Repository: citusdata/citus Branch: main Commit: 347d7236231e Files: 3056 Total size: 39.3 MB Directory structure: gitextract_h2dbs482/ ├── .codeclimate.yml ├── .codecov.yml ├── .devcontainer/ │ ├── .gdbinit │ ├── .gitignore │ ├── .psqlrc │ ├── .vscode/ │ │ ├── Pipfile │ │ ├── generate_c_cpp_properties-json.py │ │ └── launch.json │ ├── Dockerfile │ ├── Makefile │ ├── devcontainer.json │ ├── pgenv/ │ │ └── config/ │ │ └── default.conf │ ├── requirements.txt │ └── src/ │ └── test/ │ └── regress/ │ └── Pipfile ├── .editorconfig ├── .flake8 ├── .gitattributes ├── .github/ │ ├── actions/ │ │ ├── parallelization/ │ │ │ └── action.yml │ │ ├── save_logs_and_results/ │ │ │ └── action.yml │ │ ├── setup_extension/ │ │ │ └── action.yml │ │ └── upload_coverage/ │ │ └── action.yml │ ├── packaging/ │ │ ├── packaging_ignore.yml │ │ └── validate_build_output.sh │ ├── pull_request_template.md │ └── workflows/ │ ├── build_and_test.yml │ ├── codeql.yml │ ├── devcontainer.yml │ ├── flaky_test_debugging.yml │ ├── packaging-test-pipelines.yml │ └── run_tests.yml ├── .gitignore ├── .ignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVCONTAINER.md ├── EXTENSION_COMPATIBILITY.md ├── LICENSE ├── Makefile ├── Makefile.global.in ├── NOTICE ├── README.md ├── SECURITY.md ├── STYLEGUIDE.md ├── aclocal.m4 ├── autogen.sh ├── cgmanifest.json ├── ci/ │ ├── README.md │ ├── banned.h.sh │ ├── build-citus.sh │ ├── check_all_ci_scripts_are_run.sh │ ├── check_all_tests_are_run.sh │ ├── check_gucs_are_alphabetically_sorted.sh │ ├── check_migration_files.sh │ ├── check_sql_snapshots.sh │ ├── ci_helpers.sh │ ├── disallow_c_comments_in_migrations.sh │ ├── disallow_hash_comments_in_spec_files.sh │ ├── disallow_long_changelog_entries.sh │ ├── editorconfig.sh │ ├── fix_gitignore.sh │ ├── fix_style.sh │ ├── include_grouping.py │ ├── normalize_expected.sh │ ├── print_stack_trace.sh │ ├── remove_useless_declarations.sh │ └── sort_and_group_includes.sh ├── config/ │ ├── config.guess │ └── general.m4 ├── configure ├── configure.ac ├── prep_buildtree ├── pyproject.toml ├── src/ │ ├── backend/ │ │ ├── .gitignore │ │ ├── columnar/ │ │ │ ├── .gitattributes │ │ │ ├── .gitignore │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ ├── citus_columnar.control │ │ │ ├── columnar.c │ │ │ ├── columnar_compression.c │ │ │ ├── columnar_customscan.c │ │ │ ├── columnar_debug.c │ │ │ ├── columnar_metadata.c │ │ │ ├── columnar_reader.c │ │ │ ├── columnar_storage.c │ │ │ ├── columnar_tableam.c │ │ │ ├── columnar_writer.c │ │ │ ├── mod.c │ │ │ ├── sql/ │ │ │ │ ├── citus_columnar--11.1-0--11.1-1.sql │ │ │ │ ├── citus_columnar--11.1-0.sql │ │ │ │ ├── citus_columnar--11.1-1--11.2-1.sql │ │ │ │ ├── citus_columnar--11.1-1.sql │ │ │ │ ├── citus_columnar--11.2-1--11.3-1.sql │ │ │ │ ├── citus_columnar--11.3-1--12.2-1.sql │ │ │ │ ├── citus_columnar--12.2-1--13.2-1.sql │ │ │ │ ├── citus_columnar--13.2-1--14.0-1.sql │ │ │ │ ├── citus_columnar--14.0-1--15.0-1.sql │ │ │ │ ├── columnar--10.0-1--10.0-2.sql │ │ │ │ ├── columnar--10.0-3--10.1-1.sql │ │ │ │ ├── columnar--10.1-1--10.2-1.sql │ │ │ │ ├── columnar--10.2-1--10.2-2.sql │ │ │ │ ├── columnar--10.2-2--10.2-3.sql │ │ │ │ ├── columnar--10.2-3--10.2-4.sql │ │ │ │ ├── columnar--11.0-2--11.0-3.sql │ │ │ │ ├── columnar--11.0-3--11.1-1.sql │ │ │ │ ├── columnar--9.5-1--10.0-1.sql │ │ │ │ ├── downgrades/ │ │ │ │ │ ├── citus--11.0-3--11.0-2.sql │ │ │ │ │ ├── citus_columnar--11.1-1--11.1-0.sql │ │ │ │ │ ├── citus_columnar--11.2-1--11.1-1.sql │ │ │ │ │ ├── citus_columnar--11.3-1--11.2-1.sql │ │ │ │ │ ├── citus_columnar--12.2-1--11.3-1.sql │ │ │ │ │ ├── citus_columnar--13.2-1--12.2-1.sql │ │ │ │ │ ├── citus_columnar--14.0-1--13.2-1.sql │ │ │ │ │ ├── citus_columnar--15.0-1--14.0-1.sql │ │ │ │ │ ├── columnar--10.0-1--9.5-1.sql │ │ │ │ │ ├── columnar--10.0-2--10.0-1.sql │ │ │ │ │ ├── columnar--10.1-1--10.0-3.sql │ │ │ │ │ ├── columnar--10.2-1--10.1-1.sql │ │ │ │ │ ├── columnar--10.2-2--10.2-1.sql │ │ │ │ │ ├── columnar--10.2-3--10.2-2.sql │ │ │ │ │ ├── columnar--10.2-4--10.2-3.sql │ │ │ │ │ └── columnar--11.1-1--11.0-3.sql │ │ │ │ └── udfs/ │ │ │ │ ├── alter_columnar_table_reset/ │ │ │ │ │ ├── 10.0-1.sql │ │ │ │ │ ├── 11.1-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── alter_columnar_table_set/ │ │ │ │ │ ├── 10.0-1.sql │ │ │ │ │ ├── 11.1-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_ensure_am_depends_catalog/ │ │ │ │ │ ├── 10.2-4.sql │ │ │ │ │ ├── 11.1-1.sql │ │ │ │ │ ├── 11.2-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_ensure_objects_exist/ │ │ │ │ │ ├── 10.0-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_finish_pg_upgrade/ │ │ │ │ │ ├── 13.2-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_handler/ │ │ │ │ │ ├── 10.0-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ ├── downgrade_columnar_storage/ │ │ │ │ │ ├── 10.2-1.sql │ │ │ │ │ └── latest.sql │ │ │ │ └── upgrade_columnar_storage/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ └── latest.sql │ │ │ └── write_state_management.c │ │ └── distributed/ │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── cdc/ │ │ │ ├── Makefile │ │ │ ├── Makefile.decoder │ │ │ ├── cdc_decoder.c │ │ │ ├── cdc_decoder_utils.c │ │ │ └── cdc_decoder_utils.h │ │ ├── citus--11.1-1.control │ │ ├── citus.control │ │ ├── clock/ │ │ │ ├── README.md │ │ │ └── causal_clock.c │ │ ├── commands/ │ │ │ ├── README.md │ │ │ ├── alter_table.c │ │ │ ├── begin.c │ │ │ ├── call.c │ │ │ ├── cascade_table_operation_for_connected_relations.c │ │ │ ├── citus_add_local_table_to_metadata.c │ │ │ ├── citus_global_signal.c │ │ │ ├── cluster.c │ │ │ ├── collation.c │ │ │ ├── comment.c │ │ │ ├── common.c │ │ │ ├── create_distributed_table.c │ │ │ ├── database.c │ │ │ ├── dependencies.c │ │ │ ├── distribute_object_ops.c │ │ │ ├── domain.c │ │ │ ├── drop_distributed_table.c │ │ │ ├── extension.c │ │ │ ├── foreign_constraint.c │ │ │ ├── foreign_data_wrapper.c │ │ │ ├── foreign_server.c │ │ │ ├── function.c │ │ │ ├── grant.c │ │ │ ├── index.c │ │ │ ├── index_pg_source.c │ │ │ ├── local_multi_copy.c │ │ │ ├── multi_copy.c │ │ │ ├── non_main_db_distribute_object_ops.c │ │ │ ├── owned.c │ │ │ ├── policy.c │ │ │ ├── publication.c │ │ │ ├── rename.c │ │ │ ├── role.c │ │ │ ├── schema.c │ │ │ ├── schema_based_sharding.c │ │ │ ├── seclabel.c │ │ │ ├── sequence.c │ │ │ ├── serialize_distributed_ddls.c │ │ │ ├── statistics.c │ │ │ ├── subscription.c │ │ │ ├── table.c │ │ │ ├── text_search.c │ │ │ ├── trigger.c │ │ │ ├── truncate.c │ │ │ ├── type.c │ │ │ ├── utility_hook.c │ │ │ ├── vacuum.c │ │ │ ├── variableset.c │ │ │ └── view.c │ │ ├── connection/ │ │ │ ├── connection_configuration.c │ │ │ ├── connection_management.c │ │ │ ├── locally_reserved_shared_connections.c │ │ │ ├── placement_connection.c │ │ │ ├── remote_commands.c │ │ │ ├── shared_connection_stats.c │ │ │ └── worker_log_messages.c │ │ ├── deparser/ │ │ │ ├── citus_deparseutils.c │ │ │ ├── citus_grantutils.c │ │ │ ├── citus_ruleutils.c │ │ │ ├── citus_setutils.c │ │ │ ├── deparse.c │ │ │ ├── deparse_attribute_stmts.c │ │ │ ├── deparse_collation_stmts.c │ │ │ ├── deparse_comment_stmts.c │ │ │ ├── deparse_database_stmts.c │ │ │ ├── deparse_domain_stmts.c │ │ │ ├── deparse_extension_stmts.c │ │ │ ├── deparse_foreign_data_wrapper_stmts.c │ │ │ ├── deparse_foreign_server_stmts.c │ │ │ ├── deparse_function_stmts.c │ │ │ ├── deparse_owned_stmts.c │ │ │ ├── deparse_publication_stmts.c │ │ │ ├── deparse_role_stmts.c │ │ │ ├── deparse_schema_stmts.c │ │ │ ├── deparse_seclabel_stmts.c │ │ │ ├── deparse_sequence_stmts.c │ │ │ ├── deparse_statistics_stmts.c │ │ │ ├── deparse_table_stmts.c │ │ │ ├── deparse_text_search.c │ │ │ ├── deparse_type_stmts.c │ │ │ ├── deparse_view_stmts.c │ │ │ ├── format_collate.c │ │ │ ├── objectaddress.c │ │ │ ├── qualify.c │ │ │ ├── qualify_aggregate_stmts.c │ │ │ ├── qualify_collation_stmt.c │ │ │ ├── qualify_domain.c │ │ │ ├── qualify_function_stmt.c │ │ │ ├── qualify_publication_stmt.c │ │ │ ├── qualify_role_stmt.c │ │ │ ├── qualify_sequence_stmt.c │ │ │ ├── qualify_statistics_stmt.c │ │ │ ├── qualify_table_stmt.c │ │ │ ├── qualify_text_search_stmts.c │ │ │ ├── qualify_type_stmt.c │ │ │ ├── qualify_view_stmt.c │ │ │ ├── ruleutils_16.c │ │ │ ├── ruleutils_17.c │ │ │ └── ruleutils_18.c │ │ ├── executor/ │ │ │ ├── adaptive_executor.c │ │ │ ├── citus_custom_scan.c │ │ │ ├── directed_acyclic_graph_execution.c │ │ │ ├── distributed_execution_locks.c │ │ │ ├── distributed_intermediate_results.c │ │ │ ├── executor_util_params.c │ │ │ ├── executor_util_tasks.c │ │ │ ├── executor_util_tuples.c │ │ │ ├── insert_select_executor.c │ │ │ ├── intermediate_results.c │ │ │ ├── local_executor.c │ │ │ ├── merge_executor.c │ │ │ ├── multi_executor.c │ │ │ ├── multi_server_executor.c │ │ │ ├── partitioned_intermediate_results.c │ │ │ ├── placement_access.c │ │ │ ├── repartition_executor.c │ │ │ ├── repartition_join_execution.c │ │ │ ├── subplan_execution.c │ │ │ ├── transmit.c │ │ │ └── tuple_destination.c │ │ ├── metadata/ │ │ │ ├── dependency.c │ │ │ ├── distobject.c │ │ │ ├── metadata_cache.c │ │ │ ├── metadata_sync.c │ │ │ ├── metadata_utility.c │ │ │ ├── node_metadata.c │ │ │ └── pg_get_object_address_16_17_18.c │ │ ├── operations/ │ │ │ ├── citus_create_restore_point.c │ │ │ ├── citus_split_shard_by_split_points.c │ │ │ ├── citus_tools.c │ │ │ ├── create_shards.c │ │ │ ├── delete_protocol.c │ │ │ ├── health_check.c │ │ │ ├── isolate_shards.c │ │ │ ├── modify_multiple_shards.c │ │ │ ├── node_promotion.c │ │ │ ├── node_protocol.c │ │ │ ├── partitioning.c │ │ │ ├── replicate_none_dist_table_shard.c │ │ │ ├── shard_cleaner.c │ │ │ ├── shard_rebalancer.c │ │ │ ├── shard_split.c │ │ │ ├── shard_transfer.c │ │ │ ├── stage_protocol.c │ │ │ ├── worker_copy_table_to_node_udf.c │ │ │ ├── worker_node_manager.c │ │ │ ├── worker_shard_copy.c │ │ │ ├── worker_split_copy_udf.c │ │ │ ├── worker_split_shard_release_dsm_udf.c │ │ │ └── worker_split_shard_replication_setup_udf.c │ │ ├── planner/ │ │ │ ├── README.md │ │ │ ├── combine_query_planner.c │ │ │ ├── cte_inline.c │ │ │ ├── deparse_shard_query.c │ │ │ ├── distributed_planner.c │ │ │ ├── extended_op_node_utils.c │ │ │ ├── fast_path_router_planner.c │ │ │ ├── function_call_delegation.c │ │ │ ├── insert_select_planner.c │ │ │ ├── intermediate_result_pruning.c │ │ │ ├── local_distributed_join_planner.c │ │ │ ├── local_plan_cache.c │ │ │ ├── merge_planner.c │ │ │ ├── multi_explain.c │ │ │ ├── multi_join_order.c │ │ │ ├── multi_logical_optimizer.c │ │ │ ├── multi_logical_planner.c │ │ │ ├── multi_physical_planner.c │ │ │ ├── multi_router_planner.c │ │ │ ├── query_colocation_checker.c │ │ │ ├── query_pushdown_planning.c │ │ │ ├── recursive_planning.c │ │ │ ├── relation_restriction_equivalence.c │ │ │ ├── shard_pruning.c │ │ │ └── tdigest_extension.c │ │ ├── progress/ │ │ │ └── multi_progress.c │ │ ├── relay/ │ │ │ └── relay_event_utility.c │ │ ├── replication/ │ │ │ └── multi_logical_replication.c │ │ ├── shardsplit/ │ │ │ ├── shardsplit_decoder.c │ │ │ ├── shardsplit_logical_replication.c │ │ │ └── shardsplit_shared_memory.c │ │ ├── shared_library_init.c │ │ ├── sql/ │ │ │ ├── cat_upgrades/ │ │ │ │ ├── add_clone_info_to_pg_dist_node.sql │ │ │ │ └── remove_clone_info_to_pg_dist_node.sql │ │ │ ├── citus--10.0-1--10.0-2.sql │ │ │ ├── citus--10.0-2--10.0-3.sql │ │ │ ├── citus--10.0-3--10.0-4.sql │ │ │ ├── citus--10.0-4--10.1-1.sql │ │ │ ├── citus--10.1-1--10.2-1.sql │ │ │ ├── citus--10.2-1--10.2-2.sql │ │ │ ├── citus--10.2-2--10.2-3.sql │ │ │ ├── citus--10.2-3--10.2-4.sql │ │ │ ├── citus--10.2-4--10.2-5.sql │ │ │ ├── citus--10.2-5--10.2-4.sql │ │ │ ├── citus--10.2-5--11.0-1.sql │ │ │ ├── citus--11.0-1--11.0-2.sql │ │ │ ├── citus--11.0-2--11.0-3.sql │ │ │ ├── citus--11.0-3--11.0-4.sql │ │ │ ├── citus--11.0-4--11.0-3.sql │ │ │ ├── citus--11.0-4--11.1-1.sql │ │ │ ├── citus--11.1-1--11.2-1.sql │ │ │ ├── citus--11.2-1--11.2-2.sql │ │ │ ├── citus--11.2-2--11.3-1.sql │ │ │ ├── citus--11.3-1--11.3-2.sql │ │ │ ├── citus--11.3-2--12.0-1.sql │ │ │ ├── citus--12.0-1--12.1-1.sql │ │ │ ├── citus--12.1-1--13.0-1.sql │ │ │ ├── citus--13.0-1--13.1-1.sql │ │ │ ├── citus--13.1-1--13.2-1.sql │ │ │ ├── citus--13.2-1--14.0-1.sql │ │ │ ├── citus--14.0-1--15.0-1.sql │ │ │ ├── citus--8.0-1--8.0-2.sql │ │ │ ├── citus--8.0-1.sql │ │ │ ├── citus--8.0-10--8.0-11.sql │ │ │ ├── citus--8.0-11--8.0-12.sql │ │ │ ├── citus--8.0-12--8.0-13.sql │ │ │ ├── citus--8.0-13--8.1-1.sql │ │ │ ├── citus--8.0-2--8.0-3.sql │ │ │ ├── citus--8.0-3--8.0-4.sql │ │ │ ├── citus--8.0-4--8.0-5.sql │ │ │ ├── citus--8.0-5--8.0-6.sql │ │ │ ├── citus--8.0-6--8.0-7.sql │ │ │ ├── citus--8.0-7--8.0-8.sql │ │ │ ├── citus--8.0-8--8.0-9.sql │ │ │ ├── citus--8.0-9--8.0-10.sql │ │ │ ├── citus--8.1-1--8.2-1.sql │ │ │ ├── citus--8.2-1--8.2-2.sql │ │ │ ├── citus--8.2-2--8.2-3.sql │ │ │ ├── citus--8.2-3--8.2-4.sql │ │ │ ├── citus--8.2-4--8.3-1.sql │ │ │ ├── citus--8.3-1--9.0-1.sql │ │ │ ├── citus--9.0-1--9.0-2.sql │ │ │ ├── citus--9.0-2--9.1-1.sql │ │ │ ├── citus--9.1-1--9.2-1.sql │ │ │ ├── citus--9.2-1--9.2-2.sql │ │ │ ├── citus--9.2-2--9.2-4.sql │ │ │ ├── citus--9.2-4--9.3-2.sql │ │ │ ├── citus--9.3-1--9.2-4.sql │ │ │ ├── citus--9.3-2--9.4-1.sql │ │ │ ├── citus--9.4-1--9.4-2.sql │ │ │ ├── citus--9.4-1--9.5-1.sql │ │ │ ├── citus--9.4-2--9.4-1.sql │ │ │ ├── citus--9.4-2--9.4-3.sql │ │ │ ├── citus--9.4-3--9.4-2.sql │ │ │ ├── citus--9.5-1--10.0-4.sql │ │ │ ├── citus--9.5-1--9.5-2.sql │ │ │ ├── citus--9.5-2--9.5-1.sql │ │ │ ├── citus--9.5-2--9.5-3.sql │ │ │ ├── citus--9.5-3--9.5-2.sql │ │ │ ├── datatypes/ │ │ │ │ └── citus_cluster_clock/ │ │ │ │ └── 11.2-1.sql │ │ │ ├── downgrades/ │ │ │ │ ├── citus--10.0-4--9.5-1.sql │ │ │ │ ├── citus--10.1-1--10.0-4.sql │ │ │ │ ├── citus--10.2-1--10.1-1.sql │ │ │ │ ├── citus--10.2-2--10.2-1.sql │ │ │ │ ├── citus--10.2-3--10.2-2.sql │ │ │ │ ├── citus--10.2-4--10.2-3.sql │ │ │ │ ├── citus--11.0-1--10.2-4.sql │ │ │ │ ├── citus--11.0-2--11.0-1.sql │ │ │ │ ├── citus--11.0-3--11.0-2.sql │ │ │ │ ├── citus--11.1-1--11.0-4.sql │ │ │ │ ├── citus--11.2-1--11.1-1.sql │ │ │ │ ├── citus--11.2-2--11.2-1.sql │ │ │ │ ├── citus--11.3-1--11.2-2.sql │ │ │ │ ├── citus--11.3-2--11.3-1.sql │ │ │ │ ├── citus--12.0-1--11.3-2.sql │ │ │ │ ├── citus--12.1-1--12.0-1.sql │ │ │ │ ├── citus--13.0-1--12.1-1.sql │ │ │ │ ├── citus--13.1-1--13.0-1.sql │ │ │ │ ├── citus--13.2-1--13.1-1.sql │ │ │ │ ├── citus--14.0-1--13.2-1.sql │ │ │ │ ├── citus--15.0-1--14.0-1.sql │ │ │ │ ├── citus--9.2-4--9.2-2.sql │ │ │ │ ├── citus--9.3-2--9.2-4.sql │ │ │ │ ├── citus--9.4-1--9.3-2.sql │ │ │ │ └── citus--9.5-1--9.4-1.sql │ │ │ └── udfs/ │ │ │ ├── alter_distributed_table/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── alter_old_partitions_set_access_method/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── alter_role_if_exists/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── alter_table_set_access_method/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── any_value/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_activate_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_add_clone_node/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_add_inactive_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_add_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_add_rebalance_strategy/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_add_secondary_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_backend_gpid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_blocking_pids/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_calculate_gpid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_check_cluster_node_health/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_check_connection_to_node/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_cleanup_orphaned_resources/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_cleanup_orphaned_shards/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_conninfo_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_coordinator_nodeid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_copy_shard_placement/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_disable_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_local_group_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_node_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_object_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_partition_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_placement_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_shard_cache_invalidate/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_dist_stat_activity/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_drain_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_drop_trigger/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_extradata_container/ │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_finalize_upgrade_to_citus11/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.0-2.sql │ │ │ │ ├── 11.0-3.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_finish_citus_upgrade/ │ │ │ │ ├── 11.0-2.sql │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_finish_pg_upgrade/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 10.0-4.sql │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 10.2-4.sql │ │ │ │ ├── 10.2-5.sql │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.0-4.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 12.1-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ ├── 13.2-1.sql │ │ │ │ ├── 14.0-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ ├── 9.4-2.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ ├── 9.5-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_get_node_clock/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_get_transaction_clock/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_acquire_citus_advisory_object_class_lock/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_colocation_metadata/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_object_metadata/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_partition_metadata/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_placement_metadata/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_shard_metadata/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_add_tenant_schema/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_adjust_local_clock_to_remote/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_copy_single_shard_placement/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_database_command/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_delete_colocation_metadata/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_delete_partition_metadata/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_delete_placement_metadata/ │ │ │ │ ├── 12.1-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_delete_shard_metadata/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_delete_tenant_schema/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_global_blocked_processes/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_local_blocked_processes/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_mark_node_not_synced/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_unregister_tenant_schema_globally/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_update_none_dist_table_metadata/ │ │ │ │ ├── 12.1-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_update_placement_metadata/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_internal_update_relation_colocation/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_is_clock_after/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_is_coordinator/ │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_is_primary_node/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_isolation_test_session_is_blocked/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ ├── 8.0-6.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_job_cancel/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_job_list/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_job_status/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_job_wait/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_local_disk_space_stats/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_lock_waits/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_locks/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_move_shard_placement/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_node_capacity_1/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_nodeid_for_gpid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_nodename_for_nodeid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_nodeport_for_nodeid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_nodes/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_pause_node_within_txn/ │ │ │ │ ├── 12.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_pid_for_gpid/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_prepare_pg_upgrade/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ ├── 12.1-1.sql │ │ │ │ ├── 13.0-1.sql │ │ │ │ ├── 14.0-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ ├── 9.4-2.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ ├── 9.5-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_promote_clone_and_rebalance/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_rebalance_start/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_rebalance_status/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_rebalance_stop/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_rebalance_wait/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_remote_connection_stats/ │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_remove_clone_node/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_remove_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_run_local_command/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_schema_distribute/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_schema_move/ │ │ │ │ ├── 12.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_schema_undistribute/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_schemas/ │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_set_coordinator_host/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_set_default_rebalance_strategy/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_set_node_property/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shard_allowed_on_node_true/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shard_cost_1/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shard_cost_by_disk_size/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shard_indexes_on_worker/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shard_sizes/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shards/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.3-2.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_shards_on_worker/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_split_shard_by_split_points/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_activity/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_counters/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_counters_reset/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_tenants/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_tenants_local/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_tenants_local_reset/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stat_tenants_reset/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_stats/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_tables/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 10.0-4.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.3-2.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_task_wait/ │ │ │ │ ├── 11.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_total_relation_size/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 7.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_unmark_object_distributed/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_update_node/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_update_shard_statistics/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_update_table_statistics/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 10.0-3.sql │ │ │ │ └── latest.sql │ │ │ ├── citus_validate_rebalance_strategy_functions/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── commit_management_command_2pc/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── coord_binary_combine_agg/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── coord_binary_combine_agg_ffunc/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── coord_binary_combine_agg_sfunc/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── create_citus_local_table/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── create_distributed_function/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── create_distributed_table/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── create_distributed_table_concurrently/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── create_time_partitions/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 13.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── distributed_tables_colocated/ │ │ │ │ ├── 9.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── drop_old_time_partitions/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── fetch_intermediate_results/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── fix_all_partition_shard_index_names/ │ │ │ │ ├── 10.2-4.sql │ │ │ │ └── latest.sql │ │ │ ├── fix_partition_shard_index_names/ │ │ │ │ ├── 10.2-4.sql │ │ │ │ └── latest.sql │ │ │ ├── fix_pre_citus10_partitioned_table_constraint_names/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── fix_pre_citus14_colocation_group_collation_mismatches/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_all_active_transactions/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_global_active_transactions/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_missing_time_partition_ranges/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ ├── 12.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_rebalance_progress/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 11.2-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_rebalance_table_shards_plan/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ ├── 9.1-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── get_snapshot_based_node_split_plan/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── isolate_tenant_to_new_shard/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 8.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── master_add_inactive_node/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── master_add_node/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── master_drain_node/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── master_set_node_property/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── pg_cancel_backend/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── pg_dist_rebalance_strategy_trigger_func/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── pg_dist_shard_placement_trigger_func/ │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── pg_terminate_backend/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── read_intermediate_results/ │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── rebalance_table_shards/ │ │ │ │ ├── 9.0-1.sql │ │ │ │ ├── 9.1-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── remove_local_tables_from_metadata/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── repl_origin_helper/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── replicate_reference_tables/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── replicate_table_shards/ │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── run_command_on_all_nodes/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── run_command_on_coordinator/ │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── shard_name/ │ │ │ │ ├── 13.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── start_metadata_sync_to_all_nodes/ │ │ │ │ ├── 11.0-2.sql │ │ │ │ └── latest.sql │ │ │ ├── stop_metadata_sync_to_node/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── time_partition_range/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── time_partitions/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── truncate_local_data_after_distributing_table/ │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── undistribute_table/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── update_distributed_table_colocation/ │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── upgrade_to_reference_table/ │ │ │ │ ├── 8.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_adjust_identity_column_seq_ranges/ │ │ │ │ ├── 11.2-2.sql │ │ │ │ ├── 11.3-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_binary_partial_agg/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_binary_partial_agg_ffunc/ │ │ │ │ ├── 14.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_change_sequence_dependency/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_copy_table_to_node/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_create_or_alter_role/ │ │ │ │ ├── 9.3-2.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_create_or_replace_object/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 9.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_create_schema/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ ├── 9.2-2.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_drop_all_shell_tables/ │ │ │ │ ├── 11.3-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_drop_sequence_dependency/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_drop_shell_table/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_fix_partition_shard_index_names/ │ │ │ │ ├── 10.2-4.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_fix_pre_citus10_partitioned_table_constraint_names/ │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_last_saved_explain_analyze/ │ │ │ │ ├── 13.2-1.sql │ │ │ │ ├── 9.4-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_nextval/ │ │ │ │ ├── 10.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_partition_query_result/ │ │ │ │ ├── 11.0-1.sql │ │ │ │ ├── 9.2-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_partitioned_relation_size/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_partitioned_relation_total_size/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_partitioned_table_size/ │ │ │ │ ├── 10.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_record_sequence_dependency/ │ │ │ │ ├── 9.5-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_repartition_cleanup/ │ │ │ │ ├── 9.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_save_query_explain_analyze/ │ │ │ │ ├── 9.4-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_split_copy/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ ├── worker_split_shard_release_dsm/ │ │ │ │ ├── 11.1-1.sql │ │ │ │ └── latest.sql │ │ │ └── worker_split_shard_replication_setup/ │ │ │ ├── 11.1-1.sql │ │ │ ├── 11.2-1.sql │ │ │ └── latest.sql │ │ ├── stats/ │ │ │ ├── query_stats.c │ │ │ ├── stat_counters.c │ │ │ └── stat_tenants.c │ │ ├── test/ │ │ │ ├── backend_counter.c │ │ │ ├── citus_depended_object.c │ │ │ ├── citus_stat_tenants.c │ │ │ ├── colocation_utils.c │ │ │ ├── create_shards.c │ │ │ ├── deparse_function_query.c │ │ │ ├── deparse_shard_query.c │ │ │ ├── dependency.c │ │ │ ├── distributed_deadlock_detection.c │ │ │ ├── distributed_intermediate_results.c │ │ │ ├── distribution_metadata.c │ │ │ ├── fake_am.c │ │ │ ├── fake_fdw.c │ │ │ ├── foreign_key_relationship_query.c │ │ │ ├── global_pid.c │ │ │ ├── hide_shards.c │ │ │ ├── intermediate_results.c │ │ │ ├── make_external_connection.c │ │ │ ├── metadata_sync.c │ │ │ ├── partitioning_utils.c │ │ │ ├── progress_utils.c │ │ │ ├── prune_shard_list.c │ │ │ ├── relation_access_tracking.c │ │ │ ├── run_from_same_connection.c │ │ │ ├── sequential_execution.c │ │ │ ├── shard_rebalancer.c │ │ │ ├── shared_connection_counters.c │ │ │ └── xact_stats.c │ │ ├── transaction/ │ │ │ ├── backend_data.c │ │ │ ├── citus_dist_stat_activity.c │ │ │ ├── distributed_deadlock_detection.c │ │ │ ├── lock_graph.c │ │ │ ├── relation_access_tracking.c │ │ │ ├── remote_transaction.c │ │ │ ├── transaction_management.c │ │ │ ├── transaction_recovery.c │ │ │ └── worker_transaction.c │ │ ├── utils/ │ │ │ ├── acquire_lock.c │ │ │ ├── aggregate_utils.c │ │ │ ├── array_type.c │ │ │ ├── background_jobs.c │ │ │ ├── background_worker_utils.c │ │ │ ├── cancel_utils.c │ │ │ ├── citus_clauses.c │ │ │ ├── citus_copyfuncs.c │ │ │ ├── citus_depended_object.c │ │ │ ├── citus_nodefuncs.c │ │ │ ├── citus_outfuncs.c │ │ │ ├── citus_readfuncs.c │ │ │ ├── citus_safe_lib.c │ │ │ ├── citus_version.c │ │ │ ├── clonenode_utils.c │ │ │ ├── colocation_utils.c │ │ │ ├── directory.c │ │ │ ├── distribution_column.c │ │ │ ├── distribution_column_map.c │ │ │ ├── enable_ssl.c │ │ │ ├── errormessage.c │ │ │ ├── foreign_key_relationship.c │ │ │ ├── function.c │ │ │ ├── function_utils.c │ │ │ ├── hash_helpers.c │ │ │ ├── jsonbutils.c │ │ │ ├── listutils.c │ │ │ ├── log_utils.c │ │ │ ├── maintenanced.c │ │ │ ├── multi_partitioning_utils.c │ │ │ ├── namespace_utils.c │ │ │ ├── param_utils.c │ │ │ ├── priority.c │ │ │ ├── query_utils.c │ │ │ ├── reference_table_utils.c │ │ │ ├── relation_utils.c │ │ │ ├── replication_origin_session_utils.c │ │ │ ├── resource_lock.c │ │ │ ├── role.c │ │ │ ├── shard_utils.c │ │ │ ├── shardinterval_utils.c │ │ │ ├── string_utils.c │ │ │ ├── task_execution_utils.c │ │ │ ├── tenant_schema_metadata.c │ │ │ ├── tuplestore.c │ │ │ └── type_utils.c │ │ └── worker/ │ │ ├── task_tracker_protocol.c │ │ ├── worker_create_or_replace.c │ │ ├── worker_data_fetch_protocol.c │ │ ├── worker_drop_protocol.c │ │ ├── worker_partition_protocol.c │ │ ├── worker_shard_visibility.c │ │ ├── worker_sql_task_protocol.c │ │ └── worker_truncate_trigger_protocol.c │ ├── include/ │ │ ├── .gitignore │ │ ├── citus_config.h.in │ │ ├── citus_version.h.in │ │ ├── columnar/ │ │ │ ├── columnar.h │ │ │ ├── columnar_compression.h │ │ │ ├── columnar_customscan.h │ │ │ ├── columnar_metadata.h │ │ │ ├── columnar_storage.h │ │ │ ├── columnar_tableam.h │ │ │ └── columnar_version_compat.h │ │ ├── distributed/ │ │ │ ├── adaptive_executor.h │ │ │ ├── argutils.h │ │ │ ├── backend_data.h │ │ │ ├── background_jobs.h │ │ │ ├── background_worker_utils.h │ │ │ ├── cancel_utils.h │ │ │ ├── causal_clock.h │ │ │ ├── citus_acquire_lock.h │ │ │ ├── citus_clauses.h │ │ │ ├── citus_custom_scan.h │ │ │ ├── citus_depended_object.h │ │ │ ├── citus_nodefuncs.h │ │ │ ├── citus_nodes.h │ │ │ ├── citus_ruleutils.h │ │ │ ├── citus_safe_lib.h │ │ │ ├── clonenode_utils.h │ │ │ ├── colocation_utils.h │ │ │ ├── combine_query_planner.h │ │ │ ├── commands/ │ │ │ │ ├── multi_copy.h │ │ │ │ ├── sequence.h │ │ │ │ ├── serialize_distributed_ddls.h │ │ │ │ └── utility_hook.h │ │ │ ├── commands.h │ │ │ ├── comment.h │ │ │ ├── connection_management.h │ │ │ ├── coordinator_protocol.h │ │ │ ├── cte_inline.h │ │ │ ├── deparse_shard_query.h │ │ │ ├── deparser.h │ │ │ ├── directed_acyclic_graph_execution.h │ │ │ ├── distributed_deadlock_detection.h │ │ │ ├── distributed_execution_locks.h │ │ │ ├── distributed_planner.h │ │ │ ├── distribution_column.h │ │ │ ├── enterprise.h │ │ │ ├── error_codes.h │ │ │ ├── errormessage.h │ │ │ ├── executor_util.h │ │ │ ├── extended_op_node_utils.h │ │ │ ├── foreign_key_relationship.h │ │ │ ├── function_call_delegation.h │ │ │ ├── function_utils.h │ │ │ ├── hash_helpers.h │ │ │ ├── insert_select_executor.h │ │ │ ├── insert_select_planner.h │ │ │ ├── intermediate_result_pruning.h │ │ │ ├── intermediate_results.h │ │ │ ├── jsonbutils.h │ │ │ ├── listutils.h │ │ │ ├── local_distributed_join_planner.h │ │ │ ├── local_executor.h │ │ │ ├── local_multi_copy.h │ │ │ ├── local_plan_cache.h │ │ │ ├── locally_reserved_shared_connections.h │ │ │ ├── lock_graph.h │ │ │ ├── log_utils.h │ │ │ ├── maintenanced.h │ │ │ ├── memutils.h │ │ │ ├── merge_executor.h │ │ │ ├── merge_planner.h │ │ │ ├── metadata/ │ │ │ │ ├── dependency.h │ │ │ │ ├── distobject.h │ │ │ │ └── pg_dist_object.h │ │ │ ├── metadata_cache.h │ │ │ ├── metadata_sync.h │ │ │ ├── metadata_utility.h │ │ │ ├── multi_executor.h │ │ │ ├── multi_explain.h │ │ │ ├── multi_join_order.h │ │ │ ├── multi_logical_optimizer.h │ │ │ ├── multi_logical_planner.h │ │ │ ├── multi_logical_replication.h │ │ │ ├── multi_partitioning_utils.h │ │ │ ├── multi_physical_planner.h │ │ │ ├── multi_progress.h │ │ │ ├── multi_router_planner.h │ │ │ ├── multi_server_executor.h │ │ │ ├── namespace_utils.h │ │ │ ├── param_utils.h │ │ │ ├── pg_dist_background_job.h │ │ │ ├── pg_dist_background_task.h │ │ │ ├── pg_dist_backrgound_task_depend.h │ │ │ ├── pg_dist_cleanup.h │ │ │ ├── pg_dist_colocation.h │ │ │ ├── pg_dist_local_group.h │ │ │ ├── pg_dist_node.h │ │ │ ├── pg_dist_node_metadata.h │ │ │ ├── pg_dist_partition.h │ │ │ ├── pg_dist_placement.h │ │ │ ├── pg_dist_rebalance_strategy.h │ │ │ ├── pg_dist_schema.h │ │ │ ├── pg_dist_shard.h │ │ │ ├── pg_dist_transaction.h │ │ │ ├── placement_access.h │ │ │ ├── placement_connection.h │ │ │ ├── priority.h │ │ │ ├── query_colocation_checker.h │ │ │ ├── query_pushdown_planning.h │ │ │ ├── query_utils.h │ │ │ ├── recursive_planning.h │ │ │ ├── reference_table_utils.h │ │ │ ├── relation_access_tracking.h │ │ │ ├── relation_restriction_equivalence.h │ │ │ ├── relation_utils.h │ │ │ ├── relay_utility.h │ │ │ ├── remote_commands.h │ │ │ ├── remote_transaction.h │ │ │ ├── repartition_executor.h │ │ │ ├── repartition_join_execution.h │ │ │ ├── replicate_none_dist_table_shard.h │ │ │ ├── replication_origin_session_utils.h │ │ │ ├── resource_lock.h │ │ │ ├── run_from_same_connection.h │ │ │ ├── shard_cleaner.h │ │ │ ├── shard_pruning.h │ │ │ ├── shard_rebalancer.h │ │ │ ├── shard_split.h │ │ │ ├── shard_transfer.h │ │ │ ├── shard_utils.h │ │ │ ├── shardinterval_utils.h │ │ │ ├── shardsplit_logical_replication.h │ │ │ ├── shardsplit_shared_memory.h │ │ │ ├── shared_connection_stats.h │ │ │ ├── shared_library_init.h │ │ │ ├── stats/ │ │ │ │ ├── query_stats.h │ │ │ │ ├── stat_counters.h │ │ │ │ └── stat_tenants.h │ │ │ ├── string_utils.h │ │ │ ├── subplan_execution.h │ │ │ ├── task_execution_utils.h │ │ │ ├── tdigest_extension.h │ │ │ ├── tenant_schema_metadata.h │ │ │ ├── time_constants.h │ │ │ ├── transaction_identifier.h │ │ │ ├── transaction_management.h │ │ │ ├── transaction_recovery.h │ │ │ ├── transmit.h │ │ │ ├── tuple_destination.h │ │ │ ├── tuplestore.h │ │ │ ├── type_utils.h │ │ │ ├── utils/ │ │ │ │ ├── array_type.h │ │ │ │ ├── directory.h │ │ │ │ ├── distribution_column_map.h │ │ │ │ └── function.h │ │ │ ├── version_compat.h │ │ │ ├── worker_create_or_replace.h │ │ │ ├── worker_log_messages.h │ │ │ ├── worker_manager.h │ │ │ ├── worker_protocol.h │ │ │ ├── worker_shard_copy.h │ │ │ ├── worker_shard_visibility.h │ │ │ └── worker_transaction.h │ │ ├── pg_version_compat.h │ │ └── pg_version_constants.h │ └── test/ │ ├── cdc/ │ │ ├── Makefile │ │ ├── postgresql.conf │ │ └── t/ │ │ ├── 001_cdc_create_distributed_table_test.pl │ │ ├── 002_cdc_create_distributed_table_concurrently.pl │ │ ├── 003_cdc_parallel_insert.pl │ │ ├── 004_cdc_move_shard.pl │ │ ├── 005_cdc_reference_table_test.pl │ │ ├── 006_cdc_schema_change_and_move.pl │ │ ├── 007_cdc_undistributed_table_test.pl │ │ ├── 008_cdc_shard_split_test.pl │ │ ├── 009_cdc_shard_split_test_non_blocking.pl │ │ ├── 010_cdc_shard_split_parallel_insert.pl │ │ ├── 011_cdc_alter_distributed_table.pl │ │ ├── 012_cdc_restart_test.pl │ │ ├── 013_cdc_drop_last_column_for_one_shard.pl │ │ ├── 014_cdc_with_table_like_shard_name.pl │ │ ├── 015_cdc_without_citus.pl │ │ ├── 016_cdc_wal2json.pl │ │ └── cdctestlib.pm │ ├── hammerdb/ │ │ ├── README.md │ │ └── run_hammerdb.sh │ ├── regress/ │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── Pipfile │ │ ├── README.md │ │ ├── after_citus_upgrade_coord_schedule │ │ ├── after_pg_upgrade_with_columnar_schedule │ │ ├── after_pg_upgrade_without_columnar_schedule │ │ ├── base_isolation_schedule │ │ ├── base_schedule │ │ ├── before_citus_upgrade_coord_schedule │ │ ├── before_pg_upgrade_with_columnar_schedule │ │ ├── before_pg_upgrade_without_columnar_schedule │ │ ├── bin/ │ │ │ ├── copy_modified │ │ │ ├── copy_modified_wrapper │ │ │ ├── create_test.py │ │ │ ├── diff │ │ │ ├── diff-filter │ │ │ ├── normalize.sed │ │ │ ├── test/ │ │ │ │ ├── expected/ │ │ │ │ │ ├── different.out │ │ │ │ │ └── same.out │ │ │ │ ├── file.out │ │ │ │ ├── file_different.out │ │ │ │ └── file_same.out │ │ │ └── test_diff │ │ ├── citus_tests/ │ │ │ ├── __init__.py │ │ │ ├── arbitrary_configs/ │ │ │ │ ├── README.md │ │ │ │ ├── __init__.py │ │ │ │ └── citus_arbitrary_configs.py │ │ │ ├── common.py │ │ │ ├── config.py │ │ │ ├── print_test_names.py │ │ │ ├── query_generator/ │ │ │ │ ├── .gitignore │ │ │ │ ├── README.md │ │ │ │ ├── bin/ │ │ │ │ │ ├── citus_compare_dist_local_joins.sh │ │ │ │ │ ├── diff-checker.py │ │ │ │ │ └── run_query_compare_test.py │ │ │ │ ├── config/ │ │ │ │ │ ├── config.py │ │ │ │ │ ├── config.yaml │ │ │ │ │ └── config_parser.py │ │ │ │ ├── data_gen.py │ │ │ │ ├── ddl_gen.py │ │ │ │ ├── generate_queries.py │ │ │ │ ├── node_defs.py │ │ │ │ ├── out/ │ │ │ │ │ └── .gitignore │ │ │ │ ├── query_gen.py │ │ │ │ └── random_selections.py │ │ │ ├── run_test.py │ │ │ ├── test/ │ │ │ │ ├── README.md │ │ │ │ ├── __init__.py │ │ │ │ ├── conftest.py │ │ │ │ ├── test_columnar.py │ │ │ │ ├── test_extension.py │ │ │ │ └── test_prepared_statements.py │ │ │ ├── upgrade/ │ │ │ │ ├── README.md │ │ │ │ ├── __init__.py │ │ │ │ ├── citus_upgrade_test.py │ │ │ │ ├── generate_citus_tarballs.sh │ │ │ │ └── pg_upgrade_test.py │ │ │ └── utils.py │ │ ├── columnar_isolation_schedule │ │ ├── columnar_schedule │ │ ├── create_schedule │ │ ├── data/ │ │ │ ├── ads.csv │ │ │ ├── agg.data │ │ │ ├── agg_type.data │ │ │ ├── array_types.csv │ │ │ ├── campaigns.csv │ │ │ ├── clicks.csv │ │ │ ├── companies.csv │ │ │ ├── contestants.1.csv │ │ │ ├── contestants.2.csv │ │ │ ├── customer-1-10.data │ │ │ ├── customer-1-15.data │ │ │ ├── customer-1-20.data │ │ │ ├── customer-1-30.data │ │ │ ├── customer-11-20.data │ │ │ ├── customer-21-30.data │ │ │ ├── customer-subset-11-20.data │ │ │ ├── customer-subset-21-30.data │ │ │ ├── customer.1.data │ │ │ ├── customer.2.data │ │ │ ├── customer.3.data │ │ │ ├── datetime_types.csv │ │ │ ├── enum_and_composite_types.csv │ │ │ ├── events_table.data │ │ │ ├── impressions.csv │ │ │ ├── large_records.data │ │ │ ├── lineitem.1.data │ │ │ ├── lineitem.2.data │ │ │ ├── nation.data │ │ │ ├── null_values.csv │ │ │ ├── orders.1.data │ │ │ ├── orders.2.data │ │ │ ├── other_types.csv │ │ │ ├── part.data │ │ │ ├── part.more.data │ │ │ ├── range_types.csv │ │ │ ├── supplier.data │ │ │ └── users_table.data │ │ ├── enterprise_failure_schedule │ │ ├── enterprise_isolation_logicalrep_1_schedule │ │ ├── enterprise_isolation_logicalrep_2_schedule │ │ ├── enterprise_isolation_logicalrep_3_schedule │ │ ├── enterprise_isolation_schedule │ │ ├── enterprise_minimal_schedule │ │ ├── enterprise_schedule │ │ ├── expected/ │ │ │ ├── adaptive_executor.out │ │ │ ├── adaptive_executor_repartition.out │ │ │ ├── add_coordinator.out │ │ │ ├── adv_lock_permission.out │ │ │ ├── aggregate_support.out │ │ │ ├── alter_database_owner.out │ │ │ ├── alter_database_propagation.out │ │ │ ├── alter_distributed_table.out │ │ │ ├── alter_index.out │ │ │ ├── alter_role_propagation.out │ │ │ ├── alter_table_add_column.out │ │ │ ├── alter_table_set_access_method.out │ │ │ ├── alter_table_single_shard_table.out │ │ │ ├── anonymous_columns.out │ │ │ ├── arbitrary_configs_alter_table_add_constraint_without_name.out │ │ │ ├── arbitrary_configs_alter_table_add_constraint_without_name_create.out │ │ │ ├── arbitrary_configs_recurring_outer_join.out │ │ │ ├── arbitrary_configs_router.out │ │ │ ├── arbitrary_configs_router_create.out │ │ │ ├── arbitrary_configs_truncate.out │ │ │ ├── arbitrary_configs_truncate_cascade.out │ │ │ ├── arbitrary_configs_truncate_cascade_create.out │ │ │ ├── arbitrary_configs_truncate_create.out │ │ │ ├── arbitrary_configs_truncate_partition.out │ │ │ ├── arbitrary_configs_truncate_partition_create.out │ │ │ ├── auto_undist_citus_local.out │ │ │ ├── background_rebalance.out │ │ │ ├── background_rebalance_parallel.out │ │ │ ├── background_rebalance_parallel_reference_tables.out │ │ │ ├── background_task_queue_monitor.out │ │ │ ├── base_enable_mx.out │ │ │ ├── binary_protocol.out │ │ │ ├── bool_agg.out │ │ │ ├── cdc_library_path.out │ │ │ ├── ch_bench_having.out │ │ │ ├── ch_bench_having_mx.out │ │ │ ├── ch_bench_subquery_repartition.out │ │ │ ├── ch_benchmarks_1.out │ │ │ ├── ch_benchmarks_2.out │ │ │ ├── ch_benchmarks_3.out │ │ │ ├── ch_benchmarks_4.out │ │ │ ├── ch_benchmarks_5.out │ │ │ ├── ch_benchmarks_6.out │ │ │ ├── ch_benchmarks_create_load.out │ │ │ ├── chbenchmark_all_queries.out │ │ │ ├── check_cluster_state.out │ │ │ ├── check_mx.out │ │ │ ├── citus_aggregated_stats.out │ │ │ ├── citus_copy_shard_placement.out │ │ │ ├── citus_depended_object.out │ │ │ ├── citus_drain_node.out │ │ │ ├── citus_internal_access.out │ │ │ ├── citus_local_dist_joins.out │ │ │ ├── citus_local_table_triggers.out │ │ │ ├── citus_local_tables.out │ │ │ ├── citus_local_tables_ent.out │ │ │ ├── citus_local_tables_mx.out │ │ │ ├── citus_local_tables_queries.out │ │ │ ├── citus_local_tables_queries_mx.out │ │ │ ├── citus_locks.out │ │ │ ├── citus_non_blocking_split_columnar.out │ │ │ ├── citus_non_blocking_split_shard_cleanup.out │ │ │ ├── citus_non_blocking_split_shards.out │ │ │ ├── citus_run_command.out │ │ │ ├── citus_schema_distribute_undistribute.out │ │ │ ├── citus_schema_move.out │ │ │ ├── citus_shards.out │ │ │ ├── citus_split_shard_by_split_points.out │ │ │ ├── citus_split_shard_by_split_points_deferred_drop.out │ │ │ ├── citus_split_shard_by_split_points_failure.out │ │ │ ├── citus_split_shard_by_split_points_negative.out │ │ │ ├── citus_split_shard_columnar_partitioned.out │ │ │ ├── citus_stat_tenants.out │ │ │ ├── citus_table_triggers.out │ │ │ ├── citus_update_table_statistics.out │ │ │ ├── clock.out │ │ │ ├── columnar_alter.out │ │ │ ├── columnar_alter_set_type.out │ │ │ ├── columnar_analyze.out │ │ │ ├── columnar_chunk_filtering.out │ │ │ ├── columnar_chunk_filtering_0.out │ │ │ ├── columnar_citus_integration.out │ │ │ ├── columnar_clean.out │ │ │ ├── columnar_copyto.out │ │ │ ├── columnar_create.out │ │ │ ├── columnar_cursor.out │ │ │ ├── columnar_data_types.out │ │ │ ├── columnar_drop.out │ │ │ ├── columnar_empty.out │ │ │ ├── columnar_fallback_scan.out │ │ │ ├── columnar_first_row_number.out │ │ │ ├── columnar_index_concurrency.out │ │ │ ├── columnar_indexes.out │ │ │ ├── columnar_insert.out │ │ │ ├── columnar_join.out │ │ │ ├── columnar_load.out │ │ │ ├── columnar_lz4.out │ │ │ ├── columnar_lz4_0.out │ │ │ ├── columnar_matview.out │ │ │ ├── columnar_memory.out │ │ │ ├── columnar_partitioning.out │ │ │ ├── columnar_paths.out │ │ │ ├── columnar_paths_0.out │ │ │ ├── columnar_permissions.out │ │ │ ├── columnar_pg15.out │ │ │ ├── columnar_query.out │ │ │ ├── columnar_recursive.out │ │ │ ├── columnar_rollback.out │ │ │ ├── columnar_tableoptions.out │ │ │ ├── columnar_temp_tables.out │ │ │ ├── columnar_test_helpers.out │ │ │ ├── columnar_transactions.out │ │ │ ├── columnar_trigger.out │ │ │ ├── columnar_truncate.out │ │ │ ├── columnar_types_without_comparison.out │ │ │ ├── columnar_update_delete.out │ │ │ ├── columnar_vacuum.out │ │ │ ├── columnar_vacuum_vs_insert.out │ │ │ ├── columnar_write_concurrency.out │ │ │ ├── columnar_write_concurrency_index.out │ │ │ ├── columnar_zstd.out │ │ │ ├── columnar_zstd_0.out │ │ │ ├── comment_on_database.out │ │ │ ├── comment_on_role.out │ │ │ ├── connectivity_checks.out │ │ │ ├── coordinator_evaluation.out │ │ │ ├── coordinator_evaluation_modify.out │ │ │ ├── coordinator_evaluation_select.out │ │ │ ├── coordinator_shouldhaveshards.out │ │ │ ├── cpu_priority.out │ │ │ ├── create_citus_local_table_cascade.out │ │ │ ├── create_distributed_table_concurrently.out │ │ │ ├── create_drop_database_propagation.out │ │ │ ├── create_drop_database_propagation_pg15.out │ │ │ ├── create_drop_database_propagation_pg16.out │ │ │ ├── create_ref_dist_from_citus_local.out │ │ │ ├── create_role_propagation.out │ │ │ ├── create_single_shard_table.out │ │ │ ├── cross_join.out │ │ │ ├── cte_inline.out │ │ │ ├── cte_nested_modification.out │ │ │ ├── cte_prepared_modify.out │ │ │ ├── cursors.out │ │ │ ├── custom_aggregate_support.out │ │ │ ├── custom_aggregate_support_0.out │ │ │ ├── data_types.out │ │ │ ├── detect_conn_close.out │ │ │ ├── disable_object_propagation.out │ │ │ ├── distributed_collations.out │ │ │ ├── distributed_collations_conflict.out │ │ │ ├── distributed_domain.out │ │ │ ├── distributed_functions.out │ │ │ ├── distributed_functions_conflict.out │ │ │ ├── distributed_intermediate_results.out │ │ │ ├── distributed_locks.out │ │ │ ├── distributed_planning.out │ │ │ ├── distributed_planning_create_load.out │ │ │ ├── distributed_procedure.out │ │ │ ├── distributed_triggers.out │ │ │ ├── distributed_types.out │ │ │ ├── distributed_types_conflict.out │ │ │ ├── distributed_types_xact_add_enum_value.out │ │ │ ├── dml_recursive.out │ │ │ ├── drop_column_partitioned_table.out │ │ │ ├── drop_database.out │ │ │ ├── drop_partitioned_table.out │ │ │ ├── dropped_columns_1.out │ │ │ ├── dropped_columns_create_load.out │ │ │ ├── ensure_citus_columnar_not_exists.out │ │ │ ├── ensure_no_intermediate_data_leak.out │ │ │ ├── ensure_no_shared_connection_leak.out │ │ │ ├── escape_extension_name.out │ │ │ ├── escape_extension_name_0.out │ │ │ ├── executor_local_failure.out │ │ │ ├── expression_reference_join.out │ │ │ ├── failure_add_disable_node.out │ │ │ ├── failure_connection_establishment.out │ │ │ ├── failure_copy_on_hash.out │ │ │ ├── failure_copy_to_reference.out │ │ │ ├── failure_create_database.out │ │ │ ├── failure_create_distributed_table_concurrently.out │ │ │ ├── failure_create_distributed_table_non_empty.out │ │ │ ├── failure_create_index_concurrently.out │ │ │ ├── failure_create_reference_table.out │ │ │ ├── failure_create_table.out │ │ │ ├── failure_cte_subquery.out │ │ │ ├── failure_ddl.out │ │ │ ├── failure_distributed_results.out │ │ │ ├── failure_failover_to_local_execution.out │ │ │ ├── failure_insert_select_pushdown.out │ │ │ ├── failure_insert_select_repartition.out │ │ │ ├── failure_insert_select_via_coordinator.out │ │ │ ├── failure_multi_dml.out │ │ │ ├── failure_multi_row_insert.out │ │ │ ├── failure_multi_shard_update_delete.out │ │ │ ├── failure_mx_metadata_sync.out │ │ │ ├── failure_mx_metadata_sync_multi_trans.out │ │ │ ├── failure_non_main_db_2pc.out │ │ │ ├── failure_offline_move_shard_placement.out │ │ │ ├── failure_on_create_subscription.out │ │ │ ├── failure_online_move_shard_placement.out │ │ │ ├── failure_parallel_connection.out │ │ │ ├── failure_ref_tables.out │ │ │ ├── failure_replicated_partitions.out │ │ │ ├── failure_savepoints.out │ │ │ ├── failure_setup.out │ │ │ ├── failure_single_mod.out │ │ │ ├── failure_single_select.out │ │ │ ├── failure_split_cleanup.out │ │ │ ├── failure_tenant_isolation.out │ │ │ ├── failure_tenant_isolation_nonblocking.out │ │ │ ├── failure_test_helpers.out │ │ │ ├── failure_truncate.out │ │ │ ├── failure_vacuum.out │ │ │ ├── fast_path_router_modify.out │ │ │ ├── fkeys_between_local_ref.out │ │ │ ├── follower_single_node.out │ │ │ ├── forcedelegation_functions.out │ │ │ ├── foreign_key_restriction_enforcement.out │ │ │ ├── foreign_key_to_reference_shard_rebalance.out │ │ │ ├── foreign_key_to_reference_table.out │ │ │ ├── foreign_tables_mx.out │ │ │ ├── function_create.out │ │ │ ├── function_propagation.out │ │ │ ├── function_with_case_when.out │ │ │ ├── functions.out │ │ │ ├── generated_identity.out │ │ │ ├── geqo.out │ │ │ ├── global_cancel.out │ │ │ ├── grant_on_database_propagation.out │ │ │ ├── grant_on_database_propagation_from_non_maindb.out │ │ │ ├── grant_on_foreign_server_propagation.out │ │ │ ├── grant_on_function_propagation.out │ │ │ ├── grant_on_schema_propagation.out │ │ │ ├── grant_on_sequence_propagation.out │ │ │ ├── grant_on_table_propagation.out │ │ │ ├── grant_on_table_propagation_0.out │ │ │ ├── grant_role_from_non_maindb.out │ │ │ ├── having_subquery.out │ │ │ ├── hyperscale_tutorial.out │ │ │ ├── index_create.out │ │ │ ├── insert_select_connection_leak.out │ │ │ ├── insert_select_into_local_table.out │ │ │ ├── insert_select_repartition.out │ │ │ ├── insert_select_single_shard_table.out │ │ │ ├── intermediate_result_pruning.out │ │ │ ├── intermediate_result_pruning_create.out │ │ │ ├── intermediate_result_pruning_queries_1.out │ │ │ ├── intermediate_result_pruning_queries_2.out │ │ │ ├── intermediate_results.out │ │ │ ├── isolation_acquire_distributed_locks.out │ │ │ ├── isolation_add_coordinator.out │ │ │ ├── isolation_add_node_vs_reference_table_operations.out │ │ │ ├── isolation_add_remove_node.out │ │ │ ├── isolation_append_copy_vs_all.out │ │ │ ├── isolation_blocking_move_multi_shard_commands.out │ │ │ ├── isolation_blocking_move_multi_shard_commands_on_mx.out │ │ │ ├── isolation_blocking_move_single_shard_commands.out │ │ │ ├── isolation_blocking_move_single_shard_commands_on_mx.out │ │ │ ├── isolation_blocking_shard_split.out │ │ │ ├── isolation_blocking_shard_split_with_fkey_to_reference.out │ │ │ ├── isolation_cancellation.out │ │ │ ├── isolation_check_mx.out │ │ │ ├── isolation_citus_dist_activity.out │ │ │ ├── isolation_citus_locks.out │ │ │ ├── isolation_citus_pause_node.out │ │ │ ├── isolation_citus_pause_node_1.out │ │ │ ├── isolation_citus_schema_distribute_undistribute.out │ │ │ ├── isolation_cluster_management.out │ │ │ ├── isolation_concurrent_dml.out │ │ │ ├── isolation_concurrent_move_create_table.out │ │ │ ├── isolation_copy_placement_vs_copy_placement.out │ │ │ ├── isolation_copy_placement_vs_modification.out │ │ │ ├── isolation_copy_vs_all_on_mx.out │ │ │ ├── isolation_create_citus_local_table.out │ │ │ ├── isolation_create_distributed_concurrently_after_drop_column.out │ │ │ ├── isolation_create_distributed_table.out │ │ │ ├── isolation_create_distributed_table_concurrently.out │ │ │ ├── isolation_create_restore_point.out │ │ │ ├── isolation_create_table_vs_add_remove_node.out │ │ │ ├── isolation_data_migration.out │ │ │ ├── isolation_database_cmd_from_any_node.out │ │ │ ├── isolation_ddl_vs_all.out │ │ │ ├── isolation_delete_vs_all.out │ │ │ ├── isolation_dis2ref_foreign_keys_on_mx.out │ │ │ ├── isolation_distributed_deadlock_detection.out │ │ │ ├── isolation_distributed_transaction_id.out │ │ │ ├── isolation_drop_alter_index_select_for_update_on_mx.out │ │ │ ├── isolation_drop_shards.out │ │ │ ├── isolation_drop_vs_all.out │ │ │ ├── isolation_dump_global_wait_edges.out │ │ │ ├── isolation_dump_local_wait_edges.out │ │ │ ├── isolation_ensure_dependency_activate_node.out │ │ │ ├── isolation_extension_commands.out │ │ │ ├── isolation_fix_partition_shard_index_names.out │ │ │ ├── isolation_get_all_active_transactions.out │ │ │ ├── isolation_get_distributed_wait_queries_mx.out │ │ │ ├── isolation_global_pid.out │ │ │ ├── isolation_hash_copy_vs_all.out │ │ │ ├── isolation_insert_select_conflict.out │ │ │ ├── isolation_insert_select_repartition.out │ │ │ ├── isolation_insert_select_vs_all.out │ │ │ ├── isolation_insert_select_vs_all_on_mx.out │ │ │ ├── isolation_insert_vs_all.out │ │ │ ├── isolation_insert_vs_all_on_mx.out │ │ │ ├── isolation_insert_vs_vacuum.out │ │ │ ├── isolation_logical_replication_binaryless.out │ │ │ ├── isolation_logical_replication_multi_shard_commands.out │ │ │ ├── isolation_logical_replication_multi_shard_commands_on_mx.out │ │ │ ├── isolation_logical_replication_nonsu_nonbypassrls.out │ │ │ ├── isolation_logical_replication_single_shard_commands.out │ │ │ ├── isolation_logical_replication_single_shard_commands_on_mx.out │ │ │ ├── isolation_logical_replication_skip_fk_validation.out │ │ │ ├── isolation_logical_replication_with_partitioning.out │ │ │ ├── isolation_master_update_node.out │ │ │ ├── isolation_master_update_node_0.out │ │ │ ├── isolation_master_update_node_1.out │ │ │ ├── isolation_max_client_connections.out │ │ │ ├── isolation_merge.out │ │ │ ├── isolation_merge_replicated.out │ │ │ ├── isolation_metadata_sync_deadlock.out │ │ │ ├── isolation_metadata_sync_vs_all.out │ │ │ ├── isolation_modify_with_subquery_vs_dml.out │ │ │ ├── isolation_move_placement_vs_modification.out │ │ │ ├── isolation_move_placement_vs_modification_fk.out │ │ │ ├── isolation_move_placement_vs_move_placement.out │ │ │ ├── isolation_multi_shard_modify_vs_all.out │ │ │ ├── isolation_multiuser_locking.out │ │ │ ├── isolation_multiuser_locking_0.out │ │ │ ├── isolation_non_blocking_shard_split.out │ │ │ ├── isolation_non_blocking_shard_split_fkey.out │ │ │ ├── isolation_non_blocking_shard_split_with_index_as_replicaIdentity.out │ │ │ ├── isolation_partitioned_copy_vs_all.out │ │ │ ├── isolation_progress_monitoring.out │ │ │ ├── isolation_range_copy_vs_all.out │ │ │ ├── isolation_rebalancer_deferred_drop.out │ │ │ ├── isolation_ref2ref_foreign_keys.out │ │ │ ├── isolation_ref2ref_foreign_keys_enterprise.out │ │ │ ├── isolation_ref2ref_foreign_keys_on_mx.out │ │ │ ├── isolation_ref_select_for_update_vs_all_on_mx.out │ │ │ ├── isolation_ref_update_delete_upsert_vs_all_on_mx.out │ │ │ ├── isolation_reference_copy_vs_all.out │ │ │ ├── isolation_reference_on_mx.out │ │ │ ├── isolation_reference_table.out │ │ │ ├── isolation_remove_coordinator.out │ │ │ ├── isolation_replace_wait_function.out │ │ │ ├── isolation_replicate_reference_tables_to_coordinator.out │ │ │ ├── isolation_replicated_dist_on_mx.out │ │ │ ├── isolation_schema_based_sharding.out │ │ │ ├── isolation_select_for_update.out │ │ │ ├── isolation_select_vs_all.out │ │ │ ├── isolation_select_vs_all_on_mx.out │ │ │ ├── isolation_setup.out │ │ │ ├── isolation_shard_move_vs_start_metadata_sync.out │ │ │ ├── isolation_shard_rebalancer.out │ │ │ ├── isolation_shard_rebalancer_progress.out │ │ │ ├── isolation_shouldhaveshards.out │ │ │ ├── isolation_tenant_isolation.out │ │ │ ├── isolation_tenant_isolation_nonblocking.out │ │ │ ├── isolation_tenant_isolation_with_fkey_to_reference.out │ │ │ ├── isolation_transaction_recovery.out │ │ │ ├── isolation_truncate_vs_all.out │ │ │ ├── isolation_truncate_vs_all_on_mx.out │ │ │ ├── isolation_undistribute_table.out │ │ │ ├── isolation_update_delete_upsert_vs_all_on_mx.out │ │ │ ├── isolation_update_node.out │ │ │ ├── isolation_update_node_lock_writes.out │ │ │ ├── isolation_update_vs_all.out │ │ │ ├── isolation_upsert_vs_all.out │ │ │ ├── isolation_vacuum_skip_locked.out │ │ │ ├── isolation_validate_vs_insert.out │ │ │ ├── issue_5099.out │ │ │ ├── issue_5248.out │ │ │ ├── issue_5763.out │ │ │ ├── issue_6543.out │ │ │ ├── issue_6592.out │ │ │ ├── issue_6758.out │ │ │ ├── issue_7477.out │ │ │ ├── issue_7891.out │ │ │ ├── issue_8243.out │ │ │ ├── join_pushdown.out │ │ │ ├── limit_intermediate_size.out │ │ │ ├── local_dist_join.out │ │ │ ├── local_dist_join_load.out │ │ │ ├── local_dist_join_mixed.out │ │ │ ├── local_dist_join_modifications.out │ │ │ ├── local_execution_local_plan.out │ │ │ ├── local_shard_copy.out │ │ │ ├── local_shard_execution.out │ │ │ ├── local_shard_execution_dropped_column.out │ │ │ ├── local_shard_execution_replicated.out │ │ │ ├── local_shard_utility_command_execution.out │ │ │ ├── local_table_join.out │ │ │ ├── locally_execute_intermediate_results.out │ │ │ ├── logical_rep_consistency.out │ │ │ ├── logical_replication.out │ │ │ ├── materialized_view.out │ │ │ ├── merge.out │ │ │ ├── merge_arbitrary.out │ │ │ ├── merge_arbitrary_create.out │ │ │ ├── merge_partition_tables.out │ │ │ ├── merge_repartition1.out │ │ │ ├── merge_repartition2.out │ │ │ ├── merge_schema_sharding.out │ │ │ ├── merge_unsupported.out │ │ │ ├── merge_unsupported_0.out │ │ │ ├── merge_vcore.out │ │ │ ├── metadata_sync_from_non_maindb.out │ │ │ ├── metadata_sync_helpers.out │ │ │ ├── minimal_cluster_management.out │ │ │ ├── mixed_relkind_tests.out │ │ │ ├── modification_correctness.out │ │ │ ├── multi_add_node_from_backup.out │ │ │ ├── multi_add_node_from_backup_negative.out │ │ │ ├── multi_add_node_from_backup_sync_replica.out │ │ │ ├── multi_agg_approximate_distinct.out │ │ │ ├── multi_agg_approximate_distinct_0.out │ │ │ ├── multi_agg_distinct.out │ │ │ ├── multi_agg_type_conversion.out │ │ │ ├── multi_alias.out │ │ │ ├── multi_alter_table_add_constraints.out │ │ │ ├── multi_alter_table_add_constraints_without_name.out │ │ │ ├── multi_alter_table_add_foreign_key_without_name.out │ │ │ ├── multi_alter_table_row_level_security.out │ │ │ ├── multi_alter_table_row_level_security_escape.out │ │ │ ├── multi_alter_table_statements.out │ │ │ ├── multi_array_agg.out │ │ │ ├── multi_average_expression.out │ │ │ ├── multi_basic_queries.out │ │ │ ├── multi_behavioral_analytics_basics.out │ │ │ ├── multi_behavioral_analytics_create_table.out │ │ │ ├── multi_behavioral_analytics_create_table_superuser.out │ │ │ ├── multi_behavioral_analytics_single_shard_queries.out │ │ │ ├── multi_cache_invalidation.out │ │ │ ├── multi_citus_tools.out │ │ │ ├── multi_cluster_management.out │ │ │ ├── multi_colocated_shard_rebalance.out │ │ │ ├── multi_colocation_utils.out │ │ │ ├── multi_complex_count_distinct.out │ │ │ ├── multi_complex_expressions.out │ │ │ ├── multi_copy.out │ │ │ ├── multi_count_type_conversion.out │ │ │ ├── multi_create_fdw.out │ │ │ ├── multi_create_role_dependency.out │ │ │ ├── multi_create_shards.out │ │ │ ├── multi_create_table.out │ │ │ ├── multi_create_table_constraints.out │ │ │ ├── multi_create_table_superuser.out │ │ │ ├── multi_create_users.out │ │ │ ├── multi_cross_shard.out │ │ │ ├── multi_data_types.out │ │ │ ├── multi_deparse_function.out │ │ │ ├── multi_deparse_procedure.out │ │ │ ├── multi_deparse_shard_query.out │ │ │ ├── multi_distributed_transaction_id.out │ │ │ ├── multi_distribution_metadata.out │ │ │ ├── multi_drop_extension.out │ │ │ ├── multi_dropped_column_aliases.out │ │ │ ├── multi_explain.out │ │ │ ├── multi_explain_0.out │ │ │ ├── multi_extension.out │ │ │ ├── multi_fix_partition_shard_index_names.out │ │ │ ├── multi_follower_configure_followers.out │ │ │ ├── multi_follower_dml.out │ │ │ ├── multi_follower_sanity_check.out │ │ │ ├── multi_follower_select_statements.out │ │ │ ├── multi_foreign_key.out │ │ │ ├── multi_foreign_key_relation_graph.out │ │ │ ├── multi_function_evaluation.out │ │ │ ├── multi_function_in_join.out │ │ │ ├── multi_generate_ddl_commands.out │ │ │ ├── multi_hash_pruning.out │ │ │ ├── multi_having_pushdown.out │ │ │ ├── multi_index_statements.out │ │ │ ├── multi_insert_select.out │ │ │ ├── multi_insert_select_behavioral_analytics_create_table.out │ │ │ ├── multi_insert_select_conflict.out │ │ │ ├── multi_insert_select_non_pushable_queries.out │ │ │ ├── multi_insert_select_window.out │ │ │ ├── multi_join_order_additional.out │ │ │ ├── multi_join_order_tpch_repartition.out │ │ │ ├── multi_join_order_tpch_small.out │ │ │ ├── multi_join_pruning.out │ │ │ ├── multi_json_agg.out │ │ │ ├── multi_json_object_agg.out │ │ │ ├── multi_jsonb_agg.out │ │ │ ├── multi_jsonb_object_agg.out │ │ │ ├── multi_large_shardid.out │ │ │ ├── multi_level_recursive_queries.out │ │ │ ├── multi_limit_clause.out │ │ │ ├── multi_limit_clause_approximate.out │ │ │ ├── multi_load_data.out │ │ │ ├── multi_load_data_superuser.out │ │ │ ├── multi_load_more_data.out │ │ │ ├── multi_master_protocol.out │ │ │ ├── multi_metadata_access.out │ │ │ ├── multi_metadata_attributes.out │ │ │ ├── multi_metadata_sync.out │ │ │ ├── multi_metadata_sync_domain.out │ │ │ ├── multi_modifications.out │ │ │ ├── multi_modifying_xacts.out │ │ │ ├── multi_move_mx.out │ │ │ ├── multi_multiuser.out │ │ │ ├── multi_multiuser_auth.out │ │ │ ├── multi_multiuser_basic_queries.out │ │ │ ├── multi_multiuser_copy.out │ │ │ ├── multi_multiuser_grant.out │ │ │ ├── multi_multiuser_load_data.out │ │ │ ├── multi_multiuser_master_protocol.out │ │ │ ├── multi_multiuser_master_protocol_0.out │ │ │ ├── multi_mx_add_coordinator.out │ │ │ ├── multi_mx_alter_distributed_table.out │ │ │ ├── multi_mx_call.out │ │ │ ├── multi_mx_call_0.out │ │ │ ├── multi_mx_copy_data.out │ │ │ ├── multi_mx_create_table.out │ │ │ ├── multi_mx_ddl.out │ │ │ ├── multi_mx_explain.out │ │ │ ├── multi_mx_explain_0.out │ │ │ ├── multi_mx_function_call_delegation.out │ │ │ ├── multi_mx_function_call_delegation_0.out │ │ │ ├── multi_mx_function_table_reference.out │ │ │ ├── multi_mx_hide_shard_names.out │ │ │ ├── multi_mx_hide_shard_names_0.out │ │ │ ├── multi_mx_insert_select_repartition.out │ │ │ ├── multi_mx_metadata.out │ │ │ ├── multi_mx_modifications.out │ │ │ ├── multi_mx_modifications_to_reference_tables.out │ │ │ ├── multi_mx_modifying_xacts.out │ │ │ ├── multi_mx_node_metadata.out │ │ │ ├── multi_mx_partitioning.out │ │ │ ├── multi_mx_reference_table.out │ │ │ ├── multi_mx_repartition_join_w1.out │ │ │ ├── multi_mx_repartition_join_w2.out │ │ │ ├── multi_mx_repartition_udt_prepare.out │ │ │ ├── multi_mx_repartition_udt_w1.out │ │ │ ├── multi_mx_repartition_udt_w2.out │ │ │ ├── multi_mx_router_planner.out │ │ │ ├── multi_mx_schema_support.out │ │ │ ├── multi_mx_tpch_query1.out │ │ │ ├── multi_mx_tpch_query10.out │ │ │ ├── multi_mx_tpch_query12.out │ │ │ ├── multi_mx_tpch_query14.out │ │ │ ├── multi_mx_tpch_query19.out │ │ │ ├── multi_mx_tpch_query3.out │ │ │ ├── multi_mx_tpch_query6.out │ │ │ ├── multi_mx_tpch_query7.out │ │ │ ├── multi_mx_tpch_query7_nested.out │ │ │ ├── multi_mx_transaction_recovery.out │ │ │ ├── multi_mx_truncate_from_worker.out │ │ │ ├── multi_name_lengths.out │ │ │ ├── multi_name_resolution.out │ │ │ ├── multi_null_minmax_value_pruning.out │ │ │ ├── multi_orderby_limit_pushdown.out │ │ │ ├── multi_outer_join.out │ │ │ ├── multi_outer_join_columns.out │ │ │ ├── multi_outer_join_reference.out │ │ │ ├── multi_partition_pruning.out │ │ │ ├── multi_partitioning.out │ │ │ ├── multi_partitioning_utils.out │ │ │ ├── multi_poolinfo_usage.out │ │ │ ├── multi_prepare_plsql.out │ │ │ ├── multi_prepare_sql.out │ │ │ ├── multi_prune_shard_list.out │ │ │ ├── multi_query_directory_cleanup.out │ │ │ ├── multi_read_from_secondaries.out │ │ │ ├── multi_real_time_transaction.out │ │ │ ├── multi_reference_table.out │ │ │ ├── multi_remove_node_reference_table.out │ │ │ ├── multi_repartition_join_planning.out │ │ │ ├── multi_repartition_join_pruning.out │ │ │ ├── multi_repartition_join_ref.out │ │ │ ├── multi_repartition_join_task_assignment.out │ │ │ ├── multi_repartition_udt.out │ │ │ ├── multi_repartitioned_subquery_udf.out │ │ │ ├── multi_replicate_reference_table.out │ │ │ ├── multi_rls_join_distribution_key.out │ │ │ ├── multi_router_planner.out │ │ │ ├── multi_router_planner_fast_path.out │ │ │ ├── multi_row_insert.out │ │ │ ├── multi_row_router_insert.out │ │ │ ├── multi_schema_support.out │ │ │ ├── multi_select_distinct.out │ │ │ ├── multi_select_for_update.out │ │ │ ├── multi_sequence_default.out │ │ │ ├── multi_shard_update_delete.out │ │ │ ├── multi_simple_queries.out │ │ │ ├── multi_single_relation_subquery.out │ │ │ ├── multi_size_queries.out │ │ │ ├── multi_sql_function.out │ │ │ ├── multi_sql_function_0.out │ │ │ ├── multi_subquery.out │ │ │ ├── multi_subquery_behavioral_analytics.out │ │ │ ├── multi_subquery_complex_queries.out │ │ │ ├── multi_subquery_complex_reference_clause.out │ │ │ ├── multi_subquery_in_where_clause.out │ │ │ ├── multi_subquery_in_where_reference_clause.out │ │ │ ├── multi_subquery_misc.out │ │ │ ├── multi_subquery_misc_0.out │ │ │ ├── multi_subquery_union.out │ │ │ ├── multi_subquery_window_functions.out │ │ │ ├── multi_subtransactions.out │ │ │ ├── multi_table_ddl.out │ │ │ ├── multi_task_assignment_policy.out │ │ │ ├── multi_task_string_size.out │ │ │ ├── multi_tenant_isolation.out │ │ │ ├── multi_tenant_isolation_nonblocking.out │ │ │ ├── multi_test_catalog_views.out │ │ │ ├── multi_test_helpers.out │ │ │ ├── multi_test_helpers_superuser.out │ │ │ ├── multi_tpch_query1.out │ │ │ ├── multi_tpch_query10.out │ │ │ ├── multi_tpch_query12.out │ │ │ ├── multi_tpch_query14.out │ │ │ ├── multi_tpch_query19.out │ │ │ ├── multi_tpch_query3.out │ │ │ ├── multi_tpch_query6.out │ │ │ ├── multi_tpch_query7.out │ │ │ ├── multi_tpch_query7_nested.out │ │ │ ├── multi_transaction_recovery.out │ │ │ ├── multi_transaction_recovery_multiple_databases.out │ │ │ ├── multi_transactional_drop_shards.out │ │ │ ├── multi_truncate.out │ │ │ ├── multi_unsupported_worker_operations.out │ │ │ ├── multi_update_select.out │ │ │ ├── multi_upsert.out │ │ │ ├── multi_utilities.out │ │ │ ├── multi_utility_statements.out │ │ │ ├── multi_utility_warnings.out │ │ │ ├── multi_view.out │ │ │ ├── multi_working_columns.out │ │ │ ├── mx_coordinator_shouldhaveshards.out │ │ │ ├── mx_foreign_key_to_reference_table.out │ │ │ ├── mx_regular_user.out │ │ │ ├── nested_execution.out │ │ │ ├── nested_execution_create.out │ │ │ ├── node_conninfo_reload.out │ │ │ ├── non_colocated_leaf_subquery_joins.out │ │ │ ├── non_colocated_subquery_joins.out │ │ │ ├── non_super_user_cdc_library_path.out │ │ │ ├── non_super_user_object_metadata.out │ │ │ ├── null_parameters.out │ │ │ ├── object_propagation_debug.out │ │ │ ├── other_databases.out │ │ │ ├── partition_wise_join.out │ │ │ ├── partition_wise_join_0.out │ │ │ ├── partitioned_indexes_create.out │ │ │ ├── partitioned_intermediate_results.out │ │ │ ├── partitioning_issue_3970.out │ │ │ ├── pg12.out │ │ │ ├── pg13.out │ │ │ ├── pg13_propagate_statistics.out │ │ │ ├── pg13_with_ties.out │ │ │ ├── pg14.out │ │ │ ├── pg15.out │ │ │ ├── pg15_jsonpath.out │ │ │ ├── pg16.out │ │ │ ├── pg17.out │ │ │ ├── pg17_0.out │ │ │ ├── pg17_json.out │ │ │ ├── pg17_json_0.out │ │ │ ├── pg18.out │ │ │ ├── pg18_0.out │ │ │ ├── pg_dump.out │ │ │ ├── pgmerge.out │ │ │ ├── postgres.out │ │ │ ├── prepared_statements_1.out │ │ │ ├── prepared_statements_2.out │ │ │ ├── prepared_statements_3.out │ │ │ ├── prepared_statements_4.out │ │ │ ├── prepared_statements_create_load.out │ │ │ ├── propagate_extension_commands.out │ │ │ ├── propagate_foreign_servers.out │ │ │ ├── propagate_set_commands.out │ │ │ ├── propagate_statistics.out │ │ │ ├── publication.out │ │ │ ├── query_single_shard_table.out │ │ │ ├── reassign_owned.out │ │ │ ├── recurring_join_pushdown.out │ │ │ ├── recurring_outer_join.out │ │ │ ├── recursive_dml_queries_mx.out │ │ │ ├── recursive_dml_with_different_planners_executors.out │ │ │ ├── recursive_relation_planning_restriction_pushdown.out │ │ │ ├── recursive_view_local_table.out │ │ │ ├── ref_citus_local_fkeys.out │ │ │ ├── relation_access_tracking.out │ │ │ ├── relation_access_tracking_single_node.out │ │ │ ├── remove_coordinator.out │ │ │ ├── remove_coordinator_from_metadata.out │ │ │ ├── remove_non_default_nodes.out │ │ │ ├── rename_public_to_citus_schema_and_recreate.out │ │ │ ├── replicate_reference_tables_to_coordinator.out │ │ │ ├── replicated_partitioned_table.out │ │ │ ├── replicated_table_disable_node.out │ │ │ ├── resync_metadata_with_sequences.out │ │ │ ├── role_command_from_any_node.out │ │ │ ├── role_operations_from_non_maindb.out │ │ │ ├── row_types.out │ │ │ ├── run_command_on_all_nodes.out │ │ │ ├── schema_based_sharding.out │ │ │ ├── schemas.out │ │ │ ├── schemas_create.out │ │ │ ├── seclabel.out │ │ │ ├── seclabel_non_maindb.out │ │ │ ├── sequences.out │ │ │ ├── sequences_create.out │ │ │ ├── sequences_owned_by.out │ │ │ ├── sequences_with_different_types.out │ │ │ ├── sequential_modifications.out │ │ │ ├── set_operation_and_local_tables.out │ │ │ ├── set_operations.out │ │ │ ├── set_role_in_transaction.out │ │ │ ├── shard_move_constraints.out │ │ │ ├── shard_move_constraints_blocking.out │ │ │ ├── shard_move_deferred_delete.out │ │ │ ├── shard_rebalancer.out │ │ │ ├── shard_rebalancer_unit.out │ │ │ ├── shared_connection_stats.out │ │ │ ├── shared_connection_waits.out │ │ │ ├── single_hash_repartition_join.out │ │ │ ├── single_node.out │ │ │ ├── single_node_enterprise.out │ │ │ ├── single_node_truncate.out │ │ │ ├── single_shard_table_prep.out │ │ │ ├── single_shard_table_udfs.out │ │ │ ├── split_shard.out │ │ │ ├── sql_procedure.out │ │ │ ├── sqlancer_failures.out │ │ │ ├── sqlsmith_failures.out │ │ │ ├── ssl_by_default.out │ │ │ ├── start_stop_metadata_sync.out │ │ │ ├── stat_counters.out │ │ │ ├── stat_statements.out │ │ │ ├── statement_cancel_error_message.out │ │ │ ├── subqueries_deep.out │ │ │ ├── subqueries_not_supported.out │ │ │ ├── subquery_and_cte.out │ │ │ ├── subquery_append.out │ │ │ ├── subquery_basics.out │ │ │ ├── subquery_complex_target_list.out │ │ │ ├── subquery_executors.out │ │ │ ├── subquery_in_targetlist.out │ │ │ ├── subquery_in_where.out │ │ │ ├── subquery_local_tables.out │ │ │ ├── subquery_partitioning.out │ │ │ ├── subquery_prepared_statements.out │ │ │ ├── subquery_view.out │ │ │ ├── subscripting_op.out │ │ │ ├── system_queries.out │ │ │ ├── tableam.out │ │ │ ├── tablespace.out │ │ │ ├── task_tracker_assign_task.out │ │ │ ├── task_tracker_cleanup_job.out │ │ │ ├── task_tracker_create_table.out │ │ │ ├── task_tracker_partition_task.out │ │ │ ├── tdigest_aggregate_support.out │ │ │ ├── tdigest_aggregate_support_0.out │ │ │ ├── tdigest_aggregate_support_1.out │ │ │ ├── text_search.out │ │ │ ├── undistribute_table.out │ │ │ ├── undistribute_table_cascade.out │ │ │ ├── undistribute_table_cascade_mx.out │ │ │ ├── union_pushdown.out │ │ │ ├── unsupported_lateral_subqueries.out │ │ │ ├── update_colocation_mx.out │ │ │ ├── upgrade_autoconverted_after.out │ │ │ ├── upgrade_autoconverted_before.out │ │ │ ├── upgrade_basic_after.out │ │ │ ├── upgrade_basic_after_non_mixed.out │ │ │ ├── upgrade_basic_before.out │ │ │ ├── upgrade_basic_before_non_mixed.out │ │ │ ├── upgrade_citus_finish_citus_upgrade.out │ │ │ ├── upgrade_citus_locks.out │ │ │ ├── upgrade_citus_stat_activity.out │ │ │ ├── upgrade_columnar_after.out │ │ │ ├── upgrade_columnar_before.out │ │ │ ├── upgrade_distributed_function_after.out │ │ │ ├── upgrade_distributed_function_before.out │ │ │ ├── upgrade_distributed_triggers_after.out │ │ │ ├── upgrade_distributed_triggers_after_0.out │ │ │ ├── upgrade_distributed_triggers_before.out │ │ │ ├── upgrade_distributed_triggers_before_0.out │ │ │ ├── upgrade_list_citus_objects.out │ │ │ ├── upgrade_pg_dist_cleanup_after.out │ │ │ ├── upgrade_pg_dist_cleanup_after_0.out │ │ │ ├── upgrade_pg_dist_cleanup_before.out │ │ │ ├── upgrade_pg_dist_cleanup_before_0.out │ │ │ ├── upgrade_post_11_after.out │ │ │ ├── upgrade_post_11_after_0.out │ │ │ ├── upgrade_post_11_before.out │ │ │ ├── upgrade_post_11_before_0.out │ │ │ ├── upgrade_post_14_after.out │ │ │ ├── upgrade_post_14_after_0.out │ │ │ ├── upgrade_post_14_before.out │ │ │ ├── upgrade_post_14_before_0.out │ │ │ ├── upgrade_rebalance_strategy_after.out │ │ │ ├── upgrade_rebalance_strategy_before.out │ │ │ ├── upgrade_ref2ref_after.out │ │ │ ├── upgrade_ref2ref_before.out │ │ │ ├── upgrade_schema_based_sharding_after.out │ │ │ ├── upgrade_schema_based_sharding_before.out │ │ │ ├── upgrade_single_shard_table_after.out │ │ │ ├── upgrade_single_shard_table_before.out │ │ │ ├── upgrade_type_after.out │ │ │ ├── upgrade_type_before.out │ │ │ ├── validate_constraint.out │ │ │ ├── values.out │ │ │ ├── view_propagation.out │ │ │ ├── views.out │ │ │ ├── views_create.out │ │ │ ├── window_functions.out │ │ │ ├── with_basics.out │ │ │ ├── with_dml.out │ │ │ ├── with_executors.out │ │ │ ├── with_join.out │ │ │ ├── with_modifying.out │ │ │ ├── with_nested.out │ │ │ ├── with_partitioning.out │ │ │ ├── with_prepare.out │ │ │ ├── with_set_operations.out │ │ │ ├── with_transactions.out │ │ │ ├── with_where.out │ │ │ ├── worker_copy_table_to_node.out │ │ │ ├── worker_split_binary_copy_test.out │ │ │ ├── worker_split_copy_test.out │ │ │ └── worker_split_text_copy_test.out │ │ ├── failure_base_schedule │ │ ├── failure_schedule │ │ ├── flaky_tests.md │ │ ├── isolation_schedule │ │ ├── log_test_times │ │ ├── minimal_columnar_schedule │ │ ├── minimal_pg_upgrade_schedule │ │ ├── minimal_schedule │ │ ├── mitmscripts/ │ │ │ ├── .gitignore │ │ │ ├── CONTRIBUTING.md │ │ │ ├── README.md │ │ │ ├── fluent.py │ │ │ └── structs.py │ │ ├── mixed_after_citus_upgrade_schedule │ │ ├── mixed_before_citus_upgrade_schedule │ │ ├── multi_1_create_citus_schedule │ │ ├── multi_1_schedule │ │ ├── multi_add_backup_node_schedule │ │ ├── multi_follower_schedule │ │ ├── multi_mx_schedule │ │ ├── multi_schedule │ │ ├── multi_schedule_hyperscale │ │ ├── multi_schedule_hyperscale_superuser │ │ ├── mx_base_schedule │ │ ├── mx_minimal_schedule │ │ ├── operations_schedule │ │ ├── pg_regress_multi.pl │ │ ├── postgres_schedule │ │ ├── single_shard_table_prep_schedule │ │ ├── spec/ │ │ │ ├── README.md │ │ │ ├── columnar_index_concurrency.spec │ │ │ ├── columnar_temp_tables.spec │ │ │ ├── columnar_vacuum_vs_insert.spec │ │ │ ├── columnar_write_concurrency.spec │ │ │ ├── columnar_write_concurrency_index.spec │ │ │ ├── isolation_acquire_distributed_locks.spec │ │ │ ├── isolation_add_coordinator.spec │ │ │ ├── isolation_add_node_vs_reference_table_operations.spec │ │ │ ├── isolation_add_remove_node.spec │ │ │ ├── isolation_blocking_move_multi_shard_commands.spec │ │ │ ├── isolation_blocking_move_multi_shard_commands_on_mx.spec │ │ │ ├── isolation_blocking_move_single_shard_commands.spec │ │ │ ├── isolation_blocking_move_single_shard_commands_on_mx.spec │ │ │ ├── isolation_blocking_shard_split.spec │ │ │ ├── isolation_blocking_shard_split_with_fkey_to_reference.spec │ │ │ ├── isolation_cancellation.spec │ │ │ ├── isolation_check_mx.spec │ │ │ ├── isolation_citus_dist_activity.spec │ │ │ ├── isolation_citus_locks.spec │ │ │ ├── isolation_citus_pause_node.spec │ │ │ ├── isolation_citus_schema_distribute_undistribute.spec │ │ │ ├── isolation_cluster_management.spec │ │ │ ├── isolation_concurrent_dml.spec │ │ │ ├── isolation_concurrent_move_create_table.spec │ │ │ ├── isolation_copy_placement_vs_copy_placement.spec │ │ │ ├── isolation_copy_placement_vs_modification.spec │ │ │ ├── isolation_copy_vs_all_on_mx.spec │ │ │ ├── isolation_create_citus_local_table.spec │ │ │ ├── isolation_create_distributed_concurrently_after_drop_column.spec │ │ │ ├── isolation_create_distributed_table.spec │ │ │ ├── isolation_create_distributed_table_concurrently.spec │ │ │ ├── isolation_create_restore_point.spec │ │ │ ├── isolation_create_table_vs_add_remove_node.spec │ │ │ ├── isolation_data_migration.spec │ │ │ ├── isolation_database_cmd_from_any_node.spec │ │ │ ├── isolation_ddl_vs_all.spec │ │ │ ├── isolation_delete_vs_all.spec │ │ │ ├── isolation_dis2ref_foreign_keys_on_mx.spec │ │ │ ├── isolation_distributed_deadlock_detection.spec │ │ │ ├── isolation_distributed_transaction_id.spec │ │ │ ├── isolation_drop_alter_index_select_for_update_on_mx.spec │ │ │ ├── isolation_drop_shards.spec │ │ │ ├── isolation_drop_vs_all.spec │ │ │ ├── isolation_dump_global_wait_edges.spec │ │ │ ├── isolation_dump_local_wait_edges.spec │ │ │ ├── isolation_ensure_dependency_activate_node.spec │ │ │ ├── isolation_extension_commands.spec │ │ │ ├── isolation_fix_partition_shard_index_names.spec │ │ │ ├── isolation_get_all_active_transactions.spec │ │ │ ├── isolation_get_distributed_wait_queries_mx.spec │ │ │ ├── isolation_global_pid.spec │ │ │ ├── isolation_hash_copy_vs_all.spec │ │ │ ├── isolation_insert_select_conflict.spec │ │ │ ├── isolation_insert_select_repartition.spec │ │ │ ├── isolation_insert_select_vs_all.spec │ │ │ ├── isolation_insert_select_vs_all_on_mx.spec │ │ │ ├── isolation_insert_vs_all.spec │ │ │ ├── isolation_insert_vs_all_on_mx.spec │ │ │ ├── isolation_insert_vs_vacuum.spec │ │ │ ├── isolation_logical_replication_binaryless.spec │ │ │ ├── isolation_logical_replication_multi_shard_commands.spec │ │ │ ├── isolation_logical_replication_multi_shard_commands_on_mx.spec │ │ │ ├── isolation_logical_replication_nonsu_nonbypassrls.spec │ │ │ ├── isolation_logical_replication_single_shard_commands.spec │ │ │ ├── isolation_logical_replication_single_shard_commands_on_mx.spec │ │ │ ├── isolation_logical_replication_skip_fk_validation.spec │ │ │ ├── isolation_logical_replication_with_partitioning.spec │ │ │ ├── isolation_master_update_node.spec │ │ │ ├── isolation_max_client_connections.spec │ │ │ ├── isolation_merge.spec │ │ │ ├── isolation_merge_replicated.spec │ │ │ ├── isolation_metadata_sync_deadlock.spec │ │ │ ├── isolation_metadata_sync_vs_all.spec │ │ │ ├── isolation_modify_with_subquery_vs_dml.spec │ │ │ ├── isolation_move_placement_vs_modification.spec │ │ │ ├── isolation_move_placement_vs_modification_fk.spec │ │ │ ├── isolation_move_placement_vs_move_placement.spec │ │ │ ├── isolation_multi_shard_modify_vs_all.spec │ │ │ ├── isolation_multiuser_locking.spec │ │ │ ├── isolation_mx_common.include.spec │ │ │ ├── isolation_non_blocking_shard_split.spec │ │ │ ├── isolation_non_blocking_shard_split_fkey.spec │ │ │ ├── isolation_non_blocking_shard_split_with_index_as_replicaIdentity.spec │ │ │ ├── isolation_partitioned_copy_vs_all.spec │ │ │ ├── isolation_progress_monitoring.spec │ │ │ ├── isolation_range_copy_vs_all.spec │ │ │ ├── isolation_rebalancer_deferred_drop.spec │ │ │ ├── isolation_ref2ref_foreign_keys.spec │ │ │ ├── isolation_ref2ref_foreign_keys_enterprise.spec │ │ │ ├── isolation_ref2ref_foreign_keys_on_mx.spec │ │ │ ├── isolation_ref_select_for_update_vs_all_on_mx.spec │ │ │ ├── isolation_ref_update_delete_upsert_vs_all_on_mx.spec │ │ │ ├── isolation_reference_copy_vs_all.spec │ │ │ ├── isolation_reference_on_mx.spec │ │ │ ├── isolation_reference_table.spec │ │ │ ├── isolation_remove_coordinator.spec │ │ │ ├── isolation_replace_wait_function.spec │ │ │ ├── isolation_replicate_reference_tables_to_coordinator.spec │ │ │ ├── isolation_replicated_dist_on_mx.spec │ │ │ ├── isolation_schema_based_sharding.spec │ │ │ ├── isolation_select_for_update.spec │ │ │ ├── isolation_select_vs_all.spec │ │ │ ├── isolation_select_vs_all_on_mx.spec │ │ │ ├── isolation_setup.spec │ │ │ ├── isolation_shard_move_vs_start_metadata_sync.spec │ │ │ ├── isolation_shard_rebalancer.spec │ │ │ ├── isolation_shard_rebalancer_progress.spec │ │ │ ├── isolation_shouldhaveshards.spec │ │ │ ├── isolation_tenant_isolation.spec │ │ │ ├── isolation_tenant_isolation_nonblocking.spec │ │ │ ├── isolation_tenant_isolation_with_fkey_to_reference.spec │ │ │ ├── isolation_transaction_recovery.spec │ │ │ ├── isolation_truncate_vs_all.spec │ │ │ ├── isolation_truncate_vs_all_on_mx.spec │ │ │ ├── isolation_undistribute_table.spec │ │ │ ├── isolation_update_delete_upsert_vs_all_on_mx.spec │ │ │ ├── isolation_update_node.spec │ │ │ ├── isolation_update_node_lock_writes.spec │ │ │ ├── isolation_update_vs_all.spec │ │ │ ├── isolation_upsert_vs_all.spec │ │ │ ├── isolation_vacuum_skip_locked.spec │ │ │ ├── isolation_validate_vs_insert.spec │ │ │ └── shared_connection_waits.spec │ │ ├── split_schedule │ │ ├── sql/ │ │ │ ├── adaptive_executor.sql │ │ │ ├── adaptive_executor_repartition.sql │ │ │ ├── add_coordinator.sql │ │ │ ├── adv_lock_permission.sql │ │ │ ├── aggregate_support.sql │ │ │ ├── alter_database_owner.sql │ │ │ ├── alter_database_propagation.sql │ │ │ ├── alter_distributed_table.sql │ │ │ ├── alter_index.sql │ │ │ ├── alter_role_propagation.sql │ │ │ ├── alter_table_add_column.sql │ │ │ ├── alter_table_set_access_method.sql │ │ │ ├── alter_table_single_shard_table.sql │ │ │ ├── anonymous_columns.sql │ │ │ ├── arbitrary_configs_alter_table_add_constraint_without_name.sql │ │ │ ├── arbitrary_configs_alter_table_add_constraint_without_name_create.sql │ │ │ ├── arbitrary_configs_recurring_outer_join.sql │ │ │ ├── arbitrary_configs_router.sql │ │ │ ├── arbitrary_configs_router_create.sql │ │ │ ├── arbitrary_configs_truncate.sql │ │ │ ├── arbitrary_configs_truncate_cascade.sql │ │ │ ├── arbitrary_configs_truncate_cascade_create.sql │ │ │ ├── arbitrary_configs_truncate_create.sql │ │ │ ├── arbitrary_configs_truncate_partition.sql │ │ │ ├── arbitrary_configs_truncate_partition_create.sql │ │ │ ├── auto_undist_citus_local.sql │ │ │ ├── background_rebalance.sql │ │ │ ├── background_rebalance_parallel.sql │ │ │ ├── background_rebalance_parallel_reference_tables.sql │ │ │ ├── background_task_queue_monitor.sql │ │ │ ├── base_enable_mx.sql │ │ │ ├── binary_protocol.sql │ │ │ ├── bool_agg.sql │ │ │ ├── cdc_library_path.sql │ │ │ ├── ch_bench_having.sql │ │ │ ├── ch_bench_having_mx.sql │ │ │ ├── ch_bench_subquery_repartition.sql │ │ │ ├── ch_benchmarks_1.sql │ │ │ ├── ch_benchmarks_2.sql │ │ │ ├── ch_benchmarks_3.sql │ │ │ ├── ch_benchmarks_4.sql │ │ │ ├── ch_benchmarks_5.sql │ │ │ ├── ch_benchmarks_6.sql │ │ │ ├── ch_benchmarks_create_load.sql │ │ │ ├── chbenchmark_all_queries.sql │ │ │ ├── check_cluster_state.sql │ │ │ ├── check_mx.sql │ │ │ ├── citus_aggregated_stats.sql │ │ │ ├── citus_copy_shard_placement.sql │ │ │ ├── citus_depended_object.sql │ │ │ ├── citus_drain_node.sql │ │ │ ├── citus_internal_access.sql │ │ │ ├── citus_local_dist_joins.sql │ │ │ ├── citus_local_table_triggers.sql │ │ │ ├── citus_local_tables.sql │ │ │ ├── citus_local_tables_ent.sql │ │ │ ├── citus_local_tables_mx.sql │ │ │ ├── citus_local_tables_queries.sql │ │ │ ├── citus_local_tables_queries_mx.sql │ │ │ ├── citus_locks.sql │ │ │ ├── citus_non_blocking_split_columnar.sql │ │ │ ├── citus_non_blocking_split_shard_cleanup.sql │ │ │ ├── citus_non_blocking_split_shards.sql │ │ │ ├── citus_run_command.sql │ │ │ ├── citus_schema_distribute_undistribute.sql │ │ │ ├── citus_schema_move.sql │ │ │ ├── citus_shards.sql │ │ │ ├── citus_split_shard_by_split_points.sql │ │ │ ├── citus_split_shard_by_split_points_deferred_drop.sql │ │ │ ├── citus_split_shard_by_split_points_failure.sql │ │ │ ├── citus_split_shard_by_split_points_negative.sql │ │ │ ├── citus_split_shard_columnar_partitioned.sql │ │ │ ├── citus_stat_tenants.sql │ │ │ ├── citus_table_triggers.sql │ │ │ ├── citus_update_table_statistics.sql │ │ │ ├── clock.sql │ │ │ ├── columnar_alter.sql │ │ │ ├── columnar_alter_set_type.sql │ │ │ ├── columnar_analyze.sql │ │ │ ├── columnar_chunk_filtering.sql │ │ │ ├── columnar_citus_integration.sql │ │ │ ├── columnar_clean.sql │ │ │ ├── columnar_copyto.sql │ │ │ ├── columnar_create.sql │ │ │ ├── columnar_cursor.sql │ │ │ ├── columnar_data_types.sql │ │ │ ├── columnar_drop.sql │ │ │ ├── columnar_empty.sql │ │ │ ├── columnar_fallback_scan.sql │ │ │ ├── columnar_first_row_number.sql │ │ │ ├── columnar_indexes.sql │ │ │ ├── columnar_insert.sql │ │ │ ├── columnar_join.sql │ │ │ ├── columnar_load.sql │ │ │ ├── columnar_lz4.sql │ │ │ ├── columnar_matview.sql │ │ │ ├── columnar_memory.sql │ │ │ ├── columnar_partitioning.sql │ │ │ ├── columnar_paths.sql │ │ │ ├── columnar_permissions.sql │ │ │ ├── columnar_pg15.sql │ │ │ ├── columnar_query.sql │ │ │ ├── columnar_recursive.sql │ │ │ ├── columnar_rollback.sql │ │ │ ├── columnar_tableoptions.sql │ │ │ ├── columnar_test_helpers.sql │ │ │ ├── columnar_transactions.sql │ │ │ ├── columnar_trigger.sql │ │ │ ├── columnar_truncate.sql │ │ │ ├── columnar_types_without_comparison.sql │ │ │ ├── columnar_update_delete.sql │ │ │ ├── columnar_vacuum.sql │ │ │ ├── columnar_zstd.sql │ │ │ ├── comment_on_database.sql │ │ │ ├── comment_on_role.sql │ │ │ ├── connectivity_checks.sql │ │ │ ├── coordinator_evaluation.sql │ │ │ ├── coordinator_evaluation_modify.sql │ │ │ ├── coordinator_evaluation_select.sql │ │ │ ├── coordinator_shouldhaveshards.sql │ │ │ ├── cpu_priority.sql │ │ │ ├── create_citus_local_table_cascade.sql │ │ │ ├── create_distributed_table_concurrently.sql │ │ │ ├── create_drop_database_propagation.sql │ │ │ ├── create_drop_database_propagation_pg15.sql │ │ │ ├── create_drop_database_propagation_pg16.sql │ │ │ ├── create_ref_dist_from_citus_local.sql │ │ │ ├── create_role_propagation.sql │ │ │ ├── create_single_shard_table.sql │ │ │ ├── cross_join.sql │ │ │ ├── cte_inline.sql │ │ │ ├── cte_nested_modification.sql │ │ │ ├── cte_prepared_modify.sql │ │ │ ├── cursors.sql │ │ │ ├── custom_aggregate_support.sql │ │ │ ├── data_types.sql │ │ │ ├── detect_conn_close.sql │ │ │ ├── disable_object_propagation.sql │ │ │ ├── distributed_collations.sql │ │ │ ├── distributed_collations_conflict.sql │ │ │ ├── distributed_domain.sql │ │ │ ├── distributed_functions.sql │ │ │ ├── distributed_functions_conflict.sql │ │ │ ├── distributed_intermediate_results.sql │ │ │ ├── distributed_locks.sql │ │ │ ├── distributed_planning.sql │ │ │ ├── distributed_planning_create_load.sql │ │ │ ├── distributed_procedure.sql │ │ │ ├── distributed_triggers.sql │ │ │ ├── distributed_types.sql │ │ │ ├── distributed_types_conflict.sql │ │ │ ├── distributed_types_xact_add_enum_value.sql │ │ │ ├── dml_recursive.sql │ │ │ ├── drop_column_partitioned_table.sql │ │ │ ├── drop_database.sql │ │ │ ├── drop_partitioned_table.sql │ │ │ ├── dropped_columns_1.sql │ │ │ ├── dropped_columns_create_load.sql │ │ │ ├── ensure_citus_columnar_not_exists.sql │ │ │ ├── ensure_no_intermediate_data_leak.sql │ │ │ ├── ensure_no_shared_connection_leak.sql │ │ │ ├── escape_extension_name.sql │ │ │ ├── executor_local_failure.sql │ │ │ ├── expression_reference_join.sql │ │ │ ├── failure_add_disable_node.sql │ │ │ ├── failure_connection_establishment.sql │ │ │ ├── failure_copy_on_hash.sql │ │ │ ├── failure_copy_to_reference.sql │ │ │ ├── failure_create_database.sql │ │ │ ├── failure_create_distributed_table_concurrently.sql │ │ │ ├── failure_create_distributed_table_non_empty.sql │ │ │ ├── failure_create_index_concurrently.sql │ │ │ ├── failure_create_reference_table.sql │ │ │ ├── failure_create_table.sql │ │ │ ├── failure_cte_subquery.sql │ │ │ ├── failure_ddl.sql │ │ │ ├── failure_distributed_results.sql │ │ │ ├── failure_failover_to_local_execution.sql │ │ │ ├── failure_insert_select_pushdown.sql │ │ │ ├── failure_insert_select_repartition.sql │ │ │ ├── failure_insert_select_via_coordinator.sql │ │ │ ├── failure_multi_dml.sql │ │ │ ├── failure_multi_row_insert.sql │ │ │ ├── failure_multi_shard_update_delete.sql │ │ │ ├── failure_mx_metadata_sync.sql │ │ │ ├── failure_mx_metadata_sync_multi_trans.sql │ │ │ ├── failure_offline_move_shard_placement.sql │ │ │ ├── failure_on_create_subscription.sql │ │ │ ├── failure_online_move_shard_placement.sql │ │ │ ├── failure_parallel_connection.sql │ │ │ ├── failure_ref_tables.sql │ │ │ ├── failure_replicated_partitions.sql │ │ │ ├── failure_savepoints.sql │ │ │ ├── failure_setup.sql │ │ │ ├── failure_single_mod.sql │ │ │ ├── failure_single_select.sql │ │ │ ├── failure_split_cleanup.sql │ │ │ ├── failure_tenant_isolation.sql │ │ │ ├── failure_tenant_isolation_nonblocking.sql │ │ │ ├── failure_test_helpers.sql │ │ │ ├── failure_truncate.sql │ │ │ ├── failure_vacuum.sql │ │ │ ├── fast_path_router_modify.sql │ │ │ ├── fkeys_between_local_ref.sql │ │ │ ├── follower_single_node.sql │ │ │ ├── forcedelegation_functions.sql │ │ │ ├── foreign_key_restriction_enforcement.sql │ │ │ ├── foreign_key_to_reference_shard_rebalance.sql │ │ │ ├── foreign_key_to_reference_table.sql │ │ │ ├── foreign_tables_mx.sql │ │ │ ├── function_create.sql │ │ │ ├── function_propagation.sql │ │ │ ├── function_with_case_when.sql │ │ │ ├── functions.sql │ │ │ ├── generated_identity.sql │ │ │ ├── geqo.sql │ │ │ ├── global_cancel.sql │ │ │ ├── grant_on_database_propagation.sql │ │ │ ├── grant_on_foreign_server_propagation.sql │ │ │ ├── grant_on_function_propagation.sql │ │ │ ├── grant_on_schema_propagation.sql │ │ │ ├── grant_on_sequence_propagation.sql │ │ │ ├── grant_on_table_propagation.sql │ │ │ ├── having_subquery.sql │ │ │ ├── hyperscale_tutorial.sql │ │ │ ├── index_create.sql │ │ │ ├── insert_select_connection_leak.sql │ │ │ ├── insert_select_into_local_table.sql │ │ │ ├── insert_select_repartition.sql │ │ │ ├── insert_select_single_shard_table.sql │ │ │ ├── intermediate_result_pruning.sql │ │ │ ├── intermediate_result_pruning_create.sql │ │ │ ├── intermediate_result_pruning_queries_1.sql │ │ │ ├── intermediate_result_pruning_queries_2.sql │ │ │ ├── intermediate_results.sql │ │ │ ├── issue_5099.sql │ │ │ ├── issue_5248.sql │ │ │ ├── issue_5763.sql │ │ │ ├── issue_6543.sql │ │ │ ├── issue_6592.sql │ │ │ ├── issue_6758.sql │ │ │ ├── issue_7477.sql │ │ │ ├── issue_7891.sql │ │ │ ├── issue_8243.sql │ │ │ ├── join_pushdown.sql │ │ │ ├── limit_intermediate_size.sql │ │ │ ├── local_dist_join.sql │ │ │ ├── local_dist_join_load.sql │ │ │ ├── local_dist_join_mixed.sql │ │ │ ├── local_dist_join_modifications.sql │ │ │ ├── local_execution_local_plan.sql │ │ │ ├── local_shard_copy.sql │ │ │ ├── local_shard_execution.sql │ │ │ ├── local_shard_execution_dropped_column.sql │ │ │ ├── local_shard_execution_replicated.sql │ │ │ ├── local_shard_utility_command_execution.sql │ │ │ ├── local_table_join.sql │ │ │ ├── locally_execute_intermediate_results.sql │ │ │ ├── logical_rep_consistency.sql │ │ │ ├── logical_replication.sql │ │ │ ├── materialized_view.sql │ │ │ ├── merge.sql │ │ │ ├── merge_arbitrary.sql │ │ │ ├── merge_arbitrary_create.sql │ │ │ ├── merge_partition_tables.sql │ │ │ ├── merge_repartition1.sql │ │ │ ├── merge_repartition2.sql │ │ │ ├── merge_schema_sharding.sql │ │ │ ├── merge_unsupported.sql │ │ │ ├── merge_vcore.sql │ │ │ ├── metadata_sync_helpers.sql │ │ │ ├── minimal_cluster_management.sql │ │ │ ├── mixed_relkind_tests.sql │ │ │ ├── modification_correctness.sql │ │ │ ├── multi_add_node_from_backup.sql │ │ │ ├── multi_add_node_from_backup_negative.sql │ │ │ ├── multi_add_node_from_backup_sync_replica.sql │ │ │ ├── multi_agg_approximate_distinct.sql │ │ │ ├── multi_agg_distinct.sql │ │ │ ├── multi_agg_type_conversion.sql │ │ │ ├── multi_alias.sql │ │ │ ├── multi_alter_table_add_constraints.sql │ │ │ ├── multi_alter_table_add_constraints_without_name.sql │ │ │ ├── multi_alter_table_add_foreign_key_without_name.sql │ │ │ ├── multi_alter_table_row_level_security.sql │ │ │ ├── multi_alter_table_row_level_security_escape.sql │ │ │ ├── multi_alter_table_statements.sql │ │ │ ├── multi_array_agg.sql │ │ │ ├── multi_average_expression.sql │ │ │ ├── multi_basic_queries.sql │ │ │ ├── multi_behavioral_analytics_basics.sql │ │ │ ├── multi_behavioral_analytics_create_table.sql │ │ │ ├── multi_behavioral_analytics_create_table_superuser.sql │ │ │ ├── multi_behavioral_analytics_single_shard_queries.sql │ │ │ ├── multi_cache_invalidation.sql │ │ │ ├── multi_citus_tools.sql │ │ │ ├── multi_cluster_management.sql │ │ │ ├── multi_colocated_shard_rebalance.sql │ │ │ ├── multi_colocation_utils.sql │ │ │ ├── multi_complex_count_distinct.sql │ │ │ ├── multi_complex_expressions.sql │ │ │ ├── multi_copy.sql │ │ │ ├── multi_count_type_conversion.sql │ │ │ ├── multi_create_fdw.sql │ │ │ ├── multi_create_role_dependency.sql │ │ │ ├── multi_create_shards.sql │ │ │ ├── multi_create_table.sql │ │ │ ├── multi_create_table_constraints.sql │ │ │ ├── multi_create_table_superuser.sql │ │ │ ├── multi_create_users.sql │ │ │ ├── multi_cross_shard.sql │ │ │ ├── multi_data_types.sql │ │ │ ├── multi_deparse_function.sql │ │ │ ├── multi_deparse_procedure.sql │ │ │ ├── multi_deparse_shard_query.sql │ │ │ ├── multi_distributed_transaction_id.sql │ │ │ ├── multi_distribution_metadata.sql │ │ │ ├── multi_drop_extension.sql │ │ │ ├── multi_dropped_column_aliases.sql │ │ │ ├── multi_explain.sql │ │ │ ├── multi_extension.sql │ │ │ ├── multi_fix_partition_shard_index_names.sql │ │ │ ├── multi_follower_configure_followers.sql │ │ │ ├── multi_follower_dml.sql │ │ │ ├── multi_follower_sanity_check.sql │ │ │ ├── multi_follower_select_statements.sql │ │ │ ├── multi_foreign_key.sql │ │ │ ├── multi_foreign_key_relation_graph.sql │ │ │ ├── multi_function_evaluation.sql │ │ │ ├── multi_function_in_join.sql │ │ │ ├── multi_generate_ddl_commands.sql │ │ │ ├── multi_hash_pruning.sql │ │ │ ├── multi_having_pushdown.sql │ │ │ ├── multi_index_statements.sql │ │ │ ├── multi_insert_select.sql │ │ │ ├── multi_insert_select_conflict.sql │ │ │ ├── multi_insert_select_non_pushable_queries.sql │ │ │ ├── multi_insert_select_window.sql │ │ │ ├── multi_join_order_additional.sql │ │ │ ├── multi_join_order_tpch_repartition.sql │ │ │ ├── multi_join_order_tpch_small.sql │ │ │ ├── multi_join_pruning.sql │ │ │ ├── multi_json_agg.sql │ │ │ ├── multi_json_object_agg.sql │ │ │ ├── multi_jsonb_agg.sql │ │ │ ├── multi_jsonb_object_agg.sql │ │ │ ├── multi_large_shardid.sql │ │ │ ├── multi_level_recursive_queries.sql │ │ │ ├── multi_limit_clause.sql │ │ │ ├── multi_limit_clause_approximate.sql │ │ │ ├── multi_load_data.sql │ │ │ ├── multi_load_data_superuser.sql │ │ │ ├── multi_load_more_data.sql │ │ │ ├── multi_master_protocol.sql │ │ │ ├── multi_metadata_access.sql │ │ │ ├── multi_metadata_attributes.sql │ │ │ ├── multi_metadata_sync.sql │ │ │ ├── multi_metadata_sync_domain.sql │ │ │ ├── multi_modifications.sql │ │ │ ├── multi_modifying_xacts.sql │ │ │ ├── multi_move_mx.sql │ │ │ ├── multi_multiuser.sql │ │ │ ├── multi_multiuser_auth.sql │ │ │ ├── multi_multiuser_basic_queries.sql │ │ │ ├── multi_multiuser_copy.sql │ │ │ ├── multi_multiuser_grant.sql │ │ │ ├── multi_multiuser_load_data.sql │ │ │ ├── multi_multiuser_master_protocol.sql │ │ │ ├── multi_mx_add_coordinator.sql │ │ │ ├── multi_mx_alter_distributed_table.sql │ │ │ ├── multi_mx_call.sql │ │ │ ├── multi_mx_copy_data.sql │ │ │ ├── multi_mx_create_table.sql │ │ │ ├── multi_mx_ddl.sql │ │ │ ├── multi_mx_explain.sql │ │ │ ├── multi_mx_function_call_delegation.sql │ │ │ ├── multi_mx_function_table_reference.sql │ │ │ ├── multi_mx_hide_shard_names.sql │ │ │ ├── multi_mx_insert_select_repartition.sql │ │ │ ├── multi_mx_metadata.sql │ │ │ ├── multi_mx_modifications.sql │ │ │ ├── multi_mx_modifications_to_reference_tables.sql │ │ │ ├── multi_mx_modifying_xacts.sql │ │ │ ├── multi_mx_node_metadata.sql │ │ │ ├── multi_mx_partitioning.sql │ │ │ ├── multi_mx_reference_table.sql │ │ │ ├── multi_mx_repartition_join_w1.sql │ │ │ ├── multi_mx_repartition_join_w2.sql │ │ │ ├── multi_mx_repartition_udt_prepare.sql │ │ │ ├── multi_mx_repartition_udt_w1.sql │ │ │ ├── multi_mx_repartition_udt_w2.sql │ │ │ ├── multi_mx_router_planner.sql │ │ │ ├── multi_mx_schema_support.sql │ │ │ ├── multi_mx_tpch_query1.sql │ │ │ ├── multi_mx_tpch_query10.sql │ │ │ ├── multi_mx_tpch_query12.sql │ │ │ ├── multi_mx_tpch_query14.sql │ │ │ ├── multi_mx_tpch_query19.sql │ │ │ ├── multi_mx_tpch_query3.sql │ │ │ ├── multi_mx_tpch_query6.sql │ │ │ ├── multi_mx_tpch_query7.sql │ │ │ ├── multi_mx_tpch_query7_nested.sql │ │ │ ├── multi_mx_transaction_recovery.sql │ │ │ ├── multi_mx_truncate_from_worker.sql │ │ │ ├── multi_name_lengths.sql │ │ │ ├── multi_name_resolution.sql │ │ │ ├── multi_null_minmax_value_pruning.sql │ │ │ ├── multi_orderby_limit_pushdown.sql │ │ │ ├── multi_outer_join.sql │ │ │ ├── multi_outer_join_columns.sql │ │ │ ├── multi_outer_join_reference.sql │ │ │ ├── multi_partition_pruning.sql │ │ │ ├── multi_partitioning.sql │ │ │ ├── multi_partitioning_utils.sql │ │ │ ├── multi_poolinfo_usage.sql │ │ │ ├── multi_prepare_plsql.sql │ │ │ ├── multi_prepare_sql.sql │ │ │ ├── multi_prune_shard_list.sql │ │ │ ├── multi_query_directory_cleanup.sql │ │ │ ├── multi_read_from_secondaries.sql │ │ │ ├── multi_real_time_transaction.sql │ │ │ ├── multi_reference_table.sql │ │ │ ├── multi_remove_node_reference_table.sql │ │ │ ├── multi_repartition_join_planning.sql │ │ │ ├── multi_repartition_join_pruning.sql │ │ │ ├── multi_repartition_join_ref.sql │ │ │ ├── multi_repartition_join_task_assignment.sql │ │ │ ├── multi_repartition_udt.sql │ │ │ ├── multi_repartitioned_subquery_udf.sql │ │ │ ├── multi_replicate_reference_table.sql │ │ │ ├── multi_rls_join_distribution_key.sql │ │ │ ├── multi_router_planner.sql │ │ │ ├── multi_router_planner_fast_path.sql │ │ │ ├── multi_row_insert.sql │ │ │ ├── multi_row_router_insert.sql │ │ │ ├── multi_schema_support.sql │ │ │ ├── multi_select_distinct.sql │ │ │ ├── multi_select_for_update.sql │ │ │ ├── multi_sequence_default.sql │ │ │ ├── multi_shard_update_delete.sql │ │ │ ├── multi_simple_queries.sql │ │ │ ├── multi_single_relation_subquery.sql │ │ │ ├── multi_size_queries.sql │ │ │ ├── multi_sql_function.sql │ │ │ ├── multi_subquery.sql │ │ │ ├── multi_subquery_behavioral_analytics.sql │ │ │ ├── multi_subquery_complex_queries.sql │ │ │ ├── multi_subquery_complex_reference_clause.sql │ │ │ ├── multi_subquery_in_where_clause.sql │ │ │ ├── multi_subquery_in_where_reference_clause.sql │ │ │ ├── multi_subquery_misc.sql │ │ │ ├── multi_subquery_union.sql │ │ │ ├── multi_subquery_window_functions.sql │ │ │ ├── multi_subtransactions.sql │ │ │ ├── multi_table_ddl.sql │ │ │ ├── multi_task_assignment_policy.sql │ │ │ ├── multi_task_string_size.sql │ │ │ ├── multi_tenant_isolation.sql │ │ │ ├── multi_tenant_isolation_nonblocking.sql │ │ │ ├── multi_test_catalog_views.sql │ │ │ ├── multi_test_helpers.sql │ │ │ ├── multi_test_helpers_superuser.sql │ │ │ ├── multi_tpch_query1.sql │ │ │ ├── multi_tpch_query10.sql │ │ │ ├── multi_tpch_query12.sql │ │ │ ├── multi_tpch_query14.sql │ │ │ ├── multi_tpch_query19.sql │ │ │ ├── multi_tpch_query3.sql │ │ │ ├── multi_tpch_query6.sql │ │ │ ├── multi_tpch_query7.sql │ │ │ ├── multi_tpch_query7_nested.sql │ │ │ ├── multi_transaction_recovery.sql │ │ │ ├── multi_transaction_recovery_multiple_databases.sql │ │ │ ├── multi_transactional_drop_shards.sql │ │ │ ├── multi_truncate.sql │ │ │ ├── multi_unsupported_worker_operations.sql │ │ │ ├── multi_update_select.sql │ │ │ ├── multi_upsert.sql │ │ │ ├── multi_utilities.sql │ │ │ ├── multi_utility_statements.sql │ │ │ ├── multi_utility_warnings.sql │ │ │ ├── multi_view.sql │ │ │ ├── multi_working_columns.sql │ │ │ ├── mx_coordinator_shouldhaveshards.sql │ │ │ ├── mx_foreign_key_to_reference_table.sql │ │ │ ├── mx_regular_user.sql │ │ │ ├── nested_execution.sql │ │ │ ├── nested_execution_create.sql │ │ │ ├── node_conninfo_reload.sql │ │ │ ├── non_colocated_leaf_subquery_joins.sql │ │ │ ├── non_colocated_subquery_joins.sql │ │ │ ├── non_super_user_cdc_library_path.sql │ │ │ ├── non_super_user_object_metadata.sql │ │ │ ├── null_parameters.sql │ │ │ ├── object_propagation_debug.sql │ │ │ ├── partition_wise_join.sql │ │ │ ├── partitioned_indexes_create.sql │ │ │ ├── partitioned_intermediate_results.sql │ │ │ ├── partitioning_issue_3970.sql │ │ │ ├── pg12.sql │ │ │ ├── pg13.sql │ │ │ ├── pg13_propagate_statistics.sql │ │ │ ├── pg13_with_ties.sql │ │ │ ├── pg14.sql │ │ │ ├── pg15.sql │ │ │ ├── pg15_jsonpath.sql │ │ │ ├── pg16.sql │ │ │ ├── pg17.sql │ │ │ ├── pg17_json.sql │ │ │ ├── pg18.sql │ │ │ ├── pg_dump.sql │ │ │ ├── pgmerge.sql │ │ │ ├── postgres.sql │ │ │ ├── prepared_statements_1.sql │ │ │ ├── prepared_statements_2.sql │ │ │ ├── prepared_statements_3.sql │ │ │ ├── prepared_statements_4.sql │ │ │ ├── prepared_statements_create_load.sql │ │ │ ├── propagate_extension_commands.sql │ │ │ ├── propagate_foreign_servers.sql │ │ │ ├── propagate_set_commands.sql │ │ │ ├── propagate_statistics.sql │ │ │ ├── publication.sql │ │ │ ├── query_single_shard_table.sql │ │ │ ├── reassign_owned.sql │ │ │ ├── recurring_join_pushdown.sql │ │ │ ├── recurring_outer_join.sql │ │ │ ├── recursive_dml_queries_mx.sql │ │ │ ├── recursive_dml_with_different_planners_executors.sql │ │ │ ├── recursive_relation_planning_restriction_pushdown.sql │ │ │ ├── recursive_view_local_table.sql │ │ │ ├── ref_citus_local_fkeys.sql │ │ │ ├── relation_access_tracking.sql │ │ │ ├── relation_access_tracking_single_node.sql │ │ │ ├── remove_coordinator.sql │ │ │ ├── remove_coordinator_from_metadata.sql │ │ │ ├── remove_non_default_nodes.sql │ │ │ ├── rename_public_to_citus_schema_and_recreate.sql │ │ │ ├── replicate_reference_tables_to_coordinator.sql │ │ │ ├── replicated_partitioned_table.sql │ │ │ ├── replicated_table_disable_node.sql │ │ │ ├── resync_metadata_with_sequences.sql │ │ │ ├── role_command_from_any_node.sql │ │ │ ├── row_types.sql │ │ │ ├── run_command_on_all_nodes.sql │ │ │ ├── schema_based_sharding.sql │ │ │ ├── schemas.sql │ │ │ ├── schemas_create.sql │ │ │ ├── seclabel.sql │ │ │ ├── sequences.sql │ │ │ ├── sequences_create.sql │ │ │ ├── sequences_owned_by.sql │ │ │ ├── sequences_with_different_types.sql │ │ │ ├── sequential_modifications.sql │ │ │ ├── set_operation_and_local_tables.sql │ │ │ ├── set_operations.sql │ │ │ ├── set_role_in_transaction.sql │ │ │ ├── shard_move_constraints.sql │ │ │ ├── shard_move_constraints_blocking.sql │ │ │ ├── shard_move_deferred_delete.sql │ │ │ ├── shard_rebalancer.sql │ │ │ ├── shard_rebalancer_unit.sql │ │ │ ├── shared_connection_stats.sql │ │ │ ├── single_hash_repartition_join.sql │ │ │ ├── single_node.sql │ │ │ ├── single_node_enterprise.sql │ │ │ ├── single_node_truncate.sql │ │ │ ├── single_shard_table_prep.sql │ │ │ ├── single_shard_table_udfs.sql │ │ │ ├── split_shard.sql │ │ │ ├── sql_procedure.sql │ │ │ ├── sqlancer_failures.sql │ │ │ ├── sqlsmith_failures.sql │ │ │ ├── ssl_by_default.sql │ │ │ ├── start_stop_metadata_sync.sql │ │ │ ├── stat_counters.sql │ │ │ ├── stat_statements.sql │ │ │ ├── statement_cancel_error_message.sql │ │ │ ├── subqueries_deep.sql │ │ │ ├── subqueries_not_supported.sql │ │ │ ├── subquery_and_cte.sql │ │ │ ├── subquery_append.sql │ │ │ ├── subquery_basics.sql │ │ │ ├── subquery_complex_target_list.sql │ │ │ ├── subquery_executors.sql │ │ │ ├── subquery_in_targetlist.sql │ │ │ ├── subquery_in_where.sql │ │ │ ├── subquery_local_tables.sql │ │ │ ├── subquery_partitioning.sql │ │ │ ├── subquery_prepared_statements.sql │ │ │ ├── subquery_view.sql │ │ │ ├── subscripting_op.sql │ │ │ ├── system_queries.sql │ │ │ ├── tableam.sql │ │ │ ├── tablespace.sql │ │ │ ├── tdigest_aggregate_support.sql │ │ │ ├── text_search.sql │ │ │ ├── undistribute_table.sql │ │ │ ├── undistribute_table_cascade.sql │ │ │ ├── undistribute_table_cascade_mx.sql │ │ │ ├── union_pushdown.sql │ │ │ ├── unsupported_lateral_subqueries.sql │ │ │ ├── update_colocation_mx.sql │ │ │ ├── upgrade_autoconverted_after.sql │ │ │ ├── upgrade_autoconverted_before.sql │ │ │ ├── upgrade_basic_after.sql │ │ │ ├── upgrade_basic_after_non_mixed.sql │ │ │ ├── upgrade_basic_before.sql │ │ │ ├── upgrade_basic_before_non_mixed.sql │ │ │ ├── upgrade_citus_finish_citus_upgrade.sql │ │ │ ├── upgrade_citus_locks.sql │ │ │ ├── upgrade_citus_stat_activity.sql │ │ │ ├── upgrade_columnar_after.sql │ │ │ ├── upgrade_columnar_before.sql │ │ │ ├── upgrade_distributed_function_after.sql │ │ │ ├── upgrade_distributed_function_before.sql │ │ │ ├── upgrade_distributed_triggers_after.sql │ │ │ ├── upgrade_distributed_triggers_before.sql │ │ │ ├── upgrade_list_citus_objects.sql │ │ │ ├── upgrade_pg_dist_cleanup_after.sql │ │ │ ├── upgrade_pg_dist_cleanup_before.sql │ │ │ ├── upgrade_post_11_after.sql │ │ │ ├── upgrade_post_11_before.sql │ │ │ ├── upgrade_post_14_after.sql │ │ │ ├── upgrade_post_14_before.sql │ │ │ ├── upgrade_rebalance_strategy_after.sql │ │ │ ├── upgrade_rebalance_strategy_before.sql │ │ │ ├── upgrade_ref2ref_after.sql │ │ │ ├── upgrade_ref2ref_before.sql │ │ │ ├── upgrade_schema_based_sharding_after.sql │ │ │ ├── upgrade_schema_based_sharding_before.sql │ │ │ ├── upgrade_single_shard_table_after.sql │ │ │ ├── upgrade_single_shard_table_before.sql │ │ │ ├── upgrade_type_after.sql │ │ │ ├── upgrade_type_before.sql │ │ │ ├── validate_constraint.sql │ │ │ ├── values.sql │ │ │ ├── view_propagation.sql │ │ │ ├── views.sql │ │ │ ├── views_create.sql │ │ │ ├── window_functions.sql │ │ │ ├── with_basics.sql │ │ │ ├── with_dml.sql │ │ │ ├── with_executors.sql │ │ │ ├── with_join.sql │ │ │ ├── with_modifying.sql │ │ │ ├── with_nested.sql │ │ │ ├── with_partitioning.sql │ │ │ ├── with_prepare.sql │ │ │ ├── with_set_operations.sql │ │ │ ├── with_transactions.sql │ │ │ ├── with_where.sql │ │ │ ├── worker_copy_table_to_node.sql │ │ │ ├── worker_split_binary_copy_test.sql │ │ │ ├── worker_split_copy_test.sql │ │ │ └── worker_split_text_copy_test.sql │ │ ├── sql_base_schedule │ │ └── sql_schedule │ └── tap/ │ ├── Makefile │ ├── citus_helpers.pm │ ├── postgresql.conf │ └── t/ │ └── restore_point_mx.pl └── vendor/ ├── README.md └── safestringlib/ ├── .gitattributes ├── .gitignore ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── LICENSE ├── LICENSE©ING.txt ├── README.md ├── include/ │ ├── safe_lib.h │ ├── safe_lib_errno.h │ ├── safe_lib_errno.h.in │ ├── safe_mem_lib.h │ ├── safe_str_lib.h │ ├── safe_types.h │ ├── safe_types.h.in │ └── snprintf_s.h ├── makefile └── safeclib/ ├── abort_handler_s.c ├── ignore_handler_s.c ├── mem_primitives_lib.c ├── mem_primitives_lib.h ├── memcmp16_s.c ├── memcmp32_s.c ├── memcmp_s.c ├── memcpy16_s.c ├── memcpy32_s.c ├── memcpy_s.c ├── memmove16_s.c ├── memmove32_s.c ├── memmove_s.c ├── memset16_s.c ├── memset32_s.c ├── memset_s.c ├── memzero16_s.c ├── memzero32_s.c ├── memzero_s.c ├── safe_mem_constraint.c ├── safe_mem_constraint.h ├── safe_str_constraint.c ├── safe_str_constraint.h ├── safeclib_private.h ├── snprintf_support.c ├── stpcpy_s.c ├── stpncpy_s.c ├── strcasecmp_s.c ├── strcasestr_s.c ├── strcat_s.c ├── strcmp_s.c ├── strcmpfld_s.c ├── strcpy_s.c ├── strcpyfld_s.c ├── strcpyfldin_s.c ├── strcpyfldout_s.c ├── strcspn_s.c ├── strfirstchar_s.c ├── strfirstdiff_s.c ├── strfirstsame_s.c ├── strisalphanumeric_s.c ├── strisascii_s.c ├── strisdigit_s.c ├── strishex_s.c ├── strislowercase_s.c ├── strismixedcase_s.c ├── strispassword_s.c ├── strisuppercase_s.c ├── strlastchar_s.c ├── strlastdiff_s.c ├── strlastsame_s.c ├── strljustify_s.c ├── strncat_s.c ├── strncpy_s.c ├── strnlen_s.c ├── strnterminate_s.c ├── strpbrk_s.c ├── strprefix_s.c ├── strremovews_s.c ├── strspn_s.c ├── strstr_s.c ├── strtok_s.c ├── strtolowercase_s.c ├── strtouppercase_s.c ├── strzero_s.c ├── wcpcpy_s.c ├── wcscat_s.c ├── wcscpy_s.c ├── wcsncat_s.c ├── wcsncpy_s.c ├── wcsnlen_s.c ├── wmemcmp_s.c ├── wmemcpy_s.c ├── wmemmove_s.c └── wmemset_s.c ================================================ FILE CONTENTS ================================================ ================================================ FILE: .codeclimate.yml ================================================ exclude_patterns: - "src/backend/distributed/utils/citus_outfuncs.c" - "src/backend/distributed/deparser/ruleutils_*.c" - "src/include/distributed/citus_nodes.h" - "src/backend/distributed/safeclib" - "src/backend/columnar/safeclib" - "**/vendor/" ================================================ FILE: .codecov.yml ================================================ codecov: notify: require_ci_to_pass: yes coverage: precision: 2 round: down range: "70...100" ignore: - "src/backend/distributed/utils/citus_outfuncs.c" - "src/backend/distributed/deparser/ruleutils_*.c" - "src/include/distributed/citus_nodes.h" - "src/backend/distributed/safeclib" - "vendor" status: project: default: target: 87.5 threshold: 0.5 patch: default: target: 75 changes: no parsers: gcov: branch_detection: conditional: yes loop: yes method: no macro: no comment: layout: "header, diff" behavior: default require_changes: no ================================================ FILE: .devcontainer/.gdbinit ================================================ # gdbpg.py contains scripts to nicely print the postgres datastructures # while in a gdb session. Since the vscode debugger is based on gdb this # actually also works when debugging with vscode. Providing nice tools # to understand the internal datastructures we are working with. source /root/gdbpg.py # when debugging postgres it is convenient to _always_ have a breakpoint # trigger when an error is logged. Because .gdbinit is sourced before gdb # is fully attached and has the sources loaded. To make sure the breakpoint # is added when the library is loaded we temporary set the breakpoint pending # to on. After we have added out breakpoint we revert back to the default # configuration for breakpoint pending. # The breakpoint is hard to read, but at entry of the function we don't have # the level loaded in elevel. Instead we hardcode the location where the # level of the current error is stored. Also gdb doesn't understand the # ERROR symbol so we hardcode this to the value of ERROR. It is very unlikely # this value will ever change in postgres, but if it does we might need to # find a way to conditionally load the correct breakpoint. set breakpoint pending on break elog.c:errfinish if errordata[errordata_stack_depth].elevel == 21 set breakpoint pending auto echo \n echo ----------------------------------------------------------------------------------\n echo when attaching to a postgres backend a breakpoint will be set on elog.c:errfinish \n echo it will only break on errors being raised in postgres \n echo \n echo to disable this breakpoint from vscode run `-exec disable 1` in the debug console \n echo this assumes it's the first breakpoint loaded as it is loaded from .gdbinit \n echo this can be verified with `-exec info break`, enabling can be done with \n echo `-exec enable 1` \n echo ----------------------------------------------------------------------------------\n echo \n ================================================ FILE: .devcontainer/.gitignore ================================================ postgresql-*.tar.bz2 ================================================ FILE: .devcontainer/.psqlrc ================================================ \timing on \pset linestyle unicode \pset border 2 \setenv PAGER 'pspg --no-mouse -bX --no-commandbar --no-topbar' \set HISTSIZE 100000 \set PROMPT1 '\n%[%033[1m%]%M %n@%/:%> (PID: %p)%R%[%033[0m%]%# ' \set PROMPT2 ' ' ================================================ FILE: .devcontainer/.vscode/Pipfile ================================================ [[source]] url = "https://pypi.org/simple" verify_ssl = true name = "pypi" [packages] docopt = "*" [dev-packages] [requires] python_version = "3.12" ================================================ FILE: .devcontainer/.vscode/generate_c_cpp_properties-json.py ================================================ #! /usr/bin/env pipenv-shebang """Generate C/C++ properties file for VSCode. Uses pgenv to iterate postgres versions and generate a C/C++ properties file for VSCode containing the include paths for the postgres headers. Usage: generate_c_cpp_properties-json.py generate_c_cpp_properties-json.py (-h | --help) generate_c_cpp_properties-json.py --version Options: -h --help Show this screen. --version Show version. """ import json import subprocess from docopt import docopt def main(args): target_path = args[''] output = subprocess.check_output(['pgenv', 'versions']) # typical output is: # 14.8 pgsql-14.8 # * 15.3 pgsql-15.3 # 16beta2 pgsql-16beta2 # where the line marked with a * is the currently active version # # we are only interested in the first word of each line, which is the version number # thus we strip the whitespace and the * from the line and split it into words # and take the first word versions = [line.strip('* ').split()[0] for line in output.decode('utf-8').splitlines()] # create the list of configurations per version configurations = [] for version in versions: configurations.append(generate_configuration(version)) # create the json file c_cpp_properties = { "configurations": configurations, "version": 4 } # write the c_cpp_properties.json file with open(target_path, 'w') as f: json.dump(c_cpp_properties, f, indent=4) def generate_configuration(version): """Returns a configuration for the given postgres version. >>> generate_configuration('14.8') { "name": "Citus Development Configuration - Postgres 14.8", "includePath": [ "/usr/local/include", "/home/citus/.pgenv/src/postgresql-14.8/src/**", "${workspaceFolder}/**", "${workspaceFolder}/src/include/", ], "configurationProvider": "ms-vscode.makefile-tools" } """ return { "name": f"Citus Development Configuration - Postgres {version}", "includePath": [ "/usr/local/include", f"/home/citus/.pgenv/src/postgresql-{version}/src/**", "${workspaceFolder}/**", "${workspaceFolder}/src/include/", ], "configurationProvider": "ms-vscode.makefile-tools" } if __name__ == '__main__': arguments = docopt(__doc__, version='0.1.0') main(arguments) ================================================ FILE: .devcontainer/.vscode/launch.json ================================================ { "version": "0.2.0", "configurations": [ { "name": "Attach Citus (devcontainer)", "type": "cppdbg", "request": "attach", "processId": "${command:pickProcess}", "program": "/home/citus/.pgenv/pgsql/bin/postgres", "additionalSOLibSearchPath": "/home/citus/.pgenv/pgsql/lib", "setupCommands": [ { "text": "handle SIGUSR1 noprint nostop pass", "description": "let gdb not stop when SIGUSR1 is sent to process", "ignoreFailures": true } ], }, { "name": "Open core file", "type": "cppdbg", "request": "launch", "program": "/home/citus/.pgenv/pgsql/bin/postgres", "coreDumpPath": "${input:corefile}", "cwd": "${workspaceFolder}", "MIMode": "gdb", } ], "inputs": [ { "id": "corefile", "type": "command", "command": "extension.commandvariable.file.pickFile", "args": { "dialogTitle": "Select core file", "include": "**/core*", }, }, ], } ================================================ FILE: .devcontainer/Dockerfile ================================================ FROM ubuntu:22.04 AS base # environment is to make python pass an interactive shell, probably not the best timezone given a wide variety of colleagues ENV TZ=UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # install build tools RUN apt update && apt install -y \ bison \ bzip2 \ cpanminus \ curl \ docbook-xml \ docbook-xsl \ flex \ gcc \ git \ libcurl4-gnutls-dev \ libicu-dev \ libkrb5-dev \ liblz4-dev \ libpam0g-dev \ libreadline-dev \ libselinux1-dev \ libssl-dev \ libxml2-utils \ libxslt-dev \ libzstd-dev \ locales \ make \ perl \ pkg-config \ python3 \ python3-pip \ software-properties-common \ sudo \ uuid-dev \ valgrind \ xsltproc \ zlib1g-dev \ && add-apt-repository ppa:deadsnakes/ppa -y \ && apt install -y \ python3.12 python3.12-venv python3-distutils \ # software properties pulls in pkexec, which makes the debugger unusable in vscode && apt purge -y \ software-properties-common \ && apt autoremove -y \ && apt clean RUN sudo pip3 install pipenv pipenv-shebang RUN cpanm install IPC::Run RUN locale-gen en_US.UTF-8 # add the citus user to sudoers and allow all sudoers to login without a password prompt RUN useradd -ms /bin/bash citus \ && usermod -aG sudo citus \ && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers WORKDIR /home/citus USER citus # run all make commands with the number of cores available RUN echo "export MAKEFLAGS=\"-j \$(nproc)\"" >> "/home/citus/.bashrc" RUN git clone --branch v1.3.2 --depth 1 https://github.com/theory/pgenv.git .pgenv COPY --chown=citus:citus pgenv/config/ .pgenv/config/ ENV PATH="/home/citus/.pgenv/bin:${PATH}" ENV PATH="/home/citus/.pgenv/pgsql/bin:${PATH}" USER citus # build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions FROM base AS pg16 RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.11 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install # create a staging directory with all files we want to copy from our pgenv build # we will copy the contents of the staged folder into the final image at once RUN mkdir .pgenv-staging/ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg17 RUN MAKEFLAGS="-j $(nproc)" pgenv build 17.7 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install # create a staging directory with all files we want to copy from our pgenv build # we will copy the contents of the staged folder into the final image at once RUN mkdir .pgenv-staging/ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg18 RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.3 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install # Stage the pgenv artifacts for PG18 RUN mkdir .pgenv-staging/ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS uncrustify-builder RUN sudo apt update && sudo apt install -y cmake tree WORKDIR /uncrustify RUN curl -L https://github.com/uncrustify/uncrustify/archive/uncrustify-0.82.0.tar.gz | tar xz WORKDIR /uncrustify/uncrustify-uncrustify-0.82.0/ RUN mkdir build WORKDIR /uncrustify/uncrustify-uncrustify-0.82.0/build/ RUN cmake .. RUN MAKEFLAGS="-j $(nproc)" make -s RUN make install DESTDIR=/uncrustify # builder for all pipenv's to get them contained in a single layer FROM base AS pipenv WORKDIR /workspaces/citus/ # tools to sync pgenv with vscode COPY --chown=citus:citus .vscode/Pipfile .vscode/Pipfile.lock .devcontainer/.vscode/ RUN ( cd .devcontainer/.vscode && pipenv install ) # environment to run our failure tests COPY --chown=citus:citus src/ src/ RUN ( cd src/test/regress && pipenv install ) # assemble the final container by copying over the artifacts from separately build containers FROM base AS devcontainer LABEL org.opencontainers.image.source=https://github.com/citusdata/citus LABEL org.opencontainers.image.description="Development container for the Citus project" LABEL org.opencontainers.image.licenses=AGPL-3.0-only RUN yes | sudo unminimize # install developer productivity tools RUN sudo apt update \ && sudo apt install -y \ autoconf2.69 \ bash-completion \ fswatch \ gdb \ htop \ libdbd-pg-perl \ libdbi-perl \ lsof \ man \ net-tools \ psmisc \ pspg \ tree \ vim \ && sudo apt clean # Since gdb will run in the context of the root user when debugging citus we will need to both # download the gdbpg.py script as the root user, into their home directory, as well as add .gdbinit # as a file owned by root # This will make that as soon as the debugger attaches to a postgres backend (or frankly any other process) # the gdbpg.py script will be sourced and the developer can direcly use it. RUN sudo curl -o /root/gdbpg.py https://raw.githubusercontent.com/tvesely/gdbpg/6065eee7872457785f830925eac665aa535caf62/gdbpg.py COPY --chown=root:root .gdbinit /root/ # install developer dependencies in the global environment RUN --mount=type=bind,source=requirements.txt,target=requirements.txt pip install -r requirements.txt # for persistent bash history across devcontainers we need to have # a) a directory to store the history in # b) a prompt command to append the history to the file # c) specify the history file to store the history in # b and c are done in the .bashrc to make it persistent across shells only RUN sudo install -d -o citus -g citus /commandhistory \ && echo "export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" >> "/home/citus/.bashrc" # install citus-dev RUN git clone --branch develop https://github.com/citusdata/tools.git citus-tools \ && ( cd citus-tools/citus_dev && pipenv install ) \ && mkdir -p ~/.local/bin \ && ln -s /home/citus/citus-tools/citus_dev/citus_dev-pipenv .local/bin/citus_dev \ && sudo make -C citus-tools/uncrustify install bindir=/usr/local/bin pkgsysconfdir=/usr/local/etc/ \ && mkdir -p ~/.local/share/bash-completion/completions/ \ && ln -s ~/citus-tools/citus_dev/bash_completion ~/.local/share/bash-completion/completions/citus_dev # TODO some LC_ALL errors, possibly solved by locale-gen RUN git clone https://github.com/so-fancy/diff-so-fancy.git \ && mkdir -p ~/.local/bin \ && ln -s /home/citus/diff-so-fancy/diff-so-fancy .local/bin/ COPY --link --from=uncrustify-builder /uncrustify/usr/ /usr/ COPY --link --from=pg16 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ COPY --link --from=pg17 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ COPY --link --from=pg18 /home/citus/.pgenv-staging/ /home/citus/.pgenv/ COPY --link --from=pipenv /home/citus/.local/share/virtualenvs/ /home/citus/.local/share/virtualenvs/ # place to run your cluster with citus_dev VOLUME /data RUN sudo mkdir /data \ && sudo chown citus:citus /data COPY --chown=citus:citus .psqlrc . # with the copy linking of layers github actions seem to misbehave with the ownership of the # directories leading upto the link, hence a small patch layer to have to right ownerships set RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version RUN pgenv switch 18.3 # make connecting to the coordinator easy ENV PGPORT=9700 ================================================ FILE: .devcontainer/Makefile ================================================ init: ../.vscode/c_cpp_properties.json ../.vscode/launch.json ../.vscode: mkdir -p ../.vscode ../.vscode/launch.json: ../.vscode .vscode/launch.json cp .vscode/launch.json ../.vscode/launch.json ../.vscode/c_cpp_properties.json: ../.vscode ./.vscode/generate_c_cpp_properties-json.py ../.vscode/c_cpp_properties.json ================================================ FILE: .devcontainer/devcontainer.json ================================================ { "image": "ghcr.io/citusdata/citus-devcontainer:main", "runArgs": [ "--cap-add=SYS_PTRACE", "--cap-add=SYS_NICE", // allow NUMA page inquiry "--security-opt=seccomp=unconfined", // unblocks move_pages() in the container "--ulimit=core=-1", ], "forwardPorts": [ 9700 ], "customizations": { "vscode": { "extensions": [ "eamodio.gitlens", "GitHub.copilot-chat", "GitHub.copilot", "github.vscode-github-actions", "github.vscode-pull-request-github", "ms-vscode.cpptools-extension-pack", "ms-vsliveshare.vsliveshare", "rioj7.command-variable", ], "settings": { "files.exclude": { "**/*.o": true, "**/.deps/": true, } }, } }, "mounts": [ "type=volume,target=/data", "source=citus-bashhistory,target=/commandhistory,type=volume", ], "updateContentCommand": "./configure", "postCreateCommand": "make -C .devcontainer/", } ================================================ FILE: .devcontainer/pgenv/config/default.conf ================================================ PGENV_MAKE_OPTIONS=(-s) PGENV_CONFIGURE_OPTIONS=( --enable-debug --enable-depend --enable-cassert --enable-tap-tests 'CFLAGS=-ggdb -Og -g3 -fno-omit-frame-pointer -DUSE_VALGRIND' --with-openssl --with-libxml --with-libxslt --with-uuid=e2fs --with-icu --with-lz4 ) ================================================ FILE: .devcontainer/requirements.txt ================================================ black==24.3.0 click==8.1.7 isort==5.12.0 mypy-extensions==1.0.0 packaging==23.2 pathspec==0.11.2 platformdirs==4.0.0 tomli==2.0.1 typing_extensions==4.8.0 ================================================ FILE: .devcontainer/src/test/regress/Pipfile ================================================ [[source]] name = "pypi" url = "https://pypi.python.org/simple" verify_ssl = true [packages] mitmproxy = {git = "https://github.com/citusdata/mitmproxy.git", ref = "main"} "aioquic" = ">=1.2.0,<1.3.0" "mitmproxy-rs" = ">=0.12.6,<0.13.0" argon2-cffi = ">=23.1.0" bcrypt = ">=4.1.2" brotli = "<=1.2.0" h11 = "==0.16.0" h2 = "==4.3.0" tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" pytest-asyncio = "*" pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" [dev-packages] black = "==24.10.0" isort = "*" flake8 = "*" flake8-bugbear = "*" [requires] python_version = "3.12" ================================================ FILE: .editorconfig ================================================ # top-most EditorConfig file root = true # rules for all files # we use tabs with indent size 4 [*] indent_style = tab indent_size = 4 tab_width = 4 end_of_line = lf insert_final_newline = true charset = utf-8 trim_trailing_whitespace = true # Don't change test output files, pngs or test data files [*.{out,png,data}] insert_final_newline = unset trim_trailing_whitespace = unset [*.{sql,sh,py,toml}] indent_style = space indent_size = 4 tab_width = 4 [*.yml] indent_style = space indent_size = 2 tab_width = 2 ================================================ FILE: .flake8 ================================================ [flake8] # E203 is ignored for black extend-ignore = E203 # black will truncate to 88 characters usually, but long string literals it # might keep. That's fine in most cases unless it gets really excessive. max-line-length = 150 exclude = .git,__pycache__,vendor,tmp_* ================================================ FILE: .gitattributes ================================================ * whitespace=space-before-tab,trailing-space *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4 *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent *.patch -whitespace *.pl whitespace=space-before-tab,trailing-space,tabwidth=4 *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent # Avoid confusing ASCII underlines with leftover merge conflict markers README conflict-marker-size=32 README.* conflict-marker-size=32 # Certain data files that contain special whitespace, and other special cases *.data -whitespace # Test output files that contain extra whitespace *.out -whitespace # These files are maintained or generated elsewhere. We take them as is. configure -whitespace # all C files (implementation and header) use our style... *.[ch] citus-style # except these exceptions... src/backend/distributed/utils/citus_outfuncs.c -citus-style src/backend/distributed/deparser/ruleutils_16.c -citus-style src/backend/distributed/deparser/ruleutils_17.c -citus-style src/backend/distributed/deparser/ruleutils_18.c -citus-style src/backend/distributed/commands/index_pg_source.c -citus-style src/include/distributed/citus_nodes.h -citus-style /vendor/** -citus-style # Hide diff on github by default for copied udfs src/backend/distributed/sql/udfs/*/[123456789]*.sql linguist-generated=true ================================================ FILE: .github/actions/parallelization/action.yml ================================================ name: 'Parallelization matrix' inputs: count: required: false default: 32 outputs: json: value: ${{ steps.generate_matrix.outputs.json }} runs: using: "composite" steps: - name: Generate parallelization matrix id: generate_matrix shell: bash run: |- json_array="{\"include\": [" for ((i = 1; i <= ${{ inputs.count }}; i++)); do json_array+="{\"id\":\"$i\"}," done json_array=${json_array%,} json_array+=" ]}" echo "json=$json_array" >> "$GITHUB_OUTPUT" echo "json=$json_array" ================================================ FILE: .github/actions/save_logs_and_results/action.yml ================================================ name: save_logs_and_results inputs: folder: required: false default: "log" runs: using: composite steps: - uses: actions/upload-artifact@v4.6.0 name: Upload logs with: name: ${{ inputs.folder }} if-no-files-found: ignore path: | src/test/**/proxy.output src/test/**/results/ src/test/**/tmp_check/master/log src/test/**/tmp_check/worker.57638/log src/test/**/tmp_check/worker.57637/log src/test/**/*.diffs src/test/**/out/ddls.sql src/test/**/out/queries.sql src/test/**/logfile_* /tmp/pg_upgrade_newData_logs - name: Publish regression.diffs run: |- diffs="$(find src/test/regress -name "*.diffs" -exec cat {} \;)" if ! [ -z "$diffs" ]; then echo '```diff' >> $GITHUB_STEP_SUMMARY echo -E "$diffs" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo -E $diffs fi shell: bash - name: Print stack traces run: "./ci/print_stack_trace.sh" if: failure() shell: bash ================================================ FILE: .github/actions/setup_extension/action.yml ================================================ name: setup_extension inputs: pg_major: required: false skip_installation: required: false default: false type: boolean runs: using: composite steps: - name: Expose $PG_MAJOR to Github Env run: |- if [ -z "${{ inputs.pg_major }}" ]; then echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV else echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV fi shell: bash - uses: actions/download-artifact@v4.1.8 with: name: build-${{ env.PG_MAJOR }} - name: Install Extension if: ${{ inputs.skip_installation == 'false' }} run: tar xfv "install-$PG_MAJOR.tar" --directory / shell: bash - name: Configure run: |- chown -R circleci . git config --global --add safe.directory ${GITHUB_WORKSPACE} gosu circleci ./configure --without-pg-version-check shell: bash - name: Enable core dumps run: ulimit -c unlimited shell: bash ================================================ FILE: .github/actions/upload_coverage/action.yml ================================================ name: coverage inputs: flags: required: false codecov_token: required: true runs: using: composite steps: - uses: codecov/codecov-action@v3 with: flags: ${{ inputs.flags }} token: ${{ inputs.codecov_token }} verbose: true gcov: true ================================================ FILE: .github/packaging/packaging_ignore.yml ================================================ base: - ".* warning: ignoring old recipe for target [`']check'" - ".* warning: overriding recipe for target [`']check'" ================================================ FILE: .github/packaging/validate_build_output.sh ================================================ #!/bin/bash set -ex # Function to get the OS version get_rpm_os_version() { if [[ -f /etc/centos-release ]]; then cat /etc/centos-release | awk '{print $4}' elif [[ -f /etc/oracle-release ]]; then cat /etc/oracle-release | awk '{print $5}' else echo "Unknown" fi } package_type=${1} # Since $HOME is set in GH_Actions as /github/home, pyenv fails to create virtualenvs. # For this script, we set $HOME to /root and then set it back to /github/home. GITHUB_HOME="${HOME}" export HOME="/root" eval "$(pyenv init -)" pyenv versions pyenv virtualenv ${PACKAGING_PYTHON_VERSION} packaging_env pyenv activate packaging_env git clone -b v0.8.27 --depth=1 https://github.com/citusdata/tools.git tools python3 -m pip install -r tools/packaging_automation/requirements.txt echo "Package type: ${package_type}" echo "OS version: $(get_rpm_os_version)" # For RHEL 7, we need to install urllib3<2 due to below execution error # ImportError: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' # module is compiled with 'OpenSSL 1.0.2k-fips 26 Jan 2017'. # See: https://github.com/urllib3/urllib3/issues/2168 if [[ ${package_type} == "rpm" && $(get_rpm_os_version) == 7* ]]; then python3 -m pip uninstall -y urllib3 python3 -m pip install 'urllib3<2' fi python3 -m tools.packaging_automation.validate_build_output --output_file output.log \ --ignore_file .github/packaging/packaging_ignore.yml \ --package_type ${package_type} pyenv deactivate # Set $HOME back to /github/home export HOME=${GITHUB_HOME} # Print the output to the console ================================================ FILE: .github/pull_request_template.md ================================================ DESCRIPTION: PR description that will go into the change log, up to 78 characters ================================================ FILE: .github/workflows/build_and_test.yml ================================================ name: Build & Test run-name: Build & Test - ${{ github.event.pull_request.title || github.ref_name }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true on: workflow_dispatch: inputs: skip_test_flakyness: required: false default: false type: boolean push: branches: - "main" - "release-*" pull_request: types: [opened, reopened,synchronize] merge_group: jobs: # Since GHA does not interpolate env varibles in matrix context, we need to # define them in a separate job and use them in other jobs. params: runs-on: ubuntu-latest name: Initialize parameters outputs: build_image_name: "ghcr.io/citusdata/extbuilder" test_image_name: "ghcr.io/citusdata/exttester" citusupgrade_image_name: "ghcr.io/citusdata/citusupgradetester" fail_test_image_name: "ghcr.io/citusdata/failtester" pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.33" sql_snapshot_pg_version: "18.3" image_suffix: "-vac4338a" pg16_version: '{ "major": "16", "full": "16.13" }' pg17_version: '{ "major": "17", "full": "17.9" }' pg18_version: '{ "major": "18", "full": "18.3" }' upgrade_pg_versions: "16.13-17.9-18.3" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters run: echo 'noop' check-sql-snapshots: needs: params runs-on: ubuntu-latest container: image: ${{ needs.params.outputs.build_image_name }}:${{ needs.params.outputs.sql_snapshot_pg_version }}${{ needs.params.outputs.image_suffix }} options: --user root steps: - uses: actions/checkout@v4 - name: Check Snapshots run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} ci/check_sql_snapshots.sh check-style: needs: params runs-on: ubuntu-latest container: image: ${{ needs.params.outputs.style_checker_image_name }}:${{ needs.params.outputs.style_checker_tools_version }}${{ needs.params.outputs.image_suffix }} steps: - name: Check Snapshots run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check C Style run: citus_indent --check - name: Check Python style run: black --check . - name: Check Python import order run: isort --check . - name: Check Python lints run: flake8 . - name: Fix whitespace run: ci/editorconfig.sh && git diff --exit-code - name: Remove useless declarations run: ci/remove_useless_declarations.sh && git diff --cached --exit-code - name: Sort and group includes run: ci/sort_and_group_includes.sh && git diff --exit-code - name: Normalize test output run: ci/normalize_expected.sh && git diff --exit-code - name: Check for C-style comments in migration files run: ci/disallow_c_comments_in_migrations.sh && git diff --exit-code - name: 'Check for comment--cached ns that start with # character in spec files' run: ci/disallow_hash_comments_in_spec_files.sh && git diff --exit-code - name: Check for gitignore entries .for source files run: ci/fix_gitignore.sh && git diff --exit-code - name: Check for lengths of changelog entries run: ci/disallow_long_changelog_entries.sh - name: Check for banned C API usage run: ci/banned.h.sh - name: Check for tests missing in schedules run: ci/check_all_tests_are_run.sh - name: Check if all CI scripts are actually run run: ci/check_all_ci_scripts_are_run.sh - name: Check if all GUCs are sorted alphabetically run: ci/check_gucs_are_alphabetically_sorted.sh - name: Check for missing downgrade scripts run: ci/check_migration_files.sh build: needs: params name: Build for PG${{ fromJson(matrix.pg_version).major }} strategy: fail-fast: false matrix: image_name: - ${{ needs.params.outputs.build_image_name }} image_suffix: - ${{ needs.params.outputs.image_suffix}} pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} - ${{ needs.params.outputs.pg18_version }} runs-on: ubuntu-latest container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}" options: --user root steps: - uses: actions/checkout@v4 - name: Expose $PG_MAJOR to Github Env run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV shell: bash - name: Build run: "./ci/build-citus.sh" shell: bash - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- ./build-${{ env.PG_MAJOR }}/* ./install-${{ env.PG_MAJOR }}.tar test-citus: name: Test Citus uses: ./.github/workflows/run_tests.yml needs: - params - build with: pg_versions: > [ ${{ needs.params.outputs.pg16_version }}, ${{ needs.params.outputs.pg17_version }}, ${{ needs.params.outputs.pg18_version }} ] make_targets: '["check-split", "check-multi", "check-multi-1", "check-multi-1-create-citus", "check-multi-mx", "check-vanilla", "check-isolation", "check-operations", "check-follower-cluster", "check-add-backup-node", "check-columnar", "check-columnar-isolation", "check-enterprise", "check-enterprise-isolation", "check-enterprise-isolation-logicalrep-1", "check-enterprise-isolation-logicalrep-2", "check-enterprise-isolation-logicalrep-3", "check-tap"]' image_suffix: ${{ needs.params.outputs.image_suffix }} image_name: ${{ needs.params.outputs.test_image_name }} secrets: codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-failure: name: Test Citus Failure uses: ./.github/workflows/run_tests.yml needs: - params - build with: pg_versions: > [ ${{ needs.params.outputs.pg16_version }}, ${{ needs.params.outputs.pg17_version }}, ${{ needs.params.outputs.pg18_version }} ] make_targets: '["check-failure", "check-enterprise-failure", "check-pytest", "check-query-generator"]' image_suffix: ${{ needs.params.outputs.image_suffix }} image_name: ${{ needs.params.outputs.fail_test_image_name }} secrets: codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-cdc: name: Test Citus CDC uses: ./.github/workflows/run_tests.yml needs: - params - build with: pg_versions: > [ ${{ needs.params.outputs.pg16_version }}, ${{ needs.params.outputs.pg17_version }}, ${{ needs.params.outputs.pg18_version }} ] make_targets: '["installcheck"]' image_suffix: ${{ needs.params.outputs.image_suffix }} image_name: ${{ needs.params.outputs.test_image_name }} suite: cdc secrets: codecov_token: ${{ secrets.CODECOV_TOKEN }} test-arbitrary-configs: name: PG${{ fromJson(matrix.pg_version).major }} - check-arbitrary-configs-${{ matrix.parallel }} runs-on: ["self-hosted", "1ES.Pool=1es-gha-citusdata-pool"] container: image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params - build strategy: fail-fast: false matrix: image_name: - ${{ needs.params.outputs.fail_test_image_name }} pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} - ${{ needs.params.outputs.pg18_version }} parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Test arbitrary configs run: |- # we use parallel jobs to split the tests into 6 parts and run them in parallel # the script below extracts the tests for the current job N=6 # Total number of jobs (see matrix.parallel) X=${{ matrix.parallel }} # Current job number TESTS=$(src/test/regress/citus_tests/print_test_names.py | tr '\n' ',' | awk -v N="$N" -v X="$X" -F, '{ split("", parts) for (i = 1; i <= NF; i++) { parts[i % N] = parts[i % N] $i "," } print substr(parts[X], 1, length(parts[X])-1) }') echo $TESTS gosu circleci \ make -C src/test/regress \ check-arbitrary-configs parallel=4 CONFIGS=$TESTS - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} - uses: "./.github/actions/upload_coverage" if: always() with: flags: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} codecov_token: ${{ secrets.CODECOV_TOKEN }} test-pg-upgrade: name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade runs-on: ubuntu-latest container: image: "${{ needs.params.outputs.pgupgrade_image_name }}:${{ needs.params.outputs.upgrade_pg_versions }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params - build strategy: fail-fast: false matrix: include: - old_pg_major: 16 new_pg_major: 17 - old_pg_major: 17 new_pg_major: 18 - old_pg_major: 16 new_pg_major: 18 env: old_pg_major: ${{ matrix.old_pg_major }} new_pg_major: ${{ matrix.new_pg_major }} steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" with: pg_major: "${{ env.old_pg_major }}" - uses: "./.github/actions/setup_extension" with: pg_major: "${{ env.new_pg_major }}" - name: Install and test postgres upgrade run: |- gosu circleci \ make -C src/test/regress \ check-pg-upgrade \ old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin \ test-with-columnar=false gosu circleci \ make -C src/test/regress \ check-pg-upgrade \ old-bindir=/usr/lib/postgresql/${{ env.old_pg_major }}/bin \ new-bindir=/usr/lib/postgresql/${{ env.new_pg_major }}/bin \ test-with-columnar=true - name: Copy pg_upgrade logs for newData dir run: |- mkdir -p /tmp/pg_upgrade_newData_logs if ls src/test/regress/tmp_upgrade/newData/*.log 1> /dev/null 2>&1; then cp src/test/regress/tmp_upgrade/newData/*.log /tmp/pg_upgrade_newData_logs fi if: failure() - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: flags: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} test-citus-upgrade: name: PG${{ fromJson(matrix.pg_version).major }} - check-citus-upgrade runs-on: ubuntu-latest container: image: "${{ needs.params.outputs.citusupgrade_image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}" options: --user root needs: - params - build strategy: fail-fast: false matrix: pg_version: - ${{ needs.params.outputs.pg16_version }} - ${{ needs.params.outputs.pg17_version }} steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" with: skip_installation: true - name: Install and test citus upgrade run: |- # run make check-citus-upgrade for all citus versions # the image has ${CITUS_VERSIONS} set with all versions it contains the binaries of for citus_version in ${CITUS_VERSIONS}; do \ gosu circleci \ make -C src/test/regress \ check-citus-upgrade \ bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ citus-old-version=${citus_version} \ citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ done; # run make check-citus-upgrade-mixed for all citus versions # the image has ${CITUS_VERSIONS} set with all versions it contains the binaries of for citus_version in ${CITUS_VERSIONS}; do \ gosu circleci \ make -C src/test/regress \ check-citus-upgrade-mixed \ citus-old-version=${citus_version} \ bindir=/usr/lib/postgresql/${PG_MAJOR}/bin \ citus-pre-tar=/install-pg${PG_MAJOR}-citus${citus_version}.tar \ citus-post-tar=${GITHUB_WORKSPACE}/install-$PG_MAJOR.tar; \ done; - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: ${{ env.PG_MAJOR }}_citus_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: flags: ${{ env.PG_MAJOR }}_citus_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} ch_benchmark: name: CH Benchmark if: startsWith(github.ref, 'refs/heads/ch_benchmark/') runs-on: ubuntu-latest needs: - build steps: - uses: actions/checkout@v4 - uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} - name: install dependencies and run ch_benchmark tests uses: azure/CLI@v1 with: inlineScript: | cd ./src/test/hammerdb chmod +x run_hammerdb.sh run_hammerdb.sh citusbot_ch_benchmark_rg tpcc_benchmark: name: TPCC Benchmark if: startsWith(github.ref, 'refs/heads/tpcc_benchmark/') runs-on: ubuntu-latest needs: - build steps: - uses: actions/checkout@v4 - uses: azure/login@v1 with: creds: ${{ secrets.AZURE_CREDENTIALS }} - name: install dependencies and run tpcc_benchmark tests uses: azure/CLI@v1 with: inlineScript: | cd ./src/test/hammerdb chmod +x run_hammerdb.sh run_hammerdb.sh citusbot_tpcc_benchmark_rg prepare_parallelization_matrix_32: name: Prepare parallelization matrix if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} needs: test-flakyness-pre runs-on: ubuntu-latest outputs: json: ${{ steps.parallelization.outputs.json }} steps: - uses: actions/checkout@v4 - uses: "./.github/actions/parallelization" id: parallelization with: count: 32 test-flakyness-pre: name: Detect regression tests need to be ran if: ${{ !inputs.skip_test_flakyness }}} runs-on: ubuntu-latest needs: build outputs: tests: ${{ steps.detect-regression-tests.outputs.tests }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Detect regression tests need to be ran id: detect-regression-tests run: |- detected_changes=$(git diff origin/main... --name-only --diff-filter=AM | (grep 'src/test/regress/sql/.*\.sql\|src/test/regress/spec/.*\.spec\|src/test/regress/citus_tests/test/test_.*\.py' || true)) tests=${detected_changes} # split the tests to be skipped --today we only skip upgrade tests # and snapshot based node addition tests. # snapshot based node addition tests are not flaky, as they promote # the streaming replica (clone) to a PostgreSQL primary node that is one way # operation skipped_tests="" not_skipped_tests="" for test in $tests; do if [[ $test =~ ^src/test/regress/sql/upgrade_ ]] || [[ $test =~ ^src/test/regress/sql/multi_add_node_from_backup ]]; then skipped_tests="$skipped_tests $test" else not_skipped_tests="$not_skipped_tests $test" fi done if [ ! -z "$skipped_tests" ]; then echo "Skipped tests " $skipped_tests fi if [ -z "$not_skipped_tests" ]; then echo "Not detected any tests that flaky test detection should run" else echo "Detected tests " $not_skipped_tests fi echo 'tests<> $GITHUB_OUTPUT echo "$not_skipped_tests" >> "$GITHUB_OUTPUT" echo 'EOF' >> $GITHUB_OUTPUT test-flakyness: if: ${{ needs.test-flakyness-pre.outputs.tests != ''}} name: Test flakyness runs-on: ubuntu-latest container: image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg18_version).full }}${{ needs.params.outputs.image_suffix }} options: --user root env: runs: 8 needs: - params - build - test-flakyness-pre - prepare_parallelization_matrix_32 strategy: fail-fast: false matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4.1.8 - uses: "./.github/actions/setup_extension" - name: Run minimal tests run: |- tests="${{ needs.test-flakyness-pre.outputs.tests }}" tests_array=($tests) for test in "${tests_array[@]}" do test_name=$(echo "$test" | sed -r "s/.+\/(.+)\..+/\1/") gosu circleci src/test/regress/citus_tests/run_test.py $test_name --repeat ${{ env.runs }} --use-whole-schedule-line done shell: bash - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: test_flakyness_parallel_${{ matrix.id }} ================================================ FILE: .github/workflows/codeql.yml ================================================ name: "CodeQL" on: schedule: - cron: '59 23 * * 6' workflow_dispatch: jobs: analyze: name: Analyze runs-on: ubuntu-22.04 permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'cpp', 'python'] steps: - name: Checkout repository uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Install package dependencies run: | # Create the file repository configuration: sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main 15" > /etc/apt/sources.list.d/pgdg.list' # Import the repository signing key: wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update sudo apt-get install -y --no-install-recommends \ autotools-dev \ build-essential \ ca-certificates \ curl \ debhelper \ devscripts \ fakeroot \ flex \ libcurl4-openssl-dev \ libdistro-info-perl \ libedit-dev \ libfile-fcntllock-perl \ libicu-dev \ libkrb5-dev \ liblz4-1 \ liblz4-dev \ libpam0g-dev \ libreadline-dev \ libselinux1-dev \ libssl-dev \ libxslt-dev \ libzstd-dev \ libzstd1 \ lintian \ postgresql-server-dev-17 \ python3-pip \ python3-setuptools \ wget \ zlib1g-dev - name: Configure, Build and Install Citus if: matrix.language == 'cpp' run: | ./configure make -sj8 sudo make install-all - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 ================================================ FILE: .github/workflows/devcontainer.yml ================================================ name: "Build devcontainer" # Since building of containers can be quite time consuming, and take up some storage, # there is no need to finish a build for a tag if new changes are concurrently being made. # This cancels any previous builds for the same tag, and only the latest one will be kept. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: push: paths: - ".devcontainer/**" workflow_dispatch: jobs: docker: runs-on: ubuntu-latest permissions: contents: read packages: write attestations: write id-token: write steps: - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ghcr.io/citusdata/citus-devcontainer tags: | type=ref,event=branch type=sha - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: 'Login to GitHub Container Registry' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} - name: Build and push uses: docker/build-push-action@v5 with: context: "{{defaultContext}}:.devcontainer" push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max ================================================ FILE: .github/workflows/flaky_test_debugging.yml ================================================ name: Flaky test debugging run-name: Flaky test debugging - ${{ inputs.flaky_test }} (${{ inputs.flaky_test_runs_per_job }}x${{ inputs.flaky_test_parallel_jobs }}) concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true on: workflow_dispatch: inputs: flaky_test: required: true type: string description: Test to run flaky_test_runs_per_job: required: false default: 8 type: number description: Number of times to run the test flaky_test_parallel_jobs: required: false default: 32 type: number description: Number of parallel jobs to run jobs: build: name: Build Citus runs-on: ubuntu-latest container: image: ${{ vars.build_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} options: --user root steps: - uses: actions/checkout@v4 - name: Configure, Build, and Install run: | echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV ./ci/build-citus.sh shell: bash - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- ./build-${{ env.PG_MAJOR }}/* ./install-${{ env.PG_MAJOR }}.tar prepare_parallelization_matrix: name: Prepare parallelization matrix runs-on: ubuntu-latest outputs: json: ${{ steps.parallelization.outputs.json }} steps: - uses: actions/checkout@v4 - uses: "./.github/actions/parallelization" id: parallelization with: count: ${{ inputs.flaky_test_parallel_jobs }} test_flakyness: name: Test flakyness runs-on: ubuntu-latest container: image: ${{ vars.fail_test_image_name }}:${{ vars.pg16_version }}${{ vars.image_suffix }} options: --user root needs: [build, prepare_parallelization_matrix] env: test: "${{ inputs.flaky_test }}" runs: "${{ inputs.flaky_test_runs_per_job }}" skip: false strategy: fail-fast: false matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }} steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Run minimal tests run: |- gosu circleci src/test/regress/citus_tests/run_test.py ${{ env.test }} --repeat ${{ env.runs }} --use-whole-schedule-line shell: bash - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: check_flakyness_parallel_${{ matrix.id }} ================================================ FILE: .github/workflows/packaging-test-pipelines.yml ================================================ name: Build tests in packaging images on: pull_request: types: [opened, reopened,synchronize] merge_group: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: get_postgres_versions_from_file: runs-on: ubuntu-latest outputs: pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 2 - name: Get Postgres Versions id: get-postgres-versions run: | set -euxo pipefail # Postgres versions are stored in .github/workflows/build_and_test.yml # file in json strings with major and full keys. # Below command extracts the versions and get the unique values. pg_versions=$(cat .github/workflows/build_and_test.yml | grep -oE '"major": "[0-9]+", "full": "[^"]+"' | sed -E 's/.*"major": "([0-9]+)".*/\1/' | sort -n | uniq | tr '\n' ',') pg_versions_array="[ ${pg_versions} ]" echo "Supported PG Versions: ${pg_versions_array}" # Below line is needed to set the output variable to be used in the next job echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT shell: bash rpm_build_tests: name: rpm_build_tests needs: get_postgres_versions_from_file runs-on: ubuntu-latest strategy: fail-fast: false matrix: # While we use separate images for different Postgres versions in rpm # based distros # For this reason, we need to use a "matrix" to generate names of # rpm images, e.g. citus/packaging:centos-7-pg12 packaging_docker_image: - oraclelinux-8 - almalinux-8 - almalinux-9 POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }} container: image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }} options: --user root steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set Postgres and python parameters for rpm based distros run: | echo "/usr/pgsql-${{ matrix.POSTGRES_VERSION }}/bin" >> $GITHUB_PATH echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV - name: Configure run: | echo "Current Shell:$0" echo "GCC Version: $(gcc --version)" ./configure 2>&1 | tee output.log - name: Make clean run: | make clean - name: Make run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} make CFLAGS="-Wno-missing-braces" -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log # Check the exit code of the make command make_exit_code=${PIPESTATUS[0]} # If the make command returned a non-zero exit code, exit with the same code if [[ $make_exit_code -ne 0 ]]; then echo "make command failed with exit code $make_exit_code" exit $make_exit_code fi - name: Make install run: | make CFLAGS="-Wno-missing-braces" install 2>&1 | tee -a output.log - name: Validate output env: POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }} PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }} run: | echo "Postgres version: ${POSTGRES_VERSION}" ./.github/packaging/validate_build_output.sh "rpm" deb_build_tests: name: deb_build_tests needs: get_postgres_versions_from_file runs-on: ubuntu-latest strategy: fail-fast: false matrix: # On deb based distros, we use the same docker image for # builds based on different Postgres versions because deb # based images include all postgres installations. # For this reason, we have multiple runs --which is 3 today-- # for each deb based image and we use POSTGRES_VERSION to set # PG_CONFIG variable in each of those runs. packaging_docker_image: - debian-bookworm-all - debian-bullseye-all - ubuntu-focal-all - ubuntu-jammy-all POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }} exclude: # PG18 is not supported on Ubuntu focal - packaging_docker_image: ubuntu-focal-all POSTGRES_VERSION: 18 container: image: citus/packaging:${{ matrix.packaging_docker_image }} options: --user root steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set pg_config path and python parameters for deb based distros run: | echo "PG_CONFIG=/usr/lib/postgresql/${{ matrix.POSTGRES_VERSION }}/bin/pg_config" >> $GITHUB_ENV echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV - name: Configure run: | echo "Current Shell:$0" echo "GCC Version: $(gcc --version)" ./configure 2>&1 | tee output.log - name: Make clean run: | make clean - name: Make shell: bash run: | set -e git config --global --add safe.directory ${GITHUB_WORKSPACE} make -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log # Check the exit code of the make command make_exit_code=${PIPESTATUS[0]} # If the make command returned a non-zero exit code, exit with the same code if [[ $make_exit_code -ne 0 ]]; then echo "make command failed with exit code $make_exit_code" exit $make_exit_code fi - name: Make install run: | make install 2>&1 | tee -a output.log - name: Validate output env: POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }} PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }} run: | echo "Postgres version: ${POSTGRES_VERSION}" ./.github/packaging/validate_build_output.sh "deb" ================================================ FILE: .github/workflows/run_tests.yml ================================================ name: Run Tests on: workflow_call: inputs: pg_versions: required: true type: string make_targets: required: true type: string image_suffix: required: true type: string image_name: required: true type: string suite: required: false type: string default: "regress" citus_version: required: false type: string default: "" citus_libdir: required: false type: string default: "" citus_libdir_label: required: false type: string default: "" n_1_mode: required: false type: string default: "" secrets: codecov_token: required: false jobs: test: name: PG${{ matrix.pg_version.major }} - ${{ matrix.make }}${{ inputs.citus_version && format(' - {0}', inputs.citus_version) || '' }}${{ inputs.citus_libdir_label && format(' - lib-{0}', inputs.citus_libdir_label) || '' }} strategy: fail-fast: false matrix: pg_version: ${{ fromJson(inputs.pg_versions) }} make: ${{ fromJson(inputs.make_targets) }} runs-on: ubuntu-latest container: image: "${{ inputs.image_name }}:${{ matrix.pg_version.full }}${{ inputs.image_suffix }}" options: >- --user root --dns=8.8.8.8 --cap-add=SYS_NICE --security-opt seccomp=unconfined steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup_extension" - name: Fix PostgreSQL library permissions for symlink setup if: ${{ inputs.citus_libdir != '' }} run: chmod -R a+w /usr/lib/postgresql/${{ matrix.pg_version.major }}/lib/ - name: Run Test run: CITUSVERSION=${{ inputs.citus_version }} CITUSLIBDIR=${{ inputs.citus_libdir }} N_1_MODE=${{ inputs.n_1_mode }} gosu circleci make -C src/test/${{ inputs.suite }} ${{ matrix.make }} timeout-minutes: 20 - uses: "./.github/actions/save_logs_and_results" if: always() with: folder: ${{ matrix.pg_version.major }}_${{ matrix.make }}${{ inputs.citus_version && format('_{0}', inputs.citus_version) || '' }}${{ inputs.citus_libdir_label && format('_{0}', inputs.citus_libdir_label) || '' }}${{ inputs.n_1_mode && format('_{0}', inputs.n_1_mode) || '' }} - uses: "./.github/actions/upload_coverage" if: always() with: flags: ${{ env.PG_MAJOR }}_${{ inputs.suite }}_${{ matrix.make }}${{ inputs.citus_version && format('_{0}', inputs.citus_version) || '' }}${{ inputs.citus_libdir_label && format('_{0}', inputs.citus_libdir_label) || '' }}${{ inputs.n_1_mode && format('_{0}', inputs.n_1_mode) || '' }} codecov_token: ${{ secrets.codecov_token }} ================================================ FILE: .gitignore ================================================ # Global excludes across all subdirectories *.o *.so *.so.[0-9] *.so.[0-9].[0-9] *.sl *.sl.[0-9] *.sl.[0-9].[0-9] *.dylib *.dll *.a *.mo *.pot objfiles.txt .deps/ *.gcno *.gcda *.gcov *.gcov.out lcov.info coverage/ *.vcproj *.vcxproj win32ver.rc *.exe lib*dll.def lib*.pc *.bc # Local excludes in root directory /config.log /config.status /pgsql.sln /pgsql.sln.cache /Debug/ /Release/ /autom4te.cache /Makefile.global /src/Makefile.custom /compile_commands.json /src/backend/distributed/cdc/build-cdc-*/* /src/test/cdc/tmp_check/* /src/test/tap/tmp_check/* /src/test/tap/log/* # temporary files vim creates *.swp # vscode .vscode/* # output from diff normalization that shouldn't be commited *.unmodified *.modified # style related temporary outputs *.uncrustify .venv # added output when modifying check_gucs_are_alphabetically_sorted.sh guc.out ================================================ FILE: .ignore ================================================ /vendor ================================================ FILE: CHANGELOG.md ================================================ ### citus v14.0.0 (February 9, 2026) ### * Drops PG15 support (#8372) * Adds support for PostgreSQL 18 (#8065) * Adds support for virtual generated columns on distributed tables for PostgreSQL 18 (#8346) * Propagates publish_generated_columns publication option to worker nodes for distributed tables on PostgreSQL 18 (#8360) * Respects VACUUM/ANALYZE ONLY semantics for Citus tables by skipping shard propagation when ONLY is specified on PostgreSQL 18 (#8365) * Allows configuring sslkeylogfile connection parameter using citus.node_conn_info on PostgreSQL 18 (#8437) * Fixes INSERT .. SELECT planning error with GROUP BY on PostgreSQL 18 (#8256) * Fixes deparser error with named joins and whole-row references on PostgreSQL 18 (#8300) * Fixes columnar temp table access failure on PostgreSQL 18 (#8309) * Fixes multi-shard MIN/MAX on composite types by blessing record aggregates (#8429) * Fixes distributed MIN/MAX for array types (#8421) * Adds propagation of ENFORCED / NOT ENFORCED on CHECK constraints (#8349) * Stops on-demand statistics collection for clusters and deprecates citus.enable_statistics_collection GUC (#8460) * Disallows creating a distributed table or altering it to be colocated with another table if distribution key collations don't match (#8257) * Makes citus_create_restore_point MX-safe by blocking 2PC commit decisions (#8352) * Supports binary transfer from worker to coordinator for custom aggregates (#8446) * Ensures query remains safe for deparse when a modify scan needs to evaluate expressions before worker query execution (#8443) * Avoids local plan cache reuse for multi-shard queries (#8371) * Tightens distributed plan check to cover distributed subplans (#8388) * Improves performance by avoiding unnecessary shallow copy of target list when no array or json subscripts are present (#8155) * Fixes a bug that ignores reference or schema sharded tables in worker subquery task construction when no distributed tables are involved (#8440) * Fixes a bug in redundant WHERE clause detection (#8162) * Fixes a bug that causes allowing UPDATE / MERGE queries that may change the distribution column value (#8214) * Fixes a bug that causes an unexpected error when executing repartitioned MERGE (#8201) * Fixes an assertion failure in Citus maintenance daemon that can happen in very slow systems (#8158) * Removes an assertion from Postgres ruleutils that was rendered meaningless by a previous Citus commit (#8136) * Fixes incorrect rejection of colocated joins when Row Level Security policies use volatile functions (#8357) * Fixes metadata sync failure when distributed tables use domain types defined in non-public schemas (#8363) * Fixes a crash on CREATE STATISTICS with non-table expressions (#8213, #8227) * Fixes invalid input syntax for type bigint in citus_stats with large tables (#8166) * Fixes an undefined behavior that could happen when computing tenant score for citus_stat_tenants (#7954) ### citus v13.1.1 (Oct 1st, 2025) ### * Adds support for latest PG minors: 14.19, 15.14, 16.10 (#8142) * Fixes an assertion failure when an expression in the query references a CTE (#8106) * Fixes a bug that causes an unexpected error when executing repartitioned MERGE (#8201) * Fixes a bug that causes allowing UPDATE / MERGE queries that may change the distribution column value (#8214) * Updates dynamic_library_path automatically when CDC is enabled (#8025) ### citus v13.0.5 (Oct 1st, 2025) ### * Adds support for latest PG minors: 14.19, 15.14, 16.10 (#7986, #8142) * Fixes a bug that causes an unexpected error when executing repartitioned MERGE (#8201) * Fixes a bug that causes allowing UPDATE / MERGE queries that may change the distribution column value (#8214) * Fixes a bug in redundant WHERE clause detection (#8162) * Updates dynamic_library_path automatically when CDC is enabled (#8025) ### citus v12.1.10 (Oct 1, 2025) ### * Adds support for latest PG minors: 14.19, 15.14, 16.10 (#7986, #8142) * Fixes a bug that causes allowing UPDATE / MERGE queries that may change the distribution column value (#8214) * Fixes an assertion failure that happens when querying a view that is defined on distributed tables (#8136) ### citus v12.1.9 (Sep 3, 2025) ### * Adds a GUC for queries with outer joins and pseudoconstant quals (#8163) * Updates dynamic_library_path automatically when CDC is enabled (#7715) ### citus v13.2.0 (August 18, 2025) ### * Adds `citus_add_clone_node()`, `citus_add_clone_node_with_nodeid()`, `citus_remove_clone_node()` and `citus_remove_clone_node_with_nodeid()` UDFs to support snapshot-based node splits. This feature allows promoting a streaming replica (clone) to a primary node and rebalancing shards between the original and newly promoted node without requiring a full data copy. This greatly reduces rebalance times for scale-out operations when the new node already has the data via streaming replication (#8122) * Improves performance of shard rebalancer by parallelizing moves and removing bottlenecks that blocked concurrent logical-replication transfers. This reduces rebalance windows especially for clusters with large reference tables and allows multiple shard transfers to run in parallel (#7983) * Adds citus.enable_recurring_outer_join_pushdown GUC (enabled by default) to allow pushing down LEFT/RIGHT outer joins having a reference table in the outer side and a distributed table on the inner side (e.g., \ LEFT JOIN \) (#7973) * Adds citus.enable_local_fast_path_query_optimization (enabled by default) GUC to avoid unnecessary query deparsing to improve performance of fast-path queries targeting local shards (#8035) * Adds `citus_stats()` UDF that can be used to retrieve distributed `pg_stats` for the provided Citus table. (#8026) * Avoids automatically creating citus_columnar when there are no relations using it (#8081) * Makes sure to check if the distribution key is in the target list before pushing down a query with a union and an outer join (#8092) * Fixes a bug in EXPLAIN ANALYZE to prevent unintended (duplicate) execution of the (sub)plans during the explain phase (#8017) * Fixes potential memory corruptions that could happen when accessing various catalog tables after a Citus downgrade is followed by a Citus upgrade (#7950, #8120, #8124, #8121, #8114, #8146) * Fixes UPDATE statements with indirection and array/jsonb subscripting with more than one field (#7675) * Fixes an assertion failure that happens when an expression in the query references a CTE (#8106) * Fixes an assertion failure that happens when querying a view that is defined on distributed tables (#8136) ### citus v13.1.0 (May 30th, 2025) ### * Adds `citus_stat_counters` view that can be used to query stat counters that Citus collects while the feature is enabled, which is controlled by citus.enable_stat_counters. `citus_stat_counters()` can be used to query the stat counters for the provided database oid and `citus_stat_counters_reset()` can be used to reset them for the provided database oid or for the current database if nothing or 0 is provided (#7917) * Adds `citus_nodes` view that displays the node name, port role, and "active" for nodes in the cluster (#7968) * Adds `citus_is_primary_node()` UDF to determine if the current node is a primary node in the cluster (#7720) * Adds support for propagating `GRANT/REVOKE` rights on table columns (#7918) * Adds support for propagating `REASSIGN OWNED BY` commands (#7319) * Adds support for propagating `CREATE`/`DROP` database from all nodes (#7240, #7253, #7359) * Propagates `SECURITY LABEL ON ROLE` statement from any node (#7508) * Adds support for issuing role management commands from worker nodes (#7278) * Adds support for propagating `ALTER USER RENAME` commands (#7204) * Adds support for propagating `ALTER DATABASE SET ..` commands (#7181) * Adds support for propagating `SECURITY LABEL` on tables and columns (#7956) * Adds support for propagating `COMMENT ON /` commands (#7388) * Moves some of the internal citus functions from `pg_catalog` to `citus_internal` schema (#7473, #7470, #7466, 7456, 7450) * Adjusts `max_prepared_transactions` only when it's set to default on PG >= 16 (#7712) * Adds skip_qualify_public param to shard_name() UDF to allow qualifying for "public" schema when needed (#8014) * Allows `citus_*_size` on indexes on a distributed tables (#7271) * Allows `GRANT ADMIN` to now also be `INHERIT` or `SET` in support of PG16 * Makes sure `worker_copy_table_to_node` errors out with Citus tables (#7662) * Adds information to explain output when using `citus.explain_distributed_queries=false` (#7412) * Logs username in the failed connection message (#7432) * Makes sure to avoid incorrectly pushing-down the outer joins between distributed tables and recurring relations (like reference tables, local tables and `VALUES(..)` etc.) prior to PG 17 (#7937) * Prevents incorrectly pushing `nextval()` call down to workers to avoid using incorrect sequence value for some types of `INSERT .. SELECT`s (#7976) * Makes sure to prevent `INSERT INTO ... SELECT` queries involving subfield or sublink, to avoid crashes (#7912) * Makes sure to take improvement_threshold into the account in `citus_add_rebalance_strategy()` (#7247) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) * Fixes a bug that causes omitting `CASCADE` clause for the commands sent to workers for `REVOKE` commands on tables (#7958) * Fixes an issue detected using address sanitizer (#7948, #7949) * Fixes a bug in deparsing of shard query in case of "output-table column" name conflict (#7932) * Fixes a crash in columnar custom scan that happens when a columnar table is used in a join (#7703) * Fixes `MERGE` command when insert value does not have source distributed column (#7627) * Fixes performance issue when using `\d tablename` on a server with many tables (#7577) * Fixes performance issue in `GetForeignKeyOids` on systems with many constraints (#7580) * Fixes performance issue when distributing a table that depends on an extension (#7574) * Fixes performance issue when creating distributed tables if many already exist (#7575) * Fixes a crash caused by some form of `ALTER TABLE ADD COLUMN` statements. When adding multiple columns, if one of the `ADD COLUMN` statements contains a `FOREIGN` constraint ommitting the referenced columns in the statement, a `SEGFAULT` occurs (#7522) * Fixes assertion failure in maintenance daemon during Citus upgrades (#7537) * Fixes segmentation fault when using `CASE WHEN` in `DO` block functions (#7554) * Fixes undefined behavior in `master_disable_node` due to argument mismatch (#7492) * Fixes incorrect propagating of `GRANTED BY` and `CASCADE/RESTRICT` clauses for `REVOKE` statements (#7451) * Fixes the incorrect column count after `ALTER TABLE` (#7379) * Fixes timeout when underlying socket is changed for an inter-node connection (#7377) * Fixes memory leaks (#7441, #7440) * Fixes leaking of memory and memory contexts when tracking foreign keys between Citus tables (#7236) * Fixes a potential segfault for background rebalancer (#7694) * Fixes potential `NULL` dereference in casual clocks (#7704) ### citus v13.0.4 (May 29th, 2025) ### * Fixes an issue detected using address sanitizer (#7966) * Error out for queries with outer joins and pseudoconstant quals in versions prior to PG 17 (#7937) ### citus v12.1.8 (May 29, 2025) ### * Fixes a crash in left outer joins that can happen when there is an an aggregate on a column from the inner side of the join (#7904) * Fixes an issue detected using address sanitizer (#7965) * Fixes a crash when executing a prepared CALL, which is not pure SQL but available with some drivers like npgsql and jpgdbc (#7288) ### citus v13.0.3 (March 20th, 2025) ### * Fixes a version bump issue in 13.0.2 ### citus v13.0.2 (March 12th, 2025) ### * Fixes a crash in columnar custom scan that happens when a columnar table is used in a join. (#7647) * Fixes a bug that breaks `UPDATE SET (...) = (SELECT some_func(),... )` type of queries on Citus tables (#7914) * Fixes a planning error caused by a redundant WHERE clause (#7907) * Fixes a crash in left outer joins that can happen when there is an aggregate on a column from the inner side of the join. (#7901) * Fixes deadlock with transaction recovery that is possible during Citus upgrades. (#7910) * Fixes a bug that prevents inserting into Citus tables that uses a GENERATED ALWAYS AS IDENTITY column. (#7920) * Ensures that a MERGE command on a distributed table with a WHEN NOT MATCHED BY SOURCE clause runs against all shards of the distributed table. (#7900) * Fixes a bug that breaks router updates on distributed tables when a reference table is used in the subquery (#7897) ### citus v12.1.7 (Feb 6, 2025) ### * Fixes a crash that happens because of unsafe catalog access when re-assigning the global pid after `application_name` changes (#7791) * Prevents crashes when another extension skips executing the `ClientAuthentication_hook` of Citus. (#7836) ### citus v13.0.1 (February 4th, 2025) ### * Drops support for PostgreSQL 14 (#7753) ### citus v13.0.0 (January 22, 2025) ### * Adds support for PostgreSQL 17 (#7699, #7661) * Adds `JSON_TABLE()` support in distributed queries (#7816) * Propagates `MERGE ... WHEN NOT MATCHED BY SOURCE` (#7807) * Propagates `MEMORY` and `SERIALIZE` options of `EXPLAIN` (#7802) * Adds support for identity columns in distributed partitioned tables (#7785) * Allows specifying an access method for distributed partitioned tables (#7818) * Allows exclusion constraints on distributed partitioned tables (#7733) * Allows configuring sslnegotiation using `citus.node_conn_info` (#7821) * Avoids wal receiver timeouts during large shard splits (#7229) * Fixes a bug causing incorrect writing of data to target `MERGE` repartition command (#7659) * Fixes a crash that happens because of unsafe catalog access when re-assigning the global pid after `application_name` changes (#7791) * Fixes incorrect `VALID UNTIL` setting assumption made for roles when syncing them to new nodes (#7534) * Fixes segfault when calling distributed procedure with a parameterized distribution argument (#7242) * Fixes server crash when trying to execute `activate_node_snapshot()` on a single-node cluster (#7552) * Improves `citus_move_shard_placement()` to fail early if there is a new node without reference tables yet (#7467) ### citus v12.1.6 (Nov 14, 2024) ### * Propagates `SECURITY LABEL .. ON ROLE` statements (#7304) * Fixes crash caused by running queries with window partition (#7718) ### citus v12.1.5 (July 17, 2024) ### * Adds support for MERGE commands with single shard distributed target tables (#7643) * Fixes an error with MERGE commands when insert value does not have source distribution column (#7627) ### citus v12.1.4 (May 28, 2024) ### * Adds null check for node in HasRangeTableRef (#7604) ### citus v12.1.3 (April 18, 2024) ### * Allows overwriting host name for all inter-node connections by supporting "host" parameter in citus.node_conninfo (#7541) * Avoids distributed deadlocks by changing the order in which the locks are acquired for the target and reference tables (#7542) * Fixes a performance issue when distributing a table that depends on an extension (#7574) * Fixes a performance issue when using "\d tablename" on a server with many tables (#7577) * Fixes a crash caused by some form of ALTER TABLE ADD COLUMN statements. When adding multiple columns, if one of the ADD COLUMN statements contains a FOREIGN constraint omitting the referenced columns in the statement, a SEGFAULT was occurring. (#7522) * Fixes a performance issue when creating distributed tables if many already exist (#7575, #7579) * Fixes a bug when hostname in pg_dist_node resolves to multiple IPs (#7377) * Fixes performance issue when tracking foreign key constraints on systems with many constraints (#7578) * Fixes segmentation fault when using CASE WHEN in DO block within functions. (#7554) * Fixes undefined behavior in master_disable_node due to argument mismatch (#7492) * Fixes some potential bugs by correctly marking some variables as volatile (#7570) * Logs username in the failed connection message (#7432) ### citus v11.0.10 (February 15, 2024) ### * Removes pg_send_cancellation and all references (#7135) ### citus v12.1.2 (February 12, 2024) ### * Fixes the incorrect column count after ALTER TABLE (#7379) ### citus v12.0.1 (July 11, 2023) ### * Fixes incorrect default value assumption for VACUUM(PROCESS_TOAST) #7122) * Fixes a bug that causes an unexpected error when adding a column with a NULL constraint (#7093) * Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152) * Fixes a bug with deleting colocation groups (#6929) * Fixes memory and memory contexts leaks in Foreign Constraint Graphs (#7236) * Fixes shard size bug with too many shards (#7018) * Fixes the incorrect column count after ALTER TABLE (#7379) * Improves citus_tables view performance (#7050) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) * Removes pg_send_cancellation and all references (#7135) ### citus v11.3.1 (February 12, 2024) ### * Disallows MERGE when the query prunes down to zero shards (#6946) * Fixes a bug related to non-existent objects in DDL commands (#6984) * Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152) * Fixes a bug with deleting colocation groups (#6929) * Fixes incorrect results on fetching scrollable with hold cursors (#7014) * Fixes memory and memory context leaks in Foreign Constraint Graphs (#7236) * Fixes replicate reference tables task fail when user is superuser (#6930) * Fixes the incorrect column count after ALTER TABLE (#7379) * Improves citus_shard_sizes performance (#7050) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) * Removes pg_send_cancellation and all references (#7135) ### citus v11.2.2 (February 12, 2024) ### * Fixes a bug in background shard rebalancer where the replicate reference tables task fails if the current user is not a superuser (#6930) * Fixes a bug related to non-existent objects in DDL commands (#6984) * Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152) * Fixes a bug with deleting colocation groups (#6929) * Fixes incorrect results on fetching scrollable with hold cursors (#7014) * Fixes memory and memory context leaks in Foreign Constraint Graphs (#7236) * Fixes the incorrect column count after ALTER TABLE (#7379) * Improves failure handling of distributed execution (#7090) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) * Removes pg_send_cancellation (#7135) ### citus v11.1.7 (February 12, 2024) ### * Fixes memory and memory context leaks in Foreign Constraint Graphs (#7236) * Fixes a bug related to non-existent objects in DDL commands (#6984) * Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152) * Fixes a bug with deleting colocation groups (#6929) * Fixes incorrect results on fetching scrollable with hold cursors (#7014) * Fixes the incorrect column count after ALTER TABLE (#7379) * Improves failure handling of distributed execution (#7090) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) * Removes pg_send_cancellation and all references (#7135) ### citus v11.0.9 (February 12, 2024) ### * Fixes a bug that could cause COPY logic to skip data in case of OOM (#7152) * Fixes a bug with deleting colocation groups (#6929) * Fixes memory and memory context leaks in Foreign Constraint Graphs (#7236) * Fixes the incorrect column count after ALTER TABLE (#7462) * Improve failure handling of distributed execution (#7090) ### citus v12.1.1 (November 9, 2023) ### * Fixes leaking of memory and memory contexts in Citus foreign key cache (#7236) * Makes sure to disallow creating a replicated distributed table concurrently (#7219) ### citus v12.1.0 (September 12, 2023) ### * Adds support for PostgreSQL 16.0 (#7173) * Add `citus_schema_move()` function which moves tables within a distributed schema to another node (#7180) * Adds `citus_pause_node_within_txn()` UDF that allows pausing the node with given id (#7089) * Makes sure to enforce shard level colocation with the GUC `citus.enable_non_colocated_router_query_pushdown` (#7076) * Allows creating reference / distributed-schema tables from local tables added to metadata and that use identity columns (#7131) * Propagates `BUFFER_USAGE_LIMIT` option in `VACUUM` and `ANALYZE` (#7114) * Propagates `PROCESS_MAIN`, `SKIP_DATABASE_STATS`, `ONLY_DATABASE_STATS` options in `VACUUM` (#7114) * Propagates `GENERIC_PLAN` option in `EXPLAIN` (#7141) * Propagates "rules" option in `CREATE COLLATION` (#7185) * Propagates `GRANT`/ `REVOKE` for database privileges (#7109) * Adds TRUNCATE trigger support on Citus foreign tables (#7170) * Removes `pg_send_cancellation` (#7135) * Prevents unnecessarily pulling the data into coordinator for some `INSERT .. SELECT` queries that target a single-shard group (#7077) * Makes sure that rebalancer throws an error if replication factor is greater than the shard allowed node count. Also makes sure to avoid moving a shard to a node that it already exists on. (#7074) * Fixes a bug that may appear during 2PC recovery when there are multiple databases (#7174) * Fixes a bug that could cause `COPY` logic to skip data in case of out-of-memory (#7152) * Fixes a bug that causes an unexpected error when adding a column with a `NULL` constraint (#7093) * Fixes `PROCESS_TOAST` default value to `true` (#7122) * Improves the error thrown when there is datatype mismatch in `MERGE ON` (#7081) ### citus v12.0.0 (July 11, 2023) ### * Adds support for schema-based sharding. While `citus.enable_schema_based_sharding` GUC allows sharding the database based on newly created schemas, `citus_schema_distribute()` allows doing so for the existing schemas. Distributed schemas used for sharding the database can be listed by using the view `citus_schemas`, monitored by using the view `citus_stat_schemas`, and undistributed by using the udf `citus_schema_undistribute()` (#6866, #6979, #6933, #6936 and many others) * Supports MERGE command across non-colocated distributed tables/subqueries, reference tables and joins on non-distribution columns (#6927) * Drops PG13 Support (#7002, #7007) * Changes default rebalance strategy to by_disk_size (#7033) * Changes by_disk_size rebalance strategy to have a base size (#7035) * Improves citus_tables view performance (#7018) * Improves tenant monitoring performance (#6868) * Introduces the GUC `citus.stat_tenants_untracked_sample_rate` for sampling in tenant monitoring (#7026) * Adds CPU usage to citus_stat_tenants (#6844) * Propagates `ALTER SCHEMA .. OWNER TO ..` commands to worker (#6987) * Allows `ADD COLUMN` in command string with other commands (#7032) * Allows `DROP CONSTRAINT` in command string with other commands (#7012) * Makes sure to properly handle index storage options for `ADD CONSTRAINT `/ COLUMN commands (#7032) * Makes sure to properly handle `IF NOT EXISTS` for `ADD COLUMN` commands (#7032) * Allows using generated identity column based on int/smallint when creating a distributed table with the limitation of not being able perform DMLs on identity columns from worker nodes (#7008) * Supports custom cast from / to timestamptz in time partition management UDFs (#6923) * Optimizes pushdown planner on memory and cpu (#6945) * Changes citus_shard_sizes view's table_name column to shard_id (#7003) * The GUC search_path is now reported when it is updated (#6983) * Disables citus.enable_non_colocated_router_query_pushdown GUC by default to ensure generating a consistent distributed plan for the queries that reference non-colocated distributed tables (#6909) * Disallows MERGE with filters that prune down to zero shards (#6946) * Makes sure to take `shouldhaveshards` setting into account for a node when planning rebalance steps (#6887) * Improves the compatibility with other extension by forwarding to existing emit_log_hook in our log hook (#6877) * Fixes wrong result when using `NOT MATCHED` with MERGE command (#6943) * Fixes querying the view `citus_shard_sizes` when there are too many shards (#7018) * Fixes a bug related to type casts from other types to text/varchar (#6391) * Fixes propagating `CREATE SCHEMA AUTHORIZATION ..` with no schema name (#7015) * Fixes an error when creating a FOREIGN KEY without a name referencing a schema qualified table (#6986) * Fixes a rare bug which mostly happens with queries that contain both outer join and where clauses (#6857) * Fixes a bug related to propagation of schemas when pg_dist_node is empty (#6900) * Fixes a crash when a query is locally executed with explain analyze (#6892) ### citus v11.3.0 (May 2, 2023) ### * Introduces CDC implementation for Citus using logical replication (#6623, #6810, #6827) * Adds support for `MERGE` command on co-located distributed tables joined on distribution column (#6696, #6733) * Adds the view `citus_stat_tenants` that monitor statistics on tenant usages (#6725) * Adds the GUC `citus.max_background_task_executors_per_node` to control number of background task executors involving a node (#6771) * Allows parallel shard moves in background rebalancer (#6756) * Introduces the GUC `citus.metadata_sync_mode` that introduces nontransactional mode for metadata sync (#6728, #6889) * Propagates CREATE/ALTER/DROP PUBLICATION statements for distributed tables (#6776) * Adds the GUC `citus.enable_non_colocated_router_query_pushdown` to ensure generating a consistent distributed plan for the queries that reference non-colocated distributed tables when set to "false" (#6793) * Checks if all moves are able to be done via logical replication for rebalancer (#6754) * Correctly reports shard size in `citus_shards` view (#6748) * Fixes a bug in shard copy operations (#6721) * Fixes a bug that prevents enforcing identity column restrictions on worker nodes (#6738) * Fixes a bug with `INSERT .. SELECT` queries with identity columns (#6802) * Fixes an issue that caused some queries with custom aggregates to fail (#6805) * Fixes an issue when `citus_set_coordinator_host` is called more than once (#6837) * Fixes an uninitialized memory access in shard split API (#6845) * Fixes memory leak and max allocation block errors during metadata syncing (#6728) * Fixes memory leak in `undistribute_table` (#6693) * Fixes memory leak in `alter_distributed_table` (#6726) * Fixes memory leak in `create_distributed_table` (#6722) * Fixes memory leak issue with query results that returns single row (#6724) * Improves rebalancer when shard groups have placement count less than worker count (#6739) * Makes sure to stop maintenance daemon when dropping a database even without Citus extension (#6688) * Prevents using `alter_distributed_table` and `undistribute_table` UDFs when a table has identity columns (#6738) * Prevents using identity columns on data types other than `bigint` on distributed tables (#6738) ### citus v11.2.1 (April 20, 2023) ### * Correctly reports shard size in `citus_shards` view (#6748) * Fixes a bug in shard copy operations (#6721) * Fixes a bug with `INSERT .. SELECT` queries with identity columns (#6802) * Fixes an uninitialized memory access in shard split API (#6845) * Fixes compilation for PG13.10 and PG14.7 (#6711) * Fixes memory leak in `alter_distributed_table` (#6726) * Fixes memory leak issue with query results that returns single row (#6724) * Prevents using `alter_distributed_table` and `undistribute_table` UDFs when a table has identity columns (#6738) * Prevents using identity columns on data types other than `bigint` on distributed tables (#6738) ### citus v11.1.6 (April 20, 2023) ### * Correctly reports shard size in `citus_shards` view (#6748) * Fixes a bug in shard copy operations (#6721) * Fixes a bug that breaks pg upgrades if the user has a columnar table (#6624) * Fixes a bug that causes background rebalancer to fail when a reference table doesn't have a primary key (#6682) * Fixes a regression in allowed foreign keys on distributed tables (#6550) * Fixes a use-after-free bug in connection management (#6685) * Fixes an unexpected foreign table error by disallowing to drop the `table_name` option (#6669) * Fixes an uninitialized memory access in shard split API (#6845) * Fixes compilation for PG13.10 and PG14.7 (#6711) * Fixes crash that happens when trying to replicate a reference table that is actually dropped (#6595) * Fixes memory leak issue with query results that returns single row (#6724) * Fixes the modifiers for subscription and role creation (#6603) * Makes sure to quote all identifiers used for logical replication to prevent potential issues (#6604) * Makes sure to skip foreign key validations at the end of shard moves (#6640) ### citus v11.0.8 (April 20, 2023) ### * Correctly reports shard size in `citus_shards` view (#6748) * Fixes a bug that breaks pg upgrades if the user has a columnar table (#6624) * Fixes an unexpected foreign table error by disallowing to drop the `table_name` option (#6669) * Fixes compilation warning on PG13 + OpenSSL 3.0 (#6038, #6502) * Fixes crash that happens when trying to replicate a reference table that is actually dropped (#6595) * Fixes memory leak issue with query results that returns single row (#6724) * Fixes the modifiers for subscription and role creation (#6603) * Fixes two potential dangling pointer issues (#6504, #6507) * Makes sure to quote all identifiers used for logical replication to prevent potential issues (#6604) ### citus v10.2.9 (April 20, 2023) ### * Correctly reports shard size in `citus_shards` view (#6748) * Fixes a bug in `ALTER EXTENSION citus UPDATE` (#6383) * Fixes a bug that breaks pg upgrades if the user has a columnar table (#6624) * Fixes a bug that prevents retaining columnar table options after a table-rewrite (#6337) * Fixes memory leak issue with query results that returns single row (#6724) * Raises memory limits in columnar from 256MB to 1GB for reads and writes (#6419) ### citus v10.1.6 (April 20, 2023) ### * Fixes a crash that occurs when the aggregate that cannot be pushed-down returns empty result from a worker (#5679) * Fixes columnar freezing/wraparound bug (#5962) * Fixes memory leak issue with query results that returns single row (#6724) * Prevents alter table functions from dropping extensions (#5974) ### citus v10.0.8 (April 20, 2023) ### * Fixes a bug that could break `DROP SCHEMA/EXTENSON` commands when there is a columnar table (#5458) * Fixes a crash that occurs when the aggregate that cannot be pushed-down returns empty result from a worker (#5679) * Fixes columnar freezing/wraparound bug (#5962) * Fixes memory leak issue with query results that returns single row (#6724) * Prevents alter table functions from dropping extensions (#5974) ### citus v9.5.12 (April 20, 2023) ### * Fixes a crash that occurs when the aggregate that cannot be pushed-down returns empty result from a worker (#5679) * Fixes memory leak issue with query results that returns single row (#6724) * Prevents alter table functions from dropping extensions (#5974) ### citus v11.2.0 (January 30, 2023) ### * Adds support for outer joins with reference tables / complex subquery-CTEs in the outer side of the join (e.g., \ LEFT JOIN \) * Adds support for creating `PRIMARY KEY`s and `UNIQUE`/`EXCLUSION`/`CHECK`/ `FOREIGN KEY` constraints via `ALTER TABLE` command without providing a constraint name * Adds support for using identity columns on Citus tables * Adds support for `MERGE` command on local tables * Adds `citus_job_list()`, `citus_job_status()` and `citus_rebalance_status()` UDFs that allow monitoring rebalancer progress * Adds `citus_task_wait()` UDF to wait on desired task status * Adds `source_lsn`, `target_lsn` and `status` fields into `get_rebalance_progress()` * Introduces `citus_copy_shard_placement()` UDF with node id * Introduces `citus_move_shard_placement()` UDF with node id * Propagates `BEGIN` properties to worker nodes * Propagates `DROP OWNED BY` to worker nodes * Deprecates `citus.replicate_reference_tables_on_activate` and makes it always `off` * Drops GUC `citus.defer_drop_after_shard_move` * Drops GUC `citus.defer_drop_after_shard_split` * Drops `SHARD_STATE_TO_DELETE` state and uses the cleanup records instead * Allows `citus_update_node()` to work with nodes from different clusters * Adds signal handlers for queue monitor to gracefully shutdown, cancel and to see config changes * Defers cleanup after a failure in shard move or split * Extends cleanup process for replication artifacts * Improves a query that terminates compelling backends from `citus_update_node()` * Includes Citus global pid in all internal `application_name`s * Avoids leaking `search_path` to workers when executing DDL commands * Fixes `alter_table_set_access_method error()` for views * Fixes `citus_drain_node()` to allow draining the specified worker only * Fixes a bug in global pid assignment for connections opened by rebalancer internally * Fixes a bug that causes background rebalancer to fail when a reference table doesn't have a primary key * Fixes a bug that might cause failing to query the views based on tables that have renamed columns * Fixes a bug that might cause incorrectly planning the sublinks in query tree * Fixes a floating point exception during `create_distributed_table_concurrently()` * Fixes a rebalancer failure due to integer overflow in subscription and role creation * Fixes a regression in allowed foreign keys on distributed tables * Fixes a use-after-free bug in connection management * Fixes an unexpected foreign table error by disallowing to drop the table_name option * Fixes an uninitialized memory access in `create_distributed_function()` * Fixes crash that happens when trying to replicate a reference table that is actually dropped * Make sure to cleanup the shard on the target node in case of a failed/aborted shard move * Makes sure to create replication artifacts with unique names * Makes sure to disallow triggers that depend on extensions * Makes sure to quote all identifiers used for logical replication to prevent potential issues * Makes sure to skip foreign key validations at the end of shard moves * Prevents crashes on `UPDATE` with certain `RETURNING` clauses * Propagates column aliases in the shard-level commands ### citus v11.1.5 (December 12, 2022) ### * Fixes two potential dangling pointer issues * Fixes compilation warning on PG13 + OpenSSL 3.0 ### citus v11.0.7 (November 8, 2022) ### * Adds the GUC `citus.allow_unsafe_constraints` to allow unique/exclusion/ primary key constraints without distribution column * Allows `citus_internal` `application_name` with additional suffix * Disallows having `ON DELETE/UPDATE SET DEFAULT` actions on columns that default to sequences * Fixes a bug in `ALTER EXTENSION citus UPDATE` * Fixes a bug that causes a crash with empty/null password * Fixes a bug that causes not retaining trigger enable/disable settings when re-creating them on shards * Fixes a bug that might cause inserting incorrect `DEFAULT` values when applying foreign key actions * Fixes a bug that prevents retaining columnar table options after a table-rewrite * Fixes a bug that prevents setting colocation group of a partitioned distributed table to `none` * Fixes an issue that can cause logical reference table replication to fail * Raises memory limits in columnar from 256MB to 1GB for reads and writes ### citus v11.1.4 (October 24, 2022) ### * Fixes an upgrade problem for `worker_fetch_foreign_file` when upgrade path starts from 8.3 up to 11.1 * Fixes an upgrade problem for `worker_repartition_cleanup` when upgrade path starts from 9.1 up to 11.1 ### citus v11.1.3 (October 14, 2022) ### * Adds support for PostgreSQL 15.0 * Fixes a bug in `ALTER EXTENSION citus UPDATE` * Fixes a bug that causes a crash with empty/null password * Fixes a bug that causes not retaining trigger enable/disable settings when re-creating them on shards * Fixes a bug that prevents retaining columnar table options after a table-rewrite * Raises memory limits in columnar from 256MB to 1GB for reads and writes ### citus v11.1.2 (September 30, 2022) ### * Adds support for PostgreSQL 15rc1 * Disallows having `ON DELETE/UPDATE SET DEFAULT` actions on columns that default to sequences * Fixes a bug that might cause inserting incorrect `DEFAULT` values when applying foreign key actions * Fixes a performance issue related to shard-moves by creating replica identities before copying shards * Improves logging during shard-splits and resource cleanup * Makes sure to reuse connections for shard-splits and logical replication * Makes sure to try dropping replication slots a few more times after a failure at the end of the shard-split ### citus v11.1.1 (September 16, 2022) ### * Fixes a bug that prevents `create_distributed_table_concurrently()` working on an empty node ### citus v11.1.0 (September 15, 2022) ### * Adds support for PostgreSQL 15beta4 * Adds ability to run shard rebalancer in the background * Adds `create_distributed_table_concurrently()` UDF to distribute tables without interrupting the application * Adds `citus_split_shard_by_split_points()` UDF that allows splitting a shard to specified set of nodes without blocking writes and based on given split points * Adds support for non-blocking tenant isolation * Adds support for isolation tenants that use partitioned tables or columnar tables * Separates columnar table access method into a separate logical extension * Adds support for online replication in `replicate_reference_tables()` * Improves performance of blocking shard moves * Improves non-blocking shard moves with a faster custom copy logic * Creates all foreign keys quickly at the end of a shard move * Limits `get_rebalance_progress()` to show shards in moving state * Makes `citus_move_shard_placement()` idempotent if shard already exists on target node * Shows `citus_copy_shard_placement()` progress in `get_rebalance_progres()` * Supports changing CPU priorities for backends and shard moves * Adds the GUC `citus.allow_unsafe_constraints` to allow unique/exclusion/ primary key constraints without distribution column * Introduces GUC `citus.skip_constraint_validation` * Introduces `citus_locks` view * Improves `citus_tables` view by showing local tables added to metadata * Improves columnar table access method by moving old catalog tables into an internal schema and introduces more secure & informative views based on them * Adds support for `GRANT/REVOKE` on aggregates * Adds support for `NULLS NOT DISTINCT` clauses for indexes for PG15+ * Adds support for setting relation options for columnar tables using `ALTER TABLE` * Adds support for unlogged distributed sequences * Removes `do_repair` option from `citus_copy_shard_placement()` * Removes deprecated re-partitioning functions like `worker_hash_partition_table()` * Drops support for isolation tenants that use replicated tables * Checks existence of the shards before insert, delete, and update * Hides tables owned by extensions from `citus_tables` and `citus_shards` * Propagates `VACUUM` and `ANALYZE` to worker nodes * Makes non-partitioned table size calculation quicker * Improves `create_distributed_table()` by creating new colocation entries when using `colocate_with => 'none'` * Ensures that `SELECT .. FOR UPDATE` opens a transaction block when used in a function call * Prevents a segfault by disallowing usage of SQL functions referencing to a distributed table * Prevents creating a new colocation entry when replicating reference tables * Fixes a bug in query escaping in `undistribute_table()` and `alter_distributed_table()` * Fixes a bug preventing the usage of `isolate_tenant_to_new_shard()` with text column * Fixes a bug that may cause `GRANT` to propagate within `CREATE EXTENSION` * Fixes a bug that causes incorrectly marking `metadatasynced` flag for coordinator * Fixes a bug that may prevent Citus from creating function in transaction block properly * Fixes a bug that prevents promoting read-replicas as primaries * Fixes a bug that prevents setting colocation group of a partitioned distributed table to `none` * Fixes a bug that prevents using `AUTO` option for `VACUUM (INDEX_CLEANUP)` operation * Fixes a segfault in `citus_copy_shard_placement()` * Fixes an issue that can cause logical reference table replication to fail * Fixes schema name qualification for `RENAME SEQUENCE` statement * Fixes several small memory leaks * Fixes the transaction timestamp column of the `get_current_transaction_id()` on coordinator * Maps any unused parameters to a generic type in prepared statements ### citus v10.2.8 (August 19, 2022) ### * Fixes compilation warning caused by latest upgrade script changes * Fixes compilation warning on PG13 + OpenSSL 3.0 ### citus v11.0.6 (August 19, 2022) ### * Fixes a bug that could cause failures in `CREATE ROLE` statement * Fixes a bug that could cause failures in `create_distributed_table` * Fixes a bug that prevents distributing tables that depend on sequences * Fixes reference table lock contention * Fixes upgrade paths for 11.0 ### citus v10.2.7 (August 19, 2022) ### * Fixes a bug that could cause failures in `INSERT INTO .. SELECT` * Fixes a bug that could cause leaking files when materialized views are refreshed * Fixes an unexpected error for foreign tables when upgrading Postgres * Fixes columnar freezing/wraparound bug * Fixes reference table lock contention * Prevents alter table functions from dropping extensions ### citus v11.0.5 (August 1, 2022) ### * Avoids possible information leakage about existing users * Allows using `WITH HOLD` cursors with parameters * Fixes a bug that could cause failures in `INSERT INTO .. SELECT` * Fixes a bug that prevents pushing down `IN` expressions when using columnar custom scan * Fixes a concurrency bug between creating a co-located distributed table and shard moves * Fixes a crash that can happen due to catalog read in `shmem_exit` * Fixes an unexpected error caused by constraints when moving shards * Fixes an unexpected error for foreign tables when upgrading Postgres * Prevents adding local table into metadata if there is a view with circular dependencies on it * Reduces memory consumption of index name fix for partitioned tables ### citus v11.0.4 (July 13, 2022) ### * Fixes a bug that prevents promoting read-replicas as primaries ### citus v11.0.3 (July 5, 2022) ### * Fixes a bug that prevents adding local tables with materialized views to Citus metadata * Fixes a bug that prevents using `COMPRESSION` and `CONSTRAINT` on a column * Fixes upgrades to Citus 11 when there are no nodes in the metadata ### citus v11.0.2 (June 15, 2022) ### * Drops support for PostgreSQL 12 * Open sources enterprise features, see the rest of changelog items * Turns metadata syncing on by default * Introduces `citus_finish_citus_upgrade()` procedure which is necessary to upgrade from earlier versions * Open sources non-blocking shard moves/shard rebalancer (`citus.logical_replication_timeout`) * Open sources propagation of `CREATE/DROP/ALTER ROLE` statements * Open sources propagation of `GRANT` statements * Open sources propagation of `CLUSTER` statements * Open sources propagation of `ALTER DATABASE ... OWNER TO ...` * Open sources optimization for `COPY` when loading `JSON` to avoid double parsing of the `JSON` object (`citus.skip_jsonb_validation_in_copy`) * Open sources support for row level security * Open sources support for `pg_dist_authinfo`, which allows storing different authentication options for different users, e.g. you can store passwords or certificates here. * Open sources support for `pg_dist_poolinfo`, which allows using connection poolers in between coordinator and workers * Open sources tracking distributed query execution times using citus_stat_statements (`citus.stat_statements_max`, `citus.stat_statements_purge_interval`, `citus.stat_statements_track`). This is disabled by default. * Open sources tenant_isolation * Open sources support for `sslkey` and `sslcert` in `citus.node_conninfo` * Adds `citus.max_client_connections` GUC to limit non-Citus connections * Allows locally creating objects having a dependency that cannot be distributed * Distributes aggregates with `CREATE AGGREGATE` command * Distributes functions with `CREATE FUNCTION` command * Adds `citus.create_object_propagation` GUC to control DDL creation behaviour in transactions * Hides shards based on `application_name` prefix * Prevents specifying `application_name` via `citus.node_conninfo` * Starts identifying rebalancer backends by `application_name=citus_rebalancer` * Starts identifying internal backends by `application_name=citus_internal` * Adds `citus.enable_unsafe_triggers` flag to enable unsafe triggers on distributed tables * Adds `fix_partition_shard_index_names` UDF to fix currently broken names * Adds propagation for foreign server commands * Adds propagation of `DOMAIN` objects * Adds propagation of `TEXT SEARCH CONFIGURATION` objects * Adds propagation of `TEXT SEARCH DICTIONARY` objects * Adds support for `ALTER FUNCTION ... SUPPORT ...` commands * Adds support for `CREATE SCHEMA AUTHORIZATION` statements without schema name * Adds support for `CREATE/DROP/ALTER VIEW` commands * Adds support for `TRUNCATE` for foreign tables * Adds support for adding local tables to metadata using `citus_add_local_table_to_metadata()` UDF * Adds support for adding partitioned local tables to Citus metadata * Adds support for automatic binary encoding in re-partition joins when possible * Adds support for foreign tables in MX * Adds support for operator class parameters in indexes * Adds support for re-partition joins in transaction blocks * Adds support for re-partition joins on followers * Adds support for shard replication > 1 hash distributed tables on Citus MX * Adds support for `LOCK` commands on distributed tables from worker nodes * Adds support for `TABLESAMPLE` * Adds support for propagating views when syncing Citus table metadata * Improves handling of `IN`, `OUT` and `INOUT` parameters for functions * Introduces `citus_backend_gpid()` UDF to get global pid of the current backend * Introduces `citus_check_cluster_node_health` UDF to check cluster connectivity * Introduces `citus_check_connection_to_node` UDF to check node connectivity * Introduces `citus_coordinator_nodeid` UDF to find the node id of the coordinator node * Introduces `citus_stat_activity` view and drops `citus_worker_stat_activity` UDF * Introduces `citus.use_citus_managed_tables` GUC to add local tables to Citus metadata automatically * Introduces a new flag `force_delegation` in `create_distributed_function()` * Introduces `run_command_on_coordinator` UDF * Introduces `synchronous` option to `citus_disable_node()` UDF * Introduces `citus_is_coordinator` UDF to check whether a node is the coordinator * Allows adding a unique constraint with an index * Allows `create_distributed_function()` on a function owned by an extension * Allows creating distributed tables in sequential mode * Allows disabling nodes when multiple failures happen * Allows `lock_table_if_exits` to be called outside of a transaction blocks * Adds support for pushing procedures with `OUT` arguments down to the worker nodes * Overrides `pg_cancel_backend()` and `pg_terminate_backend()` to run with global pid * Delegates function calls of the form `SELECT .. FROM func()` * Adds propagation of `CREATE SCHEMA .. GRANT ON SCHEMA ..` commands * Propagates `pg_dist_object` to worker nodes * Adds propagation of `SCHEMA` operations * Adds missing version-mismatch checks for columnar tables * Adds missing version-mismatch checks for internal functions * `citus_shard_indexes_on_worker` shows all local shard indexes regardless of `search_path` * `citus_shards_on_worker` shows all local shards regardless of `search_path` * Enables distributed execution from `run_command_on_*` functions * Deprecates inactive shard state, never marks any placement inactive * Disables distributed & reference foreign tables * Prevents propagating objects having a circular dependency * Prevents propagating objects having a dependency to an object with unsupported type * Deprecates `master_get_table_metadata` UDF * Disallows remote execution from queries on shards * Drops `citus.enable_cte_inlining` GUC * Drops `citus.single_shard_commit_protocol` GUC, defaults to 2PC * Drops support for `citus.multi_shard_commit_protocol`, always use 2PC * Avoids unnecessary errors for `ALTER STATISTICS IF EXISTS` when the statistics does not exist * Fixes a bug that prevents dropping/altering indexes * Fixes a bug that prevents non-client backends from accessing shards * Fixes columnar freezing/wraparound bug * Fixes `invalid read of size 1` memory error with `citus_add_node` * Fixes schema name qualification for `ALTER/DROP SEQUENCE` * Fixes schema name qualification for `ALTER/DROP STATISTICS` * Fixes schema name qualification for `CREATE STATISTICS` * Fixes a bug that causes columnar storage pages to have zero LSN * Fixes a bug that causes issues while create dependencies from multiple sessions * Fixes a bug that causes reading columnar metapage as all-zeros when writing to a columnar table * Fixes a bug that could break `DROP SCHEMA/EXTENSON` commands when there is a columnar table * Fixes a bug that could break pg upgrades due to missing `pg_depend` records for columnar table access method * Fixes a bug that could cause `CREATE INDEX` to fail for expressions when using custom `search_path` * Fixes a bug that could cause `worker_save_query_explain_analyze` to fail on custom types * Fixes a bug that could cause failed re-partition joins to leak result tables * Fixes a bug that could cause prerequisite columnar table access method objects being not created during pg upgrades * Fixes a bug that could cause re-partition joins involving local shards to fail * Fixes a bug that could cause false positive distributed deadlocks due to local execution * Fixes a bug that could cause leaking files when materialized views are refreshed * Fixes a bug that could cause unqualified `DROP DOMAIN IF EXISTS` to fail * Fixes a bug that could cause wrong schema and ownership after `alter_distributed_table` * Fixes a bug that could cause `EXPLAIN ANALYZE` to fail for prepared statements with custom type * Fixes a bug that could cause Citus not to create function in transaction block properly * Fixes a bug that could cause returning invalid JSON when running `EXPLAIN ANALYZE` with subplans * Fixes a bug that limits usage of sequences in non-int columns * Fixes a bug that prevents `DROP SCHEMA CASCADE` * Fixes a build error that happens when `lz4` is not installed * Fixes a clog lookup failure that could occur when writing to a columnar table * Fixes a crash that occurs when the aggregate that cannot be pushed-down returns empty result from a worker * Fixes a missing `FROM` clause entry error * Fixes a possible segfault that could happen when reporting distributed deadlock * Fixes an issue that could cause unexpected errors when there is an in-progress write to a columnar table * Fixes an unexpected error that occurs due to aborted writes to a columnar table with an index * Fixes an unexpected error that occurs when writing to a columnar table created in older version * Fixes issue when compiling Citus from source with some compilers * Fixes issues on `ATTACH PARTITION` logic * Fixes naming issues of newly created partitioned indexes * Honors `enable_metadata_sync` in node operations * Improves nested execution checks and adds GUC to control (`citus.allow_nested_distributed_execution`) * Improves self-deadlock prevention for `CREATE INDEX / REINDEX CONCURRENTLY` commands for builds using PG14 or higher * Moves `pg_dist_object` to `pg_catalog` schema * Parallelizes metadata syncing on node activation * Partitions shards to be co-located with the parent shards * Prevents Citus table functions from being called on shards * Prevents creating distributed functions when there are out of sync nodes * Prevents alter table functions from dropping extensions * Refrains reading the metadata cache for all tables during upgrade * Provides notice message for idempotent `create_distributed_function` calls * Reinstates optimisation for uniform shard interval ranges * Relaxes table ownership check to privileges check while acquiring lock * Drops support for `citus.shard_placement_policy` GUC * Drops `master_append_table_to_shard` UDF * Drops `master_apply_delete_command` UDF * Removes copy into new shard logic for append-distributed tables * Drops support for distributed `cstore_fdw` tables in favor of Citus columnar table access method * Removes support for dropping distributed and local indexes in the same statement * Replaces `citus.enable_object_propagation` GUC with `citus.enable_metadata_sync` * Requires superuser for `citus_add_node()` and `citus_activate_node()` UDFs * Revokes read access to `columnar.chunk` from unprivileged user * Disallows unsupported lateral subqueries on distributed tables * Stops updating shard range in `citus_update_shard_statistics` for append tables ### citus v10.2.5 (March 15, 2022) ### * Fixes a bug that could cause `worker_save_query_explain_analyze` to fail on custom types * Fixes a bug that limits usage of sequences in non-integer columns * Fixes a crash that occurs when the aggregate that cannot be pushed-down returns empty result from a worker * Improves concurrent metadata syncing and metadata changing DDL operations ### citus v10.2.4 (February 1, 2022) ### * Adds support for operator class parameters in indexes * Fixes a bug with distributed functions that have `OUT` parameters or return `TABLE` * Fixes a build error that happens when `lz4` is not installed * Improves self-deadlock prevention for `CREATE INDEX` & `REINDEX CONCURRENTLY` commands for builds using PG14 or higher * Fixes a bug that causes commands to fail when `application_name` is set ### citus v10.1.4 (February 1, 2022) ### * Adds missing version checks for columnar tables * Fixes a bug that could break `DROP SCHEMA/EXTENSION` commands when there is a columnar table * Fixes a build error that happens when `lz4` is not installed * Fixes a missing `FROM` clause entry error * Reinstates optimisation for uniform shard interval ranges * Fixes a bug that causes commands to fail when `application_name` is set ### citus v10.2.3 (November 29, 2021) ### * Adds `fix_partition_shard_index_names` udf to fix currently broken partition index names * Fixes a bug that could break `DROP SCHEMA/EXTENSION` commands when there is a columnar table * Fixes a bug that could break pg upgrades due to missing `pg_depend` records for columnar table access method * Fixes a missing `FROM` clause entry error * Fixes an unexpected error that occurs when writing to a columnar table created in older versions * Fixes issue when compiling Citus from source with some compilers * Reinstates optimisation for uniform shard interval ranges * Relaxes table ownership check to privileges check while acquiring lock ### citus v10.0.6 (November 12, 2021) ### * Adds missing version checks for columnar tables * Fixes a bug that caused `worker_append_table_to_shard` to write as superuser * Fixes a bug with local cached plans on tables with dropped columns * Fixes a missing `FROM` clause entry error * Fixes a use after free issue that could happen when altering a distributed table * Reinstates optimisation for uniform shard interval ranges ### citus v9.5.10 (November 8, 2021) ### * Fixes a release problem in 9.5.9 ### citus v9.5.9 (November 8, 2021) ### * Fixes a bug preventing `INSERT SELECT .. ON CONFLICT` with a constraint name on local shards * Fixes a bug with local cached plans on tables with dropped columns * Fixes a crash in queries with a modifying `CTE` and a `SELECT` without `FROM` * Fixes a missing `FROM` clause entry error * Fixes a missing intermediate result when coordinator is in metadata * Reinstates optimisation for uniform shard interval ranges ### citus v9.2.8 (November 4, 2021) ### * Adds a configure flag to enforce security ### citus v9.2.7 (November 3, 2021) ### * Fixes `ALTER TABLE IF EXISTS SET SCHEMA` with non-existing table bug * Fixes `CREATE INDEX CONCURRENTLY` with no index name on a postgres table bug * Fixes a bug that could cause crashes with certain compile flags * Fixes a crash because of overflow in partition id with certain compile flags * Fixes a memory leak in subtransaction memory handling * Fixes deparsing for queries with anonymous column references ### citus v10.2.2 (October 14, 2021) ### * Fixes a bug that causes reading columnar metapage as all-zeros when writing to a columnar table * Fixes a bug that could cause prerequisite columnar table access method objects being not created during pg upgrades * Fixes a bug that could cause `CREATE INDEX` to fail for expressions when using custom `search_path` * Fixes an unexpected error that occurs due to aborted writes to a columnar table with an index ### citus v10.2.1 (September 24, 2021) ### * Adds missing version-mismatch checks for columnar tables * Adds missing version-mismatch checks for internal functions * Fixes a bug that could cause partition shards being not co-located with parent shards * Fixes a bug that prevents pushing down boolean expressions when using columnar custom scan * Fixes a clog lookup failure that could occur when writing to a columnar table * Fixes an issue that could cause unexpected errors when there is an in-progress write to a columnar table * Revokes read access to `columnar.chunk` from unprivileged user ### citus v10.1.3 (September 17, 2021) ### * Fixes a bug that caused `worker_append_table_to_shard` to write as superuser * Fixes a crash in shard rebalancer when no distributed tables exist * Fixes a use after free issue that could happen when altering a distributed table ### citus v9.5.8 (September 15, 2021) ### * Fixes a bug that caused `worker_append_table_to_shard` to write as superuser * Fixes a use after free issue that could happen when altering a distributed table ### citus v10.2.0 (September 14, 2021) ### * Adds PostgreSQL 14 support * Adds hash & btree index support for columnar tables * Adds helper UDFs for easy time partition management: `get_missing_time_partition_ranges`, `create_time_partitions`, and `drop_old_time_partitions` * Adds propagation of ALTER SEQUENCE * Adds support for ALTER INDEX ATTACH PARTITION * Adds support for CREATE INDEX ON ONLY * Allows more graceful failovers when replication factor > 1 * Enables chunk group filtering to work with Params for columnar tables * Enables qual push down for joins including columnar tables * Enables transferring of data using binary encoding by default on PG14 * Improves `master_update_table_statistics` and provides distributed deadlock detection * Includes `data_type` and `cache` in sequence definition on worker * Makes start/stop_metadata_sync_to_node() transactional * Makes sure that table exists before updating table statistics * Prevents errors with concurrent `citus_update_table_statistics` and DROP table * Reduces memory usage of columnar table scans by freeing the memory used for last stripe read * Shows projected columns for columnar tables in EXPLAIN output * Speeds up dropping partitioned tables * Synchronizes hasmetadata flag on mx workers * Uses current user while syncing metadata * Adds a parameter to cleanup metadata when metadata syncing is stopped * Fixes a bug about int and smallint sequences on MX * Fixes a bug that cause partitions to have wrong distribution key after DROP COLUMN * Fixes a bug that caused `worker_append_table_to_shard` to write as superuser * Fixes a bug that caused `worker_create_or_alter_role` to crash with NULL input * Fixes a bug that causes pruning incorrect shard of a range distributed table * Fixes a bug that may cause crash while aborting transaction * Fixes a bug that prevents attaching partitions when colocated foreign key exists * Fixes a bug with `nextval('seq_name'::text)` * Fixes a crash in shard rebalancer when no distributed tables exist * Fixes a segfault caused by use after free in when using a cached connection * Fixes a UNION pushdown issue * Fixes a use after free issue that could happen when altering a distributed table * Fixes showing target shard size in the rebalance progress monitor ### citus v10.1.2 (August 16, 2021) ### * Allows more graceful failovers when replication factor > 1 * Fixes a bug that causes partitions to have wrong distribution key after `DROP COLUMN` ### citus v10.0.5 (August 16, 2021) ### * Allows more graceful failovers when replication factor > 1 * Fixes a bug that causes partitions to have wrong distribution key after `DROP COLUMN` * Improves citus_update_table_statistics and provides distributed deadlock detection ### citus v9.5.7 (August 16, 2021) ### * Allows more graceful failovers when replication factor > 1 * Fixes a bug that causes partitions to have wrong distribution key after `DROP COLUMN` * Improves master_update_table_statistics and provides distributed deadlock detection ### citus v9.4.6 (August 8, 2021) ### * Allows more graceful failovers when replication factor > 1 * Improves master_update_table_statistics and provides distributed deadlock detection ### citus v10.1.1 (August 5, 2021) ### * Improves citus_update_table_statistics and provides distributed deadlock detection * Fixes showing target shard size in the rebalance progress monitor ### citus v10.1.0 (July 14, 2021) ### * Drops support for PostgreSQL 11 * Adds `shard_count` parameter to `create_distributed_table` function * Adds support for `ALTER DATABASE OWNER` * Adds support for temporary columnar tables * Adds support for using sequences as column default values when syncing metadata * `alter_columnar_table_set` enforces columnar table option constraints * Continues to remove shards after failure in `DropMarkedShards` * Deprecates the `citus.replication_model` GUC * Enables `citus.defer_drop_after_shard_move` by default * Ensures free disk space before moving a shard * Fetches shard size on the fly for the rebalance monitor * Ignores old placements when disabling or removing a node * Implements `improvement_threshold` at shard rebalancer moves * Improves orphaned shard cleanup logic * Improves performance of `citus_shards` * Introduces `citus.local_hostname` GUC for connections to the current node * Makes sure connection is closed after each shard move * Makes sure that target node in shard moves is eligible for shard move * Optimizes partitioned disk size calculation for shard rebalancer * Prevents connection errors by properly terminating connections * Prevents inheriting a distributed table * Prevents users from dropping & truncating known shards * Pushes down `VALUES` clause as long as not in outer part of a `JOIN` * Reduces memory usage for multi-row inserts * Reduces memory usage while rebalancing shards * Removes length limits around partition names * Removes dependencies on the existence of public schema * Executor avoids opening extra connections * Excludes orphaned shards while finding shard placements * Preserves access method of materialized views when undistributing or altering distributed tables * Fixes a bug that allowed moving of shards belonging to a reference table * Fixes a bug that can cause a crash when DEBUG4 logging is enabled * Fixes a bug that causes pruning incorrect shard of a range distributed table * Fixes a bug that causes worker_create_or_alter_role to crash with NULL input * Fixes a bug where foreign key to reference table was disallowed * Fixes a bug with local cached plans on tables with dropped columns * Fixes data race in `get_rebalance_progress` * Fixes `FROM ONLY` queries on partitioned tables * Fixes an issue that could cause citus_finish_pg_upgrade to fail * Fixes error message for local table joins * Fixes issues caused by omitting public schema in queries * Fixes nested `SELECT` query with `UNION` bug * Fixes null relationName bug at parallel execution * Fixes possible segfaults when using Citus in the middle of an upgrade * Fixes problems with concurrent calls of `DropMarkedShards` * Fixes shared dependencies that are not resident in a database * Fixes stale hostnames bug in prepared statements after `master_update_node` * Fixes the relation size bug during rebalancing * Fixes two race conditions in the get_rebalance_progress * Fixes using 2PC when it might be necessary ### citus v10.0.4 (July 14, 2021) ### * Introduces `citus.local_hostname` GUC for connections to the current node * Removes dependencies on the existence of public schema * Removes limits around long partition names * Fixes a bug that can cause a crash when DEBUG4 logging is enabled * Fixes a bug that causes pruning incorrect shard of a range distributed table * Fixes an issue that could cause citus_finish_pg_upgrade to fail * Fixes FROM ONLY queries on partitioned tables * Fixes issues caused by public schema being omitted in queries * Fixes problems with concurrent calls of DropMarkedShards * Fixes relname null bug when using parallel execution * Fixes two race conditions in the get_rebalance_progress ### citus v9.5.6 (July 8, 2021) ### * Fixes minor bug in `citus_prepare_pg_upgrade` that caused it to lose its idempotency ### citus v9.5.5 (July 7, 2021) ### * Adds a configure flag to enforce security * Fixes a bug that causes pruning incorrect shard of a range distributed table * Fixes an issue that could cause citus_finish_pg_upgrade to fail ### citus v9.4.5 (July 7, 2021) ### * Adds a configure flag to enforce security * Avoids re-using connections for intermediate results * Fixes a bug that causes pruning incorrect shard of a range distributed table * Fixes a bug that might cause self-deadlocks when COPY used in TX block * Fixes an issue that could cause citus_finish_pg_upgrade to fail ### citus v8.3.3 (March 23, 2021) ### * Fixes a bug that leads to various issues when a connection is lost * Fixes a bug where one could create a foreign key between non-colocated tables * Fixes a deadlock during transaction recovery * Fixes a memory leak in adaptive executor when query returns many columns ### citus v10.0.3 (March 16, 2021) ### * Prevents infinite recursion for queries that involve `UNION ALL` below `JOIN` * Fixes a crash in queries with a modifying `CTE` and a `SELECT` without `FROM` * Fixes upgrade and downgrade paths for `citus_update_table_statistics` * Fixes a bug that causes `SELECT` queries to use 2PC unnecessarily * Fixes a bug that might cause self-deadlocks with `CREATE INDEX` / `REINDEX CONCURRENTLY` commands * Adds `citus.max_cached_connection_lifetime` GUC to set maximum connection lifetime * Adds `citus.remote_copy_flush_threshold` GUC that controls per-shard memory usages by `COPY` * Adds `citus_get_active_worker_nodes` UDF to deprecate `master_get_active_worker_nodes` * Skips 2PC for readonly connections in a transaction * Makes sure that local execution starts coordinated transaction * Removes open temporary file warning when cancelling a query with an open tuple store * Relaxes the locks when adding an existing node ### citus v10.0.2 (March 3, 2021) ### * Adds a configure flag to enforce security * Fixes a bug due to cross join without target list * Fixes a bug with `UNION ALL` on PG 13 * Fixes a compatibility issue with pg_audit in utility calls * Fixes insert query with CTEs/sublinks/subqueries etc * Grants `SELECT` permission on `citus_tables` view to `public` * Grants `SELECT` permission on columnar metadata tables to `public` * Improves `citus_update_table_statistics` and provides distributed deadlock detection * Preserves colocation with procedures in `alter_distributed_table` * Prevents using `alter_columnar_table_set` and `alter_columnar_table_reset` on a columnar table not owned by the user * Removes limits around long table names ### citus v9.5.4 (February 19, 2021) ### * Fixes a compatibility issue with pg_audit in utility calls ### citus v10.0.1 (February 19, 2021) ### * Fixes an issue in creation of `pg_catalog.time_partitions` view ### citus v9.5.3 (February 16, 2021) ### * Avoids re-using connections for intermediate results * Fixes a bug that might cause self-deadlocks when `COPY` used in xact block * Fixes a crash that occurs when distributing table after dropping foreign key ### citus v10.0.0 (February 16, 2021) ### * Adds support for per-table option for columnar storage * Adds `rebalance_table_shards` to rebalance the shards across the nodes * Adds `citus_drain_node` to move all shards away from any node * Enables single-node Citus for production workloads * Adds support for local table and distributed table/subquery joins * Enables foreign keys between reference tables and local tables (`citus.enable_local_reference_table_foreign_keys`) * Adds support for co-located/recurring correlated subqueries * Adds support for co-located/recurring sublinks in the target list * Adds `alter_distributed_table` and `alter_table_set_access_method` UDFs * Adds `alter_old_partitions_set_access_method` UDF to compress old partitions * Adds cascade option to `undistribute_table` UDF for foreign key connected relations * Allows distributed table creation immediately after CREATE EXTENSION citus * Automatically adds coordinator to the metadata before the first distributed table created * Introduces adaptive connection management for local nodes * Adds support for local execution in `INSERT..SELECT` commands that perform re-partitioning * Adds `public.citus_tables` view * Adds `pg_catalog.citus_shards`, `pg_catalog.citus_shards_on_worker` and `pg_catalog.citus_shard_indexes_on_worker` views * Adds `pg_catalog.time_partitions` view for simple (time) partitions and their access methods * Adds `remove_local_tables_from_metadata` UDF for local tables automatically added to metadata when defining foreign keys with reference tables * Adds support for `CREATE INDEX` commands without index name on citus tables * Adds support for `CREATE TABLE ... USING table_access_method` statements * Propagates `ALTER TABLE .. SET LOGGED/UNLOGGED` statements * Propagates `ALTER INDEX .. SET STATISTICS` statements * Propagates `ALTER SCHEMA RENAME` statements * Propagates `CREATE STATISTICS` statements across workers. * Propagates `DROP STATISTICS` statements across the workers * Propagates all types of `ALTER STATISTICS` statements. * Avoids establishing new connections until the previous ones are finalized * Avoids re-using connections for intermediate results * Improves performance in some subquery pushdown cases * Removes unused `citus.binary_master_copy_format` GUC * Removes unused `citus.expire_cached_shards` GUC * Removes unused `citus.large_table_shard_count` GUC * Removes unused `citus.max_assign_task_batch_size` GUC * Removes unused `citus.max_running_tasks_per_node` GUC * Removes unused `citus.max_task_string_size` GUC * Removes unused `citus.max_tracked_tasks_per_node` GUC * Removes unused `citus.sslmode` GUC * Removes unused `citus.task_tracker_delay` GUC * Removes the word 'master' from Citus UDFs * Removes unused UDF `mark_tables_colocated` * Removes `upgrade_to_reference_table` UDF * Renames 'master' to 'distributed' for columns of `citus_dist_stat_activity` * Renames 'master' to 'distributed' for columns of `citus_worker_stat_activity` * Chooses the default co-location group deterministically * Deletes distributed transaction entries when removing a node * Drops support for `create_citus_local_table` in favor of `citus_add_local_table_to_metadata` * Fixes `EXPLAIN ANALYZE` error when query returns no columns * Fixes a bug preventing `INSERT SELECT .. ON CONFLICT` with a constraint name on local shards * Fixes a bug related to correctness of multiple joins * Fixes a bug that might cause self-deadlocks when `COPY` used in xact block * Fixes a bug with partitioned tables that block new partition creation due to wrong constraint names on workers * Fixes a crash that occurs when distributing table after dropping foreign key * Fixes a crash when adding/dropping foreign keys from reference to local tables added to metadata * Fixes an unexpected error when executing `CREATE TABLE IF NOT EXISTS PARTITION OF` commands * Fixes deadlock issue for `CREATE INDEX` in single node * Fixes distributed deadlock detection being blocked by metadata sync * Fixes handling indexes when adding local table to metadata * Fixes `undistribute_table` when table has column with `GENERATED ALWAYS AS STORED` expressions * Improves concurrent index creation failure message * Prevents empty placement creation in the coordinator when executing `master_create_empty_shard()` ### citus v9.5.2 (January 26, 2021) ### * Fixes distributed deadlock detection being blocked by metadata sync * Prevents segfaults when SAVEPOINT handling cannot recover from connection failures * Fixes possible issues that might occur with single shard distributed tables ### citus v9.4.4 (December 28, 2020) ### * Fixes a bug that could cause router queries with local tables to be pushed down * Fixes a segfault in connection management due to invalid connection hash entries * Fixes possible issues that might occur with single shard distributed tables ### citus v9.5.1 (December 1, 2020) ### * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE * Fixes a bug that could cause excessive memory consumption when a partition is created * Fixes a bug that triggers subplan executions unnecessarily with cursors * Fixes a segfault in connection management due to invalid connection hash entries ### citus v9.4.3 (November 24, 2020) ### * Enables PostgreSQL's parallel queries on EXPLAIN ANALYZE * Fixes a bug that triggers subplan executions unnecessarily with cursors ### citus v9.5.0 (November 10, 2020) ### * Adds support for PostgreSQL 13 * Removes the task-tracker executor * Introduces citus local tables * Introduces `undistribute_table` UDF to convert tables back to postgres tables * Adds support for `EXPLAIN (ANALYZE) EXECUTE` and `EXPLAIN EXECUTE` * Adds support for `EXPLAIN (ANALYZE, WAL)` for PG13 * Sorts the output of `EXPLAIN (ANALYZE)` by execution duration. * Adds support for CREATE TABLE ... USING table_access_method * Adds support for `WITH TIES` option in SELECT and INSERT SELECT queries * Avoids taking multi-shard locks on workers * Enforces `citus.max_shared_pool_size` config in COPY queries * Enables custom aggregates with multiple parameters to be executed on workers * Enforces `citus.max_intermediate_result_size` in local execution * Improves cost estimation of INSERT SELECT plans * Introduces delegation of procedures that read from reference tables * Prevents pull-push execution for simple pushdownable subqueries * Improves error message when creating a foreign key to a local table * Makes `citus_prepare_pg_upgrade` idempotent by dropping transition tables * Disallows `ON TRUE` outer joins with reference & distributed tables when reference table is outer relation to avoid incorrect results * Disallows field indirection in INSERT/UPDATE queries to avoid incorrect results * Disallows volatile functions in UPDATE subqueries to avoid incorrect results * Fixes CREATE INDEX CONCURRENTLY crash with local execution * Fixes `citus_finish_pg_upgrade` to drop all backup tables * Fixes a bug that cause failures when `RECURSIVE VIEW` joined reference table * Fixes DROP SEQUENCE failures when metadata syncing is enabled * Fixes a bug that caused CREATE TABLE with CHECK constraint to fail * Fixes a bug that could cause VACUUM to deadlock * Fixes master_update_node failure when no background worker slots are available * Fixes a bug that caused replica identity to not be propagated on shard repair * Fixes a bug that could cause crashes after connection timeouts * Fixes a bug that could cause crashes with certain compile flags * Fixes a bug that could cause deadlocks on CREATE INDEX * Fixes a bug with genetic query optimization in outer joins * Fixes a crash when aggregating empty tables * Fixes a crash with inserting domain constrained composite types * Fixes a crash with multi-row & router INSERT's in local execution * Fixes a possibility of doing temporary file cleanup more than once * Fixes incorrect setting of join related fields * Fixes memory issues around deparsing index commands * Fixes reference table access tracking for sequential execution * Fixes removal of a single node with only reference tables * Fixes sending commands to coordinator when it is added as a worker * Fixes write queries with const expressions and COLLATE in various places * Fixes wrong cancellation message about distributed deadlock ### citus v9.4.2 (October 21, 2020) ### * Fixes a bug that could lead to multiple maintenance daemons * Fixes an issue preventing views in reference table modifications ### citus v9.4.1 (September 30, 2020) ### * Fixes EXPLAIN ANALYZE output truncation * Fixes a deadlock during transaction recovery ### citus v9.4.0 (July 28, 2020) ### * Improves COPY by honoring max_adaptive_executor_pool_size config * Adds support for insert into local table select from distributed table * Adds support to partially push down tdigest aggregates * Adds support for receiving binary encoded results from workers using citus.enable_binary_protocol * Enables joins between local tables and CTEs * Adds showing query text in EXPLAIN output when explain verbose is true * Adds support for showing CTE statistics in EXPLAIN ANALYZE * Adds support for showing amount of data received in EXPLAIN ANALYZE * Introduces downgrade paths in migration scripts * Avoids returning incorrect results when changing roles in a transaction * Fixes `ALTER TABLE IF EXISTS SET SCHEMA` with non-existing table bug * Fixes `CREATE INDEX CONCURRENTLY` with no index name on a postgres table bug * Fixes a bug that could cause crashes with certain compile flags * Fixes a bug with lists of configuration values in ALTER ROLE SET statements * Fixes a bug that occurs when coordinator is added as a worker node * Fixes a crash because of overflow in partition id with certain compile flags * Fixes a crash that may happen if no worker nodes are added * Fixes a crash that occurs when inserting implicitly coerced constants * Fixes a crash when aggregating empty tables * Fixes a memory leak in subtransaction memory handling * Fixes crash when using rollback to savepoint after cancellation of DML * Fixes deparsing for queries with anonymous column references * Fixes distribution of composite types failing to include typemods * Fixes explain analyze on adaptive executor repartitions * Fixes possible error throwing in abort handle * Fixes segfault when evaluating func calls with default params on coordinator * Fixes several EXPLAIN ANALYZE issues * Fixes write queries with const expressions and COLLATE in various places * Fixes wrong cancellation message about distributed deadlocks * Reports correct INSERT/SELECT method in EXPLAIN * Disallows triggers on citus tables ### citus v9.3.5 (July 24, 2020) ### * Fixes `ALTER TABLE IF EXISTS SET SCHEMA` with non-existing table bug * Fixes `CREATE INDEX CONCURRENTLY` with no index name on a postgres table bug * Fixes a crash because of overflow in partition id with certain compile flags ### citus v9.3.4 (July 21, 2020) ### * Fixes a bug that could cause crashes with certain compile flags * Fixes a bug with lists of configuration values in ALTER ROLE SET statements * Fixes deparsing for queries with anonymous column references ### citus v9.3.3 (July 10, 2020) ### * Fixes a memory leak in subtransaction memory handling ### citus v9.3.2 (Jun 22, 2020) ### * Fixes a version bump issue in 9.3.1 ### citus v9.2.6 (Jun 22, 2020) ### * Fixes a version bump issue in 9.2.5 ### citus v9.3.1 (Jun 17, 2020) ### * Adds support to partially push down tdigest aggregates * Fixes a crash that occurs when inserting implicitly coerced constants ### citus v9.2.5 (Jun 17, 2020) ### * Adds support to partially push down tdigest aggregates * Fixes an issue with distributing tables having generated cols not at the end ### citus v9.3.0 (May 6, 2020) ### * Adds `max_shared_pool_size` to control number of connections across sessions * Adds support for window functions on coordinator * Improves shard pruning logic to understand OR-conditions * Prevents using an extra connection for intermediate result multi-casts * Adds propagation of `ALTER ROLE .. SET` statements * Adds `update_distributed_table_colocation` UDF to update colocation of tables * Introduces a UDF to truncate local data after distributing a table * Adds support for creating temp schemas in parallel * Adds support for evaluation of `nextval` in the target list on coordinator * Adds support for local execution of `COPY/TRUNCATE/DROP/DDL` commands * Adds support for local execution of shard creation * Uses local execution in a transaction block * Adds support for querying distributed table sizes concurrently * Allows `master_copy_shard_placement` to replicate placements to new nodes * Allows table type to be used in target list * Avoids having multiple maintenance daemons active for a single database * Defers reference table replication to shard creation time * Enables joins between local tables and reference tables in transaction blocks * Ignores pruned target list entries in coordinator plan * Improves `SIGTERM` handling of maintenance daemon * Increases the default of `citus.node_connection_timeout` to 30 seconds * Fixes a bug that occurs when creating remote tasks in local execution * Fixes a bug that causes some DML queries containing aggregates to fail * Fixes a bug that could cause failures in queries with subqueries or CTEs * Fixes a bug that may cause some connection failures to throw errors * Fixes a bug which caused queries with SRFs and function evalution to fail * Fixes a bug with generated columns when executing `COPY dist_table TO file` * Fixes a crash when using non-constant limit clauses * Fixes a failure when composite types used in prepared statements * Fixes a possible segfault when dropping dist. table in a transaction block * Fixes a possible segfault when non-pushdownable aggs are solely used in HAVING * Fixes a segfault when executing queries using `GROUPING` * Fixes an error when using `LEFT JOIN with GROUP BY` on primary key * Fixes an issue with distributing tables having generated cols not at the end * Fixes automatic SSL permission issue when using "initdb --allow-group-access" * Fixes errors which could occur when subqueries are parameters to aggregates * Fixes possible issues by invalidating the plan cache in `master_update_node` * Fixes timing issues which could be caused by changing system clock ### citus v9.2.4 (March 30, 2020) ### * Fixes a release problem in 9.2.3 ### citus v9.2.3 (March 25, 2020) ### * Do not use C functions that have been banned by Microsoft * Fixes a bug that causes wrong results with complex outer joins * Fixes issues found using static analysis * Fixes left join shard pruning in pushdown planner * Fixes possibility for segmentation fault in internal aggregate functions * Fixes possible segfault when non pushdownable aggregates are used in `HAVING` * Improves correctness of planning subqueries in `HAVING` * Prevents using old connections for security if `citus.node_conninfo` changed * Uses Microsoft approved cipher string for default TLS setup ### citus v9.0.2 (March 6, 2020) ### * Fixes build errors on EL/OL 6 based distros * Fixes a bug that caused maintenance daemon to fail on standby nodes * Disallows distributed function creation when replication_model is `statement` ### citus v9.2.2 (March 5, 2020) ### * Fixes a bug that caused some prepared stmts with function calls to fail * Fixes a bug that caused some prepared stmts with composite types to fail * Fixes a bug that caused missing subplan results in workers * Improves performance of re-partition joins ### citus v9.2.1 (February 14, 2020) ### * Fixes a bug that could cause crashes if distribution key is NULL ### citus v9.2.0 (February 10, 2020) ### * Adds support for `INSERT...SELECT` queries with re-partitioning * Adds `citus.coordinator_aggregation_strategy` to support more aggregates * Adds caching of local plans on shards for Citus MX * Adds compatibility support for dist. object infrastructure from old versions * Adds defering shard-pruning for fast-path router queries to execution * Adds propagation of `GRANT ... ON SCHEMA` queries * Adds support for CTE pushdown via CTE inlining in distributed planning * Adds support for `ALTER TABLE ... SET SCHEMA` propagation. * Adds support for `DROP ROUTINE` & `ALTER ROUTINE` commands * Adds support for any inner join on a reference table * Changes `citus.log_remote_commands` level to `NOTICE` * Disallows marking ref. table shards unhealthy in the presence of savepoints * Disallows placing new shards with shards in TO_DELETE state * Enables local execution of queries that do not need any data access * Fixes Makefile trying to cleanup PG directory during install * Fixes a bug causing errors when planning a query with multiple subqueries * Fixes a possible deadlock that could happen during shard moves * Fixes a problem when adding a new node due to tables referenced in func body * Fixes an issue that could cause joins with reference tables to be slow * Fixes cached metadata for shard is inconsistent issue * Fixes inserting multiple composite types as partition key in VALUES * Fixes unnecessary repartition on joins with more than 4 tables * Prevents wrong results for replicated partitioned tables after failure * Restricts LIMIT approximation for non-commutative aggregates ### citus v9.1.2 (December 30, 2019) ### * Fixes a bug that prevents installation from source ### citus v9.1.1 (December 18, 2019) ### * Fixes a bug causing SQL-executing UDFs to crash when passing in DDL * Fixes a bug that caused column_to_column_name to crash for invalid input * Fixes a bug that caused inserts into local tables w/ dist. subqueries to crash * Fixes a bug that caused some noop DML statements to fail * Fixes a bug that prevents dropping reference table columns * Fixes a crash in IN (.., NULL) queries * Fixes a crash when calling a distributed function from PL/pgSQL * Fixes an issue that caused CTEs to sometimes leak connections * Fixes strange errors in DML with unreachable sublinks * Prevents statements in SQL functions to run outside of a transaction ### citus v9.1.0 (November 21, 2019) ### * Adds extensions to distributed object propagation infrastructure * Adds support for ALTER ROLE propagation * Adds support for aggregates in create_distributed_function * Adds support for expressions in reference joins * Adds support for returning RECORD in multi-shard queries * Adds support for simple IN subqueries on unique cols in repartition joins * Adds support for subqueries in HAVING clauses * Automatically distributes unary aggs w/ combinefunc and non-internal stype * Disallows distributed func creation when replication_model is 'statement' * Drops support for deprecated real-time and router executors * Fixes a bug in local execution that could cause missing rows in RETURNING * Fixes a bug that caused maintenance daemon to fail on standby nodes * Fixes a bug that caused other CREATE EXTENSION commands to take longer * Fixes a bug that prevented REFRESH MATERIALIZED VIEW * Fixes a bug when view is used in modify statements * Fixes a memory leak in adaptive executor when query returns many columns * Fixes underflow init of default values in worker extended op node creation * Fixes potential segfault in standard_planner inlining functions * Fixes an issue that caused failures in RHEL 6 builds * Fixes queries with repartition joins and group by unique column * Improves CTE/Subquery performance by pruning intermediate rslt broadcasting * Removes `citus.worker_list_file` GUC * Revokes usage from the citus schema from public ### citus v9.0.1 (October 25, 2019) ### * Fixes a memory leak in the executor * Revokes usage from the citus schema from public ### citus v9.0.0 (October 7, 2019) ### * Adds support for PostgreSQL 12 * Adds UDFs to help with PostgreSQL upgrades * Distributes types to worker nodes * Introduces `create_distributed_function` UDF * Introduces local query execution for Citus MX * Implements infrastructure for routing `CALL` to MX workers * Implements infrastructure for routing `SELECT function()` to MX workers * Adds support for foreign key constraints between reference tables * Adds a feature flag to turn off `CREATE TYPE` propagation * Adds option `citus.single_shard_commit_protocol` * Adds support for `EXPLAIN SUMMARY` * Adds support for `GENERATE ALWAYS AS STORED` * Adds support for `serial` and `smallserial` in MX mode * Adds support for anon composite types on the target list in router queries * Avoids race condition between `create_reference_table` & `master_add_node` * Fixes a bug in schemas of distributed sequence definitions * Fixes a bug that caused `run_command_on_colocated_placements` to fail * Fixes a bug that leads to various issues when a connection is lost * Fixes a schema leak on `CREATE INDEX` statement * Fixes assert failure in bare `SELECT FROM reference table FOR UPDATE` in MX * Makes `master_update_node` MX compatible * Prevents `pg_dist_colocation` from multiple records for reference tables * Prevents segfault in `worker_partition_protocol` edgecase * Propagates `ALTER FUNCTION` statements for distributed functions * Propagates `CREATE OR REPLACE FUNCTION` for distributed functions * Propagates `REINDEX` on tables & indexes * Provides a GUC to turn of the new dependency propagation functionality * Uses 2PC in adaptive executor when dealing with replication factors above 1 ### citus v8.3.2 (August 09, 2019) ### * Fixes performance issues by skipping unnecessary relation access recordings ### citus v8.3.1 (July 29, 2019) ### * Improves Adaptive Executor performance ### citus v8.3.0 (July 10, 2019) ### * Adds a new distributed executor: Adaptive Executor * citus.enable_statistics_collection defaults to off (opt-in) * Adds support for CTEs in router planner for modification queries * Adds support for propagating SET LOCAL at xact start * Adds option to force master_update_node during failover * Deprecates master_modify_multiple_shards * Improves round robin logic on router queries * Creates all distributed schemas as superuser on a separate connection * Makes COPY adapt to connection use behaviour of previous commands * Replaces SESSION_LIFESPAN with configurable number of connections at xact end * Propagates ALTER FOREIGN TABLE commands to workers * Don't schedule tasks on inactive nodes * Makes DROP/VALIDATE CONSTRAINT tolerant of ambiguous shard extension * Fixes an issue with subquery map merge jobs as non-root * Fixes null pointers caused by partial initialization of ConnParamsHashEntry * Fixes errors caused by joins with shadowed aliases * Fixes a regression in outer joining subqueries introduced in 8.2.0 * Fixes a crash that can occur under high memory load * Fixes a bug that selects wrong worker when using round-robin assignment * Fixes savepoint rollback after multi-shard modify/copy failure * Fixes bad foreign constraint name search * Fixes a bug that prevents stack size to be adjusted ### citus v8.2.2 (June 11, 2019) ### * Fixes a bug in outer joins wrapped in subqueries ### citus v8.2.1 (April 03, 2019) ### * Fixes a bug that prevents stack size to be adjusted ### citus v8.1.2 (April 03, 2019) ### * Don't do redundant ALTER TABLE consistency checks at coordinator * Fixes a bug that prevents stack size to be adjusted * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands ### citus v8.2.0 (March 28, 2019) ### * Removes support and code for PostgreSQL 9.6 * Enable more outer joins with reference tables * Execute CREATE INDEX CONCURRENTLY in parallel * Treat functions as transaction blocks * Add support for column aliases on join clauses * Skip standard_planner() for trivial queries * Added support for function calls in joins * Round-robin task assignment policy relies on local transaction id * Relax subquery union pushdown restrictions for reference tables * Speed-up run_command_on_shards() * Address some memory issues in connection config * Restrict visibility of get_*_active_transactions functions to pg_monitor * Don't do redundant ALTER TABLE consistency checks at coordinator * Queries with only intermediate results do not rely on task assignment policy * Finish connection establishment in parallel for multiple connections * Fixes a bug related to pruning shards using a coerced value * Fix an issue with some DECLARE .. CURSOR WITH HOLD commands * Fixes a bug that could lead to infinite recursion during recursive planning * Fixes a bug that could prevent planning full outer joins with using clause * Fixes a bug that could lead to memory leak on `citus_relation_size` * Fixes a problem that could cause segmentation fault with recursive planning * Switch CI solution to CircleCI ### citus v8.0.3 (January 9, 2019) ### * Fixes maintenance daemon panic due to unreleased spinlock * Fixes an issue with having clause when used with complex joins ### citus v8.1.1 (January 7, 2019) ### * Fixes maintenance daemon panic due to unreleased spinlock * Fixes an issue with having clause when used with complex joins ### citus v8.1.0 (December 17, 2018) ### * Turns on ssl by default for new installations of citus * Restricts SSL Ciphers to TLS1.2 and above * Adds support for INSERT INTO SELECT..ON CONFLICT/RETURNING via coordinator * Adds support for round-robin task assignment for queries to reference tables * Adds support for SQL tasks using worker_execute_sql_task UDF with task-tracker * Adds support for VALIDATE CONSTRAINT queries * Adds support for disabling hash aggregate with HLL * Adds user ID suffix to intermediate files generated by task-tracker * Only allow transmit from pgsql_job_cache directory * Disallows GROUPING SET clauses in subqueries * Removes restriction on user-defined group ID in node addition functions * Relaxes multi-shard modify locks when enable_deadlock_prevention is disabled * Improves security in task-tracker protocol * Improves permission checks in internal DROP TABLE functions * Improves permission checks in cluster management functions * Cleans up UDFs and fixes permission checks * Fixes crashes caused by stack size increase under high memory load * Fixes a bug that could cause maintenance daemon panic ### citus v8.0.2 (December 13, 2018) ### * Fixes a bug that could cause maintenance daemon panic * Fixes crashes caused by stack size increase under high memory load ### citus v7.5.4 (December 11, 2018) ### * Fixes a bug that could cause maintenance daemon panic ### citus v8.0.1 (November 27, 2018) ### * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker ### citus v7.5.3 (November 27, 2018) ### * Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker ### citus v7.5.2 (November 14, 2018) ### * Fixes inconsistent metadata error when shard metadata caching get interrupted * Fixes a bug that could cause memory leak * Fixes a bug that prevents recovering wrong transactions in MX * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load * Fixes crashes caused by stack size increase under high memory load ### citus v8.0.0 (October 31, 2018) ### * Adds support for PostgreSQL 11 * Adds support for applying DML operations on reference tables from MX nodes * Adds distributed locking to truncated MX tables * Adds support for running TRUNCATE command from MX worker nodes * Adds views to provide insight about the distributed transactions * Adds support for TABLESAMPLE in router queries * Adds support for INCLUDE option in index creation * Adds option to allow simple DML commands from hot standby * Adds support for partitioned tables with replication factor > 1 * Prevents a deadlock on concurrent DROP TABLE and SELECT on Citus MX * Fixes a bug that prevents recovering wrong transactions in MX * Fixes a bug to prevent wrong memory accesses on Citus MX under very high load * Fixes a bug in MX mode, calling DROP SCHEMA with existing partitioned table * Fixes a bug that could cause modifying CTEs to select wrong execution mode * Fixes a bug preventing rollback in CREATE PROCEDURE * Fixes a bug on not being able to drop index on a partitioned table * Fixes a bug on TRUNCATE when there is a foreign key to a reference table * Fixes a performance issue in prepared INSERT..SELECT * Fixes a bug which causes errors on DROP DATABASE IF EXISTS * Fixes a bug to remove intermediate result directory in pull-push execution * Improves query pushdown planning performance * Evaluate functions anywhere in query ### citus v7.5.1 (August 28, 2018) ### * Improves query pushdown planning performance * Fixes a bug that could cause modifying CTEs to select wrong execution mode ### citus v7.4.2 (July 27, 2018) ### * Fixes a segfault in real-time executor during online shard move ### citus v7.5.0 (July 25, 2018) ### * Adds foreign key support from hash distributed to reference tables * Adds SELECT ... FOR UPDATE support for router plannable queries * Adds support for non-partition columns in count distinct * Fixes a segfault in real-time executor during online shard move * Fixes ALTER TABLE ADD COLUMN constraint check * Fixes a bug where INSERT ... SELECT was allowed to update distribution column * Allows DDL commands to be sequentialized via `citus.multi_shard_modify_mode` * Adds support for topn_union_agg and topn_add_agg across shards * Adds support for hll_union_agg and hll_add_agg across shards * Fixes a bug that might cause shards to have a wrong owner * GUC select_opens_transaction_block defers opening transaction block on workers * Utils to implement DDLs for policies in future, warn about being unsupported * Intermediate results use separate connections to avoid interfering with tasks * Adds a node_conninfo GUC to set outgoing connection settings ### citus v6.2.6 (July 06, 2018) ### * Adds support for respecting enable_hashagg in the master planner ### citus v7.4.1 (June 20, 2018) ### * Fixes a bug that could cause transactions to incorrectly proceed after failure * Fixes a bug on INSERT ... SELECT queries in prepared statements ### citus v7.4.0 (May 15, 2018) ### * Adds support for non-pushdownable subqueries and CTEs in UPDATE/DELETE queries * Adds support for pushdownable subqueries and joins in UPDATE/DELETE queries * Adds faster shard pruning for subqueries * Adds partitioning support to MX table * Adds support for (VACUUM | ANALYZE) VERBOSE * Adds support for multiple ANDs in `HAVING` for pushdown planner * Adds support for quotation needy schema names * Improves operator check time in physical planner for custom data types * Removes broadcast join logic * Deprecates `large_table_shard_count` and `master_expire_table_cache()` * Modifies master_update_node to lock write on shards hosted by node over update * `DROP TABLE` now drops shards as the currrent user instead of the superuser * Adds specialised error codes for connection failures * Improves error messages on connection failure * Fixes issue which prevented multiple `citus_table_size` calls per query * Tests are updated to use `create_distributed_table` ### citus v7.2.2 (May 4, 2018) ### * Fixes a bug that could cause SELECTs to crash during a rebalance ### citus v7.3.0 (March 15, 2018) ### * Adds support for non-colocated joins between subqueries * Adds support for window functions that can be pushed down to worker * Adds support for modifying CTEs * Adds recursive plan for subqueries in WHERE clause with recurring FROM clause * Adds support for bool_ and bit_ aggregates * Adds support for Postgres `jsonb` and `json` aggregation functions * Adds support for respecting enable_hashagg in the master plan * Adds support for renaming a distributed table * Adds support for ALTER INDEX (SET|RESET|RENAME TO) commands * Adds support for setting storage parameters on distributed tables * Performance improvements to reduce distributed planning time * Fixes a bug on planner when aggregate is used in ORDER BY * Fixes a bug on planner when DISTINCT (ON) clause is used with GROUP BY * Fixes a bug of creating coordinator planner with distinct and aggregate clause * Fixes a bug that could open a new connection on every table size function call * Fixes a bug canceling backends that are not involved in distributed deadlocks * Fixes count distinct bug on column expressions when used with subqueries * Improves error handling on worker node failures * Improves error messages for INSERT queries that have subqueries ### citus v7.2.1 (February 6, 2018) ### * Fixes count distinct bug on column expressions when used with subqueries * Adds support for respecting enable_hashagg in the master plan * Fixes a bug canceling backends that are not involved in distributed deadlocks ### citus v7.2.0 (January 16, 2018) ### * Adds support for CTEs * Adds support for subqueries that require merge step * Adds support for set operations (UNION, INTERSECT, ...) * Adds support for 2PC auto-recovery * Adds support for querying local tables in CTEs and subqueries * Adds support for more SQL coverage in subqueries for reference tables * Adds support for count(distinct) in queries with a subquery * Adds support for non-equijoins when there is already an equijoin for queries * Adds support for non-equijoins when there is already an equijoin for subquery * Adds support for real-time executor to run in transaction blocks * Adds infrastructure for storing intermediate distributed query results * Adds a new GUC named `enable_repartition_joins` for auto executor switch * Adds support for limiting the intermediate result size * Improves support for queries with unions containing filters * Improves support for queries with unions containing joins * Improves support for subqueries in the `WHERE` clause * Increases `COPY` throughput * Enables pushing down queries containing only recurring tuples and `GROUP BY` * Load-balance queries that read from 0 shards * Improves support for using functions in subqueries * Fixes a bug that could cause real-time executor to crash during cancellation * Fixes a bug that could cause real-time executor to get stuck on cancellation * Fixes a bug that could block modification queries unnecessarily * Fixes a bug that could cause assigning wrong IDs to transactions * Fixes a bug that could cause an assert failure with `ANALYZE` statements * Fixes a bug that could allow pushing down wrong set operations in subqueries * Fixes a bug that could cause a deadlock in create_distributed_table * Fixes a bug that could confuse user about `ANALYZE` usage * Fixes a bug that could lead to false positive distributed deadlock detections * Relaxes the locking for DDL commands on partitioned tables * Relaxes the locking on `COPY` with replication * Logs more remote commands when citus.log_remote_commands is set ### citus v6.2.5 (January 11, 2018) ### * Fixes a bug that could crash the coordinator while reporting a remote error ### citus v7.1.2 (January 4, 2018) ### * Fixes a bug that could cause assigning wrong IDs to transactions * Increases `COPY` throughput ### citus v7.1.1 (December 1, 2017) ### * Fixes a bug that could prevent pushing down subqueries with reference tables * Fixes a bug that could create false positive distributed deadlocks * Fixes a bug that could prevent running concurrent COPY and multi-shard DDL * Fixes a bug that could mislead users about `ANALYZE` queries ### citus v7.1.0 (November 14, 2017) ### * Adds support for native queries with multi shard `UPDATE`/`DELETE` queries * Expands reference table support in subquery pushdown * Adds window function support for subqueries and `INSERT ... SELECT` queries * Adds support for `COUNT(DISTINCT) [ON]` queries on non-partition columns * Adds support for `DISTINCT [ON]` queries on non-partition columns * Introduces basic usage statistic collector * Adds support for setting replica identity while creating distributed tables * Adds support for `ALTER TABLE ... REPLICA IDENTITY` queries * Adds pushdown support for `LIMIT` and `HAVING` grouped by partition key * Adds support for `INSERT ... SELECT` queries via worker nodes on MX clusters * Adds support for adding primary key using already defined index * Adds parameter to shard copy functions to support distinct replication models * Changes `shard_name` UDF to omit public schema name * Adds `master_move_node` UDF to make changes on nodename/nodeport more easy * Fixes a bug that could cause casting error with `INSERT ... SELECT` queries * Fixes a bug that could prevent upgrading servers from Citus 6.1 * Fixes a bug that could prevent attaching partitions to a table in schema * Fixes a bug that could prevent adding a node to cluster with reference table * Fixes a bug that could cause a crash with `INSERT ... SELECT` queries * Fixes a bug that could prevent creating a partitoned table on Cloud * Implements various performance improvements * Adds internal infrastructures and tests to improve development process * Addresses various race conditions and deadlocks * Improves and standardizes error messages ### citus v7.0.3 (October 16, 2017) ### * Fixes several bugs that could cause crash * Fixes a bug that could cause deadlock while creating reference tables * Fixes a bug that could cause false-positives in deadlock detection * Fixes a bug that could cause 2PC recovery not to work from MX workers * Fixes a bug that could cause cache incohorency * Fixes a bug that could cause maintenance daemon to skip cache invalidations * Improves performance of transaction recovery by using correct index ### citus v7.0.2 (September 28, 2017) ### * Updates task-tracker to limit file access ### citus v6.2.4 (September 28, 2017) ### * Updates task-tracker to limit file access ### citus v6.1.3 (September 28, 2017) ### * Updates task-tracker to limit file access ### citus v7.0.1 (September 12, 2017) ### * Fixes a bug that could cause memory leaks in `INSERT ... SELECT` queries * Fixes a bug that could cause incorrect execution of prepared statements * Fixes a bug that could cause excessive memory usage during COPY * Incorporates latest changes from core PostgreSQL code ### citus v7.0.0 (August 28, 2017) ### * Adds support for PostgreSQL 10 * Drops support for PostgreSQL 9.5 * Adds support for multi-row `INSERT` * Adds support for router `UPDATE` and `DELETE` queries with subqueries * Adds infrastructure for distributed deadlock detection * Deprecates `enable_deadlock_prevention` flag * Adds support for partitioned tables * Adds support for creating `UNLOGGED` tables * Adds support for `SAVEPOINT` * Adds UDF `citus_create_restore_point` for taking distributed snapshots * Adds support for evaluating non-pushable `INSERT ... SELECT` queries * Adds support for subquery pushdown on reference tables * Adds shard pruning support for `IN` and `ANY` * Adds support for `UPDATE` and `DELETE` commands that prune down to 0 shard * Enhances transaction support by relaxing some transaction restrictions * Fixes a bug causing crash if distributed table has no shards * Fixes a bug causing crash when removing inactive node * Fixes a bug causing failure during `COPY` on tables with dropped columns * Fixes a bug causing failure during `DROP EXTENSION` * Fixes a bug preventing executing `VACUUM` and `INSERT` concurrently * Fixes a bug in prepared `INSERT` statements containing an implicit cast * Fixes several issues related to statement cancellations and connections * Fixes several 2PC related issues * Removes an unnecessary dependency causing warning messages in pg_dump * Adds internal infrastructure for follower clusters * Adds internal infrastructure for progress tracking * Implements various performance improvements * Adds internal infrastructures and tests to improve development process * Addresses various race conditions and deadlocks * Improves and standardizes error messages ### citus v6.2.3 (July 13, 2017) ### * Fixes a crash during execution of local CREATE INDEX CONCURRENTLY * Fixes a bug preventing usage of quoted column names in COPY * Fixes a bug in prepared INSERTs with implicit cast in partition column * Relaxes locks in VACUUM to ensure concurrent execution with INSERT ### citus v6.2.2 (May 31, 2017) ### * Fixes a common cause of deadlocks when repairing tables with foreign keys ### citus v6.2.1 (May 24, 2017) ### * Relaxes version-check logic to avoid breaking non-distributed commands ### citus v6.2.0 (May 16, 2017) ### * Increases SQL subquery coverage by pushing down more kinds of queries * Adds CustomScan API support to allow read-only transactions * Adds support for `CREATE/DROP INDEX CONCURRENTLY` * Adds support for `ALTER TABLE ... ADD CONSTRAINT` * Adds support for `ALTER TABLE ... RENAME COLUMN` * Adds support for `DISABLE/ENABLE TRIGGER ALL` * Adds support for expressions in the partition column in INSERTs * Adds support for query parameters in combination with function evaluation * Adds support for creating distributed tables from non-empty local tables * Adds UDFs to get size of distributed tables * Adds UDFs to add a new node without replicating reference tables * Adds checks to prevent running Citus binaries with wrong metadata tables * Improves shard pruning performance for range queries * Improves planner performance for joins involving co-located tables * Improves shard copy performance by creating indexes after copy * Improves task-tracker performance by batching several status checks * Enables router planner for queries on range partitioned table * Changes `TRUNCATE` to drop local data only if `enable_ddl_propagation` is off * Starts to execute DDL on coordinator before workers * Fixes a bug causing incorrectly reading invalidated cache * Fixes a bug related to creation of schemas of in workers with incorrect owner * Fixes a bug related to concurrent run of shard drop functions * Fixes a bug related to `EXPLAIN ANALYZE` with DML queries * Fixes a bug related to SQL functions in FROM clause * Adds a GUC variable to report cross shard queries * Fixes a bug related to partition columns without native hash function * Adds internal infrastructures and tests to improve development process * Addresses various race conditions and deadlocks * Improves and standardizes error messages ### citus v6.1.2 (May 31, 2017) ### * Fixes a common cause of deadlocks when repairing tables with foreign keys ### citus v6.1.1 (May 5, 2017) ### * Fixes a crash caused by router executor use after connection timeouts * Fixes a crash caused by relation cache invalidation during COPY * Fixes bug related to DDL use within PL/pgSQL functions * Fixes a COPY bug related to types lacking binary output functions * Fixes a bug related to modifications with parameterized partition values * Fixes improper value interpolation in worker sequence generation * Guards shard pruning logic against zero-shard tables * Fixes possible NULL pointer dereference and buffer underflow (via PVS-Studio) * Fixes a INSERT ... SELECT bug that could push down non-partition column JOINs ### citus v6.1.0 (February 9, 2017) ### * Implements _reference tables_, transactionally replicated to all nodes * Adds `upgrade_to_reference_table` UDF to upgrade pre-6.1 reference tables * Expands prepared statement support to nearly all statements * Adds support for creating `VIEW`s which reference distributed tables * Adds targeted `VACUUM`/`ANALYZE` support * Adds support for the `FILTER` clause in aggregate expressions * Adds support for function evaluation within `INSERT INTO ... SELECT` * Adds support for creating foreign key constraints with `ALTER TABLE` * Adds logic to choose router planner for all queries it supports * Enhances `create_distributed_table` with parameter for explicit colocation * Adds generally useful utility UDFs previously available as "Citus Tools" * Adds user-facing UDFs for locking shard resources and metadata * Refactors connection and transaction management; giving consistent experience * Enhances `COPY` with fully transactional semantics * Improves support for cancellation for a number of queries and commands * Adds `column_to_column_name` UDF to help users understand `partkey` values * Adds `master_disable_node` UDF for temporarily disabling nodes * Adds proper MX ("masterless") metadata propagation logic * Adds `start_metadata_sync_to_node` UDF to propagate metadata changes to nodes * Enhances `SERIAL` compatibility with MX tables * Adds `node_connection_timeout` parameter to control node connection timeouts * Adds `enable_deadlock_prevention` setting to permit multi-node transactions * Adds a `replication_model` setting to specify replication of new tables * Changes the `shard_replication_factor` setting's default value to one * Adds code to automatically set `max_prepared_transactions` if not configured * Accelerates lookup of colocated shard placements * Fixes a bug affecting `INSERT INTO ... SELECT` queries using constant values * Fixes a bug by ensuring `COPY` does not mark placements inactive * Fixes a bug affecting reads from `pg_dist_shard_placement` table * Fixes a crash triggered by creating a foreign key without a column * Fixes a crash related to accessing catalog tables after aborted transactions * Fixes a bug affecting JOIN queries requiring repartitions * Fixes a bug affecting node insertions to `pg_dist_node` table * Fixes a crash triggered by queries with modifying common table expressions * Fixes a bug affecting workloads with concurrent shard appends and deletions * Addresses various race conditions and deadlocks * Improves and standardizes error messages ### citus v6.0.1 (November 29, 2016) ### * Fixes a bug causing failures during pg_upgrade * Fixes a bug preventing DML queries during colocated table creation * Fixes a bug that caused NULL parameters to be incorrectly passed as text ### citus v6.0.0 (November 7, 2016) ### * Adds compatibility with PostgreSQL 9.6, now the recommended version * Removes the `pg_worker_list.conf` file in favor of a `pg_dist_node` table * Adds `master_add_node` and `master_add_node` UDFs to manage membership * Removes the `\stage` command and corresponding csql binary in favor of `COPY` * Removes `copy_to_distributed_table` in favor of first-class `COPY` support * Adds support for multiple DDL statements within a transaction * Adds support for certain foreign key constraints * Adds support for parallel `INSERT INTO ... SELECT` against colocated tables * Adds support for the `TRUNCATE` command * Adds support for `HAVING` clauses in `SELECT` queries * Adds support for `EXCLUDE` constraints which include the partition column * Adds support for system columns in queries (`tableoid`, `ctid`, etc.) * Adds support for relation name extension within `INDEX` definitions * Adds support for no-op `UPDATE`s of the partition column * Adds several general-purpose utility UDFs to aid in Citus maintenance * Adds `master_expire_table_cache` UDF to forcibly expire cached shards * Parallelizes the processing of DDL commands which affect distributed tables * Adds support for repartition jobs using composite or custom types * Enhances object name extension to handle long names and large shard counts * Parallelizes the `master_modify_multiple_shards` UDF * Changes distributed table creation to error if the target table is not empty * Changes the `pg_dist_shard.logicalrelid` column from an `oid` to `regclass` * Adds a `placementid` column to `pg_dist_shard_placement`, replacing Oid use * Removes the `pg_dist_shard.shardalias` distribution metadata column * Adds `pg_dist_partition.repmodel` to track tables using streaming replication * Adds internal infrastructure to take snapshots of distribution metadata * Addresses the need to invalidate prepared statements on metadata changes * Adds a `mark_tables_colocated` UDF for denoting pre-6.0 manual colocation * Fixes a bug affecting prepared statement execution within PL/pgSQL * Fixes a bug affecting `COPY` commands using composite types * Fixes a bug that could cause crashes during `EXPLAIN EXECUTE` * Separates worker and master job temporary folders * Eliminates race condition between distributed modification and repair * Relaxes the requirement that shard repairs also repair colocated shards * Implements internal functions to track which tables' shards are colocated * Adds `pg_dist_partition.colocationid` to track colocation group membership * Extends shard copy and move operations to respect colocation settings * Adds `pg_dist_local_group` to prepare for future MX-related changes * Adds `create_distributed_table` to easily create shards and infer colocation ### citus v5.2.2 (November 7, 2016) ### * Adds support for `IF NOT EXISTS` clause of `CREATE INDEX` command * Adds support for `RETURN QUERY` and `FOR ... IN` PL/pgSQL features * Extends the router planner to handle more queries * Changes `COUNT` of zero-row sets to return `0` rather than an empty result * Reduces the minimum permitted `task_tracker_delay` to a single millisecond * Fixes a bug that caused crashes during joins with a `WHERE false` clause * Fixes a bug triggered by unique violation errors raised in long transactions * Fixes a bug resulting in multiple registration of transaction callbacks * Fixes a bug which could result in stale reads of distribution metadata * Fixes a bug preventing distributed modifications in some PL/pgSQL functions * Fixes some code paths that could hypothetically read uninitialized memory * Lowers log level of _waiting for activity_ messages ### citus v5.2.1 (September 6, 2016) ### * Fixes subquery pushdown to properly extract outer join qualifiers * Addresses possible memory leak during multi-shard transactions ### citus v5.2.0 (August 15, 2016) ### * Drops support for PostgreSQL 9.4; PostgreSQL 9.5 is required * Adds schema support for tables, other named objects (types, operators, etc.) * Evaluates non-immutable functions on master in all modification commands * Adds support for SERIAL types in non-partition columns * Adds support for RETURNING clause in INSERT, UPDATE, and DELETE commands * Adds support for multi-statement transactions involving a fixed set of nodes * Full SQL support for SELECT queries which can be executed on a single worker * Adds option to perform DDL changes using prepared transactions (2PC) * Adds an `enable_ddl_propagation` parameter to control DDL propagation * Accelerates shard pruning during merges * Adds `master_modify_multiple_shards` UDF to modify many shards at once * Adds COPY support for arrays of user-defined types * Now supports parameterized prepared statements for certain use cases * Extends LIMIT/OFFSET support to all executor types * Constraint violations now fail fast rather than hitting all placements * Makes `master_create_empty_shard` aware of shard placement policy * Reduces unnecessary sleep during queries processed by real-time executor * Improves task tracker executor's task cleanup logic * Relaxes restrictions on cancellation of DDL commands * Removes ONLY keyword from worker SELECT queries * Error message improvements and standardization * Moves `master_update_shard_statistics` function to `pg_catalog` schema * Fixes a bug where hash-partitioned anti-joins could return incorrect results * Now sets storage type correctly for foreign table-backed shards * Fixes `master_update_shard_statistics` issue with hash-partitioned tables * Fixes an issue related to extending table names that require escaping * Reduces risk of row counter overflows during modifications * Fixes a crash related to FILTER clause use in COUNT DISTINCT subqueries * Fixes crashes related to partition columns with high attribute numbers * Fixes certain subquery and join crashes * Detects flex for build even if PostgreSQL was built without it * Fixes assert-enabled crash when `all_modifications_commutative` is true ### citus v5.2.0-rc.1 (August 1, 2016) ### * Initial 5.2.0 candidate ### citus v5.1.1 (June 17, 2016) ### * Adds complex count distinct expression support in repartitioned subqueries * Improves task tracker job cleanup logic, addressing a memory leak * Fixes bug that generated incorrect results for LEFT JOIN queries * Improves compatibility with Debian's reproducible builds project * Fixes build issues on FreeBSD platforms ### citus v5.1.0 (May 17, 2016) ### * Adds distributed COPY to rapidly populate distributed tables * Adds support for using EXPLAIN on distributed queries * Recognizes and fast-paths single-shard SELECT statements automatically * Increases INSERT throughput via shard pruning optimizations * Improves planner performance for joins involving tables with many shards * Adds ability to pass columns as arguments to function calls in UPDATEs * Introduces transaction manager for use by multi-shard commands * Adds COUNT(DISTINCT ...) pushdown optimization for hash-partitioned tables * Adds support for certain UNIQUE indexes on hash- or range-partitioned tables * Deprecates \stage in favor of using COPY for append-partition tables * Deprecates `copy_to_distributed_table` in favor of first-class COPY support * Fixes build problems when using non-packaged PostgreSQL installs * Fixes bug that sometimes skipped pruning when partitioned by a VARCHAR column * Fixes bug impeding use of user-defined functions in repartitioned subqueries * Fixes bug involving queries with equality comparisons of boolean types * Fixes crash that prevented use alongside `pg_stat_statements` * Fixes crash arising from SELECT queries that lack a target list * Improves warning and error messages ### citus v5.1.0-rc.2 (May 10, 2016) ### * Fixes test failures * Fixes EXPLAIN output when FORMAT JSON in use ### citus v5.1.0-rc.1 (May 4, 2016) ### * Initial 5.1.0 candidate ### citus v5.0.1 (April 15, 2016) ### * Fixes issues on 32-bit systems ### citus v5.0.0 (March 24, 2016) ### * Public release under AGPLv3 * PostgreSQL extension compatible with PostgreSQL 9.5 and 9.4 ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Microsoft Open Source Code of Conduct This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). Resources: - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to Citus We're happy you want to contribute! You can help us in different ways: * Open an [issue](https://github.com/citusdata/citus/issues) with suggestions for improvements * Fork this repository and submit a pull request Before accepting any code contributions we ask that contributors sign a Contributor License Agreement (CLA). For an explanation of why we ask this as well as instructions for how to proceed, see the [Microsoft CLA](https://cla.opensource.microsoft.com/). ### Devcontainer / Github Codespaces The easiest way to start contributing is via our devcontainer. This container works both locally in visual studio code with docker-desktop/docker-for-mac as well as [Github Codespaces](https://github.com/features/codespaces). To open the project in vscode you will need the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). For codespaces you will need to [create a new codespace](https://codespace.new/citusdata/citus). With the extension installed you can run the following from the command pallet to get started ``` > Dev Containers: Clone Repository in Container Volume... ``` In the subsequent popup paste the url to the repo and hit enter. ``` https://github.com/citusdata/citus ``` This will create an isolated Workspace in vscode, complete with all tools required to build, test and run the Citus extension. We keep this container up to date with the supported postgres versions as well as the exact versions of tooling we use. To quickly start we suggest splitting your terminal once to have two shells. The left one in the `/workspaces/citus`, the second one changed to `/data`. The left terminal will be used to interact with the project, the right one with a testing cluster. To get citus installed from source we run `make install -s` in the first terminal. Once installed you can start a Citus cluster in the second terminal via `citus_dev make citus`. The cluster will run in the background, and can be interacted with via `citus_dev`. To get an overview of the available commands. With the Citus cluster running you can connect to the coordinator in the first terminal via `psql -p9700`. Because the coordinator is the most common entrypoint the `PGPORT` environment is set accordingly, so a simple `psql` will connect directly to the coordinator. ### Debugging in the VS code 1. Start Debugging: Press F5 in VS Code to start debugging. When prompted, you'll need to attach the debugger to the appropriate PostgreSQL process. 2. Identify the Process: If you're running a psql command, take note of the PID that appears in your psql prompt. For example: ``` [local] citus@citus:9700 (PID: 5436)=# ``` This PID (5436 in this case) indicates the process that you should attach the debugger to. If you are uncertain about which process to attach, you can list all running PostgreSQL processes using the following command: ``` ps aux | grep postgres ``` Look for the process associated with the PID you noted. For example: ``` citus 5436 0.0 0.0 0 0 ? S 14:00 0:00 postgres: citus citus ``` 4. Attach the Debugger: Once you've identified the correct PID, select that process when prompted in VS Code to attach the debugger. You should now be able to debug the PostgreSQL session tied to the psql command. 5. Set Breakpoints and Debug: With the debugger attached, you can set breakpoints within the code. This allows you to step through the code execution, inspect variables, and fully debug the PostgreSQL instance running in your container. ### Getting and building [PostgreSQL documentation](https://www.postgresql.org/support/versioning/) has a section on upgrade policy. We always recommend that all users run the latest available minor release [for PostgreSQL] for whatever major version is in use. We expect Citus users to honor this recommendation and use latest available PostgreSQL minor release. Failure to do so may result in failures in our test suite. There are some known improvements in PG test architecture such as [this commit](https://github.com/postgres/postgres/commit/3f323956128ff8589ce4d3a14e8b950837831803) that are missing in earlier minor versions. #### Mac 1. Install Xcode 2. Install packages with Homebrew ```bash brew update brew install git postgresql python ``` 3. Get, build, and test the code ```bash git clone https://github.com/citusdata/citus.git cd citus ./configure # If you have already installed the project, you need to clean it first make clean make make install # Optionally, you might instead want to use `make install-all` # since `multi_extension` regression test would fail due to missing downgrade scripts. cd src/test/regress pip install pipenv pipenv --rm pipenv install pipenv shell make check ``` #### Debian-based Linux (Ubuntu, Debian) 1. Install build dependencies ```bash echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - sudo apt-get update sudo apt-get install -y postgresql-server-dev-14 postgresql-14 \ autoconf flex git libcurl4-gnutls-dev libicu-dev \ libkrb5-dev liblz4-dev libpam0g-dev libreadline-dev \ libselinux1-dev libssl-dev libxslt1-dev libzstd-dev \ make uuid-dev ``` 2. Get, build, and test the code ```bash git clone https://github.com/citusdata/citus.git cd citus ./configure # If you have already installed the project previously, you need to clean it first make clean make sudo make install # Optionally, you might instead want to use `sudo make install-all` # since `multi_extension` regression test would fail due to missing downgrade scripts. cd src/test/regress pip install pipenv pipenv --rm pipenv install pipenv shell make check ``` #### Red Hat-based Linux (RHEL, CentOS, Fedora) 1. Find the RPM URL for your repo at [yum.postgresql.org](http://yum.postgresql.org/repopackages.php) 2. Register its contents with Yum: ```bash sudo yum install -y ``` 3. Register EPEL and SCL repositories for your distro. On CentOS: ```bash yum install -y centos-release-scl-rh epel-release ``` On RHEL, see [this RedHat blog post](https://developers.redhat.com/blog/2018/07/07/yum-install-gcc7-clang/) to install set-up SCL first. Then run: ```bash yum install -y epel-release ``` 4. Install build dependencies ```bash sudo yum update -y sudo yum groupinstall -y 'Development Tools' sudo yum install -y postgresql14-devel postgresql14-server \ git libcurl-devel libxml2-devel libxslt-devel \ libzstd-devel llvm-toolset-7-clang llvm5.0 lz4-devel \ openssl-devel pam-devel readline-devel git clone https://github.com/citusdata/citus.git cd citus PG_CONFIG=/usr/pgsql-14/bin/pg_config ./configure # If you have already installed the project previously, you need to clean it first make clean make sudo make install # Optionally, you might instead want to use `sudo make install-all` # since `multi_extension` regression test would fail due to missing downgrade scripts. cd src/test/regress pip install pipenv pipenv --rm pipenv install pipenv shell make check ``` ### Following our coding conventions Our coding conventions are documented in [STYLEGUIDE.md](STYLEGUIDE.md). ### Making SQL changes Sometimes you need to make change to the SQL that the citus extension runs upon creations. The way this is done is by changing the last file in `src/backend/distributed/sql`, or creating it if the last file is from a published release. If you needed to create a new file, also change the `default_version` field in `src/backend/distributed/citus.control` to match your new version. All the files in this directory are run in order based on their name. See [this page in the Postgres docs](https://www.postgresql.org/docs/current/extend-extensions.html) for more information on how Postgres runs these files. #### Changing or creating functions If you need to change any functions defined by Citus. You should check inside `src/backend/distributed/sql/udfs` to see if there is already a directory for this function, if not create one. Then change or create the file called `latest.sql` in that directory to match how it should create the function. This should be including any DROP (IF EXISTS), COMMENT and REVOKE statements for this function. Then copy the `latest.sql` file to `{version}.sql`, where `{version}` is the version for which this sql change is, e.g. `{9.0-1.sql}`. Now that you've created this stable snapshot of the function definition for your version you should use it in your actual sql file, e.g. `src/backend/distributed/sql/citus--8.3-1--9.0-1.sql`. You do this by using C style `#include` statements like this: ``` #include "udfs/myudf/9.0-1.sql" ``` #### Other SQL Any other SQL you can put directly in the main sql file, e.g. `src/backend/distributed/sql/citus--8.3-1--9.0-1.sql`. ### Backporting a commit to a release branch 1. Check out the release branch that you want to backport to `git checkout release-11.3` 2. Make sure you have the latest changes `git pull` 3. Create a new release branch with a unique name `git checkout -b release-11.3-` 4. Cherry-pick the commit that you want to backport `git cherry-pick -x ` (the `-x` is important) 5. Push the branch `git push` 6. Wait for tests to pass 7. If the cherry-pick required non-trivial merge conflicts, create a PR and ask for a review. 8. After the tests pass on CI, fast-forward the release branch `git push origin release-11.3-:release-11.3` ### Running tests See [`src/test/regress/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/README.md) ### Documentation User-facing documentation is published on [docs.citusdata.com](https://docs.citusdata.com/). When adding a new feature, function, or setting, you can open a pull request or issue against the [Citus docs repo](https://github.com/citusdata/citus_docs/). Detailed descriptions of the implementation for Citus developers are provided in the [Citus Technical Documentation](src/backend/distributed/README.md). It is currently a single file for ease of searching. Please update the documentation if you make any changes that affect the design or add major new features. # Making a pull request ready for reviews Asking for help and asking for reviews are two different things. When you're asking for help, you're asking for someone to help you with something that you're not expected to know. But when you're asking for a review, you're asking for someone to review your work and provide feedback. So, when you're asking for a review, you're expected to make sure that: * Your changes don't perform **unnecessary line addition / deletions / style changes on unrelated files / lines**. * All CI jobs are **passing**, including **style checks** and **flaky test detection jobs**. Note that if you're an external contributor, you don't have to wait CI jobs to run (and finish) because they don't get automatically triggered for external contributors. * Your PR has necessary amount of **tests** and that they're passing. * You separated as much as possible work into **separate PRs**, e.g., a prerequisite bugfix, a refactoring etc.. * Your PR doesn't introduce a typo or something that you can easily fix yourself. * After all CI jobs pass, code-coverage measurement job (CodeCov as of today) then kicks in. That's why it's important to make the **tests passing** first. At that point, you're expected to check **CodeCov annotations** that can be seen in the **Files Changed** tab and expected to make sure that it doesn't complain about any lines that are not covered. For example, it's ok if CodeCov complains about an `ereport()` call that you put for an "unexpected-but-better-than-crashing" case, but it's not ok if it complains about an uncovered `if` branch that you added. * And finally, perform a **self-review** to make sure that: * Code and code-comments reflects the idea **without requiring an extra explanation** via a chat message / email / PR comment. This is important because we don't expect developers to reach out to author / read about the whole discussion in the PR to understand the idea behind a commit merged into `main` branch. * PR description is clear enough. * If-and-only-if you're **introducing a user facing change / bugfix**, your PR has a line that starts with `DESCRIPTION: `. * **Commit messages** are clear enough if the commits are doing logically different things. ================================================ FILE: DEVCONTAINER.md ================================================ # Devcontainer ## Coredumps When postgres/citus crashes, there is the option to create a coredump. This is useful for debugging the issue. Coredumps are enabled in the devcontainer by default. However, not all environments are configured correctly out of the box. The most important configuration that is not standardized is the `core_pattern`. The configuration can be verified from the container, however, you cannot change this setting from inside the container as the filesystem containing this setting is in read only mode while inside the container. To verify if corefiles are written run the following command in a terminal. This shows the filename pattern with which the corefile will be written. ```bash cat /proc/sys/kernel/core_pattern ``` This should be configured with a relative path or simply a simple filename, such as `core`. When your environment shows an absolute path you will need to change this setting. How to change this setting depends highly on the underlying system as the setting needs to be changed on the kernel of the host running the container. You can put any pattern in `/proc/sys/kernel/core_pattern` as you see fit. eg. You can add the PID to the core pattern in one of two ways; - You either include `%p` in the core_pattern. This gets substituted with the PID of the crashing process. - Alternatively you could set `/proc/sys/kernel/core_uses_pid` to `1` in the same way as you set `core_pattern`. This will append the PID to the corefile if `%p` is not explicitly contained in the core_pattern. When a coredump is written you can use the debug/launch configuration `Open core file` which is preconfigured in the devcontainer. This will open a fileprompt that lists all coredumps that are found in your workspace. When you want to debug coredumps from `citus_dev` that are run in your `/data` directory, you can add the data directory to your workspace. In the command pallet of vscode you can run `>Workspace: Add Folder to Workspace...` and select the `/data` directory. This will allow you to open the coredumps from the `/data` directory in the `Open core file` debug configuration. ### Windows (docker desktop) When running in docker desktop on windows you will most likely need to change this setting. The linux guest in WSL2 that runs your container is the `docker-desktop` environment. The easiest way to get onto the host, where you can change this setting, is to open a powershell window and verify you have the docker-desktop environment listed. ```powershell wsl --list ``` Among others this should list both `docker-desktop` and `docker-desktop-data`. You can then open a shell in the `docker-desktop` environment. ```powershell wsl -d docker-desktop ``` Inside this shell you can verify that you have the right environment by running ```bash cat /proc/sys/kernel/core_pattern ``` This should show the same configuration as the one you see inside the devcontainer. You can then change the setting by running the following command. This will change the setting for the current session. If you want to make the change permanent you will need to add this to a startup script. ```bash echo "core" > /proc/sys/kernel/core_pattern ``` ================================================ FILE: EXTENSION_COMPATIBILITY.md ================================================ Below table is created with Citus 12.1.7 on PG16 | Extension Name | Works as Expected | Notes | |:-----------------------------|:--------------------|:--------| | address_standardizer | Yes | | | address_standardizer_data_us | Yes | | | age | Partially | Works fine side by side, but graph data cannot be distributed. | | amcheck | Yes | | | anon | Partially | Cannot anonymize distributed tables. It is possible to anonymize local tables. | | auto_explain | No | [Issue #6448](https://github.com/citusdata/citus/issues/6448) | | azure | Yes | | | azure_ai | Yes | | | azure_storage | Yes | | | bloom | Yes | | | Btree_gin | Yes | | | btree_gist | Yes | | | citext | Yes | | | Citus_columnar | Yes | | | cube | Yes | | | dblink | Yes | | | dict_int | Yes | | | dict_xsyn | Yes | | | earthdistance | Yes | | | fuzzystrmatch | Yes | | | hll | Yes | | | hstore | Yes | | | hypopg | Partially | Hypopg can work on local tables and individual shards, however, when we create a hypothetical index on a distributed table, citus does not propagate the index creation command to worker nodes, and thus, hypothetical index is not used in explain statements. | | intagg | Yes | | | intarray | Yes | | | isn | Yes | | | lo | Partially | Extension relies on triggers, but Citus does not support triggers over distributed tables | | login_hook | Yes | | | ltree | Yes | | | oracle_fdw | Yes | | | orafce | Yes | | | pageinspect | Yes | | | pg_buffercache | Yes | | | pg_cron | Yes | | | pg_diskann | Yes | | | pg_failover_slots | To be tested | | | pg_freespacemap | Partially | Users can set citus.override_table_visibility='off'; to get accurate calculation of free space map. | | pg_hint_plan | Partially | Works fine side by side, but hints are ignored for distributed queries | | pg_partman | Yes | | | pg_prewarm | Partially | In order to prewarm distributed tables, set " citus.override_table_visibility" to off, and run prewarm for each shard. This needs to be done at each node. | | pg_repack | Partially | Extension relies on triggers, but Citus does not support triggers over distributed tables. It works fine on local tables. | | pg_squeeze | Partially | It can work on local tables, but it is not aware of distributed tables. Users can set citus.override_table_visibility='off'; and then run pg_squeeze for each shard. This needs to be done at each node. | | pg_stat_statements | Yes | | | pg_trgm | Yes | | | pg_visibility | Partially | In order to get visibility map of a distributed table, customers can run the functions for shard tables. | | pgaadauth | Yes | | | pgaudit | Yes | | | pgcrypto | Yes | | | pglogical | No | | | pgrowlocks | Partially | It works only with individual shards, not with distributed table names. | | pgstattuple | Yes | | | plpgsql | Yes | | | plv8 | Yes | | | postgis | Yes | | | postgis_raster | Yes | | | postgis_sfcgal | Yes | | | postgis_tiger_geocoder | No | | | postgis_topology | No | | | postgres_fdw | Yes | | | postgres_protobuf | Yes | | | semver | Yes | | | session_variable | No | | | sslinfo | Yes | | | tablefunc | Yes | | | tdigest | Yes | | | tds_fdw | Yes | | | timescaledb | No | [Known to be incompatible with Citus](https://www.citusdata.com/blog/2021/10/22/how-to-scale-postgres-for-time-series-data-with-citus/#:~:text=Postgres%E2%80%99%20built-in%20partitioning) | | topn | Yes | | | tsm_system_rows | Yes | | | tsm_system_time | Yes | | | unaccent | Yes | | | uuid-ossp | Yes | | | vector (aka pg_vector) | Yes | | | wal2json | To be tested | | | xml2 | To be tested | | ================================================ FILE: LICENSE ================================================ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: Makefile ================================================ # Citus toplevel Makefile citus_subdir = . citus_top_builddir = . extension_dir = $(shell $(PG_CONFIG) --sharedir)/extension # Hint that configure should be run first ifeq (,$(wildcard Makefile.global)) $(error ./configure needs to be run before compiling Citus) endif include Makefile.global all: extension # build columnar only columnar: $(MAKE) -C src/backend/columnar all # build extension extension: $(citus_top_builddir)/src/include/citus_version.h columnar $(MAKE) -C src/backend/distributed/ all install-columnar: columnar $(MAKE) -C src/backend/columnar install install-extension: extension install-columnar $(MAKE) -C src/backend/distributed/ install install-headers: extension $(MKDIR_P) '$(DESTDIR)$(includedir_server)/distributed/' # generated headers are located in the build directory $(INSTALL_DATA) $(citus_top_builddir)/src/include/citus_version.h '$(DESTDIR)$(includedir_server)/' # the rest in the source tree $(INSTALL_DATA) $(citus_abs_srcdir)/src/include/distributed/*.h '$(DESTDIR)$(includedir_server)/distributed/' clean-extension: $(MAKE) -C src/backend/distributed/ clean $(MAKE) -C src/backend/columnar/ clean clean-full: $(MAKE) -C src/backend/distributed/ clean-full .PHONY: extension install-extension clean-extension clean-full install-downgrades: $(MAKE) -C src/backend/distributed/ install-downgrades install-all: install-headers $(MAKE) -C src/backend/columnar/ install-all $(MAKE) -C src/backend/distributed/ install-all # Add to generic targets install: install-extension install-headers clean: clean-extension # apply or check style reindent: ${citus_abs_top_srcdir}/ci/fix_style.sh check-style: black . --check --quiet isort . --check --quiet flake8 cd ${citus_abs_top_srcdir} && citus_indent --quiet --check .PHONY: reindent check-style # depend on install-all so that downgrade scripts are installed as well check: all install-all # explicetely does not use $(MAKE) to avoid parallelism make -C src/test/regress check .PHONY: all check clean install install-downgrades install-all ================================================ FILE: Makefile.global.in ================================================ # -*-makefile-*- # @configure_input@ # Makefile.global.in - Makefile to be included by all submakes # # This file is converted by configure into an actual Makefile, # replacing the @varname@ placeholders by actual values. # # This files is intended to contain infrastructure needed by several # makefiles, particulary central handling of compilation flags and # rules. citus_abs_srcdir:=@abs_top_srcdir@/${citus_subdir} citus_abs_top_srcdir:=@abs_top_srcdir@ postgres_abs_srcdir:=@POSTGRES_SRCDIR@ postgres_abs_builddir:=@POSTGRES_BUILDDIR@ PG_CONFIG:=@PG_CONFIG@ PGXS:=$(shell $(PG_CONFIG) --pgxs) # if both, git is installed and there is a .git directory in the working dir we set the # GIT_VERSION to a human readable gitref that resembles the version from which citus is # built. During releases it will show the tagname which by convention is the verion of the # release ifneq (@GIT_BIN@,) ifneq (@HAS_DOTGIT@,) # try to find a tag that exactly matches the current branch, swallow the error if cannot find such a tag GIT_VERSION := "$(shell @GIT_BIN@ describe --exact-match --dirty --always --tags 2>/dev/null)" # if there is not a tag that exactly matches the branch, then GIT_VERSION would still be empty # in that case, set GIT_VERSION with current branch's name and the short sha of the HEAD ifeq ($(GIT_VERSION),"") GIT_VERSION := "$(shell @GIT_BIN@ rev-parse --abbrev-ref HEAD)(sha: $(shell @GIT_BIN@ rev-parse --short HEAD))" endif endif endif # Support for VPATH builds (i.e. builds from outside the source tree) vpath_build=@vpath_build@ ifeq ($(vpath_build),yes) override VPATH:=$(citus_abs_srcdir) USE_VPATH:=$(VPATH) citus_top_srcdir:=$(citus_abs_top_srcdir) override srcdir=$(VPATH) else citus_top_srcdir:=$(citus_top_builddir) endif # Citus is built using PostgreSQL's pgxs USE_PGXS=1 include $(PGXS) # Remake Makefile.global from Makefile.global.in if the latter # changed. In order to trigger this rule, the including file must # write `include $(citus_top_builddir)/Makefile.global', not some # shortcut thereof. This makes it less likely to accidentally run # with some outdated Makefile.global. # Make internally restarts whenever included Makefiles are # regenerated. $(citus_top_builddir)/Makefile.global: $(citus_abs_top_srcdir)/configure $(citus_top_builddir)/Makefile.global.in $(citus_top_builddir)/config.status cd @abs_top_builddir@ && ./config.status Makefile.global # Ensure configuration is generated by the most recent configure, # useful for longer existing build directories. $(citus_top_builddir)/config.status: $(citus_abs_top_srcdir)/configure $(citus_abs_top_srcdir)/src/backend/distributed/citus.control cd @abs_top_builddir@ && ./config.status --recheck && ./config.status # Regenerate configure if configure.ac changed $(citus_abs_top_srcdir)/configure: $(citus_abs_top_srcdir)/configure.ac cd ${citus_abs_top_srcdir} && ./autogen.sh # If specified via configure, replace the default compiler. Normally # we'll build with the one postgres was built with. But it's useful to # be able to use a different one, especially when building against # distribution packages. ifneq (@CC@,) override CC=@CC@ endif # If detected by our configure script, override the FLEX postgres # detected. That allows to compile citus against a postgres which was # built without flex available (possible because generated files are # included) ifneq (@FLEX@,) override FLEX=@FLEX@ endif # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/... override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@ override BITCODE_CFLAGS := $(BITCODE_CFLAGS) @CITUS_BITCODE_CFLAGS@ ifneq ($(GIT_VERSION),) override CFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\" endif override CPPFLAGS := @CPPFLAGS@ @CITUS_CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' -I'${citus_top_builddir}/src/include' $(CPPFLAGS) override LDFLAGS += @LDFLAGS@ @CITUS_LDFLAGS@ # optional file with user defined, additional, rules -include ${citus_abs_srcdir}/src/Makefile.custom ================================================ FILE: NOTICE ================================================ NOTICES AND INFORMATION Do Not Translate or Localize This software incorporates material from third parties. Microsoft makes certain open source code available at https://3rdpartysource.microsoft.com, or you may send a check or money order for US $5.00, including the product name, the open source component name, platform, and version number, to: Source Code Compliance Team Microsoft Corporation One Microsoft Way Redmond, WA 98052 USA Notwithstanding any other terms, you may reverse engineer this software to the extent required to debug changes to any libraries licensed under the GNU Lesser General Public License. --------------------------------------------------------- --------------------------------------------------------- intel/safestringlib 245c4b8cff1d2e7338b7f3a82828fc8e72b29549 - MIT Copyright (c) 2014-2018 Intel Corporation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ Copyright (C) 2012, 2013 Cisco Systems All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------- postgres/postgres 29be9983a64c011eac0b9ee29895cce71e15ea77 PostgreSQL Database Management System (formerly known as Postgres, then as Postgres95) Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. --------------------------------------------------------- ================================================ FILE: README.md ================================================ | **
The Citus database is 100% open source.

Learn what's new in the [Citus 13.0 release blog](https://www.citusdata.com/blog/2025/02/06/distribute-postgresql-17-with-citus-13/) and the [Citus Updates page](https://www.citusdata.com/updates/).

**| |---|
![Citus Banner](images/citus-readme-banner.png) [![Latest Docs](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://docs.citusdata.com/) [![Stack Overflow](https://img.shields.io/badge/Stack%20Overflow-%20-545353?logo=Stack%20Overflow)](https://stackoverflow.com/questions/tagged/citus) [![Slack](https://cituscdn.azureedge.net/images/social/slack-badge.svg)](https://slack.citusdata.com/) [![Code Coverage](https://codecov.io/gh/citusdata/citus/branch/master/graph/badge.svg)](https://app.codecov.io/gh/citusdata/citus) [![Twitter](https://img.shields.io/twitter/follow/citusdata.svg?label=Follow%20@citusdata)](https://twitter.com/intent/follow?screen_name=citusdata) [![Citus Deb Packages](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/app/citusdata/community/search?q=&filter=debs) [![Citus Rpm Packages](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/app/citusdata/community/search?q=&filter=rpms) ## What is Citus? Citus is a [PostgreSQL extension](https://www.citusdata.com/blog/2017/10/25/what-it-means-to-be-a-postgresql-extension/) that transforms Postgres into a distributed database—so you can achieve high performance at any scale. With Citus, you extend your PostgreSQL database with new superpowers: - **Distributed tables** are sharded across a cluster of PostgreSQL nodes to combine their CPU, memory, storage and I/O capacity. - **References tables** are replicated to all nodes for joins and foreign keys from distributed tables and maximum read performance. - **Distributed query engine** routes and parallelizes SELECT, DML, and other operations on distributed tables across the cluster. - **Columnar storage** compresses data, speeds up scans, and supports fast projections, both on regular and distributed tables. - **Query from any node** enables you to utilize the full capacity of your cluster for distributed queries You can use these Citus superpowers to make your Postgres database scale-out ready on a single Citus node. Or you can build a large cluster capable of handling **high transaction throughputs**, especially in **multi-tenant apps**, run **fast analytical queries**, and process large amounts of **time series** or **IoT data** for **real-time analytics**. When your data size and volume grow, you can easily add more worker nodes to the cluster and rebalance the shards. Our [SIGMOD '21](https://2021.sigmod.org/) paper [Citus: Distributed PostgreSQL for Data-Intensive Applications](https://doi.org/10.1145/3448016.3457551) gives a more detailed look into what Citus is, how it works, and why it works that way. ![Citus scales out from a single node](images/citus-scale-out.png) Since Citus is an extension to Postgres, you can use Citus with the latest Postgres versions. And Citus works seamlessly with the PostgreSQL tools and extensions you are already familiar with. - [Why Citus?](#why-citus) - [Getting Started](#getting-started) - [Using Citus](#using-citus) - [Schema-based sharding](#schema-based-sharding) - [Setting up with High Availability](#setting-up-with-high-availability) - [Documentation](#documentation) - [Architecture](#architecture) - [When to Use Citus](#when-to-use-citus) - [Need Help?](#need-help) - [Contributing](#contributing) - [Stay Connected](#stay-connected) ## Why Citus? Developers choose Citus for two reasons: 1. Your application is outgrowing a single PostgreSQL node If the size and volume of your data increases over time, you may start seeing any number of performance and scalability problems on a single PostgreSQL node. For example: High CPU utilization and I/O wait times slow down your queries, SQL queries return out of memory errors, autovacuum cannot keep up and increases table bloat, etc. With Citus you can distribute and optionally compress your tables to always have enough memory, CPU, and I/O capacity to achieve high performance at scale. The distributed query engine can efficiently route transactions across the cluster, while parallelizing analytical queries and batch operations across all cores. Moreover, you can still use the PostgreSQL features and tools you know and love. 2. PostgreSQL can do things other systems can’t There are many data processing systems that are built to scale out, but few have as many powerful capabilities as PostgreSQL, including: Advanced joins and subqueries, user-defined functions, update/delete/upsert, constraints and foreign keys, powerful extensions (e.g. PostGIS, HyperLogLog), many types of indexes, time-partitioning, and sophisticated JSON support. Citus makes PostgreSQL’s most powerful capabilities work at any scale, allowing you to handle complex data-intensive workloads on a single database system. ## Getting Started The quickest way to get started with Citus is to use the [Azure Cosmos DB for PostgreSQL](https://learn.microsoft.com/azure/cosmos-db/postgresql/quickstart-create-portal) managed service in the cloud—or [set up Citus locally](https://docs.citusdata.com/en/stable/installation/single_node.html). ### Citus Managed Service on Azure You can get a fully-managed Citus cluster in minutes through the [Azure Cosmos DB for PostgreSQL portal](https://azure.microsoft.com/products/cosmos-db/). Azure will manage your backups, high availability through auto-failover, software updates, monitoring, and more for all of your servers. To get started Citus on Azure, use the [Azure Cosmos DB for PostgreSQL Quickstart](https://learn.microsoft.com/azure/cosmos-db/postgresql/quickstart-create-portal). ### Running Citus using Docker The smallest possible Citus cluster is a single PostgreSQL node with the Citus extension, which means you can try out Citus by running a single Docker container. ```bash # run PostgreSQL with Citus on port 5500 docker run -d --name citus -p 5500:5432 -e POSTGRES_PASSWORD=mypassword citusdata/citus # connect using psql within the Docker container docker exec -it citus psql -U postgres # or, connect using local psql psql -U postgres -d postgres -h localhost -p 5500 ``` ### Install Citus locally If you already have a local PostgreSQL installation, the easiest way to install Citus is to use our packaging repo Install packages on Ubuntu / Debian: ```bash curl https://install.citusdata.com/community/deb.sh > add-citus-repo.sh sudo bash add-citus-repo.sh sudo apt-get -y install postgresql-17-citus-13.0 ``` Install packages on Red Hat: ```bash curl https://install.citusdata.com/community/rpm.sh > add-citus-repo.sh sudo bash add-citus-repo.sh sudo yum install -y citus130_17 ``` To add Citus to your local PostgreSQL database, add the following to `postgresql.conf`: ``` shared_preload_libraries = 'citus' ``` After restarting PostgreSQL, connect using `psql` and run: ```sql CREATE EXTENSION citus; ```` You’re now ready to get started and use Citus tables on a single node. ### Install Citus on multiple nodes If you want to set up a multi-node cluster, you can also set up additional PostgreSQL nodes with the Citus extensions and add them to form a Citus cluster: ```sql -- before adding the first worker node, tell future worker nodes how to reach the coordinator SELECT citus_set_coordinator_host('10.0.0.1', 5432); -- add worker nodes SELECT citus_add_node('10.0.0.2', 5432); SELECT citus_add_node('10.0.0.3', 5432); -- rebalance the shards over the new worker nodes SELECT rebalance_table_shards(); ``` For more details, see our [documentation on how to set up a multi-node Citus cluster](https://docs.citusdata.com/en/stable/installation/multi_node.html) on various operating systems. ## Using Citus Once you have your Citus cluster, you can start creating distributed tables, reference tables and use columnar storage. ### Creating Distributed Tables The `create_distributed_table` UDF will transparently shard your table locally or across the worker nodes: ```sql CREATE TABLE events ( device_id bigint, event_id bigserial, event_time timestamptz default now(), data jsonb not null, PRIMARY KEY (device_id, event_id) ); -- distribute the events table across shards placed locally or on the worker nodes SELECT create_distributed_table('events', 'device_id'); ``` After this operation, queries for a specific device ID will be efficiently routed to a single worker node, while queries across device IDs will be parallelized across the cluster. ```sql -- insert some events INSERT INTO events (device_id, data) SELECT s % 100, ('{"measurement":'||random()||'}')::jsonb FROM generate_series(1,1000000) s; -- get the last 3 events for device 1, routed to a single node SELECT * FROM events WHERE device_id = 1 ORDER BY event_time DESC, event_id DESC LIMIT 3; ┌───────────┬──────────┬───────────────────────────────┬───────────────────────────────────────┐ │ device_id │ event_id │ event_time │ data │ ├───────────┼──────────┼───────────────────────────────┼───────────────────────────────────────┤ │ 1 │ 1999901 │ 2021-03-04 16:00:31.189963+00 │ {"measurement": 0.88722643925054} │ │ 1 │ 1999801 │ 2021-03-04 16:00:31.189963+00 │ {"measurement": 0.6512231304621992} │ │ 1 │ 1999701 │ 2021-03-04 16:00:31.189963+00 │ {"measurement": 0.019368766051897524} │ └───────────┴──────────┴───────────────────────────────┴───────────────────────────────────────┘ (3 rows) Time: 4.588 ms -- explain plan for a query that is parallelized across shards, which shows the plan for -- a query one of the shards and how the aggregation across shards is done EXPLAIN (VERBOSE ON) SELECT count(*) FROM events; ┌────────────────────────────────────────────────────────────────────────────────────┐ │ QUERY PLAN │ ├────────────────────────────────────────────────────────────────────────────────────┤ │ Aggregate │ │ Output: COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) │ │ -> Custom Scan (Citus Adaptive) │ │ ... │ │ -> Task │ │ Query: SELECT count(*) AS count FROM events_102008 events WHERE true │ │ Node: host=localhost port=5432 dbname=postgres │ │ -> Aggregate │ │ -> Seq Scan on public.events_102008 events │ └────────────────────────────────────────────────────────────────────────────────────┘ ``` ### Creating Distributed Tables with Co-location Distributed tables that have the same distribution column can be co-located to enable high performance distributed joins and foreign keys between distributed tables. By default, distributed tables will be co-located based on the type of the distribution column, but you define co-location explicitly with the `colocate_with` argument in `create_distributed_table`. ```sql CREATE TABLE devices ( device_id bigint primary key, device_name text, device_type_id int ); CREATE INDEX ON devices (device_type_id); -- co-locate the devices table with the events table SELECT create_distributed_table('devices', 'device_id', colocate_with := 'events'); -- insert device metadata INSERT INTO devices (device_id, device_name, device_type_id) SELECT s, 'device-'||s, 55 FROM generate_series(0, 99) s; -- optionally: make sure the application can only insert events for a known device ALTER TABLE events ADD CONSTRAINT device_id_fk FOREIGN KEY (device_id) REFERENCES devices (device_id); -- get the average measurement across all devices of type 55, parallelized across shards SELECT avg((data->>'measurement')::double precision) FROM events JOIN devices USING (device_id) WHERE device_type_id = 55; ┌────────────────────┐ │ avg │ ├────────────────────┤ │ 0.5000191877513974 │ └────────────────────┘ (1 row) Time: 209.961 ms ``` Co-location also helps you scale [INSERT..SELECT](https://docs.citusdata.com/en/stable/articles/aggregation.html), [stored procedures](https://www.citusdata.com/blog/2020/11/21/making-postgres-stored-procedures-9x-faster-in-citus/), and [distributed transactions](https://www.citusdata.com/blog/2017/06/02/scaling-complex-sql-transactions/). ### Distributing Tables without interrupting the application Some of you already start with Postgres, and decide to distribute tables later on while your application using the tables. In that case, you want to avoid downtime for both reads and writes. `create_distributed_table` command block writes (e.g., DML commands) on the table until the command is finished. Instead, with `create_distributed_table_concurrently` command, your application can continue to read and write the data even during the command. ```sql CREATE TABLE device_logs ( device_id bigint primary key, log text ); -- insert device logs INSERT INTO device_logs (device_id, log) SELECT s, 'device log:'||s FROM generate_series(0, 99) s; -- convert device_logs into a distributed table without interrupting the application SELECT create_distributed_table_concurrently('device_logs', 'device_id', colocate_with := 'devices'); -- get the count of the logs, parallelized across shards SELECT count(*) FROM device_logs; ┌───────┐ │ count │ ├───────┤ │ 100 │ └───────┘ (1 row) Time: 48.734 ms ``` ### Creating Reference Tables When you need fast joins or foreign keys that do not include the distribution column, you can use `create_reference_table` to replicate a table across all nodes in the cluster. ```sql CREATE TABLE device_types ( device_type_id int primary key, device_type_name text not null unique ); -- replicate the table across all nodes to enable foreign keys and joins on any column SELECT create_reference_table('device_types'); -- insert a device type INSERT INTO device_types (device_type_id, device_type_name) VALUES (55, 'laptop'); -- optionally: make sure the application can only insert devices with known types ALTER TABLE devices ADD CONSTRAINT device_type_fk FOREIGN KEY (device_type_id) REFERENCES device_types (device_type_id); -- get the last 3 events for devices whose type name starts with laptop, parallelized across shards SELECT device_id, event_time, data->>'measurement' AS value, device_name, device_type_name FROM events JOIN devices USING (device_id) JOIN device_types USING (device_type_id) WHERE device_type_name LIKE 'laptop%' ORDER BY event_time DESC LIMIT 3; ┌───────────┬───────────────────────────────┬─────────────────────┬─────────────┬──────────────────┐ │ device_id │ event_time │ value │ device_name │ device_type_name │ ├───────────┼───────────────────────────────┼─────────────────────┼─────────────┼──────────────────┤ │ 60 │ 2021-03-04 16:00:31.189963+00 │ 0.28902084163415864 │ device-60 │ laptop │ │ 8 │ 2021-03-04 16:00:31.189963+00 │ 0.8723803076285073 │ device-8 │ laptop │ │ 20 │ 2021-03-04 16:00:31.189963+00 │ 0.8177634801548557 │ device-20 │ laptop │ └───────────┴───────────────────────────────┴─────────────────────┴─────────────┴──────────────────┘ (3 rows) Time: 146.063 ms ``` Reference tables enable you to scale out complex data models and take full advantage of relational database features. ### Creating Tables with Columnar Storage To use columnar storage in your PostgreSQL database, all you need to do is add `USING columnar` to your `CREATE TABLE` statements and your data will be automatically compressed using the columnar access method. ```sql CREATE TABLE events_columnar ( device_id bigint, event_id bigserial, event_time timestamptz default now(), data jsonb not null ) USING columnar; -- insert some data INSERT INTO events_columnar (device_id, data) SELECT d, '{"hello":"columnar"}' FROM generate_series(1,10000000) d; -- create a row-based table to compare CREATE TABLE events_row AS SELECT * FROM events_columnar; -- see the huge size difference! \d+ List of relations ┌────────┬──────────────────────────────┬──────────┬───────┬─────────────┬────────────┬─────────────┐ │ Schema │ Name │ Type │ Owner │ Persistence │ Size │ Description │ ├────────┼──────────────────────────────┼──────────┼───────┼─────────────┼────────────┼─────────────┤ │ public │ events_columnar │ table │ marco │ permanent │ 25 MB │ │ │ public │ events_row │ table │ marco │ permanent │ 651 MB │ │ └────────┴──────────────────────────────┴──────────┴───────┴─────────────┴────────────┴─────────────┘ (2 rows) ``` You can use columnar storage by itself, or in a distributed table to combine the benefits of compression and the distributed query engine. When using columnar storage, you should only load data in batch using `COPY` or `INSERT..SELECT` to achieve good compression. Update, delete, and foreign keys are currently unsupported on columnar tables. However, you can use partitioned tables in which newer partitions use row-based storage, and older partitions are compressed using columnar storage. To learn more about columnar storage, check out the [columnar storage README](https://github.com/citusdata/citus/blob/master/src/backend/columnar/README.md). ## Schema-based sharding Available since Citus 12.0, [schema-based sharding](https://docs.citusdata.com/en/stable/get_started/concepts.html#schema-based-sharding) is the shared database, separate schema model, the schema becomes the logical shard within the database. Multi-tenant apps can a use a schema per tenant to easily shard along the tenant dimension. Query changes are not required and the application usually only needs a small modification to set the proper search_path when switching tenants. Schema-based sharding is an ideal solution for microservices, and for ISVs deploying applications that cannot undergo the changes required to onboard row-based sharding. ### Creating distributed schemas You can turn an existing schema into a distributed schema by calling `citus_schema_distribute`: ```sql SELECT citus_schema_distribute('user_service'); ``` Alternatively, you can set `citus.enable_schema_based_sharding` to have all newly created schemas be automatically converted into distributed schemas: ```sql SET citus.enable_schema_based_sharding TO ON; CREATE SCHEMA AUTHORIZATION user_service; CREATE SCHEMA AUTHORIZATION time_service; CREATE SCHEMA AUTHORIZATION ping_service; ``` ### Running queries Queries will be properly routed to schemas based on `search_path` or by explicitly using the schema name in the query. For [microservices](https://docs.citusdata.com/en/stable/get_started/tutorial_microservices.html) you would create a USER per service matching the schema name, hence the default `search_path` would contain the schema name. When connected the user queries would be automatically routed and no changes to the microservice would be required. ```sql CREATE USER user_service; CREATE SCHEMA AUTHORIZATION user_service; ``` For typical multi-tenant applications, you would set the search path to the tenant schema name in your application: ```sql SET search_path = tenant_name, public; ``` ## Setting up with High Availability One of the most popular high availability solutions for PostgreSQL, [Patroni 3.0](https://github.com/zalando/patroni), has [first class support for Citus 10.0 and above](https://patroni.readthedocs.io/en/latest/citus.html#citus), additionally since Citus 11.2 ships with improvements for smoother node switchover in Patroni. An example of patronictl list output for the Citus cluster: ```bash postgres@coord1:~$ patronictl list demo ``` ```text + Citus cluster: demo ----------+--------------+---------+----+-----------+ | Group | Member | Host | Role | State | TL | Lag in MB | +-------+---------+-------------+--------------+---------+----+-----------+ | 0 | coord1 | 172.27.0.10 | Replica | running | 1 | 0 | | 0 | coord2 | 172.27.0.6 | Sync Standby | running | 1 | 0 | | 0 | coord3 | 172.27.0.4 | Leader | running | 1 | | | 1 | work1-1 | 172.27.0.8 | Sync Standby | running | 1 | 0 | | 1 | work1-2 | 172.27.0.2 | Leader | running | 1 | | | 2 | work2-1 | 172.27.0.5 | Sync Standby | running | 1 | 0 | | 2 | work2-2 | 172.27.0.7 | Leader | running | 1 | | +-------+---------+-------------+--------------+---------+----+-----------+ ``` ## Documentation If you’re ready to get started with Citus or want to know more, we recommend reading the [Citus open source documentation](https://docs.citusdata.com/en/stable/). Or, if you are using Citus on Azure, then the [Azure Cosmos DB for PostgreSQL](https://learn.microsoft.com/azure/cosmos-db/postgresql/introduction) is the place to start. Our Citus docs contain comprehensive use case guides on how to build a [multi-tenant SaaS application](https://docs.citusdata.com/en/stable/use_cases/multi_tenant.html), [real-time analytics dashboard]( https://docs.citusdata.com/en/stable/use_cases/realtime_analytics.html), or work with [time series data](https://docs.citusdata.com/en/stable/use_cases/timeseries.html). ## Architecture A Citus database cluster grows from a single PostgreSQL node into a cluster by adding worker nodes. In a Citus cluster, the original node to which the application connects is referred to as the coordinator node. The Citus coordinator contains both the metadata of distributed tables and reference tables, as well as regular (local) tables, sequences, and other database objects (e.g. foreign tables). Data in distributed tables is stored in “shards”, which are actually just regular PostgreSQL tables on the worker nodes. When querying a distributed table on the coordinator node, Citus will send regular SQL queries to the worker nodes. That way, all the usual PostgreSQL optimizations and extensions can automatically be used with Citus. ![Citus architecture](images/citus-architecture.png) When you send a query in which all (co-located) distributed tables have the same filter on the distribution column, Citus will automatically detect that and send the whole query to the worker node that stores the data. That way, arbitrarily complex queries are supported with minimal routing overhead, which is especially useful for scaling transactional workloads. If queries do not have a specific filter, each shard is queried in parallel, which is especially useful in analytical workloads. The Citus distributed executor is adaptive and is designed to handle both query types at the same time on the same system under high concurrency, which enables large-scale mixed workloads. The schema and metadata of distributed tables and reference tables are automatically synchronized to all the nodes in the cluster. That way, you can connect to any node to run distributed queries. Schema changes and cluster administration still need to go through the coordinator. Detailed descriptions of the implementation for Citus developers are provided in the [Citus Technical Documentation](src/backend/distributed/README.md). ## When to use Citus Citus is uniquely capable of scaling both analytical and transactional workloads with up to petabytes of data. Use cases in which Citus is commonly used: - **[Customer-facing analytics dashboards](http://docs.citusdata.com/en/stable/use_cases/realtime_analytics.html)**: Citus enables you to build analytics dashboards that simultaneously ingest and process large amounts of data in the database and give sub-second response times even with a large number of concurrent users. The advanced parallel, distributed query engine in Citus combined with PostgreSQL features such as [array types](https://www.postgresql.org/docs/current/arrays.html), [JSONB](https://www.postgresql.org/docs/current/datatype-json.html), [lateral joins](https://heap.io/blog/engineering/postgresqls-powerful-new-join-type-lateral), and extensions like [HyperLogLog](https://github.com/citusdata/postgresql-hll) and [TopN](https://github.com/citusdata/postgresql-topn) allow you to build responsive analytics dashboards no matter how many customers or how much data you have. Example real-time analytics users: [Algolia](https://www.citusdata.com/customers/algolia) - **[Time series data](http://docs.citusdata.com/en/stable/use_cases/timeseries.html)**: Citus enables you to process and analyze very large amounts of time series data. The biggest Citus clusters store well over a petabyte of time series data and ingest terabytes per day. Citus integrates seamlessly with [Postgres table partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) and has [built-in functions for partitioning by time](https://www.citusdata.com/blog/2021/10/22/how-to-scale-postgres-for-time-series-data-with-citus/), which can speed up queries and writes on time series tables. You can take advantage of Citus’s parallel, distributed query engine for fast analytical queries, and use the built-in *columnar storage* to compress old partitions. Example users: [MixRank](https://www.citusdata.com/customers/mixrank) - **[Software-as-a-service (SaaS) applications](http://docs.citusdata.com/en/stable/use_cases/multi_tenant.html)**: SaaS and other multi-tenant applications need to be able to scale their database as the number of tenants/customers grows. Citus enables you to transparently shard a complex data model by the tenant dimension, so your database can grow along with your business. By distributing tables along a tenant ID column and co-locating data for the same tenant, Citus can horizontally scale complex (tenant-scoped) queries, transactions, and foreign key graphs. Reference tables and distributed DDL commands make database management a breeze compared to manual sharding. On top of that, you have a built-in distributed query engine for doing cross-tenant analytics inside the database. Example multi-tenant SaaS users: [Salesloft](https://fivetran.com/case-studies/replicating-sharded-databases-a-case-study-of-salesloft-citus-data-and-fivetran), [ConvertFlow](https://www.citusdata.com/customers/convertflow) - **[Microservices](https://docs.citusdata.com/en/stable/get_started/tutorial_microservices.html)**: Citus supports schema based sharding, which allows distributing regular database schemas across many machines. This sharding methodology fits nicely with typical Microservices architecture, where storage is fully owned by the service hence can’t share the same schema definition with other tenants. Citus allows distributing horizontally scalable state across services, solving one of the [main problems](https://stackoverflow.blog/2020/11/23/the-macro-problem-with-microservices/) of microservices. - **Geospatial**: Because of the powerful [PostGIS](https://postgis.net/) extension to Postgres that adds support for geographic objects into Postgres, many people run spatial/GIS applications on top of Postgres. And since spatial location information has become part of our daily life, well, there are more geospatial applications than ever. When your Postgres database needs to scale out to handle an increased workload, Citus is a good fit. Example geospatial users: [Helsinki Regional Transportation Authority (HSL)](https://customers.microsoft.com/story/845146-transit-authority-improves-traffic-monitoring-with-azure-database-for-postgresql-hyperscale), [MobilityDB](https://www.citusdata.com/blog/2020/11/09/analyzing-gps-trajectories-at-scale-with-postgres-mobilitydb/). ## Need Help? - **Slack**: Ask questions in our Citus community [Slack channel](https://slack.citusdata.com). - **GitHub issues**: Please submit issues via [GitHub issues](https://github.com/citusdata/citus/issues). - **Documentation**: Our [Citus docs](https://docs.citusdata.com ) have a wealth of resources, including sections on [query performance tuning](https://docs.citusdata.com/en/stable/performance/performance_tuning.html), [useful diagnostic queries](https://docs.citusdata.com/en/stable/admin_guide/diagnostic_queries.html), and [common error messages](https://docs.citusdata.com/en/stable/reference/common_errors.html). - **Docs issues**: You can also submit documentation issues via [GitHub issues for our Citus docs](https://github.com/citusdata/citus_docs/issues). - **Updates & Release Notes**: Learn about what's new in each Citus version on the [Citus Updates page](https://www.citusdata.com/updates/). ## Contributing Citus is built on and of open source, and we welcome your contributions. The [CONTRIBUTING.md](CONTRIBUTING.md) file explains how to get started developing the Citus extension itself and our code quality guidelines. ## Code of Conduct This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ## Stay Connected - **Twitter**: Follow us [@citusdata](https://twitter.com/citusdata) to track the latest posts & updates on what’s happening. - **Citus Blog**: Read our popular [Citus Open Source Blog](https://www.citusdata.com/blog/) for posts about PostgreSQL and Citus. - **Citus Newsletter**: Subscribe to our monthly technical [Citus Newsletter](https://www.citusdata.com/join-newsletter) to get a curated collection of our favorite posts, videos, docs, talks, & other Postgres goodies. - **Slack**: Our [Citus Public slack](https://slack.citusdata.com/) is a good way to stay connected, not just with us but with other Citus users. - **Sister Blog**: Read the PostgreSQL posts on the [Azure Cosmos DB for PostgreSQL blog](https://devblogs.microsoft.com/cosmosdb/category/postgresql/) about our managed service on Azure. - **Videos**: Check out this [YouTube playlist](https://www.youtube.com/playlist?list=PLixnExCn6lRq261O0iwo4ClYxHpM9qfVy) of some of our favorite Citus videos and demos. If you want to deep dive into how Citus extends PostgreSQL, you might want to check out Marco Slot’s talk at Carnegie Mellon titled [Citus: Distributed PostgreSQL as an Extension](https://youtu.be/X-aAgXJZRqM) that was part of Andy Pavlo’s Vaccination Database Talks series at CMUDB. - **Our other Postgres projects**: Our team also works on other awesome PostgreSQL open source extensions & projects, including: [pg_cron](https://github.com/citusdata/pg_cron), [HyperLogLog](https://github.com/citusdata/postgresql-hll), [TopN](https://github.com/citusdata/postgresql-topn), [pg_auto_failover](https://github.com/citusdata/pg_auto_failover), [activerecord-multi-tenant](https://github.com/citusdata/activerecord-multi-tenant), and [django-multitenant](https://github.com/citusdata/django-multitenant). ___ Copyright © Citus Data, Inc. ================================================ FILE: SECURITY.md ================================================ ## Security Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. ## Reporting Security Issues **Please do not report security vulnerabilities through public GitHub issues.** Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) * Full paths of source file(s) related to the manifestation of the issue * The location of the affected source code (tag/branch/commit or direct URL) * Any special configuration required to reproduce the issue * Step-by-step instructions to reproduce the issue * Proof-of-concept or exploit code (if possible) * Impact of the issue, including how an attacker might exploit the issue This information will help us triage your report more quickly. If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. ## Preferred Languages We prefer all communications to be in English. ## Policy Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). ================================================ FILE: STYLEGUIDE.md ================================================ # Coding style The existing code-style in our code-base is not super consistent. There are multiple reasons for that. One big reason is because our code-base is relatively old and our standards have changed over time. The second big reason is that our style-guide is different from style-guide of Postgres and some code is copied from Postgres source code and is slightly modified. The below rules are for new code. If you're changing existing code that uses a different style, use your best judgement to decide if you use the rules here or if you match the existing style. ## Using citus_indent CI pipeline will automatically reject any PRs which do not follow our coding conventions. The easiest way to ensure your PR adheres to those conventions is to use the [citus_indent](https://github.com/citusdata/tools/tree/develop/uncrustify) tool. This tool uses `uncrustify` under the hood. ```bash # Uncrustify changes the way it formats code every release a bit. To make sure # everyone formats consistently we use version 0.82.0: curl -L https://github.com/uncrustify/uncrustify/archive/uncrustify-0.82.0.tar.gz | tar xz cd uncrustify-uncrustify-0.82.0/ mkdir build cd build cmake .. make -j5 sudo make install cd ../.. git clone https://github.com/citusdata/tools.git cd tools make uncrustify/.install ``` Once you've done that, you can run the `make reindent` command from the top directory to recursively check and correct the style of any source files in the current directory. Under the hood, `make reindent` will run `citus_indent` and some other style corrections for you. You can also run the following in the directory of this repository to automatically format all the files that you have changed before committing: ```bash cat > .git/hooks/pre-commit << __EOF__ #!/bin/bash citus_indent --check --diff || { citus_indent --diff; exit 1; } __EOF__ chmod +x .git/hooks/pre-commit ``` ## Other rules we follow that citus_indent does not enforce * We almost always use **CamelCase**, when naming functions, variables etc., **not snake_case**. * We also have the habits of using a **lowerCamelCase** for some variables named from their type or from their function name, as shown in the examples: ```c bool IsCitusExtensionLoaded = false; bool IsAlterTableRenameStmt(RenameStmt *renameStmt) { AlterTableCmd *alterTableCommand = NULL; .. .. bool isAlterTableRenameStmt = false; .. } ``` * We **start functions with a comment**: ```c /* * MyNiceFunction .. * .. * .. */ MyNiceFunction(..) { .. .. } ``` * `#includes` needs to be sorted based on below ordering and then alphabetically and we should not include what we don't need in a file: * System includes (eg. #include<...>) * Postgres.h (eg. #include "postgres.h") * Toplevel imports from postgres, not contained in a directory (eg. #include "miscadmin.h") * General postgres includes (eg . #include "nodes/...") * Toplevel citus includes, not contained in a directory (eg. #include "citus_verion.h") * Columnar includes (eg. #include "columnar/...") * Distributed includes (eg. #include "distributed/...") * Comments: ```c /* single line comments start with a lower-case */ /* * We start multi-line comments with a capital letter * and keep adding a star to the beginning of each line * until we close the comment with a star and a slash. */ ``` * Order of function implementations and their declarations in a file: We define static functions after the functions that call them. For example: ```c #include<..> #include<..> .. .. typedef struct { .. .. } MyNiceStruct; .. .. PG_FUNCTION_INFO_V1(my_nice_udf1); PG_FUNCTION_INFO_V1(my_nice_udf2); .. .. // .. somewhere on top of the file … static void MyNiceStaticlyDeclaredFunction1(…); static void MyNiceStaticlyDeclaredFunction2(…); .. .. void MyNiceFunctionExternedViaHeaderFile(..) { .. .. MyNiceStaticlyDeclaredFunction1(..); .. .. MyNiceStaticlyDeclaredFunction2(..); .. } .. .. // we define this first because it's called by MyNiceFunctionExternedViaHeaderFile() // before MyNiceStaticlyDeclaredFunction2() static void MyNiceStaticlyDeclaredFunction1(…) { } .. .. // then we define this static void MyNiceStaticlyDeclaredFunction2(…) { } ``` ================================================ FILE: aclocal.m4 ================================================ dnl aclocal.m4 m4_include([config/general.m4]) ================================================ FILE: autogen.sh ================================================ #!/bin/bash # # autogen.sh converts configure.ac to configure and creates # citus_config.h.in. The resuting resulting files are checked into # the SCM, to avoid everyone needing autoconf installed. autoreconf -f ================================================ FILE: cgmanifest.json ================================================ { "Registrations": [ { "Component": { "Type": "git", "git": { "RepositoryUrl": "https://github.com/intel/safestringlib", "CommitHash": "245c4b8cff1d2e7338b7f3a82828fc8e72b29549" } }, "DevelopmentDependency": false }, { "Component": { "Type": "git", "git": { "RepositoryUrl": "https://github.com/postgres/postgres", "CommitHash": "29be9983a64c011eac0b9ee29895cce71e15ea77" } }, "license": "PostgreSQL", "licenseDetail": [ "Portions Copyright (c) 1996-2010, The PostgreSQL Global Development Group", "", "Portions Copyright (c) 1994, The Regents of the University of California", "", "Permission to use, copy, modify, and distribute this software and its documentation for ", "any purpose, without fee, and without a written agreement is hereby granted, provided ", "that the above copyright notice and this paragraph and the following two paragraphs appear ", "in all copies.", "", "IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, ", "INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS ", "SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE ", "POSSIBILITY OF SUCH DAMAGE.", "", "THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ", "THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED ", "HEREUNDER IS ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE ", "MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS." ], "version": "0.0.1", "DevelopmentDependency": false } ] } ================================================ FILE: ci/README.md ================================================ # CI scripts We have a few scripts that we run in CI to confirm that code confirms to our standards. Be sure you have followed the setup in the [Following our coding conventions](https://github.com/citusdata/citus/blob/master/CONTRIBUTING.md#following-our-coding-conventions) section of `CONTRIBUTING.md`. Once you've done that, most of them should be fixed automatically, when running: ``` make reindent ``` See the sections below for details on what a specific failing script means. ## `citus_indent` We format all our code using the coding conventions in the [citus_indent](https://github.com/citusdata/tools/tree/develop/uncrustify) tool. This tool uses `uncrustify` under the hood. See [Following our coding conventions](https://github.com/citusdata/citus/blob/master/CONTRIBUTING.md#following-our-coding-conventions) on how to install this. ## `editorconfig.sh` You should install the Editorconfig plugin for your editor/IDE https://editorconfig.org/ ## `banned.h.sh` You're using a C library function that is banned by Microsoft, mostly because of risk for buffer overflows. This page lists the Microsoft suggested replacements: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10082#guide These replacements are only available on Windows normally. Since we build for Linux we make most of them available with this header file: ```c #include "distributed/citus_safe_lib.h" ``` This uses https://github.com/intel/safestringlib to provide them. However, still not all of them are available. For those cases we provide some extra functions in `citus_safe_lib.h`, with similar functionality. If none of those replacements match your requirements you have to do one of the following: 1. Add a replacement to `citus_safe_lib.{c,h}` that handles the same error cases that the `{func_name}_s` function that Microsoft suggests. 2. Add a `/* IGNORE-BANNED */` comment to the line that complains. Doing this requires also adding a comment before explaining why this specific use of the function is safe. ## `build-citus.sh` This is the script used during the build phase of the extension. Historically this script was embedded in the docker images. This made maintenance a hassle. Now it lives in tree with the rest of the source code. When this script fails you most likely have a build error on the postgres version it was building at the time of the failure. Fix the compile error and push a new version of your code to fix. ## `check_enterprise_merge.sh` This check exists to make sure that we can always merge the `master` branch of `community` into the `enterprise-master` branch of the `enterprise` repo. There are two conditions in which this check passes: 1. There are no merge conflicts between your PR branch and `enterprise-master` and after this merge the code compiles. 2. There are merge conflicts, but there is a branch with the same name in the enterprise repo that: 1. Contains the last commit of the community branch with the same name. 2. Merges cleanly into `enterprise-master` 3. After merging, the code can be compiled. If the job already passes, you are done, nothing further required! Otherwise follow the below steps. ### Prerequisites Before continuing with the real steps make sure you have done the following (this only needs to be done once): 1. You have enabled `git rerere` in globally or in your enterprise repo ([docs](https://git-scm.com/docs/git-rerere), [very useful blog](https://medium.com/@porteneuve/fix-conflicts-only-once-with-git-rerere-7d116b2cec67#.3vui844dt)): ```bash # Enables it globally for all repos git config --global rerere.enabled true # Enables it only for the enterprise repo cd git config rerere.enabled true ``` 2. You have set up the `community` remote on your enterprise as [described in CONTRIBUTING.md](https://github.com/citusdata/citus-enterprise/blob/enterprise-master/CONTRIBUTING.md#merging-community-changes-onto-enterprise). #### Important notes on `git rerere` This is very useful as it will make sure git will automatically redo merges that you have done before. However, this has a downside too. It will also redo merges that you did, but that were incorrect. Two work around this you can use these commands. 1. Make `git rerere` forget a merge: ```bash git rerere forget ``` 2. During conflict resolution where `git rerere` already applied the bad merge, simply forgetting it is not enough. Since it is already applied. In that case you also have to undo the apply using: ```bash git checkout --conflict=merge ``` ### Actual steps After the prerequisites are met we continue on to the real steps. Say your branch name is `$PR_BRANCH`, we will refer to `$PR_BRANCH` on community as `community/$PR_BRANCH` and on enterprise as `enterprise/$PR_BRANCH`. First make sure these two things are the case: 1. Get approval from your reviewer for `community/$PR_BRANCH`. Only follow the next steps after you are about to merge the branch to community master. 2. Make sure your commits are in a nice state, since you should not do "squash and merge" on Github later. Otherwise you will certainly get duplicate commits and possibly get merge conflicts with enterprise again. Once that's done, you need to create a merged version of your PR branch on the enterprise repo. For example if `community` is added as a remote in your enterprise repo, you can do the following: ```bash export PR_BRANCH= git checkout enterprise-master git pull # Make sure your local enterprise-master is up to date git fetch community # Fetch your up to date branch name git checkout -b "$PR_BRANCH" enterprise-master ``` Now you have X in your enterprise repo, which we refer to as `enterprise/$PR_BRANCH` (even though in git commands you would reference it as `origin/$PR_BRANCH`). This branch is currently the same as `enterprise-master`. First to make review easier, you should merge community master into it. This should apply without any merge conflicts: ```bash git merge community/master ``` Now you need to merge `community/$PR_BRANCH` to `enterprise/$PR_BRANCH`. Solve any conflicts and make sure to remove any parts that should not be in enterprise even though it doesn't have a conflict, on enterprise repository: ```bash git merge "community/$PR_BRANCH" ``` 1. You should push this branch to the enterprise repo. This is so that the job on community will see this branch. 2. Wait until tests on `enterprise/$PR_BRANCH` pass. 3. Create a PR on the enterprise repo for your `enterprise/$PR_BRANCH` branch. 4. You should get approval for the merge conflict changes on `enterprise/$PR_BRANCH`, preferably from the same reviewer as they are familiar with the change. 5. You should rerun the `check-merge-to-enterprise` check on `community/$PR_BRANCH`. You can use re-run from failed option in circle CI. 6. You can now merge the PR on community. Be sure to NOT use "squash and merge", but instead use the regular "merge commit" mode. 7. You can now merge the PR on enterprise. Be sure to NOT use "squash and merge", but instead use the regular "merge commit" mode. The subsequent PRs on community will be able to pass the `check-merge-to-enterprise` check as long as they don't have a conflict with `enterprise-master`. ### What to do when your branch got outdated? So there's one issue that can occur. Your branch will become outdated with master and you have to make it up to date. There are two ways to do this using `git merge` or `git rebase`. As usual, `git merge` is a bit easier than `git rebase`, but clutters git history. This section will explain both. If you don't know which one makes the most sense, start with `git rebase`. It's possible that for whatever reason this doesn't work or becomes very complex, for instance when new merge conflicts appear. Feel free to fall back to `git merge` in that case, by using `git rebase --abort`. #### Updating both branches with `git rebase` In the community repo, first update the outdated branch using `rebase`: ```bash git checkout $PR_BRANCH # Keep a backup in case you want to fallback to the merge approach git checkout -b ${PR_BRANCH}-backup git checkout $PR_BRANCH # Actually update the branch git fetch origin git rebase origin/master git push origin $PR_BRANCH --force-with-lease ``` In the enterprise repo, rebase onto the new community branch with `--preserve-merges`: ```bash git checkout $PR_BRANCH git fetch community git rebase community/$PR_BRANCH --preserve-merges ``` Automatic merge might have failed with the above command. However, because of `git rerere` it should have re-applied your original merge resolution. If this is indeed the case it should show something like this in the output of the previous command (note the `Resolved ...` line): ``` CONFLICT (content): Merge conflict in Resolved '' using previous resolution. Automatic merge failed; fix conflicts and then commit the result. Error redoing merge ``` Confirm that the merge conflict is indeed resolved correctly. In that case you can do the following: ```bash # Add files that were conflicting git add "$(git diff --name-only --diff-filter=U)" git rebase --continue ``` Before pushing you should do a final check that the commit hash of your final non merge commit matches the commit hash that's on the community repo. If that's not the case, you should fallback to the `git merge` approach. ```bash git reset origin/$PR_BRANCH --hard ``` If the commit hashes were as expected, push the branch: ```bash git push origin $PR_BRANCH --force-with-lease ``` #### Updating both branches with `git merge` If you are falling back to the `git merge` approach after trying the `git rebase` approach, you should first restore the original branch on the community repo. ```bash git checkout $PR_BRANCH git reset ${PR_BRANCH}-backup --hard git push origin $PR_BRANCH --force-with-lease ``` In the community repo, first update the outdated branch using `merge`: ```bash git checkout $PR_BRANCH git fetch origin git merge origin/master git push origin $PR_BRANCH ``` In the enterprise repo, merge with the updated `community/$PR_BRANCH`: ```bash git checkout $PR_BRANCH git fetch community git merge community/$PR_BRANCH git push origin $PR_BRANCH ``` ## `check_sql_snapshots.sh` To allow for better diffs during review we have snapshots of SQL UDFs. This means that `latest.sql` is not up to date with the SQL file of the highest version number in the directory. The output of the script shows you what is different. ## `check_all_tests_are_run.sh` A test should always be included in a schedule file, otherwise it will not be run in CI. This is most commonly forgotten for newly added tests. In that case the dev ran it locally without running a full schedule with something like: ```bash make -C src/test/regress/ check-minimal EXTRA_TESTS='multi_create_table_new_features' ``` ## `check_all_ci_scripts_are_run.sh` This is the meta CI script. This checks that all existing CI scripts are actually run in CI. This is most commonly forgotten for newly added CI tests that the developer only ran locally. It also checks that all CI scripts have a section in this `README.md` file and that they include `ci/ci_helpers.sh`. ## `check_migration_files.sh` A branch that touches a set of upgrade scripts is also expected to touch corresponding downgrade scripts as well. If this script fails, read the output and make sure you update the downgrade scripts in the printed list. If you really don't need a downgrade to run any SQL. You can write a comment in the file explaining why a downgrade step is not necessary. ## `disallow_c_comments_in_migrations.sh` We do not use C-style comments in migration files as the stripped zero-length migration files cause warning during packaging. Instead use SQL type comments, i.e: ``` -- this is a comment ``` See [#3115](https://github.com/citusdata/citus/pull/3115) for more info. ## `disallow_hash_comments_in_spec_files.sh` We do not use comments starting with # in spec files because it creates errors from C preprocessor that expects directives after this character. Instead use C type comments, i.e: ``` // this is a single line comment /* * this is a multi line comment */ ``` ## `disallow_long_changelog_entries.sh` Having changelog items with entries that are longer than 80 characters are forbidden. It's allowed to split up the entry over multiple lines, as long as each line of the entry is 80 characters or less. ## `normalize_expected.sh` All files in `src/test/expected` should be committed in normalized form. This error mostly happens if someone added a new normalization rule and you have not rerun tests that you have added. We normalize the test output files using a `sed` script called [`normalize.sed`](https://github.com/citusdata/citus/blob/master/src/test/regress/bin/normalize.sed). The reason for this is that some output changes randomly in ways we don't care about. An example of this is when an error happens on a different port number, or a different worker shard, or a different placement, etc. Either randomly or because we are running the tests in a slightly different configuration. ## `remove_useless_declarations.sh` This script tries to make sure that we don't add useless declarations to our code. What it effectively does is replace this: ```c int a = 0; int b = 2; Assert(b == 2); a = b + b; ``` With this equivalent, but shorter version: ```c int b = 2; Assert(b == 2); int a = b + b; ``` It relies on the fact that `citus_indent` formats our code in certain ways. So before running this script, make sure that you've done that. This replacement is all done using a [regex replace](xkcd.com/1171), so it's definitely possible there's a bug in there. So far no bad ones have been found. A known issue is that it does not replace code in a block after an `#ifdef` like this. ```c int foo = 0; #ifdef SOMETHING foo = 1 #else foo = 2 #endif ``` This was deemed to be error prone and not worth the effort. ## `fix_gitignore.sh` This script checks and fixes issues with `.gitignore` rules: 1. Makes sure we do not commit any generated files that should be ignored. If there is an ignored file in the git tree, the user is expected to review the files that are removed from the git tree and commit them. ## `check_gucs_are_alphabetically_sorted.sh` This script checks the order of the GUCs defined in `shared_library_init.c`. To solve this failure, please check `shared_library_init.c` and make sure that the GUC definitions are in alphabetical order. ## `print_stack_trace.sh` This script prints stack traces for failed tests, if they left core files. ## `sort_and_group_includes.sh` This script checks and fixes issues with include grouping and sorting in C files. Includes are grouped in the following groups: - System includes (eg. `#include `) - Postgres.h include (eg. `#include "postgres.h"`) - Toplevel postgres includes (includes not in a directory eg. `#include "miscadmin.h`) - Postgres includes in a directory (eg. `#include "catalog/pg_type.h"`) - Toplevel citus includes (includes not in a directory eg. `#include "pg_version_constants.h"`) - Columnar includes (eg. `#include "columnar/columnar.h"`) - Distributed includes (eg. `#include "distributed/maintenanced.h"`) Within every group the include lines are sorted alphabetically. ================================================ FILE: ci/banned.h.sh ================================================ #!/bin/bash # Checks for the APIs that are banned by microsoft. Since we compile for Linux # we use the replacements from https://github.com/intel/safestringlib # Not all replacement functions are available in safestringlib. If it doesn't # exist and you cannot rewrite the code to not use the banned API, then you can # add a comment containing "IGNORE-BANNED" to the line where the error is and # this check will ignore that match. # # The replacement function that you should use are listed here: # https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10082#guide set -eu # shellcheck disable=SC1091 source ci/ci_helpers.sh files=$(find src -iname '*.[ch]' | git check-attr --stdin citus-style | grep -v ': unset$' | sed 's/: citus-style: set$//') # grep is allowed to fail, that means no banned matches are found set +e # Required banned from banned.h. These functions are not allowed to be used at # all. # shellcheck disable=SC2086 grep -E '\b(strcpy|strcpyA|strcpyW|wcscpy|_tcscpy|_mbscpy|StrCpy|StrCpyA|StrCpyW|lstrcpy|lstrcpyA|lstrcpyW|_tccpy|_mbccpy|_ftcscpy|strcat|strcatA|strcatW|wcscat|_tcscat|_mbscat|StrCat|StrCatA|StrCatW|lstrcat|lstrcatA|lstrcatW|StrCatBuff|StrCatBuffA|StrCatBuffW|StrCatChainW|_tccat|_mbccat|_ftcscat|sprintfW|sprintfA|wsprintf|wsprintfW|wsprintfA|sprintf|swprintf|_stprintf|wvsprintf|wvsprintfA|wvsprintfW|vsprintf|_vstprintf|vswprintf|strncpy|wcsncpy|_tcsncpy|_mbsncpy|_mbsnbcpy|StrCpyN|StrCpyNA|StrCpyNW|StrNCpy|strcpynA|StrNCpyA|StrNCpyW|lstrcpyn|lstrcpynA|lstrcpynW|strncat|wcsncat|_tcsncat|_mbsncat|_mbsnbcat|StrCatN|StrCatNA|StrCatNW|StrNCat|StrNCatA|StrNCatW|lstrncat|lstrcatnA|lstrcatnW|lstrcatn|gets|_getts|_gettws|IsBadWritePtr|IsBadHugeWritePtr|IsBadReadPtr|IsBadHugeReadPtr|IsBadCodePtr|IsBadStringPtr|memcpy|RtlCopyMemory|CopyMemory|wmemcpy|lstrlen)\(' $files \ | grep -v "IGNORE-BANNED" \ && echo "ERROR: Required banned API usage detected" && exit 1 # Required banned from table on liquid. These functions are not allowed to be # used at all. # shellcheck disable=SC2086 grep -E '\b(strcat|strcpy|strerror|strncat|strncpy|strtok|wcscat|wcscpy|wcsncat|wcsncpy|wcstok|fprintf|fwprintf|printf|snprintf|sprintf|swprintf|vfprintf|vprintf|vsnprintf|vsprintf|vswprintf|vwprintf|wprintf|fscanf|fwscanf|gets|scanf|sscanf|swscanf|vfscanf|vfwscanf|vscanf|vsscanf|vswscanf|vwscanf|wscanf|asctime|atof|atoi|atol|atoll|bsearch|ctime|fopen|freopen|getenv|gmtime|localtime|mbsrtowcs|mbstowcs|memcpy|memmove|qsort|rewind|setbuf|wmemcpy|wmemmove)\(' $files \ | grep -v "IGNORE-BANNED" \ && echo "ERROR: Required banned API usage from table detected" && exit 1 # Recommended banned from banned.h. If you can change the code not to use these # that would be great. You can use IGNORE-BANNED if you need to use it anyway. # You can also remove it from the regex, if you want to mark the API as allowed # throughout the codebase (to not have to add IGNORED-BANNED everywhere). In # that case note it in this comment that you did so. # shellcheck disable=SC2086 grep -E '\b(wnsprintf|wnsprintfA|wnsprintfW|_snwprintf|_snprintf|_sntprintf|_vsnprintf|vsnprintf|_vsnwprintf|_vsntprintf|wvnsprintf|wvnsprintfA|wvnsprintfW|strtok|_tcstok|wcstok|_mbstok|makepath|_tmakepath| _makepath|_wmakepath|_splitpath|_tsplitpath|_wsplitpath|scanf|wscanf|_tscanf|sscanf|swscanf|_stscanf|snscanf|snwscanf|_sntscanf|_itoa|_itow|_i64toa|_i64tow|_ui64toa|_ui64tot|_ui64tow|_ultoa|_ultot|_ultow|CharToOem|CharToOemA|CharToOemW|OemToChar|OemToCharA|OemToCharW|CharToOemBuffA|CharToOemBuffW|alloca|_alloca|ChangeWindowMessageFilter)\(' $files \ | grep -v "IGNORE-BANNED" \ && echo "ERROR: Recomended banned API usage detected" && exit 1 # Recommended banned from table on liquid. If you can change the code not to use these # that would be great. You can use IGNORE-BANNED if you need to use it anyway. # You can also remove it from the regex, if you want to mark the API as allowed # throughout the codebase (to not have to add IGNORED-BANNED everywhere). In # that case note it in this comment that you did so. # Banned APIs ignored throughout the codebase: # - strlen # shellcheck disable=SC2086 grep -E '\b(alloca|getwd|mktemp|tmpnam|wcrtomb|wcrtombs|wcslen|wcsrtombs|wcstombs|wctomb|class_addMethod|class_replaceMethod)\(' $files \ | grep -v "IGNORE-BANNED" \ && echo "ERROR: Recomended banned API usage detected" && exit 1 exit 0 ================================================ FILE: ci/build-citus.sh ================================================ #!/bin/bash # make bash behave set -euo pipefail IFS=$'\n\t' # shellcheck disable=SC1091 source ci/ci_helpers.sh # read pg major version, error if not provided PG_MAJOR=${PG_MAJOR:?please provide the postgres major version} # get codename from release file . /etc/os-release codename=${VERSION#*(} codename=${codename%)*} # we'll do everything with absolute paths basedir="$(pwd)" # get the project and clear out the git repo (reduce workspace size rm -rf "${basedir}/.git" build_ext() { pg_major="$1" builddir="${basedir}/build-${pg_major}" echo "Beginning build for PostgreSQL ${pg_major}..." >&2 # do everything in a subdirectory to avoid clutter in current directory mkdir -p "${builddir}" && cd "${builddir}" CFLAGS=-Werror "${basedir}/configure" PG_CONFIG="/usr/lib/postgresql/${pg_major}/bin/pg_config" --enable-coverage --with-security-flags installdir="${builddir}/install" make -j$(nproc) && mkdir -p "${installdir}" && { make DESTDIR="${installdir}" install-all || make DESTDIR="${installdir}" install ; } cd "${installdir}" && find . -type f -print > "${builddir}/files.lst" tar cvf "${basedir}/install-${pg_major}.tar" `cat ${builddir}/files.lst` cd "${builddir}" && rm -rf install files.lst && make clean } build_ext "${PG_MAJOR}" ================================================ FILE: ci/check_all_ci_scripts_are_run.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # 1. Find all *.sh files in the ci directory # 2. Strip the directory # 3. Exclude some scripts that we should not run in CI directly ci_scripts=$( find ci/ -iname "*.sh" | sed -E 's#^ci/##g' | grep -v -E '^(ci_helpers.sh|fix_style.sh)$' ) for script in $ci_scripts; do if ! grep "\\bci/$script\\b" -r .github > /dev/null; then echo "ERROR: CI script with name \"$script\" is not actually used in .github folder" exit 1 fi if ! grep "^## \`$script\`\$" ci/README.md > /dev/null; then echo "ERROR: CI script with name \"$script\" does not have a section in ci/README.md" exit 1 fi if ! grep "source ci/ci_helpers.sh" "ci/$script" > /dev/null; then echo "ERROR: CI script with name \"$script\" does not include ci/ci_helpers.sh" exit 1 fi done ================================================ FILE: ci/check_all_tests_are_run.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh cd src/test/regress # 1. Find all *.sql and *.spec files in the sql, and spec directories # 2. Strip the extension and the directory # 3. Ignore names that end with .include, those files are meant to be in an C # preprocessor #include statement. They should not be in schedules. test_names=$( find sql spec -iname "*.sql" -o -iname "*.spec" | sed -E 's#^\w+/([^/]+)\.[^.]+$#\1#g' | grep -v '.include$' ) for name in $test_names; do if ! grep "\\b$name\\b" ./*_schedule > /dev/null; then echo "ERROR: Test with name \"$name\" is not used in any of the schedule files" exit 1 fi done ================================================ FILE: ci/check_gucs_are_alphabetically_sorted.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # Find the line that exactly matches "RegisterCitusConfigVariables(void)" in # shared_library_init.c. grep command returns something like # "934:RegisterCitusConfigVariables(void)" and we extract the line number # with cut. RegisterCitusConfigVariables_begin_linenumber=$(grep -n "^RegisterCitusConfigVariables(void)$" src/backend/distributed/shared_library_init.c | cut -d: -f1) # Consider the lines starting from $RegisterCitusConfigVariables_begin_linenumber, # grep the first line that starts with "}" and extract the line number with cut # as in the previous step. RegisterCitusConfigVariables_length=$(tail -n +$RegisterCitusConfigVariables_begin_linenumber src/backend/distributed/shared_library_init.c | grep -n -m 1 "^}$" | cut -d: -f1) # extract the function definition of RegisterCitusConfigVariables into a temp file tail -n +$RegisterCitusConfigVariables_begin_linenumber src/backend/distributed/shared_library_init.c | head -n $(($RegisterCitusConfigVariables_length)) > RegisterCitusConfigVariables_func_def.out # extract citus gucs in the form of "citus.X" grep -P "^[\t][\t]\"citus\.[a-zA-Z_0-9]+\"" RegisterCitusConfigVariables_func_def.out > gucs.out LC_COLLATE=C sort -c gucs.out rm gucs.out rm RegisterCitusConfigVariables_func_def.out ================================================ FILE: ci/check_migration_files.sh ================================================ #! /bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # This file checks for the existence of downgrade scripts for every upgrade script that is changed in the branch. # create list of migration files for upgrades upgrade_files=$(git diff --name-only origin/main | { grep "src/backend/distributed/sql/citus--.*sql" || exit 0 ; }) downgrade_files=$(git diff --name-only origin/main | { grep "src/backend/distributed/sql/downgrades/citus--.*sql" || exit 0 ; }) ret_value=0 for file in $upgrade_files do # There should always be 2 matches, and no need to avoid splitting here # shellcheck disable=SC2207 versions=($(grep --only-matching --extended-regexp "[0-9]+\.[0-9]+[-.][0-9]+" <<< "$file")) from_version=${versions[0]}; to_version=${versions[1]}; downgrade_migration_file="src/backend/distributed/sql/downgrades/citus--$to_version--$from_version.sql" # check for the existence of migration scripts if [[ $(grep --line-regexp --count "$downgrade_migration_file" <<< "$downgrade_files") == 0 ]] then echo "$file is updated, but $downgrade_migration_file is not updated in branch" ret_value=1 fi done exit $ret_value; ================================================ FILE: ci/check_sql_snapshots.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh for udf_dir in src/backend/distributed/sql/udfs/* src/backend/columnar/sql/udfs/*; do # We want to find the last snapshotted sql file, to make sure it's the same # as "latest.sql". This is done by: # 1. Getting the filenames in the UDF directory (using find instead of ls, to keep shellcheck happy) # 2. Filter out latest.sql # 3. Sort using "version sort" # 4. Get the last one using tail latest_snapshot=$(\ find "$udf_dir" -iname "*.sql" -exec basename {} \; \ | { grep --invert-match latest.sql || true; } \ | sort --version-sort \ | tail --lines 1); diff --unified --color=auto "$udf_dir/latest.sql" "$udf_dir/$latest_snapshot"; \ done ================================================ FILE: ci/ci_helpers.sh ================================================ #!/bin/bash # For echo commands "set -x" would show the message effectively twice. Once as # part of the echo command shown by "set -x" and once because of the output of # the echo command. We do not want "set -x" to show the echo command. We only # want to see the actual message in the output of echo itself. This function is # a trick to do so. Read the StackOverflow post below to understand why this # works and what this works around. # Source: https://superuser.com/a/1141026/242593 shopt -s expand_aliases alias echo='{ save_flags="$-"; set +x;} 2> /dev/null && echo_and_restore' echo_and_restore() { builtin echo "$*" #shellcheck disable=SC2154 case "$save_flags" in (*x*) set -x esac } # Make sure that on a failing exit we show a useful message hint_on_fail() { exit_code=$? # Get filename of the currently running script # Source: https://stackoverflow.com/a/192337/2570866 filename=$(basename "$0") if [ $exit_code != 0 ]; then echo "HINT: To solve this failure look here: https://github.com/citusdata/citus/blob/master/ci/README.md#$filename" fi exit $exit_code } trap hint_on_fail EXIT ================================================ FILE: ci/disallow_c_comments_in_migrations.sh ================================================ #! /bin/bash set -euo pipefail # make ** match all directories and subdirectories shopt -s globstar # shellcheck disable=SC1091 source ci/ci_helpers.sh # We do not use c-style comments in migration files as the stripped # zero-length migration files cause warning during packaging # See #3115 for more info # In this file, we aim to keep the indentation intact by capturing whitespaces, # and reusing them if needed. GNU sed unfortunately does not support lookaround assertions. # /* -> -- find src/backend/{distributed,columnar}/sql/**/*.sql -print0 | xargs -0 sed -i 's#/\*#--#g' # */ -> `` (empty string) # remove all whitespaces immediately before the match find src/backend/{distributed,columnar}/sql/**/*.sql -print0 | xargs -0 sed -i 's#\s*\*/\s*##g' # * -> -- # keep the indentation # allow only whitespaces before the match find src/backend/{distributed,columnar}/sql/**/*.sql -print0 | xargs -0 sed -i 's#^\(\s*\) \*#\1--#g' # // -> -- # do not touch http:// or similar by allowing only whitespaces before // find src/backend/{distributed,columnar}/sql/**/*.sql -print0 | xargs -0 sed -i 's#^\(\s*\)//#\1--#g' ================================================ FILE: ci/disallow_hash_comments_in_spec_files.sh ================================================ #! /bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # We do not use comments starting with # in spec files because it creates warnings from # preprocessor that expects directives after this character. # `# ` -> `-- ` find src/test/regress/spec/*.spec -print0 | xargs -0 sed -i 's!# !// !g' ================================================ FILE: ci/disallow_long_changelog_entries.sh ================================================ #! /bin/bash set -eu # shellcheck disable=SC1091 source ci/ci_helpers.sh # Having changelog items with entries that are longer than 80 characters are forbidden. # Find all lines with disallowed length, and for all such lines store # - line number # - length of the line # - the line content too_long_lines=$(awk 'length() > 80 {print NR,"(",length(),"characters ) :",$0}' CHANGELOG.md) if [[ -n $too_long_lines ]] then echo "We allow at most 80 characters in CHANGELOG.md." echo "${too_long_lines}" exit 1 fi ================================================ FILE: ci/editorconfig.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh for f in $(git ls-tree -r HEAD --name-only); do if [ "$f" = "${f%.out}" ] && [ "$f" = "${f%.data}" ] && [ "$f" = "${f%.png}" ] && [ -f "$f" ] && [ "$(echo "$f" | cut -d / -f1)" != "vendor" ] && [ "$(dirname "$f")" != "src/test/regress/output" ] then # Trim trailing whitespace sed -e 's/[[:space:]]*$//' -i "./$f" # Add final newline if not there if [ -n "$(tail -c1 "$f")" ]; then echo >> "$f" fi fi done ================================================ FILE: ci/fix_gitignore.sh ================================================ #! /bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # Remove all the ignored files from git tree, and error out # find all ignored files in git tree, and use quotation marks to prevent word splitting on filenames with spaces in them # NOTE: Option --cached is needed to avoid a bug in git ls-files command. ignored_lines_in_git_tree=$(git ls-files --ignored --cached --exclude-standard | sed 's/.*/"&"/') if [[ -n $ignored_lines_in_git_tree ]] then echo "Ignored files should not be in git tree!" echo "${ignored_lines_in_git_tree}" echo "Removing these files from git tree, please review and commit" echo "$ignored_lines_in_git_tree" | xargs git rm -r --cached exit 1 fi ================================================ FILE: ci/fix_style.sh ================================================ #!/bin/sh # fail if trying to reference a variable that is not set. set -u / set -o nounset # exit immediately if a command fails set -e cidir="${0%/*}" cd ${cidir}/.. citus_indent . --quiet black . --quiet isort . --quiet ci/editorconfig.sh ci/remove_useless_declarations.sh ci/disallow_c_comments_in_migrations.sh ci/disallow_hash_comments_in_spec_files.sh ci/disallow_long_changelog_entries.sh ci/normalize_expected.sh ci/fix_gitignore.sh ci/print_stack_trace.sh ci/sort_and_group_includes.sh ================================================ FILE: ci/include_grouping.py ================================================ #!/usr/bin/env python3 """ easy command line to run against all citus-style checked files: $ git ls-files \ | git check-attr --stdin citus-style \ | grep 'citus-style: set' \ | awk '{print $1}' \ | cut -d':' -f1 \ | xargs -n1 ./ci/include_grouping.py """ import collections import os import sys def main(args): if len(args) < 2: print("Usage: include_grouping.py ") return file = args[1] if not os.path.isfile(file): sys.exit(f"File '{file}' does not exist") with open(file, "r") as in_file: with open(file + ".tmp", "w") as out_file: includes = [] skipped_lines = [] # This calls print_sorted_includes on a set of consecutive #include lines. # This implicitly keeps separation of any #include lines that are contained in # an #ifdef, because it will order the #include lines inside and after the # #ifdef completely separately. for line in in_file: # if a line starts with #include we don't want to print it yet, instead we # want to collect all consecutive #include lines if line.startswith("#include"): includes.append(line) skipped_lines = [] continue # if we have collected any #include lines, we want to print them sorted # before printing the current line. However, if the current line is empty # we want to perform a lookahead to see if the next line is an #include. # To maintain any separation between #include lines and their subsequent # lines we keep track of all lines we have skipped inbetween. if len(includes) > 0: if len(line.strip()) == 0: skipped_lines.append(line) continue # we have includes that need to be grouped before printing the current # line. print_sorted_includes(includes, file=out_file) includes = [] # print any skipped lines print("".join(skipped_lines), end="", file=out_file) skipped_lines = [] print(line, end="", file=out_file) # move out_file to file os.rename(file + ".tmp", file) def print_sorted_includes(includes, file=sys.stdout): default_group_key = 1 groups = collections.defaultdict(set) # define the groups that we separate correctly. The matchers are tested in the order # of their priority field. The first matcher that matches the include is used to # assign the include to a group. # The groups are printed in the order of their group_key. matchers = [ { "name": "system includes", "matcher": lambda x: x.startswith("<"), "group_key": -2, "priority": 0, }, { "name": "toplevel postgres includes", "matcher": lambda x: "/" not in x, "group_key": 0, "priority": 9, }, { "name": "postgres.h", "matcher": lambda x: x.strip() in ['"postgres.h"'], "group_key": -1, "priority": -1, }, { "name": "toplevel citus inlcudes", "matcher": lambda x: x.strip() in [ '"citus_version.h"', '"pg_version_compat.h"', '"pg_version_constants.h"', ], "group_key": 3, "priority": 0, }, { "name": "columnar includes", "matcher": lambda x: x.startswith('"columnar/'), "group_key": 4, "priority": 1, }, { "name": "distributed includes", "matcher": lambda x: x.startswith('"distributed/'), "group_key": 5, "priority": 1, }, ] matchers.sort(key=lambda x: x["priority"]) # throughout our codebase we have some includes where either postgres or citus # includes are wrongfully included with the syntax for system includes. Before we # try to match those we will change the <> to "" to make them match our system. This # will also rewrite the include to the correct syntax. common_system_include_error_prefixes = [" 0: print(file=file) includes = group[1] print("".join(sorted(includes)), end="", file=file) if __name__ == "__main__": main(sys.argv) ================================================ FILE: ci/normalize_expected.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh for f in $(git ls-tree -r HEAD --name-only src/test/regress/expected/*.out); do sed -Ef src/test/regress/bin/normalize.sed < "$f" > "$f.modified" mv "$f.modified" "$f" done ================================================ FILE: ci/print_stack_trace.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh # find all core files core_files=( $(find . -type f -regex .*core.*\d*.*postgres) ) if [ ${#core_files[@]} -gt 0 ]; then # print stack traces for the core files for core_file in "${core_files[@]}" do # set print frame-arguments all: show all scalars + structures in the frame # set print pretty on: show structures in indented mode # set print addr off: do not show pointer address # thread apply all bt full: show stack traces for all threads gdb --batch \ -ex "set print frame-arguments all" \ -ex "set print pretty on" \ -ex "set print addr off" \ -ex "thread apply all bt full" \ postgres "${core_file}" done fi ================================================ FILE: ci/remove_useless_declarations.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh files=$(find src -iname '*.c' -type f | git check-attr --stdin citus-style | grep -v ': unset$' | sed 's/: citus-style: set$//') while true; do # A visual version of this regex can be seen here (it is MUCH clearer): # https://www.debuggex.com/r/XodMNE9auT9e-bTx # This visual version only contains the search bit, the replacement bit is # quite simple. It looks like when extracted from the command below: # \n$+{code_between}\t$+{type}$+{variable} = # shellcheck disable=SC2086 perl -i -p0e 's/\n\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t)?(?=\b(?P=variable)\b))(?<=\n\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t$+{type}$+{variable} =/sg' $files # The following are simply the same regex, but repeated for different # indentation levels, i.e. finding declarations indented using 2, 3, 4, 5 # and 6 tabs. More than 6 don't really occur in the wild. # (this is needed because variable sized backtracking is not supported in perl) # shellcheck disable=SC2086 perl -i -p0e 's/\n\t\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t\t)?(?=\b(?P=variable)\b))(?<=\n\t\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t\t$+{type}$+{variable} =/sg' $files # shellcheck disable=SC2086 perl -i -p0e 's/\n\t\t\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t\t\t)?(?=\b(?P=variable)\b))(?<=\n\t\t\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t\t\t$+{type}$+{variable} =/sg' $files # shellcheck disable=SC2086 perl -i -p0e 's/\n\t\t\t\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t\t\t\t)?(?=\b(?P=variable)\b))(?<=\n\t\t\t\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t\t\t\t$+{type}$+{variable} =/sg' $files # shellcheck disable=SC2086 perl -i -p0e 's/\n\t\t\t\t\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t\t\t\t\t)?(?=\b(?P=variable)\b))(?<=\n\t\t\t\t\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t\t\t\t\t$+{type}$+{variable} =/sg' $files # shellcheck disable=SC2086 perl -i -p0e 's/\n\t\t\t\t\t\t(?!return )(?P(\w+ )+\**)(?>(?P\w+)( = *[\w>\s\n-]*?)?;\n(?P(?>(?P\/\*.*?\*\/|"(?>\\"|.)*?"|[^#]))*?)(\t\t\t\t\t\t)?(?=\b(?P=variable)\b))(?<=\n\t\t\t\t\t\t)(?P=variable) =(?![^;]*?[^>_]\b(?P=variable)\b[^_])/\n$+{code_between}\t\t\t\t\t\t$+{type}$+{variable} =/sg' $files # shellcheck disable=SC2086 git diff --quiet $files && break; # shellcheck disable=SC2086 git add $files; done ================================================ FILE: ci/sort_and_group_includes.sh ================================================ #!/bin/bash set -euo pipefail # shellcheck disable=SC1091 source ci/ci_helpers.sh git ls-files \ | git check-attr --stdin citus-style \ | grep 'citus-style: set' \ | awk '{print $1}' \ | cut -d':' -f1 \ | xargs -n1 ./ci/include_grouping.py ================================================ FILE: config/config.guess ================================================ #! /bin/sh # Attempt to guess a canonical system name. # Copyright 1992-2017 Free Software Foundation, Inc. timestamp='2017-09-26' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: # https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # # Please send patches to . me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] Output the configuration name of the system \`$me' is run on. Operation modes: -h, --help print this help, then exit -t, --time-stamp print date of last modification, then exit -v, --version print version number, then exit Report bugs and patches to ." version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" Try \`$me --help' for more information." # Parse command line while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) echo "$timestamp" ; exit ;; --version | -v ) echo "$version" ; exit ;; --help | --h* | -h ) echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. break ;; -* ) echo "$me: invalid option $1$help" >&2 exit 1 ;; * ) break ;; esac done if test $# != 0; then echo "$me: too many arguments$help" >&2 exit 1 fi trap 'exit 1' 1 2 15 # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. # Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still # use `HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. set_cc_for_build=' trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; : ${TMPDIR=/tmp} ; { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; dummy=$tmp/dummy ; tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; case $CC_FOR_BUILD,$HOST_CC,$CC in ,,) echo "int x;" > $dummy.c ; for c in cc gcc c89 c99 ; do if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then CC_FOR_BUILD="$c"; break ; fi ; done ; if test x"$CC_FOR_BUILD" = x ; then CC_FOR_BUILD=no_compiler_found ; fi ;; ,,*) CC_FOR_BUILD=$CC ;; ,*,*) CC_FOR_BUILD=$HOST_CC ;; esac ; set_cc_for_build= ;' # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) if (test -f /.attbin/uname) >/dev/null 2>&1 ; then PATH=$PATH:/.attbin ; export PATH fi UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown case "${UNAME_SYSTEM}" in Linux|GNU|GNU/*) # If the system lacks a compiler, then just pick glibc. # We could probably try harder. LIBC=gnu eval $set_cc_for_build cat <<-EOF > $dummy.c #include #if defined(__UCLIBC__) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc #else LIBC=gnu #endif EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` ;; esac # Note: order is significant - the case branches are not exclusive. case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward # compatibility and a consistent mechanism for selecting the # object file format. # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ /sbin/$sysctl 2>/dev/null || \ /usr/sbin/$sysctl 2>/dev/null || \ echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` machine=${arch}${endian}-unknown ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently (or will in the future) and ABI. case "${UNAME_MACHINE_ARCH}" in earm*) os=netbsdelf ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? os=netbsd else os=netbsdelf fi ;; *) os=netbsd ;; esac # Determine ABI tags. case "${UNAME_MACHINE_ARCH}" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` ;; esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. case "${UNAME_VERSION}" in Debian*) release='-gnu' ;; *) release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} exit ;; *:LibertyBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} exit ;; *:SolidBSD:*:*) echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} exit ;; macppc:MirBSD:*:*) echo powerpc-unknown-mirbsd${UNAME_RELEASE} exit ;; *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} exit ;; *:Sortix:*:*) echo ${UNAME_MACHINE}-unknown-sortix exit ;; *:Redox:*:*) echo ${UNAME_MACHINE}-unknown-redox exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on # OSF/1 and Tru64 systems produced since 1995. I hope that # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") UNAME_MACHINE=alpha ;; "EV4.5 (21064)") UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") UNAME_MACHINE=alpha ;; "EV5 (21164)") UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 exit $exitcode ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition exit ;; *:z/VM:*:*) echo s390-ibm-zvmoe exit ;; *:OS400:*:*) echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} exit ;; arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) echo hppa1.1-hitachi-hiuxmpp exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. if test "`(/bin/universe) 2>/dev/null`" = att ; then echo pyramid-pyramid-sysv3 else echo pyramid-pyramid-bsd fi exit ;; NILE*:*:*:dcosx) echo pyramid-pyramid-svr4 exit ;; DRS?6000:unix:4.0:6*) echo sparc-icl-nx6 exit ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; sun4H:SunOS:5.*:*) echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) echo i386-pc-auroraux${UNAME_RELEASE} exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 fi fi echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in Series*|S4*) UNAME_RELEASE=`uname -v` ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos${UNAME_RELEASE} exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos${UNAME_RELEASE} ;; sun4) echo sparc-sun-sunos${UNAME_RELEASE} ;; esac exit ;; aushp:SunOS:*:*) echo sparc-auspex-sunos${UNAME_RELEASE} exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor # > m68000). The system name ranges from "MiNT" over "FreeMiNT" # to the lowercase version "mint" (or "freemint"). Finally # the system name "TOS" denotes a system which is actually not # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) echo m68k-milan-mint${UNAME_RELEASE} exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) echo m68k-hades-mint${UNAME_RELEASE} exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) echo m68k-unknown-mint${UNAME_RELEASE} exit ;; m68k:machten:*:*) echo m68k-apple-machten${UNAME_RELEASE} exit ;; powerpc:machten:*:*) echo powerpc-apple-machten${UNAME_RELEASE} exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) echo mips-dec-ultrix${UNAME_RELEASE} exit ;; VAX*:ULTRIX*:*:*) echo vax-dec-ultrix${UNAME_RELEASE} exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) echo clipper-intergraph-clix${UNAME_RELEASE} exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #ifdef __cplusplus #include /* for printf() prototype */ int main (int argc, char *argv[]) { #else int main (argc, argv) int argc; char *argv[]; { #endif #if defined (host_mips) && defined (MIPSEB) #if defined (SYSTYPE_SYSV) printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_SVR4) printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); #endif #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); #endif #endif exit (-1); } EOF $CC_FOR_BUILD -o $dummy $dummy.c && dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && SYSTEM_NAME=`$dummy $dummyarg` && { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos${UNAME_RELEASE} exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax exit ;; Motorola:*:4.3:PL8-*) echo powerpc-harris-powermax exit ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) echo powerpc-harris-powermax exit ;; Night_Hawk:Power_UNIX:*:*) echo powerpc-harris-powerunix exit ;; m88k:CX/UX:7*:*) echo m88k-harris-cxux7 exit ;; m88k:*:4*:R4*) echo m88k-motorola-sysv4 exit ;; m88k:*:3*:R3*) echo m88k-motorola-sysv3 exit ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] then if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ [ ${TARGET_BINARY_INTERFACE}x = x ] then echo m88k-dg-dgux${UNAME_RELEASE} else echo m88k-dg-dguxbcs${UNAME_RELEASE} fi else echo i586-dg-dgux${UNAME_RELEASE} fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 exit ;; M88*:*:R3*:*) # Delta 88k system running SVR3 echo m88k-motorola-sysv3 exit ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) echo m88k-tektronix-sysv3 exit ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix exit ;; ia64:AIX:*:*) if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #include main() { if (!__power_pc()) exit(1); puts("powerpc-ibm-aix3.2.5"); exit(0); } EOF if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` then echo "$SYSTEM_NAME" else echo rs6000-ibm-aix3.2.5 fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then echo rs6000-ibm-aix3.2.4 else echo rs6000-ibm-aix3.2 fi exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc fi if [ -x /usr/bin/lslpp ] ; then IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${IBM_ARCH}-ibm-aix${IBM_REV} exit ;; *:AIX:*:*) echo rs6000-ibm-aix exit ;; ibmrt:4.4BSD:*|romp-ibm:BSD:*) echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx exit ;; DPX/2?00:B.O.S.:*:*) echo m68k-bull-sysv3 exit ;; 9000/[34]??:4.3bsd:1.*:*) echo m68k-hp-bsd exit ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` case "${UNAME_MACHINE}" in 9000/31? ) HP_ARCH=m68000 ;; 9000/[34]?? ) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "${sc_cpu_version}" in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "${sc_kernel_bits}" in 32) HP_ARCH=hppa2.0n ;; 64) HP_ARCH=hppa2.0w ;; '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi if [ "${HP_ARCH}" = "" ]; then eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #define _HPUX_SOURCE #include #include int main () { #if defined(_SC_KERNEL_BITS) long bits = sysconf(_SC_KERNEL_BITS); #endif long cpu = sysconf (_SC_CPU_VERSION); switch (cpu) { case CPU_PA_RISC1_0: puts ("hppa1.0"); break; case CPU_PA_RISC1_1: puts ("hppa1.1"); break; case CPU_PA_RISC2_0: #if defined(_SC_KERNEL_BITS) switch (bits) { case 64: puts ("hppa2.0w"); break; case 32: puts ("hppa2.0n"); break; default: puts ("hppa2.0"); break; } break; #else /* !defined(_SC_KERNEL_BITS) */ puts ("hppa2.0"); break; #endif default: puts ("hppa1.0"); break; } exit (0); } EOF (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac if [ ${HP_ARCH} = hppa2.0w ] then eval $set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler # generating 64-bit code. GNU and HP use different nomenclature: # # $ CC_FOR_BUILD=cc ./config.guess # => hppa2.0w-hp-hpux11.23 # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then HP_ARCH=hppa2.0w else HP_ARCH=hppa64 fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} exit ;; ia64:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` echo ia64-hp-hpux${HPUX_REV} exit ;; 3050*:HI-UX:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #include int main () { long cpu = sysconf (_SC_CPU_VERSION); /* The order matters, because CPU_IS_HP_MC68K erroneously returns true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct results, however. */ if (CPU_IS_PA_RISC (cpu)) { switch (cpu) { case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; default: puts ("hppa-hitachi-hiuxwe2"); break; } } else if (CPU_IS_HP_MC68K (cpu)) puts ("m68k-hitachi-hiuxwe2"); else puts ("unknown-hitachi-hiuxwe2"); exit (0); } EOF $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) echo hppa1.1-hp-bsd exit ;; 9000/8??:4.3bsd:*:*) echo hppa1.0-hp-bsd exit ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix exit ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) echo hppa1.1-hp-osf exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then echo ${UNAME_MACHINE}-unknown-osf1mk else echo ${UNAME_MACHINE}-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) echo hppa1.1-hp-lites exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} exit ;; sparc*:BSD/OS:*:*) echo sparc-unknown-bsdi${UNAME_RELEASE} exit ;; *:BSD/OS:*:*) echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin exit ;; *:MINGW64*:*) echo ${UNAME_MACHINE}-pc-mingw64 exit ;; *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 exit ;; *:MSYS*:*) echo ${UNAME_MACHINE}-pc-msys exit ;; i*:PW*:*) echo ${UNAME_MACHINE}-pc-pw32 exit ;; *:Interix*:*) case ${UNAME_MACHINE} in x86) echo i586-pc-interix${UNAME_RELEASE} exit ;; authenticamd | genuineintel | EM64T) echo x86_64-unknown-interix${UNAME_RELEASE} exit ;; IA64) echo ia64-unknown-interix${UNAME_RELEASE} exit ;; esac ;; i*:UWIN*:*) echo ${UNAME_MACHINE}-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) echo x86_64-unknown-cygwin exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` exit ;; *:GNU:*:*) # the GNU system echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix exit ;; aarch64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; PCA57) UNAME_MACHINE=alphapca56 ;; EV6) UNAME_MACHINE=alphaev6 ;; EV67) UNAME_MACHINE=alphaev67 ;; EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arc:Linux:*:* | arceb:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arm*:Linux:*:*) eval $set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then echo ${UNAME_MACHINE}-unknown-linux-${LIBC} else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi else echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf fi fi exit ;; avr32*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; cris:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; e2k:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; hexagon:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; i*86:Linux:*:*) echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; k1om:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; m68*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #undef CPU #undef ${UNAME_MACHINE} #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; mips64el:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; openrisc*:Linux:*:*) echo or1k-unknown-linux-${LIBC} exit ;; or32:Linux:*:* | or1k*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; padre:Linux:*:*) echo sparc-unknown-linux-${LIBC} exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) echo hppa64-unknown-linux-${LIBC} exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; *) echo hppa-unknown-linux-${LIBC} ;; esac exit ;; ppc64:Linux:*:*) echo powerpc64-unknown-linux-${LIBC} exit ;; ppc:Linux:*:*) echo powerpc-unknown-linux-${LIBC} exit ;; ppc64le:Linux:*:*) echo powerpc64le-unknown-linux-${LIBC} exit ;; ppcle:Linux:*:*) echo powerpcle-unknown-linux-${LIBC} exit ;; riscv32:Linux:*:* | riscv64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux-${LIBC} exit ;; sh64*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; sh*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; tile*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; vax:Linux:*:*) echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. echo i386-sequent-sysv4 exit ;; i*86:UNIX_SV:4.2MP:2.*) # Unixware is an offshoot of SVR4, but it has its own version # number series starting with 2... # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. echo ${UNAME_MACHINE}-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) echo ${UNAME_MACHINE}-unknown-stop exit ;; i*86:atheos:*:*) echo ${UNAME_MACHINE}-unknown-atheos exit ;; i*86:syllable:*:*) echo ${UNAME_MACHINE}-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos${UNAME_RELEASE} exit ;; i*86:*DOS:*:*) echo ${UNAME_MACHINE}-pc-msdosdjgpp exit ;; i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} else echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} fi exit ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ && UNAME_MACHINE=i586 (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 echo ${UNAME_MACHINE}-pc-sco$UNAME_REL else echo ${UNAME_MACHINE}-pc-sysv32 fi exit ;; pc:*:*:*) # Left here for compatibility: # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; Intel:Mach:3*:*) echo i386-pc-mach3 exit ;; paragon:*:*:*) echo i860-intel-osf1 exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) # "miniframe" echo m68010-convergent-sysv exit ;; mc68k:UNIX:SYSTEM5:3.51m) echo m68k-convergent-sysv exit ;; M680?0:D-NIX:5.3:*) echo m68k-diab-dnix exit ;; M68*:*:R3V[5678]*:*) test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; NCR*:*:4.2:* | MPRAS*:*:4.2:*) OS_REL='.3' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) echo m68k-unknown-lynxos${UNAME_RELEASE} exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) echo sparc-unknown-lynxos${UNAME_RELEASE} exit ;; rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos${UNAME_RELEASE} exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos${UNAME_RELEASE} exit ;; SM[BE]S:UNIX_SV:*:*) echo mips-dde-sysv${UNAME_RELEASE} exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 exit ;; RM*:SINIX-*:*:*) echo mips-sni-sysv4 exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` echo ${UNAME_MACHINE}-sni-sysv4 else echo ns32k-sni-sysv fi exit ;; PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort # says echo i586-unisys-sysv4 exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm echo hppa1.1-stratus-sysv4 exit ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. echo i860-stratus-sysv4 exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. echo ${UNAME_MACHINE}-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) echo m68k-apple-aux${UNAME_RELEASE} exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then echo mips-nec-sysv${UNAME_RELEASE} else echo mips-unknown-sysv${UNAME_RELEASE} fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos exit ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. echo powerpc-apple-beos exit ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. echo i586-pc-beos exit ;; BePC:Haiku:*:*) # Haiku running on Intel PC compatible. echo i586-pc-haiku exit ;; x86_64:Haiku:*:*) echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux${UNAME_RELEASE} exit ;; SX-5:SUPER-UX:*:*) echo sx5-nec-superux${UNAME_RELEASE} exit ;; SX-6:SUPER-UX:*:*) echo sx6-nec-superux${UNAME_RELEASE} exit ;; SX-7:SUPER-UX:*:*) echo sx7-nec-superux${UNAME_RELEASE} exit ;; SX-8:SUPER-UX:*:*) echo sx8-nec-superux${UNAME_RELEASE} exit ;; SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux${UNAME_RELEASE} exit ;; SX-ACE:SUPER-UX:*:*) echo sxace-nec-superux${UNAME_RELEASE} exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} exit ;; *:Rhapsody:*:*) echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown eval $set_cc_for_build if test "$UNAME_PROCESSOR" = unknown ; then UNAME_PROCESSOR=powerpc fi if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_PPC >/dev/null then UNAME_PROCESSOR=powerpc fi fi elif test "$UNAME_PROCESSOR" = i386 ; then # Avoid executing cc on OS X 10.9, as it ships with a stub # that puts up a graphical alert prompting to install # developer tools. Any system running Mac OS X 10.7 or # later (Darwin 11 and later) is required to have a 64-bit # processor. This is not true of the ARM version of Darwin # that Apple uses in portable devices. UNAME_PROCESSOR=x86_64 fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; NEO-*:NONSTOP_KERNEL:*:*) echo neo-tandem-nsk${UNAME_RELEASE} exit ;; NSE-*:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk${UNAME_RELEASE} exit ;; NSR-*:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; NSX-*:NONSTOP_KERNEL:*:*) echo nsx-tandem-nsk${UNAME_RELEASE} exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; BS2000:POSIX*:*:*) echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" fi echo ${UNAME_MACHINE}-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 exit ;; *:TENEX:*:*) echo pdp10-unknown-tenex exit ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) echo pdp10-dec-tops20 exit ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) echo pdp10-xkl-tops20 exit ;; *:TOPS-20:*:*) echo pdp10-unknown-tops20 exit ;; *:ITS:*:*) echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) echo mips-sei-seiux${UNAME_RELEASE} exit ;; *:DragonFly:*:*) echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` case "${UNAME_MACHINE}" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; esac ;; *:XENIX:*:SysV) echo i386-pc-xenix exit ;; i*86:skyos:*:*) echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` exit ;; i*86:rdos:*:*) echo ${UNAME_MACHINE}-pc-rdos exit ;; i*86:AROS:*:*) echo ${UNAME_MACHINE}-pc-aros exit ;; x86_64:VMkernel:*:*) echo ${UNAME_MACHINE}-unknown-esx exit ;; amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs exit ;; esac echo "$0: unable to guess system type" >&2 case "${UNAME_MACHINE}:${UNAME_SYSTEM}" in mips:Linux | mips64:Linux) # If we got here on MIPS GNU/Linux, output extra information. cat >&2 <&2 </dev/null || echo unknown` uname -r = `(uname -r) 2>/dev/null || echo unknown` uname -s = `(uname -s) 2>/dev/null || echo unknown` uname -v = `(uname -v) 2>/dev/null || echo unknown` /usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` /bin/uname -X = `(/bin/uname -X) 2>/dev/null` hostinfo = `(hostinfo) 2>/dev/null` /bin/universe = `(/bin/universe) 2>/dev/null` /usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` /bin/arch = `(/bin/arch) 2>/dev/null` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` UNAME_MACHINE = ${UNAME_MACHINE} UNAME_RELEASE = ${UNAME_RELEASE} UNAME_SYSTEM = ${UNAME_SYSTEM} UNAME_VERSION = ${UNAME_VERSION} EOF exit 1 # Local variables: # eval: (add-hook 'write-file-functions 'time-stamp) # time-stamp-start: "timestamp='" # time-stamp-format: "%:y-%02m-%02d" # time-stamp-end: "'" # End: ================================================ FILE: config/general.m4 ================================================ # config/general.m4 # Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group # Portions Copyright (c) 1994, The Regents of the University of California # This file defines new macros to process configure command line # arguments, to replace the brain-dead AC_ARG_WITH and AC_ARG_ENABLE. # The flaw in these is particularly that they only differentiate # between "given" and "not given" and do not provide enough help to # process arguments that only accept "yes/no", that require an # argument (other than "yes/no"), etc. # # The point of this implementation is to reduce code size and # redundancy in configure.ac and to improve robustness and consistency # in the option evaluation code. # Convert type and name to shell variable name (e.g., "enable_long_strings") m4_define([pgac_arg_to_variable], [$1[]_[]patsubst($2, -, _)]) # PGAC_ARG(TYPE, NAME, HELP-STRING-LHS-EXTRA, HELP-STRING-RHS, # [ACTION-IF-YES], [ACTION-IF-NO], [ACTION-IF-ARG], # [ACTION-IF-OMITTED]) # ------------------------------------------------------------ # This is the base layer. TYPE is either "with" or "enable", depending # on what you like. NAME is the rest of the option name. # HELP-STRING-LHS-EXTRA is a string to append to the option name on # the left-hand side of the help output, e.g., an argument name. If # set to "-", append nothing, but let the option appear in the # negative form (disable/without). HELP-STRING-RHS is the option # description, for the right-hand side of the help output. # ACTION-IF-YES is executed if the option is given without an argument # (or "yes", which is the same); similar for ACTION-IF-NO. AC_DEFUN([PGAC_ARG], [ m4_case([$1], enable, [ AC_ARG_ENABLE([$2], [AS_HELP_STRING([--]m4_if($3, -, disable, enable)[-$2]m4_if($3, -, , $3), [$4])], [ case [$]enableval in yes) m4_default([$5], :) ;; no) m4_default([$6], :) ;; *) $7 ;; esac ], [$8])[]dnl AC_ARG_ENABLE ], with, [ AC_ARG_WITH([$2], [AS_HELP_STRING([--]m4_if($3, -, without, with)[-$2]m4_if($3, -, , $3), [$4])], [ case [$]withval in yes) m4_default([$5], :) ;; no) m4_default([$6], :) ;; *) $7 ;; esac ], [$8])[]dnl AC_ARG_WITH ], [m4_fatal([first argument of $0 must be 'enable' or 'with', not '$1'])] ) ])# PGAC_ARG # PGAC_ARG_BOOL(TYPE, NAME, DEFAULT, HELP-STRING-RHS, # [ACTION-IF-YES], [ACTION-IF-NO]) # --------------------------------------------------- # Accept a boolean option, that is, one that only takes yes or no. # ("no" is equivalent to "disable" or "without"). DEFAULT is what # should be done if the option is omitted; it should be "yes" or "no". # (Consequently, one of ACTION-IF-YES and ACTION-IF-NO will always # execute.) AC_DEFUN([PGAC_ARG_BOOL], [dnl The following hack is necessary because in a few instances this dnl macro is called twice for the same option with different default dnl values. But we only want it to appear once in the help. We achieve dnl that by making the help string look the same, which is why we need to dnl save the default that was passed in previously. m4_define([_pgac_helpdefault], m4_ifdef([pgac_defined_$1_$2_bool], [m4_defn([pgac_defined_$1_$2_bool])], [$3]))dnl PGAC_ARG([$1], [$2], [m4_if(_pgac_helpdefault, yes, -)], [$4], [$5], [$6], [AC_MSG_ERROR([no argument expected for --$1-$2 option])], [m4_case([$3], yes, [pgac_arg_to_variable([$1], [$2])=yes $5], no, [pgac_arg_to_variable([$1], [$2])=no $6], [m4_fatal([third argument of $0 must be 'yes' or 'no', not '$3'])])])[]dnl m4_define([pgac_defined_$1_$2_bool], [$3])dnl ])# PGAC_ARG_BOOL # PGAC_ARG_REQ(TYPE, NAME, HELP-ARGNAME, HELP-STRING-RHS, # [ACTION-IF-GIVEN], [ACTION-IF-NOT-GIVEN]) # ------------------------------------------------------- # This option will require an argument; "yes" or "no" will not be # accepted. HELP-ARGNAME is a name for the argument for the help output. AC_DEFUN([PGAC_ARG_REQ], [PGAC_ARG([$1], [$2], [=$3], [$4], [AC_MSG_ERROR([argument required for --$1-$2 option])], [AC_MSG_ERROR([argument required for --$1-$2 option])], [$5], [$6])])# PGAC_ARG_REQ # PGAC_ARG_OPTARG(TYPE, NAME, HELP-ARGNAME, HELP-STRING-RHS, # [DEFAULT-ACTION], [ARG-ACTION], # [ACTION-ENABLED], [ACTION-DISABLED]) # ---------------------------------------------------------- # This will create an option that behaves as follows: If omitted, or # called with "no", then set the enable_variable to "no" and do # nothing else. If called with "yes", then execute DEFAULT-ACTION. If # called with argument, set enable_variable to "yes" and execute # ARG-ACTION. Additionally, execute ACTION-ENABLED if we ended up with # "yes" either way, else ACTION-DISABLED. # # The intent is to allow enabling a feature, and optionally pass an # additional piece of information. AC_DEFUN([PGAC_ARG_OPTARG], [PGAC_ARG([$1], [$2], [@<:@=$3@:>@], [$4], [$5], [], [pgac_arg_to_variable([$1], [$2])=yes $6], [pgac_arg_to_variable([$1], [$2])=no]) dnl Add this code only if there's a ACTION-ENABLED or ACTION-DISABLED. m4_ifval([$7[]$8], [ if test "[$]pgac_arg_to_variable([$1], [$2])" = yes; then m4_default([$7], :) m4_ifval([$8], [else $8 ])[]dnl fi ])[]dnl ])# PGAC_ARG_OPTARG ================================================ FILE: configure ================================================ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.69 for Citus 15.0devel. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. # # Copyright (c) Citus Data, Inc. ## -------------------- ## ## M4sh Initialization. ## ## -------------------- ## # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi as_nl=' ' export as_nl # Printing a long string crashes Solaris 7 /usr/bin/printf. as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo # Prefer a ksh shell builtin over an external printf program on Solaris, # but without wasting forks for bash or zsh. if test -z "$BASH_VERSION$ZSH_VERSION" \ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='print -r --' as_echo_n='print -rn --' elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='printf %s\n' as_echo_n='printf %s' else if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' as_echo_n='/usr/ucb/echo -n' else as_echo_body='eval expr "X$1" : "X\\(.*\\)"' as_echo_n_body='eval arg=$1; case $arg in #( *"$as_nl"*) expr "X$arg" : "X\\(.*\\)$as_nl"; arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; esac; expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" ' export as_echo_n_body as_echo_n='sh -c $as_echo_n_body as_echo' fi export as_echo_body as_echo='sh -c $as_echo_body as_echo' fi # The user is always right. if test "${PATH_SEPARATOR+set}" != set; then PATH_SEPARATOR=: (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || PATH_SEPARATOR=';' } fi # IFS # We need space, tab and new line, in precisely that order. Quoting is # there to prevent editors from complaining about space-tab. # (If _AS_PATH_WALK were called with IFS unset, it would disable word # splitting by setting IFS to empty value.) IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break done IFS=$as_save_IFS ;; esac # We did not find ourselves, most probably we were run as `sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi # Unset variables that we do not need and which cause bugs (e.g. in # pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" # suppresses any "Segmentation fault" message there. '((' could # trigger a bug in pdksh 5.2.14. for as_var in BASH_ENV ENV MAIL MAILPATH do eval test x\${$as_var+set} = xset \ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : done PS1='$ ' PS2='> ' PS4='+ ' # NLS nuisances. LC_ALL=C export LC_ALL LANGUAGE=C export LANGUAGE # CDPATH. (unset CDPATH) >/dev/null 2>&1 && unset CDPATH # Use a proper internal environment variable to ensure we don't fall # into an infinite loop, continuously re-executing ourselves. if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then _as_can_reexec=no; export _as_can_reexec; # We cannot yet assume a decent shell, so we have to provide a # neutralization value for shells without unset; and this also # works around shells that cannot unset nonexistent variables. # Preserve -v and -x to the replacement shell. BASH_ENV=/dev/null ENV=/dev/null (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV case $- in # (((( *v*x* | *x*v* ) as_opts=-vx ;; *v* ) as_opts=-v ;; *x* ) as_opts=-x ;; * ) as_opts= ;; esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed `exec'. $as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 as_fn_exit 255 fi # We don't want this to propagate to other subprocesses. { _as_can_reexec=; unset _as_can_reexec;} if test "x$CONFIG_SHELL" = x; then as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which # is contrary to our usage. Disable this feature. alias -g '\${1+\"\$@\"}'='\"\$@\"' setopt NO_GLOB_SUBST else case \`(set -o) 2>/dev/null\` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi " as_required="as_fn_return () { (exit \$1); } as_fn_success () { as_fn_return 0; } as_fn_failure () { as_fn_return 1; } as_fn_ret_success () { return 0; } as_fn_ret_failure () { return 1; } exitcode=0 as_fn_success || { exitcode=1; echo as_fn_success failed.; } as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; } as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; } as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : else exitcode=1; echo positional parameters were not saved. fi test x\$exitcode = x0 || exit 1 test -x / || exit 1" as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 test \$(( 1 + 1 )) = 2 || exit 1" if (eval "$as_required") 2>/dev/null; then : as_have_required=yes else as_have_required=no fi if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then : else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR as_found=false for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. as_found=: case $as_dir in #( /*) for as_base in sh bash ksh sh5; do # Try only shells that exist, to save several forks. as_shell=$as_dir/$as_base if { test -f "$as_shell" || test -f "$as_shell.exe"; } && { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then : CONFIG_SHELL=$as_shell as_have_required=yes if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then : break 2 fi fi done;; esac as_found=false done $as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } && { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then : CONFIG_SHELL=$SHELL as_have_required=yes fi; } IFS=$as_save_IFS if test "x$CONFIG_SHELL" != x; then : export CONFIG_SHELL # We cannot yet assume a decent shell, so we have to provide a # neutralization value for shells without unset; and this also # works around shells that cannot unset nonexistent variables. # Preserve -v and -x to the replacement shell. BASH_ENV=/dev/null ENV=/dev/null (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV case $- in # (((( *v*x* | *x*v* ) as_opts=-vx ;; *v* ) as_opts=-v ;; *x* ) as_opts=-x ;; * ) as_opts= ;; esac exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} # Admittedly, this is quite paranoid, since all the known shells bail # out after a failed `exec'. $as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 exit 255 fi if test x$as_have_required = xno; then : $as_echo "$0: This script requires a shell more modern than all" $as_echo "$0: the shells that I found on your system." if test x${ZSH_VERSION+set} = xset ; then $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should" $as_echo "$0: be upgraded to zsh 4.3.4 or later." else $as_echo "$0: Please tell bug-autoconf@gnu.org about your system, $0: including any error possibly output before this $0: message. Then install a modern shell, or manually run $0: the script under such a shell if you do have one." fi exit 1 fi fi fi SHELL=${CONFIG_SHELL-/bin/sh} export SHELL # Unset more variables known to interfere with behavior of common tools. CLICOLOR_FORCE= GREP_OPTIONS= unset CLICOLOR_FORCE GREP_OPTIONS ## --------------------- ## ## M4sh Shell Functions. ## ## --------------------- ## # as_fn_unset VAR # --------------- # Portably unset VAR. as_fn_unset () { { eval $1=; unset $1;} } as_unset=as_fn_unset # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. as_fn_set_status () { return $1 } # as_fn_set_status # as_fn_exit STATUS # ----------------- # Exit the shell with STATUS, even in a "trap 0" or "set -e" context. as_fn_exit () { set +e as_fn_set_status $1 exit $1 } # as_fn_exit # as_fn_mkdir_p # ------------- # Create "$as_dir" as a directory, including parents if necessary. as_fn_mkdir_p () { case $as_dir in #( -*) as_dir=./$as_dir;; esac test -d "$as_dir" || eval $as_mkdir_p || { as_dirs= while :; do case $as_dir in #( *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" as_dir=`$as_dirname -- "$as_dir" || $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p # as_fn_executable_p FILE # ----------------------- # Test if FILE is an executable regular file. as_fn_executable_p () { test -f "$1" && test -x "$1" } # as_fn_executable_p # as_fn_append VAR VALUE # ---------------------- # Append the text in VALUE to the end of the definition contained in VAR. Take # advantage of any shell optimizations that allow amortized linear growth over # repeated appends, instead of the typical quadratic growth present in naive # implementations. if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : eval 'as_fn_append () { eval $1+=\$2 }' else as_fn_append () { eval $1=\$$1\$2 } fi # as_fn_append # as_fn_arith ARG... # ------------------ # Perform arithmetic evaluation on the ARGs, and store the result in the # global $as_val. Take advantage of shells that can avoid forks. The arguments # must be portable across $(()) and expr. if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : eval 'as_fn_arith () { as_val=$(( $* )) }' else as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` } fi # as_fn_arith # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the # script with STATUS, using 1 if that was 0. as_fn_error () { as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error if expr a : '\(a\)' >/dev/null 2>&1 && test "X`expr 00001 : '.*\(...\)'`" = X001; then as_expr=expr else as_expr=false fi if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then as_basename=basename else as_basename=false fi if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then as_dirname=dirname else as_dirname=false fi as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || $as_echo X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q } /^X\/\(\/\/\)$/{ s//\1/ q } /^X\/\(\/\).*/{ s//\1/ q } s/.*/./; q'` # Avoid depending upon Character Ranges. as_cr_letters='abcdefghijklmnopqrstuvwxyz' as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' as_cr_Letters=$as_cr_letters$as_cr_LETTERS as_cr_digits='0123456789' as_cr_alnum=$as_cr_Letters$as_cr_digits as_lineno_1=$LINENO as_lineno_1a=$LINENO as_lineno_2=$LINENO as_lineno_2a=$LINENO eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" && test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || { # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-) sed -n ' p /[$]LINENO/= ' <$as_myself | sed ' s/[$]LINENO.*/&-/ t lineno b :lineno N :loop s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ t loop s/-\n.*// ' >$as_me.lineno && chmod +x "$as_me.lineno" || { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } # If we had to re-execute with $CONFIG_SHELL, we're ensured to have # already done that, so ensure we don't try to do so again and fall # in an infinite loop. This has already happened in practice. _as_can_reexec=no; export _as_can_reexec # Don't try to exec as it changes $[0], causing all sort of problems # (the dirname of $[0] is not the place where we might find the # original and so on. Autoconf is especially sensitive to this). . "./$as_me.lineno" # Exit status is that of the last command. exit } ECHO_C= ECHO_N= ECHO_T= case `echo -n x` in #((((( -n*) case `echo 'xy\c'` in *c*) ECHO_T=' ';; # ECHO_T is single tab character. xy) ECHO_C='\c';; *) echo `echo ksh88 bug on AIX 6.1` > /dev/null ECHO_T=' ';; esac;; *) ECHO_N='-n';; esac rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file else rm -f conf$$.dir mkdir conf$$.dir 2>/dev/null fi if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else as_ln_s='cp -pR' fi else as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null if mkdir -p . 2>/dev/null; then as_mkdir_p='mkdir -p "$as_dir"' else test -d ./-p && rmdir ./-p as_mkdir_p=false fi as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" # Sed expression to map a string onto a valid variable name. as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" test -n "$DJDIR" || exec 7<&0 &1 # Name of the host. # hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, # so uname gets run too. ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` # # Initializations. # ac_default_prefix=/usr/local ac_clean_files= ac_config_libobj_dir=. LIBOBJS= cross_compiling=no subdirs= MFLAGS= MAKEFLAGS= # Identity of this package. PACKAGE_NAME='Citus' PACKAGE_TARNAME='citus' PACKAGE_VERSION='15.0devel' PACKAGE_STRING='Citus 15.0devel' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ #include #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef STDC_HEADERS # include # include #else # ifdef HAVE_STDLIB_H # include # endif #endif #ifdef HAVE_STRING_H # if !defined STDC_HEADERS && defined HAVE_MEMORY_H # include # endif # include #endif #ifdef HAVE_STRINGS_H # include #endif #ifdef HAVE_INTTYPES_H # include #endif #ifdef HAVE_STDINT_H # include #endif #ifdef HAVE_UNISTD_H # include #endif" ac_subst_vars='LTLIBOBJS LIBOBJS HAS_DOTGIT POSTGRES_BUILDDIR POSTGRES_SRCDIR CITUS_LDFLAGS CITUS_CPPFLAGS CITUS_BITCODE_CFLAGS CITUS_CFLAGS GIT_BIN with_security_flags with_zstd with_lz4 EGREP GREP CPP OBJEXT EXEEXT ac_ct_CC CPPFLAGS LDFLAGS CFLAGS CC vpath_build with_pg_version_check PATH PG_CONFIG FLEX AWK SED target_alias host_alias build_alias LIBS ECHO_T ECHO_N ECHO_C DEFS mandir localedir libdir psdir pdfdir dvidir htmldir infodir docdir oldincludedir includedir localstatedir sharedstatedir sysconfdir datadir datarootdir libexecdir sbindir bindir program_transform_name prefix exec_prefix PACKAGE_URL PACKAGE_BUGREPORT PACKAGE_STRING PACKAGE_VERSION PACKAGE_TARNAME PACKAGE_NAME PATH_SEPARATOR SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking with_extra_version with_pg_version_check enable_coverage with_lz4 with_zstd with_security_flags ' ac_precious_vars='build_alias host_alias target_alias PG_CONFIG PATH CC CFLAGS LDFLAGS LIBS CPPFLAGS CPP' # Initialize some variables set by options. ac_init_help= ac_init_version=false ac_unrecognized_opts= ac_unrecognized_sep= # The variables have the same names as the options, with # dashes changed to underlines. cache_file=/dev/null exec_prefix=NONE no_create= no_recursion= prefix=NONE program_prefix=NONE program_suffix=NONE program_transform_name=s,x,x, silent= site= srcdir= verbose= x_includes=NONE x_libraries=NONE # Installation directory options. # These are left unexpanded so users can "make install exec_prefix=/foo" # and all the variables that are supposed to be based on exec_prefix # by default will actually change. # Use braces instead of parens because sh, perl, etc. also accept them. # (The list follows the same order as the GNU Coding Standards.) bindir='${exec_prefix}/bin' sbindir='${exec_prefix}/sbin' libexecdir='${exec_prefix}/libexec' datarootdir='${prefix}/share' datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' infodir='${datarootdir}/info' htmldir='${docdir}' dvidir='${docdir}' pdfdir='${docdir}' psdir='${docdir}' libdir='${exec_prefix}/lib' localedir='${datarootdir}/locale' mandir='${datarootdir}/man' ac_prev= ac_dashdash= for ac_option do # If the previous option needs an argument, assign it. if test -n "$ac_prev"; then eval $ac_prev=\$ac_option ac_prev= continue fi case $ac_option in *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; *=) ac_optarg= ;; *) ac_optarg=yes ;; esac # Accept the important Cygnus configure options, so we can diagnose typos. case $ac_dashdash$ac_option in --) ac_dashdash=yes ;; -bindir | --bindir | --bindi | --bind | --bin | --bi) ac_prev=bindir ;; -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) bindir=$ac_optarg ;; -build | --build | --buil | --bui | --bu) ac_prev=build_alias ;; -build=* | --build=* | --buil=* | --bui=* | --bu=*) build_alias=$ac_optarg ;; -cache-file | --cache-file | --cache-fil | --cache-fi \ | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) ac_prev=cache_file ;; -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) cache_file=$ac_optarg ;; --config-cache | -C) cache_file=config.cache ;; -datadir | --datadir | --datadi | --datad) ac_prev=datadir ;; -datadir=* | --datadir=* | --datadi=* | --datad=*) datadir=$ac_optarg ;; -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ | --dataroo | --dataro | --datar) ac_prev=datarootdir ;; -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) datarootdir=$ac_optarg ;; -disable-* | --disable-*) ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval enable_$ac_useropt=no ;; -docdir | --docdir | --docdi | --doc | --do) ac_prev=docdir ;; -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) docdir=$ac_optarg ;; -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) ac_prev=dvidir ;; -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) dvidir=$ac_optarg ;; -enable-* | --enable-*) ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "enable_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval enable_$ac_useropt=\$ac_optarg ;; -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ | --exec | --exe | --ex) ac_prev=exec_prefix ;; -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ | --exec=* | --exe=* | --ex=*) exec_prefix=$ac_optarg ;; -gas | --gas | --ga | --g) # Obsolete; use --with-gas. with_gas=yes ;; -help | --help | --hel | --he | -h) ac_init_help=long ;; -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) ac_init_help=recursive ;; -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) ac_init_help=short ;; -host | --host | --hos | --ho) ac_prev=host_alias ;; -host=* | --host=* | --hos=* | --ho=*) host_alias=$ac_optarg ;; -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) ac_prev=htmldir ;; -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ | --ht=*) htmldir=$ac_optarg ;; -includedir | --includedir | --includedi | --included | --include \ | --includ | --inclu | --incl | --inc) ac_prev=includedir ;; -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ | --includ=* | --inclu=* | --incl=* | --inc=*) includedir=$ac_optarg ;; -infodir | --infodir | --infodi | --infod | --info | --inf) ac_prev=infodir ;; -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) infodir=$ac_optarg ;; -libdir | --libdir | --libdi | --libd) ac_prev=libdir ;; -libdir=* | --libdir=* | --libdi=* | --libd=*) libdir=$ac_optarg ;; -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ | --libexe | --libex | --libe) ac_prev=libexecdir ;; -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ | --libexe=* | --libex=* | --libe=*) libexecdir=$ac_optarg ;; -localedir | --localedir | --localedi | --localed | --locale) ac_prev=localedir ;; -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) localedir=$ac_optarg ;; -localstatedir | --localstatedir | --localstatedi | --localstated \ | --localstate | --localstat | --localsta | --localst | --locals) ac_prev=localstatedir ;; -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) localstatedir=$ac_optarg ;; -mandir | --mandir | --mandi | --mand | --man | --ma | --m) ac_prev=mandir ;; -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) mandir=$ac_optarg ;; -nfp | --nfp | --nf) # Obsolete; use --without-fp. with_fp=no ;; -no-create | --no-create | --no-creat | --no-crea | --no-cre \ | --no-cr | --no-c | -n) no_create=yes ;; -no-recursion | --no-recursion | --no-recursio | --no-recursi \ | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) no_recursion=yes ;; -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ | --oldin | --oldi | --old | --ol | --o) ac_prev=oldincludedir ;; -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) oldincludedir=$ac_optarg ;; -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) ac_prev=prefix ;; -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) prefix=$ac_optarg ;; -program-prefix | --program-prefix | --program-prefi | --program-pref \ | --program-pre | --program-pr | --program-p) ac_prev=program_prefix ;; -program-prefix=* | --program-prefix=* | --program-prefi=* \ | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) program_prefix=$ac_optarg ;; -program-suffix | --program-suffix | --program-suffi | --program-suff \ | --program-suf | --program-su | --program-s) ac_prev=program_suffix ;; -program-suffix=* | --program-suffix=* | --program-suffi=* \ | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) program_suffix=$ac_optarg ;; -program-transform-name | --program-transform-name \ | --program-transform-nam | --program-transform-na \ | --program-transform-n | --program-transform- \ | --program-transform | --program-transfor \ | --program-transfo | --program-transf \ | --program-trans | --program-tran \ | --progr-tra | --program-tr | --program-t) ac_prev=program_transform_name ;; -program-transform-name=* | --program-transform-name=* \ | --program-transform-nam=* | --program-transform-na=* \ | --program-transform-n=* | --program-transform-=* \ | --program-transform=* | --program-transfor=* \ | --program-transfo=* | --program-transf=* \ | --program-trans=* | --program-tran=* \ | --progr-tra=* | --program-tr=* | --program-t=*) program_transform_name=$ac_optarg ;; -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) ac_prev=pdfdir ;; -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) pdfdir=$ac_optarg ;; -psdir | --psdir | --psdi | --psd | --ps) ac_prev=psdir ;; -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) psdir=$ac_optarg ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil) silent=yes ;; -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ | --sbi=* | --sb=*) sbindir=$ac_optarg ;; -sharedstatedir | --sharedstatedir | --sharedstatedi \ | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ | --sharedst | --shareds | --shared | --share | --shar \ | --sha | --sh) ac_prev=sharedstatedir ;; -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ | --sha=* | --sh=*) sharedstatedir=$ac_optarg ;; -site | --site | --sit) ac_prev=site ;; -site=* | --site=* | --sit=*) site=$ac_optarg ;; -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) ac_prev=srcdir ;; -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) srcdir=$ac_optarg ;; -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ | --syscon | --sysco | --sysc | --sys | --sy) ac_prev=sysconfdir ;; -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) sysconfdir=$ac_optarg ;; -target | --target | --targe | --targ | --tar | --ta | --t) ac_prev=target_alias ;; -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) target_alias=$ac_optarg ;; -v | -verbose | --verbose | --verbos | --verbo | --verb) verbose=yes ;; -version | --version | --versio | --versi | --vers | -V) ac_init_version=: ;; -with-* | --with-*) ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval with_$ac_useropt=\$ac_optarg ;; -without-* | --without-*) ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in *" "with_$ac_useropt" "*) ;; *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" ac_unrecognized_sep=', ';; esac eval with_$ac_useropt=no ;; --x) # Obsolete; use --with-x. with_x=yes ;; -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ | --x-incl | --x-inc | --x-in | --x-i) ac_prev=x_includes ;; -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) x_includes=$ac_optarg ;; -x-libraries | --x-libraries | --x-librarie | --x-librari \ | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) ac_prev=x_libraries ;; -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; -*) as_fn_error $? "unrecognized option: \`$ac_option' Try \`$0 --help' for more information" ;; *=*) ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; *) # FIXME: should be removed in autoconf 3.0. $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" ;; esac done if test -n "$ac_prev"; then ac_option=--`echo $ac_prev | sed 's/_/-/g'` as_fn_error $? "missing argument to $ac_option" fi if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi # Check all directory arguments for consistency. for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. case $ac_val in */ ) ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` eval $ac_var=\$ac_val;; esac # Be sure to have absolute directory names. case $ac_val in [\\/$]* | ?:[\\/]* ) continue;; NONE | '' ) case $ac_var in *prefix ) continue;; esac;; esac as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done # There might be people who depend on the old broken behavior: `$host' # used to hold the argument of --host etc. # FIXME: To remove some day. build=$build_alias host=$host_alias target=$target_alias # FIXME: To remove some day. if test "x$host_alias" != x; then if test "x$build_alias" = x; then cross_compiling=maybe elif test "x$build_alias" != "x$host_alias"; then cross_compiling=yes fi fi ac_tool_prefix= test -n "$host_alias" && ac_tool_prefix=$host_alias- test "$silent" = yes && exec 6>/dev/null ac_pwd=`pwd` && test -n "$ac_pwd" && ac_ls_di=`ls -di .` && ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || as_fn_error $? "working directory cannot be determined" test "X$ac_ls_di" = "X$ac_pwd_ls_di" || as_fn_error $? "pwd does not report name of working directory" # Find the source files, if location was not specified. if test -z "$srcdir"; then ac_srcdir_defaulted=yes # Try the directory containing this script, then the parent directory. ac_confdir=`$as_dirname -- "$as_myself" || $as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_myself" : 'X\(//\)[^/]' \| \ X"$as_myself" : 'X\(//\)$' \| \ X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_myself" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` srcdir=$ac_confdir if test ! -r "$srcdir/$ac_unique_file"; then srcdir=.. fi else ac_srcdir_defaulted=no fi if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" ac_abs_confdir=`( cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` # When building in place, set srcdir=. if test "$ac_abs_confdir" = "$ac_pwd"; then srcdir=. fi # Remove unnecessary trailing slashes from srcdir. # Double slashes in file names in object file debugging info # mess up M-x gdb in Emacs. case $srcdir in */) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; esac for ac_var in $ac_precious_vars; do eval ac_env_${ac_var}_set=\${${ac_var}+set} eval ac_env_${ac_var}_value=\$${ac_var} eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} eval ac_cv_env_${ac_var}_value=\$${ac_var} done # # Report the --help message. # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF \`configure' configures Citus 15.0devel to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. Defaults for the options are specified in brackets. Configuration: -h, --help display this help and exit --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit -q, --quiet, --silent do not print \`checking ...' messages --cache-file=FILE cache test results in FILE [disabled] -C, --config-cache alias for \`--cache-file=config.cache' -n, --no-create do not create output files --srcdir=DIR find the sources in DIR [configure dir or \`..'] Installation directories: --prefix=PREFIX install architecture-independent files in PREFIX [$ac_default_prefix] --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX [PREFIX] By default, \`make install' will install all the files in \`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify an installation prefix other than \`$ac_default_prefix' using \`--prefix', for instance \`--prefix=\$HOME'. For better control, use the options below. Fine tuning of the installation directories: --bindir=DIR user executables [EPREFIX/bin] --sbindir=DIR system admin executables [EPREFIX/sbin] --libexecdir=DIR program executables [EPREFIX/libexec] --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] --datadir=DIR read-only architecture-independent data [DATAROOTDIR] --infodir=DIR info documentation [DATAROOTDIR/info] --localedir=DIR locale-dependent data [DATAROOTDIR/locale] --mandir=DIR man documentation [DATAROOTDIR/man] --docdir=DIR documentation root [DATAROOTDIR/doc/citus] --htmldir=DIR html documentation [DOCDIR] --dvidir=DIR dvi documentation [DOCDIR] --pdfdir=DIR pdf documentation [DOCDIR] --psdir=DIR ps documentation [DOCDIR] _ACEOF cat <<\_ACEOF _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in short | recursive ) echo "Configuration of Citus 15.0devel:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-coverage build with coverage testing instrumentation Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-extra-version=STRING append STRING to version --without-pg-version-check do not check postgres version during configure --without-lz4 do not use lz4 --without-zstd do not use zstd --with-security-flags use security flags Some influential environment variables: PG_CONFIG Location to find pg_config for target PostgreSQL instalation (default PATH) PATH PATH for target PostgreSQL install pg_config CC C compiler command CFLAGS C compiler flags LDFLAGS linker flags, e.g. -L if you have libraries in a nonstandard directory LIBS libraries to pass to the linker, e.g. -l CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. Report bugs to the package provider. _ACEOF ac_status=$? fi if test "$ac_init_help" = "recursive"; then # If there are subdirs, report their specific --help. for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue test -d "$ac_dir" || { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || continue ac_builddir=. case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; esac ;; esac ac_abs_top_builddir=$ac_pwd ac_abs_builddir=$ac_pwd$ac_dir_suffix # for backward compatibility: ac_top_builddir=$ac_top_build_prefix case $srcdir in .) # We are building in place. ac_srcdir=. ac_top_srcdir=$ac_top_builddir_sub ac_abs_top_srcdir=$ac_pwd ;; [\\/]* | ?:[\\/]* ) # Absolute name. ac_srcdir=$srcdir$ac_dir_suffix; ac_top_srcdir=$srcdir ac_abs_top_srcdir=$srcdir ;; *) # Relative name. ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix ac_top_srcdir=$ac_top_build_prefix$srcdir ac_abs_top_srcdir=$ac_pwd/$srcdir ;; esac ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix cd "$ac_dir" || { ac_status=$?; continue; } # Check for guested configure. if test -f "$ac_srcdir/configure.gnu"; then echo && $SHELL "$ac_srcdir/configure.gnu" --help=recursive elif test -f "$ac_srcdir/configure"; then echo && $SHELL "$ac_srcdir/configure" --help=recursive else $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 fi || ac_status=$? cd "$ac_pwd" || { ac_status=$?; break; } done fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF Citus configure 15.0devel generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. Copyright (c) Citus Data, Inc. _ACEOF exit fi ## ------------------------ ## ## Autoconf initialization. ## ## ------------------------ ## # ac_fn_c_try_compile LINENO # -------------------------- # Try to compile conftest.$ac_ext, and return whether this succeeded. ac_fn_c_try_compile () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack rm -f conftest.$ac_objext if { { ac_try="$ac_compile" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { test -z "$ac_c_werror_flag" || test ! -s conftest.err } && test -s conftest.$ac_objext; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_compile # ac_fn_c_try_run LINENO # ---------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. Assumes # that executables *can* be run. ac_fn_c_try_run () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' { { case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_try") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; }; then : ac_retval=0 else $as_echo "$as_me: program exited with status $ac_status" >&5 $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=$ac_status fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_run # ac_fn_c_compute_int LINENO EXPR VAR INCLUDES # -------------------------------------------- # Tries to find the compile-time value of EXPR in a program that includes # INCLUDES, setting VAR accordingly. Returns whether the value could be # computed ac_fn_c_compute_int () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if test "$cross_compiling" = yes; then # Depending upon the size, compute the lo and hi bounds. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) >= 0)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_lo=0 ac_mid=0 while :; do cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=$ac_mid; break else as_fn_arith $ac_mid + 1 && ac_lo=$as_val if test $ac_lo -le $ac_mid; then ac_lo= ac_hi= break fi as_fn_arith 2 '*' $ac_mid + 1 && ac_mid=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) < 0)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=-1 ac_mid=-1 while :; do cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) >= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_lo=$ac_mid; break else as_fn_arith '(' $ac_mid ')' - 1 && ac_hi=$as_val if test $ac_mid -le $ac_hi; then ac_lo= ac_hi= break fi as_fn_arith 2 '*' $ac_mid && ac_mid=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done else ac_lo= ac_hi= fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext # Binary search between lo and hi bounds. while test "x$ac_lo" != "x$ac_hi"; do as_fn_arith '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo && ac_mid=$as_val cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 int main () { static int test_array [1 - 2 * !(($2) <= $ac_mid)]; test_array [0] = 0; return test_array [0]; ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_hi=$ac_mid else as_fn_arith '(' $ac_mid ')' + 1 && ac_lo=$as_val fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext done case $ac_lo in #(( ?*) eval "$3=\$ac_lo"; ac_retval=0 ;; '') ac_retval=1 ;; esac else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 static long int longval () { return $2; } static unsigned long int ulongval () { return $2; } #include #include int main () { FILE *f = fopen ("conftest.val", "w"); if (! f) return 1; if (($2) < 0) { long int i = longval (); if (i != ($2)) return 1; fprintf (f, "%ld", i); } else { unsigned long int i = ulongval (); if (i != ($2)) return 1; fprintf (f, "%lu", i); } /* Do not output a trailing newline, as this causes \r\n confusion on some platforms. */ return ferror (f) || fclose (f) != 0; ; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : echo >>conftest.val; read $3 &5 (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } > conftest.i && { test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || test ! -s conftest.err }; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_cpp # ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES # ------------------------------------------------------- # Tests whether HEADER exists and can be compiled using the include files in # INCLUDES, setting the cache variable VAR accordingly. ac_fn_c_check_header_compile () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> _ACEOF if ac_fn_c_try_compile "$LINENO"; then : eval "$3=yes" else eval "$3=no" fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_compile # ac_fn_c_try_link LINENO # ----------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. ac_fn_c_try_link () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack rm -f conftest.$ac_objext conftest$ac_exeext if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>conftest.err ac_status=$? if test -s conftest.err; then grep -v '^ *+' conftest.err >conftest.er1 cat conftest.er1 >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } && { test -z "$ac_c_werror_flag" || test ! -s conftest.err } && test -s conftest$ac_exeext && { test "$cross_compiling" = yes || test -x conftest$ac_exeext }; then : ac_retval=0 else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would # interfere with the next link command; also delete a directory that is # left behind by Apple's compiler. We do this before executing the actions. rm -rf conftest.dSYM conftest_ipa8_conftest.oo eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_link # ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES # ------------------------------------------------------- # Tests whether HEADER exists, giving a warning if it cannot be compiled using # the include files in INCLUDES and setting the cache variable VAR # accordingly. ac_fn_c_check_header_mongrel () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack if eval \${$3+:} false; then : { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } else # Is the header compilable? { $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 $as_echo_n "checking $2 usability... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ $4 #include <$2> _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_header_compiler=yes else ac_header_compiler=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 $as_echo "$ac_header_compiler" >&6; } # Is the header present? { $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 $as_echo_n "checking $2 presence... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include <$2> _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : ac_header_preproc=yes else ac_header_preproc=no fi rm -f conftest.err conftest.i conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 $as_echo "$ac_header_preproc" >&6; } # So? What about this header? case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( yes:no: ) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 $as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 $as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} ;; no:yes:* ) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 $as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 $as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 $as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 $as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} ;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else eval "$3=\$ac_header_compiler" fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } fi eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by Citus $as_me 15.0devel, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF exec 5>>config.log { cat <<_ASUNAME ## --------- ## ## Platform. ## ## --------- ## hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` uname -m = `(uname -m) 2>/dev/null || echo unknown` uname -r = `(uname -r) 2>/dev/null || echo unknown` uname -s = `(uname -s) 2>/dev/null || echo unknown` uname -v = `(uname -v) 2>/dev/null || echo unknown` /usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` /bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` /bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` /usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` /usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` /bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` /usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` /bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` _ASUNAME as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. $as_echo "PATH: $as_dir" done IFS=$as_save_IFS } >&5 cat >&5 <<_ACEOF ## ----------- ## ## Core tests. ## ## ----------- ## _ACEOF # Keep a trace of the command line. # Strip out --no-create and --no-recursion so they do not pile up. # Strip out --silent because we don't want to record it for future runs. # Also quote any args containing shell meta-characters. # Make two passes to allow for proper duplicate-argument suppression. ac_configure_args= ac_configure_args0= ac_configure_args1= ac_must_keep_next=false for ac_pass in 1 2 do for ac_arg do case $ac_arg in -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil) continue ;; *\'*) ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; esac case $ac_pass in 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; 2) as_fn_append ac_configure_args1 " '$ac_arg'" if test $ac_must_keep_next = true; then ac_must_keep_next=false # Got value, back to normal. else case $ac_arg in *=* | --config-cache | -C | -disable-* | --disable-* \ | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ | -with-* | --with-* | -without-* | --without-* | --x) case "$ac_configure_args0 " in "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; esac ;; -* ) ac_must_keep_next=true ;; esac fi as_fn_append ac_configure_args " '$ac_arg'" ;; esac done done { ac_configure_args0=; unset ac_configure_args0;} { ac_configure_args1=; unset ac_configure_args1;} # When interrupted or exit'd, cleanup temporary files, and complete # config.log. We remove comments because anyway the quotes in there # would cause problems or look ugly. # WARNING: Use '\'' to represent an apostrophe within the trap. # WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. trap 'exit_status=$? # Save into config.log some information that might help in debugging. { echo $as_echo "## ---------------- ## ## Cache variables. ## ## ---------------- ##" echo # The following way of writing the cache mishandles newlines in values, ( for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do eval ac_val=\$$ac_var case $ac_val in #( *${as_nl}*) case $ac_var in #( *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; esac case $ac_var in #( _ | IFS | as_nl) ;; #( BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( *) { eval $ac_var=; unset $ac_var;} ;; esac ;; esac done (set) 2>&1 | case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( *${as_nl}ac_space=\ *) sed -n \ "s/'\''/'\''\\\\'\'''\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" ;; #( *) sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | sort ) echo $as_echo "## ----------------- ## ## Output variables. ## ## ----------------- ##" echo for ac_var in $ac_subst_vars do eval ac_val=\$$ac_var case $ac_val in *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; esac $as_echo "$ac_var='\''$ac_val'\''" done | sort echo if test -n "$ac_subst_files"; then $as_echo "## ------------------- ## ## File substitutions. ## ## ------------------- ##" echo for ac_var in $ac_subst_files do eval ac_val=\$$ac_var case $ac_val in *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; esac $as_echo "$ac_var='\''$ac_val'\''" done | sort echo fi if test -s confdefs.h; then $as_echo "## ----------- ## ## confdefs.h. ## ## ----------- ##" echo cat confdefs.h echo fi test "$ac_signal" != 0 && $as_echo "$as_me: caught signal $ac_signal" $as_echo "$as_me: exit $exit_status" } >&5 rm -f core *.core core.conftest.* && rm -f -r conftest* confdefs* conf$$* $ac_clean_files && exit $exit_status ' 0 for ac_signal in 1 2 13 15; do trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal done ac_signal=0 # confdefs.h avoids OS command line length limits that DEFS can exceed. rm -f -r conftest* confdefs.h $as_echo "/* confdefs.h */" > confdefs.h # Predefined preprocessor variables. cat >>confdefs.h <<_ACEOF #define PACKAGE_NAME "$PACKAGE_NAME" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_TARNAME "$PACKAGE_TARNAME" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_VERSION "$PACKAGE_VERSION" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_STRING "$PACKAGE_STRING" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" _ACEOF cat >>confdefs.h <<_ACEOF #define PACKAGE_URL "$PACKAGE_URL" _ACEOF # Let the site file select an alternate cache file if it wants to. # Prefer an explicitly selected file to automatically selected ones. ac_site_file1=NONE ac_site_file2=NONE if test -n "$CONFIG_SITE"; then # We do not want a PATH search for config.site. case $CONFIG_SITE in #(( -*) ac_site_file1=./$CONFIG_SITE;; */*) ac_site_file1=$CONFIG_SITE;; *) ac_site_file1=./$CONFIG_SITE;; esac elif test "x$prefix" != xNONE; then ac_site_file1=$prefix/share/config.site ac_site_file2=$prefix/etc/config.site else ac_site_file1=$ac_default_prefix/share/config.site ac_site_file2=$ac_default_prefix/etc/config.site fi for ac_site_file in "$ac_site_file1" "$ac_site_file2" do test "x$ac_site_file" = xNONE && continue if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 $as_echo "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 . "$ac_site_file" \ || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "failed to load site script $ac_site_file See \`config.log' for more details" "$LINENO" 5; } fi done if test -r "$cache_file"; then # Some versions of bash will fail to source /dev/null (special files # actually), so we avoid doing that. DJGPP emulates it as a regular file. if test /dev/null != "$cache_file" && test -f "$cache_file"; then { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 $as_echo "$as_me: loading cache $cache_file" >&6;} case $cache_file in [\\/]* | ?:[\\/]* ) . "$cache_file";; *) . "./$cache_file";; esac fi else { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 $as_echo "$as_me: creating cache $cache_file" >&6;} >$cache_file fi # Check that the precious variables saved in the cache have kept the same # value. ac_cache_corrupted=false for ac_var in $ac_precious_vars; do eval ac_old_set=\$ac_cv_env_${ac_var}_set eval ac_new_set=\$ac_env_${ac_var}_set eval ac_old_val=\$ac_cv_env_${ac_var}_value eval ac_new_val=\$ac_env_${ac_var}_value case $ac_old_set,$ac_new_set in set,) { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 $as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} ac_cache_corrupted=: ;; ,set) { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 $as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} ac_cache_corrupted=: ;; ,);; *) if test "x$ac_old_val" != "x$ac_new_val"; then # differences in whitespace do not lead to failure. ac_old_val_w=`echo x $ac_old_val` ac_new_val_w=`echo x $ac_new_val` if test "$ac_old_val_w" != "$ac_new_val_w"; then { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 $as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} ac_cache_corrupted=: else { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 $as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} eval $ac_var=\$ac_old_val fi { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 $as_echo "$as_me: former value: \`$ac_old_val'" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 $as_echo "$as_me: current value: \`$ac_new_val'" >&2;} fi;; esac # Pass precious variables to config.status. if test "$ac_new_set" = set; then case $ac_new_val in *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; *) ac_arg=$ac_var=$ac_new_val ;; esac case " $ac_configure_args " in *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. *) as_fn_append ac_configure_args " '$ac_arg'" ;; esac fi done if $ac_cache_corrupted; then { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 $as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## ## Main body of script. ## ## -------------------- ## ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu # we'll need sed and awk for some of the version commands { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5 $as_echo_n "checking for a sed that does not truncate output... " >&6; } if ${ac_cv_path_SED+:} false; then : $as_echo_n "(cached) " >&6 else ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ for ac_i in 1 2 3 4 5 6 7; do ac_script="$ac_script$as_nl$ac_script" done echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed { ac_script=; unset ac_script;} if test -z "$SED"; then ac_path_SED_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in sed gsed; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_SED="$as_dir/$ac_prog$ac_exec_ext" as_fn_executable_p "$ac_path_SED" || continue # Check for GNU ac_path_SED and select it if it is found. # Check for GNU $ac_path_SED case `"$ac_path_SED" --version 2>&1` in *GNU*) ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; *) ac_count=0 $as_echo_n 0123456789 >"conftest.in" while : do cat "conftest.in" "conftest.in" >"conftest.tmp" mv "conftest.tmp" "conftest.in" cp "conftest.in" "conftest.nl" $as_echo '' >> "conftest.nl" "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break as_fn_arith $ac_count + 1 && ac_count=$as_val if test $ac_count -gt ${ac_path_SED_max-0}; then # Best one so far, save it but keep looking for a better one ac_cv_path_SED="$ac_path_SED" ac_path_SED_max=$ac_count fi # 10*(2^10) chars as input seems more than enough test $ac_count -gt 10 && break done rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac $ac_path_SED_found && break 3 done done done IFS=$as_save_IFS if test -z "$ac_cv_path_SED"; then as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5 fi else ac_cv_path_SED=$SED fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 $as_echo "$ac_cv_path_SED" >&6; } SED="$ac_cv_path_SED" rm -f conftest.sed for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_AWK+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$AWK"; then ac_cv_prog_AWK="$AWK" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_AWK="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi AWK=$ac_cv_prog_AWK if test -n "$AWK"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 $as_echo "$AWK" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$AWK" && break done # CITUS_NAME definition cat >>confdefs.h <<_ACEOF #define CITUS_NAME "$PACKAGE_NAME" _ACEOF case $PACKAGE_NAME in 'Citus Enterprise') citus_edition=enterprise ;; Citus) citus_edition=community ;; *) as_fn_error $? "Unrecognized package name." "$LINENO" 5 ;; esac # CITUS_EDITION definition cat >>confdefs.h <<_ACEOF #define CITUS_EDITION "$citus_edition" _ACEOF # CITUS_MAJORVERSION definition CITUS_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\.[0-9][0-9]*\)'` cat >>confdefs.h <<_ACEOF #define CITUS_MAJORVERSION "$CITUS_MAJORVERSION" _ACEOF # CITUS_VERSION definition # Check whether --with-extra-version was given. if test "${with_extra_version+set}" = set; then : withval=$with_extra_version; case $withval in yes) as_fn_error $? "argument required for --with-extra-version option" "$LINENO" 5 ;; no) as_fn_error $? "argument required for --with-extra-version option" "$LINENO" 5 ;; *) CITUS_VERSION="$PACKAGE_VERSION$withval" ;; esac else CITUS_VERSION="$PACKAGE_VERSION" fi cat >>confdefs.h <<_ACEOF #define CITUS_VERSION "$CITUS_VERSION" _ACEOF # CITUS_VERSION_NUM definition # awk -F is a regex on some platforms, and not on others, so make "." a tab CITUS_VERSION_NUM="`echo "$PACKAGE_VERSION" | sed 's/[A-Za-z].*$//' | tr '.' ' ' | $AWK '{printf "%d%02d%02d", $1, $2, (NF >= 3) ? $3 : 0}'`" cat >>confdefs.h <<_ACEOF #define CITUS_VERSION_NUM $CITUS_VERSION_NUM _ACEOF # CITUS_EXTENSIONVERSION definition CITUS_EXTENSIONVERSION="`grep '^default_version' $srcdir/src/backend/distributed/citus.control | cut -d\' -f2`" cat >>confdefs.h <<_ACEOF #define CITUS_EXTENSIONVERSION "$CITUS_EXTENSIONVERSION" _ACEOF # Re-check for flex. That allows to compile citus against a postgres # which was built without flex available (possible because generated # files are included) # Extract the first word of "flex", so it can be a program name with args. set dummy flex; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_FLEX+:} false; then : $as_echo_n "(cached) " >&6 else case $FLEX in [\\/]* | ?:[\\/]*) ac_cv_path_FLEX="$FLEX" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_FLEX="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS ;; esac fi FLEX=$ac_cv_path_FLEX if test -n "$FLEX"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FLEX" >&5 $as_echo "$FLEX" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi # Locate pg_config binary if test -z "$PG_CONFIG"; then # Extract the first word of "pg_config", so it can be a program name with args. set dummy pg_config; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_PG_CONFIG+:} false; then : $as_echo_n "(cached) " >&6 else case $PG_CONFIG in [\\/]* | ?:[\\/]*) ac_cv_path_PG_CONFIG="$PG_CONFIG" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_PG_CONFIG="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS ;; esac fi PG_CONFIG=$ac_cv_path_PG_CONFIG if test -n "$PG_CONFIG"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PG_CONFIG" >&5 $as_echo "$PG_CONFIG" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi fi if test -z "$PG_CONFIG"; then as_fn_error $? "Could not find pg_config. Set PG_CONFIG or PATH." "$LINENO" 5 fi # check we're building against a supported version of PostgreSQL citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) version_num=$(echo "$citusac_pg_config_version"| $SED -e 's/^PostgreSQL \([0-9]*\)\(\.[0-9]*\)\{0,1\}\(.*\)$/\1\2/') # if PostgreSQL version starts with two digits, the major version is those digits version_num=$(echo "$version_num"| $SED -e 's/^\([0-9]\{2\}\)\(.*\)$/\1/') if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 fi # Check whether --with-pg-version-check was given. if test "${with_pg_version_check+set}" = set; then : withval=$with_pg_version_check; case $withval in yes) : ;; no) : ;; *) as_fn_error $? "no argument expected for --with-pg-version-check option" "$LINENO" 5 ;; esac else with_pg_version_check=yes fi if test "$with_pg_version_check" = no; then { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;} elif test "$version_num" != '16' -a "$version_num" != '17' -a "$version_num" != '18'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 $as_echo "$as_me: building against PostgreSQL $version_num" >&6;} fi; # Check whether we're building inside the source tree, if not, prepare # the build directory. if test "$srcdir" -ef '.' ; then vpath_build=no else vpath_build=yes $as_echo_n "preparing build tree... " >&6 citusac_abs_top_srcdir=`cd "$srcdir" && pwd` $SHELL "$citusac_abs_top_srcdir/prep_buildtree" "$citusac_abs_top_srcdir" "." \ || as_fn_error $? "failed" "$LINENO" 5 { $as_echo "$as_me:${as_lineno-$LINENO}: result: done" >&5 $as_echo "done" >&6; } fi # Allow to overwrite the C compiler, default to the one postgres was # compiled with. We don't want autoconf's default CFLAGS though, so save # those. SAVE_CFLAGS="$CFLAGS" ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu if test -n "$ac_tool_prefix"; then for ac_prog in $($PG_CONFIG --cc) do # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. set dummy $ac_tool_prefix$ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then ac_cv_prog_CC="$CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_CC="$ac_tool_prefix$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi CC=$ac_cv_prog_CC if test -n "$CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 $as_echo "$CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$CC" && break done fi if test -z "$CC"; then ac_ct_CC=$CC for ac_prog in $($PG_CONFIG --cc) do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_prog_ac_ct_CC="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS fi fi ac_ct_CC=$ac_cv_prog_ac_ct_CC if test -n "$ac_ct_CC"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 $as_echo "$ac_ct_CC" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi test -n "$ac_ct_CC" && break done if test "x$ac_ct_CC" = x; then CC="" else case $cross_compiling:$ac_tool_warned in yes:) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 $as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} ac_tool_warned=yes ;; esac CC=$ac_ct_CC fi fi test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "no acceptable C compiler found in \$PATH See \`config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 set X $ac_compile ac_compiler=$2 for ac_option in --version -v -V -qversion; do { { ac_try="$ac_compiler $ac_option >&5" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compiler $ac_option >&5") 2>conftest.err ac_status=$? if test -s conftest.err; then sed '10a\ ... rest of stderr output deleted ... 10q' conftest.err >conftest.er1 cat conftest.er1 >&5 fi rm -f conftest.er1 conftest.err $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } done cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" # Try to create an executable without -o first, disregard a.out. # It will help us diagnose broken compilers, and finding out an intuition # of exeext. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5 $as_echo_n "checking whether the C compiler works... " >&6; } ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` # The possible output files: ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" ac_rmfiles= for ac_file in $ac_files do case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; * ) ac_rmfiles="$ac_rmfiles $ac_file";; esac done rm -f $ac_rmfiles if { { ac_try="$ac_link_default" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link_default") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. # So ignore a value of `no', otherwise this would lead to `EXEEXT = no' # in a Makefile. We should not override ac_cv_exeext if it was cached, # so that the user can short-circuit this test for compilers unknown to # Autoconf. for ac_file in $ac_files '' do test -f "$ac_file" || continue case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; [ab].out ) # We found the default executable, but exeext='' is most # certainly right. break;; *.* ) if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; then :; else ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` fi # We set ac_cv_exeext here because the later test for it is not # safe: cross compilers may not add the suffix if given an `-o' # argument, so we may need to know it at that point already. # Even if this section looks crufty: it has the advantage of # actually working. break;; * ) break;; esac done test "$ac_cv_exeext" = no && ac_cv_exeext= else ac_file='' fi if test -z "$ac_file"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error 77 "C compiler cannot create executables See \`config.log' for more details" "$LINENO" 5; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 $as_echo_n "checking for C compiler default output file name... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 $as_echo "$ac_file" >&6; } ac_exeext=$ac_cv_exeext rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 $as_echo_n "checking for suffix of executables... " >&6; } if { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : # If both `conftest.exe' and `conftest' are `present' (well, observable) # catch `conftest.exe'. For instance with Cygwin, `ls conftest' will # work properly (i.e., refer to `conftest.exe'), while it won't with # `rm'. for ac_file in conftest.exe conftest conftest.*; do test -f "$ac_file" || continue case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` break;; * ) break;; esac done else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of executables: cannot compile and link See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest conftest$ac_cv_exeext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 $as_echo "$ac_cv_exeext" >&6; } rm -f conftest.$ac_ext EXEEXT=$ac_cv_exeext ac_exeext=$EXEEXT cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { FILE *f = fopen ("conftest.out", "w"); return ferror (f) || fclose (f) != 0; ; return 0; } _ACEOF ac_clean_files="$ac_clean_files conftest.out" # Check that the compiler produces executables we can run. If not, either # the compiler is broken, or we cross compile. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 $as_echo_n "checking whether we are cross compiling... " >&6; } if test "$cross_compiling" != yes; then { { ac_try="$ac_link" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_link") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } if { ac_try='./conftest$ac_cv_exeext' { { case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_try") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; }; then cross_compiling=no else if test "$cross_compiling" = maybe; then cross_compiling=yes else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot run C compiled programs. If you meant to cross compile, use \`--host'. See \`config.log' for more details" "$LINENO" 5; } fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 $as_echo "$cross_compiling" >&6; } rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 $as_echo_n "checking for suffix of object files... " >&6; } if ${ac_cv_objext+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF rm -f conftest.o conftest.obj if { { ac_try="$ac_compile" case "(($ac_try" in *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; *) ac_try_echo=$ac_try;; esac eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" $as_echo "$ac_try_echo"; } >&5 (eval "$ac_compile") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then : for ac_file in conftest.o conftest.obj conftest.*; do test -f "$ac_file" || continue; case $ac_file in *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` break;; esac done else $as_echo "$as_me: failed program was:" >&5 sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "cannot compute suffix of object files: cannot compile See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest.$ac_cv_objext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 $as_echo "$ac_cv_objext" >&6; } OBJEXT=$ac_cv_objext ac_objext=$OBJEXT { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 $as_echo_n "checking whether we are using the GNU C compiler... " >&6; } if ${ac_cv_c_compiler_gnu+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { #ifndef __GNUC__ choke me #endif ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_compiler_gnu=yes else ac_compiler_gnu=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_cv_c_compiler_gnu=$ac_compiler_gnu fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 $as_echo "$ac_cv_c_compiler_gnu" >&6; } if test $ac_compiler_gnu = yes; then GCC=yes else GCC= fi ac_test_CFLAGS=${CFLAGS+set} ac_save_CFLAGS=$CFLAGS { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 $as_echo_n "checking whether $CC accepts -g... " >&6; } if ${ac_cv_prog_cc_g+:} false; then : $as_echo_n "(cached) " >&6 else ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes ac_cv_prog_cc_g=no CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_g=yes else CFLAGS="" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : else ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="-g" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_g=yes fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 $as_echo "$ac_cv_prog_cc_g" >&6; } if test "$ac_test_CFLAGS" = set; then CFLAGS=$ac_save_CFLAGS elif test $ac_cv_prog_cc_g = yes; then if test "$GCC" = yes; then CFLAGS="-g -O2" else CFLAGS="-g" fi else if test "$GCC" = yes; then CFLAGS="-O2" else CFLAGS= fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 $as_echo_n "checking for $CC option to accept ISO C89... " >&6; } if ${ac_cv_prog_cc_c89+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_prog_cc_c89=no ac_save_CC=$CC cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include struct stat; /* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ struct buf { int x; }; FILE * (*rcsopen) (struct buf *, struct stat *, int); static char *e (p, i) char **p; int i; { return p[i]; } static char *f (char * (*g) (char **, int), char **p, ...) { char *s; va_list v; va_start (v,p); s = g (p, va_arg (v,int)); va_end (v); return s; } /* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has function prototypes and stuff, but not '\xHH' hex character constants. These don't provoke an error unfortunately, instead are silently treated as 'x'. The following induces an error, until -std is added to get proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an array size at least. It's necessary to write '\x00'==0 to get something that's true only with -std. */ int osf4_cc_array ['\x00' == 0 ? 1 : -1]; /* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters inside strings and character constants. */ #define FOO(x) 'x' int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; int test (int i, double x); struct s1 {int (*f) (int a);}; struct s2 {int (*f) (double a);}; int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); int argc; char **argv; int main () { return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; ; return 0; } _ACEOF for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" do CC="$ac_save_CC $ac_arg" if ac_fn_c_try_compile "$LINENO"; then : ac_cv_prog_cc_c89=$ac_arg fi rm -f core conftest.err conftest.$ac_objext test "x$ac_cv_prog_cc_c89" != "xno" && break done rm -f conftest.$ac_ext CC=$ac_save_CC fi # AC_CACHE_VAL case "x$ac_cv_prog_cc_c89" in x) { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 $as_echo "none needed" >&6; } ;; xno) { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 $as_echo "unsupported" >&6; } ;; *) CC="$CC $ac_cv_prog_cc_c89" { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 $as_echo "$ac_cv_prog_cc_c89" >&6; } ;; esac if test "x$ac_cv_prog_cc_c89" != xno; then : fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu CFLAGS="$SAVE_CFLAGS" host_guess=`${SHELL} $srcdir/config/config.guess` # Create compiler version string if test x"$GCC" = x"yes" ; then cc_string=`${CC} --version | sed q` case $cc_string in [A-Za-z]*) ;; *) cc_string="GCC $cc_string";; esac elif test x"$SUN_STUDIO_CC" = x"yes" ; then cc_string=`${CC} -V 2>&1 | sed q` else cc_string=$CC fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 $as_echo_n "checking how to run the C preprocessor... " >&6; } # On Suns, sometimes $CPP names a directory. if test -n "$CPP" && test -d "$CPP"; then CPP= fi if test -z "$CPP"; then if ${ac_cv_prog_CPP+:} false; then : $as_echo_n "(cached) " >&6 else # Double quotes because CPP needs to be expanded for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" do ac_preproc_ok=false for ac_c_preproc_warn_flag in '' yes do # Use a header file that comes with gcc, so configuring glibc # with a fresh cross-compiler works. # Prefer to if __STDC__ is defined, since # exists even on freestanding compilers. # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. "Syntax error" is here to catch this case. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef __STDC__ # include #else # include #endif Syntax error _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : else # Broken: fails on valid input. continue fi rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : # Broken: success on invalid input. continue else # Passes both tests. ac_preproc_ok=: break fi rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : break fi done ac_cv_prog_CPP=$CPP fi CPP=$ac_cv_prog_CPP else ac_cv_prog_CPP=$CPP fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 $as_echo "$CPP" >&6; } ac_preproc_ok=false for ac_c_preproc_warn_flag in '' yes do # Use a header file that comes with gcc, so configuring glibc # with a fresh cross-compiler works. # Prefer to if __STDC__ is defined, since # exists even on freestanding compilers. # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. "Syntax error" is here to catch this case. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #ifdef __STDC__ # include #else # include #endif Syntax error _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : else # Broken: fails on valid input. continue fi rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if ac_fn_c_try_cpp "$LINENO"; then : # Broken: success on invalid input. continue else # Passes both tests. ac_preproc_ok=: break fi rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "C preprocessor \"$CPP\" fails sanity check See \`config.log' for more details" "$LINENO" 5; } fi ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 $as_echo_n "checking for grep that handles long lines and -e... " >&6; } if ${ac_cv_path_GREP+:} false; then : $as_echo_n "(cached) " >&6 else if test -z "$GREP"; then ac_path_GREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in grep ggrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" as_fn_executable_p "$ac_path_GREP" || continue # Check for GNU ac_path_GREP and select it if it is found. # Check for GNU $ac_path_GREP case `"$ac_path_GREP" --version 2>&1` in *GNU*) ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; *) ac_count=0 $as_echo_n 0123456789 >"conftest.in" while : do cat "conftest.in" "conftest.in" >"conftest.tmp" mv "conftest.tmp" "conftest.in" cp "conftest.in" "conftest.nl" $as_echo 'GREP' >> "conftest.nl" "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break as_fn_arith $ac_count + 1 && ac_count=$as_val if test $ac_count -gt ${ac_path_GREP_max-0}; then # Best one so far, save it but keep looking for a better one ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_max=$ac_count fi # 10*(2^10) chars as input seems more than enough test $ac_count -gt 10 && break done rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac $ac_path_GREP_found && break 3 done done done IFS=$as_save_IFS if test -z "$ac_cv_path_GREP"; then as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_GREP=$GREP fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 $as_echo "$ac_cv_path_GREP" >&6; } GREP="$ac_cv_path_GREP" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 $as_echo_n "checking for egrep... " >&6; } if ${ac_cv_path_EGREP+:} false; then : $as_echo_n "(cached) " >&6 else if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 then ac_cv_path_EGREP="$GREP -E" else if test -z "$EGREP"; then ac_path_EGREP_found=false # Loop through the user's path and test for each of PROGNAME-LIST as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_prog in egrep; do for ac_exec_ext in '' $ac_executable_extensions; do ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" as_fn_executable_p "$ac_path_EGREP" || continue # Check for GNU ac_path_EGREP and select it if it is found. # Check for GNU $ac_path_EGREP case `"$ac_path_EGREP" --version 2>&1` in *GNU*) ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; *) ac_count=0 $as_echo_n 0123456789 >"conftest.in" while : do cat "conftest.in" "conftest.in" >"conftest.tmp" mv "conftest.tmp" "conftest.in" cp "conftest.in" "conftest.nl" $as_echo 'EGREP' >> "conftest.nl" "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break as_fn_arith $ac_count + 1 && ac_count=$as_val if test $ac_count -gt ${ac_path_EGREP_max-0}; then # Best one so far, save it but keep looking for a better one ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_max=$ac_count fi # 10*(2^10) chars as input seems more than enough test $ac_count -gt 10 && break done rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac $ac_path_EGREP_found && break 3 done done done IFS=$as_save_IFS if test -z "$ac_cv_path_EGREP"; then as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_EGREP=$EGREP fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 $as_echo "$ac_cv_path_EGREP" >&6; } EGREP="$ac_cv_path_EGREP" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 $as_echo_n "checking for ANSI C header files... " >&6; } if ${ac_cv_header_stdc+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #include #include int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : ac_cv_header_stdc=yes else ac_cv_header_stdc=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext if test $ac_cv_header_stdc = yes; then # SunOS 4.x string.h does not declare mem*, contrary to ANSI. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | $EGREP "memchr" >/dev/null 2>&1; then : else ac_cv_header_stdc=no fi rm -f conftest* fi if test $ac_cv_header_stdc = yes; then # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include _ACEOF if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | $EGREP "free" >/dev/null 2>&1; then : else ac_cv_header_stdc=no fi rm -f conftest* fi if test $ac_cv_header_stdc = yes; then # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. if test "$cross_compiling" = yes; then : : else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include #include #if ((' ' & 0x0FF) == 0x020) # define ISLOWER(c) ('a' <= (c) && (c) <= 'z') # define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) #else # define ISLOWER(c) \ (('a' <= (c) && (c) <= 'i') \ || ('j' <= (c) && (c) <= 'r') \ || ('s' <= (c) && (c) <= 'z')) # define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) #endif #define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) int main () { int i; for (i = 0; i < 256; i++) if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) return 2; return 0; } _ACEOF if ac_fn_c_try_run "$LINENO"; then : else ac_cv_header_stdc=no fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 $as_echo "$ac_cv_header_stdc" >&6; } if test $ac_cv_header_stdc = yes; then $as_echo "#define STDC_HEADERS 1" >>confdefs.h fi # On IRIX 5.3, sys/types and inttypes.h are conflicting. for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ inttypes.h stdint.h unistd.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default " if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF fi done # The cast to long int works around a bug in the HP C Compiler # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 $as_echo_n "checking size of void *... " >&6; } if ${ac_cv_sizeof_void_p+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default"; then : else if test "$ac_cv_type_void_p" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error 77 "cannot compute sizeof (void *) See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 fi fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_void_p" >&5 $as_echo "$ac_cv_sizeof_void_p" >&6; } cat >>confdefs.h <<_ACEOF #define SIZEOF_VOID_P $ac_cv_sizeof_void_p _ACEOF cat >>confdefs.h <<_ACEOF #define CITUS_VERSION_STR "$PACKAGE_NAME $CITUS_VERSION on $host_guess, compiled by $cc_string, `expr $ac_cv_sizeof_void_p \* 8`-bit" _ACEOF # Locate source and build directory of the postgres we're building # against. Can't rely on either still being present, but e.g. optional # test infrastructure can rely on it. POSTGRES_SRCDIR=$(grep ^abs_top_srcdir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) POSTGRES_BUILDDIR=$(grep ^abs_top_builddir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) # check for a number of CFLAGS that make development easier # CITUSAC_PROG_CC_CFLAGS_OPT # ----------------------- # Given a string, check if the compiler supports the string as a # command-line option. If it does, add the string to CFLAGS. # CITUSAC_PROG_CC_CFLAGS_OPT { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -std=gnu99" >&5 $as_echo_n "checking whether $CC supports -std=gnu99... " >&6; } if ${citusac_cv_prog_cc_cflags__std_gnu99+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-std=gnu99 case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__std_gnu99=yes else citusac_cv_prog_cc_cflags__std_gnu99=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__std_gnu99" >&5 $as_echo "$citusac_cv_prog_cc_cflags__std_gnu99" >&6; } if test x"$citusac_cv_prog_cc_cflags__std_gnu99" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -std=gnu99" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wall" >&5 $as_echo_n "checking whether $CC supports -Wall... " >&6; } if ${citusac_cv_prog_cc_cflags__Wall+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wall case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wall=yes else citusac_cv_prog_cc_cflags__Wall=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wall" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wall" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wall" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wall" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wextra" >&5 $as_echo_n "checking whether $CC supports -Wextra... " >&6; } if ${citusac_cv_prog_cc_cflags__Wextra+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wextra case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wextra=yes else citusac_cv_prog_cc_cflags__Wextra=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wextra" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wextra" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wextra" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wextra" fi # disarm options included in the above, which are too noisy for now { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-unused-parameter" >&5 $as_echo_n "checking whether $CC supports -Wno-unused-parameter... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_unused_parameter+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-unused-parameter case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_unused_parameter=yes else citusac_cv_prog_cc_cflags__Wno_unused_parameter=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_unused_parameter" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_unused_parameter" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_unused_parameter" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-unused-parameter" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-sign-compare" >&5 $as_echo_n "checking whether $CC supports -Wno-sign-compare... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_sign_compare+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-sign-compare case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_sign_compare=yes else citusac_cv_prog_cc_cflags__Wno_sign_compare=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_sign_compare" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_sign_compare" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_sign_compare" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-sign-compare" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-missing-field-initializers" >&5 $as_echo_n "checking whether $CC supports -Wno-missing-field-initializers... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_missing_field_initializers+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-missing-field-initializers case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_missing_field_initializers=yes else citusac_cv_prog_cc_cflags__Wno_missing_field_initializers=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_missing_field_initializers" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_missing_field_initializers" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_missing_field_initializers" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-missing-field-initializers" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-clobbered" >&5 $as_echo_n "checking whether $CC supports -Wno-clobbered... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_clobbered+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-clobbered case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_clobbered=yes else citusac_cv_prog_cc_cflags__Wno_clobbered=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_clobbered" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_clobbered" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_clobbered" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-clobbered" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-gnu-variable-sized-type-not-at-end" >&5 $as_echo_n "checking whether $CC supports -Wno-gnu-variable-sized-type-not-at-end... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-gnu-variable-sized-type-not-at-end case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end=yes else citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-gnu-variable-sized-type-not-at-end" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-declaration-after-statement" >&5 $as_echo_n "checking whether $CC supports -Wno-declaration-after-statement... " >&6; } if ${citusac_cv_prog_cc_cflags__Wno_declaration_after_statement+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wno-declaration-after-statement case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wno_declaration_after_statement=yes else citusac_cv_prog_cc_cflags__Wno_declaration_after_statement=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_declaration_after_statement" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wno_declaration_after_statement" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wno_declaration_after_statement" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-declaration-after-statement" fi # And add a few extra warnings { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wendif-labels" >&5 $as_echo_n "checking whether $CC supports -Wendif-labels... " >&6; } if ${citusac_cv_prog_cc_cflags__Wendif_labels+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wendif-labels case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wendif_labels=yes else citusac_cv_prog_cc_cflags__Wendif_labels=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wendif_labels" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wendif_labels" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wendif_labels" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wendif-labels" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wmissing-format-attribute" >&5 $as_echo_n "checking whether $CC supports -Wmissing-format-attribute... " >&6; } if ${citusac_cv_prog_cc_cflags__Wmissing_format_attribute+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wmissing-format-attribute case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wmissing_format_attribute=yes else citusac_cv_prog_cc_cflags__Wmissing_format_attribute=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wmissing_format_attribute" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wmissing_format_attribute" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wmissing_format_attribute" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wmissing-format-attribute" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wmissing-declarations" >&5 $as_echo_n "checking whether $CC supports -Wmissing-declarations... " >&6; } if ${citusac_cv_prog_cc_cflags__Wmissing_declarations+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wmissing-declarations case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wmissing_declarations=yes else citusac_cv_prog_cc_cflags__Wmissing_declarations=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wmissing_declarations" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wmissing_declarations" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wmissing_declarations" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wmissing-declarations" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wmissing-prototypes" >&5 $as_echo_n "checking whether $CC supports -Wmissing-prototypes... " >&6; } if ${citusac_cv_prog_cc_cflags__Wmissing_prototypes+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wmissing-prototypes case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wmissing_prototypes=yes else citusac_cv_prog_cc_cflags__Wmissing_prototypes=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wmissing_prototypes" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wmissing_prototypes" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wmissing_prototypes" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wmissing-prototypes" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wshadow" >&5 $as_echo_n "checking whether $CC supports -Wshadow... " >&6; } if ${citusac_cv_prog_cc_cflags__Wshadow+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Wshadow case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Wshadow=yes else citusac_cv_prog_cc_cflags__Wshadow=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wshadow" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Wshadow" >&6; } if test x"$citusac_cv_prog_cc_cflags__Wshadow" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wshadow" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Werror=vla" >&5 $as_echo_n "checking whether $CC supports -Werror=vla... " >&6; } if ${citusac_cv_prog_cc_cflags__Werror_vla+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Werror=vla case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Werror_vla=yes else citusac_cv_prog_cc_cflags__Werror_vla=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Werror_vla" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Werror_vla" >&6; } if test x"$citusac_cv_prog_cc_cflags__Werror_vla" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Werror=vla" fi # visual studio does not support these { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Werror=implicit-int" >&5 $as_echo_n "checking whether $CC supports -Werror=implicit-int... " >&6; } if ${citusac_cv_prog_cc_cflags__Werror_implicit_int+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Werror=implicit-int case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Werror_implicit_int=yes else citusac_cv_prog_cc_cflags__Werror_implicit_int=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Werror_implicit_int" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Werror_implicit_int" >&6; } if test x"$citusac_cv_prog_cc_cflags__Werror_implicit_int" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Werror=implicit-int" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Werror=implicit-function-declaration" >&5 $as_echo_n "checking whether $CC supports -Werror=implicit-function-declaration... " >&6; } if ${citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Werror=implicit-function-declaration case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration=yes else citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration" >&6; } if test x"$citusac_cv_prog_cc_cflags__Werror_implicit_function_declaration" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Werror=implicit-function-declaration" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Werror=return-type" >&5 $as_echo_n "checking whether $CC supports -Werror=return-type... " >&6; } if ${citusac_cv_prog_cc_cflags__Werror_return_type+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-Werror=return-type case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__Werror_return_type=yes else citusac_cv_prog_cc_cflags__Werror_return_type=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Werror_return_type" >&5 $as_echo "$citusac_cv_prog_cc_cflags__Werror_return_type" >&6; } if test x"$citusac_cv_prog_cc_cflags__Werror_return_type" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Werror=return-type" fi # Security flags # Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide # We do not enforce the following flag because it is only available on GCC>=8 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -fstack-clash-protection" >&5 $as_echo_n "checking whether $CC supports -fstack-clash-protection... " >&6; } if ${citusac_cv_prog_cc_cflags__fstack_clash_protection+:} false; then : $as_echo_n "(cached) " >&6 else citusac_save_CFLAGS=$CFLAGS flag=-fstack-clash-protection case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main () { ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : citusac_cv_prog_cc_cflags__fstack_clash_protection=yes else citusac_cv_prog_cc_cflags__fstack_clash_protection=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__fstack_clash_protection" >&5 $as_echo "$citusac_cv_prog_cc_cflags__fstack_clash_protection" >&6; } if test x"$citusac_cv_prog_cc_cflags__fstack_clash_protection" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -fstack-clash-protection" fi # # --enable-coverage enables generation of code coverage metrics with gcov # # Check whether --enable-coverage was given. if test "${enable_coverage+set}" = set; then : enableval=$enable_coverage; fi if test "$enable_coverage" = yes; then CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage" CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG" CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage" fi # # LZ4 # # Check whether --with-lz4 was given. if test "${with_lz4+set}" = set; then : withval=$with_lz4; case $withval in yes) $as_echo "#define HAVE_CITUS_LIBLZ4 1" >>confdefs.h ;; no) : ;; *) as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 ;; esac else with_lz4=yes $as_echo "#define HAVE_CITUS_LIBLZ4 1" >>confdefs.h fi if test "$with_lz4" = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4_compress_default in -llz4" >&5 $as_echo_n "checking for LZ4_compress_default in -llz4... " >&6; } if ${ac_cv_lib_lz4_LZ4_compress_default+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS LIBS="-llz4 $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char LZ4_compress_default (); int main () { return LZ4_compress_default (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : ac_cv_lib_lz4_LZ4_compress_default=yes else ac_cv_lib_lz4_LZ4_compress_default=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lz4_LZ4_compress_default" >&5 $as_echo "$ac_cv_lib_lz4_LZ4_compress_default" >&6; } if test "x$ac_cv_lib_lz4_LZ4_compress_default" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_LIBLZ4 1 _ACEOF LIBS="-llz4 $LIBS" else as_fn_error $? "lz4 library not found If you have lz4 installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-lz4 to disable lz4 support." "$LINENO" 5 fi ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" if test "x$ac_cv_header_lz4_h" = xyes; then : else as_fn_error $? "lz4 header not found If you have lz4 already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-lz4 to disable lz4 support." "$LINENO" 5 fi fi # # ZSTD # # Check whether --with-zstd was given. if test "${with_zstd+set}" = set; then : withval=$with_zstd; case $withval in yes) : ;; no) : ;; *) as_fn_error $? "no argument expected for --with-zstd option" "$LINENO" 5 ;; esac else with_zstd=yes fi if test "$with_zstd" = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_decompress in -lzstd" >&5 $as_echo_n "checking for ZSTD_decompress in -lzstd... " >&6; } if ${ac_cv_lib_zstd_ZSTD_decompress+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS LIBS="-lzstd $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char ZSTD_decompress (); int main () { return ZSTD_decompress (); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : ac_cv_lib_zstd_ZSTD_decompress=yes else ac_cv_lib_zstd_ZSTD_decompress=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_decompress" >&5 $as_echo "$ac_cv_lib_zstd_ZSTD_decompress" >&6; } if test "x$ac_cv_lib_zstd_ZSTD_decompress" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_LIBZSTD 1 _ACEOF LIBS="-lzstd $LIBS" else as_fn_error $? "zstd library not found If you have zstd installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-zstd to disable zstd support." "$LINENO" 5 fi ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default" if test "x$ac_cv_header_zstd_h" = xyes; then : else as_fn_error $? "zstd header not found If you have zstd already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-zstd to disable zstd support." "$LINENO" 5 fi fi # Check whether --with-security-flags was given. if test "${with_security_flags+set}" = set; then : withval=$with_security_flags; case $withval in yes) : ;; no) : ;; *) as_fn_error $? "no argument expected for --with-security-flags option" "$LINENO" 5 ;; esac else with_security_flags=no fi if test "$with_security_flags" = yes; then # Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide # We always want to have some compiler flags for security concerns. SECURITY_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2 -z noexecstack -fpic -shared -Wl,-z,relro -Wl,-z,now -Wformat -Wformat-security -Werror=format-security" CITUS_CFLAGS="$CITUS_CFLAGS $SECURITY_CFLAGS" { $as_echo "$as_me:${as_lineno-$LINENO}: Blindly added security flags for linker: $SECURITY_CFLAGS" >&5 $as_echo "$as_me: Blindly added security flags for linker: $SECURITY_CFLAGS" >&6;} # We always want to have some clang flags for security concerns. # This doesn't include "-Wl,-z,relro -Wl,-z,now" on purpuse, because bitcode is not linked. # This doesn't include -fsanitize=cfi because it breaks builds on many distros including # Debian/Buster, Debian/Stretch, Ubuntu/Bionic, Ubuntu/Xenial and EL7. SECURITY_BITCODE_CFLAGS="-fsanitize=safe-stack -fstack-protector-strong -flto -fPIC -Wformat -Wformat-security -Werror=format-security" CITUS_BITCODE_CFLAGS="$CITUS_BITCODE_CFLAGS $SECURITY_BITCODE_CFLAGS" { $as_echo "$as_me:${as_lineno-$LINENO}: Blindly added security flags for llvm: $SECURITY_BITCODE_CFLAGS" >&5 $as_echo "$as_me: Blindly added security flags for llvm: $SECURITY_BITCODE_CFLAGS" >&6;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: If you run into issues during linking or bitcode compilation, you can use --without-security-flags." >&5 $as_echo "$as_me: WARNING: If you run into issues during linking or bitcode compilation, you can use --without-security-flags." >&2;} fi # Check if git is installed, when installed the gitref of the checkout will be baked in the application # Extract the first word of "git", so it can be a program name with args. set dummy git; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } if ${ac_cv_path_GIT_BIN+:} false; then : $as_echo_n "(cached) " >&6 else case $GIT_BIN in [\\/]* | ?:[\\/]*) ac_cv_path_GIT_BIN="$GIT_BIN" # Let the user override the test with a path. ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then ac_cv_path_GIT_BIN="$as_dir/$ac_word$ac_exec_ext" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi done done IFS=$as_save_IFS ;; esac fi GIT_BIN=$ac_cv_path_GIT_BIN if test -n "$GIT_BIN"; then { $as_echo "$as_me:${as_lineno-$LINENO}: result: $GIT_BIN" >&5 $as_echo "$GIT_BIN" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for .git" >&5 $as_echo_n "checking for .git... " >&6; } if ${ac_cv_file__git+:} false; then : $as_echo_n "(cached) " >&6 else test "$cross_compiling" = yes && as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 if test -r ".git"; then ac_cv_file__git=yes else ac_cv_file__git=no fi fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_file__git" >&5 $as_echo "$ac_cv_file__git" >&6; } if test "x$ac_cv_file__git" = xyes; then : HAS_DOTGIT=yes else HAS_DOTGIT= fi CITUS_CFLAGS="$CITUS_CFLAGS" CITUS_BITCODE_CFLAGS="$CITUS_BITCODE_CFLAGS" CITUS_CPPFLAGS="$CITUS_CPPFLAGS" CITUS_LDFLAGS="$LIBS $CITUS_LDFLAGS" POSTGRES_SRCDIR="$POSTGRES_SRCDIR" POSTGRES_BUILDDIR="$POSTGRES_BUILDDIR" HAS_DOTGIT="$HAS_DOTGIT" ac_config_files="$ac_config_files Makefile.global" ac_config_headers="$ac_config_headers src/include/citus_config.h src/include/citus_version.h" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure # scripts and configure runs, see configure's option --config-cache. # It is not useful on other systems. If it contains results you don't # want to keep, you may remove or edit it. # # config.status only pays attention to the cache file if you give it # the --recheck option to rerun configure. # # `ac_cv_env_foo' variables (set or unset) will be overridden when # loading this file, other *unset* `ac_cv_foo' will be assigned the # following values. _ACEOF # The following way of writing the cache mishandles newlines in values, # but we know of no workaround that is simple, portable, and efficient. # So, we kill variables containing newlines. # Ultrix sh set writes to stderr and can't be redirected directly, # and sets the high bit in the cache file unless we assign to the vars. ( for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do eval ac_val=\$$ac_var case $ac_val in #( *${as_nl}*) case $ac_var in #( *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; esac case $ac_var in #( _ | IFS | as_nl) ;; #( BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( *) { eval $ac_var=; unset $ac_var;} ;; esac ;; esac done (set) 2>&1 | case $as_nl`(ac_space=' '; set) 2>&1` in #( *${as_nl}ac_space=\ *) # `set' does not quote correctly, so add quotes: double-quote # substitution turns \\\\ into \\, and sed turns \\ into \. sed -n \ "s/'/'\\\\''/g; s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" ;; #( *) # `set' quotes correctly as required by POSIX, so do not add quotes. sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" ;; esac | sort ) | sed ' /^ac_cv_env_/b end t clear :clear s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ t end s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ :end' >>confcache if diff "$cache_file" confcache >/dev/null 2>&1; then :; else if test -w "$cache_file"; then if test "x$cache_file" != "x/dev/null"; then { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 $as_echo "$as_me: updating cache $cache_file" >&6;} if test ! -f "$cache_file" || test -h "$cache_file"; then cat confcache >"$cache_file" else case $cache_file in #( */* | ?:*) mv -f confcache "$cache_file"$$ && mv -f "$cache_file"$$ "$cache_file" ;; #( *) mv -f confcache "$cache_file" ;; esac fi fi else { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 $as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} fi fi rm -f confcache test "x$prefix" = xNONE && prefix=$ac_default_prefix # Let make expand exec_prefix. test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' DEFS=-DHAVE_CONFIG_H ac_libobjs= ac_ltlibobjs= U= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' ac_i=`$as_echo "$ac_i" | sed "$ac_script"` # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR # will be set to the directory where LIBOBJS objects are built. as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo' done LIBOBJS=$ac_libobjs LTLIBOBJS=$ac_ltlibobjs : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files $CONFIG_STATUS" { $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 $as_echo "$as_me: creating $CONFIG_STATUS" >&6;} as_write_fail=0 cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 #! $SHELL # Generated by $as_me. # Run this file to recreate the current configuration. # Compiler output produced by configure, useful for debugging # configure, is in config.log if it exists. debug=false ac_cs_recheck=false ac_cs_silent=false SHELL=\${CONFIG_SHELL-$SHELL} export SHELL _ASEOF cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 ## -------------------- ## ## M4sh Initialization. ## ## -------------------- ## # Be more Bourne compatible DUALCASE=1; export DUALCASE # for MKS sh if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : emulate sh NULLCMD=: # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which # is contrary to our usage. Disable this feature. alias -g '${1+"$@"}'='"$@"' setopt NO_GLOB_SUBST else case `(set -o) 2>/dev/null` in #( *posix*) : set -o posix ;; #( *) : ;; esac fi as_nl=' ' export as_nl # Printing a long string crashes Solaris 7 /usr/bin/printf. as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo # Prefer a ksh shell builtin over an external printf program on Solaris, # but without wasting forks for bash or zsh. if test -z "$BASH_VERSION$ZSH_VERSION" \ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='print -r --' as_echo_n='print -rn --' elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then as_echo='printf %s\n' as_echo_n='printf %s' else if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' as_echo_n='/usr/ucb/echo -n' else as_echo_body='eval expr "X$1" : "X\\(.*\\)"' as_echo_n_body='eval arg=$1; case $arg in #( *"$as_nl"*) expr "X$arg" : "X\\(.*\\)$as_nl"; arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; esac; expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" ' export as_echo_n_body as_echo_n='sh -c $as_echo_n_body as_echo' fi export as_echo_body as_echo='sh -c $as_echo_body as_echo' fi # The user is always right. if test "${PATH_SEPARATOR+set}" != set; then PATH_SEPARATOR=: (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || PATH_SEPARATOR=';' } fi # IFS # We need space, tab and new line, in precisely that order. Quoting is # there to prevent editors from complaining about space-tab. # (If _AS_PATH_WALK were called with IFS unset, it would disable word # splitting by setting IFS to empty value.) IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH do IFS=$as_save_IFS test -z "$as_dir" && as_dir=. test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break done IFS=$as_save_IFS ;; esac # We did not find ourselves, most probably we were run as `sh COMMAND' # in which case we are not to be found in the path. if test "x$as_myself" = x; then as_myself=$0 fi if test ! -f "$as_myself"; then $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 exit 1 fi # Unset variables that we do not need and which cause bugs (e.g. in # pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" # suppresses any "Segmentation fault" message there. '((' could # trigger a bug in pdksh 5.2.14. for as_var in BASH_ENV ENV MAIL MAILPATH do eval test x\${$as_var+set} = xset \ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : done PS1='$ ' PS2='> ' PS4='+ ' # NLS nuisances. LC_ALL=C export LC_ALL LANGUAGE=C export LANGUAGE # CDPATH. (unset CDPATH) >/dev/null 2>&1 && unset CDPATH # as_fn_error STATUS ERROR [LINENO LOG_FD] # ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the # script with STATUS, using 1 if that was 0. as_fn_error () { as_status=$1; test $as_status -eq 0 && as_status=1 if test "$4"; then as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error # as_fn_set_status STATUS # ----------------------- # Set $? to STATUS, without forking. as_fn_set_status () { return $1 } # as_fn_set_status # as_fn_exit STATUS # ----------------- # Exit the shell with STATUS, even in a "trap 0" or "set -e" context. as_fn_exit () { set +e as_fn_set_status $1 exit $1 } # as_fn_exit # as_fn_unset VAR # --------------- # Portably unset VAR. as_fn_unset () { { eval $1=; unset $1;} } as_unset=as_fn_unset # as_fn_append VAR VALUE # ---------------------- # Append the text in VALUE to the end of the definition contained in VAR. Take # advantage of any shell optimizations that allow amortized linear growth over # repeated appends, instead of the typical quadratic growth present in naive # implementations. if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : eval 'as_fn_append () { eval $1+=\$2 }' else as_fn_append () { eval $1=\$$1\$2 } fi # as_fn_append # as_fn_arith ARG... # ------------------ # Perform arithmetic evaluation on the ARGs, and store the result in the # global $as_val. Take advantage of shells that can avoid forks. The arguments # must be portable across $(()) and expr. if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : eval 'as_fn_arith () { as_val=$(( $* )) }' else as_fn_arith () { as_val=`expr "$@" || test $? -eq 1` } fi # as_fn_arith if expr a : '\(a\)' >/dev/null 2>&1 && test "X`expr 00001 : '.*\(...\)'`" = X001; then as_expr=expr else as_expr=false fi if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then as_basename=basename else as_basename=false fi if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then as_dirname=dirname else as_dirname=false fi as_me=`$as_basename -- "$0" || $as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ X"$0" : 'X\(//\)$' \| \ X"$0" : 'X\(/\)' \| . 2>/dev/null || $as_echo X/"$0" | sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/ q } /^X\/\(\/\/\)$/{ s//\1/ q } /^X\/\(\/\).*/{ s//\1/ q } s/.*/./; q'` # Avoid depending upon Character Ranges. as_cr_letters='abcdefghijklmnopqrstuvwxyz' as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' as_cr_Letters=$as_cr_letters$as_cr_LETTERS as_cr_digits='0123456789' as_cr_alnum=$as_cr_Letters$as_cr_digits ECHO_C= ECHO_N= ECHO_T= case `echo -n x` in #((((( -n*) case `echo 'xy\c'` in *c*) ECHO_T=' ';; # ECHO_T is single tab character. xy) ECHO_C='\c';; *) echo `echo ksh88 bug on AIX 6.1` > /dev/null ECHO_T=' ';; esac;; *) ECHO_N='-n';; esac rm -f conf$$ conf$$.exe conf$$.file if test -d conf$$.dir; then rm -f conf$$.dir/conf$$.file else rm -f conf$$.dir mkdir conf$$.dir 2>/dev/null fi if (echo >conf$$.file) 2>/dev/null; then if ln -s conf$$.file conf$$ 2>/dev/null; then as_ln_s='ln -s' # ... but there are two gotchas: # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. # In both cases, we have to default to `cp -pR'. ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || as_ln_s='cp -pR' elif ln conf$$.file conf$$ 2>/dev/null; then as_ln_s=ln else as_ln_s='cp -pR' fi else as_ln_s='cp -pR' fi rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file rmdir conf$$.dir 2>/dev/null # as_fn_mkdir_p # ------------- # Create "$as_dir" as a directory, including parents if necessary. as_fn_mkdir_p () { case $as_dir in #( -*) as_dir=./$as_dir;; esac test -d "$as_dir" || eval $as_mkdir_p || { as_dirs= while :; do case $as_dir in #( *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( *) as_qdir=$as_dir;; esac as_dirs="'$as_qdir' $as_dirs" as_dir=`$as_dirname -- "$as_dir" || $as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$as_dir" : 'X\(//\)[^/]' \| \ X"$as_dir" : 'X\(//\)$' \| \ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$as_dir" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p if mkdir -p . 2>/dev/null; then as_mkdir_p='mkdir -p "$as_dir"' else test -d ./-p && rmdir ./-p as_mkdir_p=false fi # as_fn_executable_p FILE # ----------------------- # Test if FILE is an executable regular file. as_fn_executable_p () { test -f "$1" && test -x "$1" } # as_fn_executable_p as_test_x='test -x' as_executable_p=as_fn_executable_p # Sed expression to map a string onto a valid CPP name. as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" # Sed expression to map a string onto a valid variable name. as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" exec 6>&1 ## ----------------------------------- ## ## Main body of $CONFIG_STATUS script. ## ## ----------------------------------- ## _ASEOF test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" This file was extended by Citus $as_me 15.0devel, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS CONFIG_COMMANDS = $CONFIG_COMMANDS $ $0 $@ on `(hostname || uname -n) 2>/dev/null | sed 1q` " _ACEOF case $ac_config_files in *" "*) set x $ac_config_files; shift; ac_config_files=$*;; esac case $ac_config_headers in *" "*) set x $ac_config_headers; shift; ac_config_headers=$*;; esac cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 # Files that config.status was made for. config_files="$ac_config_files" config_headers="$ac_config_headers" _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 ac_cs_usage="\ \`$as_me' instantiates files and other configuration actions from templates according to the current configuration. Unless the files and actions are specified as TAGs, all are instantiated by default. Usage: $0 [OPTION]... [TAG]... -h, --help print this help, then exit -V, --version print version number and configuration settings, then exit --config print configuration, then exit -q, --quiet, --silent do not print progress messages -d, --debug don't remove temporary files --recheck update $as_me by reconfiguring in the same conditions --file=FILE[:TEMPLATE] instantiate the configuration file FILE --header=FILE[:TEMPLATE] instantiate the configuration header FILE Configuration files: $config_files Configuration headers: $config_headers Report bugs to the package provider." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ Citus config.status 15.0devel configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." ac_pwd='$ac_pwd' srcdir='$srcdir' AWK='$AWK' test -n "\$AWK" || AWK=awk _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # The default lists apply if the user does not specify any file. ac_need_defaults=: while test $# != 0 do case $1 in --*=?*) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` ac_shift=: ;; --*=) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg= ac_shift=: ;; *) ac_option=$1 ac_optarg=$2 ac_shift=shift ;; esac case $ac_option in # Handling of the options. -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) ac_cs_recheck=: ;; --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) $as_echo "$ac_cs_version"; exit ;; --config | --confi | --conf | --con | --co | --c ) $as_echo "$ac_cs_config"; exit ;; --debug | --debu | --deb | --de | --d | -d ) debug=: ;; --file | --fil | --fi | --f ) $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; --header | --heade | --head | --hea ) $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; esac as_fn_append CONFIG_HEADERS " '$ac_optarg'" ac_need_defaults=false;; --he | --h) # Conflict between --help and --header as_fn_error $? "ambiguous option: \`$1' Try \`$0 --help' for more information.";; --help | --hel | -h ) $as_echo "$ac_cs_usage"; exit ;; -q | -quiet | --quiet | --quie | --qui | --qu | --q \ | -silent | --silent | --silen | --sile | --sil | --si | --s) ac_cs_silent=: ;; # This is an error. -*) as_fn_error $? "unrecognized option: \`$1' Try \`$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" ac_need_defaults=false ;; esac shift done ac_configure_extra_args= if $ac_cs_silent; then exec 6>/dev/null ac_configure_extra_args="$ac_configure_extra_args --silent" fi _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 if \$ac_cs_recheck; then set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion shift \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 CONFIG_SHELL='$SHELL' export CONFIG_SHELL exec "\$@" fi _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 exec 5>>config.log { echo sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX ## Running $as_me. ## _ASBOX $as_echo "$ac_log" } >&5 _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Handling of arguments. for ac_config_target in $ac_config_targets do case $ac_config_target in "Makefile.global") CONFIG_FILES="$CONFIG_FILES Makefile.global" ;; "src/include/citus_config.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citus_config.h" ;; "src/include/citus_version.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citus_version.h" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac done # If the user did not use the arguments to specify the items to instantiate, # then the envvar interface is used. Set only those that are not. # We use the long form for the default assignment because of an extremely # bizarre bug on SunOS 4.1.3. if $ac_need_defaults; then test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers fi # Have a temporary directory for convenience. Make it in the build tree # simply because there is no reason against having it here, and in addition, # creating and moving files from /tmp can sometimes cause problems. # Hook for its removal unless debugging. # Note that there is a small window in which the directory will not be cleaned: # after its creation but before its name has been assigned to `$tmp'. $debug || { tmp= ac_tmp= trap 'exit_status=$? : "${ac_tmp:=$tmp}" { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status ' 0 trap 'as_fn_exit 1' 1 2 13 15 } # Create a (secure) tmp directory for tmp files. { tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && test -d "$tmp" } || { tmp=./conf$$-$RANDOM (umask 077 && mkdir "$tmp") } || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. # This happens for instance with `./config.status config.h'. if test -n "$CONFIG_FILES"; then ac_cr=`echo X | tr X '\015'` # On cygwin, bash can eat \r inside `` if the user requested igncr. # But we know of no other shell where ac_cr would be empty at this # point, so we can use a bashism as a fallback. if test "x$ac_cr" = x; then eval ac_cr=\$\'\\r\' fi ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then ac_cs_awk_cr='\\r' else ac_cs_awk_cr=$ac_cr fi echo 'BEGIN {' >"$ac_tmp/subs1.awk" && _ACEOF { echo "cat >conf$$subs.awk <<_ACEOF" && echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && echo "_ACEOF" } >conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi done rm -f conf$$subs.sh cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && _ACEOF sed -n ' h s/^/S["/; s/!.*/"]=/ p g s/^[^!]*!// :repl t repl s/'"$ac_delim"'$// t delim :nl h s/\(.\{148\}\)..*/\1/ t more1 s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ p n b repl :more1 s/["\\]/\\&/g; s/^/"/; s/$/"\\/ p g s/.\{148\}// t nl :delim h s/\(.\{148\}\)..*/\1/ t more2 s/["\\]/\\&/g; s/^/"/; s/$/"/ p b :more2 s/["\\]/\\&/g; s/^/"/; s/$/"\\/ p g s/.\{148\}// t delim ' >$CONFIG_STATUS || ac_write_fail=1 rm -f conf$$subs.awk cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && for (key in S) S_is_set[key] = 1 FS = "" } { line = $ 0 nfields = split(line, field, "@") substed = 0 len = length(field[1]) for (i = 2; i < nfields; i++) { key = field[i] keylen = length(key) if (S_is_set[key]) { value = S[key] line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) len += length(value) + length(field[++i]) substed = 1 } else len += 1 + keylen } print line } _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" else cat fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 _ACEOF # VPATH may cause trouble with some makes, so we remove sole $(srcdir), # ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and # trailing colons and then remove the whole line if VPATH becomes empty # (actually we leave an empty line to preserve line numbers). if test "x$srcdir" = x.; then ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ h s/// s/^/:/ s/[ ]*$/:/ s/:\$(srcdir):/:/g s/:\${srcdir}:/:/g s/:@srcdir@:/:/g s/^:*// s/:*$// x s/\(=[ ]*\).*/\1/ G s/\n// s/^[^=]*=[ ]*$// }' fi cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 fi # test -n "$CONFIG_FILES" # Set up the scripts for CONFIG_HEADERS section. # No need to generate them if there are no CONFIG_HEADERS. # This happens for instance with `./config.status Makefile'. if test -n "$CONFIG_HEADERS"; then cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF # Transform confdefs.h into an awk script `defines.awk', embedded as # here-document in config.status, that substitutes the proper values into # config.h.in to produce config.h. # Create a delimiter string that does not exist in confdefs.h, to ease # handling of long lines. ac_delim='%!_!# ' for ac_last_try in false false :; do ac_tt=`sed -n "/$ac_delim/p" confdefs.h` if test -z "$ac_tt"; then break elif $ac_last_try; then as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi done # For the awk script, D is an array of macro values keyed by name, # likewise P contains macro parameters if any. Preserve backslash # newline sequences. ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]* sed -n ' s/.\{148\}/&'"$ac_delim"'/g t rset :rset s/^[ ]*#[ ]*define[ ][ ]*/ / t def d :def s/\\$// t bsnl s/["\\]/\\&/g s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ D["\1"]=" \3"/p s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p d :bsnl s/["\\]/\\&/g s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ D["\1"]=" \3\\\\\\n"\\/p t cont s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p t cont d :cont n s/.\{148\}/&'"$ac_delim"'/g t clear :clear s/\\$// t bsnlc s/["\\]/\\&/g; s/^/"/; s/$/"/p d :bsnlc s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p b cont ' >$CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 for (key in D) D_is_set[key] = 1 FS = "" } /^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ { line = \$ 0 split(line, arg, " ") if (arg[1] == "#") { defundef = arg[2] mac1 = arg[3] } else { defundef = substr(arg[1], 2) mac1 = arg[2] } split(mac1, mac2, "(") #) macro = mac2[1] prefix = substr(line, 1, index(line, defundef) - 1) if (D_is_set[macro]) { # Preserve the white space surrounding the "#". print prefix "define", macro P[macro] D[macro] next } else { # Replace #undef with comments. This is necessary, for example, # in the case of _POSIX_SOURCE, which is predefined and required # on some systems where configure will not decide to define it. if (defundef == "undef") { print "/*", prefix defundef, macro, "*/" next } } } { print } _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 fi # test -n "$CONFIG_HEADERS" eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS " shift for ac_tag do case $ac_tag in :[FHLC]) ac_mode=$ac_tag; continue;; esac case $ac_mode$ac_tag in :[FHL]*:*);; :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac ac_save_IFS=$IFS IFS=: set x $ac_tag IFS=$ac_save_IFS shift ac_file=$1 shift case $ac_mode in :L) ac_source=$1;; :[FH]) ac_file_inputs= for ac_f do case $ac_f in -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, # because $ac_f cannot contain `:'. test -f "$ac_f" || case $ac_f in [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" done # Let's still pretend it is `configure' which instantiates (i.e., don't # use $as_me), people would be surprised to read: # /* config.h. Generated by config.status. */ configure_input='Generated from '` $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' `' by configure.' if test x"$ac_file" != x-; then configure_input="$ac_file. $configure_input" { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 $as_echo "$as_me: creating $ac_file" >&6;} fi # Neutralize special characters interpreted by sed in replacement strings. case $configure_input in #( *\&* | *\|* | *\\* ) ac_sed_conf_input=`$as_echo "$configure_input" | sed 's/[\\\\&|]/\\\\&/g'`;; #( *) ac_sed_conf_input=$configure_input;; esac case $ac_tag in *:-:* | *:-) cat >"$ac_tmp/stdin" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; esac ;; esac ac_dir=`$as_dirname -- "$ac_file" || $as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ X"$ac_file" : 'X\(//\)[^/]' \| \ X"$ac_file" : 'X\(//\)$' \| \ X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || $as_echo X"$ac_file" | sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/ q } /^X\(\/\/\)[^/].*/{ s//\1/ q } /^X\(\/\/\)$/{ s//\1/ q } /^X\(\/\).*/{ s//\1/ q } s/.*/./; q'` as_dir="$ac_dir"; as_fn_mkdir_p ac_builddir=. case "$ac_dir" in .) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` # A ".." for each directory in $ac_dir_suffix. ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` case $ac_top_builddir_sub in "") ac_top_builddir_sub=. ac_top_build_prefix= ;; *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; esac ;; esac ac_abs_top_builddir=$ac_pwd ac_abs_builddir=$ac_pwd$ac_dir_suffix # for backward compatibility: ac_top_builddir=$ac_top_build_prefix case $srcdir in .) # We are building in place. ac_srcdir=. ac_top_srcdir=$ac_top_builddir_sub ac_abs_top_srcdir=$ac_pwd ;; [\\/]* | ?:[\\/]* ) # Absolute name. ac_srcdir=$srcdir$ac_dir_suffix; ac_top_srcdir=$srcdir ac_abs_top_srcdir=$srcdir ;; *) # Relative name. ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix ac_top_srcdir=$ac_top_build_prefix$srcdir ac_abs_top_srcdir=$ac_pwd/$srcdir ;; esac ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix case $ac_mode in :F) # # CONFIG_FILE # _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # If the template does not know about datarootdir, expand it. # FIXME: This hack should be removed a few years after 2.60. ac_datarootdir_hack=; ac_datarootdir_seen= ac_sed_dataroot=' /datarootdir/ { p q } /@datadir@/p /@docdir@/p /@infodir@/p /@localedir@/p /@mandir@/p' case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in *datarootdir*) ac_datarootdir_seen=yes;; *@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 $as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_datarootdir_hack=' s&@datadir@&$datadir&g s&@docdir@&$docdir&g s&@infodir@&$infodir&g s&@localedir@&$localedir&g s&@mandir@&$mandir&g s&\\\${datarootdir}&$datarootdir&g' ;; esac _ACEOF # Neutralize VPATH when `$srcdir' = `.'. # Shell code in configure.ac might set extrasub. # FIXME: do we really want to maintain this feature? cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_sed_extra="$ac_vpsub $extrasub _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 :t /@[a-zA-Z_][a-zA-Z_0-9]*@/!b s|@configure_input@|$ac_sed_conf_input|;t t s&@top_builddir@&$ac_top_builddir_sub&;t t s&@top_build_prefix@&$ac_top_build_prefix&;t t s&@srcdir@&$ac_srcdir&;t t s&@abs_srcdir@&$ac_abs_srcdir&;t t s&@top_srcdir@&$ac_top_srcdir&;t t s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t s&@builddir@&$ac_builddir&;t t s&@abs_builddir@&$ac_abs_builddir&;t t s&@abs_top_builddir@&$ac_abs_top_builddir&;t t $ac_datarootdir_hack " eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ "$ac_tmp/out"`; test -z "$ac_out"; } && { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' which seems to be undefined. Please make sure it is defined" >&5 $as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' which seems to be undefined. Please make sure it is defined" >&2;} rm -f "$ac_tmp/stdin" case $ac_file in -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; esac \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; :H) # # CONFIG_HEADER # if test x"$ac_file" != x-; then { $as_echo "/* $configure_input */" \ && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" } >"$ac_tmp/config.h" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 $as_echo "$as_me: $ac_file is unchanged" >&6;} else rm -f "$ac_file" mv "$ac_tmp/config.h" "$ac_file" \ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 fi else $as_echo "/* $configure_input */" \ && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ || as_fn_error $? "could not create -" "$LINENO" 5 fi ;; esac done # for ac_tag as_fn_exit 0 _ACEOF ac_clean_files=$ac_clean_files_save test $ac_write_fail = 0 || as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 # configure is writing to config.log, and then calls config.status. # config.status does its own redirection, appending to config.log. # Unfortunately, on DOS this fails, as config.log is still kept open # by configure, so config.status won't be able to write to it; its # output is simply discarded. So we exec the FD to /dev/null, # effectively closing config.log, so it can be properly (re)opened and # appended to by config.status. When coming back to configure, we # need to make the FD available again. if test "$no_create" != yes; then ac_cs_success=: ac_config_status_args= test "$silent" = yes && ac_config_status_args="$ac_config_status_args --quiet" exec 5>/dev/null $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. $ac_cs_success || as_fn_exit 1 fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi ================================================ FILE: configure.ac ================================================ # Citus autoconf input script. # # Converted into an actual configure script by autogen.sh. This # conversion only has to be done when configure.in changes. To avoid # everyone needing autoconf installed, the resulting files are checked # into the SCM. AC_INIT([Citus], [15.0devel]) AC_COPYRIGHT([Copyright (c) Citus Data, Inc.]) # we'll need sed and awk for some of the version commands AC_PROG_SED AC_PROG_AWK # CITUS_NAME definition AC_DEFINE_UNQUOTED(CITUS_NAME, "$PACKAGE_NAME", [Citus full name as a string]) case $PACKAGE_NAME in 'Citus Enterprise') citus_edition=enterprise ;; Citus) citus_edition=community ;; *) AC_MSG_ERROR([Unrecognized package name.]) ;; esac # CITUS_EDITION definition AC_DEFINE_UNQUOTED(CITUS_EDITION, "$citus_edition", [Citus edition as a string]) # CITUS_MAJORVERSION definition [CITUS_MAJORVERSION=`expr "$PACKAGE_VERSION" : '\([0-9][0-9]*\.[0-9][0-9]*\)'`] AC_DEFINE_UNQUOTED(CITUS_MAJORVERSION, "$CITUS_MAJORVERSION", [Citus major version as a string]) # CITUS_VERSION definition PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version], [CITUS_VERSION="$PACKAGE_VERSION$withval"], [CITUS_VERSION="$PACKAGE_VERSION"]) AC_DEFINE_UNQUOTED(CITUS_VERSION, "$CITUS_VERSION", [Citus version as a string]) # CITUS_VERSION_NUM definition # awk -F is a regex on some platforms, and not on others, so make "." a tab [CITUS_VERSION_NUM="`echo "$PACKAGE_VERSION" | sed 's/[A-Za-z].*$//' | tr '.' ' ' | $AWK '{printf "%d%02d%02d", $1, $2, (NF >= 3) ? $3 : 0}'`"] AC_DEFINE_UNQUOTED(CITUS_VERSION_NUM, $CITUS_VERSION_NUM, [Citus version as a number]) # CITUS_EXTENSIONVERSION definition [CITUS_EXTENSIONVERSION="`grep '^default_version' $srcdir/src/backend/distributed/citus.control | cut -d\' -f2`"] AC_DEFINE_UNQUOTED([CITUS_EXTENSIONVERSION], "$CITUS_EXTENSIONVERSION", [Extension version expected by this Citus build]) # Re-check for flex. That allows to compile citus against a postgres # which was built without flex available (possible because generated # files are included) AC_PATH_PROG([FLEX], [flex]) # Locate pg_config binary AC_ARG_VAR([PG_CONFIG], [Location to find pg_config for target PostgreSQL instalation (default PATH)]) AC_ARG_VAR([PATH], [PATH for target PostgreSQL install pg_config]) if test -z "$PG_CONFIG"; then AC_PATH_PROG(PG_CONFIG, pg_config) fi if test -z "$PG_CONFIG"; then AC_MSG_ERROR([Could not find pg_config. Set PG_CONFIG or PATH.]) fi # check we're building against a supported version of PostgreSQL citusac_pg_config_version=$($PG_CONFIG --version 2>/dev/null) version_num=$(echo "$citusac_pg_config_version"| $SED -e 's/^PostgreSQL \([[0-9]]*\)\(\.[[0-9]]*\)\{0,1\}\(.*\)$/\1\2/') # if PostgreSQL version starts with two digits, the major version is those digits version_num=$(echo "$version_num"| $SED -e 's/^\([[0-9]]\{2\}\)\(.*\)$/\1/') if test -z "$version_num"; then AC_MSG_ERROR([Could not detect PostgreSQL version from pg_config.]) fi PGAC_ARG_BOOL(with, pg-version-check, yes, [do not check postgres version during configure]) AC_SUBST(with_pg_version_check) if test "$with_pg_version_check" = no; then AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)]) elif test "$version_num" != '16' -a "$version_num" != '17' -a "$version_num" != '18'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) fi; # Check whether we're building inside the source tree, if not, prepare # the build directory. if test "$srcdir" -ef '.' ; then vpath_build=no else vpath_build=yes _AS_ECHO_N([preparing build tree... ]) citusac_abs_top_srcdir=`cd "$srcdir" && pwd` $SHELL "$citusac_abs_top_srcdir/prep_buildtree" "$citusac_abs_top_srcdir" "." \ || AC_MSG_ERROR(failed) AC_MSG_RESULT(done) fi AC_SUBST(vpath_build) # Allow to overwrite the C compiler, default to the one postgres was # compiled with. We don't want autoconf's default CFLAGS though, so save # those. SAVE_CFLAGS="$CFLAGS" AC_PROG_CC([$($PG_CONFIG --cc)]) CFLAGS="$SAVE_CFLAGS" host_guess=`${SHELL} $srcdir/config/config.guess` # Create compiler version string if test x"$GCC" = x"yes" ; then cc_string=`${CC} --version | sed q` case $cc_string in [[A-Za-z]]*) ;; *) cc_string="GCC $cc_string";; esac elif test x"$SUN_STUDIO_CC" = x"yes" ; then cc_string=`${CC} -V 2>&1 | sed q` else cc_string=$CC fi AC_CHECK_SIZEOF([void *]) AC_DEFINE_UNQUOTED(CITUS_VERSION_STR, ["$PACKAGE_NAME $CITUS_VERSION on $host_guess, compiled by $cc_string, `expr $ac_cv_sizeof_void_p \* 8`-bit"], [A string containing the version number, platform, and C compiler]) # Locate source and build directory of the postgres we're building # against. Can't rely on either still being present, but e.g. optional # test infrastructure can rely on it. POSTGRES_SRCDIR=$(grep ^abs_top_srcdir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) POSTGRES_BUILDDIR=$(grep ^abs_top_builddir $(dirname $($PG_CONFIG --pgxs))/../Makefile.global|cut -d ' ' -f3-) # check for a number of CFLAGS that make development easier # CITUSAC_PROG_CC_CFLAGS_OPT # ----------------------- # Given a string, check if the compiler supports the string as a # command-line option. If it does, add the string to CFLAGS. AC_DEFUN([CITUSAC_PROG_CC_CFLAGS_OPT], [define([Ac_cachevar], [AS_TR_SH([citusac_cv_prog_cc_cflags_$1])])dnl AC_CACHE_CHECK([whether $CC supports $1], [Ac_cachevar], [citusac_save_CFLAGS=$CFLAGS flag=$1 case $flag in -Wno*) flag=-W$(echo $flag | cut -c 6-) esac CFLAGS="$citusac_save_CFLAGS $flag" ac_save_c_werror_flag=$ac_c_werror_flag ac_c_werror_flag=yes _AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], [Ac_cachevar=yes], [Ac_cachevar=no]) ac_c_werror_flag=$ac_save_c_werror_flag CFLAGS="$citusac_save_CFLAGS"]) if test x"$Ac_cachevar" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS $1" fi undefine([Ac_cachevar])dnl ])# CITUSAC_PROG_CC_CFLAGS_OPT CITUSAC_PROG_CC_CFLAGS_OPT([-std=gnu99]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wall]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wextra]) # disarm options included in the above, which are too noisy for now CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-unused-parameter]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-sign-compare]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-missing-field-initializers]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-clobbered]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-gnu-variable-sized-type-not-at-end]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-declaration-after-statement]) # And add a few extra warnings CITUSAC_PROG_CC_CFLAGS_OPT([-Wendif-labels]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-format-attribute]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-declarations]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-prototypes]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wshadow]) CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=vla]) # visual studio does not support these CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=implicit-int]) CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=implicit-function-declaration]) CITUSAC_PROG_CC_CFLAGS_OPT([-Werror=return-type]) # Security flags # Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide # We do not enforce the following flag because it is only available on GCC>=8 CITUSAC_PROG_CC_CFLAGS_OPT([-fstack-clash-protection]) # # --enable-coverage enables generation of code coverage metrics with gcov # AC_ARG_ENABLE([coverage], AS_HELP_STRING([--enable-coverage], [build with coverage testing instrumentation])) if test "$enable_coverage" = yes; then CITUS_CFLAGS="$CITUS_CFLAGS -O0 -g --coverage" CITUS_CPPFLAGS="$CITUS_CPPFLAGS -DNDEBUG" CITUS_LDFLAGS="$CITUS_LDFLAGS --coverage" fi # # LZ4 # PGAC_ARG_BOOL(with, lz4, yes, [do not use lz4], [AC_DEFINE([HAVE_CITUS_LIBLZ4], 1, [Define to 1 to build with lz4 support. (--with-lz4)])]) AC_SUBST(with_lz4) if test "$with_lz4" = yes; then AC_CHECK_LIB(lz4, LZ4_compress_default, [], [AC_MSG_ERROR([lz4 library not found If you have lz4 installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-lz4 to disable lz4 support.])]) AC_CHECK_HEADER(lz4.h, [], [AC_MSG_ERROR([lz4 header not found If you have lz4 already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-lz4 to disable lz4 support.])]) fi # # ZSTD # PGAC_ARG_BOOL(with, zstd, yes, [do not use zstd]) AC_SUBST(with_zstd) if test "$with_zstd" = yes; then AC_CHECK_LIB(zstd, ZSTD_decompress, [], [AC_MSG_ERROR([zstd library not found If you have zstd installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-zstd to disable zstd support.])]) AC_CHECK_HEADER(zstd.h, [], [AC_MSG_ERROR([zstd header not found If you have zstd already installed, see config.log for details on the failure. It is possible the compiler isn't looking in the proper directory. Use --without-zstd to disable zstd support.])]) fi PGAC_ARG_BOOL(with, security-flags, no, [use security flags]) AC_SUBST(with_security_flags) if test "$with_security_flags" = yes; then # Flags taken from: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10203#guide # We always want to have some compiler flags for security concerns. SECURITY_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2 -z noexecstack -fpic -shared -Wl,-z,relro -Wl,-z,now -Wformat -Wformat-security -Werror=format-security" CITUS_CFLAGS="$CITUS_CFLAGS $SECURITY_CFLAGS" AC_MSG_NOTICE([Blindly added security flags for linker: $SECURITY_CFLAGS]) # We always want to have some clang flags for security concerns. # This doesn't include "-Wl,-z,relro -Wl,-z,now" on purpuse, because bitcode is not linked. # This doesn't include -fsanitize=cfi because it breaks builds on many distros including # Debian/Buster, Debian/Stretch, Ubuntu/Bionic, Ubuntu/Xenial and EL7. SECURITY_BITCODE_CFLAGS="-fsanitize=safe-stack -fstack-protector-strong -flto -fPIC -Wformat -Wformat-security -Werror=format-security" CITUS_BITCODE_CFLAGS="$CITUS_BITCODE_CFLAGS $SECURITY_BITCODE_CFLAGS" AC_MSG_NOTICE([Blindly added security flags for llvm: $SECURITY_BITCODE_CFLAGS]) AC_MSG_WARN([If you run into issues during linking or bitcode compilation, you can use --without-security-flags.]) fi # Check if git is installed, when installed the gitref of the checkout will be baked in the application AC_PATH_PROG(GIT_BIN, git) AC_CHECK_FILE(.git,[HAS_DOTGIT=yes], [HAS_DOTGIT=]) AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") AC_SUBST(CITUS_BITCODE_CFLAGS, "$CITUS_BITCODE_CFLAGS") AC_SUBST(CITUS_CPPFLAGS, "$CITUS_CPPFLAGS") AC_SUBST(CITUS_LDFLAGS, "$LIBS $CITUS_LDFLAGS") AC_SUBST(POSTGRES_SRCDIR, "$POSTGRES_SRCDIR") AC_SUBST(POSTGRES_BUILDDIR, "$POSTGRES_BUILDDIR") AC_SUBST(HAS_DOTGIT, "$HAS_DOTGIT") AC_CONFIG_FILES([Makefile.global]) AC_CONFIG_HEADERS([src/include/citus_config.h] [src/include/citus_version.h]) AH_TOP([ /* * citus_config.h.in is generated by autoconf/autoheader and * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! */ ]) AC_OUTPUT ================================================ FILE: prep_buildtree ================================================ #! /bin/sh # # Citus copy of PostgreSQL's config/prep_buildtree # # This script prepares a Citus build tree for an out-of-tree/VPATH # build. It is intended to be run by the configure script. me=`basename $0` help="\ Usage: $me sourcetree [buildtree]" if test -z "$1"; then echo "$help" 1>&2 exit 1 elif test x"$1" = x"--help"; then echo "$help" exit 0 fi unset CDPATH sourcetree=`cd $1 && pwd` buildtree=`cd ${2:-'.'} && pwd` # We must not auto-create the subdirectories holding built documentation. # If we did, it would interfere with installation of prebuilt docs from # the source tree, if a VPATH build is done from a distribution tarball. # See bug #5595. for item in `find "$sourcetree" -type d \( \( -name CVS -prune \) -o \( -name .git -prune \) -o -print \) | grep -v "$sourcetree/doc/src/sgml/\+"`; do subdir=`expr "$item" : "$sourcetree\(.*\)"` if test ! -d "$buildtree/$subdir"; then mkdir -p "$buildtree/$subdir" || exit 1 fi done for item in `find "$sourcetree" -not -path '*/.git/hg/*' \( -name Makefile -print -o -name GNUmakefile -print \)`; do filename=`expr "$item" : "$sourcetree\(.*\)"` if test ! -f "${item}.in"; then if cmp "$item" "$buildtree/$filename" >/dev/null 2>&1; then : ; else ln -fs "$item" "$buildtree/$filename" || exit 1 fi fi done exit 0 ================================================ FILE: pyproject.toml ================================================ [tool.isort] profile = 'black' [tool.black] include = '(src/test/regress/bin/diff-filter|\.pyi?|\.ipynb)$' [tool.pytest.ini_options] addopts = [ "--import-mode=importlib", "--showlocals", "--tb=short", ] pythonpath = 'src/test/regress/citus_tests' asyncio_mode = 'auto' # Make test discovery quicker from the root dir of the repo testpaths = ['src/test/regress/citus_tests/test'] # Make test discovery quicker from other directories than root directory norecursedirs = [ '*.egg', '.*', 'build', 'venv', 'ci', 'vendor', 'backend', 'bin', 'include', 'tmp_*', 'results', 'expected', 'sql', 'spec', 'data', '__pycache__', ] # Don't find files with test at the end such as run_test.py python_files = ['test_*.py'] ================================================ FILE: src/backend/.gitignore ================================================ ================================================ FILE: src/backend/columnar/.gitattributes ================================================ * whitespace=space-before-tab,trailing-space *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4 *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent *.patch -whitespace *.pl whitespace=space-before-tab,trailing-space,tabwidth=4 *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent # Avoid confusing ASCII underlines with leftover merge conflict markers README conflict-marker-size=32 README.* conflict-marker-size=32 # Certain data files that contain special whitespace, and other special cases *.data -whitespace # Test output files that contain extra whitespace *.out -whitespace # These files are maintained or generated elsewhere. We take them as is. configure -whitespace # all C files (implementation and header) use our style... *.[ch] citus-style ================================================ FILE: src/backend/columnar/.gitignore ================================================ # The directory used to store columnar sql files after pre-processing them # with 'cpp' in build-time, see src/backend/columnar/Makefile. /build/ ================================================ FILE: src/backend/columnar/Makefile ================================================ citus_subdir = src/backend/columnar citus_top_builddir = ../../.. safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib SUBDIRS = . safeclib SUBDIRS += ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS)) OBJS += \ $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c)))) MODULE_big = citus_columnar EXTENSION = citus_columnar template_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql)) template_downgrade_sql_files = $(patsubst $(citus_abs_srcdir)/sql/downgrades/%,%,$(wildcard $(citus_abs_srcdir)/sql/downgrades/*.sql)) generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_files)) generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(template_downgrade_sql_files)) DATA_built = $(generated_sql_files) PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include include $(citus_top_builddir)/Makefile.global SQL_DEPDIR=.deps/sql SQL_BUILDDIR=build/sql $(generated_sql_files): $(citus_abs_srcdir)/build/%: % @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) @# -MF is used to store dependency files(.Po) in another directory for separation @# -MT is used to change the target of the rule emitted by dependency generation. @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. @# -undef is used to not predefine any system-specific or GCC-specific macros. @# `man cpp` for further information cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ $(generated_downgrade_sql_files): $(citus_abs_srcdir)/build/sql/%: sql/downgrades/% @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) @# -MF is used to store dependency files(.Po) in another directory for separation @# -MT is used to change the target of the rule emitted by dependency generation. @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. @# -undef is used to not predefine any system-specific or GCC-specific macros. @# `man cpp` for further information cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ .PHONY: install install-downgrades install-all cleanup-before-install: rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus_columnar.control rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/columnar--* rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus_columnar--* install: cleanup-before-install # install and install-downgrades should be run sequentially install-all: install $(MAKE) install-downgrades install-downgrades: $(generated_downgrade_sql_files) $(INSTALL_DATA) $(generated_downgrade_sql_files) '$(DESTDIR)$(datadir)/$(datamoduledir)/' ================================================ FILE: src/backend/columnar/README.md ================================================ # Introduction Citus Columnar offers a per-table option for columnar storage to reduce IO requirements though compression and projection pushdown. # Design Trade-Offs Existing PostgreSQL row tables work well for OLTP: * Support `UPDATE`/`DELETE` efficiently * Efficient single-tuple lookups The Citus Columnar tables work best for analytic or DW workloads: * Compression * Doesn't read unnecessary columns * Efficient `VACUUM` # Next generation of cstore_fdw Citus Columnar is the next generation of [cstore_fdw](https://github.com/citusdata/cstore_fdw/). Benefits of Citus Columnar over cstore_fdw: * Citus Columnar is based on the [Table Access Method API](https://www.postgresql.org/docs/current/tableam.html), which allows it to behave exactly like an ordinary heap (row) table for most operations. * Supports Write-Ahead Log (WAL). * Supports ``ROLLBACK``. * Supports physical replication. * Supports recovery, including Point-In-Time Restore (PITR). * Supports ``pg_dump`` and ``pg_upgrade`` without the need for special options or extra steps. * Better user experience; simple ``USING``clause. * Supports more features that work on ordinary heap (row) tables. # Limitations * Append-only (no ``UPDATE``/``DELETE`` support) * No space reclamation (e.g. rolled-back transactions may still consume disk space) * No bitmap index scans * No tidscans * No sample scans * No TOAST support (large values supported inline) * No support for [``ON CONFLICT``](https://www.postgresql.org/docs/12/sql-insert.html#SQL-ON-CONFLICT) statements (except ``DO NOTHING`` actions with no target specified). * No support for tuple locks (``SELECT ... FOR SHARE``, ``SELECT ... FOR UPDATE``) * No support for serializable isolation level * Support for PostgreSQL server versions 12+ only * No support for foreign keys * No support for logical decoding * No support for intra-node parallel scans * No support for ``AFTER ... FOR EACH ROW`` triggers * No `UNLOGGED` columnar tables Future iterations will incrementally lift the limitations listed above. # User Experience Create a Columnar table by specifying ``USING columnar`` when creating the table. ```sql CREATE TABLE my_columnar_table ( id INT, i1 INT, i2 INT8, n NUMERIC, t TEXT ) USING columnar; ``` Insert data into the table and read from it like normal (subject to the limitations listed above). To see internal statistics about the table, use ``VACUUM VERBOSE``. Note that ``VACUUM`` (without ``FULL``) is much faster on a columnar table, because it scans only the metadata, and not the actual data. ## Options Set options using: ```sql ALTER TABLE my_columnar_table SET (columnar.compression = none, columnar.stripe_row_limit = 10000); ``` The following options are available: * **columnar.compression**: `[none|pglz|zstd|lz4|lz4hc]` - set the compression type for _newly-inserted_ data. Existing data will not be recompressed/decompressed. The default value is `zstd` (if support has been compiled in). * **columnar.compression_level**: ```` - Sets compression level. Valid settings are from 1 through 19. If the compression method does not support the level chosen, the closest level will be selected instead. * **columnar.stripe_row_limit**: ```` - the maximum number of rows per stripe for _newly-inserted_ data. Existing stripes of data will not be changed and may have more rows than this maximum value. The default value is `150000`. * **columnar.chunk_group_row_limit**: ```` - the maximum number of rows per chunk for _newly-inserted_ data. Existing chunks of data will not be changed and may have more rows than this maximum value. The default value is `10000`. View options for all tables with: ```sql SELECT * FROM columnar.options; ``` You can also adjust options with a `SET` command of one of the following GUCs: * `columnar.compression` * `columnar.compression_level` * `columnar.stripe_row_limit` * `columnar.chunk_group_row_limit` GUCs only affect newly-created *tables*, not any newly-created *stripes* on an existing table. ## Partitioning Columnar tables can be used as partitions; and a partitioned table may be made up of any combination of row and columnar partitions. ```sql CREATE TABLE parent(ts timestamptz, i int, n numeric, s text) PARTITION BY RANGE (ts); -- columnar partition CREATE TABLE p0 PARTITION OF parent FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') USING COLUMNAR; -- columnar partition CREATE TABLE p1 PARTITION OF parent FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') USING COLUMNAR; -- row partition CREATE TABLE p2 PARTITION OF parent FOR VALUES FROM ('2020-03-01') TO ('2020-04-01'); INSERT INTO parent VALUES ('2020-01-15', 10, 100, 'one thousand'); -- columnar INSERT INTO parent VALUES ('2020-02-15', 20, 200, 'two thousand'); -- columnar INSERT INTO parent VALUES ('2020-03-15', 30, 300, 'three thousand'); -- row ``` When performing operations on a partitioned table with a mix of row and columnar partitions, take note of the following behaviors for operations that are supported on row tables but not columnar (e.g. ``UPDATE``, ``DELETE``, tuple locks, etc.): * If the operation is targeted at a specific row partition (e.g. ``UPDATE p2 SET i = i + 1``), it will succeed; if targeted at a specified columnar partition (e.g. ``UPDATE p1 SET i = i + 1``), it will fail. * If the operation is targeted at the partitioned table and has a ``WHERE`` clause that excludes all columnar partitions (e.g. ``UPDATE parent SET i = i + 1 WHERE ts = '2020-03-15'``), it will succeed. * If the operation is targeted at the partitioned table, but does not exclude all columnar partitions, it will fail; even if the actual data to be updated only affects row tables (e.g. ``UPDATE parent SET i = i + 1 WHERE n = 300``). Note that Citus Columnar supports `btree` and `hash `indexes (and the constraints requiring them) but does not support `gist`, `gin`, `spgist` and `brin` indexes. For this reason, if some partitions are columnar and if the index is not supported by Citus Columnar, then it's impossible to create indexes on the partitioned (parent) table directly. In that case, you need to create the index on the individual row partitions. Similarly for the constraints that require indexes, e.g.: ```sql CREATE INDEX p2_ts_idx ON p2 (ts); CREATE UNIQUE INDEX p2_i_unique ON p2 (i); ALTER TABLE p2 ADD UNIQUE (n); ``` ## Converting Between Row and Columnar Note: ensure that you understand any advanced features that may be used with the table before converting it (e.g. row-level security, storage options, constraints, inheritance, etc.), and ensure that they are reproduced in the new table or partition appropriately. ``LIKE``, used below, is a shorthand that works only in simple cases. ```sql CREATE TABLE my_table(i INT8 DEFAULT '7'); INSERT INTO my_table VALUES(1); -- convert to columnar SELECT alter_table_set_access_method('my_table', 'columnar'); -- back to row SELECT alter_table_set_access_method('my_table', 'heap'); ``` # Performance Microbenchmark *Important*: This microbenchmark is not intended to represent any real workload. Compression ratios, and therefore performance, will depend heavily on the specific workload. This is only for the purpose of illustrating a "columnar friendly" contrived workload that showcases the benefits of columnar. ## Schema ```sql CREATE TABLE perf_row( id INT8, ts TIMESTAMPTZ, customer_id INT8, vendor_id INT8, name TEXT, description TEXT, value NUMERIC, quantity INT4 ); CREATE TABLE perf_columnar(LIKE perf_row) USING COLUMNAR; ``` ## Data ```sql CREATE OR REPLACE FUNCTION random_words(n INT4) RETURNS TEXT LANGUAGE sql AS $$ WITH words(w) AS ( SELECT ARRAY['zero','one','two','three','four','five','six','seven','eight','nine','ten'] ), random (word) AS ( SELECT w[(random()*array_length(w, 1))::int] FROM generate_series(1, $1) AS i, words ) SELECT string_agg(word, ' ') FROM random; $$; ``` ```sql INSERT INTO perf_row SELECT g, -- id '2020-01-01'::timestamptz + ('1 minute'::interval * g), -- ts (random() * 1000000)::INT4, -- customer_id (random() * 100)::INT4, -- vendor_id random_words(7), -- name random_words(100), -- description (random() * 100000)::INT4/100.0, -- value (random() * 100)::INT4 -- quantity FROM generate_series(1,75000000) g; INSERT INTO perf_columnar SELECT * FROM perf_row; ``` ## Compression Ratio ``` => SELECT pg_total_relation_size('perf_row')::numeric/pg_total_relation_size('perf_columnar') AS compression_ratio; compression_ratio -------------------- 5.3958044063457513 (1 row) ``` The overall compression ratio of columnar table, versus the same data stored with row storage, is **5.4X**. ``` => VACUUM VERBOSE perf_columnar; INFO: statistics for "perf_columnar": storage id: 10000000000 total file size: 8761368576, total data size: 8734266196 compression rate: 5.01x total row count: 75000000, stripe count: 500, average rows per stripe: 150000 chunk count: 60000, containing data for dropped columns: 0, zstd compressed: 60000 ``` ``VACUUM VERBOSE`` reports a smaller compression ratio, because it only averages the compression ratio of the individual chunks, and does not account for the metadata savings of the columnar format. ## System * Azure VM: Standard D2s v3 (2 vcpus, 8 GiB memory) * Linux (ubuntu 18.04) * Data Drive: Standard HDD (512GB, 500 IOPS Max, 60 MB/s Max) * PostgreSQL 13 (``--with-llvm``, ``--with-python``) * ``shared_buffers = 128MB`` * ``max_parallel_workers_per_gather = 0`` * ``jit = on`` Note: because this was run on a system with enough physical memory to hold a substantial fraction of the table, the IO benefits of columnar won't be entirely realized by the query runtime unless the data size is substantially increased. ## Query ```sql -- OFFSET 1000 so that no rows are returned, and we collect only timings SELECT vendor_id, SUM(quantity) FROM perf_row GROUP BY vendor_id OFFSET 1000; SELECT vendor_id, SUM(quantity) FROM perf_row GROUP BY vendor_id OFFSET 1000; SELECT vendor_id, SUM(quantity) FROM perf_row GROUP BY vendor_id OFFSET 1000; SELECT vendor_id, SUM(quantity) FROM perf_columnar GROUP BY vendor_id OFFSET 1000; SELECT vendor_id, SUM(quantity) FROM perf_columnar GROUP BY vendor_id OFFSET 1000; SELECT vendor_id, SUM(quantity) FROM perf_columnar GROUP BY vendor_id OFFSET 1000; ``` Timing (median of three runs): * row: 436s * columnar: 16s * speedup: **27X** ================================================ FILE: src/backend/columnar/citus_columnar.control ================================================ # Columnar extension comment = 'Citus Columnar extension' default_version = '15.0-1' module_pathname = '$libdir/citus_columnar' relocatable = false schema = pg_catalog ================================================ FILE: src/backend/columnar/columnar.c ================================================ /*------------------------------------------------------------------------- * * columnar.c * * This file contains... * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "miscadmin.h" #include "utils/guc.h" #include "utils/rel.h" #include "citus_version.h" #include "columnar/columnar.h" #include "columnar/columnar_tableam.h" /* Default values for option parameters */ #define DEFAULT_STRIPE_ROW_COUNT 150000 #define DEFAULT_CHUNK_ROW_COUNT 10000 #if HAVE_LIBZSTD #define DEFAULT_COMPRESSION_TYPE COMPRESSION_ZSTD #elif HAVE_CITUS_LIBLZ4 #define DEFAULT_COMPRESSION_TYPE COMPRESSION_LZ4 #else #define DEFAULT_COMPRESSION_TYPE COMPRESSION_PG_LZ #endif int columnar_compression = DEFAULT_COMPRESSION_TYPE; int columnar_stripe_row_limit = DEFAULT_STRIPE_ROW_COUNT; int columnar_chunk_group_row_limit = DEFAULT_CHUNK_ROW_COUNT; int columnar_compression_level = 3; static const struct config_enum_entry columnar_compression_options[] = { { "none", COMPRESSION_NONE, false }, { "pglz", COMPRESSION_PG_LZ, false }, #if HAVE_CITUS_LIBLZ4 { "lz4", COMPRESSION_LZ4, false }, #endif #if HAVE_LIBZSTD { "zstd", COMPRESSION_ZSTD, false }, #endif { NULL, 0, false } }; void columnar_init(void) { columnar_init_gucs(); columnar_tableam_init(); } void columnar_init_gucs() { DefineCustomEnumVariable("columnar.compression", "Compression type for columnar.", NULL, &columnar_compression, DEFAULT_COMPRESSION_TYPE, columnar_compression_options, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("columnar.compression_level", "Compression level to be used with zstd.", NULL, &columnar_compression_level, 3, COMPRESSION_LEVEL_MIN, COMPRESSION_LEVEL_MAX, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("columnar.stripe_row_limit", "Maximum number of tuples per stripe.", NULL, &columnar_stripe_row_limit, DEFAULT_STRIPE_ROW_COUNT, STRIPE_ROW_COUNT_MINIMUM, STRIPE_ROW_COUNT_MAXIMUM, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable("columnar.chunk_group_row_limit", "Maximum number of rows per chunk.", NULL, &columnar_chunk_group_row_limit, DEFAULT_CHUNK_ROW_COUNT, CHUNK_ROW_COUNT_MINIMUM, CHUNK_ROW_COUNT_MAXIMUM, PGC_USERSET, 0, NULL, NULL, NULL); } /* * ParseCompressionType converts a string to a compression type. * For compression algorithms that are invalid or not compiled, it * returns COMPRESSION_TYPE_INVALID. */ CompressionType ParseCompressionType(const char *compressionTypeString) { Assert(compressionTypeString != NULL); for (int compressionIndex = 0; columnar_compression_options[compressionIndex].name != NULL; compressionIndex++) { const char *compressionName = columnar_compression_options[compressionIndex].name; if (strncmp(compressionTypeString, compressionName, NAMEDATALEN) == 0) { return columnar_compression_options[compressionIndex].val; } } return COMPRESSION_TYPE_INVALID; } /* * CompressionTypeStr returns string representation of a compression type. * For compression algorithms that are invalid or not compiled, it * returns NULL. */ const char * CompressionTypeStr(CompressionType requestedType) { for (int compressionIndex = 0; columnar_compression_options[compressionIndex].name != NULL; compressionIndex++) { CompressionType compressionType = columnar_compression_options[compressionIndex].val; if (compressionType == requestedType) { return columnar_compression_options[compressionIndex].name; } } return NULL; } ================================================ FILE: src/backend/columnar/columnar_compression.c ================================================ /*------------------------------------------------------------------------- * * columnar_compression.c * * This file contains compression/decompression functions definitions * used for columnar. * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/pg_lzcompress.h" #include "lib/stringinfo.h" #include "citus_version.h" #include "pg_version_constants.h" #include "columnar/columnar_compression.h" #if HAVE_CITUS_LIBLZ4 #include #endif #include "varatt.h" #if HAVE_LIBZSTD #include #endif /* * The information at the start of the compressed data. This decription is taken * from pg_lzcompress in pre-9.5 version of PostgreSQL. */ typedef struct ColumnarCompressHeader { int32 vl_len_; /* varlena header (do not touch directly!) */ int32 rawsize; } ColumnarCompressHeader; /* * Utilities for manipulation of header information for compressed data */ #define COLUMNAR_COMPRESS_HDRSZ ((int32) sizeof(ColumnarCompressHeader)) #define COLUMNAR_COMPRESS_RAWSIZE(ptr) (((ColumnarCompressHeader *) (ptr))->rawsize) #define COLUMNAR_COMPRESS_RAWDATA(ptr) (((char *) (ptr)) + COLUMNAR_COMPRESS_HDRSZ) #define COLUMNAR_COMPRESS_SET_RAWSIZE(ptr, \ len) (((ColumnarCompressHeader *) (ptr))->rawsize = \ (len)) /* * CompressBuffer compresses the given buffer with the given compression type * outputBuffer enlarged to contain compressed data. The function returns true * if compression is done, returns false if compression is not done. * outputBuffer is valid only if the function returns true. */ bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, CompressionType compressionType, int compressionLevel) { switch (compressionType) { #if HAVE_CITUS_LIBLZ4 case COMPRESSION_LZ4: { int maximumLength = LZ4_compressBound(inputBuffer->len); resetStringInfo(outputBuffer); enlargeStringInfo(outputBuffer, maximumLength); int compressedSize = LZ4_compress_default(inputBuffer->data, outputBuffer->data, inputBuffer->len, maximumLength); if (compressedSize <= 0) { elog(DEBUG1, "failure in LZ4_compress_default, input size=%d, output size=%d", inputBuffer->len, maximumLength); return false; } elog(DEBUG1, "compressed %d bytes to %d bytes", inputBuffer->len, compressedSize); outputBuffer->len = compressedSize; return true; } #endif #if HAVE_LIBZSTD case COMPRESSION_ZSTD: { int maximumLength = ZSTD_compressBound(inputBuffer->len); resetStringInfo(outputBuffer); enlargeStringInfo(outputBuffer, maximumLength); size_t compressedSize = ZSTD_compress(outputBuffer->data, outputBuffer->maxlen, inputBuffer->data, inputBuffer->len, compressionLevel); if (ZSTD_isError(compressedSize)) { ereport(WARNING, (errmsg("zstd compression failed"), (errdetail("%s", ZSTD_getErrorName(compressedSize))))); return false; } outputBuffer->len = compressedSize; return true; } #endif case COMPRESSION_PG_LZ: { uint64 maximumLength = PGLZ_MAX_OUTPUT(inputBuffer->len) + COLUMNAR_COMPRESS_HDRSZ; bool compressionResult = false; resetStringInfo(outputBuffer); enlargeStringInfo(outputBuffer, maximumLength); int32 compressedByteCount = pglz_compress((const char *) inputBuffer->data, inputBuffer->len, COLUMNAR_COMPRESS_RAWDATA( outputBuffer->data), PGLZ_strategy_always); if (compressedByteCount >= 0) { COLUMNAR_COMPRESS_SET_RAWSIZE(outputBuffer->data, inputBuffer->len); SET_VARSIZE_COMPRESSED(outputBuffer->data, compressedByteCount + COLUMNAR_COMPRESS_HDRSZ); compressionResult = true; } if (compressionResult) { outputBuffer->len = VARSIZE(outputBuffer->data); } return compressionResult; } default: { return false; } } } /* * DecompressBuffer decompresses the given buffer with the given compression * type. This function returns the buffer as-is when no compression is applied. */ StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType, uint64 decompressedSize) { switch (compressionType) { case COMPRESSION_NONE: { return buffer; } #if HAVE_CITUS_LIBLZ4 case COMPRESSION_LZ4: { StringInfo decompressedBuffer = makeStringInfo(); enlargeStringInfo(decompressedBuffer, decompressedSize); int lz4DecompressSize = LZ4_decompress_safe(buffer->data, decompressedBuffer->data, buffer->len, decompressedSize); if (lz4DecompressSize != decompressedSize) { ereport(ERROR, (errmsg("cannot decompress the buffer"), errdetail("Expected %lu bytes, but received %d bytes", decompressedSize, lz4DecompressSize))); } decompressedBuffer->len = decompressedSize; return decompressedBuffer; } #endif #if HAVE_LIBZSTD case COMPRESSION_ZSTD: { StringInfo decompressedBuffer = makeStringInfo(); enlargeStringInfo(decompressedBuffer, decompressedSize); size_t zstdDecompressSize = ZSTD_decompress(decompressedBuffer->data, decompressedSize, buffer->data, buffer->len); if (ZSTD_isError(zstdDecompressSize)) { ereport(ERROR, (errmsg("zstd decompression failed"), (errdetail("%s", ZSTD_getErrorName( zstdDecompressSize))))); } if (zstdDecompressSize != decompressedSize) { ereport(ERROR, (errmsg("unexpected decompressed size"), errdetail("Expected %ld, received %ld", decompressedSize, zstdDecompressSize))); } decompressedBuffer->len = decompressedSize; return decompressedBuffer; } #endif case COMPRESSION_PG_LZ: { uint32 compressedDataSize = VARSIZE(buffer->data) - COLUMNAR_COMPRESS_HDRSZ; uint32 decompressedDataSize = COLUMNAR_COMPRESS_RAWSIZE(buffer->data); if (compressedDataSize + COLUMNAR_COMPRESS_HDRSZ != buffer->len) { ereport(ERROR, (errmsg("cannot decompress the buffer"), errdetail("Expected %u bytes, but received %u bytes", compressedDataSize, buffer->len))); } char *decompressedData = palloc0(decompressedDataSize); int32 decompressedByteCount = pglz_decompress(COLUMNAR_COMPRESS_RAWDATA( buffer->data), compressedDataSize, decompressedData, decompressedDataSize, true); if (decompressedByteCount < 0) { ereport(ERROR, (errmsg("cannot decompress the buffer"), errdetail("compressed data is corrupted"))); } StringInfo decompressedBuffer = palloc0(sizeof(StringInfoData)); decompressedBuffer->data = decompressedData; decompressedBuffer->len = decompressedDataSize; decompressedBuffer->maxlen = decompressedDataSize; return decompressedBuffer; } default: { ereport(ERROR, (errmsg("unexpected compression type: %d", compressionType))); } } } ================================================ FILE: src/backend/columnar/columnar_customscan.c ================================================ /*------------------------------------------------------------------------- * * columnar_customscan.c * * This file contains the implementation of a postgres custom scan that * we use to push down the projections into the table access methods. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "access/amapi.h" #include "access/skey.h" #include "catalog/pg_am.h" #include "catalog/pg_statistic.h" #include "commands/defrem.h" #include "columnar/columnar_version_compat.h" #if PG_VERSION_NUM >= PG_VERSION_18 #include "commands/explain_format.h" #endif #include "executor/executor.h" /* for ExecInitExprWithParams(), ExecEvalExpr() */ #include "nodes/execnodes.h" /* for ExprState, ExprContext, etc. */ #include "nodes/extensible.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "nodes/plannodes.h" #include "optimizer/cost.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/plancat.h" #include "optimizer/restrictinfo.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/selfuncs.h" #include "utils/spccache.h" #include "citus_version.h" #include "columnar/columnar.h" #include "columnar/columnar_customscan.h" #include "columnar/columnar_metadata.h" #include "columnar/columnar_tableam.h" #include "distributed/listutils.h" /* * ColumnarScanState represents the state for a columnar scan. It's a * CustomScanState with additional fields specific to columnar scans. */ typedef struct ColumnarScanState { CustomScanState custom_scanstate; /* must be first field */ ExprContext *css_RuntimeContext; List *qual; } ColumnarScanState; typedef bool (*PathPredicate)(Path *path); /* functions to cost paths in-place */ static void CostColumnarPaths(PlannerInfo *root, RelOptInfo *rel, Oid relationId); static void CostColumnarIndexPath(PlannerInfo *root, RelOptInfo *rel, Oid relationId, IndexPath *indexPath); static void CostColumnarSeqPath(RelOptInfo *rel, Oid relationId, Path *path); static void CostColumnarScan(PlannerInfo *root, RelOptInfo *rel, Oid relationId, CustomPath *cpath, int numberOfColumnsRead, int nClauses); /* functions to add new paths */ static void AddColumnarScanPaths(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void AddColumnarScanPath(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Relids required_relids); /* helper functions to be used when costing paths or altering them */ static void RemovePathsByPredicate(RelOptInfo *rel, PathPredicate removePathPredicate); static bool IsNotIndexPath(Path *path); static Cost ColumnarIndexScanAdditionalCost(PlannerInfo *root, RelOptInfo *rel, Oid relationId, IndexPath *indexPath); static int RelationIdGetNumberOfAttributes(Oid relationId); static Cost ColumnarPerStripeScanCost(RelOptInfo *rel, Oid relationId, int numberOfColumnsRead); static uint64 ColumnarTableStripeCount(Oid relationId); static Path * CreateColumnarSeqScanPath(PlannerInfo *root, RelOptInfo *rel, Oid relationId); static void AddColumnarScanPathsRec(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Relids paramRelids, Relids candidateRelids, int depthLimit); /* hooks and callbacks */ static void ColumnarSetRelPathlistHook(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte); static void ColumnarGetRelationInfoHook(PlannerInfo *root, Oid relationObjectId, bool inhparent, RelOptInfo *rel); static Plan * ColumnarScanPath_PlanCustomPath(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *best_path, List *tlist, List *clauses, List *custom_plans); static List * ColumnarScanPath_ReparameterizeCustomPathByChild(PlannerInfo *root, List *custom_private, RelOptInfo *child_rel); static Node * ColumnarScan_CreateCustomScanState(CustomScan *cscan); static void ColumnarScan_BeginCustomScan(CustomScanState *node, EState *estate, int eflags); static TupleTableSlot * ColumnarScan_ExecCustomScan(CustomScanState *node); static void ColumnarScan_EndCustomScan(CustomScanState *node); static void ColumnarScan_ReScanCustomScan(CustomScanState *node); static void ColumnarScan_ExplainCustomScan(CustomScanState *node, List *ancestors, ExplainState *es); /* helper functions to build strings for EXPLAIN */ static const char * ColumnarPushdownClausesStr(List *context, List *clauses); static const char * ColumnarProjectedColumnsStr(List *context, List *projectedColumns); static List * set_deparse_context_planstate(List *dpcontext, Node *node, List *ancestors); /* other helpers */ static List * ColumnarVarNeeded(ColumnarScanState *columnarScanState); static Bitmapset * ColumnarAttrNeeded(ScanState *ss); static Bitmapset * fixup_inherited_columns(Oid parentId, Oid childId, Bitmapset *columns); /* saved hook value in case of unload */ static set_rel_pathlist_hook_type PreviousSetRelPathlistHook = NULL; static get_relation_info_hook_type PreviousGetRelationInfoHook = NULL; static bool EnableColumnarCustomScan = true; static bool EnableColumnarQualPushdown = true; static double ColumnarQualPushdownCorrelationThreshold = 0.9; static int ColumnarMaxCustomScanPaths = 64; static int ColumnarPlannerDebugLevel = DEBUG3; const struct CustomPathMethods ColumnarScanPathMethods = { .CustomName = "ColumnarScan", .PlanCustomPath = ColumnarScanPath_PlanCustomPath, .ReparameterizeCustomPathByChild = ColumnarScanPath_ReparameterizeCustomPathByChild, }; const struct CustomScanMethods ColumnarScanScanMethods = { .CustomName = "ColumnarScan", .CreateCustomScanState = ColumnarScan_CreateCustomScanState, }; const struct CustomExecMethods ColumnarScanExecuteMethods = { .CustomName = "ColumnarScan", .BeginCustomScan = ColumnarScan_BeginCustomScan, .ExecCustomScan = ColumnarScan_ExecCustomScan, .EndCustomScan = ColumnarScan_EndCustomScan, .ReScanCustomScan = ColumnarScan_ReScanCustomScan, .ExplainCustomScan = ColumnarScan_ExplainCustomScan, }; static const struct config_enum_entry debug_level_options[] = { { "debug5", DEBUG5, false }, { "debug4", DEBUG4, false }, { "debug3", DEBUG3, false }, { "debug2", DEBUG2, false }, { "debug1", DEBUG1, false }, { "debug", DEBUG2, true }, { "info", INFO, false }, { "notice", NOTICE, false }, { "warning", WARNING, false }, { "log", LOG, false }, { NULL, 0, false } }; /* * columnar_customscan_init installs the hook required to intercept the postgres planner and * provide extra paths for columnar tables */ void columnar_customscan_init() { PreviousSetRelPathlistHook = set_rel_pathlist_hook; set_rel_pathlist_hook = ColumnarSetRelPathlistHook; PreviousGetRelationInfoHook = get_relation_info_hook; get_relation_info_hook = ColumnarGetRelationInfoHook; /* register customscan specific GUC's */ DefineCustomBoolVariable( "columnar.enable_custom_scan", gettext_noop("Enables the use of a custom scan to push projections and quals " "into the storage layer."), NULL, &EnableColumnarCustomScan, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "columnar.enable_qual_pushdown", gettext_noop("Enables qual pushdown into columnar. This has no effect unless " "columnar.enable_custom_scan is true."), NULL, &EnableColumnarQualPushdown, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomRealVariable( "columnar.qual_pushdown_correlation_threshold", gettext_noop("Correlation threshold to attempt to push a qual " "referencing the given column. A value of 0 means " "attempt to push down all quals, even if the column " "is uncorrelated."), NULL, &ColumnarQualPushdownCorrelationThreshold, 0.9, 0.0, 1.0, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "columnar.max_custom_scan_paths", gettext_noop("Maximum number of custom scan paths to generate " "for a columnar table when planning."), NULL, &ColumnarMaxCustomScanPaths, 64, 1, 1024, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "columnar.planner_debug_level", "Message level for columnar planning information.", NULL, &ColumnarPlannerDebugLevel, DEBUG3, debug_level_options, PGC_USERSET, 0, NULL, NULL, NULL); RegisterCustomScanMethods(&ColumnarScanScanMethods); } static void ColumnarSetRelPathlistHook(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { /* call into previous hook if assigned */ if (PreviousSetRelPathlistHook) { PreviousSetRelPathlistHook(root, rel, rti, rte); } if (!OidIsValid(rte->relid) || rte->rtekind != RTE_RELATION || rte->inh) { /* some calls to the pathlist hook don't have a valid relation set. Do nothing */ return; } /* * Here we want to inspect if this relation pathlist hook is accessing a columnar table. * If that is the case we want to insert an extra path that pushes down the projection * into the scan of the table to minimize the data read. */ Relation relation = RelationIdGetRelation(rte->relid); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", rte->relid))); } if (relation->rd_tableam == GetColumnarTableAmRoutine()) { if (rte->tablesample != NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("sample scans not supported on columnar tables"))); } if (list_length(rel->partial_pathlist) != 0) { /* * Parallel scans on columnar tables are already discardad by * ColumnarGetRelationInfoHook but be on the safe side. */ elog(ERROR, "parallel scans on columnar are not supported"); } /* * There are cases where IndexPath is normally more preferrable over * SeqPath for heapAM but not for columnarAM. In such cases, an * IndexPath could wrongly dominate a SeqPath based on the costs * estimated by postgres earlier. For this reason, here we manually * create a SeqPath, estimate the cost based on columnarAM and append * to pathlist. * * Before doing that, we first re-cost all the existing paths so that * add_path makes correct cost comparisons when appending our SeqPath. */ CostColumnarPaths(root, rel, rte->relid); Path *seqPath = CreateColumnarSeqScanPath(root, rel, rte->relid); add_path(rel, seqPath); if (EnableColumnarCustomScan) { ereport(DEBUG1, (errmsg("pathlist hook for columnar table am"))); /* * When columnar custom scan is enabled (columnar.enable_custom_scan), * we only consider ColumnarScanPath's & IndexPath's. For this reason, * we remove other paths and re-estimate IndexPath costs to make accurate * comparisons between them. * * Even more, we might calculate an equal cost for a * ColumnarCustomScan and a SeqPath if we are reading all columns * of given table since we don't consider chunk group filtering * when costing ColumnarCustomScan. * In that case, if we don't remove SeqPath's, we might wrongly choose * SeqPath thinking that its cost would be equal to ColumnarCustomScan. */ RemovePathsByPredicate(rel, IsNotIndexPath); AddColumnarScanPaths(root, rel, rte); } } RelationClose(relation); } static void ColumnarGetRelationInfoHook(PlannerInfo *root, Oid relationObjectId, bool inhparent, RelOptInfo *rel) { if (PreviousGetRelationInfoHook) { PreviousGetRelationInfoHook(root, relationObjectId, inhparent, rel); } if (IsColumnarTableAmTable(relationObjectId)) { /* disable parallel query */ rel->rel_parallel_workers = 0; /* disable index-only scan */ IndexOptInfo *indexOptInfo = NULL; foreach_declared_ptr(indexOptInfo, rel->indexlist) { memset(indexOptInfo->canreturn, false, indexOptInfo->ncolumns * sizeof(bool)); } } } /* * RemovePathsByPredicate removes the paths that removePathPredicate * evaluates to true from pathlist of given rel. */ static void RemovePathsByPredicate(RelOptInfo *rel, PathPredicate removePathPredicate) { List *filteredPathList = NIL; Path *path = NULL; foreach_declared_ptr(path, rel->pathlist) { if (!removePathPredicate(path)) { filteredPathList = lappend(filteredPathList, path); } } rel->pathlist = filteredPathList; } /* * IsNotIndexPath returns true if given path is not an IndexPath. */ static bool IsNotIndexPath(Path *path) { return !IsA(path, IndexPath); } /* * CreateColumnarSeqScanPath returns Path for sequential scan on columnar * table with relationId. */ static Path * CreateColumnarSeqScanPath(PlannerInfo *root, RelOptInfo *rel, Oid relationId) { /* columnar doesn't support parallel scan */ int parallelWorkers = 0; Relids requiredOuter = rel->lateral_relids; Path *path = create_seqscan_path(root, rel, requiredOuter, parallelWorkers); CostColumnarSeqPath(rel, relationId, path); return path; } /* * CostColumnarPaths re-costs paths of given RelOptInfo for * columnar table with relationId. */ static void CostColumnarPaths(PlannerInfo *root, RelOptInfo *rel, Oid relationId) { Path *path = NULL; foreach_declared_ptr(path, rel->pathlist) { if (IsA(path, IndexPath)) { /* * Since we don't provide implementations for scan_bitmap_next_block * & scan_bitmap_next_tuple, postgres doesn't generate bitmap index * scan paths for columnar tables already (see related comments in * TableAmRoutine). For this reason, we only consider IndexPath's * here. */ CostColumnarIndexPath(root, rel, relationId, (IndexPath *) path); } else if (path->pathtype == T_SeqScan) { CostColumnarSeqPath(rel, relationId, path); } } } /* * CostColumnarIndexPath re-costs given index path for columnar table with * relationId. */ static void CostColumnarIndexPath(PlannerInfo *root, RelOptInfo *rel, Oid relationId, IndexPath *indexPath) { if (!enable_indexscan) { /* costs are already set to disable_cost, don't adjust them */ return; } ereport(DEBUG4, (errmsg("columnar table index scan costs estimated by " "indexAM: startup cost = %.10f, total cost = " "%.10f", indexPath->path.startup_cost, indexPath->path.total_cost))); /* * We estimate the cost for columnar table read during index scan. Also, * instead of overwriting total cost, we "add" ours to the cost estimated * by indexAM since we should consider index traversal related costs too. */ Cost columnarIndexScanCost = ColumnarIndexScanAdditionalCost(root, rel, relationId, indexPath); indexPath->path.total_cost += columnarIndexScanCost; ereport(DEBUG4, (errmsg("columnar table index scan costs re-estimated " "by columnarAM (including indexAM costs): " "startup cost = %.10f, total cost = %.10f", indexPath->path.startup_cost, indexPath->path.total_cost))); } /* * ColumnarIndexScanAdditionalCost returns additional cost estimated for * index scan described by IndexPath for columnar table with relationId. */ static Cost ColumnarIndexScanAdditionalCost(PlannerInfo *root, RelOptInfo *rel, Oid relationId, IndexPath *indexPath) { int numberOfColumnsRead = RelationIdGetNumberOfAttributes(relationId); Cost perStripeCost = ColumnarPerStripeScanCost(rel, relationId, numberOfColumnsRead); /* * We don't need to pass correct loop count to amcostestimate since we * will only use index correlation & index selectivity, and loop count * doesn't have any effect on those two. */ double fakeLoopCount = 1; Cost fakeIndexStartupCost; Cost fakeIndexTotalCost; double fakeIndexPages; Selectivity indexSelectivity; double indexCorrelation; amcostestimate_function amcostestimate = indexPath->indexinfo->amcostestimate; amcostestimate(root, indexPath, fakeLoopCount, &fakeIndexStartupCost, &fakeIndexTotalCost, &indexSelectivity, &indexCorrelation, &fakeIndexPages); Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } uint64 rowCount = ColumnarTableRowCount(relation); RelationClose(relation); double estimatedRows = rowCount * indexSelectivity; /* * In the worst case (i.e no correlation between the column & the index), * we need to read a different stripe for each row. */ double maxStripeReadCount = estimatedRows; /* * In the best case (i.e the column is fully correlated with the index), * we wouldn't read the same stripe again and again thanks * to locality. */ double avgStripeRowCount = rowCount / (double) ColumnarTableStripeCount(relationId); double minStripeReadCount = estimatedRows / avgStripeRowCount; /* * While being close to 0 means low correlation, being close to -1 or +1 * means high correlation. For index scans on columnar tables, it doesn't * matter if the column and the index are "correlated" (+1) or * "anti-correlated" (-1) since both help us avoiding from reading the * same stripe again and again. */ double absIndexCorrelation = fabs(indexCorrelation); /* * To estimate the number of stripes that we need to read, we do linear * interpolation between minStripeReadCount & maxStripeReadCount. To do * that, we use complement to 1 of absolute correlation, where being * close to 0 means high correlation and being close to 1 means low * correlation. * In practice, we only want to do an index scan when absIndexCorrelation * is 1 (or extremely close to it), or when the absolute number of tuples * returned is very small. Other cases will have a prohibitive cost. */ double complementIndexCorrelation = 1 - absIndexCorrelation; double estimatedStripeReadCount = minStripeReadCount + complementIndexCorrelation * (maxStripeReadCount - minStripeReadCount); /* even in the best case, we will read a single stripe */ estimatedStripeReadCount = Max(estimatedStripeReadCount, 1.0); Cost scanCost = perStripeCost * estimatedStripeReadCount; ereport(DEBUG4, (errmsg("re-costing index scan for columnar table: " "selectivity = %.10f, complement abs " "correlation = %.10f, per stripe cost = %.10f, " "estimated stripe read count = %.10f, " "total additional cost = %.10f", indexSelectivity, complementIndexCorrelation, perStripeCost, estimatedStripeReadCount, scanCost))); return scanCost; } /* * CostColumnarSeqPath sets costs given seq path for columnar table with * relationId. */ static void CostColumnarSeqPath(RelOptInfo *rel, Oid relationId, Path *path) { if (!enable_seqscan) { /* costs are already set to disable_cost, don't adjust them */ return; } /* * Seq scan doesn't support projection or qual pushdown, so we will read * all the stripes and all the columns. */ double stripesToRead = ColumnarTableStripeCount(relationId); int numberOfColumnsRead = RelationIdGetNumberOfAttributes(relationId); path->startup_cost = 0; path->total_cost = stripesToRead * ColumnarPerStripeScanCost(rel, relationId, numberOfColumnsRead); } /* * RelationIdGetNumberOfAttributes returns number of attributes that relation * with relationId has. */ static int RelationIdGetNumberOfAttributes(Oid relationId) { Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } int nattrs = relation->rd_att->natts; RelationClose(relation); return nattrs; } /* * CheckVarStats() checks whether a qual involving this Var is likely to be * useful based on the correlation stats. If so, or if stats are unavailable, * return true; otherwise return false and sets absVarCorrelation in case * caller wants to use for logging purposes. */ static bool CheckVarStats(PlannerInfo *root, Var *var, Oid sortop, float4 *absVarCorrelation) { /* * Collect isunique, ndistinct, and varCorrelation. */ VariableStatData varStatData; examine_variable(root, (Node *) var, var->varno, &varStatData); if (varStatData.rel == NULL || !HeapTupleIsValid(varStatData.statsTuple)) { return true; } AttStatsSlot sslot; if (!get_attstatsslot(&sslot, varStatData.statsTuple, STATISTIC_KIND_CORRELATION, sortop, ATTSTATSSLOT_NUMBERS)) { ReleaseVariableStats(varStatData); return true; } Assert(sslot.nnumbers == 1); float4 varCorrelation = sslot.numbers[0]; ReleaseVariableStats(varStatData); /* * If the Var is not highly correlated, then the chunk's min/max bounds * will be nearly useless. */ if (fabs(varCorrelation) < ColumnarQualPushdownCorrelationThreshold) { if (absVarCorrelation) { /* * Report absVarCorrelation if caller wants to know why given * var is rejected. */ *absVarCorrelation = fabs(varCorrelation); } return false; } return true; } /* * ExprReferencesRelid returns true if any of the Expr's Vars refer to the * given relid; false otherwise. */ static bool ExprReferencesRelid(Expr *expr, Index relid) { List *exprVars = pull_var_clause( (Node *) expr, PVC_RECURSE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS | PVC_RECURSE_PLACEHOLDERS); ListCell *lc; foreach(lc, exprVars) { Var *var = (Var *) lfirst(lc); if (var->varno == relid) { return true; } } return false; } /* * ExtractPushdownClause extracts an Expr node from given clause for pushing down * into the given rel (including join clauses). This test may not be exact in * all cases; it's used to reduce the search space for parameterization. * * Note that we don't try to handle cases like "Var + ExtParam = 3". That * would require going through eval_const_expression after parameter binding, * and that doesn't seem worth the effort. Here we just look for "Var op Expr" * or "Expr op Var", where Var references rel and Expr references other rels * (or no rels at all). * * Moreover, this function also looks into BoolExpr's to recursively extract * pushdownable OpExpr's of them: * i) AND_EXPR: * Take pushdownable args of AND expressions by ignoring the other args. * ii) OR_EXPR: * Ignore the whole OR expression if we cannot exract a pushdownable Expr * from one of its args. * iii) NOT_EXPR: * Simply ignore NOT expressions since we don't expect to see them before * an expression that we can pushdown, see the comment in function. * * The reasoning for those three rules could also be summarized as such; * for any expression that we cannot push-down, we must assume that it * evaluates to true. * * For example, given following WHERE clause: * ( * (a > random() OR a < 30) * AND * a < 200 * ) OR * ( * a = 300 * OR * a > 400 * ); * Even if we can pushdown (a < 30), we cannot pushdown (a > random() OR a < 30) * due to (a > random()). However, we can pushdown (a < 200), so we extract * (a < 200) from the lhs of the top level OR expression. * * For the rhs of the top level OR expression, since we can pushdown both (a = 300) * and (a > 400), we take this part as is. * * Finally, since both sides of the top level OR expression yielded pushdownable * expressions, we will pushdown the following: * (a < 200) OR ((a = 300) OR (a > 400)) */ static Expr * ExtractPushdownClause(PlannerInfo *root, RelOptInfo *rel, Node *node) { CHECK_FOR_INTERRUPTS(); check_stack_depth(); if (node == NULL) { return NULL; } if (IsA(node, BoolExpr)) { BoolExpr *boolExpr = castNode(BoolExpr, node); if (boolExpr->boolop == NOT_EXPR) { /* * Standard planner should have already applied de-morgan rule to * simple NOT expressions. If we encounter with such an expression * here, then it can't be a pushdownable one, such as: * WHERE id NOT IN (SELECT id FROM something). */ ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "must not contain a subplan"))); return NULL; } List *pushdownableArgs = NIL; Node *boolExprArg = NULL; foreach_declared_ptr(boolExprArg, boolExpr->args) { Expr *pushdownableArg = ExtractPushdownClause(root, rel, (Node *) boolExprArg); if (pushdownableArg) { pushdownableArgs = lappend(pushdownableArgs, pushdownableArg); } else if (boolExpr->boolop == OR_EXPR) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "all arguments of an OR expression must be " "pushdownable but one of them was not, due " "to the reason given above"))); return NULL; } /* simply skip AND args that we cannot pushdown */ } int npushdownableArgs = list_length(pushdownableArgs); if (npushdownableArgs == 0) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "none of the arguments were pushdownable, " "due to the reason(s) given above "))); return NULL; } else if (npushdownableArgs == 1) { return (Expr *) linitial(pushdownableArgs); } if (boolExpr->boolop == AND_EXPR) { return make_andclause(pushdownableArgs); } else if (boolExpr->boolop == OR_EXPR) { return make_orclause(pushdownableArgs); } else { /* already discarded NOT expr, so should not be reachable */ return NULL; } } if (IsA(node, ScalarArrayOpExpr)) { if (!contain_volatile_functions(node)) { return (Expr *) node; } else { return NULL; } } if (!IsA(node, OpExpr) || list_length(((OpExpr *) node)->args) != 2) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "must be binary operator expression"))); return NULL; } OpExpr *opExpr = castNode(OpExpr, node); Expr *lhs = list_nth(opExpr->args, 0); Expr *rhs = list_nth(opExpr->args, 1); Var *varSide; Expr *exprSide; if (IsA(lhs, Var) && ((Var *) lhs)->varno == rel->relid && !ExprReferencesRelid((Expr *) rhs, rel->relid)) { varSide = castNode(Var, lhs); exprSide = rhs; } else if (IsA(rhs, Var) && ((Var *) rhs)->varno == rel->relid && !ExprReferencesRelid((Expr *) lhs, rel->relid)) { varSide = castNode(Var, rhs); exprSide = lhs; } else { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "must match 'Var Expr' or 'Expr Var'"), errhint("Var must only reference this rel, " "and Expr must not reference this rel"))); return NULL; } if (varSide->varattno <= 0) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "var is whole-row reference or system column"))); return NULL; } if (contain_volatile_functions((Node *) exprSide)) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "expr contains volatile functions"))); return NULL; } /* only the default opclass is used for qual pushdown. */ Oid varOpClass = GetDefaultOpClass(varSide->vartype, BTREE_AM_OID); Oid varOpFamily; Oid varOpcInType; if (!OidIsValid(varOpClass) || !get_opclass_opfamily_and_input_type(varOpClass, &varOpFamily, &varOpcInType)) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "cannot find default btree opclass and opfamily for type: %s", format_type_be(varSide->vartype)))); return NULL; } if (!op_in_opfamily(opExpr->opno, varOpFamily)) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "operator %d not a member of opfamily %d", opExpr->opno, varOpFamily))); return NULL; } Oid sortop = get_opfamily_member(varOpFamily, varOpcInType, varOpcInType, BTLessStrategyNumber); Assert(OidIsValid(sortop)); /* * Check that statistics on the Var support the utility of this * clause. */ float4 absVarCorrelation = 0; if (!CheckVarStats(root, varSide, sortop, &absVarCorrelation)) { ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: cannot push down clause: " "absolute correlation (%.3f) of var attribute %d is " "smaller than the value configured in " "\"columnar.qual_pushdown_correlation_threshold\" " "(%.3f)", absVarCorrelation, varSide->varattno, ColumnarQualPushdownCorrelationThreshold))); return NULL; } return (Expr *) node; } /* * FilterPushdownClauses filters for clauses that are candidates for pushing * down into rel. */ static List * FilterPushdownClauses(PlannerInfo *root, RelOptInfo *rel, List *inputClauses) { List *filteredClauses = NIL; ListCell *lc; foreach(lc, inputClauses) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); /* * Ignore clauses that don't refer to this rel, and pseudoconstants. * * XXX: A pseudoconstant may be of use, but it doesn't make sense to * push it down because it doesn't contain any Vars. Look into if * there's something we should do with pseudoconstants here. */ if (rinfo->pseudoconstant || !bms_is_member(rel->relid, rinfo->required_relids)) { continue; } Expr *pushdownableExpr = ExtractPushdownClause(root, rel, (Node *) rinfo->clause); if (!pushdownableExpr) { continue; } rinfo = copyObject(rinfo); rinfo->clause = pushdownableExpr; filteredClauses = lappend(filteredClauses, rinfo); } return filteredClauses; } /* * PushdownJoinClauseMatches is a callback that returns true, indicating that * we want all of the clauses from generate_implied_equalities_for_column(). */ static bool PushdownJoinClauseMatches(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg) { return true; } /* * FindPushdownJoinClauses finds join clauses, including those implied by ECs, * that may be pushed down. */ static List * FindPushdownJoinClauses(PlannerInfo *root, RelOptInfo *rel) { List *joinClauses = copyObject(rel->joininfo); /* * Here we are generating the clauses just so we can later extract the * interesting relids. This is somewhat wasteful, but it allows us to * filter out joinclauses, reducing the number of relids we need to * consider. * * XXX: also find additional clauses for joininfo that are implied by ECs? */ List *ecClauses = generate_implied_equalities_for_column( root, rel, PushdownJoinClauseMatches, NULL, rel->lateral_referencers); List *allClauses = list_concat(joinClauses, ecClauses); return FilterPushdownClauses(root, rel, allClauses); } /* * FindCandidateRelids identifies candidate rels for parameterization from the * list of join clauses. * * Some rels cannot be considered for parameterization, such as a partitioned * parent of the given rel. Other rels are just not useful because they don't * appear in a join clause that could be pushed down. */ static Relids FindCandidateRelids(PlannerInfo *root, RelOptInfo *rel, List *joinClauses) { Relids candidateRelids = NULL; ListCell *lc; foreach(lc, joinClauses) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); candidateRelids = bms_add_members(candidateRelids, rinfo->required_relids); } candidateRelids = bms_del_members(candidateRelids, rel->relids); candidateRelids = bms_del_members(candidateRelids, rel->lateral_relids); /* * For the relevant PG16 commit requiring this addition: * postgres/postgres@2489d76 */ candidateRelids = bms_del_members(candidateRelids, root->outer_join_rels); return candidateRelids; } /* * Combinations() calculates the number of combinations of n things taken k at * a time. When the correct result is large, the calculation may produce a * non-integer result, or overflow to inf, which caller should handle * appropriately. * * Use the following two formulae from Knuth TAoCP, 1.2.6: * (2) Combinations(n, k) = (n*(n-1)..(n-k+1)) / (k*(k-1)..1) * (5) Combinations(n, k) = Combinations(n, n-k) */ static double Combinations(int n, int k) { double v = 1; /* * If k is close to n, then both the numerator and the denominator are * close to n!, and we may overflow even if the input is reasonable * (e.g. Combinations(500, 500)). Use formula (5) to choose the smaller, * but equivalent, k. */ k = Min(k, n - k); /* calculate numerator of formula (2) first */ for (int i = n; i >= n - k + 1; i--) { v *= i; } /* * Divide by each factor in the denominator of formula (2), skipping * division by 1. */ for (int i = k; i >= 2; i--) { v /= i; } return v; } /* * ChooseDepthLimit() calculates the depth limit for the parameterization * search, given the number of candidate relations. * * The maximum number of paths generated for a given depthLimit is: * * Combinations(nCandidates, 0) + Combinations(nCandidates, 1) + ... + * Combinations(nCandidates, depthLimit) * * There's no closed formula for a partial sum of combinations, so just keep * increasing the depth until the number of combinations exceeds the limit. */ static int ChooseDepthLimit(int nCandidates) { if (!EnableColumnarQualPushdown) { return 0; } int depth = 0; double numPaths = 1; while (depth < nCandidates) { numPaths += Combinations(nCandidates, depth + 1); if (numPaths > (double) ColumnarMaxCustomScanPaths) { break; } depth++; } return depth; } /* * AddColumnarScanPaths is the entry point for recursively generating * parameterized paths. See AddColumnarScanPathsRec() for discussion. */ static void AddColumnarScanPaths(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { List *joinClauses = FindPushdownJoinClauses(root, rel); Relids candidateRelids = FindCandidateRelids(root, rel, joinClauses); int depthLimit = ChooseDepthLimit(bms_num_members(candidateRelids)); /* must always parameterize by lateral refs */ Relids paramRelids = bms_copy(rel->lateral_relids); AddColumnarScanPathsRec(root, rel, rte, paramRelids, candidateRelids, depthLimit); } /* * AddColumnarScanPathsRec is a recursive function to search the * parameterization space and add CustomPaths for columnar scans. * * The set paramRelids is the parameterization at the current level, and * candidateRelids is the set from which we draw to generate paths with * greater parameterization. * * Columnar tables resemble indexes because of the ability to push down * quals. Ordinary quals, such as x = 7, can be pushed down easily. But join * quals of the form "x = y" (where "y" comes from another rel) require the * proper parameterization. * * Paths that require more outer rels can push down more join clauses that * depend on those outer rels. But requiring more outer rels gives the planner * fewer options for the shape of the plan. That means there is a trade-off, * and we should generate plans of various parameterizations, then let the * planner choose. We always need to generate one minimally-parameterized path * (parameterized only by lateral refs, if present) to make sure that at least * one path can be chosen. Then, we generate as many parameterized paths as we * reasonably can. * * The set of all possible parameterizations is the power set of * candidateRelids. The power set has cardinality 2^N, where N is the * cardinality of candidateRelids. To avoid creating a huge number of paths, * limit the depth of the search; the depthLimit is equivalent to the maximum * number of required outer rels (beyond the minimal parameterization) for the * path. A depthLimit of zero means that only the minimally-parameterized path * will be generated. */ static void AddColumnarScanPathsRec(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Relids paramRelids, Relids candidateRelids, int depthLimit) { CHECK_FOR_INTERRUPTS(); check_stack_depth(); Assert(!bms_overlap(paramRelids, candidateRelids)); AddColumnarScanPath(root, rel, rte, paramRelids); /* recurse for all candidateRelids, unless we hit the depth limit */ Assert(depthLimit >= 0); if (depthLimit-- == 0) { return; } /* * Iterate through parameter combinations depth-first. Deeper levels * generate paths of greater parameterization (and hopefully lower * cost). */ Relids tmpCandidateRelids = bms_copy(candidateRelids); int relid = -1; while ((relid = bms_next_member(candidateRelids, relid)) >= 0) { Relids tmpParamRelids = bms_add_member( bms_copy(paramRelids), relid); /* * Because we are generating combinations (not permutations), remove * the relid from the set of candidates at this level as we descend to * the next. */ tmpCandidateRelids = bms_del_member(tmpCandidateRelids, relid); AddColumnarScanPathsRec(root, rel, rte, tmpParamRelids, tmpCandidateRelids, depthLimit); } bms_free(tmpCandidateRelids); } /* * ParameterizationAsString returns the string representation of the set of * rels given in paramRelids. * * Takes a StringInfo so that it doesn't return palloc'd memory. This makes it * easy to call this function as an argument to ereport(), such that it won't * be evaluated unless the message is going to be output somewhere. */ static char * ParameterizationAsString(PlannerInfo *root, Relids paramRelids, StringInfo buf) { bool firstTime = true; int relid = -1; if (bms_num_members(paramRelids) == 0) { return "unparameterized"; } appendStringInfoString(buf, "parameterized by rels {"); while ((relid = bms_next_member(paramRelids, relid)) >= 0) { RangeTblEntry *rte = root->simple_rte_array[relid]; const char *relname = quote_identifier(rte->eref->aliasname); appendStringInfo(buf, "%s%s", firstTime ? "" : ", ", relname); if (relname != rte->eref->aliasname) { pfree((void *) relname); } firstTime = false; } appendStringInfoString(buf, "}"); return buf->data; } /* * ContainsExecParams tests whether the node contains any exec params. The * signature accepts an extra argument for use with expression_tree_walker. */ static bool ContainsExecParams(Node *node, void *notUsed) { if (node == NULL) { return false; } else if (IsA(node, Param)) { Param *param = castNode(Param, node); if (param->paramkind == PARAM_EXEC) { return true; } } return expression_tree_walker(node, ContainsExecParams, NULL); } /* * Create and add a path with the given parameterization paramRelids. * * XXX: Consider refactoring to be more like postgresGetForeignPaths(). The * only differences are param_info and custom_private. */ static void AddColumnarScanPath(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Relids paramRelids) { /* * Must return a CustomPath, not a larger structure containing a * CustomPath as the first field. Otherwise, nodeToString() will fail to * output the additional fields. */ CustomPath *cpath = makeNode(CustomPath); cpath->methods = &ColumnarScanPathMethods; /* necessary to avoid extra Result node in PG15 */ cpath->flags = CUSTOMPATH_SUPPORT_PROJECTION; /* * populate generic path information */ Path *path = &cpath->path; path->pathtype = T_CustomScan; path->parent = rel; path->pathtarget = rel->reltarget; /* columnar scans are not parallel-aware, but they are parallel-safe */ path->parallel_safe = rel->consider_parallel; path->param_info = get_baserel_parampathinfo(root, rel, paramRelids); /* * Usable clauses for this parameterization exist in baserestrictinfo and * ppi_clauses. */ List *allClauses = copyObject(rel->baserestrictinfo); if (path->param_info != NULL) { allClauses = list_concat(allClauses, path->param_info->ppi_clauses); } allClauses = FilterPushdownClauses(root, rel, allClauses); /* * Plain clauses may contain extern params, but not exec params, and can * be evaluated at init time or rescan time. Track them in another list * that is a subset of allClauses. * * Note: although typically baserestrictinfo contains plain clauses, * that's not always true. It can also contain a qual referencing a Var at * a higher query level, which can be turned into an exec param, and * therefore it won't be a plain clause. */ List *plainClauses = NIL; ListCell *lc; foreach(lc, allClauses) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); if (bms_is_subset(rinfo->required_relids, rel->relids) && !ContainsExecParams((Node *) rinfo->clause, NULL)) { plainClauses = lappend(plainClauses, rinfo); } } /* * We can't make our own CustomPath structure, so we need to put * everything in the custom_private list. To keep the two lists separate, * we make them sublists in a 2-element list. */ if (EnableColumnarQualPushdown) { cpath->custom_private = list_make2(copyObject(plainClauses), copyObject(allClauses)); } else { cpath->custom_private = list_make2(NIL, NIL); } int numberOfColumnsRead = 0; if (rte->perminfoindex > 0) { /* * If perminfoindex > 0, that means that this relation's permission info * is directly found in the list of rteperminfos of the Query(root->parse) * So, all we have to do here is retrieve that info. */ RTEPermissionInfo *perminfo = getRTEPermissionInfo(root->parse->rteperminfos, rte); numberOfColumnsRead = bms_num_members(perminfo->selectedCols); } else { /* * If perminfoindex = 0, that means we are skipping the check for permission info * for this relation, which means that it's either a partition or an inheritance child. * In these cases, we need to access the permission info of the top parent of this relation. * After thorough checking, we found that the index of the top parent pointing to the correct * range table entry in Query's range tables (root->parse->rtable) is found under * RelOptInfo rel->top_parent->relid. * For reference, check expand_partitioned_rtentry and expand_inherited_rtentry PG functions */ Assert(rel->top_parent); RangeTblEntry *parent_rte = rt_fetch(rel->top_parent->relid, root->parse->rtable); RTEPermissionInfo *perminfo = getRTEPermissionInfo(root->parse->rteperminfos, parent_rte); numberOfColumnsRead = bms_num_members(fixup_inherited_columns(perminfo->relid, rte->relid, perminfo-> selectedCols)); } int numberOfClausesPushed = list_length(allClauses); CostColumnarScan(root, rel, rte->relid, cpath, numberOfColumnsRead, numberOfClausesPushed); StringInfoData buf; initStringInfo(&buf); ereport(ColumnarPlannerDebugLevel, (errmsg("columnar planner: adding CustomScan path for %s", rte->eref->aliasname), errdetail("%s; %d clauses pushed down", ParameterizationAsString(root, paramRelids, &buf), numberOfClausesPushed))); add_path(rel, path); } /* * fixup_inherited_columns * * Exact function Copied from PG16 as it's static. * * When user is querying on a table with children, it implicitly accesses * child tables also. So, we also need to check security label of child * tables and columns, but there is no guarantee attribute numbers are * same between the parent and children. * It returns a bitmapset which contains attribute number of the child * table based on the given bitmapset of the parent. */ static Bitmapset * fixup_inherited_columns(Oid parentId, Oid childId, Bitmapset *columns) { Bitmapset *result = NULL; /* * obviously, no need to do anything here */ if (parentId == childId) { return columns; } int index = -1; while ((index = bms_next_member(columns, index)) >= 0) { /* bit numbers are offset by FirstLowInvalidHeapAttributeNumber */ AttrNumber attno = index + FirstLowInvalidHeapAttributeNumber; /* * whole-row-reference shall be fixed-up later */ if (attno == InvalidAttrNumber) { result = bms_add_member(result, index); continue; } char *attname = get_attname(parentId, attno, false); attno = get_attnum(childId, attname); if (attno == InvalidAttrNumber) { elog(ERROR, "cache lookup failed for attribute %s of relation %u", attname, childId); } result = bms_add_member(result, attno - FirstLowInvalidHeapAttributeNumber); pfree(attname); } return result; } /* * CostColumnarScan calculates the cost of scanning the columnar table. The * cost is estimated by using all stripe metadata to estimate based on the * columns to read how many pages need to be read. */ static void CostColumnarScan(PlannerInfo *root, RelOptInfo *rel, Oid relationId, CustomPath *cpath, int numberOfColumnsRead, int nClauses) { Path *path = &cpath->path; List *allClauses = lsecond(cpath->custom_private); Selectivity clauseSel = clauselist_selectivity( root, allClauses, rel->relid, JOIN_INNER, NULL); /* * We already filtered out clauses where the overall selectivity would be * misleading, such as inequalities involving an uncorrelated column. So * we can apply the selectivity directly to the number of stripes. */ double stripesToRead = clauseSel * ColumnarTableStripeCount(relationId); stripesToRead = Max(stripesToRead, 1.0); path->rows = rel->rows; path->startup_cost = 0; path->total_cost = stripesToRead * ColumnarPerStripeScanCost(rel, relationId, numberOfColumnsRead); } /* * ColumnarPerStripeScanCost calculates the cost to scan a single stripe * of given columnar table based on number of columns that needs to be * read during scan operation. */ static Cost ColumnarPerStripeScanCost(RelOptInfo *rel, Oid relationId, int numberOfColumnsRead) { Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *stripeList = StripesForRelfilelocator(relation); RelationClose(relation); uint32 maxColumnCount = 0; uint64 totalStripeSize = 0; StripeMetadata *stripeMetadata = NULL; foreach_declared_ptr(stripeMetadata, stripeList) { totalStripeSize += stripeMetadata->dataLength; maxColumnCount = Max(maxColumnCount, stripeMetadata->columnCount); } /* * When no stripes are in the table we don't have a count in maxColumnCount. To * prevent a division by zero turning into a NaN we keep the ratio on zero. * This will result in a cost of 0 for scanning the table which is a reasonable * cost on an empty table. */ if (maxColumnCount == 0) { return 0; } double columnSelectionRatio = numberOfColumnsRead / (double) maxColumnCount; Cost tableScanCost = (double) totalStripeSize / BLCKSZ * columnSelectionRatio; Cost perStripeScanCost = tableScanCost / list_length(stripeList); /* * Finally, multiply the cost of reading a single stripe by seq page read * cost to make our estimation scale compatible with postgres. * Since we are calculating the cost for a single stripe here, we use seq * page cost instead of random page cost. This is because, random page * access only happens when switching between columns, which is pretty * much neglactable. */ double relSpaceSeqPageCost; get_tablespace_page_costs(rel->reltablespace, NULL, &relSpaceSeqPageCost); perStripeScanCost = perStripeScanCost * relSpaceSeqPageCost; return perStripeScanCost; } /* * ColumnarTableStripeCount returns the number of stripes that columnar * table with relationId has by using stripe metadata. */ static uint64 ColumnarTableStripeCount(Oid relationId) { Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *stripeList = StripesForRelfilelocator(relation); int stripeCount = list_length(stripeList); RelationClose(relation); return stripeCount; } static Plan * ColumnarScanPath_PlanCustomPath(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) { /* * Must return a CustomScan, not a larger structure containing a * CustomScan as the first field. Otherwise, copyObject() will fail to * copy the additional fields. */ CustomScan *cscan = makeNode(CustomScan); cscan->methods = &ColumnarScanScanMethods; /* XXX: also need to store projected column list for EXPLAIN */ if (EnableColumnarQualPushdown) { /* * Lists of pushed-down clauses. The Vars in custom_exprs referencing * other relations will be changed into exec Params by * create_customscan_plan(). * * Like CustomPath->custom_private, keep a list of plain clauses * separate from the list of all clauses by making them sublists of a * 2-element list. * * XXX: custom_exprs are the quals that will be pushed into the * columnar reader code; some of these may not be usable. We should * fix this by processing the quals more completely and using * ScanKeys. */ List *plainClauses = extract_actual_clauses( linitial(best_path->custom_private), false /* no pseudoconstants */); List *allClauses = extract_actual_clauses( lsecond(best_path->custom_private), false /* no pseudoconstants */); cscan->custom_exprs = copyObject(list_make2(plainClauses, allClauses)); } else { cscan->custom_exprs = list_make2(NIL, NIL); } cscan->scan.plan.qual = extract_actual_clauses( clauses, false /* no pseudoconstants */); cscan->scan.plan.targetlist = list_copy(tlist); cscan->scan.scanrelid = best_path->path.parent->relid; #if (PG_VERSION_NUM >= 150000) /* necessary to avoid extra Result node in PG15 */ cscan->flags = CUSTOMPATH_SUPPORT_PROJECTION; #endif return (Plan *) cscan; } /* * ReparameterizeMutator changes all varnos referencing the topmost parent of * child_rel to instead reference child_rel directly. */ static Node * ReparameterizeMutator(Node *node, RelOptInfo *child_rel) { if (node == NULL) { return NULL; } if (IsA(node, Var)) { Var *var = castNode(Var, node); if (bms_is_member(var->varno, child_rel->top_parent_relids)) { var = copyObject(var); var->varno = child_rel->relid; } return (Node *) var; } if (IsA(node, RestrictInfo)) { RestrictInfo *rinfo = castNode(RestrictInfo, node); rinfo = copyObject(rinfo); rinfo->clause = (Expr *) expression_tree_mutator( (Node *) rinfo->clause, ReparameterizeMutator, (void *) child_rel); return (Node *) rinfo; } return expression_tree_mutator(node, ReparameterizeMutator, (void *) child_rel); } /* * ColumnarScanPath_ReparameterizeCustomPathByChild is a method called when a * path is reparameterized directly to a child relation, rather than the * top-level parent. * * For instance, let there be a join of two partitioned columnar relations PX * and PY. A path for a ColumnarScan of PY3 might be parameterized by PX so * that the join qual "PY3.a = PX.a" (referencing the parent PX) can be pushed * down. But if the planner decides on a partition-wise join, then the path * will be reparameterized on the child table PX3 directly. * * When that happens, we need to update all Vars in the pushed-down quals to * reference PX3, not PX, to match the new parameterization. This method * notifies us that it needs to be done, and allows us to update the * information in custom_private. */ static List * ColumnarScanPath_ReparameterizeCustomPathByChild(PlannerInfo *root, List *custom_private, RelOptInfo *child_rel) { return (List *) ReparameterizeMutator((Node *) custom_private, child_rel); } static Node * ColumnarScan_CreateCustomScanState(CustomScan *cscan) { ColumnarScanState *columnarScanState = (ColumnarScanState *) newNode( sizeof(ColumnarScanState), T_CustomScanState); CustomScanState *cscanstate = &columnarScanState->custom_scanstate; cscanstate->methods = &ColumnarScanExecuteMethods; return (Node *) cscanstate; } /* * EvalParamsMutator evaluates Params in the expression and replaces them with * Consts. */ static Node * EvalParamsMutator(Node *node, ExprContext *econtext) { if (node == NULL) { return NULL; } if (IsA(node, Param)) { Param *param = (Param *) node; int16 typLen; bool typByVal; bool isnull; get_typlenbyval(param->paramtype, &typLen, &typByVal); /* XXX: should save ExprState for efficiency */ ExprState *exprState = ExecInitExprWithParams((Expr *) node, econtext->ecxt_param_list_info); Datum pval = ExecEvalExpr(exprState, econtext, &isnull); return (Node *) makeConst(param->paramtype, param->paramtypmod, param->paramcollid, (int) typLen, pval, isnull, typByVal); } return expression_tree_mutator(node, EvalParamsMutator, (void *) econtext); } static void ColumnarScan_BeginCustomScan(CustomScanState *cscanstate, EState *estate, int eflags) { CustomScan *cscan = (CustomScan *) cscanstate->ss.ps.plan; ColumnarScanState *columnarScanState = (ColumnarScanState *) cscanstate; ExprContext *stdecontext = cscanstate->ss.ps.ps_ExprContext; /* * Make a new ExprContext just like the existing one, except that we don't * reset it every tuple. */ ExecAssignExprContext(estate, &cscanstate->ss.ps); columnarScanState->css_RuntimeContext = cscanstate->ss.ps.ps_ExprContext; cscanstate->ss.ps.ps_ExprContext = stdecontext; ResetExprContext(columnarScanState->css_RuntimeContext); List *plainClauses = linitial(cscan->custom_exprs); columnarScanState->qual = (List *) EvalParamsMutator( (Node *) plainClauses, columnarScanState->css_RuntimeContext); /* scan slot is already initialized */ } /* * ColumnarAttrNeeded returns a list of AttrNumber's for the ones that are * needed during columnar custom scan. * Throws an error if finds a Var referencing to an attribute not supported * by ColumnarScan. */ static Bitmapset * ColumnarAttrNeeded(ScanState *ss) { TupleTableSlot *slot = ss->ss_ScanTupleSlot; int natts = slot->tts_tupleDescriptor->natts; Bitmapset *attr_needed = NULL; Plan *plan = ss->ps.plan; int flags = PVC_RECURSE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS | PVC_RECURSE_PLACEHOLDERS; List *vars = list_concat(pull_var_clause((Node *) plan->targetlist, flags), pull_var_clause((Node *) plan->qual, flags)); ListCell *lc; foreach(lc, vars) { Var *var = lfirst(lc); if (var->varattno < 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "UPDATE and CTID scans not supported for ColumnarScan"))); } if (var->varattno == 0) { elog(DEBUG1, "Need attribute: all"); /* all attributes are required, we don't need to add more so break*/ attr_needed = bms_add_range(attr_needed, 0, natts - 1); break; } elog(DEBUG1, "Need attribute: %d", var->varattno); attr_needed = bms_add_member(attr_needed, var->varattno - 1); } return attr_needed; } static TupleTableSlot * ColumnarScanNext(ColumnarScanState *columnarScanState) { CustomScanState *node = (CustomScanState *) columnarScanState; /* * get information from the estate and scan state */ TableScanDesc scandesc = node->ss.ss_currentScanDesc; EState *estate = node->ss.ps.state; ScanDirection direction = estate->es_direction; TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; if (scandesc == NULL) { /* the columnar access method does not use the flags, they are specific to heap */ uint32 flags = 0; Bitmapset *attr_needed = ColumnarAttrNeeded(&node->ss); /* * We reach here if the scan is not parallel, or if we're serially * executing a scan that was planned to be parallel. */ scandesc = columnar_beginscan_extended(node->ss.ss_currentRelation, estate->es_snapshot, 0, NULL, NULL, flags, attr_needed, columnarScanState->qual); bms_free(attr_needed); node->ss.ss_currentScanDesc = scandesc; } /* * get the next tuple from the table */ if (table_scan_getnextslot(scandesc, direction, slot)) { return slot; } return NULL; } /* * SeqRecheck -- access method routine to recheck a tuple in EvalPlanQual */ static bool ColumnarScanRecheck(ColumnarScanState *node, TupleTableSlot *slot) { return true; } static TupleTableSlot * ColumnarScan_ExecCustomScan(CustomScanState *node) { return ExecScan(&node->ss, (ExecScanAccessMtd) ColumnarScanNext, (ExecScanRecheckMtd) ColumnarScanRecheck); } static void ColumnarScan_EndCustomScan(CustomScanState *node) { /* * get information from node */ TableScanDesc scanDesc = node->ss.ss_currentScanDesc; /* * clean out the tuple table */ if (node->ss.ps.ps_ResultTupleSlot) { ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); } ExecClearTuple(node->ss.ss_ScanTupleSlot); /* * close heap scan */ if (scanDesc != NULL) { table_endscan(scanDesc); } } static void ColumnarScan_ReScanCustomScan(CustomScanState *node) { CustomScan *cscan = (CustomScan *) node->ss.ps.plan; ColumnarScanState *columnarScanState = (ColumnarScanState *) node; ResetExprContext(columnarScanState->css_RuntimeContext); List *allClauses = lsecond(cscan->custom_exprs); columnarScanState->qual = (List *) EvalParamsMutator( (Node *) allClauses, columnarScanState->css_RuntimeContext); TableScanDesc scanDesc = node->ss.ss_currentScanDesc; if (scanDesc != NULL) { /* XXX: hack to pass quals as scan keys */ ScanKey scanKeys = (ScanKey) columnarScanState->qual; table_rescan(node->ss.ss_currentScanDesc, scanKeys); } } static void ColumnarScan_ExplainCustomScan(CustomScanState *node, List *ancestors, ExplainState *es) { ColumnarScanState *columnarScanState = (ColumnarScanState *) node; List *context = set_deparse_context_planstate( es->deparse_cxt, (Node *) &node->ss.ps, ancestors); List *projectedColumns = ColumnarVarNeeded(columnarScanState); const char *projectedColumnsStr = ColumnarProjectedColumnsStr( context, projectedColumns); ExplainPropertyText("Columnar Projected Columns", projectedColumnsStr, es); CustomScan *cscan = castNode(CustomScan, node->ss.ps.plan); List *chunkGroupFilter = lsecond(cscan->custom_exprs); if (chunkGroupFilter != NULL) { const char *pushdownClausesStr = ColumnarPushdownClausesStr( context, chunkGroupFilter); ExplainPropertyText("Columnar Chunk Group Filters", pushdownClausesStr, es); ColumnarScanDesc columnarScanDesc = (ColumnarScanDesc) node->ss.ss_currentScanDesc; if (columnarScanDesc != NULL) { ExplainPropertyInteger( "Columnar Chunk Groups Removed by Filter", NULL, ColumnarScanChunkGroupsFiltered(columnarScanDesc), es); } } } /* * ColumnarPushdownClausesStr represents the clauses to push down as a string. */ static const char * ColumnarPushdownClausesStr(List *context, List *clauses) { Expr *conjunction; Assert(list_length(clauses) > 0); if (list_length(clauses) == 1) { conjunction = (Expr *) linitial(clauses); } else { conjunction = make_andclause(clauses); } bool useTableNamePrefix = false; bool showImplicitCast = false; return deparse_expression((Node *) conjunction, context, useTableNamePrefix, showImplicitCast); } /* * ColumnarProjectedColumnsStr generates projected column string for * explain output. */ static const char * ColumnarProjectedColumnsStr(List *context, List *projectedColumns) { if (list_length(projectedColumns) == 0) { return ""; } bool useTableNamePrefix = false; bool showImplicitCast = false; return deparse_expression((Node *) projectedColumns, context, useTableNamePrefix, showImplicitCast); } /* * ColumnarVarNeeded returns a list of Var objects for the ones that are * needed during columnar custom scan. * Throws an error if finds a Var referencing to an attribute not supported * by ColumnarScan. */ static List * ColumnarVarNeeded(ColumnarScanState *columnarScanState) { ScanState *scanState = &columnarScanState->custom_scanstate.ss; List *varList = NIL; Bitmapset *neededAttrSet = ColumnarAttrNeeded(scanState); int bmsMember = -1; while ((bmsMember = bms_next_member(neededAttrSet, bmsMember)) >= 0) { Relation columnarRelation = scanState->ss_currentRelation; /* neededAttrSet already represents 0-indexed attribute numbers */ Form_pg_attribute columnForm = TupleDescAttr(RelationGetDescr(columnarRelation), bmsMember); if (columnForm->attisdropped) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("cannot explain column with attrNum=%d " "of columnar table %s since it is dropped", bmsMember + 1, RelationGetRelationName(columnarRelation)))); } else if (columnForm->attnum <= 0) { /* * ColumnarAttrNeeded should have already thrown an error for * system columns. Similarly, it should have already expanded * whole-row references to individual attributes. */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot explain column with attrNum=%d " "of columnar table %s since it is either " "a system column or a whole-row " "reference", columnForm->attnum, RelationGetRelationName(columnarRelation)))); } /* * varlevelsup is used to figure out the (query) level of the Var * that we are investigating. Since we are dealing with a particular * relation, it is useless here. */ Index varlevelsup = 0; CustomScanState *customScanState = (CustomScanState *) columnarScanState; CustomScan *customScan = (CustomScan *) customScanState->ss.ps.plan; Index scanrelid = customScan->scan.scanrelid; Var *var = makeVar(scanrelid, columnForm->attnum, columnForm->atttypid, columnForm->atttypmod, columnForm->attcollation, varlevelsup); varList = lappend(varList, var); } return varList; } /* * set_deparse_context_planstate is a compatibility wrapper for versions 13+. */ static List * set_deparse_context_planstate(List *dpcontext, Node *node, List *ancestors) { PlanState *ps = (PlanState *) node; return set_deparse_context_plan(dpcontext, ps->plan, ancestors); } ================================================ FILE: src/backend/columnar/columnar_debug.c ================================================ /*------------------------------------------------------------------------- * * columnar_debug.c * * Helper functions to debug column store. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "access/nbtree.h" #include "access/table.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" #include "storage/fd.h" #include "storage/smgr.h" #include "utils/guc.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/tuplestore.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "columnar/columnar.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_version_compat.h" static void MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters); PG_FUNCTION_INFO_V1(columnar_store_memory_stats); PG_FUNCTION_INFO_V1(columnar_storage_info); /* * columnar_store_memory_stats returns a record of 3 values: size of * TopMemoryContext, TopTransactionContext, and Write State context. */ Datum columnar_store_memory_stats(PG_FUNCTION_ARGS) { const int resultColumnCount = 3; TupleDesc tupleDescriptor = CreateTemplateTupleDesc(resultColumnCount); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 1, "TopMemoryContext", INT8OID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 2, "TopTransactionContext", INT8OID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 3, "WriteStateContext", INT8OID, -1, 0); tupleDescriptor = BlessTupleDesc(tupleDescriptor); MemoryContextCounters transactionCounters = { 0 }; MemoryContextCounters topCounters = { 0 }; MemoryContextCounters writeStateCounters = { 0 }; MemoryContextTotals(TopTransactionContext, &transactionCounters); MemoryContextTotals(TopMemoryContext, &topCounters); MemoryContextTotals(GetWriteContextForDebug(), &writeStateCounters); bool nulls[3] = { false }; Datum values[3] = { Int64GetDatum(topCounters.totalspace), Int64GetDatum(transactionCounters.totalspace), Int64GetDatum(writeStateCounters.totalspace) }; HeapTuple tuple = heap_form_tuple(tupleDescriptor, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } /* * columnar_storage_info - UDF to return internal storage info for a columnar relation. * * DDL: * CREATE OR REPLACE FUNCTION columnar_storage_info( * rel regclass, * version_major OUT int4, * version_minor OUT int4, * storage_id OUT int8, * reserved_stripe_id OUT int8, * reserved_row_number OUT int8, * reserved_offset OUT int8) * STRICT * LANGUAGE c AS 'MODULE_PATHNAME', 'columnar_storage_info'; */ Datum columnar_storage_info(PG_FUNCTION_ARGS) { #define STORAGE_INFO_NATTS 6 Oid relid = PG_GETARG_OID(0); TupleDesc tupdesc; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) { elog(ERROR, "return type must be a row type"); } if (tupdesc->natts != STORAGE_INFO_NATTS) { elog(ERROR, "return type must have %d columns", STORAGE_INFO_NATTS); } Relation rel = table_open(relid, AccessShareLock); if (!IsColumnarTableAmTable(relid)) { ereport(ERROR, (errmsg("table \"%s\" is not a columnar table", RelationGetRelationName(rel)))); } Datum values[STORAGE_INFO_NATTS] = { 0 }; bool nulls[STORAGE_INFO_NATTS] = { 0 }; /* * Pass force = true so that we can inspect metapages that are not the * current version. * * NB: ensure the order and number of attributes correspond to DDL * declaration. */ values[0] = Int32GetDatum(ColumnarStorageGetVersionMajor(rel, true)); values[1] = Int32GetDatum(ColumnarStorageGetVersionMinor(rel, true)); values[2] = Int64GetDatum(ColumnarStorageGetStorageId(rel, true)); values[3] = Int64GetDatum(ColumnarStorageGetReservedStripeId(rel, true)); values[4] = Int64GetDatum(ColumnarStorageGetReservedRowNumber(rel, true)); values[5] = Int64GetDatum(ColumnarStorageGetReservedOffset(rel, true)); /* release lock */ table_close(rel, AccessShareLock); HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } /* * MemoryContextTotals adds stats of the given memory context and its * subtree to the given counters. */ static void MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters) { if (context == NULL) { return; } MemoryContext child; for (child = context->firstchild; child != NULL; child = child->nextchild) { MemoryContextTotals(child, counters); } context->methods->stats(context, NULL, NULL, counters, true); } ================================================ FILE: src/backend/columnar/columnar_metadata.c ================================================ /*------------------------------------------------------------------------- * * columnar_metadata.c * * Copyright (c) Citus Data, Inc. * * Manages metadata for columnar relations in separate, shared metadata tables * in the "columnar" schema. * * * holds basic stripe information including data size and row counts * * holds basic chunk and chunk group information like data offsets and * min/max values (used for Chunk Group Filtering) * * useful for fast VACUUM operations (e.g. reporting with VACUUM VERBOSE) * * useful for stats/costing * * maps logical row numbers to stripe IDs * * TODO: visibility information * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "port.h" #include "safe_lib.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "access/xact.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_collation.h" #include "catalog/pg_namespace.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" #include "lib/stringinfo.h" #include "nodes/execnodes.h" #include "parser/parse_relation.h" #include "storage/fd.h" #include "storage/lmgr.h" #include "storage/procarray.h" #include "storage/relfilelocator.h" #include "storage/smgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/relfilenumbermap.h" #include "citus_version.h" #include "pg_version_constants.h" #include "columnar/columnar.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_version_compat.h" #include "distributed/listutils.h" #define COLUMNAR_RELOPTION_NAMESPACE "columnar" #define SLOW_METADATA_ACCESS_WARNING \ "Metadata index %s is not available, this might mean slower read/writes " \ "on columnar tables. This is expected during Postgres upgrades and not " \ "expected otherwise." typedef struct { Relation rel; EState *estate; ResultRelInfo *resultRelInfo; } ModifyState; /* RowNumberLookupMode to be used in StripeMetadataLookupRowNumber */ typedef enum RowNumberLookupMode { /* * Find the stripe whose firstRowNumber is less than or equal to given * input rowNumber. */ FIND_LESS_OR_EQUAL, /* * Find the stripe whose firstRowNumber is greater than input rowNumber. */ FIND_GREATER } RowNumberLookupMode; static void ParseColumnarRelOptions(List *reloptions, ColumnarOptions *options); static void InsertEmptyStripeMetadataRow(uint64 storageId, uint64 stripeId, uint32 columnCount, uint32 chunkGroupRowCount, uint64 firstRowNumber); static void GetHighestUsedAddressAndId(uint64 storageId, uint64 *highestUsedAddress, uint64 *highestUsedId); static StripeMetadata * UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, uint64 fileOffset, uint64 dataLength, uint64 rowCount, uint64 chunkCount); static List * ReadDataFileStripeList(uint64 storageId, Snapshot snapshot); static StripeMetadata * BuildStripeMetadata(Relation columnarStripes, HeapTuple heapTuple); static uint32 * ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount, Snapshot snapshot); static Oid ColumnarStorageIdSequenceRelationId(void); static Oid ColumnarStripeRelationId(void); static Oid ColumnarStripePKeyIndexRelationId(void); static Oid ColumnarStripeFirstRowNumberIndexRelationId(void); static Oid ColumnarOptionsRelationId(void); static Oid ColumnarOptionsIndexRegclass(void); static Oid ColumnarChunkRelationId(void); static Oid ColumnarChunkGroupRelationId(void); static Oid ColumnarChunkIndexRelationId(void); static Oid ColumnarChunkGroupIndexRelationId(void); static Oid ColumnarNamespaceId(void); static uint64 LookupStorageId(Oid relationId, RelFileLocator relfilelocator); static uint64 GetHighestUsedRowNumber(uint64 storageId); static void DeleteStorageFromColumnarMetadataTable(Oid metadataTableId, AttrNumber storageIdAtrrNumber, Oid storageIdIndexId, uint64 storageId); static ModifyState * StartModifyRelation(Relation rel); static void InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values, bool *nulls); static void DeleteTupleAndEnforceConstraints(ModifyState *state, HeapTuple heapTuple); static void FinishModifyRelation(ModifyState *state); static EState * create_estate_for_relation(Relation rel); static bytea * DatumToBytea(Datum value, Form_pg_attribute attrForm); static Datum ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm); static bool WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite); static StripeMetadata * StripeMetadataLookupRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot, RowNumberLookupMode lookupMode); static void CheckStripeMetadataConsistency(StripeMetadata *stripeMetadata); PG_FUNCTION_INFO_V1(columnar_relation_storageid); /* constants for columnar.options */ #define Natts_columnar_options 5 #define Anum_columnar_options_regclass 1 #define Anum_columnar_options_chunk_group_row_limit 2 #define Anum_columnar_options_stripe_row_limit 3 #define Anum_columnar_options_compression_level 4 #define Anum_columnar_options_compression 5 /* ---------------- * columnar.options definition. * ---------------- */ typedef struct FormData_columnar_options { Oid regclass; int32 chunk_group_row_limit; int32 stripe_row_limit; int32 compressionLevel; NameData compression; #ifdef CATALOG_VARLEN /* variable-length fields start here */ #endif } FormData_columnar_options; typedef FormData_columnar_options *Form_columnar_options; /* constants for columnar.stripe */ #define Natts_columnar_stripe 9 #define Anum_columnar_stripe_storageid 1 #define Anum_columnar_stripe_stripe 2 #define Anum_columnar_stripe_file_offset 3 #define Anum_columnar_stripe_data_length 4 #define Anum_columnar_stripe_column_count 5 #define Anum_columnar_stripe_chunk_row_count 6 #define Anum_columnar_stripe_row_count 7 #define Anum_columnar_stripe_chunk_count 8 #define Anum_columnar_stripe_first_row_number 9 static int GetFirstRowNumberAttrIndexInColumnarStripe(TupleDesc tupleDesc); /* constants for columnar.chunk_group */ #define Natts_columnar_chunkgroup 4 #define Anum_columnar_chunkgroup_storageid 1 #define Anum_columnar_chunkgroup_stripe 2 #define Anum_columnar_chunkgroup_chunk 3 #define Anum_columnar_chunkgroup_row_count 4 /* constants for columnar.chunk */ #define Natts_columnar_chunk 14 #define Anum_columnar_chunk_storageid 1 #define Anum_columnar_chunk_stripe 2 #define Anum_columnar_chunk_attr 3 #define Anum_columnar_chunk_chunk 4 #define Anum_columnar_chunk_minimum_value 5 #define Anum_columnar_chunk_maximum_value 6 #define Anum_columnar_chunk_value_stream_offset 7 #define Anum_columnar_chunk_value_stream_length 8 #define Anum_columnar_chunk_exists_stream_offset 9 #define Anum_columnar_chunk_exists_stream_length 10 #define Anum_columnar_chunk_value_compression_type 11 #define Anum_columnar_chunk_value_compression_level 12 #define Anum_columnar_chunk_value_decompressed_size 13 #define Anum_columnar_chunk_value_count 14 /* * InitColumnarOptions initialized the columnar table options. Meaning it writes the * default options to the options table if not already existing. */ void InitColumnarOptions(Oid regclass) { /* * When upgrading we retain options for all columnar tables by upgrading * "columnar.options" catalog table, so we shouldn't do anything here. */ if (IsBinaryUpgrade) { return; } ColumnarOptions defaultOptions = { .chunkRowCount = columnar_chunk_group_row_limit, .stripeRowCount = columnar_stripe_row_limit, .compressionType = columnar_compression, .compressionLevel = columnar_compression_level }; WriteColumnarOptions(regclass, &defaultOptions, false); } /* * ParseColumnarRelOptions - update the given 'options' using the given list * of DefElem. */ static void ParseColumnarRelOptions(List *reloptions, ColumnarOptions *options) { ListCell *lc = NULL; foreach(lc, reloptions) { DefElem *elem = castNode(DefElem, lfirst(lc)); if (elem->defnamespace == NULL || strcmp(elem->defnamespace, COLUMNAR_RELOPTION_NAMESPACE) != 0) { ereport(ERROR, (errmsg("columnar options must have the prefix \"%s\"", COLUMNAR_RELOPTION_NAMESPACE))); } if (strcmp(elem->defname, "chunk_group_row_limit") == 0) { options->chunkRowCount = (elem->arg == NULL) ? columnar_chunk_group_row_limit : defGetInt64(elem); if (options->chunkRowCount < CHUNK_ROW_COUNT_MINIMUM || options->chunkRowCount > CHUNK_ROW_COUNT_MAXIMUM) { ereport(ERROR, (errmsg("chunk group row count limit out of range"), errhint("chunk group row count limit must be between " UINT64_FORMAT " and " UINT64_FORMAT, (uint64) CHUNK_ROW_COUNT_MINIMUM, (uint64) CHUNK_ROW_COUNT_MAXIMUM))); } } else if (strcmp(elem->defname, "stripe_row_limit") == 0) { options->stripeRowCount = (elem->arg == NULL) ? columnar_stripe_row_limit : defGetInt64(elem); if (options->stripeRowCount < STRIPE_ROW_COUNT_MINIMUM || options->stripeRowCount > STRIPE_ROW_COUNT_MAXIMUM) { ereport(ERROR, (errmsg("stripe row count limit out of range"), errhint("stripe row count limit must be between " UINT64_FORMAT " and " UINT64_FORMAT, (uint64) STRIPE_ROW_COUNT_MINIMUM, (uint64) STRIPE_ROW_COUNT_MAXIMUM))); } } else if (strcmp(elem->defname, "compression") == 0) { options->compressionType = (elem->arg == NULL) ? columnar_compression : ParseCompressionType( defGetString(elem)); if (options->compressionType == COMPRESSION_TYPE_INVALID) { ereport(ERROR, (errmsg("unknown compression type for columnar table: %s", quote_identifier(defGetString(elem))))); } } else if (strcmp(elem->defname, "compression_level") == 0) { options->compressionLevel = (elem->arg == NULL) ? columnar_compression_level : defGetInt64(elem); if (options->compressionLevel < COMPRESSION_LEVEL_MIN || options->compressionLevel > COMPRESSION_LEVEL_MAX) { ereport(ERROR, (errmsg("compression level out of range"), errhint("compression level must be between %d and %d", COMPRESSION_LEVEL_MIN, COMPRESSION_LEVEL_MAX))); } } else { ereport(ERROR, (errmsg("unrecognized columnar storage parameter \"%s\"", elem->defname))); } } } /* * ExtractColumnarOptions - extract columnar options from inOptions, appending * to inoutColumnarOptions. Return the remaining (non-columnar) options. */ List * ExtractColumnarRelOptions(List *inOptions, List **inoutColumnarOptions) { List *otherOptions = NIL; ListCell *lc = NULL; foreach(lc, inOptions) { DefElem *elem = castNode(DefElem, lfirst(lc)); if (elem->defnamespace != NULL && strcmp(elem->defnamespace, COLUMNAR_RELOPTION_NAMESPACE) == 0) { *inoutColumnarOptions = lappend(*inoutColumnarOptions, elem); } else { otherOptions = lappend(otherOptions, elem); } } /* validate options */ ColumnarOptions dummy = { 0 }; ParseColumnarRelOptions(*inoutColumnarOptions, &dummy); return otherOptions; } /* * SetColumnarRelOptions - apply the list of DefElem options to the * relation. If there are duplicates, the last one in the list takes effect. */ void SetColumnarRelOptions(RangeVar *rv, List *reloptions) { ColumnarOptions options = { 0 }; if (reloptions == NIL) { return; } Relation rel = relation_openrv(rv, AccessShareLock); Oid relid = RelationGetRelid(rel); relation_close(rel, NoLock); /* get existing or default options */ if (!ReadColumnarOptions(relid, &options)) { /* if extension doesn't exist, just return */ return; } ParseColumnarRelOptions(reloptions, &options); SetColumnarOptions(relid, &options); } /* * SetColumnarOptions writes the passed table options as the authoritive options to the * table irregardless of the optiones already existing or not. This can be used to put a * table in a certain state. */ void SetColumnarOptions(Oid regclass, ColumnarOptions *options) { WriteColumnarOptions(regclass, options, true); } /* * WriteColumnarOptions writes the options to the catalog table for a given regclass. * - If overwrite is false it will only write the values if there is not already a record * found. * - If overwrite is true it will always write the settings * * The return value indicates if the record has been written. */ static bool WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite) { /* * When upgrading we should retain the options from the previous * cluster and don't write new options. */ Assert(!IsBinaryUpgrade); bool written = false; bool nulls[Natts_columnar_options] = { 0 }; Datum values[Natts_columnar_options] = { ObjectIdGetDatum(regclass), Int32GetDatum(options->chunkRowCount), Int32GetDatum(options->stripeRowCount), Int32GetDatum(options->compressionLevel), 0, /* to be filled below */ }; NameData compressionName = { 0 }; namestrcpy(&compressionName, CompressionTypeStr(options->compressionType)); values[Anum_columnar_options_compression - 1] = NameGetDatum(&compressionName); /* create heap tuple and insert into catalog table */ Relation columnarOptions = relation_open(ColumnarOptionsRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(columnarOptions); /* find existing item to perform update if exist */ ScanKeyData scanKey[1] = { 0 }; ScanKeyInit(&scanKey[0], Anum_columnar_options_regclass, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(regclass)); Relation index = index_open(ColumnarOptionsIndexRegclass(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarOptions, index, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); if (HeapTupleIsValid(heapTuple)) { if (overwrite) { /* TODO check if the options are actually different, skip if not changed */ /* update existing record */ bool update[Natts_columnar_options] = { 0 }; update[Anum_columnar_options_chunk_group_row_limit - 1] = true; update[Anum_columnar_options_stripe_row_limit - 1] = true; update[Anum_columnar_options_compression_level - 1] = true; update[Anum_columnar_options_compression - 1] = true; HeapTuple tuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, nulls, update); CatalogTupleUpdate(columnarOptions, &tuple->t_self, tuple); written = true; } } else { /* inserting new record */ HeapTuple newTuple = heap_form_tuple(tupleDescriptor, values, nulls); CatalogTupleInsert(columnarOptions, newTuple); written = true; } if (written) { CommandCounterIncrement(); } systable_endscan_ordered(scanDescriptor); index_close(index, AccessShareLock); relation_close(columnarOptions, RowExclusiveLock); return written; } /* * DeleteColumnarTableOptions removes the columnar table options for a regclass. When * missingOk is false it will throw an error when no table options can be found. * * Returns whether a record has been removed. */ bool DeleteColumnarTableOptions(Oid regclass, bool missingOk) { bool result = false; /* * When upgrading we shouldn't delete or modify table options and * retain options from the previous cluster. */ Assert(!IsBinaryUpgrade); Relation columnarOptions = try_relation_open(ColumnarOptionsRelationId(), RowExclusiveLock); if (columnarOptions == NULL) { /* extension has been dropped */ return false; } /* find existing item to remove */ ScanKeyData scanKey[1] = { 0 }; ScanKeyInit(&scanKey[0], Anum_columnar_options_regclass, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(regclass)); Relation index = index_open(ColumnarOptionsIndexRegclass(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarOptions, index, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); if (HeapTupleIsValid(heapTuple)) { CatalogTupleDelete(columnarOptions, &heapTuple->t_self); CommandCounterIncrement(); result = true; } else if (!missingOk) { ereport(ERROR, (errmsg("missing options for regclass: %d", regclass))); } systable_endscan_ordered(scanDescriptor); index_close(index, AccessShareLock); relation_close(columnarOptions, RowExclusiveLock); return result; } bool ReadColumnarOptions(Oid regclass, ColumnarOptions *options) { ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_columnar_options_regclass, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(regclass)); Oid columnarOptionsOid = ColumnarOptionsRelationId(); Relation columnarOptions = try_relation_open(columnarOptionsOid, AccessShareLock); if (columnarOptions == NULL) { /* * Extension has been dropped. This can be called while * dropping extension or database via ObjectAccess(). */ return false; } Relation index = try_relation_open(ColumnarOptionsIndexRegclass(), AccessShareLock); if (index == NULL) { table_close(columnarOptions, AccessShareLock); /* extension has been dropped */ return false; } SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarOptions, index, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); if (HeapTupleIsValid(heapTuple)) { Form_columnar_options tupOptions = (Form_columnar_options) GETSTRUCT(heapTuple); options->chunkRowCount = tupOptions->chunk_group_row_limit; options->stripeRowCount = tupOptions->stripe_row_limit; options->compressionLevel = tupOptions->compressionLevel; options->compressionType = ParseCompressionType(NameStr(tupOptions->compression)); } else { /* populate options with system defaults */ options->compressionType = columnar_compression; options->stripeRowCount = columnar_stripe_row_limit; options->chunkRowCount = columnar_chunk_group_row_limit; options->compressionLevel = columnar_compression_level; } systable_endscan_ordered(scanDescriptor); index_close(index, AccessShareLock); relation_close(columnarOptions, AccessShareLock); return true; } /* * SaveStripeSkipList saves chunkList for a given stripe as rows * of columnar.chunk. */ void SaveStripeSkipList(Oid relid, RelFileLocator relfilelocator, uint64 stripe, StripeSkipList *chunkList, TupleDesc tupleDescriptor) { uint32 columnIndex = 0; uint32 chunkIndex = 0; uint32 columnCount = chunkList->columnCount; uint64 storageId = LookupStorageId(relid, relfilelocator); Oid columnarChunkOid = ColumnarChunkRelationId(); Relation columnarChunk = table_open(columnarChunkOid, RowExclusiveLock); ModifyState *modifyState = StartModifyRelation(columnarChunk); bool pushed_snapshot = false; if (!ActiveSnapshotSet()) { PushActiveSnapshot(GetTransactionSnapshot()); pushed_snapshot = true; } for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { for (chunkIndex = 0; chunkIndex < chunkList->chunkCount; chunkIndex++) { ColumnChunkSkipNode *chunk = &chunkList->chunkSkipNodeArray[columnIndex][chunkIndex]; Datum values[Natts_columnar_chunk] = { UInt64GetDatum(storageId), Int64GetDatum(stripe), Int32GetDatum(columnIndex + 1), Int32GetDatum(chunkIndex), 0, /* to be filled below */ 0, /* to be filled below */ Int64GetDatum(chunk->valueChunkOffset), Int64GetDatum(chunk->valueLength), Int64GetDatum(chunk->existsChunkOffset), Int64GetDatum(chunk->existsLength), Int32GetDatum(chunk->valueCompressionType), Int32GetDatum(chunk->valueCompressionLevel), Int64GetDatum(chunk->decompressedValueSize), Int64GetDatum(chunk->rowCount) }; bool nulls[Natts_columnar_chunk] = { false }; if (chunk->hasMinMax) { values[Anum_columnar_chunk_minimum_value - 1] = PointerGetDatum(DatumToBytea(chunk->minimumValue, TupleDescAttr(tupleDescriptor, columnIndex))); values[Anum_columnar_chunk_maximum_value - 1] = PointerGetDatum(DatumToBytea(chunk->maximumValue, TupleDescAttr(tupleDescriptor, columnIndex))); } else { nulls[Anum_columnar_chunk_minimum_value - 1] = true; nulls[Anum_columnar_chunk_maximum_value - 1] = true; } InsertTupleAndEnforceConstraints(modifyState, values, nulls); } } if (pushed_snapshot) { PopActiveSnapshot(); } FinishModifyRelation(modifyState); table_close(columnarChunk, RowExclusiveLock); } /* * SaveChunkGroups saves the metadata for given chunk groups in columnar.chunk_group. */ void SaveChunkGroups(Oid relid, RelFileLocator relfilelocator, uint64 stripe, List *chunkGroupRowCounts) { uint64 storageId = LookupStorageId(relid, relfilelocator); Oid columnarChunkGroupOid = ColumnarChunkGroupRelationId(); Relation columnarChunkGroup = table_open(columnarChunkGroupOid, RowExclusiveLock); ModifyState *modifyState = StartModifyRelation(columnarChunkGroup); ListCell *lc = NULL; int chunkId = 0; foreach(lc, chunkGroupRowCounts) { int64 rowCount = lfirst_int(lc); Datum values[Natts_columnar_chunkgroup] = { UInt64GetDatum(storageId), Int64GetDatum(stripe), Int32GetDatum(chunkId), Int64GetDatum(rowCount) }; bool nulls[Natts_columnar_chunkgroup] = { false }; InsertTupleAndEnforceConstraints(modifyState, values, nulls); chunkId++; } FinishModifyRelation(modifyState); table_close(columnarChunkGroup, NoLock); } /* * ReadStripeSkipList fetches chunk metadata for a given stripe. */ StripeSkipList * ReadStripeSkipList(Relation rel, uint64 stripe, TupleDesc tupleDescriptor, uint32 chunkCount, Snapshot snapshot) { int32 columnIndex = 0; HeapTuple heapTuple = NULL; uint32 columnCount = tupleDescriptor->natts; ScanKeyData scanKey[2]; uint64 storageId = LookupStorageId(RelationPrecomputeOid(rel), rel->rd_locator); Oid columnarChunkOid = ColumnarChunkRelationId(); Relation columnarChunk = table_open(columnarChunkOid, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_columnar_chunk_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); ScanKeyInit(&scanKey[1], Anum_columnar_chunk_stripe, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(stripe)); Oid indexId = ColumnarChunkIndexRelationId(); bool indexOk = OidIsValid(indexId); SysScanDesc scanDescriptor = systable_beginscan(columnarChunk, indexId, indexOk, snapshot, 2, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_pkey"))); loggedSlowMetadataAccessWarning = true; } StripeSkipList *chunkList = palloc0(sizeof(StripeSkipList)); chunkList->chunkCount = chunkCount; chunkList->columnCount = columnCount; chunkList->chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *)); for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { chunkList->chunkSkipNodeArray[columnIndex] = palloc0(chunkCount * sizeof(ColumnChunkSkipNode)); } while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Datum datumArray[Natts_columnar_chunk]; bool isNullArray[Natts_columnar_chunk]; heap_deform_tuple(heapTuple, RelationGetDescr(columnarChunk), datumArray, isNullArray); int32 attr = DatumGetInt32(datumArray[Anum_columnar_chunk_attr - 1]); int32 chunkIndex = DatumGetInt32(datumArray[Anum_columnar_chunk_chunk - 1]); if (attr <= 0 || attr > columnCount) { ereport(ERROR, (errmsg("invalid columnar chunk entry"), errdetail("Attribute number out of range: %d", attr))); } if (chunkIndex < 0 || chunkIndex >= chunkCount) { ereport(ERROR, (errmsg("invalid columnar chunk entry"), errdetail("Chunk number out of range: %d", chunkIndex))); } columnIndex = attr - 1; ColumnChunkSkipNode *chunk = &chunkList->chunkSkipNodeArray[columnIndex][chunkIndex]; chunk->rowCount = DatumGetInt64(datumArray[Anum_columnar_chunk_value_count - 1]); chunk->valueChunkOffset = DatumGetInt64(datumArray[Anum_columnar_chunk_value_stream_offset - 1]); chunk->valueLength = DatumGetInt64(datumArray[Anum_columnar_chunk_value_stream_length - 1]); chunk->existsChunkOffset = DatumGetInt64(datumArray[Anum_columnar_chunk_exists_stream_offset - 1]); chunk->existsLength = DatumGetInt64(datumArray[Anum_columnar_chunk_exists_stream_length - 1]); chunk->valueCompressionType = DatumGetInt32(datumArray[Anum_columnar_chunk_value_compression_type - 1]); chunk->valueCompressionLevel = DatumGetInt32(datumArray[Anum_columnar_chunk_value_compression_level - 1]); chunk->decompressedValueSize = DatumGetInt64(datumArray[Anum_columnar_chunk_value_decompressed_size - 1]); if (isNullArray[Anum_columnar_chunk_minimum_value - 1] || isNullArray[Anum_columnar_chunk_maximum_value - 1]) { chunk->hasMinMax = false; } else { bytea *minValue = DatumGetByteaP( datumArray[Anum_columnar_chunk_minimum_value - 1]); bytea *maxValue = DatumGetByteaP( datumArray[Anum_columnar_chunk_maximum_value - 1]); chunk->minimumValue = ByteaToDatum(minValue, TupleDescAttr(tupleDescriptor, columnIndex)); chunk->maximumValue = ByteaToDatum(maxValue, TupleDescAttr(tupleDescriptor, columnIndex)); chunk->hasMinMax = true; } } systable_endscan(scanDescriptor); table_close(columnarChunk, AccessShareLock); chunkList->chunkGroupRowCounts = ReadChunkGroupRowCounts(storageId, stripe, chunkCount, snapshot); return chunkList; } /* * FindStripeByRowNumber returns StripeMetadata for the stripe that has the * smallest firstRowNumber among the stripes whose firstRowNumber is grater * than given rowNumber. If no such stripe exists, then returns NULL. */ StripeMetadata * FindNextStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot) { return StripeMetadataLookupRowNumber(relation, rowNumber, snapshot, FIND_GREATER); } /* * FindStripeByRowNumber returns StripeMetadata for the stripe that contains * the row with rowNumber. If no such stripe exists, then returns NULL. */ StripeMetadata * FindStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot) { StripeMetadata *stripeMetadata = FindStripeWithMatchingFirstRowNumber(relation, rowNumber, snapshot); if (!stripeMetadata) { return NULL; } if (rowNumber > StripeGetHighestRowNumber(stripeMetadata)) { return NULL; } return stripeMetadata; } /* * FindStripeWithMatchingFirstRowNumber returns a StripeMetadata object for * the stripe that has the greatest firstRowNumber among the stripes whose * firstRowNumber is smaller than or equal to given rowNumber. If no such * stripe exists, then returns NULL. * * Note that this doesn't mean that found stripe certainly contains the tuple * with given rowNumber. This is because, it also needs to be verified if * highest row number that found stripe contains is greater than or equal to * given rowNumber. For this reason, unless that additional check is done, * this function is mostly useful for checking against "possible" constraint * violations due to concurrent writes that are not flushed by other backends * yet. */ StripeMetadata * FindStripeWithMatchingFirstRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot) { return StripeMetadataLookupRowNumber(relation, rowNumber, snapshot, FIND_LESS_OR_EQUAL); } /* * StripeWriteState returns write state of given stripe. */ StripeWriteStateEnum StripeWriteState(StripeMetadata *stripeMetadata) { if (stripeMetadata->aborted) { return STRIPE_WRITE_ABORTED; } else if (stripeMetadata->rowCount > 0) { return STRIPE_WRITE_FLUSHED; } else { return STRIPE_WRITE_IN_PROGRESS; } } /* * StripeGetHighestRowNumber returns rowNumber of the row with highest * rowNumber in given stripe. */ uint64 StripeGetHighestRowNumber(StripeMetadata *stripeMetadata) { return stripeMetadata->firstRowNumber + stripeMetadata->rowCount - 1; } /* * StripeMetadataLookupRowNumber returns StripeMetadata for the stripe whose * firstRowNumber is less than or equal to (FIND_LESS_OR_EQUAL), or is * greater than (FIND_GREATER) given rowNumber. * If no such stripe exists, then returns NULL. */ static StripeMetadata * StripeMetadataLookupRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot, RowNumberLookupMode lookupMode) { Assert(lookupMode == FIND_LESS_OR_EQUAL || lookupMode == FIND_GREATER); StripeMetadata *foundStripeMetadata = NULL; uint64 storageId = ColumnarStorageGetStorageId(relation, false); ScanKeyData scanKey[2]; ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); StrategyNumber strategyNumber = InvalidStrategy; RegProcedure procedure = InvalidOid; if (lookupMode == FIND_LESS_OR_EQUAL) { strategyNumber = BTLessEqualStrategyNumber; procedure = F_INT8LE; } else if (lookupMode == FIND_GREATER) { strategyNumber = BTGreaterStrategyNumber; procedure = F_INT8GT; } Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(columnarStripes); ScanKeyInit(&scanKey[1], GetFirstRowNumberAttrIndexInColumnarStripe(tupleDesc) + 1, strategyNumber, procedure, Int64GetDatum(rowNumber)); Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId(); bool indexOk = OidIsValid(indexId); SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk, snapshot, 2, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_first_row_number_idx"))); loggedSlowMetadataAccessWarning = true; } if (indexOk) { ScanDirection scanDirection = NoMovementScanDirection; if (lookupMode == FIND_LESS_OR_EQUAL) { scanDirection = BackwardScanDirection; } else if (lookupMode == FIND_GREATER) { scanDirection = ForwardScanDirection; } HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, scanDirection); if (HeapTupleIsValid(heapTuple)) { foundStripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple); } } else { HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple); if (!foundStripeMetadata) { /* first match */ foundStripeMetadata = stripe; } else if (lookupMode == FIND_LESS_OR_EQUAL && stripe->firstRowNumber > foundStripeMetadata->firstRowNumber) { /* * Among the stripes with firstRowNumber less-than-or-equal-to given, * we're looking for the one with the greatest firstRowNumber. */ foundStripeMetadata = stripe; } else if (lookupMode == FIND_GREATER && stripe->firstRowNumber < foundStripeMetadata->firstRowNumber) { /* * Among the stripes with firstRowNumber greater-than given, * we're looking for the one with the smallest firstRowNumber. */ foundStripeMetadata = stripe; } } } systable_endscan(scanDescriptor); table_close(columnarStripes, AccessShareLock); return foundStripeMetadata; } /* * CheckStripeMetadataConsistency first decides if stripe write operation for * given stripe is "flushed", "aborted" or "in-progress", then errors out if * its metadata entry contradicts with this fact. * * Checks performed here are just to catch bugs, so it is encouraged to call * this function whenever a StripeMetadata object is built from an heap tuple * of columnar.stripe. Currently, BuildStripeMetadata is the only function * that does this. */ static void CheckStripeMetadataConsistency(StripeMetadata *stripeMetadata) { bool stripeLooksInProgress = stripeMetadata->rowCount == 0 && stripeMetadata->chunkCount == 0 && stripeMetadata->fileOffset == ColumnarInvalidLogicalOffset && stripeMetadata->dataLength == 0; /* * Even if stripe is flushed, fileOffset and dataLength might be equal * to 0 for zero column tables, but those two should still be consistent * with respect to each other. */ bool stripeLooksFlushed = stripeMetadata->rowCount > 0 && stripeMetadata->chunkCount > 0 && ((stripeMetadata->fileOffset != ColumnarInvalidLogicalOffset && stripeMetadata->dataLength > 0) || (stripeMetadata->fileOffset == ColumnarInvalidLogicalOffset && stripeMetadata->dataLength == 0)); StripeWriteStateEnum stripeWriteState = StripeWriteState(stripeMetadata); if (stripeWriteState == STRIPE_WRITE_FLUSHED && stripeLooksFlushed) { /* * If stripe was flushed to disk, then we expect stripe to store * at least one tuple. */ return; } else if (stripeWriteState == STRIPE_WRITE_IN_PROGRESS && stripeLooksInProgress) { /* * If stripe was not flushed to disk, then values of given four * fields should match the columns inserted by * InsertEmptyStripeMetadataRow. */ return; } else if (stripeWriteState == STRIPE_WRITE_ABORTED && (stripeLooksInProgress || stripeLooksFlushed)) { /* * Stripe metadata entry for an aborted write can be complete or * incomplete. We might have aborted the transaction before or after * inserting into stripe metadata. */ return; } ereport(ERROR, (errmsg("unexpected stripe state, stripe metadata " "entry for stripe with id=" UINT64_FORMAT " is not consistent", stripeMetadata->id))); } /* * FindStripeWithHighestRowNumber returns StripeMetadata for the stripe that * has the row with highest rowNumber. If given relation is empty, then returns * NULL. */ StripeMetadata * FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot) { StripeMetadata *stripeWithHighestRowNumber = NULL; uint64 storageId = ColumnarStorageGetStorageId(relation, false); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock); Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId(); bool indexOk = OidIsValid(indexId); SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk, snapshot, 1, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_first_row_number_idx"))); loggedSlowMetadataAccessWarning = true; } if (indexOk) { /* do one-time fetch using the index */ HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, BackwardScanDirection); if (HeapTupleIsValid(heapTuple)) { stripeWithHighestRowNumber = BuildStripeMetadata(columnarStripes, heapTuple); } } else { HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple); if (!stripeWithHighestRowNumber || stripe->firstRowNumber > stripeWithHighestRowNumber->firstRowNumber) { /* first or a greater match */ stripeWithHighestRowNumber = stripe; } } } systable_endscan(scanDescriptor); table_close(columnarStripes, AccessShareLock); return stripeWithHighestRowNumber; } /* * ReadChunkGroupRowCounts returns an array of row counts of chunk groups for the * given stripe. */ static uint32 * ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount, Snapshot snapshot) { Oid columnarChunkGroupOid = ColumnarChunkGroupRelationId(); Relation columnarChunkGroup = table_open(columnarChunkGroupOid, AccessShareLock); ScanKeyData scanKey[2]; ScanKeyInit(&scanKey[0], Anum_columnar_chunkgroup_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); ScanKeyInit(&scanKey[1], Anum_columnar_chunkgroup_stripe, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(stripe)); Oid indexId = ColumnarChunkGroupIndexRelationId(); bool indexOk = OidIsValid(indexId); SysScanDesc scanDescriptor = systable_beginscan(columnarChunkGroup, indexId, indexOk, snapshot, 2, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_group_pkey"))); loggedSlowMetadataAccessWarning = true; } HeapTuple heapTuple = NULL; uint32 *chunkGroupRowCounts = palloc0(chunkGroupCount * sizeof(uint32)); while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Datum datumArray[Natts_columnar_chunkgroup]; bool isNullArray[Natts_columnar_chunkgroup]; heap_deform_tuple(heapTuple, RelationGetDescr(columnarChunkGroup), datumArray, isNullArray); uint32 tupleChunkGroupIndex = DatumGetUInt32(datumArray[Anum_columnar_chunkgroup_chunk - 1]); if (tupleChunkGroupIndex >= chunkGroupCount) { elog(ERROR, "unexpected chunk group"); } chunkGroupRowCounts[tupleChunkGroupIndex] = (uint32) DatumGetUInt64(datumArray[Anum_columnar_chunkgroup_row_count - 1]); } systable_endscan(scanDescriptor); table_close(columnarChunkGroup, AccessShareLock); return chunkGroupRowCounts; } /* * InsertEmptyStripeMetadataRow adds a row to columnar.stripe for the empty * stripe reservation made for stripeId. */ static void InsertEmptyStripeMetadataRow(uint64 storageId, uint64 stripeId, uint32 columnCount, uint32 chunkGroupRowCount, uint64 firstRowNumber) { Oid columnarStripesOid = ColumnarStripeRelationId(); Relation columnarStripes = table_open(columnarStripesOid, RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(columnarStripes); Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *nulls = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); values[Anum_columnar_stripe_storageid - 1] = UInt64GetDatum(storageId); values[Anum_columnar_stripe_stripe - 1] = UInt64GetDatum(stripeId); values[Anum_columnar_stripe_column_count - 1] = UInt32GetDatum(columnCount); values[Anum_columnar_stripe_chunk_row_count - 1] = UInt32GetDatum(chunkGroupRowCount); values[GetFirstRowNumberAttrIndexInColumnarStripe(tupleDescriptor)] = UInt64GetDatum(firstRowNumber); /* stripe has no rows yet, so initialize rest of the columns accordingly */ values[Anum_columnar_stripe_row_count - 1] = UInt64GetDatum(0); values[Anum_columnar_stripe_file_offset - 1] = UInt64GetDatum(ColumnarInvalidLogicalOffset); values[Anum_columnar_stripe_data_length - 1] = UInt64GetDatum(0); values[Anum_columnar_stripe_chunk_count - 1] = UInt32GetDatum(0); ModifyState *modifyState = StartModifyRelation(columnarStripes); InsertTupleAndEnforceConstraints(modifyState, values, nulls); FinishModifyRelation(modifyState); table_close(columnarStripes, RowExclusiveLock); pfree(values); pfree(nulls); } /* * StripesForRelfilelocator returns a list of StripeMetadata for stripes * of the given relfilenode. */ List * StripesForRelfilelocator(Relation rel) { uint64 storageId = LookupStorageId(RelationPrecomputeOid(rel), rel->rd_locator); /* * PG18 requires snapshot to be active or registered before it's used * Without this, we hit * Assert(snapshot->regd_count > 0 || snapshot->active_count > 0); * when reading columnar stripes. * Relevant PG18 commit: * 8076c00592e40e8dbd1fce7a98b20d4bf075e4ba */ Snapshot snapshot = RegisterSnapshot(GetTransactionSnapshot()); List *readDataFileStripeList = ReadDataFileStripeList(storageId, snapshot); UnregisterSnapshot(snapshot); return readDataFileStripeList; } /* * GetHighestUsedAddress returns the highest used address for the given * relfilenode across all active and inactive transactions. * * This is used by truncate stage of VACUUM, and VACUUM can be called * for empty tables. So this doesn't throw errors for empty tables and * returns 0. */ uint64 GetHighestUsedAddress(Relation rel) { uint64 storageId = LookupStorageId(RelationPrecomputeOid(rel), rel->rd_locator); uint64 highestUsedAddress = 0; uint64 highestUsedId = 0; GetHighestUsedAddressAndId(storageId, &highestUsedAddress, &highestUsedId); return highestUsedAddress; } /* * In case if relid hasn't been defined yet, we should use RelidByRelfilenumber * to get correct relid value. * * Now it is basically used for temp rels, because since PG18(it was backpatched * through PG13) RelidByRelfilenumber skip temp relations and we should use * alternative ways to get relid value in case of temp objects. */ Oid ColumnarRelationId(Oid relid, RelFileLocator relfilelocator) { return OidIsValid(relid) ? relid : RelidByRelfilenumber(relfilelocator.spcOid, relfilelocator.relNumber); } /* * GetHighestUsedAddressAndId returns the highest used address and id for * the given relfilenode across all active and inactive transactions. */ static void GetHighestUsedAddressAndId(uint64 storageId, uint64 *highestUsedAddress, uint64 *highestUsedId) { ListCell *stripeMetadataCell = NULL; SnapshotData SnapshotDirty; InitDirtySnapshot(SnapshotDirty); List *stripeMetadataList = ReadDataFileStripeList(storageId, &SnapshotDirty); *highestUsedId = 0; /* file starts with metapage */ *highestUsedAddress = COLUMNAR_BYTES_PER_PAGE; foreach(stripeMetadataCell, stripeMetadataList) { StripeMetadata *stripe = lfirst(stripeMetadataCell); uint64 lastByte = stripe->fileOffset + stripe->dataLength - 1; *highestUsedAddress = Max(*highestUsedAddress, lastByte); *highestUsedId = Max(*highestUsedId, stripe->id); } } /* * ReserveEmptyStripe reserves an empty stripe for given relation * and inserts it into columnar.stripe. It is guaranteed that concurrent * writes won't overwrite the returned stripe. */ EmptyStripeReservation * ReserveEmptyStripe(Relation rel, uint64 columnCount, uint64 chunkGroupRowCount, uint64 stripeRowCount) { EmptyStripeReservation *stripeReservation = palloc0(sizeof(EmptyStripeReservation)); uint64 storageId = ColumnarStorageGetStorageId(rel, false); stripeReservation->stripeId = ColumnarStorageReserveStripeId(rel); stripeReservation->stripeFirstRowNumber = ColumnarStorageReserveRowNumber(rel, stripeRowCount); /* * XXX: Instead of inserting a dummy entry to columnar.stripe and * updating it when flushing the stripe, we could have a hash table * in shared memory for the bookkeeping of ongoing writes. */ InsertEmptyStripeMetadataRow(storageId, stripeReservation->stripeId, columnCount, chunkGroupRowCount, stripeReservation->stripeFirstRowNumber); return stripeReservation; } /* * CompleteStripeReservation completes reservation of the stripe with * stripeId for given size and in-place updates related stripe metadata tuple * to complete reservation. */ StripeMetadata * CompleteStripeReservation(Relation rel, uint64 stripeId, uint64 sizeBytes, uint64 rowCount, uint64 chunkCount) { uint64 resLogicalStart = ColumnarStorageReserveData(rel, sizeBytes); uint64 storageId = ColumnarStorageGetStorageId(rel, false); return UpdateStripeMetadataRow(storageId, stripeId, resLogicalStart, sizeBytes, rowCount, chunkCount); } /* * UpdateStripeMetadataRow updates stripe metadata tuple for the stripe with * stripeId according to given newValues and update arrays. * Note that this function shouldn't be used for the cases where any indexes * of stripe metadata should be updated according to modifications done. */ static StripeMetadata * UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, uint64 fileOffset, uint64 dataLength, uint64 rowCount, uint64 chunkCount) { ScanKeyData scanKey[2]; ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); ScanKeyInit(&scanKey[1], Anum_columnar_stripe_stripe, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(stripeId)); Oid columnarStripesOid = ColumnarStripeRelationId(); Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(columnarStripes); Oid indexId = ColumnarStripePKeyIndexRelationId(); bool indexOk = OidIsValid(indexId); void *state; HeapTuple tuple; systable_inplace_update_begin(columnarStripes, indexId, indexOk, NULL, 2, scanKey, &tuple, &state); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_pkey"))); loggedSlowMetadataAccessWarning = true; } if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errmsg("attempted to modify an unexpected stripe, " "columnar storage with id=" UINT64_FORMAT " does not have stripe with id=" UINT64_FORMAT, storageId, stripeId))); } /* * systable_inplace_update_finish already doesn't allow changing size of the original * tuple, so we don't allow setting any Datum's to NULL values. */ Datum *newValues = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *newNulls = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *update = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); update[Anum_columnar_stripe_file_offset - 1] = true; update[Anum_columnar_stripe_data_length - 1] = true; update[Anum_columnar_stripe_row_count - 1] = true; update[Anum_columnar_stripe_chunk_count - 1] = true; newValues[Anum_columnar_stripe_file_offset - 1] = Int64GetDatum(fileOffset); newValues[Anum_columnar_stripe_data_length - 1] = Int64GetDatum(dataLength); newValues[Anum_columnar_stripe_row_count - 1] = UInt64GetDatum(rowCount); newValues[Anum_columnar_stripe_chunk_count - 1] = Int32GetDatum(chunkCount); tuple = heap_modify_tuple(tuple, tupleDescriptor, newValues, newNulls, update); systable_inplace_update_finish(state, tuple); StripeMetadata *modifiedStripeMetadata = BuildStripeMetadata(columnarStripes, tuple); CommandCounterIncrement(); heap_freetuple(tuple); table_close(columnarStripes, AccessShareLock); pfree(newValues); pfree(newNulls); pfree(update); /* return StripeMetadata object built from modified tuple */ return modifiedStripeMetadata; } /* * ReadDataFileStripeList reads the stripe list for a given storageId * in the given snapshot. * * Doesn't sort the stripes by their ids before returning if * stripe_first_row_number_idx is not available --normally can only happen * during pg upgrades. */ static List * ReadDataFileStripeList(uint64 storageId, Snapshot snapshot) { List *stripeMetadataList = NIL; ScanKeyData scanKey[1]; HeapTuple heapTuple; ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); Oid columnarStripesOid = ColumnarStripeRelationId(); Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock); Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId(); bool indexOk = OidIsValid(indexId); SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk, snapshot, 1, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_first_row_number_idx"))); loggedSlowMetadataAccessWarning = true; } while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { StripeMetadata *stripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple); stripeMetadataList = lappend(stripeMetadataList, stripeMetadata); } systable_endscan(scanDescriptor); table_close(columnarStripes, AccessShareLock); return stripeMetadataList; } /* * BuildStripeMetadata builds a StripeMetadata object from given heap tuple. * * NB: heapTuple must be a proper heap tuple with MVCC fields. */ static StripeMetadata * BuildStripeMetadata(Relation columnarStripes, HeapTuple heapTuple) { Assert(RelationGetRelid(columnarStripes) == ColumnarStripeRelationId()); TupleDesc tupleDescriptor = RelationGetDescr(columnarStripes); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); StripeMetadata *stripeMetadata = palloc0(sizeof(StripeMetadata)); stripeMetadata->id = DatumGetInt64(datumArray[Anum_columnar_stripe_stripe - 1]); stripeMetadata->fileOffset = DatumGetInt64( datumArray[Anum_columnar_stripe_file_offset - 1]); stripeMetadata->dataLength = DatumGetInt64( datumArray[Anum_columnar_stripe_data_length - 1]); stripeMetadata->columnCount = DatumGetInt32( datumArray[Anum_columnar_stripe_column_count - 1]); stripeMetadata->chunkCount = DatumGetInt32( datumArray[Anum_columnar_stripe_chunk_count - 1]); stripeMetadata->chunkGroupRowCount = DatumGetInt32( datumArray[Anum_columnar_stripe_chunk_row_count - 1]); stripeMetadata->rowCount = DatumGetInt64( datumArray[Anum_columnar_stripe_row_count - 1]); stripeMetadata->firstRowNumber = DatumGetUInt64( datumArray[GetFirstRowNumberAttrIndexInColumnarStripe(tupleDescriptor)]); pfree(datumArray); pfree(isNullArray); /* * If there is unflushed data in a parent transaction, then we would * have already thrown an error before starting to scan the table.. If * the data is from an earlier subxact that committed, then it would * have been flushed already. For this reason, we don't care about * subtransaction id here. */ TransactionId entryXmin = HeapTupleHeaderGetXmin(heapTuple->t_data); stripeMetadata->aborted = !TransactionIdIsInProgress(entryXmin) && TransactionIdDidAbort(entryXmin); stripeMetadata->insertedByCurrentXact = TransactionIdIsCurrentTransactionId(entryXmin); CheckStripeMetadataConsistency(stripeMetadata); return stripeMetadata; } /* * DeleteMetadataRows removes the rows with given relfilenode from columnar * metadata tables. */ void DeleteMetadataRows(Relation rel) { /* * During a restore for binary upgrade, metadata tables and indexes may or * may not exist. */ if (IsBinaryUpgrade) { return; } uint64 storageId = LookupStorageId(RelationPrecomputeOid(rel), rel->rd_locator); DeleteStorageFromColumnarMetadataTable(ColumnarStripeRelationId(), Anum_columnar_stripe_storageid, ColumnarStripePKeyIndexRelationId(), storageId); DeleteStorageFromColumnarMetadataTable(ColumnarChunkGroupRelationId(), Anum_columnar_chunkgroup_storageid, ColumnarChunkGroupIndexRelationId(), storageId); DeleteStorageFromColumnarMetadataTable(ColumnarChunkRelationId(), Anum_columnar_chunk_storageid, ColumnarChunkIndexRelationId(), storageId); } /* * DeleteStorageFromColumnarMetadataTable removes the rows with given * storageId from given columnar metadata table. */ static void DeleteStorageFromColumnarMetadataTable(Oid metadataTableId, AttrNumber storageIdAtrrNumber, Oid storageIdIndexId, uint64 storageId) { ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], storageIdAtrrNumber, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(storageId)); Relation metadataTable = try_relation_open(metadataTableId, AccessShareLock); if (metadataTable == NULL) { /* extension has been dropped */ return; } bool indexOk = OidIsValid(storageIdIndexId); SysScanDesc scanDescriptor = systable_beginscan(metadataTable, storageIdIndexId, indexOk, NULL, 1, scanKey); static bool loggedSlowMetadataAccessWarning = false; if (!indexOk && !loggedSlowMetadataAccessWarning) { ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "on a columnar metadata table"))); loggedSlowMetadataAccessWarning = true; } ModifyState *modifyState = StartModifyRelation(metadataTable); HeapTuple heapTuple; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { DeleteTupleAndEnforceConstraints(modifyState, heapTuple); } systable_endscan(scanDescriptor); FinishModifyRelation(modifyState); table_close(metadataTable, AccessShareLock); } /* * StartModifyRelation allocates resources for modifications. */ static ModifyState * StartModifyRelation(Relation rel) { EState *estate = create_estate_for_relation(rel); ResultRelInfo *resultRelInfo = makeNode(ResultRelInfo); InitResultRelInfo(resultRelInfo, rel, 1, NULL, 0); /* ExecSimpleRelationInsert, ... require caller to open indexes */ ExecOpenIndices(resultRelInfo, false); ModifyState *modifyState = palloc(sizeof(ModifyState)); modifyState->rel = rel; modifyState->estate = estate; modifyState->resultRelInfo = resultRelInfo; return modifyState; } /* * InsertTupleAndEnforceConstraints inserts a tuple into a relation and makes * sure constraints are enforced and indexes are updated. */ static void InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values, bool *nulls) { TupleDesc tupleDescriptor = RelationGetDescr(state->rel); HeapTuple tuple = heap_form_tuple(tupleDescriptor, values, nulls); TupleTableSlot *slot = ExecInitExtraTupleSlot(state->estate, tupleDescriptor, &TTSOpsHeapTuple); ExecStoreHeapTuple(tuple, slot, false); /* use ExecSimpleRelationInsert to enforce constraints */ ExecSimpleRelationInsert(state->resultRelInfo, state->estate, slot); } /* * DeleteTupleAndEnforceConstraints deletes a tuple from a relation and * makes sure constraints (e.g. FK constraints) are enforced. */ static void DeleteTupleAndEnforceConstraints(ModifyState *state, HeapTuple heapTuple) { EState *estate = state->estate; ResultRelInfo *resultRelInfo = state->resultRelInfo; ItemPointer tid = &(heapTuple->t_self); simple_heap_delete(state->rel, tid); /* execute AFTER ROW DELETE Triggers to enforce constraints */ ExecARDeleteTriggers(estate, resultRelInfo, tid, NULL, NULL, false); } /* * FinishModifyRelation cleans up resources after modifications are done. */ static void FinishModifyRelation(ModifyState *state) { ExecCloseIndices(state->resultRelInfo); AfterTriggerEndQuery(state->estate); ExecCloseResultRelations(state->estate); ExecCloseRangeTableRelations(state->estate); ExecResetTupleTable(state->estate->es_tupleTable, false); FreeExecutorState(state->estate); CommandCounterIncrement(); } /* * Based on a similar function from * postgres/src/backend/replication/logical/worker.c. * * Executor state preparation for evaluation of constraint expressions, * indexes and triggers. * * This is based on similar code in copy.c */ static EState * create_estate_for_relation(Relation rel) { EState *estate = CreateExecutorState(); RangeTblEntry *rte = makeNode(RangeTblEntry); rte->rtekind = RTE_RELATION; rte->relid = RelationGetRelid(rel); rte->relkind = rel->rd_rel->relkind; rte->rellockmode = AccessShareLock; /* Prepare permission info on PG 16+ */ List *perminfos = NIL; addRTEPermissionInfo(&perminfos, rte); /* Initialize the range table, with the right signature for each PG version */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+ needs four arguments (unpruned_relids) */ ExecInitRangeTable( estate, list_make1(rte), perminfos, NULL /* unpruned_relids: not used by columnar */ ); #else /* PG 16–17: three-arg signature (permInfos) */ ExecInitRangeTable( estate, list_make1(rte), perminfos ); #endif estate->es_output_cid = GetCurrentCommandId(true); /* Prepare to catch AFTER triggers. */ AfterTriggerBeginQuery(); return estate; } /* * DatumToBytea serializes a datum into a bytea value. * * Since we don't want to limit datum size to RSIZE_MAX unnecessarily, * we use memcpy instead of memcpy_s several places in this function. */ static bytea * DatumToBytea(Datum value, Form_pg_attribute attrForm) { int datumLength = att_addlength_datum(0, attrForm->attlen, value); bytea *result = palloc0(datumLength + VARHDRSZ); SET_VARSIZE(result, datumLength + VARHDRSZ); if (attrForm->attlen > 0) { if (attrForm->attbyval) { Datum tmp; store_att_byval(&tmp, value, attrForm->attlen); memcpy(VARDATA(result), &tmp, attrForm->attlen); /* IGNORE-BANNED */ } else { memcpy(VARDATA(result), DatumGetPointer(value), attrForm->attlen); /* IGNORE-BANNED */ } } else { memcpy(VARDATA(result), DatumGetPointer(value), datumLength); /* IGNORE-BANNED */ } return result; } /* * ByteaToDatum deserializes a value which was previously serialized using * DatumToBytea. */ static Datum ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm) { /* * We copy the data so the result of this function lives even * after the byteaDatum is freed. */ char *binaryDataCopy = palloc0(VARSIZE_ANY_EXHDR(bytes)); /* * We use IGNORE-BANNED here since we don't want to limit datum size to * RSIZE_MAX unnecessarily. */ memcpy(binaryDataCopy, VARDATA_ANY(bytes), VARSIZE_ANY_EXHDR(bytes)); /* IGNORE-BANNED */ return fetch_att(binaryDataCopy, attrForm->attbyval, attrForm->attlen); } /* * ColumnarStorageIdSequenceRelationId returns relation id of columnar.stripe. * TODO: should we cache this similar to citus? */ static Oid ColumnarStorageIdSequenceRelationId(void) { return get_relname_relid("storageid_seq", ColumnarNamespaceId()); } /* * ColumnarStripeRelationId returns relation id of columnar.stripe. * TODO: should we cache this similar to citus? */ static Oid ColumnarStripeRelationId(void) { return get_relname_relid("stripe", ColumnarNamespaceId()); } /* * ColumnarStripePKeyIndexRelationId returns relation id of columnar.stripe_pkey. * TODO: should we cache this similar to citus? */ static Oid ColumnarStripePKeyIndexRelationId(void) { return get_relname_relid("stripe_pkey", ColumnarNamespaceId()); } /* * ColumnarStripeFirstRowNumberIndexRelationId returns relation id of * columnar.stripe_first_row_number_idx. * TODO: should we cache this similar to citus? */ static Oid ColumnarStripeFirstRowNumberIndexRelationId(void) { return get_relname_relid("stripe_first_row_number_idx", ColumnarNamespaceId()); } /* * ColumnarOptionsRelationId returns relation id of columnar.options. */ static Oid ColumnarOptionsRelationId(void) { return get_relname_relid("options", ColumnarNamespaceId()); } /* * ColumnarOptionsIndexRegclass returns relation id of columnar.options_pkey. */ static Oid ColumnarOptionsIndexRegclass(void) { return get_relname_relid("options_pkey", ColumnarNamespaceId()); } /* * ColumnarChunkRelationId returns relation id of columnar.chunk. * TODO: should we cache this similar to citus? */ static Oid ColumnarChunkRelationId(void) { return get_relname_relid("chunk", ColumnarNamespaceId()); } /* * ColumnarChunkGroupRelationId returns relation id of columnar.chunk_group. * TODO: should we cache this similar to citus? */ static Oid ColumnarChunkGroupRelationId(void) { return get_relname_relid("chunk_group", ColumnarNamespaceId()); } /* * ColumnarChunkIndexRelationId returns relation id of columnar.chunk_pkey. * TODO: should we cache this similar to citus? */ static Oid ColumnarChunkIndexRelationId(void) { return get_relname_relid("chunk_pkey", ColumnarNamespaceId()); } /* * ColumnarChunkGroupIndexRelationId returns relation id of columnar.chunk_group_pkey. * TODO: should we cache this similar to citus? */ static Oid ColumnarChunkGroupIndexRelationId(void) { return get_relname_relid("chunk_group_pkey", ColumnarNamespaceId()); } /* * ColumnarNamespaceId returns namespace id of the schema we store columnar * related tables. */ static Oid ColumnarNamespaceId(void) { Oid namespace = get_namespace_oid("columnar_internal", true); /* if schema is earlier than 11.1-1 */ if (!OidIsValid(namespace)) { namespace = get_namespace_oid("columnar", false); } return namespace; } /* * LookupStorageId reads storage metapage to find the storage ID for the given relfilenode. It returns * false if the relation doesn't have a meta page yet. */ static uint64 LookupStorageId(Oid relid, RelFileLocator relfilelocator) { relid = ColumnarRelationId(relid, relfilelocator); Relation relation = relation_open(relid, AccessShareLock); uint64 storageId = ColumnarStorageGetStorageId(relation, false); table_close(relation, AccessShareLock); return storageId; } /* * ColumnarMetadataNewStorageId - create a new, unique storage id and return * it. */ uint64 ColumnarMetadataNewStorageId() { return nextval_internal(ColumnarStorageIdSequenceRelationId(), false); } /* * columnar_relation_storageid returns storage id associated with the * given relation id, or -1 if there is no associated storage id yet. */ Datum columnar_relation_storageid(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); Relation relation = relation_open(relationId, AccessShareLock); if (!object_ownercheck(RelationRelationId, relationId, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_TABLE, get_rel_name(relationId)); } if (!IsColumnarTableAmTable(relationId)) { elog(ERROR, "relation \"%s\" is not a columnar table", RelationGetRelationName(relation)); } uint64 storageId = ColumnarStorageGetStorageId(relation, false); relation_close(relation, AccessShareLock); PG_RETURN_INT64(storageId); } /* * ColumnarStorageUpdateIfNeeded - upgrade columnar storage to the current version by * using information from the metadata tables. */ void ColumnarStorageUpdateIfNeeded(Relation rel, bool isUpgrade) { if (ColumnarStorageIsCurrent(rel)) { return; } BlockNumber nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); if (nblocks < 2) { ColumnarStorageInit(RelationGetSmgr(rel), ColumnarMetadataNewStorageId()); return; } uint64 storageId = ColumnarStorageGetStorageId(rel, true); uint64 highestId; uint64 highestOffset; GetHighestUsedAddressAndId(storageId, &highestOffset, &highestId); uint64 reservedStripeId = highestId + 1; uint64 reservedOffset = highestOffset + 1; uint64 reservedRowNumber = GetHighestUsedRowNumber(storageId) + 1; ColumnarStorageUpdateCurrent(rel, isUpgrade, reservedStripeId, reservedRowNumber, reservedOffset); } /* * GetHighestUsedRowNumber returns the highest used rowNumber for given * storageId. Returns COLUMNAR_INVALID_ROW_NUMBER if storage with * storageId has no stripes. * Note that normally we would use ColumnarStorageGetReservedRowNumber * to decide that. However, this function is designed to be used when * building the metapage itself during upgrades. */ static uint64 GetHighestUsedRowNumber(uint64 storageId) { uint64 highestRowNumber = COLUMNAR_INVALID_ROW_NUMBER; List *stripeMetadataList = ReadDataFileStripeList(storageId, GetTransactionSnapshot()); StripeMetadata *stripeMetadata = NULL; foreach_declared_ptr(stripeMetadata, stripeMetadataList) { highestRowNumber = Max(highestRowNumber, StripeGetHighestRowNumber(stripeMetadata)); } return highestRowNumber; } /* * GetFirstRowNumberAttrIndexInColumnarStripe returns attrnum for first_row_number attr. * * first_row_number attr was added to table columnar.stripe using alter operation after * the version where Citus started supporting downgrades, and it's only column that we've * introduced to columnar.stripe since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_columnar_stripe and when this happens, then we know that attrnum first_row_number is * not Anum_columnar_stripe_first_row_number anymore but tupleDesc->natts - 1. */ static int GetFirstRowNumberAttrIndexInColumnarStripe(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_columnar_stripe ? (Anum_columnar_stripe_first_row_number - 1) : tupleDesc->natts - 1; } ================================================ FILE: src/backend/columnar/columnar_reader.c ================================================ /*------------------------------------------------------------------------- * * columnar_reader.c * * This file contains function definitions for reading columnar tables. This * includes the logic for reading file level metadata, reading row stripes, * and skipping unrelated row chunks and columns. * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "safe_lib.h" #include "access/nbtree.h" #include "access/xact.h" #include "catalog/pg_am.h" #include "commands/defrem.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "storage/fd.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "columnar/columnar.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" #include "distributed/listutils.h" #define UNEXPECTED_STRIPE_READ_ERR_MSG \ "attempted to read an unexpected stripe while reading columnar " \ "table %s, stripe with id=" UINT64_FORMAT " is not flushed" typedef struct ChunkGroupReadState { int64 currentRow; int64 rowCount; int columnCount; List *projectedColumnList; /* borrowed reference */ ChunkData *chunkGroupData; } ChunkGroupReadState; typedef struct StripeReadState { int columnCount; int64 rowCount; int64 currentRow; TupleDesc tupleDescriptor; Relation relation; int chunkGroupIndex; int64 chunkGroupsFiltered; MemoryContext stripeReadContext; StripeBuffers *stripeBuffers; /* allocated in stripeReadContext */ List *projectedColumnList; /* borrowed reference */ ChunkGroupReadState *chunkGroupReadState; /* owned */ } StripeReadState; struct ColumnarReadState { TupleDesc tupleDescriptor; Relation relation; StripeMetadata *currentStripeMetadata; StripeReadState *stripeReadState; /* * Integer list of attribute numbers (1-indexed) for columns needed by the * query. */ List *projectedColumnList; List *whereClauseList; List *whereClauseVars; MemoryContext stripeReadContext; int64 chunkGroupsFiltered; /* * Memory context guaranteed to be not freed during scan so we can * safely use for any memory allocations regarding ColumnarReadState * itself. */ MemoryContext scanContext; Snapshot snapshot; bool snapshotRegisteredByUs; }; /* static function declarations */ static MemoryContext CreateStripeReadMemoryContext(void); static bool ColumnarReadIsCurrentStripe(ColumnarReadState *readState, uint64 rowNumber); static StripeMetadata * ColumnarReadGetCurrentStripe(ColumnarReadState *readState); static void ReadStripeRowByRowNumber(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls); static bool StripeReadIsCurrentChunkGroup(StripeReadState *stripeReadState, int chunkGroupIndex); static void ReadChunkGroupRowByRowOffset(ChunkGroupReadState *chunkGroupReadState, StripeMetadata *stripeMetadata, uint64 stripeRowOffset, Datum *columnValues, bool *columnNulls); static bool StripeReadInProgress(ColumnarReadState *readState); static bool HasUnreadStripe(ColumnarReadState *readState); static StripeReadState * BeginStripeRead(StripeMetadata *stripeMetadata, Relation rel, TupleDesc tupleDesc, List *projectedColumnList, List *whereClauseList, List *whereClauseVars, MemoryContext stripeReadContext, Snapshot snapshot); static void AdvanceStripeRead(ColumnarReadState *readState); static bool SnapshotMightSeeUnflushedStripes(Snapshot snapshot); static bool ReadStripeNextRow(StripeReadState *stripeReadState, Datum *columnValues, bool *columnNulls); static ChunkGroupReadState * BeginChunkGroupRead(StripeBuffers *stripeBuffers, int chunkIndex, TupleDesc tupleDesc, List *projectedColumnList, MemoryContext cxt); static void EndChunkGroupRead(ChunkGroupReadState *chunkGroupReadState); static bool ReadChunkGroupNextRow(ChunkGroupReadState *chunkGroupReadState, Datum *columnValues, bool *columnNulls); static StripeBuffers * LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata, TupleDesc tupleDescriptor, List *projectedColumnList, List *whereClauseList, List *whereClauseVars, int64 *chunkGroupsFiltered, Snapshot snapshot); static ColumnBuffers * LoadColumnBuffers(Relation relation, ColumnChunkSkipNode *chunkSkipNodeArray, uint32 chunkCount, uint64 stripeOffset, Form_pg_attribute attributeForm); static bool * SelectedChunkMask(StripeSkipList *stripeSkipList, List *whereClauseList, List *whereClauseVars, int64 *chunkGroupsFiltered); static Node * BuildBaseConstraint(Var *variable); static List * GetClauseVars(List *clauses, int natts); static OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber); static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber); static void UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue); static StripeSkipList * SelectedChunkSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask, bool *selectedChunkMask); static uint32 StripeSkipListRowCount(StripeSkipList *stripeSkipList); static bool * ProjectedColumnMask(uint32 columnCount, List *projectedColumnList); static void DeserializeBoolArray(StringInfo boolArrayBuffer, bool *boolArray, uint32 boolArrayLength); static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCount, bool datumTypeByValue, int datumTypeLength, char datumTypeAlign, Datum *datumArray); static ChunkData * DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex, uint32 rowCount, TupleDesc tupleDescriptor, List *projectedColumnList); static Datum ColumnDefaultValue(TupleConstr *tupleConstraints, Form_pg_attribute attributeForm); /* * ColumnarBeginRead initializes a columnar read operation. This function returns a * read handle that's used during reading rows and finishing the read operation. * * projectedColumnList is an integer list of attribute numbers (1-indexed). */ ColumnarReadState * ColumnarBeginRead(Relation relation, TupleDesc tupleDescriptor, List *projectedColumnList, List *whereClauseList, MemoryContext scanContext, Snapshot snapshot, bool randomAccess) { /* * We allocate all stripe specific data in the stripeReadContext, and reset * this memory context before loading a new stripe. This is to avoid memory * leaks. */ MemoryContext stripeReadContext = CreateStripeReadMemoryContext(); ColumnarReadState *readState = palloc0(sizeof(ColumnarReadState)); readState->relation = relation; readState->projectedColumnList = projectedColumnList; readState->whereClauseList = whereClauseList; readState->whereClauseVars = GetClauseVars(whereClauseList, tupleDescriptor->natts); readState->chunkGroupsFiltered = 0; readState->tupleDescriptor = tupleDescriptor; readState->stripeReadContext = stripeReadContext; readState->stripeReadState = NULL; readState->scanContext = scanContext; /* * Note that ColumnarReadFlushPendingWrites might update those two by * registering a new snapshot. */ readState->snapshot = snapshot; readState->snapshotRegisteredByUs = false; if (!randomAccess) { /* * When doing random access (i.e.: index scan), we don't need to flush * pending writes until we need to read them. * columnar_index_fetch_tuple would do so when needed. */ ColumnarReadFlushPendingWrites(readState); /* * AdvanceStripeRead sets currentStripeMetadata for the first stripe * to read if not doing random access. Otherwise, reader (i.e.: * ColumnarReadRowByRowNumber) would already decide the stripe to read * on-the-fly. * * Moreover, Since we don't flush pending writes for random access, * AdvanceStripeRead might encounter with stripe metadata entries due * to current transaction's pending writes even when using an MVCC * snapshot, but AdvanceStripeRead would throw an error for that. * Note that this is not the case with for plain table scan methods * (i.e.: SeqScan and Columnar CustomScan). * * For those reasons, we don't call AdvanceStripeRead if we will do * random access. */ AdvanceStripeRead(readState); } return readState; } /* * ColumnarReadFlushPendingWrites flushes pending writes for read operation * and sets a new (registered) snapshot if necessary. * * If it sets a new snapshot, then sets snapshotRegisteredByUs to true to * indicate that caller should unregister the snapshot after finishing read * operation. * * Note that this function assumes that readState's relation and snapshot * fields are already set. */ void ColumnarReadFlushPendingWrites(ColumnarReadState *readState) { Assert(!readState->snapshotRegisteredByUs); RelFileNumber relfilenumber = readState->relation->rd_locator.relNumber; FlushWriteStateForRelfilenumber(relfilenumber, GetCurrentSubTransactionId()); if (readState->snapshot == InvalidSnapshot || !IsMVCCSnapshot(readState->snapshot)) { return; } /* * If we flushed any pending writes, then we should guarantee that * those writes are visible to us too. For this reason, if given * snapshot is an MVCC snapshot, then we set its curcid to current * command id. * * For simplicity, we do that even if we didn't flush any writes * since we don't see any problem with that. * * XXX: We should either not update cid if we are executing a FETCH * (from cursor) command, or we should have a better way to deal with * pending writes, see the discussion in * https://github.com/citusdata/citus/issues/5231. */ PushCopiedSnapshot(readState->snapshot); /* now our snapshot is the active one */ UpdateActiveSnapshotCommandId(); Snapshot newSnapshot = GetActiveSnapshot(); RegisterSnapshot(newSnapshot); /* * To be able to use UpdateActiveSnapshotCommandId, we pushed the * copied snapshot to the stack. However, we don't need to keep it * there since we will anyway rely on ColumnarReadState->snapshot * during read operation. * * Note that since we registered the snapshot already, we guarantee * that PopActiveSnapshot won't free it. */ PopActiveSnapshot(); readState->snapshot = newSnapshot; /* not forget to unregister it when finishing read operation */ readState->snapshotRegisteredByUs = true; } /* * CreateStripeReadMemoryContext creates a memory context to be used when * reading a stripe. */ static MemoryContext CreateStripeReadMemoryContext() { return AllocSetContextCreate(CurrentMemoryContext, "Stripe Read Memory Context", ALLOCSET_DEFAULT_SIZES); } /* * ColumnarReadNextRow tries to read a row from the columnar table. On success, it sets * column values, column nulls and rowNumber (if passed to be non-NULL), and returns true. * If there are no more rows to read, the function returns false. */ bool ColumnarReadNextRow(ColumnarReadState *readState, Datum *columnValues, bool *columnNulls, uint64 *rowNumber) { while (true) { if (!StripeReadInProgress(readState)) { if (!HasUnreadStripe(readState)) { return false; } readState->stripeReadState = BeginStripeRead(readState->currentStripeMetadata, readState->relation, readState->tupleDescriptor, readState->projectedColumnList, readState->whereClauseList, readState->whereClauseVars, readState->stripeReadContext, readState->snapshot); } if (!ReadStripeNextRow(readState->stripeReadState, columnValues, columnNulls)) { AdvanceStripeRead(readState); continue; } if (rowNumber) { *rowNumber = readState->currentStripeMetadata->firstRowNumber + readState->stripeReadState->currentRow - 1; } return true; } return false; } /* * ColumnarReadRowByRowNumberOrError is a wrapper around * ColumnarReadRowByRowNumber that throws an error if tuple * with rowNumber does not exist. */ void ColumnarReadRowByRowNumberOrError(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls) { if (!ColumnarReadRowByRowNumber(readState, rowNumber, columnValues, columnNulls)) { ereport(ERROR, (errmsg("cannot read from columnar table %s, tuple with " "row number " UINT64_FORMAT " does not exist", RelationGetRelationName(readState->relation), rowNumber))); } } /* * ColumnarReadRowByRowNumber reads row with rowNumber from given relation * into columnValues and columnNulls, and returns true. If no such row * exists, then returns false. */ bool ColumnarReadRowByRowNumber(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls) { if (!ColumnarReadIsCurrentStripe(readState, rowNumber)) { Relation columnarRelation = readState->relation; Snapshot snapshot = readState->snapshot; StripeMetadata *stripeMetadata = FindStripeByRowNumber(columnarRelation, rowNumber, snapshot); if (stripeMetadata == NULL) { /* no such row exists */ return false; } if (StripeWriteState(stripeMetadata) != STRIPE_WRITE_FLUSHED) { /* * Callers are expected to skip stripes that are not flushed to * disk yet or should wait for the writer xact to commit or abort, * but let's be on the safe side. */ ereport(ERROR, (errmsg(UNEXPECTED_STRIPE_READ_ERR_MSG, RelationGetRelationName(columnarRelation), stripeMetadata->id))); } /* do the cleanup before reading a new stripe */ ColumnarResetRead(readState); TupleDesc relationTupleDesc = RelationGetDescr(columnarRelation); List *whereClauseList = NIL; List *whereClauseVars = NIL; MemoryContext stripeReadContext = readState->stripeReadContext; readState->stripeReadState = BeginStripeRead(stripeMetadata, columnarRelation, relationTupleDesc, readState->projectedColumnList, whereClauseList, whereClauseVars, stripeReadContext, snapshot); readState->currentStripeMetadata = stripeMetadata; } ReadStripeRowByRowNumber(readState, rowNumber, columnValues, columnNulls); return true; } /* * ColumnarReadIsCurrentStripe returns true if stripe being read contains * row with given rowNumber. */ static bool ColumnarReadIsCurrentStripe(ColumnarReadState *readState, uint64 rowNumber) { if (!StripeReadInProgress(readState)) { return false; } StripeMetadata *currentStripeMetadata = readState->currentStripeMetadata; if (rowNumber >= currentStripeMetadata->firstRowNumber && rowNumber <= StripeGetHighestRowNumber(currentStripeMetadata)) { return true; } return false; } /* * ColumnarReadGetCurrentStripe returns StripeMetadata for the stripe that is * being read. */ static StripeMetadata * ColumnarReadGetCurrentStripe(ColumnarReadState *readState) { return readState->currentStripeMetadata; } /* * ReadStripeRowByRowNumber reads row with rowNumber from given * stripeReadState into columnValues and columnNulls. * Errors out if no such row exists in the stripe being read. */ static void ReadStripeRowByRowNumber(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls) { StripeMetadata *stripeMetadata = ColumnarReadGetCurrentStripe(readState); StripeReadState *stripeReadState = readState->stripeReadState; if (rowNumber < stripeMetadata->firstRowNumber) { /* not expected but be on the safe side */ ereport(ERROR, (errmsg("row offset cannot be negative"))); } /* find the exact chunk group to be read */ uint64 stripeRowOffset = rowNumber - stripeMetadata->firstRowNumber; int chunkGroupIndex = stripeRowOffset / stripeMetadata->chunkGroupRowCount; if (!StripeReadIsCurrentChunkGroup(stripeReadState, chunkGroupIndex)) { if (stripeReadState->chunkGroupReadState) { EndChunkGroupRead(stripeReadState->chunkGroupReadState); } stripeReadState->chunkGroupIndex = chunkGroupIndex; stripeReadState->chunkGroupReadState = BeginChunkGroupRead( stripeReadState->stripeBuffers, stripeReadState->chunkGroupIndex, stripeReadState->tupleDescriptor, stripeReadState->projectedColumnList, stripeReadState->stripeReadContext); } ReadChunkGroupRowByRowOffset(stripeReadState->chunkGroupReadState, stripeMetadata, stripeRowOffset, columnValues, columnNulls); } /* * StripeReadIsCurrentChunkGroup returns true if chunk group being read is * the has given chunkGroupIndex in its stripe. */ static bool StripeReadIsCurrentChunkGroup(StripeReadState *stripeReadState, int chunkGroupIndex) { if (!stripeReadState->chunkGroupReadState) { return false; } return (stripeReadState->chunkGroupIndex == chunkGroupIndex); } /* * ReadChunkGroupRowByRowOffset reads row with stripeRowOffset from given * chunkGroupReadState into columnValues and columnNulls. * Errors out if no such row exists in the chunk group being read. */ static void ReadChunkGroupRowByRowOffset(ChunkGroupReadState *chunkGroupReadState, StripeMetadata *stripeMetadata, uint64 stripeRowOffset, Datum *columnValues, bool *columnNulls) { /* set the exact row number to be read from given chunk roup */ chunkGroupReadState->currentRow = stripeRowOffset % stripeMetadata->chunkGroupRowCount; if (!ReadChunkGroupNextRow(chunkGroupReadState, columnValues, columnNulls)) { /* not expected but be on the safe side */ ereport(ERROR, (errmsg("could not find the row in stripe"))); } } /* * StripeReadInProgress returns true if we already started reading a stripe. */ static bool StripeReadInProgress(ColumnarReadState *readState) { return readState->stripeReadState != NULL; } /* * HasUnreadStripe returns true if we still have stripes to read during current * read operation. */ static bool HasUnreadStripe(ColumnarReadState *readState) { return readState->currentStripeMetadata != NULL; } /* * ColumnarRescan clears the position where we were scanning so that the next read starts at * the beginning again */ void ColumnarRescan(ColumnarReadState *readState, List *scanQual) { MemoryContext oldContext = MemoryContextSwitchTo(readState->scanContext); ColumnarResetRead(readState); /* set currentStripeMetadata for the first stripe to read */ AdvanceStripeRead(readState); readState->chunkGroupsFiltered = 0; readState->whereClauseList = copyObject(scanQual); MemoryContextSwitchTo(oldContext); } /* * Finishes a columnar read operation. */ void ColumnarEndRead(ColumnarReadState *readState) { if (readState->snapshotRegisteredByUs) { /* * init_columnar_read_state created a new snapshot and registered it, * so now forget it. */ UnregisterSnapshot(readState->snapshot); } MemoryContextDelete(readState->stripeReadContext); if (readState->currentStripeMetadata) { pfree(readState->currentStripeMetadata); } pfree(readState); } /* * ColumnarResetRead resets the stripe and the chunk group that is * being read currently (if any). */ void ColumnarResetRead(ColumnarReadState *readState) { if (StripeReadInProgress(readState)) { pfree(readState->currentStripeMetadata); readState->currentStripeMetadata = NULL; readState->stripeReadState = NULL; MemoryContextReset(readState->stripeReadContext); } } /* * BeginStripeRead allocates state for reading a stripe. */ static StripeReadState * BeginStripeRead(StripeMetadata *stripeMetadata, Relation rel, TupleDesc tupleDesc, List *projectedColumnList, List *whereClauseList, List *whereClauseVars, MemoryContext stripeReadContext, Snapshot snapshot) { MemoryContext oldContext = MemoryContextSwitchTo(stripeReadContext); StripeReadState *stripeReadState = palloc0(sizeof(StripeReadState)); stripeReadState->relation = rel; stripeReadState->tupleDescriptor = tupleDesc; stripeReadState->columnCount = tupleDesc->natts; stripeReadState->chunkGroupReadState = NULL; stripeReadState->projectedColumnList = projectedColumnList; stripeReadState->stripeReadContext = stripeReadContext; stripeReadState->stripeBuffers = LoadFilteredStripeBuffers(rel, stripeMetadata, tupleDesc, projectedColumnList, whereClauseList, whereClauseVars, &stripeReadState-> chunkGroupsFiltered, snapshot); stripeReadState->rowCount = stripeReadState->stripeBuffers->rowCount; MemoryContextSwitchTo(oldContext); return stripeReadState; } /* * AdvanceStripeRead updates chunkGroupsFiltered and sets * currentStripeMetadata for next stripe read. */ static void AdvanceStripeRead(ColumnarReadState *readState) { MemoryContext oldContext = MemoryContextSwitchTo(readState->scanContext); /* if not read any stripes yet, start from the first one .. */ uint64 lastReadRowNumber = COLUMNAR_INVALID_ROW_NUMBER; if (StripeReadInProgress(readState)) { /* .. otherwise, continue with the next stripe */ lastReadRowNumber = StripeGetHighestRowNumber(readState->currentStripeMetadata); readState->chunkGroupsFiltered += readState->stripeReadState->chunkGroupsFiltered; } readState->currentStripeMetadata = FindNextStripeByRowNumber(readState->relation, lastReadRowNumber, readState->snapshot); if (readState->currentStripeMetadata && StripeWriteState(readState->currentStripeMetadata) != STRIPE_WRITE_FLUSHED && !SnapshotMightSeeUnflushedStripes(readState->snapshot)) { /* * To be on the safe side, error out if we don't expect to encounter * with an un-flushed stripe. Otherwise, we will skip such stripes * until finding a flushed one. */ ereport(ERROR, (errmsg(UNEXPECTED_STRIPE_READ_ERR_MSG, RelationGetRelationName(readState->relation), readState->currentStripeMetadata->id))); } while (readState->currentStripeMetadata && StripeWriteState(readState->currentStripeMetadata) != STRIPE_WRITE_FLUSHED) { readState->currentStripeMetadata = FindNextStripeByRowNumber(readState->relation, readState->currentStripeMetadata->firstRowNumber, readState->snapshot); } readState->stripeReadState = NULL; MemoryContextReset(readState->stripeReadContext); MemoryContextSwitchTo(oldContext); } /* * SnapshotMightSeeUnflushedStripes returns true if given snapshot is * expected to see un-flushed stripes either because of other backends' * pending writes or aborted transactions. */ static bool SnapshotMightSeeUnflushedStripes(Snapshot snapshot) { if (snapshot == InvalidSnapshot) { return false; } switch (snapshot->snapshot_type) { case SNAPSHOT_ANY: case SNAPSHOT_DIRTY: case SNAPSHOT_NON_VACUUMABLE: { return true; } default: { return false; } } } /* * ReadStripeNextRow: If more rows can be read from the current stripe, fill * in non-NULL columnValues and return true. Otherwise, return false. * * On entry, all entries in columnNulls should be true; this function only * sets non-NULL entries. * */ static bool ReadStripeNextRow(StripeReadState *stripeReadState, Datum *columnValues, bool *columnNulls) { if (stripeReadState->currentRow >= stripeReadState->rowCount) { Assert(stripeReadState->currentRow == stripeReadState->rowCount); return false; } while (true) { if (stripeReadState->chunkGroupReadState == NULL) { stripeReadState->chunkGroupReadState = BeginChunkGroupRead( stripeReadState->stripeBuffers, stripeReadState-> chunkGroupIndex, stripeReadState-> tupleDescriptor, stripeReadState-> projectedColumnList, stripeReadState-> stripeReadContext); } if (!ReadChunkGroupNextRow(stripeReadState->chunkGroupReadState, columnValues, columnNulls)) { /* if this chunk group is exhausted, fetch the next one and loop */ EndChunkGroupRead(stripeReadState->chunkGroupReadState); stripeReadState->chunkGroupReadState = NULL; stripeReadState->chunkGroupIndex++; continue; } stripeReadState->currentRow++; return true; } Assert(stripeReadState->currentRow == stripeReadState->rowCount); return false; } /* * BeginChunkGroupRead allocates state for reading a chunk. */ static ChunkGroupReadState * BeginChunkGroupRead(StripeBuffers *stripeBuffers, int chunkIndex, TupleDesc tupleDesc, List *projectedColumnList, MemoryContext cxt) { uint32 chunkGroupRowCount = stripeBuffers->selectedChunkGroupRowCounts[chunkIndex]; MemoryContext oldContext = MemoryContextSwitchTo(cxt); ChunkGroupReadState *chunkGroupReadState = palloc0(sizeof(ChunkGroupReadState)); chunkGroupReadState->currentRow = 0; chunkGroupReadState->rowCount = chunkGroupRowCount; chunkGroupReadState->columnCount = tupleDesc->natts; chunkGroupReadState->projectedColumnList = projectedColumnList; chunkGroupReadState->chunkGroupData = DeserializeChunkData(stripeBuffers, chunkIndex, chunkGroupRowCount, tupleDesc, projectedColumnList); MemoryContextSwitchTo(oldContext); return chunkGroupReadState; } /* * EndChunkRead finishes a chunk read. */ static void EndChunkGroupRead(ChunkGroupReadState *chunkGroupReadState) { FreeChunkData(chunkGroupReadState->chunkGroupData); pfree(chunkGroupReadState); } /* * ReadChunkGroupNextRow: if more rows can be read from the current chunk * group, fill in non-NULL columnValues and return true. Otherwise, return * false. * * On entry, all entries in columnNulls should be true; this function only * sets non-NULL entries. */ static bool ReadChunkGroupNextRow(ChunkGroupReadState *chunkGroupReadState, Datum *columnValues, bool *columnNulls) { if (chunkGroupReadState->currentRow >= chunkGroupReadState->rowCount) { Assert(chunkGroupReadState->currentRow == chunkGroupReadState->rowCount); return false; } /* * Initialize to all-NULL. Only non-NULL projected attributes will be set. */ memset(columnNulls, true, sizeof(bool) * chunkGroupReadState->columnCount); int attno; foreach_declared_int(attno, chunkGroupReadState->projectedColumnList) { const ChunkData *chunkGroupData = chunkGroupReadState->chunkGroupData; const int rowIndex = chunkGroupReadState->currentRow; /* attno is 1-indexed; existsArray is 0-indexed */ const uint32 columnIndex = attno - 1; if (chunkGroupData->existsArray[columnIndex][rowIndex]) { columnValues[columnIndex] = chunkGroupData->valueArray[columnIndex][rowIndex]; columnNulls[columnIndex] = false; } } chunkGroupReadState->currentRow++; return true; } /* * ColumnarReadChunkGroupsFiltered * * Return the number of chunk groups filtered during this read operation. */ int64 ColumnarReadChunkGroupsFiltered(ColumnarReadState *state) { return state->chunkGroupsFiltered; } /* * CreateEmptyChunkDataArray creates data buffers to keep deserialized exist and * value arrays for requested columns in columnMask. */ ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask, uint32 chunkGroupRowCount) { uint32 columnIndex = 0; ChunkData *chunkData = palloc0(sizeof(ChunkData)); chunkData->existsArray = palloc0(columnCount * sizeof(bool *)); chunkData->valueArray = palloc0(columnCount * sizeof(Datum *)); chunkData->valueBufferArray = palloc0(columnCount * sizeof(StringInfo)); chunkData->columnCount = columnCount; chunkData->rowCount = chunkGroupRowCount; /* allocate chunk memory for deserialized data */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { if (columnMask[columnIndex]) { chunkData->existsArray[columnIndex] = palloc0(chunkGroupRowCount * sizeof(bool)); chunkData->valueArray[columnIndex] = palloc0(chunkGroupRowCount * sizeof(Datum)); chunkData->valueBufferArray[columnIndex] = NULL; } } return chunkData; } /* * FreeChunkData deallocates data buffers to keep deserialized exist and * value arrays for requested columns in columnMask. * ColumnChunkData->serializedValueBuffer lives in memory read/write context * so it is deallocated automatically when the context is deleted. */ void FreeChunkData(ChunkData *chunkData) { uint32 columnIndex = 0; if (chunkData == NULL) { return; } for (columnIndex = 0; columnIndex < chunkData->columnCount; columnIndex++) { if (chunkData->existsArray[columnIndex] != NULL) { pfree(chunkData->existsArray[columnIndex]); } if (chunkData->valueArray[columnIndex] != NULL) { pfree(chunkData->valueArray[columnIndex]); } } pfree(chunkData->existsArray); pfree(chunkData->valueArray); pfree(chunkData); } /* ColumnarTableRowCount returns the exact row count of a table using skiplists */ uint64 ColumnarTableRowCount(Relation relation) { ListCell *stripeMetadataCell = NULL; uint64 totalRowCount = 0; List *stripeList = StripesForRelfilelocator(relation); foreach(stripeMetadataCell, stripeList) { StripeMetadata *stripeMetadata = (StripeMetadata *) lfirst(stripeMetadataCell); totalRowCount += stripeMetadata->rowCount; } return totalRowCount; } /* * LoadFilteredStripeBuffers reads serialized stripe data from the given file. * The function skips over chunks whose rows are refuted by restriction qualifiers, * and only loads columns that are projected in the query. */ static StripeBuffers * LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata, TupleDesc tupleDescriptor, List *projectedColumnList, List *whereClauseList, List *whereClauseVars, int64 *chunkGroupsFiltered, Snapshot snapshot) { uint32 columnIndex = 0; uint32 columnCount = tupleDescriptor->natts; bool *projectedColumnMask = ProjectedColumnMask(columnCount, projectedColumnList); StripeSkipList *stripeSkipList = ReadStripeSkipList(relation, stripeMetadata->id, tupleDescriptor, stripeMetadata->chunkCount, snapshot); bool *selectedChunkMask = SelectedChunkMask(stripeSkipList, whereClauseList, whereClauseVars, chunkGroupsFiltered); StripeSkipList *selectedChunkSkipList = SelectedChunkSkipList(stripeSkipList, projectedColumnMask, selectedChunkMask); /* load column data for projected columns */ ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *)); for (columnIndex = 0; columnIndex < stripeMetadata->columnCount; columnIndex++) { if (projectedColumnMask[columnIndex]) { ColumnChunkSkipNode *chunkSkipNode = selectedChunkSkipList->chunkSkipNodeArray[columnIndex]; Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); uint32 chunkCount = selectedChunkSkipList->chunkCount; ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, chunkSkipNode, chunkCount, stripeMetadata->fileOffset, attributeForm); columnBuffersArray[columnIndex] = columnBuffers; } } StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers)); stripeBuffers->columnCount = columnCount; stripeBuffers->rowCount = StripeSkipListRowCount(selectedChunkSkipList); stripeBuffers->columnBuffersArray = columnBuffersArray; stripeBuffers->selectedChunkGroupRowCounts = selectedChunkSkipList->chunkGroupRowCounts; return stripeBuffers; } /* * LoadColumnBuffers reads serialized column data from the given file. These * column data are laid out as sequential chunks in the file; and chunk positions * and lengths are retrieved from the column chunk skip node array. */ static ColumnBuffers * LoadColumnBuffers(Relation relation, ColumnChunkSkipNode *chunkSkipNodeArray, uint32 chunkCount, uint64 stripeOffset, Form_pg_attribute attributeForm) { uint32 chunkIndex = 0; ColumnChunkBuffers **chunkBuffersArray = palloc0(chunkCount * sizeof(ColumnChunkBuffers *)); for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers)); } /* * We first read the "exists" chunks. We don't read "values" array here, * because "exists" chunks are stored sequentially on disk, and we want to * minimize disk seeks. */ for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex]; uint64 existsOffset = stripeOffset + chunkSkipNode->existsChunkOffset; StringInfo rawExistsBuffer = makeStringInfo(); enlargeStringInfo(rawExistsBuffer, chunkSkipNode->existsLength); rawExistsBuffer->len = chunkSkipNode->existsLength; ColumnarStorageRead(relation, existsOffset, rawExistsBuffer->data, chunkSkipNode->existsLength); chunkBuffersArray[chunkIndex]->existsBuffer = rawExistsBuffer; } /* then read "values" chunks, which are also stored sequentially on disk */ for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex]; CompressionType compressionType = chunkSkipNode->valueCompressionType; uint64 valueOffset = stripeOffset + chunkSkipNode->valueChunkOffset; StringInfo rawValueBuffer = makeStringInfo(); enlargeStringInfo(rawValueBuffer, chunkSkipNode->valueLength); rawValueBuffer->len = chunkSkipNode->valueLength; ColumnarStorageRead(relation, valueOffset, rawValueBuffer->data, chunkSkipNode->valueLength); chunkBuffersArray[chunkIndex]->valueBuffer = rawValueBuffer; chunkBuffersArray[chunkIndex]->valueCompressionType = compressionType; chunkBuffersArray[chunkIndex]->decompressedValueSize = chunkSkipNode->decompressedValueSize; } ColumnBuffers *columnBuffers = palloc0(sizeof(ColumnBuffers)); columnBuffers->chunkBuffersArray = chunkBuffersArray; return columnBuffers; } /* * SelectedChunkMask walks over each column's chunks and checks if a chunk can * be filtered without reading its data. The filtering happens when all rows in * the chunk can be refuted by the given qualifier conditions. */ static bool * SelectedChunkMask(StripeSkipList *stripeSkipList, List *whereClauseList, List *whereClauseVars, int64 *chunkGroupsFiltered) { ListCell *columnCell = NULL; uint32 chunkIndex = 0; bool *selectedChunkMask = palloc0(stripeSkipList->chunkCount * sizeof(bool)); memset(selectedChunkMask, true, stripeSkipList->chunkCount * sizeof(bool)); foreach(columnCell, whereClauseVars) { Var *column = lfirst(columnCell); uint32 columnIndex = column->varattno - 1; /* if this column's data type doesn't have a comparator, skip it */ FmgrInfo *comparisonFunction = GetFunctionInfoOrNull(column->vartype, BTREE_AM_OID, BTORDER_PROC); if (comparisonFunction == NULL) { continue; } Node *baseConstraint = BuildBaseConstraint(column); for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { ColumnChunkSkipNode *chunkSkipNodeArray = stripeSkipList->chunkSkipNodeArray[columnIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex]; /* * A column chunk with comparable data type can miss min/max values * if all values in the chunk are NULL. */ if (!chunkSkipNode->hasMinMax) { continue; } UpdateConstraint(baseConstraint, chunkSkipNode->minimumValue, chunkSkipNode->maximumValue); List *constraintList = list_make1(baseConstraint); bool predicateRefuted = predicate_refuted_by(constraintList, whereClauseList, false); if (predicateRefuted && selectedChunkMask[chunkIndex]) { selectedChunkMask[chunkIndex] = false; *chunkGroupsFiltered += 1; } } } return selectedChunkMask; } /* * GetFunctionInfoOrNull first resolves the operator for the given data type, * access method, and support procedure. The function then uses the resolved * operator's identifier to fill in a function manager object, and returns * this object. This function is based on a similar function from CitusDB's code. */ FmgrInfo * GetFunctionInfoOrNull(Oid typeId, Oid accessMethodId, int16 procedureId) { FmgrInfo *functionInfo = NULL; /* get default operator class from pg_opclass for datum type */ Oid operatorClassId = GetDefaultOpClass(typeId, accessMethodId); if (operatorClassId == InvalidOid) { return NULL; } Oid operatorFamilyId = get_opclass_family(operatorClassId); if (operatorFamilyId == InvalidOid) { return NULL; } Oid operatorId = get_opfamily_proc(operatorFamilyId, typeId, typeId, procedureId); if (operatorId != InvalidOid) { functionInfo = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); /* fill in the FmgrInfo struct using the operatorId */ fmgr_info(operatorId, functionInfo); } return functionInfo; } /* * BuildBaseConstraint builds and returns a base constraint. This constraint * implements an expression in the form of (var <= max && var >= min), where * min and max values represent a chunk's min and max values. These chunk * values are filled in after the constraint is built. This function is based * on a similar function from CitusDB's shard pruning logic. */ static Node * BuildBaseConstraint(Var *variable) { OpExpr *lessThanExpr = MakeOpExpression(variable, BTLessEqualStrategyNumber); OpExpr *greaterThanExpr = MakeOpExpression(variable, BTGreaterEqualStrategyNumber); Node *baseConstraint = make_and_qual((Node *) lessThanExpr, (Node *) greaterThanExpr); return baseConstraint; } /* * GetClauseVars extracts the Vars from the given clauses for the purpose of * building constraints that can be refuted by predicate_refuted_by(). It also * deduplicates and sorts them. */ static List * GetClauseVars(List *whereClauseList, int natts) { /* * We don't recurse into or include aggregates, window functions, or * PHVs. We don't expect any PHVs during execution; and Vars found inside * an aggregate or window function aren't going to be useful in forming * constraints that can be refuted. */ int flags = 0; List *vars = pull_var_clause((Node *) whereClauseList, flags); Var **deduplicate = palloc0(sizeof(Var *) * natts); ListCell *lc; foreach(lc, vars) { Node *node = lfirst(lc); Assert(IsA(node, Var)); Var *var = (Var *) node; int idx = var->varattno - 1; if (deduplicate[idx] != NULL) { /* if they have the same varattno, the rest should be identical */ Assert(equal(var, deduplicate[idx])); } deduplicate[idx] = var; } List *whereClauseVars = NIL; for (int i = 0; i < natts; i++) { Var *var = deduplicate[i]; if (var != NULL) { whereClauseVars = lappend(whereClauseVars, var); } } pfree(deduplicate); return whereClauseVars; } /* * MakeOpExpression builds an operator expression node. This operator expression * implements the operator clause as defined by the variable and the strategy * number. The function is copied from CitusDB's shard pruning logic. */ static OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber) { Oid typeId = variable->vartype; Oid typeModId = variable->vartypmod; Oid collationId = variable->varcollid; Oid accessMethodId = BTREE_AM_OID; /* Load the operator from system catalogs */ Oid operatorId = GetOperatorByType(typeId, accessMethodId, strategyNumber); Const *constantValue = makeNullConst(typeId, typeModId, collationId); /* Now make the expression with the given variable and a null constant */ OpExpr *expression = (OpExpr *) make_opclause(operatorId, InvalidOid, /* no result type yet */ false, /* no return set */ (Expr *) variable, (Expr *) constantValue, InvalidOid, collationId); /* Set implementing function id and result type */ expression->opfuncid = get_opcode(operatorId); expression->opresulttype = get_func_rettype(expression->opfuncid); return expression; } /* * GetOperatorByType returns operator Oid for the given type, access method, * and strategy number. Note that this function incorrectly errors out when * the given type doesn't have its own operator but can use another compatible * type's default operator. The function is copied from CitusDB's shard pruning * logic. */ static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber) { /* Get default operator class from pg_opclass */ Oid operatorClassId = GetDefaultOpClass(typeId, accessMethodId); Oid operatorFamily = get_opclass_family(operatorClassId); Oid operatorId = get_opfamily_member(operatorFamily, typeId, typeId, strategyNumber); return operatorId; } /* * UpdateConstraint updates the base constraint with the given min/max values. * The function is copied from CitusDB's shard pruning logic. */ static void UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue) { BoolExpr *andExpr = (BoolExpr *) baseConstraint; Node *lessThanExpr = (Node *) linitial(andExpr->args); Node *greaterThanExpr = (Node *) lsecond(andExpr->args); Node *minNode = get_rightop((Expr *) greaterThanExpr); Node *maxNode = get_rightop((Expr *) lessThanExpr); Assert(IsA(minNode, Const)); Assert(IsA(maxNode, Const)); Const *minConstant = (Const *) minNode; Const *maxConstant = (Const *) maxNode; minConstant->constvalue = minValue; maxConstant->constvalue = maxValue; minConstant->constisnull = false; maxConstant->constisnull = false; minConstant->constbyval = true; maxConstant->constbyval = true; } /* * SelectedChunkSkipList constructs a new StripeSkipList in which the * non-selected chunks are removed from the given stripeSkipList. */ static StripeSkipList * SelectedChunkSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask, bool *selectedChunkMask) { uint32 selectedChunkCount = 0; uint32 chunkIndex = 0; uint32 columnIndex = 0; uint32 columnCount = stripeSkipList->columnCount; uint32 selectedChunkIndex = 0; for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { if (selectedChunkMask[chunkIndex]) { selectedChunkCount++; } } ColumnChunkSkipNode **selectedChunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *)); for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { bool firstColumn = columnIndex == 0; selectedChunkIndex = 0; /* first column's chunk skip node is always read */ if (!projectedColumnMask[columnIndex] && !firstColumn) { selectedChunkSkipNodeArray[columnIndex] = NULL; continue; } Assert(stripeSkipList->chunkSkipNodeArray[columnIndex] != NULL); selectedChunkSkipNodeArray[columnIndex] = palloc0(selectedChunkCount * sizeof(ColumnChunkSkipNode)); for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { if (selectedChunkMask[chunkIndex]) { selectedChunkSkipNodeArray[columnIndex][selectedChunkIndex] = stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex]; selectedChunkIndex++; } } } selectedChunkIndex = 0; uint32 *chunkGroupRowCounts = palloc0(selectedChunkCount * sizeof(uint32)); for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { if (selectedChunkMask[chunkIndex]) { chunkGroupRowCounts[selectedChunkIndex++] = stripeSkipList->chunkGroupRowCounts[chunkIndex]; } } StripeSkipList *selectedChunkSkipList = palloc0(sizeof(StripeSkipList)); selectedChunkSkipList->chunkSkipNodeArray = selectedChunkSkipNodeArray; selectedChunkSkipList->chunkCount = selectedChunkCount; selectedChunkSkipList->columnCount = stripeSkipList->columnCount; selectedChunkSkipList->chunkGroupRowCounts = chunkGroupRowCounts; return selectedChunkSkipList; } /* * StripeSkipListRowCount counts the number of rows in the given stripeSkipList. * To do this, the function finds the first column, and sums up row counts across * all chunks for that column. */ static uint32 StripeSkipListRowCount(StripeSkipList *stripeSkipList) { uint32 stripeSkipListRowCount = 0; uint32 chunkIndex = 0; uint32 *chunkGroupRowCounts = stripeSkipList->chunkGroupRowCounts; for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { uint32 chunkGroupRowCount = chunkGroupRowCounts[chunkIndex]; stripeSkipListRowCount += chunkGroupRowCount; } return stripeSkipListRowCount; } /* * ProjectedColumnMask returns a boolean array in which the projected columns * from the projected column list are marked as true. */ static bool * ProjectedColumnMask(uint32 columnCount, List *projectedColumnList) { bool *projectedColumnMask = palloc0(columnCount * sizeof(bool)); int attno; foreach_declared_int(attno, projectedColumnList) { /* attno is 1-indexed; projectedColumnMask is 0-indexed */ int columnIndex = attno - 1; projectedColumnMask[columnIndex] = true; } return projectedColumnMask; } /* * DeserializeBoolArray reads an array of bits from the given buffer and stores * it in provided bool array. */ static void DeserializeBoolArray(StringInfo boolArrayBuffer, bool *boolArray, uint32 boolArrayLength) { uint32 boolArrayIndex = 0; uint32 maximumBoolCount = boolArrayBuffer->len * 8; if (boolArrayLength > maximumBoolCount) { ereport(ERROR, (errmsg("insufficient data for reading boolean array"))); } for (boolArrayIndex = 0; boolArrayIndex < boolArrayLength; boolArrayIndex++) { uint32 byteIndex = boolArrayIndex / 8; uint32 bitIndex = boolArrayIndex % 8; uint8 bitmask = (1 << bitIndex); uint8 shiftedBit = (boolArrayBuffer->data[byteIndex] & bitmask); if (shiftedBit == 0) { boolArray[boolArrayIndex] = false; } else { boolArray[boolArrayIndex] = true; } } } /* * DeserializeDatumArray reads an array of datums from the given buffer and stores * them in provided datumArray. If a value is marked as false in the exists array, * the function assumes that the datum isn't in the buffer, and simply skips it. */ static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCount, bool datumTypeByValue, int datumTypeLength, char datumTypeAlign, Datum *datumArray) { uint32 datumIndex = 0; uint32 currentDatumDataOffset = 0; for (datumIndex = 0; datumIndex < datumCount; datumIndex++) { if (!existsArray[datumIndex]) { continue; } char *currentDatumDataPointer = datumBuffer->data + currentDatumDataOffset; datumArray[datumIndex] = fetch_att(currentDatumDataPointer, datumTypeByValue, datumTypeLength); currentDatumDataOffset = att_addlength_datum(currentDatumDataOffset, datumTypeLength, datumArray[datumIndex]); currentDatumDataOffset = att_align_nominal(currentDatumDataOffset, datumTypeAlign); if (currentDatumDataOffset > datumBuffer->len) { ereport(ERROR, (errmsg("insufficient data left in datum buffer"))); } } } /* * DeserializeChunkGroupData deserializes requested data chunk for all columns and * stores in chunkDataArray. It uncompresses serialized data if necessary. The * function also deallocates data buffers used for previous chunk, and compressed * data buffers for the current chunk which will not be needed again. If a column * data is not present serialized buffer, then default value (or null) is used * to fill value array. */ static ChunkData * DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex, uint32 rowCount, TupleDesc tupleDescriptor, List *projectedColumnList) { int columnIndex = 0; bool *columnMask = ProjectedColumnMask(tupleDescriptor->natts, projectedColumnList); ChunkData *chunkData = CreateEmptyChunkData(tupleDescriptor->natts, columnMask, rowCount); for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; bool columnAdded = false; if (columnBuffers == NULL && columnMask[columnIndex]) { columnAdded = true; } if (columnBuffers != NULL) { ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; /* decompress and deserialize current chunk's data */ StringInfo valueBuffer = DecompressBuffer(chunkBuffers->valueBuffer, chunkBuffers->valueCompressionType, chunkBuffers->decompressedValueSize); DeserializeBoolArray(chunkBuffers->existsBuffer, chunkData->existsArray[columnIndex], rowCount); DeserializeDatumArray(valueBuffer, chunkData->existsArray[columnIndex], rowCount, attributeForm->attbyval, attributeForm->attlen, attributeForm->attalign, chunkData->valueArray[columnIndex]); /* store current chunk's data buffer to be freed at next chunk read */ chunkData->valueBufferArray[columnIndex] = valueBuffer; } else if (columnAdded) { /* * This is a column that was added after creation of this stripe. * So we use either the default value or NULL. */ if (attributeForm->atthasdef) { int rowIndex = 0; Datum defaultValue = ColumnDefaultValue(tupleDescriptor->constr, attributeForm); for (rowIndex = 0; rowIndex < rowCount; rowIndex++) { chunkData->existsArray[columnIndex][rowIndex] = true; chunkData->valueArray[columnIndex][rowIndex] = defaultValue; } } else { memset(chunkData->existsArray[columnIndex], false, rowCount * sizeof(bool)); } } } return chunkData; } /* * ColumnDefaultValue returns default value for given column. Only const values * are supported. The function errors on any other default value expressions. */ static Datum ColumnDefaultValue(TupleConstr *tupleConstraints, Form_pg_attribute attributeForm) { Node *defaultValueNode = NULL; int defValIndex = 0; for (defValIndex = 0; defValIndex < tupleConstraints->num_defval; defValIndex++) { AttrDefault attrDefault = tupleConstraints->defval[defValIndex]; if (attrDefault.adnum == attributeForm->attnum) { defaultValueNode = stringToNode(attrDefault.adbin); break; } } Assert(defaultValueNode != NULL); /* try reducing the default value node to a const node */ defaultValueNode = eval_const_expressions(NULL, defaultValueNode); if (IsA(defaultValueNode, Const)) { Const *constNode = (Const *) defaultValueNode; return constNode->constvalue; } else { const char *columnName = NameStr(attributeForm->attname); ereport(ERROR, (errmsg("unsupported default value for column \"%s\"", columnName), errhint("Expression is either mutable or " "does not evaluate to constant value"))); } } ================================================ FILE: src/backend/columnar/columnar_storage.c ================================================ /*------------------------------------------------------------------------- * * columnar_storage.c * * Copyright (c) Citus Data, Inc. * * Low-level storage layer for columnar. * - Translates columnar read/write operations on logical offsets into operations on pages/blocks. * - Emits WAL. * - Reads/writes the columnar metapage. * - Reserves data offsets, stripe numbers, and row offsets. * - Truncation. * * Higher-level columnar operations deal with logical offsets and large * contiguous buffers of data that need to be stored. But the buffer manager * and WAL depend on formatted pages with headers, so these large buffers need * to be written across many pages. This module translates the contiguous * buffers into individual block reads/writes, and performs WAL when * necessary. * * Storage layout: a metapage in block 0, followed by an empty page in block * 1, followed by logical data starting at the first byte after the page * header in block 2 (having logical offset ColumnarFirstLogicalOffset). (XXX: * Block 1 is left empty for no particular reason. Reconsider?). A columnar * table should always have at least 2 blocks. * * Reservation is done with a relation extension lock, and designed for * concurrency, so the callers only need an ordinary lock on the * relation. Initializing the metapage or truncating the relation require that * the caller holds an AccessExclusiveLock. (XXX: New reservations of data are * aligned onto a new page for no particular reason. Reconsider?). * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "safe_lib.h" #include "access/generic_xlog.h" #include "catalog/storage.h" #include "storage/bufmgr.h" #include "storage/lmgr.h" #include "pg_version_compat.h" #include "columnar/columnar.h" #include "columnar/columnar_storage.h" /* * Content of the first page in main fork, which stores metadata at file * level. */ typedef struct ColumnarMetapage { /* * Store version of file format used, so we can detect files from * previous versions if we change file format. */ uint32 versionMajor; uint32 versionMinor; /* * Each of the metadata table rows are identified by a storageId. * We store it also in the main fork so we can link metadata rows * with data files. */ uint64 storageId; uint64 reservedStripeId; /* first unused stripe id */ uint64 reservedRowNumber; /* first unused row number */ uint64 reservedOffset; /* first unused byte offset */ /* * Flag set to true in the init fork. After an unlogged table reset (due * to a crash), the init fork will be copied over the main fork. When * trying to read an unlogged table, if this flag is set to true, we must * clear the metadata for the table (because the actual data is gone, * too), and clear the flag. We can cross-check that the table is * UNLOGGED, and that the main fork is at the minimum size (no actual * data). * * XXX: Not used yet; reserved field for later support for UNLOGGED. */ bool unloggedReset; } ColumnarMetapage; /* represents a "physical" block+offset address */ typedef struct PhysicalAddr { BlockNumber blockno; uint32 offset; } PhysicalAddr; #define COLUMNAR_METAPAGE_BLOCKNO 0 #define COLUMNAR_EMPTY_BLOCKNO 1 #define COLUMNAR_INVALID_STRIPE_ID 0 #define COLUMNAR_FIRST_STRIPE_ID 1 #define OLD_METAPAGE_VERSION_HINT "Use \"VACUUM\" to upgrade the columnar table format " \ "version or run \"ALTER EXTENSION citus UPDATE\"." /* only for testing purposes */ PG_FUNCTION_INFO_V1(test_columnar_storage_write_new_page); /* * Map logical offsets to a physical page and offset where the data is kept. */ static inline PhysicalAddr LogicalToPhysical(uint64 logicalOffset) { PhysicalAddr addr; addr.blockno = logicalOffset / COLUMNAR_BYTES_PER_PAGE; addr.offset = SizeOfPageHeaderData + (logicalOffset % COLUMNAR_BYTES_PER_PAGE); return addr; } /* * Map a physical page and offset address to a logical address. */ static inline uint64 PhysicalToLogical(PhysicalAddr addr) { return COLUMNAR_BYTES_PER_PAGE * addr.blockno + addr.offset - SizeOfPageHeaderData; } static void ColumnarOverwriteMetapage(Relation relation, ColumnarMetapage columnarMetapage); static ColumnarMetapage ColumnarMetapageRead(Relation rel, bool force); static void ReadFromBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf, uint32 len, bool force); static void WriteToBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf, uint32 len, bool clear); static uint64 AlignReservation(uint64 prevReservation); static bool ColumnarMetapageIsCurrent(ColumnarMetapage *metapage); static bool ColumnarMetapageIsOlder(ColumnarMetapage *metapage); static bool ColumnarMetapageIsNewer(ColumnarMetapage *metapage); static void ColumnarMetapageCheckVersion(Relation rel, ColumnarMetapage *metapage); /* * ColumnarStorageInit - initialize a new metapage in an empty relation * with the given storageId. * * Caller must hold AccessExclusiveLock on the relation. */ void ColumnarStorageInit(SMgrRelation srel, uint64 storageId) { BlockNumber nblocks = smgrnblocks(srel, MAIN_FORKNUM); if (nblocks > 0) { elog(ERROR, "attempted to initialize metapage, but %d pages already exist", nblocks); } /* create two pages */ PGIOAlignedBlock block; Page page = block.data; /* write metapage */ PageInit(page, BLCKSZ, 0); PageHeader phdr = (PageHeader) page; ColumnarMetapage metapage = { 0 }; metapage.storageId = storageId; metapage.versionMajor = COLUMNAR_VERSION_MAJOR; metapage.versionMinor = COLUMNAR_VERSION_MINOR; metapage.reservedStripeId = COLUMNAR_FIRST_STRIPE_ID; metapage.reservedRowNumber = COLUMNAR_FIRST_ROW_NUMBER; metapage.reservedOffset = ColumnarFirstLogicalOffset; metapage.unloggedReset = false; memcpy_s(page + phdr->pd_lower, phdr->pd_upper - phdr->pd_lower, (char *) &metapage, sizeof(ColumnarMetapage)); phdr->pd_lower += sizeof(ColumnarMetapage); log_newpage(&srel->smgr_rlocator.locator, MAIN_FORKNUM, COLUMNAR_METAPAGE_BLOCKNO, page, true); PageSetChecksumInplace(page, COLUMNAR_METAPAGE_BLOCKNO); smgrextend(srel, MAIN_FORKNUM, COLUMNAR_METAPAGE_BLOCKNO, page, true); /* write empty page */ PageInit(page, BLCKSZ, 0); log_newpage(&srel->smgr_rlocator.locator, MAIN_FORKNUM, COLUMNAR_EMPTY_BLOCKNO, page, true); PageSetChecksumInplace(page, COLUMNAR_EMPTY_BLOCKNO); smgrextend(srel, MAIN_FORKNUM, COLUMNAR_EMPTY_BLOCKNO, page, true); /* * An immediate sync is required even if we xlog'd the page, because the * write did not go through shared_buffers and therefore a concurrent * checkpoint may have moved the redo pointer past our xlog record. */ smgrimmedsync(srel, MAIN_FORKNUM); } /* * ColumnarStorageUpdateCurrent - update the metapage to the current * version. No effect if the version already matches. If 'upgrade' is true, * throw an error if metapage version is newer; if 'upgrade' is false, it's a * downgrade, so throw an error if the metapage version is older. * * NB: caller must ensure that metapage already exists, which might not be the * case on 10.0. */ void ColumnarStorageUpdateCurrent(Relation rel, bool upgrade, uint64 reservedStripeId, uint64 reservedRowNumber, uint64 reservedOffset) { LockRelationForExtension(rel, ExclusiveLock); ColumnarMetapage metapage = ColumnarMetapageRead(rel, true); if (ColumnarMetapageIsCurrent(&metapage)) { /* nothing to do */ return; } if (upgrade && ColumnarMetapageIsNewer(&metapage)) { elog(ERROR, "found newer columnar metapage while upgrading"); } if (!upgrade && ColumnarMetapageIsOlder(&metapage)) { elog(ERROR, "found older columnar metapage while downgrading"); } metapage.versionMajor = COLUMNAR_VERSION_MAJOR; metapage.versionMinor = COLUMNAR_VERSION_MINOR; /* storageId remains the same */ metapage.reservedStripeId = reservedStripeId; metapage.reservedRowNumber = reservedRowNumber; metapage.reservedOffset = reservedOffset; ColumnarOverwriteMetapage(rel, metapage); UnlockRelationForExtension(rel, ExclusiveLock); } /* * ColumnarStorageGetVersionMajor - return major version from the metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetVersionMajor(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.versionMajor; } /* * ColumnarStorageGetVersionMinor - return minor version from the metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetVersionMinor(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.versionMinor; } /* * ColumnarStorageGetStorageId - return storage ID from the metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetStorageId(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.storageId; } /* * ColumnarStorageGetReservedStripeId - return reserved stripe ID from the * metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetReservedStripeId(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.reservedStripeId; } /* * ColumnarStorageGetReservedRowNumber - return reserved row number from the * metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetReservedRowNumber(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.reservedRowNumber; } /* * ColumnarStorageGetReservedOffset - return reserved offset from the metapage. * * Throw an error if the metapage is not the current version, unless * 'force' is true. */ uint64 ColumnarStorageGetReservedOffset(Relation rel, bool force) { ColumnarMetapage metapage = ColumnarMetapageRead(rel, force); return metapage.reservedOffset; } /* * ColumnarStorageIsCurrent - return true if metapage exists and is not * the current version. */ bool ColumnarStorageIsCurrent(Relation rel) { BlockNumber nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); if (nblocks < 2) { return false; } ColumnarMetapage metapage = ColumnarMetapageRead(rel, true); return ColumnarMetapageIsCurrent(&metapage); } /* * ColumnarStorageReserveRowNumber returns reservedRowNumber and advances * it for next row number reservation. */ uint64 ColumnarStorageReserveRowNumber(Relation rel, uint64 nrows) { LockRelationForExtension(rel, ExclusiveLock); ColumnarMetapage metapage = ColumnarMetapageRead(rel, false); uint64 firstRowNumber = metapage.reservedRowNumber; metapage.reservedRowNumber += nrows; ColumnarOverwriteMetapage(rel, metapage); UnlockRelationForExtension(rel, ExclusiveLock); return firstRowNumber; } /* * ColumnarStorageReserveStripeId returns stripeId and advances it for next * stripeId reservation. * Note that this function doesn't handle row number reservation. * See ColumnarStorageReserveRowNumber function. */ uint64 ColumnarStorageReserveStripeId(Relation rel) { LockRelationForExtension(rel, ExclusiveLock); ColumnarMetapage metapage = ColumnarMetapageRead(rel, false); uint64 stripeId = metapage.reservedStripeId; metapage.reservedStripeId++; ColumnarOverwriteMetapage(rel, metapage); UnlockRelationForExtension(rel, ExclusiveLock); return stripeId; } /* * ColumnarStorageReserveData - reserve logical data offsets for writing. */ uint64 ColumnarStorageReserveData(Relation rel, uint64 amount) { if (amount == 0) { return ColumnarInvalidLogicalOffset; } LockRelationForExtension(rel, ExclusiveLock); ColumnarMetapage metapage = ColumnarMetapageRead(rel, false); uint64 alignedReservation = AlignReservation(metapage.reservedOffset); uint64 nextReservation = alignedReservation + amount; metapage.reservedOffset = nextReservation; /* write new reservation */ ColumnarOverwriteMetapage(rel, metapage); /* last used PhysicalAddr of new reservation */ PhysicalAddr final = LogicalToPhysical(nextReservation - 1); /* extend with new pages */ BlockNumber nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); while (nblocks <= final.blockno) { Buffer newBuffer = ReadBuffer(rel, P_NEW); Assert(BufferGetBlockNumber(newBuffer) == nblocks); ReleaseBuffer(newBuffer); nblocks++; } UnlockRelationForExtension(rel, ExclusiveLock); return alignedReservation; } /* * ColumnarStorageRead - map the logical offset to a block and offset, then * read the buffer from multiple blocks if necessary. */ void ColumnarStorageRead(Relation rel, uint64 logicalOffset, char *data, uint32 amount) { /* if there's no work to do, succeed even with invalid offset */ if (amount == 0) { return; } if (!ColumnarLogicalOffsetIsValid(logicalOffset)) { elog(ERROR, "attempted columnar read on relation %d from invalid logical offset: " UINT64_FORMAT, rel->rd_id, logicalOffset); } uint64 read = 0; while (read < amount) { PhysicalAddr addr = LogicalToPhysical(logicalOffset + read); uint32 to_read = Min(amount - read, BLCKSZ - addr.offset); ReadFromBlock(rel, addr.blockno, addr.offset, data + read, to_read, false); read += to_read; } } /* * ColumnarStorageWrite - map the logical offset to a block and offset, then * write the buffer across multiple blocks if necessary. */ void ColumnarStorageWrite(Relation rel, uint64 logicalOffset, char *data, uint32 amount) { /* if there's no work to do, succeed even with invalid offset */ if (amount == 0) { return; } if (!ColumnarLogicalOffsetIsValid(logicalOffset)) { elog(ERROR, "attempted columnar write on relation %d to invalid logical offset: " UINT64_FORMAT, rel->rd_id, logicalOffset); } uint64 written = 0; while (written < amount) { PhysicalAddr addr = LogicalToPhysical(logicalOffset + written); uint64 to_write = Min(amount - written, BLCKSZ - addr.offset); WriteToBlock(rel, addr.blockno, addr.offset, data + written, to_write, false); written += to_write; } } /* * ColumnarStorageTruncate - truncate the columnar storage such that * newDataReservation will be the first unused logical offset available. Free * pages at the end of the relation. * * Caller must hold AccessExclusiveLock on the relation. * * Returns true if pages were truncated; false otherwise. */ bool ColumnarStorageTruncate(Relation rel, uint64 newDataReservation) { if (!ColumnarLogicalOffsetIsValid(newDataReservation)) { elog(ERROR, "attempted to truncate relation %d to " "invalid logical offset: " UINT64_FORMAT, rel->rd_id, newDataReservation); } BlockNumber old_rel_pages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); if (old_rel_pages == 0) { /* nothing to do */ return false; } LockRelationForExtension(rel, ExclusiveLock); ColumnarMetapage metapage = ColumnarMetapageRead(rel, false); if (metapage.reservedOffset < newDataReservation) { elog(ERROR, "attempted to truncate relation %d to offset " UINT64_FORMAT \ " which is higher than existing offset " UINT64_FORMAT, rel->rd_id, newDataReservation, metapage.reservedOffset); } if (metapage.reservedOffset == newDataReservation) { /* nothing to do */ UnlockRelationForExtension(rel, ExclusiveLock); return false; } metapage.reservedOffset = newDataReservation; /* write new reservation */ ColumnarOverwriteMetapage(rel, metapage); UnlockRelationForExtension(rel, ExclusiveLock); PhysicalAddr final = LogicalToPhysical(newDataReservation - 1); BlockNumber new_rel_pages = final.blockno + 1; Assert(new_rel_pages <= old_rel_pages); /* * Truncate the storage. Note that RelationTruncate() takes care of * Write Ahead Logging. */ if (new_rel_pages < old_rel_pages) { RelationTruncate(rel, new_rel_pages); return true; } return false; } /* * ColumnarOverwriteMetapage writes given columnarMetapage back to metapage * for given relation. */ static void ColumnarOverwriteMetapage(Relation relation, ColumnarMetapage columnarMetapage) { /* clear metapage because we are overwriting */ bool clear = true; WriteToBlock(relation, COLUMNAR_METAPAGE_BLOCKNO, SizeOfPageHeaderData, (char *) &columnarMetapage, sizeof(ColumnarMetapage), clear); } /* * ColumnarMetapageRead - read the current contents of the metapage. Error if * it does not exist. Throw an error if the metapage is not the current * version, unless 'force' is true. * * NB: it's safe to read a different version of a metapage because we * guarantee that fields will only be added and existing fields will never be * changed. However, it's important that we don't depend on new fields being * set properly when we read an old metapage; an old metapage should only be * read for the purposes of upgrading or error checking. */ static ColumnarMetapage ColumnarMetapageRead(Relation rel, bool force) { BlockNumber nblocks = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); if (nblocks == 0) { /* * We only expect this to happen when upgrading citus.so. This is because, * in current version of columnar, we immediately create the metapage * for columnar tables, i.e right after creating the table. * However in older versions, we were creating metapages lazily, i.e * when ingesting data to columnar table. */ ereport(ERROR, (errmsg("columnar metapage for relation \"%s\" does not exist", RelationGetRelationName(rel)), errhint(OLD_METAPAGE_VERSION_HINT))); } /* * Regardless of "force" parameter, always force read metapage block. * We will check metapage version in ColumnarMetapageCheckVersion * depending on "force". */ bool forceReadBlock = true; ColumnarMetapage metapage; ReadFromBlock(rel, COLUMNAR_METAPAGE_BLOCKNO, SizeOfPageHeaderData, (char *) &metapage, sizeof(ColumnarMetapage), forceReadBlock); if (!force) { ColumnarMetapageCheckVersion(rel, &metapage); } return metapage; } /* * ReadFromBlock - read bytes from a page at the given offset. If 'force' is * true, don't check pd_lower; useful when reading a metapage of unknown * version. */ static void ReadFromBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf, uint32 len, bool force) { Buffer buffer = ReadBuffer(rel, blockno); LockBuffer(buffer, BUFFER_LOCK_SHARE); Page page = BufferGetPage(buffer); PageHeader phdr = (PageHeader) page; if (BLCKSZ < offset + len || (!force && (phdr->pd_lower < offset + len))) { elog(ERROR, "attempt to read columnar data of length %d from offset %d of block %d of relation %d", len, offset, blockno, rel->rd_id); } memcpy_s(buf, len, page + offset, len); UnlockReleaseBuffer(buffer); } /* * WriteToBlock - append data to a block, initializing if necessary, and emit * WAL. If 'clear' is true, always clear the data on the page and reinitialize * it first, and offset must be SizeOfPageHeaderData. Otherwise, offset must * be equal to pd_lower and pd_lower will be set to the end of the written * data. */ static void WriteToBlock(Relation rel, BlockNumber blockno, uint32 offset, char *buf, uint32 len, bool clear) { Buffer buffer = ReadBuffer(rel, blockno); GenericXLogState *state = GenericXLogStart(rel); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); Page page = GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE); PageHeader phdr = (PageHeader) page; if (PageIsNew(page) || clear) { PageInit(page, BLCKSZ, 0); } if (phdr->pd_lower < offset || phdr->pd_upper - offset < len) { elog(ERROR, "attempt to write columnar data of length %d to offset %d of block %d of relation %d", len, offset, blockno, rel->rd_id); } /* * After a transaction has been rolled-back, we might be * over-writing the rolledback write, so phdr->pd_lower can be * different from addr.offset. * * We reset pd_lower to reset the rolledback write. * * Given that we always align page reservation to the next page as of * 10.2, having such a disk page is only possible if write operaion * failed in an older version of columnar, but now user attempts writing * to that table in version >= 10.2. */ if (phdr->pd_lower > offset) { ereport(DEBUG4, (errmsg("overwriting page %u", blockno), errdetail("This can happen after a roll-back."))); phdr->pd_lower = offset; } memcpy_s(page + phdr->pd_lower, phdr->pd_upper - phdr->pd_lower, buf, len); phdr->pd_lower += len; GenericXLogFinish(state); UnlockReleaseBuffer(buffer); } /* * AlignReservation - given an unused logical byte offset, align it so that it * falls at the start of a page. * * XXX: Reconsider whether we want/need to do this at all. */ static uint64 AlignReservation(uint64 prevReservation) { PhysicalAddr prevAddr = LogicalToPhysical(prevReservation); uint64 alignedReservation = prevReservation; if (prevAddr.offset != SizeOfPageHeaderData) { /* not aligned; align on beginning of next page */ PhysicalAddr initial = { 0 }; initial.blockno = prevAddr.blockno + 1; initial.offset = SizeOfPageHeaderData; alignedReservation = PhysicalToLogical(initial); } Assert(alignedReservation >= prevReservation); return alignedReservation; } /* * ColumnarMetapageIsCurrent - is the metapage at the latest version? */ static bool ColumnarMetapageIsCurrent(ColumnarMetapage *metapage) { return (metapage->versionMajor == COLUMNAR_VERSION_MAJOR && metapage->versionMinor == COLUMNAR_VERSION_MINOR); } /* * ColumnarMetapageIsOlder - is the metapage older than the current version? */ static bool ColumnarMetapageIsOlder(ColumnarMetapage *metapage) { return (metapage->versionMajor < COLUMNAR_VERSION_MAJOR || (metapage->versionMajor == COLUMNAR_VERSION_MAJOR && (int) metapage->versionMinor < (int) COLUMNAR_VERSION_MINOR)); } /* * ColumnarMetapageIsNewer - is the metapage newer than the current version? */ static bool ColumnarMetapageIsNewer(ColumnarMetapage *metapage) { return (metapage->versionMajor > COLUMNAR_VERSION_MAJOR || (metapage->versionMajor == COLUMNAR_VERSION_MAJOR && metapage->versionMinor > COLUMNAR_VERSION_MINOR)); } /* * ColumnarMetapageCheckVersion - throw an error if accessing old * version of metapage. */ static void ColumnarMetapageCheckVersion(Relation rel, ColumnarMetapage *metapage) { if (!ColumnarMetapageIsCurrent(metapage)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "attempted to access relation \"%s\", which uses an older columnar format", RelationGetRelationName(rel)), errdetail( "Columnar format version %d.%d is required, \"%s\" has version %d.%d.", COLUMNAR_VERSION_MAJOR, COLUMNAR_VERSION_MINOR, RelationGetRelationName(rel), metapage->versionMajor, metapage->versionMinor), errhint(OLD_METAPAGE_VERSION_HINT))); } } /* * test_columnar_storage_write_new_page is a UDF only used for testing * purposes. It could make more sense to define this in columnar_debug.c, * but the storage layer doesn't expose ColumnarMetapage to any other files, * so we define it here. */ Datum test_columnar_storage_write_new_page(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); Relation relation = relation_open(relationId, AccessShareLock); /* * Allocate a new page, write some data to there, and set reserved offset * to the start of that page. That way, for a subsequent write operation, * storage layer would try to overwrite the page that we allocated here. */ uint64 newPageOffset = ColumnarStorageGetReservedOffset(relation, false); ColumnarStorageReserveData(relation, 100); ColumnarStorageWrite(relation, newPageOffset, "foo_bar", 8); ColumnarMetapage metapage = ColumnarMetapageRead(relation, false); metapage.reservedOffset = newPageOffset; ColumnarOverwriteMetapage(relation, metapage); relation_close(relation, AccessShareLock); PG_RETURN_VOID(); } ================================================ FILE: src/backend/columnar/columnar_tableam.c ================================================ #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "safe_lib.h" #include "access/detoast.h" #include "access/genam.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/rewriteheap.h" #include "access/tableam.h" #include "access/tsmapi.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" #include "catalog/pg_extension.h" #include "catalog/pg_publication.h" #include "catalog/pg_trigger.h" #include "catalog/storage.h" #include "catalog/storage_xlog.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/progress.h" #include "commands/vacuum.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "optimizer/plancat.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" #include "storage/lmgr.h" #include "storage/predicate.h" #include "storage/procarray.h" #include "storage/smgr.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_rusage.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "citus_version.h" #include "pg_version_compat.h" #include "columnar/columnar.h" #include "columnar/columnar_customscan.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" #include "distributed/listutils.h" /* * Timing parameters for truncate locking heuristics. * * These are the same values from src/backend/access/heap/vacuumlazy.c */ #define VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL 50 /* ms */ #define VACUUM_TRUNCATE_LOCK_TIMEOUT 4500 /* ms */ /* * ColumnarScanDescData is the scan state passed between beginscan(), * getnextslot(), rescan(), and endscan() calls. */ typedef struct ColumnarScanDescData { TableScanDescData cs_base; ColumnarReadState *cs_readState; /* * We initialize cs_readState lazily in the first getnextslot() call. We * need the following for initialization. We save them in beginscan(). */ MemoryContext scanContext; Bitmapset *attr_needed; List *scanQual; } ColumnarScanDescData; /* * IndexFetchColumnarData is the scan state passed between index_fetch_begin, * index_fetch_reset, index_fetch_end, index_fetch_tuple calls. */ typedef struct IndexFetchColumnarData { IndexFetchTableData cs_base; ColumnarReadState *cs_readState; /* * We initialize cs_readState lazily in the first columnar_index_fetch_tuple * call. However, we want to do memory allocations in a sub MemoryContext of * columnar_index_fetch_begin. For this reason, we store scanContext in * columnar_index_fetch_begin. */ MemoryContext scanContext; } IndexFetchColumnarData; static object_access_hook_type PrevObjectAccessHook = NULL; static ProcessUtility_hook_type PrevProcessUtilityHook = NULL; /* forward declaration for static functions */ static MemoryContext CreateColumnarScanMemoryContext(void); static void ColumnarTableDropHook(Oid tgid); static void ColumnarTriggerCreateHook(Oid tgid); static void ColumnarTableAMObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg); static RangeVar * ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions); static void ColumnarProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag); static bool ConditionalLockRelationWithTimeout(Relation rel, LOCKMODE lockMode, int timeout, int retryInterval); static List * NeededColumnsList(TupleDesc tupdesc, Bitmapset *attr_needed); static void LogRelationStats(Relation rel, int elevel); static void TruncateColumnar(Relation rel, int elevel); static HeapTuple ColumnarSlotCopyHeapTuple(TupleTableSlot *slot); static void ColumnarCheckLogicalReplication(Relation rel); static Datum * detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull); static ItemPointerData row_number_to_tid(uint64 rowNumber); static uint64 tid_to_row_number(ItemPointerData tid); static void ErrorIfInvalidRowNumber(uint64 rowNumber); static void ColumnarReportTotalVirtualBlocks(Relation relation, Snapshot snapshot, int progressArrIndex); static BlockNumber ColumnarGetNumberOfVirtualBlocks(Relation relation, Snapshot snapshot); static ItemPointerData ColumnarGetHighestItemPointer(Relation relation, Snapshot snapshot); static double ColumnarReadRowsIntoIndex(TableScanDesc scan, Relation indexRelation, IndexInfo *indexInfo, bool progress, IndexBuildCallback indexCallback, void *indexCallbackState, EState *estate, ExprState *predicate); static void ColumnarReadMissingRowsIntoIndex(TableScanDesc scan, Relation indexRelation, IndexInfo *indexInfo, EState *estate, ExprState *predicate, ValidateIndexState *state); static ItemPointerData TupleSortSkipSmallerItemPointers(Tuplesortstate *tupleSort, ItemPointer targetItemPointer); /* functions for CheckCitusColumnarVersion */ static bool CheckAvailableVersionColumnar(int elevel); static bool CheckInstalledVersionColumnar(int elevel); static char * AvailableExtensionVersionColumnar(void); static char * InstalledExtensionVersionColumnar(void); static bool CitusColumnarHasBeenLoadedInternal(void); static bool CitusColumnarHasBeenLoaded(void); static bool CheckCitusColumnarVersion(int elevel); static bool MajorVersionsCompatibleColumnar(char *leftVersion, char *rightVersion); static bool MinorVersionsCompatibleRelaxedColumnar(char *leftVersion, char *rightVersion); static int ParseVersionComponent(const char *version, char **endPtr); /* global variables for CheckCitusColumnarVersion */ static bool extensionLoadedColumnar = false; static bool EnableVersionChecksColumnar = true; static bool citusVersionKnownCompatibleColumnar = false; /* Custom tuple slot ops used for columnar. Initialized in columnar_tableam_init(). */ static TupleTableSlotOps TTSOpsColumnar; static const TupleTableSlotOps * columnar_slot_callbacks(Relation relation) { return &TTSOpsColumnar; } static TableScanDesc columnar_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, uint32 flags) { CheckCitusColumnarVersion(ERROR); int natts = relation->rd_att->natts; /* attr_needed represents 0-indexed attribute numbers */ Bitmapset *attr_needed = bms_add_range(NULL, 0, natts - 1); TableScanDesc scandesc = columnar_beginscan_extended(relation, snapshot, nkeys, key, parallel_scan, flags, attr_needed, NULL); bms_free(attr_needed); return scandesc; } TableScanDesc columnar_beginscan_extended(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, uint32 flags, Bitmapset *attr_needed, List *scanQual) { CheckCitusColumnarVersion(ERROR); RelFileNumber relfilenumber = relation->rd_locator.relNumber; /* * A memory context to use for scan-wide data, including the lazily * initialized read state. We assume that beginscan is called in a * context that will last until end of scan. */ MemoryContext scanContext = CreateColumnarScanMemoryContext(); MemoryContext oldContext = MemoryContextSwitchTo(scanContext); ColumnarScanDesc scan = palloc0(sizeof(ColumnarScanDescData)); scan->cs_base.rs_rd = relation; scan->cs_base.rs_snapshot = snapshot; scan->cs_base.rs_nkeys = nkeys; scan->cs_base.rs_key = key; scan->cs_base.rs_flags = flags; scan->cs_base.rs_parallel = parallel_scan; /* * We will initialize this lazily in first tuple, where we have the actual * tuple descriptor to use for reading. In some cases like ALTER TABLE ... * ALTER COLUMN ... TYPE, the tuple descriptor of relation doesn't match * the storage which we are reading, so we need to use the tuple descriptor * of "slot" in first read. */ scan->cs_readState = NULL; scan->attr_needed = bms_copy(attr_needed); scan->scanQual = copyObject(scanQual); scan->scanContext = scanContext; if (PendingWritesInUpperTransactions(relfilenumber, GetCurrentSubTransactionId())) { elog(ERROR, "cannot read from table when there is unflushed data in upper transactions"); } MemoryContextSwitchTo(oldContext); return ((TableScanDesc) scan); } /* * CreateColumnarScanMemoryContext creates a memory context to store * ColumnarReadStare in it. */ static MemoryContext CreateColumnarScanMemoryContext(void) { return AllocSetContextCreate(CurrentMemoryContext, "Columnar Scan Context", ALLOCSET_DEFAULT_SIZES); } /* * init_columnar_read_state initializes a column store table read and returns the * state. */ static ColumnarReadState * init_columnar_read_state(Relation relation, TupleDesc tupdesc, Bitmapset *attr_needed, List *scanQual, MemoryContext scanContext, Snapshot snapshot, bool randomAccess) { MemoryContext oldContext = MemoryContextSwitchTo(scanContext); List *neededColumnList = NeededColumnsList(tupdesc, attr_needed); ColumnarReadState *readState = ColumnarBeginRead(relation, tupdesc, neededColumnList, scanQual, scanContext, snapshot, randomAccess); MemoryContextSwitchTo(oldContext); return readState; } static void columnar_endscan(TableScanDesc sscan) { ColumnarScanDesc scan = (ColumnarScanDesc) sscan; if (scan->cs_readState != NULL) { ColumnarEndRead(scan->cs_readState); scan->cs_readState = NULL; } if (scan->cs_base.rs_flags & SO_TEMP_SNAPSHOT) { UnregisterSnapshot(scan->cs_base.rs_snapshot); } } static void columnar_rescan(TableScanDesc sscan, ScanKey key, bool set_params, bool allow_strat, bool allow_sync, bool allow_pagemode) { ColumnarScanDesc scan = (ColumnarScanDesc) sscan; /* XXX: hack to pass in new quals that aren't actually scan keys */ List *scanQual = (List *) key; if (scan->cs_readState != NULL) { ColumnarRescan(scan->cs_readState, scanQual); } } static bool columnar_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot) { ColumnarScanDesc scan = (ColumnarScanDesc) sscan; /* * if this is the first row, initialize read state. */ if (scan->cs_readState == NULL) { bool randomAccess = false; scan->cs_readState = init_columnar_read_state(scan->cs_base.rs_rd, slot->tts_tupleDescriptor, scan->attr_needed, scan->scanQual, scan->scanContext, scan->cs_base.rs_snapshot, randomAccess); } ExecClearTuple(slot); uint64 rowNumber; bool nextRowFound = ColumnarReadNextRow(scan->cs_readState, slot->tts_values, slot->tts_isnull, &rowNumber); if (!nextRowFound) { return false; } ExecStoreVirtualTuple(slot); slot->tts_tid = row_number_to_tid(rowNumber); return true; } /* * row_number_to_tid maps given rowNumber to ItemPointerData. */ static ItemPointerData row_number_to_tid(uint64 rowNumber) { ErrorIfInvalidRowNumber(rowNumber); ItemPointerData tid = { 0 }; ItemPointerSetBlockNumber(&tid, rowNumber / VALID_ITEMPOINTER_OFFSETS); ItemPointerSetOffsetNumber(&tid, rowNumber % VALID_ITEMPOINTER_OFFSETS + FirstOffsetNumber); return tid; } /* * tid_to_row_number maps given ItemPointerData to rowNumber. */ static uint64 tid_to_row_number(ItemPointerData tid) { uint64 rowNumber = ItemPointerGetBlockNumber(&tid) * VALID_ITEMPOINTER_OFFSETS + ItemPointerGetOffsetNumber(&tid) - FirstOffsetNumber; ErrorIfInvalidRowNumber(rowNumber); return rowNumber; } /* * ErrorIfInvalidRowNumber errors out if given rowNumber is invalid. */ static void ErrorIfInvalidRowNumber(uint64 rowNumber) { if (rowNumber == COLUMNAR_INVALID_ROW_NUMBER) { /* not expected but be on the safe side */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unexpected row number for columnar table"))); } else if (rowNumber > COLUMNAR_MAX_ROW_NUMBER) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("columnar tables can't have row numbers " "greater than " UINT64_FORMAT, (uint64) COLUMNAR_MAX_ROW_NUMBER), errhint("Consider using VACUUM FULL for your table"))); } } static Size columnar_parallelscan_estimate(Relation rel) { elog(ERROR, "columnar_parallelscan_estimate not implemented"); } static Size columnar_parallelscan_initialize(Relation rel, ParallelTableScanDesc pscan) { elog(ERROR, "columnar_parallelscan_initialize not implemented"); } static void columnar_parallelscan_reinitialize(Relation rel, ParallelTableScanDesc pscan) { elog(ERROR, "columnar_parallelscan_reinitialize not implemented"); } static IndexFetchTableData * columnar_index_fetch_begin(Relation rel) { CheckCitusColumnarVersion(ERROR); RelFileNumber relfilenumber = rel->rd_locator.relNumber; if (PendingWritesInUpperTransactions(relfilenumber, GetCurrentSubTransactionId())) { /* XXX: maybe we can just flush the data and continue */ elog(ERROR, "cannot read from index when there is unflushed data in " "upper transactions"); } MemoryContext scanContext = CreateColumnarScanMemoryContext(); MemoryContext oldContext = MemoryContextSwitchTo(scanContext); IndexFetchColumnarData *scan = palloc0(sizeof(IndexFetchColumnarData)); scan->cs_base.rel = rel; scan->cs_readState = NULL; scan->scanContext = scanContext; MemoryContextSwitchTo(oldContext); return &scan->cs_base; } static void columnar_index_fetch_reset(IndexFetchTableData *sscan) { /* no-op */ } static void columnar_index_fetch_end(IndexFetchTableData *sscan) { columnar_index_fetch_reset(sscan); IndexFetchColumnarData *scan = (IndexFetchColumnarData *) sscan; if (scan->cs_readState) { ColumnarEndRead(scan->cs_readState); scan->cs_readState = NULL; } } static bool columnar_index_fetch_tuple(struct IndexFetchTableData *sscan, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot, bool *call_again, bool *all_dead) { /* no HOT chains are possible in columnar, directly set it to false */ *call_again = false; /* * Initialize all_dead to false if passed to be non-NULL. * * XXX: For aborted writes, we should set all_dead to true but this would * require implementing columnar_index_delete_tuples for simple deletion * of dead tuples (TM_IndexDeleteOp.bottomup = false). */ if (all_dead) { *all_dead = false; } ExecClearTuple(slot); IndexFetchColumnarData *scan = (IndexFetchColumnarData *) sscan; Relation columnarRelation = scan->cs_base.rel; /* initialize read state for the first row */ if (scan->cs_readState == NULL) { /* we need all columns */ int natts = columnarRelation->rd_att->natts; Bitmapset *attr_needed = bms_add_range(NULL, 0, natts - 1); /* no quals for index scan */ List *scanQual = NIL; bool randomAccess = true; scan->cs_readState = init_columnar_read_state(columnarRelation, slot->tts_tupleDescriptor, attr_needed, scanQual, scan->scanContext, snapshot, randomAccess); } uint64 rowNumber = tid_to_row_number(*tid); StripeMetadata *stripeMetadata = FindStripeWithMatchingFirstRowNumber(columnarRelation, rowNumber, snapshot); if (!stripeMetadata) { /* it is certain that tuple with rowNumber doesn't exist */ return false; } StripeWriteStateEnum stripeWriteState = StripeWriteState(stripeMetadata); if (stripeWriteState == STRIPE_WRITE_FLUSHED && !ColumnarReadRowByRowNumber(scan->cs_readState, rowNumber, slot->tts_values, slot->tts_isnull)) { /* * FindStripeWithMatchingFirstRowNumber doesn't verify upper row * number boundary of found stripe. For this reason, we didn't * certainly know if given row number belongs to one of the stripes. */ return false; } else if (stripeWriteState == STRIPE_WRITE_ABORTED) { /* * We only expect to see un-flushed stripes when checking against * constraint violation. In that case, indexAM provides dirty * snapshot to index_fetch_tuple callback. */ Assert(snapshot->snapshot_type == SNAPSHOT_DIRTY); return false; } else if (stripeWriteState == STRIPE_WRITE_IN_PROGRESS) { if (stripeMetadata->insertedByCurrentXact) { /* * Stripe write is in progress and its entry is inserted by current * transaction, so obviously it must be written by me. Since caller * might want to use tupleslot datums for some reason, do another * look-up, but this time by first flushing our writes. * * XXX: For index scan, this is the only case that we flush pending * writes of the current backend. If we have taught reader how to * read from WriteStateMap. then we could guarantee that * index_fetch_tuple would never flush pending writes, but this seem * to be too much work for now, but should be doable. */ ColumnarReadFlushPendingWrites(scan->cs_readState); /* * Fill the tupleslot and fall through to return true, it * certainly exists. */ ColumnarReadRowByRowNumberOrError(scan->cs_readState, rowNumber, slot->tts_values, slot->tts_isnull); } else { /* similar to aborted writes, it should be dirty snapshot */ Assert(snapshot->snapshot_type == SNAPSHOT_DIRTY); /* * Stripe that "might" contain the tuple with rowNumber is not * flushed yet. Here we set all attributes of given tupleslot to NULL * before returning true and expect the indexAM callback that called * us --possibly to check against constraint violation-- blocks until * writer transaction commits or aborts, without requiring us to fill * the tupleslot properly. * * XXX: Note that the assumption we made above for the tupleslot * holds for "unique" constraints defined on "btree" indexes. * * For the other constraints that we support, namely: * * exclusion on btree, * * exclusion on hash, * * unique on btree; * we still need to fill tts_values. * * However, for the same reason, we should have already flushed * single tuple stripes when inserting into table for those three * classes of constraints. * * This is annoying, but this also explains why this hack works for * unique constraints on btree indexes, and also explains how we * would never end up with "else" condition otherwise. */ memset(slot->tts_isnull, true, slot->tts_nvalid * sizeof(bool)); } } else { /* * At this point, we certainly know that stripe is flushed and * ColumnarReadRowByRowNumber successfully filled the tupleslot. */ Assert(stripeWriteState == STRIPE_WRITE_FLUSHED); } slot->tts_tableOid = RelationGetRelid(columnarRelation); slot->tts_tid = *tid; ExecStoreVirtualTuple(slot); return true; } static bool columnar_fetch_row_version(Relation relation, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot) { elog(ERROR, "columnar_fetch_row_version not implemented"); } static void columnar_get_latest_tid(TableScanDesc sscan, ItemPointer tid) { elog(ERROR, "columnar_get_latest_tid not implemented"); } static bool columnar_tuple_tid_valid(TableScanDesc scan, ItemPointer tid) { elog(ERROR, "columnar_tuple_tid_valid not implemented"); } static bool columnar_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, Snapshot snapshot) { CheckCitusColumnarVersion(ERROR); uint64 rowNumber = tid_to_row_number(slot->tts_tid); StripeMetadata *stripeMetadata = FindStripeByRowNumber(rel, rowNumber, snapshot); return stripeMetadata != NULL; } static TransactionId columnar_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate) { CheckCitusColumnarVersion(ERROR); /* * XXX: We didn't bother implementing index_delete_tuple for neither of * simple deletion and bottom-up deletion cases. There is no particular * reason for that, just to keep things simple. * * See the rest of this function to see how we deal with * index_delete_tuples requests made to columnarAM. */ if (delstate->bottomup) { /* * Ignore any bottom-up deletion requests. * * Currently only caller in postgres that does bottom-up deletion is * _bt_bottomupdel_pass, which in turn calls _bt_delitems_delete_check. * And this function is okay with ndeltids being set to 0 by tableAM * for bottom-up deletion. */ delstate->ndeltids = 0; return InvalidTransactionId; } else { /* * TableAM is not expected to set ndeltids to 0 for simple deletion * case, so here we cannot do the same trick that we do for * bottom-up deletion. * See the assertion around table_index_delete_tuples call in pg * function index_compute_xid_horizon_for_tuples. * * For this reason, to avoid receiving simple deletion requests for * columnar tables (bottomup = false), columnar_index_fetch_tuple * doesn't ever set all_dead to true in order to prevent triggering * simple deletion of index tuples. But let's throw an error to be on * the safe side. */ elog(ERROR, "columnar_index_delete_tuples not implemented for simple deletion"); } } static void columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate) { CheckCitusColumnarVersion(ERROR); /* * columnar_init_write_state allocates the write state in a longer * lasting context, so no need to worry about it. */ ColumnarWriteState *writeState = columnar_init_write_state(relation, RelationGetDescr(relation), slot->tts_tableOid, GetCurrentSubTransactionId()); MemoryContext oldContext = MemoryContextSwitchTo(ColumnarWritePerTupleContext( writeState)); ColumnarCheckLogicalReplication(relation); slot_getallattrs(slot); Datum *values = detoast_values(slot->tts_tupleDescriptor, slot->tts_values, slot->tts_isnull); uint64 writtenRowNumber = ColumnarWriteRow(writeState, values, slot->tts_isnull); slot->tts_tid = row_number_to_tid(writtenRowNumber); MemoryContextSwitchTo(oldContext); MemoryContextReset(ColumnarWritePerTupleContext(writeState)); } static void columnar_tuple_insert_speculative(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate, uint32 specToken) { elog(ERROR, "columnar_tuple_insert_speculative not implemented"); } static void columnar_tuple_complete_speculative(Relation relation, TupleTableSlot *slot, uint32 specToken, bool succeeded) { elog(ERROR, "columnar_tuple_complete_speculative not implemented"); } static void columnar_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, CommandId cid, int options, BulkInsertState bistate) { CheckCitusColumnarVersion(ERROR); /* * The callback to .multi_insert is table_multi_insert() and this is only used for the COPY * command, so slot[i]->tts_tableoid will always be equal to relation->id. Thus, we can send * RelationGetRelid(relation) as the tupSlotTableOid */ ColumnarWriteState *writeState = columnar_init_write_state(relation, RelationGetDescr(relation), RelationGetRelid(relation), GetCurrentSubTransactionId()); ColumnarCheckLogicalReplication(relation); MemoryContext oldContext = MemoryContextSwitchTo(ColumnarWritePerTupleContext( writeState)); for (int i = 0; i < ntuples; i++) { TupleTableSlot *tupleSlot = slots[i]; slot_getallattrs(tupleSlot); Datum *values = detoast_values(tupleSlot->tts_tupleDescriptor, tupleSlot->tts_values, tupleSlot->tts_isnull); uint64 writtenRowNumber = ColumnarWriteRow(writeState, values, tupleSlot->tts_isnull); tupleSlot->tts_tid = row_number_to_tid(writtenRowNumber); MemoryContextReset(ColumnarWritePerTupleContext(writeState)); } MemoryContextSwitchTo(oldContext); } static TM_Result columnar_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, Snapshot snapshot, Snapshot crosscheck, bool wait, TM_FailureData *tmfd, bool changingPart) { elog(ERROR, "columnar_tuple_delete not implemented"); } static TM_Result columnar_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, CommandId cid, Snapshot snapshot, Snapshot crosscheck, bool wait, TM_FailureData *tmfd, LockTupleMode *lockmode, TU_UpdateIndexes *update_indexes) { elog(ERROR, "columnar_tuple_update not implemented"); } static TM_Result columnar_tuple_lock(Relation relation, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot, CommandId cid, LockTupleMode mode, LockWaitPolicy wait_policy, uint8 flags, TM_FailureData *tmfd) { elog(ERROR, "columnar_tuple_lock not implemented"); } static void columnar_finish_bulk_insert(Relation relation, int options) { /* * Nothing to do here. We keep write states live until transaction end. */ } static void columnar_relation_set_new_filelocator(Relation rel, const RelFileLocator *newrlocator, char persistence, TransactionId *freezeXid, MultiXactId *minmulti) { CheckCitusColumnarVersion(ERROR); if (persistence == RELPERSISTENCE_UNLOGGED) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unlogged columnar tables are not supported"))); } /* * If existing and new relfilenode are different, that means the existing * storage was dropped and we also need to clean up the metadata and * state. If they are equal, this is a new relation object and we don't * need to clean anything. */ if (rel->rd_locator.relNumber != newrlocator->relNumber) { MarkRelfilenumberDropped(rel->rd_locator.relNumber, GetCurrentSubTransactionId()); DeleteMetadataRows(rel); } *freezeXid = RecentXmin; *minmulti = GetOldestMultiXactId(); SMgrRelation srel = RelationCreateStorage(*newrlocator, persistence, true); ColumnarStorageInit(srel, ColumnarMetadataNewStorageId()); InitColumnarOptions(rel->rd_id); smgrclose(srel); /* we will lazily initialize metadata in first stripe reservation */ } static void columnar_relation_nontransactional_truncate(Relation rel) { CheckCitusColumnarVersion(ERROR); RelFileLocator relfilelocator = rel->rd_locator; NonTransactionDropWriteState(relfilelocator.relNumber); /* Delete old relfilenode metadata */ DeleteMetadataRows(rel); /* * No need to set new relfilenode, since the table was created in this * transaction and no other transaction can see this relation yet. We * can just truncate the relation. * * This is similar to what is done in heapam_relation_nontransactional_truncate. */ RelationTruncate(rel, 0); uint64 storageId = ColumnarMetadataNewStorageId(); ColumnarStorageInit(RelationGetSmgr(rel), storageId); } static void columnar_relation_copy_data(Relation rel, const RelFileLocator *newrnode) { elog(ERROR, "columnar_relation_copy_data not implemented"); } /* * columnar_relation_copy_for_cluster is called on VACUUM FULL, at which * we should copy data from OldHeap to NewHeap. * * In general TableAM case this can also be called for the CLUSTER command * which is not applicable for columnar since it doesn't support indexes. */ static void columnar_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, Relation OldIndex, bool use_sort, TransactionId OldestXmin, TransactionId *xid_cutoff, MultiXactId *multi_cutoff, double *num_tuples, double *tups_vacuumed, double *tups_recently_dead) { CheckCitusColumnarVersion(ERROR); TupleDesc sourceDesc = RelationGetDescr(OldHeap); TupleDesc targetDesc = RelationGetDescr(NewHeap); if (OldIndex != NULL || use_sort) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("clustering columnar tables using indexes is " "not supported"))); } /* * copy_table_data in cluster.c assumes tuple descriptors are exactly * the same. Even dropped columns exist and are marked as attisdropped * in the target relation. */ Assert(sourceDesc->natts == targetDesc->natts); /* read settings from old heap, relfilenode will be swapped at the end */ ColumnarOptions columnarOptions = { 0 }; ReadColumnarOptions(OldHeap->rd_id, &columnarOptions); ColumnarWriteState *writeState = ColumnarBeginWrite(NewHeap, columnarOptions, targetDesc); /* we need all columns */ int natts = OldHeap->rd_att->natts; Bitmapset *attr_needed = bms_add_range(NULL, 0, natts - 1); /* no quals for table rewrite */ List *scanQual = NIL; /* use SnapshotAny when re-writing table as heapAM does */ Snapshot snapshot = SnapshotAny; MemoryContext scanContext = CreateColumnarScanMemoryContext(); bool randomAccess = false; ColumnarReadState *readState = init_columnar_read_state(OldHeap, sourceDesc, attr_needed, scanQual, scanContext, snapshot, randomAccess); Datum *values = palloc0(sourceDesc->natts * sizeof(Datum)); bool *nulls = palloc0(sourceDesc->natts * sizeof(bool)); *num_tuples = 0; /* we don't need to know rowNumber here */ while (ColumnarReadNextRow(readState, values, nulls, NULL)) { ColumnarWriteRow(writeState, values, nulls); (*num_tuples)++; } *tups_vacuumed = 0; ColumnarEndWrite(writeState); ColumnarEndRead(readState); } /* * NeededColumnsList returns a list of AttrNumber's for the columns that * are not dropped and specified by attr_needed. */ static List * NeededColumnsList(TupleDesc tupdesc, Bitmapset *attr_needed) { List *columnList = NIL; for (int i = 0; i < tupdesc->natts; i++) { if (TupleDescAttr(tupdesc, i)->attisdropped) { continue; } /* attr_needed is 0-indexed but columnList is 1-indexed */ if (bms_is_member(i, attr_needed)) { AttrNumber varattno = i + 1; columnList = lappend_int(columnList, varattno); } } return columnList; } /* * ColumnarTableTupleCount returns the number of tuples that columnar * table with relationId has by using stripe metadata. */ static uint64 ColumnarTableTupleCount(Relation relation) { List *stripeList = StripesForRelfilelocator(relation); uint64 tupleCount = 0; ListCell *lc = NULL; foreach(lc, stripeList) { StripeMetadata *stripe = lfirst(lc); tupleCount += stripe->rowCount; } return tupleCount; } /* * columnar_vacuum_rel implements VACUUM without FULL option. */ static void columnar_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { if (!CheckCitusColumnarVersion(WARNING)) { /* * Skip if the extension catalogs are not up-to-date, but avoid * erroring during auto-vacuum. */ return; } pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); /* * If metapage version of relation is older, then we hint users to VACUUM * the relation in ColumnarMetapageCheckVersion. So if needed, upgrade * the metapage before doing anything. */ bool isUpgrade = true; ColumnarStorageUpdateIfNeeded(rel, isUpgrade); int elevel = (params->options & VACOPT_VERBOSE) ? INFO : DEBUG2; /* this should have been resolved by vacuum.c until now */ Assert(params->truncate != VACOPTVALUE_UNSPECIFIED); LogRelationStats(rel, elevel); /* * We don't have updates, deletes, or concurrent updates, so all we * care for now is truncating the unused space at the end of storage. */ if (params->truncate == VACOPTVALUE_ENABLED) { TruncateColumnar(rel, elevel); } BlockNumber new_rel_pages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); /* get the number of indexes */ List *indexList = RelationGetIndexList(rel); int nindexes = list_length(indexList); struct VacuumCutoffs cutoffs; vacuum_get_cutoffs(rel, params, &cutoffs); Assert(MultiXactIdPrecedesOrEquals(cutoffs.MultiXactCutoff, cutoffs.OldestMxact)); Assert(TransactionIdPrecedesOrEquals(cutoffs.FreezeLimit, cutoffs.OldestXmin)); /* * Columnar storage doesn't hold any transaction IDs, so we can always * just advance to the most aggressive value. */ TransactionId newRelFrozenXid = cutoffs.OldestXmin; MultiXactId newRelminMxid = cutoffs.OldestMxact; double new_live_tuples = ColumnarTableTupleCount(rel); /* all visible pages are always 0 */ BlockNumber new_rel_allvisible = 0; bool frozenxid_updated; bool minmulti_updated; /* for PG 18+, vac_update_relstats gained a new “all_frozen” param */ #if PG_VERSION_NUM >= PG_VERSION_18 /* all frozen pages are always 0, because columnar stripes never store XIDs */ BlockNumber new_rel_allfrozen = 0; vac_update_relstats(rel, new_rel_pages, new_live_tuples, new_rel_allvisible, /* allvisible */ new_rel_allfrozen, /* all_frozen */ nindexes > 0, newRelFrozenXid, newRelminMxid, &frozenxid_updated, &minmulti_updated, false); #else vac_update_relstats(rel, new_rel_pages, new_live_tuples, new_rel_allvisible, nindexes > 0, newRelFrozenXid, newRelminMxid, &frozenxid_updated, &minmulti_updated, false); #endif #if PG_VERSION_NUM >= PG_VERSION_18 pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(new_live_tuples, 0), /* live tuples */ 0, /* dead tuples */ GetCurrentTimestamp()); /* start time */ #else pgstat_report_vacuum(RelationGetRelid(rel), rel->rd_rel->relisshared, Max(new_live_tuples, 0), 0); #endif pgstat_progress_end_command(); } /* * LogRelationStats logs statistics as the output of the VACUUM VERBOSE. */ static void LogRelationStats(Relation rel, int elevel) { ListCell *stripeMetadataCell = NULL; StringInfo infoBuf = makeStringInfo(); int compressionStats[COMPRESSION_COUNT] = { 0 }; uint64 totalStripeLength = 0; uint64 tupleCount = 0; uint64 chunkCount = 0; TupleDesc tupdesc = RelationGetDescr(rel); uint64 droppedChunksWithData = 0; uint64 totalDecompressedLength = 0; List *stripeList = StripesForRelfilelocator(rel); int stripeCount = list_length(stripeList); foreach(stripeMetadataCell, stripeList) { StripeMetadata *stripe = lfirst(stripeMetadataCell); Snapshot snapshot = RegisterSnapshot(GetTransactionSnapshot()); StripeSkipList *skiplist = ReadStripeSkipList(rel, stripe->id, RelationGetDescr(rel), stripe->chunkCount, snapshot); UnregisterSnapshot(snapshot); for (uint32 column = 0; column < skiplist->columnCount; column++) { bool attrDropped = TupleDescAttr(tupdesc, column)->attisdropped; for (uint32 chunk = 0; chunk < skiplist->chunkCount; chunk++) { ColumnChunkSkipNode *skipnode = &skiplist->chunkSkipNodeArray[column][chunk]; /* ignore zero length chunks for dropped attributes */ if (skipnode->valueLength > 0) { compressionStats[skipnode->valueCompressionType]++; chunkCount++; if (attrDropped) { droppedChunksWithData++; } } /* * We don't compress exists buffer, so its compressed & decompressed * lengths are the same. */ totalDecompressedLength += skipnode->existsLength; totalDecompressedLength += skipnode->decompressedValueSize; } } tupleCount += stripe->rowCount; totalStripeLength += stripe->dataLength; } uint64 relPages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); RelationCloseSmgr(rel); Datum storageId = DirectFunctionCall1(columnar_relation_storageid, ObjectIdGetDatum(RelationGetRelid(rel))); double compressionRate = totalStripeLength ? (double) totalDecompressedLength / totalStripeLength : 1.0; appendStringInfo(infoBuf, "storage id: %ld\n", DatumGetInt64(storageId)); appendStringInfo(infoBuf, "total file size: %ld, total data size: %ld\n", relPages * BLCKSZ, totalStripeLength); appendStringInfo(infoBuf, "compression rate: %.2fx\n", compressionRate); appendStringInfo(infoBuf, "total row count: %ld, stripe count: %d, " "average rows per stripe: %ld\n", tupleCount, stripeCount, stripeCount ? tupleCount / stripeCount : 0); appendStringInfo(infoBuf, "chunk count: %ld" ", containing data for dropped columns: %ld", chunkCount, droppedChunksWithData); for (int compressionType = 0; compressionType < COMPRESSION_COUNT; compressionType++) { const char *compressionName = CompressionTypeStr(compressionType); /* skip if this compression algorithm has not been compiled */ if (compressionName == NULL) { continue; } /* skip if no chunks use this compression type */ if (compressionStats[compressionType] == 0) { continue; } appendStringInfo(infoBuf, ", %s compressed: %d", compressionName, compressionStats[compressionType]); } appendStringInfoString(infoBuf, "\n"); ereport(elevel, (errmsg("statistics for \"%s\":\n%s", RelationGetRelationName(rel), infoBuf->data))); } /* * TruncateColumnar truncates the unused space at the end of main fork for * a columnar table. This unused space can be created by aborted transactions. * * This implementation is based on heap_vacuum_rel in vacuumlazy.c with some * changes so it suits columnar store relations. */ static void TruncateColumnar(Relation rel, int elevel) { PGRUsage ru0; pg_rusage_init(&ru0); /* Report that we are now truncating */ pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, PROGRESS_VACUUM_PHASE_TRUNCATE); /* * We need access exclusive lock on the relation in order to do * truncation. If we can't get it, give up rather than waiting --- we * don't want to block other backends, and we don't want to deadlock * (which is quite possible considering we already hold a lower-grade * lock). * * The decisions for AccessExclusiveLock and conditional lock with * a timeout is based on lazy_truncate_heap in vacuumlazy.c. */ if (!ConditionalLockRelationWithTimeout(rel, AccessExclusiveLock, VACUUM_TRUNCATE_LOCK_TIMEOUT, VACUUM_TRUNCATE_LOCK_WAIT_INTERVAL)) { /* * We failed to establish the lock in the specified number of * retries. This means we give up truncating. */ ereport(elevel, (errmsg("\"%s\": stopping truncate due to conflicting lock request", RelationGetRelationName(rel)))); return; } /* * Due to the AccessExclusive lock there's no danger that * new stripes be added beyond highestPhysicalAddress while * we're truncating. */ uint64 newDataReservation = Max(GetHighestUsedAddress(rel) + 1, ColumnarFirstLogicalOffset); BlockNumber old_rel_pages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); if (!ColumnarStorageTruncate(rel, newDataReservation)) { UnlockRelation(rel, AccessExclusiveLock); return; } BlockNumber new_rel_pages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); /* * We can release the exclusive lock as soon as we have truncated. * Other backends can't safely access the relation until they have * processed the smgr invalidation that smgrtruncate sent out ... but * that should happen as part of standard invalidation processing once * they acquire lock on the relation. */ UnlockRelation(rel, AccessExclusiveLock); ereport(elevel, (errmsg("\"%s\": truncated %u to %u pages", RelationGetRelationName(rel), old_rel_pages, new_rel_pages), errdetail_internal("%s", pg_rusage_show(&ru0)))); } /* * ConditionalLockRelationWithTimeout tries to acquire a relation lock until * it either succeeds or timesout. It doesn't enter wait queue and instead it * sleeps between lock tries. * * This is based on the lock loop in lazy_truncate_heap(). */ static bool ConditionalLockRelationWithTimeout(Relation rel, LOCKMODE lockMode, int timeout, int retryInterval) { int lock_retry = 0; while (true) { if (ConditionalLockRelation(rel, lockMode)) { break; } /* * Check for interrupts while trying to (re-)acquire the lock */ CHECK_FOR_INTERRUPTS(); if (++lock_retry > (timeout / retryInterval)) { return false; } pg_usleep(retryInterval * 1000L); } return true; } static bool columnar_scan_analyze_next_block(TableScanDesc scan, #if PG_VERSION_NUM >= PG_VERSION_17 ReadStream *stream) #else BlockNumber blockno, BufferAccessStrategy bstrategy) #endif { /* * Our access method is not pages based, i.e. tuples are not confined * to pages boundaries. So not much to do here. We return true anyway * so acquire_sample_rows() in analyze.c would call our * columnar_scan_analyze_next_tuple() callback. * In PG17, we return false in case there is no buffer left, since * the outer loop changed in acquire_sample_rows(), and it is * expected for the scan_analyze_next_block function to check whether * there are any blocks left in the block sampler. */ #if PG_VERSION_NUM >= PG_VERSION_17 Buffer buf = read_stream_next_buffer(stream, NULL); if (!BufferIsValid(buf)) { return false; } ReleaseBuffer(buf); #endif return true; } static bool columnar_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, double *liverows, double *deadrows, TupleTableSlot *slot) { /* * Currently we don't do anything smart to reduce number of rows returned * for ANALYZE. The TableAM API's ANALYZE functions are designed for page * based access methods where it chooses random pages, and then reads * tuples from those pages. * * We could do something like that here by choosing sample stripes or chunks, * but getting that correct might need quite some work. Since columnar_fdw's * ANALYZE scanned all rows, as a starter we do the same here and scan all * rows. */ if (columnar_getnextslot(scan, ForwardScanDirection, slot)) { (*liverows)++; return true; } return false; } static double columnar_index_build_range_scan(Relation columnarRelation, Relation indexRelation, IndexInfo *indexInfo, bool allow_sync, bool anyvisible, bool progress, BlockNumber start_blockno, BlockNumber numblocks, IndexBuildCallback callback, void *callback_state, TableScanDesc scan) { CheckCitusColumnarVersion(ERROR); if (start_blockno != 0 || numblocks != InvalidBlockNumber) { /* * Columnar utility hook already errors out for BRIN indexes on columnar * tables, but be on the safe side. */ ereport(ERROR, (errmsg("BRIN indexes on columnar tables are not supported"))); } if (scan) { /* * Parallel scans on columnar tables are already discardad by * ColumnarGetRelationInfoHook but be on the safe side. */ elog(ERROR, "parallel scans on columnar are not supported"); } /* * In a normal index build, we use SnapshotAny to retrieve all tuples. In * a concurrent build or during bootstrap, we take a regular MVCC snapshot * and index whatever's live according to that. */ TransactionId OldestXmin = InvalidTransactionId; if (!IsBootstrapProcessingMode() && !indexInfo->ii_Concurrent) { /* ignore lazy VACUUM's */ OldestXmin = GetOldestNonRemovableTransactionId(columnarRelation); } Snapshot snapshot = { 0 }; bool snapshotRegisteredByUs = false; /* * For serial index build, we begin our own scan. We may also need to * register a snapshot whose lifetime is under our direct control. */ if (!TransactionIdIsValid(OldestXmin)) { snapshot = RegisterSnapshot(GetTransactionSnapshot()); snapshotRegisteredByUs = true; } else { snapshot = SnapshotAny; } int nkeys = 0; ScanKeyData *scanKey = NULL; bool allowAccessStrategy = true; scan = table_beginscan_strat(columnarRelation, snapshot, nkeys, scanKey, allowAccessStrategy, allow_sync); if (progress) { ColumnarReportTotalVirtualBlocks(columnarRelation, snapshot, PROGRESS_SCAN_BLOCKS_TOTAL); } /* * Set up execution state for predicate, if any. * Note that this is only useful for partial indexes. */ EState *estate = CreateExecutorState(); ExprContext *econtext = GetPerTupleExprContext(estate); econtext->ecxt_scantuple = table_slot_create(columnarRelation, NULL); ExprState *predicate = ExecPrepareQual(indexInfo->ii_Predicate, estate); double reltuples = ColumnarReadRowsIntoIndex(scan, indexRelation, indexInfo, progress, callback, callback_state, estate, predicate); table_endscan(scan); if (progress) { /* report the last "virtual" block as "done" */ ColumnarReportTotalVirtualBlocks(columnarRelation, snapshot, PROGRESS_SCAN_BLOCKS_DONE); } if (snapshotRegisteredByUs) { UnregisterSnapshot(snapshot); } ExecDropSingleTupleTableSlot(econtext->ecxt_scantuple); FreeExecutorState(estate); indexInfo->ii_ExpressionsState = NIL; indexInfo->ii_PredicateState = NULL; return reltuples; } /* * ColumnarReportTotalVirtualBlocks reports progress for index build based on * number of "virtual" blocks that given relation has. * "progressArrIndex" argument determines which entry in st_progress_param * array should be updated. In this case, we only expect PROGRESS_SCAN_BLOCKS_TOTAL * or PROGRESS_SCAN_BLOCKS_DONE to specify whether we want to report calculated * number of blocks as "done" or as "total" number of "virtual" blocks to scan. */ static void ColumnarReportTotalVirtualBlocks(Relation relation, Snapshot snapshot, int progressArrIndex) { /* * Indeed, columnar tables might have gaps between row numbers, e.g * due to aborted transactions etc. Also, ItemPointer BlockNumber's * for columnar tables don't actually correspond to actual disk blocks * as in heapAM. For this reason, we call them as "virtual" blocks. At * the moment, we believe it is better to report our progress based on * this "virtual" block concept instead of doing nothing. */ Assert(progressArrIndex == PROGRESS_SCAN_BLOCKS_TOTAL || progressArrIndex == PROGRESS_SCAN_BLOCKS_DONE); BlockNumber nvirtualBlocks = ColumnarGetNumberOfVirtualBlocks(relation, snapshot); pgstat_progress_update_param(progressArrIndex, nvirtualBlocks); } /* * ColumnarGetNumberOfVirtualBlocks returns total number of "virtual" blocks * that given columnar table has based on based on ItemPointer BlockNumber's. */ static BlockNumber ColumnarGetNumberOfVirtualBlocks(Relation relation, Snapshot snapshot) { ItemPointerData highestItemPointer = ColumnarGetHighestItemPointer(relation, snapshot); if (!ItemPointerIsValid(&highestItemPointer)) { /* table is empty according to our snapshot */ return 0; } /* * Since BlockNumber is 0-based, increment it by 1 to find the total * number of "virtual" blocks. */ return ItemPointerGetBlockNumber(&highestItemPointer) + 1; } /* * ColumnarGetHighestItemPointer returns ItemPointerData for the tuple with * highest tid for given relation. * If given relation is empty, then returns invalid item pointer. */ static ItemPointerData ColumnarGetHighestItemPointer(Relation relation, Snapshot snapshot) { StripeMetadata *stripeWithHighestRowNumber = FindStripeWithHighestRowNumber(relation, snapshot); if (stripeWithHighestRowNumber == NULL || StripeGetHighestRowNumber(stripeWithHighestRowNumber) == 0) { /* table is empty according to our snapshot */ ItemPointerData invalidItemPtr; ItemPointerSetInvalid(&invalidItemPtr); return invalidItemPtr; } uint64 highestRowNumber = StripeGetHighestRowNumber(stripeWithHighestRowNumber); return row_number_to_tid(highestRowNumber); } /* * ColumnarReadRowsIntoIndex builds indexRelation tuples by reading the * actual relation based on given "scan" and returns number of tuples * scanned to build the indexRelation. */ static double ColumnarReadRowsIntoIndex(TableScanDesc scan, Relation indexRelation, IndexInfo *indexInfo, bool progress, IndexBuildCallback indexCallback, void *indexCallbackState, EState *estate, ExprState *predicate) { double reltuples = 0; BlockNumber lastReportedBlockNumber = InvalidBlockNumber; ExprContext *econtext = GetPerTupleExprContext(estate); TupleTableSlot *slot = econtext->ecxt_scantuple; while (columnar_getnextslot(scan, ForwardScanDirection, slot)) { CHECK_FOR_INTERRUPTS(); BlockNumber currentBlockNumber = ItemPointerGetBlockNumber(&slot->tts_tid); if (progress && lastReportedBlockNumber != currentBlockNumber) { /* * columnar_getnextslot guarantees that returned tuple will * always have a greater ItemPointer than the ones we fetched * before, so we directly use BlockNumber to report our progress. */ Assert(lastReportedBlockNumber == InvalidBlockNumber || currentBlockNumber >= lastReportedBlockNumber); pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, currentBlockNumber); lastReportedBlockNumber = currentBlockNumber; } MemoryContextReset(econtext->ecxt_per_tuple_memory); if (predicate != NULL && !ExecQual(predicate, econtext)) { /* for partial indexes, discard tuples that don't satisfy the predicate */ continue; } Datum indexValues[INDEX_MAX_KEYS]; bool indexNulls[INDEX_MAX_KEYS]; FormIndexDatum(indexInfo, slot, estate, indexValues, indexNulls); ItemPointerData itemPointerData = slot->tts_tid; /* currently, columnar tables can't have dead tuples */ bool tupleIsAlive = true; indexCallback(indexRelation, &itemPointerData, indexValues, indexNulls, tupleIsAlive, indexCallbackState); reltuples++; } return reltuples; } static void columnar_index_validate_scan(Relation columnarRelation, Relation indexRelation, IndexInfo *indexInfo, Snapshot snapshot, ValidateIndexState * validateIndexState) { CheckCitusColumnarVersion(ERROR); ColumnarReportTotalVirtualBlocks(columnarRelation, snapshot, PROGRESS_SCAN_BLOCKS_TOTAL); /* * Set up execution state for predicate, if any. * Note that this is only useful for partial indexes. */ EState *estate = CreateExecutorState(); ExprContext *econtext = GetPerTupleExprContext(estate); econtext->ecxt_scantuple = table_slot_create(columnarRelation, NULL); ExprState *predicate = ExecPrepareQual(indexInfo->ii_Predicate, estate); int nkeys = 0; ScanKeyData *scanKey = NULL; bool allowAccessStrategy = true; bool allowSyncScan = false; TableScanDesc scan = table_beginscan_strat(columnarRelation, snapshot, nkeys, scanKey, allowAccessStrategy, allowSyncScan); ColumnarReadMissingRowsIntoIndex(scan, indexRelation, indexInfo, estate, predicate, validateIndexState); table_endscan(scan); /* report the last "virtual" block as "done" */ ColumnarReportTotalVirtualBlocks(columnarRelation, snapshot, PROGRESS_SCAN_BLOCKS_DONE); ExecDropSingleTupleTableSlot(econtext->ecxt_scantuple); FreeExecutorState(estate); indexInfo->ii_ExpressionsState = NIL; indexInfo->ii_PredicateState = NULL; } /* * ColumnarReadMissingRowsIntoIndex inserts the tuples that are not in * the index yet by reading the actual relation based on given "scan". */ static void ColumnarReadMissingRowsIntoIndex(TableScanDesc scan, Relation indexRelation, IndexInfo *indexInfo, EState *estate, ExprState *predicate, ValidateIndexState *validateIndexState) { BlockNumber lastReportedBlockNumber = InvalidBlockNumber; bool indexTupleSortEmpty = false; ItemPointerData indexedItemPointerData; ItemPointerSetInvalid(&indexedItemPointerData); ExprContext *econtext = GetPerTupleExprContext(estate); TupleTableSlot *slot = econtext->ecxt_scantuple; while (columnar_getnextslot(scan, ForwardScanDirection, slot)) { CHECK_FOR_INTERRUPTS(); ItemPointer columnarItemPointer = &slot->tts_tid; BlockNumber currentBlockNumber = ItemPointerGetBlockNumber(columnarItemPointer); if (lastReportedBlockNumber != currentBlockNumber) { /* * columnar_getnextslot guarantees that returned tuple will * always have a greater ItemPointer than the ones we fetched * before, so we directly use BlockNumber to report our progress. */ Assert(lastReportedBlockNumber == InvalidBlockNumber || currentBlockNumber >= lastReportedBlockNumber); pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, currentBlockNumber); lastReportedBlockNumber = currentBlockNumber; } validateIndexState->htups += 1; if (!indexTupleSortEmpty && (!ItemPointerIsValid(&indexedItemPointerData) || ItemPointerCompare(&indexedItemPointerData, columnarItemPointer) < 0)) { /* * Skip indexed item pointers until we find or pass the current * columnar relation item pointer. */ indexedItemPointerData = TupleSortSkipSmallerItemPointers(validateIndexState->tuplesort, columnarItemPointer); indexTupleSortEmpty = !ItemPointerIsValid(&indexedItemPointerData); } if (!indexTupleSortEmpty && ItemPointerCompare(&indexedItemPointerData, columnarItemPointer) == 0) { /* tuple is already covered by the index, skip */ continue; } Assert(indexTupleSortEmpty || ItemPointerCompare(&indexedItemPointerData, columnarItemPointer) > 0); MemoryContextReset(econtext->ecxt_per_tuple_memory); if (predicate != NULL && !ExecQual(predicate, econtext)) { /* for partial indexes, discard tuples that don't satisfy the predicate */ continue; } Datum indexValues[INDEX_MAX_KEYS]; bool indexNulls[INDEX_MAX_KEYS]; FormIndexDatum(indexInfo, slot, estate, indexValues, indexNulls); Relation columnarRelation = scan->rs_rd; IndexUniqueCheck indexUniqueCheck = indexInfo->ii_Unique ? UNIQUE_CHECK_YES : UNIQUE_CHECK_NO; index_insert(indexRelation, indexValues, indexNulls, columnarItemPointer, columnarRelation, indexUniqueCheck, false, indexInfo); validateIndexState->tups_inserted += 1; } } /* * TupleSortSkipSmallerItemPointers iterates given tupleSort until finding an * ItemPointer that is greater than or equal to given targetItemPointer and * returns that ItemPointer. * If such an ItemPointer does not exist, then returns invalid ItemPointer. * * Note that this function assumes given tupleSort doesn't have any NULL * Datum's. */ static ItemPointerData TupleSortSkipSmallerItemPointers(Tuplesortstate *tupleSort, ItemPointer targetItemPointer) { ItemPointerData tsItemPointerData; ItemPointerSetInvalid(&tsItemPointerData); while (!ItemPointerIsValid(&tsItemPointerData) || ItemPointerCompare(&tsItemPointerData, targetItemPointer) < 0) { bool forwardDirection = true; Datum *abbrev = NULL; Datum tsDatum; bool tsDatumIsNull; if (!tuplesort_getdatum(tupleSort, forwardDirection, false, &tsDatum, &tsDatumIsNull, abbrev)) { ItemPointerSetInvalid(&tsItemPointerData); break; } Assert(!tsDatumIsNull); itemptr_decode(&tsItemPointerData, DatumGetInt64(tsDatum)); #ifndef USE_FLOAT8_BYVAL /* * If int8 is pass-by-ref, we need to free Datum memory. * See tuplesort_getdatum function's comment. */ pfree(DatumGetPointer(tsDatum)); #endif } return tsItemPointerData; } static uint64 columnar_relation_size(Relation rel, ForkNumber forkNumber) { CheckCitusColumnarVersion(ERROR); uint64 nblocks = 0; /* InvalidForkNumber indicates returning the size for all forks */ if (forkNumber == InvalidForkNumber) { for (int i = 0; i < MAX_FORKNUM; i++) { nblocks += smgrnblocks(RelationGetSmgr(rel), i); } } else { nblocks = smgrnblocks(RelationGetSmgr(rel), forkNumber); } return nblocks * BLCKSZ; } static bool columnar_relation_needs_toast_table(Relation rel) { CheckCitusColumnarVersion(ERROR); return false; } static void columnar_estimate_rel_size(Relation rel, int32 *attr_widths, BlockNumber *pages, double *tuples, double *allvisfrac) { CheckCitusColumnarVersion(ERROR); *pages = smgrnblocks(RelationGetSmgr(rel), MAIN_FORKNUM); *tuples = ColumnarTableRowCount(rel); /* * Append-only, so everything is visible except in-progress or rolled-back * transactions. */ *allvisfrac = 1.0; get_rel_data_width(rel, attr_widths); } static bool columnar_scan_sample_next_block(TableScanDesc scan, SampleScanState *scanstate) { elog(ERROR, "columnar_scan_sample_next_block not implemented"); } static bool columnar_scan_sample_next_tuple(TableScanDesc scan, SampleScanState *scanstate, TupleTableSlot *slot) { elog(ERROR, "columnar_scan_sample_next_tuple not implemented"); } static void ColumnarXactCallback(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_COMMIT: case XACT_EVENT_PARALLEL_COMMIT: case XACT_EVENT_PREPARE: { /* nothing to do */ break; } case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: { DiscardWriteStateForAllRels(GetCurrentSubTransactionId(), 0); break; } case XACT_EVENT_PRE_COMMIT: case XACT_EVENT_PARALLEL_PRE_COMMIT: case XACT_EVENT_PRE_PREPARE: { FlushWriteStateForAllRels(GetCurrentSubTransactionId(), 0); break; } } } static void ColumnarSubXactCallback(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg) { switch (event) { case SUBXACT_EVENT_START_SUB: case SUBXACT_EVENT_COMMIT_SUB: { /* nothing to do */ break; } case SUBXACT_EVENT_ABORT_SUB: { DiscardWriteStateForAllRels(mySubid, parentSubid); break; } case SUBXACT_EVENT_PRE_COMMIT_SUB: { FlushWriteStateForAllRels(mySubid, parentSubid); break; } } } void columnar_tableam_init() { RegisterXactCallback(ColumnarXactCallback, NULL); RegisterSubXactCallback(ColumnarSubXactCallback, NULL); PrevObjectAccessHook = object_access_hook; object_access_hook = ColumnarTableAMObjectAccessHook; PrevProcessUtilityHook = ProcessUtility_hook ? ProcessUtility_hook : standard_ProcessUtility; ProcessUtility_hook = ColumnarProcessUtility; columnar_customscan_init(); TTSOpsColumnar = TTSOpsVirtual; TTSOpsColumnar.copy_heap_tuple = ColumnarSlotCopyHeapTuple; DefineCustomBoolVariable( "columnar.enable_version_checks", gettext_noop("Enables Version Check for Columnar"), NULL, &EnableVersionChecksColumnar, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); } /* * Get the number of chunks filtered out during the given scan. */ int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc) { ColumnarReadState *readState = columnarScanDesc->cs_readState; /* readState is initialized lazily */ if (readState != NULL) { return ColumnarReadChunkGroupsFiltered(readState); } else { return 0; } } /* * Implementation of TupleTableSlotOps.copy_heap_tuple for TTSOpsColumnar. */ static HeapTuple ColumnarSlotCopyHeapTuple(TupleTableSlot *slot) { Assert(!TTS_EMPTY(slot)); HeapTuple tuple = heap_form_tuple(slot->tts_tupleDescriptor, slot->tts_values, slot->tts_isnull); /* slot->tts_tid is filled in columnar_getnextslot */ tuple->t_self = slot->tts_tid; return tuple; } /* * ColumnarTableDropHook * * Clean-up resources for columnar tables. */ static void ColumnarTableDropHook(Oid relid) { /* * Lock relation to prevent it from being dropped and to avoid * race conditions in the next if block. */ LockRelationOid(relid, AccessShareLock); if (IsColumnarTableAmTable(relid)) { CheckCitusColumnarVersion(ERROR); /* * Drop metadata. No need to drop storage here since for * tableam tables storage is managed by postgres. */ Relation rel = table_open(relid, AccessExclusiveLock); RelFileLocator relfilelocator = rel->rd_locator; DeleteMetadataRows(rel); DeleteColumnarTableOptions(rel->rd_id, true); MarkRelfilenumberDropped(relfilelocator.relNumber, GetCurrentSubTransactionId()); /* keep the lock since we did physical changes to the relation */ table_close(rel, NoLock); } } /* * Reject AFTER ... FOR EACH ROW triggers on columnar tables. */ static void ColumnarTriggerCreateHook(Oid tgid) { /* * Fetch the pg_trigger tuple by the Oid of the trigger */ ScanKeyData skey[1]; Relation tgrel = table_open(TriggerRelationId, AccessShareLock); ScanKeyInit(&skey[0], Anum_pg_trigger_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(tgid)); SysScanDesc tgscan = systable_beginscan(tgrel, TriggerOidIndexId, true, SnapshotSelf, 1, skey); HeapTuple tgtup = systable_getnext(tgscan); if (!HeapTupleIsValid(tgtup)) { table_close(tgrel, AccessShareLock); return; } Form_pg_trigger tgrec = (Form_pg_trigger) GETSTRUCT(tgtup); Oid tgrelid = tgrec->tgrelid; int16 tgtype = tgrec->tgtype; systable_endscan(tgscan); table_close(tgrel, AccessShareLock); if (TRIGGER_FOR_ROW(tgtype) && TRIGGER_FOR_AFTER(tgtype) && IsColumnarTableAmTable(tgrelid)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "Foreign keys and AFTER ROW triggers are not supported for columnar tables"), errhint("Consider an AFTER STATEMENT trigger instead."))); } } /* * Capture create/drop events and dispatch to the proper action. */ static void ColumnarTableAMObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg) { if (PrevObjectAccessHook) { PrevObjectAccessHook(access, classId, objectId, subId, arg); } /* dispatch to the proper action */ if (access == OAT_DROP && classId == RelationRelationId && !OidIsValid(subId)) { ColumnarTableDropHook(objectId); } else if (access == OAT_POST_CREATE && classId == TriggerRelationId) { ColumnarTriggerCreateHook(objectId); } } /* * ColumnarProcessAlterTable - if modifying a columnar table, extract columnar * options and return the table's RangeVar. */ static RangeVar * ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions) { RangeVar *columnarRangeVar = NULL; Relation rel = relation_openrv_extended(alterTableStmt->relation, AccessShareLock, alterTableStmt->missing_ok); if (rel == NULL) { return NULL; } /* track separately in case of ALTER TABLE ... SET ACCESS METHOD */ bool srcIsColumnar = rel->rd_tableam == GetColumnarTableAmRoutine(); bool destIsColumnar = srcIsColumnar; ListCell *lc = NULL; foreach(lc, alterTableStmt->cmds) { AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(lc)); if (alterTableCmd->subtype == AT_SetRelOptions || alterTableCmd->subtype == AT_ResetRelOptions) { List *options = castNode(List, alterTableCmd->def); alterTableCmd->def = (Node *) ExtractColumnarRelOptions( options, columnarOptions); if (destIsColumnar) { columnarRangeVar = alterTableStmt->relation; } } else if (alterTableCmd->subtype == AT_SetAccessMethod) { if (columnarRangeVar || *columnarOptions) { ereport(ERROR, (errmsg( "ALTER TABLE cannot alter the access method after altering storage parameters"), errhint( "Specify SET ACCESS METHOD before storage parameters, or use separate ALTER TABLE commands."))); } destIsColumnar = (strcmp(alterTableCmd->name ? alterTableCmd->name : default_table_access_method, COLUMNAR_AM_NAME) == 0); if (srcIsColumnar && !destIsColumnar) { DeleteColumnarTableOptions(RelationGetRelid(rel), true); } } } relation_close(rel, NoLock); return columnarRangeVar; } /* * Utility hook for columnar tables. */ static void ColumnarProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag) { if (readOnlyTree) { pstmt = copyObject(pstmt); } Node *parsetree = pstmt->utilityStmt; RangeVar *columnarRangeVar = NULL; List *columnarOptions = NIL; switch (nodeTag(parsetree)) { case T_IndexStmt: { IndexStmt *indexStmt = (IndexStmt *) parsetree; Relation rel = relation_openrv(indexStmt->relation, indexStmt->concurrent ? ShareUpdateExclusiveLock : ShareLock); if (rel->rd_tableam == GetColumnarTableAmRoutine()) { CheckCitusColumnarVersion(ERROR); if (!ColumnarSupportsIndexAM(indexStmt->accessMethod)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported access method for the " "index on columnar table %s", RelationGetRelationName(rel)))); } } RelationClose(rel); break; } case T_CreateStmt: { CreateStmt *createStmt = castNode(CreateStmt, parsetree); bool no_op = false; if (createStmt->if_not_exists) { Oid existing_relid; /* use same check as transformCreateStmt */ (void) RangeVarGetAndCheckCreationNamespace( createStmt->relation, AccessShareLock, &existing_relid); no_op = OidIsValid(existing_relid); } if (!no_op && createStmt->accessMethod != NULL && !strcmp(createStmt->accessMethod, COLUMNAR_AM_NAME)) { columnarRangeVar = createStmt->relation; createStmt->options = ExtractColumnarRelOptions(createStmt->options, &columnarOptions); } break; } case T_CreateTableAsStmt: { CreateTableAsStmt *createTableAsStmt = castNode(CreateTableAsStmt, parsetree); IntoClause *into = createTableAsStmt->into; bool no_op = false; if (createTableAsStmt->if_not_exists) { Oid existing_relid; /* use same check as transformCreateStmt */ (void) RangeVarGetAndCheckCreationNamespace( into->rel, AccessShareLock, &existing_relid); no_op = OidIsValid(existing_relid); } if (!no_op && into->accessMethod != NULL && !strcmp(into->accessMethod, COLUMNAR_AM_NAME)) { columnarRangeVar = into->rel; into->options = ExtractColumnarRelOptions(into->options, &columnarOptions); } break; } case T_AlterTableStmt: { AlterTableStmt *alterTableStmt = castNode(AlterTableStmt, parsetree); columnarRangeVar = ColumnarProcessAlterTable(alterTableStmt, &columnarOptions); break; } default: { /* FALL THROUGH */ break; } } if (columnarOptions != NIL && columnarRangeVar == NULL) { ereport(ERROR, (errmsg("columnar storage parameters specified on non-columnar table"))); } if (IsA(parsetree, CreateExtensionStmt)) { CheckCitusColumnarCreateExtensionStmt(parsetree); } if (IsA(parsetree, AlterExtensionStmt)) { CheckCitusColumnarAlterExtensionStmt(parsetree); } PrevProcessUtilityHook(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); if (columnarOptions != NIL) { SetColumnarRelOptions(columnarRangeVar, columnarOptions); } } /* * ColumnarSupportsIndexAM returns true if indexAM with given name is * supported by columnar tables. */ bool ColumnarSupportsIndexAM(char *indexAMName) { return strncmp(indexAMName, "btree", NAMEDATALEN) == 0 || strncmp(indexAMName, "hash", NAMEDATALEN) == 0; } /* * IsColumnarTableAmTable returns true if relation has columnar_tableam * access method. This can be called before extension creation. */ bool IsColumnarTableAmTable(Oid relationId) { if (!OidIsValid(relationId)) { return false; } /* * Lock relation to prevent it from being dropped & * avoid race conditions. */ Relation rel = relation_open(relationId, AccessShareLock); bool result = rel->rd_tableam == GetColumnarTableAmRoutine(); relation_close(rel, NoLock); return result; } /* * CheckCitusColumnarCreateExtensionStmt determines whether can install * citus_columnar per given CREATE extension statment */ void CheckCitusColumnarCreateExtensionStmt(Node *parseTree) { CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt, parseTree); if (get_extension_oid("citus_columnar", true) == InvalidOid) { if (strcmp(createExtensionStmt->extname, "citus_columnar") == 0) { DefElem *newVersionValue = GetExtensionOption( createExtensionStmt->options, "new_version"); /*we are not allowed to install citus_columnar as version 11.1-0 by cx*/ if (newVersionValue) { const char *newVersion = defGetString(newVersionValue); if (strcmp(newVersion, CITUS_COLUMNAR_INTERNAL_VERSION) == 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "unsupported citus_columnar version 11.1-0"))); } } } } } /* * CheckCitusColumnarAlterExtensionStmt determines whether can alter * citus_columnar per given ALTER extension statment */ void CheckCitusColumnarAlterExtensionStmt(Node *parseTree) { AlterExtensionStmt *alterExtensionStmt = castNode(AlterExtensionStmt, parseTree); if (strcmp(alterExtensionStmt->extname, "citus_columnar") == 0) { DefElem *newVersionValue = GetExtensionOption(alterExtensionStmt->options, "new_version"); /*we are not allowed cx to downgrade citus_columnar to 11.1-0*/ if (newVersionValue) { const char *newVersion = defGetString(newVersionValue); if (strcmp(newVersion, CITUS_COLUMNAR_INTERNAL_VERSION) == 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported citus_columnar version 11.1-0"))); } } } } static const TableAmRoutine columnar_am_methods = { .type = T_TableAmRoutine, .slot_callbacks = columnar_slot_callbacks, .scan_begin = columnar_beginscan, .scan_end = columnar_endscan, .scan_rescan = columnar_rescan, .scan_getnextslot = columnar_getnextslot, .parallelscan_estimate = columnar_parallelscan_estimate, .parallelscan_initialize = columnar_parallelscan_initialize, .parallelscan_reinitialize = columnar_parallelscan_reinitialize, .index_fetch_begin = columnar_index_fetch_begin, .index_fetch_reset = columnar_index_fetch_reset, .index_fetch_end = columnar_index_fetch_end, .index_fetch_tuple = columnar_index_fetch_tuple, .tuple_fetch_row_version = columnar_fetch_row_version, .tuple_get_latest_tid = columnar_get_latest_tid, .tuple_tid_valid = columnar_tuple_tid_valid, .tuple_satisfies_snapshot = columnar_tuple_satisfies_snapshot, .index_delete_tuples = columnar_index_delete_tuples, .tuple_insert = columnar_tuple_insert, .tuple_insert_speculative = columnar_tuple_insert_speculative, .tuple_complete_speculative = columnar_tuple_complete_speculative, .multi_insert = columnar_multi_insert, .tuple_delete = columnar_tuple_delete, .tuple_update = columnar_tuple_update, .tuple_lock = columnar_tuple_lock, .finish_bulk_insert = columnar_finish_bulk_insert, .relation_set_new_filelocator = columnar_relation_set_new_filelocator, .relation_nontransactional_truncate = columnar_relation_nontransactional_truncate, .relation_copy_data = columnar_relation_copy_data, .relation_copy_for_cluster = columnar_relation_copy_for_cluster, .relation_vacuum = columnar_vacuum_rel, .scan_analyze_next_block = columnar_scan_analyze_next_block, .scan_analyze_next_tuple = columnar_scan_analyze_next_tuple, .index_build_range_scan = columnar_index_build_range_scan, .index_validate_scan = columnar_index_validate_scan, .relation_size = columnar_relation_size, .relation_needs_toast_table = columnar_relation_needs_toast_table, .relation_estimate_size = columnar_estimate_rel_size, #if PG_VERSION_NUM < PG_VERSION_18 /* these two fields were removed in PG 18 */ .scan_bitmap_next_block = NULL, .scan_bitmap_next_tuple = NULL, #endif .scan_sample_next_block = columnar_scan_sample_next_block, .scan_sample_next_tuple = columnar_scan_sample_next_tuple }; const TableAmRoutine * GetColumnarTableAmRoutine(void) { return &columnar_am_methods; } PG_FUNCTION_INFO_V1(columnar_handler); Datum columnar_handler(PG_FUNCTION_ARGS) { PG_RETURN_POINTER(&columnar_am_methods); } /* * detoast_values * * Detoast and decompress all values. If there's no work to do, return * original pointer; otherwise return a newly-allocated values array. Should * be called in per-tuple context. */ static Datum * detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull) { int natts = tupleDesc->natts; /* copy on write to optimize for case where nothing is toasted */ Datum *values = orig_values; for (int i = 0; i < tupleDesc->natts; i++) { if (!isnull[i] && TupleDescAttr(tupleDesc, i)->attlen == -1 && VARATT_IS_EXTENDED(values[i])) { /* make a copy */ if (values == orig_values) { values = palloc(sizeof(Datum) * natts); /* * We use IGNORE-BANNED here since we don't want to limit * size of the buffer that holds the datum array to RSIZE_MAX * unnecessarily. */ memcpy(values, orig_values, sizeof(Datum) * natts); /* IGNORE-BANNED */ } /* will be freed when per-tuple context is reset */ struct varlena *new_value = (struct varlena *) DatumGetPointer(values[i]); new_value = detoast_attr(new_value); values[i] = PointerGetDatum(new_value); } } return values; } /* * ColumnarCheckLogicalReplication throws an error if the relation is * part of any publication. This should be called before any write to * a columnar table, because columnar changes are not replicated with * logical replication (similar to a row table without a replica * identity). */ static void ColumnarCheckLogicalReplication(Relation rel) { bool pubActionInsert = false; if (!is_publishable_relation(rel)) { return; } { PublicationDesc pubdesc; RelationBuildPublicationDesc(rel, &pubdesc); pubActionInsert = pubdesc.pubactions.pubinsert; } if (pubActionInsert) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot insert into columnar table that is a part of a publication"))); } } /* * alter_columnar_table_set() * * Deprecated in 11.1-1: should issue ALTER TABLE ... SET instead. Function * still available, but implemented in PL/pgSQL instead of C. * * C code is removed -- the symbol may still be required in some * upgrade/downgrade paths, but it should not be called. */ PG_FUNCTION_INFO_V1(alter_columnar_table_set); Datum alter_columnar_table_set(PG_FUNCTION_ARGS) { elog(ERROR, "alter_columnar_table_set is deprecated"); } /* * alter_columnar_table_reset() * * Deprecated in 11.1-1: should issue ALTER TABLE ... RESET instead. Function * still available, but implemented in PL/pgSQL instead of C. * * C code is removed -- the symbol may still be required in some * upgrade/downgrade paths, but it should not be called. */ PG_FUNCTION_INFO_V1(alter_columnar_table_reset); Datum alter_columnar_table_reset(PG_FUNCTION_ARGS) { elog(ERROR, "alter_columnar_table_reset is deprecated"); } /* * upgrade_columnar_storage - upgrade columnar storage to the current * version. * * DDL: * CREATE OR REPLACE FUNCTION upgrade_columnar_storage(rel regclass) * RETURNS VOID * STRICT * LANGUAGE c AS 'MODULE_PATHNAME', 'upgrade_columnar_storage'; */ PG_FUNCTION_INFO_V1(upgrade_columnar_storage); Datum upgrade_columnar_storage(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); /* * ACCESS EXCLUSIVE LOCK is not required by the low-level routines, so we * can take only an ACCESS SHARE LOCK. But all access to non-current * columnar tables will fail anyway, so it's better to take ACCESS * EXCLUSIVE LOCK now. */ Relation rel = table_open(relid, AccessExclusiveLock); if (!IsColumnarTableAmTable(relid)) { ereport(ERROR, (errmsg("table %s is not a columnar table", quote_identifier(RelationGetRelationName(rel))))); } ColumnarStorageUpdateIfNeeded(rel, true); table_close(rel, AccessExclusiveLock); PG_RETURN_VOID(); } /* * downgrade_columnar_storage - downgrade columnar storage to the * current version. * * DDL: * CREATE OR REPLACE FUNCTION downgrade_columnar_storage(rel regclass) * RETURNS VOID * STRICT * LANGUAGE c AS 'MODULE_PATHNAME', 'downgrade_columnar_storage'; */ PG_FUNCTION_INFO_V1(downgrade_columnar_storage); Datum downgrade_columnar_storage(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); /* * ACCESS EXCLUSIVE LOCK is not required by the low-level routines, so we * can take only an ACCESS SHARE LOCK. But all access to non-current * columnar tables will fail anyway, so it's better to take ACCESS * EXCLUSIVE LOCK now. */ Relation rel = table_open(relid, AccessExclusiveLock); if (!IsColumnarTableAmTable(relid)) { ereport(ERROR, (errmsg("table %s is not a columnar table", quote_identifier(RelationGetRelationName(rel))))); } ColumnarStorageUpdateIfNeeded(rel, false); table_close(rel, AccessExclusiveLock); PG_RETURN_VOID(); } /* * Code to check the Citus Version, helps remove dependency from Citus */ /* * CitusColumnarHasBeenLoaded returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. The result is cached as this is called very frequently. */ bool CitusColumnarHasBeenLoaded(void) { if (!extensionLoadedColumnar || creating_extension) { /* * Refresh if we have not determined whether the extension has been * loaded yet, or in case of ALTER EXTENSION since we want to treat * Citus as "not loaded" during ALTER EXTENSION citus. */ bool extensionLoaded = CitusColumnarHasBeenLoadedInternal(); extensionLoadedColumnar = extensionLoaded; } return extensionLoadedColumnar; } /* * CitusColumnarHasBeenLoadedInternal returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. */ static bool CitusColumnarHasBeenLoadedInternal(void) { if (IsBinaryUpgrade) { /* never use Citus logic during pg_upgrade */ return false; } Oid citusExtensionOid = get_extension_oid("citus", true); if (citusExtensionOid == InvalidOid) { /* Citus extension does not exist yet */ return false; } if (creating_extension && CurrentExtensionObject == citusExtensionOid) { /* * We do not use Citus hooks during CREATE/ALTER EXTENSION citus * since the objects used by the C code might be not be there yet. */ return false; } /* citus extension exists and has been created */ return true; } /* * CheckCitusColumnarVersion checks whether there is a version mismatch between the * available version and the loaded version or between the installed version * and the loaded version. Returns true if compatible, false otherwise. * * As a side effect, this function also sets citusVersionKnownCompatible_Columnar global * variable to true which reduces version check cost of next calls. */ bool CheckCitusColumnarVersion(int elevel) { if (citusVersionKnownCompatibleColumnar || !CitusColumnarHasBeenLoaded() || !EnableVersionChecksColumnar) { return true; } if (CheckAvailableVersionColumnar(elevel) && CheckInstalledVersionColumnar(elevel)) { citusVersionKnownCompatibleColumnar = true; return true; } else { return false; } } /* * CheckAvailableVersion compares CITUS_EXTENSIONVERSION and the currently * available version from the citus.control file. If they are not compatible, * this function logs an error with the specified elevel and returns false, * otherwise it returns true. */ bool CheckAvailableVersionColumnar(int elevel) { if (!EnableVersionChecksColumnar) { return true; } char *availableVersion = AvailableExtensionVersionColumnar(); if (!MajorVersionsCompatibleColumnar(availableVersion, CITUS_EXTENSIONVERSION)) { ereport(elevel, (errmsg("loaded Citus library version differs from latest " "available extension version"), errdetail("Loaded library requires %s, but the latest control " "file specifies %s.", CITUS_MAJORVERSION, availableVersion), errhint("Restart the database to load the latest Citus " "library."))); pfree(availableVersion); return false; } pfree(availableVersion); return true; } /* * CheckInstalledVersion compares CITUS_EXTENSIONVERSION and the * extension's current version from the pg_extension catalog table. If they * are not compatible, this function logs an error with the specified elevel, * otherwise it returns true. */ static bool CheckInstalledVersionColumnar(int elevel) { Assert(CitusColumnarHasBeenLoaded()); Assert(EnableVersionChecksColumnar); char *installedVersion = InstalledExtensionVersionColumnar(); if (!MinorVersionsCompatibleRelaxedColumnar(installedVersion, CITUS_EXTENSIONVERSION)) { ereport(elevel, (errmsg("loaded Citus library version differs from installed " "extension version"), errdetail("Loaded library requires %s, but the installed " "extension version is %s.", CITUS_MAJORVERSION, installedVersion), errhint("Run ALTER EXTENSION citus UPDATE and try again."))); pfree(installedVersion); return false; } pfree(installedVersion); return true; } /* * MajorVersionsCompatible checks whether both versions are compatible. They * are if major and minor version numbers match, the schema version is * ignored. Returns true if compatible, false otherwise. */ bool MajorVersionsCompatibleColumnar(char *leftVersion, char *rightVersion) { const char schemaVersionSeparator = '-'; char *leftSeperatorPosition = strchr(leftVersion, schemaVersionSeparator); char *rightSeperatorPosition = strchr(rightVersion, schemaVersionSeparator); int leftComparisionLimit = 0; int rightComparisionLimit = 0; if (leftSeperatorPosition != NULL) { leftComparisionLimit = leftSeperatorPosition - leftVersion; } else { leftComparisionLimit = strlen(leftVersion); } if (rightSeperatorPosition != NULL) { rightComparisionLimit = rightSeperatorPosition - rightVersion; } else { rightComparisionLimit = strlen(rightVersion); } /* we can error out early if hypens are not in the same position */ if (leftComparisionLimit != rightComparisionLimit) { return false; } return strncmp(leftVersion, rightVersion, leftComparisionLimit) == 0; } /* * ParseVersionComponent parses the integer at the current position and * advances endPtr past the parsed digits to the next character. */ static int ParseVersionComponent(const char *version, char **endPtr) { errno = 0; long int val = strtol(version, endPtr, 10); if (errno == ERANGE || val > INT_MAX || val < INT_MIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("Invalid integer in version string"))); } return (int) val; } /* * MinorVersionsCompatibleRelaxedColumnar checks if two versions have the same major * version and their minor versions differ by at most 1. The schema version * (after '-') is ignored. Returns true if compatible, false otherwise. * * Version format expected: "major.minor-schema" (e.g., "13.1-2") */ bool MinorVersionsCompatibleRelaxedColumnar(char *leftVersion, char *rightVersion) { char *leftSep; char *rightSep; int leftMajor = ParseVersionComponent(leftVersion, &leftSep); int rightMajor = ParseVersionComponent(rightVersion, &rightSep); if (leftMajor != rightMajor) { return false; } int leftMinor = (*leftSep == '.') ? ParseVersionComponent(leftSep + 1, &leftSep) : 0; int rightMinor = (*rightSep == '.') ? ParseVersionComponent(rightSep + 1, &rightSep) : 0; int diff = leftMinor - rightMinor; return diff >= -1 && diff <= 1; } /* * AvailableExtensionVersion returns the Citus version from citus.control file. It also * saves the result, thus consecutive calls to CitusExtensionAvailableVersion will * not read the citus.control file again. */ static char * AvailableExtensionVersionColumnar(void) { LOCAL_FCINFO(fcinfo, 0); FmgrInfo flinfo; bool goForward = true; bool doCopy = false; char *availableExtensionVersion; EState *estate = CreateExecutorState(); ReturnSetInfo *extensionsResultSet = makeNode(ReturnSetInfo); extensionsResultSet->econtext = GetPerTupleExprContext(estate); extensionsResultSet->allowedModes = SFRM_Materialize; fmgr_info(F_PG_AVAILABLE_EXTENSIONS, &flinfo); InitFunctionCallInfoData(*fcinfo, &flinfo, 0, InvalidOid, NULL, (Node *) extensionsResultSet); /* pg_available_extensions returns result set containing all available extensions */ (*pg_available_extensions)(fcinfo); TupleTableSlot *tupleTableSlot = MakeSingleTupleTableSlot( extensionsResultSet->setDesc, &TTSOpsMinimalTuple); bool hasTuple = tuplestore_gettupleslot(extensionsResultSet->setResult, goForward, doCopy, tupleTableSlot); while (hasTuple) { bool isNull = false; Datum extensionNameDatum = slot_getattr(tupleTableSlot, 1, &isNull); char *extensionName = NameStr(*DatumGetName(extensionNameDatum)); if (strcmp(extensionName, "citus") == 0) { Datum availableVersion = slot_getattr(tupleTableSlot, 2, &isNull); availableExtensionVersion = text_to_cstring(DatumGetTextPP(availableVersion)); ExecClearTuple(tupleTableSlot); ExecDropSingleTupleTableSlot(tupleTableSlot); return availableExtensionVersion; } ExecClearTuple(tupleTableSlot); hasTuple = tuplestore_gettupleslot(extensionsResultSet->setResult, goForward, doCopy, tupleTableSlot); } ExecDropSingleTupleTableSlot(tupleTableSlot); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not found"))); return NULL; /* keep compiler happy */ } /* * InstalledExtensionVersion returns the Citus version in PostgreSQL pg_extension table. */ static char * InstalledExtensionVersionColumnar(void) { ScanKeyData entry[1]; char *installedExtensionVersion = NULL; Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum("citus")); SysScanDesc scandesc = systable_beginscan(relation, ExtensionNameIndexId, true, NULL, 1, entry); HeapTuple extensionTuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(extensionTuple)) { int extensionIndex = Anum_pg_extension_extversion; TupleDesc tupleDescriptor = RelationGetDescr(relation); bool isNull = false; Datum installedVersion = heap_getattr(extensionTuple, extensionIndex, tupleDescriptor, &isNull); if (isNull) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension version is null"))); } installedExtensionVersion = text_to_cstring(DatumGetTextPP(installedVersion)); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not loaded"))); } systable_endscan(scandesc); table_close(relation, AccessShareLock); return installedExtensionVersion; } /* * GetExtensionOption returns DefElem * node with "defname" from "options" list */ DefElem * GetExtensionOption(List *extensionOptions, const char *defname) { DefElem *defElement = NULL; foreach_declared_ptr(defElement, extensionOptions) { if (IsA(defElement, DefElem) && strncmp(defElement->defname, defname, NAMEDATALEN) == 0) { return defElement; } } return NULL; } ================================================ FILE: src/backend/columnar/columnar_writer.c ================================================ /*------------------------------------------------------------------------- * * columnar_writer.c * * This file contains function definitions for writing columnar tables. This * includes the logic for writing file level metadata, writing row stripes, * and calculating chunk skip nodes. * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "safe_lib.h" #include "access/heapam.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "storage/fd.h" #include "storage/relfilelocator.h" #include "storage/smgr.h" #include "utils/guc.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/relfilenumbermap.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "columnar/columnar.h" #include "columnar/columnar_storage.h" #include "columnar/columnar_version_compat.h" struct ColumnarWriteState { TupleDesc tupleDescriptor; FmgrInfo **comparisonFunctionArray; RelFileLocator relfilelocator; /* * We can't rely on RelidByRelfilenumber for temp tables since * PG18(it was backpatched through PG13). */ Oid temp_relid; MemoryContext stripeWriteContext; MemoryContext perTupleContext; StripeBuffers *stripeBuffers; StripeSkipList *stripeSkipList; EmptyStripeReservation *emptyStripeReservation; ColumnarOptions options; ChunkData *chunkData; List *chunkGroupRowCounts; /* * compressionBuffer buffer is used as temporary storage during * data value compression operation. It is kept here to minimize * memory allocations. It lives in stripeWriteContext and gets * deallocated when memory context is reset. */ StringInfo compressionBuffer; }; static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 chunkRowCount, uint32 columnCount); static StripeSkipList * CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 chunkRowCount, uint32 columnCount); static void FlushStripe(ColumnarWriteState *writeState); static StringInfo SerializeBoolArray(bool *boolArray, uint32 boolArrayLength); static void SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue, int datumTypeLength, char datumTypeAlign); static void SerializeChunkData(ColumnarWriteState *writeState, uint32 chunkIndex, uint32 rowCount); static void UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode, Datum columnValue, bool columnTypeByValue, int columnTypeLength, Oid columnCollation, FmgrInfo *comparisonFunction); static Datum DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength); static StringInfo CopyStringInfo(StringInfo sourceString); /* * ColumnarBeginWrite initializes a columnar data load operation and returns a table * handle. This handle should be used for adding the row values and finishing the * data load operation. */ ColumnarWriteState * ColumnarBeginWrite(Relation rel, ColumnarOptions options, TupleDesc tupleDescriptor) { RelFileLocator relfilelocator = rel->rd_locator; /* get comparison function pointers for each of the columns */ uint32 columnCount = tupleDescriptor->natts; FmgrInfo **comparisonFunctionArray = palloc0(columnCount * sizeof(FmgrInfo *)); for (uint32 columnIndex = 0; columnIndex < columnCount; columnIndex++) { FmgrInfo *comparisonFunction = NULL; FormData_pg_attribute *attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); if (!attributeForm->attisdropped) { Oid typeId = attributeForm->atttypid; comparisonFunction = GetFunctionInfoOrNull(typeId, BTREE_AM_OID, BTORDER_PROC); } comparisonFunctionArray[columnIndex] = comparisonFunction; } /* * We allocate all stripe specific data in the stripeWriteContext, and * reset this memory context once we have flushed the stripe to the file. * This is to avoid memory leaks. */ MemoryContext stripeWriteContext = AllocSetContextCreate(CurrentMemoryContext, "Stripe Write Memory Context", ALLOCSET_DEFAULT_SIZES); bool *columnMaskArray = palloc(columnCount * sizeof(bool)); memset(columnMaskArray, true, columnCount * sizeof(bool)); ChunkData *chunkData = CreateEmptyChunkData(columnCount, columnMaskArray, options.chunkRowCount); ColumnarWriteState *writeState = palloc0(sizeof(ColumnarWriteState)); writeState->relfilelocator = relfilelocator; writeState->temp_relid = RelationPrecomputeOid(rel); writeState->options = options; writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor); writeState->comparisonFunctionArray = comparisonFunctionArray; writeState->stripeBuffers = NULL; writeState->stripeSkipList = NULL; writeState->emptyStripeReservation = NULL; writeState->stripeWriteContext = stripeWriteContext; writeState->chunkData = chunkData; writeState->compressionBuffer = NULL; writeState->perTupleContext = AllocSetContextCreate(CurrentMemoryContext, "Columnar per tuple context", ALLOCSET_DEFAULT_SIZES); return writeState; } /* * ColumnarWriteRow adds a row to the columnar table. If the stripe is not initialized, * we create structures to hold stripe data and skip list. Then, we serialize and * append data to serialized value buffer for each of the columns and update * corresponding skip nodes. Then, whole chunk data is compressed at every * rowChunkCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush * the stripe, and add its metadata to the table footer. * * Returns the "row number" assigned to written row. */ uint64 ColumnarWriteRow(ColumnarWriteState *writeState, Datum *columnValues, bool *columnNulls) { uint32 columnIndex = 0; StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeSkipList *stripeSkipList = writeState->stripeSkipList; uint32 columnCount = writeState->tupleDescriptor->natts; ColumnarOptions *options = &writeState->options; const uint32 chunkRowCount = options->chunkRowCount; ChunkData *chunkData = writeState->chunkData; MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); if (stripeBuffers == NULL) { stripeBuffers = CreateEmptyStripeBuffers(options->stripeRowCount, chunkRowCount, columnCount); stripeSkipList = CreateEmptyStripeSkipList(options->stripeRowCount, chunkRowCount, columnCount); writeState->stripeBuffers = stripeBuffers; writeState->stripeSkipList = stripeSkipList; writeState->compressionBuffer = makeStringInfo(); Oid relationId = ColumnarRelationId(writeState->temp_relid, writeState->relfilelocator); Relation relation = relation_open(relationId, NoLock); writeState->emptyStripeReservation = ReserveEmptyStripe(relation, columnCount, chunkRowCount, options->stripeRowCount); relation_close(relation, NoLock); /* * serializedValueBuffer lives in stripe write memory context so it needs to be * initialized when the stripe is created. */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { chunkData->valueBufferArray[columnIndex] = makeStringInfo(); } } uint32 chunkIndex = stripeBuffers->rowCount / chunkRowCount; uint32 chunkRowIndex = stripeBuffers->rowCount % chunkRowCount; for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { ColumnChunkSkipNode **chunkSkipNodeArray = stripeSkipList->chunkSkipNodeArray; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[columnIndex][chunkIndex]; if (columnNulls[columnIndex]) { chunkData->existsArray[columnIndex][chunkRowIndex] = false; } else { FmgrInfo *comparisonFunction = writeState->comparisonFunctionArray[columnIndex]; Form_pg_attribute attributeForm = TupleDescAttr(writeState->tupleDescriptor, columnIndex); bool columnTypeByValue = attributeForm->attbyval; int columnTypeLength = attributeForm->attlen; Oid columnCollation = attributeForm->attcollation; char columnTypeAlign = attributeForm->attalign; chunkData->existsArray[columnIndex][chunkRowIndex] = true; SerializeSingleDatum(chunkData->valueBufferArray[columnIndex], columnValues[columnIndex], columnTypeByValue, columnTypeLength, columnTypeAlign); UpdateChunkSkipNodeMinMax(chunkSkipNode, columnValues[columnIndex], columnTypeByValue, columnTypeLength, columnCollation, comparisonFunction); } chunkSkipNode->rowCount++; } stripeSkipList->chunkCount = chunkIndex + 1; /* last row of the chunk is inserted serialize the chunk */ if (chunkRowIndex == chunkRowCount - 1) { SerializeChunkData(writeState, chunkIndex, chunkRowCount); } uint64 writtenRowNumber = writeState->emptyStripeReservation->stripeFirstRowNumber + stripeBuffers->rowCount; stripeBuffers->rowCount++; if (stripeBuffers->rowCount >= options->stripeRowCount) { ColumnarFlushPendingWrites(writeState); } MemoryContextSwitchTo(oldContext); return writtenRowNumber; } /* * ColumnarEndWrite finishes a columnar data load operation. If we have an unflushed * stripe, we flush it. */ void ColumnarEndWrite(ColumnarWriteState *writeState) { ColumnarFlushPendingWrites(writeState); MemoryContextDelete(writeState->stripeWriteContext); pfree(writeState->comparisonFunctionArray); FreeChunkData(writeState->chunkData); pfree(writeState); } void ColumnarFlushPendingWrites(ColumnarWriteState *writeState) { StripeBuffers *stripeBuffers = writeState->stripeBuffers; if (stripeBuffers != NULL) { MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); FlushStripe(writeState); MemoryContextReset(writeState->stripeWriteContext); /* set stripe data and skip list to NULL so they are recreated next time */ writeState->stripeBuffers = NULL; writeState->stripeSkipList = NULL; MemoryContextSwitchTo(oldContext); } } /* * ColumnarWritePerTupleContext * * Return per-tuple context for columnar write operation. */ MemoryContext ColumnarWritePerTupleContext(ColumnarWriteState *state) { return state->perTupleContext; } /* * CreateEmptyStripeBuffers allocates an empty StripeBuffers structure with the given * column count. */ static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 chunkRowCount, uint32 columnCount) { uint32 columnIndex = 0; uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1; ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *)); for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { uint32 chunkIndex = 0; ColumnChunkBuffers **chunkBuffersArray = palloc0(maxChunkCount * sizeof(ColumnChunkBuffers *)); for (chunkIndex = 0; chunkIndex < maxChunkCount; chunkIndex++) { chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers)); chunkBuffersArray[chunkIndex]->existsBuffer = NULL; chunkBuffersArray[chunkIndex]->valueBuffer = NULL; chunkBuffersArray[chunkIndex]->valueCompressionType = COMPRESSION_NONE; } columnBuffersArray[columnIndex] = palloc0(sizeof(ColumnBuffers)); columnBuffersArray[columnIndex]->chunkBuffersArray = chunkBuffersArray; } StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers)); stripeBuffers->columnBuffersArray = columnBuffersArray; stripeBuffers->columnCount = columnCount; stripeBuffers->rowCount = 0; return stripeBuffers; } /* * CreateEmptyStripeSkipList allocates an empty StripeSkipList structure with * the given column count. This structure has enough chunks to hold statistics * for stripeMaxRowCount rows. */ static StripeSkipList * CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 chunkRowCount, uint32 columnCount) { uint32 columnIndex = 0; uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1; ColumnChunkSkipNode **chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *)); for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { chunkSkipNodeArray[columnIndex] = palloc0(maxChunkCount * sizeof(ColumnChunkSkipNode)); } StripeSkipList *stripeSkipList = palloc0(sizeof(StripeSkipList)); stripeSkipList->columnCount = columnCount; stripeSkipList->chunkCount = 0; stripeSkipList->chunkSkipNodeArray = chunkSkipNodeArray; return stripeSkipList; } /* * FlushStripe flushes current stripe data into the file. The function first ensures * the last data chunk for each column is properly serialized and compressed. Then, * the function creates the skip list and footer buffers. Finally, the function * flushes the skip list, data, and footer buffers to the file. */ static void FlushStripe(ColumnarWriteState *writeState) { uint32 columnIndex = 0; uint32 chunkIndex = 0; StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeSkipList *stripeSkipList = writeState->stripeSkipList; ColumnChunkSkipNode **columnSkipNodeArray = stripeSkipList->chunkSkipNodeArray; TupleDesc tupleDescriptor = writeState->tupleDescriptor; uint32 columnCount = tupleDescriptor->natts; uint32 chunkCount = stripeSkipList->chunkCount; uint32 chunkRowCount = writeState->options.chunkRowCount; uint32 lastChunkIndex = stripeBuffers->rowCount / chunkRowCount; uint32 lastChunkRowCount = stripeBuffers->rowCount % chunkRowCount; uint64 stripeSize = 0; uint64 stripeRowCount = stripeBuffers->rowCount; elog(DEBUG1, "Flushing Stripe of size %d", stripeBuffers->rowCount); Oid relationId = ColumnarRelationId(writeState->temp_relid, writeState->relfilelocator); Relation relation = relation_open(relationId, NoLock); /* * check if the last chunk needs serialization , the last chunk was not serialized * if it was not full yet, e.g. (rowCount > 0) */ if (lastChunkRowCount > 0) { SerializeChunkData(writeState, lastChunkIndex, lastChunkRowCount); } /* update buffer sizes in stripe skip list */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { ColumnChunkSkipNode *chunkSkipNodeArray = columnSkipNodeArray[columnIndex]; ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; uint64 existsBufferSize = chunkBuffers->existsBuffer->len; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex]; chunkSkipNode->existsChunkOffset = stripeSize; chunkSkipNode->existsLength = existsBufferSize; stripeSize += existsBufferSize; } for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) { ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; uint64 valueBufferSize = chunkBuffers->valueBuffer->len; CompressionType valueCompressionType = chunkBuffers->valueCompressionType; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex]; chunkSkipNode->valueChunkOffset = stripeSize; chunkSkipNode->valueLength = valueBufferSize; chunkSkipNode->valueCompressionType = valueCompressionType; chunkSkipNode->valueCompressionLevel = writeState->options.compressionLevel; chunkSkipNode->decompressedValueSize = chunkBuffers->decompressedValueSize; stripeSize += valueBufferSize; } } StripeMetadata *stripeMetadata = CompleteStripeReservation(relation, writeState->emptyStripeReservation->stripeId, stripeSize, stripeRowCount, chunkCount); uint64 currentFileOffset = stripeMetadata->fileOffset; /* * Each stripe has only one section: * Data section, in which we store data for each column continuously. * We store data for each for each column in chunks. For each chunk, we * store two buffers: "exists" buffer, and "value" buffer. "exists" buffer * tells which values are not NULL. "value" buffer contains values for * present values. For each column, we first store all "exists" buffers, * and then all "value" buffers. */ /* flush the data buffers */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; StringInfo existsBuffer = chunkBuffers->existsBuffer; ColumnarStorageWrite(relation, currentFileOffset, existsBuffer->data, existsBuffer->len); currentFileOffset += existsBuffer->len; } for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++) { ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; StringInfo valueBuffer = chunkBuffers->valueBuffer; ColumnarStorageWrite(relation, currentFileOffset, valueBuffer->data, valueBuffer->len); currentFileOffset += valueBuffer->len; } } SaveChunkGroups(writeState->temp_relid, writeState->relfilelocator, stripeMetadata->id, writeState->chunkGroupRowCounts); SaveStripeSkipList(writeState->temp_relid, writeState->relfilelocator, stripeMetadata->id, stripeSkipList, tupleDescriptor); writeState->chunkGroupRowCounts = NIL; relation_close(relation, NoLock); } /* * SerializeBoolArray serializes the given boolean array and returns the result * as a StringInfo. This function packs every 8 boolean values into one byte. */ static StringInfo SerializeBoolArray(bool *boolArray, uint32 boolArrayLength) { uint32 boolArrayIndex = 0; uint32 byteCount = ((boolArrayLength * sizeof(bool)) + (8 - sizeof(bool))) / 8; StringInfo boolArrayBuffer = makeStringInfo(); enlargeStringInfo(boolArrayBuffer, byteCount); boolArrayBuffer->len = byteCount; memset(boolArrayBuffer->data, 0, byteCount); for (boolArrayIndex = 0; boolArrayIndex < boolArrayLength; boolArrayIndex++) { if (boolArray[boolArrayIndex]) { uint32 byteIndex = boolArrayIndex / 8; uint32 bitIndex = boolArrayIndex % 8; boolArrayBuffer->data[byteIndex] |= (1 << bitIndex); } } return boolArrayBuffer; } /* * SerializeSingleDatum serializes the given datum value and appends it to the * provided string info buffer. * * Since we don't want to limit datum buffer size to RSIZE_MAX unnecessarily, * we use memcpy instead of memcpy_s several places in this function. */ static void SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue, int datumTypeLength, char datumTypeAlign) { uint32 datumLength = att_addlength_datum(0, datumTypeLength, datum); uint32 datumLengthAligned = att_align_nominal(datumLength, datumTypeAlign); enlargeStringInfo(datumBuffer, datumLengthAligned); char *currentDatumDataPointer = datumBuffer->data + datumBuffer->len; memset(currentDatumDataPointer, 0, datumLengthAligned); if (datumTypeLength > 0) { if (datumTypeByValue) { store_att_byval(currentDatumDataPointer, datum, datumTypeLength); } else { memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumTypeLength); /* IGNORE-BANNED */ } } else { Assert(!datumTypeByValue); memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */ } datumBuffer->len += datumLengthAligned; } /* * SerializeChunkData serializes and compresses chunk data at given chunk index with given * compression type for every column. */ static void SerializeChunkData(ColumnarWriteState *writeState, uint32 chunkIndex, uint32 rowCount) { uint32 columnIndex = 0; StripeBuffers *stripeBuffers = writeState->stripeBuffers; ChunkData *chunkData = writeState->chunkData; CompressionType requestedCompressionType = writeState->options.compressionType; int compressionLevel = writeState->options.compressionLevel; const uint32 columnCount = stripeBuffers->columnCount; StringInfo compressionBuffer = writeState->compressionBuffer; writeState->chunkGroupRowCounts = lappend_int(writeState->chunkGroupRowCounts, rowCount); /* serialize exist values, data values are already serialized */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; chunkBuffers->existsBuffer = SerializeBoolArray(chunkData->existsArray[columnIndex], rowCount); } /* * check and compress value buffers, if a value buffer is not compressable * then keep it as uncompressed, store compression information. */ for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex]; CompressionType actualCompressionType = COMPRESSION_NONE; StringInfo serializedValueBuffer = chunkData->valueBufferArray[columnIndex]; Assert(requestedCompressionType >= 0 && requestedCompressionType < COMPRESSION_COUNT); chunkBuffers->decompressedValueSize = chunkData->valueBufferArray[columnIndex]->len; /* * if serializedValueBuffer is be compressed, update serializedValueBuffer * with compressed data and store compression type. */ bool compressed = CompressBuffer(serializedValueBuffer, compressionBuffer, requestedCompressionType, compressionLevel); if (compressed) { serializedValueBuffer = compressionBuffer; actualCompressionType = requestedCompressionType; } /* store (compressed) value buffer */ chunkBuffers->valueCompressionType = actualCompressionType; chunkBuffers->valueBuffer = CopyStringInfo(serializedValueBuffer); /* valueBuffer needs to be reset for next chunk's data */ resetStringInfo(chunkData->valueBufferArray[columnIndex]); } } /* * UpdateChunkSkipNodeMinMax takes the given column value, and checks if this * value falls outside the range of minimum/maximum values of the given column * chunk skip node. If it does, the function updates the column chunk skip node * accordingly. */ static void UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode, Datum columnValue, bool columnTypeByValue, int columnTypeLength, Oid columnCollation, FmgrInfo *comparisonFunction) { bool hasMinMax = chunkSkipNode->hasMinMax; Datum previousMinimum = chunkSkipNode->minimumValue; Datum previousMaximum = chunkSkipNode->maximumValue; Datum currentMinimum = 0; Datum currentMaximum = 0; /* if type doesn't have a comparison function, skip min/max values */ if (comparisonFunction == NULL) { return; } if (!hasMinMax) { currentMinimum = DatumCopy(columnValue, columnTypeByValue, columnTypeLength); currentMaximum = DatumCopy(columnValue, columnTypeByValue, columnTypeLength); } else { Datum minimumComparisonDatum = FunctionCall2Coll(comparisonFunction, columnCollation, columnValue, previousMinimum); Datum maximumComparisonDatum = FunctionCall2Coll(comparisonFunction, columnCollation, columnValue, previousMaximum); int minimumComparison = DatumGetInt32(minimumComparisonDatum); int maximumComparison = DatumGetInt32(maximumComparisonDatum); if (minimumComparison < 0) { currentMinimum = DatumCopy(columnValue, columnTypeByValue, columnTypeLength); } else { currentMinimum = previousMinimum; } if (maximumComparison > 0) { currentMaximum = DatumCopy(columnValue, columnTypeByValue, columnTypeLength); } else { currentMaximum = previousMaximum; } } chunkSkipNode->hasMinMax = true; chunkSkipNode->minimumValue = currentMinimum; chunkSkipNode->maximumValue = currentMaximum; } /* Creates a copy of the given datum. */ static Datum DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength) { Datum datumCopy = 0; if (datumTypeByValue) { datumCopy = datum; } else { uint32 datumLength = att_addlength_datum(0, datumTypeLength, datum); char *datumData = palloc0(datumLength); /* * We use IGNORE-BANNED here since we don't want to limit datum size to * RSIZE_MAX unnecessarily. */ memcpy(datumData, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */ datumCopy = PointerGetDatum(datumData); } return datumCopy; } /* * CopyStringInfo creates a deep copy of given source string allocating only needed * amount of memory. */ static StringInfo CopyStringInfo(StringInfo sourceString) { StringInfo targetString = palloc0(sizeof(StringInfoData)); if (sourceString->len > 0) { targetString->data = palloc0(sourceString->len); targetString->len = sourceString->len; targetString->maxlen = sourceString->len; /* * We use IGNORE-BANNED here since we don't want to limit string * buffer size to RSIZE_MAX unnecessarily. */ memcpy(targetString->data, sourceString->data, sourceString->len); /* IGNORE-BANNED */ } return targetString; } bool ContainsPendingWrites(ColumnarWriteState *state) { return state->stripeBuffers != NULL && state->stripeBuffers->rowCount != 0; } ================================================ FILE: src/backend/columnar/mod.c ================================================ /*------------------------------------------------------------------------- * * mod.c * * This file contains module-level definitions. * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "citus_version.h" #include "columnar/columnar.h" #include "columnar/columnar_tableam.h" #if PG_VERSION_NUM >= PG_VERSION_18 PG_MODULE_MAGIC_EXT(.name = "citus_columnar", .version = "15.0devel"); #else PG_MODULE_MAGIC; #endif void _PG_init(void); void _PG_init(void) { columnar_init(); } ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.1-0--11.1-1.sql ================================================ -- add columnar objects back ALTER EXTENSION citus_columnar ADD SCHEMA columnar; ALTER EXTENSION citus_columnar ADD SCHEMA columnar_internal; ALTER EXTENSION citus_columnar ADD SEQUENCE columnar_internal.storageid_seq; ALTER EXTENSION citus_columnar ADD TABLE columnar_internal.options; ALTER EXTENSION citus_columnar ADD TABLE columnar_internal.stripe; ALTER EXTENSION citus_columnar ADD TABLE columnar_internal.chunk_group; ALTER EXTENSION citus_columnar ADD TABLE columnar_internal.chunk; ALTER EXTENSION citus_columnar ADD FUNCTION columnar_internal.columnar_handler; ALTER EXTENSION citus_columnar ADD ACCESS METHOD columnar; ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_set; ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_reset; ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.upgrade_columnar_storage; ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.downgrade_columnar_storage; ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog; ALTER EXTENSION citus_columnar ADD FUNCTION columnar.get_storage_id; ALTER EXTENSION citus_columnar ADD VIEW columnar.storage; ALTER EXTENSION citus_columnar ADD VIEW columnar.options; ALTER EXTENSION citus_columnar ADD VIEW columnar.stripe; ALTER EXTENSION citus_columnar ADD VIEW columnar.chunk_group; ALTER EXTENSION citus_columnar ADD VIEW columnar.chunk; -- move citus_internal functions to columnar_internal ALTER FUNCTION citus_internal.upgrade_columnar_storage(regclass) SET SCHEMA columnar_internal; ALTER FUNCTION citus_internal.downgrade_columnar_storage(regclass) SET SCHEMA columnar_internal; ALTER FUNCTION citus_internal.columnar_ensure_am_depends_catalog() SET SCHEMA columnar_internal; ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.1-0.sql ================================================ -- fake sql file 'Y' ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.1-1--11.2-1.sql ================================================ -- citus_columnar--11.1-1--11.2-1 #include "udfs/columnar_ensure_am_depends_catalog/11.2-1.sql" DELETE FROM pg_depend WHERE classid = 'pg_am'::regclass::oid AND objid IN (select oid from pg_am where amname = 'columnar') AND objsubid = 0 AND refclassid = 'pg_class'::regclass::oid AND refobjid IN ( 'columnar_internal.stripe_first_row_number_idx'::regclass::oid, 'columnar_internal.chunk_group_pkey'::regclass::oid, 'columnar_internal.chunk_pkey'::regclass::oid, 'columnar_internal.options_pkey'::regclass::oid, 'columnar_internal.stripe_first_row_number_idx'::regclass::oid, 'columnar_internal.stripe_pkey'::regclass::oid ) AND refobjsubid = 0 AND deptype = 'n'; ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.1-1.sql ================================================ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION citus_columnar" to load this file. \quit -- columnar--9.5-1--10.0-1.sql CREATE SCHEMA IF NOT EXISTS columnar; SET search_path TO columnar; CREATE SEQUENCE IF NOT EXISTS storageid_seq MINVALUE 10000000000 NO CYCLE; CREATE TABLE IF NOT EXISTS options ( regclass regclass NOT NULL PRIMARY KEY, chunk_group_row_limit int NOT NULL, stripe_row_limit int NOT NULL, compression_level int NOT NULL, compression name NOT NULL ) WITH (user_catalog_table = true); COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set'; CREATE TABLE IF NOT EXISTS stripe ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, file_offset bigint NOT NULL, data_length bigint NOT NULL, column_count int NOT NULL, chunk_row_count int NOT NULL, row_count bigint NOT NULL, chunk_group_count int NOT NULL, first_row_number bigint NOT NULL, PRIMARY KEY (storage_id, stripe_num), CONSTRAINT stripe_first_row_number_idx UNIQUE (storage_id, first_row_number) ) WITH (user_catalog_table = true); COMMENT ON TABLE stripe IS 'Columnar per stripe metadata'; CREATE TABLE IF NOT EXISTS chunk_group ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, chunk_group_num int NOT NULL, row_count bigint NOT NULL, PRIMARY KEY (storage_id, stripe_num, chunk_group_num) ); COMMENT ON TABLE chunk_group IS 'Columnar chunk group metadata'; CREATE TABLE IF NOT EXISTS chunk ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, attr_num int NOT NULL, chunk_group_num int NOT NULL, minimum_value bytea, maximum_value bytea, value_stream_offset bigint NOT NULL, value_stream_length bigint NOT NULL, exists_stream_offset bigint NOT NULL, exists_stream_length bigint NOT NULL, value_compression_type int NOT NULL, value_compression_level int NOT NULL, value_decompressed_length bigint NOT NULL, value_count bigint NOT NULL, PRIMARY KEY (storage_id, stripe_num, attr_num, chunk_group_num) ) WITH (user_catalog_table = true); COMMENT ON TABLE chunk IS 'Columnar per chunk metadata'; DO $proc$ BEGIN -- from version 12 and up we have support for tableam's if installed on pg11 we can't -- create the objects here. Instead we rely on citus_finish_pg_upgrade to be called by the -- user instead to add the missing objects IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN EXECUTE $$ --#include "udfs/columnar_handler/10.0-1.sql" CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal) RETURNS table_am_handler LANGUAGE C AS 'MODULE_PATHNAME', 'columnar_handler'; COMMENT ON FUNCTION columnar.columnar_handler(internal) IS 'internal function returning the handler for columnar tables'; -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying -- to parse the upgrade file and erroring on unknown syntax. -- normally this section would not execute on postgres 11 anyway. To trick it to pass on -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid -- syntax on 11.8 and will execute correctly in 12 DO $create_table_am$ BEGIN EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler'; END $create_table_am$; --#include "udfs/alter_columnar_table_set/10.0-1.sql" CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; --#include "udfs/alter_columnar_table_reset/10.0-1.sql" CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; $$; END IF; END$proc$; -- (this function being dropped in 10.0.3)->#include "udfs/columnar_ensure_objects_exist/10.0-1.sql" RESET search_path; -- columnar--10.0.-1 --10.0.2 GRANT USAGE ON SCHEMA columnar TO PUBLIC; GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ; -- columnar--10.0-3--10.1-1.sql -- Drop foreign keys between columnar metadata tables. -- columnar--10.1-1--10.2-1.sql -- For a proper mapping between tid & (stripe, row_num), add a new column to -- columnar.stripe and define a BTREE index on this column. -- Also include storage_id column for per-relation scans. -- Populate first_row_number column of columnar.stripe table. -- -- For simplicity, we calculate MAX(row_count) value across all the stripes -- of all the columanar tables and then use it to populate first_row_number -- column. This would introduce some gaps however we are okay with that since -- it's already the case with regular INSERT/COPY's. DO $$ DECLARE max_row_count bigint; -- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1; BEGIN SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe; UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER + (stripe_num - 1) * max_row_count; END; $$; -- columnar--10.2-1--10.2-2.sql -- revoke read access for columnar.chunk from unprivileged -- user as it contains chunk min/max values REVOKE SELECT ON columnar.chunk FROM PUBLIC; -- columnar--10.2-2--10.2-3.sql -- Since stripe_first_row_number_idx is required to scan a columnar table, we -- need to make sure that it is created before doing anything with columnar -- tables during pg upgrades. -- -- However, a plain btree index is not a dependency of a table, so pg_upgrade -- cannot guarantee that stripe_first_row_number_idx gets created when -- creating columnar.stripe, unless we make it a unique "constraint". -- -- To do that, drop stripe_first_row_number_idx and create a unique -- constraint with the same name to keep the code change at minimum. -- columnar--10.2-3--10.2-4.sql -- columnar--11.0-2--11.1-1.sql CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_set$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' SET ('; begin if (chunk_group_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit=' || chunk_group_row_limit; noop := false; end if; if (stripe_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit=' || stripe_row_limit; noop := false; end if; if (compression is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression=' || compression; noop := false; end if; if (compression_level is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level=' || compression_level; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_set$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_reset$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' RESET ('; begin if (chunk_group_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit'; noop := false; end if; if (stripe_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit'; noop := false; end if; if (compression) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression'; noop := false; end if; if (compression_level) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level'; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_reset$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; -- rename columnar schema to columnar_internal and tighten security REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA columnar FROM PUBLIC; ALTER SCHEMA columnar RENAME TO columnar_internal; REVOKE ALL PRIVILEGES ON SCHEMA columnar_internal FROM PUBLIC; -- create columnar schema with public usage privileges CREATE SCHEMA columnar; GRANT USAGE ON SCHEMA columnar TO PUBLIC; --#include "udfs/upgrade_columnar_storage/10.2-1.sql" CREATE OR REPLACE FUNCTION columnar_internal.upgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$; COMMENT ON FUNCTION columnar_internal.upgrade_columnar_storage(regclass) IS 'function to upgrade the columnar storage, if necessary'; --#include "udfs/downgrade_columnar_storage/10.2-1.sql" CREATE OR REPLACE FUNCTION columnar_internal.downgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$; COMMENT ON FUNCTION columnar_internal.downgrade_columnar_storage(regclass) IS 'function to downgrade the columnar storage, if necessary'; -- update UDF to account for columnar_internal schema CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend WITH columnar_schema_members(relid) AS ( SELECT pg_class.oid AS relid FROM pg_class WHERE relnamespace = COALESCE( (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'), (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar') ) AND relname IN ('chunk', 'chunk_group', 'chunk_group_pkey', 'chunk_pkey', 'options', 'options_pkey', 'storageid_seq', 'stripe', 'stripe_first_row_number_idx', 'stripe_pkey') ) SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to each object that is registered to pg_class and that lives -- in "columnar" schema. That contains catalog tables, indexes -- created on them and the sequences created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relid as refobjid, 0 as refobjsubid, 'n' as deptype FROM columnar_schema_members -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; SELECT columnar_internal.columnar_ensure_am_depends_catalog(); -- add utility function CREATE FUNCTION columnar.get_storage_id(regclass) RETURNS bigint LANGUAGE C STRICT AS 'citus_columnar', $$columnar_relation_storageid$$; -- create views for columnar table information CREATE VIEW columnar.storage WITH (security_barrier) AS SELECT c.oid::regclass AS relation, columnar.get_storage_id(c.oid) AS storage_id FROM pg_class c, pg_am am WHERE c.relam = am.oid AND am.amname = 'columnar' AND pg_has_role(c.relowner, 'USAGE'); COMMENT ON VIEW columnar.storage IS 'Columnar relation ID to storage ID mapping.'; GRANT SELECT ON columnar.storage TO PUBLIC; CREATE VIEW columnar.options WITH (security_barrier) AS SELECT regclass AS relation, chunk_group_row_limit, stripe_row_limit, compression, compression_level FROM columnar_internal.options o, pg_class c WHERE o.regclass = c.oid AND pg_has_role(c.relowner, 'USAGE'); COMMENT ON VIEW columnar.options IS 'Columnar options for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.options TO PUBLIC; CREATE VIEW columnar.stripe WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, file_offset, data_length, column_count, chunk_row_count, row_count, chunk_group_count, first_row_number FROM columnar_internal.stripe stripe, columnar.storage storage WHERE stripe.storage_id = storage.storage_id; COMMENT ON VIEW columnar.stripe IS 'Columnar stripe information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.stripe TO PUBLIC; CREATE VIEW columnar.chunk_group WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, chunk_group_num, row_count FROM columnar_internal.chunk_group cg, columnar.storage storage WHERE cg.storage_id = storage.storage_id; COMMENT ON VIEW columnar.chunk_group IS 'Columnar chunk group information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.chunk_group TO PUBLIC; CREATE VIEW columnar.chunk WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, attr_num, chunk_group_num, minimum_value, maximum_value, value_stream_offset, value_stream_length, exists_stream_offset, exists_stream_length, value_compression_type, value_compression_level, value_decompressed_length, value_count FROM columnar_internal.chunk chunk, columnar.storage storage WHERE chunk.storage_id = storage.storage_id; COMMENT ON VIEW columnar.chunk IS 'Columnar chunk information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.chunk TO PUBLIC; ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.2-1--11.3-1.sql ================================================ -- citus_columnar--11.2-1--11.3-1 ================================================ FILE: src/backend/columnar/sql/citus_columnar--11.3-1--12.2-1.sql ================================================ -- citus_columnar--11.3-1--12.2-1 ================================================ FILE: src/backend/columnar/sql/citus_columnar--12.2-1--13.2-1.sql ================================================ -- citus_columnar--12.2-1--13.2-1.sql #include "udfs/columnar_finish_pg_upgrade/13.2-1.sql" ================================================ FILE: src/backend/columnar/sql/citus_columnar--13.2-1--14.0-1.sql ================================================ -- citus_columnar--13.2-1--14.0-1 -- bump version to 14.0-1 ================================================ FILE: src/backend/columnar/sql/citus_columnar--14.0-1--15.0-1.sql ================================================ -- citus_columnar--14.0-1--15.0-1 -- bump version to 15.0-1 ================================================ FILE: src/backend/columnar/sql/columnar--10.0-1--10.0-2.sql ================================================ -- columnar--10.0-1--10.0-2.sql -- grant read access for columnar metadata tables to unprivileged user GRANT USAGE ON SCHEMA columnar TO PUBLIC; GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ; ================================================ FILE: src/backend/columnar/sql/columnar--10.0-3--10.1-1.sql ================================================ -- columnar--10.0-3--10.1-1.sql -- Drop foreign keys between columnar metadata tables. -- Postgres assigns different names to those foreign keys in PG11, so act accordingly. DO $proc$ BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN EXECUTE $$ ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_stripe_num_chunk_group_num_fkey; ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_stripe_num_fkey; $$; ELSE EXECUTE $$ ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_fkey; ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_fkey; $$; END IF; END$proc$; -- since we dropped pg11 support, we don't need to worry about missing -- columnar objects when upgrading postgres DROP FUNCTION citus_internal.columnar_ensure_objects_exist(); ================================================ FILE: src/backend/columnar/sql/columnar--10.1-1--10.2-1.sql ================================================ -- columnar--10.1-1--10.2-1.sql -- For a proper mapping between tid & (stripe, row_num), add a new column to -- columnar.stripe and define a BTREE index on this column. -- Also include storage_id column for per-relation scans. ALTER TABLE columnar.stripe ADD COLUMN first_row_number bigint; CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number); -- Populate first_row_number column of columnar.stripe table. -- -- For simplicity, we calculate MAX(row_count) value across all the stripes -- of all the columanar tables and then use it to populate first_row_number -- column. This would introduce some gaps however we are okay with that since -- it's already the case with regular INSERT/COPY's. DO $$ DECLARE max_row_count bigint; -- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1; BEGIN SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe; UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER + (stripe_num - 1) * max_row_count; END; $$; #include "udfs/upgrade_columnar_storage/10.2-1.sql" #include "udfs/downgrade_columnar_storage/10.2-1.sql" -- upgrade storage for all columnar relations PERFORM citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a WHERE c.relam = a.oid AND amname = 'columnar'; ================================================ FILE: src/backend/columnar/sql/columnar--10.2-1--10.2-2.sql ================================================ -- columnar--10.2-1--10.2-2.sql -- revoke read access for columnar.chunk from unprivileged -- user as it contains chunk min/max values REVOKE SELECT ON columnar.chunk FROM PUBLIC; ================================================ FILE: src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql ================================================ -- columnar--10.2-2--10.2-3.sql -- Since stripe_first_row_number_idx is required to scan a columnar table, we -- need to make sure that it is created before doing anything with columnar -- tables during pg upgrades. -- -- However, a plain btree index is not a dependency of a table, so pg_upgrade -- cannot guarantee that stripe_first_row_number_idx gets created when -- creating columnar.stripe, unless we make it a unique "constraint". -- -- To do that, drop stripe_first_row_number_idx and create a unique -- constraint with the same name to keep the code change at minimum. -- -- If we have a pg_depend entry for this index, we can not drop it as -- the extension depends on it. Remove the pg_depend entry if it exists. DELETE FROM pg_depend WHERE classid = 'pg_am'::regclass::oid AND objid IN (select oid from pg_am where amname = 'columnar') AND objsubid = 0 AND refclassid = 'pg_class'::regclass::oid AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid AND refobjsubid = 0 AND deptype = 'n'; DROP INDEX columnar.stripe_first_row_number_idx; ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx UNIQUE (storage_id, first_row_number); ================================================ FILE: src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql ================================================ -- columnar--10.2-3--10.2-4.sql #include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql" PERFORM citus_internal.columnar_ensure_am_depends_catalog(); ================================================ FILE: src/backend/columnar/sql/columnar--11.0-2--11.0-3.sql ================================================ -- no changes needed ================================================ FILE: src/backend/columnar/sql/columnar--11.0-3--11.1-1.sql ================================================ #include "udfs/alter_columnar_table_set/11.1-1.sql" #include "udfs/alter_columnar_table_reset/11.1-1.sql" -- rename columnar schema to columnar_internal and tighten security REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA columnar FROM PUBLIC; ALTER SCHEMA columnar RENAME TO columnar_internal; REVOKE ALL PRIVILEGES ON SCHEMA columnar_internal FROM PUBLIC; -- create columnar schema with public usage privileges CREATE SCHEMA columnar; GRANT USAGE ON SCHEMA columnar TO PUBLIC; -- update UDF to account for columnar_internal schema #include "udfs/columnar_ensure_am_depends_catalog/11.1-1.sql" -- add utility function CREATE FUNCTION columnar.get_storage_id(regclass) RETURNS bigint LANGUAGE C STRICT AS 'citus_columnar', $$columnar_relation_storageid$$; -- create views for columnar table information CREATE VIEW columnar.storage WITH (security_barrier) AS SELECT c.oid::regclass AS relation, columnar.get_storage_id(c.oid) AS storage_id FROM pg_class c, pg_am am WHERE c.relam = am.oid AND am.amname = 'columnar' AND pg_has_role(c.relowner, 'USAGE'); COMMENT ON VIEW columnar.storage IS 'Columnar relation ID to storage ID mapping.'; GRANT SELECT ON columnar.storage TO PUBLIC; CREATE VIEW columnar.options WITH (security_barrier) AS SELECT regclass AS relation, chunk_group_row_limit, stripe_row_limit, compression, compression_level FROM columnar_internal.options o, pg_class c WHERE o.regclass = c.oid AND pg_has_role(c.relowner, 'USAGE'); COMMENT ON VIEW columnar.options IS 'Columnar options for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.options TO PUBLIC; CREATE VIEW columnar.stripe WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, file_offset, data_length, column_count, chunk_row_count, row_count, chunk_group_count, first_row_number FROM columnar_internal.stripe stripe, columnar.storage storage WHERE stripe.storage_id = storage.storage_id; COMMENT ON VIEW columnar.stripe IS 'Columnar stripe information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.stripe TO PUBLIC; CREATE VIEW columnar.chunk_group WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, chunk_group_num, row_count FROM columnar_internal.chunk_group cg, columnar.storage storage WHERE cg.storage_id = storage.storage_id; COMMENT ON VIEW columnar.chunk_group IS 'Columnar chunk group information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.chunk_group TO PUBLIC; CREATE VIEW columnar.chunk WITH (security_barrier) AS SELECT relation, storage.storage_id, stripe_num, attr_num, chunk_group_num, minimum_value, maximum_value, value_stream_offset, value_stream_length, exists_stream_offset, exists_stream_length, value_compression_type, value_compression_level, value_decompressed_length, value_count FROM columnar_internal.chunk chunk, columnar.storage storage WHERE chunk.storage_id = storage.storage_id; COMMENT ON VIEW columnar.chunk IS 'Columnar chunk information for tables on which the current user has ownership privileges.'; GRANT SELECT ON columnar.chunk TO PUBLIC; ================================================ FILE: src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql ================================================ -- columnar--9.5-1--10.0-1.sql CREATE SCHEMA columnar; SET search_path TO columnar; CREATE SEQUENCE storageid_seq MINVALUE 10000000000 NO CYCLE; CREATE TABLE options ( regclass regclass NOT NULL PRIMARY KEY, chunk_group_row_limit int NOT NULL, stripe_row_limit int NOT NULL, compression_level int NOT NULL, compression name NOT NULL ) WITH (user_catalog_table = true); COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set'; CREATE TABLE stripe ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, file_offset bigint NOT NULL, data_length bigint NOT NULL, column_count int NOT NULL, chunk_row_count int NOT NULL, row_count bigint NOT NULL, chunk_group_count int NOT NULL, PRIMARY KEY (storage_id, stripe_num) ) WITH (user_catalog_table = true); COMMENT ON TABLE stripe IS 'Columnar per stripe metadata'; CREATE TABLE chunk_group ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, chunk_group_num int NOT NULL, row_count bigint NOT NULL, PRIMARY KEY (storage_id, stripe_num, chunk_group_num), FOREIGN KEY (storage_id, stripe_num) REFERENCES stripe(storage_id, stripe_num) ON DELETE CASCADE ); COMMENT ON TABLE chunk_group IS 'Columnar chunk group metadata'; CREATE TABLE chunk ( storage_id bigint NOT NULL, stripe_num bigint NOT NULL, attr_num int NOT NULL, chunk_group_num int NOT NULL, minimum_value bytea, maximum_value bytea, value_stream_offset bigint NOT NULL, value_stream_length bigint NOT NULL, exists_stream_offset bigint NOT NULL, exists_stream_length bigint NOT NULL, value_compression_type int NOT NULL, value_compression_level int NOT NULL, value_decompressed_length bigint NOT NULL, value_count bigint NOT NULL, PRIMARY KEY (storage_id, stripe_num, attr_num, chunk_group_num), FOREIGN KEY (storage_id, stripe_num, chunk_group_num) REFERENCES chunk_group(storage_id, stripe_num, chunk_group_num) ON DELETE CASCADE ) WITH (user_catalog_table = true); COMMENT ON TABLE chunk IS 'Columnar per chunk metadata'; DO $proc$ BEGIN -- from version 12 and up we have support for tableam's if installed on pg11 we can't -- create the objects here. Instead we rely on citus_finish_pg_upgrade to be called by the -- user instead to add the missing objects IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN EXECUTE $$ #include "udfs/columnar_handler/10.0-1.sql" #include "udfs/alter_columnar_table_set/10.0-1.sql" #include "udfs/alter_columnar_table_reset/10.0-1.sql" $$; END IF; END$proc$; #include "udfs/columnar_ensure_objects_exist/10.0-1.sql" RESET search_path; ================================================ FILE: src/backend/columnar/sql/downgrades/citus--11.0-3--11.0-2.sql ================================================ -- no changes needed ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--11.1-1--11.1-0.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to each object that is registered to pg_class and that lives -- in "columnar" schema. That contains catalog tables, indexes -- created on them and the sequences created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relname::regclass::oid as refobjid, 0 as refobjsubid, 'n' as deptype FROM (VALUES ('columnar.chunk'), ('columnar.chunk_group'), ('columnar.chunk_group_pkey'), ('columnar.chunk_pkey'), ('columnar.options'), ('columnar.options_pkey'), ('columnar.storageid_seq'), ('columnar.stripe'), ('columnar.stripe_first_row_number_idx'), ('columnar.stripe_pkey') ) columnar_schema_members(relname) -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; DROP VIEW columnar.options; DROP VIEW columnar.stripe; DROP VIEW columnar.chunk_group; DROP VIEW columnar.chunk; DROP VIEW columnar.storage; DROP FUNCTION columnar.get_storage_id(regclass); DROP SCHEMA columnar; -- move columnar_internal functions back to citus_internal ALTER FUNCTION columnar_internal.upgrade_columnar_storage(regclass) SET SCHEMA citus_internal; ALTER FUNCTION columnar_internal.downgrade_columnar_storage(regclass) SET SCHEMA citus_internal; ALTER FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() SET SCHEMA citus_internal; ALTER SCHEMA columnar_internal RENAME TO columnar; GRANT USAGE ON SCHEMA columnar TO PUBLIC; GRANT SELECT ON columnar.options TO PUBLIC; GRANT SELECT ON columnar.stripe TO PUBLIC; GRANT SELECT ON columnar.chunk_group TO PUBLIC; -- detach relations from citus_columnar ALTER EXTENSION citus_columnar DROP SCHEMA columnar; ALTER EXTENSION citus_columnar DROP SEQUENCE columnar.storageid_seq; -- columnar tables ALTER EXTENSION citus_columnar DROP TABLE columnar.options; ALTER EXTENSION citus_columnar DROP TABLE columnar.stripe; ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk_group; ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk; ALTER EXTENSION citus_columnar DROP FUNCTION columnar.columnar_handler; ALTER EXTENSION citus_columnar DROP ACCESS METHOD columnar; ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_set; ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_reset; -- functions under citus_internal for columnar ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.upgrade_columnar_storage; ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.downgrade_columnar_storage; ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog; ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--11.2-1--11.1-1.sql ================================================ -- citus_columnar--11.2-1--11.1-1 -- Note that we intentionally do not re-insert the pg_depend records that we -- deleted via citus_columnar--11.1-1--11.2-1.sql. ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--11.3-1--11.2-1.sql ================================================ -- citus_columnar--11.3-1--11.2-1 ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--12.2-1--11.3-1.sql ================================================ -- citus_columnar--12.2-1--11.3-1 ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--13.2-1--12.2-1.sql ================================================ -- citus_columnar--13.2-1--12.2-1.sql DROP FUNCTION IF EXISTS pg_catalog.columnar_finish_pg_upgrade(); ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--14.0-1--13.2-1.sql ================================================ -- citus_columnar--14.0-1--13.2-1 -- downgrade version to 13.2-1 ================================================ FILE: src/backend/columnar/sql/downgrades/citus_columnar--15.0-1--14.0-1.sql ================================================ -- citus_columnar--15.0-1--14.0-1 -- downgrade version to 14.0-1 ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql ================================================ -- columnar--10.0-1--9.5-1.sql SET search_path TO columnar; DO $proc$ BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN EXECUTE $$ DROP FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool); DROP FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int); DROP ACCESS METHOD columnar; DROP FUNCTION columnar_handler(internal); $$; END IF; END$proc$; DROP TABLE chunk; DROP TABLE chunk_group; DROP TABLE stripe; DROP TABLE options; DROP SEQUENCE storageid_seq; DROP FUNCTION citus_internal.columnar_ensure_objects_exist(); RESET search_path; DROP SCHEMA columnar; ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.0-2--10.0-1.sql ================================================ -- columnar--10.0-2--10.0-1.sql -- revoke read access for columnar metadata tables from unprivileged user REVOKE USAGE ON SCHEMA columnar FROM PUBLIC; REVOKE SELECT ON ALL tables IN SCHEMA columnar FROM PUBLIC; ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.1-1--10.0-3.sql ================================================ -- columnar--10.1-1--10.0-3.sql -- define foreign keys between columnar metadata tables ALTER TABLE columnar.chunk ADD FOREIGN KEY (storage_id, stripe_num, chunk_group_num) REFERENCES columnar.chunk_group(storage_id, stripe_num, chunk_group_num) ON DELETE CASCADE; ALTER TABLE columnar.chunk_group ADD FOREIGN KEY (storage_id, stripe_num) REFERENCES columnar.stripe(storage_id, stripe_num) ON DELETE CASCADE; -- define columnar_ensure_objects_exist again #include "../udfs/columnar_ensure_objects_exist/10.0-1.sql" ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql ================================================ -- columnar--10.2-1--10.1-1.sql -- downgrade storage for all columnar relations SELECT citus_internal.downgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a WHERE c.relam = a.oid AND amname = 'columnar'; DROP FUNCTION citus_internal.upgrade_columnar_storage(regclass); DROP FUNCTION citus_internal.downgrade_columnar_storage(regclass); -- drop "first_row_number" column and the index defined on it -- -- If we have a pg_depend entry for this index, we can not drop it as -- the extension depends on it. Remove the pg_depend entry if it exists. DELETE FROM pg_depend WHERE classid = 'pg_am'::regclass::oid AND objid IN (select oid from pg_am where amname = 'columnar') AND objsubid = 0 AND refclassid = 'pg_class'::regclass::oid AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid AND refobjsubid = 0 AND deptype = 'n'; DROP INDEX columnar.stripe_first_row_number_idx; ALTER TABLE columnar.stripe DROP COLUMN first_row_number; ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql ================================================ -- columnar--10.2-2--10.2-1.sql -- grant read access for columnar.chunk to unprivileged user GRANT SELECT ON columnar.chunk TO PUBLIC; ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql ================================================ -- columnar--10.2-3--10.2-2.sql -- -- If we have a pg_depend entry for this index, we can not drop it as -- the extension depends on it. Remove the pg_depend entry if it exists. DELETE FROM pg_depend WHERE classid = 'pg_am'::regclass::oid AND objid IN (select oid from pg_am where amname = 'columnar') AND objsubid = 0 AND refclassid = 'pg_class'::regclass::oid AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid AND refobjsubid = 0 AND deptype = 'n'; ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx; CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number); ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql ================================================ -- columnar--10.2-4--10.2-3.sql DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog(); -- Note that we intentionally do not delete pg_depend records that we inserted -- via columnar--10.2-3--10.2-4.sql (by using columnar_ensure_am_depends_catalog). ================================================ FILE: src/backend/columnar/sql/downgrades/columnar--11.1-1--11.0-3.sql ================================================ #include "../udfs/alter_columnar_table_set/10.0-1.sql" #include "../udfs/alter_columnar_table_reset/10.0-1.sql" #include "../udfs/columnar_ensure_am_depends_catalog/10.2-4.sql" DROP VIEW columnar.options; DROP VIEW columnar.stripe; DROP VIEW columnar.chunk_group; DROP VIEW columnar.chunk; DROP VIEW columnar.storage; DROP FUNCTION columnar.get_storage_id(regclass); DROP SCHEMA columnar; ALTER SCHEMA columnar_internal RENAME TO columnar; GRANT USAGE ON SCHEMA columnar TO PUBLIC; GRANT SELECT ON columnar.options TO PUBLIC; GRANT SELECT ON columnar.stripe TO PUBLIC; GRANT SELECT ON columnar.chunk_group TO PUBLIC; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_reset/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_reset$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' RESET ('; begin if (chunk_group_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit'; noop := false; end if; if (stripe_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit'; noop := false; end if; if (compression) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression'; noop := false; end if; if (compression_level) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level'; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_reset$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool DEFAULT false, stripe_row_limit bool DEFAULT false, compression bool DEFAULT false, compression_level bool DEFAULT false) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_reset$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' RESET ('; begin if (chunk_group_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit'; noop := false; end if; if (stripe_row_limit) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit'; noop := false; end if; if (compression) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression'; noop := false; end if; if (compression_level) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level'; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_reset$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool) IS 'reset on or more options on a columnar table to the system defaults'; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_set/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_set$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' SET ('; begin if (chunk_group_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit=' || chunk_group_row_limit; noop := false; end if; if (stripe_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit=' || stripe_row_limit; noop := false; end if; if (compression is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression=' || compression; noop := false; end if; if (compression_level is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level=' || compression_level; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_set$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; ================================================ FILE: src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int DEFAULT NULL, stripe_row_limit int DEFAULT NULL, compression name DEFAULT null, compression_level int DEFAULT NULL) RETURNS void LANGUAGE plpgsql AS $alter_columnar_table_set$ declare noop BOOLEAN := true; cmd TEXT := 'ALTER TABLE ' || table_name::text || ' SET ('; begin if (chunk_group_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.chunk_group_row_limit=' || chunk_group_row_limit; noop := false; end if; if (stripe_row_limit is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.stripe_row_limit=' || stripe_row_limit; noop := false; end if; if (compression is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression=' || compression; noop := false; end if; if (compression_level is not null) then if (not noop) then cmd := cmd || ', '; end if; cmd := cmd || 'columnar.compression_level=' || compression_level; noop := false; end if; cmd := cmd || ')'; if (not noop) then execute cmd; end if; return; end; $alter_columnar_table_set$; COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int) IS 'set one or more options on a columnar table, when set to NULL no change is made'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/10.2-4.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to each object that is registered to pg_class and that lives -- in "columnar" schema. That contains catalog tables, indexes -- created on them and the sequences created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relname::regclass::oid as refobjid, 0 as refobjsubid, 'n' as deptype FROM (VALUES ('columnar.chunk'), ('columnar.chunk_group'), ('columnar.chunk_group_pkey'), ('columnar.chunk_pkey'), ('columnar.options'), ('columnar.options_pkey'), ('columnar.storageid_seq'), ('columnar.stripe'), ('columnar.stripe_first_row_number_idx'), ('columnar.stripe_pkey') ) columnar_schema_members(relname) -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION citus_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend WITH columnar_schema_members(relid) AS ( SELECT pg_class.oid AS relid FROM pg_class WHERE relnamespace = COALESCE( (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'), (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar') ) AND relname IN ('chunk', 'chunk_group', 'chunk_group_pkey', 'chunk_pkey', 'options', 'options_pkey', 'storageid_seq', 'stripe', 'stripe_first_row_number_idx', 'stripe_pkey') ) SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to each object that is registered to pg_class and that lives -- in "columnar" schema. That contains catalog tables, indexes -- created on them and the sequences created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relid as refobjid, 0 as refobjsubid, 'n' as deptype FROM columnar_schema_members -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION citus_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend WITH columnar_schema_members(relid) AS ( SELECT pg_class.oid AS relid FROM pg_class WHERE relnamespace = COALESCE( (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'), (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar') ) AND relname IN ('chunk', 'chunk_group', 'options', 'storageid_seq', 'stripe') ) SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to some objects registered as regclass and that lives in -- "columnar" schema. That contains catalog tables and the sequences -- created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relid as refobjid, 0 as refobjsubid, 'n' as deptype FROM columnar_schema_members -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/latest.sql ================================================ CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $func$ BEGIN INSERT INTO pg_depend WITH columnar_schema_members(relid) AS ( SELECT pg_class.oid AS relid FROM pg_class WHERE relnamespace = COALESCE( (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'), (SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar') ) AND relname IN ('chunk', 'chunk_group', 'options', 'storageid_seq', 'stripe') ) SELECT -- Define a dependency edge from "columnar table access method" .. 'pg_am'::regclass::oid as classid, (select oid from pg_am where amname = 'columnar') as objid, 0 as objsubid, -- ... to some objects registered as regclass and that lives in -- "columnar" schema. That contains catalog tables and the sequences -- created in "columnar" schema. -- -- Given the possibility of user might have created their own objects -- in columnar schema, we explicitly specify list of objects that we -- are interested in. 'pg_class'::regclass::oid as refclassid, columnar_schema_members.relid as refobjid, 0 as refobjsubid, 'n' as deptype FROM columnar_schema_members -- Avoid inserting duplicate entries into pg_depend. EXCEPT TABLE pg_depend; END; $func$; COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() IS 'internal function responsible for creating dependencies from columnar ' 'table access method to the rel objects in columnar schema'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql ================================================ -- citus_internal.columnar_ensure_objects_exist is an internal helper function to create -- missing objects, anything related to the table access methods. -- Since the API for table access methods is only available in PG12 we can't create these -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can -- upgrade their database to PG12. Now they require the table access method objects that -- we couldn't create before. -- This internal function is called from `citus_finish_pg_upgrade` which the user is -- required to call after a PG major upgrade. CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_objects_exist() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $ceoe$ BEGIN -- when postgres is version 12 or above we need to create the tableam. If the tableam -- exist we assume all objects have been created. IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN #include "../columnar_handler/10.0-1.sql" #include "../alter_columnar_table_set/10.0-1.sql" #include "../alter_columnar_table_reset/10.0-1.sql" -- add the missing objects to the extension ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal); ALTER EXTENSION citus ADD ACCESS METHOD columnar; ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int); ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool); END IF; END IF; END; $ceoe$; COMMENT ON FUNCTION citus_internal.columnar_ensure_objects_exist() IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql ================================================ -- citus_internal.columnar_ensure_objects_exist is an internal helper function to create -- missing objects, anything related to the table access methods. -- Since the API for table access methods is only available in PG12 we can't create these -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can -- upgrade their database to PG12. Now they require the table access method objects that -- we couldn't create before. -- This internal function is called from `citus_finish_pg_upgrade` which the user is -- required to call after a PG major upgrade. CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_objects_exist() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $ceoe$ BEGIN -- when postgres is version 12 or above we need to create the tableam. If the tableam -- exist we assume all objects have been created. IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN #include "../columnar_handler/10.0-1.sql" #include "../alter_columnar_table_set/10.0-1.sql" #include "../alter_columnar_table_reset/10.0-1.sql" -- add the missing objects to the extension ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal); ALTER EXTENSION citus ADD ACCESS METHOD columnar; ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_group_row_limit int, stripe_row_limit int, compression name, compression_level int); ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_group_row_limit bool, stripe_row_limit bool, compression bool, compression_level bool); END IF; END IF; END; $ceoe$; COMMENT ON FUNCTION citus_internal.columnar_ensure_objects_exist() IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_finish_pg_upgrade/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.columnar_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); END; $cppu$; COMMENT ON FUNCTION pg_catalog.columnar_finish_pg_upgrade() IS 'perform tasks to properly complete a Postgres upgrade for columnar extension'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_finish_pg_upgrade/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.columnar_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); END; $cppu$; COMMENT ON FUNCTION pg_catalog.columnar_finish_pg_upgrade() IS 'perform tasks to properly complete a Postgres upgrade for columnar extension'; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_handler/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal) RETURNS table_am_handler LANGUAGE C AS 'MODULE_PATHNAME', 'columnar_handler'; COMMENT ON FUNCTION columnar.columnar_handler(internal) IS 'internal function returning the handler for columnar tables'; -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying -- to parse the upgrade file and erroring on unknown syntax. -- normally this section would not execute on postgres 11 anyway. To trick it to pass on -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid -- syntax on 11.8 and will execute correctly in 12 DO $create_table_am$ BEGIN EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler'; END $create_table_am$; ================================================ FILE: src/backend/columnar/sql/udfs/columnar_handler/latest.sql ================================================ CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal) RETURNS table_am_handler LANGUAGE C AS 'MODULE_PATHNAME', 'columnar_handler'; COMMENT ON FUNCTION columnar.columnar_handler(internal) IS 'internal function returning the handler for columnar tables'; -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying -- to parse the upgrade file and erroring on unknown syntax. -- normally this section would not execute on postgres 11 anyway. To trick it to pass on -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid -- syntax on 11.8 and will execute correctly in 12 DO $create_table_am$ BEGIN EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler'; END $create_table_am$; ================================================ FILE: src/backend/columnar/sql/udfs/downgrade_columnar_storage/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.downgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$; COMMENT ON FUNCTION citus_internal.downgrade_columnar_storage(regclass) IS 'function to downgrade the columnar storage, if necessary'; ================================================ FILE: src/backend/columnar/sql/udfs/downgrade_columnar_storage/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.downgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$; COMMENT ON FUNCTION citus_internal.downgrade_columnar_storage(regclass) IS 'function to downgrade the columnar storage, if necessary'; ================================================ FILE: src/backend/columnar/sql/udfs/upgrade_columnar_storage/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.upgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$; COMMENT ON FUNCTION citus_internal.upgrade_columnar_storage(regclass) IS 'function to upgrade the columnar storage, if necessary'; ================================================ FILE: src/backend/columnar/sql/udfs/upgrade_columnar_storage/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.upgrade_columnar_storage(rel regclass) RETURNS VOID STRICT LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$; COMMENT ON FUNCTION citus_internal.upgrade_columnar_storage(regclass) IS 'function to upgrade the columnar storage, if necessary'; ================================================ FILE: src/backend/columnar/write_state_management.c ================================================ #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "access/genam.h" #include "access/heapam.h" #include "access/heaptoast.h" #include "access/multixact.h" #include "access/rewriteheap.h" #include "access/tsmapi.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/index.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" #include "catalog/pg_trigger.h" #include "catalog/storage.h" #include "catalog/storage_xlog.h" #include "commands/progress.h" #include "commands/vacuum.h" #include "common/hashfn.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "optimizer/plancat.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" #include "storage/lmgr.h" #include "storage/predicate.h" #include "storage/procarray.h" #include "storage/smgr.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/pg_rusage.h" #include "utils/rel.h" #include "utils/syscache.h" #include "citus_version.h" #include "pg_version_compat.h" #include "columnar/columnar.h" #include "columnar/columnar_customscan.h" #include "columnar/columnar_tableam.h" #include "columnar/columnar_version_compat.h" /* * Mapping from relfilenode to WriteStateMapEntry. This keeps write state for * each relation. */ static HTAB *WriteStateMap = NULL; /* memory context for allocating WriteStateMap & all write states */ static MemoryContext WriteStateContext = NULL; /* * Each member of the writeStateStack in WriteStateMapEntry. This means that * we did some inserts in the subtransaction subXid, and the state of those * inserts is stored at writeState. Those writes can be flushed or unflushed. */ typedef struct SubXidWriteState { SubTransactionId subXid; ColumnarWriteState *writeState; struct SubXidWriteState *next; } SubXidWriteState; /* * An entry in WriteStateMap. */ typedef struct WriteStateMapEntry { /* key of the entry */ RelFileNumber relfilenumber; /* * If a table is dropped, we set dropped to true and set dropSubXid to the * id of the subtransaction in which the drop happened. */ bool dropped; SubTransactionId dropSubXid; /* * Stack of SubXidWriteState where first element is top of the stack. When * inserts happen, we look at top of the stack. If top of stack belongs to * current subtransaction, we forward writes to its writeState. Otherwise, * we create a new stack entry for current subtransaction and push it to * the stack, and forward writes to that. */ SubXidWriteState *writeStateStack; } WriteStateMapEntry; /* * Memory context reset callback so we reset WriteStateMap to NULL at the end * of transaction. WriteStateMap is allocated in & WriteStateMap, so its * leaked reference can cause memory issues. */ static MemoryContextCallback cleanupCallback; static void CleanupWriteStateMap(void *arg) { WriteStateMap = NULL; WriteStateContext = NULL; } ColumnarWriteState * columnar_init_write_state(Relation relation, TupleDesc tupdesc, Oid tupSlotRelationId, SubTransactionId currentSubXid) { bool found; /* * If this is the first call in current transaction, allocate the hash * table. */ if (WriteStateMap == NULL) { WriteStateContext = AllocSetContextCreate( TopTransactionContext, "Column Store Write State Management Context", ALLOCSET_DEFAULT_SIZES); HASHCTL info; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); memset(&info, 0, sizeof(info)); info.keysize = sizeof(RelFileNumber); info.hash = oid_hash; info.entrysize = sizeof(WriteStateMapEntry); info.hcxt = WriteStateContext; WriteStateMap = hash_create("column store write state map", 64, &info, hashFlags); cleanupCallback.arg = NULL; cleanupCallback.func = &CleanupWriteStateMap; cleanupCallback.next = NULL; MemoryContextRegisterResetCallback(WriteStateContext, &cleanupCallback); } WriteStateMapEntry *hashEntry = hash_search(WriteStateMap, &(relation->rd_locator.relNumber), HASH_ENTER, &found); if (!found) { hashEntry->writeStateStack = NULL; hashEntry->dropped = false; } Assert(!hashEntry->dropped); /* * If top of stack belongs to the current subtransaction, return its * writeState, ... */ if (hashEntry->writeStateStack != NULL) { SubXidWriteState *stackHead = hashEntry->writeStateStack; if (stackHead->subXid == currentSubXid) { return stackHead->writeState; } } /* * ... otherwise we need to create a new stack entry for the current * subtransaction. */ MemoryContext oldContext = MemoryContextSwitchTo(WriteStateContext); ColumnarOptions columnarOptions = { 0 }; /* * In case of a table rewrite, we need to fetch table options based on the * relation id of the source tuple slot. * * For this reason, we always pass tupSlotRelationId here; which should be * same as the target table if the write operation is not related to a table * rewrite etc. */ ReadColumnarOptions(tupSlotRelationId, &columnarOptions); SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState)); stackEntry->writeState = ColumnarBeginWrite(relation, columnarOptions, tupdesc); stackEntry->subXid = currentSubXid; stackEntry->next = hashEntry->writeStateStack; hashEntry->writeStateStack = stackEntry; MemoryContextSwitchTo(oldContext); return stackEntry->writeState; } /* * Flushes pending writes for given relfilenode in the given subtransaction. */ void FlushWriteStateForRelfilenumber(RelFileNumber relfilenumber, SubTransactionId currentSubXid) { if (WriteStateMap == NULL) { return; } WriteStateMapEntry *entry = hash_search(WriteStateMap, &relfilenumber, HASH_FIND, NULL); Assert(!entry || !entry->dropped); if (entry && entry->writeStateStack != NULL) { SubXidWriteState *stackEntry = entry->writeStateStack; if (stackEntry->subXid == currentSubXid) { ColumnarFlushPendingWrites(stackEntry->writeState); } } } /* * Helper function for FlushWriteStateForAllRels and DiscardWriteStateForAllRels. * Pops all of write states for current subtransaction, and depending on "commit" * either flushes them or discards them. This also takes into account dropped * tables, and either propagates the dropped flag to parent subtransaction or * rolls back abort. */ static void PopWriteStateForAllRels(SubTransactionId currentSubXid, SubTransactionId parentSubXid, bool commit) { HASH_SEQ_STATUS status; WriteStateMapEntry *entry; if (WriteStateMap == NULL) { return; } hash_seq_init(&status, WriteStateMap); while ((entry = hash_seq_search(&status)) != 0) { if (entry->writeStateStack == NULL) { continue; } /* * If the table has been dropped in current subtransaction, either * commit the drop or roll it back. */ if (entry->dropped) { if (entry->dropSubXid == currentSubXid) { if (commit) { /* elevate drop to the upper subtransaction */ entry->dropSubXid = parentSubXid; } else { /* abort the drop */ entry->dropped = false; } } } /* * Otherwise, commit or discard pending writes. */ else { SubXidWriteState *stackHead = entry->writeStateStack; if (stackHead->subXid == currentSubXid) { if (commit) { ColumnarEndWrite(stackHead->writeState); } entry->writeStateStack = stackHead->next; } } } } /* * Called when current subtransaction is committed. */ void FlushWriteStateForAllRels(SubTransactionId currentSubXid, SubTransactionId parentSubXid) { PopWriteStateForAllRels(currentSubXid, parentSubXid, true); } /* * Called when current subtransaction is aborted. */ void DiscardWriteStateForAllRels(SubTransactionId currentSubXid, SubTransactionId parentSubXid) { PopWriteStateForAllRels(currentSubXid, parentSubXid, false); } /* * Called when the given relfilenode is dropped. */ void MarkRelfilenumberDropped(RelFileNumber relfilenumber, SubTransactionId currentSubXid) { if (WriteStateMap == NULL) { return; } WriteStateMapEntry *entry = hash_search(WriteStateMap, &relfilenumber, HASH_FIND, NULL); if (!entry || entry->dropped) { return; } entry->dropped = true; entry->dropSubXid = currentSubXid; } /* * Called when the given relfilenode is dropped in non-transactional TRUNCATE. */ void NonTransactionDropWriteState(RelFileNumber relfilenumber) { if (WriteStateMap) { hash_search(WriteStateMap, &relfilenumber, HASH_REMOVE, false); } } /* * Returns true if there are any pending writes in upper transactions. */ bool PendingWritesInUpperTransactions(RelFileNumber relfilenumber, SubTransactionId currentSubXid) { if (WriteStateMap == NULL) { return false; } WriteStateMapEntry *entry = hash_search(WriteStateMap, &relfilenumber, HASH_FIND, NULL); if (entry && entry->writeStateStack != NULL) { SubXidWriteState *stackEntry = entry->writeStateStack; while (stackEntry != NULL) { if (stackEntry->subXid != currentSubXid && ContainsPendingWrites(stackEntry->writeState)) { return true; } stackEntry = stackEntry->next; } } return false; } /* * GetWriteContextForDebug exposes WriteStateContext for debugging * purposes. */ extern MemoryContext GetWriteContextForDebug(void) { return WriteStateContext; } ================================================ FILE: src/backend/distributed/.gitignore ================================================ # ==================== # = Project-Specific = # ==================== # regression test detritus /log/ /regression.diffs /regression.out /results/ /tmp_check* /build/ ================================================ FILE: src/backend/distributed/Makefile ================================================ # Makefile for the Citus extension citus_subdir = src/backend/distributed citus_top_builddir = ../../.. safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib safestringlib_builddir = $(citus_top_builddir)/vendor/safestringlib/build safestringlib_a = $(safestringlib_builddir)/libsafestring_static.a safestringlib_sources = $(wildcard $(safestringlib_srcdir)/safeclib/*) MODULE_big = citus EXTENSION = citus template_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql)) template_downgrade_sql_files = $(patsubst $(citus_abs_srcdir)/sql/downgrades/%,%,$(wildcard $(citus_abs_srcdir)/sql/downgrades/*.sql)) generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_files)) generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(template_downgrade_sql_files)) # All citus--*.sql files that are used to upgrade between versions DATA_built = $(generated_sql_files) # directories with source files SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib shardsplit stats test transaction utils worker clock # enterprise modules SUBDIRS += replication # Symlinks are not copied over to the build directory if a separete build # directory is used during configure (such as on CI) ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS)) # That patsubst rule searches all directories listed in SUBDIRS for .c # files, and adds the corresponding .o files to OBJS OBJS += \ $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c)))) # be explicit about the default target .PHONY: cdc all: cdc cdc: $(MAKE) -C cdc all NO_PGXS = 1 SHLIB_LINK = $(libpq) include $(citus_top_builddir)/Makefile.global # make sure citus_version.o is recompiled whenever any change is made to the binary or any # other artifact being installed to reflect the correct gitref for every build CITUS_VERSION_INVALIDATE := $(filter-out utils/citus_version.o,$(OBJS)) CITUS_VERSION_INVALIDATE += $(generated_sql_files) ifneq ($(wildcard $(citus_top_builddir)/.git/.*),) CITUS_VERSION_INVALIDATE += $(citus_top_builddir)/.git/index endif utils/citus_version.o: $(CITUS_VERSION_INVALIDATE) SHLIB_LINK += $(filter -lssl -lcrypto -lssleay32 -leay32, $(LIBS)) override CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include SQL_DEPDIR=.deps/sql SQL_BUILDDIR=build/sql $(generated_sql_files): $(citus_abs_srcdir)/build/%: % @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) @# -MF is used to store dependency files(.Po) in another directory for separation @# -MT is used to change the target of the rule emitted by dependency generation. @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. @# -undef is used to not predefine any system-specific or GCC-specific macros. @# `man cpp` for further information cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ $(generated_downgrade_sql_files): $(citus_abs_srcdir)/build/sql/%: sql/downgrades/% @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) @# -MF is used to store dependency files(.Po) in another directory for separation @# -MT is used to change the target of the rule emitted by dependency generation. @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. @# -undef is used to not predefine any system-specific or GCC-specific macros. @# `man cpp` for further information cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ SQL_Po_files := $(wildcard $(SQL_DEPDIR)/*.Po) ifneq (,$(SQL_Po_files)) include $(SQL_Po_files) endif .PHONY: clean-full install install-downgrades install-all install-cdc clean-cdc clean: clean-cdc clean-cdc: $(MAKE) -C cdc clean cleanup-before-install: rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus.control rm -f $(DESTDIR)$(datadir)/$(datamoduledir)/citus--* install: cleanup-before-install install-cdc install-cdc: $(MAKE) -C cdc install # install and install-downgrades should be run sequentially install-all: install $(MAKE) install-downgrades install-downgrades: $(generated_downgrade_sql_files) $(INSTALL_DATA) $(generated_downgrade_sql_files) '$(DESTDIR)$(datadir)/$(datamoduledir)/' clean-full: $(MAKE) clean $(MAKE) -C cdc clean-full rm -rf $(safestringlib_builddir) ================================================ FILE: src/backend/distributed/README.md ================================================ # Citus Technical Documentation The purpose of this document is to provide comprehensive technical documentation for Citus, in particular the distributed database implementation. # Table of Contents - [Citus Concepts](#citus-concepts) - [Principles](#principles) - [Use of hooks](#use-of-hooks) - [Query planner](#query-planner) - [High-level design/flow:](#high-level-designflow) - [Distributed Query Planning with Examples in Citus (as of Citus 12.1)](#distributed-query-planning-with-examples-in-citus-as-of-citus-121) - [Logical Planner & Optimizer](#logical-planner--optimizer) - [Combine query planner](#combine-query-planner) - [Restriction Equivalence](#restriction-equivalence) - [Recurring Tuples](#recurring-tuples) - [Executor](#executor) - [Custom scan](#custom-scan) - [Function evaluation](#function-evaluation) - [Prepared statements](#prepared-statements) - [Adaptive executor](#adaptive-executor) - [Local execution](#local-execution) - [Subplans](#subplans) - [Re-partitioning](#re-partitioning) - [COPY .. FROM command](#copy--from-command) - [COPY .. TO command](#copy--to-command) - [INSERT..SELECT](#insertselect) - [Merge command](#merge-command) - [DDL](#ddl) - [Object & dependency propagation](#object--dependency-propagation) - [Foreign keys](#foreign-keys) - [DROP Table](#drop-table) - [Connection management](#connection-management) - [Connection management](#connection-management-1) - [Placement connection tracking](#placement-connection-tracking) - [citus.max_cached_connections_per_worker](#citusmax_cached_connections_per_worker) - [citus.max_shared_pool_size](#citusmax_shared_pool_size) - [Transactions (2PC)](#transactions-2pc) - [Single-node transactions](#single-node-transactions) - [Multi-node transactions](#multi-node-transactions) - [No distributed snapshot isolation](#no-distributed-snapshot-isolation) - [Distributed Deadlocks](#distributed-deadlocks) - [Locking](#locking) - [Lock Levels](#lock-levels) - [Lock Monitoring](#lock-monitoring) - [Lock Types](#lock-types) - [Rebalancing](#rebalancing) - [Rebalancing algorithm](#rebalancing-algorithm) - [Shard moves](#shard-moves) - [Shard splits](#shard-splits) - [Background task runner](#background-task-runner) - [Resource cleanup](#resource-cleanup) - [Logical decoding / CDC](#logical-decoding--cdc) - [CDC ordering](#cdc-ordering) - [Global PID](#global-pid) - [Function call delegation](#function-call-delegation) - [Query from any node](#query-from-any-node) - [Why didn’t we have dedicated Query Nodes and Data Nodes?](#why-didnt-we-have-dedicated-query-nodes-and-data-nodes) - [Shard visibility](#shard-visibility) - [Statistic tracking](#statistic-tracking) - [Citus stat counters](#citus-stat-counters) # Citus Concepts **Citus table**: A table managed by Citus through PostgreSQL hooks. Whenever the table is involved in a command, the command is handled by Citus. **Shell table**: The “original” (inaccessible) PostgreSQL table which has been converted to a Citus table. Every node has shell tables and the corresponding Citus metadata. **Metadata**: The Citus catalog tables, shell tables, and dependent objects that are replicated to all nodes to enable distributed queries from those nodes. **Metadata syncing**: The process of replicating metadata from the coordinator to other nodes, which happens when adding a node or any change to the metadata. There are several types of Citus tables: **Distributed tables** are created using `SELECT create_distributed_table('table_name', 'distribution_column')`. They have a distribution column and for each row the value in the distribution column determines to which shard the row is assigned. There are 3 different partitioning schemes for distributed tables, though only the first is supported: - Hash-distributed tables have a range of hash values in shardminvalue/shardmaxvalue in pg_dist_shard - Range-distributed tables (deprecated) have shards with a distinct range of values in pg_dist_shard - Append-distributed tables (deprecated) have shards with a range of values in pg_dist_shard, though the ranges can overlap. Hash-distributed tables can be **co-located** with each other, such that the shards with the same hash value range are always on the same node. From the planner point-of-view, range-distributed tables can also be colocated, but shards are not managed by Citus. Shards can be replicated (deprecated), in which case they have multiple shard placements. Writes to the placements are performed using 2PC and use aggressive locking to avoid diverging. **Reference tables** are created using SELECT create_reference_table(..). They have a single shard, which is replicated to all nodes. It is possible to have a node without reference table replicas, but in that case the reference tables are replicated before the next rebalance. Reference tables are always co-located with each other and have a common co-location ID. Writes to a reference table are performed using 2PC and use aggressive locking to avoid diverging. **Single shard tables** are a special type of distributed table without a distribution column and with a single shard. When using schema-based sharding, tables created in a distributed schema automatically become single shard tables. Single shard tables can be co-located with each other, but not replicated. Single shard tables can be explicitly created using `SELECT create_distributed_table('table_name', NULL);`, though are meant to be auto-generated by schema-based sharding. **Citus local tables**: A single shard table that can only be placed on the coordinator and are primarily used as a drop-in replacement for regular PostgreSQL tables when creating foreign keys to/from reference tables. All Citus local tables are implicitly co-located with each other, but do not have a co-location ID. Citus local tables can be explicitly created using `SELECT citus_add_local_table_to_metadata('table_name');`, though are meant to be auto-generated by foreign keys. All Citus table types have the notion of a “shard”, though in many cases there is only a single shard. **Shard**: The table that contains the actual data in a Citus table. Shards reside in the same schema as the Citus table and are named after the Citus table with the shard ID as a suffix. Shards are hidden from catalog queries unless you SET citus.override_table_visibility TO off. Hash-distributed tables have multiple shards, each of which has distinct shardminvalue/shardmaxvalue. **Colocation group**: A set of distributed tables that have the same co-location ID, such that their shards are always co-located. **Shard group**: A set of shards with the same shardminvalue/shardmaxvalue that are part of the same co-location group. Citus guarantees that shards from the same shard group are always placed on the same node group. **Shard placement**: The assignment of a shard to a node group. There can be multiple placements of the same shard if the table is replicated (e.g. reference tables). **Shard group placement**: The assignment of a shard to a node group must be the same for all shards in a shard group, since those are always co-located. We’ll refer to the group of placements of a shard group as a shard group placement. **Node**: A single PostgreSQL/Citus server listed in pg_dist_node and added via SELECT citus_add_node(..). **Node group**: Each primary node can have 0 or more physical replicas in read replica clusters. Together the nodes form a node group identified by the groupid in pg_dist_node. Per convention, the coordinator(s) have group ID 0. Each node can know its own node groupid by reading it from pg_dist_local_group. **Coordinator**: The node with groupid 0, which can perform reads, writes, and administrative operations such as adding a node, rebalancing, and schema changes. **Worker nodes**: Nodes with groupid > 0, which can perform reads and writes, but not administrative operations. **Cluster**: The combination of worker nodes and coordinator is a cluster. When the cluster has a read replica, the nodes in the read replica are listed in pg_dist_node with a different nodecluster value, and the servers have a corresponding citus.cluster_name in their postgresql.conf. That way, nodes know which other nodes in pg_dist_node belong to their cluster, and they will ignore others. **Read replica cluster**: In a read replica cluster, every node is a physical replica of a node in a primary Citus cluster. The read replica has a distinct citus.cluster_name value and the nodes in the read replica cluster should be added to pg_dist_node on the primary coordinator with the corresponding cluster name. **Client connections**: Connections made by the client to any of the nodes in the cluster. Each client connection is backed by a postgres process/backend, which we sometimes refer to as a client session. **Internal connections**: Connections to other nodes made by a command running on a client session. Each internal connection is backed by a process, which we sometimes refer to as an internal session. In the code, you can use IsCitusInternalBackend() **Maintenance daemon**: A background worker that is started in each database that has the Citus extension. It performs distributed deadlock detection, 2PC recovery, synchronizing node metadata after citus_update_node, resource cleanup, and other tasks. In the query planner, we use the following terminology: **Distributed query**: A query sent by the client that involves a Citus table and is therefore handled by the distributed query planner. **Shard query**: An internal query on shards (at most 1 shard group). **Intermediate result**: A temporary file that contains the result of a subquery or CTE, created as a result of a broadcast or repartition operation. Intermediate results are automatically cleaned up on transaction end or restart. ## Principles Use cases: - Multi-tenant apps are the primary use case for Citus, which we can scale through distributing and co-locating by tenant ID, or through schema-based sharding. Citus is reasonably complete for this use case, but there are still SQL and operational improvements that can be made. - Real-time analytics is another popular use case due the combination of parallel distributed queries with indexes & in-database materialization (ETL). Improvement areas include automated time partitioning, better columnar storage (perf and update/delete), and incremental materialized views. - Citus works well for CRUD use cases, but would be far easier to use if we introduced a load balancer, DDL from any node (no explicit coordinator), and by better use of connection pooling for better performance (e.g. outbound pgbouncers). - Marketplace use cases could work well if we made it easier to distribute tables twice by different dimensions or made it easier to keep paired tables in sync. Schema management: - Our goal is for all DDL commands on Citus tables to work transparently, and for global DDL commands (e.g. CREATE TYPE) to be propagated to all nodes. Not all DDL is implemented yet and may either error or not propagate. - Since we cannot define custom DDL commands for sharding operations, we use functions that are called from a SELECT query. Query layer: - No incompatibilities with PostgreSQL – any query on a Citus table is supported on an equivalent PostgreSQL table. - We optimize for maximum pushdown (& performance) over complete compatibility, but our long-term goal is for all queries to be supported in all cases. - For single-shard queries, it is useful to avoid detailed query analysis through the fast path planner (simple, single table) and router planner (co-located joins) layers. However, multi-shard queries can go through disparate code paths that were added out of expediency and should eventually be unified. Transactional semantics: - Transactions scoped to a single node follow the same semantics as PostgreSQL. - Transactions across nodes are atomic, durable, and consistent, but do not have full snapshot isolation: A multi-shard query may see a concurrently committing transaction as committed on one node, but not yet committed on another node. - Read-your-writes consistency should be preserved. - Monotonic read consistency should be preserved for tables without replication, may not always be the case for replicated/reference tables. Replication model: - High availability is achieved through hot standby nodes managed by a control plane or PostgreSQL HA solution like Patroni or pg_auto_failover. - Read replicas are Citus clusters in which each node is a physical replica of a node in another Citus cluster. - Hot standby nodes are, at the time of writing, not in the metadata. Instead, the hostname/IP is replaced or rerouted at failover time. - The deprecated “statement based” replication is (as of Citus 11.0+) only useful for providing read scalability, not for HA as all modifications are done via 2PC. Reference tables do use statement-based replication. # Use of hooks A PostgreSQL extension consists of two parts: a set of SQL objects (e.g. metadata tables, functions, types) and a shared library that is loaded into PostgreSQL and can alter the behavior of PostgreSQL by setting certain hooks. You can find a high-level description of these concepts in [this talk](https://learn.microsoft.com/en-us/events/azure-cosmos-db-liftoff/foundations-of-distributed-postgresql-with-citus). Citus uses the following hooks: **User-defined functions (UDFs)** are callable from SQL queries as part of a transaction, but have an implementation in C, and are primarily used to manipulate the Citus metadata and implement remote procedure calls between servers. **Planner and executor hooks** are global function pointers that allow an extension to provide an alternative query plan and execution method. After PostgreSQL parses a query, Citus checks if the query involves a Citus table. If so, Citus generates a plan tree that contains a CustomScan operator, which encapsulates distributed query plan. **CustomScan** is an operator in a PostgreSQL query plan that returns tuples via custom function pointers. The Citus CustomScan calls the distributed query executor, which sends queries to other servers and collects the results before returning them to the PostgreSQL executor. **Transaction callbacks** are called at critical points in the lifecycle of a transaction (e.g. pre-commit, post-commit, abort). Citus uses these to implement distributed transactions. **Utility hook** is called after parsing any command that does not go through the regular query planner. Citus uses this hook primarily to apply DDL and COPY commands that affect Citus tables. **Background workers** run user-supplied code in separate processes. Citus uses this API to run a maintenance daemon. This daemon performs distributed deadlock detection, 2PC prepared transaction recovery, and cleanup. Through these hooks, Citus can intercept any interaction between the client and the PostgreSQL engine that involves Citus tables. Citus can then replace or augment PostgreSQL's behavior. # Query planner Citus has a layered planner architecture that accommodates different workloads. There are several useful presentations/papers that are relevant to Citus’ distributed planner, below we try to categorize them: ## High-level design/flow: - Distributing Queries the Citus Way: Marco’s PG Con presentation provides a good introduction: https://postgresconf.org/system/events/document/000/000/233/Distributing_Queries_the_Citus_Way.pdf - Another useful content on this topic is the Planner README.md: https://github.com/citusdata/citus/blob/main/src/backend/distributed/planner/README.md - Onder’s talk at CitusCon: https://www.youtube.com/watch?v=raw3Pwv0jb8 - Citus paper: https://dl.acm.org/doi/pdf/10.1145/3448016.3457551 - Logical planner design - 1: https://speakerdeck.com/marcocitus/scaling-out-postgre-sql - Logical Planner design - 2: https://www.youtube.com/watch?v=xJghcPs0ibQ - Logical Planner based on the paper: Correctness of query execution strategies in distributed databases: https://dl.acm.org/doi/pdf/10.1145/319996.320009 ## Distributed Query Planning with Examples in Citus (as of Citus 12.1) This part of the documentation aims to provide a comprehensive understanding of how Citus handles distributed query planning with examples. We will use a set of realistic tables to demonstrate various queries. Through these examples, we hope to offer a step-by-step guide on how Citus chooses to plan queries. Citus hooks into the PostgreSQL planner using the top-level planner_hook function pointer, which sees the query tree after parsing and analysis. If the query tree contains a Citus table, we go through several planner stages: fast path planner, router planner, recursive planning, logical planner & optimizer. Each stage can handle more complex queries than the previous, but also comes with more overhead. That way, we can handle a mixture of high throughput transactional workloads (without adding significant planning overhead), as well as more complex analytical queries (with more sophisticated distributed query execution). For specific types of queries (e.g. insert..select), we have separate planner code paths. For a more comprehensive high-level overview of the planner, go to https://postgresconf.org/system/events/document/000/000/233/Distributing_Queries_the_Citus_Way.pdf ### Table definitions used in this section ```sql -- Distributed Table: Users Table CREATE TABLE users_table ( user_id bigserial PRIMARY KEY, username VARCHAR(50) NOT NULL, email VARCHAR(50), date_of_birth DATE, country_code VARCHAR(3) ); SELECT create_distributed_table('users_table', 'user_id'); -- Distributed Table: Orders Table CREATE TABLE orders_table ( order_id bigserial, user_id BIGINT REFERENCES users_table(user_id), product_id BIGINT, order_date TIMESTAMPTZ, status VARCHAR(20) ); SELECT create_distributed_table('orders_table', 'user_id'); -- Distributed Table: Products Table CREATE TABLE products_table ( product_id bigserial PRIMARY KEY, product_name VARCHAR(100), category_id INT, price NUMERIC(10, 2) ); SELECT create_distributed_table('products_table', 'product_id'); -- Reference Table: Country Codes CREATE TABLE country_codes ( country_code VARCHAR(3) PRIMARY KEY, country_name VARCHAR(50) ); SELECT create_reference_table('country_codes'); -- Reference Table: Order Status CREATE TABLE order_status ( status VARCHAR(20) PRIMARY KEY, description TEXT ); SELECT create_reference_table('order_status'); -- Reference Table: Product Categories CREATE TABLE product_categories ( category_id INT PRIMARY KEY, category_name VARCHAR(50) ); SELECT create_reference_table('product_categories'); ``` ## Fast Path Router Planner The Fast Path Router Planner is specialized in optimizing queries that are both simple in structure and certain to touch a single shard. Importantly, it targets queries on a single shard distributed, citus local or reference tables. This does not mean the planner is restricted to trivial queries; it can handle complex SQL constructs like `GROUP BY`, `HAVING`, `DISTINCT`, etc., as long as these operate on a single table and involve an equality condition on the distribution key (`distribution_key = X`). The main SQL limitation for fast path distributed query planning is the subquery/CTE support. Those are left to the next planner: Router planner. The aim of this planner is to avoid relying on PostgreSQL's standard_planner() for planning, which performs unnecessary computations like cost estimation, irrelevant for distributed planning. Skipping the standard_planner has significant performance gains for OLTP workloads. By focusing on "shard-reachable" queries, the Fast Path Router Planner is able to bypass the need for more computationally expensive planning processes, thereby accelerating query execution. ### Main C Functions Involved: - `FastPathPlanner()`: The primary function for creating the fast-path query plan. - `FastPathRouterQuery()`: Validates if a query is eligible for fast-path routing by checking its structure and the WHERE clause. With set client_min_messages to debug4; you should see the following in the DEBUG messages: "DEBUG: Distributed planning for a fast-path router query" ```sql -- Fetches the count of users born in the same year, but only -- for a single country, with a filter on the distribution column -- Normally we have a single user with id = 15 because it's a PRIMARY KEY -- this is just to demonstrate that fast-path can handle complex queries -- with EXTRACT(), COUNT(), GROUP BY, HAVING, etc. SELECT EXTRACT(YEAR FROM date_of_birth) as birth_year, COUNT(*) FROM users_table WHERE country_code = 'USA' AND user_id = 15 GROUP BY birth_year HAVING COUNT(*) > 10; ``` ```sql -- all INSERT commands are by definition fast path -- router queries in the sense that they do not -- need any information from Postgres' standard_planner() INSERT INTO orders_table (user_id, product_id, order_date, status) VALUES (42, 555, now(), 'NEW'); ``` ```sql -- UPDATE/DELETEs can also be qualified as fast path router -- queries UPDATE products_table SET price = price * 1.1 WHERE product_id = 555; ``` Fast path queries have another important characteristic named "deferredPruning." For regular queries, Citus does the shard pruning during the planning phase, meaning that the shards that the query touches are calculated during the planning phase. However, in an ideal world, the shard pruning should happen during the execution and, for a certain class of queries, we support that. In the code, that is denoted by "Job->deferredPruning" field. Given that fast path queries are performance critical, they can be planned with prepared statements. When this is done, "Job->deferredPruning" becomes "true". And, the meaning of that is Citus can support PREPARED statements as expected. The first 6 executions of the plan do distributed planning, the rest is cached similar to Postgres' plan caching, and the shard pruning is done during the execution phase. And, if you attach a debugger, you'd see that on the first 6 executions, the debugger will stop at distributed_planner() function, but on the rest, it will not. The shard pruning for the cached command will happen in CitusBeginScan() function. To see that in action, checkout the DEBUG messages: ```sql set client_min_messages to debug4; PREPARE p1 (bigint) AS SELECT * FROM users_table WHERE user_id = $1; -- 1st execute execute p1(1); DEBUG: Deferred pruning for a fast-path router query DEBUG: Creating router plan .... (0 rows) -- 2nd execute execute p1(1); DEBUG: Deferred pruning for a fast-path router query DEBUG: Creating router plan .... (0 rows) ... execute p1(1); execute p1(1); execute p1(1); ... -- 6th execute execute p1(1); DEBUG: Deferred pruning for a fast-path router query DEBUG: Creating router plan .... (0 rows) -- now, on the 7th execute, you would **NOT** see the fast-path -- planning anymore, because the query comes from Postgres' -- query cache execute p1(1); DEBUG: constraint value: '1'::bigint DEBUG: shard count after pruning for users_table: 1 DEBUG: opening 1 new connections to localhost:9702 DEBUG: established connection to localhost:9702 for session 8 in 9499 microseconds DEBUG: task execution (0) for placement (46) on anchor shard (102049) finished in 1281 microseconds on worker node localhost:9702 DEBUG: Total number of commands sent over the session 8: 1 to node localhost:9702 (0 rows) ``` ### Delaying the Fast Path Plan As of Citus 13.2, if it can be determined at plan-time that a fast path query is against a local shard then a shortcut can be taken so that deparse and parse/plan of the shard query is avoided. Citus must be in MX mode and the shard must be local to the Citus node processing the query. If so, the OID of the distributed table is replaced by the OID of the shard in the parse tree. The parse tree is then given to the Postgres planner which returns a plan that is stored in the distributed plan's task. That plan can be repeatedly used by the local executor (described in the next section), avoiding the need to deparse and plan the shard query on each execution. We call this delayed fast path planning because if a query is eligible for fast path planning then `FastPathPlanner()` is delayed if the following properties hold: - The query is a SELECT or UPDATE on a distributed table (schema or column sharded) or Citus managed local table - The query has no volatile functions If so, then `FastPathRouterQuery()` sets a flag indicating that making the fast path plan should be delayed until after the worker job has been created. At that point the router planner uses `CheckAndBuildDelayedFastPathPlan()` to see if the task's shard placement is local (and not a dummy placement) and the metadata of the shard table and distributed table are consistent (no DDL in progress on the distributed table). If so the parse tree with OID of the distributed table replaced by the OID of the shard table is fed to `standard_planner()` and the resultant plan is saved in the task. Otherwise, if the worker job has been marked for deferred pruning or the shard is not local or the shard is local but it's not safe to swap OIDs, then `CheckAndBuildDelayedFastPathPlan()` calls `FastPathPlanner()` to ensure a complete plan context. Reference tables are not currently supported, but this may be relaxed for SELECT statements in the future. Delayed fast path planning can be disabled by turning off `citus.enable_local_fast_path_query_optimization` (it is on by default). ## Router Planner in Citus ### Overview The Router Planner plays a key role in Citus' query optimization landscape. While sharing some common traits with the Fast Path Router Planner, it offers unique capabilities as well. Router (and fast path router) planners are the bedrock for the multi-tenant use cases. #### Similarities with Fast Path Router Planner - **Single Node Routing**: Both planners send queries to a single node. Unlike the Fast Path Planner, the Router Planner can work with multiple colocated tables, provided they have filters on their distribution columns. - **Query Routing Mechanics**: Router Planner takes the query, verifies if it can be routed, and if so, it replaces original table names with their corresponding shard names, directing the query to the appropriate nodes. #### Differences - **Subqueries and CTEs**: The Router Planner can manage subqueries and Common Table Expressions (CTEs), routing the entire query to a single node as long as all involved tables have filters on their distribution columns. - **Standard Planner Reliance**: Router Planner relies on PostgreSQL's `standard_planner()` to learn the necessary filter restrictions on the tables. #### Main C Functions Involved - `PlanRouterQuery()`: Responsible for creating the router plan. - `TargetShardIntervalsForRestrictInfo()`: Retrieves the shard intervals based on restrictions provided by PostgreSQL's `standard_planner()`. ### Example Router Planner Queries ```sql -- Fetch user data and their respective orders for a given user_id SELECT u.username, o.order_id FROM users_table u, orders_table o WHERE u.user_id = o.user_id AND u.user_id = 42; -- With Subqueries: -- Fetch the username and their total order amount -- for a specific user SELECT u.username, (SELECT COUNT(*) FROM orders_table o WHERE o.user_id = 42 AND o.user_id = u.user_id) FROM users_table u WHERE u.user_id = 42; -- Router planner works with CTEs (and UPDATE/DELETE Query): -- Update the status of the most recent order for a specific user WITH RecentOrder AS ( SELECT MAX(order_id) as last_order_id FROM orders_table WHERE user_id = 42 ) UPDATE orders_table SET status = 'COMPLETED' FROM RecentOrder WHERE orders_table.user_id = 42 AND orders_table.order_id = RecentOrder.last_order_id; ``` ## Query Pushdown Planning in Citus ### Overview While Router and Fast-Path Router Planners are proficient at dealing with single-shard commands—making them ideal for multi-tenant and OLTP applications—Citus also excels in analytical use-cases. In these scenarios, a single query is broken down into multiple parallel sub-queries, which are run on various shards across multiple machines, thereby speeding up query execution times significantly. #### What is Query Pushdown Planning? Query Pushdown Planning is an extension of the Router Planning paradigm. Unlike the latter, which deals with single-shard, single-node queries, Query Pushdown can route a query to multiple shards across multiple nodes. Instead of verifying that all tables have the same filters, as in Router Planning, Query Pushdown ascertains that all tables are joined on their distribution keys. #### Core Functions The core C function responsible for this check is `RestrictionEquivalenceForPartitionKeys()`, which ensures that tables in the query are joined based on their distribution keys. Initially intended for subqueries, Query Pushdown has been extended to include other cases as well. The decision to utilize Query Pushdown is determined by the `ShouldUseSubqueryPushDown()` function. #### Understanding Query Pushdown Understanding Query Pushdown Planning and how it extends the simpler Router Planning can help you fully utilize Citus for your analytical workloads. #### Key Characteristics of Query Pushdown - **High Parallelism**: The query is broken down into multiple sub-queries, leading to parallel execution on multiple shards and nodes. - **Worker Subquery**: You will typically notice the alias `worker_subquery` in the SQL queries sent to the shards, indicating a pushdown operation. ### Examples of query pushdown #### Basic Example ```sql -- Count of distinct product_ids where user_ids from two different tables match SELECT count(DISTINCT product_id) FROM ( SELECT DISTINCT user_id as distinct_user_id FROM users_table ) foo, orders_table WHERE orders_table.user_id = distinct_user_id; ``` #### Subquery in Target List ```sql -- retrieves the most recent order date for each user SELECT (SELECT MAX(order_date) FROM orders_table o WHERE o.user_id = u.user_id) FROM users_table u; ``` #### Subquery in WHERE Clause ```sql -- Number of distinct users who have placed an order SELECT COUNT(DISTINCT u.user_id) FROM users_table u WHERE u.user_id IN ( SELECT o.user_id FROM orders_table o ); ``` #### More Examples ```sql -- Count of distinct products per user, with maximum order date from orders -- as a subquery in the target list SELECT (SELECT MAX(o.order_date) FROM orders_table o WHERE o.user_id = u.user_id), COUNT(DISTINCT o.product_id) FROM orders_table o, users_table u WHERE o.user_id = u.user_id GROUP BY u.user_id; ``` #### UPDATE and DELETE with Query Pushdown ```sql -- Update status in orders_table for users whose email ends with '@example.com' UPDATE orders_table o SET status = 'DISCOUNTED' FROM users_table u WHERE o.user_id = u.user_id AND u.email LIKE '%@example.com'; ``` ```sql -- Delete orders for users who were born before '2000-01-01' DELETE FROM orders_table o USING users_table u WHERE o.user_id = u.user_id AND u.date_of_birth < '2000-01-01'; ``` ## Recursive Planning Central to understanding Citus' approach to distributed query planning are two closely interrelated concepts: "Query Pushdown Planning" and "Recursive Planning." These dual strategies lay the foundation for Citus' capacity to manage complex query structures across multiple shards and nodes effectively. While Query Pushdown Planning optimizes queries by breaking them into smaller components that can run in parallel across multiple shards, Recursive Planning takes a more nuanced approach. It works its way through the query tree from the deepest level upwards, scrutinizing each subquery to determine its suitability for pushdown. The essence of recursive planning lies in treating each recursively planned query in isolation. This means correlated subqueries can't take advantage of recursive planning. However, (sub)queries on local tables can be done via recursive planning. This process is primarily executed in the `RecursivelyPlanSubqueryWalker()` C function. In this function, the engine goes to the innermost subquery and assesses whether it can safely be pushed down as a stand-alone query. If it can, the query engine simply moves on. However, if the subquery isn't suitable for pushdown, Citus generates a separate "sub-plan" for that subquery, substituting it with a `read_intermediate_result()` function call. These sub-plans are later executed as independent queries, a task overseen by the `ExecuteSubPlans()` function. The engine continues this way, moving upward through each level of subqueries, evaluating and, if needed, creating sub-plans until it reaches the top-level query. ### Intermediate Results as Reference Tables One of the key aspects of Recursive Planning is the use of "intermediate results." These are essentially the outcomes of subqueries that have been recursively planned and executed on worker nodes. Once these intermediate results are obtained, they are treated much like reference tables in the subsequent stages of query planning and execution. The key advantage here is that, like reference tables, these intermediate results can be joined with distributed tables on any column, not just the distribution key. ### Full SQL Coverage via Recursive Planning The practice of recursively creating sub-plans and generating intermediate results offers a workaround for achieving full SQL coverage in Citus. If each subquery in a complex SQL query can be replaced with an intermediate result, then the entire query essentially becomes a query on a reference table. This feature is a crucial aspect for many users who require comprehensive SQL support in their distributed systems. ### Trade-offs of using recursive planning While Recursive Planning brings a lot to the table, it's not without its drawbacks. First, the method inherently adds more network round-trips, as each recursively planned query is executed separately, and its results are pushed back to all worker nodes. Secondly, when functions like `read_intermediate_results` are used to fetch data from these intermediate results, it can confound the Postgres planner, particularly in the context of complex joins. As a result, query estimations may be inaccurate, leading to suboptimal execution plans. Understanding these facets of Recursive Planning can provide you with a comprehensive view of how Citus approaches distributed query planning, allowing you to better optimize your database operations. This may seem complex at first glance, but it's a bit like a step-by-step puzzle-solving process that the Citus query engine performs to optimize your database queries effectively. To help clarify these intricate mechanics, we'll present a series of examples. #### Recursive Plan Example 1: In the simplest example, we'll have a single subquery which is NOT pushdown-safe due to LIMIT 1, hence creating a subplan ```sql SET client_min_messages TO DEBUG1; SELECT count(*) FROM (SELECT * FROM users_table LIMIT 1) as foo; SET Time: 0.765 ms DEBUG: push down of limit count: 1 DEBUG: generating subplan 7_1 for subquery SELECT user_id, username, email, date_of_birth, country_code FROM public.users_table LIMIT 1 DEBUG: Plan 7 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id, intermediate_result.username, intermediate_result.email, intermediate_result.date_of_birth, intermediate_result.country_code FROM read_intermediate_result('7_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id bigint, username character varying(50), email character varying(50), date_of_birth date, country_code character varying(3))) foo ``` #### Recursive Plan Example 2: Now, we have multiple subqueries in the same level which are NOT pushdown-safe due to LIMIT 1 and GROUP BY non distribution keys, hence creating a subplan ```sql SELECT count(*) FROM (SELECT * FROM users_table LIMIT 1) as foo, (SELECT count(*) FROM users_table GROUP BY country_code) as bar; DEBUG: push down of limit count: 1 DEBUG: generating subplan 9_1 for subquery SELECT user_id, username, email, date_of_birth, country_code FROM public.users_table LIMIT 1 DEBUG: generating subplan 9_2 for subquery SELECT count(*) AS count FROM public.users_table GROUP BY country_code DEBUG: Plan 9 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id, intermediate_result.username, intermediate_result.email, intermediate_result.date_of_birth, intermediate_result.country_code FROM read_intermediate_result('9_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id bigint, username character varying(50), email character varying(50), date_of_birth date, country_code character varying(3))) foo, (SELECT intermediate_result.count FROM read_intermediate_result('9_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) bar ``` #### Recursive Plan Example 3: We have a subquery foo that is NOT safe-to-pushdown but once that subquery is replaced with an intermediate result, the rest of the query becomes safe-to-pushdown ```sql SELECT count(*) FROM (SELECT 1 FROM (SELECT user_id FROM users_table LIMIT 1) as foo, (SELECT * FROM orders_table) as o1, (SELECT * FROM users_table) as u2 WHERE foo.user_id = o1.user_id AND o1.user_id = u2.user_id) as top_level_subquery; DEBUG: push down of limit count: 1 DEBUG: generating subplan 1_1 for subquery SELECT user_id FROM public.users_table LIMIT 1 DEBUG: Plan 1 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT 1 AS "?column?" FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('1_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id bigint)) foo, (SELECT orders_table.order_id, orders_table.user_id, orders_table.product_id, orders_table.order_date, orders_table.status FROM public.orders_table) o1, (SELECT users_table.user_id, users_table.username, users_table.email, users_table.date_of_birth, users_table.country_code FROM public.users_table) u2 WHERE ((foo.user_id OPERATOR(pg_catalog.=) o1.user_id) AND (o1.user_id OPERATOR(pg_catalog.=) u2.user_id))) top_level_subquery ``` ### More advanced recursive planning constructs In the previous parts of the recursive planning examples, we only dealt with a subquery at a time. However, recursive planning is capable of considering multiple subqueries in the same query level or converting tables to subqueries in the same level. In this part of the document, let's discuss these advanced query planning capabilities. ### Set operations via recursive planning (and query pushdown) Set operations like UNION, UNION ALL, and EXCEPT are essentially two subqueries in the same query level. > **Note:** The rules for set operation planning on Citus can be confusing and should be taken carefully. Citus is capable of "pushing down" certain set operations: UNION and UNION ALL. To allow this, two rules must be met, which are defined in the `SafeToPushdownUnionSubquery()` C code. 1. The set operation cannot be on the top level; it should be wrapped into a subquery. This is purely an implementation limitation that can and should be eased. 2. For all subqueries, each leaf query should have a "distribution key" on the target list, and the ordinal positions of these "distribution keys" should match across all set operations. This second limitation is required to preserve correctness. #### Set operation query examples: ```sql -- safe to pushdown SELECT * FROM (SELECT * FROM users_table UNION SELECT * FROM users_table) as foo; ``` ```sql -- not safe to pushdown because the set operation is NOT wrapped into a subquery. -- Each leaf query is recursively planned. SELECT * FROM users_table UNION SELECT * FROM users_table; ``` ```sql -- not safe to pushdown because the distribution columns do NOT match (e.g., not existing) SELECT * FROM (SELECT username FROM users_table UNION SELECT username FROM users_table) as foo; ``` ```sql -- not safe to pushdown because the distribution columns do NOT match. SELECT * FROM (SELECT user_id + 1 FROM users_table UNION SELECT user_id - 1 FROM users_table) as foo; ``` ```sql -- EXCEPT is never safe to pushdown SELECT * FROM (SELECT * FROM users_table EXCEPT SELECT * FROM users_table) as foo; ``` ### Set operations and joins Although not very common, some users might have joins along with set operations. Example queries might look like: - `(SELECT .. t1 JOIN t2) UNION (SELECT t2 JOIN t3)` - `(SELECT .. t1 UNION SELECT t2) JOIN t3 ..` - `((SELECT .. t1 JOIN t2) UNION (SELECT t2 JOIN t3)) JOIN t4` For all these cases, similar rules apply: - JOINs should be made on the distribution keys. - SET operations should satisfy the `SafeToPushdownUnionSubquery()` conditions. When combined, all conditions should match. #### Safe to Pushdown Examples: ```sql -- All joins are on the distribution key and all the unions have the distribution key in the same ordinal position. SELECT * FROM ( (SELECT user_id FROM users_table u1 JOIN users_table u2 USING (user_id)) UNION (SELECT user_id FROM users_table u1 JOIN users_table u2 USING (user_id)) ) as foo; ``` ```sql -- All joins are on the distribution key and all the unions have the distribution key in the same ordinal position. SELECT * FROM (SELECT user_id FROM users_table u1 UNION SELECT user_id FROM users_table u2) as foo JOIN users_table u2 USING (user_id); ``` ### HAVING subqueries via recursive planning Postgres allows the HAVING clause to contain subqueries. If the subqueries in the HAVING clause don't reference the outer query (i.e., not correlated), then it's possible to recursively plan the subquery in the HAVING clause. This involves using the `RecursivelyPlanAllSubqueries()` function specifically for the HAVING clause. #### Example: ```sql -- Find user_ids who have placed more orders than the average number of orders per user. SELECT u.user_id, COUNT(o.order_id) AS total_orders FROM users_table u JOIN orders_table o ON u.user_id = o.user_id GROUP BY u.user_id HAVING COUNT(o.order_id) > (SELECT AVG(order_count) FROM ( SELECT user_id, COUNT(order_id) AS order_count FROM orders_table GROUP BY user_id) AS subquery); ``` ### Non-colocated subqueries via recursive planning Assume that there are two subqueries; each subquery is individually joined on their distribution keys. However, when the two subqueries are joined on arbitrary keys, the non-colocated subquery join logic kicks in, as described in `RecursivelyPlanNonColocatedSubqueries()`. #### Non-colocated subquery Example 1: ```sql -- Find users who do not have orders with status 'shipped' and 'pending' -- Sub1 and Sub2 are individually safe to pushdown. -- The join condition between them is: sub1.user_id != sub2.user_id, which does not preserve distribution key equality. -- Citus qualifies sub1 as the anchor subquery and checks whether all other subqueries are joined on the distribution key. -- In this case, sub2 is not joined on the distribution key, so Citus decides to recursively plan the whole sub2. SELECT sub1.user_id, sub2.user_id FROM ( SELECT u.user_id FROM users_table u JOIN orders_table o ON u.user_id = o.user_id WHERE o.status = 'shipped' GROUP BY u.user_id ) AS sub1 JOIN ( SELECT u.user_id FROM users_table u JOIN orders_table o ON u.user_id = o.user_id WHERE o.status = 'pending' GROUP BY u.user_id ) AS sub2 ON sub1.user_id != sub2.user_id; ``` #### Non-colocated subquery Example 2: ```sql -- Similar logic also applies for subqueries in the WHERE clause. -- Both the query in the FROM clause and the subquery in the WHERE clause are individually safe to pushdown. -- However, as a whole, the query is not safe to pushdown. -- Therefore, Citus decides to recursively plan the subquery in the WHERE clause. SELECT o1.order_id, o1.order_date FROM orders_table o1, users_table u1 WHERE o1.user_id = u1.user_id AND o1.order_date IN ( SELECT o2.order_date FROM orders_table o2, users_table u2 WHERE o2.user_id = u2.user_id AND o2.status = 'shipped' ); ``` ### Local table - distributed table JOINs via recursive planning In Citus, joins between a local table and a distributed table require special handling. The local table data resides on the Citus coordinator node, while the distributed table data is across multiple worker nodes. The `RecursivelyPlanLocalTableJoins()` C function handles this. #### Performance Characteristics Local and distributed table joins have specific performance traits. They push down filters and projections, meaning only relevant data is pulled to the coordinator. See the `RequiredAttrNumbersForRelation()` and `ReplaceRTERelationWithRteSubquery()` functions for more details. #### How It Works 1. Citus scans the query tree to find joins between local and distributed tables. 2. Upon finding such a join, Citus forms a sub-plan for the local table. 3. This sub-plan retrieves relevant data from the local table into an intermediate result and distributes it across worker nodes. 4. The original query is then rewritten, replacing the local table with these intermediate results. 5. Finally, this new query, now only involving distributed tables, is executed using Citus's standard query execution engine. #### Example 1 For example, consider a local table `local_users` and a distributed table `orders_table`. A query like this: ```sql SELECT * FROM local_users l, orders_table o WHERE l.user_id = o.user_id; ``` Would be internally transformed by Citus as follows: ```sql -- Create a temporary reference table and populate it with local table data CREATE TEMP TABLE temp_local_users AS SELECT * FROM local_users; SELECT create_reference_table('temp_local_users'); -- Replace the local table with the temporary distributed table in the original query SELECT * FROM temp_local_users t, orders_table o WHERE t.user_id = o.user_id; ``` #### Configuration Option By tweaking `citus.local_table_join_policy`, you can control how Citus behaves for queries involving local and distributed tables. The default behavior is to pull local table data to the coordinator, with exceptions for distributed tables filtered on primary key or unique index. #### Example 2 For instance, when the distributed table is guaranteed to return at most one row, Citus chooses to recursively plan the distributed table: ```sql SELECT * FROM local_users l, orders_table o WHERE l.user_id = o.user_id AND o.primary_key = 55; ``` ### Outer joins between reference and distributed tables In general, when the outer side of an outer join is a recurring tuple (e.g., reference table, intermediate results, or set returning functions), it is not safe to push down the join. ```sql "... ref_table LEFT JOIN distributed_table ..." "... distributed_table RIGHT JOIN ref_table ..." ``` In these situations, Citus recursively plans the "distributed" part of the join. Even though it may seem excessive to recursively plan a distributed table, remember that Citus pushes down the filters and projections. Functions involved here include `RequiredAttrNumbersForRelation()` and `ReplaceRTERelationWithRteSubquery()`. The core function handling this logic is `RecursivelyPlanRecurringTupleOuterJoinWalker()`. There are likely numerous optimizations possible (e.g., first pushing down an inner JOIN then an outer join), but these have not been implemented due to their complexity. #### Example Query Here's an example that counts the number of orders for each status, including only statuses that also appear in the reference table: ```sql SELECT os.status, COUNT(o.order_id) FROM order_status os LEFT JOIN orders_table o ON os.status = o.status GROUP BY os.status; ``` #### Debug Messages ``` DEBUG: recursively planning right side of the left join since the outer side is a recurring rel DEBUG: recursively planning distributed relation "orders_table" "o" since it is part of a distributed join node that is outer joined with a recurring rel DEBUG: Wrapping relation "orders_table" "o" to a subquery DEBUG: generating subplan 45_1 for subquery SELECT order_id, status FROM public.orders_table o WHERE true ``` As of Citus 13.2, under certain conditions, Citus can push down these types of LEFT and RIGHT outer joins by injecting constraints—derived from the shard intervals of distributed tables—into shard queries for the reference table. The eligibility rules for pushdown are defined in `CanPushdownRecurringOuterJoin()`, while the logic for computing and injecting the constraints is implemented in `UpdateWhereClauseToPushdownRecurringOuterJoin()`. #### Example Query In the example below, Citus pushes down the query by injecting interval constraints on the reference table. The injected constraints are visible in the EXPLAIN output. ```sql SELECT pc.category_name, count(pt.product_id) FROM product_categories pc LEFT JOIN products_table pt ON pc.category_id = pt.product_id GROUP BY pc.category_name; ``` #### Debug Messages ``` DEBUG: Router planner cannot handle multi-shard select queries DEBUG: a push down safe left join with recurring left side ``` #### Explain Output ``` HashAggregate Group Key: remote_scan.category_name -> Custom Scan (Citus Adaptive) Task Count: 32 Tasks Shown: One of 32 -> Task Node: host=localhost port=9701 dbname=ebru -> HashAggregate Group Key: pc.category_name -> Hash Right Join Hash Cond: (pt.product_id = pc.category_id) -> Seq Scan on products_table_102072 pt -> Hash -> Seq Scan on product_categories_102106 pc Filter: ((category_id IS NULL) OR ((btint4cmp('-2147483648'::integer, hashint8((category_id)::bigint)) < 0) AND (btint4cmp(hashint8((category_id::bigint), '-2013265921'::integer) <= 0))) ``` ### Recursive Planning When FROM Clause has Reference Table (or Recurring Tuples) This section discusses a specific scenario in Citus's recursive query planning: handling queries where the main query's `FROM` clause is recurring, but there are subqueries in the `SELECT` or `WHERE` clauses involving distributed tables. #### Definitions - **Recurring**: Here, "recurring" implies that the `FROM` clause doesn't contain any distributed tables. Instead, it may have reference tables, local tables, or set-returning functions. - **Subqueries in SELECT and WHERE**: In case the main query's `FROM` clause is recurring, then no distributed tables should be present in the `SELECT` and `WHERE` subqueries. #### Citus's Approach Citus solves this by recursively planning these problematic subqueries, effectively replacing them with calls to `read_intermediate_result()`. #### Handling the WHERE Clause For the `WHERE` clause, the function `RecursivelyPlanAllSubqueries` is called, transforming all subqueries within it. ```sql -- Main query FROM clause is recurring, but -- WHERE clause contains a pushdownable subquery from -- orders_table (distributed table) SELECT country_name FROM country_codes WHERE country_code IN (SELECT country_code FROM users_table WHERE user_id IN (SELECT user_id FROM orders_table)); ``` #### Handling the SELECT Clause Similarly, `RecursivelyPlanAllSubqueries` is called for the `SELECT` clause to replace any existing subqueries. ```sql -- Main query FROM clause is recurring, but SELECT clause contains a subquery from orders_table (distributed table) SELECT (SELECT COUNT(*) FROM orders_table WHERE status = 'shipped') AS shipped_orders, country_name FROM country_codes; ``` In both examples, since the main query's `FROM` clause is recurring and involves subqueries on distributed tables in `WHERE` or `SELECT`, Citus uses `RecursivelyPlanAllSubqueries` to manage these subqueries. ### Logical Planner & Optimizer At the high level, all multi-task queries go through the logical planner. However, when it comes to query pushdown or the recursive planner, the logical planner does very little. Most of its complexity deals with multi-shard queries that don't fall into these categories. Below, we are going to discuss those details. #### Simple Example The simplest example of a query processed by the logical planner would be: ```sql SELECT * FROM users_table; ``` #### Academic Background The logical planner implements the concepts from the paper: "Correctness of query execution strategies in distributed databases." The paper is available [here](https://dl.acm.org/doi/pdf/10.1145/319996.320009). If you find the paper hard to read, Marco provides a good introduction to the same concepts in the following presentation: - [YouTube Video](https://www.youtube.com/watch?v=xJghcPs0ibQ) - [Speaker Deck](https://speakerdeck.com/marcocitus/scaling-out-postgre-sql) #### Core Functions We assume you have either watched the video or read the paper. The core C functions involved are `MultiLogicalPlanCreate()`, `MultiNodeTree()`, and `MultiLogicalPlanOptimize()`. Citus has a rules-based optimizer. The core function `MultiLogicalPlanCreate()` maps the SQL query to a C structure (e.g., `MultiNode`). Then `MultiLogicalPlanOptimize()` applies available optimizations to the `MultiNode`. For instance, one simple optimization pushes the "filter" operation below the "MultiCollect." Such rules are defined in the function `Commutative()` in `multi_logical_optimizer.c`. The most interesting part of the optimizer is usually in the final stage, when handling the more complex operators (GROUP BY, DISTINCT window functions, ORDER BY, aggregates). These operators are conjoined in a `MultiExtendedOpNode`. In many cases, they can only partially be pushed down into the worker nodes, which results in one `MultiExtendedOpNode` above the `MultiCollect` (which will run on the coordinator and aggregates across worker nodes), and another `MultiExtendedOpNode` below the `MultiCollect` (which will be pushed down to worker nodes). The bulk of the logic for generating the two nodes lives in `MasterExtendedOpNode()` and `WorkerExtendedOpNode()`, respectively. ##### Aggregate functions [Aggregate functions](https://www.postgresql.org/docs/current/sql-createaggregate.html) can appear in the SELECT (target list) or HAVING clause of a query, often in the context of a `GROUP BY`. The aggregate primarily specify a state function (`sfunc`), which is called for every row in the group, and an `stype` which defines the data format in which intermediate state is held as a type, which maybe be `internal`. Many aggregates also have a `finalfunc`, which converts the last `stype` value to the final result of the aggregate function. Citus support distributing aggregate functions in several ways described below, each with an example. **Aggregate functions in queries that group by distribution column can be fully pushed down, since no cross-shard aggregation is needed**. This is mostly handled by the rules in `CanPushDownExpression`. Example: ```sql select x, avg(y) from test group by x; DEBUG: combine query: SELECT x, avg FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cstring(0), '(i 1)'::cstring(0)) remote_scan(x integer, avg numeric) NOTICE: issuing SELECT x, avg(y) AS avg FROM public.test_102041 test WHERE true GROUP BY x NOTICE: issuing SELECT x, avg(y) AS avg FROM public.test_102042 test WHERE true GROUP BY x ... ``` **Built-in, or well-known aggregate functions (based on their name) are distributed using custom rules**. An almost-complete list of aggregates that are handled in this way can be found in the `AggregateNames` variable. Examples are `avg`, `sum`, `count`, `min`, `max`. To distribute an aggregate function like `avg`, the optimizer implements rules such as injecting a `sum` and `count` aggregate in the worker target list, and doing a `sum(sum)/sum(count)` on the master target list. The logic is agnostic to types, so it will for work any custom type that implements aggregate functions with the same name. Example: ```sql select y, avg(x) from test group by y; DEBUG: combine query: SELECT y, (pg_catalog.sum(avg) OPERATOR(pg_catalog./) pg_catalog.sum(avg_1)) AS avg FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cs tring(0), '(i 1)'::cstring(0)) remote_scan(y integer, avg bigint, avg_1 bigint) GROUP BY y NOTICE: issuing SELECT y, sum(x) AS avg, count(x) AS avg FROM public.test_102041 test WHERE true GROUP BY y NOTICE: issuing SELECT y, sum(x) AS avg, count(x) AS avg FROM public.test_102042 test WHERE true GROUP BY y ``` **Aggregates that specify a `combinefunc` and have an non-internal `stype` are distributed using generic aggregate functions**. The `worker_partial_agg` aggregate function is pushed down to the worker runs the `sfunc` of the custom aggregate across the tuples of a shard without running the `finalfunc` (which should come after `combinefunc`). The `coord_combine_agg` aggregate function runs the `combinefunc` across the `stype` values returned by `worker_partial_agg` and runs the `finalfunc` to obtain the final result of the aggregate function. This approach currently does not support aggregates whose `stype` is `internal`. A reason we for not handling `internal` is that it is not clear that they can always be safely transferred to a different server, though that may be overly pedantic. Example: ```sql select st_memunion(geo) from test; DEBUG: combine query: SELECT coord_combine_agg('351463'::oid, st_memunion, NULL::postgis_public.geometry) AS st_memunion FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cstring(0), '(i 1)'::cstring(0)) remote_scan(st_memunion cstring) NOTICE: issuing SELECT worker_partial_agg('postgis_public.st_memunion(postgis_public.geometry)'::regprocedure, geo) AS st_memunion FROM public.test_102041 test WHERE true NOTICE: issuing SELECT worker_partial_agg('postgis_public.st_memunion(postgis_public.geometry)'::regprocedure, geo) AS st_memunion FROM public.test_102042 test WHERE true ``` **Other aggregates will be fully above the `MultiCollect` node, meaning the source data is pulled to the coordinator.** If this is undesirable due to the performance/load risk, it can be disabled using `citus.coordinator_aggregation_strategy = 'disabled'`, in which case the aggregate function calls would result in an error. Example: ```sql select st_union(geo) from test; DEBUG: combine query: SELECT postgis_public.st_union(st_union) AS st_union FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cstring(0), '(i 1)'::cstring(0)) remote_scan(st_union postgis_public.geometry) NOTICE: issuing SELECT geo AS st_union FROM public.test_102041 test WHERE true NOTICE: issuing SELECT geo AS st_union FROM public.test_102042 test WHERE true ``` ### Multi Join Order **Context and Use Case**: This query planning mechanism is primarily geared towards data warehouse type of query planning. It's worth noting that the Citus team has not actively pursued optimizations in this direction, resulting in some non-optimized code paths. **Join Order Optimization**: In Citus' logical planner, the `JoinOrderList()` function serves to choose the most efficient join order possible. However, its primary focus has been on joins that require repartitioning, as well as some specific non-repartition joins. For example, joins on distribution keys that are not eligible for pushdown planning may pass through this code path, although no optimizations are made in those cases. **Algorithm Simplicity**: The current algorithm, encapsulated in the `BestJoinOrder()` function, is relatively naive. While it aims to minimize the number of repartition joins, it does not provide a performance evaluation for each of them. This function provides room for performance optimizations, especially when dealing with complex joins that necessitate repartitioning. **Control via GUCs**: Two GUCs control the behavior of repartitioning in Citus: `citus.enable_single_hash_repartition_joins` and `citus.repartition_join_bucket_count_per_node`. - **citus.enable_single_hash_repartition_joins**: The default value is "off". When "off", both tables involved in the join are repartitioned. When "on", if one table is already joined on its distribution key, only the other table is repartitioned. - **citus.repartition_join_bucket_count_per_node**: This setting defines the level of parallelism during repartitioning. The reason for the "off" default is tied to this GUC. Opting for a fixed bucket count, rather than dynamically adjusting based on shard count, provides more stability and safety. If you ever consider changing these defaults, be cautious of the potential performance implications. ### Combine Query - **Overview**: The multi-task SELECT queries pull results to the coordinator, and the tuples returned always go through the "combine query". - **Structure and Source**: The `combineQuery` can be traced back through the `DistributedPlan->combineQuery` struct. This query is essentially constructed in the `CreatePhysicalDistributedPlan` function. However, the actual source comes from `MasterExtendedOpNode()` within the logical optimizer. For deeper insights into this logic, you can refer to the paper and video links shared under the "Logical Planner & Optimizer" section. - **Example**: The simplest example is the following where Citus sends `count(*)` to the shards, and needs to do a `sum()` on top of the results collected from the workers. ```sql SET client_min_messages TO DEBUG4; DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT count(*) AS count FROM public.users_table_102008 users_table WHERE true" .... DEBUG: combine query: SELECT COALESCE((pg_catalog.sum(count))::bigint, '0'::bigint) AS count FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cstring(0), '(i 1)'::cstring(0)) remote_scan(count bigint) D ``` ### CTE Processing - **In Postgres 13 and Later Versions**: In Postgres 13 and later versions, CTEs (Common Table Expressions) are almost like subqueries. Usually, these CTEs are transformed into subqueries during `standard_planner()`. Citus follows the same approach via `RecursivelyInlineCtesInQueryTree()`. - **Additional Consideration in Citus**: For Citus, there's an additional consideration. CTEs that aren't inlined get materialized. In the Citus context, materialization converts these CTEs into intermediate results. Some users leverage this for achieving full-SQL coverage. - **Extra CTE Check in Citus**: Citus includes an extra check before inlining CTEs, conducted by the function `TryCreateDistributedPlannedStmt`. Here, the planner first tries to inline all CTEs and then checks whether Citus can still plan the query. If not, the CTEs remain as is, leading to their materialization. If all CTEs are materialized (e.g., read_intermediate_result), then the query becomes equivalent of a query on reference table, hence full SQL. **Examples for Better Understanding**: I understand the logic might seem complex at first. Simple examples will be provided for better understanding. ```sql -- a CTE that is inlined as subquery, and does a query-pushdown WITH cte_1 AS (SELECT DISTINCT user_id FROM orders_table) SELECT * FROM cte_1; ``` So, from Citus' query planning perspective the above CTE is equivalent to the following subquery ```sql SELECT * FROM (SELECT DISTINCT user_id FROM orders_table) cte_1; ``` Once a CTE is inlined, then the rest of the query planning logic kicks in for example, below, the cte is inlined and then because the subquery is NOT safe to pushdown it is recursively planned ```sql WITH cte_1 AS (SELECT DISTINCT product_id FROM orders_table) SELECT * FROM cte_1; .. DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: generating subplan 81_1 for subquery SELECT DISTINCT product_id FROM public.orders_table DEBUG: Plan 81 query after replacing subqueries and CTEs: SELECT product_id FROM (SELECT intermediate_result.product_id FROM read_intermediate_result('81_1'::text, 'binary'::citus_copy_format) intermediate_result(product_id bigint)) cte_1; ``` - **Which CTEs Are Materialized**: Citus follows the same rules as Postgres. See [Postgres documentation](https://www.postgresql.org/docs/current/queries-with.html#id-1.5.6.12.7). ```sql -- the same query as the first query -- but due to MATERIALIZED keyword -- Citus converts the CTE to intermediate result WITH cte_1 AS MATERIALIZED (SELECT DISTINCT user_id FROM orders_table) SELECT * FROM cte_1; -- the same query as the first query -- but as the same cte used twice -- Citus converts the CTE to intermediate result WITH cte_1 AS (SELECT DISTINCT user_id FROM orders_table) SELECT * FROM cte_1 as c1 JOIN cte_1 as c2 USING (user_id); ``` - **Citus Specific Materialization**: Citus first tries to inline the CTEs, but if it decides that after inlining the query cannot be supported due Citus' SQL limitations, it lets the CTE to be materialized. As of writing this document, Citus does NOT support GROUPING SETs on distributed tables/subqueries. So, when we inline the CTE, then Citus would try to plan a query with GROUPING SETs on a distributed table, which would fail. Then, citus would materialize the cte and the final query would be GROUPING SET on an intermediate result, hence can be supported ```sql WITH users_that_have_orders AS (SELECT users_table.* FROM users_table JOIN orders_table USING (user_id)) SELECT max(date_of_birth) FROM users_that_have_orders GROUP BY GROUPING SETS (user_id, email); ... DEBUG: CTE users_that_have_orders is going to be inlined via distributed planning ... DEBUG: Planning after CTEs inlined failed with message: could not run distributed query with GROUPING SETS, CUBE, or ROLLUP hint: Consider using an equality filter on the distributed table''s partition column. ... DEBUG: generating subplan 98_1 for CTE users_that_have_orders: SELECT users_table.user_id, users_table.username, users_table.email, users_table.date_of_birth, users_table.country_code FROM (public.users_table JOIN public.orders_table USING (user_id)) ``` ### INSERT Query Planning **At a High-Level Overview**: - There are approximately 4 different ways that an INSERT command can be planned in Citus. The first one is the INSERT ... SELECT command, which will be discussed separately. **INSERT with Sublink (Not Supported)**: ```sql INSERT INTO users_table (user_id) VALUES ((SELECT count(8) FROM orders_table)); ERROR: subqueries are not supported within INSERT queries HINT: Try rewriting your queries with 'INSERT INTO ... SELECT' syntax. INSERT INTO users_table (user_id) VALUES (1) RETURNING (SELECT count(*) FROM users_table); ERROR: subqueries are not supported within INSERT queries HINT: Try rewriting your queries with 'INSERT INTO ... SELECT' syntax. ``` **Simple Inserts with a Single VALUES Clause**: -- As noted in the "fast-path router planner", these INSERT commands are planned with fast-path planning. This does not require calling into `standard_planner()`, and the distribution key should be extracted from the query itself. ```sql INSERT INTO users_table VALUES (1, 'onder', 'onderkalaci@gmail.com', now() - '5 years'::interval, 'TR'); ``` **Main Functions**: The main functions involved in this path are `RegenerateTaskListForInsert()`, `FastPathRouterQuery()`, and `RouterInsertTaskList`. For single-row INSERT tasks, `Job->deferredPruning=true`, meaning we can always do the shard pruning during execution. **Multi-row INSERTs**: For multi-row INSERTs, `RouterInsertTaskList()` becomes slightly more interesting. Citus groups rows by target shard. ```sql INSERT INTO orders_table (order_id, user_id) VALUES (1, 1), (2, 2), (3, 1), (4, 3), (5, 2); ``` **Debug Info**: Debug information shows how the query is rebuilt for different user_ids. Here, the shard_count is 4. ```sql -- for user_id: 1 DEBUG: query after rebuilding: INSERT INTO public.orders_table_102041 AS citus_table_alias (order_id, user_id) VALUES ('1'::bigint,'1'::bigint), ('3'::bigint,'1'::bigint) -- for user_id: 3 DEBUG: query after rebuilding: INSERT INTO public.orders_table_102055 AS citus_table_alias (order_id, user_id) VALUES ('4'::bigint,'3'::bigint) -- for user_id: 2 DEBUG: query after rebuilding: INSERT INTO public.orders_table_102064 AS citus_table_alias (order_id, user_id) VALUES ('2'::bigint,'2'::bigint), ('5'::bigint,'2'::bigint) ``` ### INSERT.. SELECT and MERGE Command Query Planning **Overview**: -- This section discusses `INSERT .. SELECT` and `MERGE` commands, which share almost identical planning logic. **Planning Methods**: Broadly, there are three methods to plan these commands: 1. Pushdown 2. Pull-to-coordinator 3. Repartition **Performance Considerations**: When it comes to performance and resource utilization, pushdown is generally the most efficient. For handling large data sizes, the repartition method scales better than the pull-to-coordinator method. **Further Reading**: For more detailed information on pushdown and repartition methods, refer to this [blog post](https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge/). The post focuses on the `MERGE` command but is also applicable to `INSERT .. SELECT`. **Examples**: The following section will delve into examples, starting with simple ones and moving to more complex scenarios. ### INSERT.. SELECT Query Planning **Overview**: The `INSERT .. SELECT` pushdown logic builds upon the pushdown planning for `SELECT` commands. The key requirements include colocated tables and matching distribution columns. Relevant C functions are `CreateDistributedInsertSelectPlan`, `DistributedInsertSelectSupported()`, and `AllDistributionKeysInQueryAreEqual`. **Additional Conditions for INSERT .. SELECT pushdown**: - The destination table's distribution keys should match the source query's distribution column. **Simplest INSERT .. SELECT Pushdown Example**: ```sql INSERT INTO users_table SELECT * FROM users_table; ``` **INSERT .. SELECT with Subqueries/Joins**: Provided subqueries can be pushed down, additional checks such as matching distribution columns are performed. ```sql INSERT INTO users_table SELECT users_table.* FROM users_table, (SELECT user_id FROM users_table JOIN orders_table USING (user_id)) as foo WHERE foo.user_id = users_table.user_id; ``` **Non-pushdownable Scenarios**: **Due to Distribution Key Mismatch**: Citus opts for repartitioning since no "merge step" is needed for the `SELECT` query. The deciding function is `IsRedistributablePlan()`. ```sql INSERT INTO users_table (user_id) SELECT user_id + 1 FROM users_table; ``` **Due to LIMIT**: The `SELECT` query requires a "merge step" for the `LIMIT` clause. Citus uses the pull-to-coordinator strategy. ```sql INSERT INTO users_table SELECT * FROM users_table LIMIT 5; ``` **Pull-to-Coordinator Details**: Citus typically pulls `SELECT` results and initiates a `COPY` command to the destination table. See `NonPushableInsertSelectExecScan()`. **Special Cases**: **ON CONFLICT or RETURNING**: In these cases, a simple `COPY` is insufficient. Citus pushes results as "colocated intermediate files" on the workers, which are colocated with the target table's shards. Then, Citus performs an `INSERT .. SELECT` on these colocated intermediate results. See `ExecutePlanIntoColocatedIntermediateResults()` and `GenerateTaskListWithColocatedIntermediateResults()`. **Example: Pull-to-coordinator with COPY back to shards**: ```sql INSERT INTO users_table SELECT * FROM users_table LIMIT 5; ``` **Example: Pull-to-coordinator with push as colocated intermediate results**: ```sql INSERT INTO users_table SELECT * FROM users_table LIMIT 5 ON CONFLICT(user_id) DO NOTHING; ``` ### MERGE Command Query Planning **Overview**: The `MERGE` command planning is similar to `INSERT .. SELECT`. The key difference is in the pull-to-coordinator strategy. `MERGE` always uses "colocated intermediate result" files, as the final executed command must be a `MERGE` command, not a `COPY`. The entry function in the code is `CreateMergePlan()`. **Further Reading**: For more insights, check out this [blog post](https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge/). **Pushdown MERGE Example**: The join is based on the distribution key. ```sql MERGE INTO users_table u USING orders_table o ON (u.user_id = o.user_id) WHEN MATCHED AND o.status = 'DONE' THEN DELETE; ``` **Pull-to-Coordinator MERGE Example**: The source query requires a "merge step" on the coordinator. ```sql MERGE INTO users_table u USING (SELECT * FROM orders_table ORDER BY order_date LIMIT 50) o ON (u.user_id = o.user_id) WHEN MATCHED AND o.status = 'DONE' THEN DELETE; ``` **Repartition MERGE Example**: The join is NOT on the distribution key, and the source query doesn't require a "merge step" on the coordinator. Note that this example is mostly hypothetical to illustrate the case. ```sql MERGE INTO users_table u USING (SELECT * FROM orders_table ORDER BY order_date) o ON (u.user_id = o.product_id) WHEN MATCHED AND o.status = 'DONE' THEN DELETE; ``` ### UPDATE / DELETE Planning **Overview**: The planning logic for UPDATE/DELETE queries is quite similar to what we've discussed for INSERT and MERGE commands. There are essentially four primary methods of planning: **1) Fast-Path Router Planning**: Targets a single shard and filters on the distribution key in the WHERE clause. ```sql UPDATE users_table SET email = 'new@email.com' WHERE user_id = 5; ``` **2) Router Planning**: Targets a single shard, but all the shards are on a single node and are colocated. ```sql UPDATE users_table u SET email = '' FROM orders_table o WHERE o.user_id = u.user_id AND u.user_id = 5 AND o.status = 'done'; ``` **3) Pushdown Planning**: The query can be pushed down to worker nodes, targeting multiple shards. Joins are also possible if they are on distribution keys. ```sql UPDATE users_table SET email = 'new@email.com' WHERE user_id IN (SELECT user_id FROM orders_table WHERE status = 'in progress'); ``` **Additional Example for Pushdown with Materialized CTE**: ```sql WITH high_value_users AS ( SELECT user_id FROM orders_table WHERE status = 'done' ORDER BY order_date LIMIT 50 ) UPDATE users_table SET username = 'High Value' WHERE user_id IN (SELECT user_id FROM high_value_users); ``` **4) Recursive Planning**: Used for more complex queries, like those with subqueries or joins that can't be pushed down. The queries are planned recursively. ```sql DELETE FROM users_table WHERE user_id IN (SELECT user_id FROM orders_table WHERE order_date < '2023-01-01' ORDER BY order_date LIMIT 5); ``` ### Correlated/Lateral Subqueries in Planning **Overview**: Correlated or LATERAL subqueries have special behavior in Citus. They can often be pushed down, especially when the join is on the distribution key. There are limitations for joins not on the distribution key. **Key Code Details**: For more information on the code, check the following functions: `DeferErrorIfCannotPushdownSubquery()` -> `ContainsReferencesToOuterQuery()`, `DeferErrorIfSubqueryRequiresMerge()`, `DeferredErrorIfUnsupportedLateralSubquery()`. LATERAL queries are different/unique: even if the subquery requires a merge step such as a `LIMIT`, if the correlation is on the distribution column, we can push it down. See [#4385](https://github.com/citusdata/citus/pull/4385). **Example 1**: Using LATERAL, where the join is on the distribution key. ```sql SELECT u.*, o_sum.total FROM users_table u, LATERAL (SELECT count(DISTINCT status) as total FROM orders_table o WHERE o.user_id = u.user_id) o_sum; ``` **Example 2**: Complex LATERAL with GROUP BY on a non-distribution key. It's pushdownable because the join is on the distribution key. ```sql SELECT u.*, o_info.product, o_info.total FROM users_table u, LATERAL ( SELECT o.product_id as product, count(DISTINCT o.status) as total FROM orders_table o WHERE o.user_id = u.user_id GROUP BY o.product_id ) o_info; ``` **Debug and Error Messages**: When it's not possible to push down correlated subqueries, recursive planning also can't be used. ```sql SELECT u.* FROM users_table u, LATERAL ( SELECT o.product_id as product FROM orders_table o WHERE o.user_id != u.user_id ) o_info; DEBUG: skipping recursive planning for the subquery since it contains references to outer queries ERROR: complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator ``` ### Planning Methodologies in Citus: Compatibility and Incompatibility #### Compatibilities 1. **Interleaving Recursive and Pushdown Planning**: - Recursive planning and pushdown planning can often be interleaved within a query. This allows for greater flexibility and optimized performance. 2. **Router Queries in Recursive Planning**: - Subqueries in recursive planning can often be router queries. This includes both fast-path router and regular router queries. 3. **Command Types**: - Command types like `UPDATE`, `DELETE`, `MERGE`, and `INSERT .. SELECT` can work well with both pushdown and recursive planning. #### Incompatibilities 1. **Repartition Joins**: - Repartition joins are generally incompatible with both recursive and pushdown planning. If a query uses recursive planning, it can't also use repartition joins. However, re-partition joins can be in a CTE that is recursively planned. #### Examples of Compatibility and Incompatibility ##### Recursive and Pushdown Planning ```sql -- Example 1: Recursive and Pushdown Planning Interleaved -- subquery is recursively planned multi-shard command WITH recent_orders AS ( SELECT * FROM orders_table ORDER BY order_date LIMIT 10 ) SELECT * FROM users_table WHERE user_id IN (SELECT user_id FROM recent_orders); ``` ##### Router Queries in Recursive Planning ```sql -- Example 2: Subquery as Fast-Path Router Query is recursively planned -- the rest is pushdown WITH user_info AS ( SELECT * FROM users_table WHERE user_id = 5 ORDER BY date_of_birth LIMIT 1 ) SELECT * FROM orders_table WHERE user_id IN (SELECT user_id FROM user_info); ``` ##### UPDATE Pushdown and Recursive Planning ```sql -- Example 3: UPDATE command with Pushdown, Router and Recursive Planning -- recursively planned router query and the rest is pushdown WITH high_value_users AS ( SELECT user_id FROM orders_table WHERE user_id = 15 AND status = 'done' ORDER BY order_date LIMIT 50 ) UPDATE users_table SET username = 'High Value' WHERE user_id IN (SELECT user_id FROM high_value_users); ``` #### Incompatibility with Repartition Joins ```sql -- Example 4: Incompatible Query involving Recursive Planning and Repartition Joins -- This query will fail because it tries to use recursive planning for recent_orders -- and trying to repartition joins between o2 and recent_orders WITH recent_orders AS ( SELECT * FROM orders_table WHERE order_date > '2023-01-01' LIMIT 10 ) SELECT u.* FROM users_table u JOIN recent_orders o ON u.user_id = o.product_id JOIN orders_table o2 ON o2.product_id = o.product_id; ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns ``` ## Combine query planner The combine planner is the final stage of planning for multi-shard queries. The logical & physical planner path creates a combine query, which will run on the coordinator. The combine query contains a special function call (called the extra data container), which can be observed using debug messages emitted during planning: ``` SET client_min_messages TO debug4; SELECT count(*) FROM test; … DEBUG: combine query: SELECT COALESCE((pg_catalog.sum(count))::bigint, '0'::bigint) AS count FROM pg_catalog.citus_extradata_container(10, NULL::cstring(0), NULL::cstring(0), '(i 1)'::cstring(0)) remote_scan(count bigint) ``` The reason we use a special function call is simply that it lets us put custom information in query trees. We use the same approach to pass sharding information into the deparser. In the combine query planner, we run the combine query through standard_planner and use the set_rel_pathlist_hook to inject a CustomPath plan for the function call. The CustomPath translates into the Citus Custom Scan that runs a Job. ## Restriction Equivalence In the PostgreSQL source code, an `EquivalenceClass` is a data structure used in query optimization. It is a way to represent a set of expressions in a query that are all equal. The PostgreSQL query planner uses this information to choose the most efficient execution plan for a query. For example, let's say you have a query like this: ```sql SELECT * FROM table1, table2 WHERE table1.a = table2.b AND table1.a = 5; ``` Here, `table1.a`, `table2.b`, and `5` can all be considered to belong to the same equivalence class because they are equal. Knowing this, the query optimizer might choose to use an index on `table1.a` or `table2.b` to speed up the query, among other optimizations. One level beyond that, Postgres can also apply transitivity rules for joins: ```sql SELECT * FROM table1, table2,table3 WHERE table1.a = table2.a AND table1.a = table3.a; ``` Here, `table1.a`, `table2.a`, and `table3.a` can all be considered to belong to the same equivalence class because they are (transitively) equal. Citus finds this information important. But Citus and Postgres have different structures. In Postgres, each (sub)query inside a big query is planned by itself. Citus tries to plan the whole big query at one time for performance reasons (see Query Pushdown Planning). This makes how they use Equivalence Classes different. In Postgres, each subquery has its own Equivalence Classes. But Citus needs Equivalence Classes for the whole big query. For example: ```sql SELECT count(*) FROM (SELECT a FROM t1 JOIN t2 USING(a)) as foo, (SELECT a FROM t3 JOIN t4 USING (a)) as bar USING (a); ``` For Postgres, it's enough to make Equivalence Classes for the subqueries `foo` and `bar`. Then make another one for the top-level query where `foo` and `bar` join. Postgres can plan joins this way. In Citus, we need to check if all tables in the big query (t1,t2,t3,t4) join on distribution columns. Citus makes Equivalence Classes for the whole big query. The logic is in the `AttributeEquivalenceClass` C struct. The function `GenerateAllAttributeEquivalences()` makes this structure in Citus. The idea is to simply merge all the Equivalence classes of different query levels into a one Equivalence Class (e.g., AttributeEquivalenceClass) Citus also introduces a new idea: RTEIdentity. Each table in the query gets a unique ID called RTEIdentity (see `AssignRTEIdentity()` C function). This ID helps make a new type of Equivalence Class that works for many levels of small queries. Without RTEIdentity, we can't tell tables apart in different levels of the query. We rely on a hack while assigning the RTEIdentities. We basically use a field in `RangeTblEntry` struct that we are sure it is not used for tables. In practice, this might break at some point. ## Recurring Tuples The Recurring Tuples concept in Citus helps manage expressions that give the same set of results across multiple shards. This is mainly useful for JOIN operations. The idea is to understand and handle how some tables or functions behave the same way across different shards of a distributed table. This concept helps to provide accurate error messages if such recurring tuples are used in a way that might give wrong results. The `RecurringTuplesType` enum in the code helps categorize these recurring tuples into different types. The types include: - Reference Table - Function - Empty Join Tree - Result Function - Values The main point is that recurring tuples "recur" for each shard in a multi-shard query. For example, consider a JOIN between a distributed table and a reference table. The query on each shard would look something like this: ```sql SELECT ... FROM dist_table_shard_1 JOIN ref_table_shard_1; SELECT ... FROM dist_table_shard_2 JOIN ref_table_shard_1; ... SELECT ... FROM dist_table_shard_n JOIN ref_table_shard_1; ``` Here, `ref_table_shard_1` is a recurring tuple because it appears in each shard query of the distributed table (`dist_table_shard_X`). It "recurs" for each shard, making it a recurring tuple. In summary, the Recurring Tuples concept in Citus helps in managing and identifying expressions that behave the same way across different shards, mainly to ensure accurate query results and error handling. # Executor Citus primarily hooks into the PostgreSQL executor by producing a query plan with a CustomScan. The overall hierarchy of where Citus hooks into the executor looks like this: - PostgreSQL executor - ExecutorRun_hook - Subplans are executed before regular execution - CustomScan functions are invoked as part of overall scan tree - BeginCustomScan (which steps are included depends on the query) - Function calls & distribution column parameters are evaluated - Deferred shard pruning - Lock shards to prevent concurrent move (write only) - Find placements for shards - ExecCustomScan - Adaptive Executor executes a list of tasks and concatenates the results into a tuple store - Re-partition jobs are executed - Remote tasks are executed - Local tasks are executed We describe each part in more detail below. ## Custom scan The Custom scan is the main entry point for the executor into Citus. The whole query plan might be a single Custom Scan node (e.g. single shard queries), or it can be a leaf node in a query plan that aggregates results across shards. The BeginCustomScan function evaluates function calls, parameters, and performs deferred pruning, and local plan caching, which are described in the next few sections. The ExecCustomScan function runs the adaptive executor which executes a list of tasks across the worker nodes. We also use top-level executor hooks, but primarily to capture some execution time information. The one important thing we do in the top-level ExecutorRun hook is execute subplans. That is because we allow subqueries to appear in certain parts of the combine query, and in case of a subquery on a Citus table that subquery needs to be executed before the overall plan. We use a separate custom scans for insert..select and merge commands due to the specialized nature of these commands (multiple phases). ![Diagram of CustomScan APIs](https://wiki.postgresql.org/images/0/05/CustomScan_Fig01.png) ## Function evaluation It is often necessary to evaluate function calls on the coordinator, rather than pushing them down to the worker node. One example is evaluating the `nextval('my_sequence')` in an insert, or stable functions like `now()` that should return the same value for the duration of the query. This is especially true for writes to replicated (reference) tables, since we cannot afford to push down function calls that might return different values on different nodes. We perform function evaluation on the “job query” of the distributed plan in `ExecuteCoordinatorEvaluableExpressions`, before deparsing the query. Whether a function call should be evaluated once on the coordinator, or many times (e.g. for every row) depends on the context in which the function call appears. For instance, a function call in a WHERE or SELECT clause might be evaluated many times, while a function call in a VALUES clause will only be evaluated once. On the other hand, stable & immutable functions are expected to return the same result for the same input for the whole query execution, so they should be evaluated once, unless their input can change (e.g. parameter is a column). So far, the function evaluation logic does not distinguish between different contexts within queries. Instead, we follow a simple policy: - For inserts, evaluate all function calls, including calls to volatile functions, but disallow stable/volatile functions in RETURNING - For update/delete, evaluate all function calls, but disallow volatile functions - For select, do not evaluate function calls on coordinator (not entirely correct) When DML commands appear in a CTE, the restriction only applies to the CTE. In many cases, the CTE will in that case be planned and executed separately through recursive planning. A function call that takes a column (Var) as a parameter will not be evaluated on the coordinator, since it depends on data on the worker nodes and will need to be evaluated many times. However, if we did this on a replicated table then stable/volatile functions may return different results on different nodes, in the context of an update/delete it would cause replicas diverge. That is one of the reasons why we disallow stable/volatile functions in update/delete statements, but we could permit them for regular tables with a single replica. The reason we also disallow volatile functions in regular update/delete is purely implementation related: Our current function evaluation logic does not know how to distinguish between stable & volatile functions. If we were to run it on a query that contains WHERE x > random(), it would evaluate the random() once, even though it’s supposed to be pushed down and re-evaluated for every row. ## Prepared statements Prepared statements is a feature that lets clients send a query once and then execute it multiple times. Plans may be cached across execution. Prepared statements can be created explicitly via PREPARE/EXECUTE commands, via protocol messages (what most clients do), via PL/pgSQL, and via SPI. Citus has limited prepared statement support in the sense that they functionally work, but there are only a few cases in which plans are meaningfully cached across executions. Despite the lack of meaningful optimization, prepared statements involve a lot of complexity and counterintuitive logic. Which parts are necessary and which parts are technical debt is left as an exercise to the reader. The plan of a prepared statement is only cached when the same prepared statement is executed 5 times by PostgreSQL (hard-coded value). The 5th time, the planner is called without supplying parameter values to obtain a “generic plan” and that plan is cached unless it is much costlier than using custom plan. Hence, the planner might be called twice on the 5th execution and if a generic plan is created then the planner may not be called again. There are a few important cases to distinguish in case of Citus: - Multi-shard queries vs. single shard (Fast path & router) - Custom plan vs. Generic plan. - Parameter in a filter on the distribution column vs. only on other columns - Local vs. remote execution - Combinations of parameters & function evaluation. Let’s start with the simplest case: Multi-shard queries. These queries have complex planning logic, and it would be even more complex if the planner did not know the values of parameters. Therefore, we dissuade PostgreSQL from using a generic plan by returning a mock PlannedStmt with an extremely high cost when asked for a generic plan (see `DissuadePlannerFromUsingPlan()`). That will cause PostgreSQL to keep using a custom plan with known parameter values. In addition, we replace any Params that appear in the query tree with their Const values in ResolveExternalParams before distributed planning, so the remaining planner logic does not need to concern itself with query parameters. For single shard queries, the story is a lot more complex. An important question is whether there is a parameter in the distribution column, and whether a query is single shard in the planner or not. A query like `SELECT * FROM table1 WHERE distcol = $1` will clearly go to a single shard, but for a query like `SELECT * FROM table1 WHERE distcol = $1 UNION ALL SELECT * FROM table2 WHERE distcol = $2` it may or may not be. We do not precisely distinguish all possible cases, but rather have a simple distinction: - Fast path queries are simple queries on a single table with a "distribution column" = "Param or Const" filter (or single row inserts). We know that they prune to at most 1 shard regardless of the parameter value. The case of “distcol = NULL” is false/null by definition (unlike “distcol IS NULL”) and therefore prunes to 0 shards. - Router queries are arbitrarily complex queries that prune down to a single shard at planning time based on the RestrictInfo data structures obtained from postgres planner. We can only decide whether a query is a router query in the planner, because if it is not a router query, we need to fall back to the multi-shard query planning code path. Hence, we can only support generic router plans when all distribution column filters are constant, or there are only single shard/reference tables in the query. The router planner cannot prune based on unbound parameters and will therefore return a soft error. When the planner sees a soft error, we return a mock plan with a high cost, similar to multi-shard queries. Fast path queries prune to a single shard regardless of the parameter values. If the distribution column value is a parameter, we defer additional planning decisions, in particular “shard pruning” to the executor (deferredPruning flag in the Job). Currently, we resolve the parameters in `ExecuteCoordinatorEvaluableExpressions()` which replaces the Param nodes in the query tree, and then `TargetShardIntervalForFastPathQuery()` finds "distribution column" = "Const" filters in the WHERE clause. This could perhaps be optimized but keeps the logic consistent between parameters and non-parameterized queries. For both fast path queries and router queries, the job query tree for single shard queries still has all the parameters when we get to the executor. We resolve the parameters in the query tree before deparsing when: - pruning is deferred (has WHERE distcol = $1 …) - the query is a DML that contains function calls that need to resolved The latter happens primarily because function evaluation also resolves parameters. Otherwise, it would not be able to resolve expressions like `stable_fn($1)`. If the parameters are not resolved in the executor, they are passed on to the worker node using the libpq functions that take parameters. Both fast path and router query plans can be stored in the PostgreSQL cache (plancache.c) if they are run at least five times. The way these plans are handled depends on whether or not the query includes a parameter on the distribution key. In the first case below, there is no parameter; in the second case, there is a parameter: - the query pruned to a single shard in the planner, the task is therefore static (always goes to the same shard group, with same query string) - the query uses deferred pruning, the shard group is therefore decided in the executor (not cached, query string rebuilt) Both scenarios reduce compute cycles in terms of planning the distributed query, but the plan for the shard query is never cached, except in the local execution case, which is described in the next section. The current structure is “less than ideal”, but by now it is battle hardened and has extensive regression tests that cover all the cases. It should be improved, but with caution. Caching the wrong plan could easily lead to invalid results, and there are many subtle edge cases. ### Local plan caching We currently only take advantage of plan caching for shard queries that access a single local shard group and use deferred pruning (described in the previous section). This avoids reparsing or replanning the query on the local shard. That works well in combination with smart clients that immediately connect to the right node, function call delegation, triggers, and Citus local tables. We can only know whether we are dealing with a local shard group after evaluating parameters and functions. Immediately after that, we plan the query on the local shard group and store the resulting (regular PG) plan in the distributed plan data structure (Job). The reason we store it in the distributed plan is that it is already cached by PostgreSQL, so anything we add to the plan will be cached along with it, with the correct lifecycle. We store a list of local plans, one for each shard plan. Local plan caching quite significantly improves performance for certain workloads, but it comes with a subtle caveat. For queries with deferred pruning, we only know whether the shard query is on a local shard query after evaluating parameters and function calls, which we do by replacing them in the query tree. However, to obtain a cacheable generic plan, we need to use the original query tree which still has the original function calls and parameters. That means re-execute those function calls when executing the shard query, which is unusual since we usually only execute them in the BeginCustomScan hook. Since we only do this for local execution, the function calls will still run in the same process and will therefore have the same effect, but it means we sometimes evaluate function calls twice. That is acceptable for stable functions, but not for volatile functions. We therefore skip caching when there are calls to volatile functions. ## Adaptive executor Once function and parameter evaluation are completed and the final task list is ready, we call into the adaptive executor. The goal of the adaptive executor is to efficiently execute a list of tasks. A task is typically a shard query that is to be executed on 1 placement (read) or all placements (write). It can also be an arbitrary command unrelated to shards. Implementation-wise, its primary function is to concurrently execute multiple queries on multiple remote nodes using libpq in non-blocking mode with appropriate failure handling and adaptive connection pools. The adaptive executor tries to minimize network round trips for single shard statements (transactional workloads) by using a single, cached connection per node, and parallelize queries using multiple connections per node for multi-shard statements (analytical workloads, ETL, DDL). Historically, Citus executed single shard queries via a single connection per worker node (router executor), while it executed multi-shard queries via a connection per shard to parallelize across nodes and cores (real-time executor), but this approach had several limitations. **The executor must consider preceding writes and locks on shards in the transaction**. In the past, if the router executor performed 2 inserts on different shards over the same connection, then the real-time executor could no longer run. It is not valid to query those shards over two separate connections, since only one of them would see the inserts. The executor must ensure that after a write or lock on a shard group, all subsequent queries on the shard group use the same connection until transaction end. **The executor should consider fast vs. slow multi-shard commands**. We observed many cases in which multi-shard commands only took a few milliseconds (e.g. index lookups on a non-distribution column) and opening a connection per shard was excessive, since it could add tens or hundreds of milliseconds to a query that could otherwise finish in 10-20ms. _Whether parallelization is beneficial depends on the runtime of individual tasks._ Some tasks can also take much longer than others. **The executor should gracefully handle failures**. One of the more challenging parts of doing remote, concurrent query execution is handling a variety of failures, including timeouts, failed connections, and query errors. The handling can be different for reads and writes, since reads on replicated tables can fail over to a different placement. **The executor should consider replicated shards**. Writes to reference tables (or replicated shards) need to be sent to all nodes, while reads can fail over to other replicas. Update/delete can be performed in parallel due to the exclusive lock on the shard, while inserts need to run in a consistent order to avoid deadlocks in case of constraint violations. The executor also needs to consider that replicas may be on the local node and use local execution. To fulfill the first two requirements, the adaptive executor uses a (process-local) **pool of connections per node**, which typically starts at 1 connection, but can grow based on the runtime of the query. Queries on shard groups that were already modified are assigned to the connection that performed the modification(s), while remaining queries are assigned to the pool (to be parallelized at will). Adaptive executor pools **Both the pool and the session have a “pending queue” and a “ready queue”**. The pending queue is mainly used in case of replication (e.g. reference tables). In the case of reads, whether a (pending) task on placement 2 needs to run depends on whether the (ready) task on placement 1 succeeds. In case of inserts, we write to the placements in order, so the task on placement 2 runs only once placement 1 is done. **The main loop of the adaptive executor waits for IO on the overall list of connections using a WaitEventSet**. When a connection has IO events, it triggers the connection state machine logic (ConnectionStateMachine). When the connection is ready, it enters the transaction state machine logic (TransactionStateMachine) which is responsible for sending queries and processing their results. The executor is designed with state machines, and the code has an extensive comment describing the state machines, please refer there for the details When a connection is ready, we first send BEGIN if needed, and then take tasks from the session-level ready queue, and then tasks from the pool-level ready queue. We currently process one task at a time per connection. There are opportunities for optimization like pipelining/batching, though we need to be careful not to break parallelism. **Late binding of tasks to connections via the pool-level queue has nice emergent properties**. If there is a task list with one particularly slow task, then one connection will spend most of its time on that task, while other connections complete the shorter tasks. We can also easily increase the number of connections at runtime, which we do via a process called slow start (described below). Finally, we’re not dependent on a connection being successfully established. We can finish the query when some connections fail, and we finish the query if BEGIN never terminates on some connection, which might happen if we were connecting via outbound pgbouncers. **The pool expands via “slow start”, which grows the pool every ~10ms as long as tasks remain in the pool-level queue**. The name slow start is derived from the process in TCP which expands the window size (the amount of data TCP sends at once). As in the case of TCP, the name slow is a misnomer. While it starts very conservatively, namely with 1 connection, the _rate_ at which new connections open increases by 1 every 10ms, starting at 1. That means after 50ms, the executor is allowed to open 6 additional connections. In a very typical scenario of 16 shards per node, the executor would reach maximum parallelism after ~60ms. It will open at most as many additional connections as there are tasks in the ready queue. Adaptive executor slow start example The 10ms was chosen to be higher than a typical connection-establishment time, but low enough to quickly expand the pool when the runtime of the tasks is long enough to benefit from parallelism. The 10ms has mostly proven effective, but we have seen cases in which slow connection establishment due to Azure network latencies would justify a higher value. In addition, we found that workloads with many queries in the 20-60ms range would see a relatively high number of redundant connection attempts. To reduce that, we introduced “cost-based connection establishment”, which factors in the average task execution time compared to the average connection establishment time and thereby significantly reduced the number of redundant connections. **The citus.max_adaptive_executor_pool_size setting can be used to limit the per-process pool sizes**. The default behaviour of the adaptive executor is optimized for parallel query performance. In practice, we find that there is another factor than runtime that users care about: memory. The memory usage of a query that uses 16 connections can be 16 times higher than the memory usage of a query that uses 1 connection. For that reason, users often prefer to limit the pool size to a lower number (e.g. 4) using citus.max_adaptive_executor_pool_size. **The citus.max_shared_pool_size setting can be used to limit the pool sizes globally**. It’s important to reiterate that the adaptive executor operates in the context of a single process. Each coordinating process has its own pools of connections to other nodes. This would lead to issues if e.g. the client makes 200 connections which each make 4 connections per node (800 total) concurrently while max_connections is 500. Therefore, there is a global limit on the number of connections configured by max_shared_pool_size. The citus.max_shared_pool_size is implemented in the connection management layer rather than the executor. Refer to the connection management section for details. **The comment on top of [adaptive_executor.c](executor/adaptive_executor.c) has a detailed description of the underlying data structures.** While these data structures are complex and this might look like an area technical debt, the current data structures and algorithm have proven to be a relatively elegant and robust way to meet all the different requirements. It is worth noting that a significant part of the complexity comes from dealing with replication, and shard replication is mostly a deprecated feature, but keep in mind that reference tables are also replicated tables and most of the same logic applies. ## Local execution When the adaptive executor completes all of its remote tasks, the final step is to perform local execution. We formally see this as part of the adaptive executor, though the code is largely separate (in local_executor.c). Local execution is essentially just executing the shard queries on local shards directly by invoking the planner & executor. In other words, there is no additional backends or connections are established for local execution. Some queries strictly require local execution. In particular, queries that depend on objects (types, tables) created by the current transaction, or joins between regular tables and Citus local or reference tables. In case of a multi-shard query, a downside of local execution is that there is no parallelism across shards. Therefore, the executor tries to avoid local execution for simple multi-shard queries outside of a transaction block. Instead, it will open multiple connections to localhost to run queries in parallel. In a multi-statement transaction, the executor always prefers local execution even for multi-shard queries, since the tranasaction might also perform operations that require local execution. Some queries cannot use local execution. For instance, we cannot use CREATE INDEX CONCURRENTLY as part of a bigger transaction, and we have not implemented a local version of EXPLAIN ANALYZE. We also cannot perform replication commands like creating a subscription via local execution. For the most part, these commands are typically executed outside of a transaction block or as internal commands, so it does not significantly affect the user experience. The executor always does the local execution after remote execution. That way, if there are any problems with the remote execution, Citus can still switch back (e.g., failover) to local execution. ## Subplans Execution of subplans (CTEs, subqueries that cannot be pushed down) is relatively straight-forward. The distributed plan has a list of subplans, which can be regular or distributed, and they are passed to the PostgreSQL executor sequentially. The result of each subplan is broadcast to all participating nodes via the COPY .. WITH (format ‘result’) command, which writes to an intermediate result. The intermediate results are read in subsequent shard queries via the read_intermediate_result function. A current downside of the read_intermediate_result function is that it first copies all the tuples into a tuple store, which may be flushed to disk. This could be fixed through a CustomScan, or in PostgreSQL itself. ## Re-partitioning Re-partitioning happens when joining distributed tables on columns other than the distribution column, or when the tables are not co-located. In the distributed plan, a re-partitioning operation is generally expressed through a Job which has dependent Jobs. The dependent Jobs are a special type of subplan whose results are re-partitioned. Two stages are executed to resolve the dependent jobs: - Run a query on all shards using the worker_partition_query_result function, which writes the result of the query to a set of intermediate results based on a partition column and set of hash ranges - Fetch the intermediate results to the target node using fetch_intermediate_result, for each source shard group & target hash range pair. These stages are run in parallel for all dependent jobs (read: all tables in a join) by building a combined task list and passing it to the adaptive executor. This logic primarily lives in ExecuteTasksInDependencyOrder. Once all dependent jobs are finished, the main Job is executed via the regular adaptive executor code path. The main job will include calls to read_intermediate_result that concatenate all the intermediate results for a particular hash range. Single hash re-partition join example Dependent jobs have similarities with subplans. A Job can only be a distributed query without a merge step, which is what allows the results to be repartitioned, while a subplan can be any type of plan, and is always broadcast. One could imagine a subplan also being repartitioned if it is subsequently used in a join with a distributed table. The difference between subplans and jobs in the distributed query plans is one of the most significant technical debts. ## COPY .. FROM command The COPY .. FROM command is used to load a CSV (or TSV, or binary copy format) file or stream from the client into a table. The \copy command is a psql command that can load files from the client, and internally does COPY .. FROM STDIN and sends the file contents over the socket. **Citus supports COPY into distributed tables via the ProcessUtility_hook by internally doing a COPY to each shard.** We go through the regular COPY parsing logic in PostgreSQL (BeginCopyFrom & NextCopyFrom), which reads from the socket, parses the CSV, and returns a tuple. The tuple is passed through the CitusCopyDestReceiver API. Most of the relevant logic lives in CitusSendTupleToPlacements. **The CitusCopyDestReceiver inspects the value in the distribution column and finds the corresponding shard.** It opens a connection to the node(s) on which the shard is placed, starts a COPY into the shard, and forwards the tuple. For performance reasons, we use the binary copy format over the internal connections, when possible (e.g. all types have send/receive), even if the client used CSV. **The COPY protocol does not require immediate confirmation when sending a tuple**, which means we can continue parsing the next tuple without waiting for the previous tuple to be fully processed. This creates nice pipelining behaviour where tuples are effectively ingested in parallel and can improve performance over regular PostgreSQL, even though parsing runs at the same speed. This effect will be more pronounced when insertions are relatively heavy-weight due to triggers or heavy indexes. **COPY does not always use a connection per shard.** If there were already writes to multiple shards on a specific connection earlier in the transaction (e.g. consecutive inserts), then that connection must be used for the writes done by the COPY (e.g. to be able to check unique constraints). However, we can only COPY into one table at a time. In this case, the COPY logic maps multiple shards to the same connection and switches back-and-forth between shards through multiple COPY commands (which has overhead). If we get a tuple for a shard that is currently active, we forward immediately over the connection. Otherwise, we add the tuple to a per-shard buffer, or switch the connection if we already sent `citus.copy_switchover_threshold` bytes to the current shard. There is a caveat in the current COPY logic. Citus always uses non-blocking I/O, which means libpq keeps outgoing bytes in a buffer when the socket is busy. We only run the relevant libpq functions to flush the per-connection libpq buffer (or flush the per-shard buffer) when there is a tuple for a particular connection, or when reaching the end of the stream. Ideally, we keep getting tuples for all the different shards, such that all connections are flushed. In some cases, when many consecutive tuples are for the same shard, a large amount of data can remain buffered on the coordinator in libpq or the per-shard buffer when we come to the end of the stream (CitusCopyDestReceiverShutdown). The connections and per-shard buffers are then flushed one by one. There is room for optimization where the COPY loop and final flush behave more like the adaptive executor and uses a WaitEventSet to check for I/O events on all the sockets, and flush the libpq buffer. Another (smaller) caveat is that the libpq buffer can fill up if the outgoing connection to the worker cannot keep up with the rate at which the coordinator is receiving and parsing tuples. To bound the size of the buffer and thereby avoid running out of memory, we force a flush on a connection after every `citus.remote_copy_flush_threshold` bytes that are written to a connection. We do this regardless of whether the libpq buffer is becoming large, because we do not have direct insight into its current size. Fortunately, it will only cause a very short pause if the buffer is not large or empty. For local shards, COPY can also use local execution. We use local execution by default in transaction blocks, but try to use connections to the local node for a single statement COPY because we get more parallelism benefits. ## COPY .. TO command The COPY .. TO command is used to dump the data from a table, or to get the output of a query in CSV format. The COPY (SELECT ..) TO syntax does not use any special logic. PostgreSQL’s implementation will plan and execute the query in the usual way, and Citus will intercept queries on distributed tables. That means these commands do not use COPY internally to query the shards. Instead, the results of the query are buffered in a tuple store and then converted to CSV. The COPY distributed_table TO .. syntax will typically return a lot of data and buffering it all in a tuplestore would cause issues. Therefore, Citus uses the process utility hook to propagate the COPY distributed_table TO .. command to each shard one by one. The output is forwarded directly to the client. If the user asked for a header, it is only requested from the first shard to avoid repeating it for each shard. ## INSERT..SELECT The INSERT.. SELECT command inserts the result of a SELECT query into a target table. In real-time analytics use cases, INSERT..SELECT enables transformation of an incoming stream of data inside the database. A typical example is maintaining a rollup table or converting raw data into a more structured form and adding indexes. INSERT..SELECT modes Citus has three different methods of handling INSERT..SELECT commands that insert into a distributed table as shown in the figure above. We identify these methods as: (1) co-located, where shards for the source and destination tables are co-located; (2) repartitioning, where the source and destination tables are not co-located and the operation requires a distributed reshuffle; and (3) pull to coordinator, where neither of the previous two methods can be applied. These three approaches can process around 100M, 10M, and 1M rows per second, respectively, in a single command. Co-located INSERT..SELECT is executed in a similar fashion to multi-shard update/delete commands. There is a single list of tasks with one task for each shard group, which runs via the adaptive executor. INSERT..SELECT with re-partitioning is architecturally similar to re-partition joins, but it goes via separate code path and uses more optimizations. Empty files are skipped and files traveling between the same pair of nodes are batched in a single call to fetch_intermediate_results, which saves round trips. The final step in INSERT..SELECT with re-partitioning runs queries like INSERT INTO dist_table SELECT .. FROM read_intermediate_result(…) with optional ON CONFLICT and RETURNING clauses. In principle, we could do an additional GROUP BY in the final step when grouping by the target distribution column, but that is not currently implemented and instead falls back to pull to coordinator. INSERT..SELECT via the coordinator logic uses the COPY code path to write results of an arbitrary SELECT into multiple shards at the same time. In case of ON CONFLICT or RETURNING, they are first written to intermediate results that are co-located with the destination shards. Then a co-located INSERT..SELECT between the intermediate results and the target shards is performed, similar to the final step of re-partitioning. ## Merge command Merge command the same principles as INSERT .. SELECT processing. However, due to the nature of distributed systems, there are few more additional limitations on top of the INSERT .. SELECT processing. The [MERGE blog post](https://www.citusdata.com/blog/2023/07/27/how-citus-12-supports-postgres-merge/) dives deep on this topic. # DDL DDL commands are primarily handled via the citus_ProcessUtility hook, which gets the parse tree of the DDL command. For supported DDL commands, we always follow the same sequence of steps: 1. Qualify the table names in the parse tree (simplifies deparsing, avoids sensitivity to search_path changes) 2. Pre-process logic 3. Call original previous ProcessUtility to execute the command on the local shell table 4. Post-process logic 5. Execute command on all other nodes 6. Execute command on shards (in case of table DDL) Either the pre-process or post-process step generates a "Distributed DDL Job", which contains a task list to run in steps 4 & 5 (via adaptive executor). In general pre-process should: - Acquire any locks that are needed beyond the ones PostgreSQL will acquire in step 3 - Perform upfront error checks (e.g. is this unique constrained allowed on a distributed table?) Post-process should: - Ensure dependencies of the current object exist on worker nodes (e.g. types used in parameters when creating a function) - Deparse the DDL parse tree to a string - Generate a task list using the deparsed DDL command The reason for handling dependencies and deparsing in post-process step is that in case of a CREATE/ALTER, the object exist in its intended form at that point. In case of a DROP, the opposite is true and the pre-process should be used. Most commands have either a pre-process or post-process function. We have not been particularly careful about defining what should be done in pre-process vs. post-process, so the steps are not always the same across different commands. Not all table DDL is currently deparsed. In that case, the original command sent by the client is used. That is a shortcoming in our DDL logic that causes user-facing issues and should be addressed. We do not directly construct a separate DDL command for each shard. Instead, we call the `worker_apply_shard_ddl_command(shardid bigint, ddl_command text)` function which parses the DDL command, replaces the table names with shard names in the parse tree according to the shard ID, and then executes the command. That also has some shortcomings, because we cannot support more complex DDL commands in this manner (e.g. adding multiple foreign keys). Ideally, all DDL would be deparsed, and for table DDL the deparsed query string would have shard names, similar to regular queries. ## Defining a new DDL command All commands that are propagated by Citus should be defined in DistributeObjectOps struct. Below is a sample DistributeObjectOps for ALTER DATABASE command that is defined in [distribute_object_ops.c](commands/distribute_object_ops.c) file. ```c static DistributeObjectOps Database_Alter = { .deparse = DeparseAlterDatabaseStmt, .qualify = NULL, .preprocess = PreprocessAlterDatabaseStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; ``` Each field in the struct is documented in the comments within the `DistributeObjectOps`. When defining a new DDL command, follow these guidelines: - **Returning tasks for `preprocess` and `postprocess`**: Ensure that either `preprocess` or `postprocess` returns a list of "DDLJob"s. If both functions return non-empty lists, then you would get an assertion failure. - **Generic `preprocess` and `postprocess` methods**: The generic methods, `PreprocessAlterDistributedObjectStmt` and `PostprocessAlterDistributedObjectStmt`, serve as generic pre and post methods utilized for various statements. Both of these methods find application in distributed object operations. - The `PreprocessAlterDistributedObjectStmt` method carries out the following operations: - Performs a qualification operation. - Deparses the statement and generates a task list. - As for the `PostprocessAlterDistributedObjectStmt` method, it: - Invokes the `EnsureAllObjectDependenciesExistOnAllNodes` function to propagate missing dependencies, both on the coordinator and the worker. - Before defining new `preprocess` or `postprocess` methods, it is advisable to assess whether the generic methods can be employed in your specific case. - **`deparse`**: When propagating the command to worker nodes, make sure to define `deparse`. This is necessary because it generates a query string for each worker node. - **`markDistributed`**: Set this flag to true if you want to add a record to the `pg_dist_object` table. This is particularly important for `CREATE` statements when introducing a new object to the system. - **`address`**: If `markDistributed` is set to true, you must define the `address`. Failure to do so will result in a runtime error. The `address` is required to identify the fields that will be stored in the `pg_dist_object` table. - **`markDistributed` usage in `DROP` Statements**: Please note that `markDistributed` does not apply to `DROP` statements. For `DROP` statements, instead you need to call `UnmarkObjectDistributed()` for the object either in `preprocess` or `postprocess`. Otherwise, state records in ``pg_dist_object`` table will cause errors in UDF calls such as ``citus_add_node()``, which will try to copy the non-existent db object. - **`qualify`**: The `qualify` function is used to qualify the objects based on their schemas in the parse tree. It is employed to prevent sensitivity to changes in the `search_path` on worker nodes. Note that it is not mandatory to define this function for all DDL commands. It is only required for commands that involve objects that are bound to schemas, such as; tables, types, functions and so on. After defining the `DistributeObjectOps` structure, this structure should be implemented in the `GetDistributeObjectOps()` function as shown below: ```c // Example implementation in C code const DistributeObjectOps * GetDistributeObjectOps(Node *node) { switch (nodeTag(node)) { case T_AlterDatabaseStmt: { return &Database_Alter; } ... ``` Finally, when adding support for propagation of a new DDL command, you also need to make sure that: * Use `quote_identifier()` or `quote_literal_cstr()` for the fields that might need escaping some characters or bare quotes when deparsing a DDL command. * The code is tolerant to nullable fields within given `Stmt *` object, i.e., the ones that Postgres allows not specifying at all. * You register the object into `pg_dist_object` if it's a CREATE command and you delete the object from `pg_dist_object` if it's a DROP command. * Node activation (e.g., `citus_add_node()`) properly propagates the object and its dependencies to new nodes. * Add tests cases for all the scenarios noted above. * Add test cases for different options that can be specified for the settings. For example, `CREATE DATABASE .. IS_TEMPLATE = TRUE` and `CREATE DATABASE .. IS_TEMPLATE = FALSE` should be tested separately. ## Object & dependency propagation These two topics are closely related, so we'll discuss them together. You can start the topic by reading [Nils' blog](https://www.citusdata.com/blog/2020/06/25/using-custom-types-with-citus-and-postgres/) on the topic. ### The concept of "Dependency" for Postgres/Citus Starting with the basics, Postgres already understands object dependencies. For instance, it won't allow you to execute `DROP SCHEMA` without the `CASCADE` option if tables exist within the schema. In this case, the table is a `dependent object`, and the schema serves as the `referenced object`. ```sql CREATE SCHEMA sc1; CREATE TABLE sc1.test(a int); DROP SCHEMA sc1; ERROR: cannot drop schema sc1 because other objects depend on it DETAIL: table sc1.test depends on schema sc1 HINT: Use DROP ... CASCADE to drop the dependent objects too. ``` The information about these dependencies is stored in a specific Postgres catalog table, known as [`pg_depend`](https://www.postgresql.org/docs/current/catalog-pg-depend.html). You can inspect the aforementioned dependency within this catalog using the following query: ```sql SELECT pg_identify_object_as_address(classid, objid, objsubid) as dependent_object, pg_identify_object_as_address(refclassid, refobjid, refobjsubid) as referenced_object FROM pg_depend WHERE (classid, objid, objsubid) IN (SELECT classid, objid, objsubid FROM pg_get_object_address('table', '{sc1,test}', '{}')); ┌─────────────────────────┬───────────────────┐ │ dependent_object │ referenced_object │ ├─────────────────────────┼───────────────────┤ │ (table,"{sc1,test}",{}) │ (schema,{sc1},{}) │ └─────────────────────────┴───────────────────┘ (1 row) ``` ### Citus' Approach to Object Creation and Dependency Tracking: `pg_dist_object` Citus employs its own catalog table called `pg_dist_object`. This table keeps records of all objects that need to be created on every node in the cluster. These objects are commonly referred to as `Distributed Objects`. When adding a new node to the cluster using `citus_add_node()`, Citus must ensure the creation of all dependent objects even before moving data to the new node. For instance, if a table relies on a custom type or an extension, these objects need to be created before any table is set up. In short, Citus is responsible for setting up all the dependent objects related to the tables. Similarly, when creating a new Citus table, Citus must confirm that all dependent objects, such as custom types, already exist before the shell table or shards are set up on the worker nodes. Note that this applies not just to tables; all distributed objects follow the same pattern. Here is a brief overview of `pg_dist_object`, which has a similar structure to `pg_depend` in terms of overlapping columns like `classid, objid, objsubid`: ```sql CREATE SCHEMA sc1; CREATE TABLE sc1.test(a int); SELECT create_distributed_table('sc1.test', 'a'); SELECT pg_identify_object_as_address(classid, objid, objsubid) as distributed_object FROM pg_dist_object; ┌─────────────────────────────┐ │ distributed_object │ ├─────────────────────────────┤ │ (role,{onderkalaci},{}) │ │ (database,{onderkalaci},{}) │ │ (schema,{sc1},{}) │ │ (table,"{sc1,test}",{}) │ └─────────────────────────────┘ (4 rows) ``` ### When Is `pg_dist_object` Populated? Generally, the process is straightforward: When a new object is created, Citus adds a record to `pg_dist_object`. The C functions responsible for this are `MarkObjectDistributed()` and `MarkObjectDistributedViaSuperuser()`. We'll discuss the difference between them in the next section. Citus employs a universal strategy for dealing with objects. Every object creation, alteration, or deletion event (like custom types, tables, or extensions) is represented by the C struct `DistributeObjectOps`. You can find a list of all supported object types in [`distribute_object_ops.c`](https://github.com/citusdata/citus/blob/2c190d068918d1c457894adf97f550e5b3739184/src/backend/distributed/commands/distribute_object_ops.c#L4). As of Citus 12.1, most Postgres objects are supported, although there are a few exceptions. Whenever `DistributeObjectOps->markDistributed` is set to true—usually during `CREATE` operations—Citus calls `MarkObjectDistributed()`. Citus also labels the same objects as distributed across all nodes via the `citus_internal.add_object_metadata()` UDF. Here's a simple example: ```sql -- Citus automatically creates the object on all nodes CREATE TYPE type_test AS (a int, b int); ... NOTICE: issuing SELECT worker_create_or_replace_object('CREATE TYPE public.type_test AS (a integer, b integer);'); .... WITH distributed_object_data(typetext, objnames, objargs, distargumentindex, colocationid, force_delegation) AS (VALUES ('schema', ARRAY['public']::text[], ARRAY[]::text[], -1, 0, false)) SELECT citus_internal.add_object_metadata(typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) FROM distributed_object_data; ... -- Then, check pg_dist_object. This should be consistent across all nodes. SELECT pg_identify_object_as_address(classid, objid, objsubid) as distributed_object FROM pg_dist_object WHERE (classid, objid, objsubid) IN (SELECT classid, objid, objsubid FROM pg_get_object_address('type', '{type_test}', '{}')); ┌──────────────────────────────┐ │ distributed_object │ ├──────────────────────────────┤ │ (type,{public.type_test},{}) │ └──────────────────────────────┘ (1 row) ``` In rare cases, `pg_dist_object` is updated during Citus version upgrades. If you upgrade Citus from `version X` to `version Y`, and a certain object becomes a supported distributed object in `version Y` but wasn't in `version X`, Citus marks it as such during the `ALTER EXTENSION citus` command. The details can be found in the C function `PostprocessAlterExtensionCitusUpdateStmt()`. ### How Are `pg_dist_object` and `pg_depend` Related? In the prior sections, we focused on standalone objects, but in reality, most objects have dependencies. That's where `pg_depend` and also `pg_shdepend` become crucial. Any mention of `pg_depend` in this section also applies to `pg_shdepend`. When Citus creates an object, it scans the `pg_depend` table to identify all dependencies and then automatically generates these dependencies as distributed objects. The core C function in this process is `EnsureDependenciesExistOnAllNodes()`. This function takes the object as an argument and conducts a depth-first search (DFS) on `pg_depend` and `pg_shdepend` tables. The DFS sequence is crucial because dependencies must be established in a specific order. For instance, if a table depends on a type `t1`, and `t1` relies on another type `t2`, then `t2` must be created before `t1`. The DFS ensures that this order is maintained. If Citus encounters a dependency it can't support, it will throw an error instead of silently allowing it. The rationale behind this approach is to avoid subtle issues later, especially when adding new nodes. For instance, Citus currently throws an error for circular dependencies. The main function performing these checks is `EnsureDependenciesCanBeDistributed()`. During the DFS, Citus might need to extend its search, especially for complex dependencies like array types that don't have a straightforward dependency on their element types. Citus expands the traversal to account for such cases. The main function responsible for this is `ExpandCitusSupportedTypes()`, which has extensive comments explaining the specific rules. ### Current User vs `superuser` for Object Propagation: The difference between `MarkObjectDistributed()` and `MarkObjectDistributedViaSuperuser()` is important here. Generally, Citus tries to minimize the use of `superuser` operations for security reasons. However, there are cases where it's necessary. We employ `superuser` permissions primarily when marking the dependencies of an object we are working on. This is because creating dependencies might require higher-level privileges that the current user might not have. For example, if a schema depends on a role, and the current user doesn't have the privilege to create roles, an error will be thrown. To avoid this, we use `superuser` for creating dependencies. However, there's an exception. If the dependency is created within the same transaction, we use the current user. This prevents visibility issues and is mainly relevant for `serial` columns. More details can be found in [Citus GitHub PR 7028](https://github.com/citusdata/citus/pull/7028). ### When Are the Objects in `pg_dist_object` Used? There are three main scenarios: + When adding a new node—or more precisely, activating it—Citus reads all objects listed in `pg_dist_object`, sorts them by dependency, and then creates those objects on the new node. The core C function for this is `SendDependencyCreationCommands()`, and the sorting is done by `OrderObjectAddressListInDependencyOrder()`. + When Citus creates a new object and processes its dependencies, any dependencies already marked as distributed are skipped. This is handled in `FollowNewSupportedDependencies()`, where dependencies are bypassed if `IsAnyObjectDistributed()` returns true. + When user modifies an object, Citus acts only when the object is distributed. For non-distributed object, Citus gives the control back to Postgres. ## Foreign keys Citus relies fully on Postgres to enforce foreign keys. To provide that, Citus requires the relevant shards to be colocated. That’s also why the foreign keys between distributed tables should always include the distribution key. When reference tables / citus local tables involved, Citus can relax some of the restrictions. [Onder’s talk Demystifying Postgres Foreign Key Constraints on Citus](https://www.youtube.com/watch?v=xReWGcSg7sc) at CitusCon discusses all the supported foreign key combinations within Citus. There is one tricky behavior regarding transactions when there is a foreign key from a distributed table to a reference table. If a statement in a transaction modifies the reference table, then Postgres acquires row locks on the referencing tables (e.g., shards of the distributed table) within the internal connection that modified the reference table. After that point, Citus cannot access the shards of the distributed table in parallel anymore. Otherwise, the multiple internal connections that would be opened via parallel command might compete to acquire the same locks, leading to a (self) distributed deadlock. To prevent these scenarios, Citus switches to sequential execution. The relevant function is `RecordParallelRelationAccessForTaskList()`, which documents the possible scenarios. The regression test file [foreign_key_restriction_enforcement](https://github.com/citusdata/citus/blob/2c190d068918d1c457894adf97f550e5b3739184/src/test/regress/sql/foreign_key_restriction_enforcement.sql) has lots of nice examples of this behavior. ## DROP TABLE Citus' handling of `DROP TABLE` is slightly different than other DDL operations. In this section, we aim to highlight the key differences and their reasoning. Citus implements an event trigger, [`citus_drop_trigger()`](https://github.com/citusdata/citus/blob/main/src/backend/distributed/sql/udfs/citus_drop_trigger/latest.sql). The trigger is defined as: ```sql select * from pg_event_trigger ; ┌───────┬────────────────────────────┬──────────┬──────────┬─────────┬────────────┬─────────┐ │ oid │ evtname │ evtevent │ evtowner │ evtfoid │ evtenabled │ evttags │ ├───────┼────────────────────────────┼──────────┼──────────┼─────────┼────────────┼─────────┤ │ 16676 │ citus_cascade_to_partition │ sql_drop │ 10 │ 16675 │ O │ │ └───────┴────────────────────────────┴──────────┴──────────┴─────────┴────────────┴─────────┘ (1 row) ``` The drop trigger proves useful for capturing all tables affected by the `CASCADE` operation. For instance, if you delete a parent table, Postgres will automatically execute a `DROP TABLE` command for its partitions. Citus can then seamlessly apply the same operation to all cascaded tables, eliminating the need for manual identification of tables that would be affected by the cascade. Another reason for utilizing a trigger for `DROP` processing is that after executing `standardProcess_Utility`, the `oid` of the table being dropped is eliminated from Postgres' catalog tables. This makes it more challenging to manage a dropped table in `PostProcessUtility`, as is customary for many `DDL` commands. Instead, we depend on the event trigger to supply the `oid` of the table that has been dropped. This allows us to delete all related metadata, such as entries in `pg_dist_partition` or `pg_dist_shard` from Citus' catalog tables. Additionally, we eliminate all relevant metadata from every node in the cluster. Ultimately, this enables us to remove the shard placements linked to the dropped Citus table. Also, if we were to rely on `standardProcess_Utility`, we'd need to handle all sorts of `DROP` commands that could cascade into `DROP TABLE`. With drop trigger, Postgres handles that and calls Citus' drop trigger. Generally speaking, there isn't a compelling reason to avoid using `PostProcessUtility` for managing `DROP TABLE` commands. Theoretically, one could implement all the same logic within `PostProcessUtility`. However, the drop trigger offers a convenient approach. Additional functionalities are present in [`PreprocessDropTableStmt()`](https://github.com/citusdata/citus/blob/c323f49e8378b5e8ce95457c845659b5fc14ccb1/src/backend/distributed/commands/table.c#L146), particularly concerning the handling of partitioned tables and colocation locking. These aspects are well-documented in the code, so for further details, please refer to the documentation there. # Connection management Each client session makes “internal” connections to other nodes in the cluster. Connection management is an important part of our overall execution logic. The design largely comes from the need to achieve a few different goals: - Cache and reuse connections for low latency. - Parallelize commands over multiple connections per node, to use multiple cores. - Multi-statement transactions have locks and uncommitted state that is only visible over a particular connection. We therefore need to make sure that: - After a write to a shard, any access to that shard group should use the same connection as the write. We need to cover the whole shard group because writes and locks can cascade to other shards in the shard group via foreign keys, and they might be used together in a join. - After a write to a reference tables, any subsequent read of a reference table, including joins between distributed table shards and reference tables, should use the same connection as the write. - Metadata and global object changes should always use the same connection. - We should not overload worker nodes with parallel connections. In some cases, these goals conflict. For instance, if a multi-statement transaction performs a parallel delete on a distributed table, and then inserts into a reference table, and then attempts to join the distributed table with the reference table, then there is no way to complete that transaction correctly, since there is no single connection that can see both the reference table update and all the updates to distributed table shards. The command that reaches the conflict will error out: ```sql -- may fail if delete is parallelized begin; delete from dist_table; insert into reference_table values (1,2); select * from dist_table join reference_table on (x = a); ERROR: cannot perform query with placements that were modified over multiple connections abort; ``` The workaround is to `set citus.multi_shard_modify_mode TO 'sequential';` before or at the start of the transaction, which forces the delete (multi-shard modification) command to use a single connection, such that the insert and select can use the same connection. The primary goal of the connection management layer is not to solve all these problems, but to detect them and prevent any form of incorrectness, such as not seeing preceding changes in the transaction and self-deadlocks. A lot of important error-checking logic lives in FindPlacementListConnection, which attempts to find a suitable connection given a list of shard placements out of the connections that are already open, and also checks if the intent of the caller would lead to a conflict. The connection management logic is divided into two parts: - **connection_management.c** tracks all connections for various purposes and concerns itself with connection establishment, caching, and error handling. - **placement_connections.c** concerns itself with finding the right connection for a given shard placement access based on preceding commands in the transaction. ## Connection management Connection management tracks all the connections made by the current backend to other worker nodes. The connections can exist for the lifetime of the transaction, or longer when they are cached. The connections are kept in a hash that is keyed by hostname, port, user, database, and replication (yes/no). Each hash entry has a list of connections, since there can be multiple when the executor decides to parallelize a multi-shard query. Citus operations that need a connection call `StartNodeUserDatabaseConnection` (or a wrapper), which either returns an existing connection or a new one. the caller should wait for the connection to be fully established. When a Citus operation needs a connection to a worker node (hostname, port, user, database, replication), it can ask for it in a few different ways via flags: - Pick any connection (default), open a new one if none exists - Force a new connection (FORCE_NEW_CONNECTION), even if connections already exist - Pick a connection outside of a transaction (OUTSIDE_TRANSACTION), or open a new one if none exists - Pick the connection used for metadata syncing (REQUIRE_METADATA_CONNECTION), or open a new one if none exists and mark it for metadata syncing In addition, the caller can claim a connection exclusively, in which case it will not be returned until it is unclaimed (or transaction end). For instance, the adaptive executor claims connections it uses exclusively. When it calls `StartNodeUserDatabaseConnection` again, it will always get a new connection that it can use to parallelize the query. It is important that global commands like creating a type, or a function, or changing Citus metadata, all use the same connection. Otherwise, we might end up creating a type over one connection, and a function that depends on it over another. The use of the REQUIRE_METADATA_CONNECTION flag prevents this. The FORCE_NEW_CONNECTION and OUTSIDE_TRANSACTION flags can BOTH be used to execute (and commit) commands outside of the current transaction. Many usages of the older FORCE_NEW_CONNECTION flag could perhaps be replaced by OUTSIDE_TRANSACTION. A benefit of FORCE_NEW_CONNECTION is that it can provide a more intuitive way to parallelize commands than claiming connections exclusively. For instance, the `run_command_on_shards` uses FORCE_NEW_CONNECTION for this purpose. It is worth noting that Citus currently always opens a new connection when switching to a different user (e.g. via SET ROLE), rather than propagating the SET ROLE command. That can lead to some inconsistent behaviour (e.g. cannot see uncommitted writes after SET ROLE). ## Placement connection tracking The placement connection tracking logic stores which shard group placements were accessed over which connections during the current transactions, and whether they performed a SELECT, DML, or DDL. It considers whether to use same connection for accesses to the same shard group placement in the following cases: - SELECT after SELECT - can use different connection - DML after SELECT – can use different connection - All other cases – must use same connection The key function that deals with this logic is `FindPlacementListConnection` in [placement_connection.c](/src/backend/distributed/connection/placement_connection.c), which is called via `GetConnectionIfPlacementAccessedInXact` by the adaptive executor. We sometimes allow the same shard group placement to be accessed from different connections (first two cases). Consider a transaction that does a query on a reference table followed by a join between a distributed table and a reference table. Currently Citus would parallelize the second query, but that implicitly causes the reference table to be accessed from multiple connections. After that, we can still perform writes on the reference table (second case), because they do not conflict with the reads. However, we cannot perform most DDL commands involving the reference table because the locks would conflict with the reads, such that it would self-deadlock (blocked waiting for itself). We throw an error to prevent the self-deadlock and suggest set citus.multi_shard_modify_mode is ‘sequential’. Probably some DDL commands that take weaker locks would still be permissible, but we currently treat them all the same way. A downside of the current placement connection tracking logic is that it does not consider foreign keys to reference tables, and the fact that writes and locks can cascade from a write to a reference table. We have a separate subsystem for error checking those scenarios (relation_access_tracking.c), but it would be nice if they can be unified. ## citus.max_cached_connections_per_worker An important part of the connection management is caching at least 1 outgoing connection per worker node in the session. Establishing a new connection for every query is quite expensive due to SSL establishment, forking a process on the worker node, and rebuilding caches. Transactional workloads that have a high rate of short-running queries benefit a lot from caching connections. For analytical queries that take hundreds of milliseconds or more, the relative benefit is smaller, but often still noticeable. At the end of a transaction, the connection management logic decides which connections to keep. It keeps at most `citus.max_cached_connections_per_worker` regular connections that are in a healthy state, unless they are open for more than `citus.max_cached_connection_lifetime` (10 minutes by default). For workloads with a high rate of multi-shard queries, it can be beneficial to increase `citus.max_cached_connections_per_worker`. ## citus.max_shared_pool_size **The citus.max_shared_pool_size setting can be used to limit the number of outgoing connections across processes **. Each session has its own set of connections to other nodes. We often make multiple connections to the same worker node from the same session to parallelize analytical queries, but if all session are doing that we might overload the worker nodes with connections. That is prevented by setting citus.max_shared_pool_size, which should be at least `citus.max_client_connections` on coordinator node, and at most `max_connections - citus.max_client_connections` on worker node. The principle behind `citus.max_shared_pool_size` is that under high concurrency (all client connections used) it converges to each process having 1 connection per node. To do so, we distinguish between “optional” and “required” connections. When the executor asks the connection management layer for a connection, the first connection to a node is always required, and other connections are optional. If all connection slots are in use, the connection manager blocks until one is available when asking for a required connection, or returns NULL when asking for an optional connection. That signals to the executor that it cannot currently expand its pool. It may try again later. Most Citus code paths are tweaked to be able to complete their operation with 1 connection per node, and use local execution for local shards. Note that `citus.max_shared_pool_size` can only control the number of outgoing connections on a single node. When there are many nodes, the number of possible inbound internal connections is the sum of the `citus.max_shared_pool_size` on all other nodes. To ensure this does not exceed max_connections, we recommend that `sum(citus.max_client_connections) < max_connections`. # Transactions (2PC) Citus uses the transaction callbacks in PostgreSQL for pre-commit, post-commit, and abort to implement distributed transactions. In general, distributed transactions comprise a transaction on the coordinator and one or more transactions on worker nodes. For transactions that only involve a single worker node, Citus delegates responsibility to the worker node. For transactions that involve multiple nodes, Citus uses two-phase commit for atomicity and implements distributed deadlock detection. ## Single-node transactions Most multi-tenant and high-performance CRUD workloads only involve transactions that access a single worker node (or rather, a single shard group). For example, multi-tenant applications typically distribute and co-locate tables by tenant and transactions typically only involve a single tenant. When all statements in a transaction are routed to the same worker node, the coordinator simply sends commit/abort commands to that worker node from the commit/abort callbacks. In this case, the transaction is effectively delegated to that worker node. The worker node, by definition, provides the same guarantees as a single PostgreSQL server. ## Multi-node transactions For transactions that write to multiple nodes, Citus uses the built-in two-phase commit (2PC) machinery in PostgreSQL. In the pre-commit callback, a “prepare transaction” command is sent over all connections to worker nodes with open transaction blocks, then a commit record is stored on the coordinator. In the post-commit callback, “commit prepared” commands are sent to commit on the worker nodes. The maintenance daemon takes care of recovering failed 2PC transactions by comparing the commit records on the coordinator to the list of pending prepared transactions on the worker. The presence of a record implies the transaction was committed, while the absence implies it was aborted. Pending prepared transactions are moved forward accordingly. 2PC recovery Nice animation at: [How Citus Executes Distributed Transactions on Postgres](https://www.citusdata.com/blog/2017/11/22/how-citus-executes-distributed-transactions/) ## No distributed snapshot isolation Multi-node transactions provide atomicity, consistency, and durability guarantees. Since the prepared transactions commit at different times, they do not provide distributed snapshot isolation guarantees. An example anomaly that can occur is two distributed transactions: Two inserts in a transaction block into two different shards ```sql BEGIN; INSERT INTO test (key, value) VALUES (1,2); INSERT INTO test (key, value) VALUES (2,2); END; ``` An update across shards ```sql UPDATE test SET value = 3 WHERE value = 2; ``` An update across shards ```sql UPDATE test SET value = 3 WHERE value = 2; ``` If Citus provided serializability, there could only be 2 outcomes (a happens first or b happens first). However, it can have at least 4 outcomes, because the update depends on the inserts, and it might see only one of the insert as committed. This can happen because the inserts commit using a 2PC if the shards are on different nodes, and therefore they might not become visible at exactly the same time. Since the commits happen in parallel, there are no guarantees w.r.t. which insert becomes visible first. The update could see either insert as committed, or none, or both, depending on exact timings. Hence, there is no well-defined order between a and b, theye are intertwined. If the inserts depend on the update, there may be even more possible outcomes. For instance, if there is a unique constraint on (key, value), and we do upserts concurrently with the multi-shard update: ```sql BEGIN; INSERT INTO test (key, value) VALUES (1,2) ON CONFLICT DO NOTHING; INSERT INTO test (key, value) VALUES (2,2) ON CONFLICT DO NOTHING; END; ``` Now, whether the insert proceeds or does nothing depends on whether the update is already committed or not. Hence, this scenario has 6 possible outcomes. It is hard for users to understand these semantics and their implications. Therefore, many database researchers and engineers have a strong preference for serializability. Having fewer possible outcomes means less potential for bugs and unintended situations. On the other hand, the performance impacts of snapshot isolation are generally significant, and we have not seen a lot of problems due to the lack of snapshot isolation in practice. The types of transactional workloads that scale well and therefore benefit from Citus are the types of workloads that scope their transactions to a single node and therefore get all the usual PostgreSQL guarantees. Our long-term goal is to provide snapshot isolation as an optional feature, with at least read committed guarantees (default in PostgreSQL). ## Distributed Deadlocks Deadlocks are inevitable in a database system which supports transactions, and Citus is no exception. Marco wrote a useful blog post on what are locks & deadlocks & distributed deadlocks, please read the blog post first: https://www.citusdata.com/blog/2017/08/31/databases-and-distributed-deadlocks-a-faq/ Another good introduction for distributed deadlocks can be found here: https://www.citusdata.com/blog/2017/11/22/how-citus-executes-distributed-transactions/ At a high-level, the applications should try to avoid (distributed) deadlocks. The application should avoid patterns that could cause deadlocks. If those patterns are unavailable, then the database can still resolve the deadlocks. ### Distributed Deadlock Detection Implementation Citus heavily relies on PostgreSQL’s internal locking infrastructure for detecting distributed deadlocks. The entry function for the distributed deadlock detection is ` CheckForDistributedDeadlocks ()`. Distributed deadlock detection runs in the background as part of maintenance daemon. At a high level, Citus assigns “distributed transaction ids” for all backends running distributed transactions that might be part of a distributed deadlock (e.g., BEGIN; command; or any multi-shard command). See `assign_distributed_transaction()` function: ```sql BEGIN DELETE FROM test WHERE a = 1; NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, 5, '2023-09-14 17:39:21.263272+03'); ``` Then, Citus periodically (default 2 seconds), pulls all the “lock graphs” from all the nodes and combines them. If it finds a cycle in the combined graph, Citus concludes that there is a deadlock and kills one of the (distributed) processes involved in the graph. To give an example, assume we have the following case: + Transaction 1 waits Transaction 2 on Node-1 + Transaction 2 waits Transaction 1 on Node-2 In this case, none of the transactions can continue. This is a distributed deadlock as neither `Node-1` nor `Node-2` can detect the deadlock as there is no local deadlocks. To find distributed deadlocks, we pull the `lock graph`s from each node and combine. Let’s dive a little deeper. A node-lock lock graph is created in C function ` BuildLocalWaitGraph ()`. The lock graph includes only distributed transactions that are assigned via `assign_distributed_transaction` UDF. As noted, Citus heavily relies on Postgres on which backends are waiting for others, for details see ` AddEdgesForLockWaits ()` and ` AddEdgesForWaitQueue ()` C functions. Once each local lock graph is created, then the results are combined in on the coordinator. Then, for each node in the graph, we do a DFS (depth-first search) to check if there is a cycle involving that node. If there is a cycle, we conculde that there is a distributed deadlock. While doing the DFS, we also keep track of the other backends that are involved in the cycle. Citus picks the `youngest` transaction as the candidate to cancel (e.g., sends SIGINT). The idea is that let’s allow longer running transactions to continue, such as a long running DDL. If there is a cycle in the local graph, typically Postgres’ deadlock detection kicks in before Citus’ deadlock detection, hence breaks the cycle. There is a safe race condition between Citus’ deadlock detection and Postgres’ deadlock detection. Even if the race happens, the worst-case scenario is that the multiple backends from the same cycle is cancelled. In practice, we do not see much, because Citus deadlock detection runs `2x` slower (e.g., `citus.distributed_deadlock_detection_factor`) than Postgres deadlock detection. Deadlock detection For debugging purposes, you can enable logging with distributed deadlock detection: `citus.log_distributed_deadlock_detection` With query from any node, we run the deadlock detection from all nodes. However, each node would only try to find deadlocks on the backends that are initiated on them. This helps to scale deadlock detection workload across all nodes. When there are too many active backends(>1000), creating lots of `waiting activity` (e.g., blocked on the same locks not necessarily deadlocks involved), then the deadlock detection process might become a bottleneck. There are probably some opportunities to optimize the code for these kinds of workloads. As a workaround, we suggest increase `citus.distributed_deadlock_detection_factor`. The distributed transactionId and backend/PID mapping is done via BackendData structure. For every Postgres backend, Citus keeps a `BackendData` structure. Each backends state is preserved in `MyBackendData` C structure. Assigning (and removing) distributed transaction id to a backend means to update this structure. If we were to implement distributed deadlock detection today, we would probably try to build it on top of `Global PID` concept instead of `distributed transaction id`. But, before changing that, we should make sure the `Global PID` is robust enough. `Global PID` today mostly used for observation of the cluster. We should slowly put more emphasis on the `Global PID` and once we feel confortable with it, we can consider using it for distributed deadlock detection as well. # Locking Locks in a database like Postgres (and Citus) make sure that only one user can change a piece of data at a time. This helps to keep the data correct and safe. If two users try to change the same data at the same time, it could create problems or errors. Citus, a distributed database, needs extra locks because it stores data across multiple servers. These extra locks help make sure that even when many servers are involved, the data stays correct and safe during changes made by multiple users. In PostgreSQL and Citus, there are several types of locks that serve different purposes. In this section, we’d like to go over these different types of locks and explain when/how they are useful. ## Lock Levels In database management, locking mechanisms are crucial for maintaining data integrity during concurrent operations. However, not all locks are created equal. As a building block, PostgreSQL allows different levels/modes of locks. So, this is probably different than what you have learned in your college, where if a lock is held, others have to wait. No, in PostgreSQL, some locks do not conflict with each other, whereas some do. This flexibility allows Postgres (and Citus) to implement sophisticated concurrency scenarios. For details of lock levels, please refer to PostgreSQL docs: https://www.postgresql.org/docs/current/explicit-locking.html Understanding these lock types and their levels of restrictiveness can help you better manage concurrent operations and ensure data integrity. ## Lock Monitoring Both PostgreSQL and Citus provide comprehensive views for monitoring the locks held (or waiting on) for each backend. `pg_locks`: https://www.postgresql.org/docs/current/view-pg-locks.html. In Citus, we have the same view, but they are collected from all nodes in the cluster: `citus_locks` You can find lots of examples of how `pg_locks` (and `citus_locks`) can be used in debugging systems. One of the good one is from PostgreSQL’s wiki, ` Сombination of blocked and blocking activity`: https://wiki.postgresql.org/wiki/Lock_Monitoring The same query is also implemented within Citus for the distributed cluster, with the name: citus_lock_waits: https://github.com/citusdata/citus/blob/4e46708789478d6deccd3d121f2b4da7f631ebe3/src/backend/distributed/sql/udfs/citus_lock_waits/11.0-1.sql#L4 These simple monitoring tools are essential to understanding the concurrency in PostgreSQL and Citus. ## Lock Types 1. **Table-level Locks**: Table-level locks in PostgreSQL range from less restrictive to more restrictive, allowing various levels of concurrent access. They can lock an entire table to ensure data integrity during operations like adding columns. You can also acquire any of these locks explicitly with the command LOCK. Two transactions cannot hold locks of conflicting modes on the same table at the same time. (However, a transaction never conflicts with itself. For example, it might acquire `ACCESS EXCLUSIVE` lock and later acquire `ACCESS SHARE` lock on the same table.) Marco’s blog post on locks could provide a nice read on this topic: https://www.citusdata.com/blog/2018/02/15/when-postgresql-blocks/ As a rule, in most cases, Citus relies on PostgreSQL to acquire the table-level locks. For example, + When a DDL is executed on a Citus table, Citus first executes Postgres’ `standardProcess_Utility()` function. One of the key reasons behind that is Postgres acquires the table-level lock, and Citus provides similar concurrency behavior with Postgres on Citus tables. If an `ALTER TABLE .. ADD COLUMN` is running on a distributed table, no `SELECT` command could run concurrently due to the table-level locks. + When regular commands like `INSERT`/`UPDATE`/`DELETE`/`SELECT` is executed, Citus again relies on Postgres to acquire the table-level locks. PostgreSQL acquires the table-level locks during the parsing of the statements, which makes life simple for Citus, as parsing happens even before any Citus logic kicks in. If the command doesn’t require parsing, such as prepared statements, then Postgres still acquires the same locks before using the cached plan. So, from Citus’ perspective, there mostly is nothing to do for acquiring the table-level locks. + + There is only a one expection to this rule, Citus' _local plan caching_. When Citus caches the queries by itself, Citus acquires the relevant table-level locks. See `ExecuteLocalTaskListExtended()` as the relevant C function. Citus additionally use table-level locks for certain table management operations on tables. With all these operations, Citus aims to fit into the same concurrency behaviors as Postgres. For example, when `create_distributed_table()` is executed, Citus acquires an `ExclusiveLock` on the table. We do that because we want to block `write`s on the tables – which acquire RowExclusiveLock -- but let `read-only` queries to continue – which acquire AccessShareLock. An additional benefit of this approach is that no two concurrent `create_distributed_table` on the same table can run. One another use case for table-level locks on Citus is the table-level locks acquired on the Citus metadata tables. Citus uses table-level locks on the metadata tables to define the concurrency behavior of certain operations. For example, while creating a new table or moving shards, it is common to acquire `ShareLock` on `pg_dist_node` table, and `citus_add_node` function to acquire `ExclusiveLock` on the same metadata table. The latter signals the rest of the backends that the node metadata is about to change, so it is not allowed to rely on the current state of `pg_dist_node` (or vice-versa `citus_add_node` should wait until rebalance finishes). The main C function involved in table-level locking is ` LockRelationOid ()`. So, you can put a break-point to this function and see when/how Citus and Postgres acquires table-level locks. 2. **Row-level Locks**: Row-level locks are more granular and lock only specific rows within a table. This allows multiple users to read and update different rows simultaneously, which can improve performance for multi-user scenarios. Citus does NOT involve in the row-level locks, fully relies on Postgres to acquire the locks on the shards. Marco’s blog-post gives a nice overview of row-level locks: https://www.citusdata.com/blog/2018/02/15/when-postgresql-blocks/ 3. **Advisory Locks:** Advisory locks are a special type of lock in PostgreSQL that give you more control over database operations. Unlike standard table or row-level locks that automatically lock database objects, advisory locks serve as flexible markers or flags. Developers can implement these custom locks to define their own rules for managing concurrency, making them particularly useful for extensions and custom workflows. #### Importance of Advisory locks for Citus In a distributed system like Citus, advisory locks take on an even more critical role. Because Citus spreads data across multiple nodes, managing concurrent operations becomes a complex task. Citus heavily relies on advisory locks for a variety of essential operations. Whether it's handling queries from any node, moving/splitting shards, preventing deadlocks, or managing colocation of related data, advisory locks serve as a powerful tool for ensuring smooth operation and data integrity. By employing advisory locks, Citus effectively deals with the complexities that come with distributed databases. They allow the system to implement sophisticated concurrency scenarios, ensuring that data remains consistent and operations are efficient across all nodes in the cluster. Below, we list some of the crucial advisory locks that Citus relies on: #### Citus Advisory Locks 1: Distributed Execution locks The C code is for a function called `AcquireExecutorShardLocksForExecution()`. The function has an extensive comment for the specific rules. The function's main goal is to get advisory locks on shard IDs to make sure data stays safe and consistent across different operations. These locks are sometimes referred as `ShardResourceLock`s in the code. In the context of distributed databases like Citus, "safe" generally means avoiding situations where different nodes in the cluster are trying to modify the same data at the same time in a way that could lead to errors, inconsistencies, or even deadlocks. This is critical when data is replicated across nodes (e.g., multiple copies of the same shard like reference tables) or when a single operation affects multiple shards (e.g., multi-shard update). The "consistency" here primarily refers to two things: 1. **Order of Operations on Replicated Tables**: In a replicated table setup, the same data exists on multiple nodes. The function aims to make sure that any updates, deletes, or inserts happen in the same order on all copies of the data (replicas). This way, all the replicas stay in sync. 2. **Preventing Distributed Deadlocks**: When you're running multi-shard operations that update the data, you can run into distributed deadlocks if operations on different nodes lock shards in a different order. This function ensures that the locks are acquired in a specific, consistent order, thus minimizing the risk of deadlocks. There are also options (`GUCs`) to relax these locking mechanisms based on the user's needs, but they come with the trade-off of potentially reduced consistency or increased risk of deadlocks. So, in summary, this function is about acquiring the right kind of lock based on what the operation is doing and what kind of table it's affecting, all to ensure that the data stays consistent and that operations are executed safely. #### Citus Advisory Locks 2: Metadata locks The second class of advisory locks referred as `metadata locks` in the code. See `AcquireMetadataLocks()` and `LockShardDistributionMetadata()` functions. The main goal is to prevent concurrent placement changes and query executions. Essentially ensure that the query execution always works on the accurate placement metadata (e.g., shard placements). Citus always acquire `Metadata Lock`s for shard moves and shard splits, irrespetive of blocking vs non-blocking operations. For blocking operations, the locks are held from the start of the operation whereas for non-blocking operations the locks are held briefly at the end right before metadata is updated. Citus always acquires `Metadata Lock`s for modification queries, at the `CitusBeginModifyScan` such that it serializes query modification with placement metadata changes. A modification query would always see up-to date metadata for the placements involved. Otherwise, the modification might get lost. Citus does not acquire Metadata Locks for SELECT queries. The main reason is that SELECTs are often long-running and would hold up the move. Instead, we allow SELECT commands to operate on the old placements in case of a concurrent shard move. The SELECT commands would already see the snapshot of the shard(s) when the SELECT started. So, there is no difference in terms of query correctness. We then later drop the old placements via "deferred drop" (see Resource cleanup). #### Citus Advisory Locks 3: Query from any node When users are allowed to run queries from any node, then in certain cases, we need to form a synchronization that involes multiple nodes. Advisory locks is a convinent tool for achieving these types of goals. Citus exposes few UDFs like `lock_shard_resources()` and `lock_shard_metadata()` which are simple wrappers around the metadata and executor locks we discussed above. When there are nodes with metadata, then Citus acquires some of the advisory locks on all nodes, like: ```sql select citus_move_shard_placement(102008, 'localhost', 9701, 'localhost', 9702, shard_transfer_mode:='force_logical'); .... DETAIL: on server onderkalaci@localhost:9702 connectionId: 2 NOTICE: issuing SELECT lock_shard_metadata(7, ARRAY[102008]) DETAIL: on server onderkalaci@localhost:9701 connectionId: 1 NOTICE: issuing SELECT lock_shard_metadata(7, ARRAY[102008]) DETAIL: on server onderkalaci@localhost:9702 connectionId: 2 ... ``` Another useful application of advisory locks in query from any node is modifications to reference tables. Reference tables (or in general replicated tables) have multiple placements for a given shard. So, modifications to the placements of the same shard should be serialized. We cannot allow multiple modification commands to modify the placements in different orders. It could cause diverging the contents of the data. The coordinator already serializes the modifications to reference tables (or in general all replicated tables) via `LockShardResource()` C function. When there are other nodes in the cluster, Citus sends the similar command, to the first worker node in the cluster. In general, Citus aims to serialize operations on the reference tables via acquiring advisory locks on the first worker node, see `SerializeNonCommutativeWrites()` and `LockShardListResourcesOnFirstWorker()` C functions. We use the `first worker node` as an optimization. Instead of acquiring the locks in all the nodes, each node sorts the worker nodes deterministically, and acquires the lock on the first node. Whichever distributed transaction acquires the lock, it has the autority to continue to the transaction. If there are any other transactions, they are blocked until the first distributed transaction finishes, as it would be in coordinator-only configuration. We used the first worker node as opposed to the coordinator for two reasons. First, the coordinator might already be getting lots of client queries, and we don't want to create additional load to the coordinator. Second, in some Citus deployments, the coordinator may **not** be in the metadata. Hence, the other nodes might not know about the coordinator. Getting back to the basic flow, the outcome of modifying the reference tables (or replicated tables) is the following where `lock_shard_resources` is acquired on the first node: ```sql BEGIN; -- 9701 is the first worker node in the metadata insert into reference_table VALUES (1); NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, 8, '2023-09-18 16:30:18.715259+03'); DETAIL: on server onderkalaci@localhost:9701 connectionId: 1 NOTICE: issuing SELECT lock_shard_resources(3, ARRAY[102040]) DETAIL: on server onderkalaci@localhost:9701 connectionId: 1 NOTICE: issuing INSERT INTO public.reference_table_102040 (a) VALUES (1) ``` #### Citus Advisory Locks 4: Colocation locks Although not ideal, there are currently multiple advisory locks that deal with colocation: `AcquireRebalanceColocationLock()` and `AcquirePlacementColocationLock()`. The former has a simple logic. It is to prevent concurrent shard moves of the same shards with the same table colocation. So, basically, prevent user running `citus_move_shard_placement()` for the same colocated shards. The latter is a bit more interesting. It aims to ensure the placements of the same colocation group never diverges regarding where the placements reside. This might sound a given feature of Citus. However, with concurrent `create_distributed_table` and `shard move/split`s there are might be some race conditions. The former operation might use the view of the placements before the `shard move/split`, whereas the latter changes this. As a result, the new table might have a placement that is not colocated with the other placements anymore. To prevent that, Citus acquires `AcquirePlacementColocationLock()` while the metadata of placements changed/read. This lock introduced to fix a user reported bug: https://github.com/citusdata/citus/issues/6050 4. **Low-level Locks (SpinLocks and LWLocks)**: SpinLocks and LWLocks are low-level locking mechanisms often used internally by the database system. Citus uses `LwLocks` and `SpinLocks` as described in Postgres' source code: https://github.com/postgres/postgres/tree/master/src/backend/storage/lmgr + **Spinlocks:** Spinlocks are used for very short-term locking and are meant to be held only for a few instructions. They don't have features like deadlock detection or automatic release, and waiting processes keep checking ("busy-loop") until they can acquire the lock. The lack of "automatic release" could be very critical. For other lock types, when the transaction finishes, the locks are released by Postgres automatically. It means that, say a `palloc` fails due to lack of enough memory, we rely on Postgres to release all the locks. However, this is **NOT** true for spin locks. So, do not even allocate any memory while holding spinlock, only do very simple assignments etc. Otherwise, the lock might be held until a restart of the server. In the past we had some bugs where we had a `palloc` failure while holding `SpinLock` and which prevented the lock to be released. So, be extra careful when using `SpinLock`. See the bugfix as a reference: https://github.com/citusdata/citus/pull/2568/files + **Lightweight Locks (LWLocks):** These locks are mainly used for controlling access to shared memory data structures and support both exclusive and shared lock modes. While they also lack deadlock detection, they do automatically release when errors are raised, and waiting processes block on a SysV semaphore, conserving CPU time. # Rebalancing A high-level overview of the shard rebalancer is given in [this rebalancer blog post][rebalancer-post]. It is a bit outdated though, specifically that it uses `rebalance_table_shards()` instead of the newer `citus_rebalance_start()`. The shard rebalancer consists of 4 main parts: 1. The rebalancing algorithm: Decides what moves/splits it should do to make the cluster balanced. 2. The background task runner: Runs a full rebalance according to a plan created by the planner. 3. A shard group moves/split: These are the smallest units of work that the rebalancer does, if this fails midway through the move is aborted and the shard group remains unchanged. 4. Deferred cleanup: The source shards stay present for a while after a move to let long-running read queries continue, eventually they need to be cleaned up. These parts interact, but they are pretty self-contained. Usually it's only necessary to change one of them to add a feature/fix a bug. [rebalancer-post]: https://www.citusdata.com/blog/2021/03/13/scaling-out-postgres-with-citus-open-source-shard-rebalancer/ ## Rebalancing algorithm The rebalancing algorithm tries to find an optimal placement of shard groups across nodes. This is not an easy job, because this is a [co-NP-complete problem](https://en.wikipedia.org/wiki/Knapsack_problem). So instead of going for the fully optimal solution it uses a greedy approach to reach a local optimum, which so far has proved effective in getting to a pretty optimal solution. Even though it won't result in the perfect balance, the greedy approach has two important practical benefits over a perfect solution: 1. It's relatively easy to understand why the algorithm decided on a certain move. 2. Every move makes the balance better. So if the rebalance is cancelled midway through, the cluster will always be in a better situation than before. As described in the [this rebalancer blog post][rebalance-post] the algorithm takes three inputs from the function in the `pg_dist_rebalance_strategy` table: 1. Is a shard group allowed on a certain node? 2. What is the "cost" of a shard group, relative to the other shard groups? 3. What is the "capacity" of a node, relative to the other nodes? Cost and capacity are vague on purpose, this way users can choose their own way to determine cost of a shard group, but **in practice "cost" is usually disk size** (because `by_disk_size` is the default rebalance strategy). Capacity is almost always set to 1, because almost all Citus clusters are homogeneous (they contain the same nodes, except for maybe the coordinator). The main usage for "Is a shard group allowed on a certain node?" is to be able to pin a specific shard group to a specific node. There is one last definition that you should know to understand the algorithm and that is "utilization". Utilization is the total cost of all shard groups divided by capacity. In practice this means that utilization is almost always the same as cost because as explained above capacity is almost always 1. So if you see "utilization" in the algorithm, for all intents and purposes you can read it as "cost". The way the general algorithm works is fairly straightforward. It starts by creating an in-memory representation of the cluster, and then it tries to improve that in-memory representation by making theoretical moves. So to be clear the algorithm doesn't actually do any shard group moves, it only does those moves to its in-memory representation. The way it determines what theoretical moves to make is as follows (updating utilization of in-memory nodes after every move): 1. Find all shard groups that are on a node where they are not allowed (due to "Is a shard group allowed on a certain node?") 2. Order those nodes by cost 3. Move them one-by one to nodes with the lowest utilization where they are allowed. 4. If the cluster is balanced we are done. 5. Take the most utilized node (A) and take the least utilized node (B). 6. Try moving the shard group with the highest cost from A to B. 7. If the balance is "better" commit this move and continue from step 4. (See subsection below for what is "better") 8. If the balance is worse/equal try again from step 6 with the shard group with the next highest cost on node A. If this was the lowest cost shard on node A, then try with the highest cost shard again but on the next least utilized node after node B. If no moves helped with the balance, try with the next most utilized node after node A. If we tried all moves for all nodes like this, we are done (we cannot get a better balance). Of course, the devil is in the details though. ### When is the balance better? The main way to determine if the balance is better is by comparing the utilization of node A and B, before and after the move and seeing if they are net closer to the average utilization of the nodes in the cluster. The easiest way to explain this is with a simple example: We have two nodes A and B. A has a utilization of 100GB and B has a utilization of 70GB. So we will move a shard from A to B. A move of 15GB is obviously best, it results in perfect balance (A=85GB, B=85GB). A move of a 10GB is still great, both improved in balance (A=90GB, B=80GB). A move of 20GB is also good, the result is the same as a move of 10GB only with the nodes swapped (A=80GB, B=90GB). The 10GB vs 20GB move shows a limitation of the current algorithm. The algorithm mostly makes choices based on the end state, not on the cost of moving a shard. This is usually not a huge problem in practice though. ### Thresholds The algorithm is full of thresholds, the main reason these exist is because moving shards around isn't free. - `threshold`: Used to determine if the cluster is in a good enough state. For the `by_disk_size` rebalance strategy this is 10%, so if all nodes are at most 10% above or 10% below the average utilization then no moves are necessary anymore (i.e. the nodes are balanced enough). The main reason for this threshold is that these small differences in utilization are not necessarily problematic and might very well resolve automatically over time. For example, consider a scenario in which one shard gets mostly written in during the weekend, while another one during the week. Moving shards on Monday and that you then have to move back on Friday is not very helpful given the overhead of moving data around. - `improvement_threshold`: This is used in cases where a shard group move from node A to B swaps which node now has the highest utilization (so afterwards B will have higher utilization than A). As described above this can still result in better balance. This threshold is meant to work around a particularly bad situation where we move a lot of data for very little benefit. Imagine this situation: A=200GB and B=99, thus moving a 100GB shard from A to B would bring their utilization closer to the average (A=100GB, B=199GB). But obviously that's a tiny gain for a move of 100GB, which probably takes lots of resources and time. The `improvement_threshold` is set to 50% for the `by_disk_size` rebalance strategy. This means that this move is only chosen if the utilization improvement is larger than 50% of the utilization that the shard group causes on its current node. ### How do multiple colocation groups impact the rebalancer algorithm? The previous section glossed over colocation groups a bit. The main reason for that is that the algorithm doesn't handle multiple colocation groups very well. If there are multiple colocation groups each colocation group gets balanced completely separately. For the utilization calculations only the costs are used for the shard groups in the colocation group that is currently being rebalanced. The reasoning for this is that if you have two colocation groups, you probably want to spread the shard groups from both colocation groups across multiple nodes. And not have shard groups from colocation group 1 only be on node A and shard groups from colocation group 2 only be on node B. There is an important caveat here though for colocation groups that have fewer shard groups than the number of nodes in the cluster (in practice these are usually colocation groups used by schema based sharding, i.e. with a single shard group): The rebalancer algorithm balances the shard groups from these colocation groups as if they are all all part of a single colocation group. The main reason for this is to make sure that schemas for schema based sharding are spread evenly across the nodes. ## Shard moves Shard moves move a shard group placement to a different node (group). It would be more correct if these were called "shard **group** moves", but in many places we don't due to historical reasons. Moves are orchestrated by the `citus_move_shard_placement` UDF, which is also the function that the rebalancer runs to move a shard. We implement blocking and non-blocking shard splits. Non-blocking shard moves use logical replication, which has an important limitation. If the (distributed) table does not have a replica identity (usually the primary key), then update/delete commands will error out once we create a publication. That means using a non-blocking move without a replica identity does incur some downtime. Since a blocking move is generally faster (in part because it forces out regular work), it may be less invasive. We therefore force the user to choose when trying to move a shard group that includes a table without a replica identity by supplying `shard_transfer_mode := 'force_logical'` or `shard_transfer_mode := 'block_writes'`. The blocking-move is mostly a simplified variant of the non-blocking move, where the write locks are taken upfront so that no catch-up using logical replication is needed. A non-blocking move involves the following steps: - **Create the new shard group placement on the target node**. We also create constraints that do not involve an index and set up ownership and access control. - **Create publication(s) on the source node**. We create publications containing the shards in the source shard group placement. We create one publications per table owner, mainly because we need one subscription per table owner to prevent privilege escalation issues on older versions of PostgreSQL (15 and below). - **Create replication slot(s) and export snapshot(s)**. We create a slot per table owner because we use a separate subscription per table owner, similar to publications. Subscriptions can create the replication slot, but we (nowadays) copy the data outside of the subscription because we apply several optimizations. - **Create subscription(s) in disabled state**. We create subscriptions upfront in case there are any errors (e.g. hitting resource limits, connectivity issues). We create one subscription per table owner, and we set the subscription owner to the table owner. The logical replication will happen with the permissions of the subscription owner. - **Copy the data from the source to target** by calling `worker_shard_copy` function for each source shard placement via the executor. The `worker_shard_copy` function makes a single pass over the source shard and pushes it into the target shard via `COPY`. We found this to be faster than using the `copy_data` option in the subscription because we can benefit from binary copy, optimizations in the adaptive executor, uses fewer replication slots, and it simplifies the flow. Some of these optimizations might be obsolete as of PostgreSQL 16. - **Enable subscription(s)**, which starts the replication of writes that happened on the source shard group placement during the data copy. - **Wait for subscription(s) to catch up to the current source LSN**. This can take a while since many writes could have happened during the data copy. - **Create indexes, unique/exclusion constraints, statistics, etc.**. For efficiency, we create these objects after copying the data and catching up to concurrent writes. - **Wait for subscription(s) to catch up to the current source LSN**. This can take a while since many writes could have happened during the index creation. - **Block writes to the split parent by acquiring metadata locks globally**. At this point, we wait for any ongoing INSERT/UPDATE/DELETE/COPY/MERGE to finish and block new ones. Once we acquire the locks we try to quickly finalize the split. - **Wait for subscription(s) to catch up to the current source LSN**. Some writes could still have happened before acquiring locks, we wait for those writes to be replicated. - **Update the metadata**. We globally update the `pg_dist_placement` record to point to the new node. Writes tactically acquire metadata locks just before reading from `pg_dist_placement`, so they will see the new placement as soon as we commit. - **Create foreign keys on the target shard group placement**. Creating foreign keys is deferred until the replication is fully done, because we used multiple subscriptions for tables with different owners and this is the first time that the data is guaranteed to be consistent between shards. We avoid rechecking foreign keys by using the `citus.skip_constraint_validation` setting on the session. - **Final cleanup of connections, resources**. We primarily lean on "Resource cleanup" to drop publications, replication slots, subscriptions, which ensures they are removed both in case of success and failure. The source shard group placement is dropped once all ongoing (read-only) queries are done, by repeatedly dropping with a short lock timeout until it succeeds. It is worth noting that the final commit happens in a 2PC, with all the characteristics of a 2PC. If the commit phase fails on one of the nodes, writes on the shell table remain blocked on that node until the prepared transaction is recovered, after which they will see the updated placement. The data movement generally happens outside of the 2PC, so the 2PC failing on the target node does not necessarily prevent access to the shard. A similar operation to shard moves is `citus_copy_shard_placement`, which can be used to add a replica to a shard group. We also use this function to replicate reference tables without blocking. The main difference is that dropping the old shard group placement is skipped. A workaround for the replica identity problem is to always assign REPLICA IDENTITY FULL to distributed tables / shards if they have no other replica identity. However, prior to PostgreSQL 16, replication of updates and delete to a table with replica identity full could be extremely slow (it does a sequential scan per tuple). As of PostgreSQL 16, the logical replication worker can use a regular btree index to find a matching tuple (if one exists). Even for distributed tables without any indexes, and without a replica identity, we could tactically set REPLICA IDENTITY FULL on the shards, and create a suitable index on the target shard group placement for the duration of the move. Once we implement this, we could avoid erroring for distributed tables without a replica identity. ## Shard splits Shard splits convert one shard group ("split parent") into two or more shard groups ("split children") by splitting the hash range. Just like with shard moves it would be more correct to call these "shard **group** splits", but again we often don't. The new shard groups can be placed on the node itself, or on other nodes. We implement blocking and non-blocking shard splits. The blocking variant is mostly a simplified version of non-blocking, so we only cover non-blocking here. Shard splits have many similarities to shard moves, and have the same `shard_transfer_mode` choice. The shard split is a lengthy process performed by the `NonBlockingShardSplit` function, supported by a custom output plugin to handle writes that happen during the split. There are a few different entry-points in this logic, namely: `citus_split_shard_by_split_points`, `create_distributed_table_concurrently`, and `isolate_tenant_to_node`. We currently do not build a separate .so file for the output plug-in, so it is part of citus.so and therefore the name of the output plug-in is "citus". The purpose of the output plug-in is to translate changes to the original shard group to changes to the split children, and emit them in pgoutput format (by calling the original pgoutput plug-in). In some cases, the schema of the split parent can be subtly different from the split children. In particular, some columns may have been dropped on the parent. Dropped columns remain in the metadata and remaining columns are not renumbered, but when we create the split children we only create it with current columns. When this scenario occurs, we convert the tuple in `GetTupleForTargetSchema`. A split involves the following steps: - **Create the new shard groups ("split children") on the target node(s)**. We also create constraints that do not involve an index and set up ownership and access control. - **Create "dummy" shard groups on the source node**, unless the split child is also on the source node. The reason for creating the dummy shards is primarily to make the pgoutput output plug-in happy. Our output plug-in maps changes to the split parent into changes to split children before calling pgouput, and those tables need to exist for pgoutput to be able to interpret and emit the change, even when that table is not actually used on the source node. - **Create replica identities on dummy shards**. This is also needed to keep pgoutput happy, because for updates and deletes it emits the values in the replica identity columns, so it needs to know what the replica identity is. - **Create publication(s) on the source node**, which include both the parent and children. We add the split parent for our own output plug-in to recognize which shard group it should split, and we add the split children for pgoutput to recognize that it should emit them. We might make multiple publications per shard group because we use a separate publication and subscription per table owner, to prevent privilege escalation issues on older versions of PostgreSQL (15 and below). - **Set up the shard split output plug-in**. We configure our output plug-in on the source node via `worker_split_shard_replication_setup`, which sets up a dynamic shared memory (DSM) segment that the output plug-in will read from. We currently only have one DSM segment, which would need to changed to support concurrent splits from the same node. - **Create replication slot(s) and export snapshot(s)**. We cannot perform any write to the database before this step, because this step waits for all transactions that perform writes to finish. We create multiple slots because we use a separate slot per table owner, similar to publications. - **Create subscription(s) in disabled state**. We create subscriptions upfront in case there are any errors (e.g. hitting resource limits, connectivity issues). We create a slot per table owner because we use a separate subscription per table owner, similar to publications. The logical replication will happen with the permissions of the subscription owner. - **Split the data in the split parent into the split children** using `worker_split_copy` with the exported snapshot. The `worker_split_copy` function makes a single pass over the parent shard and pushes it into the split children via `COPY`, either via a connection to another node or by invoking the COPY logic locally when the split children are on the same node. Internally, it uses the DestReceiver APIS and effectively it layers the DestReceiver used in re-partition operations on top of the DestReceiver used by `worker_shard_copy` in shard moves. We run a separate `worker_split_copy` task for every shard in the shard group and execute them via the adaptive executor, which may elect to parallelize them. - **Enable subscription(s)**, which starts the replication of writes that happened on the split parent during the data copy into the split children. - **Wait for subscription(s) to catch up to the current source LSN**. This can take a while since many writes could have happened during the data copy. - **Create indexes, unique/exclusion constraints, statistics, etc.**. For efficiency, we create these objects after copying the data and catching up to concurrent writes. - **Wait for subscription(s) to catch up to the current source LSN**. This can take a while since many writes could have happened during the index creation. - **Block writes to the split parent by acquiring metadata locks globally**. At this point, we wait for any ongoing INSERT/UPDATE/DELETE/COPY/MERGE to finish and block new ones. Once we acquire the locks we try to quickly finalize the split. - **Wait for subscription(s) to catch up to the current source LSN**. Some writes could still have happened before acquiring locks, we wait for those writes to be replicated. - **Update the metadata**. We globally delete the metadata of the split parent, and insert the metadata of the split children. In case of `create_distributed_table_concurrently` we also update `pg_dist_partition` and `pg_dist_colocation`. - **Create partitioning hierarchy and foreign keys on the split children**. Creating these relationships is deferred until the replication is fully done, because we used multiple subscriptions for tables with different owners and this is the first time that the data is guaranteed to be consistent between shards. We avoid rechecking foreign keys by using the `citus.skip_constraint_validation` setting on the session. - **Final cleanup of DSM, connections, resources**. We clean up all the resources we created such as publications, subscriptions, replication slots, dummy shards via "Resource lceanup", as well as the split parent (deferred, in case of success) or split children (in case of failure). We currently do not clean up the DSM in case of failure, but we always idempotently reset it when doing another split. A difference between splits and moves is that the old shard ID disappears. In case of a move, only the placement changes and for writes we always look up placement in the executor after acquiring locks that conflict with moves (wait until move is done). In case of a split, the query changes in more fundamental ways, and a single-shard query might actually become a multi-shard queryif it were replanned. When a writes get to the executor, after acquiring locks that conflict with the shard split (wait until split is done), we check whether the shard still exists _in the metadata_ and in case of fast path queries (which are strictly single shard), we try to reroute in `TryToRerouteFastPathModifyQuery`. Otherwise, we error in `EnsureAnchorShardsInJobExist`. In case of reads, we lean on the deferred drop logic to let the read proceed on the old shard placement. ## Background tasks In the past the only way to trigger a rebalance was to call `rebalance_table_shards()`, this function run the rebalance using the current session. This has the huge downside that the connection needs to be kept open until the rebalance completes. So eventually we [introduced `citus_rebalance_start()`](https://www.citusdata.com/blog/2022/09/19/citus-11-1-shards-postgres-tables-without-interruption/#rebalance-background), which uses a background worker to do the rebalancing, so users can disconnect their client and the rebalance continues. It even automatically retries moves if they failed for some reason. The way this works is using a general background job infrastructure that Citus has in the tables `pg_dist_backround_job` and `pg_dist_background_task`. A job (often) contains multiple tasks. In case of the rebalancer, the job is the full rebalance, and each of its tasks are separate shard group moves. ### Parallel background task execution A big benefit of the background task infrastructure is that it can execute tasks and jobs in parallel. This can make rebalancing go much faster especially in clusters with many nodes. To ensure that we're not doing too many tasks in parallel though we have a few ways to limit concurrency: 1. Tasks can depend on each other. This makes sure that one task doesn't start before all the ones that it depends on have finished. 2. The maximum number of parallel tasks being executed at the same time can be limited using `citus.max_background_task_executors`. The default for this is 4. 3. Tasks can specify which nodes are involved in the task, that way we can control that a single node is not involved into too many tasks. The rebalancer specifies both the source and target node as being involved in the task. That together with the default of 1 for `citus.max_background_task_executors_per_node` makes sure that a node doesn't handle more than a single shard move at once, while still allowing moves involving different nodes to happen in parallel. For larger machines it can be beneficial to increase the default a bit. ## Resource cleanup During a shard move/split, some PostgreSQL objects can be created that live outside of the scope of any transaction or are committed early. We need to make sure those objects are dropped once the shard move ends, either through failure or success. For instance, subscriptions and publications used for logical replication need to be dropped in case of failure, but also the target shard (in case of failure) and source shard (in case of success). To achieve that, we write records to pg_dist_cleanup before creating an object to remember that we need to clean it. We distinguish between a few scenarios: **Cleanup-always**: For most resources that require cleanup records, cleanup should happen regardless of whether the operation succeeds or fails. For instance, subscriptions and publications should always be dropped. We achieve cleanup always by writing pg_dist_cleanup records in a subtransaction, and at the end of the operation we try to clean up object immediately and if it succeeds delete the record. If cleanup fails, we do not fail the whole operation, but instead leave the pg_dist_cleanup record in place for the maintenance daemon. **Cleanup-on-failure**: Cleanup should only happen if the operation fails. The main example is the target shard of a move/split. We achieve cleanup-on-failure by writing pg_dist_cleanup records in a subtransaction (transaction on a localhost connection that commits immediately) and deleting them in the outer transaction that performs the move/split. That way, they remain in pg_dist_cleanup in case of failure, but disappear in case of success. **Cleanup-deferred-on-success**: Cleanup should only happen after the operation (move/split) succeeds. We use this to clean the source shards of a shard move. We previously dropped shards immediately as part of the transaction, but this frequently led to deadlocks at the end of a shard move. We achieve cleanup-on-success by writing pg_dist_cleanup records as part of the outer transaction that performs the move/split. Resource cleaner (currently shard_cleaner.c) is part of the maintenance daemon and periodically checks pg_dist_cleanup for cleanup tasks. It’s important to prevent cleanup of operations that are still running. Therefore, each operation has a unique operation ID (from a sequence) and takes an advisory lock on the operation ID. The resource cleaner learns the operation ID from pg_dist_cleanup and attempts to acquire this lock. If it cannot acquire the lock, the operation is not done and cleanup is skipped. If it can, the operation is done, and the resource cleaner rechecks whether the record still exists, since it could have been deleted by the operation. Cleanup records always need to be committed before creating the actual object. It’s also important for the cleanup operation to be idempotent, since the server might crash immediately after committing a cleanup record, but before actually creating the object. Hence, the object might not exist when trying to clean it up. In that case, the cleanup is seen as successful, and the cleanup record removed. # Logical decoding / CDC PostgreSQL supports change data capture (CDC) via the logical decoding interface. The basic idea behind logical decoding is that you make a replication connection (a special type of postgres connection), start replication, and then the backend process reads through the WAL and decodes the WAL records and emits it over the wire in a format defined by the output plugin. If we were to use regular logical decoding on the nodes of a Citus cluster, we would see the name of the shard in each write, and internal data transfers such as shard moves would result in inserts being emitted. We use several techniques to avoid this. All writes in PostgreSQL are marked with a replication origin (0 by default) and the decoder can make decisions on whether to emit the change based on the replication origin. We use this to filter out internal data transfers. If `citus.enable_change_data_capture` is enabled, all internal data transfers are marked with the special DoNotReplicateId replication origin by calling the `citus_internal.start_replication_origin_tracking()` UDF before writing the data. This replication origin ID is special in the sense that it does not need to be created (which prevents locking issues, especially when dropping replication origins). It is still up to output plugin to decide what to do with changes marked as DoNotReplicateId. We have very minimal control over replication commands like `CREATE_REPLICATION_SLOT`, since there are no direct hooks, and decoder names (e.g. “pgoutput”) are typically hard-coded in the client. The only method we found of overriding logical decoding behaviour is to overload the output plugin name in the dynamic library path. For specific output plugins, we supply a wrapper .so that has the same name as the existing output plugin, but in a separate directory in the PostgreSQL installation folder, and we automatically prefix the `dynamic_library_path` with this folder such that PostgreSQL will load our wrapper. The wrapper internally loads the original output plugin, and calls the original output plugin functions after applying two transformations: - Shard OIDS are mapped to distributed table OIDS - Changes marked with DoNotReplicateId are skipped Mapping the shard OIDs to distributed table OIDs not only makes the output understandable for users, but also simplifies our implementation of the CREATE PUBLICATION command, which is used to configure the pgoutput plugin (used by logical replication). We create the same publication on all nodes using the same distributed table names. Since the original pgoutput plugin only sees changes to distributed tables, it can relate those to the set of distributed tables in the publication. We have to build a .so for each wrapper separately. We currently build wrappers for [pgoutput and wal2json](https://github.com/citusdata/citus/blob/main/src/backend/distributed/cdc/Makefile#L8). This approach fulfills our main requirements, though we currently have a small correctness issue. Logical decoding always deals with a situation in the past, and to do so they build a historical snapshot of the PostgreSQL catalogs. Tables may have been renamed or dropped since the change happened, but the historical snapshot shows the schema as it was at the time of the change. However, we cannot build a historical snapshot of the Citus catalogs, and we therefore rely on the present values. The main issue that can arise is that the shard may have been dropped, in which case the change might be emitted using its original shard name, since it’s not recognized as a shard name. In many cases, this issue is avoided by caching the Citus catalogs. An open issue with CDC is that there is no good way to get a consistent snapshot followed by a change stream that starts from the snapshot. One way to resolve this is to allow reading only from local shards using an exported snapshot. That way, clients can create a replication slot and export a snapshot from each node, pull a subset of the data from each node using the snapshots, and then start replication on each node from the snapshot LSN. ## CDC ordering When you have a multi-node cluster, clients should connect to each node and combine the changes. It is important to note that there are no guarantees with regard to when and in what order changes will be emitted between nodes. It is especially important to understand that changes cannot be reordered (e.g. based on timestamp or transaction ID), because only the node-level order is correct. The lack of distributed snapshot isolation in Citus means that changes can be interleaved (a happens before b on node 1, b happens before a on node 2). The node-level decoder output will reflect that as it happened. _Do not reorder changes based on timestamp or distributed transaction ID or anything that is not guaranteed to preserve node-level order. It is never correct._ # Global PID The global PID (gpid) is used to give each client connection to the cluster a unique process identifier, and to understand which internal connections belong to a specific client connection. A gpid consists of the combination of the node ID and the PID of the coordinating process (i.e. the process serving a client connection). It can be seen in various monitoring views: ```sql SELECT citus_backend_gpid(); SELECT * FROM citus_stat_activity; SELECT * FROM citus_lock_waits; SELECT * FROM citus_locks; ``` The gpid is passed over internal connections via the application_name setting. This is one of the few settings that pgbouncer automatically propagates when reusing a server connection for a different client connection. Hence, gpids are robust to having pgbouncers in between nodes, but it means that internal sessions might switch between gpids. Additional details: Monitor distributed Postgres activity with citus_stat_activity & citus_lock_waits (citusdata.com) # Function call delegation One of the downsides of multi-statement transactions in a distributed database is the extra network round trips involved in each individual statement, even when each statement goes to the same worker node. In Citus this can be solved by marking a function or stored procedure as distributed. A distributed function has a distribution argument and can be co-located with distributed tables. It can be created using: ```sql SELECT create_distributed_function('delivery(int,int)', '$1'); ``` When a distributed function is called, the argument is treated as a distribution column filter on a co-located distributed table and delegated to the worker node that stores the corresponding shards. Ideally, every statement in the function uses the distribution argument as a distribution column filter and only accesses co-located tables, such that the transaction remains local to the worker node. Otherwise, the worker assumes the role of coordinator and performs a distributed transaction. Function call delegation is especially useful in multi-tenant applications that involve complex transactions, as those transactions can be handled in a single network round-trip and with almost no overhead on the coordinator. Citus coordinator delegates stored procedure call to worker nodes We’ve implemented function call delegation through early-stage checks in the planner hook and the utility hook (for CALL, in case of procedures). If the query matches the simple form shown in the figure above, and the function is marked as distributed, then the function call will be propagated to the right node based on the sharding metadata of a co-located table. On the target node, the function is executed as usual using the distributed tables (shell tables + metadata) on that node. The target node will hopefully find that most of the queries are on local shards and only use local execution and take advantage of _local plan caching_. # Query from any node Some Citus users have remarkably high query throughputs (>500k/s). A single-coordinator architecture could sometimes become a bottleneck for scaling such applications. To avoid that, Citus supports querying the database from any node in the cluster. In the past, this feature was referred to as Citus MX. We currently refer that as Query From Any Node. The idea is simple: Synchronize all the metadata (including the shell tables and pg_dist_XXX tables) on all the nodes. We do this for all DDL commands as well as when a new node is added to the cluster. In essence, all the nodes in the cluster always have the same metadata through 2PC. Once the metadata is synced, then each node can act as the coordinator, capable of doing the distributed query processing. We also provide some monitoring tools such that from the user’s perspective, it should not matter which node the client is connected to. The user should be able to monitor / cancel all the activity in the cluster, using the infrastructure described here: https://www.citusdata.com/blog/2022/07/21/citus-stat-activity-views-for-distributed-postgres/ One of the challenges with query from any node is the total number of connections in the cluster. In a single coordinator world, only the coordinator establishes connections per node. Now, each node connects to each other. Hence, the user should adjust the connection related settings, see here for details: https://www.citusdata.com/updates/v11-0/#connection-management By default, Citus hides all the shards from the applications because it had confused many users: https://www.citusdata.com/updates/v11-0/#shard-visibility We do it in a slightly hacky way. Read ` HideShardsFromSomeApplications ()` C function for the details. Another important piece of query from any node is that the managed service should provide a single connection string and do the load balancing for the user. It is impractical to have multiple connection strings to the database from any application’s perspective. As of writing this document (Sept. 2023), the managed service did not provide this infrastructure. Another future improvement area for query from any node is the “smart client”. A smart “pgbouncer” type of client might be able to route the client queries to the worker node with the relevant data. This could eliminate the need for additional query routing in case the query does not hit the worker node with the relevant data. Another future improvement is to allow running DDLs from any node. Currently, DDLs (including ALTER TABLE, create_distributed_table etc) all should go through the coordinator. ## Why didn’t we have dedicated Query Nodes and Data Nodes? Some distributed databases distinguish the Query Nodes and Data Nodes. As the names imply, Query Nodes would only do the query processing, whereas Data Nodes would only hold the data. In Citus, we decided not to follow that route, mostly because our initial benchmarks showed that combined nodes performed better in terms of price/performance. Still, some people argued that it might be better to have different classes of nodes such that they can be tuned /scaled-up-out differently based on the load for a given application. Dedicated query nodes benchmarks If this discussion comes up again, we suggest running some more benchmarks and ensuring the performance characteristics do not change dramatically. We do not foresee any architectural problems with that. It mostly comes down to price, performance, and product discussions. Note that you can quickly test this by disallowing certain nodes to have shards on them. You should also consider whether reference tables should still be present on query nodes, and whether there are any behavioural differences between query nodes and the coordinator. ## Shard visibility Shards live in the same schema as the distributed table they belong to, so you might expect to see them when connecting to a worker node and running `\d`. While this was previously the case, it caused confusion among users and also breaks tools like `pg_dump`. Therefore, we now aggressively hide the shards by default from any query on `pg_class`. We do this by injecting a `relation_is_a_known_shard(oid)` filter in the query tree via the planner hook when we encounter a RangeTblEntry for `pg_class`. The fact that shards are hidden from `pg_class` does not affect queries on the shards, since PostgreSQL internals will not go through the query planner when accessing `pg_class`. Shards can be revealed via two settings: - `citus.override_shard_visibility = off` disables shard hiding entirely - `citus.show_shards_for_app_name_prefixes`= 'pgAdmin,psql'` disables shard hiding only for specific application_name values, by prefix ## Statistic tracking Statistic views defined by Postgres already work well for one Citus node, like `pg_stat_database`, `pg_stat_activity`, `pg_stat_statements`, etc. And for some of them, we even provide wrapper views in Citus to have a global (i.e., cluster-wide) view of the statistics, like `citus_stat_activity`. And beside these, Citus itself also provides some additional statistic views to track the Citus-specific activities. Note that the way we collect statastics for each is quite different. - `citus_stat_tenants` (needs documentation) - `citus_stat_statements` (needs documentation) - `citus_stat_counters` ### Citus stat counters Citus keeps track of several stat counters and exposes them via the `citus_stat_counters` view. The counters are tracked once `citus.enable_stat_counters` is set to true. Also, `citus_stat_counters_reset()` can be used to reset the counters for the given database if a database id different than 0 (default, InvalidOid) is provided, otherwise, it resets the counters for the current database. Details about the implementation and its caveats can be found in the header comment of [stat_counters.c](/src/backend/distributed/stats/stat_counters.c). However, at the high level; 1. We allocate a shared memory array of length `MaxBackends` so that each backend has its own counter slot to reduce the contention while incrementing the counters at the runtime. 2. We also allocate a shared hash, whose entries correspond to individual databases. Then, when a backend exits, it first aggregates its counters to the relevant entry in the shared hash, and then it resets its own counters because the same counter slot might be reused by another backend later. Note that today we use the regular shared hash table API (`ShmemInitHash()`) to do this, but we should consider using `dshash_table()` once using many databases with Citus becomes "practically" possible because the performance of the regular shared hash table API is supposed to degrade when the number of entries in the hash table becomes much larger than the `max_size` parameter provided when creating the shared hash table. 3. So, when `citus_stat_counters` is queried, we first aggregate the counters from the shared memory array and then we add this with the counters aggregated so far in the relevant shared hash entry for the database. This means that if we weren't aggregating the counters in the shared hash when exiting, counters seen in `citus_stat_counters` could drift backwards in time. Note that `citus_stat_counters` might observe the counters for a backend twice or perhaps unsee it if the backend was concurrently exiting. However, the next call to `citus_stat_counters` will see the correct values for the counters if the same situation doesn't happen due to another backend that is exiting concurrently, so we can live with that for now. However, if one day we think that this is very undesirable, we can enforce blocking behavior between the whole period of `citus_stat_counters` queries and saving the counters in the shared hash entry. However, that will also mean that exiting backends will have to wait for any active `citus_stat_counters` queries to finish, so this needs to be carefully considered. 4. Finally, when `citus_stat_counters_reset()` is called, we reset the shared hash entry for the relevant database and also reset the relevant slots in the shared memory array for the provided database. Note that there is chance that `citus_stat_counters_reset()` might partially fail to reset the counters for of a backend slot under some rare circumstances, but this should be very rare and we choose to ignore that for the sake of lock-free counter increments. 5. Also, today neither of `citus_stat_counters` nor `citus_stat_counters_reset()` explicitly exclude the backend slots that belong to exited backends during their operations. Instead, they consider any "not unused" backend slots where the relevant `PGPROC` points to a valid database oid, which doesn't guarantee that the backend slot is actively used. However, in practice, this is not a problem for neither of these operations due to the reasons mentioned in the relevant functions. However, if we ever decide that the way that they operate slow down these operations, we can consider explicitly excluding the exited backend slots by checking (Citus) `BackendData`'s `activeBackend` field for the backend slots. 6. As of today, we don't persist stat counters on server shutdown. Although it seems quite straightforward to do so, we skipped doing that at v1. Once we decide to persist the counters, one can check the relevant functions that we have for `citus_stat_statements`, namely, `CitusQueryStatsShmemShutdown()` and `CitusQueryStatsShmemStartup()`. And since it has been quite a long time since we wrote these two functions, we should also make sure to check `pgstat_write_statsfile()` and `pgstat_read_statsfile()` in Postgres to double check if we're missing anything -it seems we have a few-. 7. Note that today we don't evict the entries of the said hash table that point to dropped databases because the wrapper view anyway filters them out (thanks to LEFT JOIN) and we don't expect a performance hit when reading from / writing to the hash table because of that unless users have a lot of databases that are dropped and recreated frequently. If one day we think that this is a problem, then we can let Citus maintenance daemon to do that for us periodically. The reason why we don't just use a shared hash table for the counters is that it could be more expensive to do hash lookups for each increment. Even more importantly, using a single counter slot for all the backends that are connected to the same database could lead to contention because that definitely requires a lock to be taken or the contended usage of atomic integers etc.. However, incrementing a counter in today's implementation doesn't require acquiring any sort of locks. Also, as of writing section, it seems quite likely that Postgres will expose their Cumulative Statistics infra starting with Postgres 18, see https://github.com/postgres/postgres/commit/7949d9594582ab49dee221e1db1aa5401ace49d4. So, once this happens, we can also consider using the same infra to track Citus stat counters. However, we can only do that once we drop support for Postgres versions older than 18. ### A side note on query stat counters Initially, we were thinking of tracking query stat counters at the very end of the planner, namely, via `FinalizePlan()` function. However, that would mean not tracking the execution of the prepared statements because a prepared statement is not planned again once its plan is cached. To give a bit more details, query plan for a prepared statement is typically cached when the same prepared statement is executed five times by Postgres (hard-coded value). Even further, a plan may even be cached after the first time it's executed if it's straightforward enough (e.g. when it doesn't have any parameters). For this reason, we track the query stat counters at appropriate places within the CustomScan implementations provided by Citus for adaptive executor and non-pushable insert-select / merge executors. ================================================ FILE: src/backend/distributed/cdc/Makefile ================================================ citus_top_builddir = ../../../.. include $(citus_top_builddir)/Makefile.global citus_subdir = src/backend/distributed/cdc SRC_DIR = $(citus_abs_top_srcdir)/$(citus_subdir) #List of supported based decoders. Add new decoders here. cdc_base_decoders :=pgoutput wal2json all: build-cdc-decoders copy-decoder-files-to-build-dir: $(eval DECODER_BUILD_DIR=build-cdc-$(DECODER)) mkdir -p $(DECODER_BUILD_DIR) @for file in $(SRC_DIR)/*.c $(SRC_DIR)/*.h; do \ if [ -f $$file ]; then \ if [ -f $(DECODER_BUILD_DIR)/$$(basename $$file) ]; then \ if ! diff -q $$file $(DECODER_BUILD_DIR)/$$(basename $$file); then \ cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \ fi \ else \ cp $$file $(DECODER_BUILD_DIR)/$$(basename $$file); \ fi \ fi \ done cp $(SRC_DIR)/Makefile.decoder $(DECODER_BUILD_DIR)/Makefile build-cdc-decoders: $(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) build-cdc-decoder;) install-cdc-decoders: $(foreach base_decoder,$(cdc_base_decoders),$(MAKE) DECODER=$(base_decoder) -C build-cdc-$(base_decoder) install;) clean-cdc-decoders: $(foreach base_decoder,$(cdc_base_decoders),rm -rf build-cdc-$(base_decoder);) build-cdc-decoder: $(MAKE) DECODER=$(DECODER) copy-decoder-files-to-build-dir $(MAKE) DECODER=$(DECODER) -C build-cdc-$(DECODER) install: install-cdc-decoders clean: clean-cdc-decoders ================================================ FILE: src/backend/distributed/cdc/Makefile.decoder ================================================ MODULE_big = citus_$(DECODER) citus_decoders_dir = $(DESTDIR)$(pkglibdir)/citus_decoders citus_top_builddir = ../../../../.. citus_subdir = src/backend/distributed/cdc/cdc_$(DECODER) OBJS += cdc_decoder.o cdc_decoder_utils.o include $(citus_top_builddir)/Makefile.global override CFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include override CPPFLAGS += -DDECODER=\"$(DECODER)\" -I$(citus_abs_top_srcdir)/include install: install-cdc install-cdc: mkdir -p '$(citus_decoders_dir)' $(INSTALL_SHLIB) citus_$(DECODER)$(DLSUFFIX) '$(citus_decoders_dir)/$(DECODER)$(DLSUFFIX)' ================================================ FILE: src/backend/distributed/cdc/cdc_decoder.c ================================================ /*------------------------------------------------------------------------- * * cdc_decoder.c * CDC Decoder plugin for Citus * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "cdc_decoder_utils.h" #include "fmgr.h" #include "access/genam.h" #include "catalog/pg_namespace.h" #include "catalog/pg_publication.h" #include "commands/extension.h" #include "common/hashfn.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/typcache.h" #include "pg_version_constants.h" PG_MODULE_MAGIC; extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); static LogicalDecodeChangeCB ouputPluginChangeCB; static void InitShardToDistributedTableMap(void); static void PublishDistributedTableChanges(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change); static bool replication_origin_filter_cb(LogicalDecodingContext *ctx, RepOriginId origin_id); static void TranslateChangesIfSchemaChanged(Relation relation, Relation targetRelation, ReorderBufferChange *change); static void TranslateAndPublishRelationForCDC(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change, Oid shardId, Oid targetRelationid); typedef struct { uint64 shardId; Oid distributedTableId; bool isReferenceTable; bool isNull; } ShardIdHashEntry; static HTAB *shardToDistributedTableMap = NULL; static void cdc_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change); /* build time macro for base decoder plugin name for CDC and Shard Split. */ #ifndef DECODER #define DECODER "pgoutput" #endif #define DECODER_INIT_FUNCTION_NAME "_PG_output_plugin_init" #define CITUS_SHARD_TRANSFER_SLOT_PREFIX "citus_shard_" #define CITUS_SHARD_TRANSFER_SLOT_PREFIX_SIZE (sizeof(CITUS_SHARD_TRANSFER_SLOT_PREFIX) - \ 1) /* * Postgres uses 'pgoutput' as default plugin for logical replication. * We want to reuse Postgres pgoutput's functionality as much as possible. * Hence we load all the functions of this plugin and override as required. */ void _PG_output_plugin_init(OutputPluginCallbacks *cb) { elog(LOG, "Initializing CDC decoder"); /* * We build custom .so files whose name matches common decoders (pgoutput, wal2json) * and place them in $libdir/citus_decoders/ such that administrators can configure * dynamic_library_path to include this directory, and users can then use the * regular decoder names when creating replications slots. * * To load the original decoder, we need to remove citus_decoders/ from the * dynamic_library_path. */ char *originalDLP = Dynamic_library_path; Dynamic_library_path = RemoveCitusDecodersFromPaths(Dynamic_library_path); LogicalOutputPluginInit plugin_init = (LogicalOutputPluginInit) (void *) load_external_function(DECODER, DECODER_INIT_FUNCTION_NAME, false, NULL); if (plugin_init == NULL) { elog(ERROR, "output plugins have to declare the _PG_output_plugin_init symbol"); } /* in case this session is used for different replication slots */ Dynamic_library_path = originalDLP; /* ask the output plugin to fill the callback struct */ plugin_init(cb); /* Initialize the Shard Id to Distributed Table id mapping hash table.*/ InitShardToDistributedTableMap(); /* actual pgoutput callback function will be called */ ouputPluginChangeCB = cb->change_cb; cb->change_cb = cdc_change_cb; cb->filter_by_origin_cb = replication_origin_filter_cb; } /* * Check if the replication slot is for Shard transfer by checking for prefix. */ inline static bool IsShardTransferSlot(char *replicationSlotName) { return strncmp(replicationSlotName, CITUS_SHARD_TRANSFER_SLOT_PREFIX, CITUS_SHARD_TRANSFER_SLOT_PREFIX_SIZE) == 0; } /* * shard_split_and_cdc_change_cb function emits the incoming tuple change * to the appropriate destination shard. */ static void cdc_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change) { /* * If Citus has not been loaded yet, pass the changes * through to the undrelying decoder plugin. */ if (!CdcCitusHasBeenLoaded()) { ouputPluginChangeCB(ctx, txn, relation, change); return; } /* check if the relation is publishable.*/ if (!is_publishable_relation(relation)) { return; } char *replicationSlotName = ctx->slot->data.name.data; if (replicationSlotName == NULL) { elog(ERROR, "Replication slot name is NULL!"); return; } /* If the slot is for internal shard operations, call the base plugin's call back. */ if (IsShardTransferSlot(replicationSlotName)) { ouputPluginChangeCB(ctx, txn, relation, change); return; } /* Transalate the changes from shard to distributes table and publish. */ PublishDistributedTableChanges(ctx, txn, relation, change); } /* * InitShardToDistributedTableMap initializes the hash table that is used to * translate the changes in the shard table to the changes in the distributed table. */ static void InitShardToDistributedTableMap() { HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(uint64); info.entrysize = sizeof(ShardIdHashEntry); info.hash = tag_hash; info.hcxt = CurrentMemoryContext; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION); shardToDistributedTableMap = hash_create("CDC Decoder translation hash table", 1024, &info, hashFlags); } /* * AddShardIdToHashTable adds the shardId to the hash table. */ static Oid AddShardIdToHashTable(uint64 shardId, ShardIdHashEntry *entry) { entry->shardId = shardId; entry->distributedTableId = CdcLookupShardRelationFromCatalog(shardId, true); entry->isReferenceTable = CdcIsReferenceTableViaCatalog(entry->distributedTableId); return entry->distributedTableId; } static Oid LookupDistributedTableIdForShardId(uint64 shardId, bool *isReferenceTable) { bool found; Oid distributedTableId = InvalidOid; ShardIdHashEntry *entry = (ShardIdHashEntry *) hash_search(shardToDistributedTableMap, &shardId, HASH_ENTER, &found); if (found) { distributedTableId = entry->distributedTableId; } else { distributedTableId = AddShardIdToHashTable(shardId, entry); } *isReferenceTable = entry->isReferenceTable; return distributedTableId; } /* * replication_origin_filter_cb call back function filters out publication of changes * originated from any other node other than the current node. This is * identified by the "origin_id" of the changes. The origin_id is set to * a non-zero value in the origin node as part of WAL replication for internal * operations like shard split/moves/create_distributed_table etc. */ static bool replication_origin_filter_cb(LogicalDecodingContext *ctx, RepOriginId origin_id) { return (origin_id != InvalidRepOriginId); } /* * This function is responsible for translating the changes in the shard table to * the changes in the shell table and publishing the changes as a change to the * distributed table so that CDD clients are not aware of the shard tables. It also * handles schema changes to the distributed table. */ static void TranslateAndPublishRelationForCDC(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change, Oid shardId, Oid targetRelationid) { /* Get the distributed table's relation for this shard.*/ Relation targetRelation = RelationIdGetRelation(targetRelationid); /* * Check if there has been a schema change (such as a dropped column), by comparing * the number of attributes in the shard table and the shell table. */ TranslateChangesIfSchemaChanged(relation, targetRelation, change); /* * Publish the change to the shard table as the change in the distributed table, * so that the CDC client can see the change in the distributed table, * instead of the shard table, by calling the pgoutput's callback function. */ ouputPluginChangeCB(ctx, txn, targetRelation, change); RelationClose(targetRelation); } /* * PublishChangesIfCdcSlot checks if the current slot is a CDC slot. If so, it publishes * the changes as the change for the distributed table instead of shard. * If not, it returns false. It also skips the Citus metadata tables. */ static void PublishDistributedTableChanges(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change) { char *shardRelationName = RelationGetRelationName(relation); /* Skip publishing CDC changes for any system relations in pg_catalog*/ if (relation->rd_rel->relnamespace == PG_CATALOG_NAMESPACE) { return; } /* Check if the relation is a distributed table by checking for shard name. */ uint64 shardId = CdcExtractShardIdFromTableName(shardRelationName, true); /* If this relation is not distributed, call the pgoutput's callback and return. */ if (shardId == INVALID_SHARD_ID) { ouputPluginChangeCB(ctx, txn, relation, change); return; } bool isReferenceTable = false; Oid distRelationId = LookupDistributedTableIdForShardId(shardId, &isReferenceTable); if (distRelationId == InvalidOid) { ouputPluginChangeCB(ctx, txn, relation, change); return; } /* Publish changes for reference table only from the coordinator node. */ if (isReferenceTable && !CdcIsCoordinator()) { return; } /* translate and publish from shard relation to distributed table relation for CDC. */ TranslateAndPublishRelationForCDC(ctx, txn, relation, change, shardId, distRelationId); } /* * GetTupleForTargetSchemaForCdc returns a heap tuple with the data from sourceRelationTuple * to match the schema in targetRelDesc. Either or both source and target relations may have * dropped columns. This function handles it by adding NULL values for dropped columns in * target relation and skipping dropped columns in source relation. It returns a heap tuple * adjusted to the current schema of the target relation. */ static HeapTuple GetTupleForTargetSchemaForCdc(HeapTuple sourceRelationTuple, TupleDesc sourceRelDesc, TupleDesc targetRelDesc) { /* Allocate memory for sourceValues and sourceNulls arrays. */ Datum *sourceValues = (Datum *) palloc0(sourceRelDesc->natts * sizeof(Datum)); bool *sourceNulls = (bool *) palloc0(sourceRelDesc->natts * sizeof(bool)); /* Deform the source tuple to sourceValues and sourceNulls arrays. */ heap_deform_tuple(sourceRelationTuple, sourceRelDesc, sourceValues, sourceNulls); /* This is the next field to Read in the source relation */ uint32 sourceIndex = 0; uint32 targetIndex = 0; /* Allocate memory for sourceValues and sourceNulls arrays. */ Datum *targetValues = (Datum *) palloc0(targetRelDesc->natts * sizeof(Datum)); bool *targetNulls = (bool *) palloc0(targetRelDesc->natts * sizeof(bool)); /* Loop through all source and target attributes one by one and handle any dropped attributes.*/ while (targetIndex < targetRelDesc->natts) { /* If this target attribute has been dropped, add a NULL attribute in targetValues and continue.*/ if (TupleDescAttr(targetRelDesc, targetIndex)->attisdropped) { Datum nullDatum = (Datum) 0; targetValues[targetIndex] = nullDatum; targetNulls[targetIndex] = true; targetIndex++; } /* If this source attribute has been dropped, just skip this source attribute.*/ else if (TupleDescAttr(sourceRelDesc, sourceIndex)->attisdropped) { sourceIndex++; continue; } /* If both source and target attributes are not dropped, add the attribute field to targetValues. */ else if (sourceIndex < sourceRelDesc->natts) { targetValues[targetIndex] = sourceValues[sourceIndex]; targetNulls[targetIndex] = sourceNulls[sourceIndex]; sourceIndex++; targetIndex++; } else { /* If there are no more source fields, add a NULL field in targetValues. */ Datum nullDatum = (Datum) 0; targetValues[targetIndex] = nullDatum; targetNulls[targetIndex] = true; targetIndex++; } } /* Form a new tuple from the target values created by the above loop. */ HeapTuple targetRelationTuple = heap_form_tuple(targetRelDesc, targetValues, targetNulls); return targetRelationTuple; } /* HasSchemaChanged function returns if there any schema changes between source and target relations.*/ static bool HasSchemaChanged(TupleDesc sourceRelationDesc, TupleDesc targetRelationDesc) { bool hasSchemaChanged = (sourceRelationDesc->natts != targetRelationDesc->natts); if (hasSchemaChanged) { return true; } for (uint32 i = 0; i < sourceRelationDesc->natts; i++) { if (TupleDescAttr(sourceRelationDesc, i)->attisdropped || TupleDescAttr(targetRelationDesc, i)->attisdropped) { hasSchemaChanged = true; break; } } return hasSchemaChanged; } /* * TranslateChangesIfSchemaChanged translates the tuples ReorderBufferChange * if there is a schema change between source and target relations. */ static void TranslateChangesIfSchemaChanged(Relation sourceRelation, Relation targetRelation, ReorderBufferChange *change) { TupleDesc sourceRelationDesc = RelationGetDescr(sourceRelation); TupleDesc targetRelationDesc = RelationGetDescr(targetRelation); /* if there are no changes between source and target relations, return. */ if (!HasSchemaChanged(sourceRelationDesc, targetRelationDesc)) { return; } #if PG_VERSION_NUM >= PG_VERSION_17 /* Check the ReorderBufferChange's action type and handle them accordingly.*/ switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { /* For insert action, only new tuple should always be translated*/ HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple = targetRelationNewTuple; break; } /* * For update changes both old and new tuples need to be translated for target relation * if the REPLICA IDENTITY is set to FULL. Otherwise, only the new tuple needs to be * translated for target relation. */ case REORDER_BUFFER_CHANGE_UPDATE: { /* For update action, new tuple should always be translated*/ /* Get the new tuple from the ReorderBufferChange, and translate it to target relation. */ HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple = targetRelationNewTuple; /* * Format oldtuple according to the target relation. If the column values of replica * identiy change, then the old tuple is non-null and needs to be formatted according * to the target relation schema. */ if (change->data.tp.oldtuple != NULL) { HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple = targetRelationOldTuple; } break; } case REORDER_BUFFER_CHANGE_DELETE: { /* For delete action, only old tuple should be translated*/ HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple = targetRelationOldTuple; break; } default: { /* Do nothing for other action types. */ break; } } #else /* Check the ReorderBufferChange's action type and handle them accordingly.*/ switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { /* For insert action, only new tuple should always be translated*/ HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple); HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple->tuple = *targetRelationNewTuple; break; } /* * For update changes both old and new tuples need to be translated for target relation * if the REPLICA IDENTITY is set to FULL. Otherwise, only the new tuple needs to be * translated for target relation. */ case REORDER_BUFFER_CHANGE_UPDATE: { /* For update action, new tuple should always be translated*/ /* Get the new tuple from the ReorderBufferChange, and translate it to target relation. */ HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple); HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple->tuple = *targetRelationNewTuple; /* * Format oldtuple according to the target relation. If the column values of replica * identiy change, then the old tuple is non-null and needs to be formatted according * to the target relation schema. */ if (change->data.tp.oldtuple != NULL) { HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple); HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple->tuple = *targetRelationOldTuple; } break; } case REORDER_BUFFER_CHANGE_DELETE: { /* For delete action, only old tuple should be translated*/ HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple); HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple->tuple = *targetRelationOldTuple; break; } default: { /* Do nothing for other action types. */ break; } } #endif } ================================================ FILE: src/backend/distributed/cdc/cdc_decoder_utils.c ================================================ /*------------------------------------------------------------------------- * * cdc_decoder_utils.c * CDC Decoder plugin utility functions for Citus * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "cdc_decoder_utils.h" #include "fmgr.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "catalog/pg_namespace.h" #include "commands/extension.h" #include "common/hashfn.h" #include "common/string.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/typcache.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/relay_utility.h" static int32 LocalGroupId = -1; static Oid PgDistLocalGroupRelationId = InvalidOid; static Oid PgDistShardRelationId = InvalidOid; static Oid PgDistShardShardidIndexId = InvalidOid; static Oid PgDistPartitionRelationId = InvalidOid; static Oid PgDistPartitionLogicalrelidIndexId = InvalidOid; static bool IsCitusExtensionLoaded = false; #define COORDINATOR_GROUP_ID 0 #define InvalidRepOriginId 0 #define Anum_pg_dist_local_groupid 1 #define GROUP_ID_UPGRADING -2 static Oid DistLocalGroupIdRelationId(void); static int32 CdcGetLocalGroupId(void); static HeapTuple CdcPgDistPartitionTupleViaCatalog(Oid relationId); /* * DistLocalGroupIdRelationId returns the relation id of the pg_dist_local_group */ static Oid DistLocalGroupIdRelationId(void) { if (PgDistLocalGroupRelationId == InvalidOid) { PgDistLocalGroupRelationId = get_relname_relid("pg_dist_local_group", PG_CATALOG_NAMESPACE); } return PgDistLocalGroupRelationId; } /* * DistShardRelationId returns the relation id of the pg_dist_shard */ static Oid DistShardRelationId(void) { if (PgDistShardRelationId == InvalidOid) { PgDistShardRelationId = get_relname_relid("pg_dist_shard", PG_CATALOG_NAMESPACE); } return PgDistShardRelationId; } /* * DistShardShardidIndexId returns the relation id of the pg_dist_shard_shardid_index */ static Oid DistShardShardidIndexId(void) { if (PgDistShardShardidIndexId == InvalidOid) { PgDistShardShardidIndexId = get_relname_relid("pg_dist_shard_shardid_index", PG_CATALOG_NAMESPACE); } return PgDistShardShardidIndexId; } /* * DistPartitionRelationId returns the relation id of the pg_dist_partition */ static Oid DistPartitionRelationId(void) { if (PgDistPartitionRelationId == InvalidOid) { PgDistPartitionRelationId = get_relname_relid("pg_dist_partition", PG_CATALOG_NAMESPACE); } return PgDistPartitionRelationId; } static Oid DistPartitionLogicalRelidIndexId(void) { if (PgDistPartitionLogicalrelidIndexId == InvalidOid) { PgDistPartitionLogicalrelidIndexId = get_relname_relid( "pg_dist_partition_logicalrelid_index", PG_CATALOG_NAMESPACE); } return PgDistPartitionLogicalrelidIndexId; } /* * CdcIsCoordinator function returns true if this node is identified as the * schema/coordinator/master node of the cluster. */ bool CdcIsCoordinator(void) { return (CdcGetLocalGroupId() == COORDINATOR_GROUP_ID); } /* * CdcCitusHasBeenLoaded function returns true if the citus extension has been loaded. */ bool CdcCitusHasBeenLoaded() { if (!IsCitusExtensionLoaded) { IsCitusExtensionLoaded = (get_extension_oid("citus", true) != InvalidOid); } return IsCitusExtensionLoaded; } /* * ExtractShardIdFromTableName tries to extract shard id from the given table name, * and returns the shard id if table name is formatted as shard name. * Else, the function returns INVALID_SHARD_ID. */ uint64 CdcExtractShardIdFromTableName(const char *tableName, bool missingOk) { char *shardIdStringEnd = NULL; /* find the last underscore and increment for shardId string */ char *shardIdString = strrchr(tableName, SHARD_NAME_SEPARATOR); if (shardIdString == NULL && !missingOk) { ereport(ERROR, (errmsg("could not extract shardId from table name \"%s\"", tableName))); } else if (shardIdString == NULL && missingOk) { return INVALID_SHARD_ID; } shardIdString++; errno = 0; uint64 shardId = strtoull(shardIdString, &shardIdStringEnd, 0); if (errno != 0 || (*shardIdStringEnd != '\0')) { if (!missingOk) { ereport(ERROR, (errmsg("could not extract shardId from table name \"%s\"", tableName))); } else { return INVALID_SHARD_ID; } } return shardId; } /* * CdcGetLocalGroupId returns the group identifier of the local node. The * function assumes that pg_dist_local_group has exactly one row and has at * least one column. Otherwise, the function errors out. */ static int32 CdcGetLocalGroupId(void) { ScanKeyData scanKey[1]; int scanKeyCount = 0; int32 groupId = 0; /* * Already set the group id, no need to read the heap again. */ if (LocalGroupId != -1) { return LocalGroupId; } Oid localGroupTableOid = DistLocalGroupIdRelationId(); if (localGroupTableOid == InvalidOid) { return 0; } Relation pgDistLocalGroupId = table_open(localGroupTableOid, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistLocalGroupId, InvalidOid, false, NULL, scanKeyCount, scanKey); TupleDesc tupleDescriptor = RelationGetDescr(pgDistLocalGroupId); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { bool isNull = false; Datum groupIdDatum = heap_getattr(heapTuple, Anum_pg_dist_local_groupid, tupleDescriptor, &isNull); groupId = DatumGetInt32(groupIdDatum); /* set the local cache variable */ LocalGroupId = groupId; } else { /* * Upgrade is happening. When upgrading postgres, pg_dist_local_group is * temporarily empty before citus_finish_pg_upgrade() finishes execution. */ groupId = GROUP_ID_UPGRADING; } systable_endscan(scanDescriptor); table_close(pgDistLocalGroupId, AccessShareLock); return groupId; } /* * CdcLookupShardRelationFromCatalog returns the logical relation oid a shard belongs to. * * Errors out if the shardId does not exist and missingOk is false. * Returns InvalidOid if the shardId does not exist and missingOk is true. */ Oid CdcLookupShardRelationFromCatalog(int64 shardId, bool missingOk) { ScanKeyData scanKey[1]; int scanKeyCount = 1; Form_pg_dist_shard shardForm = NULL; Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); Oid relationId = InvalidOid; ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistShard, DistShardShardidIndexId(), true, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple) && !missingOk) { ereport(ERROR, (errmsg("could not find valid entry for shard " UINT64_FORMAT, shardId))); } if (!HeapTupleIsValid(heapTuple)) { relationId = InvalidOid; } else { shardForm = (Form_pg_dist_shard) GETSTRUCT(heapTuple); relationId = shardForm->logicalrelid; } systable_endscan(scanDescriptor); table_close(pgDistShard, NoLock); return relationId; } /* * CdcPgDistPartitionTupleViaCatalog is a helper function that searches * pg_dist_partition for the given relationId. The caller is responsible * for ensuring that the returned heap tuple is valid before accessing * its fields. */ static HeapTuple CdcPgDistPartitionTupleViaCatalog(Oid relationId) { const int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = true; Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple partitionTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(partitionTuple)) { /* callers should have the tuple in their memory contexts */ partitionTuple = heap_copytuple(partitionTuple); } systable_endscan(scanDescriptor); table_close(pgDistPartition, AccessShareLock); return partitionTuple; } /* * CdcIsReferenceTableViaCatalog gets a relationId and returns true if the relation * is a reference table and false otherwise. */ char CdcIsReferenceTableViaCatalog(Oid relationId) { HeapTuple partitionTuple = CdcPgDistPartitionTupleViaCatalog(relationId); if (!HeapTupleIsValid(partitionTuple)) { return false; } Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(partitionTuple, tupleDescriptor, datumArray, isNullArray); if (isNullArray[Anum_pg_dist_partition_partmethod - 1] || isNullArray[Anum_pg_dist_partition_repmodel - 1]) { /* * partition method and replication model cannot be NULL, * still let's make sure */ heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return false; } Datum partitionMethodDatum = datumArray[Anum_pg_dist_partition_partmethod - 1]; char partitionMethodChar = DatumGetChar(partitionMethodDatum); Datum replicationModelDatum = datumArray[Anum_pg_dist_partition_repmodel - 1]; char replicationModelChar = DatumGetChar(replicationModelDatum); heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); /* * A table is a reference table when its partition method is 'none' * and replication model is 'two phase commit' */ return partitionMethodChar == DISTRIBUTE_BY_NONE && replicationModelChar == REPLICATION_MODEL_2PC; } /* * RemoveCitusDecodersFromPaths removes a path ending in citus_decoders * from the given input paths. */ char * RemoveCitusDecodersFromPaths(char *paths) { if (strlen(paths) == 0) { /* dynamic_library_path is empty */ return paths; } StringInfo newPaths = makeStringInfo(); char *remainingPaths = paths; for (;;) { int pathLength = 0; char *pathStart = first_path_var_separator(remainingPaths); if (pathStart == remainingPaths) { /* * This will error out in find_in_dynamic_libpath, return * original value here. */ return paths; } else if (pathStart == NULL) { /* final path */ pathLength = strlen(remainingPaths); } else { /* more paths remaining */ pathLength = pathStart - remainingPaths; } char *currentPath = palloc(pathLength + 1); strlcpy(currentPath, remainingPaths, pathLength + 1); canonicalize_path(currentPath); if (!pg_str_endswith(currentPath, "/citus_decoders")) { appendStringInfo(newPaths, "%s%s", newPaths->len > 0 ? ":" : "", currentPath); } if (remainingPaths[pathLength] == '\0') { /* end of string */ break; } remainingPaths += pathLength + 1; } return newPaths->data; } ================================================ FILE: src/backend/distributed/cdc/cdc_decoder_utils.h ================================================ /*------------------------------------------------------------------------- * * cdc_decoder_utils.h * Utility functions and declerations for cdc decoder. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_CDC_DECODER_H #define CITUS_CDC_DECODER_H #include "postgres.h" #include "c.h" #include "fmgr.h" #include "replication/logical.h" #define InvalidRepOriginId 0 #define INVALID_SHARD_ID 0 bool CdcIsCoordinator(void); uint64 CdcExtractShardIdFromTableName(const char *tableName, bool missingOk); Oid CdcLookupShardRelationFromCatalog(int64 shardId, bool missingOk); char CdcIsReferenceTableViaCatalog(Oid relationId); bool CdcCitusHasBeenLoaded(void); char * RemoveCitusDecodersFromPaths(char *paths); #endif /* CITUS_CDC_DECODER_UTILS_H */ ================================================ FILE: src/backend/distributed/citus--11.1-1.control ================================================ requires = 'citus_columnar' ================================================ FILE: src/backend/distributed/citus.control ================================================ # Citus extension comment = 'Citus distributed database' default_version = '15.0-1' module_pathname = '$libdir/citus' relocatable = false schema = pg_catalog ================================================ FILE: src/backend/distributed/clock/README.md ================================================ # Cluster Clock ### Requirement: Many distributed applications need to track the changes in the same order as they are applied on the database. The changes can be to databases or objects within them, either within a single node or across the sharded cluster. ### Definitions **Total ordering** - Every pair of change events can be placed in some order. **Causal ordering** - Only events that are causally related (an event A caused an event B) can be ordered i.e., it's only a partial order - sometimes events happen independently with no possible causal relationship, such events are treated to concurrent. **Sequential consistency** - All writes must be seen in the same order by all processes. **Causal consistency** - Causally related writes must be seen in the same order. Transactions on a single node system naturally provide a total and sequential ordering guarantees for client read and write operations as all operations are routed to the same node, but there are challenges for a multi node distributed system, such as, Citus. One possible way to totally order all the changes in the system is to timestamp all the events with a global physical clock or a centralized logical clock. Thus, observing the events in the increasing order of the timestamp will give the total ordering of events. For both the performance and cost reasons such solutions are impractical. In the absence of total ordering, a little weaker ordering is the **causal order**. Causal order is defined as a model that preserves a partial order of events in a distributed system. If an event 1. A causes another event B, every other process in the system observes the event A before observing event B. 2. Causal order is transitive: if A causes B, and B causes C, then A causes C. 3. Non causally ordered events are treated as concurrent. Causal consistency is a weak form of consistency that preserves the order of causally related operations. The causal consistency model can be refined into four session guarantees. 1. Read Your Writes: If a process performs a write, the same process later observes the result of its write. 6. Monotonic Reads: The set of writes observed (read) by a process is guaranteed to be monotonically increasing. 7. Writes Follow Reads: If some process performs a read followed by a write, and another process observes the result of the write, then it can also observe the read. 8. Monotonic Writes: If some process performs a write, followed sometime later by another write, other processes will observe them in the same order. ### Hybrid Logical Clock (HLC) HLC provides a way to get the causality relationship like logical clocks. It can also be used for backup/recovery too as the logical clock value is maintained close to the wall clock time. HLC consists of LC - Logical clock C - Counter Clock components - Unsigned 64 bit Epoch Milliseconds ( LC ) | Logical counter ( C )| |--|--| | 42 bits | 22 bits | 2^42 milliseconds - 4398046511104 milliseconds, which is ~139 years. 2^22 ticks - maximum of four million operations per millisecond. ### UDFs A new UDF `citus_get_cluster_clock`() that returns a monotonically increasing logical clock. Clock guarantees to never go back in value after restarts and makes best attempt to keep the value close to UNIX epoch time in milliseconds. A new UDF `citus_get_transaction_clock`(), when called by the user, returns the logical causal clock timestamp current transaction, Internally, this is the maximum clock among all transaction nodes, and all nodes move to the new clock. ### GUC A new GUC parameter, "**citus.enable_cluster_clock**", If clocks go bad for any reason, this serves as a safety valve to avoid the need to change the application and (re)deploy it. ### Sequence In Unix, though rare, there is a possibility of clock drifting backwards (or forward) after a restart. In such rare scenarios, we might end up with a logical clock value less than the previously used value, this violates the fundamental requirement of monotonically increasing clock. To avoid such disasters, every logical clock tick is persisted using sequences (non-transactional). After a restart, the persisted sequence value is read and clock starts from that value, which will ensure that system starts the clock from where we left off. ### Psuedo code WC - Current Wall Clock in milliseconds HLC - Current Hybrid Logical Clock in shared memory MAX_COUNTER - Four million /* Tick the clock by 1 */ IncrementClusterClock() { /* It's the counter that always ticks, once it reaches the maximum, reset the counter to 1 and increment the logical clock. */ if (HLC.C == MAX_COUNTER) { HLC.LC++; HLC.C = 0; return; } HLC.C++; } /* Tick for each event, must increase monotonically */ GetNextNodeClockValue() { IncrementClusterClock(HLC); /* From the incremented clock and current wall clock, pick which ever is highest */ NextClock = MAX(HLC, WC); /* Save the NextClock value in both the shared memory and sequence */ HLC = NextClock; SETVAL(pg_dist_clock_logical_seq, HLC); } /* Returns true if the clock1 is after clock2 */ IsEventAfter(HLC1, HLC2) { IF (HLC1.LC != HLC2.LC) return (HLC1.LC > HLC2.LC); ELSE return (HLC1.C > HLC2.C); } /* Simply returns the highest node clock value among all nodes */ GetHighestClockInTransaction() { For each node { NodeClock[N] = GetNextNodeClockValue(); } /* Return the highest clock value of all the nodes */ return MAX(NodeClock[N]); } /* Adjust the local shared memory clock to the received value (RHLC) from the remote node */ AdjustClock(RHLC) { /* local clock is ahead or equal, do nothing */ IF (HLC >= RHLC) { return; } /* Save the remote clockvalue in both the shared memory and sequence */ HLC = RHLC; SETVAL(pg_dist_clock_logical_seq, HLC); } /* All the nodes will adjust their clocks to the highest of the newly negotiated clock */ AdjustClocksToTransactionHighest(HLC) { For each node { SendCommand ("AdjustClock(HLC)"); } } /* When citus_get_transaction_clock() UDF is invoked */ PrepareAndSetTransactionClock() { /* Pick the highest logical clock value among all transaction-nodes */ txnCLock = GetHighestClockInTransaction() /* Adjust all the nodes with the new clock value */ AdjustClocksToTransactionHighest(txnCLock ) return txnClock; } /* Initialize the clock value to the highest clock persisted in sequence */ InitClockAtBoot() { /* Start with the current wall clock */ HLC = WC; IF (SEQUENCE == 1) /* clock never ticked on this node, start with the wall clock. */ return; /* get the most recent clock ever used from disk */ persistedClock = NEXT_VAL(pg_dist_clock_logical_seq...) /* Start the clock with persisted value */ AdjustLocalClock(persistedClock); } } #### Usage **Step 1** In the application, track every change of a transaction along with the unique transaction ID by calling UDF `get_current_transaction_id`() INSERT INTO track_table SET TransactionId = get_current_transaction_id(), operation = , row_key = <>, ....; **Step 2** As the transaction is about to end, and before the COMMIT, capture the causal clock timestamp along with the transaction ID in a table INSERT INTO transaction_commit_clock (TransactionId, CommitClock, timestamp) SELECT citus_get_transaction_clock(), get_current_transaction_id(), now() **Step 3** How to get all the events in the causal order? SELECT tt.row_key, tt.operation FROM track_table tt, transaction_commit_clock cc WHERE tt.TransactionId = cc.TransactionId ORDER BY cc.CommitClock Events for an object SELECT tt.row_key, tt.operation FROM track_table tt, transaction_commit_clock cc WHERE tt.TransactionId = cc.TransactionId and row_key = $1 ORDER BY cc.CommitClock Events in the last one hour SELECT tt.row_key, tt.operation FROM track_table tt, transaction_commit_clockcc WHERE cc.timestamp >= now() - interval '1 hour' and tt.TransactionId = cc.TransactionId **Note**: In Citus we use 2PC, if any node goes down after the PREPARE and before the COMMIT, we might have changes partially committed. Citus tracks such transactions in **pg_dist_transaction** and eventually will be committed when the node becomes healthy, but when we track change-data from committed transactions of **transaction_commit_clock** we will miss the changes from a bad node. To address this issue, proposal is to have a new UDF #TBD, that freezes the clock and ensures that all the 2PCs are fully complete (i.e., **pg_dist_transaction** should be empty) and return the highest clock used. All transactions in `transaction_commit_clock` with timestamp below this returned clock are visible to the application. The exact nuances, such as frequency of calling such UDF, are still TBD. Caveat is, if the node and the 2PC takes long to fully recover, the visibility of the committed transactions might stall. ### Catalog pruning The data in **transaction_commit_clock** should be ephemeral data i.e., eventually rows have to automatically be deleted. Users can install a pg_cron job to prune the catalog regularly. delete from transaction_commit_clock where timestamp < now() - interval '7 days' ### Limitations of Citus Using this transaction commit clock ordering to build a secondary, that's a mirror copy of the original, may not be feasible at this time for the following reasons. Given that there is no well-defined order between concurrent distributed transactions in Citus, we cannot retroactively apply a transaction-order that leads to an exact replica of the primary unless we preserve the original object-level ordering as it happened on individual nodes. For instance, if a multi-shard insert (transaction A) happens concurrently with a multi-shard update (transaction B) and the WHERE clause of the update matches inserted rows in multiple shards, we could have a scenario in which only a subset of the inserted rows gets updated. Effectively, transaction A might happen before transaction B on node 1, while transaction B happens before transaction A on node 2. While unfortunate, we cannot simply claim changes made by transaction A happened first based on commit timestamps, because that would lead us reorder changes to the same object ID on node 2, which might lead to a different outcome when replayed. In such scenario, even if we use causal commit clock to order changes. It is essential that the order of modifications to an object matches the original order. Otherwise, you could have above scenarios where an insert happens before an update in the primary cluster, but the update happens before the insert. Replaying the changes would then lead to a different database. In absence of a coherent transaction-ordering semantics in distributed cluster, best we can do is ensure that changes to the same object are in the correct order and ensure exactly once delivery (correct pagination). ================================================ FILE: src/backend/distributed/clock/causal_clock.c ================================================ /*------------------------------------------------------------------------- * causal_clock.c * * Core function defintions to implement hybrid logical clock. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "catalog/namespace.h" #include "commands/extension.h" #include "commands/sequence.h" #include "executor/spi.h" #include "nodes/pg_list.h" #include "postmaster/postmaster.h" #include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/s_lock.h" #include "storage/shmem.h" #include "storage/spin.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/numeric.h" #include "utils/typcache.h" #include "distributed/causal_clock.h" #include "distributed/citus_safe_lib.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/placement_connection.h" #include "distributed/remote_commands.h" #define SAVE_AND_PERSIST(c) \ do { \ Oid savedUserId = InvalidOid; \ int savedSecurityContext = 0; \ LogicalClockShmem->clusterClockValue = *(c); \ GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); \ SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); \ DirectFunctionCall2(setval_oid, \ ObjectIdGetDatum(DistClockLogicalSequenceId()), \ Int64GetDatum((c)->logical)); \ SetUserIdAndSecContext(savedUserId, savedSecurityContext); \ } while (0) PG_FUNCTION_INFO_V1(citus_get_node_clock); PG_FUNCTION_INFO_V1(citus_internal_adjust_local_clock_to_remote); PG_FUNCTION_INFO_V1(citus_is_clock_after); PG_FUNCTION_INFO_V1(citus_get_transaction_clock); /* * Current state of the logical clock */ typedef enum ClockState { CLOCKSTATE_INITIALIZED, CLOCKSTATE_UNINITIALIZED } ClockState; /* * Holds the cluster clock variables in shared memory. */ typedef struct LogicalClockShmemData { NamedLWLockTranche namedLockTranche; LWLock clockLock; /* Current logical clock value of this node */ ClusterClock clusterClockValue; /* Tracks initialization at boot */ ClockState clockInitialized; } LogicalClockShmemData; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static LogicalClockShmemData *LogicalClockShmem = NULL; static void AdjustLocalClock(ClusterClock *remoteClock); static void GetNextNodeClockValue(ClusterClock *nextClusterClockValue); static ClusterClock * GetHighestClockInTransaction(List *nodeConnectionList); static void AdjustClocksToTransactionHighest(List *nodeConnectionList, ClusterClock *transactionClockValue); static void InitClockAtFirstUse(void); static void IncrementClusterClock(ClusterClock *clusterClock); static ClusterClock * LargerClock(ClusterClock *clock1, ClusterClock *clock2); static ClusterClock * PrepareAndSetTransactionClock(void); bool EnableClusterClock = true; /* * GetEpochTimeAsClock returns the epoch value milliseconds used as logical * value in ClusterClock. */ ClusterClock * GetEpochTimeAsClock(void) { struct timeval tp = { 0 }; gettimeofday(&tp, NULL); uint64 result = (uint64) (tp.tv_sec) * 1000; result = result + (uint64) (tp.tv_usec) / 1000; ClusterClock *epochClock = (ClusterClock *) palloc(sizeof(ClusterClock)); epochClock->logical = result; epochClock->counter = 0; return epochClock; } /* * LogicalClockShmemSize returns the size that should be allocated * in the shared memory for logical clock management. */ size_t LogicalClockShmemSize(void) { Size size = 0; size = add_size(size, sizeof(LogicalClockShmemData)); return size; } /* * InitializeClusterClockMem reserves shared-memory space needed to * store LogicalClockShmemData, and sets the hook for initialization * of the same. */ void InitializeClusterClockMem(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = LogicalClockShmemInit; } /* * LogicalClockShmemInit Allocates and initializes shared memory for * cluster clock related variables. */ void LogicalClockShmemInit(void) { bool alreadyInitialized = false; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); LogicalClockShmem = (LogicalClockShmemData *) ShmemInitStruct("Logical Clock Shmem", LogicalClockShmemSize(), &alreadyInitialized); if (!alreadyInitialized) { /* A zero value indicates that the clock is not adjusted yet */ memset(&LogicalClockShmem->clusterClockValue, 0, sizeof(ClusterClock)); LogicalClockShmem->namedLockTranche.trancheName = "Cluster Clock Setup Tranche"; LogicalClockShmem->namedLockTranche.trancheId = LWLockNewTrancheId(); LWLockRegisterTranche(LogicalClockShmem->namedLockTranche.trancheId, LogicalClockShmem->namedLockTranche.trancheName); LWLockInitialize(&LogicalClockShmem->clockLock, LogicalClockShmem->namedLockTranche.trancheId); LogicalClockShmem->clockInitialized = CLOCKSTATE_UNINITIALIZED; } LWLockRelease(AddinShmemInitLock); if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * IncrementClusterClock increments the ClusterClock by 1. */ static void IncrementClusterClock(ClusterClock *clusterClock) { /* * It's the counter that always ticks, once it reaches the maximum, reset * the counter to 1 and increment the logical clock. */ if (clusterClock->counter == MAX_COUNTER) { clusterClock->logical++; clusterClock->counter = 0; return; } clusterClock->counter++; } /* * LargerClock compares two ClusterClock(s) and returns pointer to the larger one. * Note: If equal or one of the clock is NULL, non NULL clock is copied. */ static ClusterClock * LargerClock(ClusterClock *clock1, ClusterClock *clock2) { /* Check if one of the paramater is NULL */ if (!clock1 || !clock2) { Assert(clock1 || clock2); return (!clock1 ? clock2 : clock1); } if (cluster_clock_cmp_internal(clock1, clock2) > 0) { return clock1; } else { return clock2; } } /* * GetNextNodeClock implements the internal guts of the UDF citus_get_node_clock() */ static void GetNextNodeClockValue(ClusterClock *nextClusterClockValue) { static bool isClockInitChecked = false; /* serves as a local cache */ ClusterClock *epochValue = GetEpochTimeAsClock(); /* If this backend already checked for initialization, skip it */ if (!isClockInitChecked) { InitClockAtFirstUse(); /* We reach here only if CLOCKSTATE_INITIALIZED, all other cases error out. */ isClockInitChecked = true; } LWLockAcquire(&LogicalClockShmem->clockLock, LW_EXCLUSIVE); Assert(LogicalClockShmem->clockInitialized == CLOCKSTATE_INITIALIZED); /* Tick the clock */ IncrementClusterClock(&LogicalClockShmem->clusterClockValue); /* Pick the larger of the two, wallclock and logical clock. */ ClusterClock *clockValue = LargerClock(&LogicalClockShmem->clusterClockValue, epochValue); /* * Save the returned value in both the shared memory and sequences. */ SAVE_AND_PERSIST(clockValue); LWLockRelease(&LogicalClockShmem->clockLock); /* Return the clock */ *nextClusterClockValue = *clockValue; } /* * AdjustLocalClock Adjusts the local shared memory clock to the * received value from the remote node. */ void AdjustLocalClock(ClusterClock *remoteClock) { LWLockAcquire(&LogicalClockShmem->clockLock, LW_EXCLUSIVE); ClusterClock *localClock = &LogicalClockShmem->clusterClockValue; /* local clock is ahead or equal, do nothing */ if (cluster_clock_cmp_internal(localClock, remoteClock) >= 0) { LWLockRelease(&LogicalClockShmem->clockLock); return; } SAVE_AND_PERSIST(remoteClock); LWLockRelease(&LogicalClockShmem->clockLock); ereport(DEBUG1, (errmsg("adjusted to remote clock: " "", remoteClock->logical, remoteClock->counter))); } /* * GetHighestClockInTransaction takes the connection list of participating nodes in the * current transaction and polls the logical clock value of all the nodes. Returns the * highest logical clock value of all the nodes in the current distributed transaction, * which may be used as commit order for individual objects in the transaction. */ static ClusterClock * GetHighestClockInTransaction(List *nodeConnectionList) { MultiConnection *connection = NULL; foreach_declared_ptr(connection, nodeConnectionList) { int querySent = SendRemoteCommand(connection, "SELECT citus_get_node_clock();"); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } /* Check for the current node */ ClusterClock *globalClockValue = (ClusterClock *) palloc(sizeof(ClusterClock)); GetNextNodeClockValue(globalClockValue); ereport(DEBUG1, (errmsg("node(%u) transaction clock %lu:%u", PostPortNumber, globalClockValue->logical, globalClockValue->counter))); /* fetch the results and pick the highest clock value of all the nodes */ foreach_declared_ptr(connection, nodeConnectionList) { bool raiseInterrupts = true; if (PQstatus(connection->pgConn) != CONNECTION_OK) { ereport(ERROR, (errmsg("connection to %s:%d failed when " "fetching logical clock value", connection->hostname, connection->port))); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } ClusterClock *nodeClockValue = ParseClusterClockPGresult(result, 0, 0); ereport(DEBUG1, (errmsg("node(%u) transaction clock %lu:%u", connection->port, nodeClockValue->logical, nodeClockValue->counter))); globalClockValue = LargerClock(globalClockValue, nodeClockValue); PQclear(result); ForgetResults(connection); } ereport(DEBUG1, (errmsg("final global transaction clock %lu:%u", globalClockValue->logical, globalClockValue->counter))); return globalClockValue; } /* * AdjustClocksToTransactionHighest Sets the clock value of all the nodes, participated * in the PREPARE of the transaction, to the highest clock value of all the nodes. */ static void AdjustClocksToTransactionHighest(List *nodeConnectionList, ClusterClock *transactionClockValue) { StringInfo queryToSend = makeStringInfo(); /* Set the clock value on participating worker nodes */ appendStringInfo(queryToSend, "SELECT citus_internal.adjust_local_clock_to_remote" "('(%lu, %u)'::pg_catalog.cluster_clock);", transactionClockValue->logical, transactionClockValue->counter); ExecuteRemoteCommandInConnectionList(nodeConnectionList, queryToSend->data); AdjustLocalClock(transactionClockValue); } /* * PrepareAndSetTransactionClock polls all the transaction-nodes for their respective clocks, * picks the highest clock and returns it via UDF citus_get_transaction_clock. All the nodes * will now move to this newly negotiated clock. */ static ClusterClock * PrepareAndSetTransactionClock(void) { if (!EnableClusterClock) { /* citus.enable_cluster_clock is false */ ereport(WARNING, (errmsg("GUC enable_cluster_clock is off"))); return NULL; } dlist_iter iter; List *transactionNodeList = NIL; List *nodeList = NIL; /* Prepare the connection list */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); WorkerNode *workerNode = FindWorkerNode(connection->hostname, connection->port); if (!workerNode) { ereport(WARNING, errmsg("Worker node is missing")); continue; } /* Skip the node if we already in the list */ if (list_member_int(nodeList, workerNode->groupId)) { continue; } RemoteTransaction *transaction = &connection->remoteTransaction; Assert(transaction->transactionState != REMOTE_TRANS_NOT_STARTED); /* Skip a transaction that failed */ if (transaction->transactionFailed) { continue; } nodeList = lappend_int(nodeList, workerNode->groupId); transactionNodeList = lappend(transactionNodeList, connection); } /* Pick the highest logical clock value among all transaction-nodes */ ClusterClock *transactionClockValue = GetHighestClockInTransaction(transactionNodeList); /* Adjust all the nodes with the new clock value */ AdjustClocksToTransactionHighest(transactionNodeList, transactionClockValue); return transactionClockValue; } /* * InitClockAtFirstUse Initializes the shared memory clock value to the highest clock * persisted. This will protect from any clock drifts. */ static void InitClockAtFirstUse(void) { LWLockAcquire(&LogicalClockShmem->clockLock, LW_EXCLUSIVE); /* Avoid repeated and parallel initialization */ if (LogicalClockShmem->clockInitialized == CLOCKSTATE_INITIALIZED) { LWLockRelease(&LogicalClockShmem->clockLock); return; } if (DistClockLogicalSequenceId() == InvalidOid) { ereport(ERROR, (errmsg("Clock related sequence is missing"))); } /* Start with the wall clock value */ ClusterClock *epochValue = GetEpochTimeAsClock(); LogicalClockShmem->clusterClockValue = *epochValue; /* Retrieve the highest clock value persisted in the sequence */ ClusterClock persistedMaxClock = { 0 }; /* * We will get one more than the persisted value, but that's harmless and * also very _crucial_ in below scenarios * * 1) As sequences are not transactional, this will protect us from crashes * after the logical increment and before the counter increment. * * 2) If a clock drifts backwards, we should always start one clock above * the previous value, though we are not persisting the counter as the * logical value supersedes the counter, a simple increment of it will * protect us. * * Note: The first (and every 32nd) call to nextval() consumes 32 values in the * WAL. This is an optimization that postgres does to only have to write a WAL * entry every 32 invocations. Normally this is harmless, however, if the database * gets in a crashloop it could outrun the wall clock, if the database crashes at * a higher rate than once every 32 seconds. * */ Oid saveUserId = InvalidOid; int savedSecurityCtx = 0; GetUserIdAndSecContext(&saveUserId, &savedSecurityCtx); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); persistedMaxClock.logical = DirectFunctionCall1(nextval_oid, ObjectIdGetDatum(DistClockLogicalSequenceId())); SetUserIdAndSecContext(saveUserId, savedSecurityCtx); /* * Sequence 1 indicates no prior clock timestamps on this server, retain * the wall clock i.e. no adjustment needed. */ if (persistedMaxClock.logical != 1) { ereport(DEBUG1, (errmsg("adjusting the clock with persisted value: " "", persistedMaxClock.logical, persistedMaxClock.counter))); /* * Adjust the local clock according to the most recent * clock stamp value persisted in the catalog. */ if (cluster_clock_cmp_internal(&persistedMaxClock, epochValue) > 0) { SAVE_AND_PERSIST(&persistedMaxClock); ereport(NOTICE, (errmsg("clock drifted backwards, adjusted to: " "", persistedMaxClock.logical, persistedMaxClock.counter))); } } LogicalClockShmem->clockInitialized = CLOCKSTATE_INITIALIZED; LWLockRelease(&LogicalClockShmem->clockLock); } /* * citus_get_node_clock() is an UDF that returns a monotonically increasing * logical clock. Clock guarantees to never go back in value after restarts, and * makes best attempt to keep the value close to unix epoch time in milliseconds. */ Datum citus_get_node_clock(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ClusterClock *nodeClockValue = (ClusterClock *) palloc(sizeof(ClusterClock)); GetNextNodeClockValue(nodeClockValue); PG_RETURN_POINTER(nodeClockValue); } /* * citus_internal_adjust_local_clock_to_remote is an internal UDF to adjust * the local clock to the highest in the cluster. */ Datum citus_internal_adjust_local_clock_to_remote(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ClusterClock *remoteClock = (ClusterClock *) PG_GETARG_POINTER(0); AdjustLocalClock(remoteClock); PG_RETURN_VOID(); } /* * citus_is_clock_after is an UDF that accepts logical clock timestamps of * two causally related events and returns true if the argument1 happened * before argument2. */ Datum citus_is_clock_after(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* Fetch both the arguments */ ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); ereport(DEBUG1, (errmsg( "clock1 @ LC:%lu, C:%u, " "clock2 @ LC:%lu, C:%u", clock1->logical, clock1->counter, clock2->logical, clock2->counter))); bool result = (cluster_clock_cmp_internal(clock1, clock2) > 0); PG_RETURN_BOOL(result); } /* * citus_get_transaction_clock() is an UDF that returns a transaction timestamp * logical clock. Clock returned is the maximum of all transaction-nodes and the * all the nodes adjust to the this new clock value. */ Datum citus_get_transaction_clock(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ClusterClock *clusterClockValue = PrepareAndSetTransactionClock(); PG_RETURN_POINTER(clusterClockValue); } ================================================ FILE: src/backend/distributed/commands/README.md ================================================ # Commands The commands module is modeled after `backend/commands` from the postgres repository and contains the logic for Citus on how to run these commands on distributed objects. Even though the structure of the directory has some resemblence to its postgres relative, files here are somewhat more fine-grained. This is due to the nature of citus commands that are heavily focused on distributed tables. Instead of having all commands in `tablecmds.c` they are often moved to files that are named after the command. | File | Description | |------------------------------|-------------| | `create_distributed_table.c` | Implementation of UDF's for creating distributed tables | | `drop_distributed_table.c` | Implementation for dropping metadata for partitions of distributed tables | | `extension.c` | Implementation of `CREATE EXTENSION` commands for citus specific checks | | `foreign_constraint.c` | Implementation of and helper functions for foreign key constraints | | `grant.c` | Implementation of `GRANT` commands for roles/users on relations | | `index.c` | Implementation of commands specific to indices on distributed tables | | `multi_copy.c` | Implementation of `COPY` command. There are multiple different copy modes which are described in detail below | | `policy.c` | Implementation of `CREATE\ALTER POLICY` commands. | | `rename.c` | Implementation of `ALTER ... RENAME ...` commands. It implements the renaming of applicable objects, otherwise provides the user with a warning | | `schema.c` | | | `sequence.c` | Implementation of `CREATE/ALTER SEQUENCE` commands. Primarily checks correctness of sequence statements as they are not propagated to the worker nodes | | `table.c` | | | `transmit.c` | Implementation of `COPY` commands with `format transmit` set in the options. This format is used to transfer files from one node to another node | | `truncate.c` | Implementation of `TRUNCATE` commands on distributed tables | | `utility_hook.c` | This is the entry point from postgres into the commands module of citus. It contains the implementation that gets registered in postgres' `ProcessUtility_hook` callback to extends the functionality of the original ProcessUtility. This code is used to route the incoming commands to their respective implementation in Citus | | `vacuum.c` | Implementation of `VACUUM` commands on distributed tables | # COPY The copy command is overloaded for a couple of use-cases specific to citus. The syntax of the command stays the same, however the implementation might slightly differ from the stock implementation. The overloading is mostly done via extra options that Citus uses to indicate how to operate the copy. The options used are described below. ## FORMAT transmit Implemented in `transmit.c` TODO: to be written by someone with enough knowledge to write how, when and why it is used. ## FORMAT result Implemented in `multi_copy.c` TODO: to be written by someone with enough knowledge to write how, when and why it is used. ================================================ FILE: src/backend/distributed/commands/alter_table.c ================================================ /*------------------------------------------------------------------------- * * alter_table.c * Routines related to the altering of tables. * * There are three UDFs defined in this file: * undistribute_table: * Turns a distributed table to a local table * alter_distributed_table: * Alters distribution_column, shard_count or colocate_with * properties of a distributed table * alter_table_set_access_method: * Changes the access method of a table * * All three methods work in similar steps: * - Create a new table the required way (with a different * shard count, distribution column, colocate with value, * access method or local) * - Move everything to the new table * - Drop the old one * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "access/hash.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/pg_am.h" #include "catalog/pg_depend.h" #include "catalog/pg_rewrite_d.h" #include "commands/defrem.h" #include "executor/spi.h" #include "nodes/pg_list.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "columnar/columnar.h" #include "columnar/columnar_tableam.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/distribution_column.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/namespace_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/shard_utils.h" #include "distributed/shared_library_init.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* Table Conversion Types */ #define UNDISTRIBUTE_TABLE 'u' #define ALTER_DISTRIBUTED_TABLE 'a' #define ALTER_TABLE_SET_ACCESS_METHOD 'm' #define UNDISTRIBUTE_TABLE_CASCADE_HINT \ "Use cascade option to undistribute all the relations involved in " \ "a foreign key relationship with %s by executing SELECT " \ "undistribute_table($$%s$$, cascade_via_foreign_keys=>true)" typedef TableConversionReturn *(*TableConversionFunction)(struct TableConversionParameters *); /* * TableConversionState objects are used for table conversion functions: * UndistributeTable, AlterDistributedTable, AlterTableSetAccessMethod. * * They can be created using TableConversionParameters objects with * CreateTableConversion function. * * TableConversionState objects include everything TableConversionParameters * objects do and some extra to be used in the conversion process. */ typedef struct TableConversionState { /* * Determines type of conversion: UNDISTRIBUTE_TABLE, * ALTER_DISTRIBUTED_TABLE, ALTER_TABLE_SET_ACCESS_METHOD. */ char conversionType; /* Oid of the table to do conversion on */ Oid relationId; /* * Options to do conversions on the table * distributionColumn is the name of the new distribution column, * shardCountIsNull is if the shardCount variable is not given * shardCount is the new shard count, * colocateWith is the name of the table to colocate with, 'none', or * 'default' * accessMethod is the name of the new accessMethod for the table */ char *distributionColumn; bool shardCountIsNull; int shardCount; char *colocateWith; char *accessMethod; /* * cascadeToColocated determines whether the shardCount and * colocateWith will be cascaded to the currently colocated tables */ CascadeToColocatedOption cascadeToColocated; /* * cascadeViaForeignKeys determines if the conversion operation * will be cascaded to the graph connected with foreign keys * to the table */ bool cascadeViaForeignKeys; /* schema of the table */ char *schemaName; Oid schemaId; /* name of the table */ char *relationName; /* new relation oid after the conversion */ Oid newRelationId; /* temporary name for intermediate table */ char *tempName; /*hash that is appended to the name to create tempName */ uint32 hashOfName; /* shard count of the table before conversion */ int originalShardCount; /* list of the table oids of tables colocated with the table before conversion */ List *colocatedTableList; /* new distribution key, if distributionColumn variable is given */ Var *distributionKey; /* distribution key of the table before conversion */ Var *originalDistributionKey; /* access method name of the table before conversion */ char *originalAccessMethod; /* * The function that will be used for the conversion * Must comply with conversionType * UNDISTRIBUTE_TABLE -> UndistributeTable * ALTER_DISTRIBUTED_TABLE -> AlterDistributedTable * ALTER_TABLE_SET_ACCESS_METHOD -> AlterTableSetAccessMethod */ TableConversionFunction function; /* * suppressNoticeMessages determines if we want to suppress NOTICE * messages that we explicitly issue */ bool suppressNoticeMessages; } TableConversionState; static TableConversionReturn * AlterDistributedTable(TableConversionParameters *params); static TableConversionReturn * AlterTableSetAccessMethod(TableConversionParameters * params); static TableConversionReturn * ConvertTable(TableConversionState *con); static TableConversionReturn * ConvertTableInternal(TableConversionState *con); static bool SwitchToSequentialAndLocalExecutionIfShardNameTooLong(char *relationName, char *longestShardName); static void DropIndexesNotSupportedByColumnar(Oid relationId, bool suppressNoticeMessages); static char * GetIndexAccessMethodName(Oid indexId); static void DropConstraintRestrict(Oid relationId, Oid constraintId); static void DropIndexRestrict(Oid indexId); static void EnsureTableNotReferencing(Oid relationId, char conversionType); static void EnsureTableNotReferenced(Oid relationId, char conversionType); static void EnsureTableNotForeign(Oid relationId); static void EnsureTableNotPartition(Oid relationId); static void ErrorIfColocateWithTenantTable(char *colocateWith); static TableConversionState * CreateTableConversion(TableConversionParameters *params); static void CreateDistributedTableLike(TableConversionState *con); static void CreateCitusTableLike(TableConversionState *con); static void ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, bool suppressNoticeMessages); static bool HasAnyGeneratedStoredColumns(Oid relationId); static List * GetNonGeneratedStoredColumnNameList(Oid relationId); static void CheckAlterDistributedTableConversionParameters(TableConversionState *con); static char * CreateWorkerChangeSequenceDependencyCommand(char *qualifiedSequeceName, char *qualifiedSourceName, char *qualifiedTargetName); static void ErrorIfMatViewSizeExceedsTheLimit(Oid matViewOid); static char * CreateMaterializedViewDDLCommand(Oid matViewOid); static char * GetAccessMethodForMatViewIfExists(Oid viewOid); static bool WillRecreateFKeyToReferenceTable(Oid relationId, CascadeToColocatedOption cascadeOption); static void WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId); static void ErrorIfUnsupportedCascadeObjects(Oid relationId); static List * WrapTableDDLCommands(List *commandStrings); static bool DoesCascadeDropUnsupportedObject(Oid classId, Oid id, HTAB *nodeMap); static TableConversionReturn * CopyTableConversionReturnIntoCurrentContext( TableConversionReturn *tableConversionReturn); PG_FUNCTION_INFO_V1(undistribute_table); PG_FUNCTION_INFO_V1(alter_distributed_table); PG_FUNCTION_INFO_V1(alter_table_set_access_method); PG_FUNCTION_INFO_V1(worker_change_sequence_dependency); /* global variable keeping track of whether we are in a table type conversion function */ bool InTableTypeConversionFunctionCall = false; /* controlled by GUC, in MB */ int MaxMatViewSizeToAutoRecreate = 1024; /* * undistribute_table gets a distributed table name and * udistributes it. */ Datum undistribute_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); bool cascadeViaForeignKeys = PG_GETARG_BOOL(1); TableConversionParameters params = { .relationId = relationId, .cascadeViaForeignKeys = cascadeViaForeignKeys, .bypassTenantCheck = false }; UndistributeTable(¶ms); PG_RETURN_VOID(); } /* * alter_distributed_table gets a distributed table and some other * parameters and alters some properties of the table according to * the parameters. */ Datum alter_distributed_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); char *distributionColumn = NULL; if (!PG_ARGISNULL(1)) { text *distributionColumnText = PG_GETARG_TEXT_P(1); distributionColumn = text_to_cstring(distributionColumnText); } int shardCount = 0; bool shardCountIsNull = true; if (!PG_ARGISNULL(2)) { shardCount = PG_GETARG_INT32(2); shardCountIsNull = false; } char *colocateWith = NULL; if (!PG_ARGISNULL(3)) { text *colocateWithText = PG_GETARG_TEXT_P(3); colocateWith = text_to_cstring(colocateWithText); } CascadeToColocatedOption cascadeToColocated = CASCADE_TO_COLOCATED_UNSPECIFIED; if (!PG_ARGISNULL(4)) { if (PG_GETARG_BOOL(4) == true) { cascadeToColocated = CASCADE_TO_COLOCATED_YES; } else { cascadeToColocated = CASCADE_TO_COLOCATED_NO; } } TableConversionParameters params = { .relationId = relationId, .distributionColumn = distributionColumn, .shardCountIsNull = shardCountIsNull, .shardCount = shardCount, .colocateWith = colocateWith, .cascadeToColocated = cascadeToColocated }; AlterDistributedTable(¶ms); PG_RETURN_VOID(); } /* * alter_table_set_access_method gets a distributed table and an access * method and changes table's access method into that. */ Datum alter_table_set_access_method(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *accessMethodText = PG_GETARG_TEXT_P(1); char *accessMethod = text_to_cstring(accessMethodText); TableConversionParameters params = { .relationId = relationId, .accessMethod = accessMethod }; AlterTableSetAccessMethod(¶ms); PG_RETURN_VOID(); } /* * worker_change_sequence_dependency is a wrapper UDF for * changeDependencyFor function */ Datum worker_change_sequence_dependency(PG_FUNCTION_ARGS) { Oid sequenceOid = PG_GETARG_OID(0); Oid sourceRelationOid = PG_GETARG_OID(1); Oid targetRelationOid = PG_GETARG_OID(2); changeDependencyFor(RelationRelationId, sequenceOid, RelationRelationId, sourceRelationOid, targetRelationOid); PG_RETURN_VOID(); } /* * DropFKeysAndUndistributeTable drops all foreign keys that relation with * relationId is involved then undistributes it. * Note that as UndistributeTable changes relationId of relation, this * function also returns new relationId of relation. * Also note that callers are responsible for storing & recreating foreign * keys to be dropped if needed. */ Oid DropFKeysAndUndistributeTable(Oid relationId) { DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES); /* store them before calling UndistributeTable as it changes relationId */ char *relationName = get_rel_name(relationId); Oid schemaId = get_rel_namespace(relationId); /* suppress notices messages not to be too verbose */ TableConversionParameters params = { .relationId = relationId, .cascadeViaForeignKeys = false, .suppressNoticeMessages = true }; UndistributeTable(¶ms); Oid newRelationId = get_relname_relid(relationName, schemaId); /* * We don't expect this to happen but to be on the safe side let's error * out here. */ EnsureRelationExists(newRelationId); return newRelationId; } /* * UndistributeTables undistributes given relations. It first collects all foreign keys * to recreate them after the undistribution. Then, drops the foreign keys and * undistributes the relations. Finally, it recreates foreign keys. */ void UndistributeTables(List *relationIdList) { /* * Collect foreign keys for recreation and then drop fkeys and undistribute * tables. */ List *originalForeignKeyRecreationCommands = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { List *fkeyCommandsForRelation = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES); originalForeignKeyRecreationCommands = list_concat( originalForeignKeyRecreationCommands, fkeyCommandsForRelation); DropFKeysAndUndistributeTable(relationId); } /* We can skip foreign key validations as we are sure about them at start */ bool skip_validation = true; ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands, skip_validation); } /* * EnsureUndistributeTenantTableSafe ensures that it is safe to undistribute a tenant table. */ void EnsureUndistributeTenantTableSafe(Oid relationId, const char *operationName) { Oid schemaId = get_rel_namespace(relationId); Assert(IsTenantSchema(schemaId)); /* We only allow undistribute while altering schema */ if (strcmp(operationName, TenantOperationNames[TENANT_SET_SCHEMA]) != 0) { ErrorIfTenantTable(relationId, operationName); } char *tableName = get_rel_name(relationId); char *schemaName = get_namespace_name(schemaId); /* * Partition table cannot be undistributed. Otherwise, its parent table would still * be a tenant table whereas partition table would be a local table. */ if (PartitionTable(relationId)) { ereport(ERROR, (errmsg("%s is not allowed for partition table %s in distributed " "schema %s", operationName, tableName, schemaName), errdetail("partition table should be under the same distributed " "schema as its parent and be a " "distributed schema table."))); } /* * When table is referenced by or referencing to a table in the same tenant * schema, we should disallow undistributing the table since we do not allow * foreign keys from/to Citus local or Postgres local table to/from distributed * schema. */ List *fkeyCommandsWithSingleShardTables = GetFKeyCreationCommandsRelationInvolvedWithTableType( relationId, INCLUDE_SINGLE_SHARD_TABLES); if (fkeyCommandsWithSingleShardTables != NIL) { ereport(ERROR, (errmsg("%s is not allowed for table %s in distributed schema %s", operationName, tableName, schemaName), errdetail("distributed schemas cannot have foreign keys from/to " "local tables or different schema"))); } } /* * UndistributeTable undistributes the given table. It uses ConvertTable function to * create a new local table and move everything to that table. * * The local tables, tables with references, partition tables and foreign tables are * not supported. The function gives errors in these cases. */ TableConversionReturn * UndistributeTable(TableConversionParameters *params) { EnsureCoordinator(); EnsureRelationExists(params->relationId); EnsureTableOwner(params->relationId); if (!IsCitusTable(params->relationId)) { ereport(ERROR, (errmsg("cannot undistribute table " "because the table is not distributed"))); } Oid schemaId = get_rel_namespace(params->relationId); if (!params->bypassTenantCheck && IsTenantSchema(schemaId) && IsCitusTableType(params->relationId, SINGLE_SHARD_DISTRIBUTED)) { EnsureUndistributeTenantTableSafe( params->relationId, TenantOperationNames[TENANT_UNDISTRIBUTE_TABLE]); } if (!params->cascadeViaForeignKeys) { EnsureTableNotReferencing(params->relationId, UNDISTRIBUTE_TABLE); EnsureTableNotReferenced(params->relationId, UNDISTRIBUTE_TABLE); } EnsureTableNotPartition(params->relationId); if (PartitionedTable(params->relationId)) { List *partitionList = PartitionList(params->relationId); /* * This is a less common pattern where foreing key is directly from/to * the partition relation as we already handled inherited foreign keys * on partitions either by erroring out or cascading via foreign keys. * It seems an acceptable limitation for now to ask users to drop such * foreign keys manually. */ ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(partitionList); } ErrorIfUnsupportedCascadeObjects(params->relationId); params->conversionType = UNDISTRIBUTE_TABLE; params->shardCountIsNull = true; TableConversionState *con = CreateTableConversion(params); SetupReplicationOriginLocalSession(); TableConversionReturn *conv = ConvertTable(con); ResetReplicationOriginLocalSession(); return conv; } /* * AlterDistributedTable changes some properties of the given table. It uses * ConvertTable function to create a new local table and move everything to that table. * * The local and reference tables, tables with references, partition tables and foreign * tables are not supported. The function gives errors in these cases. */ TableConversionReturn * AlterDistributedTable(TableConversionParameters *params) { EnsureCoordinator(); EnsureRelationExists(params->relationId); EnsureTableOwner(params->relationId); if (!IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) { ereport(ERROR, (errmsg("cannot alter table because the table " "is not distributed"))); } ErrorIfTenantTable(params->relationId, TenantOperationNames[TENANT_ALTER_TABLE]); ErrorIfColocateWithTenantTable(params->colocateWith); EnsureTableNotForeign(params->relationId); EnsureTableNotPartition(params->relationId); EnsureHashDistributedTable(params->relationId); ErrorIfUnsupportedCascadeObjects(params->relationId); params->conversionType = ALTER_DISTRIBUTED_TABLE; TableConversionState *con = CreateTableConversion(params); CheckAlterDistributedTableConversionParameters(con); if (WillRecreateFKeyToReferenceTable(con->relationId, con->cascadeToColocated)) { ereport(DEBUG1, (errmsg("setting multi shard modify mode to sequential"))); SetLocalMultiShardModifyModeToSequential(); } return ConvertTable(con); } /* * AlterTableSetAccessMethod changes the access method of the given table. It uses * ConvertTable function to create a new table with the access method and move everything * to that table. * * The local and references tables, tables with references, partition tables and foreign * tables are not supported. The function gives errors in these cases. */ TableConversionReturn * AlterTableSetAccessMethod(TableConversionParameters *params) { EnsureRelationExists(params->relationId); EnsureTableOwner(params->relationId); if (IsCitusTable(params->relationId)) { EnsureCoordinator(); } EnsureTableNotReferencing(params->relationId, ALTER_TABLE_SET_ACCESS_METHOD); EnsureTableNotReferenced(params->relationId, ALTER_TABLE_SET_ACCESS_METHOD); EnsureTableNotForeign(params->relationId); if (!IsCitusTableType(params->relationId, SINGLE_SHARD_DISTRIBUTED) && IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) { /* we do not support non-hash distributed tables, except single shard tables */ EnsureHashDistributedTable(params->relationId); } if (PartitionedTable(params->relationId)) { ereport(ERROR, (errmsg("you cannot alter access method of a partitioned table"))); } if (get_rel_relkind(params->relationId) == RELKIND_VIEW) { ereport(ERROR, (errmsg("you cannot alter access method of a view"))); } if (PartitionTable(params->relationId) && IsCitusTableType(params->relationId, DISTRIBUTED_TABLE)) { Oid parentRelationId = PartitionParentOid(params->relationId); if (HasForeignKeyToReferenceTable(parentRelationId)) { ereport(DEBUG1, (errmsg("setting multi shard modify mode to sequential"))); SetLocalMultiShardModifyModeToSequential(); } } ErrorIfUnsupportedCascadeObjects(params->relationId); params->conversionType = ALTER_TABLE_SET_ACCESS_METHOD; params->shardCountIsNull = true; TableConversionState *con = CreateTableConversion(params); if (strcmp(con->originalAccessMethod, con->accessMethod) == 0) { ereport(ERROR, (errmsg("the access method of %s is already %s", generate_qualified_relation_name(con->relationId), con->accessMethod))); } return ConvertTable(con); } /* * ConvertTableInternal is used for converting a table into a new table with different * properties. The conversion is done by creating a new table, moving everything to the * new table and dropping the old one. So the oid of the table is not preserved. * * The new table will have the same name, columns and rows. It will also have partitions, * views, sequences of the old table. Finally it will have everything created by * GetPostLoadTableCreationCommands function, which include indexes. These will be * re-created during conversion, so their oids are not preserved either (except for * sequences). However, their names are preserved. * * The dropping of old table is done with CASCADE. Anything not mentioned here will * be dropped. * * The function returns a TableConversionReturn object that can stores variables that * can be used at the caller operations. * * To be able to provide more meaningful messages while converting a table type, * Citus keeps InTableTypeConversionFunctionCall flag. Don't forget to set it properly * in case you add a new way to return from this function. */ TableConversionReturn * ConvertTableInternal(TableConversionState *con) { InTableTypeConversionFunctionCall = true; /* * We undistribute citus local tables that are not chained with any reference * tables via foreign keys at the end of the utility hook. * Here we temporarily set the related GUC to off to disable the logic for * internally executed DDL's that might invoke this mechanism unnecessarily. */ bool oldEnableLocalReferenceForeignKeys = EnableLocalReferenceForeignKeys; SetLocalEnableLocalReferenceForeignKeys(false); /* switch to sequential execution if shard names will be too long */ SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(con->relationId, con->relationName); if (con->conversionType == UNDISTRIBUTE_TABLE && con->cascadeViaForeignKeys && (TableReferencing(con->relationId) || TableReferenced(con->relationId))) { /* * Acquire ExclusiveLock as UndistributeTable does in order to * make sure that no modifications happen on the relations. */ CascadeOperationForFkeyConnectedRelations(con->relationId, ExclusiveLock, CASCADE_FKEY_UNDISTRIBUTE_TABLE); /* * Undistributed every foreign key connected relation in our foreign key * subgraph including itself, so return here. */ SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys); InTableTypeConversionFunctionCall = false; return NULL; } char *newAccessMethod = con->accessMethod ? con->accessMethod : con->originalAccessMethod; IncludeSequenceDefaults includeSequenceDefaults = NEXTVAL_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentity = INCLUDE_IDENTITY; List *preLoadCommands = GetPreLoadTableCreationCommands(con->relationId, includeSequenceDefaults, includeIdentity, newAccessMethod); if (con->accessMethod && strcmp(con->accessMethod, "columnar") == 0) { DropIndexesNotSupportedByColumnar(con->relationId, con->suppressNoticeMessages); } /* * Since we already dropped unsupported indexes, we can safely pass * includeIndexes to be true. */ bool includeIndexes = true; bool includeReplicaIdentity = true; List *postLoadCommands = GetPostLoadTableCreationCommands(con->relationId, includeIndexes, includeReplicaIdentity); List *justBeforeDropCommands = NIL; List *attachPartitionCommands = NIL; List *createViewCommands = GetViewCreationCommandsOfTable(con->relationId); postLoadCommands = list_concat(postLoadCommands, WrapTableDDLCommands(createViewCommands)); /* need to add back to publications after dropping the original table */ bool isAdd = true; List *alterPublicationCommands = GetAlterPublicationDDLCommandsForTable(con->relationId, isAdd); postLoadCommands = list_concat(postLoadCommands, WrapTableDDLCommands(alterPublicationCommands)); List *foreignKeyCommands = NIL; if (con->conversionType == ALTER_DISTRIBUTED_TABLE) { foreignKeyCommands = GetForeignConstraintToReferenceTablesCommands( con->relationId); if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES || con->cascadeToColocated == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) { List *foreignKeyToDistributedTableCommands = GetForeignConstraintToDistributedTablesCommands(con->relationId); foreignKeyCommands = list_concat(foreignKeyCommands, foreignKeyToDistributedTableCommands); List *foreignKeyFromDistributedTableCommands = GetForeignConstraintFromDistributedTablesCommands(con->relationId); foreignKeyCommands = list_concat(foreignKeyCommands, foreignKeyFromDistributedTableCommands); } else { WarningsForDroppingForeignKeysWithDistributedTables(con->relationId); } } bool isPartitionTable = false; char *attachToParentCommand = NULL; if (PartitionTable(con->relationId)) { isPartitionTable = true; char *detachFromParentCommand = GenerateDetachPartitionCommand(con->relationId); attachToParentCommand = GenerateAlterTableAttachPartitionCommand(con->relationId); justBeforeDropCommands = lappend(justBeforeDropCommands, detachFromParentCommand); } char *qualifiedRelationName = quote_qualified_identifier(con->schemaName, con->relationName); if (PartitionedTable(con->relationId)) { if (!con->suppressNoticeMessages) { ereport(NOTICE, (errmsg("converting the partitions of %s", qualifiedRelationName))); } List *partitionList = PartitionList(con->relationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { char *tableQualifiedName = generate_qualified_relation_name( partitionRelationId); char *detachPartitionCommand = GenerateDetachPartitionCommand( partitionRelationId); char *attachPartitionCommand = GenerateAlterTableAttachPartitionCommand( partitionRelationId); /* * We first detach the partitions to be able to convert them separately. * After this they are no longer partitions, so they will not be caught by * the checks. */ ExecuteQueryViaSPI(detachPartitionCommand, SPI_OK_UTILITY); attachPartitionCommands = lappend(attachPartitionCommands, attachPartitionCommand); CascadeToColocatedOption cascadeOption = CASCADE_TO_COLOCATED_NO; if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES || con->cascadeToColocated == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) { cascadeOption = CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED; } TableConversionParameters partitionParam = { .relationId = partitionRelationId, .distributionColumn = con->distributionColumn, .shardCountIsNull = con->shardCountIsNull, .shardCount = con->shardCount, .cascadeToColocated = cascadeOption, .colocateWith = con->colocateWith, .suppressNoticeMessages = con->suppressNoticeMessages, /* * Even if we called UndistributeTable with cascade option, we * shouldn't cascade via foreign keys on partitions. Otherwise, * we might try to undistribute partitions of other tables in * our foreign key subgraph more than once. */ .cascadeViaForeignKeys = false }; TableConversionReturn *partitionReturn = con->function(&partitionParam); if (cascadeOption == CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) { foreignKeyCommands = list_concat(foreignKeyCommands, partitionReturn->foreignKeyCommands); } /* * If we are altering a partitioned distributed table by * colocateWith:none, we override con->colocationWith parameter * with the first newly created partition table to share the * same colocation group for rest of partitions and partitioned * table. */ if (con->colocateWith != NULL && IsColocateWithNone(con->colocateWith)) { con->colocateWith = tableQualifiedName; } } } if (!con->suppressNoticeMessages) { ereport(NOTICE, (errmsg("creating a new table for %s", qualifiedRelationName))); } TableDDLCommand *tableCreationCommand = NULL; foreach_declared_ptr(tableCreationCommand, preLoadCommands) { Assert(CitusIsA(tableCreationCommand, TableDDLCommand)); char *tableCreationSql = GetTableDDLCommand(tableCreationCommand); Node *parseTree = ParseTreeNode(tableCreationSql); RelayEventExtendNames(parseTree, con->schemaName, con->hashOfName); ProcessUtilityParseTree(parseTree, tableCreationSql, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } /* set columnar options */ if (con->accessMethod == NULL && con->originalAccessMethod && strcmp(con->originalAccessMethod, "columnar") == 0) { ColumnarOptions options = { 0 }; extern_ReadColumnarOptions(con->relationId, &options); ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0( sizeof(ColumnarTableDDLContext)); /* build the context */ context->schemaName = con->schemaName; context->relationName = con->relationName; context->options = options; char *columnarOptionsSql = GetShardedTableDDLCommandColumnar(con->hashOfName, context); ExecuteQueryViaSPI(columnarOptionsSql, SPI_OK_UTILITY); } con->newRelationId = get_relname_relid(con->tempName, con->schemaId); if (con->conversionType == ALTER_DISTRIBUTED_TABLE) { CreateDistributedTableLike(con); } else if (con->conversionType == ALTER_TABLE_SET_ACCESS_METHOD) { CreateCitusTableLike(con); } /* preserve colocation with procedures/functions */ if (con->conversionType == ALTER_DISTRIBUTED_TABLE) { /* * Updating the colocationId of functions is always desirable for * the following scenario: * we have shardCount or colocateWith change * AND entire co-location group is altered * The reason for the second condition is because we currently don't * remember the original table specified in the colocateWith when * distributing the function. We only remember the colocationId in * pg_dist_object table. */ if ((!con->shardCountIsNull || con->colocateWith != NULL) && (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES || list_length( con->colocatedTableList) == 1) && con->distributionColumn == NULL) { /* * Update the colocationId from the one of the old relation to the one * of the new relation for all tuples in citus.pg_dist_object */ UpdateDistributedObjectColocationId(TableColocationId(con->relationId), TableColocationId(con->newRelationId)); } } ReplaceTable(con->relationId, con->newRelationId, justBeforeDropCommands, con->suppressNoticeMessages); TableDDLCommand *tableConstructionCommand = NULL; foreach_declared_ptr(tableConstructionCommand, postLoadCommands) { Assert(CitusIsA(tableConstructionCommand, TableDDLCommand)); char *tableConstructionSQL = GetTableDDLCommand(tableConstructionCommand); ExecuteQueryViaSPI(tableConstructionSQL, SPI_OK_UTILITY); } /* * when there are many partitions, each call to ProcessUtilityParseTree * accumulates used memory. Free context after each call. */ MemoryContext citusPerPartitionContext = AllocSetContextCreate(CurrentMemoryContext, "citus_per_partition_context", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(citusPerPartitionContext); char *attachPartitionCommand = NULL; foreach_declared_ptr(attachPartitionCommand, attachPartitionCommands) { MemoryContextReset(citusPerPartitionContext); Node *parseTree = ParseTreeNode(attachPartitionCommand); ProcessUtilityParseTree(parseTree, attachPartitionCommand, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } MemoryContextSwitchTo(oldContext); MemoryContextDelete(citusPerPartitionContext); if (isPartitionTable) { ExecuteQueryViaSPI(attachToParentCommand, SPI_OK_UTILITY); } if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES) { Oid colocatedTableId = InvalidOid; /* For now we only support cascade to colocation for alter_distributed_table UDF */ Assert(con->conversionType == ALTER_DISTRIBUTED_TABLE); foreach_declared_oid(colocatedTableId, con->colocatedTableList) { if (colocatedTableId == con->relationId) { continue; } TableConversionParameters cascadeParam = { .relationId = colocatedTableId, .shardCountIsNull = con->shardCountIsNull, .shardCount = con->shardCount, .colocateWith = qualifiedRelationName, .cascadeToColocated = CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED, .suppressNoticeMessages = con->suppressNoticeMessages }; TableConversionReturn *colocatedReturn = con->function(&cascadeParam); foreignKeyCommands = list_concat(foreignKeyCommands, colocatedReturn->foreignKeyCommands); } } /* recreate foreign keys */ TableConversionReturn *ret = NULL; if (con->conversionType == ALTER_DISTRIBUTED_TABLE) { if (con->cascadeToColocated != CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED) { char *foreignKeyCommand = NULL; foreach_declared_ptr(foreignKeyCommand, foreignKeyCommands) { ExecuteQueryViaSPI(foreignKeyCommand, SPI_OK_UTILITY); } } else { ret = palloc0(sizeof(TableConversionReturn)); ret->foreignKeyCommands = foreignKeyCommands; } } /* increment command counter so that next command can see the new table */ CommandCounterIncrement(); SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys); InTableTypeConversionFunctionCall = false; return ret; } /* * CopyTableConversionReturnIntoCurrentContext copies given tableConversionReturn * into CurrentMemoryContext. */ static TableConversionReturn * CopyTableConversionReturnIntoCurrentContext(TableConversionReturn *tableConversionReturn) { TableConversionReturn *tableConversionReturnCopy = NULL; if (tableConversionReturn) { tableConversionReturnCopy = palloc0(sizeof(TableConversionReturn)); List *copyForeignKeyCommands = NIL; char *foreignKeyCommand = NULL; foreach_declared_ptr(foreignKeyCommand, tableConversionReturn->foreignKeyCommands) { char *copyForeignKeyCommand = MemoryContextStrdup(CurrentMemoryContext, foreignKeyCommand); copyForeignKeyCommands = lappend(copyForeignKeyCommands, copyForeignKeyCommand); } tableConversionReturnCopy->foreignKeyCommands = copyForeignKeyCommands; } return tableConversionReturnCopy; } /* * ConvertTable is a wrapper for ConvertTableInternal to persist only * TableConversionReturn and delete all other allocations. */ static TableConversionReturn * ConvertTable(TableConversionState *con) { /* * We do not allow alter_distributed_table and undistribute_table operations * for tables with identity columns. This is because we do not have a proper way * of keeping sequence states consistent across the cluster. */ ErrorIfTableHasIdentityColumn(con->relationId); /* * when there are many partitions or colocated tables, memory usage is * accumulated. Free context for each call to ConvertTable. */ MemoryContext convertTableContext = AllocSetContextCreate(CurrentMemoryContext, "citus_convert_table_context", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(convertTableContext); TableConversionReturn *tableConversionReturn = ConvertTableInternal(con); MemoryContextSwitchTo(oldContext); /* persist TableConversionReturn in oldContext */ TableConversionReturn *tableConversionReturnCopy = CopyTableConversionReturnIntoCurrentContext(tableConversionReturn); /* delete convertTableContext */ MemoryContextDelete(convertTableContext); return tableConversionReturnCopy; } /* * DropIndexesNotSupportedByColumnar is a helper function used during accces * method conversion to drop the indexes that are not supported by columnarAM. */ static void DropIndexesNotSupportedByColumnar(Oid relationId, bool suppressNoticeMessages) { Relation columnarRelation = RelationIdGetRelation(relationId); if (!RelationIsValid(columnarRelation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *indexIdList = RelationGetIndexList(columnarRelation); /* * Immediately close the relation since we might execute ALTER TABLE * for that relation. */ RelationClose(columnarRelation); Oid indexId = InvalidOid; foreach_declared_oid(indexId, indexIdList) { char *indexAmName = GetIndexAccessMethodName(indexId); if (extern_ColumnarSupportsIndexAM(indexAmName)) { continue; } if (!suppressNoticeMessages) { ereport(NOTICE, (errmsg("unsupported access method for index %s " "on columnar table %s, given index and " "the constraint depending on the index " "(if any) will be dropped", get_rel_name(indexId), generate_qualified_relation_name(relationId)))); } Oid constraintId = get_index_constraint(indexId); if (OidIsValid(constraintId)) { /* index is implied by a constraint, so drop the constraint itself */ DropConstraintRestrict(relationId, constraintId); } else { DropIndexRestrict(indexId); } } } /* * GetIndexAccessMethodName returns access method name of index with indexId. * If there is no such index, then errors out. */ static char * GetIndexAccessMethodName(Oid indexId) { /* fetch pg_class tuple of the index relation */ HeapTuple indexTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indexId)); if (!HeapTupleIsValid(indexTuple)) { ereport(ERROR, (errmsg("index with oid %u does not exist", indexId))); } Form_pg_class indexForm = (Form_pg_class) GETSTRUCT(indexTuple); Oid indexAMId = indexForm->relam; ReleaseSysCache(indexTuple); char *indexAmName = get_am_name(indexAMId); if (!indexAmName) { ereport(ERROR, (errmsg("access method with oid %u does not exist", indexAMId))); } return indexAmName; } /* * DropConstraintRestrict drops the constraint with constraintId by using spi. */ static void DropConstraintRestrict(Oid relationId, Oid constraintId) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); char *constraintName = get_constraint_name(constraintId); const char *quotedConstraintName = quote_identifier(constraintName); StringInfo dropConstraintCommand = makeStringInfo(); appendStringInfo(dropConstraintCommand, "ALTER TABLE %s DROP CONSTRAINT %s RESTRICT;", qualifiedRelationName, quotedConstraintName); ExecuteQueryViaSPI(dropConstraintCommand->data, SPI_OK_UTILITY); } /* * DropIndexRestrict drops the index with indexId by using spi. */ static void DropIndexRestrict(Oid indexId) { char *qualifiedIndexName = generate_qualified_relation_name(indexId); StringInfo dropIndexCommand = makeStringInfo(); appendStringInfo(dropIndexCommand, "DROP INDEX %s RESTRICT;", qualifiedIndexName); ExecuteQueryViaSPI(dropIndexCommand->data, SPI_OK_UTILITY); } /* * EnsureTableNotReferencing checks if the table has a reference to another * table and errors if it is. */ void EnsureTableNotReferencing(Oid relationId, char conversionType) { if (TableReferencing(relationId)) { if (conversionType == UNDISTRIBUTE_TABLE) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); ereport(ERROR, (errmsg("cannot complete operation " "because table %s has a foreign key", get_rel_name(relationId)), errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, qualifiedRelationName, qualifiedRelationName))); } else { ereport(ERROR, (errmsg("cannot complete operation " "because table %s has a foreign key", get_rel_name(relationId)))); } } } /* * EnsureTableNotReferenced checks if the table is referenced by another * table and errors if it is. */ void EnsureTableNotReferenced(Oid relationId, char conversionType) { if (TableReferenced(relationId)) { if (conversionType == UNDISTRIBUTE_TABLE) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); ereport(ERROR, (errmsg("cannot complete operation " "because table %s is referenced by a foreign key", get_rel_name(relationId)), errhint(UNDISTRIBUTE_TABLE_CASCADE_HINT, qualifiedRelationName, qualifiedRelationName))); } else { ereport(ERROR, (errmsg("cannot complete operation " "because table %s is referenced by a foreign key", get_rel_name(relationId)))); } } } /* * EnsureTableNotForeign checks if the table is a foreign table and errors * if it is. */ void EnsureTableNotForeign(Oid relationId) { if (IsForeignTable(relationId)) { ereport(ERROR, (errmsg("cannot complete operation " "because it is a foreign table"))); } } /* * EnsureTableNotPartition checks if the table is a partition of another * table and errors if it is. */ void EnsureTableNotPartition(Oid relationId) { if (PartitionTable(relationId)) { Oid parentRelationId = PartitionParentOid(relationId); char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errmsg("cannot complete operation " "because table is a partition"), errhint("the parent table is \"%s\"", parentRelationName))); } } /* * ErrorIfColocateWithTenantTable errors out if given colocateWith text refers to * a tenant table. */ void ErrorIfColocateWithTenantTable(char *colocateWith) { if (colocateWith != NULL && !IsColocateWithDefault(colocateWith) && !IsColocateWithNone(colocateWith)) { text *colocateWithTableNameText = cstring_to_text(colocateWith); Oid colocateWithTableId = ResolveRelationId(colocateWithTableNameText, false); ErrorIfTenantTable(colocateWithTableId, TenantOperationNames[TENANT_COLOCATE_WITH]); } } TableConversionState * CreateTableConversion(TableConversionParameters *params) { TableConversionState *con = palloc0(sizeof(TableConversionState)); con->conversionType = params->conversionType; con->relationId = params->relationId; con->distributionColumn = params->distributionColumn; con->shardCountIsNull = params->shardCountIsNull; con->shardCount = params->shardCount; con->colocateWith = params->colocateWith; con->accessMethod = params->accessMethod; con->cascadeToColocated = params->cascadeToColocated; con->cascadeViaForeignKeys = params->cascadeViaForeignKeys; con->suppressNoticeMessages = params->suppressNoticeMessages; Relation relation = try_relation_open(con->relationId, ExclusiveLock); if (relation == NULL) { ereport(ERROR, (errmsg("cannot complete operation " "because no such table exists"))); } Oid relam = relation->rd_rel->relam; relation_close(relation, NoLock); con->distributionKey = BuildDistributionKeyFromColumnName(con->relationId, con->distributionColumn, NoLock); con->originalAccessMethod = NULL; if (!PartitionedTable(con->relationId) && !IsForeignTable(con->relationId)) { HeapTuple amTuple = SearchSysCache1(AMOID, ObjectIdGetDatum( relam)); if (!HeapTupleIsValid(amTuple)) { ereport(ERROR, (errmsg("cache lookup failed for access method %d", relam))); } Form_pg_am amForm = (Form_pg_am) GETSTRUCT(amTuple); con->originalAccessMethod = NameStr(amForm->amname); ReleaseSysCache(amTuple); } con->colocatedTableList = NIL; if (IsCitusTableType(con->relationId, DISTRIBUTED_TABLE)) { con->originalDistributionKey = DistPartitionKey(con->relationId); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(con->relationId); con->originalShardCount = cacheEntry->shardIntervalArrayLength; List *colocatedTableList = ColocatedTableList(con->relationId); /* * we will not add partition tables to the colocatedTableList * since they will be handled separately. */ Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { if (PartitionTable(colocatedTableId)) { continue; } con->colocatedTableList = lappend_oid(con->colocatedTableList, colocatedTableId); } /* sort the oids to avoid deadlock */ con->colocatedTableList = SortList(con->colocatedTableList, CompareOids); } /* find relation and schema names */ con->relationName = get_rel_name(con->relationId); con->schemaId = get_rel_namespace(con->relationId); con->schemaName = get_namespace_name(con->schemaId); /* calculate a temp name for the new table */ con->tempName = pstrdup(con->relationName); con->hashOfName = hash_any((unsigned char *) con->tempName, strlen(con->tempName)); AppendShardIdToName(&con->tempName, con->hashOfName); if (con->conversionType == UNDISTRIBUTE_TABLE) { con->function = &UndistributeTable; } else if (con->conversionType == ALTER_DISTRIBUTED_TABLE) { con->function = &AlterDistributedTable; } else if (con->conversionType == ALTER_TABLE_SET_ACCESS_METHOD) { con->function = &AlterTableSetAccessMethod; } return con; } /* * CreateDistributedTableLike distributes the new table in con parameter * like the old one. It checks the distribution column, colocation and * shard count and if they are not changed sets them to the old table's values. */ void CreateDistributedTableLike(TableConversionState *con) { Var *newDistributionKey = con->distributionColumn ? con->distributionKey : con->originalDistributionKey; char *newColocateWith = con->colocateWith; if (con->colocateWith == NULL) { /* * If the new distribution column and the old one have the same data type * and the shard_count parameter is null (which means shard count will not * change) we can create the new table in the same colocation as the old one. * In this case we set the new table's colocate_with value as the old table * so we don't even change the colocation id of the table during conversion. */ if (con->originalDistributionKey->vartype == newDistributionKey->vartype && con->shardCountIsNull) { newColocateWith = quote_qualified_identifier(con->schemaName, con->relationName); } else { newColocateWith = "default"; } } int newShardCount = 0; if (con->shardCountIsNull) { newShardCount = con->originalShardCount; } else { newShardCount = con->shardCount; } /* * To get the correct column name, we use the original relation id, not the * new relation id. The reason is that the cached attributes of the original * and newly created tables are not the same if the original table has * dropped columns (dropped columns are still present in the attribute cache) * Detailed example in https://github.com/citusdata/citus/pull/6387 */ char *distributionColumnName = ColumnToColumnName(con->relationId, (Node *) newDistributionKey); Oid originalRelationId = con->relationId; if (con->originalDistributionKey != NULL && PartitionTable(originalRelationId)) { /* * Due to dropped columns, the partition tables might have different * distribution keys than their parents, see issue #5123 for details. * * At this point, we get the partitioning information from the * originalRelationId, but we get the distribution key for newRelationId. * * We have to do this, because the newRelationId is just a placeholder * at this moment, but that's going to be the table in pg_dist_partition. */ Oid parentRelationId = PartitionParentOid(originalRelationId); Var *parentDistKey = DistPartitionKeyOrError(parentRelationId); distributionColumnName = ColumnToColumnName(parentRelationId, (Node *) parentDistKey); } char partitionMethod = PartitionMethod(con->relationId); CreateDistributedTable(con->newRelationId, distributionColumnName, partitionMethod, newShardCount, true, newColocateWith); } /* * CreateCitusTableLike converts the new table to the Citus table type * of the old table. */ void CreateCitusTableLike(TableConversionState *con) { if (IsCitusTableType(con->relationId, DISTRIBUTED_TABLE)) { if (IsCitusTableType(con->relationId, SINGLE_SHARD_DISTRIBUTED)) { ColocationParam colocationParam = { .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT, .colocateWithTableName = quote_qualified_identifier(con->schemaName, con->relationName) }; CreateSingleShardTable(con->newRelationId, colocationParam); } else { CreateDistributedTableLike(con); } } else if (IsCitusTableType(con->relationId, REFERENCE_TABLE)) { CreateReferenceTable(con->newRelationId); } else if (IsCitusTableType(con->relationId, CITUS_LOCAL_TABLE)) { CitusTableCacheEntry *entry = GetCitusTableCacheEntry(con->relationId); bool autoConverted = entry->autoConverted; bool cascade = false; CreateCitusLocalTable(con->newRelationId, cascade, autoConverted); /* * creating Citus local table adds a shell table on top * so we need its oid now */ con->newRelationId = get_relname_relid(con->tempName, con->schemaId); } } /* * ErrorIfUnsupportedCascadeObjects gets oid of a relation, finds the objects * that dropping this relation cascades into and errors if there are any extensions * that would be dropped. */ static void ErrorIfUnsupportedCascadeObjects(Oid relationId) { HTAB *nodeMap = CreateSimpleHashSetWithName(Oid, "object dependency map (oid)"); bool unsupportedObjectInDepGraph = DoesCascadeDropUnsupportedObject(RelationRelationId, relationId, nodeMap); if (unsupportedObjectInDepGraph) { ereport(ERROR, (errmsg("cannot alter table because an extension depends on it"))); } } /* * DoesCascadeDropUnsupportedObject walks through the objects that depend on the * object with object id and returns true if it finds any unsupported objects. * * This function only checks extensions as unsupported objects. * * Extension dependency is different than the rest. If an object depends on an extension * dropping the object would drop the extension too. * So we check with IsAnyObjectAddressOwnedByExtension function. */ static bool DoesCascadeDropUnsupportedObject(Oid classId, Oid objectId, HTAB *nodeMap) { bool found = false; hash_search(nodeMap, &objectId, HASH_ENTER, &found); if (found) { return false; } ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, classId, objectId); if (IsAnyObjectAddressOwnedByExtension(list_make1(objectAddress), NULL)) { return true; } Oid targetObjectClassId = classId; Oid targetObjectId = objectId; List *dependencyTupleList = GetPgDependTuplesForDependingObjects(targetObjectClassId, targetObjectId); HeapTuple depTup = NULL; foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); Oid dependingOid = InvalidOid; Oid dependingClassId = InvalidOid; if (pg_depend->classid == RewriteRelationId) { dependingOid = GetDependingView(pg_depend); dependingClassId = RelationRelationId; } else { dependingOid = pg_depend->objid; dependingClassId = pg_depend->classid; } if (DoesCascadeDropUnsupportedObject(dependingClassId, dependingOid, nodeMap)) { return true; } } return false; } /* * GetViewCreationCommandsOfTable takes a table oid generates the CREATE VIEW * commands for views that depend to the given table. This includes the views * that recursively depend on the table too. */ List * GetViewCreationCommandsOfTable(Oid relationId) { List *views = GetDependingViews(relationId); List *commands = NIL; Oid viewOid = InvalidOid; foreach_declared_oid(viewOid, views) { StringInfo query = makeStringInfo(); /* See comments on CreateMaterializedViewDDLCommand for its limitations */ if (get_rel_relkind(viewOid) == RELKIND_MATVIEW) { ErrorIfMatViewSizeExceedsTheLimit(viewOid); char *matViewCreateCommands = CreateMaterializedViewDDLCommand(viewOid); appendStringInfoString(query, matViewCreateCommands); } else { char *viewCreateCommand = CreateViewDDLCommand(viewOid); appendStringInfoString(query, viewCreateCommand); } char *alterViewCommmand = AlterViewOwnerCommand(viewOid); appendStringInfoString(query, alterViewCommmand); commands = lappend(commands, query->data); } return commands; } /* * WrapTableDDLCommands takes a list of command strings and wraps them * in TableDDLCommand structs. */ static List * WrapTableDDLCommands(List *commandStrings) { List *tableDDLCommands = NIL; char *command = NULL; foreach_declared_ptr(command, commandStrings) { tableDDLCommands = lappend(tableDDLCommands, makeTableDDLCommandString(command)); } return tableDDLCommands; } /* * ErrorIfMatViewSizeExceedsTheLimit takes the oid of a materialized view and errors * out if the size of the matview exceeds the limit set by the GUC * citus.max_matview_size_to_auto_recreate. */ static void ErrorIfMatViewSizeExceedsTheLimit(Oid matViewOid) { if (MaxMatViewSizeToAutoRecreate >= 0) { /* if it's below 0, it means the user has removed the limit */ Datum relSizeDatum = DirectFunctionCall1(pg_total_relation_size, ObjectIdGetDatum(matViewOid)); uint64 matViewSize = DatumGetInt64(relSizeDatum); /* convert from MB to bytes */ uint64 limitSizeInBytes = MaxMatViewSizeToAutoRecreate * 1024L * 1024L; if (matViewSize > limitSizeInBytes) { ereport(ERROR, (errmsg("size of the materialized view %s exceeds " "citus.max_matview_size_to_auto_recreate " "(currently %d MB)", get_rel_name(matViewOid), MaxMatViewSizeToAutoRecreate), errdetail("Citus restricts automatically recreating " "materialized views that are larger than the " "limit, because it could take too long."), errhint( "Consider increasing the size limit by setting " "citus.max_matview_size_to_auto_recreate; " "or you can remove the limit by setting it to -1"))); } } } /* * CreateMaterializedViewDDLCommand creates the command to create materialized view. * Note that this function doesn't support * - Aliases * - Storage parameters * - Tablespace * - WITH [NO] DATA * options for the given materialized view. Parser functions for materialized views * should be added to handle them. * * Related issue: https://github.com/citusdata/citus/issues/5968 */ static char * CreateMaterializedViewDDLCommand(Oid matViewOid) { StringInfo query = makeStringInfo(); char *qualifiedViewName = generate_qualified_relation_name(matViewOid); /* here we need to get the access method of the view to recreate it */ char *accessMethodName = GetAccessMethodForMatViewIfExists(matViewOid); appendStringInfo(query, "CREATE MATERIALIZED VIEW %s ", qualifiedViewName); if (accessMethodName) { appendStringInfo(query, "USING %s ", accessMethodName); } /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. */ int saveNestLevel = PushEmptySearchPath(); /* * Push the transaction snapshot to be able to get vief definition with pg_get_viewdef */ PushActiveSnapshot(GetTransactionSnapshot()); Datum viewDefinitionDatum = DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matViewOid)); char *viewDefinition = TextDatumGetCString(viewDefinitionDatum); PopActiveSnapshot(); PopEmptySearchPath(saveNestLevel); appendStringInfo(query, "AS %s", viewDefinition); return query->data; } /* * ReplaceTable replaces the source table with the target table. * It moves all the rows of the source table to target table with INSERT SELECT. * Changes the dependencies of the sequences owned by source table to target table. * Then drops the source table and renames the target table to source tables name. * * Source and target tables need to be in the same schema and have the same columns. */ void ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands, bool suppressNoticeMessages) { char *sourceName = get_rel_name(sourceId); char *qualifiedSourceName = generate_qualified_relation_name(sourceId); char *qualifiedTargetName = generate_qualified_relation_name(targetId); StringInfo query = makeStringInfo(); if (!PartitionedTable(sourceId) && !IsForeignTable(sourceId)) { if (!suppressNoticeMessages) { ereport(NOTICE, (errmsg("moving the data of %s", qualifiedSourceName))); } if (!HasAnyGeneratedStoredColumns(sourceId)) { /* * Relation has no GENERATED STORED columns, copy the table via plain * "INSERT INTO .. SELECT *"". */ appendStringInfo(query, "INSERT INTO %s SELECT * FROM %s", qualifiedTargetName, qualifiedSourceName); } else { /* * Skip columns having GENERATED ALWAYS AS (...) STORED expressions * since Postgres doesn't allow inserting into such columns. * This is not bad since Postgres would already generate such columns. * Note that here we intentionally don't skip columns having DEFAULT * expressions since user might have inserted non-default values. */ List *nonStoredColumnNameList = GetNonGeneratedStoredColumnNameList(sourceId); char *insertColumnString = StringJoin(nonStoredColumnNameList, ','); appendStringInfo(query, "INSERT INTO %s (%s) OVERRIDING SYSTEM VALUE SELECT %s FROM %s", qualifiedTargetName, insertColumnString, insertColumnString, qualifiedSourceName); } ExecuteQueryViaSPI(query->data, SPI_OK_INSERT); } /* * Modify regular sequence dependencies (sequences marked as DEPENDENCY_AUTO) */ List *ownedSequences = getOwnedSequences_internal(sourceId, 0, DEPENDENCY_AUTO); Oid sequenceOid = InvalidOid; foreach_declared_oid(sequenceOid, ownedSequences) { changeDependencyFor(RelationRelationId, sequenceOid, RelationRelationId, sourceId, targetId); /* * Skip if we cannot sync metadata for target table. * Checking only for the target table is sufficient since we will * anyway drop the source table even if it was a Citus table that * has metadata on MX workers. */ if (ShouldSyncTableMetadata(targetId)) { char *qualifiedSequenceName = generate_qualified_relation_name(sequenceOid); char *workerChangeSequenceDependencyCommand = CreateWorkerChangeSequenceDependencyCommand(qualifiedSequenceName, qualifiedSourceName, qualifiedTargetName); SendCommandToWorkersWithMetadata(workerChangeSequenceDependencyCommand); } else if (ShouldSyncTableMetadata(sourceId)) { /* * We are converting a citus local table to a distributed/reference table, * so we should prevent dropping the sequence on the table. Otherwise, we'd * lose track of the previous changes in the sequence. */ char *command = WorkerDropSequenceDependencyCommand(sourceId); SendCommandToWorkersWithMetadata(command); } } char *justBeforeDropCommand = NULL; foreach_declared_ptr(justBeforeDropCommand, justBeforeDropCommands) { ExecuteQueryViaSPI(justBeforeDropCommand, SPI_OK_UTILITY); } if (!suppressNoticeMessages) { ereport(NOTICE, (errmsg("dropping the old %s", qualifiedSourceName))); } resetStringInfo(query); appendStringInfo(query, "DROP %sTABLE %s CASCADE", IsForeignTable(sourceId) ? "FOREIGN " : "", qualifiedSourceName); ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); if (!suppressNoticeMessages) { ereport(NOTICE, (errmsg("renaming the new table to %s", qualifiedSourceName))); } resetStringInfo(query); appendStringInfo(query, "ALTER TABLE %s RENAME TO %s", qualifiedTargetName, quote_identifier(sourceName)); ExecuteQueryViaSPI(query->data, SPI_OK_UTILITY); } /* * HasAnyGeneratedStoredColumns decides if relation has any columns that we * might need to copy the data of when replacing table. */ static bool HasAnyGeneratedStoredColumns(Oid relationId) { return list_length(GetNonGeneratedStoredColumnNameList(relationId)) > 0; } /* * GetNonGeneratedStoredColumnNameList returns a list of column names for * columns not having GENERATED ALWAYS AS (...) STORED expressions. */ static List * GetNonGeneratedStoredColumnNameList(Oid relationId) { List *nonStoredColumnNameList = NIL; Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); for (int columnIndex = 0; columnIndex < tupleDescriptor->natts; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); if (IsDroppedOrGenerated(currentColumn)) { /* skip dropped or generated columns */ continue; } const char *quotedColumnName = quote_identifier(NameStr(currentColumn->attname)); nonStoredColumnNameList = lappend(nonStoredColumnNameList, pstrdup(quotedColumnName)); } relation_close(relation, NoLock); return nonStoredColumnNameList; } /* * CheckAlterDistributedTableConversionParameters errors for the cases where * alter_distributed_table UDF wouldn't work. */ void CheckAlterDistributedTableConversionParameters(TableConversionState *con) { /* Changing nothing is not allowed */ if (con->distributionColumn == NULL && con->shardCountIsNull && con->colocateWith == NULL && con->cascadeToColocated != CASCADE_TO_COLOCATED_YES) { ereport(ERROR, (errmsg("you have to specify at least one of the " "distribution_column, shard_count or " "colocate_with parameters"))); } /* check if the parameters in this conversion are given and same with table's properties */ bool sameDistColumn = false; if (con->distributionColumn != NULL && equal(con->distributionKey, con->originalDistributionKey)) { sameDistColumn = true; } bool sameShardCount = false; if (!con->shardCountIsNull && con->originalShardCount == con->shardCount) { sameShardCount = true; } bool sameColocateWith = false; if (con->colocateWith != NULL && strcmp(con->colocateWith, "default") != 0 && strcmp(con->colocateWith, "none") != 0) { /* check if already colocated with colocate_with */ Oid colocatedTableOid = InvalidOid; text *colocateWithText = cstring_to_text(con->colocateWith); Oid colocateWithTableOid = ResolveRelationId(colocateWithText, false); foreach_declared_oid(colocatedTableOid, con->colocatedTableList) { if (colocateWithTableOid == colocatedTableOid) { sameColocateWith = true; break; } } /* * already found colocateWithTableOid so let's check if * colocate_with table is a distributed table */ if (!IsCitusTableType(colocateWithTableOid, DISTRIBUTED_TABLE)) { ereport(ERROR, (errmsg("cannot colocate with %s because " "it is not a distributed table", con->colocateWith))); } else if (IsCitusTableType(colocateWithTableOid, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errmsg("cannot colocate with %s because " "it is a single shard distributed table", con->colocateWith))); } } /* shard_count:=0 is not allowed */ if (!con->shardCountIsNull && con->shardCount == 0) { ereport(ERROR, (errmsg("shard_count cannot be 0"), errhint("if you no longer want this to be a " "distributed table you can try " "undistribute_table() function"))); } if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && con->distributionColumn != NULL) { ereport(ERROR, (errmsg("distribution_column cannot be " "cascaded to colocated tables"))); } if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && con->shardCountIsNull && con->colocateWith == NULL) { ereport(ERROR, (errmsg("shard_count or colocate_with is necessary " "for cascading to colocated tables"))); } /* * if every parameter is either not given or already the * same then give error */ if ((con->distributionColumn == NULL || sameDistColumn) && (con->shardCountIsNull || sameShardCount) && (con->colocateWith == NULL || sameColocateWith)) { ereport(ERROR, (errmsg("this call doesn't change any properties of the table"), errhint("check citus_tables view to see current " "properties of the table"))); } if (con->cascadeToColocated == CASCADE_TO_COLOCATED_YES && con->colocateWith != NULL && strcmp(con->colocateWith, "none") == 0) { ereport(ERROR, (errmsg("colocate_with := 'none' cannot be " "cascaded to colocated tables"))); } int colocatedTableCount = list_length(con->colocatedTableList) - 1; if (colocatedTableCount > 0 && !con->shardCountIsNull && !sameShardCount && con->cascadeToColocated == CASCADE_TO_COLOCATED_UNSPECIFIED) { ereport(ERROR, (errmsg("cascade_to_colocated parameter is necessary"), errdetail("this table is colocated with some other tables"), errhint("cascade_to_colocated := false will break the " "current colocation, cascade_to_colocated := true " "will change the shard count of colocated tables " "too."))); } if (con->colocateWith != NULL && strcmp(con->colocateWith, "default") != 0 && strcmp(con->colocateWith, "none") != 0) { text *colocateWithText = cstring_to_text(con->colocateWith); Oid colocateWithTableOid = ResolveRelationId(colocateWithText, false); CitusTableCacheEntry *colocateWithTableCacheEntry = GetCitusTableCacheEntry(colocateWithTableOid); int colocateWithTableShardCount = colocateWithTableCacheEntry->shardIntervalArrayLength; if (!con->shardCountIsNull && con->shardCount != colocateWithTableShardCount) { ereport(ERROR, (errmsg("shard_count cannot be different than the shard " "count of the table in colocate_with"), errhint("if no shard_count is specified shard count " "will be same with colocate_with table's"))); } if (colocateWithTableShardCount != con->originalShardCount) { /* * shardCount is either 0 or already same with colocateWith table's * It's ok to set shardCountIsNull to false because we assume giving a table * to colocate with and no shard count is the same with giving colocate_with * table's shard count if it is different than the original. * So it is almost like the shard_count parameter was given by the user. */ con->shardCount = colocateWithTableShardCount; con->shardCountIsNull = false; } Var *colocateWithPartKey = DistPartitionKey(colocateWithTableOid); if (colocateWithPartKey == NULL) { /* this should never happen */ ereport(ERROR, (errmsg("cannot colocate %s with %s because %s doesn't have a " "distribution column", con->relationName, con->colocateWith, con->colocateWith))); } else if (con->distributionColumn && colocateWithPartKey->vartype != con->distributionKey->vartype) { ereport(ERROR, (errmsg("cannot colocate with %s and change distribution " "column to %s because data type of column %s is " "different than the distribution column of the %s", con->colocateWith, con->distributionColumn, con->distributionColumn, con->colocateWith))); } else if (!con->distributionColumn && colocateWithPartKey->vartype != con->originalDistributionKey->vartype) { ereport(ERROR, (errmsg("cannot colocate with %s because data type of its " "distribution column is different than %s", con->colocateWith, con->relationName))); } else if (con->distributionColumn && colocateWithPartKey->varcollid != con->distributionKey->varcollid) { ereport(ERROR, (errmsg("cannot colocate with %s and change distribution " "column to %s because collation of column %s is " "different than the distribution column of the %s", con->colocateWith, con->distributionColumn, con->distributionColumn, con->colocateWith))); } else if (!con->distributionColumn && colocateWithPartKey->varcollid != con->originalDistributionKey->varcollid ) { ereport(ERROR, (errmsg("cannot colocate with %s because collation of its " "distribution column is different than %s", con->colocateWith, con->relationName))); } } if (!con->suppressNoticeMessages) { /* Notices for no operation UDF calls */ if (sameDistColumn) { ereport(NOTICE, (errmsg("table is already distributed by %s", con->distributionColumn))); } if (sameShardCount) { ereport(NOTICE, (errmsg("shard count of the table is already %d", con->shardCount))); } if (sameColocateWith) { ereport(NOTICE, (errmsg("table is already colocated with %s", con->colocateWith))); } } } /* * CreateWorkerChangeSequenceDependencyCommand creates and returns a * worker_change_sequence_dependency query with the parameters. */ static char * CreateWorkerChangeSequenceDependencyCommand(char *qualifiedSequeceName, char *qualifiedSourceName, char *qualifiedTargetName) { StringInfo query = makeStringInfo(); appendStringInfo(query, "SELECT worker_change_sequence_dependency(%s, %s, %s)", quote_literal_cstr(qualifiedSequeceName), quote_literal_cstr(qualifiedSourceName), quote_literal_cstr(qualifiedTargetName)); return query->data; } /* * GetAccessMethodForMatViewIfExists returns if there's an access method * set to the view with the given oid. Returns NULL otherwise. */ static char * GetAccessMethodForMatViewIfExists(Oid viewOid) { char *accessMethodName = NULL; Relation relation = try_relation_open(viewOid, AccessShareLock); if (relation == NULL) { ereport(ERROR, (errmsg("cannot complete operation " "because no such view exists"))); } Oid accessMethodOid = relation->rd_rel->relam; if (OidIsValid(accessMethodOid)) { accessMethodName = get_am_name(accessMethodOid); } relation_close(relation, NoLock); return accessMethodName; } /* * WillRecreateFKeyToReferenceTable checks if the table of relationId has any foreign * key to a reference table, if conversion will be cascaded to colocated table this function * also checks if any of the colocated tables have a foreign key to a reference table too */ bool WillRecreateFKeyToReferenceTable(Oid relationId, CascadeToColocatedOption cascadeOption) { if (cascadeOption == CASCADE_TO_COLOCATED_NO || cascadeOption == CASCADE_TO_COLOCATED_UNSPECIFIED) { return HasForeignKeyToReferenceTable(relationId); } else if (cascadeOption == CASCADE_TO_COLOCATED_YES) { List *colocatedTableList = ColocatedTableList(relationId); Oid colocatedTableOid = InvalidOid; foreach_declared_oid(colocatedTableOid, colocatedTableList) { if (HasForeignKeyToReferenceTable(colocatedTableOid)) { return true; } } } return false; } /* * WarningsForDroppingForeignKeysWithDistributedTables gives warnings for the * foreign keys that will be dropped because formerly colocated distributed tables * are not colocated. */ void WarningsForDroppingForeignKeysWithDistributedTables(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; List *referencingForeingKeys = GetForeignKeyOids(relationId, flags); flags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; List *referencedForeignKeys = GetForeignKeyOids(relationId, flags); List *foreignKeys = list_concat(referencingForeingKeys, referencedForeignKeys); Oid foreignKeyOid = InvalidOid; foreach_declared_oid(foreignKeyOid, foreignKeys) { ereport(WARNING, (errmsg("foreign key %s will be dropped", get_constraint_name(foreignKeyOid)))); } } /* * ExecuteQueryViaSPI connects to SPI, executes the query and checks if it * returned the OK value and finishes the SPI connection */ void ExecuteQueryViaSPI(char *query, int SPIOK) { int spiResult = SPI_connect(); if (spiResult != SPI_OK_CONNECT) { ereport(ERROR, (errmsg("could not connect to SPI manager"))); } spiResult = SPI_execute(query, false, 0); if (spiResult != SPIOK) { ereport(ERROR, (errmsg("could not run SPI query"))); } spiResult = SPI_finish(); if (spiResult != SPI_OK_FINISH) { ereport(ERROR, (errmsg("could not finish SPI connection"))); } } /* * ExecuteAndLogQueryViaSPI is a wrapper around ExecuteQueryViaSPI, that logs * the query to be executed, with the given log level. */ void ExecuteAndLogQueryViaSPI(char *query, int SPIOK, int logLevel) { ereport(logLevel, (errmsg("executing \"%s\"", query))); ExecuteQueryViaSPI(query, SPIOK); } /* * SwitchToSequentialAndLocalExecutionIfRelationNameTooLong generates the longest shard name * on the shards of a distributed table, and if exceeds the limit switches to sequential and * local execution to prevent self-deadlocks. * * In case of a RENAME, the relation name parameter should store the new table name, so * that the function can generate shard names of the renamed relations */ void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId, char *finalRelationName) { if (!IsCitusTable(relationId)) { return; } if (ShardIntervalCount(relationId) == 0) { /* * Relation has no shards, so we cannot run into "long shard relation * name" issue. */ return; } char *longestShardName = GetLongestShardName(relationId, finalRelationName); bool switchedToSequentialAndLocalExecution = SwitchToSequentialAndLocalExecutionIfShardNameTooLong(finalRelationName, longestShardName); if (switchedToSequentialAndLocalExecution) { return; } if (PartitionedTable(relationId)) { Oid longestNamePartitionId = PartitionWithLongestNameRelationId(relationId); if (!OidIsValid(longestNamePartitionId)) { /* no partitions have been created yet */ return; } char *longestPartitionName = get_rel_name(longestNamePartitionId); char *longestPartitionShardName = NULL; /* * Use the shardId values of the partition if it is distributed, otherwise use * hypothetical values */ if (IsCitusTable(longestNamePartitionId) && ShardIntervalCount(longestNamePartitionId) > 0) { longestPartitionShardName = GetLongestShardName(longestNamePartitionId, longestPartitionName); } else { longestPartitionShardName = GetLongestShardNameForLocalPartition(relationId, longestPartitionName); } SwitchToSequentialAndLocalExecutionIfShardNameTooLong(longestPartitionName, longestPartitionShardName); } } /* * SwitchToSequentialAndLocalExecutionIfShardNameTooLong switches to sequential and local * execution if the shard name is too long. * * returns true if switched to sequential and local execution. */ static bool SwitchToSequentialAndLocalExecutionIfShardNameTooLong(char *relationName, char *longestShardName) { if (strlen(longestShardName) >= NAMEDATALEN - 1) { if (ParallelQueryExecutedInTransaction()) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg( "Shard name (%s) for table (%s) is too long and could " "lead to deadlocks when executed in a transaction " "block after a parallel query", longestShardName, relationName), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else { elog(DEBUG1, "the name of the shard (%s) for relation (%s) is too long, " "switching to sequential and local execution mode to prevent " "self deadlocks", longestShardName, relationName); SetLocalMultiShardModifyModeToSequential(); SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); return true; } } return false; } /* * SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong is a wrapper for new * partitions that will be distributed after attaching to a distributed partitioned table */ void SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid parentRelationId, Oid partitionRelationId) { SwitchToSequentialAndLocalExecutionIfRelationNameTooLong( parentRelationId, get_rel_name(partitionRelationId)); } ================================================ FILE: src/backend/distributed/commands/begin.c ================================================ /*------------------------------------------------------------------------- * * begin.c * Processing of the BEGIN command. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "nodes/parsenodes.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/transaction_management.h" /* * SaveBeginCommandProperties stores the transaction properties passed * via BEGIN. */ void SaveBeginCommandProperties(TransactionStmt *transactionStmt) { DefElem *item = NULL; /* * This loop is similar to the one in standard_ProcessUtility. * * While BEGIN can be quite frequent it will rarely have options set. */ foreach_declared_ptr(item, transactionStmt->options) { A_Const *constant = (A_Const *) item->arg; if (strcmp(item->defname, "transaction_read_only") == 0) { if (intVal(&constant->val) == 1) { BeginXactReadOnly = BeginXactReadOnly_Enabled; } else { BeginXactReadOnly = BeginXactReadOnly_Disabled; } } else if (strcmp(item->defname, "transaction_deferrable") == 0) { if (intVal(&constant->val) == 1) { BeginXactDeferrable = BeginXactDeferrable_Enabled; } else { BeginXactDeferrable = BeginXactDeferrable_Disabled; } } } } ================================================ FILE: src/backend/distributed/commands/call.c ================================================ /*------------------------------------------------------------------------- * * call.c * Commands for distributing CALL for distributed procedures. * * Procedures can be distributed with create_distributed_function. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "catalog/pg_proc.h" #include "commands/defrem.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "tcop/dest.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/function_call_delegation.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/shard_pruning.h" #include "distributed/tuple_destination.h" #include "distributed/version_compat.h" #include "distributed/worker_log_messages.h" #include "distributed/worker_manager.h" /* global variable tracking whether we are in a delegated procedure call */ bool InDelegatedProcedureCall = false; /* * CallDistributedProcedureRemotely calls a stored procedure on the worker if possible. */ bool CallDistributedProcedureRemotely(CallStmt *callStmt, DestReceiver *dest) { FuncExpr *funcExpr = callStmt->funcexpr; Oid functionId = funcExpr->funcid; DistObjectCacheEntry *procedure = LookupDistObjectCacheEntry(ProcedureRelationId, functionId, 0); if (procedure == NULL || !procedure->isDistributed) { return false; } if (IsCitusInternalBackend()) { /* * We are in a citus-initiated backend handling a CALL to a distributed * procedure. That means that this is the delegated call. */ InDelegatedProcedureCall = true; return false; } if (IsMultiStatementTransaction()) { ereport(DEBUG1, (errmsg("cannot push down CALL in multi-statement transaction"))); return false; } Oid colocatedRelationId = ColocatedTableId(procedure->colocationId); if (colocatedRelationId == InvalidOid) { ereport(DEBUG1, (errmsg("stored procedure does not have co-located tables"))); return false; } if (contain_volatile_functions((Node *) funcExpr->args)) { ereport(DEBUG1, (errmsg("arguments in a distributed stored procedure must " "be constant expressions"))); return false; } CitusTableCacheEntry *distTable = GetCitusTableCacheEntry(colocatedRelationId); Var *partitionColumn = distTable->partitionColumn; bool colocatedWithReferenceTable = false; if (IsCitusTableTypeCacheEntry(distTable, REFERENCE_TABLE)) { /* This can happen if colocated with a reference table. Punt for now. */ ereport(DEBUG1, (errmsg( "will push down CALL for reference tables"))); colocatedWithReferenceTable = true; } ShardPlacement *placement = NULL; if (colocatedWithReferenceTable) { placement = ShardPlacementForFunctionColocatedWithReferenceTable(distTable); } else { List *argumentList = NIL; List *namedArgList; int numberOfArgs; Oid *argumentTypes; if (!get_merged_argument_list(callStmt, &namedArgList, &argumentTypes, &argumentList, &numberOfArgs)) { argumentList = funcExpr->args; } placement = ShardPlacementForFunctionColocatedWithDistTable(procedure, argumentList, partitionColumn, distTable, NULL); } /* return if we could not find a placement */ if (placement == NULL) { return false; } WorkerNode *workerNode = FindWorkerNode(placement->nodeName, placement->nodePort); if (workerNode == NULL || !workerNode->hasMetadata || !workerNode->metadataSynced) { ereport(DEBUG1, (errmsg("there is no worker node with metadata"))); return false; } else if (workerNode->groupId == GetLocalGroupId()) { /* * Two reasons for this: * (a) It would lead to infinite recursion as the node would * keep pushing down the procedure as it gets * (b) It doesn't have any value to pushdown as we are already * on the node itself */ ereport(DEBUG1, (errmsg("not pushing down procedure to the same node"))); return false; } ereport(DEBUG1, (errmsg("pushing down the procedure"))); /* build remote command with fully qualified names */ StringInfo callCommand = makeStringInfo(); appendStringInfo(callCommand, "CALL %s", pg_get_rule_expr((Node *) callStmt)); { Tuplestorestate *tupleStore = tuplestore_begin_heap(true, false, work_mem); TupleDesc tupleDesc = CallStmtResultDesc(callStmt); TupleTableSlot *slot = MakeSingleTupleTableSlot(tupleDesc, &TTSOpsMinimalTuple); bool expectResults = true; Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = INVALID_TASK_ID; task->taskType = DDL_TASK; SetTaskQueryString(task, callCommand->data); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NIL; task->anchorShardId = placement->shardId; task->relationShardList = NIL; task->taskPlacementList = list_make1(placement); /* * We are delegating the distributed transaction to the worker, so we * should not run the CALL in a transaction block. */ TransactionProperties xactProperties = { .errorOnAnyFailure = true, .useRemoteTransactionBlocks = TRANSACTION_BLOCKS_DISALLOWED, .requires2PC = false }; EnableWorkerMessagePropagation(); bool localExecutionSupported = true; ExecutionParams *executionParams = CreateBasicExecutionParams( ROW_MODIFY_NONE, list_make1(task), MaxAdaptiveExecutorPoolSize, localExecutionSupported ); executionParams->tupleDestination = CreateTupleStoreTupleDest(tupleStore, tupleDesc); executionParams->expectResults = expectResults; executionParams->xactProperties = xactProperties; executionParams->isUtilityCommand = true; ExecuteTaskListExtended(executionParams); DisableWorkerMessagePropagation(); while (tuplestore_gettupleslot(tupleStore, true, false, slot)) { if (!dest->receiveSlot(slot, dest)) { break; } } /* Don't call tuplestore_end(tupleStore). It'll be freed soon enough in a top level CALL, * & dest->receiveSlot could conceivably rely on slots being long lived. */ } return true; } ================================================ FILE: src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c ================================================ /*------------------------------------------------------------------------- * * cascade_table_operation_for_connected_relations.c * Routines to execute citus table functions (e.g undistribute_table, * create_citus_local_table) by cascading to foreign key connected * relations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/xact.h" #include "catalog/pg_constraint.h" #include "executor/spi.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_protocol.h" static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList); static List * GetPartitionRelationIds(List *relationIdList); static void LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode); static void ErrorIfConvertingMultiLevelPartitionedTable(List *relationIdList); static void DropRelationIdListForeignKeys(List *relationIdList, int fKeyFlags); static List * GetRelationDropFkeyCommands(Oid relationId, int fKeyFlags); static char * GetDropFkeyCascadeCommand(Oid foreignKeyId); static void ExecuteCascadeOperationForRelationIdList(List *relationIdList, CascadeOperationType cascadeOperationType); static void ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation); /* * CascadeOperationForFkeyConnectedRelations is a wrapper function which calls * CascadeOperationForRelationIdList for the foreign key connected relations, for * the given relationId. */ void CascadeOperationForFkeyConnectedRelations(Oid relationId, LOCKMODE lockMode, CascadeOperationType cascadeOperationType) { /* * As we will operate on foreign key connected relations, here we * invalidate foreign key graph to be on the safe side. */ InvalidateForeignKeyGraph(); List *fKeyConnectedRelationIdList = GetForeignKeyConnectedRelationIdList(relationId); /* early exit if there are no connected relations */ if (fKeyConnectedRelationIdList == NIL) { return; } CascadeOperationForRelationIdList(fKeyConnectedRelationIdList, lockMode, cascadeOperationType); } /* * CascadeOperationForRelationIdList executes citus table function specified * by CascadeOperationType argument on each relation in the relationIdList; * Also see CascadeOperationType enum definition for supported * citus table functions. */ void CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, CascadeOperationType cascadeOperationType) { LockRelationsWithLockMode(relationIdList, lockMode); if (cascadeOperationType == CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA || cascadeOperationType == CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA) { /* * In CreateCitusLocalTable function, this check would never error out, * since CreateCitusLocalTable gets called with partition relations, *after* * they are detached. * Instead, here, it would error out if the user tries to convert a multi-level * partitioned table, since partitioned table conversions always go through here. * Also, there can be a multi-level partitioned table, to be cascaded via foreign * keys, and they are hard to detect in CreateCitusLocalTable. * Therefore, we put this check here. */ ErrorIfConvertingMultiLevelPartitionedTable(relationIdList); } /* * Before removing any partition relations, we should error out here if any * of connected relations is a partition table involved in a foreign key that * is not inherited from its parent table. * We should handle this case here as we remove partition relations in this * function before ExecuteCascadeOperationForRelationIdList. */ ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(relationIdList); List *partitonRelationList = GetPartitionRelationIds(relationIdList); /* * Here we generate detach/attach commands, if there are any partition tables * in our "relations-to-cascade" list. */ List *detachPartitionCommands = GenerateDetachPartitionCommandRelationIdList(partitonRelationList); List *attachPartitionCommands = GenerateAttachPartitionCommandRelationIdList(partitonRelationList); /* * Our foreign key subgraph can have distributed tables which might already * be modified in current transaction. So switch to sequential execution * before executing any ddl's to prevent erroring out later in this function. */ EnsureSequentialModeForCitusTableCascadeFunction(relationIdList); /* store foreign key creation commands before dropping them */ List *fKeyCreationCommands = GetFKeyCreationCommandsForRelationIdList(relationIdList); /* * Note that here we only drop referencing foreign keys for each relation. * This is because referenced foreign keys are already captured as other * relations' referencing foreign keys. */ int fKeyFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; DropRelationIdListForeignKeys(relationIdList, fKeyFlags); ExecuteAndLogUtilityCommandList(detachPartitionCommands); ExecuteCascadeOperationForRelationIdList(relationIdList, cascadeOperationType); ExecuteAndLogUtilityCommandList(attachPartitionCommands); /* now recreate foreign keys on tables */ bool skip_validation = true; ExecuteForeignKeyCreateCommandList(fKeyCreationCommands, skip_validation); } /* * GetPartitionRelationIds returns a list of relation id's by picking * partition relation id's from given relationIdList. */ static List * GetPartitionRelationIds(List *relationIdList) { List *partitionRelationIdList = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (PartitionTable(relationId)) { partitionRelationIdList = lappend_oid(partitionRelationIdList, relationId); } } return partitionRelationIdList; } /* * LockRelationsWithLockMode sorts given relationIdList and then acquires * specified lockMode on those relations. */ static void LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode) { Oid relationId; relationIdList = SortList(relationIdList, CompareOids); foreach_declared_oid(relationId, relationIdList) { LockRelationOid(relationId, lockMode); } } /* * ErrorIfConvertingMultiLevelPartitionedTable iterates given relationIdList and checks * if there's a multi-level partitioned table involved or not. As we currently don't * support converting multi-level partitioned tables into Citus Local Tables, * this function errors out for such a case. We detect the multi-level partitioned * table if one of the relations is both partition and partitioned table. */ static void ErrorIfConvertingMultiLevelPartitionedTable(List *relationIdList) { Oid relationId; foreach_declared_oid(relationId, relationIdList) { if (PartitionedTable(relationId) && PartitionTable(relationId)) { Oid parentRelId = PartitionParentOid(relationId); char *parentRelationName = get_rel_name(parentRelId); char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Citus does not support multi-level " "partitioned tables"), errdetail("Relation \"%s\" is partitioned table itself so " "cannot be partition of relation \"%s\".", relationName, parentRelationName))); } } } /* * ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey searches given * relationIdList for a partition relation involved in a foreign key relationship * that is not inherited from its parent and errors out if such a partition * relation exists. */ void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (!PartitionTable(relationId)) { continue; } if (!RelationInvolvedInAnyNonInheritedForeignKeys(relationId)) { continue; } char *partitionRelationQualifiedName = generate_qualified_relation_name(relationId); ereport(ERROR, (errmsg("cannot cascade operation via foreign keys as " "partition table %s involved in a foreign key " "relationship that is not inherited from its " "parent table", partitionRelationQualifiedName), errhint("Remove non-inherited foreign keys from %s and " "try operation again", partitionRelationQualifiedName))); } } /* * EnsureSequentialModeForCitusTableCascadeFunction switches to sequential * execution mode if needed. If it's not possible, then errors out. */ static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList) { if (!RelationIdListHasReferenceTable(relationIdList)) { /* * We don't need to switch to sequential execution if there is no * reference table in our foreign key subgraph. */ return; } if (ParallelQueryExecutedInTransaction()) { ereport(ERROR, (errmsg("cannot execute command because there was a parallel " "operation on a distributed table in transaction"), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } ereport(DEBUG1, (errmsg("switching to sequential query execution mode because the " "operation cascades into distributed tables with foreign " "keys to reference tables"))); SetLocalMultiShardModifyModeToSequential(); } /* * RelationIdListHasReferenceTable returns true if relationIdList has a relation * id that belongs to a reference table. */ bool RelationIdListHasReferenceTable(List *relationIdList) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (IsCitusTableType(relationId, REFERENCE_TABLE)) { return true; } } return false; } /* * GetFKeyCreationCommandsForRelationIdList returns a list of DDL commands to * create foreign keys for each relation in relationIdList. */ List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList) { List *fKeyCreationCommands = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { List *relationFKeyCreationCommands = GetReferencingForeignConstaintCommands(relationId); fKeyCreationCommands = list_concat(fKeyCreationCommands, relationFKeyCreationCommands); } return fKeyCreationCommands; } /* * DropRelationIdListForeignKeys drops foreign keys for each relation in given * relation id list. */ static void DropRelationIdListForeignKeys(List *relationIdList, int fKeyFlags) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { DropRelationForeignKeys(relationId, fKeyFlags); } } /* * DropRelationForeignKeys drops foreign keys where the relation with * relationId is the referencing relation. */ void DropRelationForeignKeys(Oid relationId, int fKeyFlags) { /* * We undistribute citus local tables that are not chained with any reference * tables via foreign keys at the end of the utility hook. * Here we temporarily set the related GUC to off to disable the logic for * internally executed DDL's that might invoke this mechanism unnecessarily. */ bool oldEnableLocalReferenceForeignKeys = EnableLocalReferenceForeignKeys; SetLocalEnableLocalReferenceForeignKeys(false); List *dropFkeyCascadeCommandList = GetRelationDropFkeyCommands(relationId, fKeyFlags); ExecuteAndLogUtilityCommandList(dropFkeyCascadeCommandList); SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys); } /* * SetLocalEnableLocalReferenceForeignKeys is simply a C interface for setting * the following: * SET LOCAL citus.enable_local_reference_table_foreign_keys = 'on'|'off'; */ void SetLocalEnableLocalReferenceForeignKeys(bool state) { char *stateStr = state ? "on" : "off"; set_config_option("citus.enable_local_reference_table_foreign_keys", stateStr, (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } /* * GetRelationDropFkeyCommands returns a list of DDL commands to drop foreign * keys where the relation with relationId is the referencing relation. */ static List * GetRelationDropFkeyCommands(Oid relationId, int fKeyFlags) { List *dropFkeyCascadeCommandList = NIL; List *relationFKeyIdList = GetForeignKeyOids(relationId, fKeyFlags); Oid foreignKeyId; foreach_declared_oid(foreignKeyId, relationFKeyIdList) { char *dropFkeyCascadeCommand = GetDropFkeyCascadeCommand(foreignKeyId); dropFkeyCascadeCommandList = lappend(dropFkeyCascadeCommandList, dropFkeyCascadeCommand); } return dropFkeyCascadeCommandList; } /* * GetDropFkeyCascadeCommand returns DDL command to drop foreign key with * foreignKeyId. */ static char * GetDropFkeyCascadeCommand(Oid foreignKeyId) { /* * As we need to execute ALTER TABLE DROP CONSTRAINT command on * referencing relation, resolve it here. */ HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyId)); Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); Oid relationId = constraintForm->conrelid; char *qualifiedRelationName = generate_qualified_relation_name(relationId); ReleaseSysCache(heapTuple); char *constraintName = get_constraint_name(foreignKeyId); const char *quotedConstraintName = quote_identifier(constraintName); StringInfo dropFkeyCascadeCommand = makeStringInfo(); appendStringInfo(dropFkeyCascadeCommand, "ALTER TABLE %s DROP CONSTRAINT %s CASCADE;", qualifiedRelationName, quotedConstraintName); return dropFkeyCascadeCommand->data; } /* * ExecuteCascadeOperationForRelationIdList executes citus table function * specified by CascadeOperationType argument for given relation id * list. */ static void ExecuteCascadeOperationForRelationIdList(List *relationIdList, CascadeOperationType cascadeOperationType) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { /* * The reason behind skipping certain table types in below loop is * that we support some sort of foreign keys between postgres tables * and citus tables when enable_local_reference_table_foreign_keys is * false or when coordinator is not added to metadata. * * Also, as caller already passed the relations that we should operate * on, we don't cascade via foreign keys here. */ bool cascadeViaForeignKeys = false; switch (cascadeOperationType) { case CASCADE_FKEY_UNDISTRIBUTE_TABLE: { if (IsCitusTable(relationId)) { TableConversionParameters params = { .relationId = relationId, .cascadeViaForeignKeys = cascadeViaForeignKeys, .bypassTenantCheck = false }; UndistributeTable(¶ms); } break; } case CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA: { if (!IsCitusTable(relationId)) { bool autoConverted = false; CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted); } break; } case CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA: { if (!IsCitusTable(relationId)) { bool autoConverted = true; CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted); } break; } default: { /* * This is not expected as other create table functions don't have * cascade option yet. To be on the safe side, error out here. */ ereport(ERROR, (errmsg("citus table function could not be found"))); } } } } /* * ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI is a wrapper function * around ExecuteAndLogQueryViaSPI, that executes view creation commands * with the flag InTableTypeConversionFunctionCall set to true. */ void ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(List *utilityCmdList) { bool oldValue = InTableTypeConversionFunctionCall; InTableTypeConversionFunctionCall = true; MemoryContext savedMemoryContext = CurrentMemoryContext; PG_TRY(); { char *utilityCommand = NULL; foreach_declared_ptr(utilityCommand, utilityCmdList) { /* * CREATE MATERIALIZED VIEW commands need to be parsed/transformed, * which SPI does for us. */ ExecuteAndLogQueryViaSPI(utilityCommand, SPI_OK_UTILITY, DEBUG1); } } PG_CATCH(); { InTableTypeConversionFunctionCall = oldValue; MemoryContextSwitchTo(savedMemoryContext); ErrorData *errorData = CopyErrorData(); FlushErrorState(); if (errorData->elevel != ERROR) { PG_RE_THROW(); } ThrowErrorData(errorData); } PG_END_TRY(); InTableTypeConversionFunctionCall = oldValue; } /* * ExecuteAndLogUtilityCommandList takes a list of utility commands and calls * ExecuteAndLogUtilityCommand function for each of them. */ void ExecuteAndLogUtilityCommandList(List *utilityCmdList) { char *utilityCommand = NULL; foreach_declared_ptr(utilityCommand, utilityCmdList) { ExecuteAndLogUtilityCommand(utilityCommand); } } /* * ExecuteAndLogUtilityCommand takes a utility command and logs it in DEBUG4 log level. * Then, parses and executes it via CitusProcessUtility. */ void ExecuteAndLogUtilityCommand(const char *commandString) { ereport(DEBUG4, (errmsg("executing \"%s\"", commandString))); ExecuteUtilityCommand(commandString); } /* * ExecuteForeignKeyCreateCommandList takes a list of foreign key creation ddl commands * and calls ExecuteAndLogForeignKeyCreateCommand function for each of them. */ void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, bool skip_validation) { char *ddlCommand = NULL; foreach_declared_ptr(ddlCommand, ddlCommandList) { ExecuteForeignKeyCreateCommand(ddlCommand, skip_validation); } } /* * ExecuteForeignKeyCreateCommand takes a foreign key creation command * and logs it in DEBUG4 log level. * * Then, parses, sets skip_validation flag to considering the input and * executes the command via CitusProcessUtility. */ static void ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation) { ereport(DEBUG4, (errmsg("executing foreign key create command \"%s\"", commandString))); Node *parseTree = ParseTreeNode(commandString); /* * We might have thrown an error if IsA(parseTree, AlterTableStmt), * but that doesn't seem to provide any benefits, so assertion is * fine for this case. */ Assert(IsA(parseTree, AlterTableStmt)); if (skip_validation && IsA(parseTree, AlterTableStmt)) { SkipForeignKeyValidationIfConstraintIsFkey((AlterTableStmt *) parseTree, true); ereport(DEBUG4, (errmsg("skipping validation for foreign key create " "command \"%s\"", commandString))); } ProcessUtilityParseTree(parseTree, commandString, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } ================================================ FILE: src/backend/distributed/commands/citus_add_local_table_to_metadata.c ================================================ /*------------------------------------------------------------------------- * * citus_add_local_table_to_metadata.c * * This file contains functions to add local table to citus metadata. * * A local table added to metadata is composed of a shell relation to wrap the * the regular postgres relation as its coordinator local shard. * * As we want to hide "citus local table" concept from users, we renamed * udf and re-branded that concept as "local tables added to metadata". * Note that we might still call this local table concept as "citus local" in * many places of the code base. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "catalog/pg_constraint.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" #include "foreign/foreign.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/namespace_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" /* * Global variable for the GUC citus.use_citus_managed_tables. * This is used after every CREATE TABLE statement in utility_hook.c * If this variable is set to true, we add all created tables to metadata. */ bool AddAllLocalTablesToMetadata = false; static void citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeignKeys); static void ErrorIfAddingPartitionTableToMetadata(Oid relationId); static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation); static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId); static void EnsureIfPostgresFdwHasTableName(Oid relationId); static void ErrorIfOptionListHasNoTableName(List *optionList); static void NoticeIfAutoConvertingLocalTables(bool autoConverted, Oid relationId); static CascadeOperationType GetCascadeTypeForCitusLocalTables(bool autoConverted); static List * GetShellTableDDLEventsForCitusLocalTable(Oid relationId); static uint64 ConvertLocalTableToShard(Oid relationId); static void RenameRelationToShardRelation(Oid shellRelationId, uint64 shardId); static void RenameShardRelationConstraints(Oid shardRelationId, uint64 shardId); static List * GetConstraintNameList(Oid relationId); static char * GetRenameShardConstraintCommand(Oid relationId, char *constraintName, uint64 shardId); static void RenameShardRelationIndexes(Oid shardRelationId, uint64 shardId); static void RenameShardRelationStatistics(Oid shardRelationId, uint64 shardId); static char * GetDropTriggerCommand(Oid relationId, char *triggerName); static char * GetRenameShardIndexCommand(Oid indexOid, uint64 shardId); static char * GetRenameShardStatsCommand(char *statSchema, char *statsName, char *statsNameWithShardId); static void RenameShardRelationNonTruncateTriggers(Oid shardRelationId, uint64 shardId); static char * GetRenameShardTriggerCommand(Oid shardRelationId, char *triggerName, uint64 shardId); static void DropRelationTruncateTriggers(Oid relationId); static char * GetDropTriggerCommand(Oid relationId, char *triggerName); static void DropViewsOnTable(Oid relationId); static void DropIdentitiesOnTable(Oid relationId); static void DropTableFromPublications(Oid relationId); static List * GetRenameStatsCommandList(List *statsOidList, uint64 shardId); static List * ReversedOidList(List *oidList); static void AppendExplicitIndexIdsToList(Form_pg_index indexForm, List **explicitIndexIdList, int flags); static void DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId, Oid targetRelationId); static void DropDefaultColumnDefinition(Oid relationId, char *columnName); static void TransferSequenceOwnership(Oid ownedSequenceId, Oid targetRelationId, char *columnName); static void InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId, bool autoConverted); static void FinalizeCitusLocalTableCreation(Oid relationId); PG_FUNCTION_INFO_V1(citus_add_local_table_to_metadata); PG_FUNCTION_INFO_V1(create_citus_local_table); PG_FUNCTION_INFO_V1(remove_local_tables_from_metadata); /* * citus_add_local_table_to_metadata creates a citus local table from the table with * relationId by executing the internal method CreateCitusLocalTable. * (See CreateCitusLocalTable function's comment.) */ Datum citus_add_local_table_to_metadata(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); bool cascadeViaForeignKeys = PG_GETARG_BOOL(1); citus_add_local_table_to_metadata_internal(relationId, cascadeViaForeignKeys); PG_RETURN_VOID(); } /* * citus_add_local_table_to_metadata_internal is the internal method for * citus_add_local_table_to_metadata udf. */ static void citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeignKeys) { CheckCitusVersion(ERROR); /* enable citus_add_local_table_to_metadata on an empty node */ InsertCoordinatorIfClusterEmpty(); bool autoConverted = false; CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted); } /* * create_citus_local_table is a wrapper function for old name of * of citus_add_local_table_to_metadata. * * The only reason for having this udf in citus binary is to make * multi_extension test happy as it uses this udf when testing the * downgrade scenario from 9.5 to 9.4. */ Datum create_citus_local_table(PG_FUNCTION_ARGS) { ereport(NOTICE, (errmsg("create_citus_local_table is deprecated in favour of " "citus_add_local_table_to_metadata"))); Oid relationId = PG_GETARG_OID(0); /* * create_citus_local_table doesn't have cascadeViaForeignKeys option, * so we can't directly call citus_add_local_table_to_metadata udf itself * since create_citus_local_table doesn't specify cascadeViaForeignKeys. */ bool cascadeViaForeignKeys = false; citus_add_local_table_to_metadata_internal(relationId, cascadeViaForeignKeys); PG_RETURN_VOID(); } /* * remove_local_tables_from_metadata undistributes citus local * tables that are not chained with any reference tables via foreign keys. */ Datum remove_local_tables_from_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); UndistributeDisconnectedCitusLocalTables(); PG_RETURN_VOID(); } /* * CreateCitusLocalTable is the internal method that creates a citus table * from the table with relationId. The created table would have the following * properties: * - it will have only one shard, * - its distribution method will be DISTRIBUTE_BY_NONE, * - its replication model will be REPLICATION_MODEL_STREAMING, * - its replication factor will be set to 1. * Similar to reference tables, it has only 1 placement. In addition to that, that * single placement is only allowed to be on the coordinator. */ void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConverted) { /* * These checks should be done before acquiring any locks on relation. * This is because we don't allow creating citus local tables in worker * nodes and we don't want to acquire any locks on a table if we are not * the owner of it. */ EnsureCoordinator(); EnsureTableOwner(relationId); /* enable citus_add_local_table_to_metadata on an empty node */ InsertCoordinatorIfClusterEmpty(); /* * Creating Citus local tables relies on functions that accesses * shards locally (e.g., ExecuteAndLogUtilityCommand()). As long as * we don't teach those functions to access shards remotely, we * cannot relax this check. */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); if (!autoConverted && IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * We allow users to mark local tables already added to metadata * as "autoConverted = false". * If the user called citus_add_local_table_to_metadata for a table that is * already added to metadata, we should mark this one and connected relations * as auto-converted = false. */ UpdateAutoConvertedForConnectedRelations(list_make1_oid(relationId), autoConverted); return; } /* * Lock target relation with an AccessExclusiveLock as we don't want * multiple backends manipulating this relation. We could actually simply * lock the relation without opening it. However, we also want to check * if the relation does not exist or dropped by another backend. Also, * we open the relation with try_relation_open instead of relation_open * to give a nice error in case the table is dropped by another backend. */ LOCKMODE lockMode = AccessExclusiveLock; Relation relation = try_relation_open(relationId, lockMode); ErrorIfUnsupportedCreateCitusLocalTable(relation); ErrorIfAddingPartitionTableToMetadata(relationId); /* * We immediately close relation with NoLock right after opening it. This is * because, in this function, we may execute ALTER TABLE commands modifying * relation's column definitions and postgres does not allow us to do so when * the table is still open. (See the postgres function CheckTableNotInUse for * more information) */ relation_close(relation, NoLock); NoticeIfAutoConvertingLocalTables(autoConverted, relationId); if (TableHasExternalForeignKeys(relationId)) { if (!cascadeViaForeignKeys) { /* * We do not allow creating citus local table if the table is involved in a * foreign key relationship with "any other table", unless the option * cascadeViaForeignKeys is given true. * Note that we allow self references. */ char *qualifiedRelationName = generate_qualified_relation_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s is involved in a foreign key " "relationship with another table", qualifiedRelationName), errhint("Use cascade_via_foreign_keys option to add " "all the relations involved in a foreign key " "relationship with %s to citus metadata by " "executing SELECT citus_add_local_table_to_metadata($$%s$$, " "cascade_via_foreign_keys=>true)", qualifiedRelationName, qualifiedRelationName))); } CascadeOperationType cascadeType = GetCascadeTypeForCitusLocalTables(autoConverted); /* * By acquiring AccessExclusiveLock, make sure that no modifications happen * on the relations. */ CascadeOperationForFkeyConnectedRelations(relationId, lockMode, cascadeType); /* * We converted every foreign key connected table in our subgraph * including itself to a citus local table, so return here. */ return; } if (PartitionedTable(relationId)) { List *relationList = PartitionList(relationId); if (list_length(relationList) > 0) { relationList = lappend_oid(relationList, relationId); CascadeOperationType cascadeType = GetCascadeTypeForCitusLocalTables(autoConverted); CascadeOperationForRelationIdList(relationList, AccessExclusiveLock, cascadeType); return; } } ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*tableAddress, RelationRelationId, relationId); /* * Ensure that the sequences used in column defaults of the table * have proper types */ EnsureRelationHasCompatibleSequenceTypes(relationId); /* * Ensure dependencies exist as we will create shell table on the other nodes * in the MX case. */ EnsureAllObjectDependenciesExistOnAllNodes(list_make1(tableAddress)); /* * Make sure that existing reference tables have been replicated to all * the nodes such that we can create foreign keys and joins work * immediately after creation. */ EnsureReferenceTablesExistOnAllNodes(); List *shellTableDDLEvents = GetShellTableDDLEventsForCitusLocalTable(relationId); List *tableViewCreationCommands = GetViewCreationCommandsOfTable(relationId); bool isAdd = true; List *alterPublicationCommands = GetAlterPublicationDDLCommandsForTable(relationId, isAdd); char *relationName = get_rel_name(relationId); Oid relationSchemaId = get_rel_namespace(relationId); /* * Drop identities before local shard conversion since the shell table owns * identities */ DropIdentitiesOnTable(relationId); /* * We do not want the shard to be in the publication (subscribers are * unlikely to recognize it). */ DropTableFromPublications(relationId); /* below we convert relation with relationId to the shard relation */ uint64 shardId = ConvertLocalTableToShard(relationId); /* * As we retrieved the DDL commands necessary to create the shell table * from scratch, below we simply recreate the shell table executing them * via process utility. */ ExecuteAndLogUtilityCommandList(shellTableDDLEvents); /* * Execute the view creation commands with the shell table. * Views will be distributed via FinalizeCitusLocalTableCreation below. */ ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(tableViewCreationCommands); /* * Execute the publication creation commands with the shell table. */ ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(alterPublicationCommands); /* * Set shellRelationId as the relation with relationId now points * to the shard relation. */ Oid shardRelationId = relationId; Oid shellRelationId = get_relname_relid(relationName, relationSchemaId); /* assert that we created the shell table properly in the same schema */ Assert(OidIsValid(shellRelationId)); /* * Move sequence ownerships from shard table to shell table and also drop * DEFAULT expressions based on sequences from shard relation as we should * evaluate such columns in shell table when needed. */ DropNextValExprsAndMoveOwnedSeqOwnerships(shardRelationId, shellRelationId); InsertMetadataForCitusLocalTable(shellRelationId, shardId, autoConverted); FinalizeCitusLocalTableCreation(shellRelationId); } /* * CreateCitusLocalTablePartitionOf generates and executes the necessary commands * to create a table as partition of a partitioned Citus Local Table. * The conversion is done by CreateCitusLocalTable. */ void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, Oid parentRelationId) { if (createStatement->partspec) { /* * Since partspec represents "PARTITION BY" clause, being different than * NULL means that given CreateStmt attempts to create a parent table * at the same time. That means multi-level partitioning within this * function's context. We don't support this currently. */ char *parentRelationName = get_rel_name(parentRelationId); char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("distributing multi-level partitioned tables " "is not supported"), errdetail("Relation \"%s\" is partitioned table itself " "and it is also partition of relation \"%s\".", relationName, parentRelationName))); } /* * Since the shell table for the partition is not created yet on MX workers, * we should disable DDL propagation before the DETACH command, to avoid * getting an error on the worker. */ List *detachCommands = list_make3(DISABLE_DDL_PROPAGATION, GenerateDetachPartitionCommand(relationId), ENABLE_DDL_PROPAGATION); char *attachCommand = GenerateAlterTableAttachPartitionCommand(relationId); ExecuteAndLogUtilityCommandList(detachCommands); int fKeyFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; /* * When cascadeViaForeignKeys is false, CreateCitusLocalTable doesn't expect * any foreign keys on given relation. Note that we don't want to pass * cascadeViaForeignKeys to be true here since we don't already allow non-inherited * foreign keys on child relations, and for the inherited ones, we should have already * cascaded to the other relations when creating a citus local table from parent. * * For this reason, we drop inherited foreign keys here, they'll anyway get created * again with the attach command */ DropRelationForeignKeys(relationId, fKeyFlags); /* get the autoconverted field from the parent */ CitusTableCacheEntry *entry = GetCitusTableCacheEntry(parentRelationId); bool cascade = false; bool autoConverted = entry->autoConverted; CreateCitusLocalTable(relationId, cascade, autoConverted); ExecuteAndLogUtilityCommand(attachCommand); } /* * ErrorIfAddingPartitionTableToMetadata errors out if we try to create the * citus local table from a partition table. */ static void ErrorIfAddingPartitionTableToMetadata(Oid relationId) { if (PartitionTable(relationId)) { /* * We do not allow converting only partitions into Citus Local Tables. * Users should call the UDF citus_add_local_table_to_metadata with the * parent table; then the whole partitioned table will be converted. */ char *relationName = get_rel_name(relationId); Oid parentRelationId = PartitionParentOid(relationId); char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errmsg("cannot add local table %s to metadata since " "it is a partition of %s. Instead, add the parent " "table %s to metadata.", relationName, parentRelationName, parentRelationName))); } } /* * ErrorIfUnsupportedCreateCitusLocalTable errors out if we cannot create the * citus local table from the relation. */ static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation) { if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("cannot add local table to metadata, relation does " "not exist"))); } ErrorIfTableIsACatalogTable(relation); Oid relationId = relation->rd_id; ErrorIfCoordinatorNotAddedAsWorkerNode(); ErrorIfUnsupportedCitusLocalTableKind(relationId); EnsureTableNotDistributed(relationId); ErrorIfRelationHasUnsupportedTrigger(relationId); /* * Error out with a hint if the foreign table is using postgres_fdw and * the option table_name is not provided. * Citus relays all the Citus local foreign table logic to the placement of the * Citus local table. If table_name is NOT provided, Citus would try to talk to * the foreign postgres table over the shard's table name, which would not exist * on the remote server. */ EnsureIfPostgresFdwHasTableName(relationId); /* * When creating other citus table types, we don't need to check that case as * EnsureTableNotDistributed already errors out if the given relation implies * a citus table. However, as we don't mark the relation as citus table, i.e we * do not use the relation with relationId as the shell relation, parallel * citus_add_local_table_to_metadata executions would not error out for that relation. * Hence we need to error out for shard relations too. */ ErrorIfRelationIsAKnownShard(relationId); /* we do not support policies in citus community */ ErrorIfUnsupportedPolicy(relation); } /* * ServerUsesPostgresFdw gets a foreign server Oid and returns true if the FDW that * the server depends on is postgres_fdw. Returns false otherwise. */ bool ServerUsesPostgresFdw(Oid serverId) { ForeignServer *server = GetForeignServer(serverId); ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid); if (strcmp(fdw->fdwname, "postgres_fdw") == 0) { return true; } return false; } /* * EnsureIfPostgresFdwHasTableName errors out with a hint if the foreign table is using postgres_fdw and * the option table_name is not provided. */ static void EnsureIfPostgresFdwHasTableName(Oid relationId) { char relationKind = get_rel_relkind(relationId); if (relationKind == RELKIND_FOREIGN_TABLE) { ForeignTable *foreignTable = GetForeignTable(relationId); if (ServerUsesPostgresFdw(foreignTable->serverid)) { ErrorIfOptionListHasNoTableName(foreignTable->options); } } } /* * ErrorIfOptionListHasNoTableName gets an option list (DefElem) and errors out * if the list does not contain a table_name element. */ static void ErrorIfOptionListHasNoTableName(List *optionList) { char *table_nameString = "table_name"; DefElem *option = NULL; foreach_declared_ptr(option, optionList) { char *optionName = option->defname; if (strcmp(optionName, table_nameString) == 0) { return; } } ereport(ERROR, (errmsg( "table_name option must be provided when using postgres_fdw with Citus"), errhint("Provide the option \"table_name\" with value target table's" " name"))); } /* * ForeignTableDropsTableNameOption returns true if given option list contains * (DROP table_name). */ bool ForeignTableDropsTableNameOption(List *optionList) { char *table_nameString = "table_name"; DefElem *option = NULL; foreach_declared_ptr(option, optionList) { char *optionName = option->defname; DefElemAction optionAction = option->defaction; if (strcmp(optionName, table_nameString) == 0 && optionAction == DEFELEM_DROP) { return true; } } return false; } /* * ErrorIfUnsupportedCitusLocalTableKind errors out if the relation kind of * relation with relationId is not supported for citus local table creation. */ static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId) { const char *relationName = get_rel_name(relationId); if (IsChildTable(relationId) || IsParentTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add local table \"%s\" to metadata, local tables " "added to metadata cannot be involved in inheritance " "relationships", relationName))); } char relationKind = get_rel_relkind(relationId); if (!(relationKind == RELKIND_RELATION || relationKind == RELKIND_FOREIGN_TABLE || relationKind == RELKIND_PARTITIONED_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add local table \"%s\" to metadata, only regular " "tables, partitioned tables and foreign tables" " can be added to citus metadata ", relationName))); } if (get_rel_persistence(relationId) == RELPERSISTENCE_TEMP) { /* * Currently, we use citus local tables only to support foreign keys * between local tables and reference tables. Citus already doesn't * support creating reference tables from temp tables. * So now we are creating a citus local table from a temp table that * has a foreign key from/to a reference table with persistent storage. * In that case, we want to give the same error as postgres would do. */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("constraints on temporary tables may reference only " "temporary tables"))); } } /* * NoticeIfAutoConvertingLocalTables logs a NOTICE message to inform the user that we are * automatically adding local tables to metadata. The user should know that this table * will be undistributed automatically, if it gets disconnected from reference table(s). */ static void NoticeIfAutoConvertingLocalTables(bool autoConverted, Oid relationId) { if (autoConverted && ShouldEnableLocalReferenceForeignKeys()) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); /* * When foreign keys between reference tables and postgres tables are * enabled, we automatically undistribute citus local tables that are * not chained with any reference tables back to postgres tables. * So give a warning to user for that. */ ereport(NOTICE, (errmsg("local tables that are added to metadata automatically " "by citus, but not chained with reference tables via " "foreign keys might be automatically converted back to " "postgres tables"), errhint("Executing citus_add_local_table_to_metadata($$%s$$) " "prevents this for the given relation, and all of the " "connected relations", qualifiedRelationName))); } } /* * GetCascadeTypeForCitusLocalTables returns CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA * if autoConverted is true. Returns CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA otherwise. */ static CascadeOperationType GetCascadeTypeForCitusLocalTables(bool autoConverted) { if (autoConverted) { return CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA; } return CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA; } /* * UpdateAutoConvertedForConnectedRelations updates the autoConverted field on * pg_dist_partition for the foreign key connected relations of the given relations. * Sets it to given autoConverted value for all of the connected relations. * We don't need to update partition relations separately, since the foreign key * graph already includes them, as they have the same (inherited) fkeys as their parents. */ void UpdateAutoConvertedForConnectedRelations(List *relationIds, bool autoConverted) { InvalidateForeignKeyGraph(); List *relationIdList = NIL; Oid relid = InvalidOid; foreach_declared_oid(relid, relationIds) { List *connectedRelations = GetForeignKeyConnectedRelationIdList(relid); relationIdList = list_concat_unique_oid(relationIdList, connectedRelations); } relationIdList = SortList(relationIdList, CompareOids); foreach_declared_oid(relid, relationIdList) { UpdatePgDistPartitionAutoConverted(relid, autoConverted); } } /* * GetShellTableDDLEventsForCitusLocalTable returns a list of DDL commands * to create the shell table from scratch. */ static List * GetShellTableDDLEventsForCitusLocalTable(Oid relationId) { /* * As we don't allow foreign keys with other tables initially, below we * only pick self-referencing foreign keys. */ List *foreignConstraintCommands = GetReferencingForeignConstaintCommands(relationId); /* * Include DEFAULT clauses for columns getting their default values from * a sequence. */ IncludeSequenceDefaults includeSequenceDefaults = NEXTVAL_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = INCLUDE_IDENTITY; bool creatingShellTableOnRemoteNode = false; List *tableDDLCommands = GetFullTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, creatingShellTableOnRemoteNode); List *shellTableDDLEvents = NIL; TableDDLCommand *tableDDLCommand = NULL; foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); shellTableDDLEvents = lappend(shellTableDDLEvents, GetTableDDLCommand(tableDDLCommand)); } shellTableDDLEvents = list_concat(shellTableDDLEvents, foreignConstraintCommands); return shellTableDDLEvents; } /* * ConvertLocalTableToShard first acquires a shardId and then converts the * given relation with relationId to the shard relation with shardId. That * means, this function suffixes shardId to: * - relation name, * - all the objects "defined on" the relation. * After converting the given relation, returns the acquired shardId. */ static uint64 ConvertLocalTableToShard(Oid relationId) { uint64 shardId = GetNextShardId(); RenameRelationToShardRelation(relationId, shardId); RenameShardRelationConstraints(relationId, shardId); RenameShardRelationIndexes(relationId, shardId); RenameShardRelationStatistics(relationId, shardId); /* * We do not create truncate triggers on shard relation. This is * because truncate triggers are fired by utility hook and we would * need to disable them to prevent executing them twice if we don't * drop the trigger on shard relation. */ DropRelationTruncateTriggers(relationId); /* drop views that depend on the shard table */ DropViewsOnTable(relationId); /* * We create INSERT|DELETE|UPDATE triggers on shard relation too. * This is because citus prevents postgres executor to fire those * triggers. So, here we suffix such triggers on shard relation * with shardId. */ RenameShardRelationNonTruncateTriggers(relationId, shardId); return shardId; } /* * RenameRelationToShardRelation appends given shardId to the end of the name * of relation with shellRelationId. */ static void RenameRelationToShardRelation(Oid shellRelationId, uint64 shardId) { char *qualifiedShellRelationName = generate_qualified_relation_name(shellRelationId); char *shellRelationName = get_rel_name(shellRelationId); char *shardRelationName = pstrdup(shellRelationName); AppendShardIdToName(&shardRelationName, shardId); const char *quotedShardRelationName = quote_identifier(shardRelationName); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER TABLE %s RENAME TO %s;", qualifiedShellRelationName, quotedShardRelationName); ExecuteAndLogUtilityCommand(renameCommand->data); } /* * RenameShardRelationConstraints appends given shardId to the end of the name * of constraints "defined on" the relation with shardRelationId. This function * utilizes GetConstraintNameList to pick the constraints to be renamed, * see more details in function's comment. */ static void RenameShardRelationConstraints(Oid shardRelationId, uint64 shardId) { List *constraintNameList = GetConstraintNameList(shardRelationId); char *constraintName = NULL; foreach_declared_ptr(constraintName, constraintNameList) { const char *commandString = GetRenameShardConstraintCommand(shardRelationId, constraintName, shardId); ExecuteAndLogUtilityCommand(commandString); } } /* * GetConstraintNameList returns a list of constraint names "defined on" the * relation with relationId. Those constraints can be: * - "check" constraints or, * - "primary key" constraints or, * - "unique" constraints or, * - "trigger" constraints or, * - "exclusion" constraints or, * - "foreign key" constraints in which the relation is the "referencing" * relation (including the self-referencing foreign keys). */ static List * GetConstraintNameList(Oid relationId) { List *constraintNameList = NIL; int scanKeyCount = 1; ScanKeyData scanKey[1]; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); bool useIndex = true; SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, ConstraintRelidTypidNameIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); char *constraintName = NameStr(constraintForm->conname); constraintNameList = lappend(constraintNameList, pstrdup(constraintName)); heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgConstraint, NoLock); return constraintNameList; } /* * GetRenameShardConstraintCommand returns DDL command to append given * shardId to the constrant with constraintName on the relation with * relationId. */ static char * GetRenameShardConstraintCommand(Oid relationId, char *constraintName, uint64 shardId) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); char *shardConstraintName = pstrdup(constraintName); AppendShardIdToName(&shardConstraintName, shardId); const char *quotedShardConstraintName = quote_identifier(shardConstraintName); const char *quotedConstraintName = quote_identifier(constraintName); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER TABLE %s RENAME CONSTRAINT %s TO %s;", qualifiedRelationName, quotedConstraintName, quotedShardConstraintName); return renameCommand->data; } /* * RenameShardRelationIndexes appends given shardId to the end of the names * of shard relation indexes except the ones that are already renamed via * RenameShardRelationConstraints. This function utilizes * GetExplicitIndexOidList to pick the indexes to be renamed, see more * details in function's comment. */ static void RenameShardRelationIndexes(Oid shardRelationId, uint64 shardId) { List *indexOidList = GetExplicitIndexOidList(shardRelationId); Oid indexOid = InvalidOid; foreach_declared_oid(indexOid, indexOidList) { const char *commandString = GetRenameShardIndexCommand(indexOid, shardId); ExecuteAndLogUtilityCommand(commandString); } } /* * GetRenameShardIndexCommand returns DDL command to append given shardId to * the index with indexName. */ static char * GetRenameShardIndexCommand(Oid indexOid, uint64 shardId) { char *indexName = get_rel_name(indexOid); char *shardIndexName = pstrdup(indexName); AppendShardIdToName(&shardIndexName, shardId); const char *quotedShardIndexName = quote_identifier(shardIndexName); const char *quotedIndexName = generate_qualified_relation_name(indexOid); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER INDEX %s RENAME TO %s;", quotedIndexName, quotedShardIndexName); return renameCommand->data; } /* * RenameShardRelationStatistics appends given shardId to the end of the names * of shard relation statistics. This function utilizes GetExplicitStatsNameList * to pick the statistics to be renamed, see more details in function's comment. */ static void RenameShardRelationStatistics(Oid shardRelationId, uint64 shardId) { Relation shardRelation = RelationIdGetRelation(shardRelationId); if (!RelationIsValid(shardRelation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", shardRelationId))); } List *statsOidList = RelationGetStatExtList(shardRelation); RelationClose(shardRelation); List *statsCommandList = GetRenameStatsCommandList(statsOidList, shardId); char *command = NULL; foreach_declared_ptr(command, statsCommandList) { ExecuteAndLogUtilityCommand(command); } } /* * GetRenameShardStatsCommand returns DDL command to append given shardId to * the statistics with statName. */ static char * GetRenameShardStatsCommand(char *statSchema, char *statsName, char *statsNameWithShardId) { const char *quotedStatsNameWithShardId = quote_identifier(statsNameWithShardId); char *qualifiedStatsName = quote_qualified_identifier(statSchema, statsName); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER STATISTICS %s RENAME TO %s;", qualifiedStatsName, quotedStatsNameWithShardId); return renameCommand->data; } /* * RenameShardRelationNonTruncateTriggers appends given shardId to the end of * the names of shard relation INSERT/DELETE/UPDATE triggers that are explicitly * created. */ static void RenameShardRelationNonTruncateTriggers(Oid shardRelationId, uint64 shardId) { List *triggerIdList = GetExplicitTriggerIdList(shardRelationId); Oid triggerId = InvalidOid; foreach_declared_oid(triggerId, triggerIdList) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); if (!TRIGGER_FOR_TRUNCATE(triggerForm->tgtype)) { char *triggerName = NameStr(triggerForm->tgname); char *commandString = GetRenameShardTriggerCommand(shardRelationId, triggerName, shardId); ExecuteAndLogUtilityCommand(commandString); } heap_freetuple(triggerTuple); } } /* * GetRenameShardTriggerCommand returns DDL command to append given shardId to * the trigger with triggerName. */ static char * GetRenameShardTriggerCommand(Oid shardRelationId, char *triggerName, uint64 shardId) { char *qualifiedShardRelationName = generate_qualified_relation_name(shardRelationId); char *shardTriggerName = pstrdup(triggerName); AppendShardIdToName(&shardTriggerName, shardId); const char *quotedShardTriggerName = quote_identifier(shardTriggerName); const char *quotedTriggerName = quote_identifier(triggerName); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER TRIGGER %s ON %s RENAME TO %s;", quotedTriggerName, qualifiedShardRelationName, quotedShardTriggerName); return renameCommand->data; } /* * DropRelationTruncateTriggers drops TRUNCATE triggers that are explicitly * created on relation with relationId. */ static void DropRelationTruncateTriggers(Oid relationId) { List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; foreach_declared_oid(triggerId, triggerIdList) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); if (TRIGGER_FOR_TRUNCATE(triggerForm->tgtype)) { char *triggerName = NameStr(triggerForm->tgname); char *commandString = GetDropTriggerCommand(relationId, triggerName); ExecuteAndLogUtilityCommand(commandString); } heap_freetuple(triggerTuple); } } /* * GetDropTriggerCommand returns DDL command to drop the trigger with triggerName * on relationId. */ static char * GetDropTriggerCommand(Oid relationId, char *triggerName) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); const char *quotedTriggerName = quote_identifier(triggerName); /* * In postgres, the only possible object type that may depend on a trigger * is the "constraint" object implied by the trigger itself if it is a * constraint trigger, and it would be an internal dependency so it could * be dropped without using CASCADE. Other than this, it is also possible * to define dependencies on trigger via recordDependencyOn api by other * extensions. We don't handle those kind of dependencies, we just drop * them with CASCADE. */ StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "DROP TRIGGER %s ON %s CASCADE;", quotedTriggerName, qualifiedRelationName); return dropCommand->data; } /* * DropIdentitiesOnTable drops the identities that depend on the given relation. */ static void DropIdentitiesOnTable(Oid relationId) { Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); List *dropCommandList = NIL; for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); char *columnName = NameStr(attributeForm->attname); if (attributeForm->attidentity) { char *qualifiedTableName = generate_qualified_relation_name(relationId); StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "ALTER TABLE %s ALTER %s DROP IDENTITY", qualifiedTableName, columnName); dropCommandList = lappend(dropCommandList, dropCommand->data); } } relation_close(relation, NoLock); char *dropCommand = NULL; foreach_declared_ptr(dropCommand, dropCommandList) { /* * We need to disable/enable ddl propagation for this command, to prevent * sending unnecessary ALTER COLUMN commands for partitions, to MX workers. */ ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, dropCommand, ENABLE_DDL_PROPAGATION)); } } /* * DropTableFromPublications drops the table from all of its publications. */ static void DropTableFromPublications(Oid relationId) { bool isAdd = false; List *alterPublicationCommands = GetAlterPublicationDDLCommandsForTable(relationId, isAdd); ExecuteAndLogUtilityCommandList(alterPublicationCommands); } /* * DropViewsOnTable drops the views that depend on the given relation. */ static void DropViewsOnTable(Oid relationId) { List *views = GetDependingViews(relationId); /* * GetDependingViews returns views in the dependency order. We should drop views * in the reversed order since dropping views can cascade to other views below. */ List *reverseOrderedViews = ReversedOidList(views); Oid viewId = InvalidOid; foreach_declared_oid(viewId, reverseOrderedViews) { char *qualifiedViewName = generate_qualified_relation_name(viewId); StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "DROP %sVIEW IF EXISTS %s", get_rel_relkind(viewId) == RELKIND_MATVIEW ? "MATERIALIZED " : "", qualifiedViewName); ExecuteAndLogUtilityCommand(dropCommand->data); } } /* * ReversedOidList takes a list of oids and returns the reverse ordered version of it. */ static List * ReversedOidList(List *oidList) { List *reversed = NIL; Oid oid = InvalidOid; foreach_declared_oid(oid, oidList) { reversed = lcons_oid(oid, reversed); } return reversed; } /* * GetExplicitIndexOidList returns a list of index oids defined "explicitly" * on the relation with relationId by the "CREATE INDEX" commands. That means, * all the constraints defined on the relation except: * - primary indexes, * - unique indexes and * - exclusion indexes * that are actually applied by the related constraints. */ List * GetExplicitIndexOidList(Oid relationId) { /* flags are not applicable for AppendExplicitIndexIdsToList */ int flags = 0; return ExecuteFunctionOnEachTableIndex(relationId, AppendExplicitIndexIdsToList, flags); } /* * AppendExplicitIndexIdsToList adds the given index oid if it is * explicitly created on its relation. */ static void AppendExplicitIndexIdsToList(Form_pg_index indexForm, List **explicitIndexIdList, int flags) { if (!IndexImpliedByAConstraint(indexForm)) { *explicitIndexIdList = lappend_oid(*explicitIndexIdList, indexForm->indexrelid); } } /* * GetRenameStatsCommandList returns a list of "ALTER STATISTICS ... * RENAME TO ..._shardId" commands for given statistics oid list. */ static List * GetRenameStatsCommandList(List *statsOidList, uint64 shardId) { List *statsCommandList = NIL; Oid statsOid; foreach_declared_oid(statsOid, statsOidList) { HeapTuple tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(tup)) { ereport(WARNING, (errmsg("No stats object found with id: %u", statsOid))); continue; } Form_pg_statistic_ext statisticsForm = (Form_pg_statistic_ext) GETSTRUCT(tup); char *statsName = statisticsForm->stxname.data; Oid statsSchemaOid = statisticsForm->stxnamespace; char *statsSchema = get_namespace_name(statsSchemaOid); char *statsNameWithShardId = pstrdup(statsName); AppendShardIdToName(&statsNameWithShardId, shardId); char *renameShardStatsCommand = GetRenameShardStatsCommand(statsSchema, statsName, statsNameWithShardId); statsCommandList = lappend(statsCommandList, renameShardStatsCommand); ReleaseSysCache(tup); } return statsCommandList; } /* * DropNextValExprsAndMoveOwnedSeqOwnerships drops default column definitions * that are based on sequences for relation with sourceRelationId. * * Also, for each such column that owns a sequence, it grants ownership to the * same named column of the relation with targetRelationId. */ static void DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId, Oid targetRelationId) { List *columnNameList = NIL; List *ownedSequenceIdList = NIL; ExtractDefaultColumnsAndOwnedSequences(sourceRelationId, &columnNameList, &ownedSequenceIdList); char *columnName = NULL; Oid ownedSequenceId = InvalidOid; forboth_ptr_oid(columnName, columnNameList, ownedSequenceId, ownedSequenceIdList) { /* * We drop nextval() expressions because Citus currently evaluates * nextval() on the shell table, not on the shards. Hence, there is * no reason for keeping nextval(). Also, distributed/reference table * shards do not have - so be consistent with those. * * Note that we keep other kind of DEFAULT expressions on shards * because we still want to be able to evaluate DEFAULT expressions * that are not based on sequences on shards, e.g., for foreign key * - SET DEFAULT actions. */ AttrNumber columnAttrNumber = get_attnum(sourceRelationId, columnName); if (ColumnDefaultsToNextVal(sourceRelationId, columnAttrNumber)) { DropDefaultColumnDefinition(sourceRelationId, columnName); } /* * Column might own a sequence without having a nextval() expr on it * --e.g., due to ALTER SEQUENCE OWNED BY .. --, so check if that is * the case even if the column doesn't have a DEFAULT. */ if (OidIsValid(ownedSequenceId)) { TransferSequenceOwnership(ownedSequenceId, targetRelationId, columnName); } } } /* * DropDefaultColumnDefinition drops the DEFAULT definiton of the column with * columnName of the relation with relationId via process utility. */ static void DropDefaultColumnDefinition(Oid relationId, char *columnName) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); const char *quotedColumnName = quote_identifier(columnName); StringInfo sequenceDropCommand = makeStringInfo(); appendStringInfo(sequenceDropCommand, "ALTER TABLE %s ALTER COLUMN %s DROP DEFAULT", qualifiedRelationName, quotedColumnName); /* * We need to disable/enable ddl propagation for this command, to prevent * sending unnecessary ALTER COLUMN commands for partitions, to MX workers. */ ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, sequenceDropCommand->data, ENABLE_DDL_PROPAGATION)); } /* * TransferSequenceOwnership grants ownership of the sequence with sequenceId * to the column with targetColumnName of relation with targetRelationId via * process utility. Note that this function assumes that the target relation * has a column with targetColumnName which can default to the given sequence. */ static void TransferSequenceOwnership(Oid sequenceId, Oid targetRelationId, char *targetColumnName) { char *qualifiedSequenceName = generate_qualified_relation_name(sequenceId); char *qualifiedTargetRelationName = generate_qualified_relation_name(targetRelationId); const char *quotedTargetColumnName = quote_identifier(targetColumnName); StringInfo sequenceOwnershipCommand = makeStringInfo(); appendStringInfo(sequenceOwnershipCommand, "ALTER SEQUENCE %s OWNED BY %s.%s", qualifiedSequenceName, qualifiedTargetRelationName, quotedTargetColumnName); /* * We need to disable/enable ddl propagation for this command, to prevent * sending unnecessary ALTER SEQUENCE commands for partitions, to MX workers. * Especially for partitioned tables, where the same sequence is used for * all partitions, this might cause errors. */ ExecuteAndLogUtilityCommandList(list_make3(DISABLE_DDL_PROPAGATION, sequenceOwnershipCommand->data, ENABLE_DDL_PROPAGATION)); } /* * InsertMetadataForCitusLocalTable inserts necessary metadata for the citus * local table to the following metadata tables: * pg_dist_partition, pg_dist_shard & pg_dist_placement. */ static void InsertMetadataForCitusLocalTable(Oid citusLocalTableId, uint64 shardId, bool autoConverted) { Assert(OidIsValid(citusLocalTableId)); Assert(shardId != INVALID_SHARD_ID); char distributionMethod = DISTRIBUTE_BY_NONE; char replicationModel = REPLICATION_MODEL_STREAMING; uint32 colocationId = INVALID_COLOCATION_ID; Var *distributionColumn = NULL; InsertIntoPgDistPartition(citusLocalTableId, distributionMethod, distributionColumn, colocationId, replicationModel, autoConverted); /* set shard storage type according to relation type */ char shardStorageType = ShardStorageType(citusLocalTableId); text *shardMinValue = NULL; text *shardMaxValue = NULL; InsertShardRow(citusLocalTableId, shardId, shardStorageType, shardMinValue, shardMaxValue); List *nodeList = list_make1(CoordinatorNodeIfAddedAsWorkerOrError()); int replicationFactor = 1; int workerStartIndex = 0; InsertShardPlacementRows(citusLocalTableId, shardId, nodeList, workerStartIndex, replicationFactor); } /* * FinalizeCitusLocalTableCreation completes creation of the citus local table * with relationId by performing operations that should be done after creating * the shard and inserting the metadata. * If the cluster has metadata workers, we ensure proper propagation of the * sequences dependent with the table. */ static void FinalizeCitusLocalTableCreation(Oid relationId) { /* * PG16+ supports truncate triggers on foreign tables */ if (RegularTable(relationId) || IsForeignTable(relationId)) { CreateTruncateTrigger(relationId); } if (ShouldSyncTableMetadata(relationId)) { SyncCitusTableMetadata(relationId); } /* * We've a custom way of foreign key graph invalidation, * see InvalidateForeignKeyGraph(). */ if (TableReferenced(relationId) || TableReferencing(relationId)) { InvalidateForeignKeyGraph(); } } /* * ShouldAddNewTableToMetadata takes a relationId and returns true if we need to add a * newly created table to metadata, false otherwise. * For partitions and temporary tables, ShouldAddNewTableToMetadata returns false. * For other tables created, returns true, if we are on a coordinator that is added * as worker, and ofcourse, if the GUC use_citus_managed_tables is set to on. */ bool ShouldAddNewTableToMetadata(Oid relationId) { if (get_rel_persistence(relationId) == RELPERSISTENCE_TEMP || PartitionTableNoLock(relationId)) { /* * Shouldn't add table to metadata if it's a temp table, or a partition. * Creating partitions of a table that is added to metadata is already handled. */ return false; } if (AddAllLocalTablesToMetadata && !IsBinaryUpgrade && IsCoordinator() && CoordinatorAddedAsWorkerNode()) { /* * We have verified that the GUC is set to true, and we are not upgrading, * and we are on the coordinator that is added as worker node. * So return true here, to add this newly created table to metadata. */ return true; } return false; } ================================================ FILE: src/backend/distributed/commands/citus_global_signal.c ================================================ /*------------------------------------------------------------------------- * * citus_global_signal.c * Commands for Citus' overriden versions of pg_cancel_backend * and pg_terminate_backend statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "signal.h" #include "lib/stringinfo.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/worker_manager.h" static bool CitusSignalBackend(uint64 globalPID, uint64 timeout, int sig); PG_FUNCTION_INFO_V1(citus_cancel_backend); PG_FUNCTION_INFO_V1(citus_terminate_backend); /* * pg_cancel_backend overrides the Postgres' pg_cancel_backend to cancel * a query with a global pid so a query can be cancelled from another node. * * To cancel a query that is on another node, a pg_cancel_backend command is sent * to that node. This new command is sent with pid instead of global pid, so original * pg_cancel_backend function is used. */ Datum citus_cancel_backend(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 pid = PG_GETARG_INT64(0); int sig = SIGINT; uint64 timeout = 0; bool success = CitusSignalBackend(pid, timeout, sig); PG_RETURN_BOOL(success); } /* * pg_terminate_backend overrides the Postgres' pg_terminate_backend to terminate * a query with a global pid so a query can be terminated from another node. * * To terminate a query that is on another node, a pg_terminate_backend command is sent * to that node. This new command is sent with pid instead of global pid, so original * pg_terminate_backend function is used. */ Datum citus_terminate_backend(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 pid = PG_GETARG_INT64(0); uint64 timeout = PG_GETARG_INT64(1); int sig = SIGTERM; bool success = CitusSignalBackend(pid, timeout, sig); PG_RETURN_BOOL(success); } /* * CitusSignalBackend gets a global pid and and ends the original query with the global pid * that might have started in another node by connecting to that node and running either * pg_cancel_backend or pg_terminate_backend based on the withTerminate argument. */ static bool CitusSignalBackend(uint64 globalPID, uint64 timeout, int sig) { Assert((sig == SIGINT) || (sig == SIGTERM)); bool missingOk = false; int nodeId = ExtractNodeIdFromGlobalPID(globalPID, missingOk); int processId = ExtractProcessIdFromGlobalPID(globalPID); WorkerNode *workerNode = FindNodeWithNodeId(nodeId, missingOk); StringInfo cancelQuery = makeStringInfo(); if (sig == SIGINT) { appendStringInfo(cancelQuery, "SELECT pg_cancel_backend(%d::integer)", processId); } else { appendStringInfo(cancelQuery, "SELECT pg_terminate_backend(%d::integer, %lu::bigint)", processId, timeout); } int connectionFlags = 0; MultiConnection *connection = GetNodeConnection(connectionFlags, workerNode->workerName, workerNode->workerPort); if (!SendRemoteCommand(connection, cancelQuery->data)) { /* if we cannot connect, we warn and report false */ ReportConnectionError(connection, WARNING); return false; } bool raiseInterrupts = true; PGresult *queryResult = GetRemoteCommandResult(connection, raiseInterrupts); /* if remote node throws an error, we also throw an error */ if (!IsResponseOK(queryResult)) { ReportResultError(connection, queryResult, ERROR); } StringInfo queryResultString = makeStringInfo(); bool success = EvaluateSingleQueryResult(connection, queryResult, queryResultString); if (success && strcmp(queryResultString->data, "f") == 0) { /* worker node returned "f" */ success = false; } PQclear(queryResult); bool raiseErrors = false; ClearResults(connection, raiseErrors); return success; } ================================================ FILE: src/backend/distributed/commands/cluster.c ================================================ /*------------------------------------------------------------------------- * * cluster.c * Commands for CLUSTER statement * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_partitioning_utils.h" static bool IsClusterStmtVerbose_compat(ClusterStmt *clusterStmt); /* * PreprocessClusterStmt first determines whether a given cluster statement involves * a distributed table. If so (and if it is supported, i.e. no verbose), it * creates a DDLJob to encapsulate information needed during the worker node * portion of DDL execution before returning that DDLJob in a List. If no * distributed table is involved, this function returns NIL. */ List * PreprocessClusterStmt(Node *node, const char *clusterCommand, ProcessUtilityContext processUtilityContext) { ClusterStmt *clusterStmt = castNode(ClusterStmt, node); bool missingOK = false; if (clusterStmt->relation == NULL) { if (EnableUnsupportedFeatureMessages) { ereport(WARNING, (errmsg("not propagating CLUSTER command to worker nodes"), errhint("Provide a specific table in order to CLUSTER " "distributed tables."))); } return NIL; } /* PostgreSQL uses access exclusive lock for CLUSTER command */ Oid relationId = RangeVarGetRelid(clusterStmt->relation, AccessExclusiveLock, missingOK); /* * If the table does not exist, don't do anything here to allow PostgreSQL * to throw the appropriate error or notice message later. */ if (!OidIsValid(relationId)) { return NIL; } /* we have no planning to do unless the table is distributed */ bool isCitusRelation = IsCitusTable(relationId); if (!isCitusRelation) { return NIL; } /* * We do not support CLUSTER command on partitioned tables as it can not be run inside * transaction blocks. PostgreSQL currently does not support CLUSTER command on * partitioned tables in a transaction block. Although Citus can execute commands * outside of transaction block -- such as VACUUM -- we cannot do that here because * CLUSTER command is also not allowed from a function call as well. By default, Citus * uses `worker_apply_shard_ddl_command()`, where we should avoid it for this case. */ if (PartitionedTable(relationId)) { if (EnableUnsupportedFeatureMessages) { ereport(WARNING, (errmsg("not propagating CLUSTER command for partitioned " "table to worker nodes"), errhint("Provide a child partition table names in order to " "CLUSTER distributed partitioned tables."))); } return NIL; } if (IsClusterStmtVerbose_compat(clusterStmt)) { ereport(ERROR, (errmsg("cannot run CLUSTER command"), errdetail("VERBOSE option is currently unsupported " "for distributed tables."))); } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->metadataSyncCommand = clusterCommand; ddlJob->taskList = DDLTaskList(relationId, clusterCommand); return list_make1(ddlJob); } /* * IsClusterStmtVerbose_compat returns true if the given statement * is a cluster statement with verbose option. */ static bool IsClusterStmtVerbose_compat(ClusterStmt *clusterStmt) { DefElem *opt = NULL; foreach_declared_ptr(opt, clusterStmt->params) { if (strcmp(opt->defname, "verbose") == 0) { return defGetBoolean(opt); } } return false; } ================================================ FILE: src/backend/distributed/commands/collation.c ================================================ /*------------------------------------------------------------------------- * collation.c * * This file contains functions to create, alter and drop policies on * distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/pg_collation.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_manager.h" static char * CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollationName); /* * GetCreateCollationDDLInternal returns a CREATE COLLATE sql string for the * given collationId. * * It includes 2 out parameters to assist creation of ALTER COLLATION OWNER. * quotedCollationName must not be NULL. */ static char * CreateCollationDDLInternal(Oid collationId, Oid *collowner, char **quotedCollationName) { StringInfoData collationNameDef; HeapTuple heapTuple = SearchSysCache1(COLLOID, ObjectIdGetDatum(collationId)); if (!HeapTupleIsValid(heapTuple)) { elog(ERROR, "citus cache lookup failed for collation %u", collationId); } Form_pg_collation collationForm = (Form_pg_collation) GETSTRUCT(heapTuple); char collprovider = collationForm->collprovider; Oid collnamespace = collationForm->collnamespace; const char *collname = NameStr(collationForm->collname); bool collisdeterministic = collationForm->collisdeterministic; char *collcollate; char *collctype; /* * In PG15, there is an added option to use ICU as global locale provider. * pg_collation has three locale-related fields: collcollate and collctype, * which are libc-related fields, and a new one colliculocale, which is the * ICU-related field. Only the libc-related fields or the ICU-related field * is set, never both. */ char *colllocale; bool isnull; Datum datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collcollate, &isnull); if (!isnull) { collcollate = TextDatumGetCString(datum); } else { collcollate = NULL; } datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collctype, &isnull); if (!isnull) { collctype = TextDatumGetCString(datum); } else { collctype = NULL; } datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_colllocale, &isnull); if (!isnull) { colllocale = TextDatumGetCString(datum); } else { colllocale = NULL; } Assert((collcollate && collctype) || colllocale); if (collowner != NULL) { *collowner = collationForm->collowner; } ReleaseSysCache(heapTuple); char *schemaName = get_namespace_name(collnamespace); *quotedCollationName = quote_qualified_identifier(schemaName, collname); const char *providerString = collprovider == COLLPROVIDER_BUILTIN ? "builtin" : collprovider == COLLPROVIDER_DEFAULT ? "default" : collprovider == COLLPROVIDER_ICU ? "icu" : collprovider == COLLPROVIDER_LIBC ? "libc" : NULL; if (providerString == NULL) { elog(ERROR, "unknown collation provider: %c", collprovider); } initStringInfo(&collationNameDef); appendStringInfo(&collationNameDef, "CREATE COLLATION %s (provider = '%s'", *quotedCollationName, providerString); if (colllocale) { appendStringInfo(&collationNameDef, ", locale = %s", quote_literal_cstr(colllocale)); pfree(colllocale); } else { if (strcmp(collcollate, collctype) == 0) { appendStringInfo(&collationNameDef, ", locale = %s", quote_literal_cstr(collcollate)); } else { appendStringInfo(&collationNameDef, ", lc_collate = %s, lc_ctype = %s", quote_literal_cstr(collcollate), quote_literal_cstr(collctype)); } pfree(collcollate); pfree(collctype); } char *collicurules = NULL; datum = SysCacheGetAttr(COLLOID, heapTuple, Anum_pg_collation_collicurules, &isnull); if (!isnull) { collicurules = TextDatumGetCString(datum); appendStringInfo(&collationNameDef, ", rules = %s", quote_literal_cstr(collicurules)); } if (!collisdeterministic) { appendStringInfoString(&collationNameDef, ", deterministic = false"); } appendStringInfoChar(&collationNameDef, ')'); return collationNameDef.data; } /* * CreateCollationDDL wrap CreateCollationDDLInternal to hide the out parameters. */ char * CreateCollationDDL(Oid collationId) { char *quotedCollationName = NULL; return CreateCollationDDLInternal(collationId, NULL, "edCollationName); } /* * CreateCollationDDLsIdempotent returns a List of cstrings for creating the collation * using create_or_replace_object & includes ALTER COLLATION ROLE. */ List * CreateCollationDDLsIdempotent(Oid collationId) { StringInfoData collationAlterOwnerCommand; Oid collowner = InvalidOid; char *quotedCollationName = NULL; char *createCollationCommand = CreateCollationDDLInternal(collationId, &collowner, "edCollationName); initStringInfo(&collationAlterOwnerCommand); appendStringInfo(&collationAlterOwnerCommand, "ALTER COLLATION %s OWNER TO %s", quotedCollationName, quote_identifier(GetUserNameFromId(collowner, false))); return list_make2(WrapCreateOrReplace(createCollationCommand), collationAlterOwnerCommand.data); } List * AlterCollationOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Relation relation; Assert(stmt->objectType == OBJECT_COLLATION); ObjectAddress objectAddress = get_object_address(stmt->objectType, stmt->object, &relation, AccessExclusiveLock, missing_ok); ObjectAddress *objectAddressCopy = palloc0(sizeof(ObjectAddress)); *objectAddressCopy = objectAddress; return list_make1(objectAddressCopy); } /* * RenameCollationStmtObjectAddress returns the ObjectAddress of the type that is the object * of the RenameStmt. Errors if missing_ok is false. */ List * RenameCollationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_COLLATION); Oid collationOid = get_collation_oid((List *) stmt->object, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, CollationRelationId, collationOid); return list_make1(address); } /* * AlterCollationSchemaStmtObjectAddress returns the ObjectAddress of the type that is the * subject of the AlterObjectSchemaStmt. Errors if missing_ok is false. * * This could be called both before or after it has been applied locally. It will look in * the old schema first, if the type cannot be found in that schema it will look in the * new schema. Errors if missing_ok is false and the type cannot be found in either of the * schemas. */ List * AlterCollationSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_COLLATION); List *name = (List *) stmt->object; Oid collationOid = get_collation_oid(name, true); if (collationOid == InvalidOid) { List *newName = list_make2(makeString(stmt->newschema), lfirst(list_tail(name))); collationOid = get_collation_oid(newName, true); if (!missing_ok && collationOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("type \"%s\" does not exist", NameListToString(name)))); } } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, CollationRelationId, collationOid); return list_make1(address); } /* * GenerateBackupNameForCollationCollision generates a new collation name for an existing collation. * The name is generated in such a way that the new name doesn't overlap with an existing collation * by adding a suffix with incrementing number after the new name. */ char * GenerateBackupNameForCollationCollision(const ObjectAddress *address) { char *newName = palloc0(NAMEDATALEN); char suffix[NAMEDATALEN] = { 0 }; int count = 0; char *baseName = get_collation_name(address->objectId); int baseLength = strlen(baseName); HeapTuple collationTuple = SearchSysCache1(COLLOID, address->objectId); if (!HeapTupleIsValid(collationTuple)) { elog(ERROR, "citus cache lookup failed"); return NULL; } Form_pg_collation collationForm = (Form_pg_collation) GETSTRUCT(collationTuple); String *namespace = makeString(get_namespace_name(collationForm->collnamespace)); ReleaseSysCache(collationTuple); while (true) { int suffixLength = SafeSnprintf(suffix, NAMEDATALEN - 1, "(citus_backup_%d)", count); /* trim the base name at the end to leave space for the suffix and trailing \0 */ baseLength = Min(baseLength, NAMEDATALEN - suffixLength - 1); /* clear newName before copying the potentially trimmed baseName and suffix */ memset(newName, 0, NAMEDATALEN); strncpy_s(newName, NAMEDATALEN, baseName, baseLength); strncpy_s(newName + baseLength, NAMEDATALEN - baseLength, suffix, suffixLength); List *newCollationName = list_make2(namespace, makeString(newName)); /* don't need to rename if the input arguments don't match */ Oid existingCollationId = get_collation_oid(newCollationName, true); if (existingCollationId == InvalidOid) { return newName; } count++; } } List * DefineCollationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DefineStmt *stmt = castNode(DefineStmt, node); Assert(stmt->kind == OBJECT_COLLATION); Oid collOid = get_collation_oid(stmt->defnames, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, CollationRelationId, collOid); return list_make1(address); } ================================================ FILE: src/backend/distributed/commands/comment.c ================================================ /*------------------------------------------------------------------------- * * comment.c * Commands to interact with the comments for all database * object types. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/table.h" #include "catalog/pg_shdescription.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/rel.h" #include "distributed/comment.h" static char * GetCommentForObject(Oid classOid, Oid objectOid); List * GetCommentPropagationCommands(Oid classOid, Oid objOoid, char *objectName, ObjectType objectType) { List *commands = NIL; StringInfo commentStmt = makeStringInfo(); /* Get the comment for the database */ char *comment = GetCommentForObject(classOid, objOoid); char const *commentObjectType = ObjectTypeNames[objectType]; /* Create the SQL command to propagate the comment to other nodes */ if (comment != NULL) { appendStringInfo(commentStmt, "COMMENT ON %s %s IS %s;", commentObjectType, quote_identifier(objectName), quote_literal_cstr(comment)); } /* Add the command to the list */ if (commentStmt->len > 0) { commands = list_make1(commentStmt->data); } return commands; } static char * GetCommentForObject(Oid classOid, Oid objectOid) { HeapTuple tuple; char *comment = NULL; /* Open pg_shdescription catalog */ Relation shdescRelation = table_open(SharedDescriptionRelationId, AccessShareLock); /* Scan the table */ ScanKeyData scanKey[2]; ScanKeyInit(&scanKey[0], Anum_pg_shdescription_objoid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(objectOid)); ScanKeyInit(&scanKey[1], Anum_pg_shdescription_classoid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(classOid)); bool indexOk = true; int scanKeyCount = 2; SysScanDesc scan = systable_beginscan(shdescRelation, SharedDescriptionObjIndexId, indexOk, NULL, scanKeyCount, scanKey); if ((tuple = systable_getnext(scan)) != NULL) { bool isNull = false; TupleDesc tupdesc = RelationGetDescr(shdescRelation); Datum descDatum = heap_getattr(tuple, Anum_pg_shdescription_description, tupdesc, &isNull); /* Add the command to the list */ if (!isNull) { comment = TextDatumGetCString(descDatum); } else { comment = NULL; } } /* End the scan and close the catalog */ systable_endscan(scan); table_close(shdescRelation, AccessShareLock); return comment; } /* * CommentObjectAddress resolves the ObjectAddress for the object * on which the comment is placed. Optionally errors if the object does not * exist based on the missing_ok flag passed in by the caller. */ List * CommentObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CommentStmt *stmt = castNode(CommentStmt, node); Relation relation; ObjectAddress objectAddress = get_object_address(stmt->objtype, stmt->object, &relation, AccessExclusiveLock, missing_ok); ObjectAddress *objectAddressCopy = palloc0(sizeof(ObjectAddress)); *objectAddressCopy = objectAddress; return list_make1(objectAddressCopy); } ================================================ FILE: src/backend/distributed/commands/common.c ================================================ /*------------------------------------------------------------------------- * * common.c * * Most of the object propagation code consists of mostly the same * operations, varying slightly in parameters passed around. This * file contains most of the reusable logic in object propagation. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/objectaddress.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "nodes/parsenodes.h" #include "tcop/utility.h" #include "distributed/citus_depended_object.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" /* * PostprocessCreateDistributedObjectFromCatalogStmt is a common function that can be used * for most objects during their creation phase. After the creation has happened locally * this function creates idempotent statements to recreate the object addressed by the * ObjectAddress of resolved from the creation statement. * * Since object already need to be able to create idempotent creation sql to support * scaleout operations we can reuse this logic during the initial creation of the objects * to reduce the complexity of implementation of new DDL commands. */ List * PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, const char *queryString) { const DistributeObjectOps *ops = GetDistributeObjectOps(stmt); Assert(ops != NULL); if (!ShouldPropagate()) { return NIL; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return NIL; } if (ops->featureFlag && *ops->featureFlag == false) { /* not propagating when a configured feature flag is turned off by the user */ return NIL; } if (ops->qualify && DistOpsValidityState(stmt, ops) == ShouldQualifyAfterLocalCreation) { /* qualify the statement after local creation */ ops->qualify(stmt); } List *addresses = GetObjectAddressListFromParseTree(stmt, false, true); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); EnsureCoordinator(); EnsureSequentialMode(ops->objectType); /* If the object has any unsupported dependency warn, and only create locally */ DeferredErrorMessage *depError = DeferErrorIfAnyObjectHasUnsupportedDependency( addresses); if (depError != NULL) { if (EnableUnsupportedFeatureMessages) { RaiseDeferredError(depError, WARNING); } return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(addresses); List *commands = GetAllDependencyCreateDDLCommands(addresses); commands = lcons(DISABLE_DDL_PROPAGATION, commands); commands = lappend(commands, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessAlterDistributedObjectStmt handles any updates to distributed objects by * creating the fully qualified sql to apply to all workers after checking all * predconditions that apply to propagating changes. * * Preconditions are (in order): * - not in a CREATE/ALTER EXTENSION code block * - citus.enable_metadata_sync is turned on * - object being altered is distributed * - any object specific feature flag is turned on when a feature flag is available * * Once we conclude to propagate the changes to the workers we make sure that the command * has been executed on the coordinator and force any ongoing transaction to run in * sequential mode. If any of these steps fail we raise an error to inform the user. * * Lastly we recreate a fully qualified version of the original sql and prepare the tasks * to send these sql commands to the workers. These tasks include instructions to prevent * recursion of propagation with Citus' MX functionality. */ List * PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext) { const DistributeObjectOps *ops = GetDistributeObjectOps(stmt); Assert(ops != NULL); List *addresses = GetObjectAddressListFromParseTree(stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } if (ops->featureFlag && *ops->featureFlag == false) { /* not propagating when a configured feature flag is turned off by the user */ return NIL; } EnsureCoordinator(); EnsureSequentialMode(ops->objectType); QualifyTreeNode(stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PostprocessAlterDistributedObjectStmt is the counter part of * PreprocessAlterDistributedObjectStmt that should be executed after the object has been * changed locally. * * We perform the same precondition checks as before to skip this operation if any of the * failed during preprocessing. Since we already raised an error on other checks we don't * have to repeat them here, as they will never fail during postprocessing. * * When objects get altered they can start depending on undistributed objects. Now that * the objects has been changed locally we can find these new dependencies and make sure * they get created on the workers before we send the command list to the workers. */ List * PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString) { const DistributeObjectOps *ops = GetDistributeObjectOps(stmt); Assert(ops != NULL); List *addresses = GetObjectAddressListFromParseTree(stmt, false, true); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } if (ops->featureFlag && *ops->featureFlag == false) { /* not propagating when a configured feature flag is turned off by the user */ return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(addresses); return NIL; } /* * PreprocessDropDistributedObjectStmt is a general purpose hook that can propagate any * DROP statement. * * DROP statements are one of the few DDL statements that can work on many different * objects at once. Instead of resolving just one ObjectAddress and check it is * distributed we will need to lookup many different object addresses. Only if an object * was _not_ distributed we will need to remove it from the list of objects before we * recreate the sql statement. * * Given that we actually _do_ need to drop them locally we can't simply remove them from * the object list. Instead we create a new list where we only add distributed objects to. * Before we recreate the sql statement we put this list on the drop statement, so that * the SQL created will only contain the objects that are actually distributed in the * cluster. After we have the SQL we restore the old list so that all objects get deleted * locally. * * The reason we need to go through all this effort is taht we can't resolve the object * addresses anymore after the objects have been removed locally. Meaning during the * postprocessing we cannot understand which objects were distributed to begin with. */ List * PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *stmt = castNode(DropStmt, node); /* * We swap the list of objects to remove during deparse so we need a reference back to * the old list to put back */ List *originalObjects = stmt->objects; if (!ShouldPropagate()) { return NIL; } QualifyTreeNode(node); List *distributedObjects = NIL; List *distributedObjectAddresses = NIL; Node *object = NULL; foreach_declared_ptr(object, stmt->objects) { /* TODO understand if the lock should be sth else */ Relation rel = NULL; /* not used, but required to pass to get_object_address */ ObjectAddress address = get_object_address(stmt->removeType, object, &rel, AccessShareLock, stmt->missing_ok); ObjectAddress *addressPtr = palloc0(sizeof(ObjectAddress)); *addressPtr = address; if (IsAnyObjectDistributed(list_make1(addressPtr))) { distributedObjects = lappend(distributedObjects, object); distributedObjectAddresses = lappend(distributedObjectAddresses, addressPtr); } } if (list_length(distributedObjects) <= 0) { /* no distributed objects to drop */ return NIL; } /* * managing objects can only be done on the coordinator if ddl propagation is on. when * it is off we will never get here. MX workers don't have a notion of distributed * types, so we block the call. */ EnsureCoordinator(); /* * remove the entries for the distributed objects on dropping */ ObjectAddress *address = NULL; foreach_declared_ptr(address, distributedObjectAddresses) { UnmarkObjectDistributed(address); } /* * temporary swap the lists of objects to delete with the distributed objects and * deparse to an executable sql statement for the workers */ stmt->objects = distributedObjects; char *dropStmtSql = DeparseTreeNode((Node *) stmt); stmt->objects = originalObjects; EnsureSequentialMode(stmt->removeType); /* to prevent recursion with mx we disable ddl propagation */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, dropStmtSql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * DropTextSearchDictObjectAddress returns list of object addresses in * the drop tsdict statement. */ List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropStmt *stmt = castNode(DropStmt, node); List *objectAddresses = NIL; List *objNameList = NIL; foreach_declared_ptr(objNameList, stmt->objects) { Oid tsdictOid = get_ts_dict_oid(objNameList, missing_ok); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, TSDictionaryRelationId, tsdictOid); objectAddresses = lappend(objectAddresses, objectAddress); } return objectAddresses; } /* * DropTextSearchConfigObjectAddress returns list of object addresses in * the drop tsconfig statement. */ List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropStmt *stmt = castNode(DropStmt, node); List *objectAddresses = NIL; List *objNameList = NIL; foreach_declared_ptr(objNameList, stmt->objects) { Oid tsconfigOid = get_ts_config_oid(objNameList, missing_ok); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, TSConfigRelationId, tsconfigOid); objectAddresses = lappend(objectAddresses, objectAddress); } return objectAddresses; } ================================================ FILE: src/backend/distributed/commands/create_distributed_table.c ================================================ /*------------------------------------------------------------------------- * * create_distributed_table.c * Routines relation to the creation of distributed relations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/hash.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/index.h" #include "catalog/pg_am.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_attribute.h" #include "catalog/pg_enum.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_proc.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/sequence.h" #include "commands/tablecmds.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "parser/parse_expr.h" #include "parser/parse_node.h" #include "parser/parse_relation.h" #include "parser/parser.h" #include "postmaster/postmaster.h" #include "storage/lmgr.h" #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/distributed_execution_locks.h" #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_colocation.h" #include "distributed/pg_dist_partition.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/replicate_none_dist_table_shard.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_split.h" #include "distributed/shard_transfer.h" #include "distributed/shared_library_init.h" #include "distributed/utils/distribution_column_map.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" /* common params that apply to all Citus table types */ typedef struct { char distributionMethod; char replicationModel; } CitusTableParams; /* * Params that only apply to distributed tables, i.e., the ones that are * known as DISTRIBUTED_TABLE by Citus metadata. */ typedef struct { int shardCount; bool shardCountIsStrict; char *distributionColumnName; ColocationParam colocationParam; } DistributedTableParams; /* * once every LOG_PER_TUPLE_AMOUNT, the copy will be logged. */ #define LOG_PER_TUPLE_AMOUNT 1000000 /* local function forward declarations */ static void CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName, char distributionMethod, char *colocateWithTableName, int shardCount, bool shardCountIsStrict); static char DecideDistTableReplicationModel(char distributionMethod, char *colocateWithTableName); static List * HashSplitPointsForShardList(List *shardList); static List * HashSplitPointsForShardCount(int shardCount); static List * WorkerNodesForShardList(List *shardList); static List * RoundRobinWorkerNodeList(List *workerNodeList, int listLength); static CitusTableParams DecideCitusTableParams(CitusTableType tableType, DistributedTableParams * distributedTableParams); static void CreateCitusTable(Oid relationId, CitusTableType tableType, DistributedTableParams *distributedTableParams); static void ConvertCitusLocalTableToTableType(Oid relationId, CitusTableType tableType, DistributedTableParams * distributedTableParams); static void CreateHashDistributedTableShards(Oid relationId, int shardCount, Oid colocatedTableId, bool localTableEmpty); static void CreateSingleShardTableShard(Oid relationId, Oid colocatedTableId, uint32 colocationId); static uint32 ColocationIdForNewTable(Oid relationId, CitusTableType tableType, DistributedTableParams *distributedTableParams, Var *distributionColumn); static void EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, char distributionMethod, uint32 colocationId, char replicationModel); static void EnsureLocalTableEmpty(Oid relationId); static void EnsureRelationHasNoTriggers(Oid relationId); static Oid SupportFunctionForColumn(Var *partitionColumn, Oid accessMethodId, int16 supportFunctionNumber); static void EnsureLocalTableEmptyIfNecessary(Oid relationId, char distributionMethod); static bool ShouldLocalTableBeEmpty(Oid relationId, char distributionMethod); static void EnsureCitusTableCanBeCreated(Oid relationOid); static void PropagatePrerequisiteObjectsForDistributedTable(Oid relationId); static void EnsureDistributedSequencesHaveOneType(Oid relationId, List *seqInfoList); static void CopyLocalDataIntoShards(Oid distributedTableId); static List * TupleDescColumnNameList(TupleDesc tupleDescriptor); static bool DistributionColumnUsesNumericColumnNegativeScale(TupleDesc relationDesc, Var *distributionColumn); static int numeric_typmod_scale(int32 typmod); static bool is_valid_numeric_typmod(int32 typmod); static void DistributionColumnIsGeneratedCheck(TupleDesc relationDesc, Var *distributionColumn, const char *relationName); static bool CanUseExclusiveConnections(Oid relationId, bool localTableEmpty); static uint64 DoCopyFromLocalTableIntoShards(Relation distributedRelation, DestReceiver *copyDest, TupleTableSlot *slot, EState *estate); static void ErrorIfTemporaryTable(Oid relationId); static void ErrorIfForeignTable(Oid relationOid); static void SendAddLocalTableToMetadataCommandOutsideTransaction(Oid relationId); static void EnsureDistributableTable(Oid relationId); static void EnsureForeignKeysForDistributedTableConcurrently(Oid relationId); static void EnsureColocateWithTableIsValid(Oid relationId, char distributionMethod, char *distributionColumnName, char *colocateWithTableName); static void WarnIfTableHaveNoReplicaIdentity(Oid relationId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_create_distributed_table); PG_FUNCTION_INFO_V1(create_distributed_table_concurrently); PG_FUNCTION_INFO_V1(create_distributed_table); PG_FUNCTION_INFO_V1(create_reference_table); /* * master_create_distributed_table is a deprecated predecessor to * create_distributed_table */ Datum master_create_distributed_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("master_create_distributed_table has been deprecated"))); } /* * create_distributed_table gets a table name, distribution column, * distribution method and colocate_with option, then it creates a * distributed table. */ Datum create_distributed_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (PG_ARGISNULL(0) || PG_ARGISNULL(3)) { PG_RETURN_VOID(); } Oid relationId = PG_GETARG_OID(0); text *distributionColumnText = PG_ARGISNULL(1) ? NULL : PG_GETARG_TEXT_P(1); Oid distributionMethodOid = PG_GETARG_OID(2); text *colocateWithTableNameText = PG_GETARG_TEXT_P(3); char *colocateWithTableName = text_to_cstring(colocateWithTableNameText); bool shardCountIsStrict = false; if (distributionColumnText) { if (PG_ARGISNULL(2)) { PG_RETURN_VOID(); } int shardCount = ShardCount; if (!PG_ARGISNULL(4)) { if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone(colocateWithTableName)) { ereport(ERROR, (errmsg("Cannot use colocate_with with a table " "and shard_count at the same time"))); } shardCount = PG_GETARG_INT32(4); /* * If shard_count parameter is given, then we have to * make sure table has that many shards. */ shardCountIsStrict = true; } char *distributionColumnName = text_to_cstring(distributionColumnText); Assert(distributionColumnName != NULL); char distributionMethod = LookupDistributionMethod(distributionMethodOid); if (shardCount < 1 || shardCount > MAX_SHARD_COUNT) { ereport(ERROR, (errmsg("%d is outside the valid range for " "parameter \"shard_count\" (1 .. %d)", shardCount, MAX_SHARD_COUNT))); } CreateDistributedTable(relationId, distributionColumnName, distributionMethod, shardCount, shardCountIsStrict, colocateWithTableName); } else { if (!PG_ARGISNULL(4)) { ereport(ERROR, (errmsg("shard_count can't be specified when the " "distribution column is null because in " "that case it's automatically set to 1"))); } if (!PG_ARGISNULL(2) && LookupDistributionMethod(PG_GETARG_OID(2)) != DISTRIBUTE_BY_HASH) { /* * As we do for shard_count parameter, we could throw an error if * distribution_type is not NULL when creating a single-shard table. * However, this requires changing the default value of distribution_type * parameter to NULL and this would mean a breaking change for most * users because they're mostly using this API to create sharded * tables. For this reason, here we instead do nothing if the distribution * method is DISTRIBUTE_BY_HASH. */ ereport(ERROR, (errmsg("distribution_type can't be specified " "when the distribution column is null "))); } ColocationParam colocationParam = { .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT, .colocateWithTableName = colocateWithTableName, }; CreateSingleShardTable(relationId, colocationParam); } PG_RETURN_VOID(); } /* * create_distributed_concurrently gets a table name, distribution column, * distribution method and colocate_with option, then it creates a * distributed table. */ Datum create_distributed_table_concurrently(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (PG_ARGISNULL(0) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) { PG_RETURN_VOID(); } if (PG_ARGISNULL(1)) { ereport(ERROR, (errmsg("cannot use create_distributed_table_concurrently " "to create a distributed table with a null shard " "key, consider using create_distributed_table()"))); } Oid relationId = PG_GETARG_OID(0); text *distributionColumnText = PG_GETARG_TEXT_P(1); char *distributionColumnName = text_to_cstring(distributionColumnText); Oid distributionMethodOid = PG_GETARG_OID(2); char distributionMethod = LookupDistributionMethod(distributionMethodOid); text *colocateWithTableNameText = PG_GETARG_TEXT_P(3); char *colocateWithTableName = text_to_cstring(colocateWithTableNameText); bool shardCountIsStrict = false; int shardCount = ShardCount; if (!PG_ARGISNULL(4)) { if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) != 0 && pg_strncasecmp(colocateWithTableName, "none", NAMEDATALEN) != 0) { ereport(ERROR, (errmsg("Cannot use colocate_with with a table " "and shard_count at the same time"))); } shardCount = PG_GETARG_INT32(4); /* * if shard_count parameter is given than we have to * make sure table has that many shards */ shardCountIsStrict = true; } CreateDistributedTableConcurrently(relationId, distributionColumnName, distributionMethod, colocateWithTableName, shardCount, shardCountIsStrict); PG_RETURN_VOID(); } /* * CreateDistributedTableConcurrently distributes a table by first converting * it to a Citus local table and then splitting the shard of the Citus local * table. * * If anything goes wrong during the second phase, the table is left as a * Citus local table. */ static void CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName, char distributionMethod, char *colocateWithTableName, int shardCount, bool shardCountIsStrict) { /* * We disallow create_distributed_table_concurrently in transaction blocks * because we cannot handle preceding writes, and we block writes at the * very end of the operation so the transaction should end immediately after. */ PreventInTransactionBlock(true, "create_distributed_table_concurrently"); /* * do not allow multiple create_distributed_table_concurrently in the same * transaction. We should do that check just here because concurrent local table * conversion can cause issues. */ ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction(); /* do not allow concurrent CreateDistributedTableConcurrently operations */ AcquireCreateDistributedTableConcurrentlyLock(relationId); if (distributionMethod != DISTRIBUTE_BY_HASH) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("only hash-distributed tables can be distributed " "without blocking writes"))); } if (ShardReplicationFactor > 1) { ereport(ERROR, (errmsg("cannot distribute a table concurrently when " "citus.shard_replication_factor > 1"))); } DropOrphanedResourcesInSeparateTransaction(); EnsureCitusTableCanBeCreated(relationId); EnsureValidDistributionColumn(relationId, distributionColumnName); /* * Ensure table type is valid to be distributed. It should be either regular or citus local table. */ EnsureDistributableTable(relationId); /* * we rely on citus_add_local_table_to_metadata, so it can generate irrelevant messages. * we want to error with a user friendly message if foreign keys are not supported. * We can miss foreign key violations because we are not holding locks, so relation * can be modified until we acquire the lock for the relation, but we do as much as we can * to be user friendly on foreign key violation messages. */ EnsureForeignKeysForDistributedTableConcurrently(relationId); char replicationModel = DecideDistTableReplicationModel(distributionMethod, colocateWithTableName); /* * we fail transaction before local table conversion if the table could not be colocated with * given table. We should make those checks after local table conversion by acquiring locks to * the relation because the distribution column can be modified in that period. */ if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone( colocateWithTableName)) { if (replicationModel != REPLICATION_MODEL_STREAMING) { ereport(ERROR, (errmsg("cannot create distributed table " "concurrently because Citus allows " "concurrent table distribution only when " "citus.shard_replication_factor = 1"), errhint("table %s is requested to be colocated " "with %s which has " "citus.shard_replication_factor > 1", get_rel_name(relationId), colocateWithTableName))); } EnsureColocateWithTableIsValid(relationId, distributionMethod, distributionColumnName, colocateWithTableName); } /* * Get name of the table before possibly replacing it in * citus_add_local_table_to_metadata. */ char *tableName = get_rel_name(relationId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); RangeVar *rangeVar = makeRangeVar(schemaName, tableName, -1); /* If table is a regular table, then we need to add it into metadata. */ if (!IsCitusTable(relationId)) { /* * Before taking locks, convert the table into a Citus local table and commit * to allow shard split to see the shard. */ SendAddLocalTableToMetadataCommandOutsideTransaction(relationId); } /* * Lock target relation with a shard update exclusive lock to * block DDL, but not writes. * * If there was a concurrent drop/rename, error out by setting missingOK = false. */ bool missingOK = false; relationId = RangeVarGetRelid(rangeVar, ShareUpdateExclusiveLock, missingOK); if (PartitionedTableNoLock(relationId)) { /* also lock partitions */ LockPartitionRelations(relationId, ShareUpdateExclusiveLock); } WarnIfTableHaveNoReplicaIdentity(relationId); List *shardList = LoadShardIntervalList(relationId); /* * It's technically possible for the table to have been concurrently * distributed just after citus_add_local_table_to_metadata and just * before acquiring the lock, so double check. */ if (list_length(shardList) != 1 || !IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("table was concurrently modified"))); } /* * The table currently has one shard, we will split that shard to match the * target distribution. */ ShardInterval *shardToSplit = (ShardInterval *) linitial(shardList); PropagatePrerequisiteObjectsForDistributedTable(relationId); /* * we should re-evaluate distribution column values. It may have changed, * because we did not lock the relation at the previous check before local * table conversion. */ Var *distributionColumn = BuildDistributionKeyFromColumnName(relationId, distributionColumnName, NoLock); /* get an advisory lock to serialize concurrent default group creations */ if (IsColocateWithDefault(colocateWithTableName)) { AcquireColocationDefaultLock(); } /* * At this stage, we only want to check for an existing co-location group. * We cannot create a new co-location group until after replication slot * creation in NonBlockingShardSplit. */ uint32 colocationId = FindColocateWithColocationId(relationId, replicationModel, distributionColumn, shardCount, shardCountIsStrict, colocateWithTableName); if (IsColocateWithDefault(colocateWithTableName) && (colocationId != INVALID_COLOCATION_ID)) { /* * we can release advisory lock if there is already a default entry for given params; * else, we should keep it to prevent different default coloc entry creation by * concurrent operations. */ ReleaseColocationDefaultLock(); } EnsureRelationCanBeDistributed(relationId, distributionColumn, distributionMethod, colocationId, replicationModel); Oid colocatedTableId = InvalidOid; if (colocationId != INVALID_COLOCATION_ID) { colocatedTableId = ColocatedTableId(colocationId); } List *workerNodeList = DistributedTablePlacementNodeList(NoLock); if (workerNodeList == NIL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no worker nodes are available for placing shards"), errhint("Add more worker nodes."))); } List *workersForPlacementList; List *shardSplitPointsList; if (colocatedTableId != InvalidOid) { List *colocatedShardList = LoadShardIntervalList(colocatedTableId); /* * Match the shard ranges of an existing table. */ shardSplitPointsList = HashSplitPointsForShardList(colocatedShardList); /* * Find the node IDs of the shard placements. */ workersForPlacementList = WorkerNodesForShardList(colocatedShardList); } else { /* * Generate a new set of #shardCount shards. */ shardSplitPointsList = HashSplitPointsForShardCount(shardCount); /* * Place shards in a round-robin fashion across all data nodes. */ workersForPlacementList = RoundRobinWorkerNodeList(workerNodeList, shardCount); } /* * Make sure that existing reference tables have been replicated to all the nodes * such that we can create foreign keys and joins work immediately after creation. * We do this after applying all essential checks to error out early in case of * user error. * * Use force_logical since this function is meant to not block writes. */ EnsureReferenceTablesExistOnAllNodesExtended(TRANSFER_MODE_FORCE_LOGICAL); /* * At this point, the table is a Citus local table, which means it does * not have a partition column in the metadata. However, we cannot update * the metadata here because that would prevent us from creating a replication * slot to copy ongoing changes. Instead, we pass a hash that maps relation * IDs to partition column vars. */ DistributionColumnMap *distributionColumnOverrides = CreateDistributionColumnMap(); AddDistributionColumnForRelation(distributionColumnOverrides, relationId, distributionColumnName); /* * there is no colocation entries yet for local table, so we should * check if table has any partition and add them to same colocation * group */ List *sourceColocatedShardIntervalList = ListShardsUnderParentRelation(relationId); SplitMode splitMode = NON_BLOCKING_SPLIT; SplitOperation splitOperation = CREATE_DISTRIBUTED_TABLE; SplitShard( splitMode, splitOperation, shardToSplit->shardId, shardSplitPointsList, workersForPlacementList, distributionColumnOverrides, sourceColocatedShardIntervalList, colocationId ); } /* * EnsureForeignKeysForDistributedTableConcurrently ensures that referenced and referencing foreign * keys for the given table are supported. * * We allow distributed -> reference * distributed -> citus local * * We disallow reference -> distributed * citus local -> distributed * regular -> distributed * * Normally regular -> distributed is allowed but it is not allowed when we create the * distributed table concurrently because we rely on conversion of regular table to citus local table, * which errors with an unfriendly message. */ static void EnsureForeignKeysForDistributedTableConcurrently(Oid relationId) { /* * disallow citus local -> distributed fkeys. * disallow reference -> distributed fkeys. * disallow regular -> distributed fkeys. */ EnsureNoFKeyFromTableType(relationId, INCLUDE_CITUS_LOCAL_TABLES | INCLUDE_REFERENCE_TABLES | INCLUDE_LOCAL_TABLES); /* * disallow distributed -> regular fkeys. */ EnsureNoFKeyToTableType(relationId, INCLUDE_LOCAL_TABLES); } /* * EnsureColocateWithTableIsValid ensures given relation can be colocated with the table of given name. */ static void EnsureColocateWithTableIsValid(Oid relationId, char distributionMethod, char *distributionColumnName, char *colocateWithTableName) { char replicationModel = DecideDistTableReplicationModel(distributionMethod, colocateWithTableName); text *colocateWithTableNameText = cstring_to_text(colocateWithTableName); Oid colocateWithTableId = ResolveRelationId(colocateWithTableNameText, false); Var *distributionColumn = BuildDistributionKeyFromColumnName(relationId, distributionColumnName, NoLock); EnsureTableCanBeColocatedWith(relationId, replicationModel, distributionColumn, colocateWithTableId); } /* * AcquireCreateDistributedTableConcurrentlyLock does not allow concurrent create_distributed_table_concurrently * operations. */ void AcquireCreateDistributedTableConcurrentlyLock(Oid relationId) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = true; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_CREATE_DISTRIBUTED_TABLE_CONCURRENTLY); LockAcquireResult lockAcquired = LockAcquire(&tag, ExclusiveLock, sessionLock, dontWait); if (!lockAcquired) { ereport(ERROR, (errmsg("another create_distributed_table_concurrently " "operation is in progress"), errhint("Make sure that the concurrent operation has " "finished and re-run the command"))); } } /* * SendAddLocalTableToMetadataCommandOutsideTransaction executes metadata add local * table command locally to avoid deadlock. */ static void SendAddLocalTableToMetadataCommandOutsideTransaction(Oid relationId) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); /* * we need to allow nested distributed execution, because we start a new distributed * execution inside the pushed-down UDF citus_add_local_table_to_metadata. Normally * citus does not allow that because it cannot guarantee correctness. */ StringInfo allowNestedDistributionCommand = makeStringInfo(); appendStringInfo(allowNestedDistributionCommand, "SET LOCAL citus.allow_nested_distributed_execution to ON"); StringInfo addLocalTableToMetadataCommand = makeStringInfo(); appendStringInfo(addLocalTableToMetadataCommand, "SELECT pg_catalog.citus_add_local_table_to_metadata(%s)", quote_literal_cstr(qualifiedRelationName)); List *commands = list_make2(allowNestedDistributionCommand->data, addLocalTableToMetadataCommand->data); char *username = NULL; SendCommandListToWorkerOutsideTransaction(LocalHostName, PostPortNumber, username, commands); } /* * WarnIfTableHaveNoReplicaIdentity notices user if the given table or its partitions (if any) * do not have a replica identity which is required for logical replication to replicate * UPDATE and DELETE commands during create_distributed_table_concurrently. */ void WarnIfTableHaveNoReplicaIdentity(Oid relationId) { bool foundRelationWithNoReplicaIdentity = false; /* * Check for source relation's partitions if any. We do not need to check for the source relation * because we can replicate partitioned table even if it does not have replica identity. * Source table will have no data if it has partitions. */ if (PartitionedTable(relationId)) { List *partitionList = PartitionList(relationId); ListCell *partitionCell = NULL; foreach(partitionCell, partitionList) { Oid partitionTableId = lfirst_oid(partitionCell); if (!RelationCanPublishAllModifications(partitionTableId)) { foundRelationWithNoReplicaIdentity = true; break; } } } /* check for source relation if it is not partitioned */ else { if (!RelationCanPublishAllModifications(relationId)) { foundRelationWithNoReplicaIdentity = true; } } if (foundRelationWithNoReplicaIdentity) { char *relationName = get_rel_name(relationId); ereport(NOTICE, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s does not have a REPLICA " "IDENTITY or PRIMARY KEY", relationName), errdetail("UPDATE and DELETE commands on the relation will " "error out during create_distributed_table_concurrently unless " "there is a REPLICA IDENTITY or PRIMARY KEY. " "INSERT commands will still work."))); } } /* * HashSplitPointsForShardList returns a list of split points which match * the shard ranges of the given list of shards; */ static List * HashSplitPointsForShardList(List *shardList) { List *splitPointList = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardList) { int32 shardMaxValue = DatumGetInt32(shardInterval->maxValue); splitPointList = lappend_int(splitPointList, shardMaxValue); } /* * Split point lists only include the upper boundaries. */ splitPointList = list_delete_last(splitPointList); return splitPointList; } /* * HashSplitPointsForShardCount returns a list of split points for a given * shard count with roughly equal hash ranges. */ static List * HashSplitPointsForShardCount(int shardCount) { List *splitPointList = NIL; /* calculate the split of the hash space */ uint64 hashTokenIncrement = HASH_TOKEN_COUNT / shardCount; /* * Split points lists only include the upper boundaries, so we only * go up to shardCount - 1 and do not have to apply the correction * for the last shardmaxvalue. */ for (int64 shardIndex = 0; shardIndex < shardCount - 1; shardIndex++) { /* initialize the hash token space for this shard */ int32 shardMinValue = PG_INT32_MIN + (shardIndex * hashTokenIncrement); int32 shardMaxValue = shardMinValue + (hashTokenIncrement - 1); splitPointList = lappend_int(splitPointList, shardMaxValue); } return splitPointList; } /* * WorkerNodesForShardList returns a list of node ids reflecting the locations of * the given list of shards. */ static List * WorkerNodesForShardList(List *shardList) { List *nodeIdList = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardList) { WorkerNode *workerNode = ActiveShardPlacementWorkerNode(shardInterval->shardId); nodeIdList = lappend_int(nodeIdList, workerNode->nodeId); } return nodeIdList; } /* * RoundRobinWorkerNodeList round robins over the workers in the worker node list * and adds node ids to a list of length listLength. */ static List * RoundRobinWorkerNodeList(List *workerNodeList, int listLength) { Assert(workerNodeList != NIL); List *nodeIdList = NIL; for (int idx = 0; idx < listLength; idx++) { int nodeIdx = idx % list_length(workerNodeList); WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList, nodeIdx); nodeIdList = lappend_int(nodeIdList, workerNode->nodeId); } return nodeIdList; } /* * create_reference_table creates a distributed table with the given relationId. The * created table has one shard and replication factor is set to the active worker * count. In fact, the above is the definition of a reference table in Citus. */ Datum create_reference_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); CreateReferenceTable(relationId); PG_RETURN_VOID(); } /* * EnsureCitusTableCanBeCreated checks if * - we are on the coordinator * - the current user is the owner of the table * - relation kind is supported * - relation is not a shard */ static void EnsureCitusTableCanBeCreated(Oid relationOid) { EnsureCoordinator(); EnsureRelationExists(relationOid); EnsureTableOwner(relationOid); ErrorIfTemporaryTable(relationOid); ErrorIfForeignTable(relationOid); /* * We should do this check here since the codes in the following lines rely * on this relation to have a supported relation kind. More extensive checks * will be performed in CreateDistributedTable. */ EnsureRelationKindSupported(relationOid); /* * When coordinator is added to the metadata, or on the workers, * some of the relations of the coordinator node may/will be shards. * We disallow creating distributed tables from shard relations, by * erroring out here. */ ErrorIfRelationIsAKnownShard(relationOid); } /* * EnsureRelationExists does a basic check on whether the OID belongs to * an existing relation. */ void EnsureRelationExists(Oid relationId) { if (!RelationExists(relationId)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relation with OID %d does not exist", relationId))); } } /* * CreateReferenceTable is a wrapper around CreateCitusTable that creates a * distributed table. */ void CreateDistributedTable(Oid relationId, char *distributionColumnName, char distributionMethod, int shardCount, bool shardCountIsStrict, char *colocateWithTableName) { CitusTableType tableType; switch (distributionMethod) { case DISTRIBUTE_BY_HASH: { tableType = HASH_DISTRIBUTED; break; } case DISTRIBUTE_BY_APPEND: { tableType = APPEND_DISTRIBUTED; break; } case DISTRIBUTE_BY_RANGE: { tableType = RANGE_DISTRIBUTED; break; } default: { ereport(ERROR, (errmsg("unexpected distribution method when " "deciding Citus table type"))); break; } } DistributedTableParams distributedTableParams = { .colocationParam = { .colocateWithTableName = colocateWithTableName, .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT }, .shardCount = shardCount, .shardCountIsStrict = shardCountIsStrict, .distributionColumnName = distributionColumnName }; CreateCitusTable(relationId, tableType, &distributedTableParams); } /* * CreateReferenceTable creates a reference table. */ void CreateReferenceTable(Oid relationId) { if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * Create the shard of given Citus local table on workers to convert * it into a reference table. */ ConvertCitusLocalTableToTableType(relationId, REFERENCE_TABLE, NULL); } else { CreateCitusTable(relationId, REFERENCE_TABLE, NULL); } } /* * CreateSingleShardTable creates a single shard distributed table that * doesn't have a shard key. */ void CreateSingleShardTable(Oid relationId, ColocationParam colocationParam) { DistributedTableParams distributedTableParams = { .colocationParam = colocationParam, .shardCount = 1, .shardCountIsStrict = true, .distributionColumnName = NULL }; if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * Create the shard of given Citus local table on appropriate node * and drop the local one to convert it into a single-shard distributed * table. */ ConvertCitusLocalTableToTableType(relationId, SINGLE_SHARD_DISTRIBUTED, &distributedTableParams); } else { CreateCitusTable(relationId, SINGLE_SHARD_DISTRIBUTED, &distributedTableParams); } } /* * CreateCitusTable is the internal method that creates a Citus table in * given configuration. * * DistributedTableParams should be non-null only if we're creating a distributed * table. * * This functions contains all necessary logic to create distributed tables. It * performs necessary checks to ensure distributing the table is safe. If it is * safe to distribute the table, this function creates distributed table metadata, * creates shards and copies local data to shards. This function also handles * partitioned tables by distributing its partitions as well. */ static void CreateCitusTable(Oid relationId, CitusTableType tableType, DistributedTableParams *distributedTableParams) { if ((tableType == HASH_DISTRIBUTED || tableType == APPEND_DISTRIBUTED || tableType == SINGLE_SHARD_DISTRIBUTED || tableType == RANGE_DISTRIBUTED) != (distributedTableParams != NULL)) { ereport(ERROR, (errmsg("distributed table params must be provided " "when creating a distributed table and must " "not be otherwise"))); } EnsureCitusTableCanBeCreated(relationId); /* allow creating a Citus table on an empty cluster */ InsertCoordinatorIfClusterEmpty(); Relation relation = try_relation_open(relationId, ExclusiveLock); if (relation == NULL) { ereport(ERROR, (errmsg("could not create Citus table: " "relation does not exist"))); } relation_close(relation, NoLock); if (tableType == SINGLE_SHARD_DISTRIBUTED && ShardReplicationFactor > 1) { ereport(ERROR, (errmsg("could not create single shard table: " "citus.shard_replication_factor is greater than 1"), errhint("Consider setting citus.shard_replication_factor to 1 " "and try again"))); } /* * EnsureTableNotDistributed errors out when relation is a citus table but * we don't want to ask user to first undistribute their citus local tables * when creating distributed tables from them. * For this reason, here we undistribute citus local tables beforehand. * But since UndistributeTable does not support undistributing relations * involved in foreign key relationships, we first drop foreign keys that * given relation is involved, then we undistribute the relation and finally * we re-create dropped foreign keys at the end of this function. */ List *originalForeignKeyRecreationCommands = NIL; if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * We use ConvertCitusLocalTableToTableType instead of CreateCitusTable * to create a reference table or a single-shard table from a Citus * local table. */ Assert(tableType != REFERENCE_TABLE && tableType != SINGLE_SHARD_DISTRIBUTED); /* store foreign key creation commands that relation is involved */ originalForeignKeyRecreationCommands = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES); relationId = DropFKeysAndUndistributeTable(relationId); } /* * To support foreign keys between reference tables and local tables, * we drop & re-define foreign keys at the end of this function so * that ALTER TABLE hook does the necessary job, which means converting * local tables to citus local tables to properly support such foreign * keys. */ else if (tableType == REFERENCE_TABLE && ShouldEnableLocalReferenceForeignKeys() && HasForeignKeyWithLocalTable(relationId)) { /* * Store foreign key creation commands for foreign key relationships * that relation has with postgres tables. */ originalForeignKeyRecreationCommands = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES); /* * Soon we will convert local tables to citus local tables. As * CreateCitusLocalTable needs to use local execution, now we * switch to local execution beforehand so that reference table * creation doesn't use remote execution and we don't error out * in CreateCitusLocalTable. */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES); } LockRelationOid(relationId, ExclusiveLock); EnsureTableNotDistributed(relationId); PropagatePrerequisiteObjectsForDistributedTable(relationId); Var *distributionColumn = NULL; if (distributedTableParams && distributedTableParams->distributionColumnName) { distributionColumn = BuildDistributionKeyFromColumnName(relationId, distributedTableParams-> distributionColumnName, NoLock); } CitusTableParams citusTableParams = DecideCitusTableParams(tableType, distributedTableParams); /* * ColocationIdForNewTable assumes caller acquires lock on relationId. In our case, * our caller already acquired lock on relationId. */ uint32 colocationId = INVALID_COLOCATION_ID; if (distributedTableParams && distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_COLOCATION_ID) { colocationId = distributedTableParams->colocationParam.colocationId; } else { /* * ColocationIdForNewTable assumes caller acquires lock on relationId. In our case, * our caller already acquired lock on relationId. */ colocationId = ColocationIdForNewTable(relationId, tableType, distributedTableParams, distributionColumn); } EnsureRelationCanBeDistributed(relationId, distributionColumn, citusTableParams.distributionMethod, colocationId, citusTableParams.replicationModel); /* * Make sure that existing reference tables have been replicated to all the nodes * such that we can create foreign keys and joins work immediately after creation. * * This will take a lock on the nodes to make sure no nodes are added after we have * verified and ensured the reference tables are copied everywhere. * Although copying reference tables here for anything but creating a new colocation * group, it requires significant refactoring which we don't want to perform now. */ EnsureReferenceTablesExistOnAllNodes(); /* * While adding tables to a colocation group we need to make sure no concurrent * mutations happen on the colocation group with regards to its placements. It is * important that we have already copied any reference tables before acquiring this * lock as these are competing operations. */ LockColocationId(colocationId, ShareLock); /* we need to calculate these variables before creating distributed metadata */ bool localTableEmpty = TableEmpty(relationId); Oid colocatedTableId = ColocatedTableId(colocationId); /* setting to false since this flag is only valid for citus local tables */ bool autoConverted = false; /* create an entry for distributed table in pg_dist_partition */ InsertIntoPgDistPartition(relationId, citusTableParams.distributionMethod, distributionColumn, colocationId, citusTableParams.replicationModel, autoConverted); /* * PG16+ supports truncate triggers on foreign tables */ if (RegularTable(relationId) || IsForeignTable(relationId)) { CreateTruncateTrigger(relationId); } if (tableType == HASH_DISTRIBUTED) { /* create shards for hash distributed table */ CreateHashDistributedTableShards(relationId, distributedTableParams->shardCount, colocatedTableId, localTableEmpty); } else if (tableType == REFERENCE_TABLE) { /* create shards for reference table */ CreateReferenceTableShard(relationId); } else if (tableType == SINGLE_SHARD_DISTRIBUTED) { /* create the shard of given single-shard distributed table */ CreateSingleShardTableShard(relationId, colocatedTableId, colocationId); } if (ShouldSyncTableMetadata(relationId)) { SyncCitusTableMetadata(relationId); } /* * We've a custom way of foreign key graph invalidation, * see InvalidateForeignKeyGraph(). */ if (TableReferenced(relationId) || TableReferencing(relationId)) { InvalidateForeignKeyGraph(); } /* if this table is partitioned table, distribute its partitions too */ if (PartitionedTable(relationId)) { List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; char *parentRelationName = generate_qualified_relation_name(relationId); /* * when there are many partitions, each call to CreateDistributedTable * accumulates used memory. Create and free context for each call. */ MemoryContext citusPartitionContext = AllocSetContextCreate(CurrentMemoryContext, "citus_per_partition_context", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(citusPartitionContext); foreach_declared_oid(partitionRelationId, partitionList) { MemoryContextReset(citusPartitionContext); DistributedTableParams childDistributedTableParams = { .colocationParam = { .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT, .colocateWithTableName = parentRelationName, }, .shardCount = distributedTableParams->shardCount, .shardCountIsStrict = false, .distributionColumnName = distributedTableParams->distributionColumnName, }; CreateCitusTable(partitionRelationId, tableType, &childDistributedTableParams); } MemoryContextSwitchTo(oldContext); MemoryContextDelete(citusPartitionContext); } /* copy over data for hash distributed and reference tables */ if (tableType == HASH_DISTRIBUTED || tableType == SINGLE_SHARD_DISTRIBUTED || tableType == REFERENCE_TABLE) { if (RegularTable(relationId)) { CopyLocalDataIntoShards(relationId); } } /* * Now recreate foreign keys that we dropped beforehand. As modifications are not * allowed on the relations that are involved in the foreign key relationship, * we can skip the validation of the foreign keys. */ bool skip_validation = true; ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands, skip_validation); } /* * ConvertCitusLocalTableToTableType converts given Citus local table to * given table type. * * This only supports converting Citus local tables to reference tables * (by replicating the shard to workers) and single-shard distributed * tables (by replicating the shard to the appropriate worker and dropping * the local one). */ static void ConvertCitusLocalTableToTableType(Oid relationId, CitusTableType tableType, DistributedTableParams *distributedTableParams) { if (!IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("table is not a local table added to metadata"))); } if (tableType != REFERENCE_TABLE && tableType != SINGLE_SHARD_DISTRIBUTED) { ereport(ERROR, (errmsg("table type is not supported for conversion"))); } if ((tableType == SINGLE_SHARD_DISTRIBUTED) != (distributedTableParams != NULL)) { ereport(ERROR, (errmsg("distributed table params must be provided " "when creating a distributed table and must " "not be otherwise"))); } EnsureCitusTableCanBeCreated(relationId); Relation relation = try_relation_open(relationId, ExclusiveLock); if (relation == NULL) { ereport(ERROR, (errmsg("could not create Citus table: " "relation does not exist"))); } relation_close(relation, NoLock); if (tableType == SINGLE_SHARD_DISTRIBUTED && ShardReplicationFactor > 1) { ereport(ERROR, (errmsg("could not create single shard table: " "citus.shard_replication_factor is greater than 1"), errhint("Consider setting citus.shard_replication_factor to 1 " "and try again"))); } LockRelationOid(relationId, ExclusiveLock); Var *distributionColumn = NULL; CitusTableParams citusTableParams = DecideCitusTableParams(tableType, distributedTableParams); uint32 colocationId = INVALID_COLOCATION_ID; if (distributedTableParams && distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_COLOCATION_ID) { colocationId = distributedTableParams->colocationParam.colocationId; } else { colocationId = ColocationIdForNewTable(relationId, tableType, distributedTableParams, distributionColumn); } /* check constraints etc. on table based on new distribution params */ EnsureRelationCanBeDistributed(relationId, distributionColumn, citusTableParams.distributionMethod, colocationId, citusTableParams.replicationModel); /* * Regarding the foreign key relationships that given relation is involved, * EnsureRelationCanBeDistributed() only checks the ones where the relation * is the referencing table. And given that the table at hand is a Citus * local table, right now it may only be referenced by a reference table * or a Citus local table. However, given that neither of those two cases * are not applicable for a distributed table, here we throw an error if * that's the case. * * Note that we don't need to check the same if we're creating a reference * table from a Citus local table because all the foreign keys referencing * Citus local tables are supported by reference tables. */ if (tableType == SINGLE_SHARD_DISTRIBUTED) { EnsureNoFKeyFromTableType(relationId, INCLUDE_CITUS_LOCAL_TABLES | INCLUDE_REFERENCE_TABLES); } EnsureReferenceTablesExistOnAllNodes(); LockColocationId(colocationId, ShareLock); /* * When converting to a single shard table, we want to drop the placement * on the coordinator, but only if transferring to a different node. In that * case, shouldDropLocalPlacement is true. When converting to a reference * table, we always keep the placement on the coordinator, so for reference * tables shouldDropLocalPlacement is always false. */ bool shouldDropLocalPlacement = false; List *targetNodeList = NIL; if (tableType == SINGLE_SHARD_DISTRIBUTED) { uint32 targetNodeId = SingleShardTableColocationNodeId(colocationId); if (targetNodeId != CoordinatorNodeIfAddedAsWorkerOrError()->nodeId) { bool missingOk = false; WorkerNode *targetNode = FindNodeWithNodeId(targetNodeId, missingOk); targetNodeList = list_make1(targetNode); shouldDropLocalPlacement = true; } } else if (tableType == REFERENCE_TABLE) { targetNodeList = ActivePrimaryNonCoordinatorNodeList(ShareLock); targetNodeList = SortList(targetNodeList, CompareWorkerNodes); } bool autoConverted = false; UpdateNoneDistTableMetadataGlobally( relationId, citusTableParams.replicationModel, colocationId, autoConverted); /* create the shard placement on workers and insert into pg_dist_placement globally */ if (list_length(targetNodeList) > 0) { NoneDistTableReplicateCoordinatorPlacement(relationId, targetNodeList); } if (shouldDropLocalPlacement) { /* * We don't yet drop the local placement before handling partitions. * Otherewise, local shard placements of the partitions will be gone * before we create them on workers. * * However, we need to delete the related entry from pg_dist_placement * before distributing partitions (if any) because we need a sane metadata * state before doing so. */ NoneDistTableDeleteCoordinatorPlacement(relationId); } /* if this table is partitioned table, distribute its partitions too */ if (PartitionedTable(relationId)) { /* right now we don't allow partitioned reference tables */ Assert(tableType == SINGLE_SHARD_DISTRIBUTED); List *partitionList = PartitionList(relationId); char *parentRelationName = generate_qualified_relation_name(relationId); /* * When there are many partitions, each call to * ConvertCitusLocalTableToTableType accumulates used memory. * Create and free citus_per_partition_context for each call. */ MemoryContext citusPartitionContext = AllocSetContextCreate(CurrentMemoryContext, "citus_per_partition_context", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(citusPartitionContext); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { MemoryContextReset(citusPartitionContext); DistributedTableParams childDistributedTableParams = { .colocationParam = { .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT, .colocateWithTableName = parentRelationName, }, .shardCount = distributedTableParams->shardCount, .shardCountIsStrict = false, .distributionColumnName = distributedTableParams->distributionColumnName, }; ConvertCitusLocalTableToTableType(partitionRelationId, tableType, &childDistributedTableParams); } MemoryContextSwitchTo(oldContext); MemoryContextDelete(citusPartitionContext); } if (shouldDropLocalPlacement) { NoneDistTableDropCoordinatorPlacementTable(relationId); } } /* * DecideCitusTableParams decides CitusTableParams based on given CitusTableType * and DistributedTableParams if it's a distributed table. * * DistributedTableParams should be non-null only if CitusTableType corresponds * to a distributed table. */ static CitusTableParams DecideCitusTableParams(CitusTableType tableType, DistributedTableParams *distributedTableParams) { CitusTableParams citusTableParams = { 0 }; switch (tableType) { case HASH_DISTRIBUTED: { Assert(distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_TABLE_LIKE_OPT); citusTableParams.distributionMethod = DISTRIBUTE_BY_HASH; citusTableParams.replicationModel = DecideDistTableReplicationModel(DISTRIBUTE_BY_HASH, distributedTableParams->colocationParam. colocateWithTableName); break; } case APPEND_DISTRIBUTED: { Assert(distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_TABLE_LIKE_OPT); citusTableParams.distributionMethod = DISTRIBUTE_BY_APPEND; citusTableParams.replicationModel = DecideDistTableReplicationModel(APPEND_DISTRIBUTED, distributedTableParams->colocationParam. colocateWithTableName); break; } case RANGE_DISTRIBUTED: { Assert(distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_TABLE_LIKE_OPT); citusTableParams.distributionMethod = DISTRIBUTE_BY_RANGE; citusTableParams.replicationModel = DecideDistTableReplicationModel(RANGE_DISTRIBUTED, distributedTableParams->colocationParam. colocateWithTableName); break; } case SINGLE_SHARD_DISTRIBUTED: { citusTableParams.distributionMethod = DISTRIBUTE_BY_NONE; citusTableParams.replicationModel = REPLICATION_MODEL_STREAMING; break; } case REFERENCE_TABLE: { citusTableParams.distributionMethod = DISTRIBUTE_BY_NONE; citusTableParams.replicationModel = REPLICATION_MODEL_2PC; break; } default: { ereport(ERROR, (errmsg("unexpected table type when deciding Citus " "table params"))); break; } } return citusTableParams; } /* * PropagatePrerequisiteObjectsForDistributedTable ensures we can create shards * on all nodes by ensuring all dependent objects exist on all node. */ static void PropagatePrerequisiteObjectsForDistributedTable(Oid relationId) { /* * Ensure that the sequences used in column defaults of the table * have proper types */ EnsureRelationHasCompatibleSequenceTypes(relationId); /* * distributed tables might have dependencies on different objects, since we create * shards for a distributed table via multiple sessions these objects will be created * via their own connection and committed immediately so they become visible to all * sessions creating shards. */ ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*tableAddress, RelationRelationId, relationId); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(tableAddress)); TrackPropagatedTableAndSequences(relationId); } /* * EnsureSequenceTypeSupported ensures that the type of the column that uses * a sequence on its DEFAULT is consistent with previous uses (if any) of the * sequence in distributed tables. * If any other distributed table uses the input sequence, it checks whether * the types of the columns using the sequence match. If they don't, it errors out. * Otherwise, the condition is ensured. * Since the owner of the sequence may not distributed yet, it should be added * explicitly. */ void EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid ownerRelationId) { Oid attrDefOid; List *attrDefOids = GetAttrDefsFromSequence(seqOid); foreach_declared_oid(attrDefOid, attrDefOids) { ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); /* * If another distributed table is using the same sequence * in one of its column defaults, make sure the types of the * columns match. * * We skip non-distributed tables, but we need to check the current * table as it might reference the same sequence multiple times. */ if (columnAddress.objectId != ownerRelationId && !IsCitusTable(columnAddress.objectId)) { continue; } Oid currentAttributeTypId = GetAttributeTypeOid(columnAddress.objectId, columnAddress.objectSubId); if (attributeTypeId != currentAttributeTypId) { char *sequenceName = generate_qualified_relation_name( seqOid); char *citusTableName = generate_qualified_relation_name(columnAddress.objectId); ereport(ERROR, (errmsg( "The sequence %s is already used for a different" " type in column %d of the table %s", sequenceName, columnAddress.objectSubId, citusTableName))); } } } /* * AlterSequenceType alters the given sequence's type to the given type. */ void AlterSequenceType(Oid seqOid, Oid typeOid) { Form_pg_sequence sequenceData = pg_get_sequencedef(seqOid); Oid currentSequenceTypeOid = sequenceData->seqtypid; if (currentSequenceTypeOid != typeOid) { AlterSeqStmt *alterSequenceStatement = makeNode(AlterSeqStmt); char *seqNamespace = get_namespace_name(get_rel_namespace(seqOid)); char *seqName = get_rel_name(seqOid); alterSequenceStatement->sequence = makeRangeVar(seqNamespace, seqName, -1); Node *asTypeNode = (Node *) makeTypeNameFromOid(typeOid, -1); SetDefElemArg(alterSequenceStatement, "as", asTypeNode); ParseState *pstate = make_parsestate(NULL); AlterSequence(pstate, alterSequenceStatement); CommandCounterIncrement(); } } /* * EnsureRelationHasCompatibleSequenceTypes ensures that sequences used for columns * of the table have compatible types both with the column type on that table and * all other distributed tables' columns they have used for */ void EnsureRelationHasCompatibleSequenceTypes(Oid relationId) { List *seqInfoList = NIL; GetDependentSequencesWithRelation(relationId, &seqInfoList, 0, DEPENDENCY_AUTO); EnsureDistributedSequencesHaveOneType(relationId, seqInfoList); } /* * EnsureDistributedSequencesHaveOneType first ensures that the type of the column * in which the sequence is used as default is supported for each sequence in input * dependentSequenceList, and then alters the sequence type if not the same with the column type. */ static void EnsureDistributedSequencesHaveOneType(Oid relationId, List *seqInfoList) { SequenceInfo *seqInfo = NULL; foreach_declared_ptr(seqInfo, seqInfoList) { if (!seqInfo->isNextValDefault) { /* * If a sequence is not on the nextval, we don't need any check. * This is a dependent sequence via ALTER SEQUENCE .. OWNED BY col */ continue; } /* * We should make sure that the type of the column that uses * that sequence is supported */ Oid sequenceOid = seqInfo->sequenceOid; AttrNumber attnum = seqInfo->attributeNumber; Oid attributeTypeId = GetAttributeTypeOid(relationId, attnum); EnsureSequenceTypeSupported(sequenceOid, attributeTypeId, relationId); /* * Alter the sequence's data type in the coordinator if needed. * * First, we should only change the sequence type if the column * is a supported sequence type. For example, if a sequence is used * in an expression which then becomes a text, we should not try to * alter the sequence type to text. Postgres only supports int2, int4 * and int8 as the sequence type. * * A sequence's type is bigint by default and it doesn't change even if * it's used in an int column. We should change the type if needed, * and not allow future ALTER SEQUENCE ... TYPE ... commands for * sequences used as defaults in distributed tables. */ if (attributeTypeId == INT2OID || attributeTypeId == INT4OID || attributeTypeId == INT8OID) { AlterSequenceType(sequenceOid, attributeTypeId); } } } /* * DecideDistTableReplicationModel function decides which replication model should be * used for a distributed table depending on given distribution configuration. */ static char DecideDistTableReplicationModel(char distributionMethod, char *colocateWithTableName) { Assert(distributionMethod != DISTRIBUTE_BY_NONE); if (!IsColocateWithDefault(colocateWithTableName) && !IsColocateWithNone(colocateWithTableName)) { text *colocateWithTableNameText = cstring_to_text(colocateWithTableName); Oid colocatedRelationId = ResolveRelationId(colocateWithTableNameText, false); CitusTableCacheEntry *targetTableEntry = GetCitusTableCacheEntry( colocatedRelationId); char replicationModel = targetTableEntry->replicationModel; return replicationModel; } else if (distributionMethod == DISTRIBUTE_BY_HASH && !DistributedTableReplicationIsEnabled()) { return REPLICATION_MODEL_STREAMING; } else { return REPLICATION_MODEL_COORDINATOR; } /* we should not reach to this point */ return REPLICATION_MODEL_INVALID; } /* * CreateHashDistributedTableShards creates shards of given hash distributed table. */ static void CreateHashDistributedTableShards(Oid relationId, int shardCount, Oid colocatedTableId, bool localTableEmpty) { bool useExclusiveConnection = false; /* * Decide whether to use exclusive connections per placement or not. Note that * if the local table is not empty, we cannot use sequential mode since the COPY * operation that'd load the data into shards currently requires exclusive * connections. */ if (RegularTable(relationId)) { useExclusiveConnection = CanUseExclusiveConnections(relationId, localTableEmpty); } if (colocatedTableId != InvalidOid) { /* * We currently allow concurrent distribution of colocated tables (which * we probably should not be allowing because of foreign keys / * partitioning etc). * * We also prevent concurrent shard moves / copy / splits) while creating * a colocated table. */ AcquirePlacementColocationLock(colocatedTableId, ShareLock, "colocate distributed table"); CreateColocatedShards(relationId, colocatedTableId, useExclusiveConnection); } else { /* * This path is only reached by create_distributed_table for the distributed * tables which will not be part of an existing colocation group. Therefore, * we can directly use ShardReplicationFactor global variable here. */ CreateShardsWithRoundRobinPolicy(relationId, shardCount, ShardReplicationFactor, useExclusiveConnection); } } /* * CreateSingleShardTableShard creates the shard of given single-shard * distributed table. */ static void CreateSingleShardTableShard(Oid relationId, Oid colocatedTableId, uint32 colocationId) { if (colocatedTableId != InvalidOid) { /* * We currently allow concurrent distribution of colocated tables (which * we probably should not be allowing because of foreign keys / * partitioning etc). * * We also prevent concurrent shard moves / copy / splits) while creating * a colocated table. */ AcquirePlacementColocationLock(colocatedTableId, ShareLock, "colocate distributed table"); /* * We don't need to force using exclusive connections because we're anyway * creating a single shard. */ bool useExclusiveConnection = false; CreateColocatedShards(relationId, colocatedTableId, useExclusiveConnection); } else { CreateSingleShardTableShardWithRoundRobinPolicy(relationId, colocationId); } } /* * ColocationIdForNewTable returns a colocation id for given table * according to given configuration. If there is no such configuration, it * creates one and returns colocation id of newly the created colocation group. * Note that DistributedTableParams and the distribution column Var should be * non-null only if CitusTableType corresponds to a distributed table. * * For append and range distributed tables, this function errors out if * colocateWithTableName parameter is not NULL, otherwise directly returns * INVALID_COLOCATION_ID. * * For reference tables, returns the common reference table colocation id. * * This function assumes its caller take necessary lock on relationId to * prevent possible changes on it. */ static uint32 ColocationIdForNewTable(Oid relationId, CitusTableType tableType, DistributedTableParams *distributedTableParams, Var *distributionColumn) { CitusTableParams citusTableParams = DecideCitusTableParams(tableType, distributedTableParams); uint32 colocationId = INVALID_COLOCATION_ID; if (tableType == APPEND_DISTRIBUTED || tableType == RANGE_DISTRIBUTED) { Assert(distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_TABLE_LIKE_OPT); char *colocateWithTableName = distributedTableParams->colocationParam.colocateWithTableName; if (!IsColocateWithDefault(colocateWithTableName)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute relation"), errdetail("Currently, colocate_with option is not supported " "for append / range distributed tables."))); } return colocationId; } else if (tableType == REFERENCE_TABLE) { return CreateReferenceTableColocationId(); } else { /* * Get an exclusive lock on the colocation system catalog. Therefore, we * can be sure that there will no modifications on the colocation table * until this transaction is committed. */ /* distributionColumn can only be null for single-shard tables */ Oid distributionColumnType = distributionColumn ? distributionColumn->vartype : InvalidOid; Oid distributionColumnCollation = distributionColumn ? distributionColumn->varcollid : InvalidOid; Assert(distributedTableParams->colocationParam.colocationParamType == COLOCATE_WITH_TABLE_LIKE_OPT); char *colocateWithTableName = distributedTableParams->colocationParam.colocateWithTableName; /* get an advisory lock to serialize concurrent default group creations */ if (IsColocateWithDefault(colocateWithTableName)) { AcquireColocationDefaultLock(); } colocationId = FindColocateWithColocationId(relationId, citusTableParams.replicationModel, distributionColumn, distributedTableParams->shardCount, distributedTableParams-> shardCountIsStrict, colocateWithTableName); if (IsColocateWithDefault(colocateWithTableName) && (colocationId != INVALID_COLOCATION_ID)) { /* * we can release advisory lock if there is already a default entry for given params; * else, we should keep it to prevent different default coloc entry creation by * concurrent operations. */ ReleaseColocationDefaultLock(); } if (colocationId == INVALID_COLOCATION_ID) { if (IsColocateWithDefault(colocateWithTableName)) { /* * Generate a new colocation ID and insert a pg_dist_colocation * record. */ colocationId = CreateColocationGroup(distributedTableParams->shardCount, ShardReplicationFactor, distributionColumnType, distributionColumnCollation); } else if (IsColocateWithNone(colocateWithTableName)) { /* * Generate a new colocation ID and insert a pg_dist_colocation * record. */ colocationId = CreateColocationGroup(distributedTableParams->shardCount, ShardReplicationFactor, distributionColumnType, distributionColumnCollation); } } } return colocationId; } /* * EnsureRelationCanBeDistributed checks whether Citus can safely distribute given * relation with the given configuration. We perform almost all safety checks for * distributing table here. If there is an unsatisfied requirement, we error out * and do not distribute the table. * * This function assumes, callers have already acquired necessary locks to ensure * there will not be any change in the given relation. */ static void EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, char distributionMethod, uint32 colocationId, char replicationModel) { Oid parentRelationId = InvalidOid; EnsureLocalTableEmptyIfNecessary(relationId, distributionMethod); /* user really wants triggers? */ if (EnableUnsafeTriggers) { ErrorIfRelationHasUnsupportedTrigger(relationId); } else { EnsureRelationHasNoTriggers(relationId); } /* we assume callers took necessary locks */ Relation relation = relation_open(relationId, NoLock); TupleDesc relationDesc = RelationGetDescr(relation); char *relationName = RelationGetRelationName(relation); ErrorIfTableIsACatalogTable(relation); /* verify target relation is not distributed by a generated stored column */ if (distributionMethod != DISTRIBUTE_BY_NONE) { DistributionColumnIsGeneratedCheck(relationDesc, distributionColumn, relationName); } /* verify target relation is not distributed by a column of type numeric with negative scale */ if (distributionMethod != DISTRIBUTE_BY_NONE && DistributionColumnUsesNumericColumnNegativeScale(relationDesc, distributionColumn)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute relation: %s", relationName), errdetail("Distribution column must not use numeric type " "with negative scale"))); } /* check for support function needed by specified partition method */ if (distributionMethod == DISTRIBUTE_BY_HASH) { Oid hashSupportFunction = SupportFunctionForColumn(distributionColumn, HASH_AM_OID, HASHSTANDARD_PROC); if (hashSupportFunction == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not identify a hash function for type %s", format_type_be(distributionColumn->vartype)), errdatatype(distributionColumn->vartype), errdetail("Partition column types must have a hash function " "defined to use hash partitioning."))); } if (distributionColumn->varcollid != InvalidOid && !get_collation_isdeterministic(distributionColumn->varcollid)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Hash distributed partition columns may not use " "a non deterministic collation"))); } } else if (distributionMethod == DISTRIBUTE_BY_RANGE) { Oid btreeSupportFunction = SupportFunctionForColumn(distributionColumn, BTREE_AM_OID, BTORDER_PROC); if (btreeSupportFunction == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not identify a comparison function for type %s", format_type_be(distributionColumn->vartype)), errdatatype(distributionColumn->vartype), errdetail("Partition column types must have a comparison function " "defined to use range partitioning."))); } } if (PartitionTableNoLock(relationId)) { parentRelationId = PartitionParentOid(relationId); } /* partitions cannot be distributed if their parent is not distributed */ if (PartitionTableNoLock(relationId) && !IsCitusTable(parentRelationId)) { char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errmsg("cannot distribute relation \"%s\" which is partition of " "\"%s\"", relationName, parentRelationName), errdetail("Citus does not support distributing partitions " "if their parent is not distributed table."), errhint("Distribute the partitioned table \"%s\" instead.", parentRelationName))); } /* * These checks are mostly for partitioned tables not partitions because we prevent * distributing partitions directly in the above check. However, partitions can still * reach this point because, we call CreateDistributedTable for partitions if their * parent table is distributed. */ if (PartitionedTableNoLock(relationId)) { /* * Distributing partitioned tables is only supported for hash-distribution * or single-shard tables. */ bool isSingleShardTable = distributionMethod == DISTRIBUTE_BY_NONE && replicationModel == REPLICATION_MODEL_STREAMING && colocationId != INVALID_COLOCATION_ID; if (distributionMethod != DISTRIBUTE_BY_HASH && !isSingleShardTable) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("distributing partitioned tables in only supported " "for hash-distributed tables"))); } /* we don't support distributing tables with multi-level partitioning */ if (PartitionTableNoLock(relationId)) { char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("distributing multi-level partitioned tables " "is not supported"), errdetail("Relation \"%s\" is partitioned table itself and " "it is also partition of relation \"%s\".", relationName, parentRelationName))); } } ErrorIfUnsupportedConstraint(relation, distributionMethod, replicationModel, distributionColumn, colocationId); ErrorIfUnsupportedPolicy(relation); relation_close(relation, NoLock); } /* * ErrorIfTemporaryTable errors out if the given table is a temporary table. */ static void ErrorIfTemporaryTable(Oid relationId) { if (get_rel_persistence(relationId) == RELPERSISTENCE_TEMP) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute a temporary table"))); } } /* * ErrorIfTableIsACatalogTable is a helper function to error out for citus * table creation from a catalog table. */ void ErrorIfTableIsACatalogTable(Relation relation) { if (relation->rd_rel->relnamespace != PG_CATALOG_NAMESPACE) { return; } ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create a citus table from a catalog table"))); } /* * EnsureLocalTableEmptyIfNecessary errors out if the function should be empty * according to ShouldLocalTableBeEmpty but it is not. */ static void EnsureLocalTableEmptyIfNecessary(Oid relationId, char distributionMethod) { if (ShouldLocalTableBeEmpty(relationId, distributionMethod)) { EnsureLocalTableEmpty(relationId); } } /* * ShouldLocalTableBeEmpty returns true if the local table should be empty * before creating a citus table. * In some cases, it is possible and safe to send local data to shards while * distributing the table. In those cases, we can distribute non-empty local * tables. This function checks the distributionMethod and relation kind to * see whether we need to be ensure emptiness of local table. */ static bool ShouldLocalTableBeEmpty(Oid relationId, char distributionMethod) { bool shouldLocalTableBeEmpty = false; if (distributionMethod != DISTRIBUTE_BY_HASH && distributionMethod != DISTRIBUTE_BY_NONE) { /* * We only support hash distributed tables and reference tables * for initial data loading */ shouldLocalTableBeEmpty = true; } else if (!RegularTable(relationId)) { /* * We only support tables and partitioned tables for initial * data loading */ shouldLocalTableBeEmpty = true; } return shouldLocalTableBeEmpty; } /* * EnsureLocalTableEmpty errors out if the local table is not empty. */ static void EnsureLocalTableEmpty(Oid relationId) { char *relationName = get_rel_name(relationId); bool localTableEmpty = TableEmpty(relationId); if (!localTableEmpty) { ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot distribute relation \"%s\"", relationName), errdetail("Relation \"%s\" contains data.", relationName), errhint("Empty your table before distributing it."))); } } /* * EnsureDistributableTable ensures the given table type is appropriate to * be distributed. Table type should be regular or citus local table. */ static void EnsureDistributableTable(Oid relationId) { bool isLocalTable = IsCitusTableType(relationId, CITUS_LOCAL_TABLE); bool isRegularTable = !IsCitusTableType(relationId, ANY_CITUS_TABLE_TYPE); if (!isLocalTable && !isRegularTable) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("table \"%s\" is already distributed", relationName))); } } /* * EnsureTableNotDistributed errors out if the table is distributed. */ void EnsureTableNotDistributed(Oid relationId) { char *relationName = get_rel_name(relationId); bool isCitusTable = IsCitusTable(relationId); if (isCitusTable) { ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("table \"%s\" is already distributed", relationName))); } } /* * EnsureRelationHasNoTriggers errors out if the given table has triggers on * it. See also GetExplicitTriggerIdList function's comment for the triggers this * function errors out. */ static void EnsureRelationHasNoTriggers(Oid relationId) { List *explicitTriggerIds = GetExplicitTriggerIdList(relationId); if (list_length(explicitTriggerIds) > 0) { char *relationName = get_rel_name(relationId); Assert(relationName != NULL); ereport(ERROR, (errmsg("cannot distribute relation \"%s\" because it " "has triggers", relationName), errhint("Consider dropping all the triggers on \"%s\" " "and retry.", relationName))); } } /* * LookupDistributionMethod maps the oids of citus.distribution_type enum * values to pg_dist_partition.partmethod values. * * The passed in oid has to belong to a value of citus.distribution_type. */ char LookupDistributionMethod(Oid distributionMethodOid) { char distributionMethod = 0; HeapTuple enumTuple = SearchSysCache1(ENUMOID, ObjectIdGetDatum( distributionMethodOid)); if (!HeapTupleIsValid(enumTuple)) { ereport(ERROR, (errmsg("invalid internal value for enum: %u", distributionMethodOid))); } Form_pg_enum enumForm = (Form_pg_enum) GETSTRUCT(enumTuple); const char *enumLabel = NameStr(enumForm->enumlabel); if (strncmp(enumLabel, "append", NAMEDATALEN) == 0) { distributionMethod = DISTRIBUTE_BY_APPEND; } else if (strncmp(enumLabel, "hash", NAMEDATALEN) == 0) { distributionMethod = DISTRIBUTE_BY_HASH; } else if (strncmp(enumLabel, "range", NAMEDATALEN) == 0) { distributionMethod = DISTRIBUTE_BY_RANGE; } else { ereport(ERROR, (errmsg("invalid label for enum: %s", enumLabel))); } ReleaseSysCache(enumTuple); return distributionMethod; } /* * SupportFunctionForColumn locates a support function given a column, an access method, * and and id of a support function. This function returns InvalidOid if there is no * support function for the operator class family of the column, but if the data type * of the column has no default operator class whatsoever, this function errors out. */ static Oid SupportFunctionForColumn(Var *partitionColumn, Oid accessMethodId, int16 supportFunctionNumber) { Oid columnOid = partitionColumn->vartype; Oid operatorClassId = GetDefaultOpClass(columnOid, accessMethodId); /* currently only support using the default operator class */ if (operatorClassId == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("data type %s has no default operator class for specified" " partition method", format_type_be(columnOid)), errdatatype(columnOid), errdetail("Partition column types must have a default operator" " class defined."))); } Oid operatorFamilyId = get_opclass_family(operatorClassId); Oid operatorClassInputType = get_opclass_input_type(operatorClassId); Oid supportFunctionOid = get_opfamily_proc(operatorFamilyId, operatorClassInputType, operatorClassInputType, supportFunctionNumber); return supportFunctionOid; } /* * TableEmpty function checks whether given table contains any row and * returns false if there is any data. */ bool TableEmpty(Oid tableId) { Oid schemaId = get_rel_namespace(tableId); char *schemaName = get_namespace_name(schemaId); char *tableName = get_rel_name(tableId); char *tableQualifiedName = quote_qualified_identifier(schemaName, tableName); StringInfo selectTrueQueryString = makeStringInfo(); bool readOnly = true; int spiConnectionResult = SPI_connect(); if (spiConnectionResult != SPI_OK_CONNECT) { ereport(ERROR, (errmsg("could not connect to SPI manager"))); } appendStringInfo(selectTrueQueryString, SELECT_TRUE_QUERY, tableQualifiedName); int spiQueryResult = SPI_execute(selectTrueQueryString->data, readOnly, 0); if (spiQueryResult != SPI_OK_SELECT) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", selectTrueQueryString->data))); } /* we expect that SELECT TRUE query will return single value in a single row OR empty set */ Assert(SPI_processed == 1 || SPI_processed == 0); bool localTableEmpty = !SPI_processed; SPI_finish(); return localTableEmpty; } /* * CanUseExclusiveConnections checks if we can open parallel connections * while creating shards. We simply error out if we need to execute * sequentially but there is data in the table, since we cannot copy the * data to shards sequentially. */ static bool CanUseExclusiveConnections(Oid relationId, bool localTableEmpty) { bool hasForeignKeyToReferenceTable = HasForeignKeyToReferenceTable(relationId); bool shouldRunSequential = MultiShardConnectionType == SEQUENTIAL_CONNECTION || hasForeignKeyToReferenceTable; if (shouldRunSequential && ParallelQueryExecutedInTransaction()) { /* * We decided to use sequential execution. It's either because relation * has a pre-existing foreign key to a reference table or because we * decided to use sequential execution due to a query executed in the * current xact beforehand. * We have specific error messages for either cases. */ char *relationName = get_rel_name(relationId); if (hasForeignKeyToReferenceTable) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg("cannot distribute relation \"%s\" in this " "transaction because it has a foreign key to " "a reference table", relationName), errdetail("If a hash distributed table has a foreign key " "to a reference table, it has to be created " "in sequential mode before any parallel commands " "have been executed in the same transaction"), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { ereport(ERROR, (errmsg("cannot distribute \"%s\" in sequential mode because " "a parallel query was executed in this transaction", relationName), errhint("If you have manually set " "citus.multi_shard_modify_mode to 'sequential', " "try with 'parallel' option. "))); } } else if (shouldRunSequential) { return false; } else if (!localTableEmpty || IsMultiStatementTransaction()) { return true; } return false; } /* * CreateTruncateTrigger creates a truncate trigger on table identified by relationId * and assigns citus_truncate_trigger() as handler. */ void CreateTruncateTrigger(Oid relationId) { StringInfo triggerName = makeStringInfo(); bool internal = true; appendStringInfo(triggerName, "truncate_trigger"); CreateTrigStmt *trigger = makeNode(CreateTrigStmt); trigger->trigname = triggerName->data; trigger->relation = NULL; trigger->funcname = SystemFuncName(CITUS_TRUNCATE_TRIGGER_NAME); trigger->args = NIL; trigger->row = false; trigger->timing = TRIGGER_TYPE_AFTER; trigger->events = TRIGGER_TYPE_TRUNCATE; trigger->columns = NIL; trigger->whenClause = NULL; trigger->isconstraint = false; CreateTrigger(trigger, NULL, relationId, InvalidOid, InvalidOid, InvalidOid, InvalidOid, InvalidOid, NULL, internal, false); } /* * RegularTable function returns true if given table's relation kind is RELKIND_RELATION * or RELKIND_PARTITIONED_TABLE otherwise it returns false. */ bool RegularTable(Oid relationId) { char relationKind = get_rel_relkind(relationId); if (relationKind == RELKIND_RELATION || relationKind == RELKIND_PARTITIONED_TABLE) { return true; } return false; } /* * CopyLocalDataIntoShards is a wrapper around CopyFromLocalTableIntoDistTable * to copy data from the local table, which is hidden after converting it to a * distributed table, into the shards of the distributed table. * * After copying local data into the distributed table, the local data remains * in place and should be truncated at a later time. */ static void CopyLocalDataIntoShards(Oid distributedTableId) { uint64 rowsCopied = CopyFromLocalTableIntoDistTable(distributedTableId, distributedTableId); if (rowsCopied > 0) { char *qualifiedRelationName = generate_qualified_relation_name(distributedTableId); ereport(NOTICE, (errmsg("copying the data has completed"), errdetail("The local data in the table is no longer visible, " "but is still on disk."), errhint("To remove the local data, run: SELECT " "truncate_local_data_after_distributing_table($$%s$$)", qualifiedRelationName))); } } /* * CopyFromLocalTableIntoDistTable copies data from given local table into * the shards of given distributed table. * * For partitioned tables, this functions returns without copying the data * because we call this function for both partitioned tables and its partitions. * Returning early saves us from copying data to workers twice. * * This function uses CitusCopyDestReceiver to invoke the distributed COPY logic. * We cannot use a regular COPY here since that cannot read from a table. Instead * we read from the table and pass each tuple to the CitusCopyDestReceiver which * opens a connection and starts a COPY for each shard placement that will have * data. * * We assume that the local table might indeed be a distributed table and the * caller would want to read the local data from the shell table in that case. * For this reason, to keep it simple, we perform a heap scan directly on the * table instead of using SELECT. * * We read from the table and pass each tuple to the CitusCopyDestReceiver which * opens a connection and starts a COPY for each shard placement that will have * data. */ uint64 CopyFromLocalTableIntoDistTable(Oid localTableId, Oid distributedTableId) { /* take an ExclusiveLock to block all operations except SELECT */ Relation localRelation = table_open(localTableId, ExclusiveLock); /* * Skip copying from partitioned tables, we will copy the data from * partition to partition's shards. */ if (PartitionedTable(distributedTableId)) { table_close(localRelation, NoLock); return 0; } /* * All writes have finished, make sure that we can see them by using the * latest snapshot. We use GetLatestSnapshot instead of * GetTransactionSnapshot since the latter would not reveal all writes * in serializable or repeatable read mode. Note that subsequent reads * from the distributed table would reveal those writes, temporarily * violating the isolation level. However, this seems preferable over * dropping the writes entirely. */ PushActiveSnapshot(GetLatestSnapshot()); Relation distributedRelation = RelationIdGetRelation(distributedTableId); /* get the table columns for distributed table */ TupleDesc destTupleDescriptor = RelationGetDescr(distributedRelation); List *columnNameList = TupleDescColumnNameList(destTupleDescriptor); RelationClose(distributedRelation); int partitionColumnIndex = INVALID_PARTITION_COLUMN_INDEX; /* determine the partition column in the tuple descriptor */ Var *partitionColumn = PartitionColumn(distributedTableId, 0); if (partitionColumn != NULL) { partitionColumnIndex = partitionColumn->varattno - 1; } /* create tuple slot for local relation */ TupleDesc sourceTupleDescriptor = RelationGetDescr(localRelation); TupleTableSlot *slot = table_slot_create(localRelation, NULL); /* initialise per-tuple memory context */ EState *estate = CreateExecutorState(); ExprContext *econtext = GetPerTupleExprContext(estate); econtext->ecxt_scantuple = slot; const bool nonPublishableData = false; /* we don't track query counters when distributing a table */ const bool trackQueryCounters = false; DestReceiver *copyDest = (DestReceiver *) CreateCitusCopyDestReceiver(distributedTableId, columnNameList, partitionColumnIndex, estate, NULL, nonPublishableData, trackQueryCounters); /* initialise state for writing to shards, we'll open connections on demand */ copyDest->rStartup(copyDest, 0, sourceTupleDescriptor); uint64 rowsCopied = DoCopyFromLocalTableIntoShards(localRelation, copyDest, slot, estate); /* finish writing into the shards */ copyDest->rShutdown(copyDest); copyDest->rDestroy(copyDest); /* free memory and close the relation */ ExecDropSingleTupleTableSlot(slot); FreeExecutorState(estate); table_close(localRelation, NoLock); PopActiveSnapshot(); return rowsCopied; } /* * DoCopyFromLocalTableIntoShards performs a copy operation * from local tables into shards. * * Returns the number of rows copied. */ static uint64 DoCopyFromLocalTableIntoShards(Relation localRelation, DestReceiver *copyDest, TupleTableSlot *slot, EState *estate) { /* begin reading from local table */ TableScanDesc scan = table_beginscan(localRelation, GetActiveSnapshot(), 0, NULL); MemoryContext oldContext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); uint64 rowsCopied = 0; while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { /* send tuple it to a shard */ copyDest->receiveSlot(slot, copyDest); /* clear tuple memory */ ResetPerTupleExprContext(estate); /* make sure we roll back on cancellation */ CHECK_FOR_INTERRUPTS(); if (rowsCopied == 0) { ereport(NOTICE, (errmsg("Copying data from local table..."))); } rowsCopied++; if (rowsCopied % LOG_PER_TUPLE_AMOUNT == 0) { ereport(DEBUG1, (errmsg("Copied " UINT64_FORMAT " rows", rowsCopied))); } } if (rowsCopied % LOG_PER_TUPLE_AMOUNT != 0) { ereport(DEBUG1, (errmsg("Copied " UINT64_FORMAT " rows", rowsCopied))); } MemoryContextSwitchTo(oldContext); /* finish reading from the local table */ table_endscan(scan); return rowsCopied; } /* * TupleDescColumnNameList returns a list of column names for the given tuple * descriptor as plain strings. */ static List * TupleDescColumnNameList(TupleDesc tupleDescriptor) { List *columnNameList = NIL; for (int columnIndex = 0; columnIndex < tupleDescriptor->natts; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); char *columnName = NameStr(currentColumn->attname); if (IsDroppedOrGenerated(currentColumn)) { continue; } columnNameList = lappend(columnNameList, columnName); } return columnNameList; } /* * is_valid_numeric_typmod checks if the typmod value is valid * * Because of the offset, valid numeric typmods are at least VARHDRSZ * * Copied from PG. See numeric.c for understanding how this works. */ static bool is_valid_numeric_typmod(int32 typmod) { return typmod >= (int32) VARHDRSZ; } /* * numeric_typmod_scale extracts the scale from a numeric typmod. * * Copied from PG. See numeric.c for understanding how this works. * */ static int numeric_typmod_scale(int32 typmod) { return (((typmod - VARHDRSZ) & 0x7ff) ^ 1024) - 1024; } /* * DistributionColumnUsesNumericColumnNegativeScale returns whether a given relation uses * numeric data type with negative scale on distribution column */ static bool DistributionColumnUsesNumericColumnNegativeScale(TupleDesc relationDesc, Var *distributionColumn) { Form_pg_attribute attributeForm = TupleDescAttr(relationDesc, distributionColumn->varattno - 1); if (attributeForm->atttypid == NUMERICOID && is_valid_numeric_typmod(attributeForm->atttypmod) && numeric_typmod_scale(attributeForm->atttypmod) < 0) { return true; } return false; } /* * DistributionColumnIsGeneratedCheck throws an error if a given relation uses * GENERATED ALWAYS AS (...) STORED | VIRTUAL on distribution column */ static void DistributionColumnIsGeneratedCheck(TupleDesc relationDesc, Var *distributionColumn, const char *relationName) { Form_pg_attribute attributeForm = TupleDescAttr(relationDesc, distributionColumn->varattno - 1); switch (attributeForm->attgenerated) { case ATTRIBUTE_GENERATED_STORED: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute relation: %s", relationName), errdetail("Distribution column must not use GENERATED ALWAYS " "AS (...) STORED."))); break; } #if PG_VERSION_NUM >= PG_VERSION_18 case ATTRIBUTE_GENERATED_VIRTUAL: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute relation: %s", relationName), errdetail("Distribution column must not use GENERATED ALWAYS " "AS (...) VIRTUAL."))); break; } #endif default: { break; } } } /* * ErrorIfForeignTable errors out if the relation with given relationOid * is a foreign table. */ static void ErrorIfForeignTable(Oid relationOid) { if (IsForeignTable(relationOid)) { char *relname = get_rel_name(relationOid); char *qualifiedRelname = generate_qualified_relation_name(relationOid); ereport(ERROR, (errmsg("foreign tables cannot be distributed"), (errhint("Can add foreign table \"%s\" to metadata by running: " "SELECT citus_add_local_table_to_metadata($$%s$$);", relname, qualifiedRelname)))); } } ================================================ FILE: src/backend/distributed/commands/database.c ================================================ /*------------------------------------------------------------------------- * * database.c * Commands to interact with the database object in a distributed * environment. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" #include "catalog/objectaddress.h" #include "catalog/pg_collation.h" #include "catalog/pg_database.h" #include "catalog/pg_database_d.h" #include "catalog/pg_tablespace.h" #include "commands/dbcommands.h" #include "commands/defrem.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "distributed/adaptive_executor.h" #include "distributed/commands.h" #include "distributed/commands/serialize_distributed_ddls.h" #include "distributed/commands/utility_hook.h" #include "distributed/comment.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/shard_cleaner.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* * Used to save original name of the database before it is replaced with a * temporary name for failure handling purposes in PreprocessCreateDatabaseStmt(). */ static char *CreateDatabaseCommandOriginalDbName = NULL; /* * The format string used when creating a temporary databases for failure * handling purposes. * * The fields are as follows to ensure using a unique name for each temporary * database: * - operationId: The operation id returned by RegisterOperationNeedingCleanup(). * - groupId: The group id of the worker node where CREATE DATABASE command * is issued from. */ #define TEMP_DATABASE_NAME_FMT "citus_temp_database_%lu_%d" /* * DatabaseCollationInfo is used to store collation related information of a database. */ typedef struct DatabaseCollationInfo { char *datcollate; char *datctype; char *daticulocale; char *datcollversion; char *daticurules; } DatabaseCollationInfo; static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm); static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid); static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); static char * GetLocaleProviderString(char datlocprovider); static char * GetTablespaceName(Oid tablespaceOid); static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk); static List * FilterDistributedDatabases(List *databases); static Oid get_database_owner(Oid dbId); /* controlled via GUC */ bool EnableCreateDatabasePropagation = false; bool EnableAlterDatabaseOwner = true; /* * AlterDatabaseOwnerObjectAddress returns the ObjectAddress of the database that is the * object of the AlterOwnerStmt. Errors if missing_ok is false. */ List * AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_DATABASE); Oid databaseOid = get_database_oid(strVal((String *) stmt->object), missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, DatabaseRelationId, databaseOid); return list_make1(address); } /* * DatabaseOwnerDDLCommands returns a list of sql statements to idempotently apply a * change of the database owner on the workers so that the database is owned by the same * user on all nodes in the cluster. */ List * DatabaseOwnerDDLCommands(const ObjectAddress *address) { Node *stmt = (Node *) RecreateAlterDatabaseOwnerStmt(address->objectId); return list_make1(DeparseTreeNode(stmt)); } /* * RecreateAlterDatabaseOwnerStmt creates an AlterOwnerStmt that represents the operation * of changing the owner of the database to its current owner. */ static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid) { AlterOwnerStmt *stmt = makeNode(AlterOwnerStmt); stmt->objectType = OBJECT_DATABASE; stmt->object = (Node *) makeString(get_database_name(databaseOid)); Oid ownerOid = get_database_owner(databaseOid); stmt->newowner = makeNode(RoleSpec); stmt->newowner->roletype = ROLESPEC_CSTRING; stmt->newowner->rolename = GetUserNameFromId(ownerOid, false); return stmt; } /* * get_database_owner returns the Oid of the role owning the database */ static Oid get_database_owner(Oid dbId) { HeapTuple tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(dbId)); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_DATABASE), errmsg("database with OID %u does not exist", dbId))); } Oid dba = ((Form_pg_database) GETSTRUCT(tuple))->datdba; ReleaseSysCache(tuple); return dba; } /* * PreprocessGrantOnDatabaseStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on databases. */ List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagate()) { return NIL; } GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_DATABASE); List *distributedDatabases = FilterDistributedDatabases(stmt->objects); if (list_length(distributedDatabases) == 0) { return NIL; } EnsureCoordinator(); List *originalObjects = stmt->objects; stmt->objects = distributedDatabases; char *sql = DeparseTreeNode((Node *) stmt); stmt->objects = originalObjects; List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * FilterDistributedDatabases filters the database list and returns the distributed ones, * as a list. */ static List * FilterDistributedDatabases(List *databases) { List *distributedDatabases = NIL; String *databaseName = NULL; foreach_declared_ptr(databaseName, databases) { bool missingOk = true; ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(strVal(databaseName), missingOk); if (IsAnyObjectDistributed(list_make1(dbAddress))) { distributedDatabases = lappend(distributedDatabases, databaseName); } } return distributedDatabases; } /* * IsSetTablespaceStatement returns true if the statement is a SET TABLESPACE statement, * false otherwise. */ static bool IsSetTablespaceStatement(AlterDatabaseStmt *stmt) { DefElem *def = NULL; foreach_declared_ptr(def, stmt->options) { if (strcmp(def->defname, "tablespace") == 0) { return true; } } return false; } /* * PreprocessAlterDatabaseStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on databases. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. */ List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { bool missingOk = false; AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missingOk); if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } EnsureCoordinator(); SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, sql, ENABLE_DDL_PROPAGATION); if (IsSetTablespaceStatement(stmt)) { /* * Set tablespace does not work inside a transaction.Therefore, we need to use * NontransactionalNodeDDLTask to run the command on the workers outside * the transaction block. */ bool warnForPartialFailure = true; return NontransactionalNodeDDLTaskList(NON_COORDINATOR_NODES, commands, warnForPartialFailure); } else { return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } } /* * PreprocessAlterDatabaseRefreshCollStmt is executed before the statement is applied to * the local postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on databases. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. */ List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { bool missingOk = true; AlterDatabaseRefreshCollStmt *stmt = castNode(AlterDatabaseRefreshCollStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missingOk); if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } EnsureCoordinator(); SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessAlterDatabaseRenameStmt is executed before the statement is applied to * the local postgres instance. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. * * We acquire this lock here instead of PostprocessAlterDatabaseRenameStmt because the * command renames the database and SerializeDistributedDDLsOnObjectClass resolves the * object on workers based on database name. For this reason, we need to acquire the lock * before the command is applied to the local postgres instance. */ List * PreprocessAlterDatabaseRenameStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { bool missingOk = true; RenameStmt *stmt = castNode(RenameStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->subname, missingOk); if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } EnsureCoordinator(); /* * Different than other ALTER DATABASE commands, we first acquire a lock * by providing InvalidOid because we want ALTER TABLE .. RENAME TO .. * commands to block not only with ALTER DATABASE operations but also * with CREATE DATABASE operations because they might cause name conflicts * and that could also cause deadlocks too. */ SerializeDistributedDDLsOnObjectClass(OCLASS_DATABASE); SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->subname); return NIL; } /* * PostprocessAlterDatabaseRenameStmt is executed after the statement is applied to the local * postgres instance. In this stage we prepare ALTER DATABASE RENAME statement to be run on * all workers. */ List * PostprocessAlterDatabaseRenameStmt(Node *node, const char *queryString) { bool missingOk = false; RenameStmt *stmt = castNode(RenameStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->newname, missingOk); if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } EnsureCoordinator(); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on databases. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. */ List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); bool missingOk = true; ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missingOk); if (!ShouldPropagate() || !IsAnyObjectDistributed(list_make1(dbAddress))) { return NIL; } EnsureCoordinator(); SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessCreateDatabaseStmt is executed before the statement is applied to the local * Postgres instance. * * In this stage, we perform validations that we want to ensure before delegating to * previous utility hooks because it might not be convenient to throw an error in an * implicit transaction that creates a database. Also in this stage, we save the original * database name and replace dbname field with a temporary name for failure handling * purposes. We let Postgres create the database with the temporary name, insert a cleanup * record for the temporary database name on all nodes and let PostprocessCreateDatabaseStmt() * to return the distributed DDL job that both creates the database with the temporary name * and then renames it back to its original name. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. */ List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } EnsureCoordinatorIsInMetadata(); CreatedbStmt *stmt = castNode(CreatedbStmt, node); EnsureSupportedCreateDatabaseCommand(stmt); SerializeDistributedDDLsOnObjectClass(OCLASS_DATABASE); OperationId operationId = RegisterOperationNeedingCleanup(); char *tempDatabaseName = psprintf(TEMP_DATABASE_NAME_FMT, operationId, GetLocalGroupId()); List *remoteNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); WorkerNode *remoteNode = NULL; foreach_declared_ptr(remoteNode, remoteNodes) { InsertCleanupRecordOutsideTransaction( CLEANUP_OBJECT_DATABASE, pstrdup(quote_identifier(tempDatabaseName)), remoteNode->groupId, CLEANUP_ON_FAILURE ); } CreateDatabaseCommandOriginalDbName = stmt->dbname; stmt->dbname = tempDatabaseName; /* * Delete cleanup records in the same transaction so that if the current * transactions fails for some reason, then the cleanup records won't be * deleted. In the happy path, we will delete the cleanup records without * deferring them to the background worker. */ FinalizeOperationNeedingCleanupOnSuccess("create database"); return NIL; } /* * PostprocessCreateDatabaseStmt is executed after the statement is applied to the local * postgres instance. * * In this stage, we first rename the temporary database back to its original name for * local node and then return a list of distributed DDL jobs to create the database with * the temporary name and then to rename it back to its original name. That way, if CREATE * DATABASE fails on any of the nodes, the temporary database will be cleaned up by the * cleanup records that we inserted in PreprocessCreateDatabaseStmt() and in case of a * failure, we won't leak any databases called as the name that user intended to use for * the database. */ List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) { if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); /* * Given that CREATE DATABASE doesn't support "IF NOT EXISTS" and we're * in the post-process, database must exist, hence missingOk = false. */ bool missingOk = false; bool isPostProcess = true; List *addresses = GetObjectAddressListFromParseTree(node, missingOk, isPostProcess); EnsureAllObjectDependenciesExistOnAllNodes(addresses); char *createDatabaseCommand = DeparseTreeNode(node); List *createDatabaseCommands = list_make3(DISABLE_DDL_PROPAGATION, (void *) createDatabaseCommand, ENABLE_DDL_PROPAGATION); /* * Since the CREATE DATABASE statements cannot be executed in a transaction * block, we need to use NontransactionalNodeDDLTaskList() to send the CREATE * DATABASE statement to the workers. */ bool warnForPartialFailure = false; List *createDatabaseDDLJobList = NontransactionalNodeDDLTaskList(REMOTE_NODES, createDatabaseCommands, warnForPartialFailure); CreatedbStmt *stmt = castNode(CreatedbStmt, node); char *renameDatabaseCommand = psprintf("ALTER DATABASE %s RENAME TO %s", quote_identifier(stmt->dbname), quote_identifier(CreateDatabaseCommandOriginalDbName)); List *renameDatabaseCommands = list_make3(DISABLE_DDL_PROPAGATION, renameDatabaseCommand, ENABLE_DDL_PROPAGATION); /* * We use NodeDDLTaskList() to send the RENAME DATABASE statement to the * workers because we want to execute it in a coordinated transaction. */ List *renameDatabaseDDLJobList = NodeDDLTaskList(REMOTE_NODES, renameDatabaseCommands); /* * Temporarily disable citus.enable_ddl_propagation before issuing * rename command locally because we don't want to execute it on remote * nodes yet. We will execute it on remote nodes by returning it as a * distributed DDL job. * * The reason why we don't want to execute it on remote nodes yet is that * the database is not created on remote nodes yet. */ int saveNestLevel = NewGUCNestLevel(); set_config_option("citus.enable_ddl_propagation", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); ExecuteUtilityCommand(renameDatabaseCommand); AtEOXact_GUC(true, saveNestLevel); /* * Restore the original database name because MarkObjectDistributed() * resolves oid of the object based on the database name and is called * after executing the distributed DDL job that renames temporary database. */ stmt->dbname = CreateDatabaseCommandOriginalDbName; return list_concat(createDatabaseDDLJobList, renameDatabaseDDLJobList); } /* * PreprocessDropDatabaseStmt is executed before the statement is applied to the local * postgres instance. In this stage we can prepare the commands that need to be run on * all workers to drop the database. * * We also serialize database commands globally by acquiring a Citus specific advisory * lock based on OCLASS_DATABASE on the first primary worker node. */ List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); DropdbStmt *stmt = (DropdbStmt *) node; bool isPostProcess = false; List *addresses = GetObjectAddressListFromParseTree(node, stmt->missing_ok, isPostProcess); if (list_length(addresses) != 1) { ereport(ERROR, (errmsg("unexpected number of objects found when " "executing DROP DATABASE command"))); } ObjectAddress *address = (ObjectAddress *) linitial(addresses); if (address->objectId == InvalidOid || !IsAnyObjectDistributed(list_make1(address))) { return NIL; } SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname); char *dropDatabaseCommand = DeparseTreeNode(node); List *dropDatabaseCommands = list_make3(DISABLE_DDL_PROPAGATION, (void *) dropDatabaseCommand, ENABLE_DDL_PROPAGATION); /* * Due to same reason stated in PostprocessCreateDatabaseStmt(), we need to * use NontransactionalNodeDDLTaskList() to send the DROP DATABASE statement * to the workers. */ bool warnForPartialFailure = true; List *dropDatabaseDDLJobList = NontransactionalNodeDDLTaskList(REMOTE_NODES, dropDatabaseCommands, warnForPartialFailure); return dropDatabaseDDLJobList; } /* * DropDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the * object of the DropdbStmt. */ List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { DropdbStmt *stmt = castNode(DropdbStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missingOk); return list_make1(dbAddress); } /* * CreateDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the * object of the CreatedbStmt. */ List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missingOk); return list_make1(dbAddress); } /* * EnsureSupportedCreateDatabaseCommand validates the options provided for the CREATE * DATABASE command. * * Parameters: * stmt: A CreatedbStmt struct representing a CREATE DATABASE command. * The options field is a list of DefElem structs, each representing an option. * * Currently, this function checks for the following: * - The "oid" option is not supported. * - The "template" option is only supported with the value "template1". * - The "strategy" option is only supported with the value "wal_log". */ void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt) { DefElem *option = NULL; foreach_declared_ptr(option, stmt->options) { if (strcmp(option->defname, "oid") == 0) { ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", option->defname)); } char *optionValue = defGetString(option); if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, "template1") != 0) { ereport(ERROR, errmsg("Only template1 is supported as template " "parameter for CREATE DATABASE")); } if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != 0) { ereport(ERROR, errmsg("Only wal_log is supported as strategy " "parameter for CREATE DATABASE")); } } } /* * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress * of the database. */ static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) { Oid databaseOid = get_database_oid(databaseName, missingOk); ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, databaseOid); return dbObjectAddress; } /* * GetTablespaceName gets the tablespace oid and returns the tablespace name. */ static char * GetTablespaceName(Oid tablespaceOid) { HeapTuple tuple = SearchSysCache1(TABLESPACEOID, ObjectIdGetDatum(tablespaceOid)); if (!HeapTupleIsValid(tuple)) { return NULL; } Form_pg_tablespace tablespaceForm = (Form_pg_tablespace) GETSTRUCT(tuple); char *tablespaceName = pstrdup(NameStr(tablespaceForm->spcname)); ReleaseSysCache(tuple); return tablespaceName; } /* * GetDatabaseMetadataSyncCommands returns a list of sql statements * for the given database id. The list contains the database ddl command, * grant commands and comment propagation commands. */ List * GetDatabaseMetadataSyncCommands(Oid dbOid) { char *databaseName = get_database_name(dbOid); char *databaseDDLCommand = CreateDatabaseDDLCommand(dbOid); List *ddlCommands = list_make1(databaseDDLCommand); List *grantDDLCommands = GrantOnDatabaseDDLCommands(dbOid); List *commentDDLCommands = GetCommentPropagationCommands(DatabaseRelationId, dbOid, databaseName, OBJECT_DATABASE); ddlCommands = list_concat(ddlCommands, grantDDLCommands); ddlCommands = list_concat(ddlCommands, commentDDLCommands); return ddlCommands; } /* * GetDatabaseCollation gets oid of a database and returns all the collation related information * We need this method since collation related info in Form_pg_database is not accessible. */ static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid) { DatabaseCollationInfo info; memset(&info, 0, sizeof(DatabaseCollationInfo)); Relation rel = table_open(DatabaseRelationId, AccessShareLock); HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, dbOid); if (!HeapTupleIsValid(tup)) { elog(ERROR, "cache lookup failed for database %u", dbOid); } bool isNull = false; TupleDesc tupdesc = RelationGetDescr(rel); Datum collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); info.datcollate = TextDatumGetCString(collationDatum); Datum ctypeDatum = heap_getattr(tup, Anum_pg_database_datctype, tupdesc, &isNull); info.datctype = TextDatumGetCString(ctypeDatum); Datum icuLocaleDatum = heap_getattr(tup, Anum_pg_database_datlocale, tupdesc, &isNull); if (!isNull) { info.daticulocale = TextDatumGetCString(icuLocaleDatum); } Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); if (!isNull) { info.datcollversion = TextDatumGetCString(collverDatum); } Datum icurulesDatum = heap_getattr(tup, Anum_pg_database_daticurules, tupdesc, &isNull); if (!isNull) { info.daticurules = TextDatumGetCString(icurulesDatum); } table_close(rel, AccessShareLock); heap_freetuple(tup); return info; } /* * GetLocaleProviderString gets the datlocprovider stored in pg_database * and returns the string representation of the datlocprovider */ static char * GetLocaleProviderString(char datlocprovider) { switch (datlocprovider) { case 'c': { return "libc"; } case 'i': { return "icu"; } default: { ereport(ERROR, (errmsg("unexpected datlocprovider value: %c", datlocprovider))); } } } /* * GenerateCreateDatabaseStatementFromPgDatabase gets the pg_database tuple and returns the * CREATE DATABASE statement that can be used to create given database. * * Note that this doesn't deparse OID of the database and this is not a * problem as we anyway don't allow specifying custom OIDs for databases * when creating them. */ static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) { DatabaseCollationInfo collInfo = GetDatabaseCollation(databaseForm->oid); StringInfoData str; initStringInfo(&str); appendStringInfo(&str, "CREATE DATABASE %s", quote_identifier(NameStr(databaseForm->datname))); appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); appendStringInfo(&str, " IS_TEMPLATE = %s", quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); appendStringInfo(&str, " LC_COLLATE = %s", quote_literal_cstr(collInfo.datcollate)); appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.datctype)); appendStringInfo(&str, " OWNER = %s", quote_identifier(GetUserNameFromId(databaseForm->datdba, false))); appendStringInfo(&str, " TABLESPACE = %s", quote_identifier(GetTablespaceName(databaseForm->dattablespace))); appendStringInfo(&str, " ENCODING = %s", quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); if (collInfo.datcollversion != NULL) { appendStringInfo(&str, " COLLATION_VERSION = %s", quote_identifier(collInfo.datcollversion)); } if (collInfo.daticulocale != NULL) { appendStringInfo(&str, " ICU_LOCALE = %s", quote_identifier( collInfo.daticulocale)); } appendStringInfo(&str, " LOCALE_PROVIDER = %s", quote_identifier(GetLocaleProviderString( databaseForm->datlocprovider))); if (collInfo.daticurules != NULL) { appendStringInfo(&str, " ICU_RULES = %s", quote_identifier( collInfo.daticurules)); } return str.data; } /* * CreateDatabaseDDLCommand returns a CREATE DATABASE command to create given * database * * Command is wrapped by citus_internal_database_command() UDF * to avoid from transaction block restrictions that apply to database commands. */ char * CreateDatabaseDDLCommand(Oid dbId) { HeapTuple tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(dbId)); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_DATABASE), errmsg("database with OID %u does not exist", dbId))); } Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); StringInfo outerDbStmt = makeStringInfo(); /* Generate the CREATE DATABASE statement */ appendStringInfo(outerDbStmt, "SELECT citus_internal.database_command(%s)", quote_literal_cstr(createStmt)); ReleaseSysCache(tuple); return outerDbStmt->data; } ================================================ FILE: src/backend/distributed/commands/dependencies.c ================================================ /*------------------------------------------------------------------------- * * dependencies.c * Commands to create dependencies of an object on all workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/dependency.h" #include "catalog/objectaddress.h" #include "commands/extension.h" #include "storage/lmgr.h" #include "utils/lsyscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" typedef enum RequiredObjectSet { REQUIRE_ONLY_DEPENDENCIES = 1, REQUIRE_OBJECT_AND_DEPENDENCIES = 2, } RequiredObjectSet; static void EnsureDependenciesCanBeDistributed(const ObjectAddress *relationAddress); static void ErrorIfCircularDependencyExists(const ObjectAddress *objectAddress); static int ObjectAddressComparator(const void *a, const void *b); static void EnsureDependenciesExistOnAllNodes(const ObjectAddress *target); static void EnsureRequiredObjectSetExistOnAllNodes(const ObjectAddress *target, RequiredObjectSet requiredObjectSet); static List * GetDependencyCreateDDLCommands(const ObjectAddress *dependency); static bool ShouldPropagateObject(const ObjectAddress *address); static char * DropTableIfExistsCommand(Oid relationId); /* * EnsureObjectAndDependenciesExistOnAllNodes is a wrapper around * EnsureRequiredObjectSetExistOnAllNodes to ensure the "object itself" (together * with its dependencies) is available on all nodes. * * Different than EnsureDependenciesExistOnAllNodes, we return early if the * target object is distributed already. * * The reason why we don't do the same in EnsureDependenciesExistOnAllNodes * is that it's is used when altering an object too and hence the target object * may instantly have a dependency that needs to be propagated now. For example, * when "⁠GRANT non_dist_role TO dist_role" is executed, we need to propagate * "non_dist_role" to all nodes before propagating the "GRANT" command itself. * For this reason, we call EnsureDependenciesExistOnAllNodes for "dist_role" * and it would automatically discover that "non_dist_role" is a dependency of * "dist_role" and propagate it beforehand. * * However, when we're requested to create the target object itself (and * implicitly its dependencies), we're sure that we're not altering the target * object itself, hence we can return early if the target object is already * distributed. This is the case, for example, when * "REASSIGN OWNED BY dist_role TO non_dist_role" is executed. In that case, * "non_dist_role" is not a dependency of "dist_role" but we want to distribute * "non_dist_role" beforehand and we call this function for "non_dist_role", * not for "dist_role". * * See EnsureRequiredObjectExistOnAllNodes to learn more about how this * function deals with an object created within the same transaction. */ void EnsureObjectAndDependenciesExistOnAllNodes(const ObjectAddress *target) { if (IsAnyObjectDistributed(list_make1((ObjectAddress *) target))) { return; } EnsureRequiredObjectSetExistOnAllNodes(target, REQUIRE_OBJECT_AND_DEPENDENCIES); } /* * EnsureDependenciesExistOnAllNodes is a wrapper around * EnsureRequiredObjectSetExistOnAllNodes to ensure "all dependencies" of given * object --but not the object itself-- are available on all nodes. * * See EnsureRequiredObjectSetExistOnAllNodes to learn more about how this * function deals with an object created within the same transaction. */ static void EnsureDependenciesExistOnAllNodes(const ObjectAddress *target) { EnsureRequiredObjectSetExistOnAllNodes(target, REQUIRE_ONLY_DEPENDENCIES); } /* * EnsureRequiredObjectSetExistOnAllNodes finds all the dependencies that we support and makes * sure these are available on all nodes if required object set is REQUIRE_ONLY_DEPENDENCIES. * Otherwise, i.e., if required object set is REQUIRE_OBJECT_AND_DEPENDENCIES, then this * function creates the object itself on all nodes too. This function ensures that each * of the dependencies are supported by Citus but doesn't check the same for the target * object itself (when REQUIRE_OBJECT_AND_DEPENDENCIES) is provided because we assume that * callers don't call this function for an unsupported function at all. * * If not available, they will be created on the nodes via a separate session that will be * committed directly so that the objects are visible to potentially multiple sessions creating * the shards. * * Note; only the actual objects are created via a separate session, the records to * pg_dist_object are created in this session. As a side effect the objects could be * created on the nodes without a catalog entry. Updates to the objects on local node * are not propagated to the remote nodes until the record is visible on local node. * * This is solved by creating the dependencies in an idempotent manner, either via * postgres native CREATE IF NOT EXISTS, or citus helper functions. */ static void EnsureRequiredObjectSetExistOnAllNodes(const ObjectAddress *target, RequiredObjectSet requiredObjectSet) { Assert(requiredObjectSet == REQUIRE_ONLY_DEPENDENCIES || requiredObjectSet == REQUIRE_OBJECT_AND_DEPENDENCIES); List *objectsWithCommands = NIL; List *ddlCommands = NULL; /* * If there is any unsupported dependency or circular dependency exists, Citus can * not ensure dependencies will exist on all nodes. * * Note that we don't check whether "target" is distributable (in case * REQUIRE_OBJECT_AND_DEPENDENCIES is provided) because we expect callers * to not even call this function if Citus doesn't know how to propagate * "target" object itself. */ EnsureDependenciesCanBeDistributed(target); /* collect all dependencies in creation order and get their ddl commands */ List *objectsToBeCreated = GetDependenciesForObject(target); /* * Append the target object to make sure that it's created after its * dependencies are created, if requested. */ if (requiredObjectSet == REQUIRE_OBJECT_AND_DEPENDENCIES) { ObjectAddress *targetCopy = palloc(sizeof(ObjectAddress)); *targetCopy = *target; objectsToBeCreated = lappend(objectsToBeCreated, targetCopy); } ObjectAddress *object = NULL; foreach_declared_ptr(object, objectsToBeCreated) { List *dependencyCommands = GetDependencyCreateDDLCommands(object); ddlCommands = list_concat(ddlCommands, dependencyCommands); /* create a new list with objects that actually created commands */ if (list_length(dependencyCommands) > 0) { objectsWithCommands = lappend(objectsWithCommands, object); } } if (list_length(ddlCommands) <= 0) { /* no ddl commands to be executed */ return; } /* since we are executing ddl commands lets disable propagation, primarily for mx */ ddlCommands = list_concat(list_make1(DISABLE_DDL_PROPAGATION), ddlCommands); /* * Make sure that no new nodes are added after this point until the end of the * transaction by taking a RowShareLock on pg_dist_node, which conflicts with the * ExclusiveLock taken by citus_add_node. * This guarantees that all active nodes will have the object, because they will * either get it now, or get it in citus_add_node after this transaction finishes and * the pg_dist_object record becomes visible. */ List *remoteNodeList = ActivePrimaryRemoteNodeList(RowShareLock); /* * Lock objects to be created explicitly to make sure same DDL command won't be sent * multiple times from parallel sessions. * * Sort the objects that will be created on workers to not to have any deadlock * issue if different sessions are creating different objects. */ List *addressSortedDependencies = SortList(objectsWithCommands, ObjectAddressComparator); foreach_declared_ptr(object, addressSortedDependencies) { LockDatabaseObject(object->classId, object->objectId, object->objectSubId, ExclusiveLock); } /* * We need to propagate objects via the current user's metadata connection if * any of the objects that we're interested in are created in the current transaction. * Our assumption is that if we rely on an object created in the current transaction, * then the current user, most probably, has permissions to create the target object * as well. * * Note that, user still may not be able to create the target due to no permissions * for any of its dependencies. But this is ok since it should be rare. * * If we opted to use a separate superuser connection for the target, then we would * have visibility issues since propagated dependencies would be invisible to * the separate connection until we locally commit. */ List *createdObjectList = GetAllSupportedDependenciesForObject(target); /* consider target as well if we're requested to create it too */ if (requiredObjectSet == REQUIRE_OBJECT_AND_DEPENDENCIES) { ObjectAddress *targetCopy = palloc(sizeof(ObjectAddress)); *targetCopy = *target; createdObjectList = lappend(createdObjectList, targetCopy); } if (HasAnyObjectInPropagatedObjects(createdObjectList)) { SendCommandListToRemoteNodesWithMetadata(ddlCommands); } else { WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, remoteNodeList) { const char *nodeName = workerNode->workerName; uint32 nodePort = workerNode->workerPort; SendCommandListToWorkerOutsideTransaction(nodeName, nodePort, CitusExtensionOwnerName(), ddlCommands); } } /* * We do this after creating the objects on remote nodes, we make sure * that objects have been created on remote nodes before marking them * distributed, so MarkObjectDistributed wouldn't fail. */ foreach_declared_ptr(object, objectsWithCommands) { /* * pg_dist_object entries must be propagated with the super user, since * the owner of the target object may not own dependencies but we must * propagate as we send objects itself with the superuser. * * Only dependent object's metadata should be propagated with super user. * Metadata of the table itself must be propagated with the current user. */ MarkObjectDistributedViaSuperUser(object); } } /* * EnsureAllObjectDependenciesExistOnAllNodes iteratively calls EnsureDependenciesExistOnAllNodes * for given targets. */ void EnsureAllObjectDependenciesExistOnAllNodes(const List *targets) { ObjectAddress *target = NULL; foreach_declared_ptr(target, targets) { EnsureDependenciesExistOnAllNodes(target); } } /* * EnsureDependenciesCanBeDistributed ensures all dependencies of the given object * can be distributed. */ static void EnsureDependenciesCanBeDistributed(const ObjectAddress *objectAddress) { /* If the object circularcly depends to itself, Citus can not handle it */ ErrorIfCircularDependencyExists(objectAddress); /* If the object has any unsupported dependency, error out */ DeferredErrorMessage *depError = DeferErrorIfAnyObjectHasUnsupportedDependency( list_make1((ObjectAddress *) objectAddress)); if (depError != NULL) { /* override error detail as it is not applicable here*/ depError->detail = NULL; RaiseDeferredError(depError, ERROR); } } /* * ErrorIfCircularDependencyExists is a wrapper around * DeferErrorIfCircularDependencyExists(), and throws error * if circular dependency exists. */ static void ErrorIfCircularDependencyExists(const ObjectAddress *objectAddress) { DeferredErrorMessage *depError = DeferErrorIfCircularDependencyExists(objectAddress); if (depError != NULL) { RaiseDeferredError(depError, ERROR); } } /* * DeferErrorIfCircularDependencyExists checks whether given object has * circular dependency with itself. If so, returns a deferred error. */ DeferredErrorMessage * DeferErrorIfCircularDependencyExists(const ObjectAddress *objectAddress) { List *dependencies = GetAllDependenciesForObject(objectAddress); ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { if (dependency->classId == objectAddress->classId && dependency->objectId == objectAddress->objectId && dependency->objectSubId == objectAddress->objectSubId) { char *objectDescription = getObjectDescription(objectAddress, false); StringInfo detailInfo = makeStringInfo(); appendStringInfo(detailInfo, "\"%s\" circularly depends itself, resolve " "circular dependency first", objectDescription); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Citus can not handle circular dependencies " "between distributed objects", detailInfo->data, NULL); } } return NULL; } /* * Copied from PG object_address_comparator function to compare ObjectAddresses. */ static int ObjectAddressComparator(const void *a, const void *b) { const ObjectAddress *obja = (const ObjectAddress *) a; const ObjectAddress *objb = (const ObjectAddress *) b; /* * Primary sort key is OID descending. */ if (obja->objectId > objb->objectId) { return -1; } if (obja->objectId < objb->objectId) { return 1; } /* * Next sort on catalog ID, in case identical OIDs appear in different * catalogs. Sort direction is pretty arbitrary here. */ if (obja->classId < objb->classId) { return -1; } if (obja->classId > objb->classId) { return 1; } /* * Last, sort on object subId. */ if ((unsigned int) obja->objectSubId < (unsigned int) objb->objectSubId) { return -1; } if ((unsigned int) obja->objectSubId > (unsigned int) objb->objectSubId) { return 1; } return 0; } /* * GetDistributableDependenciesForObject finds all the dependencies that Citus * can distribute and returns those dependencies regardless of their existency * on nodes. */ List * GetDistributableDependenciesForObject(const ObjectAddress *target) { /* local variables to work with dependencies */ List *distributableDependencies = NIL; /* collect all dependencies in creation order */ List *dependencies = GetDependenciesForObject(target); /* filter the ones that can be distributed */ ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { /* * TODO: maybe we can optimize the logic applied in below line. Actually we * do not need to create ddl commands as we are not ensuring their existence * in nodes, but we utilize logic it follows to choose the objects that could * be distributed */ List *dependencyCommands = GetDependencyCreateDDLCommands(dependency); /* create a new list with dependencies that actually created commands */ if (list_length(dependencyCommands) > 0) { distributableDependencies = lappend(distributableDependencies, dependency); } } return distributableDependencies; } /* * DropTableIfExistsCommand returns command to drop given table if exists. */ static char * DropTableIfExistsCommand(Oid relationId) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); StringInfo dropTableCommand = makeStringInfo(); appendStringInfo(dropTableCommand, "DROP TABLE IF EXISTS %s CASCADE", qualifiedRelationName); return dropTableCommand->data; } /* * GetDependencyCreateDDLCommands returns a list (potentially empty or NIL) of ddl * commands to execute on a worker to create the object. */ static List * GetDependencyCreateDDLCommands(const ObjectAddress *dependency) { switch (getObjectClass(dependency)) { case OCLASS_CLASS: { char relKind = get_rel_relkind(dependency->objectId); /* * types have an intermediate dependency on a relation (aka class), so we do * support classes when the relkind is composite */ if (relKind == RELKIND_COMPOSITE_TYPE) { return NIL; } /* * Indices are created separately, however, they do show up in the dependency * list for a table since they will have potentially their own dependencies. * The commands will be added to both shards and metadata tables via the table * creation commands. */ if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { return NIL; } if (relKind == RELKIND_RELATION || relKind == RELKIND_PARTITIONED_TABLE || relKind == RELKIND_FOREIGN_TABLE) { Oid relationId = dependency->objectId; List *commandList = NIL; if (IsCitusTable(relationId)) { bool creatingShellTableOnRemoteNode = true; List *tableDDLCommands = GetFullTableCreationCommands(relationId, WORKER_NEXTVAL_SEQUENCE_DEFAULTS, INCLUDE_IDENTITY, creatingShellTableOnRemoteNode); TableDDLCommand *tableDDLCommand = NULL; foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); commandList = lappend(commandList, GetTableDDLCommand( tableDDLCommand)); } /* * We need to drop table, if exists, first to make table creation * idempotent. Before dropping the table, we should also break * dependencies with sequences since `drop cascade table` would also * drop depended sequences. This is safe as we still record dependency * with the sequence during table creation. */ commandList = lcons(DropTableIfExistsCommand(relationId), commandList); commandList = lcons(WorkerDropSequenceDependencyCommand(relationId), commandList); } return commandList; } if (relKind == RELKIND_SEQUENCE) { char *sequenceOwnerName = TableOwner(dependency->objectId); return DDLCommandsForSequence(dependency->objectId, sequenceOwnerName); } if (relKind == RELKIND_VIEW) { char *createViewCommand = CreateViewDDLCommand(dependency->objectId); char *alterViewOwnerCommand = AlterViewOwnerCommand(dependency->objectId); return list_make2(createViewCommand, alterViewOwnerCommand); } /* if this relation is not supported, break to the error at the end */ break; } case OCLASS_COLLATION: { return CreateCollationDDLsIdempotent(dependency->objectId); } case OCLASS_CONSTRAINT: { /* * Constraints can only be reached by domains, they resolve functions. * Constraints themself are recreated by the domain recreation. */ return NIL; } case OCLASS_DATABASE: { /* * For the database where Citus is installed, only propagate the ownership of the * database, only when the feature is on. * * This is because this database must exist on all nodes already so we shouldn't * need to "CREATE" it on other nodes. However, we still need to correctly reflect * its owner on other nodes too. */ if (dependency->objectId == MyDatabaseId && EnableAlterDatabaseOwner) { return DatabaseOwnerDDLCommands(dependency); } /* * For the other databases, create the database on all nodes, only when the feature * is on. */ if (dependency->objectId != MyDatabaseId && EnableCreateDatabasePropagation) { return GetDatabaseMetadataSyncCommands(dependency->objectId); } return NIL; } case OCLASS_PROC: { List *DDLCommands = CreateFunctionDDLCommandsIdempotent(dependency); List *grantDDLCommands = GrantOnFunctionDDLCommands(dependency->objectId); DDLCommands = list_concat(DDLCommands, grantDDLCommands); return DDLCommands; } case OCLASS_PUBLICATION: { return CreatePublicationDDLCommandsIdempotent(dependency); } case OCLASS_ROLE: { return GenerateCreateOrAlterRoleCommand(dependency->objectId); } case OCLASS_SCHEMA: { char *schemaDDLCommand = CreateSchemaDDLCommand(dependency->objectId); List *DDLCommands = list_make1(schemaDDLCommand); List *grantDDLCommands = GrantOnSchemaDDLCommands(dependency->objectId); DDLCommands = list_concat(DDLCommands, grantDDLCommands); return DDLCommands; } case OCLASS_TSCONFIG: { return CreateTextSearchConfigDDLCommandsIdempotent(dependency); } case OCLASS_TSDICT: { return CreateTextSearchDictDDLCommandsIdempotent(dependency); } case OCLASS_TYPE: { return CreateTypeDDLCommandsIdempotent(dependency); } case OCLASS_EXTENSION: { return CreateExtensionDDLCommand(dependency); } case OCLASS_FOREIGN_SERVER: { Oid serverId = dependency->objectId; List *DDLCommands = GetForeignServerCreateDDLCommand(serverId); List *grantDDLCommands = GrantOnForeignServerDDLCommands(serverId); DDLCommands = list_concat(DDLCommands, grantDDLCommands); return DDLCommands; } default: { break; } } /* * make sure it fails hard when in debug mode, leave a hint for the user if this ever * happens in production */ Assert(false); ereport(ERROR, (errmsg("unsupported object %s for distribution by citus", getObjectTypeDescription(dependency, /* missingOk: */ false)), errdetail( "citus tries to recreate an unsupported object on its workers"), errhint("please report a bug as this should not be happening"))); } /* * GetAllDependencyCreateDDLCommands iteratively calls GetDependencyCreateDDLCommands * for given dependencies. */ List * GetAllDependencyCreateDDLCommands(const List *dependencies) { List *commands = NIL; ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { commands = list_concat(commands, GetDependencyCreateDDLCommands(dependency)); } return commands; } /* * ShouldPropagate determines if we should be propagating anything */ bool ShouldPropagate(void) { if (creating_extension) { /* * extensions should be created separately on the workers, types cascading from an * extension should therefore not be propagated. */ return false; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return false; } return true; } /* * ShouldPropagateCreateInCoordinatedTransction returns based the current state of the * session and policies if Citus needs to propagate the creation of new objects. * * Creation of objects on other nodes could be postponed till the object is actually used * in a sharded object (eg. distributed table or index on a distributed table). In certain * use cases the opportunity for parallelism in a transaction block is preferred. When * configured like that the creation of an object might be postponed and backfilled till * the object is actually used. */ bool ShouldPropagateCreateInCoordinatedTransction() { if (!IsMultiStatementTransaction()) { /* * If we are in a single statement transaction we will always propagate the * creation of objects. There are no downsides in regard to performance or * transactional limitations. These only arise with transaction blocks consisting * of multiple statements. */ return true; } if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { /* * If we are in a transaction that is already switched to sequential, either by * the user, or automatically by an other command, we will always propagate the * creation of new objects to the workers. * * This guarantees no strange anomalies when the transaction aborts or on * visibility of the newly created object. */ return true; } switch (CreateObjectPropagationMode) { case CREATE_OBJECT_PROPAGATION_DEFERRED: { /* * We prefer parallelism at this point. Since we did not already return while * checking for sequential mode we are still in parallel mode. We don't want * to switch that now, thus not propagating the creation. */ return false; } case CREATE_OBJECT_PROPAGATION_AUTOMATIC: { /* * When we run in optimistic mode we want to switch to sequential mode, only * if this would _not_ give an error to the user. Meaning, we either are * already in sequential mode (checked earlier), or there has been no parallel * execution in the current transaction block. * * If switching to sequential would throw an error we would stay in parallel * mode while creating new objects. We will rely on Citus' mechanism to ensure * the existence if the object would be used in the same transaction. */ if (ParallelQueryExecutedInTransaction()) { return false; } return true; } case CREATE_OBJECT_PROPAGATION_IMMEDIATE: { return true; } default: { elog(ERROR, "unsupported ddl propagation mode"); } } } /* * ShouldPropagateObject determines if we should be propagating DDLs based * on their object address. */ static bool ShouldPropagateObject(const ObjectAddress *address) { if (!ShouldPropagate()) { return false; } if (!IsAnyObjectDistributed(list_make1((ObjectAddress *) address))) { /* do not propagate for non-distributed types */ return false; } return true; } /* * ShouldPropagateAnyObject determines if we should be propagating DDLs based * on their object addresses. */ bool ShouldPropagateAnyObject(List *addresses) { ObjectAddress *address = NULL; foreach_declared_ptr(address, addresses) { if (ShouldPropagateObject(address)) { return true; } } return false; } /* * FilterObjectAddressListByPredicate takes a list of ObjectAddress *'s and returns a list * only containing the ObjectAddress *'s for which the predicate returned true. */ List * FilterObjectAddressListByPredicate(List *objectAddressList, AddressPredicate predicate) { List *result = NIL; ObjectAddress *address = NULL; foreach_declared_ptr(address, objectAddressList) { if (predicate(address)) { result = lappend(result, address); } } return result; } ================================================ FILE: src/backend/distributed/commands/distribute_object_ops.c ================================================ /*------------------------------------------------------------------------- * * distribute_object_ops.c * * Contains declarations for DistributeObjectOps, along with their * lookup function, GetDistributeObjectOps. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/comment.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" static DistributeObjectOps NoDistributeOps = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = NULL, .operationType = DIST_OPS_NONE, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Aggregate_AlterObjectSchema = { .deparse = DeparseAlterFunctionSchemaStmt, .qualify = QualifyAlterFunctionSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Aggregate_AlterOwner = { .deparse = DeparseAlterFunctionOwnerStmt, .qualify = QualifyAlterFunctionOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Aggregate_Define = { .deparse = NULL, .qualify = QualifyDefineAggregateStmt, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_AGGREGATE, .operationType = DIST_OPS_CREATE, .address = DefineAggregateStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Aggregate_Drop = { .deparse = DeparseDropFunctionStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Aggregate_Rename = { .deparse = DeparseRenameFunctionStmt, .qualify = QualifyRenameFunctionStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = RenameFunctionStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Aggregate_Grant = { .deparse = DeparseGrantOnFunctionStmt, .qualify = NULL, .preprocess = PreprocessGrantOnFunctionStmt, .postprocess = PostprocessGrantOnFunctionStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_AlterEnum = { .deparse = DeparseAlterEnumStmt, .qualify = QualifyAlterEnumStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_ALTER, .address = AlterEnumStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterExtension = { .deparse = DeparseAlterExtensionStmt, .qualify = NULL, .preprocess = PreprocessAlterExtensionUpdateStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterExtensionUpdateStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterExtensionContents = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterExtensionContentsStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_AlterForeignServer = { .deparse = DeparseAlterForeignServerStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FOREIGN_SERVER, .operationType = DIST_OPS_ALTER, .address = AlterForeignServerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterFunction = { .deparse = DeparseAlterFunctionStmt, .qualify = QualifyAlterFunctionStmt, .preprocess = PreprocessAlterFunctionStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterFunctionStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterPolicy = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterPolicyStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_AlterRole = { .deparse = DeparseAlterRoleStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessAlterRoleStmt, .operationType = DIST_OPS_ALTER, .address = AlterRoleStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterRoleRename = { .deparse = DeparseRenameRoleStmt, .qualify = NULL, .preprocess = PreprocessAlterRoleRenameStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = RenameRoleStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterRoleSet = { .deparse = DeparseAlterRoleSetStmt, .qualify = QualifyAlterRoleSetStmt, .preprocess = PreprocessAlterRoleSetStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterRoleSetStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_AlterTableMoveAll = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterTableMoveAllStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_Cluster = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessClusterStmt, .postprocess = NULL, .operationType = DIST_OPS_NONE, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_CompositeType = { .deparse = DeparseCompositeTypeStmt, .qualify = QualifyCompositeTypeStmt, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_CREATE, .featureFlag = &EnableCreateTypePropagation, .address = CompositeTypeStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateDomain = { .deparse = DeparseCreateDomainStmt, .qualify = QualifyCreateDomainStmt, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_CREATE, .address = CreateDomainStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateEnum = { .deparse = DeparseCreateEnumStmt, .qualify = QualifyCreateEnumStmt, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_CREATE, .featureFlag = &EnableCreateTypePropagation, .address = CreateEnumStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateExtension = { .deparse = DeparseCreateExtensionStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateExtensionStmt, .operationType = DIST_OPS_CREATE, .address = CreateExtensionStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateFunction = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessCreateFunctionStmt, .postprocess = PostprocessCreateFunctionStmt, .operationType = DIST_OPS_CREATE, .address = CreateFunctionStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_View = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessViewStmt, .postprocess = PostprocessViewStmt, .operationType = DIST_OPS_CREATE, .address = ViewStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreatePolicy = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreatePolicyStmt, .operationType = DIST_OPS_CREATE, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_CreatePublication = { .deparse = DeparseCreatePublicationStmt, .qualify = QualifyCreatePublicationStmt, .preprocess = NULL, .postprocess = PostProcessCreatePublicationStmt, .operationType = DIST_OPS_CREATE, .address = CreatePublicationStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateRole = { .deparse = DeparseCreateRoleStmt, .qualify = NULL, .preprocess = PreprocessCreateRoleStmt, .postprocess = NULL, .operationType = DIST_OPS_CREATE, .address = CreateRoleStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_ReassignOwned = { .deparse = DeparseReassignOwnedStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessReassignOwnedStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_DropOwned = { .deparse = DeparseDropOwnedStmt, .qualify = NULL, .preprocess = PreprocessDropOwnedStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_DropRole = { .deparse = DeparseDropRoleStmt, .qualify = NULL, .preprocess = PreprocessDropRoleStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Role_Comment = { .deparse = DeparseCommentStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = CommentObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_CreateForeignServer = { .deparse = DeparseCreateForeignServerStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_FOREIGN_SERVER, .operationType = DIST_OPS_CREATE, .address = CreateForeignServerStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateSchema = { .deparse = DeparseCreateSchemaStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateSchemaStmt, .operationType = DIST_OPS_CREATE, .address = CreateSchemaStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Any_CreateStatistics = { .deparse = DeparseCreateStatisticsStmt, .qualify = QualifyCreateStatisticsStmt, .preprocess = PreprocessCreateStatisticsStmt, .postprocess = PostprocessCreateStatisticsStmt, .operationType = DIST_OPS_CREATE, .address = CreateStatisticsStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_CreateTrigger = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateTriggerStmt, .operationType = DIST_OPS_CREATE, .address = CreateTriggerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_Grant = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessGrantStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_GrantRole = { .deparse = DeparseGrantRoleStmt, .qualify = NULL, .preprocess = PreprocessGrantRoleStmt, .postprocess = PostprocessGrantRoleStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_Index = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessIndexStmt, .postprocess = PostprocessIndexStmt, .operationType = DIST_OPS_CREATE, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_Reindex = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessReindexStmt, .postprocess = NULL, .operationType = DIST_OPS_NONE, .address = ReindexStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Any_Rename = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessRenameStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Any_SecLabel = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessAnySecLabelStmt, .operationType = DIST_OPS_ALTER, .address = SecLabelStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Role_SecLabel = { .deparse = DeparseRoleSecLabelStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessRoleSecLabelStmt, .operationType = DIST_OPS_ALTER, .address = SecLabelStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Table_SecLabel = { .deparse = DeparseTableSecLabelStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessTableOrColumnSecLabelStmt, .operationType = DIST_OPS_ALTER, .address = SecLabelStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Column_SecLabel = { .deparse = DeparseColumnSecLabelStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessTableOrColumnSecLabelStmt, .operationType = DIST_OPS_ALTER, .address = SecLabelStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Attribute_Rename = { .deparse = DeparseRenameAttributeStmt, .qualify = QualifyRenameAttributeStmt, .preprocess = PreprocessRenameAttributeStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = RenameAttributeStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Collation_AlterObjectSchema = { .deparse = DeparseAlterCollationSchemaStmt, .qualify = QualifyAlterCollationSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_COLLATION, .operationType = DIST_OPS_ALTER, .address = AlterCollationSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Collation_AlterOwner = { .deparse = DeparseAlterCollationOwnerStmt, .qualify = QualifyAlterCollationOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_COLLATION, .operationType = DIST_OPS_ALTER, .address = AlterCollationOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Collation_Define = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_COLLATION, .operationType = DIST_OPS_CREATE, .address = DefineCollationStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Collation_Drop = { .deparse = DeparseDropCollationStmt, .qualify = QualifyDropCollationStmt, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Collation_Rename = { .deparse = DeparseRenameCollationStmt, .qualify = QualifyRenameCollationStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_COLLATION, .operationType = DIST_OPS_ALTER, .address = RenameCollationStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Database_AlterOwner = { .deparse = DeparseAlterDatabaseOwnerStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .featureFlag = &EnableAlterDatabaseOwner, .address = AlterDatabaseOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Database_Grant = { .deparse = DeparseGrantOnDatabaseStmt, .qualify = NULL, .preprocess = PreprocessGrantOnDatabaseStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Database_Alter = { .deparse = DeparseAlterDatabaseStmt, .qualify = NULL, .preprocess = PreprocessAlterDatabaseStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Database_Create = { .deparse = DeparseCreateDatabaseStmt, .qualify = NULL, .preprocess = PreprocessCreateDatabaseStmt, .postprocess = PostprocessCreateDatabaseStmt, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_CREATE, .address = CreateDatabaseStmtObjectAddress, .markDistributed = true, }; static DistributeObjectOps Database_Drop = { .deparse = DeparseDropDatabaseStmt, .qualify = NULL, .preprocess = PreprocessDropDatabaseStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_DROP, .address = DropDatabaseStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Database_RefreshColl = { .deparse = DeparseAlterDatabaseRefreshCollStmt, .qualify = NULL, .preprocess = PreprocessAlterDatabaseRefreshCollStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Database_Set = { .deparse = DeparseAlterDatabaseSetStmt, .qualify = NULL, .preprocess = PreprocessAlterDatabaseSetStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Database_Comment = { .deparse = DeparseCommentStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = CommentObjectAddress, .markDistributed = false, }; static DistributeObjectOps Database_Rename = { .deparse = DeparseAlterDatabaseRenameStmt, .qualify = NULL, .preprocess = PreprocessAlterDatabaseRenameStmt, .postprocess = PostprocessAlterDatabaseRenameStmt, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Domain_Alter = { .deparse = DeparseAlterDomainStmt, .qualify = QualifyAlterDomainStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_ALTER, .address = AlterDomainStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Domain_AlterObjectSchema = { .deparse = DeparseAlterDomainSchemaStmt, .qualify = QualifyAlterDomainSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_ALTER, .address = AlterTypeSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Domain_AlterOwner = { .deparse = DeparseAlterDomainOwnerStmt, .qualify = QualifyAlterDomainOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_ALTER, .address = AlterDomainOwnerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Domain_Drop = { .deparse = DeparseDropDomainStmt, .qualify = QualifyDropDomainStmt, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Domain_Rename = { .deparse = DeparseRenameDomainStmt, .qualify = QualifyRenameDomainStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_ALTER, .address = RenameDomainStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Domain_RenameConstraint = { .deparse = DeparseDomainRenameConstraintStmt, .qualify = QualifyDomainRenameConstraintStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_DOMAIN, .operationType = DIST_OPS_ALTER, .address = DomainRenameConstraintStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Extension_AlterObjectSchema = { .deparse = DeparseAlterExtensionSchemaStmt, .qualify = NULL, .preprocess = PreprocessAlterExtensionSchemaStmt, .postprocess = PostprocessAlterExtensionSchemaStmt, .operationType = DIST_OPS_ALTER, .address = AlterExtensionSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Extension_Drop = { .deparse = DeparseDropExtensionStmt, .qualify = NULL, .preprocess = PreprocessDropExtensionStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps FDW_Grant = { .deparse = DeparseGrantOnFDWStmt, .qualify = NULL, .preprocess = PreprocessGrantOnFDWStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps ForeignServer_Drop = { .deparse = DeparseDropForeignServerStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps ForeignServer_Grant = { .deparse = DeparseGrantOnForeignServerStmt, .qualify = NULL, .preprocess = PreprocessGrantOnForeignServerStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps ForeignServer_Rename = { .deparse = DeparseAlterForeignServerRenameStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FOREIGN_SERVER, .operationType = DIST_OPS_ALTER, .address = RenameForeignServerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps ForeignServer_AlterOwner = { .deparse = DeparseAlterForeignServerOwnerStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FOREIGN_SERVER, .operationType = DIST_OPS_ALTER, .address = AlterForeignServerOwnerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps ForeignTable_AlterTable = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterTableStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Function_AlterObjectDepends = { .deparse = DeparseAlterFunctionDependsStmt, .qualify = QualifyAlterFunctionDependsStmt, .preprocess = PreprocessAlterFunctionDependsStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterFunctionDependsStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Function_AlterObjectSchema = { .deparse = DeparseAlterFunctionSchemaStmt, .qualify = QualifyAlterFunctionSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Function_AlterOwner = { .deparse = DeparseAlterFunctionOwnerStmt, .qualify = QualifyAlterFunctionOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Function_Drop = { .deparse = DeparseDropFunctionStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Function_Grant = { .deparse = DeparseGrantOnFunctionStmt, .qualify = NULL, .preprocess = PreprocessGrantOnFunctionStmt, .postprocess = PostprocessGrantOnFunctionStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps View_Drop = { .deparse = DeparseDropViewStmt, .qualify = QualifyDropViewStmt, .preprocess = PreprocessDropViewStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = DropViewStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Function_Rename = { .deparse = DeparseRenameFunctionStmt, .qualify = QualifyRenameFunctionStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = RenameFunctionStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Index_AlterTable = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterTableStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Index_Drop = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessDropIndexStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Policy_Drop = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessDropPolicyStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Procedure_AlterObjectDepends = { .deparse = DeparseAlterFunctionDependsStmt, .qualify = QualifyAlterFunctionDependsStmt, .preprocess = PreprocessAlterFunctionDependsStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterFunctionDependsStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Procedure_AlterObjectSchema = { .deparse = DeparseAlterFunctionSchemaStmt, .qualify = QualifyAlterFunctionSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Procedure_AlterOwner = { .deparse = DeparseAlterFunctionOwnerStmt, .qualify = QualifyAlterFunctionOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Procedure_Drop = { .deparse = DeparseDropFunctionStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Procedure_Grant = { .deparse = DeparseGrantOnFunctionStmt, .qualify = NULL, .preprocess = PreprocessGrantOnFunctionStmt, .postprocess = PostprocessGrantOnFunctionStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Procedure_Rename = { .deparse = DeparseRenameFunctionStmt, .qualify = QualifyRenameFunctionStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = RenameFunctionStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Publication_Alter = { .deparse = DeparseAlterPublicationStmt, .qualify = QualifyAlterPublicationStmt, .preprocess = PreprocessAlterPublicationStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_PUBLICATION, .operationType = DIST_OPS_ALTER, .address = AlterPublicationStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Publication_AlterOwner = { .deparse = DeparseAlterPublicationOwnerStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_PUBLICATION, .operationType = DIST_OPS_ALTER, .address = AlterPublicationOwnerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Publication_Drop = { .deparse = DeparseDropPublicationStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Publication_Rename = { .deparse = DeparseRenamePublicationStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_PUBLICATION, .operationType = DIST_OPS_ALTER, .address = RenamePublicationStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Routine_AlterObjectDepends = { .deparse = DeparseAlterFunctionDependsStmt, .qualify = QualifyAlterFunctionDependsStmt, .preprocess = PreprocessAlterFunctionDependsStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterFunctionDependsStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_Alter = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterSequenceStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterSequenceStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_AlterObjectSchema = { .deparse = DeparseAlterSequenceSchemaStmt, .qualify = QualifyAlterSequenceSchemaStmt, .preprocess = PreprocessAlterSequenceSchemaStmt, .postprocess = PostprocessAlterSequenceSchemaStmt, .operationType = DIST_OPS_ALTER, .address = AlterSequenceSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_AlterOwner = { .deparse = DeparseAlterSequenceOwnerStmt, .qualify = QualifyAlterSequenceOwnerStmt, .preprocess = PreprocessAlterSequenceOwnerStmt, .postprocess = PostprocessAlterSequenceOwnerStmt, .operationType = DIST_OPS_ALTER, .address = AlterSequenceOwnerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_AlterPersistence = { .deparse = DeparseAlterSequencePersistenceStmt, .qualify = QualifyAlterSequencePersistenceStmt, .preprocess = PreprocessAlterSequencePersistenceStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = AlterSequencePersistenceStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_Drop = { .deparse = DeparseDropSequenceStmt, .qualify = QualifyDropSequenceStmt, .preprocess = PreprocessDropSequenceStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = SequenceDropStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Sequence_Grant = { .deparse = DeparseGrantOnSequenceStmt, .qualify = QualifyGrantOnSequenceStmt, .preprocess = PreprocessGrantOnSequenceStmt, .postprocess = PostprocessGrantOnSequenceStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Sequence_Rename = { .deparse = DeparseRenameSequenceStmt, .qualify = QualifyRenameSequenceStmt, .preprocess = PreprocessRenameSequenceStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = RenameSequenceStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_Alter = { .deparse = DeparseAlterTextSearchConfigurationStmt, .qualify = QualifyAlterTextSearchConfigurationStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchConfigurationStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_AlterObjectSchema = { .deparse = DeparseAlterTextSearchConfigurationSchemaStmt, .qualify = QualifyAlterTextSearchConfigurationSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchConfigurationSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_AlterOwner = { .deparse = DeparseAlterTextSearchConfigurationOwnerStmt, .qualify = QualifyAlterTextSearchConfigurationOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchConfigurationOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_Comment = { .deparse = DeparseCommentStmt, /* TODO: When adding new comment types we should create an abstracted * qualify function, just like we have an abstract deparse * and adress function */ .qualify = QualifyTextSearchConfigurationCommentStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_ALTER, .address = CommentObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_Define = { .deparse = DeparseCreateTextSearchConfigurationStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_CREATE, .address = CreateTextSearchConfigurationObjectAddress, .markDistributed = true, }; static DistributeObjectOps TextSearchConfig_Drop = { .deparse = DeparseDropTextSearchConfigurationStmt, .qualify = QualifyDropTextSearchConfigurationStmt, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = DropTextSearchConfigObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchConfig_Rename = { .deparse = DeparseRenameTextSearchConfigurationStmt, .qualify = QualifyRenameTextSearchConfigurationStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSCONFIGURATION, .operationType = DIST_OPS_ALTER, .address = RenameTextSearchConfigurationStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_Alter = { .deparse = DeparseAlterTextSearchDictionaryStmt, .qualify = QualifyAlterTextSearchDictionaryStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchDictionaryStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_AlterObjectSchema = { .deparse = DeparseAlterTextSearchDictionarySchemaStmt, .qualify = QualifyAlterTextSearchDictionarySchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchDictionarySchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_AlterOwner = { .deparse = DeparseAlterTextSearchDictionaryOwnerStmt, .qualify = QualifyAlterTextSearchDictionaryOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_ALTER, .address = AlterTextSearchDictOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_Comment = { .deparse = DeparseCommentStmt, .qualify = QualifyTextSearchDictionaryCommentStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_ALTER, .address = CommentObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_Define = { .deparse = DeparseCreateTextSearchDictionaryStmt, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessCreateDistributedObjectFromCatalogStmt, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_CREATE, .address = CreateTextSearchDictObjectAddress, .markDistributed = true, }; static DistributeObjectOps TextSearchDict_Drop = { .deparse = DeparseDropTextSearchDictionaryStmt, .qualify = QualifyDropTextSearchDictionaryStmt, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = DropTextSearchDictObjectAddress, .markDistributed = false, }; static DistributeObjectOps TextSearchDict_Rename = { .deparse = DeparseRenameTextSearchDictionaryStmt, .qualify = QualifyRenameTextSearchDictionaryStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TSDICTIONARY, .operationType = DIST_OPS_ALTER, .address = RenameTextSearchDictionaryStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Trigger_AlterObjectDepends = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessAlterTriggerDependsStmt, .postprocess = PostprocessAlterTriggerDependsStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Routine_AlterObjectSchema = { .deparse = DeparseAlterFunctionSchemaStmt, .qualify = QualifyAlterFunctionSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Routine_AlterOwner = { .deparse = DeparseAlterFunctionOwnerStmt, .qualify = QualifyAlterFunctionOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = AlterFunctionOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Routine_Drop = { .deparse = DeparseDropFunctionStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Routine_Grant = { .deparse = DeparseGrantOnFunctionStmt, .qualify = NULL, .preprocess = PreprocessGrantOnFunctionStmt, .postprocess = PostprocessGrantOnFunctionStmt, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Routine_Rename = { .deparse = DeparseRenameFunctionStmt, .qualify = QualifyRenameFunctionStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_FUNCTION, .operationType = DIST_OPS_ALTER, .address = RenameFunctionStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Schema_AlterOwner = { .deparse = DeparseAlterSchemaOwnerStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .operationType = DIST_OPS_ALTER, .postprocess = NULL, .address = AlterSchemaOwnerStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Schema_Drop = { .deparse = DeparseDropSchemaStmt, .qualify = NULL, .preprocess = PreprocessDropSchemaStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Schema_Grant = { .deparse = DeparseGrantOnSchemaStmt, .qualify = NULL, .preprocess = PreprocessGrantOnSchemaStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Schema_Rename = { .deparse = DeparseAlterSchemaRenameStmt, .qualify = NULL, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_SCHEMA, .operationType = DIST_OPS_ALTER, .address = AlterSchemaRenameStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Statistics_Alter = { .deparse = DeparseAlterStatisticsStmt, .qualify = QualifyAlterStatisticsStmt, .preprocess = PreprocessAlterStatisticsStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Statistics_AlterObjectSchema = { .deparse = DeparseAlterStatisticsSchemaStmt, .qualify = QualifyAlterStatisticsSchemaStmt, .preprocess = PreprocessAlterStatisticsSchemaStmt, .postprocess = PostprocessAlterStatisticsSchemaStmt, .operationType = DIST_OPS_ALTER, .address = AlterStatisticsSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Statistics_AlterOwner = { .deparse = DeparseAlterStatisticsOwnerStmt, .qualify = QualifyAlterStatisticsOwnerStmt, .preprocess = PreprocessAlterStatisticsOwnerStmt, .operationType = DIST_OPS_ALTER, .postprocess = PostprocessAlterStatisticsOwnerStmt, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Statistics_Drop = { .deparse = NULL, .qualify = QualifyDropStatisticsStmt, .preprocess = PreprocessDropStatisticsStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = DropStatisticsObjectAddress, .markDistributed = false, }; static DistributeObjectOps Statistics_Rename = { .deparse = DeparseAlterStatisticsRenameStmt, .qualify = QualifyAlterStatisticsRenameStmt, .preprocess = PreprocessAlterStatisticsRenameStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Table_AlterTable = { .deparse = DeparseAlterTableStmt, .qualify = NULL, .preprocess = PreprocessAlterTableStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Table_AlterObjectSchema = { .deparse = DeparseAlterTableSchemaStmt, .qualify = QualifyAlterTableSchemaStmt, .preprocess = PreprocessAlterTableSchemaStmt, .postprocess = PostprocessAlterTableSchemaStmt, .operationType = DIST_OPS_ALTER, .address = AlterTableSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Table_Drop = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessDropTableStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Type_AlterObjectSchema = { .deparse = DeparseAlterTypeSchemaStmt, .qualify = QualifyAlterTypeSchemaStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_ALTER, .address = AlterTypeSchemaStmtObjectAddress, .markDistributed = false, }; /* * PreprocessAlterViewSchemaStmt and PostprocessAlterViewSchemaStmt functions can be called * internally by ALTER TABLE view_name SET SCHEMA ... if the ALTER TABLE command targets a * view. In other words ALTER VIEW view_name SET SCHEMA will use the View_AlterObjectSchema * but ALTER TABLE view_name SET SCHEMA will use Table_AlterObjectSchema but call process * functions of View_AlterObjectSchema internally. */ static DistributeObjectOps View_AlterObjectSchema = { .deparse = DeparseAlterViewSchemaStmt, .qualify = QualifyAlterViewSchemaStmt, .preprocess = PreprocessAlterViewSchemaStmt, .postprocess = PostprocessAlterViewSchemaStmt, .operationType = DIST_OPS_ALTER, .address = AlterViewSchemaStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Type_AlterOwner = { .deparse = DeparseAlterTypeOwnerStmt, .qualify = QualifyAlterTypeOwnerStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = PostprocessAlterDistributedObjectStmt, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_ALTER, .address = AlterTypeOwnerObjectAddress, .markDistributed = false, }; static DistributeObjectOps Type_AlterTable = { .deparse = DeparseAlterTypeStmt, .qualify = QualifyAlterTypeStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_ALTER, .address = AlterTypeStmtObjectAddress, .markDistributed = false, }; /* * PreprocessAlterViewStmt and PostprocessAlterViewStmt functions can be called internally * by ALTER TABLE view_name SET/RESET ... if the ALTER TABLE command targets a view. In * other words ALTER VIEW view_name SET/RESET will use the View_AlterView * but ALTER TABLE view_name SET/RESET will use Table_AlterTable but call process * functions of View_AlterView internally. */ static DistributeObjectOps View_AlterView = { .deparse = DeparseAlterViewStmt, .qualify = QualifyAlterViewStmt, .preprocess = PreprocessAlterViewStmt, .postprocess = PostprocessAlterViewStmt, .operationType = DIST_OPS_ALTER, .address = AlterViewStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Type_Drop = { .deparse = DeparseDropTypeStmt, .qualify = NULL, .preprocess = PreprocessDropDistributedObjectStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Trigger_Drop = { .deparse = NULL, .qualify = NULL, .preprocess = PreprocessDropTriggerStmt, .postprocess = NULL, .operationType = DIST_OPS_DROP, .address = NULL, .markDistributed = false, }; static DistributeObjectOps Type_Rename = { .deparse = DeparseRenameTypeStmt, .qualify = QualifyRenameTypeStmt, .preprocess = PreprocessAlterDistributedObjectStmt, .postprocess = NULL, .objectType = OBJECT_TYPE, .operationType = DIST_OPS_ALTER, .address = RenameTypeStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Vacuum_Analyze = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .postprocess = PostprocessVacuumStmt, .operationType = DIST_OPS_NONE, .address = NULL, .markDistributed = false, }; /* * PreprocessRenameViewStmt function can be called internally by ALTER TABLE view_name * RENAME ... if the ALTER TABLE command targets a view or a view's column. In other words * ALTER VIEW view_name RENAME will use the View_Rename but ALTER TABLE view_name RENAME * will use Any_Rename but call process functions of View_Rename internally. */ static DistributeObjectOps View_Rename = { .deparse = DeparseRenameViewStmt, .qualify = QualifyRenameViewStmt, .preprocess = PreprocessRenameViewStmt, .postprocess = NULL, .operationType = DIST_OPS_ALTER, .address = RenameViewStmtObjectAddress, .markDistributed = false, }; static DistributeObjectOps Trigger_Rename = { .deparse = NULL, .qualify = NULL, .preprocess = NULL, .operationType = DIST_OPS_ALTER, .postprocess = PostprocessAlterTriggerRenameStmt, .address = NULL, .markDistributed = false, }; /* * GetDistributeObjectOps looks up the DistributeObjectOps which handles the node. * * Never returns NULL. */ const DistributeObjectOps * GetDistributeObjectOps(Node *node) { switch (nodeTag(node)) { case T_AlterDatabaseStmt: { return &Database_Alter; } case T_CreatedbStmt: { return &Database_Create; } case T_DropdbStmt: { return &Database_Drop; } case T_AlterDatabaseRefreshCollStmt: { return &Database_RefreshColl; } case T_AlterDatabaseSetStmt: { return &Database_Set; } case T_AlterDomainStmt: { return &Domain_Alter; } case T_AlterEnumStmt: { return &Any_AlterEnum; } case T_AlterExtensionStmt: { return &Any_AlterExtension; } case T_AlterExtensionContentsStmt: { return &Any_AlterExtensionContents; } case T_AlterFunctionStmt: { return &Any_AlterFunction; } case T_AlterForeignServerStmt: { return &Any_AlterForeignServer; } case T_AlterObjectDependsStmt: { AlterObjectDependsStmt *stmt = castNode(AlterObjectDependsStmt, node); switch (stmt->objectType) { case OBJECT_FUNCTION: { return &Function_AlterObjectDepends; } case OBJECT_PROCEDURE: { return &Procedure_AlterObjectDepends; } case OBJECT_ROUTINE: { return &Routine_AlterObjectDepends; } case OBJECT_TRIGGER: { return &Trigger_AlterObjectDepends; } default: { return &NoDistributeOps; } } } case T_AlterObjectSchemaStmt: { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); switch (stmt->objectType) { case OBJECT_AGGREGATE: { return &Aggregate_AlterObjectSchema; } case OBJECT_COLLATION: { return &Collation_AlterObjectSchema; } case OBJECT_DOMAIN: { return &Domain_AlterObjectSchema; } case OBJECT_EXTENSION: { return &Extension_AlterObjectSchema; } case OBJECT_FUNCTION: { return &Function_AlterObjectSchema; } case OBJECT_PROCEDURE: { return &Procedure_AlterObjectSchema; } case OBJECT_ROUTINE: { return &Routine_AlterObjectSchema; } case OBJECT_SEQUENCE: { return &Sequence_AlterObjectSchema; } case OBJECT_STATISTIC_EXT: { return &Statistics_AlterObjectSchema; } case OBJECT_FOREIGN_TABLE: case OBJECT_TABLE: { return &Table_AlterObjectSchema; } case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_AlterObjectSchema; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_AlterObjectSchema; } case OBJECT_TYPE: { return &Type_AlterObjectSchema; } case OBJECT_VIEW: { return &View_AlterObjectSchema; } default: { return &NoDistributeOps; } } } case T_AlterOwnerStmt: { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); switch (stmt->objectType) { case OBJECT_AGGREGATE: { return &Aggregate_AlterOwner; } case OBJECT_COLLATION: { return &Collation_AlterOwner; } case OBJECT_DATABASE: { return &Database_AlterOwner; } case OBJECT_DOMAIN: { return &Domain_AlterOwner; } case OBJECT_FOREIGN_SERVER: { return &ForeignServer_AlterOwner; } case OBJECT_FUNCTION: { return &Function_AlterOwner; } case OBJECT_PROCEDURE: { return &Procedure_AlterOwner; } case OBJECT_PUBLICATION: { return &Publication_AlterOwner; } case OBJECT_ROUTINE: { return &Routine_AlterOwner; } case OBJECT_SCHEMA: { return &Schema_AlterOwner; } case OBJECT_STATISTIC_EXT: { return &Statistics_AlterOwner; } case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_AlterOwner; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_AlterOwner; } case OBJECT_TYPE: { return &Type_AlterOwner; } default: { return &NoDistributeOps; } } } case T_AlterPolicyStmt: { return &Any_AlterPolicy; } case T_AlterPublicationStmt: { return &Publication_Alter; } case T_AlterRoleStmt: { return &Any_AlterRole; } case T_AlterRoleSetStmt: { return &Any_AlterRoleSet; } case T_AlterSeqStmt: { return &Sequence_Alter; } case T_AlterStatsStmt: { return &Statistics_Alter; } case T_AlterTableStmt: { AlterTableStmt *stmt = castNode(AlterTableStmt, node); switch (stmt->objtype) { case OBJECT_TYPE: { return &Type_AlterTable; } case OBJECT_TABLE: { return &Table_AlterTable; } case OBJECT_FOREIGN_TABLE: { return &ForeignTable_AlterTable; } case OBJECT_INDEX: { return &Index_AlterTable; } case OBJECT_SEQUENCE: { ListCell *cmdCell = NULL; foreach(cmdCell, stmt->cmds) { AlterTableCmd *cmd = castNode(AlterTableCmd, lfirst(cmdCell)); switch (cmd->subtype) { case AT_ChangeOwner: { return &Sequence_AlterOwner; } case AT_SetLogged: { return &Sequence_AlterPersistence; } case AT_SetUnLogged: { return &Sequence_AlterPersistence; } default: { return &NoDistributeOps; } } } /* * Prior to PG15, the only Alter Table statement * with Sequence as its object was an * Alter Owner statement */ return &Sequence_AlterOwner; } case OBJECT_VIEW: { return &View_AlterView; } default: { return &NoDistributeOps; } } } case T_AlterTableMoveAllStmt: { return &Any_AlterTableMoveAll; } case T_AlterTSConfigurationStmt: { return &TextSearchConfig_Alter; } case T_AlterTSDictionaryStmt: { return &TextSearchDict_Alter; } case T_ClusterStmt: { return &Any_Cluster; } case T_CommentStmt: { CommentStmt *stmt = castNode(CommentStmt, node); switch (stmt->objtype) { case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_Comment; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_Comment; } case OBJECT_DATABASE: { return &Database_Comment; } case OBJECT_ROLE: { return &Role_Comment; } default: { return &NoDistributeOps; } } } case T_CompositeTypeStmt: { return &Any_CompositeType; } case T_CreateDomainStmt: { return &Any_CreateDomain; } case T_CreateEnumStmt: { return &Any_CreateEnum; } case T_CreateExtensionStmt: { return &Any_CreateExtension; } case T_CreateFunctionStmt: { return &Any_CreateFunction; } case T_CreateForeignServerStmt: { return &Any_CreateForeignServer; } case T_CreatePolicyStmt: { return &Any_CreatePolicy; } case T_CreatePublicationStmt: { return &Any_CreatePublication; } case T_CreateRoleStmt: { return &Any_CreateRole; } case T_CreateSchemaStmt: { return &Any_CreateSchema; } case T_CreateStatsStmt: { return &Any_CreateStatistics; } case T_CreateTrigStmt: { return &Any_CreateTrigger; } case T_DefineStmt: { DefineStmt *stmt = castNode(DefineStmt, node); switch (stmt->kind) { case OBJECT_AGGREGATE: { return &Aggregate_Define; } case OBJECT_COLLATION: { return &Collation_Define; } case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_Define; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_Define; } default: { return &NoDistributeOps; } } } case T_DropRoleStmt: { return &Any_DropRole; } case T_DropOwnedStmt: { return &Any_DropOwned; } case T_ReassignOwnedStmt: { return &Any_ReassignOwned; } case T_DropStmt: { DropStmt *stmt = castNode(DropStmt, node); switch (stmt->removeType) { case OBJECT_AGGREGATE: { return &Aggregate_Drop; } case OBJECT_COLLATION: { return &Collation_Drop; } case OBJECT_DOMAIN: { return &Domain_Drop; } case OBJECT_EXTENSION: { return &Extension_Drop; } case OBJECT_FUNCTION: { return &Function_Drop; } case OBJECT_FOREIGN_SERVER: { return &ForeignServer_Drop; } case OBJECT_INDEX: { return &Index_Drop; } case OBJECT_POLICY: { return &Policy_Drop; } case OBJECT_PROCEDURE: { return &Procedure_Drop; } case OBJECT_PUBLICATION: { return &Publication_Drop; } case OBJECT_ROUTINE: { return &Routine_Drop; } case OBJECT_SCHEMA: { return &Schema_Drop; } case OBJECT_SEQUENCE: { return &Sequence_Drop; } case OBJECT_STATISTIC_EXT: { return &Statistics_Drop; } case OBJECT_TABLE: { return &Table_Drop; } case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_Drop; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_Drop; } case OBJECT_TYPE: { return &Type_Drop; } case OBJECT_TRIGGER: { return &Trigger_Drop; } case OBJECT_VIEW: { return &View_Drop; } default: { return &NoDistributeOps; } } } case T_GrantRoleStmt: { return &Any_GrantRole; } case T_GrantStmt: { GrantStmt *stmt = castNode(GrantStmt, node); switch (stmt->objtype) { case OBJECT_SCHEMA: { return &Schema_Grant; } case OBJECT_SEQUENCE: { return &Sequence_Grant; } case OBJECT_FDW: { return &FDW_Grant; } case OBJECT_FOREIGN_SERVER: { return &ForeignServer_Grant; } case OBJECT_FUNCTION: { return &Function_Grant; } case OBJECT_AGGREGATE: { return &Aggregate_Grant; } case OBJECT_PROCEDURE: { return &Procedure_Grant; } case OBJECT_ROUTINE: { return &Routine_Grant; } case OBJECT_DATABASE: { return &Database_Grant; } default: { return &Any_Grant; } } } case T_IndexStmt: { return &Any_Index; } case T_ViewStmt: { return &Any_View; } case T_ReindexStmt: { return &Any_Reindex; } case T_VacuumStmt: { return &Vacuum_Analyze; } case T_SecLabelStmt: { SecLabelStmt *stmt = castNode(SecLabelStmt, node); switch (stmt->objtype) { case OBJECT_ROLE: { return &Role_SecLabel; } case OBJECT_TABLE: { return &Table_SecLabel; } case OBJECT_COLUMN: { return &Column_SecLabel; } default: { return &Any_SecLabel; } } } case T_RenameStmt: { RenameStmt *stmt = castNode(RenameStmt, node); switch (stmt->renameType) { case OBJECT_AGGREGATE: { return &Aggregate_Rename; } case OBJECT_ATTRIBUTE: { return &Attribute_Rename; } case OBJECT_COLLATION: { return &Collation_Rename; } case OBJECT_DATABASE: { return &Database_Rename; } case OBJECT_DOMAIN: { return &Domain_Rename; } case OBJECT_DOMCONSTRAINT: { return &Domain_RenameConstraint; } case OBJECT_FOREIGN_SERVER: { return &ForeignServer_Rename; } case OBJECT_FUNCTION: { return &Function_Rename; } case OBJECT_PROCEDURE: { return &Procedure_Rename; } case OBJECT_PUBLICATION: { return &Publication_Rename; } case OBJECT_ROLE: { return &Any_AlterRoleRename; } case OBJECT_ROUTINE: { return &Routine_Rename; } case OBJECT_SCHEMA: { return &Schema_Rename; } case OBJECT_SEQUENCE: { return &Sequence_Rename; } case OBJECT_STATISTIC_EXT: { return &Statistics_Rename; } case OBJECT_TSCONFIGURATION: { return &TextSearchConfig_Rename; } case OBJECT_TSDICTIONARY: { return &TextSearchDict_Rename; } case OBJECT_TYPE: { return &Type_Rename; } case OBJECT_TRIGGER: { return &Trigger_Rename; } case OBJECT_VIEW: { return &View_Rename; } case OBJECT_COLUMN: { switch (stmt->relationType) { case OBJECT_VIEW: { return &View_Rename; } default: { return &Any_Rename; } } } default: { return &Any_Rename; } } } default: { return &NoDistributeOps; } } } ================================================ FILE: src/backend/distributed/commands/domain.c ================================================ /*------------------------------------------------------------------------- * * domain.c * Hooks to handle the creation, altering and removal of domains. * These hooks are responsible for duplicating the changes to the * workers nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/regproc.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_transaction.h" static CollateClause * MakeCollateClauseFromOid(Oid collationOid); static List * GetDomainAddressByName(TypeName *domainName, bool missing_ok); /* * GetDomainAddressByName returns the ObjectAddress of the domain identified by * domainName. When missing_ok is true the object id part of the ObjectAddress can be * InvalidOid. When missing_ok is false this function will raise an error instead when the * domain can't be found. */ static List * GetDomainAddressByName(TypeName *domainName, bool missing_ok) { ObjectAddress *address = palloc0(sizeof(ObjectAddress)); Oid domainOid = LookupTypeNameOid(NULL, domainName, missing_ok); ObjectAddressSet(*address, TypeRelationId, domainOid); return list_make1(address); } /* * RecreateDomainStmt returns a CreateDomainStmt pointer where the statement represents * the creation of the domain to recreate the domain on a different postgres node based on * the current representation in the local catalog. */ CreateDomainStmt * RecreateDomainStmt(Oid domainOid) { CreateDomainStmt *stmt = makeNode(CreateDomainStmt); stmt->domainname = stringToQualifiedNameList(format_type_be_qualified(domainOid), NULL); HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(domainOid)); if (!HeapTupleIsValid(tup)) { elog(ERROR, "cache lookup failed for type %u", domainOid); } Form_pg_type typTup = (Form_pg_type) GETSTRUCT(tup); if (typTup->typtype != TYPTYPE_DOMAIN) { elog(ERROR, "type is not a domain type"); } stmt->typeName = makeTypeNameFromOid(typTup->typbasetype, typTup->typtypmod); if (OidIsValid(typTup->typcollation)) { stmt->collClause = MakeCollateClauseFromOid(typTup->typcollation); } /* * typdefault and typdefaultbin are potentially null, so don't try to * access 'em as struct fields. Must do it the hard way with * SysCacheGetAttr. */ bool isNull = false; Datum typeDefaultDatum = SysCacheGetAttr(TYPEOID, tup, Anum_pg_type_typdefaultbin, &isNull); if (!isNull) { /* when not null there is default value which we should add as a constraint */ Constraint *constraint = makeNode(Constraint); constraint->contype = CONSTR_DEFAULT; constraint->cooked_expr = TextDatumGetCString(typeDefaultDatum); stmt->constraints = lappend(stmt->constraints, constraint); } /* NOT NULL constraints are non-named on the actual type */ if (typTup->typnotnull) { Constraint *constraint = makeNode(Constraint); constraint->contype = CONSTR_NOTNULL; stmt->constraints = lappend(stmt->constraints, constraint); } /* lookup and look all constraints to add them to the CreateDomainStmt */ Relation conRel = table_open(ConstraintRelationId, AccessShareLock); /* Look for CHECK Constraints on this domain */ ScanKeyData key[1]; ScanKeyInit(&key[0], Anum_pg_constraint_contypid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(domainOid)); SysScanDesc scan = systable_beginscan(conRel, ConstraintTypidIndexId, true, NULL, 1, key); HeapTuple conTup = NULL; while (HeapTupleIsValid(conTup = systable_getnext(scan))) { Form_pg_constraint c = (Form_pg_constraint) GETSTRUCT(conTup); if (c->contype != CONSTRAINT_CHECK) { /* Ignore non-CHECK constraints, shouldn't be any */ continue; } /* * We create a constraint, completely ignoring c->convalidated because we can't * create a domain with an invalidated constraint. Once a constraint is added to * a domain -even non valid-, all new data is validated. Meaning, creating a * domain with a non-valid constraint doesn't make any sense. * * Given it will be too hard to defer the creation of a constraint till we * validate the constraint on the coordinator we will simply create the * non-validated constraint to ad hear to validating all new data. * * An edgecase here would be when moving existing data, that hasn't been validated * before to an other node. This behaviour is consistent with sending it to an * already existing node (that has the constraint created but not validated) and a * new node. */ Constraint *constraint = makeNode(Constraint); constraint->conname = pstrdup(NameStr(c->conname)); constraint->contype = CONSTR_CHECK; /* we only come here with check constraints */ /* Not expecting conbin to be NULL, but we'll test for it anyway */ Datum conbin = heap_getattr(conTup, Anum_pg_constraint_conbin, conRel->rd_att, &isNull); if (isNull) { elog(ERROR, "domain \"%s\" constraint \"%s\" has NULL conbin", NameStr(typTup->typname), NameStr(c->conname)); } /* * The conbin containes the cooked expression from when the constraint was * inserted into the catalog. We store it here for the deparser to distinguish * between cooked expressions and raw expressions. * * There is no supported way to go from a cooked expression to a raw expression. */ constraint->cooked_expr = TextDatumGetCString(conbin); stmt->constraints = lappend(stmt->constraints, constraint); } systable_endscan(scan); table_close(conRel, NoLock); ReleaseSysCache(tup); QualifyTreeNode((Node *) stmt); return stmt; } /* * MakeCollateClauseFromOid returns a CollateClause describing the COLLATE segment of a * CREATE DOMAIN statement based on the Oid of the collation used for the domain. */ static CollateClause * MakeCollateClauseFromOid(Oid collationOid) { CollateClause *collateClause = makeNode(CollateClause); ObjectAddress collateAddress = { 0 }; ObjectAddressSet(collateAddress, CollationRelationId, collationOid); List *objName = NIL; List *objArgs = NIL; getObjectIdentityParts(&collateAddress, &objName, &objArgs, false); char *name = NULL; foreach_declared_ptr(name, objName) { collateClause->collname = lappend(collateClause->collname, makeString(name)); } collateClause->location = -1; return collateClause; } /* * CreateDomainStmtObjectAddress returns the ObjectAddress of the domain that would be * created by the statement. When missing_ok is false the function will raise an error if * the domain cannot be found in the local catalog. */ List * CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateDomainStmt *stmt = castNode(CreateDomainStmt, node); TypeName *typeName = makeTypeNameFromNameList(stmt->domainname); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * AlterDomainStmtObjectAddress returns the ObjectAddress of the domain being altered. * When missing_ok is false this function will raise an error when the domain is not * found. */ List * AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterDomainStmt *stmt = castNode(AlterDomainStmt, node); TypeName *domainName = makeTypeNameFromNameList(stmt->typeName); List *domainObjectAddresses = GetDomainAddressByName(domainName, missing_ok); /* the code-path only supports a single object */ Assert(list_length(domainObjectAddresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(domainObjectAddresses); Oid domainOid = address->objectId; bool isDropConstraintStmt = (stmt->subtype == 'X'); if (!isPostprocess && isDropConstraintStmt && OidIsValid(domainOid)) { /* * we validate constraint if we are not in postprocess yet. It should have * been already dropped at postprocess, so we do not validate in postprocess. */ char *constraintName = stmt->name; Oid constraintOid = get_domain_constraint_oid(domainOid, constraintName, missing_ok); if (!OidIsValid(constraintOid)) { /* * Although the domain is valid, the constraint is not. Eventually, PG will * throw an error. To prevent diverging outputs between Citus and PG, we treat * the domain as invalid. */ address->objectId = InvalidOid; } } return list_make1(address); } /* * DomainRenameConstraintStmtObjectAddress returns the ObjectAddress of the domain for * which the constraint is being renamed. When missing_ok this function will raise an * error if the domain cannot be found. */ List * DomainRenameConstraintStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); TypeName *domainName = makeTypeNameFromNameList(castNode(List, stmt->object)); return GetDomainAddressByName(domainName, missing_ok); } /* * AlterDomainOwnerStmtObjectAddress returns the ObjectAddress for which the owner is * being changed. When missing_ok is false this function will raise an error if the domain * cannot be found. */ List * AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_DOMAIN); TypeName *domainName = makeTypeNameFromNameList(castNode(List, stmt->object)); return GetDomainAddressByName(domainName, missing_ok); } /* * RenameDomainStmtObjectAddress returns the ObjectAddress of the domain being renamed. * When missing_ok is false this function will raise an error when the domain cannot be * found. */ List * RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_DOMAIN); TypeName *domainName = makeTypeNameFromNameList(castNode(List, stmt->object)); return GetDomainAddressByName(domainName, missing_ok); } /* * get_constraint_typid returns the contypid of a constraint. This field is only set for * constraints on domain types. Returns InvalidOid if conoid is an invalid constraint, as * well as for constraints that are not on domain types. */ Oid get_constraint_typid(Oid conoid) { HeapTuple tp = SearchSysCache1(CONSTROID, ObjectIdGetDatum(conoid)); if (HeapTupleIsValid(tp)) { Form_pg_constraint contup = (Form_pg_constraint) GETSTRUCT(tp); Oid result = contup->contypid; ReleaseSysCache(tp); return result; } else { return InvalidOid; } } ================================================ FILE: src/backend/distributed/commands/drop_distributed_table.c ================================================ /*------------------------------------------------------------------------- * * drop_distributed_table.c * Routines related to dropping distributed relations from a trigger. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_transaction.h" /* local function forward declarations */ static void MasterRemoveDistributedTableMetadataFromWorkers(Oid relationId, char *schemaName, char *tableName); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_drop_distributed_table_metadata); PG_FUNCTION_INFO_V1(master_remove_partition_metadata); PG_FUNCTION_INFO_V1(master_remove_distributed_table_metadata_from_workers); PG_FUNCTION_INFO_V1(notify_constraint_dropped); /* * master_drop_distributed_table_metadata UDF is a stub UDF to install Citus flawlessly. * Otherwise we need to delete them from our sql files, which is confusing and not a * common operation in the code-base. * * This function is basically replaced with * master_remove_distributed_table_metadata_from_workers() followed by * master_remove_partition_metadata(). */ Datum master_drop_distributed_table_metadata(PG_FUNCTION_ARGS) { ereport(INFO, (errmsg("this function is deprecated and no longer is used"))); PG_RETURN_VOID(); } /* * master_remove_partition_metadata removes the entry of the specified distributed * table from pg_dist_partition. */ Datum master_remove_partition_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *schemaNameText = PG_GETARG_TEXT_P(1); text *tableNameText = PG_GETARG_TEXT_P(2); char *schemaName = text_to_cstring(schemaNameText); char *tableName = text_to_cstring(tableNameText); uint32 colocationId = ColocationIdViaCatalog(relationId); /* * The SQL_DROP trigger calls this function even for tables that are * not distributed. In that case, silently ignore. This is not very * user-friendly, but this function is really only meant to be called * from the trigger. */ if (!IsCitusTableViaCatalog(relationId) || !EnableDDLPropagation) { PG_RETURN_VOID(); } EnsureCoordinator(); CheckTableSchemaNameForDrop(relationId, &schemaName, &tableName); DeletePartitionRow(relationId); /* * We want to keep using the same colocation group for the tenant even if * all the tables that belong to it are dropped and new tables are created * for the tenant etc. For this reason, if a colocation group belongs to a * tenant schema, we don't delete the colocation group even if there are no * tables that belong to it. * * We do the same if system catalog cannot find the schema of the table * because this means that the whole schema is dropped. * * In that case, we want to delete the colocation group regardless of * whether the schema is a tenant schema or not. Even more, calling * IsTenantSchema() with InvalidOid would cause an error, hence we check * whether the schema is valid beforehand. */ bool missingOk = true; Oid schemaId = get_namespace_oid(schemaName, missingOk); if (!OidIsValid(schemaId) || !IsTenantSchema(schemaId)) { DeleteColocationGroupIfNoTablesBelong(colocationId); } PG_RETURN_VOID(); } /* * master_remove_distributed_table_metadata_from_workers removes the entry of the * specified distributed table from pg_dist_partition and drops the table from * the workers if needed. */ Datum master_remove_distributed_table_metadata_from_workers(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *schemaNameText = PG_GETARG_TEXT_P(1); text *tableNameText = PG_GETARG_TEXT_P(2); char *schemaName = text_to_cstring(schemaNameText); char *tableName = text_to_cstring(tableNameText); CheckTableSchemaNameForDrop(relationId, &schemaName, &tableName); MasterRemoveDistributedTableMetadataFromWorkers(relationId, schemaName, tableName); PG_RETURN_VOID(); } /* * MasterRemoveDistributedTableMetadataFromWorkers drops the table and removes * all the metadata belonging the distributed table in the worker nodes * with metadata. The function doesn't drop the tables that are * the shards on the workers. * * The function is a no-op for non-distributed tables and clusters that don't * have any workers with metadata. Also, the function errors out if called * from a worker node. * * This function assumed that it is called via a trigger. But we cannot do the * typical CALLED_AS_TRIGGER check because this is called via another trigger, * which CALLED_AS_TRIGGER does not cover. */ static void MasterRemoveDistributedTableMetadataFromWorkers(Oid relationId, char *schemaName, char *tableName) { /* * The SQL_DROP trigger calls this function even for tables that are * not distributed. In that case, silently ignore. This is not very * user-friendly, but this function is really only meant to be called * from the trigger. */ if (!IsCitusTableViaCatalog(relationId) || !EnableDDLPropagation) { return; } EnsureCoordinator(); if (!ShouldSyncTableMetadataViaCatalog(relationId)) { return; } if (PartitionTable(relationId)) { /* * MasterRemoveDistributedTableMetadataFromWorkers is only called from drop trigger. * When parent is dropped in a drop trigger, we remove all the corresponding * partitions via the parent, mostly for performance reasons. */ return; } /* drop the distributed table metadata on the workers */ char *deleteDistributionCommand = DistributionDeleteCommand(schemaName, tableName); SendCommandToWorkersWithMetadata(deleteDistributionCommand); } /* * notify_constraint_dropped simply calls NotifyUtilityHookConstraintDropped * to set ConstraintDropped to true. * This udf is designed to be called from citus_drop_trigger to tell us we * dropped a table constraint. */ Datum notify_constraint_dropped(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* * We reset this only in utility hook, so we should not set this flag * otherwise if we are not in utility hook. * In some cases -where dropping foreign key not issued via utility * hook-, we would not be able to undistribute such citus local tables * but we are ok with that. */ if (UtilityHookLevel >= 1) { NotifyUtilityHookConstraintDropped(); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/commands/extension.c ================================================ /*------------------------------------------------------------------------- * * extension.c * Commands for creating and altering extensions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension_d.h" #include "catalog/pg_foreign_data_wrapper.h" #include "commands/defrem.h" #include "commands/extension.h" #include "foreign/foreign.h" #include "nodes/makefuncs.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #if PG_VERSION_NUM < PG_VERSION_17 #include "catalog/pg_am_d.h" #endif #include "citus_version.h" #include "columnar/columnar.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/transaction_management.h" /* Local functions forward declarations for helper functions */ static char * ExtractNewExtensionVersion(Node *parseTree); static void AddSchemaFieldIfMissing(CreateExtensionStmt *stmt); static List * FilterDistributedExtensions(List *extensionObjectList); static List * ExtensionNameListToObjectAddressList(List *extensionObjectList); static void MarkExistingObjectDependenciesDistributedIfSupported(void); static List * GetAllViews(void); static bool ShouldPropagateExtensionCommand(Node *parseTree); static bool IsAlterExtensionSetSchemaCitus(Node *parseTree); static bool HasAnyRelationsUsingOldColumnar(void); static Oid GetOldColumnarAMIdIfExists(void); static bool AccessMethodDependsOnAnyExtensions(Oid accessMethodId); static bool HasAnyRelationsUsingAccessMethod(Oid accessMethodId); static Node * RecreateExtensionStmt(Oid extensionOid); static List * GenerateGrantCommandsOnExtensionDependentFDWs(Oid extensionId); /* * ErrorIfUnstableCreateOrAlterExtensionStmt compares CITUS_EXTENSIONVERSION * and version given CREATE/ALTER EXTENSION statement will create/update to. If * they are not same in major or minor version numbers, this function errors * out. It ignores the schema version. */ void ErrorIfUnstableCreateOrAlterExtensionStmt(Node *parseTree) { char *newExtensionVersion = ExtractNewExtensionVersion(parseTree); if (newExtensionVersion != NULL) { /* explicit version provided in CREATE or ALTER EXTENSION UPDATE; verify */ if (!MinorVersionsCompatibleRelaxed(newExtensionVersion, CITUS_EXTENSIONVERSION)) { ereport(ERROR, (errmsg("specified version incompatible with loaded " "Citus library"), errdetail("Loaded library requires %s, but %s was specified.", CITUS_MAJORVERSION, newExtensionVersion), errhint("If a newer library is present, restart the database " "and try the command again."))); } } else { /* * No version was specified, so PostgreSQL will use the default_version * from the citus.control file. */ CheckAvailableVersion(ERROR); } } /* * ExtractNewExtensionVersion returns the palloc'd new extension version specified * by a CREATE or ALTER EXTENSION statement. Other inputs are not permitted. * This function returns NULL for statements with no explicit version specified. */ static char * ExtractNewExtensionVersion(Node *parseTree) { List *optionsList = NIL; if (IsA(parseTree, CreateExtensionStmt)) { optionsList = ((CreateExtensionStmt *) parseTree)->options; } else if (IsA(parseTree, AlterExtensionStmt)) { optionsList = ((AlterExtensionStmt *) parseTree)->options; } else { /* input must be one of the two above types */ Assert(false); } DefElem *newVersionValue = GetExtensionOption(optionsList, "new_version"); /* return target string safely */ if (newVersionValue) { const char *newVersion = defGetString(newVersionValue); return pstrdup(newVersion); } else { return NULL; } } /* * PostprocessCreateExtensionStmt is called after the creation of an extension. * We decide if the extension needs to be replicated to the worker, and * if that is the case return a list of DDLJob's that describe how and * where the extension needs to be created. * * As we now have access to ObjectAddress of the extension that is just * created, we can mark it as distributed to make sure that its * dependencies exist on all nodes. */ List * PostprocessCreateExtensionStmt(Node *node, const char *queryString) { CreateExtensionStmt *stmt = castNode(CreateExtensionStmt, node); if (!ShouldPropagateExtensionCommand(node)) { return NIL; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return NIL; } /* extension management can only be done via coordinator node */ EnsureCoordinator(); /* * Make sure that the current transaction is already in sequential mode, * or can still safely be put in sequential mode */ EnsureSequentialMode(OBJECT_EXTENSION); /* * Here we append "schema" field to the "options" list (if not specified) * to satisfy the schema consistency between worker nodes and the coordinator. */ AddSchemaFieldIfMissing(stmt); /* always send commands with IF NOT EXISTS */ stmt->if_not_exists = true; const char *createExtensionStmtSql = DeparseTreeNode(node); /* * To prevent recursive propagation in mx architecture, we disable ddl * propagation before sending the command to workers. */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) createExtensionStmtSql, ENABLE_DDL_PROPAGATION); List *extensionAddresses = GetObjectAddressListFromParseTree(node, false, true); /* the code-path only supports a single object */ Assert(list_length(extensionAddresses) == 1); /* * If the extension is already distributed, don't do anything to avoid * ownership checks on workers. This is important when a non-owner user runs * CREATE EXTENSION IF NOT EXISTS for an existing extension - PostgreSQL's * standard_ProcessUtility succeeds (extension exists, no-op), but metadata * propagation would fail the ownership check. */ if (IsAnyObjectDistributed(extensionAddresses)) { return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(extensionAddresses); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * AddSchemaFieldIfMissing adds DefElem item for "schema" (if not specified * in statement) to "options" list before deparsing the statement to satisfy * the schema consistency between worker nodes and the coordinator. */ static void AddSchemaFieldIfMissing(CreateExtensionStmt *createExtensionStmt) { List *optionsList = createExtensionStmt->options; DefElem *schemaNameValue = GetExtensionOption(optionsList, "schema"); if (!schemaNameValue) { /* * As we already created the extension by standard_ProcessUtility, * we actually know the schema it belongs to */ const bool missingOk = false; Oid extensionOid = get_extension_oid(createExtensionStmt->extname, missingOk); Oid extensionSchemaOid = get_extension_schema(extensionOid); char *extensionSchemaName = get_namespace_name(extensionSchemaOid); Node *schemaNameArg = (Node *) makeString(extensionSchemaName); /* set location to -1 as it is unknown */ int location = -1; DefElem *newDefElement = makeDefElem("schema", schemaNameArg, location); createExtensionStmt->options = lappend(createExtensionStmt->options, newDefElement); } } /* * PreprocessDropExtensionStmt is called to drop extension(s) in coordinator and * in worker nodes if distributed before. * We first ensure that we keep only the distributed ones before propagating * the statement to worker nodes. * If no extensions in the drop list are distributed, then no calls will * be made to the workers. */ List * PreprocessDropExtensionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *stmt = castNode(DropStmt, node); if (!ShouldPropagateExtensionCommand(node)) { return NIL; } /* get distributed extensions to be dropped in worker nodes as well */ List *allDroppedExtensions = stmt->objects; List *distributedExtensions = FilterDistributedExtensions(allDroppedExtensions); if (list_length(distributedExtensions) <= 0) { /* no distributed extensions to drop */ return NIL; } /* extension management can only be done via coordinator node */ EnsureCoordinator(); /* * Make sure that the current transaction is already in sequential mode, * or can still safely be put in sequential mode */ EnsureSequentialMode(OBJECT_EXTENSION); List *distributedExtensionAddresses = ExtensionNameListToObjectAddressList( distributedExtensions); /* unmark each distributed extension */ ObjectAddress *address = NULL; foreach_declared_ptr(address, distributedExtensionAddresses) { UnmarkObjectDistributed(address); } /* * Temporary swap the lists of objects to delete with the distributed * objects and deparse to an sql statement for the workers. * Then switch back to allDroppedExtensions to drop all specified * extensions in coordinator after PreprocessDropExtensionStmt completes * its execution. */ stmt->objects = distributedExtensions; const char *deparsedStmt = DeparseTreeNode((Node *) stmt); stmt->objects = allDroppedExtensions; /* * To prevent recursive propagation in mx architecture, we disable ddl * propagation before sending the command to workers. */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) deparsedStmt, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * FilterDistributedExtensions returns the distributed objects in an "objects" * list of a DropStmt, a list having the format of a "DropStmt.objects" list. * That is, in turn, a list of string "Value"s. */ static List * FilterDistributedExtensions(List *extensionObjectList) { List *extensionNameList = NIL; String *objectName = NULL; foreach_declared_ptr(objectName, extensionObjectList) { const char *extensionName = strVal(objectName); const bool missingOk = true; Oid extensionOid = get_extension_oid(extensionName, missingOk); if (!OidIsValid(extensionOid)) { continue; } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ExtensionRelationId, extensionOid); if (!IsAnyObjectDistributed(list_make1(address))) { continue; } extensionNameList = lappend(extensionNameList, objectName); } return extensionNameList; } /* * ExtensionNameListToObjectAddressList returns the object addresses in * an ObjectAddress list for an "objects" list of a DropStmt. * Callers of this function should ensure that all the objects in the list * are valid and distributed. */ static List * ExtensionNameListToObjectAddressList(List *extensionObjectList) { List *extensionObjectAddressList = NIL; String *objectName; foreach_declared_ptr(objectName, extensionObjectList) { /* * We set missingOk to false as we assume all the objects in * extensionObjectList list are valid and distributed. */ const char *extensionName = strVal(objectName); const bool missingOk = false; ObjectAddress *address = palloc0(sizeof(ObjectAddress)); Oid extensionOid = get_extension_oid(extensionName, missingOk); ObjectAddressSet(*address, ExtensionRelationId, extensionOid); extensionObjectAddressList = lappend(extensionObjectAddressList, address); } return extensionObjectAddressList; } /* * PreprocessAlterExtensionSchemaStmt is invoked for alter extension set schema statements. */ List * PreprocessAlterExtensionSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagateExtensionCommand(node)) { return NIL; } /* extension management can only be done via coordinator node */ EnsureCoordinator(); /* * Make sure that the current transaction is already in sequential mode, * or can still safely be put in sequential mode */ EnsureSequentialMode(OBJECT_EXTENSION); const char *alterExtensionStmtSql = DeparseTreeNode(node); /* * To prevent recursive propagation in mx architecture, we disable ddl * propagation before sending the command to workers. */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) alterExtensionStmtSql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PostprocessAlterExtensionSchemaStmt is executed after the change has been applied * locally, we can now use the new dependencies (schema) of the extension to ensure * all its dependencies exist on the workers before we apply the commands remotely. */ List * PostprocessAlterExtensionSchemaStmt(Node *node, const char *queryString) { List *extensionAddresses = GetObjectAddressListFromParseTree(node, false, true); /* the code-path only supports a single object */ Assert(list_length(extensionAddresses) == 1); if (!ShouldPropagateExtensionCommand(node)) { return NIL; } /* dependencies (schema) have changed let's ensure they exist */ EnsureAllObjectDependenciesExistOnAllNodes(extensionAddresses); return NIL; } /* * PreprocessAlterExtensionUpdateStmt is invoked for alter extension update statements. */ List * PreprocessAlterExtensionUpdateStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterExtensionStmt *alterExtensionStmt = castNode(AlterExtensionStmt, node); if (!ShouldPropagateExtensionCommand((Node *) alterExtensionStmt)) { return NIL; } /* extension management can only be done via coordinator node */ EnsureCoordinator(); /* * Make sure that the current transaction is already in sequential mode, * or can still safely be put in sequential mode */ EnsureSequentialMode(OBJECT_EXTENSION); const char *alterExtensionStmtSql = DeparseTreeNode((Node *) alterExtensionStmt); /* * To prevent recursive propagation in mx architecture, we disable ddl * propagation before sending the command to workers. */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) alterExtensionStmtSql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PostprocessAlterExtensionCitusUpdateStmt is called after ALTER EXTENSION * citus UPDATE command is executed by standard utility hook. * * Actually, we do not need such a post process function for ALTER EXTENSION * UPDATE commands unless the extension is Citus itself. This is because we * need to mark existing objects that are not included in distributed object * infrastructure in older versions of Citus (but now should be) as distributed * if we really updated Citus to the available version successfully via standard * utility hook. */ void PostprocessAlterExtensionCitusUpdateStmt(Node *node) { bool citusIsUpdatedToLatestVersion = InstalledAndAvailableVersionsSame(); /* * Knowing that Citus version was different than the available version before * standard process utility runs ALTER EXTENSION command, we perform post * process operations if Citus is updated to that available version */ if (!citusIsUpdatedToLatestVersion) { return; } /* * Finally, mark existing objects that are not included in distributed object * infrastructure in older versions of Citus (but now should be) as distributed */ MarkExistingObjectDependenciesDistributedIfSupported(); } /* * MarkExistingObjectDependenciesDistributedIfSupported marks all objects that could * be distributed by resolving dependencies of "existing distributed tables" and * "already distributed objects" to introduce the objects created in older versions * of Citus to distributed object infrastructure as well. * * Note that this function is not responsible for ensuring if dependencies exist on * nodes and satisfying these dependendencies if not exists, which is already done by * EnsureAllObjectDependenciesExistOnAllNodes on demand. Hence, this function is just designed * to be used when "ALTER EXTENSION citus UPDATE" is executed. * This is because we want to add existing objects that would have already been in * pg_dist_object if we had created them in new version of Citus to pg_dist_object. */ static void MarkExistingObjectDependenciesDistributedIfSupported() { /* resulting object addresses to be marked as distributed */ List *resultingObjectAddresses = NIL; /* resolve dependencies of citus tables */ List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid citusTableId = InvalidOid; foreach_declared_oid(citusTableId, citusTableIdList) { if (!ShouldMarkRelationDistributed(citusTableId)) { continue; } /* refrain reading the metadata cache for all tables */ if (ShouldSyncTableMetadataViaCatalog(citusTableId)) { ObjectAddress tableAddress = { 0 }; ObjectAddressSet(tableAddress, RelationRelationId, citusTableId); /* * We mark tables distributed immediately because we also need to mark * views as distributed. We check whether the views that depend on * the table has any auto-distirbutable dependencies below. Citus * currently cannot "auto" distribute tables as dependencies, so we * mark them distributed immediately. */ MarkObjectDistributedLocally(&tableAddress); /* * All the distributable dependencies of a table should be marked as * distributed. */ List *distributableDependencyObjectAddresses = GetDistributableDependenciesForObject(&tableAddress); resultingObjectAddresses = list_concat(resultingObjectAddresses, distributableDependencyObjectAddresses); } } /* * As of Citus 11, views on Citus tables that do not have any unsupported * dependency should also be distributed. * * In general, we mark views distributed as long as it does not have * any unsupported dependencies. */ List *viewList = GetAllViews(); Oid viewOid = InvalidOid; foreach_declared_oid(viewOid, viewList) { if (!ShouldMarkRelationDistributed(viewOid)) { continue; } ObjectAddress viewAddress = { 0 }; ObjectAddressSet(viewAddress, RelationRelationId, viewOid); /* * If a view depends on multiple views, that view will be marked * as distributed while it is processed for the last view * table. */ MarkObjectDistributedLocally(&viewAddress); /* we need to pass pointer allocated in the heap */ ObjectAddress *addressPointer = palloc0(sizeof(ObjectAddress)); *addressPointer = viewAddress; List *distributableDependencyObjectAddresses = GetDistributableDependenciesForObject(&viewAddress); resultingObjectAddresses = list_concat(resultingObjectAddresses, distributableDependencyObjectAddresses); } /* resolve dependencies of the objects in pg_dist_object*/ List *distributedObjectAddressList = GetDistributedObjectAddressList(); ObjectAddress *distributedObjectAddress = NULL; foreach_declared_ptr(distributedObjectAddress, distributedObjectAddressList) { List *distributableDependencyObjectAddresses = GetDistributableDependenciesForObject(distributedObjectAddress); resultingObjectAddresses = list_concat(resultingObjectAddresses, distributableDependencyObjectAddresses); } /* remove duplicates from object addresses list for efficiency */ List *uniqueObjectAddresses = GetUniqueDependenciesList(resultingObjectAddresses); /* * We should sync the new dependencies during ALTER EXTENSION because * we cannot know whether the nodes has already been upgraded or not. If * the nodes are not upgraded at this point, we cannot sync the object. Also, * when the workers upgraded, they'd get the same objects anyway. */ bool prevMetadataSyncValue = EnableMetadataSync; SetLocalEnableMetadataSync(false); ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, uniqueObjectAddresses) { MarkObjectDistributed(objectAddress); } SetLocalEnableMetadataSync(prevMetadataSyncValue); } /* * GetAllViews returns list of view oids that exists on this server. */ static List * GetAllViews(void) { List *viewOidList = NIL; Relation pgClass = table_open(RelationRelationId, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgClass, InvalidOid, false, NULL, 0, NULL); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_class relationForm = (Form_pg_class) GETSTRUCT(heapTuple); /* we're only interested in views */ if (relationForm->relkind == RELKIND_VIEW) { viewOidList = lappend_oid(viewOidList, relationForm->oid); } heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgClass, NoLock); return viewOidList; } /* * PreprocessAlterExtensionContentsStmt issues a notice. It does not propagate. */ List * PreprocessAlterExtensionContentsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { ereport(NOTICE, (errmsg( "Citus does not propagate adding/dropping member objects"), errhint( "You can add/drop the member objects on the workers as well."))); return NIL; } /* * ShouldPropagateExtensionCommand determines whether to propagate an extension * command to the worker nodes. */ static bool ShouldPropagateExtensionCommand(Node *parseTree) { /* if we disabled object propagation, then we should not propagate anything. */ if (!EnableMetadataSync) { return false; } /* * If extension command is run for/on citus, leave the rest to standard utility hook * by returning false. */ if (IsCreateAlterExtensionUpdateCitusStmt(parseTree)) { return false; } else if (IsDropCitusExtensionStmt(parseTree)) { return false; } else if (IsAlterExtensionSetSchemaCitus(parseTree)) { return false; } return true; } /* * IsCreateAlterExtensionUpdateCitusStmt returns whether a given utility is a * CREATE or ALTER EXTENSION UPDATE statement which references the citus extension. * This function returns false for all other inputs. */ bool IsCreateAlterExtensionUpdateCitusStmt(Node *parseTree) { const char *extensionName = NULL; if (IsA(parseTree, CreateExtensionStmt)) { extensionName = ((CreateExtensionStmt *) parseTree)->extname; } else if (IsA(parseTree, AlterExtensionStmt)) { extensionName = ((AlterExtensionStmt *) parseTree)->extname; } else { /* * If it is not a Create Extension or a Alter Extension stmt, * it does not matter if the it is about citus */ return false; } /* * Now that we have CreateExtensionStmt or AlterExtensionStmt, * check if it is run for/on citus */ return (strncasecmp(extensionName, "citus", NAMEDATALEN) == 0); } /* * PreProcessCreateExtensionCitusStmtForColumnar determines whether need to * install citus_columnar first or citus_columnar is supported on current * citus version, when a given utility is a CREATE statement */ void PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree) { /*CREATE EXTENSION CITUS (version Z) */ CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt, parsetree); if (strcmp(createExtensionStmt->extname, "citus") == 0) { int versionNumber = (int) (100 * strtod(CITUS_MAJORVERSION, NULL)); DefElem *newVersionValue = GetExtensionOption(createExtensionStmt->options, "new_version"); /*create extension citus version xxx*/ if (newVersionValue) { char *newVersion = pstrdup(defGetString(newVersionValue)); versionNumber = GetExtensionVersionNumber(newVersion); } /*citus version >= 11.1 requires install citus_columnar first*/ if (versionNumber >= 1110 && !CitusHasBeenLoaded()) { if (get_extension_oid("citus_columnar", true) == InvalidOid && (versionNumber < 1320 || HasAnyRelationsUsingOldColumnar())) { CreateExtensionWithVersion("citus_columnar", NULL); } } } /*Edge case check: citus_columnar are supported on citus version >= 11.1*/ if (strcmp(createExtensionStmt->extname, "citus_columnar") == 0) { Oid citusOid = get_extension_oid("citus", true); if (citusOid != InvalidOid) { char *curCitusVersion = pstrdup(get_extension_version(citusOid)); int curCitusVersionNum = GetExtensionVersionNumber(curCitusVersion); if (curCitusVersionNum < 1110) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "must upgrade citus to version 11.1-1 first before install citus_columnar"))); } } } } /* * IsDropCitusExtensionStmt iterates the objects to be dropped in a drop statement * and try to find citus extension there. */ bool IsDropCitusExtensionStmt(Node *parseTree) { /* if it is not a DropStmt, it is needless to search for citus */ if (!IsA(parseTree, DropStmt)) { return false; } DropStmt *dropStmt = (DropStmt *) parseTree; /* check if the drop command is a DROP EXTENSION command */ if (dropStmt->removeType != OBJECT_EXTENSION) { return false; } /* now that we have a DropStmt, check if citus extension is among the objects to dropped */ String *objectName; foreach_declared_ptr(objectName, dropStmt->objects) { const char *extensionName = strVal(objectName); if (strncasecmp(extensionName, "citus", NAMEDATALEN) == 0) { return true; } } return false; } /* * IsAlterExtensionSetSchemaCitus returns whether a given utility is an * ALTER EXTENSION SET SCHEMA statement which references the citus extension. * This function returns false for all other inputs. */ static bool IsAlterExtensionSetSchemaCitus(Node *parseTree) { const char *extensionName = NULL; if (IsA(parseTree, AlterObjectSchemaStmt)) { AlterObjectSchemaStmt *alterExtensionSetSchemaStmt = (AlterObjectSchemaStmt *) parseTree; if (alterExtensionSetSchemaStmt->objectType == OBJECT_EXTENSION) { extensionName = strVal(((AlterObjectSchemaStmt *) parseTree)->object); /* * Now that we have AlterObjectSchemaStmt for an extension, * check if it is run for/on citus */ return (strncasecmp(extensionName, "citus", NAMEDATALEN) == 0); } } return false; } /* * PreprocessAlterExtensionCitusStmtForCitusColumnar pre-process the case when upgrade citus * to version that support citus_columnar, or downgrade citus to lower version that * include columnar inside citus extension */ void PreprocessAlterExtensionCitusStmtForCitusColumnar(Node *parseTree) { /*upgrade citus: alter extension citus update to 'xxx' */ DefElem *newVersionValue = GetExtensionOption( ((AlterExtensionStmt *) parseTree)->options, "new_version"); Oid citusColumnarOid = get_extension_oid("citus_columnar", true); if (newVersionValue) { char *newVersion = defGetString(newVersionValue); double newVersionNumber = GetExtensionVersionNumber(pstrdup(newVersion)); /*alter extension citus update to version >= 11.1-1, and no citus_columnar installed */ if (newVersionNumber >= 1110 && citusColumnarOid == InvalidOid && (newVersionNumber < 1320 || HasAnyRelationsUsingOldColumnar())) { /*it's upgrade citus to 11.1-1 or further version and there are relations using old columnar */ CreateExtensionWithVersion("citus_columnar", CITUS_COLUMNAR_INTERNAL_VERSION); } else if (newVersionNumber < 1110 && citusColumnarOid != InvalidOid) { /*downgrade citus, need downgrade citus_columnar to Y */ AlterExtensionUpdateStmt("citus_columnar", CITUS_COLUMNAR_INTERNAL_VERSION); } } else { /*alter extension citus update without specifying the version*/ int versionNumber = (int) (100 * strtod(CITUS_MAJORVERSION, NULL)); if (versionNumber >= 1110) { if (citusColumnarOid == InvalidOid && (versionNumber < 1320 || HasAnyRelationsUsingOldColumnar())) { CreateExtensionWithVersion("citus_columnar", CITUS_COLUMNAR_INTERNAL_VERSION); } } } } /* * HasAnyRelationsUsingOldColumnar returns true if there are any relations * using the old columnar access method. */ static bool HasAnyRelationsUsingOldColumnar(void) { Oid oldColumnarAMId = GetOldColumnarAMIdIfExists(); return OidIsValid(oldColumnarAMId) && HasAnyRelationsUsingAccessMethod(oldColumnarAMId); } /* * GetOldColumnarAMIdIfExists returns the oid of the old columnar access * method, i.e., the columnar access method that we had as part of "citus" * extension before we split it into "citus_columnar" at version 11.1, if * it exists. Otherwise, it returns InvalidOid. * * We know that it's "old columnar" only if the access method doesn't depend * on any extensions. This is because, in citus--11.0-4--11.1-1.sql, we * detach the columnar objects (including the access method) from citus * in preparation for splitting of the columnar into a separate extension. */ static Oid GetOldColumnarAMIdIfExists(void) { Oid columnarAMId = get_am_oid("columnar", true); if (OidIsValid(columnarAMId) && !AccessMethodDependsOnAnyExtensions(columnarAMId)) { return columnarAMId; } return InvalidOid; } /* * AccessMethodDependsOnAnyExtensions returns true if the access method * with the given accessMethodId depends on any extensions. */ static bool AccessMethodDependsOnAnyExtensions(Oid accessMethodId) { ScanKeyData key[3]; Relation pgDepend = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(AccessMethodRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(accessMethodId)); ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(0)); SysScanDesc scan = systable_beginscan(pgDepend, DependDependerIndexId, true, NULL, 3, key); bool result = false; HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scan))) { Form_pg_depend dependForm = (Form_pg_depend) GETSTRUCT(heapTuple); if (dependForm->refclassid == ExtensionRelationId) { result = true; break; } } systable_endscan(scan); table_close(pgDepend, AccessShareLock); return result; } /* * HasAnyRelationsUsingAccessMethod returns true if there are any relations * using the access method with the given accessMethodId. */ static bool HasAnyRelationsUsingAccessMethod(Oid accessMethodId) { ScanKeyData key[1]; Relation pgClass = table_open(RelationRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_class_relam, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(accessMethodId)); SysScanDesc scan = systable_beginscan(pgClass, InvalidOid, false, NULL, 1, key); bool result = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); table_close(pgClass, AccessShareLock); return result; } /* * PostprocessAlterExtensionCitusStmtForCitusColumnar process the case when upgrade citus * to version that support citus_columnar, or downgrade citus to lower version that * include columnar inside citus extension */ void PostprocessAlterExtensionCitusStmtForCitusColumnar(Node *parseTree) { DefElem *newVersionValue = GetExtensionOption( ((AlterExtensionStmt *) parseTree)->options, "new_version"); Oid citusColumnarOid = get_extension_oid("citus_columnar", true); if (newVersionValue) { char *newVersion = defGetString(newVersionValue); double newVersionNumber = GetExtensionVersionNumber(pstrdup(newVersion)); if (newVersionNumber >= 1110 && citusColumnarOid != InvalidOid) { /*upgrade citus, after "ALTER EXTENSION citus update to xxx" updates citus_columnar Y to version Z. */ char *curColumnarVersion = get_extension_version(citusColumnarOid); if (strcmp(curColumnarVersion, CITUS_COLUMNAR_INTERNAL_VERSION) == 0) { AlterExtensionUpdateStmt("citus_columnar", "11.1-1"); } } else if (newVersionNumber < 1110 && citusColumnarOid != InvalidOid) { /*downgrade citus, after "ALTER EXTENSION citus update to xxx" drops citus_columnar extension */ char *curColumnarVersion = get_extension_version(citusColumnarOid); if (strcmp(curColumnarVersion, CITUS_COLUMNAR_INTERNAL_VERSION) == 0) { RemoveExtensionById(citusColumnarOid); } } } else { /*alter extension citus update, need upgrade citus_columnar from Y to Z*/ int versionNumber = (int) (100 * strtod(CITUS_MAJORVERSION, NULL)); if (versionNumber >= 1110 && citusColumnarOid != InvalidOid) { char *curColumnarVersion = get_extension_version(citusColumnarOid); if (strcmp(curColumnarVersion, CITUS_COLUMNAR_INTERNAL_VERSION) == 0) { AlterExtensionUpdateStmt("citus_columnar", "11.1-1"); } } } } /* * CreateExtensionDDLCommand returns a list of DDL statements (const char *) to be * executed on a node to recreate the extension addressed by the extensionAddress. */ List * CreateExtensionDDLCommand(const ObjectAddress *extensionAddress) { /* generate a statement for creation of the extension in "if not exists" construct */ Node *stmt = RecreateExtensionStmt(extensionAddress->objectId); /* capture ddl command for the create statement */ const char *ddlCommand = DeparseTreeNode(stmt); List *ddlCommands = list_make1((void *) ddlCommand); /* any privilege granted on FDWs that belong to the extension should be included */ List *FDWGrants = GenerateGrantCommandsOnExtensionDependentFDWs(extensionAddress->objectId); ddlCommands = list_concat(ddlCommands, FDWGrants); return ddlCommands; } /* * RecreateExtensionStmt returns a parsetree for a CREATE EXTENSION statement that would * recreate the given extension on a new node. */ static Node * RecreateExtensionStmt(Oid extensionOid) { CreateExtensionStmt *createExtensionStmt = makeNode(CreateExtensionStmt); char *extensionName = get_extension_name(extensionOid); if (!extensionName) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("extension with oid %u does not exist", extensionOid))); } /* schema DefElement related variables */ /* set location to -1 as it is unknown */ int location = -1; /* set extension name and if_not_exists fields */ createExtensionStmt->extname = extensionName; createExtensionStmt->if_not_exists = true; /* get schema name that extension was created on */ Oid extensionSchemaOid = get_extension_schema(extensionOid); char *extensionSchemaName = get_namespace_name(extensionSchemaOid); /* make DefEleme for extensionSchemaName */ Node *schemaNameArg = (Node *) makeString(extensionSchemaName); DefElem *schemaDefElement = makeDefElem("schema", schemaNameArg, location); /* append the schema name DefElem finally */ createExtensionStmt->options = lappend(createExtensionStmt->options, schemaDefElement); char *extensionVersion = get_extension_version(extensionOid); if (extensionVersion != NULL) { Node *extensionVersionArg = (Node *) makeString(extensionVersion); DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg, location); createExtensionStmt->options = lappend(createExtensionStmt->options, extensionVersionElement); } return (Node *) createExtensionStmt; } /* * GenerateGrantCommandsOnExtensionDependentFDWs returns a list of commands that GRANTs * the privileges on FDWs that are depending on the given extension. */ static List * GenerateGrantCommandsOnExtensionDependentFDWs(Oid extensionId) { List *commands = NIL; List *FDWOids = GetDependentFDWsToExtension(extensionId); Oid FDWOid = InvalidOid; foreach_declared_oid(FDWOid, FDWOids) { Acl *aclEntry = GetPrivilegesForFDW(FDWOid); if (aclEntry == NULL) { continue; } AclItem *privileges = ACL_DAT(aclEntry); int numberOfPrivsGranted = ACL_NUM(aclEntry); for (int i = 0; i < numberOfPrivsGranted; i++) { commands = list_concat(commands, GenerateGrantOnFDWQueriesFromAclItem(FDWOid, &privileges[i])); } } return commands; } /* * GetDependentFDWsToExtension gets an extension oid and returns the list of oids of FDWs * that are depending on the given extension. */ List * GetDependentFDWsToExtension(Oid extensionId) { List *extensionFDWs = NIL; ScanKeyData key[1]; HeapTuple tup; Relation pgDepend = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ForeignDataWrapperRelationId)); SysScanDesc scan = systable_beginscan(pgDepend, DependDependerIndexId, true, NULL, lengthof(key), key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend pgDependEntry = (Form_pg_depend) GETSTRUCT(tup); if (pgDependEntry->deptype == DEPENDENCY_EXTENSION && pgDependEntry->refclassid == ExtensionRelationId && pgDependEntry->refobjid == extensionId) { extensionFDWs = lappend_oid(extensionFDWs, pgDependEntry->objid); } } systable_endscan(scan); table_close(pgDepend, AccessShareLock); return extensionFDWs; } /* * AlterExtensionSchemaStmtObjectAddress returns the ObjectAddress of the extension that is * the subject of the AlterObjectSchemaStmt. Errors if missing_ok is false. */ List * AlterExtensionSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_EXTENSION); const char *extensionName = strVal(stmt->object); Oid extensionOid = get_extension_oid(extensionName, missing_ok); if (extensionOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("extension \"%s\" does not exist", extensionName))); } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ExtensionRelationId, extensionOid); return list_make1(address); } /* * AlterExtensionUpdateStmtObjectAddress returns the ObjectAddress of the extension that is * the subject of the AlterExtensionStmt. Errors if missing_ok is false. */ List * AlterExtensionUpdateStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterExtensionStmt *stmt = castNode(AlterExtensionStmt, node); const char *extensionName = stmt->extname; Oid extensionOid = get_extension_oid(extensionName, missing_ok); if (extensionOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("extension \"%s\" does not exist", extensionName))); } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ExtensionRelationId, extensionOid); return list_make1(address); } /* * CreateExtensionWithVersion builds and execute create extension statements * per given extension name and extension verision */ void CreateExtensionWithVersion(char *extname, char *extVersion) { CreateExtensionStmt *createExtensionStmt = makeNode(CreateExtensionStmt); /* set location to -1 as it is unknown */ int location = -1; /* set extension name and if_not_exists fields */ createExtensionStmt->extname = extname; createExtensionStmt->if_not_exists = true; if (extVersion == NULL) { createExtensionStmt->options = NIL; } else { Node *extensionVersionArg = (Node *) makeString(extVersion); DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg, location); createExtensionStmt->options = lappend(createExtensionStmt->options, extensionVersionElement); } CreateExtension(NULL, createExtensionStmt); CommandCounterIncrement(); } /* * GetExtensionVersionNumber convert extension version to real value */ int GetExtensionVersionNumber(char *extVersion) { char *strtokPosition = NULL; char *versionVal = strtok_r(extVersion, "-", &strtokPosition); double versionNumber = strtod(versionVal, NULL); return (int) (versionNumber * 100); } /* * AlterExtensionUpdateStmt builds and execute Alter extension statements * per given extension name and updates extension verision */ void AlterExtensionUpdateStmt(char *extname, char *extVersion) { AlterExtensionStmt *alterExtensionStmt = makeNode(AlterExtensionStmt); /* set location to -1 as it is unknown */ int location = -1; alterExtensionStmt->extname = extname; if (extVersion == NULL) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("alter extension \"%s\" should not be empty", extVersion))); } Node *extensionVersionArg = (Node *) makeString(extVersion); DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg, location); alterExtensionStmt->options = lappend(alterExtensionStmt->options, extensionVersionElement); ExecAlterExtensionStmt(NULL, alterExtensionStmt); CommandCounterIncrement(); } ================================================ FILE: src/backend/distributed/commands/foreign_constraint.c ================================================ /*------------------------------------------------------------------------- * * foreign_constraint.c * * This file contains functions to create, alter and drop foreign * constraints on distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_type.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/coordinator_protocol.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/multi_join_order.h" #include "distributed/namespace_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" #define BehaviorIsRestrictOrNoAction(x) \ ((x) == FKCONSTR_ACTION_NOACTION || (x) == FKCONSTR_ACTION_RESTRICT) #define USE_CREATE_REFERENCE_TABLE_HINT \ "You could use SELECT create_reference_table('%s') " \ "to replicate the referenced table to all nodes or " \ "consider dropping the foreign key" typedef bool (*CheckRelationFunc)(Oid); /* Local functions forward declarations */ static void EnsureReferencingTableNotReplicated(Oid referencingTableId); static void EnsureSupportedFKeyOnDistKey(Form_pg_constraint constraintForm); static bool ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple); static List * ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple); static void EnsureSupportedFKeyBetweenCitusLocalAndRefTable(Form_pg_constraint constraintForm, char referencingReplicationModel, char referencedReplicationModel, Oid referencedTableId); static bool HeapTupleOfForeignConstraintIncludesColumn(HeapTuple heapTuple, Oid relationId, int pgConstraintKey, char *columnName); static Oid FindForeignKeyOidWithName(List *foreignKeyOids, const char *inputConstraintName); static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple, Var *referencingDistColumn, Var *referencedDistColumn, int *referencingAttrIndex, int *referencedAttrIndex); static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId, int searchForeignKeyColumnFlags); static List * GetForeignKeysWithLocalTables(Oid relationId); static bool IsTableTypeIncluded(Oid relationId, int flags); /* * ConstraintIsAForeignKeyToReferenceTable checks if the given constraint is a * foreign key constraint from the given relation to any reference table. */ bool ConstraintIsAForeignKeyToReferenceTable(char *inputConstaintName, Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_REFERENCE_TABLES; List *foreignKeyOids = GetForeignKeyOids(relationId, flags); Oid foreignKeyOid = FindForeignKeyOidWithName(foreignKeyOids, inputConstaintName); return OidIsValid(foreignKeyOid); } /* * EnsureNoFKeyFromTableType ensures that given relation is not referenced by any table specified * by table type flag. */ void EnsureNoFKeyFromTableType(Oid relationId, int tableTypeFlag) { int flags = INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | tableTypeFlag; List *referencedFKeyOids = GetForeignKeyOids(relationId, flags); if (list_length(referencedFKeyOids) > 0) { Oid referencingFKeyOid = linitial_oid(referencedFKeyOids); Oid referencingTableId = GetReferencingTableId(referencingFKeyOid); char *referencingRelName = get_rel_name(referencingTableId); char *referencedRelName = get_rel_name(relationId); char *referencingTableTypeName = GetTableTypeName(referencingTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s is referenced by a foreign key from %s", referencedRelName, referencingRelName), errdetail( "foreign keys from a %s to a distributed table are not supported.", referencingTableTypeName))); } } /* * EnsureNoFKeyToTableType ensures that given relation is not referencing any table specified * by table type flag. */ void EnsureNoFKeyToTableType(Oid relationId, int tableTypeFlag) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | tableTypeFlag; List *referencingFKeyOids = GetForeignKeyOids(relationId, flags); if (list_length(referencingFKeyOids) > 0) { Oid referencedFKeyOid = linitial_oid(referencingFKeyOids); Oid referencedTableId = GetReferencedTableId(referencedFKeyOid); char *referencedRelName = get_rel_name(referencedTableId); char *referencingRelName = get_rel_name(relationId); char *referencedTableTypeName = GetTableTypeName(referencedTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s is referenced by a foreign key from %s", referencedRelName, referencingRelName), errdetail( "foreign keys from a distributed table to a %s are not supported.", referencedTableTypeName))); } } /* * ErrorIfUnsupportedForeignConstraintExists runs checks related to foreign * constraints and errors out if it is not possible to create one of the * foreign constraint in distributed environment. * * To support foreign constraints, we require that; * - If referencing and referenced tables are hash-distributed * - Referencing and referenced tables are co-located. * - Foreign constraint is defined over distribution column. * - ON DELETE/UPDATE SET NULL, ON DELETE/UPDATE SET DEFAULT and * ON UPDATE CASCADE options * are not used. * - Replication factors of referencing and referenced table are 1. * - If referenced table is a reference table * - ON DELETE/UPDATE SET NULL, ON DELETE/UPDATE SET DEFAULT and * ON UPDATE CASCADE options are not used on the distribution key * of the referencing column. * - If referencing table is a reference table, error out if the referenced * table is a distributed table. * - If referencing table is a reference table and referenced table is a * citus local table: * - ON DELETE/UPDATE SET NULL, ON DELETE/UPDATE SET DEFAULT and * ON CASCADE options are not used. * - If referencing or referenced table is distributed table, then the * other table is not a citus local table. */ void ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDistMethod, char referencingReplicationModel, Var *referencingDistKey, uint32 referencingColocationId) { Oid referencingTableId = relation->rd_id; int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *foreignKeyOids = GetForeignKeyOids(referencingTableId, flags); Oid foreignKeyOid = InvalidOid; foreach_declared_oid(foreignKeyOid, foreignKeyOids) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyOid)); Assert(HeapTupleIsValid(heapTuple)); Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); int referencingAttrIndex = -1; Var *referencedDistKey = NULL; int referencedAttrIndex = -1; uint32 referencedColocationId = INVALID_COLOCATION_ID; Oid referencedTableId = constraintForm->confrelid; bool referencedIsCitus = IsCitusTable(referencedTableId); bool selfReferencingTable = (referencingTableId == referencedTableId); if (!referencedIsCitus && !selfReferencingTable) { if (IsCitusLocalTableByDistParams(referencingDistMethod, referencingReplicationModel, referencingColocationId)) { ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(referencedTableId); } char *referencedTableName = get_rel_name(referencedTableId); ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("referenced table \"%s\" must be a distributed table" " or a reference table", referencedTableName), errdetail("To enforce foreign keys, the referencing and " "referenced rows need to be stored on the same " "node."), errhint(USE_CREATE_REFERENCE_TABLE_HINT, referencedTableName))); } /* set referenced table related variables here if table is referencing itself */ char referencedDistMethod = 0; char referencedReplicationModel = REPLICATION_MODEL_INVALID; if (!selfReferencingTable) { referencedDistMethod = PartitionMethod(referencedTableId); referencedDistKey = !HasDistributionKey(referencedTableId) ? NULL : DistPartitionKey(referencedTableId); referencedColocationId = TableColocationId(referencedTableId); referencedReplicationModel = TableReplicationModel(referencedTableId); } else { referencedDistMethod = referencingDistMethod; referencedDistKey = referencingDistKey; referencedColocationId = referencingColocationId; referencedReplicationModel = referencingReplicationModel; } /* * Given that we drop DEFAULT nextval('sequence') expressions from * shard relation columns, allowing ON DELETE/UPDATE SET DEFAULT * on such columns causes inserting NULL values to referencing relation * as a result of a delete/update operation on referenced relation. * * For this reason, we disallow ON DELETE/UPDATE SET DEFAULT actions * on columns that default to sequences. */ if (ForeignKeySetsNextValColumnToDefault(heapTuple)) { ereport(ERROR, (errmsg("cannot create foreign key constraint " "since Citus does not support ON DELETE " "/ UPDATE SET DEFAULT actions on the " "columns that default to sequences"))); } bool referencingIsCitusLocalOrRefTable = IsCitusLocalTableByDistParams(referencingDistMethod, referencingReplicationModel, referencingColocationId) || IsReferenceTableByDistParams(referencingDistMethod, referencingReplicationModel); bool referencedIsCitusLocalOrRefTable = IsCitusLocalTableByDistParams(referencedDistMethod, referencedReplicationModel, referencedColocationId) || IsReferenceTableByDistParams(referencedDistMethod, referencedReplicationModel); if (referencingIsCitusLocalOrRefTable && referencedIsCitusLocalOrRefTable) { EnsureSupportedFKeyBetweenCitusLocalAndRefTable(constraintForm, referencingReplicationModel, referencedReplicationModel, referencedTableId); ReleaseSysCache(heapTuple); continue; } /* * Foreign keys from citus local tables or reference tables to distributed * tables are not supported. * * We could support foreign keys from references tables to single-shard * tables but this doesn't seem useful a lot. However, if we decide supporting * this, then we need to expand relation access tracking check for the single-shard * tables too. */ if (referencingIsCitusLocalOrRefTable && !referencedIsCitusLocalOrRefTable) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint " "since foreign keys from reference tables " "and local tables to distributed tables " "are not supported"), errdetail("Reference tables and local tables " "can only have foreign keys to reference " "tables and local tables"))); } /* * To enforce foreign constraints, tables must be co-located unless a * reference table is referenced. */ bool referencedIsReferenceTable = IsReferenceTableByDistParams(referencedDistMethod, referencedReplicationModel); if (!referencedIsReferenceTable && ( referencingColocationId == INVALID_COLOCATION_ID || referencingColocationId != referencedColocationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint since " "relations are not colocated or not referencing " "a reference table"), errdetail( "A distributed table can only have foreign keys " "if it is referencing another colocated hash " "distributed table or a reference table"))); } ForeignConstraintFindDistKeys(heapTuple, referencingDistKey, referencedDistKey, &referencingAttrIndex, &referencedAttrIndex); bool referencingColumnsIncludeDistKey = (referencingAttrIndex != -1); bool foreignConstraintOnDistKey = (referencingColumnsIncludeDistKey && referencingAttrIndex == referencedAttrIndex); /* * If columns in the foreign key includes the distribution key from the * referencing side, we do not allow update/delete operations through * foreign key constraints (e.g. ... ON UPDATE SET NULL) */ if (referencingColumnsIncludeDistKey) { EnsureSupportedFKeyOnDistKey(constraintForm); } /* * if tables are hash-distributed and colocated, we need to make sure that * the distribution key is included in foreign constraint. */ bool referencedIsSingleShardTable = IsSingleShardTableByDistParams(referencedDistMethod, referencedReplicationModel, referencedColocationId); if (!referencedIsCitusLocalOrRefTable && !referencedIsSingleShardTable && !foreignConstraintOnDistKey) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint"), errdetail("Foreign keys are supported in two cases, " "either in between two colocated tables including " "partition column in the same ordinal in the both " "tables or from distributed to reference tables"))); } /* * We do not allow to create foreign constraints if shard replication factor is * greater than 1. Because in our current design, multiple replicas may cause * locking problems and inconsistent shard contents. * * Note that we allow referenced table to be a reference table (e.g., not a * single replicated table). This is allowed since (a) we are sure that * placements always be in the same state (b) executors are aware of reference * tables and handle concurrency related issues accordingly. */ EnsureReferencingTableNotReplicated(referencingTableId); ReleaseSysCache(heapTuple); } } /* * ForeignKeySetsNextValColumnToDefault returns true if at least one of the * columns specified in ON DELETE / UPDATE SET DEFAULT clauses default to * nextval(). */ static bool ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple) { Form_pg_constraint pgConstraintForm = (Form_pg_constraint) GETSTRUCT(pgConstraintTuple); List *setDefaultAttrs = ForeignKeyGetDefaultingAttrs(pgConstraintTuple); AttrNumber setDefaultAttr = InvalidAttrNumber; foreach_declared_int(setDefaultAttr, setDefaultAttrs) { if (ColumnDefaultsToNextVal(pgConstraintForm->conrelid, setDefaultAttr)) { return true; } } return false; } /* * ForeignKeyGetDefaultingAttrs returns a list of AttrNumbers * might be set to default ON DELETE or ON UPDATE. * * For example; if the foreign key has SET DEFAULT clause for * both actions, then returns a superset of the attributes that * might be set to DEFAULT on either of those actions. */ static List * ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple) { bool isNull = false; Datum referencingColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple, Anum_pg_constraint_conkey, &isNull); if (isNull) { ereport(ERROR, (errmsg("got NULL conkey from catalog"))); } List *referencingColumns = IntegerArrayTypeToList(DatumGetArrayTypeP(referencingColumnsDatum)); Form_pg_constraint pgConstraintForm = (Form_pg_constraint) GETSTRUCT(pgConstraintTuple); if (pgConstraintForm->confupdtype == FKCONSTR_ACTION_SETDEFAULT) { /* * Postgres doesn't allow specifying SET DEFAULT for a subset of * the referencing columns for ON UPDATE action, so in that case * we return all referencing columns regardless of what ON DELETE * action says. */ return referencingColumns; } if (pgConstraintForm->confdeltype != FKCONSTR_ACTION_SETDEFAULT) { return NIL; } List *onDeleteSetDefColumnList = NIL; Datum onDeleteSetDefColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple, Anum_pg_constraint_confdelsetcols, &isNull); /* * confdelsetcols being NULL means that "ON DELETE SET DEFAULT" doesn't * specify which subset of columns should be set to DEFAULT, so fetching * NULL from the catalog is also possible. */ if (!isNull) { onDeleteSetDefColumnList = IntegerArrayTypeToList(DatumGetArrayTypeP(onDeleteSetDefColumnsDatum)); } if (list_length(onDeleteSetDefColumnList) == 0) { /* * That means that all referencing columns need to be set to * DEFAULT. */ return referencingColumns; } else { return onDeleteSetDefColumnList; } } /* * EnsureSupportedFKeyBetweenCitusLocalAndRefTable is a helper function that * takes a foreign key constraint form for a foreign key between two citus * tables that are either citus local table or reference table and errors * out if it it an unsupported foreign key from a reference table to a citus * local table according to given replication model parameters. */ static void EnsureSupportedFKeyBetweenCitusLocalAndRefTable(Form_pg_constraint fKeyConstraintForm, char referencingReplicationModel, char referencedReplicationModel, Oid referencedTableId) { bool referencingIsReferenceTable = (referencingReplicationModel == REPLICATION_MODEL_2PC); bool referencedIsCitusLocalTable = (referencedReplicationModel != REPLICATION_MODEL_2PC); if (referencingIsReferenceTable && referencedIsCitusLocalTable) { /* * We only support RESTRICT and NO ACTION behaviors for the * foreign keys from reference tables to citus local tables. * This is because, we can't cascade dml operations from citus * local tables's coordinator placement to the remote placements * of the reference table. * Note that for the foreign keys from citus local tables to * reference tables, we support all foreign key behaviors. */ if (!(BehaviorIsRestrictOrNoAction(fKeyConstraintForm->confdeltype) && BehaviorIsRestrictOrNoAction(fKeyConstraintForm->confupdtype))) { char *referencedTableName = get_rel_name(referencedTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot define foreign key constraint, " "foreign keys from reference tables to " "local tables can only be defined " "with NO ACTION or RESTRICT behaviors"), errhint(USE_CREATE_REFERENCE_TABLE_HINT, referencedTableName))); } } } /* * EnsureSupportedFKeyOnDistKey errors out if given foreign key constraint form * implies an unsupported ON DELETE/UPDATE behavior assuming the referencing column * is the distribution column of the referencing distributed table. */ static void EnsureSupportedFKeyOnDistKey(Form_pg_constraint fKeyConstraintForm) { /* * ON DELETE SET NULL and ON DELETE SET DEFAULT is not supported. Because we do * not want to set partition column to NULL or default value. */ if (fKeyConstraintForm->confdeltype == FKCONSTR_ACTION_SETNULL || fKeyConstraintForm->confdeltype == FKCONSTR_ACTION_SETDEFAULT) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint"), errdetail("SET NULL or SET DEFAULT is not supported " "in ON DELETE operation when distribution " "key is included in the foreign key constraint"))); } /* * ON UPDATE SET NULL, ON UPDATE SET DEFAULT and UPDATE CASCADE is not supported. * Because we do not want to set partition column to NULL or default value. Also * cascading update operation would require re-partitioning. Updating partition * column value is not allowed anyway even outside of foreign key concept. */ if (fKeyConstraintForm->confupdtype == FKCONSTR_ACTION_SETNULL || fKeyConstraintForm->confupdtype == FKCONSTR_ACTION_SETDEFAULT || fKeyConstraintForm->confupdtype == FKCONSTR_ACTION_CASCADE) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint"), errdetail("SET NULL, SET DEFAULT or CASCADE is not " "supported in ON UPDATE operation when " "distribution key included in the foreign " "constraint."))); } } /* * EnsureReferencingTableNotReplicated takes referencingTableId for the * referencing table of the foreign key and errors out if it's not a single * replicated table. */ static void EnsureReferencingTableNotReplicated(Oid referencingTableId) { bool referencingNotReplicated = true; bool referencingIsCitus = IsCitusTable(referencingTableId); if (referencingIsCitus) { /* ALTER TABLE command is applied over single replicated table */ referencingNotReplicated = SingleReplicatedTable(referencingTableId); } else { /* Creating single replicated table with foreign constraint */ referencingNotReplicated = !DistributedTableReplicationIsEnabled(); } if (!referencingNotReplicated) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint"), errdetail("Citus currently supports foreign key constraints " "only for \"citus.shard_replication_factor = 1\"."), errhint("Please change \"citus.shard_replication_factor to " "1\". To learn more about using foreign keys with " "other replication factors, please contact us at " "https://citusdata.com/about/contact_us."))); } } /* * ErrorOutForFKeyBetweenPostgresAndCitusLocalTable is a helper function to * error out for foreign keys between postgres local tables and citus local * tables. */ void ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(Oid localTableId) { char *localTableName = get_rel_name(localTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create foreign key constraint as \"%s\" is " "a postgres local table", localTableName), errhint("first add local table to citus metadata " "by using SELECT citus_add_local_table_to_metadata('%s') " "and execute the ALTER TABLE command to create the " "foreign key to local table", localTableName))); } /* * ForeignConstraintFindDistKeys finds the index of the given distribution columns * in the given foreign key constraint and returns them in referencingAttrIndex * and referencedAttrIndex. If one of them is not found, it returns -1 instead. */ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple, Var *referencingDistColumn, Var *referencedDistColumn, int *referencingAttrIndex, int *referencedAttrIndex) { Datum *referencingColumnArray = NULL; int referencingColumnCount = 0; Datum *referencedColumnArray = NULL; int referencedColumnCount = 0; bool isNull = false; *referencedAttrIndex = -1; *referencedAttrIndex = -1; /* * Column attributes are not available in Form_pg_constraint, therefore we need * to find them in the system catalog. After finding them, we iterate over column * attributes together because partition column must be at the same place in both * referencing and referenced side of the foreign key constraint. */ Datum referencingColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple, Anum_pg_constraint_conkey, &isNull); Datum referencedColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple, Anum_pg_constraint_confkey, &isNull); deconstruct_array(DatumGetArrayTypeP(referencingColumnsDatum), INT2OID, 2, true, 's', &referencingColumnArray, NULL, &referencingColumnCount); deconstruct_array(DatumGetArrayTypeP(referencedColumnsDatum), INT2OID, 2, true, 's', &referencedColumnArray, NULL, &referencedColumnCount); Assert(referencingColumnCount == referencedColumnCount); for (int attrIdx = 0; attrIdx < referencingColumnCount; ++attrIdx) { AttrNumber referencingAttrNo = DatumGetInt16(referencingColumnArray[attrIdx]); AttrNumber referencedAttrNo = DatumGetInt16(referencedColumnArray[attrIdx]); if (referencedDistColumn != NULL && referencedDistColumn->varattno == referencedAttrNo) { *referencedAttrIndex = attrIdx; } if (referencingDistColumn != NULL && referencingDistColumn->varattno == referencingAttrNo) { *referencingAttrIndex = attrIdx; } } } /* * ColumnAppearsInForeignKey returns true if there is a foreign key constraint * from/to given column. False otherwise. */ bool ColumnAppearsInForeignKey(char *columnName, Oid relationId) { int searchForeignKeyColumnFlags = SEARCH_REFERENCING_RELATION | SEARCH_REFERENCED_RELATION; List *foreignKeysColumnAppeared = GetForeignKeyIdsForColumn(columnName, relationId, searchForeignKeyColumnFlags); return list_length(foreignKeysColumnAppeared) > 0; } /* * ColumnAppearsInForeignKeyToReferenceTable checks if there is a foreign key * constraint from/to any reference table on the given column. */ bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId) { int searchForeignKeyColumnFlags = SEARCH_REFERENCING_RELATION | SEARCH_REFERENCED_RELATION; List *foreignKeyIdsColumnAppeared = GetForeignKeyIdsForColumn(columnName, relationId, searchForeignKeyColumnFlags); Oid foreignKeyId = InvalidOid; foreach_declared_oid(foreignKeyId, foreignKeyIdsColumnAppeared) { Oid referencedTableId = GetReferencedTableId(foreignKeyId); if (IsCitusTableType(referencedTableId, REFERENCE_TABLE)) { return true; } } return false; } /* * GetForeignKeyIdsForColumn takes columnName and relationId for the owning * relation, and returns a list of OIDs for foreign constraints that the column * with columnName is involved according to "searchForeignKeyColumnFlags" argument. * See SearchForeignKeyColumnFlags enum definition for usage. */ static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId, int searchForeignKeyColumnFlags) { bool searchReferencing = searchForeignKeyColumnFlags & SEARCH_REFERENCING_RELATION; bool searchReferenced = searchForeignKeyColumnFlags & SEARCH_REFERENCED_RELATION; /* at least one of them should be true */ Assert(searchReferencing || searchReferenced); List *foreignKeyIdsColumnAppeared = NIL; ScanKeyData scanKey[1]; int scanKeyCount = 1; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, InvalidOid, false, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { int pgConstraintKey = 0; Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); Oid referencedTableId = constraintForm->confrelid; Oid referencingTableId = constraintForm->conrelid; if (referencedTableId == relationId && searchReferenced) { pgConstraintKey = Anum_pg_constraint_confkey; } else if (referencingTableId == relationId && searchReferencing) { pgConstraintKey = Anum_pg_constraint_conkey; } else { /* * If the constraint is not from/to the given relation, we should simply * skip. */ heapTuple = systable_getnext(scanDescriptor); continue; } if (HeapTupleOfForeignConstraintIncludesColumn(heapTuple, relationId, pgConstraintKey, columnName)) { foreignKeyIdsColumnAppeared = lappend_oid(foreignKeyIdsColumnAppeared, constraintForm->oid); } heapTuple = systable_getnext(scanDescriptor); } /* clean up scan and close system catalog */ systable_endscan(scanDescriptor); table_close(pgConstraint, NoLock); return foreignKeyIdsColumnAppeared; } /* * GetReferencingForeignConstaintCommands takes in a relationId, and * returns the list of foreign constraint commands needed to reconstruct * foreign key constraints that the table is involved in as the "referencing" * one. */ List * GetReferencingForeignConstaintCommands(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; return GetForeignConstraintCommandsInternal(relationId, flags); } /* * GetForeignConstraintToReferenceTablesCommands takes in a relationId, and * returns the list of foreign constraint commands needed to reconstruct * foreign key constraints that the table is involved in as the "referencing" * one and the "referenced" table is a reference table. */ List * GetForeignConstraintToReferenceTablesCommands(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_REFERENCE_TABLES; return GetForeignConstraintCommandsInternal(relationId, flags); } /* * GetForeignConstraintToReferenceTablesCommands takes in a relationId, and * returns the list of foreign constraint commands needed to reconstruct * foreign key constraints that the table is involved in as the "referenced" * one and the "referencing" table is a reference table. */ List * GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId) { int flags = INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | INCLUDE_REFERENCE_TABLES; return GetForeignConstraintCommandsInternal(relationId, flags); } /* * GetForeignConstraintToDistributedTablesCommands takes in a relationId, and * returns the list of foreign constraint commands needed to reconstruct * foreign key constraints that the table is involved in as the "referencing" * one and the "referenced" table is a distributed table. */ List * GetForeignConstraintToDistributedTablesCommands(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; return GetForeignConstraintCommandsInternal(relationId, flags); } /* * GetForeignConstraintFromDistributedTablesCommands takes in a relationId, and * returns the list of foreign constraint commands needed to reconstruct * foreign key constraints that the table is involved in as the "referenced" * one and the "referencing" table is a distributed table. */ List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId) { int flags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_DISTRIBUTED_TABLES; return GetForeignConstraintCommandsInternal(relationId, flags); } /* * GetForeignConstraintCommandsInternal is a wrapper function to get the * DDL commands to recreate the foreign key constraints returned by * GetForeignKeyOids. See more details at the underlying function. */ List * GetForeignConstraintCommandsInternal(Oid relationId, int flags) { List *foreignKeyOids = GetForeignKeyOids(relationId, flags); List *foreignKeyCommands = NIL; int saveNestLevel = PushEmptySearchPath(); Oid foreignKeyOid = InvalidOid; foreach_declared_oid(foreignKeyOid, foreignKeyOids) { char *statementDef = pg_get_constraintdef_command(foreignKeyOid); foreignKeyCommands = lappend(foreignKeyCommands, statementDef); } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return foreignKeyCommands; } /* * GetFKeyCreationCommandsRelationInvolvedWithTableType returns a list of DDL * commands to recreate the foreign keys that relation with relationId is involved * with given table type. */ List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag) { int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS | tableTypeFlag; List *referencingFKeyCreationCommands = GetForeignConstraintCommandsInternal(relationId, referencingFKeysFlag); /* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */ int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | tableTypeFlag; List *referencedFKeyCreationCommands = GetForeignConstraintCommandsInternal(relationId, referencedFKeysFlag); return list_concat(referencingFKeyCreationCommands, referencedFKeyCreationCommands); } /* * DropFKeysRelationInvolvedWithTableType drops foreign keys that relation * with relationId is involved with given table type. */ void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag) { int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS | tableTypeFlag; DropRelationForeignKeys(relationId, referencingFKeysFlag); /* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */ int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | tableTypeFlag; DropRelationForeignKeys(relationId, referencedFKeysFlag); } /* * HasForeignKeyWithLocalTable returns true if relation has foreign key * relationship with a local table. */ bool HasForeignKeyWithLocalTable(Oid relationId) { List *foreignKeysWithLocalTables = GetForeignKeysWithLocalTables(relationId); return list_length(foreignKeysWithLocalTables) > 0; } /* * GetForeignKeysWithLocalTables returns a list of foreign keys for foreign key * relationships that relation has with local tables. */ static List * GetForeignKeysWithLocalTables(Oid relationId) { int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_LOCAL_TABLES; List *referencingFKeyList = GetForeignKeyOids(relationId, referencingFKeysFlag); /* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */ int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | INCLUDE_LOCAL_TABLES; List *referencedFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag); return list_concat(referencingFKeyList, referencedFKeyList); } /* * GetForeignKeysFromLocalTables returns a list of foreign keys where the referencing * relation is a local table. */ List * GetForeignKeysFromLocalTables(Oid relationId) { int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_LOCAL_TABLES; List *referencingFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag); return referencingFKeyList; } /* * HasForeignKeyToReferenceTable returns true if any of the foreign key * constraints on the relation with relationId references to a reference * table. */ bool HasForeignKeyToReferenceTable(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_REFERENCE_TABLES; List *foreignKeyOids = GetForeignKeyOids(relationId, flags); return list_length(foreignKeyOids) > 0; } /* * TableReferenced function checks whether given table is referenced by another table * via foreign constraints. If it is referenced, this function returns true. */ bool TableReferenced(Oid relationId) { int flags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *foreignKeyOids = GetForeignKeyOids(relationId, flags); return list_length(foreignKeyOids) > 0; } /* * HeapTupleOfForeignConstraintIncludesColumn fetches the columns from the foreign * constraint and checks if the given column name matches one of them. */ static bool HeapTupleOfForeignConstraintIncludesColumn(HeapTuple heapTuple, Oid relationId, int pgConstraintKey, char *columnName) { Datum *columnArray = NULL; int columnCount = 0; bool isNull = false; Datum columnsDatum = SysCacheGetAttr(CONSTROID, heapTuple, pgConstraintKey, &isNull); deconstruct_array(DatumGetArrayTypeP(columnsDatum), INT2OID, 2, true, 's', &columnArray, NULL, &columnCount); for (int attrIdx = 0; attrIdx < columnCount; ++attrIdx) { AttrNumber attrNo = DatumGetInt16(columnArray[attrIdx]); char *colName = get_attname(relationId, attrNo, false); if (strncmp(colName, columnName, NAMEDATALEN) == 0) { return true; } } return false; } /* * TableReferencing function checks whether given table is referencing to another * table via foreign key constraints. If it is referencing, this function returns * true. */ bool TableReferencing(Oid relationId) { int flags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *foreignKeyOids = GetForeignKeyOids(relationId, flags); return list_length(foreignKeyOids) > 0; } /* * ConstraintIsAUniquenessConstraint is a wrapper around ConstraintWithNameIsOfType * that returns true if given constraint name identifies a uniqueness constraint, i.e: * - primary key constraint, or * - unique constraint */ bool ConstraintIsAUniquenessConstraint(char *inputConstaintName, Oid relationId) { bool isUniqueConstraint = ConstraintWithNameIsOfType(inputConstaintName, relationId, CONSTRAINT_UNIQUE); bool isPrimaryConstraint = ConstraintWithNameIsOfType(inputConstaintName, relationId, CONSTRAINT_PRIMARY); return isUniqueConstraint || isPrimaryConstraint; } /* * ConstraintIsAForeignKey is a wrapper around ConstraintWithNameIsOfType that returns true * if given constraint name identifies a foreign key constraint. */ bool ConstraintIsAForeignKey(char *inputConstaintName, Oid relationId) { return ConstraintWithNameIsOfType(inputConstaintName, relationId, CONSTRAINT_FOREIGN); } /* * ConstraintWithNameIsOfType is a wrapper around get_relation_constraint_oid that * returns true if given constraint name identifies a valid constraint defined * on relation with relationId and its type matches the input constraint type. */ bool ConstraintWithNameIsOfType(char *inputConstaintName, Oid relationId, char targetConstraintType) { bool missingOk = true; Oid constraintId = get_relation_constraint_oid(relationId, inputConstaintName, missingOk); return ConstraintWithIdIsOfType(constraintId, targetConstraintType); } /* * ConstraintWithIdIsOfType returns true if constraint with constraintId exists * and is of type targetConstraintType. */ bool ConstraintWithIdIsOfType(Oid constraintId, char targetConstraintType) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(constraintId)); if (!HeapTupleIsValid(heapTuple)) { /* no such constraint */ return false; } Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); char constraintType = constraintForm->contype; bool constraintTypeMatches = (constraintType == targetConstraintType); ReleaseSysCache(heapTuple); return constraintTypeMatches; } /* * FindForeignKeyOidWithName searches the foreign key constraint with * inputConstraintName in the given list of foreign key constraint OIDs. * Returns the OID of the matching constraint. If there no matching constraint * in the given list, then returns InvalidOid. */ static Oid FindForeignKeyOidWithName(List *foreignKeyOids, const char *inputConstraintName) { Oid foreignKeyOid = InvalidOid; foreach_declared_oid(foreignKeyOid, foreignKeyOids) { char *constraintName = get_constraint_name(foreignKeyOid); Assert(constraintName != NULL); if (strncmp(constraintName, inputConstraintName, NAMEDATALEN) == 0) { return foreignKeyOid; } } return InvalidOid; } /* * TableHasExternalForeignKeys returns true if the relation with relationId is * involved in a foreign key relationship other than the self-referencing ones. */ bool TableHasExternalForeignKeys(Oid relationId) { int flags = (INCLUDE_REFERENCING_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | INCLUDE_ALL_TABLE_TYPES); List *foreignKeyIdsTableReferencing = GetForeignKeyOids(relationId, flags); flags = (INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | INCLUDE_ALL_TABLE_TYPES); List *foreignKeyIdsTableReferenced = GetForeignKeyOids(relationId, flags); List *foreignKeysWithOtherTables = list_concat(foreignKeyIdsTableReferencing, foreignKeyIdsTableReferenced); if (list_length(foreignKeysWithOtherTables) == 0) { return false; } return true; } /* * ForeignConstraintMatchesFlags is a function with logic that's very specific * to GetForeignKeyOids. There's no reason to use it in any other context. */ static bool ForeignConstraintMatchesFlags(Form_pg_constraint constraintForm, int flags) { if (constraintForm->contype != CONSTRAINT_FOREIGN) { return false; } bool inheritedConstraint = OidIsValid(constraintForm->conparentid); if (inheritedConstraint) { /* * We only consider the constraints that are explicitly created on * the table as we already process the constraints from parent tables * implicitly when a command is issued */ return false; } bool excludeSelfReference = (flags & EXCLUDE_SELF_REFERENCES); bool isSelfReference = (constraintForm->conrelid == constraintForm->confrelid); if (excludeSelfReference && isSelfReference) { return false; } Oid otherTableId = InvalidOid; if (flags & INCLUDE_REFERENCING_CONSTRAINTS) { otherTableId = constraintForm->confrelid; } else { otherTableId = constraintForm->conrelid; } return IsTableTypeIncluded(otherTableId, flags); } /* * GetForeignKeyOidsForReferencedTable returns a list of foreign key OIDs that * reference the relationId and match the given flags. * * This is separated from GetForeignKeyOids because we need to scan pg_depend * instead of pg_constraint directly. The reason for this is that there is no * index on the confrelid of pg_constraint, so searching by that column * requires a seqscan. */ static List * GetForeignKeyOidsForReferencedTable(Oid relationId, int flags) { HTAB *foreignKeyOidsSet = CreateSimpleHashSetWithName( Oid, "ReferencingForeignKeyOidsSet"); List *foreignKeyOidsList = NIL; ScanKeyData key[2]; HeapTuple dependTup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, lengthof(key), key); while (HeapTupleIsValid(dependTup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(dependTup); if (deprec->classid != ConstraintRelationId || deprec->deptype != DEPENDENCY_NORMAL || hash_search(foreignKeyOidsSet, &deprec->objid, HASH_FIND, NULL)) { continue; } HeapTuple constraintTup = SearchSysCache1(CONSTROID, ObjectIdGetDatum( deprec->objid)); if (!HeapTupleIsValid(constraintTup)) /* can happen during DROP TABLE */ { continue; } Form_pg_constraint constraint = (Form_pg_constraint) GETSTRUCT(constraintTup); if (constraint->confrelid == relationId && ForeignConstraintMatchesFlags(constraint, flags)) { foreignKeyOidsList = lappend_oid(foreignKeyOidsList, constraint->oid); hash_search(foreignKeyOidsSet, &constraint->oid, HASH_ENTER, NULL); } ReleaseSysCache(constraintTup); } systable_endscan(scan); table_close(depRel, AccessShareLock); return foreignKeyOidsList; } /* * GetForeignKeyOids takes in a relationId, and returns a list of OIDs for * foreign constraints that the relation with relationId is involved according * to "flags" argument. See ExtractForeignKeyConstraintsMode enum definition * for usage of the flags. */ List * GetForeignKeyOids(Oid relationId, int flags) { bool extractReferencing PG_USED_FOR_ASSERTS_ONLY = (flags & INCLUDE_REFERENCING_CONSTRAINTS); bool extractReferenced = (flags & INCLUDE_REFERENCED_CONSTRAINTS); /* * Only one of them should be passed at a time since the way we scan * pg_constraint differs for those columns. Anum_pg_constraint_conrelid * supports index scan while Anum_pg_constraint_confrelid does not. */ Assert(!(extractReferencing && extractReferenced)); Assert(extractReferencing || extractReferenced); if (extractReferenced) { return GetForeignKeyOidsForReferencedTable(relationId, flags); } List *foreignKeyOids = NIL; ScanKeyData scanKey[1]; int scanKeyCount = 1; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, ConstraintRelidTypidNameIndexId, true, NULL, scanKeyCount, scanKey); HeapTuple heapTuple; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); if (ForeignConstraintMatchesFlags(constraintForm, flags)) { foreignKeyOids = lappend_oid(foreignKeyOids, constraintForm->oid); } } systable_endscan(scanDescriptor); /* * Do not release AccessShareLock yet to prevent modifications to be done * on pg_constraint to make sure that caller will process valid foreign key * constraints through the transaction. */ table_close(pgConstraint, NoLock); return foreignKeyOids; } /* * GetReferencedTableId returns OID of the referenced relation for the foreign * key with foreignKeyId. If there is no such foreign key, then this function * returns InvalidOid. */ Oid GetReferencedTableId(Oid foreignKeyId) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyId)); if (!HeapTupleIsValid(heapTuple)) { /* no such foreign key */ return InvalidOid; } Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); Oid referencedTableId = constraintForm->confrelid; ReleaseSysCache(heapTuple); return referencedTableId; } /* * GetReferencingTableId returns OID of the referencing relation for the foreign * key with foreignKeyId. If there is no such foreign key, then this function * returns InvalidOid. */ Oid GetReferencingTableId(Oid foreignKeyId) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyId)); if (!HeapTupleIsValid(heapTuple)) { /* no such foreign key */ return InvalidOid; } Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); Oid referencingTableId = constraintForm->conrelid; ReleaseSysCache(heapTuple); return referencingTableId; } /* * IsTableTypeIncluded returns true if type of the table with relationId (distributed, * reference, Citus local or Postgres local) is included in the flags, false if not */ static bool IsTableTypeIncluded(Oid relationId, int flags) { if (!IsCitusTable(relationId)) { return (flags & INCLUDE_LOCAL_TABLES) != 0; } else if (IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { return (flags & INCLUDE_SINGLE_SHARD_TABLES) != 0; } else if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { return (flags & INCLUDE_DISTRIBUTED_TABLES) != 0; } else if (IsCitusTableType(relationId, REFERENCE_TABLE)) { return (flags & INCLUDE_REFERENCE_TABLES) != 0; } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { return (flags & INCLUDE_CITUS_LOCAL_TABLES) != 0; } return false; } /* * RelationInvolvedInAnyNonInheritedForeignKeys returns true if relation involved * in a foreign key that is not inherited from its parent relation. */ bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId) { List *referencingForeignKeys = GetForeignKeyOids(relationId, INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES); /* * We already capture self-referencing foreign keys above, so use * EXCLUDE_SELF_REFERENCES here */ List *referencedForeignKeys = GetForeignKeyOids(relationId, INCLUDE_REFERENCED_CONSTRAINTS | EXCLUDE_SELF_REFERENCES | INCLUDE_ALL_TABLE_TYPES); List *foreignKeysRelationInvolved = list_concat(referencingForeignKeys, referencedForeignKeys); Oid foreignKeyId = InvalidOid; foreach_declared_oid(foreignKeyId, foreignKeysRelationInvolved) { HeapTuple heapTuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(foreignKeyId)); if (!HeapTupleIsValid(heapTuple)) { /* not possible but be on the safe side */ continue; } Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); Oid parentConstraintId = constraintForm->conparentid; if (!OidIsValid(parentConstraintId)) { return true; } } return false; } ================================================ FILE: src/backend/distributed/commands/foreign_data_wrapper.c ================================================ /*------------------------------------------------------------------------- * * foreign_data_wrapper.c * Commands for FOREIGN DATA WRAPPER statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_foreign_data_wrapper.h" #include "foreign/foreign.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" static bool NameListHasFDWOwnedByDistributedExtension(List *FDWNames); static ObjectAddress GetObjectAddressByFDWName(char *FDWName, bool missing_ok); /* * PreprocessGrantOnFDWStmt is executed before the statement is applied to the * local postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on foreign data wrappers. */ List * PreprocessGrantOnFDWStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_FDW); if (!NameListHasFDWOwnedByDistributedExtension(stmt->objects)) { /* * We propagate granted privileges on a FDW only if it belongs to a distributed * extension. For now, we skip for custom FDWs, as most of the users prefer * extension FDWs. */ return NIL; } if (list_length(stmt->objects) > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot grant on FDW with other FDWs"), errhint("Try granting on each object in separate commands"))); } if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); /* the code-path only supports a single object */ Assert(list_length(stmt->objects) == 1); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * NameListHasFDWOwnedByDistributedExtension takes a namelist of FDWs and returns true * if at least one of them depends on a distributed extension. Returns false otherwise. */ static bool NameListHasFDWOwnedByDistributedExtension(List *FDWNames) { String *FDWValue = NULL; foreach_declared_ptr(FDWValue, FDWNames) { /* captures the extension address during lookup */ ObjectAddress *extensionAddress = palloc0(sizeof(ObjectAddress)); ObjectAddress FDWAddress = GetObjectAddressByFDWName(strVal(FDWValue), false); ObjectAddress *copyFDWAddress = palloc0(sizeof(ObjectAddress)); *copyFDWAddress = FDWAddress; if (IsAnyObjectAddressOwnedByExtension(list_make1(copyFDWAddress), extensionAddress)) { if (IsAnyObjectDistributed(list_make1(extensionAddress))) { return true; } } } return false; } /* * GetObjectAddressByFDWName takes a FDW name and returns the object address. */ static ObjectAddress GetObjectAddressByFDWName(char *FDWName, bool missing_ok) { ForeignDataWrapper *FDW = GetForeignDataWrapperByName(FDWName, missing_ok); Oid FDWId = FDW->fdwid; ObjectAddress address = { 0 }; ObjectAddressSet(address, ForeignDataWrapperRelationId, FDWId); return address; } /* * GetPrivilegesForFDW takes a FDW object id and returns the privileges granted * on that FDW as a Acl object. Returns NULL if there is no privilege granted. */ Acl * GetPrivilegesForFDW(Oid FDWOid) { HeapTuple fdwtup = SearchSysCache1(FOREIGNDATAWRAPPEROID, ObjectIdGetDatum(FDWOid)); bool isNull = true; Datum aclDatum = SysCacheGetAttr(FOREIGNDATAWRAPPEROID, fdwtup, Anum_pg_foreign_data_wrapper_fdwacl, &isNull); if (isNull) { ReleaseSysCache(fdwtup); return NULL; } Acl *aclEntry = DatumGetAclPCopy(aclDatum); ReleaseSysCache(fdwtup); return aclEntry; } ================================================ FILE: src/backend/distributed/commands/foreign_server.c ================================================ /*------------------------------------------------------------------------- * * foreign_server.c * Commands for FOREIGN SERVER statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/pg_foreign_server.h" #include "foreign/foreign.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "utils/builtins.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" static char * GetForeignServerAlterOwnerCommand(Oid serverId); static Node * RecreateForeignServerStmt(Oid serverId); static bool NameListHasDistributedServer(List *serverNames); static List * GetObjectAddressByServerName(char *serverName, bool missing_ok); /* * CreateForeignServerStmtObjectAddress finds the ObjectAddress for the server * that is created by given CreateForeignServerStmt. If missingOk is false and if * the server does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateForeignServerStmt *stmt = castNode(CreateForeignServerStmt, node); return GetObjectAddressByServerName(stmt->servername, missing_ok); } /* * AlterForeignServerStmtObjectAddress finds the ObjectAddress for the server that is * changed by given AlterForeignServerStmt. If missingOk is false and if * the server does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterForeignServerStmt *stmt = castNode(AlterForeignServerStmt, node); return GetObjectAddressByServerName(stmt->servername, missing_ok); } /* * PreprocessGrantOnForeignServerStmt is executed before the statement is applied to the * local postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on servers. */ List * PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_FOREIGN_SERVER); bool includesDistributedServer = NameListHasDistributedServer(stmt->objects); if (!includesDistributedServer) { return NIL; } if (list_length(stmt->objects) > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot grant on distributed server with other servers"), errhint("Try granting on each object in separate commands"))); } if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); /* the code-path only supports a single object */ Assert(list_length(stmt->objects) == 1); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * RenameForeignServerStmtObjectAddress finds the ObjectAddress for the server that is * renamed by given RenmaeStmt. If missingOk is false and if the server does not exist, * then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_FOREIGN_SERVER); return GetObjectAddressByServerName(strVal(stmt->object), missing_ok); } /* * AlterForeignServerOwnerStmtObjectAddress finds the ObjectAddress for the server * given in AlterOwnerStmt. If missingOk is false and if * the server does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); char *serverName = strVal(stmt->object); return GetObjectAddressByServerName(serverName, missing_ok); } /* * GetForeignServerCreateDDLCommand returns a list that includes the CREATE SERVER * command that would recreate the given server on a new node. */ List * GetForeignServerCreateDDLCommand(Oid serverId) { /* generate a statement for creation of the server in "if not exists" construct */ Node *stmt = RecreateForeignServerStmt(serverId); /* capture ddl command for the create statement */ const char *createCommand = DeparseTreeNode(stmt); const char *alterOwnerCommand = GetForeignServerAlterOwnerCommand(serverId); List *ddlCommands = list_make2((void *) createCommand, (void *) alterOwnerCommand); return ddlCommands; } /* * GetForeignServerAlterOwnerCommand returns "ALTER SERVER .. OWNER TO .." statement * for the specified foreign server. */ static char * GetForeignServerAlterOwnerCommand(Oid serverId) { ForeignServer *server = GetForeignServer(serverId); Oid ownerId = server->owner; char *ownerName = GetUserNameFromId(ownerId, false); StringInfo alterCommand = makeStringInfo(); appendStringInfo(alterCommand, "ALTER SERVER %s OWNER TO %s;", quote_identifier(server->servername), quote_identifier(ownerName)); return alterCommand->data; } /* * RecreateForeignServerStmt returns a parsetree for a CREATE SERVER statement * that would recreate the given server on a new node. */ static Node * RecreateForeignServerStmt(Oid serverId) { ForeignServer *server = GetForeignServer(serverId); CreateForeignServerStmt *createStmt = makeNode(CreateForeignServerStmt); /* set server name and if_not_exists fields */ createStmt->servername = pstrdup(server->servername); createStmt->if_not_exists = true; /* set foreign data wrapper */ ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid); createStmt->fdwname = pstrdup(fdw->fdwname); /* set all fields using the existing server */ if (server->servertype != NULL) { createStmt->servertype = pstrdup(server->servertype); } if (server->serverversion != NULL) { createStmt->version = pstrdup(server->serverversion); } createStmt->options = NIL; int location = -1; DefElem *option = NULL; foreach_declared_ptr(option, server->options) { DefElem *copyOption = makeDefElem(option->defname, option->arg, location); createStmt->options = lappend(createStmt->options, copyOption); } return (Node *) createStmt; } /* * NameListHasDistributedServer takes a namelist of servers and returns true if at least * one of them is distributed. Returns false otherwise. */ static bool NameListHasDistributedServer(List *serverNames) { String *serverValue = NULL; foreach_declared_ptr(serverValue, serverNames) { List *addresses = GetObjectAddressByServerName(strVal(serverValue), false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); if (IsAnyObjectDistributed(list_make1(address))) { return true; } } return false; } static List * GetObjectAddressByServerName(char *serverName, bool missing_ok) { ForeignServer *server = GetForeignServerByName(serverName, missing_ok); Oid serverOid = (server) ? server->serverid : InvalidOid; ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ForeignServerRelationId, serverOid); return list_make1(address); } ================================================ FILE: src/backend/distributed/commands/function.c ================================================ /*------------------------------------------------------------------------- * * function.c * Commands for FUNCTION statements. * * We currently support replicating function definitions on the * coordinator in all the worker nodes in the form of * * CREATE OR REPLACE FUNCTION ... queries and * GRANT ... ON FUNCTION queries * * * ALTER or DROP operations are not yet propagated. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/extension.h" #include "nodes/makefuncs.h" #include "parser/parse_coerce.h" #include "parser/parse_type.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_depended_object.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/maintenanced.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/namespace_utils.h" #include "distributed/pg_dist_node.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/version_compat.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_transaction.h" #define DISABLE_LOCAL_CHECK_FUNCTION_BODIES "SET LOCAL check_function_bodies TO off;" #define RESET_CHECK_FUNCTION_BODIES "RESET check_function_bodies;" #define argumentStartsWith(arg, prefix) \ (strncmp(arg, prefix, strlen(prefix)) == 0) /* forward declaration for helper functions*/ static bool RecreateSameNonColocatedFunction(ObjectAddress functionAddress, char *distributionArgumentName, bool colocateWithTableNameDefault, bool *forceDelegationAddress); static void ErrorIfAnyNodeDoesNotHaveMetadata(void); static char * GetAggregateDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); static char * GetFunctionAlterOwnerCommand(const RegProcedure funcOid); static int GetDistributionArgIndex(Oid functionOid, char *distributionArgumentName, Oid *distributionArgumentOid); static int GetFunctionColocationId(Oid functionOid, char *colocateWithName, Oid distributionArgumentOid); static void EnsureFunctionCanBeColocatedWithTable(Oid functionOid, Oid distributionColumnType, Oid sourceRelationId); static bool ShouldPropagateCreateFunction(CreateFunctionStmt *stmt); static bool ShouldPropagateAlterFunction(const ObjectAddress *address); static bool ShouldAddFunctionSignature(FunctionParameterMode mode); static List * FunctionToObjectAddress(ObjectType objectType, ObjectWithArgs *objectWithArgs, bool missing_ok); static void ErrorIfUnsupportedAlterFunctionStmt(AlterFunctionStmt *stmt); static char * quote_qualified_func_name(Oid funcOid); static void DistributeFunctionWithDistributionArgument(RegProcedure funcOid, char *distributionArgumentName, Oid distributionArgumentOid, char *colocateWithTableName, bool *forceDelegationAddress, const ObjectAddress * functionAddress); static void DistributeFunctionColocatedWithDistributedTable(RegProcedure funcOid, char *colocateWithTableName, const ObjectAddress * functionAddress); static void DistributeFunctionColocatedWithSingleShardTable(const ObjectAddress * functionAddress, text * colocateWithText); static void DistributeFunctionColocatedWithReferenceTable(const ObjectAddress *functionAddress); static List * FilterDistributedFunctions(GrantStmt *grantStmt); static void EnsureExtensionFunctionCanBeDistributed(const ObjectAddress functionAddress, const ObjectAddress extensionAddress, char *distributionArgumentName); PG_FUNCTION_INFO_V1(create_distributed_function); /* * create_distributed_function gets a function or procedure name with their list of * argument types in parantheses, then it creates a new distributed function. */ Datum create_distributed_function(PG_FUNCTION_ARGS) { RegProcedure funcOid = PG_GETARG_OID(0); text *distributionArgumentNameText = NULL; /* optional */ text *colocateWithText = NULL; /* optional */ StringInfoData ddlCommand = { 0 }; ObjectAddress *functionAddress = palloc0(sizeof(ObjectAddress)); Oid distributionArgumentOid = InvalidOid; bool colocatedWithReferenceTable = false; bool colocatedWithSingleShardTable = false; char *distributionArgumentName = NULL; char *colocateWithTableName = NULL; bool colocateWithTableNameDefault = false; bool *forceDelegationAddress = NULL; bool forceDelegation = false; ObjectAddress extensionAddress = { 0 }; /* if called on NULL input, error out */ if (funcOid == InvalidOid) { ereport(ERROR, (errmsg("the first parameter for create_distributed_function() " "should be a single a valid function or procedure name " "followed by a list of parameters in parantheses"), errhint("skip the parameters with OUT argtype as they are not " "part of the signature in PostgreSQL"))); } if (PG_ARGISNULL(1)) { /* * Using the default value, so distribute the function but do not set * the distribution argument. */ distributionArgumentName = NULL; } else { distributionArgumentNameText = PG_GETARG_TEXT_P(1); distributionArgumentName = text_to_cstring(distributionArgumentNameText); } if (PG_ARGISNULL(2)) { ereport(ERROR, (errmsg("colocate_with parameter should not be NULL"), errhint("To use the default value, set colocate_with option " "to \"default\""))); } else { colocateWithText = PG_GETARG_TEXT_P(2); colocateWithTableName = text_to_cstring(colocateWithText); if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) == 0) { colocateWithTableNameDefault = true; } /* check if the colocation belongs to a reference table */ if (!colocateWithTableNameDefault) { Oid colocationRelationId = ResolveRelationId(colocateWithText, false); colocatedWithReferenceTable = IsCitusTableType(colocationRelationId, REFERENCE_TABLE); colocatedWithSingleShardTable = IsCitusTableType(colocationRelationId, SINGLE_SHARD_DISTRIBUTED); } } /* check if the force_delegation flag is explicitly set (default is NULL) */ if (PG_ARGISNULL(3)) { forceDelegationAddress = NULL; } else { forceDelegation = PG_GETARG_BOOL(3); forceDelegationAddress = &forceDelegation; } EnsureCoordinator(); EnsureFunctionOwner(funcOid); ObjectAddressSet(*functionAddress, ProcedureRelationId, funcOid); if (RecreateSameNonColocatedFunction(*functionAddress, distributionArgumentName, colocateWithTableNameDefault, forceDelegationAddress)) { char *schemaName = get_namespace_name(get_func_namespace(funcOid)); char *functionName = get_func_name(funcOid); char *qualifiedName = quote_qualified_identifier(schemaName, functionName); ereport(NOTICE, (errmsg("procedure %s is already distributed", qualifiedName), errdetail("Citus distributes procedures with CREATE " "[PROCEDURE|FUNCTION|AGGREGATE] commands"))); PG_RETURN_VOID(); } /* * If the function is owned by an extension, only update the * pg_dist_object, and not propagate the CREATE FUNCTION. Function * will be created by the virtue of the extension creation. */ if (IsAnyObjectAddressOwnedByExtension(list_make1(functionAddress), &extensionAddress)) { EnsureExtensionFunctionCanBeDistributed(*functionAddress, extensionAddress, distributionArgumentName); } else { /* * when we allow propagation within a transaction block we should make sure * to only allow this in sequential mode. */ EnsureSequentialMode(OBJECT_FUNCTION); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(functionAddress)); const char *createFunctionSQL = GetFunctionDDLCommand(funcOid, true); const char *alterFunctionOwnerSQL = GetFunctionAlterOwnerCommand(funcOid); initStringInfo(&ddlCommand); appendStringInfo(&ddlCommand, "%s;%s;%s", DISABLE_METADATA_SYNC, createFunctionSQL, alterFunctionOwnerSQL); List *grantDDLCommands = GrantOnFunctionDDLCommands(funcOid); char *grantOnFunctionSQL = NULL; foreach_declared_ptr(grantOnFunctionSQL, grantDDLCommands) { appendStringInfo(&ddlCommand, ";%s", grantOnFunctionSQL); } appendStringInfo(&ddlCommand, ";%s", ENABLE_METADATA_SYNC); SendCommandToWorkersAsUser(NON_COORDINATOR_NODES, CurrentUserName(), ddlCommand.data); } MarkObjectDistributed(functionAddress); if (distributionArgumentName != NULL) { /* * Prior to Citus 11, this code was triggering metadata * syncing. However, with Citus 11+, we expect the metadata * has already been synced. */ ErrorIfAnyNodeDoesNotHaveMetadata(); DistributeFunctionWithDistributionArgument(funcOid, distributionArgumentName, distributionArgumentOid, colocateWithTableName, forceDelegationAddress, functionAddress); } else if (!colocatedWithReferenceTable && !colocatedWithSingleShardTable) { DistributeFunctionColocatedWithDistributedTable(funcOid, colocateWithTableName, functionAddress); } else if (colocatedWithSingleShardTable) { DistributeFunctionColocatedWithSingleShardTable(functionAddress, colocateWithText); } else if (colocatedWithReferenceTable) { /* * Prior to Citus 11, this code was triggering metadata * syncing. However, with Citus 11+, we expect the metadata * has already been synced. */ ErrorIfAnyNodeDoesNotHaveMetadata(); DistributeFunctionColocatedWithReferenceTable(functionAddress); } PG_RETURN_VOID(); } /* * RecreateSameNonColocatedFunction returns true if the given parameters of * create_distributed_function will not change anything on the given function. * Returns false otherwise. */ static bool RecreateSameNonColocatedFunction(ObjectAddress functionAddress, char *distributionArgumentName, bool colocateWithTableNameDefault, bool *forceDelegationAddress) { DistObjectCacheEntry *cacheEntry = LookupDistObjectCacheEntry(ProcedureRelationId, functionAddress.objectId, InvalidOid); if (cacheEntry == NULL || !cacheEntry->isValid || !cacheEntry->isDistributed) { return false; } /* * If the colocationId, forceDelegation and distributionArgIndex fields of a * pg_dist_object entry of a distributed function are all set to zero, it means * that function is either automatically distributed by ddl propagation, without * calling create_distributed_function. Or, it could be distributed via * create_distributed_function, but with no parameters. * * For these cases, calling create_distributed_function for that function, * without parameters would be idempotent. Hence we can simply early return here, * by providing a notice message to the user. */ /* are pg_dist_object fields set to zero? */ bool functionDistributedWithoutParams = cacheEntry->colocationId == 0 && cacheEntry->forceDelegation == 0 && cacheEntry->distributionArgIndex == 0; /* called create_distributed_function without parameters? */ bool distributingAgainWithNoParams = distributionArgumentName == NULL && colocateWithTableNameDefault && forceDelegationAddress == NULL; return functionDistributedWithoutParams && distributingAgainWithNoParams; } /* * ErrorIfAnyNodeDoesNotHaveMetadata throws error if any * of the worker nodes does not have the metadata. */ static void ErrorIfAnyNodeDoesNotHaveMetadata(void) { List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(ShareLock); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if (!workerNode->hasMetadata) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot process the distributed function " "since the node %s:%d does not have metadata " "synced and this command requires all the nodes " "have the metadata sycned", workerNode->workerName, workerNode->workerPort), errhint("To sync the metadata execute: " "SELECT enable_citus_mx_for_pre_citus11();"))); } } } /* * DistributeFunctionWithDistributionArgument updates pg_dist_object records for * a function/procedure that has a distribution argument, and triggers metadata * sync so that the functions can be delegated on workers. */ static void DistributeFunctionWithDistributionArgument(RegProcedure funcOid, char *distributionArgumentName, Oid distributionArgumentOid, char *colocateWithTableName, bool *forceDelegationAddress, const ObjectAddress *functionAddress) { /* get the argument index, or error out if we cannot find a valid index */ int distributionArgumentIndex = GetDistributionArgIndex(funcOid, distributionArgumentName, &distributionArgumentOid); /* get the colocation id, or error out if we cannot find an appropriate one */ int colocationId = GetFunctionColocationId(funcOid, colocateWithTableName, distributionArgumentOid); /* record the distribution argument and colocationId */ UpdateFunctionDistributionInfo(functionAddress, &distributionArgumentIndex, &colocationId, forceDelegationAddress); } /* * DistributeFunctionColocatedWithDistributedTable updates pg_dist_object records for * a function/procedure that is colocated with a distributed table. */ static void DistributeFunctionColocatedWithDistributedTable(RegProcedure funcOid, char *colocateWithTableName, const ObjectAddress *functionAddress) { /* * cannot provide colocate_with without distribution_arg_name when the function * is not colocated with a reference table */ if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) != 0) { char *functionName = get_func_name(funcOid); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot distribute the function \"%s\" since the " "distribution argument is not valid ", functionName), errhint("To provide \"colocate_with\" option with a" " distributed table, the distribution argument" " parameter should also be provided"))); } /* set distribution argument and colocationId to NULL */ UpdateFunctionDistributionInfo(functionAddress, NULL, NULL, NULL); } /* * DistributeFunctionColocatedWithSingleShardTable updates pg_dist_object records for * a function/procedure that is colocated with a single shard table. */ static void DistributeFunctionColocatedWithSingleShardTable(const ObjectAddress *functionAddress, text *colocateWithText) { /* get the single shard table's colocation id */ int colocationId = TableColocationId(ResolveRelationId(colocateWithText, false)); /* set distribution argument to NULL */ int *distributionArgumentIndex = NULL; UpdateFunctionDistributionInfo(functionAddress, distributionArgumentIndex, &colocationId, NULL); } /* * DistributeFunctionColocatedWithReferenceTable updates pg_dist_object records for * a function/procedure that is colocated with a reference table. */ static void DistributeFunctionColocatedWithReferenceTable(const ObjectAddress *functionAddress) { /* get the reference table colocation id */ int colocationId = CreateReferenceTableColocationId(); /* set distribution argument to NULL and colocationId to the reference table colocation id */ int *distributionArgumentIndex = NULL; UpdateFunctionDistributionInfo(functionAddress, distributionArgumentIndex, &colocationId, NULL); } /* * CreateFunctionDDLCommandsIdempotent returns a list of DDL statements (const char *) to be * executed on a node to recreate the function addressed by the functionAddress. */ List * CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress) { Assert(functionAddress->classId == ProcedureRelationId); char *ddlCommand = GetFunctionDDLCommand(functionAddress->objectId, true); char *alterFunctionOwnerSQL = GetFunctionAlterOwnerCommand(functionAddress->objectId); return list_make4( DISABLE_LOCAL_CHECK_FUNCTION_BODIES, ddlCommand, alterFunctionOwnerSQL, RESET_CHECK_FUNCTION_BODIES); } /* * GetDistributionArgIndex calculates the distribution argument with the given * parameters. The function errors out if no valid argument is found. */ static int GetDistributionArgIndex(Oid functionOid, char *distributionArgumentName, Oid *distributionArgumentOid) { int distributionArgumentIndex = -1; Oid *argTypes = NULL; char **argNames = NULL; char *argModes = NULL; *distributionArgumentOid = InvalidOid; HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "cache lookup failed for function %u", functionOid); } int numberOfArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); if (argumentStartsWith(distributionArgumentName, "$")) { /* skip the first character, we're safe because text_to_cstring pallocs */ distributionArgumentName++; /* throws error if the input is not an integer */ distributionArgumentIndex = pg_strtoint32(distributionArgumentName); if (distributionArgumentIndex < 1 || distributionArgumentIndex > numberOfArgs) { char *functionName = get_func_name(functionOid); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot distribute the function \"%s\" since " "the distribution argument is not valid", functionName), errhint("Either provide a valid function argument name " "or a valid \"$paramIndex\" to " "create_distributed_function()"))); } /* * Internal representation for the distributionArgumentIndex * starts from 0 whereas user facing API starts from 1. */ distributionArgumentIndex -= 1; *distributionArgumentOid = argTypes[distributionArgumentIndex]; ReleaseSysCache(proctup); Assert(*distributionArgumentOid != InvalidOid); return distributionArgumentIndex; } /* * The user didn't provid "$paramIndex" but potentially the name of the parameter. * So, loop over the arguments and try to find the argument name that matches * the parameter that user provided. */ for (int argIndex = 0; argIndex < numberOfArgs; ++argIndex) { char *argNameOnIndex = argNames != NULL ? argNames[argIndex] : NULL; if (argNameOnIndex != NULL && pg_strncasecmp(argNameOnIndex, distributionArgumentName, NAMEDATALEN) == 0) { distributionArgumentIndex = argIndex; *distributionArgumentOid = argTypes[argIndex]; /* we found, no need to continue */ break; } } /* we still couldn't find the argument, so error out */ if (distributionArgumentIndex == -1) { char *functionName = get_func_name(functionOid); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot distribute the function \"%s\" since the " "distribution argument is not valid ", functionName), errhint("Either provide a valid function argument name " "or a valid \"$paramIndex\" to " "create_distributed_function()"))); } ReleaseSysCache(proctup); Assert(*distributionArgumentOid != InvalidOid); return distributionArgumentIndex; } /* * GetFunctionColocationId gets the parameters for deciding the colocationId * of the function that is being distributed. The function errors out if it is * not possible to assign a colocationId to the input function. */ static int GetFunctionColocationId(Oid functionOid, char *colocateWithTableName, Oid distributionArgumentOid) { int colocationId = INVALID_COLOCATION_ID; Relation pgDistColocation = table_open(DistColocationRelationId(), ShareLock); if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) == 0) { /* check for default colocation group */ colocationId = ColocationId(ShardCount, ShardReplicationFactor, distributionArgumentOid, get_typcollation( distributionArgumentOid)); if (colocationId == INVALID_COLOCATION_ID) { char *functionName = get_func_name(functionOid); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot distribute the function \"%s\" since there " "is no table to colocate with", functionName), errhint("Provide a distributed table via \"colocate_with\" " "option to create_distributed_function()"))); } Oid colocatedTableId = ColocatedTableId(colocationId); if (colocatedTableId != InvalidOid) { EnsureFunctionCanBeColocatedWithTable(functionOid, distributionArgumentOid, colocatedTableId); } } else { Oid sourceRelationId = ResolveRelationId(cstring_to_text(colocateWithTableName), false); EnsureFunctionCanBeColocatedWithTable(functionOid, distributionArgumentOid, sourceRelationId); colocationId = TableColocationId(sourceRelationId); } /* keep the lock */ table_close(pgDistColocation, NoLock); return colocationId; } /* * EnsureFunctionCanBeColocatedWithTable checks whether the given arguments are * suitable to distribute the function to be colocated with given source table. */ static void EnsureFunctionCanBeColocatedWithTable(Oid functionOid, Oid distributionColumnType, Oid sourceRelationId) { CitusTableCacheEntry *sourceTableEntry = GetCitusTableCacheEntry(sourceRelationId); char sourceReplicationModel = sourceTableEntry->replicationModel; if (IsCitusTableTypeCacheEntry(sourceTableEntry, SINGLE_SHARD_DISTRIBUTED) && distributionColumnType != InvalidOid) { char *functionName = get_func_name(functionOid); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot colocate function \"%s\" and table \"%s\" because " "distribution arguments are not supported when " "colocating with single shard distributed tables.", functionName, sourceRelationName))); } if (!IsCitusTableTypeCacheEntry(sourceTableEntry, HASH_DISTRIBUTED) && !IsCitusTableTypeCacheEntry(sourceTableEntry, REFERENCE_TABLE)) { char *functionName = get_func_name(functionOid); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot colocate function \"%s\" and table \"%s\" because " "colocate_with option is only supported for hash " "distributed tables and reference tables.", functionName, sourceRelationName))); } if (IsCitusTableTypeCacheEntry(sourceTableEntry, REFERENCE_TABLE) && distributionColumnType != InvalidOid) { char *functionName = get_func_name(functionOid); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot colocate function \"%s\" and table \"%s\" because " "distribution arguments are not supported when " "colocating with reference tables.", functionName, sourceRelationName))); } if (sourceReplicationModel != REPLICATION_MODEL_STREAMING) { char *functionName = get_func_name(functionOid); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errmsg("cannot colocate function \"%s\" and table \"%s\"", functionName, sourceRelationName), errdetail("Citus currently only supports colocating function " "with distributed tables that are created using " "streaming replication model."), errhint("When distributing tables make sure that " "citus.shard_replication_factor = 1"))); } /* * If the types are the same, we're good. If not, we still check if there * is any coercion path between the types. */ Var *sourceDistributionColumn = DistPartitionKeyOrError(sourceRelationId); Oid sourceDistributionColumnType = sourceDistributionColumn->vartype; if (sourceDistributionColumnType != distributionColumnType) { Oid coercionFuncId = InvalidOid; CoercionPathType coercionType = find_coercion_pathway(distributionColumnType, sourceDistributionColumnType, COERCION_EXPLICIT, &coercionFuncId); /* if there is no path for coercion, error out*/ if (coercionType == COERCION_PATH_NONE) { char *functionName = get_func_name(functionOid); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errmsg("cannot colocate function \"%s\" and table \"%s\" " "because distribution column types don't match and " "there is no coercion path", sourceRelationName, functionName))); } } } /* * UpdateFunctionDistributionInfo gets object address of a function and * updates its distribution_argument_index and colocationId in pg_dist_object. * Then update pg_dist_object on nodes with metadata if object propagation is on. */ void UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, int *distribution_argument_index, int *colocationId, bool *forceDelegation) { const bool indexOK = true; ScanKeyData scanKey[3]; Relation pgDistObjectRel = table_open(DistObjectRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistObjectRel); Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc0(tupleDescriptor->natts * sizeof(bool)); int forseDelegationIndex = GetForceDelegationAttrIndexInPgDistObject(tupleDescriptor); /* scan pg_dist_object for classid = $1 AND objid = $2 AND objsubid = $3 via index */ ScanKeyInit(&scanKey[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distAddress->classId)); ScanKeyInit(&scanKey[1], Anum_pg_dist_object_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distAddress->objectId)); ScanKeyInit(&scanKey[2], Anum_pg_dist_object_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(distAddress->objectSubId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistObjectRel, DistObjectPrimaryKeyIndexId(), indexOK, NULL, 3, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for node \"%d,%d,%d\" " "in pg_dist_object", distAddress->classId, distAddress->objectId, distAddress->objectSubId))); } replace[Anum_pg_dist_object_distribution_argument_index - 1] = true; if (distribution_argument_index != NULL) { values[Anum_pg_dist_object_distribution_argument_index - 1] = Int32GetDatum( *distribution_argument_index); isnull[Anum_pg_dist_object_distribution_argument_index - 1] = false; } else { isnull[Anum_pg_dist_object_distribution_argument_index - 1] = true; } replace[Anum_pg_dist_object_colocationid - 1] = true; if (colocationId != NULL) { values[Anum_pg_dist_object_colocationid - 1] = Int32GetDatum(*colocationId); isnull[Anum_pg_dist_object_colocationid - 1] = false; } else { isnull[Anum_pg_dist_object_colocationid - 1] = true; } replace[forseDelegationIndex] = true; if (forceDelegation != NULL) { values[forseDelegationIndex] = BoolGetDatum(*forceDelegation); isnull[forseDelegationIndex] = false; } else { isnull[forseDelegationIndex] = true; } heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistObjectRel, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistObjectRelationId()); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistObjectRel, NoLock); pfree(values); pfree(isnull); pfree(replace); if (EnableMetadataSync) { List *objectAddressList = list_make1((ObjectAddress *) distAddress); List *distArgumentIndexList = NIL; List *colocationIdList = NIL; List *forceDelegationList = NIL; if (distribution_argument_index == NULL) { distArgumentIndexList = list_make1_int(INVALID_DISTRIBUTION_ARGUMENT_INDEX); } else { distArgumentIndexList = list_make1_int(*distribution_argument_index); } if (colocationId == NULL) { colocationIdList = list_make1_int(INVALID_COLOCATION_ID); } else { colocationIdList = list_make1_int(*colocationId); } if (forceDelegation == NULL) { forceDelegationList = list_make1_int(NO_FORCE_PUSHDOWN); } else { forceDelegationList = list_make1_int(*forceDelegation); } char *workerPgDistObjectUpdateCommand = MarkObjectsDistributedCreateCommand(objectAddressList, NIL, distArgumentIndexList, colocationIdList, forceDelegationList); SendCommandToWorkersWithMetadata(workerPgDistObjectUpdateCommand); } } /* * GetFunctionDDLCommand returns the complete "CREATE OR REPLACE FUNCTION ..." statement for * the specified function. * * useCreateOrReplace is ignored for non-aggregate functions. */ char * GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace) { char *createFunctionSQL = NULL; if (get_func_prokind(funcOid) == PROKIND_AGGREGATE) { createFunctionSQL = GetAggregateDDLCommand(funcOid, useCreateOrReplace); } else { Datum sqlTextDatum = (Datum) 0; int saveNestLevel = PushEmptySearchPath(); sqlTextDatum = DirectFunctionCall1(pg_get_functiondef, ObjectIdGetDatum(funcOid)); createFunctionSQL = TextDatumGetCString(sqlTextDatum); /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } return createFunctionSQL; } /* * GetFunctionAlterOwnerCommand returns "ALTER FUNCTION .. SET OWNER .." statement for * the specified function. */ static char * GetFunctionAlterOwnerCommand(const RegProcedure funcOid) { HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcOid)); StringInfo alterCommand = makeStringInfo(); Oid procOwner = InvalidOid; if (HeapTupleIsValid(proctup)) { Form_pg_proc procform = (Form_pg_proc) GETSTRUCT(proctup); procOwner = procform->proowner; ReleaseSysCache(proctup); } else if (!OidIsValid(funcOid) || !HeapTupleIsValid(proctup)) { ereport(ERROR, (errmsg("cannot find function with oid: %d", funcOid))); } /* * If the function exists we want to use format_procedure_qualified to * serialize its canonical arguments */ char *functionSignature = format_procedure_qualified(funcOid); char *functionOwner = GetUserNameFromId(procOwner, false); appendStringInfo(alterCommand, "ALTER ROUTINE %s OWNER TO %s;", functionSignature, quote_identifier(functionOwner)); return alterCommand->data; } /* * GetAggregateDDLCommand returns a string for creating an aggregate. * A second parameter useCreateOrReplace signals whether to * to create a plain CREATE AGGREGATE or not. */ static char * GetAggregateDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace) { StringInfoData buf = { 0 }; int i = 0; Oid *argtypes = NULL; char **argnames = NULL; char *argmodes = NULL; int insertorderbyat = -1; int argsprinted = 0; HeapTuple proctup = SearchSysCache1(PROCOID, funcOid); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "cache lookup failed for %d", funcOid); } Form_pg_proc proc = (Form_pg_proc) GETSTRUCT(proctup); Assert(proc->prokind == PROKIND_AGGREGATE); initStringInfo(&buf); const char *name = NameStr(proc->proname); const char *nsp = get_namespace_name(proc->pronamespace); if (useCreateOrReplace) { appendStringInfo(&buf, "CREATE OR REPLACE AGGREGATE %s(", quote_qualified_identifier(nsp, name)); } else { appendStringInfo(&buf, "CREATE AGGREGATE %s(", quote_qualified_identifier(nsp, name)); } /* Parameters, borrows heavily from print_function_arguments in postgres */ int numargs = get_func_arg_info(proctup, &argtypes, &argnames, &argmodes); HeapTuple aggtup = SearchSysCache1(AGGFNOID, funcOid); if (!HeapTupleIsValid(aggtup)) { elog(ERROR, "cache lookup failed for %d", funcOid); } Form_pg_aggregate agg = (Form_pg_aggregate) GETSTRUCT(aggtup); if (AGGKIND_IS_ORDERED_SET(agg->aggkind)) { insertorderbyat = agg->aggnumdirectargs; } /* * For zero-argument aggregate, write * in place of the list of arguments */ if (numargs == 0) { appendStringInfo(&buf, "*"); } for (i = 0; i < numargs; i++) { Oid argtype = argtypes[i]; char *argname = argnames ? argnames[i] : NULL; char argmode = argmodes ? argmodes[i] : PROARGMODE_IN; const char *modename; switch (argmode) { case PROARGMODE_IN: { modename = ""; break; } case PROARGMODE_VARIADIC: { modename = "VARIADIC "; break; } default: { elog(ERROR, "unexpected parameter mode '%c'", argmode); modename = NULL; break; } } if (argsprinted == insertorderbyat) { appendStringInfoString(&buf, " ORDER BY "); } else if (argsprinted) { appendStringInfoString(&buf, ", "); } appendStringInfoString(&buf, modename); if (argname && argname[0]) { appendStringInfo(&buf, "%s ", quote_identifier(argname)); } appendStringInfoString(&buf, format_type_be_qualified(argtype)); argsprinted++; /* nasty hack: print the last arg twice for variadic ordered-set agg */ if (argsprinted == insertorderbyat && i == numargs - 1) { i--; } } appendStringInfo(&buf, ") (STYPE = %s,SFUNC = %s", format_type_be_qualified(agg->aggtranstype), quote_qualified_func_name(agg->aggtransfn)); if (agg->aggtransspace != 0) { appendStringInfo(&buf, ", SSPACE = %d", agg->aggtransspace); } if (agg->aggfinalfn != InvalidOid) { const char *finalmodifystring = NULL; switch (agg->aggfinalmodify) { case AGGMODIFY_READ_ONLY: { finalmodifystring = "READ_ONLY"; break; } case AGGMODIFY_SHAREABLE: { finalmodifystring = "SHAREABLE"; break; } case AGGMODIFY_READ_WRITE: { finalmodifystring = "READ_WRITE"; break; } } appendStringInfo(&buf, ", FINALFUNC = %s", quote_qualified_func_name(agg->aggfinalfn)); if (finalmodifystring != NULL) { appendStringInfo(&buf, ", FINALFUNC_MODIFY = %s", finalmodifystring); } if (agg->aggfinalextra) { appendStringInfoString(&buf, ", FINALFUNC_EXTRA"); } } if (agg->aggmtransspace != 0) { appendStringInfo(&buf, ", MSSPACE = %d", agg->aggmtransspace); } if (agg->aggmfinalfn) { const char *mfinalmodifystring = NULL; switch (agg->aggfinalmodify) { case AGGMODIFY_READ_ONLY: { mfinalmodifystring = "READ_ONLY"; break; } case AGGMODIFY_SHAREABLE: { mfinalmodifystring = "SHAREABLE"; break; } case AGGMODIFY_READ_WRITE: { mfinalmodifystring = "READ_WRITE"; break; } } appendStringInfo(&buf, ", MFINALFUNC = %s", quote_qualified_func_name(agg->aggmfinalfn)); if (mfinalmodifystring != NULL) { appendStringInfo(&buf, ", MFINALFUNC_MODIFY = %s", mfinalmodifystring); } if (agg->aggmfinalextra) { appendStringInfoString(&buf, ", MFINALFUNC_EXTRA"); } } if (agg->aggmtransfn) { appendStringInfo(&buf, ", MSFUNC = %s", quote_qualified_func_name(agg->aggmtransfn)); if (agg->aggmtranstype) { appendStringInfo(&buf, ", MSTYPE = %s", format_type_be_qualified(agg->aggmtranstype)); } } if (agg->aggtransspace != 0) { appendStringInfo(&buf, ", SSPACE = %d", agg->aggtransspace); } if (agg->aggminvtransfn) { appendStringInfo(&buf, ", MINVFUNC = %s", quote_qualified_func_name(agg->aggminvtransfn)); } if (agg->aggcombinefn) { appendStringInfo(&buf, ", COMBINEFUNC = %s", quote_qualified_func_name(agg->aggcombinefn)); } if (agg->aggserialfn) { appendStringInfo(&buf, ", SERIALFUNC = %s", quote_qualified_func_name(agg->aggserialfn)); } if (agg->aggdeserialfn) { appendStringInfo(&buf, ", DESERIALFUNC = %s", quote_qualified_func_name(agg->aggdeserialfn)); } if (agg->aggsortop != InvalidOid) { appendStringInfo(&buf, ", SORTOP = %s", generate_operator_name(agg->aggsortop, argtypes[0], argtypes[0])); } { const char *parallelstring = NULL; switch (proc->proparallel) { case PROPARALLEL_SAFE: { parallelstring = "SAFE"; break; } case PROPARALLEL_RESTRICTED: { parallelstring = "RESTRICTED"; break; } case PROPARALLEL_UNSAFE: { break; } default: { elog(WARNING, "Unknown parallel option, ignoring: %c", proc->proparallel); break; } } if (parallelstring != NULL) { appendStringInfo(&buf, ", PARALLEL = %s", parallelstring); } } { bool isNull = false; Datum textInitVal = SysCacheGetAttr(AGGFNOID, aggtup, Anum_pg_aggregate_agginitval, &isNull); if (!isNull) { char *strInitVal = TextDatumGetCString(textInitVal); char *strInitValQuoted = quote_literal_cstr(strInitVal); appendStringInfo(&buf, ", INITCOND = %s", strInitValQuoted); pfree(strInitValQuoted); pfree(strInitVal); } } { bool isNull = false; Datum textInitVal = SysCacheGetAttr(AGGFNOID, aggtup, Anum_pg_aggregate_aggminitval, &isNull); if (!isNull) { char *strInitVal = TextDatumGetCString(textInitVal); char *strInitValQuoted = quote_literal_cstr(strInitVal); appendStringInfo(&buf, ", MINITCOND = %s", strInitValQuoted); pfree(strInitValQuoted); pfree(strInitVal); } } if (agg->aggkind == AGGKIND_HYPOTHETICAL) { appendStringInfoString(&buf, ", HYPOTHETICAL"); } appendStringInfoChar(&buf, ')'); ReleaseSysCache(aggtup); ReleaseSysCache(proctup); return buf.data; } /* * ShouldPropagateCreateFunction tests if we need to propagate a CREATE FUNCTION * statement. */ static bool ShouldPropagateCreateFunction(CreateFunctionStmt *stmt) { if (!ShouldPropagate()) { return false; } if (!ShouldPropagateCreateInCoordinatedTransction()) { return false; } return true; } /* * ShouldPropagateAlterFunction returns, based on the address of a function, if alter * statements targeting the function should be propagated. */ static bool ShouldPropagateAlterFunction(const ObjectAddress *address) { if (creating_extension) { /* * extensions should be created separately on the workers, functions cascading * from an extension should therefore not be propagated. */ return false; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return false; } if (!IsAnyObjectDistributed(list_make1((ObjectAddress *) address))) { /* do not propagate alter function for non-distributed functions */ return false; } return true; } /* * PreprocessCreateFunctionStmt is called during the planning phase for CREATE [OR REPLACE] * FUNCTION before it is created on the local node internally. * * Since we use pg_get_functiondef to get the ddl command we actually do not do any * planning here, instead we defer the plan creation to the postprocessing step. * * Instead we do our basic housekeeping where we make sure we are on the coordinator and * can propagate the function in sequential mode. */ List * PreprocessCreateFunctionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { CreateFunctionStmt *stmt = castNode(CreateFunctionStmt, node); if (!ShouldPropagateCreateFunction(stmt)) { return NIL; } EnsureCoordinator(); EnsureSequentialMode(OBJECT_FUNCTION); /* * ddl jobs will be generated during the postprocessing phase as we need the function to * be updated in the catalog to get its sql representation */ return NIL; } /* * PostprocessCreateFunctionStmt actually creates the plan we need to execute for function * propagation. This is the downside of using pg_get_functiondef to get the sql statement. * * If function depends on any non-distributed relation (except sequence and composite type), * Citus can not distribute it. In order to not to prevent users from creating local * functions on the coordinator WARNING message will be sent to the customer about the case * instead of erroring out. * * Besides creating the plan we also make sure all (new) dependencies of the function are * created on all nodes. */ List * PostprocessCreateFunctionStmt(Node *node, const char *queryString) { CreateFunctionStmt *stmt = castNode(CreateFunctionStmt, node); if (!ShouldPropagateCreateFunction(stmt)) { return NIL; } List *functionAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, true); /* the code-path only supports a single object */ Assert(list_length(functionAddresses) == 1); if (IsAnyObjectAddressOwnedByExtension(functionAddresses, NULL)) { return NIL; } /* If the function has any unsupported dependency, create it locally */ DeferredErrorMessage *errMsg = DeferErrorIfAnyObjectHasUnsupportedDependency( functionAddresses); if (errMsg != NULL) { if (EnableUnsupportedFeatureMessages) { RaiseDeferredError(errMsg, WARNING); } return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(functionAddresses); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *functionAddress = linitial(functionAddresses); List *commands = list_make1(DISABLE_DDL_PROPAGATION); commands = list_concat(commands, CreateFunctionDDLCommandsIdempotent( functionAddress)); commands = list_concat(commands, list_make1(ENABLE_DDL_PROPAGATION)); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * CreateFunctionStmtObjectAddress returns the ObjectAddress for the subject of the * CREATE [OR REPLACE] FUNCTION statement. If missing_ok is false it will error with the * normal postgres error for unfound functions. */ List * CreateFunctionStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateFunctionStmt *stmt = castNode(CreateFunctionStmt, node); ObjectType objectType = OBJECT_FUNCTION; if (stmt->is_procedure) { objectType = OBJECT_PROCEDURE; } ObjectWithArgs *objectWithArgs = makeNode(ObjectWithArgs); objectWithArgs->objname = stmt->funcname; FunctionParameter *funcParam = NULL; foreach_declared_ptr(funcParam, stmt->parameters) { if (ShouldAddFunctionSignature(funcParam->mode)) { objectWithArgs->objargs = lappend(objectWithArgs->objargs, funcParam->argType); } } int OldClientMinMessage = client_min_messages; /* suppress NOTICE if running under pg vanilla tests */ SetLocalClientMinMessagesIfRunningPGTests(WARNING); List *funcAddresses = FunctionToObjectAddress(objectType, objectWithArgs, missing_ok); /* set it back */ SetLocalClientMinMessagesIfRunningPGTests(OldClientMinMessage); return funcAddresses; } /* * DefineAggregateStmtObjectAddress finds the ObjectAddress for the composite type described * by the DefineStmtObjectAddress. If missing_ok is false this function throws an error if the * aggregate does not exist. * * objectId in the address can be invalid if missing_ok was set to true. */ List * DefineAggregateStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DefineStmt *stmt = castNode(DefineStmt, node); Assert(stmt->kind == OBJECT_AGGREGATE); ObjectWithArgs *objectWithArgs = makeNode(ObjectWithArgs); objectWithArgs->objname = stmt->defnames; if (stmt->args != NIL) { FunctionParameter *funcParam = NULL; foreach_declared_ptr(funcParam, linitial(stmt->args)) { objectWithArgs->objargs = lappend(objectWithArgs->objargs, funcParam->argType); } } else { DefElem *defItem = NULL; foreach_declared_ptr(defItem, stmt->definition) { /* * If no explicit args are given, pg includes basetype in the signature. * If the basetype given is a type, like int4, we should include it in the * signature. In that case, defItem->arg would be a TypeName. * If the basetype given is a string, like "ANY", we shouldn't include it. */ if (strcmp(defItem->defname, "basetype") == 0 && IsA(defItem->arg, TypeName)) { objectWithArgs->objargs = lappend(objectWithArgs->objargs, defItem->arg); } } } return FunctionToObjectAddress(OBJECT_AGGREGATE, objectWithArgs, missing_ok); } /* * PreprocessAlterFunctionStmt is invoked for alter function statements with actions. Here we * plan the jobs to be executed on the workers for functions that have been distributed in * the cluster. */ List * PreprocessAlterFunctionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterFunctionStmt *stmt = castNode(AlterFunctionStmt, node); AssertObjectTypeIsFunctional(stmt->objtype); List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); if (!ShouldPropagateAlterFunction(address)) { return NIL; } EnsureCoordinator(); ErrorIfUnsupportedAlterFunctionStmt(stmt); EnsureSequentialMode(OBJECT_FUNCTION); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessAlterFunctionDependsStmt is called during the planning phase of an * ALTER FUNCION ... DEPENDS ON EXTENSION ... statement. Since functions depending on * extensions are assumed to be Owned by an extension we assume the extension to keep the * function in sync. * * If we would allow users to create a dependency between a distributed function and an * extension our pruning logic for which objects to distribute as dependencies of other * objects will change significantly which could cause issues adding new workers. Hence we * don't allow this dependency to be created. */ List * PreprocessAlterFunctionDependsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectDependsStmt *stmt = castNode(AlterObjectDependsStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); if (creating_extension) { /* * extensions should be created separately on the workers, types cascading from an * extension should therefore not be propagated here. */ return NIL; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return NIL; } List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, true, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!IsAnyObjectDistributed(addresses)) { return NIL; } /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); /* * Distributed objects should not start depending on an extension, this will break * the dependency resolving mechanism we use to replicate distributed objects to new * workers */ const char *functionName = getObjectIdentity(address, /* missingOk: */ false); ereport(ERROR, (errmsg("distrtibuted functions are not allowed to depend on an " "extension"), errdetail("Function \"%s\" is already distributed. Functions from " "extensions are expected to be created on the workers by " "the extension they depend on.", functionName))); } /* * AlterFunctionDependsStmtObjectAddress resolves the ObjectAddress of the function that * is the subject of an ALTER FUNCTION ... DEPENS ON EXTENSION ... statement. If * missing_ok is set to false the lookup will raise an error. */ List * AlterFunctionDependsStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectDependsStmt *stmt = castNode(AlterObjectDependsStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); return FunctionToObjectAddress(stmt->objectType, castNode(ObjectWithArgs, stmt->object), missing_ok); } /* * AlterFunctionStmtObjectAddress returns the ObjectAddress of the subject in the * AlterFunctionStmt. If missing_ok is set to false an error will be raised if postgres * was unable to find the function/procedure that was the target of the statement. */ List * AlterFunctionStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterFunctionStmt *stmt = castNode(AlterFunctionStmt, node); return FunctionToObjectAddress(stmt->objtype, stmt->func, missing_ok); } /* * RenameFunctionStmtObjectAddress returns the ObjectAddress of the function that is the * subject of the RenameStmt. Errors if missing_ok is false. */ List * RenameFunctionStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); return FunctionToObjectAddress(stmt->renameType, castNode(ObjectWithArgs, stmt->object), missing_ok); } /* * AlterFunctionOwnerObjectAddress returns the ObjectAddress of the function that is the * subject of the AlterOwnerStmt. Errors if missing_ok is false. */ List * AlterFunctionOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); return FunctionToObjectAddress(stmt->objectType, castNode(ObjectWithArgs, stmt->object), missing_ok); } /* * AlterFunctionSchemaStmtObjectAddress returns the ObjectAddress of the function that is * the subject of the AlterObjectSchemaStmt. Errors if missing_ok is false. * * This could be called both before or after it has been applied locally. It will look in * the old schema first, if the function cannot be found in that schema it will look in * the new schema. Errors if missing_ok is false and the type cannot be found in either of * the schemas. */ List * AlterFunctionSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); ObjectWithArgs *objectWithArgs = castNode(ObjectWithArgs, stmt->object); Oid funcOid = LookupFuncWithArgs(stmt->objectType, objectWithArgs, true); List *names = objectWithArgs->objname; if (funcOid == InvalidOid) { /* * couldn't find the function, might have already been moved to the new schema, we * construct a new objname that uses the new schema to search in. */ /* the name of the function is the last in the list of names */ String *funcNameStr = lfirst(list_tail(names)); List *newNames = list_make2(makeString(stmt->newschema), funcNameStr); /* * we don't error here either, as the error would be not a good user facing * error if the type didn't exist in the first place. */ objectWithArgs->objname = newNames; funcOid = LookupFuncWithArgs(stmt->objectType, objectWithArgs, true); objectWithArgs->objname = names; /* restore the original names */ /* * if the function is still invalid we couldn't find the function, cause postgres * to error by preforming a lookup once more. Since we know the */ if (!missing_ok && funcOid == InvalidOid) { /* * this will most probably throw an error, unless for some reason the function * has just been created (if possible at all). For safety we assign the * funcOid. */ funcOid = LookupFuncWithArgs(stmt->objectType, objectWithArgs, missing_ok); } } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ProcedureRelationId, funcOid); return list_make1(address); } /* * GenerateBackupNameForProcCollision generates a new proc name for an existing proc. The * name is generated in such a way that the new name doesn't overlap with an existing proc * by adding a suffix with incrementing number after the new name. */ char * GenerateBackupNameForProcCollision(const ObjectAddress *address) { char *newName = palloc0(NAMEDATALEN); char suffix[NAMEDATALEN] = { 0 }; int count = 0; String *namespace = makeString(get_namespace_name(get_func_namespace( address->objectId))); char *baseName = get_func_name(address->objectId); int baseLength = strlen(baseName); Oid *argtypes = NULL; char **argnames = NULL; char *argmodes = NULL; HeapTuple proctup = SearchSysCache1(PROCOID, address->objectId); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "citus cache lookup failed."); } int numargs = get_func_arg_info(proctup, &argtypes, &argnames, &argmodes); ReleaseSysCache(proctup); while (true) { int suffixLength = SafeSnprintf(suffix, NAMEDATALEN - 1, "(citus_backup_%d)", count); /* trim the base name at the end to leave space for the suffix and trailing \0 */ baseLength = Min(baseLength, NAMEDATALEN - suffixLength - 1); /* clear newName before copying the potentially trimmed baseName and suffix */ memset(newName, 0, NAMEDATALEN); strncpy_s(newName, NAMEDATALEN, baseName, baseLength); strncpy_s(newName + baseLength, NAMEDATALEN - baseLength, suffix, suffixLength); List *newProcName = list_make2(namespace, makeString(newName)); /* don't need to rename if the input arguments don't match */ FuncCandidateList clist = FuncnameGetCandidates(newProcName, numargs, NIL, false, false, false, true); for (; clist; clist = clist->next) { if (memcmp(clist->args, argtypes, sizeof(Oid) * numargs) == 0) { break; } } if (!clist) { return newName; } count++; } } /* * ObjectWithArgsFromOid returns the corresponding ObjectWithArgs node for a given pg_proc oid */ ObjectWithArgs * ObjectWithArgsFromOid(Oid funcOid) { ObjectWithArgs *objectWithArgs = makeNode(ObjectWithArgs); List *objargs = NIL; Oid *argTypes = NULL; char **argNames = NULL; char *argModes = NULL; HeapTuple proctup = SearchSysCache1(PROCOID, funcOid); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "citus cache lookup failed."); } int numargs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); objectWithArgs->objname = list_make2( makeString(get_namespace_name(get_func_namespace(funcOid))), makeString(get_func_name(funcOid)) ); for (int i = 0; i < numargs; i++) { if (argModes == NULL || ShouldAddFunctionSignature(argModes[i])) { objargs = lappend(objargs, makeTypeNameFromOid(argTypes[i], -1)); } } objectWithArgs->objargs = objargs; ReleaseSysCache(proctup); return objectWithArgs; } /* * ShouldAddFunctionSignature takes a FunctionParameterMode and returns true if it should * be included in the function signature. Returns false otherwise. */ static bool ShouldAddFunctionSignature(FunctionParameterMode mode) { /* only input parameters should be added to the generated signature */ switch (mode) { case FUNC_PARAM_IN: case FUNC_PARAM_INOUT: case FUNC_PARAM_VARIADIC: { return true; } case FUNC_PARAM_OUT: case FUNC_PARAM_TABLE: { return false; } default: { return true; } } } /* * FunctionToObjectAddress returns the ObjectAddress of a Function, Procedure or * Aggregate based on its type and ObjectWithArgs describing the * Function/Procedure/Aggregate. If missing_ok is set to false an error will be * raised by postgres explaining the Function/Procedure could not be found. */ static List * FunctionToObjectAddress(ObjectType objectType, ObjectWithArgs *objectWithArgs, bool missing_ok) { AssertObjectTypeIsFunctional(objectType); Oid funcOid = LookupFuncWithArgs(objectType, objectWithArgs, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, ProcedureRelationId, funcOid); return list_make1(address); } /* * ErrorIfUnsupportedAlterFunctionStmt raises an error if the AlterFunctionStmt contains a * construct that is not supported to be altered on a distributed function. It is assumed * the statement passed in is already tested to be targeting a distributed function, and * will only execute the checks to error on unsupported constructs. * * Unsupported Constructs: * - ALTER FUNCTION ... SET ... FROM CURRENT */ static void ErrorIfUnsupportedAlterFunctionStmt(AlterFunctionStmt *stmt) { DefElem *action = NULL; foreach_declared_ptr(action, stmt->actions) { if (strcmp(action->defname, "set") == 0) { VariableSetStmt *setStmt = castNode(VariableSetStmt, action->arg); if (setStmt->kind == VAR_SET_CURRENT) { /* check if the set action is a SET ... FROM CURRENT */ ereport(ERROR, (errmsg("unsupported ALTER FUNCTION ... SET ... FROM " "CURRENT for a distributed function"), errhint("SET FROM CURRENT is not supported for " "distributed functions, instead use the SET ... " "TO ... syntax with a constant value."))); } } } } /* returns the quoted qualified name of a given function oid */ static char * quote_qualified_func_name(Oid funcOid) { return quote_qualified_identifier( get_namespace_name(get_func_namespace(funcOid)), get_func_name(funcOid)); } /* * EnsureExtensionFuncionCanBeCreated checks if the dependent objects * (including extension) exists on all nodes, if not, creates them. In * addition, it also checks if distribution argument is passed. */ static void EnsureExtensionFunctionCanBeDistributed(const ObjectAddress functionAddress, const ObjectAddress extensionAddress, char *distributionArgumentName) { if (CitusExtensionObject(&extensionAddress)) { /* * Citus extension is a special case. It's the extension that * provides the 'distributed capabilities' in the first place. * Trying to distribute its own function(s) doesn't make sense. */ ereport(ERROR, (errmsg("Citus extension functions(%s) " "cannot be distributed.", get_func_name(functionAddress.objectId)))); } /* * Distributing functions from extensions has the most benefit when * distribution argument is specified. */ if (distributionArgumentName == NULL) { ereport(ERROR, (errmsg("Extension functions(%s) " "without distribution argument " "are not supported.", get_func_name(functionAddress.objectId)))); } /* * Ensure corresponding extension is in pg_dist_object. * Functions owned by an extension are depending internally on that extension, * hence EnsureAllObjectDependenciesExistOnAllNodes() creates the extension, which in * turn creates the function, and thus we don't have to create it ourself like * we do for non-extension functions. */ ereport(DEBUG1, (errmsg("Extension(%s) owning the " "function(%s) is not distributed, " "attempting to propogate the extension", get_extension_name(extensionAddress.objectId), get_func_name(functionAddress.objectId)))); ObjectAddress *copyFunctionAddress = palloc0(sizeof(ObjectAddress)); *copyFunctionAddress = functionAddress; EnsureAllObjectDependenciesExistOnAllNodes(list_make1(copyFunctionAddress)); } /* * PreprocessGrantOnFunctionStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on distributed functions, procedures, routines. */ List * PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(isFunction(stmt->objtype)); List *distributedFunctions = FilterDistributedFunctions(stmt); if (list_length(distributedFunctions) == 0 || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); List *grantFunctionList = NIL; ObjectAddress *functionAddress = NULL; foreach_declared_ptr(functionAddress, distributedFunctions) { ObjectWithArgs *distFunction = ObjectWithArgsFromOid( functionAddress->objectId); grantFunctionList = lappend(grantFunctionList, distFunction); } List *originalObjects = stmt->objects; GrantTargetType originalTargtype = stmt->targtype; stmt->objects = grantFunctionList; stmt->targtype = ACL_TARGET_OBJECT; char *sql = DeparseTreeNode((Node *) stmt); stmt->objects = originalObjects; stmt->targtype = originalTargtype; List *commandList = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commandList); } /* * PostprocessGrantOnFunctionStmt makes sure dependencies of each * distributed function in the statement exist on all nodes */ List * PostprocessGrantOnFunctionStmt(Node *node, const char *queryString) { GrantStmt *stmt = castNode(GrantStmt, node); List *distributedFunctions = FilterDistributedFunctions(stmt); if (list_length(distributedFunctions) == 0) { return NIL; } ObjectAddress *functionAddress = NULL; foreach_declared_ptr(functionAddress, distributedFunctions) { EnsureAllObjectDependenciesExistOnAllNodes(list_make1(functionAddress)); } return NIL; } /* * FilterDistributedFunctions determines and returns a list of distributed functions * ObjectAddress-es from given grant statement. */ static List * FilterDistributedFunctions(GrantStmt *grantStmt) { List *grantFunctionList = NIL; bool grantOnFunctionCommand = (grantStmt->targtype == ACL_TARGET_OBJECT && isFunction(grantStmt->objtype)); bool grantAllFunctionsOnSchemaCommand = (grantStmt->targtype == ACL_TARGET_ALL_IN_SCHEMA && isFunction(grantStmt->objtype)); /* we are only interested in function/procedure/routine level grants */ if (!grantOnFunctionCommand && !grantAllFunctionsOnSchemaCommand) { return NIL; } if (grantAllFunctionsOnSchemaCommand) { List *distributedFunctionList = DistributedFunctionList(); ObjectAddress *distributedFunction = NULL; List *namespaceOidList = NIL; /* iterate over all namespace names provided to get their oid's */ String *namespaceValue = NULL; foreach_declared_ptr(namespaceValue, grantStmt->objects) { char *nspname = strVal(namespaceValue); bool missing_ok = false; Oid namespaceOid = get_namespace_oid(nspname, missing_ok); namespaceOidList = list_append_unique_oid(namespaceOidList, namespaceOid); } /* * iterate over all distributed functions to filter the ones * that belong to one of the namespaces from above */ foreach_declared_ptr(distributedFunction, distributedFunctionList) { Oid namespaceOid = get_func_namespace(distributedFunction->objectId); /* * if this distributed function's schema is one of the schemas * specified in the GRANT .. ALL FUNCTIONS IN SCHEMA .. * add it to the list */ if (list_member_oid(namespaceOidList, namespaceOid)) { grantFunctionList = lappend(grantFunctionList, distributedFunction); } } } else { bool missingOk = false; ObjectWithArgs *objectWithArgs = NULL; foreach_declared_ptr(objectWithArgs, grantStmt->objects) { ObjectAddress *functionAddress = palloc0(sizeof(ObjectAddress)); functionAddress->classId = ProcedureRelationId; functionAddress->objectId = LookupFuncWithArgs(grantStmt->objtype, objectWithArgs, missingOk); functionAddress->objectSubId = 0; /* * if this function from GRANT .. ON FUNCTION .. is a distributed * function, add it to the list */ if (IsAnyObjectDistributed(list_make1(functionAddress))) { grantFunctionList = lappend(grantFunctionList, functionAddress); } } } return grantFunctionList; } ================================================ FILE: src/backend/distributed/commands/grant.c ================================================ /*------------------------------------------------------------------------- * * grant.c * Commands for granting access to distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/lsyscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/version_compat.h" /* Local functions forward declarations for helper functions */ static List * CollectGrantTableIdList(GrantStmt *grantStmt); /* * PreprocessGrantStmt determines whether a given GRANT/REVOKE statement involves * a distributed table. If so, it creates DDLJobs to encapsulate information * needed during the worker node portion of DDL execution before returning the * DDLJobs in a List. If no distributed table is involved, this returns NIL. * */ List * PreprocessGrantStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { GrantStmt *grantStmt = castNode(GrantStmt, node); StringInfoData privsString; StringInfoData granteesString; StringInfoData targetString; StringInfoData ddlString; ListCell *granteeCell = NULL; ListCell *tableListCell = NULL; bool isFirst = true; List *ddlJobs = NIL; initStringInfo(&privsString); initStringInfo(&granteesString); initStringInfo(&targetString); initStringInfo(&ddlString); /* * So far only table level grants are supported. Most other types of * grants aren't interesting anyway. */ if (grantStmt->objtype != OBJECT_TABLE) { return NIL; } List *tableIdList = CollectGrantTableIdList(grantStmt); /* nothing to do if there is no distributed table in the grant list */ if (tableIdList == NIL) { return NIL; } EnsureCoordinator(); /* deparse the privileges */ if (grantStmt->privileges == NIL) { /* this is used for table level only */ appendStringInfo(&privsString, "ALL"); } else { ListCell *privilegeCell = NULL; isFirst = true; foreach(privilegeCell, grantStmt->privileges) { AccessPriv *priv = lfirst(privilegeCell); if (!isFirst) { appendStringInfoString(&privsString, ", "); } if (priv->priv_name) { appendStringInfo(&privsString, "%s", priv->priv_name); } /* * ALL can only be set alone. * And ALL is not added as a keyword in priv_name by parser, but * because there are column(s) defined, a grantStmt->privileges is * defined. So we need to handle this special case here (see if * condition above). */ else if (isFirst) { /* this is used for column level only */ appendStringInfo(&privsString, "ALL"); } /* * Instead of relying only on the syntax check done by Postgres and * adding an assert here, add a default ERROR if ALL is not first * and no priv_name is defined. */ else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Cannot parse GRANT/REVOKE privileges"))); } isFirst = false; if (priv->cols != NIL) { StringInfoData colsString; initStringInfo(&colsString); AppendColumnNameList(&colsString, priv->cols); appendStringInfo(&privsString, "%s", colsString.data); } } } /* deparse the grantees */ isFirst = true; foreach(granteeCell, grantStmt->grantees) { RoleSpec *spec = lfirst(granteeCell); if (!isFirst) { appendStringInfoString(&granteesString, ", "); } isFirst = false; appendStringInfoString(&granteesString, RoleSpecString(spec, true)); } /* * Deparse the target objects, and issue the deparsed statements to * workers, if applicable. That's so we easily can replicate statements * only to distributed relations. */ isFirst = true; foreach(tableListCell, tableIdList) { Oid relationId = lfirst_oid(tableListCell); const char *grantOption = ""; resetStringInfo(&targetString); appendStringInfo(&targetString, "%s", generate_relation_name(relationId, NIL)); if (grantStmt->is_grant) { if (grantStmt->grant_option) { grantOption = " WITH GRANT OPTION"; } appendStringInfo(&ddlString, "GRANT %s ON %s TO %s%s", privsString.data, targetString.data, granteesString.data, grantOption); } else { if (grantStmt->grant_option) { grantOption = "GRANT OPTION FOR "; } appendStringInfo(&ddlString, "REVOKE %s%s ON %s FROM %s", grantOption, privsString.data, targetString.data, granteesString.data); if (grantStmt->behavior == DROP_CASCADE) { appendStringInfoString(&ddlString, " CASCADE"); } else { appendStringInfoString(&ddlString, " RESTRICT"); } } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->metadataSyncCommand = pstrdup(ddlString.data); ddlJob->taskList = NIL; if (IsCitusTable(relationId)) { ddlJob->taskList = DDLTaskList(relationId, ddlString.data); } ddlJobs = lappend(ddlJobs, ddlJob); resetStringInfo(&ddlString); } return ddlJobs; } /* * CollectGrantTableIdList determines and returns a list of distributed table * Oids from grant statement. * Grant statement may appear in two forms * 1 - grant on table: * each distributed table oid in grant object list is added to returned list. * 2 - grant all tables in schema: * Collect namespace oid list from grant statement * Add each distributed table oid in the target namespace list to the returned list. */ static List * CollectGrantTableIdList(GrantStmt *grantStmt) { List *grantTableList = NIL; bool grantOnTableCommand = (grantStmt->targtype == ACL_TARGET_OBJECT && grantStmt->objtype == OBJECT_TABLE); bool grantAllTablesOnSchemaCommand = (grantStmt->targtype == ACL_TARGET_ALL_IN_SCHEMA && grantStmt->objtype == OBJECT_TABLE); /* we are only interested in table level grants */ if (!grantOnTableCommand && !grantAllTablesOnSchemaCommand) { return NIL; } if (grantAllTablesOnSchemaCommand) { List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); ListCell *citusTableIdCell = NULL; List *namespaceOidList = NIL; ListCell *objectCell = NULL; foreach(objectCell, grantStmt->objects) { char *nspname = strVal(lfirst(objectCell)); bool missing_ok = false; Oid namespaceOid = get_namespace_oid(nspname, missing_ok); Assert(namespaceOid != InvalidOid); namespaceOidList = list_append_unique_oid(namespaceOidList, namespaceOid); } foreach(citusTableIdCell, citusTableIdList) { Oid relationId = lfirst_oid(citusTableIdCell); Oid namespaceOid = get_rel_namespace(relationId); if (list_member_oid(namespaceOidList, namespaceOid)) { grantTableList = lappend_oid(grantTableList, relationId); } } } else { ListCell *objectCell = NULL; foreach(objectCell, grantStmt->objects) { RangeVar *relvar = (RangeVar *) lfirst(objectCell); Oid relationId = RangeVarGetRelid(relvar, NoLock, false); if (IsCitusTable(relationId)) { grantTableList = lappend_oid(grantTableList, relationId); continue; } /* check for distributed sequences included in GRANT ON TABLE statement */ ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, relationId); if (IsAnyObjectDistributed(list_make1(sequenceAddress))) { grantTableList = lappend_oid(grantTableList, relationId); } } } return grantTableList; } ================================================ FILE: src/backend/distributed/commands/index.c ================================================ /*------------------------------------------------------------------------- * * index.c * Commands for creating and altering indices on distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_namespace.h" #include "commands/defrem.h" #include "commands/tablecmds.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "parser/parse_utilcmd.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/namespace_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/relation_utils.h" #include "distributed/resource_lock.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" /* Local functions forward declarations for helper functions */ static void ErrorIfCreateIndexHasTooManyColumns(IndexStmt *createIndexStatement); static int GetNumberOfIndexParameters(IndexStmt *createIndexStatement); static bool IndexAlreadyExists(IndexStmt *createIndexStatement); static Oid CreateIndexStmtGetIndexId(IndexStmt *createIndexStatement); static Oid CreateIndexStmtGetSchemaId(IndexStmt *createIndexStatement); static void SwitchToSequentialAndLocalExecutionIfIndexNameTooLong(IndexStmt * createIndexStatement); static char * GenerateLongestShardPartitionIndexName(IndexStmt *createIndexStatement); static char * GenerateDefaultIndexName(IndexStmt *createIndexStatement); static List * GenerateIndexParameters(IndexStmt *createIndexStatement); static DDLJob * GenerateCreateIndexDDLJob(IndexStmt *createIndexStatement, const char *createIndexCommand); static Oid CreateIndexStmtGetRelationId(IndexStmt *createIndexStatement); static List * CreateIndexTaskList(IndexStmt *indexStmt); static List * CreateReindexTaskList(Oid relationId, ReindexStmt *reindexStmt); static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg); static void RangeVarCallbackForReindexIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg); static void ErrorIfUnsupportedIndexStmt(IndexStmt *createIndexStatement); static void ErrorIfUnsupportedDropIndexStmt(DropStmt *dropIndexStatement); static List * DropIndexTaskList(Oid relationId, Oid indexId, DropStmt *dropStmt); static Oid ReindexStmtFindRelationOid(ReindexStmt *reindexStmt, bool missingOk); /* * This struct defines the state for the callback for drop statements. * It is copied as it is from commands/tablecmds.c in Postgres source. */ struct DropRelationCallbackState { char relkind; Oid heapOid; bool concurrent; }; /* * This struct defines the state for the callback for reindex statements. * It is copied as it is from commands/indexcmds.c in Postgres source. */ struct ReindexIndexCallbackState { bool concurrent; Oid locked_table_oid; }; /* * IsIndexRenameStmt returns whether the passed-in RenameStmt is the following * form: * * - ALTER INDEX RENAME */ bool IsIndexRenameStmt(RenameStmt *renameStmt) { bool isIndexRenameStmt = false; if (renameStmt->renameType == OBJECT_INDEX) { isIndexRenameStmt = true; } return isIndexRenameStmt; } /* * PreprocessIndexStmt determines whether a given CREATE INDEX statement involves * a distributed table. If so (and if the statement does not use unsupported * options), it modifies the input statement to ensure proper execution against * the coordinator node table and creates a DDLJob to encapsulate information needed * during the worker node portion of DDL execution before returning that DDLJob * in a List. If no distributed table is involved, this function returns NIL. */ List * PreprocessIndexStmt(Node *node, const char *createIndexCommand, ProcessUtilityContext processUtilityContext) { IndexStmt *createIndexStatement = castNode(IndexStmt, node); RangeVar *relationRangeVar = createIndexStatement->relation; if (relationRangeVar == NULL) { /* let's be on the safe side */ return NIL; } /* * We first check whether a distributed relation is affected. For that, * we need to open the relation. To prevent race conditions with later * lookups, lock the table. * * XXX: Consider using RangeVarGetRelidExtended with a permission * checking callback. Right now we'll acquire the lock before having * checked permissions, and will only fail when executing the actual * index statements. */ LOCKMODE lockMode = GetCreateIndexRelationLockMode(createIndexStatement); Relation relation = table_openrv(relationRangeVar, lockMode); /* * Before we do any further processing, fix the schema name to make sure * that a (distributed) table with the same name does not appear on the * search_path in front of the current schema. We do this even if the * table is not distributed, since a distributed table may appear on the * search_path by the time postgres starts processing this command. */ if (relationRangeVar->schemaname == NULL) { /* ensure we copy string into proper context */ MemoryContext relationContext = GetMemoryChunkContext(relationRangeVar); char *namespaceName = RelationGetNamespaceName(relation); relationRangeVar->schemaname = MemoryContextStrdup(relationContext, namespaceName); } Oid relationOid = relation->rd_id; table_close(relation, NoLock); Oid relationId = CreateIndexStmtGetRelationId(createIndexStatement); if (!IsCitusTable(relationId)) { return NIL; } EnsureCoordinator(); if (createIndexStatement->idxname == NULL) { /* * Postgres does not support indexes with over INDEX_MAX_KEYS columns * and we should not attempt to generate an index name for such cases. */ ErrorIfCreateIndexHasTooManyColumns(createIndexStatement); /* * If there are expressions on the index, we should first transform * the statement as the default index name depends on that. We do * it on a copy not to interfere with standard process utility. */ IndexStmt *copyCreateIndexStatement = transformIndexStmt(relationOid, copyObject(createIndexStatement), createIndexCommand); /* ensure we copy string into proper context */ MemoryContext relationContext = GetMemoryChunkContext(relationRangeVar); char *defaultIndexName = GenerateDefaultIndexName(copyCreateIndexStatement); createIndexStatement->idxname = MemoryContextStrdup(relationContext, defaultIndexName); } if (IndexAlreadyExists(createIndexStatement)) { /* * Let standard_processUtility to error out or skip if command has * IF NOT EXISTS. */ return NIL; } ErrorIfUnsupportedIndexStmt(createIndexStatement); /* * Citus has the logic to truncate the long shard names to prevent * various issues, including self-deadlocks. However, for partitioned * tables, when index is created on the parent table, the index names * on the partitions are auto-generated by Postgres. We use the same * Postgres function to generate the index names on the shards of the * partitions. If the length exceeds the limit, we switch to sequential * execution mode. * * The root cause of the problem is that postgres truncates the * table/index names if they are longer than "NAMEDATALEN - 1". * From Citus' perspective, running commands in parallel on the * shards could mean these table/index names are truncated to be * the same, and thus forming a self-deadlock as these tables/ * indexes are inserted into postgres' metadata tables, like pg_class. */ SwitchToSequentialAndLocalExecutionIfIndexNameTooLong(createIndexStatement); DDLJob *ddlJob = GenerateCreateIndexDDLJob(createIndexStatement, createIndexCommand); return list_make1(ddlJob); } /* * ErrorIfCreateIndexHasTooManyColumns errors out if given CREATE INDEX command * would use more than INDEX_MAX_KEYS columns. */ static void ErrorIfCreateIndexHasTooManyColumns(IndexStmt *createIndexStatement) { int numberOfIndexParameters = GetNumberOfIndexParameters(createIndexStatement); if (numberOfIndexParameters <= INDEX_MAX_KEYS) { return; } ereport(ERROR, (errcode(ERRCODE_TOO_MANY_COLUMNS), errmsg("cannot use more than %d columns in an index", INDEX_MAX_KEYS))); } /* * GetNumberOfIndexParameters returns number of parameters to be used when * creating the index to be defined by given CREATE INDEX command. */ static int GetNumberOfIndexParameters(IndexStmt *createIndexStatement) { List *indexParams = createIndexStatement->indexParams; List *indexIncludingParams = createIndexStatement->indexIncludingParams; return list_length(indexParams) + list_length(indexIncludingParams); } /* * IndexAlreadyExists returns true if index to be created by given CREATE INDEX * command already exists. */ static bool IndexAlreadyExists(IndexStmt *createIndexStatement) { Oid indexRelationId = CreateIndexStmtGetIndexId(createIndexStatement); return OidIsValid(indexRelationId); } /* * CreateIndexStmtGetIndexId returns OID of the index that given CREATE INDEX * command attempts to create if it's already created before. Otherwise, returns * InvalidOid. */ static Oid CreateIndexStmtGetIndexId(IndexStmt *createIndexStatement) { char *indexName = createIndexStatement->idxname; Oid namespaceId = CreateIndexStmtGetSchemaId(createIndexStatement); Oid indexRelationId = get_relname_relid(indexName, namespaceId); return indexRelationId; } /* * CreateIndexStmtGetSchemaId returns schemaId of the schema that given * CREATE INDEX command operates on. */ static Oid CreateIndexStmtGetSchemaId(IndexStmt *createIndexStatement) { RangeVar *relationRangeVar = createIndexStatement->relation; char *schemaName = relationRangeVar->schemaname; bool missingOk = false; Oid namespaceId = get_namespace_oid(schemaName, missingOk); return namespaceId; } /* * ExecuteFunctionOnEachTableIndex executes the given pgIndexProcessor function on each * index of the given relation. * It returns a list that is filled by the pgIndexProcessor. */ List * ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcessor, int indexFlags) { List *result = NIL; Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *indexIdList = RelationGetIndexList(relation); Oid indexId = InvalidOid; foreach_declared_oid(indexId, indexIdList) { HeapTuple indexTuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(indexId)); if (!HeapTupleIsValid(indexTuple)) { ereport(ERROR, (errmsg("cache lookup failed for index with oid %u", indexId))); } Form_pg_index indexForm = (Form_pg_index) GETSTRUCT(indexTuple); pgIndexProcessor(indexForm, &result, indexFlags); ReleaseSysCache(indexTuple); } RelationClose(relation); return result; } /* * SwitchToSequentialAndLocalExecutionIfIndexNameTooLong generates the longest index name * on the shards of the partitions, and if exceeds the limit switches to sequential and * local execution to prevent self-deadlocks. */ static void SwitchToSequentialAndLocalExecutionIfIndexNameTooLong(IndexStmt *createIndexStatement) { Oid relationId = CreateIndexStmtGetRelationId(createIndexStatement); if (!PartitionedTable(relationId)) { /* Citus already handles long names for regular tables */ return; } if (ShardIntervalCount(relationId) == 0) { /* * Relation has no shards, so we cannot run into "long shard index * name" issue. */ return; } char *indexName = GenerateLongestShardPartitionIndexName(createIndexStatement); if (indexName && strnlen(indexName, NAMEDATALEN) >= NAMEDATALEN - 1) { if (ParallelQueryExecutedInTransaction()) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg( "The index name (%s) on a shard is too long and could lead " "to deadlocks when executed in a transaction " "block after a parallel query", indexName), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else { elog(DEBUG1, "the index name on the shards of the partition " "is too long, switching to sequential and local execution " "mode to prevent self deadlocks: %s", indexName); SetLocalMultiShardModifyModeToSequential(); SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); } } } /* * GenerateLongestShardPartitionIndexName emulates Postgres index name * generation for partitions on the shards. It returns the longest * possible index name. */ static char * GenerateLongestShardPartitionIndexName(IndexStmt *createIndexStatement) { Oid relationId = CreateIndexStmtGetRelationId(createIndexStatement); Oid longestNamePartitionId = PartitionWithLongestNameRelationId(relationId); if (!OidIsValid(longestNamePartitionId)) { /* no partitions have been created yet */ return NULL; } char *longestPartitionShardName = get_rel_name(longestNamePartitionId); ShardInterval *shardInterval = LoadShardIntervalWithLongestShardName( longestNamePartitionId); AppendShardIdToName(&longestPartitionShardName, shardInterval->shardId); IndexStmt *createLongestShardIndexStmt = copyObject(createIndexStatement); createLongestShardIndexStmt->relation->relname = longestPartitionShardName; char *choosenIndexName = GenerateDefaultIndexName(createLongestShardIndexStmt); return choosenIndexName; } /* * GenerateDefaultIndexName is a wrapper around postgres function ChooseIndexName * that generates default index name for the index to be created by given CREATE * INDEX statement as postgres would do. * * (See DefineIndex at postgres/src/backend/commands/indexcmds.c) */ static char * GenerateDefaultIndexName(IndexStmt *createIndexStatement) { char *relationName = createIndexStatement->relation->relname; Oid namespaceId = CreateIndexStmtGetSchemaId(createIndexStatement); List *indexParams = GenerateIndexParameters(createIndexStatement); List *indexColNames = ChooseIndexColumnNames(indexParams); char *indexName = ChooseIndexName(relationName, namespaceId, indexColNames, createIndexStatement->excludeOpNames, createIndexStatement->primary, createIndexStatement->isconstraint); return indexName; } /* * GenerateIndexParameters is a helper function that creates a list of parameters * required to assign a default index name for the index to be created by given * CREATE INDEX command. */ static List * GenerateIndexParameters(IndexStmt *createIndexStatement) { List *indexParams = createIndexStatement->indexParams; List *indexIncludingParams = createIndexStatement->indexIncludingParams; List *allIndexParams = list_concat(list_copy(indexParams), list_copy(indexIncludingParams)); return allIndexParams; } /* * GenerateCreateIndexDDLJob returns DDLJob for given CREATE INDEX command. */ static DDLJob * GenerateCreateIndexDDLJob(IndexStmt *createIndexStatement, const char *createIndexCommand) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, CreateIndexStmtGetRelationId(createIndexStatement)); ddlJob->startNewTransaction = createIndexStatement->concurrent; ddlJob->metadataSyncCommand = createIndexCommand; ddlJob->taskList = CreateIndexTaskList(createIndexStatement); ddlJob->warnForPartialFailure = true; return ddlJob; } /* * CreateIndexStmtGetRelationId returns relationId for relation that given * CREATE INDEX command operates on. */ static Oid CreateIndexStmtGetRelationId(IndexStmt *createIndexStatement) { RangeVar *relationRangeVar = createIndexStatement->relation; LOCKMODE lockMode = GetCreateIndexRelationLockMode(createIndexStatement); bool missingOk = false; Oid relationId = RangeVarGetRelid(relationRangeVar, lockMode, missingOk); return relationId; } /* * GetCreateIndexRelationLockMode returns required lock mode to open the * relation that given CREATE INDEX command operates on. */ LOCKMODE GetCreateIndexRelationLockMode(IndexStmt *createIndexStatement) { if (createIndexStatement->concurrent) { return ShareUpdateExclusiveLock; } else { return ShareLock; } } /* * ReindexStmtFindRelationOid returns the oid of the relation on which the index exist * if the object is an index in the reindex stmt. It returns the oid of the relation * if the object is a table in the reindex stmt. It also acquires the relevant lock * for the statement. */ static Oid ReindexStmtFindRelationOid(ReindexStmt *reindexStmt, bool missingOk) { Assert(reindexStmt->relation != NULL); Assert(reindexStmt->kind == REINDEX_OBJECT_INDEX || reindexStmt->kind == REINDEX_OBJECT_TABLE); Oid relationId = InvalidOid; LOCKMODE lockmode = IsReindexWithParam_compat(reindexStmt, "concurrently") ? ShareUpdateExclusiveLock : AccessExclusiveLock; if (reindexStmt->kind == REINDEX_OBJECT_INDEX) { struct ReindexIndexCallbackState state; state.concurrent = IsReindexWithParam_compat(reindexStmt, "concurrently"); state.locked_table_oid = InvalidOid; Oid indOid = RangeVarGetRelidExtended(reindexStmt->relation, lockmode, (missingOk) ? RVR_MISSING_OK : 0, RangeVarCallbackForReindexIndex, &state); relationId = IndexGetRelation(indOid, missingOk); } else { relationId = RangeVarGetRelidExtended(reindexStmt->relation, lockmode, (missingOk) ? RVR_MISSING_OK : 0, RangeVarCallbackOwnsTable, NULL); } return relationId; } /* * PreprocessReindexStmt determines whether a given REINDEX statement involves * a distributed table. If so (and if the statement does not use unsupported * options), it modifies the input statement to ensure proper execution against * the coordinator node table and creates a DDLJob to encapsulate information needed * during the worker node portion of DDL execution before returning that DDLJob * in a List. If no distributed table is involved, this function returns NIL. */ List * PreprocessReindexStmt(Node *node, const char *reindexCommand, ProcessUtilityContext processUtilityContext) { ReindexStmt *reindexStatement = castNode(ReindexStmt, node); List *ddlJobs = NIL; /* * We first check whether a distributed relation is affected. For that, we need to * open the relation. To prevent race conditions with later lookups, lock the table, * and modify the rangevar to include the schema. */ if (reindexStatement->relation != NULL) { Oid relationId = ReindexStmtFindRelationOid(reindexStatement, false); MemoryContext relationContext = NULL; Relation relation = NULL; if (reindexStatement->kind == REINDEX_OBJECT_INDEX) { Oid indOid = RangeVarGetRelid(reindexStatement->relation, NoLock, 0); relation = index_open(indOid, NoLock); } else { relation = table_openrv(reindexStatement->relation, NoLock); } bool isCitusRelation = IsCitusTable(relationId); if (reindexStatement->relation->schemaname == NULL) { /* * Before we do any further processing, fix the schema name to make sure * that a (distributed) table with the same name does not appear on the * search path in front of the current schema. We do this even if the * table is not distributed, since a distributed table may appear on the * search path by the time postgres starts processing this statement. */ char *namespaceName = get_namespace_name(RelationGetNamespace(relation)); /* ensure we copy string into proper context */ relationContext = GetMemoryChunkContext(reindexStatement->relation); reindexStatement->relation->schemaname = MemoryContextStrdup(relationContext, namespaceName); } if (reindexStatement->kind == REINDEX_OBJECT_INDEX) { index_close(relation, NoLock); } else { table_close(relation, NoLock); } if (isCitusRelation) { if (PartitionedTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("REINDEX TABLE queries on distributed partitioned " "tables are not supported"))); } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = IsReindexWithParam_compat(reindexStatement, "concurrently"); ddlJob->metadataSyncCommand = reindexCommand; ddlJob->taskList = CreateReindexTaskList(relationId, reindexStatement); ddlJob->warnForPartialFailure = true; ddlJobs = list_make1(ddlJob); } } return ddlJobs; } /* * ReindexStmtObjectAddress returns list of object addresses in the reindex * statement. We add the address if the object is either index or table; * else, we add invalid address. */ List * ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess) { ReindexStmt *reindexStatement = castNode(ReindexStmt, stmt); Oid relationId = InvalidOid; if (reindexStatement->relation != NULL) { /* we currently only support reindex commands on tables */ relationId = ReindexStmtFindRelationOid(reindexStatement, missing_ok); } ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, RelationRelationId, relationId); return list_make1(objectAddress); } /* * PreprocessDropIndexStmt determines whether a given DROP INDEX statement involves * a distributed table. If so (and if the statement does not use unsupported * options), it modifies the input statement to ensure proper execution against * the coordinator node table and creates a DDLJob to encapsulate information needed * during the worker node portion of DDL execution before returning that DDLJob * in a List. If no distributed table is involved, this function returns NIL. */ List * PreprocessDropIndexStmt(Node *node, const char *dropIndexCommand, ProcessUtilityContext processUtilityContext) { DropStmt *dropIndexStatement = castNode(DropStmt, node); List *ddlJobs = NIL; Oid distributedIndexId = InvalidOid; Oid distributedRelationId = InvalidOid; Assert(dropIndexStatement->removeType == OBJECT_INDEX); /* check if any of the indexes being dropped belong to a distributed table */ List *objectNameList = NULL; foreach_declared_ptr(objectNameList, dropIndexStatement->objects) { struct DropRelationCallbackState state; uint32 rvrFlags = RVR_MISSING_OK; LOCKMODE lockmode = AccessExclusiveLock; RangeVar *rangeVar = makeRangeVarFromNameList(objectNameList); /* * We don't support concurrently dropping indexes for distributed * tables, but till this point, we don't know if it is a regular or a * distributed table. */ if (dropIndexStatement->concurrent) { lockmode = ShareUpdateExclusiveLock; } /* * The next few statements are based on RemoveRelations() in * commands/tablecmds.c in Postgres source. */ AcceptInvalidationMessages(); state.relkind = RELKIND_INDEX; state.heapOid = InvalidOid; state.concurrent = dropIndexStatement->concurrent; Oid indexId = RangeVarGetRelidExtended(rangeVar, lockmode, rvrFlags, RangeVarCallbackForDropIndex, (void *) &state); /* * If the index does not exist, we don't do anything here, and allow * postgres to throw appropriate error or notice message later. */ if (!OidIsValid(indexId)) { continue; } Oid relationId = IndexGetRelation(indexId, false); bool isCitusRelation = IsCitusTable(relationId); if (isCitusRelation) { distributedIndexId = indexId; distributedRelationId = relationId; break; } } if (OidIsValid(distributedIndexId)) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ErrorIfUnsupportedDropIndexStmt(dropIndexStatement); if (AnyForeignKeyDependsOnIndex(distributedIndexId)) { MarkInvalidateForeignKeyGraph(); } ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, distributedRelationId); /* * We do not want DROP INDEX CONCURRENTLY to commit locally before * sending commands, because if there is a failure we would like to * to be able to repeat the DROP INDEX later. */ ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = dropIndexCommand; ddlJob->taskList = DropIndexTaskList(distributedRelationId, distributedIndexId, dropIndexStatement); ddlJob->warnForPartialFailure = true; ddlJobs = list_make1(ddlJob); } return ddlJobs; } /* * PostprocessIndexStmt marks new indexes invalid if they were created using the * CONCURRENTLY flag. This (non-transactional) change provides the fallback * state if an error is raised, otherwise a sub-sequent change to valid will be * committed. */ List * PostprocessIndexStmt(Node *node, const char *queryString) { IndexStmt *indexStmt = castNode(IndexStmt, node); /* this logic only applies to the coordinator */ if (!IsCoordinator()) { return NIL; } /* * We make sure schema name is not null in the PreprocessIndexStmt */ Oid schemaId = get_namespace_oid(indexStmt->relation->schemaname, true); Oid relationId = get_relname_relid(indexStmt->relation->relname, schemaId); if (!IsCitusTable(relationId)) { return NIL; } Oid indexRelationId = get_relname_relid(indexStmt->idxname, schemaId); /* ensure dependencies of index exist on all nodes */ ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, RelationRelationId, indexRelationId); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(address)); /* furtheron we are only processing CONCURRENT index statements */ if (!indexStmt->concurrent) { return NIL; } /* * EnsureAllObjectDependenciesExistOnAllNodes could have distributed objects that are required * by this index. During the propagation process an active snapshout might be left as * a side effect of inserting the local tuples via SPI. To not leak a snapshot like * that we will pop any snapshot if we have any right before we commit. */ if (ActiveSnapshotSet()) { PopActiveSnapshot(); } /* commit the current transaction and start anew */ CommitTransactionCommand(); StartTransactionCommand(); /* get the affected relation and index */ Relation relation = table_openrv(indexStmt->relation, ShareUpdateExclusiveLock); Relation indexRelation = index_open(indexRelationId, RowExclusiveLock); /* close relations but retain locks */ table_close(relation, NoLock); index_close(indexRelation, NoLock); PushActiveSnapshot(GetTransactionSnapshot()); /* mark index as invalid, in-place (cannot be rolled back) */ index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); PopActiveSnapshot(); /* re-open a transaction command from here on out */ CommitTransactionCommand(); StartTransactionCommand(); return NIL; } /* * ErrorIfUnsupportedAlterIndexStmt checks if the corresponding alter index * statement is supported for distributed tables and errors out if it is not. * Currently, only the following commands are supported. * * ALTER INDEX SET () * ALTER INDEX RESET () * ALTER INDEX ALTER COLUMN SET STATISTICS */ void ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement) { /* error out if any of the subcommands are unsupported */ List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; switch (alterTableType) { case AT_SetRelOptions: /* SET (...) */ case AT_ResetRelOptions: /* RESET (...) */ case AT_ReplaceRelOptions: /* replace entire option list */ case AT_SetStatistics: /* SET STATISTICS */ case AT_AttachPartition: /* ATTACH PARTITION */ { /* this command is supported by Citus */ break; } /* unsupported create index statements */ case AT_SetTableSpace: default: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("alter index ... set tablespace ... " "is currently unsupported"), errdetail("Only RENAME TO, SET (), RESET (), ATTACH PARTITION " "and SET STATISTICS are supported."))); return; /* keep compiler happy */ } } } } /* * CreateIndexTaskList builds a list of tasks to execute a CREATE INDEX command. */ static List * CreateIndexTaskList(IndexStmt *indexStmt) { List *taskList = NIL; Oid relationId = CreateIndexStmtGetRelationId(indexStmt); List *shardIntervalList = LoadShardIntervalList(relationId); StringInfoData ddlString; uint64 jobId = INVALID_JOB_ID; int taskId = 1; initStringInfo(&ddlString); /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; deparse_shard_index_statement(indexStmt, relationId, shardId, &ddlString); Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, pstrdup(ddlString.data)); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); task->cannotBeExecutedInTransaction = indexStmt->concurrent; taskList = lappend(taskList, task); resetStringInfo(&ddlString); } return taskList; } /* * CreateReindexTaskList builds a list of tasks to execute a REINDEX command * against a specified distributed table. */ static List * CreateReindexTaskList(Oid relationId, ReindexStmt *reindexStmt) { List *taskList = NIL; List *shardIntervalList = LoadShardIntervalList(relationId); StringInfoData ddlString; uint64 jobId = INVALID_JOB_ID; int taskId = 1; initStringInfo(&ddlString); /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; deparse_shard_reindex_statement(reindexStmt, relationId, shardId, &ddlString); Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, pstrdup(ddlString.data)); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); task->cannotBeExecutedInTransaction = IsReindexWithParam_compat(reindexStmt, "concurrently"); taskList = lappend(taskList, task); resetStringInfo(&ddlString); } return taskList; } /* * Before acquiring a table lock, check whether we have sufficient rights. * In the case of DROP INDEX, also try to lock the table before the index. * * This code is heavily borrowed from RangeVarCallbackForDropRelation() in * commands/tablecmds.c in Postgres source. We need this to ensure the right * order of locking while dealing with DROP INDEX statements. Because we are * exclusively using this callback for INDEX processing, the PARTITION-related * logic from PostgreSQL's similar callback has been omitted as unneeded. */ static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg) { /* *INDENT-OFF* */ HeapTuple tuple; char relkind; char expected_relkind; LOCKMODE heap_lockmode; struct DropRelationCallbackState *state = (struct DropRelationCallbackState *) arg; relkind = state->relkind; heap_lockmode = state->concurrent ? ShareUpdateExclusiveLock : AccessExclusiveLock; Assert(relkind == RELKIND_INDEX); /* * If we previously locked some other index's heap, and the name we're * looking up no longer refers to that relation, release the now-useless * lock. */ if (relOid != oldRelOid && OidIsValid(state->heapOid)) { UnlockRelationOid(state->heapOid, heap_lockmode); state->heapOid = InvalidOid; } /* Didn't find a relation, so no need for locking or permission checks. */ if (!OidIsValid(relOid)) return; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relOid)); if (!HeapTupleIsValid(tuple)) return; /* concurrently dropped, so nothing to do */ Form_pg_class classform = (Form_pg_class) GETSTRUCT(tuple); /* * PG 11 sends relkind as partitioned index for an index * on partitioned table. It is handled the same * as regular index as far as we are concerned here. * * See tablecmds.c:RangeVarCallbackForDropRelation() */ expected_relkind = classform->relkind; if (expected_relkind == RELKIND_PARTITIONED_INDEX) expected_relkind = RELKIND_INDEX; if (expected_relkind != relkind) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", rel->relname))); /* Allow DROP to either table owner or schema owner */ if (!object_ownercheck(RelationRelationId, relOid, GetUserId()) && !object_ownercheck(NamespaceRelationId, classform->relnamespace, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, rel->relname); } if (!allowSystemTableMods && IsSystemClass(relOid, classform)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied: \"%s\" is a system catalog", rel->relname))); ReleaseSysCache(tuple); /* * In DROP INDEX, attempt to acquire lock on the parent table before * locking the index. index_drop() will need this anyway, and since * regular queries lock tables before their indexes, we risk deadlock if * we do it the other way around. No error if we don't find a pg_index * entry, though --- the relation may have been dropped. */ if (relkind == RELKIND_INDEX && relOid != oldRelOid) { state->heapOid = IndexGetRelation(relOid, true); if (OidIsValid(state->heapOid)) LockRelationOid(state->heapOid, heap_lockmode); } /* *INDENT-ON* */ } /* * Check permissions on table before acquiring relation lock; also lock * the heap before the RangeVarGetRelidExtended takes the index lock, to avoid * deadlocks. * * This code is borrowed from RangeVarCallbackForReindexIndex() in * commands/indexcmds.c in Postgres source. We need this to ensure the right * order of locking while dealing with REINDEX statements. */ static void RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg) { /* *INDENT-OFF* */ char relkind; struct ReindexIndexCallbackState *state = arg; LOCKMODE table_lockmode; Oid table_oid; /* * Lock level here should match table lock in reindex_index() for * non-concurrent case and table locks used by index_concurrently_*() for * concurrent case. */ table_lockmode = state->concurrent ? ShareUpdateExclusiveLock : ShareLock; /* * If we previously locked some other index's heap, and the name we're * looking up no longer refers to that relation, release the now-useless * lock. */ if (relId != oldRelId && OidIsValid(oldRelId)) { UnlockRelationOid(state->locked_table_oid, table_lockmode); state->locked_table_oid = InvalidOid; } /* If the relation does not exist, there's nothing more to do. */ if (!OidIsValid(relId)) return; /* * If the relation does exist, check whether it's an index. But note that * the relation might have been dropped between the time we did the name * lookup and now. In that case, there's nothing to do. */ relkind = get_rel_relkind(relId); if (!relkind) return; if (relkind != RELKIND_INDEX && relkind != RELKIND_PARTITIONED_INDEX) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", relation->relname))); /* Check permissions */ #if PG_VERSION_NUM >= PG_VERSION_17 table_oid = IndexGetRelation(relId, true); if (OidIsValid(table_oid)) { AclResult aclresult = pg_class_aclcheck(table_oid, GetUserId(), ACL_MAINTAIN); if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_INDEX, relation->relname); } #else if (!object_ownercheck(RelationRelationId, relId, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relation->relname); #endif /* Lock heap before index to avoid deadlock. */ if (relId != oldRelId) { table_oid = IndexGetRelation(relId, true); /* * If the OID isn't valid, it means the index was concurrently * dropped, which is not a problem for us; just return normally. */ if (OidIsValid(table_oid)) { LockRelationOid(table_oid, table_lockmode); state->locked_table_oid = table_oid; } } /* *INDENT-ON* */ } /* * ErrorIfUnsupportedIndexStmt checks if the corresponding index statement is * supported for distributed tables and errors out if it is not. */ static void ErrorIfUnsupportedIndexStmt(IndexStmt *createIndexStatement) { if (createIndexStatement->tableSpace != NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("specifying tablespaces with CREATE INDEX statements is " "currently unsupported"))); } if (createIndexStatement->unique) { RangeVar *relation = createIndexStatement->relation; bool missingOk = false; /* caller uses ShareLock for non-concurrent indexes, use the same lock here */ LOCKMODE lockMode = ShareLock; Oid relationId = RangeVarGetRelid(relation, lockMode, missingOk); bool indexContainsPartitionColumn = false; /* * Non-distributed tables do not have partition key, and unique constraints * are allowed for them. Thus, we added a short-circuit for non-distributed tables. */ if (!HasDistributionKey(relationId)) { return; } if (IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("creating unique indexes on append-partitioned tables " "is currently unsupported"))); } if (AllowUnsafeConstraints) { /* * The user explicitly wants to allow the constraint without * distribution column. */ return; } Var *partitionKey = DistPartitionKeyOrError(relationId); List *indexParameterList = createIndexStatement->indexParams; IndexElem *indexElement = NULL; foreach_declared_ptr(indexElement, indexParameterList) { const char *columnName = indexElement->name; /* column name is null for index expressions, skip it */ if (columnName == NULL) { continue; } AttrNumber attributeNumber = get_attnum(relationId, columnName); if (attributeNumber == partitionKey->varattno) { indexContainsPartitionColumn = true; } } if (!indexContainsPartitionColumn) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("creating unique indexes on non-partition " "columns is currently unsupported"))); } } } /* * ErrorIfUnsupportedDropIndexStmt checks if the corresponding drop index statement is * supported for distributed tables and errors out if it is not. */ static void ErrorIfUnsupportedDropIndexStmt(DropStmt *dropIndexStatement) { Assert(dropIndexStatement->removeType == OBJECT_INDEX); if (list_length(dropIndexStatement->objects) > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop multiple distributed objects in a " "single command"), errhint("Try dropping each object in a separate DROP " "command."))); } } /* * DropIndexTaskList builds a list of tasks to execute a DROP INDEX command * against a specified distributed table. */ static List * DropIndexTaskList(Oid relationId, Oid indexId, DropStmt *dropStmt) { List *taskList = NIL; List *shardIntervalList = LoadShardIntervalList(relationId); char *indexName = get_rel_name(indexId); Oid schemaId = get_rel_namespace(indexId); char *schemaName = get_namespace_name(schemaId); StringInfoData ddlString; uint64 jobId = INVALID_JOB_ID; int taskId = 1; initStringInfo(&ddlString); /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardIndexName = pstrdup(indexName); AppendShardIdToName(&shardIndexName, shardId); /* deparse shard-specific DROP INDEX command */ appendStringInfo(&ddlString, "DROP INDEX %s %s %s %s", (dropStmt->concurrent ? "CONCURRENTLY" : ""), (dropStmt->missing_ok ? "IF EXISTS" : ""), quote_qualified_identifier(schemaName, shardIndexName), (dropStmt->behavior == DROP_RESTRICT ? "RESTRICT" : "CASCADE")); Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, pstrdup(ddlString.data)); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); task->cannotBeExecutedInTransaction = dropStmt->concurrent; taskList = lappend(taskList, task); resetStringInfo(&ddlString); } return taskList; } /* * MarkIndexValid marks an index as valid after a CONCURRENTLY command. We mark * indexes invalid in PostProcessIndexStmt and then commit, such that any failure * leaves behind an invalid index. We mark it as valid here when the command * completes. */ void MarkIndexValid(IndexStmt *indexStmt) { Assert(indexStmt->concurrent); Assert(IsCoordinator()); /* * We make sure schema name is not null in the PreprocessIndexStmt */ bool missingOk = false; Oid schemaId = get_namespace_oid(indexStmt->relation->schemaname, missingOk); Oid relationId PG_USED_FOR_ASSERTS_ONLY = get_relname_relid(indexStmt->relation->relname, schemaId); Assert(IsCitusTable(relationId)); /* get the affected relation and index */ Relation relation = table_openrv(indexStmt->relation, ShareUpdateExclusiveLock); Oid indexRelationId = get_relname_relid(indexStmt->idxname, schemaId); Relation indexRelation = index_open(indexRelationId, RowExclusiveLock); PushActiveSnapshot(GetTransactionSnapshot()); /* mark index as valid, in-place (cannot be rolled back) */ index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); PopActiveSnapshot(); table_close(relation, NoLock); index_close(indexRelation, NoLock); } ================================================ FILE: src/backend/distributed/commands/index_pg_source.c ================================================ /*------------------------------------------------------------------------- * * index_pg_source.c * Helper functions copy & pasted from PostgreSQL source code. * All the functions in this file is copied from * postgres/src/backend/commands/indexcmds.c * * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/distributed/commands/index_pg_source.c * *------------------------------------------------------------------------- */ #include "postgres.h" #include "commands/defrem.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "mb/pg_wchar.h" /* *INDENT-OFF* */ /* * This function is copy & paste from Postgres source code: * postgres/src/backend/commands/indexcmds.c * * Select the name to be used for an index. * * The argument list is pretty ad-hoc :-( */ char * ChooseIndexName(const char *tabname, Oid namespaceId, List *colnames, List *exclusionOpNames, bool primary, bool isconstraint) { char *indexname; if (primary) { /* the primary key's name does not depend on the specific column(s) */ indexname = ChooseRelationName(tabname, NULL, "pkey", namespaceId, true); } else if (exclusionOpNames != NIL) { indexname = ChooseRelationName(tabname, ChooseIndexNameAddition(colnames), "excl", namespaceId, true); } else if (isconstraint) { indexname = ChooseRelationName(tabname, ChooseIndexNameAddition(colnames), "key", namespaceId, true); } else { indexname = ChooseRelationName(tabname, ChooseIndexNameAddition(colnames), "idx", namespaceId, false); } return indexname; } /* * This function is copy & paste from Postgres source code: * postgres/src/backend/commands/indexcmds.c * * Generate "name2" for a new index given the list of column names for it * (as produced by ChooseIndexColumnNames). This will be passed to * ChooseRelationName along with the parent table name and a suitable label. * * We know that less than NAMEDATALEN characters will actually be used, * so we can truncate the result once we've generated that many. * * XXX See also ChooseForeignKeyConstraintNameAddition and * ChooseExtendedStatisticNameAddition. */ char * ChooseIndexNameAddition(List *colnames) { char buf[NAMEDATALEN * 2]; int buflen = 0; ListCell *lc; buf[0] = '\0'; foreach(lc, colnames) { const char *name = (const char *) lfirst(lc); if (buflen > 0) { buf[buflen++] = '_'; /* insert _ between names */ } /* * At this point we have buflen <= NAMEDATALEN. name should be less * than NAMEDATALEN already, but use strlcpy for paranoia. */ strlcpy(buf + buflen, name, NAMEDATALEN); buflen += strlen(buf + buflen); if (buflen >= NAMEDATALEN) { break; } } return pstrdup(buf); } /* * * This function is copy & paste from Postgres source code: * postgres/src/backend/commands/indexcmds.c * * Select the actual names to be used for the columns of an index, given the * list of IndexElems for the columns. This is mostly about ensuring the * names are unique so we don't get a conflicting-attribute-names error. * * Returns a List of plain strings (char *, not String nodes). */ List * ChooseIndexColumnNames(List *indexElems) { List *result = NIL; ListCell *lc; foreach(lc, indexElems) { IndexElem *ielem = (IndexElem *) lfirst(lc); const char *origname; const char *curname; int i; char buf[NAMEDATALEN]; /* Get the preliminary name from the IndexElem */ if (ielem->indexcolname) { origname = ielem->indexcolname; /* caller-specified name */ } else if (ielem->name) { origname = ielem->name; /* simple column reference */ } else { origname = "expr"; /* default name for expression */ } /* If it conflicts with any previous column, tweak it */ curname = origname; for (i = 1;; i++) { ListCell *lc2; char nbuf[32]; int nlen; foreach(lc2, result) { if (strcmp(curname, (char *) lfirst(lc2)) == 0) { break; } } if (lc2 == NULL) { break; /* found nonconflicting name */ } sprintf(nbuf, "%d", i); /* lgtm[cpp/banned-api-usage-required-any] */ /* Ensure generated names are shorter than NAMEDATALEN */ nlen = pg_mbcliplen(origname, strlen(origname), NAMEDATALEN - 1 - strlen(nbuf)); memcpy(buf, origname, nlen); /* lgtm[cpp/banned-api-usage-required-any] */ strcpy(buf + nlen, nbuf); /* lgtm[cpp/banned-api-usage-required-any] */ curname = buf; } /* And attach to the result list */ result = lappend(result, pstrdup(curname)); } return result; } /* *INDENT-ON* */ ================================================ FILE: src/backend/distributed/commands/local_multi_copy.c ================================================ /*------------------------------------------------------------------------- * * local_multi_copy.c * Commands for running a copy locally * * For each local placement, we have a buffer. When we receive a slot * from a copy, the slot will be put to the corresponding buffer based * on the shard id. When the buffer size exceeds the threshold a local * copy will be done. Also If we reach to the end of copy, we will send * the current buffer for local copy. * * The existing logic from multi_copy.c and format are used, therefore * even if user did not do a copy with binary format, it is possible that * we are going to be using binary format internally. * * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include /* for htons */ #include "postgres.h" #include "safe_lib.h" #include "catalog/namespace.h" #include "commands/copy.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "utils/lsyscache.h" #include "distributed/commands/multi_copy.h" #include "distributed/intermediate_results.h" #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/shard_utils.h" #include "distributed/transmit.h" #include "distributed/version_compat.h" /* managed via GUC, default is 512 kB */ int LocalCopyFlushThresholdByte = 512 * 1024; static void AddSlotToBuffer(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, CopyOutState localCopyOutState); static bool ShouldAddBinaryHeaders(StringInfo buffer, bool isBinary); static bool ShouldSendCopyNow(StringInfo buffer); static void DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStatement, bool isEndOfCopy, bool isPublishable); static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead); /* * LocalCopyBuffer is used in copy callback to return the copied rows. * The reason this is a global variable is that we cannot pass an additional * argument to the copy callback. */ static StringInfo LocalCopyBuffer; /* * WriteTupleToLocalShard adds the given slot and does a local copy if * this is the end of copy, or the buffer size exceeds the threshold. */ void WriteTupleToLocalShard(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localCopyOutState) { /* * Since we are doing a local copy, the following statements should * use local execution to see the changes */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); bool isBinaryCopy = localCopyOutState->binary; if (ShouldAddBinaryHeaders(localCopyOutState->fe_msgbuf, isBinaryCopy)) { AppendCopyBinaryHeaders(localCopyOutState); } AddSlotToBuffer(slot, copyDest, localCopyOutState); if (ShouldSendCopyNow(localCopyOutState->fe_msgbuf)) { if (isBinaryCopy) { /* * We're going to flush the buffer to disk by effectively doing a full * COPY command. Hence we also need to add footers to the current buffer. */ AppendCopyBinaryFooters(localCopyOutState); } bool isEndOfCopy = false; DoLocalCopy(localCopyOutState->fe_msgbuf, copyDest->distributedRelationId, shardId, copyDest->copyStatement, isEndOfCopy, copyDest->isPublishable); resetStringInfo(localCopyOutState->fe_msgbuf); } } /* * WriteTupleToLocalFile adds the given slot and does a local copy to the * file if the buffer size exceeds the threshold. */ void WriteTupleToLocalFile(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localFileCopyOutState, FileCompat *fileCompat) { AddSlotToBuffer(slot, copyDest, localFileCopyOutState); if (ShouldSendCopyNow(localFileCopyOutState->fe_msgbuf)) { WriteToLocalFile(localFileCopyOutState->fe_msgbuf, fileCompat); resetStringInfo(localFileCopyOutState->fe_msgbuf); } } /* * FinishLocalCopyToShard finishes local copy for the given shard with the shard id. */ void FinishLocalCopyToShard(CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localCopyOutState) { bool isBinaryCopy = localCopyOutState->binary; if (isBinaryCopy) { AppendCopyBinaryFooters(localCopyOutState); } bool isEndOfCopy = true; DoLocalCopy(localCopyOutState->fe_msgbuf, copyDest->distributedRelationId, shardId, copyDest->copyStatement, isEndOfCopy, copyDest->isPublishable); } /* * FinishLocalCopyToFile finishes local copy for the given file. */ void FinishLocalCopyToFile(CopyOutState localFileCopyOutState, FileCompat *fileCompat) { StringInfo data = localFileCopyOutState->fe_msgbuf; bool isBinaryCopy = localFileCopyOutState->binary; if (isBinaryCopy) { AppendCopyBinaryFooters(localFileCopyOutState); } WriteToLocalFile(data, fileCompat); resetStringInfo(localFileCopyOutState->fe_msgbuf); FileClose(fileCompat->fd); } /* * AddSlotToBuffer serializes the given slot and adds it to * the buffer in localCopyOutState. */ static void AddSlotToBuffer(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, CopyOutState localCopyOutState) { Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; FmgrInfo *columnOutputFunctions = copyDest->columnOutputFunctions; CopyCoercionData *columnCoercionPaths = copyDest->columnCoercionPaths; AppendCopyRowData(columnValues, columnNulls, copyDest->tupleDescriptor, localCopyOutState, columnOutputFunctions, columnCoercionPaths); } /* * ShouldSendCopyNow returns true if the given buffer size exceeds the * local copy buffer size threshold. */ static bool ShouldSendCopyNow(StringInfo buffer) { /* LocalCopyFlushThreshold is in bytes */ return buffer->len > LocalCopyFlushThresholdByte; } /* * DoLocalCopy finds the shard table from the distributed relation id, and copies the given * buffer into the shard. * CopyFrom calls ReadFromLocalBufferCallback to read bytes from the buffer * as though it was reading from stdin. It then parses the tuples and * writes them to the shardOid table. */ static void DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStatement, bool isEndOfCopy, bool isPublishable) { /* * Set the buffer as a global variable to allow ReadFromLocalBufferCallback * to read from it. We cannot pass additional arguments to * ReadFromLocalBufferCallback. */ LocalCopyBuffer = buffer; if (!isPublishable) { SetupReplicationOriginLocalSession(); } Oid shardOid = GetTableLocalShardOid(relationId, shardId); Relation shard = table_open(shardOid, RowExclusiveLock); ParseState *pState = make_parsestate(NULL); (void) addRangeTableEntryForRelation(pState, shard, AccessShareLock, NULL, false, false); CopyFromState cstate = BeginCopyFrom(pState, shard, NULL, NULL, false, ReadFromLocalBufferCallback, copyStatement->attlist, copyStatement->options); CopyFrom(cstate); EndCopyFrom(cstate); table_close(shard, NoLock); if (!isPublishable) { ResetReplicationOriginLocalSession(); } free_parsestate(pState); } /* * ShouldAddBinaryHeaders returns true if the given buffer * is empty and the format is binary. */ static bool ShouldAddBinaryHeaders(StringInfo buffer, bool isBinary) { if (!isBinary) { return false; } return buffer->len == 0; } /* * ReadFromLocalBufferCallback is the copy callback. * It always tries to copy maxRead bytes. */ static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead) { int bytesRead = 0; int avail = LocalCopyBuffer->len - LocalCopyBuffer->cursor; int bytesToRead = Min(avail, maxRead); if (bytesToRead > 0) { memcpy_s(outBuf, bytesToRead, &LocalCopyBuffer->data[LocalCopyBuffer->cursor], bytesToRead); } bytesRead += bytesToRead; LocalCopyBuffer->cursor += bytesToRead; return bytesRead; } ================================================ FILE: src/backend/distributed/commands/multi_copy.c ================================================ /*------------------------------------------------------------------------- * * multi_copy.c * This file contains implementation of COPY utility for distributed * tables. * * The CitusCopyFrom function should be called from the utility hook to process * COPY ... FROM commands on distributed tables. CitusCopyFrom parses the input * from stdin, a program, or a file, and decides to copy new rows to existing * shards or new shards based on the partition method of the distributed table. * * If this is the first command in the transaction, we open a new connection for * every shard placement. Otherwise we open as many connections as we can to * not conflict with previous commands in transactions, in which case some shards * may share connections. See the comments of CopyConnectionState for how we * operate in that case. * * We use the PQputCopyData function to copy the data. Because PQputCopyData * transmits data asynchronously, the workers will ingest data at least partially * in parallel. * * For hash-partitioned tables, if it fails to connect to a worker, the master * rollbacks the distributed transaction, similar to the way DML statements * are handled. If a failure occurs after connecting, the transaction * is rolled back on all the workers. Note that, * in the case of append-partitioned tables, if a fail occurs, immediately * metadata changes are rolled back on the master node, but shard placements * are left on the worker nodes. * * By default, COPY uses normal transactions on the workers. In the case of * hash or range-partitioned tables, this can cause a problem when some of the * transactions fail to commit while others have succeeded. To ensure no data * is lost, COPY uses two-phase commit. * * Parsing options are processed and enforced on the node where copy command * is run, while constraints are enforced on the worker. In either case, * failure causes the whole COPY to roll back. * * Copyright (c) Citus Data, Inc. * * With contributions from Postgres Professional. * *------------------------------------------------------------------------- */ #include /* for htons */ #include /* for htons */ #include #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/sdir.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_attribute.h" #include "catalog/pg_type.h" #include "commands/copy.h" #include "commands/defrem.h" #include "commands/progress.h" #include "executor/executor.h" #include "foreign/foreign.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_func.h" #include "parser/parse_type.h" #include "tcop/cmdtag.h" #include "tsearch/ts_locale.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/hash_helpers.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #include "distributed/relation_utils.h" #include "distributed/remote_commands.h" #include "distributed/remote_transaction.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/resource_lock.h" #include "distributed/shard_pruning.h" #include "distributed/shared_connection_stats.h" #include "distributed/stats/stat_counters.h" #include "distributed/transmit.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* constant used in binary protocol */ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* if true, skip validation of JSONB columns during COPY */ bool SkipJsonbValidationInCopy = true; /* custom Citus option for appending to a shard */ #define APPEND_TO_SHARD_OPTION "append_to_shard" /* * Data size threshold to switch over the active placement for a connection. * If this is too low, overhead of starting COPY commands will hurt the * performance. If this is too high, buffered data will use lots of memory. * 4MB is a good balance between memory usage and performance. Note that this * is irrelevant in the common case where we open one connection per placement. */ int CopySwitchOverThresholdBytes = 4 * 1024 * 1024; #define FILE_IS_OPEN(x) (x > -1) typedef struct CopyShardState CopyShardState; typedef struct CopyPlacementState CopyPlacementState; /* * Multiple shard placements can share one connection. Each connection has one * of those placements as the activePlacementState, and others in the * bufferedPlacementList. When we want to send a tuple to a CopyPlacementState, * we check if it is the active one in its connectionState, and in this case we * directly put data on wire. Otherwise, we buffer it so we can put it on wire * later, when copy ends or a switch-over happens. See CitusSendTupleToPlacements() * for more details. * * This is done so we are compatible with adaptive_executor. If a previous command * in the current transaction has been executed using adaptive_executor.c, then * CopyGetPlacementConnection() might return the same connection for multiple * placements. We support that case by the buffering mechanism described above. * * If no previous command in the current transaction has used adaptive_executor.c, * then CopyGetPlacementConnection() returns one connection per placement and no * buffering happens and we put the copy data directly on connection. */ typedef struct CopyConnectionState { /* Used as hash key. Equal to PQsocket(connection->pgConn). */ int socket; MultiConnection *connection; /* * Placement for which we have an active COPY going on over connection. * Can be NULL. */ CopyPlacementState *activePlacementState; /* * Other placements that we are buffering data for. Later when a switch-over * happens, we remove an item from this list and set it to activePlacementState. * In this case, old activePlacementState isn't NULL, is added to this list. */ dlist_head bufferedPlacementList; /* length of bufferedPlacementList, to avoid iterations over the list when needed */ int bufferedPlacementCount; } CopyConnectionState; struct CopyPlacementState { /* Connection state to which the placemement is assigned to. */ CopyConnectionState *connectionState; /* State of shard to which the placement belongs to. */ CopyShardState *shardState; /* node group ID of the placement */ int32 groupId; /* * Buffered COPY data. When the placement is activePlacementState of * some connection, this is empty. Because in that case we directly * send the data over connection. */ StringInfo data; /* List node for CopyConnectionState->bufferedPlacementList. */ dlist_node bufferedPlacementNode; }; struct CopyShardState { /* Used as hash key. */ uint64 shardId; /* used for doing local copy, either for a shard or a co-located file */ CopyOutState copyOutState; /* used when copy is targeting co-located file */ FileCompat fileDest; /* containsLocalPlacement is true if we have a local placement for the shard id of this state */ bool containsLocalPlacement; /* List of CopyPlacementStates for all active placements of the shard. */ List *placementStateList; }; /* * Represents the state for allowing copy via local * execution. */ typedef enum LocalCopyStatus { LOCAL_COPY_REQUIRED, LOCAL_COPY_OPTIONAL, LOCAL_COPY_DISABLED } LocalCopyStatus; /* Local functions forward declarations */ static void CopyToExistingShards(CopyStmt *copyStatement, QueryCompletion *completionTag); static bool IsCopyInBinaryFormat(CopyStmt *copyStatement); static List * FindJsonbInputColumns(TupleDesc tupleDescriptor, List *inputColumnNameList); static List * RemoveOptionFromList(List *optionList, char *optionName); static bool BinaryOutputFunctionDefined(Oid typeId); static bool BinaryInputFunctionDefined(Oid typeId); static void SendCopyBinaryHeaders(CopyOutState copyOutState, int64 shardId, List *connectionList); static void SendCopyBinaryFooters(CopyOutState copyOutState, int64 shardId, List *connectionList); static StringInfo ConstructCopyStatement(CopyStmt *copyStatement, int64 shardId); static void SendCopyDataToAll(StringInfo dataBuffer, int64 shardId, List *connectionList); static void SendCopyDataToPlacement(StringInfo dataBuffer, int64 shardId, MultiConnection *connection); static uint32 AvailableColumnCount(TupleDesc tupleDescriptor); static Oid TypeForColumnName(Oid relationId, TupleDesc tupleDescriptor, char *columnName); static Oid * TypeArrayFromTupleDescriptor(TupleDesc tupleDescriptor); static CopyCoercionData * ColumnCoercionPaths(TupleDesc destTupleDescriptor, TupleDesc inputTupleDescriptor, Oid destRelId, List *columnNameList, Oid *finalColumnTypeArray); static FmgrInfo * TypeOutputFunctions(uint32 columnCount, Oid *typeIdArray, bool binaryFormat); static bool CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName); static void CitusCopyFrom(CopyStmt *copyStatement, QueryCompletion *completionTag); static void EnsureCopyCanRunOnRelation(Oid relationId); static HTAB * CreateConnectionStateHash(MemoryContext memoryContext); static HTAB * CreateShardStateHash(MemoryContext memoryContext); static CopyConnectionState * GetConnectionState(HTAB *connectionStateHash, MultiConnection *connection); static CopyShardState * GetShardState(uint64 shardId, HTAB *shardStateHash, HTAB *connectionStateHash, bool *found, bool shouldUseLocalCopy, CopyOutState copyOutState, bool isColocatedIntermediateResult, bool isPublishable); static MultiConnection * CopyGetPlacementConnection(HTAB *connectionStateHash, ShardPlacement *placement, bool colocatedIntermediateResult); static bool HasReachedAdaptiveExecutorPoolSize(List *connectionStateHash); static MultiConnection * GetLeastUtilisedCopyConnection(List *connectionStateList, char *nodeName, int nodePort); static List * ConnectionStateList(HTAB *connectionStateHash); static List * ConnectionStateListToNode(HTAB *connectionStateHash, const char *hostname, int32 port); static void InitializeCopyShardState(CopyShardState *shardState, HTAB *connectionStateHash, uint64 shardId, bool canUseLocalCopy, CopyOutState copyOutState, bool colocatedIntermediateResult, bool isPublishable); static void StartPlacementStateCopyCommand(CopyPlacementState *placementState, CopyStmt *copyStatement, CopyOutState copyOutState); static void EndPlacementStateCopyCommand(CopyPlacementState *placementState, CopyOutState copyOutState); static void UnclaimCopyConnections(List *connectionStateList); static void ShutdownCopyConnectionState(CopyConnectionState *connectionState, CitusCopyDestReceiver *copyDest); static SelectStmt * CitusCopySelect(CopyStmt *copyStatement); static void CitusCopyTo(CopyStmt *copyStatement, QueryCompletion *completionTag); static int64 ForwardCopyDataFromConnection(CopyOutState copyOutState, MultiConnection *connection); static void ErrorIfCopyHasOnErrorLogVerbosity(CopyStmt *copyStatement); /* Private functions copied and adapted from copy.c in PostgreSQL */ static void SendCopyBegin(CopyOutState cstate); static void SendCopyEnd(CopyOutState cstate); static void CopySendData(CopyOutState outputState, const void *databuf, int datasize); static void CopySendString(CopyOutState outputState, const char *str); static void CopySendChar(CopyOutState outputState, char c); static void CopySendInt32(CopyOutState outputState, int32 val); static void CopySendInt16(CopyOutState outputState, int16 val); static void CopySendEndOfRow(CopyOutState cstate, bool includeEndOfLine); static void CopyAttributeOutText(CopyOutState outputState, char *string); static inline void CopyFlushOutput(CopyOutState outputState, char *start, char *pointer); static bool CitusSendTupleToPlacements(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest); static void AddPlacementStateToCopyConnectionStateBuffer(CopyConnectionState * connectionState, CopyPlacementState * placementState); static void RemovePlacementStateFromCopyConnectionStateBuffer(CopyConnectionState * connectionState, CopyPlacementState * placementState); static uint64 ProcessAppendToShardOption(Oid relationId, CopyStmt *copyStatement); static uint64 ShardIdForTuple(CitusCopyDestReceiver *copyDest, Datum *columnValues, bool *columnNulls); /* CitusCopyDestReceiver functions */ static void CitusCopyDestReceiverStartup(DestReceiver *copyDest, int operation, TupleDesc inputTupleDesc); static bool CitusCopyDestReceiverReceive(TupleTableSlot *slot, DestReceiver *copyDest); static void CitusCopyDestReceiverShutdown(DestReceiver *destReceiver); static void CitusCopyDestReceiverDestroy(DestReceiver *destReceiver); static bool ContainsLocalPlacement(int64 shardId); static void CompleteCopyQueryTagCompat(QueryCompletion *completionTag, uint64 processedRowCount); static void FinishLocalCopy(CitusCopyDestReceiver *copyDest); static void CreateLocalColocatedIntermediateFile(CitusCopyDestReceiver *copyDest, CopyShardState *shardState); static void FinishLocalColocatedIntermediateFiles(CitusCopyDestReceiver *copyDest); static void CloneCopyOutStateForLocalCopy(CopyOutState from, CopyOutState to); static LocalCopyStatus GetLocalCopyStatus(void); static bool ShardIntervalListHasLocalPlacements(List *shardIntervalList); static void LogLocalCopyToRelationExecution(uint64 shardId); static void LogLocalCopyToFileExecution(uint64 shardId); static void ErrorIfMergeInCopy(CopyStmt *copyStatement); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_text_send_as_jsonb); /* * CitusCopyFrom implements the COPY table_name FROM. It dispacthes the copy * statement to related subfunctions based on where the copy command is run * and the partition method of the distributed table. */ static void CitusCopyFrom(CopyStmt *copyStatement, QueryCompletion *completionTag) { UseCoordinatedTransaction(); /* disallow COPY to/from file or program except for superusers */ if (copyStatement->filename != NULL && !superuser()) { if (copyStatement->is_program) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to COPY to or from an external program"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } else { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to COPY to or from a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } } Oid relationId = RangeVarGetRelid(copyStatement->relation, NoLock, false); EnsureCopyCanRunOnRelation(relationId); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); /* disallow modifications to a partition table which have rep. factor > 1 */ EnsurePartitionTableNotReplicated(relationId); if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, APPEND_DISTRIBUTED) || !HasDistributionKeyCacheEntry(cacheEntry)) { CopyToExistingShards(copyStatement, completionTag); } else { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported partition method"))); } XactModificationLevel = XACT_MODIFICATION_DATA; } /* * EnsureCopyCanRunOnRelation throws error is the database in read-only mode. */ static void EnsureCopyCanRunOnRelation(Oid relationId) { /* first, do the regular check and give consistent errors with regular queries */ EnsureModificationsCanRunOnRelation(relationId); /* * We use 2PC for all COPY commands. It means that we cannot allow any COPY * on replicas even if the user allows via WritableStandbyCoordinator GUC. */ if (RecoveryInProgress() && WritableStandbyCoordinator) { ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("COPY command to Citus tables is not allowed in " "read-only mode"), errhint("All COPY commands to citus tables happen via 2PC, " "and 2PC requires the database to be in a writable state."), errdetail("the database is read-only"))); } } /* * CopyToExistingShards implements the COPY table_name FROM ... for hash or * range-partitioned tables where there are already shards into which to copy * rows. */ static void CopyToExistingShards(CopyStmt *copyStatement, QueryCompletion *completionTag) { Oid tableId = RangeVarGetRelid(copyStatement->relation, NoLock, false); List *columnNameList = NIL; int partitionColumnIndex = INVALID_PARTITION_COLUMN_INDEX; bool isInputFormatBinary = IsCopyInBinaryFormat(copyStatement); uint64 processedRowCount = 0; ErrorContextCallback errorCallback; /* allocate column values and nulls arrays */ Relation distributedRelation = table_open(tableId, RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); uint32 columnCount = tupleDescriptor->natts; Datum *columnValues = palloc0(columnCount * sizeof(Datum)); bool *columnNulls = palloc0(columnCount * sizeof(bool)); /* set up a virtual tuple table slot */ TupleTableSlot *tupleTableSlot = MakeSingleTupleTableSlot(tupleDescriptor, &TTSOpsVirtual); tupleTableSlot->tts_nvalid = columnCount; tupleTableSlot->tts_values = columnValues; tupleTableSlot->tts_isnull = columnNulls; /* determine the partition column index in the tuple descriptor */ Var *partitionColumn = PartitionColumn(tableId, 0); if (partitionColumn != NULL) { partitionColumnIndex = partitionColumn->varattno - 1; } /* build the list of column names for remote COPY statements */ for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); char *columnName = NameStr(currentColumn->attname); if (IsDroppedOrGenerated(currentColumn)) { continue; } columnNameList = lappend(columnNameList, columnName); } EState *executorState = CreateExecutorState(); MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState); ExprContext *executorExpressionContext = GetPerTupleExprContext(executorState); /* set up the destination for the COPY */ const bool publishableData = true; /* we want to track query counters for "COPY (to) distributed-table .." commands */ const bool trackQueryCounters = true; CitusCopyDestReceiver *copyDest = CreateCitusCopyDestReceiver(tableId, columnNameList, partitionColumnIndex, executorState, NULL, publishableData, trackQueryCounters); /* if the user specified an explicit append-to_shard option, write to it */ uint64 appendShardId = ProcessAppendToShardOption(tableId, copyStatement); if (appendShardId != INVALID_SHARD_ID) { copyDest->appendShardId = appendShardId; } DestReceiver *dest = (DestReceiver *) copyDest; dest->rStartup(dest, 0, tupleDescriptor); /* * Below, we change a few fields in the Relation to control the behaviour * of BeginCopyFrom. However, we obviously should not do this in relcache * and therefore make a copy of the Relation. */ Relation copiedDistributedRelation = (Relation) palloc(sizeof(RelationData)); Form_pg_class copiedDistributedRelationTuple = (Form_pg_class) palloc(CLASS_TUPLE_SIZE); /* * There is no need to deep copy everything. We will just deep copy of the fields * we will change. */ *copiedDistributedRelation = *distributedRelation; *copiedDistributedRelationTuple = *distributedRelation->rd_rel; copiedDistributedRelation->rd_rel = copiedDistributedRelationTuple; copiedDistributedRelation->rd_att = CreateTupleDescCopyConstr(tupleDescriptor); /* * BeginCopyFrom opens all partitions of given partitioned table with relation_open * and it expects its caller to close those relations. We do not have direct access * to opened relations, thus we are changing relkind of partitioned tables so that * Postgres will treat those tables as regular relations and will not open its * partitions. */ if (PartitionedTable(tableId)) { copiedDistributedRelationTuple->relkind = RELKIND_RELATION; } /* * We make an optimisation to skip JSON parsing for JSONB columns, because many * Citus users have large objects in this column and parsing it on the coordinator * causes significant CPU overhead. We do this by forcing BeginCopyFrom and * NextCopyFrom to parse the column as text and then encoding it as JSON again * by using citus_text_send_as_jsonb as the binary output function. * * The main downside of enabling this optimisation is that it defers validation * until the object is parsed by the worker, which is unable to give an accurate * line number. */ if (SkipJsonbValidationInCopy && !isInputFormatBinary) { CopyOutState copyOutState = copyDest->copyOutState; ListCell *jsonbColumnIndexCell = NULL; /* get the column indices for all JSONB columns that appear in the input */ List *jsonbColumnIndexList = FindJsonbInputColumns( copiedDistributedRelation->rd_att, copyStatement->attlist); foreach(jsonbColumnIndexCell, jsonbColumnIndexList) { int jsonbColumnIndex = lfirst_int(jsonbColumnIndexCell); Form_pg_attribute currentColumn = TupleDescAttr(copiedDistributedRelation->rd_att, jsonbColumnIndex); if (jsonbColumnIndex == partitionColumnIndex) { /* * In the curious case of using a JSONB column as partition column, * we leave it as is because we want to make sure the hashing works * correctly. */ continue; } ereport(DEBUG1, (errmsg("parsing JSONB column %s as text", NameStr(currentColumn->attname)))); /* parse the column as text instead of JSONB */ currentColumn->atttypid = TEXTOID; if (copyOutState->binary) { Oid textSendAsJsonbFunctionId = CitusTextSendAsJsonbFunctionId(); /* * If we're using binary encoding between coordinator and workers * then we should honour the format expected by jsonb_recv, which * is a version number followed by text. We therefore use an output * function which sends the text as if it were jsonb, namely by * prepending a version number. */ fmgr_info(textSendAsJsonbFunctionId, ©Dest->columnOutputFunctions[jsonbColumnIndex]); } else { Oid textoutFunctionId = TextOutFunctionId(); fmgr_info(textoutFunctionId, ©Dest->columnOutputFunctions[jsonbColumnIndex]); } } } /* initialize copy state to read from COPY data source */ CopyFromState copyState = BeginCopyFrom(NULL, copiedDistributedRelation, NULL, copyStatement->filename, copyStatement->is_program, NULL, copyStatement->attlist, copyStatement->options); /* set up callback to identify error line number */ errorCallback.callback = CopyFromErrorCallback; errorCallback.arg = (void *) copyState; errorCallback.previous = error_context_stack; error_context_stack = &errorCallback; while (true) { ResetPerTupleExprContext(executorState); MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); /* parse a row from the input */ bool nextRowFound = NextCopyFrom(copyState, executorExpressionContext, columnValues, columnNulls); if (!nextRowFound) { MemoryContextSwitchTo(oldContext); break; } CHECK_FOR_INTERRUPTS(); MemoryContextSwitchTo(oldContext); dest->receiveSlot(tupleTableSlot, dest); ++processedRowCount; pgstat_progress_update_param(PROGRESS_COPY_TUPLES_PROCESSED, processedRowCount); } EndCopyFrom(copyState); /* all lines have been copied, stop showing line number in errors */ error_context_stack = errorCallback.previous; /* finish the COPY commands */ dest->rShutdown(dest); dest->rDestroy(dest); ExecDropSingleTupleTableSlot(tupleTableSlot); FreeExecutorState(executorState); table_close(distributedRelation, NoLock); CHECK_FOR_INTERRUPTS(); if (completionTag != NULL) { CompleteCopyQueryTagCompat(completionTag, processedRowCount); } } /* * IsCopyInBinaryFormat determines whether the given COPY statement has the * WITH (format binary) option. */ static bool IsCopyInBinaryFormat(CopyStmt *copyStatement) { ListCell *optionCell = NULL; foreach(optionCell, copyStatement->options) { DefElem *defel = lfirst_node(DefElem, optionCell); if (strcmp(defel->defname, "format") == 0 && strcmp(defGetString(defel), "binary") == 0) { return true; } } return false; } /* * FindJsonbInputColumns finds columns in the tuple descriptor that have * the JSONB type and appear in inputColumnNameList. If the list is empty then * all JSONB columns are returned. */ static List * FindJsonbInputColumns(TupleDesc tupleDescriptor, List *inputColumnNameList) { List *jsonbColumnIndexList = NIL; int columnCount = tupleDescriptor->natts; for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); if (currentColumn->attisdropped) { continue; } if (currentColumn->atttypid != JSONBOID) { continue; } if (inputColumnNameList != NIL) { ListCell *inputColumnCell = NULL; bool isInputColumn = false; foreach(inputColumnCell, inputColumnNameList) { char *inputColumnName = strVal(lfirst(inputColumnCell)); if (namestrcmp(¤tColumn->attname, inputColumnName) == 0) { isInputColumn = true; break; } } if (!isInputColumn) { continue; } } jsonbColumnIndexList = lappend_int(jsonbColumnIndexList, columnIndex); } return jsonbColumnIndexList; } static void CompleteCopyQueryTagCompat(QueryCompletion *completionTag, uint64 processedRowCount) { SetQueryCompletion(completionTag, CMDTAG_COPY, processedRowCount); } /* * RemoveOptionFromList removes an option from a list of options in a * COPY .. WITH (..) statement by name and returns the resulting list. */ static List * RemoveOptionFromList(List *optionList, char *optionName) { ListCell *optionCell = NULL; foreach(optionCell, optionList) { DefElem *option = (DefElem *) lfirst(optionCell); if (strncmp(option->defname, optionName, NAMEDATALEN) == 0) { return list_delete_cell(optionList, optionCell); } } return optionList; } /* * CanUseBinaryCopyFormat iterates over columns of the relation and looks for a * column whose type is array of user-defined type or composite type. If it finds * such column, that means we cannot use binary format for COPY, because binary * format sends Oid of the types, which are generally not same in master and * worker nodes for user-defined types. If the function can not detect a binary * output function for any of the column, it returns false. */ bool CanUseBinaryCopyFormat(TupleDesc tupleDescription) { bool useBinaryCopyFormat = true; int totalColumnCount = tupleDescription->natts; for (int columnIndex = 0; columnIndex < totalColumnCount; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescription, columnIndex); if (IsDroppedOrGenerated(currentColumn)) { continue; } Oid typeId = currentColumn->atttypid; if (!CanUseBinaryCopyFormatForType(typeId)) { useBinaryCopyFormat = false; break; } } return useBinaryCopyFormat; } /* * CanUseBinaryCopyFormatForTargetList returns true if we can use binary * copy format for all columns of the given target list. */ bool CanUseBinaryCopyFormatForTargetList(List *targetEntryList) { ListCell *targetEntryCell = NULL; foreach(targetEntryCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Node *targetExpr = (Node *) targetEntry->expr; Oid columnType = exprType(targetExpr); if (!CanUseBinaryCopyFormatForType(columnType)) { return false; } } return true; } /* * CanUseBinaryCopyFormatForType determines whether it is safe to use the * binary copy format for the given type. See contents of the function for * details of when it's safe to use binary copy. */ bool CanUseBinaryCopyFormatForType(Oid typeId) { if (!BinaryOutputFunctionDefined(typeId)) { return false; } if (!BinaryInputFunctionDefined(typeId)) { return false; } /* * A row type can contain any types, possibly types that don't have * the binary input and output functions defined. */ if (type_is_rowtype(typeId)) { /* * TODO: Inspect the types inside the record and check if all of them * can be binary encoded. If so, it's safe to use binary encoding. * * IMPORTANT: When implementing this todo keep the following in mind: * * In PG versions before PG14 the record_recv function would error out * more than necessary. * * It errors out when any of the columns in the row have a type oid * that doesn't match with the oid in the received data. This happens * pretty much always for non built in types, because their oids differ * between postgres intallations. So for those Postgres versions we * would need a check like the following for each column: * * if (columnType >= FirstNormalObjectId) { * return false * } */ return false; } HeapTuple typeTup = typeidType(typeId); Form_pg_type type = (Form_pg_type) GETSTRUCT(typeTup); Oid elementType = type->typelem; ReleaseSysCache(typeTup); /* * Any type that is a wrapper around an element type (e.g. arrays and * ranges) require the element type to also has support for binary * encoding. */ if (elementType != InvalidOid) { if (!CanUseBinaryCopyFormatForType(elementType)) { return false; } } /* * For domains, make sure that the underlying type can be binary copied. */ Oid baseTypeId = getBaseType(typeId); if (typeId != baseTypeId) { if (!CanUseBinaryCopyFormatForType(baseTypeId)) { return false; } } return true; } /* * BinaryOutputFunctionDefined checks whether binary output function is defined * for the given type. */ static bool BinaryOutputFunctionDefined(Oid typeId) { Oid typeFunctionId = InvalidOid; Oid typeIoParam = InvalidOid; int16 typeLength = 0; bool typeByVal = false; char typeAlign = 0; char typeDelim = 0; get_type_io_data(typeId, IOFunc_send, &typeLength, &typeByVal, &typeAlign, &typeDelim, &typeIoParam, &typeFunctionId); return OidIsValid(typeFunctionId); } /* * BinaryInputFunctionDefined checks whether binary output function is defined * for the given type. */ static bool BinaryInputFunctionDefined(Oid typeId) { Oid typeFunctionId = InvalidOid; Oid typeIoParam = InvalidOid; int16 typeLength = 0; bool typeByVal = false; char typeAlign = 0; char typeDelim = 0; get_type_io_data(typeId, IOFunc_receive, &typeLength, &typeByVal, &typeAlign, &typeDelim, &typeIoParam, &typeFunctionId); return OidIsValid(typeFunctionId); } /* Send copy binary headers to given connections */ static void SendCopyBinaryHeaders(CopyOutState copyOutState, int64 shardId, List *connectionList) { resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryHeaders(copyOutState); SendCopyDataToAll(copyOutState->fe_msgbuf, shardId, connectionList); } /* Send copy binary footers to given connections */ static void SendCopyBinaryFooters(CopyOutState copyOutState, int64 shardId, List *connectionList) { resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryFooters(copyOutState); SendCopyDataToAll(copyOutState->fe_msgbuf, shardId, connectionList); } /* * ConstructCopyStatement constructs the text of a COPY statement for a particular * shard. */ static StringInfo ConstructCopyStatement(CopyStmt *copyStatement, int64 shardId) { StringInfo command = makeStringInfo(); char *schemaName = copyStatement->relation->schemaname; char *relationName = copyStatement->relation->relname; char *shardName = pstrdup(relationName); AppendShardIdToName(&shardName, shardId); char *shardQualifiedName = quote_qualified_identifier(schemaName, shardName); appendStringInfo(command, "COPY %s ", shardQualifiedName); if (copyStatement->attlist != NIL) { ListCell *columnNameCell = NULL; bool appendedFirstName = false; foreach(columnNameCell, copyStatement->attlist) { char *columnName = strVal(lfirst(columnNameCell)); const char *quotedColumnName = quote_identifier(columnName); if (!appendedFirstName) { appendStringInfo(command, "(%s", quotedColumnName); appendedFirstName = true; } else { appendStringInfo(command, ", %s", quotedColumnName); } } appendStringInfoString(command, ") "); } if (copyStatement->is_from) { appendStringInfoString(command, "FROM STDIN"); } else { appendStringInfoString(command, "TO STDOUT"); } if (copyStatement->options != NIL) { ListCell *optionCell = NULL; appendStringInfoString(command, " WITH ("); foreach(optionCell, copyStatement->options) { DefElem *defel = (DefElem *) lfirst(optionCell); if (optionCell != list_head(copyStatement->options)) { appendStringInfoString(command, ", "); } appendStringInfo(command, "%s", defel->defname); if (defel->arg == NULL) { /* option without value */ } else if (IsA(defel->arg, String)) { char *value = defGetString(defel); /* make sure strings are quoted (may contain reserved characters) */ appendStringInfo(command, " %s", quote_literal_cstr(value)); } else if (IsA(defel->arg, List)) { List *nameList = defGetStringList(defel); appendStringInfo(command, " (%s)", NameListToQuotedString(nameList)); } else { char *value = defGetString(defel); /* numeric options or * should not have quotes */ appendStringInfo(command, " %s", value); } } appendStringInfoString(command, ")"); } return command; } /* * SendCopyDataToAll sends copy data to all connections in a list. */ static void SendCopyDataToAll(StringInfo dataBuffer, int64 shardId, List *connectionList) { ListCell *connectionCell = NULL; foreach(connectionCell, connectionList) { MultiConnection *connection = (MultiConnection *) lfirst(connectionCell); SendCopyDataToPlacement(dataBuffer, shardId, connection); } } /* * SendCopyDataToPlacement sends serialized COPY data to a specific shard placement * over the given connection. */ static void SendCopyDataToPlacement(StringInfo dataBuffer, int64 shardId, MultiConnection *connection) { if (!PutRemoteCopyData(connection, dataBuffer->data, dataBuffer->len)) { ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("failed to COPY to shard " INT64_FORMAT " on %s:%d", shardId, connection->hostname, connection->port), errdetail("failed to send %d bytes %s", dataBuffer->len, dataBuffer->data))); } } /* * EndRemoteCopy ends the COPY input on all connections, and unclaims connections. * This reports an error on failure. */ void EndRemoteCopy(int64 shardId, List *connectionList) { ListCell *connectionCell = NULL; foreach(connectionCell, connectionList) { MultiConnection *connection = (MultiConnection *) lfirst(connectionCell); bool raiseInterrupts = true; /* end the COPY input */ if (!PutRemoteCopyEnd(connection, NULL)) { ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("failed to COPY to shard " INT64_FORMAT " on %s:%d", shardId, connection->hostname, connection->port))); } /* check whether there were any COPY errors */ PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (PQresultStatus(result) != PGRES_COMMAND_OK) { ReportCopyError(connection, result); } PQclear(result); ForgetResults(connection); UnclaimConnection(connection); } } /* * ReportCopyError tries to report a useful error message for the user from * the remote COPY error messages. */ void ReportCopyError(MultiConnection *connection, PGresult *result) { char *remoteMessage = PQresultErrorField(result, PG_DIAG_MESSAGE_PRIMARY); if (remoteMessage != NULL) { /* probably a constraint violation, show remote message and detail */ char *remoteDetail = PQresultErrorField(result, PG_DIAG_MESSAGE_DETAIL); bool haveDetail = remoteDetail != NULL; ereport(ERROR, (errmsg("%s", remoteMessage), haveDetail ? errdetail("%s", remoteDetail) : 0)); } else { /* trim the trailing characters */ remoteMessage = pchomp(PQerrorMessage(connection->pgConn)); ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("failed to complete COPY on %s:%d", connection->hostname, connection->port), errdetail("%s", remoteMessage))); } } /* * ConversionPathForTypes fills *result with all the data necessary for converting * Datums of type inputType to Datums of type destType. */ void ConversionPathForTypes(Oid inputType, Oid destType, CopyCoercionData *result) { Oid coercionFuncId = InvalidOid; CoercionPathType coercionType = COERCION_PATH_RELABELTYPE; if (destType == inputType) { result->coercionType = COERCION_PATH_RELABELTYPE; return; } coercionType = find_coercion_pathway(destType, inputType, COERCION_EXPLICIT, &coercionFuncId); switch (coercionType) { case COERCION_PATH_NONE: { ereport(ERROR, (errmsg("cannot cast %d to %d", inputType, destType))); return; } case COERCION_PATH_ARRAYCOERCE: { Oid inputBaseType = get_base_element_type(inputType); Oid destBaseType = get_base_element_type(destType); CoercionPathType baseCoercionType = COERCION_PATH_NONE; if (inputBaseType != InvalidOid && destBaseType != InvalidOid) { baseCoercionType = find_coercion_pathway(inputBaseType, destBaseType, COERCION_EXPLICIT, &coercionFuncId); } if (baseCoercionType != COERCION_PATH_COERCEVIAIO) { ereport(ERROR, (errmsg("can not run query which uses an implicit coercion" " between array types"))); } } /* fallthrough */ case COERCION_PATH_COERCEVIAIO: { result->coercionType = COERCION_PATH_COERCEVIAIO; { bool typisvarlena = false; /* ignored */ Oid iofunc = InvalidOid; getTypeOutputInfo(inputType, &iofunc, &typisvarlena); fmgr_info(iofunc, &(result->outputFunction)); } { Oid iofunc = InvalidOid; getTypeInputInfo(destType, &iofunc, &(result->typioparam)); fmgr_info(iofunc, &(result->inputFunction)); } return; } case COERCION_PATH_FUNC: { result->coercionType = COERCION_PATH_FUNC; fmgr_info(coercionFuncId, &(result->coerceFunction)); return; } case COERCION_PATH_RELABELTYPE: { result->coercionType = COERCION_PATH_RELABELTYPE; return; /* the types are binary compatible, no need to call a function */ } default: { Assert(false); /* there are no other options for this enum */ } } } /* * Returns the type of the provided column of the provided tuple. Throws an error if the * column does not exist or is dropped. * * tupleDescriptor and relationId must refer to the same table. */ static Oid TypeForColumnName(Oid relationId, TupleDesc tupleDescriptor, char *columnName) { AttrNumber destAttrNumber = get_attnum(relationId, columnName); if (destAttrNumber == InvalidAttrNumber) { ereport(ERROR, (errmsg("invalid attr? %s", columnName))); } Form_pg_attribute attr = TupleDescAttr(tupleDescriptor, destAttrNumber - 1); return attr->atttypid; } /* * Walks a TupleDesc and returns an array of the types of each attribute. * Returns InvalidOid in the place of dropped or generated attributes. */ static Oid * TypeArrayFromTupleDescriptor(TupleDesc tupleDescriptor) { int columnCount = tupleDescriptor->natts; Oid *typeArray = palloc0(columnCount * sizeof(Oid)); for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { Form_pg_attribute attr = TupleDescAttr(tupleDescriptor, columnIndex); if (IsDroppedOrGenerated(attr)) { typeArray[columnIndex] = InvalidOid; } else { typeArray[columnIndex] = attr->atttypid; } } return typeArray; } /* * ColumnCoercionPaths scans the input and output tuples looking for mismatched types, * it then returns an array of coercion functions to use on the input tuples, and an * array of types which descript the output tuple */ static CopyCoercionData * ColumnCoercionPaths(TupleDesc destTupleDescriptor, TupleDesc inputTupleDescriptor, Oid destRelId, List *columnNameList, Oid *finalColumnTypeArray) { int columnCount = inputTupleDescriptor->natts; CopyCoercionData *coercePaths = palloc0(columnCount * sizeof(CopyCoercionData)); Oid *inputTupleTypes = TypeArrayFromTupleDescriptor(inputTupleDescriptor); ListCell *currentColumnName = list_head(columnNameList); for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { Oid inputTupleType = inputTupleTypes[columnIndex]; char *columnName = lfirst(currentColumnName); if (inputTupleType == InvalidOid) { /* TypeArrayFromTupleDescriptor decided to skip this column */ continue; } Oid destTupleType = TypeForColumnName(destRelId, destTupleDescriptor, columnName); finalColumnTypeArray[columnIndex] = destTupleType; ConversionPathForTypes(inputTupleType, destTupleType, &coercePaths[columnIndex]); currentColumnName = lnext(columnNameList, currentColumnName); if (currentColumnName == NULL) { /* the rest of inputTupleDescriptor are dropped columns, return early! */ break; } } return coercePaths; } /* * TypeOutputFunctions takes an array of types and returns an array of output functions * for those types. */ static FmgrInfo * TypeOutputFunctions(uint32 columnCount, Oid *typeIdArray, bool binaryFormat) { FmgrInfo *columnOutputFunctions = palloc0(columnCount * sizeof(FmgrInfo)); for (uint32 columnIndex = 0; columnIndex < columnCount; columnIndex++) { FmgrInfo *currentOutputFunction = &columnOutputFunctions[columnIndex]; Oid columnTypeId = typeIdArray[columnIndex]; bool typeVariableLength = false; Oid outputFunctionId = InvalidOid; if (columnTypeId == InvalidOid) { /* TypeArrayFromTupleDescriptor decided to skip this column */ continue; } else if (binaryFormat) { getTypeBinaryOutputInfo(columnTypeId, &outputFunctionId, &typeVariableLength); } else { getTypeOutputInfo(columnTypeId, &outputFunctionId, &typeVariableLength); } fmgr_info(outputFunctionId, currentOutputFunction); } return columnOutputFunctions; } /* * ColumnOutputFunctions is a wrapper around TypeOutputFunctions, it takes a * tupleDescriptor and returns an array of output functions, one for each column in * the tuple. */ FmgrInfo * ColumnOutputFunctions(TupleDesc rowDescriptor, bool binaryFormat) { uint32 columnCount = (uint32) rowDescriptor->natts; Oid *columnTypes = TypeArrayFromTupleDescriptor(rowDescriptor); FmgrInfo *outputFunctions = TypeOutputFunctions(columnCount, columnTypes, binaryFormat); return outputFunctions; } /* * citus_text_send_as_jsonb sends a text as if it was a JSONB. This should only * be used if the text is indeed valid JSON. */ Datum citus_text_send_as_jsonb(PG_FUNCTION_ARGS) { text *inputText = PG_GETARG_TEXT_PP(0); StringInfoData buf; int version = 1; pq_begintypsend(&buf); pq_sendint(&buf, version, 1); pq_sendtext(&buf, VARDATA_ANY(inputText), VARSIZE_ANY_EXHDR(inputText)); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* * AppendCopyRowData serializes one row using the column output functions, * and appends the data to the row output state object's message buffer. * This function is modeled after the CopyOneRowTo() function in * commands/copy.c, but only implements a subset of that functionality. * Note that the caller of this function should reset row memory context * to not bloat memory usage. */ void AppendCopyRowData(Datum *valueArray, bool *isNullArray, TupleDesc rowDescriptor, CopyOutState rowOutputState, FmgrInfo *columnOutputFunctions, CopyCoercionData *columnCoercionPaths) { uint32 totalColumnCount = (uint32) rowDescriptor->natts; uint32 availableColumnCount = AvailableColumnCount(rowDescriptor); uint32 appendedColumnCount = 0; MemoryContext oldContext = MemoryContextSwitchTo(rowOutputState->rowcontext); if (rowOutputState->binary) { CopySendInt16(rowOutputState, availableColumnCount); } for (uint32 columnIndex = 0; columnIndex < totalColumnCount; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(rowDescriptor, columnIndex); Datum value = valueArray[columnIndex]; bool isNull = isNullArray[columnIndex]; bool lastColumn = false; if (!isNull && columnCoercionPaths != NULL) { value = CoerceColumnValue(value, &columnCoercionPaths[columnIndex]); } if (IsDroppedOrGenerated(currentColumn)) { continue; } else if (rowOutputState->binary) { if (!isNull) { FmgrInfo *outputFunctionPointer = &columnOutputFunctions[columnIndex]; bytea *outputBytes = SendFunctionCall(outputFunctionPointer, value); CopySendInt32(rowOutputState, VARSIZE(outputBytes) - VARHDRSZ); CopySendData(rowOutputState, VARDATA(outputBytes), VARSIZE(outputBytes) - VARHDRSZ); } else { CopySendInt32(rowOutputState, -1); } } else { if (!isNull) { FmgrInfo *outputFunctionPointer = &columnOutputFunctions[columnIndex]; char *columnText = OutputFunctionCall(outputFunctionPointer, value); CopyAttributeOutText(rowOutputState, columnText); } else { CopySendString(rowOutputState, rowOutputState->null_print_client); } lastColumn = ((appendedColumnCount + 1) == availableColumnCount); if (!lastColumn) { CopySendChar(rowOutputState, rowOutputState->delim[0]); } } appendedColumnCount++; } if (!rowOutputState->binary) { /* append default line termination string depending on the platform */ #ifndef WIN32 CopySendChar(rowOutputState, '\n'); #else CopySendString(rowOutputState, "\r\n"); #endif } MemoryContextSwitchTo(oldContext); } /* * CoerceColumnValue follows the instructions in *coercionPath and uses them to convert * inputValue into a Datum of the correct type. */ Datum CoerceColumnValue(Datum inputValue, CopyCoercionData *coercionPath) { switch (coercionPath->coercionType) { case COERCION_PATH_NONE: { return inputValue; /* this was a dropped column */ } case COERCION_PATH_RELABELTYPE: { return inputValue; /* no need to do anything */ } case COERCION_PATH_FUNC: { FmgrInfo *coerceFunction = &(coercionPath->coerceFunction); Datum outputValue = FunctionCall1(coerceFunction, inputValue); return outputValue; } case COERCION_PATH_COERCEVIAIO: { FmgrInfo *outFunction = &(coercionPath->outputFunction); Datum textRepr = FunctionCall1(outFunction, inputValue); FmgrInfo *inFunction = &(coercionPath->inputFunction); Oid typioparam = coercionPath->typioparam; Datum outputValue = FunctionCall3(inFunction, textRepr, typioparam, Int32GetDatum(-1)); return outputValue; } default: { /* this should never happen */ ereport(ERROR, (errmsg("unsupported coercion type"))); } } } /* * AvailableColumnCount returns the number of columns in a tuple descriptor, excluding * columns that were dropped. */ static uint32 AvailableColumnCount(TupleDesc tupleDescriptor) { uint32 columnCount = 0; for (uint32 columnIndex = 0; columnIndex < tupleDescriptor->natts; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); if (!IsDroppedOrGenerated(currentColumn)) { columnCount++; } } return columnCount; } /* * AppendCopyBinaryHeaders appends binary headers to the copy buffer in * headerOutputState. */ void AppendCopyBinaryHeaders(CopyOutState headerOutputState) { const int32 zero = 0; MemoryContext oldContext = MemoryContextSwitchTo(headerOutputState->rowcontext); /* Signature */ CopySendData(headerOutputState, BinarySignature, 11); /* Flags field (no OIDs) */ CopySendInt32(headerOutputState, zero); /* No header extension */ CopySendInt32(headerOutputState, zero); MemoryContextSwitchTo(oldContext); } /* * AppendCopyBinaryFooters appends binary footers to the copy buffer in * footerOutputState. */ void AppendCopyBinaryFooters(CopyOutState footerOutputState) { int16 negative = -1; MemoryContext oldContext = MemoryContextSwitchTo(footerOutputState->rowcontext); CopySendInt16(footerOutputState, negative); MemoryContextSwitchTo(oldContext); } /* *INDENT-OFF* */ /* * Send copy start/stop messages for frontend copies. These have changed * in past protocol redesigns. */ static void SendCopyBegin(CopyOutState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); int16 format = (cstate->binary ? 1 : 0); int i; pq_beginmessage(&buf, 'H'); pq_sendbyte(&buf, format); /* overall format */ pq_sendint16(&buf, natts); for (i = 0; i < natts; i++) pq_sendint16(&buf, format); /* per-column formats */ pq_endmessage(&buf); cstate->copy_dest = COPY_FRONTEND; } /* End a copy stream sent to the client */ static void SendCopyEnd(CopyOutState cstate) { /* Shouldn't have any unsent data */ Assert(cstate->fe_msgbuf->len == 0); /* Send Copy Done message */ pq_putemptymessage('c'); } /* Append data to the copy buffer in outputState */ static void CopySendData(CopyOutState outputState, const void *databuf, int datasize) { appendBinaryStringInfo(outputState->fe_msgbuf, databuf, datasize); } /* Append a striong to the copy buffer in outputState. */ static void CopySendString(CopyOutState outputState, const char *str) { appendBinaryStringInfo(outputState->fe_msgbuf, str, strlen(str)); } /* Append a char to the copy buffer in outputState. */ static void CopySendChar(CopyOutState outputState, char c) { appendStringInfoCharMacro(outputState->fe_msgbuf, c); } /* Append an int32 to the copy buffer in outputState. */ static void CopySendInt32(CopyOutState outputState, int32 val) { uint32 buf = htonl((uint32) val); CopySendData(outputState, &buf, sizeof(buf)); } /* Append an int16 to the copy buffer in outputState. */ static void CopySendInt16(CopyOutState outputState, int16 val) { uint16 buf = htons((uint16) val); CopySendData(outputState, &buf, sizeof(buf)); } /* Send the row to the appropriate destination */ static void CopySendEndOfRow(CopyOutState cstate, bool includeEndOfLine) { StringInfo fe_msgbuf = cstate->fe_msgbuf; switch (cstate->copy_dest) { case COPY_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ if (!cstate->binary && includeEndOfLine) CopySendChar(cstate, '\n'); /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len); break; case COPY_FILE: case COPY_CALLBACK: Assert(false); /* Not yet supported. */ break; } resetStringInfo(fe_msgbuf); } /* * Send text representation of one column, with conversion and escaping. * * NB: This function is based on commands/copy.c and doesn't fully conform to * our coding style. The function should be kept in sync with copy.c. */ static void CopyAttributeOutText(CopyOutState cstate, char *string) { char *pointer = NULL; char c = '\0'; char delimc = cstate->delim[0]; if (cstate->need_transcoding) { pointer = pg_server_to_any(string, strlen(string), cstate->file_encoding); } else { pointer = string; } /* * We have to grovel through the string searching for control characters * and instances of the delimiter character. In most cases, though, these * are infrequent. To avoid overhead from calling CopySendData once per * character, we dump out all characters between escaped characters in a * single call. The loop invariant is that the data from "start" to "pointer" * can be sent literally, but hasn't yet been. * * As all encodings here are safe, i.e. backend supported ones, we can * skip doing pg_encoding_mblen(), because in valid backend encodings, * extra bytes of a multibyte character never look like ASCII. */ char *start = pointer; while ((c = *pointer) != '\0') { if ((unsigned char) c < (unsigned char) 0x20) { /* * \r and \n must be escaped, the others are traditional. We * prefer to dump these using the C-like notation, rather than * a backslash and the literal character, because it makes the * dump file a bit more proof against Microsoftish data * mangling. */ switch (c) { case '\b': c = 'b'; break; case '\f': c = 'f'; break; case '\n': c = 'n'; break; case '\r': c = 'r'; break; case '\t': c = 't'; break; case '\v': c = 'v'; break; default: /* If it's the delimiter, must backslash it */ if (c == delimc) break; /* All ASCII control chars are length 1 */ pointer++; continue; /* fall to end of loop */ } /* if we get here, we need to convert the control char */ CopyFlushOutput(cstate, start, pointer); CopySendChar(cstate, '\\'); CopySendChar(cstate, c); start = ++pointer; /* do not include char in next run */ } else if (c == '\\' || c == delimc) { CopyFlushOutput(cstate, start, pointer); CopySendChar(cstate, '\\'); start = pointer++; /* we include char in next run */ } else { pointer++; } } CopyFlushOutput(cstate, start, pointer); } /* *INDENT-ON* */ /* Helper function to send pending copy output */ static inline void CopyFlushOutput(CopyOutState cstate, char *start, char *pointer) { if (pointer > start) { CopySendData(cstate, start, pointer - start); } } /* * CreateCitusCopyDestReceiver creates a DestReceiver that copies into * a distributed table. * * The caller should provide the list of column names to use in the * remote COPY statement, and the partition column index in the tuple * descriptor (*not* the column name list). * * If intermediateResultIdPrefix is not NULL, the COPY will go into a set * of intermediate results that are co-located with the actual table. * The names of the intermediate results with be of the form: * intermediateResultIdPrefix_ * * If trackQueryCounters is true, the COPY will increment the query stat * counters as needed at the end of the COPY. */ CitusCopyDestReceiver * CreateCitusCopyDestReceiver(Oid tableId, List *columnNameList, int partitionColumnIndex, EState *executorState, char *intermediateResultIdPrefix, bool isPublishable, bool trackQueryCounters) { CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) palloc0( sizeof(CitusCopyDestReceiver)); /* set up the DestReceiver function pointers */ copyDest->pub.receiveSlot = CitusCopyDestReceiverReceive; copyDest->pub.rStartup = CitusCopyDestReceiverStartup; copyDest->pub.rShutdown = CitusCopyDestReceiverShutdown; copyDest->pub.rDestroy = CitusCopyDestReceiverDestroy; copyDest->pub.mydest = DestCopyOut; /* set up output parameters */ copyDest->distributedRelationId = tableId; copyDest->columnNameList = columnNameList; copyDest->partitionColumnIndex = partitionColumnIndex; copyDest->executorState = executorState; copyDest->colocatedIntermediateResultIdPrefix = intermediateResultIdPrefix; copyDest->memoryContext = CurrentMemoryContext; copyDest->isPublishable = isPublishable; copyDest->trackQueryCounters = trackQueryCounters; return copyDest; } /* * GetLocalCopyStatus returns the status for executing copy locally. * If LOCAL_COPY_DISABLED or LOCAL_COPY_REQUIRED, the caller has to * follow that. Else, the caller may decide to use local or remote * execution depending on other information. */ static LocalCopyStatus GetLocalCopyStatus(void) { if (!EnableLocalExecution || GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_DISABLED) { return LOCAL_COPY_DISABLED; } else if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_REQUIRED) { /* * For various reasons, including the transaction visibility * rules (e.g., read-your-own-writes), we have to use local * execution again if it has already happened within this * transaction block. * * We might error out later in the execution if it is not suitable * to execute the tasks locally. */ Assert(IsMultiStatementTransaction() || InCoordinatedTransaction()); /* * TODO: A future improvement could be to keep track of which placements * have been locally executed. At this point, only use local execution for * those placements. That'd help to benefit more from parallelism. */ return LOCAL_COPY_REQUIRED; } else if (IsMultiStatementTransaction()) { return LOCAL_COPY_REQUIRED; } return LOCAL_COPY_OPTIONAL; } /* * ShardIntervalListHasLocalPlacements returns true if any of the input * shard placement has a local placement; */ static bool ShardIntervalListHasLocalPlacements(List *shardIntervalList) { int32 localGroupId = GetLocalGroupId(); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { if (ActiveShardPlacementOnGroup(localGroupId, shardInterval->shardId) != NULL) { return true; } } return false; } /* * CitusCopyDestReceiverStartup implements the rStartup interface of * CitusCopyDestReceiver. It opens the relation, acquires necessary * locks, and initializes the state required for doing the copy. */ static void CitusCopyDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor) { CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) dest; Oid tableId = copyDest->distributedRelationId; char *relationName = get_rel_name(tableId); Oid schemaOid = get_rel_namespace(tableId); char *schemaName = get_namespace_name(schemaOid); List *columnNameList = copyDest->columnNameList; List *attributeList = NIL; ListCell *columnNameCell = NULL; const char *delimiterCharacter = "\t"; const char *nullPrintCharacter = "\\N"; /* look up table properties */ Relation distributedRelation = table_open(tableId, RowExclusiveLock); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(tableId); copyDest->distributedRelation = distributedRelation; copyDest->tupleDescriptor = inputTupleDescriptor; /* load the list of shards and verify that we have shards to copy into */ List *shardIntervalList = LoadShardIntervalList(tableId); if (shardIntervalList == NIL) { if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find any shards into which to copy"), errdetail("No shards exist for distributed table \"%s\".", relationName), errhint("Run master_create_worker_shards to create shards " "and try again."))); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find any shards into which to copy"), errdetail("No shards exist for distributed table \"%s\".", relationName))); } } /* error if any shard missing min/max values */ if (cacheEntry->hasUninitializedShardInterval) { if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not start copy"), errdetail("Distributed relation \"%s\" has shards " "with missing shardminvalue/shardmaxvalue.", relationName))); } } /* prevent concurrent placement changes and non-commutative DML statements */ LockShardListMetadata(shardIntervalList, ShareLock); /* * Prevent concurrent UPDATE/DELETE on replication factor >1 * (see AcquireExecutorMultiShardLocks() at multi_router_executor.c) */ SerializeNonCommutativeWrites(shardIntervalList, RowExclusiveLock); UseCoordinatedTransaction(); /* all modifications use 2PC */ Use2PCForCoordinatedTransaction(); /* define how tuples will be serialised */ CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; copyOutState->null_print = (char *) nullPrintCharacter; copyOutState->null_print_client = (char *) nullPrintCharacter; copyOutState->binary = CanUseBinaryCopyFormat(inputTupleDescriptor); copyOutState->fe_msgbuf = makeStringInfo(); copyOutState->rowcontext = GetPerTupleMemoryContext(copyDest->executorState); copyDest->copyOutState = copyOutState; copyDest->multiShardCopy = false; /* prepare functions to call on received tuples */ { TupleDesc destTupleDescriptor = distributedRelation->rd_att; int columnCount = inputTupleDescriptor->natts; Oid *finalTypeArray = palloc0(columnCount * sizeof(Oid)); /* * To ensure the proper co-location and distribution of the target table, * the entire process of repartitioning intermediate files requires the * destReceiver to be created on the target rather than the source. * * Within this specific code path, it is assumed that the employed model * is for insert-select. Consequently, it validates the column types of * destTupleDescriptor(target) during the intermediate result generation * process. However, this approach varies significantly for MERGE operations, * where the source tuple(s) can have arbitrary types and are not required to * align with the target column names. * * Despite this minor setback, a significant portion of the code responsible * for repartitioning intermediate files can be reused for the MERGE * operation. By leveraging the ability to perform actual coercion during * the writing process to the target table, we can bypass this specific route. */ if (copyDest->skipCoercions) { copyDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor, copyOutState->binary); } else { copyDest->columnCoercionPaths = ColumnCoercionPaths(destTupleDescriptor, inputTupleDescriptor, tableId, columnNameList, finalTypeArray); copyDest->columnOutputFunctions = TypeOutputFunctions(columnCount, finalTypeArray, copyOutState->binary); } } /* wrap the column names as Values */ foreach(columnNameCell, columnNameList) { char *columnName = (char *) lfirst(columnNameCell); String *columnNameValue = makeString(columnName); attributeList = lappend(attributeList, columnNameValue); } if (IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE) && !IsCitusTableTypeCacheEntry(cacheEntry, SINGLE_SHARD_DISTRIBUTED) && copyDest->partitionColumnIndex == INVALID_PARTITION_COLUMN_INDEX) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("the partition column of table %s should have a value", quote_qualified_identifier(schemaName, relationName)))); } /* define the template for the COPY statement that is sent to workers */ CopyStmt *copyStatement = makeNode(CopyStmt); bool colocatedIntermediateResults = copyDest->colocatedIntermediateResultIdPrefix != NULL; if (colocatedIntermediateResults) { copyStatement->relation = makeRangeVar(NULL, copyDest-> colocatedIntermediateResultIdPrefix, -1); DefElem *formatResultOption = makeDefElem("format", (Node *) makeString("result"), -1); copyStatement->options = list_make1(formatResultOption); } else { copyStatement->relation = makeRangeVar(schemaName, relationName, -1); copyStatement->options = NIL; if (copyOutState->binary) { DefElem *binaryFormatOption = makeDefElem("format", (Node *) makeString("binary"), -1); copyStatement->options = lappend(copyStatement->options, binaryFormatOption); } } copyStatement->query = NULL; copyStatement->attlist = attributeList; copyStatement->is_from = true; copyStatement->is_program = false; copyStatement->filename = NULL; copyDest->copyStatement = copyStatement; copyDest->shardStateHash = CreateShardStateHash(TopTransactionContext); copyDest->connectionStateHash = CreateConnectionStateHash(TopTransactionContext); RecordRelationAccessIfNonDistTable(tableId, PLACEMENT_ACCESS_DML); /* * Colocated intermediate results do not honor citus.max_shared_pool_size, * so we don't need to reserve any connections. Each result file is sent * over a single connection. */ if (!colocatedIntermediateResults) { /* * For all the primary (e.g., writable) remote nodes, reserve a shared * connection. We do this upfront because we cannot know which nodes * are going to be accessed. Since the order of the reservation is * important, we need to do it right here. For the details on why the * order important, see EnsureConnectionPossibilityForNodeList(). * * We don't need to care about local node because we either get a * connection or use local connection, so it cannot be part of * the starvation. As an edge case, if it cannot get a connection * and cannot switch to local execution (e.g., disabled by user), * COPY would fail hinting the user to change the relevant settiing. */ EnsureConnectionPossibilityForRemotePrimaryNodes(); } LocalCopyStatus localCopyStatus = GetLocalCopyStatus(); if (localCopyStatus == LOCAL_COPY_DISABLED) { copyDest->shouldUseLocalCopy = false; } else if (localCopyStatus == LOCAL_COPY_REQUIRED) { copyDest->shouldUseLocalCopy = true; } else if (localCopyStatus == LOCAL_COPY_OPTIONAL) { /* * At this point, there is no requirements for doing the copy locally. * However, if there are local placements, we can try to reserve * a connection to local node. If we cannot reserve, we can still use * local execution. * * NB: It is not advantageous to use remote execution just with a * single remote connection. In other words, a single remote connection * would not perform better than local execution. However, we prefer to * do this because it is likely that the COPY would get more connections * to parallelize the operation. In the future, we might relax this * requirement and failover to local execution as on connection attempt * failures as the executor does. */ if (ShardIntervalListHasLocalPlacements(shardIntervalList)) { bool reservedConnection = TryConnectionPossibilityForLocalPrimaryNode(); copyDest->shouldUseLocalCopy = !reservedConnection; } } } /* * CitusCopyDestReceiverReceive implements the receiveSlot function of * CitusCopyDestReceiver. It takes a TupleTableSlot and sends the contents to * the appropriate shard placement(s). */ static bool CitusCopyDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) { bool result = false; CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) dest; PG_TRY(); { result = CitusSendTupleToPlacements(slot, copyDest); } PG_CATCH(); { /* * We might be able to recover from errors with ROLLBACK TO SAVEPOINT, * so unclaim the connections before throwing errors. */ List *connectionStateList = ConnectionStateList(copyDest->connectionStateHash); UnclaimCopyConnections(connectionStateList); PG_RE_THROW(); } PG_END_TRY(); return result; } /* * CitusSendTupleToPlacements sends the given TupleTableSlot to the appropriate * shard placement(s). */ static bool CitusSendTupleToPlacements(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest) { TupleDesc tupleDescriptor = copyDest->tupleDescriptor; CopyStmt *copyStatement = copyDest->copyStatement; CopyOutState copyOutState = copyDest->copyOutState; FmgrInfo *columnOutputFunctions = copyDest->columnOutputFunctions; CopyCoercionData *columnCoercionPaths = copyDest->columnCoercionPaths; ListCell *placementStateCell = NULL; bool cachedShardStateFound = false; bool firstTupleInShard = false; EState *executorState = copyDest->executorState; MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState); MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); slot_getallattrs(slot); Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; int64 shardId = ShardIdForTuple(copyDest, columnValues, columnNulls); /* connections hash is kept in memory context */ MemoryContextSwitchTo(copyDest->memoryContext); bool isColocatedIntermediateResult = copyDest->colocatedIntermediateResultIdPrefix != NULL; CopyShardState *shardState = GetShardState(shardId, copyDest->shardStateHash, copyDest->connectionStateHash, &cachedShardStateFound, copyDest->shouldUseLocalCopy, copyDest->copyOutState, isColocatedIntermediateResult, copyDest->isPublishable); if (!cachedShardStateFound) { firstTupleInShard = true; } if (firstTupleInShard && !copyDest->multiShardCopy && hash_get_num_entries(copyDest->shardStateHash) == 2) { Oid relationId = copyDest->distributedRelationId; /* mark as multi shard to skip doing the same thing over and over */ copyDest->multiShardCopy = true; if (MultiShardConnectionType != SEQUENTIAL_CONNECTION) { /* when we see multiple shard connections, we mark COPY as parallel modify */ RecordParallelModifyAccess(relationId); } } if (isColocatedIntermediateResult && copyDest->shouldUseLocalCopy && shardState->containsLocalPlacement) { if (firstTupleInShard) { CreateLocalColocatedIntermediateFile(copyDest, shardState); } WriteTupleToLocalFile(slot, copyDest, shardId, shardState->copyOutState, &shardState->fileDest); } else if (copyDest->shouldUseLocalCopy && shardState->containsLocalPlacement) { WriteTupleToLocalShard(slot, copyDest, shardId, shardState->copyOutState); } foreach(placementStateCell, shardState->placementStateList) { CopyPlacementState *currentPlacementState = lfirst(placementStateCell); CopyConnectionState *connectionState = currentPlacementState->connectionState; CopyPlacementState *activePlacementState = connectionState->activePlacementState; bool switchToCurrentPlacement = false; bool sendTupleOverConnection = false; if (activePlacementState == NULL) { switchToCurrentPlacement = true; } else if (currentPlacementState != activePlacementState && currentPlacementState->data->len > CopySwitchOverThresholdBytes) { switchToCurrentPlacement = true; /* before switching, make sure to finish the copy */ EndPlacementStateCopyCommand(activePlacementState, copyOutState); AddPlacementStateToCopyConnectionStateBuffer(connectionState, activePlacementState); } if (switchToCurrentPlacement) { StartPlacementStateCopyCommand(currentPlacementState, copyStatement, copyOutState); RemovePlacementStateFromCopyConnectionStateBuffer(connectionState, currentPlacementState); connectionState->activePlacementState = currentPlacementState; /* send previously buffered tuples */ SendCopyDataToPlacement(currentPlacementState->data, shardId, connectionState->connection); resetStringInfo(currentPlacementState->data); /* additionaly, we need to send the current tuple too */ sendTupleOverConnection = true; } else if (currentPlacementState != activePlacementState) { /* buffer data */ StringInfo copyBuffer = copyOutState->fe_msgbuf; resetStringInfo(copyBuffer); AppendCopyRowData(columnValues, columnNulls, tupleDescriptor, copyOutState, columnOutputFunctions, columnCoercionPaths); appendBinaryStringInfo(currentPlacementState->data, copyBuffer->data, copyBuffer->len); } else { Assert(currentPlacementState == activePlacementState); sendTupleOverConnection = true; } if (sendTupleOverConnection) { resetStringInfo(copyOutState->fe_msgbuf); AppendCopyRowData(columnValues, columnNulls, tupleDescriptor, copyOutState, columnOutputFunctions, columnCoercionPaths); SendCopyDataToPlacement(copyOutState->fe_msgbuf, shardId, connectionState->connection); } } MemoryContextSwitchTo(oldContext); copyDest->tuplesSent++; /* * Release per tuple memory allocated in this function. If we're writing * the results of an INSERT ... SELECT then the SELECT execution will use * its own executor state and reset the per tuple expression context * separately. */ ResetPerTupleExprContext(executorState); return true; } /* * AddPlacementStateToCopyConnectionStateBuffer is a helper function to add a placement * state to connection state's placement buffer. In addition to that, keep the counter * up to date. */ static void AddPlacementStateToCopyConnectionStateBuffer(CopyConnectionState *connectionState, CopyPlacementState *placementState) { dlist_push_head(&connectionState->bufferedPlacementList, &placementState->bufferedPlacementNode); connectionState->bufferedPlacementCount++; } /* * RemovePlacementStateFromCopyConnectionStateBuffer is a helper function to removes a placement * state from connection state's placement buffer. In addition to that, keep the counter * up to date. */ static void RemovePlacementStateFromCopyConnectionStateBuffer(CopyConnectionState *connectionState, CopyPlacementState *placementState) { dlist_delete(&placementState->bufferedPlacementNode); connectionState->bufferedPlacementCount--; } /* * ProcessAppendToShardOption returns the value of append_to_shard if set, * and removes the option from the options list. */ static uint64 ProcessAppendToShardOption(Oid relationId, CopyStmt *copyStatement) { uint64 appendShardId = INVALID_SHARD_ID; bool appendToShardSet = false; DefElem *defel = NULL; foreach_declared_ptr(defel, copyStatement->options) { if (strncmp(defel->defname, APPEND_TO_SHARD_OPTION, NAMEDATALEN) == 0) { appendShardId = defGetInt64(defel); appendToShardSet = true; break; } } if (appendToShardSet) { if (!IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { ereport(ERROR, (errmsg(APPEND_TO_SHARD_OPTION " is only valid for " "append-distributed tables"))); } /* throws an error if shard does not exist */ ShardInterval *shardInterval = LoadShardInterval(appendShardId); /* also check whether shard belongs to table */ if (shardInterval->relationId != relationId) { ereport(ERROR, (errmsg("shard " UINT64_FORMAT " does not belong to table %s", appendShardId, get_rel_name(relationId)))); } copyStatement->options = RemoveOptionFromList(copyStatement->options, APPEND_TO_SHARD_OPTION); } else if (IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { ereport(ERROR, (errmsg("COPY into append-distributed table requires using the " APPEND_TO_SHARD_OPTION " option"))); } return appendShardId; } /* * ContainsLocalPlacement returns true if the current node has * a local placement for the given shard id. */ static bool ContainsLocalPlacement(int64 shardId) { ListCell *placementCell = NULL; List *activePlacementList = ActiveShardPlacementList(shardId); int32 localGroupId = GetLocalGroupId(); foreach(placementCell, activePlacementList) { ShardPlacement *placement = (ShardPlacement *) lfirst(placementCell); if (placement->groupId == localGroupId) { return true; } } return false; } /* * ShardIdForTuple returns id of the shard to which the given tuple belongs to. */ static uint64 ShardIdForTuple(CitusCopyDestReceiver *copyDest, Datum *columnValues, bool *columnNulls) { int partitionColumnIndex = copyDest->partitionColumnIndex; Datum partitionColumnValue = 0; CopyCoercionData *columnCoercionPaths = copyDest->columnCoercionPaths; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(copyDest->distributedRelationId); if (IsCitusTableTypeCacheEntry(cacheEntry, APPEND_DISTRIBUTED)) { return copyDest->appendShardId; } /* * Find the partition column value and corresponding shard interval * for non-reference tables. * Get the existing (and only a single) shard interval for the reference * tables. Note that, reference tables has NULL partition column values so * skip the check. */ if (partitionColumnIndex != INVALID_PARTITION_COLUMN_INDEX) { CopyCoercionData *coercePath = &columnCoercionPaths[partitionColumnIndex]; if (columnNulls[partitionColumnIndex]) { char *qualifiedTableName = generate_qualified_relation_name( copyDest->distributedRelationId); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("the partition column of table %s cannot be NULL", qualifiedTableName))); } /* find the partition column value */ partitionColumnValue = columnValues[partitionColumnIndex]; if (!copyDest->skipCoercions) { /* annoyingly this is evaluated twice, but at least we don't crash! */ partitionColumnValue = CoerceColumnValue(partitionColumnValue, coercePath); } } /* * Find the shard interval and id for the partition column value for * non-reference tables. * * For reference table, and single shard distributed table this function blindly returns the tables single * shard. */ ShardInterval *shardInterval = FindShardInterval(partitionColumnValue, cacheEntry); if (shardInterval == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find shard for partition column " "value"))); } return shardInterval->shardId; } /* * CitusCopyDestReceiverShutdown implements the rShutdown interface of * CitusCopyDestReceiver. It ends the COPY on all the open connections, closes * the relation and increments the query stat counters based on the shards * copied into if requested. */ static void CitusCopyDestReceiverShutdown(DestReceiver *destReceiver) { CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) destReceiver; HTAB *connectionStateHash = copyDest->connectionStateHash; ListCell *connectionStateCell = NULL; Relation distributedRelation = copyDest->distributedRelation; /* * Increment the query stat counters based on the shards copied into * if requested. */ if (copyDest->trackQueryCounters) { int copiedShardCount = copyDest->shardStateHash ? hash_get_num_entries(copyDest->shardStateHash) : 0; if (copiedShardCount <= 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } } List *connectionStateList = ConnectionStateList(connectionStateHash); FinishLocalColocatedIntermediateFiles(copyDest); FinishLocalCopy(copyDest); PG_TRY(); { foreach(connectionStateCell, connectionStateList) { CopyConnectionState *connectionState = (CopyConnectionState *) lfirst(connectionStateCell); ShutdownCopyConnectionState(connectionState, copyDest); } } PG_CATCH(); { /* * We might be able to recover from errors with ROLLBACK TO SAVEPOINT, * so unclaim the connections before throwing errors. */ UnclaimCopyConnections(connectionStateList); PG_RE_THROW(); } PG_END_TRY(); table_close(distributedRelation, NoLock); } /* * FinishLocalCopy sends the remaining copies for local placements. */ static void FinishLocalCopy(CitusCopyDestReceiver *copyDest) { HTAB *shardStateHash = copyDest->shardStateHash; HASH_SEQ_STATUS status; CopyShardState *copyShardState; foreach_htab(copyShardState, &status, shardStateHash) { if (copyShardState->copyOutState != NULL && copyShardState->copyOutState->fe_msgbuf->len > 0) { FinishLocalCopyToShard(copyDest, copyShardState->shardId, copyShardState->copyOutState); } } } /* * CreateLocalColocatedIntermediateFile creates a co-located file for the given * shard, and appends the binary headers if needed. The function also modifies * shardState to set the fileDest and copyOutState. */ static void CreateLocalColocatedIntermediateFile(CitusCopyDestReceiver *copyDest, CopyShardState *shardState) { /* make sure the directory exists */ CreateIntermediateResultsDirectory(); const int fileFlags = (O_CREAT | O_RDWR | O_TRUNC); StringInfo filePath = makeStringInfo(); appendStringInfo(filePath, "%s_%ld", copyDest->colocatedIntermediateResultIdPrefix, shardState->shardId); const char *fileName = QueryResultFileName(filePath->data); shardState->fileDest = FileCompatFromFileStart(FileOpenForTransmit(fileName, fileFlags)); CopyOutState localFileCopyOutState = shardState->copyOutState; bool isBinaryCopy = localFileCopyOutState->binary; if (isBinaryCopy) { AppendCopyBinaryHeaders(localFileCopyOutState); } } /* * FinishLocalColocatedIntermediateFiles iterates over all the colocated * intermediate files and finishes the COPY on all of them. */ static void FinishLocalColocatedIntermediateFiles(CitusCopyDestReceiver *copyDest) { HTAB *shardStateHash = copyDest->shardStateHash; HASH_SEQ_STATUS status; CopyShardState *copyShardState; foreach_htab(copyShardState, &status, shardStateHash) { if (copyShardState->copyOutState != NULL && FILE_IS_OPEN(copyShardState->fileDest.fd)) { FinishLocalCopyToFile(copyShardState->copyOutState, ©ShardState->fileDest); } } } /* * ShutdownCopyConnectionState ends the copy command for the current active * placement on connection, and then sends the rest of the buffers over the * connection. */ static void ShutdownCopyConnectionState(CopyConnectionState *connectionState, CitusCopyDestReceiver *copyDest) { CopyOutState copyOutState = copyDest->copyOutState; CopyStmt *copyStatement = copyDest->copyStatement; dlist_iter iter; CopyPlacementState *activePlacementState = connectionState->activePlacementState; if (activePlacementState != NULL) { EndPlacementStateCopyCommand(activePlacementState, copyOutState); if (!copyDest->isPublishable) { ResetReplicationOriginRemoteSession( activePlacementState->connectionState->connection); } } dlist_foreach(iter, &connectionState->bufferedPlacementList) { CopyPlacementState *placementState = dlist_container(CopyPlacementState, bufferedPlacementNode, iter.cur); uint64 shardId = placementState->shardState->shardId; StartPlacementStateCopyCommand(placementState, copyStatement, copyOutState); SendCopyDataToPlacement(placementState->data, shardId, connectionState->connection); EndPlacementStateCopyCommand(placementState, copyOutState); if (!copyDest->isPublishable) { ResetReplicationOriginRemoteSession(connectionState->connection); } } } /* * CitusCopyDestReceiverDestroy frees the DestReceiver */ static void CitusCopyDestReceiverDestroy(DestReceiver *destReceiver) { CitusCopyDestReceiver *copyDest = (CitusCopyDestReceiver *) destReceiver; if (copyDest->copyOutState) { pfree(copyDest->copyOutState); } if (copyDest->columnOutputFunctions) { pfree(copyDest->columnOutputFunctions); } if (copyDest->columnCoercionPaths) { pfree(copyDest->columnCoercionPaths); } if (copyDest->shardStateHash) { hash_destroy(copyDest->shardStateHash); } if (copyDest->connectionStateHash) { hash_destroy(copyDest->connectionStateHash); } pfree(copyDest); } /* * IsCopyResultStmt determines whether the given copy statement is a * COPY "resultkey" FROM STDIN WITH (format result) statement, which is used * to copy query results from the coordinator into workers. */ bool IsCopyResultStmt(CopyStmt *copyStatement) { return CopyStatementHasFormat(copyStatement, "result"); } /* * CopyStatementHasFormat checks whether the COPY statement has the given * format. */ static bool CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName) { ListCell *optionCell = NULL; bool hasFormat = false; /* extract WITH (...) options from the COPY statement */ foreach(optionCell, copyStatement->options) { DefElem *defel = (DefElem *) lfirst(optionCell); if (strncmp(defel->defname, "format", NAMEDATALEN) == 0 && strncmp(defGetString(defel), formatName, NAMEDATALEN) == 0) { hasFormat = true; break; } } return hasFormat; } /* * ErrorIfCopyHasOnErrorLogVerbosity errors out if the COPY statement * has on_error option or log_verbosity option specified */ static void ErrorIfCopyHasOnErrorLogVerbosity(CopyStmt *copyStatement) { #if PG_VERSION_NUM >= PG_VERSION_17 bool log_verbosity = false; foreach_ptr(DefElem, option, copyStatement->options) { if (strcmp(option->defname, "on_error") == 0) { ereport(ERROR, (errmsg("Citus does not support " "COPY FROM with ON_ERROR option."))); } else if (strcmp(option->defname, "log_verbosity") == 0) { log_verbosity = true; } } /* * Given that log_verbosity is currently used in COPY FROM * when ON_ERROR option is set to ignore, it makes more * sense to error out for ON_ERROR option first. For this reason, * we don't error out in the previous loop directly. * Relevant PG17 commit: https://github.com/postgres/postgres/commit/f5a227895 */ if (log_verbosity) { ereport(ERROR, (errmsg("Citus does not support " "COPY FROM with LOG_VERBOSITY option."))); } #endif } /* * ErrorIfMergeInCopy Raises an exception if the MERGE is called in the COPY * where Citus tables are involved, as we don't support this yet * Relevant PG17 commit: c649fa24a */ static void ErrorIfMergeInCopy(CopyStmt *copyStatement) { #if PG_VERSION_NUM < 170000 return; #else if (!copyStatement->relation && (IsA(copyStatement->query, MergeStmt))) { /* * This path is currently not reachable because Merge in COPY can * only work with a RETURNING clause, and a RETURNING check * will error out sooner for Citus */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("MERGE with Citus tables " "is not yet supported in COPY"))); } #endif } /* * ProcessCopyStmt handles Citus specific concerns for COPY like supporting * COPYing from distributed tables and preventing unsupported actions. The * function returns a modified COPY statement to be executed, or NULL if no * further processing is needed. */ Node * ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletion *completionTag, const char *queryString) { /* * Handle special COPY "resultid" FROM STDIN WITH (format result) commands * for sending intermediate results to workers. */ if (IsCopyResultStmt(copyStatement)) { const char *resultId = copyStatement->relation->relname; if (copyStatement->is_from) { ReceiveQueryResultViaCopy(resultId); } else { SendQueryResultViaCopy(resultId); } return NULL; } /* * We check whether a distributed relation is affected. For that, we need to open the * relation. To prevent race conditions with later lookups, lock the table, and modify * the rangevar to include the schema. */ if (copyStatement->relation != NULL) { ErrorIfMergeInCopy(copyStatement); bool isFrom = copyStatement->is_from; /* consider using RangeVarGetRelidExtended to check perms before locking */ Relation copiedRelation = table_openrv(copyStatement->relation, isFrom ? RowExclusiveLock : AccessShareLock); bool isCitusRelation = IsCitusTable(RelationGetRelid(copiedRelation)); /* ensure future lookups hit the same relation */ char *schemaName = get_namespace_name(RelationGetNamespace(copiedRelation)); /* ensure we copy string into proper context */ MemoryContext relationContext = GetMemoryChunkContext( copyStatement->relation); schemaName = MemoryContextStrdup(relationContext, schemaName); copyStatement->relation->schemaname = schemaName; table_close(copiedRelation, NoLock); if (isCitusRelation) { if (copyStatement->is_from) { if (copyStatement->whereClause) { /* * Update progress reporting for tuples progressed so that the * progress is reflected on pg_stat_progress_copy. Citus currently * does not support COPY .. WHERE clause so TUPLES_EXCLUDED is not * handled. When we remove this check, we should implement progress * reporting as well. */ ereport(ERROR, (errmsg( "Citus does not support COPY FROM with WHERE"))); } ErrorIfCopyHasOnErrorLogVerbosity(copyStatement); /* check permissions, we're bypassing postgres' normal checks */ CheckCopyPermissions(copyStatement); CitusCopyFrom(copyStatement, completionTag); return NULL; } else if (copyStatement->filename == NULL && !copyStatement->is_program && !CopyStatementHasFormat(copyStatement, "binary")) { /* * COPY table TO STDOUT is handled by specialized logic to * avoid buffering the table on the coordinator. This enables * pg_dump of large tables. */ CitusCopyTo(copyStatement, completionTag); return NULL; } else { /* * COPY table TO PROGRAM / file is handled by wrapping the table * in a SELECT and going through the resulting COPY logic. */ SelectStmt *selectStmt = CitusCopySelect(copyStatement); /* replace original statement */ copyStatement = copyObject(copyStatement); copyStatement->relation = NULL; copyStatement->query = (Node *) selectStmt; } } } return (Node *) copyStatement; } /* * CitusCopySelect generates a SelectStmt such that table may be replaced in * "COPY table FROM" for an equivalent result. */ static SelectStmt * CitusCopySelect(CopyStmt *copyStatement) { SelectStmt *selectStmt = makeNode(SelectStmt); selectStmt->fromClause = list_make1(copyObject(copyStatement->relation)); Relation distributedRelation = table_openrv(copyStatement->relation, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); List *targetList = NIL; for (int i = 0; i < tupleDescriptor->natts; i++) { Form_pg_attribute attr = TupleDescAttr(tupleDescriptor, i); if (attr->attisdropped || attr->attgenerated ) { continue; } ColumnRef *column = makeNode(ColumnRef); column->fields = list_make1(makeString(pstrdup(attr->attname.data))); column->location = -1; ResTarget *selectTarget = makeNode(ResTarget); selectTarget->name = NULL; selectTarget->indirection = NIL; selectTarget->val = (Node *) column; selectTarget->location = -1; targetList = lappend(targetList, selectTarget); } table_close(distributedRelation, NoLock); selectStmt->targetList = targetList; return selectStmt; } /* * CitusCopyTo runs a COPY .. TO STDOUT command on each shard to do a full * table dump. */ static void CitusCopyTo(CopyStmt *copyStatement, QueryCompletion *completionTag) { ListCell *shardIntervalCell = NULL; int64 tuplesSent = 0; Relation distributedRelation = table_openrv(copyStatement->relation, AccessShareLock); Oid relationId = RelationGetRelid(distributedRelation); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->fe_msgbuf = makeStringInfo(); copyOutState->binary = false; copyOutState->attnumlist = CopyGetAttnums(tupleDescriptor, distributedRelation, copyStatement->attlist); SendCopyBegin(copyOutState); List *shardIntervalList = LoadShardIntervalList(relationId); foreach(shardIntervalCell, shardIntervalList) { ShardInterval *shardInterval = lfirst(shardIntervalCell); List *shardPlacementList = ActiveShardPlacementList(shardInterval->shardId); ListCell *shardPlacementCell = NULL; int placementIndex = 0; StringInfo copyCommand = ConstructCopyStatement(copyStatement, shardInterval->shardId); foreach(shardPlacementCell, shardPlacementList) { ShardPlacement *shardPlacement = lfirst(shardPlacementCell); int connectionFlags = 0; char *userName = NULL; const bool raiseErrors = true; MultiConnection *connection = GetPlacementConnection(connectionFlags, shardPlacement, userName); /* * This code-path doesn't support optional connections, so we don't expect * NULL connections. */ Assert(connection != NULL); if (placementIndex == list_length(shardPlacementList) - 1) { /* last chance for this shard */ MarkRemoteTransactionCritical(connection); } if (PQstatus(connection->pgConn) != CONNECTION_OK) { ReportConnectionError(connection, ERROR); continue; } RemoteTransactionBeginIfNecessary(connection); if (!SendRemoteCommand(connection, copyCommand->data)) { ReportConnectionError(connection, ERROR); continue; } PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (PQresultStatus(result) != PGRES_COPY_OUT) { ReportResultError(connection, result, ERROR); } PQclear(result); tuplesSent += ForwardCopyDataFromConnection(copyOutState, connection); break; } if (shardIntervalCell == list_head(shardIntervalList)) { /* remove header after the first shard */ copyStatement->options = RemoveOptionFromList(copyStatement->options, "header"); } } SendCopyEnd(copyOutState); if (list_length(shardIntervalList) <= 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } table_close(distributedRelation, AccessShareLock); if (completionTag != NULL) { CompleteCopyQueryTagCompat(completionTag, tuplesSent); } } /* * ForwardCopyDataFromConnection forwards copy data received over the given connection * to the client or file descriptor. */ static int64 ForwardCopyDataFromConnection(CopyOutState copyOutState, MultiConnection *connection) { char *receiveBuffer = NULL; const int useAsync = 0; bool raiseErrors = true; int64 tuplesSent = 0; /* receive copy data message in a synchronous manner */ int receiveLength = PQgetCopyData(connection->pgConn, &receiveBuffer, useAsync); while (receiveLength > 0) { bool includeEndOfLine = false; CopySendData(copyOutState, receiveBuffer, receiveLength); CopySendEndOfRow(copyOutState, includeEndOfLine); tuplesSent++; PQfreemem(receiveBuffer); receiveLength = PQgetCopyData(connection->pgConn, &receiveBuffer, useAsync); } if (receiveLength != -1) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ClearResults(connection, raiseErrors); return tuplesSent; } /* * Check whether the current user has the permission to execute a COPY * statement, raise ERROR if not. In some cases we have to do this separately * from postgres' copy.c, because we have to execute the copy with elevated * privileges. * * Copied from postgres, where it's part of DoCopy(). */ void CheckCopyPermissions(CopyStmt *copyStatement) { /* *INDENT-OFF* */ bool is_from = copyStatement->is_from; Relation rel; List *range_table = NIL; TupleDesc tupDesc; AclMode required_access = (is_from ? ACL_INSERT : ACL_SELECT); List *attnums; ListCell *cur; rel = table_openrv(copyStatement->relation, is_from ? RowExclusiveLock : AccessShareLock); range_table = CreateRangeTable(rel); RangeTblEntry *rte = (RangeTblEntry*) linitial(range_table); tupDesc = RelationGetDescr(rel); /* create permission info for rte */ RTEPermissionInfo *perminfo = GetFilledPermissionInfo(rel->rd_id, rte->inh, required_access); attnums = CopyGetAttnums(tupDesc, rel, copyStatement->attlist); foreach(cur, attnums) { int attno = lfirst_int(cur) - FirstLowInvalidHeapAttributeNumber; if (is_from) { perminfo->insertedCols = bms_add_member(perminfo->insertedCols, attno); } else { perminfo->selectedCols = bms_add_member(perminfo->selectedCols, attno); } } /* link rte to its permission info then check permissions */ rte->perminfoindex = 1; ExecCheckPermissions(list_make1(rte), list_make1(perminfo), true); /* TODO: Perform RLS checks once supported */ table_close(rel, NoLock); /* *INDENT-ON* */ } /* * CreateRangeTable creates a range table with the given relation. */ List * CreateRangeTable(Relation rel) { RangeTblEntry *rte = makeNode(RangeTblEntry); rte->rtekind = RTE_RELATION; rte->relid = rel->rd_id; rte->relkind = rel->rd_rel->relkind; return list_make1(rte); } /* * CreateConnectionStateHash constructs a hash table which maps from socket * number to CopyConnectionState, passing the provided MemoryContext to * hash_create for hash allocations. */ static HTAB * CreateConnectionStateHash(MemoryContext memoryContext) { HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(int); info.entrysize = sizeof(CopyConnectionState); info.hcxt = memoryContext; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); HTAB *connectionStateHash = hash_create("Copy Connection State Hash", 128, &info, hashFlags); return connectionStateHash; } /* * CreateShardStateHash constructs a hash table which maps from shard * identifier to CopyShardState, passing the provided MemoryContext to * hash_create for hash allocations. */ static HTAB * CreateShardStateHash(MemoryContext memoryContext) { HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(uint64); info.entrysize = sizeof(CopyShardState); info.hcxt = memoryContext; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); HTAB *shardStateHash = hash_create("Copy Shard State Hash", 128, &info, hashFlags); return shardStateHash; } /* * GetConnectionState finds existing CopyConnectionState for a connection in the * provided hash. If not found, then a default structure is returned. */ static CopyConnectionState * GetConnectionState(HTAB *connectionStateHash, MultiConnection *connection) { bool found = false; int sock = PQsocket(connection->pgConn); Assert(sock != -1); CopyConnectionState *connectionState = (CopyConnectionState *) hash_search( connectionStateHash, &sock, HASH_ENTER, &found); if (!found) { connectionState->socket = sock; connectionState->connection = connection; connectionState->activePlacementState = NULL; connectionState->bufferedPlacementCount = 0; dlist_init(&connectionState->bufferedPlacementList); } return connectionState; } /* * ConnectionStateList returns all CopyConnectionState structures in * the given hash. */ static List * ConnectionStateList(HTAB *connectionStateHash) { List *connectionStateList = NIL; HASH_SEQ_STATUS status; hash_seq_init(&status, connectionStateHash); CopyConnectionState *connectionState = (CopyConnectionState *) hash_seq_search( &status); while (connectionState != NULL) { connectionStateList = lappend(connectionStateList, connectionState); connectionState = (CopyConnectionState *) hash_seq_search(&status); } return connectionStateList; } /* * ConnectionStateListToNode returns all CopyConnectionState structures in * the given hash for a given hostname and port values. */ static List * ConnectionStateListToNode(HTAB *connectionStateHash, const char *hostname, int32 port) { List *connectionStateList = NIL; HASH_SEQ_STATUS status; hash_seq_init(&status, connectionStateHash); CopyConnectionState *connectionState = (CopyConnectionState *) hash_seq_search(&status); while (connectionState != NULL) { char *connectionHostname = connectionState->connection->hostname; if (strncmp(connectionHostname, hostname, MAX_NODE_LENGTH) == 0 && connectionState->connection->port == port) { connectionStateList = lappend(connectionStateList, connectionState); } connectionState = (CopyConnectionState *) hash_seq_search(&status); } return connectionStateList; } /* * GetShardState finds existing CopyShardState for a shard in the provided * hash. If not found, then a new shard state is returned with all related * CopyPlacementStates initialized. */ static CopyShardState * GetShardState(uint64 shardId, HTAB *shardStateHash, HTAB *connectionStateHash, bool *found, bool shouldUseLocalCopy, CopyOutState copyOutState, bool isColocatedIntermediateResult, bool isPublishable) { CopyShardState *shardState = (CopyShardState *) hash_search(shardStateHash, &shardId, HASH_ENTER, found); if (!*found) { InitializeCopyShardState(shardState, connectionStateHash, shardId, shouldUseLocalCopy, copyOutState, isColocatedIntermediateResult, isPublishable); } return shardState; } /* * InitializeCopyShardState initializes the given shardState. It finds all * placements for the given shardId, assignes connections to them, and * adds them to shardState->placementStateList. */ static void InitializeCopyShardState(CopyShardState *shardState, HTAB *connectionStateHash, uint64 shardId, bool shouldUseLocalCopy, CopyOutState copyOutState, bool colocatedIntermediateResult, bool isPublishable) { ListCell *placementCell = NULL; int failedPlacementCount = 0; bool hasRemoteCopy = false; MemoryContext localContext = AllocSetContextCreateInternal(CurrentMemoryContext, "InitializeCopyShardState", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); /* release active placement list at the end of this function */ MemoryContext oldContext = MemoryContextSwitchTo(localContext); List *activePlacementList = ActiveShardPlacementList(shardId); MemoryContextSwitchTo(oldContext); shardState->shardId = shardId; shardState->placementStateList = NIL; shardState->copyOutState = NULL; shardState->containsLocalPlacement = ContainsLocalPlacement(shardId); shardState->fileDest.fd = -1; foreach(placementCell, activePlacementList) { ShardPlacement *placement = (ShardPlacement *) lfirst(placementCell); if (shouldUseLocalCopy && placement->groupId == GetLocalGroupId()) { shardState->copyOutState = (CopyOutState) palloc0(sizeof(*copyOutState)); CloneCopyOutStateForLocalCopy(copyOutState, shardState->copyOutState); if (colocatedIntermediateResult) { LogLocalCopyToFileExecution(shardId); } else { LogLocalCopyToRelationExecution(shardId); } continue; } hasRemoteCopy = true; MultiConnection *connection = CopyGetPlacementConnection(connectionStateHash, placement, colocatedIntermediateResult); if (connection == NULL) { failedPlacementCount++; continue; } CopyConnectionState *connectionState = GetConnectionState(connectionStateHash, connection); /* * If this is the first time we are using this connection for copying a * shard, send begin if necessary. */ if (connectionState->activePlacementState == NULL) { RemoteTransactionBeginIfNecessary(connection); } if (!isPublishable) { SetupReplicationOriginRemoteSession(connection); } CopyPlacementState *placementState = palloc0(sizeof(CopyPlacementState)); placementState->shardState = shardState; placementState->data = makeStringInfo(); placementState->groupId = placement->groupId; placementState->connectionState = connectionState; /* * We don't set connectionState->activePlacementState here even if it * is NULL. Later in CitusSendTupleToPlacements() we set it at the * same time as calling StartPlacementStateCopyCommand() so we actually * know the COPY operation for the placement is ongoing. */ AddPlacementStateToCopyConnectionStateBuffer(connectionState, placementState); shardState->placementStateList = lappend(shardState->placementStateList, placementState); } /* if all placements failed, error out */ if (failedPlacementCount == list_length(activePlacementList)) { ereport(ERROR, (errmsg("could not connect to any active placements"))); } EnsureTaskExecutionAllowed(hasRemoteCopy); /* * We just error out and code execution should never reach to this * point. This is the case for all tables. */ Assert(failedPlacementCount == 0); MemoryContextReset(localContext); } /* * CloneCopyOutStateForLocalCopy creates a shallow copy of the CopyOutState with a new * fe_msgbuf. We keep a separate CopyOutState for every local shard placement, because * in case of local copy we serialize and buffer incoming tuples into fe_msgbuf for each * placement and the serialization functions take a CopyOutState as a parameter. */ static void CloneCopyOutStateForLocalCopy(CopyOutState from, CopyOutState to) { to->attnumlist = from->attnumlist; to->binary = from->binary; to->copy_dest = from->copy_dest; to->delim = from->delim; to->file_encoding = from->file_encoding; to->need_transcoding = from->need_transcoding; to->null_print = from->null_print; to->null_print_client = from->null_print_client; to->rowcontext = from->rowcontext; to->fe_msgbuf = makeStringInfo(); } /* * LogLocalCopyToRelationExecution logs that the copy will be done * locally for the given shard. */ static void LogLocalCopyToRelationExecution(uint64 shardId) { if (!(LogRemoteCommands || LogLocalCommands)) { return; } ereport(NOTICE, (errmsg("executing the copy locally for shard %lu", shardId))); } /* * LogLocalCopyToFileExecution logs that the copy will be done locally for * a file colocated to the given shard. */ static void LogLocalCopyToFileExecution(uint64 shardId) { if (!(LogRemoteCommands || LogLocalCommands)) { return; } ereport(NOTICE, (errmsg("executing the copy locally for colocated file with " "shard %lu", shardId))); } /* * CopyGetPlacementConnection assigns a connection to the given placement. If * a connection has already been assigned the placement in the current transaction * then it reuses the connection. Otherwise, it requests a connection for placement. */ static MultiConnection * CopyGetPlacementConnection(HTAB *connectionStateHash, ShardPlacement *placement, bool colocatedIntermediateResult) { if (colocatedIntermediateResult) { /* * Colocated intermediate results are just files and not required to use * the same connections with their co-located shards. So, we are free to * use any connection we can get. * * Also, the current connection re-use logic does not know how to handle * intermediate results as the intermediate results always truncates the * existing files. That's why we we use one connection per intermediate * result. * * Also note that we are breaking the guarantees of citus.shared_pool_size * as we cannot rely on optional connections. */ uint32 connectionFlagsForIntermediateResult = 0; MultiConnection *connection = GetNodeConnection(connectionFlagsForIntermediateResult, placement->nodeName, placement->nodePort); /* * As noted above, we want each intermediate file to go over * a separate connection. */ ClaimConnectionExclusively(connection); /* and, we cannot afford to handle failures when anything goes wrong */ MarkRemoteTransactionCritical(connection); return connection; } /* * Determine whether the task has to be assigned to a particular connection * due to a preceding access to the placement in the same transaction. */ ShardPlacementAccess *placementAccess = CreatePlacementAccess(placement, PLACEMENT_ACCESS_DML); uint32 connectionFlags = FOR_DML; MultiConnection *connection = GetConnectionIfPlacementAccessedInXact(connectionFlags, list_make1(placementAccess), NULL); if (connection != NULL) { /* * Errors are supposed to cause immediate aborts (i.e. we don't * want to/can't invalidate placements), mark the connection as * critical so later errors cause failures. */ MarkRemoteTransactionCritical(connection); return connection; } /* * If we exceeded citus.max_adaptive_executor_pool_size, we should re-use the * existing connections to multiplex multiple COPY commands on shards over a * single connection. */ char *nodeName = placement->nodeName; int nodePort = placement->nodePort; List *copyConnectionStateList = ConnectionStateListToNode(connectionStateHash, nodeName, nodePort); if (HasReachedAdaptiveExecutorPoolSize(copyConnectionStateList)) { /* * If we've already reached the executor pool size, there should be at * least one connection to any given node. * * Note that we don't need to mark the connection as critical, since the * connection was already returned by this function before. */ connection = GetLeastUtilisedCopyConnection(copyConnectionStateList, nodeName, nodePort); /* * Make sure that the connection management remembers that Citus * accesses this placement over the connection. */ AssignPlacementListToConnection(list_make1(placementAccess), connection); return connection; } if (IsReservationPossible()) { /* * Enforce the requirements for adaptive connection management * (a.k.a., throttle connections if citus.max_shared_pool_size * reached). * * Given that we have done reservations per node, we do not ever * need to pass WAIT_FOR_CONNECTION, we are sure that there is a * connection either reserved for this backend or already established * by the previous commands in the same transaction block. */ int adaptiveConnectionManagementFlag = OPTIONAL_CONNECTION; connectionFlags |= adaptiveConnectionManagementFlag; } /* * For placements that haven't been assigned a connection by a previous command * in the current transaction, we use a separate connection per placement for * hash-distributed tables in order to get the maximum performance. */ if (placement->partitionMethod == DISTRIBUTE_BY_HASH && MultiShardConnectionType != SEQUENTIAL_CONNECTION) { /* * Claiming the connection exclusively (done below) would also have the * effect of opening multiple connections, but claiming the connection * exclusively prevents GetConnectionIfPlacementAccessedInXact from returning * the connection if it is needed for a different shard placement. * * By setting the REQUIRE_CLEAN_CONNECTION flag we are guaranteed to get * connection that will not be returned by GetConnectionIfPlacementAccessedInXact * for the remainder of the COPY, hence it safe to claim the connection * exclusively. Claiming a connection exclusively prevents it from being * used in other distributed queries that happen during the COPY (e.g. if * the copy logic calls a function to calculate a default value, and the * function does a distributed query). */ connectionFlags |= REQUIRE_CLEAN_CONNECTION; } char *nodeUser = CurrentUserName(); connection = GetPlacementConnection(connectionFlags, placement, nodeUser); if (connection == NULL) { if (list_length(copyConnectionStateList) > 0) { /* * The connection manager throttled any new connections, so pick an existing * connection with least utilization. * * Note that we don't need to mark the connection as critical, since the * connection was already returned by this function before. */ connection = GetLeastUtilisedCopyConnection(copyConnectionStateList, nodeName, nodePort); /* * Make sure that the connection management remembers that Citus * accesses this placement over the connection. */ AssignPlacementListToConnection(list_make1(placementAccess), connection); } else { /* * For this COPY command, we have not established any connections * and adaptive connection management throttled the new connection * request. This could only happen if this COPY command is the * second (or later) COPY command in a transaction block as the * first COPY command always gets a connection per node thanks to * the connection reservation. * * As we know that there has been at least one COPY command happened * earlier, we need to find the connection to that node, and use it. */ connection = ConnectionAvailableToNode(nodeName, nodePort, CurrentUserName(), CurrentDatabaseName()); /* * We do not expect this to happen, but still instead of an assert, * we prefer explicit error message. */ if (connection == NULL) { ereport(ERROR, (errmsg("could not find an available connection"), errhint("Set citus.max_shared_pool_size TO -1 to let " "COPY command finish"))); } } return connection; } if (PQstatus(connection->pgConn) != CONNECTION_OK) { ReportConnectionError(connection, ERROR); } /* * Errors are supposed to cause immediate aborts (i.e. we don't * want to/can't invalidate placements), mark the connection as * critical so later errors cause failures. */ MarkRemoteTransactionCritical(connection); if (MultiShardConnectionType != SEQUENTIAL_CONNECTION) { ClaimConnectionExclusively(connection); } return connection; } /* * HasReachedAdaptiveExecutorPoolSize returns true if the number of entries in input * connection list has greater than or equal to citus.max_adaptive_executor_pool_size. */ static bool HasReachedAdaptiveExecutorPoolSize(List *connectionStateList) { if (list_length(connectionStateList) >= MaxAdaptiveExecutorPoolSize) { /* * We've not reached MaxAdaptiveExecutorPoolSize number of * connections, so we're allowed to establish a new * connection to the given node. */ return true; } return false; } /* * GetLeastUtilisedCopyConnection returns a MultiConnection to the given node * with the least number of placements assigned to it. * * It is assumed that there exists at least one connection to the node. */ static MultiConnection * GetLeastUtilisedCopyConnection(List *connectionStateList, char *nodeName, int nodePort) { MultiConnection *connection = NULL; int minPlacementCount = PG_INT32_MAX; ListCell *connectionStateCell = NULL; /* * We only pick the least utilised connection when some connection limits are * reached such as max_shared_pool_size or max_adaptive_executor_pool_size. * * Therefore there should be some connections to choose from. */ Assert(list_length(connectionStateList) > 0); foreach(connectionStateCell, connectionStateList) { CopyConnectionState *connectionState = lfirst(connectionStateCell); int currentConnectionPlacementCount = connectionState->bufferedPlacementCount; if (connectionState->activePlacementState != NULL) { currentConnectionPlacementCount++; } Assert(currentConnectionPlacementCount > 0); if (currentConnectionPlacementCount < minPlacementCount) { minPlacementCount = currentConnectionPlacementCount; connection = connectionState->connection; } } return connection; } /* * StartPlacementStateCopyCommand sends the COPY for the given placement. It also * sends binary headers if this is a binary COPY. */ static void StartPlacementStateCopyCommand(CopyPlacementState *placementState, CopyStmt *copyStatement, CopyOutState copyOutState) { MultiConnection *connection = placementState->connectionState->connection; uint64 shardId = placementState->shardState->shardId; bool raiseInterrupts = true; bool binaryCopy = copyOutState->binary; StringInfo copyCommand = ConstructCopyStatement(copyStatement, shardId); if (!SendRemoteCommand(connection, copyCommand->data)) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (PQresultStatus(result) != PGRES_COPY_IN) { ReportResultError(connection, result, ERROR); } PQclear(result); if (binaryCopy) { SendCopyBinaryHeaders(copyOutState, shardId, list_make1(connection)); } } /* * EndPlacementStateCopyCommand ends the COPY for the given placement. It also * sends binary footers if this is a binary COPY. */ static void EndPlacementStateCopyCommand(CopyPlacementState *placementState, CopyOutState copyOutState) { MultiConnection *connection = placementState->connectionState->connection; uint64 shardId = placementState->shardState->shardId; bool binaryCopy = copyOutState->binary; /* send footers and end copy command */ if (binaryCopy) { SendCopyBinaryFooters(copyOutState, shardId, list_make1(connection)); } EndRemoteCopy(shardId, list_make1(connection)); } /* * UnclaimCopyConnections unclaims all the connections used for COPY. */ static void UnclaimCopyConnections(List *connectionStateList) { ListCell *connectionStateCell = NULL; foreach(connectionStateCell, connectionStateList) { CopyConnectionState *connectionState = lfirst(connectionStateCell); UnclaimConnection(connectionState->connection); } } /* * IsDroppedOrGenerated - helper function for determining if an attribute is * dropped or generated. Used by COPY and Citus DDL to skip such columns. */ inline bool IsDroppedOrGenerated(Form_pg_attribute attr) { /* * If the "is dropped" flag is true or the generated column flag * is not the default nul character (in which case its value is 's' * for ATTRIBUTE_GENERATED_STORED or possibly 'v' with PG18+ for * ATTRIBUTE_GENERATED_VIRTUAL) then return true. */ return attr->attisdropped || (attr->attgenerated != '\0'); } ================================================ FILE: src/backend/distributed/commands/non_main_db_distribute_object_ops.c ================================================ /*------------------------------------------------------------------------- * * non_main_db_distribute_object_ops.c * * Routines to support node-wide object management commands from non-main * databases. * * RunPreprocessNonMainDBCommand and RunPostprocessNonMainDBCommand are * the entrypoints for this module. These functions are called from * utility_hook.c to support some of the node-wide object management * commands from non-main databases. * * To add support for a new command type, one needs to define a new * NonMainDbDistributeObjectOps object within OperationArray. Also, if * the command requires marking or unmarking some objects as distributed, * the necessary operations can be implemented in * RunPreprocessNonMainDBCommand and RunPostprocessNonMainDBCommand. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/xact.h" #include "catalog/pg_authid_d.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/remote_transaction.h" #define EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER \ "SELECT citus_internal.execute_command_on_remote_nodes_as_user(%s, %s)" #define START_MANAGEMENT_TRANSACTION \ "SELECT citus_internal.start_management_transaction('%lu')" #define MARK_OBJECT_DISTRIBUTED \ "SELECT citus_internal.mark_object_distributed(%d, %s, %d, %s)" #define UNMARK_OBJECT_DISTRIBUTED \ "SELECT pg_catalog.citus_unmark_object_distributed(%d, %d, %d, %s)" /* * NonMainDbDistributeObjectOps contains the necessary callbacks / flags to * support node-wide object management commands from non-main databases. * * cannotBeExecutedInTransaction: * Indicates whether the statement cannot be executed in a transaction. If * this is set to true, the statement will be executed directly on the main * database because there are no transactional visibility issues for such * commands. * * checkSupportedObjectType: * Callback function that checks whether type of the object referred to by * given statement is supported. Can be NULL if not applicable for the * statement type. */ typedef struct NonMainDbDistributeObjectOps { bool cannotBeExecutedInTransaction; bool (*checkSupportedObjectType)(Node *parsetree); } NonMainDbDistributeObjectOps; /* * checkSupportedObjectType callbacks for OperationArray. */ static bool CreateDbStmtCheckSupportedObjectType(Node *node); static bool DropDbStmtCheckSupportedObjectType(Node *node); static bool GrantStmtCheckSupportedObjectType(Node *node); static bool SecLabelStmtCheckSupportedObjectType(Node *node); /* * OperationArray that holds NonMainDbDistributeObjectOps for different command types. */ static const NonMainDbDistributeObjectOps *const OperationArray[] = { [T_CreateRoleStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = NULL }, [T_DropRoleStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = NULL }, [T_AlterRoleStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = NULL }, [T_GrantRoleStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = NULL }, [T_CreatedbStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = true, .checkSupportedObjectType = CreateDbStmtCheckSupportedObjectType }, [T_DropdbStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = true, .checkSupportedObjectType = DropDbStmtCheckSupportedObjectType }, [T_GrantStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = GrantStmtCheckSupportedObjectType }, [T_SecLabelStmt] = &(NonMainDbDistributeObjectOps) { .cannotBeExecutedInTransaction = false, .checkSupportedObjectType = SecLabelStmtCheckSupportedObjectType }, }; /* other static function declarations */ const NonMainDbDistributeObjectOps * GetNonMainDbDistributeObjectOps(Node *parsetree); static void CreateRoleStmtMarkDistGloballyOnMainDbs(CreateRoleStmt *createRoleStmt); static void DropRoleStmtUnmarkDistOnLocalMainDb(DropRoleStmt *dropRoleStmt); static void MarkObjectDistributedGloballyOnMainDbs(Oid catalogRelId, Oid objectId, char *objectName); static void UnmarkObjectDistributedOnLocalMainDb(uint16 catalogRelId, Oid objectId); /* * RunPreprocessNonMainDBCommand runs the necessary commands for a query, in main * database before query is run on the local node with PrevProcessUtility. * * Returns true if previous utility hook needs to be skipped after completing * preprocess phase. */ bool RunPreprocessNonMainDBCommand(Node *parsetree) { if (IsMainDB) { return false; } const NonMainDbDistributeObjectOps *ops = GetNonMainDbDistributeObjectOps(parsetree); if (!ops) { return false; } char *queryString = DeparseTreeNode(parsetree); /* * For the commands that cannot be executed in a transaction, there are no * transactional visibility issues. We directly route them to main database * so that we only have to consider one code-path for such commands. */ if (ops->cannotBeExecutedInTransaction) { IsMainDBCommandInXact = false; RunCitusMainDBQuery((char *) queryString); return true; } IsMainDBCommandInXact = true; StringInfo mainDBQuery = makeStringInfo(); appendStringInfo(mainDBQuery, START_MANAGEMENT_TRANSACTION, GetCurrentFullTransactionId().value); RunCitusMainDBQuery(mainDBQuery->data); mainDBQuery = makeStringInfo(); appendStringInfo(mainDBQuery, EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER, quote_literal_cstr(queryString), quote_literal_cstr(CurrentUserName())); RunCitusMainDBQuery(mainDBQuery->data); if (IsA(parsetree, DropRoleStmt)) { DropRoleStmtUnmarkDistOnLocalMainDb((DropRoleStmt *) parsetree); } return false; } /* * RunPostprocessNonMainDBCommand runs the necessary commands for a query, in main * database after query is run on the local node with PrevProcessUtility. */ void RunPostprocessNonMainDBCommand(Node *parsetree) { if (IsMainDB || !GetNonMainDbDistributeObjectOps(parsetree)) { return; } if (IsA(parsetree, CreateRoleStmt)) { CreateRoleStmtMarkDistGloballyOnMainDbs((CreateRoleStmt *) parsetree); } } /* * GetNonMainDbDistributeObjectOps returns the NonMainDbDistributeObjectOps for given * command if it's node-wide object management command that's supported from non-main * databases. */ const NonMainDbDistributeObjectOps * GetNonMainDbDistributeObjectOps(Node *parsetree) { NodeTag tag = nodeTag(parsetree); if (tag >= lengthof(OperationArray)) { return NULL; } const NonMainDbDistributeObjectOps *ops = OperationArray[tag]; if (ops == NULL) { return NULL; } if (!ops->checkSupportedObjectType || ops->checkSupportedObjectType(parsetree)) { return ops; } return NULL; } /* * CreateRoleStmtMarkDistGloballyOnMainDbs marks the role as * distributed on all main databases globally. */ static void CreateRoleStmtMarkDistGloballyOnMainDbs(CreateRoleStmt *createRoleStmt) { /* object must exist as we've just created it */ bool missingOk = false; Oid roleId = get_role_oid(createRoleStmt->role, missingOk); MarkObjectDistributedGloballyOnMainDbs(AuthIdRelationId, roleId, createRoleStmt->role); } /* * DropRoleStmtUnmarkDistOnLocalMainDb unmarks the roles as * distributed on the local main database. */ static void DropRoleStmtUnmarkDistOnLocalMainDb(DropRoleStmt *dropRoleStmt) { RoleSpec *roleSpec = NULL; foreach_declared_ptr(roleSpec, dropRoleStmt->roles) { Oid roleOid = get_role_oid(roleSpec->rolename, dropRoleStmt->missing_ok); if (roleOid == InvalidOid) { continue; } UnmarkObjectDistributedOnLocalMainDb(AuthIdRelationId, roleOid); } } /* * MarkObjectDistributedGloballyOnMainDbs marks an object as * distributed on all main databases globally. */ static void MarkObjectDistributedGloballyOnMainDbs(Oid catalogRelId, Oid objectId, char *objectName) { StringInfo mainDBQuery = makeStringInfo(); appendStringInfo(mainDBQuery, MARK_OBJECT_DISTRIBUTED, catalogRelId, quote_literal_cstr(objectName), objectId, quote_literal_cstr(CurrentUserName())); RunCitusMainDBQuery(mainDBQuery->data); } /* * UnmarkObjectDistributedOnLocalMainDb unmarks an object as * distributed on the local main database. */ static void UnmarkObjectDistributedOnLocalMainDb(uint16 catalogRelId, Oid objectId) { const int subObjectId = 0; const char *checkObjectExistence = "false"; StringInfo query = makeStringInfo(); appendStringInfo(query, UNMARK_OBJECT_DISTRIBUTED, catalogRelId, objectId, subObjectId, checkObjectExistence); RunCitusMainDBQuery(query->data); } /* * checkSupportedObjectTypes callbacks for OperationArray lie below. */ static bool CreateDbStmtCheckSupportedObjectType(Node *node) { /* * We don't try to send the query to the main database if the CREATE * DATABASE command is for the main database itself, this is a very * rare case but it's exercised by our test suite. */ CreatedbStmt *stmt = castNode(CreatedbStmt, node); return strcmp(stmt->dbname, MainDb) != 0; } static bool DropDbStmtCheckSupportedObjectType(Node *node) { /* * We don't try to send the query to the main database if the DROP * DATABASE command is for the main database itself, this is a very * rare case but it's exercised by our test suite. */ DropdbStmt *stmt = castNode(DropdbStmt, node); return strcmp(stmt->dbname, MainDb) != 0; } static bool GrantStmtCheckSupportedObjectType(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); return stmt->objtype == OBJECT_DATABASE; } static bool SecLabelStmtCheckSupportedObjectType(Node *node) { SecLabelStmt *stmt = castNode(SecLabelStmt, node); return stmt->objtype == OBJECT_ROLE; } ================================================ FILE: src/backend/distributed/commands/owned.c ================================================ /*------------------------------------------------------------------------- * * owned.c * Commands for DROP OWNED statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/objectaddress.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_db_role_setting.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "parser/scansup.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/version_compat.h" #include "distributed/worker_transaction.h" static ObjectAddress * GetNewRoleAddress(ReassignOwnedStmt *stmt); /* * PreprocessDropOwnedStmt finds the distributed role out of the ones * being dropped and unmarks them distributed and creates the drop statements * for the workers. */ List * PreprocessDropOwnedStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropOwnedStmt *stmt = castNode(DropOwnedStmt, node); List *allDropRoles = stmt->roles; List *distributedDropRoles = FilterDistributedRoles(allDropRoles); if (list_length(distributedDropRoles) <= 0) { return NIL; } if (!ShouldPropagate()) { return NIL; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return NIL; } EnsureCoordinator(); stmt->roles = distributedDropRoles; char *sql = DeparseTreeNode((Node *) stmt); stmt->roles = allDropRoles; List *commands = list_make3(DISABLE_DDL_PROPAGATION, sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PostprocessReassignOwnedStmt takes a Node pointer representing a REASSIGN * OWNED statement and performs any necessary post-processing after the statement * has been executed locally. * * We filter out local roles in OWNED BY clause before deparsing the command, * meaning that we skip reassigning what is owned by local roles. However, * if the role specified in TO clause is local, we automatically distribute * it before deparsing the command. */ List * PostprocessReassignOwnedStmt(Node *node, const char *queryString) { ReassignOwnedStmt *stmt = castNode(ReassignOwnedStmt, node); List *allReassignRoles = stmt->roles; List *distributedReassignRoles = FilterDistributedRoles(allReassignRoles); if (list_length(distributedReassignRoles) <= 0) { return NIL; } if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); stmt->roles = distributedReassignRoles; char *sql = DeparseTreeNode((Node *) stmt); stmt->roles = allReassignRoles; ObjectAddress *newRoleAddress = GetNewRoleAddress(stmt); /* * We temporarily enable create / alter role propagation to properly * propagate the role specified in TO clause. */ int saveNestLevel = NewGUCNestLevel(); set_config_option("citus.enable_create_role_propagation", "on", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); set_config_option("citus.enable_alter_role_propagation", "on", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); set_config_option("citus.enable_alter_role_set_propagation", "on", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); EnsureObjectAndDependenciesExistOnAllNodes(newRoleAddress); /* rollback GUCs to the state before this session */ AtEOXact_GUC(true, saveNestLevel); List *commands = list_make3(DISABLE_DDL_PROPAGATION, sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * GetNewRoleAddress returns the ObjectAddress of the new role */ static ObjectAddress * GetNewRoleAddress(ReassignOwnedStmt *stmt) { Oid roleOid = get_role_oid(stmt->newrole->rolename, false); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, AuthIdRelationId, roleOid); return address; } ================================================ FILE: src/backend/distributed/commands/policy.c ================================================ /*------------------------------------------------------------------------- * policy.c * * This file contains functions to create, alter and drop policies on * distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/namespace.h" #include "commands/policy.h" #include "nodes/makefuncs.h" #include "parser/parse_clause.h" #include "parser/parse_relation.h" #include "rewrite/rewriteManip.h" #include "rewrite/rowsecurity.h" #include "utils/builtins.h" #include "utils/ruleutils.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" static const char * unparse_policy_command(const char aclchar); static RowSecurityPolicy * GetPolicyByName(Oid relationId, const char *policyName); static List * GetPolicyListForRelation(Oid relationId); static char * CreatePolicyCommandForPolicy(Oid relationId, RowSecurityPolicy *policy); /* * CreatePolicyCommands takes in a relationId, and returns the list of create policy * commands needed to reconstruct the policies of that table. */ List * CreatePolicyCommands(Oid relationId) { List *commands = NIL; List *policyList = GetPolicyListForRelation(relationId); RowSecurityPolicy *policy; foreach_declared_ptr(policy, policyList) { char *createPolicyCommand = CreatePolicyCommandForPolicy(relationId, policy); commands = lappend(commands, makeTableDDLCommandString(createPolicyCommand)); } return commands; } /* * GetPolicyListForRelation returns a list of RowSecurityPolicy objects identifying * the policies on the relation with relationId. Note that this function acquires * AccessShareLock on relation and does not release it in the end to make sure that * caller will process valid policies through the transaction. */ static List * GetPolicyListForRelation(Oid relationId) { Relation relation = table_open(relationId, AccessShareLock); if (!relation_has_policies(relation)) { table_close(relation, NoLock); return NIL; } if (relation->rd_rsdesc == NULL) { /* * there are policies, but since RLS is not enabled they are not loaded into * cache, we will do so here for us to access */ RelationBuildRowSecurity(relation); } List *policyList = NIL; RowSecurityPolicy *policy; foreach_declared_ptr(policy, relation->rd_rsdesc->policies) { policyList = lappend(policyList, policy); } table_close(relation, NoLock); return policyList; } /* * CreatePolicyCommandForPolicy takes a relationId and a policy, returns * the CREATE POLICY command needed to reconstruct the policy identified * by the "policy" object on the relation with relationId. */ static char * CreatePolicyCommandForPolicy(Oid relationId, RowSecurityPolicy *policy) { char *relationName = generate_qualified_relation_name(relationId); List *relationContext = deparse_context_for(relationName, relationId); StringInfo createPolicyCommand = makeStringInfo(); appendStringInfo(createPolicyCommand, "CREATE POLICY %s ON %s FOR %s", quote_identifier(policy->policy_name), relationName, unparse_policy_command(policy->polcmd)); appendStringInfoString(createPolicyCommand, " TO "); /* * iterate over all roles and append them to the ddl command with commas * separating the role names */ Oid *roles = (Oid *) ARR_DATA_PTR(policy->roles); for (int roleIndex = 0; roleIndex < ARR_DIMS(policy->roles)[0]; roleIndex++) { const char *roleName; if (roleIndex > 0) { appendStringInfoString(createPolicyCommand, ", "); } if (roles[roleIndex] == ACL_ID_PUBLIC) { roleName = "PUBLIC"; } else { roleName = quote_identifier(GetUserNameFromId(roles[roleIndex], false)); } appendStringInfoString(createPolicyCommand, roleName); } if (policy->qual) { char *qualString = deparse_expression((Node *) (policy->qual), relationContext, false, false); appendStringInfo(createPolicyCommand, " USING (%s)", qualString); } if (policy->with_check_qual) { char *withCheckQualString = deparse_expression( (Node *) (policy->with_check_qual), relationContext, false, false); appendStringInfo(createPolicyCommand, " WITH CHECK (%s)", withCheckQualString); } return createPolicyCommand->data; } /* * unparse_policy_command takes the type of a policy command and converts it to its full * command string. This function is the exact inverse of parse_policy_command that is in * postgres. */ static const char * unparse_policy_command(const char aclchar) { switch (aclchar) { case '*': { return "ALL"; } case ACL_SELECT_CHR: { return "SELECT"; } case ACL_INSERT_CHR: { return "INSERT"; } case ACL_UPDATE_CHR: { return "UPDATE"; } case ACL_DELETE_CHR: { return "DELETE"; } default: { elog(ERROR, "unrecognized aclchar: %d", aclchar); return NULL; } } } /* * PostprocessCreatePolicyStmt determines when a CREATE POLICY statement involves * a distributed table. If so, it creates DDLJobs to encapsulate information * needed during the worker node portion of DDL execution before returning the * DDLJobs in a List. If no distributed table is involved, this returns NIL. */ List * PostprocessCreatePolicyStmt(Node *node, const char *queryString) { CreatePolicyStmt *stmt = castNode(CreatePolicyStmt, node); /* load relation information */ RangeVar *relvar = stmt->table; Oid relationId = RangeVarGetRelid(relvar, NoLock, false); if (!IsCitusTable(relationId)) { return NIL; } Relation relation = table_open(relationId, AccessShareLock); ParseState *qual_pstate = make_parsestate(NULL); AddRangeTableEntryToQueryCompat(qual_pstate, relation); Node *qual = transformWhereClause(qual_pstate, copyObject(stmt->qual), EXPR_KIND_POLICY, "POLICY"); if (qual) { ErrorIfUnsupportedPolicyExpr(qual); } ParseState *with_check_pstate = make_parsestate(NULL); AddRangeTableEntryToQueryCompat(with_check_pstate, relation); Node *with_check_qual = transformWhereClause(with_check_pstate, copyObject(stmt->with_check), EXPR_KIND_POLICY, "POLICY"); if (with_check_qual) { ErrorIfUnsupportedPolicyExpr(with_check_qual); } RowSecurityPolicy *policy = GetPolicyByName(relationId, stmt->policy_name); if (policy == NULL) { /* * As this function is executed after standard process utility created the * policy, we should be able to find & deparse the policy with policy_name. * But to be more safe, error out here. */ ereport(ERROR, (errmsg("cannot create policy, policy does not exist."))); } EnsureCoordinator(); char *ddlCommand = CreatePolicyCommandForPolicy(relationId, policy); /* * create the DDLJob that needs to be executed both on the local relation and all its * placements. */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->metadataSyncCommand = pstrdup(ddlCommand); ddlJob->taskList = DDLTaskList(relationId, ddlCommand); relation_close(relation, NoLock); return list_make1(ddlJob); } /* * AddRangeTableEntryToQueryCompat adds the given relation to query. * This method is a compatibility wrapper. */ void AddRangeTableEntryToQueryCompat(ParseState *parseState, Relation relation) { ParseNamespaceItem *rte = addRangeTableEntryForRelation(parseState, relation, AccessShareLock, NULL, false, false); addNSItemToQuery(parseState, rte, false, true, true); } /* * GetPolicyByName takes a relationId and a policyName, returns RowSecurityPolicy * object which identifies the policy with name "policyName" on the relation * with relationId. If there does not exist such a policy, then this function * returns NULL. */ static RowSecurityPolicy * GetPolicyByName(Oid relationId, const char *policyName) { List *policyList = GetPolicyListForRelation(relationId); RowSecurityPolicy *policy = NULL; foreach_declared_ptr(policy, policyList) { if (strncmp(policy->policy_name, policyName, NAMEDATALEN) == 0) { return policy; } } return NULL; } /* * PreprocessAlterPolicyStmt determines whether a given ALTER POLICY statement involves a * distributed table. If so, it creates DDLJobs to encapsulate information needed during * the worker node portion of DDL execution before returning the DDLJobs in a list. If no * distributed table is involved this returns NIL. */ List * PreprocessAlterPolicyStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterPolicyStmt *stmt = castNode(AlterPolicyStmt, node); StringInfoData ddlString; ListCell *roleCell = NULL; /* load relation information */ RangeVar *relvar = stmt->table; Oid relOid = RangeVarGetRelid(relvar, NoLock, false); if (!IsCitusTable(relOid)) { return NIL; } initStringInfo(&ddlString); Relation relation = relation_open(relOid, AccessShareLock); char *relationName = generate_relation_name(relOid, NIL); appendStringInfo(&ddlString, "ALTER POLICY %s ON %s", quote_identifier(stmt->policy_name), relationName ); if (stmt->roles) { appendStringInfoString(&ddlString, " TO "); foreach(roleCell, stmt->roles) { RoleSpec *roleSpec = (RoleSpec *) lfirst(roleCell); appendStringInfoString(&ddlString, RoleSpecString(roleSpec, true)); if (lnext(stmt->roles, roleCell) != NULL) { appendStringInfoString(&ddlString, ", "); } } } List *relationContext = deparse_context_for(relationName, relOid); ParseState *qual_pstate = make_parsestate(NULL); AddRangeTableEntryToQueryCompat(qual_pstate, relation); Node *qual = transformWhereClause(qual_pstate, copyObject(stmt->qual), EXPR_KIND_POLICY, "POLICY"); if (qual) { ErrorIfUnsupportedPolicyExpr(qual); char *qualString = deparse_expression(qual, relationContext, false, false); appendStringInfo(&ddlString, " USING (%s)", qualString); } ParseState *with_check_pstate = make_parsestate(NULL); AddRangeTableEntryToQueryCompat(with_check_pstate, relation); Node *with_check_qual = transformWhereClause(with_check_pstate, copyObject(stmt->with_check), EXPR_KIND_POLICY, "POLICY"); if (with_check_qual) { ErrorIfUnsupportedPolicyExpr(with_check_qual); char *withCheckString = deparse_expression(with_check_qual, relationContext, false, false); appendStringInfo(&ddlString, " WITH CHECK (%s)", withCheckString); } /* * create the DDLJob that needs to be executed both on the local relation and all its * placements. */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relOid); ddlJob->metadataSyncCommand = pstrdup(ddlString.data); ddlJob->taskList = DDLTaskList(relOid, ddlString.data); relation_close(relation, NoLock); return list_make1(ddlJob); } /* * ErrorIfUnsupportedPolicy runs checks related to a Relation their Policies and errors * out if it is not possible to create one of the policies in a distributed environment. * * To support policies we require that: * - Policy expressions do not contain subqueries. */ void ErrorIfUnsupportedPolicy(Relation relation) { ListCell *policyCell = NULL; if (!relation_has_policies(relation)) { return; } /* * even if a relation has policies they might not be loaded on the Relation yet. This * happens if policies are on a Relation without Row Level Security enabled. We need * to make sure the policies installed are valid for distribution if RLS gets enabled * after the table has been distributed. Therefore we force a build of the policies on * the cached Relation */ if (relation->rd_rsdesc == NULL) { RelationBuildRowSecurity(relation); } foreach(policyCell, relation->rd_rsdesc->policies) { RowSecurityPolicy *policy = (RowSecurityPolicy *) lfirst(policyCell); ErrorIfUnsupportedPolicyExpr((Node *) policy->qual); ErrorIfUnsupportedPolicyExpr((Node *) policy->with_check_qual); } } /* * ErrorIfUnsupportedPolicyExpr tests if the provided expression for a policy is * supported on a distributed table. */ void ErrorIfUnsupportedPolicyExpr(Node *expr) { /* * We do not allow any sublink to prevent expressions with subqueries to be used as an * expression in policies on distributed tables. */ if (checkExprHasSubLink(expr)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create policy"), errdetail("Subqueries are not supported in policies on distributed " "tables"))); } } /* * PreprocessDropPolicyStmt determines whether a given DROP POLICY statement involves a * distributed table. If so it creates DDLJobs to encapsulate information needed during * the worker node portion of DDL execution before returning the DDLJobs in a List. If no * distributed table is involved this returns NIL. */ List * PreprocessDropPolicyStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *stmt = castNode(DropStmt, node); List *ddlJobs = NIL; ListCell *dropObjectCell = NULL; Assert(stmt->removeType == OBJECT_POLICY); foreach(dropObjectCell, stmt->objects) { List *names = (List *) lfirst(dropObjectCell); /* * the last element in the list of names is the name of the policy. The ones * before are describing the relation. By removing the last item from the list we * can use makeRangeVarFromNameList to get to the relation. As list_truncate * changes the list in place we make a copy before. */ names = list_copy(names); names = list_truncate(names, list_length(names) - 1); RangeVar *relation = makeRangeVarFromNameList(names); Oid relOid = RangeVarGetRelid(relation, NoLock, false); if (!IsCitusTable(relOid)) { continue; } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relOid); ddlJob->metadataSyncCommand = queryString; ddlJob->taskList = DDLTaskList(relOid, queryString); ddlJobs = lappend(ddlJobs, ddlJob); } return ddlJobs; } /* * IsPolicyRenameStmt returns wherher the passed-in RenameStmt is one of the following * forms: * * - ALTER POLICY ... ON ... RENAME TO ... */ bool IsPolicyRenameStmt(RenameStmt *stmt) { return stmt->renameType == OBJECT_POLICY; } /* * CreatePolicyEventExtendNames extends relation names in the given CreatePolicyStmt tree. * This function has side effects on the tree as the names are replaced inplace. */ void CreatePolicyEventExtendNames(CreatePolicyStmt *stmt, const char *schemaName, uint64 shardId) { RangeVar *relation = stmt->table; char **relationName = &(relation->relname); char **relationSchemaName = &(relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); } /* * AlterPolicyEventExtendNames extends relation names in the given AlterPolicyStatement * tree. This function has side effects on the tree as the names are replaced inplace. */ void AlterPolicyEventExtendNames(AlterPolicyStmt *stmt, const char *schemaName, uint64 shardId) { RangeVar *relation = stmt->table; char **relationName = &(relation->relname); char **relationSchemaName = &(relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); } /* * RenamePolicyEventExtendNames extends relation names in the given RenameStmt tree. This * function has side effects on the tree as the names are replaced inline. */ void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 shardId) { char **relationName = &(stmt->relation->relname); char **objectSchemaName = &(stmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(objectSchemaName, schemaName); AppendShardIdToName(relationName, shardId); } /* * DropPolicyEventExtendNames extends relation names in the given DropStmt tree specific * to policies. This function has side effects on the tree as the names are replaced * inplace. */ void DropPolicyEventExtendNames(DropStmt *dropStmt, const char *schemaName, uint64 shardId) { String *relationSchemaNameValue = NULL; String *relationNameValue = NULL; uint32 dropCount = list_length(dropStmt->objects); if (dropCount > 1) { ereport(ERROR, (errmsg("cannot extend name for multiple drop objects"))); } List *relationNameList = (List *) linitial(dropStmt->objects); int relationNameListLength = list_length(relationNameList); switch (relationNameListLength) { case 2: { relationNameValue = linitial(relationNameList); break; } case 3: { relationSchemaNameValue = linitial(relationNameList); relationNameValue = lsecond(relationNameList); break; } default: { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("improper policy name: \"%s\"", NameListToString(relationNameList)))); break; } } /* prefix with schema name if it is not added already */ if (relationSchemaNameValue == NULL) { String *schemaNameValue = makeString(pstrdup(schemaName)); relationNameList = lcons(schemaNameValue, relationNameList); } char **relationName = &(strVal(relationNameValue)); AppendShardIdToName(relationName, shardId); } ================================================ FILE: src/backend/distributed/commands/publication.c ================================================ /*------------------------------------------------------------------------- * * publication.c * Commands for creating publications * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_rel.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_compat.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/reference_table_utils.h" #include "distributed/worker_create_or_replace.h" static CreatePublicationStmt * BuildCreatePublicationStmt(Oid publicationId); static PublicationObjSpec * BuildPublicationRelationObjSpec(Oid relationId, Oid publicationId, bool tableOnly); static void AppendPublishOptionList(StringInfo str, List *strings); static char * AlterPublicationOwnerCommand(Oid publicationId); static bool ShouldPropagateCreatePublication(CreatePublicationStmt *stmt); static List * ObjectAddressForPublicationName(char *publicationName, bool missingOk); /* * PostProcessCreatePublicationStmt handles CREATE PUBLICATION statements * that contain distributed tables. */ List * PostProcessCreatePublicationStmt(Node *node, const char *queryString) { CreatePublicationStmt *stmt = castNode(CreatePublicationStmt, node); if (!ShouldPropagateCreatePublication(stmt)) { /* should not propagate right now */ return NIL; } /* call into CreatePublicationStmtObjectAddress */ List *publicationAddresses = GetObjectAddressListFromParseTree(node, false, true); /* the code-path only supports a single object */ Assert(list_length(publicationAddresses) == 1); if (IsAnyObjectAddressOwnedByExtension(publicationAddresses, NULL)) { /* should not propagate publications owned by extensions */ return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(publicationAddresses); const ObjectAddress *pubAddress = linitial(publicationAddresses); List *commands = NIL; commands = lappend(commands, DISABLE_DDL_PROPAGATION); commands = lappend(commands, CreatePublicationDDLCommand(pubAddress->objectId)); commands = lappend(commands, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * CreatePublicationDDLCommandsIdempotent returns a list of DDL statements to be * executed on a node to recreate the publication addressed by the publicationAddress. */ List * CreatePublicationDDLCommandsIdempotent(const ObjectAddress *publicationAddress) { Assert(publicationAddress->classId == PublicationRelationId); char *ddlCommand = CreatePublicationDDLCommand(publicationAddress->objectId); char *alterPublicationOwnerSQL = AlterPublicationOwnerCommand(publicationAddress->objectId); return list_make2( WrapCreateOrReplace(ddlCommand), alterPublicationOwnerSQL); } /* * CreatePublicationDDLCommand returns the CREATE PUBLICATION string that * can be used to recreate a given publication. */ char * CreatePublicationDDLCommand(Oid publicationId) { CreatePublicationStmt *createPubStmt = BuildCreatePublicationStmt(publicationId); /* we took the WHERE clause from the catalog where it is already transformed */ bool whereClauseRequiresTransform = false; /* only propagate Citus tables in publication */ bool includeLocalTables = false; return DeparseCreatePublicationStmtExtended((Node *) createPubStmt, whereClauseRequiresTransform, includeLocalTables); } /* * BuildCreatePublicationStmt constructs a CreatePublicationStmt struct for the * given publication. */ static CreatePublicationStmt * BuildCreatePublicationStmt(Oid publicationId) { CreatePublicationStmt *createPubStmt = makeNode(CreatePublicationStmt); HeapTuple publicationTuple = SearchSysCache1(PUBLICATIONOID, ObjectIdGetDatum(publicationId)); if (!HeapTupleIsValid(publicationTuple)) { ereport(ERROR, (errmsg("cannot find publication with oid: %d", publicationId))); } Form_pg_publication publicationForm = (Form_pg_publication) GETSTRUCT(publicationTuple); /* CREATE PUBLICATION */ createPubStmt->pubname = pstrdup(NameStr(publicationForm->pubname)); /* FOR ALL TABLES */ createPubStmt->for_all_tables = publicationForm->puballtables; ReleaseSysCache(publicationTuple); List *schemaIds = GetPublicationSchemas(publicationId); Oid schemaId = InvalidOid; foreach_declared_oid(schemaId, schemaIds) { char *schemaName = get_namespace_name(schemaId); PublicationObjSpec *publicationObject = makeNode(PublicationObjSpec); publicationObject->pubobjtype = PUBLICATIONOBJ_TABLES_IN_SCHEMA; publicationObject->pubtable = NULL; publicationObject->name = schemaName; publicationObject->location = -1; createPubStmt->pubobjects = lappend(createPubStmt->pubobjects, publicationObject); } List *relationIds = GetPublicationRelations(publicationId, publicationForm->pubviaroot ? PUBLICATION_PART_ROOT : PUBLICATION_PART_LEAF); Oid relationId = InvalidOid; /* mainly for consistent ordering in test output */ relationIds = SortList(relationIds, CompareOids); foreach_declared_oid(relationId, relationIds) { bool tableOnly = false; /* since postgres 15, tables can have a column list and filter */ PublicationObjSpec *publicationObject = BuildPublicationRelationObjSpec(relationId, publicationId, tableOnly); createPubStmt->pubobjects = lappend(createPubStmt->pubobjects, publicationObject); } /* WITH (publish_via_partition_root = true) option */ bool publishViaRoot = publicationForm->pubviaroot; char *publishViaRootString = publishViaRoot ? "true" : "false"; DefElem *pubViaRootOption = makeDefElem("publish_via_partition_root", (Node *) makeString(publishViaRootString), -1); createPubStmt->options = lappend(createPubStmt->options, pubViaRootOption); /* WITH (publish_generated_columns = ...) option (PG18+) */ #if PG_VERSION_NUM >= PG_VERSION_18 if (publicationForm->pubgencols == 's') /* stored */ { DefElem *pubGenColsOption = makeDefElem("publish_generated_columns", (Node *) makeString("stored"), -1); createPubStmt->options = lappend(createPubStmt->options, pubGenColsOption); } else if (publicationForm->pubgencols != 'n') /* 'n' = none (default) */ { ereport(ERROR, (errmsg("unexpected pubgencols value '%c' for publication %u", publicationForm->pubgencols, publicationId))); } #endif /* WITH (publish = 'insert, update, delete, truncate') option */ List *publishList = NIL; if (publicationForm->pubinsert) { publishList = lappend(publishList, makeString("insert")); } if (publicationForm->pubupdate) { publishList = lappend(publishList, makeString("update")); } if (publicationForm->pubdelete) { publishList = lappend(publishList, makeString("delete")); } if (publicationForm->pubtruncate) { publishList = lappend(publishList, makeString("truncate")); } if (list_length(publishList) > 0) { StringInfo optionValue = makeStringInfo(); AppendPublishOptionList(optionValue, publishList); DefElem *publishOption = makeDefElem("publish", (Node *) makeString(optionValue->data), -1); createPubStmt->options = lappend(createPubStmt->options, publishOption); } return createPubStmt; } /* * AppendPublishOptionList appends a list of publication options in * comma-separate form. */ static void AppendPublishOptionList(StringInfo str, List *options) { ListCell *stringCell = NULL; foreach(stringCell, options) { const char *string = strVal(lfirst(stringCell)); if (stringCell != list_head(options)) { appendStringInfoString(str, ", "); } /* we cannot escape these strings */ appendStringInfoString(str, string); } } /* * BuildPublicationRelationObjSpec returns a PublicationObjSpec that * can be included in a CREATE or ALTER PUBLICATION statement. */ static PublicationObjSpec * BuildPublicationRelationObjSpec(Oid relationId, Oid publicationId, bool tableOnly) { HeapTuple pubRelationTuple = SearchSysCache2(PUBLICATIONRELMAP, ObjectIdGetDatum(relationId), ObjectIdGetDatum(publicationId)); if (!HeapTupleIsValid(pubRelationTuple)) { ereport(ERROR, (errmsg("cannot find relation with oid %d in publication " "with oid %d", relationId, publicationId))); } List *columnNameList = NIL; Node *whereClause = NULL; /* build the column list */ if (!tableOnly) { bool isNull = false; Datum attributesDatum = SysCacheGetAttr(PUBLICATIONRELMAP, pubRelationTuple, Anum_pg_publication_rel_prattrs, &isNull); if (!isNull) { ArrayType *attributesArray = DatumGetArrayTypeP(attributesDatum); int attributeCount = ARR_DIMS(attributesArray)[0]; int16 *elems = (int16 *) ARR_DATA_PTR(attributesArray); for (int attNumIndex = 0; attNumIndex < attributeCount; attNumIndex++) { AttrNumber attributeNumber = elems[attNumIndex]; char *columnName = get_attname(relationId, attributeNumber, false); columnNameList = lappend(columnNameList, makeString(columnName)); } } /* build the WHERE clause */ Datum whereClauseDatum = SysCacheGetAttr(PUBLICATIONRELMAP, pubRelationTuple, Anum_pg_publication_rel_prqual, &isNull); if (!isNull) { /* * We use the already-transformed parse tree form here, which does * not match regular CreatePublicationStmt */ whereClause = stringToNode(TextDatumGetCString(whereClauseDatum)); } } ReleaseSysCache(pubRelationTuple); char *schemaName = get_namespace_name(get_rel_namespace(relationId)); char *tableName = get_rel_name(relationId); RangeVar *rangeVar = makeRangeVar(schemaName, tableName, -1); /* build the FOR TABLE */ PublicationTable *publicationTable = makeNode(PublicationTable); publicationTable->relation = rangeVar; publicationTable->whereClause = whereClause; publicationTable->columns = columnNameList; PublicationObjSpec *publicationObject = makeNode(PublicationObjSpec); publicationObject->pubobjtype = PUBLICATIONOBJ_TABLE; publicationObject->pubtable = publicationTable; publicationObject->name = NULL; publicationObject->location = -1; return publicationObject; } /* * PreprocessAlterPublicationStmt handles ALTER PUBLICATION statements * in a way that is mostly similar to PreprocessAlterDistributedObjectStmt, * except we do not ensure sequential mode (publications do not interact with * shards) and can handle NULL deparse commands for ALTER PUBLICATION commands * that only involve local tables. */ List * PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext) { List *addresses = GetObjectAddressListFromParseTree(stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode(stmt); const char *sql = DeparseTreeNode((Node *) stmt); if (sql == NULL) { /* * Deparsing logic decided that there is nothing to propagate, e.g. * because the command only concerns local tables. */ return NIL; } List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * GetAlterPublicationDDLCommandsForTable gets a list of ALTER PUBLICATION .. ADD/DROP * commands for the given table. * * If isAdd is true, it return ALTER PUBLICATION .. ADD TABLE commands for all * publications. * * Otherwise, it returns ALTER PUBLICATION .. DROP TABLE commands for all * publications. */ List * GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd) { List *commands = NIL; List *publicationIds = GetRelationPublications(relationId); Oid publicationId = InvalidOid; foreach_declared_oid(publicationId, publicationIds) { char *command = GetAlterPublicationTableDDLCommand(publicationId, relationId, isAdd); commands = lappend(commands, command); } return commands; } /* * GetAlterPublicationTableDDLCommand generates an ALTer PUBLICATION .. ADD/DROP TABLE * command for the given publication and relation ID. * * If isAdd is true, it return an ALTER PUBLICATION .. ADD TABLE command. * Otherwise, it returns ALTER PUBLICATION .. DROP TABLE command. */ char * GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, bool isAdd) { HeapTuple pubTuple = SearchSysCache1(PUBLICATIONOID, ObjectIdGetDatum(publicationId)); if (!HeapTupleIsValid(pubTuple)) { ereport(ERROR, (errmsg("cannot find publication with oid: %d", publicationId))); } Form_pg_publication pubForm = (Form_pg_publication) GETSTRUCT(pubTuple); AlterPublicationStmt *alterPubStmt = makeNode(AlterPublicationStmt); alterPubStmt->pubname = pstrdup(NameStr(pubForm->pubname)); ReleaseSysCache(pubTuple); bool tableOnly = !isAdd; /* since postgres 15, tables can have a column list and filter */ PublicationObjSpec *publicationObject = BuildPublicationRelationObjSpec(relationId, publicationId, tableOnly); alterPubStmt->pubobjects = lappend(alterPubStmt->pubobjects, publicationObject); alterPubStmt->action = isAdd ? AP_AddObjects : AP_DropObjects; /* we take the WHERE clause from the catalog where it is already transformed */ bool whereClauseNeedsTransform = false; /* * We use these commands to restore publications before/after transforming a * table, including transformations to/from local tables. */ bool includeLocalTables = true; char *command = DeparseAlterPublicationStmtExtended((Node *) alterPubStmt, whereClauseNeedsTransform, includeLocalTables); return command; } /* * AlterPublicationOwnerCommand returns "ALTER PUBLICATION .. OWNER TO .." * statement for the specified publication. */ static char * AlterPublicationOwnerCommand(Oid publicationId) { HeapTuple publicationTuple = SearchSysCache1(PUBLICATIONOID, ObjectIdGetDatum(publicationId)); if (!HeapTupleIsValid(publicationTuple)) { ereport(ERROR, (errmsg("cannot find publication with oid: %d", publicationId))); } Form_pg_publication publicationForm = (Form_pg_publication) GETSTRUCT(publicationTuple); char *publicationName = NameStr(publicationForm->pubname); Oid publicationOwnerId = publicationForm->pubowner; char *publicationOwnerName = GetUserNameFromId(publicationOwnerId, false); StringInfo alterCommand = makeStringInfo(); appendStringInfo(alterCommand, "ALTER PUBLICATION %s OWNER TO %s", quote_identifier(publicationName), quote_identifier(publicationOwnerName)); ReleaseSysCache(publicationTuple); return alterCommand->data; } /* * ShouldPropagateCreatePublication tests if we need to propagate a CREATE PUBLICATION * statement. */ static bool ShouldPropagateCreatePublication(CreatePublicationStmt *stmt) { if (!ShouldPropagate()) { return false; } if (!ShouldPropagateCreateInCoordinatedTransction()) { return false; } return true; } /* * AlterPublicationStmtObjectAddress generates the object address for the * publication altered by a regular ALTER PUBLICATION .. statement. */ List * AlterPublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess) { AlterPublicationStmt *stmt = castNode(AlterPublicationStmt, node); return ObjectAddressForPublicationName(stmt->pubname, missingOk); } /* * AlterPublicationOwnerStmtObjectAddress generates the object address for the * publication altered by the given ALTER PUBLICATION .. OWNER TO statement. */ List * AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); return ObjectAddressForPublicationName(strVal(stmt->object), missingOk); } /* * CreatePublicationStmtObjectAddress generates the object address for the * publication created by the given CREATE PUBLICATION statement. */ List * CreatePublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess) { CreatePublicationStmt *stmt = castNode(CreatePublicationStmt, node); return ObjectAddressForPublicationName(stmt->pubname, missingOk); } /* * RenamePublicationStmtObjectAddress generates the object address for the * publication altered by the given ALter PUBLICATION .. RENAME TO statement. */ List * RenamePublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); return ObjectAddressForPublicationName(strVal(stmt->object), missingOk); } /* * ObjectAddressForPublicationName returns the object address for a given publication * name. */ static List * ObjectAddressForPublicationName(char *publicationName, bool missingOk) { Oid publicationId = InvalidOid; HeapTuple publicationTuple = SearchSysCache1(PUBLICATIONNAME, CStringGetDatum(publicationName)); if (HeapTupleIsValid(publicationTuple)) { Form_pg_publication publicationForm = (Form_pg_publication) GETSTRUCT(publicationTuple); publicationId = publicationForm->oid; ReleaseSysCache(publicationTuple); } else if (!missingOk) { /* it should have just been created */ ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("publication \"%s\" does not exist", publicationName))); } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, PublicationRelationId, publicationId); return list_make1(address); } ================================================ FILE: src/backend/distributed/commands/rename.c ================================================ /*------------------------------------------------------------------------- * * rename.c * Commands for renaming objects related to distributed tables * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "nodes/parsenodes.h" #include "utils/lsyscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/metadata_cache.h" /* * PreprocessRenameStmt first determines whether a given rename statement involves * a distributed table. If so (and if it is supported, i.e. renames a column), * it creates a DDLJob to encapsulate information needed during the worker node * portion of DDL execution before returning that DDLJob in a List. If no dis- * tributed table is involved, this function returns NIL. */ List * PreprocessRenameStmt(Node *node, const char *renameCommand, ProcessUtilityContext processUtilityContext) { RenameStmt *renameStmt = castNode(RenameStmt, node); Oid objectRelationId = InvalidOid; /* SQL Object OID */ Oid tableRelationId = InvalidOid; /* Relation OID, maybe not the same. */ /* * We only support some of the PostgreSQL supported RENAME statements, and * our list include only renaming table, index, policy and view (related) objects. */ if (!IsAlterTableRenameStmt(renameStmt) && !IsIndexRenameStmt(renameStmt) && !IsPolicyRenameStmt(renameStmt) && !IsViewRenameStmt(renameStmt)) { return NIL; } /* * The lock levels here should be same as the ones taken in * RenameRelation(), renameatt() and RenameConstraint(). All statements * have identical lock levels except alter index rename. */ LOCKMODE lockmode = (IsIndexRenameStmt(renameStmt)) ? ShareUpdateExclusiveLock : AccessExclusiveLock; objectRelationId = RangeVarGetRelid(renameStmt->relation, lockmode, renameStmt->missing_ok); /* * If the table does not exist, don't do anything here to allow PostgreSQL * to throw the appropriate error or notice message later. */ if (!OidIsValid(objectRelationId)) { return NIL; } /* * Check whether we are dealing with a sequence or view here and route queries * accordingly to the right processor function. We need to check both objects here * since PG supports targeting sequences and views with ALTER TABLE commands. */ char relKind = get_rel_relkind(objectRelationId); if (relKind == RELKIND_SEQUENCE) { RenameStmt *stmtCopy = copyObject(renameStmt); stmtCopy->renameType = OBJECT_SEQUENCE; return PreprocessRenameSequenceStmt((Node *) stmtCopy, renameCommand, processUtilityContext); } else if (relKind == RELKIND_VIEW) { RenameStmt *stmtCopy = copyObject(renameStmt); stmtCopy->relationType = OBJECT_VIEW; if (stmtCopy->renameType == OBJECT_TABLE) { stmtCopy->renameType = OBJECT_VIEW; } return PreprocessRenameViewStmt((Node *) stmtCopy, renameCommand, processUtilityContext); } /* we have no planning to do unless the table is distributed */ switch (renameStmt->renameType) { case OBJECT_TABLE: case OBJECT_FOREIGN_TABLE: case OBJECT_COLUMN: case OBJECT_TABCONSTRAINT: case OBJECT_POLICY: { if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { /* * Although weird, postgres allows ALTER TABLE .. RENAME command * on indexes. We don't want to break non-distributed tables, * so allow. */ tableRelationId = IndexGetRelation(objectRelationId, false); break; } /* the target object is our tableRelationId. */ tableRelationId = objectRelationId; break; } case OBJECT_INDEX: { if (relKind == RELKIND_RELATION || relKind == RELKIND_PARTITIONED_TABLE) { /* * Although weird, postgres allows ALTER INDEX .. RENAME command * on tables. We don't want to break non-distributed tables, * so allow. * Because of the weird syntax, we locked with wrong level, so relock * the relation to acquire true level of lock. Same logic * can be found in the function RenameRelation(RenameStmt) at tablecmds.c */ UnlockRelationOid(objectRelationId, lockmode); objectRelationId = RangeVarGetRelid(renameStmt->relation, AccessExclusiveLock, renameStmt->missing_ok); tableRelationId = objectRelationId; break; } /* * here, objRelationId points to the index relation entry, and we * are interested into the entry of the table on which the index is * defined. */ tableRelationId = IndexGetRelation(objectRelationId, false); break; } default: { /* * Nodes that are not supported by Citus: we pass-through to the * main PostgreSQL executor. Any Citus-supported RenameStmt * renameType must appear above in the switch, explicitly. */ return NIL; } } bool isCitusRelation = IsCitusTable(tableRelationId); if (!isCitusRelation) { return NIL; } /* * We might ERROR out on some commands, but only for Citus tables. * That's why this test comes this late in the function. */ ErrorIfUnsupportedRenameStmt(renameStmt); if (renameStmt->renameType == OBJECT_TABLE || renameStmt->renameType == OBJECT_FOREIGN_TABLE) { SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(tableRelationId, renameStmt->newname); } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, tableRelationId); ddlJob->metadataSyncCommand = renameCommand; ddlJob->taskList = DDLTaskList(tableRelationId, renameCommand); return list_make1(ddlJob); } /* * ErrorIfUnsupportedRenameStmt errors out if the corresponding rename statement * operates on any part of a distributed table other than a column. * * Note: This function handles RenameStmt applied to relations handed by Citus. * At the moment of writing this comment, it could be either tables or indexes. */ void ErrorIfUnsupportedRenameStmt(RenameStmt *renameStmt) { if (IsAlterTableRenameStmt(renameStmt) && renameStmt->renameType == OBJECT_TABCONSTRAINT) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("renaming constraints belonging to distributed tables is " "currently unsupported"))); } } /* * PreprocessRenameAttributeStmt called for RenameStmt's that are targetting an attribute eg. * type attributes. Based on the relation type the attribute gets renamed it dispatches to * a specialized implementation if present, otherwise return an empty list for its DDLJobs */ List * PreprocessRenameAttributeStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); switch (stmt->relationType) { case OBJECT_TYPE: { return PreprocessRenameTypeAttributeStmt(node, queryString, processUtilityContext); } default: { /* unsupported relation for attribute rename, do nothing */ return NIL; } } } ================================================ FILE: src/backend/distributed/commands/role.c ================================================ /*------------------------------------------------------------------------- * * role.c * Commands for ALTER ROLE statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/table.h" #include "access/xact.h" #include "catalog/catalog.h" #include "catalog/objectaddress.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_db_role_setting.h" #include "catalog/pg_shseclabel.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "parser/scansup.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/guc_tables.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/comment.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/version_compat.h" #include "distributed/worker_transaction.h" static const char * ExtractEncryptedPassword(Oid roleOid); static const char * CreateAlterRoleIfExistsCommand(AlterRoleStmt *stmt); static const char * CreateAlterRoleSetIfExistsCommand(AlterRoleSetStmt *stmt); static char * CreateCreateOrAlterRoleCommand(const char *roleName, CreateRoleStmt *createRoleStmt, AlterRoleStmt *alterRoleStmt); static DefElem * makeDefElemInt(char *name, int value); static DefElem * makeDefElemBool(char *name, bool value); static List * GenerateRoleOptionsList(HeapTuple tuple); static List * GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options); static List * GenerateGrantRoleStmtsOfRole(Oid roleid); static List * GenerateSecLabelOnRoleStmts(Oid roleid, char *rolename); static void EnsureSequentialModeForRoleDDL(void); static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); static char * GetDatabaseNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); #if PG_VERSION_NUM < PG_VERSION_17 static Node * makeStringConst(char *str, int location); #endif static Node * makeIntConst(int val, int location); static Node * makeFloatConst(char *str, int location); static const char * WrapQueryInAlterRoleIfExistsCall(const char *query, RoleSpec *role); static VariableSetStmt * MakeVariableSetStmt(const char *config); static int ConfigGenericNameCompare(const void *lhs, const void *rhs); static List * RoleSpecToObjectAddress(RoleSpec *role, bool missing_ok); /* controlled via GUC */ bool EnableCreateRolePropagation = true; bool EnableAlterRolePropagation = true; bool EnableAlterRoleSetPropagation = true; /* * AlterRoleStmtObjectAddress returns the ObjectAddress of the role in the * AlterRoleStmt. If missing_ok is set to false an error will be raised if postgres * was unable to find the role that was the target of the statement. */ List * AlterRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterRoleStmt *stmt = castNode(AlterRoleStmt, node); return RoleSpecToObjectAddress(stmt->role, missing_ok); } /* * AlterRoleSetStmtObjectAddress returns the ObjectAddress of the role in the * AlterRoleSetStmt. If missing_ok is set to false an error will be raised if postgres * was unable to find the role that was the target of the statement. */ List * AlterRoleSetStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterRoleSetStmt *stmt = castNode(AlterRoleSetStmt, node); return RoleSpecToObjectAddress(stmt->role, missing_ok); } /* * RoleSpecToObjectAddress returns the ObjectAddress of a Role associated with a * RoleSpec. If missing_ok is set to false an error will be raised by postgres * explaining the Role could not be found. */ static List * RoleSpecToObjectAddress(RoleSpec *role, bool missing_ok) { ObjectAddress *address = palloc0(sizeof(ObjectAddress)); if (role != NULL) { /* roles can be NULL for statements on ALL roles eg. ALTER ROLE ALL SET ... */ Oid roleOid = get_rolespec_oid(role, missing_ok); ObjectAddressSet(*address, AuthIdRelationId, roleOid); } return list_make1(address); } /* * PostprocessAlterRoleStmt actually creates the plan we need to execute for alter * role statement. We need to do it this way because we need to use the encrypted * password, which is, in some cases, created at standardProcessUtility. */ List * PostprocessAlterRoleStmt(Node *node, const char *queryString) { List *addresses = GetObjectAddressListFromParseTree(node, false, true); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } if (!EnableAlterRolePropagation) { return NIL; } EnsurePropagationToCoordinator(); AlterRoleStmt *stmt = castNode(AlterRoleStmt, node); DefElem *option = NULL; foreach_declared_ptr(option, stmt->options) { if (strcasecmp(option->defname, "password") == 0) { Oid roleOid = get_rolespec_oid(stmt->role, true); const char *encryptedPassword = ExtractEncryptedPassword(roleOid); if (encryptedPassword != NULL) { String *encryptedPasswordValue = makeString((char *) encryptedPassword); option->arg = (Node *) encryptedPasswordValue; } else { option->arg = NULL; } break; } } List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) CreateAlterRoleIfExistsCommand(stmt), ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commands); } /* * PreprocessAlterRoleSetStmt actually creates the plan we need to execute for alter * role set statement. */ List * PreprocessAlterRoleSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagate()) { return NIL; } if (!EnableAlterRoleSetPropagation) { return NIL; } AlterRoleSetStmt *stmt = castNode(AlterRoleSetStmt, node); /* don't propagate if the statement is scoped to another database */ if (stmt->database != NULL && strcmp(stmt->database, get_database_name(MyDatabaseId)) != 0) { return NIL; } List *addresses = GetObjectAddressListFromParseTree(node, false, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); /* * stmt->role could be NULL when the statement is on 'ALL' roles, we do propagate for * ALL roles. If it is not NULL the role is for a specific role. If that role is not * distributed we will not propagate the statement */ if (stmt->role != NULL && !IsAnyObjectDistributed(addresses)) { return NIL; } EnsurePropagationToCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commandList = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commandList); } /* * CreateAlterRoleIfExistsCommand creates ALTER ROLE command, from the alter role node * using the alter_role_if_exists() UDF. */ static const char * CreateAlterRoleIfExistsCommand(AlterRoleStmt *stmt) { const char *alterRoleQuery = DeparseTreeNode((Node *) stmt); return WrapQueryInAlterRoleIfExistsCall(alterRoleQuery, stmt->role); } /* * CreateAlterRoleSetIfExistsCommand creates ALTER ROLE .. SET command, from the * AlterRoleSetStmt node. * * If the statement affects a single user, the query is wrapped in a * alter_role_if_exists() to make sure that it is run on workers that has a user * with the same name. If the query is a ALTER ROLE ALL .. SET query, the query * is sent to the workers as is. */ static const char * CreateAlterRoleSetIfExistsCommand(AlterRoleSetStmt *stmt) { char *alterRoleSetQuery = DeparseTreeNode((Node *) stmt); /* ALTER ROLE ALL .. SET queries should not be wrapped in a alter_role_if_exists() call */ if (stmt->role == NULL) { return alterRoleSetQuery; } else { return WrapQueryInAlterRoleIfExistsCall(alterRoleSetQuery, stmt->role); } } /* * WrapQueryInAlterRoleIfExistsCall wraps a given query in a alter_role_if_exists() * UDF. */ static const char * WrapQueryInAlterRoleIfExistsCall(const char *query, RoleSpec *role) { StringInfoData buffer = { 0 }; const char *roleName = RoleSpecString(role, false); initStringInfo(&buffer); appendStringInfo(&buffer, "SELECT alter_role_if_exists(%s, %s)", quote_literal_cstr(roleName), quote_literal_cstr(query)); return buffer.data; } /* * CreateCreateOrAlterRoleCommand creates ALTER ROLE command, from the alter role node * using the alter_role_if_exists() UDF. */ static char * CreateCreateOrAlterRoleCommand(const char *roleName, CreateRoleStmt *createRoleStmt, AlterRoleStmt *alterRoleStmt) { StringInfoData createOrAlterRoleQueryBuffer = { 0 }; const char *createRoleQuery = "null"; const char *alterRoleQuery = "null"; if (createRoleStmt != NULL) { createRoleQuery = quote_literal_cstr(DeparseTreeNode((Node *) createRoleStmt)); } if (alterRoleStmt != NULL) { alterRoleQuery = quote_literal_cstr(DeparseTreeNode((Node *) alterRoleStmt)); } initStringInfo(&createOrAlterRoleQueryBuffer); appendStringInfo(&createOrAlterRoleQueryBuffer, "SELECT worker_create_or_alter_role(%s, %s, %s)", quote_literal_cstr(roleName), createRoleQuery, alterRoleQuery); return createOrAlterRoleQueryBuffer.data; } /* * ExtractEncryptedPassword extracts the encrypted password of a role. The function * gets the password from the pg_authid table. */ static const char * ExtractEncryptedPassword(Oid roleOid) { Relation pgAuthId = table_open(AuthIdRelationId, AccessShareLock); TupleDesc pgAuthIdDescription = RelationGetDescr(pgAuthId); HeapTuple tuple = SearchSysCache1(AUTHOID, roleOid); bool isNull = true; if (!HeapTupleIsValid(tuple)) { return NULL; } Datum passwordDatum = heap_getattr(tuple, Anum_pg_authid_rolpassword, pgAuthIdDescription, &isNull); /* * In PG, an empty password is treated the same as NULL. * So we propagate NULL password to the other nodes, even if * the user supplied an empty password */ char *passwordCstring = NULL; if (!isNull) { passwordCstring = pstrdup(TextDatumGetCString(passwordDatum)); } table_close(pgAuthId, AccessShareLock); ReleaseSysCache(tuple); return passwordCstring; } /* * GenerateAlterRoleSetIfExistsCommandList generate a list of ALTER ROLE .. SET commands that * copies a role session defaults from the pg_db_role_settings table. */ static List * GenerateAlterRoleSetIfExistsCommandList(HeapTuple tuple, TupleDesc DbRoleSettingDescription) { AlterRoleSetStmt *stmt = makeNode(AlterRoleSetStmt); List *commandList = NIL; bool isnull = false; char *databaseName = GetDatabaseNameFromDbRoleSetting(tuple, DbRoleSettingDescription); if (databaseName != NULL) { stmt->database = databaseName; } char *roleName = GetRoleNameFromDbRoleSetting(tuple, DbRoleSettingDescription); if (roleName != NULL) { stmt->role = makeNode(RoleSpec); stmt->role->location = -1; stmt->role->roletype = ROLESPEC_CSTRING; stmt->role->rolename = roleName; } Datum setconfig = heap_getattr(tuple, Anum_pg_db_role_setting_setconfig, DbRoleSettingDescription, &isnull); Datum *configs; int nconfigs; int i; deconstruct_array(DatumGetArrayTypeP(setconfig), TEXTOID, -1, false, 'i', &configs, NULL, &nconfigs); /* * A tuple might contain one or more settings that apply to the user-database combination. * ALTER ROLE ... SET ... only allows to set one at a time. We will create a statement for every * configuration contained in the tuple. */ for (i = 0; i < nconfigs; i++) { char *config = TextDatumGetCString(configs[i]); stmt->setstmt = MakeVariableSetStmt(config); commandList = lappend(commandList, (void *) CreateAlterRoleSetIfExistsCommand(stmt)); } return commandList; } /* * MakeVariableSetStmt takes a "some-option=some value" string and creates a * VariableSetStmt Node. */ static VariableSetStmt * MakeVariableSetStmt(const char *config) { char *name = NULL; char *value = NULL; ParseLongOption(config, &name, &value); VariableSetStmt *variableSetStmt = makeNode(VariableSetStmt); variableSetStmt->kind = VAR_SET_VALUE; variableSetStmt->name = name; variableSetStmt->args = MakeSetStatementArguments(name, value); return variableSetStmt; } /* * GenerateRoleOptionsList returns the list of options set on a user based on the record * in pg_authid. It requires the HeapTuple for a user entry to access both its fixed * length and variable length fields. */ static List * GenerateRoleOptionsList(HeapTuple tuple) { Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(tuple)); List *options = NIL; options = lappend(options, makeDefElemBool("superuser", role->rolsuper)); options = lappend(options, makeDefElemBool("createdb", role->rolcreatedb)); options = lappend(options, makeDefElemBool("createrole", role->rolcreaterole)); options = lappend(options, makeDefElemBool("inherit", role->rolinherit)); options = lappend(options, makeDefElemBool("canlogin", role->rolcanlogin)); options = lappend(options, makeDefElemBool("isreplication", role->rolreplication)); options = lappend(options, makeDefElemBool("bypassrls", role->rolbypassrls)); options = lappend(options, makeDefElemInt("connectionlimit", role->rolconnlimit)); /* load password from heap tuple, use NULL if not set */ bool isNull = true; Datum rolPasswordDatum = SysCacheGetAttr(AUTHNAME, tuple, Anum_pg_authid_rolpassword, &isNull); if (!isNull) { char *rolPassword = pstrdup(TextDatumGetCString(rolPasswordDatum)); Node *passwordStringNode = (Node *) makeString(rolPassword); DefElem *passwordOption = makeDefElem("password", passwordStringNode, -1); options = lappend(options, passwordOption); } else { options = lappend(options, makeDefElem("password", NULL, -1)); } /* load valid until data from the heap tuple */ Datum rolValidUntilDatum = SysCacheGetAttr(AUTHNAME, tuple, Anum_pg_authid_rolvaliduntil, &isNull); if (!isNull) { char *rolValidUntil = pstrdup((char *) timestamptz_to_str(rolValidUntilDatum)); Node *validUntilStringNode = (Node *) makeString(rolValidUntil); DefElem *validUntilOption = makeDefElem("validUntil", validUntilStringNode, -1); options = lappend(options, validUntilOption); } return options; } /* * GenerateCreateOrAlterRoleCommand generates ALTER ROLE command that copies a role from * the pg_authid table. */ List * GenerateCreateOrAlterRoleCommand(Oid roleOid) { HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); char *rolename = pstrdup(NameStr(role->rolname)); CreateRoleStmt *createRoleStmt = NULL; if (EnableCreateRolePropagation) { createRoleStmt = makeNode(CreateRoleStmt); createRoleStmt->stmt_type = ROLESTMT_ROLE; createRoleStmt->role = rolename; createRoleStmt->options = GenerateRoleOptionsList(roleTuple); } AlterRoleStmt *alterRoleStmt = NULL; if (EnableAlterRolePropagation) { alterRoleStmt = makeNode(AlterRoleStmt); alterRoleStmt->role = makeNode(RoleSpec); alterRoleStmt->role->roletype = ROLESPEC_CSTRING; alterRoleStmt->role->location = -1; alterRoleStmt->role->rolename = rolename; alterRoleStmt->action = 1; alterRoleStmt->options = GenerateRoleOptionsList(roleTuple); } ReleaseSysCache(roleTuple); List *completeRoleList = NIL; if (createRoleStmt != NULL || alterRoleStmt != NULL) { /* add a worker_create_or_alter_role command if any of them are set */ char *createOrAlterRoleQuery = CreateCreateOrAlterRoleCommand( rolename, createRoleStmt, alterRoleStmt); completeRoleList = lappend(completeRoleList, createOrAlterRoleQuery); } if (EnableAlterRoleSetPropagation) { /* append ALTER ROLE ... SET commands fot this specific user */ List *alterRoleSetCommands = GenerateAlterRoleSetCommandForRole(roleOid); completeRoleList = list_concat(completeRoleList, alterRoleSetCommands); } if (EnableCreateRolePropagation) { List *grantRoleStmts = GenerateGrantRoleStmtsOfRole(roleOid); Node *stmt = NULL; foreach_declared_ptr(stmt, grantRoleStmts) { completeRoleList = lappend(completeRoleList, DeparseTreeNode(stmt)); } /* * append SECURITY LABEL ON ROLE commands for this specific user * When we propagate user creation, we also want to make sure that we propagate * all the security labels it has been given. For this, we check pg_shseclabel * for the ROLE entry corresponding to roleOid, and generate the relevant * SecLabel stmts to be run in the new node. */ List *secLabelOnRoleStmts = GenerateSecLabelOnRoleStmts(roleOid, rolename); stmt = NULL; foreach_declared_ptr(stmt, secLabelOnRoleStmts) { completeRoleList = lappend(completeRoleList, DeparseTreeNode(stmt)); } /* * append COMMENT ON ROLE commands for this specific user * When we propagate user creation, we also want to make sure that we propagate * all the comments it has been given. For this, we check pg_shdescription * for the ROLE entry corresponding to roleOid, and generate the relevant * Comment stmts to be run in the new node. */ List *commentStmts = GetCommentPropagationCommands(AuthIdRelationId, roleOid, rolename, OBJECT_ROLE); completeRoleList = list_concat(completeRoleList, commentStmts); } return completeRoleList; } /* * GenerateAlterRoleSetCommandForRole returns the list of database wide settings for a * specifc role. If the roleid is InvalidOid it returns the commands that apply to all * users for the database or postgres wide. */ List * GenerateAlterRoleSetCommandForRole(Oid roleid) { Relation DbRoleSetting = table_open(DbRoleSettingRelationId, AccessShareLock); TupleDesc DbRoleSettingDescription = RelationGetDescr(DbRoleSetting); HeapTuple tuple = NULL; List *commands = NIL; TableScanDesc scan = table_beginscan_catalog(DbRoleSetting, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_db_role_setting roleSetting = (Form_pg_db_role_setting) GETSTRUCT(tuple); if (roleSetting->setrole != roleid) { /* not the user we are looking for */ continue; } if (OidIsValid(roleSetting->setdatabase) && roleSetting->setdatabase != MyDatabaseId) { /* setting is database specific for a different database */ continue; } List *alterRoleSetQueries = GenerateAlterRoleSetIfExistsCommandList(tuple, DbRoleSettingDescription); commands = list_concat(commands, alterRoleSetQueries); } heap_endscan(scan); table_close(DbRoleSetting, AccessShareLock); return commands; } /* * makeDefElemInt creates a DefElem with integer typed value with -1 as location. */ static DefElem * makeDefElemInt(char *name, int value) { return makeDefElem(name, (Node *) makeInteger(value), -1); } /* * makeDefElemBool creates a DefElem with boolean typed value with -1 as location. */ static DefElem * makeDefElemBool(char *name, bool value) { return makeDefElem(name, (Node *) makeBoolean(value), -1); } /* * GetDatabaseNameFromDbRoleSetting performs a lookup, and finds the database name * associated DbRoleSetting Tuple */ static char * GetDatabaseNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription) { bool isnull; Datum setdatabase = heap_getattr(tuple, Anum_pg_db_role_setting_setdatabase, DbRoleSettingDescription, &isnull); if (isnull) { return NULL; } Oid databaseId = DatumGetObjectId(setdatabase); char *databaseName = get_database_name(databaseId); return databaseName; } /* * GetRoleNameFromDbRoleSetting performs a lookup, and finds the role name * associated DbRoleSetting Tuple */ static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription) { bool isnull; Datum setrole = heap_getattr(tuple, Anum_pg_db_role_setting_setrole, DbRoleSettingDescription, &isnull); if (isnull) { return NULL; } Oid roleId = DatumGetObjectId(setrole); char *roleName = GetUserNameFromId(roleId, true); return roleName; } /* * MakeSetStatementArgs parses a configuration value and creates an List of A_Const * Nodes with appropriate types. * * The allowed A_Const types are Integer, Float, and String. */ List * MakeSetStatementArguments(char *configurationName, char *configurationValue) { List *args = NIL; char **key = &configurationName; /* Perform a lookup on GUC variables to find the config type and units. * All user-defined GUCs have string values, but we need to perform a search * on all the GUCs to understand if it is a user-defined one or not. * * Note: get_guc_variables() is intended for internal use only, but there * is no other way to determine allowed units, and value types other than * using this function */ int gucCount = 0; struct config_generic **gucVariables = get_guc_variables(&gucCount); struct config_generic **matchingConfig = (struct config_generic **) SafeBsearch((void *) &key, (void *) gucVariables, gucCount, sizeof(struct config_generic *), ConfigGenericNameCompare); /* If the config is not user-defined, lookup the variable type to contruct the arguments */ if (matchingConfig != NULL) { switch ((*matchingConfig)->vartype) { /* We use postgresql parser so that we will parse the units only if * the configuration paramater allows it. * * e.g. `SET statement_timeout = '1min'` will be parsed as 60000 since * the value is stored in units of ms internally. */ case PGC_INT: { int intValue; parse_int(configurationValue, &intValue, (*matchingConfig)->flags, NULL); Node *arg = makeIntConst(intValue, -1); args = lappend(args, arg); break; } case PGC_REAL: { Node *arg = makeFloatConst(configurationValue, -1); args = lappend(args, arg); break; } case PGC_BOOL: case PGC_STRING: case PGC_ENUM: { List *configurationList = NIL; if ((*matchingConfig)->flags & GUC_LIST_INPUT) { char *configurationValueCopy = pstrdup(configurationValue); SplitIdentifierString(configurationValueCopy, ',', &configurationList); } else { configurationList = list_make1(configurationValue); } char *configuration = NULL; foreach_declared_ptr(configuration, configurationList) { Node *arg = makeStringConst(configuration, -1); args = lappend(args, arg); } break; } default: { ereport(ERROR, (errmsg("Unrecognized run-time parameter type for %s", configurationName))); break; } } } else { Node *arg = makeStringConst(configurationValue, -1); args = lappend(args, arg); } return args; } /* * GenerateGrantRoleStmtsFromOptions gets a RoleSpec of a role that is being * created and a list of options of CreateRoleStmt to generate GrantRoleStmts * for the role's memberships. */ static List * GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options) { List *stmts = NIL; DefElem *option = NULL; foreach_declared_ptr(option, options) { if (strcmp(option->defname, "adminmembers") != 0 && strcmp(option->defname, "rolemembers") != 0 && strcmp(option->defname, "addroleto") != 0) { continue; } GrantRoleStmt *grantRoleStmt = makeNode(GrantRoleStmt); grantRoleStmt->is_grant = true; if (strcmp(option->defname, "adminmembers") == 0 || strcmp(option->defname, "rolemembers") == 0) { grantRoleStmt->granted_roles = list_make1(roleSpec); grantRoleStmt->grantee_roles = (List *) option->arg; } else { grantRoleStmt->granted_roles = (List *) option->arg; grantRoleStmt->grantee_roles = list_make1(roleSpec); } if (strcmp(option->defname, "adminmembers") == 0) { DefElem *opt = makeDefElem("admin", (Node *) makeBoolean(true), -1); grantRoleStmt->opt = list_make1(opt); } stmts = lappend(stmts, grantRoleStmt); } return stmts; } /* * GenerateGrantRoleStmtsOfRole generates the GrantRoleStmts for the memberships * of the role whose oid is roleid. */ static List * GenerateGrantRoleStmtsOfRole(Oid roleid) { Relation pgAuthMembers = table_open(AuthMemRelationId, AccessShareLock); HeapTuple tuple = NULL; List *stmts = NIL; ScanKeyData skey[1]; ScanKeyInit(&skey[0], Anum_pg_auth_members_member, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(roleid)); SysScanDesc scan = systable_beginscan(pgAuthMembers, AuthMemMemRoleIndexId, true, NULL, 1, &skey[0]); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_auth_members membership = (Form_pg_auth_members) GETSTRUCT(tuple); ObjectAddress *roleAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*roleAddress, AuthIdRelationId, membership->grantor); if (!IsAnyObjectDistributed(list_make1(roleAddress))) { /* we only need to propagate the grant if the grantor is distributed */ continue; } GrantRoleStmt *grantRoleStmt = makeNode(GrantRoleStmt); grantRoleStmt->is_grant = true; RoleSpec *grantedRole = makeNode(RoleSpec); grantedRole->roletype = ROLESPEC_CSTRING; grantedRole->location = -1; grantedRole->rolename = GetUserNameFromId(membership->roleid, true); grantRoleStmt->granted_roles = list_make1(grantedRole); RoleSpec *granteeRole = makeNode(RoleSpec); granteeRole->roletype = ROLESPEC_CSTRING; granteeRole->location = -1; granteeRole->rolename = GetUserNameFromId(membership->member, true); grantRoleStmt->grantee_roles = list_make1(granteeRole); RoleSpec *grantorRole = makeNode(RoleSpec); grantorRole->roletype = ROLESPEC_CSTRING; grantorRole->location = -1; grantorRole->rolename = GetUserNameFromId(membership->grantor, false); grantRoleStmt->grantor = grantorRole; /* inherit option is always included */ DefElem *inherit_opt; if (membership->inherit_option) { inherit_opt = makeDefElem("inherit", (Node *) makeBoolean(true), -1); } else { inherit_opt = makeDefElem("inherit", (Node *) makeBoolean(false), -1); } grantRoleStmt->opt = list_make1(inherit_opt); /* admin option is false by default, only include true case */ if (membership->admin_option) { DefElem *admin_opt = makeDefElem("admin", (Node *) makeBoolean(true), -1); grantRoleStmt->opt = lappend(grantRoleStmt->opt, admin_opt); } /* set option is true by default, only include false case */ if (!membership->set_option) { DefElem *set_opt = makeDefElem("set", (Node *) makeBoolean(false), -1); grantRoleStmt->opt = lappend(grantRoleStmt->opt, set_opt); } stmts = lappend(stmts, grantRoleStmt); } systable_endscan(scan); table_close(pgAuthMembers, AccessShareLock); return stmts; } /* * GenerateSecLabelOnRoleStmts generates the SecLabelStmts for the role * whose oid is roleid. */ static List * GenerateSecLabelOnRoleStmts(Oid roleid, char *rolename) { List *secLabelStmts = NIL; /* * Note that roles are shared database objects, therefore their * security labels are stored in pg_shseclabel instead of pg_seclabel. */ Relation pg_shseclabel = table_open(SharedSecLabelRelationId, AccessShareLock); ScanKeyData skey[1]; ScanKeyInit(&skey[0], Anum_pg_shseclabel_objoid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(roleid)); SysScanDesc scan = systable_beginscan(pg_shseclabel, SharedSecLabelObjectIndexId, true, NULL, 1, &skey[0]); HeapTuple tuple = NULL; while (HeapTupleIsValid(tuple = systable_getnext(scan))) { SecLabelStmt *secLabelStmt = makeNode(SecLabelStmt); secLabelStmt->objtype = OBJECT_ROLE; secLabelStmt->object = (Node *) makeString(pstrdup(rolename)); Datum datumArray[Natts_pg_shseclabel]; bool isNullArray[Natts_pg_shseclabel]; heap_deform_tuple(tuple, RelationGetDescr(pg_shseclabel), datumArray, isNullArray); secLabelStmt->provider = TextDatumGetCString( datumArray[Anum_pg_shseclabel_provider - 1]); secLabelStmt->label = TextDatumGetCString( datumArray[Anum_pg_shseclabel_label - 1]); secLabelStmts = lappend(secLabelStmts, secLabelStmt); } systable_endscan(scan); table_close(pg_shseclabel, AccessShareLock); return secLabelStmts; } /* * PreprocessCreateRoleStmt creates a worker_create_or_alter_role query for the * role that is being created. With that query we can create the role in the * workers or if they exist we alter them to the way they are being created * right now. */ List * PreprocessCreateRoleStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!EnableCreateRolePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); EnsureSequentialModeForRoleDDL(); LockRelationOid(DistNodeRelationId(), RowShareLock); CreateRoleStmt *createRoleStmt = castNode(CreateRoleStmt, node); AlterRoleStmt *alterRoleStmt = makeNode(AlterRoleStmt); alterRoleStmt->role = makeNode(RoleSpec); alterRoleStmt->role->roletype = ROLESPEC_CSTRING; alterRoleStmt->role->location = -1; alterRoleStmt->role->rolename = pstrdup(createRoleStmt->role); alterRoleStmt->action = 1; alterRoleStmt->options = createRoleStmt->options; List *grantRoleStmts = GenerateGrantRoleStmtsFromOptions(alterRoleStmt->role, createRoleStmt->options); char *createOrAlterRoleQuery = CreateCreateOrAlterRoleCommand(createRoleStmt->role, createRoleStmt, alterRoleStmt); List *commands = NIL; commands = lappend(commands, DISABLE_DDL_PROPAGATION); commands = lappend(commands, createOrAlterRoleQuery); /* deparse all grant statements and add them to the commands list */ Node *stmt = NULL; foreach_declared_ptr(stmt, grantRoleStmts) { commands = lappend(commands, DeparseTreeNode(stmt)); } commands = lappend(commands, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commands); } #if PG_VERSION_NUM < PG_VERSION_17 /* * makeStringConst creates a Const Node that stores a given string * * copied from backend/parser/gram.c */ static Node * makeStringConst(char *str, int location) { A_Const *n = makeNode(A_Const); n->val.sval.type = T_String; n->val.sval.sval = str; n->location = location; return (Node *) n; } #endif /* * makeIntConst creates a Const Node that stores a given integer * * copied from backend/parser/gram.c */ static Node * makeIntConst(int val, int location) { A_Const *n = makeNode(A_Const); n->val.ival.type = T_Integer; n->val.ival.ival = val; n->location = location; return (Node *) n; } /* * makeIntConst creates a Const Node that stores a given Float * * copied from backend/parser/gram.c */ static Node * makeFloatConst(char *str, int location) { A_Const *n = makeNode(A_Const); n->val.fval.type = T_Float; n->val.fval.fval = str; n->location = location; return (Node *) n; } /* * PreprocessDropRoleStmt finds the distributed role out of the ones * being dropped and unmarks them distributed and creates the drop statements * for the workers. */ List * PreprocessDropRoleStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropRoleStmt *stmt = castNode(DropRoleStmt, node); List *allDropRoles = stmt->roles; List *distributedDropRoles = FilterDistributedRoles(allDropRoles); if (list_length(distributedDropRoles) <= 0) { return NIL; } if (!EnableCreateRolePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); EnsureSequentialModeForRoleDDL(); stmt->roles = distributedDropRoles; char *sql = DeparseTreeNode((Node *) stmt); stmt->roles = allDropRoles; List *commands = list_make3(DISABLE_DDL_PROPAGATION, sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commands); } /* * UnmarkRolesDistributed unmarks the roles in the RoleSpec list distributed. */ void UnmarkRolesDistributed(List *roles) { Node *roleNode = NULL; foreach_declared_ptr(roleNode, roles) { RoleSpec *role = castNode(RoleSpec, roleNode); ObjectAddress roleAddress = { 0 }; Oid roleOid = get_rolespec_oid(role, true); if (roleOid == InvalidOid) { /* * If the role is dropped (concurrently), we might get an inactive oid for the * role. If it is invalid oid, skip. */ continue; } ObjectAddressSet(roleAddress, AuthIdRelationId, roleOid); UnmarkObjectDistributed(&roleAddress); } } /* * FilterDistributedRoles filters the list of RoleSpecs and returns the ones * that are distributed. */ List * FilterDistributedRoles(List *roles) { List *distributedRoles = NIL; Node *roleNode = NULL; foreach_declared_ptr(roleNode, roles) { RoleSpec *role = castNode(RoleSpec, roleNode); Oid roleOid = get_rolespec_oid(role, true); if (roleOid == InvalidOid) { /* * Non-existing roles are ignored silently here. Postgres will * handle to give an error or not for these roles. */ continue; } ObjectAddress *roleAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*roleAddress, AuthIdRelationId, roleOid); if (IsAnyObjectDistributed(list_make1(roleAddress))) { distributedRoles = lappend(distributedRoles, role); } } return distributedRoles; } /* * PreprocessGrantRoleStmt finds the distributed grantee roles and creates the * query to run on the workers. */ List * PreprocessGrantRoleStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!EnableCreateRolePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); GrantRoleStmt *stmt = castNode(GrantRoleStmt, node); List *allGranteeRoles = stmt->grantee_roles; RoleSpec *grantor = stmt->grantor; List *distributedGranteeRoles = FilterDistributedRoles(allGranteeRoles); if (list_length(distributedGranteeRoles) <= 0) { return NIL; } stmt->grantee_roles = distributedGranteeRoles; char *sql = DeparseTreeNode((Node *) stmt); stmt->grantee_roles = allGranteeRoles; stmt->grantor = grantor; List *commands = list_make3(DISABLE_DDL_PROPAGATION, sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commands); } /* * PostprocessGrantRoleStmt actually creates the plan we need to execute for grant * role statement. */ List * PostprocessGrantRoleStmt(Node *node, const char *queryString) { if (!EnableCreateRolePropagation || !ShouldPropagate()) { return NIL; } EnsurePropagationToCoordinator(); GrantRoleStmt *stmt = castNode(GrantRoleStmt, node); RoleSpec *role = NULL; foreach_declared_ptr(role, stmt->grantee_roles) { Oid roleOid = get_rolespec_oid(role, false); ObjectAddress *roleAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*roleAddress, AuthIdRelationId, roleOid); if (IsAnyObjectDistributed(list_make1(roleAddress))) { EnsureAllObjectDependenciesExistOnAllNodes(list_make1(roleAddress)); } } return NIL; } /* * ConfigGenericNameCompare compares two config_generic structs based on their * name fields. If the name fields contain the same strings two structs are * considered to be equal. * * copied from guc_var_compare in utils/misc/guc.c */ static int ConfigGenericNameCompare(const void *a, const void *b) { const struct config_generic *confa = *(struct config_generic *const *) a; const struct config_generic *confb = *(struct config_generic *const *) b; /* * guc_var_compare used a custom comparison function here to allow stable * ordering, but we do not need it here as we only perform a lookup, and do * not use this function to order the guc list. */ return pg_strcasecmp(confa->name, confb->name); } /* * CreateRoleStmtObjectAddress finds the ObjectAddress for the role described * by the CreateRoleStmt. If missing_ok is false this function throws an error if the * role does not exist. * * Never returns NULL, but the objid in the address could be invalid if missing_ok was set * to true. */ List * CreateRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateRoleStmt *stmt = castNode(CreateRoleStmt, node); Oid roleOid = get_role_oid(stmt->role, missing_ok); ObjectAddress *roleAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*roleAddress, AuthIdRelationId, roleOid); return list_make1(roleAddress); } /* * EnsureSequentialModeForRoleDDL makes sure that the current transaction is already in * sequential mode, or can still safely be put in sequential mode, it errors if that is * not possible. The error contains information for the user to retry the transaction with * sequential mode set from the begining. * * As roles are node scoped objects there exists only 1 instance of the role used by * potentially multiple shards. To make sure all shards in the transaction can interact * with the role the role needs to be visible on all connections used by the transaction, * meaning we can only use 1 connection per node. */ static void EnsureSequentialModeForRoleDDL(void) { if (!IsTransactionBlock()) { /* we do not need to switch to sequential mode if we are not in a transaction */ return; } if (ParallelQueryExecutedInTransaction()) { ereport(ERROR, (errmsg("cannot create or modify role because there was a " "parallel operation on a distributed table in the " "transaction"), errdetail("When creating or altering a role, Citus needs to " "perform all operations over a single connection per " "node to ensure consistency."), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail("Role is created or altered. To make sure subsequent " "commands see the role correctly we need to make sure to " "use only one connection for all future commands"))); SetLocalMultiShardModifyModeToSequential(); } /* * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on databases. */ List * PreprocessAlterRoleRenameStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagate()) { return NIL; } if (!EnableAlterRolePropagation) { return NIL; } RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ROLE); EnsurePropagationToCoordinator(); char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commands); } List * RenameRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ROLE); Oid roleOid = get_role_oid(stmt->subname, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, AuthIdRelationId, roleOid); return list_make1(address); } ================================================ FILE: src/backend/distributed/commands/schema.c ================================================ /*------------------------------------------------------------------------- * * schema.c * Commands for creating and altering schemas for distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_namespace.h" #include "nodes/parsenodes.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/relcache.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/version_compat.h" static List * GetObjectAddressBySchemaName(char *schemaName, bool missing_ok); static List * FilterDistributedSchemas(List *schemas); static bool SchemaHasDistributedTableWithFKey(char *schemaName); static bool ShouldPropagateCreateSchemaStmt(void); static List * GetGrantCommandsFromCreateSchemaStmt(Node *node); static bool CreateSchemaStmtCreatesTable(CreateSchemaStmt *stmt); /* * PostprocessCreateSchemaStmt is called during the planning phase for * CREATE SCHEMA .. */ List * PostprocessCreateSchemaStmt(Node *node, const char *queryString) { CreateSchemaStmt *createSchemaStmt = castNode(CreateSchemaStmt, node); if (!ShouldPropagateCreateSchemaStmt()) { return NIL; } EnsureCoordinator(); EnsureSequentialMode(OBJECT_SCHEMA); bool missingOk = createSchemaStmt->if_not_exists; List *schemaAdressList = CreateSchemaStmtObjectAddress(node, missingOk, true); Assert(list_length(schemaAdressList) == 1); ObjectAddress *schemaAdress = linitial(schemaAdressList); Oid schemaId = schemaAdress->objectId; if (!OidIsValid(schemaId)) { return NIL; } /* to prevent recursion with mx we disable ddl propagation */ List *commands = list_make1(DISABLE_DDL_PROPAGATION); /* deparse sql*/ const char *sql = DeparseTreeNode(node); commands = lappend(commands, (void *) sql); commands = list_concat(commands, GetGrantCommandsFromCreateSchemaStmt(node)); char *schemaName = get_namespace_name(schemaId); if (ShouldUseSchemaBasedSharding(schemaName)) { /* for now, we don't allow creating tenant tables when creating the schema itself */ if (CreateSchemaStmtCreatesTable(createSchemaStmt)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create distributed schema and table in a " "single statement"), errhint("SET citus.enable_schema_based_sharding TO off, " "or create the schema and table in separate " "commands."))); } /* * Skip if the schema is already inserted into pg_dist_schema. * This could occur when trying to create an already existing schema, * with IF NOT EXISTS clause. */ if (!IsTenantSchema(schemaId)) { /* * Register the tenant schema on the coordinator and save the command * to register it on the workers. */ int shardCount = 1; int replicationFactor = 1; Oid distributionColumnType = InvalidOid; Oid distributionColumnCollation = InvalidOid; uint32 colocationId = CreateColocationGroup( shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); InsertTenantSchemaLocally(schemaId, colocationId); commands = lappend(commands, TenantSchemaInsertCommand(schemaId, colocationId)); } } commands = lappend(commands, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessDropSchemaStmt invalidates the foreign key cache if any table created * under dropped schema involved in any foreign key relationship. */ List * PreprocessDropSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *dropStatement = castNode(DropStmt, node); Assert(dropStatement->removeType == OBJECT_SCHEMA); List *distributedSchemas = FilterDistributedSchemas(dropStatement->objects); if (list_length(distributedSchemas) < 1) { return NIL; } if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); EnsureSequentialMode(OBJECT_SCHEMA); String *schemaVal = NULL; foreach_declared_ptr(schemaVal, distributedSchemas) { if (SchemaHasDistributedTableWithFKey(strVal(schemaVal))) { MarkInvalidateForeignKeyGraph(); break; } } /* * We swap around the schema's in the statement to only contain the distributed * schemas before deparsing. We need to restore the original list as postgres * will execute on this statement locally, which requires all original schemas * from the user to be present. */ List *originalObjects = dropStatement->objects; dropStatement->objects = distributedSchemas; const char *sql = DeparseTreeNode(node); dropStatement->objects = originalObjects; /* to prevent recursion with mx we disable ddl propagation */ List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * PreprocessGrantOnSchemaStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on schemas. Only grant statements for distributed schema are propagated. */ List * PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagate()) { return NIL; } GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SCHEMA); List *distributedSchemas = FilterDistributedSchemas(stmt->objects); if (list_length(distributedSchemas) == 0) { return NIL; } EnsureCoordinator(); List *originalObjects = stmt->objects; stmt->objects = distributedSchemas; char *sql = DeparseTreeNode((Node *) stmt); stmt->objects = originalObjects; List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * CreateSchemaStmtObjectAddress returns the ObjectAddress of the schema that is * the object of the CreateSchemaStmt. Errors if missing_ok is false. */ List * CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateSchemaStmt *stmt = castNode(CreateSchemaStmt, node); StringInfoData schemaName = { 0 }; initStringInfo(&schemaName); if (stmt->schemaname == NULL) { /* * If the schema name is not provided, the schema will be created * with the name of the authorizated user. */ Assert(stmt->authrole != NULL); appendStringInfoString(&schemaName, RoleSpecString(stmt->authrole, true)); } else { appendStringInfoString(&schemaName, stmt->schemaname); } return GetObjectAddressBySchemaName(schemaName.data, missing_ok); } /* * AlterSchemaOwnerStmtObjectAddress returns the ObjectAddress of the schema that is * the object of the AlterOwnerStmt. Errors if missing_ok is false. */ List * AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_SCHEMA); return GetObjectAddressBySchemaName(strVal(stmt->object), missing_ok); } /* * AlterSchemaRenameStmtObjectAddress returns the ObjectAddress of the schema that is * the object of the RenameStmt. Errors if missing_ok is false. */ List * AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_SCHEMA); return GetObjectAddressBySchemaName(stmt->subname, missing_ok); } /* * GetObjectAddressBySchemaName returns the ObjectAddress of the schema with the * given name. Errors out if schema is not found and missing_ok is false. */ List * GetObjectAddressBySchemaName(char *schemaName, bool missing_ok) { Oid schemaOid = get_namespace_oid(schemaName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, NamespaceRelationId, schemaOid); return list_make1(address); } /* * FilterDistributedSchemas filters the schema list and returns the distributed ones * as a list */ static List * FilterDistributedSchemas(List *schemas) { List *distributedSchemas = NIL; String *schemaValue = NULL; foreach_declared_ptr(schemaValue, schemas) { const char *schemaName = strVal(schemaValue); Oid schemaOid = get_namespace_oid(schemaName, true); if (!OidIsValid(schemaOid)) { continue; } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, NamespaceRelationId, schemaOid); if (!IsAnyObjectDistributed(list_make1(address))) { continue; } distributedSchemas = lappend(distributedSchemas, schemaValue); } return distributedSchemas; } /* * SchemaHasDistributedTableWithFKey takes a schema name and scans the relations within * that schema. If any one of the relations has a foreign key relationship, it returns * true. Returns false otherwise. */ static bool SchemaHasDistributedTableWithFKey(char *schemaName) { ScanKeyData scanKey[1]; int scanKeyCount = 1; Oid scanIndexId = InvalidOid; bool useIndex = false; Oid namespaceOid = get_namespace_oid(schemaName, true); if (namespaceOid == InvalidOid) { return false; } Relation pgClass = table_open(RelationRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(namespaceOid)); SysScanDesc scanDescriptor = systable_beginscan(pgClass, scanIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_class relationForm = (Form_pg_class) GETSTRUCT(heapTuple); char *relationName = NameStr(relationForm->relname); Oid relationId = get_relname_relid(relationName, namespaceOid); /* we're not interested in non-valid, non-distributed relations */ if (relationId == InvalidOid || !IsCitusTable(relationId)) { heapTuple = systable_getnext(scanDescriptor); continue; } /* invalidate foreign key cache if the table involved in any foreign key */ if (TableReferenced(relationId) || TableReferencing(relationId)) { systable_endscan(scanDescriptor); table_close(pgClass, NoLock); return true; } heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgClass, NoLock); return false; } /* * ShouldPropagateCreateSchemaStmt gets called only for CreateSchemaStmt's. * This function wraps the ShouldPropagate function which is commonly used * for all object types; additionally it checks whether there's a multi-statement * transaction ongoing or not. For transaction blocks, we require sequential mode * with this function, for CREATE SCHEMA statements. If Citus has not already * switched to sequential mode, we don't propagate. */ static bool ShouldPropagateCreateSchemaStmt() { if (!ShouldPropagate()) { return false; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return false; } return true; } /* * GetGrantCommandsFromCreateSchemaStmt takes a CreateSchemaStmt and returns the * list of deparsed queries of the inner GRANT commands of the given statement. * Ignores commands other than GRANT statements. */ static List * GetGrantCommandsFromCreateSchemaStmt(Node *node) { List *commands = NIL; CreateSchemaStmt *stmt = castNode(CreateSchemaStmt, node); Node *element = NULL; foreach_declared_ptr(element, stmt->schemaElts) { if (!IsA(element, GrantStmt)) { continue; } GrantStmt *grantStmt = castNode(GrantStmt, element); switch (grantStmt->objtype) { /* we only propagate GRANT ON SCHEMA in community */ case OBJECT_SCHEMA: { commands = lappend(commands, DeparseGrantOnSchemaStmt(element)); break; } default: { break; } } } return commands; } /* * CreateSchemaStmtCreatesTable returns true if given CreateSchemaStmt * creates a table using "schema_element" list. */ static bool CreateSchemaStmtCreatesTable(CreateSchemaStmt *stmt) { Node *element = NULL; foreach_declared_ptr(element, stmt->schemaElts) { /* * CREATE TABLE AS and CREATE FOREIGN TABLE commands cannot be * used as schema_elements anyway, so we don't need to check them. */ if (IsA(element, CreateStmt)) { return true; } } return false; } ================================================ FILE: src/backend/distributed/commands/schema_based_sharding.c ================================================ /*------------------------------------------------------------------------- * schema_based_sharding.c * * Routines for schema-based sharding. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "catalog/catalog.h" #include "catalog/pg_namespace_d.h" #include "commands/extension.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/argutils.h" #include "distributed/backend_data.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/shard_transfer.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/worker_shard_visibility.h" /* return value of CreateCitusMoveSchemaParams() */ typedef struct { uint64 anchorShardId; uint32 sourceNodeId; char *sourceNodeName; uint32 sourceNodePort; } CitusMoveSchemaParams; static void UnregisterTenantSchemaGlobally(Oid schemaId, char *schemaName); static List * SchemaGetNonShardTableIdList(Oid schemaId); static void EnsureSchemaCanBeDistributed(Oid schemaId, List *schemaTableIdList); static void EnsureTenantSchemaNameAllowed(Oid schemaId); static void EnsureTableKindSupportedForTenantSchema(Oid relationId); static void EnsureFKeysForTenantTable(Oid relationId); static void EnsureSchemaExist(Oid schemaId); static CitusMoveSchemaParams * CreateCitusMoveSchemaParams(Oid schemaId); static uint64 TenantSchemaPickAnchorShardId(Oid schemaId); /* controlled via citus.enable_schema_based_sharding GUC */ bool EnableSchemaBasedSharding = false; const char *TenantOperationNames[TOTAL_TENANT_OPERATION] = { "undistribute_table", "alter_distributed_table", "colocate_with", "update_distributed_table_colocation", "set schema", }; PG_FUNCTION_INFO_V1(citus_internal_unregister_tenant_schema_globally); PG_FUNCTION_INFO_V1(citus_schema_distribute); PG_FUNCTION_INFO_V1(citus_schema_undistribute); PG_FUNCTION_INFO_V1(citus_schema_move); PG_FUNCTION_INFO_V1(citus_schema_move_with_nodeid); /* * ShouldUseSchemaBasedSharding returns true if schema given name should be * used as a tenant schema. */ bool ShouldUseSchemaBasedSharding(char *schemaName) { if (!EnableSchemaBasedSharding) { return false; } if (IsBinaryUpgrade) { return false; } /* * Citus utility hook skips processing CREATE SCHEMA commands while an * extension is being created. For this reason, we don't expect to get * here while an extension is being created. */ Assert(!creating_extension); /* * CREATE SCHEMA commands issued by internal backends are not meant to * create tenant schemas but to sync metadata. * * On workers, Citus utility hook skips processing CREATE SCHEMA commands * because we temporarily disable DDL propagation on workers when sending * CREATE SCHEMA commands. For this reason, right now this check is a bit * redundant but we prefer to keep it here to be on the safe side. */ if (IsCitusInternalBackend() || IsRebalancerInternalBackend()) { return false; } /* * Not do an oid comparison based on PG_PUBLIC_NAMESPACE because * we want to treat "public" schema in the same way even if it's * recreated. */ if (strcmp(schemaName, "public") == 0) { return false; } return true; } /* * ShouldCreateTenantSchemaTable returns true if we should create a tenant * schema table for given relationId. */ bool ShouldCreateTenantSchemaTable(Oid relationId) { if (IsBinaryUpgrade) { return false; } /* * CREATE TABLE commands issued by internal backends are not meant to * create tenant tables but to sync metadata. */ if (IsCitusInternalBackend() || IsRebalancerInternalBackend()) { return false; } Oid schemaId = get_rel_namespace(relationId); return IsTenantSchema(schemaId); } /* * EnsureTableKindSupportedForTenantSchema ensures that given table's kind is * supported by a tenant schema. */ static void EnsureTableKindSupportedForTenantSchema(Oid relationId) { if (IsForeignTable(relationId)) { ereport(ERROR, (errmsg("cannot create a foreign table in a distributed " "schema"))); } if (PartitionTable(relationId)) { ErrorIfIllegalPartitioningInTenantSchema(PartitionParentOid(relationId), relationId); } if (PartitionedTable(relationId)) { List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { ErrorIfIllegalPartitioningInTenantSchema(relationId, partitionRelationId); } } if (IsChildTable(relationId) || IsParentTable(relationId)) { ereport(ERROR, (errmsg("tables in a distributed schema cannot inherit or " "be inherited"))); } } /* * EnsureFKeysForTenantTable ensures that all referencing and referenced foreign * keys are allowed for given table. */ static void EnsureFKeysForTenantTable(Oid relationId) { Oid tenantSchemaId = get_rel_namespace(relationId); int fKeyReferencingFlags = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *referencingForeignKeys = GetForeignKeyOids(relationId, fKeyReferencingFlags); Oid foreignKeyId = InvalidOid; foreach_declared_oid(foreignKeyId, referencingForeignKeys) { Oid referencingTableId = GetReferencingTableId(foreignKeyId); Oid referencedTableId = GetReferencedTableId(foreignKeyId); Oid referencedTableSchemaId = get_rel_namespace(referencedTableId); /* We allow foreign keys to a table in the same schema */ if (tenantSchemaId == referencedTableSchemaId) { continue; } /* * Allow foreign keys to the other schema only if the referenced table is * a reference table. */ if (!IsCitusTable(referencedTableId) || !IsCitusTableType(referencedTableId, REFERENCE_TABLE)) { ereport(ERROR, (errmsg("foreign keys from distributed schemas can only " "point to the same distributed schema or reference " "tables in regular schemas"), errdetail("\"%s\" references \"%s\" via foreign key " "constraint \"%s\"", generate_qualified_relation_name( referencingTableId), generate_qualified_relation_name(referencedTableId), get_constraint_name(foreignKeyId)))); } } int fKeyReferencedFlags = INCLUDE_REFERENCED_CONSTRAINTS | INCLUDE_ALL_TABLE_TYPES; List *referencedForeignKeys = GetForeignKeyOids(relationId, fKeyReferencedFlags); foreach_declared_oid(foreignKeyId, referencedForeignKeys) { Oid referencingTableId = GetReferencingTableId(foreignKeyId); Oid referencedTableId = GetReferencedTableId(foreignKeyId); Oid referencingTableSchemaId = get_rel_namespace(referencingTableId); /* We allow foreign keys from a table in the same schema */ if (tenantSchemaId == referencingTableSchemaId) { continue; } /* Not allow any foreign keys from the other schema */ ereport(ERROR, (errmsg("cannot create foreign keys to tables in a distributed " "schema from another schema"), errdetail("\"%s\" references \"%s\" via foreign key " "constraint \"%s\"", generate_qualified_relation_name(referencingTableId), generate_qualified_relation_name(referencedTableId), get_constraint_name(foreignKeyId)))); } } /* * CreateTenantSchemaTable creates a tenant table with given relationId. * * This means creating a single shard distributed table without a shard * key and colocating it with the other tables in its schema. */ void CreateTenantSchemaTable(Oid relationId) { if (!IsCoordinator()) { /* * We don't support creating tenant tables from workers. We could * let ShouldCreateTenantSchemaTable() to return false to allow users * to create a local table as usual but that would be confusing because * it might sound like we allow creating tenant tables from workers. * For this reason, we prefer to throw an error instead. * * Indeed, CreateSingleShardTable() would already do so but we * prefer to throw an error with a more meaningful message, rather * than saying "operation is not allowed on this node". */ ereport(ERROR, (errmsg("cannot create tables in a distributed schema from " "a worker node"), errhint("Connect to the coordinator node and try again."))); } EnsureTableKindSupportedForTenantSchema(relationId); /* * We don't expect this to happen because ShouldCreateTenantSchemaTable() * should've already verified that; but better to check. */ Oid schemaId = get_rel_namespace(relationId); uint32 colocationId = SchemaIdGetTenantColocationId(schemaId); if (colocationId == INVALID_COLOCATION_ID) { ereport(ERROR, (errmsg("schema \"%s\" is not distributed", get_namespace_name(schemaId)))); } ColocationParam colocationParam = { .colocationParamType = COLOCATE_WITH_COLOCATION_ID, .colocationId = colocationId, }; CreateSingleShardTable(relationId, colocationParam); } /* * ErrorIfIllegalPartitioningInTenantSchema throws an error if the * partitioning relationship between the parent and the child is illegal * because they are in different schemas while one of them is a tenant table. * * This function assumes that either the parent or the child are in a tenant * schema. */ void ErrorIfIllegalPartitioningInTenantSchema(Oid parentRelationId, Oid partitionRelationId) { if (get_rel_namespace(partitionRelationId) != get_rel_namespace(parentRelationId)) { ereport(ERROR, (errmsg("partitioning within a distributed schema is not " "supported when the parent and the child " "are in different schemas"))); } } /* * CreateTenantSchemaColocationId returns new colocation id for a tenant schema. */ uint32 CreateTenantSchemaColocationId(void) { int shardCount = 1; int replicationFactor = 1; Oid distributionColumnType = InvalidOid; Oid distributionColumnCollation = InvalidOid; uint32 schemaColocationId = CreateColocationGroup( shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); return schemaColocationId; } /* * SchemaGetNonShardTableIdList returns all nonshard relation ids * inside given schema. */ static List * SchemaGetNonShardTableIdList(Oid schemaId) { List *relationIdList = NIL; /* scan all relations in pg_class and return all tables inside given schema */ Relation relationRelation = relation_open(RelationRelationId, AccessShareLock); ScanKeyData scanKey[1] = { 0 }; ScanKeyInit(&scanKey[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(schemaId)); SysScanDesc scanDescriptor = systable_beginscan(relationRelation, ClassNameNspIndexId, true, NULL, 1, scanKey); HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Form_pg_class relationForm = (Form_pg_class) GETSTRUCT(heapTuple); char *relationName = NameStr(relationForm->relname); Oid relationId = get_relname_relid(relationName, schemaId); if (!OidIsValid(relationId)) { ereport(ERROR, errmsg("table %s is dropped by a concurrent operation", relationName)); } /* skip shards */ if (RelationIsAKnownShard(relationId)) { continue; } if (RegularTable(relationId) || PartitionTable(relationId) || IsForeignTable(relationId)) { relationIdList = lappend_oid(relationIdList, relationId); } } systable_endscan(scanDescriptor); relation_close(relationRelation, AccessShareLock); return relationIdList; } /* * EnsureSchemaCanBeDistributed ensures the schema can be distributed. * Caller should take required the lock on relations and the schema. * * It checks: * - Schema name is in the allowed-list, * - Schema does not depend on an extension (created by extension), * - No extension depends on the schema (CREATE EXTENSION SCHEMA ), * - Some checks for the table for being a valid tenant table. */ static void EnsureSchemaCanBeDistributed(Oid schemaId, List *schemaTableIdList) { /* Ensure schema name is allowed */ EnsureTenantSchemaNameAllowed(schemaId); /* Any schema owned by extension is not allowed */ char *schemaName = get_namespace_name(schemaId); ObjectAddress *schemaAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*schemaAddress, NamespaceRelationId, schemaId); if (IsAnyObjectAddressOwnedByExtension(list_make1(schemaAddress), NULL)) { ereport(ERROR, (errmsg("schema %s, which is owned by an extension, cannot " "be distributed", schemaName))); } /* Extension schemas are not allowed */ ObjectAddress *extensionAddress = FirstExtensionWithSchema(schemaId); if (extensionAddress) { char *extensionName = get_extension_name(extensionAddress->objectId); ereport(ERROR, (errmsg("schema %s cannot be distributed since it is the schema " "of extension %s", schemaName, extensionName))); } Oid relationId = InvalidOid; foreach_declared_oid(relationId, schemaTableIdList) { EnsureTenantTable(relationId, "citus_schema_distribute"); } } /* * EnsureTenantTable ensures the table can be a valid tenant table. * - Current user should be the owner of table, * - Table kind is supported, * - Referencing and referenced foreign keys for the table are supported, * - Table is not owned by an extension, * - Table should be Citus local or Postgres local table. */ void EnsureTenantTable(Oid relationId, char *operationName) { /* Ensure table owner */ EnsureTableOwner(relationId); /* Check relation kind */ EnsureTableKindSupportedForTenantSchema(relationId); /* Check foreign keys */ EnsureFKeysForTenantTable(relationId); /* Check table not owned by an extension */ ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*tableAddress, RelationRelationId, relationId); if (IsAnyObjectAddressOwnedByExtension(list_make1(tableAddress), NULL)) { Oid schemaId = get_rel_namespace(relationId); char *tableName = get_namespace_name(schemaId); ereport(ERROR, (errmsg("schema cannot be distributed since it has " "table %s which is owned by an extension", tableName))); } /* Postgres local tables are allowed */ if (!IsCitusTable(relationId)) { return; } /* Only Citus local tables, amongst Citus table types, are allowed */ if (!IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("distributed schema cannot have distributed tables"), errhint("Undistribute distributed tables before " "'%s'.", operationName))); } } /* * EnsureTenantSchemaNameAllowed ensures if given schema is applicable for registering * as a tenant schema. */ static void EnsureTenantSchemaNameAllowed(Oid schemaId) { char *schemaName = get_namespace_name(schemaId); /* public schema is not allowed */ if (strcmp(schemaName, "public") == 0) { ereport(ERROR, (errmsg("public schema cannot be distributed"))); } /* information_schema schema is not allowed */ if (strcmp(schemaName, "information_schema") == 0) { ereport(ERROR, (errmsg("information_schema schema cannot be distributed"))); } /* pg_temp_xx and pg_toast_temp_xx schemas are not allowed */ if (isAnyTempNamespace(schemaId)) { ereport(ERROR, (errmsg("temporary schema cannot be distributed"))); } /* pg_catalog schema is not allowed */ if (IsCatalogNamespace(schemaId)) { ereport(ERROR, (errmsg("pg_catalog schema cannot be distributed"))); } /* pg_toast schema is not allowed */ if (IsToastNamespace(schemaId)) { ereport(ERROR, (errmsg("pg_toast schema cannot be distributed"))); } } /* * EnsureSchemaExist ensures that schema exists. Caller is responsible to take * the required lock on the schema. */ static void EnsureSchemaExist(Oid schemaId) { if (!SearchSysCacheExists1(NAMESPACEOID, ObjectIdGetDatum(schemaId))) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_SCHEMA), errmsg("schema with OID %u does not exist", schemaId))); } } /* * UnregisterTenantSchemaGlobally removes given schema from the tenant schema * metadata table, deletes the colocation group of the schema and sends the * command to do the same on the workers. */ static void UnregisterTenantSchemaGlobally(Oid schemaId, char *schemaName) { uint32 tenantSchemaColocationId = SchemaIdGetTenantColocationId(schemaId); DeleteTenantSchemaLocally(schemaId); if (EnableMetadataSync) { SendCommandToWorkersWithMetadata(TenantSchemaDeleteCommand(schemaName)); } DeleteColocationGroup(tenantSchemaColocationId); } /* * citus_internal_unregister_tenant_schema_globally, called by Citus drop hook, * unregisters the schema when a tenant schema is dropped. * * NOTE: We need to pass schema_name as an argument. We cannot use schema id * to obtain schema name since the schema would have already been dropped when this * udf is called by the drop hook. */ Datum citus_internal_unregister_tenant_schema_globally(PG_FUNCTION_ARGS) { PG_ENSURE_ARGNOTNULL(0, "schema_id"); Oid schemaId = PG_GETARG_OID(0); PG_ENSURE_ARGNOTNULL(1, "schema_name"); text *schemaName = PG_GETARG_TEXT_PP(1); char *schemaNameStr = text_to_cstring(schemaName); /* * Skip on workers because we expect this to be called from the coordinator * only via drop hook. */ if (!IsCoordinator()) { PG_RETURN_VOID(); } /* make sure that the schema is dropped already */ HeapTuple namespaceTuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(schemaId)); if (HeapTupleIsValid(namespaceTuple)) { ReleaseSysCache(namespaceTuple); ereport(ERROR, (errmsg("schema is expected to be already dropped " "because this function is only expected to " "be called from Citus drop hook"))); } UnregisterTenantSchemaGlobally(schemaId, schemaNameStr); PG_RETURN_VOID(); } /* * citus_schema_distribute gets a regular schema name, then converts it to a tenant * schema. */ Datum citus_schema_distribute(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid schemaId = PG_GETARG_OID(0); EnsureSchemaExist(schemaId); EnsureSchemaOwner(schemaId); /* Prevent concurrent table creation under the schema */ LockDatabaseObject(NamespaceRelationId, schemaId, 0, AccessExclusiveLock); /* * We should ensure the existence of the schema after taking the lock since * the schema could have been dropped before we acquired the lock. */ EnsureSchemaExist(schemaId); EnsureSchemaOwner(schemaId); /* Return if the schema is already a tenant schema */ char *schemaName = get_namespace_name(schemaId); if (IsTenantSchema(schemaId)) { ereport(NOTICE, (errmsg("schema %s is already distributed", schemaName))); PG_RETURN_VOID(); } /* Take lock on the relations and filter out partition tables */ List *tableIdListInSchema = SchemaGetNonShardTableIdList(schemaId); List *tableIdListToConvert = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, tableIdListInSchema) { /* prevent concurrent drop of the relation */ LockRelationOid(relationId, AccessShareLock); EnsureRelationExists(relationId); /* * Skip partitions as they would be distributed by the parent table. * * We should filter out partitions here before distributing the schema. * Otherwise, converted partitioned table would change oid of partitions and its * partition tables would fail with oid not exist. */ if (PartitionTable(relationId)) { continue; } tableIdListToConvert = lappend_oid(tableIdListToConvert, relationId); } /* Makes sure the schema can be distributed. */ EnsureSchemaCanBeDistributed(schemaId, tableIdListInSchema); ereport(NOTICE, (errmsg("distributing the schema %s", schemaName))); /* Create colocation id and then single shard tables with the colocation id */ uint32 colocationId = CreateTenantSchemaColocationId(); ColocationParam colocationParam = { .colocationParamType = COLOCATE_WITH_COLOCATION_ID, .colocationId = colocationId, }; /* * Collect foreign keys for recreation and then drop fkeys and create single shard * tables. */ List *originalForeignKeyRecreationCommands = NIL; foreach_declared_oid(relationId, tableIdListToConvert) { List *fkeyCommandsForRelation = GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES); originalForeignKeyRecreationCommands = list_concat( originalForeignKeyRecreationCommands, fkeyCommandsForRelation); DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES); CreateSingleShardTable(relationId, colocationParam); } /* We can skip foreign key validations as we are sure about them at start */ bool skip_validation = true; ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands, skip_validation); /* Register the schema locally and sync it to workers */ InsertTenantSchemaLocally(schemaId, colocationId); char *registerSchemaCommand = TenantSchemaInsertCommand(schemaId, colocationId); if (EnableMetadataSync) { SendCommandToWorkersWithMetadata(registerSchemaCommand); } PG_RETURN_VOID(); } /* * citus_schema_undistribute gets a tenant schema name, then converts it to a regular * schema by undistributing all tables under it. */ Datum citus_schema_undistribute(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid schemaId = PG_GETARG_OID(0); EnsureSchemaExist(schemaId); EnsureSchemaOwner(schemaId); /* Prevent concurrent table creation under the schema */ LockDatabaseObject(NamespaceRelationId, schemaId, 0, AccessExclusiveLock); /* * We should ensure the existence of the schema after taking the lock since * the schema could have been dropped before we acquired the lock. */ EnsureSchemaExist(schemaId); EnsureSchemaOwner(schemaId); /* The schema should be a tenant schema */ char *schemaName = get_namespace_name(schemaId); if (!IsTenantSchema(schemaId)) { ereport(ERROR, (errmsg("schema %s is not distributed", schemaName))); } ereport(NOTICE, (errmsg("undistributing schema %s", schemaName))); /* Take lock on the relations and filter out partition tables */ List *tableIdListInSchema = SchemaGetNonShardTableIdList(schemaId); List *tableIdListToConvert = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, tableIdListInSchema) { /* prevent concurrent drop of the relation */ LockRelationOid(relationId, AccessShareLock); EnsureRelationExists(relationId); /* * Skip partitions as they would be undistributed by the parent table. * * We should filter out partitions here before undistributing the schema. * Otherwise, converted partitioned table would change oid of partitions and its * partition tables would fail with oid not exist. */ if (PartitionTable(relationId)) { continue; } tableIdListToConvert = lappend_oid(tableIdListToConvert, relationId); /* Only single shard tables are expected during the undistribution of the schema */ Assert(IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)); } /* * First, we need to delete schema metadata and sync it to workers. Otherwise, * we would get error from `ErrorIfTenantTable` while undistributing the tables. */ UnregisterTenantSchemaGlobally(schemaId, schemaName); UndistributeTables(tableIdListToConvert); PG_RETURN_VOID(); } /* * citus_schema_move moves the shards that belong to given distributed tenant * schema from one node to the other node by using citus_move_shard_placement(). */ Datum citus_schema_move(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid schemaId = PG_GETARG_OID(0); CitusMoveSchemaParams *params = CreateCitusMoveSchemaParams(schemaId); DirectFunctionCall6(citus_move_shard_placement, UInt64GetDatum(params->anchorShardId), CStringGetTextDatum(params->sourceNodeName), UInt32GetDatum(params->sourceNodePort), PG_GETARG_DATUM(1), PG_GETARG_DATUM(2), PG_GETARG_DATUM(3)); PG_RETURN_VOID(); } /* * citus_schema_move_with_nodeid does the same as citus_schema_move(), but * accepts node id as parameter instead of hostname and port, hence uses * citus_move_shard_placement_with_nodeid(). */ Datum citus_schema_move_with_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid schemaId = PG_GETARG_OID(0); CitusMoveSchemaParams *params = CreateCitusMoveSchemaParams(schemaId); DirectFunctionCall4(citus_move_shard_placement_with_nodeid, UInt64GetDatum(params->anchorShardId), UInt32GetDatum(params->sourceNodeId), PG_GETARG_DATUM(1), PG_GETARG_DATUM(2)); PG_RETURN_VOID(); } /* * CreateCitusMoveSchemaParams is a helper function for * citus_schema_move() and citus_schema_move_with_nodeid() * that validates input schema and returns the parameters to be used in underlying * shard transfer functions. */ static CitusMoveSchemaParams * CreateCitusMoveSchemaParams(Oid schemaId) { EnsureSchemaExist(schemaId); EnsureSchemaOwner(schemaId); if (!IsTenantSchema(schemaId)) { ereport(ERROR, (errmsg("schema %s is not a distributed schema", get_namespace_name(schemaId)))); } uint64 anchorShardId = TenantSchemaPickAnchorShardId(schemaId); if (anchorShardId == INVALID_SHARD_ID) { ereport(ERROR, (errmsg("cannot move distributed schema %s because it is empty", get_namespace_name(schemaId)))); } uint32 colocationId = SchemaIdGetTenantColocationId(schemaId); uint32 sourceNodeId = SingleShardTableColocationNodeId(colocationId); bool missingOk = false; WorkerNode *sourceNode = FindNodeWithNodeId(sourceNodeId, missingOk); CitusMoveSchemaParams *params = palloc0(sizeof(CitusMoveSchemaParams)); params->anchorShardId = anchorShardId; params->sourceNodeId = sourceNodeId; params->sourceNodeName = sourceNode->workerName; params->sourceNodePort = sourceNode->workerPort; return params; } /* * TenantSchemaPickAnchorShardId returns the id of one of the shards * created in given tenant schema. * * Returns INVALID_SHARD_ID if the schema was initially empty or if it's not * a tenant schema. * * Throws an error if all the tables in the schema are concurrently dropped. */ static uint64 TenantSchemaPickAnchorShardId(Oid schemaId) { uint32 colocationId = SchemaIdGetTenantColocationId(schemaId); List *tablesInSchema = ColocationGroupTableList(colocationId, 0); if (list_length(tablesInSchema) == 0) { return INVALID_SHARD_ID; } Oid relationId = InvalidOid; foreach_declared_oid(relationId, tablesInSchema) { /* * Make sure the relation isn't dropped for the remainder of * the transaction. */ LockRelationOid(relationId, AccessShareLock); /* * The relation might have been dropped just before we locked it. * Let's look it up. */ Relation relation = RelationIdGetRelation(relationId); if (RelationIsValid(relation)) { /* relation still exists, we can use it */ RelationClose(relation); return GetFirstShardId(relationId); } } ereport(ERROR, (errmsg("tables in schema %s are concurrently dropped", get_namespace_name(schemaId)))); } /* * ErrorIfTenantTable errors out with the given operation name, * if the given relation is a tenant table. */ void ErrorIfTenantTable(Oid relationId, const char *operationName) { if (IsTenantSchema(get_rel_namespace(relationId))) { ereport(ERROR, (errmsg("%s is not allowed for %s because it belongs to " "a distributed schema", generate_qualified_relation_name(relationId), operationName))); } } ================================================ FILE: src/backend/distributed/commands/seclabel.c ================================================ /*------------------------------------------------------------------------- * * seclabel.c * * This file contains the logic of SECURITY LABEL statement propagation. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" /* * PostprocessRoleSecLabelStmt prepares the commands that need to be run on all workers to assign * security labels on distributed roles. It also ensures that all object dependencies exist on all * nodes for the role in the SecLabelStmt. */ List * PostprocessRoleSecLabelStmt(Node *node, const char *queryString) { if (!EnableAlterRolePropagation || !ShouldPropagate()) { return NIL; } SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); List *objectAddresses = GetObjectAddressListFromParseTree(node, false, true); if (!IsAnyObjectDistributed(objectAddresses)) { return NIL; } EnsurePropagationToCoordinator(); EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); const char *secLabelCommands = DeparseTreeNode((Node *) secLabelStmt); List *commandList = list_make3(DISABLE_DDL_PROPAGATION, (void *) secLabelCommands, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(REMOTE_NODES, commandList); } /* * PostprocessTableOrColumnSecLabelStmt prepares the commands that need to be run on all * workers to assign security labels on distributed tables or the columns of a distributed * table. It also ensures that all object dependencies exist on all nodes for the table in * the SecLabelStmt. */ List * PostprocessTableOrColumnSecLabelStmt(Node *node, const char *queryString) { if (!EnableAlterRolePropagation || !ShouldPropagate()) { return NIL; } SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); List *objectAddresses = GetObjectAddressListFromParseTree(node, false, true); if (!IsAnyParentObjectDistributed(objectAddresses)) { return NIL; } EnsurePropagationToCoordinator(); EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); const char *secLabelCommands = DeparseTreeNode((Node *) secLabelStmt); List *commandList = list_make3(DISABLE_DDL_PROPAGATION, (void *) secLabelCommands, ENABLE_DDL_PROPAGATION); List *DDLJobs = NodeDDLTaskList(REMOTE_NODES, commandList); ListCell *lc = NULL; /* * The label is for a table or a column, so we need to set the targetObjectAddress * of the DDLJob to the relationId of the table. This is needed to ensure that * the search path is correctly set for the remote security label command; it * needs to be able to resolve the table that the label is being defined on. */ Assert(list_length(objectAddresses) == 1); ObjectAddress *target = linitial(objectAddresses); Oid relationId = target->objectId; Assert(relationId != InvalidOid); foreach(lc, DDLJobs) { DDLJob *ddlJob = (DDLJob *) lfirst(lc); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); } return DDLJobs; } /* * PostprocessAnySecLabelStmt is used for any other object types * that are not supported by Citus. It issues a notice to the client * if appropriate. Is effectively a nop. */ List * PostprocessAnySecLabelStmt(Node *node, const char *queryString) { /* * If we are not in the coordinator, we don't want to interrupt the security * label command with notices, the user expects that from the worker node * the command will not be propagated */ if (EnableUnsupportedFeatureMessages && IsCoordinator()) { ereport(NOTICE, (errmsg("not propagating SECURITY LABEL commands whose " "object type is not role or table or column"), errhint("Connect to worker nodes directly to manually " "run the same SECURITY LABEL command."))); } return NIL; } /* * SecLabelStmtObjectAddress returns the object address of the object on * which this statement operates (secLabelStmt->object). Note that it has no limitation * on the object type being OBJECT_ROLE. This is intentionally implemented like this * since it is fairly simple to implement and we might extend SECURITY LABEL propagation * in the future to include more object types. */ List * SecLabelStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); Relation rel = NULL; ObjectAddress address = get_object_address(secLabelStmt->objtype, secLabelStmt->object, &rel, AccessShareLock, missing_ok); if (rel != NULL) { relation_close(rel, AccessShareLock); } ObjectAddress *addressPtr = palloc0(sizeof(ObjectAddress)); *addressPtr = address; return list_make1(addressPtr); } /* * citus_test_object_relabel is a dummy function for check_object_relabel_type hook. * It is meant to be used in tests combined with citus_test_register_label_provider */ void citus_test_object_relabel(const ObjectAddress *object, const char *seclabel) { if (seclabel == NULL || strcmp(seclabel, "citus_unclassified") == 0 || strcmp(seclabel, "citus_classified") == 0 || strcmp(seclabel, "citus '!unclassified") == 0) { return; } ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("'%s' is not a valid security label for Citus tests.", seclabel))); } ================================================ FILE: src/backend/distributed/commands/sequence.c ================================================ /*------------------------------------------------------------------------- * * sequence.c * This file contains implementation of CREATE and ALTER SEQUENCE * statement functions to run in a distributed setting * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/pg_attrdef.h" #include "commands/defrem.h" #include "commands/extension.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "rewrite/rewriteHandler.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/commands.h" #include "distributed/commands/sequence.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/worker_create_or_replace.h" /* Local functions forward declarations for helper functions */ static bool OptionsSpecifyOwnedBy(List *optionList, Oid *ownedByTableId); static Oid SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char depType); static List * FilterDistributedSequences(GrantStmt *stmt); /* * ErrorIfUnsupportedSeqStmt errors out if the provided create sequence * statement specifies a distributed table in its OWNED BY clause. */ void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt) { Oid ownedByTableId = InvalidOid; /* create is easy: just prohibit any distributed OWNED BY */ if (OptionsSpecifyOwnedBy(createSeqStmt->options, &ownedByTableId)) { if (IsCitusTable(ownedByTableId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create sequences that specify a distributed " "table in their OWNED BY option"), errhint("Use a sequence in a distributed table by specifying " "a serial column type before creating any shards."))); } } } /* * ErrorIfDistributedAlterSeqOwnedBy errors out if the provided alter sequence * statement attempts to change the owned by property of a distributed sequence * or attempt to change a local sequence to be owned by a distributed table. */ void ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt) { Oid sequenceId = RangeVarGetRelid(alterSeqStmt->sequence, AccessShareLock, alterSeqStmt->missing_ok); Oid ownedByTableId = InvalidOid; Oid newOwnedByTableId = InvalidOid; int32 ownedByColumnId = 0; bool hasDistributedOwner = false; /* alter statement referenced nonexistent sequence; return */ if (sequenceId == InvalidOid) { return; } bool sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_AUTO, &ownedByTableId, &ownedByColumnId); if (!sequenceOwned) { sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_INTERNAL, &ownedByTableId, &ownedByColumnId); } /* see whether the sequence is already owned by a distributed table */ if (sequenceOwned) { hasDistributedOwner = IsCitusTable(ownedByTableId); } if (OptionsSpecifyOwnedBy(alterSeqStmt->options, &newOwnedByTableId)) { /* if a distributed sequence tries to change owner, error */ if (hasDistributedOwner && ownedByTableId != newOwnedByTableId) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter OWNED BY option of a sequence " "already owned by a distributed table"))); } } } /* * OptionsSpecifyOwnedBy processes the options list of either a CREATE or ALTER * SEQUENCE command, extracting the first OWNED BY option it encounters. The * identifier for the specified table is placed in the Oid out parameter before * returning true. Returns false if no such option is found. Still returns true * for OWNED BY NONE, but leaves the out paramter set to InvalidOid. */ static bool OptionsSpecifyOwnedBy(List *optionList, Oid *ownedByTableId) { DefElem *defElem = NULL; foreach_declared_ptr(defElem, optionList) { if (strcmp(defElem->defname, "owned_by") == 0) { List *ownedByNames = defGetQualifiedName(defElem); int nameCount = list_length(ownedByNames); /* if only one name is present, this is OWNED BY NONE */ if (nameCount == 1) { *ownedByTableId = InvalidOid; return true; } else { /* * Otherwise, we have a list of schema, table, column, which we * need to truncate to simply the schema and table to determine * the relevant relation identifier. */ List *relNameList = list_truncate(list_copy(ownedByNames), nameCount - 1); RangeVar *rangeVar = makeRangeVarFromNameList(relNameList); bool failOK = true; *ownedByTableId = RangeVarGetRelid(rangeVar, NoLock, failOK); return true; } } } return false; } /* * ExtractDefaultColumnsAndOwnedSequences finds each column of relation with * relationId that has a DEFAULT expression and each sequence owned by such * columns (if any). Then, appends the column name and id of the owned sequence * -that the column defaults- to the lists passed as NIL initially. */ void ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList, List **ownedSequenceIdList) { Assert(*columnNameList == NIL && *ownedSequenceIdList == NIL); Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); if (IsDroppedOrGenerated(attributeForm)) { /* skip dropped columns and columns with GENERATED AS ALWAYS expressions */ continue; } char *columnName = NameStr(attributeForm->attname); List *columnOwnedSequences = getOwnedSequences_internal(relationId, attributeIndex + 1, DEPENDENCY_AUTO); if (attributeForm->atthasdef && list_length(columnOwnedSequences) == 0) { /* * Even if there are no owned sequences, the code path still * expects the columnName to be filled such that it can DROP * DEFAULT for the existing nextval('seq') columns. */ *ownedSequenceIdList = lappend_oid(*ownedSequenceIdList, InvalidOid); *columnNameList = lappend(*columnNameList, columnName); continue; } Oid ownedSequenceId = InvalidOid; foreach_declared_oid(ownedSequenceId, columnOwnedSequences) { /* * A column might have multiple sequences one via OWNED BY one another * via bigserial/default nextval. */ *ownedSequenceIdList = lappend_oid(*ownedSequenceIdList, ownedSequenceId); *columnNameList = lappend(*columnNameList, columnName); } } relation_close(relation, NoLock); } /* * ColumnDefaultsToNextVal returns true if the column with attrNumber * has a default expression that contains nextval(). */ bool ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber) { Assert(AttributeNumberIsValid(attrNumber)); Relation relation = RelationIdGetRelation(relationId); Node *defExpr = build_column_default(relation, attrNumber); RelationClose(relation); if (defExpr == NULL) { /* column doesn't have a DEFAULT expression */ return false; } return contain_nextval_expression_walker(defExpr, NULL); } /* * PreprocessDropSequenceStmt gets called during the planning phase of a DROP SEQUENCE statement * and returns a list of DDLJob's that will drop any distributed sequences from the * workers. * * The DropStmt could have multiple objects to drop, the list of objects will be filtered * to only keep the distributed sequences for deletion on the workers. Non-distributed * sequences will still be dropped locally but not on the workers. */ List * PreprocessDropSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *stmt = castNode(DropStmt, node); List *distributedSequencesList = NIL; List *distributedSequenceAddresses = NIL; Assert(stmt->removeType == OBJECT_SEQUENCE); if (creating_extension) { /* * extensions should be created separately on the workers, sequences cascading * from an extension should therefore not be propagated here. */ return NIL; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return NIL; } /* * Our statements need to be fully qualified so we can drop them from the right schema * on the workers */ QualifyTreeNode((Node *) stmt); /* * iterate over all sequences to be dropped and filter to keep only distributed * sequences. */ List *deletingSequencesList = stmt->objects; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, deletingSequencesList) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); if (!IsAnyObjectDistributed(list_make1(sequenceAddress))) { continue; } /* collect information for all distributed sequences */ distributedSequenceAddresses = lappend(distributedSequenceAddresses, sequenceAddress); distributedSequencesList = lappend(distributedSequencesList, objectNameList); } if (list_length(distributedSequencesList) <= 0) { /* no distributed functions to drop */ return NIL; } /* * managing sequences can only be done on the coordinator if ddl propagation is on. when * it is off we will never get here. MX workers don't have a notion of distributed * sequences, so we block the call. */ EnsureCoordinator(); /* remove the entries for the distributed objects on dropping */ ObjectAddress *address = NULL; foreach_declared_ptr(address, distributedSequenceAddresses) { UnmarkObjectDistributed(address); } /* * Swap the list of objects before deparsing and restore the old list after. This * ensures we only have distributed sequences in the deparsed drop statement. */ DropStmt *stmtCopy = copyObject(stmt); stmtCopy->objects = distributedSequencesList; const char *dropStmtSql = DeparseTreeNode((Node *) stmtCopy); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) dropStmtSql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * SequenceDropStmtObjectAddress returns list of object addresses in the drop sequence * statement. */ List * SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess) { DropStmt *dropSeqStmt = castNode(DropStmt, stmt); List *objectAddresses = NIL; List *droppingSequencesList = dropSeqStmt->objects; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, droppingSequencesList) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); Oid seqOid = RangeVarGetRelid(seq, AccessShareLock, missing_ok); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, SequenceRelationId, seqOid); objectAddresses = lappend(objectAddresses, objectAddress); } return objectAddresses; } /* * PreprocessRenameSequenceStmt is called when the user is renaming a sequence. The invocation * happens before the statement is applied locally. * * As the sequence already exists we have access to the ObjectAddress, this is used to * check if it is distributed. If so the rename is executed on all the workers to keep the * types in sync across the cluster. */ List * PreprocessRenameSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_SEQUENCE); List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, stmt->missing_ok, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * RenameSequenceStmtObjectAddress returns the ObjectAddress of the sequence that is the * subject of the RenameStmt. */ List * RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_SEQUENCE); RangeVar *sequence = stmt->relation; Oid seqOid = RangeVarGetRelid(sequence, NoLock, missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); return list_make1(sequenceAddress); } /* * PreprocessAlterSequenceStmt gets called during the planning phase of an ALTER SEQUENCE statement * of one of the following forms: * ALTER SEQUENCE [ IF EXISTS ] name * [ AS data_type ] * [ INCREMENT [ BY ] increment ] * [ MINVALUE minvalue | NO MINVALUE ] [ MAXVALUE maxvalue | NO MAXVALUE ] * [ START [ WITH ] start ] * [ RESTART [ [ WITH ] restart ] ] * [ CACHE cache ] [ [ NO ] CYCLE ] * [ OWNED BY { table_name.column_name | NONE } ] * * For distributed sequences, this operation will not be allowed for now. * The reason is that we change sequence parameters when distributing it, so we don't want to * touch those parameters for now. */ List * PreprocessAlterSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterSeqStmt *stmt = castNode(AlterSeqStmt, node); List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, stmt->missing_ok, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); /* error out if the sequence is distributed */ if (IsAnyObjectDistributed(addresses) || SequenceUsedInDistributedTable(address, DEPENDENCY_INTERNAL)) { ereport(ERROR, (errmsg("Altering a distributed sequence " "is currently not supported."))); } /* * error out if the sequence is used in a distributed table * and this is an ALTER SEQUENCE .. AS .. statement */ Oid citusTableId = SequenceUsedInDistributedTable(address, DEPENDENCY_AUTO); if (citusTableId != InvalidOid) { List *options = stmt->options; DefElem *defel = NULL; foreach_declared_ptr(defel, options) { if (strcmp(defel->defname, "as") == 0) { if (IsCitusTableType(citusTableId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg( "Altering a sequence used in a local table that" " is added to metadata is currently not supported."))); } ereport(ERROR, (errmsg( "Altering a sequence used in a distributed" " table is currently not supported."))); } } } return NIL; } /* * SequenceUsedInDistributedTable returns true if the argument sequence * is used as the default value of a column in a distributed table. * Returns false otherwise * See DependencyType for the possible values of depType. * We use DEPENDENCY_INTERNAL for sequences created by identity column. * DEPENDENCY_AUTO for regular sequences. */ static Oid SequenceUsedInDistributedTable(const ObjectAddress *sequenceAddress, char depType) { Oid relationId; List *relations = GetDependentRelationsWithSequence(sequenceAddress->objectId, depType); foreach_declared_oid(relationId, relations) { if (IsCitusTable(relationId)) { return relationId; } } return InvalidOid; } /* * AlterSequenceStmtObjectAddress returns the ObjectAddress of the sequence that is the * subject of the AlterSeqStmt. */ List * AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterSeqStmt *stmt = castNode(AlterSeqStmt, node); RangeVar *sequence = stmt->sequence; Oid seqOid = RangeVarGetRelid(sequence, NoLock, stmt->missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); return list_make1(sequenceAddress); } /* * PreprocessAlterSequenceSchemaStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers. */ List * PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_SEQUENCE); List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, stmt->missing_ok, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * AlterSequenceSchemaStmtObjectAddress returns the ObjectAddress of the sequence that is * the subject of the AlterObjectSchemaStmt. */ List * AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_SEQUENCE); RangeVar *sequence = stmt->relation; Oid seqOid = RangeVarGetRelid(sequence, NoLock, true); if (seqOid == InvalidOid) { /* * couldn't find the sequence, might have already been moved to the new schema, we * construct a new sequence name that uses the new schema to search in. */ const char *newSchemaName = stmt->newschema; Oid newSchemaOid = get_namespace_oid(newSchemaName, true); seqOid = get_relname_relid(sequence->relname, newSchemaOid); if (!missing_ok && seqOid == InvalidOid) { /* * if the sequence is still invalid we couldn't find the sequence, error with the same * message postgres would error with if missing_ok is false (not ok to miss) */ const char *quotedSequenceName = quote_qualified_identifier(sequence->schemaname, sequence->relname); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", quotedSequenceName))); } } ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); return list_make1(sequenceAddress); } /* * PostprocessAlterSequenceSchemaStmt is executed after the change has been applied locally, * we can now use the new dependencies of the sequence to ensure all its dependencies * exist on the workers before we apply the commands remotely. */ List * PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_SEQUENCE); List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, stmt->missing_ok, true); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); if (!ShouldPropagateAnyObject(addresses)) { return NIL; } /* dependencies have changed (schema) let's ensure they exist */ EnsureAllObjectDependenciesExistOnAllNodes(addresses); return NIL; } /* * PreprocessAlterSequenceOwnerStmt is called for change of ownership of sequences before the * ownership is changed on the local instance. * * If the sequence for which the owner is changed is distributed we execute the change on * all the workers to keep the type in sync across the cluster. */ List * PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); List *sequenceAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(sequenceAddresses) == 1); if (!ShouldPropagateAnyObject(sequenceAddresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * AlterSequenceOwnerStmtObjectAddress returns the ObjectAddress of the sequence that is the * subject of the AlterOwnerStmt. */ List * AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *sequence = stmt->relation; Oid seqOid = RangeVarGetRelid(sequence, NoLock, missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); return list_make1(sequenceAddress); } /* * PostprocessAlterSequenceOwnerStmt is executed after the change has been applied locally, * we can now use the new dependencies of the sequence to ensure all its dependencies * exist on the workers before we apply the commands remotely. */ List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); List *sequenceAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, true); /* the code-path only supports a single object */ Assert(list_length(sequenceAddresses) == 1); if (!ShouldPropagateAnyObject(sequenceAddresses)) { return NIL; } /* dependencies have changed (owner) let's ensure they exist */ EnsureAllObjectDependenciesExistOnAllNodes(sequenceAddresses); return NIL; } /* * PreprocessAlterSequencePersistenceStmt is called for change of persistence * of sequences before the persistence is changed on the local instance. * * If the sequence for which the persistence is changed is distributed, we execute * the change on all the workers to keep the type in sync across the cluster. */ List * PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); List *sequenceAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(sequenceAddresses) == 1); if (!ShouldPropagateAnyObject(sequenceAddresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * AlterSequencePersistenceStmtObjectAddress returns the ObjectAddress of the * sequence that is the subject of the AlterPersistenceStmt. */ List * AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *sequence = stmt->relation; Oid seqOid = RangeVarGetRelid(sequence, NoLock, missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, seqOid); return list_make1(sequenceAddress); } /* * PreprocessSequenceAlterTableStmt is called for change of persistence or owner * of sequences before the persistence/owner is changed on the local instance. * * Altering persistence or owner are the only ALTER commands of a sequence * that may pass through an AlterTableStmt as well */ List * PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); ListCell *cmdCell = NULL; foreach(cmdCell, stmt->cmds) { AlterTableCmd *cmd = castNode(AlterTableCmd, lfirst(cmdCell)); switch (cmd->subtype) { case AT_ChangeOwner: { return PreprocessAlterSequenceOwnerStmt(node, queryString, processUtilityContext); } case AT_SetLogged: { return PreprocessAlterSequencePersistenceStmt(node, queryString, processUtilityContext); } case AT_SetUnLogged: { return PreprocessAlterSequencePersistenceStmt(node, queryString, processUtilityContext); } default: { /* normally we shouldn't ever reach this */ ereport(ERROR, (errmsg("unsupported subtype for alter sequence command"), errdetail("sub command type: %d", cmd->subtype))); } } } return NIL; } /* * PreprocessGrantOnSequenceStmt is executed before the statement is applied to the local * postgres instance. * * In this stage we can prepare the commands that need to be run on all workers to grant * on distributed sequences. */ List * PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); if (creating_extension) { /* * extensions should be created separately on the workers, sequences cascading * from an extension should therefore not be propagated here. */ return NIL; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return NIL; } List *distributedSequences = FilterDistributedSequences(stmt); if (list_length(distributedSequences) == 0) { return NIL; } EnsureCoordinator(); GrantStmt *stmtCopy = copyObject(stmt); stmtCopy->objects = distributedSequences; /* * if the original command was targeting schemas, we have expanded to the distributed * sequences in these schemas through FilterDistributedSequences. */ stmtCopy->targtype = ACL_TARGET_OBJECT; QualifyTreeNode((Node *) stmtCopy); char *sql = DeparseTreeNode((Node *) stmtCopy); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_METADATA_NODES, commands); } /* * PostprocessGrantOnSequenceStmt makes sure dependencies of each * distributed sequence in the statement exist on all nodes */ List * PostprocessGrantOnSequenceStmt(Node *node, const char *queryString) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); List *distributedSequences = FilterDistributedSequences(stmt); if (list_length(distributedSequences) == 0) { return NIL; } EnsureCoordinator(); RangeVar *sequence = NULL; foreach_declared_ptr(sequence, distributedSequences) { ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); Oid sequenceOid = RangeVarGetRelid(sequence, NoLock, false); ObjectAddressSet(*sequenceAddress, RelationRelationId, sequenceOid); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(sequenceAddress)); } return NIL; } /* * GenerateBackupNameForSequenceCollision generates a new sequence name for an existing * sequence. The name is generated in such a way that the new name doesn't overlap with * an existing relation by adding a suffix with incrementing number after the new name. */ char * GenerateBackupNameForSequenceCollision(const ObjectAddress *address) { char *newName = palloc0(NAMEDATALEN); char suffix[NAMEDATALEN] = { 0 }; int count = 0; char *namespaceName = get_namespace_name(get_rel_namespace(address->objectId)); Oid schemaId = get_namespace_oid(namespaceName, false); char *baseName = get_rel_name(address->objectId); int baseLength = strlen(baseName); while (true) { int suffixLength = SafeSnprintf(suffix, NAMEDATALEN - 1, "(citus_backup_%d)", count); /* trim the base name at the end to leave space for the suffix and trailing \0 */ baseLength = Min(baseLength, NAMEDATALEN - suffixLength - 1); /* clear newName before copying the potentially trimmed baseName and suffix */ memset(newName, 0, NAMEDATALEN); strncpy_s(newName, NAMEDATALEN, baseName, baseLength); strncpy_s(newName + baseLength, NAMEDATALEN - baseLength, suffix, suffixLength); Oid newRelationId = get_relname_relid(newName, schemaId); if (newRelationId == InvalidOid) { return newName; } count++; } } /* * FilterDistributedSequences determines and returns a list of distributed sequences * RangeVar-s from given grant statement. * - If the stmt's targtype is ACL_TARGET_OBJECT, i.e. of the form GRANT ON SEQUENCE ... * it returns the distributed sequences in the list of sequences in the statement * - If targtype is ACL_TARGET_ALL_IN_SCHEMA, i.e. GRANT ON ALL SEQUENCES IN SCHEMA ... * it expands the ALL IN SCHEMA to the actual sequences, and returns the distributed * sequences from those. */ static List * FilterDistributedSequences(GrantStmt *stmt) { bool grantOnSequenceCommand = (stmt->targtype == ACL_TARGET_OBJECT && stmt->objtype == OBJECT_SEQUENCE); bool grantOnAllSequencesInSchemaCommand = (stmt->targtype == ACL_TARGET_ALL_IN_SCHEMA && stmt->objtype == OBJECT_SEQUENCE); /* we are only interested in sequence level grants */ if (!grantOnSequenceCommand && !grantOnAllSequencesInSchemaCommand) { return NIL; } List *grantSequenceList = NIL; if (grantOnAllSequencesInSchemaCommand) { /* iterate over all namespace names provided to get their oid's */ List *namespaceOidList = NIL; String *namespaceValue = NULL; foreach_declared_ptr(namespaceValue, stmt->objects) { char *nspname = strVal(namespaceValue); bool missing_ok = false; Oid namespaceOid = get_namespace_oid(nspname, missing_ok); namespaceOidList = list_append_unique_oid(namespaceOidList, namespaceOid); } /* * iterate over all distributed sequences to filter the ones * that belong to one of the namespaces from above */ List *distributedSequenceList = DistributedSequenceList(); ObjectAddress *sequenceAddress = NULL; foreach_declared_ptr(sequenceAddress, distributedSequenceList) { Oid namespaceOid = get_rel_namespace(sequenceAddress->objectId); /* * if this distributed sequence's schema is one of the schemas * specified in the GRANT .. ALL SEQUENCES IN SCHEMA .. * add it to the list */ if (list_member_oid(namespaceOidList, namespaceOid)) { RangeVar *distributedSequence = makeRangeVar( get_namespace_name(namespaceOid), get_rel_name(sequenceAddress->objectId), -1); grantSequenceList = lappend(grantSequenceList, distributedSequence); } } } else { bool missing_ok = false; RangeVar *sequenceRangeVar = NULL; foreach_declared_ptr(sequenceRangeVar, stmt->objects) { Oid sequenceOid = RangeVarGetRelid(sequenceRangeVar, NoLock, missing_ok); ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, sequenceOid); /* * if this sequence from GRANT .. ON SEQUENCE .. is a distributed * sequence, add it to the list */ if (IsAnyObjectDistributed(list_make1(sequenceAddress))) { grantSequenceList = lappend(grantSequenceList, sequenceRangeVar); } } } return grantSequenceList; } /* * RenameExistingSequenceWithDifferentTypeIfExists renames the sequence's type if * that sequence exists and the desired sequence type is different than its type. */ void RenameExistingSequenceWithDifferentTypeIfExists(RangeVar *sequence, Oid desiredSeqTypeId) { Oid sequenceOid; RangeVarGetAndCheckCreationNamespace(sequence, NoLock, &sequenceOid); if (OidIsValid(sequenceOid)) { Form_pg_sequence pgSequenceForm = pg_get_sequencedef(sequenceOid); if (pgSequenceForm->seqtypid != desiredSeqTypeId) { ObjectAddress sequenceAddress = { 0 }; ObjectAddressSet(sequenceAddress, RelationRelationId, sequenceOid); char *newName = GenerateBackupNameForCollision(&sequenceAddress); RenameStmt *renameStmt = CreateRenameStatement(&sequenceAddress, newName); const char *sqlRenameStmt = DeparseTreeNode((Node *) renameStmt); ProcessUtilityParseTree((Node *) renameStmt, sqlRenameStmt, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); CommandCounterIncrement(); } } } ================================================ FILE: src/backend/distributed/commands/serialize_distributed_ddls.c ================================================ /*------------------------------------------------------------------------- * * serialize_distributed_ddls.c * * This file contains functions for serializing distributed DDLs. * * If you're adding support for serializing a new DDL, you should * extend the following functions to support the new object class: * AcquireCitusAdvisoryObjectClassLockGetOid() * AcquireCitusAdvisoryObjectClassLockCheckPrivileges() * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/dependency.h" #include "catalog/pg_database_d.h" #include "commands/dbcommands.h" #include "storage/lock.h" #include "utils/builtins.h" #include "pg_version_compat.h" #include "distributed/adaptive_executor.h" #include "distributed/argutils.h" #include "distributed/commands/serialize_distributed_ddls.h" #include "distributed/deparse_shard_query.h" #include "distributed/resource_lock.h" PG_FUNCTION_INFO_V1(citus_internal_acquire_citus_advisory_object_class_lock); static void SerializeDistributedDDLsOnObjectClassInternal(ObjectClass objectClass, char *qualifiedObjectName); static char * AcquireCitusAdvisoryObjectClassLockCommand(ObjectClass objectClass, char *qualifiedObjectName); static void AcquireCitusAdvisoryObjectClassLock(ObjectClass objectClass, char *qualifiedObjectName); static Oid AcquireCitusAdvisoryObjectClassLockGetOid(ObjectClass objectClass, char *qualifiedObjectName); static void AcquireCitusAdvisoryObjectClassLockCheckPrivileges(ObjectClass objectClass, Oid oid); /* * citus_internal_acquire_citus_advisory_object_class_lock is an internal UDF * to call AcquireCitusAdvisoryObjectClassLock(). */ Datum citus_internal_acquire_citus_advisory_object_class_lock(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "object_class"); ObjectClass objectClass = PG_GETARG_INT32(0); char *qualifiedObjectName = PG_ARGISNULL(1) ? NULL : PG_GETARG_CSTRING(1); AcquireCitusAdvisoryObjectClassLock(objectClass, qualifiedObjectName); PG_RETURN_VOID(); } /* * SerializeDistributedDDLsOnObjectClass is a wrapper around * SerializeDistributedDDLsOnObjectClassInternal to acquire the lock on given * object class itself, see the comment in header file for more details about * the difference between this function and * SerializeDistributedDDLsOnObjectClassObject(). */ void SerializeDistributedDDLsOnObjectClass(ObjectClass objectClass) { SerializeDistributedDDLsOnObjectClassInternal(objectClass, NULL); } /* * SerializeDistributedDDLsOnObjectClassObject is a wrapper around * SerializeDistributedDDLsOnObjectClassInternal to acquire the lock on given * object that belongs to given object class, see the comment in header file * for more details about the difference between this function and * SerializeDistributedDDLsOnObjectClass(). */ void SerializeDistributedDDLsOnObjectClassObject(ObjectClass objectClass, char *qualifiedObjectName) { if (qualifiedObjectName == NULL) { elog(ERROR, "qualified object name cannot be NULL"); } SerializeDistributedDDLsOnObjectClassInternal(objectClass, qualifiedObjectName); } /* * SerializeDistributedDDLsOnObjectClassInternal serializes distributed DDLs * that target given object class by acquiring a Citus specific advisory lock * on the first primary worker node if there are any workers in the cluster. * * The lock is acquired via a coordinated transaction. For this reason, * it automatically gets released when (maybe implicit) transaction on * current server commits or rolls back. * * If qualifiedObjectName is provided to be non-null, then the oid of the * object is first resolved on the first primary worker node and then the * lock is acquired on that oid. If qualifiedObjectName is null, then the * lock is acquired on the object class itself. * * Note that those two lock types don't conflict with each other and are * acquired for different purposes. The lock on the object class * (qualifiedObjectName = NULL) is used to serialize DDLs that target the * object class itself, e.g., when creating a new object of that class, and * the latter is used to serialize DDLs that target a specific object of * that class, e.g., when altering an object. * * In some cases, we may want to acquire both locks at the same time. For * example, when renaming a database, we want to acquire both lock types * because while the object class lock is used to ensure that another session * doesn't create a new database with the same name, the object lock is used * to ensure that another session doesn't alter the same database. */ static void SerializeDistributedDDLsOnObjectClassInternal(ObjectClass objectClass, char *qualifiedObjectName) { WorkerNode *firstWorkerNode = GetFirstPrimaryWorkerNode(); if (firstWorkerNode == NULL) { /* * If there are no worker nodes in the cluster, then we don't need * to acquire the lock at all; and we cannot indeed. */ return; } /* * Indeed we would already ensure permission checks in remote node * --via AcquireCitusAdvisoryObjectClassLock()-- but we first do so on * the local node to avoid from reporting confusing error messages. */ Oid oid = AcquireCitusAdvisoryObjectClassLockGetOid(objectClass, qualifiedObjectName); AcquireCitusAdvisoryObjectClassLockCheckPrivileges(objectClass, oid); Task *task = CitusMakeNode(Task); task->taskType = DDL_TASK; char *command = AcquireCitusAdvisoryObjectClassLockCommand(objectClass, qualifiedObjectName); SetTaskQueryString(task, command); ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(targetPlacement, firstWorkerNode); task->taskPlacementList = list_make1(targetPlacement); /* need to be in a transaction to acquire a lock that's bound to transactions */ UseCoordinatedTransaction(); bool localExecutionSupported = true; ExecuteUtilityTaskList(list_make1(task), localExecutionSupported); } /* * AcquireCitusAdvisoryObjectClassLockCommand returns a command to call * citus_internal.acquire_citus_advisory_object_class_lock(). */ static char * AcquireCitusAdvisoryObjectClassLockCommand(ObjectClass objectClass, char *qualifiedObjectName) { /* safe to cast to int as it's an enum */ int objectClassInt = (int) objectClass; char *quotedObjectName = !qualifiedObjectName ? "NULL" : quote_literal_cstr(qualifiedObjectName); StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.acquire_citus_advisory_object_class_lock(%d, %s)", objectClassInt, quotedObjectName); return command->data; } /* * AcquireCitusAdvisoryObjectClassLock acquires a Citus specific advisory * ExclusiveLock based on given object class. */ static void AcquireCitusAdvisoryObjectClassLock(ObjectClass objectClass, char *qualifiedObjectName) { Oid oid = AcquireCitusAdvisoryObjectClassLockGetOid(objectClass, qualifiedObjectName); AcquireCitusAdvisoryObjectClassLockCheckPrivileges(objectClass, oid); LOCKTAG locktag; SET_LOCKTAG_GLOBAL_DDL_SERIALIZATION(locktag, objectClass, oid); LOCKMODE lockmode = ExclusiveLock; bool sessionLock = false; bool dontWait = false; LockAcquire(&locktag, lockmode, sessionLock, dontWait); } /* * AcquireCitusAdvisoryObjectClassLockGetOid returns the oid of given object * that belongs to given object class. If qualifiedObjectName is NULL, then * it returns InvalidOid. */ static Oid AcquireCitusAdvisoryObjectClassLockGetOid(ObjectClass objectClass, char *qualifiedObjectName) { if (qualifiedObjectName == NULL) { return InvalidOid; } bool missingOk = false; switch (objectClass) { case OCLASS_DATABASE: { return get_database_oid(qualifiedObjectName, missingOk); } default: { elog(ERROR, "unsupported object class: %d", objectClass); } } } /* * AcquireCitusAdvisoryObjectClassLockCheckPrivileges is used to perform privilege checks * before acquiring the Citus specific advisory lock on given object class and oid. */ static void AcquireCitusAdvisoryObjectClassLockCheckPrivileges(ObjectClass objectClass, Oid oid) { switch (objectClass) { case OCLASS_DATABASE: { if (OidIsValid(oid) && !object_ownercheck(DatabaseRelationId, oid, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, get_database_name(oid)); } else if (!OidIsValid(oid) && !have_createdb_privilege()) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied to create / rename database"))); } break; } default: { elog(ERROR, "unsupported object class: %d", objectClass); } } } ================================================ FILE: src/backend/distributed/commands/statistics.c ================================================ /*------------------------------------------------------------------------- * * statistics.c * Commands for STATISTICS statements. * * We currently support replicating statistics definitions on the * coordinator in all the worker nodes in the form of * * CREATE STATISTICS ... queries. * * We also support dropping statistics from all the worker nodes in form of * * DROP STATISTICS ... queries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_namespace.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_type.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/namespace_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/resource_lock.h" #include "distributed/worker_transaction.h" #define DEFAULT_STATISTICS_TARGET -1 #define ALTER_INDEX_COLUMN_SET_STATS_COMMAND \ "ALTER INDEX %s ALTER COLUMN %d SET STATISTICS %d" static char * GenerateAlterIndexColumnSetStatsCommand(char *indexNameWithSchema, int16 attnum, int32 attstattarget); static Oid GetRelIdByStatsOid(Oid statsOid); static char * CreateAlterCommandIfOwnerNotDefault(Oid statsOid); static char * CreateAlterCommandIfTargetNotDefault(Oid statsOid); /* * PreprocessCreateStatisticsStmt is called during the planning phase for * CREATE STATISTICS. */ List * PreprocessCreateStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { CreateStatsStmt *stmt = castNode(CreateStatsStmt, node); Node *relationNode = (Node *) linitial(stmt->relations); if (!IsA(relationNode, RangeVar)) { return NIL; } RangeVar *relation = (RangeVar *) relationNode; Oid relationId = RangeVarGetRelid(relation, ShareUpdateExclusiveLock, false); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); if (!(stmt->defnames)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create statistics without a name on a " "Citus table"), errhint("Consider specifying a name for the statistics"))); } QualifyTreeNode((Node *) stmt); Oid statsOid = get_statistics_object_oid(stmt->defnames, true); if (statsOid != InvalidOid) { /* if stats object already exists, don't create DDLJobs */ return NIL; } char *ddlCommand = DeparseTreeNode((Node *) stmt); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * PostprocessCreateStatisticsStmt is called after a CREATE STATISTICS command has * been executed by standard process utility. */ List * PostprocessCreateStatisticsStmt(Node *node, const char *queryString) { CreateStatsStmt *stmt = castNode(CreateStatsStmt, node); Assert(stmt->type == T_CreateStatsStmt); RangeVar *relation = (RangeVar *) linitial(stmt->relations); Oid relationId = RangeVarGetRelid(relation, ShareUpdateExclusiveLock, false); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } bool missingOk = false; List *objectAddresses = GetObjectAddressListFromParseTree((Node *) stmt, missingOk, true); /* the code-path only supports a single object */ Assert(list_length(objectAddresses) == 1); EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); return NIL; } /* * CreateStatisticsStmtObjectAddress finds the ObjectAddress for the statistics that * is created by given CreateStatsStmt. If missingOk is false and if statistics * does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { CreateStatsStmt *stmt = castNode(CreateStatsStmt, node); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); Oid statsOid = get_statistics_object_oid(stmt->defnames, missingOk); ObjectAddressSet(*address, StatisticExtRelationId, statsOid); return list_make1(address); } /* * PreprocessDropStatisticsStmt is called during the planning phase for * DROP STATISTICS. */ List * PreprocessDropStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *dropStatisticsStmt = castNode(DropStmt, node); Assert(dropStatisticsStmt->removeType == OBJECT_STATISTIC_EXT); if (!ShouldPropagate()) { return NIL; } QualifyTreeNode((Node *) dropStatisticsStmt); List *ddlJobs = NIL; List *processedStatsOids = NIL; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { Oid statsOid = get_statistics_object_oid(objectNameList, dropStatisticsStmt->missing_ok); if (list_member_oid(processedStatsOids, statsOid)) { /* skip duplicate entries in DROP STATISTICS */ continue; } processedStatsOids = lappend_oid(processedStatsOids, statsOid); Oid relationId = GetRelIdByStatsOid(statsOid); if (!OidIsValid(relationId) || !IsCitusTable(relationId)) { continue; } char *ddlCommand = DeparseDropStatisticsStmt(objectNameList, dropStatisticsStmt->missing_ok); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); ddlJobs = lappend(ddlJobs, ddlJob); } return ddlJobs; } /* * DropStatisticsObjectAddress returns list of object addresses in the drop statistics * statement. */ List * DropStatisticsObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropStmt *dropStatisticsStmt = castNode(DropStmt, node); Assert(dropStatisticsStmt->removeType == OBJECT_STATISTIC_EXT); List *objectAddresses = NIL; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { Oid statsOid = get_statistics_object_oid(objectNameList, dropStatisticsStmt->missing_ok); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, StatisticExtRelationId, statsOid); objectAddresses = lappend(objectAddresses, objectAddress); } return objectAddresses; } /* * PreprocessAlterStatisticsRenameStmt is called during the planning phase for * ALTER STATISTICS RENAME. */ List * PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { RenameStmt *renameStmt = castNode(RenameStmt, node); Assert(renameStmt->renameType == OBJECT_STATISTIC_EXT); Oid statsOid = get_statistics_object_oid((List *) renameStmt->object, false); Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) renameStmt); char *ddlCommand = DeparseTreeNode((Node *) renameStmt); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * PreprocessAlterStatisticsSchemaStmt is called during the planning phase for * ALTER STATISTICS SET SCHEMA. */ List * PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); Oid statsOid = get_statistics_object_oid((List *) stmt->object, false); Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); char *ddlCommand = DeparseTreeNode((Node *) stmt); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * PostprocessAlterStatisticsSchemaStmt is called after a ALTER STATISTICS SCHEMA * command has been executed by standard process utility. */ List * PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); String *statName = llast((List *) stmt->object); Oid statsOid = get_statistics_object_oid(list_make2(makeString(stmt->newschema), statName), false); Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } bool missingOk = false; List *objectAddresses = GetObjectAddressListFromParseTree((Node *) stmt, missingOk, true); /* the code-path only supports a single object */ Assert(list_length(objectAddresses) == 1); EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); return NIL; } /* * AlterStatisticsSchemaStmtObjectAddress finds the ObjectAddress for the statistics * that is altered by given AlterObjectSchemaStmt. If missingOk is false and if * the statistics does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); Oid statsOid = InvalidOid; List *statName = (List *) stmt->object; if (isPostprocess) { /* * we should search the object in the new schema because the method is * called during postprocess, standard_utility should have already moved * the stat into new schema. */ List *newStatName = list_make2(makeString(stmt->newschema), llast(statName)); statsOid = get_statistics_object_oid(newStatName, missingOk); } else { statsOid = get_statistics_object_oid(statName, missingOk); } ObjectAddressSet(*address, StatisticExtRelationId, statsOid); return list_make1(address); } /* * PreprocessAlterStatisticsStmt is called during the planning phase for * ALTER STATISTICS .. SET STATISTICS. */ List * PreprocessAlterStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterStatsStmt *stmt = castNode(AlterStatsStmt, node); Oid statsOid = get_statistics_object_oid(stmt->defnames, stmt->missing_ok); if (!OidIsValid(statsOid)) { /* * If statsOid is invalid, here we can assume that the query includes * IF EXISTS clause, since get_statistics_object_oid would error out otherwise. * So here we can safely return NIL here without checking stmt->missing_ok. */ return NIL; } Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); char *ddlCommand = DeparseTreeNode((Node *) stmt); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * PreprocessAlterStatisticsOwnerStmt is called during the planning phase for * ALTER STATISTICS .. OWNER TO. */ List * PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); Oid statsOid = get_statistics_object_oid((List *) stmt->object, false); Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); char *ddlCommand = DeparseTreeNode((Node *) stmt); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; ddlJob->taskList = DDLTaskList(relationId, ddlCommand); List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * PostprocessAlterStatisticsOwnerStmt is invoked after the owner has been changed locally. * Since changing the owner could result in new dependencies being found for this object * we re-ensure all the dependencies for the statistics do exist. * * This is solely to propagate the new owner (and all its dependencies) if it was not * already distributed in the cluster. */ List * PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); Oid statsOid = get_statistics_object_oid((List *) stmt->object, false); Oid relationId = GetRelIdByStatsOid(statsOid); if (!IsCitusTable(relationId) || !ShouldPropagate()) { return NIL; } ObjectAddress *statisticsAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*statisticsAddress, StatisticExtRelationId, statsOid); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(statisticsAddress)); return NIL; } /* * GetExplicitStatisticsCommandList returns the list of DDL commands to create * or alter statistics that are explicitly created for the table with relationId. * This function gets called when distributing the table with relationId. */ List * GetExplicitStatisticsCommandList(Oid relationId) { List *explicitStatisticsCommandList = NIL; Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *statisticsIdList = RelationGetStatExtList(relation); RelationClose(relation); /* generate fully-qualified names */ int saveNestLevel = PushEmptySearchPath(); Oid statisticsId = InvalidOid; foreach_declared_oid(statisticsId, statisticsIdList) { /* we need create commands for already created stats before distribution */ Datum commandText = DirectFunctionCall1(pg_get_statisticsobjdef, ObjectIdGetDatum(statisticsId)); /* * pg_get_statisticsobjdef doesn't throw an error if there is no such * statistics object, be on the safe side. */ if (DatumGetPointer(commandText) == NULL) { ereport(ERROR, (errmsg("statistics with oid %u does not exist", statisticsId))); } char *createStatisticsCommand = TextDatumGetCString(commandText); explicitStatisticsCommandList = lappend(explicitStatisticsCommandList, makeTableDDLCommandString(createStatisticsCommand)); /* we need to alter stats' target if it's getting distributed after creation */ char *alterStatisticsTargetCommand = CreateAlterCommandIfTargetNotDefault(statisticsId); if (alterStatisticsTargetCommand != NULL) { explicitStatisticsCommandList = lappend(explicitStatisticsCommandList, makeTableDDLCommandString(alterStatisticsTargetCommand)); } /* we need to alter stats' owner if it's getting distributed after creation */ char *alterStatisticsOwnerCommand = CreateAlterCommandIfOwnerNotDefault(statisticsId); if (alterStatisticsOwnerCommand != NULL) { explicitStatisticsCommandList = lappend(explicitStatisticsCommandList, makeTableDDLCommandString(alterStatisticsOwnerCommand)); } } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return explicitStatisticsCommandList; } /* * GetExplicitStatisticsSchemaIdList returns the list of schema ids of statistics' * which are created on relation with given relation id. */ List * GetExplicitStatisticsSchemaIdList(Oid relationId) { List *schemaIdList = NIL; Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", relationId))); } List *statsIdList = RelationGetStatExtList(relation); RelationClose(relation); Oid statsId = InvalidOid; foreach_declared_oid(statsId, statsIdList) { HeapTuple heapTuple = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsId)); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("cache lookup failed for statistics " "object with oid %u", statsId))); } FormData_pg_statistic_ext *statisticsForm = (FormData_pg_statistic_ext *) GETSTRUCT(heapTuple); Oid schemaId = statisticsForm->stxnamespace; if (!list_member_oid(schemaIdList, schemaId)) { schemaIdList = lappend_oid(schemaIdList, schemaId); } ReleaseSysCache(heapTuple); } return schemaIdList; } /* * GetAlterIndexStatisticsCommands returns the list of ALTER INDEX .. ALTER COLUMN .. * SET STATISTICS commands, based on non default targets of the index with given id. * Note that this function only looks for expression indexes, since this command is * valid for only expression indexes. */ List * GetAlterIndexStatisticsCommands(Oid indexOid) { List *alterIndexStatisticsCommandList = NIL; int16 exprCount = 1; while (true) { HeapTuple attTuple = SearchSysCacheAttNum(indexOid, exprCount); if (!HeapTupleIsValid(attTuple)) { break; } Form_pg_attribute targetAttr = (Form_pg_attribute) GETSTRUCT(attTuple); int32 targetAttstattarget = getAttstattarget_compat(attTuple); if (targetAttstattarget != DEFAULT_STATISTICS_TARGET) { char *indexNameWithSchema = generate_qualified_relation_name(indexOid); char *command = GenerateAlterIndexColumnSetStatsCommand(indexNameWithSchema, targetAttr->attnum, targetAttstattarget); alterIndexStatisticsCommandList = lappend(alterIndexStatisticsCommandList, makeTableDDLCommandString(command)); } ReleaseSysCache(attTuple); exprCount++; } return alterIndexStatisticsCommandList; } /* * GenerateAlterIndexColumnSetStatsCommand returns a string in form of 'ALTER INDEX .. * ALTER COLUMN .. SET STATISTICS ..' which will be used to create a DDL command to * send to workers. */ static char * GenerateAlterIndexColumnSetStatsCommand(char *indexNameWithSchema, int16 attnum, int32 attstattarget) { StringInfoData command; initStringInfo(&command); appendStringInfo(&command, ALTER_INDEX_COLUMN_SET_STATS_COMMAND, indexNameWithSchema, attnum, attstattarget); return command.data; } /* * GetRelIdByStatsOid returns the relation id for given statistics oid. * If statistics doesn't exist, returns InvalidOid. */ static Oid GetRelIdByStatsOid(Oid statsOid) { HeapTuple tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(tup)) { return InvalidOid; } Form_pg_statistic_ext statisticsForm = (Form_pg_statistic_ext) GETSTRUCT(tup); ReleaseSysCache(tup); return statisticsForm->stxrelid; } /* * CreateAlterCommandIfOwnerNotDefault returns an ALTER STATISTICS .. OWNER TO * command if the stats object with given id has an owner different than the default one. * Returns NULL otherwise. */ static char * CreateAlterCommandIfOwnerNotDefault(Oid statsOid) { HeapTuple tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(tup)) { ereport(WARNING, (errmsg("No stats object found with id: %u", statsOid))); return NULL; } Form_pg_statistic_ext statisticsForm = (Form_pg_statistic_ext) GETSTRUCT(tup); ReleaseSysCache(tup); if (statisticsForm->stxowner == GetUserId()) { return NULL; } char *schemaName = get_namespace_name(statisticsForm->stxnamespace); char *statName = NameStr(statisticsForm->stxname); char *ownerName = GetUserNameFromId(statisticsForm->stxowner, false); StringInfoData str; initStringInfo(&str); appendStringInfo(&str, "ALTER STATISTICS %s OWNER TO %s", NameListToQuotedString(list_make2(makeString(schemaName), makeString(statName))), quote_identifier(ownerName)); return str.data; } /* * CreateAlterCommandIfTargetNotDefault returns an ALTER STATISTICS .. SET STATISTICS * command if the stats object with given id has a target different than the default one. * Returns NULL otherwise. */ static char * CreateAlterCommandIfTargetNotDefault(Oid statsOid) { HeapTuple tup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(tup)) { ereport(WARNING, (errmsg("No stats object found with id: %u", statsOid))); return NULL; } Form_pg_statistic_ext statisticsForm = (Form_pg_statistic_ext) GETSTRUCT(tup); int16 currentStxstattarget = getStxstattarget_compat(tup); ReleaseSysCache(tup); if (currentStxstattarget == -1) { return NULL; } AlterStatsStmt *alterStatsStmt = makeNode(AlterStatsStmt); char *schemaName = get_namespace_name(statisticsForm->stxnamespace); char *statName = NameStr(statisticsForm->stxname); alterStatsStmt->stxstattarget = getAlterStatsStxstattarget_compat( currentStxstattarget); alterStatsStmt->defnames = list_make2(makeString(schemaName), makeString(statName)); return DeparseAlterStatisticsStmt((Node *) alterStatsStmt); } ================================================ FILE: src/backend/distributed/commands/subscription.c ================================================ /*------------------------------------------------------------------------- * * subscription.c * Commands for creating subscriptions * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "libpq-fe.h" #include "safe_lib.h" #include "commands/defrem.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/connection_management.h" #include "distributed/version_compat.h" static char * GenerateConninfoWithAuth(char *conninfo); /* * ProcessCreateSubscriptionStmt looks for a special citus_use_authinfo option. * If it is set to true, then we'll expand the node's authinfo into the create * statement (see GenerateConninfoWithAuth). */ Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt) { ListCell *currCell = NULL; bool useAuthinfo = false; foreach(currCell, createSubStmt->options) { DefElem *defElem = (DefElem *) lfirst(currCell); if (strcmp(defElem->defname, "citus_use_authinfo") == 0) { useAuthinfo = defGetBoolean(defElem); createSubStmt->options = list_delete_cell(createSubStmt->options, currCell); break; } } if (useAuthinfo) { createSubStmt->conninfo = GenerateConninfoWithAuth(createSubStmt->conninfo); } return (Node *) createSubStmt; } /* * GenerateConninfoWithAuth extracts the host and port from the provided libpq * conninfo string, using them to find an appropriate authinfo for the target * host. If such an authinfo is found, it is added to the (repalloc'd) string, * which is then returned. */ static char * GenerateConninfoWithAuth(char *conninfo) { StringInfo connInfoWithAuth = makeStringInfo(); char *host = NULL, *user = NULL; int32 port = -1; PQconninfoOption *option = NULL, *optionArray = NULL; optionArray = PQconninfoParse(conninfo, NULL); if (optionArray == NULL) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("not a valid libpq connection info string: %s", conninfo))); } for (option = optionArray; option->keyword != NULL; option++) { if (option->val == NULL || option->val[0] == '\0') { continue; } if (strcmp(option->keyword, "host") == 0) { host = option->val; } else if (strcmp(option->keyword, "port") == 0) { port = pg_strtoint32(option->val); } else if (strcmp(option->keyword, "user") == 0) { user = option->val; } } /* * In case of repetition of parameters in connection strings, last value * wins. So first add the provided connection string, then global * connection parameters, then node specific ones. * * Note that currently lists of parameters in pg_dist_authnode and * citus.node_conninfo do not overlap. * * The only overlapping parameter between these three lists is * connect_timeout, which is assigned in conninfo (generated * by CreateShardMoveSubscription) and is also allowed in * citus.node_conninfo. Prioritizing the value in citus.node_conninfo * over conninfo gives user the power to control this value. */ appendStringInfo(connInfoWithAuth, "%s %s", conninfo, NodeConninfo); if (host != NULL && port > 0 && user != NULL) { char *nodeAuthInfo = GetAuthinfo(host, port, user); appendStringInfo(connInfoWithAuth, " %s", nodeAuthInfo); } PQconninfoFree(optionArray); return connInfoWithAuth->data; } ================================================ FILE: src/backend/distributed/commands/table.c ================================================ /*------------------------------------------------------------------------- * * table.c * Commands for creating and altering distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/index.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_type.h" #include "commands/tablecmds.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "parser/parse_expr.h" #include "parser/parse_type.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/distribution_column.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/resource_lock.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" /* controlled via GUC, should be accessed via GetEnableLocalReferenceForeignKeys() */ bool EnableLocalReferenceForeignKeys = true; /* * GUC that controls whether to allow unique/exclude constraints without * distribution column. */ bool AllowUnsafeConstraints = false; /* Local functions forward declarations for unsupported command checks */ static void PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement); static void PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const char *queryString); static void PreprocessAttachPartitionToCitusTable(Oid parentRelationId, Oid partitionRelationId); static void PreprocessAttachCitusPartitionToCitusTable(Oid parentCitusRelationId, Oid partitionRelationId); static void DistributePartitionUsingParent(Oid parentRelationId, Oid partitionRelationId); static void ErrorIfMultiLevelPartitioning(Oid parentRelationId, Oid partitionRelationId); static void ErrorIfAttachCitusTableToPgLocalTable(Oid parentRelationId, Oid partitionRelationId); static bool DeparserSupportsAlterTableAddColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *addColumnSubCommand); static bool ATDefinesFKeyBetweenPostgresAndCitusLocalOrRef(AlterTableStmt * alterTableStatement); static bool ShouldMarkConnectedRelationsNotAutoConverted(Oid leftRelationId, Oid rightRelationId); static bool RelationIdListContainsCitusTableType(List *relationIdList, CitusTableType citusTableType); static bool RelationIdListContainsPostgresTable(List *relationIdList); static void ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt * alterTableStatement); static bool RangeVarListHasLocalRelationConvertedByUser(List *relationRangeVarList, AlterTableStmt * alterTableStatement); static int CompareRangeVarsByOid(const void *leftElement, const void *rightElement); static List * GetAlterTableAddFKeyRightRelationIdList(AlterTableStmt * alterTableStatement); static List * GetAlterTableAddFKeyRightRelationRangeVarList(AlterTableStmt * alterTableStatement); static List * GetAlterTableAddFKeyConstraintList(AlterTableStmt *alterTableStatement); static List * GetAlterTableCommandFKeyConstraintList(AlterTableCmd *command); static List * GetRangeVarListFromFKeyConstraintList(List *fKeyConstraintList); static List * GetRelationIdListFromRangeVarList(List *rangeVarList, LOCKMODE lockmode, bool missingOk); static bool AlterTableCommandTypeIsTrigger(AlterTableType alterTableType); static bool AlterTableDropsForeignKey(AlterTableStmt *alterTableStatement); static void ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement); static bool AlterInvolvesPartitionColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *command); static bool AlterColumnInvolvesIdentityColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *command); static void ErrorIfUnsupportedAlterAddConstraintStmt(AlterTableStmt *alterTableStatement); static List * CreateRightShardListForInterShardDDLTask(Oid rightRelationId, Oid leftRelationId, List *leftShardList); static void SetInterShardDDLTaskPlacementList(Task *task, ShardInterval *leftShardInterval, ShardInterval *rightShardInterval); static void SetInterShardDDLTaskRelationShardList(Task *task, ShardInterval *leftShardInterval, ShardInterval *rightShardInterval); static Oid get_attrdef_oid(Oid relationId, AttrNumber attnum); static char * GetAddColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, TypeName *typeName, bool ifNotExists); static void ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid relationId); /* * We need to run some of the commands sequentially if there is a foreign constraint * from/to reference table. */ static bool SetupExecutionModeForAlterTable(Oid relationId, AlterTableCmd *command); /* * PreprocessDropTableStmt processes DROP TABLE commands for partitioned tables. * If we are trying to DROP partitioned tables, we first need to go to MX nodes * and DETACH partitions from their parents. Otherwise, we process DROP command * multiple times in MX workers. For shards, we send DROP commands with IF EXISTS * parameter which solves problem of processing same command multiple times. * However, for distributed table itself, we directly remove related table from * Postgres catalogs via performDeletion function, thus we need to be cautious * about not processing same DROP command twice. */ List * PreprocessDropTableStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *dropTableStatement = castNode(DropStmt, node); Assert(dropTableStatement->removeType == OBJECT_TABLE); List *tableNameList = NULL; foreach_declared_ptr(tableNameList, dropTableStatement->objects) { RangeVar *tableRangeVar = makeRangeVarFromNameList(tableNameList); bool missingOK = true; Oid relationId = RangeVarGetRelid(tableRangeVar, AccessShareLock, missingOK); ErrorIfIllegallyChangingKnownShard(relationId); /* we're not interested in non-valid, non-distributed relations */ if (relationId == InvalidOid || !IsCitusTable(relationId)) { continue; } /* * While changing the tables that are part of a colocation group we need to * prevent concurrent mutations to the placements of the shard groups. */ CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (cacheEntry->colocationId != INVALID_COLOCATION_ID) { LockColocationId(cacheEntry->colocationId, ShareLock); } /* invalidate foreign key cache if the table involved in any foreign key */ if ((TableReferenced(relationId) || TableReferencing(relationId))) { MarkInvalidateForeignKeyGraph(); } /* we're only interested in partitioned and mx tables */ if (!ShouldSyncTableMetadata(relationId) || !PartitionedTable(relationId)) { continue; } EnsureCoordinator(); List *partitionList = PartitionList(relationId); if (list_length(partitionList) == 0) { continue; } SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { char *detachPartitionCommand = GenerateDetachPartitionCommand(partitionRelationId); SendCommandToWorkersWithMetadata(detachPartitionCommand); } SendCommandToWorkersWithMetadata(ENABLE_DDL_PROPAGATION); } return NIL; } /* * PostprocessCreateTableStmt takes CreateStmt object as a parameter and * processes foreign keys on relation via PostprocessCreateTableStmtForeignKeys * function. * * This function also processes CREATE TABLE ... PARTITION OF statements via * PostprocessCreateTableStmtPartitionOf function. * * Also CREATE TABLE ... INHERITS ... commands are filtered here. If the inherited * table is a distributed table, this function errors out, as we currently don't * support local tables inheriting a distributed table. */ void PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString) { PostprocessCreateTableStmtForeignKeys(createStatement); bool missingOk = false; Oid relationId = RangeVarGetRelid(createStatement->relation, NoLock, missingOk); Oid schemaId = get_rel_namespace(relationId); if (createStatement->ofTypename && IsTenantSchema(schemaId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create tables in a distributed schema using " "CREATE TABLE OF syntax"))); } if (createStatement->inhRelations != NIL) { if (createStatement->partbound != NULL) { /* process CREATE TABLE ... PARTITION OF command */ PostprocessCreateTableStmtPartitionOf(createStatement, queryString); } else { /* process CREATE TABLE ... INHERITS ... */ if (IsTenantSchema(schemaId)) { ereport(ERROR, (errmsg("tables in a distributed schema cannot inherit " "or be inherited"))); } RangeVar *parentRelation = NULL; foreach_declared_ptr(parentRelation, createStatement->inhRelations) { Oid parentRelationId = RangeVarGetRelid(parentRelation, NoLock, missingOk); Assert(parentRelationId != InvalidOid); /* * Throw a better error message if the user tries to inherit a * tenant table or if the user tries to inherit from a tenant * table. */ if (IsTenantSchema(get_rel_namespace(parentRelationId))) { ereport(ERROR, (errmsg("tables in a distributed schema cannot " "inherit or be inherited"))); } else if (IsCitusTable(parentRelationId)) { /* here we error out if inheriting a distributed table */ ereport(ERROR, (errmsg("non-distributed tables cannot inherit " "distributed tables"))); } } } } } /* * PostprocessCreateTableStmtForeignKeys drops ands re-defines foreign keys * defined by given CREATE TABLE command if command defined any foreign to * reference or citus local tables. */ static void PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement) { if (!ShouldEnableLocalReferenceForeignKeys()) { /* * Either the user disabled foreign keys from/to local/reference tables * or the coordinator is not in the metadata */ return; } /* * Relation must exist and it is already locked as standard process utility * is already executed. */ bool missingOk = false; Oid relationId = RangeVarGetRelid(createStatement->relation, NoLock, missingOk); if (ShouldCreateTenantSchemaTable(relationId)) { /* * Avoid unnecessarily adding the table into metadata if we will * distribute it as a tenant table later. */ return; } /* * As we are just creating the table, we cannot have foreign keys that our * relation is referenced. So we use INCLUDE_REFERENCING_CONSTRAINTS here. * Reason behind using other two flags is explained below. */ int nonDistTableFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS | INCLUDE_CITUS_LOCAL_TABLES | INCLUDE_REFERENCE_TABLES; List *nonDistTableForeignKeyIdList = GetForeignKeyOids(relationId, nonDistTableFKeysFlag); bool hasForeignKeyToNonDistTable = list_length(nonDistTableForeignKeyIdList) != 0; if (hasForeignKeyToNonDistTable) { /* * To support foreign keys from postgres tables to reference or citus * local tables, we drop and re-define foreign keys so that our ALTER * TABLE hook does the necessary job. */ List *relationFKeyCreationCommands = GetForeignConstraintCommandsInternal(relationId, nonDistTableFKeysFlag); DropRelationForeignKeys(relationId, nonDistTableFKeysFlag); bool skip_validation = true; ExecuteForeignKeyCreateCommandList(relationFKeyCreationCommands, skip_validation); } } /* * ShouldEnableLocalReferenceForeignKeys is a wrapper around getting the GUC * EnableLocalReferenceForeignKeys. If the coordinator is not added * to the metadata, the function returns false. Else, the function returns * the value set by the user * */ bool ShouldEnableLocalReferenceForeignKeys(void) { if (!EnableLocalReferenceForeignKeys) { return false; } return CoordinatorAddedAsWorkerNode(); } /* * PostprocessCreateTableStmtPartitionOf processes CREATE TABLE ... PARTITION OF * statements and it checks if user creates the table as a partition of a distributed * table. In that case, it distributes partition as well. Since the table itself is a * partition, CreateDistributedTable will attach it to its parent table automatically * after distributing it. */ static void PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement, const char *queryString) { RangeVar *parentRelation = linitial(createStatement->inhRelations); bool missingOk = false; Oid parentRelationId = RangeVarGetRelid(parentRelation, NoLock, missingOk); /* a partition can only inherit from single parent table */ Assert(list_length(createStatement->inhRelations) == 1); Assert(parentRelationId != InvalidOid); Oid relationId = RangeVarGetRelid(createStatement->relation, NoLock, missingOk); /* * In case of an IF NOT EXISTS statement, Postgres lets it pass through the * standardProcess_Utility, and gets into this Post-process hook by * ignoring the statement if the table already exists. Thus, we need to make * sure Citus behaves like plain PG in case the relation already exists. */ if (createStatement->if_not_exists) { if (IsCitusTable(relationId)) { /* * Ignore if the relation is already distributed. */ return; } else if (!PartitionTable(relationId) || PartitionParentOid(relationId) != parentRelationId) { /* * Ignore if the relation is not a partition, or if that * partition's parent is not the current parent from parentRelationId */ return; } } if (IsTenantSchema(get_rel_namespace(parentRelationId)) || IsTenantSchema(get_rel_namespace(relationId))) { ErrorIfIllegalPartitioningInTenantSchema(parentRelationId, relationId); } /* * If a partition is being created and if its parent is a distributed * table, we will distribute this table as well. */ if (IsCitusTable(parentRelationId)) { /* * We can create Citus local tables right away, without switching to * sequential mode, because they are going to have only one shard. */ if (IsCitusTableType(parentRelationId, CITUS_LOCAL_TABLE)) { CreateCitusLocalTablePartitionOf(createStatement, relationId, parentRelationId); return; } DistributePartitionUsingParent(parentRelationId, relationId); } } /* * PreprocessAlterTableStmtAttachPartition takes AlterTableStmt object as * parameter but it only processes into ALTER TABLE ... ATTACH PARTITION * commands and distributes the partition if necessary. There are four cases * to consider; * * Parent is not distributed, partition is not distributed: We do not need to * do anything in this case. * * Parent is not distributed, partition is distributed: This can happen if * user first distributes a table and tries to attach it to a non-distributed * table. Non-distributed tables cannot have distributed partitions, thus we * simply error out in this case. * * Parent is distributed, partition is not distributed: We should distribute * the table and attach it to its parent in workers. CreateDistributedTable * perform both of these operations. Thus, we will not propagate ALTER TABLE * ... ATTACH PARTITION command to workers. * * Parent is distributed, partition is distributed: Partition is already * distributed, we only need to attach it to its parent in workers. Attaching * operation will be performed via propagating this ALTER TABLE ... ATTACH * PARTITION command to workers. * * This function does nothing if the provided CreateStmt is not an ALTER TABLE ... * ATTACH PARTITION OF command. */ List * PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, const char *queryString) { List *commandList = alterTableStatement->cmds; AlterTableCmd *alterTableCommand = NULL; foreach_declared_ptr(alterTableCommand, commandList) { if (alterTableCommand->subtype == AT_AttachPartition) { /* * We acquire the lock on the parent and child as we are in the pre-process * and want to ensure we acquire the locks in the same order with Postgres */ LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid parentRelationId = AlterTableLookupRelation(alterTableStatement, lockmode); PartitionCmd *partitionCommand = (PartitionCmd *) alterTableCommand->def; bool partitionMissingOk = true; Oid partitionRelationId = RangeVarGetRelid(partitionCommand->name, lockmode, partitionMissingOk); if (!OidIsValid(partitionRelationId)) { /* * We can stop propagation here instead of logging error. Pg will complain * in standard_utility, so we are safe to stop here. We pass missing_ok * as true to not diverge from pg output. */ return NIL; } if (IsTenantSchema(get_rel_namespace(parentRelationId)) || IsTenantSchema(get_rel_namespace(partitionRelationId))) { ErrorIfIllegalPartitioningInTenantSchema(parentRelationId, partitionRelationId); } if (!IsCitusTable(parentRelationId)) { /* * If the parent is a regular Postgres table, but the partition is a * Citus table, we error out. */ ErrorIfAttachCitusTableToPgLocalTable(parentRelationId, partitionRelationId); /* * If both the parent and the child table are Postgres tables, * we can just skip preprocessing this command. */ continue; } /* Citus doesn't support multi-level partitioned tables */ ErrorIfMultiLevelPartitioning(parentRelationId, partitionRelationId); /* attaching to a Citus table */ PreprocessAttachPartitionToCitusTable(parentRelationId, partitionRelationId); } } return NIL; } /* * PreprocessAttachPartitionToCitusTable takes a parent relation, which is a Citus table, * and a partition to be attached to it. * If the partition table is a regular Postgres table: * - Converts the partition to Citus Local Table, if the parent is a Citus Local Table. * - Distributes the partition, if the parent is a distributed table. * If not, calls PreprocessAttachCitusPartitionToCitusTable to attach given partition to * the parent relation. */ static void PreprocessAttachPartitionToCitusTable(Oid parentRelationId, Oid partitionRelationId) { Assert(IsCitusTable(parentRelationId)); /* reference tables cannot be partitioned */ Assert(!IsCitusTableType(parentRelationId, REFERENCE_TABLE)); /* if parent of this table is distributed, distribute this table too */ if (!IsCitusTable(partitionRelationId)) { if (IsCitusTableType(parentRelationId, CITUS_LOCAL_TABLE)) { /* * We pass the cascade option as false, since Citus Local Table partitions * cannot have non-inherited foreign keys. */ bool cascadeViaForeignKeys = false; CitusTableCacheEntry *entry = GetCitusTableCacheEntry(parentRelationId); bool autoConverted = entry->autoConverted; CreateCitusLocalTable(partitionRelationId, cascadeViaForeignKeys, autoConverted); } else if (IsCitusTableType(parentRelationId, DISTRIBUTED_TABLE)) { DistributePartitionUsingParent(parentRelationId, partitionRelationId); } } else { /* both the parent and child are Citus tables */ PreprocessAttachCitusPartitionToCitusTable(parentRelationId, partitionRelationId); } } /* * PreprocessAttachCitusPartitionToCitusTable takes a parent relation, and a partition * to be attached to it. Both of them are Citus tables. * Errors out if the partition is a reference table. * Errors out if the partition is distributed and the parent is a Citus Local Table. * Distributes the partition, if it's a Citus Local Table, and the parent is distributed. */ static void PreprocessAttachCitusPartitionToCitusTable(Oid parentCitusRelationId, Oid partitionRelationId) { if (IsCitusTableType(partitionRelationId, REFERENCE_TABLE)) { ereport(ERROR, (errmsg("partitioned reference tables are not supported"))); } else if (IsCitusTableType(partitionRelationId, DISTRIBUTED_TABLE) && IsCitusTableType(parentCitusRelationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("non-distributed partitioned tables cannot have " "distributed partitions"))); } else if (IsCitusTableType(partitionRelationId, CITUS_LOCAL_TABLE) && IsCitusTableType(parentCitusRelationId, DISTRIBUTED_TABLE)) { /* if the parent is a distributed table, distribute the partition too */ DistributePartitionUsingParent(parentCitusRelationId, partitionRelationId); } else if (IsCitusTableType(partitionRelationId, CITUS_LOCAL_TABLE) && IsCitusTableType(parentCitusRelationId, CITUS_LOCAL_TABLE)) { /* * We should ensure that the partition relation has no foreign keys, * as Citus Local Table partitions can only have inherited foreign keys. */ if (TableHasExternalForeignKeys(partitionRelationId)) { ereport(ERROR, (errmsg("partition local tables added to citus metadata " "cannot have non-inherited foreign keys"))); } } /* * We don't need to add other cases here, like distributed - distributed and * citus_local - citus_local, as PreprocessAlterTableStmt and standard process * utility would do the work to attach partitions to shell and shard relations. */ } /* * DistributePartitionUsingParent takes a parent and a partition relation and * distributes the partition, using the same distribution column as the parent, if the * parent has a distribution column. It creates a *hash* distributed table by default, as * partitioned tables can only be distributed by hash, unless it's null key distributed. * * If the parent has no distribution key, we distribute the partition with null key too. */ static void DistributePartitionUsingParent(Oid parentCitusRelationId, Oid partitionRelationId) { char *parentRelationName = generate_qualified_relation_name(parentCitusRelationId); /* * We can create tenant tables and single shard tables right away, without * switching to sequential mode, because they are going to have only one shard. */ if (ShouldCreateTenantSchemaTable(partitionRelationId)) { CreateTenantSchemaTable(partitionRelationId); return; } else if (!HasDistributionKey(parentCitusRelationId)) { /* * If the parent is null key distributed, we should distribute the partition * with null distribution key as well. */ ColocationParam colocationParam = { .colocationParamType = COLOCATE_WITH_TABLE_LIKE_OPT, .colocateWithTableName = parentRelationName, }; CreateSingleShardTable(partitionRelationId, colocationParam); return; } Var *distributionColumn = DistPartitionKeyOrError(parentCitusRelationId); char *distributionColumnName = ColumnToColumnName(parentCitusRelationId, (Node *) distributionColumn); char distributionMethod = DISTRIBUTE_BY_HASH; SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong( parentCitusRelationId, partitionRelationId); CreateDistributedTable(partitionRelationId, distributionColumnName, distributionMethod, ShardCount, false, parentRelationName); } /* * ErrorIfMultiLevelPartitioning takes a parent, and a partition relation to be attached * and errors out if the partition is also a partitioned table, which means we are * trying to build a multi-level partitioned table. */ static void ErrorIfMultiLevelPartitioning(Oid parentRelationId, Oid partitionRelationId) { if (PartitionedTable(partitionRelationId)) { char *relationName = get_rel_name(partitionRelationId); char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Citus doesn't support multi-level " "partitioned tables"), errdetail("Relation \"%s\" is partitioned table " "itself and it is also partition of " "relation \"%s\".", relationName, parentRelationName))); } } /* * ErrorIfAttachCitusTableToPgLocalTable takes a parent, and a partition relation * to be attached. Errors out if the partition is a Citus table, and the parent is a * regular Postgres table. */ static void ErrorIfAttachCitusTableToPgLocalTable(Oid parentRelationId, Oid partitionRelationId) { if (!IsCitusTable(parentRelationId) && IsCitusTable(partitionRelationId)) { char *parentRelationName = get_rel_name(parentRelationId); ereport(ERROR, (errmsg("non-citus partitioned tables cannot have " "citus table partitions"), errhint("Distribute the partitioned table \"%s\" " "instead, or add it to metadata", parentRelationName))); } } /* * PostprocessAlterTableSchemaStmt is executed after the change has been applied * locally, we can now use the new dependencies of the table to ensure all its * dependencies exist on the workers before we apply the commands remotely. */ List * PostprocessAlterTableSchemaStmt(Node *node, const char *queryString) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); /* * We will let Postgres deal with missing_ok */ List *tableAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, true); /* the code-path only supports a single object */ Assert(list_length(tableAddresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *tableAddress = linitial(tableAddresses); /* * Check whether we are dealing with a sequence or view here and route queries * accordingly to the right processor function. */ char relKind = get_rel_relkind(tableAddress->objectId); if (relKind == RELKIND_SEQUENCE) { stmt->objectType = OBJECT_SEQUENCE; return PostprocessAlterSequenceSchemaStmt((Node *) stmt, queryString); } else if (relKind == RELKIND_VIEW) { stmt->objectType = OBJECT_VIEW; return PostprocessAlterViewSchemaStmt((Node *) stmt, queryString); } if (!ShouldPropagate() || !IsCitusTable(tableAddress->objectId)) { return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(tableAddresses); return NIL; } /* * ChooseForeignKeyConstraintNameAddition returns the string of column names to be used when generating a foreign * key constraint name. This function is copied from postgres codebase. */ static char * ChooseForeignKeyConstraintNameAddition(List *columnNames) { char buf[NAMEDATALEN * 2]; int buflen = 0; buf[0] = '\0'; String *columnNameString = NULL; foreach_declared_ptr(columnNameString, columnNames) { const char *name = strVal(columnNameString); if (buflen > 0) { buf[buflen++] = '_'; /* insert _ between names */ } /* * At this point we have buflen <= NAMEDATALEN. name should be less * than NAMEDATALEN already, but use strlcpy for paranoia. */ strlcpy(buf + buflen, name, NAMEDATALEN); buflen += strlen(buf + buflen); if (buflen >= NAMEDATALEN) { break; } } return pstrdup(buf); } /* * GenerateConstraintName creates and returns a default name for the constraints Citus supports * for default naming. See ConstTypeCitusCanDefaultName function for the supported constraint types. */ static char * GenerateConstraintName(const char *tableName, Oid namespaceId, Constraint *constraint) { char *conname = NULL; switch (constraint->contype) { case CONSTR_PRIMARY: { conname = ChooseIndexName(tableName, namespaceId, NULL, NULL, true, true); break; } case CONSTR_UNIQUE: { ListCell *lc; List *indexParams = NIL; foreach(lc, constraint->keys) { IndexElem *iparam = makeNode(IndexElem); iparam->name = pstrdup(strVal(lfirst(lc))); indexParams = lappend(indexParams, iparam); } conname = ChooseIndexName(tableName, namespaceId, ChooseIndexColumnNames(indexParams), NULL, false, true); break; } case CONSTR_EXCLUSION: { ListCell *lc; List *indexParams = NIL; List *excludeOpNames = NIL; foreach(lc, constraint->exclusions) { List *pair = (List *) lfirst(lc); Assert(list_length(pair) == 2); IndexElem *elem = linitial_node(IndexElem, pair); List *opname = lsecond_node(List, pair); indexParams = lappend(indexParams, elem); excludeOpNames = lappend(excludeOpNames, opname); } conname = ChooseIndexName(tableName, namespaceId, ChooseIndexColumnNames(indexParams), excludeOpNames, false, true); break; } case CONSTR_CHECK: { conname = ChooseConstraintName(tableName, NULL, "check", namespaceId, NIL); break; } case CONSTR_FOREIGN: { conname = ChooseConstraintName(tableName, ChooseForeignKeyConstraintNameAddition( constraint->fk_attrs), "fkey", namespaceId, NIL); break; } default: { ereport(ERROR, (errmsg( "unsupported constraint type for generating a constraint name: %d", constraint->contype))); break; } } return conname; } /* * EnsureSequentialModeForAlterTableOperation makes sure that the current transaction is already in * sequential mode, or can still safely be put in sequential mode, it errors if that is * not possible. The error contains information for the user to retry the transaction with * sequential mode set from the beginning. */ static void EnsureSequentialModeForAlterTableOperation(void) { const char *objTypeString = "ALTER TABLE ... ADD FOREIGN KEY"; if (ParallelQueryExecutedInTransaction()) { ereport(ERROR, (errmsg("cannot run %s command because there was a " "parallel operation on a distributed table in the " "transaction", objTypeString), errdetail("When running command on/for a distributed %s, Citus " "needs to perform all operations over a single " "connection per node to ensure consistency.", objTypeString), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail( "A command for a distributed %s is run. To make sure subsequent " "commands see the %s correctly we need to make sure to " "use only one connection for all future commands", objTypeString, objTypeString))); SetLocalMultiShardModifyModeToSequential(); } /* * SwitchToSequentialAndLocalExecutionIfConstraintNameTooLong generates the longest index constraint name * among the shards of the partitions, and if exceeds the limit switches to sequential and * local execution to prevent self-deadlocks. */ static void SwitchToSequentialAndLocalExecutionIfConstraintNameTooLong(Oid relationId, Constraint *constraint) { if (!PartitionedTable(relationId)) { /* Citus already handles long names for regular tables */ return; } if (ShardIntervalCount(relationId) == 0) { /* * Relation has no shards, so we cannot run into "long shard index * name" issue. */ return; } Oid longestNamePartitionId = PartitionWithLongestNameRelationId(relationId); if (!OidIsValid(longestNamePartitionId)) { /* no partitions have been created yet */ return; } char *longestPartitionShardName = get_rel_name(longestNamePartitionId); ShardInterval *shardInterval = LoadShardIntervalWithLongestShardName( longestNamePartitionId); AppendShardIdToName(&longestPartitionShardName, shardInterval->shardId); Relation rel = RelationIdGetRelation(longestNamePartitionId); Oid namespaceOid = RelationGetNamespace(rel); RelationClose(rel); char *longestConname = GenerateConstraintName(longestPartitionShardName, namespaceOid, constraint); if (longestConname && strnlen(longestConname, NAMEDATALEN) >= NAMEDATALEN - 1) { if (ParallelQueryExecutedInTransaction()) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg( "The constraint name (%s) on a shard is too long and could lead " "to deadlocks when executed in a transaction " "block after a parallel query", longestConname), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else { elog(DEBUG1, "the constraint name on the shards of the partition " "is too long, switching to sequential and local execution " "mode to prevent self deadlocks: %s", longestConname); SetLocalMultiShardModifyModeToSequential(); SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); } } } /* * PreprocessAlterTableAddConstraint creates a new constraint name for {PRIMARY KEY, UNIQUE, EXCLUDE, CHECK, FOREIGN KEY} * and changes the original alterTableCommand run by the standard utility hook to use the new constraint name. * Then it converts the ALTER TABLE ... ADD {PRIMARY KEY, UNIQUE, EXCLUDE, CHECK, FOREIGN KEY} ... command * into ALTER TABLE ... ADD CONSTRAINT {PRIMARY KEY, UNIQUE, EXCLUDE, CHECK, FOREIGN KEY} format and returns the DDLJob * to run this command in the workers. */ static List * PreprocessAlterTableAddConstraint(AlterTableStmt *alterTableStatement, Oid relationId, Constraint *constraint) { PrepareAlterTableStmtForConstraint(alterTableStatement, relationId, constraint); char *ddlCommand = DeparseTreeNode((Node *) alterTableStatement); DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->startNewTransaction = false; ddlJob->metadataSyncCommand = ddlCommand; if (constraint->contype == CONSTR_FOREIGN) { Oid rightRelationId = RangeVarGetRelid(constraint->pktable, NoLock, false); /* * If one of the relations involved in the FOREIGN KEY constraint is not a distributed table, citus errors out eventually. * PreprocessAlterTableStmt function returns an empty tasklist in those cases. * leftRelation is checked in PreprocessAlterTableStmt before * calling PreprocessAlterTableAddConstraint. However, we need to handle the rightRelation since PreprocessAlterTableAddConstraint * returns early. */ bool referencedIsLocalTable = !IsCitusTable(rightRelationId); if (referencedIsLocalTable) { ddlJob->taskList = NIL; } else { ddlJob->taskList = InterShardDDLTaskList(relationId, rightRelationId, ddlCommand); } } else { ddlJob->taskList = DDLTaskList(relationId, ddlCommand); } return list_make1(ddlJob); } /* * PrepareAlterTableStmtForConstraint assigns a name to the constraint if it * does not have one and switches to sequential and local execution if the * constraint name is too long. */ void PrepareAlterTableStmtForConstraint(AlterTableStmt *alterTableStatement, Oid relationId, Constraint *constraint) { if (constraint->conname == NULL && constraint->indexname == NULL) { Relation rel = RelationIdGetRelation(relationId); /* * Change the alterTableCommand so that the standard utility * hook runs it with the name we created. */ constraint->conname = GenerateConstraintName(RelationGetRelationName(rel), RelationGetNamespace(rel), constraint); RelationClose(rel); } SwitchToSequentialAndLocalExecutionIfConstraintNameTooLong(relationId, constraint); if (constraint->contype == CONSTR_FOREIGN) { Oid rightRelationId = RangeVarGetRelid(constraint->pktable, NoLock, false); if (IsCitusTableType(rightRelationId, REFERENCE_TABLE)) { EnsureSequentialModeForAlterTableOperation(); } } } /* * PreprocessAlterTableStmt determines whether a given ALTER TABLE statement * involves a distributed table. If so (and if the statement does not use * unsupported options), it modifies the input statement to ensure proper * execution against the master node table and creates a DDLJob to encapsulate * information needed during the worker node portion of DDL execution before * returning that DDLJob in a List. If no distributed table is involved, this * function returns NIL. */ List * PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, ProcessUtilityContext processUtilityContext) { AlterTableStmt *alterTableStatement = castNode(AlterTableStmt, node); /* first check whether a distributed relation is affected */ if (alterTableStatement->relation == NULL) { return NIL; } LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid leftRelationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!OidIsValid(leftRelationId)) { return NIL; } /* * check whether we are dealing with a sequence or view here */ char relKind = get_rel_relkind(leftRelationId); if (relKind == RELKIND_SEQUENCE) { AlterTableStmt *stmtCopy = copyObject(alterTableStatement); stmtCopy->objtype = OBJECT_SEQUENCE; /* * it must be ALTER TABLE .. OWNER TO .. * or ALTER TABLE .. SET LOGGED/UNLOGGED command * since these are the only ALTER commands of a sequence that * pass through an AlterTableStmt */ return PreprocessSequenceAlterTableStmt((Node *) stmtCopy, alterTableCommand, processUtilityContext); } else if (relKind == RELKIND_VIEW) { /* * it must be ALTER TABLE .. OWNER TO .. command * since this is the only ALTER command of a view that * passes through an AlterTableStmt */ AlterTableStmt *stmtCopy = copyObject(alterTableStatement); stmtCopy->objtype = OBJECT_VIEW; return PreprocessAlterViewStmt((Node *) stmtCopy, alterTableCommand, processUtilityContext); } /* * AlterTableStmt applies also to INDEX relations, and we have support for * SET/SET storage parameters in Citus, so we might have to check for * another relation here. * * ALTER INDEX ATTACH PARTITION also applies to INDEX relation, so we might * check another relation for that option as well. */ char leftRelationKind = get_rel_relkind(leftRelationId); if (leftRelationKind == RELKIND_INDEX || leftRelationKind == RELKIND_PARTITIONED_INDEX) { bool missingOk = false; leftRelationId = IndexGetRelation(leftRelationId, missingOk); } if (ShouldEnableLocalReferenceForeignKeys() && processUtilityContext != PROCESS_UTILITY_SUBCOMMAND && ATDefinesFKeyBetweenPostgresAndCitusLocalOrRef(alterTableStatement)) { /* * We don't process subcommands generated by postgres. * This is mainly because postgres issues ALTER TABLE commands * for some set of objects that are defined via CREATE TABLE commands. * However, citus already has a separate logic for CREATE TABLE * commands. * * To support foreign keys from/to postgres local tables to/from reference * or citus local tables, we convert given postgres local table -and the * other postgres tables that it is connected via a fkey graph- to a citus * local table. * * Note that we don't convert postgres tables to citus local tables if * coordinator is not added to metadata as CreateCitusLocalTable requires * this. In this case, we assume user is about to create reference or * distributed table from local table and we don't want to break user * experience by asking to add coordinator to metadata. */ ConvertPostgresLocalTablesToCitusLocalTables(alterTableStatement); /* * CreateCitusLocalTable converts relation to a shard relation and creates * shell table from scratch. * For this reason we should re-enter to PreprocessAlterTableStmt to operate * on shell table relation id. */ return PreprocessAlterTableStmt(node, alterTableCommand, processUtilityContext); } if (AlterTableDropsForeignKey(alterTableStatement)) { /* * The foreign key graph keeps track of the foreign keys including local tables. * So, even if a foreign key on a local table is dropped, we should invalidate * the graph so that the next commands can see the graph up-to-date. * We are aware that utility hook would still invalidate foreign key graph * even when command fails, but currently we are ok with that. */ MarkInvalidateForeignKeyGraph(); } bool referencingIsLocalTable = !IsCitusTable(leftRelationId); if (referencingIsLocalTable) { return NIL; } /* * The PostgreSQL parser dispatches several commands into the node type * AlterTableStmt, from ALTER INDEX to ALTER SEQUENCE or ALTER VIEW. Here * we have a special implementation for ALTER INDEX, and a specific error * message in case of unsupported sub-command. */ if (leftRelationKind == RELKIND_INDEX || leftRelationKind == RELKIND_PARTITIONED_INDEX) { ErrorIfUnsupportedAlterIndexStmt(alterTableStatement); } else { /* this function also accepts more than just RELKIND_RELATION... */ ErrorIfUnsupportedAlterTableStmt(alterTableStatement); } EnsureCoordinator(); /* these will be set in below loop according to subcommands */ Oid rightRelationId = InvalidOid; bool executeSequentially = false; /* * We check if there is: * - an ADD/DROP FOREIGN CONSTRAINT command in sub commands * list. If there is we assign referenced relation id to rightRelationId and * we also set skip_validation to true to prevent PostgreSQL to verify validity * of the foreign constraint in master. Validity will be checked in workers * anyway. * - an ADD COLUMN .. that is the only subcommand in the list OR * - an ADD COLUMN .. DEFAULT nextval('..') OR * an ADD COLUMN .. SERIAL pseudo-type OR * an ALTER COLUMN .. SET DEFAULT nextval('..'). If there is we set * deparseAT variable to true which means we will deparse the statement * before we propagate the command to shards. For shards, all the defaults * coming from a user-defined sequence will be replaced by * NOT NULL constraint. */ List *commandList = alterTableStatement->cmds; /* * if deparsing is needed, we will use a different version of the original * alterTableStmt */ bool deparseAT = false; bool propagateCommandToWorkers = true; /* * Sometimes we want to run a different DDL Command string in MX workers * For example, in cases where worker_nextval should be used instead * of nextval() in column defaults with type int and smallint */ bool useInitialDDLCommandString = true; AlterTableStmt *newStmt = copyObject(alterTableStatement); AlterTableCmd *newCmd = makeNode(AlterTableCmd); AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; /* * if deparsing is needed, we will use a different version of the original * AlterTableCmd */ newCmd = copyObject(command); if (alterTableType == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { /* * We only support ALTER TABLE ADD CONSTRAINT ... FOREIGN KEY, if it is * only subcommand of ALTER TABLE. It was already checked in * ErrorIfUnsupportedAlterTableStmt. */ Assert(list_length(commandList) == 1); rightRelationId = RangeVarGetRelid(constraint->pktable, lockmode, alterTableStatement->missing_ok); if (processUtilityContext != PROCESS_UTILITY_SUBCOMMAND && ShouldMarkConnectedRelationsNotAutoConverted(leftRelationId, rightRelationId)) { List *relationList = list_make2_oid(leftRelationId, rightRelationId); UpdateAutoConvertedForConnectedRelations(relationList, false); } /* * Foreign constraint validations will be done in workers. If we do not * set this flag, PostgreSQL tries to do additional checking when we drop * to standard_ProcessUtility. standard_ProcessUtility tries to open new * connections to workers to verify foreign constraints while original * transaction is in process, which causes deadlock. */ constraint->skip_validation = true; if (constraint->conname == NULL) { return PreprocessAlterTableAddConstraint(alterTableStatement, leftRelationId, constraint); } } /* * When constraint->indexname is not NULL we are handling an * ADD {PRIMARY KEY, UNIQUE} USING INDEX command. In this case * we do not have to create a name and change the command. * The existing index name will be used by the postgres. */ else if (constraint->conname == NULL && constraint->indexname == NULL) { if (ConstrTypeCitusCanDefaultName(constraint->contype)) { /* * Create a constraint name. Convert ALTER TABLE ... ADD PRIMARY ... command into * ALTER TABLE ... ADD CONSTRAINT PRIMARY KEY ... form and create the ddl jobs * for running this form of the command on the workers. */ return PreprocessAlterTableAddConstraint(alterTableStatement, leftRelationId, constraint); } } } else if (alterTableType == AT_DropConstraint) { char *constraintName = command->name; if (ConstraintIsAForeignKey(constraintName, leftRelationId)) { /* * We only support ALTER TABLE DROP CONSTRAINT ... FOREIGN KEY, if it is * only subcommand of ALTER TABLE. It was already checked in * ErrorIfUnsupportedAlterTableStmt. */ Assert(list_length(commandList) == 1); bool missingOk = false; Oid foreignKeyId = get_relation_constraint_oid(leftRelationId, constraintName, missingOk); rightRelationId = GetReferencedTableId(foreignKeyId); } /* * We support deparsing for DROP CONSTRAINT, but currently deparsing is only * possible if all subcommands are supported. */ if (list_length(commandList) == 1 && alterTableStatement->objtype == OBJECT_TABLE) { deparseAT = true; } } else if (alterTableType == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { rightRelationId = RangeVarGetRelid(constraint->pktable, lockmode, alterTableStatement->missing_ok); /* * Foreign constraint validations will be done in workers. If we do not * set this flag, PostgreSQL tries to do additional checking when we drop * to standard_ProcessUtility. standard_ProcessUtility tries to open new * connections to workers to verify foreign constraints while original * transaction is in process, which causes deadlock. */ constraint->skip_validation = true; break; } } if (DeparserSupportsAlterTableAddColumn(alterTableStatement, command)) { deparseAT = true; constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (ConstrTypeCitusCanDefaultName(constraint->contype)) { PrepareAlterTableStmtForConstraint(alterTableStatement, leftRelationId, constraint); } } /* * Copy the constraints to the new subcommand because now we * might have assigned names to some of them. */ ColumnDef *newColumnDef = (ColumnDef *) newCmd->def; newColumnDef->constraints = copyObject(columnConstraints); } /* * We check for ADD COLUMN .. DEFAULT expr * if expr contains nextval('user_defined_seq') * we should deparse the statement */ constraint = NULL; int constraintIdx = 0; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { if (constraint->raw_expr != NULL) { ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, constraint->raw_expr, EXPR_KIND_COLUMN_DEFAULT); if (contain_nextval_expression_walker(expr, NULL)) { deparseAT = true; useInitialDDLCommandString = false; /* drop the default expression from new subcomand */ ColumnDef *newColumnDef = (ColumnDef *) newCmd->def; newColumnDef->constraints = list_delete_nth_cell(newColumnDef->constraints, constraintIdx); } } /* there can only be one DEFAULT constraint that can be used per column */ break; } constraintIdx++; } /* * We check for ADD COLUMN .. SERIAL pseudo-type * if that's the case, we should deparse the statement * The structure of this check is copied from transformColumnDefinition. */ if (columnDefinition->typeName && list_length( columnDefinition->typeName->names) == 1 && !columnDefinition->typeName->pct_type) { char *typeName = strVal(linitial(columnDefinition->typeName->names)); if (strcmp(typeName, "smallserial") == 0 || strcmp(typeName, "serial2") == 0 || strcmp(typeName, "serial") == 0 || strcmp(typeName, "serial4") == 0 || strcmp(typeName, "bigserial") == 0 || strcmp(typeName, "serial8") == 0) { deparseAT = true; ColumnDef *newColDef = copyObject(columnDefinition); newColDef->is_not_null = false; if (strcmp(typeName, "smallserial") == 0 || strcmp(typeName, "serial2") == 0) { newColDef->typeName->names = NIL; newColDef->typeName->typeOid = INT2OID; } else if (strcmp(typeName, "serial") == 0 || strcmp(typeName, "serial4") == 0) { newColDef->typeName->names = NIL; newColDef->typeName->typeOid = INT4OID; } else if (strcmp(typeName, "bigserial") == 0 || strcmp(typeName, "serial8") == 0) { newColDef->typeName->names = NIL; newColDef->typeName->typeOid = INT8OID; } newCmd->def = (Node *) newColDef; } } } /* * We check for ALTER COLUMN .. SET/DROP DEFAULT * we should not propagate anything to shards */ else if (alterTableType == AT_ColumnDefault) { ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, command->def, EXPR_KIND_COLUMN_DEFAULT); if (contain_nextval_expression_walker(expr, NULL)) { propagateCommandToWorkers = false; useInitialDDLCommandString = false; } } else if (alterTableType == AT_AttachPartition) { PartitionCmd *partitionCommand = (PartitionCmd *) command->def; Oid attachedRelationId = RangeVarGetRelid(partitionCommand->name, NoLock, false); char attachedRelationKind = get_rel_relkind(attachedRelationId); /* * We support ALTER INDEX ATTACH PARTITION and ALTER TABLE ATTACH PARTITION * if it is only subcommand of ALTER TABLE command. Since the attached relation * type is index for ALTER INDEX ATTACH PARTITION, we need to use the relation * id this index is created for. * * Both were already checked in ErrorIfUnsupportedAlterIndexStmt and * ErrorIfUnsupportedAlterTableStmt. */ if (attachedRelationKind == RELKIND_INDEX) { bool missingOk = false; rightRelationId = IndexGetRelation(attachedRelationId, missingOk); /* * Since left relation is checked above to make sure it is Citus table, * partition of that must be Citus table as well. */ Assert(IsCitusTable(rightRelationId)); } else if (attachedRelationKind == RELKIND_RELATION || attachedRelationKind == RELKIND_FOREIGN_TABLE) { Assert(list_length(commandList) <= 1); /* * Do not generate tasks if relation is distributed and the partition * is not distributed. Because, we'll manually convert the partition into * distributed table and co-locate with its parent. */ if (!IsCitusTable(attachedRelationId)) { return NIL; } rightRelationId = attachedRelationId; } } else if (alterTableType == AT_DetachPartition) { PartitionCmd *partitionCommand = (PartitionCmd *) command->def; /* * We only support ALTER TABLE DETACH PARTITION, if it is only subcommand of * ALTER TABLE. It was already checked in ErrorIfUnsupportedAlterTableStmt. */ Assert(list_length(commandList) <= 1); rightRelationId = RangeVarGetRelid(partitionCommand->name, NoLock, false); } else if (AlterTableCommandTypeIsTrigger(alterTableType)) { char *triggerName = command->name; return CitusCreateTriggerCommandDDLJob(leftRelationId, triggerName, alterTableCommand); } /* * We check and set the execution mode only if we fall into either of first two * conditional blocks, otherwise we already continue the loop */ executeSequentially |= SetupExecutionModeForAlterTable(leftRelationId, command); } if (executeSequentially) { SetLocalMultiShardModifyModeToSequential(); } /* fill them here as it is possible to use them in some conditional blocks below */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, leftRelationId); if (deparseAT) { newStmt->cmds = list_make1(newCmd); alterTableCommand = DeparseTreeNode((Node *) newStmt); } ddlJob->metadataSyncCommand = useInitialDDLCommandString ? alterTableCommand : NULL; if (OidIsValid(rightRelationId)) { bool referencedIsLocalTable = !IsCitusTable(rightRelationId); if (referencedIsLocalTable || !propagateCommandToWorkers) { ddlJob->taskList = NIL; } else { /* if foreign key or attaching partition index related, use specialized task list function ... */ ddlJob->taskList = InterShardDDLTaskList(leftRelationId, rightRelationId, alterTableCommand); } } else { /* ... otherwise use standard DDL task list function */ ddlJob->taskList = DDLTaskList(leftRelationId, alterTableCommand); if (!propagateCommandToWorkers) { ddlJob->taskList = NIL; } } List *ddlJobs = list_make1(ddlJob); return ddlJobs; } /* * DeparserSupportsAlterTableAddColumn returns true if it's safe to deparse * the given ALTER TABLE statement that is known to contain given ADD COLUMN * subcommand. */ static bool DeparserSupportsAlterTableAddColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *addColumnSubCommand) { /* * We support deparsing for ADD COLUMN only of it's the only * subcommand. */ if (list_length(alterTableStatement->cmds) == 1 && alterTableStatement->objtype == OBJECT_TABLE) { ColumnDef *columnDefinition = (ColumnDef *) addColumnSubCommand->def; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnDefinition->constraints) { if (constraint->contype == CONSTR_CHECK) { /* * Given that we're in the preprocess, any reference to the * column that we're adding would break the deparser. This * can only be the case with CHECK constraints. For this * reason, we skip deparsing the command and fall back to * legacy behavior that we follow for ADD COLUMN subcommands. * * For other constraint types, we prepare the constraint to * make sure that we can deparse it. */ return false; } } return true; } return false; } /* * ATDefinesFKeyBetweenPostgresAndCitusLocalOrRef returns true if given * alter table command defines foreign key between a postgres table and a * reference or citus local table. */ static bool ATDefinesFKeyBetweenPostgresAndCitusLocalOrRef(AlterTableStmt *alterTableStatement) { List *foreignKeyConstraintList = GetAlterTableAddFKeyConstraintList(alterTableStatement); if (list_length(foreignKeyConstraintList) == 0) { /* we are not defining any foreign keys */ return false; } List *rightRelationIdList = GetAlterTableAddFKeyRightRelationIdList(alterTableStatement); LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid leftRelationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!IsCitusTable(leftRelationId)) { return RelationIdListContainsCitusTableType(rightRelationIdList, CITUS_LOCAL_TABLE) || RelationIdListContainsCitusTableType(rightRelationIdList, REFERENCE_TABLE); } else if (IsCitusTableType(leftRelationId, CITUS_LOCAL_TABLE) || IsCitusTableType(leftRelationId, REFERENCE_TABLE)) { return RelationIdListContainsPostgresTable(rightRelationIdList); } return false; } /* * ShouldMarkConnectedRelationsNotAutoConverted takes two relations. * If both of them are Citus Local Tables, and one of them is auto-converted while the * other one is not; then it returns true. False otherwise. */ static bool ShouldMarkConnectedRelationsNotAutoConverted(Oid leftRelationId, Oid rightRelationId) { if (!IsCitusTableType(leftRelationId, CITUS_LOCAL_TABLE)) { return false; } if (!IsCitusTableType(rightRelationId, CITUS_LOCAL_TABLE)) { return false; } CitusTableCacheEntry *entryLeft = GetCitusTableCacheEntry(leftRelationId); CitusTableCacheEntry *entryRight = GetCitusTableCacheEntry(rightRelationId); return entryLeft->autoConverted != entryRight->autoConverted; } /* * RelationIdListContainsCitusTableType returns true if given relationIdList * contains a citus table with given type. */ static bool RelationIdListContainsCitusTableType(List *relationIdList, CitusTableType citusTableType) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (IsCitusTableType(relationId, citusTableType)) { return true; } } return false; } /* * RelationIdListContainsPostgresTable returns true if given relationIdList * contains a postgres table. */ static bool RelationIdListContainsPostgresTable(List *relationIdList) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (OidIsValid(relationId) && !IsCitusTable(relationId)) { return true; } } return false; } /* * ConvertPostgresLocalTablesToCitusLocalTables converts each postgres table * involved in foreign keys to be defined by given alter table command and the * other tables connected to them via a foreign key graph to citus local tables. */ static void ConvertPostgresLocalTablesToCitusLocalTables(AlterTableStmt *alterTableStatement) { List *rightRelationRangeVarList = GetAlterTableAddFKeyRightRelationRangeVarList(alterTableStatement); RangeVar *leftRelationRangeVar = alterTableStatement->relation; List *relationRangeVarList = lappend(rightRelationRangeVarList, leftRelationRangeVar); /* * To prevent deadlocks, sort the list before converting each postgres local * table to a citus local table. */ relationRangeVarList = SortList(relationRangeVarList, CompareRangeVarsByOid); bool containsAnyUserConvertedLocalRelation = RangeVarListHasLocalRelationConvertedByUser(relationRangeVarList, alterTableStatement); /* * Here we should operate on RangeVar objects since relations oid's would * change in below loop due to CreateCitusLocalTable. */ RangeVar *relationRangeVar; foreach_declared_ptr(relationRangeVar, relationRangeVarList) { List *commandList = alterTableStatement->cmds; LOCKMODE lockMode = AlterTableGetLockLevel(commandList); bool missingOk = alterTableStatement->missing_ok; Oid relationId = RangeVarGetRelid(relationRangeVar, lockMode, missingOk); if (!OidIsValid(relationId)) { /* * As we are in preprocess, missingOk might be true and relation * might not exist. */ continue; } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { CitusTableCacheEntry *entry = GetCitusTableCacheEntry(relationId); if (!entry->autoConverted) { /* * This citus local table is already added to the metadata * by the user, so no further operation needed. */ continue; } else if (!containsAnyUserConvertedLocalRelation) { /* * We are safe to skip this relation because none of the citus local * tables involved are manually added to the metadata by the user. * This implies that all the Citus local tables involved are marked * as autoConverted = true and there is no chance to update * autoConverted = false. */ continue; } } else if (IsCitusTable(relationId)) { /* we can directly skip for table types other than citus local tables */ continue; } /* * The only reason behind using a try/catch block here is giving a proper * error message. For example, when creating a citus local table we might * give an error telling that partitioned tables are not supported for * citus local table creation. But as a user it wouldn't make much sense * to see such an error. So here we extend error message to tell that we * actually ended up with this error when trying to define the foreign key. * * Also, as CopyErrorData() requires (CurrentMemoryContext != ErrorContext), * so we store CurrentMemoryContext here. */ MemoryContext savedMemoryContext = CurrentMemoryContext; PG_TRY(); { bool cascade = true; /* * Without this check, we would be erroring out in CreateCitusLocalTable * for this case anyway. The purpose of this check&error is to provide * a more meaningful message for the user. */ if (PartitionTable(relationId)) { ereport(ERROR, (errmsg("cannot build foreign key between" " reference table and a partition"), errhint("Try using parent table: %s", get_rel_name(PartitionParentOid(relationId))))); } else { /* * There might be two scenarios: * * a) A user created foreign key from a reference table * to Postgres local table(s) or Citus local table(s) * where all of the citus local tables involved are auto * converted. In that case, we mark the new table as auto * converted as well. * * b) A user created foreign key from a reference table * to Postgres local table(s) or Citus local table(s) * where at least one of the citus local tables * involved is not auto converted. In that case, we mark * this new Citus local table as autoConverted = false * as well. Because our logic is to keep all the connected * Citus local tables to have the same autoConverted value. */ bool autoConverted = containsAnyUserConvertedLocalRelation ? false : true; CreateCitusLocalTable(relationId, cascade, autoConverted); } } PG_CATCH(); { MemoryContextSwitchTo(savedMemoryContext); ErrorData *errorData = CopyErrorData(); FlushErrorState(); if (errorData->elevel != ERROR) { PG_RE_THROW(); } /* override error detail */ errorData->detail = "When adding a foreign key from a local table to " "a reference table, Citus applies a conversion to " "all the local tables in the foreign key graph"; ThrowErrorData(errorData); } PG_END_TRY(); } } /* * RangeVarListHasLocalRelationConvertedByUser takes a list of relations and returns true * if any of these relations is marked as auto-converted = false. Returns true otherwise. * This function also takes the current alterTableStatement command, to obtain the * necessary locks. */ static bool RangeVarListHasLocalRelationConvertedByUser(List *relationRangeVarList, AlterTableStmt *alterTableStatement) { RangeVar *relationRangeVar; foreach_declared_ptr(relationRangeVar, relationRangeVarList) { /* * Here we iterate the relation list, and if at least one of the relations * is marked as not-auto-converted, we should mark all of them as * not-auto-converted. In that case, we return true here. */ List *commandList = alterTableStatement->cmds; LOCKMODE lockMode = AlterTableGetLockLevel(commandList); bool missingOk = alterTableStatement->missing_ok; Oid relationId = RangeVarGetRelid(relationRangeVar, lockMode, missingOk); if (OidIsValid(relationId) && IsCitusTable(relationId) && IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { CitusTableCacheEntry *entry = GetCitusTableCacheEntry(relationId); if (!entry->autoConverted) { return true; } } } return false; } /* * CompareRangeVarsByOid is a comparison function to sort RangeVar object list. */ static int CompareRangeVarsByOid(const void *leftElement, const void *rightElement) { RangeVar *leftRangeVar = *((RangeVar **) leftElement); RangeVar *rightRangeVar = *((RangeVar **) rightElement); /* * Any way we will check their existence, so it's okay to map non-existing * relations to InvalidOid when sorting. */ bool missingOk = true; /* * As this is an object comparator function, there is no way to understand * proper lock mode. So assume caller already locked relations. */ LOCKMODE lockMode = NoLock; Oid leftRelationId = RangeVarGetRelid(leftRangeVar, lockMode, missingOk); Oid rightRelationId = RangeVarGetRelid(rightRangeVar, lockMode, missingOk); return CompareOids(&leftRelationId, &rightRelationId); } /* * GetAlterTableAddFKeyRightRelationIdList returns a list of oid's for right * relations involved in foreign keys to be defined by given ALTER TABLE command. */ static List * GetAlterTableAddFKeyRightRelationIdList(AlterTableStmt *alterTableStatement) { List *rightRelationRangeVarList = GetAlterTableAddFKeyRightRelationRangeVarList(alterTableStatement); List *commandList = alterTableStatement->cmds; LOCKMODE lockMode = AlterTableGetLockLevel(commandList); bool missingOk = alterTableStatement->missing_ok; List *rightRelationIdList = GetRelationIdListFromRangeVarList(rightRelationRangeVarList, lockMode, missingOk); return rightRelationIdList; } /* * GetAlterTableAddFKeyRightRelationRangeVarList returns a list of RangeVar * objects for right relations involved in foreign keys to be defined by * given ALTER TABLE command. */ static List * GetAlterTableAddFKeyRightRelationRangeVarList(AlterTableStmt *alterTableStatement) { List *fKeyConstraintList = GetAlterTableAddFKeyConstraintList(alterTableStatement); List *rightRelationRangeVarList = GetRangeVarListFromFKeyConstraintList(fKeyConstraintList); return rightRelationRangeVarList; } /* * GetAlterTableAddFKeyConstraintList returns a list of Constraint objects for * foreign keys that given ALTER TABLE to be defined by given ALTER TABLE command. */ static List * GetAlterTableAddFKeyConstraintList(AlterTableStmt *alterTableStatement) { List *foreignKeyConstraintList = NIL; List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { List *commandForeignKeyConstraintList = GetAlterTableCommandFKeyConstraintList(command); foreignKeyConstraintList = list_concat(foreignKeyConstraintList, commandForeignKeyConstraintList); } return foreignKeyConstraintList; } /* * GetAlterTableCommandFKeyConstraintList returns a list of Constraint objects * for the foreign keys that given ALTER TABLE subcommand defines. Note that * this is only possible if it is an: * - ADD CONSTRAINT subcommand (explicitly defines) or, * - ADD COLUMN subcommand (implicitly defines by adding a new column that * references to another table. */ static List * GetAlterTableCommandFKeyConstraintList(AlterTableCmd *command) { List *fkeyConstraintList = NIL; AlterTableType alterTableType = command->subtype; if (alterTableType == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { fkeyConstraintList = lappend(fkeyConstraintList, constraint); } } else if (alterTableType == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { fkeyConstraintList = lappend(fkeyConstraintList, constraint); } } } return fkeyConstraintList; } /* * GetRangeVarListFromFKeyConstraintList returns a list of RangeVar objects for * right relations in fKeyConstraintList. */ static List * GetRangeVarListFromFKeyConstraintList(List *fKeyConstraintList) { List *rightRelationRangeVarList = NIL; Constraint *fKeyConstraint = NULL; foreach_declared_ptr(fKeyConstraint, fKeyConstraintList) { RangeVar *rightRelationRangeVar = fKeyConstraint->pktable; rightRelationRangeVarList = lappend(rightRelationRangeVarList, rightRelationRangeVar); } return rightRelationRangeVarList; } /* * GetRelationIdListFromRangeVarList returns relation id list for relations * identified by RangeVar objects in given list. */ static List * GetRelationIdListFromRangeVarList(List *rangeVarList, LOCKMODE lockMode, bool missingOk) { List *relationIdList = NIL; RangeVar *rangeVar = NULL; foreach_declared_ptr(rangeVar, rangeVarList) { Oid rightRelationId = RangeVarGetRelid(rangeVar, lockMode, missingOk); relationIdList = lappend_oid(relationIdList, rightRelationId); } return relationIdList; } /* * AlterTableCommandTypeIsTrigger returns true if given alter table command type * is identifies an ALTER TABLE .. TRIGGER .. command. */ static bool AlterTableCommandTypeIsTrigger(AlterTableType alterTableType) { switch (alterTableType) { case AT_EnableTrig: case AT_EnableAlwaysTrig: case AT_EnableReplicaTrig: case AT_EnableTrigUser: case AT_DisableTrig: case AT_DisableTrigUser: case AT_EnableTrigAll: case AT_DisableTrigAll: { return true; } default: { return false; } } } /* * ConstrTypeUsesIndex returns true if the given constraint type uses an index */ bool ConstrTypeUsesIndex(ConstrType constrType) { return constrType == CONSTR_PRIMARY || constrType == CONSTR_UNIQUE || constrType == CONSTR_EXCLUSION; } /* * ConstrTypeSupportsDefaultNaming returns true if we can generate a default name for the given constraint type */ bool ConstrTypeCitusCanDefaultName(ConstrType constrType) { return constrType == CONSTR_PRIMARY || constrType == CONSTR_UNIQUE || constrType == CONSTR_EXCLUSION || constrType == CONSTR_CHECK || constrType == CONSTR_FOREIGN; } /* * AlterTableDropsForeignKey returns true if the given AlterTableStmt drops * a foreign key. False otherwise. */ static bool AlterTableDropsForeignKey(AlterTableStmt *alterTableStatement) { LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); AlterTableCmd *command = NULL; foreach_declared_ptr(command, alterTableStatement->cmds) { AlterTableType alterTableType = command->subtype; if (alterTableType == AT_DropColumn) { char *columnName = command->name; if (ColumnAppearsInForeignKey(columnName, relationId)) { /* dropping a column in the either side of the fkey will drop the fkey */ return true; } } /* * In order to drop the foreign key, other than DROP COLUMN, the command must be * DROP CONSTRAINT command. */ if (alterTableType != AT_DropConstraint) { continue; } char *constraintName = command->name; if (ConstraintIsAForeignKey(constraintName, relationId)) { return true; } else if (ConstraintIsAUniquenessConstraint(constraintName, relationId)) { /* * If the uniqueness constraint of the column that the foreign key depends on * is getting dropped, then the foreign key will also be dropped. */ bool missingOk = false; Oid uniquenessConstraintId = get_relation_constraint_oid(relationId, constraintName, missingOk); Oid indexId = get_constraint_index(uniquenessConstraintId); if (AnyForeignKeyDependsOnIndex(indexId)) { return true; } } } return false; } /* * AnyForeignKeyDependsOnIndex scans pg_depend and returns true if given index * is valid and any foreign key depends on it. */ bool AnyForeignKeyDependsOnIndex(Oid indexId) { Oid dependentObjectClassId = RelationRelationId; Oid dependentObjectId = indexId; List *dependencyTupleList = GetPgDependTuplesForDependingObjects(dependentObjectClassId, dependentObjectId); HeapTuple dependencyTuple = NULL; foreach_declared_ptr(dependencyTuple, dependencyTupleList) { Form_pg_depend dependencyForm = (Form_pg_depend) GETSTRUCT(dependencyTuple); Oid dependingClassId = dependencyForm->classid; if (dependingClassId != ConstraintRelationId) { continue; } Oid dependingObjectId = dependencyForm->objid; if (ConstraintWithIdIsOfType(dependingObjectId, CONSTRAINT_FOREIGN)) { return true; } } return false; } /* * PreprocessAlterTableStmt issues a warning. * ALTER TABLE ALL IN TABLESPACE statements have their node type as * AlterTableMoveAllStmt. At the moment we do not support this functionality in * the distributed environment. We warn out here. */ List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (EnableUnsupportedFeatureMessages) { ereport(WARNING, (errmsg("not propagating ALTER TABLE ALL IN TABLESPACE " "commands to worker nodes"), errhint("Connect to worker nodes directly to manually " "move all tables."))); } return NIL; } /* * PreprocessAlterTableSchemaStmt is executed before the statement is applied * to the local postgres instance. * * In this stage we can prepare the commands that will alter the schemas of the * shards. */ List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); if (stmt->relation == NULL) { return NIL; } List *addresses = GetObjectAddressListFromParseTree((Node *) stmt, stmt->missing_ok, false); /* the code-path only supports a single object */ Assert(list_length(addresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); Oid relationId = address->objectId; /* * Check whether we are dealing with a sequence or view here and route queries * accordingly to the right processor function. We need to check both objects here * since PG supports targeting sequences and views with ALTER TABLE commands. */ char relKind = get_rel_relkind(relationId); if (relKind == RELKIND_SEQUENCE) { AlterObjectSchemaStmt *stmtCopy = copyObject(stmt); stmtCopy->objectType = OBJECT_SEQUENCE; return PreprocessAlterSequenceSchemaStmt((Node *) stmtCopy, queryString, processUtilityContext); } else if (relKind == RELKIND_VIEW) { AlterObjectSchemaStmt *stmtCopy = copyObject(stmt); stmtCopy->objectType = OBJECT_VIEW; return PreprocessAlterViewSchemaStmt((Node *) stmtCopy, queryString, processUtilityContext); } /* first check whether a distributed relation is affected */ if (!OidIsValid(relationId) || !IsCitusTable(relationId)) { return NIL; } Oid oldSchemaId = get_rel_namespace(relationId); Oid newSchemaId = get_namespace_oid(stmt->newschema, stmt->missing_ok); if (!OidIsValid(oldSchemaId) || !OidIsValid(newSchemaId)) { return NIL; } /* Do nothing if new schema is the same as old schema */ if (newSchemaId == oldSchemaId) { return NIL; } /* Undistribute table if its old schema is a tenant schema */ if (IsTenantSchema(oldSchemaId) && IsCoordinator()) { EnsureUndistributeTenantTableSafe(relationId, TenantOperationNames[TENANT_SET_SCHEMA]); char *oldSchemaName = get_namespace_name(oldSchemaId); char *tableName = stmt->relation->relname; ereport(NOTICE, (errmsg("undistributing table %s in distributed schema %s " "before altering its schema", tableName, oldSchemaName))); /* Undistribute tenant table by suppressing weird notices */ TableConversionParameters params = { .relationId = relationId, .cascadeViaForeignKeys = false, .bypassTenantCheck = true, .suppressNoticeMessages = true, }; UndistributeTable(¶ms); /* relation id changes after undistribute_table */ relationId = get_relname_relid(tableName, oldSchemaId); /* * After undistribution, the table could be Citus table or Postgres table. * If it is Postgres table, do not propagate the `ALTER TABLE SET SCHEMA` * command to workers. */ if (!IsCitusTable(relationId)) { return NIL; } } DDLJob *ddlJob = palloc0(sizeof(DDLJob)); QualifyTreeNode((Node *) stmt); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->metadataSyncCommand = DeparseTreeNode((Node *) stmt); ddlJob->taskList = DDLTaskList(relationId, ddlJob->metadataSyncCommand); return list_make1(ddlJob); } /* * SkipForeignKeyValidationIfConstraintIsFkey checks and processes the alter table * statement to be worked on the distributed table. Currently, it only processes * ALTER TABLE ... ADD FOREIGN KEY command to skip the validation step. */ void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement, bool processLocalRelation) { /* first check whether a distributed relation is affected */ if (alterTableStatement->relation == NULL) { return; } LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid leftRelationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!OidIsValid(leftRelationId)) { return; } if (!IsCitusTable(leftRelationId) && !processLocalRelation) { return; } /* * We check if there is a ADD FOREIGN CONSTRAINT command in sub commands * list. We set skip_validation to true to prevent PostgreSQL to verify * validity of the foreign constraint. Validity will be checked on the * shards anyway. */ AlterTableCmd *command = NULL; foreach_declared_ptr(command, alterTableStatement->cmds) { AlterTableType alterTableType = command->subtype; if (alterTableType == AT_AddConstraint) { /* skip only if the constraint is a foreign key */ Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { /* foreign constraint validations will be done in shards. */ constraint->skip_validation = true; } } } } /* * IsAlterTableRenameStmt returns whether the passed-in RenameStmt is one of * the following forms: * * - ALTER TABLE RENAME * - ALTER TABLE RENAME COLUMN * - ALTER TABLE RENAME CONSTRAINT */ bool IsAlterTableRenameStmt(RenameStmt *renameStmt) { bool isAlterTableRenameStmt = false; if (renameStmt->renameType == OBJECT_TABLE || renameStmt->renameType == OBJECT_FOREIGN_TABLE) { isAlterTableRenameStmt = true; } else if (renameStmt->renameType == OBJECT_COLUMN && (renameStmt->relationType == OBJECT_TABLE || renameStmt->relationType == OBJECT_FOREIGN_TABLE)) { isAlterTableRenameStmt = true; } else if (renameStmt->renameType == OBJECT_TABCONSTRAINT) { isAlterTableRenameStmt = true; } return isAlterTableRenameStmt; } /* * ErrorIfAlterDropsPartitionColumn checks if any subcommands of the given alter table * command is a DROP COLUMN command which drops the partition column of a distributed * table. If there is such a subcommand, this function errors out. */ void ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement) { /* first check whether a distributed relation is affected */ if (alterTableStatement->relation == NULL) { return; } LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid leftRelationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!OidIsValid(leftRelationId)) { return; } bool isCitusRelation = IsCitusTable(leftRelationId); if (!isCitusRelation) { return; } /* then check if any of subcommands drop partition column.*/ List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; if (alterTableType == AT_DropColumn) { if (AlterInvolvesPartitionColumn(alterTableStatement, command)) { ereport(ERROR, (errmsg("cannot execute ALTER TABLE command " "dropping partition column"))); } } } } /* * PostprocessAlterTableStmt runs after the ALTER TABLE command has already run * on the master, so we are checking constraints over the table with constraints * already defined (to make the constraint check process same for ALTER TABLE and * CREATE TABLE). If constraints do not fulfill the rules we defined, they will be * removed and the table will return back to the state before the ALTER TABLE command. */ void PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) { LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (relationId != InvalidOid) { /* * check whether we are dealing with a sequence here * if yes, it must be ALTER TABLE .. OWNER TO .. command * since this is the only ALTER command of a sequence that * passes through an AlterTableStmt */ char relKind = get_rel_relkind(relationId); if (relKind == RELKIND_SEQUENCE) { alterTableStatement->objtype = OBJECT_SEQUENCE; PostprocessAlterSequenceOwnerStmt((Node *) alterTableStatement, NULL); return; } else if (relKind == RELKIND_VIEW) { alterTableStatement->objtype = OBJECT_VIEW; PostprocessAlterViewStmt((Node *) alterTableStatement, NULL); return; } /* * Before ensuring each dependency exist, update dependent sequences * types if necessary. */ EnsureRelationHasCompatibleSequenceTypes(relationId); /* changing a relation could introduce new dependencies */ ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*tableAddress, RelationRelationId, relationId); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(tableAddress)); } /* for the new sequences coming with this ALTER TABLE statement */ bool needMetadataSyncForNewSequences = false; char *alterTableDefaultNextvalCmd = NULL; List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; if (alterTableType == AT_AddConstraint) { Assert(list_length(commandList) == 1); ErrorIfUnsupportedAlterAddConstraintStmt(alterTableStatement); if (!OidIsValid(relationId)) { continue; } Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { InvalidateForeignKeyGraph(); } } else if (alterTableType == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; List *columnConstraints = columnDefinition->constraints; if (columnConstraints) { ErrorIfUnsupportedAlterAddConstraintStmt(alterTableStatement); } if (!OidIsValid(relationId)) { continue; } Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->conname == NULL && (constraint->contype == CONSTR_PRIMARY || constraint->contype == CONSTR_UNIQUE || constraint->contype == CONSTR_FOREIGN || constraint->contype == CONSTR_CHECK)) { ErrorUnsupportedAlterTableAddColumn(relationId, command, constraint); } } /* * We check for ADD COLUMN .. DEFAULT expr * if expr contains nextval('user_defined_seq') * we should make sure that the type of the column that uses * that sequence is supported */ constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { if (constraint->raw_expr != NULL) { ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, constraint->raw_expr, EXPR_KIND_COLUMN_DEFAULT); /* * We should make sure that the type of the column that uses * that sequence is supported */ if (contain_nextval_expression_walker(expr, NULL)) { AttrNumber attnum = get_attnum(relationId, columnDefinition->colname); Oid seqOid = GetSequenceOid(relationId, attnum); if (seqOid != InvalidOid) { if (ShouldSyncTableMetadata(relationId)) { needMetadataSyncForNewSequences = true; alterTableDefaultNextvalCmd = GetAddColumnWithNextvalDefaultCmd(seqOid, relationId, columnDefinition ->colname, columnDefinition ->typeName, command-> missing_ok); } } } } } } } /* * We check for ALTER COLUMN .. SET DEFAULT nextval('user_defined_seq') * we should make sure that the type of the column that uses * that sequence is supported */ else if (alterTableType == AT_ColumnDefault) { ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, command->def, EXPR_KIND_COLUMN_DEFAULT); if (contain_nextval_expression_walker(expr, NULL)) { AttrNumber attnum = get_attnum(relationId, command->name); Oid seqOid = GetSequenceOid(relationId, attnum); if (seqOid != InvalidOid) { if (ShouldSyncTableMetadata(relationId)) { needMetadataSyncForNewSequences = true; bool missingTableOk = false; alterTableDefaultNextvalCmd = GetAlterColumnWithNextvalDefaultCmd( seqOid, relationId, command->name, missingTableOk); } } } } } if (needMetadataSyncForNewSequences) { /* prevent recursive propagation */ SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); /* * It's easy to retrieve the sequence id to create the proper commands * in postprocess, after the dependency between the sequence and the table * has been created. We already return ddlJobs in PreprocessAlterTableStmt, * hence we can't return ddlJobs in PostprocessAlterTableStmt. * That's why we execute the following here instead of * in ExecuteDistributedDDLJob */ SendCommandToWorkersWithMetadata(alterTableDefaultNextvalCmd); SendCommandToWorkersWithMetadata(ENABLE_DDL_PROPAGATION); } } /* * FixAlterTableStmtIndexNames runs after the ALTER TABLE command * has already run on the coordinator, and also after the distributed DDL * Jobs have been executed on the workers. * * We might have wrong index names generated on indexes of shards of partitions, * see https://github.com/citusdata/citus/pull/5397 for the details. So we * perform the relevant checks and index renaming here. */ void FixAlterTableStmtIndexNames(AlterTableStmt *alterTableStatement) { LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!(OidIsValid(relationId) && IsCitusTable(relationId) && PartitionedTable(relationId))) { /* we are only interested in partitioned Citus tables */ return; } List *commandList = alterTableStatement->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; /* * If this a partitioned table, and the constraint type uses an index * UNIQUE, PRIMARY KEY, EXCLUDE constraint, * we have wrong index names generated on indexes of shards of * partitions of this table, so we should fix them */ Constraint *constraint = (Constraint *) command->def; if (alterTableType == AT_AddConstraint && ConstrTypeUsesIndex(constraint->contype)) { bool missingOk = false; const char *constraintName = constraint->conname; Oid constraintId = get_relation_constraint_oid(relationId, constraintName, missingOk); /* fix only the relevant index */ Oid parentIndexOid = get_constraint_index(constraintId); FixPartitionShardIndexNames(relationId, parentIndexOid); } /* * If this is an ALTER TABLE .. ATTACH PARTITION command * we have wrong index names generated on indexes of shards of * the current partition being attached, so we should fix them */ else if (alterTableType == AT_AttachPartition) { PartitionCmd *partitionCommand = (PartitionCmd *) command->def; bool partitionMissingOk = false; Oid partitionRelationId = RangeVarGetRelid(partitionCommand->name, lockmode, partitionMissingOk); Oid parentIndexOid = InvalidOid; /* fix all the indexes */ FixPartitionShardIndexNames(partitionRelationId, parentIndexOid); } } } /* * GetSequenceOid returns the oid of the sequence used as default value * of the attribute with given attnum of the given table relationId * If there is no sequence used it returns InvalidOid. */ Oid GetSequenceOid(Oid relationId, AttrNumber attnum) { /* get attrdefoid from the given relationId and attnum */ Oid attrdefOid = get_attrdef_oid(relationId, attnum); /* retrieve the sequence id of the sequence found in nextval('seq') */ List *sequencesFromAttrDef = GetSequencesFromAttrDef(attrdefOid); if (list_length(sequencesFromAttrDef) == 0) { /* * We need this check because sometimes there are cases where the * dependency between the table and the sequence is not formed * One example is when the default is defined by * DEFAULT nextval('seq_name'::text) (not by DEFAULT nextval('seq_name')) * In these cases, sequencesFromAttrDef with be empty. */ return InvalidOid; } if (list_length(sequencesFromAttrDef) > 1) { /* to simplify and eliminate cases like "DEFAULT nextval('..') - nextval('..')" */ ereport(ERROR, (errmsg( "More than one sequence in a column default" " is not supported for distribution " "or for adding local tables to metadata"))); } return lfirst_oid(list_head(sequencesFromAttrDef)); } /* * get_attrdef_oid gets the oid of the attrdef that has dependency with * the given relationId (refobjid) and attnum (refobjsubid). * If there is no such attrdef it returns InvalidOid. * NOTE: we are iterating pg_depend here since this function is used together * with other functions that iterate pg_depend. Normally, a look at pg_attrdef * would make more sense. */ static Oid get_attrdef_oid(Oid relationId, AttrNumber attnum) { Oid resultAttrdefOid = InvalidOid; ScanKeyData key[3]; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); HeapTuple tup; while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if (deprec->classid == AttrDefaultRelationId) { resultAttrdefOid = deprec->objid; } } systable_endscan(scan); table_close(depRel, AccessShareLock); return resultAttrdefOid; } /* * GetAlterColumnWithNextvalDefaultCmd returns a string representing: * ALTER TABLE ALTER COLUMN .. SET DEFAULT nextval() * If sequence type is not bigint, we use worker_nextval() instead of nextval(). */ char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, bool missingTableOk) { char *qualifiedSequenceName = generate_qualified_relation_name(sequenceOid); char *qualifiedRelationName = generate_qualified_relation_name(relationId); char *nextvalFunctionName = "nextval"; bool useWorkerNextval = (pg_get_sequencedef(sequenceOid)->seqtypid != INT8OID); if (useWorkerNextval) { /* * We use worker_nextval for int and smallint types. * Check issue #5126 and PR #5254 for details. * https://github.com/citusdata/citus/issues/5126 */ nextvalFunctionName = "worker_nextval"; } StringInfoData str = { 0 }; initStringInfo(&str); appendStringInfo(&str, "ALTER TABLE "); if (missingTableOk) { appendStringInfo(&str, "IF EXISTS "); } appendStringInfo(&str, "%s ALTER COLUMN %s " "SET DEFAULT %s(%s::regclass)", qualifiedRelationName, colname, quote_qualified_identifier("pg_catalog", nextvalFunctionName), quote_literal_cstr(qualifiedSequenceName)); return str.data; } /* * GetAddColumnWithNextvalDefaultCmd returns a string representing: * ALTER TABLE ADD COLUMN .. DEFAULT nextval() * If sequence type is not bigint, we use worker_nextval() instead of nextval(). */ static char * GetAddColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, TypeName *typeName, bool ifNotExists) { char *qualifiedSequenceName = generate_qualified_relation_name(sequenceOid); char *qualifiedRelationName = generate_qualified_relation_name(relationId); char *nextvalFunctionName = "nextval"; bool useWorkerNextval = (pg_get_sequencedef(sequenceOid)->seqtypid != INT8OID); if (useWorkerNextval) { /* * We use worker_nextval for int and smallint types. * Check issue #5126 and PR #5254 for details. * https://github.com/citusdata/citus/issues/5126 */ nextvalFunctionName = "worker_nextval"; } int32 typmod = 0; Oid typeOid = InvalidOid; bits16 formatFlags = FORMAT_TYPE_TYPEMOD_GIVEN | FORMAT_TYPE_FORCE_QUALIFY; typenameTypeIdAndMod(NULL, typeName, &typeOid, &typmod); StringInfoData str = { 0 }; initStringInfo(&str); appendStringInfo(&str, "ALTER TABLE %s ADD COLUMN %s %s %s " "DEFAULT %s(%s::regclass)", qualifiedRelationName, ifNotExists ? "IF NOT EXISTS" : "", colname, format_type_extended(typeOid, typmod, formatFlags), quote_qualified_identifier("pg_catalog", nextvalFunctionName), quote_literal_cstr(qualifiedSequenceName)); return str.data; } void ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *command, Constraint *constraint) { ColumnDef *columnDefinition = (ColumnDef *) command->def; char *colName = columnDefinition->colname; char *errMsg = "cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints"; StringInfo errHint = makeStringInfo(); appendStringInfo(errHint, "You can issue each command separately such as "); appendStringInfo(errHint, "ALTER TABLE %s ADD COLUMN %s data_type; ALTER TABLE %s ADD CONSTRAINT constraint_name ", get_rel_name(relationId), colName, get_rel_name(relationId)); if (constraint->contype == CONSTR_UNIQUE) { appendStringInfo(errHint, "UNIQUE (%s)", colName); } else if (constraint->contype == CONSTR_PRIMARY) { appendStringInfo(errHint, "PRIMARY KEY (%s)", colName); } else if (constraint->contype == CONSTR_CHECK) { appendStringInfo(errHint, "CHECK (check_expression)"); } else if (constraint->contype == CONSTR_FOREIGN) { RangeVar *referencedTable = constraint->pktable; Oid referencedRelationId = RangeVarGetRelid(referencedTable, NoLock, false); appendStringInfo(errHint, "FOREIGN KEY (%s) REFERENCES %s", colName, get_rel_name(referencedRelationId)); if (list_length(constraint->pk_attrs) > 0) { AppendColumnNameList(errHint, constraint->pk_attrs); } if (constraint->fk_del_action == FKCONSTR_ACTION_SETNULL) { appendStringInfo(errHint, " %s", "ON DELETE SET NULL"); } else if (constraint->fk_del_action == FKCONSTR_ACTION_CASCADE) { appendStringInfo(errHint, " %s", "ON DELETE CASCADE"); } else if (constraint->fk_del_action == FKCONSTR_ACTION_SETDEFAULT) { appendStringInfo(errHint, " %s", "ON DELETE SET DEFAULT"); } else if (constraint->fk_del_action == FKCONSTR_ACTION_RESTRICT) { appendStringInfo(errHint, " %s", "ON DELETE RESTRICT"); } if (constraint->fk_upd_action == FKCONSTR_ACTION_SETNULL) { appendStringInfo(errHint, " %s", "ON UPDATE SET NULL"); } else if (constraint->fk_upd_action == FKCONSTR_ACTION_CASCADE) { appendStringInfo(errHint, " %s", "ON UPDATE CASCADE"); } else if (constraint->fk_upd_action == FKCONSTR_ACTION_SETDEFAULT) { appendStringInfo(errHint, " %s", "ON UPDATE SET DEFAULT"); } else if (constraint->fk_upd_action == FKCONSTR_ACTION_RESTRICT) { appendStringInfo(errHint, " %s", "ON UPDATE RESTRICT"); } } appendStringInfo(errHint, "%s", ";"); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s", errMsg), errhint("%s", errHint->data), errdetail("Adding a column with a constraint in " "one command is not supported because " "all constraints in Citus must have " "explicit names"))); } /* * ErrorIfUnsupportedConstraint runs checks related to unique index / exclude * constraints. * * The function skips the uniqeness checks for reference tables (i.e., distribution * method is 'none'). * * Forbid UNIQUE, PRIMARY KEY, or EXCLUDE constraints on append partitioned * tables, since currently there is no way of enforcing uniqueness for * overlapping shards. * * Similarly, do not allow such constraints if they do not include partition * column. This check is important for two reasons: * i. First, currently Citus does not enforce uniqueness constraint on multiple * shards. * ii. Second, INSERT INTO .. ON CONFLICT (i.e., UPSERT) queries can be executed * with no further check for constraints. */ void ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, char referencingReplicationModel, Var *distributionColumn, uint32 colocationId) { /* * We first perform check for foreign constraints. It is important to do this * check before next check, because other types of constraints are allowed on * reference tables and we return early for those constraints thanks to next * check. Therefore, for reference tables, we first check for foreign constraints * and if they are OK, we do not error out for other types of constraints. */ ErrorIfUnsupportedForeignConstraintExists(relation, distributionMethod, referencingReplicationModel, distributionColumn, colocationId); /* * Citus supports any kind of uniqueness constraints for reference tables * given that they only consist of a single shard and we can simply rely on * Postgres. */ if (distributionMethod == DISTRIBUTE_BY_NONE) { return; } if (distributionColumn == NULL) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("distribution column of distributed table is NULL"))); } char *relationName = RelationGetRelationName(relation); List *indexOidList = RelationGetIndexList(relation); Oid indexOid = InvalidOid; foreach_declared_oid(indexOid, indexOidList) { Relation indexDesc = index_open(indexOid, RowExclusiveLock); bool hasDistributionColumn = false; /* extract index key information from the index's pg_index info */ IndexInfo *indexInfo = BuildIndexInfo(indexDesc); /* only check unique indexes and exclusion constraints. */ if (indexInfo->ii_Unique == false && indexInfo->ii_ExclusionOps == NULL) { index_close(indexDesc, NoLock); continue; } /* * Citus cannot enforce uniqueness/exclusion constraints with overlapping shards. * Thus, emit a warning for unique indexes and exclusion constraints on * append partitioned tables. */ if (distributionMethod == DISTRIBUTE_BY_APPEND) { ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table \"%s\" has a UNIQUE or EXCLUDE constraint", relationName), errdetail("UNIQUE constraints, EXCLUDE constraints, " "and PRIMARY KEYs on " "append-partitioned tables cannot be enforced."), errhint("Consider using hash partitioning."))); } if (AllowUnsafeConstraints) { /* * The user explicitly wants to allow the constraint without * distribution column. */ index_close(indexDesc, NoLock); continue; } int attributeCount = indexInfo->ii_NumIndexAttrs; AttrNumber *attributeNumberArray = indexInfo->ii_IndexAttrNumbers; for (int attributeIndex = 0; attributeIndex < attributeCount; attributeIndex++) { AttrNumber attributeNumber = attributeNumberArray[attributeIndex]; if (distributionColumn->varattno != attributeNumber) { continue; } bool uniqueConstraint = indexInfo->ii_Unique; bool exclusionConstraintWithEquality = (indexInfo->ii_ExclusionOps != NULL && OperatorImplementsEquality( indexInfo->ii_ExclusionOps[ attributeIndex])); if (uniqueConstraint || exclusionConstraintWithEquality) { hasDistributionColumn = true; break; } } if (!hasDistributionColumn) { ereport(ERROR, ( errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create constraint on \"%s\"", relationName), errdetail("Distributed relations cannot have UNIQUE, " "EXCLUDE, or PRIMARY KEY constraints that do not " "include the partition column (with an equality " "operator if EXCLUDE)."))); } index_close(indexDesc, NoLock); } } /* * ErrorIfAlterTableDropTableNameFromPostgresFdw errors if given alter foreign table * option list drops 'table_name' from a postgresfdw foreign table which is * inside metadata. */ static void ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid relationId) { char relationKind PG_USED_FOR_ASSERTS_ONLY = get_rel_relkind(relationId); Assert(relationKind == RELKIND_FOREIGN_TABLE); ForeignTable *foreignTable = GetForeignTable(relationId); Oid serverId = foreignTable->serverid; if (!ServerUsesPostgresFdw(serverId)) { return; } if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE) && ForeignTableDropsTableNameOption(optionList)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "alter foreign table alter options (drop table_name) command " "is not allowed for Citus tables"), errdetail( "Table_name option can not be dropped from a foreign table " "which is inside metadata."), errhint( "Try to undistribute foreign table before dropping table_name option."))); } } /* * ErrorIfUnsupportedAlterTableStmt checks if the corresponding alter table * statement is supported for distributed tables and errors out if it is not. * Currently, only the following commands are supported. * * ALTER TABLE ADD|DROP COLUMN * ALTER TABLE ALTER COLUMN SET DATA TYPE * ALTER TABLE SET|DROP NOT NULL * ALTER TABLE SET|DROP DEFAULT * ALTER TABLE ADD|DROP CONSTRAINT * ALTER TABLE REPLICA IDENTITY * ALTER TABLE SET () * ALTER TABLE ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY * ALTER TABLE RESET () * ALTER TABLE ENABLE/DISABLE TRIGGER (if enable_unsafe_triggers is not set, we only support triggers for citus local tables) */ static void ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) { List *commandList = alterTableStatement->cmds; LOCKMODE lockmode = AlterTableGetLockLevel(commandList); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); /* error out if any of the subcommands are unsupported */ AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { AlterTableType alterTableType = command->subtype; switch (alterTableType) { case AT_AddColumn: { if (IsA(command->def, ColumnDef)) { ColumnDef *column = (ColumnDef *) command->def; /* * Check for SERIAL pseudo-types. The structure of this * check is copied from transformColumnDefinition. */ if (column->typeName && list_length(column->typeName->names) == 1 && !column->typeName->pct_type) { char *typeName = strVal(linitial(column->typeName->names)); if (strcmp(typeName, "smallserial") == 0 || strcmp(typeName, "serial2") == 0 || strcmp(typeName, "serial") == 0 || strcmp(typeName, "serial4") == 0 || strcmp(typeName, "bigserial") == 0 || strcmp(typeName, "serial8") == 0) { /* * We currently don't support adding a serial column for an MX table * TODO: record the dependency in the workers */ if (ShouldSyncTableMetadata(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot execute ADD COLUMN commands involving serial" " pseudotypes when metadata is synchronized to workers"))); } /* * we only allow adding a serial column if it is the only subcommand * and it has no constraints */ if (commandList->length > 1 || column->constraints) { ereport(ERROR, (errcode( ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot execute ADD COLUMN commands involving " "serial pseudotypes with other subcommands/constraints"), errhint( "You can issue each subcommand separately"))); } /* * Currently we don't support backfilling the new column with default values * if the table is not empty */ if (!TableEmpty(relationId)) { ereport(ERROR, (errcode( ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "Cannot add a column involving serial pseudotypes " "because the table is not empty"), errhint( "You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint\n" "Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..')"))); } } } Constraint *columnConstraint = NULL; foreach_declared_ptr(columnConstraint, column->constraints) { if (columnConstraint->contype == CONSTR_IDENTITY) { /* * We currently don't support adding an identity column for an MX table */ if (ShouldSyncTableMetadata(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot execute ADD COLUMN commands involving identity" " columns when metadata is synchronized to workers"))); } /* * Currently we don't support backfilling the new identity column with default values * if the table is not empty */ if (!TableEmpty(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "Cannot add an identity column because the table is not empty"))); } } } List *columnConstraints = column->constraints; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_DEFAULT) { if (constraint->raw_expr != NULL) { ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, constraint->raw_expr, EXPR_KIND_COLUMN_DEFAULT); if (contain_nextval_expression_walker(expr, NULL)) { /* * we only allow adding a column with non_const default * if its the only subcommand and has no other constraints */ if (commandList->length > 1 || columnConstraints->length > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot execute ADD COLUMN .. DEFAULT nextval('..')" " command with other subcommands/constraints"), errhint( "You can issue each subcommand separately"))); } /* * Currently we don't support backfilling the new column with default values * if the table is not empty */ if (!TableEmpty(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot add a column involving DEFAULT nextval('..') " "because the table is not empty"), errhint( "You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint\n" "Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..')"))); } } } } } } break; } case AT_ColumnDefault: { if (AlterInvolvesPartitionColumn(alterTableStatement, command)) { ereport(ERROR, (errmsg("cannot execute ALTER TABLE command " "involving partition column"))); } ParseState *pstate = make_parsestate(NULL); Node *expr = transformExpr(pstate, command->def, EXPR_KIND_COLUMN_DEFAULT); if (contain_nextval_expression_walker(expr, NULL)) { /* * we only allow altering a column's default to non_const expr * if its the only subcommand */ if (commandList->length > 1) { ereport(ERROR, (errcode( ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot execute ALTER COLUMN COLUMN .. SET DEFAULT " "nextval('..') command with other subcommands"), errhint( "You can issue each subcommand separately"))); } } break; } case AT_AlterColumnType: { if (AlterInvolvesPartitionColumn(alterTableStatement, command)) { ereport(ERROR, (errmsg("cannot execute ALTER TABLE command " "involving partition column"))); } /* * We check for ALTER COLUMN TYPE ... * if the column is an identity column, * changing the type of the column * should not be allowed for now */ if (AlterColumnInvolvesIdentityColumn(alterTableStatement, command)) { ereport(ERROR, (errmsg("cannot execute ALTER COLUMN command " "involving identity column"))); } /* * We check for ALTER COLUMN TYPE ... * if the column has default coming from a user-defined sequence * changing the type of the column * should not be allowed for now */ AttrNumber attnum = get_attnum(relationId, command->name); List *seqInfoList = NIL; GetDependentSequencesWithRelation(relationId, &seqInfoList, attnum, DEPENDENCY_AUTO); if (seqInfoList != NIL) { ereport(ERROR, (errmsg("cannot execute ALTER COLUMN TYPE .. command " "because the column involves a default coming " "from a sequence"))); } break; } case AT_DropColumn: case AT_DropNotNull: { if (AlterInvolvesPartitionColumn(alterTableStatement, command)) { ereport(ERROR, (errmsg("cannot execute ALTER TABLE command " "involving partition column"))); } break; } case AT_AddConstraint: { /* we only allow constraints if they are only subcommand */ if (commandList->length > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot execute ADD CONSTRAINT command with " "other subcommands"), errhint("You can issue each subcommand separately"))); } break; } case AT_AttachPartition: { PartitionCmd *partitionCommand = (PartitionCmd *) command->def; bool missingOK = false; Oid partitionRelationId = RangeVarGetRelid(partitionCommand->name, lockmode, missingOK); /* we only allow partitioning commands if they are only subcommand */ if (commandList->length > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot execute ATTACH PARTITION command " "with other subcommands"), errhint("You can issue each subcommand " "separately."))); } if (IsCitusTableType(partitionRelationId, CITUS_LOCAL_TABLE) || IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * Citus Local Tables cannot be colocated with other tables. * If either of two relations is not a Citus Local Table, then we * don't need to check colocation since CreateCitusLocalTable would * anyway throw an error. */ break; } if (IsCitusTable(partitionRelationId) && !TablesColocated(relationId, partitionRelationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("distributed tables cannot have " "non-colocated distributed tables as a " "partition "))); } break; } case AT_DetachPartitionFinalize: { ereport(ERROR, (errmsg("ALTER TABLE .. DETACH PARTITION .. FINALIZE " "commands are currently unsupported."))); break; } case AT_DetachPartition: { /* we only allow partitioning commands if they are only subcommand */ if (commandList->length > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot execute DETACH PARTITION command " "with other subcommands"), errhint("You can issue each subcommand " "separately."))); } PartitionCmd *partitionCommand = (PartitionCmd *) command->def; if (partitionCommand->concurrent) { ereport(ERROR, (errmsg("ALTER TABLE .. DETACH PARTITION .. " "CONCURRENTLY commands are currently " "unsupported."))); } break; } case AT_EnableTrig: case AT_EnableAlwaysTrig: case AT_EnableReplicaTrig: case AT_EnableTrigUser: case AT_DisableTrig: case AT_DisableTrigUser: case AT_EnableTrigAll: case AT_DisableTrigAll: { /* * Postgres already does not allow executing ALTER TABLE TRIGGER * commands with other subcommands, but let's be on the safe side. */ if (commandList->length > 1) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cannot execute ENABLE/DISABLE TRIGGER " "command with other subcommands"), errhint("You can issue each subcommand separately"))); } ErrorOutForTriggerIfNotSupported(relationId); break; } #if PG_VERSION_NUM >= PG_VERSION_17 case AT_SetExpression: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "ALTER TABLE ... ALTER COLUMN ... SET EXPRESSION commands " "are currently unsupported."))); break; } #endif case AT_SetAccessMethod: { /* * If command->name == NULL, that means the user is trying to use * ALTER TABLE ... SET ACCESS METHOD DEFAULT * which we don't support currently. */ if (command->name == NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "DEFAULT option in ALTER TABLE ... SET ACCESS METHOD " "is currently unsupported."), errhint( "You can rerun the command by explicitly writing the access method name."))); } break; } case AT_SetNotNull: case AT_ReplicaIdentity: case AT_ChangeOwner: case AT_EnableRowSecurity: case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: case AT_ValidateConstraint: case AT_DropConstraint: /* we do the check for invalidation in AlterTableDropsForeignKey */ case AT_SetCompression: { /* * We will not perform any special check for: * ALTER TABLE .. SET ACCESS METHOD .. * ALTER TABLE .. ALTER COLUMN .. SET NOT NULL * ALTER TABLE .. REPLICA IDENTITY .. * ALTER TABLE .. VALIDATE CONSTRAINT .. * ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. */ break; } case AT_SetRelOptions: /* SET (...) */ case AT_ResetRelOptions: /* RESET (...) */ case AT_ReplaceRelOptions: /* replace entire option list */ case AT_SetLogged: case AT_SetUnLogged: { /* this command is supported by Citus */ break; } case AT_GenericOptions: { if (IsForeignTable(relationId)) { List *optionList = (List *) command->def; ErrorIfAlterTableDropTableNameFromPostgresFdw(optionList, relationId); break; } } /* fallthrough */ default: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("alter table command is currently unsupported"), errdetail("Only ADD|DROP COLUMN, SET|DROP NOT NULL, " "SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, " "SET (), RESET (), " "ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, " "ATTACH|DETACH PARTITION and TYPE subcommands " "are supported."))); } } } } /* * SetupExecutionModeForAlterTable is the function that is responsible * for two things for practical purpose for not doing the same checks * twice: * (a) For any command, decide and return whether we should * run the command in sequential mode * (b) For commands in a transaction block, set the transaction local * multi-shard modify mode to sequential when necessary * * The commands that operate on the same reference table shard in parallel * is in the interest of (a), where the return value indicates the executor * to run the command sequentially to prevent self-deadlocks. * * The commands that both operate on the same reference table shard in parallel * and cascades to run any parallel operation is in the interest of (b). By * setting the multi-shard mode, we ensure that the cascading parallel commands * are executed sequentially to prevent self-deadlocks. * * One final note on the function is that if the function decides to execute * the command in sequential mode, and a parallel command has already been * executed in the same transaction, the function errors out. See the comment * in the function for the rationale. */ static bool SetupExecutionModeForAlterTable(Oid relationId, AlterTableCmd *command) { bool executeSequentially = false; AlterTableType alterTableType = command->subtype; if (alterTableType == AT_DropConstraint) { char *constraintName = command->name; if (ConstraintIsAForeignKeyToReferenceTable(constraintName, relationId)) { executeSequentially = true; } } else if (alterTableType == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { Oid rightRelationId = RangeVarGetRelid(constraint->pktable, NoLock, false); if (IsCitusTableType(rightRelationId, REFERENCE_TABLE)) { executeSequentially = true; } } } } else if (alterTableType == AT_DropColumn || alterTableType == AT_AlterColumnType) { char *affectedColumnName = command->name; if (ColumnAppearsInForeignKeyToReferenceTable(affectedColumnName, relationId)) { if (alterTableType == AT_AlterColumnType) { SetLocalMultiShardModifyModeToSequential(); } executeSequentially = true; } } else if (alterTableType == AT_AddConstraint) { /* * We need to execute the ddls working with reference tables on the * right side sequentially, because parallel ddl operations * relating to one and only shard of a reference table on a worker * may cause self-deadlocks. */ Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { Oid rightRelationId = RangeVarGetRelid(constraint->pktable, NoLock, false); if (IsCitusTableType(rightRelationId, REFERENCE_TABLE)) { executeSequentially = true; } } } else if (alterTableType == AT_DetachPartition || alterTableType == AT_AttachPartition) { /* check if there are foreign constraints to reference tables */ if (HasForeignKeyToReferenceTable(relationId)) { executeSequentially = true; } } /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers for * the distributed tables, thus contradicting our purpose of using * sequential mode. */ if (executeSequentially && HasDistributionKey(relationId) && ParallelQueryExecutedInTransaction()) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errmsg("cannot modify table \"%s\" because there " "was a parallel operation on a distributed table " "in the transaction", relationName), errdetail("When there is a foreign key to a reference " "table, Citus needs to perform all operations " "over a single connection per node to ensure " "consistency."), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } return executeSequentially; } /* * InterShardDDLTaskList builds a list of tasks to execute a inter shard DDL command on a * shards of given list of distributed table. At the moment this function is used to run * foreign key, partitioning and attaching partition index command on worker node. * * leftRelationId is the relation id of actual distributed table which given command is * applied. rightRelationId is the relation id of either index or distributed table which * given command refers to. */ List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, const char *commandString) { List *leftShardList = LoadShardIntervalList(leftRelationId); List *rightShardList = CreateRightShardListForInterShardDDLTask(rightRelationId, leftRelationId, leftShardList); /* lock metadata before getting placement lists */ LockShardListMetadata(leftShardList, ShareLock); uint64 jobId = INVALID_JOB_ID; int taskId = 1; Oid leftSchemaId = get_rel_namespace(leftRelationId); char *leftSchemaName = get_namespace_name(leftSchemaId); char *escapedLeftSchemaName = quote_literal_cstr(leftSchemaName); Oid rightSchemaId = get_rel_namespace(rightRelationId); char *rightSchemaName = get_namespace_name(rightSchemaId); char *escapedRightSchemaName = quote_literal_cstr(rightSchemaName); char *escapedCommandString = quote_literal_cstr(commandString); List *taskList = NIL; ShardInterval *leftShardInterval = NULL; ShardInterval *rightShardInterval = NULL; forboth_ptr(leftShardInterval, leftShardList, rightShardInterval, rightShardList) { uint64 leftShardId = leftShardInterval->shardId; uint64 rightShardId = rightShardInterval->shardId; StringInfo applyCommand = makeStringInfo(); appendStringInfo(applyCommand, WORKER_APPLY_INTER_SHARD_DDL_COMMAND, leftShardId, escapedLeftSchemaName, rightShardId, escapedRightSchemaName, escapedCommandString); Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, applyCommand->data); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = leftShardId; SetInterShardDDLTaskPlacementList(task, leftShardInterval, rightShardInterval); SetInterShardDDLTaskRelationShardList(task, leftShardInterval, rightShardInterval); taskList = lappend(taskList, task); } return taskList; } /* * CreateRightShardListForInterShardDDLTask is a helper function that creates * shard list for the right relation for InterShardDDLTaskList. */ static List * CreateRightShardListForInterShardDDLTask(Oid rightRelationId, Oid leftRelationId, List *leftShardList) { List *rightShardList = LoadShardIntervalList(rightRelationId); if (!IsCitusTableType(leftRelationId, CITUS_LOCAL_TABLE) && IsCitusTableType(rightRelationId, REFERENCE_TABLE)) { /* * If the right relation is a reference table and left relation is not * a citus local table, we need to make sure that the tasks are created * in a way that the right shard stays the same since we only have one * placement per worker. * If left relation is a citus local table, then we don't need to populate * reference table shards as we will execute ADD/DROP constraint command * only for coordinator placement of reference table. */ ShardInterval *rightShard = (ShardInterval *) linitial(rightShardList); int leftShardCount = list_length(leftShardList); rightShardList = GenerateListFromElement(rightShard, leftShardCount); } return rightShardList; } /* * SetInterShardDDLTaskPlacementList sets taskPlacementList field of given * inter-shard DDL task according to passed shard interval arguments. */ static void SetInterShardDDLTaskPlacementList(Task *task, ShardInterval *leftShardInterval, ShardInterval *rightShardInterval) { uint64 leftShardId = leftShardInterval->shardId; List *leftShardPlacementList = ActiveShardPlacementList(leftShardId); uint64 rightShardId = rightShardInterval->shardId; List *rightShardPlacementList = ActiveShardPlacementList(rightShardId); List *intersectedPlacementList = NIL; ShardPlacement *leftShardPlacement = NULL; foreach_declared_ptr(leftShardPlacement, leftShardPlacementList) { ShardPlacement *rightShardPlacement = NULL; foreach_declared_ptr(rightShardPlacement, rightShardPlacementList) { if (leftShardPlacement->nodeId == rightShardPlacement->nodeId) { intersectedPlacementList = lappend(intersectedPlacementList, leftShardPlacement); } } } task->taskPlacementList = intersectedPlacementList; } /* * SetInterShardDDLTaskRelationShardList sets relationShardList field of given * inter-shard DDL task according to passed shard interval arguments. */ static void SetInterShardDDLTaskRelationShardList(Task *task, ShardInterval *leftShardInterval, ShardInterval *rightShardInterval) { RelationShard *leftRelationShard = CitusMakeNode(RelationShard); leftRelationShard->relationId = leftShardInterval->relationId; leftRelationShard->shardId = leftShardInterval->shardId; RelationShard *rightRelationShard = CitusMakeNode(RelationShard); rightRelationShard->relationId = rightShardInterval->relationId; rightRelationShard->shardId = rightShardInterval->shardId; task->relationShardList = list_make2(leftRelationShard, rightRelationShard); } /* * AlterColumnInvolvesIdentityColumn checks if the given alter column command * involves relation's identity column. */ static bool AlterColumnInvolvesIdentityColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *command) { bool involvesIdentityColumn = false; char *alterColumnName = command->name; LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!OidIsValid(relationId)) { return false; } HeapTuple tuple = SearchSysCacheAttName(relationId, alterColumnName); if (HeapTupleIsValid(tuple)) { Form_pg_attribute targetAttr = (Form_pg_attribute) GETSTRUCT(tuple); if (targetAttr->attidentity) { involvesIdentityColumn = true; } ReleaseSysCache(tuple); } return involvesIdentityColumn; } /* * AlterInvolvesPartitionColumn checks if the given alter table command * involves relation's partition column. */ static bool AlterInvolvesPartitionColumn(AlterTableStmt *alterTableStatement, AlterTableCmd *command) { bool involvesPartitionColumn = false; char *alterColumnName = command->name; LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); if (!OidIsValid(relationId)) { return false; } Var *partitionColumn = DistPartitionKey(relationId); HeapTuple tuple = SearchSysCacheAttName(relationId, alterColumnName); if (HeapTupleIsValid(tuple)) { Form_pg_attribute targetAttr = (Form_pg_attribute) GETSTRUCT(tuple); /* reference tables do not have partition column, so allow them */ if (partitionColumn != NULL && targetAttr->attnum == partitionColumn->varattno) { involvesPartitionColumn = true; } ReleaseSysCache(tuple); } return involvesPartitionColumn; } /* * ErrorIfUnsupportedAlterAddConstraintStmt runs the constraint checks on distributed * table using the same logic with create_distributed_table. */ static void ErrorIfUnsupportedAlterAddConstraintStmt(AlterTableStmt *alterTableStatement) { LOCKMODE lockmode = AlterTableGetLockLevel(alterTableStatement->cmds); Oid relationId = AlterTableLookupRelation(alterTableStatement, lockmode); char distributionMethod = PartitionMethod(relationId); char referencingReplicationModel = TableReplicationModel(relationId); Var *distributionColumn = DistPartitionKey(relationId); uint32 colocationId = TableColocationId(relationId); Relation relation = relation_open(relationId, ExclusiveLock); ErrorIfUnsupportedConstraint(relation, distributionMethod, referencingReplicationModel, distributionColumn, colocationId); relation_close(relation, NoLock); } /* * AlterTableSchemaStmtObjectAddress returns the ObjectAddress of the table that * is the object of the AlterObjectSchemaStmt. * * This could be called both before or after it has been applied locally. It will * look in the old schema first, if the table cannot be found in that schema it * will look in the new schema. Errors if missing_ok is false and the table cannot * be found in either of the schemas. */ List * AlterTableSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); const char *tableName = stmt->relation->relname; Oid tableOid = InvalidOid; if (stmt->relation->schemaname) { const char *schemaName = stmt->relation->schemaname; Oid schemaOid = get_namespace_oid(schemaName, missing_ok); tableOid = get_relname_relid(tableName, schemaOid); } else { tableOid = RelnameGetRelid(stmt->relation->relname); } if (tableOid == InvalidOid) { const char *newSchemaName = stmt->newschema; Oid newSchemaOid = get_namespace_oid(newSchemaName, true); tableOid = get_relname_relid(tableName, newSchemaOid); if (!missing_ok && tableOid == InvalidOid) { const char *quotedTableName = quote_qualified_identifier(stmt->relation->schemaname, tableName); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", quotedTableName))); } } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, RelationRelationId, tableOid); return list_make1(address); } /* * MakeNameListFromRangeVar makes a namelist from a RangeVar. Its behaviour * should be the exact opposite of postgres' makeRangeVarFromNameList. */ List * MakeNameListFromRangeVar(const RangeVar *rel) { if (rel->catalogname != NULL) { Assert(rel->schemaname != NULL); Assert(rel->relname != NULL); return list_make3(makeString(rel->catalogname), makeString(rel->schemaname), makeString(rel->relname)); } else if (rel->schemaname != NULL) { Assert(rel->relname != NULL); return list_make2(makeString(rel->schemaname), makeString(rel->relname)); } else { Assert(rel->relname != NULL); return list_make1(makeString(rel->relname)); } } /* * ErrorIfTableHasIdentityColumn errors out if the given table has identity column */ void ErrorIfTableHasIdentityColumn(Oid relationId) { Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); if (attributeForm->attidentity) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot complete operation on a table with identity column"))); } } relation_close(relation, NoLock); } /* * ConvertNewTableIfNecessary converts the given table to a tenant schema * table or a Citus managed table if necessary. * * Input node is expected to be a CreateStmt or a CreateTableAsStmt. */ void ConvertNewTableIfNecessary(Node *createStmt) { /* * Need to increment command counter so that next command * can see the new table. */ CommandCounterIncrement(); if (IsA(createStmt, CreateTableAsStmt)) { CreateTableAsStmt *createTableAsStmt = (CreateTableAsStmt *) createStmt; bool missingOk = false; Oid createdRelationId = RangeVarGetRelid(createTableAsStmt->into->rel, NoLock, missingOk); if (ShouldCreateTenantSchemaTable(createdRelationId)) { /* not try to convert the table if it already exists and IF NOT EXISTS syntax is used */ if (createTableAsStmt->if_not_exists && IsCitusTable(createdRelationId)) { return; } /* * We allow mat views in a distributed schema but do not make them a tenant * table. We should skip converting them. */ if (get_rel_relkind(createdRelationId) == RELKIND_MATVIEW) { return; } CreateTenantSchemaTable(createdRelationId); } /* * We simply ignore the tables created by using that syntax when using * Citus managed tables. */ return; } CreateStmt *baseCreateTableStmt = (CreateStmt *) createStmt; bool missingOk = false; Oid createdRelationId = RangeVarGetRelid(baseCreateTableStmt->relation, NoLock, missingOk); /* not try to convert the table if it already exists and IF NOT EXISTS syntax is used */ if (baseCreateTableStmt->if_not_exists && IsCitusTable(createdRelationId)) { return; } /* * Check ShouldCreateTenantSchemaTable() before ShouldAddNewTableToMetadata() * because we don't want to unnecessarily add the table into metadata * (as a Citus managed table) before distributing it as a tenant table. */ if (ShouldCreateTenantSchemaTable(createdRelationId)) { /* * We skip creating tenant schema table if the table is a partition * table because in that case PostprocessCreateTableStmt() should've * already created a tenant schema table from the partition table. */ if (!PartitionTable(createdRelationId)) { CreateTenantSchemaTable(createdRelationId); } } else if (ShouldAddNewTableToMetadata(createdRelationId)) { /* * Here we set autoConverted to false, since the user explicitly * wants these tables to be added to metadata, by setting the * GUC use_citus_managed_tables to true. */ bool autoConverted = false; bool cascade = true; CreateCitusLocalTable(createdRelationId, cascade, autoConverted); } } /* * ConvertToTenantTableIfNecessary converts given relation to a tenant table if its * schema changed to a distributed schema. */ void ConvertToTenantTableIfNecessary(AlterObjectSchemaStmt *stmt) { Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); if (!IsCoordinator()) { return; } /* * We will let Postgres deal with missing_ok */ List *tableAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, true); /* the code-path only supports a single object */ Assert(list_length(tableAddresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *tableAddress = linitial(tableAddresses); char relKind = get_rel_relkind(tableAddress->objectId); if (relKind == RELKIND_SEQUENCE || relKind == RELKIND_VIEW) { return; } Oid relationId = tableAddress->objectId; Oid schemaId = get_namespace_oid(stmt->newschema, stmt->missing_ok); if (!OidIsValid(schemaId)) { return; } /* * Make table a tenant table when its schema actually changed. When its schema * is not changed as in `ALTER TABLE SET SCHEMA `, we detect * that by seeing the table is still a single shard table. (i.e. not undistributed * at `preprocess` step) */ if (!IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED) && IsTenantSchema(schemaId)) { EnsureTenantTable(relationId, "ALTER TABLE SET SCHEMA"); char *schemaName = get_namespace_name(schemaId); char *tableName = stmt->relation->relname; ereport(NOTICE, (errmsg("Moving %s into distributed schema %s", tableName, schemaName))); CreateTenantSchemaTable(relationId); } } ================================================ FILE: src/backend/distributed/commands/text_search.c ================================================ /*------------------------------------------------------------------------- * * text_search.c * Commands for creating and altering TEXT SEARCH objects * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "access/genam.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/objectaddress.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_config_map.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_parser.h" #include "catalog/pg_ts_template.h" #include "commands/comment.h" #include "commands/defrem.h" #include "commands/extension.h" #include "nodes/makefuncs.h" #include "tsearch/ts_cache.h" #include "tsearch/ts_public.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_create_or_replace.h" static DefineStmt * GetTextSearchConfigDefineStmt(Oid tsconfigOid); static DefineStmt * GetTextSearchDictionaryDefineStmt(Oid tsdictOid); static List * GetTextSearchDictionaryInitOptions(HeapTuple tup, Form_pg_ts_dict dict); static List * GetTextSearchConfigCommentStmt(Oid tsconfigOid); static List * GetTextSearchDictionaryCommentStmt(Oid tsconfigOid); static List * get_ts_parser_namelist(Oid tsparserOid); static List * GetTextSearchConfigMappingStmt(Oid tsconfigOid); static List * GetTextSearchConfigOwnerStmts(Oid tsconfigOid); static List * GetTextSearchDictionaryOwnerStmts(Oid tsdictOid); static List * get_ts_dict_namelist(Oid tsdictOid); static List * get_ts_template_namelist(Oid tstemplateOid); static Oid get_ts_config_parser_oid(Oid tsconfigOid); static char * get_ts_parser_tokentype_name(Oid parserOid, int32 tokentype); List * GetCreateTextSearchConfigStatements(const ObjectAddress *address) { Assert(address->classId == TSConfigRelationId); List *stmts = NIL; /* CREATE TEXT SEARCH CONFIGURATION ...*/ stmts = lappend(stmts, GetTextSearchConfigDefineStmt(address->objectId)); /* ALTER TEXT SEARCH CONFIGURATION ... OWNER TO ...*/ stmts = list_concat(stmts, GetTextSearchConfigOwnerStmts(address->objectId)); /* COMMENT ON TEXT SEARCH CONFIGURATION ... */ stmts = list_concat(stmts, GetTextSearchConfigCommentStmt(address->objectId)); /* ALTER TEXT SEARCH CONFIGURATION ... ADD MAPPING FOR ... WITH ... */ stmts = list_concat(stmts, GetTextSearchConfigMappingStmt(address->objectId)); return stmts; } List * GetCreateTextSearchDictionaryStatements(const ObjectAddress *address) { Assert(address->classId == TSDictionaryRelationId); List *stmts = NIL; /* CREATE TEXT SEARCH DICTIONARY ...*/ stmts = lappend(stmts, GetTextSearchDictionaryDefineStmt(address->objectId)); /* ALTER TEXT SEARCH DICTIONARY ... OWNER TO ...*/ stmts = list_concat(stmts, GetTextSearchDictionaryOwnerStmts(address->objectId)); /* COMMENT ON TEXT SEARCH DICTIONARY ... */ stmts = list_concat(stmts, GetTextSearchDictionaryCommentStmt(address->objectId)); return stmts; } /* * CreateTextSearchConfigDDLCommandsIdempotent creates a list of ddl commands to recreate * a TEXT SERACH CONFIGURATION object in an idempotent manner on workers. */ List * CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address) { List *stmts = GetCreateTextSearchConfigStatements(address); List *sqls = DeparseTreeNodes(stmts); return list_make1(WrapCreateOrReplaceList(sqls)); } /* * CreateTextSearchDictDDLCommandsIdempotent creates a list of ddl commands to recreate * a TEXT SEARCH CONFIGURATION object in an idempotent manner on workers. */ List * CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address) { List *stmts = GetCreateTextSearchDictionaryStatements(address); List *sqls = DeparseTreeNodes(stmts); return list_make1(WrapCreateOrReplaceList(sqls)); } /* * GetTextSearchConfigDefineStmt returns the DefineStmt for a TEXT SEARCH CONFIGURATION * based on the configuration as defined in the catalog identified by tsconfigOid. * * This statement will only contain the parser, as all other properties for text search * configurations are stored as mappings in a different catalog. */ static DefineStmt * GetTextSearchConfigDefineStmt(Oid tsconfigOid) { HeapTuple tup = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(tsconfigOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search configuration %u", tsconfigOid); } Form_pg_ts_config config = (Form_pg_ts_config) GETSTRUCT(tup); DefineStmt *stmt = makeNode(DefineStmt); stmt->kind = OBJECT_TSCONFIGURATION; stmt->defnames = get_ts_config_namelist(tsconfigOid); List *parserNameList = get_ts_parser_namelist(config->cfgparser); TypeName *parserTypeName = makeTypeNameFromNameList(parserNameList); stmt->definition = list_make1(makeDefElem("parser", (Node *) parserTypeName, -1)); ReleaseSysCache(tup); return stmt; } /* * GetTextSearchDictionaryDefineStmt returns the DefineStmt for a TEXT SEARCH DICTIONARY * based on the dictionary as defined in the catalog identified by tsdictOid. * * This statement will contain the template along with all initilaization options. */ static DefineStmt * GetTextSearchDictionaryDefineStmt(Oid tsdictOid) { HeapTuple tup = SearchSysCache1(TSDICTOID, ObjectIdGetDatum(tsdictOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search dictionary %u", tsdictOid); } Form_pg_ts_dict dict = (Form_pg_ts_dict) GETSTRUCT(tup); DefineStmt *stmt = makeNode(DefineStmt); stmt->kind = OBJECT_TSDICTIONARY; stmt->defnames = get_ts_dict_namelist(tsdictOid); stmt->definition = GetTextSearchDictionaryInitOptions(tup, dict); ReleaseSysCache(tup); return stmt; } /* * GetTextSearchDictionaryInitOptions returns the list of DefElem for the initialization * options for a TEXT SEARCH DICTIONARY. * * The initialization options contain both the template name, and template specific key, * value pairs that are supplied when the dictionary was first created. */ static List * GetTextSearchDictionaryInitOptions(HeapTuple tup, Form_pg_ts_dict dict) { List *templateNameList = get_ts_template_namelist(dict->dicttemplate); TypeName *templateTypeName = makeTypeNameFromNameList(templateNameList); DefElem *templateDefElem = makeDefElem("template", (Node *) templateTypeName, -1); Relation TSDictionaryRelation = table_open(TSDictionaryRelationId, AccessShareLock); TupleDesc TSDictDescription = RelationGetDescr(TSDictionaryRelation); bool isnull = false; Datum dictinitoption = heap_getattr(tup, Anum_pg_ts_dict_dictinitoption, TSDictDescription, &isnull); List *initOptionDefElemList = NIL; if (!isnull) { initOptionDefElemList = deserialize_deflist(dictinitoption); } table_close(TSDictionaryRelation, AccessShareLock); return lcons(templateDefElem, initOptionDefElemList); } /* * GetTextSearchConfigCommentStmt returns a list containing all entries to recreate a * comment on the configuration identified by tsconfigOid. The list could be empty if * there is no comment on a configuration. * * The reason for a list is for easy use when building a list of all statements to invoke * to recreate the text search configuration. An empty list can easily be concatinated * without inspection, contrary to a NULL ptr if we would return the CommentStmt struct. */ static List * GetTextSearchConfigCommentStmt(Oid tsconfigOid) { char *comment = GetComment(tsconfigOid, TSConfigRelationId, 0); if (!comment) { return NIL; } CommentStmt *stmt = makeNode(CommentStmt); stmt->objtype = OBJECT_TSCONFIGURATION; stmt->object = (Node *) get_ts_config_namelist(tsconfigOid); stmt->comment = comment; return list_make1(stmt); } /* * GetTextSearchDictionaryCommentStmt returns a list containing all entries to recreate a * comment on the dictionary identified by tsconfigOid. The list could be empty if * there is no comment on a dictionary. * * The reason for a list is for easy use when building a list of all statements to invoke * to recreate the text search dictionary. An empty list can easily be concatinated * without inspection, contrary to a NULL ptr if we would return the CommentStmt struct. */ static List * GetTextSearchDictionaryCommentStmt(Oid tsdictOid) { char *comment = GetComment(tsdictOid, TSDictionaryRelationId, 0); if (!comment) { return NIL; } CommentStmt *stmt = makeNode(CommentStmt); stmt->objtype = OBJECT_TSDICTIONARY; stmt->object = (Node *) get_ts_dict_namelist(tsdictOid); stmt->comment = comment; return list_make1(stmt); } /* * GetTextSearchConfigMappingStmt returns a list of all mappings from token_types to * dictionaries configured on a text search configuration identified by tsconfigOid. * * Many mappings can exist on a configuration which all require their own statement to * recreate. */ static List * GetTextSearchConfigMappingStmt(Oid tsconfigOid) { ScanKeyData mapskey = { 0 }; /* mapcfg = tsconfigOid */ ScanKeyInit(&mapskey, Anum_pg_ts_config_map_mapcfg, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(tsconfigOid)); Relation maprel = table_open(TSConfigMapRelationId, AccessShareLock); Relation mapidx = index_open(TSConfigMapIndexId, AccessShareLock); SysScanDesc mapscan = systable_beginscan_ordered(maprel, mapidx, NULL, 1, &mapskey); List *stmts = NIL; AlterTSConfigurationStmt *stmt = NULL; /* * We iterate the config mappings on the index order filtered by mapcfg. Meaning we * get equal maptokentype's in 1 run. By comparing the current tokentype to the last * we know when we can create a new stmt and append the previous constructed one to * the list. */ int lastTokType = -1; /* * We read all mappings filtered by config id, hence we only need to load the name * once and can reuse for every statement. */ List *configName = get_ts_config_namelist(tsconfigOid); Oid parserOid = get_ts_config_parser_oid(tsconfigOid); HeapTuple maptup = NULL; while ((maptup = systable_getnext_ordered(mapscan, ForwardScanDirection)) != NULL) { Form_pg_ts_config_map cfgmap = (Form_pg_ts_config_map) GETSTRUCT(maptup); if (lastTokType != cfgmap->maptokentype) { /* creating a new statement, appending the previous one (if existing) */ if (stmt != NULL) { stmts = lappend(stmts, stmt); } stmt = makeNode(AlterTSConfigurationStmt); stmt->cfgname = configName; stmt->kind = ALTER_TSCONFIG_ADD_MAPPING; stmt->tokentype = list_make1(makeString( get_ts_parser_tokentype_name(parserOid, cfgmap-> maptokentype))); lastTokType = cfgmap->maptokentype; } stmt->dicts = lappend(stmt->dicts, get_ts_dict_namelist(cfgmap->mapdict)); } /* * If we have ran atleast 1 iteration above we have the last stmt not added to the * stmts list. */ if (stmt != NULL) { stmts = lappend(stmts, stmt); stmt = NULL; } systable_endscan_ordered(mapscan); index_close(mapidx, NoLock); table_close(maprel, NoLock); return stmts; } /* * GetTextSearchConfigOwnerStmts returns a potentially empty list of statements to change * the ownership of a TEXT SEARCH CONFIGURATION object. * * The list is for convenience when building a full list of statements to recreate the * configuration. */ static List * GetTextSearchConfigOwnerStmts(Oid tsconfigOid) { HeapTuple tup = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(tsconfigOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search configuration %u", tsconfigOid); } Form_pg_ts_config config = (Form_pg_ts_config) GETSTRUCT(tup); AlterOwnerStmt *stmt = makeNode(AlterOwnerStmt); stmt->objectType = OBJECT_TSCONFIGURATION; stmt->object = (Node *) get_ts_config_namelist(tsconfigOid); stmt->newowner = GetRoleSpecObjectForUser(config->cfgowner); ReleaseSysCache(tup); return list_make1(stmt); } /* * GetTextSearchDictionaryOwnerStmts returns a potentially empty list of statements to change * the ownership of a TEXT SEARCH DICTIONARY object. * * The list is for convenience when building a full list of statements to recreate the * dictionary. */ static List * GetTextSearchDictionaryOwnerStmts(Oid tsdictOid) { HeapTuple tup = SearchSysCache1(TSDICTOID, ObjectIdGetDatum(tsdictOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search dictionary %u", tsdictOid); } Form_pg_ts_dict dict = (Form_pg_ts_dict) GETSTRUCT(tup); AlterOwnerStmt *stmt = makeNode(AlterOwnerStmt); stmt->objectType = OBJECT_TSDICTIONARY; stmt->object = (Node *) get_ts_dict_namelist(tsdictOid); stmt->newowner = GetRoleSpecObjectForUser(dict->dictowner); ReleaseSysCache(tup); return list_make1(stmt); } /* * get_ts_config_namelist based on the tsconfigOid this function creates the namelist that * identifies the configuration in a fully qualified manner, irregardless of the schema * existing on the search_path. */ List * get_ts_config_namelist(Oid tsconfigOid) { HeapTuple tup = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(tsconfigOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search configuration %u", tsconfigOid); } Form_pg_ts_config config = (Form_pg_ts_config) GETSTRUCT(tup); char *schema = get_namespace_name(config->cfgnamespace); char *configName = pstrdup(NameStr(config->cfgname)); List *names = list_make2(makeString(schema), makeString(configName)); ReleaseSysCache(tup); return names; } /* * get_ts_dict_namelist based on the tsdictOid this function creates the namelist that * identifies the dictionary in a fully qualified manner, irregardless of the schema * existing on the search_path. */ static List * get_ts_dict_namelist(Oid tsdictOid) { HeapTuple tup = SearchSysCache1(TSDICTOID, ObjectIdGetDatum(tsdictOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search dictionary %u", tsdictOid); } Form_pg_ts_dict dict = (Form_pg_ts_dict) GETSTRUCT(tup); char *schema = get_namespace_name(dict->dictnamespace); char *dictName = pstrdup(NameStr(dict->dictname)); List *names = list_make2(makeString(schema), makeString(dictName)); ReleaseSysCache(tup); return names; } /* * get_ts_template_namelist based on the tstemplateOid this function creates the namelist * that identifies the template in a fully qualified manner, irregardless of the schema * existing on the search_path. */ static List * get_ts_template_namelist(Oid tstemplateOid) { HeapTuple tup = SearchSysCache1(TSTEMPLATEOID, ObjectIdGetDatum(tstemplateOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search template %u", tstemplateOid); } Form_pg_ts_template template = (Form_pg_ts_template) GETSTRUCT(tup); char *schema = get_namespace_name(template->tmplnamespace); char *templateName = pstrdup(NameStr(template->tmplname)); List *names = list_make2(makeString(schema), makeString(templateName)); ReleaseSysCache(tup); return names; } /* * get_ts_config_parser_oid based on the tsconfigOid this function returns the Oid of the * parser used in the configuration. */ static Oid get_ts_config_parser_oid(Oid tsconfigOid) { HeapTuple tup = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(tsconfigOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search configuration %u", tsconfigOid); } Form_pg_ts_config config = (Form_pg_ts_config) GETSTRUCT(tup); Oid parserOid = config->cfgparser; ReleaseSysCache(tup); return parserOid; } /* * get_ts_parser_tokentype_name returns the name of the token as known to the parser by * its tokentype identifier. The parser used to resolve the token name is identified by * parserOid and should be the same that emitted the tokentype to begin with. */ static char * get_ts_parser_tokentype_name(Oid parserOid, int32 tokentype) { TSParserCacheEntry *parserCache = lookup_ts_parser_cache(parserOid); if (!OidIsValid(parserCache->lextypeOid)) { elog(ERROR, "method lextype isn't defined for text search parser %u", parserOid); } /* take lextypes from parser */ LexDescr *tokenlist = (LexDescr *) DatumGetPointer( OidFunctionCall1(parserCache->lextypeOid, Int32GetDatum(0))); /* and find the one with lexid = tokentype */ int tokenIndex = 0; while (tokenlist && tokenlist[tokenIndex].lexid) { if (tokenlist[tokenIndex].lexid == tokentype) { return pstrdup(tokenlist[tokenIndex].alias); } tokenIndex++; } /* we haven't found the token */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("token type \"%d\" does not exist in parser", tokentype))); } /* * get_ts_parser_namelist based on the tsparserOid this function creates the namelist that * identifies the parser in a fully qualified manner, irregardless of the schema existing * on the search_path. */ static List * get_ts_parser_namelist(Oid tsparserOid) { HeapTuple tup = SearchSysCache1(TSPARSEROID, ObjectIdGetDatum(tsparserOid)); if (!HeapTupleIsValid(tup)) /* should not happen */ { elog(ERROR, "cache lookup failed for text search parser %u", tsparserOid); } Form_pg_ts_parser parser = (Form_pg_ts_parser) GETSTRUCT(tup); char *schema = get_namespace_name(parser->prsnamespace); char *parserName = pstrdup(NameStr(parser->prsname)); List *names = list_make2(makeString(schema), makeString(parserName)); ReleaseSysCache(tup); return names; } /* * CreateTextSearchConfigurationObjectAddress resolves the ObjectAddress for the object * being created. If missing_pk is false the function will error, explaining to the user * the text search configuration described in the statement doesn't exist. */ List * CreateTextSearchConfigurationObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DefineStmt *stmt = castNode(DefineStmt, node); Assert(stmt->kind == OBJECT_TSCONFIGURATION); Oid objid = get_ts_config_oid(stmt->defnames, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSConfigRelationId, objid); return list_make1(address); } /* * CreateTextSearchDictObjectAddress resolves the ObjectAddress for the object * being created. If missing_pk is false the function will error, explaining to the user * the text search dictionary described in the statement doesn't exist. */ List * CreateTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DefineStmt *stmt = castNode(DefineStmt, node); Assert(stmt->kind == OBJECT_TSDICTIONARY); Oid objid = get_ts_dict_oid(stmt->defnames, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSDictionaryRelationId, objid); return list_make1(address); } /* * RenameTextSearchConfigurationStmtObjectAddress resolves the ObjectAddress for the TEXT * SEARCH CONFIGURATION being renamed. Optionally errors if the configuration does not * exist based on the missing_ok flag passed in by the caller. */ List * RenameTextSearchConfigurationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSCONFIGURATION); Oid objid = get_ts_config_oid(castNode(List, stmt->object), missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSConfigRelationId, objid); return list_make1(address); } /* * RenameTextSearchDictionaryStmtObjectAddress resolves the ObjectAddress for the TEXT * SEARCH DICTIONARY being renamed. Optionally errors if the dictionary does not * exist based on the missing_ok flag passed in by the caller. */ List * RenameTextSearchDictionaryStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSDICTIONARY); Oid objid = get_ts_dict_oid(castNode(List, stmt->object), missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSDictionaryRelationId, objid); return list_make1(address); } /* * AlterTextSearchConfigurationStmtObjectAddress resolves the ObjectAddress for the TEXT * SEARCH CONFIGURATION being altered. Optionally errors if the configuration does not * exist based on the missing_ok flag passed in by the caller. */ List * AlterTextSearchConfigurationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTSConfigurationStmt *stmt = castNode(AlterTSConfigurationStmt, node); Oid objid = get_ts_config_oid(stmt->cfgname, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSConfigRelationId, objid); return list_make1(address); } /* * AlterTextSearchDictionaryStmtObjectAddress resolves the ObjectAddress for the TEXT * SEARCH CONFIGURATION being altered. Optionally errors if the configuration does not * exist based on the missing_ok flag passed in by the caller. */ List * AlterTextSearchDictionaryStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTSDictionaryStmt *stmt = castNode(AlterTSDictionaryStmt, node); Oid objid = get_ts_dict_oid(stmt->dictname, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TSDictionaryRelationId, objid); return list_make1(address); } /* * AlterTextSearchConfigurationSchemaStmtObjectAddress resolves the ObjectAddress for the * TEXT SEARCH CONFIGURATION being moved to a different schema. Optionally errors if the * configuration does not exist based on the missing_ok flag passed in by the caller. * * This can be called, either before or after the move of schema has been executed, hence * the triple checking before the error might be thrown. Errors for non-existing schema's * in edgecases will be raised by postgres while executing the move. */ List * AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSCONFIGURATION); Oid objid = get_ts_config_oid(castNode(List, stmt->object), true); if (!OidIsValid(objid)) { /* * couldn't find the text search configuration, might have already been moved to * the new schema, we construct a new sequence name that uses the new schema to * search in. */ char *schemaname = NULL; char *config_name = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaname, &config_name); char *newSchemaName = stmt->newschema; List *names = list_make2(makeString(newSchemaName), makeString(config_name)); objid = get_ts_config_oid(names, true); if (!missing_ok && !OidIsValid(objid)) { /* * if the text search config id is still invalid we couldn't find it, error * with the same message postgres would error with if missing_ok is false * (not ok to miss) */ ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("text search configuration \"%s\" does not exist", NameListToString(castNode(List, stmt->object))))); } } ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, TSConfigRelationId, objid); return list_make1(sequenceAddress); } /* * AlterTextSearchDictionarySchemaStmtObjectAddress resolves the ObjectAddress for the * TEXT SEARCH DICTIONARY being moved to a different schema. Optionally errors if the * dictionary does not exist based on the missing_ok flag passed in by the caller. * * This can be called, either before or after the move of schema has been executed, hence * the triple checking before the error might be thrown. Errors for non-existing schema's * in edgecases will be raised by postgres while executing the move. */ List * AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSDICTIONARY); Oid objid = get_ts_dict_oid(castNode(List, stmt->object), true); if (!OidIsValid(objid)) { /* * couldn't find the text search dictionary, might have already been moved to * the new schema, we construct a new sequence name that uses the new schema to * search in. */ char *schemaname = NULL; char *dict_name = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaname, &dict_name); char *newSchemaName = stmt->newschema; List *names = list_make2(makeString(newSchemaName), makeString(dict_name)); objid = get_ts_dict_oid(names, true); if (!missing_ok && !OidIsValid(objid)) { /* * if the text search dict id is still invalid we couldn't find it, error * with the same message postgres would error with if missing_ok is false * (not ok to miss) */ ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("text search dictionary \"%s\" does not exist", NameListToString(castNode(List, stmt->object))))); } } ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, TSDictionaryRelationId, objid); return list_make1(sequenceAddress); } /* * AlterTextSearchConfigurationOwnerObjectAddress resolves the ObjectAddress for the TEXT * SEARCH CONFIGURATION for which the owner is changed. Optionally errors if the * configuration does not exist based on the missing_ok flag passed in by the caller. */ List * AlterTextSearchConfigurationOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Relation relation = NULL; Assert(stmt->objectType == OBJECT_TSCONFIGURATION); ObjectAddress objectAddress = get_object_address(stmt->objectType, stmt->object, &relation, AccessShareLock, missing_ok); ObjectAddress *objectAddressCopy = palloc0(sizeof(ObjectAddress)); *objectAddressCopy = objectAddress; return list_make1(objectAddressCopy); } /* * AlterTextSearchDictOwnerObjectAddress resolves the ObjectAddress for the TEXT * SEARCH DICTIONARY for which the owner is changed. Optionally errors if the * configuration does not exist based on the missing_ok flag passed in by the caller. */ List * AlterTextSearchDictOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Relation relation = NULL; Assert(stmt->objectType == OBJECT_TSDICTIONARY); ObjectAddress objectAddress = get_object_address(stmt->objectType, stmt->object, &relation, AccessShareLock, missing_ok); ObjectAddress *objectAddressCopy = palloc0(sizeof(ObjectAddress)); *objectAddressCopy = objectAddress; return list_make1(objectAddressCopy); } /* * GenerateBackupNameForTextSearchConfiguration generates a safe name that is not in use * already that can be used to rename an existing TEXT SEARCH CONFIGURATION to allow the * configuration with a specific name to be created, even if this would not have been * possible due to name collisions. */ char * GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address) { Assert(address->classId == TSConfigRelationId); List *names = get_ts_config_namelist(address->objectId); RangeVar *rel = makeRangeVarFromNameList(names); char *newName = palloc0(NAMEDATALEN); char suffix[NAMEDATALEN] = { 0 }; char *baseName = rel->relname; int baseLength = strlen(baseName); int count = 0; while (true) { int suffixLength = SafeSnprintf(suffix, NAMEDATALEN - 1, "(citus_backup_%d)", count); /* trim the base name at the end to leave space for the suffix and trailing \0 */ baseLength = Min(baseLength, NAMEDATALEN - suffixLength - 1); /* clear newName before copying the potentially trimmed baseName and suffix */ memset(newName, 0, NAMEDATALEN); strncpy_s(newName, NAMEDATALEN, baseName, baseLength); strncpy_s(newName + baseLength, NAMEDATALEN - baseLength, suffix, suffixLength); rel->relname = newName; List *newNameList = MakeNameListFromRangeVar(rel); Oid tsconfigOid = get_ts_config_oid(newNameList, true); if (!OidIsValid(tsconfigOid)) { return newName; } count++; } } ================================================ FILE: src/backend/distributed/commands/trigger.c ================================================ /*------------------------------------------------------------------------- * trigger.c * * This file contains functions to create and process trigger objects on * citus tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/table.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_trigger.h" #include "commands/extension.h" #include "commands/trigger.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/namespace_utils.h" #include "distributed/shard_utils.h" #include "distributed/worker_protocol.h" /* appropriate lock modes for the owner relation according to postgres */ #define CREATE_TRIGGER_LOCK_MODE ShareRowExclusiveLock #define ALTER_TRIGGER_LOCK_MODE AccessExclusiveLock #define DROP_TRIGGER_LOCK_MODE AccessExclusiveLock /* local function forward declarations */ static char * GetAlterTriggerStateCommand(Oid triggerId); static bool IsCreateCitusTruncateTriggerStmt(CreateTrigStmt *createTriggerStmt); static String * GetAlterTriggerDependsTriggerNameValue(AlterObjectDependsStmt * alterTriggerDependsStmt); static void ErrorIfUnsupportedDropTriggerCommand(DropStmt *dropTriggerStmt); static RangeVar * GetDropTriggerStmtRelation(DropStmt *dropTriggerStmt); static void ExtractDropStmtTriggerAndRelationName(DropStmt *dropTriggerStmt, char **triggerName, char **relationName); static void ErrorIfDropStmtDropsMultipleTriggers(DropStmt *dropTriggerStmt); static char * GetTriggerNameById(Oid triggerId); static int16 GetTriggerTypeById(Oid triggerId); /* GUC that overrides trigger checks for distributed tables and reference tables */ bool EnableUnsafeTriggers = false; /* * GetExplicitTriggerCommandList returns the list of DDL commands to create * triggers that are explicitly created for the table with relationId. See * comment of GetExplicitTriggerIdList function. */ List * GetExplicitTriggerCommandList(Oid relationId) { List *createTriggerCommandList = NIL; int saveNestLevel = PushEmptySearchPath(); List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; foreach_declared_oid(triggerId, triggerIdList) { bool prettyOutput = false; Datum commandText = DirectFunctionCall2(pg_get_triggerdef_ext, ObjectIdGetDatum(triggerId), BoolGetDatum(prettyOutput)); /* * pg_get_triggerdef_ext doesn't throw an error if there is no such * trigger, be on the safe side. */ if (DatumGetPointer(commandText) == NULL) { ereport(ERROR, (errmsg("trigger with oid %u does not exist", triggerId))); } char *createTriggerCommand = TextDatumGetCString(commandText); createTriggerCommandList = lappend( createTriggerCommandList, makeTableDDLCommandString(createTriggerCommand)); /* * Appends the commands for the trigger settings that are not covered * by CREATE TRIGGER command, such as ALTER TABLE ENABLE/DISABLE . */ char *alterTriggerStateCommand = GetAlterTriggerStateCommand(triggerId); createTriggerCommandList = lappend( createTriggerCommandList, makeTableDDLCommandString(alterTriggerStateCommand)); } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return createTriggerCommandList; } /* * GetAlterTriggerStateCommand returns the DDL command to set enable/disable * state for given trigger. Throws an error if no such trigger exists. */ static char * GetAlterTriggerStateCommand(Oid triggerId) { StringInfo alterTriggerStateCommand = makeStringInfo(); bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); char *qualifiedRelName = generate_qualified_relation_name(triggerForm->tgrelid); const char *quotedTrigName = quote_identifier(NameStr(triggerForm->tgname)); char enableDisableState = triggerForm->tgenabled; const char *alterTriggerStateStr = NULL; switch (enableDisableState) { case TRIGGER_FIRES_ON_ORIGIN: { /* default mode */ alterTriggerStateStr = "ENABLE"; break; } case TRIGGER_FIRES_ALWAYS: { alterTriggerStateStr = "ENABLE ALWAYS"; break; } case TRIGGER_FIRES_ON_REPLICA: { alterTriggerStateStr = "ENABLE REPLICA"; break; } case TRIGGER_DISABLED: { alterTriggerStateStr = "DISABLE"; break; } default: { elog(ERROR, "unexpected trigger state"); } } appendStringInfo(alterTriggerStateCommand, "ALTER TABLE %s %s TRIGGER %s;", qualifiedRelName, alterTriggerStateStr, quotedTrigName); /* * Free triggerTuple at the end since quote_identifier() might not return * a palloc'd string if given identifier doesn't need to be quoted, and in * that case quotedTrigName would still be bound to triggerTuple. */ heap_freetuple(triggerTuple); return alterTriggerStateCommand->data; } /* * GetTriggerTupleById returns copy of the heap tuple from pg_trigger for * the trigger with triggerId. If no such trigger exists, this function returns * NULL or errors out depending on missingOk. */ HeapTuple GetTriggerTupleById(Oid triggerId, bool missingOk) { Relation pgTrigger = table_open(TriggerRelationId, AccessShareLock); int scanKeyCount = 1; ScanKeyData scanKey[1]; AttrNumber attrNumber = Anum_pg_trigger_oid; ScanKeyInit(&scanKey[0], attrNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(triggerId)); bool useIndex = true; SysScanDesc scanDescriptor = systable_beginscan(pgTrigger, TriggerOidIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple targetHeapTuple = NULL; HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { targetHeapTuple = heap_copytuple(heapTuple); } systable_endscan(scanDescriptor); table_close(pgTrigger, NoLock); if (targetHeapTuple == NULL && missingOk == false) { ereport(ERROR, (errmsg("could not find heap tuple for trigger with " "OID %d", triggerId))); } return targetHeapTuple; } /* * GetExplicitTriggerIdList returns a list of OIDs corresponding to the triggers * that are explicitly created on the relation with relationId. That means, * this function discards internal triggers implicitly created by postgres for * foreign key constraint validation and the citus_truncate_trigger. */ List * GetExplicitTriggerIdList(Oid relationId) { List *triggerIdList = NIL; Relation pgTrigger = table_open(TriggerRelationId, AccessShareLock); int scanKeyCount = 1; ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_trigger_tgrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); bool useIndex = true; SysScanDesc scanDescriptor = systable_beginscan(pgTrigger, TriggerRelidNameIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(heapTuple); /* * Note that we mark truncate trigger that we create on citus tables as * internal. Hence, below we discard citus_truncate_trigger as well as * the implicit triggers created by postgres for foreign key validation. * * Pre PG15, tgisinternal is true for a "child" trigger on a partition * cloned from the trigger on the parent. * In PG15, tgisinternal is false in that case. However, we don't want to * create this trigger on the partition since it will create a conflict * when we try to attach the partition to the parent table: * ERROR: trigger "..." for relation "{partition_name}" already exists * Hence we add an extra check on whether the parent id is invalid to * make sure this is not a child trigger */ if (!triggerForm->tgisinternal && (triggerForm->tgparentid == InvalidOid)) { triggerIdList = lappend_oid(triggerIdList, triggerForm->oid); } heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgTrigger, NoLock); return triggerIdList; } /* * PostprocessCreateTriggerStmt is called after a CREATE TRIGGER command has * been executed by standard process utility. This function errors out for * unsupported commands or creates ddl job for supported CREATE TRIGGER commands. */ List * PostprocessCreateTriggerStmt(Node *node, const char *queryString) { CreateTrigStmt *createTriggerStmt = castNode(CreateTrigStmt, node); if (IsCreateCitusTruncateTriggerStmt(createTriggerStmt)) { return NIL; } RangeVar *relation = createTriggerStmt->relation; bool missingOk = false; Oid relationId = RangeVarGetRelid(relation, CREATE_TRIGGER_LOCK_MODE, missingOk); if (!IsCitusTable(relationId)) { return NIL; } EnsureCoordinator(); ErrorOutForTriggerIfNotSupported(relationId); List *objectAddresses = GetObjectAddressListFromParseTree(node, missingOk, true); /* the code-path only supports a single object */ Assert(list_length(objectAddresses) == 1); EnsureAllObjectDependenciesExistOnAllNodes(objectAddresses); char *triggerName = createTriggerStmt->trigname; return CitusCreateTriggerCommandDDLJob(relationId, triggerName, queryString); } /* * CreateTriggerStmtObjectAddress finds the ObjectAddress for the trigger that * is created by given CreateTriggerStmt. If missingOk is false and if trigger * does not exist, then it errors out. * * Never returns NULL, but the objid in the address can be invalid if missingOk * was set to true. */ List * CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { CreateTrigStmt *createTriggerStmt = castNode(CreateTrigStmt, node); RangeVar *relation = createTriggerStmt->relation; Oid relationId = RangeVarGetRelid(relation, CREATE_TRIGGER_LOCK_MODE, missingOk); char *triggerName = createTriggerStmt->trigname; Oid triggerId = get_trigger_oid(relationId, triggerName, missingOk); if (triggerId == InvalidOid && missingOk == false) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("trigger \"%s\" on relation \"%s\" does not exist", triggerName, relationName))); } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TriggerRelationId, triggerId); return list_make1(address); } /* * IsCreateCitusTruncateTriggerStmt returns true if given createTriggerStmt * creates citus_truncate_trigger. */ static bool IsCreateCitusTruncateTriggerStmt(CreateTrigStmt *createTriggerStmt) { List *functionNameList = createTriggerStmt->funcname; RangeVar *functionRangeVar = makeRangeVarFromNameList(functionNameList); char *functionName = functionRangeVar->relname; if (strncmp(functionName, CITUS_TRUNCATE_TRIGGER_NAME, NAMEDATALEN) == 0) { return true; } return false; } /* * CreateTriggerEventExtendNames extends relation name and trigger name with * shardId, and sets schema name in given CreateTrigStmt. */ void CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaName, uint64 shardId) { RangeVar *relation = createTriggerStmt->relation; char **relationName = &(relation->relname); AppendShardIdToName(relationName, shardId); char **triggerName = &(createTriggerStmt->trigname); AppendShardIdToName(triggerName, shardId); char **relationSchemaName = &(relation->schemaname); SetSchemaNameIfNotExist(relationSchemaName, schemaName); } /* * PostprocessAlterTriggerRenameStmt is called after a ALTER TRIGGER RENAME * command has been executed by standard process utility. This function errors * out for unsupported commands or creates ddl job for supported ALTER TRIGGER * RENAME commands. */ List * PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString) { RenameStmt *renameTriggerStmt = castNode(RenameStmt, node); Assert(renameTriggerStmt->renameType == OBJECT_TRIGGER); RangeVar *relation = renameTriggerStmt->relation; bool missingOk = false; Oid relationId = RangeVarGetRelid(relation, ALTER_TRIGGER_LOCK_MODE, missingOk); if (!IsCitusTable(relationId)) { return NIL; } EnsureCoordinator(); ErrorOutForTriggerIfNotSupported(relationId); /* use newname as standard process utility already renamed it */ char *triggerName = renameTriggerStmt->newname; return CitusCreateTriggerCommandDDLJob(relationId, triggerName, queryString); } /* * AlterTriggerRenameEventExtendNames extends relation name, old and new trigger * name with shardId, and sets schema name in given RenameStmt. */ void AlterTriggerRenameEventExtendNames(RenameStmt *renameTriggerStmt, char *schemaName, uint64 shardId) { Assert(renameTriggerStmt->renameType == OBJECT_TRIGGER); RangeVar *relation = renameTriggerStmt->relation; char **relationName = &(relation->relname); AppendShardIdToName(relationName, shardId); char **triggerOldName = &(renameTriggerStmt->subname); AppendShardIdToName(triggerOldName, shardId); char **triggerNewName = &(renameTriggerStmt->newname); AppendShardIdToName(triggerNewName, shardId); char **relationSchemaName = &(relation->schemaname); SetSchemaNameIfNotExist(relationSchemaName, schemaName); } /* * PreprocessAlterTriggerDependsStmt is called during the planning phase of an * ALTER TRIGGER ... DEPENDS ON EXTENSION ... statement. Since triggers depending on * extensions are assumed to be Owned by an extension we assume the extension to keep * the trigger in sync. * * If we would allow users to create a dependency between a distributed trigger and an * extension, our pruning logic for which objects to distribute as dependencies of other * objects will change significantly, which could cause issues adding new workers. Hence * we don't allow this dependency to be created. */ List * PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectDependsStmt *alterTriggerDependsStmt = castNode(AlterObjectDependsStmt, node); Assert(alterTriggerDependsStmt->objectType == OBJECT_TRIGGER); if (creating_extension) { /* * extensions should be created separately on the workers, triggers cascading * from an extension should therefore not be propagated here. */ return NIL; } if (!EnableMetadataSync) { /* * we are configured to disable object propagation, should not propagate anything */ return NIL; } RangeVar *relation = alterTriggerDependsStmt->relation; bool missingOk = false; Oid relationId = RangeVarGetRelid(relation, ALTER_TRIGGER_LOCK_MODE, missingOk); if (!IsCitusTable(relationId)) { return NIL; } /* * Distributed objects should not start depending on an extension, this will break * the dependency resolving mechanism we use to replicate distributed objects to new * workers */ String *triggerNameValue = GetAlterTriggerDependsTriggerNameValue(alterTriggerDependsStmt); ereport(ERROR, (errmsg("trigger \"%s\" depends on an extension and this " "is not supported for distributed tables and " "local tables added to metadata", strVal(triggerNameValue)), errdetail("Triggers from extensions are expected to be " "created on the workers by the extension they " "depend on."))); /* not reachable, keep compiler happy */ return NIL; } /* * PostprocessAlterTriggerDependsStmt is called after a ALTER TRIGGER DEPENDS ON * command has been executed by standard process utility. This function errors out * for unsupported commands or creates ddl job for supported ALTER TRIGGER DEPENDS * ON commands. */ List * PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString) { AlterObjectDependsStmt *alterTriggerDependsStmt = castNode(AlterObjectDependsStmt, node); Assert(alterTriggerDependsStmt->objectType == OBJECT_TRIGGER); RangeVar *relation = alterTriggerDependsStmt->relation; bool missingOk = false; Oid relationId = RangeVarGetRelid(relation, ALTER_TRIGGER_LOCK_MODE, missingOk); if (!IsCitusTable(relationId)) { return NIL; } EnsureCoordinator(); ErrorOutForTriggerIfNotSupported(relationId); String *triggerNameValue = GetAlterTriggerDependsTriggerNameValue(alterTriggerDependsStmt); return CitusCreateTriggerCommandDDLJob(relationId, strVal(triggerNameValue), queryString); } /* * AlterTriggerDependsEventExtendNames extends relation name and trigger name * with shardId, and sets schema name in given AlterObjectDependsStmt. */ void AlterTriggerDependsEventExtendNames(AlterObjectDependsStmt *alterTriggerDependsStmt, char *schemaName, uint64 shardId) { Assert(alterTriggerDependsStmt->objectType == OBJECT_TRIGGER); RangeVar *relation = alterTriggerDependsStmt->relation; char **relationName = &(relation->relname); AppendShardIdToName(relationName, shardId); String *triggerNameValue = GetAlterTriggerDependsTriggerNameValue(alterTriggerDependsStmt); AppendShardIdToName(&strVal(triggerNameValue), shardId); char **relationSchemaName = &(relation->schemaname); SetSchemaNameIfNotExist(relationSchemaName, schemaName); } /* * GetAlterTriggerDependsTriggerName returns Value object for the trigger * name that given AlterObjectDependsStmt is executed for. */ static String * GetAlterTriggerDependsTriggerNameValue(AlterObjectDependsStmt *alterTriggerDependsStmt) { List *triggerObjectNameList = (List *) alterTriggerDependsStmt->object; /* * Before standard process utility, we only have trigger name in "object" * list. However, standard process utility prepends that list with the * relationNameList retrieved from AlterObjectDependsStmt->RangeVar and * we call this method after standard process utility. So, for the further * usages, it is certain that the last element in "object" list will always * be the name of the trigger in either before or after standard process * utility. */ String *triggerNameValue = llast(triggerObjectNameList); return triggerNameValue; } /* * PreprocessDropTriggerStmt is called before a DROP TRIGGER command has been * executed by standard process utility. This function errors out for * unsupported commands or creates ddl job for supported DROP TRIGGER commands. * The reason we process drop trigger commands before standard process utility * (unlike the other type of trigger commands) is that we act according to trigger * type in CitusCreateTriggerCommandDDLJob but trigger wouldn't exist after * standard process utility. */ List * PreprocessDropTriggerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *dropTriggerStmt = castNode(DropStmt, node); Assert(dropTriggerStmt->removeType == OBJECT_TRIGGER); RangeVar *relation = GetDropTriggerStmtRelation(dropTriggerStmt); bool missingOk = true; Oid relationId = RangeVarGetRelid(relation, DROP_TRIGGER_LOCK_MODE, missingOk); if (!OidIsValid(relationId)) { /* let standard process utility to error out */ return NIL; } if (!IsCitusTable(relationId)) { return NIL; } ErrorIfUnsupportedDropTriggerCommand(dropTriggerStmt); char *triggerName = NULL; ExtractDropStmtTriggerAndRelationName(dropTriggerStmt, &triggerName, NULL); return CitusCreateTriggerCommandDDLJob(relationId, triggerName, queryString); } /* * ErrorIfUnsupportedDropTriggerCommand errors out for unsupported * "DROP TRIGGER triggerName ON relationName" commands. */ static void ErrorIfUnsupportedDropTriggerCommand(DropStmt *dropTriggerStmt) { RangeVar *relation = GetDropTriggerStmtRelation(dropTriggerStmt); bool missingOk = false; Oid relationId = RangeVarGetRelid(relation, DROP_TRIGGER_LOCK_MODE, missingOk); if (!IsCitusTable(relationId)) { return; } EnsureCoordinator(); ErrorOutForTriggerIfNotSupported(relationId); } /* * ErrorOutForTriggerIfNotSupported is a helper function to error * out for unsupported trigger commands depending on the citus table type. */ void ErrorOutForTriggerIfNotSupported(Oid relationId) { if (EnableUnsafeTriggers) { /* user really wants triggers */ return; } else if (IsCitusTableType(relationId, REFERENCE_TABLE)) { ereport(ERROR, (errmsg("triggers are not supported on reference tables"))); } else if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { ereport(ERROR, (errmsg("triggers are not supported on distributed tables"))); } /* we always support triggers on citus local tables */ } /* * ErrorIfRelationHasUnsupportedTrigger throws an error if given relation has * a trigger that is not supported by Citus. */ void ErrorIfRelationHasUnsupportedTrigger(Oid relationId) { List *relationTriggerList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; foreach_declared_oid(triggerId, relationTriggerList) { ObjectAddress triggerObjectAddress = InvalidObjectAddress; ObjectAddressSet(triggerObjectAddress, TriggerRelationId, triggerId); /* triggers that depend on extensions are not supported */ if (ObjectAddressDependsOnExtension(&triggerObjectAddress)) { ereport(ERROR, (errmsg("trigger \"%s\" depends on an extension and this " "is not supported for distributed tables and " "local tables added to metadata", GetTriggerNameById(triggerId)))); } } } /* * GetDropTriggerStmtRelation takes a DropStmt for a trigger object and returns * RangeVar for the relation that owns the trigger. */ static RangeVar * GetDropTriggerStmtRelation(DropStmt *dropTriggerStmt) { Assert(dropTriggerStmt->removeType == OBJECT_TRIGGER); ErrorIfDropStmtDropsMultipleTriggers(dropTriggerStmt); List *targetObjectList = dropTriggerStmt->objects; List *triggerObjectNameList = linitial(targetObjectList); /* * The name list that identifies the trigger to be dropped looks like: * [catalogName, schemaName, relationName, triggerName], where, the first * two elements are optional. We should take all elements except the * triggerName to create the range var object that defines the owner * relation. */ int relationNameListLength = list_length(triggerObjectNameList) - 1; List *relationNameList = list_truncate(list_copy(triggerObjectNameList), relationNameListLength); return makeRangeVarFromNameList(relationNameList); } /* * DropTriggerEventExtendNames extends relation name and trigger name with * shardId, and sets schema name in given DropStmt by recreating "objects" * list. */ void DropTriggerEventExtendNames(DropStmt *dropTriggerStmt, char *schemaName, uint64 shardId) { Assert(dropTriggerStmt->removeType == OBJECT_TRIGGER); char *triggerName = NULL; char *relationName = NULL; ExtractDropStmtTriggerAndRelationName(dropTriggerStmt, &triggerName, &relationName); AppendShardIdToName(&triggerName, shardId); String *triggerNameValue = makeString(triggerName); AppendShardIdToName(&relationName, shardId); String *relationNameValue = makeString(relationName); String *schemaNameValue = makeString(pstrdup(schemaName)); List *shardTriggerNameList = list_make3(schemaNameValue, relationNameValue, triggerNameValue); dropTriggerStmt->objects = list_make1(shardTriggerNameList); } /* * ExtractDropStmtTriggerAndRelationName extracts triggerName and relationName * from given dropTriggerStmt if arguments are passed as non-null pointers. */ static void ExtractDropStmtTriggerAndRelationName(DropStmt *dropTriggerStmt, char **triggerName, char **relationName) { ErrorIfDropStmtDropsMultipleTriggers(dropTriggerStmt); List *targetObjectList = dropTriggerStmt->objects; List *triggerObjectNameList = linitial(targetObjectList); int objectNameListLength = list_length(triggerObjectNameList); if (triggerName != NULL) { int triggerNameindex = objectNameListLength - 1; *triggerName = strVal(safe_list_nth(triggerObjectNameList, triggerNameindex)); } if (relationName != NULL) { int relationNameIndex = objectNameListLength - 2; *relationName = strVal(safe_list_nth(triggerObjectNameList, relationNameIndex)); } } /* * ErrorIfDropStmtDropsMultipleTriggers errors out if given drop trigger * command drops more than one trigger. Actually, this can't be the case * as postgres doesn't support dropping multiple triggers, but we should * be on the safe side. */ static void ErrorIfDropStmtDropsMultipleTriggers(DropStmt *dropTriggerStmt) { List *targetObjectList = dropTriggerStmt->objects; if (list_length(targetObjectList) > 1) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cannot execute DROP TRIGGER command for multiple " "triggers"))); } } /* * CitusCreateTriggerCommandDDLJob creates a ddl job to execute given * queryString trigger command on shell relation(s) in mx worker(s) and to * execute necessary ddl task on citus local table shard (if needed). */ List * CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, const char *queryString) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ObjectAddressSet(ddlJob->targetObjectAddress, RelationRelationId, relationId); ddlJob->metadataSyncCommand = queryString; if (!triggerName) { /* * ENABLE/DISABLE TRIGGER ALL/USER commands do not specify trigger * name. */ ddlJob->taskList = DDLTaskList(relationId, queryString); return list_make1(ddlJob); } bool missingOk = true; Oid triggerId = get_trigger_oid(relationId, triggerName, missingOk); if (!OidIsValid(triggerId)) { /* * For DROP, ENABLE/DISABLE, ENABLE REPLICA/ALWAYS TRIGGER commands, * we create ddl job in preprocess. So trigger may not exist. */ return NIL; } int16 triggerType = GetTriggerTypeById(triggerId); /* we don't have truncate triggers on shard relations */ if (!TRIGGER_FOR_TRUNCATE(triggerType)) { ddlJob->taskList = DDLTaskList(relationId, queryString); } return list_make1(ddlJob); } /* * GetTriggerNameById returns name of given trigger if such a trigger exists. * Otherwise, errors out. */ static char * GetTriggerNameById(Oid triggerId) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); char *triggerName = pstrdup(NameStr(triggerForm->tgname)); heap_freetuple(triggerTuple); return triggerName; } /* * GetTriggerTypeById returns trigger type (tgtype) of the trigger identified * by triggerId if it exists. Otherwise, errors out. */ static int16 GetTriggerTypeById(Oid triggerId) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); int16 triggerType = triggerForm->tgtype; heap_freetuple(triggerTuple); return triggerType; } /* * GetTriggerFunctionId returns OID of the function that the trigger with * triggerId executes if the trigger exists. Otherwise, errors out. */ Oid GetTriggerFunctionId(Oid triggerId) { bool missingOk = false; HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk); Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple); Oid functionId = triggerForm->tgfoid; heap_freetuple(triggerTuple); return functionId; } ================================================ FILE: src/backend/distributed/commands/truncate.c ================================================ /*------------------------------------------------------------------------- * * truncate.c * Commands for truncating distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "commands/tablecmds.h" #include "commands/trigger.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/rel.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" /* Local functions forward declarations for unsupported command checks */ static void ErrorIfUnsupportedTruncateStmt(TruncateStmt *truncateStatement); static void ExecuteTruncateStmtSequentialIfNecessary(TruncateStmt *command); static void EnsurePartitionTableNotReplicatedForTruncate(TruncateStmt *truncateStatement); static List * TruncateTaskList(Oid relationId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_truncate_trigger); PG_FUNCTION_INFO_V1(truncate_local_data_after_distributing_table); void EnsureLocalTableCanBeTruncated(Oid relationId); /* * citus_truncate_trigger is called as a trigger when a distributed * table is truncated. */ Datum citus_truncate_trigger(PG_FUNCTION_ARGS) { if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } TriggerData *triggerData = (TriggerData *) fcinfo->context; Relation truncatedRelation = triggerData->tg_relation; Oid relationId = RelationGetRelid(truncatedRelation); if (!EnableDDLPropagation) { PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* we might be truncating multiple relations */ UseCoordinatedTransaction(); if (IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *relationName = get_rel_name(relationId); bool dropShardsMetadataOnly = false; DirectFunctionCall4(citus_drop_all_shards, ObjectIdGetDatum(relationId), CStringGetTextDatum(schemaName), CStringGetTextDatum(relationName), BoolGetDatum(dropShardsMetadataOnly)); } else { List *taskList = TruncateTaskList(relationId); /* * If it is a local placement of a distributed table or a reference table, * then execute TRUNCATE command locally. */ bool localExecutionSupported = true; ExecuteUtilityTaskList(taskList, localExecutionSupported); } PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * TruncateTaskList returns a list of tasks to execute a TRUNCATE on a * distributed table. This is handled separately from other DDL commands * because we handle it via the TRUNCATE trigger, which is called whenever * a truncate cascades. */ static List * TruncateTaskList(Oid relationId) { /* resulting task list */ List *taskList = NIL; /* enumerate the tasks when putting them to the taskList */ int taskId = 1; Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *relationName = get_rel_name(relationId); List *shardIntervalList = LoadShardIntervalList(relationId); /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardRelationName = pstrdup(relationName); /* build shard relation name */ AppendShardIdToName(&shardRelationName, shardId); char *quotedShardName = quote_qualified_identifier(schemaName, shardRelationName); StringInfo shardQueryString = makeStringInfo(); appendStringInfo(shardQueryString, "TRUNCATE TABLE %s CASCADE", quotedShardName); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, shardQueryString->data); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); taskList = lappend(taskList, task); } return taskList; } /* * truncate_local_data_after_distributing_table truncates the local records of a distributed table. * * The main advantage of this function is to truncate all local records after creating a * distributed table, and prevent constraints from failing due to outdated local records. */ Datum truncate_local_data_after_distributing_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid relationId = PG_GETARG_OID(0); EnsureLocalTableCanBeTruncated(relationId); TruncateStmt *truncateStmt = makeNode(TruncateStmt); char *relationName = generate_qualified_relation_name(relationId); List *names = stringToQualifiedNameList(relationName, NULL); truncateStmt->relations = list_make1(makeRangeVarFromNameList(names)); truncateStmt->restart_seqs = false; truncateStmt->behavior = DROP_CASCADE; set_config_option("citus.enable_ddl_propagation", "false", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); ExecuteTruncate(truncateStmt); set_config_option("citus.enable_ddl_propagation", "true", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); PG_RETURN_VOID(); } /* * EnsureLocalTableCanBeTruncated performs the necessary checks to make sure it * is safe to truncate the local table of a distributed table */ void EnsureLocalTableCanBeTruncated(Oid relationId) { /* error out if the relation is not a distributed table */ if (!IsCitusTable(relationId)) { ereport(ERROR, (errmsg("supplied parameter is not a distributed relation"), errdetail("This UDF only truncates local records of distributed " "tables."))); } List *referencingForeignConstaintsFromLocalTables = GetForeignKeysFromLocalTables(relationId); if (list_length(referencingForeignConstaintsFromLocalTables) > 0) { Oid foreignKeyId = linitial_oid(referencingForeignConstaintsFromLocalTables); Oid referencingRelation = GetReferencingTableId(foreignKeyId); char *referencedRelationName = get_rel_name(relationId); char *referencingRelationName = get_rel_name(referencingRelation); ereport(ERROR, (errmsg("cannot truncate a table referenced in a " "foreign key constraint by a local table"), errdetail("Table \"%s\" references \"%s\"", referencingRelationName, referencedRelationName))); } } /* * PreprocessTruncateStatement handles few things that should be * done before standard process utility is called for truncate * command. */ void PreprocessTruncateStatement(TruncateStmt *truncateStatement) { ErrorIfUnsupportedTruncateStmt(truncateStatement); EnsurePartitionTableNotReplicatedForTruncate(truncateStatement); ExecuteTruncateStmtSequentialIfNecessary(truncateStatement); uint32 lockAcquiringMode = truncateStatement->behavior == DROP_CASCADE ? DIST_LOCK_REFERENCING_TABLES : DIST_LOCK_DEFAULT; AcquireDistributedLockOnRelations(truncateStatement->relations, AccessExclusiveLock, lockAcquiringMode); } /* * ErrorIfUnsupportedTruncateStmt errors out if the command attempts to * truncate a distributed foreign table. */ static void ErrorIfUnsupportedTruncateStmt(TruncateStmt *truncateStatement) { List *relationList = truncateStatement->relations; RangeVar *rangeVar = NULL; foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); ErrorIfIllegallyChangingKnownShard(relationId); /* * We allow truncating foreign tables that are added to metadata * only on the coordinator, as user mappings are not propagated. */ if (IsForeignTable(relationId) && IsCitusTableType(relationId, CITUS_LOCAL_TABLE) && !IsCoordinator()) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("truncating foreign tables that are added to metadata " "can only be executed on the coordinator"))); } } } /* * EnsurePartitionTableNotReplicatedForTruncate a simple wrapper around * EnsurePartitionTableNotReplicated for TRUNCATE command. */ static void EnsurePartitionTableNotReplicatedForTruncate(TruncateStmt *truncateStatement) { RangeVar *rangeVar = NULL; foreach_declared_ptr(rangeVar, truncateStatement->relations) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); if (!IsCitusTable(relationId)) { continue; } EnsurePartitionTableNotReplicated(relationId); } } /* * ExecuteTruncateStmtSequentialIfNecessary decides if the TRUNCATE stmt needs * to run sequential. If so, it calls SetLocalMultiShardModifyModeToSequential(). * * If a reference table which has a foreign key from a distributed table is truncated * we need to execute the command sequentially to avoid self-deadlock. */ static void ExecuteTruncateStmtSequentialIfNecessary(TruncateStmt *command) { List *relationList = command->relations; bool failOK = false; RangeVar *rangeVar = NULL; foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, failOK); if ((IsCitusTableType(relationId, REFERENCE_TABLE) || IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) && TableReferenced(relationId)) { char *relationName = get_rel_name(relationId); ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail( "Table \"%s\" is modified, which might lead " "to data inconsistencies or distributed deadlocks via " "parallel accesses to hash distributed tables due to " "foreign keys. Any parallel modification to " "those hash distributed tables in the same " "transaction can only be executed in sequential query " "execution mode", relationName))); SetLocalMultiShardModifyModeToSequential(); /* nothing to do more */ return; } } } ================================================ FILE: src/backend/distributed/commands/type.c ================================================ /*------------------------------------------------------------------------- * * type.c * Commands for TYPE statements. * The following types are supported in citus * - Composite Types * - Enum Types * - Array Types * * Types that are currently not supporter: * - Range Types * - Base Types * * Range types have a dependency on functions. We can only support Range * types after we have function distribution sorted. * * Base types are more complex and often involve c code from extensions. * These types should be created by creating the extension on all the * workers as well. Therefore types created during the creation of an * extension are not propagated to the worker nodes. * * Types will be created on the workers during the following situations: * - on type creation (except if called in a transaction) * By not distributing types directly when in a transaction allows * the type to be used in a newly created table that will be * distributed in the same transaction. In that case the type will be * created just-in-time to allow citus' parallelism to work. * - just-in-time * When the type is not already distributed but used in an object * that will distribute now. This allows distributed tables to use * types that have not yet been propagated, either due to the * transaction case abvove, or due to a type predating the citus * extension. * - node activation * Together with all objects that are marked as distributed in citus * types will be created during the activation of a new node to allow * reference tables to use this type. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_enum.h" #include "catalog/pg_type.h" #include "commands/extension.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" #define AlterEnumIsRename(stmt) (stmt->oldVal != NULL) #define AlterEnumIsAddValue(stmt) (stmt->oldVal == NULL) #define ALTER_TYPE_OWNER_COMMAND "ALTER TYPE %s OWNER TO %s;" /* guc to turn of the automatic type distribution */ bool EnableCreateTypePropagation = true; /* forward declaration for helper functions*/ static TypeName * MakeTypeNameFromRangeVar(const RangeVar *relation); static Oid GetTypeOwner(Oid typeOid); static Oid LookupNonAssociatedArrayTypeNameOid(ParseState *pstate, const TypeName *typeName, bool missing_ok); /* recreate functions */ static CompositeTypeStmt * RecreateCompositeTypeStmt(Oid typeOid); static List * CompositeTypeColumnDefList(Oid typeOid); static CreateEnumStmt * RecreateEnumStmt(Oid typeOid); static List * EnumValsList(Oid typeOid); /* * PreprocessRenameTypeAttributeStmt is called for changes of attribute names for composite * types. Planning is called before the statement is applied locally. * * For distributed types we apply the attribute renames directly on all the workers to * keep the type in sync across the cluster. */ List * PreprocessRenameTypeAttributeStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); Assert(stmt->relationType == OBJECT_TYPE); List *typeAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, false); /* the code-path only supports a single object */ Assert(list_length(typeAddresses) == 1); if (!ShouldPropagateAnyObject(typeAddresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); EnsureSequentialMode(OBJECT_TYPE); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * CreateTypeStmtByObjectAddress returns a parsetree for the CREATE TYPE statement to * recreate the type by its object address. */ Node * CreateTypeStmtByObjectAddress(const ObjectAddress *address) { Assert(address->classId == TypeRelationId); switch (get_typtype(address->objectId)) { case TYPTYPE_ENUM: { return (Node *) RecreateEnumStmt(address->objectId); } case TYPTYPE_COMPOSITE: { return (Node *) RecreateCompositeTypeStmt(address->objectId); } case TYPTYPE_DOMAIN: { return (Node *) RecreateDomainStmt(address->objectId); } default: { ereport(ERROR, (errmsg("unsupported type to generate create statement for"), errdetail("only enum and composite types can be recreated"))); } } } /* * RecreateCompositeTypeStmt is called for composite types to create its parsetree for the * CREATE TYPE statement that would recreate the composite type. */ static CompositeTypeStmt * RecreateCompositeTypeStmt(Oid typeOid) { Assert(get_typtype(typeOid) == TYPTYPE_COMPOSITE); CompositeTypeStmt *stmt = makeNode(CompositeTypeStmt); List *names = stringToQualifiedNameList(format_type_be_qualified(typeOid), NULL); stmt->typevar = makeRangeVarFromNameList(names); stmt->coldeflist = CompositeTypeColumnDefList(typeOid); return stmt; } /* * attributeFormToColumnDef returns a ColumnDef * describing the field and its property * for a pg_attribute entry. */ static ColumnDef * attributeFormToColumnDef(Form_pg_attribute attributeForm) { return makeColumnDef(NameStr(attributeForm->attname), attributeForm->atttypid, attributeForm->atttypmod, attributeForm->attcollation); } /* * CompositeTypeColumnDefList returns a list of ColumnDef *'s that make up all the fields * of the composite type. */ static List * CompositeTypeColumnDefList(Oid typeOid) { List *columnDefs = NIL; Oid relationId = typeidTypeRelid(typeOid); Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); if (attributeForm->attisdropped) { /* skip logically hidden attributes */ continue; } columnDefs = lappend(columnDefs, attributeFormToColumnDef(attributeForm)); } relation_close(relation, AccessShareLock); return columnDefs; } /* * RecreateEnumStmt returns a parsetree for a CREATE TYPE ... AS ENUM statement that would * recreate the given enum type. */ static CreateEnumStmt * RecreateEnumStmt(Oid typeOid) { Assert(get_typtype(typeOid) == TYPTYPE_ENUM); CreateEnumStmt *stmt = makeNode(CreateEnumStmt); stmt->typeName = stringToQualifiedNameList(format_type_be_qualified(typeOid), NULL); stmt->vals = EnumValsList(typeOid); return stmt; } /* * EnumValsList returns a list of String values containing the enum values for the given * enum type. */ static List * EnumValsList(Oid typeOid) { HeapTuple enum_tuple = NULL; ScanKeyData skey = { 0 }; List *vals = NIL; /* Scan pg_enum for the members of the target enum type. */ ScanKeyInit(&skey, Anum_pg_enum_enumtypid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(typeOid)); Relation enum_rel = table_open(EnumRelationId, AccessShareLock); SysScanDesc enum_scan = systable_beginscan(enum_rel, EnumTypIdSortOrderIndexId, true, NULL, 1, &skey); /* collect all value names in CREATE TYPE ... AS ENUM stmt */ while (HeapTupleIsValid(enum_tuple = systable_getnext(enum_scan))) { Form_pg_enum en = (Form_pg_enum) GETSTRUCT(enum_tuple); vals = lappend(vals, makeString(pstrdup(NameStr(en->enumlabel)))); } systable_endscan(enum_scan); table_close(enum_rel, AccessShareLock); return vals; } /* * CompositeTypeStmtObjectAddress finds the ObjectAddress for the composite type described * by the CompositeTypeStmt. If missing_ok is false this function throws an error if the * type does not exist. * * Never returns NULL, but the objid in the address could be invalid if missing_ok was set * to true. */ List * CompositeTypeStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CompositeTypeStmt *stmt = castNode(CompositeTypeStmt, node); TypeName *typeName = MakeTypeNameFromRangeVar(stmt->typevar); Oid typeOid = LookupNonAssociatedArrayTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * CreateEnumStmtObjectAddress finds the ObjectAddress for the enum type described by the * CreateEnumStmt. If missing_ok is false this function throws an error if the type does * not exist. * * Never returns NULL, but the objid in the address could be invalid if missing_ok was set * to true. */ List * CreateEnumStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateEnumStmt *stmt = castNode(CreateEnumStmt, node); TypeName *typeName = makeTypeNameFromNameList(stmt->typeName); Oid typeOid = LookupNonAssociatedArrayTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * AlterTypeStmtObjectAddress finds the ObjectAddress for the type described by the ALTER * TYPE statement. If missing_ok is false this function throws an error if the type does * not exist. * * Never returns NULL, but the objid in the address could be invalid if missing_ok was set * to true. */ List * AlterTypeStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_TYPE); TypeName *typeName = MakeTypeNameFromRangeVar(stmt->relation); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * AlterEnumStmtObjectAddress return the ObjectAddress of the enum type that is the * object of the AlterEnumStmt. Errors is missing_ok is false. */ List * AlterEnumStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterEnumStmt *stmt = castNode(AlterEnumStmt, node); TypeName *typeName = makeTypeNameFromNameList(stmt->typeName); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * RenameTypeStmtObjectAddress returns the ObjectAddress of the type that is the object * of the RenameStmt. Errors if missing_ok is false. */ List * RenameTypeStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TYPE); TypeName *typeName = makeTypeNameFromNameList((List *) stmt->object); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * AlterTypeSchemaStmtObjectAddress returns the ObjectAddress of the type that is the * object of the AlterObjectSchemaStmt. Errors if missing_ok is false. * * This could be called both before or after it has been applied locally. It will look in * the old schema first, if the type cannot be found in that schema it will look in the * new schema. Errors if missing_ok is false and the type cannot be found in either of the * schemas. */ List * AlterTypeSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TYPE || stmt->objectType == OBJECT_DOMAIN); List *names = (List *) stmt->object; /* * we hardcode missing_ok here during LookupTypeNameOid because if we can't find it it * might have already been moved in this transaction. */ TypeName *typeName = makeTypeNameFromNameList(names); Oid typeOid = LookupTypeNameOid(NULL, typeName, true); if (typeOid == InvalidOid) { /* * couldn't find the type, might have already been moved to the new schema, we * construct a new typename that uses the new schema to search in. */ /* typename is the last in the list of names */ String *typeNameStr = lfirst(list_tail(names)); /* * we don't error here either, as the error would be not a good user facing * error if the type didn't exist in the first place. */ List *newNames = list_make2(makeString(stmt->newschema), typeNameStr); TypeName *newTypeName = makeTypeNameFromNameList(newNames); typeOid = LookupTypeNameOid(NULL, newTypeName, true); /* * if the type is still invalid we couldn't find the type, error with the same * message postgres would error with it missing_ok is false (not ok to miss) */ if (!missing_ok && typeOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("type \"%s\" does not exist", TypeNameToString(typeName)))); } } ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * RenameTypeAttributeStmtObjectAddress returns the ObjectAddress of the type that is the * object of the RenameStmt. Errors if missing_ok is false. * * The ObjectAddress is that of the type, not that of the attributed for which the name is * changed as Attributes are not distributed on their own but as a side effect of the * whole type distribution. */ List * RenameTypeAttributeStmtObjectAddress(Node *node, bool missing_ok) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); Assert(stmt->relationType == OBJECT_TYPE); TypeName *typeName = MakeTypeNameFromRangeVar(stmt->relation); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * AlterTypeOwnerObjectAddress returns the ObjectAddress of the type that is the object * of the AlterOwnerStmt. Errors if missing_ok is false. */ List * AlterTypeOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TYPE); TypeName *typeName = makeTypeNameFromNameList((List *) stmt->object); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, TypeRelationId, typeOid); return list_make1(address); } /* * CreateTypeDDLCommandsIdempotent returns a list of DDL statements (const char *) to be * executed on a node to recreate the type addressed by the typeAddress. */ List * CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress) { List *ddlCommands = NIL; StringInfoData buf = { 0 }; Assert(typeAddress->classId == TypeRelationId); if (type_is_array(typeAddress->objectId)) { /* * array types cannot be created on their own, but could be a direct dependency of * a table. In that case they are on the dependency graph and tried to be created. * * By returning an empty list we will not send any commands to create this type. */ return NIL; } HeapTuple tup = SearchSysCacheCopy1(TYPEOID, ObjectIdGetDatum(typeAddress->objectId)); if (!HeapTupleIsValid(tup)) { elog(ERROR, "cache lookup failed for type %u", typeAddress->objectId); } /* Don't send any command if the type is a table's row type */ Form_pg_type typTup = (Form_pg_type) GETSTRUCT(tup); if (typTup->typtype == TYPTYPE_COMPOSITE && get_rel_relkind(typTup->typrelid) != RELKIND_COMPOSITE_TYPE) { return NIL; } Node *stmt = CreateTypeStmtByObjectAddress(typeAddress); /* capture ddl command for recreation and wrap in create if not exists construct */ const char *ddlCommand = DeparseTreeNode(stmt); ddlCommand = WrapCreateOrReplace(ddlCommand); ddlCommands = lappend(ddlCommands, (void *) ddlCommand); /* add owner ship change so the creation command can be run as a different user */ const char *username = GetUserNameFromId(GetTypeOwner(typeAddress->objectId), false); initStringInfo(&buf); appendStringInfo(&buf, ALTER_TYPE_OWNER_COMMAND, getObjectIdentity(typeAddress, false), quote_identifier(username)); ddlCommands = lappend(ddlCommands, buf.data); return ddlCommands; } /* * GenerateBackupNameForTypeCollision generates a new type name for an existing type. The * name is generated in such a way that the new name doesn't overlap with an existing type * by adding a suffix with incrementing number after the new name. */ char * GenerateBackupNameForTypeCollision(const ObjectAddress *address) { List *names = stringToQualifiedNameList(format_type_be_qualified(address->objectId), NULL); RangeVar *rel = makeRangeVarFromNameList(names); char *newName = palloc0(NAMEDATALEN); char suffix[NAMEDATALEN] = { 0 }; char *baseName = rel->relname; int baseLength = strlen(baseName); int count = 0; while (true) { int suffixLength = SafeSnprintf(suffix, NAMEDATALEN - 1, "(citus_backup_%d)", count); /* trim the base name at the end to leave space for the suffix and trailing \0 */ baseLength = Min(baseLength, NAMEDATALEN - suffixLength - 1); /* clear newName before copying the potentially trimmed baseName and suffix */ memset(newName, 0, NAMEDATALEN); strncpy_s(newName, NAMEDATALEN, baseName, baseLength); strncpy_s(newName + baseLength, NAMEDATALEN - baseLength, suffix, suffixLength); rel->relname = newName; TypeName *newTypeName = makeTypeNameFromNameList(MakeNameListFromRangeVar(rel)); Oid typeOid = LookupTypeNameOid(NULL, newTypeName, true); if (typeOid == InvalidOid) { return newName; } count++; } } /* * GetTypeOwner * * Given the type OID, find its owner */ static Oid GetTypeOwner(Oid typeOid) { Oid result = InvalidOid; HeapTuple tp = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeOid)); if (HeapTupleIsValid(tp)) { Form_pg_type typtup = (Form_pg_type) GETSTRUCT(tp); result = typtup->typowner; ReleaseSysCache(tp); } return result; } /* * MakeTypeNameFromRangeVar creates a TypeName based on a RangeVar. */ static TypeName * MakeTypeNameFromRangeVar(const RangeVar *relation) { List *names = NIL; if (relation->schemaname) { names = lappend(names, makeString(relation->schemaname)); } names = lappend(names, makeString(relation->relname)); return makeTypeNameFromNameList(names); } /* * LookupNonAssociatedArrayTypeNameOid returns the oid of the type with the given type name * that is not an array type that is associated to another user defined type. */ static Oid LookupNonAssociatedArrayTypeNameOid(ParseState *pstate, const TypeName *typeName, bool missing_ok) { Type tup = LookupTypeName(NULL, typeName, NULL, missing_ok); Oid typeOid = InvalidOid; if (tup != NULL) { if (((Form_pg_type) GETSTRUCT(tup))->typelem == 0) { typeOid = ((Form_pg_type) GETSTRUCT(tup))->oid; } ReleaseSysCache(tup); } if (!missing_ok && typeOid == InvalidOid) { elog(ERROR, "type \"%s\" that is not an array type associated with " "another type does not exist", TypeNameToString(typeName)); } return typeOid; } ================================================ FILE: src/backend/distributed/commands/utility_hook.c ================================================ /*------------------------------------------------------------------------- * utility_hook.c * Citus utility hook and related functionality. * * The utility hook is called by PostgreSQL when processing any command * that is not SELECT, UPDATE, DELETE, INSERT, in place of the regular * PostprocessUtility function. We use this primarily to implement (or in * some cases prevent) DDL commands and COPY on distributed tables. * * For DDL commands that affect distributed tables, we check whether * they are valid (and implemented) for the distributed table and then * propagate the command to all shards and, in case of MX, to distributed * tables on other nodes. We still call the original ProcessUtility * function to apply catalog changes on the coordinator. * * For COPY into a distributed table, we provide an alternative * implementation in ProcessCopyStmt that sends rows to shards based * on their distribution column value instead of writing it to the local * table on the coordinator. For COPY from a distributed table, we * replace the table with a SELECT * FROM table and pass it back to * PostprocessUtility, which will plan the query via the distributed planner * hook. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/attnum.h" #include "access/heapam.h" #include "access/htup_details.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/pg_authid.h" #include "catalog/pg_database.h" #include "commands/dbcommands.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/tablecmds.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "postmaster/postmaster.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "citus_version.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_depended_object.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" /* IWYU pragma: keep */ #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" #include "distributed/executor_util.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/maintenanced.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_logical_replication.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/string_utils.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" bool EnableDDLPropagation = true; /* ddl propagation is enabled */ int CreateObjectPropagationMode = CREATE_OBJECT_PROPAGATION_IMMEDIATE; PropSetCmdBehavior PropagateSetCommands = PROPSETCMD_NONE; /* SET prop off */ static bool shouldInvalidateForeignKeyGraph = false; static int activeAlterTables = 0; static int activeDropSchemaOrDBs = 0; static bool ConstraintDropped = false; ProcessUtility_hook_type PrevProcessUtility = NULL; int UtilityHookLevel = 0; /* Local functions forward declarations for helper functions */ static void citus_ProcessUtilityInternal(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag); static void set_indexsafe_procflags(void); static char * CurrentSearchPath(void); static void IncrementUtilityHookCountersIfNecessary(Node *parsetree); static void PostStandardProcessUtility(Node *parsetree); static void DecrementUtilityHookCountersIfNecessary(Node *parsetree); static bool IsDropSchemaOrDB(Node *parsetree); static bool ShouldCheckUndistributeCitusLocalTables(void); /* * ProcessUtilityParseTree is a convenience method to create a PlannedStmt out of * pieces of a utility statement before invoking ProcessUtility. */ void ProcessUtilityParseTree(Node *node, const char *queryString, ProcessUtilityContext context, ParamListInfo params, DestReceiver *dest, QueryCompletion *completionTag) { PlannedStmt *plannedStmt = makeNode(PlannedStmt); plannedStmt->commandType = CMD_UTILITY; plannedStmt->utilityStmt = node; ProcessUtility(plannedStmt, queryString, false, context, params, NULL, dest, completionTag); } /* * citus_ProcessUtility is the main entry hook for implementing Citus-specific * utility behavior. Its primary responsibilities are intercepting COPY and DDL * commands and augmenting the coordinator's command with corresponding tasks * to be run on worker nodes, after suitably ensuring said commands' options * are fully supported by Citus. Much of the DDL behavior is toggled by Citus' * enable_ddl_propagation GUC. In addition to DDL and COPY, utilities such as * TRUNCATE and VACUUM are also supported. */ void citus_ProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag) { if (readOnlyTree) { pstmt = copyObject(pstmt); } Node *parsetree = pstmt->utilityStmt; if (IsA(parsetree, TransactionStmt)) { TransactionStmt *transactionStmt = (TransactionStmt *) parsetree; if (context == PROCESS_UTILITY_TOPLEVEL && (transactionStmt->kind == TRANS_STMT_BEGIN || transactionStmt->kind == TRANS_STMT_START)) { SaveBeginCommandProperties(transactionStmt); } } if (IsA(parsetree, TransactionStmt) || IsA(parsetree, ListenStmt) || IsA(parsetree, NotifyStmt) || IsA(parsetree, ExecuteStmt) || IsA(parsetree, PrepareStmt) || IsA(parsetree, DiscardStmt) || IsA(parsetree, DeallocateStmt) || IsA(parsetree, DeclareCursorStmt) || IsA(parsetree, FetchStmt)) { /* * Skip additional checks for common commands that do not have any * Citus-specific logic. * * Transaction statements (e.g. ABORT, COMMIT) can be run in aborted * transactions in which case a lot of checks cannot be done safely in * that state. Since we never need to intercept transaction statements, * skip our checks and immediately fall into standard_ProcessUtility. */ PrevProcessUtility(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); return; } bool isCreateAlterExtensionUpdateCitusStmt = IsCreateAlterExtensionUpdateCitusStmt( parsetree); if (EnableVersionChecks && isCreateAlterExtensionUpdateCitusStmt) { ErrorIfUnstableCreateOrAlterExtensionStmt(parsetree); } if (IsA(parsetree, CreateExtensionStmt)) { /* * Postgres forbids creating/altering other extensions from within an extension script, so we use a utility hook instead * This preprocess check whether citus_columnar should be installed first before citus */ PreprocessCreateExtensionStmtForCitusColumnar(parsetree); } if (isCreateAlterExtensionUpdateCitusStmt || IsDropCitusExtensionStmt(parsetree)) { /* * Citus maintains a higher level cache. We use the cache invalidation mechanism * of Postgres to achieve cache coherency between backends. Any change to citus * extension should be made known to other backends. We do this by invalidating the * relcache and therefore invoking the citus registered callback that invalidates * the citus cache in other backends. */ CacheInvalidateRelcacheAll(); } /* * Make sure that on DROP DATABASE we terminate the background daemon * associated with it. */ if (IsA(parsetree, DropdbStmt)) { const bool missingOK = true; DropdbStmt *dropDbStatement = (DropdbStmt *) parsetree; char *dbname = dropDbStatement->dbname; Oid databaseOid = get_database_oid(dbname, missingOK); if (OidIsValid(databaseOid)) { StopMaintenanceDaemon(databaseOid); } } if (!CitusHasBeenLoaded()) { /* * Process the command via RunPreprocessNonMainDBCommand and * RunPostprocessNonMainDBCommand hooks if we're in a non-main database * and if the command is a node-wide object management command that we * support from non-main databases. */ bool shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree); if (!shouldSkipPrevUtilityHook) { /* * Ensure that utility commands do not behave any differently until CREATE * EXTENSION is invoked. */ PrevProcessUtility(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); } RunPostprocessNonMainDBCommand(parsetree); return; } else if (IsA(parsetree, CallStmt)) { CallStmt *callStmt = (CallStmt *) parsetree; /* * If the procedure is distributed and we are using MX then we have the * possibility of calling it on the worker. If the data is located on * the worker this can avoid making many network round trips. */ if (context == PROCESS_UTILITY_TOPLEVEL && CallDistributedProcedureRemotely(callStmt, dest)) { return; } /* * Stored procedures are a bit strange in the sense that some statements * are not in a transaction block, but can be rolled back. We need to * make sure we send all statements in a transaction block. The * StoredProcedureLevel variable signals this to the router executor * and indicates how deep in the call stack we are in case of nested * stored procedures. */ StoredProcedureLevel += 1; PG_TRY(); { PrevProcessUtility(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); StoredProcedureLevel -= 1; if (InDelegatedProcedureCall && StoredProcedureLevel == 0) { InDelegatedProcedureCall = false; } } PG_CATCH(); { StoredProcedureLevel -= 1; if (InDelegatedProcedureCall && StoredProcedureLevel == 0) { InDelegatedProcedureCall = false; } PG_RE_THROW(); } PG_END_TRY(); return; } else if (IsA(parsetree, DoStmt)) { /* * All statements in a DO block are executed in a single transaction, * so we need to keep track of whether we are inside a DO block. */ DoBlockLevel += 1; PG_TRY(); { PrevProcessUtility(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); DoBlockLevel -= 1; } PG_CATCH(); { DoBlockLevel -= 1; PG_RE_THROW(); } PG_END_TRY(); return; } UtilityHookLevel++; PG_TRY(); { citus_ProcessUtilityInternal(pstmt, queryString, context, params, queryEnv, dest, completionTag); if (UtilityHookLevel == 1) { /* * When Citus local tables are disconnected from the foreign key graph, which * can happen due to various kinds of drop commands, we immediately * undistribute them at the end of the command. */ if (ShouldCheckUndistributeCitusLocalTables()) { UndistributeDisconnectedCitusLocalTables(); } ResetConstraintDropped(); /* * We're only interested in top-level CREATE TABLE commands * to create a tenant schema table or a Citus managed table. */ if (context == PROCESS_UTILITY_TOPLEVEL && (IsA(parsetree, CreateStmt) || IsA(parsetree, CreateForeignTableStmt) || IsA(parsetree, CreateTableAsStmt))) { Node *createStmt = NULL; if (IsA(parsetree, CreateTableAsStmt)) { createStmt = parsetree; } else { /* * Not directly cast to CreateStmt to guard against the case where * the definition of CreateForeignTableStmt changes in future. */ createStmt = IsA(parsetree, CreateStmt) ? parsetree : (Node *) &(((CreateForeignTableStmt *) parsetree)->base); } ConvertNewTableIfNecessary(createStmt); } if (context == PROCESS_UTILITY_TOPLEVEL && IsA(parsetree, AlterObjectSchemaStmt)) { AlterObjectSchemaStmt *alterSchemaStmt = castNode(AlterObjectSchemaStmt, parsetree); if (alterSchemaStmt->objectType == OBJECT_TABLE || alterSchemaStmt->objectType == OBJECT_FOREIGN_TABLE) { ConvertToTenantTableIfNecessary(alterSchemaStmt); } } } UtilityHookLevel--; } PG_CATCH(); { if (UtilityHookLevel == 1) { ResetConstraintDropped(); } UtilityHookLevel--; PG_RE_THROW(); } PG_END_TRY(); } /* * citus_ProcessUtilityInternal is a helper function for citus_ProcessUtility where majority * of the Citus specific utility statements are handled here. The distinction between * both functions is that Citus_ProcessUtility does not handle CALL and DO statements. * The reason for the distinction is implemented to be able to find the "top-level" DDL * commands (not internal/cascading ones). UtilityHookLevel variable is used to achieve * this goal. */ static void citus_ProcessUtilityInternal(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag) { Node *parsetree = pstmt->utilityStmt; List *ddlJobs = NIL; DistOpsValidationState distOpsValidationState = HasNoneValidObject; if (IsA(parsetree, ExplainStmt) && IsA(((ExplainStmt *) parsetree)->query, Query)) { ExplainStmt *explainStmt = (ExplainStmt *) parsetree; if (IsTransactionBlock()) { bool analyze = false; DefElem *option = NULL; foreach_declared_ptr(option, explainStmt->options) { if (strcmp(option->defname, "analyze") == 0) { analyze = defGetBoolean(option); } /* don't "break", as explain.c will use the last value */ } if (analyze) { /* * Since we cannot execute EXPLAIN ANALYZE locally, we * cannot continue. */ ErrorIfTransactionAccessedPlacementsLocally(); } } /* * EXPLAIN ANALYZE is tricky with local execution, and there is not * much difference between the local and distributed execution in terms * of the actual EXPLAIN output. * * TODO: It might be nice to have a way to show that the query is locally * executed. Shall we add a INFO output? */ DisableLocalExecution(); } if (IsA(parsetree, CreateSubscriptionStmt)) { CreateSubscriptionStmt *createSubStmt = (CreateSubscriptionStmt *) parsetree; parsetree = ProcessCreateSubscriptionStmt(createSubStmt); } /* * For security and reliability reasons we disallow altering and dropping * subscriptions created by citus by non superusers. We could probably * disallow this for all subscriptions without issues. But out of an * abundance of caution for breaking subscription logic created by users * for other purposes, we only disallow it for the subscriptions that we * create i.e. ones that start with "citus_". */ if (IsA(parsetree, AlterSubscriptionStmt)) { AlterSubscriptionStmt *alterSubStmt = (AlterSubscriptionStmt *) parsetree; if (!superuser() && StringStartsWith(alterSubStmt->subname, "citus_")) { ereport(ERROR, ( errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg( "Only superusers can alter subscriptions that are created by citus"))); } } if (IsA(parsetree, DropSubscriptionStmt)) { DropSubscriptionStmt *dropSubStmt = (DropSubscriptionStmt *) parsetree; if (!superuser() && StringStartsWith(dropSubStmt->subname, "citus_")) { ereport(ERROR, ( errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg( "Only superusers can drop subscriptions that are created by citus"))); } } /* * Process SET LOCAL and SET TRANSACTION statements in multi-statement * transactions. */ if (IsA(parsetree, VariableSetStmt)) { VariableSetStmt *setStmt = (VariableSetStmt *) parsetree; /* at present, we only implement the NONE and LOCAL behaviors */ Assert(PropagateSetCommands == PROPSETCMD_NONE || PropagateSetCommands == PROPSETCMD_LOCAL); if (IsMultiStatementTransaction() && ShouldPropagateSetCommand(setStmt)) { PostprocessVariableSetStmt(setStmt, queryString); } } if (IsA(parsetree, CopyStmt)) { MemoryContext planContext = GetMemoryChunkContext(parsetree); parsetree = copyObject(parsetree); parsetree = ProcessCopyStmt((CopyStmt *) parsetree, completionTag, queryString); if (parsetree == NULL) { return; } MemoryContext previousContext = MemoryContextSwitchTo(planContext); parsetree = copyObject(parsetree); MemoryContextSwitchTo(previousContext); /* * we need to set the parsetree here already as we copy and replace the original * parsetree during ddl propagation. In reality we need to refactor the code above * to not juggle copy the parsetree and leak it to a potential cache above the * utility hook. */ pstmt->utilityStmt = parsetree; } /* we're mostly in DDL (and VACUUM/TRUNCATE) territory at this point... */ if (IsA(parsetree, CreateSeqStmt)) { ErrorIfUnsupportedSeqStmt((CreateSeqStmt *) parsetree); } if (IsA(parsetree, AlterSeqStmt)) { ErrorIfDistributedAlterSeqOwnedBy((AlterSeqStmt *) parsetree); } if (IsA(parsetree, TruncateStmt)) { PreprocessTruncateStatement((TruncateStmt *) parsetree); } if (IsA(parsetree, LockStmt)) { /* * PreprocessLockStatement might lock the relations locally if the * node executing the command is in pg_dist_node. Even though the process * utility will re-acquire the locks across the same relations if the node * is in the metadata (in the pg_dist_node table) that should not be a problem, * plus it ensures consistent locking order between the nodes. */ PreprocessLockStatement((LockStmt *) parsetree, context); } /* * We only process ALTER TABLE ... ATTACH PARTITION commands in the function below * and distribute the partition if necessary. */ if (IsA(parsetree, AlterTableStmt)) { AlterTableStmt *alterTableStatement = (AlterTableStmt *) parsetree; PreprocessAlterTableStmtAttachPartition(alterTableStatement, queryString); } /* only generate worker DDLJobs if propagation is enabled */ const DistributeObjectOps *ops = NULL; if (EnableDDLPropagation) { /* copy planned statement since we might scribble on it or its utilityStmt */ pstmt = copyObject(pstmt); parsetree = pstmt->utilityStmt; ops = GetDistributeObjectOps(parsetree); /* * Preprocess and qualify steps can cause pg tests to fail because of the * unwanted citus related warnings or early error logs related to invalid address. * Therefore, we first check if any address in the given statement is valid. * Then, we do not execute qualify and preprocess if none of the addresses are valid * or any address violates ownership rules to prevent before-mentioned citus related * messages. PG will complain about the invalid address or ownership violation, so we * are safe to not execute qualify and preprocess. Also note that we should not guard * any step after standardProcess_Utility with the enum state distOpsValidationState * because PG would have already failed the transaction. */ distOpsValidationState = DistOpsValidityState(parsetree, ops); /* * For some statements Citus defines a Qualify function. The goal of this function * is to take any ambiguity from the statement that is contextual on either the * search_path or current settings. * Instead of relying on the search_path and settings we replace any deduced bits * and fill them out how postgres would resolve them. This makes subsequent * deserialize calls for the statement portable to other postgres servers, the * workers in our case. * If there are no valid objects or any object violates ownership, let's skip * the qualify and preprocess, and do not diverge from Postgres in terms of * error messages. */ if (ops && ops->qualify && DistOpsInValidState(distOpsValidationState)) { ops->qualify(parsetree); } if (ops && ops->preprocess && DistOpsInValidState(distOpsValidationState)) { ddlJobs = ops->preprocess(parsetree, queryString, context); } } else { /* * citus.enable_ddl_propagation is disabled, which means that PostgreSQL * should handle the DDL command on a distributed table directly, without * Citus intervening. The only exception is partition column drop, in * which case we error out. Advanced Citus users use this to implement their * own DDL propagation. We also use it to avoid re-propagating DDL commands * when changing MX tables on workers. Below, we also make sure that DDL * commands don't run queries that might get intercepted by Citus and error * out during planning, specifically we skip validation in foreign keys. */ if (IsA(parsetree, AlterTableStmt)) { AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree; if (alterTableStmt->objtype == OBJECT_TABLE || alterTableStmt->objtype == OBJECT_FOREIGN_TABLE) { ErrorIfAlterDropsPartitionColumn(alterTableStmt); /* * When issuing an ALTER TABLE ... ADD FOREIGN KEY command, the * the validation step should be skipped on the distributed table. * Therefore, we check whether the given ALTER TABLE statement is a * FOREIGN KEY constraint and if so disable the validation step. * Note validation is done on the shard level when DDL propagation * is enabled. The following eagerly executes some tasks on workers. */ SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt, false); } } } /* * If we've explicitly set the citus.skip_constraint_validation GUC, then * we skip validation of any added constraints. */ if (IsA(parsetree, AlterTableStmt) && SkipConstraintValidation) { AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree; AlterTableCmd *command = NULL; foreach_declared_ptr(command, alterTableStmt->cmds) { AlterTableType alterTableType = command->subtype; /* * XXX: In theory we could probably use this GUC to skip validation * of VALIDATE CONSTRAINT and ALTER CONSTRAINT too. But currently * this is not needed, so we make its behaviour only apply to ADD * CONSTRAINT. */ if (alterTableType == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; constraint->skip_validation = true; } } } /* inform the user about potential caveats */ if (IsA(parsetree, CreatedbStmt) && !EnableCreateDatabasePropagation) { if (EnableUnsupportedFeatureMessages) { ereport(NOTICE, (errmsg("Citus partially supports CREATE DATABASE for " "distributed databases"), errdetail("Citus does not propagate CREATE DATABASE " "command to other nodes"), errhint("You can manually create a database and its " "extensions on other nodes."))); } } else if (IsA(parsetree, CreateRoleStmt) && !EnableCreateRolePropagation) { ereport(NOTICE, (errmsg("not propagating CREATE ROLE/USER commands to other" " nodes"), errhint("Connect to other nodes directly to manually create all" " necessary users and roles."))); } else if (IsA(parsetree, SecLabelStmt) && !EnableAlterRolePropagation) { ereport(NOTICE, (errmsg("not propagating SECURITY LABEL commands to other" " nodes"), errhint("Connect to other nodes directly to manually assign" " necessary labels."))); } /* * Make sure that on DROP EXTENSION we terminate the background daemon * associated with it. */ if (IsDropCitusExtensionStmt(parsetree)) { StopMaintenanceDaemon(MyDatabaseId); } /* * Make sure that dropping node-wide objects deletes the pg_dist_object * entries. There is a separate logic for node-wide objects (such as role * and databases), since they are not included as dropped objects in the * drop event trigger. To handle it both on worker and coordinator nodes, * it is not implemented as a part of process functions but here. */ UnmarkNodeWideObjectsDistributed(parsetree); pstmt->utilityStmt = parsetree; PG_TRY(); { IncrementUtilityHookCountersIfNecessary(parsetree); /* * Check if we are running ALTER EXTENSION citus UPDATE (TO "") command and * the available version is different than the current version of Citus. In this case, * ALTER EXTENSION citus UPDATE command can actually update Citus to a new version. */ bool isCreateAlterExtensionUpdateCitusStmt = IsCreateAlterExtensionUpdateCitusStmt(parsetree); bool isAlterExtensionUpdateCitusStmt = isCreateAlterExtensionUpdateCitusStmt && IsA(parsetree, AlterExtensionStmt); bool citusCanBeUpdatedToAvailableVersion = false; if (isAlterExtensionUpdateCitusStmt) { citusCanBeUpdatedToAvailableVersion = !InstalledAndAvailableVersionsSame(); /* * Check whether need to install/drop citus_columnar when upgrade/downgrade citus */ PreprocessAlterExtensionCitusStmtForCitusColumnar(parsetree); } PrevProcessUtility(pstmt, queryString, false, context, params, queryEnv, dest, completionTag); if (isAlterExtensionUpdateCitusStmt) { /* * Post process, upgrade citus_columnar from fake internal version to normal version if upgrade citus * or drop citus_columnar fake version when downgrade citus to older version that do not support * citus_columnar */ PostprocessAlterExtensionCitusStmtForCitusColumnar(parsetree); } /* * if we are running ALTER EXTENSION citus UPDATE (to "") command, we may need * to mark existing objects as distributed depending on the "version" parameter if * specified in "ALTER EXTENSION citus UPDATE" command */ if (isAlterExtensionUpdateCitusStmt && citusCanBeUpdatedToAvailableVersion) { PostprocessAlterExtensionCitusUpdateStmt(parsetree); } PostStandardProcessUtility(parsetree); } PG_CATCH(); { PostStandardProcessUtility(parsetree); PG_RE_THROW(); } PG_END_TRY(); /* * Post process for ddl statements */ if (EnableDDLPropagation) { if (ops && ops->postprocess) { List *processJobs = ops->postprocess(parsetree, queryString); if (processJobs) { Assert(ddlJobs == NIL); /* jobs should not have been set before */ ddlJobs = processJobs; } } } if (IsA(parsetree, CreateStmt)) { CreateStmt *createStatement = (CreateStmt *) parsetree; PostprocessCreateTableStmt(createStatement, queryString); } if (IsA(parsetree, CreateForeignTableStmt)) { CreateForeignTableStmt *createForeignTableStmt = (CreateForeignTableStmt *) parsetree; CreateStmt *createTableStmt = (CreateStmt *) (&createForeignTableStmt->base); PostprocessCreateTableStmt(createTableStmt, queryString); } /* after local command has completed, finish by executing worker DDLJobs, if any */ if (ddlJobs != NIL) { if (IsA(parsetree, AlterTableStmt)) { PostprocessAlterTableStmt(castNode(AlterTableStmt, parsetree)); } if (IsA(parsetree, GrantStmt)) { GrantStmt *grantStmt = (GrantStmt *) parsetree; if (grantStmt->targtype == ACL_TARGET_ALL_IN_SCHEMA) { /* * Grant .. IN SCHEMA causes a deadlock if we don't use local execution * because standard process utility processes the shard placements as well * and the row-level locks in pg_class will not be released until the current * transaction commits. We could skip the local shard placements after standard * process utility, but for simplicity we just prefer using local execution. */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); } } DDLJob *ddlJob = NULL; foreach_declared_ptr(ddlJob, ddlJobs) { ExecuteDistributedDDLJob(ddlJob); } if (IsA(parsetree, AlterTableStmt)) { /* * This postprocess needs to be done after the distributed ddl jobs have * been executed in the workers, hence is separate from PostprocessAlterTableStmt. * We might have wrong index names generated on indexes of shards of partitions, * so we perform the relevant checks and index renaming here. */ FixAlterTableStmtIndexNames(castNode(AlterTableStmt, parsetree)); } /* * For CREATE/DROP/REINDEX CONCURRENTLY we mark the index as valid * after successfully completing the distributed DDL job. */ if (IsA(parsetree, IndexStmt)) { IndexStmt *indexStmt = (IndexStmt *) parsetree; if (indexStmt->concurrent) { /* no failures during CONCURRENTLY, mark the index as valid */ MarkIndexValid(indexStmt); } /* * We make sure schema name is not null in the PreprocessIndexStmt. */ Oid schemaId = get_namespace_oid(indexStmt->relation->schemaname, true); Oid relationId = get_relname_relid(indexStmt->relation->relname, schemaId); /* * If this a partitioned table, and CREATE INDEX was not run with ONLY, * we have wrong index names generated on indexes of shards of * partitions of this table, so we should fix them. */ if (IsCitusTable(relationId) && PartitionedTable(relationId) && indexStmt->relation->inh) { /* only fix this specific index */ Oid indexRelationId = get_relname_relid(indexStmt->idxname, schemaId); FixPartitionShardIndexNames(relationId, indexRelationId); } } /* * Since we must have objects on workers before distributing them, * mark object distributed as the last step. */ if (ops && ops->markDistributed) { List *addresses = GetObjectAddressListFromParseTree(parsetree, false, true); ObjectAddress *address = NULL; foreach_declared_ptr(address, addresses) { MarkObjectDistributed(address); TrackPropagatedObject(address); } } } } /* * UndistributeDisconnectedCitusLocalTables undistributes citus local tables that * are not connected to any reference tables via their individual foreign key * subgraphs. Note that this function undistributes only the auto-converted tables, * i.e the ones that are converted by Citus by cascading through foreign keys. */ void UndistributeDisconnectedCitusLocalTables(void) { List *citusLocalTableIdList = CitusTableTypeIdList(CITUS_LOCAL_TABLE); citusLocalTableIdList = SortList(citusLocalTableIdList, CompareOids); Oid citusLocalTableId = InvalidOid; foreach_declared_oid(citusLocalTableId, citusLocalTableIdList) { /* acquire ShareRowExclusiveLock to prevent concurrent foreign key creation */ LOCKMODE lockMode = ShareRowExclusiveLock; LockRelationOid(citusLocalTableId, lockMode); HeapTuple heapTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(citusLocalTableId)); if (!HeapTupleIsValid(heapTuple)) { /* * UndistributeTable drops relation, skip if already undistributed * via cascade. */ continue; } ReleaseSysCache(heapTuple); if (PartitionTable(citusLocalTableId)) { /* we skip here, we'll undistribute from the parent if necessary */ UnlockRelationOid(citusLocalTableId, lockMode); continue; } if (!ShouldUndistributeCitusLocalTable(citusLocalTableId)) { /* still connected to a reference table, skip it */ UnlockRelationOid(citusLocalTableId, lockMode); continue; } /* * Citus local table is not connected to any reference tables, then * undistribute it via cascade. Here, instead of first dropping foreing * keys then undistributing the table, we just set cascadeViaForeignKeys * to true for simplicity. * * We suppress notices messages not to be too verbose. On the other hand, * as UndistributeTable moves data to a new table, we want to inform user * as it might take some time. */ ereport(NOTICE, (errmsg("removing table %s from metadata as it is not " "connected to any reference tables via foreign keys", generate_qualified_relation_name(citusLocalTableId)))); TableConversionParameters params = { .relationId = citusLocalTableId, .cascadeViaForeignKeys = true, .suppressNoticeMessages = true, .bypassTenantCheck = false }; UndistributeTable(¶ms); } } /* * ShouldCheckUndistributeCitusLocalTables returns true if we might need to check * citus local tables for undistributing automatically. */ static bool ShouldCheckUndistributeCitusLocalTables(void) { if (!ConstraintDropped) { /* * citus_drop_trigger executes notify_constraint_dropped to set * ConstraintDropped to true, which means that last command dropped * a table constraint. */ return false; } if (!CitusHasBeenLoaded()) { /* * If we are dropping citus, we should not try to undistribute citus * local tables as they will also be dropped. */ return false; } if (!InCoordinatedTransaction()) { /* not interacting with any Citus objects */ return false; } if (IsCitusInternalBackend() || IsRebalancerInternalBackend()) { /* connection from the coordinator operating on a shard */ return false; } if (!ShouldEnableLocalReferenceForeignKeys()) { /* * If foreign keys between reference tables and local tables are * disabled, then user might be using citus_add_local_table_to_metadata for * their own purposes. In that case, we should not undistribute * citus local tables. */ return false; } if (!IsCoordinator()) { /* we should not perform this operation in worker nodes */ return false; } return true; } /* * NotifyUtilityHookConstraintDropped sets ConstraintDropped to true to tell us * last command dropped a table constraint. */ void NotifyUtilityHookConstraintDropped(void) { ConstraintDropped = true; } /* * ResetConstraintDropped sets ConstraintDropped to false. */ void ResetConstraintDropped(void) { ConstraintDropped = false; } /* * IsDropSchemaOrDB returns true if parsetree represents DROP SCHEMA ...or * a DROP DATABASE. */ static bool IsDropSchemaOrDB(Node *parsetree) { if (!IsA(parsetree, DropStmt)) { return false; } DropStmt *dropStatement = (DropStmt *) parsetree; return (dropStatement->removeType == OBJECT_SCHEMA) || (dropStatement->removeType == OBJECT_DATABASE); } /* * ExecuteDistributedDDLJob simply executes a provided DDLJob in a distributed trans- * action, including metadata sync if needed. If the multi shard commit protocol is * in its default value of '1pc', then a notice message indicating that '2pc' might be * used for extra safety. In the commit protocol, a BEGIN is sent after connection to * each shard placement and COMMIT/ROLLBACK is handled by * CoordinatedTransactionCallback function. * * The function errors out if the DDL is on a partitioned table which has replication * factor > 1, or if the the coordinator is not added into metadata and we're on a * worker node because we want to make sure that distributed DDL jobs are executed * on the coordinator node too. See EnsurePropagationToCoordinator() for more details. */ void ExecuteDistributedDDLJob(DDLJob *ddlJob) { bool shouldSyncMetadata = false; EnsurePropagationToCoordinator(); ObjectAddress targetObjectAddress = ddlJob->targetObjectAddress; if (OidIsValid(targetObjectAddress.classId)) { /* * Only for ddlJobs that are targetting an object we want to sync * its metadata. */ shouldSyncMetadata = ShouldSyncUserCommandForObject(targetObjectAddress); if (targetObjectAddress.classId == RelationRelationId) { EnsurePartitionTableNotReplicated(targetObjectAddress.objectId); } } bool localExecutionSupported = true; if (!TaskListCannotBeExecutedInTransaction(ddlJob->taskList)) { if (shouldSyncMetadata) { SendCommandToRemoteNodesWithMetadata(DISABLE_DDL_PROPAGATION); char *currentSearchPath = CurrentSearchPath(); /* * Given that we're relaying the query to the remote nodes directly, * we should set the search path exactly the same when necessary. */ if (currentSearchPath != NULL) { SendCommandToRemoteNodesWithMetadata( psprintf("SET LOCAL search_path TO %s;", currentSearchPath)); } if (ddlJob->metadataSyncCommand != NULL) { SendCommandToRemoteNodesWithMetadata( (char *) ddlJob->metadataSyncCommand); } } ExecuteUtilityTaskList(ddlJob->taskList, localExecutionSupported); } else { localExecutionSupported = false; /* * Start a new transaction to make sure CONCURRENTLY commands * on localhost do not block waiting for this transaction to finish. * * In addition to doing that, we also need to tell other backends * --including the ones spawned for connections opened to localhost to * build indexes on shards of this relation-- that concurrent index * builds can safely ignore us. * * Normally, DefineIndex() only does that if index doesn't have any * predicates (i.e.: where clause) and no index expressions at all. * However, now that we already called standard process utility, * index build on the shell table is finished anyway. * * The reason behind doing so is that we cannot guarantee not * grabbing any snapshots via adaptive executor, and the backends * creating indexes on local shards (if any) might block on waiting * for current xact of the current backend to finish, which would * cause self deadlocks that are not detectable. */ if (ddlJob->startNewTransaction) { /* * Since it is not certain whether the code-path that we followed * until reaching here caused grabbing any snapshots or not, we * need to pop the active snapshot if we had any, to ensure not * leaking any snapshots. * * For example, EnsureCoordinator might return without grabbing * any snapshots if we didn't receive any invalidation messages * but the otherwise is also possible. */ if (ActiveSnapshotSet()) { PopActiveSnapshot(); } CommitTransactionCommand(); StartTransactionCommand(); /* * Tell other backends to ignore us, even if we grab any * snapshots via adaptive executor. */ set_indexsafe_procflags(); /* * We should not have any CREATE INDEX commands go through the * local backend as we signaled other backends that this backend * is executing a "safe" index command (PROC_IN_SAFE_IC), which * is NOT true, we are only faking postgres based on the reasoning * given above. */ Assert(localExecutionSupported == false); } MemoryContext savedContext = CurrentMemoryContext; PG_TRY(); { ExecuteUtilityTaskList(ddlJob->taskList, localExecutionSupported); if (shouldSyncMetadata) { List *commandList = list_make1(DISABLE_DDL_PROPAGATION); char *currentSearchPath = CurrentSearchPath(); /* * Given that we're relaying the query to the remote nodes directly, * we should set the search path exactly the same when necessary. */ if (currentSearchPath != NULL) { commandList = lappend(commandList, psprintf("SET search_path TO %s;", currentSearchPath)); } commandList = lappend(commandList, (char *) ddlJob->metadataSyncCommand); SendBareCommandListToRemoteMetadataNodes(commandList); } } PG_CATCH(); { /* CopyErrorData() requires (CurrentMemoryContext != ErrorContext) */ MemoryContextSwitchTo(savedContext); ErrorData *edata = CopyErrorData(); /* * In concurrent index creation, if a worker index with the same name already * exists, prompt to DROP the current index and retry the original command */ if (edata->sqlerrcode == ERRCODE_DUPLICATE_TABLE) { ereport(ERROR, (errmsg("CONCURRENTLY-enabled index command failed"), errdetail( "CONCURRENTLY-enabled index commands can fail partially, " "leaving behind an INVALID index."), errhint("Use DROP INDEX CONCURRENTLY IF EXISTS to remove the " "invalid index, then retry the original command."))); } else if (ddlJob->warnForPartialFailure) { ereport(WARNING, (errmsg( "Commands that are not transaction-safe may result in " "partial failure, potentially leading to an inconsistent " "state.\nIf the problematic command is a CREATE operation, " "consider using the 'IF EXISTS' syntax to drop the object," "\nif applicable, and then re-attempt " "the original command."))); } PG_RE_THROW(); } PG_END_TRY(); } } /* * set_indexsafe_procflags sets PROC_IN_SAFE_IC flag in MyProc->statusFlags. * * The flag is reset automatically at transaction end, so it must be set * for each transaction. * * Copied from pg/src/backend/commands/indexcmds.c * Also see pg commit c98763bf51bf610b3ee7e209fc76c3ff9a6b3163. */ static void set_indexsafe_procflags(void) { Assert(MyProc->xid == InvalidTransactionId && MyProc->xmin == InvalidTransactionId); LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->statusFlags |= PROC_IN_SAFE_IC; ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags; LWLockRelease(ProcArrayLock); } /* * CurrentSearchPath is a C interface for calling current_schemas(bool) that * PostgreSQL exports. * * CurrentSchemas returns all the schemas in the seach_path that are seperated * with comma (,) sign. The returned string can be used to set the search_path. * * The function omits implicit schemas. * * The function returns NULL if there are no valid schemas in the search_path, * mimicing current_schemas(false) function. */ static char * CurrentSearchPath(void) { StringInfo currentSearchPath = makeStringInfo(); List *searchPathList = fetch_search_path(false); bool schemaAdded = false; Oid searchPathOid = InvalidOid; foreach_declared_oid(searchPathOid, searchPathList) { char *schemaName = get_namespace_name(searchPathOid); /* watch out for deleted namespace */ if (schemaName) { if (schemaAdded) { appendStringInfoString(currentSearchPath, ","); schemaAdded = false; } appendStringInfoString(currentSearchPath, quote_identifier(schemaName)); schemaAdded = true; } } /* fetch_search_path() returns a palloc'd list that we should free now */ list_free(searchPathList); return (currentSearchPath->len > 0 ? currentSearchPath->data : NULL); } /* * IncrementUtilityHookCountersIfNecessary increments activeAlterTables and * activeDropSchemaOrDBs counters if utility command being processed implies * to do so. */ static void IncrementUtilityHookCountersIfNecessary(Node *parsetree) { if (IsA(parsetree, AlterTableStmt)) { activeAlterTables++; } if (IsDropSchemaOrDB(parsetree)) { activeDropSchemaOrDBs++; } } /* * PostStandardProcessUtility performs operations to alter (backend) global * state of citus utility hook. Those operations should be done after standard * process utility executes even if it errors out. */ static void PostStandardProcessUtility(Node *parsetree) { DecrementUtilityHookCountersIfNecessary(parsetree); /* * Re-forming the foreign key graph relies on the command being executed * on the local table first. However, in order to decide whether the * command leads to an invalidation, we need to check before the command * is being executed since we read pg_constraint table. Thus, we maintain a * local flag and do the invalidation after citus_ProcessUtility, * before ExecuteDistributedDDLJob(). */ InvalidateForeignKeyGraphForDDL(); } /* * DecrementUtilityHookCountersIfNecessary decrements activeAlterTables and * activeDropSchemaOrDBs counters if utility command being processed implies * to do so. */ static void DecrementUtilityHookCountersIfNecessary(Node *parsetree) { if (IsA(parsetree, AlterTableStmt)) { activeAlterTables--; } if (IsDropSchemaOrDB(parsetree)) { activeDropSchemaOrDBs--; } } /* * MarkInvalidateForeignKeyGraph marks whether the foreign key graph should be * invalidated due to a DDL. */ void MarkInvalidateForeignKeyGraph() { shouldInvalidateForeignKeyGraph = true; } /* * InvalidateForeignKeyGraphForDDL simply keeps track of whether * the foreign key graph should be invalidated due to a DDL. */ void InvalidateForeignKeyGraphForDDL(void) { if (shouldInvalidateForeignKeyGraph) { InvalidateForeignKeyGraph(); shouldInvalidateForeignKeyGraph = false; } } /* * DDLTaskList builds a list of tasks to execute a DDL command on a * given list of shards. */ List * DDLTaskList(Oid relationId, const char *commandString) { List *taskList = NIL; List *shardIntervalList = LoadShardIntervalList(relationId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *escapedSchemaName = quote_literal_cstr(schemaName); char *escapedCommandString = quote_literal_cstr(commandString); uint64 jobId = INVALID_JOB_ID; int taskId = 1; /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; StringInfo applyCommand = makeStringInfo(); /* * If rightRelationId is not InvalidOid, instead of worker_apply_shard_ddl_command * we use worker_apply_inter_shard_ddl_command. */ appendStringInfo(applyCommand, WORKER_APPLY_SHARD_DDL_COMMAND, shardId, escapedSchemaName, escapedCommandString); Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, applyCommand->data); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); taskList = lappend(taskList, task); } return taskList; } /* * NontransactionalNodeDDLTaskList builds a list of tasks to execute a DDL command on a * given target set of nodes with cannotBeExecutedInTransaction is set to make sure * that task list is executed outside a transaction block. * * Also sets warnForPartialFailure for the returned DDLJobs. */ List * NontransactionalNodeDDLTaskList(TargetWorkerSet targets, List *commands, bool warnForPartialFailure) { List *ddlJobs = NodeDDLTaskList(targets, commands); DDLJob *ddlJob = NULL; foreach_declared_ptr(ddlJob, ddlJobs) { Task *task = NULL; foreach_declared_ptr(task, ddlJob->taskList) { task->cannotBeExecutedInTransaction = true; } ddlJob->warnForPartialFailure = warnForPartialFailure; } return ddlJobs; } /* * NodeDDLTaskList builds a list of tasks to execute a DDL command on a * given target set of nodes. */ List * NodeDDLTaskList(TargetWorkerSet targets, List *commands) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = InvalidObjectAddress; ddlJob->metadataSyncCommand = NULL; /* don't allow concurrent node list changes that require an exclusive lock */ List *workerNodes = TargetWorkerSetNodeList(targets, RowShareLock); /* * if there are no nodes we don't have to plan any ddl tasks. Planning them would * cause the executor to stop responding. */ if (list_length(workerNodes) > 0) { Task *task = CitusMakeNode(Task); task->taskType = DDL_TASK; SetTaskQueryStringList(task, commands); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodes) { ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); targetPlacement->nodeName = workerNode->workerName; targetPlacement->nodePort = workerNode->workerPort; targetPlacement->groupId = workerNode->groupId; task->taskPlacementList = lappend(task->taskPlacementList, targetPlacement); } ddlJob->taskList = list_make1(task); } return list_make1(ddlJob); } /* * AlterTableInProgress returns true if we're processing an ALTER TABLE command * right now. */ bool AlterTableInProgress(void) { return activeAlterTables > 0; } /* * DropSchemaOrDBInProgress returns true if we're processing a DROP SCHEMA * or a DROP DATABASE command right now. */ bool DropSchemaOrDBInProgress(void) { return activeDropSchemaOrDBs > 0; } ================================================ FILE: src/backend/distributed/commands/vacuum.c ================================================ /*------------------------------------------------------------------------- * * vacuum.c * Commands for vacuuming distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/xact.h" #include "commands/defrem.h" #include "commands/vacuum.h" #include "postmaster/bgworker_internals.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #define VACUUM_PARALLEL_NOTSET -2 /* * Subset of VacuumParams we care about */ typedef struct CitusVacuumParams { int options; VacOptValue truncate; VacOptValue index_cleanup; int nworkers; int ring_size; } CitusVacuumParams; /* * Information we track per VACUUM/ANALYZE target relation. */ typedef struct CitusVacuumRelation { VacuumRelation *vacuumRelation; Oid relationId; } CitusVacuumRelation; /* Local functions forward declarations for processing distributed table commands */ static bool IsDistributedVacuumStmt(List *vacuumRelationList); static List * VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColumnList); static char * DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams); static char * DeparseVacuumColumnNames(List *columnNameList); static void ExecuteVacuumOnDistributedTables(VacuumStmt *vacuumStmt, List *relationList, CitusVacuumParams vacuumParams); static void ExecuteUnqualifiedVacuumTasks(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumParams); static CitusVacuumParams VacuumStmtParams(VacuumStmt *vacstmt); static List * VacuumRelationList(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumParams); /* * PostprocessVacuumStmt processes vacuum statements that may need propagation to * citus tables only if ddl propagation is enabled. If a VACUUM or ANALYZE command * references a citus table or no table, it is propagated to all involved nodes; otherwise, * the statements will not be propagated. * * Unlike most other Process functions within this file, this function does not * return a modified parse node, as it is expected that the local VACUUM or * ANALYZE has already been processed. */ List * PostprocessVacuumStmt(Node *node, const char *vacuumCommand) { VacuumStmt *vacuumStmt = castNode(VacuumStmt, node); CitusVacuumParams vacuumParams = VacuumStmtParams(vacuumStmt); if (vacuumParams.options & VACOPT_VACUUM) { /* * We commit the current transaction here so that the global lock * taken from the shell table for VACUUM is released, which would block execution * of shard placements. We don't do this in case of "ANALYZE " command because * its semantics are different than VACUUM and it doesn't acquire the global lock. */ CommitTransactionCommand(); StartTransactionCommand(); } /* * when no table is specified propagate the command as it is; * otherwise, only propagate when there is at least 1 citus table */ List *vacuumRelationList = VacuumRelationList(vacuumStmt, vacuumParams); if (list_length(vacuumStmt->rels) == 0) { /* no table is specified (unqualified vacuum) */ ExecuteUnqualifiedVacuumTasks(vacuumStmt, vacuumParams); } else if (IsDistributedVacuumStmt(vacuumRelationList)) { /* there is at least 1 citus table specified */ ExecuteVacuumOnDistributedTables(vacuumStmt, vacuumRelationList, vacuumParams); } /* else only local tables are specified */ return NIL; } /* * VacuumRelationList returns the list of relations in the given vacuum statement, * along with their resolved Oids (if they can be locked). */ static List * VacuumRelationList(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumParams) { LOCKMODE lockMode = (vacuumParams.options & VACOPT_FULL) ? AccessExclusiveLock : ShareUpdateExclusiveLock; bool skipLocked = (vacuumParams.options & VACOPT_SKIP_LOCKED); List *relationList = NIL; VacuumRelation *vacuumRelation = NULL; foreach_declared_ptr(vacuumRelation, vacuumStmt->rels) { Oid relationId = InvalidOid; /* * If skip_locked option is enabled, we are skipping that relation * if the lock for it is currently not available; otherwise, we get the lock. */ if (vacuumRelation->relation) { relationId = RangeVarGetRelidExtended(vacuumRelation->relation, lockMode, skipLocked ? RVR_SKIP_LOCKED : 0, NULL, NULL); } else if (OidIsValid(vacuumRelation->oid)) { /* fall back to the Oid directly when provided */ if (!skipLocked || ConditionalLockRelationOid(vacuumRelation->oid, lockMode)) { if (!skipLocked) { LockRelationOid(vacuumRelation->oid, lockMode); } relationId = vacuumRelation->oid; } } if (OidIsValid(relationId)) { CitusVacuumRelation *relation = palloc(sizeof(CitusVacuumRelation)); relation->vacuumRelation = vacuumRelation; relation->relationId = relationId; relationList = lappend(relationList, relation); } } return relationList; } /* * IsDistributedVacuumStmt returns true if there is any citus table in the relation id list; * otherwise, it returns false. */ static bool IsDistributedVacuumStmt(List *vacuumRelationList) { CitusVacuumRelation *vacuumRelation = NULL; foreach_declared_ptr(vacuumRelation, vacuumRelationList) { if (OidIsValid(vacuumRelation->relationId) && IsCitusTable(vacuumRelation->relationId)) { return true; } } return false; } /* * ExecuteVacuumOnDistributedTables executes the vacuum for the shard placements of given tables * if they are citus tables. */ static void ExecuteVacuumOnDistributedTables(VacuumStmt *vacuumStmt, List *relationList, CitusVacuumParams vacuumParams) { CitusVacuumRelation *vacuumRelationEntry = NULL; foreach_declared_ptr(vacuumRelationEntry, relationList) { Oid relationId = vacuumRelationEntry->relationId; VacuumRelation *vacuumRelation = vacuumRelationEntry->vacuumRelation; RangeVar *relation = vacuumRelation->relation; if (relation != NULL && !relation->inh) { /* ONLY specified, so don't recurse to shard placements */ continue; } if (IsCitusTable(relationId)) { List *vacuumColumnList = vacuumRelation->va_cols; List *taskList = VacuumTaskList(relationId, vacuumParams, vacuumColumnList); /* local execution is not implemented for VACUUM commands */ bool localExecutionSupported = false; ExecuteUtilityTaskList(taskList, localExecutionSupported); } } } /* * VacuumTaskList returns a list of tasks to be executed as part of processing * a VacuumStmt which targets a distributed relation. */ static List * VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColumnList) { LOCKMODE lockMode = (vacuumParams.options & VACOPT_FULL) ? AccessExclusiveLock : ShareUpdateExclusiveLock; /* resulting task list */ List *taskList = NIL; /* enumerate the tasks when putting them to the taskList */ int taskId = 1; Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *relationName = get_rel_name(relationId); const char *vacuumStringPrefix = DeparseVacuumStmtPrefix(vacuumParams); const char *columnNames = DeparseVacuumColumnNames(vacuumColumnList); /* * We obtain ShareUpdateExclusiveLock here to not conflict with INSERT's * RowExclusiveLock. However if VACUUM FULL is used, we already obtain * AccessExclusiveLock before reaching to that point and INSERT's will be * blocked anyway. This is inline with PostgreSQL's own behaviour. * Also note that if skip locked option is enabled, we try to acquire the lock * in nonblocking way. If lock is not available, vacuum just skip that relation. */ if (!(vacuumParams.options & VACOPT_SKIP_LOCKED)) { LockRelationOid(relationId, lockMode); } else { if (!ConditionalLockRelationOid(relationId, lockMode)) { return NIL; } } List *shardIntervalList = LoadShardIntervalList(relationId); /* grab shard lock before getting placement list */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; char *shardRelationName = pstrdup(relationName); /* build shard relation name */ AppendShardIdToName(&shardRelationName, shardId); char *quotedShardName = quote_qualified_identifier(schemaName, shardRelationName); /* copy base vacuum string and build the shard specific command */ StringInfo vacuumStringForShard = makeStringInfo(); appendStringInfoString(vacuumStringForShard, vacuumStringPrefix); appendStringInfoString(vacuumStringForShard, quotedShardName); appendStringInfoString(vacuumStringForShard, columnNames); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = VACUUM_ANALYZE_TASK; SetTaskQueryString(task, vacuumStringForShard->data); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); taskList = lappend(taskList, task); } return taskList; } /* * DeparseVacuumStmtPrefix returns a StringInfo appropriate for use as a prefix * during distributed execution of a VACUUM or ANALYZE statement. Callers may * reuse this prefix within a loop to generate shard-specific VACUUM or ANALYZE * statements. */ static char * DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) { int vacuumFlags = vacuumParams.options; StringInfo vacuumPrefix = makeStringInfo(); /* determine actual command and block out its bits */ if (vacuumFlags & VACOPT_VACUUM) { appendStringInfoString(vacuumPrefix, "VACUUM "); vacuumFlags &= ~VACOPT_VACUUM; } else { Assert((vacuumFlags & VACOPT_ANALYZE) != 0); appendStringInfoString(vacuumPrefix, "ANALYZE "); vacuumFlags &= ~VACOPT_ANALYZE; if (vacuumFlags & VACOPT_VERBOSE) { appendStringInfoString(vacuumPrefix, "VERBOSE "); vacuumFlags &= ~VACOPT_VERBOSE; } } /* if no flags remain, exit early */ if (vacuumFlags & VACOPT_PROCESS_TOAST && vacuumFlags & VACOPT_PROCESS_MAIN) { /* process toast and process main are true by default */ if (((vacuumFlags & ~VACOPT_PROCESS_TOAST) & ~VACOPT_PROCESS_MAIN) == 0 && vacuumParams.ring_size == -1 && vacuumParams.truncate == VACOPTVALUE_UNSPECIFIED && vacuumParams.index_cleanup == VACOPTVALUE_UNSPECIFIED && vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET ) { return vacuumPrefix->data; } } /* otherwise, handle options */ appendStringInfoChar(vacuumPrefix, '('); if (vacuumFlags & VACOPT_ANALYZE) { appendStringInfoString(vacuumPrefix, "ANALYZE,"); } if (vacuumFlags & VACOPT_DISABLE_PAGE_SKIPPING) { appendStringInfoString(vacuumPrefix, "DISABLE_PAGE_SKIPPING,"); } if (vacuumFlags & VACOPT_FREEZE) { appendStringInfoString(vacuumPrefix, "FREEZE,"); } if (vacuumFlags & VACOPT_FULL) { appendStringInfoString(vacuumPrefix, "FULL,"); } if (vacuumFlags & VACOPT_VERBOSE) { appendStringInfoString(vacuumPrefix, "VERBOSE,"); } if (vacuumFlags & VACOPT_SKIP_LOCKED) { appendStringInfoString(vacuumPrefix, "SKIP_LOCKED,"); } if (!(vacuumFlags & VACOPT_PROCESS_TOAST)) { appendStringInfoString(vacuumPrefix, "PROCESS_TOAST FALSE,"); } if (!(vacuumFlags & VACOPT_PROCESS_MAIN)) { appendStringInfoString(vacuumPrefix, "PROCESS_MAIN FALSE,"); } if (vacuumFlags & VACOPT_SKIP_DATABASE_STATS) { appendStringInfoString(vacuumPrefix, "SKIP_DATABASE_STATS,"); } if (vacuumFlags & VACOPT_ONLY_DATABASE_STATS) { appendStringInfoString(vacuumPrefix, "ONLY_DATABASE_STATS,"); } if (vacuumParams.ring_size != -1) { appendStringInfo(vacuumPrefix, "BUFFER_USAGE_LIMIT %d,", vacuumParams.ring_size); } if (vacuumParams.truncate != VACOPTVALUE_UNSPECIFIED) { appendStringInfoString(vacuumPrefix, vacuumParams.truncate == VACOPTVALUE_ENABLED ? "TRUNCATE," : "TRUNCATE false," ); } if (vacuumParams.index_cleanup != VACOPTVALUE_UNSPECIFIED) { switch (vacuumParams.index_cleanup) { case VACOPTVALUE_ENABLED: { appendStringInfoString(vacuumPrefix, "INDEX_CLEANUP true,"); break; } case VACOPTVALUE_DISABLED: { appendStringInfoString(vacuumPrefix, "INDEX_CLEANUP false,"); break; } case VACOPTVALUE_AUTO: { appendStringInfoString(vacuumPrefix, "INDEX_CLEANUP auto,"); break; } default: { break; } } } if (vacuumParams.nworkers != VACUUM_PARALLEL_NOTSET) { appendStringInfo(vacuumPrefix, "PARALLEL %d,", vacuumParams.nworkers); } vacuumPrefix->data[vacuumPrefix->len - 1] = ')'; appendStringInfoChar(vacuumPrefix, ' '); return vacuumPrefix->data; } /* * DeparseVacuumColumnNames joins the list of strings using commas as a * delimiter. The whole thing is placed in parenthesis and set off with a * single space in order to facilitate appending it to the end of any VACUUM * or ANALYZE command which uses explicit column names. If the provided list * is empty, this function returns an empty string to keep the calling code * simplest. */ static char * DeparseVacuumColumnNames(List *columnNameList) { StringInfo columnNames = makeStringInfo(); if (columnNameList == NIL) { return columnNames->data; } appendStringInfoString(columnNames, " ("); String *columnName = NULL; foreach_declared_ptr(columnName, columnNameList) { appendStringInfo(columnNames, "%s,", strVal(columnName)); } columnNames->data[columnNames->len - 1] = ')'; return columnNames->data; } /* * VacuumStmtParams returns a CitusVacuumParams based on the supplied VacuumStmt. */ /* * This is mostly ExecVacuum from Postgres's commands/vacuum.c * Note that ExecVacuum does an actual vacuum as well and we don't want * that to happen in the coordinator hence we copied the rest here. */ static CitusVacuumParams VacuumStmtParams(VacuumStmt *vacstmt) { CitusVacuumParams params; bool verbose = false; bool skip_locked = false; bool analyze = false; bool freeze = false; bool full = false; bool disable_page_skipping = false; bool process_toast = true; bool process_main = true; bool skip_database_stats = false; bool only_database_stats = false; params.ring_size = -1; /* Set default value */ params.index_cleanup = VACOPTVALUE_UNSPECIFIED; params.truncate = VACOPTVALUE_UNSPECIFIED; params.nworkers = VACUUM_PARALLEL_NOTSET; /* Parse options list */ DefElem *opt = NULL; foreach_declared_ptr(opt, vacstmt->options) { /* Parse common options for VACUUM and ANALYZE */ if (strcmp(opt->defname, "verbose") == 0) { verbose = defGetBoolean(opt); } else if (strcmp(opt->defname, "skip_locked") == 0) { skip_locked = defGetBoolean(opt); } else if (strcmp(opt->defname, "buffer_usage_limit") == 0) { char *vac_buffer_size = defGetString(opt); parse_int(vac_buffer_size, ¶ms.ring_size, GUC_UNIT_KB, NULL); } else if (!vacstmt->is_vacuumcmd) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized ANALYZE option \"%s\"", opt->defname))); } /* Parse options available on VACUUM */ else if (strcmp(opt->defname, "analyze") == 0) { analyze = defGetBoolean(opt); } else if (strcmp(opt->defname, "freeze") == 0) { freeze = defGetBoolean(opt); } else if (strcmp(opt->defname, "full") == 0) { full = defGetBoolean(opt); } else if (strcmp(opt->defname, "disable_page_skipping") == 0) { disable_page_skipping = defGetBoolean(opt); } else if (strcmp(opt->defname, "process_main") == 0) { process_main = defGetBoolean(opt); } else if (strcmp(opt->defname, "skip_database_stats") == 0) { skip_database_stats = defGetBoolean(opt); } else if (strcmp(opt->defname, "only_database_stats") == 0) { only_database_stats = defGetBoolean(opt); } else if (strcmp(opt->defname, "process_toast") == 0) { process_toast = defGetBoolean(opt); } else if (strcmp(opt->defname, "index_cleanup") == 0) { /* Interpret no string as the default, which is 'auto' */ if (!opt->arg) { params.index_cleanup = VACOPTVALUE_AUTO; } else { char *sval = defGetString(opt); /* Try matching on 'auto' string, or fall back on boolean */ if (pg_strcasecmp(sval, "auto") == 0) { params.index_cleanup = VACOPTVALUE_AUTO; } else { params.index_cleanup = defGetBoolean(opt) ? VACOPTVALUE_ENABLED : VACOPTVALUE_DISABLED; } } } else if (strcmp(opt->defname, "truncate") == 0) { params.truncate = defGetBoolean(opt) ? VACOPTVALUE_ENABLED : VACOPTVALUE_DISABLED; } else if (strcmp(opt->defname, "parallel") == 0) { if (opt->arg == NULL) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("parallel option requires a value between 0 and %d", MAX_PARALLEL_WORKER_LIMIT))); } else { int nworkers = defGetInt32(opt); if (nworkers < 0 || nworkers > MAX_PARALLEL_WORKER_LIMIT) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("parallel vacuum degree must be between 0 and %d", MAX_PARALLEL_WORKER_LIMIT))); } params.nworkers = nworkers; } } else { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized VACUUM option \"%s\"", opt->defname) )); } } params.options = (vacstmt->is_vacuumcmd ? VACOPT_VACUUM : VACOPT_ANALYZE) | (verbose ? VACOPT_VERBOSE : 0) | (skip_locked ? VACOPT_SKIP_LOCKED : 0) | (analyze ? VACOPT_ANALYZE : 0) | (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | (process_main ? VACOPT_PROCESS_MAIN : 0) | (skip_database_stats ? VACOPT_SKIP_DATABASE_STATS : 0) | (only_database_stats ? VACOPT_ONLY_DATABASE_STATS : 0) | (process_toast ? VACOPT_PROCESS_TOAST : 0) | (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0); return params; } /* * ExecuteUnqualifiedVacuumTasks executes tasks for unqualified vacuum commands */ static void ExecuteUnqualifiedVacuumTasks(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumParams) { /* don't allow concurrent node list changes that require an exclusive lock */ List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); if (list_length(workerNodes) == 0) { return; } const char *vacuumStringPrefix = DeparseVacuumStmtPrefix(vacuumParams); StringInfo vacuumCommand = makeStringInfo(); appendStringInfoString(vacuumCommand, vacuumStringPrefix); List *unqualifiedVacuumCommands = list_make3(DISABLE_DDL_PROPAGATION, vacuumCommand->data, ENABLE_DDL_PROPAGATION); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskType = VACUUM_ANALYZE_TASK; SetTaskQueryStringList(task, unqualifiedVacuumCommands); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); bool hasPeerWorker = false; int32 localNodeGroupId = GetLocalGroupId(); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodes) { if (workerNode->groupId != localNodeGroupId) { ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); targetPlacement->nodeName = workerNode->workerName; targetPlacement->nodePort = workerNode->workerPort; targetPlacement->groupId = workerNode->groupId; task->taskPlacementList = lappend(task->taskPlacementList, targetPlacement); hasPeerWorker = true; } } if (hasPeerWorker) { bool localExecution = false; ExecuteUtilityTaskList(list_make1(task), localExecution); } } ================================================ FILE: src/backend/distributed/commands/variableset.c ================================================ /*------------------------------------------------------------------------- * * variableset.c * Support for propagation of SET (commands to set variables) * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "common/string.h" #include "lib/ilist.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/varlena.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" /* * ShouldPropagateSetCommand determines whether a SET or RESET command should be * propagated to the workers. * * We currently propagate: * - SET LOCAL (for allowed settings) * - SET TRANSACTION * - RESET (for allowed settings) * - RESET ALL */ bool ShouldPropagateSetCommand(VariableSetStmt *setStmt) { if (PropagateSetCommands != PROPSETCMD_LOCAL) { /* SET propagation is disabled */ return false; } switch (setStmt->kind) { case VAR_SET_VALUE: case VAR_SET_CURRENT: case VAR_SET_DEFAULT: { /* SET LOCAL on a safe setting */ return setStmt->is_local && IsSettingSafeToPropagate(setStmt->name); } case VAR_RESET: { /* may need to reset prior SET LOCAL */ return IsSettingSafeToPropagate(setStmt->name); } case VAR_RESET_ALL: { /* always propagate RESET ALL since it might affect prior SET LOCALs */ return true; } case VAR_SET_MULTI: default: { /* SET TRANSACTION is similar to SET LOCAL */ return strcmp(setStmt->name, "TRANSACTION") == 0; } } } /* * PostprocessVariableSetStmt actually does the work of propagating a provided SET stmt * to currently-participating worker nodes and adding the SET command test to a string * keeping track of all propagated SET commands since (sub-)xact start. */ void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *setStmtString) { dlist_iter iter; const bool raiseInterrupts = true; List *connectionList = NIL; /* at present we only support SET LOCAL and SET TRANSACTION */ Assert(ShouldPropagateSetCommand(setStmt)); /* haven't seen any SET stmts so far in this (sub-)xact: initialize StringInfo */ if (activeSetStmts == NULL) { /* see comments in PushSubXact on why we allocate this in TopTransactionContext */ MemoryContext old_context = MemoryContextSwitchTo(TopTransactionContext); activeSetStmts = makeStringInfo(); MemoryContextSwitchTo(old_context); } /* send text of SET stmt to participating nodes... */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } if (!SendRemoteCommand(connection, setStmtString)) { const bool raiseErrors = true; HandleRemoteTransactionConnectionError(connection, raiseErrors); } connectionList = lappend(connectionList, connection); } WaitForAllConnections(connectionList, raiseInterrupts); /* ... and wait for the results */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); const bool raiseErrors = true; RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } ClearResults(connection, raiseErrors); } /* SET propagation successful: add to active SET stmt string */ appendStringInfoString(activeSetStmts, setStmtString); /* ensure semicolon on end to allow appending future SET stmts */ if (!pg_str_endswith(setStmtString, ";")) { appendStringInfoChar(activeSetStmts, ';'); } } ================================================ FILE: src/backend/distributed/commands/view.c ================================================ /*------------------------------------------------------------------------- * * view.c * Commands for distributing CREATE OR REPLACE VIEW statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "access/genam.h" #include "catalog/objectaddress.h" #include "commands/extension.h" #include "executor/spi.h" #include "nodes/nodes.h" #include "nodes/pg_list.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/errormessage.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/multi_executor.h" #include "distributed/namespace_utils.h" #include "distributed/worker_transaction.h" /* * GUC controls some restrictions for local objects. For example, * if it is disabled, a local view with no distributed relation dependency * will be created even if it has circular dependency and will not * log error or warning. Citus will normally restrict that behaviour for the * local views. e.g CREATE TABLE local_t (a int); * CREATE VIEW vv1 as SELECT * FROM local_t; * CREATE OR REPLACE VIEW vv2 as SELECT * FROM vv1; * * When the GUC is disabled, citus also wont distribute the local * view which has no citus relation dependency to workers. Otherwise, it distributes * them by default. e.g create view v as select 1; */ bool EnforceLocalObjectRestrictions = true; static List * FilterNameListForDistributedViews(List *viewNamesList, bool missing_ok); static void AppendQualifiedViewNameToCreateViewCommand(StringInfo buf, Oid viewOid); static void AppendViewDefinitionToCreateViewCommand(StringInfo buf, Oid viewOid); static void AppendAliasesToCreateViewCommand(StringInfo createViewCommand, Oid viewOid); static void AppendOptionsToCreateViewCommand(StringInfo createViewCommand, Oid viewOid); static bool ViewHasDistributedRelationDependency(ObjectAddress *viewObjectAddress); /* * ViewHasDistributedRelationDependency returns true if given view at address has * any distributed relation dependency. */ static bool ViewHasDistributedRelationDependency(ObjectAddress *viewObjectAddress) { List *dependencies = GetAllDependenciesForObject(viewObjectAddress); ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { if (dependency->classId == RelationRelationId && IsAnyObjectDistributed( list_make1(dependency))) { return true; } } return false; } /* * PreprocessViewStmt is called during the planning phase for CREATE OR REPLACE VIEW * before it is created on the local node internally. */ List * PreprocessViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { if (!ShouldPropagate()) { return NIL; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return NIL; } EnsureCoordinator(); return NIL; } /* * PostprocessViewStmt actually creates the commmands we need to run on workers to * propagate views. * * If view depends on any undistributable object, Citus can not distribute it. In order to * not to prevent users from creating local views on the coordinator WARNING message will * be sent to the customer about the case instead of erroring out. If no worker nodes exist * at all, view will be created locally without any WARNING message. * * Besides creating the plan we also make sure all (new) dependencies of the view are * created on all nodes. */ List * PostprocessViewStmt(Node *node, const char *queryString) { ViewStmt *stmt = castNode(ViewStmt, node); if (!ShouldPropagate()) { return NIL; } /* check creation against multi-statement transaction policy */ if (!ShouldPropagateCreateInCoordinatedTransction()) { return NIL; } List *viewAddresses = GetObjectAddressListFromParseTree((Node *) stmt, false, true); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (IsAnyObjectAddressOwnedByExtension(viewAddresses, NULL)) { return NIL; } /* If the view has any unsupported dependency, create it locally */ if (ErrorOrWarnIfAnyObjectHasUnsupportedDependency(viewAddresses)) { return NIL; } /* * If it is disabled, a local view with no distributed relation dependency * will be created even if it has circular dependency and will not * log error or warning. Citus will normally restrict that behaviour for the * local views. e.g CREATE TABLE local_t (a int); * CREATE VIEW vv1 as SELECT * FROM local_t; * CREATE OR REPLACE VIEW vv2 as SELECT * FROM vv1; * * When the GUC is disabled, citus also wont distribute the local * view which has no citus relation dependency to workers. Otherwise, it distributes * them by default. e.g create view v as select 1; * * We disable local object restrictions during pg vanilla tests to not diverge * from Postgres in terms of error messages. */ if (!EnforceLocalObjectRestrictions) { /* we asserted that we have only one address. */ ObjectAddress *viewAddress = linitial(viewAddresses); if (!ViewHasDistributedRelationDependency(viewAddress)) { /* * The local view has no distributed relation dependency, so we can allow * it to be created even if it has circular dependency. */ return NIL; } } EnsureAllObjectDependenciesExistOnAllNodes(viewAddresses); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *viewAddress = linitial(viewAddresses); char *command = CreateViewDDLCommand(viewAddress->objectId); /* * We'd typically use NodeDDLTaskList() for generating node-level DDL commands, * such as when creating a type. However, views are different in a sense that * views do not depend on citus tables. Instead, they are `depending` on citus tables. * * When NodeDDLTaskList() used, it should be accompanied with sequential execution. * Here, we do something equivalent to NodeDDLTaskList(), but using metadataSyncCommand * field. This hack allows us to use the metadata connection * (see `REQUIRE_METADATA_CONNECTION` flag). Meaning that, view creation is treated as * a metadata operation. * * We do this mostly for performance reasons, because we cannot afford to switch to * sequential execution, for instance when we are altering or creating distributed * tables -- which may require significant resources. * * The downside of using this hack is that if a view is re-used in the same transaction * that creates the view on the workers, we might get errors such as the below which * we consider a decent trade-off currently: * * BEGIN; * CREATE VIEW dist_view .. * CRETAE TABLE t2(id int, val dist_view); * * -- shard creation fails on one of the connections * SELECT create_distributed_table('t2', 'id'); * ERROR: type "public.dist_view" does not exist * */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = *viewAddress; ddlJob->metadataSyncCommand = command; ddlJob->taskList = NIL; return list_make1(ddlJob); } /* * ViewStmtObjectAddress returns the ObjectAddress for the subject of the * CREATE [OR REPLACE] VIEW statement. */ List * ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { ViewStmt *stmt = castNode(ViewStmt, node); Oid viewOid = RangeVarGetRelid(stmt->view, NoLock, missing_ok); ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); return list_make1(viewAddress); } /* * PreprocessDropViewStmt gets called during the planning phase of a DROP VIEW statement * and returns a list of DDLJob's that will drop any distributed view from the * workers. * * The DropStmt could have multiple objects to drop, the list of objects will be filtered * to only keep the distributed views for deletion on the workers. Non-distributed * views will still be dropped locally but not on the workers. */ List * PreprocessDropViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { DropStmt *stmt = castNode(DropStmt, node); if (!ShouldPropagate()) { return NIL; } List *distributedViewNames = FilterNameListForDistributedViews(stmt->objects, stmt->missing_ok); if (list_length(distributedViewNames) < 1) { /* no distributed view to drop */ return NIL; } EnsureCoordinator(); EnsureSequentialMode(OBJECT_VIEW); /* * Swap the list of objects before deparsing and restore the old list after. This * ensures we only have distributed views in the deparsed drop statement. */ DropStmt *stmtCopy = copyObject(stmt); stmtCopy->objects = distributedViewNames; QualifyTreeNode((Node *) stmtCopy); const char *dropStmtSql = DeparseTreeNode((Node *) stmtCopy); List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) dropStmtSql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* * DropViewStmtObjectAddress returns list of object addresses in the drop view * statement. */ List * DropViewStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess) { DropStmt *dropStmt = castNode(DropStmt, stmt); List *objectAddresses = NIL; List *possiblyQualifiedViewName = NULL; foreach_declared_ptr(possiblyQualifiedViewName, dropStmt->objects) { RangeVar *viewRangeVar = makeRangeVarFromNameList(possiblyQualifiedViewName); Oid viewOid = RangeVarGetRelid(viewRangeVar, AccessShareLock, missing_ok); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, RelationRelationId, viewOid); objectAddresses = lappend(objectAddresses, objectAddress); } return objectAddresses; } /* * FilterNameListForDistributedViews takes a list of view names and filters against the * views that are distributed. * * The original list will not be touched, a new list will be created with only the objects * in there. */ static List * FilterNameListForDistributedViews(List *viewNamesList, bool missing_ok) { List *distributedViewNames = NIL; List *possiblyQualifiedViewName = NULL; foreach_declared_ptr(possiblyQualifiedViewName, viewNamesList) { char *viewName = NULL; char *schemaName = NULL; DeconstructQualifiedName(possiblyQualifiedViewName, &schemaName, &viewName); if (schemaName == NULL) { char *objName = NULL; Oid schemaOid = QualifiedNameGetCreationNamespace(possiblyQualifiedViewName, &objName); schemaName = get_namespace_name(schemaOid); } Oid schemaId = get_namespace_oid(schemaName, missing_ok); Oid viewOid = get_relname_relid(viewName, schemaId); if (!OidIsValid(viewOid)) { continue; } if (IsViewDistributed(viewOid)) { distributedViewNames = lappend(distributedViewNames, possiblyQualifiedViewName); } } return distributedViewNames; } /* * CreateViewDDLCommand returns the DDL command to create the view addressed by * the viewAddress. */ char * CreateViewDDLCommand(Oid viewOid) { StringInfo createViewCommand = makeStringInfo(); appendStringInfoString(createViewCommand, "CREATE OR REPLACE VIEW "); AppendQualifiedViewNameToCreateViewCommand(createViewCommand, viewOid); AppendAliasesToCreateViewCommand(createViewCommand, viewOid); AppendOptionsToCreateViewCommand(createViewCommand, viewOid); AppendViewDefinitionToCreateViewCommand(createViewCommand, viewOid); return createViewCommand->data; } /* * AppendQualifiedViewNameToCreateViewCommand adds the qualified view of the given view * oid to the given create view command. */ static void AppendQualifiedViewNameToCreateViewCommand(StringInfo buf, Oid viewOid) { char *qualifiedViewName = generate_qualified_relation_name(viewOid); appendStringInfo(buf, "%s ", qualifiedViewName); } /* * AppendAliasesToCreateViewCommand appends aliases to the create view * command for the existing view. */ static void AppendAliasesToCreateViewCommand(StringInfo createViewCommand, Oid viewOid) { /* Get column name aliases from pg_attribute */ ScanKeyData key[1]; ScanKeyInit(&key[0], Anum_pg_attribute_attrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(viewOid)); Relation maprel = table_open(AttributeRelationId, AccessShareLock); Relation mapidx = index_open(AttributeRelidNumIndexId, AccessShareLock); SysScanDesc pgAttributeScan = systable_beginscan_ordered(maprel, mapidx, NULL, 1, key); bool isInitialAlias = true; bool hasAlias = false; HeapTuple attributeTuple; while (HeapTupleIsValid(attributeTuple = systable_getnext_ordered(pgAttributeScan, ForwardScanDirection))) { Form_pg_attribute att = (Form_pg_attribute) GETSTRUCT(attributeTuple); const char *aliasName = quote_identifier(NameStr(att->attname)); if (isInitialAlias) { appendStringInfoString(createViewCommand, "("); } else { appendStringInfoString(createViewCommand, ","); } appendStringInfoString(createViewCommand, aliasName); hasAlias = true; isInitialAlias = false; } if (hasAlias) { appendStringInfoString(createViewCommand, ") "); } systable_endscan_ordered(pgAttributeScan); index_close(mapidx, AccessShareLock); table_close(maprel, AccessShareLock); } /* * AppendOptionsToCreateViewCommand add relation options to create view command * for an existing view */ static void AppendOptionsToCreateViewCommand(StringInfo createViewCommand, Oid viewOid) { /* Add rel options to create view command */ char *relOptions = flatten_reloptions(viewOid); if (relOptions != NULL) { appendStringInfo(createViewCommand, "WITH (%s) ", relOptions); } } /* * AppendViewDefinitionToCreateViewCommand adds the definition of the given view to the * given create view command. */ static void AppendViewDefinitionToCreateViewCommand(StringInfo buf, Oid viewOid) { /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. */ int saveNestLevel = PushEmptySearchPath(); /* * Push the transaction snapshot to be able to get vief definition with pg_get_viewdef */ PushActiveSnapshot(GetTransactionSnapshot()); Datum viewDefinitionDatum = DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(viewOid)); char *viewDefinition = TextDatumGetCString(viewDefinitionDatum); PopActiveSnapshot(); PopEmptySearchPath(saveNestLevel); appendStringInfo(buf, "AS %s ", viewDefinition); } /* * AlterViewOwnerCommand returns the command to alter view owner command for the * given view or materialized view oid. */ char * AlterViewOwnerCommand(Oid viewOid) { /* Add alter owner commmand */ StringInfo alterOwnerCommand = makeStringInfo(); char *viewName = get_rel_name(viewOid); Oid schemaOid = get_rel_namespace(viewOid); char *schemaName = get_namespace_name(schemaOid); char *viewOwnerName = TableOwner(viewOid); char *qualifiedViewName = NameListToQuotedString(list_make2(makeString(schemaName), makeString(viewName))); if (get_rel_relkind(viewOid) == RELKIND_MATVIEW) { appendStringInfo(alterOwnerCommand, "ALTER MATERIALIZED VIEW %s ", qualifiedViewName); } else { appendStringInfo(alterOwnerCommand, "ALTER VIEW %s ", qualifiedViewName); } appendStringInfo(alterOwnerCommand, "OWNER TO %s", quote_identifier(viewOwnerName)); return alterOwnerCommand->data; } /* * IsViewDistributed checks if a view is distributed */ bool IsViewDistributed(Oid viewOid) { Assert(get_rel_relkind(viewOid) == RELKIND_VIEW || get_rel_relkind(viewOid) == RELKIND_MATVIEW); ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); return IsAnyObjectDistributed(list_make1(viewAddress)); } /* * PreprocessAlterViewStmt is invoked for alter view statements. */ List * PreprocessAlterViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); List *viewAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, false); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (!ShouldPropagateAnyObject(viewAddresses)) { return NIL; } QualifyTreeNode((Node *) stmt); EnsureCoordinator(); /* reconstruct alter statement in a portable fashion */ const char *alterViewStmtSql = DeparseTreeNode((Node *) stmt); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *viewAddress = linitial(viewAddresses); /* * To avoid sequential mode, we are using metadata connection. For the * detailed explanation, please check the comment on PostprocessViewStmt. */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = *viewAddress; ddlJob->metadataSyncCommand = alterViewStmtSql; ddlJob->taskList = NIL; return list_make1(ddlJob); } /* * PostprocessAlterViewStmt is invoked for alter view statements. */ List * PostprocessAlterViewStmt(Node *node, const char *queryString) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_VIEW); List *viewAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, true); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (!ShouldPropagateAnyObject(viewAddresses)) { return NIL; } if (IsAnyObjectAddressOwnedByExtension(viewAddresses, NULL)) { return NIL; } /* If the view has any unsupported dependency, create it locally */ if (ErrorOrWarnIfAnyObjectHasUnsupportedDependency(viewAddresses)) { return NIL; } EnsureAllObjectDependenciesExistOnAllNodes(viewAddresses); return NIL; } /* * AlterViewStmtObjectAddress returns the ObjectAddress for the subject of the * ALTER VIEW statement. */ List * AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Oid viewOid = RangeVarGetRelid(stmt->relation, NoLock, missing_ok); ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); return list_make1(viewAddress); } /* * PreprocessRenameViewStmt is called when the user is renaming the view or the column of * the view. */ List * PreprocessRenameViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { List *viewAddresses = GetObjectAddressListFromParseTree(node, true, false); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (!ShouldPropagateAnyObject(viewAddresses)) { return NIL; } EnsureCoordinator(); /* fully qualify */ QualifyTreeNode(node); /* deparse sql*/ const char *renameStmtSql = DeparseTreeNode(node); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *viewAddress = linitial(viewAddresses); /* * To avoid sequential mode, we are using metadata connection. For the * detailed explanation, please check the comment on PostprocessViewStmt. */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = *viewAddress; ddlJob->metadataSyncCommand = renameStmtSql; ddlJob->taskList = NIL; return list_make1(ddlJob); } /* * RenameViewStmtObjectAddress returns the ObjectAddress of the view that is the object * of the RenameStmt. Errors if missing_ok is false. */ List * RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Oid viewOid = RangeVarGetRelid(stmt->relation, NoLock, missing_ok); ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); return list_make1(viewAddress); } /* * PreprocessAlterViewSchemaStmt is executed before the statement is applied to the local * postgres instance. */ List * PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); List *viewAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, false); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (!ShouldPropagateAnyObject(viewAddresses)) { return NIL; } EnsureCoordinator(); QualifyTreeNode((Node *) stmt); const char *sql = DeparseTreeNode((Node *) stmt); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *viewAddress = linitial(viewAddresses); /* * To avoid sequential mode, we are using metadata connection. For the * detailed explanation, please check the comment on PostprocessViewStmt. */ DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = *viewAddress; ddlJob->metadataSyncCommand = sql; ddlJob->taskList = NIL; return list_make1(ddlJob); } /* * PostprocessAlterViewSchemaStmt is executed after the change has been applied locally, we * can now use the new dependencies of the view to ensure all its dependencies exist on * the workers before we apply the commands remotely. */ List * PostprocessAlterViewSchemaStmt(Node *node, const char *queryString) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); List *viewAddresses = GetObjectAddressListFromParseTree((Node *) stmt, true, true); /* the code-path only supports a single object */ Assert(list_length(viewAddresses) == 1); if (!ShouldPropagateAnyObject(viewAddresses)) { return NIL; } /* dependencies have changed (schema) let's ensure they exist */ EnsureAllObjectDependenciesExistOnAllNodes(viewAddresses); return NIL; } /* * AlterViewSchemaStmtObjectAddress returns the ObjectAddress of the view that is the object * of the alter schema statement. */ List * AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Oid viewOid = RangeVarGetRelid(stmt->relation, NoLock, true); /* * Since it can be called both before and after executing the standardProcess utility, * we need to check both old and new schemas */ if (viewOid == InvalidOid) { Oid schemaId = get_namespace_oid(stmt->newschema, missing_ok); viewOid = get_relname_relid(stmt->relation->relname, schemaId); /* * if the view is still invalid we couldn't find the view, error with the same * message postgres would error with it missing_ok is false (not ok to miss) */ if (!missing_ok && viewOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("view \"%s\" does not exist", stmt->relation->relname))); } } ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); return list_make1(viewAddress); } /* * IsViewRenameStmt returns whether the passed-in RenameStmt is the following * form: * * - ALTER VIEW RENAME * - ALTER VIEW RENAME COLUMN */ bool IsViewRenameStmt(RenameStmt *renameStmt) { bool isViewRenameStmt = false; if (renameStmt->renameType == OBJECT_VIEW || (renameStmt->renameType == OBJECT_COLUMN && renameStmt->relationType == OBJECT_VIEW)) { isViewRenameStmt = true; } return isViewRenameStmt; } ================================================ FILE: src/backend/distributed/connection/connection_configuration.c ================================================ /*------------------------------------------------------------------------- * * connection_configuration.c * Functions for controlling configuration of Citus connections * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/transam.h" #include "access/xact.h" #include "mb/pg_wchar.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/metadata_cache.h" #include "distributed/worker_manager.h" /* stores the string representation of our node connection GUC */ #ifdef USE_SSL char *NodeConninfo = "sslmode=require"; #else char *NodeConninfo = "sslmode=prefer"; #endif /* * Previously we would use an empty initial value for NodeConnInfo * PG16 however requires same initial and boot values for configuration parameters * Therefore we now use this flag in NodeConninfoGucAssignHook */ bool checkAtBootPassed = false; char *LocalHostName = "localhost"; /* represents a list of libpq parameter settings */ typedef struct ConnParamsInfo { char **keywords; /* libpq keywords */ char **values; /* desired values for above */ Size size; /* current used size of arrays */ Size maxSize; /* maximum allocated size of arrays (similar to e.g. StringInfo) */ } ConnParamsInfo; /* * Stores parsed global libpq parameter settings. static because all access * is encapsulated in the other public functions in this file. */ static ConnParamsInfo ConnParams; /* helper functions for processing connection info */ static ConnectionHashKey * GetEffectiveConnKey(ConnectionHashKey *key); static Size CalculateMaxSize(void); static int uri_prefix_length(const char *connstr); /* * InitConnParams initializes the ConnParams field to point to enough memory to * store settings for every valid libpq value, though these regions are set to * zeros from the outset and the size appropriately also set to zero. * * This function must be called before others in this file, though calling it * after use of the initialized ConnParams structure will result in any * populated parameter settings being lost. */ void InitConnParams() { Size maxSize = CalculateMaxSize(); ConnParamsInfo connParams = { .keywords = malloc(maxSize * sizeof(char *)), .values = malloc(maxSize * sizeof(char *)), .size = 0, .maxSize = maxSize }; memset(connParams.keywords, 0, maxSize * sizeof(char *)); memset(connParams.values, 0, maxSize * sizeof(char *)); ConnParams = connParams; } /* * ResetConnParams frees all strings in the keywords and parameters arrays, * sets their elements to null, and resets the ConnParamsSize to zero before * adding back any hardcoded global connection settings (at present, there * are no). */ void ResetConnParams() { for (Size paramIdx = 0; paramIdx < ConnParams.size; paramIdx++) { free((void *) ConnParams.keywords[paramIdx]); free((void *) ConnParams.values[paramIdx]); ConnParams.keywords[paramIdx] = ConnParams.values[paramIdx] = NULL; } ConnParams.size = 0; InvalidateConnParamsHashEntries(); } /* * AddConnParam adds a parameter setting to the global libpq settings according * to the provided keyword and value. */ void AddConnParam(const char *keyword, const char *value) { if (ConnParams.size + 1 >= ConnParams.maxSize) { /* hopefully this error is only seen by developers */ ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("ConnParams arrays bound check failed"))); } /* * Don't use pstrdup here to avoid being tied to a memory context, we free * these later using ResetConnParams */ ConnParams.keywords[ConnParams.size] = strdup(keyword); ConnParams.values[ConnParams.size] = strdup(value); ConnParams.size++; ConnParams.keywords[ConnParams.size] = ConnParams.values[ConnParams.size] = NULL; } /* * CheckConninfo is a building block to help implement check constraints and * other check hooks against libpq-like conninfo strings. In particular, the * provided conninfo must: * * - Not use a uri-prefix such as postgres:// (it must be only keys and values) * - Parse using PQconninfoParse * - Only set keywords contained in the provided allowedConninfoKeywords * * This function returns true if all of the above are satisfied, otherwise it * returns false. If the provided errmsg pointer is not NULL, it will be set * to an appropriate message if the check fails. * * The provided allowedConninfoKeywords must be sorted in a manner usable by bsearch, * though this is only validated during assert-enabled builds. */ bool CheckConninfo(const char *conninfo, const char **allowedConninfoKeywords, Size allowedConninfoKeywordsLength, char **errorMsg) { PQconninfoOption *option = NULL; char *errorMsgString = NULL; /* * If the user doesn't need a message, just overwrite errmsg with a stack * variable so we can always safely write to it. */ if (errorMsg == NULL) { errorMsg = &errorMsgString; } /* sure, it can be null */ if (conninfo == NULL) { return true; } /* the libpq prefix form is more complex than we need; ban it */ if (uri_prefix_length(conninfo) != 0) { *errorMsg = "Citus connection info strings must be in " "'k1=v1 k2=v2 [...] kn=vn' format"; return false; } /* this should at least parse */ PQconninfoOption *optionArray = PQconninfoParse(conninfo, NULL); if (optionArray == NULL) { *errorMsg = "Provided string is not a valid libpq connection info string"; return false; } #ifdef USE_ASSERT_CHECKING /* verify that the allowedConninfoKeywords is in ascending order */ for (Size keywordIdx = 1; keywordIdx < allowedConninfoKeywordsLength; keywordIdx++) { const char *prev = allowedConninfoKeywords[keywordIdx - 1]; const char *curr = allowedConninfoKeywords[keywordIdx]; Assert(strcmp(prev, curr) < 0); } #endif for (option = optionArray; option->keyword != NULL; option++) { if (option->val == NULL || option->val[0] == '\0') { continue; } void *matchingKeyword = SafeBsearch(&option->keyword, allowedConninfoKeywords, allowedConninfoKeywordsLength, sizeof(char *), pg_qsort_strcmp); if (matchingKeyword == NULL) { /* the allowedConninfoKeywords lacks this keyword; error out! */ StringInfoData msgString; initStringInfo(&msgString); appendStringInfo(&msgString, "Prohibited conninfo keyword detected: %s", option->keyword); *errorMsg = msgString.data; break; } } PQconninfoFree(optionArray); /* if error message is set we found an invalid keyword */ return (*errorMsg == NULL); } /* * GetConnParams uses the provided key to determine libpq parameters needed to * establish a connection using that key. The keywords and values are placed in * the like-named out parameters. All parameter strings are allocated in the * context provided by the caller, to save the caller needing to copy strings * into an appropriate context later. */ void GetConnParams(ConnectionHashKey *key, char ***keywords, char ***values, Index *runtimeParamStart, MemoryContext context) { /* * make space for the port as a string: sign, 10 digits, NUL. We keep it on the stack * till we can later copy it to the right context. By having the declaration here * already we can add a pointer to the runtimeValues. */ char nodePortString[12] = ""; ConnectionHashKey *effectiveKey = GetEffectiveConnKey(key); StringInfo applicationName = makeStringInfo(); appendStringInfo(applicationName, "%s%ld", CITUS_APPLICATION_NAME_PREFIX, GetGlobalPID()); /* * This function has three sections: * - Initialize the keywords and values (to be copied later) of global parameters * - Append user/host-specific parameters calculated from the given key * - (Enterprise-only) append user/host-specific authentication params * * The global parameters have already been assigned from a GUC, so begin by * calculating the key-specific parameters (basically just the fields of * the key and the active database encoding). * * We allocate everything in the provided context so as to facilitate using * pfree on all runtime parameters when connections using these entries are * invalidated during config reloads. * * Also, when "host" is already provided in global parameters, we use hostname * from the key as "hostaddr" instead of "host" to avoid host name lookup. In * that case, the value for "host" becomes useful only if the authentication * method requires it. */ bool gotHostParamFromGlobalParams = false; for (Size paramIndex = 0; paramIndex < ConnParams.size; paramIndex++) { if (strcmp(ConnParams.keywords[paramIndex], "host") == 0) { gotHostParamFromGlobalParams = true; break; } } const char *runtimeKeywords[] = { gotHostParamFromGlobalParams ? "hostaddr" : "host", "port", "dbname", "user", "client_encoding", "application_name" }; const char *runtimeValues[] = { effectiveKey->hostname, nodePortString, effectiveKey->database, effectiveKey->user, GetDatabaseEncodingName(), applicationName->data }; /* * remember where global/GUC params end and runtime ones start, all entries after this * point should be allocated in context and will be freed upon * FreeConnParamsHashEntryFields */ *runtimeParamStart = ConnParams.size; /* * Declare local params for readability; * * assignment is done directly to not lose the pointers if any of the later * allocations cause an error. FreeConnParamsHashEntryFields knows about the * possibility of half initialized keywords or values and correctly reclaims them when * the cache is reused. * * Need to zero enough space for all possible libpq parameters. */ char **connKeywords = *keywords = MemoryContextAllocZero(context, ConnParams.maxSize * sizeof(char *)); char **connValues = *values = MemoryContextAllocZero(context, ConnParams.maxSize * sizeof(char *)); /* auth keywords will begin after global and runtime ones are appended */ Index authParamsIdx = ConnParams.size + lengthof(runtimeKeywords); if (ConnParams.size + lengthof(runtimeKeywords) >= ConnParams.maxSize) { /* hopefully this error is only seen by developers */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("too many connParams entries"))); } pg_ltoa(effectiveKey->port, nodePortString); /* populate node port string with port */ /* first step: copy global parameters to beginning of array */ for (Size paramIndex = 0; paramIndex < ConnParams.size; paramIndex++) { /* copy the keyword&value pointers to the new array */ connKeywords[paramIndex] = ConnParams.keywords[paramIndex]; connValues[paramIndex] = ConnParams.values[paramIndex]; } /* second step: begin after global params and copy runtime params into our context */ for (Index runtimeParamIndex = 0; runtimeParamIndex < lengthof(runtimeKeywords); runtimeParamIndex++) { /* copy the keyword & value into our context and append to the new array */ connKeywords[ConnParams.size + runtimeParamIndex] = MemoryContextStrdup(context, runtimeKeywords[runtimeParamIndex]); connValues[ConnParams.size + runtimeParamIndex] = MemoryContextStrdup(context, runtimeValues[runtimeParamIndex]); } /* we look up authinfo by original key, not effective one */ char *authinfo = GetAuthinfo(key->hostname, key->port, key->user); char *pqerr = NULL; PQconninfoOption *optionArray = PQconninfoParse(authinfo, &pqerr); if (optionArray == NULL) { /* PQconninfoParse failed, it's unsafe to continue as this has caused segfaults in production */ if (pqerr == NULL) { /* parse failed without an error message, treat as OOM error */ ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), errdetail("Failed to parse authentication information via libpq"))); } else { /* * Parse error, should not be possible as the validity is checked upon insert into pg_dist_authinfo, * however, better safe than sorry */ /* * errmsg is populated by PQconninfoParse which requires us to free the message. Since we want to * incorporate the parse error into the detail of our message we need to copy the error message before * freeing it. Not freeing the message will leak memory. */ char *pqerrcopy = pstrdup(pqerr); PQfreemem(pqerr); ereport(ERROR, (errmsg( "failed to parse node authentication information for %s@%s:%d", key->user, key->hostname, key->port), errdetail("%s", pqerrcopy))); } } for (PQconninfoOption *option = optionArray; option->keyword != NULL; option++) { if (option->val == NULL || option->val[0] == '\0') { continue; } connKeywords[authParamsIdx] = MemoryContextStrdup(context, option->keyword); connValues[authParamsIdx] = MemoryContextStrdup(context, option->val); authParamsIdx++; } if (key->replicationConnParam) { connKeywords[authParamsIdx] = MemoryContextStrdup(context, "replication"); connValues[authParamsIdx] = MemoryContextStrdup(context, "database"); authParamsIdx++; } PQconninfoFree(optionArray); /* final step: add terminal NULL, required by libpq */ connKeywords[authParamsIdx] = connValues[authParamsIdx] = NULL; } /* * GetConnParam finds the keyword in the configured connection parameters and returns its * value. */ const char * GetConnParam(const char *keyword) { for (Size i = 0; i < ConnParams.size; i++) { if (strcmp(keyword, ConnParams.keywords[i]) == 0) { return ConnParams.values[i]; } } return NULL; } /* * GetEffectiveConnKey checks whether there is any pooler configuration for the * provided key (host/port combination). If a corresponding row is found in the * poolinfo table, a modified (effective) key is returned with the node, port, * and dbname overridden, as applicable, otherwise, the original key is returned * unmodified. * * In the case of Citus non-main databases we just return the key, since we * would not have access to tables with worker information. */ ConnectionHashKey * GetEffectiveConnKey(ConnectionHashKey *key) { PQconninfoOption *option = NULL, *optionArray = NULL; if (!IsTransactionState()) { /* we're in the task tracker, so should only see loopback */ Assert(strncmp(LocalHostName, key->hostname, MAX_NODE_LENGTH) == 0 && PostPortNumber == key->port); return key; } if (!CitusHasBeenLoaded()) { /* * This happens when we connect to main database over localhost * from some non Citus database. */ return key; } WorkerNode *worker = FindWorkerNode(key->hostname, key->port); if (worker == NULL) { /* this can be hit when the key references an unknown node */ return key; } char *poolinfo = GetPoolinfoViaCatalog(worker->nodeId); if (poolinfo == NULL) { return key; } /* copy the key to provide defaults for all fields */ ConnectionHashKey *effectiveKey = palloc(sizeof(ConnectionHashKey)); *effectiveKey = *key; optionArray = PQconninfoParse(poolinfo, NULL); for (option = optionArray; option->keyword != NULL; option++) { if (option->val == NULL || option->val[0] == '\0') { continue; } if (strcmp(option->keyword, "host") == 0) { strlcpy(effectiveKey->hostname, option->val, MAX_NODE_LENGTH); } else if (strcmp(option->keyword, "port") == 0) { effectiveKey->port = pg_strtoint32(option->val); } else if (strcmp(option->keyword, "dbname") == 0) { /* permit dbname for poolers which can key pools based on dbname */ strlcpy(effectiveKey->database, option->val, NAMEDATALEN); } else { ereport(FATAL, (errmsg("unrecognized poolinfo keyword"))); } } PQconninfoFree(optionArray); return effectiveKey; } /* * GetAuthinfo simply returns the string representation of authentication info * for a specified hostname/port/user combination. If the current transaction * is valid, then we use the catalog, otherwise a shared memory hash is used, * a mode that is currently only useful for getting authentication information * to the Task Tracker, which lacks a database connection and transaction. */ char * GetAuthinfo(char *hostname, int32 port, char *user) { char *authinfo = NULL; bool isLoopback = (strncmp(LocalHostName, hostname, MAX_NODE_LENGTH) == 0 && PostPortNumber == port); /* * Citus will not be loaded when we run a global DDL command from a * Citus non-main database. */ if (!CitusHasBeenLoaded()) { /* * We don't expect non-main databases to connect to a node other than * the local one. */ Assert(isLoopback); return ""; } if (IsTransactionState()) { int64 nodeId = WILDCARD_NODE_ID; /* -1 is a special value for loopback connections (task tracker) */ if (isLoopback) { nodeId = LOCALHOST_NODE_ID; } else { WorkerNode *worker = FindWorkerNode(hostname, port); if (worker != NULL) { nodeId = worker->nodeId; } } authinfo = GetAuthinfoViaCatalog(user, nodeId); } return (authinfo != NULL) ? authinfo : ""; } /* * CalculateMaxSize simply counts the number of elements returned by * PQconnDefaults, including the final NULL. This helps us know how space would * be used if a connection utilizes every known libpq parameter. */ static Size CalculateMaxSize() { PQconninfoOption *defaults = PQconndefaults(); Size maxSize = 0; for (PQconninfoOption *option = defaults; option->keyword != NULL; option++, maxSize++) { /* do nothing, we're just counting the elements */ } PQconninfoFree(defaults); /* we've counted elements but libpq needs a final NULL, so add one */ maxSize++; return maxSize; } /* *INDENT-OFF* */ /* * Checks if connection string starts with either of the valid URI prefix * designators. * * Returns the URI prefix length, 0 if the string doesn't contain a URI prefix. * * This implementation (mostly) taken from libpq/fe-connect.c. */ static int uri_prefix_length(const char *connstr) { const char uri_designator[] = "postgresql://"; const char short_uri_designator[] = "postgres://"; if (strncmp(connstr, uri_designator, sizeof(uri_designator) - 1) == 0) return sizeof(uri_designator) - 1; if (strncmp(connstr, short_uri_designator, sizeof(short_uri_designator) - 1) == 0) return sizeof(short_uri_designator) - 1; return 0; } /* *INDENT-ON* */ ================================================ FILE: src/backend/distributed/connection/connection_management.c ================================================ /*------------------------------------------------------------------------- * * connection_management.c * Central management of connections and their life-cycle * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pg_config.h" #include "pgstat.h" #include "safe_lib.h" #include "access/hash.h" #include "commands/dbcommands.h" #include "mb/pg_wchar.h" #include "portability/instr_time.h" #include "postmaster/postmaster.h" #include "storage/ipc.h" #include "utils/hsearch.h" #include "utils/memutils.h" #include "distributed/backend_data.h" #include "distributed/cancel_utils.h" #include "distributed/connection_management.h" #include "distributed/error_codes.h" #include "distributed/errormessage.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/memutils.h" #include "distributed/metadata_cache.h" #include "distributed/placement_connection.h" #include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" #include "distributed/shared_connection_stats.h" #include "distributed/stats/stat_counters.h" #include "distributed/time_constants.h" #include "distributed/version_compat.h" #include "distributed/worker_log_messages.h" int NodeConnectionTimeout = 30000; int MaxCachedConnectionsPerWorker = 1; int MaxCachedConnectionLifetime = 10 * MS_PER_MINUTE; HTAB *ConnectionHash = NULL; HTAB *ConnParamsHash = NULL; MemoryContext ConnectionContext = NULL; static uint32 ConnectionHashHash(const void *key, Size keysize); static int ConnectionHashCompare(const void *a, const void *b, Size keysize); static void StartConnectionEstablishment(MultiConnection *connectionn, ConnectionHashKey *key); static MultiConnection * FindAvailableConnection(dlist_head *connections, uint32 flags); static void ErrorIfMultipleMetadataConnectionExists(dlist_head *connections); static void FreeConnParamsHashEntryFields(ConnParamsHashEntry *entry); static void AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit); static bool ShouldShutdownConnection(MultiConnection *connection, const int cachedConnectionCount); static bool RemoteTransactionIdle(MultiConnection *connection); static int EventSetSizeForConnectionList(List *connections); /* types for async connection management */ enum MultiConnectionPhase { MULTI_CONNECTION_PHASE_CONNECTING, MULTI_CONNECTION_PHASE_CONNECTED, MULTI_CONNECTION_PHASE_ERROR, }; typedef struct MultiConnectionPollState { enum MultiConnectionPhase phase; MultiConnection *connection; PostgresPollingStatusType pollmode; } MultiConnectionPollState; /* helper functions for async connection management */ static bool MultiConnectionStatePoll(MultiConnectionPollState *connectionState); static WaitEventSet * WaitEventSetFromMultiConnectionStates(List *connections, int *waitCount); static void CloseNotReadyMultiConnectionStates(List *connectionStates); static uint32 MultiConnectionStateEventMask(MultiConnectionPollState *connectionState); static void CitusPQFinish(MultiConnection *connection); static ConnParamsHashEntry * FindOrCreateConnParamsEntry(ConnectionHashKey *key); /* * Initialize per-backend connection management infrastructure. */ void InitializeConnectionManagement(void) { HASHCTL info, connParamsInfo; /* * Create a single context for connection and transaction related memory * management. Doing so, instead of allocating in TopMemoryContext, makes * it easier to associate used memory. */ ConnectionContext = AllocSetContextCreateInternal(TopMemoryContext, "Connection Context", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); /* create (host,port,user,database) -> [connection] hash */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(ConnectionHashKey); info.entrysize = sizeof(ConnectionHashEntry); info.hash = ConnectionHashHash; info.match = ConnectionHashCompare; info.hcxt = ConnectionContext; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE); /* connParamsInfo is same as info, except for entrysize */ connParamsInfo = info; connParamsInfo.entrysize = sizeof(ConnParamsHashEntry); ConnectionHash = hash_create("citus connection cache (host,port,user,database)", 64, &info, hashFlags); ConnParamsHash = hash_create("citus connparams cache (host,port,user,database)", 64, &connParamsInfo, hashFlags); } /* * InvalidateConnParamsHashEntries sets every hash entry's isValid flag to false. */ void InvalidateConnParamsHashEntries(void) { if (ConnParamsHash != NULL) { ConnParamsHashEntry *entry = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, ConnParamsHash); while ((entry = (ConnParamsHashEntry *) hash_seq_search(&status)) != NULL) { entry->isValid = false; } } } /* * AfterXactConnectionHandling performs connection management activity after the end of a transaction. Both * COMMIT and ABORT paths are handled here. * * This is called by Citus' global transaction callback. */ void AfterXactConnectionHandling(bool isCommit) { HASH_SEQ_STATUS status; ConnectionHashEntry *entry; hash_seq_init(&status, ConnectionHash); while ((entry = (ConnectionHashEntry *) hash_seq_search(&status)) != 0) { if (!entry->isValid) { /* skip invalid connection hash entries */ continue; } AfterXactHostConnectionHandling(entry, isCommit); /* * NB: We leave the hash entry in place, even if there's no individual * connections in it anymore. There seems no benefit in deleting it, * and it'll save a bit of work in the next transaction. */ } } /* * GetNodeConnection() establishes a connection to remote node, using default * user and database. * * See StartNodeUserDatabaseConnection for details. */ MultiConnection * GetNodeConnection(uint32 flags, const char *hostname, int32 port) { return GetNodeUserDatabaseConnection(flags, hostname, port, NULL, NULL); } /* * StartNodeConnection initiates a connection to remote node, using default * user and database. * * See StartNodeUserDatabaseConnection for details. */ MultiConnection * StartNodeConnection(uint32 flags, const char *hostname, int32 port) { MultiConnection *connection = StartNodeUserDatabaseConnection(flags, hostname, port, NULL, NULL); /* * connection can only be NULL for optional connections, which we don't * support in this codepath. */ Assert((flags & OPTIONAL_CONNECTION) == 0); Assert(connection != NULL); return connection; } /* * GetNodeUserDatabaseConnection establishes connection to remote node. * * See StartNodeUserDatabaseConnection for details. */ MultiConnection * GetNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port, const char *user, const char *database) { MultiConnection *connection = StartNodeUserDatabaseConnection(flags, hostname, port, user, database); /* * connection can only be NULL for optional connections, which we don't * support in this codepath. */ Assert((flags & OPTIONAL_CONNECTION) == 0); Assert(connection != NULL); FinishConnectionEstablishment(connection); return connection; } /* * GetConnectionForLocalQueriesOutsideTransaction returns a localhost connection for * subtransaction. To avoid creating excessive connections, we reuse an * existing connection. */ MultiConnection * GetConnectionForLocalQueriesOutsideTransaction(char *userName) { int connectionFlag = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag, LocalHostName, PostPortNumber, userName, get_database_name(MyDatabaseId)); return connection; } /* * StartNodeUserDatabaseConnection() initiates a connection to a remote node. * * If user or database are NULL, the current session's defaults are used. The * following flags influence connection establishment behaviour: * - FORCE_NEW_CONNECTION - a new connection is required * * The returned connection has only been initiated, not fully * established. That's useful to allow parallel connection establishment. If * that's not desired use the Get* variant. */ MultiConnection * StartNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port, const char *user, const char *database) { ConnectionHashKey key; bool found; /* do some minimal input checks */ if (strlen(hostname) > MAX_NODE_LENGTH) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hostname exceeds the maximum length of %d", MAX_NODE_LENGTH))); } strlcpy(key.hostname, hostname, MAX_NODE_LENGTH); key.port = port; if (user) { strlcpy(key.user, user, NAMEDATALEN); } else { strlcpy(key.user, CurrentUserName(), NAMEDATALEN); } if (database) { strlcpy(key.database, database, NAMEDATALEN); } else { strlcpy(key.database, CurrentDatabaseName(), NAMEDATALEN); } if (flags & REQUIRE_REPLICATION_CONNECTION_PARAM) { key.replicationConnParam = true; } else { key.replicationConnParam = false; } if (CurrentCoordinatedTransactionState == COORD_TRANS_NONE) { CurrentCoordinatedTransactionState = COORD_TRANS_IDLE; } /* * Lookup relevant hash entry. We always enter. If only a cached * connection is desired, and there's none, we'll simply leave the * connection list empty. */ ConnectionHashEntry *entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found); if (!found || !entry->isValid) { /* * We are just building hash entry or previously it was left in an * invalid state as we couldn't allocate memory for it. * So initialize entry->connections list here. */ entry->isValid = false; entry->connections = MemoryContextAlloc(ConnectionContext, sizeof(dlist_head)); dlist_init(entry->connections); /* * If MemoryContextAlloc errors out -e.g. during an OOM-, entry->connections * stays as NULL. So entry->isValid should be set to true right after we * initialize entry->connections properly. */ entry->isValid = true; } /* if desired, check whether there's a usable connection */ if (!(flags & FORCE_NEW_CONNECTION)) { /* check connection cache for a connection that's not already in use */ MultiConnection *connection = FindAvailableConnection(entry->connections, flags); if (connection) { /* * Increment the connection stat counter for the connections that are * reused only if the connection is in a good state. Here we don't * bother shutting down the connection or such if it is not in a good * state but we mostly want to avoid incrementing the connection stat * counter for a connection that the caller cannot really use. */ if (PQstatus(connection->pgConn) == CONNECTION_OK) { IncrementStatCounterForMyDb(STAT_CONNECTION_REUSED); } return connection; } } else if (flags & REQUIRE_METADATA_CONNECTION) { /* FORCE_NEW_CONNECTION and REQUIRE_METADATA_CONNECTION are incompatible */ ereport(ERROR, (errmsg("metadata connections cannot be forced to open " "a new connection"))); } /* * Either no caching desired, or no pre-established, non-claimed, * connection present. Initiate connection establishment. */ MultiConnection *connection = MemoryContextAllocZero(ConnectionContext, sizeof(MultiConnection)); connection->initializationState = POOL_STATE_NOT_INITIALIZED; dlist_push_tail(entry->connections, &connection->connectionNode); /* these two flags are by nature cannot happen at the same time */ Assert(!((flags & WAIT_FOR_CONNECTION) && (flags & OPTIONAL_CONNECTION))); if (flags & WAIT_FOR_CONNECTION) { WaitLoopForSharedConnection(hostname, port); } else if (flags & OPTIONAL_CONNECTION) { /* * We can afford to skip establishing an optional connection. For * non-optional connections, we first retry for some time. If we still * cannot reserve the right to establish a connection, we prefer to * error out. */ if (!TryToIncrementSharedConnectionCounter(hostname, port)) { /* do not track the connection anymore */ dlist_delete(&connection->connectionNode); pfree(connection); /* * Here we don't increment the connection stat counter for the optional * connections that we gave up establishing due to connection throttling * because the callers who request optional connections know how to * survive without them. */ return NULL; } } else { /* * The caller doesn't want the connection manager to wait * until a connection slot is available on the remote node. * In the end, we might fail to establish connection to the * remote node as it might not have any space in * max_connections for this connection establishment. * * Still, we keep track of the connection counter. */ IncrementSharedConnectionCounter(hostname, port); } /* * We've already incremented the counter above, so we should decrement * when we're done with the connection. */ connection->initializationState = POOL_STATE_COUNTER_INCREMENTED; StartConnectionEstablishment(connection, &key); ResetShardPlacementAssociation(connection); if ((flags & REQUIRE_METADATA_CONNECTION)) { connection->useForMetadataOperations = true; } /* fully initialized the connection, record it */ connection->initializationState = POOL_STATE_INITIALIZED; return connection; } /* * FindAvailableConnection searches the given list of connections for one that * is not claimed exclusively. * * If no connection is available, FindAvailableConnection returns NULL. */ static MultiConnection * FindAvailableConnection(dlist_head *connections, uint32 flags) { List *metadataConnectionCandidateList = NIL; dlist_iter iter; dlist_foreach(iter, connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); if (flags & OUTSIDE_TRANSACTION) { /* don't return connections that are used in transactions */ if (connection-> remoteTransaction.transactionState != REMOTE_TRANS_NOT_STARTED) { continue; } } /* don't return claimed connections */ if (connection->claimedExclusively) { /* connection is in use for an ongoing operation */ continue; } if (connection->forceCloseAtTransactionEnd && !connection->remoteTransaction.beginSent) { /* * This is a connection that should be closed, probably because * of old connection options or removing a node. This will * automatically be closed at the end of the transaction. But, if we are still * inside a transaction, we should keep using this connection as long as a remote * transaction is in progress over the connection. The main use for this case * is having some commands inside a transaction block after removing nodes. And, we * currently allow very limited operations after removing a node inside a * transaction block (e.g., no placement access can happen). */ continue; } if (connection->initializationState != POOL_STATE_INITIALIZED) { /* * If the connection has not been initialized, it should not be * considered as available. */ continue; } if ((flags & REQUIRE_METADATA_CONNECTION) && !connection->useForMetadataOperations) { /* * The caller requested a metadata connection, and this is not the * metadata connection. Still, this is a candidate for becoming a * metadata connection. */ metadataConnectionCandidateList = lappend(metadataConnectionCandidateList, connection); continue; } return connection; } if ((flags & REQUIRE_METADATA_CONNECTION) && list_length(metadataConnectionCandidateList) > 0) { /* * Caller asked a metadata connection, and we couldn't find a connection * that has already been used for metadata operations. * * So, we pick the first connection as the metadata connection. */ MultiConnection *metadataConnection = linitial(metadataConnectionCandidateList); Assert(!metadataConnection->claimedExclusively); /* remember that we use this connection for metadata operations */ metadataConnection->useForMetadataOperations = true; /* * We cannot have multiple metadata connections. If we see * this error, it is likely that there is a bug in connection * management. */ ErrorIfMultipleMetadataConnectionExists(connections); return metadataConnection; } return NULL; } /* * ErrorIfMultipleMetadataConnectionExists throws an error if the * input connection dlist contains more than one metadata connections. */ static void ErrorIfMultipleMetadataConnectionExists(dlist_head *connections) { bool foundMetadataConnection = false; dlist_iter iter; dlist_foreach(iter, connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); if (connection->useForMetadataOperations) { if (foundMetadataConnection) { ereport(ERROR, (errmsg("cannot have multiple metadata connections"))); } foundMetadataConnection = true; } } } /* * CloseAllConnectionsAfterTransaction sets the forceClose flag of all the * connections. This is mainly done when citus.node_conninfo changes. */ void CloseAllConnectionsAfterTransaction(void) { if (ConnectionHash == NULL) { return; } HASH_SEQ_STATUS status; ConnectionHashEntry *entry; hash_seq_init(&status, ConnectionHash); while ((entry = (ConnectionHashEntry *) hash_seq_search(&status)) != 0) { if (!entry->isValid) { /* skip invalid connection hash entries */ continue; } dlist_iter iter; dlist_head *connections = entry->connections; dlist_foreach(iter, connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); connection->forceCloseAtTransactionEnd = true; } } } /* * ConnectionAvailableToNode returns a MultiConnection if the session has at least * one connection established and avaliable to use to the give node. Else, returns * false. */ MultiConnection * ConnectionAvailableToNode(char *hostName, int nodePort, const char *userName, const char *database) { ConnectionHashKey key; bool found = false; strlcpy(key.hostname, hostName, MAX_NODE_LENGTH); key.port = nodePort; strlcpy(key.user, userName, NAMEDATALEN); strlcpy(key.database, database, NAMEDATALEN); key.replicationConnParam = false; ConnectionHashEntry *entry = (ConnectionHashEntry *) hash_search(ConnectionHash, &key, HASH_FIND, &found); if (!found || !entry->isValid) { return false; } int flags = 0; MultiConnection *connection = FindAvailableConnection(entry->connections, flags); return connection; } /* * CloseNodeConnectionsAfterTransaction sets the forceClose flag of the connections * to a particular node as true such that the connections are no longer cached. This * is mainly used when a worker leaves the cluster. */ void CloseNodeConnectionsAfterTransaction(char *nodeName, int nodePort) { HASH_SEQ_STATUS status; ConnectionHashEntry *entry; hash_seq_init(&status, ConnectionHash); while ((entry = (ConnectionHashEntry *) hash_seq_search(&status)) != 0) { if (!entry->isValid) { /* skip invalid connection hash entries */ continue; } dlist_iter iter; if (strcmp(entry->key.hostname, nodeName) != 0 || entry->key.port != nodePort) { continue; } dlist_head *connections = entry->connections; dlist_foreach(iter, connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); connection->forceCloseAtTransactionEnd = true; } } } /* * Close a previously established connection. */ void CloseConnection(MultiConnection *connection) { ConnectionHashKey key; bool found; /* close connection */ CitusPQFinish(connection); strlcpy(key.hostname, connection->hostname, MAX_NODE_LENGTH); key.port = connection->port; key.replicationConnParam = connection->requiresReplication; strlcpy(key.user, connection->user, NAMEDATALEN); strlcpy(key.database, connection->database, NAMEDATALEN); hash_search(ConnectionHash, &key, HASH_FIND, &found); if (found) { /* unlink from list of open connections */ dlist_delete(&connection->connectionNode); /* same for transaction state and shard/placement machinery */ CloseShardPlacementAssociation(connection); ResetRemoteTransaction(connection); /* we leave the per-host entry alive */ pfree(connection); } else { ereport(ERROR, (errmsg("closing untracked connection"))); } } /* * ShutdownAllConnections shutdowns all the MultiConnections in the * ConnectionHash. * * This function is intended to be called atexit() of the backend, so * that the cached connections are closed properly. Calling this function * at another point in the code could be dangerous, so think twice if you * need to call this function. */ void ShutdownAllConnections(void) { ConnectionHashEntry *entry = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, ConnectionHash); while ((entry = (ConnectionHashEntry *) hash_seq_search(&status)) != NULL) { if (!entry->isValid) { /* skip invalid connection hash entries */ continue; } dlist_iter iter; dlist_foreach(iter, entry->connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); ShutdownConnection(connection); } } } /* * ShutdownConnection, if necessary cancels the currently running statement, * and then closes the underlying libpq connection. The MultiConnection * itself is left intact. * * NB: Cancelling a statement requires network IO, and currently is not * interruptible. Unfortunately libpq does not provide a non-blocking * implementation of PQcancel(), so we don't have much choice for now. */ void ShutdownConnection(MultiConnection *connection) { /* * Only cancel statement if there's currently one running, and the * connection is in an OK state. */ if (PQstatus(connection->pgConn) == CONNECTION_OK && PQtransactionStatus(connection->pgConn) == PQTRANS_ACTIVE) { SendCancelationRequest(connection); } CitusPQFinish(connection); } /* * MultiConnectionStatePoll executes a PQconnectPoll on the connection to progress the * connection establishment. The return value of this function indicates if the * MultiConnectionPollState has been changed, which could require a change to the WaitEventSet */ static bool MultiConnectionStatePoll(MultiConnectionPollState *connectionState) { MultiConnection *connection = connectionState->connection; ConnStatusType status = PQstatus(connection->pgConn); PostgresPollingStatusType oldPollmode = connectionState->pollmode; Assert(connectionState->phase == MULTI_CONNECTION_PHASE_CONNECTING); if (status == CONNECTION_OK) { connectionState->phase = MULTI_CONNECTION_PHASE_CONNECTED; return true; } else if (status == CONNECTION_BAD) { /* FIXME: retries? */ connectionState->phase = MULTI_CONNECTION_PHASE_ERROR; return true; } else { connectionState->phase = MULTI_CONNECTION_PHASE_CONNECTING; } connectionState->pollmode = PQconnectPoll(connection->pgConn); /* * FIXME: Do we want to add transparent retry support here? */ if (connectionState->pollmode == PGRES_POLLING_FAILED) { connectionState->phase = MULTI_CONNECTION_PHASE_ERROR; return true; } else if (connectionState->pollmode == PGRES_POLLING_OK) { connectionState->phase = MULTI_CONNECTION_PHASE_CONNECTED; return true; } else { Assert(connectionState->pollmode == PGRES_POLLING_WRITING || connectionState->pollmode == PGRES_POLLING_READING); } return (oldPollmode != connectionState->pollmode) ? true : false; } /* * EventSetSizeForConnectionList calculates the space needed for a WaitEventSet based on a * list of connections. */ inline static int EventSetSizeForConnectionList(List *connections) { /* we need space for 2 postgres events in the waitset on top of the connections */ return list_length(connections) + 2; } /* * WaitEventSetFromMultiConnectionStates takes a list of MultiConnectionStates and adds * all sockets of the connections that are still in the connecting phase to a WaitSet, * taking into account the maximum number of connections that could be added in total to * a WaitSet. * * waitCount populates the number of connections added to the WaitSet in case when a * non-NULL pointer is provided. */ static WaitEventSet * WaitEventSetFromMultiConnectionStates(List *connections, int *waitCount) { const int eventSetSize = EventSetSizeForConnectionList(connections); int numEventsAdded = 0; if (waitCount) { *waitCount = 0; } WaitEventSet *waitEventSet = CreateWaitEventSet(WaitEventSetTracker_compat, eventSetSize); EnsureReleaseResource((MemoryContextCallbackFunction) (&FreeWaitEventSet), waitEventSet); /* * Put the wait events for the signal latch and postmaster death at the end such that * event index + pendingConnectionsStartIndex = the connection index in the array. */ AddWaitEventToSet(waitEventSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); AddWaitEventToSet(waitEventSet, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); numEventsAdded += 2; MultiConnectionPollState *connectionState = NULL; foreach_declared_ptr(connectionState, connections) { if (numEventsAdded >= eventSetSize) { /* room for events to schedule is exhausted */ break; } if (connectionState->phase != MULTI_CONNECTION_PHASE_CONNECTING) { /* connections that are not connecting will not be added to the WaitSet */ continue; } int sock = PQsocket(connectionState->connection->pgConn); int eventMask = MultiConnectionStateEventMask(connectionState); int waitEventSetIndex = CitusAddWaitEventSetToSet(waitEventSet, eventMask, sock, NULL, (void *) connectionState); if (waitEventSetIndex == WAIT_EVENT_SET_INDEX_FAILED) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection establishment for node %s:%d failed", connectionState->connection->hostname, connectionState->connection->port), errhint("Check both the local and remote server logs for the " "connection establishment errors."))); } numEventsAdded++; if (waitCount) { *waitCount = *waitCount + 1; } } return waitEventSet; } /* * MultiConnectionStateEventMask returns the eventMask use by the WaitEventSet for the * for the socket associated with the connection based on the pollmode PQconnectPoll * returned in its last invocation */ static uint32 MultiConnectionStateEventMask(MultiConnectionPollState *connectionState) { uint32 eventMask = 0; if (connectionState->pollmode == PGRES_POLLING_READING) { eventMask |= WL_SOCKET_READABLE; } else { eventMask |= WL_SOCKET_WRITEABLE; } return eventMask; } /* * FinishConnectionListEstablishment takes a list of MultiConnection and finishes the * connections establishment asynchronously for all connections not already fully * connected. */ void FinishConnectionListEstablishment(List *multiConnectionList) { instr_time connectionStart; INSTR_TIME_SET_CURRENT(connectionStart); List *connectionStates = NULL; WaitEventSet *waitEventSet = NULL; bool waitEventSetRebuild = true; int waitCount = 0; MultiConnection *connection = NULL; foreach_declared_ptr(connection, multiConnectionList) { MultiConnectionPollState *connectionState = palloc0(sizeof(MultiConnectionPollState)); connectionState->connection = connection; /* * before we can build the waitset to wait for asynchronous IO we need to know the * pollmode to use for the sockets. This is populated by executing one round of * PQconnectPoll. This updates the MultiConnectionPollState struct with its phase and * its next poll mode. */ MultiConnectionStatePoll(connectionState); connectionStates = lappend(connectionStates, connectionState); if (connectionState->phase == MULTI_CONNECTION_PHASE_CONNECTING) { waitCount++; } else if (connectionState->phase == MULTI_CONNECTION_PHASE_ERROR) { /* * Here we count the connections establishments that failed and that * we won't wait anymore. */ IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); } } /* prepare space for socket events */ WaitEvent *events = (WaitEvent *) palloc0(EventSetSizeForConnectionList( connectionStates) * sizeof(WaitEvent)); /* * for high connection counts with lots of round trips we could potentially have a lot * of (big) waitsets that we'd like to clean right after we have used them. To do this * we switch to a temporary memory context for this loop which gets reset at the end */ MemoryContext oldContext = MemoryContextSwitchTo( AllocSetContextCreate(CurrentMemoryContext, "connection establishment temporary context", ALLOCSET_DEFAULT_SIZES)); while (waitCount > 0) { long timeout = MillisecondsToTimeout(connectionStart, NodeConnectionTimeout); if (waitEventSetRebuild) { MemoryContextReset(CurrentMemoryContext); waitEventSet = WaitEventSetFromMultiConnectionStates(connectionStates, &waitCount); waitEventSetRebuild = false; if (waitCount <= 0) { break; } } int eventCount = WaitEventSetWait(waitEventSet, timeout, events, waitCount, WAIT_EVENT_CLIENT_READ); for (int eventIndex = 0; eventIndex < eventCount; eventIndex++) { WaitEvent *event = &events[eventIndex]; MultiConnectionPollState *connectionState = (MultiConnectionPollState *) event->user_data; if (event->events & WL_POSTMASTER_DEATH) { /* * Here we don't increment the connection stat counter for the * optional failed connections because this is not a connection * failure, but a postmaster death in the local node. */ ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } if (event->events & WL_LATCH_SET) { ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); if (IsHoldOffCancellationReceived()) { /* * because we can't break from 2 loops easily we need to not forget to * reset the memory context */ MemoryContextDelete(MemoryContextSwitchTo(oldContext)); /* * Similarly, we don't increment the connection stat counter for the * failed connections here because this is not a connection failure * but a cancellation request is received. */ return; } continue; } bool beforePollSocket = PQsocket(connectionState->connection->pgConn); bool connectionStateChanged = MultiConnectionStatePoll(connectionState); if (beforePollSocket != PQsocket(connectionState->connection->pgConn)) { /* rebuild the wait events if MultiConnectionStatePoll() changed the socket */ waitEventSetRebuild = true; } if (connectionStateChanged) { if (connectionState->phase != MULTI_CONNECTION_PHASE_CONNECTING) { /* we cannot stop waiting for connection, so rebuild the event set */ waitEventSetRebuild = true; } else { /* connection state changed, reset the event mask */ uint32 eventMask = MultiConnectionStateEventMask(connectionState); bool success = CitusModifyWaitEvent(waitEventSet, event->pos, eventMask, NULL); if (!success) { IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection establishment for node %s:%d " "failed", connection->hostname, connection->port), errhint("Check both the local and remote server " "logs for the connection establishment " "errors."))); } } /* * The state has changed to connected, update the connection's * state as well. */ if (connectionState->phase == MULTI_CONNECTION_PHASE_CONNECTED) { /* * Since WaitEventSetFromMultiConnectionStates() only adds the * connections that we haven't completed the connection * establishment yet, here we always have a new connection. * In other words, at this point, we surely know that we're * not dealing with a cached connection. */ bool newConnection = true; MarkConnectionConnected(connectionState->connection, newConnection); } } } if (eventCount == 0) { /* * timeout has occurred on waitset, double check the timeout since * connectionStart and if passed close all non-finished connections */ if (MillisecondsPassedSince(connectionStart) >= NodeConnectionTimeout) { /* * showing as a warning, can't be an error. In some cases queries can * proceed with only some of the connections being fully established. * Queries that can't will error then and there */ ereport(WARNING, (errmsg("could not establish connection after %u ms", NodeConnectionTimeout))); /* * Close all connections that have not been fully established. */ CloseNotReadyMultiConnectionStates(connectionStates); break; } } } MemoryContextDelete(MemoryContextSwitchTo(oldContext)); } /* * MillisecondsPassedSince returns the number of milliseconds elapsed between an * instr_time & the current time. */ double MillisecondsPassedSince(instr_time moment) { instr_time timeSinceMoment; INSTR_TIME_SET_CURRENT(timeSinceMoment); INSTR_TIME_SUBTRACT(timeSinceMoment, moment); return INSTR_TIME_GET_MILLISEC(timeSinceMoment); } /* * MillisecondsToTimeout returns the numer of milliseconds that still need to elapse * before msAfterStart milliseconds have passed since start. The outcome can be used to * pass to the Wait of an EventSet to make sure it returns after the timeout has passed. */ long MillisecondsToTimeout(instr_time start, long msAfterStart) { return msAfterStart - MillisecondsPassedSince(start); } /* * CloseNotReadyMultiConnectionStates calls CloseConnection for all MultiConnection's * tracked in the MultiConnectionPollState list passed in, only if the connection is not yet * fully established. * * This function removes the pointer to the MultiConnection data after the Connections are * closed since they should not be used anymore. */ static void CloseNotReadyMultiConnectionStates(List *connectionStates) { MultiConnectionPollState *connectionState = NULL; foreach_declared_ptr(connectionState, connectionStates) { MultiConnection *connection = connectionState->connection; if (connectionState->phase != MULTI_CONNECTION_PHASE_CONNECTING) { continue; } /* close connection, otherwise we take up resource on the other side */ CitusPQFinish(connection); IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); } } /* * CitusPQFinish is a wrapper around PQfinish and does book keeping on shared connection * counters. */ static void CitusPQFinish(MultiConnection *connection) { if (connection->pgConn != NULL) { PQfinish(connection->pgConn); connection->pgConn = NULL; } /* behave idempotently, there is no gurantee that CitusPQFinish() is called once */ if (connection->initializationState >= POOL_STATE_COUNTER_INCREMENTED) { DecrementSharedConnectionCounter(connection->hostname, connection->port); connection->initializationState = POOL_STATE_NOT_INITIALIZED; } } /* * Close connections on timeout in FinishConnectionListEstablishment * Synchronously finish connection establishment of an individual connection. * This function is a convenience wrapped around FinishConnectionListEstablishment. */ void FinishConnectionEstablishment(MultiConnection *connection) { FinishConnectionListEstablishment(list_make1(connection)); } /* * ForceConnectionCloseAtTransactionEnd marks connection to be closed at the end of the * transaction. */ void ForceConnectionCloseAtTransactionEnd(MultiConnection *connection) { connection->forceCloseAtTransactionEnd = true; } /* * ClaimConnectionExclusively signals that this connection is actively being * used. That means it'll not be, again, returned by * StartNodeUserDatabaseConnection() et al until releases with * UnclaimConnection(). */ void ClaimConnectionExclusively(MultiConnection *connection) { Assert(!connection->claimedExclusively); connection->claimedExclusively = true; } /* * UnclaimConnection signals that this connection is not being used * anymore. That means it again may be returned by * StartNodeUserDatabaseConnection() et al. */ void UnclaimConnection(MultiConnection *connection) { connection->claimedExclusively = false; } static uint32 ConnectionHashHash(const void *key, Size keysize) { ConnectionHashKey *entry = (ConnectionHashKey *) key; uint32 hash = string_hash(entry->hostname, NAMEDATALEN); hash = hash_combine(hash, hash_uint32(entry->port)); hash = hash_combine(hash, string_hash(entry->user, NAMEDATALEN)); hash = hash_combine(hash, string_hash(entry->database, NAMEDATALEN)); hash = hash_combine(hash, hash_uint32(entry->replicationConnParam)); return hash; } static int ConnectionHashCompare(const void *a, const void *b, Size keysize) { ConnectionHashKey *ca = (ConnectionHashKey *) a; ConnectionHashKey *cb = (ConnectionHashKey *) b; if (strncmp(ca->hostname, cb->hostname, MAX_NODE_LENGTH) != 0 || ca->port != cb->port || ca->replicationConnParam != cb->replicationConnParam || strncmp(ca->user, cb->user, NAMEDATALEN) != 0 || strncmp(ca->database, cb->database, NAMEDATALEN) != 0) { return 1; } else { return 0; } } /* * Asynchronously establish connection to a remote node, but don't wait for * that to finish. DNS lookups etc. are performed synchronously though. */ static void StartConnectionEstablishment(MultiConnection *connection, ConnectionHashKey *key) { static uint64 connectionId = 1; ConnParamsHashEntry *entry = FindOrCreateConnParamsEntry(key); strlcpy(connection->hostname, key->hostname, MAX_NODE_LENGTH); connection->port = key->port; strlcpy(connection->database, key->database, NAMEDATALEN); strlcpy(connection->user, key->user, NAMEDATALEN); connection->requiresReplication = key->replicationConnParam; connection->pgConn = PQconnectStartParams((const char **) entry->keywords, (const char **) entry->values, false); INSTR_TIME_SET_CURRENT(connection->connectionEstablishmentStart); /* do not increment for restarted connections */ if (connection->connectionId == 0) { connection->connectionId = connectionId++; } /* * To avoid issues with interrupts not getting caught all our connections * are managed in a non-blocking manner. remote_commands.c provides * wrappers emulating blocking behaviour. */ PQsetnonblocking(connection->pgConn, true); SetCitusNoticeReceiver(connection); } /* * FindOrCreateConnParamsEntry searches ConnParamsHash for the given key, * if it is not found, it is created. */ static ConnParamsHashEntry * FindOrCreateConnParamsEntry(ConnectionHashKey *key) { bool found = false; /* search our cache for precomputed connection settings */ ConnParamsHashEntry *entry = hash_search(ConnParamsHash, key, HASH_ENTER, &found); if (!found || !entry->isValid) { if (!found) { /* * Zero out entry, but not the key part. * Avoids leaving invalid pointers in hash table if GetConnParam throws with MemoryContextAllocZero. */ memset(((char *) entry) + sizeof(ConnectionHashKey), 0, sizeof(ConnParamsHashEntry) - sizeof(ConnectionHashKey)); } /* avoid leaking memory in the keys and values arrays */ if (found && !entry->isValid) { FreeConnParamsHashEntryFields(entry); } /* if not found or not valid, compute them from GUC, runtime, etc. */ GetConnParams(key, &entry->keywords, &entry->values, &entry->runtimeParamStart, ConnectionContext); entry->isValid = true; } return entry; } /* * FreeConnParamsHashEntryFields frees any dynamically allocated memory reachable * from the fields of the provided ConnParamsHashEntry. This includes all runtime * libpq keywords and values, as well as the actual arrays storing them. */ static void FreeConnParamsHashEntryFields(ConnParamsHashEntry *entry) { /* * if there was a memory error during the initialization of ConnParamHashEntry in * GetConnParams the keywords or values might not have been initialized completely. * We check if they have been initialized before freeing them. * * We only iteratively free the lists starting at the index pointed to by * entry->runtimeParamStart as all entries before are settings that are managed * separately. */ if (entry->keywords != NULL) { char **keyword = &entry->keywords[entry->runtimeParamStart]; while (*keyword != NULL) { pfree(*keyword); keyword++; } pfree(entry->keywords); entry->keywords = NULL; } if (entry->values != NULL) { char **value = &entry->values[entry->runtimeParamStart]; while (*value != NULL) { pfree(*value); value++; } pfree(entry->values); entry->values = NULL; } entry->runtimeParamStart = 0; } /* * AfterXactHostConnectionHandling closes all remote connections if not necessary anymore (i.e. not session * lifetime), or if in a failed state. */ static void AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit) { if (!entry || !entry->isValid) { /* callers only pass valid hash entries but let's be on the safe side */ ereport(ERROR, (errmsg("connection hash entry is NULL or invalid"))); } dlist_mutable_iter iter; int cachedConnectionCount = 0; dlist_foreach_modify(iter, entry->connections) { MultiConnection *connection = dlist_container(MultiConnection, connectionNode, iter.cur); /* * To avoid leaking connections we warn if connections are * still claimed exclusively. We can only do so if the transaction is * committed, as it's normal that code didn't have chance to clean * up after errors. */ if (isCommit && connection->claimedExclusively) { ereport(WARNING, (errmsg("connection claimed exclusively at transaction commit"))); } if (ShouldShutdownConnection(connection, cachedConnectionCount)) { ShutdownConnection(connection); /* remove from transactionlist before free-ing */ ResetRemoteTransaction(connection); /* unlink from list */ dlist_delete(iter.cur); pfree(connection); } else { /* * reset healthy session lifespan connections. */ ResetRemoteTransaction(connection); UnclaimConnection(connection); cachedConnectionCount++; } } } /* * ShouldShutdownConnection returns true if either one of the followings is true: * - The connection is citus initiated. * - Current cached connections is already at MaxCachedConnectionsPerWorker * - Connection is forced to close at the end of transaction * - Connection is not in OK state * - Connection has a replication origin setup * - A transaction is still in progress (usually because we are cancelling a distributed transaction) * - A connection reached its maximum lifetime */ static bool ShouldShutdownConnection(MultiConnection *connection, const int cachedConnectionCount) { /* * When we are in a backend that was created to serve an internal connection * from the coordinator or another worker, we disable connection caching to avoid * escalating the number of cached connections. We can recognize such backends * from their application name. */ return (IsCitusInternalBackend() || IsRebalancerInternalBackend()) || connection->initializationState != POOL_STATE_INITIALIZED || cachedConnectionCount >= MaxCachedConnectionsPerWorker || connection->forceCloseAtTransactionEnd || PQstatus(connection->pgConn) != CONNECTION_OK || !RemoteTransactionIdle(connection) || connection->requiresReplication || connection->isReplicationOriginSessionSetup || (MaxCachedConnectionLifetime >= 0 && MillisecondsToTimeout(connection->connectionEstablishmentStart, MaxCachedConnectionLifetime) <= 0); } /* * RestartConnection starts a new connection attempt for the given * MultiConnection. * * The internal state of the MultiConnection is preserved. For example, we * assume that we already went through all the other initialization steps in * StartNodeUserDatabaseConnection, such as incrementing shared connection * counters. * * This function should be used cautiously. If a connection is already * involved in a remote transaction, we cannot restart the underlying * connection. The caller is responsible for enforcing the restrictions * on this. */ void RestartConnection(MultiConnection *connection) { /* we cannot restart any connection that refers to a placement */ Assert(dlist_is_empty(&connection->referencedPlacements)); /* we cannot restart any connection that is part of a transaction */ Assert(connection->remoteTransaction.transactionState == REMOTE_TRANS_NOT_STARTED); ConnectionHashKey key; strlcpy(key.hostname, connection->hostname, MAX_NODE_LENGTH); key.port = connection->port; strlcpy(key.user, connection->user, NAMEDATALEN); strlcpy(key.database, connection->database, NAMEDATALEN); key.replicationConnParam = connection->requiresReplication; /* * With low-level APIs, we shutdown and restart the connection. * The main trick here is that we are using the same MultiConnection * * such that all the state of the connection is preserved. */ ShutdownConnection(connection); StartConnectionEstablishment(connection, &key); /* * We are restarting an already initialized connection which has * gone through StartNodeUserDatabaseConnection(). That's why we * can safely mark the state initialized. * * Not that we have to do this because ShutdownConnection() sets the * state to not initialized. */ connection->initializationState = POOL_STATE_INITIALIZED; connection->connectionState = MULTI_CONNECTION_CONNECTING; } /* * RemoteTransactionIdle function returns true if we manually * set flag on run_commands_on_session_level_connection_to_node to true to * force connection API keeping connection open or the status of the connection * is idle. */ static bool RemoteTransactionIdle(MultiConnection *connection) { /* * This is a very special case where we're running isolation tests on MX. * We don't care whether the transaction is idle or not when we're * running MX isolation tests. Thus, let the caller act as if the remote * transactions is idle. */ if (AllowNonIdleTransactionOnXactHandling()) { return true; } return PQtransactionStatus(connection->pgConn) == PQTRANS_IDLE; } /* * MarkConnectionConnected is a helper function which sets the connection * connectionState to MULTI_CONNECTION_CONNECTED, and also updates connection * establishment time when necessary. */ void MarkConnectionConnected(MultiConnection *connection, bool newConnection) { connection->connectionState = MULTI_CONNECTION_CONNECTED; if (INSTR_TIME_IS_ZERO(connection->connectionEstablishmentEnd)) { INSTR_TIME_SET_CURRENT(connection->connectionEstablishmentEnd); } if (newConnection) { IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_SUCCEEDED); } } /* * CitusAddWaitEventSetToSet is a wrapper around Postgres' AddWaitEventToSet(). * * AddWaitEventToSet() may throw hard errors. For example, when the * underlying socket for a connection is closed by the remote server * and already reflected by the OS, however Citus hasn't had a chance * to get this information. In that case, if replication factor is >1, * Citus can failover to other nodes for executing the query. Even if * replication factor = 1, Citus can give much nicer errors. * * So CitusAddWaitEventSetToSet simply puts ModifyWaitEvent into a * PG_TRY/PG_CATCH block in order to catch any hard errors, and * returns this information to the caller. */ int CitusAddWaitEventSetToSet(WaitEventSet *set, uint32 events, pgsocket fd, Latch *latch, void *user_data) { volatile int waitEventSetIndex = WAIT_EVENT_SET_INDEX_NOT_INITIALIZED; MemoryContext savedContext = CurrentMemoryContext; PG_TRY(); { waitEventSetIndex = AddWaitEventToSet(set, events, fd, latch, (void *) user_data); } PG_CATCH(); { /* * We might be in an arbitrary memory context when the * error is thrown and we should get back to one we had * at PG_TRY() time, especially because we are not * re-throwing the error. */ MemoryContextSwitchTo(savedContext); FlushErrorState(); /* let the callers know about the failure */ waitEventSetIndex = WAIT_EVENT_SET_INDEX_FAILED; } PG_END_TRY(); return waitEventSetIndex; } /* * CitusModifyWaitEvent is a wrapper around Postgres' ModifyWaitEvent(). * * ModifyWaitEvent may throw hard errors. For example, when the underlying * socket for a connection is closed by the remote server and already * reflected by the OS, however Citus hasn't had a chance to get this * information. In that case, if replication factor is >1, Citus can * failover to other nodes for executing the query. Even if replication * factor = 1, Citus can give much nicer errors. * * So CitusModifyWaitEvent simply puts ModifyWaitEvent into a PG_TRY/PG_CATCH * block in order to catch any hard errors, and returns this information to the * caller. */ bool CitusModifyWaitEvent(WaitEventSet *set, int pos, uint32 events, Latch *latch) { volatile bool success = true; MemoryContext savedContext = CurrentMemoryContext; PG_TRY(); { ModifyWaitEvent(set, pos, events, latch); } PG_CATCH(); { /* * We might be in an arbitrary memory context when the * error is thrown and we should get back to one we had * at PG_TRY() time, especially because we are not * re-throwing the error. */ MemoryContextSwitchTo(savedContext); FlushErrorState(); /* let the callers know about the failure */ success = false; } PG_END_TRY(); return success; } ================================================ FILE: src/backend/distributed/connection/locally_reserved_shared_connections.c ================================================ /*------------------------------------------------------------------------- * * locally_reserved_shared_connections.c * * Keeps track of the number of reserved connections to remote nodes * for this backend. The primary goal is to complement the logic * implemented in shared_connections.c which aims to prevent excessive * number of connections (typically > max_connections) to any worker node. * With this locally reserved connection stats, we enforce the same * constraints considering these locally reserved shared connections. * * To be more precise, shared connection stats are incremented only with two * operations: (a) Establishing a connection to a remote node * (b) Reserving connections, the logic that this * file implements. * * Finally, as the name already implies, once a node has reserved a shared * connection, it is guaranteed to have the right to establish a connection * to the given remote node when needed. * * For COPY command, we use this fact to reserve connections to the remote nodes * in the same order as the adaptive executor in order to prevent any resource * starvations. We need to do this because COPY establishes connections when it * receives a tuple that targets a remote node. This is a valuable optimization * to prevent unnecessary connection establishments, which are pretty expensive. * Instead, COPY command can reserve connections upfront, and utilize them when * they are actually needed. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/hash.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "utils/builtins.h" #include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/placement_connection.h" #include "distributed/shared_connection_stats.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" #define RESERVED_CONNECTION_COLUMNS 4 /* session specific hash map*/ static HTAB *SessionLocalReservedConnections = NULL; /* * Hash key for connection reservations */ typedef struct ReservedConnectionHashKey { char hostname[MAX_NODE_LENGTH]; int32 port; Oid databaseOid; Oid userId; } ReservedConnectionHashKey; /* * Hash entry for per worker information. The rules are as follows: * - If there is no entry in the hash, we can make a reservation. * - If usedReservation is false, we have a reservation that we can use. * - If usedReservation is true, we used the reservation and cannot make more reservations. */ typedef struct ReservedConnectionHashEntry { ReservedConnectionHashKey key; bool usedReservation; } ReservedConnectionHashEntry; static void StoreAllReservedConnections(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); static ReservedConnectionHashEntry * AllocateOrGetReservedConnectionEntry(char *hostName, int nodePort, Oid userId, Oid databaseOid, bool *found); static void EnsureConnectionPossibilityForNodeList(List *nodeList); static bool EnsureConnectionPossibilityForNode(WorkerNode *workerNode, bool waitForConnection); static uint32 LocalConnectionReserveHashHash(const void *key, Size keysize); static int LocalConnectionReserveHashCompare(const void *a, const void *b, Size keysize); PG_FUNCTION_INFO_V1(citus_reserved_connection_stats); /* * citus_reserved_connection_stats returns all the avaliable information about all * the reserved connections. This function is used mostly for testing. */ Datum citus_reserved_connection_stats(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); StoreAllReservedConnections(tupleStore, tupleDescriptor); PG_RETURN_VOID(); } /* * StoreAllReservedConnections gets connections established from the current node * and inserts them into the given tuplestore. * * We don't need to enforce any access privileges as the number of backends * on any node is already visible on pg_stat_activity to all users. */ static void StoreAllReservedConnections(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { Datum values[RESERVED_CONNECTION_COLUMNS]; bool isNulls[RESERVED_CONNECTION_COLUMNS]; HASH_SEQ_STATUS status; ReservedConnectionHashEntry *connectionEntry = NULL; hash_seq_init(&status, SessionLocalReservedConnections); while ((connectionEntry = (ReservedConnectionHashEntry *) hash_seq_search(&status)) != 0) { /* get ready for the next tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); char *databaseName = get_database_name(connectionEntry->key.databaseOid); if (databaseName == NULL) { /* database might have been dropped */ continue; } values[0] = PointerGetDatum(cstring_to_text(connectionEntry->key.hostname)); values[1] = Int32GetDatum(connectionEntry->key.port); values[2] = PointerGetDatum(cstring_to_text(databaseName)); values[3] = BoolGetDatum(connectionEntry->usedReservation); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } } /* * InitializeLocallyReservedSharedConnections initializes the hashmap in * ConnectionContext. */ void InitializeLocallyReservedSharedConnections(void) { HASHCTL reservedConnectionInfo; memset(&reservedConnectionInfo, 0, sizeof(reservedConnectionInfo)); reservedConnectionInfo.keysize = sizeof(ReservedConnectionHashKey); reservedConnectionInfo.entrysize = sizeof(ReservedConnectionHashEntry); /* * ConnectionContext is the session local memory context that is used for * tracking remote connections. */ reservedConnectionInfo.hcxt = ConnectionContext; reservedConnectionInfo.hash = LocalConnectionReserveHashHash; reservedConnectionInfo.match = LocalConnectionReserveHashCompare; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE); SessionLocalReservedConnections = hash_create("citus session level reserved connections (host,port,database,user)", 64, &reservedConnectionInfo, hashFlags); } /* * CanUseReservedConnection returns true if we have already reserved at least * one shared connection in this session that is not used. */ bool CanUseReservedConnection(const char *hostName, int nodePort, Oid userId, Oid databaseOid) { ReservedConnectionHashKey key; strlcpy(key.hostname, hostName, MAX_NODE_LENGTH); key.userId = userId; key.port = nodePort; key.databaseOid = databaseOid; bool found = false; ReservedConnectionHashEntry *entry = (ReservedConnectionHashEntry *) hash_search(SessionLocalReservedConnections, &key, HASH_FIND, &found); if (!found || !entry) { return false; } return !entry->usedReservation; } /* * DeallocateReservedConnections is responsible for two things. First, if the operation * has reserved a connection but not used, it gives back the connection back to the * shared memory pool. Second, for all cases, it deallocates the session local entry from * the hash. */ void DeallocateReservedConnections(void) { HASH_SEQ_STATUS status; ReservedConnectionHashEntry *entry; hash_seq_init(&status, SessionLocalReservedConnections); while ((entry = (ReservedConnectionHashEntry *) hash_seq_search(&status)) != 0) { if (!entry->usedReservation) { /* * We have not used this reservation, make sure to clean-up from * the shared memory as well. */ DecrementSharedConnectionCounter(entry->key.hostname, entry->key.port); /* for completeness, set it to true */ entry->usedReservation = true; } /* * We cleaned up all the entries because we may not need reserved connections * in the next iteration. */ bool found = false; hash_search(SessionLocalReservedConnections, entry, HASH_REMOVE, &found); Assert(found); } } /* * MarkReservedConnectionUsed sets the local hash that the reservation is used. */ void MarkReservedConnectionUsed(const char *hostName, int nodePort, Oid userId, Oid databaseOid) { ReservedConnectionHashKey key; strlcpy(key.hostname, hostName, MAX_NODE_LENGTH); key.userId = userId; key.port = nodePort; key.databaseOid = databaseOid; bool found = false; ReservedConnectionHashEntry *entry = (ReservedConnectionHashEntry *) hash_search( SessionLocalReservedConnections, &key, HASH_FIND, &found); if (!found) { ereport(ERROR, (errmsg("BUG: untracked reserved connection"), errhint("Set citus.max_shared_pool_size TO -1 to " "disable reserved connection counters"))); } /* a reservation can only be used once */ Assert(!entry->usedReservation); entry->usedReservation = true; } /* * EnsureConnectionPossibilityForRemotePrimaryNodes is a wrapper around * EnsureConnectionPossibilityForNodeList. */ void EnsureConnectionPossibilityForRemotePrimaryNodes(void) { /* * By using NoLock there is a tiny risk of that we miss to reserve a * connection for a concurrently added node. However, that doesn't * seem to cause any problems as none of the placements that we are * going to access would be on the new node. */ List *remoteNodeList = ActivePrimaryRemoteNodeList(NoLock); EnsureConnectionPossibilityForNodeList(remoteNodeList); } /* * TryConnectionPossibilityForLocalPrimaryNode returns true if the primary * local node is in the metadata an we can reserve a connection for the node. * If not, the function returns false. */ bool TryConnectionPossibilityForLocalPrimaryNode(void) { bool nodeIsInMetadata = false; WorkerNode *localNode = PrimaryNodeForGroup(GetLocalGroupId(), &nodeIsInMetadata); if (localNode == NULL) { /* * If the local node is not a primary node, we should not try to * reserve a connection as there cannot be any shards. */ return false; } bool waitForConnection = false; return EnsureConnectionPossibilityForNode(localNode, waitForConnection); } /* * EnsureConnectionPossibilityForNodeList reserves a shared connection * counter per node in the nodeList unless: * - Reservation is possible/allowed (see IsReservationPossible()) * - there is at least one connection to the node so that we are guaranteed * to get a connection * - An earlier call already reserved a connection (e.g., we allow only a * single reservation per backend) */ static void EnsureConnectionPossibilityForNodeList(List *nodeList) { /* * We sort the workerList because adaptive connection management * (e.g., OPTIONAL_CONNECTION) requires any concurrent executions * to wait for the connections in the same order to prevent any * starvation. If we don't sort, we might end up with: * Execution 1: Get connection for worker 1, wait for worker 2 * Execution 2: Get connection for worker 2, wait for worker 1 * * and, none could proceed. Instead, we enforce every execution establish * the required connections to workers in the same order. */ nodeList = SortList(nodeList, CompareWorkerNodes); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, nodeList) { bool waitForConnection = true; EnsureConnectionPossibilityForNode(workerNode, waitForConnection); } } /* * EnsureConnectionPossibilityForNode reserves a shared connection * counter per node in the nodeList unless: * - Reservation is not possible/allowed (see IsReservationPossible()) * - there is at least one connection to the node so that we are guranteed * to get a connection * - An earlier call already reserved a connection (e.g., we allow only a * single reservation per backend) * - waitForConnection is false. When this is false, the function still tries * to ensure connection possibility. If it fails (e.g., we * reached max_shared_pool_size), it doesn't wait to get the connection. Instead, * return false. */ static bool EnsureConnectionPossibilityForNode(WorkerNode *workerNode, bool waitForConnection) { if (!IsReservationPossible()) { return false; } char *databaseName = get_database_name(MyDatabaseId); Oid userId = GetUserId(); char *userName = GetUserNameFromId(userId, false); if (ConnectionAvailableToNode(workerNode->workerName, workerNode->workerPort, userName, databaseName) != NULL) { /* * The same user has already an active connection for the node. It * means that the execution can use the same connection, so reservation * is not necessary. */ return true; } /* * We are trying to be defensive here by ensuring that the required hash * table entry can be allocated. The main goal is that we don't want to be * in a situation where shared connection counter is incremented but not * the local reserved counter due to out-of-memory. * * Note that shared connection stats operate on the shared memory, and we * pre-allocate all the necessary memory. In other words, it would never * throw out of memory error. */ bool found = false; ReservedConnectionHashEntry *hashEntry = AllocateOrGetReservedConnectionEntry(workerNode->workerName, workerNode->workerPort, userId, MyDatabaseId, &found); if (found) { /* * We have already reserved a connection for this user and database * on the worker. We only allow a single reservation per * transaction block. The reason is that the earlier command (either in * a transaction block or a function call triggered by a single command) * was able to reserve or establish a connection. That connection is * guranteed to be available for us. */ return true; } if (waitForConnection) { /* * Increment the shared counter, we may need to wait if there are * no space left. */ WaitLoopForSharedConnection(workerNode->workerName, workerNode->workerPort); } else { bool incremented = TryToIncrementSharedConnectionCounter(workerNode->workerName, workerNode->workerPort); if (!incremented) { /* * We could not reserve a connection. First, remove the entry from the * hash. The reason is that we allow single reservation per transaction * block and leaving the entry in the hash would be qualified as there is a * reserved connection to the node. */ bool foundForRemove = false; hash_search(SessionLocalReservedConnections, hashEntry, HASH_REMOVE, &foundForRemove); Assert(foundForRemove); return false; } } /* locally mark that we have one connection reserved */ hashEntry->usedReservation = false; return true; } /* * IsReservationPossible returns true if the state of the current * session is eligible for shared connection reservation. */ bool IsReservationPossible(void) { if (GetMaxSharedPoolSize() == DISABLE_CONNECTION_THROTTLING) { /* connection throttling disabled */ return false; } if (UseConnectionPerPlacement()) { /* * For this case, we are not enforcing adaptive * connection management anyway. */ return false; } if (SessionLocalReservedConnections == NULL) { /* * This is unexpected as SessionLocalReservedConnections hash table is * created at startup. Still, let's be defensive. */ return false; } return true; } /* * AllocateOrGetReservedConnectionEntry allocates the required entry in the hash * map by HASH_ENTER. The function throws an error if it cannot allocate * the entry. */ static ReservedConnectionHashEntry * AllocateOrGetReservedConnectionEntry(char *hostName, int nodePort, Oid userId, Oid databaseOid, bool *found) { ReservedConnectionHashKey key; *found = false; strlcpy(key.hostname, hostName, MAX_NODE_LENGTH); key.userId = userId; key.port = nodePort; key.databaseOid = databaseOid; /* * Entering a new entry with HASH_ENTER flag is enough as it would * throw out-of-memory error as it internally does palloc. */ ReservedConnectionHashEntry *entry = (ReservedConnectionHashEntry *) hash_search(SessionLocalReservedConnections, &key, HASH_ENTER, found); if (!*found) { /* * Until we reserve connection in the shared memory, we treat * as if have used the reservation. */ entry->usedReservation = true; } return entry; } /* * LocalConnectionReserveHashHash is a utilty function to calculate hash of * ReservedConnectionHashKey. */ static uint32 LocalConnectionReserveHashHash(const void *key, Size keysize) { ReservedConnectionHashKey *entry = (ReservedConnectionHashKey *) key; uint32 hash = string_hash(entry->hostname, MAX_NODE_LENGTH); hash = hash_combine(hash, hash_uint32(entry->userId)); hash = hash_combine(hash, hash_uint32(entry->port)); hash = hash_combine(hash, hash_uint32(entry->databaseOid)); return hash; } /* * LocalConnectionReserveHashCompare is a utilty function to compare * ReservedConnectionHashKeys. */ static int LocalConnectionReserveHashCompare(const void *a, const void *b, Size keysize) { ReservedConnectionHashKey *ca = (ReservedConnectionHashKey *) a; ReservedConnectionHashKey *cb = (ReservedConnectionHashKey *) b; if (ca->port != cb->port || ca->databaseOid != cb->databaseOid || ca->userId != cb->userId || strncmp(ca->hostname, cb->hostname, MAX_NODE_LENGTH) != 0) { return 1; } else { return 0; } } ================================================ FILE: src/backend/distributed/connection/placement_connection.c ================================================ /*------------------------------------------------------------------------- * * placement_connection.c * Per placement connection handling. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/hash.h" #include "common/hashfn.h" #include "utils/hsearch.h" #include "utils/memutils.h" #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" /* * A connection reference is used to register that a connection has been used * to read or modify either a) a shard placement as a particular user b) a * group of colocated placements (which depend on whether the reference is * from ConnectionPlacementHashEntry or ColocatedPlacementHashEntry). */ typedef struct ConnectionReference { /* * The user used to read/modify the placement. We cannot reuse connections * that were performed using a different role, since it would not have the * right permissions. */ const char *userName; /* the connection */ MultiConnection *connection; /* * Information about what the connection is used for. There can only be * one connection executing DDL/DML for a placement to avoid deadlock * issues/read-your-own-writes violations. The difference between DDL/DML * currently is only used to emit more precise error messages. */ bool hadDML; bool hadDDL; /* colocation group of the placement, if any */ uint32 colocationGroupId; uint32 representativeValue; /* placementId of the placement, used only for append distributed tables */ uint64 placementId; /* membership in MultiConnection->referencedPlacements */ dlist_node connectionNode; } ConnectionReference; struct ColocatedPlacementsHashEntry; /* * Hash table mapping placements to a list of connections. * * This stores a list of connections for each placement, because multiple * connections to the same placement may exist at the same time. E.g. an * adaptive executor query may reference the same placement in several * sub-tasks. * * We keep track about a connection having executed DML or DDL, since we can * only ever allow a single transaction to do either to prevent deadlocks and * consistency violations (e.g. read-your-own-writes). */ /* hash key */ typedef struct ConnectionPlacementHashKey { uint64 placementId; } ConnectionPlacementHashKey; /* hash entry */ typedef struct ConnectionPlacementHashEntry { ConnectionPlacementHashKey key; /* did any remote transactions fail? */ bool failed; /* primary connection used to access the placement */ ConnectionReference *primaryConnection; /* are any other connections reading from the placements? */ bool hasSecondaryConnections; /* entry for the set of co-located placements */ struct ColocatedPlacementsHashEntry *colocatedEntry; /* membership in ConnectionShardHashEntry->placementConnections */ dlist_node shardNode; } ConnectionPlacementHashEntry; /* hash table */ static HTAB *ConnectionPlacementHash; /* * A hash-table mapping colocated placements to connections. Colocated * placements being the set of placements on a single node that represent the * same value range. This is needed because connections for colocated * placements (i.e. the corresponding placements for different colocated * distributed tables) need to share connections. Otherwise things like * foreign keys can very easily lead to unprincipled deadlocks. This means * that there can only be one DML/DDL connection for a set of colocated * placements. * * A set of colocated placements is identified, besides node identifying * information, by the associated colocation group id and the placement's * 'representativeValue' which currently is the lower boundary of it's * hash-range. * * Note that this hash-table only contains entries for hash-partitioned * tables, because others so far don't support colocation. */ /* hash key */ typedef struct ColocatedPlacementsHashKey { /* to identify host - database can't differ */ uint32 nodeId; /* colocation group, or invalid */ uint32 colocationGroupId; /* to represent the value range */ uint32 representativeValue; } ColocatedPlacementsHashKey; /* hash entry */ typedef struct ColocatedPlacementsHashEntry { ColocatedPlacementsHashKey key; /* primary connection used to access the co-located placements */ ConnectionReference *primaryConnection; /* are any other connections reading from the placements? */ bool hasSecondaryConnections; } ColocatedPlacementsHashEntry; static HTAB *ColocatedPlacementsHash; /* * Hash table mapping shard ids to placements. * * This is used to track whether placements of a shard have to be marked * invalid after a failure, or whether a coordinated transaction has to be * aborted, to avoid all placements of a shard to be marked invalid. */ /* hash key */ typedef struct ConnectionShardHashKey { uint64 shardId; } ConnectionShardHashKey; /* hash entry */ typedef struct ConnectionShardHashEntry { ConnectionShardHashKey key; dlist_head placementConnections; } ConnectionShardHashEntry; /* hash table */ static HTAB *ConnectionShardHash; static MultiConnection * FindPlacementListConnection(int flags, List *placementAccessList, const char *userName); static ConnectionPlacementHashEntry * FindOrCreatePlacementEntry(ShardPlacement * placement); static bool CanUseExistingConnection(uint32 flags, const char *userName, ConnectionReference *placementConnection); static bool ConnectionAccessedDifferentPlacement(MultiConnection *connection, ShardPlacement *placement); static void AssociatePlacementWithShard(ConnectionPlacementHashEntry *placementEntry, ShardPlacement *placement); static bool HasModificationFailedForShard(ConnectionShardHashEntry *shardEntry); static uint32 ColocatedPlacementsHashHash(const void *key, Size keysize); static int ColocatedPlacementsHashCompare(const void *a, const void *b, Size keysize); /* * GetPlacementConnection establishes a connection for a placement. * * See StartPlacementConnection for details. */ MultiConnection * GetPlacementConnection(uint32 flags, ShardPlacement *placement, const char *userName) { MultiConnection *connection = StartPlacementConnection(flags, placement, userName); if (connection == NULL) { /* connection can only be NULL for optional connections */ Assert((flags & OPTIONAL_CONNECTION)); return NULL; } FinishConnectionEstablishment(connection); return connection; } /* * StartPlacementConnection initiates a connection to a remote node, * associated with the placement and transaction. * * The connection is established for the current database. If userName is NULL * the current user is used, otherwise the provided one. * * See StartNodeUserDatabaseConnection for details. * * Flags have the corresponding meaning from StartNodeUserDatabaseConnection, * except that two additional flags have an effect: * - FOR_DML - signal that connection is going to be used for DML (modifications) * - FOR_DDL - signal that connection is going to be used for DDL * * Only one connection associated with the placement may have FOR_DML or * FOR_DDL set. For hash-partitioned tables only one connection for a set of * colocated placements may have FOR_DML/DDL set. This restriction prevents * deadlocks and wrong results due to in-progress transactions. */ MultiConnection * StartPlacementConnection(uint32 flags, ShardPlacement *placement, const char *userName) { ShardPlacementAccess *placementAccess = (ShardPlacementAccess *) palloc0(sizeof(ShardPlacementAccess)); placementAccess->placement = placement; if (flags & FOR_DDL) { placementAccess->accessType = PLACEMENT_ACCESS_DDL; } else if (flags & FOR_DML) { placementAccess->accessType = PLACEMENT_ACCESS_DML; } else { placementAccess->accessType = PLACEMENT_ACCESS_SELECT; } return StartPlacementListConnection(flags, list_make1(placementAccess), userName); } /* * StartPlacementListConnection returns a connection to a remote node suitable for * a placement accesses (SELECT, DML, DDL) or throws an error if no suitable * connection can be established if would cause a self-deadlock or consistency * violation. */ MultiConnection * StartPlacementListConnection(uint32 flags, List *placementAccessList, const char *userName) { char *freeUserName = NULL; if (userName == NULL) { userName = freeUserName = CurrentUserName(); } MultiConnection *chosenConnection = FindPlacementListConnection(flags, placementAccessList, userName); if (chosenConnection == NULL) { /* use the first placement from the list to extract nodename and nodeport */ ShardPlacementAccess *placementAccess = (ShardPlacementAccess *) linitial(placementAccessList); ShardPlacement *placement = placementAccess->placement; char *nodeName = placement->nodeName; int nodePort = placement->nodePort; /* * No suitable connection in the placement->connection mapping, get one from * the node->connection pool. */ chosenConnection = StartNodeUserDatabaseConnection(flags, nodeName, nodePort, userName, NULL); if (chosenConnection == NULL) { /* connection can only be NULL for optional connections */ Assert((flags & OPTIONAL_CONNECTION)); return NULL; } if ((flags & REQUIRE_CLEAN_CONNECTION) && ConnectionAccessedDifferentPlacement(chosenConnection, placement)) { /* * Cached connection accessed a non-co-located placement in the same * table or co-location group, while the caller asked for a clean * connection. Open a new connection instead. * * We use this for situations in which we want to use a different * connection for every placement, such as COPY. If we blindly returned * a cached connection that already modified a different, non-co-located * placement B in the same table or in a table with the same co-location * ID as the current placement, then we'd no longer able to write to * placement B later in the COPY. */ chosenConnection = StartNodeUserDatabaseConnection(flags | FORCE_NEW_CONNECTION, nodeName, nodePort, userName, NULL); if (chosenConnection == NULL) { /* connection can only be NULL for optional connections */ Assert((flags & OPTIONAL_CONNECTION)); return NULL; } Assert(!ConnectionAccessedDifferentPlacement(chosenConnection, placement)); } } /* remember which connection we're going to use to access the placements */ AssignPlacementListToConnection(placementAccessList, chosenConnection); if (freeUserName) { pfree(freeUserName); } return chosenConnection; } /* * AssignPlacementListToConnection assigns a set of shard placement accesses to a * given connection, meaning that connection must be used for all (conflicting) * accesses of the same shard placements to make sure reads see writes and to * make sure we don't take conflicting locks. */ void AssignPlacementListToConnection(List *placementAccessList, MultiConnection *connection) { const char *userName = connection->user; ShardPlacementAccess *placementAccess = NULL; foreach_declared_ptr(placementAccess, placementAccessList) { ShardPlacement *placement = placementAccess->placement; ShardPlacementAccessType accessType = placementAccess->accessType; if (placement->shardId == INVALID_SHARD_ID) { /* * When a SELECT prunes down to 0 shard, we use a dummy placement * which is only used to route the query to a worker node, but * the SELECT doesn't actually access any shard placement. * * FIXME: this can be removed if we evaluate empty SELECTs locally. */ continue; } ConnectionPlacementHashEntry *placementEntry = FindOrCreatePlacementEntry( placement); ConnectionReference *placementConnection = placementEntry->primaryConnection; if (placementConnection->connection == connection) { /* using the connection that was already assigned to the placement */ } else if (placementConnection->connection == NULL) { /* placement does not have a connection assigned yet */ placementConnection->connection = connection; placementConnection->hadDDL = false; placementConnection->hadDML = false; placementConnection->userName = MemoryContextStrdup(TopTransactionContext, userName); placementConnection->placementId = placementAccess->placement->placementId; /* record association with connection */ dlist_push_tail(&connection->referencedPlacements, &placementConnection->connectionNode); } else { /* using a different connection than the one assigned to the placement */ if (accessType != PLACEMENT_ACCESS_SELECT) { /* * We previously read from the placement, but now we're writing to * it (if we had written to the placement, we would have either chosen * the same connection, or errored out). Update the connection reference * to point to the connection used for writing. We don't need to remember * the existing connection since we won't be able to reuse it for * accessing the placement. However, we do register that it exists in * hasSecondaryConnections. */ placementConnection->connection = connection; placementConnection->userName = MemoryContextStrdup(TopTransactionContext, userName); Assert(!placementConnection->hadDDL); Assert(!placementConnection->hadDML); /* record association with connection */ dlist_push_tail(&connection->referencedPlacements, &placementConnection->connectionNode); } /* * There are now multiple connections that read from the placement * and DDL commands are forbidden. */ placementEntry->hasSecondaryConnections = true; if (placementEntry->colocatedEntry != NULL) { /* we also remember this for co-located placements */ placementEntry->colocatedEntry->hasSecondaryConnections = true; } } /* * Remember that we used the current connection for writes. */ if (accessType == PLACEMENT_ACCESS_DDL) { placementConnection->hadDDL = true; } if (accessType == PLACEMENT_ACCESS_DML) { placementConnection->hadDML = true; } /* record the relation access */ Oid relationId = RelationIdForShard(placement->shardId); RecordRelationAccessIfNonDistTable(relationId, accessType); } } /* * GetConnectionIfPlacementAccessedInXact returns the connection over which * the placement has been access in the transaction. If not found, returns * NULL. */ MultiConnection * GetConnectionIfPlacementAccessedInXact(int flags, List *placementAccessList, const char *userName) { char *freeUserName = NULL; if (userName == NULL) { userName = freeUserName = CurrentUserName(); } MultiConnection *connection = FindPlacementListConnection(flags, placementAccessList, userName); if (freeUserName != NULL) { pfree(freeUserName); } return connection; } /* * FindPlacementListConnection determines whether there is a connection that must * be used to perform the given placement accesses. * * If a placement was only read in this transaction, then the same connection must * be used for DDL to prevent self-deadlock. If a placement was modified in this * transaction, then the same connection must be used for all subsequent accesses * to ensure read-your-writes consistency and prevent self-deadlock. If those * conditions cannot be met, because a connection is in use or the placements in * the placement access list were modified over multiple connections, then this * function throws an error. * * The function returns the connection that needs to be used, if such a connection * exists. */ static MultiConnection * FindPlacementListConnection(int flags, List *placementAccessList, const char *userName) { bool foundModifyingConnection = false; MultiConnection *chosenConnection = NULL; /* * Go through all placement accesses to find a suitable connection. * * If none of the placements have been accessed in this transaction, connection * remains NULL. * * If one or more of the placements have been modified in this transaction, then * use the connection that performed the write. If placements have been written * over multiple connections or the connection is not available, error out. * * If placements have only been read in this transaction, then use the last * suitable connection found for a placement in the placementAccessList. */ ShardPlacementAccess *placementAccess = NULL; foreach_declared_ptr(placementAccess, placementAccessList) { ShardPlacement *placement = placementAccess->placement; ShardPlacementAccessType accessType = placementAccess->accessType; if (placement->shardId == INVALID_SHARD_ID) { /* * When a SELECT prunes down to 0 shard, we use a dummy placement. * In that case, we can fall back to the default connection. * * FIXME: this can be removed if we evaluate empty SELECTs locally. */ continue; } ConnectionPlacementHashEntry *placementEntry = FindOrCreatePlacementEntry( placement); ColocatedPlacementsHashEntry *colocatedEntry = placementEntry->colocatedEntry; ConnectionReference *placementConnection = placementEntry->primaryConnection; /* note: the Asserts below are primarily for clarifying the conditions */ if (placementConnection->connection == NULL) { /* no connection has been chosen for the placement */ } else if (accessType == PLACEMENT_ACCESS_DDL && placementEntry->hasSecondaryConnections) { /* * If a placement has been read over multiple connections (typically as * a result of a reference table join) then a DDL command on the placement * would create a self-deadlock. */ Assert(placementConnection != NULL); ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot perform DDL on placement " UINT64_FORMAT ", which has been read over multiple connections", placement->placementId))); } else if (accessType == PLACEMENT_ACCESS_DDL && colocatedEntry != NULL && colocatedEntry->hasSecondaryConnections) { /* * If a placement has been read over multiple (uncommitted) connections * then a DDL command on a co-located placement may create a self-deadlock * if there exist some relationship between the co-located placements * (e.g. foreign key, partitioning). */ Assert(placementConnection != NULL); ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot perform DDL on placement " UINT64_FORMAT " since a co-located placement has been read over multiple connections", placement->placementId))); } else if (foundModifyingConnection) { /* * We already found a connection that performed writes on of the placements * and must use it. */ if ((placementConnection->hadDDL || placementConnection->hadDML) && placementConnection->connection != chosenConnection) { /* * The current placement may have been modified over a different * connection. Neither connection is guaranteed to see all uncomitted * writes and therefore we cannot proceed. */ ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot perform query with placements that were " "modified over multiple connections"))); } } else if (accessType == PLACEMENT_ACCESS_SELECT && placementEntry->hasSecondaryConnections && !placementConnection->hadDDL && !placementConnection->hadDML) { /* * Two separate connections have already selected from this placement * and it was not modified. There is no benefit to using this connection. */ } else if (CanUseExistingConnection(flags, userName, placementConnection)) { /* * There is an existing connection for the placement and we can use it. */ Assert(placementConnection != NULL); chosenConnection = placementConnection->connection; if (placementConnection->hadDDL || placementConnection->hadDML) { /* this connection performed writes, we must use it */ foundModifyingConnection = true; } } else if (placementConnection->hadDDL || placementConnection->hadDML) { if (strcmp(placementConnection->userName, userName) != 0) { ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot perform query on placements that were " "modified in this transaction by a different " "user"))); } ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot perform query, because modifications were " "made over a connection that cannot be used at " "this time. This is most likely a Citus bug so " "please report it" ))); } } return chosenConnection; } /* * FindOrCreatePlacementEntry finds a placement entry in either the * placement->connection hash or the co-located placements->connection hash, * or adds a new entry if the placement has not yet been accessed in the * current transaction. */ static ConnectionPlacementHashEntry * FindOrCreatePlacementEntry(ShardPlacement *placement) { ConnectionPlacementHashKey connKey; bool found = false; connKey.placementId = placement->placementId; ConnectionPlacementHashEntry *placementEntry = hash_search(ConnectionPlacementHash, &connKey, HASH_ENTER, &found); if (!found) { /* no connection has been chosen for this placement */ placementEntry->failed = false; placementEntry->primaryConnection = NULL; placementEntry->hasSecondaryConnections = false; placementEntry->colocatedEntry = NULL; if (placement->partitionMethod == DISTRIBUTE_BY_HASH || placement->partitionMethod == DISTRIBUTE_BY_NONE) { ColocatedPlacementsHashKey coloKey; coloKey.nodeId = placement->nodeId; coloKey.colocationGroupId = placement->colocationGroupId; coloKey.representativeValue = placement->representativeValue; /* look for a connection assigned to co-located placements */ ColocatedPlacementsHashEntry *colocatedEntry = hash_search( ColocatedPlacementsHash, &coloKey, HASH_ENTER, &found); if (!found) { void *conRef = MemoryContextAllocZero(TopTransactionContext, sizeof(ConnectionReference)); ConnectionReference *connectionReference = (ConnectionReference *) conRef; /* * Store the co-location group information such that we can later * determine whether a connection accessed different placements * of the same co-location group. */ connectionReference->colocationGroupId = placement->colocationGroupId; connectionReference->representativeValue = placement->representativeValue; /* * Create a connection reference that can be used for the entire * set of co-located placements. */ colocatedEntry->primaryConnection = connectionReference; colocatedEntry->hasSecondaryConnections = false; } /* * Assign the connection reference for the set of co-located placements * to the current placement. */ placementEntry->primaryConnection = colocatedEntry->primaryConnection; placementEntry->colocatedEntry = colocatedEntry; } else { void *conRef = MemoryContextAllocZero(TopTransactionContext, sizeof(ConnectionReference)); placementEntry->primaryConnection = (ConnectionReference *) conRef; } } /* record association with shard, for invalidation */ AssociatePlacementWithShard(placementEntry, placement); return placementEntry; } /* * CanUseExistingConnection is a helper function for CheckExistingConnections() * that checks whether an existing connection can be reused. */ static bool CanUseExistingConnection(uint32 flags, const char *userName, ConnectionReference *connectionReference) { MultiConnection *connection = connectionReference->connection; if (!connection) { /* if already closed connection obviously not usable */ return false; } else if (connection->claimedExclusively) { /* already used */ return false; } else if (flags & FORCE_NEW_CONNECTION) { /* no connection reuse desired */ return false; } else if (strcmp(connectionReference->userName, userName) != 0) { /* connection for different user, check for conflict */ return false; } else { return true; } } /* * ConnectionAccessedDifferentPlacement returns true if the connection accessed another * placement in the same colocation group with a different representative value, * meaning it's not strictly colocated. */ static bool ConnectionAccessedDifferentPlacement(MultiConnection *connection, ShardPlacement *placement) { dlist_iter placementIter; dlist_foreach(placementIter, &connection->referencedPlacements) { ConnectionReference *connectionReference = dlist_container(ConnectionReference, connectionNode, placementIter.cur); /* handle append and range distributed tables */ if (placement->partitionMethod != DISTRIBUTE_BY_HASH && placement->placementId != connectionReference->placementId) { return true; } /* handle hash distributed tables */ if (placement->colocationGroupId != INVALID_COLOCATION_ID && placement->colocationGroupId == connectionReference->colocationGroupId && placement->representativeValue != connectionReference->representativeValue) { /* non-co-located placements from the same co-location group */ return true; } } return false; } /* * ConnectionModifiedPlacement returns true if any DML or DDL is executed over * the connection on any placement/table. */ bool ConnectionModifiedPlacement(MultiConnection *connection) { dlist_iter placementIter; if (connection->remoteTransaction.transactionState == REMOTE_TRANS_NOT_STARTED) { /* * When StartPlacementListConnection() is called, we set the * hadDDL/hadDML even before the actual command is sent to * remote nodes. And, if this function is called at that * point, we should not assume that the connection has already * done any modifications. */ return false; } if (dlist_is_empty(&connection->referencedPlacements)) { /* * When referencesPlacements are empty, it means that we come here * from an API that uses a node connection (e.g., not placement connection), * which doesn't set placements. * In that case, the command sent could be either write or read, so we assume * it is write to be on the safe side. */ return true; } dlist_foreach(placementIter, &connection->referencedPlacements) { ConnectionReference *connectionReference = dlist_container(ConnectionReference, connectionNode, placementIter.cur); if (connectionReference->hadDDL || connectionReference->hadDML) { return true; } } return false; } /* * AssociatePlacementWithShard records shard->placement relation in * ConnectionShardHash. * * That association is later used, in CheckForFailedPlacements, to invalidate * shard placements if necessary. */ static void AssociatePlacementWithShard(ConnectionPlacementHashEntry *placementEntry, ShardPlacement *placement) { ConnectionShardHashKey shardKey; bool found = false; dlist_iter placementIter; shardKey.shardId = placement->shardId; ConnectionShardHashEntry *shardEntry = hash_search(ConnectionShardHash, &shardKey, HASH_ENTER, &found); if (!found) { dlist_init(&shardEntry->placementConnections); } /* * Check if placement is already associated with shard (happens if there's * multiple connections for a placement). There'll usually only be few * placement per shard, so the price of iterating isn't large. */ dlist_foreach(placementIter, &shardEntry->placementConnections) { ConnectionPlacementHashEntry *currPlacementEntry = dlist_container(ConnectionPlacementHashEntry, shardNode, placementIter.cur); if (currPlacementEntry->key.placementId == placement->placementId) { return; } } /* otherwise add */ dlist_push_tail(&shardEntry->placementConnections, &placementEntry->shardNode); } /* * CloseShardPlacementAssociation handles a connection being closed before * transaction end. * * This should only be called by connection_management.c. */ void CloseShardPlacementAssociation(struct MultiConnection *connection) { dlist_iter placementIter; /* set connection to NULL for all references to the connection */ dlist_foreach(placementIter, &connection->referencedPlacements) { ConnectionReference *reference = dlist_container(ConnectionReference, connectionNode, placementIter.cur); reference->connection = NULL; /* * Note that we don't reset ConnectionPlacementHashEntry's * primaryConnection here, that'd be more complicated than it seems * worth. That means we'll error out spuriously if a DML/DDL * executing connection is closed earlier in a transaction. */ } } /* * ResetShardPlacementAssociation resets the association of connections to * shard placements at the end of a transaction. * * This should only be called by connection_management.c. */ void ResetShardPlacementAssociation(struct MultiConnection *connection) { dlist_init(&connection->referencedPlacements); } /* * ResetPlacementConnectionManagement() dissociates connections from * placements and shards. This will be called at the end of XACT_EVENT_COMMIT * and XACT_EVENT_ABORT. */ void ResetPlacementConnectionManagement(void) { /* Simply delete all entries */ hash_delete_all(ConnectionPlacementHash); hash_delete_all(ConnectionShardHash); hash_delete_all(ColocatedPlacementsHash); /* * NB: memory for ConnectionReference structs and subordinate data is * deleted by virtue of being allocated in TopTransactionContext. */ } /* * ErrorIfPostCommitFailedShardPlacements throws an error if any of the placements * that modified the database and involved in the transaction has failed. * * Note that Citus already fails queries/commands in case of any failures during query * processing. However, there are certain failures that can only be detected on the * COMMIT time. And, this check mainly ensures to catch errors that happens on the * COMMIT time on the placements. * * The most common example for this case is the deferred errors that are thrown by * triggers or constraints at the COMMIT time. */ void ErrorIfPostCommitFailedShardPlacements(void) { HASH_SEQ_STATUS status; ConnectionShardHashEntry *shardEntry = NULL; hash_seq_init(&status, ConnectionShardHash); while ((shardEntry = (ConnectionShardHashEntry *) hash_seq_search(&status)) != 0) { if (HasModificationFailedForShard(shardEntry)) { ereport(ERROR, (errmsg("could not commit transaction for shard " INT64_FORMAT " on at least one active node", shardEntry->key.shardId))); } } } /* * HasModificationFailedForShard is a helper function for * ErrorIfPostCommitFailedShardPlacements that performs the per-shard work. * * The function returns true if any placement of the input shard is modified * and any failures has happened (either connection failures or transaction * failures). */ static bool HasModificationFailedForShard(ConnectionShardHashEntry *shardEntry) { dlist_iter placementIter; dlist_foreach(placementIter, &shardEntry->placementConnections) { ConnectionPlacementHashEntry *placementEntry = dlist_container(ConnectionPlacementHashEntry, shardNode, placementIter.cur); ConnectionReference *primaryConnection = placementEntry->primaryConnection; /* we only consider shards that are modified */ if (primaryConnection == NULL || !(primaryConnection->hadDDL || primaryConnection->hadDML)) { continue; } MultiConnection *connection = primaryConnection->connection; if (!connection || connection->remoteTransaction.transactionFailed) { return true; } } return false; } /* * InitPlacementConnectionManagement performs initialization of the * infrastructure in this file at server start. */ void InitPlacementConnectionManagement(void) { HASHCTL info; /* create (placementId) -> [ConnectionReference] hash */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(ConnectionPlacementHashKey); info.entrysize = sizeof(ConnectionPlacementHashEntry); info.hash = tag_hash; info.hcxt = ConnectionContext; uint32 hashFlags = (HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); ConnectionPlacementHash = hash_create("citus connection cache (placementid)", 64, &info, hashFlags); /* create (colocated placement identity) -> [ConnectionReference] hash */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(ColocatedPlacementsHashKey); info.entrysize = sizeof(ColocatedPlacementsHashEntry); info.hash = ColocatedPlacementsHashHash; info.match = ColocatedPlacementsHashCompare; info.hcxt = ConnectionContext; hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE); ColocatedPlacementsHash = hash_create("citus connection cache (colocated placements)", 64, &info, hashFlags); /* create (shardId) -> [ConnectionShardHashEntry] hash */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(ConnectionShardHashKey); info.entrysize = sizeof(ConnectionShardHashEntry); info.hash = tag_hash; info.hcxt = ConnectionContext; hashFlags = (HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); ConnectionShardHash = hash_create("citus connection cache (shardid)", 64, &info, hashFlags); } /* * UseConnectionPerPlacement returns whether we should use as separate connection * per placement even if another connection is idle. We mostly use this in testing * scenarios. */ bool UseConnectionPerPlacement(void) { return ForceMaxQueryParallelization && MultiShardConnectionType != SEQUENTIAL_CONNECTION; } static uint32 ColocatedPlacementsHashHash(const void *key, Size keysize) { ColocatedPlacementsHashKey *entry = (ColocatedPlacementsHashKey *) key; uint32 hash = hash_uint32(entry->nodeId); hash = hash_combine(hash, hash_uint32(entry->colocationGroupId)); hash = hash_combine(hash, hash_uint32(entry->representativeValue)); return hash; } static int ColocatedPlacementsHashCompare(const void *a, const void *b, Size keysize) { ColocatedPlacementsHashKey *ca = (ColocatedPlacementsHashKey *) a; ColocatedPlacementsHashKey *cb = (ColocatedPlacementsHashKey *) b; if (ca->nodeId != cb->nodeId || ca->colocationGroupId != cb->colocationGroupId || ca->representativeValue != cb->representativeValue) { return 1; } else { return 0; } } ================================================ FILE: src/backend/distributed/connection/remote_commands.c ================================================ /*------------------------------------------------------------------------- * * remote_commands.c * Helpers to make it easier to execute command on remote nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "catalog/pg_collation.h" #include "lib/stringinfo.h" #include "storage/latch.h" #include "utils/builtins.h" #include "utils/fmgrprotos.h" #include "utils/palloc.h" #include "distributed/cancel_utils.h" #include "distributed/connection_management.h" #include "distributed/errormessage.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/remote_commands.h" /* * Setting that controls how many bytes of COPY data libpq is allowed to buffer * internally before we force a flush. */ int RemoteCopyFlushThreshold = 8 * 1024 * 1024; /* GUC, determining whether statements sent to remote nodes are logged */ bool LogRemoteCommands = false; char *GrepRemoteCommands = ""; static bool ClearResultsInternal(MultiConnection *connection, bool raiseErrors, bool discardWarnings); static bool FinishConnectionIO(MultiConnection *connection, bool raiseInterrupts); static WaitEventSet * BuildWaitEventSet(MultiConnection **allConnections, int totalConnectionCount, int pendingConnectionsStartIndex); /* simple helpers */ /* * IsResponseOK checks whether the result is a successful one. */ bool IsResponseOK(PGresult *result) { ExecStatusType resultStatus = PQresultStatus(result); if (resultStatus == PGRES_SINGLE_TUPLE || resultStatus == PGRES_TUPLES_OK || resultStatus == PGRES_COMMAND_OK) { return true; } return false; } /* * ForgetResults clears a connection from pending activity. * * Note that this might require network IO. If that's not acceptable, use * ClearResultsIfReady(). * * ClearResults is variant of this function which can also raise errors. */ void ForgetResults(MultiConnection *connection) { ClearResults(connection, false); } /* * ClearResults clears a connection from pending activity, * returns true if all pending commands return success. It raises * error if raiseErrors flag is set, any command fails and transaction * is marked critical. * * Note that this might require network IO. If that's not acceptable, use * ClearResultsIfReady(). */ bool ClearResults(MultiConnection *connection, bool raiseErrors) { return ClearResultsInternal(connection, raiseErrors, false); } /* * ClearResultsDiscardWarnings does the same thing as ClearResults, but doesn't * emit warnings. */ bool ClearResultsDiscardWarnings(MultiConnection *connection, bool raiseErrors) { return ClearResultsInternal(connection, raiseErrors, true); } /* * ClearResultsInternal is used by ClearResults and ClearResultsDiscardWarnings. */ static bool ClearResultsInternal(MultiConnection *connection, bool raiseErrors, bool discardWarnings) { bool success = true; while (true) { PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (result == NULL) { break; } /* * End any pending copy operation. Transaction will be marked * as failed by the following part. */ if (PQresultStatus(result) == PGRES_COPY_IN) { PQputCopyEnd(connection->pgConn, NULL); } if (!IsResponseOK(result)) { if (!discardWarnings) { ReportResultError(connection, result, WARNING); } MarkRemoteTransactionFailed(connection, raiseErrors); success = false; /* an error happened, there is nothing we can do more */ if (PQresultStatus(result) == PGRES_FATAL_ERROR) { PQclear(result); break; } } PQclear(result); } return success; } /* * ClearResultsIfReady clears a connection from pending activity if doing * so does not require network IO. Returns true if successful, false * otherwise. */ bool ClearResultsIfReady(MultiConnection *connection) { PGconn *pgConn = connection->pgConn; if (PQstatus(pgConn) != CONNECTION_OK) { return false; } Assert(PQisnonblocking(pgConn)); while (true) { /* * If busy, there might still be results already received and buffered * by the OS. As connection is in non-blocking mode, we can check for * that without blocking. */ if (PQisBusy(pgConn)) { if (PQflush(pgConn) == -1) { /* write failed */ return false; } if (PQconsumeInput(pgConn) == 0) { /* some low-level failure */ return false; } } /* clearing would require blocking IO, return */ if (PQisBusy(pgConn)) { return false; } PGresult *result = PQgetResult(pgConn); if (result == NULL) { /* no more results available */ return true; } ExecStatusType resultStatus = PQresultStatus(result); /* only care about the status, can clear now */ PQclear(result); if (resultStatus == PGRES_COPY_IN || resultStatus == PGRES_COPY_OUT) { /* in copy, can't reliably recover without blocking */ return false; } if (!(resultStatus == PGRES_SINGLE_TUPLE || resultStatus == PGRES_TUPLES_OK || resultStatus == PGRES_COMMAND_OK)) { /* an error occurred just when we were aborting */ return false; } /* check if there are more results to consume */ } pg_unreachable(); } /* report errors & warnings */ /* * Report libpq failure that's not associated with a result. */ void ReportConnectionError(MultiConnection *connection, int elevel) { char *userName = connection->user; char *nodeName = connection->hostname; int nodePort = connection->port; PGconn *pgConn = connection->pgConn; char *messageDetail = NULL; if (pgConn != NULL) { messageDetail = pchomp(PQerrorMessage(pgConn)); if (messageDetail == NULL || messageDetail[0] == '\0') { /* give a similar messages to Postgres */ messageDetail = "connection not open"; } } if (messageDetail) { ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection to the remote node %s@%s:%d failed with the " "following error: %s", userName, nodeName, nodePort, messageDetail))); } else { ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection to the remote node %s@%s:%d failed", userName, nodeName, nodePort))); } } /* * ReportResultError reports libpq failure associated with a result. */ void ReportResultError(MultiConnection *connection, PGresult *result, int elevel) { /* we release PQresult when throwing an error because the caller can't */ PG_TRY(); { char *sqlStateString = PQresultErrorField(result, PG_DIAG_SQLSTATE); char *messagePrimary = PQresultErrorField(result, PG_DIAG_MESSAGE_PRIMARY); char *messageDetail = PQresultErrorField(result, PG_DIAG_MESSAGE_DETAIL); char *messageHint = PQresultErrorField(result, PG_DIAG_MESSAGE_HINT); char *messageContext = PQresultErrorField(result, PG_DIAG_CONTEXT); char *nodeName = connection->hostname; int nodePort = connection->port; int sqlState = ERRCODE_INTERNAL_ERROR; if (sqlStateString != NULL) { sqlState = MAKE_SQLSTATE(sqlStateString[0], sqlStateString[1], sqlStateString[2], sqlStateString[3], sqlStateString[4]); } /* * If the PGresult did not contain a message, the connection may provide a * suitable top level one. At worst, this is an empty string. */ if (messagePrimary == NULL) { messagePrimary = pchomp(PQerrorMessage(connection->pgConn)); } ereport(elevel, (errcode(sqlState), errmsg("%s", messagePrimary), messageDetail ? errdetail("%s", messageDetail) : 0, messageHint ? errhint("%s", messageHint) : 0, messageContext ? errcontext("%s", messageContext) : 0, errcontext("while executing command on %s:%d", nodeName, nodePort))); } PG_CATCH(); { PQclear(result); PG_RE_THROW(); } PG_END_TRY(); } /* *INDENT-ON* */ /* * LogRemoteCommand logs commands send to remote nodes if * citus.log_remote_commands wants us to do so. */ void LogRemoteCommand(MultiConnection *connection, const char *command) { if (!LogRemoteCommands) { return; } if (!CommandMatchesLogGrepPattern(command)) { return; } ereport(NOTICE, (errmsg("issuing %s", command), errdetail("on server %s@%s:%d connectionId: %ld", connection->user, connection->hostname, connection->port, connection->connectionId))); } /* * CommandMatchesLogGrepPattern returns true of the input command matches * the pattern specified by citus.grep_remote_commands. * * If citus.grep_remote_commands set to an empty string, all commands are * considered as a match. */ bool CommandMatchesLogGrepPattern(const char *command) { if (GrepRemoteCommands && strnlen(GrepRemoteCommands, NAMEDATALEN) > 0) { Datum boolDatum = DirectFunctionCall2Coll(textlike, DEFAULT_COLLATION_OID, CStringGetTextDatum(command), CStringGetTextDatum(GrepRemoteCommands)); return DatumGetBool(boolDatum); } return true; } /* wrappers around libpq functions, with command logging support */ /* * ExecuteCriticalRemoteCommandList calls ExecuteCriticalRemoteCommand for every * command in the commandList. */ void ExecuteCriticalRemoteCommandList(MultiConnection *connection, List *commandList) { const char *command = NULL; foreach_declared_ptr(command, commandList) { ExecuteCriticalRemoteCommand(connection, command); } } /* * ExecuteCriticalRemoteCommand executes a remote command that is critical * to the transaction. If the command fails then the transaction aborts. */ void ExecuteCriticalRemoteCommand(MultiConnection *connection, const char *command) { bool raiseInterrupts = true; int querySent = SendRemoteCommand(connection, command); if (querySent == 0) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ForgetResults(connection); } /* * ExecuteRemoteCommandInConnectionList executes a remote command, on all connections * given in the list, that is critical to the transaction. If the command fails then * the transaction aborts. */ void ExecuteRemoteCommandInConnectionList(List *nodeConnectionList, const char *command) { MultiConnection *connection = NULL; foreach_declared_ptr(connection, nodeConnectionList) { int querySent = SendRemoteCommand(connection, command); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } /* Process the result */ foreach_declared_ptr(connection, nodeConnectionList) { bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ForgetResults(connection); } } /* * ExecuteOptionalRemoteCommand executes a remote command. If the command fails a WARNING * is emitted but execution continues. * * could return 0, QUERY_SEND_FAILED, or RESPONSE_NOT_OKAY * result is only set if there was no error */ int ExecuteOptionalRemoteCommand(MultiConnection *connection, const char *command, PGresult **result) { bool raiseInterrupts = true; int querySent = SendRemoteCommand(connection, command); if (querySent == 0) { ReportConnectionError(connection, WARNING); return QUERY_SEND_FAILED; } PGresult *localResult = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(localResult)) { ReportResultError(connection, localResult, WARNING); PQclear(localResult); ForgetResults(connection); return RESPONSE_NOT_OKAY; } /* * store result if result has been set, when the user is not interested in the result * a NULL pointer could be passed and the result will be cleared. */ if (result != NULL) { *result = localResult; } else { PQclear(localResult); ForgetResults(connection); } return RESPONSE_OKAY; } /* * SendRemoteCommandParams is a PQsendQueryParams wrapper that logs remote commands, * and accepts a MultiConnection instead of a plain PGconn. It makes sure it can * send commands asynchronously without blocking (at the potential expense of * an additional memory allocation). The command string can only include a single * command since PQsendQueryParams() supports only that. */ int SendRemoteCommandParams(MultiConnection *connection, const char *command, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues, bool binaryResults) { PGconn *pgConn = connection->pgConn; LogRemoteCommand(connection, command); /* * Don't try to send command if connection is entirely gone * (PQisnonblocking() would crash). */ if (!pgConn || PQstatus(pgConn) != CONNECTION_OK) { return 0; } Assert(PQisnonblocking(pgConn)); int rc = PQsendQueryParams(pgConn, command, parameterCount, parameterTypes, parameterValues, NULL, NULL, binaryResults ? 1 : 0); return rc; } /* * SendRemoteCommand is a PQsendQuery wrapper that logs remote commands, and * accepts a MultiConnection instead of a plain PGconn. It makes sure it can * send commands asynchronously without blocking (at the potential expense of * an additional memory allocation). The command string can include multiple * commands since PQsendQuery() supports that. */ int SendRemoteCommand(MultiConnection *connection, const char *command) { PGconn *pgConn = connection->pgConn; LogRemoteCommand(connection, command); /* * Don't try to send command if connection is entirely gone * (PQisnonblocking() would crash). */ if (!pgConn || PQstatus(pgConn) != CONNECTION_OK) { return 0; } Assert(PQisnonblocking(pgConn)); int rc = PQsendQuery(pgConn, command); return rc; } /* * ExecuteRemoteCommandAndCheckResult executes the given command in the remote node and * checks if the result is equal to the expected result. If the result is equal to the * expected result, the function returns true, otherwise it returns false. */ bool ExecuteRemoteCommandAndCheckResult(MultiConnection *connection, char *command, char *expected) { if (!SendRemoteCommand(connection, command)) { /* if we cannot connect, we warn and report false */ ReportConnectionError(connection, WARNING); return false; } bool raiseInterrupts = true; PGresult *queryResult = GetRemoteCommandResult(connection, raiseInterrupts); /* if remote node throws an error, we also throw an error */ if (!IsResponseOK(queryResult)) { ReportResultError(connection, queryResult, ERROR); } StringInfo queryResultString = makeStringInfo(); /* Evaluate the queryResult and store it into the queryResultString */ bool success = EvaluateSingleQueryResult(connection, queryResult, queryResultString); bool result = false; if (success && strcmp(queryResultString->data, expected) == 0) { result = true; } PQclear(queryResult); ForgetResults(connection); return result; } /* * ReadFirstColumnAsText reads the first column of result tuples from the given * PGresult struct and returns them in a StringInfo list. */ List * ReadFirstColumnAsText(PGresult *queryResult) { List *resultRowList = NIL; const int columnIndex = 0; int64 rowCount = 0; ExecStatusType status = PQresultStatus(queryResult); if (status == PGRES_TUPLES_OK) { rowCount = PQntuples(queryResult); } for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { char *rowValue = PQgetvalue(queryResult, rowIndex, columnIndex); StringInfo rowValueString = makeStringInfo(); appendStringInfoString(rowValueString, rowValue); resultRowList = lappend(resultRowList, rowValueString); } return resultRowList; } /* * GetRemoteCommandResult is a wrapper around PQgetResult() that handles interrupts. * * If raiseInterrupts is true and an interrupt arrives, e.g. the query is * being cancelled, CHECK_FOR_INTERRUPTS() will be called, which then throws * an error. * * If raiseInterrupts is false and an interrupt arrives that'd otherwise raise * an error, GetRemoteCommandResult returns NULL, and the transaction is * marked as having failed. While that's not a perfect way to signal failure, * callers will usually treat that as an error, and it's easy to use. * * Handling of interrupts is important to allow queries being cancelled while * waiting on remote nodes. In a distributed deadlock scenario cancelling * might be the only way to resolve the deadlock. */ PGresult * GetRemoteCommandResult(MultiConnection *connection, bool raiseInterrupts) { PGconn *pgConn = connection->pgConn; /* * Short circuit tests around the more expensive parts of this * routine. This'd also trigger a return in the, unlikely, case of a * failed/nonexistant connection. */ if (!PQisBusy(pgConn)) { return PQgetResult(connection->pgConn); } if (!FinishConnectionIO(connection, raiseInterrupts)) { /* some error(s) happened while doing the I/O, signal the callers */ if (PQstatus(pgConn) == CONNECTION_BAD) { return PQmakeEmptyPGresult(pgConn, PGRES_FATAL_ERROR); } return NULL; } /* no IO should be necessary to get result */ Assert(!PQisBusy(pgConn)); PGresult *result = PQgetResult(connection->pgConn); return result; } /* * PutRemoteCopyData is a wrapper around PQputCopyData() that handles * interrupts. * * Returns false if PQputCopyData() failed, true otherwise. */ bool PutRemoteCopyData(MultiConnection *connection, const char *buffer, int nbytes) { PGconn *pgConn = connection->pgConn; bool allowInterrupts = true; if (PQstatus(pgConn) != CONNECTION_OK) { return false; } Assert(PQisnonblocking(pgConn)); int copyState = PQputCopyData(pgConn, buffer, nbytes); if (copyState <= 0) { return false; } /* * PQputCopyData may have queued up part of the data even if it managed * to send some of it successfully. We provide back pressure by waiting * until the socket is writable to prevent the internal libpq buffers * from growing excessively. * * We currently allow the internal buffer to grow to 8MB before * providing back pressure based on experimentation that showed * throughput get worse at 4MB and lower due to the number of CPU * cycles spent in networking system calls. */ connection->copyBytesWrittenSinceLastFlush += nbytes; if (connection->copyBytesWrittenSinceLastFlush > RemoteCopyFlushThreshold) { connection->copyBytesWrittenSinceLastFlush = 0; return FinishConnectionIO(connection, allowInterrupts); } return true; } /* * PutRemoteCopyEnd is a wrapper around PQputCopyEnd() that handles * interrupts. * * Returns false if PQputCopyEnd() failed, true otherwise. */ bool PutRemoteCopyEnd(MultiConnection *connection, const char *errormsg) { PGconn *pgConn = connection->pgConn; bool allowInterrupts = true; if (PQstatus(pgConn) != CONNECTION_OK) { return false; } Assert(PQisnonblocking(pgConn)); int copyState = PQputCopyEnd(pgConn, errormsg); if (copyState == -1) { return false; } /* see PutRemoteCopyData() */ connection->copyBytesWrittenSinceLastFlush = 0; return FinishConnectionIO(connection, allowInterrupts); } /* * FinishConnectionIO performs pending IO for the connection, while accepting * interrupts. * * See GetRemoteCommandResult() for documentation of interrupt handling * behaviour. * * Returns true if IO was successfully completed, false otherwise. */ static bool FinishConnectionIO(MultiConnection *connection, bool raiseInterrupts) { PGconn *pgConn = connection->pgConn; int sock = PQsocket(pgConn); Assert(pgConn); Assert(PQisnonblocking(pgConn)); if (raiseInterrupts) { CHECK_FOR_INTERRUPTS(); } /* perform the necessary IO */ while (true) { int waitFlags = WL_POSTMASTER_DEATH | WL_LATCH_SET; /* try to send all pending data */ int sendStatus = PQflush(pgConn); /* if sending failed, there's nothing more we can do */ if (sendStatus == -1) { return false; } else if (sendStatus == 1) { waitFlags |= WL_SOCKET_WRITEABLE; } /* if reading fails, there's not much we can do */ if (PQconsumeInput(pgConn) == 0) { return false; } if (PQisBusy(pgConn)) { waitFlags |= WL_SOCKET_READABLE; } if ((waitFlags & (WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE)) == 0) { /* no IO necessary anymore, we're done */ return true; } int rc = WaitLatchOrSocket(MyLatch, waitFlags, sock, 0, PG_WAIT_EXTENSION); if (rc & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } if (rc & WL_LATCH_SET) { ResetLatch(MyLatch); /* if allowed raise errors */ if (raiseInterrupts) { CHECK_FOR_INTERRUPTS(); } /* * If raising errors allowed, or called within in a section with * interrupts held, return instead, and mark the transaction as * failed. */ if (IsHoldOffCancellationReceived()) { connection->remoteTransaction.transactionFailed = true; break; } } } return false; } /* * WaitForAllConnections blocks until all connections in the list are no * longer busy, meaning the pending command has either finished or failed. */ void WaitForAllConnections(List *connectionList, bool raiseInterrupts) { int totalConnectionCount = list_length(connectionList); int pendingConnectionsStartIndex = 0; int connectionIndex = 0; MultiConnection **allConnections = palloc(totalConnectionCount * sizeof(MultiConnection *)); WaitEvent *events = palloc(totalConnectionCount * sizeof(WaitEvent)); bool *connectionReady = palloc(totalConnectionCount * sizeof(bool)); WaitEventSet *volatile waitEventSet = NULL; /* convert connection list to an array such that we can move items around */ MultiConnection *connectionItem = NULL; foreach_declared_ptr(connectionItem, connectionList) { allConnections[connectionIndex] = connectionItem; connectionReady[connectionIndex] = false; connectionIndex++; } /* make an initial pass to check for failed and idle connections */ for (connectionIndex = 0; connectionIndex < totalConnectionCount; connectionIndex++) { MultiConnection *connection = allConnections[connectionIndex]; if (PQstatus(connection->pgConn) == CONNECTION_BAD || !PQisBusy(connection->pgConn)) { /* connection is already done; keep non-ready connections at the end */ allConnections[connectionIndex] = allConnections[pendingConnectionsStartIndex]; pendingConnectionsStartIndex++; } } PG_TRY(); { bool rebuildWaitEventSet = true; while (pendingConnectionsStartIndex < totalConnectionCount) { bool cancellationReceived = false; int eventIndex = 0; long timeout = -1; int pendingConnectionCount = totalConnectionCount - pendingConnectionsStartIndex; /* rebuild the WaitEventSet whenever connections are ready */ if (rebuildWaitEventSet) { if (waitEventSet != NULL) { FreeWaitEventSet(waitEventSet); } waitEventSet = BuildWaitEventSet(allConnections, totalConnectionCount, pendingConnectionsStartIndex); rebuildWaitEventSet = false; } /* wait for I/O events */ int eventCount = WaitEventSetWait(waitEventSet, timeout, events, pendingConnectionCount, WAIT_EVENT_CLIENT_READ); /* process I/O events */ for (; eventIndex < eventCount; eventIndex++) { WaitEvent *event = &events[eventIndex]; bool connectionIsReady = false; if (event->events & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } if (event->events & WL_LATCH_SET) { ResetLatch(MyLatch); if (raiseInterrupts) { CHECK_FOR_INTERRUPTS(); } if (IsHoldOffCancellationReceived()) { /* * Break out of event loop immediately in case of cancellation. * We cannot use "return" here inside a PG_TRY() block since * then the exception stack won't be reset. */ cancellationReceived = true; break; } continue; } MultiConnection *connection = (MultiConnection *) event->user_data; if (event->events & WL_SOCKET_WRITEABLE) { int sendStatus = PQflush(connection->pgConn); if (sendStatus == -1) { /* send failed, done with this connection */ connectionIsReady = true; } else if (sendStatus == 0) { /* done writing, only wait for read events */ bool success = CitusModifyWaitEvent(waitEventSet, event->pos, WL_SOCKET_READABLE, NULL); if (!success) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection establishment for " "node %s:%d failed", connection->hostname, connection->port), errhint("Check both the local and remote " "server logs for the connection " "establishment errors."))); } } } /* * Check whether the connection is done is the socket is either readable * or writable. If it was only writable, we performed a PQflush which * might have read from the socket, meaning we may not see the socket * becoming readable again, so better to check it now. */ if (event->events & (WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE)) { int receiveStatus = PQconsumeInput(connection->pgConn); if (receiveStatus == 0) { /* receive failed, done with this connection */ connectionIsReady = true; } else if (!PQisBusy(connection->pgConn)) { /* result was received */ connectionIsReady = true; } } if (connectionIsReady) { /* * All pending connections are kept at the end of the allConnections * array and the connectionReady array matches the allConnections * array. The wait event set corresponds to the pending connections * subarray so we can get the index in the allConnections array by * taking the event index + the offset of the subarray. */ connectionIndex = event->pos + pendingConnectionsStartIndex; connectionReady[connectionIndex] = true; /* * When a connection is ready, we should build a new wait event * set that excludes this connection. */ rebuildWaitEventSet = true; } } if (cancellationReceived) { break; } /* move non-ready connections to the back of the array */ for (connectionIndex = pendingConnectionsStartIndex; connectionIndex < totalConnectionCount; connectionIndex++) { if (connectionReady[connectionIndex]) { /* * Replace the ready connection with a connection from * the start of the pending connections subarray. This * may be the connection itself, in which case this is * a noop. */ allConnections[connectionIndex] = allConnections[pendingConnectionsStartIndex]; /* offset of the pending connections subarray is now 1 higher */ pendingConnectionsStartIndex++; /* * We've moved a pending connection into this position, * so we must reset the ready flag. Otherwise, we'd * falsely interpret it as ready in the next round. */ connectionReady[connectionIndex] = false; } } } if (waitEventSet != NULL) { FreeWaitEventSet(waitEventSet); waitEventSet = NULL; } pfree(allConnections); pfree(events); pfree(connectionReady); } PG_CATCH(); { /* make sure the epoll file descriptor is always closed */ if (waitEventSet != NULL) { FreeWaitEventSet(waitEventSet); waitEventSet = NULL; } pfree(allConnections); pfree(events); pfree(connectionReady); PG_RE_THROW(); } PG_END_TRY(); } /* * BuildWaitEventSet creates a WaitEventSet for the given array of connections * which can be used to wait for any of the sockets to become read-ready or * write-ready. */ static WaitEventSet * BuildWaitEventSet(MultiConnection **allConnections, int totalConnectionCount, int pendingConnectionsStartIndex) { int pendingConnectionCount = totalConnectionCount - pendingConnectionsStartIndex; /* * subtract 3 to make room for WL_POSTMASTER_DEATH, WL_LATCH_SET, and * pgwin32_signal_event. */ if (pendingConnectionCount > FD_SETSIZE - 3) { pendingConnectionCount = FD_SETSIZE - 3; } /* allocate pending connections + 2 for the signal latch and postmaster death */ /* (CreateWaitEventSet makes room for pgwin32_signal_event automatically) */ WaitEventSet *waitEventSet = CreateWaitEventSet(WaitEventSetTracker_compat, pendingConnectionCount + 2); for (int connectionIndex = 0; connectionIndex < pendingConnectionCount; connectionIndex++) { MultiConnection *connection = allConnections[pendingConnectionsStartIndex + connectionIndex]; int sock = PQsocket(connection->pgConn); /* * Always start by polling for both readability (server sent bytes) * and writeability (server is ready to receive bytes). */ int eventMask = WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE; int waitEventSetIndex = CitusAddWaitEventSetToSet(waitEventSet, eventMask, sock, NULL, (void *) connection); if (waitEventSetIndex == WAIT_EVENT_SET_INDEX_FAILED) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("connection establishment for node %s:%d failed", connection->hostname, connection->port), errhint("Check both the local and remote server logs for the " "connection establishment errors."))); } } /* * Put the wait events for the signal latch and postmaster death at the end such that * event index + pendingConnectionsStartIndex = the connection index in the array. */ AddWaitEventToSet(waitEventSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); AddWaitEventToSet(waitEventSet, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); return waitEventSet; } /* * SendCancelationRequest sends a cancelation request on the given connection. * Return value indicates whether the cancelation request was sent successfully. */ bool SendCancelationRequest(MultiConnection *connection) { char errorBuffer[ERROR_BUFFER_SIZE] = { 0 }; PGcancel *cancelObject = PQgetCancel(connection->pgConn); if (cancelObject == NULL) { /* this can happen if connection is invalid */ return false; } bool cancelSent = PQcancel(cancelObject, errorBuffer, sizeof(errorBuffer)); if (!cancelSent) { ereport(WARNING, (errmsg("could not issue cancel request"), errdetail("Client error: %s", errorBuffer))); } PQfreeCancel(cancelObject); return cancelSent; } /* * EvaluateSingleQueryResult gets the query result from connection and returns * true if the query is executed successfully, false otherwise. A query result * or an error message is returned in queryResultString. The function requires * that the query returns a single column/single row result. It returns an * error otherwise. */ bool EvaluateSingleQueryResult(MultiConnection *connection, PGresult *queryResult, StringInfo queryResultString) { bool success = false; ExecStatusType resultStatus = PQresultStatus(queryResult); if (resultStatus == PGRES_COMMAND_OK) { char *commandStatus = PQcmdStatus(queryResult); appendStringInfo(queryResultString, "%s", commandStatus); success = true; } else if (resultStatus == PGRES_TUPLES_OK) { int ntuples = PQntuples(queryResult); int nfields = PQnfields(queryResult); /* error if query returns more than 1 rows, or more than 1 fields */ if (nfields != 1) { appendStringInfo(queryResultString, "expected a single column in query target"); } else if (ntuples > 1) { appendStringInfo(queryResultString, "expected a single row in query result"); } else { int row = 0; int column = 0; if (!PQgetisnull(queryResult, row, column)) { char *queryResultValue = PQgetvalue(queryResult, row, column); appendStringInfo(queryResultString, "%s", queryResultValue); } success = true; } } else { StoreErrorMessage(connection, queryResultString); } return success; } /* * StoreErrorMessage gets the error message from connection and stores it * in queryResultString. It should be called only when error is present * otherwise it would return a default error message. */ void StoreErrorMessage(MultiConnection *connection, StringInfo queryResultString) { char *errorMessage = PQerrorMessage(connection->pgConn); if (errorMessage != NULL) { /* copy the error message to a writable memory */ errorMessage = pnstrdup(errorMessage, strlen(errorMessage)); char *firstNewlineIndex = strchr(errorMessage, '\n'); /* trim the error message at the line break */ if (firstNewlineIndex != NULL) { *firstNewlineIndex = '\0'; } } else { /* put a default error message if no error message is reported */ errorMessage = "An error occurred while running the query"; } appendStringInfo(queryResultString, "%s", errorMessage); } /* * IsSettingSafeToPropagate returns whether a SET LOCAL is safe to propagate. * * We exclude settings that are highly specific to the client or session and also * ban propagating the citus.propagate_set_commands setting (not for correctness, * more to avoid confusion). */ bool IsSettingSafeToPropagate(const char *name) { /* if this list grows considerably we should switch to bsearch */ const char *skipSettings[] = { "application_name", "citus.propagate_set_commands", "client_encoding", "exit_on_error", "max_stack_depth" }; for (Index settingIndex = 0; settingIndex < lengthof(skipSettings); settingIndex++) { if (pg_strcasecmp(skipSettings[settingIndex], name) == 0) { return false; } } return true; } ================================================ FILE: src/backend/distributed/connection/shared_connection_stats.c ================================================ /*------------------------------------------------------------------------- * * shared_connection_stats.c * Keeps track of the number of connections to remote nodes across * backends. The primary goal is to prevent excessive number of * connections (typically > max_connections) to any worker node. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "access/hash.h" #include "access/htup_details.h" #include "catalog/pg_authid.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "storage/ipc.h" #include "utils/builtins.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/cancel_utils.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/placement_connection.h" #include "distributed/shared_connection_stats.h" #include "distributed/time_constants.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" #define REMOTE_CONNECTION_STATS_COLUMNS 4 /* * The data structure used to store data in shared memory. This data structure is only * used for storing the lock. The actual statistics about the connections are stored * in the hashmap, which is allocated separately, as Postgres provides different APIs * for allocating hashmaps in the shared memory. */ typedef struct ConnectionStatsSharedData { int sharedConnectionHashTrancheId; char *sharedConnectionHashTrancheName; LWLock sharedConnectionHashLock; ConditionVariable waitersConditionVariable; } ConnectionStatsSharedData; typedef struct SharedConnStatsHashKey { /* * We keep the entries in the shared memory even after master_update_node() * as there might be some cached connections to the old node. * That's why, we prefer to use "hostname/port" over nodeId. */ char hostname[MAX_NODE_LENGTH]; int32 port; /* * Given that citus.shared_max_pool_size can be defined per database, we * should keep track of shared connections per database. */ Oid databaseOid; } SharedConnStatsHashKey; /* hash entry for per worker stats */ typedef struct SharedConnStatsHashEntry { SharedConnStatsHashKey key; int connectionCount; } SharedConnStatsHashEntry; /* * Controlled via a GUC, never access directly, use GetMaxSharedPoolSize(). * "0" means adjust MaxSharedPoolSize automatically by using MaxConnections. * "-1" means do not apply connection throttling * Anything else means use that number */ int MaxSharedPoolSize = 0; /* * Controlled via a GUC, never access directly, use GetLocalSharedPoolSize(). * "0" means adjust LocalSharedPoolSize automatically by using MaxConnections. * "-1" means do not use any remote connections for local tasks * Anything else means use that number */ int LocalSharedPoolSize = 0; /* number of connections reserved for Citus */ int MaxClientConnections = ALLOW_ALL_EXTERNAL_CONNECTIONS; /* the following two structs are used for accessing shared memory */ static HTAB *SharedConnStatsHash = NULL; static ConnectionStatsSharedData *ConnectionStatsSharedState = NULL; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* local function declarations */ static void StoreAllRemoteConnectionStats(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); static void LockConnectionSharedMemory(LWLockMode lockMode); static void UnLockConnectionSharedMemory(void); static bool ShouldWaitForConnection(int currentConnectionCount); static uint32 SharedConnectionHashHash(const void *key, Size keysize); static int SharedConnectionHashCompare(const void *a, const void *b, Size keysize); PG_FUNCTION_INFO_V1(citus_remote_connection_stats); /* * citus_remote_connection_stats returns all the avaliable information about all * the remote connections (a.k.a., connections to remote nodes). */ Datum citus_remote_connection_stats(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); StoreAllRemoteConnectionStats(tupleStore, tupleDescriptor); PG_RETURN_VOID(); } /* * StoreAllRemoteConnectionStats gets connections established from the current node * and inserts them into the given tuplestore. * * We don't need to enforce any access privileges as the number of backends * on any node is already visible on pg_stat_activity to all users. */ static void StoreAllRemoteConnectionStats(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { Datum values[REMOTE_CONNECTION_STATS_COLUMNS]; bool isNulls[REMOTE_CONNECTION_STATS_COLUMNS]; /* we're reading all shared connections, prevent any changes */ LockConnectionSharedMemory(LW_SHARED); HASH_SEQ_STATUS status; SharedConnStatsHashEntry *connectionEntry = NULL; hash_seq_init(&status, SharedConnStatsHash); while ((connectionEntry = (SharedConnStatsHashEntry *) hash_seq_search(&status)) != 0) { /* get ready for the next tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); char *databaseName = get_database_name(connectionEntry->key.databaseOid); if (databaseName == NULL) { /* database might have been dropped */ continue; } values[0] = PointerGetDatum(cstring_to_text(connectionEntry->key.hostname)); values[1] = Int32GetDatum(connectionEntry->key.port); values[2] = PointerGetDatum(cstring_to_text(databaseName)); values[3] = Int32GetDatum(connectionEntry->connectionCount); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } UnLockConnectionSharedMemory(); } /* * GetMaxClientConnections returns the value of citus.max_client_connections, * or max_connections when it is -1 or when connecting as superuser. * * The latter is done because citus.max_client_connections does not apply to * superuser. */ int GetMaxClientConnections(void) { if (MaxClientConnections == ALLOW_ALL_EXTERNAL_CONNECTIONS || superuser()) { return MaxConnections; } return MaxClientConnections; } /* * GetMaxSharedPoolSize is a wrapper around MaxSharedPoolSize which is controlled * via a GUC. * "0" means adjust MaxSharedPoolSize automatically by using MaxConnections * "-1" means do not apply connection throttling * Anything else means use that number */ int GetMaxSharedPoolSize(void) { if (MaxSharedPoolSize == ADJUST_POOLSIZE_AUTOMATICALLY) { return GetMaxClientConnections(); } return MaxSharedPoolSize; } /* * GetLocalSharedPoolSize is a wrapper around LocalSharedPoolSize which is * controlled via a GUC. * "0" means adjust MaxSharedPoolSize automatically by using MaxConnections * "-1" means do not use any remote connections for local tasks * Anything else means use that number */ int GetLocalSharedPoolSize(void) { if (LocalSharedPoolSize == ADJUST_POOLSIZE_AUTOMATICALLY) { return GetMaxClientConnections() * 0.5; } return LocalSharedPoolSize; } /* * WaitLoopForSharedConnection tries to increment the shared connection * counter for the given hostname/port and the current database in * SharedConnStatsHash. * * The function implements a retry mechanism via a condition variable. */ void WaitLoopForSharedConnection(const char *hostname, int port) { while (!TryToIncrementSharedConnectionCounter(hostname, port)) { CHECK_FOR_INTERRUPTS(); WaitForSharedConnection(); } ConditionVariableCancelSleep(); } /* * TryToIncrementSharedConnectionCounter tries to increment the shared * connection counter for the given nodeId and the current database in * SharedConnStatsHash. * * If the function returns true, the caller is allowed (and expected) * to establish a new connection to the given node. Else, the caller * is not allowed to establish a new connection. */ bool TryToIncrementSharedConnectionCounter(const char *hostname, int port) { if (GetMaxSharedPoolSize() == DISABLE_CONNECTION_THROTTLING) { /* connection throttling disabled */ return true; } bool counterIncremented = false; SharedConnStatsHashKey connKey; strlcpy(connKey.hostname, hostname, MAX_NODE_LENGTH); if (strlen(hostname) > MAX_NODE_LENGTH) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hostname exceeds the maximum length of %d", MAX_NODE_LENGTH))); } /* * The local session might already have some reserved connections to the given * node. In that case, we don't need to go through the shared memory. */ Oid userId = GetUserId(); if (CanUseReservedConnection(hostname, port, userId, MyDatabaseId)) { MarkReservedConnectionUsed(hostname, port, userId, MyDatabaseId); return true; } connKey.port = port; connKey.databaseOid = MyDatabaseId; /* * Handle adaptive connection management for the local node slightly different * as local node can failover to local execution. */ bool connectionToLocalNode = false; int activeBackendCount = 0; WorkerNode *workerNode = FindWorkerNode(hostname, port); if (workerNode) { connectionToLocalNode = (workerNode->groupId == GetLocalGroupId()); if (connectionToLocalNode && GetLocalSharedPoolSize() == DISABLE_REMOTE_CONNECTIONS_FOR_LOCAL_QUERIES) { /* * This early return is required as LocalNodeParallelExecutionFactor * is ignored for the first connection below. This check makes the * user experience is more accurate and also makes it easy for * having regression tests which emulates the local node adaptive * connection management. */ return false; } activeBackendCount = GetExternalClientBackendCount(); } LockConnectionSharedMemory(LW_EXCLUSIVE); /* * As the hash map is allocated in shared memory, it doesn't rely on palloc for * memory allocation, so we could get NULL via HASH_ENTER_NULL when there is no * space in the shared memory. That's why we prefer continuing the execution * instead of throwing an error. */ bool entryFound = false; SharedConnStatsHashEntry *connectionEntry = hash_search(SharedConnStatsHash, &connKey, HASH_ENTER_NULL, &entryFound); /* * It is possible to throw an error at this point, but that doesn't help us in anyway. * Instead, we try our best, let the connection establishment continue by-passing the * connection throttling. */ if (!connectionEntry) { UnLockConnectionSharedMemory(); return true; } if (!entryFound) { /* we successfully allocated the entry for the first time, so initialize it */ connectionEntry->connectionCount = 1; counterIncremented = true; } else if (connectionToLocalNode) { /* * For local nodes, solely relying on citus.max_shared_pool_size or * max_connections might not be sufficient. The former gives us * a preview of the future (e.g., we let the new connections to establish, * but they are not established yet). The latter gives us the close to * precise view of the past (e.g., the active number of client backends). * * Overall, we want to limit both of the metrics. The former limit typically * kicks in under regular loads, where the load of the database increases in * a reasonable pace. The latter limit typically kicks in when the database * is issued lots of concurrent sessions at the same time, such as benchmarks. */ if (activeBackendCount + 1 > GetLocalSharedPoolSize()) { counterIncremented = false; } else if (connectionEntry->connectionCount + 1 > GetLocalSharedPoolSize()) { counterIncremented = false; } else { connectionEntry->connectionCount++; counterIncremented = true; } } else if (connectionEntry->connectionCount + 1 > GetMaxSharedPoolSize()) { /* there is no space left for this connection */ counterIncremented = false; } else { connectionEntry->connectionCount++; counterIncremented = true; } UnLockConnectionSharedMemory(); return counterIncremented; } /* * IncrementSharedConnectionCounter increments the shared counter * for the given hostname and port. */ void IncrementSharedConnectionCounter(const char *hostname, int port) { SharedConnStatsHashKey connKey; if (MaxSharedPoolSize == DISABLE_CONNECTION_THROTTLING) { /* connection throttling disabled */ return; } strlcpy(connKey.hostname, hostname, MAX_NODE_LENGTH); if (strlen(hostname) > MAX_NODE_LENGTH) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hostname exceeds the maximum length of %d", MAX_NODE_LENGTH))); } connKey.port = port; connKey.databaseOid = MyDatabaseId; LockConnectionSharedMemory(LW_EXCLUSIVE); /* * As the hash map is allocated in shared memory, it doesn't rely on palloc for * memory allocation, so we could get NULL via HASH_ENTER_NULL. That's why we prefer * continuing the execution instead of throwing an error. */ bool entryFound = false; SharedConnStatsHashEntry *connectionEntry = hash_search(SharedConnStatsHash, &connKey, HASH_ENTER_NULL, &entryFound); /* * It is possible to throw an error at this point, but that doesn't help us in anyway. * Instead, we try our best, let the connection establishment continue by-passing the * connection throttling. */ if (!connectionEntry) { UnLockConnectionSharedMemory(); ereport(DEBUG4, (errmsg("No entry found for node %s:%d while incrementing " "connection counter", hostname, port))); return; } if (!entryFound) { /* we successfully allocated the entry for the first time, so initialize it */ connectionEntry->connectionCount = 0; } connectionEntry->connectionCount += 1; UnLockConnectionSharedMemory(); } /* * DecrementSharedConnectionCounter decrements the shared counter * for the given hostname and port for the given count. */ void DecrementSharedConnectionCounter(const char *hostname, int port) { SharedConnStatsHashKey connKey; /* * Do not call GetMaxSharedPoolSize() here, since it may read from * the catalog and we may be in the process exit handler. */ if (MaxSharedPoolSize == DISABLE_CONNECTION_THROTTLING) { /* connection throttling disabled */ return; } strlcpy(connKey.hostname, hostname, MAX_NODE_LENGTH); if (strlen(hostname) > MAX_NODE_LENGTH) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hostname exceeds the maximum length of %d", MAX_NODE_LENGTH))); } connKey.port = port; connKey.databaseOid = MyDatabaseId; LockConnectionSharedMemory(LW_EXCLUSIVE); bool entryFound = false; SharedConnStatsHashEntry *connectionEntry = hash_search(SharedConnStatsHash, &connKey, HASH_FIND, &entryFound); /* this worker node is removed or updated, no need to care */ if (!entryFound) { UnLockConnectionSharedMemory(); /* wake up any waiters in case any backend is waiting for this node */ WakeupWaiterBackendsForSharedConnection(); ereport(DEBUG4, (errmsg("No entry found for node %s:%d while decrementing " "connection counter", hostname, port))); return; } /* we should never go below 0 */ Assert(connectionEntry->connectionCount > 0); connectionEntry->connectionCount -= 1; if (connectionEntry->connectionCount == 0) { /* * We don't have to remove at this point as the node might be still active * and will have new connections open to it. Still, this seems like a convenient * place to remove the entry, as connectionCount == 0 implies that the server is * not busy, and given the default value of MaxCachedConnectionsPerWorker = 1, * we're unlikely to trigger this often. */ hash_search(SharedConnStatsHash, &connKey, HASH_REMOVE, &entryFound); } UnLockConnectionSharedMemory(); WakeupWaiterBackendsForSharedConnection(); } /* * LockConnectionSharedMemory is a utility function that should be used when * accessing to the SharedConnStatsHash, which is in the shared memory. */ static void LockConnectionSharedMemory(LWLockMode lockMode) { LWLockAcquire(&ConnectionStatsSharedState->sharedConnectionHashLock, lockMode); } /* * UnLockConnectionSharedMemory is a utility function that should be used after * LockConnectionSharedMemory(). */ static void UnLockConnectionSharedMemory(void) { LWLockRelease(&ConnectionStatsSharedState->sharedConnectionHashLock); } /* * WakeupWaiterBackendsForSharedConnection is a wrapper around the condition variable * broadcast operation. * * We use a single condition variable, for all worker nodes, to implement the connection * throttling mechanism. Combination of all the backends are allowed to establish * MaxSharedPoolSize number of connections per worker node. If a backend requires a * non-optional connection (see WAIT_FOR_CONNECTION for details), it is not allowed * to establish it immediately if the total connections are equal to MaxSharedPoolSize. * Instead, the backend waits on the condition variable. When any other backend * terminates an existing connection to any remote node, this function is called. * The main goal is to trigger all waiting backends to try getting a connection slot * in MaxSharedPoolSize. The ones which can get connection slot are allowed to continue * with the connection establishments. Others should wait another backend to call * this function. */ void WakeupWaiterBackendsForSharedConnection(void) { ConditionVariableBroadcast(&ConnectionStatsSharedState->waitersConditionVariable); } /* * WaitForSharedConnection is a wrapper around the condition variable sleep operation. * * For the details of the use of the condition variable, see * WakeupWaiterBackendsForSharedConnection(). */ void WaitForSharedConnection(void) { ConditionVariableSleep(&ConnectionStatsSharedState->waitersConditionVariable, PG_WAIT_EXTENSION); } /* * InitializeSharedConnectionStats requests the necessary shared memory * from Postgres and sets up the shared memory startup hook. */ void InitializeSharedConnectionStats(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = SharedConnectionStatsShmemInit; } /* * SharedConnectionStatsShmemSize returns the size that should be allocated * on the shared memory for shared connection stats. */ size_t SharedConnectionStatsShmemSize(void) { Size size = 0; size = add_size(size, sizeof(ConnectionStatsSharedData)); Size hashSize = hash_estimate_size(MaxWorkerNodesTracked, sizeof(SharedConnStatsHashEntry)); size = add_size(size, hashSize); return size; } /* * SharedConnectionStatsShmemInit initializes the shared memory used * for keeping track of connection stats across backends. */ void SharedConnectionStatsShmemInit(void) { bool alreadyInitialized = false; HASHCTL info; /* create (hostname, port, database) -> [counter] */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(SharedConnStatsHashKey); info.entrysize = sizeof(SharedConnStatsHashEntry); info.hash = SharedConnectionHashHash; info.match = SharedConnectionHashCompare; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_COMPARE); /* * Currently the lock isn't required because allocation only happens at * startup in postmaster, but it doesn't hurt, and makes things more * consistent with other extensions. */ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); ConnectionStatsSharedState = (ConnectionStatsSharedData *) ShmemInitStruct( "Shared Connection Stats Data", sizeof(ConnectionStatsSharedData), &alreadyInitialized); if (!alreadyInitialized) { ConnectionStatsSharedState->sharedConnectionHashTrancheId = LWLockNewTrancheId(); ConnectionStatsSharedState->sharedConnectionHashTrancheName = "Shared Connection Tracking Hash Tranche"; LWLockRegisterTranche( ConnectionStatsSharedState->sharedConnectionHashTrancheId, ConnectionStatsSharedState->sharedConnectionHashTrancheName); LWLockInitialize(&ConnectionStatsSharedState->sharedConnectionHashLock, ConnectionStatsSharedState->sharedConnectionHashTrancheId); ConditionVariableInit(&ConnectionStatsSharedState->waitersConditionVariable); } /* allocate hash table */ SharedConnStatsHash = ShmemInitHash("Shared Conn. Stats Hash", MaxWorkerNodesTracked, MaxWorkerNodesTracked, &info, hashFlags); LWLockRelease(AddinShmemInitLock); Assert(SharedConnStatsHash != NULL); Assert(ConnectionStatsSharedState->sharedConnectionHashTrancheId != 0); if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * AdaptiveConnectionManagementFlag returns the appropriate connection flag, * regarding the adaptive connection management, based on the given * activeConnectionCount to remote nodes. * * This function should only be called if the code-path is capable of handling * optional connections. */ int AdaptiveConnectionManagementFlag(bool connectToLocalNode, int activeConnectionCount) { if (UseConnectionPerPlacement()) { /* * User wants one connection per placement, so no throttling is desired * and we do not set any flags. * * The primary reason for this is that allowing multiple backends to use * connection per placement could lead to unresolved self deadlocks. In other * words, each backend may stuck waiting for other backends to get a slot * in the shared connection counters. */ return 0; } else if (connectToLocalNode) { /* * Connection to local node is always optional because the executor is capable * of falling back to local execution. */ return OPTIONAL_CONNECTION; } else if (ShouldWaitForConnection(activeConnectionCount)) { /* * We need this connection to finish the execution. If it is not * available based on the current number of connections to the worker * then wait for it. */ return WAIT_FOR_CONNECTION; } else { /* * The execution can be finished the execution with a single connection, * remaining are optional. If the execution can get more connections, * it can increase the parallelism. */ return OPTIONAL_CONNECTION; } } /* * ShouldWaitForConnection returns true if the workerPool should wait to * get the next connection until one slot is empty within * citus.max_shared_pool_size on the worker. Note that, if there is an * empty slot, the connection will not wait anyway. */ static bool ShouldWaitForConnection(int currentConnectionCount) { if (currentConnectionCount == 0) { /* * We definitely need at least 1 connection to finish the execution. * All single shard queries hit here with the default settings. */ return true; } if (currentConnectionCount < MaxCachedConnectionsPerWorker) { /* * Until this session caches MaxCachedConnectionsPerWorker connections, * this might lead some optional connections to be considered as non-optional * when MaxCachedConnectionsPerWorker > 1. * * However, once the session caches MaxCachedConnectionsPerWorker (which is * the second transaction executed in the session), Citus would utilize the * cached connections as much as possible. */ return true; } return false; } static uint32 SharedConnectionHashHash(const void *key, Size keysize) { SharedConnStatsHashKey *entry = (SharedConnStatsHashKey *) key; uint32 hash = string_hash(entry->hostname, NAMEDATALEN); hash = hash_combine(hash, hash_uint32(entry->port)); hash = hash_combine(hash, hash_uint32(entry->databaseOid)); return hash; } static int SharedConnectionHashCompare(const void *a, const void *b, Size keysize) { SharedConnStatsHashKey *ca = (SharedConnStatsHashKey *) a; SharedConnStatsHashKey *cb = (SharedConnStatsHashKey *) b; if (strncmp(ca->hostname, cb->hostname, MAX_NODE_LENGTH) != 0 || ca->port != cb->port || ca->databaseOid != cb->databaseOid) { return 1; } else { return 0; } } ================================================ FILE: src/backend/distributed/connection/worker_log_messages.c ================================================ /*------------------------------------------------------------------------- * * worker_log_messages.c * Logic for handling log messages from workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/elog.h" #include "utils/memutils.h" /* for TopTransactionContext */ #include "distributed/connection_management.h" #include "distributed/error_codes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/worker_log_messages.h" /* * WorkerMinMessages reflects the value of the citus.worker_min_messages setting which * control the minimum log level of messages from the worker that are propagated to the * client and the log on the coordinator. */ int WorkerMinMessages = NOTICE; /* * PreserveWorkerMessageLogLevel specifies whether to propagate messages from workers * to the client and the log on the coordinator with their original log level. When * false, messages are propagated using DEBUG1. * * This flag used to suppress redundant notices in some commands (e.g. VACUUM, DROP * TABLE). */ static bool PreserveWorkerMessageLogLevel = false; /* * WorkerErrorIndication can contain a warning that arrives to use from one session, but occurred * because another session in the same distributed transaction threw an error. We store * this warning in case we do not get an error, in which case the warning should have * been an error (and usually indicates a bug). */ DeferredErrorMessage *WorkerErrorIndication = NULL; /* list of log level names we might see from the worker */ static const char *LogLevelNames[] = { "DEBUG", "NOTICE", "INFO", "WARNING", "ERROR", "FATAL", "PANIC", NULL }; /* postgres log level values corresponding to LogLevelNames */ static const int LogLevels[] = { DEBUG1, NOTICE, INFO, WARNING, ERROR, FATAL, PANIC }; static void DefaultCitusNoticeReceiver(void *arg, const PGresult *result); static int LogLevelNameToLogLevel(char *levelName); static char * TrimLogLevel(const char *message); /* * SetCitusNoticeReceiver sets the NoticeReceiver to DefaultCitusNoticeReceivere */ void SetCitusNoticeReceiver(MultiConnection *connection) { PQsetNoticeReceiver(connection->pgConn, DefaultCitusNoticeReceiver, connection); } /* * EnableWorkerMessagePropagation indicates that we want to propagate messages * from workers to the client using the same log level. */ void EnableWorkerMessagePropagation(void) { PreserveWorkerMessageLogLevel = true; } /* * DisableWorkerMessagePropagation indiciates that we want all messages from the * workers to only be sent to the client as debug messages. */ void DisableWorkerMessagePropagation(void) { PreserveWorkerMessageLogLevel = false; } /* * DefaultCitusNoticeReceiver is used to redirect worker notices * from logfile to console. */ static void DefaultCitusNoticeReceiver(void *arg, const PGresult *result) { MultiConnection *connection = (MultiConnection *) arg; char *nodeName = connection->hostname; uint32 nodePort = connection->port; char *message = PQresultErrorMessage(result); char *trimmedMessage = TrimLogLevel(message); char *levelName = PQresultErrorField(result, PG_DIAG_SEVERITY); int logLevel = LogLevelNameToLogLevel(levelName); int sqlState = ERRCODE_INTERNAL_ERROR; char *sqlStateString = PQresultErrorField(result, PG_DIAG_SQLSTATE); if (sqlStateString != NULL) { sqlState = MAKE_SQLSTATE(sqlStateString[0], sqlStateString[1], sqlStateString[2], sqlStateString[3], sqlStateString[4]); } /* * When read_intermediate_result cannot find a file it might mean that the * transaction that created the file already deleted it because it aborted. * That's an expected situation, unless there is no actual error. We * therefore store a DeferredError and raise it if we reach the end of * execution without errors. */ if (sqlState == ERRCODE_CITUS_INTERMEDIATE_RESULT_NOT_FOUND && logLevel == WARNING) { if (WorkerErrorIndication == NULL) { /* we'll at most need this for the lifetime of the transaction */ MemoryContext oldContext = MemoryContextSwitchTo(TopTransactionContext); WorkerErrorIndication = DeferredError(sqlState, pstrdup(trimmedMessage), NULL, NULL); MemoryContextSwitchTo(oldContext); } /* if we get the error we're expecting, the user does not need to know */ logLevel = DEBUG4; } if (logLevel < WorkerMinMessages || WorkerMinMessages == CITUS_LOG_LEVEL_OFF) { /* user does not want to see message */ return; } if (!PreserveWorkerMessageLogLevel) { /* * We sometimes want to suppress notices (e.g. DROP TABLE cascading), * since the user already gets the relevant notices for the distributed * table. In that case, we change the log level to DEBUG1. */ logLevel = DEBUG1; } ereport(logLevel, (errcode(sqlState), errmsg("%s", trimmedMessage), errdetail("from %s:%d", nodeName, nodePort))); } /* * TrimLogLevel returns a copy of the string with the leading log level * and spaces removed such as * From: * INFO: "normal2_102070": scanned 0 of 0 pages... * To: * "normal2_102070": scanned 0 of 0 pages... */ static char * TrimLogLevel(const char *message) { char *chompedMessage = pchomp(message); size_t n = 0; while (n < strlen(chompedMessage) && chompedMessage[n] != ':') { n++; } do { n++; } while (n < strlen(chompedMessage) && chompedMessage[n] == ' '); return chompedMessage + n; } /* * LogLevelNameToLogLevel translates the prefix of Postgres log messages * back to a native log level. */ static int LogLevelNameToLogLevel(char *levelName) { int levelIndex = 0; while (LogLevelNames[levelIndex] != NULL) { if (strcmp(levelName, LogLevelNames[levelIndex]) == 0) { return LogLevels[levelIndex]; } levelIndex++; } return DEBUG1; } /* * ErrorIfWorkerErrorIndicationReceived throws the deferred error in * WorkerErrorIndication, if any. * * A fatal warning arrives to us as a WARNING in one session, that is triggered * by an ERROR in another session in the same distributed transaction. We therefore * do not expect to throw it, unless there is a bug in Citus. */ void ErrorIfWorkerErrorIndicationReceived(void) { if (WorkerErrorIndication != NULL) { RaiseDeferredError(WorkerErrorIndication, ERROR); } } /* * ResetWorkerErrorIndication resets the fatal warning if one was received. */ void ResetWorkerErrorIndication(void) { WorkerErrorIndication = NULL; } ================================================ FILE: src/backend/distributed/deparser/citus_deparseutils.c ================================================ /*------------------------------------------------------------------------- * * citus_deparseutils.c * * This file contains common functions used for deparsing PostgreSQL * statements to their equivalent SQL representation. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "commands/defrem.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/deparser.h" /** * DefElemOptionToStatement converts a DefElem option to a SQL statement and * appends it to the given StringInfo buffer. * * @param buf The StringInfo buffer to append the SQL statement to. * @param option The DefElem option to convert to a SQL statement. * @param optionFormats The option format specification to use for the conversion. * @param optionFormatsLen The number of option formats in the opt_formats array. */ void DefElemOptionToStatement(StringInfo buf, DefElem *option, const DefElemOptionFormat *optionFormats, int optionFormatsLen) { const char *name = option->defname; int i; for (i = 0; i < optionFormatsLen; i++) { if (strcmp(name, optionFormats[i].name) == 0) { switch (optionFormats[i].type) { case OPTION_FORMAT_STRING: { char *value = defGetString(option); appendStringInfo(buf, optionFormats[i].format, quote_identifier( value)); break; } case OPTION_FORMAT_INTEGER: { int32 value = defGetInt32(option); appendStringInfo(buf, optionFormats[i].format, value); break; } case OPTION_FORMAT_BOOLEAN: { bool value = defGetBoolean(option); appendStringInfo(buf, optionFormats[i].format, value ? "true" : "false"); break; } case OPTION_FORMAT_LITERAL_CSTR: { char *value = defGetString(option); appendStringInfo(buf, optionFormats[i].format, quote_literal_cstr( value)); break; } default: { elog(ERROR, "unrecognized option type: %d", optionFormats[i].type); break; } } } } } ================================================ FILE: src/backend/distributed/deparser/citus_grantutils.c ================================================ #include "postgres.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" /* * Append the 'WITH GRANT OPTION' clause to the given buffer if the given * statement is a 'GRANT' statement and the grant option is specified. */ void AppendWithGrantOption(StringInfo buf, GrantStmt *stmt) { if (stmt->is_grant && stmt->grant_option) { appendStringInfo(buf, " WITH GRANT OPTION"); } } /* * Append the 'GRANT OPTION FOR' clause to the given buffer if the given * statement is a 'REVOKE' statement and the grant option is specified. */ void AppendGrantOptionFor(StringInfo buf, GrantStmt *stmt) { if (!stmt->is_grant && stmt->grant_option) { appendStringInfo(buf, "GRANT OPTION FOR "); } } /* * Append the 'RESTRICT' or 'CASCADE' clause to the given buffer if the given * statement is a 'REVOKE' statement and the behavior is specified. */ void AppendGrantRestrictAndCascadeForRoleSpec(StringInfo buf, DropBehavior behavior, bool isGrant) { if (!isGrant) { if (behavior == DROP_RESTRICT) { appendStringInfo(buf, " RESTRICT"); } else if (behavior == DROP_CASCADE) { appendStringInfo(buf, " CASCADE"); } } } /* * Append the 'RESTRICT' or 'CASCADE' clause to the given buffer using 'GrantStmt', * if the given statement is a 'REVOKE' statement and the behavior is specified. */ void AppendGrantRestrictAndCascade(StringInfo buf, GrantStmt *stmt) { AppendGrantRestrictAndCascadeForRoleSpec(buf, stmt->behavior, stmt->is_grant); } /* * Append the 'GRANTED BY' clause to the given buffer if the given statement is a * 'GRANT' statement and the grantor is specified. */ void AppendGrantedByInGrantForRoleSpec(StringInfo buf, RoleSpec *grantor, bool isGrant) { if (grantor) { appendStringInfo(buf, " GRANTED BY %s", RoleSpecString(grantor, true)); } } /* * Append the 'GRANTED BY' clause to the given buffer using 'GrantStmt', * if the given statement is a 'GRANT' statement and the grantor is specified. */ void AppendGrantedByInGrant(StringInfo buf, GrantStmt *stmt) { AppendGrantedByInGrantForRoleSpec(buf, stmt->grantor, stmt->is_grant); } void AppendGrantSharedPrefix(StringInfo buf, GrantStmt *stmt) { appendStringInfo(buf, "%s ", stmt->is_grant ? "GRANT" : "REVOKE"); AppendGrantOptionFor(buf, stmt); AppendGrantPrivileges(buf, stmt); } void AppendGrantSharedSuffix(StringInfo buf, GrantStmt *stmt) { AppendGrantGrantees(buf, stmt); AppendWithGrantOption(buf, stmt); AppendGrantRestrictAndCascade(buf, stmt); AppendGrantedByInGrant(buf, stmt); appendStringInfo(buf, ";"); } ================================================ FILE: src/backend/distributed/deparser/citus_ruleutils.c ================================================ /*------------------------------------------------------------------------- * * citus_ruleutils.c * Version independent ruleutils wrapper * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "access/attnum.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/stratnum.h" #include "access/sysattr.h" #include "access/toast_compression.h" #include "access/tupdesc.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" #include "catalog/pg_attribute.h" #include "catalog/pg_authid.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_index.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/sequence.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "parser/parse_utilcmd.h" #include "parser/parser.h" #include "storage/lock.h" #include "utils/acl.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/namespace_utils.h" #include "distributed/relay_utility.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" static void deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparseContext); static void AppendStorageParametersToString(StringInfo stringBuffer, List *optionList); static const char * convert_aclright_to_string(int aclright); static void simple_quote_literal(StringInfo buf, const char *val); static SubscriptingRef * TargetEntryExprFindSubsRef(Expr *expr); static void AddVacuumParams(ReindexStmt *reindexStmt, StringInfo buffer); static void process_acl_items(Acl *acl, const char *relationName, const char *attributeName, List **defs); /* * pg_get_extensiondef_string finds the foreign data wrapper that corresponds to * the given foreign tableId, and checks if an extension owns this foreign data * wrapper. If it does, the function returns the extension's definition. If not, * the function returns null. */ char * pg_get_extensiondef_string(Oid tableRelationId) { ForeignTable *foreignTable = GetForeignTable(tableRelationId); ForeignServer *server = GetForeignServer(foreignTable->serverid); ForeignDataWrapper *foreignDataWrapper = GetForeignDataWrapper(server->fdwid); StringInfoData buffer = { NULL, 0, 0, 0 }; Oid classId = ForeignDataWrapperRelationId; Oid objectId = server->fdwid; Oid extensionId = getExtensionOfObject(classId, objectId); if (OidIsValid(extensionId)) { char *extensionName = get_extension_name(extensionId); Oid extensionSchemaId = get_extension_schema(extensionId); char *extensionSchema = get_namespace_name(extensionSchemaId); initStringInfo(&buffer); appendStringInfo(&buffer, "CREATE EXTENSION IF NOT EXISTS %s WITH SCHEMA %s", quote_identifier(extensionName), quote_identifier(extensionSchema)); } else { ereport(NOTICE, (errmsg("foreign-data wrapper \"%s\" does not have an " "extension defined", foreignDataWrapper->fdwname))); } return (buffer.data); } /* * get_extension_version - given an extension OID, fetch its extversion * or NULL if not found. */ char * get_extension_version(Oid extensionId) { char *versionName = NULL; Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyData entry[1]; ScanKeyInit(&entry[0], Anum_pg_extension_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(extensionId)); SysScanDesc scanDesc = systable_beginscan(relation, ExtensionOidIndexId, true, NULL, 1, entry); HeapTuple tuple = systable_getnext(scanDesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) { bool isNull = false; Datum versionDatum = heap_getattr(tuple, Anum_pg_extension_extversion, RelationGetDescr(relation), &isNull); if (!isNull) { versionName = text_to_cstring(DatumGetTextPP(versionDatum)); } } systable_endscan(scanDesc); table_close(relation, AccessShareLock); return versionName; } /* * get_extension_schema - given an extension OID, fetch its extnamespace * * Returns InvalidOid if no such extension. */ Oid get_extension_schema(Oid ext_oid) { /* *INDENT-OFF* */ Oid result; Relation rel; HeapTuple tuple; ScanKeyData entry[1]; rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ext_oid)); SysScanDesc scandesc = systable_beginscan(rel, ExtensionOidIndexId, true, NULL, 1, entry); tuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace; else result = InvalidOid; systable_endscan(scandesc); table_close(rel, AccessShareLock); return result; /* *INDENT-ON* */ } /* * pg_get_serverdef_string finds the foreign server that corresponds to the * given foreign tableId, and returns this server's definition. */ char * pg_get_serverdef_string(Oid tableRelationId) { ForeignTable *foreignTable = GetForeignTable(tableRelationId); ForeignServer *server = GetForeignServer(foreignTable->serverid); ForeignDataWrapper *foreignDataWrapper = GetForeignDataWrapper(server->fdwid); StringInfoData buffer = { NULL, 0, 0, 0 }; initStringInfo(&buffer); appendStringInfo(&buffer, "CREATE SERVER IF NOT EXISTS %s", quote_identifier(server->servername)); if (server->servertype != NULL) { appendStringInfo(&buffer, " TYPE %s", quote_literal_cstr(server->servertype)); } if (server->serverversion != NULL) { appendStringInfo(&buffer, " VERSION %s", quote_literal_cstr(server->serverversion)); } appendStringInfo(&buffer, " FOREIGN DATA WRAPPER %s", quote_identifier(foreignDataWrapper->fdwname)); /* append server options, if any */ AppendOptionListToString(&buffer, server->options); return (buffer.data); } /* * pg_get_sequencedef_string returns the definition of a given sequence. This * definition includes explicit values for all CREATE SEQUENCE options. */ char * pg_get_sequencedef_string(Oid sequenceRelationId) { Form_pg_sequence pgSequenceForm = pg_get_sequencedef(sequenceRelationId); /* build our DDL command */ char *qualifiedSequenceName = generate_qualified_relation_name(sequenceRelationId); char *typeName = format_type_be(pgSequenceForm->seqtypid); char *sequenceDef = psprintf(CREATE_SEQUENCE_COMMAND, get_rel_persistence(sequenceRelationId) == RELPERSISTENCE_UNLOGGED ? "UNLOGGED " : "", qualifiedSequenceName, typeName, pgSequenceForm->seqincrement, pgSequenceForm->seqmin, pgSequenceForm->seqmax, pgSequenceForm->seqstart, pgSequenceForm->seqcache, pgSequenceForm->seqcycle ? "" : "NO "); return sequenceDef; } /* * pg_get_sequencedef returns the Form_pg_sequence data about the sequence with the given * object id. */ Form_pg_sequence pg_get_sequencedef(Oid sequenceRelationId) { HeapTuple heapTuple = SearchSysCache1(SEQRELID, sequenceRelationId); if (!HeapTupleIsValid(heapTuple)) { elog(ERROR, "cache lookup failed for sequence %u", sequenceRelationId); } Form_pg_sequence pgSequenceForm = (Form_pg_sequence) GETSTRUCT(heapTuple); ReleaseSysCache(heapTuple); return pgSequenceForm; } /* * pg_get_tableschemadef_string returns the definition of a given table. This * definition includes table's schema, default column values, not null and check * constraints. The definition does not include constraints that trigger index * creations; specifically, unique and primary key constraints are excluded. * When includeSequenceDefaults is NEXTVAL_SEQUENCE_DEFAULTS, the function also creates * DEFAULT clauses for columns getting their default values from a sequence. * When it's WORKER_NEXTVAL_SEQUENCE_DEFAULTS, the function creates the DEFAULT * clause using worker_nextval('sequence') and not nextval('sequence') * When IncludeIdentities is NO_IDENTITY, the function does not include identity column * specifications. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. */ char * pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, char *accessMethod) { bool firstAttributePrinted = false; AttrNumber defaultValueIndex = 0; AttrNumber constraintIndex = 0; AttrNumber constraintCount = 0; bool relIsPartition = false; StringInfoData buffer = { NULL, 0, 0, 0 }; /* * Instead of retrieving values from system catalogs as other functions in * ruleutils.c do, we follow an unusual approach here: we open the relation, * and fetch the relation's tuple descriptor. We do this because the tuple * descriptor already contains information harnessed from pg_attrdef, * pg_attribute, pg_constraint, and pg_class; and therefore using the * descriptor saves us from a lot of additional work. */ Relation relation = relation_open(tableRelationId, AccessShareLock); char *relationName = generate_relation_name(tableRelationId, NIL); EnsureRelationKindSupported(tableRelationId); initStringInfo(&buffer); if (RegularTable(tableRelationId)) { appendStringInfoString(&buffer, "CREATE "); if (relation->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED) { appendStringInfoString(&buffer, "UNLOGGED "); } appendStringInfo(&buffer, "TABLE %s (", relationName); relIsPartition = relation->rd_rel->relispartition; } else { appendStringInfo(&buffer, "CREATE FOREIGN TABLE %s (", relationName); } /* * Iterate over the table's columns. If a particular column is not dropped * and is not inherited from another table, print the column's name and its * formatted type. */ TupleDesc tupleDescriptor = RelationGetDescr(relation); TupleConstr *tupleConstraints = tupleDescriptor->constr; for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); /* * We disregard the inherited attributes (i.e., attinhcount > 0) here. The * reasoning behind this is that Citus implements declarative partitioning * by creating the partitions first and then sending * "ALTER TABLE parent_table ATTACH PARTITION .." command. This may not play * well with regular inherited tables, which isn't a big concern from Citus' * perspective. */ if (!attributeForm->attisdropped) { if (firstAttributePrinted) { appendStringInfoString(&buffer, ", "); } firstAttributePrinted = true; const char *attributeName = NameStr(attributeForm->attname); appendStringInfo(&buffer, "%s ", quote_identifier(attributeName)); const char *attributeTypeName = format_type_with_typemod( attributeForm->atttypid, attributeForm-> atttypmod); appendStringInfoString(&buffer, attributeTypeName); if (CompressionMethodIsValid(attributeForm->attcompression)) { appendStringInfo(&buffer, " COMPRESSION %s", GetCompressionMethodName(attributeForm->attcompression)); } /* * If this is an identity column include its identity definition in the * DDL only if its relation is not a partition. If it is a partition, any * identity is inherited from the parent table by ATTACH PARTITION. This * is Postgres 17+ behavior (commit 699586315); prior PG versions did not * support identity columns in partitioned tables. */ if (attributeForm->attidentity && includeIdentityDefaults && !relIsPartition) { bool missing_ok = false; Oid seqOid = getIdentitySequence(identitySequenceRelation_compat( relation), attributeForm->attnum, missing_ok); if (includeIdentityDefaults == INCLUDE_IDENTITY) { Form_pg_sequence pgSequenceForm = pg_get_sequencedef(seqOid); char *sequenceDef = psprintf( " GENERATED %s AS IDENTITY (INCREMENT BY " INT64_FORMAT \ " MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \ " START WITH " INT64_FORMAT " CACHE " INT64_FORMAT " %sCYCLE)", attributeForm->attidentity == ATTRIBUTE_IDENTITY_ALWAYS ? "ALWAYS" : "BY DEFAULT", pgSequenceForm->seqincrement, pgSequenceForm->seqmin, pgSequenceForm->seqmax, pgSequenceForm->seqstart, pgSequenceForm->seqcache, pgSequenceForm->seqcycle ? "" : "NO "); appendStringInfo(&buffer, "%s", sequenceDef); } } /* if this column has a default value, append the default value */ if (attributeForm->atthasdef) { List *defaultContext = NULL; char *defaultString = NULL; Assert(tupleConstraints != NULL); AttrDefault *defaultValueList = tupleConstraints->defval; Assert(defaultValueList != NULL); AttrDefault *defaultValue = &(defaultValueList[defaultValueIndex]); defaultValueIndex++; Assert(defaultValue->adnum == (attributeIndex + 1)); Assert(defaultValueIndex <= tupleConstraints->num_defval); /* convert expression to node tree, and prepare deparse context */ Node *defaultNode = (Node *) stringToNode(defaultValue->adbin); /* * if column default value is explicitly requested, or it is * not set from a sequence then we include DEFAULT clause for * this column. */ if (includeSequenceDefaults || !contain_nextval_expression_walker(defaultNode, NULL)) { defaultContext = deparse_context_for(relationName, tableRelationId); /* deparse default value string */ defaultString = deparse_expression(defaultNode, defaultContext, false, false); if (attributeForm->attgenerated == ATTRIBUTE_GENERATED_STORED) { appendStringInfo(&buffer, " GENERATED ALWAYS AS (%s) STORED", defaultString); } #if PG_VERSION_NUM >= PG_VERSION_18 else if (attributeForm->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) { appendStringInfo(&buffer, " GENERATED ALWAYS AS (%s) VIRTUAL", defaultString); } #endif else { Oid seqOid = GetSequenceOid(tableRelationId, defaultValue->adnum); if (includeSequenceDefaults == WORKER_NEXTVAL_SEQUENCE_DEFAULTS && seqOid != InvalidOid && pg_get_sequencedef(seqOid)->seqtypid != INT8OID) { /* * We use worker_nextval for int and smallint types. * Check issue #5126 and PR #5254 for details. * https://github.com/citusdata/citus/issues/5126 */ char *sequenceName = generate_qualified_relation_name( seqOid); appendStringInfo(&buffer, " DEFAULT worker_nextval(%s::regclass)", quote_literal_cstr(sequenceName)); } else { appendStringInfo(&buffer, " DEFAULT %s", defaultString); } } } } /* if this column has a not null constraint, append the constraint */ if (attributeForm->attnotnull) { appendStringInfoString(&buffer, " NOT NULL"); } if (attributeForm->attcollation != InvalidOid && attributeForm->attcollation != DEFAULT_COLLATION_OID) { appendStringInfo(&buffer, " COLLATE %s", generate_collation_name( attributeForm->attcollation)); } } } /* * Now check if the table has any constraints. If it does, set the number of * check constraints here. Then iterate over all check constraints and print * them. */ if (tupleConstraints != NULL) { constraintCount = tupleConstraints->num_check; } for (constraintIndex = 0; constraintIndex < constraintCount; constraintIndex++) { ConstrCheck *checkConstraintList = tupleConstraints->check; ConstrCheck *checkConstraint = &(checkConstraintList[constraintIndex]); /* if an attribute or constraint has been printed, format properly */ if (firstAttributePrinted || constraintIndex > 0) { appendStringInfoString(&buffer, ", "); } appendStringInfo(&buffer, "CONSTRAINT %s CHECK ", quote_identifier(checkConstraint->ccname)); /* convert expression to node tree, and prepare deparse context */ Node *checkNode = (Node *) stringToNode(checkConstraint->ccbin); List *checkContext = deparse_context_for(relationName, tableRelationId); /* deparse check constraint string */ char *checkString = deparse_expression(checkNode, checkContext, false, false); appendStringInfoString(&buffer, "("); appendStringInfoString(&buffer, checkString); appendStringInfoString(&buffer, ")"); #if PG_VERSION_NUM >= PG_VERSION_18 if (!checkConstraint->ccenforced) { appendStringInfoString(&buffer, " NOT ENFORCED"); } #endif } /* close create table's outer parentheses */ appendStringInfoString(&buffer, ")"); /* * If the relation is a foreign table, append the server name and options to * the create table statement. */ char relationKind = relation->rd_rel->relkind; if (relationKind == RELKIND_FOREIGN_TABLE) { ForeignTable *foreignTable = GetForeignTable(tableRelationId); ForeignServer *foreignServer = GetForeignServer(foreignTable->serverid); char *serverName = foreignServer->servername; appendStringInfo(&buffer, " SERVER %s", quote_identifier(serverName)); AppendOptionListToString(&buffer, foreignTable->options); } else if (relationKind == RELKIND_PARTITIONED_TABLE) { char *partitioningInformation = GeneratePartitioningInformation(tableRelationId); appendStringInfo(&buffer, " PARTITION BY %s ", partitioningInformation); } /* * Add table access methods when the table is configured with an * access method */ if (accessMethod) { appendStringInfo(&buffer, " USING %s", quote_identifier(accessMethod)); } else if (OidIsValid(relation->rd_rel->relam)) { HeapTuple amTup = SearchSysCache1(AMOID, ObjectIdGetDatum( relation->rd_rel->relam)); if (!HeapTupleIsValid(amTup)) { elog(ERROR, "cache lookup failed for access method %u", relation->rd_rel->relam); } Form_pg_am amForm = (Form_pg_am) GETSTRUCT(amTup); appendStringInfo(&buffer, " USING %s", quote_identifier(NameStr(amForm->amname))); ReleaseSysCache(amTup); } /* * Add any reloptions (storage parameters) defined on the table in a WITH * clause. */ { char *reloptions = flatten_reloptions(tableRelationId); if (reloptions) { appendStringInfo(&buffer, " WITH (%s)", reloptions); pfree(reloptions); } } relation_close(relation, AccessShareLock); return (buffer.data); } /* * EnsureRelationKindSupported errors out if the given relation is not supported * as a distributed relation. */ void EnsureRelationKindSupported(Oid relationId) { char relationKind = get_rel_relkind(relationId); if (!relationKind) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relation with OID %d does not exist", relationId))); } bool supportedRelationKind = RegularTable(relationId) || relationKind == RELKIND_FOREIGN_TABLE; /* * Citus doesn't support bare inherited tables (i.e., not a partition or * partitioned table) */ supportedRelationKind = supportedRelationKind && !(IsChildTable(relationId) || IsParentTable(relationId)); if (!supportedRelationKind) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s is not a regular, foreign or partitioned table", relationName))); } } /* * pg_get_tablecolumnoptionsdef_string returns column storage type and column * statistics definitions for given table, _if_ these definitions differ from * their default values. The function returns null if all columns use default * values for their storage types and statistics. */ char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId) { List *columnOptionList = NIL; ListCell *columnOptionCell = NULL; bool firstOptionPrinted = false; StringInfoData buffer = { NULL, 0, 0, 0 }; /* * Instead of retrieving values from system catalogs, we open the relation, * and use the relation's tuple descriptor to access attribute information. * This is primarily to maintain symmetry with pg_get_tableschemadef. */ Relation relation = relation_open(tableRelationId, AccessShareLock); EnsureRelationKindSupported(tableRelationId); /* * Iterate over the table's columns. If a particular column is not dropped * and is not inherited from another table, check if column storage or * statistics statements need to be printed. */ TupleDesc tupleDescriptor = RelationGetDescr(relation); if (tupleDescriptor->natts > MaxAttrNumber) { ereport(ERROR, (errmsg("bad number of tuple descriptor attributes"))); } AttrNumber natts = tupleDescriptor->natts; for (AttrNumber attributeIndex = 0; attributeIndex < natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); char *attributeName = NameStr(attributeForm->attname); char defaultStorageType = get_typstorage(attributeForm->atttypid); if (!attributeForm->attisdropped && attributeForm->attinhcount == 0) { /* * If the user changed the column's default storage type, create * alter statement and add statement to a list for later processing. */ if (attributeForm->attstorage != defaultStorageType) { char *storageName = 0; StringInfoData statement = { NULL, 0, 0, 0 }; initStringInfo(&statement); switch (attributeForm->attstorage) { case 'p': { storageName = "PLAIN"; break; } case 'e': { storageName = "EXTERNAL"; break; } case 'm': { storageName = "MAIN"; break; } case 'x': { storageName = "EXTENDED"; break; } default: { ereport(ERROR, (errmsg("unrecognized storage type: %c", attributeForm->attstorage))); break; } } appendStringInfo(&statement, "ALTER COLUMN %s ", quote_identifier(attributeName)); appendStringInfo(&statement, "SET STORAGE %s", storageName); columnOptionList = lappend(columnOptionList, statement.data); } /* * If the user changed the column's statistics target, create * alter statement and add statement to a list for later processing. */ HeapTuple atttuple = SearchSysCache2(ATTNUM, ObjectIdGetDatum(tableRelationId), Int16GetDatum(attributeForm->attnum)); if (!HeapTupleIsValid(atttuple)) { elog(ERROR, "cache lookup failed for attribute %d of relation %u", attributeForm->attnum, tableRelationId); } int32 targetAttstattarget = getAttstattarget_compat(atttuple); ReleaseSysCache(atttuple); if (targetAttstattarget >= 0) { StringInfoData statement = { NULL, 0, 0, 0 }; initStringInfo(&statement); appendStringInfo(&statement, "ALTER COLUMN %s ", quote_identifier(attributeName)); appendStringInfo(&statement, "SET STATISTICS %d", targetAttstattarget); columnOptionList = lappend(columnOptionList, statement.data); } } } /* * Iterate over column storage and statistics statements that we created, * and append them to a single alter table statement. */ foreach(columnOptionCell, columnOptionList) { if (!firstOptionPrinted) { initStringInfo(&buffer); appendStringInfo(&buffer, "ALTER TABLE ONLY %s ", generate_relation_name(tableRelationId, NIL)); } else { appendStringInfoString(&buffer, ", "); } firstOptionPrinted = true; char *columnOptionStatement = (char *) lfirst(columnOptionCell); appendStringInfoString(&buffer, columnOptionStatement); pfree(columnOptionStatement); } list_free(columnOptionList); relation_close(relation, AccessShareLock); return (buffer.data); } /* * deparse_shard_index_statement uses the provided CREATE INDEX node, dist. * relation, and shard identifier to populate a provided buffer with a string * representation of a shard-extended version of that command. */ void deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid, StringInfo buffer) { IndexStmt *indexStmt = copyObject(origStmt); /* copy to avoid modifications */ /* extend relation and index name using shard identifier */ AppendShardIdToName(&(indexStmt->relation->relname), shardid); AppendShardIdToName(&(indexStmt->idxname), shardid); char *relationName = indexStmt->relation->relname; char *indexName = indexStmt->idxname; /* use extended shard name and transformed stmt for deparsing */ List *deparseContext = deparse_context_for(relationName, distrelid); indexStmt = transformIndexStmt(distrelid, indexStmt, NULL); appendStringInfo(buffer, "CREATE %s INDEX %s %s %s ON %s %s USING %s ", (indexStmt->unique ? "UNIQUE" : ""), (indexStmt->concurrent ? "CONCURRENTLY" : ""), (indexStmt->if_not_exists ? "IF NOT EXISTS" : ""), quote_identifier(indexName), (indexStmt->relation->inh ? "" : "ONLY"), quote_qualified_identifier(indexStmt->relation->schemaname, relationName), indexStmt->accessMethod); /* * Switch to empty search_path to deparse_index_columns to produce fully- * qualified names in expressions. */ int saveNestLevel = PushEmptySearchPath(); /* index column or expression list begins here */ appendStringInfoChar(buffer, '('); deparse_index_columns(buffer, indexStmt->indexParams, deparseContext); appendStringInfoString(buffer, ") "); /* column/expressions for INCLUDE list */ if (indexStmt->indexIncludingParams != NIL) { appendStringInfoString(buffer, "INCLUDE ("); deparse_index_columns(buffer, indexStmt->indexIncludingParams, deparseContext); appendStringInfoString(buffer, ") "); } if (indexStmt->nulls_not_distinct) { appendStringInfoString(buffer, "NULLS NOT DISTINCT "); } if (indexStmt->options != NIL) { appendStringInfoString(buffer, "WITH ("); AppendStorageParametersToString(buffer, indexStmt->options); appendStringInfoString(buffer, ") "); } if (indexStmt->whereClause != NULL) { appendStringInfo(buffer, "WHERE %s", deparse_expression(indexStmt->whereClause, deparseContext, false, false)); } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } /* * deparse_shard_reindex_statement uses the provided REINDEX node, dist. * relation, and shard identifier to populate a provided buffer with a string * representation of a shard-extended version of that command. */ void deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shardid, StringInfo buffer) { ReindexStmt *reindexStmt = copyObject(origStmt); /* copy to avoid modifications */ char *relationName = NULL; const char *concurrentlyString = IsReindexWithParam_compat(reindexStmt, "concurrently") ? "CONCURRENTLY " : ""; if (reindexStmt->kind == REINDEX_OBJECT_INDEX || reindexStmt->kind == REINDEX_OBJECT_TABLE) { /* extend relation and index name using shard identifier */ AppendShardIdToName(&(reindexStmt->relation->relname), shardid); relationName = reindexStmt->relation->relname; } appendStringInfoString(buffer, "REINDEX "); AddVacuumParams(reindexStmt, buffer); switch (reindexStmt->kind) { case REINDEX_OBJECT_INDEX: { appendStringInfo(buffer, "INDEX %s%s", concurrentlyString, quote_qualified_identifier(reindexStmt->relation->schemaname, relationName)); break; } case REINDEX_OBJECT_TABLE: { appendStringInfo(buffer, "TABLE %s%s", concurrentlyString, quote_qualified_identifier(reindexStmt->relation->schemaname, relationName)); break; } case REINDEX_OBJECT_SCHEMA: { appendStringInfo(buffer, "SCHEMA %s%s", concurrentlyString, quote_identifier(reindexStmt->name)); break; } case REINDEX_OBJECT_SYSTEM: { appendStringInfo(buffer, "SYSTEM %s%s", concurrentlyString, quote_identifier(reindexStmt->name)); break; } case REINDEX_OBJECT_DATABASE: { appendStringInfo(buffer, "DATABASE %s%s", concurrentlyString, quote_identifier(reindexStmt->name)); break; } } } /* * IsReindexWithParam_compat returns true if the given parameter * exists for the given reindexStmt. */ bool IsReindexWithParam_compat(ReindexStmt *reindexStmt, char *param) { DefElem *opt = NULL; foreach_declared_ptr(opt, reindexStmt->params) { if (strcmp(opt->defname, param) == 0) { return defGetBoolean(opt); } } return false; } /* * AddVacuumParams adds vacuum params to the given buffer. */ static void AddVacuumParams(ReindexStmt *reindexStmt, StringInfo buffer) { StringInfo temp = makeStringInfo(); if (IsReindexWithParam_compat(reindexStmt, "verbose")) { appendStringInfoString(temp, "VERBOSE"); } char *tableSpaceName = NULL; DefElem *opt = NULL; foreach_declared_ptr(opt, reindexStmt->params) { if (strcmp(opt->defname, "tablespace") == 0) { tableSpaceName = defGetString(opt); break; } } if (tableSpaceName) { if (temp->len > 0) { appendStringInfo(temp, ", TABLESPACE %s", tableSpaceName); } else { appendStringInfo(temp, "TABLESPACE %s", tableSpaceName); } } if (temp->len > 0) { appendStringInfo(buffer, "(%s) ", temp->data); } } /* deparse_index_columns appends index or include parameters to the provided buffer */ static void deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparseContext) { ListCell *indexParameterCell = NULL; foreach(indexParameterCell, indexParameterList) { IndexElem *indexElement = (IndexElem *) lfirst(indexParameterCell); /* use commas to separate subsequent elements */ if (indexParameterCell != list_head(indexParameterList)) { appendStringInfoChar(buffer, ','); } if (indexElement->name) { appendStringInfo(buffer, "%s ", quote_identifier(indexElement->name)); } else if (indexElement->expr) { appendStringInfo(buffer, "(%s)", deparse_expression(indexElement->expr, deparseContext, false, false)); } if (indexElement->collation != NIL) { appendStringInfo(buffer, "COLLATE %s ", NameListToQuotedString(indexElement->collation)); } if (indexElement->opclass != NIL) { appendStringInfo(buffer, "%s ", NameListToQuotedString(indexElement->opclass)); } /* Commit on postgres: 911e70207703799605f5a0e8aad9f06cff067c63*/ if (indexElement->opclassopts != NIL) { appendStringInfoString(buffer, "("); AppendStorageParametersToString(buffer, indexElement->opclassopts); appendStringInfoString(buffer, ") "); } if (indexElement->ordering != SORTBY_DEFAULT) { bool sortAsc = (indexElement->ordering == SORTBY_ASC); appendStringInfo(buffer, "%s ", (sortAsc ? "ASC" : "DESC")); } if (indexElement->nulls_ordering != SORTBY_NULLS_DEFAULT) { bool nullsFirst = (indexElement->nulls_ordering == SORTBY_NULLS_FIRST); appendStringInfo(buffer, "NULLS %s ", (nullsFirst ? "FIRST" : "LAST")); } } } /* * pg_get_indexclusterdef_string returns the definition of a cluster statement * for given index. The function returns null if the table is not clustered on * given index. */ char * pg_get_indexclusterdef_string(Oid indexRelationId) { StringInfoData buffer = { NULL, 0, 0, 0 }; HeapTuple indexTuple = SearchSysCache(INDEXRELID, ObjectIdGetDatum(indexRelationId), 0, 0, 0); if (!HeapTupleIsValid(indexTuple)) { ereport(ERROR, (errmsg("cache lookup failed for index %u", indexRelationId))); } Form_pg_index indexForm = (Form_pg_index) GETSTRUCT(indexTuple); Oid tableRelationId = indexForm->indrelid; /* check if the table is clustered on this index */ if (indexForm->indisclustered) { char *qualifiedRelationName = generate_qualified_relation_name(tableRelationId); char *indexName = get_rel_name(indexRelationId); /* needs to be quoted */ initStringInfo(&buffer); appendStringInfo(&buffer, "ALTER TABLE %s CLUSTER ON %s", qualifiedRelationName, quote_identifier(indexName)); } ReleaseSysCache(indexTuple); return (buffer.data); } /* * pg_get_table_grants returns a list of sql statements which recreate the * permissions for a specific table, including attributes privileges. * */ List * pg_get_table_grants(Oid relationId) { /* *INDENT-OFF* */ StringInfoData buffer; List *defs = NIL; bool isNull = false; Relation relation = relation_open(relationId, AccessShareLock); char *relationName = generate_relation_name(relationId, NIL); initStringInfo(&buffer); /* lookup all table level grants */ HeapTuple classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relationId)); if (!HeapTupleIsValid(classTuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation with OID %u does not exist", relationId))); } Form_pg_class classForm = (Form_pg_class) GETSTRUCT(classTuple); AttrNumber nattrs = classForm->relnatts; Datum aclDatum = SysCacheGetAttr(RELOID, classTuple, Anum_pg_class_relacl, &isNull); ReleaseSysCache(classTuple); if (!isNull) { /* * First revoke all default permissions, so we can start adding the * exact permissions from the master. Note that we only do so if there * are any actual grants; an empty grant set signals default * permissions. * * Note: This doesn't work correctly if default permissions have been * changed with ALTER DEFAULT PRIVILEGES - but that's hard to fix * properly currently. */ appendStringInfo(&buffer, "REVOKE ALL ON %s FROM PUBLIC", relationName); defs = lappend(defs, pstrdup(buffer.data)); resetStringInfo(&buffer); /* iterate through the acl datastructure, emit GRANTs */ Acl *acl = DatumGetAclP(aclDatum); process_acl_items(acl, relationName, NULL, &defs); /* if we have a detoasted copy, free it */ if ((Pointer) acl != DatumGetPointer(aclDatum)) pfree(acl); } resetStringInfo(&buffer); /* lookup all attribute level grants */ for (AttrNumber attNum = 1; attNum <= nattrs; attNum++) { HeapTuple attTuple = SearchSysCache2(ATTNUM, ObjectIdGetDatum(relationId), Int16GetDatum(attNum)); if (!HeapTupleIsValid(attTuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("attribute with OID %u does not exist", attNum))); } Form_pg_attribute thisAttribute = (Form_pg_attribute) GETSTRUCT(attTuple); /* ignore dropped columns */ if (thisAttribute->attisdropped) { ReleaseSysCache(attTuple); continue; } Datum aclAttDatum = SysCacheGetAttr(ATTNUM, attTuple, Anum_pg_attribute_attacl, &isNull); if (!isNull) { /* iterate through the acl datastructure, emit GRANTs */ Acl *acl = DatumGetAclP(aclAttDatum); process_acl_items(acl, relationName, NameStr(thisAttribute->attname), &defs); /* if we have a detoasted copy, free it */ if ((Pointer) acl != DatumGetPointer(aclAttDatum)) pfree(acl); } ReleaseSysCache(attTuple); } relation_close(relation, NoLock); return defs; } /* * Helper function to process ACL items. * If attributeName is NULL, the function emits table-level GRANT commands; * otherwise it emits column-level GRANT commands. * This function was modeled after aclexplode(), previously in pg_get_table_grants(). */ static void process_acl_items(Acl *acl, const char *relationName, const char *attributeName, List **defs) { AclItem *aidat = ACL_DAT(acl); int i = 0; int offtype = -1; StringInfoData buffer; initStringInfo(&buffer); while (i < ACL_NUM(acl)) { offtype++; if (offtype == N_ACL_RIGHTS) { offtype = 0; i++; if (i >= ACL_NUM(acl)) /* done */ { break; } } AclItem *aidata = &aidat[i]; AclMode priv_bit = 1 << offtype; if (ACLITEM_GET_PRIVS(*aidata) & priv_bit) { const char *roleName = NULL; const char *withGrant = ""; if (aidata->ai_grantee != 0) { roleName = quote_identifier(GetUserNameFromId(aidata->ai_grantee, false)); } else { roleName = "PUBLIC"; } if ((ACLITEM_GET_GOPTIONS(*aidata) & priv_bit) != 0) { withGrant = " WITH GRANT OPTION"; } if (attributeName) { appendStringInfo(&buffer, "GRANT %s(%s) ON %s TO %s%s", convert_aclright_to_string(priv_bit), quote_identifier(attributeName), relationName, roleName, withGrant); } else { appendStringInfo(&buffer, "GRANT %s ON %s TO %s%s", convert_aclright_to_string(priv_bit), relationName, roleName, withGrant); } *defs = lappend(*defs, pstrdup(buffer.data)); resetStringInfo(&buffer); } } } /* * generate_qualified_relation_name computes the schema-qualified name to display for a * relation specified by OID. */ char * generate_qualified_relation_name(Oid relid) { HeapTuple tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) { elog(ERROR, "cache lookup failed for relation %u", relid); } Form_pg_class reltup = (Form_pg_class) GETSTRUCT(tp); char *relname = NameStr(reltup->relname); char *nspname = get_namespace_name(reltup->relnamespace); if (!nspname) { elog(ERROR, "cache lookup failed for namespace %u", reltup->relnamespace); } char *result = quote_qualified_identifier(nspname, relname); ReleaseSysCache(tp); return result; } /* * AppendOptionListToString converts the option list to its textual format, and * appends this text to the given string buffer. */ void AppendOptionListToString(StringInfo stringBuffer, List *optionList) { if (optionList != NIL) { ListCell *optionCell = NULL; bool firstOptionPrinted = false; appendStringInfo(stringBuffer, " OPTIONS ("); foreach(optionCell, optionList) { DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; char *optionValue = defGetString(option); if (firstOptionPrinted) { appendStringInfo(stringBuffer, ", "); } firstOptionPrinted = true; appendStringInfo(stringBuffer, "%s ", quote_identifier(optionName)); appendStringInfo(stringBuffer, "%s", quote_literal_cstr(optionValue)); } appendStringInfo(stringBuffer, ")"); } } /* * AppendStorageParametersToString converts the storage parameter list to its * textual format, and appends this text to the given string buffer. */ static void AppendStorageParametersToString(StringInfo stringBuffer, List *optionList) { ListCell *optionCell = NULL; bool firstOptionPrinted = false; foreach(optionCell, optionList) { DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; char *optionValue = defGetString(option); if (firstOptionPrinted) { appendStringInfo(stringBuffer, ", "); } firstOptionPrinted = true; appendStringInfo(stringBuffer, "%s = %s ", quote_identifier(optionName), quote_literal_cstr(optionValue)); } } /* copy of postgresql's function, which is static as well */ static const char * convert_aclright_to_string(int aclright) { /* *INDENT-OFF* */ switch (aclright) { case ACL_INSERT: return "INSERT"; case ACL_SELECT: return "SELECT"; case ACL_UPDATE: return "UPDATE"; case ACL_DELETE: return "DELETE"; case ACL_TRUNCATE: return "TRUNCATE"; case ACL_REFERENCES: return "REFERENCES"; case ACL_TRIGGER: return "TRIGGER"; case ACL_EXECUTE: return "EXECUTE"; case ACL_USAGE: return "USAGE"; case ACL_CREATE: return "CREATE"; case ACL_CREATE_TEMP: return "TEMPORARY"; case ACL_CONNECT: return "CONNECT"; #if PG_VERSION_NUM >= PG_VERSION_17 case ACL_MAINTAIN: return "MAINTAIN"; #endif default: elog(ERROR, "unrecognized aclright: %d", aclright); return NULL; } /* *INDENT-ON* */ } /* * contain_nextval_expression_walker walks over expression tree and returns * true if it contains call to 'nextval' function or it has an identity column. */ bool contain_nextval_expression_walker(Node *node, void *context) { if (node == NULL) { return false; } /* check if the node contains an identity column */ if (IsA(node, NextValueExpr)) { return true; } /* check if the node contains call to 'nextval' */ if (IsA(node, FuncExpr)) { FuncExpr *funcExpr = (FuncExpr *) node; if (funcExpr->funcid == F_NEXTVAL) { return true; } } return expression_tree_walker(node, contain_nextval_expression_walker, context); } /* * pg_get_replica_identity_command function returns the required ALTER .. TABLE * command to define the replica identity. */ char * pg_get_replica_identity_command(Oid tableRelationId) { StringInfo buf = makeStringInfo(); Relation relation = table_open(tableRelationId, AccessShareLock); char replicaIdentity = relation->rd_rel->relreplident; char *relationName = generate_qualified_relation_name(tableRelationId); if (replicaIdentity == REPLICA_IDENTITY_INDEX) { Oid indexId = RelationGetReplicaIndex(relation); if (OidIsValid(indexId)) { appendStringInfo(buf, "ALTER TABLE %s REPLICA IDENTITY USING INDEX %s ", relationName, quote_identifier(get_rel_name(indexId))); } } else if (replicaIdentity == REPLICA_IDENTITY_NOTHING) { appendStringInfo(buf, "ALTER TABLE %s REPLICA IDENTITY NOTHING", relationName); } else if (replicaIdentity == REPLICA_IDENTITY_FULL) { appendStringInfo(buf, "ALTER TABLE %s REPLICA IDENTITY FULL", relationName); } table_close(relation, AccessShareLock); return (buf->len > 0) ? buf->data : NULL; } /* * pg_get_row_level_security_commands function returns the required ALTER .. TABLE * commands to define the row level security settings for a relation. */ List * pg_get_row_level_security_commands(Oid relationId) { StringInfoData buffer; List *commands = NIL; initStringInfo(&buffer); Relation relation = table_open(relationId, AccessShareLock); if (relation->rd_rel->relrowsecurity) { char *relationName = generate_qualified_relation_name(relationId); appendStringInfo(&buffer, "ALTER TABLE %s ENABLE ROW LEVEL SECURITY", relationName); commands = lappend(commands, pstrdup(buffer.data)); resetStringInfo(&buffer); } if (relation->rd_rel->relforcerowsecurity) { char *relationName = generate_qualified_relation_name(relationId); appendStringInfo(&buffer, "ALTER TABLE %s FORCE ROW LEVEL SECURITY", relationName); commands = lappend(commands, pstrdup(buffer.data)); resetStringInfo(&buffer); } table_close(relation, AccessShareLock); return commands; } /* * Generate a C string representing a relation's reloptions, or NULL if none. * * This function comes from PostgreSQL source code in * src/backend/utils/adt/ruleutils.c */ char * flatten_reloptions(Oid relid) { char *result = NULL; bool isnull; HeapTuple tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "cache lookup failed for relation %u", relid); } Datum reloptions = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, &isnull); if (!isnull) { StringInfoData buf; Datum *options; int noptions; int i; initStringInfo(&buf); deconstruct_array(DatumGetArrayTypeP(reloptions), TEXTOID, -1, false, 'i', &options, NULL, &noptions); for (i = 0; i < noptions; i++) { char *option = TextDatumGetCString(options[i]); char *value; /* * Each array element should have the form name=value. If the "=" * is missing for some reason, treat it like an empty value. */ char *name = option; char *separator = strchr(option, '='); if (separator) { *separator = '\0'; value = separator + 1; } else { value = ""; } if (i > 0) { appendStringInfoString(&buf, ", "); } appendStringInfo(&buf, "%s=", quote_identifier(name)); /* * In general we need to quote the value; but to avoid unnecessary * clutter, do not quote if it is an identifier that would not * need quoting. (We could also allow numbers, but that is a bit * trickier than it looks --- for example, are leading zeroes * significant? We don't want to assume very much here about what * custom reloptions might mean.) */ if (quote_identifier(value) == value) { appendStringInfoString(&buf, value); } else { simple_quote_literal(&buf, value); } pfree(option); } result = buf.data; } ReleaseSysCache(tuple); return result; } /* * simple_quote_literal - Format a string as a SQL literal, append to buf * * This function comes from PostgreSQL source code in * src/backend/utils/adt/ruleutils.c */ static void simple_quote_literal(StringInfo buf, const char *val) { /* * We form the string literal according to the prevailing setting of * standard_conforming_strings; we never use E''. User is responsible for * making sure result is used correctly. */ appendStringInfoChar(buf, '\''); for (const char *valptr = val; *valptr; valptr++) { char ch = *valptr; if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) { appendStringInfoChar(buf, ch); } appendStringInfoChar(buf, ch); } appendStringInfoChar(buf, '\''); } /* * RoleSpecString resolves the role specification to its string form that is suitable for transport to a worker node. * This function resolves the following identifiers from the current context so they are safe to transfer. * * CURRENT_USER - resolved to the user name of the current role being used * SESSION_USER - resolved to the user name of the user that opened the session * CURRENT_ROLE - same as CURRENT_USER, resolved to the user name of the current role being used * Postgres treats CURRENT_ROLE is equivalent to CURRENT_USER, and we follow the same approach. * * withQuoteIdentifier is used, because if the results will be used in a query the quotes are needed but if not there * should not be extra quotes. */ const char * RoleSpecString(RoleSpec *spec, bool withQuoteIdentifier) { switch (spec->roletype) { case ROLESPEC_CSTRING: { return withQuoteIdentifier ? quote_identifier(spec->rolename) : spec->rolename; } case ROLESPEC_CURRENT_ROLE: case ROLESPEC_CURRENT_USER: { return withQuoteIdentifier ? quote_identifier(GetUserNameFromId(GetUserId(), false)) : GetUserNameFromId(GetUserId(), false); } case ROLESPEC_SESSION_USER: { return withQuoteIdentifier ? quote_identifier(GetUserNameFromId(GetSessionUserId(), false)) : GetUserNameFromId(GetSessionUserId(), false); } case ROLESPEC_PUBLIC: { return "PUBLIC"; } default: { elog(ERROR, "unexpected role type %d", spec->roletype); } } } /* * Recursively search an expression for a Param and return its paramid * Intended for indirection management: UPDATE SET () = (SELECT ) * Does not cover all options but those supported by Citus. */ static int GetParamId(Node *expr) { int paramid = 0; if (expr == NULL) { return paramid; } /* If it's a Param, return its attnum */ if (IsA(expr, Param)) { Param *param = (Param *) expr; paramid = param->paramid; } /* If it's a FuncExpr, search in arguments */ else if (IsA(expr, FuncExpr)) { FuncExpr *func = (FuncExpr *) expr; ListCell *lc; foreach(lc, func->args) { paramid = GetParamId((Node *) lfirst(lc)); if (paramid != 0) { break; /* Stop at the first valid paramid */ } } } return paramid; } /* * list_sort comparator to sort target list by paramid (in MULTIEXPR) * Intended for indirection management: UPDATE SET () = (SELECT ) */ static int target_list_cmp(const ListCell *a, const ListCell *b) { TargetEntry *tleA = lfirst(a); TargetEntry *tleB = lfirst(b); /* * Deal with resjunk entries; sublinks are marked resjunk and * are placed at the end of the target list so this logic * ensures they stay grouped at the end of the target list: */ if (tleA->resjunk || tleB->resjunk) { return tleA->resjunk - tleB->resjunk; } int la = GetParamId((Node *) tleA->expr); int lb = GetParamId((Node *) tleB->expr); /* * Should be looking at legitimate param ids */ Assert(la > 0); Assert(lb > 0); /* * Return -1, 0 or 1 depending on if la is less than, * equal to or greater than lb */ return (la > lb) - (la < lb); } /* * Used by get_update_query_targetlist_def() (in ruleutils) to reorder the target * list on the left side of the update: * SET () = (SELECT ) * Reordering the SELECT side only does not work, consider a case like: * SET (col_1, col3) = (SELECT 1, 3), (col_2) = (SELECT 2) * Without ensure_update_targetlist_in_param_order(), this will lead to an incorrect * deparsed query: * SET (col_1, col2) = (SELECT 1, 3), (col_3) = (SELECT 2) */ void ensure_update_targetlist_in_param_order(List *targetList) { bool need_to_sort_target_list = false; int previous_paramid = 0; ListCell *l; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (!tle->resjunk) { int paramid = GetParamId((Node *) tle->expr); if (paramid < previous_paramid) { need_to_sort_target_list = true; break; } previous_paramid = paramid; } } if (need_to_sort_target_list) { list_sort(targetList, target_list_cmp); } } /* * isSubsRef checks if a given node is a SubscriptingRef or can be * reached through an implicit coercion. */ static bool isSubsRef(Node *node) { if (node == NULL) { return false; } if (IsA(node, CoerceToDomain)) { CoerceToDomain *coerceToDomain = (CoerceToDomain *) node; if (coerceToDomain->coercionformat != COERCE_IMPLICIT_CAST) { /* not an implicit coercion, cannot reach to a SubscriptingRef */ return false; } node = (Node *) coerceToDomain->arg; } return (IsA(node, SubscriptingRef)); } /* * checkTlistForSubsRef - checks if any target entry in the list contains a * SubscriptingRef or can be reached through an implicit coercion. Used by * ExpandMergedSubscriptingRefEntries() to identify if any target entries * need to be expanded - if not the original target list is preserved. */ static bool checkTlistForSubsRef(List *targetEntryList) { ListCell *tgtCell = NULL; foreach(tgtCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(tgtCell); Expr *expr = targetEntry->expr; if (isSubsRef((Node *) expr)) { return true; } } return false; } /* * ExpandMergedSubscriptingRefEntries takes a list of target entries and expands * each one that references a SubscriptingRef node that indicates multiple (field) * updates on the same attribute, which is applicable for array/json types atm. */ List * ExpandMergedSubscriptingRefEntries(List *targetEntryList) { List *newTargetEntryList = NIL; ListCell *tgtCell = NULL; if (!checkTlistForSubsRef(targetEntryList)) { /* No subscripting refs found, return original list */ return targetEntryList; } foreach(tgtCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(tgtCell); List *expandedTargetEntries = NIL; Expr *expr = targetEntry->expr; while (expr) { SubscriptingRef *subsRef = TargetEntryExprFindSubsRef(expr); if (!subsRef) { break; } /* * Remove refexpr from the SubscriptingRef that we are about to * wrap in a new TargetEntry and save it for the next one. */ Expr *refexpr = subsRef->refexpr; subsRef->refexpr = NULL; /* * Wrap the Expr that holds SubscriptingRef (directly or indirectly) * in a new TargetEntry; note that it doesn't have a refexpr anymore. */ TargetEntry *newTargetEntry = copyObject(targetEntry); newTargetEntry->expr = expr; expandedTargetEntries = lappend(expandedTargetEntries, newTargetEntry); /* now inspect the refexpr that SubscriptingRef at hand were holding */ expr = refexpr; } if (expandedTargetEntries == NIL) { /* return original entry since it doesn't hold a SubscriptingRef node */ newTargetEntryList = lappend(newTargetEntryList, targetEntry); } else { /* * Need to concat expanded target list entries in reverse order * to preserve ordering of the original target entry list. */ List *reversedTgtEntries = NIL; ListCell *revCell = NULL; foreach(revCell, expandedTargetEntries) { TargetEntry *tgtEntry = (TargetEntry *) lfirst(revCell); reversedTgtEntries = lcons(tgtEntry, reversedTgtEntries); } newTargetEntryList = list_concat(newTargetEntryList, reversedTgtEntries); } } return newTargetEntryList; } /* * TargetEntryExprFindSubsRef searches given Expr --assuming that it is part * of a target list entry-- to see if it directly (i.e.: itself) or indirectly * (e.g.: behind some level of coercions) holds a SubscriptingRef node. * * Returns the original SubscriptingRef node on success or NULL otherwise. * * Note that it wouldn't add much value to use expression_tree_walker here * since we are only interested in a subset of the fields of a few certain * node types. */ static SubscriptingRef * TargetEntryExprFindSubsRef(Expr *expr) { Node *node = (Node *) expr; while (node) { if (IsA(node, FieldStore)) { /* * ModifyPartialQuerySupported doesn't allow INSERT/UPDATE via * FieldStore. If we decide supporting such commands, then we * should take the first element of "newvals" list into account * here. This is because, to support such commands, we will need * to expand merged FieldStore into separate target entries too. * * For this reason, this block is not reachable atm and need to * uncomment the following if we decide supporting such commands. * * """ * FieldStore *fieldStore = (FieldStore *) node; * node = (Node *) linitial(fieldStore->newvals); * """ */ ereport(ERROR, (errmsg("unexpectedly got FieldStore object when " "generating shard query"))); } else if (IsA(node, CoerceToDomain)) { CoerceToDomain *coerceToDomain = (CoerceToDomain *) node; if (coerceToDomain->coercionformat != COERCE_IMPLICIT_CAST) { /* not an implicit coercion, cannot reach to a SubscriptingRef */ break; } node = (Node *) coerceToDomain->arg; } else if (IsA(node, SubscriptingRef)) { return (SubscriptingRef *) node; } else { /* got a node that we are not interested in */ break; } } return NULL; } ================================================ FILE: src/backend/distributed/deparser/citus_setutils.c ================================================ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/print.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/log_utils.h" void AppendVarSetValue(StringInfo buf, VariableSetStmt *setStmt); /* * AppendVarSetValueDb deparses a VariableSetStmt with VAR_SET_VALUE kind. * It takes from flatten_set_variable_args in postgres's utils/misc/guc.c, * however flatten_set_variable_args does not apply correct quoting. */ void AppendVarSetValue(StringInfo buf, VariableSetStmt *setStmt) { ListCell *varArgCell = NULL; ListCell *firstCell = list_head(setStmt->args); Assert(setStmt->kind == VAR_SET_VALUE); foreach(varArgCell, setStmt->args) { Node *varArgNode = lfirst(varArgCell); A_Const *varArgConst = NULL; TypeName *typeName = NULL; if (IsA(varArgNode, A_Const)) { varArgConst = (A_Const *) varArgNode; } else if (IsA(varArgNode, TypeCast)) { TypeCast *varArgTypeCast = (TypeCast *) varArgNode; varArgConst = castNode(A_Const, varArgTypeCast->arg); typeName = varArgTypeCast->typeName; } else { elog(ERROR, "unrecognized node type: %d", varArgNode->type); } /* don't know how to start SET until we inspect first arg */ if (varArgCell != firstCell) { appendStringInfoChar(buf, ','); } else if (typeName != NULL) { appendStringInfoString(buf, " SET TIME ZONE"); } else { appendStringInfo(buf, " SET %s =", quote_identifier(setStmt->name)); } Node *value = (Node *) &varArgConst->val; switch (value->type) { case T_Integer: { appendStringInfo(buf, " %d", intVal(value)); break; } case T_Float: { appendStringInfo(buf, " %s", nodeToString(value)); break; } case T_String: { if (typeName != NULL) { /* * Must be a ConstInterval argument for TIME ZONE. Coerce * to interval and back to normalize the value and account * for any typmod. */ Oid typoid = InvalidOid; int32 typmod = -1; typenameTypeIdAndMod(NULL, typeName, &typoid, &typmod); Assert(typoid == INTERVALOID); Datum interval = DirectFunctionCall3(interval_in, CStringGetDatum(strVal(value)), ObjectIdGetDatum(InvalidOid), Int32GetDatum(typmod)); char *intervalout = DatumGetCString(DirectFunctionCall1(interval_out, interval)); appendStringInfo(buf, " INTERVAL '%s'", intervalout); } else { appendStringInfo(buf, " %s", quote_literal_cstr(strVal(value))); } break; } default: { elog(ERROR, "Unexpected Value type in VAR_SET_VALUE arguments."); break; } } } } /* * AppendVariableSetDb appends a string representing the VariableSetStmt to a buffer */ void AppendVariableSet(StringInfo buf, VariableSetStmt *setStmt) { switch (setStmt->kind) { case VAR_SET_VALUE: { AppendVarSetValue(buf, setStmt); break; } case VAR_SET_CURRENT: { appendStringInfo(buf, " SET %s FROM CURRENT", quote_identifier( setStmt->name)); break; } case VAR_SET_DEFAULT: { appendStringInfo(buf, " SET %s TO DEFAULT", quote_identifier(setStmt->name)); break; } case VAR_RESET: { appendStringInfo(buf, " RESET %s", quote_identifier(setStmt->name)); break; } case VAR_RESET_ALL: { appendStringInfoString(buf, " RESET ALL"); break; } /* VAR_SET_MULTI is a special case for SET TRANSACTION that should not occur here */ case VAR_SET_MULTI: default: { ereport(ERROR, (errmsg("Unable to deparse SET statement"))); break; } } } ================================================ FILE: src/backend/distributed/deparser/deparse.c ================================================ /*------------------------------------------------------------------------- * * deparse.c * Entrypoint for deparsing parsetrees. * * The goal of deparsing parsetrees is to reconstruct sql statements * from any parsed sql statement by ParseTreeNode. Deparsed statements * can be used to reapply them on remote postgres nodes like the citus * workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" /* * DeparseTreeNode aims to be the inverse of postgres' ParseTreeNode. Currently with * limited support. Check support before using, and add support for new statements as * required. */ char * DeparseTreeNode(Node *stmt) { const DistributeObjectOps *ops = GetDistributeObjectOps(stmt); if (!ops->deparse) { ereport(ERROR, (errmsg("unsupported statement for deparsing"))); } return ops->deparse(stmt); } /* * DeparseTreeNodes deparses all stmts in the list from the statement datastructure into * sql statements. */ List * DeparseTreeNodes(List *stmts) { List *sqls = NIL; Node *stmt = NULL; foreach_declared_ptr(stmt, stmts) { sqls = lappend(sqls, DeparseTreeNode(stmt)); } return sqls; } ================================================ FILE: src/backend/distributed/deparser/deparse_attribute_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_attribute_stmts.c * All routines to deparse attribute statements. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/deparser.h" char * DeparseRenameAttributeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); switch (stmt->relationType) { case OBJECT_TYPE: { return DeparseRenameTypeAttributeStmt(node); } default: { ereport(ERROR, (errmsg("unsupported rename attribute statement for" " deparsing"))); } } } ================================================ FILE: src/backend/distributed/deparser/deparse_collation_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_collation_stmts.c * All routines to deparse collation statements. * This file contains all entry points specific for type statement deparsing as well as * functions that are currently only used for deparsing of the type statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/value.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" static void AppendDropCollationStmt(StringInfo buf, DropStmt *stmt); static void AppendRenameCollationStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterCollationSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterCollationOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendNameList(StringInfo buf, List *objects); /* * DeparseDropCollationStmt builds and returns a string representing the DropStmt */ char * DeparseDropCollationStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->removeType == OBJECT_COLLATION); AppendDropCollationStmt(&str, stmt); return str.data; } /* * AppendDropCollationStmt appends a string representing the DropStmt to a buffer */ static void AppendDropCollationStmt(StringInfo buf, DropStmt *stmt) { appendStringInfoString(buf, "DROP COLLATION "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendNameList(buf, stmt->objects); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } /* * DeparseRenameCollationStmt builds and returns a string representing the RenameStmt */ char * DeparseRenameCollationStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_COLLATION); AppendRenameCollationStmt(&str, stmt); return str.data; } /* * AppendRenameCollationStmt appends a string representing the RenameStmt to a buffer */ static void AppendRenameCollationStmt(StringInfo buf, RenameStmt *stmt) { List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER COLLATION %s RENAME TO %s;", NameListToQuotedString(names), quote_identifier(stmt->newname)); } /* * DeparseAlterCollationSchemaStmt builds and returns a string representing the AlterObjectSchemaStmt */ char * DeparseAlterCollationSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_COLLATION); AppendAlterCollationSchemaStmt(&str, stmt); return str.data; } /* * AppendAlterCollationSchemaStmt appends a string representing the AlterObjectSchemaStmt to a buffer */ static void AppendAlterCollationSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { Assert(stmt->objectType == OBJECT_COLLATION); List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER COLLATION %s SET SCHEMA %s;", NameListToQuotedString( names), quote_identifier(stmt->newschema)); } /* * DeparseAlterCollationOwnerStmt builds and returns a string representing the AlterOwnerStmt */ char * DeparseAlterCollationOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_COLLATION); AppendAlterCollationOwnerStmt(&str, stmt); return str.data; } /* * AppendAlterCollationOwnerStmt appends a string representing the AlterOwnerStmt to a buffer */ static void AppendAlterCollationOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { Assert(stmt->objectType == OBJECT_COLLATION); List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER COLLATION %s OWNER TO %s;", NameListToQuotedString( names), RoleSpecString(stmt->newowner, true)); } static void AppendNameList(StringInfo buf, List *objects) { ListCell *objectCell = NULL; foreach(objectCell, objects) { List *name = castNode(List, lfirst(objectCell)); if (objectCell != list_head(objects)) { appendStringInfo(buf, ", "); } appendStringInfoString(buf, NameListToQuotedString(name)); } } ================================================ FILE: src/backend/distributed/deparser/deparse_comment_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_coment_stmts.c * * All routines to deparse comment statements. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/elog.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/comment.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" const char *ObjectTypeNames[] = { [OBJECT_DATABASE] = "DATABASE", [OBJECT_ROLE] = "ROLE", [OBJECT_TSCONFIGURATION] = "TEXT SEARCH CONFIGURATION", [OBJECT_TSDICTIONARY] = "TEXT SEARCH DICTIONARY", /* When support for propagating comments to new objects is introduced, an entry for each * statement type should be added to this list. The first element in each entry is the 'object_type' keyword * that will be included in the 'COMMENT ON ..' statement (i.e. DATABASE,). The second element is the type of * stmt->object, which represents the name of the propagated object. */ }; char * DeparseCommentStmt(Node *node) { CommentStmt *stmt = castNode(CommentStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); const char *objectName = NULL; if (IsA(stmt->object, String)) { objectName = quote_identifier(strVal(stmt->object)); } else if (IsA(stmt->object, List)) { objectName = NameListToQuotedString(castNode(List, stmt->object)); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unknown object type"))); } const char *objectType = ObjectTypeNames[stmt->objtype]; char *comment = stmt->comment != NULL ? quote_literal_cstr(stmt->comment) : "NULL"; appendStringInfo(&str, "COMMENT ON %s %s IS %s;", objectType, objectName, comment); return str.data; } ================================================ FILE: src/backend/distributed/deparser/deparse_database_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_database_stmts.c * * All routines to deparse database statements. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendAlterDatabaseSetStmt(StringInfo buf, AlterDatabaseSetStmt *stmt); static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt); static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt); static void AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt); static void AppendBasicAlterDatabaseOptions(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendGrantDatabases(StringInfo buf, GrantStmt *stmt); static void AppendAlterDatabaseSetTablespace(StringInfo buf, DefElem *def, char *dbname); const DefElemOptionFormat createDatabaseOptionFormats[] = { { "owner", " OWNER %s", OPTION_FORMAT_STRING }, { "template", " TEMPLATE %s", OPTION_FORMAT_STRING }, { "encoding", " ENCODING %s", OPTION_FORMAT_LITERAL_CSTR }, { "strategy", " STRATEGY %s", OPTION_FORMAT_LITERAL_CSTR }, { "locale", " LOCALE %s", OPTION_FORMAT_LITERAL_CSTR }, { "lc_collate", " LC_COLLATE %s", OPTION_FORMAT_LITERAL_CSTR }, { "lc_ctype", " LC_CTYPE %s", OPTION_FORMAT_LITERAL_CSTR }, { "icu_locale", " ICU_LOCALE %s", OPTION_FORMAT_LITERAL_CSTR }, { "icu_rules", " ICU_RULES %s", OPTION_FORMAT_LITERAL_CSTR }, { "locale_provider", " LOCALE_PROVIDER %s", OPTION_FORMAT_LITERAL_CSTR }, { "collation_version", " COLLATION_VERSION %s", OPTION_FORMAT_LITERAL_CSTR }, { "tablespace", " TABLESPACE %s", OPTION_FORMAT_STRING }, { "allow_connections", " ALLOW_CONNECTIONS %s", OPTION_FORMAT_BOOLEAN }, { "connection_limit", " CONNECTION LIMIT %d", OPTION_FORMAT_INTEGER }, { "is_template", " IS_TEMPLATE %s", OPTION_FORMAT_BOOLEAN } }; const DefElemOptionFormat alterDatabaseOptionFormats[] = { { "is_template", " IS_TEMPLATE %s", OPTION_FORMAT_BOOLEAN }, { "allow_connections", " ALLOW_CONNECTIONS %s", OPTION_FORMAT_BOOLEAN }, { "connection_limit", " CONNECTION LIMIT %d", OPTION_FORMAT_INTEGER }, }; char * DeparseAlterDatabaseOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_DATABASE); AppendAlterDatabaseOwnerStmt(&str, stmt); return str.data; } static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { Assert(stmt->objectType == OBJECT_DATABASE); appendStringInfo(buf, "ALTER DATABASE %s OWNER TO %s;", quote_identifier(strVal((String *) stmt->object)), RoleSpecString(stmt->newowner, true)); } static void AppendGrantDatabases(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON DATABASE "); foreach(cell, stmt->objects) { char *database = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(database)); if (cell != list_tail(stmt->objects)) { appendStringInfo(buf, ", "); } } } static void AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_DATABASE); AppendGrantSharedPrefix(buf, stmt); AppendGrantDatabases(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) { if (list_length(stmt->options) == 0) { elog(ERROR, "got unexpected number of options for ALTER DATABASE"); } if (stmt->options) { DefElem *firstOption = linitial(stmt->options); if (strcmp(firstOption->defname, "tablespace") == 0) { AppendAlterDatabaseSetTablespace(buf, firstOption, stmt->dbname); /* SET tablespace cannot be combined with other options */ return; } appendStringInfo(buf, "ALTER DATABASE %s WITH", quote_identifier(stmt->dbname)); AppendBasicAlterDatabaseOptions(buf, stmt); } appendStringInfo(buf, ";"); } static void AppendAlterDatabaseSetTablespace(StringInfo buf, DefElem *def, char *dbname) { appendStringInfo(buf, "ALTER DATABASE %s SET TABLESPACE %s", quote_identifier(dbname), quote_identifier(defGetString(def))); } /* * AppendBasicAlterDatabaseOptions appends basic ALTER DATABASE options to a string buffer. * Basic options are those that can be appended to the ALTER DATABASE statement * after the "WITH" keyword.(i.e. ALLOW_CONNECTIONS, CONNECTION LIMIT, IS_TEMPLATE) * For example, the tablespace option is not a basic option since it is defined via SET keyword. * * This function takes a string buffer and an AlterDatabaseStmt as input. * It appends the basic options to the string buffer. * */ static void AppendBasicAlterDatabaseOptions(StringInfo buf, AlterDatabaseStmt *stmt) { DefElem *def = NULL; foreach_declared_ptr(def, stmt->options) { DefElemOptionToStatement(buf, def, alterDatabaseOptionFormats, lengthof( alterDatabaseOptionFormats)); } } char * DeparseGrantOnDatabaseStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_DATABASE); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnDatabaseStmt(&str, stmt); return str.data; } char * DeparseAlterDatabaseStmt(Node *node) { AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterDatabaseStmt(&str, stmt); return str.data; } char * DeparseAlterDatabaseRefreshCollStmt(Node *node) { AlterDatabaseRefreshCollStmt *stmt = (AlterDatabaseRefreshCollStmt *) node; StringInfoData str; initStringInfo(&str); appendStringInfo(&str, "ALTER DATABASE %s REFRESH COLLATION VERSION;", quote_identifier( stmt->dbname)); return str.data; } static void AppendAlterDatabaseSetStmt(StringInfo buf, AlterDatabaseSetStmt *stmt) { appendStringInfo(buf, "ALTER DATABASE %s", quote_identifier(stmt->dbname)); VariableSetStmt *varSetStmt = castNode(VariableSetStmt, stmt->setstmt); AppendVariableSet(buf, varSetStmt); } char * DeparseAlterDatabaseRenameStmt(Node *node) { RenameStmt *stmt = (RenameStmt *) node; StringInfoData str; initStringInfo(&str); appendStringInfo(&str, "ALTER DATABASE %s RENAME TO %s", quote_identifier(stmt->subname), quote_identifier(stmt->newname)); return str.data; } char * DeparseAlterDatabaseSetStmt(Node *node) { AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterDatabaseSetStmt(&str, stmt); return str.data; } static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { /* * Make sure that we don't try to deparse something that this * function doesn't expect. * * This is also useful to throw an error for unsupported CREATE * DATABASE options when the command is issued from non-main dbs * because we use the same function to deparse CREATE DATABASE * commands there too. */ EnsureSupportedCreateDatabaseCommand(stmt); appendStringInfo(buf, "CREATE DATABASE %s", quote_identifier(stmt->dbname)); DefElem *option = NULL; foreach_declared_ptr(option, stmt->options) { DefElemOptionToStatement(buf, option, createDatabaseOptionFormats, lengthof(createDatabaseOptionFormats)); } } char * DeparseCreateDatabaseStmt(Node *node) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendCreateDatabaseStmt(&str, stmt); return str.data; } static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { char *ifExistsStatement = stmt->missing_ok ? "IF EXISTS" : ""; appendStringInfo(buf, "DROP DATABASE %s %s", ifExistsStatement, quote_identifier(stmt->dbname)); if (list_length(stmt->options) > 1) { /* FORCE is the only option that can be provided for this command */ elog(ERROR, "got unexpected number of options for DROP DATABASE"); } else if (list_length(stmt->options) == 1) { DefElem *option = linitial(stmt->options); appendStringInfo(buf, " WITH ( "); if (strcmp(option->defname, "force") == 0) { appendStringInfo(buf, "FORCE"); } else { /* FORCE is the only option that can be provided for this command */ ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized DROP DATABASE option \"%s\"", option->defname))); } appendStringInfo(buf, " )"); } } char * DeparseDropDatabaseStmt(Node *node) { DropdbStmt *stmt = castNode(DropdbStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendDropDatabaseStmt(&str, stmt); return str.data; } ================================================ FILE: src/backend/distributed/deparser/deparse_domain_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_domain_stmts.c * Functions to turn all Statement structures related to domains back * into sql. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/heap.h" #include "catalog/namespace.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "parser/parse_coerce.h" #include "parser/parse_collate.h" #include "parser/parse_expr.h" #include "parser/parse_node.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/namespace_utils.h" static void AppendConstraint(StringInfo buf, Constraint *constraint, List *domainName, TypeName *typeName); static Node * replace_domain_constraint_value(ParseState *pstate, ColumnRef *cref); static Node * TransformDefaultExpr(Node *expr, List *domainName, TypeName *typeName); static Node * TransformConstraintExpr(Node *expr, TypeName *typeName); static CoerceToDomainValue * GetCoerceDomainValue(TypeName *typeName); static char * TypeNameAsIdentifier(TypeName *typeName); static Oid DomainGetBaseTypeOid(List *names, int32 *baseTypeMod); static void AppendAlterDomainStmtSetDefault(StringInfo buf, AlterDomainStmt *stmt); static void AppendAlterDomainStmtAddConstraint(StringInfo buf, AlterDomainStmt *stmt); static void AppendAlterDomainStmtDropConstraint(StringInfo buf, AlterDomainStmt *stmt); /* * DeparseCreateDomainStmt returns the sql representation for the CREATE DOMAIN statement. */ char * DeparseCreateDomainStmt(Node *node) { CreateDomainStmt *stmt = castNode(CreateDomainStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); const char *domainIdentifier = NameListToQuotedString(stmt->domainname); const char *typeIdentifier = TypeNameAsIdentifier(stmt->typeName); appendStringInfo(&buf, "CREATE DOMAIN %s AS %s", domainIdentifier, typeIdentifier); if (stmt->collClause) { const char *collateIdentifier = NameListToQuotedString(stmt->collClause->collname); appendStringInfo(&buf, " COLLATE %s", collateIdentifier); } Constraint *constraint = NULL; foreach_declared_ptr(constraint, stmt->constraints) { AppendConstraint(&buf, constraint, stmt->domainname, stmt->typeName); } appendStringInfoString(&buf, ";"); return buf.data; } /* * TypeNameAsIdentifier returns the sql identifier of a TypeName. This is more complex * than concatenating the schema name and typename since certain types contain modifiers * that need to be correctly represented. */ static char * TypeNameAsIdentifier(TypeName *typeName) { int32 typmod = 0; Oid typeOid = InvalidOid; bits16 formatFlags = FORMAT_TYPE_TYPEMOD_GIVEN | FORMAT_TYPE_FORCE_QUALIFY; typenameTypeIdAndMod(NULL, typeName, &typeOid, &typmod); return format_type_extended(typeOid, typmod, formatFlags); } /* * DeparseDropDomainStmt returns the sql for teh DROP DOMAIN statement. */ char * DeparseDropDomainStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfoString(&buf, "DROP DOMAIN "); if (stmt->missing_ok) { appendStringInfoString(&buf, "IF EXISTS "); } TypeName *domainName = NULL; bool first = true; foreach_declared_ptr(domainName, stmt->objects) { if (!first) { appendStringInfoString(&buf, ", "); } first = false; const char *identifier = NameListToQuotedString(domainName->names); appendStringInfoString(&buf, identifier); } if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(&buf, " CASCADE"); } appendStringInfoString(&buf, ";"); return buf.data; } /* * DeparseAlterDomainStmt returns the sql representation of the DOMAIN specific ALTER * statements. */ char * DeparseAlterDomainStmt(Node *node) { AlterDomainStmt *stmt = castNode(AlterDomainStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER DOMAIN %s ", NameListToQuotedString(stmt->typeName)); switch (stmt->subtype) { case 'T': /* SET DEFAULT */ { AppendAlterDomainStmtSetDefault(&buf, stmt); break; } case 'N': /* DROP NOT NULL */ { appendStringInfoString(&buf, "DROP NOT NULL"); break; } case 'O': /* SET NOT NULL */ { appendStringInfoString(&buf, "SET NOT NULL"); break; } case 'C': /* ADD [CONSTRAINT name] */ { AppendAlterDomainStmtAddConstraint(&buf, stmt); break; } case 'X': /* DROP CONSTRAINT */ { AppendAlterDomainStmtDropConstraint(&buf, stmt); break; } case 'V': /* VALIDATE CONSTRAINT */ { appendStringInfo(&buf, "VALIDATE CONSTRAINT %s", quote_identifier(stmt->name)); break; } default: { elog(ERROR, "unsupported alter domain statement for distribution"); } } appendStringInfoChar(&buf, ';'); return buf.data; } /* * DeparseDomainRenameConstraintStmt returns the sql representation of the domain * constraint renaming. */ char * DeparseDomainRenameConstraintStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); char *domainIdentifier = NameListToQuotedString(castNode(List, stmt->object)); appendStringInfo(&buf, "ALTER DOMAIN %s RENAME CONSTRAINT %s TO %s;", domainIdentifier, quote_identifier(stmt->subname), quote_identifier(stmt->newname)); return buf.data; } /* * DeparseAlterDomainOwnerStmt returns the sql representation of the ALTER DOMAIN OWNER * statement. */ char * DeparseAlterDomainOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); List *domainName = castNode(List, stmt->object); char *domainIdentifier = NameListToQuotedString(domainName); appendStringInfo(&buf, "ALTER DOMAIN %s OWNER TO %s;", domainIdentifier, RoleSpecString(stmt->newowner, true)); return buf.data; } /* * DeparseRenameDomainStmt returns the sql representation of the ALTER DOMAIN RENAME * statement. */ char * DeparseRenameDomainStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); List *domainName = castNode(List, stmt->object); char *domainIdentifier = NameListToQuotedString(domainName); appendStringInfo(&buf, "ALTER DOMAIN %s RENAME TO %s;", domainIdentifier, quote_identifier(stmt->newname)); return buf.data; } /* * DeparseAlterDomainSchemaStmt returns the sql representation of the ALTER DOMAIN SET * SCHEMA statement. */ char * DeparseAlterDomainSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); List *domainName = castNode(List, stmt->object); char *domainIdentifier = NameListToQuotedString(domainName); appendStringInfo(&buf, "ALTER DOMAIN %s SET SCHEMA %s;", domainIdentifier, quote_identifier(stmt->newschema)); return buf.data; } /* * DomainGetBaseTypeOid returns the type Oid and the type modifiers of the type underlying * a domain addresses by the namelist provided as the names argument. The type modifier is * only provided if the baseTypeMod pointer is a valid pointer on where to write the * modifier (not a NULL pointer). * * If the type cannot be found this function will raise a non-userfacing error. Care needs * to be taken by the caller that the domain is actually existing. */ static Oid DomainGetBaseTypeOid(List *names, int32 *baseTypeMod) { TypeName *domainName = makeTypeNameFromNameList(names); Oid domainoid = typenameTypeId(NULL, domainName); HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(domainoid)); if (!HeapTupleIsValid(tup)) { elog(ERROR, "cache lookup failed for type %u", domainoid); } Form_pg_type typTup = (Form_pg_type) GETSTRUCT(tup); Oid baseTypeOid = typTup->typbasetype; if (baseTypeMod) { *baseTypeMod = typTup->typtypmod; } ReleaseSysCache(tup); return baseTypeOid; } /* * AppendAlterDomainStmtSetDefault is a helper function that appends the default value * portion of an ALTER DOMAIN statement that is changing the default value of the domain. */ static void AppendAlterDomainStmtSetDefault(StringInfo buf, AlterDomainStmt *stmt) { if (stmt->def == NULL) { /* no default expression is a DROP DEFAULT statment */ appendStringInfoString(buf, "DROP DEFAULT"); return; } int32 baseTypMod = 0; Oid baseOid = DomainGetBaseTypeOid(stmt->typeName, &baseTypMod); TypeName *baseTypeName = makeTypeNameFromOid(baseOid, baseTypMod); /* cook the default expression, without cooking we can't deparse */ Node *expr = stmt->def; expr = TransformDefaultExpr(expr, stmt->typeName, baseTypeName); /* deparse while the searchpath is cleared to force qualification of identifiers */ int saveNestLevel = PushEmptySearchPath(); char *exprSql = deparse_expression(expr, NIL, true, true); PopEmptySearchPath(saveNestLevel); appendStringInfo(buf, "SET DEFAULT %s", exprSql); } /* * AppendAlterDomainStmtAddConstraint is a helper function that appends the constraint * specification for an ALTER DOMAIN statement that adds a constraint to the domain. */ static void AppendAlterDomainStmtAddConstraint(StringInfo buf, AlterDomainStmt *stmt) { if (stmt->def == NULL || !IsA(stmt->def, Constraint)) { ereport(ERROR, (errmsg("unable to deparse ALTER DOMAIN statement due to " "unexpected contents"))); } Constraint *constraint = castNode(Constraint, stmt->def); appendStringInfoString(buf, "ADD"); int32 baseTypMod = 0; Oid baseOid = DomainGetBaseTypeOid(stmt->typeName, &baseTypMod); TypeName *baseTypeName = makeTypeNameFromOid(baseOid, baseTypMod); AppendConstraint(buf, constraint, stmt->typeName, baseTypeName); if (!constraint->initially_valid) { appendStringInfoString(buf, " NOT VALID"); } } /* * AppendAlterDomainStmtDropConstraint is a helper function that appends the DROP * CONSTRAINT part of an ALTER DOMAIN statement for an alter statement that drops a * constraint. */ static void AppendAlterDomainStmtDropConstraint(StringInfo buf, AlterDomainStmt *stmt) { appendStringInfoString(buf, "DROP CONSTRAINT "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfoString(buf, quote_identifier(stmt->name)); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } /* * AppendConstraint is a helper function that appends a constraint specification to a sql * string that is adding a constraint. * * There are multiple places where a constraint specification is appended to sql strings. * * Given the complexities of serializing a constraint they all use this routine. */ static void AppendConstraint(StringInfo buf, Constraint *constraint, List *domainName, TypeName *typeName) { if (constraint->conname) { appendStringInfo(buf, " CONSTRAINT %s", quote_identifier(constraint->conname)); } switch (constraint->contype) { case CONSTR_CHECK: { Node *expr = NULL; if (constraint->raw_expr) { /* the expression was parsed from sql, still needs to transform */ expr = TransformConstraintExpr(constraint->raw_expr, typeName); } else if (constraint->cooked_expr) { /* expression was read from the catalog, no cooking required just parse */ expr = stringToNode(constraint->cooked_expr); } else { elog(ERROR, "missing expression for domain constraint"); } int saveNestLevel = PushEmptySearchPath(); char *exprSql = deparse_expression(expr, NIL, true, true); PopEmptySearchPath(saveNestLevel); appendStringInfo(buf, " CHECK (%s)", exprSql); return; } case CONSTR_DEFAULT: { Node *expr = NULL; if (constraint->raw_expr) { /* the expression was parsed from sql, still needs to transform */ expr = TransformDefaultExpr(constraint->raw_expr, domainName, typeName); } else if (constraint->cooked_expr) { /* expression was read from the catalog, no cooking required just parse */ expr = stringToNode(constraint->cooked_expr); } else { elog(ERROR, "missing expression for domain default"); } int saveNestLevel = PushEmptySearchPath(); char *exprSql = deparse_expression(expr, NIL, true, true); PopEmptySearchPath(saveNestLevel); appendStringInfo(buf, " DEFAULT %s", exprSql); return; } case CONSTR_NOTNULL: { appendStringInfoString(buf, " NOT NULL"); return; } case CONSTR_NULL: { appendStringInfoString(buf, " NULL"); return; } default: { ereport(ERROR, (errmsg("unsupported constraint for distributed domain"))); } } } /* * TransformDefaultExpr transforms a default expression from the expression passed on the * AST to a cooked version that postgres uses internally. * * Only the cooked version can be easily turned back into a sql string, hence its use in * the deparser. This is only called for default expressions that don't have a cooked * variant stored. */ static Node * TransformDefaultExpr(Node *expr, List *domainName, TypeName *typeName) { const char *domainNameStr = NameListToQuotedString(domainName); int32 basetypeMod = 0; /* capture typeMod during lookup */ Type tup = typenameType(NULL, typeName, &basetypeMod); Oid basetypeoid = typeTypeId(tup); ReleaseSysCache(tup); ParseState *pstate = make_parsestate(NULL); Node *defaultExpr = cookDefault(pstate, expr, basetypeoid, basetypeMod, domainNameStr, 0); return defaultExpr; } /* * TransformConstraintExpr transforms a constraint expression from the expression passed * on the AST to a cooked version that postgres uses internally. * * Only the cooked version can be easily turned back into a sql string, hence its use in * the deparser. This is only called for default expressions that don't have a cooked * variant stored. */ static Node * TransformConstraintExpr(Node *expr, TypeName *typeName) { /* * Convert the A_EXPR in raw_expr into an EXPR */ ParseState *pstate = make_parsestate(NULL); /* * Set up a CoerceToDomainValue to represent the occurrence of VALUE in * the expression. Note that it will appear to have the type of the base * type, not the domain. This seems correct since within the check * expression, we should not assume the input value can be considered a * member of the domain. */ CoerceToDomainValue *domVal = GetCoerceDomainValue(typeName); pstate->p_pre_columnref_hook = replace_domain_constraint_value; pstate->p_ref_hook_state = (void *) domVal; expr = transformExpr(pstate, expr, EXPR_KIND_DOMAIN_CHECK); /* * Make sure it yields a boolean result. */ expr = coerce_to_boolean(pstate, expr, "CHECK"); /* * Fix up collation information. */ assign_expr_collations(pstate, expr); return expr; } /* * GetCoerceDomainValue creates a stub CoerceToDomainValue struct representing the type * referenced by the typeName. */ static CoerceToDomainValue * GetCoerceDomainValue(TypeName *typeName) { int32 typMod = 0; /* capture typeMod during lookup */ Type tup = LookupTypeName(NULL, typeName, &typMod, false); if (tup == NULL) { elog(ERROR, "unable to lookup type information for %s", NameListToQuotedString(typeName->names)); } CoerceToDomainValue *domVal = makeNode(CoerceToDomainValue); domVal->typeId = typeTypeId(tup); domVal->typeMod = typMod; domVal->collation = typeTypeCollation(tup); domVal->location = -1; ReleaseSysCache(tup); return domVal; } /* Parser pre_columnref_hook for domain CHECK constraint parsing */ static Node * replace_domain_constraint_value(ParseState *pstate, ColumnRef *cref) { /* * Check for a reference to "value", and if that's what it is, replace * with a CoerceToDomainValue as prepared for us by domainAddConstraint. * (We handle VALUE as a name, not a keyword, to avoid breaking a lot of * applications that have used VALUE as a column name in the past.) */ if (list_length(cref->fields) == 1) { Node *field1 = (Node *) linitial(cref->fields); Assert(IsA(field1, String)); char *colname = strVal(field1); if (strcmp(colname, "value") == 0) { CoerceToDomainValue *domVal = copyObject(pstate->p_ref_hook_state); /* Propagate location knowledge, if any */ domVal->location = cref->location; return (Node *) domVal; } } return NULL; } ================================================ FILE: src/backend/distributed/deparser/deparse_extension_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_extension_stmts.c * All routines to deparse extension statements. * This file contains deparse functions for extension statement deparsing * as well as related helper functions. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "utils/builtins.h" #include "distributed/deparser.h" #include "distributed/listutils.h" /* Local functions forward declarations for helper functions */ static void AppendCreateExtensionStmt(StringInfo buf, CreateExtensionStmt *stmt); static void AppendCreateExtensionStmtOptions(StringInfo buf, List *options); static void AppendDropExtensionStmt(StringInfo buf, DropStmt *stmt); static void AppendExtensionNameList(StringInfo buf, List *objects); static void AppendAlterExtensionSchemaStmt(StringInfo buf, AlterObjectSchemaStmt * alterExtensionSchemaStmt); static void AppendAlterExtensionStmt(StringInfo buf, AlterExtensionStmt *alterExtensionStmt); /* * GetExtensionOption returns DefElem * node with "defname" from "options" list */ DefElem * GetExtensionOption(List *extensionOptions, const char *defname) { DefElem *defElement = NULL; foreach_declared_ptr(defElement, extensionOptions) { if (IsA(defElement, DefElem) && strncmp(defElement->defname, defname, NAMEDATALEN) == 0) { return defElement; } } return NULL; } /* * DeparseCreateExtensionStmt builds and returns a string representing the * CreateExtensionStmt to be sent to worker nodes. */ char * DeparseCreateExtensionStmt(Node *node) { CreateExtensionStmt *stmt = castNode(CreateExtensionStmt, node); StringInfoData sql = { 0 }; initStringInfo(&sql); AppendCreateExtensionStmt(&sql, stmt); return sql.data; } /* * AppendCreateExtensionStmt appends a string representing the CreateExtensionStmt to a buffer */ static void AppendCreateExtensionStmt(StringInfo buf, CreateExtensionStmt *createExtensionStmt) { appendStringInfoString(buf, "CREATE EXTENSION "); if (createExtensionStmt->if_not_exists) { appendStringInfoString(buf, "IF NOT EXISTS "); } /* * Up until here we have been ending the statement in a space, which makes it possible * to just append the quoted extname. From here onwards we will not have the string * ending in a space so appends should begin with a whitespace. */ appendStringInfoString(buf, quote_identifier(createExtensionStmt->extname)); AppendCreateExtensionStmtOptions(buf, createExtensionStmt->options); appendStringInfoString(buf, ";"); } /* * AppendCreateExtensionStmtOptions takes the option list of a CreateExtensionStmt and * loops over the options to add them to the statement we are building. * * An error will be thrown if we run into an unsupported option, comparable to how * postgres gives an error when parsing this list. */ static void AppendCreateExtensionStmtOptions(StringInfo buf, List *options) { if (list_length(options) > 0) { /* only append WITH if we actual will add options to the statement */ appendStringInfoString(buf, " WITH"); } /* Add the options to the statement */ DefElem *defElem = NULL; foreach_declared_ptr(defElem, options) { if (strcmp(defElem->defname, "schema") == 0) { const char *schemaName = defGetString(defElem); appendStringInfo(buf, " SCHEMA %s", quote_identifier(schemaName)); } else if (strcmp(defElem->defname, "new_version") == 0) { const char *newVersion = defGetString(defElem); appendStringInfo(buf, " VERSION %s", quote_identifier(newVersion)); } else if (strcmp(defElem->defname, "old_version") == 0) { const char *oldVersion = defGetString(defElem); appendStringInfo(buf, " FROM %s", quote_identifier(oldVersion)); } else if (strcmp(defElem->defname, "cascade") == 0) { bool cascade = defGetBoolean(defElem); if (cascade) { appendStringInfoString(buf, " CASCADE"); } } else { elog(ERROR, "unrecognized option: %s", defElem->defname); } } } /* * DeparseAlterExtensionStmt builds and returns a string representing the * AlterExtensionStmt to be sent to worker nodes. */ char * DeparseAlterExtensionStmt(Node *node) { AlterExtensionStmt *stmt = castNode(AlterExtensionStmt, node); StringInfoData sql = { 0 }; initStringInfo(&sql); AppendAlterExtensionStmt(&sql, stmt); return sql.data; } /* * AppendAlterExtensionStmt appends a string representing the AlterExtensionStmt to a buffer */ static void AppendAlterExtensionStmt(StringInfo buf, AlterExtensionStmt *alterExtensionStmt) { List *optionsList = alterExtensionStmt->options; const char *extensionName = alterExtensionStmt->extname; extensionName = quote_identifier(extensionName); appendStringInfo(buf, "ALTER EXTENSION %s UPDATE", extensionName); /* * Append the options for ALTER EXTENSION ... UPDATE * Currently there is only 1 option, but this structure follows how postgres parses * the options. */ DefElem *option = NULL; foreach_declared_ptr(option, optionsList) { if (strcmp(option->defname, "new_version") == 0) { const char *newVersion = defGetString(option); appendStringInfo(buf, " TO %s", quote_identifier(newVersion)); } else { elog(ERROR, "unrecognized option: %s", option->defname); } } appendStringInfoString(buf, ";"); } /* * DeparseDropExtensionStmt builds and returns a string representing the DropStmt */ char * DeparseDropExtensionStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendDropExtensionStmt(&str, stmt); return str.data; } /* * AppendDropExtensionStmt appends a string representing the DropStmt for * an extension to a buffer. */ static void AppendDropExtensionStmt(StringInfo str, DropStmt *dropStmt) { /* we append "IF NOT EXISTS" clause regardless of the content of the statement. */ appendStringInfoString(str, "DROP EXTENSION IF EXISTS "); /* * Pick the distributed ones from the "objects" list that is storing * the object names to be deleted. */ AppendExtensionNameList(str, dropStmt->objects); /* depending on behaviour field of DropStmt, we should append CASCADE or RESTRICT */ if (dropStmt->behavior == DROP_CASCADE) { appendStringInfoString(str, " CASCADE;"); } else { appendStringInfoString(str, " RESTRICT;"); } } /* * AppendExtensionNameList appends a string representing the list of * extension names to a buffer. */ static void AppendExtensionNameList(StringInfo str, List *objects) { ListCell *objectCell = NULL; foreach(objectCell, objects) { const char *extensionName = strVal(lfirst(objectCell)); extensionName = quote_identifier(extensionName); if (objectCell != list_head(objects)) { appendStringInfo(str, ", "); } appendStringInfoString(str, extensionName); } } /* * DeparseAlterExtensionSchemaStmt builds and returns a string representing the * AlterObjectSchemaStmt (ALTER EXTENSION SET SCHEMA). */ char * DeparseAlterExtensionSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_EXTENSION); AppendAlterExtensionSchemaStmt(&str, stmt); return str.data; } /* * AppendAlterExtensionSchemaStmt appends a string representing the AlterObjectSchemaStmt * for an extension to a buffer. */ static void AppendAlterExtensionSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *alterExtensionSchemaStmt) { Assert(alterExtensionSchemaStmt->objectType == OBJECT_EXTENSION); const char *extensionName = strVal(alterExtensionSchemaStmt->object); const char *newSchemaName = alterExtensionSchemaStmt->newschema; extensionName = quote_identifier(extensionName); newSchemaName = quote_identifier(newSchemaName); appendStringInfo(buf, "ALTER EXTENSION %s SET SCHEMA %s;", extensionName, newSchemaName); } ================================================ FILE: src/backend/distributed/deparser/deparse_foreign_data_wrapper_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_foreign_data_wrapper_stmts.c * All routines to deparse foreign data wrapper statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" static void AppendGrantOnFDWStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnFDWNames(StringInfo buf, GrantStmt *stmt); char * DeparseGrantOnFDWStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_FDW); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnFDWStmt(&str, stmt); return str.data; } static void AppendGrantOnFDWStmt(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_FDW); AppendGrantSharedPrefix(buf, stmt); AppendGrantOnFDWNames(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } static void AppendGrantOnFDWNames(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON FOREIGN DATA WRAPPER "); foreach(cell, stmt->objects) { char *fdwname = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(fdwname)); if (cell != list_tail(stmt->objects)) { appendStringInfo(buf, ", "); } } } ================================================ FILE: src/backend/distributed/deparser/deparse_foreign_server_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_foreign_server_stmts.c * All routines to deparse foreign server statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" static void AppendCreateForeignServerStmt(StringInfo buf, CreateForeignServerStmt *stmt); static void AppendAlterForeignServerStmt(StringInfo buf, AlterForeignServerStmt *stmt); static void AppendAlterForeignServerOptions(StringInfo buf, AlterForeignServerStmt *stmt); static void AppendAlterForeignServerRenameStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterForeignServerOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendDropForeignServerStmt(StringInfo buf, DropStmt *stmt); static void AppendServerNames(StringInfo buf, DropStmt *stmt); static void AppendBehavior(StringInfo buf, DropStmt *stmt); static char * GetDefElemActionString(DefElemAction action); static void AppendGrantOnForeignServerStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnForeignServerServers(StringInfo buf, GrantStmt *stmt); char * DeparseCreateForeignServerStmt(Node *node) { CreateForeignServerStmt *stmt = castNode(CreateForeignServerStmt, node); StringInfoData str; initStringInfo(&str); AppendCreateForeignServerStmt(&str, stmt); return str.data; } char * DeparseAlterForeignServerStmt(Node *node) { AlterForeignServerStmt *stmt = castNode(AlterForeignServerStmt, node); StringInfoData str; initStringInfo(&str); AppendAlterForeignServerStmt(&str, stmt); return str.data; } char * DeparseAlterForeignServerRenameStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_FOREIGN_SERVER); StringInfoData str; initStringInfo(&str); AppendAlterForeignServerRenameStmt(&str, stmt); return str.data; } char * DeparseAlterForeignServerOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_FOREIGN_SERVER); StringInfoData str; initStringInfo(&str); AppendAlterForeignServerOwnerStmt(&str, stmt); return str.data; } char * DeparseDropForeignServerStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_FOREIGN_SERVER); StringInfoData str; initStringInfo(&str); AppendDropForeignServerStmt(&str, stmt); return str.data; } char * DeparseGrantOnForeignServerStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_FOREIGN_SERVER); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnForeignServerStmt(&str, stmt); return str.data; } static void AppendCreateForeignServerStmt(StringInfo buf, CreateForeignServerStmt *stmt) { appendStringInfoString(buf, "CREATE SERVER "); if (stmt->if_not_exists) { appendStringInfoString(buf, "IF NOT EXISTS "); } appendStringInfo(buf, "%s ", quote_identifier(stmt->servername)); if (stmt->servertype) { appendStringInfo(buf, "TYPE %s ", quote_literal_cstr(stmt->servertype)); } if (stmt->version) { appendStringInfo(buf, "VERSION %s ", quote_literal_cstr(stmt->version)); } appendStringInfo(buf, "FOREIGN DATA WRAPPER %s ", quote_identifier(stmt->fdwname)); AppendOptionListToString(buf, stmt->options); } static void AppendAlterForeignServerStmt(StringInfo buf, AlterForeignServerStmt *stmt) { appendStringInfo(buf, "ALTER SERVER %s ", quote_identifier(stmt->servername)); if (stmt->has_version) { appendStringInfo(buf, "VERSION %s ", quote_literal_cstr(stmt->version)); } AppendAlterForeignServerOptions(buf, stmt); } static void AppendAlterForeignServerOptions(StringInfo buf, AlterForeignServerStmt *stmt) { if (list_length(stmt->options) <= 0) { return; } appendStringInfoString(buf, "OPTIONS ("); DefElemAction action = DEFELEM_UNSPEC; DefElem *def = NULL; foreach_declared_ptr(def, stmt->options) { if (def->defaction != DEFELEM_UNSPEC) { action = def->defaction; char *actionString = GetDefElemActionString(action); appendStringInfo(buf, "%s ", actionString); } appendStringInfo(buf, "%s", quote_identifier(def->defname)); if (action != DEFELEM_DROP) { const char *value = quote_literal_cstr(defGetString(def)); appendStringInfo(buf, " %s", value); } if (def != llast(stmt->options)) { appendStringInfoString(buf, ", "); } } appendStringInfoString(buf, ")"); } static void AppendAlterForeignServerRenameStmt(StringInfo buf, RenameStmt *stmt) { appendStringInfo(buf, "ALTER SERVER %s RENAME TO %s", quote_identifier(strVal(stmt->object)), quote_identifier(stmt->newname)); } static void AppendAlterForeignServerOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { const char *servername = quote_identifier(strVal(stmt->object)); appendStringInfo(buf, "ALTER SERVER %s OWNER TO ", servername); appendStringInfo(buf, "%s", RoleSpecString(stmt->newowner, true)); } static void AppendDropForeignServerStmt(StringInfo buf, DropStmt *stmt) { appendStringInfoString(buf, "DROP SERVER "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendServerNames(buf, stmt); AppendBehavior(buf, stmt); } static void AppendServerNames(StringInfo buf, DropStmt *stmt) { String *serverValue = NULL; foreach_declared_ptr(serverValue, stmt->objects) { const char *serverString = quote_identifier(strVal(serverValue)); appendStringInfo(buf, "%s", serverString); if (serverValue != llast(stmt->objects)) { appendStringInfoString(buf, ", "); } } } static void AppendBehavior(StringInfo buf, DropStmt *stmt) { if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } else if (stmt->behavior == DROP_RESTRICT) { appendStringInfoString(buf, " RESTRICT"); } } static char * GetDefElemActionString(DefElemAction action) { switch (action) { case DEFELEM_ADD: { return "ADD"; } case DEFELEM_SET: { return "SET"; } case DEFELEM_DROP: { return "DROP"; } default: { return ""; } } } static void AppendGrantOnForeignServerStmt(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_FOREIGN_SERVER); AppendGrantSharedPrefix(buf, stmt); AppendGrantOnForeignServerServers(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } static void AppendGrantOnForeignServerServers(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON FOREIGN SERVER "); foreach(cell, stmt->objects) { char *servername = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(servername)); if (cell != list_tail(stmt->objects)) { appendStringInfo(buf, ", "); } } } ================================================ FILE: src/backend/distributed/deparser/deparse_function_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_function_stmts.c * * All routines to deparse function and procedure statements. * This file contains all entry points specific for function and procedure statement * deparsing * * Functions that could move later are AppendDefElem, AppendDefElemStrict, etc. These * should be reused across multiple statements and should live in their own deparse * file. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "nodes/value.h" #include "parser/parse_func.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/regproc.h" #include "utils/syscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" /* forward declaration for deparse functions */ static char * ObjectTypeToKeyword(ObjectType objtype); static void AppendAlterFunctionStmt(StringInfo buf, AlterFunctionStmt *stmt); static void AppendDropFunctionStmt(StringInfo buf, DropStmt *stmt); static void AppendFunctionName(StringInfo buf, ObjectWithArgs *func, ObjectType objtype); static void AppendFunctionNameList(StringInfo buf, List *objects, ObjectType objtype); static void AppendDefElem(StringInfo buf, DefElem *def); static void AppendDefElemStrict(StringInfo buf, DefElem *def); static void AppendDefElemVolatility(StringInfo buf, DefElem *def); static void AppendDefElemLeakproof(StringInfo buf, DefElem *def); static void AppendDefElemSecurity(StringInfo buf, DefElem *def); static void AppendDefElemParallel(StringInfo buf, DefElem *def); static void AppendDefElemCost(StringInfo buf, DefElem *def); static void AppendDefElemRows(StringInfo buf, DefElem *def); static void AppendDefElemSet(StringInfo buf, DefElem *def); static void AppendDefElemSupport(StringInfo buf, DefElem *def); static void AppendRenameFunctionStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterFunctionSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterFunctionOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendAlterFunctionDependsStmt(StringInfo buf, AlterObjectDependsStmt *stmt); static void AppendGrantOnFunctionStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnFunctionFunctions(StringInfo buf, GrantStmt *stmt); static char * CopyAndConvertToUpperCase(const char *str); /* * DeparseAlterFunctionStmt builds and returns a string representing the AlterFunctionStmt */ char * DeparseAlterFunctionStmt(Node *node) { AlterFunctionStmt *stmt = castNode(AlterFunctionStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterFunctionStmt(&str, stmt); return str.data; } /* * ObjectTypeToKeyword returns an appropriate string for the given ObjectType * Where the string will be one of "FUNCTION", "PROCEDURE", or "AGGREGATE" */ static char * ObjectTypeToKeyword(ObjectType objtype) { switch (objtype) { case OBJECT_FUNCTION: { return "FUNCTION"; } case OBJECT_PROCEDURE: { return "PROCEDURE"; } case OBJECT_AGGREGATE: { return "AGGREGATE"; } case OBJECT_ROUTINE: { return "ROUTINE"; } default: { elog(ERROR, "Unknown object type: %d", objtype); return NULL; } } } /* * AppendAlterFunctionStmt appends a string representing the AlterFunctionStmt to a buffer */ static void AppendAlterFunctionStmt(StringInfo buf, AlterFunctionStmt *stmt) { ListCell *actionCell = NULL; appendStringInfo(buf, "ALTER %s ", ObjectTypeToKeyword(stmt->objtype)); AppendFunctionName(buf, stmt->func, stmt->objtype); foreach(actionCell, stmt->actions) { DefElem *def = castNode(DefElem, lfirst(actionCell)); AppendDefElem(buf, def); } appendStringInfoString(buf, ";"); } /* * AppendDefElem appends a string representing the DefElem to a buffer */ static void AppendDefElem(StringInfo buf, DefElem *def) { if (strcmp(def->defname, "strict") == 0) { AppendDefElemStrict(buf, def); } else if (strcmp(def->defname, "volatility") == 0) { AppendDefElemVolatility(buf, def); } else if (strcmp(def->defname, "leakproof") == 0) { AppendDefElemLeakproof(buf, def); } else if (strcmp(def->defname, "security") == 0) { AppendDefElemSecurity(buf, def); } else if (strcmp(def->defname, "parallel") == 0) { AppendDefElemParallel(buf, def); } else if (strcmp(def->defname, "cost") == 0) { AppendDefElemCost(buf, def); } else if (strcmp(def->defname, "rows") == 0) { AppendDefElemRows(buf, def); } else if (strcmp(def->defname, "set") == 0) { AppendDefElemSet(buf, def); } else if (strcmp(def->defname, "support") == 0) { AppendDefElemSupport(buf, def); } } /* * AppendDefElemStrict appends a string representing the DefElem to a buffer */ static void AppendDefElemStrict(StringInfo buf, DefElem *def) { if (boolVal(def->arg)) { appendStringInfo(buf, " STRICT"); } else { appendStringInfo(buf, " CALLED ON NULL INPUT"); } } /* * AppendDefElemVolatility appends a string representing the DefElem to a buffer */ static void AppendDefElemVolatility(StringInfo buf, DefElem *def) { appendStringInfo(buf, " %s", CopyAndConvertToUpperCase(strVal(def->arg))); } /* * AppendDefElemLeakproof appends a string representing the DefElem to a buffer */ static void AppendDefElemLeakproof(StringInfo buf, DefElem *def) { if (!boolVal(def->arg)) { appendStringInfo(buf, " NOT"); } appendStringInfo(buf, " LEAKPROOF"); } /* * AppendDefElemSecurity appends a string representing the DefElem to a buffer */ static void AppendDefElemSecurity(StringInfo buf, DefElem *def) { if (!boolVal(def->arg)) { appendStringInfo(buf, " SECURITY INVOKER"); } else { appendStringInfo(buf, " SECURITY DEFINER"); } } /* * AppendDefElemParallel appends a string representing the DefElem to a buffer */ static void AppendDefElemParallel(StringInfo buf, DefElem *def) { appendStringInfo(buf, " PARALLEL %s", CopyAndConvertToUpperCase(strVal(def->arg))); } /* * AppendDefElemCost appends a string representing the DefElem to a buffer */ static void AppendDefElemCost(StringInfo buf, DefElem *def) { appendStringInfo(buf, " COST %lf", defGetNumeric(def)); } /* * AppendDefElemRows appends a string representing the DefElem to a buffer */ static void AppendDefElemRows(StringInfo buf, DefElem *def) { appendStringInfo(buf, " ROWS %lf", defGetNumeric(def)); } /* * AppendDefElemSet appends a string representing the DefElem to a buffer */ static void AppendDefElemSet(StringInfo buf, DefElem *def) { VariableSetStmt *setStmt = castNode(VariableSetStmt, def->arg); AppendVariableSet(buf, setStmt); } /* * AppendDefElemSupport appends a string representing the DefElem to a buffer */ static void AppendDefElemSupport(StringInfo buf, DefElem *def) { appendStringInfo(buf, " SUPPORT %s", defGetString(def)); } /* * DeparseRenameFunctionStmt builds and returns a string representing the RenameStmt */ char * DeparseRenameFunctionStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AssertObjectTypeIsFunctional(stmt->renameType); AppendRenameFunctionStmt(&str, stmt); return str.data; } /* * AppendRenameFunctionStmt appends a string representing the RenameStmt to a buffer */ static void AppendRenameFunctionStmt(StringInfo buf, RenameStmt *stmt) { ObjectWithArgs *func = castNode(ObjectWithArgs, stmt->object); appendStringInfo(buf, "ALTER %s ", ObjectTypeToKeyword(stmt->renameType)); AppendFunctionName(buf, func, stmt->renameType); appendStringInfo(buf, " RENAME TO %s;", quote_identifier(stmt->newname)); } /* * DeparseAlterFunctionSchemaStmt builds and returns a string representing the AlterObjectSchemaStmt */ char * DeparseAlterFunctionSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AssertObjectTypeIsFunctional(stmt->objectType); AppendAlterFunctionSchemaStmt(&str, stmt); return str.data; } /* * AppendAlterFunctionSchemaStmt appends a string representing the AlterObjectSchemaStmt to a buffer */ static void AppendAlterFunctionSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { ObjectWithArgs *func = castNode(ObjectWithArgs, stmt->object); appendStringInfo(buf, "ALTER %s ", ObjectTypeToKeyword(stmt->objectType)); AppendFunctionName(buf, func, stmt->objectType); appendStringInfo(buf, " SET SCHEMA %s;", quote_identifier(stmt->newschema)); } /* * DeparseAlterFunctionOwnerStmt builds and returns a string representing the AlterOwnerStmt */ char * DeparseAlterFunctionOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AssertObjectTypeIsFunctional(stmt->objectType); AppendAlterFunctionOwnerStmt(&str, stmt); return str.data; } /* * AppendAlterFunctionOwnerStmt appends a string representing the AlterOwnerStmt to a buffer */ static void AppendAlterFunctionOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { ObjectWithArgs *func = castNode(ObjectWithArgs, stmt->object); appendStringInfo(buf, "ALTER %s ", ObjectTypeToKeyword(stmt->objectType)); AppendFunctionName(buf, func, stmt->objectType); appendStringInfo(buf, " OWNER TO %s;", RoleSpecString(stmt->newowner, true)); } /* * DeparseAlterFunctionDependsStmt builds and returns a string representing the AlterObjectDependsStmt */ char * DeparseAlterFunctionDependsStmt(Node *node) { AlterObjectDependsStmt *stmt = castNode(AlterObjectDependsStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AssertObjectTypeIsFunctional(stmt->objectType); AppendAlterFunctionDependsStmt(&str, stmt); return str.data; } /* * AppendAlterFunctionDependsStmt appends a string representing the AlterObjectDependsStmt to a buffer */ static void AppendAlterFunctionDependsStmt(StringInfo buf, AlterObjectDependsStmt *stmt) { ObjectWithArgs *func = castNode(ObjectWithArgs, stmt->object); appendStringInfo(buf, "ALTER %s ", ObjectTypeToKeyword(stmt->objectType)); AppendFunctionName(buf, func, stmt->objectType); appendStringInfo(buf, " DEPENDS ON EXTENSION %s;", strVal(stmt->extname)); } /* * DeparseDropFunctionStmt builds and returns a string representing the DropStmt */ char * DeparseDropFunctionStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AssertObjectTypeIsFunctional(stmt->removeType); AppendDropFunctionStmt(&str, stmt); return str.data; } /* * AppendDropFunctionStmt appends a string representing the DropStmt to a buffer */ static void AppendDropFunctionStmt(StringInfo buf, DropStmt *stmt) { appendStringInfo(buf, "DROP %s ", ObjectTypeToKeyword(stmt->removeType)); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendFunctionNameList(buf, stmt->objects, stmt->removeType); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } appendStringInfoString(buf, ";"); } /* * AppendFunctionNameList appends a string representing the list of function names to a buffer */ static void AppendFunctionNameList(StringInfo buf, List *objects, ObjectType objtype) { ListCell *objectCell = NULL; foreach(objectCell, objects) { Node *object = lfirst(objectCell); if (objectCell != list_head(objects)) { appendStringInfo(buf, ", "); } ObjectWithArgs *func = castNode(ObjectWithArgs, object); AppendFunctionName(buf, func, objtype); } } /* * AppendFunctionName appends a string representing a single function name to a buffer */ static void AppendFunctionName(StringInfo buf, ObjectWithArgs *func, ObjectType objtype) { Oid funcid = LookupFuncWithArgs(objtype, func, true); if (funcid == InvalidOid) { /* * DROP FUNCTION IF EXISTS absent_function arrives here * * There is no namespace associated with the nonexistent function, * thus we return the function name as it is provided */ char *functionName = NULL; char *schemaName = NULL; DeconstructQualifiedName(func->objname, &schemaName, &functionName); char *qualifiedFunctionName = quote_qualified_identifier(schemaName, functionName); appendStringInfoString(buf, qualifiedFunctionName); if (!func->args_unspecified) { /* * The function is not found, but there is an argument list specified, this has * some known issues with the "any" type. However this is mostly a bug in * postgres' TypeNameListToString. For now the best we can do until we understand * the underlying cause better. */ const char *args = TypeNameListToString(func->objargs); appendStringInfo(buf, "(%s)", args); } } else { char *functionSignature = format_procedure_qualified(funcid); appendStringInfoString(buf, functionSignature); } } /* * CopyAndConvertToUpperCase copies a string and converts all characters to uppercase */ static char * CopyAndConvertToUpperCase(const char *str) { char *result, *p; result = pstrdup(str); for (p = result; *p; p++) { *p = pg_toupper((unsigned char) *p); } return result; } /* * DeparseGrantOnFunctionStmt builds and returns a string representing the GrantOnFunctionStmt */ char * DeparseGrantOnFunctionStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(isFunction(stmt->objtype)); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnFunctionStmt(&str, stmt); return str.data; } /* * AppendGrantOnFunctionStmt builds and returns an SQL command representing a * GRANT .. ON FUNCTION command from given GrantStmt object. */ static void AppendGrantOnFunctionStmt(StringInfo buf, GrantStmt *stmt) { Assert(isFunction(stmt->objtype)); if (stmt->targtype == ACL_TARGET_ALL_IN_SCHEMA) { elog(ERROR, "GRANT .. ALL FUNCTIONS/PROCEDURES IN SCHEMA is not supported for formatting."); } AppendGrantSharedPrefix(buf, stmt); AppendGrantOnFunctionFunctions(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } /* * AppendGrantOnFunctionFunctions appends the function names along with their arguments * to the given StringInfo from the given GrantStmt */ static void AppendGrantOnFunctionFunctions(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; /* * The FUNCTION syntax works for plain functions, aggregate functions, and window * functions, but not for procedures; use PROCEDURE for those. Alternatively, use * ROUTINE to refer to a function, aggregate function, window function, or procedure * regardless of its precise type. * https://www.postgresql.org/docs/current/sql-grant.html */ appendStringInfo(buf, " ON ROUTINE "); foreach(cell, stmt->objects) { /* * GrantOnFunction statement keeps its objects (functions) as * a list of ObjectWithArgs */ ObjectWithArgs *function = (ObjectWithArgs *) lfirst(cell); appendStringInfoString(buf, NameListToString(function->objname)); if (!function->args_unspecified) { /* if args are specified, we should append "(arg1, arg2, ...)" to the function name */ const char *args = TypeNameListToString(function->objargs); appendStringInfo(buf, "(%s)", args); } if (cell != list_tail(stmt->objects)) { appendStringInfoString(buf, ", "); } } } /* * isFunction returns true if the given ObjectType is a function, a procedure, a routine * or an aggregate otherwise returns false */ bool isFunction(ObjectType objectType) { return (objectType == OBJECT_FUNCTION || objectType == OBJECT_PROCEDURE || objectType == OBJECT_ROUTINE || objectType == OBJECT_AGGREGATE); } ================================================ FILE: src/backend/distributed/deparser/deparse_owned_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_owned_stmts.c * Functions to turn all Statement structures related to owned back * into sql. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" static void AppendDropOwnedStmt(StringInfo buf, DropOwnedStmt *stmt); static void AppendRoleList(StringInfo buf, List *roleList); /* * DeparseDropOwnedStmt builds and returns a string representing of the * DropOwnedStmt for application on a remote server. */ char * DeparseDropOwnedStmt(Node *node) { DropOwnedStmt *stmt = castNode(DropOwnedStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendDropOwnedStmt(&buf, stmt); return buf.data; } /* * AppendDropOwnedStmt generates the string representation of the * DropOwnedStmt and appends it to the buffer. */ static void AppendDropOwnedStmt(StringInfo buf, DropOwnedStmt *stmt) { appendStringInfo(buf, "DROP OWNED BY "); AppendRoleList(buf, stmt->roles); if (stmt->behavior == DROP_RESTRICT) { appendStringInfo(buf, " RESTRICT"); } else if (stmt->behavior == DROP_CASCADE) { appendStringInfo(buf, " CASCADE"); } } static void AppendRoleList(StringInfo buf, List *roleList) { ListCell *cell = NULL; foreach(cell, roleList) { Node *roleNode = (Node *) lfirst(cell); Assert(IsA(roleNode, RoleSpec) || IsA(roleNode, AccessPriv)); const char *rolename = NULL; if (IsA(roleNode, RoleSpec)) { rolename = RoleSpecString((RoleSpec *) roleNode, true); } appendStringInfoString(buf, rolename); if (cell != list_tail(roleList)) { appendStringInfo(buf, ", "); } } } static void AppendReassignOwnedStmt(StringInfo buf, ReassignOwnedStmt *stmt) { appendStringInfo(buf, "REASSIGN OWNED BY "); AppendRoleList(buf, stmt->roles); const char *newRoleName = RoleSpecString(stmt->newrole, true); appendStringInfo(buf, " TO %s", newRoleName); } char * DeparseReassignOwnedStmt(Node *node) { ReassignOwnedStmt *stmt = castNode(ReassignOwnedStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendReassignOwnedStmt(&buf, stmt); return buf.data; } ================================================ FILE: src/backend/distributed/deparser/deparse_publication_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_publication_stmts.c * All routines to deparse publication statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/relation.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/value.h" #include "parser/parse_clause.h" #include "parser/parse_collate.h" #include "parser/parse_node.h" #include "parser/parse_relation.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/namespace_utils.h" static void AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables); static bool AppendPublicationObjects(StringInfo buf, List *publicationObjects, bool whereClauseNeedsTransform, bool includeLocalTables); static void AppendWhereClauseExpression(StringInfo buf, RangeVar *tableName, Node *whereClause, bool whereClauseNeedsTransform); static void AppendAlterPublicationAction(StringInfo buf, AlterPublicationAction action); static bool AppendAlterPublicationStmt(StringInfo buf, AlterPublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables); static void AppendDropPublicationStmt(StringInfo buf, DropStmt *stmt); static void AppendRenamePublicationStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterPublicationOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendPublicationOptions(StringInfo stringBuffer, List *optionList); static void AppendIdentifierList(StringInfo buf, List *objects); /* * DeparseCreatePublicationStmt builds and returns a string representing a * CreatePublicationStmt. */ char * DeparseCreatePublicationStmt(Node *node) { /* regular deparsing function takes CREATE PUBLICATION from the parser */ bool whereClauseNeedsTransform = false; /* for regular CREATE PUBLICATION we do not propagate local tables */ bool includeLocalTables = false; return DeparseCreatePublicationStmtExtended(node, whereClauseNeedsTransform, includeLocalTables); } /* * DeparseCreatePublicationStmtExtended builds and returns a string representing a * CreatePublicationStmt, which may have already-transformed expressions. */ char * DeparseCreatePublicationStmtExtended(Node *node, bool whereClauseNeedsTransform, bool includeLocalTables) { CreatePublicationStmt *stmt = castNode(CreatePublicationStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendCreatePublicationStmt(&str, stmt, whereClauseNeedsTransform, includeLocalTables); return str.data; } /* * AppendCreatePublicationStmt appends a string representing a * CreatePublicationStmt to a buffer. */ static void AppendCreatePublicationStmt(StringInfo buf, CreatePublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables) { appendStringInfo(buf, "CREATE PUBLICATION %s", quote_identifier(stmt->pubname)); if (stmt->for_all_tables) { appendStringInfoString(buf, " FOR ALL TABLES"); } else if (stmt->pubobjects != NIL) { bool hasObjects = false; PublicationObjSpec *publicationObject = NULL; /* * Check whether there are objects to propagate, mainly to know whether * we should include "FOR". */ foreach_declared_ptr(publicationObject, stmt->pubobjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { /* FOR TABLE ... */ PublicationTable *publicationTable = publicationObject->pubtable; if (includeLocalTables || IsCitusTableRangeVar(publicationTable->relation, NoLock, false)) { hasObjects = true; break; } } else { hasObjects = true; break; } } if (hasObjects) { appendStringInfoString(buf, " FOR"); AppendPublicationObjects(buf, stmt->pubobjects, whereClauseNeedsTransform, includeLocalTables); } } if (stmt->options != NIL) { appendStringInfoString(buf, " WITH ("); AppendPublicationOptions(buf, stmt->options); appendStringInfoString(buf, ")"); } } /* * AppendPublicationObjects appends a string representing a list of publication * objects to a buffer. * * For instance: TABLE users, departments, TABLES IN SCHEMA production */ static bool AppendPublicationObjects(StringInfo buf, List *publicationObjects, bool whereClauseNeedsTransform, bool includeLocalTables) { PublicationObjSpec *publicationObject = NULL; bool appendedObject = false; foreach_declared_ptr(publicationObject, publicationObjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { /* FOR TABLE ... */ PublicationTable *publicationTable = publicationObject->pubtable; RangeVar *rangeVar = publicationTable->relation; char *schemaName = rangeVar->schemaname; char *tableName = rangeVar->relname; if (!includeLocalTables && !IsCitusTableRangeVar(rangeVar, NoLock, false)) { /* do not propagate local tables */ continue; } if (schemaName != NULL) { /* qualified table name */ appendStringInfo(buf, "%s TABLE %s", appendedObject ? "," : "", quote_qualified_identifier(schemaName, tableName)); } else { /* unqualified table name */ appendStringInfo(buf, "%s TABLE %s", appendedObject ? "," : "", quote_identifier(tableName)); } if (publicationTable->columns != NIL) { appendStringInfoString(buf, " ("); AppendIdentifierList(buf, publicationTable->columns); appendStringInfoString(buf, ")"); } if (publicationTable->whereClause != NULL) { appendStringInfoString(buf, " WHERE ("); AppendWhereClauseExpression(buf, rangeVar, publicationTable->whereClause, whereClauseNeedsTransform); appendStringInfoString(buf, ")"); } } else { /* FOR TABLES IN SCHEMA */ char *schemaName = publicationObject->name; if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLES_IN_CUR_SCHEMA) { List *searchPath = fetch_search_path(false); if (searchPath == NIL) { ereport(ERROR, errcode(ERRCODE_UNDEFINED_SCHEMA), errmsg("no schema has been selected for " "CURRENT_SCHEMA")); } schemaName = get_namespace_name(linitial_oid(searchPath)); } appendStringInfo(buf, "%s TABLES IN SCHEMA %s", appendedObject ? "," : "", quote_identifier(schemaName)); } appendedObject = true; } return appendedObject; } /* * AppendWhereClauseExpression appends a deparsed expression that can * contain a filter on the given table. If whereClauseNeedsTransform is set * the expression is first tranformed. */ static void AppendWhereClauseExpression(StringInfo buf, RangeVar *tableName, Node *whereClause, bool whereClauseNeedsTransform) { Relation relation = relation_openrv(tableName, AccessShareLock); if (whereClauseNeedsTransform) { ParseState *pstate = make_parsestate(NULL); pstate->p_sourcetext = ""; ParseNamespaceItem *nsitem = addRangeTableEntryForRelation(pstate, relation, AccessShareLock, NULL, false, false); addNSItemToQuery(pstate, nsitem, false, true, true); whereClause = transformWhereClause(pstate, copyObject(whereClause), EXPR_KIND_WHERE, "PUBLICATION WHERE"); assign_expr_collations(pstate, whereClause); } List *relationContext = deparse_context_for(tableName->relname, relation->rd_id); int saveNestLevel = PushEmptySearchPath(); char *whereClauseString = deparse_expression(whereClause, relationContext, true, true); PopEmptySearchPath(saveNestLevel); appendStringInfoString(buf, whereClauseString); relation_close(relation, AccessShareLock); } /* * DeparseAlterPublicationSchemaStmt builds and returns a string representing * an AlterPublicationStmt. */ char * DeparseAlterPublicationStmt(Node *node) { /* regular deparsing function takes ALTER PUBLICATION from the parser */ bool whereClauseNeedsTransform = true; /* for regular ALTER PUBLICATION we do not propagate local tables */ bool includeLocalTables = false; return DeparseAlterPublicationStmtExtended(node, whereClauseNeedsTransform, includeLocalTables); } /* * DeparseAlterPublicationStmtExtended builds and returns a string representing a * AlterPublicationStmt, which may have already-transformed expressions. */ char * DeparseAlterPublicationStmtExtended(Node *node, bool whereClauseNeedsTransform, bool includeLocalTables) { AlterPublicationStmt *stmt = castNode(AlterPublicationStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); if (!AppendAlterPublicationStmt(&str, stmt, whereClauseNeedsTransform, includeLocalTables)) { Assert(!includeLocalTables); /* * When there are no objects to propagate, then there is no * valid ALTER PUBLICATION to construct. */ return NULL; } return str.data; } /* * AppendAlterPublicationStmt appends a string representing an AlterPublicationStmt * of the form ALTER PUBLICATION .. ADD/SET/DROP */ static bool AppendAlterPublicationStmt(StringInfo buf, AlterPublicationStmt *stmt, bool whereClauseNeedsTransform, bool includeLocalTables) { appendStringInfo(buf, "ALTER PUBLICATION %s", quote_identifier(stmt->pubname)); if (stmt->options) { appendStringInfoString(buf, " SET ("); AppendPublicationOptions(buf, stmt->options); appendStringInfoString(buf, ")"); /* changing options cannot be combined with other actions */ return true; } AppendAlterPublicationAction(buf, stmt->action); return AppendPublicationObjects(buf, stmt->pubobjects, whereClauseNeedsTransform, includeLocalTables); } /* * AppendAlterPublicationAction appends a string representing an AlterPublicationAction * to a buffer. */ static void AppendAlterPublicationAction(StringInfo buf, AlterPublicationAction action) { switch (action) { case AP_AddObjects: { appendStringInfoString(buf, " ADD"); break; } case AP_DropObjects: { appendStringInfoString(buf, " DROP"); break; } case AP_SetObjects: { appendStringInfoString(buf, " SET"); break; } default: { ereport(ERROR, (errmsg("unrecognized publication action: %d", action))); } } } /* * DeparseDropPublicationStmt builds and returns a string representing the DropStmt */ char * DeparseDropPublicationStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->removeType == OBJECT_PUBLICATION); AppendDropPublicationStmt(&str, stmt); return str.data; } /* * AppendDropPublicationStmt appends a string representing the DropStmt to a buffer */ static void AppendDropPublicationStmt(StringInfo buf, DropStmt *stmt) { appendStringInfoString(buf, "DROP PUBLICATION "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendIdentifierList(buf, stmt->objects); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } /* * DeparseRenamePublicationStmt builds and returns a string representing the RenameStmt */ char * DeparseRenamePublicationStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_PUBLICATION); AppendRenamePublicationStmt(&str, stmt); return str.data; } /* * AppendRenamePublicationStmt appends a string representing the RenameStmt to a buffer */ static void AppendRenamePublicationStmt(StringInfo buf, RenameStmt *stmt) { appendStringInfo(buf, "ALTER PUBLICATION %s RENAME TO %s;", quote_identifier(strVal(stmt->object)), quote_identifier(stmt->newname)); } /* * DeparseAlterPublicationOwnerStmt builds and returns a string representing the AlterOwnerStmt */ char * DeparseAlterPublicationOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_PUBLICATION); AppendAlterPublicationOwnerStmt(&str, stmt); return str.data; } /* * AppendAlterPublicationOwnerStmt appends a string representing the AlterOwnerStmt to a buffer */ static void AppendAlterPublicationOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { Assert(stmt->objectType == OBJECT_PUBLICATION); appendStringInfo(buf, "ALTER PUBLICATION %s OWNER TO %s;", quote_identifier(strVal(stmt->object)), RoleSpecString(stmt->newowner, true)); } /* * AppendPublicationOptions appends a string representing a list of publication opions. */ static void AppendPublicationOptions(StringInfo stringBuffer, List *optionList) { ListCell *optionCell = NULL; bool firstOptionPrinted = false; foreach(optionCell, optionList) { DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; char *optionValue = defGetString(option); NodeTag valueType = nodeTag(option->arg); if (firstOptionPrinted) { appendStringInfo(stringBuffer, ", "); } firstOptionPrinted = true; appendStringInfo(stringBuffer, "%s = ", quote_identifier(optionName)); if (valueType == T_Integer || valueType == T_Float || valueType == T_Boolean) { /* string escaping is unnecessary for numeric types and can cause issues */ appendStringInfo(stringBuffer, "%s", optionValue); } else { appendStringInfo(stringBuffer, "%s", quote_literal_cstr(optionValue)); } } } /* * AppendIdentifierList appends a string representing a list of * identifiers (of String type). */ static void AppendIdentifierList(StringInfo buf, List *objects) { ListCell *objectCell = NULL; foreach(objectCell, objects) { char *name = strVal(lfirst(objectCell)); if (objectCell != list_head(objects)) { appendStringInfo(buf, ", "); } appendStringInfoString(buf, quote_identifier(name)); } } ================================================ FILE: src/backend/distributed/deparser/deparse_role_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_role_stmts.c * All routines to deparse role statements. * This file contains all entry points specific for ALTER ROLE statement * deparsing as well as functions that are currently only used for deparsing * ALTER ROLE statements. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void AppendAlterRoleStmt(StringInfo buf, AlterRoleStmt *stmt); static void AppendAlterRoleSetStmt(StringInfo buf, AlterRoleSetStmt *stmt); static void AppendCreateRoleStmt(StringInfo buf, CreateRoleStmt *stmt); static void AppendRoleOption(StringInfo buf, ListCell *optionCell); static void AppendRoleList(StringInfo buf, List *roleList); static void AppendDropRoleStmt(StringInfo buf, DropRoleStmt *stmt); static void AppendGrantRoleStmt(StringInfo buf, GrantRoleStmt *stmt); static void AppendRevokeAdminOptionFor(StringInfo buf, GrantRoleStmt *stmt); static void AppendGrantWithAdminOption(StringInfo buf, GrantRoleStmt *stmt); /* * DeparseAlterRoleStmt builds and returns a string representing of the * AlterRoleStmt for application on a remote server. */ char * DeparseAlterRoleStmt(Node *node) { AlterRoleStmt *stmt = castNode(AlterRoleStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendAlterRoleStmt(&buf, stmt); return buf.data; } /* * DeparseAlterRoleSetStmt builds and returns a string representing of the * AlterRoleSetStmt for application on a remote server. */ char * DeparseAlterRoleSetStmt(Node *node) { AlterRoleSetStmt *stmt = castNode(AlterRoleSetStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendAlterRoleSetStmt(&buf, stmt); return buf.data; } /* * AppendAlterRoleStmt generates the string representation of the * AlterRoleStmt and appends it to the buffer. */ static void AppendAlterRoleStmt(StringInfo buf, AlterRoleStmt *stmt) { ListCell *optionCell = NULL; RoleSpec *role = stmt->role; /* * If the role_specification used is CURRENT_USER or SESSION_USER, * it will be converted to thats roles role name. */ appendStringInfo(buf, "ALTER ROLE %s", RoleSpecString(role, true)); foreach(optionCell, stmt->options) { AppendRoleOption(buf, optionCell); } } /* * AppendRoleOption generates the string representation for the role options * and appends it to the buffer. * * This function only generates strings for common role options of ALTER ROLE * and CREATE ROLE statements. The extra options for CREATE ROLE are handled * seperately. */ static void AppendRoleOption(StringInfo buf, ListCell *optionCell) { DefElem *option = (DefElem *) lfirst(optionCell); if (strcmp(option->defname, "superuser") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " SUPERUSER"); } else if (strcmp(option->defname, "superuser") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOSUPERUSER"); } else if (strcmp(option->defname, "createdb") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " CREATEDB"); } else if (strcmp(option->defname, "createdb") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOCREATEDB"); } else if (strcmp(option->defname, "createrole") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " CREATEROLE"); } else if (strcmp(option->defname, "createrole") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOCREATEROLE"); } else if (strcmp(option->defname, "inherit") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " INHERIT"); } else if (strcmp(option->defname, "inherit") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOINHERIT"); } else if (strcmp(option->defname, "canlogin") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " LOGIN"); } else if (strcmp(option->defname, "canlogin") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOLOGIN"); } else if (strcmp(option->defname, "isreplication") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " REPLICATION"); } else if (strcmp(option->defname, "isreplication") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOREPLICATION"); } else if (strcmp(option->defname, "bypassrls") == 0 && boolVal(option->arg)) { appendStringInfo(buf, " BYPASSRLS"); } else if (strcmp(option->defname, "bypassrls") == 0 && !boolVal(option->arg)) { appendStringInfo(buf, " NOBYPASSRLS"); } else if (strcmp(option->defname, "connectionlimit") == 0) { appendStringInfo(buf, " CONNECTION LIMIT %d", intVal(option->arg)); } else if (strcmp(option->defname, "password") == 0) { if (option->arg != NULL) { appendStringInfo(buf, " PASSWORD %s", quote_literal_cstr(strVal( option->arg))); } else { appendStringInfo(buf, " PASSWORD NULL"); } } else if (strcmp(option->defname, "validUntil") == 0) { appendStringInfo(buf, " VALID UNTIL %s", quote_literal_cstr(strVal(option->arg))); } } /* * DeparseCreateRoleStmt builds and returns a string representing of the * CreateRoleStmt for application on a remote server. */ char * DeparseCreateRoleStmt(Node *node) { CreateRoleStmt *stmt = castNode(CreateRoleStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendCreateRoleStmt(&buf, stmt); return buf.data; } static void AppendSysIdStatement(StringInfo buf, ListCell *optionCell) { DefElem *option = (DefElem *) lfirst(optionCell); if (strcmp(option->defname, "sysid") == 0) { appendStringInfo(buf, " SYSID %d", intVal(option->arg)); } } /* * AppendInlinePriviliges generates the string representation for the inline * privileges of the role in create statement and appends it to the buffer. */ static void AppendInlinePriviliges(StringInfo buf, ListCell *optionCell) { DefElem *option = (DefElem *) lfirst(optionCell); if (strcmp(option->defname, "adminmembers") == 0) { appendStringInfo(buf, " ADMIN "); AppendRoleList(buf, (List *) option->arg); } else if (strcmp(option->defname, "rolemembers") == 0) { appendStringInfo(buf, " ROLE "); AppendRoleList(buf, (List *) option->arg); } else if (strcmp(option->defname, "addroleto") == 0) { appendStringInfo(buf, " IN ROLE "); AppendRoleList(buf, (List *) option->arg); } } /* * AppendStatementType generates the string representation for the statement * type (role, user or group) in alter/create statement and appends it to the buffer. */ static void AppendStatementType(StringInfo buf, CreateRoleStmt *stmt) { switch (stmt->stmt_type) { case ROLESTMT_ROLE: { appendStringInfo(buf, "ROLE "); break; } case ROLESTMT_USER: { appendStringInfo(buf, "USER "); break; } case ROLESTMT_GROUP: { appendStringInfo(buf, "GROUP "); break; } } } /* * AppendCreateRoleStmt generates the string representation of the * CreateRoleStmt and appends it to the buffer. */ static void AppendCreateRoleStmt(StringInfo buf, CreateRoleStmt *stmt) { ListCell *optionCell = NULL; appendStringInfo(buf, "CREATE "); AppendStatementType(buf, stmt); appendStringInfo(buf, "%s", quote_identifier(stmt->role)); foreach(optionCell, stmt->options) { AppendRoleOption(buf, optionCell); AppendInlinePriviliges(buf, optionCell); AppendSysIdStatement(buf, optionCell); } } /* * DeparseDropRoleStmt builds and returns a string representing of the * DropRoleStmt for application on a remote server. */ char * DeparseDropRoleStmt(Node *node) { DropRoleStmt *stmt = castNode(DropRoleStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendDropRoleStmt(&buf, stmt); return buf.data; } /* * AppendDropRoleStmt generates the string representation of the * DropRoleStmt and appends it to the buffer. */ static void AppendDropRoleStmt(StringInfo buf, DropRoleStmt *stmt) { appendStringInfo(buf, "DROP ROLE "); if (stmt->missing_ok) { appendStringInfo(buf, "IF EXISTS "); } AppendRoleList(buf, stmt->roles); } static void AppendRoleList(StringInfo buf, List *roleList) { ListCell *cell = NULL; foreach(cell, roleList) { Node *roleNode = (Node *) lfirst(cell); Assert(IsA(roleNode, RoleSpec) || IsA(roleNode, AccessPriv)); char const *rolename = NULL; if (IsA(roleNode, RoleSpec)) { rolename = RoleSpecString((RoleSpec *) roleNode, true); } if (IsA(roleNode, AccessPriv)) { rolename = quote_identifier(((AccessPriv *) roleNode)->priv_name); } appendStringInfoString(buf, rolename); if (cell != list_tail(roleList)) { appendStringInfo(buf, ", "); } } } char * DeparseRenameRoleStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_ROLE); appendStringInfo(&str, "ALTER ROLE %s RENAME TO %s;", quote_identifier(stmt->subname), quote_identifier(stmt->newname)); return str.data; } /* * DeparseGrantRoleStmt builds and returns a string representing of the * GrantRoleStmt for application on a remote server. */ char * DeparseGrantRoleStmt(Node *node) { GrantRoleStmt *stmt = castNode(GrantRoleStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); AppendGrantRoleStmt(&buf, stmt); return buf.data; } /* * Append the 'RESTRICT' or 'CASCADE' clause to the given buffer if the given * statement is a 'REVOKE' statement and the behavior is specified. * After PostgreSQL 16, the behavior is specified in the 'opt' field of * GrantRoleStmt and may have multiple values. * Here, compile time version is checked to support both versions. */ static void AppendRevokeAdminOptionFor(StringInfo buf, GrantRoleStmt *stmt) { if (!stmt->is_grant) { DefElem *opt = NULL; foreach_declared_ptr(opt, stmt->opt) { if (strcmp(opt->defname, "admin") == 0) { appendStringInfo(buf, "ADMIN OPTION FOR "); break; } else if (strcmp(opt->defname, "inherit") == 0) { appendStringInfo(buf, "INHERIT OPTION FOR "); break; } else if (strcmp(opt->defname, "set") == 0) { appendStringInfo(buf, "SET OPTION FOR "); break; } } } } static void AppendGrantWithAdminOption(StringInfo buf, GrantRoleStmt *stmt) { if (stmt->is_grant) { int opt_count = 0; DefElem *opt = NULL; foreach_declared_ptr(opt, stmt->opt) { char *optval = defGetString(opt); bool option_value = false; if (parse_bool(optval, &option_value)) { opt_count++; char *prefix = opt_count > 1 ? "," : " WITH"; if (strcmp(opt->defname, "inherit") == 0) { appendStringInfo(buf, "%s INHERIT %s", prefix, option_value ? "TRUE" : "FALSE"); } else if (strcmp(opt->defname, "admin") == 0 && option_value) { appendStringInfo(buf, "%s ADMIN OPTION", prefix); } else if (strcmp(opt->defname, "set") == 0 && !option_value) { appendStringInfo(buf, "%s SET FALSE", prefix); } } } } } /* * AppendGrantRoleStmt generates the string representation of the * GrantRoleStmt and appends it to the buffer. */ static void AppendGrantRoleStmt(StringInfo buf, GrantRoleStmt *stmt) { appendStringInfo(buf, "%s ", stmt->is_grant ? "GRANT" : "REVOKE"); AppendRevokeAdminOptionFor(buf, stmt); AppendRoleList(buf, stmt->granted_roles); appendStringInfo(buf, "%s ", stmt->is_grant ? " TO " : " FROM "); AppendRoleList(buf, stmt->grantee_roles); AppendGrantWithAdminOption(buf, stmt); AppendGrantedByInGrantForRoleSpec(buf, stmt->grantor, stmt->is_grant); AppendGrantRestrictAndCascadeForRoleSpec(buf, stmt->behavior, stmt->is_grant); appendStringInfo(buf, ";"); } /* * AppendAlterRoleSetStmt generates the string representation of the * AlterRoleSetStmt and appends it to the buffer. */ static void AppendAlterRoleSetStmt(StringInfo buf, AlterRoleSetStmt *stmt) { RoleSpec *role = stmt->role; const char *roleSpecStr = NULL; if (role == NULL) { /* * If all roles are be affected, role field is left blank in an * AlterRoleSetStmt. */ roleSpecStr = "ALL"; } else { /* * If the role_specification used is CURRENT_USER or SESSION_USER, * it will be converted to thats roles role name. * * We also set withQuoteIdentifier parameter to true. Since the * roleSpecStr will be used in a query, the quotes are needed. */ roleSpecStr = RoleSpecString(role, true); } appendStringInfo(buf, "ALTER ROLE %s", roleSpecStr); if (stmt->database != NULL) { appendStringInfo(buf, " IN DATABASE %s", quote_identifier(stmt->database)); } VariableSetStmt *setStmt = castNode(VariableSetStmt, stmt->setstmt); AppendVariableSet(buf, setStmt); } ================================================ FILE: src/backend/distributed/deparser/deparse_schema_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_schema_stmts.c * All routines to deparse schema statements. * This file contains all entry points specific for schema statement deparsing * as well as functions that are currently only used for deparsing of the * schema statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void AppendCreateSchemaStmt(StringInfo buf, CreateSchemaStmt *stmt); static void AppendDropSchemaStmt(StringInfo buf, DropStmt *stmt); static void AppendGrantOnSchemaStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnSchemaSchemas(StringInfo buf, GrantStmt *stmt); static void AppendAlterSchemaRenameStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterSchemaOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); char * DeparseCreateSchemaStmt(Node *node) { CreateSchemaStmt *stmt = castNode(CreateSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendCreateSchemaStmt(&str, stmt); return str.data; } char * DeparseDropSchemaStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendDropSchemaStmt(&str, stmt); return str.data; } char * DeparseGrantOnSchemaStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SCHEMA); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnSchemaStmt(&str, stmt); return str.data; } char * DeparseAlterSchemaOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterSchemaOwnerStmt(&str, stmt); return str.data; } static void AppendAlterSchemaOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { Assert(stmt->objectType == OBJECT_SCHEMA); appendStringInfo(buf, "ALTER SCHEMA %s OWNER TO %s;", quote_identifier(strVal(stmt->object)), RoleSpecString(stmt->newowner, true)); } char * DeparseAlterSchemaRenameStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterSchemaRenameStmt(&str, stmt); return str.data; } static void AppendCreateSchemaStmt(StringInfo buf, CreateSchemaStmt *stmt) { appendStringInfoString(buf, "CREATE SCHEMA "); if (stmt->if_not_exists) { appendStringInfoString(buf, "IF NOT EXISTS "); } if (stmt->schemaname != NULL) { appendStringInfo(buf, "%s ", quote_identifier(stmt->schemaname)); } else { /* * If the schema name is not provided, the schema will be created * with the name of the authorizated user. */ Assert(stmt->authrole != NULL); } if (stmt->authrole != NULL) { appendStringInfo(buf, "AUTHORIZATION %s", RoleSpecString(stmt->authrole, true)); } } static void AppendDropSchemaStmt(StringInfo buf, DropStmt *stmt) { Assert(stmt->removeType == OBJECT_SCHEMA); appendStringInfoString(buf, "DROP SCHEMA "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } String *schemaValue = NULL; foreach_declared_ptr(schemaValue, stmt->objects) { const char *schemaString = quote_identifier(strVal(schemaValue)); appendStringInfo(buf, "%s", schemaString); if (schemaValue != llast(stmt->objects)) { appendStringInfoString(buf, ", "); } } if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } else if (stmt->behavior == DROP_RESTRICT) { appendStringInfoString(buf, " RESTRICT"); } } static void AppendGrantOnSchemaStmt(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_SCHEMA); AppendGrantSharedPrefix(buf, stmt); AppendGrantOnSchemaSchemas(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } void AppendGrantPrivileges(StringInfo buf, GrantStmt *stmt) { if (list_length(stmt->privileges) == 0) { appendStringInfo(buf, "ALL PRIVILEGES"); } else { ListCell *cell = NULL; foreach(cell, stmt->privileges) { AccessPriv *privilege = (AccessPriv *) lfirst(cell); appendStringInfoString(buf, privilege->priv_name); if (cell != list_tail(stmt->privileges)) { appendStringInfo(buf, ", "); } } } } static void AppendGrantOnSchemaSchemas(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON SCHEMA "); foreach(cell, stmt->objects) { char *schema = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(schema)); if (cell != list_tail(stmt->objects)) { appendStringInfo(buf, ", "); } } } void AppendGrantGrantees(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " %s ", stmt->is_grant ? "TO" : "FROM"); foreach(cell, stmt->grantees) { RoleSpec *grantee = (RoleSpec *) lfirst(cell); appendStringInfoString(buf, RoleSpecString(grantee, true)); if (cell != list_tail(stmt->grantees)) { appendStringInfo(buf, ", "); } } } static void AppendAlterSchemaRenameStmt(StringInfo buf, RenameStmt *stmt) { Assert(stmt->renameType == OBJECT_SCHEMA); appendStringInfo(buf, "ALTER SCHEMA %s RENAME TO %s;", quote_identifier(stmt->subname), quote_identifier(stmt->newname)); } ================================================ FILE: src/backend/distributed/deparser/deparse_seclabel_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_seclabel_stmts.c * All routines to deparse SECURITY LABEL statements. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "distributed/deparser.h" static void BeginSecLabel(StringInfo buf, SecLabelStmt *stmt) { initStringInfo(buf); appendStringInfoString(buf, "SECURITY LABEL "); if (stmt->provider != NULL) { appendStringInfo(buf, "FOR %s ", quote_identifier(stmt->provider)); } appendStringInfoString(buf, "ON "); } static void EndSecLabel(StringInfo buf, SecLabelStmt *stmt) { appendStringInfo(buf, "IS %s", (stmt->label != NULL) ? quote_literal_cstr(stmt->label) : "NULL"); } /* * DeparseRoleSecLabelStmt builds and returns a string representation of the * SecLabelStmt for application on a remote server. The SecLabelStmt is for * a role object. */ char * DeparseRoleSecLabelStmt(Node *node) { SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); char *role_name = strVal(secLabelStmt->object); StringInfoData buf = { 0 }; BeginSecLabel(&buf, secLabelStmt); appendStringInfo(&buf, "ROLE %s ", quote_identifier(role_name)); EndSecLabel(&buf, secLabelStmt); return buf.data; } /* * DeparseTableSecLabelStmt builds and returns a string representation of the * SecLabelStmt for application on a remote server. The SecLabelStmt is for a * table. */ char * DeparseTableSecLabelStmt(Node *node) { SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); List *names = (List *) secLabelStmt->object; StringInfoData buf = { 0 }; BeginSecLabel(&buf, secLabelStmt); appendStringInfo(&buf, "TABLE %s", quote_identifier(strVal(linitial(names)))); if (list_length(names) > 1) { appendStringInfo(&buf, ".%s", quote_identifier(strVal(lsecond(names)))); } appendStringInfoString(&buf, " "); EndSecLabel(&buf, secLabelStmt); return buf.data; } /* * DeparseColumnSecLabelStmt builds and returns a string representation of the * SecLabelStmt for application on a remote server. The SecLabelStmt is for a * column of a distributed table. */ char * DeparseColumnSecLabelStmt(Node *node) { SecLabelStmt *secLabelStmt = castNode(SecLabelStmt, node); List *names = (List *) secLabelStmt->object; StringInfoData buf = { 0 }; BeginSecLabel(&buf, secLabelStmt); appendStringInfo(&buf, "COLUMN %s.%s", quote_identifier(strVal(linitial(names))), quote_identifier(strVal(lsecond(names)))); if (list_length(names) > 2) { appendStringInfo(&buf, ".%s", quote_identifier(strVal(lthird(names)))); } appendStringInfoString(&buf, " "); EndSecLabel(&buf, secLabelStmt); return buf.data; } ================================================ FILE: src/backend/distributed/deparser/deparse_sequence_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_sequence_stmts.c * * All routines to deparse sequence statements. * This file contains all entry points specific for sequence statement * deparsing * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" /* forward declaration for deparse functions */ static void AppendDropSequenceStmt(StringInfo buf, DropStmt *stmt); static void AppendSequenceNameList(StringInfo buf, List *objects, ObjectType objtype); static void AppendRenameSequenceStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterSequenceSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterSequenceOwnerStmt(StringInfo buf, AlterTableStmt *stmt); static void AppendAlterSequencePersistenceStmt(StringInfo buf, AlterTableStmt *stmt); static void AppendGrantOnSequenceStmt(StringInfo buf, GrantStmt *stmt); static void AppendGrantOnSequenceSequences(StringInfo buf, GrantStmt *stmt); /* * DeparseDropSequenceStmt builds and returns a string representing the DropStmt */ char * DeparseDropSequenceStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->removeType == OBJECT_SEQUENCE); AppendDropSequenceStmt(&str, stmt); return str.data; } /* * AppendDropSequenceStmt appends a string representing the DropStmt to a buffer */ static void AppendDropSequenceStmt(StringInfo buf, DropStmt *stmt) { appendStringInfoString(buf, "DROP SEQUENCE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendSequenceNameList(buf, stmt->objects, stmt->removeType); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } appendStringInfoString(buf, ";"); } /* * AppendSequenceNameList appends a string representing the list of sequence names to a buffer */ static void AppendSequenceNameList(StringInfo buf, List *objects, ObjectType objtype) { ListCell *objectCell = NULL; foreach(objectCell, objects) { if (objectCell != list_head(objects)) { appendStringInfo(buf, ", "); } RangeVar *seq = makeRangeVarFromNameList((List *) lfirst(objectCell)); char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); appendStringInfoString(buf, qualifiedSequenceName); } } /* * DeparseRenameSequenceStmt builds and returns a string representing the RenameStmt */ char * DeparseRenameSequenceStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_SEQUENCE); AppendRenameSequenceStmt(&str, stmt); return str.data; } /* * AppendRenameSequenceStmt appends a string representing the RenameStmt to a buffer */ static void AppendRenameSequenceStmt(StringInfo buf, RenameStmt *stmt) { RangeVar *seq = stmt->relation; char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); appendStringInfoString(buf, "ALTER SEQUENCE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfoString(buf, qualifiedSequenceName); appendStringInfo(buf, " RENAME TO %s", quote_identifier(stmt->newname)); } /* * DeparseAlterSequenceSchemaStmt builds and returns a string representing the AlterObjectSchemaStmt */ char * DeparseAlterSequenceSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_SEQUENCE); AppendAlterSequenceSchemaStmt(&str, stmt); return str.data; } /* * AppendAlterSequenceSchemaStmt appends a string representing the AlterObjectSchemaStmt to a buffer */ static void AppendAlterSequenceSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { RangeVar *seq = stmt->relation; char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); appendStringInfoString(buf, "ALTER SEQUENCE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfoString(buf, qualifiedSequenceName); appendStringInfo(buf, " SET SCHEMA %s;", quote_identifier(stmt->newschema)); } /* * DeparseAlterSequenceOwnerStmt builds and returns a string representing the AlterTableStmt * consisting of changing the owner of a sequence */ char * DeparseAlterSequenceOwnerStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objtype == OBJECT_SEQUENCE); AppendAlterSequenceOwnerStmt(&str, stmt); return str.data; } /* * AppendAlterSequenceOwnerStmt appends a string representing the AlterTableStmt to a buffer * consisting of changing the owner of a sequence */ static void AppendAlterSequenceOwnerStmt(StringInfo buf, AlterTableStmt *stmt) { Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); appendStringInfoString(buf, "ALTER SEQUENCE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfoString(buf, qualifiedSequenceName); ListCell *cmdCell = NULL; foreach(cmdCell, stmt->cmds) { if (cmdCell != list_head(stmt->cmds)) { /* * normally we shouldn't ever reach this * because we enter this function after making sure we have only * one subcommand of the type AT_ChangeOwner */ ereport(ERROR, (errmsg("More than one subcommand is not supported " "for ALTER SEQUENCE"))); } AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(cmdCell)); switch (alterTableCmd->subtype) { case AT_ChangeOwner: { appendStringInfo(buf, " OWNER TO %s;", get_rolespec_name( alterTableCmd->newowner)); break; } default: { /* * normally we shouldn't ever reach this * because we enter this function after making sure this stmt is of the form * ALTER SEQUENCE .. OWNER TO .. */ ereport(ERROR, (errmsg("unsupported subtype for alter sequence command"), errdetail("sub command type: %d", alterTableCmd->subtype))); } } } } /* * DeparseAlterSequencePersistenceStmt builds and returns a string representing * the AlterTableStmt consisting of changing the persistence of a sequence */ char * DeparseAlterSequencePersistenceStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objtype == OBJECT_SEQUENCE); AppendAlterSequencePersistenceStmt(&str, stmt); return str.data; } /* * AppendAlterSequencePersistenceStmt appends a string representing the * AlterTableStmt to a buffer consisting of changing the persistence of a sequence */ static void AppendAlterSequencePersistenceStmt(StringInfo buf, AlterTableStmt *stmt) { Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); appendStringInfoString(buf, "ALTER SEQUENCE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfoString(buf, qualifiedSequenceName); ListCell *cmdCell = NULL; foreach(cmdCell, stmt->cmds) { if (cmdCell != list_head(stmt->cmds)) { /* * As of PG15, we cannot reach this code because ALTER SEQUENCE * is only supported for a single sequence. Still, let's be * defensive for future PG changes */ ereport(ERROR, (errmsg("More than one subcommand is not supported " "for ALTER SEQUENCE"))); } AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(cmdCell)); switch (alterTableCmd->subtype) { case AT_SetLogged: { appendStringInfoString(buf, " SET LOGGED;"); break; } case AT_SetUnLogged: { appendStringInfoString(buf, " SET UNLOGGED;"); break; } default: { /* * normally we shouldn't ever reach this * because we enter this function after making sure this stmt is of the form * ALTER SEQUENCE .. SET LOGGED/UNLOGGED */ ereport(ERROR, (errmsg("unsupported subtype for alter sequence command"), errdetail("sub command type: %d", alterTableCmd->subtype))); } } } } /* * DeparseGrantOnSequenceStmt builds and returns a string representing the GrantOnSequenceStmt */ char * DeparseGrantOnSequenceStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnSequenceStmt(&str, stmt); return str.data; } /* * AppendGrantOnSequenceStmt builds and returns an SQL command representing a * GRANT .. ON SEQUENCE command from given GrantStmt object. */ static void AppendGrantOnSequenceStmt(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_SEQUENCE); if (stmt->targtype == ACL_TARGET_ALL_IN_SCHEMA) { /* * Normally we shouldn't reach this * We deparse a GrantStmt with OBJECT_SEQUENCE after setting targtype * to ACL_TARGET_OBJECT */ elog(ERROR, "GRANT .. ALL SEQUENCES IN SCHEMA is not supported for formatting."); } AppendGrantSharedPrefix(buf, stmt); AppendGrantOnSequenceSequences(buf, stmt); AppendGrantSharedSuffix(buf, stmt); } /* * AppendGrantOnSequenceSequences appends the sequence names along with their arguments * to the given StringInfo from the given GrantStmt */ static void AppendGrantOnSequenceSequences(StringInfo buf, GrantStmt *stmt) { Assert(stmt->objtype == OBJECT_SEQUENCE); appendStringInfoString(buf, " ON SEQUENCE "); ListCell *cell = NULL; foreach(cell, stmt->objects) { /* * GrantOnSequence statement keeps its objects (sequences) as * a list of RangeVar-s */ RangeVar *sequence = (RangeVar *) lfirst(cell); /* * We have qualified the statement beforehand */ appendStringInfoString(buf, quote_qualified_identifier(sequence->schemaname, sequence->relname)); if (cell != list_tail(stmt->objects)) { appendStringInfoString(buf, ", "); } } } ================================================ FILE: src/backend/distributed/deparser/deparse_statistics_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_statistics_stmts.c * All routines to deparse statistics statements. * This file contains all entry points specific for statistics statement deparsing * as well as functions that are currently only used for deparsing of the statistics * statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "parser/parse_expr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" static void AppendCreateStatisticsStmt(StringInfo buf, CreateStatsStmt *stmt); static void AppendDropStatisticsStmt(StringInfo buf, List *nameList, bool ifExists); static void AppendAlterStatisticsRenameStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterStatisticsSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterStatisticsStmt(StringInfo buf, AlterStatsStmt *stmt); static void AppendAlterStatisticsOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendStatisticsName(StringInfo buf, CreateStatsStmt *stmt); static void AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt); static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt); static void AppendTableName(StringInfo buf, CreateStatsStmt *stmt); char * DeparseCreateStatisticsStmt(Node *node) { CreateStatsStmt *stmt = castNode(CreateStatsStmt, node); StringInfoData str; initStringInfo(&str); AppendCreateStatisticsStmt(&str, stmt); return str.data; } char * DeparseDropStatisticsStmt(List *nameList, bool ifExists) { StringInfoData str; initStringInfo(&str); AppendDropStatisticsStmt(&str, nameList, ifExists); return str.data; } char * DeparseAlterStatisticsRenameStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str; initStringInfo(&str); AppendAlterStatisticsRenameStmt(&str, stmt); return str.data; } char * DeparseAlterStatisticsSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str; initStringInfo(&str); AppendAlterStatisticsSchemaStmt(&str, stmt); return str.data; } char * DeparseAlterStatisticsStmt(Node *node) { AlterStatsStmt *stmt = castNode(AlterStatsStmt, node); StringInfoData str; initStringInfo(&str); AppendAlterStatisticsStmt(&str, stmt); return str.data; } char * DeparseAlterStatisticsOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); StringInfoData str; initStringInfo(&str); AppendAlterStatisticsOwnerStmt(&str, stmt); return str.data; } static void AppendCreateStatisticsStmt(StringInfo buf, CreateStatsStmt *stmt) { appendStringInfoString(buf, "CREATE STATISTICS "); if (stmt->if_not_exists) { appendStringInfoString(buf, "IF NOT EXISTS "); } AppendStatisticsName(buf, stmt); AppendStatTypes(buf, stmt); appendStringInfoString(buf, " ON "); AppendColumnNames(buf, stmt); appendStringInfoString(buf, " FROM "); AppendTableName(buf, stmt); } static void AppendDropStatisticsStmt(StringInfo buf, List *nameList, bool ifExists) { appendStringInfoString(buf, "DROP STATISTICS "); if (ifExists) { appendStringInfoString(buf, "IF EXISTS "); } appendStringInfo(buf, "%s", NameListToQuotedString(nameList)); } static void AppendAlterStatisticsRenameStmt(StringInfo buf, RenameStmt *stmt) { appendStringInfo(buf, "ALTER STATISTICS %s RENAME TO %s", NameListToQuotedString((List *) stmt->object), quote_identifier( stmt->newname)); } static void AppendAlterStatisticsSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { appendStringInfo(buf, "ALTER STATISTICS %s SET SCHEMA %s", NameListToQuotedString((List *) stmt->object), quote_identifier( stmt->newschema)); } static void AppendAlterStatisticsStmt(StringInfo buf, AlterStatsStmt *stmt) { appendStringInfo(buf, "ALTER STATISTICS %s SET STATISTICS %d", NameListToQuotedString(stmt->defnames), getIntStxstattarget_compat(stmt->stxstattarget)); } static void AppendAlterStatisticsOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER STATISTICS %s OWNER TO %s", NameListToQuotedString(names), RoleSpecString(stmt->newowner, true)); } static void AppendStatisticsName(StringInfo buf, CreateStatsStmt *stmt) { String *schemaNameVal = (String *) linitial(stmt->defnames); const char *schemaName = quote_identifier(strVal(schemaNameVal)); String *statNameVal = (String *) lsecond(stmt->defnames); const char *statName = quote_identifier(strVal(statNameVal)); appendStringInfo(buf, "%s.%s", schemaName, statName); } static void AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) { if (list_length(stmt->stat_types) == 0) { return; } appendStringInfoString(buf, " ("); String *statType = NULL; foreach_declared_ptr(statType, stmt->stat_types) { appendStringInfoString(buf, strVal(statType)); if (statType != llast(stmt->stat_types)) { appendStringInfoString(buf, ", "); } } appendStringInfoString(buf, ")"); } /* See ruleutils.c in postgres for the logic here. */ static bool looks_like_function(Node *node) { if (node == NULL) { return false; /* probably shouldn't happen */ } switch (nodeTag(node)) { case T_FuncExpr: { /* OK, unless it's going to deparse as a cast */ return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); } case T_NullIfExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: { /* these are all accepted by func_expr_common_subexpr */ return true; } default: { break; } } return false; } static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { StatsElem *column = NULL; foreach_declared_ptr(column, stmt->exprs) { if (!column->name) { /* * Since these expressions are parser statements, we first call * transform to get the transformed Expr tree, and then deparse * the transformed tree. This is similar to the logic found in * deparse_table_stmts for check constraints. */ if (list_length(stmt->relations) != 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot use expressions in CREATE STATISTICS with multiple relations"))); } RangeVar *rel = (RangeVar *) linitial(stmt->relations); bool missingOk = false; Oid relOid = RangeVarGetRelid(rel, AccessShareLock, missingOk); /* Add table name to the name space in parse state. Otherwise column names * cannot be found. */ Relation relation = table_open(relOid, AccessShareLock); ParseState *pstate = make_parsestate(NULL); AddRangeTableEntryToQueryCompat(pstate, relation); Node *exprCooked = transformExpr(pstate, column->expr, EXPR_KIND_STATS_EXPRESSION); char *relationName = get_rel_name(relOid); List *relationCtx = deparse_context_for(relationName, relOid); char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); relation_close(relation, NoLock); /* Need parens if it's not a bare function call */ if (looks_like_function(exprCooked)) { appendStringInfoString(buf, exprSql); } else { appendStringInfo(buf, "(%s)", exprSql); } } else { const char *columnName = quote_identifier(column->name); appendStringInfoString(buf, columnName); } if (column != llast(stmt->exprs)) { appendStringInfoString(buf, ", "); } } } static void AppendTableName(StringInfo buf, CreateStatsStmt *stmt) { /* statistics' can be created with only one relation */ Assert(list_length(stmt->relations) == 1); RangeVar *relation = (RangeVar *) linitial(stmt->relations); char *relationName = relation->relname; char *schemaName = relation->schemaname; appendStringInfoString(buf, quote_qualified_identifier(schemaName, relationName)); } ================================================ FILE: src/backend/distributed/deparser/deparse_table_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_table_stmts.c * All routines to deparse table statements. * This file contains all entry points specific for table statement deparsing as well as * functions that are currently only used for deparsing of the table statements. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/heap.h" #include "commands/defrem.h" #include "commands/tablecmds.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "parser/parse_expr.h" #include "parser/parse_relation.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/namespace_utils.h" #include "distributed/version_compat.h" static void AppendAlterTableSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterTableStmt(StringInfo buf, AlterTableStmt *stmt); static void AppendAlterTableCmd(StringInfo buf, AlterTableCmd *alterTableCmd, AlterTableStmt *stmt); static void AppendAlterTableCmdAddColumn(StringInfo buf, AlterTableCmd *alterTableCmd, AlterTableStmt *stmt); static void AppendAlterTableCmdDropConstraint(StringInfo buf, AlterTableCmd *alterTableCmd); char * DeparseAlterTableSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); AppendAlterTableSchemaStmt(&str, stmt); return str.data; } static void AppendAlterTableSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); bool isForeignTable = stmt->objectType == OBJECT_FOREIGN_TABLE; appendStringInfo(buf, "ALTER %sTABLE ", isForeignTable ? "FOREIGN " : ""); if (stmt->missing_ok) { appendStringInfo(buf, "IF EXISTS "); } char *tableName = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); const char *newSchemaName = quote_identifier(stmt->newschema); appendStringInfo(buf, "%s SET SCHEMA %s;", tableName, newSchemaName); } /* * DeparseAlterTableStmt builds and returns a string representing the * AlterTableStmt where the object acted upon is of kind OBJECT_TABLE */ char * DeparseAlterTableStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objtype == OBJECT_TABLE); AppendAlterTableStmt(&str, stmt); return str.data; } /* * AppendAlterTableStmt builds and returns an SQL command representing an * ALTER TABLE statement from given AlterTableStmt object where the object * acted upon is of kind OBJECT_TABLE */ static void AppendAlterTableStmt(StringInfo buf, AlterTableStmt *stmt) { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); ListCell *cmdCell = NULL; Assert(stmt->objtype == OBJECT_TABLE); appendStringInfo(buf, "ALTER TABLE %s", identifier); foreach(cmdCell, stmt->cmds) { if (cmdCell != list_head(stmt->cmds)) { appendStringInfoString(buf, ", "); } AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(cmdCell)); AppendAlterTableCmd(buf, alterTableCmd, stmt); } appendStringInfoString(buf, ";"); } /* * AppendColumnNameList converts a list of columns into comma separated string format * (colname_1, colname_2, .., colname_n). */ void AppendColumnNameList(StringInfo buf, List *columns) { appendStringInfoString(buf, " ("); ListCell *lc; bool firstkey = true; foreach(lc, columns) { if (firstkey == false) { appendStringInfoString(buf, ", "); } appendStringInfo(buf, "%s", quote_identifier(strVal(lfirst(lc)))); firstkey = false; } appendStringInfoString(buf, " )"); } /* * AppendAlterTableCmdConstraint builds a string required to create given * constraint as part of an ADD CONSTRAINT or an ADD COLUMN subcommand, * and appends it to the buf. */ static void AppendAlterTableCmdConstraint(StringInfo buf, Constraint *constraint, AlterTableStmt *stmt, AlterTableType subtype) { if (subtype != AT_AddConstraint && subtype != AT_AddColumn) { ereport(ERROR, (errmsg("Unsupported alter table subtype: %d", (int) subtype))); } /* Need to deparse the alter table constraint command only if we are adding a constraint name.*/ if (constraint->conname == NULL) { ereport(ERROR, (errmsg( "Constraint name can not be NULL when deparsing the constraint."))); } if (subtype == AT_AddConstraint) { appendStringInfoString(buf, " ADD CONSTRAINT "); } else { appendStringInfoString(buf, " CONSTRAINT "); } appendStringInfo(buf, "%s ", quote_identifier(constraint->conname)); if (constraint->contype == CONSTR_PRIMARY || constraint->contype == CONSTR_UNIQUE) { if (constraint->contype == CONSTR_PRIMARY) { appendStringInfoString(buf, " PRIMARY KEY "); } else { appendStringInfoString(buf, " UNIQUE"); if (constraint->nulls_not_distinct == true) { appendStringInfoString(buf, " NULLS NOT DISTINCT"); } } if (subtype == AT_AddConstraint) { AppendColumnNameList(buf, constraint->keys); } if (constraint->including != NULL) { appendStringInfoString(buf, " INCLUDE "); AppendColumnNameList(buf, constraint->including); } if (constraint->options != NIL) { appendStringInfoString(buf, " WITH("); ListCell *defListCell; foreach(defListCell, constraint->options) { DefElem *def = (DefElem *) lfirst(defListCell); bool first = (defListCell == list_head(constraint->options)); appendStringInfo(buf, "%s%s=%s", first ? "" : ",", quote_identifier(def->defname), quote_literal_cstr(defGetString(def))); } appendStringInfoChar(buf, ')'); } } else if (constraint->contype == CONSTR_EXCLUSION) { /* * This block constructs the EXCLUDE clause which is in the following form: * EXCLUDE [ USING index_method ] ( exclude_element WITH operator [, ... ] ) */ appendStringInfoString(buf, " EXCLUDE "); if (constraint->access_method != NULL) { appendStringInfoString(buf, "USING "); appendStringInfo(buf, "%s ", quote_identifier( constraint->access_method)); } appendStringInfoString(buf, " ("); ListCell *lc; bool firstOp = true; foreach(lc, constraint->exclusions) { List *pair = (List *) lfirst(lc); Assert(list_length(pair) == 2); IndexElem *elem = linitial_node(IndexElem, pair); List *opname = lsecond_node(List, pair); if (firstOp == false) { appendStringInfoString(buf, " ,"); } ListCell *lc2; foreach(lc2, opname) { appendStringInfo(buf, "%s WITH %s", quote_identifier(elem->name), strVal(lfirst(lc2))); } firstOp = false; } appendStringInfoString(buf, " )"); } else if (constraint->contype == CONSTR_CHECK) { if (subtype == AT_AddColumn) { /* * Preprocess should've rejected deparsing such an ALTER TABLE * command but be on the safe side. */ ereport(ERROR, (errmsg("cannot add check constraint to column by " "using ADD COLUMN command"), errhint("Consider using ALTER TABLE ... ADD CONSTRAINT " "... CHECK command after adding the column"))); } LOCKMODE lockmode = AlterTableGetLockLevel(stmt->cmds); Oid leftRelationId = AlterTableLookupRelation(stmt, lockmode); /* To be able to use deparse_expression function, which creates an expression string, * the expression should be provided in its cooked form. We transform the raw expression * to cooked form. */ ParseState *pstate = make_parsestate(NULL); Relation relation = table_open(leftRelationId, AccessShareLock); /* Add table name to the name space in parse state. Otherwise column names * cannot be found. */ AddRangeTableEntryToQueryCompat(pstate, relation); Node *exprCooked = transformExpr(pstate, constraint->raw_expr, EXPR_KIND_CHECK_CONSTRAINT); char *relationName = get_rel_name(leftRelationId); List *relationCtx = deparse_context_for(relationName, leftRelationId); char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); relation_close(relation, NoLock); appendStringInfo(buf, " CHECK (%s)", exprSql); if (constraint->is_no_inherit) { appendStringInfo(buf, " NO INHERIT"); } } else if (constraint->contype == CONSTR_FOREIGN) { if (subtype == AT_AddConstraint) { appendStringInfoString(buf, " FOREIGN KEY"); AppendColumnNameList(buf, constraint->fk_attrs); } appendStringInfoString(buf, " REFERENCES"); appendStringInfo(buf, " %s", quote_qualified_identifier( constraint->pktable->schemaname, constraint->pktable->relname)); if (list_length(constraint->pk_attrs) > 0) { AppendColumnNameList(buf, constraint->pk_attrs); } /* Append supported options if provided */ /* FKCONSTR_MATCH_SIMPLE is default. Append matchtype if not default */ if (constraint->fk_matchtype == FKCONSTR_MATCH_FULL) { appendStringInfoString(buf, " MATCH FULL"); } switch (constraint->fk_del_action) { case FKCONSTR_ACTION_SETDEFAULT: { appendStringInfoString(buf, " ON DELETE SET DEFAULT"); break; } case FKCONSTR_ACTION_SETNULL: { appendStringInfoString(buf, " ON DELETE SET NULL"); break; } case FKCONSTR_ACTION_NOACTION: { appendStringInfoString(buf, " ON DELETE NO ACTION"); break; } case FKCONSTR_ACTION_RESTRICT: { appendStringInfoString(buf, " ON DELETE RESTRICT"); break; } case FKCONSTR_ACTION_CASCADE: { appendStringInfoString(buf, " ON DELETE CASCADE"); break; } default: { elog(ERROR, "unsupported FK delete action type: %d", (int) constraint->fk_del_action); break; } } switch (constraint->fk_upd_action) { case FKCONSTR_ACTION_SETDEFAULT: { appendStringInfoString(buf, " ON UPDATE SET DEFAULT"); break; } case FKCONSTR_ACTION_SETNULL: { appendStringInfoString(buf, " ON UPDATE SET NULL"); break; } case FKCONSTR_ACTION_NOACTION: { appendStringInfoString(buf, " ON UPDATE NO ACTION"); break; } case FKCONSTR_ACTION_RESTRICT: { appendStringInfoString(buf, " ON UPDATE RESTRICT"); break; } case FKCONSTR_ACTION_CASCADE: { appendStringInfoString(buf, " ON UPDATE CASCADE"); break; } default: { elog(ERROR, "unsupported FK update action type: %d", (int) constraint->fk_upd_action); break; } } } /* * For ADD CONSTRAINT subcommand, FOREIGN KEY and CHECK constraints migth * have NOT VALID option. * * Note that skip_validation might be true for an ADD COLUMN too but this * is not because Postgres supports this but because Citus sets this flag * to true for foreign key constraints added via ADD COLUMN. So we don't * check for skip_validation for ADD COLUMN subcommand. */ if (subtype == AT_AddConstraint && constraint->skip_validation) { appendStringInfoString(buf, " NOT VALID "); } if (subtype == AT_AddColumn && (constraint->deferrable || constraint->initdeferred)) { /* * For ADD COLUMN subcommand, the fact that whether given constraint * is deferrable or initially deferred is indicated by another Constraint * object, not via deferrable / initdeferred fields. */ ereport(ERROR, (errmsg("unexpected value set for deferrable/initdeferred " "field for an ADD COLUMN subcommand"))); } if (constraint->deferrable) { appendStringInfoString(buf, " DEFERRABLE"); if (constraint->initdeferred) { appendStringInfoString(buf, " INITIALLY DEFERRED"); } } } /* * AppendAlterTableCmd builds and appends to the given buffer a command * from given AlterTableCmd object. Currently supported commands are of type * AT_AddColumn, AT_SetNotNull and AT_AddConstraint {PRIMARY KEY, UNIQUE, EXCLUDE}. */ static void AppendAlterTableCmd(StringInfo buf, AlterTableCmd *alterTableCmd, AlterTableStmt *stmt) { switch (alterTableCmd->subtype) { case AT_AddColumn: { AppendAlterTableCmdAddColumn(buf, alterTableCmd, stmt); break; } case AT_DropConstraint: { AppendAlterTableCmdDropConstraint(buf, alterTableCmd); break; } case AT_AddConstraint: { Constraint *constraint = (Constraint *) alterTableCmd->def; /* We need to deparse ALTER TABLE ... ADD {PRIMARY KEY, UNIQUE, EXCLUSION} commands into * ALTER TABLE ... ADD CONSTRAINT {PRIMARY KEY, UNIQUE, EXCLUSION} ... format to be able * add a constraint name. */ if (ConstrTypeCitusCanDefaultName(constraint->contype)) { AppendAlterTableCmdConstraint(buf, constraint, stmt, AT_AddConstraint); break; } } /* fallthrough */ default: { ereport(ERROR, (errmsg("unsupported subtype for alter table command"), errdetail("sub command type: %d", alterTableCmd->subtype))); } } } /* * GeneratedWhenStr returns the char representation of given generated_when * value. */ static const char * GeneratedWhenStr(char generatedWhen) { switch (generatedWhen) { case 'a': { return "ALWAYS"; } case 'd': { return "BY DEFAULT"; } default: { ereport(ERROR, (errmsg("unrecognized generated_when: %d", generatedWhen))); } } } /* * DeparseRawExprForColumnDefault returns string representation of given * rawExpr based on given column type information. */ static char * DeparseRawExprForColumnDefault(Oid relationId, Oid columnTypeId, int32 columnTypeMod, char *columnName, char attgenerated, Node *rawExpr) { ParseState *pstate = make_parsestate(NULL); Relation relation = RelationIdGetRelation(relationId); AddRangeTableEntryToQueryCompat(pstate, relation); Node *defaultExpr = cookDefault(pstate, rawExpr, columnTypeId, columnTypeMod, columnName, attgenerated); List *deparseContext = deparse_context_for(get_rel_name(relationId), relationId); int saveNestLevel = PushEmptySearchPath(); char *defaultExprStr = deparse_expression(defaultExpr, deparseContext, false, false); PopEmptySearchPath(saveNestLevel); RelationClose(relation); return defaultExprStr; } /* * AppendAlterTableCmd builds and appends to the given buffer an AT_AddColumn command * from given AlterTableCmd object in the form ADD COLUMN ... */ static void AppendAlterTableCmdAddColumn(StringInfo buf, AlterTableCmd *alterTableCmd, AlterTableStmt *stmt) { Assert(alterTableCmd->subtype == AT_AddColumn); Oid relationId = AlterTableLookupRelation(stmt, NoLock); appendStringInfoString(buf, " ADD COLUMN "); if (alterTableCmd->missing_ok) { appendStringInfoString(buf, "IF NOT EXISTS "); } ColumnDef *columnDefinition = (ColumnDef *) alterTableCmd->def; if (columnDefinition->colname) { appendStringInfo(buf, "%s ", quote_identifier(columnDefinition->colname)); } int32 typmod = 0; Oid typeOid = InvalidOid; bits16 formatFlags = FORMAT_TYPE_TYPEMOD_GIVEN | FORMAT_TYPE_FORCE_QUALIFY; typenameTypeIdAndMod(NULL, columnDefinition->typeName, &typeOid, &typmod); appendStringInfo(buf, "%s", format_type_extended(typeOid, typmod, formatFlags)); if (columnDefinition->compression) { appendStringInfo(buf, " COMPRESSION %s", quote_identifier(columnDefinition->compression)); } Oid collationOid = GetColumnDefCollation(NULL, columnDefinition, typeOid); if (OidIsValid(collationOid)) { const char *identifier = FormatCollateBEQualified(collationOid); appendStringInfo(buf, " COLLATE %s", identifier); } ListCell *constraintCell = NULL; foreach(constraintCell, columnDefinition->constraints) { Constraint *constraint = (Constraint *) lfirst(constraintCell); if (constraint->contype == CONSTR_NOTNULL) { appendStringInfoString(buf, " NOT NULL"); } else if (constraint->contype == CONSTR_NULL) { appendStringInfoString(buf, " NULL"); } else if (constraint->contype == CONSTR_DEFAULT) { char attgenerated = '\0'; appendStringInfo(buf, " DEFAULT %s", DeparseRawExprForColumnDefault(relationId, typeOid, typmod, columnDefinition->colname, attgenerated, constraint->raw_expr)); } else if (constraint->contype == CONSTR_IDENTITY) { /* * Citus doesn't support adding identity columns via ALTER TABLE, * so we don't bother teaching the deparser about them. */ ereport(ERROR, (errmsg("unexpectedly found identity column " "definition in ALTER TABLE command"))); } else if (constraint->contype == CONSTR_GENERATED) { char attgenerated = ATTRIBUTE_GENERATED_STORED; #if PG_VERSION_NUM >= PG_VERSION_18 attgenerated = constraint->generated_kind; #endif appendStringInfo(buf, " GENERATED %s AS (%s) %s", GeneratedWhenStr(constraint->generated_when), DeparseRawExprForColumnDefault(relationId, typeOid, typmod, columnDefinition->colname, attgenerated, constraint->raw_expr), (attgenerated == ATTRIBUTE_GENERATED_STORED ? "STORED" : "VIRTUAL")); } else if (constraint->contype == CONSTR_CHECK || constraint->contype == CONSTR_PRIMARY || constraint->contype == CONSTR_UNIQUE || constraint->contype == CONSTR_EXCLUSION || constraint->contype == CONSTR_FOREIGN) { AppendAlterTableCmdConstraint(buf, constraint, stmt, AT_AddColumn); } else if (constraint->contype == CONSTR_ATTR_DEFERRABLE) { appendStringInfoString(buf, " DEFERRABLE"); } else if (constraint->contype == CONSTR_ATTR_NOT_DEFERRABLE) { appendStringInfoString(buf, " NOT DEFERRABLE"); } else if (constraint->contype == CONSTR_ATTR_DEFERRED) { appendStringInfoString(buf, " INITIALLY DEFERRED"); } else if (constraint->contype == CONSTR_ATTR_IMMEDIATE) { appendStringInfoString(buf, " INITIALLY IMMEDIATE"); } else { ereport(ERROR, (errmsg("unsupported constraint type"), errdetail("constraint type: %d", constraint->contype))); } } } /* * AppendAlterTableCmdDropConstraint builds and appends to the given buffer an * AT_DropConstraint command from given AlterTableCmd object in the form * DROP CONSTRAINT ... */ static void AppendAlterTableCmdDropConstraint(StringInfo buf, AlterTableCmd *alterTableCmd) { appendStringInfoString(buf, " DROP CONSTRAINT"); if (alterTableCmd->missing_ok) { appendStringInfoString(buf, " IF EXISTS"); } appendStringInfo(buf, " %s", quote_identifier(alterTableCmd->name)); if (alterTableCmd->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } ================================================ FILE: src/backend/distributed/deparser/deparse_text_search.c ================================================ /*------------------------------------------------------------------------- * * deparse_text_search.c * All routines to deparse text search statements. * This file contains all entry points specific for text search statement deparsing. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void AppendDefElemList(StringInfo buf, List *defelems, char *objectName); static void AppendStringInfoTokentypeList(StringInfo buf, List *tokentypes); static void AppendStringInfoDictnames(StringInfo buf, List *dicts); /* * DeparseCreateTextSearchConfigurationStmt returns the sql for a DefineStmt defining a * TEXT SEARCH CONFIGURATION * * Although the syntax is mutually exclusive on the two arguments that can be passed in * the deparser will syntactically correct multiple definitions if provided. * */ char * DeparseCreateTextSearchConfigurationStmt(Node *node) { DefineStmt *stmt = castNode(DefineStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); const char *identifier = NameListToQuotedString(stmt->defnames); appendStringInfo(&buf, "CREATE TEXT SEARCH CONFIGURATION %s ", identifier); appendStringInfoString(&buf, "("); AppendDefElemList(&buf, stmt->definition, "CONFIGURATION"); appendStringInfoString(&buf, ");"); return buf.data; } /* * DeparseCreateTextSearchDictionaryStmt returns the sql for a DefineStmt defining a * TEXT SEARCH DICTIONARY * * Although the syntax is mutually exclusive on the two arguments that can be passed in * the deparser will syntactically correct multiple definitions if provided. * */ char * DeparseCreateTextSearchDictionaryStmt(Node *node) { DefineStmt *stmt = castNode(DefineStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); const char *identifier = NameListToQuotedString(stmt->defnames); appendStringInfo(&buf, "CREATE TEXT SEARCH DICTIONARY %s ", identifier); appendStringInfoString(&buf, "("); AppendDefElemList(&buf, stmt->definition, "DICTIONARY"); appendStringInfoString(&buf, ");"); return buf.data; } /* * AppendDefElemList is a helper to append a comma separated list of definitions to a * define statement. * * The extra objectName parameter is used to create meaningful error messages. */ static void AppendDefElemList(StringInfo buf, List *defelems, char *objectName) { DefElem *defelem = NULL; bool first = true; foreach_declared_ptr(defelem, defelems) { if (!first) { appendStringInfoString(buf, ", "); } first = false; /* * There are some operations that can omit the argument. In that case, we only use * the defname. * * For example, omitting [ = value ] in the next query results in resetting the * option to defaults: * ALTER TEXT SEARCH DICTIONARY name ( option [ = value ] ); */ if (defelem->arg == NULL) { appendStringInfo(buf, "%s", defelem->defname); continue; } /* extract value from defelem */ const char *value = defGetString(defelem); /* stringify */ appendStringInfo(buf, "%s = %s", defelem->defname, value); } } /* * DeparseDropTextSearchConfigurationStmt returns the sql representation for a DROP TEXT * SEARCH CONFIGURATION ... statment. Supports dropping multiple configurations at once. */ char * DeparseDropTextSearchConfigurationStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_TSCONFIGURATION); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfoString(&buf, "DROP TEXT SEARCH CONFIGURATION "); List *nameList = NIL; bool first = true; foreach_declared_ptr(nameList, stmt->objects) { if (!first) { appendStringInfoString(&buf, ", "); } first = false; appendStringInfoString(&buf, NameListToQuotedString(nameList)); } if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(&buf, " CASCADE"); } appendStringInfoString(&buf, ";"); return buf.data; } /* * DeparseDropTextSearchDictionaryStmt returns the sql representation for a DROP TEXT SEARCH * DICTIONARY ... statment. Supports dropping multiple dictionaries at once. */ char * DeparseDropTextSearchDictionaryStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_TSDICTIONARY); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfoString(&buf, "DROP TEXT SEARCH DICTIONARY "); List *nameList = NIL; bool first = true; foreach_declared_ptr(nameList, stmt->objects) { if (!first) { appendStringInfoString(&buf, ", "); } first = false; appendStringInfoString(&buf, NameListToQuotedString(nameList)); } if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(&buf, " CASCADE"); } appendStringInfoString(&buf, ";"); return buf.data; } /* * DeparseRenameTextSearchConfigurationStmt returns the sql representation of a ALTER TEXT * SEARCH CONFIGURATION ... RENAME TO ... statement. */ char * DeparseRenameTextSearchConfigurationStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSCONFIGURATION); StringInfoData buf = { 0 }; initStringInfo(&buf); char *identifier = NameListToQuotedString(castNode(List, stmt->object)); appendStringInfo(&buf, "ALTER TEXT SEARCH CONFIGURATION %s RENAME TO %s;", identifier, quote_identifier(stmt->newname)); return buf.data; } /* * DeparseRenameTextSearchDictionaryStmt returns the sql representation of a ALTER TEXT SEARCH * DICTIONARY ... RENAME TO ... statement. */ char * DeparseRenameTextSearchDictionaryStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSDICTIONARY); StringInfoData buf = { 0 }; initStringInfo(&buf); char *identifier = NameListToQuotedString(castNode(List, stmt->object)); appendStringInfo(&buf, "ALTER TEXT SEARCH DICTIONARY %s RENAME TO %s;", identifier, quote_identifier(stmt->newname)); return buf.data; } /* * DeparseAlterTextSearchConfigurationStmt returns the sql representation of any generic * ALTER TEXT SEARCH CONFIGURATION .... statement. The statements supported include: * - ALTER TEXT SEARCH CONFIGURATIONS ... ADD MAPPING FOR [, ...] WITH [, ...] * - ALTER TEXT SEARCH CONFIGURATIONS ... ALTER MAPPING FOR [, ...] WITH [, ...] * - ALTER TEXT SEARCH CONFIGURATIONS ... ALTER MAPPING REPLACE ... WITH ... * - ALTER TEXT SEARCH CONFIGURATIONS ... ALTER MAPPING FOR [, ...] REPLACE ... WITH ... * - ALTER TEXT SEARCH CONFIGURATIONS ... DROP MAPPING [ IF EXISTS ] FOR ... */ char * DeparseAlterTextSearchConfigurationStmt(Node *node) { AlterTSConfigurationStmt *stmt = castNode(AlterTSConfigurationStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); char *identifier = NameListToQuotedString(castNode(List, stmt->cfgname)); appendStringInfo(&buf, "ALTER TEXT SEARCH CONFIGURATION %s", identifier); switch (stmt->kind) { case ALTER_TSCONFIG_ADD_MAPPING: { appendStringInfoString(&buf, " ADD MAPPING FOR "); AppendStringInfoTokentypeList(&buf, stmt->tokentype); appendStringInfoString(&buf, " WITH "); AppendStringInfoDictnames(&buf, stmt->dicts); break; } case ALTER_TSCONFIG_ALTER_MAPPING_FOR_TOKEN: { appendStringInfoString(&buf, " ALTER MAPPING FOR "); AppendStringInfoTokentypeList(&buf, stmt->tokentype); appendStringInfoString(&buf, " WITH "); AppendStringInfoDictnames(&buf, stmt->dicts); break; } case ALTER_TSCONFIG_REPLACE_DICT: case ALTER_TSCONFIG_REPLACE_DICT_FOR_TOKEN: { appendStringInfoString(&buf, " ALTER MAPPING"); if (list_length(stmt->tokentype) > 0) { appendStringInfoString(&buf, " FOR "); AppendStringInfoTokentypeList(&buf, stmt->tokentype); } if (list_length(stmt->dicts) != 2) { elog(ERROR, "unexpected number of dictionaries while deparsing ALTER " "TEXT SEARCH CONFIGURATION ... ALTER MAPPING [FOR ...] REPLACE " "statement."); } appendStringInfo(&buf, " REPLACE %s", NameListToQuotedString(linitial(stmt->dicts))); appendStringInfo(&buf, " WITH %s", NameListToQuotedString(lsecond(stmt->dicts))); break; } case ALTER_TSCONFIG_DROP_MAPPING: { appendStringInfoString(&buf, " DROP MAPPING"); if (stmt->missing_ok) { appendStringInfoString(&buf, " IF EXISTS"); } appendStringInfoString(&buf, " FOR "); AppendStringInfoTokentypeList(&buf, stmt->tokentype); break; } default: { elog(ERROR, "unable to deparse unsupported ALTER TEXT SEARCH STATEMENT"); } } appendStringInfoString(&buf, ";"); return buf.data; } /* * DeparseAlterTextSearchConfigurationStmt returns the sql representation of any generic * ALTER TEXT SEARCH DICTIONARY .... statement. The statements supported include * - ALTER TEXT SEARCH DICTIONARY name ( option [ = value ] [, ... ] ) */ char * DeparseAlterTextSearchDictionaryStmt(Node *node) { AlterTSDictionaryStmt *stmt = castNode(AlterTSDictionaryStmt, node); StringInfoData buf = { 0 }; initStringInfo(&buf); char *identifier = NameListToQuotedString(castNode(List, stmt->dictname)); appendStringInfo(&buf, "ALTER TEXT SEARCH DICTIONARY %s ( ", identifier); AppendDefElemList(&buf, stmt->options, "DICTIONARY"); appendStringInfoString(&buf, " );"); return buf.data; } /* * DeparseAlterTextSearchConfigurationSchemaStmt returns the sql statement representing * ALTER TEXT SEARCH CONFIGURATION ... SET SCHEMA ... statements. */ char * DeparseAlterTextSearchConfigurationSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSCONFIGURATION); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER TEXT SEARCH CONFIGURATION %s SET SCHEMA %s;", NameListToQuotedString(castNode(List, stmt->object)), quote_identifier(stmt->newschema)); return buf.data; } /* * DeparseAlterTextSearchDictionarySchemaStmt returns the sql statement representing ALTER TEXT * SEARCH DICTIONARY ... SET SCHEMA ... statements. */ char * DeparseAlterTextSearchDictionarySchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSDICTIONARY); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER TEXT SEARCH DICTIONARY %s SET SCHEMA %s;", NameListToQuotedString(castNode(List, stmt->object)), quote_identifier(stmt->newschema)); return buf.data; } /* * AppendStringInfoTokentypeList specializes in adding a comma separated list of * token_tyoe's to TEXT SEARCH CONFIGURATION commands */ static void AppendStringInfoTokentypeList(StringInfo buf, List *tokentypes) { String *tokentype = NULL; bool first = true; foreach_declared_ptr(tokentype, tokentypes) { if (nodeTag(tokentype) != T_String) { elog(ERROR, "unexpected tokentype for deparsing in text search configuration"); } if (!first) { appendStringInfoString(buf, ", "); } first = false; appendStringInfoString(buf, strVal(tokentype)); } } /* * AppendStringInfoDictnames specializes in appending a comma separated list of * dictionaries to TEXT SEARCH CONFIGURATION commands. */ static void AppendStringInfoDictnames(StringInfo buf, List *dicts) { List *dictNames = NIL; bool first = true; foreach_declared_ptr(dictNames, dicts) { if (!first) { appendStringInfoString(buf, ", "); } first = false; char *dictIdentifier = NameListToQuotedString(dictNames); appendStringInfoString(buf, dictIdentifier); } } /* * DeparseAlterTextSearchConfigurationOwnerStmt returns the sql statement representing * ALTER TEXT SEARCH CONFIGURATION ... ONWER TO ... commands. */ char * DeparseAlterTextSearchConfigurationOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TSCONFIGURATION); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER TEXT SEARCH CONFIGURATION %s OWNER TO %s;", NameListToQuotedString(castNode(List, stmt->object)), RoleSpecString(stmt->newowner, true)); return buf.data; } /* * DeparseAlterTextSearchDictionaryOwnerStmt returns the sql statement representing ALTER TEXT * SEARCH DICTIONARY ... ONWER TO ... commands. */ char * DeparseAlterTextSearchDictionaryOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TSDICTIONARY); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER TEXT SEARCH DICTIONARY %s OWNER TO %s;", NameListToQuotedString(castNode(List, stmt->object)), RoleSpecString(stmt->newowner, true)); return buf.data; } ================================================ FILE: src/backend/distributed/deparser/deparse_type_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_type_stmts.c * All routines to deparse type statements. * This file contains all entry points specific for type statement deparsing as well as * functions that are currently only used for deparsing of the type statements. * * Functions that could move later are AppendColumnDef, AppendColumnDefList, etc. These * should be reused across multiple statements and should live in their own deparse * file. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "parser/parse_type.h" #include "utils/builtins.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" #define AlterEnumIsRename(stmt) (stmt->oldVal != NULL) #define AlterEnumIsAddValue(stmt) (stmt->oldVal == NULL) /* forward declaration for deparse functions */ static void AppendCompositeTypeStmt(StringInfo str, CompositeTypeStmt *stmt); static void AppendColumnDef(StringInfo str, ColumnDef *columnDef); static void AppendColumnDefList(StringInfo str, List *columnDefs); static void AppendCreateEnumStmt(StringInfo str, CreateEnumStmt *stmt); static void AppendStringList(StringInfo str, List *strings); static void AppendDropTypeStmt(StringInfo buf, DropStmt *stmt); static void AppendTypeNameList(StringInfo buf, List *objects); static void AppendAlterEnumStmt(StringInfo buf, AlterEnumStmt *stmt); static void AppendAlterTypeStmt(StringInfo buf, AlterTableStmt *stmt); static void AppendAlterTypeCmd(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterTypeCmdAddColumn(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterTypeCmdDropColumn(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterTypeCmdAlterColumnType(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendRenameTypeStmt(StringInfo buf, RenameStmt *stmt); static void AppendRenameTypeAttributeStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterTypeSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); static void AppendAlterTypeOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); /* * DeparseCompositeTypeStmt builds and returns a string representing the * CompositeTypeStmt for application on a remote server. */ char * DeparseCompositeTypeStmt(Node *node) { CompositeTypeStmt *stmt = castNode(CompositeTypeStmt, node); StringInfoData sql = { 0 }; initStringInfo(&sql); AppendCompositeTypeStmt(&sql, stmt); return sql.data; } char * DeparseCreateEnumStmt(Node *node) { CreateEnumStmt *stmt = castNode(CreateEnumStmt, node); StringInfoData sql = { 0 }; initStringInfo(&sql); AppendCreateEnumStmt(&sql, stmt); return sql.data; } char * DeparseAlterEnumStmt(Node *node) { AlterEnumStmt *stmt = castNode(AlterEnumStmt, node); StringInfoData sql = { 0 }; initStringInfo(&sql); AppendAlterEnumStmt(&sql, stmt); return sql.data; } char * DeparseDropTypeStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->removeType == OBJECT_TYPE); AppendDropTypeStmt(&str, stmt); return str.data; } char * DeparseAlterTypeStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objtype == OBJECT_TYPE); AppendAlterTypeStmt(&str, stmt); return str.data; } static void AppendAlterTypeStmt(StringInfo buf, AlterTableStmt *stmt) { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); ListCell *cmdCell = NULL; Assert(stmt->objtype == OBJECT_TYPE); appendStringInfo(buf, "ALTER TYPE %s", identifier); foreach(cmdCell, stmt->cmds) { if (cmdCell != list_head(stmt->cmds)) { appendStringInfoString(buf, ", "); } AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(cmdCell)); AppendAlterTypeCmd(buf, alterTableCmd); } appendStringInfoString(buf, ";"); } static void AppendAlterTypeCmd(StringInfo buf, AlterTableCmd *alterTableCmd) { switch (alterTableCmd->subtype) { case AT_AddColumn: { AppendAlterTypeCmdAddColumn(buf, alterTableCmd); break; } case AT_DropColumn: { AppendAlterTypeCmdDropColumn(buf, alterTableCmd); break; } case AT_AlterColumnType: { AppendAlterTypeCmdAlterColumnType(buf, alterTableCmd); break; } default: { ereport(ERROR, (errmsg("unsupported subtype for alter table command"), errdetail("sub command type: %d", alterTableCmd->subtype))); } } } static void AppendAlterTypeCmdAddColumn(StringInfo buf, AlterTableCmd *alterTableCmd) { Assert(alterTableCmd->subtype == AT_AddColumn); appendStringInfoString(buf, " ADD ATTRIBUTE "); AppendColumnDef(buf, castNode(ColumnDef, alterTableCmd->def)); } static void AppendAlterTypeCmdDropColumn(StringInfo buf, AlterTableCmd *alterTableCmd) { Assert(alterTableCmd->subtype == AT_DropColumn); appendStringInfo(buf, " DROP ATTRIBUTE %s", quote_identifier(alterTableCmd->name)); if (alterTableCmd->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } static void AppendAlterTypeCmdAlterColumnType(StringInfo buf, AlterTableCmd *alterTableCmd) { Assert(alterTableCmd->subtype == AT_AlterColumnType); appendStringInfo(buf, " ALTER ATTRIBUTE %s SET DATA TYPE ", quote_identifier( alterTableCmd->name)); AppendColumnDef(buf, castNode(ColumnDef, alterTableCmd->def)); if (alterTableCmd->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } } static void AppendAlterEnumStmt(StringInfo buf, AlterEnumStmt *stmt) { appendStringInfo(buf, "ALTER TYPE %s", NameListToQuotedString(stmt->typeName)); if (AlterEnumIsRename(stmt)) { /* Rename an existing label */ appendStringInfo(buf, " RENAME VALUE %s TO %s;", quote_literal_cstr(stmt->oldVal), quote_literal_cstr(stmt->newVal)); } else if (AlterEnumIsAddValue(stmt)) { /* Add a new label */ appendStringInfoString(buf, " ADD VALUE "); if (stmt->skipIfNewValExists) { appendStringInfoString(buf, "IF NOT EXISTS "); } appendStringInfoString(buf, quote_literal_cstr(stmt->newVal)); if (stmt->newValNeighbor) { appendStringInfo(buf, " %s %s", stmt->newValIsAfter ? "AFTER" : "BEFORE", quote_literal_cstr(stmt->newValNeighbor)); } appendStringInfoString(buf, ";"); } } static void AppendDropTypeStmt(StringInfo buf, DropStmt *stmt) { /* * already tested at call site, but for future it might be collapsed in a * DeparseDropStmt so be safe and check again */ Assert(stmt->removeType == OBJECT_TYPE); appendStringInfo(buf, "DROP TYPE "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendTypeNameList(buf, stmt->objects); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } appendStringInfoString(buf, ";"); } static void AppendTypeNameList(StringInfo buf, List *objects) { ListCell *objectCell = NULL; foreach(objectCell, objects) { TypeName *typeName = castNode(TypeName, lfirst(objectCell)); Oid typeOid = LookupTypeNameOid(NULL, typeName, false); const char *identifier = format_type_be_qualified(typeOid); if (objectCell != list_head(objects)) { appendStringInfo(buf, ", "); } appendStringInfoString(buf, identifier); } } /* * AppendCompositeTypeStmt appends the sql string to recreate a CompositeTypeStmt to the * provided buffer, ending in a ; for concatination of multiple statements. */ static void AppendCompositeTypeStmt(StringInfo str, CompositeTypeStmt *stmt) { const char *identifier = quote_qualified_identifier(stmt->typevar->schemaname, stmt->typevar->relname); appendStringInfo(str, "CREATE TYPE %s AS (", identifier); AppendColumnDefList(str, stmt->coldeflist); appendStringInfo(str, ");"); } static void AppendCreateEnumStmt(StringInfo str, CreateEnumStmt *stmt) { RangeVar *typevar = makeRangeVarFromNameList(stmt->typeName); /* create the identifier from the fully qualified rangevar */ const char *identifier = quote_qualified_identifier(typevar->schemaname, typevar->relname); appendStringInfo(str, "CREATE TYPE %s AS ENUM (", identifier); AppendStringList(str, stmt->vals); appendStringInfo(str, ");"); } static void AppendStringList(StringInfo str, List *strings) { ListCell *stringCell = NULL; foreach(stringCell, strings) { const char *string = strVal(lfirst(stringCell)); if (stringCell != list_head(strings)) { appendStringInfoString(str, ", "); } string = quote_literal_cstr(string); appendStringInfoString(str, string); } } /* * AppendColumnDefList appends the definition of a list of ColumnDef items to the provided * buffer, adding separators as necessary. */ static void AppendColumnDefList(StringInfo str, List *columnDefs) { ListCell *columnDefCell = NULL; foreach(columnDefCell, columnDefs) { if (columnDefCell != list_head(columnDefs)) { appendStringInfoString(str, ", "); } AppendColumnDef(str, castNode(ColumnDef, lfirst(columnDefCell))); } } /* * AppendColumnDef appends the definition of one ColumnDef completely qualified to the * provided buffer. * * If the colname is not set that part is ommitted. This is the case in alter column type * statements. */ static void AppendColumnDef(StringInfo str, ColumnDef *columnDef) { int32 typmod = 0; Oid typeOid = InvalidOid; bits16 formatFlags = FORMAT_TYPE_TYPEMOD_GIVEN | FORMAT_TYPE_FORCE_QUALIFY; typenameTypeIdAndMod(NULL, columnDef->typeName, &typeOid, &typmod); Oid collationOid = GetColumnDefCollation(NULL, columnDef, typeOid); Assert(!columnDef->is_not_null); /* not null is not supported on composite types */ if (columnDef->colname) { appendStringInfo(str, "%s ", quote_identifier(columnDef->colname)); } appendStringInfo(str, "%s", format_type_extended(typeOid, typmod, formatFlags)); if (OidIsValid(collationOid)) { const char *identifier = FormatCollateBEQualified(collationOid); appendStringInfo(str, " COLLATE %s", identifier); } } char * DeparseRenameTypeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_TYPE); AppendRenameTypeStmt(&str, stmt); return str.data; } static void AppendRenameTypeStmt(StringInfo buf, RenameStmt *stmt) { List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER TYPE %s RENAME TO %s;", NameListToQuotedString(names), quote_identifier(stmt->newname)); } char * DeparseRenameTypeAttributeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->renameType == OBJECT_ATTRIBUTE); Assert(stmt->relationType == OBJECT_TYPE); AppendRenameTypeAttributeStmt(&str, stmt); return str.data; } static void AppendRenameTypeAttributeStmt(StringInfo buf, RenameStmt *stmt) { appendStringInfo(buf, "ALTER TYPE %s RENAME ATTRIBUTE %s TO %s", quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname), quote_identifier(stmt->subname), quote_identifier(stmt->newname)); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } appendStringInfoString(buf, ";"); } char * DeparseAlterTypeSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_TYPE); AppendAlterTypeSchemaStmt(&str, stmt); return str.data; } static void AppendAlterTypeSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { Assert(stmt->objectType == OBJECT_TYPE); List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER TYPE %s SET SCHEMA %s;", NameListToQuotedString(names), quote_identifier(stmt->newschema)); } char * DeparseAlterTypeOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_TYPE); AppendAlterTypeOwnerStmt(&str, stmt); return str.data; } static void AppendAlterTypeOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { Assert(stmt->objectType == OBJECT_TYPE); List *names = (List *) stmt->object; appendStringInfo(buf, "ALTER TYPE %s OWNER TO %s;", NameListToQuotedString(names), RoleSpecString(stmt->newowner, true)); } ================================================ FILE: src/backend/distributed/deparser/deparse_view_stmts.c ================================================ /*------------------------------------------------------------------------- * * deparse_view_stmts.c * * All routines to deparse view statements. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void AppendDropViewStmt(StringInfo buf, DropStmt *stmt); static void AppendViewNameList(StringInfo buf, List *objects); static void AppendAlterViewStmt(StringInfo buf, AlterTableStmt *stmt); static void AppendAlterViewCmd(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterViewOwnerStmt(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterViewSetOptionsStmt(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendAlterViewResetOptionsStmt(StringInfo buf, AlterTableCmd *alterTableCmd); static void AppendRenameViewStmt(StringInfo buf, RenameStmt *stmt); static void AppendAlterViewSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt); /* * DeparseDropViewStmt deparses the given DROP VIEW statement. */ char * DeparseDropViewStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->removeType == OBJECT_VIEW); AppendDropViewStmt(&str, stmt); return str.data; } /* * AppendDropViewStmt appends the deparsed representation of given drop stmt * to the given string info buffer. */ static void AppendDropViewStmt(StringInfo buf, DropStmt *stmt) { /* * already tested at call site, but for future it might be collapsed in a * DeparseDropStmt so be safe and check again */ Assert(stmt->removeType == OBJECT_VIEW); appendStringInfo(buf, "DROP VIEW "); if (stmt->missing_ok) { appendStringInfoString(buf, "IF EXISTS "); } AppendViewNameList(buf, stmt->objects); if (stmt->behavior == DROP_CASCADE) { appendStringInfoString(buf, " CASCADE"); } appendStringInfoString(buf, ";"); } /* * AppendViewNameList appends the qualified view names by constructing them from the given * objects list to the given string info buffer. Note that, objects must hold schema * qualified view names as its' members. */ static void AppendViewNameList(StringInfo buf, List *viewNamesList) { bool isFirstView = true; List *qualifiedViewName = NULL; foreach_declared_ptr(qualifiedViewName, viewNamesList) { char *quotedQualifiedVieName = NameListToQuotedString(qualifiedViewName); if (!isFirstView) { appendStringInfo(buf, ", "); } appendStringInfoString(buf, quotedQualifiedVieName); isFirstView = false; } } /* * DeparseAlterViewStmt deparses the given ALTER VIEW statement. */ char * DeparseAlterViewStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterViewStmt(&str, stmt); return str.data; } static void AppendAlterViewStmt(StringInfo buf, AlterTableStmt *stmt) { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); appendStringInfo(buf, "ALTER VIEW %s ", identifier); AlterTableCmd *alterTableCmd = castNode(AlterTableCmd, lfirst(list_head(stmt->cmds))); AppendAlterViewCmd(buf, alterTableCmd); appendStringInfoString(buf, ";"); } static void AppendAlterViewCmd(StringInfo buf, AlterTableCmd *alterTableCmd) { switch (alterTableCmd->subtype) { case AT_ChangeOwner: { AppendAlterViewOwnerStmt(buf, alterTableCmd); break; } case AT_SetRelOptions: { AppendAlterViewSetOptionsStmt(buf, alterTableCmd); break; } case AT_ResetRelOptions: { AppendAlterViewResetOptionsStmt(buf, alterTableCmd); break; } case AT_ColumnDefault: { elog(ERROR, "Citus doesn't support setting or resetting default values for a " "column of view"); break; } default: { /* * ALTER VIEW command only supports for the cases checked above but an * ALTER TABLE commands targeting views may have different cases. To let * PG throw the right error locally, we don't throw any error here */ break; } } } static void AppendAlterViewOwnerStmt(StringInfo buf, AlterTableCmd *alterTableCmd) { appendStringInfo(buf, "OWNER TO %s", RoleSpecString(alterTableCmd->newowner, true)); } static void AppendAlterViewSetOptionsStmt(StringInfo buf, AlterTableCmd *alterTableCmd) { ListCell *lc = NULL; bool initialOption = true; foreach(lc, (List *) alterTableCmd->def) { DefElem *def = (DefElem *) lfirst(lc); if (initialOption) { appendStringInfo(buf, "SET ("); initialOption = false; } else { appendStringInfo(buf, ","); } appendStringInfo(buf, "%s", def->defname); if (def->arg != NULL) { appendStringInfo(buf, "="); appendStringInfo(buf, "%s", defGetString(def)); } } appendStringInfo(buf, ")"); } static void AppendAlterViewResetOptionsStmt(StringInfo buf, AlterTableCmd *alterTableCmd) { ListCell *lc = NULL; bool initialOption = true; foreach(lc, (List *) alterTableCmd->def) { DefElem *def = (DefElem *) lfirst(lc); if (initialOption) { appendStringInfo(buf, "RESET ("); initialOption = false; } else { appendStringInfo(buf, ","); } appendStringInfo(buf, "%s", def->defname); } appendStringInfo(buf, ")"); } char * DeparseRenameViewStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendRenameViewStmt(&str, stmt); return str.data; } static void AppendRenameViewStmt(StringInfo buf, RenameStmt *stmt) { switch (stmt->renameType) { case OBJECT_COLUMN: { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); appendStringInfo(buf, "ALTER VIEW %s RENAME COLUMN %s TO %s;", identifier, quote_identifier(stmt->subname), quote_identifier( stmt->newname)); break; } case OBJECT_VIEW: { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); appendStringInfo(buf, "ALTER VIEW %s RENAME TO %s;", identifier, quote_identifier(stmt->newname)); break; } default: { ereport(ERROR, (errmsg("unsupported subtype for alter view rename command"), errdetail("sub command type: %d", stmt->renameType))); } } } char * DeparseAlterViewSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterViewSchemaStmt(&str, stmt); return str.data; } static void AppendAlterViewSchemaStmt(StringInfo buf, AlterObjectSchemaStmt *stmt) { const char *identifier = quote_qualified_identifier(stmt->relation->schemaname, stmt->relation->relname); appendStringInfo(buf, "ALTER VIEW %s SET SCHEMA %s;", identifier, quote_identifier( stmt->newschema)); } ================================================ FILE: src/backend/distributed/deparser/format_collate.c ================================================ /*------------------------------------------------------------------------- * * format_collate.c * Display collate names "nicely". * * This file is modeled after postgres' utils/adt/format_*.c files * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_collation.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/deparser.h" /* * This version returns a name that is always qualified. */ char * FormatCollateBEQualified(Oid collate_oid) { return FormatCollateExtended(collate_oid, FORMAT_COLLATE_FORCE_QUALIFY); } /* * FormatCollateExtended - inspired by format_type_extended * Generate a possibly-qualified collate name. * * The default behavior is to only qualify if the type is not in the search * path, and to raise an error if a non-existent collate_oid is given. * * The following bits in 'flags' modify the behavior: * - FORMAT_COLLATE_FORCE_QUALIFY * always schema-qualify collate names, regardless of search_path * * Returns a palloc'd string. */ char * FormatCollateExtended(Oid collid, bits16 flags) { char *nspname = NULL; if (collid == InvalidOid && (flags & FORMAT_COLLATE_ALLOW_INVALID) != 0) { return pstrdup("-"); } HeapTuple tuple = SearchSysCache1(COLLOID, ObjectIdGetDatum(collid)); if (!HeapTupleIsValid(tuple)) { if ((flags & FORMAT_COLLATE_ALLOW_INVALID) != 0) { return pstrdup("???"); } else { elog(ERROR, "cache lookup failed for collate %u", collid); } } Form_pg_collation collform = (Form_pg_collation) GETSTRUCT(tuple); if ((flags & FORMAT_COLLATE_FORCE_QUALIFY) == 0 && CollationIsVisible(collid)) { nspname = NULL; } else { nspname = get_namespace_name_or_temp(collform->collnamespace); } char *typname = NameStr(collform->collname); char *buf = quote_qualified_identifier(nspname, typname); ReleaseSysCache(tuple); return buf; } ================================================ FILE: src/backend/distributed/deparser/objectaddress.c ================================================ /*------------------------------------------------------------------------- * * objectaddress.c * Parstrees almost always target a object that postgres can address by * an ObjectAddress. Here we have a walker for parsetrees to find the * address of the object targeted. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/objectaddress.h" #include "catalog/pg_extension_d.h" #include "commands/extension.h" #include "distributed/commands.h" #include "distributed/deparser.h" /* * GetObjectAddressListFromParseTree returns the list of ObjectAddress of the main target of the parse * tree. */ List * GetObjectAddressListFromParseTree(Node *parseTree, bool missing_ok, bool isPostprocess) { const DistributeObjectOps *ops = GetDistributeObjectOps(parseTree); if (!ops->address) { ereport(ERROR, (errmsg("unsupported statement to get object address for"))); } return ops->address(parseTree, missing_ok, isPostprocess); } List * RenameAttributeStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); switch (stmt->relationType) { case OBJECT_TYPE: { return RenameTypeAttributeStmtObjectAddress(node, missing_ok); } default: { ereport(ERROR, (errmsg("unsupported alter rename attribute statement to get " "object address for"))); } } } /* * CreateExtensionStmtObjectAddress finds the ObjectAddress for the extension described * by the CreateExtensionStmt. If missing_ok is false, then this function throws an * error if the extension does not exist. * * Never returns NULL, but the objid in the address could be invalid if missing_ok was set * to true. */ List * CreateExtensionStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreateExtensionStmt *stmt = castNode(CreateExtensionStmt, node); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); const char *extensionName = stmt->extname; Oid extensionoid = get_extension_oid(extensionName, missing_ok); /* if we couldn't find the extension, error if missing_ok is false */ if (!missing_ok && extensionoid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("extension \"%s\" does not exist", extensionName))); } ObjectAddressSet(*address, ExtensionRelationId, extensionoid); return list_make1(address); } ================================================ FILE: src/backend/distributed/deparser/qualify.c ================================================ /*------------------------------------------------------------------------- * * qualify.c * The deparser relies on fully qualified names on all statements to * work transparently on a remote worker node. Here we have helpers to * fully qualify parsetrees. * * Fully qualified parsetrees contain names for all identifiers that * are search_path agnostic. Meaning we need to include the schema name * for each and every identifier in the parsetree. * * This file contains mostly the distpatching functions to specialized * functions for each class of objects. eg qualify_type_stmt.c contains * all functions related to fully qualifying parsetrees that interact * with types. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/commands.h" #include "distributed/deparser.h" /* * QualifyTreeNode transforms the statement in place and makes all (supported) statements * fully qualified. Fully qualified statements allow for application on a remote postgres * server irregardless of their search_path. */ void QualifyTreeNode(Node *stmt) { const DistributeObjectOps *ops = GetDistributeObjectOps(stmt); if (ops && ops->qualify) { ops->qualify(stmt); } } /* * QualifyRenameAttributeStmt transforms a RENAME ATTRIBUTE statement in place and makes all (supported) * statements fully qualified. */ void QualifyRenameAttributeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); switch (stmt->relationType) { case OBJECT_TYPE: { QualifyRenameTypeAttributeStmt(node); return; } default: { return; } } } ================================================ FILE: src/backend/distributed/deparser/qualify_aggregate_stmts.c ================================================ /*------------------------------------------------------------------------- * * qualify_aggregate_stmts.c * Functions specialized in fully qualifying all aggregate statements. * These functions are dispatched from qualify.c * * Fully qualifying aggregate statements consists of adding the schema name * to the subject of the types as well as any other branch of the parsetree. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "nodes/makefuncs.h" #include "utils/lsyscache.h" #include "distributed/deparser.h" void QualifyDefineAggregateStmt(Node *node) { DefineStmt *stmt = castNode(DefineStmt, node); if (list_length(stmt->defnames) == 1) { char *objname = NULL; Oid creationSchema = QualifiedNameGetCreationNamespace(stmt->defnames, &objname); stmt->defnames = list_make2(makeString(get_namespace_name(creationSchema)), linitial(stmt->defnames)); } } ================================================ FILE: src/backend/distributed/deparser/qualify_collation_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_collation_stmt.c * Functions specialized in fully qualifying all collation statements. These * functions are dispatched from qualify.c * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_collation.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static Node * QualifyCollationName(List *func); /* * QualifyRenameCollationStmt transforms a * ALTER COLLATION .. RENAME TO .. * statement in place and makes the collation name fully qualified. */ void QualifyRenameCollationStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_COLLATION); stmt->object = QualifyCollationName(castNode(List, stmt->object)); } /* * QualifyAlterCollationSchemaStmt transforms a * ALTER COLLATION .. SET SCHEMA .. * statement in place and makes the collation name fully qualified. */ void QualifyAlterCollationSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_COLLATION); stmt->object = QualifyCollationName(castNode(List, stmt->object)); } /* * QualifyAlterCollationOwnerStmt transforms a * ALTER COLLATION .. OWNER TO .. * statement in place and makes the collation name fully qualified. */ void QualifyAlterCollationOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_COLLATION); stmt->object = QualifyCollationName(castNode(List, stmt->object)); } /* * QualifyDropCollationStmt transforms a * DROP COLLATION .. * statement in place and makes the collation name fully qualified. */ void QualifyDropCollationStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); List *names = NIL; List *name = NIL; foreach_declared_ptr(name, stmt->objects) { names = lappend(names, QualifyCollationName(name)); } stmt->objects = names; } /* * QualifyCollation transforms a collation in place and makes its name fully qualified. */ Node * QualifyCollationName(List *name) { char *collationName = NULL; char *schemaName = NULL; /* check if the collation name is already qualified */ DeconstructQualifiedName(name, &schemaName, &collationName); /* do a lookup for the schema name if the statement does not include one */ if (schemaName == NULL) { Oid collid = get_collation_oid(name, true); if (collid == InvalidOid) { return (Node *) name; } HeapTuple colltup = SearchSysCache1(COLLOID, collid); if (!HeapTupleIsValid(colltup)) { return (Node *) name; } Form_pg_collation collationForm = (Form_pg_collation) GETSTRUCT(colltup); schemaName = get_namespace_name(collationForm->collnamespace); name = list_make2(makeString(schemaName), makeString(collationName)); ReleaseSysCache(colltup); } return (Node *) name; } ================================================ FILE: src/backend/distributed/deparser/qualify_domain.c ================================================ /*------------------------------------------------------------------------- * * qualify_domain.c * Functions to fully qualify, make the statements independent of * search_path settings, for all domain related statements. This * mostly consists of adding the schema name to all the domain * names referencing domains. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void QualifyTypeName(TypeName *typeName, bool missing_ok); static void QualifyCollate(CollateClause *collClause, bool missing_ok); /* * QualifyCreateDomainStmt modifies the CreateDomainStmt passed to become search_path * independent. */ void QualifyCreateDomainStmt(Node *node) { CreateDomainStmt *stmt = castNode(CreateDomainStmt, node); char *schemaName = NULL; char *domainName = NULL; /* fully qualify domain name */ DeconstructQualifiedName(stmt->domainname, &schemaName, &domainName); if (!schemaName) { RangeVar *var = makeRangeVarFromNameList(stmt->domainname); Oid creationSchema = RangeVarGetCreationNamespace(var); schemaName = get_namespace_name(creationSchema); stmt->domainname = list_make2(makeString(schemaName), makeString(domainName)); } /* referenced types should be fully qualified */ QualifyTypeName(stmt->typeName, false); QualifyCollate(stmt->collClause, false); } /* * QualifyDropDomainStmt modifies the DropStmt for DOMAIN's to be search_path independent. */ void QualifyDropDomainStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); TypeName *domainName = NULL; foreach_declared_ptr(domainName, stmt->objects) { QualifyTypeName(domainName, stmt->missing_ok); } } /* * QualifyAlterDomainStmt modifies the AlterDomainStmt to be search_path independent. */ void QualifyAlterDomainStmt(Node *node) { AlterDomainStmt *stmt = castNode(AlterDomainStmt, node); if (list_length(stmt->typeName) == 1) { TypeName *typeName = makeTypeNameFromNameList(stmt->typeName); QualifyTypeName(typeName, false); stmt->typeName = typeName->names; } } /* * QualifyDomainRenameConstraintStmt modifies the RenameStmt for domain constraints to be * search_path independent. */ void QualifyDomainRenameConstraintStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_DOMCONSTRAINT); List *domainName = castNode(List, stmt->object); if (list_length(domainName) == 1) { TypeName *typeName = makeTypeNameFromNameList(domainName); QualifyTypeName(typeName, false); stmt->object = (Node *) typeName->names; } } /* * QualifyAlterDomainOwnerStmt modifies the AlterOwnerStmt for DOMAIN's to be search_oath * independent. */ void QualifyAlterDomainOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_DOMAIN); List *domainName = castNode(List, stmt->object); if (list_length(domainName) == 1) { TypeName *typeName = makeTypeNameFromNameList(domainName); QualifyTypeName(typeName, false); stmt->object = (Node *) typeName->names; } } /* * QualifyRenameDomainStmt modifies the RenameStmt for the Domain to be search_path * independent. */ void QualifyRenameDomainStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_DOMAIN); List *domainName = castNode(List, stmt->object); if (list_length(domainName) == 1) { TypeName *typeName = makeTypeNameFromNameList(domainName); QualifyTypeName(typeName, false); stmt->object = (Node *) typeName->names; } } /* * QualifyAlterDomainSchemaStmt modifies the AlterObjectSchemaStmt to be search_path * independent. */ void QualifyAlterDomainSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_DOMAIN); List *domainName = castNode(List, stmt->object); if (list_length(domainName) == 1) { TypeName *typeName = makeTypeNameFromNameList(domainName); QualifyTypeName(typeName, false); stmt->object = (Node *) typeName->names; } } /* * QualifyTypeName qualifies a TypeName object in place. When missing_ok is false it might * throw an error if the type can't be found based on its name. When an oid is provided * missing_ok is ignored and treated as false. Meaning, even if missing_ok is true the * function might raise an error for non-existing types if the oid can't be found. */ static void QualifyTypeName(TypeName *typeName, bool missing_ok) { if (OidIsValid(typeName->typeOid)) { /* * When the typeName is provided as oid, fill in the names. * missing_ok is ignored for oid's */ Type typeTup = typeidType(typeName->typeOid); char *name = typeTypeName(typeTup); Oid namespaceOid = TypeOidGetNamespaceOid(typeName->typeOid); char *schemaName = get_namespace_name(namespaceOid); typeName->names = list_make2(makeString(schemaName), makeString(name)); ReleaseSysCache(typeTup); } else { char *name = NULL; char *schemaName = NULL; DeconstructQualifiedName(typeName->names, &schemaName, &name); if (!schemaName) { Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); if (OidIsValid(typeOid)) { Oid namespaceOid = TypeOidGetNamespaceOid(typeOid); schemaName = get_namespace_name(namespaceOid); typeName->names = list_make2(makeString(schemaName), makeString(name)); } } } } /* * QualifyCollate qualifies any given CollateClause by adding any missing schema name to * the collation being identified. * * If collClause is a NULL pointer this function is a no-nop. */ static void QualifyCollate(CollateClause *collClause, bool missing_ok) { if (collClause == NULL) { /* no collate clause, nothing to qualify*/ return; } if (list_length(collClause->collname) != 1) { /* already qualified */ return; } Oid collOid = get_collation_oid(collClause->collname, missing_ok); ObjectAddress collationAddress = { 0 }; ObjectAddressSet(collationAddress, CollationRelationId, collOid); List *objName = NIL; List *objArgs = NIL; getObjectIdentityParts(&collationAddress, &objName, &objArgs, false); collClause->collname = NIL; char *name = NULL; foreach_declared_ptr(name, objName) { collClause->collname = lappend(collClause->collname, makeString(name)); } } ================================================ FILE: src/backend/distributed/deparser/qualify_function_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_function_stmt.c * Functions specialized in fully qualifying all function statements. These * functions are dispatched from qualify.c * * Fully qualifying function statements consists of adding the schema name * to the subject of the function and types as well as any other branch of * the parsetree. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_proc.h" #include "parser/parse_func.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" /* forward declaration for qualify functions */ static void QualifyFunction(ObjectWithArgs *func, ObjectType type); static void QualifyFunctionSchemaName(ObjectWithArgs *func, ObjectType type); /* AssertObjectTypeIsFunctionType asserts we aren't receiving something we shouldn't */ void AssertObjectTypeIsFunctional(ObjectType type) { Assert(type == OBJECT_AGGREGATE || type == OBJECT_FUNCTION || type == OBJECT_PROCEDURE || type == OBJECT_ROUTINE); } /* * QualifyAlterFunctionStmt transforms a * ALTER {AGGREGATE|FUNCTION|PROCEDURE} .. * statement in place and makes all (supported) statements fully qualified. * * Note that not all queries of this form are valid AlterFunctionStmt * (e.g. ALTER FUNCTION .. RENAME .. queries are RenameStmt ) */ void QualifyAlterFunctionStmt(Node *node) { AlterFunctionStmt *stmt = castNode(AlterFunctionStmt, node); AssertObjectTypeIsFunctional(stmt->objtype); QualifyFunction(stmt->func, stmt->objtype); } /* * QualifyRenameFunctionStmt transforms a * ALTER {AGGREGATE|FUNCTION|PROCEDURE} .. RENAME TO .. * statement in place and makes the function name fully qualified. */ void QualifyRenameFunctionStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); AssertObjectTypeIsFunctional(stmt->renameType); QualifyFunction(castNode(ObjectWithArgs, stmt->object), stmt->renameType); } /* * QualifyAlterFunctionSchemaStmt transforms a * ALTER {AGGREGATE|FUNCTION|PROCEDURE} .. SET SCHEMA .. * statement in place and makes the function name fully qualified. */ void QualifyAlterFunctionSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); QualifyFunction(castNode(ObjectWithArgs, stmt->object), stmt->objectType); } /* * QualifyAlterFunctionOwnerStmt transforms a * ALTER {AGGREGATE|FUNCTION|PROCEDURE} .. OWNER TO .. * statement in place and makes the function name fully qualified. */ void QualifyAlterFunctionOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); QualifyFunction(castNode(ObjectWithArgs, stmt->object), stmt->objectType); } /* * QualifyAlterFunctionDependsStmt transforms a * ALTER {FUNCTION|PROCEDURE} .. DEPENDS ON EXTENSION .. * statement in place and makes the function name fully qualified. */ void QualifyAlterFunctionDependsStmt(Node *node) { AlterObjectDependsStmt *stmt = castNode(AlterObjectDependsStmt, node); AssertObjectTypeIsFunctional(stmt->objectType); QualifyFunction(castNode(ObjectWithArgs, stmt->object), stmt->objectType); } /* * QualifyFunction transforms a function in place and makes its name fully qualified. */ void QualifyFunction(ObjectWithArgs *func, ObjectType type) { char *functionName = NULL; char *schemaName = NULL; /* check if the function name is already qualified */ DeconstructQualifiedName(func->objname, &schemaName, &functionName); /* do a lookup for the schema name if the statement does not include one */ if (schemaName == NULL) { QualifyFunctionSchemaName(func, type); } } /* * QualifyFunction transforms a function in place using a catalog lookup for its schema name to make it fully qualified. */ void QualifyFunctionSchemaName(ObjectWithArgs *func, ObjectType type) { char *schemaName = NULL; char *functionName = NULL; Oid funcid = LookupFuncWithArgs(type, func, true); HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); /* * We can not qualify the function if the catalogs do not have any records. * * e.g. DROP FUNC IF EXISTS non_existent_func() do not result in a valid heap tuple */ if (HeapTupleIsValid(proctup)) { Form_pg_proc procform = (Form_pg_proc) GETSTRUCT(proctup); schemaName = get_namespace_name(procform->pronamespace); functionName = NameStr(procform->proname); functionName = pstrdup(functionName); /* we release the tuple before used */ ReleaseSysCache(proctup); /* update the function using the schema name */ func->objname = list_make2(makeString(schemaName), makeString(functionName)); } } ================================================ FILE: src/backend/distributed/deparser/qualify_publication_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_publication_stmt.c * Functions specialized in fully qualifying all publication statements. These * functions are dispatched from qualify.c * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "nodes/nodes.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void QualifyPublicationObjects(List *publicationObjects); static void QualifyPublicationRangeVar(RangeVar *publication); /* * QualifyCreatePublicationStmt quailifies the publication names of the * CREATE PUBLICATION statement. */ void QualifyCreatePublicationStmt(Node *node) { CreatePublicationStmt *stmt = castNode(CreatePublicationStmt, node); QualifyPublicationObjects(stmt->pubobjects); } /* * QualifyPublicationObjects ensures all table names in a list of * publication objects are fully qualified. */ static void QualifyPublicationObjects(List *publicationObjects) { PublicationObjSpec *publicationObject = NULL; foreach_declared_ptr(publicationObject, publicationObjects) { if (publicationObject->pubobjtype == PUBLICATIONOBJ_TABLE) { /* FOR TABLE ... */ PublicationTable *publicationTable = publicationObject->pubtable; QualifyPublicationRangeVar(publicationTable->relation); } } } /* * QualifyPublicationObjects ensures all table names in a list of * publication objects are fully qualified. */ void QualifyAlterPublicationStmt(Node *node) { AlterPublicationStmt *stmt = castNode(AlterPublicationStmt, node); QualifyPublicationObjects(stmt->pubobjects); } /* * QualifyPublicationRangeVar qualifies the given publication RangeVar if it is not qualified. */ static void QualifyPublicationRangeVar(RangeVar *publication) { if (publication->schemaname == NULL) { Oid publicationOid = RelnameGetRelid(publication->relname); Oid schemaOid = get_rel_namespace(publicationOid); publication->schemaname = get_namespace_name(schemaOid); } } ================================================ FILE: src/backend/distributed/deparser/qualify_role_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_role_stmt.c * Functions specialized in fully qualifying all role statements. These * functions are dispatched from qualify.c * * Fully qualifying role statements consists of adding the current database * name, session user, current use, and current configuration values. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/nodes.h" #include "utils/guc.h" #include "distributed/deparser.h" static void QualifyVarSetCurrent(VariableSetStmt *setStmt); /* * QualifyAlterRoleSetStmt transforms a * ALTER ROLE .. SET .. * statement in place and makes the settings fully qualified. */ void QualifyAlterRoleSetStmt(Node *node) { AlterRoleSetStmt *stmt = castNode(AlterRoleSetStmt, node); VariableSetStmt *setStmt = stmt->setstmt; if (setStmt->kind == VAR_SET_CURRENT) { QualifyVarSetCurrent(setStmt); } } /* * QualifyVarSetCurrent transforms a * FROM CURRENT * into a * SET config_name TO 'config_value' * VariableSetStmt in place and hence makes it fully qualified. */ static void QualifyVarSetCurrent(VariableSetStmt *setStmt) { char *configurationName = setStmt->name; char *configValue = GetConfigOptionByName(configurationName, NULL, false); setStmt->kind = VAR_SET_VALUE; setStmt->args = list_make1(MakeSetStatementArguments(configurationName, configValue)); } ================================================ FILE: src/backend/distributed/deparser/qualify_sequence_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_sequence_stmt.c * Functions specialized in fully qualifying all sequence statements. These * functions are dispatched from qualify.c * * Fully qualifying sequence statements consists of adding the schema name * to the subject of the sequence. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "parser/parse_func.h" #include "utils/lsyscache.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/version_compat.h" /* * QualifyAlterSequenceOwnerStmt transforms a * ALTER SEQUENCE .. OWNER TO .. * statement in place and makes the sequence name fully qualified. */ void QualifyAlterSequenceOwnerStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; if (seq->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); if (OidIsValid(seqOid)) { Oid schemaOid = get_rel_namespace(seqOid); seq->schemaname = get_namespace_name(schemaOid); } } } /* * QualifyAlterSequencePersistenceStmt transforms a * ALTER SEQUENCE .. SET LOGGED/UNLOGGED * statement in place and makes the sequence name fully qualified. */ void QualifyAlterSequencePersistenceStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; if (seq->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); if (OidIsValid(seqOid)) { Oid schemaOid = get_rel_namespace(seqOid); seq->schemaname = get_namespace_name(schemaOid); } } } /* * QualifyAlterSequenceSchemaStmt transforms a * ALTER SEQUENCE .. SET SCHEMA .. * statement in place and makes the sequence name fully qualified. */ void QualifyAlterSequenceSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; if (seq->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); if (OidIsValid(seqOid)) { Oid schemaOid = get_rel_namespace(seqOid); seq->schemaname = get_namespace_name(schemaOid); } } } /* * QualifyRenameSequenceStmt transforms a * ALTER SEQUENCE .. RENAME TO .. * statement in place and makes the sequence name fully qualified. */ void QualifyRenameSequenceStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; if (seq->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); if (OidIsValid(seqOid)) { Oid schemaOid = get_rel_namespace(seqOid); seq->schemaname = get_namespace_name(schemaOid); } } } /* * QualifyDropSequenceStmt transforms a DROP SEQUENCE * statement in place and makes the sequence name fully qualified. */ void QualifyDropSequenceStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_SEQUENCE); List *objectNameListWithSchema = NIL; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, stmt->objects) { RangeVar *seq = makeRangeVarFromNameList(objectNameList); if (seq->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(seq, NoLock, stmt->missing_ok); if (OidIsValid(seqOid)) { Oid schemaOid = get_rel_namespace(seqOid); seq->schemaname = get_namespace_name(schemaOid); } } objectNameListWithSchema = lappend(objectNameListWithSchema, MakeNameListFromRangeVar(seq)); } stmt->objects = objectNameListWithSchema; } /* * QualifyGrantOnSequenceStmt transforms a * GRANT ON SEQUENCE ... * statement in place and makes the sequence names fully qualified. */ void QualifyGrantOnSequenceStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_SEQUENCE); /* * The other option would be GRANT ALL SEQUENCES ON SCHEMA ... * For that we don't need to qualify */ if (stmt->targtype != ACL_TARGET_OBJECT) { return; } List *qualifiedSequenceRangeVars = NIL; RangeVar *sequenceRangeVar = NULL; foreach_declared_ptr(sequenceRangeVar, stmt->objects) { if (sequenceRangeVar->schemaname == NULL) { Oid seqOid = RangeVarGetRelid(sequenceRangeVar, NoLock, false); Oid schemaOid = get_rel_namespace(seqOid); sequenceRangeVar->schemaname = get_namespace_name(schemaOid); } qualifiedSequenceRangeVars = lappend(qualifiedSequenceRangeVars, sequenceRangeVar); } stmt->objects = qualifiedSequenceRangeVars; } ================================================ FILE: src/backend/distributed/deparser/qualify_statistics_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_statistics_stmt.c * Functions specialized in fully qualifying all statistics statements. * These functions are dispatched from qualify.c * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "catalog/pg_statistic_ext.h" #include "nodes/parsenodes.h" #include "nodes/value.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static Oid GetStatsNamespaceOid(Oid statsOid); void QualifyCreateStatisticsStmt(Node *node) { CreateStatsStmt *stmt = castNode(CreateStatsStmt, node); Node *relationNode = (Node *) linitial(stmt->relations); if (!IsA(relationNode, RangeVar)) { return; } RangeVar *relation = (RangeVar *) relationNode; if (relation->schemaname == NULL) { Oid tableOid = RelnameGetRelid(relation->relname); Oid schemaOid = get_rel_namespace(tableOid); relation->schemaname = get_namespace_name(schemaOid); } if (list_length(stmt->defnames) < 1) { /* no name to qualify */ return; } RangeVar *stat = makeRangeVarFromNameList(stmt->defnames); if (stat->schemaname == NULL) { Oid schemaOid = RangeVarGetCreationNamespace(stat); stat->schemaname = get_namespace_name(schemaOid); stmt->defnames = MakeNameListFromRangeVar(stat); } } /* * QualifyDropStatisticsStmt qualifies DropStmt's with schema name for * DROP STATISTICS statements. */ void QualifyDropStatisticsStmt(Node *node) { DropStmt *dropStatisticsStmt = castNode(DropStmt, node); Assert(dropStatisticsStmt->removeType == OBJECT_STATISTIC_EXT); List *objectNameListWithSchema = NIL; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, dropStatisticsStmt->objects) { RangeVar *stat = makeRangeVarFromNameList(objectNameList); if (stat->schemaname == NULL) { Oid statsOid = get_statistics_object_oid(objectNameList, dropStatisticsStmt->missing_ok); if (OidIsValid(statsOid)) { Oid schemaOid = GetStatsNamespaceOid(statsOid); stat->schemaname = get_namespace_name(schemaOid); } } objectNameListWithSchema = lappend(objectNameListWithSchema, MakeNameListFromRangeVar(stat)); } dropStatisticsStmt->objects = objectNameListWithSchema; } /* * QualifyAlterStatisticsRenameStmt qualifies RenameStmt's with schema name for * ALTER STATISTICS RENAME statements. */ void QualifyAlterStatisticsRenameStmt(Node *node) { RenameStmt *renameStmt = castNode(RenameStmt, node); Assert(renameStmt->renameType == OBJECT_STATISTIC_EXT); List *nameList = (List *) renameStmt->object; if (list_length(nameList) == 1) { RangeVar *stat = makeRangeVarFromNameList(nameList); Oid statsOid = get_statistics_object_oid(nameList, renameStmt->missing_ok); if (!OidIsValid(statsOid)) { return; } Oid schemaOid = GetStatsNamespaceOid(statsOid); stat->schemaname = get_namespace_name(schemaOid); renameStmt->object = (Node *) MakeNameListFromRangeVar(stat); } } /* * QualifyAlterStatisticsSchemaStmt qualifies AlterObjectSchemaStmt's with schema name for * ALTER STATISTICS RENAME statements. */ void QualifyAlterStatisticsSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); List *nameList = (List *) stmt->object; if (list_length(nameList) == 1) { RangeVar *stat = makeRangeVarFromNameList(nameList); Oid statsOid = get_statistics_object_oid(nameList, stmt->missing_ok); if (!OidIsValid(statsOid)) { return; } Oid schemaOid = GetStatsNamespaceOid(statsOid); stat->schemaname = get_namespace_name(schemaOid); stmt->object = (Node *) MakeNameListFromRangeVar(stat); } } /* * QualifyAlterStatisticsStmt qualifies AlterStatsStmt's with schema name for * ALTER STATISTICS .. SET STATISTICS statements. */ void QualifyAlterStatisticsStmt(Node *node) { AlterStatsStmt *stmt = castNode(AlterStatsStmt, node); if (list_length(stmt->defnames) == 1) { RangeVar *stat = makeRangeVarFromNameList(stmt->defnames); Oid statsOid = get_statistics_object_oid(stmt->defnames, stmt->missing_ok); if (!OidIsValid(statsOid)) { return; } Oid schemaOid = GetStatsNamespaceOid(statsOid); stat->schemaname = get_namespace_name(schemaOid); stmt->defnames = MakeNameListFromRangeVar(stat); } } /* * QualifyAlterStatisticsOwnerStmt qualifies AlterOwnerStmt's with schema * name for ALTER STATISTICS .. OWNER TO statements. */ void QualifyAlterStatisticsOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_STATISTIC_EXT); List *nameList = (List *) stmt->object; if (list_length(nameList) == 1) { RangeVar *stat = makeRangeVarFromNameList(nameList); Oid statsOid = get_statistics_object_oid(nameList, /* missing_ok */ true); if (!OidIsValid(statsOid)) { return; } Oid schemaOid = GetStatsNamespaceOid(statsOid); stat->schemaname = get_namespace_name(schemaOid); stmt->object = (Node *) MakeNameListFromRangeVar(stat); } } /* * GetStatsNamespaceOid takes the id of a Statistics object and returns * the id of the schema that the statistics object belongs to. * Errors out if the stats object is not found. */ static Oid GetStatsNamespaceOid(Oid statsOid) { HeapTuple heapTuple = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statsOid)); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("cache lookup failed for statistics " "object with oid %u", statsOid))); } FormData_pg_statistic_ext *statisticsForm = (FormData_pg_statistic_ext *) GETSTRUCT(heapTuple); Oid result = statisticsForm->stxnamespace; ReleaseSysCache(heapTuple); return result; } ================================================ FILE: src/backend/distributed/deparser/qualify_table_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_table_stmt.c * Functions specialized in fully qualifying all table statements. These * functions are dispatched from qualify.c * * Fully qualifying table statements consists of adding the schema name * to the subject of the table as well as any other branch of the * parsetree. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "nodes/parsenodes.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "distributed/deparser.h" void QualifyAlterTableSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TABLE || stmt->objectType == OBJECT_FOREIGN_TABLE); if (stmt->relation->schemaname == NULL) { Oid tableOid = RelnameGetRelid(stmt->relation->relname); Oid schemaOid = get_rel_namespace(tableOid); stmt->relation->schemaname = get_namespace_name(schemaOid); } } ================================================ FILE: src/backend/distributed/deparser/qualify_text_search_stmts.c ================================================ /*------------------------------------------------------------------------- * * qualify_text_search_stmts.c * Functions specialized in fully qualifying all text search statements. These * functions are dispatched from qualify.c * * Fully qualifying text search statements consists of adding the schema name * to the subject of the types as well as any other branch of the parsetree. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static Oid get_ts_config_namespace(Oid tsconfigOid); static Oid get_ts_dict_namespace(Oid tsdictOid); /* * QualifyDropTextSearchConfigurationStmt adds any missing schema names to text search * configurations being dropped. All configurations are expected to exists before fully * qualifying the statement. Errors will be raised for objects not existing. Non-existing * objects are expected to not be distributed. */ void QualifyDropTextSearchConfigurationStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_TSCONFIGURATION); List *qualifiedObjects = NIL; List *objName = NIL; foreach_declared_ptr(objName, stmt->objects) { char *schemaName = NULL; char *tsconfigName = NULL; DeconstructQualifiedName(objName, &schemaName, &tsconfigName); if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(objName, stmt->missing_ok); if (OidIsValid(tsconfigOid)) { Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); objName = list_make2(makeString(schemaName), makeString(tsconfigName)); } } qualifiedObjects = lappend(qualifiedObjects, objName); } stmt->objects = qualifiedObjects; } /* * QualifyDropTextSearchDictionaryStmt adds any missing schema names to text search * dictionaries being dropped. All dictionaries are expected to exists before fully * qualifying the statement. Errors will be raised for objects not existing. Non-existing * objects are expected to not be distributed. */ void QualifyDropTextSearchDictionaryStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); Assert(stmt->removeType == OBJECT_TSDICTIONARY); List *qualifiedObjects = NIL; List *objName = NIL; foreach_declared_ptr(objName, stmt->objects) { char *schemaName = NULL; char *tsdictName = NULL; DeconstructQualifiedName(objName, &schemaName, &tsdictName); if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(objName, stmt->missing_ok); if (OidIsValid(tsdictOid)) { Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); objName = list_make2(makeString(schemaName), makeString(tsdictName)); } } qualifiedObjects = lappend(qualifiedObjects, objName); } stmt->objects = qualifiedObjects; } /* * QualifyAlterTextSearchConfigurationStmt adds the schema name (if missing) to the name * of the text search configurations, as well as the dictionaries referenced. */ void QualifyAlterTextSearchConfigurationStmt(Node *node) { AlterTSConfigurationStmt *stmt = castNode(AlterTSConfigurationStmt, node); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(stmt->cfgname, &schemaName, &objName); /* fully qualify the cfgname being altered */ if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(stmt->cfgname, false); Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); stmt->cfgname = list_make2(makeString(schemaName), makeString(objName)); } /* fully qualify the dicts */ bool useNewDicts = false; List *dicts = NULL; List *dictName = NIL; foreach_declared_ptr(dictName, stmt->dicts) { DeconstructQualifiedName(dictName, &schemaName, &objName); /* fully qualify the cfgname being altered */ if (!schemaName) { Oid dictOid = get_ts_dict_oid(dictName, false); Oid namespaceOid = get_ts_dict_namespace(dictOid); schemaName = get_namespace_name(namespaceOid); useNewDicts = true; dictName = list_make2(makeString(schemaName), makeString(objName)); } dicts = lappend(dicts, dictName); } if (useNewDicts) { /* swap original dicts with the new list */ stmt->dicts = dicts; } else { /* we don't use the new list, everything was already qualified, free-ing */ list_free(dicts); } } /* * QualifyAlterTextSearchDictionaryStmt adds the schema name (if missing) to the name * of the text search dictionary. */ void QualifyAlterTextSearchDictionaryStmt(Node *node) { AlterTSDictionaryStmt *stmt = castNode(AlterTSDictionaryStmt, node); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(stmt->dictname, &schemaName, &objName); /* fully qualify the dictname being altered */ if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(stmt->dictname, false); Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); stmt->dictname = list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyRenameTextSearchConfigurationStmt adds the schema name (if missing) to the * configuration being renamed. The new name will kept be without schema name since this * command cannot be used to change the schema of a configuration. */ void QualifyRenameTextSearchConfigurationStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSCONFIGURATION); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); /* fully qualify the cfgname being altered */ if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyRenameTextSearchDictionaryStmt adds the schema name (if missing) to the * dictionary being renamed. The new name will kept be without schema name since this * command cannot be used to change the schema of a dictionary. */ void QualifyRenameTextSearchDictionaryStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_TSDICTIONARY); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); /* fully qualify the dictname being altered */ if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyAlterTextSearchConfigurationSchemaStmt adds the schema name (if missing) for the * text search config being moved to a new schema. */ void QualifyAlterTextSearchConfigurationSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSCONFIGURATION); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyAlterTextSearchDictionarySchemaStmt adds the schema name (if missing) for the * text search dictionary being moved to a new schema. */ void QualifyAlterTextSearchDictionarySchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TSDICTIONARY); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyTextSearchConfigurationCommentStmt adds the schema name (if missing) to the * configuration name on which the comment is created. */ void QualifyTextSearchConfigurationCommentStmt(Node *node) { CommentStmt *stmt = castNode(CommentStmt, node); Assert(stmt->objtype == OBJECT_TSCONFIGURATION); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyTextSearchDictionaryCommentStmt adds the schema name (if missing) to the * dictionary name on which the comment is created. */ void QualifyTextSearchDictionaryCommentStmt(Node *node) { CommentStmt *stmt = castNode(CommentStmt, node); Assert(stmt->objtype == OBJECT_TSDICTIONARY); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyAlterTextSearchConfigurationOwnerStmt adds the schema name (if missing) to the * configuration for which the owner is changing. */ void QualifyAlterTextSearchConfigurationOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TSCONFIGURATION); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsconfigOid = get_ts_config_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_config_namespace(tsconfigOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * QualifyAlterTextSearchDictionaryOwnerStmt adds the schema name (if missing) to the * dictionary for which the owner is changing. */ void QualifyAlterTextSearchDictionaryOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TSDICTIONARY); char *schemaName = NULL; char *objName = NULL; DeconstructQualifiedName(castNode(List, stmt->object), &schemaName, &objName); if (!schemaName) { Oid tsdictOid = get_ts_dict_oid(castNode(List, stmt->object), false); Oid namespaceOid = get_ts_dict_namespace(tsdictOid); schemaName = get_namespace_name(namespaceOid); stmt->object = (Node *) list_make2(makeString(schemaName), makeString(objName)); } } /* * get_ts_config_namespace returns the oid of the namespace which is housing the text * search configuration identified by tsconfigOid. */ static Oid get_ts_config_namespace(Oid tsconfigOid) { HeapTuple tup = SearchSysCache1(TSCONFIGOID, ObjectIdGetDatum(tsconfigOid)); if (HeapTupleIsValid(tup)) { Form_pg_ts_config cfgform = (Form_pg_ts_config) GETSTRUCT(tup); Oid namespaceOid = cfgform->cfgnamespace; ReleaseSysCache(tup); return namespaceOid; } return InvalidOid; } /* * get_ts_dict_namespace returns the oid of the namespace which is housing the text * search dictionary identified by tsdictOid. */ static Oid get_ts_dict_namespace(Oid tsdictOid) { HeapTuple tup = SearchSysCache1(TSDICTOID, ObjectIdGetDatum(tsdictOid)); if (HeapTupleIsValid(tup)) { Form_pg_ts_dict cfgform = (Form_pg_ts_dict) GETSTRUCT(tup); Oid namespaceOid = cfgform->dictnamespace; ReleaseSysCache(tup); return namespaceOid; } return InvalidOid; } ================================================ FILE: src/backend/distributed/deparser/qualify_type_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_type_stmt.c * Functions specialized in fully qualifying all type statements. These * functions are dispatched from qualify.c * * Fully qualifying type statements consists of adding the schema name * to the subject of the types as well as any other branch of the * parsetree. * * Goal would be that the deparser functions for these statements can * serialize the statement without any external lookups. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/heapam.h" #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/objectaddress.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/version_compat.h" /* * GetTypeNamespaceNameByNameList resolved the schema name of a type by its namelist. */ char * GetTypeNamespaceNameByNameList(List *names) { TypeName *typeName = makeTypeNameFromNameList(names); Oid typeOid = LookupTypeNameOid(NULL, typeName, false); Oid namespaceOid = TypeOidGetNamespaceOid(typeOid); char *nspname = get_namespace_name_or_temp(namespaceOid); return nspname; } /* * TypeOidGetNamespaceOid resolves the namespace oid for a type identified by its type oid */ Oid TypeOidGetNamespaceOid(Oid typeOid) { HeapTuple typeTuple = SearchSysCache1(TYPEOID, typeOid); if (!HeapTupleIsValid(typeTuple)) { elog(ERROR, "citus cache lookup failed"); return InvalidOid; } Form_pg_type typeData = (Form_pg_type) GETSTRUCT(typeTuple); Oid typnamespace = typeData->typnamespace; ReleaseSysCache(typeTuple); return typnamespace; } void QualifyRenameTypeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); List *names = (List *) stmt->object; Assert(stmt->renameType == OBJECT_TYPE); if (list_length(names) == 1) { /* not qualified, lookup name and add namespace name to names */ char *nspname = GetTypeNamespaceNameByNameList(names); names = list_make2(makeString(nspname), linitial(names)); stmt->object = (Node *) names; } } void QualifyRenameTypeAttributeStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); Assert(stmt->renameType == OBJECT_ATTRIBUTE); Assert(stmt->relationType == OBJECT_TYPE); if (stmt->relation->schemaname == NULL) { List *names = list_make1(makeString(stmt->relation->relname)); char *nspname = GetTypeNamespaceNameByNameList(names); stmt->relation->schemaname = nspname; } } void QualifyAlterEnumStmt(Node *node) { AlterEnumStmt *stmt = castNode(AlterEnumStmt, node); List *names = stmt->typeName; if (list_length(names) == 1) { /* not qualified, lookup name and add namespace name to names */ char *nspname = GetTypeNamespaceNameByNameList(names); names = list_make2(makeString(nspname), linitial(names)); stmt->typeName = names; } } void QualifyAlterTypeStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); Assert(stmt->objtype == OBJECT_TYPE); if (stmt->relation->schemaname == NULL) { List *names = MakeNameListFromRangeVar(stmt->relation); char *nspname = GetTypeNamespaceNameByNameList(names); stmt->relation->schemaname = nspname; } } void QualifyCompositeTypeStmt(Node *node) { CompositeTypeStmt *stmt = castNode(CompositeTypeStmt, node); if (stmt->typevar->schemaname == NULL) { Oid creationSchema = RangeVarGetCreationNamespace(stmt->typevar); stmt->typevar->schemaname = get_namespace_name(creationSchema); } } void QualifyCreateEnumStmt(Node *node) { CreateEnumStmt *stmt = castNode(CreateEnumStmt, node); if (list_length(stmt->typeName) == 1) { char *objname = NULL; Oid creationSchema = QualifiedNameGetCreationNamespace(stmt->typeName, &objname); stmt->typeName = list_make2(makeString(get_namespace_name(creationSchema)), linitial(stmt->typeName)); } } void QualifyAlterTypeSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); Assert(stmt->objectType == OBJECT_TYPE); List *names = (List *) stmt->object; if (list_length(names) == 1) { /* not qualified with schema, lookup type and its schema s*/ char *nspname = GetTypeNamespaceNameByNameList(names); names = list_make2(makeString(nspname), linitial(names)); stmt->object = (Node *) names; } } void QualifyAlterTypeOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_TYPE); List *names = (List *) stmt->object; if (list_length(names) == 1) { /* not qualified with schema, lookup type and its schema s*/ char *nspname = GetTypeNamespaceNameByNameList(names); names = list_make2(makeString(nspname), linitial(names)); stmt->object = (Node *) names; } } ================================================ FILE: src/backend/distributed/deparser/qualify_view_stmt.c ================================================ /*------------------------------------------------------------------------- * * qualify_view_stmt.c * Functions specialized in fully qualifying all view statements. These * functions are dispatched from qualify.c * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "nodes/nodes.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "distributed/deparser.h" #include "distributed/listutils.h" static void QualifyViewRangeVar(RangeVar *view); /* * QualifyDropViewStmt quailifies the view names of the DROP VIEW statement. */ void QualifyDropViewStmt(Node *node) { DropStmt *stmt = castNode(DropStmt, node); List *qualifiedViewNames = NIL; List *possiblyQualifiedViewName = NULL; foreach_declared_ptr(possiblyQualifiedViewName, stmt->objects) { char *viewName = NULL; char *schemaName = NULL; List *viewNameToAdd = possiblyQualifiedViewName; DeconstructQualifiedName(possiblyQualifiedViewName, &schemaName, &viewName); if (schemaName == NULL) { RangeVar *viewRangeVar = makeRangeVarFromNameList(possiblyQualifiedViewName); Oid viewOid = RangeVarGetRelid(viewRangeVar, AccessExclusiveLock, stmt->missing_ok); /* * If DROP VIEW IF EXISTS called and the view doesn't exist, oid can be invalid. * Do not try to qualify it. */ if (OidIsValid(viewOid)) { Oid schemaOid = get_rel_namespace(viewOid); schemaName = get_namespace_name(schemaOid); List *qualifiedViewName = list_make2(makeString(schemaName), makeString(viewName)); viewNameToAdd = qualifiedViewName; } } qualifiedViewNames = lappend(qualifiedViewNames, viewNameToAdd); } stmt->objects = qualifiedViewNames; } /* * QualifyAlterViewStmt quailifies the view name of the ALTER VIEW statement. */ void QualifyAlterViewStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); RangeVar *view = stmt->relation; QualifyViewRangeVar(view); } /* * QualifyRenameViewStmt quailifies the view name of the ALTER VIEW ... RENAME statement. */ void QualifyRenameViewStmt(Node *node) { RenameStmt *stmt = castNode(RenameStmt, node); RangeVar *view = stmt->relation; QualifyViewRangeVar(view); } /* * QualifyAlterViewSchemaStmt quailifies the view name of the ALTER VIEW ... SET SCHEMA statement. */ void QualifyAlterViewSchemaStmt(Node *node) { AlterObjectSchemaStmt *stmt = castNode(AlterObjectSchemaStmt, node); RangeVar *view = stmt->relation; QualifyViewRangeVar(view); } /* * QualifyViewRangeVar qualifies the given view RangeVar if it is not qualified. */ static void QualifyViewRangeVar(RangeVar *view) { if (view->schemaname == NULL) { Oid viewOid = RelnameGetRelid(view->relname); Oid schemaOid = get_rel_namespace(viewOid); view->schemaname = get_namespace_name(schemaOid); } } ================================================ FILE: src/backend/distributed/deparser/ruleutils_16.c ================================================ /*------------------------------------------------------------------------- * * ruleutils_16.c * Functions to convert stored expressions/querytrees back to * source text * * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/distributed/deparser/ruleutils_16.c * * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ #include "pg_version_constants.h" #include "pg_config.h" #if (PG_VERSION_NUM >= PG_VERSION_16) && (PG_VERSION_NUM < PG_VERSION_17) #include "postgres.h" #include #include #include #include "access/amapi.h" #include "access/htup_details.h" #include "access/relation.h" #include "access/sysattr.h" #include "access/table.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_authid.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_language.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_proc.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/tablespace.h" #include "common/keywords.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/multi_router_planner.h" #include "distributed/namespace_utils.h" #include "executor/spi.h" #include "foreign/foreign.h" #include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "optimizer/optimizer.h" #include "parser/parse_node.h" #include "parser/parse_agg.h" #include "parser/parse_func.h" #include "parser/parse_oper.h" #include "parser/parse_relation.h" #include "parser/parser.h" #include "parser/parsetree.h" #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSupport.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/ruleutils.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "utils/varlena.h" #include "utils/xml.h" /* ---------- * Pretty formatting constants * ---------- */ /* Indent counts */ #define PRETTYINDENT_STD 8 #define PRETTYINDENT_JOIN 4 #define PRETTYINDENT_VAR 4 #define PRETTYINDENT_LIMIT 40 /* wrap limit */ /* Pretty flags */ #define PRETTYFLAG_PAREN 0x0001 #define PRETTYFLAG_INDENT 0x0002 /* Default line length for pretty-print wrapping: 0 means wrap always */ #define WRAP_COLUMN_DEFAULT 0 /* macros to test if pretty action needed */ #define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) #define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) /* ---------- * Local data types * ---------- */ /* Context info needed for invoking a recursive querytree display routine */ typedef struct { StringInfo buf; /* output buffer to append to */ List *namespaces; /* List of deparse_namespace nodes */ TupleDesc resultDesc; /* if top level of a view, the view's tupdesc */ List *targetList; /* Current query level's SELECT targetlist */ List *windowClause; /* Current query level's WINDOW clause */ int prettyFlags; /* enabling of pretty-print functions */ int wrapColumn; /* max line length, or -1 for no limit */ int indentLevel; /* current indent level for prettyprint */ bool varprefix; /* true to print prefixes on Vars */ Oid distrelid; /* the distributed table being modified, if valid */ int64 shardid; /* a distributed table's shardid, if positive */ bool colNamesVisible; /* do we care about output column names? */ bool inGroupBy; /* deparsing GROUP BY clause? */ bool varInOrderBy; /* deparsing simple Var in ORDER BY? */ Bitmapset *appendparents; /* if not null, map child Vars of these relids * back to the parent rel */ } deparse_context; /* * Each level of query context around a subtree needs a level of Var namespace. * A Var having varlevelsup=N refers to the N'th item (counting from 0) in * the current context's namespaces list. * * The rangetable is the list of actual RTEs from the query tree, and the * cte list is the list of actual CTEs. * * rtable_names holds the alias name to be used for each RTE (either a C * string, or NULL for nameless RTEs such as unnamed joins). * rtable_columns holds the column alias names to be used for each RTE. * * In some cases we need to make names of merged JOIN USING columns unique * across the whole query, not only per-RTE. If so, unique_using is true * and using_names is a list of C strings representing names already assigned * to USING columns. * * When deparsing plan trees, there is always just a single item in the * deparse_namespace list (since a plan tree never contains Vars with * varlevelsup > 0). We store the PlanState node that is the immediate * parent of the expression to be deparsed, as well as a list of that * PlanState's ancestors. In addition, we store its outer and inner subplan * state nodes, as well as their plan nodes' targetlists, and the index tlist * if the current plan node might contain INDEX_VAR Vars. (These fields could * be derived on-the-fly from the current PlanState, but it seems notationally * clearer to set them up as separate fields.) */ typedef struct { List *rtable; /* List of RangeTblEntry nodes */ List *rtable_names; /* Parallel list of names for RTEs */ List *rtable_columns; /* Parallel list of deparse_columns structs */ List *subplans; /* List of Plan trees for SubPlans */ List *ctes; /* List of CommonTableExpr nodes */ AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ /* Workspace for column alias assignment: */ bool unique_using; /* Are we making USING names globally unique */ List *using_names; /* List of assigned names for USING columns */ /* Remaining fields are used only when deparsing a Plan tree: */ Plan *plan; /* immediate parent of current expression */ List *ancestors; /* ancestors of planstate */ Plan *outer_plan; /* outer subnode, or NULL if none */ Plan *inner_plan; /* inner subnode, or NULL if none */ List *outer_tlist; /* referent for OUTER_VAR Vars */ List *inner_tlist; /* referent for INNER_VAR Vars */ List *index_tlist; /* referent for INDEX_VAR Vars */ /* Special namespace representing a function signature: */ char *funcname; int numargs; char **argnames; } deparse_namespace; /* Callback signature for resolve_special_varno() */ typedef void (*rsv_callback) (Node *node, deparse_context *context, void *callback_arg); /* * Per-relation data about column alias names. * * Selecting aliases is unreasonably complicated because of the need to dump * rules/views whose underlying tables may have had columns added, deleted, or * renamed since the query was parsed. We must nonetheless print the rule/view * in a form that can be reloaded and will produce the same results as before. * * For each RTE used in the query, we must assign column aliases that are * unique within that RTE. SQL does not require this of the original query, * but due to factors such as *-expansion we need to be able to uniquely * reference every column in a decompiled query. As long as we qualify all * column references, per-RTE uniqueness is sufficient for that. * * However, we can't ensure per-column name uniqueness for unnamed join RTEs, * since they just inherit column names from their input RTEs, and we can't * rename the columns at the join level. Most of the time this isn't an issue * because we don't need to reference the join's output columns as such; we * can reference the input columns instead. That approach can fail for merged * JOIN USING columns, however, so when we have one of those in an unnamed * join, we have to make that column's alias globally unique across the whole * query to ensure it can be referenced unambiguously. * * Another problem is that a JOIN USING clause requires the columns to be * merged to have the same aliases in both input RTEs, and that no other * columns in those RTEs or their children conflict with the USING names. * To handle that, we do USING-column alias assignment in a recursive * traversal of the query's jointree. When descending through a JOIN with * USING, we preassign the USING column names to the child columns, overriding * other rules for column alias assignment. We also mark each RTE with a list * of all USING column names selected for joins containing that RTE, so that * when we assign other columns' aliases later, we can avoid conflicts. * * Another problem is that if a JOIN's input tables have had columns added or * deleted since the query was parsed, we must generate a column alias list * for the join that matches the current set of input columns --- otherwise, a * change in the number of columns in the left input would throw off matching * of aliases to columns of the right input. Thus, positions in the printable * column alias list are not necessarily one-for-one with varattnos of the * JOIN, so we need a separate new_colnames[] array for printing purposes. */ typedef struct { /* * colnames is an array containing column aliases to use for columns that * existed when the query was parsed. Dropped columns have NULL entries. * This array can be directly indexed by varattno to get a Var's name. * * Non-NULL entries are guaranteed unique within the RTE, *except* when * this is for an unnamed JOIN RTE. In that case we merely copy up names * from the two input RTEs. * * During the recursive descent in set_using_names(), forcible assignment * of a child RTE's column name is represented by pre-setting that element * of the child's colnames array. So at that stage, NULL entries in this * array just mean that no name has been preassigned, not necessarily that * the column is dropped. */ int num_cols; /* length of colnames[] array */ char **colnames; /* array of C strings and NULLs */ /* * new_colnames is an array containing column aliases to use for columns * that would exist if the query was re-parsed against the current * definitions of its base tables. This is what to print as the column * alias list for the RTE. This array does not include dropped columns, * but it will include columns added since original parsing. Indexes in * it therefore have little to do with current varattno values. As above, * entries are unique unless this is for an unnamed JOIN RTE. (In such an * RTE, we never actually print this array, but we must compute it anyway * for possible use in computing column names of upper joins.) The * parallel array is_new_col marks which of these columns are new since * original parsing. Entries with is_new_col false must match the * non-NULL colnames entries one-for-one. */ int num_new_cols; /* length of new_colnames[] array */ char **new_colnames; /* array of C strings */ bool *is_new_col; /* array of bool flags */ /* This flag tells whether we should actually print a column alias list */ bool printaliases; /* This list has all names used as USING names in joins above this RTE */ List *parentUsing; /* names assigned to parent merged columns */ /* * If this struct is for a JOIN RTE, we fill these fields during the * set_using_names() pass to describe its relationship to its child RTEs. * * leftattnos and rightattnos are arrays with one entry per existing * output column of the join (hence, indexable by join varattno). For a * simple reference to a column of the left child, leftattnos[i] is the * child RTE's attno and rightattnos[i] is zero; and conversely for a * column of the right child. But for merged columns produced by JOIN * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. * Also, if the column has been dropped, both are zero. * * If it's a JOIN USING, usingNames holds the alias names selected for the * merged columns (these might be different from the original USING list, * if we had to modify names to achieve uniqueness). */ int leftrti; /* rangetable index of left child */ int rightrti; /* rangetable index of right child */ int *leftattnos; /* left-child varattnos of join cols, or 0 */ int *rightattnos; /* right-child varattnos of join cols, or 0 */ List *usingNames; /* names assigned to merged columns */ } deparse_columns; /* This macro is analogous to rt_fetch(), but for deparse_columns structs */ #define deparse_columns_fetch(rangetable_index, dpns) \ ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) /* * Entry in set_rtable_names' hash table */ typedef struct { char name[NAMEDATALEN]; /* Hash key --- must be first */ int counter; /* Largest addition used so far for name */ } NameHashEntry; /* ---------- * Local functions * * Most of these functions used to use fixed-size buffers to build their * results. Now, they take an (already initialized) StringInfo object * as a parameter, and append their text output to its contents. * ---------- */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used); static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces); static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing); static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static char *make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static void expand_colnames_array_to(deparse_columns *colinfo, int n); static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo); static char *get_rtable_name(int rtindex, deparse_context *context); static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); static Plan *find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan); static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns); static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns); static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_values_def(List *values_lists, deparse_context *context); static void get_with_clause(Query *query, deparse_context *context); static void get_select_query_def(Query *query, deparse_context *context); static void get_insert_query_def(Query *query, deparse_context *context); static void get_update_query_def(Query *query, deparse_context *context); static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte); static void get_delete_query_def(Query *query, deparse_context *context); static void get_merge_query_def(Query *query, deparse_context *context); static void get_utility_query_def(Query *query, deparse_context *context); static void get_basic_select_query(Query *query, deparse_context *context); static void get_target_list(List *targetList, deparse_context *context); static void get_setop_query(Node *setOp, Query *query, deparse_context *context); static Node *get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context); static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context); static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context); static void get_rule_windowclause(Query *query, deparse_context *context); static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context); static char *get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context); static void get_special_variable(Node *node, deparse_context *context, void *callback_arg); static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg); static Node *find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p); static void get_parameter(Param *param, deparse_context *context); static const char *get_simple_binary_op_name(OpExpr *expr); static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus); static void removeStringInfoSpaces(StringInfo str); static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit); static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit); static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit); static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit); static bool looks_like_function(Node *node); static void get_oper_expr(OpExpr *expr, deparse_context *context); static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit); static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit); static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref); static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg); static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg); static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg); static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode); static void get_const_expr(Const *constval, deparse_context *context, int showtype); static void get_const_collation(Const *constval, deparse_context *context); static void get_json_format(JsonFormat *format, StringInfo buf); static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit); static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf); static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg); static void simple_quote_literal(StringInfo buf, const char *val); static void get_sublink_expr(SubLink *sublink, deparse_context *context); static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit); static void get_from_clause(Query *query, const char *prefix, deparse_context *context); static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context); static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context); static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context); static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context); static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context); static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf); static Node *processIndirection(Node *node, deparse_context *context); static void printSubscripts(SubscriptingRef *aref, deparse_context *context); static char *get_relation_name(Oid relid); static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces); static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); static char *generate_fragment_name(char *schemaName, char *tableName); static char *generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy); static List *get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte); #define only_marker(rte) ((rte)->inh ? "" : "ONLY ") /* * pg_get_query_def parses back one query tree, and outputs the resulting query * string into given buffer. */ void pg_get_query_def(Query *query, StringInfo buffer) { get_query_def(query, buffer, NIL, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* * get_merged_argument_list merges both the IN and OUT arguments lists into one and * also eliminates the INOUT duplicates(present in both the lists). After merging both * the lists, it returns all the named-arguments in a list(mergedNamedArgList) along * with their types(mergedNamedArgTypes), final argument list(mergedArgumentList), and * the total number of arguments(totalArguments). */ bool get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, Oid **mergedNamedArgTypes, List **mergedArgumentList, int *totalArguments) { Oid functionOid = stmt->funcexpr->funcid; List *namedArgList = NIL; List *finalArgumentList = NIL; Oid *finalArgTypes; Oid *argTypes = NULL; char *argModes = NULL; char **argNames = NULL; int argIndex = 0; HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "cache lookup failed for function %u", functionOid); } int defArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); ReleaseSysCache(proctup); if (argModes == NULL) { /* No OUT arguments */ return false; } /* * Passed arguments Includes IN, OUT, INOUT (in both the lists) and VARIADIC arguments, * which means INOUT arguments are double counted. */ int numberOfArgs = list_length(stmt->funcexpr->args) + list_length(stmt->outargs); int totalInoutArgs = 0; /* Let's count INOUT arguments from the defined number of arguments */ for (argIndex=0; argIndex < defArgs; ++argIndex) { if (argModes[argIndex] == PROARGMODE_INOUT) totalInoutArgs++; } /* Remove the duplicate INOUT counting */ numberOfArgs = numberOfArgs - totalInoutArgs; finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs); ListCell *inArgCell = list_head(stmt->funcexpr->args); ListCell *outArgCell = list_head(stmt->outargs); for (argIndex=0; argIndex < numberOfArgs; ++argIndex) { switch (argModes[argIndex]) { case PROARGMODE_IN: case PROARGMODE_VARIADIC: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); break; } case PROARGMODE_OUT: { Node *arg = (Node *) lfirst(outArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_INOUT: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_TABLE: default: { elog(ERROR, "Unhandled procedure argument mode[%d]", argModes[argIndex]); break; } } } /* * After eliminating INOUT duplicates and merging OUT arguments, we now * have the final list of arguments. */ if (defArgs != list_length(finalArgumentList)) { elog(ERROR, "Insufficient number of args passed[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)); } if (list_length(finalArgumentList) > FUNC_MAX_ARGS) { ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)))); } *mergedNamedArgList = namedArgList; *mergedNamedArgTypes = finalArgTypes; *mergedArgumentList = finalArgumentList; *totalArguments = numberOfArgs; return true; } /* * pg_get_rule_expr deparses an expression and returns the result as a string. */ char * pg_get_rule_expr(Node *expression) { bool showImplicitCasts = true; deparse_context context; StringInfo buffer = makeStringInfo(); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buffer; context.namespaces = NIL; context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = false; context.prettyFlags = 0; context.wrapColumn = WRAP_COLUMN_DEFAULT; context.indentLevel = 0; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.distrelid = InvalidOid; context.shardid = INVALID_SHARD_ID; get_rule_expr(expression, &context, showImplicitCasts); /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return buffer->data; } /* * set_rtable_names: select RTE aliases to be used in printing a query * * We fill in dpns->rtable_names with a list of names that is one-for-one with * the already-filled dpns->rtable list. Each RTE name is unique among those * in the new namespace plus any ancestor namespaces listed in * parent_namespaces. * * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. * * Note that this function is only concerned with relation names, not column * names. */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used) { HASHCTL hash_ctl; HTAB *names_hash; NameHashEntry *hentry; bool found; int rtindex; ListCell *lc; dpns->rtable_names = NIL; /* nothing more to do if empty rtable */ if (dpns->rtable == NIL) return; /* * We use a hash table to hold known names, so that this process is O(N) * not O(N^2) for N names. */ hash_ctl.keysize = NAMEDATALEN; hash_ctl.entrysize = sizeof(NameHashEntry); hash_ctl.hcxt = CurrentMemoryContext; names_hash = hash_create("set_rtable_names names", list_length(dpns->rtable), &hash_ctl, HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); /* Preload the hash table with names appearing in parent_namespaces */ foreach(lc, parent_namespaces) { deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); ListCell *lc2; foreach(lc2, olddpns->rtable_names) { char *oldname = (char *) lfirst(lc2); if (oldname == NULL) continue; hentry = (NameHashEntry *) hash_search(names_hash, oldname, HASH_ENTER, &found); /* we do not complain about duplicate names in parent namespaces */ hentry->counter = 0; } } /* Now we can scan the rtable */ rtindex = 1; foreach(lc, dpns->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); char *refname; /* Just in case this takes an unreasonable amount of time ... */ CHECK_FOR_INTERRUPTS(); if (rels_used && !bms_is_member(rtindex, rels_used)) { /* Ignore unreferenced RTE */ refname = NULL; } else if (rte->alias) { /* If RTE has a user-defined alias, prefer that */ refname = rte->alias->aliasname; } else if (rte->rtekind == RTE_RELATION) { /* Use the current actual name of the relation */ refname = get_rel_name(rte->relid); } else if (rte->rtekind == RTE_JOIN) { /* Unnamed join has no refname */ refname = NULL; } else { /* Otherwise use whatever the parser assigned */ refname = rte->eref->aliasname; } /* * If the selected name isn't unique, append digits to make it so, and * make a new hash entry for it once we've got a unique name. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (refname) { hentry = (NameHashEntry *) hash_search(names_hash, refname, HASH_ENTER, &found); if (found) { /* Name already in use, must choose a new one */ int refnamelen = strlen(refname); char *modname = (char *) palloc(refnamelen + 16); NameHashEntry *hentry2; do { hentry->counter++; for (;;) { memcpy(modname, refname, refnamelen); sprintf(modname + refnamelen, "_%d", hentry->counter); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from refname to keep all the digits */ refnamelen = pg_mbcliplen(refname, refnamelen, refnamelen - 1); } hentry2 = (NameHashEntry *) hash_search(names_hash, modname, HASH_ENTER, &found); } while (found); hentry2->counter = 0; /* init new hash entry */ refname = modname; } else { /* Name not previously used, need only initialize hentry */ hentry->counter = 0; } } dpns->rtable_names = lappend(dpns->rtable_names, refname); rtindex++; } hash_destroy(names_hash); } /* * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree * * For convenience, this is defined to initialize the deparse_namespace struct * from scratch. */ static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces) { ListCell *lc; ListCell *lc2; /* Initialize *dpns and fill rtable/ctes links */ memset(dpns, 0, sizeof(deparse_namespace)); dpns->rtable = query->rtable; dpns->subplans = NIL; dpns->ctes = query->cteList; dpns->appendrels = NULL; /* Assign a unique relation alias to each RTE */ set_rtable_names(dpns, parent_namespaces, NULL); /* Initialize dpns->rtable_columns to contain zeroed structs */ dpns->rtable_columns = NIL; while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) dpns->rtable_columns = lappend(dpns->rtable_columns, palloc0(sizeof(deparse_columns))); /* If it's a utility query, it won't have a jointree */ if (query->jointree) { /* Detect whether global uniqueness of USING names is needed */ dpns->unique_using = has_dangerous_join_using(dpns, (Node *) query->jointree); /* * Select names for columns merged by USING, via a recursive pass over * the query jointree. */ set_using_names(dpns, (Node *) query->jointree, NIL); } /* * Now assign remaining column aliases for each RTE. We do this in a * linear scan of the rtable, so as to process RTEs whether or not they * are in the jointree (we mustn't miss NEW.*, INSERT target relations, * etc). JOIN RTEs must be processed after their children, but this is * okay because they appear later in the rtable list than their children * (cf Asserts in identify_join_columns()). */ forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); if (rte->rtekind == RTE_JOIN) set_join_column_names(dpns, rte, colinfo); else set_relation_column_names(dpns, rte, colinfo); } } /* * has_dangerous_join_using: search jointree for unnamed JOIN USING * * Merged columns of a JOIN USING may act differently from either of the input * columns, either because they are merged with COALESCE (in a FULL JOIN) or * because an implicit coercion of the underlying input column is required. * In such a case the column must be referenced as a column of the JOIN not as * a column of either input. And this is problematic if the join is unnamed * (alias-less): we cannot qualify the column's name with an RTE name, since * there is none. (Forcibly assigning an alias to the join is not a solution, * since that will prevent legal references to tables below the join.) * To ensure that every column in the query is unambiguously referenceable, * we must assign such merged columns names that are globally unique across * the whole query, aliasing other columns out of the way as necessary. * * Because the ensuing re-aliasing is fairly damaging to the readability of * the query, we don't do this unless we have to. So, we must pre-scan * the join tree to see if we have to, before starting set_using_names(). */ static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do here */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) { if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) return true; } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; /* Is it an unnamed JOIN with USING? */ if (j->alias == NULL && j->usingClause) { /* * Yes, so check each join alias var to see if any of them are not * simple references to underlying columns. If so, we have a * dangerous situation and must pick unique aliases. */ RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); /* We need only examine the merged columns */ for (int i = 0; i < jrte->joinmergedcols; i++) { Node *aliasvar = list_nth(jrte->joinaliasvars, i); if (!IsA(aliasvar, Var)) return true; } } /* Nope, but inspect children */ if (has_dangerous_join_using(dpns, j->larg)) return true; if (has_dangerous_join_using(dpns, j->rarg)) return true; } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); return false; } /* * set_using_names: select column aliases to be used for merged USING columns * * We do this during a recursive descent of the query jointree. * dpns->unique_using must already be set to determine the global strategy. * * Column alias info is saved in the dpns->rtable_columns list, which is * assumed to be filled with pre-zeroed deparse_columns structs. * * parentUsing is a list of all USING aliases assigned in parent joins of * the current jointree node. (The passed-in list must not be modified.) */ static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do now */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) set_using_names(dpns, (Node *) lfirst(lc), parentUsing); } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); int *leftattnos; int *rightattnos; deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; int i; ListCell *lc; /* Get info about the shape of the join */ identify_join_columns(j, rte, colinfo); leftattnos = colinfo->leftattnos; rightattnos = colinfo->rightattnos; /* Look up the not-yet-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * If this join is unnamed, then we cannot substitute new aliases at * this level, so any name requirements pushed down to here must be * pushed down again to the children. */ if (rte->alias == NULL) { for (i = 0; i < colinfo->num_cols; i++) { char *colname = colinfo->colnames[i]; if (colname == NULL) continue; /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } } } /* * If there's a USING clause, select the USING column names and push * those names down to the children. We have two strategies: * * If dpns->unique_using is true, we force all USING names to be * unique across the whole query level. In principle we'd only need * the names of dangerous USING columns to be globally unique, but to * safely assign all USING names in a single pass, we have to enforce * the same uniqueness rule for all of them. However, if a USING * column's name has been pushed down from the parent, we should use * it as-is rather than making a uniqueness adjustment. This is * necessary when we're at an unnamed join, and it creates no risk of * ambiguity. Also, if there's a user-written output alias for a * merged column, we prefer to use that rather than the input name; * this simplifies the logic and seems likely to lead to less aliasing * overall. * * If dpns->unique_using is false, we only need USING names to be * unique within their own join RTE. We still need to honor * pushed-down names, though. * * Though significantly different in results, these two strategies are * implemented by the same code, with only the difference of whether * to put assigned names into dpns->using_names. */ if (j->usingClause) { /* Copy the input parentUsing list so we don't modify it */ parentUsing = list_copy(parentUsing); /* USING names must correspond to the first join output columns */ expand_colnames_array_to(colinfo, list_length(j->usingClause)); i = 0; foreach(lc, j->usingClause) { char *colname = strVal(lfirst(lc)); /* Assert it's a merged column */ Assert(leftattnos[i] != 0 && rightattnos[i] != 0); /* Adopt passed-down name if any, else select unique name */ if (colinfo->colnames[i] != NULL) colname = colinfo->colnames[i]; else { /* Prefer user-written output alias if any */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); /* Make it appropriately unique */ colname = make_colname_unique(colname, dpns, colinfo); if (dpns->unique_using) dpns->using_names = lappend(dpns->using_names, colname); /* Save it as output column name, too */ colinfo->colnames[i] = colname; } /* Remember selected names for use later */ colinfo->usingNames = lappend(colinfo->usingNames, colname); parentUsing = lappend(parentUsing, colname); /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } i++; } } /* Mark child deparse_columns structs with correct parentUsing info */ leftcolinfo->parentUsing = parentUsing; rightcolinfo->parentUsing = parentUsing; /* Now recursively assign USING column names in children */ set_using_names(dpns, j->larg, parentUsing); set_using_names(dpns, j->rarg, parentUsing); } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * set_relation_column_names: select column aliases for a non-join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. */ static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { int ncolumns; char **real_colnames; bool changed_any; bool has_anonymous; int noldcolumns; int i; int j; /* * Construct an array of the current "real" column names of the RTE. * real_colnames[] will be indexed by physical column number, with NULL * entries for dropped columns. */ if (rte->rtekind == RTE_RELATION || GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* Relation --- look to the system catalogs for up-to-date info */ Relation rel; TupleDesc tupdesc; rel = relation_open(rte->relid, AccessShareLock); tupdesc = RelationGetDescr(rel); ncolumns = tupdesc->natts; real_colnames = (char **) palloc(ncolumns * sizeof(char *)); for (i = 0; i < ncolumns; i++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); if (attr->attisdropped) real_colnames[i] = NULL; else real_colnames[i] = pstrdup(NameStr(attr->attname)); } relation_close(rel, AccessShareLock); } else { /* Otherwise get the column names from eref or expandRTE() */ List *colnames; ListCell *lc; /* * Functions returning composites have the annoying property that some * of the composite type's columns might have been dropped since the * query was parsed. If possible, use expandRTE() to handle that * case, since it has the tedious logic needed to find out about * dropped columns. However, if we're explaining a plan, then we * don't have rte->functions because the planner thinks that won't be * needed later, and that breaks expandRTE(). So in that case we have * to rely on rte->eref, which may lead us to report a dropped * column's old name; that seems close enough for EXPLAIN's purposes. * * For non-RELATION, non-FUNCTION RTEs, we can just look at rte->eref, * which should be sufficiently up-to-date: no other RTE types can * have columns get dropped from under them after parsing. */ if (rte->rtekind == RTE_FUNCTION && rte->functions != NIL) { /* Since we're not creating Vars, rtindex etc. don't matter */ expandRTE(rte, 1, 0, -1, true /* include dropped */ , &colnames, NULL); } else colnames = rte->eref->colnames; ncolumns = list_length(colnames); real_colnames = (char **) palloc(ncolumns * sizeof(char *)); i = 0; foreach(lc, colnames) { /* * If the column name we find here is an empty string, then it's a * dropped column, so change to NULL. */ char *cname = strVal(lfirst(lc)); if (cname[0] == '\0') cname = NULL; real_colnames[i] = cname; i++; } } /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that there are now more columns than there were when the * query was parsed, ie colnames could be longer than rte->eref->colnames. * We must assign unique aliases to the new columns too, else there could * be unresolved conflicts when the view/rule is reloaded. */ expand_colnames_array_to(colinfo, ncolumns); Assert(colinfo->num_cols == ncolumns); /* * Make sufficiently large new_colnames and is_new_col arrays, too. * * Note: because we leave colinfo->num_new_cols zero until after the loop, * colname_is_unique will not consult that array, which is fine because it * would only be duplicate effort. */ colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); /* * Scan the columns, select a unique alias for each one, and store it in * colinfo->colnames and colinfo->new_colnames. The former array has NULL * entries for dropped columns, the latter omits them. Also mark * new_colnames entries as to whether they are new since parse time; this * is the case for entries beyond the length of rte->eref->colnames. */ noldcolumns = list_length(rte->eref->colnames); changed_any = false; has_anonymous = false; j = 0; for (i = 0; i < ncolumns; i++) { char *real_colname = real_colnames[i]; char *colname = colinfo->colnames[i]; /* Skip dropped columns */ if (real_colname == NULL) { Assert(colname == NULL); /* colnames[i] is already NULL */ continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; } /* Put names of non-dropped columns in new_colnames[] too */ colinfo->new_colnames[j] = colname; /* And mark them as new or not */ colinfo->is_new_col[j] = (i >= noldcolumns); j++; /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; /* * Remember if there is a reference to an anonymous column as named by * char * FigureColname(Node *node) */ if (!has_anonymous && strcmp(real_colname, "?column?") == 0) has_anonymous = true; } /* * Set correct length for new_colnames[] array. (Note: if columns have * been added, colinfo->num_cols includes them, which is not really quite * right but is harmless, since any new columns must be at the end where * they won't affect varattnos of pre-existing columns.) */ colinfo->num_new_cols = j; /* * For a relation RTE, we need only print the alias column names if any * are different from the underlying "real" names. For a function RTE, * always emit a complete column alias list; this is to protect against * possible instability of the default column names (eg, from altering * parameter names). For tablefunc RTEs, we never print aliases, because * the column names are part of the clause itself. For other RTE types, * print if we changed anything OR if there were user-written column * aliases (since the latter would be part of the underlying "reality"). */ if (rte->rtekind == RTE_RELATION) colinfo->printaliases = changed_any; else if (rte->rtekind == RTE_FUNCTION) colinfo->printaliases = true; else if (rte->rtekind == RTE_TABLEFUNC) colinfo->printaliases = false; else if (rte->alias && rte->alias->colnames != NIL) colinfo->printaliases = true; else colinfo->printaliases = changed_any || has_anonymous; } /* * set_join_column_names: select column aliases for a join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. Also, names for USING columns were already chosen by * set_using_names(). We further expect that column alias selection has been * completed for both input RTEs. */ static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; bool changed_any; int noldcolumns; int nnewcolumns; Bitmapset *leftmerged = NULL; Bitmapset *rightmerged = NULL; int i; int j; int ic; int jc; /* Look up the previously-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that one or both inputs now have more columns than there * were when the query was parsed, but we'll deal with that below. We * only need entries in colnames for pre-existing columns. */ noldcolumns = list_length(rte->eref->colnames); expand_colnames_array_to(colinfo, noldcolumns); Assert(colinfo->num_cols == noldcolumns); /* * Scan the join output columns, select an alias for each one, and store * it in colinfo->colnames. If there are USING columns, set_using_names() * already selected their names, so we can start the loop at the first * non-merged column. */ changed_any = false; for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) { char *colname = colinfo->colnames[i]; char *real_colname; /* Join column must refer to at least one input column */ Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); /* Get the child column name */ if (colinfo->leftattnos[i] > 0) real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; else if (colinfo->rightattnos[i] > 0) real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; else { /* We're joining system columns --- use eref name */ real_colname = strVal(list_nth(rte->eref->colnames, i)); } /* If child col has been dropped, no need to assign a join colname */ if (real_colname == NULL) { colinfo->colnames[i] = NULL; continue; } /* In an unnamed join, just report child column names as-is */ if (rte->alias == NULL) { colinfo->colnames[i] = real_colname; continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; } /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; } /* * Calculate number of columns the join would have if it were re-parsed * now, and create storage for the new_colnames and is_new_col arrays. * * Note: colname_is_unique will be consulting new_colnames[] during the * loops below, so its not-yet-filled entries must be zeroes. */ nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - list_length(colinfo->usingNames); colinfo->num_new_cols = nnewcolumns; colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); /* * Generating the new_colnames array is a bit tricky since any new columns * added since parse time must be inserted in the right places. This code * must match the parser, which will order a join's columns as merged * columns first (in USING-clause order), then non-merged columns from the * left input (in attnum order), then non-merged columns from the right * input (ditto). If one of the inputs is itself a join, its columns will * be ordered according to the same rule, which means newly-added columns * might not be at the end. We can figure out what's what by consulting * the leftattnos and rightattnos arrays plus the input is_new_col arrays. * * In these loops, i indexes leftattnos/rightattnos (so it's join varattno * less one), j indexes new_colnames/is_new_col, and ic/jc have similar * meanings for the current child RTE. */ /* Handle merged columns; they are first and can't be new */ i = j = 0; while (i < noldcolumns && colinfo->leftattnos[i] != 0 && colinfo->rightattnos[i] != 0) { /* column name is already determined and known unique */ colinfo->new_colnames[j] = colinfo->colnames[i]; colinfo->is_new_col[j] = false; /* build bitmapsets of child attnums of merged columns */ if (colinfo->leftattnos[i] > 0) leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); if (colinfo->rightattnos[i] > 0) rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); i++, j++; } /* Handle non-merged left-child columns */ ic = 0; for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) { char *child_colname = leftcolinfo->new_colnames[jc]; if (!leftcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of left child */ while (ic < leftcolinfo->num_cols && leftcolinfo->colnames[ic] == NULL) ic++; Assert(ic < leftcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, leftmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->leftattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; } colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; j++; } /* Handle non-merged right-child columns in exactly the same way */ ic = 0; for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) { char *child_colname = rightcolinfo->new_colnames[jc]; if (!rightcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of right child */ while (ic < rightcolinfo->num_cols && rightcolinfo->colnames[ic] == NULL) ic++; Assert(ic < rightcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, rightmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->rightattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; } colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; j++; } /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING for (int col_index = 0; col_index < colinfo->num_cols; col_index++) { /* * In the above processing-loops, "i" advances only if * the column is not new, check if this is a new column. */ if (colinfo->is_new_col[col_index]) i++; } Assert(j == nnewcolumns); #endif /* * For a named join, print column aliases if we changed any from the child * names. Unnamed joins cannot print aliases. */ if (rte->alias != NULL) colinfo->printaliases = changed_any; else colinfo->printaliases = false; } /* * colname_is_unique: is colname distinct from already-chosen column names? * * dpns is query-wide info, colinfo is for the column's RTE */ static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { int i; ListCell *lc; /* Check against already-assigned column aliases within RTE */ for (i = 0; i < colinfo->num_cols; i++) { char *oldname = colinfo->colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* * If we're building a new_colnames array, check that too (this will be * partially but not completely redundant with the previous checks) */ for (i = 0; i < colinfo->num_new_cols; i++) { char *oldname = colinfo->new_colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* Also check against USING-column names that must be globally unique */ foreach(lc, dpns->using_names) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } /* Also check against names already assigned for parent-join USING cols */ foreach(lc, colinfo->parentUsing) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } return true; } /* * make_colname_unique: modify colname if necessary to make it unique * * dpns is query-wide info, colinfo is for the column's RTE */ static char * make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { /* * If the selected name isn't unique, append digits to make it so. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (!colname_is_unique(colname, dpns, colinfo)) { int colnamelen = strlen(colname); char *modname = (char *) palloc(colnamelen + 16); int i = 0; do { i++; for (;;) { memcpy(modname, colname, colnamelen); sprintf(modname + colnamelen, "_%d", i); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from colname to keep all the digits */ colnamelen = pg_mbcliplen(colname, colnamelen, colnamelen - 1); } } while (!colname_is_unique(modname, dpns, colinfo)); colname = modname; } return colname; } /* * expand_colnames_array_to: make colinfo->colnames at least n items long * * Any added array entries are initialized to zero. */ static void expand_colnames_array_to(deparse_columns *colinfo, int n) { if (n > colinfo->num_cols) { if (colinfo->colnames == NULL) colinfo->colnames = palloc0_array(char *, n); else { colinfo->colnames = repalloc0_array(colinfo->colnames, char *, colinfo->num_cols, n); } colinfo->num_cols = n; } } /* * identify_join_columns: figure out where columns of a join come from * * Fills the join-specific fields of the colinfo struct, except for * usingNames which is filled later. */ static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo) { int numjoincols; int jcolno; int rcolno; ListCell *lc; /* Extract left/right child RT indexes */ if (IsA(j->larg, RangeTblRef)) colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; else if (IsA(j->larg, JoinExpr)) colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->larg)); if (IsA(j->rarg, RangeTblRef)) colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; else if (IsA(j->rarg, JoinExpr)) colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->rarg)); /* Assert children will be processed earlier than join in second pass */ Assert(colinfo->leftrti < j->rtindex); Assert(colinfo->rightrti < j->rtindex); /* Initialize result arrays with zeroes */ numjoincols = list_length(jrte->joinaliasvars); Assert(numjoincols == list_length(jrte->eref->colnames)); colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); /* * Deconstruct RTE's joinleftcols/joinrightcols into desired format. * Recall that the column(s) merged due to USING are the first column(s) * of the join output. We need not do anything special while scanning * joinleftcols, but while scanning joinrightcols we must distinguish * merged from unmerged columns. */ jcolno = 0; foreach(lc, jrte->joinleftcols) { int leftattno = lfirst_int(lc); colinfo->leftattnos[jcolno++] = leftattno; } rcolno = 0; foreach(lc, jrte->joinrightcols) { int rightattno = lfirst_int(lc); if (rcolno < jrte->joinmergedcols) /* merged column? */ colinfo->rightattnos[rcolno] = rightattno; else colinfo->rightattnos[jcolno++] = rightattno; rcolno++; } Assert(jcolno == numjoincols); } /* * get_rtable_name: convenience function to get a previously assigned RTE alias * * The RTE must belong to the topmost namespace level in "context". */ static char * get_rtable_name(int rtindex, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); return (char *) list_nth(dpns->rtable_names, rtindex - 1); } /* * set_deparse_plan: set up deparse_namespace to parse subexpressions * of a given Plan node * * This sets the plan, outer_planstate, inner_planstate, outer_tlist, * inner_tlist, and index_tlist fields. Caller is responsible for adjusting * the ancestors list if necessary. Note that the rtable and ctes fields do * not need to change when shifting attention to different plan nodes in a * single plan tree. */ static void set_deparse_plan(deparse_namespace *dpns, Plan *plan) { dpns->plan = plan; /* * We special-case Append and MergeAppend to pretend that the first child * plan is the OUTER referent; we have to interpret OUTER Vars in their * tlists according to one of the children, and the first one is the most * natural choice. */ if (IsA(plan, Append)) dpns->outer_plan = linitial(((Append *) plan)->appendplans); else if (IsA(plan, MergeAppend)) dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); else dpns->outer_plan = outerPlan(plan); if (dpns->outer_plan) dpns->outer_tlist = dpns->outer_plan->targetlist; else dpns->outer_tlist = NIL; /* * For a SubqueryScan, pretend the subplan is INNER referent. (We don't * use OUTER because that could someday conflict with the normal meaning.) * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. * For a WorkTableScan, locate the parent RecursiveUnion plan node and use * that as INNER referent. * * For MERGE, make the inner tlist point to the merge source tlist, which * is same as the targetlist that the ModifyTable's source plan provides. * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the * excluded expression's tlist. (Similar to the SubqueryScan we don't want * to reuse OUTER, it's used for RETURNING in some modify table cases, * although not INSERT .. CONFLICT). */ if (IsA(plan, SubqueryScan)) dpns->inner_plan = ((SubqueryScan *) plan)->subplan; else if (IsA(plan, CteScan)) dpns->inner_plan = list_nth(dpns->subplans, ((CteScan *) plan)->ctePlanId - 1); else if (IsA(plan, WorkTableScan)) dpns->inner_plan = find_recursive_union(dpns, (WorkTableScan *) plan); else if (IsA(plan, ModifyTable)) dpns->inner_plan = plan; else dpns->inner_plan = innerPlan(plan); if (IsA(plan, ModifyTable)) { if (((ModifyTable *) plan)->operation == CMD_MERGE) dpns->inner_tlist = dpns->outer_tlist; else dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; } else if (dpns->inner_plan) dpns->inner_tlist = dpns->inner_plan->targetlist; else dpns->inner_tlist = NIL; /* Set up referent for INDEX_VAR Vars, if needed */ if (IsA(plan, IndexOnlyScan)) dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; else if (IsA(plan, ForeignScan)) dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; else if (IsA(plan, CustomScan)) dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; else dpns->index_tlist = NIL; } /* * Locate the ancestor plan node that is the RecursiveUnion generating * the WorkTableScan's work table. We can match on wtParam, since that * should be unique within the plan tree. */ static Plan * find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan) { ListCell *lc; foreach(lc, dpns->ancestors) { Plan *ancestor = (Plan *) lfirst(lc); if (IsA(ancestor, RecursiveUnion) && ((RecursiveUnion *) ancestor)->wtParam == wtscan->wtParam) return ancestor; } elog(ERROR, "could not find RecursiveUnion for WorkTableScan with wtParam %d", wtscan->wtParam); return NULL; } /* * push_child_plan: temporarily transfer deparsing attention to a child plan * * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the * deparse context in case the referenced expression itself uses * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid * affecting levelsup issues (although in a Plan tree there really shouldn't * be any). * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_child_plan. */ static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns) { /* Save state for restoration later */ *save_dpns = *dpns; /* Link current plan node into ancestors list */ dpns->ancestors = lcons(dpns->plan, dpns->ancestors); /* Set attention on selected child */ set_deparse_plan(dpns, plan); } /* * pop_child_plan: undo the effects of push_child_plan */ static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { List *ancestors; /* Get rid of ancestors list cell added by push_child_plan */ ancestors = list_delete_first(dpns->ancestors); /* Restore fields changed by push_child_plan */ *dpns = *save_dpns; /* Make sure dpns->ancestors is right (may be unnecessary) */ dpns->ancestors = ancestors; } /* * push_ancestor_plan: temporarily transfer deparsing attention to an * ancestor plan * * When expanding a Param reference, we must adjust the deparse context * to match the plan node that contains the expression being printed; * otherwise we'd fail if that expression itself contains a Param or * OUTER_VAR/INNER_VAR/INDEX_VAR variable. * * The target ancestor is conveniently identified by the ListCell holding it * in dpns->ancestors. * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_ancestor_plan. */ static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns) { Plan *plan = (Plan *) lfirst(ancestor_cell); /* Save state for restoration later */ *save_dpns = *dpns; /* Build a new ancestor list with just this node's ancestors */ dpns->ancestors = list_copy_tail(dpns->ancestors, list_cell_number(dpns->ancestors, ancestor_cell) + 1); /* Set attention on selected ancestor */ set_deparse_plan(dpns, plan); } /* * pop_ancestor_plan: undo the effects of push_ancestor_plan */ static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { /* Free the ancestor list made in push_ancestor_plan */ list_free(dpns->ancestors); /* Restore fields changed by push_ancestor_plan */ *dpns = *save_dpns; } /* ---------- * deparse_shard_query - Parse back a query for execution on a shard * * Builds an SQL string to perform the provided query on a specific shard and * places this string into the provided buffer. * ---------- */ void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer) { get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* ---------- * get_query_def - Parse back one query parsetree * * query: parsetree to be displayed * buf: output text is appended to buf * parentnamespace: list (initially empty) of outer-level deparse_namespace's * resultDesc: if not NULL, the output tuple descriptor for the view * represented by a SELECT query. We use the column names from it * to label SELECT output columns, in preference to names in the query * colNamesVisible: true if the surrounding context cares about the output * column names at all (as, for example, an EXISTS() context does not); * when false, we can suppress dummy column labels such as "?column?" * prettyFlags: bitmask of PRETTYFLAG_XXX options * wrapColumn: maximum line length, or -1 to disable wrapping * startIndent: initial indentation amount * ---------- */ static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, colNamesVisible, prettyFlags, wrapColumn, startIndent); } /* ---------- * get_query_def_extended - Parse back one query parsetree, optionally * with extension using a shard identifier. * * If distrelid is valid and shardid is positive, the provided shardid is added * any time the provided relid is deparsed, so that the query may be executed * on a placement for the given shard. * ---------- */ static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { deparse_context context; deparse_namespace dpns; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); /* * Before we begin to examine the query, acquire locks on referenced * relations, and fix up deleted columns in JOIN RTEs. This ensures * consistent results. Note we assume it's OK to scribble on the passed * querytree! * * We are only deparsing the query (we are not about to execute it), so we * only need AccessShareLock on the relations it mentions. */ AcquireRewriteLocks(query, false, false); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buf; context.namespaces = lcons(&dpns, list_copy(parentnamespace)); context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = (parentnamespace != NIL || list_length(query->rtable) != 1); context.prettyFlags = prettyFlags; context.wrapColumn = wrapColumn; context.indentLevel = startIndent; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.appendparents = NULL; context.distrelid = distrelid; context.shardid = shardid; set_deparse_for_query(&dpns, query, parentnamespace); switch (query->commandType) { case CMD_SELECT: /* We set context.resultDesc only if it's a SELECT */ context.resultDesc = resultDesc; get_select_query_def(query, &context); break; case CMD_UPDATE: get_update_query_def(query, &context); break; case CMD_INSERT: get_insert_query_def(query, &context); break; case CMD_DELETE: get_delete_query_def(query, &context); break; case CMD_MERGE: get_merge_query_def(query, &context); break; case CMD_NOTHING: appendStringInfoString(buf, "NOTHING"); break; case CMD_UTILITY: get_utility_query_def(query, &context); break; default: elog(ERROR, "unrecognized query command type: %d", query->commandType); break; } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } /* ---------- * get_values_def - Parse back a VALUES list * ---------- */ static void get_values_def(List *values_lists, deparse_context *context) { StringInfo buf = context->buf; bool first_list = true; ListCell *vtl; appendStringInfoString(buf, "VALUES "); foreach(vtl, values_lists) { List *sublist = (List *) lfirst(vtl); bool first_col = true; ListCell *lc; if (first_list) first_list = false; else appendStringInfoString(buf, ", "); appendStringInfoChar(buf, '('); foreach(lc, sublist) { Node *col = (Node *) lfirst(lc); if (first_col) first_col = false; else appendStringInfoChar(buf, ','); /* * Print the value. Whole-row Vars need special treatment. */ get_rule_expr_toplevel(col, context, false); } appendStringInfoChar(buf, ')'); } } /* ---------- * get_with_clause - Parse back a WITH clause * ---------- */ static void get_with_clause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; if (query->cteList == NIL) return; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } if (query->hasRecursive) sep = "WITH RECURSIVE "; else sep = "WITH "; foreach(l, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); appendStringInfoString(buf, sep); appendStringInfoString(buf, quote_identifier(cte->ctename)); if (cte->aliascolnames) { bool first = true; ListCell *col; appendStringInfoChar(buf, '('); foreach(col, cte->aliascolnames) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(col)))); } appendStringInfoChar(buf, ')'); } appendStringInfoString(buf, " AS "); switch (cte->ctematerialized) { case CTEMaterializeDefault: break; case CTEMaterializeAlways: appendStringInfoString(buf, "MATERIALIZED "); break; case CTEMaterializeNever: appendStringInfoString(buf, "NOT MATERIALIZED "); break; } appendStringInfoChar(buf, '('); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); appendStringInfoChar(buf, ')'); if (cte->search_clause) { bool first = true; ListCell *lc; appendStringInfo(buf, " SEARCH %s FIRST BY ", cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); foreach(lc, cte->search_clause->search_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); } if (cte->cycle_clause) { bool first = true; ListCell *lc; appendStringInfoString(buf, " CYCLE "); foreach(lc, cte->cycle_clause->cycle_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); { Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) { appendStringInfoString(buf, " TO "); get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); appendStringInfoString(buf, " DEFAULT "); get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); } } appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); } sep = ", "; } if (PRETTY_INDENT(context)) { context->indentLevel -= PRETTYINDENT_STD; appendContextKeyword(context, "", 0, 0, 0); } else appendStringInfoChar(buf, ' '); } /* ---------- * get_select_query_def - Parse back a SELECT parsetree * ---------- */ static void get_select_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; bool force_colno; ListCell *l; /* Insert the WITH clause if given */ get_with_clause(query, context); /* Subroutines may need to consult the SELECT targetlist and windowClause */ context->targetList = query->targetList; context->windowClause = query->windowClause; /* * If the Query node has a setOperations tree, then it's the top level of * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT * fields are interesting in the top query itself. */ if (query->setOperations) { get_setop_query(query->setOperations, query, context); /* ORDER BY clauses must be simple in this case */ force_colno = true; } else { get_basic_select_query(query, context); force_colno = false; } /* Add the ORDER BY clause if given */ if (query->sortClause != NIL) { appendContextKeyword(context, " ORDER BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_orderby(query->sortClause, query->targetList, force_colno, context); } /* * Add the LIMIT/OFFSET clauses if given. If non-default options, use the * standard spelling of LIMIT. */ if (query->limitOffset != NULL) { appendContextKeyword(context, " OFFSET ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitOffset, context, false); } if (query->limitCount != NULL) { if (query->limitOption == LIMIT_OPTION_WITH_TIES) { // had to add '(' and ')' here because it fails with casting appendContextKeyword(context, " FETCH FIRST (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitCount, context, false); appendStringInfoString(buf, ") ROWS WITH TIES"); } else { appendContextKeyword(context, " LIMIT ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); if (IsA(query->limitCount, Const) && ((Const *) query->limitCount)->constisnull) appendStringInfoString(buf, "ALL"); else get_rule_expr(query->limitCount, context, false); } } /* Add FOR [KEY] UPDATE/SHARE clauses if present */ if (query->hasForUpdate) { foreach(l, query->rowMarks) { RowMarkClause *rc = (RowMarkClause *) lfirst(l); /* don't print implicit clauses */ if (rc->pushedDown) continue; switch (rc->strength) { case LCS_NONE: /* we intentionally throw an error for LCS_NONE */ elog(ERROR, "unrecognized LockClauseStrength %d", (int) rc->strength); break; case LCS_FORKEYSHARE: appendContextKeyword(context, " FOR KEY SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORSHARE: appendContextKeyword(context, " FOR SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORNOKEYUPDATE: appendContextKeyword(context, " FOR NO KEY UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORUPDATE: appendContextKeyword(context, " FOR UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; } appendStringInfo(buf, " OF %s", quote_identifier(get_rtable_name(rc->rti, context))); if (rc->waitPolicy == LockWaitError) appendStringInfoString(buf, " NOWAIT"); else if (rc->waitPolicy == LockWaitSkip) appendStringInfoString(buf, " SKIP LOCKED"); } } } /* * Detect whether query looks like SELECT ... FROM VALUES(); * if so, return the VALUES RTE. Otherwise return NULL. */ static RangeTblEntry * get_simple_values_rte(Query *query, TupleDesc resultDesc) { RangeTblEntry *result = NULL; ListCell *lc; int colno; /* * We want to return true even if the Query also contains OLD or NEW rule * RTEs. So the idea is to scan the rtable and see if there is only one * inFromCl RTE that is a VALUES RTE. */ foreach(lc, query->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); if (rte->rtekind == RTE_VALUES && rte->inFromCl) { if (result) return NULL; /* multiple VALUES (probably not possible) */ result = rte; } else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) continue; /* ignore rule entries */ else return NULL; /* something else -> not simple VALUES */ } /* * We don't need to check the targetlist in any great detail, because * parser/analyze.c will never generate a "bare" VALUES RTE --- they only * appear inside auto-generated sub-queries with very restricted * structure. However, DefineView might have modified the tlist by * injecting new column aliases; so compare tlist resnames against the * RTE's names to detect that. */ if (result) { ListCell *lcn; if (list_length(query->targetList) != list_length(result->eref->colnames)) return NULL; /* this probably cannot happen */ colno = 0; forboth(lc, query->targetList, lcn, result->eref->colnames) { TargetEntry *tle = (TargetEntry *) lfirst(lc); char *cname = strVal(lfirst(lcn)); char *colname; if (tle->resjunk) return NULL; /* this probably cannot happen */ /* compute name that get_target_list would use for column */ colno++; if (resultDesc && colno <= resultDesc->natts) colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); else colname = tle->resname; /* does it match the VALUES RTE? */ if (colname == NULL || strcmp(colname, cname) != 0) return NULL; /* column name has been changed */ } } return result; } static void get_basic_select_query(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *values_rte; char *sep; ListCell *l; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } /* * If the query looks like SELECT * FROM (VALUES ...), then print just the * VALUES part. This reverses what transformValuesClause() did at parse * time. */ values_rte = get_simple_values_rte(query, context->resultDesc); if (values_rte) { get_values_def(values_rte->values_lists, context); return; } /* * Build up the query string - first we say SELECT */ if (query->isReturn) appendStringInfoString(buf, "RETURN"); else appendStringInfoString(buf, "SELECT"); /* Add the DISTINCT clause if given */ if (query->distinctClause != NIL) { if (query->hasDistinctOn) { appendStringInfoString(buf, " DISTINCT ON ("); sep = ""; foreach(l, query->distinctClause) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, false, context); sep = ", "; } appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DISTINCT"); } /* Then we tell what to select (the targetlist) */ get_target_list(query->targetList, context); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add the WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add the GROUP BY clause if given */ if (query->groupClause != NULL || query->groupingSets != NULL) { bool save_ingroupby; appendContextKeyword(context, " GROUP BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); if (query->groupDistinct) appendStringInfoString(buf, "DISTINCT "); save_ingroupby = context->inGroupBy; context->inGroupBy = true; if (query->groupingSets == NIL) { sep = ""; foreach(l, query->groupClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, false, context); sep = ", "; } } else { sep = ""; foreach(l, query->groupingSets) { GroupingSet *grp = lfirst(l); appendStringInfoString(buf, sep); get_rule_groupingset(grp, query->targetList, true, context); sep = ", "; } } context->inGroupBy = save_ingroupby; } /* Add the HAVING clause if given */ if (query->havingQual != NULL) { appendContextKeyword(context, " HAVING ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->havingQual, context, false); } /* Add the WINDOW clause if needed */ if (query->windowClause != NIL) get_rule_windowclause(query, context); } /* ---------- * get_target_list - Parse back a SELECT target list * * This is also used for RETURNING lists in INSERT/UPDATE/DELETE/MERGE. * ---------- */ static void get_target_list(List *targetList, deparse_context *context) { StringInfo buf = context->buf; StringInfoData targetbuf; bool last_was_multiline = false; char *sep; int colno; ListCell *l; /* we use targetbuf to hold each TLE's text temporarily */ initStringInfo(&targetbuf); sep = " "; colno = 0; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); char *colname; char *attname; if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; colno++; /* * Put the new field text into targetbuf so we can decide after we've * got it whether or not it needs to go on a new line. */ resetStringInfo(&targetbuf); context->buf = &targetbuf; /* * We special-case Var nodes rather than using get_rule_expr. This is * needed because get_rule_expr will display a whole-row Var as * "foo.*", which is the preferred notation in most contexts, but at * the top level of a SELECT list it's not right (the parser will * expand that notation into multiple columns, yielding behavior * different from a whole-row Var). We need to call get_variable * directly so that we can tell it to do the right thing, and so that * we can get the attribute name which is the default AS label. */ if (tle->expr && (IsA(tle->expr, Var))) { attname = get_variable((Var *) tle->expr, 0, true, context); } else { get_rule_expr((Node *) tle->expr, context, true); /* * When colNamesVisible is true, we should always show the * assigned column name explicitly. Otherwise, show it only if * it's not FigureColname's fallback. */ attname = context->colNamesVisible ? NULL : "?column?"; } /* * Figure out what the result column should be called. In the context * of a view, use the view's tuple descriptor (so as to pick up the * effects of any column RENAME that's been done on the view). * Otherwise, just use what we can find in the TLE. */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; /* Show AS unless the column's name is correct as-is */ if (colname) /* resname could be NULL */ { if (attname == NULL || strcmp(attname, colname) != 0) appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); } /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { int leading_nl_pos; /* Does the new field start with a new line? */ if (targetbuf.len > 0 && targetbuf.data[0] == '\n') leading_nl_pos = 0; else leading_nl_pos = -1; /* If so, we shouldn't add anything */ if (leading_nl_pos >= 0) { /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the output buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new field is * not the first and either the new field would cause an * overflow or the last field used more than one line. */ if (colno > 1 && ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || last_was_multiline)) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } /* Remember this field's multiline status for next iteration */ last_was_multiline = (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); } /* Add the new field */ appendStringInfoString(buf, targetbuf.data); } /* clean up */ pfree(targetbuf.data); } static void get_setop_query(Node *setOp, Query *query, deparse_context *context) { StringInfo buf = context->buf; bool need_paren; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); if (IsA(setOp, RangeTblRef)) { RangeTblRef *rtr = (RangeTblRef *) setOp; RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); Query *subquery = rte->subquery; Assert(subquery != NULL); Assert(subquery->setOperations == NULL); /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ need_paren = (subquery->cteList || subquery->sortClause || subquery->rowMarks || subquery->limitOffset || subquery->limitCount); if (need_paren) appendStringInfoChar(buf, '('); get_query_def(subquery, buf, context->namespaces, context->resultDesc, context->colNamesVisible, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoChar(buf, ')'); } else if (IsA(setOp, SetOperationStmt)) { SetOperationStmt *op = (SetOperationStmt *) setOp; int subindent; bool save_colnamesvisible; /* * We force parens when nesting two SetOperationStmts, except when the * lefthand input is another setop of the same kind. Syntactically, * we could omit parens in rather more cases, but it seems best to use * parens to flag cases where the setop operator changes. If we use * parens, we also increase the indentation level for the child query. * * There are some cases in which parens are needed around a leaf query * too, but those are more easily handled at the next level down (see * code above). */ if (IsA(op->larg, SetOperationStmt)) { SetOperationStmt *lop = (SetOperationStmt *) op->larg; if (op->op == lop->op && op->all == lop->all) need_paren = false; else need_paren = true; } else need_paren = false; if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; appendContextKeyword(context, "", subindent, 0, 0); } else subindent = 0; get_setop_query(op->larg, query, context); if (need_paren) appendContextKeyword(context, ") ", -subindent, 0, 0); else if (PRETTY_INDENT(context)) appendContextKeyword(context, "", -subindent, 0, 0); else appendStringInfoChar(buf, ' '); switch (op->op) { case SETOP_UNION: appendStringInfoString(buf, "UNION "); break; case SETOP_INTERSECT: appendStringInfoString(buf, "INTERSECT "); break; case SETOP_EXCEPT: appendStringInfoString(buf, "EXCEPT "); break; default: elog(ERROR, "unrecognized set op: %d", (int) op->op); } if (op->all) appendStringInfoString(buf, "ALL "); /* Always parenthesize if RHS is another setop */ need_paren = IsA(op->rarg, SetOperationStmt); /* * The indentation code here is deliberately a bit different from that * for the lefthand input, because we want the line breaks in * different places. */ if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; } else subindent = 0; appendContextKeyword(context, "", subindent, 0, 0); /* * The output column names of the RHS sub-select don't matter. */ save_colnamesvisible = context->colNamesVisible; context->colNamesVisible = false; get_setop_query(op->rarg, query, context); context->colNamesVisible = save_colnamesvisible; if (PRETTY_INDENT(context)) context->indentLevel -= subindent; if (need_paren) appendContextKeyword(context, ")", 0, 0, 0); } else { elog(ERROR, "unrecognized node type: %d", (int) nodeTag(setOp)); } } /* * Display a sort/group clause. * * Also returns the expression tree, so caller need not find it again. */ static Node * get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; TargetEntry *tle; Node *expr; tle = get_sortgroupref_tle(ref, tlist); expr = (Node *) tle->expr; /* * Use column-number form if requested by caller. Otherwise, if * expression is a constant, force it to be dumped with an explicit cast * as decoration --- this is because a simple integer constant is * ambiguous (and will be misinterpreted by findTargetlistEntrySQL92()) if * we dump it without any decoration. Similarly, if it's just a Var, * there is risk of misinterpretation if the column name is reassigned in * the SELECT list, so we may need to force table qualification. And, if * it's anything more complex than a simple Var, then force extra parens * around it, to ensure it can't be misinterpreted as a cube() or rollup() * construct. */ if (force_colno) { Assert(!tle->resjunk); appendStringInfo(buf, "%d", tle->resno); } else if (!expr) /* do nothing, probably can't happen */ ; else if (IsA(expr, Const)) get_const_expr((Const *) expr, context, 1); else if (IsA(expr, Var)) { /* Tell get_variable to check for name conflict */ bool save_varinorderby = context->varInOrderBy; context->varInOrderBy = true; (void) get_variable((Var *) expr, 0, false, context); context->varInOrderBy = save_varinorderby; } else { /* * We must force parens for function-like expressions even if * PRETTY_PAREN is off, since those are the ones in danger of * misparsing. For other expressions we need to force them only if * PRETTY_PAREN is on, since otherwise the expression will output them * itself. (We can't skip the parens.) */ bool need_paren = (PRETTY_PAREN(context) || IsA(expr, FuncExpr) || IsA(expr, Aggref) || IsA(expr, WindowFunc) || IsA(expr, JsonConstructorExpr)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, true); if (need_paren) appendStringInfoChar(context->buf, ')'); } return expr; } /* * Display a GroupingSet */ static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context) { ListCell *l; StringInfo buf = context->buf; bool omit_child_parens = true; char *sep = ""; switch (gset->kind) { case GROUPING_SET_EMPTY: appendStringInfoString(buf, "()"); return; case GROUPING_SET_SIMPLE: { if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, '('); foreach(l, gset->content) { Index ref = lfirst_int(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(ref, targetlist, false, context); sep = ", "; } if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, ')'); } return; case GROUPING_SET_ROLLUP: appendStringInfoString(buf, "ROLLUP("); break; case GROUPING_SET_CUBE: appendStringInfoString(buf, "CUBE("); break; case GROUPING_SET_SETS: appendStringInfoString(buf, "GROUPING SETS ("); omit_child_parens = false; break; } foreach(l, gset->content) { appendStringInfoString(buf, sep); get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); sep = ", "; } appendStringInfoChar(buf, ')'); } /* * Display an ORDER BY list. */ static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = ""; foreach(l, orderList) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); Node *sortexpr; Oid sortcoltype; TypeCacheEntry *typentry; appendStringInfoString(buf, sep); sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, force_colno, context); sortcoltype = exprType(sortexpr); /* See whether operator is default < or > for datatype */ typentry = lookup_type_cache(sortcoltype, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); if (srt->sortop == typentry->lt_opr) { /* ASC is default, so emit nothing for it */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); } else if (srt->sortop == typentry->gt_opr) { appendStringInfoString(buf, " DESC"); /* DESC defaults to NULLS FIRST */ if (!srt->nulls_first) appendStringInfoString(buf, " NULLS LAST"); } else { appendStringInfo(buf, " USING %s", generate_operator_name(srt->sortop, sortcoltype, sortcoltype)); /* be specific to eliminate ambiguity */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); else appendStringInfoString(buf, " NULLS LAST"); } sep = ", "; } } /* * Display a WINDOW clause. * * Note that the windowClause list might contain only anonymous window * specifications, in which case we should print nothing here. */ static void get_rule_windowclause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = NULL; foreach(l, query->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->name == NULL) continue; /* ignore anonymous windows */ if (sep == NULL) appendContextKeyword(context, " WINDOW ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); else appendStringInfoString(buf, sep); appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); get_rule_windowspec(wc, query->targetList, context); sep = ", "; } } /* * Display a window definition */ static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context) { StringInfo buf = context->buf; bool needspace = false; const char *sep; ListCell *l; appendStringInfoChar(buf, '('); if (wc->refname) { appendStringInfoString(buf, quote_identifier(wc->refname)); needspace = true; } /* partition clauses are always inherited, so only print if no refname */ if (wc->partitionClause && !wc->refname) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "PARTITION BY "); sep = ""; foreach(l, wc->partitionClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, false, context); sep = ", "; } needspace = true; } /* print ordering clause only if not inherited */ if (wc->orderClause && !wc->copiedOrder) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "ORDER BY "); get_rule_orderby(wc->orderClause, targetList, false, context); needspace = true; } /* framing clause is never inherited, so print unless it's default */ if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) { if (needspace) appendStringInfoChar(buf, ' '); if (wc->frameOptions & FRAMEOPTION_RANGE) appendStringInfoString(buf, "RANGE "); else if (wc->frameOptions & FRAMEOPTION_ROWS) appendStringInfoString(buf, "ROWS "); else if (wc->frameOptions & FRAMEOPTION_GROUPS) appendStringInfoString(buf, "GROUPS "); else Assert(false); if (wc->frameOptions & FRAMEOPTION_BETWEEN) appendStringInfoString(buf, "BETWEEN "); if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) appendStringInfoString(buf, "UNBOUNDED PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) { get_rule_expr(wc->startOffset, context, false); if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); if (wc->frameOptions & FRAMEOPTION_BETWEEN) { appendStringInfoString(buf, "AND "); if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) { get_rule_expr(wc->endOffset, context, false); if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); } if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) appendStringInfoString(buf, "EXCLUDE GROUP "); else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) appendStringInfoString(buf, "EXCLUDE TIES "); /* we will now have a trailing space; remove it */ buf->len--; } appendStringInfoChar(buf, ')'); } /* ---------- * get_insert_query_def - Parse back an INSERT parsetree * ---------- */ static void get_insert_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *select_rte = NULL; RangeTblEntry *values_rte = NULL; RangeTblEntry *rte; ListCell *l; List *strippedexprs = NIL; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * If it's an INSERT ... SELECT or multi-row VALUES, there will be a * single RTE for the SELECT or VALUES. Plain VALUES has neither. */ foreach(l, query->rtable) { rte = (RangeTblEntry *) lfirst(l); if (rte->rtekind == RTE_SUBQUERY) { if (select_rte) elog(ERROR, "too many subquery RTEs in INSERT"); select_rte = rte; } if (rte->rtekind == RTE_VALUES) { if (values_rte) elog(ERROR, "too many values RTEs in INSERT"); values_rte = rte; } } if (select_rte && values_rte) elog(ERROR, "both subquery and values RTEs in INSERT"); /* * Start the query with INSERT INTO relname */ rte = rt_fetch(query->resultRelation, query->rtable); Assert(rte->rtekind == RTE_RELATION); if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } appendStringInfo(buf, "INSERT INTO %s", generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed; INSERT requires explicit AS */ get_rte_alias(rte, query->resultRelation, true, context); /* always want a space here */ appendStringInfoChar(buf, ' '); /* * Add the insert-column-names list. Any indirection decoration needed on * the column names can be inferred from the top targetlist. */ if (query->targetList) { strippedexprs = get_insert_column_names_list(query->targetList, buf, context, rte); } if (query->override) { if (query->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); else if (query->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, "OVERRIDING USER VALUE "); } if (select_rte) { /* Add the SELECT */ get_query_def(select_rte->subquery, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); } else if (values_rte) { /* Add the multi-VALUES expression lists */ get_values_def(values_rte->values_lists, context); } else if (strippedexprs) { /* Add the single-VALUES expression list */ appendContextKeyword(context, "VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else { /* No expressions, so it must be DEFAULT VALUES */ appendStringInfoString(buf, "DEFAULT VALUES"); } /* Add ON CONFLICT if present */ if (query->onConflict) { OnConflictExpr *confl = query->onConflict; appendStringInfoString(buf, " ON CONFLICT"); if (confl->arbiterElems) { /* Add the single-VALUES expression list */ appendStringInfoChar(buf, '('); get_rule_expr((Node *) confl->arbiterElems, context, false); appendStringInfoChar(buf, ')'); /* Add a WHERE clause (for partial indexes) if given */ if (confl->arbiterWhere != NULL) { bool save_varprefix; /* * Force non-prefixing of Vars, since parser assumes that they * belong to target relation. WHERE clause does not use * InferenceElem, so this is separately required. */ save_varprefix = context->varprefix; context->varprefix = false; appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->arbiterWhere, context, false); context->varprefix = save_varprefix; } } else if (OidIsValid(confl->constraint)) { char *constraint = get_constraint_name(confl->constraint); int64 shardId = context->shardid; if (shardId > 0) { AppendShardIdToName(&constraint, shardId); } if (!constraint) elog(ERROR, "cache lookup failed for constraint %u", confl->constraint); appendStringInfo(buf, " ON CONSTRAINT %s", quote_identifier(constraint)); } if (confl->action == ONCONFLICT_NOTHING) { appendStringInfoString(buf, " DO NOTHING"); } else { appendStringInfoString(buf, " DO UPDATE SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, confl->onConflictSet, context, rte); /* Add a WHERE clause if given */ if (confl->onConflictWhere != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->onConflictWhere, context, false); } } } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_update_query_def - Parse back an UPDATE parsetree * ---------- */ static void get_update_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with UPDATE relname SET */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } appendStringInfoString(buf, " SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, query->targetList, context, rte); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_update_query_targetlist_def - Parse back an UPDATE targetlist * ---------- */ static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte) { StringInfo buf = context->buf; ListCell *l; ListCell *next_ma_cell; int remaining_ma_columns; const char *sep; SubLink *cur_ma_sublink; List *ma_sublinks; targetList = ExpandMergedSubscriptingRefEntries(targetList); /* * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks * into a list. We expect them to appear, in ID order, in resjunk tlist * entries. */ ma_sublinks = NIL; if (query->hasSubLinks) /* else there can't be any */ { foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk && IsA(tle->expr, SubLink)) { SubLink *sl = (SubLink *) tle->expr; if (sl->subLinkType == MULTIEXPR_SUBLINK) { ma_sublinks = lappend(ma_sublinks, sl); Assert(sl->subLinkId == list_length(ma_sublinks)); } } } ensure_update_targetlist_in_param_order(targetList); } next_ma_cell = list_head(ma_sublinks); cur_ma_sublink = NULL; remaining_ma_columns = 0; /* Add the comma separated list of 'attname = value' */ sep = ""; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *expr; if (tle->resjunk) continue; /* ignore junk entries */ /* Emit separator (OK whether we're in multiassignment or not) */ appendStringInfoString(buf, sep); sep = ", "; /* * Check to see if we're starting a multiassignment group: if so, * output a left paren. */ if (next_ma_cell != NULL && cur_ma_sublink == NULL) { /* * We must dig down into the expr to see if it's a PARAM_MULTIEXPR * Param. That could be buried under FieldStores and * SubscriptingRefs and CoerceToDomains (cf processIndirection()), * and underneath those there could be an implicit type coercion. * Because we would ignore implicit type coercions anyway, we * don't need to be as careful as processIndirection() is about * descending past implicit CoerceToDomains. */ expr = (Node *) tle->expr; while (expr) { if (IsA(expr, FieldStore)) { FieldStore *fstore = (FieldStore *) expr; expr = (Node *) linitial(fstore->newvals); } else if (IsA(expr, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) expr; if (sbsref->refassgnexpr == NULL) break; expr = (Node *) sbsref->refassgnexpr; } else if (IsA(expr, CoerceToDomain)) { CoerceToDomain *cdomain = (CoerceToDomain *) expr; if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; expr = (Node *) cdomain->arg; } else break; } expr = strip_implicit_coercions(expr); if (expr && IsA(expr, Param) && ((Param *) expr)->paramkind == PARAM_MULTIEXPR) { cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); next_ma_cell = lnext(ma_sublinks, next_ma_cell); remaining_ma_columns = count_nonjunk_tlist_entries( ((Query *) cur_ma_sublink->subselect)->targetList); Assert(((Param *) expr)->paramid == ((cur_ma_sublink->subLinkId << 16) | 1)); appendStringInfoChar(buf, '('); } } /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. */ expr = processIndirection((Node *) tle->expr, context); /* * If we're in a multiassignment, skip printing anything more, unless * this is the last column; in which case, what we print should be the * sublink, not the Param. */ if (cur_ma_sublink != NULL) { if (--remaining_ma_columns > 0) continue; /* not the last column of multiassignment */ appendStringInfoChar(buf, ')'); expr = (Node *) cur_ma_sublink; cur_ma_sublink = NULL; } appendStringInfoString(buf, " = "); get_rule_expr(expr, context, false); } } /* ---------- * get_delete_query_def - Parse back a DELETE parsetree * ---------- */ static void get_delete_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with DELETE FROM relname */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } /* Add the USING clause if given */ get_from_clause(query, " USING ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_merge_query_def - Parse back a MERGE parsetree * ---------- */ static void get_merge_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; ListCell *lc; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with MERGE INTO relname */ rte = ExtractResultRelationRTE(query); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); if (rte->alias != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } /* Print the source relation and join clause */ get_from_clause(query, " USING ", context); appendContextKeyword(context, " ON ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_expr(query->jointree->quals, context, false); /* Print each merge action */ foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); appendContextKeyword(context, " WHEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); appendStringInfo(buf, "%sMATCHED", action->matched ? "" : "NOT "); if (action->qual) { appendContextKeyword(context, " AND ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); get_rule_expr(action->qual, context, false); } appendContextKeyword(context, " THEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); if (action->commandType == CMD_INSERT) { /* This generally matches get_insert_query_def() */ List *strippedexprs = NIL; const char *sep = ""; ListCell *lc2; appendStringInfoString(buf, "INSERT"); if (action->targetList) appendStringInfoString(buf, " ("); foreach(lc2, action->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc2); Assert(!tle->resjunk); appendStringInfoString(buf, sep); sep = ", "; appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } if (action->targetList) appendStringInfoChar(buf, ')'); if (action->override) { if (action->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, " OVERRIDING SYSTEM VALUE"); else if (action->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, " OVERRIDING USER VALUE"); } if (strippedexprs) { appendContextKeyword(context, " VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 4); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DEFAULT VALUES"); } else if (action->commandType == CMD_UPDATE) { appendStringInfoString(buf, "UPDATE SET "); get_update_query_targetlist_def(query, action->targetList, context, rte); } else if (action->commandType == CMD_DELETE) appendStringInfoString(buf, "DELETE"); else if (action->commandType == CMD_NOTHING) appendStringInfoString(buf, "DO NOTHING"); } /* No RETURNING support in MERGE yet */ Assert(query->returningList == NIL); ereport(DEBUG1, (errmsg("", buf->data))); } /* ---------- * get_utility_query_def - Parse back a UTILITY parsetree * ---------- */ static void get_utility_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) { NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "NOTIFY %s", quote_identifier(stmt->conditionname)); if (stmt->payload) { appendStringInfoString(buf, ", "); simple_quote_literal(buf, stmt->payload); } } else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) { TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; List *relationList = stmt->relations; ListCell *relationCell = NULL; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "TRUNCATE TABLE"); foreach(relationCell, relationList) { RangeVar *relationVar = (RangeVar *) lfirst(relationCell); Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); char *relationName = generate_relation_or_shard_name(relationId, context->distrelid, context->shardid, NIL); appendStringInfo(buf, " %s", relationName); if (lnext(relationList, relationCell) != NULL) { appendStringInfo(buf, ","); } } if (stmt->restart_seqs) { appendStringInfo(buf, " RESTART IDENTITY"); } if (stmt->behavior == DROP_CASCADE) { appendStringInfo(buf, " CASCADE"); } } else { /* Currently only NOTIFY utility commands can appear in rules */ elog(ERROR, "unexpected utility statement type"); } } /* * Display a Var appropriately. * * In some cases (currently only when recursing into an unnamed join) * the Var's varlevelsup has to be interpreted with respect to a context * above the current one; levelsup indicates the offset. * * If istoplevel is true, the Var is at the top level of a SELECT's * targetlist, which means we need special treatment of whole-row Vars. * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a * dirty hack to prevent "tab.*" from being expanded into multiple columns. * (The parser will strip the useless coercion, so no inefficiency is added in * dump and reload.) We used to print just "tab" in such cases, but that is * ambiguous and will yield the wrong result if "tab" is also a plain column * name in the query. * * Returns the attname of the Var, or NULL if the Var has no attname (because * it is a whole-row Var or a subplan output reference). */ static char * get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; AttrNumber attnum; int varno; AttrNumber varattno; int netlevelsup; deparse_namespace *dpns; deparse_columns *colinfo; char *refname; char *attname; bool need_prefix; /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. Also * find the aliases previously assigned for this RTE. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { /* * We might have been asked to map child Vars to some parent relation. */ if (context->appendparents && dpns->appendrels) { int pvarno = varno; AttrNumber pvarattno = varattno; AppendRelInfo *appinfo = dpns->appendrels[pvarno]; bool found = false; /* Only map up to inheritance parents, not UNION ALL appendrels */ while (appinfo && rt_fetch(appinfo->parent_relid, dpns->rtable)->rtekind == RTE_RELATION) { found = false; if (pvarattno > 0) /* system columns stay as-is */ { if (pvarattno > appinfo->num_child_cols) break; /* safety check */ pvarattno = appinfo->parent_colnos[pvarattno - 1]; if (pvarattno == 0) break; /* Var is local to child */ } pvarno = appinfo->parent_relid; found = true; /* If the parent is itself a child, continue up. */ Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); appinfo = dpns->appendrels[pvarno]; } /* * If we found an ancestral rel, and that rel is included in * appendparents, print that column not the original one. */ if (found && bms_is_member(pvarno, context->appendparents)) { varno = pvarno; varattno = pvarattno; } } rte = rt_fetch(varno, dpns->rtable); refname = (char *) list_nth(dpns->rtable_names, varno - 1); colinfo = deparse_columns_fetch(varno, dpns); attnum = varattno; } else { resolve_special_varno((Node *) var, context, get_special_variable, NULL); return NULL; } /* * The planner will sometimes emit Vars referencing resjunk elements of a * subquery's target list (this is currently only possible if it chooses * to generate a "physical tlist" for a SubqueryScan or CteScan node). * Although we prefer to print subquery-referencing Vars using the * subquery's alias, that's not possible for resjunk items since they have * no alias. So in that case, drill down to the subplan and print the * contents of the referenced tlist item. This works because in a plan * tree, such Vars can only occur in a SubqueryScan or CteScan node, and * we'll have set dpns->inner_plan to reference the child plan node. */ if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && attnum > list_length(rte->eref->colnames) && dpns->inner_plan) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); /* * Force parentheses because our caller probably assumed a Var is a * simple expression. */ if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, '('); get_rule_expr((Node *) tle->expr, context, true); if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, ')'); pop_child_plan(dpns, &save_dpns); return NULL; } /* * If it's an unnamed join, look at the expansion of the alias variable. * If it's a simple reference to one of the input vars, then recursively * print the name of that var instead. When it's not a simple reference, * we have to just print the unqualified join column name. (This can only * happen with "dangerous" merged columns in a JOIN USING; we took pains * previously to make the unqualified column name unique in such cases.) * * This wouldn't work in decompiling plan trees, because we don't store * joinaliasvars lists after planning; but a plan tree should never * contain a join alias variable. */ if (rte->rtekind == RTE_JOIN && rte->alias == NULL) { if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); if (attnum > 0) { Var *aliasvar; aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); /* we intentionally don't strip implicit coercions here */ if (aliasvar && IsA(aliasvar, Var)) { return get_variable(aliasvar, var->varlevelsup + levelsup, istoplevel, context); } } /* * Unnamed join has no refname. (Note: since it's unnamed, there is * no way the user could have referenced it to create a whole-row Var * for it. So we don't have to cover that case below.) */ Assert(refname == NULL); } if (attnum == InvalidAttrNumber) attname = NULL; else if (attnum > 0) { /* Get column name to use from the colinfo struct */ if (attnum > colinfo->num_cols) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); attname = colinfo->colnames[attnum - 1]; /* * If we find a Var referencing a dropped column, it seems better to * print something (anything) than to fail. In general this should * not happen, but it used to be possible for some cases involving * functions returning named composite types, and perhaps there are * still bugs out there. */ if (attname == NULL) attname = "?dropped?column?"; } else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* System column on a Citus shard */ attname = get_attname(rte->relid, attnum, false); } else { /* System column - name is fixed, get it from the catalog */ attname = get_rte_attribute_name(rte, attnum); } need_prefix = (context->varprefix || attname == NULL); /* * If we're considering a plain Var in an ORDER BY (but not GROUP BY) * clause, we may need to add a table-name prefix to prevent * findTargetlistEntrySQL92 from misinterpreting the name as an * output-column name. To avoid cluttering the output with unnecessary * prefixes, do so only if there is a name match to a SELECT tlist item * that is different from the Var. */ if (context->varInOrderBy && !context->inGroupBy && !need_prefix) { int colno = 0; ListCell *l; foreach(l, context->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); char *colname; if (tle->resjunk) continue; /* ignore junk entries */ colno++; /* This must match colname-choosing logic in get_target_list() */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; if (colname && strcmp(colname, attname) == 0 && !equal(var, tle->expr)) { need_prefix = true; break; } } } if (refname && need_prefix) { appendStringInfoString(buf, quote_identifier(refname)); appendStringInfoChar(buf, '.'); } if (attname) appendStringInfoString(buf, quote_identifier(attname)); else { appendStringInfoChar(buf, '*'); if (istoplevel) { if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* use rel.*::shard_name instead of rel.*::table_name */ appendStringInfo(buf, "::%s", generate_rte_shard_name(rte)); } else { appendStringInfo(buf, "::%s", format_type_with_typemod(var->vartype, var->vartypmod)); } } } return attname; } /* * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This * routine is actually a callback for get_special_varno, which handles finding * the correct TargetEntry. We get the expression contained in that * TargetEntry and just need to deparse it, a job we can throw back on * get_rule_expr. */ static void get_special_variable(Node *node, deparse_context *context, void *callback_arg) { StringInfo buf = context->buf; /* * For a non-Var referent, force parentheses because our caller probably * assumed a Var is a simple expression. */ if (!IsA(node, Var)) appendStringInfoChar(buf, '('); get_rule_expr(node, context, true); if (!IsA(node, Var)) appendStringInfoChar(buf, ')'); } /* * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, * invoke the callback provided. */ static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) { Var *var; deparse_namespace *dpns; /* This function is recursive, so let's be paranoid. */ check_stack_depth(); /* If it's not a Var, invoke the callback. */ if (!IsA(node, Var)) { (*callback) (node, context, callback_arg); return; } /* Find appropriate nesting depth */ var = (Var *) node; dpns = (deparse_namespace *) list_nth(context->namespaces, var->varlevelsup); /* * It's a special RTE, so recurse. */ if (var->varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; Bitmapset *save_appendparents; tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); /* If we're descending to the first child of an Append or MergeAppend, * update appendparents. This will affect deparsing of all Vars * appearing within the eventually-resolved subexpression. */ save_appendparents = context->appendparents; if (IsA(dpns->plan, Append)) context->appendparents = bms_union(context->appendparents, ((Append *) dpns->plan)->apprelids); else if (IsA(dpns->plan, MergeAppend)) context->appendparents = bms_union(context->appendparents, ((MergeAppend *) dpns->plan)->apprelids); push_child_plan(dpns, dpns->outer_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); context->appendparents = save_appendparents; return; } else if (var->varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); push_child_plan(dpns, dpns->inner_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); return; } else if (var->varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; tle = get_tle_by_resno(dpns->index_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); return; } else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) elog(ERROR, "bogus varno: %d", var->varno); /* Not special. Just invoke the callback. */ (*callback) (node, context, callback_arg); } /* * Get the name of a field of an expression of composite type. The * expression is usually a Var, but we handle other cases too. * * levelsup is an extra offset to interpret the Var's varlevelsup correctly. * * This is fairly straightforward when the expression has a named composite * type; we need only look up the type in the catalogs. However, the type * could also be RECORD. Since no actual table or view column is allowed to * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE * or to a subquery output. We drill down to find the ultimate defining * expression and attempt to infer the field name from it. We ereport if we * can't determine the name. * * Similarly, a PARAM of type RECORD has to refer to some expression of * a determinable composite type. */ static const char * get_name_for_var_field(Var *var, int fieldno, int levelsup, deparse_context *context) { RangeTblEntry *rte; AttrNumber attnum; int netlevelsup; deparse_namespace *dpns; int varno; AttrNumber varattno; TupleDesc tupleDesc; Node *expr; /* * If it's a RowExpr that was expanded from a whole-row Var, use the * column names attached to it. (We could let get_expr_result_tupdesc() * handle this, but it's much cheaper to just pull out the name we need.) */ if (IsA(var, RowExpr)) { RowExpr *r = (RowExpr *) var; if (fieldno > 0 && fieldno <= list_length(r->colnames)) return strVal(list_nth(r->colnames, fieldno - 1)); } /* * If it's a Param of type RECORD, try to find what the Param refers to. */ if (IsA(var, Param)) { Param *param = (Param *) var; ListCell *ancestor_cell; expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so recurse to decipher the field name */ deparse_namespace save_dpns; const char *result; push_ancestor_plan(dpns, ancestor_cell, &save_dpns); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); pop_ancestor_plan(dpns, &save_dpns); return result; } } /* * If it's a Var of type RECORD, we have to find what the Var refers to; * if not, we can use get_expr_result_tupdesc(). */ if (!IsA(var, Var) || var->vartype != RECORDOID) { tupleDesc = get_expr_result_tupdesc((Node *) var, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { rte = rt_fetch(varno, dpns->rtable); attnum = varattno; } else if (varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->outer_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->outer_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->inner_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; const char *result; tle = get_tle_by_resno(dpns->index_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); Assert(netlevelsup == 0); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); return result; } else { elog(ERROR, "bogus varno: %d", varno); return NULL; /* keep compiler quiet */ } if (attnum == InvalidAttrNumber) { /* Var is whole-row reference to RTE, so select the right field */ return get_rte_attribute_name(rte, fieldno); } /* * This part has essentially the same logic as the parser's * expandRecordVariable() function, but we are dealing with a different * representation of the input context, and we only need one field name * not a TupleDesc. Also, we need special cases for finding subquery and * CTE subplans when deparsing Plan trees. */ expr = (Node *) var; /* default if we can't drill down */ switch (rte->rtekind) { case RTE_RELATION: case RTE_VALUES: case RTE_NAMEDTUPLESTORE: case RTE_RESULT: /* * This case should not occur: a column of a table or values list * shouldn't have type RECORD. Fall through and fail (most * likely) at the bottom. */ break; case RTE_SUBQUERY: /* Subselect-in-FROM: examine sub-select's output expr */ { if (rte->subquery) { TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "subquery %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the sub-select to see what its Var * refers to. We have to build an additional level of * namespace to keep in step with varlevelsup in the * subselect. */ deparse_namespace mydpns; const char *result; set_deparse_for_query(&mydpns, rte->subquery, context->namespaces); context->namespaces = lcons(&mydpns, context->namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = list_delete_first(context->namespaces); return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have complete * RTE entries (in particular, rte->subquery is NULL). But * the only place we'd see a Var directly referencing a * SUBQUERY RTE is in a SubqueryScan plan node, and we can * look into the child plan's tlist instead. */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) elog(ERROR, "failed to find plan for subquery %s", rte->eref->aliasname); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; case RTE_JOIN: /* Join RTE --- recursively inspect the alias variable */ if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); Assert(expr != NULL); /* we intentionally don't strip implicit coercions here */ if (IsA(expr, Var)) return get_name_for_var_field((Var *) expr, fieldno, var->varlevelsup + levelsup, context); /* else fall through to inspect the expression */ break; case RTE_FUNCTION: case RTE_TABLEFUNC: /* * We couldn't get here unless a function is declared with one of * its result columns as RECORD, which is not allowed. */ break; case RTE_CTE: /* CTE reference: examine subquery's output expr */ { CommonTableExpr *cte = NULL; Index ctelevelsup; ListCell *lc; /* * Try to find the referenced CTE using the namespace stack. */ ctelevelsup = rte->ctelevelsup + netlevelsup; if (ctelevelsup >= list_length(context->namespaces)) lc = NULL; else { deparse_namespace *ctedpns; ctedpns = (deparse_namespace *) list_nth(context->namespaces, ctelevelsup); foreach(lc, ctedpns->ctes) { cte = (CommonTableExpr *) lfirst(lc); if (strcmp(cte->ctename, rte->ctename) == 0) break; } } if (lc != NULL) { Query *ctequery = (Query *) cte->ctequery; TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "subquery %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the CTE to see what its Var refers to. * We have to build an additional level of namespace * to keep in step with varlevelsup in the CTE. * Furthermore it could be an outer CTE, so we may * have to delete some levels of namespace. */ List *save_nslist = context->namespaces; List *new_nslist; deparse_namespace mydpns; const char *result; set_deparse_for_query(&mydpns, ctequery, context->namespaces); new_nslist = list_copy_tail(context->namespaces, ctelevelsup); context->namespaces = lcons(&mydpns, new_nslist); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = save_nslist; return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have a CTE * list. But the only places we'd see a Var directly * referencing a CTE RTE are in CteScan or WorkTableScan * plan nodes. For those cases, set_deparse_plan arranged * for dpns->inner_plan to be the plan node that emits the * CTE or RecursiveUnion result, and we can look at its * tlist instead. */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) elog(ERROR, "failed to find plan for CTE %s", rte->eref->aliasname); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; } /* * We now have an expression we can't expand any more, so see if * get_expr_result_tupdesc() can do anything with it. */ tupleDesc = get_expr_result_tupdesc(expr, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* * Try to find the referenced expression for a PARAM_EXEC Param that might * reference a parameter supplied by an upper NestLoop or SubPlan plan node. * * If successful, return the expression and set *dpns_p and *ancestor_cell_p * appropriately for calling push_ancestor_plan(). If no referent can be * found, return NULL. */ static Node * find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p) { /* Initialize output parameters to prevent compiler warnings */ *dpns_p = NULL; *ancestor_cell_p = NULL; /* * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or * SubPlan argument. This will necessarily be in some ancestor of the * current expression's Plan. */ if (param->paramkind == PARAM_EXEC) { deparse_namespace *dpns; Plan *child_plan; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); child_plan = dpns->plan; foreach(lc, dpns->ancestors) { Node *ancestor = (Node *) lfirst(lc); ListCell *lc2; /* * NestLoops transmit params to their inner child only. */ if (IsA(ancestor, NestLoop) && child_plan == innerPlan(ancestor)) { NestLoop *nl = (NestLoop *) ancestor; foreach(lc2, nl->nestParams) { NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); if (nlp->paramno == param->paramid) { /* Found a match, so return it */ *dpns_p = dpns; *ancestor_cell_p = lc; return (Node *) nlp->paramval; } } } /* * Check to see if we're crawling up from a subplan. */ if(IsA(ancestor, SubPlan)) { SubPlan *subplan = (SubPlan *) ancestor; ListCell *lc3; ListCell *lc4; /* Matched subplan, so check its arguments */ forboth(lc3, subplan->parParam, lc4, subplan->args) { int paramid = lfirst_int(lc3); Node *arg = (Node *) lfirst(lc4); if (paramid == param->paramid) { /* * Found a match, so return it. But, since Vars in * the arg are to be evaluated in the surrounding * context, we have to point to the next ancestor item * that is *not* a SubPlan. */ ListCell *rest; for_each_cell(rest, dpns->ancestors, lnext(dpns->ancestors, lc)) { Node *ancestor2 = (Node *) lfirst(rest); if (!IsA(ancestor2, SubPlan)) { *dpns_p = dpns; *ancestor_cell_p = rest; return arg; } } elog(ERROR, "SubPlan cannot be outermost ancestor"); } } /* SubPlan isn't a kind of Plan, so skip the rest */ continue; } /* * We need not consider the ancestor's initPlan list, since * initplans never have any parParams. */ /* No luck, crawl up to next ancestor */ child_plan = (Plan *) ancestor; } } /* No referent found */ return NULL; } /* * Display a Param appropriately. */ static void get_parameter(Param *param, deparse_context *context) { Node *expr; deparse_namespace *dpns; ListCell *ancestor_cell; /* * If it's a PARAM_EXEC parameter, try to locate the expression from which * the parameter was computed. Note that failing to find a referent isn't * an error, since the Param might well be a subplan output rather than an * input. */ expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so print it */ deparse_namespace save_dpns; bool save_varprefix; bool need_paren; /* Switch attention to the ancestor plan node */ push_ancestor_plan(dpns, ancestor_cell, &save_dpns); /* * Force prefixing of Vars, since they won't belong to the relation * being scanned in the original plan node. */ save_varprefix = context->varprefix; context->varprefix = true; /* * A Param's expansion is typically a Var, Aggref, GroupingFunc, or * upper-level Param, which wouldn't need extra parentheses. * Otherwise, insert parens to ensure the expression looks atomic. */ need_paren = !(IsA(expr, Var) || IsA(expr, Aggref) || IsA(expr, GroupingFunc) || IsA(expr, Param)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, false); if (need_paren) appendStringInfoChar(context->buf, ')'); context->varprefix = save_varprefix; pop_ancestor_plan(dpns, &save_dpns); return; } /* * If it's an external parameter, see if the outermost namespace provides * function argument names. */ if (param->paramkind == PARAM_EXTERN && context->namespaces != NIL) { dpns = llast(context->namespaces); if (dpns->argnames && param->paramid > 0 && param->paramid <= dpns->numargs) { char *argname = dpns->argnames[param->paramid - 1]; if (argname) { bool should_qualify = false; ListCell *lc; /* * Qualify the parameter name if there are any other deparse * namespaces with range tables. This avoids qualifying in * trivial cases like "RETURN a + b", but makes it safe in all * other cases. */ foreach(lc, context->namespaces) { deparse_namespace *depns = lfirst(lc); if (depns->rtable_names != NIL) { should_qualify = true; break; } } if (should_qualify) { appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); appendStringInfoChar(context->buf, '.'); } appendStringInfoString(context->buf, quote_identifier(argname)); return; } } } /* * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. * For composite types, add cast to the parameter to ease remote node detect * the type. */ if (param->paramtype >= FirstNormalObjectId) { char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); } else { appendStringInfo(context->buf, "$%d", param->paramid); } } /* * get_simple_binary_op_name * * helper function for isSimpleNode * will return single char binary operator name, or NULL if it's not */ static const char * get_simple_binary_op_name(OpExpr *expr) { List *args = expr->args; if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); const char *op; op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); if (strlen(op) == 1) return op; } return NULL; } /* * isSimpleNode - check if given node is simple (doesn't need parenthesizing) * * true : simple in the context of parent node's type * false : not simple */ static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags) { if (!node) return false; switch (nodeTag(node)) { case T_Var: case T_Const: case T_Param: case T_CoerceToDomainValue: case T_SetToDefault: case T_CurrentOfExpr: /* single words: always simple */ return true; case T_SubscriptingRef: case T_ArrayExpr: case T_RowExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: case T_NextValueExpr: case T_NullIfExpr: case T_Aggref: case T_GroupingFunc: case T_WindowFunc: case T_FuncExpr: case T_JsonConstructorExpr: /* function-like: name(..) or name[..] */ return true; /* CASE keywords act as parentheses */ case T_CaseExpr: return true; case T_FieldSelect: /* * appears simple since . has top precedence, unless parent is * T_FieldSelect itself! */ return !IsA(parentNode, FieldSelect); case T_FieldStore: /* * treat like FieldSelect (probably doesn't matter) */ return !IsA(parentNode, FieldStore); case T_CoerceToDomain: /* maybe simple, check args */ return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, node, prettyFlags); case T_RelabelType: return isSimpleNode((Node *) ((RelabelType *) node)->arg, node, prettyFlags); case T_CoerceViaIO: return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, node, prettyFlags); case T_ArrayCoerceExpr: return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, node, prettyFlags); case T_ConvertRowtypeExpr: return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, node, prettyFlags); case T_OpExpr: { /* depends on parent node type; needs further checking */ if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) { const char *op; const char *parentOp; bool is_lopriop; bool is_hipriop; bool is_lopriparent; bool is_hipriparent; op = get_simple_binary_op_name((OpExpr *) node); if (!op) return false; /* We know only the basic operators + - and * / % */ is_lopriop = (strchr("+-", *op) != NULL); is_hipriop = (strchr("*/%", *op) != NULL); if (!(is_lopriop || is_hipriop)) return false; parentOp = get_simple_binary_op_name((OpExpr *) parentNode); if (!parentOp) return false; is_lopriparent = (strchr("+-", *parentOp) != NULL); is_hipriparent = (strchr("*/%", *parentOp) != NULL); if (!(is_lopriparent || is_hipriparent)) return false; if (is_hipriop && is_lopriparent) return true; /* op binds tighter than parent */ if (is_lopriop && is_hipriparent) return false; /* * Operators are same priority --- can skip parens only if * we have (a - b) - c, not a - (b - c). */ if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) return true; return false; } /* else do the same stuff as for T_SubLink et al. */ } /* FALLTHROUGH */ case T_SubLink: case T_NullTest: case T_BooleanTest: case T_DistinctExpr: case T_JsonIsPredicate: switch (nodeTag(parentNode)) { case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_BoolExpr: /* lower precedence */ case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ return true; default: return false; } case T_BoolExpr: switch (nodeTag(parentNode)) { case T_BoolExpr: if (prettyFlags & PRETTYFLAG_PAREN) { BoolExprType type; BoolExprType parentType; type = ((BoolExpr *) node)->boolop; parentType = ((BoolExpr *) parentNode)->boolop; switch (type) { case NOT_EXPR: case AND_EXPR: if (parentType == AND_EXPR || parentType == OR_EXPR) return true; break; case OR_EXPR: if (parentType == OR_EXPR) return true; break; } } return false; case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ return true; default: return false; } case T_JsonValueExpr: /* maybe simple, check args */ return isSimpleNode((Node *) ((JsonValueExpr *) node)->raw_expr, node, prettyFlags); default: break; } /* those we don't know: in dubio complexo */ return false; } /* * appendContextKeyword - append a keyword to buffer * * If prettyPrint is enabled, perform a line break, and adjust indentation. * Otherwise, just append the keyword. */ static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus) { StringInfo buf = context->buf; if (PRETTY_INDENT(context)) { int indentAmount; context->indentLevel += indentBefore; /* remove any trailing spaces currently in the buffer ... */ removeStringInfoSpaces(buf); /* ... then add a newline and some spaces */ appendStringInfoChar(buf, '\n'); if (context->indentLevel < PRETTYINDENT_LIMIT) indentAmount = Max(context->indentLevel, 0) + indentPlus; else { /* * If we're indented more than PRETTYINDENT_LIMIT characters, try * to conserve horizontal space by reducing the per-level * indentation. For best results the scale factor here should * divide all the indent amounts that get added to indentLevel * (PRETTYINDENT_STD, etc). It's important that the indentation * not grow unboundedly, else deeply-nested trees use O(N^2) * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. */ indentAmount = PRETTYINDENT_LIMIT + (context->indentLevel - PRETTYINDENT_LIMIT) / (PRETTYINDENT_STD / 2); indentAmount %= PRETTYINDENT_LIMIT; /* scale/wrap logic affects indentLevel, but not indentPlus */ indentAmount += indentPlus; } appendStringInfoSpaces(buf, indentAmount); appendStringInfoString(buf, str); context->indentLevel += indentAfter; if (context->indentLevel < 0) context->indentLevel = 0; } else appendStringInfoString(buf, str); } /* * removeStringInfoSpaces - delete trailing spaces from a buffer. * * Possibly this should move to stringinfo.c at some point. */ static void removeStringInfoSpaces(StringInfo str) { while (str->len > 0 && str->data[str->len - 1] == ' ') str->data[--(str->len)] = '\0'; } /* * get_rule_expr_paren - deparse expr using get_rule_expr, * embracing the string with parentheses if necessary for prettyPrint. * * Never embrace if prettyFlags=0, because it's done in the calling node. * * Any node that does *not* embrace its argument node by sql syntax (with * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should * use get_rule_expr_paren instead of get_rule_expr so parentheses can be * added. */ static void get_rule_expr_paren(Node *node, deparse_context *context, bool showimplicit, Node *parentNode) { bool need_paren; need_paren = PRETTY_PAREN(context) && !isSimpleNode(node, parentNode, context->prettyFlags); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(node, context, showimplicit); if (need_paren) appendStringInfoChar(context->buf, ')'); } /* ---------- * get_rule_expr - Parse back an expression * * Note: showimplicit determines whether we display any implicit cast that * is present at the top of the expression tree. It is a passed argument, * not a field of the context struct, because we change the value as we * recurse down into the expression. In general we suppress implicit casts * when the result type is known with certainty (eg, the arguments of an * OR must be boolean). We display implicit casts for arguments of functions * and operators, since this is needed to be certain that the same function * or operator will be chosen when the expression is re-parsed. * ---------- */ static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; if (node == NULL) return; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); /* * Each level of get_rule_expr must emit an indivisible term * (parenthesized if necessary) to ensure result is reparsed into the same * expression tree. The only exception is that when the input is a List, * we emit the component items comma-separated with no surrounding * decoration; this is convenient for most callers. */ switch (nodeTag(node)) { case T_Var: (void) get_variable((Var *) node, 0, false, context); break; case T_Const: get_const_expr((Const *) node, context, 0); break; case T_Param: get_parameter((Param *) node, context); break; case T_Aggref: get_agg_expr((Aggref *) node, context, (Aggref *) node); break; case T_GroupingFunc: { GroupingFunc *gexpr = (GroupingFunc *) node; appendStringInfoString(buf, "GROUPING("); get_rule_expr((Node *) gexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_WindowFunc: get_windowfunc_expr((WindowFunc *) node, context); break; case T_SubscriptingRef: { SubscriptingRef *sbsref = (SubscriptingRef *) node; bool need_parens; /* * If the argument is a CaseTestExpr, we must be inside a * FieldStore, ie, we are assigning to an element of an array * within a composite column. Since we already punted on * displaying the FieldStore's target information, just punt * here too, and display only the assignment source * expression. */ if (IsA(sbsref->refexpr, CaseTestExpr)) { Assert(sbsref->refassgnexpr); get_rule_expr((Node *) sbsref->refassgnexpr, context, showimplicit); break; } /* * Parenthesize the argument unless it's a simple Var or a * FieldSelect. (In particular, if it's another * SubscriptingRef, we *must* parenthesize to avoid * confusion.) */ need_parens = !IsA(sbsref->refexpr, Var) && !IsA(sbsref->refexpr, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); /* * If there's a refassgnexpr, we want to print the node in the * format "container[subscripts] := refassgnexpr". This is * not legal SQL, so decompilation of INSERT or UPDATE * statements should always use processIndirection as part of * the statement-level syntax. We should only see this when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. */ if (sbsref->refassgnexpr) { Node *refassgnexpr; /* * Use processIndirection to print this node's subscripts * as well as any additional field selections or * subscripting in immediate descendants. It returns the * RHS expr that is actually being "assigned". */ refassgnexpr = processIndirection(node, context); appendStringInfoString(buf, " := "); get_rule_expr(refassgnexpr, context, showimplicit); } else { /* Just an ordinary container fetch, so print subscripts */ printSubscripts(sbsref, context); } } break; case T_FuncExpr: get_func_expr((FuncExpr *) node, context, showimplicit); break; case T_NamedArgExpr: { NamedArgExpr *na = (NamedArgExpr *) node; appendStringInfo(buf, "%s => ", quote_identifier(na->name)); get_rule_expr((Node *) na->arg, context, showimplicit); } break; case T_OpExpr: get_oper_expr((OpExpr *) node, context); break; case T_DistinctExpr: { DistinctExpr *expr = (DistinctExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfoString(buf, " IS DISTINCT FROM "); get_rule_expr_paren(arg2, context, true, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_NullIfExpr: { NullIfExpr *nullifexpr = (NullIfExpr *) node; appendStringInfoString(buf, "NULLIF("); get_rule_expr((Node *) nullifexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfo(buf, " %s %s (", generate_operator_name(expr->opno, exprType(arg1), get_base_element_type(exprType(arg2))), expr->useOr ? "ANY" : "ALL"); get_rule_expr_paren(arg2, context, true, node); /* * There's inherent ambiguity in "x op ANY/ALL (y)" when y is * a bare sub-SELECT. Since we're here, the sub-SELECT must * be meant as a scalar sub-SELECT yielding an array value to * be used in ScalarArrayOpExpr; but the grammar will * preferentially interpret such a construct as an ANY/ALL * SubLink. To prevent misparsing the output that way, insert * a dummy coercion (which will be stripped by parse analysis, * so no inefficiency is added in dump and reload). This is * indeed most likely what the user wrote to get the construct * accepted in the first place. */ if (IsA(arg2, SubLink) && ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) appendStringInfo(buf, "::%s", format_type_with_typemod(exprType(arg2), exprTypmod(arg2))); appendStringInfoChar(buf, ')'); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BoolExpr: { BoolExpr *expr = (BoolExpr *) node; Node *first_arg = linitial(expr->args); ListCell *arg; switch (expr->boolop) { case AND_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " AND "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case OR_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " OR "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case NOT_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); appendStringInfoString(buf, "NOT "); get_rule_expr_paren(first_arg, context, false, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; default: elog(ERROR, "unrecognized boolop: %d", (int) expr->boolop); } } break; case T_SubLink: get_sublink_expr((SubLink *) node, context); break; case T_SubPlan: { SubPlan *subplan = (SubPlan *) node; /* * We cannot see an already-planned subplan in rule deparsing, * only while EXPLAINing a query plan. We don't try to * reconstruct the original SQL, just reference the subplan * that appears elsewhere in EXPLAIN's result. */ if (subplan->useHashTable) appendStringInfo(buf, "(hashed %s)", subplan->plan_name); else appendStringInfo(buf, "(%s)", subplan->plan_name); } break; case T_AlternativeSubPlan: { AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; ListCell *lc; /* * This case cannot be reached in normal usage, since no * AlternativeSubPlan can appear either in parsetrees or * finished plan trees. We keep it just in case somebody * wants to use this code to print planner data structures. */ appendStringInfoString(buf, "(alternatives: "); foreach(lc, asplan->subplans) { SubPlan *splan = lfirst_node(SubPlan, lc); if (splan->useHashTable) appendStringInfo(buf, "hashed %s", splan->plan_name); else appendStringInfoString(buf, splan->plan_name); if (lnext(asplan->subplans, lc)) appendStringInfoString(buf, " or "); } appendStringInfoChar(buf, ')'); } break; case T_FieldSelect: { FieldSelect *fselect = (FieldSelect *) node; Node *arg = (Node *) fselect->arg; int fno = fselect->fieldnum; const char *fieldname; bool need_parens; /* * Parenthesize the argument unless it's an SubscriptingRef or * another FieldSelect. Note in particular that it would be * WRONG to not parenthesize a Var argument; simplicity is not * the issue here, having the right number of names is. */ need_parens = !IsA(arg, SubscriptingRef) && !IsA(arg, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr(arg, context, true); if (need_parens) appendStringInfoChar(buf, ')'); /* * Get and print the field name. */ fieldname = get_name_for_var_field((Var *) arg, fno, 0, context); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); } break; case T_FieldStore: { FieldStore *fstore = (FieldStore *) node; bool need_parens; /* * There is no good way to represent a FieldStore as real SQL, * so decompilation of INSERT or UPDATE statements should * always use processIndirection as part of the * statement-level syntax. We should only get here when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. The plan case is even harder than * ordinary rules would be, because the planner tries to * collapse multiple assignments to the same field or subfield * into one FieldStore; so we can see a list of target fields * not just one, and the arguments could be FieldStores * themselves. We don't bother to try to print the target * field names; we just print the source arguments, with a * ROW() around them if there's more than one. This isn't * terribly complete, but it's probably good enough for * EXPLAIN's purposes; especially since anything more would be * either hopelessly confusing or an even poorer * representation of what the plan is actually doing. */ need_parens = (list_length(fstore->newvals) != 1); if (need_parens) appendStringInfoString(buf, "ROW("); get_rule_expr((Node *) fstore->newvals, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); } break; case T_RelabelType: { RelabelType *relabel = (RelabelType *) node; Node *arg = (Node *) relabel->arg; if (relabel->relabelformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, relabel->resulttype, relabel->resulttypmod, node); } } break; case T_CoerceViaIO: { CoerceViaIO *iocoerce = (CoerceViaIO *) node; Node *arg = (Node *) iocoerce->arg; if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, iocoerce->resulttype, -1, node); } } break; case T_ArrayCoerceExpr: { ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; Node *arg = (Node *) acoerce->arg; if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, acoerce->resulttype, acoerce->resulttypmod, node); } } break; case T_ConvertRowtypeExpr: { ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; Node *arg = (Node *) convert->arg; if (convert->convertformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, convert->resulttype, -1, node); } } break; case T_CollateExpr: { CollateExpr *collate = (CollateExpr *) node; Node *arg = (Node *) collate->arg; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, showimplicit, node); appendStringInfo(buf, " COLLATE %s", generate_collation_name(collate->collOid)); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CaseExpr: { CaseExpr *caseexpr = (CaseExpr *) node; ListCell *temp; appendContextKeyword(context, "CASE", 0, PRETTYINDENT_VAR, 0); if (caseexpr->arg) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) caseexpr->arg, context, true); } foreach(temp, caseexpr->args) { CaseWhen *when = (CaseWhen *) lfirst(temp); Node *w = (Node *) when->expr; if (caseexpr->arg) { /* * The parser should have produced WHEN clauses of the * form "CaseTestExpr = RHS", possibly with an * implicit coercion inserted above the CaseTestExpr. * For accurate decompilation of rules it's essential * that we show just the RHS. However in an * expression that's been through the optimizer, the * WHEN clause could be almost anything (since the * equality operator could have been expanded into an * inline function). If we don't recognize the form * of the WHEN clause, just punt and display it as-is. */ if (IsA(w, OpExpr)) { List *args = ((OpExpr *) w)->args; if (list_length(args) == 2 && IsA(strip_implicit_coercions(linitial(args)), CaseTestExpr)) w = (Node *) lsecond(args); } } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "WHEN ", 0, 0, 0); get_rule_expr(w, context, false); appendStringInfoString(buf, " THEN "); get_rule_expr((Node *) when->result, context, true); } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "ELSE ", 0, 0, 0); get_rule_expr((Node *) caseexpr->defresult, context, true); if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "END", -PRETTYINDENT_VAR, 0, 0); } break; case T_CaseTestExpr: { /* * Normally we should never get here, since for expressions * that can contain this node type we attempt to avoid * recursing to it. But in an optimized expression we might * be unable to avoid that (see comments for CaseExpr). If we * do see one, print it as CASE_TEST_EXPR. */ appendStringInfoString(buf, "CASE_TEST_EXPR"); } break; case T_ArrayExpr: { ArrayExpr *arrayexpr = (ArrayExpr *) node; appendStringInfoString(buf, "ARRAY["); get_rule_expr((Node *) arrayexpr->elements, context, true); appendStringInfoChar(buf, ']'); /* * If the array isn't empty, we assume its elements are * coerced to the desired type. If it's empty, though, we * need an explicit coercion to the array type. */ if (arrayexpr->elements == NIL) appendStringInfo(buf, "::%s", format_type_with_typemod(arrayexpr->array_typeid, -1)); } break; case T_RowExpr: { RowExpr *rowexpr = (RowExpr *) node; TupleDesc tupdesc = NULL; ListCell *arg; int i; char *sep; /* * If it's a named type and not RECORD, we may have to skip * dropped columns and/or claim there are NULLs for added * columns. */ if (rowexpr->row_typeid != RECORDOID) { tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); Assert(list_length(rowexpr->args) <= tupdesc->natts); } /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. */ appendStringInfoString(buf, "ROW("); sep = ""; i = 0; foreach(arg, rowexpr->args) { Node *e = (Node *) lfirst(arg); if (tupdesc == NULL || !TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); /* Whole-row Vars need special treatment here */ get_rule_expr_toplevel(e, context, true); sep = ", "; } i++; } if (tupdesc != NULL) { while (i < tupdesc->natts) { if (!TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); appendStringInfoString(buf, "NULL"); sep = ", "; } i++; } ReleaseTupleDesc(tupdesc); } appendStringInfoChar(buf, ')'); if (rowexpr->row_format == COERCE_EXPLICIT_CAST) appendStringInfo(buf, "::%s", format_type_with_typemod(rowexpr->row_typeid, -1)); } break; case T_RowCompareExpr: { RowCompareExpr *rcexpr = (RowCompareExpr *) node; /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. Within * a ROW expression, whole-row Vars need special treatment, so * use get_rule_list_toplevel. */ appendStringInfoString(buf, "(ROW("); get_rule_list_toplevel(rcexpr->largs, context, true); /* * We assume that the name of the first-column operator will * do for all the rest too. This is definitely open to * failure, eg if some but not all operators were renamed * since the construct was parsed, but there seems no way to * be perfect. */ appendStringInfo(buf, ") %s ROW(", generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs)))); get_rule_list_toplevel(rcexpr->rargs, context, true); appendStringInfoString(buf, "))"); } break; case T_CoalesceExpr: { CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; appendStringInfoString(buf, "COALESCE("); get_rule_expr((Node *) coalesceexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_MinMaxExpr: { MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; switch (minmaxexpr->op) { case IS_GREATEST: appendStringInfoString(buf, "GREATEST("); break; case IS_LEAST: appendStringInfoString(buf, "LEAST("); break; } get_rule_expr((Node *) minmaxexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_SQLValueFunction: { SQLValueFunction *svf = (SQLValueFunction *) node; /* * Note: this code knows that typmod for time, timestamp, and * timestamptz just prints as integer. */ switch (svf->op) { case SVFOP_CURRENT_DATE: appendStringInfoString(buf, "CURRENT_DATE"); break; case SVFOP_CURRENT_TIME: appendStringInfoString(buf, "CURRENT_TIME"); break; case SVFOP_CURRENT_TIME_N: appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); break; case SVFOP_CURRENT_TIMESTAMP: appendStringInfoString(buf, "CURRENT_TIMESTAMP"); break; case SVFOP_CURRENT_TIMESTAMP_N: appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", svf->typmod); break; case SVFOP_LOCALTIME: appendStringInfoString(buf, "LOCALTIME"); break; case SVFOP_LOCALTIME_N: appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); break; case SVFOP_LOCALTIMESTAMP: appendStringInfoString(buf, "LOCALTIMESTAMP"); break; case SVFOP_LOCALTIMESTAMP_N: appendStringInfo(buf, "LOCALTIMESTAMP(%d)", svf->typmod); break; case SVFOP_CURRENT_ROLE: appendStringInfoString(buf, "CURRENT_ROLE"); break; case SVFOP_CURRENT_USER: appendStringInfoString(buf, "CURRENT_USER"); break; case SVFOP_USER: appendStringInfoString(buf, "USER"); break; case SVFOP_SESSION_USER: appendStringInfoString(buf, "SESSION_USER"); break; case SVFOP_CURRENT_CATALOG: appendStringInfoString(buf, "CURRENT_CATALOG"); break; case SVFOP_CURRENT_SCHEMA: appendStringInfoString(buf, "CURRENT_SCHEMA"); break; } } break; case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; bool needcomma = false; ListCell *arg; ListCell *narg; Const *con; switch (xexpr->op) { case IS_XMLCONCAT: appendStringInfoString(buf, "XMLCONCAT("); break; case IS_XMLELEMENT: appendStringInfoString(buf, "XMLELEMENT("); break; case IS_XMLFOREST: appendStringInfoString(buf, "XMLFOREST("); break; case IS_XMLPARSE: appendStringInfoString(buf, "XMLPARSE("); break; case IS_XMLPI: appendStringInfoString(buf, "XMLPI("); break; case IS_XMLROOT: appendStringInfoString(buf, "XMLROOT("); break; case IS_XMLSERIALIZE: appendStringInfoString(buf, "XMLSERIALIZE("); break; case IS_DOCUMENT: break; } if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) { if (xexpr->xmloption == XMLOPTION_DOCUMENT) appendStringInfoString(buf, "DOCUMENT "); else appendStringInfoString(buf, "CONTENT "); } if (xexpr->name) { appendStringInfo(buf, "NAME %s", quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); needcomma = true; } if (xexpr->named_args) { if (xexpr->op != IS_XMLFOREST) { if (needcomma) appendStringInfoString(buf, ", "); appendStringInfoString(buf, "XMLATTRIBUTES("); needcomma = false; } forboth(arg, xexpr->named_args, narg, xexpr->arg_names) { Node *e = (Node *) lfirst(arg); char *argname = strVal(lfirst(narg)); if (needcomma) appendStringInfoString(buf, ", "); get_rule_expr((Node *) e, context, true); appendStringInfo(buf, " AS %s", quote_identifier(map_xml_name_to_sql_identifier(argname))); needcomma = true; } if (xexpr->op != IS_XMLFOREST) appendStringInfoChar(buf, ')'); } if (xexpr->args) { if (needcomma) appendStringInfoString(buf, ", "); switch (xexpr->op) { case IS_XMLCONCAT: case IS_XMLELEMENT: case IS_XMLFOREST: case IS_XMLPI: case IS_XMLSERIALIZE: /* no extra decoration needed */ get_rule_expr((Node *) xexpr->args, context, true); break; case IS_XMLPARSE: Assert(list_length(xexpr->args) == 2); get_rule_expr((Node *) linitial(xexpr->args), context, true); con = lsecond_node(Const, xexpr->args); Assert(!con->constisnull); if (DatumGetBool(con->constvalue)) appendStringInfoString(buf, " PRESERVE WHITESPACE"); else appendStringInfoString(buf, " STRIP WHITESPACE"); break; case IS_XMLROOT: Assert(list_length(xexpr->args) == 3); get_rule_expr((Node *) linitial(xexpr->args), context, true); appendStringInfoString(buf, ", VERSION "); con = (Const *) lsecond(xexpr->args); if (IsA(con, Const) && con->constisnull) appendStringInfoString(buf, "NO VALUE"); else get_rule_expr((Node *) con, context, false); con = lthird_node(Const, xexpr->args); if (con->constisnull) /* suppress STANDALONE NO VALUE */ ; else { switch (DatumGetInt32(con->constvalue)) { case XML_STANDALONE_YES: appendStringInfoString(buf, ", STANDALONE YES"); break; case XML_STANDALONE_NO: appendStringInfoString(buf, ", STANDALONE NO"); break; case XML_STANDALONE_NO_VALUE: appendStringInfoString(buf, ", STANDALONE NO VALUE"); break; default: break; } } break; case IS_DOCUMENT: get_rule_expr_paren((Node *) xexpr->args, context, false, node); break; } } if (xexpr->op == IS_XMLSERIALIZE) appendStringInfo(buf, " AS %s", format_type_with_typemod(xexpr->type, xexpr->typmod)); if (xexpr->op == IS_DOCUMENT) appendStringInfoString(buf, " IS DOCUMENT"); else appendStringInfoChar(buf, ')'); } break; case T_NullTest: { NullTest *ntest = (NullTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) ntest->arg, context, true, node); /* * For scalar inputs, we prefer to print as IS [NOT] NULL, * which is shorter and traditional. If it's a rowtype input * but we're applying a scalar test, must print IS [NOT] * DISTINCT FROM NULL to be semantically correct. */ if (ntest->argisrow || !type_is_rowtype(exprType((Node *) ntest->arg))) { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS NOT NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } else { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS DISTINCT FROM NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BooleanTest: { BooleanTest *btest = (BooleanTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) btest->arg, context, false, node); switch (btest->booltesttype) { case IS_TRUE: appendStringInfoString(buf, " IS TRUE"); break; case IS_NOT_TRUE: appendStringInfoString(buf, " IS NOT TRUE"); break; case IS_FALSE: appendStringInfoString(buf, " IS FALSE"); break; case IS_NOT_FALSE: appendStringInfoString(buf, " IS NOT FALSE"); break; case IS_UNKNOWN: appendStringInfoString(buf, " IS UNKNOWN"); break; case IS_NOT_UNKNOWN: appendStringInfoString(buf, " IS NOT UNKNOWN"); break; default: elog(ERROR, "unrecognized booltesttype: %d", (int) btest->booltesttype); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CoerceToDomain: { CoerceToDomain *ctest = (CoerceToDomain *) node; Node *arg = (Node *) ctest->arg; if (ctest->coercionformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr(arg, context, false); } else { get_coercion_expr(arg, context, ctest->resulttype, ctest->resulttypmod, node); } } break; case T_CoerceToDomainValue: appendStringInfoString(buf, "VALUE"); break; case T_SetToDefault: appendStringInfoString(buf, "DEFAULT"); break; case T_CurrentOfExpr: { CurrentOfExpr *cexpr = (CurrentOfExpr *) node; if (cexpr->cursor_name) appendStringInfo(buf, "CURRENT OF %s", quote_identifier(cexpr->cursor_name)); else appendStringInfo(buf, "CURRENT OF $%d", cexpr->cursor_param); } break; case T_NextValueExpr: { NextValueExpr *nvexpr = (NextValueExpr *) node; /* * This isn't exactly nextval(), but that seems close enough * for EXPLAIN's purposes. */ appendStringInfoString(buf, "nextval("); simple_quote_literal(buf, generate_relation_name(nvexpr->seqid, NIL)); appendStringInfoChar(buf, ')'); } break; case T_InferenceElem: { InferenceElem *iexpr = (InferenceElem *) node; bool save_varprefix; bool need_parens; /* * InferenceElem can only refer to target relation, so a * prefix is not useful, and indeed would cause parse errors. */ save_varprefix = context->varprefix; context->varprefix = false; /* * Parenthesize the element unless it's a simple Var or a bare * function call. Follows pg_get_indexdef_worker(). */ need_parens = !IsA(iexpr->expr, Var); if (IsA(iexpr->expr, FuncExpr) && ((FuncExpr *) iexpr->expr)->funcformat == COERCE_EXPLICIT_CALL) need_parens = false; if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) iexpr->expr, context, false); if (need_parens) appendStringInfoChar(buf, ')'); context->varprefix = save_varprefix; if (iexpr->infercollid) appendStringInfo(buf, " COLLATE %s", generate_collation_name(iexpr->infercollid)); /* Add the operator class name, if not default */ if (iexpr->inferopclass) { Oid inferopclass = iexpr->inferopclass; Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); get_opclass_name(inferopclass, inferopcinputtype, buf); } } break; case T_PartitionBoundSpec: { PartitionBoundSpec *spec = (PartitionBoundSpec *) node; ListCell *cell; char *sep; if (spec->is_default) { appendStringInfoString(buf, "DEFAULT"); break; } switch (spec->strategy) { case PARTITION_STRATEGY_HASH: Assert(spec->modulus > 0 && spec->remainder >= 0); Assert(spec->modulus > spec->remainder); appendStringInfoString(buf, "FOR VALUES"); appendStringInfo(buf, " WITH (modulus %d, remainder %d)", spec->modulus, spec->remainder); break; case PARTITION_STRATEGY_LIST: Assert(spec->listdatums != NIL); appendStringInfoString(buf, "FOR VALUES IN ("); sep = ""; foreach(cell, spec->listdatums) { Const *val = lfirst_node(Const, cell); appendStringInfoString(buf, sep); get_const_expr(val, context, -1); sep = ", "; } appendStringInfoChar(buf, ')'); break; case PARTITION_STRATEGY_RANGE: Assert(spec->lowerdatums != NIL && spec->upperdatums != NIL && list_length(spec->lowerdatums) == list_length(spec->upperdatums)); appendStringInfo(buf, "FOR VALUES FROM %s TO %s", get_range_partbound_string(spec->lowerdatums), get_range_partbound_string(spec->upperdatums)); break; default: elog(ERROR, "unrecognized partition strategy: %d", (int) spec->strategy); break; } } break; case T_JsonValueExpr: { JsonValueExpr *jve = (JsonValueExpr *) node; get_rule_expr((Node *) jve->raw_expr, context, false); get_json_format(jve->format, context->buf); } break; case T_JsonConstructorExpr: get_json_constructor((JsonConstructorExpr *) node, context, false); break; case T_JsonIsPredicate: { JsonIsPredicate *pred = (JsonIsPredicate *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, '('); get_rule_expr_paren(pred->expr, context, true, node); appendStringInfoString(context->buf, " IS JSON"); /* TODO: handle FORMAT clause */ switch (pred->item_type) { case JS_TYPE_SCALAR: appendStringInfoString(context->buf, " SCALAR"); break; case JS_TYPE_ARRAY: appendStringInfoString(context->buf, " ARRAY"); break; case JS_TYPE_OBJECT: appendStringInfoString(context->buf, " OBJECT"); break; default: break; } if (pred->unique_keys) appendStringInfoString(context->buf, " WITH UNIQUE KEYS"); if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, ')'); } break; case T_List: { char *sep; ListCell *l; sep = ""; foreach(l, (List *) node) { appendStringInfoString(buf, sep); get_rule_expr((Node *) lfirst(l), context, showimplicit); sep = ", "; } } break; case T_TableFunc: get_tablefunc((TableFunc *) node, context, showimplicit); break; case T_CallStmt: get_proc_expr((CallStmt *) node, context, showimplicit); break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); break; } } /* * get_rule_expr_toplevel - Parse back a toplevel expression * * Same as get_rule_expr(), except that if the expr is just a Var, we pass * istoplevel = true not false to get_variable(). This causes whole-row Vars * to get printed with decoration that will prevent expansion of "*". * We need to use this in contexts such as ROW() and VALUES(), where the * parser would expand "foo.*" appearing at top level. (In principle we'd * use this in get_target_list() too, but that has additional worries about * whether to print AS, so it needs to invoke get_variable() directly anyway.) */ static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit) { if (node && IsA(node, Var)) (void) get_variable((Var *) node, 0, true, context); else get_rule_expr(node, context, showimplicit); } /* * get_rule_list_toplevel - Parse back a list of toplevel expressions * * Apply get_rule_expr_toplevel() to each element of a List. * * This adds commas between the expressions, but caller is responsible * for printing surrounding decoration. */ static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit) { const char *sep; ListCell *lc; sep = ""; foreach(lc, lst) { Node *e = (Node *) lfirst(lc); appendStringInfoString(context->buf, sep); get_rule_expr_toplevel(e, context, showimplicit); sep = ", "; } } /* * get_rule_expr_funccall - Parse back a function-call expression * * Same as get_rule_expr(), except that we guarantee that the output will * look like a function call, or like one of the things the grammar treats as * equivalent to a function call (see the func_expr_windowless production). * This is needed in places where the grammar uses func_expr_windowless and * you can't substitute a parenthesized a_expr. If what we have isn't going * to look like a function call, wrap it in a dummy CAST() expression, which * will satisfy the grammar --- and, indeed, is likely what the user wrote to * produce such a thing. */ static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit) { if (looks_like_function(node)) get_rule_expr(node, context, showimplicit); else { StringInfo buf = context->buf; appendStringInfoString(buf, "CAST("); /* no point in showing any top-level implicit cast */ get_rule_expr(node, context, false); appendStringInfo(buf, " AS %s)", format_type_with_typemod(exprType(node), exprTypmod(node))); } } /* * Helper function to identify node types that satisfy func_expr_windowless. * If in doubt, "false" is always a safe answer. */ static bool looks_like_function(Node *node) { if (node == NULL) return false; /* probably shouldn't happen */ switch (nodeTag(node)) { case T_FuncExpr: /* OK, unless it's going to deparse as a cast */ return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); case T_NullIfExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: /* these are all accepted by func_expr_common_subexpr */ return true; default: break; } return false; } /* * get_oper_expr - Parse back an OpExpr node */ static void get_oper_expr(OpExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid opno = expr->opno; List *args = expr->args; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); get_rule_expr_paren(arg1, context, true, (Node *) expr); appendStringInfo(buf, " %s ", generate_operator_name(opno, exprType(arg1), exprType(arg2))); get_rule_expr_paren(arg2, context, true, (Node *) expr); } else { /* prefix operator */ Node *arg = (Node *) linitial(args); appendStringInfo(buf, "%s ", generate_operator_name(opno, InvalidOid, exprType(arg))); get_rule_expr_paren(arg, context, true, (Node *) expr); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } /* * get_func_expr - Parse back a FuncExpr node */ static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; bool use_variadic; ListCell *l; /* * If the function call came from an implicit coercion, then just show the * first argument --- unless caller wants to see implicit coercions. */ if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) { get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); return; } /* * If the function call came from a cast, then show the first argument * plus an explicit cast operation. */ if (expr->funcformat == COERCE_EXPLICIT_CAST || expr->funcformat == COERCE_IMPLICIT_CAST) { Node *arg = linitial(expr->args); Oid rettype = expr->funcresulttype; int32 coercedTypmod; /* Get the typmod if this is a length-coercion function */ (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); get_coercion_expr(arg, context, rettype, coercedTypmod, (Node *) expr); return; } /* * If the function was called using one of the SQL spec's random special * syntaxes, try to reproduce that. If we don't recognize the function, * fall through. */ if (expr->funcformat == COERCE_SQL_SYNTAX) { if (get_func_sql_syntax(expr, context)) return; } /* * Normal function: display as proname(args). First we need to extract * the argument datatypes. */ if (list_length(expr->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, expr->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } appendStringInfo(buf, "%s(", generate_function_name(funcoid, nargs, argnames, argtypes, expr->funcvariadic, &use_variadic, context->inGroupBy)); nargs = 0; foreach(l, expr->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(expr->args, l) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(l), context, true); } appendStringInfoChar(buf, ')'); } /* * get_proc_expr - Parse back a CallStmt node */ static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid functionOid = stmt->funcexpr->funcid; bool use_variadic; Oid *argumentTypes; List *finalArgumentList = NIL; ListCell *argumentCell; List *namedArgList = NIL; int numberOfArgs = -1; if (!get_merged_argument_list(stmt, &namedArgList, &argumentTypes, &finalArgumentList, &numberOfArgs)) { /* Nothing merged i.e. no OUT arguments */ get_func_expr((FuncExpr *) stmt->funcexpr, context, showimplicit); return; } appendStringInfo(buf, "%s(", generate_function_name(functionOid, numberOfArgs, namedArgList, argumentTypes, stmt->funcexpr->funcvariadic, &use_variadic, context->inGroupBy)); int argNumber = 0; foreach(argumentCell, finalArgumentList) { if (argNumber++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(finalArgumentList, argumentCell) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(argumentCell), context, true); argNumber++; } appendStringInfoChar(buf, ')'); } /* * get_agg_expr - Parse back an Aggref node */ static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref) { get_agg_expr_helper(aggref, context, original_aggref, NULL, NULL, false); } /* * get_agg_expr_helper - subroutine for get_agg_expr and * get_json_agg_constructor */ static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; bool use_variadic = false; /* * For a combining aggregate, we look up and deparse the corresponding * partial aggregate instead. This is necessary because our input * argument list has been replaced; the new argument list always has just * one element, which will point to a partial Aggref that supplies us with * transition states to combine. */ if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) { TargetEntry *tle; Assert(list_length(aggref->args) == 1); tle = linitial_node(TargetEntry, aggref->args); resolve_special_varno((Node *) tle->expr, context, get_agg_combine_expr, original_aggref); return; } /* * Mark as PARTIAL, if appropriate. We look to the original aggref so as * to avoid printing this when recursing from the code just above. */ if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) appendStringInfoString(buf, "PARTIAL "); /* Extract the argument types as seen by the parser */ nargs = get_aggregate_argtypes(aggref, argtypes); if (!funcname) funcname = generate_function_name(aggref->aggfnoid, nargs, NIL, argtypes, aggref->aggvariadic, &use_variadic, context->inGroupBy); /* Print the aggregate name, schema-qualified if needed */ appendStringInfo(buf, "%s(%s", funcname, (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) { /* * Ordered-set aggregates do not use "*" syntax. Also, we needn't * worry about inserting VARIADIC. So we can just dump the direct * args as-is. */ Assert(!aggref->aggvariadic); get_rule_expr((Node *) aggref->aggdirectargs, context, true); Assert(aggref->aggorder != NIL); appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } else { /* aggstar can be set only in zero-argument aggregates */ if (aggref->aggstar) appendStringInfoChar(buf, '*'); else { ListCell *l; int i; i = 0; foreach(l, aggref->args) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *arg = (Node *) tle->expr; Assert(!IsA(arg, NamedArgExpr)); if (tle->resjunk) continue; if (i++ > 0) { if (is_json_objectagg) { /* * the ABSENT ON NULL and WITH UNIQUE args are printed * separately, so ignore them here */ if (i > 2) break; appendStringInfoString(buf, " : "); } else appendStringInfoString(buf, ", "); } if (use_variadic && i == nargs) appendStringInfoString(buf, "VARIADIC "); get_rule_expr(arg, context, true); } } if (aggref->aggorder != NIL) { appendStringInfoString(buf, " ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } } if (options) appendStringInfoString(buf, options); if (aggref->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) aggref->aggfilter, context, false); } appendStringInfoChar(buf, ')'); } /* * This is a helper function for get_agg_expr(). It's used when we deparse * a combining Aggref; resolve_special_varno locates the corresponding partial * Aggref and then calls this. */ static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) { Aggref *aggref; Aggref *original_aggref = callback_arg; if (!IsA(node, Aggref)) elog(ERROR, "combining Aggref does not point to an Aggref"); aggref = (Aggref *) node; get_agg_expr(aggref, context, original_aggref); } /* * get_windowfunc_expr - Parse back a WindowFunc node */ static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) { get_windowfunc_expr_helper(wfunc, context, NULL, NULL, false); } /* * get_windowfunc_expr_helper - subroutine for get_windowfunc_expr and * get_json_agg_constructor */ static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; ListCell *l; if (list_length(wfunc->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, wfunc->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } if (!funcname) funcname = generate_function_name(wfunc->winfnoid, nargs, argnames, argtypes, false, NULL, context->inGroupBy); appendStringInfo(buf, "%s(", funcname); /* winstar can be set only in zero-argument aggregates */ if (wfunc->winstar) appendStringInfoChar(buf, '*'); else { if (is_json_objectagg) { get_rule_expr((Node *) linitial(wfunc->args), context, false); appendStringInfoString(buf, " : "); get_rule_expr((Node *) lsecond(wfunc->args), context, false); } else get_rule_expr((Node *) wfunc->args, context, true); } if (options) appendStringInfoString(buf, options); if (wfunc->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) wfunc->aggfilter, context, false); } appendStringInfoString(buf, ") OVER "); foreach(l, context->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->winref == wfunc->winref) { if (wc->name) appendStringInfoString(buf, quote_identifier(wc->name)); else get_rule_windowspec(wc, context->targetList, context); break; } } if (l == NULL) { if (context->windowClause) elog(ERROR, "could not find window clause for winref %u", wfunc->winref); /* * In EXPLAIN, we don't have window context information available, so * we have to settle for this: */ appendStringInfoString(buf, "(?)"); } } /* * get_func_sql_syntax - Parse back a SQL-syntax function call * * Returns true if we successfully deparsed, false if we did not * recognize the function. */ static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; switch (funcoid) { case F_TIMEZONE_INTERVAL_TIMESTAMP: case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: case F_TIMEZONE_INTERVAL_TIMETZ: case F_TIMEZONE_TEXT_TIMESTAMP: case F_TIMEZONE_TEXT_TIMESTAMPTZ: case F_TIMEZONE_TEXT_TIMETZ: /* AT TIME ZONE ... note reversed argument order */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) lsecond(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " AT TIME ZONE "); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoChar(buf, ')'); return true; case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: case F_OVERLAPS_TIME_TIME_TIME_TIME: /* (x1, x2) OVERLAPS (y1, y2) */ appendStringInfoString(buf, "(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") OVERLAPS ("); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfourth(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_EXTRACT_TEXT_DATE: case F_EXTRACT_TEXT_TIME: case F_EXTRACT_TEXT_TIMETZ: case F_EXTRACT_TEXT_TIMESTAMP: case F_EXTRACT_TEXT_TIMESTAMPTZ: case F_EXTRACT_TEXT_INTERVAL: /* EXTRACT (x FROM y) */ appendStringInfoString(buf, "EXTRACT("); { Const *con = (Const *) linitial(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_IS_NORMALIZED: /* IS xxx NORMALIZED */ appendStringInfoString(buf, "("); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " IS"); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, " %s", TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " NORMALIZED)"); return true; case F_PG_COLLATION_FOR: /* COLLATION FOR */ appendStringInfoString(buf, "COLLATION FOR ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_NORMALIZE: /* NORMALIZE() */ appendStringInfoString(buf, "NORMALIZE("); get_rule_expr((Node *) linitial(expr->args), context, false); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, ", %s", TextDatumGetCString(con->constvalue)); } appendStringInfoChar(buf, ')'); return true; case F_OVERLAY_BIT_BIT_INT4: case F_OVERLAY_BIT_BIT_INT4_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: case F_OVERLAY_TEXT_TEXT_INT4: case F_OVERLAY_TEXT_TEXT_INT4_INT4: /* OVERLAY() */ appendStringInfoString(buf, "OVERLAY("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " PLACING "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lthird(expr->args), context, false); if (list_length(expr->args) == 4) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lfourth(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_POSITION_BIT_BIT: case F_POSITION_BYTEA_BYTEA: case F_POSITION_TEXT_TEXT: /* POSITION() ... extra parens since args are b_expr not a_expr */ appendStringInfoString(buf, "POSITION(("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") IN ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_SUBSTRING_BIT_INT4: case F_SUBSTRING_BIT_INT4_INT4: case F_SUBSTRING_BYTEA_INT4: case F_SUBSTRING_BYTEA_INT4_INT4: case F_SUBSTRING_TEXT_INT4: case F_SUBSTRING_TEXT_INT4_INT4: /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); if (list_length(expr->args) == 3) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lthird(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_SUBSTRING_TEXT_TEXT_TEXT: /* SUBSTRING SIMILAR/ESCAPE */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " SIMILAR "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " ESCAPE "); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_BTRIM_BYTEA_BYTEA: case F_BTRIM_TEXT: case F_BTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(BOTH"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_LTRIM_BYTEA_BYTEA: case F_LTRIM_TEXT: case F_LTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(LEADING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_RTRIM_BYTEA_BYTEA: case F_RTRIM_TEXT: case F_RTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(TRAILING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_SYSTEM_USER: appendStringInfoString(buf, "SYSTEM_USER"); return true; case F_XMLEXISTS: /* XMLEXISTS ... extra parens because args are c_expr */ appendStringInfoString(buf, "XMLEXISTS(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, "))"); return true; } return false; } /* ---------- * get_coercion_expr * * Make a string representation of a value coerced to a specific type * ---------- */ static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode) { StringInfo buf = context->buf; /* * Since parse_coerce.c doesn't immediately collapse application of * length-coercion functions to constants, what we'll typically see in * such cases is a Const with typmod -1 and a length-coercion function * right above it. Avoid generating redundant output. However, beware of * suppressing casts when the user actually wrote something like * 'foo'::text::char(3). * * Note: it might seem that we are missing the possibility of needing to * print a COLLATE clause for such a Const. However, a Const could only * have nondefault collation in a post-constant-folding tree, in which the * length coercion would have been folded too. See also the special * handling of CollateExpr in coerce_to_target_type(): any collation * marking will be above the coercion node, not below it. */ if (arg && IsA(arg, Const) && ((Const *) arg)->consttype == resulttype && ((Const *) arg)->consttypmod == -1) { /* Show the constant without normal ::typename decoration */ get_const_expr((Const *) arg, context, -1); } else { if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, false, parentNode); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } appendStringInfo(buf, "::%s", format_type_with_typemod(resulttype, resulttypmod)); } /* ---------- * get_const_expr * * Make a string representation of a Const * * showtype can be -1 to never show "::typename" decoration, or +1 to always * show it, or 0 to show it only if the constant wouldn't be assumed to be * the right type by default. * * If the Const's collation isn't default for its type, show that too. * We mustn't do this when showtype is -1 (since that means the caller will * print "::typename", and we can't put a COLLATE clause in between). It's * caller's responsibility that collation isn't missed in such cases. * ---------- */ static void get_const_expr(Const *constval, deparse_context *context, int showtype) { StringInfo buf = context->buf; Oid typoutput; bool typIsVarlena; char *extval; bool needlabel = false; if (constval->constisnull) { /* * Always label the type of a NULL constant to prevent misdecisions * about type when reparsing. */ appendStringInfoString(buf, "NULL"); if (showtype >= 0) { appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } return; } getTypeOutputInfo(constval->consttype, &typoutput, &typIsVarlena); extval = OidOutputFunctionCall(typoutput, constval->constvalue); switch (constval->consttype) { case INT4OID: /* * INT4 can be printed without any decoration, unless it is * negative; in that case print it as '-nnn'::integer to ensure * that the output will re-parse as a constant, not as a constant * plus operator. In most cases we could get away with printing * (-nnn) instead, because of the way that gram.y handles negative * literals; but that doesn't work for INT_MIN, and it doesn't * seem that much prettier anyway. */ if (extval[0] != '-') appendStringInfoString(buf, extval); else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case NUMERICOID: /* * NUMERIC can be printed without quotes if it looks like a float * constant (not an integer, and not Infinity or NaN) and doesn't * have a leading sign (for the same reason as for INT4). */ if (isdigit((unsigned char) extval[0]) && strcspn(extval, "eE.") != strlen(extval)) { appendStringInfoString(buf, extval); } else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case BITOID: case VARBITOID: appendStringInfo(buf, "B'%s'", extval); break; case BOOLOID: if (strcmp(extval, "t") == 0) appendStringInfoString(buf, "true"); else appendStringInfoString(buf, "false"); break; default: simple_quote_literal(buf, extval); break; } pfree(extval); if (showtype < 0) return; /* * For showtype == 0, append ::typename unless the constant will be * implicitly typed as the right type when it is read in. * * XXX this code has to be kept in sync with the behavior of the parser, * especially make_const. */ switch (constval->consttype) { case BOOLOID: case UNKNOWNOID: /* These types can be left unlabeled */ needlabel = false; break; case INT4OID: /* We determined above whether a label is needed */ break; case NUMERICOID: /* * Float-looking constants will be typed as numeric, which we * checked above; but if there's a nondefault typmod we need to * show it. */ needlabel |= (constval->consttypmod >= 0); break; default: needlabel = true; break; } if (needlabel || showtype > 0) appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } /* * helper for get_const_expr: append COLLATE if needed */ static void get_const_collation(Const *constval, deparse_context *context) { StringInfo buf = context->buf; if (OidIsValid(constval->constcollid)) { Oid typcollation = get_typcollation(constval->consttype); if (constval->constcollid != typcollation) { appendStringInfo(buf, " COLLATE %s", generate_collation_name(constval->constcollid)); } } } /* * get_json_format - Parse back a JsonFormat node */ static void get_json_format(JsonFormat *format, StringInfo buf) { if (format->format_type == JS_FORMAT_DEFAULT) return; appendStringInfoString(buf, format->format_type == JS_FORMAT_JSONB ? " FORMAT JSONB" : " FORMAT JSON"); if (format->encoding != JS_ENC_DEFAULT) { const char *encoding; encoding = format->encoding == JS_ENC_UTF16 ? "UTF16" : format->encoding == JS_ENC_UTF32 ? "UTF32" : "UTF8"; appendStringInfo(buf, " ENCODING %s", encoding); } } /* * get_json_returning - Parse back a JsonReturning structure */ static void get_json_returning(JsonReturning *returning, StringInfo buf, bool json_format_by_default) { if (!OidIsValid(returning->typid)) return; appendStringInfo(buf, " RETURNING %s", format_type_with_typemod(returning->typid, returning->typmod)); if (!json_format_by_default || returning->format->format_type != (returning->typid == JSONBOID ? JS_FORMAT_JSONB : JS_FORMAT_JSON)) get_json_format(returning->format, buf); } /* * get_json_constructor - Parse back a JsonConstructorExpr node */ static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; const char *funcname; bool is_json_object; int curridx; ListCell *lc; if (ctor->type == JSCTOR_JSON_OBJECTAGG) { get_json_agg_constructor(ctor, context, "JSON_OBJECTAGG", true); return; } else if (ctor->type == JSCTOR_JSON_ARRAYAGG) { get_json_agg_constructor(ctor, context, "JSON_ARRAYAGG", false); return; } switch (ctor->type) { case JSCTOR_JSON_OBJECT: funcname = "JSON_OBJECT"; break; case JSCTOR_JSON_ARRAY: funcname = "JSON_ARRAY"; break; default: elog(ERROR, "invalid JsonConstructorType %d", ctor->type); } appendStringInfo(buf, "%s(", funcname); is_json_object = ctor->type == JSCTOR_JSON_OBJECT; foreach(lc, ctor->args) { curridx = foreach_current_index(lc); if (curridx > 0) { const char *sep; sep = (is_json_object && (curridx % 2) != 0) ? " : " : ", "; appendStringInfoString(buf, sep); } get_rule_expr((Node *) lfirst(lc), context, true); } get_json_constructor_options(ctor, buf); appendStringInfo(buf, ")"); } /* * Append options, if any, to the JSON constructor being deparsed */ static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf) { if (ctor->absent_on_null) { if (ctor->type == JSCTOR_JSON_OBJECT || ctor->type == JSCTOR_JSON_OBJECTAGG) appendStringInfoString(buf, " ABSENT ON NULL"); } else { if (ctor->type == JSCTOR_JSON_ARRAY || ctor->type == JSCTOR_JSON_ARRAYAGG) appendStringInfoString(buf, " NULL ON NULL"); } if (ctor->unique) appendStringInfoString(buf, " WITH UNIQUE KEYS"); get_json_returning(ctor->returning, buf, true); } /* * get_json_agg_constructor - Parse back an aggregate JsonConstructorExpr node */ static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg) { StringInfoData options; initStringInfo(&options); get_json_constructor_options(ctor, &options); if (IsA(ctor->func, Aggref)) get_agg_expr_helper((Aggref *) ctor->func, context, (Aggref *) ctor->func, funcname, options.data, is_json_objectagg); else if (IsA(ctor->func, WindowFunc)) get_windowfunc_expr_helper((WindowFunc *) ctor->func, context, funcname, options.data, is_json_objectagg); else elog(ERROR, "invalid JsonConstructorExpr underlying node type: %d", nodeTag(ctor->func)); } /* * simple_quote_literal - Format a string as a SQL literal, append to buf */ static void simple_quote_literal(StringInfo buf, const char *val) { const char *valptr; /* * We form the string literal according to the prevailing setting of * standard_conforming_strings; we never use E''. User is responsible for * making sure result is used correctly. */ appendStringInfoChar(buf, '\''); for (valptr = val; *valptr; valptr++) { char ch = *valptr; if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) appendStringInfoChar(buf, ch); appendStringInfoChar(buf, ch); } appendStringInfoChar(buf, '\''); } /* ---------- * get_sublink_expr - Parse back a sublink * ---------- */ static void get_sublink_expr(SubLink *sublink, deparse_context *context) { StringInfo buf = context->buf; Query *query = (Query *) (sublink->subselect); char *opname = NULL; bool need_paren; if (sublink->subLinkType == ARRAY_SUBLINK) appendStringInfoString(buf, "ARRAY("); else appendStringInfoChar(buf, '('); /* * Note that we print the name of only the first operator, when there are * multiple combining operators. This is an approximation that could go * wrong in various scenarios (operators in different schemas, renamed * operators, etc) but there is not a whole lot we can do about it, since * the syntax allows only one operator to be shown. */ if (sublink->testexpr) { if (IsA(sublink->testexpr, OpExpr)) { /* single combining operator */ OpExpr *opexpr = (OpExpr *) sublink->testexpr; get_rule_expr(linitial(opexpr->args), context, true); opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); } else if (IsA(sublink->testexpr, BoolExpr)) { /* multiple combining operators, = or <> cases */ char *sep; ListCell *l; appendStringInfoChar(buf, '('); sep = ""; foreach(l, ((BoolExpr *) sublink->testexpr)->args) { OpExpr *opexpr = lfirst_node(OpExpr, l); appendStringInfoString(buf, sep); get_rule_expr(linitial(opexpr->args), context, true); if (!opname) opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); sep = ", "; } appendStringInfoChar(buf, ')'); } else if (IsA(sublink->testexpr, RowCompareExpr)) { /* multiple combining operators, < <= > >= cases */ RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; appendStringInfoChar(buf, '('); get_rule_expr((Node *) rcexpr->largs, context, true); opname = generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs))); appendStringInfoChar(buf, ')'); } else elog(ERROR, "unrecognized testexpr type: %d", (int) nodeTag(sublink->testexpr)); } need_paren = true; switch (sublink->subLinkType) { case EXISTS_SUBLINK: appendStringInfoString(buf, "EXISTS "); break; case ANY_SUBLINK: if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ appendStringInfoString(buf, " IN "); else appendStringInfo(buf, " %s ANY ", opname); break; case ALL_SUBLINK: appendStringInfo(buf, " %s ALL ", opname); break; case ROWCOMPARE_SUBLINK: appendStringInfo(buf, " %s ", opname); break; case EXPR_SUBLINK: case MULTIEXPR_SUBLINK: case ARRAY_SUBLINK: need_paren = false; break; case CTE_SUBLINK: /* shouldn't occur in a SubLink */ default: elog(ERROR, "unrecognized sublink type: %d", (int) sublink->subLinkType); break; } if (need_paren) appendStringInfoChar(buf, '('); get_query_def(query, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoString(buf, "))"); else appendStringInfoChar(buf, ')'); } /* ---------- * get_tablefunc - Parse back a table function * ---------- */ static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; /* XMLTABLE is the only existing implementation. */ appendStringInfoString(buf, "XMLTABLE("); if (tf->ns_uris != NIL) { ListCell *lc1, *lc2; bool first = true; appendStringInfoString(buf, "XMLNAMESPACES ("); forboth(lc1, tf->ns_uris, lc2, tf->ns_names) { Node *expr = (Node *) lfirst(lc1); char *name = strVal(lfirst(lc2)); if (!first) appendStringInfoString(buf, ", "); else first = false; if (name != NULL) { get_rule_expr(expr, context, showimplicit); appendStringInfo(buf, " AS %s", name); } else { appendStringInfoString(buf, "DEFAULT "); get_rule_expr(expr, context, showimplicit); } } appendStringInfoString(buf, "), "); } appendStringInfoChar(buf, '('); get_rule_expr((Node *) tf->rowexpr, context, showimplicit); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) tf->docexpr, context, showimplicit); appendStringInfoChar(buf, ')'); if (tf->colexprs != NIL) { ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; ListCell *l5; int colnum = 0; appendStringInfoString(buf, " COLUMNS "); forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, l4, tf->colexprs, l5, tf->coldefexprs) { char *colname = strVal(lfirst(l1)); Oid typid = lfirst_oid(l2); int32 typmod = lfirst_int(l3); Node *colexpr = (Node *) lfirst(l4); Node *coldefexpr = (Node *) lfirst(l5); bool ordinality = (tf->ordinalitycol == colnum); bool notnull = bms_is_member(colnum, tf->notnulls); if (colnum > 0) appendStringInfoString(buf, ", "); colnum++; appendStringInfo(buf, "%s %s", quote_identifier(colname), ordinality ? "FOR ORDINALITY" : format_type_with_typemod(typid, typmod)); if (ordinality) continue; if (coldefexpr != NULL) { appendStringInfoString(buf, " DEFAULT ("); get_rule_expr((Node *) coldefexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (colexpr != NULL) { appendStringInfoString(buf, " PATH ("); get_rule_expr((Node *) colexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (notnull) appendStringInfoString(buf, " NOT NULL"); } } appendStringInfoChar(buf, ')'); } /* ---------- * get_from_clause - Parse back a FROM clause * * "prefix" is the keyword that denotes the start of the list of FROM * elements. It is FROM when used to parse back SELECT and UPDATE, but * is USING when parsing back DELETE. * ---------- */ static void get_from_clause(Query *query, const char *prefix, deparse_context *context) { StringInfo buf = context->buf; bool first = true; ListCell *l; /* * We use the query's jointree as a guide to what to print. However, we * must ignore auto-added RTEs that are marked not inFromCl. (These can * only appear at the top level of the jointree, so it's sufficient to * check here.) This check also ensures we ignore the rule pseudo-RTEs * for NEW and OLD. */ foreach(l, query->jointree->fromlist) { Node *jtnode = (Node *) lfirst(l); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); if (!rte->inFromCl) continue; } if (first) { appendContextKeyword(context, prefix, -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); first = false; get_from_clause_item(jtnode, query, context); } else { StringInfoData itembuf; appendStringInfoString(buf, ", "); /* * Put the new FROM item's text into itembuf so we can decide * after we've got it whether or not it needs to go on a new line. */ initStringInfo(&itembuf); context->buf = &itembuf; get_from_clause_item(jtnode, query, context); /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { /* Does the new item start with a new line? */ if (itembuf.len > 0 && itembuf.data[0] == '\n') { /* If so, we shouldn't add anything */ /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new item * would cause an overflow. */ if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } } /* Add the new item */ appendStringInfoString(buf, itembuf.data); /* clean up */ pfree(itembuf.data); } } } static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) { StringInfo buf = context->buf; deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); RangeTblFunction *rtfunc1 = NULL; CitusRTEKind rteKind = GetRangeTblKind(rte); if (rte->lateral) appendStringInfoString(buf, "LATERAL "); /* Print the FROM item proper */ switch (rte->rtekind) { case RTE_RELATION: /* Normal relation RTE */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, context->namespaces)); break; case RTE_SUBQUERY: /* Subquery RTE */ appendStringInfoChar(buf, '('); get_query_def(rte->subquery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); appendStringInfoChar(buf, ')'); break; case RTE_FUNCTION: /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); break; } /* Function RTE */ rtfunc1 = (RangeTblFunction *) linitial(rte->functions); /* * Omit ROWS FROM() syntax for just one function, unless it * has both a coldeflist and WITH ORDINALITY. If it has both, * we must use ROWS FROM() syntax to avoid ambiguity about * whether the coldeflist includes the ordinality column. */ if (list_length(rte->functions) == 1 && (rtfunc1->funccolnames == NIL || !rte->funcordinality)) { get_rule_expr_funccall(rtfunc1->funcexpr, context, true); /* we'll print the coldeflist below, if it has one */ } else { bool all_unnest; ListCell *lc; /* * If all the function calls in the list are to unnest, * and none need a coldeflist, then collapse the list back * down to UNNEST(args). (If we had more than one * built-in unnest function, this would get more * difficult.) * * XXX This is pretty ugly, since it makes not-terribly- * future-proof assumptions about what the parser would do * with the output; but the alternative is to emit our * nonstandard ROWS FROM() notation for what might have * been a perfectly spec-compliant multi-argument * UNNEST(). */ all_unnest = true; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (!IsA(rtfunc->funcexpr, FuncExpr) || ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || rtfunc->funccolnames != NIL) { all_unnest = false; break; } } if (all_unnest) { List *allargs = NIL; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); List *args = ((FuncExpr *) rtfunc->funcexpr)->args; allargs = list_concat(allargs, args); } appendStringInfoString(buf, "UNNEST("); get_rule_expr((Node *) allargs, context, true); appendStringInfoChar(buf, ')'); } else { int funcno = 0; appendStringInfoString(buf, "ROWS FROM("); foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (funcno > 0) appendStringInfoString(buf, ", "); get_rule_expr_funccall(rtfunc->funcexpr, context, true); if (rtfunc->funccolnames != NIL) { /* Reconstruct the column definition list */ appendStringInfoString(buf, " AS "); get_from_clause_coldeflist(rtfunc, NULL, context); } funcno++; } appendStringInfoChar(buf, ')'); } /* prevent printing duplicate coldeflist below */ rtfunc1 = NULL; } if (rte->funcordinality) appendStringInfoString(buf, " WITH ORDINALITY"); break; case RTE_TABLEFUNC: get_tablefunc(rte->tablefunc, context, true); break; case RTE_VALUES: /* Values list RTE */ appendStringInfoChar(buf, '('); get_values_def(rte->values_lists, context); appendStringInfoChar(buf, ')'); break; case RTE_CTE: appendStringInfoString(buf, quote_identifier(rte->ctename)); break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); break; } /* Print the relation alias, if needed */ get_rte_alias(rte, varno, false, context); /* Print the column definitions or aliases, if needed */ if (rtfunc1 && rtfunc1->funccolnames != NIL) { /* Reconstruct the columndef list, which is also the aliases */ get_from_clause_coldeflist(rtfunc1, colinfo, context); } else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD || (rte->alias != NULL && rte->alias->colnames != NIL)) { /* Else print column aliases as needed */ get_column_alias_list(colinfo, context); } /* check if column's are given aliases in distributed tables */ else if (colinfo->parentUsing != NIL) { Assert(colinfo->printaliases); get_column_alias_list(colinfo, context); } /* Tablesample clause must go after any alias */ if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && rte->tablesample) { get_tablesample_def(rte->tablesample, context); } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); bool need_paren_on_right; need_paren_on_right = PRETTY_PAREN(context) && !IsA(j->rarg, RangeTblRef) && !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, '('); get_from_clause_item(j->larg, query, context); switch (j->jointype) { case JOIN_INNER: if (j->quals) appendContextKeyword(context, " JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); else appendContextKeyword(context, " CROSS JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_LEFT: appendContextKeyword(context, " LEFT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_FULL: appendContextKeyword(context, " FULL JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_RIGHT: appendContextKeyword(context, " RIGHT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; default: elog(ERROR, "unrecognized join type: %d", (int) j->jointype); } if (need_paren_on_right) appendStringInfoChar(buf, '('); get_from_clause_item(j->rarg, query, context); if (need_paren_on_right) appendStringInfoChar(buf, ')'); if (j->usingClause) { ListCell *lc; bool first = true; appendStringInfoString(buf, " USING ("); /* Use the assigned names, not what's in usingClause */ foreach(lc, colinfo->usingNames) { char *colname = (char *) lfirst(lc); if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } appendStringInfoChar(buf, ')'); if (j->join_using_alias) appendStringInfo(buf, " AS %s", quote_identifier(j->join_using_alias->aliasname)); } else if (j->quals) { appendStringInfoString(buf, " ON "); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr(j->quals, context, false); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } else if (j->jointype != JOIN_INNER) { /* If we didn't say CROSS JOIN above, we must provide an ON */ appendStringInfoString(buf, " ON TRUE"); } if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, ')'); /* Yes, it's correct to put alias after the right paren ... */ if (j->alias != NULL) { /* * Note that it's correct to emit an alias clause if and only if * there was one originally. Otherwise we'd be converting a named * join to unnamed or vice versa, which creates semantic * subtleties we don't want. However, we might print a different * alias name than was there originally. */ appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(j->rtindex, context))); get_column_alias_list(colinfo, context); } } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * get_rte_alias - print the relation's alias, if needed * * If printed, the alias is preceded by a space, or by " AS " if use_as is true. */ static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); char *refname = get_rtable_name(varno, context); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); bool printalias = false; if (rte->alias != NULL) { /* Always print alias if user provided one */ printalias = true; } else if (colinfo->printaliases) { /* Always print alias if we need to print column aliases */ printalias = true; } else if (rte->rtekind == RTE_RELATION) { /* * No need to print alias if it's same as relation name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, get_relation_name(rte->relid)) != 0) printalias = true; } else if (rte->rtekind == RTE_FUNCTION) { /* * For a function RTE, always print alias. This covers possible * renaming of the function and/or instability of the FigureColname * rules for things that aren't simple functions. Note we'd need to * force it anyway for the columndef list case. */ printalias = true; } else if (rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_VALUES) { /* * For a subquery, always print alias. This makes the output * SQL-spec-compliant, even though we allow such aliases to be omitted * on input. */ printalias = true; } else if (rte->rtekind == RTE_CTE) { /* * No need to print alias if it's same as CTE name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, rte->ctename) != 0) printalias = true; } if (printalias) appendStringInfo(context->buf, "%s%s", use_as ? " AS " : " ", quote_identifier(refname)); } /* * get_column_alias_list - print column alias list for an RTE * * Caller must already have printed the relation's alias name. */ static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; int i; bool first = true; /* Don't print aliases if not needed */ if (!colinfo->printaliases) return; for (i = 0; i < colinfo->num_new_cols; i++) { char *colname = colinfo->new_colnames[i]; if (first) { appendStringInfoChar(buf, '('); first = false; } else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } if (!first) appendStringInfoChar(buf, ')'); } /* * get_from_clause_coldeflist - reproduce FROM clause coldeflist * * When printing a top-level coldeflist (which is syntactically also the * relation's column alias list), use column names from colinfo. But when * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the * original coldeflist's names, which are available in rtfunc->funccolnames. * Pass NULL for colinfo to select the latter behavior. * * The coldeflist is appended immediately (no space) to buf. Caller is * responsible for ensuring that an alias or AS is present before it. */ static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; int i; appendStringInfoChar(buf, '('); i = 0; forfour(l1, rtfunc->funccoltypes, l2, rtfunc->funccoltypmods, l3, rtfunc->funccolcollations, l4, rtfunc->funccolnames) { Oid atttypid = lfirst_oid(l1); int32 atttypmod = lfirst_int(l2); Oid attcollation = lfirst_oid(l3); char *attname; if (colinfo) attname = colinfo->colnames[i]; else attname = strVal(lfirst(l4)); Assert(attname); /* shouldn't be any dropped columns here */ if (i > 0) appendStringInfoString(buf, ", "); appendStringInfo(buf, "%s %s", quote_identifier(attname), format_type_with_typemod(atttypid, atttypmod)); if (OidIsValid(attcollation) && attcollation != get_typcollation(atttypid)) appendStringInfo(buf, " COLLATE %s", generate_collation_name(attcollation)); i++; } appendStringInfoChar(buf, ')'); } /* * get_tablesample_def - print a TableSampleClause */ static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) { StringInfo buf = context->buf; Oid argtypes[1]; int nargs; ListCell *l; /* * We should qualify the handler's function name if it wouldn't be * resolved by lookup in the current search path. */ argtypes[0] = INTERNALOID; appendStringInfo(buf, " TABLESAMPLE %s (", generate_function_name(tablesample->tsmhandler, 1, NIL, argtypes, false, NULL, false)); nargs = 0; foreach(l, tablesample->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfirst(l), context, false); } appendStringInfoChar(buf, ')'); if (tablesample->repeatable != NULL) { appendStringInfoString(buf, " REPEATABLE ("); get_rule_expr((Node *) tablesample->repeatable, context, false); appendStringInfoChar(buf, ')'); } } /* * get_opclass_name - fetch name of an index operator class * * The opclass name is appended (after a space) to buf. * * Output is suppressed if the opclass is the default for the given * actual_datatype. (If you don't want this behavior, just pass * InvalidOid for actual_datatype.) */ static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf) { HeapTuple ht_opc; Form_pg_opclass opcrec; char *opcname; char *nspname; ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); if (!HeapTupleIsValid(ht_opc)) elog(ERROR, "cache lookup failed for opclass %u", opclass); opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); if (!OidIsValid(actual_datatype) || GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) { /* Okay, we need the opclass name. Do we need to qualify it? */ opcname = NameStr(opcrec->opcname); if (OpclassIsVisible(opclass)) appendStringInfo(buf, " %s", quote_identifier(opcname)); else { nspname = get_namespace_name_or_temp(opcrec->opcnamespace); appendStringInfo(buf, " %s.%s", quote_identifier(nspname), quote_identifier(opcname)); } } ReleaseSysCache(ht_opc); } /* * processIndirection - take care of array and subfield assignment * * We strip any top-level FieldStore or assignment SubscriptingRef nodes that * appear in the input, printing them as decoration for the base column * name (which we assume the caller just printed). We might also need to * strip CoerceToDomain nodes, but only ones that appear above assignment * nodes. * * Returns the subexpression that's to be assigned. */ static Node * processIndirection(Node *node, deparse_context *context) { StringInfo buf = context->buf; CoerceToDomain *cdomain = NULL; for (;;) { if (node == NULL) break; if (IsA(node, FieldStore)) { FieldStore *fstore = (FieldStore *) node; Oid typrelid; char *fieldname; /* lookup tuple type */ typrelid = get_typ_typrelid(fstore->resulttype); if (!OidIsValid(typrelid)) elog(ERROR, "argument type %s of FieldStore is not a tuple type", format_type_be(fstore->resulttype)); /* * Print the field name. There should only be one target field in * stored rules. There could be more than that in executable * target lists, but this function cannot be used for that case. */ Assert(list_length(fstore->fieldnums) == 1); fieldname = get_attname(typrelid, linitial_int(fstore->fieldnums), false); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); /* * We ignore arg since it should be an uninteresting reference to * the target column or subcolumn. */ node = (Node *) linitial(fstore->newvals); } else if (IsA(node, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) node; if (sbsref->refassgnexpr == NULL) break; printSubscripts(sbsref, context); /* * We ignore refexpr since it should be an uninteresting reference * to the target column or subcolumn. */ node = (Node *) sbsref->refassgnexpr; } else if (IsA(node, CoerceToDomain)) { cdomain = (CoerceToDomain *) node; /* If it's an explicit domain coercion, we're done */ if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; /* Tentatively descend past the CoerceToDomain */ node = (Node *) cdomain->arg; } else break; } /* * If we descended past a CoerceToDomain whose argument turned out not to * be a FieldStore or array assignment, back up to the CoerceToDomain. * (This is not enough to be fully correct if there are nested implicit * CoerceToDomains, but such cases shouldn't ever occur.) */ if (cdomain && node == (Node *) cdomain->arg) node = (Node *) cdomain; return node; } static void printSubscripts(SubscriptingRef *sbsref, deparse_context *context) { StringInfo buf = context->buf; ListCell *lowlist_item; ListCell *uplist_item; lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ foreach(uplist_item, sbsref->refupperindexpr) { appendStringInfoChar(buf, '['); if (lowlist_item) { /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(lowlist_item), context, false); appendStringInfoChar(buf, ':'); lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); } /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(uplist_item), context, false); appendStringInfoChar(buf, ']'); } } /* * get_relation_name * Get the unqualified name of a relation specified by OID * * This differs from the underlying get_rel_name() function in that it will * throw error instead of silently returning NULL if the OID is bad. */ static char * get_relation_name(Oid relid) { char *relname = get_rel_name(relid); if (!relname) elog(ERROR, "cache lookup failed for relation %u", relid); return relname; } /* * generate_relation_or_shard_name * Compute the name to display for a relation or shard * * If the provided relid is equal to the provided distrelid, this function * returns a shard-extended relation name; otherwise, it falls through to a * simple generate_relation_name call. */ static char * generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces) { char *relname = NULL; if (relid == distrelid) { relname = get_relation_name(relid); if (shardid > 0) { Oid schemaOid = get_rel_namespace(relid); char *schemaName = get_namespace_name_or_temp(schemaOid); AppendShardIdToName(&relname, shardid); relname = quote_qualified_identifier(schemaName, relname); } } else { relname = generate_relation_name(relid, namespaces); } return relname; } /* * generate_relation_name * Compute the name to display for a relation specified by OID * * The result includes all necessary quoting and schema-prefixing. * * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. * We will forcibly qualify the relation name if it equals any CTE name * visible in the namespace list. */ char * generate_relation_name(Oid relid, List *namespaces) { HeapTuple tp; Form_pg_class reltup; bool need_qual; ListCell *nslist; char *relname; char *nspname; char *result; tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for relation %u", relid); reltup = (Form_pg_class) GETSTRUCT(tp); relname = NameStr(reltup->relname); /* Check for conflicting CTE name */ need_qual = false; foreach(nslist, namespaces) { deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); ListCell *ctlist; foreach(ctlist, dpns->ctes) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); if (strcmp(cte->ctename, relname) == 0) { need_qual = true; break; } } if (need_qual) break; } /* Otherwise, qualify the name if not visible in search path */ if (!need_qual) need_qual = !RelationIsVisible(relid); if (need_qual) nspname = get_namespace_name_or_temp(reltup->relnamespace); else nspname = NULL; result = quote_qualified_identifier(nspname, relname); ReleaseSysCache(tp); return result; } /* * generate_rte_shard_name returns the qualified name of the shard given a * CITUS_RTE_SHARD range table entry. */ static char * generate_rte_shard_name(RangeTblEntry *rangeTableEntry) { char *shardSchemaName = NULL; char *shardTableName = NULL; Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, NULL); return generate_fragment_name(shardSchemaName, shardTableName); } /* * generate_fragment_name * Compute the name to display for a shard or merged table * * The result includes all necessary quoting and schema-prefixing. The schema * name can be NULL for regular shards. For merged tables, they are always * declared within a job-specific schema, and therefore can't have null schema * names. */ static char * generate_fragment_name(char *schemaName, char *tableName) { StringInfo fragmentNameString = makeStringInfo(); if (schemaName != NULL) { appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), quote_identifier(tableName)); } else { appendStringInfoString(fragmentNameString, quote_identifier(tableName)); } return fragmentNameString->data; } /* * generate_function_name * Compute the name to display for a function specified by OID, * given that it is being called with the specified actual arg names and * types. (Those matter because of ambiguous-function resolution rules.) * * If we're dealing with a potentially variadic function (in practice, this * means a FuncExpr or Aggref, not some other way of calling a function), then * has_variadic must specify whether variadic arguments have been merged, * and *use_variadic_p will be set to indicate whether to print VARIADIC in * the output. For non-FuncExpr cases, has_variadic should be false and * use_variadic_p can be NULL. * * inGroupBy must be true if we're deparsing a GROUP BY clause. * * The result includes all necessary quoting and schema-prefixing. */ static char * generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy) { char *result; HeapTuple proctup; Form_pg_proc procform; char *proname; bool use_variadic; char *nspname; FuncDetailCode p_result; Oid p_funcid; Oid p_rettype; bool p_retset; int p_nvargs; Oid p_vatype; Oid *p_true_typeids; bool force_qualify = false; proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); if (!HeapTupleIsValid(proctup)) elog(ERROR, "cache lookup failed for function %u", funcid); procform = (Form_pg_proc) GETSTRUCT(proctup); proname = NameStr(procform->proname); /* * Due to parser hacks to avoid needing to reserve CUBE, we need to force * qualification of some function names within GROUP BY. */ if (inGroupBy) { if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) force_qualify = true; } /* * Determine whether VARIADIC should be printed. We must do this first * since it affects the lookup rules in func_get_detail(). * * Currently, we always print VARIADIC if the function has a merged * variadic-array argument. Note that this is always the case for * functions taking a VARIADIC argument type other than VARIADIC ANY. * * In principle, if VARIADIC wasn't originally specified and the array * actual argument is deconstructable, we could print the array elements * separately and not print VARIADIC, thus more nearly reproducing the * original input. For the moment that seems like too much complication * for the benefit, and anyway we do not know whether VARIADIC was * originally specified if it's a non-ANY type. */ if (use_variadic_p) { /* Parser should not have set funcvariadic unless fn is variadic */ Assert(!has_variadic || OidIsValid(procform->provariadic)); use_variadic = has_variadic; *use_variadic_p = use_variadic; } else { Assert(!has_variadic); use_variadic = false; } /* * The idea here is to schema-qualify only if the parser would fail to * resolve the correct function given the unqualified func name with the * specified argtypes and VARIADIC flag. But if we already decided to * force qualification, then we can skip the lookup and pretend we didn't * find it. */ if (!force_qualify) p_result = func_get_detail(list_make1(makeString(proname)), NIL, argnames, nargs, argtypes, !use_variadic, true, false, &p_funcid, &p_rettype, &p_retset, &p_nvargs, &p_vatype, &p_true_typeids, NULL); else { p_result = FUNCDETAIL_NOTFOUND; p_funcid = InvalidOid; } if ((p_result == FUNCDETAIL_NORMAL || p_result == FUNCDETAIL_AGGREGATE || p_result == FUNCDETAIL_WINDOWFUNC) && p_funcid == funcid) nspname = NULL; else nspname = get_namespace_name_or_temp(procform->pronamespace); result = quote_qualified_identifier(nspname, proname); ReleaseSysCache(proctup); return result; } /* * generate_operator_name * Compute the name to display for an operator specified by OID, * given that it is being called with the specified actual arg types. * (Arg types matter because of ambiguous-operator resolution rules. * Pass InvalidOid for unused arg of a unary operator.) * * The result includes all necessary quoting and schema-prefixing, * plus the OPERATOR() decoration needed to use a qualified operator name * in an expression. */ char * generate_operator_name(Oid operid, Oid arg1, Oid arg2) { StringInfoData buf; HeapTuple opertup; Form_pg_operator operform; char *oprname; char *nspname; initStringInfo(&buf); opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); if (!HeapTupleIsValid(opertup)) elog(ERROR, "cache lookup failed for operator %u", operid); operform = (Form_pg_operator) GETSTRUCT(opertup); oprname = NameStr(operform->oprname); /* * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, * we don't check if the operator is in current namespace or not. This is * because this check is costly when the operator is not in current namespace. */ nspname = get_namespace_name_or_temp(operform->oprnamespace); Assert(nspname != NULL); appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); appendStringInfoString(&buf, oprname); appendStringInfoChar(&buf, ')'); ReleaseSysCache(opertup); return buf.data; } /* * get_one_range_partition_bound_string * A C string representation of one range partition bound */ char * get_range_partbound_string(List *bound_datums) { deparse_context context; StringInfo buf = makeStringInfo(); ListCell *cell; char *sep; memset(&context, 0, sizeof(deparse_context)); context.buf = buf; appendStringInfoChar(buf, '('); sep = ""; foreach(cell, bound_datums) { PartitionRangeDatum *datum = lfirst_node(PartitionRangeDatum, cell); appendStringInfoString(buf, sep); if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) appendStringInfoString(buf, "MINVALUE"); else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) appendStringInfoString(buf, "MAXVALUE"); else { Const *val = castNode(Const, datum->value); get_const_expr(val, &context, -1); } sep = ", "; } appendStringInfoChar(buf, ')'); return buf->data; } /* * Collect a list of OIDs of all sequences owned by the specified relation, * and column if specified. If deptype is not zero, then only find sequences * with the specified dependency type. */ List * getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) { List *result = NIL; Relation depRel; ScanKeyData key[3]; SysScanDesc scan; HeapTuple tup; depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); if (attnum) ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* * We assume any auto or internal dependency of a sequence on a column * must be what we are looking for. (We need the relkind test because * indexes can also have auto dependencies on columns.) */ if (deprec->classid == RelationRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { if (!deptype || deprec->deptype == deptype) result = lappend_oid(result, deprec->objid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return result; } /* * get_insert_column_names_list Prepares the insert-column-names list. Any indirection * decoration needed on the column names can be inferred from the top targetlist. */ static List * get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte) { char *sep; ListCell *l; List *strippedexprs; strippedexprs = NIL; sep = ""; appendStringInfoChar(buf, '('); foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. * Add the stripped expressions to strippedexprs. (If it's a * single-VALUES statement, the stripped expressions are the VALUES to * print below. Otherwise they're just Vars and not really * interesting.) */ strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } appendStringInfoString(buf, ") "); return strippedexprs; } #endif /* (PG_VERSION_NUM >= PG_VERSION_16) && (PG_VERSION_NUM < PG_VERSION_17) */ ================================================ FILE: src/backend/distributed/deparser/ruleutils_17.c ================================================ /*------------------------------------------------------------------------- * * ruleutils_16.c * Functions to convert stored expressions/querytrees back to * source text * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/distributed/deparser/ruleutils_16.c * * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ #include "pg_version_constants.h" #include "pg_config.h" #if (PG_VERSION_NUM >= PG_VERSION_17) && (PG_VERSION_NUM < PG_VERSION_18) #include "postgres.h" #include #include #include #include "access/amapi.h" #include "access/htup_details.h" #include "access/relation.h" #include "access/table.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_authid.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_language.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_proc.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/tablespace.h" #include "common/keywords.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/multi_router_planner.h" #include "distributed/namespace_utils.h" #include "executor/spi.h" #include "foreign/foreign.h" #include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "optimizer/optimizer.h" #include "parser/parse_node.h" #include "parser/parse_agg.h" #include "parser/parse_func.h" #include "parser/parse_oper.h" #include "parser/parse_relation.h" #include "parser/parser.h" #include "parser/parsetree.h" #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSupport.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/ruleutils.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "utils/varlena.h" #include "utils/xml.h" /* ---------- * Pretty formatting constants * ---------- */ /* Indent counts */ #define PRETTYINDENT_STD 8 #define PRETTYINDENT_JOIN 4 #define PRETTYINDENT_VAR 4 #define PRETTYINDENT_LIMIT 40 /* wrap limit */ /* Pretty flags */ #define PRETTYFLAG_PAREN 0x0001 #define PRETTYFLAG_INDENT 0x0002 /* Default line length for pretty-print wrapping: 0 means wrap always */ #define WRAP_COLUMN_DEFAULT 0 /* macros to test if pretty action needed */ #define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) #define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) /* ---------- * Local data types * ---------- */ /* Context info needed for invoking a recursive querytree display routine */ typedef struct { StringInfo buf; /* output buffer to append to */ List *namespaces; /* List of deparse_namespace nodes */ TupleDesc resultDesc; /* if top level of a view, the view's tupdesc */ List *targetList; /* Current query level's SELECT targetlist */ List *windowClause; /* Current query level's WINDOW clause */ int prettyFlags; /* enabling of pretty-print functions */ int wrapColumn; /* max line length, or -1 for no limit */ int indentLevel; /* current indent level for prettyprint */ bool varprefix; /* true to print prefixes on Vars */ Oid distrelid; /* the distributed table being modified, if valid */ int64 shardid; /* a distributed table's shardid, if positive */ bool colNamesVisible; /* do we care about output column names? */ bool inGroupBy; /* deparsing GROUP BY clause? */ bool varInOrderBy; /* deparsing simple Var in ORDER BY? */ Bitmapset *appendparents; /* if not null, map child Vars of these relids * back to the parent rel */ } deparse_context; /* * Each level of query context around a subtree needs a level of Var namespace. * A Var having varlevelsup=N refers to the N'th item (counting from 0) in * the current context's namespaces list. * * The rangetable is the list of actual RTEs from the query tree, and the * cte list is the list of actual CTEs. * * rtable_names holds the alias name to be used for each RTE (either a C * string, or NULL for nameless RTEs such as unnamed joins). * rtable_columns holds the column alias names to be used for each RTE. * * In some cases we need to make names of merged JOIN USING columns unique * across the whole query, not only per-RTE. If so, unique_using is true * and using_names is a list of C strings representing names already assigned * to USING columns. * * When deparsing plan trees, there is always just a single item in the * deparse_namespace list (since a plan tree never contains Vars with * varlevelsup > 0). We store the PlanState node that is the immediate * parent of the expression to be deparsed, as well as a list of that * PlanState's ancestors. In addition, we store its outer and inner subplan * state nodes, as well as their plan nodes' targetlists, and the index tlist * if the current plan node might contain INDEX_VAR Vars. (These fields could * be derived on-the-fly from the current PlanState, but it seems notationally * clearer to set them up as separate fields.) */ typedef struct { List *rtable; /* List of RangeTblEntry nodes */ List *rtable_names; /* Parallel list of names for RTEs */ List *rtable_columns; /* Parallel list of deparse_columns structs */ List *subplans; /* List of Plan trees for SubPlans */ List *ctes; /* List of CommonTableExpr nodes */ AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ /* Workspace for column alias assignment: */ bool unique_using; /* Are we making USING names globally unique */ List *using_names; /* List of assigned names for USING columns */ /* Remaining fields are used only when deparsing a Plan tree: */ Plan *plan; /* immediate parent of current expression */ List *ancestors; /* ancestors of planstate */ Plan *outer_plan; /* outer subnode, or NULL if none */ Plan *inner_plan; /* inner subnode, or NULL if none */ List *outer_tlist; /* referent for OUTER_VAR Vars */ List *inner_tlist; /* referent for INNER_VAR Vars */ List *index_tlist; /* referent for INDEX_VAR Vars */ /* Special namespace representing a function signature: */ char *funcname; int numargs; char **argnames; } deparse_namespace; /* Callback signature for resolve_special_varno() */ typedef void (*rsv_callback) (Node *node, deparse_context *context, void *callback_arg); /* * Per-relation data about column alias names. * * Selecting aliases is unreasonably complicated because of the need to dump * rules/views whose underlying tables may have had columns added, deleted, or * renamed since the query was parsed. We must nonetheless print the rule/view * in a form that can be reloaded and will produce the same results as before. * * For each RTE used in the query, we must assign column aliases that are * unique within that RTE. SQL does not require this of the original query, * but due to factors such as *-expansion we need to be able to uniquely * reference every column in a decompiled query. As long as we qualify all * column references, per-RTE uniqueness is sufficient for that. * * However, we can't ensure per-column name uniqueness for unnamed join RTEs, * since they just inherit column names from their input RTEs, and we can't * rename the columns at the join level. Most of the time this isn't an issue * because we don't need to reference the join's output columns as such; we * can reference the input columns instead. That approach can fail for merged * JOIN USING columns, however, so when we have one of those in an unnamed * join, we have to make that column's alias globally unique across the whole * query to ensure it can be referenced unambiguously. * * Another problem is that a JOIN USING clause requires the columns to be * merged to have the same aliases in both input RTEs, and that no other * columns in those RTEs or their children conflict with the USING names. * To handle that, we do USING-column alias assignment in a recursive * traversal of the query's jointree. When descending through a JOIN with * USING, we preassign the USING column names to the child columns, overriding * other rules for column alias assignment. We also mark each RTE with a list * of all USING column names selected for joins containing that RTE, so that * when we assign other columns' aliases later, we can avoid conflicts. * * Another problem is that if a JOIN's input tables have had columns added or * deleted since the query was parsed, we must generate a column alias list * for the join that matches the current set of input columns --- otherwise, a * change in the number of columns in the left input would throw off matching * of aliases to columns of the right input. Thus, positions in the printable * column alias list are not necessarily one-for-one with varattnos of the * JOIN, so we need a separate new_colnames[] array for printing purposes. */ typedef struct { /* * colnames is an array containing column aliases to use for columns that * existed when the query was parsed. Dropped columns have NULL entries. * This array can be directly indexed by varattno to get a Var's name. * * Non-NULL entries are guaranteed unique within the RTE, *except* when * this is for an unnamed JOIN RTE. In that case we merely copy up names * from the two input RTEs. * * During the recursive descent in set_using_names(), forcible assignment * of a child RTE's column name is represented by pre-setting that element * of the child's colnames array. So at that stage, NULL entries in this * array just mean that no name has been preassigned, not necessarily that * the column is dropped. */ int num_cols; /* length of colnames[] array */ char **colnames; /* array of C strings and NULLs */ /* * new_colnames is an array containing column aliases to use for columns * that would exist if the query was re-parsed against the current * definitions of its base tables. This is what to print as the column * alias list for the RTE. This array does not include dropped columns, * but it will include columns added since original parsing. Indexes in * it therefore have little to do with current varattno values. As above, * entries are unique unless this is for an unnamed JOIN RTE. (In such an * RTE, we never actually print this array, but we must compute it anyway * for possible use in computing column names of upper joins.) The * parallel array is_new_col marks which of these columns are new since * original parsing. Entries with is_new_col false must match the * non-NULL colnames entries one-for-one. */ int num_new_cols; /* length of new_colnames[] array */ char **new_colnames; /* array of C strings */ bool *is_new_col; /* array of bool flags */ /* This flag tells whether we should actually print a column alias list */ bool printaliases; /* This list has all names used as USING names in joins above this RTE */ List *parentUsing; /* names assigned to parent merged columns */ /* * If this struct is for a JOIN RTE, we fill these fields during the * set_using_names() pass to describe its relationship to its child RTEs. * * leftattnos and rightattnos are arrays with one entry per existing * output column of the join (hence, indexable by join varattno). For a * simple reference to a column of the left child, leftattnos[i] is the * child RTE's attno and rightattnos[i] is zero; and conversely for a * column of the right child. But for merged columns produced by JOIN * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. * Also, if the column has been dropped, both are zero. * * If it's a JOIN USING, usingNames holds the alias names selected for the * merged columns (these might be different from the original USING list, * if we had to modify names to achieve uniqueness). */ int leftrti; /* rangetable index of left child */ int rightrti; /* rangetable index of right child */ int *leftattnos; /* left-child varattnos of join cols, or 0 */ int *rightattnos; /* right-child varattnos of join cols, or 0 */ List *usingNames; /* names assigned to merged columns */ } deparse_columns; /* This macro is analogous to rt_fetch(), but for deparse_columns structs */ #define deparse_columns_fetch(rangetable_index, dpns) \ ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) /* * Entry in set_rtable_names' hash table */ typedef struct { char name[NAMEDATALEN]; /* Hash key --- must be first */ int counter; /* Largest addition used so far for name */ } NameHashEntry; /* ---------- * Local functions * * Most of these functions used to use fixed-size buffers to build their * results. Now, they take an (already initialized) StringInfo object * as a parameter, and append their text output to its contents. * ---------- */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used); static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces); static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing); static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static char *make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static void expand_colnames_array_to(deparse_columns *colinfo, int n); static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo); static char *get_rtable_name(int rtindex, deparse_context *context); static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); static Plan *find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan); static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns); static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns); static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_values_def(List *values_lists, deparse_context *context); static void get_with_clause(Query *query, deparse_context *context); static void get_select_query_def(Query *query, deparse_context *context); static void get_insert_query_def(Query *query, deparse_context *context); static void get_update_query_def(Query *query, deparse_context *context); static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte); static void get_delete_query_def(Query *query, deparse_context *context); static void get_merge_query_def(Query *query, deparse_context *context); static void get_utility_query_def(Query *query, deparse_context *context); static void get_basic_select_query(Query *query, deparse_context *context); static void get_target_list(List *targetList, deparse_context *context); static void get_setop_query(Node *setOp, Query *query, deparse_context *context); static Node *get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context); static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context); static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context); static void get_rule_windowclause(Query *query, deparse_context *context); static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context); static char *get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context); static void get_special_variable(Node *node, deparse_context *context, void *callback_arg); static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg); static Node *find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p); static SubPlan *find_param_generator(Param *param, deparse_context *context, int *column_p); static SubPlan *find_param_generator_initplan(Param *param, Plan *plan, int *column_p); static void get_parameter(Param *param, deparse_context *context); static const char *get_simple_binary_op_name(OpExpr *expr); static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus); static void removeStringInfoSpaces(StringInfo str); static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit); static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit); static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit); static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit); static bool looks_like_function(Node *node); static void get_oper_expr(OpExpr *expr, deparse_context *context); static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit); static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit); static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref); static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg); static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg); static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg); static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode); static void get_const_expr(Const *constval, deparse_context *context, int showtype); static void get_const_collation(Const *constval, deparse_context *context); static void get_json_format(JsonFormat *format, StringInfo buf); static void get_json_returning(JsonReturning *returning, StringInfo buf, bool json_format_by_default); static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit); static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf); static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg); static void simple_quote_literal(StringInfo buf, const char *val); static void get_sublink_expr(SubLink *sublink, deparse_context *context); static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit); static void get_from_clause(Query *query, const char *prefix, deparse_context *context); static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context); static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context); static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context); static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context); static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context); static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf); static Node *processIndirection(Node *node, deparse_context *context); static void printSubscripts(SubscriptingRef *aref, deparse_context *context); static char *get_relation_name(Oid relid); static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces); static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); static char *generate_fragment_name(char *schemaName, char *tableName); static char *generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy); static List *get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte); static void get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit); static void get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, deparse_context *context, bool showimplicit); static void get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, deparse_context *context, bool showimplicit, bool needcomma); #define only_marker(rte) ((rte)->inh ? "" : "ONLY ") /* * pg_get_query_def parses back one query tree, and outputs the resulting query * string into given buffer. */ void pg_get_query_def(Query *query, StringInfo buffer) { get_query_def(query, buffer, NIL, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* * get_merged_argument_list merges both the IN and OUT arguments lists into one and * also eliminates the INOUT duplicates(present in both the lists). After merging both * the lists, it returns all the named-arguments in a list(mergedNamedArgList) along * with their types(mergedNamedArgTypes), final argument list(mergedArgumentList), and * the total number of arguments(totalArguments). */ bool get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, Oid **mergedNamedArgTypes, List **mergedArgumentList, int *totalArguments) { Oid functionOid = stmt->funcexpr->funcid; List *namedArgList = NIL; List *finalArgumentList = NIL; Oid *finalArgTypes; Oid *argTypes = NULL; char *argModes = NULL; char **argNames = NULL; int argIndex = 0; HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "cache lookup failed for function %u", functionOid); } int defArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); ReleaseSysCache(proctup); if (argModes == NULL) { /* No OUT arguments */ return false; } /* * Passed arguments Includes IN, OUT, INOUT (in both the lists) and VARIADIC arguments, * which means INOUT arguments are double counted. */ int numberOfArgs = list_length(stmt->funcexpr->args) + list_length(stmt->outargs); int totalInoutArgs = 0; /* Let's count INOUT arguments from the defined number of arguments */ for (argIndex=0; argIndex < defArgs; ++argIndex) { if (argModes[argIndex] == PROARGMODE_INOUT) totalInoutArgs++; } /* Remove the duplicate INOUT counting */ numberOfArgs = numberOfArgs - totalInoutArgs; finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs); ListCell *inArgCell = list_head(stmt->funcexpr->args); ListCell *outArgCell = list_head(stmt->outargs); for (argIndex=0; argIndex < numberOfArgs; ++argIndex) { switch (argModes[argIndex]) { case PROARGMODE_IN: case PROARGMODE_VARIADIC: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); break; } case PROARGMODE_OUT: { Node *arg = (Node *) lfirst(outArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_INOUT: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_TABLE: default: { elog(ERROR, "Unhandled procedure argument mode[%d]", argModes[argIndex]); break; } } } /* * After eliminating INOUT duplicates and merging OUT arguments, we now * have the final list of arguments. */ if (defArgs != list_length(finalArgumentList)) { elog(ERROR, "Insufficient number of args passed[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)); } if (list_length(finalArgumentList) > FUNC_MAX_ARGS) { ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)))); } *mergedNamedArgList = namedArgList; *mergedNamedArgTypes = finalArgTypes; *mergedArgumentList = finalArgumentList; *totalArguments = numberOfArgs; return true; } /* * pg_get_rule_expr deparses an expression and returns the result as a string. */ char * pg_get_rule_expr(Node *expression) { bool showImplicitCasts = true; deparse_context context; StringInfo buffer = makeStringInfo(); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buffer; context.namespaces = NIL; context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = false; context.prettyFlags = 0; context.wrapColumn = WRAP_COLUMN_DEFAULT; context.indentLevel = 0; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.distrelid = InvalidOid; context.shardid = INVALID_SHARD_ID; get_rule_expr(expression, &context, showImplicitCasts); /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return buffer->data; } /* * set_rtable_names: select RTE aliases to be used in printing a query * * We fill in dpns->rtable_names with a list of names that is one-for-one with * the already-filled dpns->rtable list. Each RTE name is unique among those * in the new namespace plus any ancestor namespaces listed in * parent_namespaces. * * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. * * Note that this function is only concerned with relation names, not column * names. */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used) { HASHCTL hash_ctl; HTAB *names_hash; NameHashEntry *hentry; bool found; int rtindex; ListCell *lc; dpns->rtable_names = NIL; /* nothing more to do if empty rtable */ if (dpns->rtable == NIL) return; /* * We use a hash table to hold known names, so that this process is O(N) * not O(N^2) for N names. */ hash_ctl.keysize = NAMEDATALEN; hash_ctl.entrysize = sizeof(NameHashEntry); hash_ctl.hcxt = CurrentMemoryContext; names_hash = hash_create("set_rtable_names names", list_length(dpns->rtable), &hash_ctl, HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); /* Preload the hash table with names appearing in parent_namespaces */ foreach(lc, parent_namespaces) { deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); ListCell *lc2; foreach(lc2, olddpns->rtable_names) { char *oldname = (char *) lfirst(lc2); if (oldname == NULL) continue; hentry = (NameHashEntry *) hash_search(names_hash, oldname, HASH_ENTER, &found); /* we do not complain about duplicate names in parent namespaces */ hentry->counter = 0; } } /* Now we can scan the rtable */ rtindex = 1; foreach(lc, dpns->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); char *refname; /* Just in case this takes an unreasonable amount of time ... */ CHECK_FOR_INTERRUPTS(); if (rels_used && !bms_is_member(rtindex, rels_used)) { /* Ignore unreferenced RTE */ refname = NULL; } else if (rte->alias) { /* If RTE has a user-defined alias, prefer that */ refname = rte->alias->aliasname; } else if (rte->rtekind == RTE_RELATION) { /* Use the current actual name of the relation */ refname = get_rel_name(rte->relid); } else if (rte->rtekind == RTE_JOIN) { /* Unnamed join has no refname */ refname = NULL; } else { /* Otherwise use whatever the parser assigned */ refname = rte->eref->aliasname; } /* * If the selected name isn't unique, append digits to make it so, and * make a new hash entry for it once we've got a unique name. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (refname) { hentry = (NameHashEntry *) hash_search(names_hash, refname, HASH_ENTER, &found); if (found) { /* Name already in use, must choose a new one */ int refnamelen = strlen(refname); char *modname = (char *) palloc(refnamelen + 16); NameHashEntry *hentry2; do { hentry->counter++; for (;;) { memcpy(modname, refname, refnamelen); sprintf(modname + refnamelen, "_%d", hentry->counter); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from refname to keep all the digits */ refnamelen = pg_mbcliplen(refname, refnamelen, refnamelen - 1); } hentry2 = (NameHashEntry *) hash_search(names_hash, modname, HASH_ENTER, &found); } while (found); hentry2->counter = 0; /* init new hash entry */ refname = modname; } else { /* Name not previously used, need only initialize hentry */ hentry->counter = 0; } } dpns->rtable_names = lappend(dpns->rtable_names, refname); rtindex++; } hash_destroy(names_hash); } /* * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree * * For convenience, this is defined to initialize the deparse_namespace struct * from scratch. */ static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces) { ListCell *lc; ListCell *lc2; /* Initialize *dpns and fill rtable/ctes links */ memset(dpns, 0, sizeof(deparse_namespace)); dpns->rtable = query->rtable; dpns->subplans = NIL; dpns->ctes = query->cteList; dpns->appendrels = NULL; /* Assign a unique relation alias to each RTE */ set_rtable_names(dpns, parent_namespaces, NULL); /* Initialize dpns->rtable_columns to contain zeroed structs */ dpns->rtable_columns = NIL; while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) dpns->rtable_columns = lappend(dpns->rtable_columns, palloc0(sizeof(deparse_columns))); /* If it's a utility query, it won't have a jointree */ if (query->jointree) { /* Detect whether global uniqueness of USING names is needed */ dpns->unique_using = has_dangerous_join_using(dpns, (Node *) query->jointree); /* * Select names for columns merged by USING, via a recursive pass over * the query jointree. */ set_using_names(dpns, (Node *) query->jointree, NIL); } /* * Now assign remaining column aliases for each RTE. We do this in a * linear scan of the rtable, so as to process RTEs whether or not they * are in the jointree (we mustn't miss NEW.*, INSERT target relations, * etc). JOIN RTEs must be processed after their children, but this is * okay because they appear later in the rtable list than their children * (cf Asserts in identify_join_columns()). */ forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); if (rte->rtekind == RTE_JOIN) set_join_column_names(dpns, rte, colinfo); else set_relation_column_names(dpns, rte, colinfo); } } /* * has_dangerous_join_using: search jointree for unnamed JOIN USING * * Merged columns of a JOIN USING may act differently from either of the input * columns, either because they are merged with COALESCE (in a FULL JOIN) or * because an implicit coercion of the underlying input column is required. * In such a case the column must be referenced as a column of the JOIN not as * a column of either input. And this is problematic if the join is unnamed * (alias-less): we cannot qualify the column's name with an RTE name, since * there is none. (Forcibly assigning an alias to the join is not a solution, * since that will prevent legal references to tables below the join.) * To ensure that every column in the query is unambiguously referenceable, * we must assign such merged columns names that are globally unique across * the whole query, aliasing other columns out of the way as necessary. * * Because the ensuing re-aliasing is fairly damaging to the readability of * the query, we don't do this unless we have to. So, we must pre-scan * the join tree to see if we have to, before starting set_using_names(). */ static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do here */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) { if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) return true; } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; /* Is it an unnamed JOIN with USING? */ if (j->alias == NULL && j->usingClause) { /* * Yes, so check each join alias var to see if any of them are not * simple references to underlying columns. If so, we have a * dangerous situation and must pick unique aliases. */ RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); /* We need only examine the merged columns */ for (int i = 0; i < jrte->joinmergedcols; i++) { Node *aliasvar = list_nth(jrte->joinaliasvars, i); if (!IsA(aliasvar, Var)) return true; } } /* Nope, but inspect children */ if (has_dangerous_join_using(dpns, j->larg)) return true; if (has_dangerous_join_using(dpns, j->rarg)) return true; } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); return false; } /* * set_using_names: select column aliases to be used for merged USING columns * * We do this during a recursive descent of the query jointree. * dpns->unique_using must already be set to determine the global strategy. * * Column alias info is saved in the dpns->rtable_columns list, which is * assumed to be filled with pre-zeroed deparse_columns structs. * * parentUsing is a list of all USING aliases assigned in parent joins of * the current jointree node. (The passed-in list must not be modified.) */ static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do now */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) set_using_names(dpns, (Node *) lfirst(lc), parentUsing); } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); int *leftattnos; int *rightattnos; deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; int i; ListCell *lc; /* Get info about the shape of the join */ identify_join_columns(j, rte, colinfo); leftattnos = colinfo->leftattnos; rightattnos = colinfo->rightattnos; /* Look up the not-yet-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * If this join is unnamed, then we cannot substitute new aliases at * this level, so any name requirements pushed down to here must be * pushed down again to the children. */ if (rte->alias == NULL) { for (i = 0; i < colinfo->num_cols; i++) { char *colname = colinfo->colnames[i]; if (colname == NULL) continue; /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } } } /* * If there's a USING clause, select the USING column names and push * those names down to the children. We have two strategies: * * If dpns->unique_using is true, we force all USING names to be * unique across the whole query level. In principle we'd only need * the names of dangerous USING columns to be globally unique, but to * safely assign all USING names in a single pass, we have to enforce * the same uniqueness rule for all of them. However, if a USING * column's name has been pushed down from the parent, we should use * it as-is rather than making a uniqueness adjustment. This is * necessary when we're at an unnamed join, and it creates no risk of * ambiguity. Also, if there's a user-written output alias for a * merged column, we prefer to use that rather than the input name; * this simplifies the logic and seems likely to lead to less aliasing * overall. * * If dpns->unique_using is false, we only need USING names to be * unique within their own join RTE. We still need to honor * pushed-down names, though. * * Though significantly different in results, these two strategies are * implemented by the same code, with only the difference of whether * to put assigned names into dpns->using_names. */ if (j->usingClause) { /* Copy the input parentUsing list so we don't modify it */ parentUsing = list_copy(parentUsing); /* USING names must correspond to the first join output columns */ expand_colnames_array_to(colinfo, list_length(j->usingClause)); i = 0; foreach(lc, j->usingClause) { char *colname = strVal(lfirst(lc)); /* Assert it's a merged column */ Assert(leftattnos[i] != 0 && rightattnos[i] != 0); /* Adopt passed-down name if any, else select unique name */ if (colinfo->colnames[i] != NULL) colname = colinfo->colnames[i]; else { /* Prefer user-written output alias if any */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); /* Make it appropriately unique */ colname = make_colname_unique(colname, dpns, colinfo); if (dpns->unique_using) dpns->using_names = lappend(dpns->using_names, colname); /* Save it as output column name, too */ colinfo->colnames[i] = colname; } /* Remember selected names for use later */ colinfo->usingNames = lappend(colinfo->usingNames, colname); parentUsing = lappend(parentUsing, colname); /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } i++; } } /* Mark child deparse_columns structs with correct parentUsing info */ leftcolinfo->parentUsing = parentUsing; rightcolinfo->parentUsing = parentUsing; /* Now recursively assign USING column names in children */ set_using_names(dpns, j->larg, parentUsing); set_using_names(dpns, j->rarg, parentUsing); } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * set_relation_column_names: select column aliases for a non-join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. */ static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { int ncolumns; char **real_colnames; bool changed_any; bool has_anonymous; int noldcolumns; int i; int j; /* * Construct an array of the current "real" column names of the RTE. * real_colnames[] will be indexed by physical column number, with NULL * entries for dropped columns. */ if (rte->rtekind == RTE_RELATION || GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* Relation --- look to the system catalogs for up-to-date info */ Relation rel; TupleDesc tupdesc; rel = relation_open(rte->relid, AccessShareLock); tupdesc = RelationGetDescr(rel); ncolumns = tupdesc->natts; real_colnames = (char **) palloc(ncolumns * sizeof(char *)); for (i = 0; i < ncolumns; i++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); if (attr->attisdropped) real_colnames[i] = NULL; else real_colnames[i] = pstrdup(NameStr(attr->attname)); } relation_close(rel, AccessShareLock); } else { /* Otherwise get the column names from eref or expandRTE() */ List *colnames; ListCell *lc; /* * Functions returning composites have the annoying property that some * of the composite type's columns might have been dropped since the * query was parsed. If possible, use expandRTE() to handle that * case, since it has the tedious logic needed to find out about * dropped columns. However, if we're explaining a plan, then we * don't have rte->functions because the planner thinks that won't be * needed later, and that breaks expandRTE(). So in that case we have * to rely on rte->eref, which may lead us to report a dropped * column's old name; that seems close enough for EXPLAIN's purposes. * * For non-RELATION, non-FUNCTION RTEs, we can just look at rte->eref, * which should be sufficiently up-to-date: no other RTE types can * have columns get dropped from under them after parsing. */ if (rte->rtekind == RTE_FUNCTION && rte->functions != NIL) { /* Since we're not creating Vars, rtindex etc. don't matter */ expandRTE(rte, 1, 0, -1, true /* include dropped */ , &colnames, NULL); } else colnames = rte->eref->colnames; ncolumns = list_length(colnames); real_colnames = (char **) palloc(ncolumns * sizeof(char *)); i = 0; foreach(lc, colnames) { /* * If the column name we find here is an empty string, then it's a * dropped column, so change to NULL. */ char *cname = strVal(lfirst(lc)); if (cname[0] == '\0') cname = NULL; real_colnames[i] = cname; i++; } } /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that there are now more columns than there were when the * query was parsed, ie colnames could be longer than rte->eref->colnames. * We must assign unique aliases to the new columns too, else there could * be unresolved conflicts when the view/rule is reloaded. */ expand_colnames_array_to(colinfo, ncolumns); Assert(colinfo->num_cols == ncolumns); /* * Make sufficiently large new_colnames and is_new_col arrays, too. * * Note: because we leave colinfo->num_new_cols zero until after the loop, * colname_is_unique will not consult that array, which is fine because it * would only be duplicate effort. */ colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); /* * Scan the columns, select a unique alias for each one, and store it in * colinfo->colnames and colinfo->new_colnames. The former array has NULL * entries for dropped columns, the latter omits them. Also mark * new_colnames entries as to whether they are new since parse time; this * is the case for entries beyond the length of rte->eref->colnames. */ noldcolumns = list_length(rte->eref->colnames); changed_any = false; has_anonymous = false; j = 0; for (i = 0; i < ncolumns; i++) { char *real_colname = real_colnames[i]; char *colname = colinfo->colnames[i]; /* Skip dropped columns */ if (real_colname == NULL) { Assert(colname == NULL); /* colnames[i] is already NULL */ continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; } /* Put names of non-dropped columns in new_colnames[] too */ colinfo->new_colnames[j] = colname; /* And mark them as new or not */ colinfo->is_new_col[j] = (i >= noldcolumns); j++; /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; /* * Remember if there is a reference to an anonymous column as named by * char * FigureColname(Node *node) */ if (!has_anonymous && strcmp(real_colname, "?column?") == 0) has_anonymous = true; } /* * Set correct length for new_colnames[] array. (Note: if columns have * been added, colinfo->num_cols includes them, which is not really quite * right but is harmless, since any new columns must be at the end where * they won't affect varattnos of pre-existing columns.) */ colinfo->num_new_cols = j; /* * For a relation RTE, we need only print the alias column names if any * are different from the underlying "real" names. For a function RTE, * always emit a complete column alias list; this is to protect against * possible instability of the default column names (eg, from altering * parameter names). For tablefunc RTEs, we never print aliases, because * the column names are part of the clause itself. For other RTE types, * print if we changed anything OR if there were user-written column * aliases (since the latter would be part of the underlying "reality"). */ if (rte->rtekind == RTE_RELATION) colinfo->printaliases = changed_any; else if (rte->rtekind == RTE_FUNCTION) colinfo->printaliases = true; else if (rte->rtekind == RTE_TABLEFUNC) colinfo->printaliases = false; else if (rte->alias && rte->alias->colnames != NIL) colinfo->printaliases = true; else colinfo->printaliases = changed_any || has_anonymous; } /* * set_join_column_names: select column aliases for a join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. Also, names for USING columns were already chosen by * set_using_names(). We further expect that column alias selection has been * completed for both input RTEs. */ static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; bool changed_any; int noldcolumns; int nnewcolumns; Bitmapset *leftmerged = NULL; Bitmapset *rightmerged = NULL; int i; int j; int ic; int jc; /* Look up the previously-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that one or both inputs now have more columns than there * were when the query was parsed, but we'll deal with that below. We * only need entries in colnames for pre-existing columns. */ noldcolumns = list_length(rte->eref->colnames); expand_colnames_array_to(colinfo, noldcolumns); Assert(colinfo->num_cols == noldcolumns); /* * Scan the join output columns, select an alias for each one, and store * it in colinfo->colnames. If there are USING columns, set_using_names() * already selected their names, so we can start the loop at the first * non-merged column. */ changed_any = false; for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) { char *colname = colinfo->colnames[i]; char *real_colname; /* Join column must refer to at least one input column */ Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); /* Get the child column name */ if (colinfo->leftattnos[i] > 0) real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; else if (colinfo->rightattnos[i] > 0) real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; else { /* We're joining system columns --- use eref name */ real_colname = strVal(list_nth(rte->eref->colnames, i)); } /* If child col has been dropped, no need to assign a join colname */ if (real_colname == NULL) { colinfo->colnames[i] = NULL; continue; } /* In an unnamed join, just report child column names as-is */ if (rte->alias == NULL) { colinfo->colnames[i] = real_colname; continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; } /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; } /* * Calculate number of columns the join would have if it were re-parsed * now, and create storage for the new_colnames and is_new_col arrays. * * Note: colname_is_unique will be consulting new_colnames[] during the * loops below, so its not-yet-filled entries must be zeroes. */ nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - list_length(colinfo->usingNames); colinfo->num_new_cols = nnewcolumns; colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); /* * Generating the new_colnames array is a bit tricky since any new columns * added since parse time must be inserted in the right places. This code * must match the parser, which will order a join's columns as merged * columns first (in USING-clause order), then non-merged columns from the * left input (in attnum order), then non-merged columns from the right * input (ditto). If one of the inputs is itself a join, its columns will * be ordered according to the same rule, which means newly-added columns * might not be at the end. We can figure out what's what by consulting * the leftattnos and rightattnos arrays plus the input is_new_col arrays. * * In these loops, i indexes leftattnos/rightattnos (so it's join varattno * less one), j indexes new_colnames/is_new_col, and ic/jc have similar * meanings for the current child RTE. */ /* Handle merged columns; they are first and can't be new */ i = j = 0; while (i < noldcolumns && colinfo->leftattnos[i] != 0 && colinfo->rightattnos[i] != 0) { /* column name is already determined and known unique */ colinfo->new_colnames[j] = colinfo->colnames[i]; colinfo->is_new_col[j] = false; /* build bitmapsets of child attnums of merged columns */ if (colinfo->leftattnos[i] > 0) leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); if (colinfo->rightattnos[i] > 0) rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); i++, j++; } /* Handle non-merged left-child columns */ ic = 0; for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) { char *child_colname = leftcolinfo->new_colnames[jc]; if (!leftcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of left child */ while (ic < leftcolinfo->num_cols && leftcolinfo->colnames[ic] == NULL) ic++; Assert(ic < leftcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, leftmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->leftattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; } colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; j++; } /* Handle non-merged right-child columns in exactly the same way */ ic = 0; for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) { char *child_colname = rightcolinfo->new_colnames[jc]; if (!rightcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of right child */ while (ic < rightcolinfo->num_cols && rightcolinfo->colnames[ic] == NULL) ic++; Assert(ic < rightcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, rightmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->rightattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; } colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; j++; } /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING for (int col_index = 0; col_index < colinfo->num_cols; col_index++) { /* * In the above processing-loops, "i" advances only if * the column is not new, check if this is a new column. */ if (colinfo->is_new_col[col_index]) i++; } Assert(j == nnewcolumns); #endif /* * For a named join, print column aliases if we changed any from the child * names. Unnamed joins cannot print aliases. */ if (rte->alias != NULL) colinfo->printaliases = changed_any; else colinfo->printaliases = false; } /* * colname_is_unique: is colname distinct from already-chosen column names? * * dpns is query-wide info, colinfo is for the column's RTE */ static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { int i; ListCell *lc; /* Check against already-assigned column aliases within RTE */ for (i = 0; i < colinfo->num_cols; i++) { char *oldname = colinfo->colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* * If we're building a new_colnames array, check that too (this will be * partially but not completely redundant with the previous checks) */ for (i = 0; i < colinfo->num_new_cols; i++) { char *oldname = colinfo->new_colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* Also check against USING-column names that must be globally unique */ foreach(lc, dpns->using_names) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } /* Also check against names already assigned for parent-join USING cols */ foreach(lc, colinfo->parentUsing) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } return true; } /* * make_colname_unique: modify colname if necessary to make it unique * * dpns is query-wide info, colinfo is for the column's RTE */ static char * make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { /* * If the selected name isn't unique, append digits to make it so. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (!colname_is_unique(colname, dpns, colinfo)) { int colnamelen = strlen(colname); char *modname = (char *) palloc(colnamelen + 16); int i = 0; do { i++; for (;;) { memcpy(modname, colname, colnamelen); sprintf(modname + colnamelen, "_%d", i); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from colname to keep all the digits */ colnamelen = pg_mbcliplen(colname, colnamelen, colnamelen - 1); } } while (!colname_is_unique(modname, dpns, colinfo)); colname = modname; } return colname; } /* * expand_colnames_array_to: make colinfo->colnames at least n items long * * Any added array entries are initialized to zero. */ static void expand_colnames_array_to(deparse_columns *colinfo, int n) { if (n > colinfo->num_cols) { if (colinfo->colnames == NULL) colinfo->colnames = palloc0_array(char *, n); else { colinfo->colnames = repalloc0_array(colinfo->colnames, char *, colinfo->num_cols, n); } colinfo->num_cols = n; } } /* * identify_join_columns: figure out where columns of a join come from * * Fills the join-specific fields of the colinfo struct, except for * usingNames which is filled later. */ static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo) { int numjoincols; int jcolno; int rcolno; ListCell *lc; /* Extract left/right child RT indexes */ if (IsA(j->larg, RangeTblRef)) colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; else if (IsA(j->larg, JoinExpr)) colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->larg)); if (IsA(j->rarg, RangeTblRef)) colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; else if (IsA(j->rarg, JoinExpr)) colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->rarg)); /* Assert children will be processed earlier than join in second pass */ Assert(colinfo->leftrti < j->rtindex); Assert(colinfo->rightrti < j->rtindex); /* Initialize result arrays with zeroes */ numjoincols = list_length(jrte->joinaliasvars); Assert(numjoincols == list_length(jrte->eref->colnames)); colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); /* * Deconstruct RTE's joinleftcols/joinrightcols into desired format. * Recall that the column(s) merged due to USING are the first column(s) * of the join output. We need not do anything special while scanning * joinleftcols, but while scanning joinrightcols we must distinguish * merged from unmerged columns. */ jcolno = 0; foreach(lc, jrte->joinleftcols) { int leftattno = lfirst_int(lc); colinfo->leftattnos[jcolno++] = leftattno; } rcolno = 0; foreach(lc, jrte->joinrightcols) { int rightattno = lfirst_int(lc); if (rcolno < jrte->joinmergedcols) /* merged column? */ colinfo->rightattnos[rcolno] = rightattno; else colinfo->rightattnos[jcolno++] = rightattno; rcolno++; } Assert(jcolno == numjoincols); } /* * get_rtable_name: convenience function to get a previously assigned RTE alias * * The RTE must belong to the topmost namespace level in "context". */ static char * get_rtable_name(int rtindex, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); return (char *) list_nth(dpns->rtable_names, rtindex - 1); } /* * set_deparse_plan: set up deparse_namespace to parse subexpressions * of a given Plan node * * This sets the plan, outer_planstate, inner_planstate, outer_tlist, * inner_tlist, and index_tlist fields. Caller is responsible for adjusting * the ancestors list if necessary. Note that the rtable and ctes fields do * not need to change when shifting attention to different plan nodes in a * single plan tree. */ static void set_deparse_plan(deparse_namespace *dpns, Plan *plan) { dpns->plan = plan; /* * We special-case Append and MergeAppend to pretend that the first child * plan is the OUTER referent; we have to interpret OUTER Vars in their * tlists according to one of the children, and the first one is the most * natural choice. */ if (IsA(plan, Append)) dpns->outer_plan = linitial(((Append *) plan)->appendplans); else if (IsA(plan, MergeAppend)) dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); else dpns->outer_plan = outerPlan(plan); if (dpns->outer_plan) dpns->outer_tlist = dpns->outer_plan->targetlist; else dpns->outer_tlist = NIL; /* * For a SubqueryScan, pretend the subplan is INNER referent. (We don't * use OUTER because that could someday conflict with the normal meaning.) * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. * For a WorkTableScan, locate the parent RecursiveUnion plan node and use * that as INNER referent. * * For MERGE, pretend the ModifyTable's source plan (its outer plan) is * INNER referent. This is the join from the target relation to the data * source, and all INNER_VAR Vars in other parts of the query refer to its * targetlist. * * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the * excluded expression's tlist. (Similar to the SubqueryScan we don't want * to reuse OUTER, it's used for RETURNING in some modify table cases, * although not INSERT .. CONFLICT). */ if (IsA(plan, SubqueryScan)) dpns->inner_plan = ((SubqueryScan *) plan)->subplan; else if (IsA(plan, CteScan)) dpns->inner_plan = list_nth(dpns->subplans, ((CteScan *) plan)->ctePlanId - 1); else if (IsA(plan, WorkTableScan)) dpns->inner_plan = find_recursive_union(dpns, (WorkTableScan *) plan); else if (IsA(plan, ModifyTable)) { if (((ModifyTable *) plan)->operation == CMD_MERGE) dpns->inner_plan = outerPlan(plan); else dpns->inner_plan = plan; } else dpns->inner_plan = innerPlan(plan); if (IsA(plan, ModifyTable) && ((ModifyTable *) plan)->operation == CMD_INSERT) dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; else if (dpns->inner_plan) dpns->inner_tlist = dpns->inner_plan->targetlist; else dpns->inner_tlist = NIL; /* Set up referent for INDEX_VAR Vars, if needed */ if (IsA(plan, IndexOnlyScan)) dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; else if (IsA(plan, ForeignScan)) dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; else if (IsA(plan, CustomScan)) dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; else dpns->index_tlist = NIL; } /* * Locate the ancestor plan node that is the RecursiveUnion generating * the WorkTableScan's work table. We can match on wtParam, since that * should be unique within the plan tree. */ static Plan * find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan) { ListCell *lc; foreach(lc, dpns->ancestors) { Plan *ancestor = (Plan *) lfirst(lc); if (IsA(ancestor, RecursiveUnion) && ((RecursiveUnion *) ancestor)->wtParam == wtscan->wtParam) return ancestor; } elog(ERROR, "could not find RecursiveUnion for WorkTableScan with wtParam %d", wtscan->wtParam); return NULL; } /* * push_child_plan: temporarily transfer deparsing attention to a child plan * * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the * deparse context in case the referenced expression itself uses * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid * affecting levelsup issues (although in a Plan tree there really shouldn't * be any). * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_child_plan. */ static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns) { /* Save state for restoration later */ *save_dpns = *dpns; /* Link current plan node into ancestors list */ dpns->ancestors = lcons(dpns->plan, dpns->ancestors); /* Set attention on selected child */ set_deparse_plan(dpns, plan); } /* * pop_child_plan: undo the effects of push_child_plan */ static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { List *ancestors; /* Get rid of ancestors list cell added by push_child_plan */ ancestors = list_delete_first(dpns->ancestors); /* Restore fields changed by push_child_plan */ *dpns = *save_dpns; /* Make sure dpns->ancestors is right (may be unnecessary) */ dpns->ancestors = ancestors; } /* * push_ancestor_plan: temporarily transfer deparsing attention to an * ancestor plan * * When expanding a Param reference, we must adjust the deparse context * to match the plan node that contains the expression being printed; * otherwise we'd fail if that expression itself contains a Param or * OUTER_VAR/INNER_VAR/INDEX_VAR variable. * * The target ancestor is conveniently identified by the ListCell holding it * in dpns->ancestors. * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_ancestor_plan. */ static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns) { Plan *plan = (Plan *) lfirst(ancestor_cell); /* Save state for restoration later */ *save_dpns = *dpns; /* Build a new ancestor list with just this node's ancestors */ dpns->ancestors = list_copy_tail(dpns->ancestors, list_cell_number(dpns->ancestors, ancestor_cell) + 1); /* Set attention on selected ancestor */ set_deparse_plan(dpns, plan); } /* * pop_ancestor_plan: undo the effects of push_ancestor_plan */ static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { /* Free the ancestor list made in push_ancestor_plan */ list_free(dpns->ancestors); /* Restore fields changed by push_ancestor_plan */ *dpns = *save_dpns; } /* ---------- * deparse_shard_query - Parse back a query for execution on a shard * * Builds an SQL string to perform the provided query on a specific shard and * places this string into the provided buffer. * ---------- */ void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer) { get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* ---------- * get_query_def - Parse back one query parsetree * * query: parsetree to be displayed * buf: output text is appended to buf * parentnamespace: list (initially empty) of outer-level deparse_namespace's * resultDesc: if not NULL, the output tuple descriptor for the view * represented by a SELECT query. We use the column names from it * to label SELECT output columns, in preference to names in the query * colNamesVisible: true if the surrounding context cares about the output * column names at all (as, for example, an EXISTS() context does not); * when false, we can suppress dummy column labels such as "?column?" * prettyFlags: bitmask of PRETTYFLAG_XXX options * wrapColumn: maximum line length, or -1 to disable wrapping * startIndent: initial indentation amount * ---------- */ static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, colNamesVisible, prettyFlags, wrapColumn, startIndent); } /* ---------- * get_query_def_extended - Parse back one query parsetree, optionally * with extension using a shard identifier. * * If distrelid is valid and shardid is positive, the provided shardid is added * any time the provided relid is deparsed, so that the query may be executed * on a placement for the given shard. * ---------- */ static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { deparse_context context; deparse_namespace dpns; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); /* * Before we begin to examine the query, acquire locks on referenced * relations, and fix up deleted columns in JOIN RTEs. This ensures * consistent results. Note we assume it's OK to scribble on the passed * querytree! * * We are only deparsing the query (we are not about to execute it), so we * only need AccessShareLock on the relations it mentions. */ AcquireRewriteLocks(query, false, false); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buf; context.namespaces = lcons(&dpns, list_copy(parentnamespace)); context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = (parentnamespace != NIL || list_length(query->rtable) != 1); context.prettyFlags = prettyFlags; context.wrapColumn = wrapColumn; context.indentLevel = startIndent; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.appendparents = NULL; context.distrelid = distrelid; context.shardid = shardid; set_deparse_for_query(&dpns, query, parentnamespace); switch (query->commandType) { case CMD_SELECT: /* We set context.resultDesc only if it's a SELECT */ context.resultDesc = resultDesc; get_select_query_def(query, &context); break; case CMD_UPDATE: get_update_query_def(query, &context); break; case CMD_INSERT: get_insert_query_def(query, &context); break; case CMD_DELETE: get_delete_query_def(query, &context); break; case CMD_MERGE: get_merge_query_def(query, &context); break; case CMD_NOTHING: appendStringInfoString(buf, "NOTHING"); break; case CMD_UTILITY: get_utility_query_def(query, &context); break; default: elog(ERROR, "unrecognized query command type: %d", query->commandType); break; } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } /* ---------- * get_values_def - Parse back a VALUES list * ---------- */ static void get_values_def(List *values_lists, deparse_context *context) { StringInfo buf = context->buf; bool first_list = true; ListCell *vtl; appendStringInfoString(buf, "VALUES "); foreach(vtl, values_lists) { List *sublist = (List *) lfirst(vtl); bool first_col = true; ListCell *lc; if (first_list) first_list = false; else appendStringInfoString(buf, ", "); appendStringInfoChar(buf, '('); foreach(lc, sublist) { Node *col = (Node *) lfirst(lc); if (first_col) first_col = false; else appendStringInfoChar(buf, ','); /* * Print the value. Whole-row Vars need special treatment. */ get_rule_expr_toplevel(col, context, false); } appendStringInfoChar(buf, ')'); } } /* ---------- * get_with_clause - Parse back a WITH clause * ---------- */ static void get_with_clause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; if (query->cteList == NIL) return; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } if (query->hasRecursive) sep = "WITH RECURSIVE "; else sep = "WITH "; foreach(l, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); appendStringInfoString(buf, sep); appendStringInfoString(buf, quote_identifier(cte->ctename)); if (cte->aliascolnames) { bool first = true; ListCell *col; appendStringInfoChar(buf, '('); foreach(col, cte->aliascolnames) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(col)))); } appendStringInfoChar(buf, ')'); } appendStringInfoString(buf, " AS "); switch (cte->ctematerialized) { case CTEMaterializeDefault: break; case CTEMaterializeAlways: appendStringInfoString(buf, "MATERIALIZED "); break; case CTEMaterializeNever: appendStringInfoString(buf, "NOT MATERIALIZED "); break; } appendStringInfoChar(buf, '('); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); appendStringInfoChar(buf, ')'); if (cte->search_clause) { bool first = true; ListCell *lc; appendStringInfo(buf, " SEARCH %s FIRST BY ", cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); foreach(lc, cte->search_clause->search_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); } if (cte->cycle_clause) { bool first = true; ListCell *lc; appendStringInfoString(buf, " CYCLE "); foreach(lc, cte->cycle_clause->cycle_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); { Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) { appendStringInfoString(buf, " TO "); get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); appendStringInfoString(buf, " DEFAULT "); get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); } } appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); } sep = ", "; } if (PRETTY_INDENT(context)) { context->indentLevel -= PRETTYINDENT_STD; appendContextKeyword(context, "", 0, 0, 0); } else appendStringInfoChar(buf, ' '); } /* ---------- * get_select_query_def - Parse back a SELECT parsetree * ---------- */ static void get_select_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; bool force_colno; ListCell *l; /* Insert the WITH clause if given */ get_with_clause(query, context); /* Subroutines may need to consult the SELECT targetlist and windowClause */ context->targetList = query->targetList; context->windowClause = query->windowClause; /* * If the Query node has a setOperations tree, then it's the top level of * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT * fields are interesting in the top query itself. */ if (query->setOperations) { get_setop_query(query->setOperations, query, context); /* ORDER BY clauses must be simple in this case */ force_colno = true; } else { get_basic_select_query(query, context); force_colno = false; } /* Add the ORDER BY clause if given */ if (query->sortClause != NIL) { appendContextKeyword(context, " ORDER BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_orderby(query->sortClause, query->targetList, force_colno, context); } /* * Add the LIMIT/OFFSET clauses if given. If non-default options, use the * standard spelling of LIMIT. */ if (query->limitOffset != NULL) { appendContextKeyword(context, " OFFSET ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitOffset, context, false); } if (query->limitCount != NULL) { if (query->limitOption == LIMIT_OPTION_WITH_TIES) { // had to add '(' and ')' here because it fails with casting appendContextKeyword(context, " FETCH FIRST (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitCount, context, false); appendStringInfoString(buf, ") ROWS WITH TIES"); } else { appendContextKeyword(context, " LIMIT ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); if (IsA(query->limitCount, Const) && ((Const *) query->limitCount)->constisnull) appendStringInfoString(buf, "ALL"); else get_rule_expr(query->limitCount, context, false); } } /* Add FOR [KEY] UPDATE/SHARE clauses if present */ if (query->hasForUpdate) { foreach(l, query->rowMarks) { RowMarkClause *rc = (RowMarkClause *) lfirst(l); /* don't print implicit clauses */ if (rc->pushedDown) continue; switch (rc->strength) { case LCS_NONE: /* we intentionally throw an error for LCS_NONE */ elog(ERROR, "unrecognized LockClauseStrength %d", (int) rc->strength); break; case LCS_FORKEYSHARE: appendContextKeyword(context, " FOR KEY SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORSHARE: appendContextKeyword(context, " FOR SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORNOKEYUPDATE: appendContextKeyword(context, " FOR NO KEY UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORUPDATE: appendContextKeyword(context, " FOR UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; } appendStringInfo(buf, " OF %s", quote_identifier(get_rtable_name(rc->rti, context))); if (rc->waitPolicy == LockWaitError) appendStringInfoString(buf, " NOWAIT"); else if (rc->waitPolicy == LockWaitSkip) appendStringInfoString(buf, " SKIP LOCKED"); } } } /* * Detect whether query looks like SELECT ... FROM VALUES(); * if so, return the VALUES RTE. Otherwise return NULL. */ static RangeTblEntry * get_simple_values_rte(Query *query, TupleDesc resultDesc) { RangeTblEntry *result = NULL; ListCell *lc; int colno; /* * We want to return true even if the Query also contains OLD or NEW rule * RTEs. So the idea is to scan the rtable and see if there is only one * inFromCl RTE that is a VALUES RTE. */ foreach(lc, query->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); if (rte->rtekind == RTE_VALUES && rte->inFromCl) { if (result) return NULL; /* multiple VALUES (probably not possible) */ result = rte; } else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) continue; /* ignore rule entries */ else return NULL; /* something else -> not simple VALUES */ } /* * We don't need to check the targetlist in any great detail, because * parser/analyze.c will never generate a "bare" VALUES RTE --- they only * appear inside auto-generated sub-queries with very restricted * structure. However, DefineView might have modified the tlist by * injecting new column aliases; so compare tlist resnames against the * RTE's names to detect that. */ if (result) { ListCell *lcn; if (list_length(query->targetList) != list_length(result->eref->colnames)) return NULL; /* this probably cannot happen */ colno = 0; forboth(lc, query->targetList, lcn, result->eref->colnames) { TargetEntry *tle = (TargetEntry *) lfirst(lc); char *cname = strVal(lfirst(lcn)); char *colname; if (tle->resjunk) return NULL; /* this probably cannot happen */ /* compute name that get_target_list would use for column */ colno++; if (resultDesc && colno <= resultDesc->natts) colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); else colname = tle->resname; /* does it match the VALUES RTE? */ if (colname == NULL || strcmp(colname, cname) != 0) return NULL; /* column name has been changed */ } } return result; } static void get_basic_select_query(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *values_rte; char *sep; ListCell *l; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } /* * If the query looks like SELECT * FROM (VALUES ...), then print just the * VALUES part. This reverses what transformValuesClause() did at parse * time. */ values_rte = get_simple_values_rte(query, context->resultDesc); if (values_rte) { get_values_def(values_rte->values_lists, context); return; } /* * Build up the query string - first we say SELECT */ if (query->isReturn) appendStringInfoString(buf, "RETURN"); else appendStringInfoString(buf, "SELECT"); /* Add the DISTINCT clause if given */ if (query->distinctClause != NIL) { if (query->hasDistinctOn) { appendStringInfoString(buf, " DISTINCT ON ("); sep = ""; foreach(l, query->distinctClause) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, false, context); sep = ", "; } appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DISTINCT"); } /* Then we tell what to select (the targetlist) */ get_target_list(query->targetList, context); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add the WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add the GROUP BY clause if given */ if (query->groupClause != NULL || query->groupingSets != NULL) { bool save_ingroupby; appendContextKeyword(context, " GROUP BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); if (query->groupDistinct) appendStringInfoString(buf, "DISTINCT "); save_ingroupby = context->inGroupBy; context->inGroupBy = true; if (query->groupingSets == NIL) { sep = ""; foreach(l, query->groupClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, false, context); sep = ", "; } } else { sep = ""; foreach(l, query->groupingSets) { GroupingSet *grp = lfirst(l); appendStringInfoString(buf, sep); get_rule_groupingset(grp, query->targetList, true, context); sep = ", "; } } context->inGroupBy = save_ingroupby; } /* Add the HAVING clause if given */ if (query->havingQual != NULL) { appendContextKeyword(context, " HAVING ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->havingQual, context, false); } /* Add the WINDOW clause if needed */ if (query->windowClause != NIL) get_rule_windowclause(query, context); } /* ---------- * get_target_list - Parse back a SELECT target list * * This is also used for RETURNING lists in INSERT/UPDATE/DELETE/MERGE. * ---------- */ static void get_target_list(List *targetList, deparse_context *context) { StringInfo buf = context->buf; StringInfoData targetbuf; bool last_was_multiline = false; char *sep; int colno; ListCell *l; /* we use targetbuf to hold each TLE's text temporarily */ initStringInfo(&targetbuf); sep = " "; colno = 0; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); char *colname; char *attname; if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; colno++; /* * Put the new field text into targetbuf so we can decide after we've * got it whether or not it needs to go on a new line. */ resetStringInfo(&targetbuf); context->buf = &targetbuf; /* * We special-case Var nodes rather than using get_rule_expr. This is * needed because get_rule_expr will display a whole-row Var as * "foo.*", which is the preferred notation in most contexts, but at * the top level of a SELECT list it's not right (the parser will * expand that notation into multiple columns, yielding behavior * different from a whole-row Var). We need to call get_variable * directly so that we can tell it to do the right thing, and so that * we can get the attribute name which is the default AS label. */ if (tle->expr && (IsA(tle->expr, Var))) { attname = get_variable((Var *) tle->expr, 0, true, context); } else { get_rule_expr((Node *) tle->expr, context, true); /* * When colNamesVisible is true, we should always show the * assigned column name explicitly. Otherwise, show it only if * it's not FigureColname's fallback. */ attname = context->colNamesVisible ? NULL : "?column?"; } /* * Figure out what the result column should be called. In the context * of a view, use the view's tuple descriptor (so as to pick up the * effects of any column RENAME that's been done on the view). * Otherwise, just use what we can find in the TLE. */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; /* Show AS unless the column's name is correct as-is */ if (colname) /* resname could be NULL */ { if (attname == NULL || strcmp(attname, colname) != 0) appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); } /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { int leading_nl_pos; /* Does the new field start with a new line? */ if (targetbuf.len > 0 && targetbuf.data[0] == '\n') leading_nl_pos = 0; else leading_nl_pos = -1; /* If so, we shouldn't add anything */ if (leading_nl_pos >= 0) { /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the output buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new field is * not the first and either the new field would cause an * overflow or the last field used more than one line. */ if (colno > 1 && ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || last_was_multiline)) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } /* Remember this field's multiline status for next iteration */ last_was_multiline = (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); } /* Add the new field */ appendStringInfoString(buf, targetbuf.data); } /* clean up */ pfree(targetbuf.data); } static void get_setop_query(Node *setOp, Query *query, deparse_context *context) { StringInfo buf = context->buf; bool need_paren; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); if (IsA(setOp, RangeTblRef)) { RangeTblRef *rtr = (RangeTblRef *) setOp; RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); Query *subquery = rte->subquery; Assert(subquery != NULL); Assert(subquery->setOperations == NULL); /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ need_paren = (subquery->cteList || subquery->sortClause || subquery->rowMarks || subquery->limitOffset || subquery->limitCount); if (need_paren) appendStringInfoChar(buf, '('); get_query_def(subquery, buf, context->namespaces, context->resultDesc, context->colNamesVisible, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoChar(buf, ')'); } else if (IsA(setOp, SetOperationStmt)) { SetOperationStmt *op = (SetOperationStmt *) setOp; int subindent; bool save_colnamesvisible; /* * We force parens when nesting two SetOperationStmts, except when the * lefthand input is another setop of the same kind. Syntactically, * we could omit parens in rather more cases, but it seems best to use * parens to flag cases where the setop operator changes. If we use * parens, we also increase the indentation level for the child query. * * There are some cases in which parens are needed around a leaf query * too, but those are more easily handled at the next level down (see * code above). */ if (IsA(op->larg, SetOperationStmt)) { SetOperationStmt *lop = (SetOperationStmt *) op->larg; if (op->op == lop->op && op->all == lop->all) need_paren = false; else need_paren = true; } else need_paren = false; if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; appendContextKeyword(context, "", subindent, 0, 0); } else subindent = 0; get_setop_query(op->larg, query, context); if (need_paren) appendContextKeyword(context, ") ", -subindent, 0, 0); else if (PRETTY_INDENT(context)) appendContextKeyword(context, "", -subindent, 0, 0); else appendStringInfoChar(buf, ' '); switch (op->op) { case SETOP_UNION: appendStringInfoString(buf, "UNION "); break; case SETOP_INTERSECT: appendStringInfoString(buf, "INTERSECT "); break; case SETOP_EXCEPT: appendStringInfoString(buf, "EXCEPT "); break; default: elog(ERROR, "unrecognized set op: %d", (int) op->op); } if (op->all) appendStringInfoString(buf, "ALL "); /* Always parenthesize if RHS is another setop */ need_paren = IsA(op->rarg, SetOperationStmt); /* * The indentation code here is deliberately a bit different from that * for the lefthand input, because we want the line breaks in * different places. */ if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; } else subindent = 0; appendContextKeyword(context, "", subindent, 0, 0); /* * The output column names of the RHS sub-select don't matter. */ save_colnamesvisible = context->colNamesVisible; context->colNamesVisible = false; get_setop_query(op->rarg, query, context); context->colNamesVisible = save_colnamesvisible; if (PRETTY_INDENT(context)) context->indentLevel -= subindent; if (need_paren) appendContextKeyword(context, ")", 0, 0, 0); } else { elog(ERROR, "unrecognized node type: %d", (int) nodeTag(setOp)); } } /* * Display a sort/group clause. * * Also returns the expression tree, so caller need not find it again. */ static Node * get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; TargetEntry *tle; Node *expr; tle = get_sortgroupref_tle(ref, tlist); expr = (Node *) tle->expr; /* * Use column-number form if requested by caller. Otherwise, if * expression is a constant, force it to be dumped with an explicit cast * as decoration --- this is because a simple integer constant is * ambiguous (and will be misinterpreted by findTargetlistEntrySQL92()) if * we dump it without any decoration. Similarly, if it's just a Var, * there is risk of misinterpretation if the column name is reassigned in * the SELECT list, so we may need to force table qualification. And, if * it's anything more complex than a simple Var, then force extra parens * around it, to ensure it can't be misinterpreted as a cube() or rollup() * construct. */ if (force_colno) { Assert(!tle->resjunk); appendStringInfo(buf, "%d", tle->resno); } else if (!expr) /* do nothing, probably can't happen */ ; else if (IsA(expr, Const)) get_const_expr((Const *) expr, context, 1); else if (IsA(expr, Var)) { /* Tell get_variable to check for name conflict */ bool save_varinorderby = context->varInOrderBy; context->varInOrderBy = true; (void) get_variable((Var *) expr, 0, false, context); context->varInOrderBy = save_varinorderby; } else { /* * We must force parens for function-like expressions even if * PRETTY_PAREN is off, since those are the ones in danger of * misparsing. For other expressions we need to force them only if * PRETTY_PAREN is on, since otherwise the expression will output them * itself. (We can't skip the parens.) */ bool need_paren = (PRETTY_PAREN(context) || IsA(expr, FuncExpr) || IsA(expr, Aggref) || IsA(expr, WindowFunc) || IsA(expr, JsonConstructorExpr)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, true); if (need_paren) appendStringInfoChar(context->buf, ')'); } return expr; } /* * Display a GroupingSet */ static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context) { ListCell *l; StringInfo buf = context->buf; bool omit_child_parens = true; char *sep = ""; switch (gset->kind) { case GROUPING_SET_EMPTY: appendStringInfoString(buf, "()"); return; case GROUPING_SET_SIMPLE: { if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, '('); foreach(l, gset->content) { Index ref = lfirst_int(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(ref, targetlist, false, context); sep = ", "; } if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, ')'); } return; case GROUPING_SET_ROLLUP: appendStringInfoString(buf, "ROLLUP("); break; case GROUPING_SET_CUBE: appendStringInfoString(buf, "CUBE("); break; case GROUPING_SET_SETS: appendStringInfoString(buf, "GROUPING SETS ("); omit_child_parens = false; break; } foreach(l, gset->content) { appendStringInfoString(buf, sep); get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); sep = ", "; } appendStringInfoChar(buf, ')'); } /* * Display an ORDER BY list. */ static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = ""; foreach(l, orderList) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); Node *sortexpr; Oid sortcoltype; TypeCacheEntry *typentry; appendStringInfoString(buf, sep); sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, force_colno, context); sortcoltype = exprType(sortexpr); /* See whether operator is default < or > for datatype */ typentry = lookup_type_cache(sortcoltype, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); if (srt->sortop == typentry->lt_opr) { /* ASC is default, so emit nothing for it */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); } else if (srt->sortop == typentry->gt_opr) { appendStringInfoString(buf, " DESC"); /* DESC defaults to NULLS FIRST */ if (!srt->nulls_first) appendStringInfoString(buf, " NULLS LAST"); } else { appendStringInfo(buf, " USING %s", generate_operator_name(srt->sortop, sortcoltype, sortcoltype)); /* be specific to eliminate ambiguity */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); else appendStringInfoString(buf, " NULLS LAST"); } sep = ", "; } } /* * Display a WINDOW clause. * * Note that the windowClause list might contain only anonymous window * specifications, in which case we should print nothing here. */ static void get_rule_windowclause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = NULL; foreach(l, query->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->name == NULL) continue; /* ignore anonymous windows */ if (sep == NULL) appendContextKeyword(context, " WINDOW ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); else appendStringInfoString(buf, sep); appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); get_rule_windowspec(wc, query->targetList, context); sep = ", "; } } /* * Display a window definition */ static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context) { StringInfo buf = context->buf; bool needspace = false; const char *sep; ListCell *l; appendStringInfoChar(buf, '('); if (wc->refname) { appendStringInfoString(buf, quote_identifier(wc->refname)); needspace = true; } /* partition clauses are always inherited, so only print if no refname */ if (wc->partitionClause && !wc->refname) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "PARTITION BY "); sep = ""; foreach(l, wc->partitionClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, false, context); sep = ", "; } needspace = true; } /* print ordering clause only if not inherited */ if (wc->orderClause && !wc->copiedOrder) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "ORDER BY "); get_rule_orderby(wc->orderClause, targetList, false, context); needspace = true; } /* framing clause is never inherited, so print unless it's default */ if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) { if (needspace) appendStringInfoChar(buf, ' '); if (wc->frameOptions & FRAMEOPTION_RANGE) appendStringInfoString(buf, "RANGE "); else if (wc->frameOptions & FRAMEOPTION_ROWS) appendStringInfoString(buf, "ROWS "); else if (wc->frameOptions & FRAMEOPTION_GROUPS) appendStringInfoString(buf, "GROUPS "); else Assert(false); if (wc->frameOptions & FRAMEOPTION_BETWEEN) appendStringInfoString(buf, "BETWEEN "); if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) appendStringInfoString(buf, "UNBOUNDED PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) { get_rule_expr(wc->startOffset, context, false); if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); if (wc->frameOptions & FRAMEOPTION_BETWEEN) { appendStringInfoString(buf, "AND "); if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) { get_rule_expr(wc->endOffset, context, false); if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); } if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) appendStringInfoString(buf, "EXCLUDE GROUP "); else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) appendStringInfoString(buf, "EXCLUDE TIES "); /* we will now have a trailing space; remove it */ buf->len--; } appendStringInfoChar(buf, ')'); } /* ---------- * get_insert_query_def - Parse back an INSERT parsetree * ---------- */ static void get_insert_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *select_rte = NULL; RangeTblEntry *values_rte = NULL; RangeTblEntry *rte; ListCell *l; List *strippedexprs = NIL; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * If it's an INSERT ... SELECT or multi-row VALUES, there will be a * single RTE for the SELECT or VALUES. Plain VALUES has neither. */ foreach(l, query->rtable) { rte = (RangeTblEntry *) lfirst(l); if (rte->rtekind == RTE_SUBQUERY) { if (select_rte) elog(ERROR, "too many subquery RTEs in INSERT"); select_rte = rte; } if (rte->rtekind == RTE_VALUES) { if (values_rte) elog(ERROR, "too many values RTEs in INSERT"); values_rte = rte; } } if (select_rte && values_rte) elog(ERROR, "both subquery and values RTEs in INSERT"); /* * Start the query with INSERT INTO relname */ rte = rt_fetch(query->resultRelation, query->rtable); Assert(rte->rtekind == RTE_RELATION); if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } appendStringInfo(buf, "INSERT INTO %s", generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed; INSERT requires explicit AS */ get_rte_alias(rte, query->resultRelation, true, context); /* always want a space here */ appendStringInfoChar(buf, ' '); /* * Add the insert-column-names list. Any indirection decoration needed on * the column names can be inferred from the top targetlist. */ if (query->targetList) { strippedexprs = get_insert_column_names_list(query->targetList, buf, context, rte); } if (query->override) { if (query->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); else if (query->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, "OVERRIDING USER VALUE "); } if (select_rte) { /* Add the SELECT */ get_query_def(select_rte->subquery, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); } else if (values_rte) { /* Add the multi-VALUES expression lists */ get_values_def(values_rte->values_lists, context); } else if (strippedexprs) { /* Add the single-VALUES expression list */ appendContextKeyword(context, "VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else { /* No expressions, so it must be DEFAULT VALUES */ appendStringInfoString(buf, "DEFAULT VALUES"); } /* Add ON CONFLICT if present */ if (query->onConflict) { OnConflictExpr *confl = query->onConflict; appendStringInfoString(buf, " ON CONFLICT"); if (confl->arbiterElems) { /* Add the single-VALUES expression list */ appendStringInfoChar(buf, '('); get_rule_expr((Node *) confl->arbiterElems, context, false); appendStringInfoChar(buf, ')'); /* Add a WHERE clause (for partial indexes) if given */ if (confl->arbiterWhere != NULL) { bool save_varprefix; /* * Force non-prefixing of Vars, since parser assumes that they * belong to target relation. WHERE clause does not use * InferenceElem, so this is separately required. */ save_varprefix = context->varprefix; context->varprefix = false; appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->arbiterWhere, context, false); context->varprefix = save_varprefix; } } else if (OidIsValid(confl->constraint)) { char *constraint = get_constraint_name(confl->constraint); int64 shardId = context->shardid; if (shardId > 0) { AppendShardIdToName(&constraint, shardId); } if (!constraint) elog(ERROR, "cache lookup failed for constraint %u", confl->constraint); appendStringInfo(buf, " ON CONSTRAINT %s", quote_identifier(constraint)); } if (confl->action == ONCONFLICT_NOTHING) { appendStringInfoString(buf, " DO NOTHING"); } else { appendStringInfoString(buf, " DO UPDATE SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, confl->onConflictSet, context, rte); /* Add a WHERE clause if given */ if (confl->onConflictWhere != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->onConflictWhere, context, false); } } } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_update_query_def - Parse back an UPDATE parsetree * ---------- */ static void get_update_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with UPDATE relname SET */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } appendStringInfoString(buf, " SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, query->targetList, context, rte); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_update_query_targetlist_def - Parse back an UPDATE targetlist * ---------- */ static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte) { StringInfo buf = context->buf; ListCell *l; ListCell *next_ma_cell; int remaining_ma_columns; const char *sep; SubLink *cur_ma_sublink; List *ma_sublinks; targetList = ExpandMergedSubscriptingRefEntries(targetList); /* * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks * into a list. We expect them to appear, in ID order, in resjunk tlist * entries. */ ma_sublinks = NIL; if (query->hasSubLinks) /* else there can't be any */ { foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk && IsA(tle->expr, SubLink)) { SubLink *sl = (SubLink *) tle->expr; if (sl->subLinkType == MULTIEXPR_SUBLINK) { ma_sublinks = lappend(ma_sublinks, sl); Assert(sl->subLinkId == list_length(ma_sublinks)); } } } ensure_update_targetlist_in_param_order(targetList); } next_ma_cell = list_head(ma_sublinks); cur_ma_sublink = NULL; remaining_ma_columns = 0; /* Add the comma separated list of 'attname = value' */ sep = ""; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *expr; if (tle->resjunk) continue; /* ignore junk entries */ /* Emit separator (OK whether we're in multiassignment or not) */ appendStringInfoString(buf, sep); sep = ", "; /* * Check to see if we're starting a multiassignment group: if so, * output a left paren. */ if (next_ma_cell != NULL && cur_ma_sublink == NULL) { /* * We must dig down into the expr to see if it's a PARAM_MULTIEXPR * Param. That could be buried under FieldStores and * SubscriptingRefs and CoerceToDomains (cf processIndirection()), * and underneath those there could be an implicit type coercion. * Because we would ignore implicit type coercions anyway, we * don't need to be as careful as processIndirection() is about * descending past implicit CoerceToDomains. */ expr = (Node *) tle->expr; while (expr) { if (IsA(expr, FieldStore)) { FieldStore *fstore = (FieldStore *) expr; expr = (Node *) linitial(fstore->newvals); } else if (IsA(expr, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) expr; if (sbsref->refassgnexpr == NULL) break; expr = (Node *) sbsref->refassgnexpr; } else if (IsA(expr, CoerceToDomain)) { CoerceToDomain *cdomain = (CoerceToDomain *) expr; if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; expr = (Node *) cdomain->arg; } else break; } expr = strip_implicit_coercions(expr); if (expr && IsA(expr, Param) && ((Param *) expr)->paramkind == PARAM_MULTIEXPR) { cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); next_ma_cell = lnext(ma_sublinks, next_ma_cell); remaining_ma_columns = count_nonjunk_tlist_entries( ((Query *) cur_ma_sublink->subselect)->targetList); Assert(((Param *) expr)->paramid == ((cur_ma_sublink->subLinkId << 16) | 1)); appendStringInfoChar(buf, '('); } } /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. */ expr = processIndirection((Node *) tle->expr, context); /* * If we're in a multiassignment, skip printing anything more, unless * this is the last column; in which case, what we print should be the * sublink, not the Param. */ if (cur_ma_sublink != NULL) { if (--remaining_ma_columns > 0) continue; /* not the last column of multiassignment */ appendStringInfoChar(buf, ')'); expr = (Node *) cur_ma_sublink; cur_ma_sublink = NULL; } appendStringInfoString(buf, " = "); get_rule_expr(expr, context, false); } } /* ---------- * get_delete_query_def - Parse back a DELETE parsetree * ---------- */ static void get_delete_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with DELETE FROM relname */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } /* Add the USING clause if given */ get_from_clause(query, " USING ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } } /* ---------- * get_merge_query_def - Parse back a MERGE parsetree * ---------- */ static void get_merge_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; ListCell *lc; bool haveNotMatchedBySource; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with MERGE INTO relname */ rte = ExtractResultRelationRTE(query); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); if (rte->alias != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } /* Print the source relation and join clause */ get_from_clause(query, " USING ", context); appendContextKeyword(context, " ON ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_expr(query->mergeJoinCondition, context, false); /* * Test for any NOT MATCHED BY SOURCE actions. If there are none, then * any NOT MATCHED BY TARGET actions are output as "WHEN NOT MATCHED", per * SQL standard. Otherwise, we have a non-SQL-standard query, so output * "BY SOURCE" / "BY TARGET" qualifiers for all NOT MATCHED actions, to be * more explicit. */ haveNotMatchedBySource = false; foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); if (action->matchKind == MERGE_WHEN_NOT_MATCHED_BY_SOURCE) { haveNotMatchedBySource = true; break; } } /* Print each merge action */ foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); appendContextKeyword(context, " WHEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); switch (action->matchKind) { case MERGE_WHEN_MATCHED: appendStringInfoString(buf, "MATCHED"); break; case MERGE_WHEN_NOT_MATCHED_BY_SOURCE: appendStringInfoString(buf, "NOT MATCHED BY SOURCE"); break; case MERGE_WHEN_NOT_MATCHED_BY_TARGET: if (haveNotMatchedBySource) appendStringInfoString(buf, "NOT MATCHED BY TARGET"); else appendStringInfoString(buf, "NOT MATCHED"); break; default: elog(ERROR, "unrecognized matchKind: %d", (int) action->matchKind); } if (action->qual) { appendContextKeyword(context, " AND ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); get_rule_expr(action->qual, context, false); } appendContextKeyword(context, " THEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); if (action->commandType == CMD_INSERT) { /* This generally matches get_insert_query_def() */ List *strippedexprs = NIL; const char *sep = ""; ListCell *lc2; appendStringInfoString(buf, "INSERT"); if (action->targetList) appendStringInfoString(buf, " ("); foreach(lc2, action->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc2); Assert(!tle->resjunk); appendStringInfoString(buf, sep); sep = ", "; appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } if (action->targetList) appendStringInfoChar(buf, ')'); if (action->override) { if (action->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, " OVERRIDING SYSTEM VALUE"); else if (action->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, " OVERRIDING USER VALUE"); } if (strippedexprs) { appendContextKeyword(context, " VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 4); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DEFAULT VALUES"); } else if (action->commandType == CMD_UPDATE) { appendStringInfoString(buf, "UPDATE SET "); get_update_query_targetlist_def(query, action->targetList, context, rte); } else if (action->commandType == CMD_DELETE) appendStringInfoString(buf, "DELETE"); else if (action->commandType == CMD_NOTHING) appendStringInfoString(buf, "DO NOTHING"); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } ereport(DEBUG1, (errmsg("", buf->data))); } /* ---------- * get_utility_query_def - Parse back a UTILITY parsetree * ---------- */ static void get_utility_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) { NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "NOTIFY %s", quote_identifier(stmt->conditionname)); if (stmt->payload) { appendStringInfoString(buf, ", "); simple_quote_literal(buf, stmt->payload); } } else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) { TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; List *relationList = stmt->relations; ListCell *relationCell = NULL; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "TRUNCATE TABLE"); foreach(relationCell, relationList) { RangeVar *relationVar = (RangeVar *) lfirst(relationCell); Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); char *relationName = generate_relation_or_shard_name(relationId, context->distrelid, context->shardid, NIL); appendStringInfo(buf, " %s", relationName); if (lnext(relationList, relationCell) != NULL) { appendStringInfo(buf, ","); } } if (stmt->restart_seqs) { appendStringInfo(buf, " RESTART IDENTITY"); } if (stmt->behavior == DROP_CASCADE) { appendStringInfo(buf, " CASCADE"); } } else { /* Currently only NOTIFY utility commands can appear in rules */ elog(ERROR, "unexpected utility statement type"); } } /* * Display a Var appropriately. * * In some cases (currently only when recursing into an unnamed join) * the Var's varlevelsup has to be interpreted with respect to a context * above the current one; levelsup indicates the offset. * * If istoplevel is true, the Var is at the top level of a SELECT's * targetlist, which means we need special treatment of whole-row Vars. * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a * dirty hack to prevent "tab.*" from being expanded into multiple columns. * (The parser will strip the useless coercion, so no inefficiency is added in * dump and reload.) We used to print just "tab" in such cases, but that is * ambiguous and will yield the wrong result if "tab" is also a plain column * name in the query. * * Returns the attname of the Var, or NULL if the Var has no attname (because * it is a whole-row Var or a subplan output reference). */ static char * get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; AttrNumber attnum; int varno; AttrNumber varattno; int netlevelsup; deparse_namespace *dpns; deparse_columns *colinfo; char *refname; char *attname; bool need_prefix; /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. Also * find the aliases previously assigned for this RTE. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { /* * We might have been asked to map child Vars to some parent relation. */ if (context->appendparents && dpns->appendrels) { int pvarno = varno; AttrNumber pvarattno = varattno; AppendRelInfo *appinfo = dpns->appendrels[pvarno]; bool found = false; /* Only map up to inheritance parents, not UNION ALL appendrels */ while (appinfo && rt_fetch(appinfo->parent_relid, dpns->rtable)->rtekind == RTE_RELATION) { found = false; if (pvarattno > 0) /* system columns stay as-is */ { if (pvarattno > appinfo->num_child_cols) break; /* safety check */ pvarattno = appinfo->parent_colnos[pvarattno - 1]; if (pvarattno == 0) break; /* Var is local to child */ } pvarno = appinfo->parent_relid; found = true; /* If the parent is itself a child, continue up. */ Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); appinfo = dpns->appendrels[pvarno]; } /* * If we found an ancestral rel, and that rel is included in * appendparents, print that column not the original one. */ if (found && bms_is_member(pvarno, context->appendparents)) { varno = pvarno; varattno = pvarattno; } } rte = rt_fetch(varno, dpns->rtable); refname = (char *) list_nth(dpns->rtable_names, varno - 1); colinfo = deparse_columns_fetch(varno, dpns); attnum = varattno; } else { resolve_special_varno((Node *) var, context, get_special_variable, NULL); return NULL; } /* * The planner will sometimes emit Vars referencing resjunk elements of a * subquery's target list (this is currently only possible if it chooses * to generate a "physical tlist" for a SubqueryScan or CteScan node). * Although we prefer to print subquery-referencing Vars using the * subquery's alias, that's not possible for resjunk items since they have * no alias. So in that case, drill down to the subplan and print the * contents of the referenced tlist item. This works because in a plan * tree, such Vars can only occur in a SubqueryScan or CteScan node, and * we'll have set dpns->inner_plan to reference the child plan node. */ if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && attnum > list_length(rte->eref->colnames) && dpns->inner_plan) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); /* * Force parentheses because our caller probably assumed a Var is a * simple expression. */ if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, '('); get_rule_expr((Node *) tle->expr, context, true); if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, ')'); pop_child_plan(dpns, &save_dpns); return NULL; } /* * If it's an unnamed join, look at the expansion of the alias variable. * If it's a simple reference to one of the input vars, then recursively * print the name of that var instead. When it's not a simple reference, * we have to just print the unqualified join column name. (This can only * happen with "dangerous" merged columns in a JOIN USING; we took pains * previously to make the unqualified column name unique in such cases.) * * This wouldn't work in decompiling plan trees, because we don't store * joinaliasvars lists after planning; but a plan tree should never * contain a join alias variable. */ if (rte->rtekind == RTE_JOIN && rte->alias == NULL) { if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); if (attnum > 0) { Var *aliasvar; aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); /* we intentionally don't strip implicit coercions here */ if (aliasvar && IsA(aliasvar, Var)) { return get_variable(aliasvar, var->varlevelsup + levelsup, istoplevel, context); } } /* * Unnamed join has no refname. (Note: since it's unnamed, there is * no way the user could have referenced it to create a whole-row Var * for it. So we don't have to cover that case below.) */ Assert(refname == NULL); } if (attnum == InvalidAttrNumber) attname = NULL; else if (attnum > 0) { /* Get column name to use from the colinfo struct */ if (attnum > colinfo->num_cols) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); attname = colinfo->colnames[attnum - 1]; /* * If we find a Var referencing a dropped column, it seems better to * print something (anything) than to fail. In general this should * not happen, but it used to be possible for some cases involving * functions returning named composite types, and perhaps there are * still bugs out there. */ if (attname == NULL) attname = "?dropped?column?"; } else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* System column on a Citus shard */ attname = get_attname(rte->relid, attnum, false); } else { /* System column - name is fixed, get it from the catalog */ attname = get_rte_attribute_name(rte, attnum); } need_prefix = (context->varprefix || attname == NULL); /* * If we're considering a plain Var in an ORDER BY (but not GROUP BY) * clause, we may need to add a table-name prefix to prevent * findTargetlistEntrySQL92 from misinterpreting the name as an * output-column name. To avoid cluttering the output with unnecessary * prefixes, do so only if there is a name match to a SELECT tlist item * that is different from the Var. */ if (context->varInOrderBy && !context->inGroupBy && !need_prefix) { int colno = 0; foreach_node(TargetEntry, tle, context->targetList) { char *colname; if (tle->resjunk) continue; /* ignore junk entries */ colno++; /* This must match colname-choosing logic in get_target_list() */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; if (colname && strcmp(colname, attname) == 0 && !equal(var, tle->expr)) { need_prefix = true; break; } } } if (refname && need_prefix) { appendStringInfoString(buf, quote_identifier(refname)); appendStringInfoChar(buf, '.'); } if (attname) appendStringInfoString(buf, quote_identifier(attname)); else { appendStringInfoChar(buf, '*'); if (istoplevel) { if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* use rel.*::shard_name instead of rel.*::table_name */ appendStringInfo(buf, "::%s", generate_rte_shard_name(rte)); } else { appendStringInfo(buf, "::%s", format_type_with_typemod(var->vartype, var->vartypmod)); } } } return attname; } /* * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This * routine is actually a callback for get_special_varno, which handles finding * the correct TargetEntry. We get the expression contained in that * TargetEntry and just need to deparse it, a job we can throw back on * get_rule_expr. */ static void get_special_variable(Node *node, deparse_context *context, void *callback_arg) { StringInfo buf = context->buf; /* * For a non-Var referent, force parentheses because our caller probably * assumed a Var is a simple expression. */ if (!IsA(node, Var)) appendStringInfoChar(buf, '('); get_rule_expr(node, context, true); if (!IsA(node, Var)) appendStringInfoChar(buf, ')'); } /* * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, * invoke the callback provided. */ static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) { Var *var; deparse_namespace *dpns; /* This function is recursive, so let's be paranoid. */ check_stack_depth(); /* If it's not a Var, invoke the callback. */ if (!IsA(node, Var)) { (*callback) (node, context, callback_arg); return; } /* Find appropriate nesting depth */ var = (Var *) node; dpns = (deparse_namespace *) list_nth(context->namespaces, var->varlevelsup); /* * It's a special RTE, so recurse. */ if (var->varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; Bitmapset *save_appendparents; tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); /* If we're descending to the first child of an Append or MergeAppend, * update appendparents. This will affect deparsing of all Vars * appearing within the eventually-resolved subexpression. */ save_appendparents = context->appendparents; if (IsA(dpns->plan, Append)) context->appendparents = bms_union(context->appendparents, ((Append *) dpns->plan)->apprelids); else if (IsA(dpns->plan, MergeAppend)) context->appendparents = bms_union(context->appendparents, ((MergeAppend *) dpns->plan)->apprelids); push_child_plan(dpns, dpns->outer_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); context->appendparents = save_appendparents; return; } else if (var->varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); push_child_plan(dpns, dpns->inner_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); return; } else if (var->varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; tle = get_tle_by_resno(dpns->index_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); return; } else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) elog(ERROR, "bogus varno: %d", var->varno); /* Not special. Just invoke the callback. */ (*callback) (node, context, callback_arg); } /* * Get the name of a field of an expression of composite type. The * expression is usually a Var, but we handle other cases too. * * levelsup is an extra offset to interpret the Var's varlevelsup correctly. * * This is fairly straightforward when the expression has a named composite * type; we need only look up the type in the catalogs. However, the type * could also be RECORD. Since no actual table or view column is allowed to * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE * or to a subquery output. We drill down to find the ultimate defining * expression and attempt to infer the field name from it. We ereport if we * can't determine the name. * * Similarly, a PARAM of type RECORD has to refer to some expression of * a determinable composite type. */ static const char * get_name_for_var_field(Var *var, int fieldno, int levelsup, deparse_context *context) { RangeTblEntry *rte; AttrNumber attnum; int netlevelsup; deparse_namespace *dpns; int varno; AttrNumber varattno; TupleDesc tupleDesc; Node *expr; /* * If it's a RowExpr that was expanded from a whole-row Var, use the * column names attached to it. (We could let get_expr_result_tupdesc() * handle this, but it's much cheaper to just pull out the name we need.) */ if (IsA(var, RowExpr)) { RowExpr *r = (RowExpr *) var; if (fieldno > 0 && fieldno <= list_length(r->colnames)) return strVal(list_nth(r->colnames, fieldno - 1)); } /* * If it's a Param of type RECORD, try to find what the Param refers to. */ if (IsA(var, Param)) { Param *param = (Param *) var; ListCell *ancestor_cell; expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so recurse to decipher the field name */ deparse_namespace save_dpns; const char *result; push_ancestor_plan(dpns, ancestor_cell, &save_dpns); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); pop_ancestor_plan(dpns, &save_dpns); return result; } } /* * If it's a Var of type RECORD, we have to find what the Var refers to; * if not, we can use get_expr_result_tupdesc(). */ if (!IsA(var, Var) || var->vartype != RECORDOID) { tupleDesc = get_expr_result_tupdesc((Node *) var, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { rte = rt_fetch(varno, dpns->rtable); attnum = varattno; } else if (varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->outer_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->outer_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->inner_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; const char *result; tle = get_tle_by_resno(dpns->index_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); Assert(netlevelsup == 0); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); return result; } else { elog(ERROR, "bogus varno: %d", varno); return NULL; /* keep compiler quiet */ } if (attnum == InvalidAttrNumber) { /* Var is whole-row reference to RTE, so select the right field */ return get_rte_attribute_name(rte, fieldno); } /* * This part has essentially the same logic as the parser's * expandRecordVariable() function, but we are dealing with a different * representation of the input context, and we only need one field name * not a TupleDesc. Also, we need special cases for finding subquery and * CTE subplans when deparsing Plan trees. */ expr = (Node *) var; /* default if we can't drill down */ switch (rte->rtekind) { case RTE_RELATION: case RTE_VALUES: case RTE_NAMEDTUPLESTORE: case RTE_RESULT: /* * This case should not occur: a column of a table or values list * shouldn't have type RECORD. Fall through and fail (most * likely) at the bottom. */ break; case RTE_SUBQUERY: /* Subselect-in-FROM: examine sub-select's output expr */ { if (rte->subquery) { TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "subquery %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the sub-select to see what its Var * refers to. We have to build an additional level of * namespace to keep in step with varlevelsup in the * subselect; furthermore, the subquery RTE might be * from an outer query level, in which case the * namespace for the subselect must have that outer * level as parent namespace. */ List *save_nslist = context->namespaces; List *parent_namespaces; deparse_namespace mydpns; const char *result; parent_namespaces = list_copy_tail(context->namespaces, netlevelsup); set_deparse_for_query(&mydpns, rte->subquery, parent_namespaces); context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = save_nslist; return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have complete * RTE entries (in particular, rte->subquery is NULL). But * the only place we'd normally see a Var directly * referencing a SUBQUERY RTE is in a SubqueryScan plan * node, and we can look into the child plan's tlist * instead. An exception occurs if the subquery was * proven empty and optimized away: then we'd find such a * Var in a childless Result node, and there's nothing in * the plan tree that would let us figure out what it had * originally referenced. In that case, fall back on * printing "fN", analogously to the default column names * for RowExprs. */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) { char *dummy_name = palloc(32); Assert(dpns->plan && IsA(dpns->plan, Result)); snprintf(dummy_name, 32, "f%d", fieldno); return dummy_name; } Assert(dpns->plan && IsA(dpns->plan, SubqueryScan)); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; case RTE_JOIN: /* Join RTE --- recursively inspect the alias variable */ if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); Assert(expr != NULL); /* we intentionally don't strip implicit coercions here */ if (IsA(expr, Var)) return get_name_for_var_field((Var *) expr, fieldno, var->varlevelsup + levelsup, context); /* else fall through to inspect the expression */ break; case RTE_FUNCTION: case RTE_TABLEFUNC: /* * We couldn't get here unless a function is declared with one of * its result columns as RECORD, which is not allowed. */ break; case RTE_CTE: /* CTE reference: examine subquery's output expr */ { CommonTableExpr *cte = NULL; Index ctelevelsup; ListCell *lc; /* * Try to find the referenced CTE using the namespace stack. */ ctelevelsup = rte->ctelevelsup + netlevelsup; if (ctelevelsup >= list_length(context->namespaces)) lc = NULL; else { deparse_namespace *ctedpns; ctedpns = (deparse_namespace *) list_nth(context->namespaces, ctelevelsup); foreach(lc, ctedpns->ctes) { cte = (CommonTableExpr *) lfirst(lc); if (strcmp(cte->ctename, rte->ctename) == 0) break; } } if (lc != NULL) { Query *ctequery = (Query *) cte->ctequery; TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "CTE %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the CTE to see what its Var refers to. * We have to build an additional level of namespace * to keep in step with varlevelsup in the CTE; * furthermore it could be an outer CTE (compare * SUBQUERY case above). */ List *save_nslist = context->namespaces; List *parent_namespaces; deparse_namespace mydpns; const char *result; parent_namespaces = list_copy_tail(context->namespaces, ctelevelsup); set_deparse_for_query(&mydpns, ctequery, parent_namespaces); context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = save_nslist; return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have a CTE * list. But the only places we'd normally see a Var * directly referencing a CTE RTE are in CteScan or * WorkTableScan plan nodes. For those cases, * set_deparse_plan arranged for dpns->inner_plan to be * the plan node that emits the CTE or RecursiveUnion * result, and we can look at its tlist instead. As * above, this can fail if the CTE has been proven empty, * in which case fall back to "fN". */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) { char *dummy_name = palloc(32); Assert(dpns->plan && IsA(dpns->plan, Result)); snprintf(dummy_name, 32, "f%d", fieldno); return dummy_name; } Assert(dpns->plan && (IsA(dpns->plan, CteScan) || IsA(dpns->plan, WorkTableScan))); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; } /* * We now have an expression we can't expand any more, so see if * get_expr_result_tupdesc() can do anything with it. */ tupleDesc = get_expr_result_tupdesc(expr, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* * Try to find the referenced expression for a PARAM_EXEC Param that might * reference a parameter supplied by an upper NestLoop or SubPlan plan node. * * If successful, return the expression and set *dpns_p and *ancestor_cell_p * appropriately for calling push_ancestor_plan(). If no referent can be * found, return NULL. */ static Node * find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p) { /* Initialize output parameters to prevent compiler warnings */ *dpns_p = NULL; *ancestor_cell_p = NULL; /* * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or * SubPlan argument. This will necessarily be in some ancestor of the * current expression's Plan. */ if (param->paramkind == PARAM_EXEC) { deparse_namespace *dpns; Plan *child_plan; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); child_plan = dpns->plan; foreach(lc, dpns->ancestors) { Node *ancestor = (Node *) lfirst(lc); ListCell *lc2; /* * NestLoops transmit params to their inner child only. */ if (IsA(ancestor, NestLoop) && child_plan == innerPlan(ancestor)) { NestLoop *nl = (NestLoop *) ancestor; foreach(lc2, nl->nestParams) { NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); if (nlp->paramno == param->paramid) { /* Found a match, so return it */ *dpns_p = dpns; *ancestor_cell_p = lc; return (Node *) nlp->paramval; } } } /* * Check to see if we're crawling up from a subplan. */ if(IsA(ancestor, SubPlan)) { SubPlan *subplan = (SubPlan *) ancestor; ListCell *lc3; ListCell *lc4; /* Matched subplan, so check its arguments */ forboth(lc3, subplan->parParam, lc4, subplan->args) { int paramid = lfirst_int(lc3); Node *arg = (Node *) lfirst(lc4); if (paramid == param->paramid) { /* * Found a match, so return it. But, since Vars in * the arg are to be evaluated in the surrounding * context, we have to point to the next ancestor item * that is *not* a SubPlan. */ ListCell *rest; for_each_cell(rest, dpns->ancestors, lnext(dpns->ancestors, lc)) { Node *ancestor2 = (Node *) lfirst(rest); if (!IsA(ancestor2, SubPlan)) { *dpns_p = dpns; *ancestor_cell_p = rest; return arg; } } elog(ERROR, "SubPlan cannot be outermost ancestor"); } } /* SubPlan isn't a kind of Plan, so skip the rest */ continue; } /* * We need not consider the ancestor's initPlan list, since * initplans never have any parParams. */ /* No luck, crawl up to next ancestor */ child_plan = (Plan *) ancestor; } } /* No referent found */ return NULL; } /* * Try to find a subplan/initplan that emits the value for a PARAM_EXEC Param. * * If successful, return the generating subplan/initplan and set *column_p * to the subplan's 0-based output column number. * Otherwise, return NULL. */ static SubPlan * find_param_generator(Param *param, deparse_context *context, int *column_p) { /* Initialize output parameter to prevent compiler warnings */ *column_p = 0; /* * If it's a PARAM_EXEC parameter, search the current plan node as well as * ancestor nodes looking for a subplan or initplan that emits the value * for the Param. It could appear in the setParams of an initplan or * MULTIEXPR_SUBLINK subplan, or in the paramIds of an ancestral SubPlan. */ if (param->paramkind == PARAM_EXEC) { SubPlan *result; deparse_namespace *dpns; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); /* First check the innermost plan node's initplans */ result = find_param_generator_initplan(param, dpns->plan, column_p); if (result) return result; /* * The plan's targetlist might contain MULTIEXPR_SUBLINK SubPlans, * which can be referenced by Params elsewhere in the targetlist. * (Such Params should always be in the same targetlist, so there's no * need to do this work at upper plan nodes.) */ foreach_node(TargetEntry, tle, dpns->plan->targetlist) { if (tle->expr && IsA(tle->expr, SubPlan)) { SubPlan *subplan = (SubPlan *) tle->expr; if (subplan->subLinkType == MULTIEXPR_SUBLINK) { foreach_int(paramid, subplan->setParam) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } } } } /* No luck, so check the ancestor nodes */ foreach(lc, dpns->ancestors) { Node *ancestor = (Node *) lfirst(lc); /* * If ancestor is a SubPlan, check the paramIds it provides. */ if (IsA(ancestor, SubPlan)) { SubPlan *subplan = (SubPlan *) ancestor; foreach_int(paramid, subplan->paramIds) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } /* SubPlan isn't a kind of Plan, so skip the rest */ continue; } /* * Otherwise, it's some kind of Plan node, so check its initplans. */ result = find_param_generator_initplan(param, (Plan *) ancestor, column_p); if (result) return result; /* No luck, crawl up to next ancestor */ } } /* No generator found */ return NULL; } /* * Subroutine for find_param_generator: search one Plan node's initplans */ static SubPlan * find_param_generator_initplan(Param *param, Plan *plan, int *column_p) { foreach_node(SubPlan, subplan, plan->initPlan) { foreach_int(paramid, subplan->setParam) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } } return NULL; } /* * Display a Param appropriately. */ static void get_parameter(Param *param, deparse_context *context) { Node *expr; deparse_namespace *dpns; ListCell *ancestor_cell; SubPlan *subplan; int column; /* * If it's a PARAM_EXEC parameter, try to locate the expression from which * the parameter was computed. This stanza handles only cases in which * the Param represents an input to the subplan we are currently in. */ expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so print it */ deparse_namespace save_dpns; bool save_varprefix; bool need_paren; /* Switch attention to the ancestor plan node */ push_ancestor_plan(dpns, ancestor_cell, &save_dpns); /* * Force prefixing of Vars, since they won't belong to the relation * being scanned in the original plan node. */ save_varprefix = context->varprefix; context->varprefix = true; /* * A Param's expansion is typically a Var, Aggref, GroupingFunc, or * upper-level Param, which wouldn't need extra parentheses. * Otherwise, insert parens to ensure the expression looks atomic. */ need_paren = !(IsA(expr, Var) || IsA(expr, Aggref) || IsA(expr, GroupingFunc) || IsA(expr, Param)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, false); if (need_paren) appendStringInfoChar(context->buf, ')'); context->varprefix = save_varprefix; pop_ancestor_plan(dpns, &save_dpns); return; } /* * Alternatively, maybe it's a subplan output, which we print as a * reference to the subplan. (We could drill down into the subplan and * print the relevant targetlist expression, but that has been deemed too * confusing since it would violate normal SQL scope rules. Also, we're * relying on this reference to show that the testexpr containing the * Param has anything to do with that subplan at all.) */ subplan = find_param_generator(param, context, &column); if (subplan) { appendStringInfo(context->buf, "(%s%s).col%d", subplan->useHashTable ? "hashed " : "", subplan->plan_name, column + 1); return; } /* * If it's an external parameter, see if the outermost namespace provides * function argument names. */ if (param->paramkind == PARAM_EXTERN && context->namespaces != NIL) { dpns = llast(context->namespaces); if (dpns->argnames && param->paramid > 0 && param->paramid <= dpns->numargs) { char *argname = dpns->argnames[param->paramid - 1]; if (argname) { bool should_qualify = false; ListCell *lc; /* * Qualify the parameter name if there are any other deparse * namespaces with range tables. This avoids qualifying in * trivial cases like "RETURN a + b", but makes it safe in all * other cases. */ foreach(lc, context->namespaces) { deparse_namespace *depns = lfirst(lc); if (depns->rtable_names != NIL) { should_qualify = true; break; } } if (should_qualify) { appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); appendStringInfoChar(context->buf, '.'); } appendStringInfoString(context->buf, quote_identifier(argname)); return; } } } /* * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. * For composite types, add cast to the parameter to ease remote node detect * the type. * * It's a bug if we get here for anything except PARAM_EXTERN Params, but * in production builds printing $N seems more useful than failing. */ Assert(param->paramkind == PARAM_EXTERN); if (param->paramtype >= FirstNormalObjectId) { char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); } else { appendStringInfo(context->buf, "$%d", param->paramid); } } /* * get_simple_binary_op_name * * helper function for isSimpleNode * will return single char binary operator name, or NULL if it's not */ static const char * get_simple_binary_op_name(OpExpr *expr) { List *args = expr->args; if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); const char *op; op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); if (strlen(op) == 1) return op; } return NULL; } /* * isSimpleNode - check if given node is simple (doesn't need parenthesizing) * * true : simple in the context of parent node's type * false : not simple */ static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags) { if (!node) return false; switch (nodeTag(node)) { case T_Var: case T_Const: case T_Param: case T_CoerceToDomainValue: case T_SetToDefault: case T_CurrentOfExpr: /* single words: always simple */ return true; case T_SubscriptingRef: case T_ArrayExpr: case T_RowExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: case T_NextValueExpr: case T_NullIfExpr: case T_Aggref: case T_GroupingFunc: case T_WindowFunc: case T_MergeSupportFunc: case T_FuncExpr: case T_JsonConstructorExpr: case T_JsonExpr: /* function-like: name(..) or name[..] */ return true; /* CASE keywords act as parentheses */ case T_CaseExpr: return true; case T_FieldSelect: /* * appears simple since . has top precedence, unless parent is * T_FieldSelect itself! */ return !IsA(parentNode, FieldSelect); case T_FieldStore: /* * treat like FieldSelect (probably doesn't matter) */ return !IsA(parentNode, FieldStore); case T_CoerceToDomain: /* maybe simple, check args */ return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, node, prettyFlags); case T_RelabelType: return isSimpleNode((Node *) ((RelabelType *) node)->arg, node, prettyFlags); case T_CoerceViaIO: return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, node, prettyFlags); case T_ArrayCoerceExpr: return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, node, prettyFlags); case T_ConvertRowtypeExpr: return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, node, prettyFlags); case T_OpExpr: { /* depends on parent node type; needs further checking */ if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) { const char *op; const char *parentOp; bool is_lopriop; bool is_hipriop; bool is_lopriparent; bool is_hipriparent; op = get_simple_binary_op_name((OpExpr *) node); if (!op) return false; /* We know only the basic operators + - and * / % */ is_lopriop = (strchr("+-", *op) != NULL); is_hipriop = (strchr("*/%", *op) != NULL); if (!(is_lopriop || is_hipriop)) return false; parentOp = get_simple_binary_op_name((OpExpr *) parentNode); if (!parentOp) return false; is_lopriparent = (strchr("+-", *parentOp) != NULL); is_hipriparent = (strchr("*/%", *parentOp) != NULL); if (!(is_lopriparent || is_hipriparent)) return false; if (is_hipriop && is_lopriparent) return true; /* op binds tighter than parent */ if (is_lopriop && is_hipriparent) return false; /* * Operators are same priority --- can skip parens only if * we have (a - b) - c, not a - (b - c). */ if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) return true; return false; } /* else do the same stuff as for T_SubLink et al. */ } /* FALLTHROUGH */ case T_SubLink: case T_NullTest: case T_BooleanTest: case T_DistinctExpr: case T_JsonIsPredicate: switch (nodeTag(parentNode)) { case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_BoolExpr: /* lower precedence */ case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ return true; default: return false; } case T_BoolExpr: switch (nodeTag(parentNode)) { case T_BoolExpr: if (prettyFlags & PRETTYFLAG_PAREN) { BoolExprType type; BoolExprType parentType; type = ((BoolExpr *) node)->boolop; parentType = ((BoolExpr *) parentNode)->boolop; switch (type) { case NOT_EXPR: case AND_EXPR: if (parentType == AND_EXPR || parentType == OR_EXPR) return true; break; case OR_EXPR: if (parentType == OR_EXPR) return true; break; } } return false; case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ case T_JsonExpr: /* own parentheses */ return true; default: return false; } case T_JsonValueExpr: /* maybe simple, check args */ return isSimpleNode((Node *) ((JsonValueExpr *) node)->raw_expr, node, prettyFlags); default: break; } /* those we don't know: in dubio complexo */ return false; } /* * appendContextKeyword - append a keyword to buffer * * If prettyPrint is enabled, perform a line break, and adjust indentation. * Otherwise, just append the keyword. */ static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus) { StringInfo buf = context->buf; if (PRETTY_INDENT(context)) { int indentAmount; context->indentLevel += indentBefore; /* remove any trailing spaces currently in the buffer ... */ removeStringInfoSpaces(buf); /* ... then add a newline and some spaces */ appendStringInfoChar(buf, '\n'); if (context->indentLevel < PRETTYINDENT_LIMIT) indentAmount = Max(context->indentLevel, 0) + indentPlus; else { /* * If we're indented more than PRETTYINDENT_LIMIT characters, try * to conserve horizontal space by reducing the per-level * indentation. For best results the scale factor here should * divide all the indent amounts that get added to indentLevel * (PRETTYINDENT_STD, etc). It's important that the indentation * not grow unboundedly, else deeply-nested trees use O(N^2) * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. */ indentAmount = PRETTYINDENT_LIMIT + (context->indentLevel - PRETTYINDENT_LIMIT) / (PRETTYINDENT_STD / 2); indentAmount %= PRETTYINDENT_LIMIT; /* scale/wrap logic affects indentLevel, but not indentPlus */ indentAmount += indentPlus; } appendStringInfoSpaces(buf, indentAmount); appendStringInfoString(buf, str); context->indentLevel += indentAfter; if (context->indentLevel < 0) context->indentLevel = 0; } else appendStringInfoString(buf, str); } /* * removeStringInfoSpaces - delete trailing spaces from a buffer. * * Possibly this should move to stringinfo.c at some point. */ static void removeStringInfoSpaces(StringInfo str) { while (str->len > 0 && str->data[str->len - 1] == ' ') str->data[--(str->len)] = '\0'; } /* * get_rule_expr_paren - deparse expr using get_rule_expr, * embracing the string with parentheses if necessary for prettyPrint. * * Never embrace if prettyFlags=0, because it's done in the calling node. * * Any node that does *not* embrace its argument node by sql syntax (with * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should * use get_rule_expr_paren instead of get_rule_expr so parentheses can be * added. */ static void get_rule_expr_paren(Node *node, deparse_context *context, bool showimplicit, Node *parentNode) { bool need_paren; need_paren = PRETTY_PAREN(context) && !isSimpleNode(node, parentNode, context->prettyFlags); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(node, context, showimplicit); if (need_paren) appendStringInfoChar(context->buf, ')'); } static void get_json_behavior(JsonBehavior *behavior, deparse_context *context, const char *on) { /* * The order of array elements must correspond to the order of * JsonBehaviorType members. */ const char *behavior_names[] = { " NULL", " ERROR", " EMPTY", " TRUE", " FALSE", " UNKNOWN", " EMPTY ARRAY", " EMPTY OBJECT", " DEFAULT " }; if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names)) elog(ERROR, "invalid json behavior type: %d", behavior->btype); appendStringInfoString(context->buf, behavior_names[behavior->btype]); if (behavior->btype == JSON_BEHAVIOR_DEFAULT) get_rule_expr(behavior->expr, context, false); appendStringInfo(context->buf, " ON %s", on); } /* * get_json_expr_options * * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and * JSON_TABLE columns. */ static void get_json_expr_options(JsonExpr *jsexpr, deparse_context *context, JsonBehaviorType default_behavior) { if (jsexpr->op == JSON_QUERY_OP) { if (jsexpr->wrapper == JSW_CONDITIONAL) appendStringInfoString(context->buf, " WITH CONDITIONAL WRAPPER"); else if (jsexpr->wrapper == JSW_UNCONDITIONAL) appendStringInfoString(context->buf, " WITH UNCONDITIONAL WRAPPER"); /* The default */ else if (jsexpr->wrapper == JSW_NONE || jsexpr->wrapper == JSW_UNSPEC) appendStringInfoString(context->buf, " WITHOUT WRAPPER"); if (jsexpr->omit_quotes) appendStringInfoString(context->buf, " OMIT QUOTES"); /* The default */ else appendStringInfoString(context->buf, " KEEP QUOTES"); } if (jsexpr->on_empty && jsexpr->on_empty->btype != default_behavior) get_json_behavior(jsexpr->on_empty, context, "EMPTY"); if (jsexpr->on_error && jsexpr->on_error->btype != default_behavior) get_json_behavior(jsexpr->on_error, context, "ERROR"); } /* ---------- * get_rule_expr - Parse back an expression * * Note: showimplicit determines whether we display any implicit cast that * is present at the top of the expression tree. It is a passed argument, * not a field of the context struct, because we change the value as we * recurse down into the expression. In general we suppress implicit casts * when the result type is known with certainty (eg, the arguments of an * OR must be boolean). We display implicit casts for arguments of functions * and operators, since this is needed to be certain that the same function * or operator will be chosen when the expression is re-parsed. * ---------- */ static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; if (node == NULL) return; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); /* * Each level of get_rule_expr must emit an indivisible term * (parenthesized if necessary) to ensure result is reparsed into the same * expression tree. The only exception is that when the input is a List, * we emit the component items comma-separated with no surrounding * decoration; this is convenient for most callers. */ switch (nodeTag(node)) { case T_Var: (void) get_variable((Var *) node, 0, false, context); break; case T_Const: get_const_expr((Const *) node, context, 0); break; case T_Param: get_parameter((Param *) node, context); break; case T_Aggref: get_agg_expr((Aggref *) node, context, (Aggref *) node); break; case T_GroupingFunc: { GroupingFunc *gexpr = (GroupingFunc *) node; appendStringInfoString(buf, "GROUPING("); get_rule_expr((Node *) gexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_WindowFunc: get_windowfunc_expr((WindowFunc *) node, context); break; case T_MergeSupportFunc: appendStringInfoString(buf, "MERGE_ACTION()"); break; case T_SubscriptingRef: { SubscriptingRef *sbsref = (SubscriptingRef *) node; bool need_parens; /* * If the argument is a CaseTestExpr, we must be inside a * FieldStore, ie, we are assigning to an element of an array * within a composite column. Since we already punted on * displaying the FieldStore's target information, just punt * here too, and display only the assignment source * expression. */ if (IsA(sbsref->refexpr, CaseTestExpr)) { Assert(sbsref->refassgnexpr); get_rule_expr((Node *) sbsref->refassgnexpr, context, showimplicit); break; } /* * Parenthesize the argument unless it's a simple Var or a * FieldSelect. (In particular, if it's another * SubscriptingRef, we *must* parenthesize to avoid * confusion.) */ need_parens = !IsA(sbsref->refexpr, Var) && !IsA(sbsref->refexpr, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); /* * If there's a refassgnexpr, we want to print the node in the * format "container[subscripts] := refassgnexpr". This is * not legal SQL, so decompilation of INSERT or UPDATE * statements should always use processIndirection as part of * the statement-level syntax. We should only see this when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. */ if (sbsref->refassgnexpr) { Node *refassgnexpr; /* * Use processIndirection to print this node's subscripts * as well as any additional field selections or * subscripting in immediate descendants. It returns the * RHS expr that is actually being "assigned". */ refassgnexpr = processIndirection(node, context); appendStringInfoString(buf, " := "); get_rule_expr(refassgnexpr, context, showimplicit); } else { /* Just an ordinary container fetch, so print subscripts */ printSubscripts(sbsref, context); } } break; case T_FuncExpr: get_func_expr((FuncExpr *) node, context, showimplicit); break; case T_NamedArgExpr: { NamedArgExpr *na = (NamedArgExpr *) node; appendStringInfo(buf, "%s => ", quote_identifier(na->name)); get_rule_expr((Node *) na->arg, context, showimplicit); } break; case T_OpExpr: get_oper_expr((OpExpr *) node, context); break; case T_DistinctExpr: { DistinctExpr *expr = (DistinctExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfoString(buf, " IS DISTINCT FROM "); get_rule_expr_paren(arg2, context, true, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_NullIfExpr: { NullIfExpr *nullifexpr = (NullIfExpr *) node; appendStringInfoString(buf, "NULLIF("); get_rule_expr((Node *) nullifexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfo(buf, " %s %s (", generate_operator_name(expr->opno, exprType(arg1), get_base_element_type(exprType(arg2))), expr->useOr ? "ANY" : "ALL"); get_rule_expr_paren(arg2, context, true, node); /* * There's inherent ambiguity in "x op ANY/ALL (y)" when y is * a bare sub-SELECT. Since we're here, the sub-SELECT must * be meant as a scalar sub-SELECT yielding an array value to * be used in ScalarArrayOpExpr; but the grammar will * preferentially interpret such a construct as an ANY/ALL * SubLink. To prevent misparsing the output that way, insert * a dummy coercion (which will be stripped by parse analysis, * so no inefficiency is added in dump and reload). This is * indeed most likely what the user wrote to get the construct * accepted in the first place. */ if (IsA(arg2, SubLink) && ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) appendStringInfo(buf, "::%s", format_type_with_typemod(exprType(arg2), exprTypmod(arg2))); appendStringInfoChar(buf, ')'); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BoolExpr: { BoolExpr *expr = (BoolExpr *) node; Node *first_arg = linitial(expr->args); ListCell *arg; switch (expr->boolop) { case AND_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " AND "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case OR_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " OR "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case NOT_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); appendStringInfoString(buf, "NOT "); get_rule_expr_paren(first_arg, context, false, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; default: elog(ERROR, "unrecognized boolop: %d", (int) expr->boolop); } } break; case T_SubLink: get_sublink_expr((SubLink *) node, context); break; case T_SubPlan: { SubPlan *subplan = (SubPlan *) node; /* * We cannot see an already-planned subplan in rule deparsing, * only while EXPLAINing a query plan. We don't try to * reconstruct the original SQL, just reference the subplan * that appears elsewhere in EXPLAIN's result. It does seem * useful to show the subLinkType and testexpr (if any), and * we also note whether the subplan will be hashed. */ switch (subplan->subLinkType) { case EXISTS_SUBLINK: appendStringInfoString(buf, "EXISTS("); Assert(subplan->testexpr == NULL); break; case ALL_SUBLINK: appendStringInfoString(buf, "(ALL "); Assert(subplan->testexpr != NULL); break; case ANY_SUBLINK: appendStringInfoString(buf, "(ANY "); Assert(subplan->testexpr != NULL); break; case ROWCOMPARE_SUBLINK: /* Parenthesizing the testexpr seems sufficient */ appendStringInfoChar(buf, '('); Assert(subplan->testexpr != NULL); break; case EXPR_SUBLINK: /* No need to decorate these subplan references */ appendStringInfoChar(buf, '('); Assert(subplan->testexpr == NULL); break; case MULTIEXPR_SUBLINK: /* MULTIEXPR isn't executed in the normal way */ appendStringInfoString(buf, "(rescan "); Assert(subplan->testexpr == NULL); break; case ARRAY_SUBLINK: appendStringInfoString(buf, "ARRAY("); Assert(subplan->testexpr == NULL); break; case CTE_SUBLINK: /* This case is unreachable within expressions */ appendStringInfoString(buf, "CTE("); Assert(subplan->testexpr == NULL); break; } if (subplan->testexpr != NULL) { deparse_namespace *dpns; /* * Push SubPlan into ancestors list while deparsing * testexpr, so that we can handle PARAM_EXEC references * to the SubPlan's paramIds. (This makes it look like * the SubPlan is an "ancestor" of the current plan node, * which is a little weird, but it does no harm.) In this * path, we don't need to mention the SubPlan explicitly, * because the referencing Params will show its existence. */ dpns = (deparse_namespace *) linitial(context->namespaces); dpns->ancestors = lcons(subplan, dpns->ancestors); get_rule_expr(subplan->testexpr, context, showimplicit); appendStringInfoChar(buf, ')'); dpns->ancestors = list_delete_first(dpns->ancestors); } else { /* No referencing Params, so show the SubPlan's name */ if (subplan->useHashTable) appendStringInfo(buf, "hashed %s)", subplan->plan_name); else appendStringInfo(buf, "%s)", subplan->plan_name); } } break; case T_AlternativeSubPlan: { AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; ListCell *lc; /* * This case cannot be reached in normal usage, since no * AlternativeSubPlan can appear either in parsetrees or * finished plan trees. We keep it just in case somebody * wants to use this code to print planner data structures. */ appendStringInfoString(buf, "(alternatives: "); foreach(lc, asplan->subplans) { SubPlan *splan = lfirst_node(SubPlan, lc); if (splan->useHashTable) appendStringInfo(buf, "hashed %s", splan->plan_name); else appendStringInfoString(buf, splan->plan_name); if (lnext(asplan->subplans, lc)) appendStringInfoString(buf, " or "); } appendStringInfoChar(buf, ')'); } break; case T_FieldSelect: { FieldSelect *fselect = (FieldSelect *) node; Node *arg = (Node *) fselect->arg; int fno = fselect->fieldnum; const char *fieldname; bool need_parens; /* * Parenthesize the argument unless it's an SubscriptingRef or * another FieldSelect. Note in particular that it would be * WRONG to not parenthesize a Var argument; simplicity is not * the issue here, having the right number of names is. */ need_parens = !IsA(arg, SubscriptingRef) && !IsA(arg, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr(arg, context, true); if (need_parens) appendStringInfoChar(buf, ')'); /* * Get and print the field name. */ fieldname = get_name_for_var_field((Var *) arg, fno, 0, context); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); } break; case T_FieldStore: { FieldStore *fstore = (FieldStore *) node; bool need_parens; /* * There is no good way to represent a FieldStore as real SQL, * so decompilation of INSERT or UPDATE statements should * always use processIndirection as part of the * statement-level syntax. We should only get here when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. The plan case is even harder than * ordinary rules would be, because the planner tries to * collapse multiple assignments to the same field or subfield * into one FieldStore; so we can see a list of target fields * not just one, and the arguments could be FieldStores * themselves. We don't bother to try to print the target * field names; we just print the source arguments, with a * ROW() around them if there's more than one. This isn't * terribly complete, but it's probably good enough for * EXPLAIN's purposes; especially since anything more would be * either hopelessly confusing or an even poorer * representation of what the plan is actually doing. */ need_parens = (list_length(fstore->newvals) != 1); if (need_parens) appendStringInfoString(buf, "ROW("); get_rule_expr((Node *) fstore->newvals, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); } break; case T_RelabelType: { RelabelType *relabel = (RelabelType *) node; Node *arg = (Node *) relabel->arg; if (relabel->relabelformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, relabel->resulttype, relabel->resulttypmod, node); } } break; case T_CoerceViaIO: { CoerceViaIO *iocoerce = (CoerceViaIO *) node; Node *arg = (Node *) iocoerce->arg; if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, iocoerce->resulttype, -1, node); } } break; case T_ArrayCoerceExpr: { ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; Node *arg = (Node *) acoerce->arg; if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, acoerce->resulttype, acoerce->resulttypmod, node); } } break; case T_ConvertRowtypeExpr: { ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; Node *arg = (Node *) convert->arg; if (convert->convertformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, convert->resulttype, -1, node); } } break; case T_CollateExpr: { CollateExpr *collate = (CollateExpr *) node; Node *arg = (Node *) collate->arg; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, showimplicit, node); appendStringInfo(buf, " COLLATE %s", generate_collation_name(collate->collOid)); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CaseExpr: { CaseExpr *caseexpr = (CaseExpr *) node; ListCell *temp; appendContextKeyword(context, "CASE", 0, PRETTYINDENT_VAR, 0); if (caseexpr->arg) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) caseexpr->arg, context, true); } foreach(temp, caseexpr->args) { CaseWhen *when = (CaseWhen *) lfirst(temp); Node *w = (Node *) when->expr; if (caseexpr->arg) { /* * The parser should have produced WHEN clauses of the * form "CaseTestExpr = RHS", possibly with an * implicit coercion inserted above the CaseTestExpr. * For accurate decompilation of rules it's essential * that we show just the RHS. However in an * expression that's been through the optimizer, the * WHEN clause could be almost anything (since the * equality operator could have been expanded into an * inline function). If we don't recognize the form * of the WHEN clause, just punt and display it as-is. */ if (IsA(w, OpExpr)) { List *args = ((OpExpr *) w)->args; if (list_length(args) == 2 && IsA(strip_implicit_coercions(linitial(args)), CaseTestExpr)) w = (Node *) lsecond(args); } } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "WHEN ", 0, 0, 0); get_rule_expr(w, context, false); appendStringInfoString(buf, " THEN "); get_rule_expr((Node *) when->result, context, true); } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "ELSE ", 0, 0, 0); get_rule_expr((Node *) caseexpr->defresult, context, true); if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "END", -PRETTYINDENT_VAR, 0, 0); } break; case T_CaseTestExpr: { /* * Normally we should never get here, since for expressions * that can contain this node type we attempt to avoid * recursing to it. But in an optimized expression we might * be unable to avoid that (see comments for CaseExpr). If we * do see one, print it as CASE_TEST_EXPR. */ appendStringInfoString(buf, "CASE_TEST_EXPR"); } break; case T_ArrayExpr: { ArrayExpr *arrayexpr = (ArrayExpr *) node; appendStringInfoString(buf, "ARRAY["); get_rule_expr((Node *) arrayexpr->elements, context, true); appendStringInfoChar(buf, ']'); /* * If the array isn't empty, we assume its elements are * coerced to the desired type. If it's empty, though, we * need an explicit coercion to the array type. */ if (arrayexpr->elements == NIL) appendStringInfo(buf, "::%s", format_type_with_typemod(arrayexpr->array_typeid, -1)); } break; case T_RowExpr: { RowExpr *rowexpr = (RowExpr *) node; TupleDesc tupdesc = NULL; ListCell *arg; int i; char *sep; /* * If it's a named type and not RECORD, we may have to skip * dropped columns and/or claim there are NULLs for added * columns. */ if (rowexpr->row_typeid != RECORDOID) { tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); Assert(list_length(rowexpr->args) <= tupdesc->natts); } /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. */ appendStringInfoString(buf, "ROW("); sep = ""; i = 0; foreach(arg, rowexpr->args) { Node *e = (Node *) lfirst(arg); if (tupdesc == NULL || !TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); /* Whole-row Vars need special treatment here */ get_rule_expr_toplevel(e, context, true); sep = ", "; } i++; } if (tupdesc != NULL) { while (i < tupdesc->natts) { if (!TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); appendStringInfoString(buf, "NULL"); sep = ", "; } i++; } ReleaseTupleDesc(tupdesc); } appendStringInfoChar(buf, ')'); if (rowexpr->row_format == COERCE_EXPLICIT_CAST) appendStringInfo(buf, "::%s", format_type_with_typemod(rowexpr->row_typeid, -1)); } break; case T_RowCompareExpr: { RowCompareExpr *rcexpr = (RowCompareExpr *) node; /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. Within * a ROW expression, whole-row Vars need special treatment, so * use get_rule_list_toplevel. */ appendStringInfoString(buf, "(ROW("); get_rule_list_toplevel(rcexpr->largs, context, true); /* * We assume that the name of the first-column operator will * do for all the rest too. This is definitely open to * failure, eg if some but not all operators were renamed * since the construct was parsed, but there seems no way to * be perfect. */ appendStringInfo(buf, ") %s ROW(", generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs)))); get_rule_list_toplevel(rcexpr->rargs, context, true); appendStringInfoString(buf, "))"); } break; case T_CoalesceExpr: { CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; appendStringInfoString(buf, "COALESCE("); get_rule_expr((Node *) coalesceexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_MinMaxExpr: { MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; switch (minmaxexpr->op) { case IS_GREATEST: appendStringInfoString(buf, "GREATEST("); break; case IS_LEAST: appendStringInfoString(buf, "LEAST("); break; } get_rule_expr((Node *) minmaxexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_SQLValueFunction: { SQLValueFunction *svf = (SQLValueFunction *) node; /* * Note: this code knows that typmod for time, timestamp, and * timestamptz just prints as integer. */ switch (svf->op) { case SVFOP_CURRENT_DATE: appendStringInfoString(buf, "CURRENT_DATE"); break; case SVFOP_CURRENT_TIME: appendStringInfoString(buf, "CURRENT_TIME"); break; case SVFOP_CURRENT_TIME_N: appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); break; case SVFOP_CURRENT_TIMESTAMP: appendStringInfoString(buf, "CURRENT_TIMESTAMP"); break; case SVFOP_CURRENT_TIMESTAMP_N: appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", svf->typmod); break; case SVFOP_LOCALTIME: appendStringInfoString(buf, "LOCALTIME"); break; case SVFOP_LOCALTIME_N: appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); break; case SVFOP_LOCALTIMESTAMP: appendStringInfoString(buf, "LOCALTIMESTAMP"); break; case SVFOP_LOCALTIMESTAMP_N: appendStringInfo(buf, "LOCALTIMESTAMP(%d)", svf->typmod); break; case SVFOP_CURRENT_ROLE: appendStringInfoString(buf, "CURRENT_ROLE"); break; case SVFOP_CURRENT_USER: appendStringInfoString(buf, "CURRENT_USER"); break; case SVFOP_USER: appendStringInfoString(buf, "USER"); break; case SVFOP_SESSION_USER: appendStringInfoString(buf, "SESSION_USER"); break; case SVFOP_CURRENT_CATALOG: appendStringInfoString(buf, "CURRENT_CATALOG"); break; case SVFOP_CURRENT_SCHEMA: appendStringInfoString(buf, "CURRENT_SCHEMA"); break; } } break; case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; bool needcomma = false; ListCell *arg; ListCell *narg; Const *con; switch (xexpr->op) { case IS_XMLCONCAT: appendStringInfoString(buf, "XMLCONCAT("); break; case IS_XMLELEMENT: appendStringInfoString(buf, "XMLELEMENT("); break; case IS_XMLFOREST: appendStringInfoString(buf, "XMLFOREST("); break; case IS_XMLPARSE: appendStringInfoString(buf, "XMLPARSE("); break; case IS_XMLPI: appendStringInfoString(buf, "XMLPI("); break; case IS_XMLROOT: appendStringInfoString(buf, "XMLROOT("); break; case IS_XMLSERIALIZE: appendStringInfoString(buf, "XMLSERIALIZE("); break; case IS_DOCUMENT: break; } if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) { if (xexpr->xmloption == XMLOPTION_DOCUMENT) appendStringInfoString(buf, "DOCUMENT "); else appendStringInfoString(buf, "CONTENT "); } if (xexpr->name) { appendStringInfo(buf, "NAME %s", quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); needcomma = true; } if (xexpr->named_args) { if (xexpr->op != IS_XMLFOREST) { if (needcomma) appendStringInfoString(buf, ", "); appendStringInfoString(buf, "XMLATTRIBUTES("); needcomma = false; } forboth(arg, xexpr->named_args, narg, xexpr->arg_names) { Node *e = (Node *) lfirst(arg); char *argname = strVal(lfirst(narg)); if (needcomma) appendStringInfoString(buf, ", "); get_rule_expr((Node *) e, context, true); appendStringInfo(buf, " AS %s", quote_identifier(map_xml_name_to_sql_identifier(argname))); needcomma = true; } if (xexpr->op != IS_XMLFOREST) appendStringInfoChar(buf, ')'); } if (xexpr->args) { if (needcomma) appendStringInfoString(buf, ", "); switch (xexpr->op) { case IS_XMLCONCAT: case IS_XMLELEMENT: case IS_XMLFOREST: case IS_XMLPI: case IS_XMLSERIALIZE: /* no extra decoration needed */ get_rule_expr((Node *) xexpr->args, context, true); break; case IS_XMLPARSE: Assert(list_length(xexpr->args) == 2); get_rule_expr((Node *) linitial(xexpr->args), context, true); con = lsecond_node(Const, xexpr->args); Assert(!con->constisnull); if (DatumGetBool(con->constvalue)) appendStringInfoString(buf, " PRESERVE WHITESPACE"); else appendStringInfoString(buf, " STRIP WHITESPACE"); break; case IS_XMLROOT: Assert(list_length(xexpr->args) == 3); get_rule_expr((Node *) linitial(xexpr->args), context, true); appendStringInfoString(buf, ", VERSION "); con = (Const *) lsecond(xexpr->args); if (IsA(con, Const) && con->constisnull) appendStringInfoString(buf, "NO VALUE"); else get_rule_expr((Node *) con, context, false); con = lthird_node(Const, xexpr->args); if (con->constisnull) /* suppress STANDALONE NO VALUE */ ; else { switch (DatumGetInt32(con->constvalue)) { case XML_STANDALONE_YES: appendStringInfoString(buf, ", STANDALONE YES"); break; case XML_STANDALONE_NO: appendStringInfoString(buf, ", STANDALONE NO"); break; case XML_STANDALONE_NO_VALUE: appendStringInfoString(buf, ", STANDALONE NO VALUE"); break; default: break; } } break; case IS_DOCUMENT: get_rule_expr_paren((Node *) xexpr->args, context, false, node); break; } } if (xexpr->op == IS_XMLSERIALIZE) appendStringInfo(buf, " AS %s", format_type_with_typemod(xexpr->type, xexpr->typmod)); if (xexpr->op == IS_DOCUMENT) appendStringInfoString(buf, " IS DOCUMENT"); else appendStringInfoChar(buf, ')'); } break; case T_NullTest: { NullTest *ntest = (NullTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) ntest->arg, context, true, node); /* * For scalar inputs, we prefer to print as IS [NOT] NULL, * which is shorter and traditional. If it's a rowtype input * but we're applying a scalar test, must print IS [NOT] * DISTINCT FROM NULL to be semantically correct. */ if (ntest->argisrow || !type_is_rowtype(exprType((Node *) ntest->arg))) { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS NOT NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } else { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS DISTINCT FROM NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BooleanTest: { BooleanTest *btest = (BooleanTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) btest->arg, context, false, node); switch (btest->booltesttype) { case IS_TRUE: appendStringInfoString(buf, " IS TRUE"); break; case IS_NOT_TRUE: appendStringInfoString(buf, " IS NOT TRUE"); break; case IS_FALSE: appendStringInfoString(buf, " IS FALSE"); break; case IS_NOT_FALSE: appendStringInfoString(buf, " IS NOT FALSE"); break; case IS_UNKNOWN: appendStringInfoString(buf, " IS UNKNOWN"); break; case IS_NOT_UNKNOWN: appendStringInfoString(buf, " IS NOT UNKNOWN"); break; default: elog(ERROR, "unrecognized booltesttype: %d", (int) btest->booltesttype); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CoerceToDomain: { CoerceToDomain *ctest = (CoerceToDomain *) node; Node *arg = (Node *) ctest->arg; if (ctest->coercionformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr(arg, context, false); } else { get_coercion_expr(arg, context, ctest->resulttype, ctest->resulttypmod, node); } } break; case T_CoerceToDomainValue: appendStringInfoString(buf, "VALUE"); break; case T_SetToDefault: appendStringInfoString(buf, "DEFAULT"); break; case T_CurrentOfExpr: { CurrentOfExpr *cexpr = (CurrentOfExpr *) node; if (cexpr->cursor_name) appendStringInfo(buf, "CURRENT OF %s", quote_identifier(cexpr->cursor_name)); else appendStringInfo(buf, "CURRENT OF $%d", cexpr->cursor_param); } break; case T_NextValueExpr: { NextValueExpr *nvexpr = (NextValueExpr *) node; /* * This isn't exactly nextval(), but that seems close enough * for EXPLAIN's purposes. */ appendStringInfoString(buf, "nextval("); simple_quote_literal(buf, generate_relation_name(nvexpr->seqid, NIL)); appendStringInfoChar(buf, ')'); } break; case T_InferenceElem: { InferenceElem *iexpr = (InferenceElem *) node; bool save_varprefix; bool need_parens; /* * InferenceElem can only refer to target relation, so a * prefix is not useful, and indeed would cause parse errors. */ save_varprefix = context->varprefix; context->varprefix = false; /* * Parenthesize the element unless it's a simple Var or a bare * function call. Follows pg_get_indexdef_worker(). */ need_parens = !IsA(iexpr->expr, Var); if (IsA(iexpr->expr, FuncExpr) && ((FuncExpr *) iexpr->expr)->funcformat == COERCE_EXPLICIT_CALL) need_parens = false; if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) iexpr->expr, context, false); if (need_parens) appendStringInfoChar(buf, ')'); context->varprefix = save_varprefix; if (iexpr->infercollid) appendStringInfo(buf, " COLLATE %s", generate_collation_name(iexpr->infercollid)); /* Add the operator class name, if not default */ if (iexpr->inferopclass) { Oid inferopclass = iexpr->inferopclass; Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); get_opclass_name(inferopclass, inferopcinputtype, buf); } } break; case T_PartitionBoundSpec: { PartitionBoundSpec *spec = (PartitionBoundSpec *) node; ListCell *cell; char *sep; if (spec->is_default) { appendStringInfoString(buf, "DEFAULT"); break; } switch (spec->strategy) { case PARTITION_STRATEGY_HASH: Assert(spec->modulus > 0 && spec->remainder >= 0); Assert(spec->modulus > spec->remainder); appendStringInfoString(buf, "FOR VALUES"); appendStringInfo(buf, " WITH (modulus %d, remainder %d)", spec->modulus, spec->remainder); break; case PARTITION_STRATEGY_LIST: Assert(spec->listdatums != NIL); appendStringInfoString(buf, "FOR VALUES IN ("); sep = ""; foreach(cell, spec->listdatums) { Const *val = lfirst_node(Const, cell); appendStringInfoString(buf, sep); get_const_expr(val, context, -1); sep = ", "; } appendStringInfoChar(buf, ')'); break; case PARTITION_STRATEGY_RANGE: Assert(spec->lowerdatums != NIL && spec->upperdatums != NIL && list_length(spec->lowerdatums) == list_length(spec->upperdatums)); appendStringInfo(buf, "FOR VALUES FROM %s TO %s", get_range_partbound_string(spec->lowerdatums), get_range_partbound_string(spec->upperdatums)); break; default: elog(ERROR, "unrecognized partition strategy: %d", (int) spec->strategy); break; } } break; case T_JsonValueExpr: { JsonValueExpr *jve = (JsonValueExpr *) node; get_rule_expr((Node *) jve->raw_expr, context, false); get_json_format(jve->format, context->buf); } break; case T_JsonConstructorExpr: get_json_constructor((JsonConstructorExpr *) node, context, false); break; case T_JsonIsPredicate: { JsonIsPredicate *pred = (JsonIsPredicate *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, '('); get_rule_expr_paren(pred->expr, context, true, node); appendStringInfoString(context->buf, " IS JSON"); /* TODO: handle FORMAT clause */ switch (pred->item_type) { case JS_TYPE_SCALAR: appendStringInfoString(context->buf, " SCALAR"); break; case JS_TYPE_ARRAY: appendStringInfoString(context->buf, " ARRAY"); break; case JS_TYPE_OBJECT: appendStringInfoString(context->buf, " OBJECT"); break; default: break; } if (pred->unique_keys) appendStringInfoString(context->buf, " WITH UNIQUE KEYS"); if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, ')'); } break; case T_JsonExpr: { JsonExpr *jexpr = (JsonExpr *) node; switch (jexpr->op) { case JSON_EXISTS_OP: appendStringInfoString(buf, "JSON_EXISTS("); break; case JSON_QUERY_OP: appendStringInfoString(buf, "JSON_QUERY("); break; case JSON_VALUE_OP: appendStringInfoString(buf, "JSON_VALUE("); break; default: elog(ERROR, "unrecognized JsonExpr op: %d", (int) jexpr->op); } get_rule_expr(jexpr->formatted_expr, context, showimplicit); appendStringInfoString(buf, ", "); get_json_path_spec(jexpr->path_spec, context, showimplicit); if (jexpr->passing_values) { ListCell *lc1, *lc2; bool needcomma = false; appendStringInfoString(buf, " PASSING "); forboth(lc1, jexpr->passing_names, lc2, jexpr->passing_values) { if (needcomma) appendStringInfoString(buf, ", "); needcomma = true; get_rule_expr((Node *) lfirst(lc2), context, showimplicit); appendStringInfo(buf, " AS %s", ((String *) lfirst_node(String, lc1))->sval); } } if (jexpr->op != JSON_EXISTS_OP || jexpr->returning->typid != BOOLOID) get_json_returning(jexpr->returning, context->buf, jexpr->op == JSON_QUERY_OP); get_json_expr_options(jexpr, context, jexpr->op != JSON_EXISTS_OP ? JSON_BEHAVIOR_NULL : JSON_BEHAVIOR_FALSE); appendStringInfoChar(buf, ')'); } break; case T_List: { char *sep; ListCell *l; sep = ""; foreach(l, (List *) node) { appendStringInfoString(buf, sep); get_rule_expr((Node *) lfirst(l), context, showimplicit); sep = ", "; } } break; case T_TableFunc: get_tablefunc((TableFunc *) node, context, showimplicit); break; case T_CallStmt: get_proc_expr((CallStmt *) node, context, showimplicit); break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); break; } } /* * get_rule_expr_toplevel - Parse back a toplevel expression * * Same as get_rule_expr(), except that if the expr is just a Var, we pass * istoplevel = true not false to get_variable(). This causes whole-row Vars * to get printed with decoration that will prevent expansion of "*". * We need to use this in contexts such as ROW() and VALUES(), where the * parser would expand "foo.*" appearing at top level. (In principle we'd * use this in get_target_list() too, but that has additional worries about * whether to print AS, so it needs to invoke get_variable() directly anyway.) */ static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit) { if (node && IsA(node, Var)) (void) get_variable((Var *) node, 0, true, context); else get_rule_expr(node, context, showimplicit); } /* * get_rule_list_toplevel - Parse back a list of toplevel expressions * * Apply get_rule_expr_toplevel() to each element of a List. * * This adds commas between the expressions, but caller is responsible * for printing surrounding decoration. */ static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit) { const char *sep; ListCell *lc; sep = ""; foreach(lc, lst) { Node *e = (Node *) lfirst(lc); appendStringInfoString(context->buf, sep); get_rule_expr_toplevel(e, context, showimplicit); sep = ", "; } } /* * get_rule_expr_funccall - Parse back a function-call expression * * Same as get_rule_expr(), except that we guarantee that the output will * look like a function call, or like one of the things the grammar treats as * equivalent to a function call (see the func_expr_windowless production). * This is needed in places where the grammar uses func_expr_windowless and * you can't substitute a parenthesized a_expr. If what we have isn't going * to look like a function call, wrap it in a dummy CAST() expression, which * will satisfy the grammar --- and, indeed, is likely what the user wrote to * produce such a thing. */ static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit) { if (looks_like_function(node)) get_rule_expr(node, context, showimplicit); else { StringInfo buf = context->buf; appendStringInfoString(buf, "CAST("); /* no point in showing any top-level implicit cast */ get_rule_expr(node, context, false); appendStringInfo(buf, " AS %s)", format_type_with_typemod(exprType(node), exprTypmod(node))); } } /* * Helper function to identify node types that satisfy func_expr_windowless. * If in doubt, "false" is always a safe answer. */ static bool looks_like_function(Node *node) { if (node == NULL) return false; /* probably shouldn't happen */ switch (nodeTag(node)) { case T_FuncExpr: /* OK, unless it's going to deparse as a cast */ return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); case T_NullIfExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: case T_JsonExpr: /* these are all accepted by func_expr_common_subexpr */ return true; default: break; } return false; } /* * get_oper_expr - Parse back an OpExpr node */ static void get_oper_expr(OpExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid opno = expr->opno; List *args = expr->args; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); get_rule_expr_paren(arg1, context, true, (Node *) expr); appendStringInfo(buf, " %s ", generate_operator_name(opno, exprType(arg1), exprType(arg2))); get_rule_expr_paren(arg2, context, true, (Node *) expr); } else { /* prefix operator */ Node *arg = (Node *) linitial(args); appendStringInfo(buf, "%s ", generate_operator_name(opno, InvalidOid, exprType(arg))); get_rule_expr_paren(arg, context, true, (Node *) expr); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } /* * get_func_expr - Parse back a FuncExpr node */ static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; bool use_variadic; ListCell *l; /* * If the function call came from an implicit coercion, then just show the * first argument --- unless caller wants to see implicit coercions. */ if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) { get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); return; } /* * If the function call came from a cast, then show the first argument * plus an explicit cast operation. */ if (expr->funcformat == COERCE_EXPLICIT_CAST || expr->funcformat == COERCE_IMPLICIT_CAST) { Node *arg = linitial(expr->args); Oid rettype = expr->funcresulttype; int32 coercedTypmod; /* Get the typmod if this is a length-coercion function */ (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); get_coercion_expr(arg, context, rettype, coercedTypmod, (Node *) expr); return; } /* * If the function was called using one of the SQL spec's random special * syntaxes, try to reproduce that. If we don't recognize the function, * fall through. */ if (expr->funcformat == COERCE_SQL_SYNTAX) { if (get_func_sql_syntax(expr, context)) return; } /* * Normal function: display as proname(args). First we need to extract * the argument datatypes. */ if (list_length(expr->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, expr->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } appendStringInfo(buf, "%s(", generate_function_name(funcoid, nargs, argnames, argtypes, expr->funcvariadic, &use_variadic, context->inGroupBy)); nargs = 0; foreach(l, expr->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(expr->args, l) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(l), context, true); } appendStringInfoChar(buf, ')'); } /* * get_proc_expr - Parse back a CallStmt node */ static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid functionOid = stmt->funcexpr->funcid; bool use_variadic; Oid *argumentTypes; List *finalArgumentList = NIL; ListCell *argumentCell; List *namedArgList = NIL; int numberOfArgs = -1; if (!get_merged_argument_list(stmt, &namedArgList, &argumentTypes, &finalArgumentList, &numberOfArgs)) { /* Nothing merged i.e. no OUT arguments */ get_func_expr((FuncExpr *) stmt->funcexpr, context, showimplicit); return; } appendStringInfo(buf, "%s(", generate_function_name(functionOid, numberOfArgs, namedArgList, argumentTypes, stmt->funcexpr->funcvariadic, &use_variadic, context->inGroupBy)); int argNumber = 0; foreach(argumentCell, finalArgumentList) { if (argNumber++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(finalArgumentList, argumentCell) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(argumentCell), context, true); argNumber++; } appendStringInfoChar(buf, ')'); } /* * get_agg_expr - Parse back an Aggref node */ static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref) { get_agg_expr_helper(aggref, context, original_aggref, NULL, NULL, false); } /* * get_agg_expr_helper - subroutine for get_agg_expr and * get_json_agg_constructor */ static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; bool use_variadic = false; /* * For a combining aggregate, we look up and deparse the corresponding * partial aggregate instead. This is necessary because our input * argument list has been replaced; the new argument list always has just * one element, which will point to a partial Aggref that supplies us with * transition states to combine. */ if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) { TargetEntry *tle; Assert(list_length(aggref->args) == 1); tle = linitial_node(TargetEntry, aggref->args); resolve_special_varno((Node *) tle->expr, context, get_agg_combine_expr, original_aggref); return; } /* * Mark as PARTIAL, if appropriate. We look to the original aggref so as * to avoid printing this when recursing from the code just above. */ if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) appendStringInfoString(buf, "PARTIAL "); /* Extract the argument types as seen by the parser */ nargs = get_aggregate_argtypes(aggref, argtypes); if (!funcname) funcname = generate_function_name(aggref->aggfnoid, nargs, NIL, argtypes, aggref->aggvariadic, &use_variadic, context->inGroupBy); /* Print the aggregate name, schema-qualified if needed */ appendStringInfo(buf, "%s(%s", funcname, (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) { /* * Ordered-set aggregates do not use "*" syntax. Also, we needn't * worry about inserting VARIADIC. So we can just dump the direct * args as-is. */ Assert(!aggref->aggvariadic); get_rule_expr((Node *) aggref->aggdirectargs, context, true); Assert(aggref->aggorder != NIL); appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } else { /* aggstar can be set only in zero-argument aggregates */ if (aggref->aggstar) appendStringInfoChar(buf, '*'); else { ListCell *l; int i; i = 0; foreach(l, aggref->args) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *arg = (Node *) tle->expr; Assert(!IsA(arg, NamedArgExpr)); if (tle->resjunk) continue; if (i++ > 0) { if (is_json_objectagg) { /* * the ABSENT ON NULL and WITH UNIQUE args are printed * separately, so ignore them here */ if (i > 2) break; appendStringInfoString(buf, " : "); } else appendStringInfoString(buf, ", "); } if (use_variadic && i == nargs) appendStringInfoString(buf, "VARIADIC "); get_rule_expr(arg, context, true); } } if (aggref->aggorder != NIL) { appendStringInfoString(buf, " ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } } if (options) appendStringInfoString(buf, options); if (aggref->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) aggref->aggfilter, context, false); } appendStringInfoChar(buf, ')'); } /* * This is a helper function for get_agg_expr(). It's used when we deparse * a combining Aggref; resolve_special_varno locates the corresponding partial * Aggref and then calls this. */ static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) { Aggref *aggref; Aggref *original_aggref = callback_arg; if (!IsA(node, Aggref)) elog(ERROR, "combining Aggref does not point to an Aggref"); aggref = (Aggref *) node; get_agg_expr(aggref, context, original_aggref); } /* * get_windowfunc_expr - Parse back a WindowFunc node */ static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) { get_windowfunc_expr_helper(wfunc, context, NULL, NULL, false); } /* * get_windowfunc_expr_helper - subroutine for get_windowfunc_expr and * get_json_agg_constructor */ static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; ListCell *l; if (list_length(wfunc->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, wfunc->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } if (!funcname) funcname = generate_function_name(wfunc->winfnoid, nargs, argnames, argtypes, false, NULL, context->inGroupBy); appendStringInfo(buf, "%s(", funcname); /* winstar can be set only in zero-argument aggregates */ if (wfunc->winstar) appendStringInfoChar(buf, '*'); else { if (is_json_objectagg) { get_rule_expr((Node *) linitial(wfunc->args), context, false); appendStringInfoString(buf, " : "); get_rule_expr((Node *) lsecond(wfunc->args), context, false); } else get_rule_expr((Node *) wfunc->args, context, true); } if (options) appendStringInfoString(buf, options); if (wfunc->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) wfunc->aggfilter, context, false); } appendStringInfoString(buf, ") OVER "); foreach(l, context->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->winref == wfunc->winref) { if (wc->name) appendStringInfoString(buf, quote_identifier(wc->name)); else get_rule_windowspec(wc, context->targetList, context); break; } } if (l == NULL) { if (context->windowClause) elog(ERROR, "could not find window clause for winref %u", wfunc->winref); /* * In EXPLAIN, we don't have window context information available, so * we have to settle for this: */ appendStringInfoString(buf, "(?)"); } } /* * get_func_sql_syntax - Parse back a SQL-syntax function call * * Returns true if we successfully deparsed, false if we did not * recognize the function. */ static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; switch (funcoid) { case F_TIMEZONE_INTERVAL_TIMESTAMP: case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: case F_TIMEZONE_INTERVAL_TIMETZ: case F_TIMEZONE_TEXT_TIMESTAMP: case F_TIMEZONE_TEXT_TIMESTAMPTZ: case F_TIMEZONE_TEXT_TIMETZ: /* AT TIME ZONE ... note reversed argument order */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) lsecond(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " AT TIME ZONE "); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoChar(buf, ')'); return true; case F_TIMEZONE_TIMESTAMP: case F_TIMEZONE_TIMESTAMPTZ: case F_TIMEZONE_TIMETZ: /* AT LOCAL */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " AT LOCAL)"); return true; case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: case F_OVERLAPS_TIME_TIME_TIME_TIME: /* (x1, x2) OVERLAPS (y1, y2) */ appendStringInfoString(buf, "(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") OVERLAPS ("); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfourth(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_EXTRACT_TEXT_DATE: case F_EXTRACT_TEXT_TIME: case F_EXTRACT_TEXT_TIMETZ: case F_EXTRACT_TEXT_TIMESTAMP: case F_EXTRACT_TEXT_TIMESTAMPTZ: case F_EXTRACT_TEXT_INTERVAL: /* EXTRACT (x FROM y) */ appendStringInfoString(buf, "EXTRACT("); { Const *con = (Const *) linitial(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_IS_NORMALIZED: /* IS xxx NORMALIZED */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " IS"); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, " %s", TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " NORMALIZED)"); return true; case F_PG_COLLATION_FOR: /* COLLATION FOR */ appendStringInfoString(buf, "COLLATION FOR ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_NORMALIZE: /* NORMALIZE() */ appendStringInfoString(buf, "NORMALIZE("); get_rule_expr((Node *) linitial(expr->args), context, false); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, ", %s", TextDatumGetCString(con->constvalue)); } appendStringInfoChar(buf, ')'); return true; case F_OVERLAY_BIT_BIT_INT4: case F_OVERLAY_BIT_BIT_INT4_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: case F_OVERLAY_TEXT_TEXT_INT4: case F_OVERLAY_TEXT_TEXT_INT4_INT4: /* OVERLAY() */ appendStringInfoString(buf, "OVERLAY("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " PLACING "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lthird(expr->args), context, false); if (list_length(expr->args) == 4) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lfourth(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_POSITION_BIT_BIT: case F_POSITION_BYTEA_BYTEA: case F_POSITION_TEXT_TEXT: /* POSITION() ... extra parens since args are b_expr not a_expr */ appendStringInfoString(buf, "POSITION(("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") IN ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_SUBSTRING_BIT_INT4: case F_SUBSTRING_BIT_INT4_INT4: case F_SUBSTRING_BYTEA_INT4: case F_SUBSTRING_BYTEA_INT4_INT4: case F_SUBSTRING_TEXT_INT4: case F_SUBSTRING_TEXT_INT4_INT4: /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); if (list_length(expr->args) == 3) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lthird(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_SUBSTRING_TEXT_TEXT_TEXT: /* SUBSTRING SIMILAR/ESCAPE */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " SIMILAR "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " ESCAPE "); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_BTRIM_BYTEA_BYTEA: case F_BTRIM_TEXT: case F_BTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(BOTH"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_LTRIM_BYTEA_BYTEA: case F_LTRIM_TEXT: case F_LTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(LEADING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_RTRIM_BYTEA_BYTEA: case F_RTRIM_TEXT: case F_RTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(TRAILING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_SYSTEM_USER: appendStringInfoString(buf, "SYSTEM_USER"); return true; case F_XMLEXISTS: /* XMLEXISTS ... extra parens because args are c_expr */ appendStringInfoString(buf, "XMLEXISTS(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, "))"); return true; } return false; } /* ---------- * get_coercion_expr * * Make a string representation of a value coerced to a specific type * ---------- */ static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode) { StringInfo buf = context->buf; /* * Since parse_coerce.c doesn't immediately collapse application of * length-coercion functions to constants, what we'll typically see in * such cases is a Const with typmod -1 and a length-coercion function * right above it. Avoid generating redundant output. However, beware of * suppressing casts when the user actually wrote something like * 'foo'::text::char(3). * * Note: it might seem that we are missing the possibility of needing to * print a COLLATE clause for such a Const. However, a Const could only * have nondefault collation in a post-constant-folding tree, in which the * length coercion would have been folded too. See also the special * handling of CollateExpr in coerce_to_target_type(): any collation * marking will be above the coercion node, not below it. */ if (arg && IsA(arg, Const) && ((Const *) arg)->consttype == resulttype && ((Const *) arg)->consttypmod == -1) { /* Show the constant without normal ::typename decoration */ get_const_expr((Const *) arg, context, -1); } else { if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, false, parentNode); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } appendStringInfo(buf, "::%s", format_type_with_typemod(resulttype, resulttypmod)); } /* ---------- * get_const_expr * * Make a string representation of a Const * * showtype can be -1 to never show "::typename" decoration, or +1 to always * show it, or 0 to show it only if the constant wouldn't be assumed to be * the right type by default. * * If the Const's collation isn't default for its type, show that too. * We mustn't do this when showtype is -1 (since that means the caller will * print "::typename", and we can't put a COLLATE clause in between). It's * caller's responsibility that collation isn't missed in such cases. * ---------- */ static void get_const_expr(Const *constval, deparse_context *context, int showtype) { StringInfo buf = context->buf; Oid typoutput; bool typIsVarlena; char *extval; bool needlabel = false; if (constval->constisnull) { /* * Always label the type of a NULL constant to prevent misdecisions * about type when reparsing. */ appendStringInfoString(buf, "NULL"); if (showtype >= 0) { appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } return; } getTypeOutputInfo(constval->consttype, &typoutput, &typIsVarlena); extval = OidOutputFunctionCall(typoutput, constval->constvalue); switch (constval->consttype) { case INT4OID: /* * INT4 can be printed without any decoration, unless it is * negative; in that case print it as '-nnn'::integer to ensure * that the output will re-parse as a constant, not as a constant * plus operator. In most cases we could get away with printing * (-nnn) instead, because of the way that gram.y handles negative * literals; but that doesn't work for INT_MIN, and it doesn't * seem that much prettier anyway. */ if (extval[0] != '-') appendStringInfoString(buf, extval); else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case NUMERICOID: /* * NUMERIC can be printed without quotes if it looks like a float * constant (not an integer, and not Infinity or NaN) and doesn't * have a leading sign (for the same reason as for INT4). */ if (isdigit((unsigned char) extval[0]) && strcspn(extval, "eE.") != strlen(extval)) { appendStringInfoString(buf, extval); } else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case BITOID: case VARBITOID: appendStringInfo(buf, "B'%s'", extval); break; case BOOLOID: if (strcmp(extval, "t") == 0) appendStringInfoString(buf, "true"); else appendStringInfoString(buf, "false"); break; default: simple_quote_literal(buf, extval); break; } pfree(extval); if (showtype < 0) return; /* * For showtype == 0, append ::typename unless the constant will be * implicitly typed as the right type when it is read in. * * XXX this code has to be kept in sync with the behavior of the parser, * especially make_const. */ switch (constval->consttype) { case BOOLOID: case UNKNOWNOID: /* These types can be left unlabeled */ needlabel = false; break; case INT4OID: /* We determined above whether a label is needed */ break; case NUMERICOID: /* * Float-looking constants will be typed as numeric, which we * checked above; but if there's a nondefault typmod we need to * show it. */ needlabel |= (constval->consttypmod >= 0); break; default: needlabel = true; break; } if (needlabel || showtype > 0) appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } /* * helper for get_const_expr: append COLLATE if needed */ static void get_const_collation(Const *constval, deparse_context *context) { StringInfo buf = context->buf; if (OidIsValid(constval->constcollid)) { Oid typcollation = get_typcollation(constval->consttype); if (constval->constcollid != typcollation) { appendStringInfo(buf, " COLLATE %s", generate_collation_name(constval->constcollid)); } } } /* * get_json_path_spec - Parse back a JSON path specification */ static void get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit) { if (IsA(path_spec, Const)) get_const_expr((Const *) path_spec, context, -1); else get_rule_expr(path_spec, context, showimplicit); } /* * get_json_format - Parse back a JsonFormat node */ static void get_json_format(JsonFormat *format, StringInfo buf) { if (format->format_type == JS_FORMAT_DEFAULT) return; appendStringInfoString(buf, format->format_type == JS_FORMAT_JSONB ? " FORMAT JSONB" : " FORMAT JSON"); if (format->encoding != JS_ENC_DEFAULT) { const char *encoding; encoding = format->encoding == JS_ENC_UTF16 ? "UTF16" : format->encoding == JS_ENC_UTF32 ? "UTF32" : "UTF8"; appendStringInfo(buf, " ENCODING %s", encoding); } } /* * get_json_returning - Parse back a JsonReturning structure */ static void get_json_returning(JsonReturning *returning, StringInfo buf, bool json_format_by_default) { if (!OidIsValid(returning->typid)) return; appendStringInfo(buf, " RETURNING %s", format_type_with_typemod(returning->typid, returning->typmod)); if (!json_format_by_default || returning->format->format_type != (returning->typid == JSONBOID ? JS_FORMAT_JSONB : JS_FORMAT_JSON)) get_json_format(returning->format, buf); } /* * get_json_constructor - Parse back a JsonConstructorExpr node */ static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; const char *funcname; bool is_json_object; int curridx; ListCell *lc; if (ctor->type == JSCTOR_JSON_OBJECTAGG) { get_json_agg_constructor(ctor, context, "JSON_OBJECTAGG", true); return; } else if (ctor->type == JSCTOR_JSON_ARRAYAGG) { get_json_agg_constructor(ctor, context, "JSON_ARRAYAGG", false); return; } switch (ctor->type) { case JSCTOR_JSON_OBJECT: funcname = "JSON_OBJECT"; break; case JSCTOR_JSON_ARRAY: funcname = "JSON_ARRAY"; break; case JSCTOR_JSON_PARSE: funcname = "JSON"; break; case JSCTOR_JSON_SCALAR: funcname = "JSON_SCALAR"; break; case JSCTOR_JSON_SERIALIZE: funcname = "JSON_SERIALIZE"; break; default: elog(ERROR, "invalid JsonConstructorType %d", ctor->type); } appendStringInfo(buf, "%s(", funcname); is_json_object = ctor->type == JSCTOR_JSON_OBJECT; foreach(lc, ctor->args) { curridx = foreach_current_index(lc); if (curridx > 0) { const char *sep; sep = (is_json_object && (curridx % 2) != 0) ? " : " : ", "; appendStringInfoString(buf, sep); } get_rule_expr((Node *) lfirst(lc), context, true); } get_json_constructor_options(ctor, buf); appendStringInfoChar(buf, ')'); } /* * Append options, if any, to the JSON constructor being deparsed */ static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf) { if (ctor->absent_on_null) { if (ctor->type == JSCTOR_JSON_OBJECT || ctor->type == JSCTOR_JSON_OBJECTAGG) appendStringInfoString(buf, " ABSENT ON NULL"); } else { if (ctor->type == JSCTOR_JSON_ARRAY || ctor->type == JSCTOR_JSON_ARRAYAGG) appendStringInfoString(buf, " NULL ON NULL"); } if (ctor->unique) appendStringInfoString(buf, " WITH UNIQUE KEYS"); /* * Append RETURNING clause if needed; JSON() and JSON_SCALAR() don't * support one. */ if (ctor->type != JSCTOR_JSON_PARSE && ctor->type != JSCTOR_JSON_SCALAR) get_json_returning(ctor->returning, buf, true); } /* * get_json_agg_constructor - Parse back an aggregate JsonConstructorExpr node */ static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg) { StringInfoData options; initStringInfo(&options); get_json_constructor_options(ctor, &options); if (IsA(ctor->func, Aggref)) get_agg_expr_helper((Aggref *) ctor->func, context, (Aggref *) ctor->func, funcname, options.data, is_json_objectagg); else if (IsA(ctor->func, WindowFunc)) get_windowfunc_expr_helper((WindowFunc *) ctor->func, context, funcname, options.data, is_json_objectagg); else elog(ERROR, "invalid JsonConstructorExpr underlying node type: %d", nodeTag(ctor->func)); } /* * simple_quote_literal - Format a string as a SQL literal, append to buf */ static void simple_quote_literal(StringInfo buf, const char *val) { const char *valptr; /* * We form the string literal according to the prevailing setting of * standard_conforming_strings; we never use E''. User is responsible for * making sure result is used correctly. */ appendStringInfoChar(buf, '\''); for (valptr = val; *valptr; valptr++) { char ch = *valptr; if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) appendStringInfoChar(buf, ch); appendStringInfoChar(buf, ch); } appendStringInfoChar(buf, '\''); } /* ---------- * get_sublink_expr - Parse back a sublink * ---------- */ static void get_sublink_expr(SubLink *sublink, deparse_context *context) { StringInfo buf = context->buf; Query *query = (Query *) (sublink->subselect); char *opname = NULL; bool need_paren; if (sublink->subLinkType == ARRAY_SUBLINK) appendStringInfoString(buf, "ARRAY("); else appendStringInfoChar(buf, '('); /* * Note that we print the name of only the first operator, when there are * multiple combining operators. This is an approximation that could go * wrong in various scenarios (operators in different schemas, renamed * operators, etc) but there is not a whole lot we can do about it, since * the syntax allows only one operator to be shown. */ if (sublink->testexpr) { if (IsA(sublink->testexpr, OpExpr)) { /* single combining operator */ OpExpr *opexpr = (OpExpr *) sublink->testexpr; get_rule_expr(linitial(opexpr->args), context, true); opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); } else if (IsA(sublink->testexpr, BoolExpr)) { /* multiple combining operators, = or <> cases */ char *sep; ListCell *l; appendStringInfoChar(buf, '('); sep = ""; foreach(l, ((BoolExpr *) sublink->testexpr)->args) { OpExpr *opexpr = lfirst_node(OpExpr, l); appendStringInfoString(buf, sep); get_rule_expr(linitial(opexpr->args), context, true); if (!opname) opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); sep = ", "; } appendStringInfoChar(buf, ')'); } else if (IsA(sublink->testexpr, RowCompareExpr)) { /* multiple combining operators, < <= > >= cases */ RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; appendStringInfoChar(buf, '('); get_rule_expr((Node *) rcexpr->largs, context, true); opname = generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs))); appendStringInfoChar(buf, ')'); } else elog(ERROR, "unrecognized testexpr type: %d", (int) nodeTag(sublink->testexpr)); } need_paren = true; switch (sublink->subLinkType) { case EXISTS_SUBLINK: appendStringInfoString(buf, "EXISTS "); break; case ANY_SUBLINK: if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ appendStringInfoString(buf, " IN "); else appendStringInfo(buf, " %s ANY ", opname); break; case ALL_SUBLINK: appendStringInfo(buf, " %s ALL ", opname); break; case ROWCOMPARE_SUBLINK: appendStringInfo(buf, " %s ", opname); break; case EXPR_SUBLINK: case MULTIEXPR_SUBLINK: case ARRAY_SUBLINK: need_paren = false; break; case CTE_SUBLINK: /* shouldn't occur in a SubLink */ default: elog(ERROR, "unrecognized sublink type: %d", (int) sublink->subLinkType); break; } if (need_paren) appendStringInfoChar(buf, '('); get_query_def(query, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoString(buf, "))"); else appendStringInfoChar(buf, ')'); } /* ---------- * get_xmltable - Parse back a XMLTABLE function * ---------- */ static void get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; appendStringInfoString(buf, "XMLTABLE("); if (tf->ns_uris != NIL) { ListCell *lc1, *lc2; bool first = true; appendStringInfoString(buf, "XMLNAMESPACES ("); forboth(lc1, tf->ns_uris, lc2, tf->ns_names) { Node *expr = (Node *) lfirst(lc1); char *name = strVal(lfirst(lc2)); if (!first) appendStringInfoString(buf, ", "); else first = false; if (name != NULL) { get_rule_expr(expr, context, showimplicit); appendStringInfo(buf, " AS %s", name); } else { appendStringInfoString(buf, "DEFAULT "); get_rule_expr(expr, context, showimplicit); } } appendStringInfoString(buf, "), "); } appendStringInfoChar(buf, '('); get_rule_expr((Node *) tf->rowexpr, context, showimplicit); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) tf->docexpr, context, showimplicit); appendStringInfoChar(buf, ')'); if (tf->colexprs != NIL) { ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; ListCell *l5; int colnum = 0; appendStringInfoString(buf, " COLUMNS "); forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, l4, tf->colexprs, l5, tf->coldefexprs) { char *colname = strVal(lfirst(l1)); Oid typid = lfirst_oid(l2); int32 typmod = lfirst_int(l3); Node *colexpr = (Node *) lfirst(l4); Node *coldefexpr = (Node *) lfirst(l5); bool ordinality = (tf->ordinalitycol == colnum); bool notnull = bms_is_member(colnum, tf->notnulls); if (colnum > 0) appendStringInfoString(buf, ", "); colnum++; appendStringInfo(buf, "%s %s", quote_identifier(colname), ordinality ? "FOR ORDINALITY" : format_type_with_typemod(typid, typmod)); if (ordinality) continue; if (coldefexpr != NULL) { appendStringInfoString(buf, " DEFAULT ("); get_rule_expr((Node *) coldefexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (colexpr != NULL) { appendStringInfoString(buf, " PATH ("); get_rule_expr((Node *) colexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (notnull) appendStringInfoString(buf, " NOT NULL"); } } appendStringInfoChar(buf, ')'); } /* * get_json_table_nested_columns - Parse back nested JSON_TABLE columns */ static void get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, deparse_context *context, bool showimplicit, bool needcomma) { if (IsA(plan, JsonTablePathScan)) { JsonTablePathScan *scan = castNode(JsonTablePathScan, plan); if (needcomma) appendStringInfoChar(context->buf, ','); appendStringInfoChar(context->buf, ' '); appendContextKeyword(context, "NESTED PATH ", 0, 0, 0); get_const_expr(scan->path->value, context, -1); appendStringInfo(context->buf, " AS %s", quote_identifier(scan->path->name)); get_json_table_columns(tf, scan, context, showimplicit); } else if (IsA(plan, JsonTableSiblingJoin)) { JsonTableSiblingJoin *join = (JsonTableSiblingJoin *) plan; get_json_table_nested_columns(tf, join->lplan, context, showimplicit, needcomma); get_json_table_nested_columns(tf, join->rplan, context, showimplicit, true); } } /* * get_json_table_columns - Parse back JSON_TABLE columns */ static void get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; ListCell *lc_colname; ListCell *lc_coltype; ListCell *lc_coltypmod; ListCell *lc_colvalexpr; int colnum = 0; appendStringInfoChar(buf, ' '); appendContextKeyword(context, "COLUMNS (", 0, 0, 0); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; forfour(lc_colname, tf->colnames, lc_coltype, tf->coltypes, lc_coltypmod, tf->coltypmods, lc_colvalexpr, tf->colvalexprs) { char *colname = strVal(lfirst(lc_colname)); JsonExpr *colexpr; Oid typid; int32 typmod; bool ordinality; JsonBehaviorType default_behavior; typid = lfirst_oid(lc_coltype); typmod = lfirst_int(lc_coltypmod); colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr)); /* Skip columns that don't belong to this scan. */ if (scan->colMin < 0 || colnum < scan->colMin) { colnum++; continue; } if (colnum > scan->colMax) break; if (colnum > scan->colMin) appendStringInfoString(buf, ", "); colnum++; ordinality = !colexpr; appendContextKeyword(context, "", 0, 0, 0); appendStringInfo(buf, "%s %s", quote_identifier(colname), ordinality ? "FOR ORDINALITY" : format_type_with_typemod(typid, typmod)); if (ordinality) continue; /* * Set default_behavior to guide get_json_expr_options() on whether to * to emit the ON ERROR / EMPTY clauses. */ if (colexpr->op == JSON_EXISTS_OP) { appendStringInfoString(buf, " EXISTS"); default_behavior = JSON_BEHAVIOR_FALSE; } else { if (colexpr->op == JSON_QUERY_OP) { char typcategory; bool typispreferred; get_type_category_preferred(typid, &typcategory, &typispreferred); if (typcategory == TYPCATEGORY_STRING) appendStringInfoString(buf, colexpr->format->format_type == JS_FORMAT_JSONB ? " FORMAT JSONB" : " FORMAT JSON"); } default_behavior = JSON_BEHAVIOR_NULL; } appendStringInfoString(buf, " PATH "); get_json_path_spec(colexpr->path_spec, context, showimplicit); get_json_expr_options(colexpr, context, default_behavior); } if (scan->child) get_json_table_nested_columns(tf, scan->child, context, showimplicit, scan->colMin >= 0); if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; appendContextKeyword(context, ")", 0, 0, 0); } /* ---------- * get_json_table - Parse back a JSON_TABLE function * ---------- */ static void get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr); JsonTablePathScan *root = castNode(JsonTablePathScan, tf->plan); appendStringInfoString(buf, "JSON_TABLE("); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; appendContextKeyword(context, "", 0, 0, 0); get_rule_expr(jexpr->formatted_expr, context, showimplicit); appendStringInfoString(buf, ", "); get_const_expr(root->path->value, context, -1); appendStringInfo(buf, " AS %s", quote_identifier(root->path->name)); if (jexpr->passing_values) { ListCell *lc1, *lc2; bool needcomma = false; appendStringInfoChar(buf, ' '); appendContextKeyword(context, "PASSING ", 0, 0, 0); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; forboth(lc1, jexpr->passing_names, lc2, jexpr->passing_values) { if (needcomma) appendStringInfoString(buf, ", "); needcomma = true; appendContextKeyword(context, "", 0, 0, 0); get_rule_expr((Node *) lfirst(lc2), context, false); appendStringInfo(buf, " AS %s", quote_identifier((lfirst_node(String, lc1))->sval) ); } if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; } get_json_table_columns(tf, castNode(JsonTablePathScan, tf->plan), context, showimplicit); if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY_ARRAY) get_json_behavior(jexpr->on_error, context, "ERROR"); if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; appendContextKeyword(context, ")", 0, 0, 0); } /* ---------- * get_tablefunc - Parse back a table function * ---------- */ static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) { /* XMLTABLE and JSON_TABLE are the only existing implementations. */ if (tf->functype == TFT_XMLTABLE) get_xmltable(tf, context, showimplicit); else if (tf->functype == TFT_JSON_TABLE) get_json_table(tf, context, showimplicit); } /* ---------- * get_from_clause - Parse back a FROM clause * * "prefix" is the keyword that denotes the start of the list of FROM * elements. It is FROM when used to parse back SELECT and UPDATE, but * is USING when parsing back DELETE. * ---------- */ static void get_from_clause(Query *query, const char *prefix, deparse_context *context) { StringInfo buf = context->buf; bool first = true; ListCell *l; /* * We use the query's jointree as a guide to what to print. However, we * must ignore auto-added RTEs that are marked not inFromCl. (These can * only appear at the top level of the jointree, so it's sufficient to * check here.) This check also ensures we ignore the rule pseudo-RTEs * for NEW and OLD. */ foreach(l, query->jointree->fromlist) { Node *jtnode = (Node *) lfirst(l); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); if (!rte->inFromCl) continue; } if (first) { appendContextKeyword(context, prefix, -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); first = false; get_from_clause_item(jtnode, query, context); } else { StringInfoData itembuf; appendStringInfoString(buf, ", "); /* * Put the new FROM item's text into itembuf so we can decide * after we've got it whether or not it needs to go on a new line. */ initStringInfo(&itembuf); context->buf = &itembuf; get_from_clause_item(jtnode, query, context); /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { /* Does the new item start with a new line? */ if (itembuf.len > 0 && itembuf.data[0] == '\n') { /* If so, we shouldn't add anything */ /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new item * would cause an overflow. */ if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } } /* Add the new item */ appendStringInfoString(buf, itembuf.data); /* clean up */ pfree(itembuf.data); } } } static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) { StringInfo buf = context->buf; deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); RangeTblFunction *rtfunc1 = NULL; CitusRTEKind rteKind = GetRangeTblKind(rte); if (rte->lateral) appendStringInfoString(buf, "LATERAL "); /* Print the FROM item proper */ switch (rte->rtekind) { case RTE_RELATION: /* Normal relation RTE */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, context->namespaces)); break; case RTE_SUBQUERY: /* Subquery RTE */ appendStringInfoChar(buf, '('); get_query_def(rte->subquery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); appendStringInfoChar(buf, ')'); break; case RTE_FUNCTION: /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); break; } /* Function RTE */ rtfunc1 = (RangeTblFunction *) linitial(rte->functions); /* * Omit ROWS FROM() syntax for just one function, unless it * has both a coldeflist and WITH ORDINALITY. If it has both, * we must use ROWS FROM() syntax to avoid ambiguity about * whether the coldeflist includes the ordinality column. */ if (list_length(rte->functions) == 1 && (rtfunc1->funccolnames == NIL || !rte->funcordinality)) { get_rule_expr_funccall(rtfunc1->funcexpr, context, true); /* we'll print the coldeflist below, if it has one */ } else { bool all_unnest; ListCell *lc; /* * If all the function calls in the list are to unnest, * and none need a coldeflist, then collapse the list back * down to UNNEST(args). (If we had more than one * built-in unnest function, this would get more * difficult.) * * XXX This is pretty ugly, since it makes not-terribly- * future-proof assumptions about what the parser would do * with the output; but the alternative is to emit our * nonstandard ROWS FROM() notation for what might have * been a perfectly spec-compliant multi-argument * UNNEST(). */ all_unnest = true; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (!IsA(rtfunc->funcexpr, FuncExpr) || ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || rtfunc->funccolnames != NIL) { all_unnest = false; break; } } if (all_unnest) { List *allargs = NIL; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); List *args = ((FuncExpr *) rtfunc->funcexpr)->args; allargs = list_concat(allargs, args); } appendStringInfoString(buf, "UNNEST("); get_rule_expr((Node *) allargs, context, true); appendStringInfoChar(buf, ')'); } else { int funcno = 0; appendStringInfoString(buf, "ROWS FROM("); foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (funcno > 0) appendStringInfoString(buf, ", "); get_rule_expr_funccall(rtfunc->funcexpr, context, true); if (rtfunc->funccolnames != NIL) { /* Reconstruct the column definition list */ appendStringInfoString(buf, " AS "); get_from_clause_coldeflist(rtfunc, NULL, context); } funcno++; } appendStringInfoChar(buf, ')'); } /* prevent printing duplicate coldeflist below */ rtfunc1 = NULL; } if (rte->funcordinality) appendStringInfoString(buf, " WITH ORDINALITY"); break; case RTE_TABLEFUNC: get_tablefunc(rte->tablefunc, context, true); break; case RTE_VALUES: /* Values list RTE */ appendStringInfoChar(buf, '('); get_values_def(rte->values_lists, context); appendStringInfoChar(buf, ')'); break; case RTE_CTE: appendStringInfoString(buf, quote_identifier(rte->ctename)); break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); break; } /* Print the relation alias, if needed */ get_rte_alias(rte, varno, false, context); /* Print the column definitions or aliases, if needed */ if (rtfunc1 && rtfunc1->funccolnames != NIL) { /* Reconstruct the columndef list, which is also the aliases */ get_from_clause_coldeflist(rtfunc1, colinfo, context); } else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD || (rte->alias != NULL && rte->alias->colnames != NIL)) { /* Else print column aliases as needed */ get_column_alias_list(colinfo, context); } /* check if column's are given aliases in distributed tables */ else if (colinfo->parentUsing != NIL) { Assert(colinfo->printaliases); get_column_alias_list(colinfo, context); } /* Tablesample clause must go after any alias */ if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && rte->tablesample) { get_tablesample_def(rte->tablesample, context); } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); bool need_paren_on_right; need_paren_on_right = PRETTY_PAREN(context) && !IsA(j->rarg, RangeTblRef) && !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, '('); get_from_clause_item(j->larg, query, context); switch (j->jointype) { case JOIN_INNER: if (j->quals) appendContextKeyword(context, " JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); else appendContextKeyword(context, " CROSS JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_LEFT: appendContextKeyword(context, " LEFT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_FULL: appendContextKeyword(context, " FULL JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_RIGHT: appendContextKeyword(context, " RIGHT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; default: elog(ERROR, "unrecognized join type: %d", (int) j->jointype); } if (need_paren_on_right) appendStringInfoChar(buf, '('); get_from_clause_item(j->rarg, query, context); if (need_paren_on_right) appendStringInfoChar(buf, ')'); if (j->usingClause) { ListCell *lc; bool first = true; appendStringInfoString(buf, " USING ("); /* Use the assigned names, not what's in usingClause */ foreach(lc, colinfo->usingNames) { char *colname = (char *) lfirst(lc); if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } appendStringInfoChar(buf, ')'); if (j->join_using_alias) appendStringInfo(buf, " AS %s", quote_identifier(j->join_using_alias->aliasname)); } else if (j->quals) { appendStringInfoString(buf, " ON "); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr(j->quals, context, false); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } else if (j->jointype != JOIN_INNER) { /* If we didn't say CROSS JOIN above, we must provide an ON */ appendStringInfoString(buf, " ON TRUE"); } if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, ')'); /* Yes, it's correct to put alias after the right paren ... */ if (j->alias != NULL) { /* * Note that it's correct to emit an alias clause if and only if * there was one originally. Otherwise we'd be converting a named * join to unnamed or vice versa, which creates semantic * subtleties we don't want. However, we might print a different * alias name than was there originally. */ appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(j->rtindex, context))); get_column_alias_list(colinfo, context); } } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * get_rte_alias - print the relation's alias, if needed * * If printed, the alias is preceded by a space, or by " AS " if use_as is true. */ static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); char *refname = get_rtable_name(varno, context); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); bool printalias = false; if (rte->alias != NULL) { /* Always print alias if user provided one */ printalias = true; } else if (colinfo->printaliases) { /* Always print alias if we need to print column aliases */ printalias = true; } else if (rte->rtekind == RTE_RELATION) { /* * No need to print alias if it's same as relation name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, get_relation_name(rte->relid)) != 0) printalias = true; } else if (rte->rtekind == RTE_FUNCTION) { /* * For a function RTE, always print alias. This covers possible * renaming of the function and/or instability of the FigureColname * rules for things that aren't simple functions. Note we'd need to * force it anyway for the columndef list case. */ printalias = true; } else if (rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_VALUES) { /* * For a subquery, always print alias. This makes the output * SQL-spec-compliant, even though we allow such aliases to be omitted * on input. */ printalias = true; } else if (rte->rtekind == RTE_CTE) { /* * No need to print alias if it's same as CTE name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, rte->ctename) != 0) printalias = true; } if (printalias) appendStringInfo(context->buf, "%s%s", use_as ? " AS " : " ", quote_identifier(refname)); } /* * get_column_alias_list - print column alias list for an RTE * * Caller must already have printed the relation's alias name. */ static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; int i; bool first = true; /* Don't print aliases if not needed */ if (!colinfo->printaliases) return; for (i = 0; i < colinfo->num_new_cols; i++) { char *colname = colinfo->new_colnames[i]; if (first) { appendStringInfoChar(buf, '('); first = false; } else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } if (!first) appendStringInfoChar(buf, ')'); } /* * get_from_clause_coldeflist - reproduce FROM clause coldeflist * * When printing a top-level coldeflist (which is syntactically also the * relation's column alias list), use column names from colinfo. But when * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the * original coldeflist's names, which are available in rtfunc->funccolnames. * Pass NULL for colinfo to select the latter behavior. * * The coldeflist is appended immediately (no space) to buf. Caller is * responsible for ensuring that an alias or AS is present before it. */ static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; int i; appendStringInfoChar(buf, '('); i = 0; forfour(l1, rtfunc->funccoltypes, l2, rtfunc->funccoltypmods, l3, rtfunc->funccolcollations, l4, rtfunc->funccolnames) { Oid atttypid = lfirst_oid(l1); int32 atttypmod = lfirst_int(l2); Oid attcollation = lfirst_oid(l3); char *attname; if (colinfo) attname = colinfo->colnames[i]; else attname = strVal(lfirst(l4)); Assert(attname); /* shouldn't be any dropped columns here */ if (i > 0) appendStringInfoString(buf, ", "); appendStringInfo(buf, "%s %s", quote_identifier(attname), format_type_with_typemod(atttypid, atttypmod)); if (OidIsValid(attcollation) && attcollation != get_typcollation(atttypid)) appendStringInfo(buf, " COLLATE %s", generate_collation_name(attcollation)); i++; } appendStringInfoChar(buf, ')'); } /* * get_tablesample_def - print a TableSampleClause */ static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) { StringInfo buf = context->buf; Oid argtypes[1]; int nargs; ListCell *l; /* * We should qualify the handler's function name if it wouldn't be * resolved by lookup in the current search path. */ argtypes[0] = INTERNALOID; appendStringInfo(buf, " TABLESAMPLE %s (", generate_function_name(tablesample->tsmhandler, 1, NIL, argtypes, false, NULL, false)); nargs = 0; foreach(l, tablesample->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfirst(l), context, false); } appendStringInfoChar(buf, ')'); if (tablesample->repeatable != NULL) { appendStringInfoString(buf, " REPEATABLE ("); get_rule_expr((Node *) tablesample->repeatable, context, false); appendStringInfoChar(buf, ')'); } } /* * get_opclass_name - fetch name of an index operator class * * The opclass name is appended (after a space) to buf. * * Output is suppressed if the opclass is the default for the given * actual_datatype. (If you don't want this behavior, just pass * InvalidOid for actual_datatype.) */ static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf) { HeapTuple ht_opc; Form_pg_opclass opcrec; char *opcname; char *nspname; ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); if (!HeapTupleIsValid(ht_opc)) elog(ERROR, "cache lookup failed for opclass %u", opclass); opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); if (!OidIsValid(actual_datatype) || GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) { /* Okay, we need the opclass name. Do we need to qualify it? */ opcname = NameStr(opcrec->opcname); if (OpclassIsVisible(opclass)) appendStringInfo(buf, " %s", quote_identifier(opcname)); else { nspname = get_namespace_name_or_temp(opcrec->opcnamespace); appendStringInfo(buf, " %s.%s", quote_identifier(nspname), quote_identifier(opcname)); } } ReleaseSysCache(ht_opc); } /* * processIndirection - take care of array and subfield assignment * * We strip any top-level FieldStore or assignment SubscriptingRef nodes that * appear in the input, printing them as decoration for the base column * name (which we assume the caller just printed). We might also need to * strip CoerceToDomain nodes, but only ones that appear above assignment * nodes. * * Returns the subexpression that's to be assigned. */ static Node * processIndirection(Node *node, deparse_context *context) { StringInfo buf = context->buf; CoerceToDomain *cdomain = NULL; for (;;) { if (node == NULL) break; if (IsA(node, FieldStore)) { FieldStore *fstore = (FieldStore *) node; Oid typrelid; char *fieldname; /* lookup tuple type */ typrelid = get_typ_typrelid(fstore->resulttype); if (!OidIsValid(typrelid)) elog(ERROR, "argument type %s of FieldStore is not a tuple type", format_type_be(fstore->resulttype)); /* * Print the field name. There should only be one target field in * stored rules. There could be more than that in executable * target lists, but this function cannot be used for that case. */ Assert(list_length(fstore->fieldnums) == 1); fieldname = get_attname(typrelid, linitial_int(fstore->fieldnums), false); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); /* * We ignore arg since it should be an uninteresting reference to * the target column or subcolumn. */ node = (Node *) linitial(fstore->newvals); } else if (IsA(node, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) node; if (sbsref->refassgnexpr == NULL) break; printSubscripts(sbsref, context); /* * We ignore refexpr since it should be an uninteresting reference * to the target column or subcolumn. */ node = (Node *) sbsref->refassgnexpr; } else if (IsA(node, CoerceToDomain)) { cdomain = (CoerceToDomain *) node; /* If it's an explicit domain coercion, we're done */ if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; /* Tentatively descend past the CoerceToDomain */ node = (Node *) cdomain->arg; } else break; } /* * If we descended past a CoerceToDomain whose argument turned out not to * be a FieldStore or array assignment, back up to the CoerceToDomain. * (This is not enough to be fully correct if there are nested implicit * CoerceToDomains, but such cases shouldn't ever occur.) */ if (cdomain && node == (Node *) cdomain->arg) node = (Node *) cdomain; return node; } static void printSubscripts(SubscriptingRef *sbsref, deparse_context *context) { StringInfo buf = context->buf; ListCell *lowlist_item; ListCell *uplist_item; lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ foreach(uplist_item, sbsref->refupperindexpr) { appendStringInfoChar(buf, '['); if (lowlist_item) { /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(lowlist_item), context, false); appendStringInfoChar(buf, ':'); lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); } /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(uplist_item), context, false); appendStringInfoChar(buf, ']'); } } /* * get_relation_name * Get the unqualified name of a relation specified by OID * * This differs from the underlying get_rel_name() function in that it will * throw error instead of silently returning NULL if the OID is bad. */ static char * get_relation_name(Oid relid) { char *relname = get_rel_name(relid); if (!relname) elog(ERROR, "cache lookup failed for relation %u", relid); return relname; } /* * generate_relation_or_shard_name * Compute the name to display for a relation or shard * * If the provided relid is equal to the provided distrelid, this function * returns a shard-extended relation name; otherwise, it falls through to a * simple generate_relation_name call. */ static char * generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces) { char *relname = NULL; if (relid == distrelid) { relname = get_relation_name(relid); if (shardid > 0) { Oid schemaOid = get_rel_namespace(relid); char *schemaName = get_namespace_name_or_temp(schemaOid); AppendShardIdToName(&relname, shardid); relname = quote_qualified_identifier(schemaName, relname); } } else { relname = generate_relation_name(relid, namespaces); } return relname; } /* * generate_relation_name * Compute the name to display for a relation specified by OID * * The result includes all necessary quoting and schema-prefixing. * * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. * We will forcibly qualify the relation name if it equals any CTE name * visible in the namespace list. */ char * generate_relation_name(Oid relid, List *namespaces) { HeapTuple tp; Form_pg_class reltup; bool need_qual; ListCell *nslist; char *relname; char *nspname; char *result; tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for relation %u", relid); reltup = (Form_pg_class) GETSTRUCT(tp); relname = NameStr(reltup->relname); /* Check for conflicting CTE name */ need_qual = false; foreach(nslist, namespaces) { deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); ListCell *ctlist; foreach(ctlist, dpns->ctes) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); if (strcmp(cte->ctename, relname) == 0) { need_qual = true; break; } } if (need_qual) break; } /* Otherwise, qualify the name if not visible in search path */ if (!need_qual) need_qual = !RelationIsVisible(relid); if (need_qual) nspname = get_namespace_name_or_temp(reltup->relnamespace); else nspname = NULL; result = quote_qualified_identifier(nspname, relname); ReleaseSysCache(tp); return result; } /* * generate_rte_shard_name returns the qualified name of the shard given a * CITUS_RTE_SHARD range table entry. */ static char * generate_rte_shard_name(RangeTblEntry *rangeTableEntry) { char *shardSchemaName = NULL; char *shardTableName = NULL; Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, NULL); return generate_fragment_name(shardSchemaName, shardTableName); } /* * generate_fragment_name * Compute the name to display for a shard or merged table * * The result includes all necessary quoting and schema-prefixing. The schema * name can be NULL for regular shards. For merged tables, they are always * declared within a job-specific schema, and therefore can't have null schema * names. */ static char * generate_fragment_name(char *schemaName, char *tableName) { StringInfo fragmentNameString = makeStringInfo(); if (schemaName != NULL) { appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), quote_identifier(tableName)); } else { appendStringInfoString(fragmentNameString, quote_identifier(tableName)); } return fragmentNameString->data; } /* * generate_function_name * Compute the name to display for a function specified by OID, * given that it is being called with the specified actual arg names and * types. (Those matter because of ambiguous-function resolution rules.) * * If we're dealing with a potentially variadic function (in practice, this * means a FuncExpr or Aggref, not some other way of calling a function), then * has_variadic must specify whether variadic arguments have been merged, * and *use_variadic_p will be set to indicate whether to print VARIADIC in * the output. For non-FuncExpr cases, has_variadic should be false and * use_variadic_p can be NULL. * * inGroupBy must be true if we're deparsing a GROUP BY clause. * * The result includes all necessary quoting and schema-prefixing. */ static char * generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy) { char *result; HeapTuple proctup; Form_pg_proc procform; char *proname; bool use_variadic; char *nspname; FuncDetailCode p_result; Oid p_funcid; Oid p_rettype; bool p_retset; int p_nvargs; Oid p_vatype; Oid *p_true_typeids; bool force_qualify = false; proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); if (!HeapTupleIsValid(proctup)) elog(ERROR, "cache lookup failed for function %u", funcid); procform = (Form_pg_proc) GETSTRUCT(proctup); proname = NameStr(procform->proname); /* * Due to parser hacks to avoid needing to reserve CUBE, we need to force * qualification of some function names within GROUP BY. */ if (inGroupBy) { if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) force_qualify = true; } /* * Determine whether VARIADIC should be printed. We must do this first * since it affects the lookup rules in func_get_detail(). * * Currently, we always print VARIADIC if the function has a merged * variadic-array argument. Note that this is always the case for * functions taking a VARIADIC argument type other than VARIADIC ANY. * * In principle, if VARIADIC wasn't originally specified and the array * actual argument is deconstructable, we could print the array elements * separately and not print VARIADIC, thus more nearly reproducing the * original input. For the moment that seems like too much complication * for the benefit, and anyway we do not know whether VARIADIC was * originally specified if it's a non-ANY type. */ if (use_variadic_p) { /* Parser should not have set funcvariadic unless fn is variadic */ Assert(!has_variadic || OidIsValid(procform->provariadic)); use_variadic = has_variadic; *use_variadic_p = use_variadic; } else { Assert(!has_variadic); use_variadic = false; } /* * The idea here is to schema-qualify only if the parser would fail to * resolve the correct function given the unqualified func name with the * specified argtypes and VARIADIC flag. But if we already decided to * force qualification, then we can skip the lookup and pretend we didn't * find it. */ if (!force_qualify) p_result = func_get_detail(list_make1(makeString(proname)), NIL, argnames, nargs, argtypes, !use_variadic, true, false, &p_funcid, &p_rettype, &p_retset, &p_nvargs, &p_vatype, &p_true_typeids, NULL); else { p_result = FUNCDETAIL_NOTFOUND; p_funcid = InvalidOid; } if ((p_result == FUNCDETAIL_NORMAL || p_result == FUNCDETAIL_AGGREGATE || p_result == FUNCDETAIL_WINDOWFUNC) && p_funcid == funcid) nspname = NULL; else nspname = get_namespace_name_or_temp(procform->pronamespace); result = quote_qualified_identifier(nspname, proname); ReleaseSysCache(proctup); return result; } /* * generate_operator_name * Compute the name to display for an operator specified by OID, * given that it is being called with the specified actual arg types. * (Arg types matter because of ambiguous-operator resolution rules. * Pass InvalidOid for unused arg of a unary operator.) * * The result includes all necessary quoting and schema-prefixing, * plus the OPERATOR() decoration needed to use a qualified operator name * in an expression. */ char * generate_operator_name(Oid operid, Oid arg1, Oid arg2) { StringInfoData buf; HeapTuple opertup; Form_pg_operator operform; char *oprname; char *nspname; initStringInfo(&buf); opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); if (!HeapTupleIsValid(opertup)) elog(ERROR, "cache lookup failed for operator %u", operid); operform = (Form_pg_operator) GETSTRUCT(opertup); oprname = NameStr(operform->oprname); /* * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, * we don't check if the operator is in current namespace or not. This is * because this check is costly when the operator is not in current namespace. */ nspname = get_namespace_name_or_temp(operform->oprnamespace); Assert(nspname != NULL); appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); appendStringInfoString(&buf, oprname); appendStringInfoChar(&buf, ')'); ReleaseSysCache(opertup); return buf.data; } /* * get_one_range_partition_bound_string * A C string representation of one range partition bound */ char * get_range_partbound_string(List *bound_datums) { deparse_context context; StringInfo buf = makeStringInfo(); ListCell *cell; char *sep; memset(&context, 0, sizeof(deparse_context)); context.buf = buf; appendStringInfoChar(buf, '('); sep = ""; foreach(cell, bound_datums) { PartitionRangeDatum *datum = lfirst_node(PartitionRangeDatum, cell); appendStringInfoString(buf, sep); if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) appendStringInfoString(buf, "MINVALUE"); else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) appendStringInfoString(buf, "MAXVALUE"); else { Const *val = castNode(Const, datum->value); get_const_expr(val, &context, -1); } sep = ", "; } appendStringInfoChar(buf, ')'); return buf->data; } /* * Collect a list of OIDs of all sequences owned by the specified relation, * and column if specified. If deptype is not zero, then only find sequences * with the specified dependency type. */ List * getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) { List *result = NIL; Relation depRel; ScanKeyData key[3]; SysScanDesc scan; HeapTuple tup; depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); if (attnum) ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* * We assume any auto or internal dependency of a sequence on a column * must be what we are looking for. (We need the relkind test because * indexes can also have auto dependencies on columns.) */ if (deprec->classid == RelationRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { if (!deptype || deprec->deptype == deptype) result = lappend_oid(result, deprec->objid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return result; } /* * get_insert_column_names_list Prepares the insert-column-names list. Any indirection * decoration needed on the column names can be inferred from the top targetlist. */ static List * get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte) { char *sep; ListCell *l; List *strippedexprs; strippedexprs = NIL; sep = ""; appendStringInfoChar(buf, '('); foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. * Add the stripped expressions to strippedexprs. (If it's a * single-VALUES statement, the stripped expressions are the VALUES to * print below. Otherwise they're just Vars and not really * interesting.) */ strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } appendStringInfoString(buf, ") "); return strippedexprs; } #endif /* (PG_VERSION_NUM >= PG_VERSION_17) && (PG_VERSION_NUM < PG_VERSION_18) */ ================================================ FILE: src/backend/distributed/deparser/ruleutils_18.c ================================================ /*------------------------------------------------------------------------- * * ruleutils_18.c * Functions to convert stored expressions/querytrees back to * source text * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/distributed/deparser/ruleutils_18.c * * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ #include "pg_version_constants.h" #include "pg_config.h" #if (PG_VERSION_NUM >= PG_VERSION_18) && (PG_VERSION_NUM < PG_VERSION_19) #include "postgres.h" #include #include #include #include "access/amapi.h" #include "access/htup_details.h" #include "access/relation.h" #include "access/table.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_authid.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_language.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_partitioned_table.h" #include "catalog/pg_proc.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/extension.h" #include "commands/tablespace.h" #include "common/keywords.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/multi_router_planner.h" #include "distributed/namespace_utils.h" #include "executor/spi.h" #include "foreign/foreign.h" #include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "optimizer/optimizer.h" #include "parser/parse_node.h" #include "parser/parse_agg.h" #include "parser/parse_func.h" #include "parser/parse_oper.h" #include "parser/parse_relation.h" #include "parser/parser.h" #include "parser/parsetree.h" #include "rewrite/rewriteHandler.h" #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSupport.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/ruleutils.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "utils/varlena.h" #include "utils/xml.h" /* ---------- * Pretty formatting constants * ---------- */ /* Indent counts */ #define PRETTYINDENT_STD 8 #define PRETTYINDENT_JOIN 4 #define PRETTYINDENT_VAR 4 #define PRETTYINDENT_LIMIT 40 /* wrap limit */ /* Pretty flags */ #define PRETTYFLAG_PAREN 0x0001 #define PRETTYFLAG_INDENT 0x0002 /* Default line length for pretty-print wrapping: 0 means wrap always */ #define WRAP_COLUMN_DEFAULT 0 /* macros to test if pretty action needed */ #define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) #define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) /* ---------- * Local data types * ---------- */ /* Context info needed for invoking a recursive querytree display routine */ typedef struct { StringInfo buf; /* output buffer to append to */ List *namespaces; /* List of deparse_namespace nodes */ TupleDesc resultDesc; /* if top level of a view, the view's tupdesc */ List *targetList; /* Current query level's SELECT targetlist */ List *windowClause; /* Current query level's WINDOW clause */ int prettyFlags; /* enabling of pretty-print functions */ int wrapColumn; /* max line length, or -1 for no limit */ int indentLevel; /* current indent level for prettyprint */ bool varprefix; /* true to print prefixes on Vars */ Oid distrelid; /* the distributed table being modified, if valid */ int64 shardid; /* a distributed table's shardid, if positive */ bool colNamesVisible; /* do we care about output column names? */ bool inGroupBy; /* deparsing GROUP BY clause? */ bool varInOrderBy; /* deparsing simple Var in ORDER BY? */ Bitmapset *appendparents; /* if not null, map child Vars of these relids * back to the parent rel */ } deparse_context; /* * Each level of query context around a subtree needs a level of Var namespace. * A Var having varlevelsup=N refers to the N'th item (counting from 0) in * the current context's namespaces list. * * The rangetable is the list of actual RTEs from the query tree, and the * cte list is the list of actual CTEs. * * rtable_names holds the alias name to be used for each RTE (either a C * string, or NULL for nameless RTEs such as unnamed joins). * rtable_columns holds the column alias names to be used for each RTE. * * In some cases we need to make names of merged JOIN USING columns unique * across the whole query, not only per-RTE. If so, unique_using is true * and using_names is a list of C strings representing names already assigned * to USING columns. * * When deparsing plan trees, there is always just a single item in the * deparse_namespace list (since a plan tree never contains Vars with * varlevelsup > 0). We store the PlanState node that is the immediate * parent of the expression to be deparsed, as well as a list of that * PlanState's ancestors. In addition, we store its outer and inner subplan * state nodes, as well as their plan nodes' targetlists, and the index tlist * if the current plan node might contain INDEX_VAR Vars. (These fields could * be derived on-the-fly from the current PlanState, but it seems notationally * clearer to set them up as separate fields.) */ typedef struct { List *rtable; /* List of RangeTblEntry nodes */ List *rtable_names; /* Parallel list of names for RTEs */ List *rtable_columns; /* Parallel list of deparse_columns structs */ List *subplans; /* List of Plan trees for SubPlans */ List *ctes; /* List of CommonTableExpr nodes */ AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ char *ret_old_alias; /* alias for OLD in RETURNING list */ char *ret_new_alias; /* alias for NEW in RETURNING list */ /* Workspace for column alias assignment: */ bool unique_using; /* Are we making USING names globally unique */ List *using_names; /* List of assigned names for USING columns */ /* Remaining fields are used only when deparsing a Plan tree: */ Plan *plan; /* immediate parent of current expression */ List *ancestors; /* ancestors of planstate */ Plan *outer_plan; /* outer subnode, or NULL if none */ Plan *inner_plan; /* inner subnode, or NULL if none */ List *outer_tlist; /* referent for OUTER_VAR Vars */ List *inner_tlist; /* referent for INNER_VAR Vars */ List *index_tlist; /* referent for INDEX_VAR Vars */ /* Special namespace representing a function signature: */ char *funcname; int numargs; char **argnames; } deparse_namespace; /* Callback signature for resolve_special_varno() */ typedef void (*rsv_callback) (Node *node, deparse_context *context, void *callback_arg); /* * Per-relation data about column alias names. * * Selecting aliases is unreasonably complicated because of the need to dump * rules/views whose underlying tables may have had columns added, deleted, or * renamed since the query was parsed. We must nonetheless print the rule/view * in a form that can be reloaded and will produce the same results as before. * * For each RTE used in the query, we must assign column aliases that are * unique within that RTE. SQL does not require this of the original query, * but due to factors such as *-expansion we need to be able to uniquely * reference every column in a decompiled query. As long as we qualify all * column references, per-RTE uniqueness is sufficient for that. * * However, we can't ensure per-column name uniqueness for unnamed join RTEs, * since they just inherit column names from their input RTEs, and we can't * rename the columns at the join level. Most of the time this isn't an issue * because we don't need to reference the join's output columns as such; we * can reference the input columns instead. That approach can fail for merged * JOIN USING columns, however, so when we have one of those in an unnamed * join, we have to make that column's alias globally unique across the whole * query to ensure it can be referenced unambiguously. * * Another problem is that a JOIN USING clause requires the columns to be * merged to have the same aliases in both input RTEs, and that no other * columns in those RTEs or their children conflict with the USING names. * To handle that, we do USING-column alias assignment in a recursive * traversal of the query's jointree. When descending through a JOIN with * USING, we preassign the USING column names to the child columns, overriding * other rules for column alias assignment. We also mark each RTE with a list * of all USING column names selected for joins containing that RTE, so that * when we assign other columns' aliases later, we can avoid conflicts. * * Another problem is that if a JOIN's input tables have had columns added or * deleted since the query was parsed, we must generate a column alias list * for the join that matches the current set of input columns --- otherwise, a * change in the number of columns in the left input would throw off matching * of aliases to columns of the right input. Thus, positions in the printable * column alias list are not necessarily one-for-one with varattnos of the * JOIN, so we need a separate new_colnames[] array for printing purposes. * * Finally, when dealing with wide tables we risk O(N^2) costs in assigning * non-duplicate column names. We ameliorate that by using a hash table that * holds all the strings appearing in colnames, new_colnames, and parentUsing. */ typedef struct { /* * colnames is an array containing column aliases to use for columns that * existed when the query was parsed. Dropped columns have NULL entries. * This array can be directly indexed by varattno to get a Var's name. * * Non-NULL entries are guaranteed unique within the RTE, *except* when * this is for an unnamed JOIN RTE. In that case we merely copy up names * from the two input RTEs. * * During the recursive descent in set_using_names(), forcible assignment * of a child RTE's column name is represented by pre-setting that element * of the child's colnames array. So at that stage, NULL entries in this * array just mean that no name has been preassigned, not necessarily that * the column is dropped. */ int num_cols; /* length of colnames[] array */ char **colnames; /* array of C strings and NULLs */ /* * new_colnames is an array containing column aliases to use for columns * that would exist if the query was re-parsed against the current * definitions of its base tables. This is what to print as the column * alias list for the RTE. This array does not include dropped columns, * but it will include columns added since original parsing. Indexes in * it therefore have little to do with current varattno values. As above, * entries are unique unless this is for an unnamed JOIN RTE. (In such an * RTE, we never actually print this array, but we must compute it anyway * for possible use in computing column names of upper joins.) The * parallel array is_new_col marks which of these columns are new since * original parsing. Entries with is_new_col false must match the * non-NULL colnames entries one-for-one. */ int num_new_cols; /* length of new_colnames[] array */ char **new_colnames; /* array of C strings */ bool *is_new_col; /* array of bool flags */ /* This flag tells whether we should actually print a column alias list */ bool printaliases; /* This list has all names used as USING names in joins above this RTE */ List *parentUsing; /* names assigned to parent merged columns */ /* * If this struct is for a JOIN RTE, we fill these fields during the * set_using_names() pass to describe its relationship to its child RTEs. * * leftattnos and rightattnos are arrays with one entry per existing * output column of the join (hence, indexable by join varattno). For a * simple reference to a column of the left child, leftattnos[i] is the * child RTE's attno and rightattnos[i] is zero; and conversely for a * column of the right child. But for merged columns produced by JOIN * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. * Also, if the column has been dropped, both are zero. * * If it's a JOIN USING, usingNames holds the alias names selected for the * merged columns (these might be different from the original USING list, * if we had to modify names to achieve uniqueness). */ int leftrti; /* rangetable index of left child */ int rightrti; /* rangetable index of right child */ int *leftattnos; /* left-child varattnos of join cols, or 0 */ int *rightattnos; /* right-child varattnos of join cols, or 0 */ List *usingNames; /* names assigned to merged columns */ /* * Hash table holding copies of all the strings appearing in this struct's * colnames, new_colnames, and parentUsing. We use a hash table only for * sufficiently wide relations, and only during the colname-assignment * functions set_relation_column_names and set_join_column_names; * otherwise, names_hash is NULL. */ HTAB *names_hash; /* entries are just strings */ } deparse_columns; /* This macro is analogous to rt_fetch(), but for deparse_columns structs */ #define deparse_columns_fetch(rangetable_index, dpns) \ ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) /* * Entry in set_rtable_names' hash table */ typedef struct { char name[NAMEDATALEN]; /* Hash key --- must be first */ int counter; /* Largest addition used so far for name */ } NameHashEntry; /* ---------- * Local functions * * Most of these functions used to use fixed-size buffers to build their * results. Now, they take an (already initialized) StringInfo object * as a parameter, and append their text output to its contents. * ---------- */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used); static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces); static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing); static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo); static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static char *make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo); static void expand_colnames_array_to(deparse_columns *colinfo, int n); static void build_colinfo_names_hash(deparse_columns *colinfo); static void add_to_names_hash(deparse_columns *colinfo, const char *name); static void destroy_colinfo_names_hash(deparse_columns *colinfo); static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo); static char *get_rtable_name(int rtindex, deparse_context *context); static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); static Plan *find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan); static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns); static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns); static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent); static void get_values_def(List *values_lists, deparse_context *context); static void get_with_clause(Query *query, deparse_context *context); static void get_select_query_def(Query *query, deparse_context *context); static void get_insert_query_def(Query *query, deparse_context *context); static void get_update_query_def(Query *query, deparse_context *context); static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte); static void get_delete_query_def(Query *query, deparse_context *context); static void get_merge_query_def(Query *query, deparse_context *context); static void get_utility_query_def(Query *query, deparse_context *context); static void get_basic_select_query(Query *query, deparse_context *context); static void get_target_list(List *targetList, deparse_context *context); static void get_returning_clause(Query *query, deparse_context *context); static void get_setop_query(Node *setOp, Query *query, deparse_context *context); static Node *get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context); static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context); static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context); static void get_rule_windowclause(Query *query, deparse_context *context); static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context); static void get_window_frame_options(int frameOptions, Node *startOffset, Node *endOffset, deparse_context *context); static char *get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context); static void get_special_variable(Node *node, deparse_context *context, void *callback_arg); static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg); static Node *find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p); static SubPlan *find_param_generator(Param *param, deparse_context *context, int *column_p); static SubPlan *find_param_generator_initplan(Param *param, Plan *plan, int *column_p); static void get_parameter(Param *param, deparse_context *context); static const char *get_simple_binary_op_name(OpExpr *expr); static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus); static void removeStringInfoSpaces(StringInfo str); static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit); static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit); static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit); static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit); static bool looks_like_function(Node *node); static void get_oper_expr(OpExpr *expr, deparse_context *context); static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit); static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit); static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref); static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg); static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg); static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg); static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode); static void get_const_expr(Const *constval, deparse_context *context, int showtype); static void get_const_collation(Const *constval, deparse_context *context); static void get_json_format(JsonFormat *format, StringInfo buf); static void get_json_returning(JsonReturning *returning, StringInfo buf, bool json_format_by_default); static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit); static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf); static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg); static void simple_quote_literal(StringInfo buf, const char *val); static void get_sublink_expr(SubLink *sublink, deparse_context *context); static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit); static void get_from_clause(Query *query, const char *prefix, deparse_context *context); static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context); static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context); static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context); static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context); static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context); static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf); static Node *processIndirection(Node *node, deparse_context *context); static void printSubscripts(SubscriptingRef *aref, deparse_context *context); static char *get_relation_name(Oid relid); static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces); static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); static char *generate_fragment_name(char *schemaName, char *tableName); static char *generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy); static List *get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte); static void get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit); static void get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, deparse_context *context, bool showimplicit); static void get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, deparse_context *context, bool showimplicit, bool needcomma); static void map_var_through_join_alias(deparse_namespace *dpns, Var *v); static Var *unwrap_simple_var(Node *expr); static bool dpns_has_named_join(const deparse_namespace *dpns); static inline bool var_matches_base(const Var *v, Index want_varno, AttrNumber want_attno); #define only_marker(rte) ((rte)->inh ? "" : "ONLY ") /* * pg_get_query_def parses back one query tree, and outputs the resulting query * string into given buffer. */ void pg_get_query_def(Query *query, StringInfo buffer) { get_query_def(query, buffer, NIL, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* * get_merged_argument_list merges both the IN and OUT arguments lists into one and * also eliminates the INOUT duplicates(present in both the lists). After merging both * the lists, it returns all the named-arguments in a list(mergedNamedArgList) along * with their types(mergedNamedArgTypes), final argument list(mergedArgumentList), and * the total number of arguments(totalArguments). */ bool get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, Oid **mergedNamedArgTypes, List **mergedArgumentList, int *totalArguments) { Oid functionOid = stmt->funcexpr->funcid; List *namedArgList = NIL; List *finalArgumentList = NIL; Oid *finalArgTypes; Oid *argTypes = NULL; char *argModes = NULL; char **argNames = NULL; int argIndex = 0; HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { elog(ERROR, "cache lookup failed for function %u", functionOid); } int defArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); ReleaseSysCache(proctup); if (argModes == NULL) { /* No OUT arguments */ return false; } /* * Passed arguments Includes IN, OUT, INOUT (in both the lists) and VARIADIC arguments, * which means INOUT arguments are double counted. */ int numberOfArgs = list_length(stmt->funcexpr->args) + list_length(stmt->outargs); int totalInoutArgs = 0; /* Let's count INOUT arguments from the defined number of arguments */ for (argIndex=0; argIndex < defArgs; ++argIndex) { if (argModes[argIndex] == PROARGMODE_INOUT) totalInoutArgs++; } /* Remove the duplicate INOUT counting */ numberOfArgs = numberOfArgs - totalInoutArgs; finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs); ListCell *inArgCell = list_head(stmt->funcexpr->args); ListCell *outArgCell = list_head(stmt->outargs); for (argIndex=0; argIndex < numberOfArgs; ++argIndex) { switch (argModes[argIndex]) { case PROARGMODE_IN: case PROARGMODE_VARIADIC: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); break; } case PROARGMODE_OUT: { Node *arg = (Node *) lfirst(outArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_INOUT: { Node *arg = (Node *) lfirst(inArgCell); if (IsA(arg, NamedArgExpr)) namedArgList = lappend(namedArgList, ((NamedArgExpr *) arg)->name); finalArgTypes[argIndex] = exprType(arg); finalArgumentList = lappend(finalArgumentList, arg); inArgCell = lnext(stmt->funcexpr->args, inArgCell); outArgCell = lnext(stmt->outargs, outArgCell); break; } case PROARGMODE_TABLE: default: { elog(ERROR, "Unhandled procedure argument mode[%d]", argModes[argIndex]); break; } } } /* * After eliminating INOUT duplicates and merging OUT arguments, we now * have the final list of arguments. */ if (defArgs != list_length(finalArgumentList)) { elog(ERROR, "Insufficient number of args passed[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)); } if (list_length(finalArgumentList) > FUNC_MAX_ARGS) { ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments[%d] for function[%s]", list_length(finalArgumentList), get_func_name(functionOid)))); } *mergedNamedArgList = namedArgList; *mergedNamedArgTypes = finalArgTypes; *mergedArgumentList = finalArgumentList; *totalArguments = numberOfArgs; return true; } /* * pg_get_rule_expr deparses an expression and returns the result as a string. */ char * pg_get_rule_expr(Node *expression) { bool showImplicitCasts = true; deparse_context context; StringInfo buffer = makeStringInfo(); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buffer; context.namespaces = NIL; context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = false; context.prettyFlags = 0; context.wrapColumn = WRAP_COLUMN_DEFAULT; context.indentLevel = 0; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.distrelid = InvalidOid; context.shardid = INVALID_SHARD_ID; get_rule_expr(expression, &context, showImplicitCasts); /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return buffer->data; } /* * set_rtable_names: select RTE aliases to be used in printing a query * * We fill in dpns->rtable_names with a list of names that is one-for-one with * the already-filled dpns->rtable list. Each RTE name is unique among those * in the new namespace plus any ancestor namespaces listed in * parent_namespaces. * * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. * * Note that this function is only concerned with relation names, not column * names. */ static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used) { HASHCTL hash_ctl; HTAB *names_hash; NameHashEntry *hentry; bool found; int rtindex; ListCell *lc; dpns->rtable_names = NIL; /* nothing more to do if empty rtable */ if (dpns->rtable == NIL) return; /* * We use a hash table to hold known names, so that this process is O(N) * not O(N^2) for N names. */ hash_ctl.keysize = NAMEDATALEN; hash_ctl.entrysize = sizeof(NameHashEntry); hash_ctl.hcxt = CurrentMemoryContext; names_hash = hash_create("set_rtable_names names", list_length(dpns->rtable), &hash_ctl, HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); /* Preload the hash table with names appearing in parent_namespaces */ foreach(lc, parent_namespaces) { deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); ListCell *lc2; foreach(lc2, olddpns->rtable_names) { char *oldname = (char *) lfirst(lc2); if (oldname == NULL) continue; hentry = (NameHashEntry *) hash_search(names_hash, oldname, HASH_ENTER, &found); /* we do not complain about duplicate names in parent namespaces */ hentry->counter = 0; } } /* Now we can scan the rtable */ rtindex = 1; foreach(lc, dpns->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); char *refname; /* Just in case this takes an unreasonable amount of time ... */ CHECK_FOR_INTERRUPTS(); if (rels_used && !bms_is_member(rtindex, rels_used)) { /* Ignore unreferenced RTE */ refname = NULL; } else if (rte->alias) { /* If RTE has a user-defined alias, prefer that */ refname = rte->alias->aliasname; } else if (rte->rtekind == RTE_RELATION) { /* Use the current actual name of the relation */ refname = get_rel_name(rte->relid); } else if (rte->rtekind == RTE_JOIN) { /* Unnamed join has no refname */ refname = NULL; } else if (rte->rtekind == RTE_GROUP) { /* Use the name of the group */ refname = NULL; } else { /* Otherwise use whatever the parser assigned */ refname = rte->eref->aliasname; } /* * If the selected name isn't unique, append digits to make it so, and * make a new hash entry for it once we've got a unique name. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (refname) { hentry = (NameHashEntry *) hash_search(names_hash, refname, HASH_ENTER, &found); if (found) { /* Name already in use, must choose a new one */ int refnamelen = strlen(refname); char *modname = (char *) palloc(refnamelen + 16); NameHashEntry *hentry2; do { hentry->counter++; for (;;) { memcpy(modname, refname, refnamelen); sprintf(modname + refnamelen, "_%d", hentry->counter); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from refname to keep all the digits */ refnamelen = pg_mbcliplen(refname, refnamelen, refnamelen - 1); } hentry2 = (NameHashEntry *) hash_search(names_hash, modname, HASH_ENTER, &found); } while (found); hentry2->counter = 0; /* init new hash entry */ refname = modname; } else { /* Name not previously used, need only initialize hentry */ hentry->counter = 0; } } dpns->rtable_names = lappend(dpns->rtable_names, refname); rtindex++; } hash_destroy(names_hash); } /* * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree * * For convenience, this is defined to initialize the deparse_namespace struct * from scratch. */ static void set_deparse_for_query(deparse_namespace *dpns, Query *query, List *parent_namespaces) { ListCell *lc; ListCell *lc2; /* Initialize *dpns and fill rtable/ctes links */ memset(dpns, 0, sizeof(deparse_namespace)); dpns->rtable = query->rtable; dpns->subplans = NIL; dpns->ctes = query->cteList; dpns->appendrels = NULL; dpns->ret_old_alias = query->returningOldAlias; dpns->ret_new_alias = query->returningNewAlias; /* Assign a unique relation alias to each RTE */ set_rtable_names(dpns, parent_namespaces, NULL); /* Initialize dpns->rtable_columns to contain zeroed structs */ dpns->rtable_columns = NIL; while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) dpns->rtable_columns = lappend(dpns->rtable_columns, palloc0(sizeof(deparse_columns))); /* If it's a utility query, it won't have a jointree */ if (query->jointree) { /* Detect whether global uniqueness of USING names is needed */ dpns->unique_using = has_dangerous_join_using(dpns, (Node *) query->jointree); /* * Select names for columns merged by USING, via a recursive pass over * the query jointree. */ set_using_names(dpns, (Node *) query->jointree, NIL); } /* * Now assign remaining column aliases for each RTE. We do this in a * linear scan of the rtable, so as to process RTEs whether or not they * are in the jointree (we mustn't miss NEW.*, INSERT target relations, * etc). JOIN RTEs must be processed after their children, but this is * okay because they appear later in the rtable list than their children * (cf Asserts in identify_join_columns()). */ forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); if (rte->rtekind == RTE_JOIN) set_join_column_names(dpns, rte, colinfo); else set_relation_column_names(dpns, rte, colinfo); } } /* * has_dangerous_join_using: search jointree for unnamed JOIN USING * * Merged columns of a JOIN USING may act differently from either of the input * columns, either because they are merged with COALESCE (in a FULL JOIN) or * because an implicit coercion of the underlying input column is required. * In such a case the column must be referenced as a column of the JOIN not as * a column of either input. And this is problematic if the join is unnamed * (alias-less): we cannot qualify the column's name with an RTE name, since * there is none. (Forcibly assigning an alias to the join is not a solution, * since that will prevent legal references to tables below the join.) * To ensure that every column in the query is unambiguously referenceable, * we must assign such merged columns names that are globally unique across * the whole query, aliasing other columns out of the way as necessary. * * Because the ensuing re-aliasing is fairly damaging to the readability of * the query, we don't do this unless we have to. So, we must pre-scan * the join tree to see if we have to, before starting set_using_names(). */ static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do here */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) { if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) return true; } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; /* Is it an unnamed JOIN with USING? */ if (j->alias == NULL && j->usingClause) { /* * Yes, so check each join alias var to see if any of them are not * simple references to underlying columns. If so, we have a * dangerous situation and must pick unique aliases. */ RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); /* We need only examine the merged columns */ for (int i = 0; i < jrte->joinmergedcols; i++) { Node *aliasvar = list_nth(jrte->joinaliasvars, i); if (!IsA(aliasvar, Var)) return true; } } /* Nope, but inspect children */ if (has_dangerous_join_using(dpns, j->larg)) return true; if (has_dangerous_join_using(dpns, j->rarg)) return true; } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); return false; } /* * set_using_names: select column aliases to be used for merged USING columns * * We do this during a recursive descent of the query jointree. * dpns->unique_using must already be set to determine the global strategy. * * Column alias info is saved in the dpns->rtable_columns list, which is * assumed to be filled with pre-zeroed deparse_columns structs. * * parentUsing is a list of all USING aliases assigned in parent joins of * the current jointree node. (The passed-in list must not be modified.) * * Note that we do not use per-deparse_columns hash tables in this function. * The number of names that need to be assigned should be small enough that * we don't need to trouble with that. */ static void set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) { if (IsA(jtnode, RangeTblRef)) { /* nothing to do now */ } else if (IsA(jtnode, FromExpr)) { FromExpr *f = (FromExpr *) jtnode; ListCell *lc; foreach(lc, f->fromlist) set_using_names(dpns, (Node *) lfirst(lc), parentUsing); } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); int *leftattnos; int *rightattnos; deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; int i; ListCell *lc; /* Get info about the shape of the join */ identify_join_columns(j, rte, colinfo); leftattnos = colinfo->leftattnos; rightattnos = colinfo->rightattnos; /* Look up the not-yet-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * If this join is unnamed, then we cannot substitute new aliases at * this level, so any name requirements pushed down to here must be * pushed down again to the children. */ if (rte->alias == NULL) { for (i = 0; i < colinfo->num_cols; i++) { char *colname = colinfo->colnames[i]; if (colname == NULL) continue; /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } } } /* * If there's a USING clause, select the USING column names and push * those names down to the children. We have two strategies: * * If dpns->unique_using is true, we force all USING names to be * unique across the whole query level. In principle we'd only need * the names of dangerous USING columns to be globally unique, but to * safely assign all USING names in a single pass, we have to enforce * the same uniqueness rule for all of them. However, if a USING * column's name has been pushed down from the parent, we should use * it as-is rather than making a uniqueness adjustment. This is * necessary when we're at an unnamed join, and it creates no risk of * ambiguity. Also, if there's a user-written output alias for a * merged column, we prefer to use that rather than the input name; * this simplifies the logic and seems likely to lead to less aliasing * overall. * * If dpns->unique_using is false, we only need USING names to be * unique within their own join RTE. We still need to honor * pushed-down names, though. * * Though significantly different in results, these two strategies are * implemented by the same code, with only the difference of whether * to put assigned names into dpns->using_names. */ if (j->usingClause) { /* Copy the input parentUsing list so we don't modify it */ parentUsing = list_copy(parentUsing); /* USING names must correspond to the first join output columns */ expand_colnames_array_to(colinfo, list_length(j->usingClause)); i = 0; foreach(lc, j->usingClause) { char *colname = strVal(lfirst(lc)); /* Assert it's a merged column */ Assert(leftattnos[i] != 0 && rightattnos[i] != 0); /* Adopt passed-down name if any, else select unique name */ if (colinfo->colnames[i] != NULL) colname = colinfo->colnames[i]; else { /* Prefer user-written output alias if any */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); /* Make it appropriately unique */ colname = make_colname_unique(colname, dpns, colinfo); if (dpns->unique_using) dpns->using_names = lappend(dpns->using_names, colname); /* Save it as output column name, too */ colinfo->colnames[i] = colname; } /* Remember selected names for use later */ colinfo->usingNames = lappend(colinfo->usingNames, colname); parentUsing = lappend(parentUsing, colname); /* Push down to left column, unless it's a system column */ if (leftattnos[i] > 0) { expand_colnames_array_to(leftcolinfo, leftattnos[i]); leftcolinfo->colnames[leftattnos[i] - 1] = colname; } /* Same on the righthand side */ if (rightattnos[i] > 0) { expand_colnames_array_to(rightcolinfo, rightattnos[i]); rightcolinfo->colnames[rightattnos[i] - 1] = colname; } i++; } } /* Mark child deparse_columns structs with correct parentUsing info */ leftcolinfo->parentUsing = parentUsing; rightcolinfo->parentUsing = parentUsing; /* Now recursively assign USING column names in children */ set_using_names(dpns, j->larg, parentUsing); set_using_names(dpns, j->rarg, parentUsing); } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * set_relation_column_names: select column aliases for a non-join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. */ static void set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { int ncolumns; char **real_colnames; bool changed_any; bool has_anonymous; int noldcolumns; int i; int j; /* * Construct an array of the current "real" column names of the RTE. * real_colnames[] will be indexed by physical column number, with NULL * entries for dropped columns. */ if ((rte->rtekind == RTE_RELATION || GetRangeTblKind(rte) == CITUS_RTE_SHARD) && OidIsValid(rte->relid)) { /* Relation --- look to the system catalogs for up-to-date info */ Relation rel; TupleDesc tupdesc; rel = relation_open(rte->relid, AccessShareLock); tupdesc = RelationGetDescr(rel); ncolumns = tupdesc->natts; real_colnames = (char **) palloc(ncolumns * sizeof(char *)); for (i = 0; i < ncolumns; i++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); if (attr->attisdropped) real_colnames[i] = NULL; else real_colnames[i] = pstrdup(NameStr(attr->attname)); } relation_close(rel, AccessShareLock); } else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* shard RTE without relid (pulled-up clone in PG18) */ /* use the column aliases already stored in rte->eref->colnames */ ncolumns = list_length(rte->eref->colnames); real_colnames = (char **) palloc0(ncolumns * sizeof(char *)); for (i = 0; i < ncolumns; i++) real_colnames[i] = pstrdup(strVal(list_nth(rte->eref->colnames, i))); /* keep changed_any / has_anonymous defaults */ } else if (rte->rtekind == RTE_GROUP) { /* ----- synthetic PG 18 RTE for GROUP BY / HAVING ----- */ ncolumns = list_length(rte->eref->colnames); real_colnames = (char **) palloc0(ncolumns * sizeof(char *)); for (i = 0; i < ncolumns; i++) real_colnames[i] = pstrdup(strVal(list_nth(rte->eref->colnames, i))); } else { /* Otherwise get the column names from eref or expandRTE() */ List *colnames; ListCell *lc; /* * Functions returning composites have the annoying property that some * of the composite type's columns might have been dropped since the * query was parsed. If possible, use expandRTE() to handle that * case, since it has the tedious logic needed to find out about * dropped columns. However, if we're explaining a plan, then we * don't have rte->functions because the planner thinks that won't be * needed later, and that breaks expandRTE(). So in that case we have * to rely on rte->eref, which may lead us to report a dropped * column's old name; that seems close enough for EXPLAIN's purposes. * * For non-RELATION, non-FUNCTION RTEs, we can just look at rte->eref, * which should be sufficiently up-to-date: no other RTE types can * have columns get dropped from under them after parsing. */ if (rte->rtekind == RTE_FUNCTION && rte->functions != NIL) { /* Since we're not creating Vars, rtindex etc. don't matter */ expandRTE(rte, 1, 0, VAR_RETURNING_DEFAULT, -1, true /* include dropped */ , &colnames, NULL); } else colnames = rte->eref->colnames; ncolumns = list_length(colnames); real_colnames = (char **) palloc(ncolumns * sizeof(char *)); i = 0; foreach(lc, colnames) { /* * If the column name we find here is an empty string, then it's a * dropped column, so change to NULL. */ char *cname = strVal(lfirst(lc)); if (cname[0] == '\0') cname = NULL; real_colnames[i] = cname; i++; } } /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that there are now more columns than there were when the * query was parsed, ie colnames could be longer than rte->eref->colnames. * We must assign unique aliases to the new columns too, else there could * be unresolved conflicts when the view/rule is reloaded. */ expand_colnames_array_to(colinfo, ncolumns); Assert(colinfo->num_cols == ncolumns); /* * Make sufficiently large new_colnames and is_new_col arrays, too. * * Note: because we leave colinfo->num_new_cols zero until after the loop, * colname_is_unique will not consult that array, which is fine because it * would only be duplicate effort. */ colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); /* If the RTE is wide enough, use a hash table to avoid O(N^2) costs */ build_colinfo_names_hash(colinfo); /* * Scan the columns, select a unique alias for each one, and store it in * colinfo->colnames and colinfo->new_colnames. The former array has NULL * entries for dropped columns, the latter omits them. Also mark * new_colnames entries as to whether they are new since parse time; this * is the case for entries beyond the length of rte->eref->colnames. */ noldcolumns = list_length(rte->eref->colnames); changed_any = false; has_anonymous = false; j = 0; for (i = 0; i < ncolumns; i++) { char *real_colname = real_colnames[i]; char *colname = colinfo->colnames[i]; /* Skip dropped columns */ if (real_colname == NULL) { Assert(colname == NULL); /* colnames[i] is already NULL */ continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; add_to_names_hash(colinfo, colname); } /* Put names of non-dropped columns in new_colnames[] too */ colinfo->new_colnames[j] = colname; /* And mark them as new or not */ colinfo->is_new_col[j] = (i >= noldcolumns); j++; /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; /* * Remember if there is a reference to an anonymous column as named by * char * FigureColname(Node *node) */ if (!has_anonymous && strcmp(real_colname, "?column?") == 0) has_anonymous = true; } /* We're now done needing the colinfo's names_hash */ destroy_colinfo_names_hash(colinfo); /* * Set correct length for new_colnames[] array. (Note: if columns have * been added, colinfo->num_cols includes them, which is not really quite * right but is harmless, since any new columns must be at the end where * they won't affect varattnos of pre-existing columns.) */ colinfo->num_new_cols = j; /* * For a relation RTE, we need only print the alias column names if any * are different from the underlying "real" names. For a function RTE, * always emit a complete column alias list; this is to protect against * possible instability of the default column names (eg, from altering * parameter names). For tablefunc RTEs, we never print aliases, because * the column names are part of the clause itself. For other RTE types, * print if we changed anything OR if there were user-written column * aliases (since the latter would be part of the underlying "reality"). */ if (rte->rtekind == RTE_RELATION) colinfo->printaliases = changed_any; else if (rte->rtekind == RTE_FUNCTION) colinfo->printaliases = true; else if (rte->rtekind == RTE_TABLEFUNC) colinfo->printaliases = false; else if (rte->alias && rte->alias->colnames != NIL) colinfo->printaliases = true; else colinfo->printaliases = changed_any || has_anonymous; } /* * set_join_column_names: select column aliases for a join RTE * * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. * If any colnames entries are already filled in, those override local * choices. Also, names for USING columns were already chosen by * set_using_names(). We further expect that column alias selection has been * completed for both input RTEs. */ static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, deparse_columns *colinfo) { deparse_columns *leftcolinfo; deparse_columns *rightcolinfo; bool changed_any; int noldcolumns; int nnewcolumns; Bitmapset *leftmerged = NULL; Bitmapset *rightmerged = NULL; int i; int j; int ic; int jc; /* Look up the previously-filled-in child deparse_columns structs */ leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); /* * Ensure colinfo->colnames has a slot for each column. (It could be long * enough already, if we pushed down a name for the last column.) Note: * it's possible that one or both inputs now have more columns than there * were when the query was parsed, but we'll deal with that below. We * only need entries in colnames for pre-existing columns. */ noldcolumns = list_length(rte->eref->colnames); expand_colnames_array_to(colinfo, noldcolumns); Assert(colinfo->num_cols == noldcolumns); /* If the RTE is wide enough, use a hash table to avoid O(N^2) costs */ build_colinfo_names_hash(colinfo); /* * Scan the join output columns, select an alias for each one, and store * it in colinfo->colnames. If there are USING columns, set_using_names() * already selected their names, so we can start the loop at the first * non-merged column. */ changed_any = false; for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) { char *colname = colinfo->colnames[i]; char *real_colname; /* Join column must refer to at least one input column */ Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); /* Get the child column name */ if (colinfo->leftattnos[i] > 0) real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; else if (colinfo->rightattnos[i] > 0) real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; else { /* We're joining system columns --- use eref name */ real_colname = strVal(list_nth(rte->eref->colnames, i)); } /* If child col has been dropped, no need to assign a join colname */ if (real_colname == NULL) { colinfo->colnames[i] = NULL; continue; } /* In an unnamed join, just report child column names as-is */ if (rte->alias == NULL) { colinfo->colnames[i] = real_colname; add_to_names_hash(colinfo, real_colname); continue; } /* If alias already assigned, that's what to use */ if (colname == NULL) { /* If user wrote an alias, prefer that over real column name */ if (rte->alias && i < list_length(rte->alias->colnames)) colname = strVal(list_nth(rte->alias->colnames, i)); else colname = real_colname; /* Unique-ify and insert into colinfo */ colname = make_colname_unique(colname, dpns, colinfo); colinfo->colnames[i] = colname; add_to_names_hash(colinfo, colname); } /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; } /* * Calculate number of columns the join would have if it were re-parsed * now, and create storage for the new_colnames and is_new_col arrays. * * Note: colname_is_unique will be consulting new_colnames[] during the * loops below, so its not-yet-filled entries must be zeroes. */ nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - list_length(colinfo->usingNames); colinfo->num_new_cols = nnewcolumns; colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); /* * Generating the new_colnames array is a bit tricky since any new columns * added since parse time must be inserted in the right places. This code * must match the parser, which will order a join's columns as merged * columns first (in USING-clause order), then non-merged columns from the * left input (in attnum order), then non-merged columns from the right * input (ditto). If one of the inputs is itself a join, its columns will * be ordered according to the same rule, which means newly-added columns * might not be at the end. We can figure out what's what by consulting * the leftattnos and rightattnos arrays plus the input is_new_col arrays. * * In these loops, i indexes leftattnos/rightattnos (so it's join varattno * less one), j indexes new_colnames/is_new_col, and ic/jc have similar * meanings for the current child RTE. */ /* Handle merged columns; they are first and can't be new */ i = j = 0; while (i < noldcolumns && colinfo->leftattnos[i] != 0 && colinfo->rightattnos[i] != 0) { /* column name is already determined and known unique */ colinfo->new_colnames[j] = colinfo->colnames[i]; colinfo->is_new_col[j] = false; /* build bitmapsets of child attnums of merged columns */ if (colinfo->leftattnos[i] > 0) leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); if (colinfo->rightattnos[i] > 0) rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); i++, j++; } /* Handle non-merged left-child columns */ ic = 0; for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) { char *child_colname = leftcolinfo->new_colnames[jc]; if (!leftcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of left child */ while (ic < leftcolinfo->num_cols && leftcolinfo->colnames[ic] == NULL) ic++; Assert(ic < leftcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, leftmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->leftattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; add_to_names_hash(colinfo, colinfo->new_colnames[j]); } colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; j++; } /* Handle non-merged right-child columns in exactly the same way */ ic = 0; for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) { char *child_colname = rightcolinfo->new_colnames[jc]; if (!rightcolinfo->is_new_col[jc]) { /* Advance ic to next non-dropped old column of right child */ while (ic < rightcolinfo->num_cols && rightcolinfo->colnames[ic] == NULL) ic++; Assert(ic < rightcolinfo->num_cols); ic++; /* If it is a merged column, we already processed it */ if (bms_is_member(ic, rightmerged)) continue; /* Else, advance i to the corresponding existing join column */ while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) i++; Assert(i < colinfo->num_cols); Assert(ic == colinfo->rightattnos[i]); /* Use the already-assigned name of this column */ colinfo->new_colnames[j] = colinfo->colnames[i]; i++; } else { /* * Unique-ify the new child column name and assign, unless we're * in an unnamed join, in which case just copy */ if (rte->alias != NULL) { colinfo->new_colnames[j] = make_colname_unique(child_colname, dpns, colinfo); if (!changed_any && strcmp(colinfo->new_colnames[j], child_colname) != 0) changed_any = true; } else colinfo->new_colnames[j] = child_colname; add_to_names_hash(colinfo, colinfo->new_colnames[j]); } colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; j++; } /* Assert we processed the right number of columns */ #ifdef USE_ASSERT_CHECKING for (int col_index = 0; col_index < colinfo->num_cols; col_index++) { /* * In the above processing-loops, "i" advances only if * the column is not new, check if this is a new column. */ if (colinfo->is_new_col[col_index]) i++; } Assert(j == nnewcolumns); #endif /* We're now done needing the colinfo's names_hash */ destroy_colinfo_names_hash(colinfo); /* * For a named join, print column aliases if we changed any from the child * names. Unnamed joins cannot print aliases. */ if (rte->alias != NULL) colinfo->printaliases = changed_any; else colinfo->printaliases = false; } /* * colname_is_unique: is colname distinct from already-chosen column names? * * dpns is query-wide info, colinfo is for the column's RTE */ static bool colname_is_unique(const char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { int i; ListCell *lc; /* * If we have a hash table, consult that instead of linearly scanning the * colinfo's strings. */ if (colinfo->names_hash) { if (hash_search(colinfo->names_hash, colname, HASH_FIND, NULL) != NULL) return false; } else { /* Check against already-assigned column aliases within RTE */ for (i = 0; i < colinfo->num_cols; i++) { char *oldname = colinfo->colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* * If we're building a new_colnames array, check that too (this will * be partially but not completely redundant with the previous checks) */ for (i = 0; i < colinfo->num_new_cols; i++) { char *oldname = colinfo->new_colnames[i]; if (oldname && strcmp(oldname, colname) == 0) return false; } /* * Also check against names already assigned for parent-join USING * cols */ foreach(lc, colinfo->parentUsing) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } } /* * Also check against USING-column names that must be globally unique. * These are not hashed, but there should be few of them. */ foreach(lc, dpns->using_names) { char *oldname = (char *) lfirst(lc); if (strcmp(oldname, colname) == 0) return false; } return true; } /* * make_colname_unique: modify colname if necessary to make it unique * * dpns is query-wide info, colinfo is for the column's RTE */ static char * make_colname_unique(char *colname, deparse_namespace *dpns, deparse_columns *colinfo) { /* * If the selected name isn't unique, append digits to make it so. For a * very long input name, we might have to truncate to stay within * NAMEDATALEN. */ if (!colname_is_unique(colname, dpns, colinfo)) { int colnamelen = strlen(colname); char *modname = (char *) palloc(colnamelen + 16); int i = 0; do { i++; for (;;) { memcpy(modname, colname, colnamelen); sprintf(modname + colnamelen, "_%d", i); if (strlen(modname) < NAMEDATALEN) break; /* drop chars from colname to keep all the digits */ colnamelen = pg_mbcliplen(colname, colnamelen, colnamelen - 1); } } while (!colname_is_unique(modname, dpns, colinfo)); colname = modname; } return colname; } /* * expand_colnames_array_to: make colinfo->colnames at least n items long * * Any added array entries are initialized to zero. */ static void expand_colnames_array_to(deparse_columns *colinfo, int n) { if (n > colinfo->num_cols) { if (colinfo->colnames == NULL) colinfo->colnames = palloc0_array(char *, n); else { colinfo->colnames = repalloc0_array(colinfo->colnames, char *, colinfo->num_cols, n); } colinfo->num_cols = n; } } /* * build_colinfo_names_hash: optionally construct a hash table for colinfo */ static void build_colinfo_names_hash(deparse_columns *colinfo) { HASHCTL hash_ctl; int i; ListCell *lc; /* * Use a hash table only for RTEs with at least 32 columns. (The cutoff * is somewhat arbitrary, but let's choose it so that this code does get * exercised in the regression tests.) */ if (colinfo->num_cols < 32) return; /* * Set up the hash table. The entries are just strings with no other * payload. */ hash_ctl.keysize = NAMEDATALEN; hash_ctl.entrysize = NAMEDATALEN; hash_ctl.hcxt = CurrentMemoryContext; colinfo->names_hash = hash_create("deparse_columns names", colinfo->num_cols + colinfo->num_new_cols, &hash_ctl, HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); /* * Preload the hash table with any names already present (these would have * come from set_using_names). */ for (i = 0; i < colinfo->num_cols; i++) { char *oldname = colinfo->colnames[i]; if (oldname) add_to_names_hash(colinfo, oldname); } for (i = 0; i < colinfo->num_new_cols; i++) { char *oldname = colinfo->new_colnames[i]; if (oldname) add_to_names_hash(colinfo, oldname); } foreach(lc, colinfo->parentUsing) { char *oldname = (char *) lfirst(lc); add_to_names_hash(colinfo, oldname); } } /* * add_to_names_hash: add a string to the names_hash, if we're using one */ static void add_to_names_hash(deparse_columns *colinfo, const char *name) { if (colinfo->names_hash) (void) hash_search(colinfo->names_hash, name, HASH_ENTER, NULL); } /* * destroy_colinfo_names_hash: destroy hash table when done with it */ static void destroy_colinfo_names_hash(deparse_columns *colinfo) { if (colinfo->names_hash) { hash_destroy(colinfo->names_hash); colinfo->names_hash = NULL; } } /* * identify_join_columns: figure out where columns of a join come from * * Fills the join-specific fields of the colinfo struct, except for * usingNames which is filled later. */ static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo) { int numjoincols; int jcolno; int rcolno; ListCell *lc; /* Extract left/right child RT indexes */ if (IsA(j->larg, RangeTblRef)) colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; else if (IsA(j->larg, JoinExpr)) colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->larg)); if (IsA(j->rarg, RangeTblRef)) colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; else if (IsA(j->rarg, JoinExpr)) colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; else elog(ERROR, "unrecognized node type in jointree: %d", (int) nodeTag(j->rarg)); /* Assert children will be processed earlier than join in second pass */ Assert(colinfo->leftrti < j->rtindex); Assert(colinfo->rightrti < j->rtindex); /* Initialize result arrays with zeroes */ numjoincols = list_length(jrte->joinaliasvars); Assert(numjoincols == list_length(jrte->eref->colnames)); colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); /* * Deconstruct RTE's joinleftcols/joinrightcols into desired format. * Recall that the column(s) merged due to USING are the first column(s) * of the join output. We need not do anything special while scanning * joinleftcols, but while scanning joinrightcols we must distinguish * merged from unmerged columns. */ jcolno = 0; foreach(lc, jrte->joinleftcols) { int leftattno = lfirst_int(lc); colinfo->leftattnos[jcolno++] = leftattno; } rcolno = 0; foreach(lc, jrte->joinrightcols) { int rightattno = lfirst_int(lc); if (rcolno < jrte->joinmergedcols) /* merged column? */ colinfo->rightattnos[rcolno] = rightattno; else colinfo->rightattnos[jcolno++] = rightattno; rcolno++; } Assert(jcolno == numjoincols); } /* * get_rtable_name: convenience function to get a previously assigned RTE alias * * The RTE must belong to the topmost namespace level in "context". */ static char * get_rtable_name(int rtindex, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); return (char *) list_nth(dpns->rtable_names, rtindex - 1); } /* * set_deparse_plan: set up deparse_namespace to parse subexpressions * of a given Plan node * * This sets the plan, outer_planstate, inner_planstate, outer_tlist, * inner_tlist, and index_tlist fields. Caller is responsible for adjusting * the ancestors list if necessary. Note that the rtable and ctes fields do * not need to change when shifting attention to different plan nodes in a * single plan tree. */ static void set_deparse_plan(deparse_namespace *dpns, Plan *plan) { dpns->plan = plan; /* * We special-case Append and MergeAppend to pretend that the first child * plan is the OUTER referent; we have to interpret OUTER Vars in their * tlists according to one of the children, and the first one is the most * natural choice. */ if (IsA(plan, Append)) dpns->outer_plan = linitial(((Append *) plan)->appendplans); else if (IsA(plan, MergeAppend)) dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); else dpns->outer_plan = outerPlan(plan); if (dpns->outer_plan) dpns->outer_tlist = dpns->outer_plan->targetlist; else dpns->outer_tlist = NIL; /* * For a SubqueryScan, pretend the subplan is INNER referent. (We don't * use OUTER because that could someday conflict with the normal meaning.) * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. * For a WorkTableScan, locate the parent RecursiveUnion plan node and use * that as INNER referent. * * For MERGE, pretend the ModifyTable's source plan (its outer plan) is * INNER referent. This is the join from the target relation to the data * source, and all INNER_VAR Vars in other parts of the query refer to its * targetlist. * * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the * excluded expression's tlist. (Similar to the SubqueryScan we don't want * to reuse OUTER, it's used for RETURNING in some modify table cases, * although not INSERT .. CONFLICT). */ if (IsA(plan, SubqueryScan)) dpns->inner_plan = ((SubqueryScan *) plan)->subplan; else if (IsA(plan, CteScan)) dpns->inner_plan = list_nth(dpns->subplans, ((CteScan *) plan)->ctePlanId - 1); else if (IsA(plan, WorkTableScan)) dpns->inner_plan = find_recursive_union(dpns, (WorkTableScan *) plan); else if (IsA(plan, ModifyTable)) { if (((ModifyTable *) plan)->operation == CMD_MERGE) dpns->inner_plan = outerPlan(plan); else dpns->inner_plan = plan; } else dpns->inner_plan = innerPlan(plan); if (IsA(plan, ModifyTable) && ((ModifyTable *) plan)->operation == CMD_INSERT) dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; else if (dpns->inner_plan) dpns->inner_tlist = dpns->inner_plan->targetlist; else dpns->inner_tlist = NIL; /* Set up referent for INDEX_VAR Vars, if needed */ if (IsA(plan, IndexOnlyScan)) dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; else if (IsA(plan, ForeignScan)) dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; else if (IsA(plan, CustomScan)) dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; else dpns->index_tlist = NIL; } /* * Locate the ancestor plan node that is the RecursiveUnion generating * the WorkTableScan's work table. We can match on wtParam, since that * should be unique within the plan tree. */ static Plan * find_recursive_union(deparse_namespace *dpns, WorkTableScan *wtscan) { ListCell *lc; foreach(lc, dpns->ancestors) { Plan *ancestor = (Plan *) lfirst(lc); if (IsA(ancestor, RecursiveUnion) && ((RecursiveUnion *) ancestor)->wtParam == wtscan->wtParam) return ancestor; } elog(ERROR, "could not find RecursiveUnion for WorkTableScan with wtParam %d", wtscan->wtParam); return NULL; } /* * push_child_plan: temporarily transfer deparsing attention to a child plan * * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the * deparse context in case the referenced expression itself uses * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid * affecting levelsup issues (although in a Plan tree there really shouldn't * be any). * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_child_plan. */ static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns) { /* Save state for restoration later */ *save_dpns = *dpns; /* Link current plan node into ancestors list */ dpns->ancestors = lcons(dpns->plan, dpns->ancestors); /* Set attention on selected child */ set_deparse_plan(dpns, plan); } /* * pop_child_plan: undo the effects of push_child_plan */ static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { List *ancestors; /* Get rid of ancestors list cell added by push_child_plan */ ancestors = list_delete_first(dpns->ancestors); /* Restore fields changed by push_child_plan */ *dpns = *save_dpns; /* Make sure dpns->ancestors is right (may be unnecessary) */ dpns->ancestors = ancestors; } /* * push_ancestor_plan: temporarily transfer deparsing attention to an * ancestor plan * * When expanding a Param reference, we must adjust the deparse context * to match the plan node that contains the expression being printed; * otherwise we'd fail if that expression itself contains a Param or * OUTER_VAR/INNER_VAR/INDEX_VAR variable. * * The target ancestor is conveniently identified by the ListCell holding it * in dpns->ancestors. * * Caller must provide a local deparse_namespace variable to save the * previous state for pop_ancestor_plan. */ static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, deparse_namespace *save_dpns) { Plan *plan = (Plan *) lfirst(ancestor_cell); /* Save state for restoration later */ *save_dpns = *dpns; /* Build a new ancestor list with just this node's ancestors */ dpns->ancestors = list_copy_tail(dpns->ancestors, list_cell_number(dpns->ancestors, ancestor_cell) + 1); /* Set attention on selected ancestor */ set_deparse_plan(dpns, plan); } /* * pop_ancestor_plan: undo the effects of push_ancestor_plan */ static void pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) { /* Free the ancestor list made in push_ancestor_plan */ list_free(dpns->ancestors); /* Restore fields changed by push_ancestor_plan */ *dpns = *save_dpns; } /* ---------- * deparse_shard_query - Parse back a query for execution on a shard * * Builds an SQL string to perform the provided query on a specific shard and * places this string into the provided buffer. * ---------- */ void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer) { get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, false, 0, WRAP_COLUMN_DEFAULT, 0); } /* ---------- * get_query_def - Parse back one query parsetree * * query: parsetree to be displayed * buf: output text is appended to buf * parentnamespace: list (initially empty) of outer-level deparse_namespace's * resultDesc: if not NULL, the output tuple descriptor for the view * represented by a SELECT query. We use the column names from it * to label SELECT output columns, in preference to names in the query * colNamesVisible: true if the surrounding context cares about the output * column names at all (as, for example, an EXISTS() context does not); * when false, we can suppress dummy column labels such as "?column?" * prettyFlags: bitmask of PRETTYFLAG_XXX options * wrapColumn: maximum line length, or -1 to disable wrapping * startIndent: initial indentation amount * ---------- */ static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, colNamesVisible, prettyFlags, wrapColumn, startIndent); } /* ---------- * get_query_def_extended - Parse back one query parsetree, optionally * with extension using a shard identifier. * * If distrelid is valid and shardid is positive, the provided shardid is added * any time the provided relid is deparsed, so that the query may be executed * on a placement for the given shard. * ---------- */ static void get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, Oid distrelid, int64 shardid, TupleDesc resultDesc, bool colNamesVisible, int prettyFlags, int wrapColumn, int startIndent) { deparse_context context; deparse_namespace dpns; int rtable_size; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); rtable_size = query->hasGroupRTE ? list_length(query->rtable) - 1 : list_length(query->rtable); /* * Replace any Vars in the query's targetlist and havingQual that * reference GROUP outputs with the underlying grouping expressions. */ if (query->hasGroupRTE) { query->targetList = (List *) flatten_group_exprs(NULL, query, (Node *) query->targetList); query->havingQual = flatten_group_exprs(NULL, query, query->havingQual); } /* * Before we begin to examine the query, acquire locks on referenced * relations, and fix up deleted columns in JOIN RTEs. This ensures * consistent results. Note we assume it's OK to scribble on the passed * querytree! * * We are only deparsing the query (we are not about to execute it), so we * only need AccessShareLock on the relations it mentions. */ AcquireRewriteLocks(query, false, false); /* * Set search_path to NIL so that all objects outside of pg_catalog will be * schema-prefixed. pg_catalog will be added automatically when we call * PushEmptySearchPath(). */ int saveNestLevel = PushEmptySearchPath(); context.buf = buf; context.namespaces = lcons(&dpns, list_copy(parentnamespace)); context.resultDesc = NULL; context.targetList = NIL; context.windowClause = NIL; context.varprefix = (parentnamespace != NIL || rtable_size != 1); context.prettyFlags = prettyFlags; context.wrapColumn = wrapColumn; context.indentLevel = startIndent; context.colNamesVisible = true; context.inGroupBy = false; context.varInOrderBy = false; context.appendparents = NULL; context.distrelid = distrelid; context.shardid = shardid; set_deparse_for_query(&dpns, query, parentnamespace); switch (query->commandType) { case CMD_SELECT: /* We set context.resultDesc only if it's a SELECT */ context.resultDesc = resultDesc; get_select_query_def(query, &context); break; case CMD_UPDATE: get_update_query_def(query, &context); break; case CMD_INSERT: get_insert_query_def(query, &context); break; case CMD_DELETE: get_delete_query_def(query, &context); break; case CMD_MERGE: get_merge_query_def(query, &context); break; case CMD_NOTHING: appendStringInfoString(buf, "NOTHING"); break; case CMD_UTILITY: get_utility_query_def(query, &context); break; default: elog(ERROR, "unrecognized query command type: %d", query->commandType); break; } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } /* ---------- * get_values_def - Parse back a VALUES list * ---------- */ static void get_values_def(List *values_lists, deparse_context *context) { StringInfo buf = context->buf; bool first_list = true; ListCell *vtl; appendStringInfoString(buf, "VALUES "); foreach(vtl, values_lists) { List *sublist = (List *) lfirst(vtl); bool first_col = true; ListCell *lc; if (first_list) first_list = false; else appendStringInfoString(buf, ", "); appendStringInfoChar(buf, '('); foreach(lc, sublist) { Node *col = (Node *) lfirst(lc); if (first_col) first_col = false; else appendStringInfoChar(buf, ','); /* * Print the value. Whole-row Vars need special treatment. */ get_rule_expr_toplevel(col, context, false); } appendStringInfoChar(buf, ')'); } } /* ---------- * get_with_clause - Parse back a WITH clause * ---------- */ static void get_with_clause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; if (query->cteList == NIL) return; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } if (query->hasRecursive) sep = "WITH RECURSIVE "; else sep = "WITH "; foreach(l, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); appendStringInfoString(buf, sep); appendStringInfoString(buf, quote_identifier(cte->ctename)); if (cte->aliascolnames) { bool first = true; ListCell *col; appendStringInfoChar(buf, '('); foreach(col, cte->aliascolnames) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(col)))); } appendStringInfoChar(buf, ')'); } appendStringInfoString(buf, " AS "); switch (cte->ctematerialized) { case CTEMaterializeDefault: break; case CTEMaterializeAlways: appendStringInfoString(buf, "MATERIALIZED "); break; case CTEMaterializeNever: appendStringInfoString(buf, "NOT MATERIALIZED "); break; } appendStringInfoChar(buf, '('); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); appendStringInfoChar(buf, ')'); if (cte->search_clause) { bool first = true; ListCell *lc; appendStringInfo(buf, " SEARCH %s FIRST BY ", cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); foreach(lc, cte->search_clause->search_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); } if (cte->cycle_clause) { bool first = true; ListCell *lc; appendStringInfoString(buf, " CYCLE "); foreach(lc, cte->cycle_clause->cycle_col_list) { if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(strVal(lfirst(lc)))); } appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); { Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) { appendStringInfoString(buf, " TO "); get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); appendStringInfoString(buf, " DEFAULT "); get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); } } appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); } sep = ", "; } if (PRETTY_INDENT(context)) { context->indentLevel -= PRETTYINDENT_STD; appendContextKeyword(context, "", 0, 0, 0); } else appendStringInfoChar(buf, ' '); } /* ---------- * get_select_query_def - Parse back a SELECT parsetree * ---------- */ static void get_select_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; bool force_colno; ListCell *l; /* Insert the WITH clause if given */ get_with_clause(query, context); /* Subroutines may need to consult the SELECT targetlist and windowClause */ context->targetList = query->targetList; context->windowClause = query->windowClause; /* * If the Query node has a setOperations tree, then it's the top level of * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT * fields are interesting in the top query itself. */ if (query->setOperations) { get_setop_query(query->setOperations, query, context); /* ORDER BY clauses must be simple in this case */ force_colno = true; } else { get_basic_select_query(query, context); force_colno = false; } /* Add the ORDER BY clause if given */ if (query->sortClause != NIL) { appendContextKeyword(context, " ORDER BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_orderby(query->sortClause, query->targetList, force_colno, context); } /* * Add the LIMIT/OFFSET clauses if given. If non-default options, use the * standard spelling of LIMIT. */ if (query->limitOffset != NULL) { appendContextKeyword(context, " OFFSET ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitOffset, context, false); } if (query->limitCount != NULL) { if (query->limitOption == LIMIT_OPTION_WITH_TIES) { /* * The limitCount arg is a c_expr, so it needs parens. Simple * literals and function expressions would not need parens, but * unfortunately it's hard to tell if the expression will be * printed as a simple literal like 123 or as a typecast * expression, like '-123'::int4. The grammar accepts the former * without quoting, but not the latter. */ // had to add '(' and ')' here because it fails with casting appendContextKeyword(context, " FETCH FIRST (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); appendStringInfoChar(buf, '('); get_rule_expr(query->limitCount, context, false); appendStringInfoChar(buf, ')'); appendStringInfoString(buf, ") ROWS WITH TIES"); } else { appendContextKeyword(context, " LIMIT ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); if (IsA(query->limitCount, Const) && ((Const *) query->limitCount)->constisnull) appendStringInfoString(buf, "ALL"); else get_rule_expr(query->limitCount, context, false); } } /* Add FOR [KEY] UPDATE/SHARE clauses if present */ if (query->hasForUpdate) { foreach(l, query->rowMarks) { RowMarkClause *rc = (RowMarkClause *) lfirst(l); /* don't print implicit clauses */ if (rc->pushedDown) continue; switch (rc->strength) { case LCS_NONE: /* we intentionally throw an error for LCS_NONE */ elog(ERROR, "unrecognized LockClauseStrength %d", (int) rc->strength); break; case LCS_FORKEYSHARE: appendContextKeyword(context, " FOR KEY SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORSHARE: appendContextKeyword(context, " FOR SHARE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORNOKEYUPDATE: appendContextKeyword(context, " FOR NO KEY UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; case LCS_FORUPDATE: appendContextKeyword(context, " FOR UPDATE", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); break; } appendStringInfo(buf, " OF %s", quote_identifier(get_rtable_name(rc->rti, context))); if (rc->waitPolicy == LockWaitError) appendStringInfoString(buf, " NOWAIT"); else if (rc->waitPolicy == LockWaitSkip) appendStringInfoString(buf, " SKIP LOCKED"); } } } /* * Detect whether query looks like SELECT ... FROM VALUES(); * if so, return the VALUES RTE. Otherwise return NULL. */ static RangeTblEntry * get_simple_values_rte(Query *query, TupleDesc resultDesc) { RangeTblEntry *result = NULL; ListCell *lc; int colno; /* * We want to return true even if the Query also contains OLD or NEW rule * RTEs. So the idea is to scan the rtable and see if there is only one * inFromCl RTE that is a VALUES RTE. */ foreach(lc, query->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); if (rte->rtekind == RTE_VALUES && rte->inFromCl) { if (result) return NULL; /* multiple VALUES (probably not possible) */ result = rte; } else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) continue; /* ignore rule entries */ else return NULL; /* something else -> not simple VALUES */ } /* * We don't need to check the targetlist in any great detail, because * parser/analyze.c will never generate a "bare" VALUES RTE --- they only * appear inside auto-generated sub-queries with very restricted * structure. However, DefineView might have modified the tlist by * injecting new column aliases; so compare tlist resnames against the * RTE's names to detect that. */ if (result) { ListCell *lcn; if (list_length(query->targetList) != list_length(result->eref->colnames)) return NULL; /* this probably cannot happen */ colno = 0; forboth(lc, query->targetList, lcn, result->eref->colnames) { TargetEntry *tle = (TargetEntry *) lfirst(lc); char *cname = strVal(lfirst(lcn)); char *colname; if (tle->resjunk) return NULL; /* this probably cannot happen */ /* compute name that get_target_list would use for column */ colno++; if (resultDesc && colno <= resultDesc->natts) colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); else colname = tle->resname; /* does it match the VALUES RTE? */ if (colname == NULL || strcmp(colname, cname) != 0) return NULL; /* column name has been changed */ } } return result; } static void get_basic_select_query(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *values_rte; char *sep; ListCell *l; if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } /* * If the query looks like SELECT * FROM (VALUES ...), then print just the * VALUES part. This reverses what transformValuesClause() did at parse * time. */ values_rte = get_simple_values_rte(query, context->resultDesc); if (values_rte) { get_values_def(values_rte->values_lists, context); return; } /* * Build up the query string - first we say SELECT */ if (query->isReturn) appendStringInfoString(buf, "RETURN"); else appendStringInfoString(buf, "SELECT"); /* Add the DISTINCT clause if given */ if (query->distinctClause != NIL) { if (query->hasDistinctOn) { appendStringInfoString(buf, " DISTINCT ON ("); sep = ""; foreach(l, query->distinctClause) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, false, context); sep = ", "; } appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DISTINCT"); } /* Then we tell what to select (the targetlist) */ get_target_list(query->targetList, context); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add the WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add the GROUP BY clause if given */ if (query->groupClause != NULL || query->groupingSets != NULL) { bool save_ingroupby; appendContextKeyword(context, " GROUP BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); if (query->groupDistinct) appendStringInfoString(buf, "DISTINCT "); save_ingroupby = context->inGroupBy; context->inGroupBy = true; if (query->groupingSets == NIL) { sep = ""; foreach(l, query->groupClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, false, context); sep = ", "; } } else { sep = ""; foreach(l, query->groupingSets) { GroupingSet *grp = lfirst(l); appendStringInfoString(buf, sep); get_rule_groupingset(grp, query->targetList, true, context); sep = ", "; } } context->inGroupBy = save_ingroupby; } /* Add the HAVING clause if given */ if (query->havingQual != NULL) { appendContextKeyword(context, " HAVING ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->havingQual, context, false); } /* Add the WINDOW clause if needed */ if (query->windowClause != NIL) get_rule_windowclause(query, context); } /* ---------- * get_target_list - Parse back a SELECT target list * * This is also used for RETURNING lists in INSERT/UPDATE/DELETE/MERGE. * ---------- */ static void get_target_list(List *targetList, deparse_context *context) { StringInfo buf = context->buf; StringInfoData targetbuf; bool last_was_multiline = false; char *sep; int colno; ListCell *l; /* we use targetbuf to hold each TLE's text temporarily */ initStringInfo(&targetbuf); sep = " "; colno = 0; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); char *colname; char *attname; if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; colno++; /* * Put the new field text into targetbuf so we can decide after we've * got it whether or not it needs to go on a new line. */ resetStringInfo(&targetbuf); context->buf = &targetbuf; /* * We special-case Var nodes rather than using get_rule_expr. This is * needed because get_rule_expr will display a whole-row Var as * "foo.*", which is the preferred notation in most contexts, but at * the top level of a SELECT list it's not right (the parser will * expand that notation into multiple columns, yielding behavior * different from a whole-row Var). We need to call get_variable * directly so that we can tell it to do the right thing, and so that * we can get the attribute name which is the default AS label. */ if (tle->expr && (IsA(tle->expr, Var))) { attname = get_variable((Var *) tle->expr, 0, true, context); } else { get_rule_expr((Node *) tle->expr, context, true); /* * When colNamesVisible is true, we should always show the * assigned column name explicitly. Otherwise, show it only if * it's not FigureColname's fallback. */ attname = context->colNamesVisible ? NULL : "?column?"; } /* * Figure out what the result column should be called. In the context * of a view, use the view's tuple descriptor (so as to pick up the * effects of any column RENAME that's been done on the view). * Otherwise, just use what we can find in the TLE. */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; /* Show AS unless the column's name is correct as-is */ if (colname) /* resname could be NULL */ { if (attname == NULL || strcmp(attname, colname) != 0) appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); } /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { int leading_nl_pos; /* Does the new field start with a new line? */ if (targetbuf.len > 0 && targetbuf.data[0] == '\n') leading_nl_pos = 0; else leading_nl_pos = -1; /* If so, we shouldn't add anything */ if (leading_nl_pos >= 0) { /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the output buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new field is * not the first and either the new field would cause an * overflow or the last field used more than one line. */ if (colno > 1 && ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || last_was_multiline)) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } /* Remember this field's multiline status for next iteration */ last_was_multiline = (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); } /* Add the new field */ appendStringInfoString(buf, targetbuf.data); } /* clean up */ pfree(targetbuf.data); } static void get_returning_clause(Query *query, deparse_context *context) { StringInfo buf = context->buf; if (query->returningList) { bool have_with = false; appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); /* Add WITH (OLD/NEW) options, if they're not the defaults */ if (query->returningOldAlias && strcmp(query->returningOldAlias, "old") != 0) { appendStringInfo(buf, " WITH (OLD AS %s", quote_identifier(query->returningOldAlias)); have_with = true; } if (query->returningNewAlias && strcmp(query->returningNewAlias, "new") != 0) { if (have_with) appendStringInfo(buf, ", NEW AS %s", quote_identifier(query->returningNewAlias)); else { appendStringInfo(buf, " WITH (NEW AS %s", quote_identifier(query->returningNewAlias)); have_with = true; } } if (have_with) appendStringInfoChar(buf, ')'); /* Add the returning expressions themselves */ get_target_list(query->returningList, context); } } static void get_setop_query(Node *setOp, Query *query, deparse_context *context) { StringInfo buf = context->buf; bool need_paren; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); if (IsA(setOp, RangeTblRef)) { RangeTblRef *rtr = (RangeTblRef *) setOp; RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); Query *subquery = rte->subquery; Assert(subquery != NULL); /* * We need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y. * Also add parens if the leaf query contains its own set operations. * (That shouldn't happen unless one of the other clauses is also * present, see transformSetOperationTree; but let's be safe.) */ need_paren = (subquery->cteList || subquery->sortClause || subquery->rowMarks || subquery->limitOffset || subquery->limitCount || subquery->setOperations); if (need_paren) appendStringInfoChar(buf, '('); get_query_def(subquery, buf, context->namespaces, context->resultDesc, context->colNamesVisible, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoChar(buf, ')'); } else if (IsA(setOp, SetOperationStmt)) { SetOperationStmt *op = (SetOperationStmt *) setOp; int subindent; bool save_colnamesvisible; /* * We force parens when nesting two SetOperationStmts, except when the * lefthand input is another setop of the same kind. Syntactically, * we could omit parens in rather more cases, but it seems best to use * parens to flag cases where the setop operator changes. If we use * parens, we also increase the indentation level for the child query. * * There are some cases in which parens are needed around a leaf query * too, but those are more easily handled at the next level down (see * code above). */ if (IsA(op->larg, SetOperationStmt)) { SetOperationStmt *lop = (SetOperationStmt *) op->larg; if (op->op == lop->op && op->all == lop->all) need_paren = false; else need_paren = true; } else need_paren = false; if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; appendContextKeyword(context, "", subindent, 0, 0); } else subindent = 0; get_setop_query(op->larg, query, context); if (need_paren) appendContextKeyword(context, ") ", -subindent, 0, 0); else if (PRETTY_INDENT(context)) appendContextKeyword(context, "", -subindent, 0, 0); else appendStringInfoChar(buf, ' '); switch (op->op) { case SETOP_UNION: appendStringInfoString(buf, "UNION "); break; case SETOP_INTERSECT: appendStringInfoString(buf, "INTERSECT "); break; case SETOP_EXCEPT: appendStringInfoString(buf, "EXCEPT "); break; default: elog(ERROR, "unrecognized set op: %d", (int) op->op); } if (op->all) appendStringInfoString(buf, "ALL "); /* Always parenthesize if RHS is another setop */ need_paren = IsA(op->rarg, SetOperationStmt); /* * The indentation code here is deliberately a bit different from that * for the lefthand input, because we want the line breaks in * different places. */ if (need_paren) { appendStringInfoChar(buf, '('); subindent = PRETTYINDENT_STD; } else subindent = 0; appendContextKeyword(context, "", subindent, 0, 0); /* * The output column names of the RHS sub-select don't matter. */ save_colnamesvisible = context->colNamesVisible; context->colNamesVisible = false; get_setop_query(op->rarg, query, context); context->colNamesVisible = save_colnamesvisible; if (PRETTY_INDENT(context)) context->indentLevel -= subindent; if (need_paren) appendContextKeyword(context, ")", 0, 0, 0); } else { elog(ERROR, "unrecognized node type: %d", (int) nodeTag(setOp)); } } /* * Display a sort/group clause. * * Also returns the expression tree, so caller need not find it again. */ static Node * get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; TargetEntry *tle; Node *expr; tle = get_sortgroupref_tle(ref, tlist); expr = (Node *) tle->expr; /* * Use column-number form if requested by caller. Otherwise, if * expression is a constant, force it to be dumped with an explicit cast * as decoration --- this is because a simple integer constant is * ambiguous (and will be misinterpreted by findTargetlistEntrySQL92()) if * we dump it without any decoration. Similarly, if it's just a Var, * there is risk of misinterpretation if the column name is reassigned in * the SELECT list, so we may need to force table qualification. And, if * it's anything more complex than a simple Var, then force extra parens * around it, to ensure it can't be misinterpreted as a cube() or rollup() * construct. */ if (force_colno) { Assert(!tle->resjunk); appendStringInfo(buf, "%d", tle->resno); } else if (!expr) /* do nothing, probably can't happen */ ; else if (IsA(expr, Const)) get_const_expr((Const *) expr, context, 1); else if (IsA(expr, Var)) { /* Tell get_variable to check for name conflict */ bool save_varinorderby = context->varInOrderBy; context->varInOrderBy = true; (void) get_variable((Var *) expr, 0, false, context); context->varInOrderBy = save_varinorderby; } else { /* * We must force parens for function-like expressions even if * PRETTY_PAREN is off, since those are the ones in danger of * misparsing. For other expressions we need to force them only if * PRETTY_PAREN is on, since otherwise the expression will output them * itself. (We can't skip the parens.) */ bool need_paren = (PRETTY_PAREN(context) || IsA(expr, FuncExpr) || IsA(expr, Aggref) || IsA(expr, WindowFunc) || IsA(expr, JsonConstructorExpr)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, true); if (need_paren) appendStringInfoChar(context->buf, ')'); } return expr; } /* * Display a GroupingSet */ static void get_rule_groupingset(GroupingSet *gset, List *targetlist, bool omit_parens, deparse_context *context) { ListCell *l; StringInfo buf = context->buf; bool omit_child_parens = true; char *sep = ""; switch (gset->kind) { case GROUPING_SET_EMPTY: appendStringInfoString(buf, "()"); return; case GROUPING_SET_SIMPLE: { if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, '('); foreach(l, gset->content) { Index ref = lfirst_int(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(ref, targetlist, false, context); sep = ", "; } if (!omit_parens || list_length(gset->content) != 1) appendStringInfoChar(buf, ')'); } return; case GROUPING_SET_ROLLUP: appendStringInfoString(buf, "ROLLUP("); break; case GROUPING_SET_CUBE: appendStringInfoString(buf, "CUBE("); break; case GROUPING_SET_SETS: appendStringInfoString(buf, "GROUPING SETS ("); omit_child_parens = false; break; } foreach(l, gset->content) { appendStringInfoString(buf, sep); get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); sep = ", "; } appendStringInfoChar(buf, ')'); } /* * Display an ORDER BY list. */ static void get_rule_orderby(List *orderList, List *targetList, bool force_colno, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = ""; foreach(l, orderList) { SortGroupClause *srt = (SortGroupClause *) lfirst(l); Node *sortexpr; Oid sortcoltype; TypeCacheEntry *typentry; appendStringInfoString(buf, sep); sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, force_colno, context); sortcoltype = exprType(sortexpr); /* See whether operator is default < or > for datatype */ typentry = lookup_type_cache(sortcoltype, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); if (srt->sortop == typentry->lt_opr) { /* ASC is default, so emit nothing for it */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); } else if (srt->sortop == typentry->gt_opr) { appendStringInfoString(buf, " DESC"); /* DESC defaults to NULLS FIRST */ if (!srt->nulls_first) appendStringInfoString(buf, " NULLS LAST"); } else { appendStringInfo(buf, " USING %s", generate_operator_name(srt->sortop, sortcoltype, sortcoltype)); /* be specific to eliminate ambiguity */ if (srt->nulls_first) appendStringInfoString(buf, " NULLS FIRST"); else appendStringInfoString(buf, " NULLS LAST"); } sep = ", "; } } /* * Display a WINDOW clause. * * Note that the windowClause list might contain only anonymous window * specifications, in which case we should print nothing here. */ static void get_rule_windowclause(Query *query, deparse_context *context) { StringInfo buf = context->buf; const char *sep; ListCell *l; sep = NULL; foreach(l, query->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->name == NULL) continue; /* ignore anonymous windows */ if (sep == NULL) appendContextKeyword(context, " WINDOW ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); else appendStringInfoString(buf, sep); appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); get_rule_windowspec(wc, query->targetList, context); sep = ", "; } } /* * Display a window definition */ static void get_rule_windowspec(WindowClause *wc, List *targetList, deparse_context *context) { StringInfo buf = context->buf; bool needspace = false; const char *sep; ListCell *l; appendStringInfoChar(buf, '('); if (wc->refname) { appendStringInfoString(buf, quote_identifier(wc->refname)); needspace = true; } /* partition clauses are always inherited, so only print if no refname */ if (wc->partitionClause && !wc->refname) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "PARTITION BY "); sep = ""; foreach(l, wc->partitionClause) { SortGroupClause *grp = (SortGroupClause *) lfirst(l); appendStringInfoString(buf, sep); get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, false, context); sep = ", "; } needspace = true; } /* print ordering clause only if not inherited */ if (wc->orderClause && !wc->copiedOrder) { if (needspace) appendStringInfoChar(buf, ' '); appendStringInfoString(buf, "ORDER BY "); get_rule_orderby(wc->orderClause, targetList, false, context); needspace = true; } /* framing clause is never inherited, so print unless it's default */ if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) { if (needspace) appendStringInfoChar(buf, ' '); get_window_frame_options(wc->frameOptions, wc->startOffset, wc->endOffset, context); } appendStringInfoChar(buf, ')'); } /* * Append the description of a window's framing options to context->buf */ static void get_window_frame_options(int frameOptions, Node *startOffset, Node *endOffset, deparse_context *context) { StringInfo buf = context->buf; if (frameOptions & FRAMEOPTION_NONDEFAULT) { if (frameOptions & FRAMEOPTION_RANGE) appendStringInfoString(buf, "RANGE "); else if (frameOptions & FRAMEOPTION_ROWS) appendStringInfoString(buf, "ROWS "); else if (frameOptions & FRAMEOPTION_GROUPS) appendStringInfoString(buf, "GROUPS "); else Assert(false); if (frameOptions & FRAMEOPTION_BETWEEN) appendStringInfoString(buf, "BETWEEN "); if (frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) appendStringInfoString(buf, "UNBOUNDED PRECEDING "); else if (frameOptions & FRAMEOPTION_START_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (frameOptions & FRAMEOPTION_START_OFFSET) { get_rule_expr(startOffset, context, false); if (frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); if (frameOptions & FRAMEOPTION_BETWEEN) { appendStringInfoString(buf, "AND "); if (frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); else if (frameOptions & FRAMEOPTION_END_CURRENT_ROW) appendStringInfoString(buf, "CURRENT ROW "); else if (frameOptions & FRAMEOPTION_END_OFFSET) { get_rule_expr(endOffset, context, false); if (frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) appendStringInfoString(buf, " PRECEDING "); else if (frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) appendStringInfoString(buf, " FOLLOWING "); else Assert(false); } else Assert(false); } if (frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); else if (frameOptions & FRAMEOPTION_EXCLUDE_GROUP) appendStringInfoString(buf, "EXCLUDE GROUP "); else if (frameOptions & FRAMEOPTION_EXCLUDE_TIES) appendStringInfoString(buf, "EXCLUDE TIES "); /* we will now have a trailing space; remove it */ buf->data[--(buf->len)] = '\0'; } } /* ---------- * get_insert_query_def - Parse back an INSERT parsetree * ---------- */ static void get_insert_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *select_rte = NULL; RangeTblEntry *values_rte = NULL; RangeTblEntry *rte; ListCell *l; List *strippedexprs = NIL; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * If it's an INSERT ... SELECT or multi-row VALUES, there will be a * single RTE for the SELECT or VALUES. Plain VALUES has neither. */ foreach(l, query->rtable) { rte = (RangeTblEntry *) lfirst(l); if (rte->rtekind == RTE_SUBQUERY) { if (select_rte) elog(ERROR, "too many subquery RTEs in INSERT"); select_rte = rte; } if (rte->rtekind == RTE_VALUES) { if (values_rte) elog(ERROR, "too many values RTEs in INSERT"); values_rte = rte; } } if (select_rte && values_rte) elog(ERROR, "both subquery and values RTEs in INSERT"); /* * Start the query with INSERT INTO relname */ rte = rt_fetch(query->resultRelation, query->rtable); Assert(rte->rtekind == RTE_RELATION); if (PRETTY_INDENT(context)) { context->indentLevel += PRETTYINDENT_STD; appendStringInfoChar(buf, ' '); } appendStringInfo(buf, "INSERT INTO %s", generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed; INSERT requires explicit AS */ get_rte_alias(rte, query->resultRelation, true, context); /* always want a space here */ appendStringInfoChar(buf, ' '); /* * Add the insert-column-names list. Any indirection decoration needed on * the column names can be inferred from the top targetlist. */ if (query->targetList) { strippedexprs = get_insert_column_names_list(query->targetList, buf, context, rte); } if (query->override) { if (query->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); else if (query->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, "OVERRIDING USER VALUE "); } if (select_rte) { /* Add the SELECT */ get_query_def(select_rte->subquery, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); } else if (values_rte) { /* Add the multi-VALUES expression lists */ get_values_def(values_rte->values_lists, context); } else if (strippedexprs) { /* Add the single-VALUES expression list */ appendContextKeyword(context, "VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else { /* No expressions, so it must be DEFAULT VALUES */ appendStringInfoString(buf, "DEFAULT VALUES"); } /* Add ON CONFLICT if present */ if (query->onConflict) { OnConflictExpr *confl = query->onConflict; appendStringInfoString(buf, " ON CONFLICT"); if (confl->arbiterElems) { /* Add the single-VALUES expression list */ appendStringInfoChar(buf, '('); get_rule_expr((Node *) confl->arbiterElems, context, false); appendStringInfoChar(buf, ')'); /* Add a WHERE clause (for partial indexes) if given */ if (confl->arbiterWhere != NULL) { bool save_varprefix; /* * Force non-prefixing of Vars, since parser assumes that they * belong to target relation. WHERE clause does not use * InferenceElem, so this is separately required. */ save_varprefix = context->varprefix; context->varprefix = false; appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->arbiterWhere, context, false); context->varprefix = save_varprefix; } } else if (OidIsValid(confl->constraint)) { char *constraint = get_constraint_name(confl->constraint); int64 shardId = context->shardid; if (shardId > 0) { AppendShardIdToName(&constraint, shardId); } if (!constraint) elog(ERROR, "cache lookup failed for constraint %u", confl->constraint); appendStringInfo(buf, " ON CONSTRAINT %s", quote_identifier(constraint)); } if (confl->action == ONCONFLICT_NOTHING) { appendStringInfoString(buf, " DO NOTHING"); } else { appendStringInfoString(buf, " DO UPDATE SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, confl->onConflictSet, context, rte); /* Add a WHERE clause if given */ if (confl->onConflictWhere != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(confl->onConflictWhere, context, false); } } } /* Add RETURNING if present */ if (query->returningList) { get_returning_clause(query, context); } } /* ---------- * get_update_query_def - Parse back an UPDATE parsetree * ---------- */ static void get_update_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with UPDATE relname SET */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "UPDATE %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } appendStringInfoString(buf, " SET "); /* Deparse targetlist */ get_update_query_targetlist_def(query, query->targetList, context, rte); /* Add the FROM clause if needed */ get_from_clause(query, " FROM ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { get_returning_clause(query, context); } } /* ---------- * get_update_query_targetlist_def - Parse back an UPDATE targetlist * ---------- */ static void get_update_query_targetlist_def(Query *query, List *targetList, deparse_context *context, RangeTblEntry *rte) { StringInfo buf = context->buf; ListCell *l; ListCell *next_ma_cell; int remaining_ma_columns; const char *sep; SubLink *cur_ma_sublink; List *ma_sublinks; targetList = ExpandMergedSubscriptingRefEntries(targetList); /* * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks * into a list. We expect them to appear, in ID order, in resjunk tlist * entries. */ ma_sublinks = NIL; if (query->hasSubLinks) /* else there can't be any */ { foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk && IsA(tle->expr, SubLink)) { SubLink *sl = (SubLink *) tle->expr; if (sl->subLinkType == MULTIEXPR_SUBLINK) { ma_sublinks = lappend(ma_sublinks, sl); Assert(sl->subLinkId == list_length(ma_sublinks)); } } } ensure_update_targetlist_in_param_order(targetList); } next_ma_cell = list_head(ma_sublinks); cur_ma_sublink = NULL; remaining_ma_columns = 0; /* Add the comma separated list of 'attname = value' */ sep = ""; foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *expr; if (tle->resjunk) continue; /* ignore junk entries */ /* Emit separator (OK whether we're in multiassignment or not) */ appendStringInfoString(buf, sep); sep = ", "; /* * Check to see if we're starting a multiassignment group: if so, * output a left paren. */ if (next_ma_cell != NULL && cur_ma_sublink == NULL) { /* * We must dig down into the expr to see if it's a PARAM_MULTIEXPR * Param. That could be buried under FieldStores and * SubscriptingRefs and CoerceToDomains (cf processIndirection()), * and underneath those there could be an implicit type coercion. * Because we would ignore implicit type coercions anyway, we * don't need to be as careful as processIndirection() is about * descending past implicit CoerceToDomains. */ expr = (Node *) tle->expr; while (expr) { if (IsA(expr, FieldStore)) { FieldStore *fstore = (FieldStore *) expr; expr = (Node *) linitial(fstore->newvals); } else if (IsA(expr, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) expr; if (sbsref->refassgnexpr == NULL) break; expr = (Node *) sbsref->refassgnexpr; } else if (IsA(expr, CoerceToDomain)) { CoerceToDomain *cdomain = (CoerceToDomain *) expr; if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; expr = (Node *) cdomain->arg; } else break; } expr = strip_implicit_coercions(expr); if (expr && IsA(expr, Param) && ((Param *) expr)->paramkind == PARAM_MULTIEXPR) { cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); next_ma_cell = lnext(ma_sublinks, next_ma_cell); remaining_ma_columns = count_nonjunk_tlist_entries( ((Query *) cur_ma_sublink->subselect)->targetList); Assert(((Param *) expr)->paramid == ((cur_ma_sublink->subLinkId << 16) | 1)); appendStringInfoChar(buf, '('); } } /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. */ expr = processIndirection((Node *) tle->expr, context); /* * If we're in a multiassignment, skip printing anything more, unless * this is the last column; in which case, what we print should be the * sublink, not the Param. */ if (cur_ma_sublink != NULL) { if (--remaining_ma_columns > 0) continue; /* not the last column of multiassignment */ appendStringInfoChar(buf, ')'); expr = (Node *) cur_ma_sublink; cur_ma_sublink = NULL; } appendStringInfoString(buf, " = "); get_rule_expr(expr, context, false); } } /* ---------- * get_delete_query_def - Parse back a DELETE parsetree * ---------- */ static void get_delete_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with DELETE FROM relname */ rte = rt_fetch(query->resultRelation, query->rtable); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "DELETE FROM %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); /* Print the relation alias, if needed */ get_rte_alias(rte, query->resultRelation, false, context); } /* Add the USING clause if given */ get_from_clause(query, " USING ", context); /* Add a WHERE clause if given */ if (query->jointree->quals != NULL) { appendContextKeyword(context, " WHERE ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_rule_expr(query->jointree->quals, context, false); } /* Add RETURNING if present */ if (query->returningList) { get_returning_clause(query, context); } } /* ---------- * get_merge_query_def - Parse back a MERGE parsetree * ---------- */ static void get_merge_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; ListCell *lc; bool haveNotMatchedBySource; /* Insert the WITH clause if given */ get_with_clause(query, context); /* * Start the query with MERGE INTO relname */ rte = ExtractResultRelationRTE(query); if (PRETTY_INDENT(context)) { appendStringInfoChar(buf, ' '); context->indentLevel += PRETTYINDENT_STD; } /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); if(rte->eref != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } else { appendStringInfo(buf, "MERGE INTO %s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, NIL)); if (rte->alias != NULL) appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(query->resultRelation, context))); } /* Print the source relation and join clause */ get_from_clause(query, " USING ", context); appendContextKeyword(context, " ON ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); get_rule_expr(query->mergeJoinCondition, context, false); /* * Test for any NOT MATCHED BY SOURCE actions. If there are none, then * any NOT MATCHED BY TARGET actions are output as "WHEN NOT MATCHED", per * SQL standard. Otherwise, we have a non-SQL-standard query, so output * "BY SOURCE" / "BY TARGET" qualifiers for all NOT MATCHED actions, to be * more explicit. */ haveNotMatchedBySource = false; foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); if (action->matchKind == MERGE_WHEN_NOT_MATCHED_BY_SOURCE) { haveNotMatchedBySource = true; break; } } /* Print each merge action */ foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); appendContextKeyword(context, " WHEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); switch (action->matchKind) { case MERGE_WHEN_MATCHED: appendStringInfoString(buf, "MATCHED"); break; case MERGE_WHEN_NOT_MATCHED_BY_SOURCE: appendStringInfoString(buf, "NOT MATCHED BY SOURCE"); break; case MERGE_WHEN_NOT_MATCHED_BY_TARGET: if (haveNotMatchedBySource) appendStringInfoString(buf, "NOT MATCHED BY TARGET"); else appendStringInfoString(buf, "NOT MATCHED"); break; default: elog(ERROR, "unrecognized matchKind: %d", (int) action->matchKind); } if (action->qual) { appendContextKeyword(context, " AND ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); get_rule_expr(action->qual, context, false); } appendContextKeyword(context, " THEN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 3); if (action->commandType == CMD_INSERT) { /* This generally matches get_insert_query_def() */ List *strippedexprs = NIL; const char *sep = ""; ListCell *lc2; appendStringInfoString(buf, "INSERT"); if (action->targetList) appendStringInfoString(buf, " ("); foreach(lc2, action->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc2); Assert(!tle->resjunk); appendStringInfoString(buf, sep); sep = ", "; appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } if (action->targetList) appendStringInfoChar(buf, ')'); if (action->override) { if (action->override == OVERRIDING_SYSTEM_VALUE) appendStringInfoString(buf, " OVERRIDING SYSTEM VALUE"); else if (action->override == OVERRIDING_USER_VALUE) appendStringInfoString(buf, " OVERRIDING USER VALUE"); } if (strippedexprs) { appendContextKeyword(context, " VALUES (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 4); get_rule_list_toplevel(strippedexprs, context, false); appendStringInfoChar(buf, ')'); } else appendStringInfoString(buf, " DEFAULT VALUES"); } else if (action->commandType == CMD_UPDATE) { appendStringInfoString(buf, "UPDATE SET "); get_update_query_targetlist_def(query, action->targetList, context, rte); } else if (action->commandType == CMD_DELETE) appendStringInfoString(buf, "DELETE"); else if (action->commandType == CMD_NOTHING) appendStringInfoString(buf, "DO NOTHING"); } /* Add RETURNING if present */ if (query->returningList) { appendContextKeyword(context, " RETURNING", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); get_target_list(query->returningList, context); } ereport(DEBUG1, (errmsg("", buf->data))); } /* ---------- * get_utility_query_def - Parse back a UTILITY parsetree * ---------- */ static void get_utility_query_def(Query *query, deparse_context *context) { StringInfo buf = context->buf; if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) { NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "NOTIFY %s", quote_identifier(stmt->conditionname)); if (stmt->payload) { appendStringInfoString(buf, ", "); simple_quote_literal(buf, stmt->payload); } } else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) { TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; List *relationList = stmt->relations; ListCell *relationCell = NULL; appendContextKeyword(context, "", 0, PRETTYINDENT_STD, 1); appendStringInfo(buf, "TRUNCATE TABLE"); foreach(relationCell, relationList) { RangeVar *relationVar = (RangeVar *) lfirst(relationCell); Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); char *relationName = generate_relation_or_shard_name(relationId, context->distrelid, context->shardid, NIL); appendStringInfo(buf, " %s", relationName); if (lnext(relationList, relationCell) != NULL) { appendStringInfo(buf, ","); } } if (stmt->restart_seqs) { appendStringInfo(buf, " RESTART IDENTITY"); } if (stmt->behavior == DROP_CASCADE) { appendStringInfo(buf, " CASCADE"); } } else { /* Currently only NOTIFY utility commands can appear in rules */ elog(ERROR, "unexpected utility statement type"); } } /* * Display a Var appropriately. * * In some cases (currently only when recursing into an unnamed join) * the Var's varlevelsup has to be interpreted with respect to a context * above the current one; levelsup indicates the offset. * * If istoplevel is true, the Var is at the top level of a SELECT's * targetlist, which means we need special treatment of whole-row Vars. * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a * dirty hack to prevent "tab.*" from being expanded into multiple columns. * (The parser will strip the useless coercion, so no inefficiency is added in * dump and reload.) We used to print just "tab" in such cases, but that is * ambiguous and will yield the wrong result if "tab" is also a plain column * name in the query. * * Returns the attname of the Var, or NULL if the Var has no attname (because * it is a whole-row Var or a subplan output reference). */ static char * get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) { StringInfo buf = context->buf; RangeTblEntry *rte; AttrNumber attnum; int varno; AttrNumber varattno; int netlevelsup; deparse_namespace *dpns; deparse_columns *colinfo; char *refname; char *attname; bool need_prefix; /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. Also * find the aliases previously assigned for this RTE. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { /* * We might have been asked to map child Vars to some parent relation. */ if (context->appendparents && dpns->appendrels) { int pvarno = varno; AttrNumber pvarattno = varattno; AppendRelInfo *appinfo = dpns->appendrels[pvarno]; bool found = false; /* Only map up to inheritance parents, not UNION ALL appendrels */ while (appinfo && rt_fetch(appinfo->parent_relid, dpns->rtable)->rtekind == RTE_RELATION) { found = false; if (pvarattno > 0) /* system columns stay as-is */ { if (pvarattno > appinfo->num_child_cols) break; /* safety check */ pvarattno = appinfo->parent_colnos[pvarattno - 1]; if (pvarattno == 0) break; /* Var is local to child */ } pvarno = appinfo->parent_relid; found = true; /* If the parent is itself a child, continue up. */ Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); appinfo = dpns->appendrels[pvarno]; } /* * If we found an ancestral rel, and that rel is included in * appendparents, print that column not the original one. */ if (found && bms_is_member(pvarno, context->appendparents)) { varno = pvarno; varattno = pvarattno; } } rte = rt_fetch(varno, dpns->rtable); /* might be returning old/new column value */ if (var->varreturningtype == VAR_RETURNING_OLD) refname = dpns->ret_old_alias; else if (var->varreturningtype == VAR_RETURNING_NEW) refname = dpns->ret_new_alias; else refname = (char *) list_nth(dpns->rtable_names, varno - 1); colinfo = deparse_columns_fetch(varno, dpns); attnum = varattno; } else { resolve_special_varno((Node *) var, context, get_special_variable, NULL); return NULL; } /* * The planner will sometimes emit Vars referencing resjunk elements of a * subquery's target list (this is currently only possible if it chooses * to generate a "physical tlist" for a SubqueryScan or CteScan node). * Although we prefer to print subquery-referencing Vars using the * subquery's alias, that's not possible for resjunk items since they have * no alias. So in that case, drill down to the subplan and print the * contents of the referenced tlist item. This works because in a plan * tree, such Vars can only occur in a SubqueryScan or CteScan node, and * we'll have set dpns->inner_plan to reference the child plan node. */ if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && attnum > list_length(rte->eref->colnames) && dpns->inner_plan) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); /* * Force parentheses because our caller probably assumed a Var is a * simple expression. */ if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, '('); get_rule_expr((Node *) tle->expr, context, true); if (!IsA(tle->expr, Var)) appendStringInfoChar(buf, ')'); pop_child_plan(dpns, &save_dpns); return NULL; } /* * If it's an unnamed join, look at the expansion of the alias variable. * If it's a simple reference to one of the input vars, then recursively * print the name of that var instead. When it's not a simple reference, * we have to just print the unqualified join column name. (This can only * happen with "dangerous" merged columns in a JOIN USING; we took pains * previously to make the unqualified column name unique in such cases.) * * This wouldn't work in decompiling plan trees, because we don't store * joinaliasvars lists after planning; but a plan tree should never * contain a join alias variable. */ if (rte->rtekind == RTE_JOIN && rte->alias == NULL) { if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); if (attnum > 0) { Var *aliasvar; aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); /* we intentionally don't strip implicit coercions here */ if (aliasvar && IsA(aliasvar, Var)) { return get_variable(aliasvar, var->varlevelsup + levelsup, istoplevel, context); } } /* * Unnamed join has no refname. (Note: since it's unnamed, there is * no way the user could have referenced it to create a whole-row Var * for it. So we don't have to cover that case below.) */ Assert(refname == NULL); } if (attnum == InvalidAttrNumber) attname = NULL; else if (attnum > 0) { /* Get column name to use from the colinfo struct */ if (attnum > colinfo->num_cols) elog(ERROR, "invalid attnum %d for relation \"%s\"", attnum, rte->eref->aliasname); attname = colinfo->colnames[attnum - 1]; /* * If we find a Var referencing a dropped column, it seems better to * print something (anything) than to fail. In general this should * not happen, but it used to be possible for some cases involving * functions returning named composite types, and perhaps there are * still bugs out there. */ if (attname == NULL) attname = "?dropped?column?"; } else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* System column on a Citus shard */ attname = get_attname(rte->relid, attnum, false); } else { /* System column - name is fixed, get it from the catalog */ attname = get_rte_attribute_name(rte, attnum); } need_prefix = (context->varprefix || attname == NULL || var->varreturningtype != VAR_RETURNING_DEFAULT); /* * If we're considering a plain Var in an ORDER BY (but not GROUP BY) * clause, we may need to add a table-name prefix to prevent * findTargetlistEntrySQL92 from misinterpreting the name as an * output-column name. To avoid cluttering the output with unnecessary * prefixes, do so only if there is a name match to a SELECT tlist item * that is different from the Var. */ if (context->varInOrderBy && !context->inGroupBy && !need_prefix) { int colno = 0; foreach_node(TargetEntry, tle, context->targetList) { char *colname; if (tle->resjunk) continue; /* ignore junk entries */ colno++; /* This must match colname-choosing logic in get_target_list() */ if (context->resultDesc && colno <= context->resultDesc->natts) colname = NameStr(TupleDescAttr(context->resultDesc, colno - 1)->attname); else colname = tle->resname; if (colname && strcmp(colname, attname) == 0 && !equal(var, tle->expr)) { need_prefix = true; break; } } } if (refname && need_prefix) { appendStringInfoString(buf, quote_identifier(refname)); appendStringInfoChar(buf, '.'); } if (attname) appendStringInfoString(buf, quote_identifier(attname)); else { appendStringInfoChar(buf, '*'); if (istoplevel) { if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { /* use rel.*::shard_name instead of rel.*::table_name */ appendStringInfo(buf, "::%s", generate_rte_shard_name(rte)); } else { appendStringInfo(buf, "::%s", format_type_with_typemod(var->vartype, var->vartypmod)); } } } return attname; } /* Any named join with joinaliasvars hides its inner aliases. */ static inline bool dpns_has_named_join(const deparse_namespace *dpns) { if (!dpns || dpns->rtable == NIL) return false; ListCell *lc; foreach (lc, dpns->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); if (rte && rte->rtekind == RTE_JOIN && rte->alias != NULL && rte->joinaliasvars != NIL) return true; } return false; } /* Unwrap trivial wrappers around a Var; return Var* or NULL. */ static Var * unwrap_simple_var(Node *expr) { for (;;) { if (expr == NULL) return NULL; if (IsA(expr, Var)) return (Var *) expr; if (IsA(expr, RelabelType)) { expr = (Node *) ((RelabelType *) expr)->arg; continue; } if (IsA(expr, CoerceToDomain)) { expr = (Node *) ((CoerceToDomain *) expr)->arg; continue; } if (IsA(expr, CollateExpr)) { expr = (Node *) ((CollateExpr *) expr)->arg; continue; } /* Not a simple Var */ return NULL; } } /* Base identity (canonical/synonym) match against a wanted (varno,attno) pair. */ static inline bool var_matches_base(const Var *v, Index want_varno, AttrNumber want_attno) { if (v->varlevelsup != 0) return false; if (v->varno == want_varno && v->varattno == want_attno) return true; if (v->varnosyn > 0 && v->varattnosyn > 0 && v->varnosyn == want_varno && v->varattnosyn == want_attno) return true; return false; } /* Mutate v in place: if v maps to a named JOIN's column, set varnosyn/varattnosyn. * Returns true iff SYN fields were set. Query deparse only (dpns->plan == NULL). */ static void map_var_through_join_alias(deparse_namespace *dpns, Var *v) { if (!dpns || dpns->plan != NULL || !v || v->varlevelsup != 0 || v->varattno <= 0) return; int rti = 0; ListCell *lc; foreach (lc, dpns->rtable) { rti++; RangeTblEntry *jrte = (RangeTblEntry *) lfirst(lc); if (!jrte || jrte->rtekind != RTE_JOIN || jrte->alias == NULL || jrte->joinaliasvars == NIL) continue; AttrNumber jattno = 0; ListCell *vlc; foreach (vlc, jrte->joinaliasvars) { jattno++; Var *aliasVar = unwrap_simple_var((Node *) lfirst(vlc)); if (!aliasVar) continue; if (var_matches_base(aliasVar, v->varno, v->varattno)) { v->varnosyn = (Index) rti; v->varattnosyn = jattno; return; } } } } /* * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This * routine is actually a callback for get_special_varno, which handles finding * the correct TargetEntry. We get the expression contained in that * TargetEntry and just need to deparse it, a job we can throw back on * get_rule_expr. */ static void get_special_variable(Node *node, deparse_context *context, void *callback_arg) { StringInfo buf = context->buf; /* * For a non-Var referent, force parentheses because our caller probably * assumed a Var is a simple expression. */ if (!IsA(node, Var)) appendStringInfoChar(buf, '('); get_rule_expr(node, context, true); if (!IsA(node, Var)) appendStringInfoChar(buf, ')'); } /* * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, * invoke the callback provided. */ static void resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) { Var *var; deparse_namespace *dpns; /* This function is recursive, so let's be paranoid. */ check_stack_depth(); /* If it's not a Var, invoke the callback. */ if (!IsA(node, Var)) { (*callback) (node, context, callback_arg); return; } /* Find appropriate nesting depth */ var = (Var *) node; dpns = (deparse_namespace *) list_nth(context->namespaces, var->varlevelsup); /* * It's a special RTE, so recurse. */ if (var->varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; Bitmapset *save_appendparents; tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); /* If we're descending to the first child of an Append or MergeAppend, * update appendparents. This will affect deparsing of all Vars * appearing within the eventually-resolved subexpression. */ save_appendparents = context->appendparents; if (IsA(dpns->plan, Append)) context->appendparents = bms_union(context->appendparents, ((Append *) dpns->plan)->apprelids); else if (IsA(dpns->plan, MergeAppend)) context->appendparents = bms_union(context->appendparents, ((MergeAppend *) dpns->plan)->apprelids); push_child_plan(dpns, dpns->outer_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); context->appendparents = save_appendparents; return; } else if (var->varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); push_child_plan(dpns, dpns->inner_plan, &save_dpns); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); return; } else if (var->varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; tle = get_tle_by_resno(dpns->index_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); return; } else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) elog(ERROR, "bogus varno: %d", var->varno); /* Not special. Just invoke the callback. */ (*callback) (node, context, callback_arg); } /* * Get the name of a field of an expression of composite type. The * expression is usually a Var, but we handle other cases too. * * levelsup is an extra offset to interpret the Var's varlevelsup correctly. * * This is fairly straightforward when the expression has a named composite * type; we need only look up the type in the catalogs. However, the type * could also be RECORD. Since no actual table or view column is allowed to * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE * or to a subquery output. We drill down to find the ultimate defining * expression and attempt to infer the field name from it. We ereport if we * can't determine the name. * * Similarly, a PARAM of type RECORD has to refer to some expression of * a determinable composite type. */ static const char * get_name_for_var_field(Var *var, int fieldno, int levelsup, deparse_context *context) { RangeTblEntry *rte; AttrNumber attnum; int netlevelsup; deparse_namespace *dpns; int varno; AttrNumber varattno; TupleDesc tupleDesc; Node *expr; /* * If it's a RowExpr that was expanded from a whole-row Var, use the * column names attached to it. (We could let get_expr_result_tupdesc() * handle this, but it's much cheaper to just pull out the name we need.) */ if (IsA(var, RowExpr)) { RowExpr *r = (RowExpr *) var; if (fieldno > 0 && fieldno <= list_length(r->colnames)) return strVal(list_nth(r->colnames, fieldno - 1)); } /* * If it's a Param of type RECORD, try to find what the Param refers to. */ if (IsA(var, Param)) { Param *param = (Param *) var; ListCell *ancestor_cell; expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so recurse to decipher the field name */ deparse_namespace save_dpns; const char *result; push_ancestor_plan(dpns, ancestor_cell, &save_dpns); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); pop_ancestor_plan(dpns, &save_dpns); return result; } } /* * If it's a Var of type RECORD, we have to find what the Var refers to; * if not, we can use get_expr_result_tupdesc(). */ if (!IsA(var, Var) || var->vartype != RECORDOID) { tupleDesc = get_expr_result_tupdesc((Node *) var, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* Find appropriate nesting depth */ netlevelsup = var->varlevelsup + levelsup; if (netlevelsup >= list_length(context->namespaces)) elog(ERROR, "bogus varlevelsup: %d offset %d", var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); varno = var->varno; varattno = var->varattno; if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); /* * if the rte var->varnosyn points to is not a regular table and it is a join * then the correct relname will be found with var->varnosyn and var->varattnosyn */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; } } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. */ if (varno >= 1 && varno <= list_length(dpns->rtable)) { rte = rt_fetch(varno, dpns->rtable); attnum = varattno; } else if (varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->outer_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->outer_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; tle = get_tle_by_resno(dpns->inner_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } else if (varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; const char *result; tle = get_tle_by_resno(dpns->index_tlist, varattno); if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); Assert(netlevelsup == 0); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); return result; } else { elog(ERROR, "bogus varno: %d", varno); return NULL; /* keep compiler quiet */ } if (attnum == InvalidAttrNumber) { /* Var is whole-row reference to RTE, so select the right field */ return get_rte_attribute_name(rte, fieldno); } /* * This part has essentially the same logic as the parser's * expandRecordVariable() function, but we are dealing with a different * representation of the input context, and we only need one field name * not a TupleDesc. Also, we need special cases for finding subquery and * CTE subplans when deparsing Plan trees. */ expr = (Node *) var; /* default if we can't drill down */ switch (rte->rtekind) { case RTE_RELATION: case RTE_VALUES: case RTE_NAMEDTUPLESTORE: case RTE_RESULT: case RTE_SUBQUERY: /* Subselect-in-FROM: examine sub-select's output expr */ { if (rte->subquery) { TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "subquery %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the sub-select to see what its Var * refers to. We have to build an additional level of * namespace to keep in step with varlevelsup in the * subselect; furthermore, the subquery RTE might be * from an outer query level, in which case the * namespace for the subselect must have that outer * level as parent namespace. */ List *save_nslist = context->namespaces; List *parent_namespaces; deparse_namespace mydpns; const char *result; parent_namespaces = list_copy_tail(context->namespaces, netlevelsup); set_deparse_for_query(&mydpns, rte->subquery, parent_namespaces); context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = save_nslist; return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have complete * RTE entries (in particular, rte->subquery is NULL). But * the only place we'd normally see a Var directly * referencing a SUBQUERY RTE is in a SubqueryScan plan * node, and we can look into the child plan's tlist * instead. An exception occurs if the subquery was * proven empty and optimized away: then we'd find such a * Var in a childless Result node, and there's nothing in * the plan tree that would let us figure out what it had * originally referenced. In that case, fall back on * printing "fN", analogously to the default column names * for RowExprs. */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) { char *dummy_name = palloc(32); Assert(dpns->plan && IsA(dpns->plan, Result)); snprintf(dummy_name, 32, "f%d", fieldno); return dummy_name; } Assert(dpns->plan && IsA(dpns->plan, SubqueryScan)); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; case RTE_JOIN: /* Join RTE --- recursively inspect the alias variable */ if (rte->joinaliasvars == NIL) elog(ERROR, "cannot decompile join alias var in plan tree"); Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); Assert(expr != NULL); /* we intentionally don't strip implicit coercions here */ if (IsA(expr, Var)) return get_name_for_var_field((Var *) expr, fieldno, var->varlevelsup + levelsup, context); /* else fall through to inspect the expression */ break; case RTE_FUNCTION: case RTE_TABLEFUNC: /* * We couldn't get here unless a function is declared with one of * its result columns as RECORD, which is not allowed. */ break; case RTE_CTE: /* CTE reference: examine subquery's output expr */ { CommonTableExpr *cte = NULL; Index ctelevelsup; ListCell *lc; /* * Try to find the referenced CTE using the namespace stack. */ ctelevelsup = rte->ctelevelsup + netlevelsup; if (ctelevelsup >= list_length(context->namespaces)) lc = NULL; else { deparse_namespace *ctedpns; ctedpns = (deparse_namespace *) list_nth(context->namespaces, ctelevelsup); foreach(lc, ctedpns->ctes) { cte = (CommonTableExpr *) lfirst(lc); if (strcmp(cte->ctename, rte->ctename) == 0) break; } } if (lc != NULL) { Query *ctequery = (Query *) cte->ctequery; TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), attnum); if (ste == NULL || ste->resjunk) elog(ERROR, "CTE %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) { /* * Recurse into the CTE to see what its Var refers to. * We have to build an additional level of namespace * to keep in step with varlevelsup in the CTE; * furthermore it could be an outer CTE (compare * SUBQUERY case above). */ List *save_nslist = context->namespaces; List *parent_namespaces; deparse_namespace mydpns; const char *result; parent_namespaces = list_copy_tail(context->namespaces, ctelevelsup); set_deparse_for_query(&mydpns, ctequery, parent_namespaces); context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); context->namespaces = save_nslist; return result; } /* else fall through to inspect the expression */ } else { /* * We're deparsing a Plan tree so we don't have a CTE * list. But the only places we'd normally see a Var * directly referencing a CTE RTE are in CteScan or * WorkTableScan plan nodes. For those cases, * set_deparse_plan arranged for dpns->inner_plan to be * the plan node that emits the CTE or RecursiveUnion * result, and we can look at its tlist instead. As * above, this can fail if the CTE has been proven empty, * in which case fall back to "fN". */ TargetEntry *tle; deparse_namespace save_dpns; const char *result; if (!dpns->inner_plan) { char *dummy_name = palloc(32); Assert(dpns->plan && IsA(dpns->plan, Result)); snprintf(dummy_name, 32, "f%d", fieldno); return dummy_name; } Assert(dpns->plan && (IsA(dpns->plan, CteScan) || IsA(dpns->plan, WorkTableScan))); tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); pop_child_plan(dpns, &save_dpns); return result; } } break; case RTE_GROUP: /* * We couldn't get here: any Vars that reference the RTE_GROUP RTE * should have been replaced with the underlying grouping * expressions. */ break; } /* * We now have an expression we can't expand any more, so see if * get_expr_result_tupdesc() can do anything with it. */ tupleDesc = get_expr_result_tupdesc(expr, false); /* Got the tupdesc, so we can extract the field name */ Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); } /* * Try to find the referenced expression for a PARAM_EXEC Param that might * reference a parameter supplied by an upper NestLoop or SubPlan plan node. * * If successful, return the expression and set *dpns_p and *ancestor_cell_p * appropriately for calling push_ancestor_plan(). If no referent can be * found, return NULL. */ static Node * find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p) { /* Initialize output parameters to prevent compiler warnings */ *dpns_p = NULL; *ancestor_cell_p = NULL; /* * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or * SubPlan argument. This will necessarily be in some ancestor of the * current expression's Plan. */ if (param->paramkind == PARAM_EXEC) { deparse_namespace *dpns; Plan *child_plan; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); child_plan = dpns->plan; foreach(lc, dpns->ancestors) { Node *ancestor = (Node *) lfirst(lc); ListCell *lc2; /* * NestLoops transmit params to their inner child only. */ if (IsA(ancestor, NestLoop) && child_plan == innerPlan(ancestor)) { NestLoop *nl = (NestLoop *) ancestor; foreach(lc2, nl->nestParams) { NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); if (nlp->paramno == param->paramid) { /* Found a match, so return it */ *dpns_p = dpns; *ancestor_cell_p = lc; return (Node *) nlp->paramval; } } } /* * Check to see if we're crawling up from a subplan. */ if(IsA(ancestor, SubPlan)) { SubPlan *subplan = (SubPlan *) ancestor; ListCell *lc3; ListCell *lc4; /* Matched subplan, so check its arguments */ forboth(lc3, subplan->parParam, lc4, subplan->args) { int paramid = lfirst_int(lc3); Node *arg = (Node *) lfirst(lc4); if (paramid == param->paramid) { /* * Found a match, so return it. But, since Vars in * the arg are to be evaluated in the surrounding * context, we have to point to the next ancestor item * that is *not* a SubPlan. */ ListCell *rest; for_each_cell(rest, dpns->ancestors, lnext(dpns->ancestors, lc)) { Node *ancestor2 = (Node *) lfirst(rest); if (!IsA(ancestor2, SubPlan)) { *dpns_p = dpns; *ancestor_cell_p = rest; return arg; } } elog(ERROR, "SubPlan cannot be outermost ancestor"); } } /* SubPlan isn't a kind of Plan, so skip the rest */ continue; } /* * We need not consider the ancestor's initPlan list, since * initplans never have any parParams. */ /* No luck, crawl up to next ancestor */ child_plan = (Plan *) ancestor; } } /* No referent found */ return NULL; } /* * Try to find a subplan/initplan that emits the value for a PARAM_EXEC Param. * * If successful, return the generating subplan/initplan and set *column_p * to the subplan's 0-based output column number. * Otherwise, return NULL. */ static SubPlan * find_param_generator(Param *param, deparse_context *context, int *column_p) { /* Initialize output parameter to prevent compiler warnings */ *column_p = 0; /* * If it's a PARAM_EXEC parameter, search the current plan node as well as * ancestor nodes looking for a subplan or initplan that emits the value * for the Param. It could appear in the setParams of an initplan or * MULTIEXPR_SUBLINK subplan, or in the paramIds of an ancestral SubPlan. */ if (param->paramkind == PARAM_EXEC) { SubPlan *result; deparse_namespace *dpns; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); /* First check the innermost plan node's initplans */ result = find_param_generator_initplan(param, dpns->plan, column_p); if (result) return result; /* * The plan's targetlist might contain MULTIEXPR_SUBLINK SubPlans, * which can be referenced by Params elsewhere in the targetlist. * (Such Params should always be in the same targetlist, so there's no * need to do this work at upper plan nodes.) */ foreach_node(TargetEntry, tle, dpns->plan->targetlist) { if (tle->expr && IsA(tle->expr, SubPlan)) { SubPlan *subplan = (SubPlan *) tle->expr; if (subplan->subLinkType == MULTIEXPR_SUBLINK) { foreach_int(paramid, subplan->setParam) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } } } } /* No luck, so check the ancestor nodes */ foreach(lc, dpns->ancestors) { Node *ancestor = (Node *) lfirst(lc); /* * If ancestor is a SubPlan, check the paramIds it provides. */ if (IsA(ancestor, SubPlan)) { SubPlan *subplan = (SubPlan *) ancestor; foreach_int(paramid, subplan->paramIds) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } /* SubPlan isn't a kind of Plan, so skip the rest */ continue; } /* * Otherwise, it's some kind of Plan node, so check its initplans. */ result = find_param_generator_initplan(param, (Plan *) ancestor, column_p); if (result) return result; /* No luck, crawl up to next ancestor */ } } /* No generator found */ return NULL; } /* * Subroutine for find_param_generator: search one Plan node's initplans */ static SubPlan * find_param_generator_initplan(Param *param, Plan *plan, int *column_p) { foreach_node(SubPlan, subplan, plan->initPlan) { foreach_int(paramid, subplan->setParam) { if (paramid == param->paramid) { /* Found a match, so return it. */ *column_p = foreach_current_index(paramid); return subplan; } } } return NULL; } /* * Display a Param appropriately. */ static void get_parameter(Param *param, deparse_context *context) { Node *expr; deparse_namespace *dpns; ListCell *ancestor_cell; SubPlan *subplan; int column; /* * If it's a PARAM_EXEC parameter, try to locate the expression from which * the parameter was computed. This stanza handles only cases in which * the Param represents an input to the subplan we are currently in. */ expr = find_param_referent(param, context, &dpns, &ancestor_cell); if (expr) { /* Found a match, so print it */ deparse_namespace save_dpns; bool save_varprefix; bool need_paren; /* Switch attention to the ancestor plan node */ push_ancestor_plan(dpns, ancestor_cell, &save_dpns); /* * Force prefixing of Vars, since they won't belong to the relation * being scanned in the original plan node. */ save_varprefix = context->varprefix; context->varprefix = true; /* * A Param's expansion is typically a Var, Aggref, GroupingFunc, or * upper-level Param, which wouldn't need extra parentheses. * Otherwise, insert parens to ensure the expression looks atomic. */ need_paren = !(IsA(expr, Var) || IsA(expr, Aggref) || IsA(expr, GroupingFunc) || IsA(expr, Param)); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(expr, context, false); if (need_paren) appendStringInfoChar(context->buf, ')'); context->varprefix = save_varprefix; pop_ancestor_plan(dpns, &save_dpns); return; } /* * Alternatively, maybe it's a subplan output, which we print as a * reference to the subplan. (We could drill down into the subplan and * print the relevant targetlist expression, but that has been deemed too * confusing since it would violate normal SQL scope rules. Also, we're * relying on this reference to show that the testexpr containing the * Param has anything to do with that subplan at all.) */ subplan = find_param_generator(param, context, &column); if (subplan) { appendStringInfo(context->buf, "(%s%s).col%d", subplan->useHashTable ? "hashed " : "", subplan->plan_name, column + 1); return; } /* * If it's an external parameter, see if the outermost namespace provides * function argument names. */ if (param->paramkind == PARAM_EXTERN && context->namespaces != NIL) { dpns = llast(context->namespaces); if (dpns->argnames && param->paramid > 0 && param->paramid <= dpns->numargs) { char *argname = dpns->argnames[param->paramid - 1]; if (argname) { bool should_qualify = false; ListCell *lc; /* * Qualify the parameter name if there are any other deparse * namespaces with range tables. This avoids qualifying in * trivial cases like "RETURN a + b", but makes it safe in all * other cases. */ foreach(lc, context->namespaces) { deparse_namespace *depns = lfirst(lc); if (depns->rtable_names != NIL) { should_qualify = true; break; } } if (should_qualify) { appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); appendStringInfoChar(context->buf, '.'); } appendStringInfoString(context->buf, quote_identifier(argname)); return; } } } /* * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. * For composite types, add cast to the parameter to ease remote node detect * the type. * * It's a bug if we get here for anything except PARAM_EXTERN Params, but * in production builds printing $N seems more useful than failing. */ Assert(param->paramkind == PARAM_EXTERN); if (param->paramtype >= FirstNormalObjectId) { char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); } else { appendStringInfo(context->buf, "$%d", param->paramid); } } /* * get_simple_binary_op_name * * helper function for isSimpleNode * will return single char binary operator name, or NULL if it's not */ static const char * get_simple_binary_op_name(OpExpr *expr) { List *args = expr->args; if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); const char *op; op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); if (strlen(op) == 1) return op; } return NULL; } /* * isSimpleNode - check if given node is simple (doesn't need parenthesizing) * * true : simple in the context of parent node's type * false : not simple */ static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags) { if (!node) return false; switch (nodeTag(node)) { case T_Var: case T_Const: case T_Param: case T_CoerceToDomainValue: case T_SetToDefault: case T_CurrentOfExpr: /* single words: always simple */ return true; case T_SubscriptingRef: case T_ArrayExpr: case T_RowExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: case T_NextValueExpr: case T_NullIfExpr: case T_Aggref: case T_GroupingFunc: case T_WindowFunc: case T_MergeSupportFunc: case T_FuncExpr: case T_JsonConstructorExpr: case T_JsonExpr: /* function-like: name(..) or name[..] */ return true; /* CASE keywords act as parentheses */ case T_CaseExpr: return true; case T_FieldSelect: /* * appears simple since . has top precedence, unless parent is * T_FieldSelect itself! */ return !IsA(parentNode, FieldSelect); case T_FieldStore: /* * treat like FieldSelect (probably doesn't matter) */ return !IsA(parentNode, FieldStore); case T_CoerceToDomain: /* maybe simple, check args */ return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, node, prettyFlags); case T_RelabelType: return isSimpleNode((Node *) ((RelabelType *) node)->arg, node, prettyFlags); case T_CoerceViaIO: return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, node, prettyFlags); case T_ArrayCoerceExpr: return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, node, prettyFlags); case T_ConvertRowtypeExpr: return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, node, prettyFlags); case T_ReturningExpr: return isSimpleNode((Node *) ((ReturningExpr *) node)->retexpr, node, prettyFlags); case T_OpExpr: { /* depends on parent node type; needs further checking */ if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) { const char *op; const char *parentOp; bool is_lopriop; bool is_hipriop; bool is_lopriparent; bool is_hipriparent; op = get_simple_binary_op_name((OpExpr *) node); if (!op) return false; /* We know only the basic operators + - and * / % */ is_lopriop = (strchr("+-", *op) != NULL); is_hipriop = (strchr("*/%", *op) != NULL); if (!(is_lopriop || is_hipriop)) return false; parentOp = get_simple_binary_op_name((OpExpr *) parentNode); if (!parentOp) return false; is_lopriparent = (strchr("+-", *parentOp) != NULL); is_hipriparent = (strchr("*/%", *parentOp) != NULL); if (!(is_lopriparent || is_hipriparent)) return false; if (is_hipriop && is_lopriparent) return true; /* op binds tighter than parent */ if (is_lopriop && is_hipriparent) return false; /* * Operators are same priority --- can skip parens only if * we have (a - b) - c, not a - (b - c). */ if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) return true; return false; } /* else do the same stuff as for T_SubLink et al. */ } /* FALLTHROUGH */ case T_SubLink: case T_NullTest: case T_BooleanTest: case T_DistinctExpr: case T_JsonIsPredicate: switch (nodeTag(parentNode)) { case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_BoolExpr: /* lower precedence */ case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ return true; default: return false; } case T_BoolExpr: switch (nodeTag(parentNode)) { case T_BoolExpr: if (prettyFlags & PRETTYFLAG_PAREN) { BoolExprType type; BoolExprType parentType; type = ((BoolExpr *) node)->boolop; parentType = ((BoolExpr *) parentNode)->boolop; switch (type) { case NOT_EXPR: case AND_EXPR: if (parentType == AND_EXPR || parentType == OR_EXPR) return true; break; case OR_EXPR: if (parentType == OR_EXPR) return true; break; } } return false; case T_FuncExpr: { /* special handling for casts and COERCE_SQL_SYNTAX */ CoercionForm type = ((FuncExpr *) parentNode)->funcformat; if (type == COERCE_EXPLICIT_CAST || type == COERCE_IMPLICIT_CAST || type == COERCE_SQL_SYNTAX) return false; return true; /* own parentheses */ } case T_SubscriptingRef: /* other separators */ case T_ArrayExpr: /* other separators */ case T_RowExpr: /* other separators */ case T_CoalesceExpr: /* own parentheses */ case T_MinMaxExpr: /* own parentheses */ case T_XmlExpr: /* own parentheses */ case T_NullIfExpr: /* other separators */ case T_Aggref: /* own parentheses */ case T_GroupingFunc: /* own parentheses */ case T_WindowFunc: /* own parentheses */ case T_CaseExpr: /* other separators */ case T_JsonExpr: /* own parentheses */ return true; default: return false; } case T_JsonValueExpr: /* maybe simple, check args */ return isSimpleNode((Node *) ((JsonValueExpr *) node)->raw_expr, node, prettyFlags); default: break; } /* those we don't know: in dubio complexo */ return false; } /* * appendContextKeyword - append a keyword to buffer * * If prettyPrint is enabled, perform a line break, and adjust indentation. * Otherwise, just append the keyword. */ static void appendContextKeyword(deparse_context *context, const char *str, int indentBefore, int indentAfter, int indentPlus) { StringInfo buf = context->buf; if (PRETTY_INDENT(context)) { int indentAmount; context->indentLevel += indentBefore; /* remove any trailing spaces currently in the buffer ... */ removeStringInfoSpaces(buf); /* ... then add a newline and some spaces */ appendStringInfoChar(buf, '\n'); if (context->indentLevel < PRETTYINDENT_LIMIT) indentAmount = Max(context->indentLevel, 0) + indentPlus; else { /* * If we're indented more than PRETTYINDENT_LIMIT characters, try * to conserve horizontal space by reducing the per-level * indentation. For best results the scale factor here should * divide all the indent amounts that get added to indentLevel * (PRETTYINDENT_STD, etc). It's important that the indentation * not grow unboundedly, else deeply-nested trees use O(N^2) * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. */ indentAmount = PRETTYINDENT_LIMIT + (context->indentLevel - PRETTYINDENT_LIMIT) / (PRETTYINDENT_STD / 2); indentAmount %= PRETTYINDENT_LIMIT; /* scale/wrap logic affects indentLevel, but not indentPlus */ indentAmount += indentPlus; } appendStringInfoSpaces(buf, indentAmount); appendStringInfoString(buf, str); context->indentLevel += indentAfter; if (context->indentLevel < 0) context->indentLevel = 0; } else appendStringInfoString(buf, str); } /* * removeStringInfoSpaces - delete trailing spaces from a buffer. * * Possibly this should move to stringinfo.c at some point. */ static void removeStringInfoSpaces(StringInfo str) { while (str->len > 0 && str->data[str->len - 1] == ' ') str->data[--(str->len)] = '\0'; } /* * get_rule_expr_paren - deparse expr using get_rule_expr, * embracing the string with parentheses if necessary for prettyPrint. * * Never embrace if prettyFlags=0, because it's done in the calling node. * * Any node that does *not* embrace its argument node by sql syntax (with * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should * use get_rule_expr_paren instead of get_rule_expr so parentheses can be * added. */ static void get_rule_expr_paren(Node *node, deparse_context *context, bool showimplicit, Node *parentNode) { bool need_paren; need_paren = PRETTY_PAREN(context) && !isSimpleNode(node, parentNode, context->prettyFlags); if (need_paren) appendStringInfoChar(context->buf, '('); get_rule_expr(node, context, showimplicit); if (need_paren) appendStringInfoChar(context->buf, ')'); } static void get_json_behavior(JsonBehavior *behavior, deparse_context *context, const char *on) { /* * The order of array elements must correspond to the order of * JsonBehaviorType members. */ const char *behavior_names[] = { " NULL", " ERROR", " EMPTY", " TRUE", " FALSE", " UNKNOWN", " EMPTY ARRAY", " EMPTY OBJECT", " DEFAULT " }; if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names)) elog(ERROR, "invalid json behavior type: %d", behavior->btype); appendStringInfoString(context->buf, behavior_names[behavior->btype]); if (behavior->btype == JSON_BEHAVIOR_DEFAULT) get_rule_expr(behavior->expr, context, false); appendStringInfo(context->buf, " ON %s", on); } /* * get_json_expr_options * * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and * JSON_TABLE columns. */ static void get_json_expr_options(JsonExpr *jsexpr, deparse_context *context, JsonBehaviorType default_behavior) { if (jsexpr->op == JSON_QUERY_OP) { if (jsexpr->wrapper == JSW_CONDITIONAL) appendStringInfoString(context->buf, " WITH CONDITIONAL WRAPPER"); else if (jsexpr->wrapper == JSW_UNCONDITIONAL) appendStringInfoString(context->buf, " WITH UNCONDITIONAL WRAPPER"); /* The default */ else if (jsexpr->wrapper == JSW_NONE || jsexpr->wrapper == JSW_UNSPEC) appendStringInfoString(context->buf, " WITHOUT WRAPPER"); if (jsexpr->omit_quotes) appendStringInfoString(context->buf, " OMIT QUOTES"); /* The default */ else appendStringInfoString(context->buf, " KEEP QUOTES"); } if (jsexpr->on_empty && jsexpr->on_empty->btype != default_behavior) get_json_behavior(jsexpr->on_empty, context, "EMPTY"); if (jsexpr->on_error && jsexpr->on_error->btype != default_behavior) get_json_behavior(jsexpr->on_error, context, "ERROR"); } /* ---------- * get_rule_expr - Parse back an expression * * Note: showimplicit determines whether we display any implicit cast that * is present at the top of the expression tree. It is a passed argument, * not a field of the context struct, because we change the value as we * recurse down into the expression. In general we suppress implicit casts * when the result type is known with certainty (eg, the arguments of an * OR must be boolean). We display implicit casts for arguments of functions * and operators, since this is needed to be certain that the same function * or operator will be chosen when the expression is re-parsed. * ---------- */ static void get_rule_expr(Node *node, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; if (node == NULL) return; /* Guard against excessively long or deeply-nested queries */ CHECK_FOR_INTERRUPTS(); check_stack_depth(); /* * Each level of get_rule_expr must emit an indivisible term * (parenthesized if necessary) to ensure result is reparsed into the same * expression tree. The only exception is that when the input is a List, * we emit the component items comma-separated with no surrounding * decoration; this is convenient for most callers. */ switch (nodeTag(node)) { case T_Var: (void) get_variable((Var *) node, 0, false, context); break; case T_Const: get_const_expr((Const *) node, context, 0); break; case T_Param: get_parameter((Param *) node, context); break; case T_Aggref: get_agg_expr((Aggref *) node, context, (Aggref *) node); break; case T_GroupingFunc: { GroupingFunc *gexpr = (GroupingFunc *) node; appendStringInfoString(buf, "GROUPING("); get_rule_expr((Node *) gexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_WindowFunc: get_windowfunc_expr((WindowFunc *) node, context); break; case T_MergeSupportFunc: appendStringInfoString(buf, "MERGE_ACTION()"); break; case T_SubscriptingRef: { SubscriptingRef *sbsref = (SubscriptingRef *) node; bool need_parens; /* * If the argument is a CaseTestExpr, we must be inside a * FieldStore, ie, we are assigning to an element of an array * within a composite column. Since we already punted on * displaying the FieldStore's target information, just punt * here too, and display only the assignment source * expression. */ if (IsA(sbsref->refexpr, CaseTestExpr)) { Assert(sbsref->refassgnexpr); get_rule_expr((Node *) sbsref->refassgnexpr, context, showimplicit); break; } /* * Parenthesize the argument unless it's a simple Var or a * FieldSelect. (In particular, if it's another * SubscriptingRef, we *must* parenthesize to avoid * confusion.) */ need_parens = !IsA(sbsref->refexpr, Var) && !IsA(sbsref->refexpr, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); /* * If there's a refassgnexpr, we want to print the node in the * format "container[subscripts] := refassgnexpr". This is * not legal SQL, so decompilation of INSERT or UPDATE * statements should always use processIndirection as part of * the statement-level syntax. We should only see this when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. */ if (sbsref->refassgnexpr) { Node *refassgnexpr; /* * Use processIndirection to print this node's subscripts * as well as any additional field selections or * subscripting in immediate descendants. It returns the * RHS expr that is actually being "assigned". */ refassgnexpr = processIndirection(node, context); appendStringInfoString(buf, " := "); get_rule_expr(refassgnexpr, context, showimplicit); } else { /* Just an ordinary container fetch, so print subscripts */ printSubscripts(sbsref, context); } } break; case T_FuncExpr: get_func_expr((FuncExpr *) node, context, showimplicit); break; case T_NamedArgExpr: { NamedArgExpr *na = (NamedArgExpr *) node; appendStringInfo(buf, "%s => ", quote_identifier(na->name)); get_rule_expr((Node *) na->arg, context, showimplicit); } break; case T_OpExpr: get_oper_expr((OpExpr *) node, context); break; case T_DistinctExpr: { DistinctExpr *expr = (DistinctExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfoString(buf, " IS DISTINCT FROM "); get_rule_expr_paren(arg2, context, true, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_NullIfExpr: { NullIfExpr *nullifexpr = (NullIfExpr *) node; appendStringInfoString(buf, "NULLIF("); get_rule_expr((Node *) nullifexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; List *args = expr->args; Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg1, context, true, node); appendStringInfo(buf, " %s %s (", generate_operator_name(expr->opno, exprType(arg1), get_base_element_type(exprType(arg2))), expr->useOr ? "ANY" : "ALL"); get_rule_expr_paren(arg2, context, true, node); /* * There's inherent ambiguity in "x op ANY/ALL (y)" when y is * a bare sub-SELECT. Since we're here, the sub-SELECT must * be meant as a scalar sub-SELECT yielding an array value to * be used in ScalarArrayOpExpr; but the grammar will * preferentially interpret such a construct as an ANY/ALL * SubLink. To prevent misparsing the output that way, insert * a dummy coercion (which will be stripped by parse analysis, * so no inefficiency is added in dump and reload). This is * indeed most likely what the user wrote to get the construct * accepted in the first place. */ if (IsA(arg2, SubLink) && ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) appendStringInfo(buf, "::%s", format_type_with_typemod(exprType(arg2), exprTypmod(arg2))); appendStringInfoChar(buf, ')'); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BoolExpr: { BoolExpr *expr = (BoolExpr *) node; Node *first_arg = linitial(expr->args); ListCell *arg; switch (expr->boolop) { case AND_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " AND "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case OR_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " OR "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; case NOT_EXPR: if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); appendStringInfoString(buf, "NOT "); get_rule_expr_paren(first_arg, context, false, node); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); break; default: elog(ERROR, "unrecognized boolop: %d", (int) expr->boolop); } } break; case T_SubLink: get_sublink_expr((SubLink *) node, context); break; case T_SubPlan: { SubPlan *subplan = (SubPlan *) node; /* * We cannot see an already-planned subplan in rule deparsing, * only while EXPLAINing a query plan. We don't try to * reconstruct the original SQL, just reference the subplan * that appears elsewhere in EXPLAIN's result. It does seem * useful to show the subLinkType and testexpr (if any), and * we also note whether the subplan will be hashed. */ switch (subplan->subLinkType) { case EXISTS_SUBLINK: appendStringInfoString(buf, "EXISTS("); Assert(subplan->testexpr == NULL); break; case ALL_SUBLINK: appendStringInfoString(buf, "(ALL "); Assert(subplan->testexpr != NULL); break; case ANY_SUBLINK: appendStringInfoString(buf, "(ANY "); Assert(subplan->testexpr != NULL); break; case ROWCOMPARE_SUBLINK: /* Parenthesizing the testexpr seems sufficient */ appendStringInfoChar(buf, '('); Assert(subplan->testexpr != NULL); break; case EXPR_SUBLINK: /* No need to decorate these subplan references */ appendStringInfoChar(buf, '('); Assert(subplan->testexpr == NULL); break; case MULTIEXPR_SUBLINK: /* MULTIEXPR isn't executed in the normal way */ appendStringInfoString(buf, "(rescan "); Assert(subplan->testexpr == NULL); break; case ARRAY_SUBLINK: appendStringInfoString(buf, "ARRAY("); Assert(subplan->testexpr == NULL); break; case CTE_SUBLINK: /* This case is unreachable within expressions */ appendStringInfoString(buf, "CTE("); Assert(subplan->testexpr == NULL); break; } if (subplan->testexpr != NULL) { deparse_namespace *dpns; /* * Push SubPlan into ancestors list while deparsing * testexpr, so that we can handle PARAM_EXEC references * to the SubPlan's paramIds. (This makes it look like * the SubPlan is an "ancestor" of the current plan node, * which is a little weird, but it does no harm.) In this * path, we don't need to mention the SubPlan explicitly, * because the referencing Params will show its existence. */ dpns = (deparse_namespace *) linitial(context->namespaces); dpns->ancestors = lcons(subplan, dpns->ancestors); get_rule_expr(subplan->testexpr, context, showimplicit); appendStringInfoChar(buf, ')'); dpns->ancestors = list_delete_first(dpns->ancestors); } else { /* No referencing Params, so show the SubPlan's name */ if (subplan->useHashTable) appendStringInfo(buf, "hashed %s)", subplan->plan_name); else appendStringInfo(buf, "%s)", subplan->plan_name); } } break; case T_AlternativeSubPlan: { AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; ListCell *lc; /* * This case cannot be reached in normal usage, since no * AlternativeSubPlan can appear either in parsetrees or * finished plan trees. We keep it just in case somebody * wants to use this code to print planner data structures. */ appendStringInfoString(buf, "(alternatives: "); foreach(lc, asplan->subplans) { SubPlan *splan = lfirst_node(SubPlan, lc); if (splan->useHashTable) appendStringInfo(buf, "hashed %s", splan->plan_name); else appendStringInfoString(buf, splan->plan_name); if (lnext(asplan->subplans, lc)) appendStringInfoString(buf, " or "); } appendStringInfoChar(buf, ')'); } break; case T_FieldSelect: { FieldSelect *fselect = (FieldSelect *) node; Node *arg = (Node *) fselect->arg; int fno = fselect->fieldnum; const char *fieldname; bool need_parens; /* * Parenthesize the argument unless it's an SubscriptingRef or * another FieldSelect. Note in particular that it would be * WRONG to not parenthesize a Var argument; simplicity is not * the issue here, having the right number of names is. */ need_parens = !IsA(arg, SubscriptingRef) && !IsA(arg, FieldSelect); if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr(arg, context, true); if (need_parens) appendStringInfoChar(buf, ')'); /* * Get and print the field name. */ fieldname = get_name_for_var_field((Var *) arg, fno, 0, context); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); } break; case T_FieldStore: { FieldStore *fstore = (FieldStore *) node; bool need_parens; /* * There is no good way to represent a FieldStore as real SQL, * so decompilation of INSERT or UPDATE statements should * always use processIndirection as part of the * statement-level syntax. We should only get here when * EXPLAIN tries to print the targetlist of a plan resulting * from such a statement. The plan case is even harder than * ordinary rules would be, because the planner tries to * collapse multiple assignments to the same field or subfield * into one FieldStore; so we can see a list of target fields * not just one, and the arguments could be FieldStores * themselves. We don't bother to try to print the target * field names; we just print the source arguments, with a * ROW() around them if there's more than one. This isn't * terribly complete, but it's probably good enough for * EXPLAIN's purposes; especially since anything more would be * either hopelessly confusing or an even poorer * representation of what the plan is actually doing. */ need_parens = (list_length(fstore->newvals) != 1); if (need_parens) appendStringInfoString(buf, "ROW("); get_rule_expr((Node *) fstore->newvals, context, showimplicit); if (need_parens) appendStringInfoChar(buf, ')'); } break; case T_RelabelType: { RelabelType *relabel = (RelabelType *) node; Node *arg = (Node *) relabel->arg; if (relabel->relabelformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, relabel->resulttype, relabel->resulttypmod, node); } } break; case T_CoerceViaIO: { CoerceViaIO *iocoerce = (CoerceViaIO *) node; Node *arg = (Node *) iocoerce->arg; if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, iocoerce->resulttype, -1, node); } } break; case T_ArrayCoerceExpr: { ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; Node *arg = (Node *) acoerce->arg; if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, acoerce->resulttype, acoerce->resulttypmod, node); } } break; case T_ConvertRowtypeExpr: { ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; Node *arg = (Node *) convert->arg; if (convert->convertformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr_paren(arg, context, false, node); } else { get_coercion_expr(arg, context, convert->resulttype, -1, node); } } break; case T_CollateExpr: { CollateExpr *collate = (CollateExpr *) node; Node *arg = (Node *) collate->arg; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, showimplicit, node); appendStringInfo(buf, " COLLATE %s", generate_collation_name(collate->collOid)); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CaseExpr: { CaseExpr *caseexpr = (CaseExpr *) node; ListCell *temp; appendContextKeyword(context, "CASE", 0, PRETTYINDENT_VAR, 0); if (caseexpr->arg) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) caseexpr->arg, context, true); } foreach(temp, caseexpr->args) { CaseWhen *when = (CaseWhen *) lfirst(temp); Node *w = (Node *) when->expr; if (caseexpr->arg) { /* * The parser should have produced WHEN clauses of the * form "CaseTestExpr = RHS", possibly with an * implicit coercion inserted above the CaseTestExpr. * For accurate decompilation of rules it's essential * that we show just the RHS. However in an * expression that's been through the optimizer, the * WHEN clause could be almost anything (since the * equality operator could have been expanded into an * inline function). If we don't recognize the form * of the WHEN clause, just punt and display it as-is. */ if (IsA(w, OpExpr)) { List *args = ((OpExpr *) w)->args; if (list_length(args) == 2 && IsA(strip_implicit_coercions(linitial(args)), CaseTestExpr)) w = (Node *) lsecond(args); } } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "WHEN ", 0, 0, 0); get_rule_expr(w, context, false); appendStringInfoString(buf, " THEN "); get_rule_expr((Node *) when->result, context, true); } if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "ELSE ", 0, 0, 0); get_rule_expr((Node *) caseexpr->defresult, context, true); if (!PRETTY_INDENT(context)) appendStringInfoChar(buf, ' '); appendContextKeyword(context, "END", -PRETTYINDENT_VAR, 0, 0); } break; case T_CaseTestExpr: { /* * Normally we should never get here, since for expressions * that can contain this node type we attempt to avoid * recursing to it. But in an optimized expression we might * be unable to avoid that (see comments for CaseExpr). If we * do see one, print it as CASE_TEST_EXPR. */ appendStringInfoString(buf, "CASE_TEST_EXPR"); } break; case T_ArrayExpr: { ArrayExpr *arrayexpr = (ArrayExpr *) node; appendStringInfoString(buf, "ARRAY["); get_rule_expr((Node *) arrayexpr->elements, context, true); appendStringInfoChar(buf, ']'); /* * If the array isn't empty, we assume its elements are * coerced to the desired type. If it's empty, though, we * need an explicit coercion to the array type. */ if (arrayexpr->elements == NIL) appendStringInfo(buf, "::%s", format_type_with_typemod(arrayexpr->array_typeid, -1)); } break; case T_RowExpr: { RowExpr *rowexpr = (RowExpr *) node; TupleDesc tupdesc = NULL; ListCell *arg; int i; char *sep; /* * If it's a named type and not RECORD, we may have to skip * dropped columns and/or claim there are NULLs for added * columns. */ if (rowexpr->row_typeid != RECORDOID) { tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); Assert(list_length(rowexpr->args) <= tupdesc->natts); } /* Precompute deparse ns and whether we even need to try mapping */ deparse_namespace *dpns = (context->namespaces != NIL) ? (deparse_namespace *) linitial(context->namespaces) : NULL; bool try_map = (dpns && dpns->plan == NULL && dpns_has_named_join(dpns)); /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. */ appendStringInfoString(buf, "ROW("); sep = ""; i = 0; foreach(arg, rowexpr->args) { Node *e = (Node *) lfirst(arg); if (tupdesc == NULL || !TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); /* PG18: if element is a simple base Var, set its SYN to the JOIN alias */ if (try_map) { Var *v = unwrap_simple_var(e); if (v) { map_var_through_join_alias(dpns, v); } } /* Whole-row Vars need special treatment here */ get_rule_expr_toplevel(e, context, true); sep = ", "; } i++; } if (tupdesc != NULL) { while (i < tupdesc->natts) { if (!TupleDescAttr(tupdesc, i)->attisdropped) { appendStringInfoString(buf, sep); appendStringInfoString(buf, "NULL"); sep = ", "; } i++; } ReleaseTupleDesc(tupdesc); } appendStringInfoChar(buf, ')'); if (rowexpr->row_format == COERCE_EXPLICIT_CAST) appendStringInfo(buf, "::%s", format_type_with_typemod(rowexpr->row_typeid, -1)); } break; case T_RowCompareExpr: { RowCompareExpr *rcexpr = (RowCompareExpr *) node; /* * SQL99 allows "ROW" to be omitted when there is more than * one column, but for simplicity we always print it. Within * a ROW expression, whole-row Vars need special treatment, so * use get_rule_list_toplevel. */ appendStringInfoString(buf, "(ROW("); get_rule_list_toplevel(rcexpr->largs, context, true); /* * We assume that the name of the first-column operator will * do for all the rest too. This is definitely open to * failure, eg if some but not all operators were renamed * since the construct was parsed, but there seems no way to * be perfect. */ appendStringInfo(buf, ") %s ROW(", generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs)))); get_rule_list_toplevel(rcexpr->rargs, context, true); appendStringInfoString(buf, "))"); } break; case T_CoalesceExpr: { CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; appendStringInfoString(buf, "COALESCE("); get_rule_expr((Node *) coalesceexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_MinMaxExpr: { MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; switch (minmaxexpr->op) { case IS_GREATEST: appendStringInfoString(buf, "GREATEST("); break; case IS_LEAST: appendStringInfoString(buf, "LEAST("); break; } get_rule_expr((Node *) minmaxexpr->args, context, true); appendStringInfoChar(buf, ')'); } break; case T_SQLValueFunction: { SQLValueFunction *svf = (SQLValueFunction *) node; /* * Note: this code knows that typmod for time, timestamp, and * timestamptz just prints as integer. */ switch (svf->op) { case SVFOP_CURRENT_DATE: appendStringInfoString(buf, "CURRENT_DATE"); break; case SVFOP_CURRENT_TIME: appendStringInfoString(buf, "CURRENT_TIME"); break; case SVFOP_CURRENT_TIME_N: appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); break; case SVFOP_CURRENT_TIMESTAMP: appendStringInfoString(buf, "CURRENT_TIMESTAMP"); break; case SVFOP_CURRENT_TIMESTAMP_N: appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", svf->typmod); break; case SVFOP_LOCALTIME: appendStringInfoString(buf, "LOCALTIME"); break; case SVFOP_LOCALTIME_N: appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); break; case SVFOP_LOCALTIMESTAMP: appendStringInfoString(buf, "LOCALTIMESTAMP"); break; case SVFOP_LOCALTIMESTAMP_N: appendStringInfo(buf, "LOCALTIMESTAMP(%d)", svf->typmod); break; case SVFOP_CURRENT_ROLE: appendStringInfoString(buf, "CURRENT_ROLE"); break; case SVFOP_CURRENT_USER: appendStringInfoString(buf, "CURRENT_USER"); break; case SVFOP_USER: appendStringInfoString(buf, "USER"); break; case SVFOP_SESSION_USER: appendStringInfoString(buf, "SESSION_USER"); break; case SVFOP_CURRENT_CATALOG: appendStringInfoString(buf, "CURRENT_CATALOG"); break; case SVFOP_CURRENT_SCHEMA: appendStringInfoString(buf, "CURRENT_SCHEMA"); break; } } break; case T_XmlExpr: { XmlExpr *xexpr = (XmlExpr *) node; bool needcomma = false; ListCell *arg; ListCell *narg; Const *con; switch (xexpr->op) { case IS_XMLCONCAT: appendStringInfoString(buf, "XMLCONCAT("); break; case IS_XMLELEMENT: appendStringInfoString(buf, "XMLELEMENT("); break; case IS_XMLFOREST: appendStringInfoString(buf, "XMLFOREST("); break; case IS_XMLPARSE: appendStringInfoString(buf, "XMLPARSE("); break; case IS_XMLPI: appendStringInfoString(buf, "XMLPI("); break; case IS_XMLROOT: appendStringInfoString(buf, "XMLROOT("); break; case IS_XMLSERIALIZE: appendStringInfoString(buf, "XMLSERIALIZE("); break; case IS_DOCUMENT: break; } if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) { if (xexpr->xmloption == XMLOPTION_DOCUMENT) appendStringInfoString(buf, "DOCUMENT "); else appendStringInfoString(buf, "CONTENT "); } if (xexpr->name) { appendStringInfo(buf, "NAME %s", quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); needcomma = true; } if (xexpr->named_args) { if (xexpr->op != IS_XMLFOREST) { if (needcomma) appendStringInfoString(buf, ", "); appendStringInfoString(buf, "XMLATTRIBUTES("); needcomma = false; } forboth(arg, xexpr->named_args, narg, xexpr->arg_names) { Node *e = (Node *) lfirst(arg); char *argname = strVal(lfirst(narg)); if (needcomma) appendStringInfoString(buf, ", "); get_rule_expr((Node *) e, context, true); appendStringInfo(buf, " AS %s", quote_identifier(map_xml_name_to_sql_identifier(argname))); needcomma = true; } if (xexpr->op != IS_XMLFOREST) appendStringInfoChar(buf, ')'); } if (xexpr->args) { if (needcomma) appendStringInfoString(buf, ", "); switch (xexpr->op) { case IS_XMLCONCAT: case IS_XMLELEMENT: case IS_XMLFOREST: case IS_XMLPI: case IS_XMLSERIALIZE: /* no extra decoration needed */ get_rule_expr((Node *) xexpr->args, context, true); break; case IS_XMLPARSE: Assert(list_length(xexpr->args) == 2); get_rule_expr((Node *) linitial(xexpr->args), context, true); con = lsecond_node(Const, xexpr->args); Assert(!con->constisnull); if (DatumGetBool(con->constvalue)) appendStringInfoString(buf, " PRESERVE WHITESPACE"); else appendStringInfoString(buf, " STRIP WHITESPACE"); break; case IS_XMLROOT: Assert(list_length(xexpr->args) == 3); get_rule_expr((Node *) linitial(xexpr->args), context, true); appendStringInfoString(buf, ", VERSION "); con = (Const *) lsecond(xexpr->args); if (IsA(con, Const) && con->constisnull) appendStringInfoString(buf, "NO VALUE"); else get_rule_expr((Node *) con, context, false); con = lthird_node(Const, xexpr->args); if (con->constisnull) /* suppress STANDALONE NO VALUE */ ; else { switch (DatumGetInt32(con->constvalue)) { case XML_STANDALONE_YES: appendStringInfoString(buf, ", STANDALONE YES"); break; case XML_STANDALONE_NO: appendStringInfoString(buf, ", STANDALONE NO"); break; case XML_STANDALONE_NO_VALUE: appendStringInfoString(buf, ", STANDALONE NO VALUE"); break; default: break; } } break; case IS_DOCUMENT: get_rule_expr_paren((Node *) xexpr->args, context, false, node); break; } } if (xexpr->op == IS_XMLSERIALIZE) { appendStringInfo(buf, " AS %s", format_type_with_typemod(xexpr->type, xexpr->typmod)); if (xexpr->indent) appendStringInfoString(buf, " INDENT"); else appendStringInfoString(buf, " NO INDENT"); } if (xexpr->op == IS_DOCUMENT) appendStringInfoString(buf, " IS DOCUMENT"); else appendStringInfoChar(buf, ')'); } break; case T_NullTest: { NullTest *ntest = (NullTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) ntest->arg, context, true, node); /* * For scalar inputs, we prefer to print as IS [NOT] NULL, * which is shorter and traditional. If it's a rowtype input * but we're applying a scalar test, must print IS [NOT] * DISTINCT FROM NULL to be semantically correct. */ if (ntest->argisrow || !type_is_rowtype(exprType((Node *) ntest->arg))) { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS NOT NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } else { switch (ntest->nulltesttype) { case IS_NULL: appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); break; case IS_NOT_NULL: appendStringInfoString(buf, " IS DISTINCT FROM NULL"); break; default: elog(ERROR, "unrecognized nulltesttype: %d", (int) ntest->nulltesttype); } } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_BooleanTest: { BooleanTest *btest = (BooleanTest *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) btest->arg, context, false, node); switch (btest->booltesttype) { case IS_TRUE: appendStringInfoString(buf, " IS TRUE"); break; case IS_NOT_TRUE: appendStringInfoString(buf, " IS NOT TRUE"); break; case IS_FALSE: appendStringInfoString(buf, " IS FALSE"); break; case IS_NOT_FALSE: appendStringInfoString(buf, " IS NOT FALSE"); break; case IS_UNKNOWN: appendStringInfoString(buf, " IS UNKNOWN"); break; case IS_NOT_UNKNOWN: appendStringInfoString(buf, " IS NOT UNKNOWN"); break; default: elog(ERROR, "unrecognized booltesttype: %d", (int) btest->booltesttype); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } break; case T_CoerceToDomain: { CoerceToDomain *ctest = (CoerceToDomain *) node; Node *arg = (Node *) ctest->arg; if (ctest->coercionformat == COERCE_IMPLICIT_CAST && !showimplicit) { /* don't show the implicit cast */ get_rule_expr(arg, context, false); } else { get_coercion_expr(arg, context, ctest->resulttype, ctest->resulttypmod, node); } } break; case T_CoerceToDomainValue: appendStringInfoString(buf, "VALUE"); break; case T_SetToDefault: appendStringInfoString(buf, "DEFAULT"); break; case T_CurrentOfExpr: { CurrentOfExpr *cexpr = (CurrentOfExpr *) node; if (cexpr->cursor_name) appendStringInfo(buf, "CURRENT OF %s", quote_identifier(cexpr->cursor_name)); else appendStringInfo(buf, "CURRENT OF $%d", cexpr->cursor_param); } break; case T_NextValueExpr: { NextValueExpr *nvexpr = (NextValueExpr *) node; /* * This isn't exactly nextval(), but that seems close enough * for EXPLAIN's purposes. */ appendStringInfoString(buf, "nextval("); simple_quote_literal(buf, generate_relation_name(nvexpr->seqid, NIL)); appendStringInfoChar(buf, ')'); } break; case T_InferenceElem: { InferenceElem *iexpr = (InferenceElem *) node; bool save_varprefix; bool need_parens; /* * InferenceElem can only refer to target relation, so a * prefix is not useful, and indeed would cause parse errors. */ save_varprefix = context->varprefix; context->varprefix = false; /* * Parenthesize the element unless it's a simple Var or a bare * function call. Follows pg_get_indexdef_worker(). */ need_parens = !IsA(iexpr->expr, Var); if (IsA(iexpr->expr, FuncExpr) && ((FuncExpr *) iexpr->expr)->funcformat == COERCE_EXPLICIT_CALL) need_parens = false; if (need_parens) appendStringInfoChar(buf, '('); get_rule_expr((Node *) iexpr->expr, context, false); if (need_parens) appendStringInfoChar(buf, ')'); context->varprefix = save_varprefix; if (iexpr->infercollid) appendStringInfo(buf, " COLLATE %s", generate_collation_name(iexpr->infercollid)); /* Add the operator class name, if not default */ if (iexpr->inferopclass) { Oid inferopclass = iexpr->inferopclass; Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); get_opclass_name(inferopclass, inferopcinputtype, buf); } } break; case T_ReturningExpr: { ReturningExpr *retExpr = (ReturningExpr *) node; /* * We cannot see a ReturningExpr in rule deparsing, only while * EXPLAINing a query plan (ReturningExpr nodes are only ever * adding during query rewriting). Just display the expression * returned (an expanded view column). */ get_rule_expr((Node *) retExpr->retexpr, context, showimplicit); } break; case T_PartitionBoundSpec: { PartitionBoundSpec *spec = (PartitionBoundSpec *) node; ListCell *cell; char *sep; if (spec->is_default) { appendStringInfoString(buf, "DEFAULT"); break; } switch (spec->strategy) { case PARTITION_STRATEGY_HASH: Assert(spec->modulus > 0 && spec->remainder >= 0); Assert(spec->modulus > spec->remainder); appendStringInfoString(buf, "FOR VALUES"); appendStringInfo(buf, " WITH (modulus %d, remainder %d)", spec->modulus, spec->remainder); break; case PARTITION_STRATEGY_LIST: Assert(spec->listdatums != NIL); appendStringInfoString(buf, "FOR VALUES IN ("); sep = ""; foreach(cell, spec->listdatums) { Const *val = lfirst_node(Const, cell); appendStringInfoString(buf, sep); get_const_expr(val, context, -1); sep = ", "; } appendStringInfoChar(buf, ')'); break; case PARTITION_STRATEGY_RANGE: Assert(spec->lowerdatums != NIL && spec->upperdatums != NIL && list_length(spec->lowerdatums) == list_length(spec->upperdatums)); appendStringInfo(buf, "FOR VALUES FROM %s TO %s", get_range_partbound_string(spec->lowerdatums), get_range_partbound_string(spec->upperdatums)); break; default: elog(ERROR, "unrecognized partition strategy: %d", (int) spec->strategy); break; } } break; case T_JsonValueExpr: { JsonValueExpr *jve = (JsonValueExpr *) node; get_rule_expr((Node *) jve->raw_expr, context, false); get_json_format(jve->format, context->buf); } break; case T_JsonConstructorExpr: get_json_constructor((JsonConstructorExpr *) node, context, false); break; case T_JsonIsPredicate: { JsonIsPredicate *pred = (JsonIsPredicate *) node; if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, '('); get_rule_expr_paren(pred->expr, context, true, node); appendStringInfoString(context->buf, " IS JSON"); /* TODO: handle FORMAT clause */ switch (pred->item_type) { case JS_TYPE_SCALAR: appendStringInfoString(context->buf, " SCALAR"); break; case JS_TYPE_ARRAY: appendStringInfoString(context->buf, " ARRAY"); break; case JS_TYPE_OBJECT: appendStringInfoString(context->buf, " OBJECT"); break; default: break; } if (pred->unique_keys) appendStringInfoString(context->buf, " WITH UNIQUE KEYS"); if (!PRETTY_PAREN(context)) appendStringInfoChar(context->buf, ')'); } break; case T_JsonExpr: { JsonExpr *jexpr = (JsonExpr *) node; switch (jexpr->op) { case JSON_EXISTS_OP: appendStringInfoString(buf, "JSON_EXISTS("); break; case JSON_QUERY_OP: appendStringInfoString(buf, "JSON_QUERY("); break; case JSON_VALUE_OP: appendStringInfoString(buf, "JSON_VALUE("); break; default: elog(ERROR, "unrecognized JsonExpr op: %d", (int) jexpr->op); } get_rule_expr(jexpr->formatted_expr, context, showimplicit); appendStringInfoString(buf, ", "); get_json_path_spec(jexpr->path_spec, context, showimplicit); if (jexpr->passing_values) { ListCell *lc1, *lc2; bool needcomma = false; appendStringInfoString(buf, " PASSING "); forboth(lc1, jexpr->passing_names, lc2, jexpr->passing_values) { if (needcomma) appendStringInfoString(buf, ", "); needcomma = true; get_rule_expr((Node *) lfirst(lc2), context, showimplicit); appendStringInfo(buf, " AS %s", quote_identifier(lfirst_node(String, lc1)->sval)); } } if (jexpr->op != JSON_EXISTS_OP || jexpr->returning->typid != BOOLOID) get_json_returning(jexpr->returning, context->buf, jexpr->op == JSON_QUERY_OP); get_json_expr_options(jexpr, context, jexpr->op != JSON_EXISTS_OP ? JSON_BEHAVIOR_NULL : JSON_BEHAVIOR_FALSE); appendStringInfoChar(buf, ')'); } break; case T_List: { char *sep; ListCell *l; sep = ""; foreach(l, (List *) node) { appendStringInfoString(buf, sep); get_rule_expr((Node *) lfirst(l), context, showimplicit); sep = ", "; } } break; case T_TableFunc: get_tablefunc((TableFunc *) node, context, showimplicit); break; case T_CallStmt: get_proc_expr((CallStmt *) node, context, showimplicit); break; default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); break; } } /* * get_rule_expr_toplevel - Parse back a toplevel expression * * Same as get_rule_expr(), except that if the expr is just a Var, we pass * istoplevel = true not false to get_variable(). This causes whole-row Vars * to get printed with decoration that will prevent expansion of "*". * We need to use this in contexts such as ROW() and VALUES(), where the * parser would expand "foo.*" appearing at top level. (In principle we'd * use this in get_target_list() too, but that has additional worries about * whether to print AS, so it needs to invoke get_variable() directly anyway.) */ static void get_rule_expr_toplevel(Node *node, deparse_context *context, bool showimplicit) { if (node && IsA(node, Var)) (void) get_variable((Var *) node, 0, true, context); else get_rule_expr(node, context, showimplicit); } /* * get_rule_list_toplevel - Parse back a list of toplevel expressions * * Apply get_rule_expr_toplevel() to each element of a List. * * This adds commas between the expressions, but caller is responsible * for printing surrounding decoration. */ static void get_rule_list_toplevel(List *lst, deparse_context *context, bool showimplicit) { const char *sep; ListCell *lc; sep = ""; foreach(lc, lst) { Node *e = (Node *) lfirst(lc); appendStringInfoString(context->buf, sep); get_rule_expr_toplevel(e, context, showimplicit); sep = ", "; } } /* * get_rule_expr_funccall - Parse back a function-call expression * * Same as get_rule_expr(), except that we guarantee that the output will * look like a function call, or like one of the things the grammar treats as * equivalent to a function call (see the func_expr_windowless production). * This is needed in places where the grammar uses func_expr_windowless and * you can't substitute a parenthesized a_expr. If what we have isn't going * to look like a function call, wrap it in a dummy CAST() expression, which * will satisfy the grammar --- and, indeed, is likely what the user wrote to * produce such a thing. */ static void get_rule_expr_funccall(Node *node, deparse_context *context, bool showimplicit) { if (looks_like_function(node)) get_rule_expr(node, context, showimplicit); else { StringInfo buf = context->buf; appendStringInfoString(buf, "CAST("); /* no point in showing any top-level implicit cast */ get_rule_expr(node, context, false); appendStringInfo(buf, " AS %s)", format_type_with_typemod(exprType(node), exprTypmod(node))); } } /* * Helper function to identify node types that satisfy func_expr_windowless. * If in doubt, "false" is always a safe answer. */ static bool looks_like_function(Node *node) { if (node == NULL) return false; /* probably shouldn't happen */ switch (nodeTag(node)) { case T_FuncExpr: /* OK, unless it's going to deparse as a cast */ return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); case T_NullIfExpr: case T_CoalesceExpr: case T_MinMaxExpr: case T_SQLValueFunction: case T_XmlExpr: case T_JsonExpr: /* these are all accepted by func_expr_common_subexpr */ return true; default: break; } return false; } /* * get_oper_expr - Parse back an OpExpr node */ static void get_oper_expr(OpExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid opno = expr->opno; List *args = expr->args; if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); if (list_length(args) == 2) { /* binary operator */ Node *arg1 = (Node *) linitial(args); Node *arg2 = (Node *) lsecond(args); get_rule_expr_paren(arg1, context, true, (Node *) expr); appendStringInfo(buf, " %s ", generate_operator_name(opno, exprType(arg1), exprType(arg2))); get_rule_expr_paren(arg2, context, true, (Node *) expr); } else { /* prefix operator */ Node *arg = (Node *) linitial(args); appendStringInfo(buf, "%s ", generate_operator_name(opno, InvalidOid, exprType(arg))); get_rule_expr_paren(arg, context, true, (Node *) expr); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } /* * get_func_expr - Parse back a FuncExpr node */ static void get_func_expr(FuncExpr *expr, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; bool use_variadic; ListCell *l; /* * If the function call came from an implicit coercion, then just show the * first argument --- unless caller wants to see implicit coercions. */ if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) { get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); return; } /* * If the function call came from a cast, then show the first argument * plus an explicit cast operation. */ if (expr->funcformat == COERCE_EXPLICIT_CAST || expr->funcformat == COERCE_IMPLICIT_CAST) { Node *arg = linitial(expr->args); Oid rettype = expr->funcresulttype; int32 coercedTypmod; /* Get the typmod if this is a length-coercion function */ (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); get_coercion_expr(arg, context, rettype, coercedTypmod, (Node *) expr); return; } /* * If the function was called using one of the SQL spec's random special * syntaxes, try to reproduce that. If we don't recognize the function, * fall through. */ if (expr->funcformat == COERCE_SQL_SYNTAX) { if (get_func_sql_syntax(expr, context)) return; } /* * Normal function: display as proname(args). First we need to extract * the argument datatypes. */ if (list_length(expr->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, expr->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } appendStringInfo(buf, "%s(", generate_function_name(funcoid, nargs, argnames, argtypes, expr->funcvariadic, &use_variadic, context->inGroupBy)); nargs = 0; foreach(l, expr->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(expr->args, l) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(l), context, true); } appendStringInfoChar(buf, ')'); } /* * get_proc_expr - Parse back a CallStmt node */ static void get_proc_expr(CallStmt *stmt, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; Oid functionOid = stmt->funcexpr->funcid; bool use_variadic; Oid *argumentTypes; List *finalArgumentList = NIL; ListCell *argumentCell; List *namedArgList = NIL; int numberOfArgs = -1; if (!get_merged_argument_list(stmt, &namedArgList, &argumentTypes, &finalArgumentList, &numberOfArgs)) { /* Nothing merged i.e. no OUT arguments */ get_func_expr((FuncExpr *) stmt->funcexpr, context, showimplicit); return; } appendStringInfo(buf, "%s(", generate_function_name(functionOid, numberOfArgs, namedArgList, argumentTypes, stmt->funcexpr->funcvariadic, &use_variadic, context->inGroupBy)); int argNumber = 0; foreach(argumentCell, finalArgumentList) { if (argNumber++ > 0) appendStringInfoString(buf, ", "); if (use_variadic && lnext(finalArgumentList, argumentCell) == NULL) appendStringInfoString(buf, "VARIADIC "); get_rule_expr((Node *) lfirst(argumentCell), context, true); argNumber++; } appendStringInfoChar(buf, ')'); } /* * get_agg_expr - Parse back an Aggref node */ static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref) { get_agg_expr_helper(aggref, context, original_aggref, NULL, NULL, false); } /* * get_agg_expr_helper - subroutine for get_agg_expr and * get_json_agg_constructor */ static void get_agg_expr_helper(Aggref *aggref, deparse_context *context, Aggref *original_aggref, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; bool use_variadic = false; /* * For a combining aggregate, we look up and deparse the corresponding * partial aggregate instead. This is necessary because our input * argument list has been replaced; the new argument list always has just * one element, which will point to a partial Aggref that supplies us with * transition states to combine. */ if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) { TargetEntry *tle; Assert(list_length(aggref->args) == 1); tle = linitial_node(TargetEntry, aggref->args); resolve_special_varno((Node *) tle->expr, context, get_agg_combine_expr, original_aggref); return; } /* * Mark as PARTIAL, if appropriate. We look to the original aggref so as * to avoid printing this when recursing from the code just above. */ if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) appendStringInfoString(buf, "PARTIAL "); /* Extract the argument types as seen by the parser */ nargs = get_aggregate_argtypes(aggref, argtypes); if (!funcname) funcname = generate_function_name(aggref->aggfnoid, nargs, NIL, argtypes, aggref->aggvariadic, &use_variadic, context->inGroupBy); /* Print the aggregate name, schema-qualified if needed */ appendStringInfo(buf, "%s(%s", funcname, (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) { /* * Ordered-set aggregates do not use "*" syntax. Also, we needn't * worry about inserting VARIADIC. So we can just dump the direct * args as-is. */ Assert(!aggref->aggvariadic); get_rule_expr((Node *) aggref->aggdirectargs, context, true); Assert(aggref->aggorder != NIL); appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } else { /* aggstar can be set only in zero-argument aggregates */ if (aggref->aggstar) appendStringInfoChar(buf, '*'); else { ListCell *l; int i; i = 0; foreach(l, aggref->args) { TargetEntry *tle = (TargetEntry *) lfirst(l); Node *arg = (Node *) tle->expr; Assert(!IsA(arg, NamedArgExpr)); if (tle->resjunk) continue; if (i++ > 0) { if (is_json_objectagg) { /* * the ABSENT ON NULL and WITH UNIQUE args are printed * separately, so ignore them here */ if (i > 2) break; appendStringInfoString(buf, " : "); } else appendStringInfoString(buf, ", "); } if (use_variadic && i == nargs) appendStringInfoString(buf, "VARIADIC "); get_rule_expr(arg, context, true); } } if (aggref->aggorder != NIL) { appendStringInfoString(buf, " ORDER BY "); get_rule_orderby(aggref->aggorder, aggref->args, false, context); } } if (options) appendStringInfoString(buf, options); if (aggref->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) aggref->aggfilter, context, false); } appendStringInfoChar(buf, ')'); } /* * This is a helper function for get_agg_expr(). It's used when we deparse * a combining Aggref; resolve_special_varno locates the corresponding partial * Aggref and then calls this. */ static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) { Aggref *aggref; Aggref *original_aggref = callback_arg; if (!IsA(node, Aggref)) elog(ERROR, "combining Aggref does not point to an Aggref"); aggref = (Aggref *) node; get_agg_expr(aggref, context, original_aggref); } /* * get_windowfunc_expr - Parse back a WindowFunc node */ static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) { get_windowfunc_expr_helper(wfunc, context, NULL, NULL, false); } /* * get_windowfunc_expr_helper - subroutine for get_windowfunc_expr and * get_json_agg_constructor */ static void get_windowfunc_expr_helper(WindowFunc *wfunc, deparse_context *context, const char *funcname, const char *options, bool is_json_objectagg) { StringInfo buf = context->buf; Oid argtypes[FUNC_MAX_ARGS]; int nargs; List *argnames; ListCell *l; if (list_length(wfunc->args) > FUNC_MAX_ARGS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_ARGUMENTS), errmsg("too many arguments"))); nargs = 0; argnames = NIL; foreach(l, wfunc->args) { Node *arg = (Node *) lfirst(l); if (IsA(arg, NamedArgExpr)) argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); argtypes[nargs] = exprType(arg); nargs++; } if (!funcname) funcname = generate_function_name(wfunc->winfnoid, nargs, argnames, argtypes, false, NULL, context->inGroupBy); appendStringInfo(buf, "%s(", funcname); /* winstar can be set only in zero-argument aggregates */ if (wfunc->winstar) appendStringInfoChar(buf, '*'); else { if (is_json_objectagg) { get_rule_expr((Node *) linitial(wfunc->args), context, false); appendStringInfoString(buf, " : "); get_rule_expr((Node *) lsecond(wfunc->args), context, false); } else get_rule_expr((Node *) wfunc->args, context, true); } if (options) appendStringInfoString(buf, options); if (wfunc->aggfilter != NULL) { appendStringInfoString(buf, ") FILTER (WHERE "); get_rule_expr((Node *) wfunc->aggfilter, context, false); } appendStringInfoString(buf, ") OVER "); if (context->windowClause) { /* Query-decompilation case: search the windowClause list */ foreach(l, context->windowClause) { WindowClause *wc = (WindowClause *) lfirst(l); if (wc->winref == wfunc->winref) { if (wc->name) appendStringInfoString(buf, quote_identifier(wc->name)); else get_rule_windowspec(wc, context->targetList, context); break; } } if (l == NULL) elog(ERROR, "could not find window clause for winref %u", wfunc->winref); } else { /* * In EXPLAIN, search the namespace stack for a matching WindowAgg * node (probably it's always the first entry), and print winname. */ foreach(l, context->namespaces) { deparse_namespace *dpns = (deparse_namespace *) lfirst(l); if (dpns->plan && IsA(dpns->plan, WindowAgg)) { WindowAgg *wagg = (WindowAgg *) dpns->plan; if (wagg->winref == wfunc->winref) { appendStringInfoString(buf, quote_identifier(wagg->winname)); break; } } } if (l == NULL) elog(ERROR, "could not find window clause for winref %u", wfunc->winref); } } /* * get_func_sql_syntax - Parse back a SQL-syntax function call * * Returns true if we successfully deparsed, false if we did not * recognize the function. */ static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context) { StringInfo buf = context->buf; Oid funcoid = expr->funcid; switch (funcoid) { case F_TIMEZONE_INTERVAL_TIMESTAMP: case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: case F_TIMEZONE_INTERVAL_TIMETZ: case F_TIMEZONE_TEXT_TIMESTAMP: case F_TIMEZONE_TEXT_TIMESTAMPTZ: case F_TIMEZONE_TEXT_TIMETZ: /* AT TIME ZONE ... note reversed argument order */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) lsecond(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " AT TIME ZONE "); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoChar(buf, ')'); return true; case F_TIMEZONE_TIMESTAMP: case F_TIMEZONE_TIMESTAMPTZ: case F_TIMEZONE_TIMETZ: /* AT LOCAL */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " AT LOCAL)"); return true; case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: case F_OVERLAPS_TIME_TIME_TIME_TIME: /* (x1, x2) OVERLAPS (y1, y2) */ appendStringInfoString(buf, "(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") OVERLAPS ("); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfourth(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_EXTRACT_TEXT_DATE: case F_EXTRACT_TEXT_TIME: case F_EXTRACT_TEXT_TIMETZ: case F_EXTRACT_TEXT_TIMESTAMP: case F_EXTRACT_TEXT_TIMESTAMPTZ: case F_EXTRACT_TEXT_INTERVAL: /* EXTRACT (x FROM y) */ appendStringInfoString(buf, "EXTRACT("); { Const *con = (Const *) linitial(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_IS_NORMALIZED: /* IS xxx NORMALIZED */ appendStringInfoChar(buf, '('); get_rule_expr_paren((Node *) linitial(expr->args), context, false, (Node *) expr); appendStringInfoString(buf, " IS"); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, " %s", TextDatumGetCString(con->constvalue)); } appendStringInfoString(buf, " NORMALIZED)"); return true; case F_PG_COLLATION_FOR: /* COLLATION FOR */ appendStringInfoString(buf, "COLLATION FOR ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_NORMALIZE: /* NORMALIZE() */ appendStringInfoString(buf, "NORMALIZE("); get_rule_expr((Node *) linitial(expr->args), context, false); if (list_length(expr->args) == 2) { Const *con = (Const *) lsecond(expr->args); Assert(IsA(con, Const) && con->consttype == TEXTOID && !con->constisnull); appendStringInfo(buf, ", %s", TextDatumGetCString(con->constvalue)); } appendStringInfoChar(buf, ')'); return true; case F_OVERLAY_BIT_BIT_INT4: case F_OVERLAY_BIT_BIT_INT4_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4: case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: case F_OVERLAY_TEXT_TEXT_INT4: case F_OVERLAY_TEXT_TEXT_INT4_INT4: /* OVERLAY() */ appendStringInfoString(buf, "OVERLAY("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " PLACING "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lthird(expr->args), context, false); if (list_length(expr->args) == 4) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lfourth(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_POSITION_BIT_BIT: case F_POSITION_BYTEA_BYTEA: case F_POSITION_TEXT_TEXT: /* POSITION() ... extra parens since args are b_expr not a_expr */ appendStringInfoString(buf, "POSITION(("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, ") IN ("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, "))"); return true; case F_SUBSTRING_BIT_INT4: case F_SUBSTRING_BIT_INT4_INT4: case F_SUBSTRING_BYTEA_INT4: case F_SUBSTRING_BYTEA_INT4_INT4: case F_SUBSTRING_TEXT_INT4: case F_SUBSTRING_TEXT_INT4_INT4: /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) lsecond(expr->args), context, false); if (list_length(expr->args) == 3) { appendStringInfoString(buf, " FOR "); get_rule_expr((Node *) lthird(expr->args), context, false); } appendStringInfoChar(buf, ')'); return true; case F_SUBSTRING_TEXT_TEXT_TEXT: /* SUBSTRING SIMILAR/ESCAPE */ appendStringInfoString(buf, "SUBSTRING("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, " SIMILAR "); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, " ESCAPE "); get_rule_expr((Node *) lthird(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_BTRIM_BYTEA_BYTEA: case F_BTRIM_TEXT: case F_BTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(BOTH"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_LTRIM_BYTEA_BYTEA: case F_LTRIM_TEXT: case F_LTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(LEADING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_RTRIM_BYTEA_BYTEA: case F_RTRIM_TEXT: case F_RTRIM_TEXT_TEXT: /* TRIM() */ appendStringInfoString(buf, "TRIM(TRAILING"); if (list_length(expr->args) == 2) { appendStringInfoChar(buf, ' '); get_rule_expr((Node *) lsecond(expr->args), context, false); } appendStringInfoString(buf, " FROM "); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoChar(buf, ')'); return true; case F_SYSTEM_USER: appendStringInfoString(buf, "SYSTEM_USER"); return true; case F_XMLEXISTS: /* XMLEXISTS ... extra parens because args are c_expr */ appendStringInfoString(buf, "XMLEXISTS(("); get_rule_expr((Node *) linitial(expr->args), context, false); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) lsecond(expr->args), context, false); appendStringInfoString(buf, "))"); return true; } return false; } /* ---------- * get_coercion_expr * * Make a string representation of a value coerced to a specific type * ---------- */ static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode) { StringInfo buf = context->buf; /* * Since parse_coerce.c doesn't immediately collapse application of * length-coercion functions to constants, what we'll typically see in * such cases is a Const with typmod -1 and a length-coercion function * right above it. Avoid generating redundant output. However, beware of * suppressing casts when the user actually wrote something like * 'foo'::text::char(3). * * Note: it might seem that we are missing the possibility of needing to * print a COLLATE clause for such a Const. However, a Const could only * have nondefault collation in a post-constant-folding tree, in which the * length coercion would have been folded too. See also the special * handling of CollateExpr in coerce_to_target_type(): any collation * marking will be above the coercion node, not below it. */ if (arg && IsA(arg, Const) && ((Const *) arg)->consttype == resulttype && ((Const *) arg)->consttypmod == -1) { /* Show the constant without normal ::typename decoration */ get_const_expr((Const *) arg, context, -1); } else { if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr_paren(arg, context, false, parentNode); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } appendStringInfo(buf, "::%s", format_type_with_typemod(resulttype, resulttypmod)); } /* ---------- * get_const_expr * * Make a string representation of a Const * * showtype can be -1 to never show "::typename" decoration, or +1 to always * show it, or 0 to show it only if the constant wouldn't be assumed to be * the right type by default. * * If the Const's collation isn't default for its type, show that too. * We mustn't do this when showtype is -1 (since that means the caller will * print "::typename", and we can't put a COLLATE clause in between). It's * caller's responsibility that collation isn't missed in such cases. * ---------- */ static void get_const_expr(Const *constval, deparse_context *context, int showtype) { StringInfo buf = context->buf; Oid typoutput; bool typIsVarlena; char *extval; bool needlabel = false; if (constval->constisnull) { /* * Always label the type of a NULL constant to prevent misdecisions * about type when reparsing. */ appendStringInfoString(buf, "NULL"); if (showtype >= 0) { appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } return; } getTypeOutputInfo(constval->consttype, &typoutput, &typIsVarlena); extval = OidOutputFunctionCall(typoutput, constval->constvalue); switch (constval->consttype) { case INT4OID: /* * INT4 can be printed without any decoration, unless it is * negative; in that case print it as '-nnn'::integer to ensure * that the output will re-parse as a constant, not as a constant * plus operator. In most cases we could get away with printing * (-nnn) instead, because of the way that gram.y handles negative * literals; but that doesn't work for INT_MIN, and it doesn't * seem that much prettier anyway. */ if (extval[0] != '-') appendStringInfoString(buf, extval); else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case NUMERICOID: /* * NUMERIC can be printed without quotes if it looks like a float * constant (not an integer, and not Infinity or NaN) and doesn't * have a leading sign (for the same reason as for INT4). */ if (isdigit((unsigned char) extval[0]) && strcspn(extval, "eE.") != strlen(extval)) { appendStringInfoString(buf, extval); } else { appendStringInfo(buf, "'%s'", extval); needlabel = true; /* we must attach a cast */ } break; case BITOID: case VARBITOID: appendStringInfo(buf, "B'%s'", extval); break; case BOOLOID: if (strcmp(extval, "t") == 0) appendStringInfoString(buf, "true"); else appendStringInfoString(buf, "false"); break; default: simple_quote_literal(buf, extval); break; } pfree(extval); if (showtype < 0) return; /* * For showtype == 0, append ::typename unless the constant will be * implicitly typed as the right type when it is read in. * * XXX this code has to be kept in sync with the behavior of the parser, * especially make_const. */ switch (constval->consttype) { case BOOLOID: case UNKNOWNOID: /* These types can be left unlabeled */ needlabel = false; break; case INT4OID: /* We determined above whether a label is needed */ break; case NUMERICOID: /* * Float-looking constants will be typed as numeric, which we * checked above; but if there's a nondefault typmod we need to * show it. */ needlabel |= (constval->consttypmod >= 0); break; default: needlabel = true; break; } if (needlabel || showtype > 0) appendStringInfo(buf, "::%s", format_type_with_typemod(constval->consttype, constval->consttypmod)); get_const_collation(constval, context); } /* * helper for get_const_expr: append COLLATE if needed */ static void get_const_collation(Const *constval, deparse_context *context) { StringInfo buf = context->buf; if (OidIsValid(constval->constcollid)) { Oid typcollation = get_typcollation(constval->consttype); if (constval->constcollid != typcollation) { appendStringInfo(buf, " COLLATE %s", generate_collation_name(constval->constcollid)); } } } /* * get_json_path_spec - Parse back a JSON path specification */ static void get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit) { if (IsA(path_spec, Const)) get_const_expr((Const *) path_spec, context, -1); else get_rule_expr(path_spec, context, showimplicit); } /* * get_json_format - Parse back a JsonFormat node */ static void get_json_format(JsonFormat *format, StringInfo buf) { if (format->format_type == JS_FORMAT_DEFAULT) return; appendStringInfoString(buf, format->format_type == JS_FORMAT_JSONB ? " FORMAT JSONB" : " FORMAT JSON"); if (format->encoding != JS_ENC_DEFAULT) { const char *encoding; encoding = format->encoding == JS_ENC_UTF16 ? "UTF16" : format->encoding == JS_ENC_UTF32 ? "UTF32" : "UTF8"; appendStringInfo(buf, " ENCODING %s", encoding); } } /* * get_json_returning - Parse back a JsonReturning structure */ static void get_json_returning(JsonReturning *returning, StringInfo buf, bool json_format_by_default) { if (!OidIsValid(returning->typid)) return; appendStringInfo(buf, " RETURNING %s", format_type_with_typemod(returning->typid, returning->typmod)); if (!json_format_by_default || returning->format->format_type != (returning->typid == JSONBOID ? JS_FORMAT_JSONB : JS_FORMAT_JSON)) get_json_format(returning->format, buf); } /* * get_json_constructor - Parse back a JsonConstructorExpr node */ static void get_json_constructor(JsonConstructorExpr *ctor, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; const char *funcname; bool is_json_object; int curridx; ListCell *lc; if (ctor->type == JSCTOR_JSON_OBJECTAGG) { get_json_agg_constructor(ctor, context, "JSON_OBJECTAGG", true); return; } else if (ctor->type == JSCTOR_JSON_ARRAYAGG) { get_json_agg_constructor(ctor, context, "JSON_ARRAYAGG", false); return; } switch (ctor->type) { case JSCTOR_JSON_OBJECT: funcname = "JSON_OBJECT"; break; case JSCTOR_JSON_ARRAY: funcname = "JSON_ARRAY"; break; case JSCTOR_JSON_PARSE: funcname = "JSON"; break; case JSCTOR_JSON_SCALAR: funcname = "JSON_SCALAR"; break; case JSCTOR_JSON_SERIALIZE: funcname = "JSON_SERIALIZE"; break; default: elog(ERROR, "invalid JsonConstructorType %d", ctor->type); } appendStringInfo(buf, "%s(", funcname); is_json_object = ctor->type == JSCTOR_JSON_OBJECT; foreach(lc, ctor->args) { curridx = foreach_current_index(lc); if (curridx > 0) { const char *sep; sep = (is_json_object && (curridx % 2) != 0) ? " : " : ", "; appendStringInfoString(buf, sep); } get_rule_expr((Node *) lfirst(lc), context, true); } get_json_constructor_options(ctor, buf); appendStringInfoChar(buf, ')'); } /* * Append options, if any, to the JSON constructor being deparsed */ static void get_json_constructor_options(JsonConstructorExpr *ctor, StringInfo buf) { if (ctor->absent_on_null) { if (ctor->type == JSCTOR_JSON_OBJECT || ctor->type == JSCTOR_JSON_OBJECTAGG) appendStringInfoString(buf, " ABSENT ON NULL"); } else { if (ctor->type == JSCTOR_JSON_ARRAY || ctor->type == JSCTOR_JSON_ARRAYAGG) appendStringInfoString(buf, " NULL ON NULL"); } if (ctor->unique) appendStringInfoString(buf, " WITH UNIQUE KEYS"); /* * Append RETURNING clause if needed; JSON() and JSON_SCALAR() don't * support one. */ if (ctor->type != JSCTOR_JSON_PARSE && ctor->type != JSCTOR_JSON_SCALAR) get_json_returning(ctor->returning, buf, true); } /* * get_json_agg_constructor - Parse back an aggregate JsonConstructorExpr node */ static void get_json_agg_constructor(JsonConstructorExpr *ctor, deparse_context *context, const char *funcname, bool is_json_objectagg) { StringInfoData options; initStringInfo(&options); get_json_constructor_options(ctor, &options); if (IsA(ctor->func, Aggref)) get_agg_expr_helper((Aggref *) ctor->func, context, (Aggref *) ctor->func, funcname, options.data, is_json_objectagg); else if (IsA(ctor->func, WindowFunc)) get_windowfunc_expr_helper((WindowFunc *) ctor->func, context, funcname, options.data, is_json_objectagg); else elog(ERROR, "invalid JsonConstructorExpr underlying node type: %d", nodeTag(ctor->func)); } /* * simple_quote_literal - Format a string as a SQL literal, append to buf */ static void simple_quote_literal(StringInfo buf, const char *val) { const char *valptr; /* * We form the string literal according to the prevailing setting of * standard_conforming_strings; we never use E''. User is responsible for * making sure result is used correctly. */ appendStringInfoChar(buf, '\''); for (valptr = val; *valptr; valptr++) { char ch = *valptr; if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) appendStringInfoChar(buf, ch); appendStringInfoChar(buf, ch); } appendStringInfoChar(buf, '\''); } /* ---------- * get_sublink_expr - Parse back a sublink * ---------- */ static void get_sublink_expr(SubLink *sublink, deparse_context *context) { StringInfo buf = context->buf; Query *query = (Query *) (sublink->subselect); char *opname = NULL; bool need_paren; if (sublink->subLinkType == ARRAY_SUBLINK) appendStringInfoString(buf, "ARRAY("); else appendStringInfoChar(buf, '('); /* * Note that we print the name of only the first operator, when there are * multiple combining operators. This is an approximation that could go * wrong in various scenarios (operators in different schemas, renamed * operators, etc) but there is not a whole lot we can do about it, since * the syntax allows only one operator to be shown. */ if (sublink->testexpr) { if (IsA(sublink->testexpr, OpExpr)) { /* single combining operator */ OpExpr *opexpr = (OpExpr *) sublink->testexpr; get_rule_expr(linitial(opexpr->args), context, true); opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); } else if (IsA(sublink->testexpr, BoolExpr)) { /* multiple combining operators, = or <> cases */ char *sep; ListCell *l; appendStringInfoChar(buf, '('); sep = ""; foreach(l, ((BoolExpr *) sublink->testexpr)->args) { OpExpr *opexpr = lfirst_node(OpExpr, l); appendStringInfoString(buf, sep); get_rule_expr(linitial(opexpr->args), context, true); if (!opname) opname = generate_operator_name(opexpr->opno, exprType(linitial(opexpr->args)), exprType(lsecond(opexpr->args))); sep = ", "; } appendStringInfoChar(buf, ')'); } else if (IsA(sublink->testexpr, RowCompareExpr)) { /* multiple combining operators, < <= > >= cases */ RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; appendStringInfoChar(buf, '('); get_rule_expr((Node *) rcexpr->largs, context, true); opname = generate_operator_name(linitial_oid(rcexpr->opnos), exprType(linitial(rcexpr->largs)), exprType(linitial(rcexpr->rargs))); appendStringInfoChar(buf, ')'); } else elog(ERROR, "unrecognized testexpr type: %d", (int) nodeTag(sublink->testexpr)); } need_paren = true; switch (sublink->subLinkType) { case EXISTS_SUBLINK: appendStringInfoString(buf, "EXISTS "); break; case ANY_SUBLINK: if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ appendStringInfoString(buf, " IN "); else appendStringInfo(buf, " %s ANY ", opname); break; case ALL_SUBLINK: appendStringInfo(buf, " %s ALL ", opname); break; case ROWCOMPARE_SUBLINK: appendStringInfo(buf, " %s ", opname); break; case EXPR_SUBLINK: case MULTIEXPR_SUBLINK: case ARRAY_SUBLINK: need_paren = false; break; case CTE_SUBLINK: /* shouldn't occur in a SubLink */ default: elog(ERROR, "unrecognized sublink type: %d", (int) sublink->subLinkType); break; } if (need_paren) appendStringInfoChar(buf, '('); get_query_def(query, buf, context->namespaces, NULL, false, context->prettyFlags, context->wrapColumn, context->indentLevel); if (need_paren) appendStringInfoString(buf, "))"); else appendStringInfoChar(buf, ')'); } /* ---------- * get_xmltable - Parse back a XMLTABLE function * ---------- */ static void get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; appendStringInfoString(buf, "XMLTABLE("); if (tf->ns_uris != NIL) { ListCell *lc1, *lc2; bool first = true; appendStringInfoString(buf, "XMLNAMESPACES ("); forboth(lc1, tf->ns_uris, lc2, tf->ns_names) { Node *expr = (Node *) lfirst(lc1); char *name = strVal(lfirst(lc2)); if (!first) appendStringInfoString(buf, ", "); else first = false; if (name != NULL) { get_rule_expr(expr, context, showimplicit); appendStringInfo(buf, " AS %s", quote_identifier(name)); } else { appendStringInfoString(buf, "DEFAULT "); get_rule_expr(expr, context, showimplicit); } } appendStringInfoString(buf, "), "); } appendStringInfoChar(buf, '('); get_rule_expr((Node *) tf->rowexpr, context, showimplicit); appendStringInfoString(buf, ") PASSING ("); get_rule_expr((Node *) tf->docexpr, context, showimplicit); appendStringInfoChar(buf, ')'); if (tf->colexprs != NIL) { ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; ListCell *l5; int colnum = 0; appendStringInfoString(buf, " COLUMNS "); forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, l4, tf->colexprs, l5, tf->coldefexprs) { char *colname = strVal(lfirst(l1)); Oid typid = lfirst_oid(l2); int32 typmod = lfirst_int(l3); Node *colexpr = (Node *) lfirst(l4); Node *coldefexpr = (Node *) lfirst(l5); bool ordinality = (tf->ordinalitycol == colnum); bool notnull = bms_is_member(colnum, tf->notnulls); if (colnum > 0) appendStringInfoString(buf, ", "); colnum++; appendStringInfo(buf, "%s %s", quote_identifier(colname), ordinality ? "FOR ORDINALITY" : format_type_with_typemod(typid, typmod)); if (ordinality) continue; if (coldefexpr != NULL) { appendStringInfoString(buf, " DEFAULT ("); get_rule_expr((Node *) coldefexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (colexpr != NULL) { appendStringInfoString(buf, " PATH ("); get_rule_expr((Node *) colexpr, context, showimplicit); appendStringInfoChar(buf, ')'); } if (notnull) appendStringInfoString(buf, " NOT NULL"); } } appendStringInfoChar(buf, ')'); } /* * get_json_table_nested_columns - Parse back nested JSON_TABLE columns */ static void get_json_table_nested_columns(TableFunc *tf, JsonTablePlan *plan, deparse_context *context, bool showimplicit, bool needcomma) { if (IsA(plan, JsonTablePathScan)) { JsonTablePathScan *scan = castNode(JsonTablePathScan, plan); if (needcomma) appendStringInfoChar(context->buf, ','); appendStringInfoChar(context->buf, ' '); appendContextKeyword(context, "NESTED PATH ", 0, 0, 0); get_const_expr(scan->path->value, context, -1); appendStringInfo(context->buf, " AS %s", quote_identifier(scan->path->name)); get_json_table_columns(tf, scan, context, showimplicit); } else if (IsA(plan, JsonTableSiblingJoin)) { JsonTableSiblingJoin *join = (JsonTableSiblingJoin *) plan; get_json_table_nested_columns(tf, join->lplan, context, showimplicit, needcomma); get_json_table_nested_columns(tf, join->rplan, context, showimplicit, true); } } /* * get_json_table_columns - Parse back JSON_TABLE columns */ static void get_json_table_columns(TableFunc *tf, JsonTablePathScan *scan, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; ListCell *lc_colname; ListCell *lc_coltype; ListCell *lc_coltypmod; ListCell *lc_colvalexpr; int colnum = 0; appendStringInfoChar(buf, ' '); appendContextKeyword(context, "COLUMNS (", 0, 0, 0); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; forfour(lc_colname, tf->colnames, lc_coltype, tf->coltypes, lc_coltypmod, tf->coltypmods, lc_colvalexpr, tf->colvalexprs) { char *colname = strVal(lfirst(lc_colname)); JsonExpr *colexpr; Oid typid; int32 typmod; bool ordinality; JsonBehaviorType default_behavior; typid = lfirst_oid(lc_coltype); typmod = lfirst_int(lc_coltypmod); colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr)); /* Skip columns that don't belong to this scan. */ if (scan->colMin < 0 || colnum < scan->colMin) { colnum++; continue; } if (colnum > scan->colMax) break; if (colnum > scan->colMin) appendStringInfoString(buf, ", "); colnum++; ordinality = !colexpr; appendContextKeyword(context, "", 0, 0, 0); appendStringInfo(buf, "%s %s", quote_identifier(colname), ordinality ? "FOR ORDINALITY" : format_type_with_typemod(typid, typmod)); if (ordinality) continue; /* * Set default_behavior to guide get_json_expr_options() on whether to * emit the ON ERROR / EMPTY clauses. */ if (colexpr->op == JSON_EXISTS_OP) { appendStringInfoString(buf, " EXISTS"); default_behavior = JSON_BEHAVIOR_FALSE; } else { if (colexpr->op == JSON_QUERY_OP) { char typcategory; bool typispreferred; get_type_category_preferred(typid, &typcategory, &typispreferred); if (typcategory == TYPCATEGORY_STRING) appendStringInfoString(buf, colexpr->format->format_type == JS_FORMAT_JSONB ? " FORMAT JSONB" : " FORMAT JSON"); } default_behavior = JSON_BEHAVIOR_NULL; } appendStringInfoString(buf, " PATH "); get_json_path_spec(colexpr->path_spec, context, showimplicit); get_json_expr_options(colexpr, context, default_behavior); } if (scan->child) get_json_table_nested_columns(tf, scan->child, context, showimplicit, scan->colMin >= 0); if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; appendContextKeyword(context, ")", 0, 0, 0); } /* ---------- * get_json_table - Parse back a JSON_TABLE function * ---------- */ static void get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit) { StringInfo buf = context->buf; JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr); JsonTablePathScan *root = castNode(JsonTablePathScan, tf->plan); appendStringInfoString(buf, "JSON_TABLE("); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; appendContextKeyword(context, "", 0, 0, 0); get_rule_expr(jexpr->formatted_expr, context, showimplicit); appendStringInfoString(buf, ", "); get_const_expr(root->path->value, context, -1); appendStringInfo(buf, " AS %s", quote_identifier(root->path->name)); if (jexpr->passing_values) { ListCell *lc1, *lc2; bool needcomma = false; appendStringInfoChar(buf, ' '); appendContextKeyword(context, "PASSING ", 0, 0, 0); if (PRETTY_INDENT(context)) context->indentLevel += PRETTYINDENT_VAR; forboth(lc1, jexpr->passing_names, lc2, jexpr->passing_values) { if (needcomma) appendStringInfoString(buf, ", "); needcomma = true; appendContextKeyword(context, "", 0, 0, 0); get_rule_expr((Node *) lfirst(lc2), context, false); appendStringInfo(buf, " AS %s", quote_identifier((lfirst_node(String, lc1))->sval) ); } if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; } get_json_table_columns(tf, castNode(JsonTablePathScan, tf->plan), context, showimplicit); if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY_ARRAY) get_json_behavior(jexpr->on_error, context, "ERROR"); if (PRETTY_INDENT(context)) context->indentLevel -= PRETTYINDENT_VAR; appendContextKeyword(context, ")", 0, 0, 0); } /* ---------- * get_tablefunc - Parse back a table function * ---------- */ static void get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) { /* XMLTABLE and JSON_TABLE are the only existing implementations. */ if (tf->functype == TFT_XMLTABLE) get_xmltable(tf, context, showimplicit); else if (tf->functype == TFT_JSON_TABLE) get_json_table(tf, context, showimplicit); } /* ---------- * get_from_clause - Parse back a FROM clause * * "prefix" is the keyword that denotes the start of the list of FROM * elements. It is FROM when used to parse back SELECT and UPDATE, but * is USING when parsing back DELETE. * ---------- */ static void get_from_clause(Query *query, const char *prefix, deparse_context *context) { StringInfo buf = context->buf; bool first = true; ListCell *l; /* * We use the query's jointree as a guide to what to print. However, we * must ignore auto-added RTEs that are marked not inFromCl. (These can * only appear at the top level of the jointree, so it's sufficient to * check here.) This check also ensures we ignore the rule pseudo-RTEs * for NEW and OLD. */ foreach(l, query->jointree->fromlist) { Node *jtnode = (Node *) lfirst(l); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); if (!rte->inFromCl) continue; } if (first) { appendContextKeyword(context, prefix, -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); first = false; get_from_clause_item(jtnode, query, context); } else { StringInfoData itembuf; appendStringInfoString(buf, ", "); /* * Put the new FROM item's text into itembuf so we can decide * after we've got it whether or not it needs to go on a new line. */ initStringInfo(&itembuf); context->buf = &itembuf; get_from_clause_item(jtnode, query, context); /* Restore context's output buffer */ context->buf = buf; /* Consider line-wrapping if enabled */ if (PRETTY_INDENT(context) && context->wrapColumn >= 0) { /* Does the new item start with a new line? */ if (itembuf.len > 0 && itembuf.data[0] == '\n') { /* If so, we shouldn't add anything */ /* instead, remove any trailing spaces currently in buf */ removeStringInfoSpaces(buf); } else { char *trailing_nl; /* Locate the start of the current line in the buffer */ trailing_nl = strrchr(buf->data, '\n'); if (trailing_nl == NULL) trailing_nl = buf->data; else trailing_nl++; /* * Add a newline, plus some indentation, if the new item * would cause an overflow. */ if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) appendContextKeyword(context, "", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_VAR); } } /* Add the new item */ appendStringInfoString(buf, itembuf.data); /* clean up */ pfree(itembuf.data); } } } static void get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) { StringInfo buf = context->buf; deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); if (IsA(jtnode, RangeTblRef)) { int varno = ((RangeTblRef *) jtnode)->rtindex; RangeTblEntry *rte = rt_fetch(varno, query->rtable); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); RangeTblFunction *rtfunc1 = NULL; CitusRTEKind rteKind = GetRangeTblKind(rte); if (rte->lateral) appendStringInfoString(buf, "LATERAL "); /* Print the FROM item proper */ switch (rte->rtekind) { case RTE_RELATION: /* Normal relation RTE */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_relation_or_shard_name(rte->relid, context->distrelid, context->shardid, context->namespaces)); break; case RTE_SUBQUERY: /* Subquery RTE */ appendStringInfoChar(buf, '('); get_query_def(rte->subquery, buf, context->namespaces, NULL, true, context->prettyFlags, context->wrapColumn, context->indentLevel); appendStringInfoChar(buf, ')'); break; case RTE_FUNCTION: /* if it's a shard, do differently */ if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) { char *fragmentSchemaName = NULL; char *fragmentTableName = NULL; ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); /* use schema and table name from the remote alias */ appendStringInfo(buf, "%s%s", only_marker(rte), generate_fragment_name(fragmentSchemaName, fragmentTableName)); break; } /* Function RTE */ rtfunc1 = (RangeTblFunction *) linitial(rte->functions); /* * Omit ROWS FROM() syntax for just one function, unless it * has both a coldeflist and WITH ORDINALITY. If it has both, * we must use ROWS FROM() syntax to avoid ambiguity about * whether the coldeflist includes the ordinality column. */ if (list_length(rte->functions) == 1 && (rtfunc1->funccolnames == NIL || !rte->funcordinality)) { get_rule_expr_funccall(rtfunc1->funcexpr, context, true); /* we'll print the coldeflist below, if it has one */ } else { bool all_unnest; ListCell *lc; /* * If all the function calls in the list are to unnest, * and none need a coldeflist, then collapse the list back * down to UNNEST(args). (If we had more than one * built-in unnest function, this would get more * difficult.) * * XXX This is pretty ugly, since it makes not-terribly- * future-proof assumptions about what the parser would do * with the output; but the alternative is to emit our * nonstandard ROWS FROM() notation for what might have * been a perfectly spec-compliant multi-argument * UNNEST(). */ all_unnest = true; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (!IsA(rtfunc->funcexpr, FuncExpr) || ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || rtfunc->funccolnames != NIL) { all_unnest = false; break; } } if (all_unnest) { List *allargs = NIL; foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); List *args = ((FuncExpr *) rtfunc->funcexpr)->args; allargs = list_concat(allargs, args); } appendStringInfoString(buf, "UNNEST("); get_rule_expr((Node *) allargs, context, true); appendStringInfoChar(buf, ')'); } else { int funcno = 0; appendStringInfoString(buf, "ROWS FROM("); foreach(lc, rte->functions) { RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (funcno > 0) appendStringInfoString(buf, ", "); get_rule_expr_funccall(rtfunc->funcexpr, context, true); if (rtfunc->funccolnames != NIL) { /* Reconstruct the column definition list */ appendStringInfoString(buf, " AS "); get_from_clause_coldeflist(rtfunc, NULL, context); } funcno++; } appendStringInfoChar(buf, ')'); } /* prevent printing duplicate coldeflist below */ rtfunc1 = NULL; } if (rte->funcordinality) appendStringInfoString(buf, " WITH ORDINALITY"); break; case RTE_TABLEFUNC: get_tablefunc(rte->tablefunc, context, true); break; case RTE_VALUES: /* Values list RTE */ appendStringInfoChar(buf, '('); get_values_def(rte->values_lists, context); appendStringInfoChar(buf, ')'); break; case RTE_CTE: appendStringInfoString(buf, quote_identifier(rte->ctename)); break; default: elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); break; } /* Print the relation alias, if needed */ get_rte_alias(rte, varno, false, context); /* Print the column definitions or aliases, if needed */ if (rtfunc1 && rtfunc1->funccolnames != NIL) { /* Reconstruct the columndef list, which is also the aliases */ get_from_clause_coldeflist(rtfunc1, colinfo, context); } else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD || (rte->alias != NULL && rte->alias->colnames != NIL)) { /* Else print column aliases as needed */ get_column_alias_list(colinfo, context); } /* check if column's are given aliases in distributed tables */ else if (colinfo->parentUsing != NIL) { Assert(colinfo->printaliases); get_column_alias_list(colinfo, context); } /* Tablesample clause must go after any alias */ if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && rte->tablesample) { get_tablesample_def(rte->tablesample, context); } } else if (IsA(jtnode, JoinExpr)) { JoinExpr *j = (JoinExpr *) jtnode; deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); bool need_paren_on_right; need_paren_on_right = PRETTY_PAREN(context) && !IsA(j->rarg, RangeTblRef) && !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, '('); get_from_clause_item(j->larg, query, context); switch (j->jointype) { case JOIN_INNER: if (j->quals) appendContextKeyword(context, " JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); else appendContextKeyword(context, " CROSS JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_LEFT: appendContextKeyword(context, " LEFT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_FULL: appendContextKeyword(context, " FULL JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; case JOIN_RIGHT: appendContextKeyword(context, " RIGHT JOIN ", -PRETTYINDENT_STD, PRETTYINDENT_STD, PRETTYINDENT_JOIN); break; default: elog(ERROR, "unrecognized join type: %d", (int) j->jointype); } if (need_paren_on_right) appendStringInfoChar(buf, '('); get_from_clause_item(j->rarg, query, context); if (need_paren_on_right) appendStringInfoChar(buf, ')'); if (j->usingClause) { ListCell *lc; bool first = true; appendStringInfoString(buf, " USING ("); /* Use the assigned names, not what's in usingClause */ foreach(lc, colinfo->usingNames) { char *colname = (char *) lfirst(lc); if (first) first = false; else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } appendStringInfoChar(buf, ')'); if (j->join_using_alias) appendStringInfo(buf, " AS %s", quote_identifier(j->join_using_alias->aliasname)); } else if (j->quals) { appendStringInfoString(buf, " ON "); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, '('); get_rule_expr(j->quals, context, false); if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); } else if (j->jointype != JOIN_INNER) { /* If we didn't say CROSS JOIN above, we must provide an ON */ appendStringInfoString(buf, " ON TRUE"); } if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, ')'); /* Yes, it's correct to put alias after the right paren ... */ if (j->alias != NULL) { /* * Note that it's correct to emit an alias clause if and only if * there was one originally. Otherwise we'd be converting a named * join to unnamed or vice versa, which creates semantic * subtleties we don't want. However, we might print a different * alias name than was there originally. */ appendStringInfo(buf, " %s", quote_identifier(get_rtable_name(j->rtindex, context))); get_column_alias_list(colinfo, context); } } else elog(ERROR, "unrecognized node type: %d", (int) nodeTag(jtnode)); } /* * get_rte_alias - print the relation's alias, if needed * * If printed, the alias is preceded by a space, or by " AS " if use_as is true. */ static void get_rte_alias(RangeTblEntry *rte, int varno, bool use_as, deparse_context *context) { deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); char *refname = get_rtable_name(varno, context); deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); bool printalias = false; if (rte->alias != NULL) { /* Always print alias if user provided one */ printalias = true; } else if (colinfo->printaliases) { /* Always print alias if we need to print column aliases */ printalias = true; } else if (rte->rtekind == RTE_RELATION) { /* * No need to print alias if it's same as relation name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, get_relation_name(rte->relid)) != 0) printalias = true; } else if (rte->rtekind == RTE_FUNCTION) { /* * For a function RTE, always print alias. This covers possible * renaming of the function and/or instability of the FigureColname * rules for things that aren't simple functions. Note we'd need to * force it anyway for the columndef list case. */ printalias = true; } else if (rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_VALUES) { /* * For a subquery, always print alias. This makes the output * SQL-spec-compliant, even though we allow such aliases to be omitted * on input. */ printalias = true; } else if (rte->rtekind == RTE_CTE) { /* * No need to print alias if it's same as CTE name (this would * normally be the case, but not if set_rtable_names had to resolve a * conflict). */ if (strcmp(refname, rte->ctename) != 0) printalias = true; } if (printalias) appendStringInfo(context->buf, "%s%s", use_as ? " AS " : " ", quote_identifier(refname)); } /* * get_column_alias_list - print column alias list for an RTE * * Caller must already have printed the relation's alias name. */ static void get_column_alias_list(deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; int i; bool first = true; /* Don't print aliases if not needed */ if (!colinfo->printaliases) return; for (i = 0; i < colinfo->num_new_cols; i++) { char *colname = colinfo->new_colnames[i]; if (first) { appendStringInfoChar(buf, '('); first = false; } else appendStringInfoString(buf, ", "); appendStringInfoString(buf, quote_identifier(colname)); } if (!first) appendStringInfoChar(buf, ')'); } /* * get_from_clause_coldeflist - reproduce FROM clause coldeflist * * When printing a top-level coldeflist (which is syntactically also the * relation's column alias list), use column names from colinfo. But when * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the * original coldeflist's names, which are available in rtfunc->funccolnames. * Pass NULL for colinfo to select the latter behavior. * * The coldeflist is appended immediately (no space) to buf. Caller is * responsible for ensuring that an alias or AS is present before it. */ static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_columns *colinfo, deparse_context *context) { StringInfo buf = context->buf; ListCell *l1; ListCell *l2; ListCell *l3; ListCell *l4; int i; appendStringInfoChar(buf, '('); i = 0; forfour(l1, rtfunc->funccoltypes, l2, rtfunc->funccoltypmods, l3, rtfunc->funccolcollations, l4, rtfunc->funccolnames) { Oid atttypid = lfirst_oid(l1); int32 atttypmod = lfirst_int(l2); Oid attcollation = lfirst_oid(l3); char *attname; if (colinfo) attname = colinfo->colnames[i]; else attname = strVal(lfirst(l4)); Assert(attname); /* shouldn't be any dropped columns here */ if (i > 0) appendStringInfoString(buf, ", "); appendStringInfo(buf, "%s %s", quote_identifier(attname), format_type_with_typemod(atttypid, atttypmod)); if (OidIsValid(attcollation) && attcollation != get_typcollation(atttypid)) appendStringInfo(buf, " COLLATE %s", generate_collation_name(attcollation)); i++; } appendStringInfoChar(buf, ')'); } /* * get_tablesample_def - print a TableSampleClause */ static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) { StringInfo buf = context->buf; Oid argtypes[1]; int nargs; ListCell *l; /* * We should qualify the handler's function name if it wouldn't be * resolved by lookup in the current search path. */ argtypes[0] = INTERNALOID; appendStringInfo(buf, " TABLESAMPLE %s (", generate_function_name(tablesample->tsmhandler, 1, NIL, argtypes, false, NULL, false)); nargs = 0; foreach(l, tablesample->args) { if (nargs++ > 0) appendStringInfoString(buf, ", "); get_rule_expr((Node *) lfirst(l), context, false); } appendStringInfoChar(buf, ')'); if (tablesample->repeatable != NULL) { appendStringInfoString(buf, " REPEATABLE ("); get_rule_expr((Node *) tablesample->repeatable, context, false); appendStringInfoChar(buf, ')'); } } /* * get_opclass_name - fetch name of an index operator class * * The opclass name is appended (after a space) to buf. * * Output is suppressed if the opclass is the default for the given * actual_datatype. (If you don't want this behavior, just pass * InvalidOid for actual_datatype.) */ static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf) { HeapTuple ht_opc; Form_pg_opclass opcrec; char *opcname; char *nspname; ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); if (!HeapTupleIsValid(ht_opc)) elog(ERROR, "cache lookup failed for opclass %u", opclass); opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); if (!OidIsValid(actual_datatype) || GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) { /* Okay, we need the opclass name. Do we need to qualify it? */ opcname = NameStr(opcrec->opcname); if (OpclassIsVisible(opclass)) appendStringInfo(buf, " %s", quote_identifier(opcname)); else { nspname = get_namespace_name_or_temp(opcrec->opcnamespace); appendStringInfo(buf, " %s.%s", quote_identifier(nspname), quote_identifier(opcname)); } } ReleaseSysCache(ht_opc); } /* * processIndirection - take care of array and subfield assignment * * We strip any top-level FieldStore or assignment SubscriptingRef nodes that * appear in the input, printing them as decoration for the base column * name (which we assume the caller just printed). We might also need to * strip CoerceToDomain nodes, but only ones that appear above assignment * nodes. * * Returns the subexpression that's to be assigned. */ static Node * processIndirection(Node *node, deparse_context *context) { StringInfo buf = context->buf; CoerceToDomain *cdomain = NULL; for (;;) { if (node == NULL) break; if (IsA(node, FieldStore)) { FieldStore *fstore = (FieldStore *) node; Oid typrelid; char *fieldname; /* lookup tuple type */ typrelid = get_typ_typrelid(fstore->resulttype); if (!OidIsValid(typrelid)) elog(ERROR, "argument type %s of FieldStore is not a tuple type", format_type_be(fstore->resulttype)); /* * Print the field name. There should only be one target field in * stored rules. There could be more than that in executable * target lists, but this function cannot be used for that case. */ Assert(list_length(fstore->fieldnums) == 1); fieldname = get_attname(typrelid, linitial_int(fstore->fieldnums), false); appendStringInfo(buf, ".%s", quote_identifier(fieldname)); /* * We ignore arg since it should be an uninteresting reference to * the target column or subcolumn. */ node = (Node *) linitial(fstore->newvals); } else if (IsA(node, SubscriptingRef)) { SubscriptingRef *sbsref = (SubscriptingRef *) node; if (sbsref->refassgnexpr == NULL) break; printSubscripts(sbsref, context); /* * We ignore refexpr since it should be an uninteresting reference * to the target column or subcolumn. */ node = (Node *) sbsref->refassgnexpr; } else if (IsA(node, CoerceToDomain)) { cdomain = (CoerceToDomain *) node; /* If it's an explicit domain coercion, we're done */ if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) break; /* Tentatively descend past the CoerceToDomain */ node = (Node *) cdomain->arg; } else break; } /* * If we descended past a CoerceToDomain whose argument turned out not to * be a FieldStore or array assignment, back up to the CoerceToDomain. * (This is not enough to be fully correct if there are nested implicit * CoerceToDomains, but such cases shouldn't ever occur.) */ if (cdomain && node == (Node *) cdomain->arg) node = (Node *) cdomain; return node; } static void printSubscripts(SubscriptingRef *sbsref, deparse_context *context) { StringInfo buf = context->buf; ListCell *lowlist_item; ListCell *uplist_item; lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ foreach(uplist_item, sbsref->refupperindexpr) { appendStringInfoChar(buf, '['); if (lowlist_item) { /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(lowlist_item), context, false); appendStringInfoChar(buf, ':'); lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); } /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(uplist_item), context, false); appendStringInfoChar(buf, ']'); } } /* * get_relation_name * Get the unqualified name of a relation specified by OID * * This differs from the underlying get_rel_name() function in that it will * throw error instead of silently returning NULL if the OID is bad. */ static char * get_relation_name(Oid relid) { char *relname = get_rel_name(relid); if (!relname) elog(ERROR, "cache lookup failed for relation %u", relid); return relname; } /* * generate_relation_or_shard_name * Compute the name to display for a relation or shard * * If the provided relid is equal to the provided distrelid, this function * returns a shard-extended relation name; otherwise, it falls through to a * simple generate_relation_name call. */ static char * generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, List *namespaces) { char *relname = NULL; if (relid == distrelid) { relname = get_relation_name(relid); if (shardid > 0) { Oid schemaOid = get_rel_namespace(relid); char *schemaName = get_namespace_name_or_temp(schemaOid); AppendShardIdToName(&relname, shardid); relname = quote_qualified_identifier(schemaName, relname); } } else { relname = generate_relation_name(relid, namespaces); } return relname; } /* * generate_relation_name * Compute the name to display for a relation specified by OID * * The result includes all necessary quoting and schema-prefixing. * * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. * We will forcibly qualify the relation name if it equals any CTE name * visible in the namespace list. */ char * generate_relation_name(Oid relid, List *namespaces) { HeapTuple tp; Form_pg_class reltup; bool need_qual; ListCell *nslist; char *relname; char *nspname; char *result; tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for relation %u", relid); reltup = (Form_pg_class) GETSTRUCT(tp); relname = NameStr(reltup->relname); /* Check for conflicting CTE name */ need_qual = false; foreach(nslist, namespaces) { deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); ListCell *ctlist; foreach(ctlist, dpns->ctes) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); if (strcmp(cte->ctename, relname) == 0) { need_qual = true; break; } } if (need_qual) break; } /* Otherwise, qualify the name if not visible in search path */ if (!need_qual) need_qual = !RelationIsVisible(relid); if (need_qual) nspname = get_namespace_name_or_temp(reltup->relnamespace); else nspname = NULL; result = quote_qualified_identifier(nspname, relname); ReleaseSysCache(tp); return result; } /* * generate_rte_shard_name returns the qualified name of the shard given a * CITUS_RTE_SHARD range table entry. */ static char * generate_rte_shard_name(RangeTblEntry *rangeTableEntry) { char *shardSchemaName = NULL; char *shardTableName = NULL; Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, NULL); return generate_fragment_name(shardSchemaName, shardTableName); } /* * generate_fragment_name * Compute the name to display for a shard or merged table * * The result includes all necessary quoting and schema-prefixing. The schema * name can be NULL for regular shards. For merged tables, they are always * declared within a job-specific schema, and therefore can't have null schema * names. */ static char * generate_fragment_name(char *schemaName, char *tableName) { StringInfo fragmentNameString = makeStringInfo(); if (schemaName != NULL) { appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), quote_identifier(tableName)); } else { appendStringInfoString(fragmentNameString, quote_identifier(tableName)); } return fragmentNameString->data; } /* * generate_function_name * Compute the name to display for a function specified by OID, * given that it is being called with the specified actual arg names and * types. (Those matter because of ambiguous-function resolution rules.) * * If we're dealing with a potentially variadic function (in practice, this * means a FuncExpr or Aggref, not some other way of calling a function), then * has_variadic must specify whether variadic arguments have been merged, * and *use_variadic_p will be set to indicate whether to print VARIADIC in * the output. For non-FuncExpr cases, has_variadic should be false and * use_variadic_p can be NULL. * * inGroupBy must be true if we're deparsing a GROUP BY clause. * * The result includes all necessary quoting and schema-prefixing. */ static char * generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, bool has_variadic, bool *use_variadic_p, bool inGroupBy) { char *result; HeapTuple proctup; Form_pg_proc procform; char *proname; bool use_variadic; char *nspname; FuncDetailCode p_result; Oid p_funcid; Oid p_rettype; bool p_retset; int p_nvargs; Oid p_vatype; Oid *p_true_typeids; bool force_qualify = false; proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); if (!HeapTupleIsValid(proctup)) elog(ERROR, "cache lookup failed for function %u", funcid); procform = (Form_pg_proc) GETSTRUCT(proctup); proname = NameStr(procform->proname); /* * Due to parser hacks to avoid needing to reserve CUBE, we need to force * qualification of some function names within GROUP BY. */ if (inGroupBy) { if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) force_qualify = true; } /* * Determine whether VARIADIC should be printed. We must do this first * since it affects the lookup rules in func_get_detail(). * * Currently, we always print VARIADIC if the function has a merged * variadic-array argument. Note that this is always the case for * functions taking a VARIADIC argument type other than VARIADIC ANY. * * In principle, if VARIADIC wasn't originally specified and the array * actual argument is deconstructable, we could print the array elements * separately and not print VARIADIC, thus more nearly reproducing the * original input. For the moment that seems like too much complication * for the benefit, and anyway we do not know whether VARIADIC was * originally specified if it's a non-ANY type. */ if (use_variadic_p) { /* Parser should not have set funcvariadic unless fn is variadic */ Assert(!has_variadic || OidIsValid(procform->provariadic)); use_variadic = has_variadic; *use_variadic_p = use_variadic; } else { Assert(!has_variadic); use_variadic = false; } /* * The idea here is to schema-qualify only if the parser would fail to * resolve the correct function given the unqualified func name with the * specified argtypes and VARIADIC flag. But if we already decided to * force qualification, then we can skip the lookup and pretend we didn't * find it. */ if (!force_qualify) p_result = func_get_detail(list_make1(makeString(proname)), NIL, argnames, nargs, argtypes, !use_variadic, true, false, &p_funcid, &p_rettype, &p_retset, &p_nvargs, &p_vatype, &p_true_typeids, NULL); else { p_result = FUNCDETAIL_NOTFOUND; p_funcid = InvalidOid; } if ((p_result == FUNCDETAIL_NORMAL || p_result == FUNCDETAIL_AGGREGATE || p_result == FUNCDETAIL_WINDOWFUNC) && p_funcid == funcid) nspname = NULL; else nspname = get_namespace_name_or_temp(procform->pronamespace); result = quote_qualified_identifier(nspname, proname); ReleaseSysCache(proctup); return result; } /* * generate_operator_name * Compute the name to display for an operator specified by OID, * given that it is being called with the specified actual arg types. * (Arg types matter because of ambiguous-operator resolution rules. * Pass InvalidOid for unused arg of a unary operator.) * * The result includes all necessary quoting and schema-prefixing, * plus the OPERATOR() decoration needed to use a qualified operator name * in an expression. */ char * generate_operator_name(Oid operid, Oid arg1, Oid arg2) { StringInfoData buf; HeapTuple opertup; Form_pg_operator operform; char *oprname; char *nspname; initStringInfo(&buf); opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); if (!HeapTupleIsValid(opertup)) elog(ERROR, "cache lookup failed for operator %u", operid); operform = (Form_pg_operator) GETSTRUCT(opertup); oprname = NameStr(operform->oprname); /* * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, * we don't check if the operator is in current namespace or not. This is * because this check is costly when the operator is not in current namespace. */ nspname = get_namespace_name_or_temp(operform->oprnamespace); Assert(nspname != NULL); appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); appendStringInfoString(&buf, oprname); appendStringInfoChar(&buf, ')'); ReleaseSysCache(opertup); return buf.data; } /* * get_one_range_partition_bound_string * A C string representation of one range partition bound */ char * get_range_partbound_string(List *bound_datums) { deparse_context context; StringInfo buf = makeStringInfo(); ListCell *cell; char *sep; memset(&context, 0, sizeof(deparse_context)); context.buf = buf; appendStringInfoChar(buf, '('); sep = ""; foreach(cell, bound_datums) { PartitionRangeDatum *datum = lfirst_node(PartitionRangeDatum, cell); appendStringInfoString(buf, sep); if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) appendStringInfoString(buf, "MINVALUE"); else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) appendStringInfoString(buf, "MAXVALUE"); else { Const *val = castNode(Const, datum->value); get_const_expr(val, &context, -1); } sep = ", "; } appendStringInfoChar(buf, ')'); return buf->data; } /* * Collect a list of OIDs of all sequences owned by the specified relation, * and column if specified. If deptype is not zero, then only find sequences * with the specified dependency type. */ List * getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) { List *result = NIL; Relation depRel; ScanKeyData key[3]; SysScanDesc scan; HeapTuple tup; depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); if (attnum) ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* * We assume any auto or internal dependency of a sequence on a column * must be what we are looking for. (We need the relkind test because * indexes can also have auto dependencies on columns.) */ if (deprec->classid == RelationRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { if (!deptype || deprec->deptype == deptype) result = lappend_oid(result, deprec->objid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return result; } /* * get_insert_column_names_list Prepares the insert-column-names list. Any indirection * decoration needed on the column names can be inferred from the top targetlist. */ static List * get_insert_column_names_list(List *targetList, StringInfo buf, deparse_context *context, RangeTblEntry *rte) { char *sep; ListCell *l; List *strippedexprs; strippedexprs = NIL; sep = ""; appendStringInfoChar(buf, '('); foreach(l, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (tle->resjunk) continue; /* ignore junk entries */ appendStringInfoString(buf, sep); sep = ", "; /* * Put out name of target column; look in the catalogs, not at * tle->resname, since resname will fail to track RENAME. */ appendStringInfoString(buf, quote_identifier(get_attname(rte->relid, tle->resno, false))); /* * Print any indirection needed (subfields or subscripts), and strip * off the top-level nodes representing the indirection assignments. * Add the stripped expressions to strippedexprs. (If it's a * single-VALUES statement, the stripped expressions are the VALUES to * print below. Otherwise they're just Vars and not really * interesting.) */ strippedexprs = lappend(strippedexprs, processIndirection((Node *) tle->expr, context)); } appendStringInfoString(buf, ") "); return strippedexprs; } #endif /* (PG_VERSION_NUM >= PG_VERSION_17) && (PG_VERSION_NUM < PG_VERSION_18) */ ================================================ FILE: src/backend/distributed/executor/adaptive_executor.c ================================================ /*------------------------------------------------------------------------- * * adaptive_executor.c * * The adaptive executor executes a list of tasks (queries on shards) over * a connection pool per worker node. The results of the queries, if any, * are written to a tuple store. * * The concepts in the executor are modelled in a set of structs: * * - DistributedExecution: * Execution of a Task list over a set of WorkerPools. * - WorkerPool * Pool of WorkerSessions for the same worker which opportunistically * executes "unassigned" tasks from a queue. * - WorkerSession: * Connection to a worker that is used to execute "assigned" tasks * from a queue and may execute unassigned tasks from the WorkerPool. * - ShardCommandExecution: * Execution of a Task across a list of placements. * - TaskPlacementExecution: * Execution of a Task on a specific placement. * Used in the WorkerPool and WorkerSession queues. * * Every connection pool (WorkerPool) and every connection (WorkerSession) * have a queue of tasks that are ready to execute (readyTaskQueue) and a * queue/set of pending tasks that may become ready later in the execution * (pendingTaskQueue). The tasks are wrapped in a ShardCommandExecution, * which keeps track of the state of execution and is referenced from a * TaskPlacementExecution, which is the data structure that is actually * added to the queues and describes the state of the execution of a task * on a particular worker node. * * When the task list is part of a bigger distributed transaction, the * shards that are accessed or modified by the task may have already been * accessed earlier in the transaction. We need to make sure we use the * same connection since it may hold relevant locks or have uncommitted * writes. In that case we "assign" the task to a connection by adding * it to the task queue of specific connection (in * AssignTasksToConnectionsOrWorkerPool). Otherwise we consider the task * unassigned and add it to the task queue of a worker pool, which means * that it can be executed over any connection in the pool. * * A task may be executed on multiple placements in case of a reference * table or a replicated distributed table. Depending on the type of * task, it may not be ready to be executed on a worker node immediately. * For instance, INSERTs on a reference table are executed serially across * placements to avoid deadlocks when concurrent INSERTs take conflicting * locks. At the beginning, only the "first" placement is ready to execute * and therefore added to the readyTaskQueue in the pool or connection. * The remaining placements are added to the pendingTaskQueue. Once * execution on the first placement is done the second placement moves * from pendingTaskQueue to readyTaskQueue. The same approach is used to * fail over read-only tasks to another placement. * * Once all the tasks are added to a queue, the main loop in * RunDistributedExecution repeatedly does the following: * * For each pool: * - ManageWorkPool evaluates whether to open additional connections * based on the number unassigned tasks that are ready to execute * and the targetPoolSize of the execution. * * Poll all connections: * - We use a WaitEventSet that contains all (non-failed) connections * and is rebuilt whenever the set of active connections or any of * their wait flags change. * * We almost always check for WL_SOCKET_READABLE because a session * can emit notices at any time during execution, but it will only * wake up WaitEventSetWait when there are actual bytes to read. * * We check for WL_SOCKET_WRITEABLE just after sending bytes in case * there is not enough space in the TCP buffer. Since a socket is * almost always writable we also use WL_SOCKET_WRITEABLE as a * mechanism to wake up WaitEventSetWait for non-I/O events, e.g. * when a task moves from pending to ready. * * For each connection that is ready: * - ConnectionStateMachine handles connection establishment and failure * as well as command execution via TransactionStateMachine. * * When a connection is ready to execute a new task, it first checks its * own readyTaskQueue and otherwise takes a task from the worker pool's * readyTaskQueue (on a first-come-first-serve basis). * * In cases where the tasks finish quickly (e.g. <1ms), a single * connection will often be sufficient to finish all tasks. It is * therefore not necessary that all connections are established * successfully or open a transaction (which may be blocked by an * intermediate pgbouncer in transaction pooling mode). It is therefore * essential that we take a task from the queue only after opening a * transaction block. * * When a command on a worker finishes or the connection is lost, we call * PlacementExecutionDone, which then updates the state of the task * based on whether we need to run it on other placements. When a * connection fails or all connections to a worker fail, we also call * PlacementExecutionDone for all queued tasks to try the next placement * and, if necessary, mark shard placements as inactive. If a task fails * to execute on all placements, the execution fails and the distributed * transaction rolls back. * * For multi-row INSERTs, tasks are executed sequentially by * SequentialRunDistributedExecution instead of in parallel, which allows * a high degree of concurrency without high risk of deadlocks. * Conversely, multi-row UPDATE/DELETE/DDL commands take aggressive locks * which forbids concurrency, but allows parallelism without high risk * of deadlocks. Note that this is unrelated to SEQUENTIAL_CONNECTION, * which indicates that we should use at most one connection per node, but * can run tasks in parallel across nodes. This is used when there are * writes to a reference table that has foreign keys from a distributed * table. * * Execution finishes when all tasks are done, the query errors out, or * the user cancels the query. * *------------------------------------------------------------------------- */ #include #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "commands/schemacmds.h" #include "lib/ilist.h" #include "portability/instr_time.h" #include "storage/fd.h" #include "storage/latch.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" #include "utils/timestamp.h" #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/cancel_utils.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_execution_locks.h" #include "distributed/executor_util.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/param_utils.h" #include "distributed/placement_access.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/repartition_join_execution.h" #include "distributed/resource_lock.h" #include "distributed/shared_connection_stats.h" #include "distributed/stats/stat_counters.h" #include "distributed/subplan_execution.h" #include "distributed/transaction_identifier.h" #include "distributed/transaction_management.h" #include "distributed/tuple_destination.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #define SLOW_START_DISABLED 0 /* * DistributedExecution represents the execution of a distributed query * plan. */ typedef struct DistributedExecution { /* the corresponding distributed plan's modLevel */ RowModifyLevel modLevel; /* * remoteAndLocalTaskList contains all the tasks required to finish the * execution. remoteTaskList contains all the tasks required to * finish the remote execution. localTaskList contains all the * local tasks required to finish the local execution. * * remoteAndLocalTaskList is the union of remoteTaskList and localTaskList. */ List *remoteAndLocalTaskList; List *remoteTaskList; List *localTaskList; /* * If a task specific destination is not provided for a task, then use * defaultTupleDest. */ TupleDestination *defaultTupleDest; /* Parameters for parameterized plans. Can be NULL. */ ParamListInfo paramListInfo; /* list of workers involved in the execution */ List *workerList; /* list of all connections used for distributed execution */ List *sessionList; /* * Flag to indiciate that the set of connections we are interested * in has changed and waitEventSet needs to be rebuilt. */ bool rebuildWaitEventSet; /* * Flag to indiciate that the set of wait events we are interested * in might have changed and waitEventSet needs to be updated. * * Note that we set this flag whenever we assign a value to waitFlags, * but we don't check that the waitFlags is actually different from the * previous value. So we might have some false positives for this flag, * which is OK, because in this case ModifyWaitEvent() is noop. */ bool waitFlagsChanged; /* * WaitEventSet used for waiting for I/O events. * * This could also be local to RunDistributedExecution(), but in that case * we had to mark it as "volatile" to avoid PG_TRY()/PG_CATCH() issues, and * cast it to non-volatile when doing WaitEventSetFree(). We thought that * would make code a bit harder to read than making this non-local, so we * move it here. See comments for PG_TRY() in postgres/src/include/elog.h * and "man 3 siglongjmp" for more context. * * Another reason for keeping these here is to cache a single * WaitEventSet/WaitEvent within the execution pair until we * need to rebuild the waitEvents. */ WaitEventSet *waitEventSet; WaitEvent *events; int eventSetSize; /* * The number of connections we aim to open per worker. * * If there are no more tasks to assigned, the actual number may be lower. * If there are already more connections, the actual number may be higher. */ int targetPoolSize; /* total number of tasks to execute */ int totalTaskCount; /* number of tasks that still need to be executed */ int unfinishedTaskCount; /* * Flag to indicate whether throwing errors on cancellation is * allowed. */ bool raiseInterrupts; /* transactional properties of the current execution */ TransactionProperties *transactionProperties; /* indicates whether distributed execution has failed */ bool failed; /* * For SELECT commands or INSERT/UPDATE/DELETE commands with RETURNING, * the total number of rows received from the workers. For * INSERT/UPDATE/DELETE commands without RETURNING, the total number of * tuples modified. * * Note that for replicated tables (e.g., reference tables), we only consider * a single replica's rows that are processed. */ uint64 rowsProcessed; /* * The following fields are used while receiving results from remote nodes. * We store this information here to avoid re-allocating it every time. * * columnArray field is reset/calculated per row, so might be useless for * other contexts. The benefit of keeping it here is to avoid allocating * the array over and over again. */ uint32 allocatedColumnCount; void **columnArray; StringInfoData *stringInfoDataArray; /* * jobIdList contains all jobs in the job tree, this is used to * do cleanup for repartition queries. */ List *jobIdList; /* * Indicates whether we can execute tasks locally during distributed * execution. In other words, this flag must be set to false when * executing a command that we surely know that local execution would * fail, such as CREATE INDEX CONCURRENTLY. */ bool localExecutionSupported; } DistributedExecution; /* * WorkerPoolFailureState indicates the current state of the * pool. */ typedef enum WorkerPoolFailureState { /* safe to continue execution*/ WORKER_POOL_NOT_FAILED, /* if a pool fails, the execution fails */ WORKER_POOL_FAILED, /* * The remote execution over the pool failed, but we failed over * to the local execution and still finish the execution. */ WORKER_POOL_FAILED_OVER_TO_LOCAL } WorkerPoolFailureState; /* * WorkerPool represents a pool of sessions on the same worker. * * A WorkerPool has two queues containing the TaskPlacementExecutions that need * to be executed on the worker. * * TaskPlacementExecutions that are ready to execute are in readyTaskQueue. * TaskPlacementExecutions that may need to be executed once execution on * another worker finishes or fails are in pendingTaskQueue. * * In TransactionStateMachine, the sessions opportunistically take * TaskPlacementExecutions from the readyQueue when they are ready and have no * assigned tasks. * * We track connection timeouts per WorkerPool. When the first connection is * established we set the poolStartTime and if no connection can be established * before NodeConnectionTime, the WorkerPool fails. There is some specialised * logic in case citus.force_max_query_parallelization is enabled because we * may fail to establish a connection per placement after already establishing * some connections earlier in the execution. * * A WorkerPool fails if all connection attempts failed or all connections * are lost. In that case, all TaskPlacementExecutions in the queues are * marked as failed in PlacementExecutionDone, which typically causes the * task and therefore the distributed execution to fail. In case of a * replicated table or a SELECT on a reference table, the remaining placements * will be tried by moving them from a pendingTaskQueue to a readyTaskQueue. */ typedef struct WorkerPool { /* distributed execution in which the worker participates */ DistributedExecution *distributedExecution; /* worker node on which we have a pool of sessions */ char *nodeName; int nodePort; /* all sessions on the worker that are part of the current execution */ List *sessionList; /* number of connections that were established */ int activeConnectionCount; /* * Keep track of how many connections are ready for execution, in * order to (efficiently) know whether more connections to the worker * are needed. */ int idleConnectionCount; /* number of connections that did not send a command */ int unusedConnectionCount; /* number of failed connections */ int failedConnectionCount; /* * Placement executions destined for worker node, but not assigned to any * connection and not yet ready to start (depends on other placement * executions). */ dlist_head pendingTaskQueue; /* * Placement executions destined for worker node, but not assigned to any * connection and ready to start. */ dlist_head readyTaskQueue; int readyTaskCount; /* * We keep this for enforcing the connection timeouts. In our definition, a pool * starts when the first connection establishment starts. */ instr_time poolStartTime; /* indicates whether to check for the connection timeout */ bool checkForPoolTimeout; /* last time we opened a connection */ instr_time lastConnectionOpenTime; /* maximum number of connections we are allowed to open at once */ uint32 maxNewConnectionsPerCycle; /* * Set to true if the pool is to local node. We use this value to * avoid re-calculating often. */ bool poolToLocalNode; /* * This is only set in WorkerPoolFailed() function. Once a pool fails, we do not * use it anymore. */ WorkerPoolFailureState failureState; /* execution statistics per pool, in microseconds */ uint64 totalTaskExecutionTime; int totalExecutedTasks; } WorkerPool; struct TaskPlacementExecution; /* * WorkerSession represents a session on a worker that can execute tasks * (sequentially) and is part of a WorkerPool. * * Each WorkerSession has two queues containing TaskPlacementExecutions that * need to be executed within this particular session because the session * accessed the same or co-located placements earlier in the transaction. * * TaskPlacementExecutions that are ready to execute are in readyTaskQueue. * TaskPlacementExecutions that may need to be executed once execution on * another worker finishes or fails are in pendingTaskQueue. */ typedef struct WorkerSession { /* only useful for debugging */ uint64 sessionId; /* worker pool of which this session is part */ WorkerPool *workerPool; /* connection over which the session is established */ MultiConnection *connection; /* tasks that need to be executed on this connection, but are not ready to start */ dlist_head pendingTaskQueue; /* tasks that need to be executed on this connection and are ready to start */ dlist_head readyTaskQueue; /* task the worker should work on or NULL */ struct TaskPlacementExecution *currentTask; /* * The number of commands sent to the worker over the session. Excludes * distributed transaction related commands such as BEGIN/COMMIT etc. */ uint64 commandsSent; /* index in the wait event set */ int waitEventSetIndex; /* events reported by the latest call to WaitEventSetWait */ int latestUnconsumedWaitEvents; /* for some restricted scenarios, we allow a single connection retry */ bool connectionRetried; /* keep track of if the session has an active connection */ bool sessionHasActiveConnection; } WorkerSession; /* GUC, determining whether Citus opens 1 connection per task */ bool ForceMaxQueryParallelization = false; int MaxAdaptiveExecutorPoolSize = 16; bool EnableBinaryProtocol = true; /* GUC, number of ms to wait between opening connections to the same worker */ int ExecutorSlowStartInterval = 10; bool EnableCostBasedConnectionEstablishment = true; bool PreventIncompleteConnectionEstablishment = true; /* * TaskExecutionState indicates whether or not a command on a shard * has finished, or whether it has failed. */ typedef enum TaskExecutionState { TASK_EXECUTION_NOT_FINISHED, TASK_EXECUTION_FINISHED, TASK_EXECUTION_FAILED, TASK_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION } TaskExecutionState; /* * PlacementExecutionOrder indicates whether a command should be executed * on any replica, on all replicas sequentially (in order), or on all * replicas in parallel. In other words, EXECUTION_ORDER_ANY is used for * SELECTs, EXECUTION_ORDER_SEQUENTIAL/EXECUTION_ORDER_PARALLEL is used for * DML/DDL. */ typedef enum PlacementExecutionOrder { EXECUTION_ORDER_ANY, EXECUTION_ORDER_SEQUENTIAL, EXECUTION_ORDER_PARALLEL, } PlacementExecutionOrder; /* * ShardCommandExecution represents an execution of a command on a shard * that may (need to) run across multiple placements. */ typedef struct ShardCommandExecution { /* description of the task */ Task *task; /* cached AttInMetadata for task */ AttInMetadata **attributeInputMetadata; /* indicates whether the attributeInputMetadata has binary or text * encoding/decoding functions */ bool binaryResults; /* order in which the command should be replicated on replicas */ PlacementExecutionOrder executionOrder; /* executions of the command on the placements of the shard */ struct TaskPlacementExecution **placementExecutions; int placementExecutionCount; /* * RETURNING results from other shard placements can be ignored * after we got results from the first placements. */ bool gotResults; TaskExecutionState executionState; /* * Indicates whether given shard command can be executed locally on * placements. Normally determined by DistributedExecution's same field. */ bool localExecutionSupported; } ShardCommandExecution; /* * TaskPlacementExecutionState indicates whether a command is running * on a shard placement, or finished or failed. */ typedef enum TaskPlacementExecutionState { PLACEMENT_EXECUTION_NOT_READY, PLACEMENT_EXECUTION_READY, PLACEMENT_EXECUTION_RUNNING, PLACEMENT_EXECUTION_FINISHED, PLACEMENT_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION, PLACEMENT_EXECUTION_FAILED } TaskPlacementExecutionState; /* * TaskPlacementExecution represents the execution of a command * on a shard placement. */ typedef struct TaskPlacementExecution { /* shard command execution of which this placement execution is part */ ShardCommandExecution *shardCommandExecution; /* shard placement on which this command runs */ ShardPlacement *shardPlacement; /* state of the execution of the command on the placement */ TaskPlacementExecutionState executionState; /* * Task query can contain multiple queries. queryIndex tracks results of * which query we are waiting for. */ uint32 queryIndex; /* worker pool on which the placement needs to be executed */ WorkerPool *workerPool; /* the session the placement execution is assigned to or NULL */ WorkerSession *assignedSession; /* membership in assigned task queue of a particular session */ dlist_node sessionPendingQueueNode; /* membership in ready-to-start assigned task queue of a particular session */ dlist_node sessionReadyQueueNode; /* membership in assigned task queue of worker */ dlist_node workerPendingQueueNode; /* membership in ready-to-start task queue of worker */ dlist_node workerReadyQueueNode; /* index in array of placement executions in a ShardCommandExecution */ int placementExecutionIndex; /* execution time statistics for this placement execution */ instr_time startTime; instr_time endTime; } TaskPlacementExecution; /* local functions */ static DistributedExecution * CreateDistributedExecution(RowModifyLevel modLevel, List *taskList, ParamListInfo paramListInfo, int targetPoolSize, TupleDestination * defaultTupleDest, TransactionProperties * xactProperties, List *jobIdList, bool localExecutionSupported); static TransactionProperties DecideTaskListTransactionProperties(RowModifyLevel modLevel, List *taskList, bool excludeFromTransaction); static void StartDistributedExecution(DistributedExecution *execution); static void RunLocalExecution(CitusScanState *scanState, DistributedExecution *execution); static void RunDistributedExecution(DistributedExecution *execution); static void SequentialRunDistributedExecution(DistributedExecution *execution); static void FinishDistributedExecution(DistributedExecution *execution); static void CleanUpSessions(DistributedExecution *execution); static bool DistributedExecutionModifiesDatabase(DistributedExecution *execution); static void AssignTasksToConnectionsOrWorkerPool(DistributedExecution *execution); static void UnclaimAllSessionConnections(List *sessionList); static PlacementExecutionOrder ExecutionOrderForTask(RowModifyLevel modLevel, Task *task); static WorkerPool * FindOrCreateWorkerPool(DistributedExecution *execution, char *nodeName, int nodePort); static WorkerSession * FindOrCreateWorkerSession(WorkerPool *workerPool, MultiConnection *connection); static void ManageWorkerPool(WorkerPool *workerPool); static bool ShouldWaitForSlowStart(WorkerPool *workerPool); static int CalculateNewConnectionCount(WorkerPool *workerPool); static bool UsingExistingSessionsCheaperThanEstablishingNewConnections(int readyTaskCount, WorkerPool * workerPool); static double AvgTaskExecutionTimeApproximation(WorkerPool *workerPool); static double AvgConnectionEstablishmentTime(WorkerPool *workerPool); static void OpenNewConnections(WorkerPool *workerPool, int newConnectionCount, TransactionProperties *transactionProperties); static void CheckConnectionTimeout(WorkerPool *workerPool); static void MarkEstablishingSessionsTimedOut(WorkerPool *workerPool); static int UsableConnectionCount(WorkerPool *workerPool); static long NextEventTimeout(DistributedExecution *execution); static WaitEventSet * BuildWaitEventSet(List *sessionList); static void FreeExecutionWaitEvents(DistributedExecution *execution); static void AddSessionToWaitEventSet(WorkerSession *session, WaitEventSet *waitEventSet); static void RebuildWaitEventSetFlags(WaitEventSet *waitEventSet, List *sessionList); static TaskPlacementExecution * PopPlacementExecution(WorkerSession *session); static TaskPlacementExecution * PopAssignedPlacementExecution(WorkerSession *session); static TaskPlacementExecution * PopUnassignedPlacementExecution(WorkerPool *workerPool); static bool StartPlacementExecutionOnSession(TaskPlacementExecution *placementExecution, WorkerSession *session); static bool SendNextQuery(TaskPlacementExecution *placementExecution, WorkerSession *session); static void ConnectionStateMachine(WorkerSession *session); static bool HasUnfinishedTaskForSession(WorkerSession *session); static void HandleMultiConnectionSuccess(WorkerSession *session, bool newConnection); static bool HasAnyConnectionFailure(WorkerPool *workerPool); static void Activate2PCIfModifyingTransactionExpandsToNewNode(WorkerSession *session); static bool TransactionModifiedDistributedTable(DistributedExecution *execution); static void TransactionStateMachine(WorkerSession *session); static void UpdateConnectionWaitFlags(WorkerSession *session, int waitFlags); static bool CheckConnectionReady(WorkerSession *session); static bool ReceiveResults(WorkerSession *session, bool storeRows); static void WorkerSessionFailed(WorkerSession *session); static void WorkerPoolFailed(WorkerPool *workerPool); static void PlacementExecutionDone(TaskPlacementExecution *placementExecution, bool succeeded); static void ScheduleNextPlacementExecution(TaskPlacementExecution *placementExecution, bool succeeded); static bool CanFailoverPlacementExecutionToLocalExecution(TaskPlacementExecution * placementExecution); static void PlacementExecutionReady(TaskPlacementExecution *placementExecution); static TaskExecutionState TaskExecutionStateMachine(ShardCommandExecution * shardCommandExecution); static int GetEventSetSize(List *sessionList); static bool ProcessSessionsWithFailedWaitEventSetOperations(DistributedExecution * execution); static bool HasIncompleteConnectionEstablishment(DistributedExecution *execution); static void RebuildWaitEventSet(DistributedExecution *execution); static void RebuildWaitEventSetForSessions(DistributedExecution *execution); static void AddLatchWaitEventToExecution(DistributedExecution *execution); static void ProcessWaitEvents(DistributedExecution *execution, WaitEvent *events, int eventCount, bool *cancellationReceived); static void RemoteSocketClosedForAnySession(DistributedExecution *execution); static void ProcessWaitEventsForSocketClosed(WaitEvent *events, int eventCount); static long MillisecondsBetweenTimestamps(instr_time startTime, instr_time endTime); static uint64 MicrosecondsBetweenTimestamps(instr_time startTime, instr_time endTime); static int WorkerPoolCompare(const void *lhsKey, const void *rhsKey); static void SetAttributeInputMetadata(DistributedExecution *execution, ShardCommandExecution *shardCommandExecution); static ExecutionParams * CreateDefaultExecutionParams(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults, ParamListInfo paramListInfo); /* * AdaptiveExecutorPreExecutorRun gets called right before postgres starts its executor * run. Given that the result of our subplans would be evaluated before the first call to * the exec function of our custom scan we make sure our subplans have executed before. */ void AdaptiveExecutorPreExecutorRun(CitusScanState *scanState) { if (scanState->finishedPreScan) { /* * Cursors (and hence RETURN QUERY syntax in pl/pgsql functions) * may trigger AdaptiveExecutorPreExecutorRun() on every fetch * operation. Though, we should only execute PreScan once. */ return; } DistributedPlan *distributedPlan = scanState->distributedPlan; /* * PostgreSQL takes locks on all partitions in the executor. It's not entirely * clear why this is necessary (instead of locking the parent during DDL), but * we do the same for consistency. */ LockPartitionsForDistributedPlan(distributedPlan); ExecuteSubPlans(distributedPlan, RequestedForExplainAnalyze(scanState)); scanState->finishedPreScan = true; } /* * AdaptiveExecutor is called via CitusExecScan on the * first call of CitusExecScan. The function fills the tupleStore * of the input scanScate. */ TupleTableSlot * AdaptiveExecutor(CitusScanState *scanState) { TupleTableSlot *resultSlot = NULL; DistributedPlan *distributedPlan = scanState->distributedPlan; EState *executorState = ScanStateGetExecutorState(scanState); ParamListInfo paramListInfo = executorState->es_param_list_info; bool randomAccess = true; bool interTransactions = false; int targetPoolSize = MaxAdaptiveExecutorPoolSize; List *jobIdList = NIL; Job *job = distributedPlan->workerJob; List *taskList = job->taskList; /* we should only call this once before the scan finished */ Assert(!scanState->finishedRemoteScan); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "AdaptiveExecutor", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); /* Reset Task fields that are only valid for a single execution */ ResetExplainAnalyzeData(taskList); scanState->tuplestorestate = tuplestore_begin_heap(randomAccess, interTransactions, work_mem); TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); TupleDestination *defaultTupleDest = CreateTupleStoreTupleDest(scanState->tuplestorestate, tupleDescriptor); bool localExecutionSupported = true; if (RequestedForExplainAnalyze(scanState)) { /* * We use multiple queries per task in EXPLAIN ANALYZE which need to * be part of the same transaction. */ UseCoordinatedTransaction(); taskList = ExplainAnalyzeTaskList(taskList, defaultTupleDest, tupleDescriptor, paramListInfo); /* * Multiple queries per task is not supported with local execution. See the Assert in * TupleDestDestReceiverReceive. */ localExecutionSupported = false; } bool hasDependentJobs = job->dependentJobList != NIL; if (hasDependentJobs) { /* jobs use intermediate results, which require a distributed transaction */ UseCoordinatedTransaction(); jobIdList = ExecuteDependentTasks(taskList, job); } if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { /* defer decision after ExecuteSubPlans() */ targetPoolSize = 1; } bool excludeFromXact = false; TransactionProperties xactProperties = DecideTaskListTransactionProperties( distributedPlan->modLevel, taskList, excludeFromXact); /* * In some rare cases, we have prepared statements that pass a parameter * and never used in the query, mark such parameters' type as Invalid(0), * which will be used later in ExtractParametersFromParamList() to map them * to a generic datatype. Skip for dynamic parameters. */ if (paramListInfo && !paramListInfo->paramFetch) { paramListInfo = copyParamList(paramListInfo); MarkUnreferencedExternParams((Node *) job->jobQuery, paramListInfo); } DistributedExecution *execution = CreateDistributedExecution( distributedPlan->modLevel, taskList, paramListInfo, targetPoolSize, defaultTupleDest, &xactProperties, jobIdList, localExecutionSupported); /* * Make sure that we acquire the appropriate locks even if the local tasks * are going to be executed with local execution. */ StartDistributedExecution(execution); if (ShouldRunTasksSequentially(execution->remoteTaskList)) { SequentialRunDistributedExecution(execution); } else { RunDistributedExecution(execution); } /* execute tasks local to the node (if any) */ if (list_length(execution->localTaskList) > 0) { /* now execute the local tasks */ RunLocalExecution(scanState, execution); } CmdType commandType = job->jobQuery->commandType; if (commandType != CMD_SELECT) { executorState->es_processed = execution->rowsProcessed; } FinishDistributedExecution(execution); if (SortReturning && distributedPlan->expectResults && commandType != CMD_SELECT) { SortTupleStore(scanState); } MemoryContextSwitchTo(oldContext); return resultSlot; } /* * RunLocalExecution runs the localTaskList in the execution, fills the tuplestore * and sets the es_processed if necessary. * * It also sorts the tuplestore if there are no remote tasks remaining. */ static void RunLocalExecution(CitusScanState *scanState, DistributedExecution *execution) { EState *estate = ScanStateGetExecutorState(scanState); bool isUtilityCommand = false; uint64 rowsProcessed = ExecuteLocalTaskListExtended(execution->localTaskList, estate->es_param_list_info, scanState->distributedPlan, execution->defaultTupleDest, isUtilityCommand); execution->rowsProcessed += rowsProcessed; } /* * ExecuteUtilityTaskList is a wrapper around executing task * list for utility commands. */ uint64 ExecuteUtilityTaskList(List *utilityTaskList, bool localExecutionSupported) { RowModifyLevel modLevel = ROW_MODIFY_NONE; ExecutionParams *executionParams = CreateBasicExecutionParams( modLevel, utilityTaskList, MaxAdaptiveExecutorPoolSize, localExecutionSupported ); executionParams->xactProperties = DecideTaskListTransactionProperties(modLevel, utilityTaskList, false); executionParams->isUtilityCommand = true; return ExecuteTaskListExtended(executionParams); } /* * ExecuteUtilityTaskListExtended is a wrapper around executing task * list for utility commands. */ uint64 ExecuteUtilityTaskListExtended(List *utilityTaskList, int poolSize, bool localExecutionSupported) { RowModifyLevel modLevel = ROW_MODIFY_NONE; ExecutionParams *executionParams = CreateBasicExecutionParams( modLevel, utilityTaskList, poolSize, localExecutionSupported ); bool excludeFromXact = false; executionParams->xactProperties = DecideTaskListTransactionProperties(modLevel, utilityTaskList, excludeFromXact); executionParams->isUtilityCommand = true; return ExecuteTaskListExtended(executionParams); } /* * ExecuteTaskList is a proxy to ExecuteTaskListExtended * with defaults for some of the arguments. */ uint64 ExecuteTaskList(RowModifyLevel modLevel, List *taskList) { bool localExecutionSupported = true; ExecutionParams *executionParams = CreateBasicExecutionParams( modLevel, taskList, MaxAdaptiveExecutorPoolSize, localExecutionSupported ); bool excludeFromXact = false; executionParams->xactProperties = DecideTaskListTransactionProperties( modLevel, taskList, excludeFromXact); return ExecuteTaskListExtended(executionParams); } /* * ExecuteTaskListOutsideTransaction is a proxy to ExecuteTaskListExtended * with defaults for some of the arguments. */ uint64 ExecuteTaskListOutsideTransaction(RowModifyLevel modLevel, List *taskList, int targetPoolSize, List *jobIdList) { /* * As we are going to run the tasks outside transaction, we shouldn't use local execution. * However, there is some problem when using local execution related to * repartition joins, when we solve that problem, we can execute the tasks * coming to this path with local execution. See PR:3711 */ bool localExecutionSupported = false; ExecutionParams *executionParams = CreateBasicExecutionParams( modLevel, taskList, targetPoolSize, localExecutionSupported ); executionParams->xactProperties = DecideTaskListTransactionProperties( modLevel, taskList, true); return ExecuteTaskListExtended(executionParams); } /* * CreateDefaultExecutionParams returns execution params based on given (possibly null) * bind params (presumably from executor state) with defaults for some of the arguments. */ static ExecutionParams * CreateDefaultExecutionParams(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults, ParamListInfo paramListInfo) { int targetPoolSize = MaxAdaptiveExecutorPoolSize; bool localExecutionSupported = true; ExecutionParams *executionParams = CreateBasicExecutionParams( modLevel, taskList, targetPoolSize, localExecutionSupported ); executionParams->xactProperties = DecideTaskListTransactionProperties( modLevel, taskList, false); executionParams->expectResults = expectResults; executionParams->tupleDestination = tupleDest; executionParams->paramListInfo = paramListInfo; return executionParams; } /* * ExecuteTaskListIntoTupleDestWithParam is a proxy to ExecuteTaskListExtended() which uses * bind params from executor state, and with defaults for some of the arguments. */ uint64 ExecuteTaskListIntoTupleDestWithParam(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults, ParamListInfo paramListInfo) { ExecutionParams *executionParams = CreateDefaultExecutionParams(modLevel, taskList, tupleDest, expectResults, paramListInfo); return ExecuteTaskListExtended(executionParams); } /* * ExecuteTaskListIntoTupleDest is a proxy to ExecuteTaskListExtended() with defaults * for some of the arguments. */ uint64 ExecuteTaskListIntoTupleDest(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults) { ParamListInfo paramListInfo = NULL; ExecutionParams *executionParams = CreateDefaultExecutionParams(modLevel, taskList, tupleDest, expectResults, paramListInfo); return ExecuteTaskListExtended(executionParams); } /* * ExecuteTaskListExtended sets up the execution for given task list and * runs it. */ uint64 ExecuteTaskListExtended(ExecutionParams *executionParams) { /* if there are no tasks to execute, we can return early */ if (list_length(executionParams->taskList) == 0) { return 0; } uint64 locallyProcessedRows = 0; TupleDestination *defaultTupleDest = executionParams->tupleDestination; if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { executionParams->targetPoolSize = 1; } DistributedExecution *execution = CreateDistributedExecution( executionParams->modLevel, executionParams->taskList, executionParams->paramListInfo, executionParams->targetPoolSize, defaultTupleDest, &executionParams->xactProperties, executionParams->jobIdList, executionParams->localExecutionSupported); /* * If current transaction accessed local placements and task list includes * tasks that should be executed locally (accessing any of the local placements), * then we should error out as it would cause inconsistencies across the * remote connection and local execution. */ EnsureCompatibleLocalExecutionState(execution->remoteTaskList); /* run the remote execution */ StartDistributedExecution(execution); RunDistributedExecution(execution); FinishDistributedExecution(execution); /* now, switch back to the local execution */ if (executionParams->isUtilityCommand) { locallyProcessedRows += ExecuteLocalUtilityTaskList(execution->localTaskList); } else { locallyProcessedRows += ExecuteLocalTaskList(execution->localTaskList, defaultTupleDest); } return execution->rowsProcessed + locallyProcessedRows; } /* * CreateBasicExecutionParams creates basic execution parameters with some common * fields. */ ExecutionParams * CreateBasicExecutionParams(RowModifyLevel modLevel, List *taskList, int targetPoolSize, bool localExecutionSupported) { ExecutionParams *executionParams = palloc0(sizeof(ExecutionParams)); executionParams->modLevel = modLevel; executionParams->taskList = taskList; executionParams->targetPoolSize = targetPoolSize; executionParams->localExecutionSupported = localExecutionSupported; executionParams->tupleDestination = CreateTupleDestNone(); executionParams->expectResults = false; executionParams->isUtilityCommand = false; executionParams->jobIdList = NIL; executionParams->paramListInfo = NULL; return executionParams; } /* * CreateDistributedExecution creates a distributed execution data structure for * a distributed plan. */ static DistributedExecution * CreateDistributedExecution(RowModifyLevel modLevel, List *taskList, ParamListInfo paramListInfo, int targetPoolSize, TupleDestination *defaultTupleDest, TransactionProperties *xactProperties, List *jobIdList, bool localExecutionSupported) { DistributedExecution *execution = (DistributedExecution *) palloc0(sizeof(DistributedExecution)); execution->modLevel = modLevel; execution->remoteAndLocalTaskList = taskList; execution->transactionProperties = xactProperties; /* we are going to calculate this values below */ execution->localTaskList = NIL; execution->remoteTaskList = NIL; execution->paramListInfo = paramListInfo; execution->workerList = NIL; execution->sessionList = NIL; execution->targetPoolSize = targetPoolSize; execution->defaultTupleDest = defaultTupleDest; execution->rowsProcessed = 0; execution->raiseInterrupts = true; execution->rebuildWaitEventSet = false; execution->waitFlagsChanged = false; execution->jobIdList = jobIdList; execution->localExecutionSupported = localExecutionSupported; /* * Since task can have multiple queries, we are not sure how many columns we should * allocate for. We start with 16, and reallocate when we need more. */ execution->allocatedColumnCount = 16; execution->columnArray = palloc0(execution->allocatedColumnCount * sizeof(void *)); if (EnableBinaryProtocol) { /* * Initialize enough StringInfos for each column. These StringInfos * (and thus the backing buffers) will be reused for each row. * We will reference these StringInfos in the columnArray if the value * is not NULL. * * NOTE: StringInfos are always grown in the memory context in which * they were initially created. So appending in any memory context will * result in bufferes that are still valid after removing that memory * context. */ execution->stringInfoDataArray = palloc0( execution->allocatedColumnCount * sizeof(StringInfoData)); for (int i = 0; i < execution->allocatedColumnCount; i++) { initStringInfo(&execution->stringInfoDataArray[i]); } } if (execution->localExecutionSupported && ShouldExecuteTasksLocally(taskList)) { bool readOnlyPlan = !TaskListModifiesDatabase(modLevel, taskList); ExtractLocalAndRemoteTasks(readOnlyPlan, taskList, &execution->localTaskList, &execution->remoteTaskList); } else { /* * Get a shallow copy of the list as we rely on remoteAndLocalTaskList * across the execution. */ execution->remoteTaskList = list_copy(execution->remoteAndLocalTaskList); } execution->totalTaskCount = list_length(execution->remoteTaskList); execution->unfinishedTaskCount = list_length(execution->remoteTaskList); return execution; } /* * DecideTaskListTransactionProperties decides whether to use remote transaction * blocks, whether to use 2PC for the given task list, and whether to error on any * failure. * * Since these decisions have specific dependencies on each other (e.g. 2PC implies * errorOnAnyFailure, but not the other way around) we keep them in the same place. */ static TransactionProperties DecideTaskListTransactionProperties(RowModifyLevel modLevel, List *taskList, bool excludeFromTransaction) { TransactionProperties xactProperties; /* ensure uninitialized padding doesn't escape the function */ memset_struct_0(xactProperties); xactProperties.errorOnAnyFailure = false; xactProperties.useRemoteTransactionBlocks = TRANSACTION_BLOCKS_ALLOWED; xactProperties.requires2PC = false; if (taskList == NIL) { /* nothing to do, return defaults */ return xactProperties; } if (excludeFromTransaction) { xactProperties.useRemoteTransactionBlocks = TRANSACTION_BLOCKS_DISALLOWED; return xactProperties; } if (TaskListCannotBeExecutedInTransaction(taskList)) { /* * We prefer to error on any failures for CREATE INDEX * CONCURRENTLY or VACUUM//VACUUM ANALYZE (e.g., COMMIT_PROTOCOL_BARE). */ xactProperties.errorOnAnyFailure = true; xactProperties.useRemoteTransactionBlocks = TRANSACTION_BLOCKS_DISALLOWED; return xactProperties; } if (TaskListRequiresRollback(taskList)) { /* transaction blocks are required if the task list needs to roll back */ xactProperties.useRemoteTransactionBlocks = TRANSACTION_BLOCKS_REQUIRED; if (TaskListRequires2PC(taskList)) { /* * Although using two phase commit protocol is an independent decision than * failing on any error, we prefer to couple them. Our motivation is that * the failures are rare, and we prefer to avoid marking placements invalid * in case of failures. */ xactProperties.errorOnAnyFailure = true; xactProperties.requires2PC = true; } } else if (InCoordinatedTransaction()) { /* * If we are already in a coordinated transaction then transaction blocks * are required even if they are not strictly required for the current * execution. */ xactProperties.useRemoteTransactionBlocks = TRANSACTION_BLOCKS_REQUIRED; } return xactProperties; } /* * StartDistributedExecution sets up the coordinated transaction and 2PC for * the execution whenever necessary. It also keeps track of parallel relation * accesses to enforce restrictions that arise due to foreign keys to reference * tables. */ void StartDistributedExecution(DistributedExecution *execution) { TransactionProperties *xactProperties = execution->transactionProperties; if (xactProperties->useRemoteTransactionBlocks == TRANSACTION_BLOCKS_REQUIRED) { UseCoordinatedTransaction(); } if (xactProperties->requires2PC) { Use2PCForCoordinatedTransaction(); } /* * Prevent unsafe concurrent modifications of replicated shards by taking * locks. * * When modifying a reference tables in MX mode, we take the lock via RPC * to the first worker in a transaction block, which activates a coordinated * transaction. We need to do this before determining whether the execution * should use transaction blocks (see below). * * We acquire the locks for both the remote and local tasks. */ AcquireExecutorShardLocksForExecution(execution->modLevel, execution->remoteAndLocalTaskList); /* * We should not record parallel access if the target pool size is less than 2. * The reason is that we define parallel access as at least two connections * accessing established to worker node. * * It is not ideal to have this check here, it'd have been better if we simply passed * DistributedExecution directly to the RecordParallelAccess*() function. However, * since we have two other executors that rely on the function, we had to only pass * the tasklist to have a common API. */ if (execution->targetPoolSize > 1) { /* * Record the access for both the local and remote tasks. The main goal * is to make sure that Citus behaves consistently even if the local * shards are moved away. */ RecordParallelRelationAccessForTaskList(execution->remoteAndLocalTaskList); } /* make sure we are not doing remote execution from within a task */ if (execution->remoteTaskList != NIL) { bool isRemote = true; EnsureTaskExecutionAllowed(isRemote); } } /* * DistributedExecutionModifiesDatabase returns true if the execution modifies the data * or the schema. */ static bool DistributedExecutionModifiesDatabase(DistributedExecution *execution) { return TaskListModifiesDatabase(execution->modLevel, execution->remoteAndLocalTaskList); } /* * FinishDistributedExecution cleans up resources associated with a * distributed execution. */ static void FinishDistributedExecution(DistributedExecution *execution) { if (DistributedExecutionModifiesDatabase(execution)) { /* prevent copying shards in same transaction */ XactModificationLevel = XACT_MODIFICATION_DATA; } } /* * AssignTasksToConnectionsOrWorkerPool goes through the list of tasks to determine whether any * task placements need to be assigned to particular connections because of preceding * operations in the transaction. It then adds those connections to the pool and adds * the task placement executions to the assigned task queue of the connection. */ static void AssignTasksToConnectionsOrWorkerPool(DistributedExecution *execution) { RowModifyLevel modLevel = execution->modLevel; List *taskList = execution->remoteTaskList; Task *task = NULL; foreach_declared_ptr(task, taskList) { bool placementExecutionReady = true; int placementExecutionIndex = 0; int placementExecutionCount = list_length(task->taskPlacementList); /* * Execution of a command on a shard, which may have multiple replicas. */ ShardCommandExecution *shardCommandExecution = (ShardCommandExecution *) palloc0(sizeof(ShardCommandExecution)); shardCommandExecution->task = task; shardCommandExecution->executionOrder = ExecutionOrderForTask(modLevel, task); shardCommandExecution->executionState = TASK_EXECUTION_NOT_FINISHED; shardCommandExecution->localExecutionSupported = execution->localExecutionSupported; shardCommandExecution->placementExecutions = (TaskPlacementExecution **) palloc0(placementExecutionCount * sizeof(TaskPlacementExecution *)); shardCommandExecution->placementExecutionCount = placementExecutionCount; SetAttributeInputMetadata(execution, shardCommandExecution); ShardPlacement *taskPlacement = NULL; foreach_declared_ptr(taskPlacement, task->taskPlacementList) { int connectionFlags = 0; char *nodeName = NULL; int nodePort = 0; LookupTaskPlacementHostAndPort(taskPlacement, &nodeName, &nodePort); WorkerPool *workerPool = FindOrCreateWorkerPool(execution, nodeName, nodePort); /* * Execution of a command on a shard placement, which may not always * happen if the query is read-only and the shard has multiple placements. */ TaskPlacementExecution *placementExecution = (TaskPlacementExecution *) palloc0(sizeof(TaskPlacementExecution)); placementExecution->shardCommandExecution = shardCommandExecution; placementExecution->shardPlacement = taskPlacement; placementExecution->workerPool = workerPool; placementExecution->placementExecutionIndex = placementExecutionIndex; placementExecution->queryIndex = 0; INSTR_TIME_SET_ZERO(placementExecution->startTime); INSTR_TIME_SET_ZERO(placementExecution->endTime); if (placementExecutionReady) { placementExecution->executionState = PLACEMENT_EXECUTION_READY; } else { placementExecution->executionState = PLACEMENT_EXECUTION_NOT_READY; } shardCommandExecution->placementExecutions[placementExecutionIndex] = placementExecution; placementExecutionIndex++; List *placementAccessList = PlacementAccessListForTask(task, taskPlacement); MultiConnection *connection = NULL; if (execution->transactionProperties->useRemoteTransactionBlocks != TRANSACTION_BLOCKS_DISALLOWED) { /* * Determine whether the task has to be assigned to a particular connection * due to a preceding access to the placement in the same transaction. */ connection = GetConnectionIfPlacementAccessedInXact( connectionFlags, placementAccessList, NULL); } if (connection != NULL) { /* * Note: We may get the same connection for multiple task placements. * FindOrCreateWorkerSession ensures that we only have one session per * connection. */ WorkerSession *session = FindOrCreateWorkerSession(workerPool, connection); ereport(DEBUG4, (errmsg("Session %ld (%s:%d) has an assigned task", session->sessionId, connection->hostname, connection->port))); placementExecution->assignedSession = session; /* if executed, this task placement must use this session */ if (placementExecutionReady) { dlist_push_tail(&session->readyTaskQueue, &placementExecution->sessionReadyQueueNode); } else { dlist_push_tail(&session->pendingTaskQueue, &placementExecution->sessionPendingQueueNode); } /* always poll the connection in the first round */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); /* If the connections are already avaliable, make sure to activate * 2PC when necessary. */ Activate2PCIfModifyingTransactionExpandsToNewNode(session); } else { placementExecution->assignedSession = NULL; if (placementExecutionReady) { /* task is ready to execute on any session */ dlist_push_tail(&workerPool->readyTaskQueue, &placementExecution->workerReadyQueueNode); workerPool->readyTaskCount++; } else { /* task can be executed on any session, but is not yet ready */ dlist_push_tail(&workerPool->pendingTaskQueue, &placementExecution->workerPendingQueueNode); } } if (shardCommandExecution->executionOrder != EXECUTION_ORDER_PARALLEL) { /* * Except for commands that can be executed across all placements * in parallel, only the first placement execution is immediately * ready. Set placementExecutionReady to false for the remaining * placements. */ placementExecutionReady = false; } } } /* * We sort the workerList because adaptive connection management * (e.g., OPTIONAL_CONNECTION) requires any concurrent executions * to wait for the connections in the same order to prevent any * starvation. If we don't sort, we might end up with: * Execution 1: Get connection for worker 1, wait for worker 2 * Execution 2: Get connection for worker 2, wait for worker 1 * * and, none could proceed. Instead, we enforce every execution establish * the required connections to workers in the same order. */ execution->workerList = SortList(execution->workerList, WorkerPoolCompare); /* * The executor claims connections exclusively to make sure that calls to * StartNodeUserDatabaseConnection do not return the same connections. * * We need to do this after assigning tasks to connections because the same * connection may be be returned multiple times by GetPlacementListConnectionIfCached. */ WorkerSession *session = NULL; foreach_declared_ptr(session, execution->sessionList) { MultiConnection *connection = session->connection; ClaimConnectionExclusively(connection); } } /* * WorkerPoolCompare is based on WorkerNodeCompare function. The function * compares two worker nodes by their host name and port number. */ static int WorkerPoolCompare(const void *lhsKey, const void *rhsKey) { const WorkerPool *workerLhs = *(const WorkerPool **) lhsKey; const WorkerPool *workerRhs = *(const WorkerPool **) rhsKey; return NodeNamePortCompare(workerLhs->nodeName, workerRhs->nodeName, workerLhs->nodePort, workerRhs->nodePort); } /* * SetAttributeInputMetadata sets attributeInputMetadata in * shardCommandExecution for all the queries that are part of its task. * This contains the deserialization functions for the tuples that will be * received. It also sets binaryResults when applicable. */ static void SetAttributeInputMetadata(DistributedExecution *execution, ShardCommandExecution *shardCommandExecution) { TupleDestination *tupleDest = shardCommandExecution->task->tupleDest ? shardCommandExecution->task->tupleDest : execution->defaultTupleDest; uint32 queryCount = shardCommandExecution->task->queryCount; shardCommandExecution->attributeInputMetadata = palloc0(queryCount * sizeof(AttInMetadata *)); for (uint32 queryIndex = 0; queryIndex < queryCount; queryIndex++) { AttInMetadata *attInMetadata = NULL; TupleDesc tupleDescriptor = tupleDest->tupleDescForQuery(tupleDest, queryIndex); if (tupleDescriptor == NULL) { attInMetadata = NULL; } else if (EnableBinaryProtocol && CanUseBinaryCopyFormat(tupleDescriptor)) { attInMetadata = TupleDescGetAttBinaryInMetadata(tupleDescriptor); shardCommandExecution->binaryResults = true; } else { attInMetadata = TupleDescGetAttInMetadata(tupleDescriptor); } shardCommandExecution->attributeInputMetadata[queryIndex] = attInMetadata; } } /* * ExecutionOrderForTask gives the appropriate execution order for a task. */ static PlacementExecutionOrder ExecutionOrderForTask(RowModifyLevel modLevel, Task *task) { switch (task->taskType) { case READ_TASK: { return EXECUTION_ORDER_ANY; } case MODIFY_TASK: { /* * For non-commutative modifications we take aggressive locks, so * there is no risk of deadlock and we can run them in parallel. * When the modification is commutative, we take no additional * locks, so we take a conservative approach and execute sequentially * to avoid deadlocks. */ if (modLevel < ROW_MODIFY_NONCOMMUTATIVE) { return EXECUTION_ORDER_SEQUENTIAL; } else { return EXECUTION_ORDER_PARALLEL; } } case DDL_TASK: case VACUUM_ANALYZE_TASK: case MAP_TASK: case MERGE_TASK: case MAP_OUTPUT_FETCH_TASK: case MERGE_FETCH_TASK: { return EXECUTION_ORDER_PARALLEL; } default: { ereport(ERROR, (errmsg("unsupported task type %d in adaptive executor", task->taskType))); } } } /* * FindOrCreateWorkerPool gets the pool of connections for a particular worker. */ static WorkerPool * FindOrCreateWorkerPool(DistributedExecution *execution, char *nodeName, int nodePort) { WorkerPool *workerPool = NULL; foreach_declared_ptr(workerPool, execution->workerList) { if (strncmp(nodeName, workerPool->nodeName, WORKER_LENGTH) == 0 && nodePort == workerPool->nodePort) { return workerPool; } } workerPool = (WorkerPool *) palloc0(sizeof(WorkerPool)); workerPool->nodeName = pstrdup(nodeName); workerPool->nodePort = nodePort; WorkerNode *workerNode = FindWorkerNode(nodeName, nodePort); if (workerNode) { workerPool->poolToLocalNode = workerNode->groupId == GetLocalGroupId(); } /* "open" connections aggressively when there are cached connections */ int nodeConnectionCount = MaxCachedConnectionsPerWorker; workerPool->maxNewConnectionsPerCycle = Max(1, nodeConnectionCount); dlist_init(&workerPool->pendingTaskQueue); dlist_init(&workerPool->readyTaskQueue); workerPool->distributedExecution = execution; execution->workerList = lappend(execution->workerList, workerPool); return workerPool; } /* * FindOrCreateWorkerSession returns a session with the given connection, * either existing or new. New sessions are added to the worker pool and * the distributed execution. */ static WorkerSession * FindOrCreateWorkerSession(WorkerPool *workerPool, MultiConnection *connection) { DistributedExecution *execution = workerPool->distributedExecution; static uint64 sessionId = 1; WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { if (session->connection == connection) { return session; } } session = (WorkerSession *) palloc0(sizeof(WorkerSession)); session->sessionId = sessionId++; session->connection = connection; session->workerPool = workerPool; session->commandsSent = 0; session->waitEventSetIndex = WAIT_EVENT_SET_INDEX_NOT_INITIALIZED; /* always detect closed sockets */ UpdateConnectionWaitFlags(session, WL_SOCKET_CLOSED); dlist_init(&session->pendingTaskQueue); dlist_init(&session->readyTaskQueue); if (connection->connectionState == MULTI_CONNECTION_CONNECTED) { /* keep track of how many connections are ready */ workerPool->activeConnectionCount++; workerPool->idleConnectionCount++; session->sessionHasActiveConnection = true; } workerPool->unusedConnectionCount++; /* * Record the first connection establishment time to the pool. We need this * to enforce NodeConnectionTimeout. */ if (list_length(workerPool->sessionList) == 0) { INSTR_TIME_SET_CURRENT(workerPool->poolStartTime); workerPool->checkForPoolTimeout = true; } workerPool->sessionList = lappend(workerPool->sessionList, session); execution->sessionList = lappend(execution->sessionList, session); return session; } /* * RemoteSocketClosedForNewSession is a helper function for detecting whether * the remote socket corresponding to the input session is closed. This is * mostly common there is a cached connection and remote server restarted * (due to failover or restart etc.). * * The function is not a generic function that can be called at the start of * the execution. The function is not generic because it does not check all * the events, even ignores cancellation events. Future callers of this * function should consider its limitations. */ static void RemoteSocketClosedForAnySession(DistributedExecution *execution) { if (!WaitEventSetCanReportClosed()) { /* we cannot detect for this OS */ return; } long timeout = 0;/* don't wait */ int eventCount = WaitEventSetWait(execution->waitEventSet, timeout, execution->events, execution->eventSetSize, WAIT_EVENT_CLIENT_READ); ProcessWaitEventsForSocketClosed(execution->events, eventCount); } /* * SequentialRunDistributedExecution gets a distributed execution and * executes each individual task in the execution sequentially, one * task at a time. See related function ShouldRunTasksSequentially() * for more detail on the definition of SequentialRun. */ static void SequentialRunDistributedExecution(DistributedExecution *execution) { List *taskList = execution->remoteTaskList; int connectionMode = MultiShardConnectionType; /* * There are some implicit assumptions about this setting for the sequential * executions, so make sure to set it. */ MultiShardConnectionType = SEQUENTIAL_CONNECTION; Task *taskToExecute = NULL; foreach_declared_ptr(taskToExecute, taskList) { execution->remoteAndLocalTaskList = list_make1(taskToExecute); execution->remoteTaskList = list_make1(taskToExecute); execution->totalTaskCount = 1; execution->unfinishedTaskCount = 1; CHECK_FOR_INTERRUPTS(); if (IsHoldOffCancellationReceived()) { break; } /* simply call the regular execution function */ RunDistributedExecution(execution); } /* set back the original execution mode */ MultiShardConnectionType = connectionMode; } /* * RunDistributedExecution runs a distributed execution to completion. It first opens * connections for distributed execution and assigns each task with shard placements * that have previously been modified in the current transaction to the connection * that modified them. Then, it creates a wait event set to listen for events on * any of the connections and runs the connection state machine when a connection * has an event. */ void RunDistributedExecution(DistributedExecution *execution) { AssignTasksToConnectionsOrWorkerPool(execution); PG_TRY(); { /* Preemptively step state machines in case of immediate errors */ WorkerSession *session = NULL; foreach_declared_ptr(session, execution->sessionList) { ConnectionStateMachine(session); } bool cancellationReceived = false; /* always (re)build the wait event set the first time */ execution->rebuildWaitEventSet = true; /* * Iterate until all the tasks are finished. Once all the tasks * are finished, ensure that all the connection initializations * are also finished. Otherwise, those connections are terminated * abruptly before they are established (or failed). Instead, we let * the ConnectionStateMachine() to properly handle them. * * Note that we could have the connections that are not established * as a side effect of slow-start algorithm. At the time the algorithm * decides to establish new connections, the execution might have tasks * to finish. But, the execution might finish before the new connections * are established. * * Note that the rules explained above could be overriden by any * cancellation to the query. In that case, we terminate the execution * irrespective of the current status of the tasks or the connections. */ while (!cancellationReceived && (execution->unfinishedTaskCount > 0 || HasIncompleteConnectionEstablishment(execution))) { WorkerPool *workerPool = NULL; foreach_declared_ptr(workerPool, execution->workerList) { ManageWorkerPool(workerPool); } bool skipWaitEvents = false; if (execution->remoteTaskList == NIL) { /* * All the tasks are failed over to the local execution, no need * to wait for any connection activity. */ continue; } else if (execution->rebuildWaitEventSet) { RebuildWaitEventSet(execution); skipWaitEvents = ProcessSessionsWithFailedWaitEventSetOperations(execution); } else if (execution->waitFlagsChanged) { RebuildWaitEventSetFlags(execution->waitEventSet, execution->sessionList); execution->waitFlagsChanged = false; skipWaitEvents = ProcessSessionsWithFailedWaitEventSetOperations(execution); } if (skipWaitEvents) { /* * Some operation on the wait event set is failed, retry * as we already removed the problematic connections. */ execution->rebuildWaitEventSet = true; continue; } /* wait for I/O events */ long timeout = NextEventTimeout(execution); int eventCount = WaitEventSetWait(execution->waitEventSet, timeout, execution->events, execution->eventSetSize, WAIT_EVENT_CLIENT_READ); ProcessWaitEvents(execution, execution->events, eventCount, &cancellationReceived); } FreeExecutionWaitEvents(execution); CleanUpSessions(execution); } PG_CATCH(); { /* * We can still recover from error using ROLLBACK TO SAVEPOINT, * unclaim all connections to allow that. */ UnclaimAllSessionConnections(execution->sessionList); FreeExecutionWaitEvents(execution); PG_RE_THROW(); } PG_END_TRY(); } /* * ProcessSessionsWithFailedWaitEventSetOperations goes over the session list * and processes sessions with failed wait event set operations. * * Failed sessions are not going to generate any further events, so it is our * only chance to process the failure by calling into `ConnectionStateMachine`. * * The function returns true if any session failed. */ static bool ProcessSessionsWithFailedWaitEventSetOperations(DistributedExecution *execution) { bool foundFailedSession = false; WorkerSession *session = NULL; foreach_declared_ptr(session, execution->sessionList) { if (session->waitEventSetIndex == WAIT_EVENT_SET_INDEX_FAILED) { /* * We can only lost only already connected connections, * others are regular failures. */ MultiConnection *connection = session->connection; if (connection->connectionState == MULTI_CONNECTION_CONNECTED) { connection->connectionState = MULTI_CONNECTION_LOST; } else { connection->connectionState = MULTI_CONNECTION_FAILED; IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); } ConnectionStateMachine(session); session->waitEventSetIndex = WAIT_EVENT_SET_INDEX_NOT_INITIALIZED; foundFailedSession = true; } } return foundFailedSession; } /* * HasIncompleteConnectionEstablishment returns true if any of the connections * that has been initiated by the executor is in initialization stage. */ static bool HasIncompleteConnectionEstablishment(DistributedExecution *execution) { if (!PreventIncompleteConnectionEstablishment) { return false; } WorkerSession *session = NULL; foreach_declared_ptr(session, execution->sessionList) { MultiConnection *connection = session->connection; if (connection->connectionState == MULTI_CONNECTION_INITIAL || connection->connectionState == MULTI_CONNECTION_CONNECTING) { return true; } } return false; } /* * RebuildWaitEventSet updates the waitEventSet for the distributed execution. * This happens when the connection set for the distributed execution is changed, * which means that we need to update which connections we wait on for events. */ static void RebuildWaitEventSet(DistributedExecution *execution) { RebuildWaitEventSetForSessions(execution); AddLatchWaitEventToExecution(execution); } /* * AddLatchWaitEventToExecution is a helper function that adds the latch * wait event to the execution->waitEventSet. Note that this function assumes * that execution->waitEventSet has already allocated enough slot for latch * event. */ static void AddLatchWaitEventToExecution(DistributedExecution *execution) { CitusAddWaitEventSetToSet(execution->waitEventSet, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); } /* * RebuildWaitEventSetForSessions re-creates the waitEventSet for the * sessions involved in the distributed execution. * * Most of the time you need RebuildWaitEventSet() which also includes * adds the Latch wait event to the set. */ static void RebuildWaitEventSetForSessions(DistributedExecution *execution) { FreeExecutionWaitEvents(execution); execution->waitEventSet = BuildWaitEventSet(execution->sessionList); execution->eventSetSize = GetEventSetSize(execution->sessionList); execution->events = palloc0(execution->eventSetSize * sizeof(WaitEvent)); CitusAddWaitEventSetToSet(execution->waitEventSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); execution->rebuildWaitEventSet = false; execution->waitFlagsChanged = false; } /* * ProcessWaitEvents processes the received events from connections. */ static void ProcessWaitEvents(DistributedExecution *execution, WaitEvent *events, int eventCount, bool *cancellationReceived) { int eventIndex = 0; /* process I/O events */ for (; eventIndex < eventCount; eventIndex++) { WaitEvent *event = &events[eventIndex]; if (event->events & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } if (event->events & WL_LATCH_SET) { ResetLatch(MyLatch); if (execution->raiseInterrupts) { CHECK_FOR_INTERRUPTS(); } if (IsHoldOffCancellationReceived()) { /* * Break out of event loop immediately in case of cancellation. * We cannot use "return" here inside a PG_TRY() block since * then the exception stack won't be reset. */ *cancellationReceived = true; } continue; } WorkerSession *session = (WorkerSession *) event->user_data; session->latestUnconsumedWaitEvents = event->events; ConnectionStateMachine(session); } } /* * ProcessWaitEventsForSocketClosed mainly checks for WL_SOCKET_CLOSED event. * If WL_SOCKET_CLOSED is found, the function sets the underlying connection's * state as MULTI_CONNECTION_LOST. */ static void ProcessWaitEventsForSocketClosed(WaitEvent *events, int eventCount) { int eventIndex = 0; /* process I/O events */ for (; eventIndex < eventCount; eventIndex++) { WaitEvent *event = &events[eventIndex]; if (event->events & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } WorkerSession *session = (WorkerSession *) event->user_data; session->latestUnconsumedWaitEvents = event->events; if (session->latestUnconsumedWaitEvents & WL_SOCKET_CLOSED) { /* let the ConnectionStateMachine handle the rest */ session->connection->connectionState = MULTI_CONNECTION_LOST; } } } /* * ManageWorkerPool ensures the worker pool has the appropriate number of connections * based on the number of pending tasks. */ static void ManageWorkerPool(WorkerPool *workerPool) { DistributedExecution *execution = workerPool->distributedExecution; /* we do not expand the pool further if there was any failure */ if (HasAnyConnectionFailure(workerPool)) { return; } /* we wait until a slow start interval has passed before expanding the pool */ if (ShouldWaitForSlowStart(workerPool)) { return; } int newConnectionCount = CalculateNewConnectionCount(workerPool); if (newConnectionCount <= 0) { return; } /* increase the open rate every cycle (like TCP slow start) */ workerPool->maxNewConnectionsPerCycle += 1; OpenNewConnections(workerPool, newConnectionCount, execution->transactionProperties); /* * Cannot establish new connections to the local host, most probably because the * local node cannot accept new connections (e.g., hit max_connections). Switch * the tasks to the local execution. * * We prefer initiatedConnectionCount over the new connection establishments happen * in this iteration via OpenNewConnections(). The reason is that it is expected for * OpenNewConnections() to not open any new connections as long as the connections * are optional (e.g., the second or later connections in the pool). But, for * initiatedConnectionCount to be zero, the connection to the local pool should have * been failed. */ int initiatedConnectionCount = list_length(workerPool->sessionList); if (initiatedConnectionCount == 0) { /* * Only the pools to the local node are allowed to have optional * connections for the first connection. Hence, initiatedConnectionCount * could only be zero for poolToLocalNode. For other pools, the connection * manager would wait until it gets at least one connection. */ Assert(workerPool->poolToLocalNode); WorkerPoolFailed(workerPool); if (execution->failed) { const char *errHint = execution->localExecutionSupported ? "This command supports local execution. Consider enabling " "local execution using SET citus.enable_local_execution " "TO true;" : "Consider using a higher value for max_connections"; ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("the total number of connections on the " "server is more than max_connections(%d)", MaxConnections), errhint("%s", errHint))); } return; } INSTR_TIME_SET_CURRENT(workerPool->lastConnectionOpenTime); } /* * HasAnyConnectionFailure returns true if worker pool has failed, * or connection timed out or we have a failure in connections. */ static bool HasAnyConnectionFailure(WorkerPool *workerPool) { if (workerPool->failureState == WORKER_POOL_FAILED || workerPool->failureState == WORKER_POOL_FAILED_OVER_TO_LOCAL) { /* connection pool failed */ return true; } /* we might fail the execution or warn the user about connection timeouts */ if (workerPool->checkForPoolTimeout) { CheckConnectionTimeout(workerPool); } int failedConnectionCount = workerPool->failedConnectionCount; if (failedConnectionCount >= 1) { /* do not attempt to open more connections after one failed */ return true; } return false; } /* * ShouldWaitForSlowStart returns true if we should wait before * opening a new connection because of slow start algorithm. */ static bool ShouldWaitForSlowStart(WorkerPool *workerPool) { /* if we can use a connection per placement, we don't need to wait for slowstart */ if (UseConnectionPerPlacement()) { return false; } /* if slow start is disabled, we can open new connections */ if (ExecutorSlowStartInterval == SLOW_START_DISABLED) { return false; } double milliSecondsPassedSince = MillisecondsPassedSince( workerPool->lastConnectionOpenTime); if (milliSecondsPassedSince < ExecutorSlowStartInterval) { return true; } /* * Refrain from establishing new connections unless we have already * finalized all the earlier connection attempts. This prevents unnecessary * load on the remote nodes and emulates the TCP slow-start algorithm. */ int initiatedConnectionCount = list_length(workerPool->sessionList); int finalizedConnectionCount = workerPool->activeConnectionCount + workerPool->failedConnectionCount; if (finalizedConnectionCount < initiatedConnectionCount) { return true; } return false; } /* * CalculateNewConnectionCount returns the amount of connections * that we can currently open. */ static int CalculateNewConnectionCount(WorkerPool *workerPool) { DistributedExecution *execution = workerPool->distributedExecution; int targetPoolSize = execution->targetPoolSize; int initiatedConnectionCount = list_length(workerPool->sessionList); int activeConnectionCount PG_USED_FOR_ASSERTS_ONLY = workerPool->activeConnectionCount; int idleConnectionCount PG_USED_FOR_ASSERTS_ONLY = workerPool->idleConnectionCount; int readyTaskCount = workerPool->readyTaskCount; int newConnectionCount = 0; /* we should always have more (or equal) active connections than idle connections */ Assert(activeConnectionCount >= idleConnectionCount); /* we should always have more (or equal) initiated connections than active connections */ Assert(initiatedConnectionCount >= activeConnectionCount); /* we should never have less than 0 connections ever */ Assert(activeConnectionCount >= 0 && idleConnectionCount >= 0); if (UseConnectionPerPlacement()) { int unusedConnectionCount = workerPool->unusedConnectionCount; /* * If force_max_query_parallelization is enabled then we ignore pool size * and idle connections. Instead, we open new connections as long as there * are more tasks than unused connections. */ newConnectionCount = Max(readyTaskCount - unusedConnectionCount, 0); } else { /* cannot open more than targetPoolSize connections */ int maxNewConnectionCount = targetPoolSize - initiatedConnectionCount; /* total number of connections that are (almost) available for tasks */ int usableConnectionCount = UsableConnectionCount(workerPool); /* * Number of additional connections we would need to run all ready tasks in * parallel. */ int newConnectionsForReadyTasks = Max(0, readyTaskCount - usableConnectionCount); /* If Slow start is enabled we need to update the maxNewConnection to the current cycle's maximum.*/ if (ExecutorSlowStartInterval != SLOW_START_DISABLED) { maxNewConnectionCount = Min(workerPool->maxNewConnectionsPerCycle, maxNewConnectionCount); } /* * Open enough connections to handle all tasks that are ready, but no more * than the target pool size. */ newConnectionCount = Min(newConnectionsForReadyTasks, maxNewConnectionCount); if (EnableCostBasedConnectionEstablishment && newConnectionCount > 0 && initiatedConnectionCount <= MaxCachedConnectionsPerWorker && UsingExistingSessionsCheaperThanEstablishingNewConnections( readyTaskCount, workerPool)) { /* * Before giving the decision, we do one more check. If the cost of * executing the remaining tasks over the existing sessions in the * pool is cheaper than establishing new connections and executing * the tasks over the new connections, we prefer the former. * * For cached connections we should ignore any optimizations as * cached connections are almost free to get. In other words, * as long as there are cached connections that the pool has * not used yet, aggressively use these already established * connections. * * Note that until MaxCachedConnectionsPerWorker has already been * established within the session, we still need to establish * the connections right now. * * Also remember that we are not trying to find the optimal number * of connections for the remaining tasks here. Our goal is to prevent * connection establishments that are absolutely unnecessary. In the * future, we may improve the calculations below to find the optimal * number of new connections required. */ return 0; } } return newConnectionCount; } /* * UsingExistingSessionsCheaperThanEstablishingNewConnections returns true if * using the already established connections takes less time compared to opening * new connections based on the current execution's stats. * * The function returns false if the current execution has not established any connections * or finished any tasks (e.g., no stats to act on). */ static bool UsingExistingSessionsCheaperThanEstablishingNewConnections(int readyTaskCount, WorkerPool *workerPool) { int activeConnectionCount = workerPool->activeConnectionCount; if (workerPool->totalExecutedTasks < 1 || activeConnectionCount < 1) { /* * The pool has not finished any connection establishment or * task yet. So, we refrain from optimizing the execution. */ return false; } double avgTaskExecutionTime = AvgTaskExecutionTimeApproximation(workerPool); double avgConnectionEstablishmentTime = AvgConnectionEstablishmentTime(workerPool); /* we assume that we are halfway through the execution */ double remainingTimeForActiveTaskExecutionsToFinish = avgTaskExecutionTime / 2; /* * We use "newConnectionCount" as if it is the task count as * we are only interested in this iteration of CalculateNewConnectionCount(). */ double totalTimeToExecuteNewTasks = avgTaskExecutionTime * readyTaskCount; double estimatedExecutionTimeForNewTasks = floor(totalTimeToExecuteNewTasks / activeConnectionCount); /* * First finish the already running tasks, and then use the connections * to execute the new tasks. */ double costOfExecutingTheTasksOverExistingConnections = remainingTimeForActiveTaskExecutionsToFinish + estimatedExecutionTimeForNewTasks; /* * For every task, the executor is supposed to establish one * connection and then execute the task over the connection. */ double costOfExecutingTheTasksOverNewConnection = (avgTaskExecutionTime + avgConnectionEstablishmentTime); return (costOfExecutingTheTasksOverExistingConnections <= costOfExecutingTheTasksOverNewConnection); } /* * AvgTaskExecutionTimeApproximation returns the approximation of the average task * execution times on the workerPool. */ static double AvgTaskExecutionTimeApproximation(WorkerPool *workerPool) { uint64 totalTaskExecutionTime = workerPool->totalTaskExecutionTime; int taskCount = workerPool->totalExecutedTasks; instr_time now; INSTR_TIME_SET_CURRENT(now); WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { /* * Involve the tasks that are currently running. We do this to * make sure that the execution responds with new connections * quickly if the actively running tasks */ TaskPlacementExecution *placementExecution = session->currentTask; if (placementExecution != NULL && placementExecution->executionState == PLACEMENT_EXECUTION_RUNNING) { uint64 durationInMicroSecs = MicrosecondsBetweenTimestamps(placementExecution->startTime, now); /* * Our approximation is that we assume that the task execution is * just in the halfway through. */ totalTaskExecutionTime += (2 * durationInMicroSecs); taskCount += 1; } } return taskCount == 0 ? 0 : ((double) totalTaskExecutionTime / taskCount); } /* * AvgConnectionEstablishmentTime calculates the average connection establishment times * for the input workerPool. */ static double AvgConnectionEstablishmentTime(WorkerPool *workerPool) { double totalTimeMicrosec = 0; int sessionCount = 0; WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; /* * There could be MaxCachedConnectionsPerWorker connections that are * already connected. Those connections might skew the average * connection establishment times for the current execution. The reason * is that they are established earlier and the connection establishment * times might be different at the moment those connections are established. */ if (connection->connectionState == MULTI_CONNECTION_CONNECTED) { long connectionEstablishmentTime = MicrosecondsBetweenTimestamps(connection->connectionEstablishmentStart, connection->connectionEstablishmentEnd); totalTimeMicrosec += connectionEstablishmentTime; ++sessionCount; } } return (sessionCount == 0) ? 0 : (totalTimeMicrosec / sessionCount); } /* * OpenNewConnections opens the given amount of connections for the given workerPool. */ static void OpenNewConnections(WorkerPool *workerPool, int newConnectionCount, TransactionProperties *transactionProperties) { ereport(DEBUG4, (errmsg("opening %d new connections to %s:%d", newConnectionCount, workerPool->nodeName, workerPool->nodePort))); List *newSessionsList = NIL; for (int connectionIndex = 0; connectionIndex < newConnectionCount; connectionIndex++) { /* experimental: just to see the perf benefits of caching connections */ int connectionFlags = 0; if (transactionProperties->useRemoteTransactionBlocks == TRANSACTION_BLOCKS_DISALLOWED) { connectionFlags |= OUTSIDE_TRANSACTION; } /* * Enforce the requirements for adaptive connection management (a.k.a., * throttle connections if citus.max_shared_pool_size reached) */ int adaptiveConnectionManagementFlag = AdaptiveConnectionManagementFlag(workerPool->poolToLocalNode, list_length(workerPool->sessionList)); connectionFlags |= adaptiveConnectionManagementFlag; /* open a new connection to the worker */ MultiConnection *connection = StartNodeUserDatabaseConnection( connectionFlags, workerPool->nodeName, workerPool->nodePort, NULL, NULL); if (!connection) { /* connection can only be NULL for optional connections */ Assert((connectionFlags & OPTIONAL_CONNECTION)); continue; } /* * Assign the initial state in the connection state machine. The connection * may already be open, but ConnectionStateMachine will immediately detect * this. */ connection->connectionState = MULTI_CONNECTION_CONNECTING; /* * Ensure that subsequent calls to StartNodeUserDatabaseConnection get a * different connection. */ connection->claimedExclusively = true; if (list_length(workerPool->sessionList) == 0) { /* * The worker pool has just started to establish connections. We need to * defer this initialization after StartNodeUserDatabaseConnection() * because for non-optional connections, we have some logic to wait * until a connection is allowed to be established. */ INSTR_TIME_SET_ZERO(workerPool->poolStartTime); } /* create a session for the connection */ WorkerSession *session = FindOrCreateWorkerSession(workerPool, connection); newSessionsList = lappend(newSessionsList, session); } if (list_length(newSessionsList) == 0) { /* nothing to do as no new connections happened */ return; } DistributedExecution *execution = workerPool->distributedExecution; /* we added new connections, rebuild the waitEventSet */ RebuildWaitEventSetForSessions(execution); /* * If there are any closed sockets, mark connection lost such that * we can re-connect. */ RemoteSocketClosedForAnySession(execution); /* * For RemoteSocketClosedForAnySession() purposes, we explicitly skip * the latch because we want to handle all cancellations to be caught * on the main execution loop, not here. We mostly skip cancellations * on RemoteSocketClosedForAnySession() for simplicity. Handling * cancellations on the main execution loop much easier to break out * of the execution. */ AddLatchWaitEventToExecution(execution); WorkerSession *session = NULL; foreach_declared_ptr(session, newSessionsList) { /* immediately run the state machine to handle potential failure */ ConnectionStateMachine(session); } } /* * CheckConnectionTimeout makes sure that the execution enforces the connection * establishment timeout defined by the user (NodeConnectionTimeout). * * The rule is that if a worker pool has already initiated connection establishment * and has not succeeded to finish establishments that are necessary to execute tasks, * take an action. For the types of actions, see the comments in the function. * * Enforcing the timeout per pool (over per session) helps the execution to continue * even if we can establish a single connection as we expect to have target pool size * number of connections. In the end, the executor is capable of using one connection * to execute multiple tasks. */ static void CheckConnectionTimeout(WorkerPool *workerPool) { DistributedExecution *execution = workerPool->distributedExecution; instr_time poolStartTime = workerPool->poolStartTime; instr_time now; INSTR_TIME_SET_CURRENT(now); int initiatedConnectionCount = list_length(workerPool->sessionList); int activeConnectionCount = workerPool->activeConnectionCount; int requiredActiveConnectionCount = 1; if (initiatedConnectionCount == 0) { /* no connection has been planned for the pool yet */ Assert(INSTR_TIME_IS_ZERO(poolStartTime)); return; } /* * This is a special case where we assign tasks to sessions even before * the connections are established. So, make sure to apply similar * restrictions. In this case, make sure that we get all the connections * established. */ if (UseConnectionPerPlacement()) { requiredActiveConnectionCount = initiatedConnectionCount; } if (MillisecondsBetweenTimestamps(poolStartTime, now) >= NodeConnectionTimeout) { if (activeConnectionCount < requiredActiveConnectionCount) { int logLevel = WARNING; /* * First fail the pool and create an opportunity to execute tasks * over other pools when tasks have more than one placement to execute. */ WorkerPoolFailed(workerPool); if (workerPool->failureState == WORKER_POOL_FAILED_OVER_TO_LOCAL) { /* * * When the pool is failed over to local execution, warning * the user just creates chatter as the executor is capable of * finishing the execution. */ logLevel = DEBUG1; } else if (execution->transactionProperties->errorOnAnyFailure || execution->failed) { /* * The enforcement is not always erroring out. For example, if a SELECT task * has two different placements, we'd warn the user, fail the pool and continue * with the next placement. */ logLevel = ERROR; } /* * We hit the connection timeout. In that case, we should not let the * connection establishment to continue because the execution logic * pretends that failed sessions are not going to be used anymore. * * That's why we mark the connection as timed out to trigger the state * changes in the executor, if we don't throw an error below. */ MarkEstablishingSessionsTimedOut(workerPool); ereport(logLevel, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not establish any connections to the node " "%s:%d after %u ms", workerPool->nodeName, workerPool->nodePort, NodeConnectionTimeout))); } else { /* stop interrupting WaitEventSetWait for timeouts */ workerPool->checkForPoolTimeout = false; } } } /* * MarkEstablishingSessionsTimedOut goes over the sessions in the given * workerPool and marks them timed out. ConnectionStateMachine() * later cleans up the sessions. */ static void MarkEstablishingSessionsTimedOut(WorkerPool *workerPool) { WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; if (connection->connectionState == MULTI_CONNECTION_CONNECTING || connection->connectionState == MULTI_CONNECTION_INITIAL) { connection->connectionState = MULTI_CONNECTION_TIMED_OUT; IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); } } } /* * UsableConnectionCount returns the number of connections in the worker pool * that are (soon to be) usable for sending commands, this includes both idle * connections and connections that are still establishing. */ static int UsableConnectionCount(WorkerPool *workerPool) { int initiatedConnectionCount = list_length(workerPool->sessionList); int activeConnectionCount = workerPool->activeConnectionCount; int failedConnectionCount = workerPool->failedConnectionCount; int idleConnectionCount = workerPool->idleConnectionCount; /* connections that are still establishing will soon be available for tasks */ int establishingConnectionCount = initiatedConnectionCount - activeConnectionCount - failedConnectionCount; int usableConnectionCount = idleConnectionCount + establishingConnectionCount; return usableConnectionCount; } /* * NextEventTimeout finds the earliest time at which we need to interrupt * WaitEventSetWait because of a timeout and returns the number of milliseconds * until that event with a minimum of 1ms and a maximum of 1000ms. * * This code may be sensitive to clock jumps, but only has the effect of waking * up WaitEventSetWait slightly earlier to later. */ static long NextEventTimeout(DistributedExecution *execution) { instr_time now; INSTR_TIME_SET_CURRENT(now); long eventTimeout = 1000; /* milliseconds */ WorkerPool *workerPool = NULL; foreach_declared_ptr(workerPool, execution->workerList) { if (workerPool->failureState == WORKER_POOL_FAILED) { /* worker pool may have already timed out */ continue; } if (!INSTR_TIME_IS_ZERO(workerPool->poolStartTime) && workerPool->checkForPoolTimeout) { long timeSincePoolStartMs = MillisecondsBetweenTimestamps(workerPool->poolStartTime, now); /* * This could go into the negative if the connection timeout just passed. * In that case we want to wake up as soon as possible. Once the timeout * has been processed, checkForPoolTimeout will be false so we will skip * this check. */ long timeUntilConnectionTimeoutMs = NodeConnectionTimeout - timeSincePoolStartMs; if (timeUntilConnectionTimeoutMs < eventTimeout) { eventTimeout = timeUntilConnectionTimeoutMs; } } int initiatedConnectionCount = list_length(workerPool->sessionList); /* * If there are connections to open we wait at most up to the end of the * current slow start interval. */ if (workerPool->readyTaskCount > UsableConnectionCount(workerPool) && initiatedConnectionCount < execution->targetPoolSize) { long timeSinceLastConnectMs = MillisecondsBetweenTimestamps(workerPool->lastConnectionOpenTime, now); long timeUntilSlowStartInterval = ExecutorSlowStartInterval - timeSinceLastConnectMs; if (timeUntilSlowStartInterval < eventTimeout) { eventTimeout = timeUntilSlowStartInterval; } } } return Max(1, eventTimeout); } /* * MillisecondsBetweenTimestamps is a helper to get the number of milliseconds * between timestamps when it is expected to be small enough to fit in a * long. */ static long MillisecondsBetweenTimestamps(instr_time startTime, instr_time endTime) { INSTR_TIME_SUBTRACT(endTime, startTime); return INSTR_TIME_GET_MILLISEC(endTime); } /* * MicrosecondsBetweenTimestamps is a helper to get the number of microseconds * between timestamps. */ static uint64 MicrosecondsBetweenTimestamps(instr_time startTime, instr_time endTime) { INSTR_TIME_SUBTRACT(endTime, startTime); return INSTR_TIME_GET_MICROSEC(endTime); } /* * ConnectionStateMachine opens a connection and descends into the transaction * state machine when ready. */ static void ConnectionStateMachine(WorkerSession *session) { WorkerPool *workerPool = session->workerPool; DistributedExecution *execution = workerPool->distributedExecution; MultiConnection *connection = session->connection; MultiConnectionState currentState; do { currentState = connection->connectionState; switch (currentState) { case MULTI_CONNECTION_INITIAL: { /* simply iterate the state machine */ connection->connectionState = MULTI_CONNECTION_CONNECTING; break; } case MULTI_CONNECTION_TIMED_OUT: { /* * When the connection timeout happens, the connection * might still be able to successfuly established. However, * the executor should not try to use this connection as * the state machines might have already progressed and used * new pools/sessions instead. That's why we terminate the * connection, clear any state associated with it. * * Note that here we don't increment the failed connection * stat counter because MarkEstablishingSessionsTimedOut() * already did that. */ connection->connectionState = MULTI_CONNECTION_FAILED; break; } case MULTI_CONNECTION_CONNECTING: { ConnStatusType status = PQstatus(connection->pgConn); if (status == CONNECTION_OK) { /* * Connection was already established, possibly a cached * connection. */ bool newConnection = false; HandleMultiConnectionSuccess(session, newConnection); UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); break; } else if (status == CONNECTION_BAD) { connection->connectionState = MULTI_CONNECTION_FAILED; IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); break; } int beforePollSocket = PQsocket(connection->pgConn); PostgresPollingStatusType pollMode = PQconnectPoll(connection->pgConn); if (beforePollSocket != PQsocket(connection->pgConn)) { /* rebuild the wait events if PQconnectPoll() changed the socket */ execution->rebuildWaitEventSet = true; } if (pollMode == PGRES_POLLING_FAILED) { connection->connectionState = MULTI_CONNECTION_FAILED; IncrementStatCounterForMyDb(STAT_CONNECTION_ESTABLISHMENT_FAILED); } else if (pollMode == PGRES_POLLING_READING) { UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE); /* we should have a valid socket */ Assert(PQsocket(connection->pgConn) != -1); } else if (pollMode == PGRES_POLLING_WRITING) { UpdateConnectionWaitFlags(session, WL_SOCKET_WRITEABLE); /* we should have a valid socket */ Assert(PQsocket(connection->pgConn) != -1); } else { /* * Connection was not established befoore (!= CONNECTION_OK) * but PQconnectPoll() did so now. */ bool newConnection = true; HandleMultiConnectionSuccess(session, newConnection); UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); /* we should have a valid socket */ Assert(PQsocket(connection->pgConn) != -1); } break; } case MULTI_CONNECTION_CONNECTED: { if (HasUnfinishedTaskForSession(session)) { /* * Connection is ready, and we have unfinished tasks. * So, run the transaction state machine. */ TransactionStateMachine(session); } else { /* * Connection is ready, but we don't have any unfinished * tasks that this session can execute. * * Note that we can be in a situation where the executor * decides to establish a connection, but not need to * use it at the time the connection is established. This could * happen when the earlier connections manages to finish all the * tasks after this connection * * As no tasks are ready to be executed at the moment, we * mark the socket readable to get any notices if exists. */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE); } break; } case MULTI_CONNECTION_LOST: { /* * If a connection is lost, we retry the connection for some * very restricted scenarios. The main use case is to retry * connection establishment when a cached connection is used * in the executor while remote server has restarted / failedover * etc. * * For simplicity, we only allow retrying connection establishment * a single time. * * We can only retry connection when the remote transaction has * not started over the connection. Otherwise, we'd have to deal * with restoring the transaction state, which is beyond our * purpose at this time. */ RemoteTransaction *transaction = &connection->remoteTransaction; if (!session->connectionRetried && transaction->transactionState == REMOTE_TRANS_NOT_STARTED) { /* * Try to connect again, we will reuse the same MultiConnection * and keep it as claimed. */ RestartConnection(connection); /* socket have changed */ execution->rebuildWaitEventSet = true; session->latestUnconsumedWaitEvents = 0; session->connectionRetried = true; break; } /* * Here we don't increment the connection stat counter for failed * connections because we don't track the connections that we could * establish but lost later. */ connection->connectionState = MULTI_CONNECTION_FAILED; break; } case MULTI_CONNECTION_FAILED: { /* managed to connect, but connection was lost */ if (session->sessionHasActiveConnection) { workerPool->activeConnectionCount--; if (session->currentTask == NULL) { /* this was an idle connection */ workerPool->idleConnectionCount--; } session->sessionHasActiveConnection = false; } /* connection failed or was lost */ int totalConnectionCount = list_length(workerPool->sessionList); workerPool->failedConnectionCount++; /* if the connection executed a critical command it should fail */ MarkRemoteTransactionFailed(connection, false); /* mark all assigned placement executions as failed */ WorkerSessionFailed(session); if (workerPool->failedConnectionCount >= totalConnectionCount) { /* * All current connection attempts have failed. * Mark all unassigned placement executions as failed. * * We do not currently retry if the first connection * attempt fails. */ WorkerPoolFailed(workerPool); } /* * The execution may have failed as a result of WorkerSessionFailed * or WorkerPoolFailed. * * Even if this execution has not failed -- but just a single session is * failed -- and an earlier execution in this transaction which marked * the remote transaction as critical, we should fail right away as the * transaction will fail anyway on PREPARE/COMMIT time. */ RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionCritical || execution->failed || (execution->transactionProperties->errorOnAnyFailure && workerPool->failureState != WORKER_POOL_FAILED_OVER_TO_LOCAL)) { /* a task has failed due to this connection failure */ ReportConnectionError(connection, ERROR); } else if (workerPool->activeConnectionCount > 0 || workerPool->failureState == WORKER_POOL_FAILED_OVER_TO_LOCAL) { /* * We already have active connection(s) to the node, and the * executor is capable of using those connections to successfully * finish the execution. So, there is not much value in warning * the user. * * Similarly when the pool is failed over to local execution, warning * the user just creates chatter. */ ReportConnectionError(connection, DEBUG1); } else { ReportConnectionError(connection, WARNING); } /* remove the connection */ UnclaimConnection(connection); /* * We forcefully close the underlying libpq connection because * we don't want any subsequent execution (either subPlan executions * or new command executions within a transaction block) use the * connection. * * However, we prefer to keep the MultiConnection around until * the end of FinishDistributedExecution() to simplify the code. * Thus, we prefer ShutdownConnection() over CloseConnection(). */ ShutdownConnection(connection); /* remove connection from wait event set */ execution->rebuildWaitEventSet = true; /* * Reset the transaction state machine since CloseConnection() * relies on it and even if we're not inside a distributed transaction * we set the transaction state (e.g., REMOTE_TRANS_SENT_COMMAND). */ if (!connection->remoteTransaction.beginSent) { connection->remoteTransaction.transactionState = REMOTE_TRANS_NOT_STARTED; } break; } default: { break; } } } while (connection->connectionState != currentState); } /* * HasUnfinishedTaskForSession gets a session and returns true if there * are any tasks that this session can execute. */ static bool HasUnfinishedTaskForSession(WorkerSession *session) { if (session->currentTask != NULL) { /* the session is executing a command right now */ return true; } dlist_head *sessionReadyTaskQueue = &(session->readyTaskQueue); if (!dlist_is_empty(sessionReadyTaskQueue)) { /* session has an assigned task, which is ready for execution */ return true; } WorkerPool *workerPool = session->workerPool; dlist_head *poolReadyTaskQueue = &(workerPool->readyTaskQueue); if (!dlist_is_empty(poolReadyTaskQueue)) { /* * Pool has unassigned tasks that can be executed * by the input session. */ return true; } return false; } /* * HandleMultiConnectionSuccess logs the established connection and updates * connection's state. */ static void HandleMultiConnectionSuccess(WorkerSession *session, bool newConnection) { MultiConnection *connection = session->connection; WorkerPool *workerPool = session->workerPool; MarkConnectionConnected(connection, newConnection); ereport(DEBUG4, (errmsg("established connection to %s:%d for " "session %ld in %ld microseconds", connection->hostname, connection->port, session->sessionId, MicrosecondsBetweenTimestamps( connection->connectionEstablishmentStart, connection->connectionEstablishmentEnd)))); workerPool->activeConnectionCount++; workerPool->idleConnectionCount++; session->sessionHasActiveConnection = true; } /* * Activate2PCIfModifyingTransactionExpandsToNewNode sets the coordinated * transaction to use 2PC under the following circumstances: * - We're already in a transaction block * - At least one of the previous commands in the transaction block * made a modification, which have not set 2PC itself because it * was a single shard command * - The input "session" is used for a distributed execution which * modifies the database. However, the session (and hence the * connection) is established to a different worker than the ones * that is used previously in the transaction. * * To give an example, * BEGIN; * -- assume that the following INSERT goes to worker-A * -- also note that this single command does not activate * -- 2PC itself since it is a single shard modification * INSERT INTO distributed_table (dist_key) VALUES (1); * * -- do one more single shard UPDATE hitting the same * shard (or worker node in general) * -- this wouldn't activate 2PC, since we're operating on the * -- same worker node that we've modified earlier * -- so the executor would use the same connection * UPDATE distributed_table SET value = 10 WHERE dist_key = 1; * * -- now, do one more INSERT, which goes to worker-B * -- At this point, this function would activate 2PC * -- since we're now expanding to a new node * -- for example, if this command were a SELECT, we wouldn't * -- activate 2PC since we're only interested in modifications/DDLs * INSERT INTO distributed_table (dist_key) VALUES (2); */ static void Activate2PCIfModifyingTransactionExpandsToNewNode(WorkerSession *session) { DistributedExecution *execution = session->workerPool->distributedExecution; if (TransactionModifiedDistributedTable(execution) && DistributedExecutionModifiesDatabase(execution) && !ConnectionModifiedPlacement(session->connection)) { /* * We already did a modification, but not on the connection that we * just opened, which means we're now going to make modifications * over multiple connections. Activate 2PC! */ Use2PCForCoordinatedTransaction(); } } /* * TransactionModifiedDistributedTable returns true if the current transaction already * executed a command which modified at least one distributed table in the current * transaction. */ static bool TransactionModifiedDistributedTable(DistributedExecution *execution) { /* * We need to explicitly check for TRANSACTION_BLOCKS_REQUIRED due to * citus.function_opens_transaction_block flag. When set to false, we * should not be pretending that we're in a coordinated transaction even * if XACT_MODIFICATION_DATA is set. That's why we implemented this workaround. */ return execution->transactionProperties->useRemoteTransactionBlocks == TRANSACTION_BLOCKS_REQUIRED && XactModificationLevel == XACT_MODIFICATION_DATA; } /* * TransactionStateMachine manages the execution of tasks over a connection. */ static void TransactionStateMachine(WorkerSession *session) { WorkerPool *workerPool = session->workerPool; DistributedExecution *execution = workerPool->distributedExecution; TransactionBlocksUsage useRemoteTransactionBlocks = execution->transactionProperties->useRemoteTransactionBlocks; MultiConnection *connection = session->connection; RemoteTransaction *transaction = &(connection->remoteTransaction); RemoteTransactionState currentState; do { currentState = transaction->transactionState; if (!CheckConnectionReady(session)) { /* connection is busy, no state transitions to make */ break; } switch (currentState) { case REMOTE_TRANS_NOT_STARTED: { if (useRemoteTransactionBlocks == TRANSACTION_BLOCKS_REQUIRED) { /* if we're expanding the nodes in a transaction, use 2PC */ Activate2PCIfModifyingTransactionExpandsToNewNode(session); /* need to open a transaction block first */ StartRemoteTransactionBegin(connection); transaction->transactionState = REMOTE_TRANS_CLEARING_RESULTS; } else { TaskPlacementExecution *placementExecution = PopPlacementExecution( session); if (placementExecution == NULL) { /* * No tasks are ready to be executed at the moment. But we * still mark the socket readable to get any notices if exists. */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE); break; } bool placementExecutionStarted = StartPlacementExecutionOnSession(placementExecution, session); if (!placementExecutionStarted) { /* no need to continue, connection is lost */ Assert(session->connection->connectionState == MULTI_CONNECTION_LOST); return; } transaction->transactionState = REMOTE_TRANS_SENT_COMMAND; } UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); break; } case REMOTE_TRANS_SENT_BEGIN: case REMOTE_TRANS_CLEARING_RESULTS: { PGresult *result = PQgetResult(connection->pgConn); if (result != NULL) { if (!IsResponseOK(result)) { /* query failures are always hard errors */ ReportResultError(connection, result, ERROR); } PQclear(result); /* wake up WaitEventSetWait */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); break; } if (session->currentTask != NULL) { TaskPlacementExecution *placementExecution = session->currentTask; bool succeeded = true; /* * Once we finished a task on a connection, we no longer * allow that connection to fail. */ MarkRemoteTransactionCritical(connection); session->currentTask = NULL; PlacementExecutionDone(placementExecution, succeeded); /* connection is ready to use for executing commands */ workerPool->idleConnectionCount++; } /* connection needs to be writeable to send next command */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); if (transaction->beginSent) { transaction->transactionState = REMOTE_TRANS_STARTED; } else { transaction->transactionState = REMOTE_TRANS_NOT_STARTED; } break; } case REMOTE_TRANS_STARTED: { TaskPlacementExecution *placementExecution = PopPlacementExecution( session); if (placementExecution == NULL) { /* no tasks are ready to be executed at the moment */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE); break; } bool placementExecutionStarted = StartPlacementExecutionOnSession(placementExecution, session); if (!placementExecutionStarted) { /* no need to continue, connection is lost */ Assert(session->connection->connectionState == MULTI_CONNECTION_LOST); return; } transaction->transactionState = REMOTE_TRANS_SENT_COMMAND; break; } case REMOTE_TRANS_SENT_COMMAND: { TaskPlacementExecution *placementExecution = session->currentTask; if (placementExecution == NULL) { /* * We have seen accounts in production where the placementExecution * could inadvertently be not set. Investigation documented on * https://github.com/citusdata/citus-enterprise/issues/493 * (due to sensitive data in the initial report it is not discussed * in our community repository) * * Currently we don't have a reliable way of reproducing this issue. * Erroring here seems to be a more desirable approach compared to a * SEGFAULT on the dereference of placementExecution, with a possible * crash recovery as a result. */ ereport(ERROR, (errmsg( "unable to recover from inconsistent state in " "the connection state machine on coordinator"))); } ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; Task *task = shardCommandExecution->task; /* * In EXPLAIN ANALYZE we need to store results except for multiple placements, * regardless of query type. In other cases, doing the same doesn't seem to have * a drawback. */ bool storeRows = true; if (shardCommandExecution->gotResults) { /* already received results from another replica */ storeRows = false; } else if (task->partiallyLocalOrRemote) { /* * For the tasks that involves placements from both * remote and local placments, such as modifications * to reference tables, we store the rows during the * local placement/execution. */ storeRows = false; } bool fetchDone = ReceiveResults(session, storeRows); if (!fetchDone) { break; } /* if this is a multi-query task, send the next query */ if (placementExecution->queryIndex < task->queryCount) { bool querySent = SendNextQuery(placementExecution, session); if (!querySent) { /* no need to continue, connection is lost */ Assert(session->connection->connectionState == MULTI_CONNECTION_LOST); return; } /* * At this point the query might be just in pgconn buffers. We * need to wait until it becomes writeable to actually send * the query. */ UpdateConnectionWaitFlags(session, WL_SOCKET_WRITEABLE | WL_SOCKET_READABLE); transaction->transactionState = REMOTE_TRANS_SENT_COMMAND; break; } shardCommandExecution->gotResults = true; transaction->transactionState = REMOTE_TRANS_CLEARING_RESULTS; break; } default: { break; } } } /* iterate in case we can perform multiple transitions at once */ while (transaction->transactionState != currentState); } /* * UpdateConnectionWaitFlags is a wrapper around setting waitFlags of the connection. * * This function might further improved in a sense that to use use ModifyWaitEvent on * waitFlag changes as opposed to what we do now: always rebuild the wait event sets. * Our initial benchmarks didn't show any significant performance improvements, but * good to keep in mind the potential improvements. */ static void UpdateConnectionWaitFlags(WorkerSession *session, int waitFlags) { MultiConnection *connection = session->connection; DistributedExecution *execution = session->workerPool->distributedExecution; /* do not take any actions if the flags not changed */ if (connection->waitFlags == waitFlags) { return; } /* always detect closed sockets */ connection->waitFlags = waitFlags | WL_SOCKET_CLOSED; /* without signalling the execution, the flag changes won't be reflected */ execution->waitFlagsChanged = true; } /* * CheckConnectionReady returns true if the connection is ready to * read or write, or false if it still has bytes to send/receive. */ static bool CheckConnectionReady(WorkerSession *session) { MultiConnection *connection = session->connection; int waitFlags = WL_SOCKET_READABLE; bool connectionReady = false; ConnStatusType status = PQstatus(connection->pgConn); if (status == CONNECTION_BAD) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } if ((session->latestUnconsumedWaitEvents & WL_SOCKET_CLOSED) != 0) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } /* try to send all pending data */ int sendStatus = PQflush(connection->pgConn); if (sendStatus == -1) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } else if (sendStatus == 1) { /* more data to send, wait for socket to become writable */ waitFlags = waitFlags | WL_SOCKET_WRITEABLE; } if ((session->latestUnconsumedWaitEvents & WL_SOCKET_READABLE) != 0) { if (PQconsumeInput(connection->pgConn) == 0) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } } if (!PQisBusy(connection->pgConn)) { connectionReady = true; } UpdateConnectionWaitFlags(session, waitFlags); /* don't consume input redundantly if we cycle back into CheckConnectionReady */ session->latestUnconsumedWaitEvents = 0; return connectionReady; } /* * PopPlacementExecution returns the next available assigned or unassigned * placement execution for the given session. */ static TaskPlacementExecution * PopPlacementExecution(WorkerSession *session) { WorkerPool *workerPool = session->workerPool; TaskPlacementExecution *placementExecution = PopAssignedPlacementExecution(session); if (placementExecution == NULL) { if (session->commandsSent > 0 && UseConnectionPerPlacement()) { /* * Only send one command per connection if force_max_query_parallelisation * is enabled, unless it's an assigned placement execution. */ return NULL; } /* no more assigned tasks, pick an unassigned task */ placementExecution = PopUnassignedPlacementExecution(workerPool); } return placementExecution; } /* * PopAssignedPlacementExecution finds an executable task from the queue of assigned tasks. */ static TaskPlacementExecution * PopAssignedPlacementExecution(WorkerSession *session) { dlist_head *readyTaskQueue = &(session->readyTaskQueue); if (dlist_is_empty(readyTaskQueue)) { return NULL; } TaskPlacementExecution *placementExecution = dlist_container(TaskPlacementExecution, sessionReadyQueueNode, dlist_pop_head_node( readyTaskQueue)); return placementExecution; } /* * PopUnAssignedPlacementExecution finds an executable task from the queue of unassigned tasks. */ static TaskPlacementExecution * PopUnassignedPlacementExecution(WorkerPool *workerPool) { dlist_head *readyTaskQueue = &(workerPool->readyTaskQueue); if (dlist_is_empty(readyTaskQueue)) { return NULL; } TaskPlacementExecution *placementExecution = dlist_container(TaskPlacementExecution, workerReadyQueueNode, dlist_pop_head_node( readyTaskQueue)); workerPool->readyTaskCount--; return placementExecution; } /* * StartPlacementExecutionOnSession gets a TaskPlacementExecution and * WorkerSession, the task's query is sent to the worker via the session. * * The function does some bookkeeping such as associating the placement * accesses with the connection and updating session's local variables. For * details read the comments in the function. * * The function returns true if the query is successfully sent over the * connection, otherwise false. */ static bool StartPlacementExecutionOnSession(TaskPlacementExecution *placementExecution, WorkerSession *session) { WorkerPool *workerPool = session->workerPool; DistributedExecution *execution = workerPool->distributedExecution; MultiConnection *connection = session->connection; ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; Task *task = shardCommandExecution->task; ShardPlacement *taskPlacement = placementExecution->shardPlacement; List *placementAccessList = PlacementAccessListForTask(task, taskPlacement); if (execution->transactionProperties->useRemoteTransactionBlocks != TRANSACTION_BLOCKS_DISALLOWED) { /* * Make sure that subsequent commands on the same placement * use the same connection. */ AssignPlacementListToConnection(placementAccessList, connection); } if (session->commandsSent == 0) { /* first time we send a command, consider the connection used (not unused) */ workerPool->unusedConnectionCount--; } /* connection is going to be in use */ workerPool->idleConnectionCount--; session->currentTask = placementExecution; placementExecution->executionState = PLACEMENT_EXECUTION_RUNNING; Assert(INSTR_TIME_IS_ZERO(placementExecution->startTime)); /* * The same TaskPlacementExecution can be used to have * call SendNextQuery() several times if queryIndex is * non-zero. Still, all are executed under the current * placementExecution, so we can start the timer right * now. */ INSTR_TIME_SET_CURRENT(placementExecution->startTime); bool querySent = SendNextQuery(placementExecution, session); if (querySent) { session->commandsSent++; if (workerPool->poolToLocalNode) { /* * As we started remote execution to the local node, * we cannot switch back to local execution as that * would cause self-deadlocks and breaking * read-your-own-writes consistency. */ SetLocalExecutionStatus(LOCAL_EXECUTION_DISABLED); } } return querySent; } /* * SendNextQuery sends the next query for placementExecution on the given * session. */ static bool SendNextQuery(TaskPlacementExecution *placementExecution, WorkerSession *session) { WorkerPool *workerPool = session->workerPool; DistributedExecution *execution = workerPool->distributedExecution; MultiConnection *connection = session->connection; ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; bool binaryResults = shardCommandExecution->binaryResults; Task *task = shardCommandExecution->task; ParamListInfo paramListInfo = execution->paramListInfo; int querySent = 0; uint32 queryIndex = placementExecution->queryIndex; Assert(queryIndex < task->queryCount); char *queryString = TaskQueryStringAtIndex(task, queryIndex); if (paramListInfo != NULL && !task->parametersInQueryStringResolved) { int parameterCount = paramListInfo->numParams; Oid *parameterTypes = NULL; const char **parameterValues = NULL; /* force evaluation of bound params */ paramListInfo = copyParamList(paramListInfo); ExtractParametersForRemoteExecution(paramListInfo, ¶meterTypes, ¶meterValues); querySent = SendRemoteCommandParams(connection, queryString, parameterCount, parameterTypes, parameterValues, binaryResults); } else { /* * We only need to use SendRemoteCommandParams when we desire * binaryResults. One downside of SendRemoteCommandParams is that it * only supports one query in the query string. In some cases we have * more than one query. In those cases we already make sure before that * binaryResults is false. * * XXX: It also seems that SendRemoteCommandParams does something * strange/incorrectly with select statements. In * isolation_select_vs_all.spec, when doing an s1-router-select in one * session blocked an s2-ddl-create-index-concurrently in another. */ if (!binaryResults) { querySent = SendRemoteCommand(connection, queryString); } else { querySent = SendRemoteCommandParams(connection, queryString, 0, NULL, NULL, binaryResults); } } if (querySent == 0) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } int singleRowMode = PQsetSingleRowMode(connection->pgConn); if (singleRowMode == 0) { connection->connectionState = MULTI_CONNECTION_LOST; return false; } return true; } /* * ReceiveResults reads the result of a command or query and writes returned * rows to the tuple store of the scan state. It returns whether fetching results * were done. On failure, it throws an error. */ static bool ReceiveResults(WorkerSession *session, bool storeRows) { bool fetchDone = false; MultiConnection *connection = session->connection; WorkerPool *workerPool = session->workerPool; DistributedExecution *execution = workerPool->distributedExecution; TaskPlacementExecution *placementExecution = session->currentTask; ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; Task *task = placementExecution->shardCommandExecution->task; TupleDestination *tupleDest = task->tupleDest ? task->tupleDest : execution->defaultTupleDest; /* * We use this context while converting each row fetched from remote node * into tuple. The context is reseted on every row, thus we create it at the * start of the loop and reset on every iteration. */ MemoryContext rowContext = AllocSetContextCreate(CurrentMemoryContext, "RowContext", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); while (!PQisBusy(connection->pgConn)) { uint32 columnIndex = 0; uint32 rowsProcessed = 0; PGresult *result = PQgetResult(connection->pgConn); if (result == NULL) { /* no more results, break out of loop and free allocated memory */ fetchDone = true; break; } ExecStatusType resultStatus = PQresultStatus(result); if (resultStatus == PGRES_COMMAND_OK) { char *currentAffectedTupleString = PQcmdTuples(result); int64 currentAffectedTupleCount = 0; /* if there are multiple replicas, make sure to consider only one */ if (storeRows && *currentAffectedTupleString != '\0') { currentAffectedTupleCount = pg_strtoint64(currentAffectedTupleString); Assert(currentAffectedTupleCount >= 0); execution->rowsProcessed += currentAffectedTupleCount; } PQclear(result); /* task query might contain multiple queries, so fetch until we reach NULL */ placementExecution->queryIndex++; continue; } else if (resultStatus == PGRES_TUPLES_OK) { /* * We've already consumed all the tuples, no more results. Break out * of loop and free allocated memory before returning. */ Assert(PQntuples(result) == 0); PQclear(result); /* task query might contain multiple queries, so fetch until we reach NULL */ placementExecution->queryIndex++; continue; } else if (resultStatus != PGRES_SINGLE_TUPLE) { /* query failures are always hard errors */ ReportResultError(connection, result, ERROR); } else if (!storeRows) { /* * Already receieved rows from executing on another shard placement or * doesn't need at all (e.g., DDL). */ PQclear(result); continue; } uint32 queryIndex = placementExecution->queryIndex; if (queryIndex >= task->queryCount) { ereport(ERROR, (errmsg("unexpected query index while processing" " query results"))); } TupleDesc tupleDescriptor = tupleDest->tupleDescForQuery(tupleDest, queryIndex); if (tupleDescriptor == NULL) { PQclear(result); continue; } rowsProcessed = PQntuples(result); uint32 columnCount = PQnfields(result); uint32 expectedColumnCount = tupleDescriptor->natts; if (columnCount != expectedColumnCount) { ereport(ERROR, (errmsg("unexpected number of columns from worker: %d, " "expected %d", columnCount, expectedColumnCount))); } if (columnCount > execution->allocatedColumnCount) { pfree(execution->columnArray); int oldColumnCount = execution->allocatedColumnCount; execution->allocatedColumnCount = columnCount; execution->columnArray = palloc0(execution->allocatedColumnCount * sizeof(void *)); if (EnableBinaryProtocol) { /* * Using repalloc here, to not throw away any previously * created StringInfos. */ execution->stringInfoDataArray = repalloc( execution->stringInfoDataArray, execution->allocatedColumnCount * sizeof(StringInfoData)); for (int i = oldColumnCount; i < columnCount; i++) { initStringInfo(&execution->stringInfoDataArray[i]); } } } void **columnArray = execution->columnArray; StringInfoData *stringInfoDataArray = execution->stringInfoDataArray; bool binaryResults = shardCommandExecution->binaryResults; /* * stringInfoDataArray is NULL when EnableBinaryProtocol is false. So * we make sure binaryResults is also false in that case. Otherwise we * cannot store them anywhere. */ Assert(EnableBinaryProtocol || !binaryResults); for (uint32 rowIndex = 0; rowIndex < rowsProcessed; rowIndex++) { uint64 tupleLibpqSize = 0; /* * Switch to a temporary memory context that we reset after each * tuple. This protects us from any memory leaks that might be * present in anything we do to parse a tuple. */ MemoryContext oldContext = MemoryContextSwitchTo(rowContext); memset(columnArray, 0, columnCount * sizeof(void *)); for (columnIndex = 0; columnIndex < columnCount; columnIndex++) { if (PQgetisnull(result, rowIndex, columnIndex)) { columnArray[columnIndex] = NULL; } else { int valueLength = PQgetlength(result, rowIndex, columnIndex); char *value = PQgetvalue(result, rowIndex, columnIndex); if (binaryResults) { if (PQfformat(result, columnIndex) == 0) { ereport(ERROR, (errmsg("unexpected text result"))); } resetStringInfo(&stringInfoDataArray[columnIndex]); appendBinaryStringInfo(&stringInfoDataArray[columnIndex], value, valueLength); columnArray[columnIndex] = &stringInfoDataArray[columnIndex]; } else { if (PQfformat(result, columnIndex) == 1) { ereport(ERROR, (errmsg("unexpected binary result"))); } columnArray[columnIndex] = value; } tupleLibpqSize += valueLength; } } AttInMetadata *attInMetadata = shardCommandExecution->attributeInputMetadata[queryIndex]; HeapTuple heapTuple; if (binaryResults) { heapTuple = BuildTupleFromBytes(attInMetadata, (fmStringInfo *) columnArray); } else { heapTuple = BuildTupleFromCStrings(attInMetadata, (char **) columnArray); } MemoryContextSwitchTo(oldContext); tupleDest->putTuple(tupleDest, task, placementExecution->placementExecutionIndex, queryIndex, heapTuple, tupleLibpqSize); MemoryContextReset(rowContext); execution->rowsProcessed++; } PQclear(result); } /* the context is local to the function, so not needed anymore */ MemoryContextDelete(rowContext); return fetchDone; } /* * WorkerPoolFailed marks a worker pool and all the placement executions scheduled * on it as failed. */ static void WorkerPoolFailed(WorkerPool *workerPool) { bool succeeded = false; dlist_iter iter; /* * A pool cannot fail multiple times, the necessary actions * has already be taken, so bail out. */ if (workerPool->failureState == WORKER_POOL_FAILED || workerPool->failureState == WORKER_POOL_FAILED_OVER_TO_LOCAL) { return; } dlist_foreach(iter, &workerPool->pendingTaskQueue) { TaskPlacementExecution *placementExecution = dlist_container(TaskPlacementExecution, workerPendingQueueNode, iter.cur); PlacementExecutionDone(placementExecution, succeeded); } dlist_foreach(iter, &workerPool->readyTaskQueue) { TaskPlacementExecution *placementExecution = dlist_container(TaskPlacementExecution, workerReadyQueueNode, iter.cur); PlacementExecutionDone(placementExecution, succeeded); } WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { WorkerSessionFailed(session); } /* we do not want more connections in this pool */ workerPool->readyTaskCount = 0; if (workerPool->failureState != WORKER_POOL_FAILED_OVER_TO_LOCAL) { /* we prefer not to override WORKER_POOL_FAILED_OVER_TO_LOCAL */ workerPool->failureState = WORKER_POOL_FAILED; } /* * The reason is that when replication factor is > 1 and we are performing * a SELECT, then we only establish connections for the specific placements * that we will read from. However, when a worker pool fails, we will need * to establish multiple new connection to other workers and the query * can only succeed if all those connections are established. */ if (UseConnectionPerPlacement()) { List *workerList = workerPool->distributedExecution->workerList; WorkerPool *pool = NULL; foreach_declared_ptr(pool, workerList) { /* failed pools or pools without any connection attempts ignored */ if (pool->failureState == WORKER_POOL_FAILED || INSTR_TIME_IS_ZERO(pool->poolStartTime)) { continue; } /* * This should give another NodeConnectionTimeout until all * the necessary connections are established. */ INSTR_TIME_SET_CURRENT(pool->poolStartTime); pool->checkForPoolTimeout = true; } } } /* * WorkerSessionFailed marks all placement executions scheduled on the * connection as failed. */ static void WorkerSessionFailed(WorkerSession *session) { TaskPlacementExecution *placementExecution = session->currentTask; bool succeeded = false; dlist_iter iter; if (placementExecution != NULL) { /* connection failed while a task was active */ PlacementExecutionDone(placementExecution, succeeded); } dlist_foreach(iter, &session->pendingTaskQueue) { placementExecution = dlist_container(TaskPlacementExecution, sessionPendingQueueNode, iter.cur); PlacementExecutionDone(placementExecution, succeeded); } dlist_foreach(iter, &session->readyTaskQueue) { placementExecution = dlist_container(TaskPlacementExecution, sessionReadyQueueNode, iter.cur); PlacementExecutionDone(placementExecution, succeeded); } } /* * PlacementExecutionDone marks the given placement execution as done when * the results have been received or a failure occurred and sets the succeeded * flag accordingly. It also adds other placement executions of the same * task to the appropriate ready queues. */ static void PlacementExecutionDone(TaskPlacementExecution *placementExecution, bool succeeded) { WorkerPool *workerPool = placementExecution->workerPool; DistributedExecution *execution = workerPool->distributedExecution; ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; TaskExecutionState executionState = shardCommandExecution->executionState; bool failedPlacementExecutionIsOnPendingQueue = false; if (placementExecution->executionState == PLACEMENT_EXECUTION_FAILED) { /* * We may mark placements as failed multiple times, but should only act * the first time. Nor should we accept success after failure. */ return; } if (succeeded) { /* mark the placement execution as finished */ placementExecution->executionState = PLACEMENT_EXECUTION_FINISHED; Assert(INSTR_TIME_IS_ZERO(placementExecution->endTime)); INSTR_TIME_SET_CURRENT(placementExecution->endTime); uint64 durationMicrosecs = MicrosecondsBetweenTimestamps(placementExecution->startTime, placementExecution->endTime); workerPool->totalTaskExecutionTime += durationMicrosecs; workerPool->totalExecutedTasks += 1; if (IsLoggableLevel(DEBUG4)) { ereport(DEBUG4, (errmsg("task execution (%d) for placement (%ld) on anchor " "shard (%ld) finished in %ld microseconds on worker " "node %s:%d", shardCommandExecution->task->taskId, placementExecution->shardPlacement->placementId, shardCommandExecution->task->anchorShardId, durationMicrosecs, workerPool->nodeName, workerPool->nodePort))); } } else if (CanFailoverPlacementExecutionToLocalExecution(placementExecution)) { /* * The placement execution can be done over local execution, so it is a soft * failure for now. */ placementExecution->executionState = PLACEMENT_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION; } else { if (placementExecution->executionState == PLACEMENT_EXECUTION_NOT_READY) { /* * If the placement is in NOT_READY state, it means that the placement * execution is assigned to the pending queue of a failed pool or * session. So, we should not schedule the next placement execution based * on this failure. */ failedPlacementExecutionIsOnPendingQueue = true; } placementExecution->executionState = PLACEMENT_EXECUTION_FAILED; } if (executionState != TASK_EXECUTION_NOT_FINISHED) { /* * Task execution has already been finished, no need to continue the * next placement. */ return; } /* * Update unfinishedTaskCount only when state changes from not finished to * finished or failed state. */ TaskExecutionState newExecutionState = TaskExecutionStateMachine(shardCommandExecution); if (newExecutionState == TASK_EXECUTION_FINISHED) { execution->unfinishedTaskCount--; return; } else if (newExecutionState == TASK_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION) { execution->unfinishedTaskCount--; /* move the task to the local execution */ Task *task = shardCommandExecution->task; execution->localTaskList = lappend(execution->localTaskList, task); /* remove the task from the remote execution list */ execution->remoteTaskList = list_delete_ptr(execution->remoteTaskList, task); /* * As we decided to failover this task to local execution, we cannot * allow remote execution to this pool during this distributedExecution. */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); workerPool->failureState = WORKER_POOL_FAILED_OVER_TO_LOCAL; ereport(DEBUG4, (errmsg("Task %d execution is failed over to local execution", task->taskId))); return; } else if (newExecutionState == TASK_EXECUTION_FAILED) { execution->unfinishedTaskCount--; /* * Even if a single task execution fails, there is no way to * successfully finish the execution. */ execution->failed = true; return; } else if (!failedPlacementExecutionIsOnPendingQueue) { ScheduleNextPlacementExecution(placementExecution, succeeded); } } /* * CanFailoverPlacementExecutionToLocalExecution returns true if the input * TaskPlacementExecution can be fail overed to local execution. In other words, * the execution can be deferred to local execution. */ static bool CanFailoverPlacementExecutionToLocalExecution(TaskPlacementExecution *placementExecution) { if (!EnableLocalExecution) { /* the user explicitly disabled local execution */ return false; } if (!placementExecution->shardCommandExecution->localExecutionSupported) { /* cannot execute given task locally */ return false; } if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_DISABLED) { /* * If the current transaction accessed the local node over a connection * then we can't use local execution because of visibility issues. */ return false; } WorkerPool *workerPool = placementExecution->workerPool; if (!workerPool->poolToLocalNode) { /* we can only fail over tasks to local execution for local pools */ return false; } if (workerPool->activeConnectionCount > 0) { /* * The pool has already active connections, the executor is capable * of using those active connections. So, no need to failover * to the local execution. */ return false; } if (placementExecution->assignedSession != NULL) { /* * If the placement execution has been assigned to a specific session, * it has to be executed over that session. Otherwise, it would cause * self-deadlocks and break read-your-own-writes consistency. */ return false; } return true; } /* * ScheduleNextPlacementExecution is triggered if the query needs to be * executed on any or all placements in order and there is a placement on * which the execution has not happened yet. If so make that placement * ready-to-start by adding it to the appropriate queue. */ static void ScheduleNextPlacementExecution(TaskPlacementExecution *placementExecution, bool succeeded) { ShardCommandExecution *shardCommandExecution = placementExecution->shardCommandExecution; PlacementExecutionOrder executionOrder = shardCommandExecution->executionOrder; if ((executionOrder == EXECUTION_ORDER_ANY && !succeeded) || executionOrder == EXECUTION_ORDER_SEQUENTIAL) { TaskPlacementExecution *nextPlacementExecution = NULL; /* find a placement execution that is not yet marked as failed */ do { int nextPlacementExecutionIndex = placementExecution->placementExecutionIndex + 1; /* * If all tasks failed then we should already have errored out. * Still, be defensive and throw error instead of crashes. */ int placementExecutionCount = shardCommandExecution->placementExecutionCount; if (nextPlacementExecutionIndex >= placementExecutionCount) { WorkerPool *workerPool = placementExecution->workerPool; ereport(ERROR, (errmsg("execution cannot recover from multiple " "connection failures. Last node failed " "%s:%d", workerPool->nodeName, workerPool->nodePort))); } /* get the next placement in the planning order */ nextPlacementExecution = shardCommandExecution->placementExecutions[nextPlacementExecutionIndex]; if (nextPlacementExecution->executionState == PLACEMENT_EXECUTION_NOT_READY) { /* move the placement execution to the ready queue */ PlacementExecutionReady(nextPlacementExecution); } } while (nextPlacementExecution->executionState == PLACEMENT_EXECUTION_FAILED); } } /* * PlacementExecutionReady adds a placement execution to the ready queue when * its dependent placement executions have finished. */ static void PlacementExecutionReady(TaskPlacementExecution *placementExecution) { WorkerPool *workerPool = placementExecution->workerPool; if (placementExecution->assignedSession != NULL) { WorkerSession *session = placementExecution->assignedSession; MultiConnection *connection = session->connection; RemoteTransaction *transaction = &(connection->remoteTransaction); RemoteTransactionState transactionState = transaction->transactionState; if (placementExecution->executionState == PLACEMENT_EXECUTION_NOT_READY) { /* remove from not-ready task queue */ dlist_delete(&placementExecution->sessionPendingQueueNode); /* add to ready-to-start task queue */ dlist_push_tail(&session->readyTaskQueue, &placementExecution->sessionReadyQueueNode); } if (transactionState == REMOTE_TRANS_NOT_STARTED || transactionState == REMOTE_TRANS_STARTED) { /* * If the connection is idle, wake it up by checking whether * the connection is writeable. */ UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); } } else { if (placementExecution->executionState == PLACEMENT_EXECUTION_NOT_READY) { /* remove from not-ready task queue */ dlist_delete(&placementExecution->workerPendingQueueNode); /* add to ready-to-start task queue */ dlist_push_tail(&workerPool->readyTaskQueue, &placementExecution->workerReadyQueueNode); } workerPool->readyTaskCount++; /* wake up an idle connection by checking whether the connection is writeable */ WorkerSession *session = NULL; foreach_declared_ptr(session, workerPool->sessionList) { MultiConnection *connection = session->connection; RemoteTransaction *transaction = &(connection->remoteTransaction); RemoteTransactionState transactionState = transaction->transactionState; if (transactionState == REMOTE_TRANS_NOT_STARTED || transactionState == REMOTE_TRANS_STARTED) { UpdateConnectionWaitFlags(session, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); break; } } } /* update the state to ready for further processing */ placementExecution->executionState = PLACEMENT_EXECUTION_READY; } /* * TaskExecutionStateMachine returns whether a shard command execution * finished or failed according to its execution order. If the task is * already finished, simply return the state. Else, calculate the state * and return it. */ static TaskExecutionState TaskExecutionStateMachine(ShardCommandExecution *shardCommandExecution) { PlacementExecutionOrder executionOrder = shardCommandExecution->executionOrder; int donePlacementCount = 0; int failedPlacementCount = 0; int failedOverPlacementCount = 0; int placementCount = 0; int placementExecutionIndex = 0; int placementExecutionCount = shardCommandExecution->placementExecutionCount; TaskExecutionState currentTaskExecutionState = shardCommandExecution->executionState; if (currentTaskExecutionState != TASK_EXECUTION_NOT_FINISHED) { /* we've already calculated the state, simply return it */ return currentTaskExecutionState; } for (; placementExecutionIndex < placementExecutionCount; placementExecutionIndex++) { TaskPlacementExecution *placementExecution = shardCommandExecution->placementExecutions[placementExecutionIndex]; TaskPlacementExecutionState executionState = placementExecution->executionState; if (executionState == PLACEMENT_EXECUTION_FINISHED) { donePlacementCount++; } else if (executionState == PLACEMENT_EXECUTION_FAILED) { failedPlacementCount++; } else if (executionState == PLACEMENT_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION) { failedOverPlacementCount++; } placementCount++; } if (failedPlacementCount == placementCount) { currentTaskExecutionState = TASK_EXECUTION_FAILED; } else if (executionOrder != EXECUTION_ORDER_ANY && failedPlacementCount > 0) { currentTaskExecutionState = TASK_EXECUTION_FAILED; } else if (executionOrder == EXECUTION_ORDER_ANY && donePlacementCount > 0) { currentTaskExecutionState = TASK_EXECUTION_FINISHED; } else if (donePlacementCount + failedPlacementCount == placementCount) { currentTaskExecutionState = TASK_EXECUTION_FINISHED; } else if (failedOverPlacementCount + donePlacementCount + failedPlacementCount == placementCount) { /* * For any given task, we could have 3 end states: * - "donePlacementCount" indicates the successful placement executions * - "failedPlacementCount" indicates the failed placement executions * - "failedOverPlacementCount" indicates the placement executions that * are failed when using remote execution due to connection errors, * but there is still a possibility of being successful via * local execution. So, for now they are considered as soft * errors. */ currentTaskExecutionState = TASK_EXECUTION_FAILOVER_TO_LOCAL_EXECUTION; } else { currentTaskExecutionState = TASK_EXECUTION_NOT_FINISHED; } shardCommandExecution->executionState = currentTaskExecutionState; return shardCommandExecution->executionState; } /* * BuildWaitEventSet creates a WaitEventSet for the given array of connections * which can be used to wait for any of the sockets to become read-ready or * write-ready. */ static WaitEventSet * BuildWaitEventSet(List *sessionList) { /* additional 2 is for postmaster and latch */ int eventSetSize = GetEventSetSize(sessionList); WaitEventSet *waitEventSet = CreateWaitEventSet(WaitEventSetTracker_compat, eventSetSize); WorkerSession *session = NULL; foreach_declared_ptr(session, sessionList) { AddSessionToWaitEventSet(session, waitEventSet); } return waitEventSet; } /* * FreeExecutionWaitEvents is a helper function that gets * a DistributedExecution and frees events and waitEventSet. */ static void FreeExecutionWaitEvents(DistributedExecution *execution) { if (execution->events != NULL) { pfree(execution->events); execution->events = NULL; } if (execution->waitEventSet != NULL) { FreeWaitEventSet(execution->waitEventSet); execution->waitEventSet = NULL; } } /* * AddSessionToWaitEventSet is a helper function which adds the session to * the waitEventSet. The function does certain checks before adding the session * to the waitEventSet. */ static void AddSessionToWaitEventSet(WorkerSession *session, WaitEventSet *waitEventSet) { MultiConnection *connection = session->connection; if (connection->pgConn == NULL) { /* connection died earlier in the transaction */ return; } if (connection->waitFlags == 0) { /* not currently waiting for this connection */ return; } int sock = PQsocket(connection->pgConn); if (sock == -1) { /* connection was closed */ return; } int waitEventSetIndex = CitusAddWaitEventSetToSet(waitEventSet, connection->waitFlags, sock, NULL, (void *) session); session->waitEventSetIndex = waitEventSetIndex; /* * Inform failed to add to wait event set with a debug message as this * is too detailed information for users. */ if (session->waitEventSetIndex == WAIT_EVENT_SET_INDEX_FAILED) { ereport(DEBUG1, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("Adding wait event for node %s:%d failed. " "The socket was: %d", session->workerPool->nodeName, session->workerPool->nodePort, sock))); } } /* * GetEventSetSize returns the event set size for a list of sessions. */ static int GetEventSetSize(List *sessionList) { /* additional 2 is for postmaster and latch */ return list_length(sessionList) + 2; } /* * RebuildWaitEventSetFlags modifies the given waitEventSet with the wait flags * for connections in the sessionList. */ static void RebuildWaitEventSetFlags(WaitEventSet *waitEventSet, List *sessionList) { WorkerSession *session = NULL; foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; int waitEventSetIndex = session->waitEventSetIndex; if (connection->pgConn == NULL) { /* connection died earlier in the transaction */ continue; } if (connection->waitFlags == 0) { /* not currently waiting for this connection */ continue; } int sock = PQsocket(connection->pgConn); if (sock == -1) { /* connection was closed */ continue; } bool success = CitusModifyWaitEvent(waitEventSet, waitEventSetIndex, connection->waitFlags, NULL); if (!success) { ereport(DEBUG1, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("modifying wait event for node %s:%d failed. " "The wait event index was: %d", connection->hostname, connection->port, waitEventSetIndex))); session->waitEventSetIndex = WAIT_EVENT_SET_INDEX_FAILED; } } } /* * CleanUpSessions does any clean-up necessary for the session used * during the execution. We only reach the function after successfully * completing all the tasks and we expect no tasks are still in progress. */ static void CleanUpSessions(DistributedExecution *execution) { List *sessionList = execution->sessionList; /* we get to this function only after successful executions */ Assert(!execution->failed && execution->unfinishedTaskCount == 0); /* always trigger wait event set in the first round */ WorkerSession *session = NULL; foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; ereport(DEBUG4, (errmsg("Total number of commands sent over the session %ld: %ld " "to node %s:%d", session->sessionId, session->commandsSent, connection->hostname, connection->port))); UnclaimConnection(connection); if (connection->connectionState == MULTI_CONNECTION_CONNECTING || connection->connectionState == MULTI_CONNECTION_FAILED || connection->connectionState == MULTI_CONNECTION_LOST || connection->connectionState == MULTI_CONNECTION_TIMED_OUT) { /* * We want the MultiConnection go away and not used in * the subsequent executions. * * We cannot get MULTI_CONNECTION_LOST via the ConnectionStateMachine, * but we might get it via the connection API and find us here before * changing any states in the ConnectionStateMachine. * */ CloseConnection(connection); } else if (connection->connectionState == MULTI_CONNECTION_CONNECTED) { RemoteTransaction *transaction = &(connection->remoteTransaction); RemoteTransactionState transactionState = transaction->transactionState; if (transactionState == REMOTE_TRANS_CLEARING_RESULTS) { /* * We might have established the connection, and even sent BEGIN, but not * get to the point where we assigned a task to this specific connection * (because other connections in the pool already finished all the tasks). */ Assert(session->commandsSent == 0); ClearResults(connection, false); } else if (!(transactionState == REMOTE_TRANS_NOT_STARTED || transactionState == REMOTE_TRANS_STARTED)) { /* * We don't have to handle anything else. Note that the execution * could only finish on connectionStates of MULTI_CONNECTION_CONNECTING, * MULTI_CONNECTION_FAILED and MULTI_CONNECTION_CONNECTED. The first two * are already handled above. * * When we're on MULTI_CONNECTION_CONNECTED, TransactionStateMachine * ensures that all the necessary commands are successfully sent over * the connection and everything is cleared up. Otherwise, we'd have been * on MULTI_CONNECTION_FAILED state. */ ereport(WARNING, (errmsg("unexpected transaction state at the end of " "execution: %d", transactionState))); } /* get ready for the next executions if we need use the same connection */ connection->waitFlags = WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE; } else { ereport(WARNING, (errmsg("unexpected connection state at the end of " "execution: %d", connection->connectionState))); } } } /* * UnclaimAllSessionConnections unclaims all of the connections for the given * sessionList. */ static void UnclaimAllSessionConnections(List *sessionList) { WorkerSession *session = NULL; foreach_declared_ptr(session, sessionList) { MultiConnection *connection = session->connection; UnclaimConnection(connection); } } /* * SetLocalForceMaxQueryParallelization is simply a C interface for setting * the following: * SET LOCAL citus.force_max_query_parallelization TO on; */ void SetLocalForceMaxQueryParallelization(void) { set_config_option("citus.force_max_query_parallelization", "on", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } ================================================ FILE: src/backend/distributed/executor/citus_custom_scan.c ================================================ /*------------------------------------------------------------------------- * * citus_custom_scan.c * * Definitions of custom scan methods for all executor types. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/pg_operator.h" #include "commands/copy.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/citus_clauses.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_execution_locks.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_plan_cache.h" #include "distributed/merge_executor.h" #include "distributed/merge_planner.h" #include "distributed/multi_executor.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/shard_utils.h" #include "distributed/stats/query_stats.h" #include "distributed/stats/stat_counters.h" #include "distributed/subplan_execution.h" #include "distributed/worker_log_messages.h" #include "distributed/worker_protocol.h" extern AllowedDistributionColumn AllowedDistributionColumnValue; /* functions for creating custom scan nodes */ static Node * AdaptiveExecutorCreateScan(CustomScan *scan); static Node * NonPushableInsertSelectCreateScan(CustomScan *scan); static Node * DelayedErrorCreateScan(CustomScan *scan); static Node * NonPushableMergeCommandCreateScan(CustomScan *scan); /* functions that are common to different scans */ static void CitusBeginScan(CustomScanState *node, EState *estate, int eflags); static void CitusBeginReadOnlyScan(CustomScanState *node, EState *estate, int eflags); static void CitusBeginModifyScan(CustomScanState *node, EState *estate, int eflags); static void CitusPreExecScan(CitusScanState *scanState); static bool ModifyJobNeedsEvaluation(Job *workerJob); static void RegenerateTaskForFasthPathQuery(Job *workerJob); static void RegenerateTaskListForInsert(Job *workerJob); static DistributedPlan * CopyDistributedPlanWithoutCache(DistributedPlan * originalDistributedPlan); static void CitusEndScan(CustomScanState *node); static void CitusReScan(CustomScanState *node); static void EnsureForceDelegationDistributionKey(Job *job); static void EnsureAnchorShardsInJobExist(Job *job); static bool AnchorShardsInTaskListExist(List *taskList); static void TryToRerouteFastPathModifyQuery(Job *job); static void CheckQueryDeparseSafety(Query *query); /* create custom scan methods for all executors */ CustomScanMethods AdaptiveExecutorCustomScanMethods = { "Citus Adaptive", AdaptiveExecutorCreateScan }; CustomScanMethods NonPushableInsertSelectCustomScanMethods = { "Citus INSERT ... SELECT", NonPushableInsertSelectCreateScan }; CustomScanMethods DelayedErrorCustomScanMethods = { "Citus Delayed Error", DelayedErrorCreateScan }; CustomScanMethods NonPushableMergeCommandCustomScanMethods = { "Citus MERGE INTO ...", NonPushableMergeCommandCreateScan }; /* * Define executor methods for the different executor types. */ static CustomExecMethods AdaptiveExecutorCustomExecMethods = { .CustomName = "AdaptiveExecutorScan", .BeginCustomScan = CitusBeginScan, .ExecCustomScan = CitusExecScan, .EndCustomScan = CitusEndScan, .ReScanCustomScan = CitusReScan, .ExplainCustomScan = CitusExplainScan }; static CustomExecMethods NonPushableInsertSelectCustomExecMethods = { .CustomName = "NonPushableInsertSelectScan", .BeginCustomScan = CitusBeginScan, .ExecCustomScan = NonPushableInsertSelectExecScan, .EndCustomScan = CitusEndScan, .ReScanCustomScan = CitusReScan, .ExplainCustomScan = NonPushableInsertSelectExplainScan }; static CustomExecMethods NonPushableMergeCommandCustomExecMethods = { .CustomName = "NonPushableMergeCommandScan", .BeginCustomScan = CitusBeginScan, .ExecCustomScan = NonPushableMergeCommandExecScan, .EndCustomScan = CitusEndScan, .ReScanCustomScan = CitusReScan, .ExplainCustomScan = NonPushableMergeCommandExplainScan }; /* * IsCitusCustomState returns if a given PlanState node is a CitusCustomState node. */ bool IsCitusCustomState(PlanState *planState) { if (!IsA(planState, CustomScanState)) { return false; } CustomScanState *css = castNode(CustomScanState, planState); if (css->methods == &AdaptiveExecutorCustomExecMethods || css->methods == &NonPushableInsertSelectCustomExecMethods || css->methods == &NonPushableMergeCommandCustomExecMethods) { return true; } return false; } /* * Let PostgreSQL know about Citus' custom scan nodes. */ void RegisterCitusCustomScanMethods(void) { RegisterCustomScanMethods(&AdaptiveExecutorCustomScanMethods); RegisterCustomScanMethods(&NonPushableInsertSelectCustomScanMethods); RegisterCustomScanMethods(&DelayedErrorCustomScanMethods); RegisterCustomScanMethods(&NonPushableMergeCommandCustomScanMethods); } /* * CitusBeginScan sets the coordinator backend initiated by Citus for queries using * that function as the BeginCustomScan callback. * * The function also handles deferred shard pruning along with function evaluations. */ static void CitusBeginScan(CustomScanState *node, EState *estate, int eflags) { CitusScanState *scanState = (CitusScanState *) node; /* * Make sure we can see notices during regular queries, which would typically * be the result of a function that raises a notices being called. */ EnableWorkerMessagePropagation(); /* * Since we are using a tuplestore we cannot use the virtual tuples postgres had * already setup on the CustomScan. Instead we need to reinitialize the tuples as * minimal. * * During initialization postgres also created the projection information and the * quals, but both are 'compiled' to be executed on virtual tuples. Since we replaced * the tuples with minimal tuples we also compile both the projection and the quals * on to these 'new' tuples. */ ExecInitResultSlot(&scanState->customScanState.ss.ps, &TTSOpsMinimalTuple); ExecInitScanTupleSlot(node->ss.ps.state, &node->ss, node->ss.ps.scandesc, &TTSOpsMinimalTuple); ExecAssignScanProjectionInfoWithVarno(&node->ss, INDEX_VAR); node->ss.ps.qual = ExecInitQual(node->ss.ps.plan->qual, (PlanState *) node); DistributedPlan *distributedPlan = scanState->distributedPlan; if (distributedPlan->modifyQueryViaCoordinatorOrRepartition != NULL) { /* * INSERT..SELECT / MERGE via coordinator or re-partitioning are special because * the SELECT part is planned separately. */ return; } else if (distributedPlan->modLevel == ROW_MODIFY_READONLY) { CitusBeginReadOnlyScan(node, estate, eflags); } else { CitusBeginModifyScan(node, estate, eflags); } /* * If there is force_delgation functions' distribution argument set, * enforce it */ if (AllowedDistributionColumnValue.isActive) { Job *workerJob = scanState->distributedPlan->workerJob; EnsureForceDelegationDistributionKey(workerJob); } /* * In case of a prepared statement, we will see this distributed plan again * on the next execution with a higher usage counter. */ distributedPlan->numberOfTimesExecuted++; } /* * CitusPreExecScan is called right before postgres' executor starts pulling tuples. */ static void CitusPreExecScan(CitusScanState *scanState) { AdaptiveExecutorPreExecutorRun(scanState); } /* * CitusExecScan is called when a tuple is pulled from a custom scan. * On the first call, it executes the distributed query and writes the * results to a tuple store. The postgres executor calls this function * repeatedly to read tuples from the tuple store. */ TupleTableSlot * CitusExecScan(CustomScanState *node) { CitusScanState *scanState = (CitusScanState *) node; if (!scanState->finishedRemoteScan) { bool isMultiTaskPlan = IsMultiTaskPlan(scanState->distributedPlan); AdaptiveExecutor(scanState); if (isMultiTaskPlan) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } scanState->finishedRemoteScan = true; } return ReturnTupleFromTuplestore(scanState); } /* * CitusBeginReadOnlyScan handles deferred pruning and plan caching for SELECTs. */ static void CitusBeginReadOnlyScan(CustomScanState *node, EState *estate, int eflags) { CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *originalDistributedPlan = scanState->distributedPlan; Assert(originalDistributedPlan->workerJob->jobQuery->commandType == CMD_SELECT); if (!originalDistributedPlan->workerJob->deferredPruning) { /* * For SELECT queries that have already been pruned we can proceed straight * to execution, since none of the prepared statement logic applies. */ return; } /* * Create a copy of the generic plan for the current execution, but make a shallow * copy of the plan cache. That means we'll be able to access the plan cache via * currentPlan->workerJob->localPlannedStatements, but it will be preserved across * executions by the prepared statement logic. */ DistributedPlan *currentPlan = CopyDistributedPlanWithoutCache(originalDistributedPlan); scanState->distributedPlan = currentPlan; Job *workerJob = currentPlan->workerJob; Query *jobQuery = workerJob->jobQuery; PlanState *planState = &(scanState->customScanState.ss.ps); /* * We only do deferred pruning for fast path queries, which have a single * partition column value. */ Assert(currentPlan->fastPathRouterPlan || !EnableFastPathRouterPlanner); /* * Evaluate parameters, because the parameters are only available on the * coordinator and are required for pruning. * * We don't evaluate functions for read-only queries on the coordinator * at the moment. Most function calls would be in a context where they * should be re-evaluated for every row in case of volatile functions. * * TODO: evaluate stable functions */ ExecuteCoordinatorEvaluableExpressions(jobQuery, planState); /* job query no longer has parameters, so we should not send any */ workerJob->parametersInJobQueryResolved = true; /* parameters are filled in, so we can generate a task for this execution */ RegenerateTaskForFasthPathQuery(workerJob); if (IsLocalPlanCachingSupported(workerJob, originalDistributedPlan)) { Task *task = linitial(workerJob->taskList); /* * We are going to execute this task locally. If it's not already in * the cache, create a local plan now and add it to the cache. During * execution, we will get the plan from the cache. * * The plan will be cached across executions when originalDistributedPlan * represents a prepared statement. */ CacheLocalPlanForShardQuery(task, originalDistributedPlan, estate->es_param_list_info); } } /* * CitusBeginModifyScan prepares the scan state for a modification. * * Modifications are special because: * a) we evaluate function calls (e.g. nextval) here and the outcome may * determine which shards are affected by this query. * b) we need to take metadata locks to make sure no write is left behind * when finalizing a shard move. */ static void CitusBeginModifyScan(CustomScanState *node, EState *estate, int eflags) { CitusScanState *scanState = (CitusScanState *) node; PlanState *planState = &(scanState->customScanState.ss.ps); DistributedPlan *originalDistributedPlan = scanState->distributedPlan; MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CitusBeginModifyScan", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); DistributedPlan *currentPlan = CopyDistributedPlanWithoutCache(originalDistributedPlan); scanState->distributedPlan = currentPlan; Job *workerJob = currentPlan->workerJob; Query *jobQuery = workerJob->jobQuery; if (ModifyJobNeedsEvaluation(workerJob)) { ExecuteCoordinatorEvaluableExpressions(jobQuery, planState); /* job query no longer has parameters, so we should not send any */ workerJob->parametersInJobQueryResolved = true; } if (workerJob->deferredPruning) { /* * At this point, we're about to do the shard pruning for fast-path queries. * Given that pruning is deferred always for INSERTs, we get here * !EnableFastPathRouterPlanner as well. Given that INSERT statements with * CTEs/sublinks etc are not eligible for fast-path router plan, we get here * jobQuery->commandType == CMD_INSERT as well. */ Assert(currentPlan->fastPathRouterPlan || !EnableFastPathRouterPlanner || jobQuery->commandType == CMD_INSERT); /* * We can only now decide which shard to use, so we need to build a new task * list. */ if (jobQuery->commandType == CMD_INSERT) { RegenerateTaskListForInsert(workerJob); } else { RegenerateTaskForFasthPathQuery(workerJob); } } else if (workerJob->requiresCoordinatorEvaluation) { /* * When there is no deferred pruning, but we did evaluate functions, then * we only rebuild the query strings in the existing tasks. * * Note that because we did evaluate functions we should also * sanity check the query first, to prevent issues like #8198 * where 'false IN (SELECT ..)' is constant folded to 'NOT * (SELECT ..)' triggering an assert in ruleutils. */ CheckQueryDeparseSafety(workerJob->jobQuery); RebuildQueryStrings(workerJob); } /* We skip shard related things if the job contains only local tables */ if (!ModifyLocalTableJob(workerJob)) { /* * Now that we know the shard ID(s) we can acquire the necessary shard metadata * locks. Once we have the locks it's safe to load the placement metadata. */ /* prevent concurrent placement changes */ AcquireMetadataLocks(workerJob->taskList); /* * In case of a split, the shard might no longer be available. In that * case try to reroute. We can only do this for fast path queries. */ if (currentPlan->fastPathRouterPlan && !AnchorShardsInTaskListExist(workerJob->taskList)) { TryToRerouteFastPathModifyQuery(workerJob); } /* ensure there is no invalid shard */ EnsureAnchorShardsInJobExist(workerJob); /* modify tasks are always assigned using first-replica policy */ workerJob->taskList = FirstReplicaAssignTaskList(workerJob->taskList); } /* * Now that we have populated the task placements we can determine whether * any of them are local to this node and cache a plan if needed. */ if (IsLocalPlanCachingSupported(workerJob, originalDistributedPlan)) { Task *task = linitial(workerJob->taskList); /* * We are going to execute this task locally. If it's not already in * the cache, create a local plan now and add it to the cache. During * execution, we will get the plan from the cache. * * WARNING: In this function we'll use the original plan with the original * query tree, meaning parameters and function calls are back and we'll * redo evaluation in the local (Postgres) executor. The reason we do this * is that we only need to cache one generic plan per shard. * * The plan will be cached across executions when originalDistributedPlan * represents a prepared statement. */ CacheLocalPlanForShardQuery(task, originalDistributedPlan, estate->es_param_list_info); } MemoryContextSwitchTo(oldContext); } /* * TryToRerouteFastPathModifyQuery tries to reroute non-existent shards in given job if it finds any such shard, * only for fastpath queries. * * Should only be called if the job belongs to a fastpath modify query */ static void TryToRerouteFastPathModifyQuery(Job *job) { if (job->jobQuery->commandType == CMD_INSERT) { RegenerateTaskListForInsert(job); } else { RegenerateTaskForFasthPathQuery(job); RebuildQueryStrings(job); } } /* * EnsureAnchorShardsInJobExist ensures all shards are valid in job. * If it finds a non-existent shard in given job, it throws an error. */ static void EnsureAnchorShardsInJobExist(Job *job) { if (!AnchorShardsInTaskListExist(job->taskList)) { ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("shard for the given value does not exist"), errdetail( "A concurrent shard split may have moved the data into a new set of shards."), errhint("Retry the query."))); } } /* * AnchorShardsInTaskListExist checks whether all the anchor shards in the task list * still exist. */ static bool AnchorShardsInTaskListExist(List *taskList) { Task *task = NULL; foreach_declared_ptr(task, taskList) { if (!ShardExists(task->anchorShardId)) { return false; } } return true; } /* * ModifyJobNeedsEvaluation checks whether the functions and parameters in the job query * need to be evaluated before we can build task query strings. */ static bool ModifyJobNeedsEvaluation(Job *workerJob) { if (workerJob->requiresCoordinatorEvaluation) { /* query contains functions that need to be evaluated on the coordinator */ return true; } if (workerJob->partitionKeyValue != NULL) { /* the value of the distribution column is already known */ return false; } /* pruning was deferred due to a parameter in the partition column */ return workerJob->deferredPruning; } /* * CopyDistributedPlanWithoutCache is a helper function which copies the * distributedPlan into the current memory context. * * We must not change the distributed plan since it may be reused across multiple * executions of a prepared statement. Instead we create a deep copy that we only * use for the current execution. * * We also exclude localPlannedStatements from the copyObject call for performance * reasons, as they are immutable, so no need to have a deep copy. */ static DistributedPlan * CopyDistributedPlanWithoutCache(DistributedPlan *originalDistributedPlan) { List *localPlannedStatements = originalDistributedPlan->workerJob->localPlannedStatements; originalDistributedPlan->workerJob->localPlannedStatements = NIL; DistributedPlan *distributedPlan = copyObject(originalDistributedPlan); /* set back the immutable field */ originalDistributedPlan->workerJob->localPlannedStatements = localPlannedStatements; distributedPlan->workerJob->localPlannedStatements = localPlannedStatements; return distributedPlan; } /* * RegenerateTaskListForInsert does the shard pruning for an INSERT query * queries and rebuilds the query strings. */ static void RegenerateTaskListForInsert(Job *workerJob) { Query *jobQuery = workerJob->jobQuery; bool parametersInJobQueryResolved = workerJob->parametersInJobQueryResolved; DeferredErrorMessage *planningError = NULL; /* need to perform shard pruning, rebuild the task list from scratch */ List *taskList = RouterInsertTaskList(jobQuery, parametersInJobQueryResolved, &planningError); if (planningError != NULL) { RaiseDeferredError(planningError, ERROR); } workerJob->taskList = taskList; if (workerJob->partitionKeyValue == NULL) { /* * If we were not able to determine the partition key value in the planner, * take another shot now. It may still be NULL in case of a multi-row * insert. */ workerJob->partitionKeyValue = ExtractInsertPartitionKeyValue(jobQuery); } RebuildQueryStrings(workerJob); } /* * RegenerateTaskForFasthPathQuery does the shard pruning for * UPDATE/DELETE/SELECT fast path router queries and rebuilds the query strings. */ static void RegenerateTaskForFasthPathQuery(Job *workerJob) { bool isMultiShardQuery = false; List *shardIntervalList = TargetShardIntervalForFastPathQuery(workerJob->jobQuery, &isMultiShardQuery, NULL, &workerJob->partitionKeyValue); /* * A fast-path router query can only yield multiple shards when the parameter * cannot be resolved properly, which can be triggered by SQL function. */ if (isMultiShardQuery) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning on this " "query because parameterized queries for SQL " "functions referencing distributed tables are " "not supported"), errhint("Consider using PL/pgSQL functions instead."))); } bool shardsPresent = false; List *relationShardList = RelationShardListForShardIntervalList(shardIntervalList, &shardsPresent); UpdateRelationToShardNames((Node *) workerJob->jobQuery, relationShardList); /* fast path queries cannot have local tables */ bool hasLocalRelation = false; List *placementList = CreateTaskPlacementListForShardIntervals(shardIntervalList, shardsPresent, true, hasLocalRelation); uint64 shardId = INVALID_SHARD_ID; if (shardsPresent) { shardId = GetAnchorShardId(shardIntervalList); } bool isLocalTableModification = false; bool delayedFastPath = false; GenerateSingleShardRouterTaskList(workerJob, relationShardList, placementList, shardId, isLocalTableModification, delayedFastPath); } /* * AdaptiveExecutorCreateScan creates the scan state for the adaptive executor. */ static Node * AdaptiveExecutorCreateScan(CustomScan *scan) { CitusScanState *scanState = palloc0(sizeof(CitusScanState)); scanState->executorType = MULTI_EXECUTOR_ADAPTIVE; scanState->customScanState.ss.ps.type = T_CustomScanState; scanState->distributedPlan = GetDistributedPlan(scan); scanState->customScanState.methods = &AdaptiveExecutorCustomExecMethods; scanState->PreExecScan = &CitusPreExecScan; scanState->finishedPreScan = false; scanState->finishedRemoteScan = false; return (Node *) scanState; } /* * NonPushableInsertSelectCrateScan creates the scan state for executing * INSERT..SELECT into a distributed table via the coordinator. */ static Node * NonPushableInsertSelectCreateScan(CustomScan *scan) { CitusScanState *scanState = palloc0(sizeof(CitusScanState)); scanState->executorType = MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT; scanState->customScanState.ss.ps.type = T_CustomScanState; scanState->distributedPlan = GetDistributedPlan(scan); scanState->customScanState.methods = &NonPushableInsertSelectCustomExecMethods; scanState->finishedPreScan = false; scanState->finishedRemoteScan = false; return (Node *) scanState; } /* * DelayedErrorCreateScan is only called if we could not plan for the given * query. This is the case when a plan is not ready for execution because * CreateDistributedPlan() couldn't find a plan due to unresolved prepared * statement parameters, but didn't error out, because we expect custom plans * to come to our rescue. But sql (not plpgsql) functions unfortunately don't * go through a codepath supporting custom plans. Here, we error out with this * delayed error message. */ static Node * DelayedErrorCreateScan(CustomScan *scan) { DistributedPlan *distributedPlan = GetDistributedPlan(scan); /* raise the deferred error */ RaiseDeferredError(distributedPlan->planningError, ERROR); return NULL; } /* * NonPushableMergeCommandCreateScan creates the scan state for executing * MERGE INTO ... into a distributed table with repartition of source rows. */ static Node * NonPushableMergeCommandCreateScan(CustomScan *scan) { CitusScanState *scanState = palloc0(sizeof(CitusScanState)); scanState->executorType = MULTI_EXECUTOR_NON_PUSHABLE_MERGE_QUERY; scanState->customScanState.ss.ps.type = T_CustomScanState; scanState->distributedPlan = GetDistributedPlan(scan); scanState->customScanState.methods = &NonPushableMergeCommandCustomExecMethods; scanState->finishedPreScan = false; scanState->finishedRemoteScan = false; return (Node *) scanState; } /* * CitusEndScan is used to clean up tuple store of the given custom scan state. */ static void CitusEndScan(CustomScanState *node) { CitusScanState *scanState = (CitusScanState *) node; Job *workerJob = scanState->distributedPlan->workerJob; uint64 queryId = scanState->distributedPlan->queryId; MultiExecutorType executorType = scanState->executorType; Const *partitionKeyConst = NULL; char *partitionKeyString = NULL; /* stop propagating notices */ DisableWorkerMessagePropagation(); /* * Check whether we received warnings that should not have been * ignored. */ ErrorIfWorkerErrorIndicationReceived(); if (workerJob != NULL) { partitionKeyConst = workerJob->partitionKeyValue; } /* * queryId is not set if pg_stat_statements is not installed, * it can be set with as of pg14: set compute_query_id to on; */ if (queryId != 0) { if (partitionKeyConst != NULL && executorType == MULTI_EXECUTOR_ADAPTIVE) { partitionKeyString = DatumToString(partitionKeyConst->constvalue, partitionKeyConst->consttype); } /* queries without partition key are also recorded */ CitusQueryStatsExecutorsEntry(queryId, executorType, partitionKeyString); } if (scanState->tuplestorestate) { tuplestore_end(scanState->tuplestorestate); scanState->tuplestorestate = NULL; } } /* * CitusReScan is not normally called, except in certain cases of * DECLARE .. CURSOR WITH HOLD .. */ static void CitusReScan(CustomScanState *node) { if (node->ss.ps.ps_ResultTupleSlot) { ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); } ExecScanReScan(&node->ss); CitusScanState *scanState = (CitusScanState *) node; if (scanState->tuplestorestate) { tuplestore_rescan(scanState->tuplestorestate); } } /* * ScanStateGetTupleDescriptor returns the tuple descriptor for the given * scan state. */ TupleDesc ScanStateGetTupleDescriptor(CitusScanState *scanState) { return scanState->customScanState.ss.ss_ScanTupleSlot->tts_tupleDescriptor; } /* * ScanStateGetExecutorState returns the executor state for the given scan * state. */ EState * ScanStateGetExecutorState(CitusScanState *scanState) { return scanState->customScanState.ss.ps.state; } /* * FetchCitusCustomScanIfExists traverses a given plan and returns a Citus CustomScan * if it has any. */ CustomScan * FetchCitusCustomScanIfExists(Plan *plan) { if (plan == NULL) { return NULL; } if (IsCitusCustomScan(plan)) { return (CustomScan *) plan; } CustomScan *customScan = FetchCitusCustomScanIfExists(plan->lefttree); if (customScan == NULL) { customScan = FetchCitusCustomScanIfExists(plan->righttree); } return customScan; } /* * IsCitusPlan returns whether a Plan contains a CustomScan generated by Citus * by recursively walking through the plan tree. */ bool IsCitusPlan(Plan *plan) { if (plan == NULL) { return false; } if (IsCitusCustomScan(plan)) { return true; } return IsCitusPlan(plan->lefttree) || IsCitusPlan(plan->righttree); } /* * IsCitusCustomScan returns whether Plan node is a CustomScan generated by Citus. */ bool IsCitusCustomScan(Plan *plan) { if (plan == NULL) { return false; } if (!IsA(plan, CustomScan)) { return false; } CustomScan *customScan = (CustomScan *) plan; if (list_length(customScan->custom_private) == 0) { return false; } Node *privateNode = (Node *) linitial(customScan->custom_private); if (!CitusIsA(privateNode, DistributedPlan)) { return false; } return true; } /* * In a Job, given a list of relations, if all them belong to the same * colocation group, the Job's colocation ID is set to the group ID, else, * it will be set to INVALID_COLOCATION_ID. */ void SetJobColocationId(Job *job) { uint32 jobColocationId = INVALID_COLOCATION_ID; List *rangeTableList = ExtractRangeTableEntryList(job->jobQuery); ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); Oid relationId = rangeTableEntry->relid; if (!IsCitusTable(relationId)) { /* ignore the non distributed table */ continue; } uint32 colocationId = TableColocationId(relationId); if (jobColocationId == INVALID_COLOCATION_ID) { /* Initialize the ID */ jobColocationId = colocationId; } else if (jobColocationId != colocationId) { /* Tables' colocationId is not the same */ jobColocationId = INVALID_COLOCATION_ID; break; } } job->colocationId = jobColocationId; } /* * Any function with force_delegate flag(true) must ensure that the Job's * partition key match with the functions' distribution argument. */ static void EnsureForceDelegationDistributionKey(Job *job) { /* If the Job has the subquery, punt the shard-key-check to the subquery */ if (job->subqueryPushdown) { return; } /* * If the query doesn't have shard key, nothing to check, only exception is when * the query doesn't have distributed tables but an RTE with intermediate_results * function (a subquery plan). */ if (!job->partitionKeyValue) { bool queryContainsDistributedTable = FindNodeMatchingCheckFunction((Node *) job->jobQuery, IsDistributedTableRTE); if (!queryContainsDistributedTable) { return; } } /* We should match both the key and the colocation ID */ SetJobColocationId(job); if (!IsShardKeyValueAllowed(job->partitionKeyValue, job->colocationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "queries must filter by the distribution argument in the same " "colocation group when using the forced function pushdown"), errhint( "consider disabling forced delegation through " "create_distributed_table(..., force_delegation := false)"))); } } /* * CheckDeparseWalker is used to walk over an expression tree and ensure * that any SubLink testexpr is safe to deparse after coordinator-side * evaluation. * * Specifically, we convert any non-OpExpr testexpr into an OpExpr of the * form TRUE = testexpr, and NOT expressions into FALSE = arg. If the deparser * sees something other than an OpExpr in a SubLink testexpr, it will * raise an assertion failure or error out. */ static bool CheckDeparseWalker(Node *expr, void *context) { if (expr == NULL) { return false; } if (IsA(expr, SubLink)) { SubLink *sublink = (SubLink *) expr; Node *testexpr = sublink->testexpr; if (testexpr && IsA(testexpr, BoolExpr) && ((BoolExpr *) testexpr)->boolop == NOT_EXPR) { Node *arg = linitial(((BoolExpr *) testexpr)->args); /* * testexpr is of the form: NOT (arg) * convert NOT (arg) to FALSE = arg */ sublink->testexpr = (Node *) make_opclause(BooleanEqualOperator, BOOLOID, false, (Expr *) makeBoolConst(false, false), (Expr *) arg, InvalidOid, InvalidOid); } else if (testexpr && !IsA(testexpr, OpExpr)) { /* * testexpr is something other than an OpExpr * convert testexpr to TRUE = testexpr */ sublink->testexpr = (Node *) make_opclause(BooleanEqualOperator, BOOLOID, false, (Expr *) makeBoolConst(true, false), (Expr *) testexpr, InvalidOid, InvalidOid); } } return expression_tree_walker(expr, CheckDeparseWalker, NULL); } /* * CheckQueryDeparseSafety is used by CitusBeginModifyScan to ensure that the * query's quals are safe to deparse after coordinator-required evaluation; * ExecuteCoordinatorEvaluableExpressions() may have reduced or constant-folded * expressions in a way that could trigger asserts in the deparser. * * Note that this is only needed for UPDATE and DELTE queries, since INSERTs always * use deferred pruning. So we just need to check the query's quals. */ static void CheckQueryDeparseSafety(Query *query) { CheckDeparseWalker(query->jointree->quals, NULL); } ================================================ FILE: src/backend/distributed/executor/directed_acyclic_graph_execution.c ================================================ /*------------------------------------------------------------------------- * * directed_acyclic_graph_execution_logic.c * * Logic to run tasks in their dependency order. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "access/hash.h" #include "distributed/adaptive_executor.h" #include "distributed/directed_acyclic_graph_execution.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/transaction_management.h" #include "distributed/transmit.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" typedef struct TaskHashKey { uint64 jobId; uint32 taskId; /* * The padding field is needed to make sure the struct contains no * automatic padding, which is not allowed for hashmap keys. */ uint32 padding; }TaskHashKey; typedef struct TaskHashEntry { TaskHashKey key; Task *task; }TaskHashEntry; static bool IsAllDependencyCompleted(Task *task, HTAB *completedTasks); static void AddCompletedTasks(List *curCompletedTasks, HTAB *completedTasks); static List * FindExecutableTasks(List *allTasks, HTAB *completedTasks); static List * RemoveMergeTasks(List *taskList); static bool IsTaskAlreadyCompleted(Task *task, HTAB *completedTasks); /* * ExecuteTasksInDependencyOrder executes the given tasks except the excluded * tasks in their dependency order. To do so, it iterates all * the tasks and finds the ones that can be executed at that time, it tries to * execute all of them in parallel. The parallelism is bound by MaxAdaptiveExecutorPoolSize. */ void ExecuteTasksInDependencyOrder(List *allTasks, List *excludedTasks, List *jobIds) { assert_valid_hash_key3(TaskHashKey, jobId, taskId, padding); HTAB *completedTasks = CreateSimpleHash(TaskHashKey, TaskHashEntry); /* We only execute depended jobs' tasks, therefore to not execute */ /* top level tasks, we add them to the completedTasks. */ AddCompletedTasks(excludedTasks, completedTasks); while (true) { List *curTasks = FindExecutableTasks(allTasks, completedTasks); if (list_length(curTasks) == 0) { break; } /* merge tasks do not need to be executed */ List *executableTasks = RemoveMergeTasks(curTasks); if (list_length(executableTasks) > 0) { ExecuteTaskList(ROW_MODIFY_NONE, executableTasks); } AddCompletedTasks(curTasks, completedTasks); curTasks = NIL; } } /* * FindExecutableTasks finds the tasks that can be executed currently, * which means that all of their dependencies are executed. If a task * is already executed, it is not added to the result. */ static List * FindExecutableTasks(List *allTasks, HTAB *completedTasks) { List *curTasks = NIL; Task *task = NULL; foreach_declared_ptr(task, allTasks) { if (IsAllDependencyCompleted(task, completedTasks) && !IsTaskAlreadyCompleted(task, completedTasks)) { curTasks = lappend(curTasks, task); } } return curTasks; } /* * RemoveMergeTasks returns a copy of taskList that excludes all the * merge tasks. We do this because merge tasks are currently only a * logical concept that does not need to be executed. */ static List * RemoveMergeTasks(List *taskList) { List *prunedTaskList = NIL; Task *task = NULL; foreach_declared_ptr(task, taskList) { if (task->taskType != MERGE_TASK) { prunedTaskList = lappend(prunedTaskList, task); } } return prunedTaskList; } /* * AddCompletedTasks adds the givens tasks to completedTasks HTAB. */ static void AddCompletedTasks(List *curCompletedTasks, HTAB *completedTasks) { bool found; Task *task = NULL; foreach_declared_ptr(task, curCompletedTasks) { TaskHashKey taskKey = { task->jobId, task->taskId }; hash_search(completedTasks, &taskKey, HASH_ENTER, &found); } } /* * IsTaskAlreadyCompleted returns true if the given task * is found in the completedTasks HTAB. */ static bool IsTaskAlreadyCompleted(Task *task, HTAB *completedTasks) { bool found; TaskHashKey taskKey = { task->jobId, task->taskId }; hash_search(completedTasks, &taskKey, HASH_ENTER, &found); return found; } /* * IsAllDependencyCompleted return true if the given task's * dependencies are completed. */ static bool IsAllDependencyCompleted(Task *targetTask, HTAB *completedTasks) { bool found = false; Task *task = NULL; foreach_declared_ptr(task, targetTask->dependentTaskList) { TaskHashKey taskKey = { task->jobId, task->taskId }; hash_search(completedTasks, &taskKey, HASH_FIND, &found); if (!found) { return false; } } return true; } ================================================ FILE: src/backend/distributed/executor/distributed_execution_locks.c ================================================ /*------------------------------------------------------------------------- * * distributed_execution_locks.c * * Definitions of the functions used in executing distributed * execution locking. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "distributed/coordinator_protocol.h" #include "distributed/distributed_execution_locks.h" #include "distributed/executor_util.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_partition.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" /* * AcquireExecutorShardLocksForExecution acquires advisory lock on shard IDs * to prevent unsafe concurrent modifications of shards. * * We prevent concurrent modifications of shards in two cases: * 1. Any non-commutative writes to a replicated table * 2. Multi-shard writes that are executed in parallel * * The first case ensures we do not apply updates in different orders on * different replicas (e.g. of a reference table), which could lead the * replicas to diverge. * * The second case prevents deadlocks due to out-of-order execution. * * There are two GUCs that can override the default behaviors. * 'citus.all_modifications_commutative' relaxes locking * that's done for the purpose of keeping replicas consistent. * 'citus.enable_deadlock_prevention' relaxes locking done for * the purpose of avoiding deadlocks between concurrent * multi-shard commands. * * We do not take executor shard locks for utility commands such as * TRUNCATE because the table locks already prevent concurrent access. */ void AcquireExecutorShardLocksForExecution(RowModifyLevel modLevel, List *taskList) { if (modLevel <= ROW_MODIFY_READONLY && !SelectForUpdateOnReferenceTable(taskList)) { /* * Executor locks only apply to DML commands and SELECT FOR UPDATE queries * touching reference tables. */ return; } bool requiresParallelExecutionLocks = !(list_length(taskList) == 1 || ShouldRunTasksSequentially(taskList)); bool modifiedTableReplicated = ModifiedTableReplicated(taskList); if (!modifiedTableReplicated && !requiresParallelExecutionLocks) { /* * When a distributed query on tables with replication * factor == 1 and command hits only a single shard, we * rely on Postgres to handle the serialization of the * concurrent modifications on the workers. * * For reference tables, even if their placements are replicated * ones (e.g., single node), we acquire the distributed execution * locks to be consistent when new node(s) are added. So, they * do not return at this point. */ return; } /* * We first assume that all the remaining modifications are going to * be serialized. So, start with an ExclusiveLock and lower the lock level * as much as possible. */ int lockMode = ExclusiveLock; /* * In addition to honouring commutativity rules, we currently only * allow a single multi-shard command on a shard at a time. Otherwise, * concurrent multi-shard commands may take row-level locks on the * shard placements in a different order and create a distributed * deadlock. This applies even when writes are commutative and/or * there is no replication. This can be relaxed via * EnableDeadlockPrevention. * * 1. If citus.all_modifications_commutative is set to true, then all locks * are acquired as RowExclusiveLock. * * 2. If citus.all_modifications_commutative is false, then only the shards * with more than one replicas are locked with ExclusiveLock. Otherwise, the * lock is acquired with ShareUpdateExclusiveLock. * * ShareUpdateExclusiveLock conflicts with itself such that only one * multi-shard modification at a time is allowed on a shard. It also conflicts * with ExclusiveLock, which ensures that updates/deletes/upserts are applied * in the same order on all placements. It does not conflict with * RowExclusiveLock, which is normally obtained by single-shard, commutative * writes. */ if (!modifiedTableReplicated && requiresParallelExecutionLocks) { /* * When there is no replication then we only need to prevent * concurrent multi-shard commands on the same shards. This is * because concurrent, parallel commands may modify the same * set of shards, but in different orders. The order of the * accesses might trigger distributed deadlocks that are not * possible to happen on non-distributed systems such * regular Postgres. * * As an example, assume that we have two queries: query-1 and query-2. * Both queries access shard-1 and shard-2. If query-1 first accesses to * shard-1 then shard-2, and query-2 accesses shard-2 then shard-1, these * two commands might block each other in case they modify the same rows * (e.g., cause distributed deadlocks). * * In either case, ShareUpdateExclusive has the desired effect, since * it conflicts with itself and ExclusiveLock (taken by non-commutative * writes). * * However, some users find this too restrictive, so we allow them to * reduce to a RowExclusiveLock when citus.enable_deadlock_prevention * is enabled, which lets multi-shard modifications run in parallel as * long as they all disable the GUC. */ lockMode = EnableDeadlockPrevention ? ShareUpdateExclusiveLock : RowExclusiveLock; if (!IsCoordinator()) { /* * We also skip taking a heavy-weight lock when running a multi-shard * commands from workers, since we currently do not prevent concurrency * across workers anyway. */ lockMode = RowExclusiveLock; } } else if (modifiedTableReplicated) { /* * When we are executing distributed queries on replicated tables, our * default behaviour is to prevent any concurrency. This is valid * for when parallel execution is happening or not. * * The reason is that we cannot control the order of the placement accesses * of two distributed queries to the same shards. The order of the accesses * might cause the replicas of the same shard placements diverge. This is * not possible to happen on non-distributed systems such regular Postgres. * * As an example, assume that we have two queries: query-1 and query-2. * Both queries only access the placements of shard-1, say p-1 and p-2. * * And, assume that these queries are non-commutative, such as: * query-1: UPDATE table SET b = 1 WHERE key = 1; * query-2: UPDATE table SET b = 2 WHERE key = 1; * * If query-1 accesses to p-1 then p-2, and query-2 accesses * p-2 then p-1, these two commands would leave the p-1 and p-2 * diverged (e.g., the values for the column "b" would be different). * * The only exception to this rule is the single shard commutative * modifications, such as INSERTs. In that case, we can allow * concurrency among such backends, hence lowering the lock level * to RowExclusiveLock. */ if (!requiresParallelExecutionLocks && modLevel < ROW_MODIFY_NONCOMMUTATIVE) { lockMode = RowExclusiveLock; } } if (AllModificationsCommutative) { /* * The mapping is overridden when all_modifications_commutative is set to true. * In that case, all modifications are treated as commutative, which can be used * to communicate that the application is only generating commutative * UPDATE/DELETE/UPSERT commands and exclusive locks are unnecessary. This * is irrespective of single-shard/multi-shard or replicated tables. */ lockMode = RowExclusiveLock; } /* now, iterate on the tasks and acquire the executor locks on the shards */ List *anchorShardIntervalList = NIL; List *relationRowLockList = NIL; List *requiresConsistentSnapshotRelationShardList = NIL; Task *task = NULL; foreach_declared_ptr(task, taskList) { ShardInterval *anchorShardInterval = LoadShardInterval(task->anchorShardId); anchorShardIntervalList = lappend(anchorShardIntervalList, anchorShardInterval); /* Acquire additional locks for SELECT .. FOR UPDATE on reference tables */ AcquireExecutorShardLocksForRelationRowLockList(task->relationRowLockList); relationRowLockList = list_concat(relationRowLockList, task->relationRowLockList); /* * If the task has a subselect, then we may need to lock the shards from which * the query selects as well to prevent the subselects from seeing different * results on different replicas. */ if (RequiresConsistentSnapshot(task)) { /* * ExclusiveLock conflicts with all lock types used by modifications * and therefore prevents other modifications from running * concurrently. */ requiresConsistentSnapshotRelationShardList = list_concat(requiresConsistentSnapshotRelationShardList, task->relationShardList); } } /* * Acquire the locks in a sorted way to avoid deadlocks due to lock * ordering across concurrent sessions. */ anchorShardIntervalList = SortList(anchorShardIntervalList, CompareShardIntervalsById); /* * If we are dealing with a partition we are also taking locks on parent table * to prevent deadlocks on concurrent operations on a partition and its parent. * * Note that this function currently does not acquire any remote locks as that * is necessary to control the concurrency across multiple nodes for replicated * tables. That is because Citus currently does not allow modifications to * partitions from any node other than the coordinator. */ LockParentShardResourceIfPartition(anchorShardIntervalList, lockMode); /* Acquire distribution execution locks on the affected shards */ SerializeNonCommutativeWrites(anchorShardIntervalList, lockMode); if (relationRowLockList != NIL) { /* Acquire additional locks for SELECT .. FOR UPDATE on reference tables */ AcquireExecutorShardLocksForRelationRowLockList(relationRowLockList); } if (requiresConsistentSnapshotRelationShardList != NIL) { /* * If the task has a subselect, then we may need to lock the shards from which * the query selects as well to prevent the subselects from seeing different * results on different replicas. * * ExclusiveLock conflicts with all lock types used by modifications * and therefore prevents other modifications from running * concurrently. */ LockRelationShardResources(requiresConsistentSnapshotRelationShardList, ExclusiveLock); } } /* * RequiresConsistentSnapshot returns true if the given task need to take * the necessary locks to ensure that a subquery in the modify query * returns the same output for all task placements. */ bool RequiresConsistentSnapshot(Task *task) { bool requiresIsolation = false; if (!task->modifyWithSubquery) { /* * Other commands do not read from other shards. */ requiresIsolation = false; } else if (list_length(task->taskPlacementList) == 1) { /* * If there is only one replica then we fully rely on PostgreSQL to * provide SELECT isolation. In this case, we do not provide isolation * across the shards, but that was never our intention. */ requiresIsolation = false; } else if (AllModificationsCommutative) { /* * An INSERT/SELECT is commutative with other writes if it excludes * any ongoing writes based on the filter conditions. Without knowing * whether this is true, we assume the user took this into account * when enabling citus.all_modifications_commutative. This option * gives users an escape from aggressive locking during INSERT/SELECT. */ requiresIsolation = false; } else { /* * If this is a non-commutative write, then we need to block ongoing * writes to make sure that the subselect returns the same result * on all placements. */ requiresIsolation = true; } return requiresIsolation; } /* * AcquireMetadataLocks acquires metadata locks on each of the anchor * shards in the task list to prevent a shard being modified while it * is being copied. */ void AcquireMetadataLocks(List *taskList) { /* * Note: to avoid the overhead of additional sorting, we assume tasks * to be already sorted by shard ID such that deadlocks are avoided. * This is true for INSERT/SELECT, which is the only multi-shard * command right now. */ Task *task = NULL; foreach_declared_ptr(task, taskList) { LockShardDistributionMetadata(task->anchorShardId, ShareLock); } } void AcquireExecutorShardLocksForRelationRowLockList(List *relationRowLockList) { LOCKMODE rowLockMode = NoLock; if (relationRowLockList == NIL) { return; } /* * If lock clause exists and it affects any reference table, we need to get * lock on shard resource. Type of lock is determined by the type of row lock * given in the query. If the type of row lock is either FOR NO KEY UPDATE or * FOR UPDATE we get ExclusiveLock on shard resource. We get ShareLock if it * is FOR KEY SHARE or FOR KEY SHARE. * * We have selected these lock types according to conflict table given in the * Postgres documentation. It is given that FOR UPDATE and FOR NO KEY UPDATE * must be conflict with each other modify command. By getting ExlcusiveLock * we guarantee that. Note that, getting ExclusiveLock does not mimic the * behaviour of Postgres exactly. Getting row lock with FOR NO KEY UPDATE and * FOR KEY SHARE do not conflict in Postgres, yet they block each other in * our implementation. Since FOR SHARE and FOR KEY SHARE does not conflict * with each other but conflicts with modify commands, we get ShareLock for * them. */ RelationRowLock *relationRowLock = NULL; foreach_declared_ptr(relationRowLock, relationRowLockList) { LockClauseStrength rowLockStrength = relationRowLock->rowLockStrength; Oid relationId = relationRowLock->relationId; if (IsCitusTableType(relationId, REFERENCE_TABLE)) { List *shardIntervalList = LoadShardIntervalList(relationId); if (rowLockStrength == LCS_FORKEYSHARE || rowLockStrength == LCS_FORSHARE) { rowLockMode = ShareLock; } else if (rowLockStrength == LCS_FORNOKEYUPDATE || rowLockStrength == LCS_FORUPDATE) { rowLockMode = ExclusiveLock; } SerializeNonCommutativeWrites(shardIntervalList, rowLockMode); } } } /* * LockPartitionsInRelationList iterates over given list and acquires locks on * partitions of each partitioned table. It does nothing for non-partitioned tables. */ void LockPartitionsInRelationList(List *relationIdList, LOCKMODE lockmode) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (PartitionedTable(relationId)) { LockPartitionRelations(relationId, lockmode); } } } /* * LockPartitionRelations acquires relation lock on all partitions of given * partitioned relation. This function expects that given relation is a * partitioned relation. */ void LockPartitionRelations(Oid relationId, LOCKMODE lockMode) { /* * PartitionList function generates partition list in the same order * as PostgreSQL. Therefore we do not need to sort it before acquiring * locks. */ List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { LockRelationOid(partitionRelationId, lockMode); } } /* * LockPartitionsForDistributedPlan ensures commands take locks on all partitions * of a distributed table that appears in the query. We do this primarily out of * consistency with PostgreSQL locking. */ void LockPartitionsForDistributedPlan(DistributedPlan *plan) { if (TaskListModifiesDatabase(plan->modLevel, plan->workerJob->taskList)) { Oid targetRelationId = plan->targetRelationId; LockPartitionsInRelationList(list_make1_oid(targetRelationId), RowExclusiveLock); } /* * Lock partitions of tables that appear in a SELECT or subquery. In the * DML case this also includes the target relation, but since we already * have a stronger lock this doesn't do any harm. */ LockPartitionsInRelationList(plan->relationIdList, AccessShareLock); } ================================================ FILE: src/backend/distributed/executor/distributed_intermediate_results.c ================================================ /*------------------------------------------------------------------------- * * distributed_intermediate_results.c * Functions for reading and writing distributed intermediate results. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "port.h" #include "access/htup_details.h" #include "access/tupdesc.h" #include "catalog/pg_type.h" #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "pg_version_constants.h" #include "distributed/deparse_shard_query.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/transaction_management.h" #include "distributed/tuple_destination.h" #include "distributed/tuplestore.h" #include "distributed/worker_protocol.h" /* * PartitioningTupleDest is internal representation of a TupleDestination * which consumes queries constructed in WrapTasksForPartitioning. */ typedef struct PartitioningTupleDest { TupleDestination pub; CitusTableCacheEntry *targetRelation; /* MemoryContext in which we add new fragments */ MemoryContext fragmentContext; /* list of DistributedResultFragment pointer */ List *fragmentList; /* what do tuples look like */ TupleDesc tupleDesc; } PartitioningTupleDest; /* forward declarations of local functions */ static List * WrapTasksForPartitioning(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *targetRelation, bool binaryFormat); static List * ExecutePartitionTaskList(List *partitionTaskList, CitusTableCacheEntry *targetRelation); static PartitioningTupleDest * CreatePartitioningTupleDest(CitusTableCacheEntry * targetRelation); static void PartitioningTupleDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize); static TupleDesc PartitioningTupleDestTupleDescForQuery(TupleDestination *self, int queryNumber); static char * SourceShardPrefix(const char *resultPrefix, uint64 shardId); static DistributedResultFragment * TupleToDistributedResultFragment(HeapTuple heapTuple, TupleDesc tupleDesc, CitusTableCacheEntry * targetRelation, uint32 sourceNodeId); static void ExecuteSelectTasksIntoTupleDest(List *taskList, TupleDestination *tupleDestination, bool errorOnAnyFailure); static List ** ColocateFragmentsWithRelation(List *fragmentList, CitusTableCacheEntry *targetRelation); static List * ColocationTransfers(List *fragmentList, CitusTableCacheEntry *targetRelation); static List * FragmentTransferTaskList(List *fragmentListTransfers); static void ExecuteFetchTaskList(List *fetchTaskList); /* * RedistributeTaskListResults partitions the results of given task list using * shard ranges and partition method of given targetRelation, and then colocates * the result files with shards. * * If a shard has a replication factor > 1, corresponding result files are copied * to all nodes containing that shard. * * returnValue[shardIndex] is list of cstrings each of which is a resultId which * correspond to targetRelation->sortedShardIntervalArray[shardIndex]. * * partitionColumnIndex determines the column in the selectTaskList to use for * partitioning. */ List ** RedistributeTaskListResults(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *targetRelation, bool binaryFormat) { /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ UseCoordinatedTransaction(); List *fragmentList = PartitionTasklistResults(resultIdPrefix, selectTaskList, partitionColumnIndex, targetRelation, binaryFormat); return ColocateFragmentsWithRelation(fragmentList, targetRelation); } /* * PartitionTasklistResults executes the given task list, and partitions results * of each task based on targetRelation's distribution method and intervals. * Each of the result partitions are stored in the node where task was executed, * and are named as $resultIdPrefix_from_$sourceShardId_to_$targetShardIndex. * * Result is list of DistributedResultFragment, each of which represents a * partition of results. Empty results are omitted. Therefore, if we have N tasks * and target relation has M shards, we will have NxM-(number of empty results) * fragments. * * partitionColumnIndex determines the column in the selectTaskList to use for * partitioning. */ List * PartitionTasklistResults(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *targetRelation, bool binaryFormat) { if (!IsCitusTableTypeCacheEntry(targetRelation, HASH_DISTRIBUTED) && !IsCitusTableTypeCacheEntry(targetRelation, RANGE_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("repartitioning results of a tasklist is only supported " "when target relation is hash or range partitioned."))); } /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ UseCoordinatedTransaction(); selectTaskList = WrapTasksForPartitioning(resultIdPrefix, selectTaskList, partitionColumnIndex, targetRelation, binaryFormat); return ExecutePartitionTaskList(selectTaskList, targetRelation); } /* * WrapTasksForPartitioning wraps the query for each of the tasks by a call * to worker_partition_query_result(). Target list of the wrapped query should * match the tuple descriptor in ExecutePartitionTaskList(). */ static List * WrapTasksForPartitioning(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *targetRelation, bool binaryFormat) { List *wrappedTaskList = NIL; ShardInterval **shardIntervalArray = targetRelation->sortedShardIntervalArray; int shardCount = targetRelation->shardIntervalArrayLength; ArrayType *minValueArray = NULL; ArrayType *maxValueArray = NULL; Var *partitionColumn = targetRelation->partitionColumn; Oid intervalTypeId = InvalidOid; int32 intervalTypeMod = 0; Oid intervalTypeOutFunc = InvalidOid; bool intervalTypeVarlena = false; GetIntervalTypeInfo(targetRelation->partitionMethod, partitionColumn, &intervalTypeId, &intervalTypeMod); getTypeOutputInfo(intervalTypeId, &intervalTypeOutFunc, &intervalTypeVarlena); ShardMinMaxValueArrays(shardIntervalArray, shardCount, intervalTypeOutFunc, &minValueArray, &maxValueArray); StringInfo minValuesString = ArrayObjectToString(minValueArray, TEXTOID, intervalTypeMod); StringInfo maxValuesString = ArrayObjectToString(maxValueArray, TEXTOID, intervalTypeMod); Task *selectTask = NULL; foreach_declared_ptr(selectTask, selectTaskList) { char *taskPrefix = SourceShardPrefix(resultIdPrefix, selectTask->anchorShardId); char *partitionMethodString = targetRelation->partitionMethod == 'h' ? "hash" : "range"; const char *binaryFormatString = binaryFormat ? "true" : "false"; Task *wrappedSelectTask = copyObject(selectTask); StringInfo wrappedQuery = makeStringInfo(); appendStringInfo(wrappedQuery, "SELECT partition_index" ", %s || '_' || partition_index::text " ", rows_written " "FROM worker_partition_query_result" "(%s,%s,%d,%s,%s,%s,%s) WHERE rows_written > 0", quote_literal_cstr(taskPrefix), quote_literal_cstr(taskPrefix), quote_literal_cstr(TaskQueryString(selectTask)), partitionColumnIndex, quote_literal_cstr(partitionMethodString), minValuesString->data, maxValuesString->data, binaryFormatString); SetTaskQueryString(wrappedSelectTask, wrappedQuery->data); wrappedTaskList = lappend(wrappedTaskList, wrappedSelectTask); } return wrappedTaskList; } /* * CreatePartitioningTupleDest creates a TupleDestination which consumes results of * tasks constructed in WrapTasksForPartitioning. */ static PartitioningTupleDest * CreatePartitioningTupleDest(CitusTableCacheEntry *targetRelation) { int resultColumnCount = 3; TupleDesc tupleDescriptor = CreateTemplateTupleDesc(resultColumnCount); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 1, "partition_index", INT4OID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 2, "result_id", TEXTOID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 3, "rows_written", INT8OID, -1, 0); PartitioningTupleDest *tupleDest = palloc0(sizeof(PartitioningTupleDest)); tupleDest->targetRelation = targetRelation; tupleDest->tupleDesc = tupleDescriptor; tupleDest->fragmentContext = CurrentMemoryContext; tupleDest->pub.putTuple = PartitioningTupleDestPutTuple; tupleDest->pub.tupleDescForQuery = PartitioningTupleDestTupleDescForQuery; return tupleDest; } /* * PartitioningTupleDestPutTuple implements TupleDestination->putTuple for * PartitioningTupleDest. */ static void PartitioningTupleDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize) { PartitioningTupleDest *tupleDest = (PartitioningTupleDest *) self; ShardPlacement *placement = list_nth(task->taskPlacementList, placementIndex); /* * We may be deep inside a nested execution, make sure we can use the * fragment list at the top. */ MemoryContext oldContext = MemoryContextSwitchTo(tupleDest->fragmentContext); DistributedResultFragment *fragment = TupleToDistributedResultFragment(heapTuple, tupleDest->tupleDesc, tupleDest->targetRelation, placement->nodeId); tupleDest->fragmentList = lappend(tupleDest->fragmentList, fragment); MemoryContextSwitchTo(oldContext); } /* * PartitioningTupleDestTupleDescForQuery implements TupleDestination->TupleDescForQuery * for PartitioningTupleDest. */ static TupleDesc PartitioningTupleDestTupleDescForQuery(TupleDestination *self, int queryNumber) { Assert(queryNumber == 0); PartitioningTupleDest *tupleDest = (PartitioningTupleDest *) self; return tupleDest->tupleDesc; } /* * SourceShardPrefix returns result id prefix for partitions which have the * given anchor shard id. */ static char * SourceShardPrefix(const char *resultPrefix, uint64 shardId) { StringInfo taskPrefix = makeStringInfo(); appendStringInfo(taskPrefix, "%s_from_" UINT64_FORMAT "_to", resultPrefix, shardId); return taskPrefix->data; } /* * ShardMinMaxValueArrays returns min values and max values of given shard * intervals. Returned arrays are text arrays. */ void ShardMinMaxValueArrays(ShardInterval **shardIntervalArray, int shardCount, Oid intervalTypeOutFunc, ArrayType **minValueArray, ArrayType **maxValueArray) { Datum *minValues = palloc0(shardCount * sizeof(Datum)); bool *minValueNulls = palloc0(shardCount * sizeof(bool)); Datum *maxValues = palloc0(shardCount * sizeof(Datum)); bool *maxValueNulls = palloc0(shardCount * sizeof(bool)); for (int shardIndex = 0; shardIndex < shardCount; shardIndex++) { minValueNulls[shardIndex] = !shardIntervalArray[shardIndex]->minValueExists; maxValueNulls[shardIndex] = !shardIntervalArray[shardIndex]->maxValueExists; if (!minValueNulls[shardIndex]) { Datum minValue = shardIntervalArray[shardIndex]->minValue; char *minValueStr = DatumGetCString(OidFunctionCall1(intervalTypeOutFunc, minValue)); minValues[shardIndex] = CStringGetTextDatum(minValueStr); } if (!maxValueNulls[shardIndex]) { Datum maxValue = shardIntervalArray[shardIndex]->maxValue; char *maxValueStr = DatumGetCString(OidFunctionCall1(intervalTypeOutFunc, maxValue)); maxValues[shardIndex] = CStringGetTextDatum(maxValueStr); } } *minValueArray = CreateArrayFromDatums(minValues, minValueNulls, shardCount, TEXTOID); *maxValueArray = CreateArrayFromDatums(maxValues, maxValueNulls, shardCount, TEXTOID); } /* * CreateArrayFromDatums creates an array consisting of given values and nulls. */ ArrayType * CreateArrayFromDatums(Datum *datumArray, bool *nullsArray, int datumCount, Oid typeId) { bool typeByValue = false; char typeAlignment = 0; int16 typeLength = 0; int dimensions[1] = { datumCount }; int lowerbounds[1] = { 1 }; get_typlenbyvalalign(typeId, &typeLength, &typeByValue, &typeAlignment); ArrayType *datumArrayObject = construct_md_array(datumArray, nullsArray, 1, dimensions, lowerbounds, typeId, typeLength, typeByValue, typeAlignment); return datumArrayObject; } /* * ExecutePartitionTaskList executes the queries formed in WrapTasksForPartitioning(), * and returns its results as a list of DistributedResultFragment. */ static List * ExecutePartitionTaskList(List *taskList, CitusTableCacheEntry *targetRelation) { PartitioningTupleDest *tupleDest = CreatePartitioningTupleDest(targetRelation); bool errorOnAnyFailure = false; ExecuteSelectTasksIntoTupleDest(taskList, (TupleDestination *) tupleDest, errorOnAnyFailure); return tupleDest->fragmentList; } /* * TupleToDistributedResultFragment converts a tuple returned by the query in * WrapTasksForPartitioning() to a DistributedResultFragment. */ static DistributedResultFragment * TupleToDistributedResultFragment(HeapTuple tuple, TupleDesc tupleDesc, CitusTableCacheEntry *targetRelation, uint32 sourceNodeId) { bool isNull = false; uint32 targetShardIndex = DatumGetUInt32(heap_getattr(tuple, 1, tupleDesc, &isNull)); text *resultId = DatumGetTextP(heap_getattr(tuple, 2, tupleDesc, &isNull)); int64 rowCount = DatumGetInt64(heap_getattr(tuple, 3, tupleDesc, &isNull)); Assert(targetShardIndex < targetRelation->shardIntervalArrayLength); ShardInterval *shardInterval = targetRelation->sortedShardIntervalArray[targetShardIndex]; DistributedResultFragment *distributedResultFragment = palloc0(sizeof(DistributedResultFragment)); distributedResultFragment->nodeId = sourceNodeId; distributedResultFragment->targetShardIndex = targetShardIndex; distributedResultFragment->targetShardId = shardInterval->shardId; distributedResultFragment->resultId = text_to_cstring(resultId); distributedResultFragment->rowCount = rowCount; return distributedResultFragment; } /* * ExecuteSelectTasksIntoTupleDest executes the given tasks and forwards its result * to the given destination. */ static void ExecuteSelectTasksIntoTupleDest(List *taskList, TupleDestination *tupleDestination, bool errorOnAnyFailure) { bool expectResults = true; int targetPoolSize = MaxAdaptiveExecutorPoolSize; TransactionProperties xactProperties = { .errorOnAnyFailure = errorOnAnyFailure, .useRemoteTransactionBlocks = TRANSACTION_BLOCKS_REQUIRED, .requires2PC = false }; bool localExecutionSupported = true; ExecutionParams *executionParams = CreateBasicExecutionParams( ROW_MODIFY_READONLY, taskList, targetPoolSize, localExecutionSupported ); executionParams->tupleDestination = tupleDestination; executionParams->xactProperties = xactProperties; executionParams->expectResults = expectResults; ExecuteTaskListExtended(executionParams); } /* * ColocateFragmentsWithRelation moves the fragments in the cluster so they are * colocated with the shards of target relation. These transfers are done by * calls to fetch_intermediate_results() between nodes. * * returnValue[shardIndex] is list of result Ids that are colocated with * targetRelation->sortedShardIntervalArray[shardIndex] after fetch tasks are * done. */ static List ** ColocateFragmentsWithRelation(List *fragmentList, CitusTableCacheEntry *targetRelation) { List *fragmentListTransfers = ColocationTransfers(fragmentList, targetRelation); List *fragmentTransferTaskList = FragmentTransferTaskList(fragmentListTransfers); ExecuteFetchTaskList(fragmentTransferTaskList); int shardCount = targetRelation->shardIntervalArrayLength; List **shardResultIdList = palloc0(shardCount * sizeof(List *)); DistributedResultFragment *sourceFragment = NULL; foreach_declared_ptr(sourceFragment, fragmentList) { int shardIndex = sourceFragment->targetShardIndex; Assert(shardIndex < shardCount); shardResultIdList[shardIndex] = lappend(shardResultIdList[shardIndex], sourceFragment->resultId); } return shardResultIdList; } /* * ColocationTransfers returns a list of transfers to colocate given fragments with * shards of the target relation. These transfers also take into account replicated * target relations. This prunes away transfers with same source and target */ static List * ColocationTransfers(List *fragmentList, CitusTableCacheEntry *targetRelation) { HASHCTL transferHashInfo; MemSet(&transferHashInfo, 0, sizeof(HASHCTL)); transferHashInfo.keysize = sizeof(NodePair); transferHashInfo.entrysize = sizeof(NodeToNodeFragmentsTransfer); transferHashInfo.hcxt = CurrentMemoryContext; HTAB *transferHash = hash_create("Fragment Transfer Hash", 32, &transferHashInfo, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); DistributedResultFragment *fragment = NULL; foreach_declared_ptr(fragment, fragmentList) { List *placementList = ActiveShardPlacementList(fragment->targetShardId); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, placementList) { NodePair transferKey = { .sourceNodeId = fragment->nodeId, .targetNodeId = placement->nodeId }; if (transferKey.sourceNodeId == transferKey.targetNodeId) { continue; } bool foundInCache = false; NodeToNodeFragmentsTransfer *fragmentListTransfer = hash_search(transferHash, &transferKey, HASH_ENTER, &foundInCache); if (!foundInCache) { fragmentListTransfer->nodes = transferKey; fragmentListTransfer->fragmentList = NIL; } fragmentListTransfer->fragmentList = lappend(fragmentListTransfer->fragmentList, fragment); } } List *fragmentListTransfers = NIL; NodeToNodeFragmentsTransfer *transfer = NULL; HASH_SEQ_STATUS hashSeqStatus; hash_seq_init(&hashSeqStatus, transferHash); while ((transfer = hash_seq_search(&hashSeqStatus)) != NULL) { fragmentListTransfers = lappend(fragmentListTransfers, transfer); } return fragmentListTransfers; } /* * FragmentTransferTaskList returns a list of tasks which performs the given list of * transfers. Each of the transfers are done by a SQL call to fetch_intermediate_results. * See QueryStringForFragmentsTransfer for how the query is constructed. */ static List * FragmentTransferTaskList(List *fragmentListTransfers) { List *fetchTaskList = NIL; NodeToNodeFragmentsTransfer *fragmentsTransfer = NULL; foreach_declared_ptr(fragmentsTransfer, fragmentListTransfers) { uint32 targetNodeId = fragmentsTransfer->nodes.targetNodeId; /* these should have already been pruned away in ColocationTransfers */ Assert(targetNodeId != fragmentsTransfer->nodes.sourceNodeId); WorkerNode *workerNode = LookupNodeByNodeIdOrError(targetNodeId); ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(targetPlacement, workerNode); Task *task = CitusMakeNode(Task); task->taskType = READ_TASK; SetTaskQueryString(task, QueryStringForFragmentsTransfer(fragmentsTransfer)); task->taskPlacementList = list_make1(targetPlacement); fetchTaskList = lappend(fetchTaskList, task); } return fetchTaskList; } /* * QueryStringForFragmentsTransfer returns a query which fetches distributed * result fragments from source node to target node. See the structure of * NodeToNodeFragmentsTransfer for details of how these are decided. */ char * QueryStringForFragmentsTransfer(NodeToNodeFragmentsTransfer *fragmentsTransfer) { StringInfo queryString = makeStringInfo(); StringInfo fragmentNamesArrayString = makeStringInfo(); int fragmentCount = 0; NodePair *nodePair = &fragmentsTransfer->nodes; uint32 sourceNodeId = nodePair->sourceNodeId; /* * If the placement is dummy, for example, queries that generate * intermediate results at the coordinator that need to be redistributed * to worker nodes, we need the local id. */ if (sourceNodeId == LOCAL_NODE_ID) { nodePair->sourceNodeId = GetLocalNodeId(); } WorkerNode *sourceNode = LookupNodeByNodeIdOrError(nodePair->sourceNodeId); appendStringInfoString(fragmentNamesArrayString, "ARRAY["); DistributedResultFragment *fragment = NULL; foreach_declared_ptr(fragment, fragmentsTransfer->fragmentList) { const char *fragmentName = fragment->resultId; if (fragmentCount > 0) { appendStringInfoString(fragmentNamesArrayString, ","); } appendStringInfoString(fragmentNamesArrayString, quote_literal_cstr(fragmentName)); fragmentCount++; } appendStringInfoString(fragmentNamesArrayString, "]::text[]"); appendStringInfo(queryString, "SELECT bytes FROM fetch_intermediate_results(%s,%s,%d) bytes", fragmentNamesArrayString->data, quote_literal_cstr(sourceNode->workerName), sourceNode->workerPort); ereport(DEBUG4, (errmsg("fetch task on %s:%d: %s", sourceNode->workerName, sourceNode->workerPort, queryString->data))); return queryString->data; } /* * ExecuteFetchTaskList executes a list of fetch_intermediate_results() tasks. * It ignores the byte_count result of the fetch_intermediate_results() calls. */ static void ExecuteFetchTaskList(List *taskList) { int resultColumnCount = 1; TupleDesc resultDescriptor = CreateTemplateTupleDesc(resultColumnCount); TupleDescInitEntry(resultDescriptor, (AttrNumber) 1, "byte_count", INT8OID, -1, 0); TupleDestination *tupleDestination = CreateTupleDestNone(); bool errorOnAnyFailure = true; ExecuteSelectTasksIntoTupleDest(taskList, tupleDestination, errorOnAnyFailure); } ================================================ FILE: src/backend/distributed/executor/executor_util_params.c ================================================ /*------------------------------------------------------------------------- * * executor_util_tasks.c * * Utility functions for dealing with task lists in the executor. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "utils/lsyscache.h" #include "distributed/executor_util.h" /* * ExtractParametersForRemoteExecution extracts parameter types and values from * the given ParamListInfo structure, and fills parameter type and value arrays. * It changes oid of custom types to InvalidOid so that they are the same in workers * and coordinators. */ void ExtractParametersForRemoteExecution(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues) { ExtractParametersFromParamList(paramListInfo, parameterTypes, parameterValues, false); } /* * ExtractParametersFromParamList extracts parameter types and values from * the given ParamListInfo structure, and fills parameter type and value arrays. * If useOriginalCustomTypeOids is true, it uses the original oids for custom types. */ void ExtractParametersFromParamList(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues, bool useOriginalCustomTypeOids) { int parameterCount = paramListInfo->numParams; *parameterTypes = (Oid *) palloc0(parameterCount * sizeof(Oid)); *parameterValues = (const char **) palloc0(parameterCount * sizeof(char *)); /* get parameter types and values */ for (int parameterIndex = 0; parameterIndex < parameterCount; parameterIndex++) { ParamExternData *parameterData = ¶mListInfo->params[parameterIndex]; Oid typeOutputFunctionId = InvalidOid; bool variableLengthType = false; /* * Use 0 for data types where the oid values can be different on * the coordinator and worker nodes. Therefore, the worker nodes can * infer the correct oid. */ if (parameterData->ptype >= FirstNormalObjectId && !useOriginalCustomTypeOids) { (*parameterTypes)[parameterIndex] = 0; } else { (*parameterTypes)[parameterIndex] = parameterData->ptype; } /* * If the parameter is not referenced / used (ptype == 0) and * would otherwise have errored out inside standard_planner()), * don't pass a value to the remote side, and pass text oid to prevent * undetermined data type errors on workers. */ if (parameterData->ptype == 0) { (*parameterValues)[parameterIndex] = NULL; (*parameterTypes)[parameterIndex] = TEXTOID; continue; } /* * If the parameter is NULL then we preserve its type, but * don't need to evaluate its value. */ if (parameterData->isnull) { (*parameterValues)[parameterIndex] = NULL; continue; } getTypeOutputInfo(parameterData->ptype, &typeOutputFunctionId, &variableLengthType); (*parameterValues)[parameterIndex] = OidOutputFunctionCall(typeOutputFunctionId, parameterData->value); } } ================================================ FILE: src/backend/distributed/executor/executor_util_tasks.c ================================================ /*------------------------------------------------------------------------- * * executor_util_tasks.c * * Utility functions for dealing with task lists in the executor. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "distributed/executor_util.h" #include "distributed/listutils.h" #include "distributed/shardinterval_utils.h" /* * TaskListModifiesDatabase is a helper function for DistributedExecutionModifiesDatabase and * DistributedPlanModifiesDatabase. */ bool TaskListModifiesDatabase(RowModifyLevel modLevel, List *taskList) { if (modLevel > ROW_MODIFY_READONLY) { return true; } /* * If we cannot decide by only checking the row modify level, * we should look closer to the tasks. */ if (list_length(taskList) < 1) { /* is this ever possible? */ return false; } Task *firstTask = (Task *) linitial(taskList); return !ReadOnlyTask(firstTask->taskType); } /* * TaskListRequiresRollback returns true if the distributed * execution should start a CoordinatedTransaction. In other words, if the * function returns true, the execution sends BEGIN; to every connection * involved in the distributed execution. */ bool TaskListRequiresRollback(List *taskList) { int taskCount = list_length(taskList); if (taskCount == 0) { return false; } Task *task = (Task *) linitial(taskList); if (task->cannotBeExecutedInTransaction) { /* vacuum, create index concurrently etc. */ return false; } bool selectForUpdate = task->relationRowLockList != NIL; if (selectForUpdate) { /* * Do not check SelectOpensTransactionBlock, always open transaction block * if SELECT FOR UPDATE is executed inside a distributed transaction. */ return IsMultiStatementTransaction(); } if (ReadOnlyTask(task->taskType)) { return SelectOpensTransactionBlock && IsTransactionBlock(); } if (IsMultiStatementTransaction()) { return true; } if (list_length(taskList) > 1) { return true; } if (list_length(task->taskPlacementList) > 1) { /* * Single DML/DDL tasks with replicated tables (including * reference and non-reference tables) should require * BEGIN/COMMIT/ROLLBACK. */ return true; } if (task->queryCount > 1) { /* * When there are multiple sequential queries in a task * we need to run those as a transaction. */ return true; } return false; } /* * TaskListRequires2PC determines whether the given task list requires 2PC. */ bool TaskListRequires2PC(List *taskList) { if (taskList == NIL) { return false; } Task *task = (Task *) linitial(taskList); if (ReadOnlyTask(task->taskType)) { /* we do not trigger 2PC for ReadOnly queries */ return false; } bool singleTask = list_length(taskList) == 1; if (singleTask && list_length(task->taskPlacementList) == 1) { /* we do not trigger 2PC for modifications that are: * - single task * - single placement */ return false; } /* * Otherwise, all modifications are done via 2PC. This includes: * - Multi-shard commands irrespective of the replication factor * - Single-shard commands that are targeting more than one replica */ return true; } /* * TaskListCannotBeExecutedInTransaction returns true if any of the * tasks in the input cannot be executed in a transaction. These are * tasks like VACUUM or CREATE INDEX CONCURRENTLY etc. */ bool TaskListCannotBeExecutedInTransaction(List *taskList) { Task *task = NULL; foreach_declared_ptr(task, taskList) { if (task->cannotBeExecutedInTransaction) { return true; } } return false; } /* * SelectForUpdateOnReferenceTable returns true if the input task * contains a FOR UPDATE clause that locks any reference tables. */ bool SelectForUpdateOnReferenceTable(List *taskList) { if (list_length(taskList) != 1) { /* we currently do not support SELECT FOR UPDATE on multi task queries */ return false; } Task *task = (Task *) linitial(taskList); RelationRowLock *relationRowLock = NULL; foreach_declared_ptr(relationRowLock, task->relationRowLockList) { Oid relationId = relationRowLock->relationId; if (IsCitusTableType(relationId, REFERENCE_TABLE)) { return true; } } return false; } /* * ReadOnlyTask returns true if the input task does a read-only operation * on the database. */ bool ReadOnlyTask(TaskType taskType) { switch (taskType) { case READ_TASK: case MAP_OUTPUT_FETCH_TASK: case MAP_TASK: case MERGE_TASK: { return true; } default: { return false; } } } /* * ModifiedTableReplicated iterates on the task list and returns true * if any of the tasks' anchor shard is a replicated table. We qualify * replicated tables as any reference table or any distributed table with * replication factor > 1. */ bool ModifiedTableReplicated(List *taskList) { Task *task = NULL; foreach_declared_ptr(task, taskList) { int64 shardId = task->anchorShardId; if (shardId == INVALID_SHARD_ID) { continue; } if (ReferenceTableShardId(shardId)) { return true; } Oid relationId = RelationIdForShard(shardId); if (!SingleReplicatedTable(relationId)) { return true; } } return false; } /* * ShouldRunTasksSequentially returns true if each of the individual tasks * should be executed one by one. Note that this is different than * MultiShardConnectionType == SEQUENTIAL_CONNECTION case. In that case, * running the tasks across the nodes in parallel is acceptable and implemented * in that way. * * However, the executions that are qualified here would perform poorly if the * tasks across the workers are executed in parallel. We currently qualify only * one class of distributed queries here, multi-row INSERTs. If we do not enforce * true sequential execution, concurrent multi-row upserts could easily form * a distributed deadlock when the upserts touch the same rows. */ bool ShouldRunTasksSequentially(List *taskList) { if (list_length(taskList) < 2) { /* single task plans are already qualified as sequential by definition */ return false; } /* all the tasks are the same, so we only look one */ Task *initialTask = (Task *) linitial(taskList); if (initialTask->rowValuesLists != NIL) { /* found a multi-row INSERT */ return true; } return false; } ================================================ FILE: src/backend/distributed/executor/executor_util_tuples.c ================================================ /*------------------------------------------------------------------------- * * executor_util_tuples.c * * Utility functions for handling tuples during remote execution. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "utils/lsyscache.h" #include "distributed/executor_util.h" /* * TupleDescGetAttBinaryInMetadata - Build an AttInMetadata structure based on * the supplied TupleDesc. AttInMetadata can be used in conjunction with * fmStringInfos containing binary encoded types to produce a properly formed * tuple. * * NOTE: This function is a copy of the PG function TupleDescGetAttInMetadata, * except that it uses getTypeBinaryInputInfo instead of getTypeInputInfo. */ AttInMetadata * TupleDescGetAttBinaryInMetadata(TupleDesc tupdesc) { int natts = tupdesc->natts; int i; Oid atttypeid; Oid attinfuncid; AttInMetadata *attinmeta = (AttInMetadata *) palloc(sizeof(AttInMetadata)); /* "Bless" the tupledesc so that we can make rowtype datums with it */ attinmeta->tupdesc = BlessTupleDesc(tupdesc); /* * Gather info needed later to call the "in" function for each attribute */ FmgrInfo *attinfuncinfo = (FmgrInfo *) palloc0(natts * sizeof(FmgrInfo)); Oid *attioparams = (Oid *) palloc0(natts * sizeof(Oid)); int32 *atttypmods = (int32 *) palloc0(natts * sizeof(int32)); for (i = 0; i < natts; i++) { Form_pg_attribute att = TupleDescAttr(tupdesc, i); /* Ignore dropped attributes */ if (!att->attisdropped) { atttypeid = att->atttypid; getTypeBinaryInputInfo(atttypeid, &attinfuncid, &attioparams[i]); fmgr_info(attinfuncid, &attinfuncinfo[i]); atttypmods[i] = att->atttypmod; } } attinmeta->attinfuncs = attinfuncinfo; attinmeta->attioparams = attioparams; attinmeta->atttypmods = atttypmods; return attinmeta; } /* * BuildTupleFromBytes - build a HeapTuple given user data in binary form. * values is an array of StringInfos, one for each attribute of the return * tuple. A NULL StringInfo pointer indicates we want to create a NULL field. * * NOTE: This function is a copy of the PG function BuildTupleFromCStrings, * except that it uses ReceiveFunctionCall instead of InputFunctionCall. */ HeapTuple BuildTupleFromBytes(AttInMetadata *attinmeta, fmStringInfo *values) { TupleDesc tupdesc = attinmeta->tupdesc; int natts = tupdesc->natts; int i; Datum *dvalues = (Datum *) palloc(natts * sizeof(Datum)); bool *nulls = (bool *) palloc(natts * sizeof(bool)); /* * Call the "in" function for each non-dropped attribute, even for nulls, * to support domains. */ for (i = 0; i < natts; i++) { if (!TupleDescAttr(tupdesc, i)->attisdropped) { /* Non-dropped attributes */ dvalues[i] = ReceiveFunctionCall(&attinmeta->attinfuncs[i], values[i], attinmeta->attioparams[i], attinmeta->atttypmods[i]); if (values[i] != NULL) { nulls[i] = false; } else { nulls[i] = true; } } else { /* Handle dropped attributes by setting to NULL */ dvalues[i] = (Datum) 0; nulls[i] = true; } } /* * Form a tuple */ HeapTuple tuple = heap_form_tuple(tupdesc, dvalues, nulls); /* * Release locally palloc'd space. XXX would probably be good to pfree * values of pass-by-reference datums, as well. */ pfree(dvalues); pfree(nulls); return tuple; } ================================================ FILE: src/backend/distributed/executor/insert_select_executor.c ================================================ /*------------------------------------------------------------------------- * * insert_select_executor.c * * Executor logic for INSERT..SELECT. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "executor/executor.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h" #include "parser/parse_coerce.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/lsyscache.h" #include "utils/portal.h" #include "utils/rel.h" #include "utils/snapmgr.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands/multi_copy.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_execution_locks.h" #include "distributed/distributed_planner.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/merge_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/recursive_planning.h" #include "distributed/relation_access_tracking.h" #include "distributed/repartition_executor.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/stats/stat_counters.h" #include "distributed/subplan_execution.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" /* Config variables managed via guc.c */ bool EnableRepartitionedInsertSelect = true; static void ExecutePlanIntoRelation(Oid targetRelationId, List *insertTargetList, PlannedStmt *selectPlan, EState *executorState); static HTAB * ExecutePlanIntoColocatedIntermediateResults(Oid targetRelationId, List *insertTargetList, PlannedStmt *selectPlan, EState *executorState, char * intermediateResultIdPrefix); static int PartitionColumnIndexFromColumnList(Oid relationId, List *columnNameList); static void WrapTaskListForProjection(List *taskList, List *projectedTargetEntries); /* * NonPushableInsertSelectExecScan executes an INSERT INTO distributed_table * SELECT .. query either by routing via coordinator or by repartitioning * task results and moving data directly between nodes. */ TupleTableSlot * NonPushableInsertSelectExecScan(CustomScanState *node) { CitusScanState *scanState = (CitusScanState *) node; if (!scanState->finishedRemoteScan) { EState *executorState = ScanStateGetExecutorState(scanState); DistributedPlan *distributedPlan = scanState->distributedPlan; Query *insertSelectQuery = copyObject(distributedPlan->modifyQueryViaCoordinatorOrRepartition); List *insertTargetList = insertSelectQuery->targetList; RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery); RangeTblEntry *insertRte = ExtractResultRelationRTE(insertSelectQuery); Oid targetRelationId = insertRte->relid; char *intermediateResultIdPrefix = distributedPlan->intermediateResultIdPrefix; bool hasReturning = distributedPlan->expectResults; HTAB *shardStateHash = NULL; Query *selectQuery = selectRte->subquery; PlannedStmt *selectPlan = copyObject(distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition); /* * If we are dealing with partitioned table, we also need to lock its * partitions. Here we only lock targetRelation, we acquire necessary * locks on selected tables during execution of those select queries. */ if (PartitionedTable(targetRelationId)) { LockPartitionRelations(targetRelationId, RowExclusiveLock); } if (distributedPlan->modifyWithSelectMethod == MODIFY_WITH_SELECT_REPARTITION) { ereport(DEBUG1, (errmsg("performing repartitioned INSERT ... SELECT"))); DistributedPlan *distSelectPlan = GetDistributedPlan((CustomScan *) selectPlan->planTree); Job *distSelectJob = distSelectPlan->workerJob; List *distSelectTaskList = distSelectJob->taskList; bool randomAccess = true; bool interTransactions = false; bool binaryFormat = CanUseBinaryCopyFormatForTargetList(selectQuery->targetList); ExecuteSubPlans(distSelectPlan, RequestedForExplainAnalyze(scanState)); /* * We have a separate directory for each transaction, so choosing * the same result prefix won't cause filename conflicts. Results * directory name also includes node id and database id, so we don't * need to include them in the filename. We include job id here for * the case "INSERT/SELECTs" are executed recursively. */ StringInfo distResultPrefixString = makeStringInfo(); appendStringInfo(distResultPrefixString, "repartitioned_results_" UINT64_FORMAT, distSelectJob->jobId); char *distResultPrefix = distResultPrefixString->data; CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(targetRelationId); int distributionColumnIndex = DistributionColumnIndex(insertTargetList, targetRelation->partitionColumn); if (distributionColumnIndex == -1) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg( "the partition column of table %s should have a value", generate_qualified_relation_name(targetRelationId)))); } TargetEntry *selectPartitionTE = list_nth(selectQuery->targetList, distributionColumnIndex); const char *partitionColumnName = selectPartitionTE->resname ? selectPartitionTE->resname : "(none)"; ereport(DEBUG2, (errmsg( "partitioning SELECT query by column index %d with name %s", distributionColumnIndex, quote_literal_cstr( partitionColumnName)))); /* * ExpandWorkerTargetEntry() can add additional columns to the worker * query. Modify the task queries to only select columns we need. */ int requiredColumnCount = list_length(insertTargetList); List *jobTargetList = distSelectJob->jobQuery->targetList; if (list_length(jobTargetList) > requiredColumnCount) { List *projectedTargetEntries = ListTake(jobTargetList, requiredColumnCount); WrapTaskListForProjection(distSelectTaskList, projectedTargetEntries); } List **redistributedResults = RedistributeTaskListResults(distResultPrefix, distSelectTaskList, distributionColumnIndex, targetRelation, binaryFormat); if (list_length(distSelectTaskList) <= 1) { /* * Probably we will never get here for a repartitioned * INSERT..SELECT because when the source is a single shard * table, we should most probably choose to use * MODIFY_WITH_SELECT_VIA_COORDINATOR, but we still keep this * here. */ IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } /* * At this point select query has been executed on workers and results * have been fetched in such a way that they are colocated with corresponding * target shard. Create and execute a list of tasks of form * INSERT INTO ... SELECT * FROM read_intermediate_results(...); */ List *taskList = GenerateTaskListWithRedistributedResults(insertSelectQuery, targetRelation, redistributedResults, binaryFormat); scanState->tuplestorestate = tuplestore_begin_heap(randomAccess, interTransactions, work_mem); TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); TupleDestination *tupleDest = CreateTupleStoreTupleDest( scanState->tuplestorestate, tupleDescriptor); uint64 rowsInserted = ExecuteTaskListIntoTupleDest(ROW_MODIFY_COMMUTATIVE, taskList, tupleDest, hasReturning); if (list_length(taskList) <= 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } executorState->es_processed = rowsInserted; if (SortReturning && hasReturning) { SortTupleStore(scanState); } } else if (insertSelectQuery->onConflict || hasReturning) { ereport(DEBUG1, (errmsg( "Collecting INSERT ... SELECT results on coordinator"))); /* * If we also have a workerJob that means there is a second step * to the INSERT...SELECT. This happens when there is a RETURNING * or ON CONFLICT clause which is implemented as a separate * distributed INSERT...SELECT from a set of intermediate results * to the target relation. */ List *prunedTaskList = NIL; shardStateHash = ExecutePlanIntoColocatedIntermediateResults( targetRelationId, insertTargetList, selectPlan, executorState, intermediateResultIdPrefix); /* generate tasks for the INSERT..SELECT phase */ List *taskList = GenerateTaskListWithColocatedIntermediateResults( targetRelationId, insertSelectQuery, intermediateResultIdPrefix); /* * We cannot actually execute INSERT...SELECT tasks that read from * intermediate results that weren't created because no rows were * written to them. Prune those tasks out by only including tasks * on shards with connections. */ Task *task = NULL; foreach_declared_ptr(task, taskList) { uint64 shardId = task->anchorShardId; bool shardModified = false; hash_search(shardStateHash, &shardId, HASH_FIND, &shardModified); if (shardModified) { prunedTaskList = lappend(prunedTaskList, task); } } if (prunedTaskList != NIL) { bool randomAccess = true; bool interTransactions = false; Assert(scanState->tuplestorestate == NULL); scanState->tuplestorestate = tuplestore_begin_heap(randomAccess, interTransactions, work_mem); TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); TupleDestination *tupleDest = CreateTupleStoreTupleDest( scanState->tuplestorestate, tupleDescriptor); ExecuteTaskListIntoTupleDest(ROW_MODIFY_COMMUTATIVE, prunedTaskList, tupleDest, hasReturning); if (SortReturning && hasReturning) { SortTupleStore(scanState); } } if (list_length(prunedTaskList) <= 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } } else { ereport(DEBUG1, (errmsg( "Collecting INSERT ... SELECT results on coordinator"))); ExecutePlanIntoRelation(targetRelationId, insertTargetList, selectPlan, executorState); } scanState->finishedRemoteScan = true; } TupleTableSlot *resultSlot = ReturnTupleFromTuplestore(scanState); return resultSlot; } /* * ExecutePlanIntoColocatedIntermediateResults executes the given PlannedStmt * and inserts tuples into a set of intermediate results that are colocated with * the target table for further processing of ON CONFLICT or RETURNING. It also * returns the hash of shard states that were used to insert tuplesinto the target * relation. */ static HTAB * ExecutePlanIntoColocatedIntermediateResults(Oid targetRelationId, List *insertTargetList, PlannedStmt *selectPlan, EState *executorState, char *intermediateResultIdPrefix) { ParamListInfo paramListInfo = executorState->es_param_list_info; /* Get column name list and partition column index for the target table */ List *columnNameList = BuildColumnNameListFromTargetList(targetRelationId, insertTargetList); int partitionColumnIndex = PartitionColumnIndexFromColumnList(targetRelationId, columnNameList); /* * We don't track query counters for the COPY commands that are executed to * prepare intermediate results. */ const bool trackQueryCounters = false; /* set up a DestReceiver that copies into the intermediate table */ const bool publishableData = true; CitusCopyDestReceiver *copyDest = CreateCitusCopyDestReceiver(targetRelationId, columnNameList, partitionColumnIndex, executorState, intermediateResultIdPrefix, publishableData, trackQueryCounters); ExecutePlanIntoDestReceiver(selectPlan, paramListInfo, (DestReceiver *) copyDest); executorState->es_processed = copyDest->tuplesSent; XactModificationLevel = XACT_MODIFICATION_DATA; return copyDest->shardStateHash; } /* * ExecutePlanIntoRelation executes the given plan and inserts the * results into the target relation, which is assumed to be a distributed * table. */ static void ExecutePlanIntoRelation(Oid targetRelationId, List *insertTargetList, PlannedStmt *selectPlan, EState *executorState) { ParamListInfo paramListInfo = executorState->es_param_list_info; /* Get column name list and partition column index for the target table */ List *columnNameList = BuildColumnNameListFromTargetList(targetRelationId, insertTargetList); int partitionColumnIndex = PartitionColumnIndexFromColumnList(targetRelationId, columnNameList); /* * We want to track query counters for the COPY commands that are executed to * perform the final INSERT for such INSERT..SELECT queries. */ const bool trackQueryCounters = true; /* set up a DestReceiver that copies into the distributed table */ const bool publishableData = true; CitusCopyDestReceiver *copyDest = CreateCitusCopyDestReceiver(targetRelationId, columnNameList, partitionColumnIndex, executorState, NULL, publishableData, trackQueryCounters); ExecutePlanIntoDestReceiver(selectPlan, paramListInfo, (DestReceiver *) copyDest); executorState->es_processed = copyDest->tuplesSent; XactModificationLevel = XACT_MODIFICATION_DATA; } /* * BuildColumnNameListForCopyStatement build the column name list given the insert * target list. */ List * BuildColumnNameListFromTargetList(Oid targetRelationId, List *insertTargetList) { List *columnNameList = NIL; /* build the list of column names for the COPY statement */ TargetEntry *insertTargetEntry = NULL; foreach_declared_ptr(insertTargetEntry, insertTargetList) { columnNameList = lappend(columnNameList, insertTargetEntry->resname); } return columnNameList; } /* * PartitionColumnIndexFromColumnList returns the index of partition column from given * column name list and relation ID. If given list doesn't contain the partition * column, it returns -1. */ static int PartitionColumnIndexFromColumnList(Oid relationId, List *columnNameList) { Var *partitionColumn = PartitionColumn(relationId, 0); int partitionColumnIndex = 0; const char *columnName = NULL; foreach_declared_ptr(columnName, columnNameList) { AttrNumber attrNumber = get_attnum(relationId, columnName); /* check whether this is the partition column */ if (partitionColumn != NULL && attrNumber == partitionColumn->varattno) { return partitionColumnIndex; } partitionColumnIndex++; } return -1; } /* * DistributionColumnIndex finds the index of given distribution column in the * given target list. */ int DistributionColumnIndex(List *insertTargetList, Var *distributionColumn) { TargetEntry *insertTargetEntry = NULL; int targetEntryIndex = 0; foreach_declared_ptr(insertTargetEntry, insertTargetList) { if (insertTargetEntry->resno == distributionColumn->varattno) { return targetEntryIndex; } targetEntryIndex++; } return -1; } /* * WrapTaskListForProjection wraps task query string to only select given * projected columns. It modifies the taskList. */ static void WrapTaskListForProjection(List *taskList, List *projectedTargetEntries) { StringInfo projectedColumnsString = makeStringInfo(); int entryIndex = 0; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, projectedTargetEntries) { if (entryIndex != 0) { appendStringInfoChar(projectedColumnsString, ','); } char *columnName = targetEntry->resname; Assert(columnName != NULL); appendStringInfoString(projectedColumnsString, quote_identifier(columnName)); entryIndex++; } Task *task = NULL; foreach_declared_ptr(task, taskList) { StringInfo wrappedQuery = makeStringInfo(); appendStringInfo(wrappedQuery, "SELECT %s FROM (%s) subquery", projectedColumnsString->data, TaskQueryString(task)); SetTaskQueryString(task, wrappedQuery->data); } } ================================================ FILE: src/backend/distributed/executor/intermediate_results.c ================================================ /*------------------------------------------------------------------------- * * intermediate_results.c * Functions for writing and reading intermediate results. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "catalog/pg_enum.h" #include "catalog/pg_type.h" #include "commands/copy.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "storage/fd.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/error_codes.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/remote_commands.h" #include "distributed/transaction_identifier.h" #include "distributed/transmit.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/utils/directory.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" static List *CreatedResultsDirectories = NIL; /* CopyDestReceiver can be used to stream results into a distributed table */ typedef struct RemoteFileDestReceiver { /* public DestReceiver interface */ DestReceiver pub; const char *resultId; /* descriptor of the tuples that are sent to the worker */ TupleDesc tupleDescriptor; /* EState for per-tuple memory allocation */ EState *executorState; /* MemoryContext for DestReceiver session */ MemoryContext memoryContext; /* worker nodes to send data to */ List *initialNodeList; List *connectionList; /* whether to write to a local file */ bool writeLocalFile; FileCompat fileCompat; /* state on how to copy out data types */ CopyOutState copyOutState; FmgrInfo *columnOutputFunctions; /* statistics */ uint64 tuplesSent; uint64 bytesSent; } RemoteFileDestReceiver; /* Enumeration to track one copy query's status on the client */ typedef enum CopyStatus { CLIENT_INVALID_COPY = 0, CLIENT_COPY_MORE = 1, CLIENT_COPY_FAILED = 2, CLIENT_COPY_DONE = 3 } CopyStatus; static void RemoteFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static void PrepareIntermediateResultBroadcast(RemoteFileDestReceiver *resultDest); static StringInfo ConstructCopyResultStatement(const char *resultId); static bool RemoteFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); static void BroadcastCopyData(StringInfo dataBuffer, List *connectionList); static void SendCopyDataOverConnection(StringInfo dataBuffer, MultiConnection *connection); static void RemoteFileDestReceiverShutdown(DestReceiver *destReceiver); static void RemoteFileDestReceiverDestroy(DestReceiver *destReceiver); static char * IntermediateResultsDirectory(void); static void ReadIntermediateResultsIntoFuncOutput(FunctionCallInfo fcinfo, char *copyFormat, Datum *resultIdArray, int resultCount); static uint64 FetchRemoteIntermediateResult(MultiConnection *connection, char *resultId); static CopyStatus CopyDataFromConnection(MultiConnection *connection, FileCompat *fileCompat, uint64 *bytesReceived); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(read_intermediate_result); PG_FUNCTION_INFO_V1(read_intermediate_result_array); PG_FUNCTION_INFO_V1(broadcast_intermediate_result); PG_FUNCTION_INFO_V1(create_intermediate_result); PG_FUNCTION_INFO_V1(fetch_intermediate_results); /* * broadcast_intermediate_result executes a query and streams the results * into a file on all workers. */ Datum broadcast_intermediate_result(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *resultIdText = PG_GETARG_TEXT_P(0); char *resultIdString = text_to_cstring(resultIdText); text *queryText = PG_GETARG_TEXT_P(1); char *queryString = text_to_cstring(queryText); bool writeLocalFile = false; ParamListInfo paramListInfo = NULL; /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ UseCoordinatedTransaction(); List *nodeList = ActivePrimaryNonCoordinatorNodeList(NoLock); EState *estate = CreateExecutorState(); RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) CreateRemoteFileDestReceiver(resultIdString, estate, nodeList, writeLocalFile); ExecuteQueryStringIntoDestReceiver(queryString, paramListInfo, (DestReceiver *) resultDest); FreeExecutorState(estate); PG_RETURN_INT64(resultDest->tuplesSent); } /* * create_intermediate_result executes a query and writes the results * into a local file. */ Datum create_intermediate_result(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *resultIdText = PG_GETARG_TEXT_P(0); char *resultIdString = text_to_cstring(resultIdText); text *queryText = PG_GETARG_TEXT_P(1); char *queryString = text_to_cstring(queryText); List *nodeList = NIL; bool writeLocalFile = true; ParamListInfo paramListInfo = NULL; /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ UseCoordinatedTransaction(); EState *estate = CreateExecutorState(); RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) CreateRemoteFileDestReceiver(resultIdString, estate, nodeList, writeLocalFile); ExecuteQueryStringIntoDestReceiver(queryString, paramListInfo, (DestReceiver *) resultDest); FreeExecutorState(estate); PG_RETURN_INT64(resultDest->tuplesSent); } /* * CreateRemoteFileDestReceiver creates a DestReceiver that streams results * to a set of worker nodes. If the scope of the intermediate result is a * distributed transaction, then it's up to the caller to ensure that a * coordinated transaction is started prior to using the DestReceiver. */ DestReceiver * CreateRemoteFileDestReceiver(const char *resultId, EState *executorState, List *initialNodeList, bool writeLocalFile) { RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) palloc0( sizeof(RemoteFileDestReceiver)); /* set up the DestReceiver function pointers */ resultDest->pub.receiveSlot = RemoteFileDestReceiverReceive; resultDest->pub.rStartup = RemoteFileDestReceiverStartup; resultDest->pub.rShutdown = RemoteFileDestReceiverShutdown; resultDest->pub.rDestroy = RemoteFileDestReceiverDestroy; resultDest->pub.mydest = DestCopyOut; /* set up output parameters */ resultDest->resultId = resultId; resultDest->executorState = executorState; resultDest->initialNodeList = initialNodeList; resultDest->memoryContext = CurrentMemoryContext; resultDest->writeLocalFile = writeLocalFile; return (DestReceiver *) resultDest; } /* * RemoteFileDestReceiverBytesSent returns number of bytes sent per remote worker. */ uint64 RemoteFileDestReceiverBytesSent(DestReceiver *destReceiver) { RemoteFileDestReceiver *remoteDestReceiver = (RemoteFileDestReceiver *) destReceiver; return remoteDestReceiver->bytesSent; } /* * RemoteFileDestReceiverStartup implements the rStartup interface of * RemoteFileDestReceiver. It opens connections to the nodes in initialNodeList, * and sends the COPY command on all connections. */ static void RemoteFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor) { RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) dest; const char *delimiterCharacter = "\t"; const char *nullPrintCharacter = "\\N"; resultDest->tupleDescriptor = inputTupleDescriptor; /* define how tuples will be serialised */ CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; copyOutState->null_print = (char *) nullPrintCharacter; copyOutState->null_print_client = (char *) nullPrintCharacter; copyOutState->binary = CanUseBinaryCopyFormat(inputTupleDescriptor); copyOutState->fe_msgbuf = makeStringInfo(); copyOutState->rowcontext = GetPerTupleMemoryContext(resultDest->executorState); resultDest->copyOutState = copyOutState; resultDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor, copyOutState->binary); } /* * PrepareIntermediateResultBroadcast gets a RemoteFileDestReceiver and does * the necessary initializations including initiating the remote connections * and creating the local file, which is necessary (it might be both). */ static void PrepareIntermediateResultBroadcast(RemoteFileDestReceiver *resultDest) { List *initialNodeList = resultDest->initialNodeList; const char *resultId = resultDest->resultId; List *connectionList = NIL; CopyOutState copyOutState = resultDest->copyOutState; if (resultDest->writeLocalFile) { const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); /* make sure the directory exists */ CreateIntermediateResultsDirectory(); const char *fileName = QueryResultFileName(resultId); resultDest->fileCompat = FileCompatFromFileStart(FileOpenForTransmit(fileName, fileFlags)); } WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, initialNodeList) { int flags = 0; const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; MultiConnection *connection = StartNodeConnection(flags, nodeName, nodePort); ClaimConnectionExclusively(connection); MarkRemoteTransactionCritical(connection); connectionList = lappend(connectionList, connection); } FinishConnectionListEstablishment(connectionList); /* must open transaction blocks to use intermediate results */ RemoteTransactionsBeginIfNecessary(connectionList); MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { StringInfo copyCommand = ConstructCopyResultStatement(resultId); bool querySent = SendRemoteCommand(connection, copyCommand->data); if (!querySent) { ReportConnectionError(connection, ERROR); } } foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (PQresultStatus(result) != PGRES_COPY_IN) { ReportResultError(connection, result, ERROR); } PQclear(result); } if (copyOutState->binary) { /* send headers when using binary encoding */ resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryHeaders(copyOutState); BroadcastCopyData(copyOutState->fe_msgbuf, connectionList); if (resultDest->writeLocalFile) { WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } } resultDest->connectionList = connectionList; } /* * ConstructCopyResultStatement constructs the text of a COPY statement * for copying into a result file. */ static StringInfo ConstructCopyResultStatement(const char *resultId) { StringInfo command = makeStringInfo(); appendStringInfo(command, "COPY \"%s\" FROM STDIN WITH (format result)", resultId); return command; } /* * RemoteFileDestReceiverReceive implements the receiveSlot function of * RemoteFileDestReceiver. It takes a TupleTableSlot and sends the contents to * all worker nodes. */ static bool RemoteFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) { RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) dest; if (resultDest->tuplesSent == 0) { /* * We get the first tuple, lets initialize the remote connections * and/or the local file. */ PrepareIntermediateResultBroadcast(resultDest); } TupleDesc tupleDescriptor = resultDest->tupleDescriptor; List *connectionList = resultDest->connectionList; CopyOutState copyOutState = resultDest->copyOutState; FmgrInfo *columnOutputFunctions = resultDest->columnOutputFunctions; StringInfo copyData = copyOutState->fe_msgbuf; EState *executorState = resultDest->executorState; MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState); MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); slot_getallattrs(slot); Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; resetStringInfo(copyData); /* construct row in COPY format */ AppendCopyRowData(columnValues, columnNulls, tupleDescriptor, copyOutState, columnOutputFunctions, NULL); /* send row to nodes */ BroadcastCopyData(copyData, connectionList); /* write to local file (if applicable) */ if (resultDest->writeLocalFile) { WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } MemoryContextSwitchTo(oldContext); resultDest->tuplesSent++; resultDest->bytesSent += copyData->len; ResetPerTupleExprContext(executorState); return true; } /* * WriteToLocalResultsFile writes the bytes in a StringInfo to a local file. */ void WriteToLocalFile(StringInfo copyData, FileCompat *fileCompat) { int bytesWritten = FileWriteCompat(fileCompat, copyData->data, copyData->len, PG_WAIT_IO); if (bytesWritten < 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not append to file: %m"))); } } /* * RemoteFileDestReceiverShutdown implements the rShutdown interface of * RemoteFileDestReceiver. It ends the COPY on all the open connections and closes * the relation. */ static void RemoteFileDestReceiverShutdown(DestReceiver *destReceiver) { RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) destReceiver; if (resultDest->tuplesSent == 0) { /* * We have not received any tuples (when the intermediate result * returns zero rows). Still, we want to create the necessary * intermediate result files even if they are empty, as the query * execution requires the files to be present. */ PrepareIntermediateResultBroadcast(resultDest); } List *connectionList = resultDest->connectionList; CopyOutState copyOutState = resultDest->copyOutState; if (copyOutState->binary) { /* send footers when using binary encoding */ resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryFooters(copyOutState); BroadcastCopyData(copyOutState->fe_msgbuf, connectionList); if (resultDest->writeLocalFile) { WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } } /* close the COPY input */ EndRemoteCopy(0, connectionList); if (resultDest->writeLocalFile) { FileClose(resultDest->fileCompat.fd); } } /* * BroadcastCopyData sends copy data to all connections in a list. */ static void BroadcastCopyData(StringInfo dataBuffer, List *connectionList) { MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { SendCopyDataOverConnection(dataBuffer, connection); } } /* * SendCopyDataOverConnection sends serialized COPY data over the given * connection. */ static void SendCopyDataOverConnection(StringInfo dataBuffer, MultiConnection *connection) { if (!PutRemoteCopyData(connection, dataBuffer->data, dataBuffer->len)) { ReportConnectionError(connection, ERROR); } } /* * RemoteFileDestReceiverDestroy frees memory allocated as part of the * RemoteFileDestReceiver and closes file descriptors. */ static void RemoteFileDestReceiverDestroy(DestReceiver *destReceiver) { RemoteFileDestReceiver *resultDest = (RemoteFileDestReceiver *) destReceiver; if (resultDest->copyOutState) { pfree(resultDest->copyOutState); } if (resultDest->columnOutputFunctions) { pfree(resultDest->columnOutputFunctions); } pfree(resultDest); } /* * SendQueryResultViaCopy is called when a COPY "resultid" TO STDOUT * WITH (format result) command is received from the client. The * contents of the file are sent directly to the client. */ void SendQueryResultViaCopy(const char *resultId) { const char *resultFileName = QueryResultFileName(resultId); SendRegularFile(resultFileName); } /* * ReceiveQueryResultViaCopy is called when a COPY "resultid" FROM * STDIN WITH (format result) command is received from the client. * The command is followed by the raw copy data stream, which is * redirected to a file. * * File names are automatically prefixed with the user OID. Users * are only allowed to read query results from their own directory. */ void ReceiveQueryResultViaCopy(const char *resultId) { CreateIntermediateResultsDirectory(); const char *resultFileName = QueryResultFileName(resultId); RedirectCopyDataToRegularFile(resultFileName); } /* * CreateIntermediateResultsDirectory creates the intermediate result * directory for the current transaction if it does not exist and ensures * that the directory is removed at the end of the transaction. */ char * CreateIntermediateResultsDirectory(void) { char *resultDirectory = IntermediateResultsDirectory(); int makeOK = MakePGDirectory(resultDirectory); if (makeOK != 0) { if (errno == EEXIST) { /* someone else beat us to it, that's ok */ return resultDirectory; } ereport(ERROR, (errcode_for_file_access(), errmsg("could not create intermediate results directory " "\"%s\": %m", resultDirectory))); } MemoryContext oldContext = MemoryContextSwitchTo(TopTransactionContext); CreatedResultsDirectories = lappend(CreatedResultsDirectories, pstrdup(resultDirectory)); MemoryContextSwitchTo(oldContext); return resultDirectory; } /* * QueryResultFileName returns the file name in which to store * an intermediate result with the given key in the per transaction * result directory. */ char * QueryResultFileName(const char *resultId) { StringInfo resultFileName = makeStringInfo(); const char *resultDirectory = IntermediateResultsDirectory(); char *checkChar = (char *) resultId; for (; *checkChar; checkChar++) { if (!((*checkChar >= 'a' && *checkChar <= 'z') || (*checkChar >= 'A' && *checkChar <= 'Z') || (*checkChar >= '0' && *checkChar <= '9') || (*checkChar == '_') || (*checkChar == '-'))) { ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), errmsg("result key \"%s\" contains invalid character", resultId), errhint("Result keys may only contain letters, numbers, " "underscores and hyphens."))); } } appendStringInfo(resultFileName, "%s/%s.data", resultDirectory, resultId); return resultFileName->data; } /* * IntermediateResultsDirectory returns the directory to use for a query result * file with a particular key. The filename includes the user OID, such * that users can never read each other's files. * * In a distributed transaction, the directory has the form: * base/pgsql_job_cache/__/ * * In a non-distributed transaction, the directory has the form: * base/pgsql_job_cache/_/ * * The latter form can be used for testing COPY ... WITH (format result) without * assigning a distributed transaction ID. * * The pgsql_job_cache directory is emptied on restart in case of failure. */ static char * IntermediateResultsDirectory(void) { StringInfo resultFileName = makeStringInfo(); Oid userId = GetUserId(); DistributedTransactionId *transactionId = GetCurrentDistributedTransactionId(); int initiatorNodeIdentifier = transactionId->initiatorNodeIdentifier; uint64 transactionNumber = transactionId->transactionNumber; if (transactionNumber > 0) { appendStringInfo(resultFileName, "base/" PG_JOB_CACHE_DIR "/%u_%u_%lu", userId, initiatorNodeIdentifier, transactionNumber); } else { appendStringInfo(resultFileName, "base/" PG_JOB_CACHE_DIR "/%u_%u", userId, MyProcPid); } return resultFileName->data; } /* * RemoveIntermediateResultsDirectories removes the intermediate result directory * for the current distributed transaction, if any was created. */ void RemoveIntermediateResultsDirectories(void) { char *directoryElement = NULL; foreach_declared_ptr(directoryElement, CreatedResultsDirectories) { /* * The shared directory is renamed before deleting it. Otherwise it * would be possible for another backend to write a file, while we are * deleting the directory. Since rename is atomic by POSIX standards * that's not possible. The current PID is included in the new * filename, so there can be no collisions with other backends. */ char *sharedName = directoryElement; StringInfo privateName = makeStringInfo(); appendStringInfo(privateName, "%s.removed-by-%d", sharedName, MyProcPid); if (rename(sharedName, privateName->data)) { ereport(LOG, (errcode_for_file_access(), errmsg( "could not rename intermediate results directory \"%s\" to \"%s\": %m", sharedName, privateName->data))); /* rename failed for some reason, we do a best effort removal of * the shared directory */ PathNameDeleteTemporaryDir(sharedName); } else { PathNameDeleteTemporaryDir(privateName->data); } } /* cleanup */ list_free_deep(CreatedResultsDirectories); CreatedResultsDirectories = NIL; } /* * IntermediateResultSize returns the file size of the intermediate result * or -1 if the file does not exist. */ int64 IntermediateResultSize(const char *resultId) { struct stat fileStat; char *resultFileName = QueryResultFileName(resultId); int statOK = stat(resultFileName, &fileStat); if (statOK < 0) { return -1; } return (int64) fileStat.st_size; } /* * read_intermediate_result is a UDF that returns a COPY-formatted intermediate * result file as a set of records. The file is parsed according to the columns * definition list specified by the user, e.g.: * * SELECT * FROM read_intermediate_result('foo', 'csv') AS (a int, b int) * * The file is read from the directory returned by IntermediateResultsDirectory, * which includes the user ID. * * read_intermediate_result is a volatile function because it cannot be * evaluated until execution time, but for distributed planning purposes we can * treat it in the same way as immutable functions and reference tables, since * we know it will return the same result on all nodes. */ Datum read_intermediate_result(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Datum resultId = PG_GETARG_DATUM(0); Datum copyFormatOidDatum = PG_GETARG_DATUM(1); Datum copyFormatLabelDatum = DirectFunctionCall1(enum_out, copyFormatOidDatum); char *copyFormatLabel = DatumGetCString(copyFormatLabelDatum); ReadIntermediateResultsIntoFuncOutput(fcinfo, copyFormatLabel, &resultId, 1); PG_RETURN_DATUM(0); } /* * read_intermediate_result_array returns the set of records in a set of given * COPY-formatted intermediate result files. * * The usage and semantics of this is same as read_intermediate_result(), except * that its first argument is an array of result ids. */ Datum read_intermediate_result_array(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ArrayType *resultIdObject = PG_GETARG_ARRAYTYPE_P(0); Datum copyFormatOidDatum = PG_GETARG_DATUM(1); Datum copyFormatLabelDatum = DirectFunctionCall1(enum_out, copyFormatOidDatum); char *copyFormatLabel = DatumGetCString(copyFormatLabelDatum); int32 resultCount = ArrayGetNItems(ARR_NDIM(resultIdObject), ARR_DIMS( resultIdObject)); Datum *resultIdArray = DeconstructArrayObject(resultIdObject); ReadIntermediateResultsIntoFuncOutput(fcinfo, copyFormatLabel, resultIdArray, resultCount); PG_RETURN_DATUM(0); } /* * ReadIntermediateResultsIntoFuncOutput reads the given result files and stores * them at the function's output tuple store. Errors out if any of the result files * don't exist. */ static void ReadIntermediateResultsIntoFuncOutput(FunctionCallInfo fcinfo, char *copyFormat, Datum *resultIdArray, int resultCount) { TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); for (int resultIndex = 0; resultIndex < resultCount; resultIndex++) { char *resultId = TextDatumGetCString(resultIdArray[resultIndex]); char *resultFileName = QueryResultFileName(resultId); struct stat fileStat; int statOK = stat(resultFileName, &fileStat); if (statOK != 0) { /* * When the file does not exist, it could mean two different things. * First -- and a lot more common -- case is that a failure happened * in a concurrent backend on the same distributed transaction. And, * one of the backends in that transaction has already been roll * backed, which has already removed the file. If we throw an error * here, the user might see this error instead of the actual error * message. Instead, we prefer to WARN the user and pretend that the * file has no data in it. In the end, the user would see the actual * error message for the failure. * * Second, in case of any bugs in intermediate result broadcasts, * we could try to read a non-existing file. That is most likely * to happen during development. */ ereport(WARNING, (errcode(ERRCODE_CITUS_INTERMEDIATE_RESULT_NOT_FOUND), errmsg("Query could not find the intermediate result file " "\"%s\", it was mostly likely deleted due to an " "error in a parallel process within the same " "distributed transaction", resultId))); } else { ReadFileIntoTupleStore(resultFileName, copyFormat, tupleDescriptor, tupleStore); } } } /* * fetch_intermediate_results fetches a set of intermediate results defined in an * array of result IDs from a remote node and writes them to a local intermediate * result with the same ID. */ Datum fetch_intermediate_results(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ArrayType *resultIdObject = PG_GETARG_ARRAYTYPE_P(0); Datum *resultIdArray = DeconstructArrayObject(resultIdObject); int32 resultCount = ArrayObjectCount(resultIdObject); text *remoteHostText = PG_GETARG_TEXT_P(1); char *remoteHost = text_to_cstring(remoteHostText); int remotePort = PG_GETARG_INT32(2); int connectionFlags = FORCE_NEW_CONNECTION; int resultIndex = 0; int64 totalBytesWritten = 0L; if (resultCount == 0) { PG_RETURN_INT64(0); } if (!IsMultiStatementTransaction()) { ereport(ERROR, (errmsg("fetch_intermediate_results can only be used in a " "distributed transaction"))); } /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ EnsureDistributedTransactionId(); MultiConnection *connection = GetNodeConnection(connectionFlags, remoteHost, remotePort); if (PQstatus(connection->pgConn) != CONNECTION_OK) { ereport(ERROR, (errmsg("cannot connect to %s:%d to fetch intermediate results", remoteHost, remotePort))); } StringInfo beginAndSetXactId = BeginAndSetDistributedTransactionIdCommand(); ExecuteCriticalRemoteCommand(connection, beginAndSetXactId->data); CreateIntermediateResultsDirectory(); for (resultIndex = 0; resultIndex < resultCount; resultIndex++) { char *resultId = TextDatumGetCString(resultIdArray[resultIndex]); totalBytesWritten += FetchRemoteIntermediateResult(connection, resultId); } ExecuteCriticalRemoteCommand(connection, "END"); CloseConnection(connection); PG_RETURN_INT64(totalBytesWritten); } /* * FetchRemoteIntermediateResult fetches a remote intermediate result over * the given connection. */ static uint64 FetchRemoteIntermediateResult(MultiConnection *connection, char *resultId) { char *localPath = QueryResultFileName(resultId); struct stat fileStat; int statOK = stat(localPath, &fileStat); if (statOK == 0) { /* * File exists, most likely because we are trying to fetch a * a file from a node to itself. Skip doing work. */ return fileStat.st_size; } uint64 totalBytesWritten = 0; StringInfo copyCommand = makeStringInfo(); const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); PGconn *pgConn = connection->pgConn; int socket = PQsocket(pgConn); bool raiseErrors = true; appendStringInfo(copyCommand, "COPY \"%s\" TO STDOUT WITH (format result)", resultId); if (!SendRemoteCommand(connection, copyCommand->data)) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (PQresultStatus(result) != PGRES_COPY_OUT) { ReportResultError(connection, result, ERROR); } PQclear(result); File fileDesc = FileOpenForTransmit(localPath, fileFlags); FileCompat fileCompat = FileCompatFromFileStart(fileDesc); while (true) { int waitFlags = WL_SOCKET_READABLE | WL_POSTMASTER_DEATH; CopyStatus copyStatus = CopyDataFromConnection(connection, &fileCompat, &totalBytesWritten); if (copyStatus == CLIENT_COPY_FAILED) { ereport(ERROR, (errmsg("failed to read result \"%s\" from node %s:%d", resultId, connection->hostname, connection->port))); } else if (copyStatus == CLIENT_COPY_DONE) { break; } Assert(copyStatus == CLIENT_COPY_MORE); int rc = WaitLatchOrSocket(MyLatch, waitFlags, socket, 0, PG_WAIT_EXTENSION); if (rc & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } if (rc & WL_LATCH_SET) { ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } } FileClose(fileDesc); ClearResults(connection, raiseErrors); return totalBytesWritten; } /* * CopyDataFromConnection reads a row of copy data from connection and writes it * to the given file. */ static CopyStatus CopyDataFromConnection(MultiConnection *connection, FileCompat *fileCompat, uint64 *bytesReceived) { /* * Consume input to handle the case where previous copy operation might have * received zero bytes. */ int consumed = PQconsumeInput(connection->pgConn); if (consumed == 0) { return CLIENT_COPY_FAILED; } /* receive copy data message in an asynchronous manner */ char *receiveBuffer = NULL; bool asynchronous = true; int receiveLength = PQgetCopyData(connection->pgConn, &receiveBuffer, asynchronous); while (receiveLength > 0) { /* received copy data; append these data to file */ errno = 0; int bytesWritten = FileWriteCompat(fileCompat, receiveBuffer, receiveLength, PG_WAIT_IO); if (bytesWritten != receiveLength) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not append to file: %m"))); } *bytesReceived += receiveLength; PQfreemem(receiveBuffer); receiveLength = PQgetCopyData(connection->pgConn, &receiveBuffer, asynchronous); } if (receiveLength == 0) { /* we cannot read more data without blocking */ return CLIENT_COPY_MORE; } else if (receiveLength == -1) { /* received copy done message */ bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); ExecStatusType resultStatus = PQresultStatus(result); CopyStatus copyStatus = 0; if (resultStatus == PGRES_COMMAND_OK) { copyStatus = CLIENT_COPY_DONE; } else { copyStatus = CLIENT_COPY_FAILED; ReportResultError(connection, result, WARNING); } PQclear(result); ForgetResults(connection); return copyStatus; } else { Assert(receiveLength == -2); ReportConnectionError(connection, WARNING); return CLIENT_COPY_FAILED; } } ================================================ FILE: src/backend/distributed/executor/local_executor.c ================================================ /* * local_executor.c * * The scope of the local execution is locally executing the queries on the * shards. In other words, local execution does not deal with any local tables * that are not on shards on the node that the query is being executed. In that * sense, the local executor is only triggered if the node has both the metadata * and the shards (e.g., only Citus MX worker nodes). * * The goal of the local execution is to skip the unnecessary network round-trip * happening on the node itself. Instead, identify the locally executable tasks * and simply call PostgreSQL's planner and executor. * * The local executor is an extension of the adaptive executor. So, the executor * uses adaptive executor's custom scan nodes. * * One thing to note is that Citus MX is only supported with replication factor * to be equal to 1, so keep that in mind while continuing the comments below. * * On the high level, there are 3 slightly different ways of utilizing local * execution: * * (1) Execution of local single shard queries of a distributed table * * This is the simplest case. The executor kicks at the start of the adaptive * executor, and since the query is only a single task the execution finishes * without going to the network at all. * * Even if there is a transaction block (or recursively planned CTEs), as * long as the queries hit the shards on the same node, the local execution * will kick in. * * (2) Execution of local single queries and remote multi-shard queries * * The rule is simple. If a transaction block starts with a local query * execution, * all the other queries in the same transaction block that touch any local * shard have to use the local execution. Although this sounds restrictive, * we prefer to implement it in this way, otherwise we'd end-up with as * complex scenarios as we have in the connection managements due to foreign * keys. * * See the following example: * BEGIN; * -- assume that the query is executed locally * SELECT count(*) FROM test WHERE key = 1; * * -- at this point, all the shards that reside on the * -- node is executed locally one-by-one. After those finishes * -- the remaining tasks are handled by adaptive executor * SELECT count(*) FROM test; * * * (3) Modifications of reference tables * * Modifications to reference tables have to be executed on all nodes. So, * after the local execution, the adaptive executor keeps continuing the * execution on the other nodes. * * Note that for read-only queries, after the local execution, there is no * need to kick in adaptive executor. * * (4) Execution of multi shards local queries and * remote multi-shard queries within a transaction block * * We prefer local execution when we are inside a transaction block, because not using * local execution might create some limitations for other commands in the transaction * block. To simplify things, whenever we are inside a transaction block, we prefer local * execution if possible. * * There are also a few limitations/trade-offs that are worth mentioning. * - The local execution on multiple shards might be slow because the execution * has to happen one task at a time (e.g., no parallelism). * - Related with the previous item, COPY command cannot be mixed with local * execution in a transaction. The implication of that is any part of INSERT..SELECT * via coordinator cannot happen via the local execution. */ #include "postgres.h" #include "miscadmin.h" #include "executor/tstoreReceiver.h" #include "executor/tuptable.h" #include "nodes/params.h" #include "optimizer/optimizer.h" #include "utils/snapmgr.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/executor_util.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_plan_cache.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_server_executor.h" #include "distributed/query_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" /* to access LogRemoteCommands */ #include "distributed/stats/stat_tenants.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* controlled via a GUC */ bool EnableLocalExecution = true; bool LogLocalCommands = false; /* global variable that tracks whether the local execution is on a shard */ uint64 LocalExecutorShardId = INVALID_SHARD_ID; static LocalExecutionStatus CurrentLocalExecutionStatus = LOCAL_EXECUTION_OPTIONAL; static void SplitLocalAndRemotePlacements(List *taskPlacementList, List **localTaskPlacementList, List **remoteTaskPlacementList); static uint64 LocallyExecuteTaskPlan(PlannedStmt *taskPlan, char *queryString, TupleDestination *tupleDest, Task *task, ParamListInfo paramListInfo); static uint64 ExecuteTaskPlan(PlannedStmt *taskPlan, char *queryString, TupleDestination *tupleDest, Task *task, ParamListInfo paramListInfo); static void RecordNonDistTableAccessesForTask(Task *task); static void LogLocalCommand(Task *task); static uint64 LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tupleDest, Task *task); static void SetColocationIdAndPartitionKeyValueForTasks(List *taskList, Job *distributedPlan); static void LocallyExecuteUtilityTask(Task *task); static void ExecuteUdfTaskQuery(Query *localUdfCommandQuery); static void EnsureTransitionPossible(LocalExecutionStatus from, LocalExecutionStatus to); /* * GetCurrentLocalExecutionStatus returns the current local execution status. */ LocalExecutionStatus GetCurrentLocalExecutionStatus(void) { return CurrentLocalExecutionStatus; } /* * ExecuteLocalTaskList executes the given tasks locally. * * The function goes over the task list and executes them locally. * The returning tuples (if any) is stored in the tupleStoreState. * * The function returns totalRowsProcessed. */ uint64 ExecuteLocalTaskList(List *taskList, TupleDestination *defaultTupleDest) { if (list_length(taskList) == 0) { return 0; } DistributedPlan *distributedPlan = NULL; ParamListInfo paramListInfo = NULL; bool isUtilityCommand = false; return ExecuteLocalTaskListExtended(taskList, paramListInfo, distributedPlan, defaultTupleDest, isUtilityCommand); } /* * ExecuteLocalUtilityTaskList executes the given tasks locally. * * The function returns totalRowsProcessed. */ uint64 ExecuteLocalUtilityTaskList(List *utilityTaskList) { if (list_length(utilityTaskList) == 0) { return 0; } DistributedPlan *distributedPlan = NULL; ParamListInfo paramListInfo = NULL; TupleDestination *defaultTupleDest = CreateTupleDestNone(); bool isUtilityCommand = true; return ExecuteLocalTaskListExtended(utilityTaskList, paramListInfo, distributedPlan, defaultTupleDest, isUtilityCommand); } /* * ExecuteLocalTaskListExtended executes the given tasks locally. * * The function goes over the task list and executes them locally. * The returning tuples (if any) is stored in the tupleStoreState. * * It uses a cached plan if distributedPlan is found in cache. * * The function returns totalRowsProcessed. */ uint64 ExecuteLocalTaskListExtended(List *taskList, ParamListInfo orig_paramListInfo, DistributedPlan *distributedPlan, TupleDestination *defaultTupleDest, bool isUtilityCommand) { ParamListInfo paramListInfo = copyParamList(orig_paramListInfo); uint64 totalRowsProcessed = 0; int numParams = 0; Oid *parameterTypes = NULL; if (paramListInfo != NULL) { /* not used anywhere, so declare here */ const char **parameterValues = NULL; ExtractParametersForLocalExecution(paramListInfo, ¶meterTypes, ¶meterValues); numParams = paramListInfo->numParams; } if (taskList != NIL) { bool isRemote = false; EnsureTaskExecutionAllowed(isRemote); } /* * If workerJob has a partitionKeyValue, we need to set the colocation id * and partition key value for each task before we start executing them * because tenant stats are collected based on these values of a task. */ if (distributedPlan != NULL && distributedPlan->workerJob != NULL && taskList != NIL) { SetJobColocationId(distributedPlan->workerJob); SetColocationIdAndPartitionKeyValueForTasks(taskList, distributedPlan->workerJob); } /* * Use a new memory context that gets reset after every task to free * the deparsed query string and query plan. */ MemoryContext loopContext = AllocSetContextCreate(CurrentMemoryContext, "ExecuteLocalTaskListExtended", ALLOCSET_DEFAULT_SIZES); Task *task = NULL; foreach_declared_ptr(task, taskList) { MemoryContext oldContext = MemoryContextSwitchTo(loopContext); TupleDestination *tupleDest = task->tupleDest ? task->tupleDest : defaultTupleDest; /* * If we have a valid shard id, a distributed table will be accessed * during execution. Record it to apply the restrictions related to * local execution. */ if (task->anchorShardId != INVALID_SHARD_ID) { SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); } if (!ReadOnlyTask(task->taskType)) { /* * Any modification on the local execution should enable 2PC. If remote * queries are also ReadOnly, our 2PC logic is smart enough to skip sending * PREPARE to those connections. */ Use2PCForCoordinatedTransaction(); } LogLocalCommand(task); if (isUtilityCommand) { LocallyExecuteUtilityTask(task); MemoryContextSwitchTo(oldContext); MemoryContextReset(loopContext); continue; } PlannedStmt *localPlan = GetCachedLocalPlan(task, distributedPlan); /* * If the plan is already cached, don't need to re-plan, just * acquire necessary locks. */ if (localPlan != NULL) { Query *jobQuery = distributedPlan->workerJob->jobQuery; LOCKMODE lockMode = GetQueryLockMode(jobQuery); Oid relationId = InvalidOid; foreach_declared_oid(relationId, localPlan->relationOids) { LockRelationOid(relationId, lockMode); } } else { int taskNumParams = numParams; Oid *taskParameterTypes = parameterTypes; int taskType = GetTaskQueryType(task); if (task->parametersInQueryStringResolved) { /* * Parameters were removed from the query string so do not pass them * here. Otherwise, we might see errors when passing custom types, * since their OIDs were set to 0 and their type is normally * inferred from */ taskNumParams = 0; taskParameterTypes = NULL; } /* * for concatenated strings, we set queryStringList so that we can access * each query string. */ if (taskType == TASK_QUERY_TEXT_LIST) { List *queryStringList = task->taskQuery.data.queryStringList; totalRowsProcessed += LocallyPlanAndExecuteMultipleQueries(queryStringList, tupleDest, task); MemoryContextSwitchTo(oldContext); MemoryContextReset(loopContext); continue; } if (taskType != TASK_QUERY_LOCAL_PLAN) { Query *shardQuery = ParseQueryString(TaskQueryString(task), taskParameterTypes, taskNumParams); int cursorOptions = CURSOR_OPT_PARALLEL_OK; /* * Altough the shardQuery is local to this node, we prefer planner() * over standard_planner(). The primary reason for that is Citus itself * is not very tolarent standard_planner() calls that doesn't go through * distributed_planner() because of the way that restriction hooks are * implemented. So, let planner to call distributed_planner() which * eventually calls standard_planner(). */ localPlan = planner(shardQuery, NULL, cursorOptions, paramListInfo); } else { ereport(DEBUG2, (errmsg( "Local executor: Using task's cached local plan for task %u", task->taskId))); localPlan = TaskQueryLocalPlan(task); } } char *shardQueryString = NULL; if (GetTaskQueryType(task) == TASK_QUERY_TEXT) { shardQueryString = TaskQueryString(task); } else { /* avoid the overhead of deparsing when using local execution */ shardQueryString = ""; } totalRowsProcessed += LocallyExecuteTaskPlan(localPlan, shardQueryString, tupleDest, task, paramListInfo); MemoryContextSwitchTo(oldContext); MemoryContextReset(loopContext); } return totalRowsProcessed; } /* * SetColocationIdAndPartitionKeyValueForTasks sets colocationId and partitionKeyValue * for the tasks in the taskList. */ static void SetColocationIdAndPartitionKeyValueForTasks(List *taskList, Job *workerJob) { if (workerJob->colocationId != INVALID_COLOCATION_ID) { Task *task = NULL; foreach_declared_ptr(task, taskList) { task->colocationId = workerJob->colocationId; task->partitionKeyValue = workerJob->partitionKeyValue; } } } /* * LocallyPlanAndExecuteMultipleQueries plans and executes the given query strings * one by one. */ static uint64 LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tupleDest, Task *task) { char *queryString = NULL; uint64 totalProcessedRows = 0; foreach_declared_ptr(queryString, queryStrings) { Query *shardQuery = ParseQueryString(queryString, NULL, 0); int cursorOptions = 0; ParamListInfo paramListInfo = NULL; PlannedStmt *localPlan = planner(shardQuery, NULL, cursorOptions, paramListInfo); totalProcessedRows += LocallyExecuteTaskPlan(localPlan, queryString, tupleDest, task, paramListInfo); } return totalProcessedRows; } /* * ExtractParametersForLocalExecution extracts parameter types and values * from the given ParamListInfo structure, and fills parameter type and * value arrays. It does not change the oid of custom types, because the * query will be run locally. */ void ExtractParametersForLocalExecution(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues) { ExtractParametersFromParamList(paramListInfo, parameterTypes, parameterValues, true); } /* * LocallyExecuteUtilityTask runs a utility command via local execution. */ static void LocallyExecuteUtilityTask(Task *task) { /* keep the parity with multi-node clusters */ RecordNonDistTableAccessesForTask(task); /* * If we roll back to a savepoint, we may no longer be in a query on * a shard. Reset the value as we go back up the stack. */ uint64 prevLocalExecutorShardId = LocalExecutorShardId; if (task->anchorShardId != INVALID_SHARD_ID) { LocalExecutorShardId = task->anchorShardId; } PG_TRY(); { ExecuteUtilityCommand(TaskQueryString(task)); } PG_CATCH(); { LocalExecutorShardId = prevLocalExecutorShardId; PG_RE_THROW(); } PG_END_TRY(); LocalExecutorShardId = prevLocalExecutorShardId; } /* * ExecuteUtilityCommand executes the given task query in the current * session. */ void ExecuteUtilityCommand(const char *taskQueryCommand) { List *parseTreeList = pg_parse_query(taskQueryCommand); RawStmt *taskRawStmt = NULL; foreach_declared_ptr(taskRawStmt, parseTreeList) { Node *taskRawParseTree = taskRawStmt->stmt; /* * The query passed to this function would mostly be a utility * command. However, some utility commands trigger udf calls * (e.g alter_columnar_table_set()). In that case, we execute * the query with the udf call in below conditional block. */ if (IsA(taskRawParseTree, SelectStmt)) { /* we have no external parameters to rewrite the UDF call RawStmt */ Query *udfTaskQuery = RewriteRawQueryStmt(taskRawStmt, taskQueryCommand, NULL, 0); ExecuteUdfTaskQuery(udfTaskQuery); } else { /* * It is a regular utility command we should execute it via * process utility. */ ProcessUtilityParseTree(taskRawParseTree, taskQueryCommand, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } } } /* * ExecuteUdfTaskQuery executes the given udf command. A udf command * is simply a "SELECT udf_call()" query and so it cannot be executed * via process utility. */ static void ExecuteUdfTaskQuery(Query *udfTaskQuery) { /* we do not expect any results */ ExecuteQueryIntoDestReceiver(udfTaskQuery, NULL, None_Receiver); } /* * LogLocalCommand logs commands executed locally on this node. Although we're * talking about local execution, the function relies on citus.log_remote_commands * GUC. This makes sense because the local execution is still on a shard of a * distributed table, meaning it is part of distributed execution. */ static void LogLocalCommand(Task *task) { if (!(LogRemoteCommands || LogLocalCommands)) { return; } const char *command = TaskQueryString(task); if (!CommandMatchesLogGrepPattern(command)) { return; } ereport(NOTICE, (errmsg("executing the command locally: %s", command))); } /* * ExtractLocalAndRemoteTasks gets a taskList and generates two * task lists namely localTaskList and remoteTaskList. The function goes * over the input taskList and puts the tasks that are local to the node * into localTaskList and the remaining to the remoteTaskList. Either of * the lists could be NIL depending on the input taskList. * * One slightly different case is modifications to replicated tables * (e.g., reference tables) where a single task ends in two separate tasks * and the local task is added to localTaskList and the remaining ones to * the remoteTaskList. */ void ExtractLocalAndRemoteTasks(bool readOnly, List *taskList, List **localTaskList, List **remoteTaskList) { *remoteTaskList = NIL; *localTaskList = NIL; Task *task = NULL; foreach_declared_ptr(task, taskList) { List *localTaskPlacementList = NULL; List *remoteTaskPlacementList = NULL; SplitLocalAndRemotePlacements( task->taskPlacementList, &localTaskPlacementList, &remoteTaskPlacementList); /* either the local or the remote should be non-nil */ Assert(!(localTaskPlacementList == NIL && remoteTaskPlacementList == NIL)); if (localTaskPlacementList == NIL) { *remoteTaskList = lappend(*remoteTaskList, task); } else if (remoteTaskPlacementList == NIL) { *localTaskList = lappend(*localTaskList, task); } else { /* * At this point, we're dealing with a task that has placements on both * local and remote nodes. */ Task *localTask = copyObject(task); localTask->partiallyLocalOrRemote = true; localTask->taskPlacementList = localTaskPlacementList; *localTaskList = lappend(*localTaskList, localTask); if (readOnly) { /* read-only tasks should only be executed on the local machine */ } else { /* since shard replication factor > 1, we should have at least 1 remote task */ Assert(remoteTaskPlacementList != NIL); Task *remoteTask = copyObject(task); remoteTask->partiallyLocalOrRemote = true; remoteTask->taskPlacementList = remoteTaskPlacementList; *remoteTaskList = lappend(*remoteTaskList, remoteTask); } } } } /* * SplitLocalAndRemotePlacements is a helper function which iterates over the * input taskPlacementList and puts the placements into corresponding list of * either localTaskPlacementList or remoteTaskPlacementList. */ static void SplitLocalAndRemotePlacements(List *taskPlacementList, List **localTaskPlacementList, List **remoteTaskPlacementList) { int32 localGroupId = GetLocalGroupId(); *localTaskPlacementList = NIL; *remoteTaskPlacementList = NIL; ShardPlacement *taskPlacement = NULL; foreach_declared_ptr(taskPlacement, taskPlacementList) { if (taskPlacement->groupId == localGroupId) { *localTaskPlacementList = lappend(*localTaskPlacementList, taskPlacement); } else { *remoteTaskPlacementList = lappend(*remoteTaskPlacementList, taskPlacement); } } } /* * ExecuteLocalTaskPlan gets a planned statement which can be executed locally. * The function simply follows the steps to have a local execution, sets the * tupleStore if necessary. The function returns the number of rows affected in * case of DML. */ static uint64 LocallyExecuteTaskPlan(PlannedStmt *taskPlan, char *queryString, TupleDestination *tupleDest, Task *task, ParamListInfo paramListInfo) { volatile uint64 processedRows = 0; /* * If we roll back to a savepoint, we may no longer be in a query on * a shard. Reset the value as we go back up the stack. */ uint64 prevLocalExecutorShardId = LocalExecutorShardId; if (task->anchorShardId != INVALID_SHARD_ID) { LocalExecutorShardId = task->anchorShardId; } char *partitionKeyValueString = NULL; if (task->partitionKeyValue != NULL) { partitionKeyValueString = DatumToString(task->partitionKeyValue->constvalue, task->partitionKeyValue->consttype); } AttributeTask(partitionKeyValueString, task->colocationId, taskPlan->commandType); PG_TRY(); { processedRows = ExecuteTaskPlan(taskPlan, queryString, tupleDest, task, paramListInfo); } PG_CATCH(); { LocalExecutorShardId = prevLocalExecutorShardId; PG_RE_THROW(); } PG_END_TRY(); LocalExecutorShardId = prevLocalExecutorShardId; return processedRows; } /* * ExecuteTaskPlan executes the given planned statement and writes the results * to tupleDest. */ static uint64 ExecuteTaskPlan(PlannedStmt *taskPlan, char *queryString, TupleDestination *tupleDest, Task *task, ParamListInfo paramListInfo) { ScanDirection scanDirection = ForwardScanDirection; QueryEnvironment *queryEnv = create_queryEnv(); int eflags = 0; uint64 totalRowsProcessed = 0; RecordNonDistTableAccessesForTask(task); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "ExecuteTaskPlan", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); /* * Some tuple destinations look at task->taskPlacementList to determine * where the result came from using the placement index. Since a local * task can only ever have 1 placement, we set the index to 0. */ int localPlacementIndex = 0; /* * Use the tupleStore provided by the scanState because it is shared across * the other task executions and the adaptive executor. * * Also note that as long as the tupleDest is provided, local execution always * stores the tuples. This is also valid for partiallyLocalOrRemote tasks * as well. */ DestReceiver *destReceiver = tupleDest ? CreateTupleDestDestReceiver(tupleDest, task, localPlacementIndex) : CreateDestReceiver(DestNone); QueryDesc *queryDesc = CreateQueryDesc( taskPlan, /* PlannedStmt *plannedstmt */ queryString, /* const char *sourceText */ GetActiveSnapshot(), /* Snapshot snapshot */ InvalidSnapshot, /* Snapshot crosscheck_snapshot */ destReceiver, /* DestReceiver *dest */ paramListInfo, /* ParamListInfo params */ queryEnv, /* QueryEnvironment *queryEnv */ 0 /* int instrument_options */ ); ExecutorStart(queryDesc, eflags); /* run the plan: count = 0 (all rows) */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+ dropped the “execute_once” boolean */ ExecutorRun(queryDesc, scanDirection, 0L); #else /* PG 17 and prevs still expect the 4th ‘once’ argument */ ExecutorRun(queryDesc, scanDirection, 0L, true); #endif /* * We'll set the executorState->es_processed later, for now only remember * the count. */ if (taskPlan->commandType != CMD_SELECT) { totalRowsProcessed = queryDesc->estate->es_processed; } ExecutorFinish(queryDesc); ExecutorEnd(queryDesc); FreeQueryDesc(queryDesc); MemoryContextSwitchTo(oldContext); MemoryContextDelete(localContext); return totalRowsProcessed; } /* * RecordNonDistTableAccessesForTask records relation accesses for the non-distributed * relations that given task will access (if any). */ static void RecordNonDistTableAccessesForTask(Task *task) { List *taskPlacementList = task->taskPlacementList; if (list_length(taskPlacementList) == 0) { /* * We should never get here, but prefer to throw an error over crashing * if we're wrong. */ ereport(ERROR, (errmsg("shard " UINT64_FORMAT " does not have any shard " "placements", task->anchorShardId))); } /* * We use only the first placement to find the relation accesses. It is * sufficient as PlacementAccessListForTask iterates relationShardList * field of the task and generates accesses per relation in the task. * As we are only interested in relations, not the placements, we can * skip rest of the placements. * Also, here we don't need to iterate relationShardList field of task * to mark each accessed relation because PlacementAccessListForTask * already computes and returns relations that task accesses. */ ShardPlacement *taskPlacement = linitial(taskPlacementList); List *placementAccessList = PlacementAccessListForTask(task, taskPlacement); ShardPlacementAccess *placementAccess = NULL; foreach_declared_ptr(placementAccess, placementAccessList) { uint64 placementAccessShardId = placementAccess->placement->shardId; if (placementAccessShardId == INVALID_SHARD_ID) { /* * When a SELECT prunes down to 0 shard, we still may pass through * the local executor. In that case, we don't need to record any * relation access as we don't actually access any shard placement. */ continue; } Oid accessedRelationId = RelationIdForShard(placementAccessShardId); ShardPlacementAccessType shardPlacementAccessType = placementAccess->accessType; RecordRelationAccessIfNonDistTable(accessedRelationId, shardPlacementAccessType); } } /* * SetLocalExecutionStatus sets the local execution status to * the given status, it errors if the transition is not possible from the * current status. */ void SetLocalExecutionStatus(LocalExecutionStatus newStatus) { EnsureTransitionPossible(GetCurrentLocalExecutionStatus(), newStatus); CurrentLocalExecutionStatus = newStatus; } /* * EnsureTransitionPossible errors if we cannot switch to the 'to' status * from the 'from' status. */ static void EnsureTransitionPossible(LocalExecutionStatus from, LocalExecutionStatus to) { if (from == LOCAL_EXECUTION_REQUIRED && to == LOCAL_EXECUTION_DISABLED) { ereport(ERROR, (errmsg( "cannot switch local execution status from local execution required " "to local execution disabled since it can cause " "visibility problems in the current transaction"))); } if (from == LOCAL_EXECUTION_DISABLED && to == LOCAL_EXECUTION_REQUIRED) { ereport(ERROR, (errmsg( "cannot switch local execution status from local execution disabled " "to local execution enabled since it can cause " "visibility problems in the current transaction"))); } } /* * ShouldExecuteTasksLocally gets a task list and returns true if the * any of the tasks should be executed locally. This function does not * guarantee that any task have to be executed locally. */ bool ShouldExecuteTasksLocally(List *taskList) { if (!EnableLocalExecution) { return false; } if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_DISABLED) { /* * if the current transaction accessed the local node over a connection * then we can't use local execution because of visibility problems. */ return false; } if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_REQUIRED) { /* * If we already used local execution for a previous command * we should stick to it for read-your-writes policy, this can be a * case when we are inside a transaction block. Such as: * * BEGIN; * some-command; -- executed via local execution * another-command; -- this should be executed via local execution for visibility * COMMIT; * * We may need to use local execution even if we are not inside a transaction block, * however the state will go back to LOCAL_EXECUTION_OPTIONAL at the end of transaction. */ return true; } bool singleTask = (list_length(taskList) == 1); if (singleTask && TaskAccessesLocalNode((Task *) linitial(taskList))) { /* * This is the valuable time to use the local execution. We are likely * to avoid any network round-trips by simply executing the command * within this session. * * We cannot avoid network round trips if the task is not a read only * task and accesses multiple placements. For example, modifications to * distributed tables (with replication factor == 1) would avoid network * round-trips. However, modifications to reference tables still needs * to go to over the network to do the modification on the other placements. * Still, we'll be avoding the network round trip for this node. * * Note that we shouldn't use local execution if any distributed execution * has happened because that'd break transaction visibility rules and * many other things. */ return true; } if (!singleTask) { /* * For multi-task executions, we prefer to use connections for parallelism, * except for two cases. First, when in a multi-statement transaction since * there could be other commands that require local execution. Second, the * task list already requires sequential execution. In that case, connection * establishment becomes an unnecessary operation. */ return (IsMultiStatementTransaction() || ShouldRunTasksSequentially(taskList)) && AnyTaskAccessesLocalNode(taskList); } return false; } /* * AnyTaskAccessesLocalNode returns true if a task within the task list accesses * to the local node. */ bool AnyTaskAccessesLocalNode(List *taskList) { Task *task = NULL; foreach_declared_ptr(task, taskList) { if (TaskAccessesLocalNode(task)) { return true; } } return false; } /* * TaskAccessesLocalNode returns true if any placements of the task reside on * the node that we're executing the query. */ bool TaskAccessesLocalNode(Task *task) { int32 localGroupId = GetLocalGroupId(); ShardPlacement *taskPlacement = NULL; foreach_declared_ptr(taskPlacement, task->taskPlacementList) { if (taskPlacement->groupId == localGroupId) { return true; } } return false; } /* * EnsureCompatibleLocalExecutionState makes sure that the tasks won't have * any visibility problems because of local execution. */ void EnsureCompatibleLocalExecutionState(List *taskList) { /* * We have LOCAL_EXECUTION_REQUIRED check here to avoid unnecessarily * iterating the task list in AnyTaskAccessesLocalNode. */ if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_REQUIRED && AnyTaskAccessesLocalNode(taskList)) { ErrorIfTransactionAccessedPlacementsLocally(); } } /* * ErrorIfTransactionAccessedPlacementsLocally errors out if a local query * on any shard has already been executed in the same transaction. * * This check is required because Citus currently hasn't implemented local * execution infrastructure for all the commands/executors. As we implement * local execution for the command/executor that this function call exists, * we should simply remove the check. */ void ErrorIfTransactionAccessedPlacementsLocally(void) { if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_REQUIRED) { ereport(ERROR, (errmsg("cannot execute command because a local execution has " "accessed a placement in the transaction"), errhint("Try re-running the transaction with " "\"SET LOCAL citus.enable_local_execution TO OFF;\""), errdetail("Some parallel commands cannot be executed if a " "previous command has already been executed locally"))); } } /* * DisableLocalExecution is simply a C interface for setting the following: * SET LOCAL citus.enable_local_execution TO off; */ void DisableLocalExecution(void) { set_config_option("citus.enable_local_execution", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } ================================================ FILE: src/backend/distributed/executor/merge_executor.c ================================================ /*------------------------------------------------------------------------- * * merge_executor.c * * Executor logic for MERGE SQL statement. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "distributed/distributed_execution_locks.h" #include "distributed/insert_select_executor.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/merge_executor.h" #include "distributed/merge_planner.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_router_planner.h" #include "distributed/repartition_executor.h" #include "distributed/stats/stat_counters.h" #include "distributed/subplan_execution.h" static void ExecuteSourceAtWorkerAndRepartition(CitusScanState *scanState); static void ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState); static HTAB * ExecuteMergeSourcePlanIntoColocatedIntermediateResults(Oid targetRelationId, Query *mergeQuery, List * sourceTargetList, PlannedStmt * sourcePlan, EState * executorState, char * intermediateResultIdPrefix, int partitionColumnIndex) ; /* * NonPushableMergeCommandExecScan performs an MERGE INTO distributed_table * USING (source-query) ... command. This can be done either by aggregating * task results at the coordinator and repartitioning the results, or by * repartitioning task results and directly transferring data between nodes. */ TupleTableSlot * NonPushableMergeCommandExecScan(CustomScanState *node) { CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *distributedPlan = scanState->distributedPlan; if (!scanState->finishedRemoteScan) { switch (distributedPlan->modifyWithSelectMethod) { case MODIFY_WITH_SELECT_REPARTITION: { ExecuteSourceAtWorkerAndRepartition(scanState); break; } case MODIFY_WITH_SELECT_VIA_COORDINATOR: { ExecuteSourceAtCoordAndRedistribution(scanState); break; } default: { ereport(ERROR, (errmsg("Unexpected MERGE execution method(%d)", distributedPlan->modifyWithSelectMethod))); } } scanState->finishedRemoteScan = true; } TupleTableSlot *resultSlot = ReturnTupleFromTuplestore(scanState); return resultSlot; } /* * ExecuteSourceAtWorkerAndRepartition Executes the Citus distributed plan, including any * sub-plans, and captures the results in intermediate files. Subsequently, redistributes * the result files to ensure colocation with the target, and directs the MERGE SQL * operation to the target shards on the worker nodes, utilizing the colocated * intermediate files as the data source. */ static void ExecuteSourceAtWorkerAndRepartition(CitusScanState *scanState) { DistributedPlan *distributedPlan = scanState->distributedPlan; Query *mergeQuery = copyObject(distributedPlan->modifyQueryViaCoordinatorOrRepartition); RangeTblEntry *targetRte = ExtractResultRelationRTE(mergeQuery); RangeTblEntry *sourceRte = ExtractMergeSourceRangeTableEntry(mergeQuery, false); Oid targetRelationId = targetRte->relid; bool hasReturning = distributedPlan->expectResults; Query *sourceQuery = sourceRte->subquery; PlannedStmt *sourcePlan = copyObject(distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition); EState *executorState = ScanStateGetExecutorState(scanState); /* * If we are dealing with partitioned table, we also need to lock its * partitions. Here we only lock targetRelation, we acquire necessary * locks on source tables during execution of those source queries. */ if (PartitionedTable(targetRelationId)) { LockPartitionRelations(targetRelationId, RowExclusiveLock); } bool randomAccess = true; bool interTransactions = false; DistributedPlan *distSourcePlan = GetDistributedPlan((CustomScan *) sourcePlan->planTree); Job *distSourceJob = distSourcePlan->workerJob; List *distSourceTaskList = distSourceJob->taskList; bool binaryFormat = CanUseBinaryCopyFormatForTargetList(sourceQuery->targetList); ereport(DEBUG1, (errmsg("Executing subplans of the source query and " "storing the results at the respective node(s)"))); ExecuteSubPlans(distSourcePlan, RequestedForExplainAnalyze(scanState)); /* * We have a separate directory for each transaction, so choosing * the same result prefix won't cause filename conflicts. Results * directory name also includes node id and database id, so we don't * need to include them in the filename. We include job id here for * the case "MERGE USING " is executed recursively. */ StringInfo distResultPrefixString = makeStringInfo(); appendStringInfo(distResultPrefixString, "repartitioned_results_" UINT64_FORMAT, distSourceJob->jobId); char *distResultPrefix = distResultPrefixString->data; CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(targetRelationId); ereport(DEBUG1, (errmsg("Redistributing source result rows across nodes"))); /* * partitionColumnIndex determines the column in the selectTaskList to * use for (re)partitioning of the source result, which will colocate * the result data with the target. */ int partitionColumnIndex = distributedPlan->sourceResultRepartitionColumnIndex; /* * Below call partitions the results using shard ranges and partition method of * targetRelation, and then colocates the result files with shards. These * transfers are done by calls to fetch_intermediate_results() between nodes. */ List **redistributedResults = RedistributeTaskListResults(distResultPrefix, distSourceTaskList, partitionColumnIndex, targetRelation, binaryFormat); if (list_length(distSourceTaskList) <= 1) { /* * Probably we will never get here for a repartitioned MERGE * because when the source is a single shard table, we should * most probably choose to use ExecuteSourceAtCoordAndRedistribution(), * but we still keep this here. */ IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } ereport(DEBUG1, (errmsg("Executing final MERGE on workers using " "intermediate results"))); /* * At this point source query has been executed on workers and results * have been fetched in such a way that they are colocated with corresponding * target shard(s). Create and execute a list of tasks of form * MERGE INTO ... USING SELECT * FROM read_intermediate_results(...); */ List *taskList = GenerateTaskListWithRedistributedResults(mergeQuery, targetRelation, redistributedResults, binaryFormat); scanState->tuplestorestate = tuplestore_begin_heap(randomAccess, interTransactions, work_mem); ParamListInfo paramListInfo = executorState->es_param_list_info; TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); TupleDestination *tupleDest = CreateTupleStoreTupleDest(scanState->tuplestorestate, tupleDescriptor); uint64 rowsMerged = ExecuteTaskListIntoTupleDestWithParam(ROW_MODIFY_NONCOMMUTATIVE, taskList, tupleDest, hasReturning, paramListInfo); if (list_length(taskList) <= 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } executorState->es_processed = rowsMerged; } /* * ExecuteSourceAtCoordAndRedistribution Executes the plan that necessitates evaluation * at the coordinator and redistributes the resulting rows to intermediate files, * ensuring colocation with the target shards. Directs the MERGE SQL operation to the * target shards on the worker nodes, utilizing the colocated intermediate files as the * data source. */ void ExecuteSourceAtCoordAndRedistribution(CitusScanState *scanState) { EState *executorState = ScanStateGetExecutorState(scanState); DistributedPlan *distributedPlan = scanState->distributedPlan; Query *mergeQuery = copyObject(distributedPlan->modifyQueryViaCoordinatorOrRepartition); RangeTblEntry *targetRte = ExtractResultRelationRTE(mergeQuery); RangeTblEntry *sourceRte = ExtractMergeSourceRangeTableEntry(mergeQuery, false); Query *sourceQuery = sourceRte->subquery; Oid targetRelationId = targetRte->relid; PlannedStmt *sourcePlan = copyObject(distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition); char *intermediateResultIdPrefix = distributedPlan->intermediateResultIdPrefix; bool hasReturning = distributedPlan->expectResults; bool hasNotMatchedBySource = HasMergeNotMatchedBySource(mergeQuery); int partitionColumnIndex = distributedPlan->sourceResultRepartitionColumnIndex; /* * If we are dealing with partitioned table, we also need to lock its * partitions. Here we only lock targetRelation, we acquire necessary * locks on source tables during execution of those source queries. */ if (PartitionedTable(targetRelationId)) { LockPartitionRelations(targetRelationId, RowExclusiveLock); } ereport(DEBUG1, (errmsg("Collect source query results on coordinator"))); List *prunedTaskList = NIL, *emptySourceTaskList = NIL; HTAB *shardStateHash = ExecuteMergeSourcePlanIntoColocatedIntermediateResults( targetRelationId, mergeQuery, sourceQuery->targetList, sourcePlan, executorState, intermediateResultIdPrefix, partitionColumnIndex); ereport(DEBUG1, (errmsg("Create a MERGE task list that needs to be routed"))); /* generate tasks for the .. phase */ List *taskList = GenerateTaskListWithColocatedIntermediateResults(targetRelationId, mergeQuery, intermediateResultIdPrefix); /* * We cannot actually execute MERGE INTO ... tasks that read from * intermediate results that weren't created because no rows were * written to them. Prune those tasks out by only including tasks * on shards with connections; however, if the MERGE INTO includes * a NOT MATCHED BY SOURCE clause we need to include the task. */ Task *task = NULL; foreach_declared_ptr(task, taskList) { uint64 shardId = task->anchorShardId; bool shardModified = false; hash_search(shardStateHash, &shardId, HASH_FIND, &shardModified); if (shardModified) { prunedTaskList = lappend(prunedTaskList, task); } else if (hasNotMatchedBySource) { emptySourceTaskList = lappend(emptySourceTaskList, task); } } if (emptySourceTaskList != NIL) { ereport(DEBUG1, (errmsg("MERGE has NOT MATCHED BY SOURCE clause, " "execute MERGE on all shards"))); AdjustTaskQueryForEmptySource(targetRelationId, mergeQuery, emptySourceTaskList, intermediateResultIdPrefix); prunedTaskList = list_concat(prunedTaskList, emptySourceTaskList); } if (prunedTaskList == NIL) { /* * No task to execute, but we still increment STAT_QUERY_EXECUTION_SINGLE_SHARD * as per our convention. */ IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); return; } ereport(DEBUG1, (errmsg("Execute MERGE task list"))); bool randomAccess = true; bool interTransactions = false; Assert(scanState->tuplestorestate == NULL); scanState->tuplestorestate = tuplestore_begin_heap(randomAccess, interTransactions, work_mem); TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); ParamListInfo paramListInfo = executorState->es_param_list_info; TupleDestination *tupleDest = CreateTupleStoreTupleDest(scanState->tuplestorestate, tupleDescriptor); uint64 rowsMerged = ExecuteTaskListIntoTupleDestWithParam(ROW_MODIFY_NONCOMMUTATIVE, prunedTaskList, tupleDest, hasReturning, paramListInfo); if (list_length(prunedTaskList) == 1) { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_SINGLE_SHARD); } else { IncrementStatCounterForMyDb(STAT_QUERY_EXECUTION_MULTI_SHARD); } executorState->es_processed = rowsMerged; } /* * ExecuteMergeSourcePlanIntoColocatedIntermediateResults Executes the given PlannedStmt * and inserts tuples into a set of intermediate results that are colocated with the * target table for further processing MERGE INTO. It also returns the hash of shard * states that were used to insert tuplesinto the target relation. */ static HTAB * ExecuteMergeSourcePlanIntoColocatedIntermediateResults(Oid targetRelationId, Query *mergeQuery, List *sourceTargetList, PlannedStmt *sourcePlan, EState *executorState, char *intermediateResultIdPrefix, int partitionColumnIndex) { ParamListInfo paramListInfo = executorState->es_param_list_info; /* Get column name list and partition column index for the target table */ List *columnNameList = BuildColumnNameListFromTargetList(targetRelationId, sourceTargetList); /* * We don't track query counters for the COPY commands that are executed to * prepare intermediate results. */ const bool trackQueryCounters = false; /* set up a DestReceiver that copies into the intermediate file */ const bool publishableData = false; CitusCopyDestReceiver *copyDest = CreateCitusCopyDestReceiver(targetRelationId, columnNameList, partitionColumnIndex, executorState, intermediateResultIdPrefix, publishableData, trackQueryCounters); /* We can skip when writing to intermediate files */ copyDest->skipCoercions = true; ExecutePlanIntoDestReceiver(sourcePlan, paramListInfo, (DestReceiver *) copyDest); executorState->es_processed = copyDest->tuplesSent; XactModificationLevel = XACT_MODIFICATION_DATA; return copyDest->shardStateHash; } ================================================ FILE: src/backend/distributed/executor/multi_executor.c ================================================ /*------------------------------------------------------------------------- * * multi_executor.c * * Entrypoint into distributed query execution. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "commands/copy.h" #include "executor/execdebug.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_oper.h" #include "parser/parsetree.h" #include "storage/lmgr.h" #include "tcop/dest.h" #include "tcop/pquery.h" #include "tcop/utility.h" #include "utils/fmgrprotos.h" #include "utils/memutils.h" #include "utils/snapmgr.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/citus_custom_scan.h" #include "distributed/combine_query_planner.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/multi_executor.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" /* * Controls the connection type for multi shard modifications, DDLs * TRUNCATE and multi-shard SELECT queries. */ int MultiShardConnectionType = PARALLEL_CONNECTION; bool WritableStandbyCoordinator = false; bool AllowModificationsFromWorkersToReplicatedTables = true; /* * Controlled by the GUC citus.skip_constraint_validation */ bool SkipConstraintValidation = false; /* * Setting that controls whether distributed queries should be * allowed within a task execution. */ bool AllowNestedDistributedExecution = false; /* * Pointer to bound parameters of the current ongoing call to ExecutorRun. * If executor is not running, then this value is meaningless. */ ParamListInfo executorBoundParams = NULL; /* sort the returning to get consistent outputs, used only for testing */ bool SortReturning = false; /* * How many nested executors have we started? This can happen for SQL * UDF calls. The outer query starts an executor, then postgres opens * another executor to run the SQL UDF. */ int ExecutorLevel = 0; /* local function forward declarations */ static Relation StubRelation(TupleDesc tupleDescriptor); static char * GetObjectTypeString(ObjectType objType); static bool AlterTableConstraintCheck(QueryDesc *queryDesc); static List * FindCitusCustomScanStates(PlanState *planState); static bool CitusCustomScanStateWalker(PlanState *planState, List **citusCustomScanStates); static bool IsTaskExecutionAllowed(bool isRemote); static bool InLocalTaskExecutionOnShard(void); static bool MaybeInRemoteTaskExecution(void); static bool InTrigger(void); /* * CitusExecutorStart is the ExecutorStart_hook that gets called when * Postgres prepares for execution or EXPLAIN. */ void CitusExecutorStart(QueryDesc *queryDesc, int eflags) { PlannedStmt *plannedStmt = queryDesc->plannedstmt; /* * We cannot modify XactReadOnly on Windows because it is not * declared with PGDLLIMPORT. */ #ifndef WIN32 if (RecoveryInProgress() && WritableStandbyCoordinator && IsCitusPlan(plannedStmt->planTree)) { PG_TRY(); { /* * To enable writes from a hot standby we cheat our way through * the checks in standard_ExecutorStart by temporarily setting * XactReadOnly to false. */ XactReadOnly = false; standard_ExecutorStart(queryDesc, eflags); XactReadOnly = true; } PG_CATCH(); { XactReadOnly = true; PG_RE_THROW(); } PG_END_TRY(); } else #endif { standard_ExecutorStart(queryDesc, eflags); } } /* * CitusExecutorRun is the ExecutorRun_hook that gets called when postgres * executes a query. */ void CitusExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { DestReceiver *dest = queryDesc->dest; ParamListInfo savedBoundParams = executorBoundParams; /* * Save a pointer to query params so UDFs can access them by calling * ExecutorBoundParams(). */ executorBoundParams = queryDesc->params; /* * We do some potentially time consuming operations ourself now before we hand off * control to postgres' executor. To make sure that time spent is accurately measured * we remove the totaltime instrumentation from the queryDesc. Instead we will start * and stop the instrumentation of the total time and put it back on the queryDesc * before returning (or rethrowing) from this function. */ Instrumentation *volatile totalTime = queryDesc->totaltime; queryDesc->totaltime = NULL; PG_TRY(); { ExecutorLevel++; if (totalTime) { InstrStartNode(totalTime); } /* * Disable execution of ALTER TABLE constraint validation queries. These * constraints will be validated in worker nodes, so running these queries * from the coordinator would be redundant. * * For example, ALTER TABLE ... ATTACH PARTITION checks that the new * partition doesn't violate constraints of the parent table, which * might involve running some SELECT queries. * * Ideally we'd completely skip these checks in the coordinator, but we don't * have any means to tell postgres to skip the checks. So the best we can do is * to not execute the queries and return an empty result set, as if this table has * no rows, so no constraints will be violated. */ if (AlterTableConstraintCheck(queryDesc)) { EState *estate = queryDesc->estate; estate->es_processed = 0; /* start and shutdown tuple receiver to simulate empty result */ dest->rStartup(queryDesc->dest, CMD_SELECT, queryDesc->tupDesc); dest->rShutdown(dest); } else { /* switch into per-query memory context before calling PreExecScan */ MemoryContext oldcontext = MemoryContextSwitchTo( queryDesc->estate->es_query_cxt); /* * Call PreExecScan for all citus custom scan nodes prior to starting the * postgres exec scan to give some citus scan nodes some time to initialize * state that would be too late if it were to initialize when the first tuple * would need to return. */ List *citusCustomScanStates = FindCitusCustomScanStates(queryDesc->planstate); CitusScanState *citusScanState = NULL; foreach_declared_ptr(citusScanState, citusCustomScanStates) { if (citusScanState->PreExecScan) { citusScanState->PreExecScan(citusScanState); } } /* postgres will switch here again and will restore back on its own */ MemoryContextSwitchTo(oldcontext); #if PG_VERSION_NUM >= PG_VERSION_18 /* PG18+ drops the “execute_once” argument */ standard_ExecutorRun(queryDesc, direction, count); #else /* PG17-: original four-arg signature */ standard_ExecutorRun(queryDesc, direction, count, execute_once); #endif } if (totalTime) { InstrStopNode(totalTime, queryDesc->estate->es_processed); queryDesc->totaltime = totalTime; } executorBoundParams = savedBoundParams; ExecutorLevel--; if (ExecutorLevel == 0 && PlannerLevel == 0) { /* * We are leaving Citus code so no one should have any references to * cache entries. Release them now to not hold onto memory in long * transactions. */ CitusTableCacheFlushInvalidatedEntries(); InTopLevelDelegatedFunctionCall = false; } /* * Within a 2PC, when a function is delegated to a remote node, we pin * the distribution argument as the shard key for all the SQL in the * function's block. The restriction is imposed to not to access other * nodes from the current node, and violate the transactional integrity * of the 2PC. Now that the query is ending, reset the shard key to NULL. */ CheckAndResetAllowedShardKeyValueIfNeeded(); } PG_CATCH(); { if (totalTime) { queryDesc->totaltime = totalTime; } executorBoundParams = savedBoundParams; ExecutorLevel--; if (ExecutorLevel == 0 && PlannerLevel == 0) { InTopLevelDelegatedFunctionCall = false; } /* * In case of an exception, reset the pinned shard-key, for more * details see the function header. */ CheckAndResetAllowedShardKeyValueIfNeeded(); PG_RE_THROW(); } PG_END_TRY(); } /* * FindCitusCustomScanStates returns a list of all citus custom scan states in it. */ static List * FindCitusCustomScanStates(PlanState *planState) { List *citusCustomScanStates = NIL; CitusCustomScanStateWalker(planState, &citusCustomScanStates); return citusCustomScanStates; } /* * CitusCustomScanStateWalker walks a planState tree structure and adds all * CitusCustomState nodes to the list passed by reference as the second argument. */ static bool CitusCustomScanStateWalker(PlanState *planState, List **citusCustomScanStates) { if (IsCitusCustomState(planState)) { CitusScanState *css = (CitusScanState *) planState; *citusCustomScanStates = lappend(*citusCustomScanStates, css); /* breaks the walking of this tree */ return true; } return planstate_tree_walker(planState, CitusCustomScanStateWalker, citusCustomScanStates); } /* * ReturnTupleFromTuplestore reads the next tuple from the tuple store of the * given Citus scan node and returns it. It returns null if all tuples are read * from the tuple store. */ TupleTableSlot * ReturnTupleFromTuplestore(CitusScanState *scanState) { Tuplestorestate *tupleStore = scanState->tuplestorestate; bool forwardScanDirection = true; if (tupleStore == NULL) { return NULL; } EState *executorState = ScanStateGetExecutorState(scanState); ScanDirection scanDirection = executorState->es_direction; Assert(ScanDirectionIsValid(scanDirection)); if (ScanDirectionIsBackward(scanDirection)) { forwardScanDirection = false; } ExprState *qual = scanState->customScanState.ss.ps.qual; ProjectionInfo *projInfo = scanState->customScanState.ss.ps.ps_ProjInfo; ExprContext *econtext = scanState->customScanState.ss.ps.ps_ExprContext; if (!qual && !projInfo) { /* no quals, nor projections return directly from the tuple store. */ TupleTableSlot *slot = scanState->customScanState.ss.ss_ScanTupleSlot; tuplestore_gettupleslot(tupleStore, forwardScanDirection, false, slot); return slot; } for (;;) { /* * If there is a very selective qual on the Citus Scan node we might block * interrupts for a longer time if we would not check for interrupts in this loop */ CHECK_FOR_INTERRUPTS(); /* * Reset per-tuple memory context to free any expression evaluation * storage allocated in the previous tuple cycle. */ ResetExprContext(econtext); TupleTableSlot *slot = scanState->customScanState.ss.ss_ScanTupleSlot; tuplestore_gettupleslot(tupleStore, forwardScanDirection, false, slot); if (TupIsNull(slot)) { /* * When the tuple is null we have reached the end of the tuplestore. We will * return a null tuple, however, depending on the existence of a projection we * need to either return the scan tuple or the projected tuple. */ if (projInfo) { return ExecClearTuple(projInfo->pi_state.resultslot); } else { return slot; } } /* place the current tuple into the expr context */ econtext->ecxt_scantuple = slot; if (!ExecQual(qual, econtext)) { /* skip nodes that do not satisfy the qual (filter) */ InstrCountFiltered1(scanState, 1); continue; } /* found a satisfactory scan tuple */ if (projInfo) { /* * Form a projection tuple, store it in the result tuple slot and return it. * ExecProj works on the ecxt_scantuple on the context stored earlier. */ return ExecProject(projInfo); } else { /* Here, we aren't projecting, so just return scan tuple */ return slot; } } } /* * ReadFileIntoTupleStore parses the records in a COPY-formatted file according * according to the given tuple descriptor and stores the records in a tuple * store. */ void ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc tupleDescriptor, Tuplestorestate *tupstore) { /* * Trick BeginCopyFrom into using our tuple descriptor by pretending it belongs * to a relation. */ Relation stubRelation = StubRelation(tupleDescriptor); EState *executorState = CreateExecutorState(); MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState); ExprContext *executorExpressionContext = GetPerTupleExprContext(executorState); int columnCount = tupleDescriptor->natts; Datum *columnValues = palloc0(columnCount * sizeof(Datum)); bool *columnNulls = palloc0(columnCount * sizeof(bool)); List *copyOptions = NIL; int location = -1; /* "unknown" token location */ DefElem *copyOption = makeDefElem("format", (Node *) makeString(copyFormat), location); copyOptions = lappend(copyOptions, copyOption); CopyFromState copyState = BeginCopyFrom(NULL, stubRelation, NULL, fileName, false, NULL, NULL, copyOptions); while (true) { ResetPerTupleExprContext(executorState); MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); bool nextRowFound = NextCopyFrom(copyState, executorExpressionContext, columnValues, columnNulls); if (!nextRowFound) { MemoryContextSwitchTo(oldContext); break; } tuplestore_putvalues(tupstore, tupleDescriptor, columnValues, columnNulls); MemoryContextSwitchTo(oldContext); } EndCopyFrom(copyState); pfree(columnValues); pfree(columnNulls); } /* * SortTupleStore gets a CitusScanState and sorts the tuplestore by all the * entries in the target entry list, starting from the first one and * ending with the last entry. * * The sorting is done in ASC order. */ void SortTupleStore(CitusScanState *scanState) { TupleDesc tupleDescriptor = ScanStateGetTupleDescriptor(scanState); Tuplestorestate *tupleStore = scanState->tuplestorestate; List *targetList = scanState->customScanState.ss.ps.plan->targetlist; uint32 expectedColumnCount = list_length(targetList); /* Convert list-ish representation to arrays wanted by executor */ int numberOfSortKeys = expectedColumnCount; AttrNumber *sortColIdx = (AttrNumber *) palloc(numberOfSortKeys * sizeof(AttrNumber)); Oid *sortOperators = (Oid *) palloc(numberOfSortKeys * sizeof(Oid)); Oid *collations = (Oid *) palloc(numberOfSortKeys * sizeof(Oid)); bool *nullsFirst = (bool *) palloc(numberOfSortKeys * sizeof(bool)); int sortKeyIndex = 0; /* * Iterate on the returning target list and generate the necessary information * for sorting the tuples. */ TargetEntry *returningEntry = NULL; foreach_declared_ptr(returningEntry, targetList) { Oid sortop = InvalidOid; /* determine the sortop, we don't need anything else */ get_sort_group_operators(exprType((Node *) returningEntry->expr), true, false, false, &sortop, NULL, NULL, NULL); sortColIdx[sortKeyIndex] = sortKeyIndex + 1; sortOperators[sortKeyIndex] = sortop; collations[sortKeyIndex] = exprCollation((Node *) returningEntry->expr); nullsFirst[sortKeyIndex] = false; sortKeyIndex++; } Tuplesortstate *tuplesortstate = tuplesort_begin_heap(tupleDescriptor, numberOfSortKeys, sortColIdx, sortOperators, collations, nullsFirst, work_mem, NULL, false); while (true) { TupleTableSlot *slot = ReturnTupleFromTuplestore(scanState); if (TupIsNull(slot)) { break; } /* tuplesort_puttupleslot copies the slot into sort context */ tuplesort_puttupleslot(tuplesortstate, slot); } /* perform the actual sort operation */ tuplesort_performsort(tuplesortstate); /* * Truncate the existing tupleStore, because we'll fill it back * from the sorted tuplestore. */ tuplestore_clear(tupleStore); /* iterate over all the sorted tuples, add them to original tuplestore */ while (true) { TupleTableSlot *newSlot = MakeSingleTupleTableSlot(tupleDescriptor, &TTSOpsMinimalTuple); bool found = tuplesort_gettupleslot(tuplesortstate, true, false, newSlot, NULL); if (!found) { break; } /* tuplesort_puttupleslot copies the slot into the tupleStore context */ tuplestore_puttupleslot(tupleStore, newSlot); } tuplestore_rescan(scanState->tuplestorestate); /* terminate the sort, clear unnecessary resources */ tuplesort_end(tuplesortstate); } /* * StubRelation creates a stub Relation from the given tuple descriptor. * To be able to use copy.c, we need a Relation descriptor. As there is no * relation corresponding to the data loaded from workers, we need to fake one. * We just need the bare minimal set of fields accessed by BeginCopyFrom(). */ static Relation StubRelation(TupleDesc tupleDescriptor) { Relation stubRelation = palloc0(sizeof(RelationData)); stubRelation->rd_att = tupleDescriptor; stubRelation->rd_rel = palloc0(sizeof(FormData_pg_class)); stubRelation->rd_rel->relkind = RELKIND_RELATION; return stubRelation; } /* * ExecuteQueryStringIntoDestReceiver plans and executes a query and sends results * to the given DestReceiver. */ void ExecuteQueryStringIntoDestReceiver(const char *queryString, ParamListInfo params, DestReceiver *dest) { Query *query = ParseQueryString(queryString, NULL, 0); ExecuteQueryIntoDestReceiver(query, params, dest); } /* * ParseQuery parses query string and returns a Query struct. */ Query * ParseQueryString(const char *queryString, Oid *paramOids, int numParams) { RawStmt *rawStmt = (RawStmt *) ParseTreeRawStmt(queryString); /* rewrite the parsed RawStmt to produce a Query */ Query *query = RewriteRawQueryStmt(rawStmt, queryString, paramOids, numParams); return query; } /* * RewriteRawQueryStmt rewrites the given parsed RawStmt according to the other * parameters and returns a Query struct. */ Query * RewriteRawQueryStmt(RawStmt *rawStmt, const char *queryString, Oid *paramOids, int numParams) { List *queryTreeList = pg_analyze_and_rewrite_fixedparams(rawStmt, queryString, paramOids, numParams, NULL); if (list_length(queryTreeList) != 1) { ereport(ERROR, (errmsg("can only execute a single query"))); } Query *query = (Query *) linitial(queryTreeList); return query; } /* * ExecuteQueryIntoDestReceiver plans and executes a query and sends results to the given * DestReceiver. */ void ExecuteQueryIntoDestReceiver(Query *query, ParamListInfo params, DestReceiver *dest) { int cursorOptions = CURSOR_OPT_PARALLEL_OK; if (query->commandType == CMD_UTILITY) { /* can only execute DML/SELECT via this path */ ereport(ERROR, (errmsg("cannot execute utility commands"))); } /* plan the subquery, this may be another distributed query */ PlannedStmt *queryPlan = pg_plan_query(query, NULL, cursorOptions, params); ExecutePlanIntoDestReceiver(queryPlan, params, dest); } /* * ExecutePlanIntoDestReceiver executes a query plan and sends results to the given * DestReceiver. */ uint64 ExecutePlanIntoDestReceiver(PlannedStmt *queryPlan, ParamListInfo params, DestReceiver *dest) { int eflags = 0; long count = FETCH_ALL; /* create a new portal for executing the query */ Portal portal = CreateNewPortal(); /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; PortalDefineQuery( portal, NULL, /* no prepared statement name */ "", /* query text */ CMDTAG_SELECT, /* command tag */ list_make1(queryPlan),/* list of PlannedStmt* */ NULL /* no CachedPlan */ ); PortalStart(portal, params, eflags, GetActiveSnapshot()); QueryCompletion qc = { 0 }; #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+: six-arg signature (drop the run_once bool) */ PortalRun(portal, count, /* how many rows to fetch */ false, /* isTopLevel */ dest, /* DestReceiver *dest */ dest, /* DestReceiver *altdest */ &qc); /* QueryCompletion *qc */ #else /* PG 17-: original seven-arg signature */ PortalRun(portal, count, /* how many rows to fetch */ false, /* isTopLevel */ true, /* run_once */ dest, /* DestReceiver *dest */ dest, /* DestReceiver *altdest */ &qc); /* QueryCompletion *qc */ #endif PortalDrop(portal, false); return qc.nprocessed; } /* * SetLocalMultiShardModifyModeToSequential is simply a C interface for setting * the following: * SET LOCAL citus.multi_shard_modify_mode = 'sequential'; */ void SetLocalMultiShardModifyModeToSequential() { set_config_option("citus.multi_shard_modify_mode", "sequential", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } /* * EnsureSequentialMode makes sure that the current transaction is already in * sequential mode, or can still safely be put in sequential mode, it errors if that is * not possible. The error contains information for the user to retry the transaction with * sequential mode set from the beginning. * * Takes an ObjectType to use in the error/debug messages. */ void EnsureSequentialMode(ObjectType objType) { char *objTypeString = GetObjectTypeString(objType); if (ParallelQueryExecutedInTransaction()) { ereport(ERROR, (errmsg("cannot run %s command because there was a " "parallel operation on a distributed table in the " "transaction", objTypeString), errdetail("When running command on/for a distributed %s, Citus " "needs to perform all operations over a single " "connection per node to ensure consistency.", objTypeString), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail( "A command for a distributed %s is run. To make sure subsequent " "commands see the %s correctly we need to make sure to " "use only one connection for all future commands", objTypeString, objTypeString))); SetLocalMultiShardModifyModeToSequential(); } /* * GetObjectTypeString takes an ObjectType and returns the string version of it. * We (for now) call this function only in EnsureSequentialMode, and use the returned * string to generate error/debug messages. * * If GetObjectTypeString gets called with an ObjectType that is not in the switch * statement, the function will return the string "object", and emit a debug message. * In that case, make sure you've added the newly supported type to the switch statement. */ static char * GetObjectTypeString(ObjectType objType) { switch (objType) { case OBJECT_AGGREGATE: { return "aggregate"; } case OBJECT_COLLATION: { return "collation"; } case OBJECT_DATABASE: { return "database"; } case OBJECT_DOMAIN: { return "domain"; } case OBJECT_EXTENSION: { return "extension"; } case OBJECT_FOREIGN_SERVER: { return "foreign server"; } case OBJECT_FUNCTION: { return "function"; } case OBJECT_PUBLICATION: { return "publication"; } case OBJECT_SCHEMA: { return "schema"; } case OBJECT_TSCONFIGURATION: { return "text search configuration"; } case OBJECT_TSDICTIONARY: { return "text search dictionary"; } case OBJECT_TYPE: { return "type"; } case OBJECT_VIEW: { return "view"; } default: { ereport(DEBUG1, (errmsg("unsupported object type"), errdetail("Please add string conversion for the object."))); return "object"; } } } /* * AlterTableConstraintCheck returns if the given query is an ALTER TABLE * constraint check query. * * Postgres uses SPI to execute these queries. To see examples of how these * constraint check queries look like, see RI_Initial_Check() and RI_Fkey_check(). */ static bool AlterTableConstraintCheck(QueryDesc *queryDesc) { if (!AlterTableInProgress()) { return false; } /* * These queries are one or more SELECT queries, where postgres checks * their results either for NULL values or existence of a row at all. */ if (queryDesc->plannedstmt->commandType != CMD_SELECT) { return false; } /* * While an ALTER TABLE is in progress, we might do SELECTs on some * catalog tables too. For example, when dropping a column, citus_drop_trigger() * runs some SELECTs on catalog tables. These are not constraint check queries. */ if (!IsCitusPlan(queryDesc->plannedstmt->planTree)) { return false; } return true; } /* * ExecutorBoundParams returns the bound parameters of the current ongoing call * to ExecutorRun. This is meant to be used by UDFs which need to access bound * parameters. */ ParamListInfo ExecutorBoundParams(void) { Assert(ExecutorLevel > 0); return executorBoundParams; } /* * EnsureTaskExecutionAllowed ensures that we do not perform remote * execution from within a task. That could happen when the user calls * a function in a query that gets pushed down to the worker, and the * function performs a query on a distributed table. */ void EnsureTaskExecutionAllowed(bool isRemote) { if (IsTaskExecutionAllowed(isRemote)) { return; } ereport(ERROR, (errmsg("cannot execute a distributed query from a query on a " "shard"), errdetail("Executing a distributed query in a function call that " "may be pushed to a remote node can lead to incorrect " "results."), errhint("Avoid nesting of distributed queries or use alter user " "current_user set citus.allow_nested_distributed_execution " "to on to allow it with possible incorrectness."))); } /* * IsTaskExecutionAllowed determines whether task execution is currently allowed. * In general, nested distributed execution is not allowed, except in a few cases * (forced function call delegation, triggers). * * We distinguish between local and remote tasks because triggers only disallow * remote task execution. */ static bool IsTaskExecutionAllowed(bool isRemote) { if (AllowNestedDistributedExecution) { /* user explicitly allows nested execution */ return true; } if (!isRemote) { if (AllowedDistributionColumnValue.isActive) { /* * When we are in a forced delegated function call, we explicitly check * whether local tasks use the same distribution column value in * EnsureForceDelegationDistributionKey. */ return true; } if (InTrigger()) { /* * In triggers on shards we only disallow remote tasks. This has a few * reasons: * * - We want to enable access to co-located shards, but do not have additional * checks yet. * - Users need to explicitly set enable_unsafe_triggers in order to create * triggers on distributed tables. * - Triggers on Citus local tables should be able to access other Citus local * tables. */ return true; } } return !InLocalTaskExecutionOnShard() && !MaybeInRemoteTaskExecution(); } /* * InLocalTaskExecutionOnShard returns whether we are currently in the local executor * and it is working on a shard of a distributed table. * * In general, we can allow distributed queries inside of local executor, because * we can correctly assign tasks to connections. However, we preemptively protect * against distributed queries inside of queries on shards of a distributed table, * because those might start failing after a shard move. */ static bool InLocalTaskExecutionOnShard(void) { if (LocalExecutorShardId == INVALID_SHARD_ID) { /* local executor is not active or is processing a task without shards */ return false; } if (!DistributedTableShardId(LocalExecutorShardId)) { /* * Local executor is processing a query on a shard, but the shard belongs * to a reference table or Citus local table. We do not expect those to * move. */ return false; } return true; } /* * MaybeInRemoteTaskExecution returns whether we could in a remote task execution. * * We consider anything that happens in a Citus-internal backend, except deleged * function or procedure calls as a potential task execution. * * This function will also return true in other scenarios, such as during metadata * syncing. However, since this function is mainly used for restricting (dangerous) * nested executions, it is good to be pessimistic. */ static bool MaybeInRemoteTaskExecution(void) { if (!IsCitusInternalBackend()) { /* in a regular, client-initiated backend doing a regular task */ return false; } if (InTopLevelDelegatedFunctionCall || InDelegatedProcedureCall) { /* in a citus-initiated backend, but also in a delegated a procedure call */ return false; } return true; } /* * InTrigger returns whether the execution is currently in a trigger. */ static bool InTrigger(void) { return DatumGetInt32(pg_trigger_depth(NULL)) > 0; } ================================================ FILE: src/backend/distributed/executor/multi_server_executor.c ================================================ /*------------------------------------------------------------------------- * * multi_server_executor.c * * Function definitions for distributed task execution for adaptive * executor. * routines are implemented backend-side logic; and they trigger executions * on the client-side via function hooks that they load. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "utils/lsyscache.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/subplan_execution.h" #include "distributed/tuple_destination.h" #include "distributed/worker_protocol.h" int RemoteTaskCheckInterval = 10; /* per cycle sleep interval in millisecs */ int TaskExecutorType = MULTI_EXECUTOR_ADAPTIVE; /* distributed executor type */ bool EnableRepartitionJoins = false; /* * JobExecutorType selects the executor type for the given distributedPlan using the task * executor type config value. The function then checks if the given distributedPlan needs * more resources than those provided to it by other config values, and issues * warnings accordingly. If the selected executor type cannot execute the given * distributedPlan, the function errors out. */ MultiExecutorType JobExecutorType(DistributedPlan *distributedPlan) { Job *job = distributedPlan->workerJob; if (distributedPlan->modifyQueryViaCoordinatorOrRepartition != NULL) { if (IsMergeQuery(distributedPlan->modifyQueryViaCoordinatorOrRepartition)) { return MULTI_EXECUTOR_NON_PUSHABLE_MERGE_QUERY; } /* * We go through * MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT because * the executor already knows how to handle adaptive * executor when necessary. */ return MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT; } /* * If we have repartition jobs with adaptive executor and repartition * joins are not enabled, error out. */ int dependentJobCount = list_length(job->dependentJobList); if (!EnableRepartitionJoins && dependentJobCount > 0) { ereport(ERROR, (errmsg("the query contains a join that requires repartitioning"), errhint("Set citus.enable_repartition_joins to on to enable " "repartitioning"))); } /* * Debug distribution column value if possible. The distributed planner sometimes * defers creating the tasks, so the task list might be NIL. Still, it sets the * partitionKeyValue and we print it here. */ if (list_length(job->taskList) <= 1 && IsLoggableLevel(DEBUG2)) { Const *partitionValueConst = job->partitionKeyValue; if (partitionValueConst != NULL && !partitionValueConst->constisnull) { Datum partitionColumnValue = partitionValueConst->constvalue; Oid partitionColumnType = partitionValueConst->consttype; char *partitionColumnString = DatumToString(partitionColumnValue, partitionColumnType); ereport(DEBUG2, (errmsg("query has a single distribution column value: " "%s", partitionColumnString))); } } return MULTI_EXECUTOR_ADAPTIVE; } ================================================ FILE: src/backend/distributed/executor/partitioned_intermediate_results.c ================================================ /*------------------------------------------------------------------------- * * partition_intermediate_results.c * Functions for writing partitioned intermediate results. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "port.h" #include "access/hash.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/primnodes.h" #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/typcache.h" #include "distributed/intermediate_results.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/pg_dist_shard.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* * PartitionedResultDestReceiver is used for streaming tuples into a set of * partitioned result files. */ typedef struct PartitionedResultDestReceiver { /* public DestReceiver interface */ DestReceiver pub; /* on lazy startup we only startup the DestReceiver once they receive a tuple */ bool lazyStartup; /* * Stores the arguments passed to the PartidionedResultDestReceiver's rStarup * function. These arguments are reused when lazyStartup has been set to true. * On the processing of a first tuple for a partitionDestReceiver since rStartup it * will pass the arguments here to the rStartup function of partitionDestReceiver to * prepare it for receiving tuples. * * Even though not used without lazyStartup we just always populate these with the * last invoked arguments for our rStartup. */ struct { /* * operation as passed to rStartup, mostly the CmdType of the command being * streamed into this DestReceiver */ int operation; /* * TupleDesc describing the layout of the tuples being streamed into the * DestReceiver. */ TupleDesc tupleDescriptor; } startupArguments; /* which column of streamed tuples to use as partition column */ int partitionColumnIndex; /* The number of partitions being partitioned into */ int partitionCount; /* used for deciding which partition a shard belongs to. */ CitusTableCacheEntry *shardSearchInfo; /* Tuples matching shardSearchInfo[i] are sent to partitionDestReceivers[i]. */ DestReceiver **partitionDestReceivers; /* keeping track of which partitionDestReceivers have been started */ Bitmapset *startedDestReceivers; /* whether NULL partition column values are allowed */ bool allowNullPartitionColumnValues; } PartitionedResultDestReceiver; static Portal StartPortalForQueryExecution(const char *queryString); static void PartitionedResultDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static bool PartitionedResultDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); static void PartitionedResultDestReceiverShutdown(DestReceiver *dest); static void PartitionedResultDestReceiverDestroy(DestReceiver *copyDest); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(worker_partition_query_result); /* * worker_partition_query_result executes a query and writes the results into a * set of local files according to the partition scheme and the partition column. */ Datum worker_partition_query_result(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ReturnSetInfo *resultInfo = (ReturnSetInfo *) fcinfo->resultinfo; text *resultIdPrefixText = PG_GETARG_TEXT_P(0); char *resultIdPrefixString = text_to_cstring(resultIdPrefixText); /* verify that resultIdPrefix doesn't contain invalid characters */ QueryResultFileName(resultIdPrefixString); text *queryText = PG_GETARG_TEXT_P(1); char *queryString = text_to_cstring(queryText); int partitionColumnIndex = PG_GETARG_INT32(2); Oid partitionMethodOid = PG_GETARG_OID(3); char partitionMethod = LookupDistributionMethod(partitionMethodOid); if (partitionMethod != DISTRIBUTE_BY_HASH && partitionMethod != DISTRIBUTE_BY_RANGE) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("only hash and range partitiong schemes are supported"))); } ArrayType *minValuesArray = PG_GETARG_ARRAYTYPE_P(4); int32 minValuesCount = ArrayObjectCount(minValuesArray); ArrayType *maxValuesArray = PG_GETARG_ARRAYTYPE_P(5); int32 maxValuesCount = ArrayObjectCount(maxValuesArray); bool binaryCopy = PG_GETARG_BOOL(6); bool allowNullPartitionColumnValues = PG_GETARG_BOOL(7); bool generateEmptyResults = PG_GETARG_BOOL(8); if (!IsMultiStatementTransaction()) { ereport(ERROR, (errmsg("worker_partition_query_result can only be used in a " "transaction block"))); } /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ EnsureDistributedTransactionId(); CreateIntermediateResultsDirectory(); if (minValuesCount != maxValuesCount) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "min values and max values must have the same number of elements"))); } int partitionCount = minValuesCount; if (partitionCount == 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of partitions cannot be 0"))); } /* start execution early in order to extract the tuple descriptor */ Portal portal = StartPortalForQueryExecution(queryString); /* extract the partition column */ TupleDesc tupleDescriptor = portal->tupDesc; if (tupleDescriptor == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("query must generate a set of rows"))); } if (partitionColumnIndex < 0 || partitionColumnIndex >= tupleDescriptor->natts) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("partition column index must be between 0 and %d", tupleDescriptor->natts - 1))); } FormData_pg_attribute *partitionColumnAttr = TupleDescAttr(tupleDescriptor, partitionColumnIndex); Var *partitionColumn = makeVar(partitionColumnIndex, partitionColumnIndex, partitionColumnAttr->atttypid, partitionColumnAttr->atttypmod, partitionColumnAttr->attcollation, 0); /* construct an artificial CitusTableCacheEntry for shard pruning */ CitusTableCacheEntry *shardSearchInfo = QueryTupleShardSearchInfo(minValuesArray, maxValuesArray, partitionMethod, partitionColumn); /* prepare the output destination */ EState *estate = CreateExecutorState(); MemoryContext tupleContext = GetPerTupleMemoryContext(estate); /* create all dest receivers */ DestReceiver **dests = palloc0(partitionCount * sizeof(DestReceiver *)); for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) { StringInfo resultId = makeStringInfo(); appendStringInfo(resultId, "%s_%d", resultIdPrefixString, partitionIndex); char *filePath = QueryResultFileName(resultId->data); DestReceiver *partitionDest = CreateFileDestReceiver(filePath, tupleContext, binaryCopy); dests[partitionIndex] = partitionDest; } /* * If we are asked to generated empty results, use non-lazy startup. * * The rStartup of the FileDestReceiver will be called for all partitions * and generate empty files, which may still have binary header/footer. */ const bool lazyStartup = !generateEmptyResults; DestReceiver *dest = CreatePartitionedResultDestReceiver( partitionColumnIndex, partitionCount, shardSearchInfo, dests, lazyStartup, allowNullPartitionColumnValues); /* execute the query */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG18+: drop the “run_once” bool */ PortalRun(portal, FETCH_ALL, /* count */ false, /* isTopLevel */ dest, /* dest receiver */ dest, /* alternative dest */ NULL); /* QueryCompletion *qc */ #else /* PG15–17: original seven‐arg signature */ PortalRun(portal, FETCH_ALL, /* count */ false, /* isTopLevel */ true, /* run_once */ dest, /* dest receiver */ dest, /* alternative dest */ NULL); /* QueryCompletion *qc */ #endif /* construct the output result */ TupleDesc returnTupleDesc = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &returnTupleDesc); resultInfo->returnMode = SFRM_Materialize; resultInfo->setResult = tupleStore; resultInfo->setDesc = returnTupleDesc; for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) { uint64 recordsWritten = 0; uint64 bytesWritten = 0; Datum values[3]; bool nulls[3]; FileDestReceiverStats(dests[partitionIndex], &recordsWritten, &bytesWritten); memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = Int32GetDatum(partitionIndex); values[1] = UInt64GetDatum(recordsWritten); values[2] = UInt64GetDatum(bytesWritten); tuplestore_putvalues(tupleStore, returnTupleDesc, values, nulls); } PortalDrop(portal, false); FreeExecutorState(estate); dest->rDestroy(dest); PG_RETURN_INT64(1); } /* * StartPortalForQueryExecution creates and starts a portal which can be * used for running the given query. */ static Portal StartPortalForQueryExecution(const char *queryString) { Query *query = ParseQueryString(queryString, NULL, 0); int cursorOptions = CURSOR_OPT_PARALLEL_OK; PlannedStmt *queryPlan = pg_plan_query(query, NULL, cursorOptions, NULL); Portal portal = CreateNewPortal(); /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; PortalDefineQuery( portal, NULL, queryString, CMDTAG_SELECT, list_make1(queryPlan), NULL /* no CachedPlan */ ); int eflags = 0; PortalStart(portal, NULL, eflags, GetActiveSnapshot()); return portal; } /* * QueryTupleShardSearchInfo returns a CitusTableCacheEntry which has enough * information so that FindShardInterval() can find the shard corresponding * to a tuple. */ CitusTableCacheEntry * QueryTupleShardSearchInfo(ArrayType *minValuesArray, ArrayType *maxValuesArray, char partitionMethod, Var *partitionColumn) { Datum *minValues = 0; Datum *maxValues = 0; bool *minValueNulls = 0; bool *maxValueNulls = 0; int minValuesCount = 0; int maxValuesCount = 0; Oid intervalTypeId = InvalidOid; int32 intervalTypeMod = 0; deconstruct_array(minValuesArray, TEXTOID, -1, false, 'i', &minValues, &minValueNulls, &minValuesCount); deconstruct_array(maxValuesArray, TEXTOID, -1, false, 'i', &maxValues, &maxValueNulls, &maxValuesCount); int partitionCount = minValuesCount; Assert(maxValuesCount == partitionCount); GetIntervalTypeInfo(partitionMethod, partitionColumn, &intervalTypeId, &intervalTypeMod); FmgrInfo *shardColumnCompare = GetFunctionInfo(partitionColumn->vartype, BTREE_AM_OID, BTORDER_PROC); FmgrInfo *shardIntervalCompare = GetFunctionInfo(intervalTypeId, BTREE_AM_OID, BTORDER_PROC); FmgrInfo *hashFunction = NULL; if (partitionMethod == DISTRIBUTE_BY_HASH) { TypeCacheEntry *typeEntry = lookup_type_cache(partitionColumn->vartype, TYPECACHE_HASH_PROC_FINFO); hashFunction = palloc0(sizeof(FmgrInfo)); fmgr_info_copy(hashFunction, &(typeEntry->hash_proc_finfo), CurrentMemoryContext); if (!OidIsValid(hashFunction->fn_oid)) { ereport(ERROR, (errmsg("no hash function defined for type %s", format_type_be(partitionColumn->vartype)))); } } ShardInterval **shardIntervalArray = palloc0(partitionCount * sizeof(ShardInterval *)); for (int partitionIndex = 0; partitionIndex < partitionCount; partitionIndex++) { Datum datumArray[Natts_pg_dist_shard] = { [Anum_pg_dist_shard_logicalrelid - 1] = InvalidOid, [Anum_pg_dist_shard_shardid - 1] = partitionIndex, [Anum_pg_dist_shard_shardstorage - 1] = SHARD_STORAGE_VIRTUAL, [Anum_pg_dist_shard_shardminvalue - 1] = minValues[partitionIndex], [Anum_pg_dist_shard_shardmaxvalue - 1] = maxValues[partitionIndex] }; bool nullsArray[Natts_pg_dist_shard] = { [Anum_pg_dist_shard_shardminvalue - 1] = minValueNulls[partitionIndex], [Anum_pg_dist_shard_shardmaxvalue - 1] = maxValueNulls[partitionIndex] }; shardIntervalArray[partitionIndex] = DeformedDistShardTupleToShardInterval(datumArray, nullsArray, intervalTypeId, intervalTypeMod); shardIntervalArray[partitionIndex]->shardIndex = partitionIndex; } CitusTableCacheEntry *result = palloc0(sizeof(CitusTableCacheEntry)); result->partitionMethod = partitionMethod; result->partitionColumn = partitionColumn; result->shardIntervalCompareFunction = shardIntervalCompare; result->shardColumnCompareFunction = shardColumnCompare; result->hashFunction = hashFunction; result->sortedShardIntervalArray = SortShardIntervalArray(shardIntervalArray, partitionCount, partitionColumn->varcollid, shardIntervalCompare); result->hasUninitializedShardInterval = HasUninitializedShardInterval(result->sortedShardIntervalArray, partitionCount); result->hasOverlappingShardInterval = result->hasUninitializedShardInterval || HasOverlappingShardInterval(result->sortedShardIntervalArray, partitionCount, partitionColumn->varcollid, shardIntervalCompare); ErrorIfInconsistentShardIntervals(result); result->shardIntervalArrayLength = partitionCount; return result; } /* * CreatePartitionedResultDestReceiver sets up a partitioned dest receiver. */ DestReceiver * CreatePartitionedResultDestReceiver(int partitionColumnIndex, int partitionCount, CitusTableCacheEntry *shardSearchInfo, DestReceiver **partitionedDestReceivers, bool lazyStartup, bool allowNullPartitionColumnValues) { PartitionedResultDestReceiver *resultDest = palloc0(sizeof(PartitionedResultDestReceiver)); /* set up the DestReceiver function pointers */ resultDest->pub.receiveSlot = PartitionedResultDestReceiverReceive; resultDest->pub.rStartup = PartitionedResultDestReceiverStartup; resultDest->pub.rShutdown = PartitionedResultDestReceiverShutdown; resultDest->pub.rDestroy = PartitionedResultDestReceiverDestroy; resultDest->pub.mydest = DestCopyOut; /* setup routing parameters */ resultDest->partitionColumnIndex = partitionColumnIndex; resultDest->partitionCount = partitionCount; resultDest->shardSearchInfo = shardSearchInfo; resultDest->partitionDestReceivers = partitionedDestReceivers; resultDest->startedDestReceivers = NULL; resultDest->lazyStartup = lazyStartup; resultDest->allowNullPartitionColumnValues = allowNullPartitionColumnValues; return (DestReceiver *) resultDest; } /* * PartitionedResultDestReceiverStartup implements the rStartup interface of * PartitionedResultDestReceiver. */ static void PartitionedResultDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor) { PartitionedResultDestReceiver *self = (PartitionedResultDestReceiver *) dest; self->startupArguments.tupleDescriptor = CreateTupleDescCopy(inputTupleDescriptor); self->startupArguments.operation = operation; if (self->lazyStartup) { /* we are initialized, rest happens when needed*/ return; } /* no lazy startup, lets startup our partitionedDestReceivers */ for (int partitionIndex = 0; partitionIndex < self->partitionCount; partitionIndex++) { DestReceiver *partitionDest = self->partitionDestReceivers[partitionIndex]; partitionDest->rStartup(partitionDest, operation, inputTupleDescriptor); self->startedDestReceivers = bms_add_member(self->startedDestReceivers, partitionIndex); } } /* * PartitionedResultDestReceiverReceive implements the receiveSlot interface of * PartitionedResultDestReceiver. */ static bool PartitionedResultDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) { PartitionedResultDestReceiver *self = (PartitionedResultDestReceiver *) dest; slot_getallattrs(slot); Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; int partitionIndex; if (columnNulls[self->partitionColumnIndex]) { if (self->allowNullPartitionColumnValues) { /* * NULL values go into the first partition for both hash- and range- * partitioning, since that is the only way to guarantee that there is * always a partition for NULL and that it is always the same partition. */ partitionIndex = 0; } else { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("the partition column value cannot be NULL"))); } } else { Datum partitionColumnValue = columnValues[self->partitionColumnIndex]; ShardInterval *shardInterval = FindShardInterval(partitionColumnValue, self->shardSearchInfo); if (shardInterval == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find shard for partition column " "value"))); } partitionIndex = shardInterval->shardIndex; } DestReceiver *partitionDest = self->partitionDestReceivers[partitionIndex]; /* check if this partitionDestReceiver has been started before, start if not */ if (!bms_is_member(partitionIndex, self->startedDestReceivers)) { partitionDest->rStartup(partitionDest, self->startupArguments.operation, self->startupArguments.tupleDescriptor); self->startedDestReceivers = bms_add_member(self->startedDestReceivers, partitionIndex); } /* forward the tuple to the appropriate dest receiver */ partitionDest->receiveSlot(slot, partitionDest); return true; } /* * PartitionedResultDestReceiverShutdown implements the rShutdown interface of * PartitionedResultDestReceiver by calling rShutdown on all started * partitionedDestReceivers. */ static void PartitionedResultDestReceiverShutdown(DestReceiver *dest) { PartitionedResultDestReceiver *self = (PartitionedResultDestReceiver *) dest; int i = -1; while ((i = bms_next_member(self->startedDestReceivers, i)) >= 0) { DestReceiver *partitionDest = self->partitionDestReceivers[i]; partitionDest->rShutdown(partitionDest); } /* empty the set of started receivers which allows them to be restarted again */ bms_free(self->startedDestReceivers); self->startedDestReceivers = NULL; } /* * PartitionedResultDestReceiverDestroy implements the rDestroy interface of * PartitionedResultDestReceiver. */ static void PartitionedResultDestReceiverDestroy(DestReceiver *dest) { PartitionedResultDestReceiver *self = (PartitionedResultDestReceiver *) dest; /* we destroy all dest receivers, irregardless if they have been started or not */ for (int partitionIndex = 0; partitionIndex < self->partitionCount; partitionIndex++) { DestReceiver *partitionDest = self->partitionDestReceivers[partitionIndex]; if (partitionDest != NULL) { partitionDest->rDestroy(partitionDest); } } } ================================================ FILE: src/backend/distributed/executor/placement_access.c ================================================ /*------------------------------------------------------------------------- * * citus_custom_scan.c * * Definitions of the functions used in generating the placement accesses * for distributed query execution. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/placement_access.h" static List * BuildPlacementSelectList(int32 groupId, List *relationShardList); static List * BuildPlacementDDLList(int32 groupId, List *relationShardList); static List * BuildPlacementAccessList(int32 groupId, List *relationShardList, ShardPlacementAccessType accessType); /* * PlacementAccessListForTask returns a list of placement accesses for a given * task and task placement. */ List * PlacementAccessListForTask(Task *task, ShardPlacement *taskPlacement) { List *placementAccessList = NIL; List *relationShardList = task->relationShardList; bool addAnchorAccess = false; ShardPlacementAccessType accessType = PLACEMENT_ACCESS_SELECT; if (task->taskType == MODIFY_TASK) { /* DML command */ addAnchorAccess = true; accessType = PLACEMENT_ACCESS_DML; } else if (task->taskType == DDL_TASK || task->taskType == VACUUM_ANALYZE_TASK) { /* DDL command */ addAnchorAccess = true; accessType = PLACEMENT_ACCESS_DDL; } else if (relationShardList == NIL) { /* SELECT query that does not touch any shard placements */ addAnchorAccess = true; accessType = PLACEMENT_ACCESS_SELECT; } if (addAnchorAccess) { ShardPlacementAccess *placementAccess = CreatePlacementAccess(taskPlacement, accessType); placementAccessList = lappend(placementAccessList, placementAccess); } /* * We've already added anchor shardId's placement access to the list. Now, * add the other placements in the relationShardList. */ if (accessType == PLACEMENT_ACCESS_DDL) { /* * All relations appearing inter-shard DDL commands should be marked * with DDL access. */ List *relationShardAccessList = BuildPlacementDDLList(taskPlacement->groupId, relationShardList); placementAccessList = list_concat(placementAccessList, relationShardAccessList); } else { /* * In case of SELECTs or DML's, we add SELECT placement accesses to the * elements in relationShardList. For SELECT queries, it is trivial, since * the query is literally accesses the relationShardList in the same query. * * For DMLs, create placement accesses for placements that appear in a * subselect. */ List *relationShardAccessList = BuildPlacementSelectList(taskPlacement->groupId, relationShardList); placementAccessList = list_concat(placementAccessList, relationShardAccessList); } return placementAccessList; } /* * BuildPlacementSelectList builds a list of SELECT placement accesses * which can be used to call StartPlacementListConnection or * GetPlacementListConnection. If the node group does not have a placement * (e.g. in case of a broadcast join) then the shard is skipped. */ static List * BuildPlacementSelectList(int32 groupId, List *relationShardList) { return BuildPlacementAccessList(groupId, relationShardList, PLACEMENT_ACCESS_SELECT); } /* * BuildPlacementDDLList is a warpper around BuildPlacementAccessList() for DDL access. */ static List * BuildPlacementDDLList(int32 groupId, List *relationShardList) { return BuildPlacementAccessList(groupId, relationShardList, PLACEMENT_ACCESS_DDL); } /* * BuildPlacementAccessList returns a list of placement accesses for the given * relationShardList and the access type. */ static List * BuildPlacementAccessList(int32 groupId, List *relationShardList, ShardPlacementAccessType accessType) { List *placementAccessList = NIL; RelationShard *relationShard = NULL; foreach_declared_ptr(relationShard, relationShardList) { ShardPlacement *placement = ActiveShardPlacementOnGroup(groupId, relationShard->shardId); if (placement == NULL) { continue; } ShardPlacementAccess *placementAccess = CreatePlacementAccess(placement, accessType); placementAccessList = lappend(placementAccessList, placementAccess); } return placementAccessList; } /* * CreatePlacementAccess returns a new ShardPlacementAccess for the given placement * and access type. */ ShardPlacementAccess * CreatePlacementAccess(ShardPlacement *placement, ShardPlacementAccessType accessType) { ShardPlacementAccess *placementAccess = (ShardPlacementAccess *) palloc0( sizeof(ShardPlacementAccess)); placementAccess->placement = placement; placementAccess->accessType = accessType; return placementAccess; } ================================================ FILE: src/backend/distributed/executor/repartition_executor.c ================================================ /*------------------------------------------------------------------- * * repartition_executor.c * * Definitions for public functions and types related to repartition * of select query results. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" #include "distributed/citus_custom_scan.h" #include "distributed/deparse_shard_query.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/recursive_planning.h" #include "distributed/repartition_executor.h" #include "distributed/resource_lock.h" /* * IsSupportedRedistributionTarget determines whether re-partitioning into the * given target relation is supported. */ bool IsSupportedRedistributionTarget(Oid targetRelationId) { CitusTableCacheEntry *tableEntry = GetCitusTableCacheEntry(targetRelationId); if (!IsCitusTableTypeCacheEntry(tableEntry, HASH_DISTRIBUTED) && !IsCitusTableTypeCacheEntry(tableEntry, RANGE_DISTRIBUTED)) { return false; } return true; } /* * IsRedistributablePlan returns true if the given plan is a distributable plan. */ bool IsRedistributablePlan(Plan *selectPlan) { if (!EnableRepartitionedInsertSelect) { return false; } /* * Don't redistribute if query is not distributed or requires * merge on coordinator */ if (!IsCitusCustomScan(selectPlan)) { return false; } DistributedPlan *distSelectPlan = GetDistributedPlan((CustomScan *) selectPlan); Job *distSelectJob = distSelectPlan->workerJob; List *distSelectTaskList = distSelectJob->taskList; /* * Don't use redistribution if only one task. This is to keep the existing * behaviour for CTEs that the last step is a read_intermediate_result() * call. It doesn't hurt much in other cases too. */ if (list_length(distSelectTaskList) <= 1) { return false; } /* don't use redistribution for repartition joins for now */ if (distSelectJob->dependentJobList != NIL) { return false; } if (distSelectPlan->combineQuery != NULL) { Query *combineQuery = (Query *) distSelectPlan->combineQuery; if (contain_nextval_expression_walker((Node *) combineQuery->targetList, NULL)) { /* nextval needs to be evaluated on the coordinator */ return false; } } return true; } /* * HasMergeNotMatchedBySource returns true if the MERGE query has a * WHEN NOT MATCHED BY SOURCE clause. If it does, we need to execute * the MERGE query on all shards of the target table, regardless of * whether or not the source shard has any rows. */ bool HasMergeNotMatchedBySource(Query *query) { if (!IsMergeQuery(query)) { return false; } bool haveNotMatchedBySource = false; #if PG_VERSION_NUM >= PG_VERSION_17 ListCell *lc; foreach(lc, query->mergeActionList) { MergeAction *action = lfirst_node(MergeAction, lc); if (action->matchKind == MERGE_WHEN_NOT_MATCHED_BY_SOURCE) { haveNotMatchedBySource = true; break; } } #endif return haveNotMatchedBySource; } /* * GenerateTaskListWithColocatedIntermediateResults generates a list of tasks * for a query that inserts into a target relation and selects from a set of * co-located intermediate results. */ List * GenerateTaskListWithColocatedIntermediateResults(Oid targetRelationId, Query * modifyQueryViaCoordinatorOrRepartition, char *resultIdPrefix) { List *taskList = NIL; /* * Make a copy of the ... SELECT. We'll repeatedly replace * the subquery of modifyResultQuery for different intermediate results and * then deparse it. */ Query *modifyWithResultQuery = copyObject(modifyQueryViaCoordinatorOrRepartition); RangeTblEntry *insertRte = ExtractResultRelationRTE(modifyWithResultQuery); RangeTblEntry *selectRte = ExtractSourceResultRangeTableEntry(modifyWithResultQuery); CitusTableCacheEntry *targetCacheEntry = GetCitusTableCacheEntry(targetRelationId); int shardCount = targetCacheEntry->shardIntervalArrayLength; uint32 taskIdIndex = 1; uint64 jobId = INVALID_JOB_ID; for (int shardOffset = 0; shardOffset < shardCount; shardOffset++) { ShardInterval *targetShardInterval = targetCacheEntry->sortedShardIntervalArray[shardOffset]; uint64 shardId = targetShardInterval->shardId; List *columnAliasList = NIL; StringInfo queryString = makeStringInfo(); StringInfo resultId = makeStringInfo(); /* during COPY, the shard ID is appended to the result name */ appendStringInfo(resultId, "%s_" UINT64_FORMAT, resultIdPrefix, shardId); /* * For MERGE SQL, use the USING clause list, the main query target list * is NULL */ List *targetList = IsMergeQuery(modifyQueryViaCoordinatorOrRepartition) ? selectRte->subquery->targetList : modifyQueryViaCoordinatorOrRepartition->targetList; /* generate the query on the intermediate result */ Query *resultSelectQuery = BuildSubPlanResultQuery(targetList, columnAliasList, resultId->data); /* put the intermediate result query in the INSERT..SELECT */ selectRte->subquery = resultSelectQuery; /* setting an alias simplifies deparsing of RETURNING */ if (insertRte->alias == NULL) { Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); insertRte->alias = alias; } /* * Generate a query string for the query that inserts into a shard and reads * from an intermediate result. * * Since CTEs have already been converted to intermediate results, they need * to removed from the query. Otherwise, worker queries include both * intermediate results and CTEs in the query. */ modifyWithResultQuery->cteList = NIL; deparse_shard_query(modifyWithResultQuery, targetRelationId, shardId, queryString); ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data))); LockShardDistributionMetadata(shardId, ShareLock); List *insertShardPlacementList = ActiveShardPlacementList(shardId); RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = targetShardInterval->relationId; relationShard->shardId = targetShardInterval->shardId; Task *modifyTask = CreateBasicTask(jobId, taskIdIndex, MODIFY_TASK, queryString->data); modifyTask->dependentTaskList = NIL; modifyTask->anchorShardId = shardId; modifyTask->taskPlacementList = insertShardPlacementList; modifyTask->relationShardList = list_make1(relationShard); modifyTask->replicationModel = targetCacheEntry->replicationModel; taskList = lappend(taskList, modifyTask); taskIdIndex++; } return taskList; } /* * AdjustTaskQueryForEmptySource adjusts the query for tasks that read from an * intermediate result to instead read from an empty relation. This ensures that * the MERGE query is executed on all shards of the target table, because it has * a NOT MATCHED BY SOURCE clause, which will be true for all target shards where * the source shard has no rows. */ void AdjustTaskQueryForEmptySource(Oid targetRelationId, Query *mergeQuery, List *tasks, char *resultIdPrefix) { Query *mergeQueryCopy = copyObject(mergeQuery); RangeTblEntry *selectRte = ExtractSourceResultRangeTableEntry(mergeQueryCopy); RangeTblEntry *mergeRte = ExtractResultRelationRTE(mergeQueryCopy); List *targetList = selectRte->subquery->targetList; ListCell *taskCell = NULL; foreach(taskCell, tasks) { Task *task = lfirst(taskCell); uint64 shardId = task->anchorShardId; StringInfo queryString = makeStringInfo(); StringInfo resultId = makeStringInfo(); appendStringInfo(resultId, "%s_" UINT64_FORMAT, resultIdPrefix, shardId); /* Generate a query for an empty relation */ selectRte->subquery = BuildEmptyResultQuery(targetList, resultId->data); /* setting an alias simplifies deparsing of RETURNING */ if (mergeRte->alias == NULL) { Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); mergeRte->alias = alias; } /* * Generate a query string for the query that merges into a shard and reads * from an empty relation. * * Since CTEs have already been converted to intermediate results, they need * to removed from the query. Otherwise, worker queries include both * intermediate results and CTEs in the query. */ mergeQueryCopy->cteList = NIL; deparse_shard_query(mergeQueryCopy, targetRelationId, shardId, queryString); ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data))); SetTaskQueryString(task, queryString->data); } } /* * GenerateTaskListWithRedistributedResults returns a task list to insert given * redistributedResults into the given target relation. * redistributedResults[shardIndex] is list of cstrings each of which is * a result name which should be inserted into * targetRelation->sortedShardIntervalArray[shardIndex]. */ List * GenerateTaskListWithRedistributedResults(Query *modifyQueryViaCoordinatorOrRepartition, CitusTableCacheEntry *targetRelation, List **redistributedResults, bool useBinaryFormat) { List *taskList = NIL; /* * Make a copy of the ... SELECT. We'll repeatedly replace * the subquery of modifyResultQuery for different intermediate results and * then deparse it. */ Query *modifyResultQuery = copyObject(modifyQueryViaCoordinatorOrRepartition); RangeTblEntry *insertRte = ExtractResultRelationRTE(modifyResultQuery); Oid targetRelationId = targetRelation->relationId; bool hasNotMatchedBySource = HasMergeNotMatchedBySource(modifyResultQuery); int shardCount = targetRelation->shardIntervalArrayLength; int shardOffset = 0; uint32 taskIdIndex = 1; uint64 jobId = INVALID_JOB_ID; RangeTblEntry *selectRte = ExtractSourceResultRangeTableEntry(modifyResultQuery); List *selectTargetList = selectRte->subquery->targetList; for (shardOffset = 0; shardOffset < shardCount; shardOffset++) { ShardInterval *targetShardInterval = targetRelation->sortedShardIntervalArray[shardOffset]; List *resultIdList = redistributedResults[targetShardInterval->shardIndex]; uint64 shardId = targetShardInterval->shardId; StringInfo queryString = makeStringInfo(); /* skip empty tasks */ if (resultIdList == NIL && !hasNotMatchedBySource) { continue; } Query *fragmentSetQuery = NULL; if (resultIdList != NIL) { /* sort result ids for consistent test output */ List *sortedResultIds = SortList(resultIdList, pg_qsort_strcmp); /* generate the query on the intermediate result */ fragmentSetQuery = BuildReadIntermediateResultsArrayQuery(selectTargetList, NIL, sortedResultIds, useBinaryFormat); } else { /* No source data, but MERGE query has NOT MATCHED BY SOURCE */ StringInfo emptyFragmentId = makeStringInfo(); appendStringInfo(emptyFragmentId, "%s_" UINT64_FORMAT, "temp_empty_rel_", shardId); fragmentSetQuery = BuildEmptyResultQuery(selectTargetList, emptyFragmentId->data); } /* put the intermediate result query in the INSERT..SELECT */ selectRte->subquery = fragmentSetQuery; /* setting an alias simplifies deparsing of RETURNING */ if (insertRte->alias == NULL) { Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); insertRte->alias = alias; } /* * Generate a query string for the query that inserts into a shard and reads * from an intermediate result. * * Since CTEs have already been converted to intermediate results, they need * to removed from the query. Otherwise, worker queries include both * intermediate results and CTEs in the query. */ modifyResultQuery->cteList = NIL; deparse_shard_query(modifyResultQuery, targetRelationId, shardId, queryString); ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data))); LockShardDistributionMetadata(shardId, ShareLock); List *insertShardPlacementList = ActiveShardPlacementList(shardId); RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = targetShardInterval->relationId; relationShard->shardId = targetShardInterval->shardId; Task *modifyTask = CreateBasicTask(jobId, taskIdIndex, MODIFY_TASK, queryString->data); modifyTask->dependentTaskList = NIL; modifyTask->anchorShardId = shardId; modifyTask->taskPlacementList = insertShardPlacementList; modifyTask->relationShardList = list_make1(relationShard); modifyTask->replicationModel = targetRelation->replicationModel; taskList = lappend(taskList, modifyTask); taskIdIndex++; } return taskList; } ================================================ FILE: src/backend/distributed/executor/repartition_join_execution.c ================================================ /*------------------------------------------------------------------------- * * repartition_join_execution.c * * This file contains repartition specific logic. * ExecuteDependentTasks takes a list of top level tasks. Its logic is as follows: * - It generates all the tasks by descending in the tasks tree. Note that each task * has a dependentTaskList. * - It generates FetchTask queryStrings with the MapTask queries. It uses the first replicate to * fetch data when replication factor is > 1. Note that if a task fails in any replica adaptive executor * gives an error, so if we come to a fetchTask we know for sure that its dependedMapTask is executed in all * replicas. * - It creates schemas in each worker in a single transaction to store intermediate results. * - It iterates all tasks and finds the ones whose dependencies are already executed, and executes them with * adaptive executor logic. * * * Repartition queries do not begin a transaction even if we are in * a transaction block. As we don't begin a transaction, they won't see the * DDLs that happened earlier in the transaction because we don't have that * transaction id with repartition queries. Therefore we error in this case. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "miscadmin.h" #include "access/hash.h" #include "utils/builtins.h" #include "distributed/adaptive_executor.h" #include "distributed/directed_acyclic_graph_execution.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/repartition_join_execution.h" #include "distributed/task_execution_utils.h" #include "distributed/transaction_management.h" #include "distributed/transmit.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" static List * ExtractJobsInJobTree(Job *job); static void TraverseJobTree(Job *curJob, List **jobs); /* * ExecuteDependentTasks executes all tasks except the top level tasks * in order from the task tree. At a time, it can execute different tasks from * different jobs. */ List * ExecuteDependentTasks(List *topLevelTasks, Job *topLevelJob) { List *allTasks = CreateTaskListForJobTree(topLevelTasks); List *jobIds = ExtractJobsInJobTree(topLevelJob); ExecuteTasksInDependencyOrder(allTasks, topLevelTasks, jobIds); return jobIds; } /* * ExtractJobsInJobTree returns all job ids in the job tree * where the given job is root. */ static List * ExtractJobsInJobTree(Job *job) { List *jobIds = NIL; TraverseJobTree(job, &jobIds); return jobIds; } /* * TraverseJobTree does a dfs in the current job and adds * all of its job ids. */ static void TraverseJobTree(Job *curJob, List **jobIds) { uint64 *jobIdPointer = palloc(sizeof(uint64)); *jobIdPointer = curJob->jobId; *jobIds = lappend(*jobIds, jobIdPointer); Job *childJob = NULL; foreach_declared_ptr(childJob, curJob->dependentJobList) { TraverseJobTree(childJob, jobIds); } } ================================================ FILE: src/backend/distributed/executor/subplan_execution.c ================================================ /*------------------------------------------------------------------------- * * subplan_execution.c * * Functions for execution subplans prior to distributed table execution. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "executor/executor.h" #include "utils/datetime.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/recursive_planning.h" #include "distributed/subplan_execution.h" #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" #define SECOND_TO_MILLI_SECOND 1000 #define MICRO_TO_MILLI_SECOND 0.001 int MaxIntermediateResult = 1048576; /* maximum size in KB the intermediate result can grow to */ /* when this is true, we enforce intermediate result size limit in all executors */ int SubPlanLevel = 0; /* * SubPlanExplainAnalyzeContext is both a memory context for storing * subplans’ EXPLAIN ANALYZE output and a flag indicating that execution * is running under EXPLAIN ANALYZE for subplans. */ MemoryContext SubPlanExplainAnalyzeContext = NULL; SubPlanExplainOutputData *SubPlanExplainOutput; extern uint8 TotalExplainOutputCapacity; extern uint8 NumTasksOutput; /* * ExecuteSubPlans executes a list of subplans from a distributed plan * by sequentially executing each plan from the top. */ void ExecuteSubPlans(DistributedPlan *distributedPlan, bool explainAnalyzeEnabled) { uint64 planId = distributedPlan->planId; List *subPlanList = distributedPlan->subPlanList; if (subPlanList == NIL) { /* no subplans to execute */ return; } /* * If the root DistributedPlan has EXPLAIN ANALYZE enabled, * its subplans should also have EXPLAIN ANALYZE enabled. */ if (explainAnalyzeEnabled) { SubPlanExplainAnalyzeContext = GetMemoryChunkContext(distributedPlan); } else { SubPlanExplainAnalyzeContext = NULL; } HTAB *intermediateResultsHash = MakeIntermediateResultHTAB(); RecordSubplanExecutionsOnNodes(intermediateResultsHash, distributedPlan); /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results of subplans will be stored in a directory that is * derived from the distributed transaction ID. */ UseCoordinatedTransaction(); DistributedSubPlan *subPlan = NULL; foreach_declared_ptr(subPlan, subPlanList) { PlannedStmt *plannedStmt = subPlan->plan; uint32 subPlanId = subPlan->subPlanId; ParamListInfo params = NULL; char *resultId = GenerateResultId(planId, subPlanId); List *remoteWorkerNodeList = FindAllWorkerNodesUsingSubplan(intermediateResultsHash, resultId); IntermediateResultsHashEntry *entry = SearchIntermediateResult(intermediateResultsHash, resultId); SubPlanLevel++; EState *estate = CreateExecutorState(); DestReceiver *copyDest = CreateRemoteFileDestReceiver(resultId, estate, remoteWorkerNodeList, entry->writeLocalFile); TimestampTz startTimestamp = GetCurrentTimestamp(); uint64 nprocessed; PG_TRY(); { nprocessed = ExecutePlanIntoDestReceiver(plannedStmt, params, copyDest); } PG_CATCH(); { SubPlanExplainAnalyzeContext = NULL; SubPlanExplainOutput = NULL; TotalExplainOutputCapacity = 0; NumTasksOutput = 0; PG_RE_THROW(); } PG_END_TRY(); /* * EXPLAIN ANALYZE instrumentations. Calculating these are very light-weight, * so always populate them regardless of EXPLAIN ANALYZE or not. */ long durationSeconds = 0.0; int durationMicrosecs = 0; TimestampDifference(startTimestamp, GetCurrentTimestamp(), &durationSeconds, &durationMicrosecs); subPlan->durationMillisecs = durationSeconds * SECOND_TO_MILLI_SECOND; subPlan->durationMillisecs += durationMicrosecs * MICRO_TO_MILLI_SECOND; subPlan->bytesSentPerWorker = RemoteFileDestReceiverBytesSent(copyDest); subPlan->ntuples = nprocessed; subPlan->remoteWorkerCount = list_length(remoteWorkerNodeList); subPlan->writeLocalFile = entry->writeLocalFile; SubPlanLevel--; /* * Save the EXPLAIN ANALYZE output(s) for later extraction in ExplainSubPlans(). * Because the SubPlan context isn’t available during distributed execution, * pass the pointer as a global variable in SubPlanExplainOutput. */ subPlan->totalExplainOutput = SubPlanExplainOutput; subPlan->numTasksOutput = NumTasksOutput; SubPlanExplainOutput = NULL; TotalExplainOutputCapacity = 0; NumTasksOutput = 0; FreeExecutorState(estate); } SubPlanExplainAnalyzeContext = NULL; } ================================================ FILE: src/backend/distributed/executor/transmit.c ================================================ /*------------------------------------------------------------------------- * * transmit.c * Routines for transmitting regular files between two nodes. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "commands/defrem.h" #include "common/file_perm.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "storage/fd.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" #include "distributed/transmit.h" #include "distributed/utils/directory.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* Local functions forward declarations */ static void SendCopyInStart(void); static void SendCopyOutStart(void); static void SendCopyDone(void); static void SendCopyData(StringInfo fileBuffer); static bool ReceiveCopyData(StringInfo copyData); static void FreeStringInfo(StringInfo stringInfo); /* * RedirectCopyDataToRegularFile receives data from stdin using the standard copy * protocol. The function then creates or truncates a file with the given * filename, and appends received data to this file. */ void RedirectCopyDataToRegularFile(const char *filename) { StringInfo copyData = makeStringInfo(); const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); File fileDesc = FileOpenForTransmit(filename, fileFlags); FileCompat fileCompat = FileCompatFromFileStart(fileDesc); SendCopyInStart(); bool copyDone = ReceiveCopyData(copyData); while (!copyDone) { /* if received data has contents, append to regular file */ if (copyData->len > 0) { int appended = FileWriteCompat(&fileCompat, copyData->data, copyData->len, PG_WAIT_IO); if (appended != copyData->len) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not append to received file: %m"))); } } resetStringInfo(copyData); copyDone = ReceiveCopyData(copyData); } FreeStringInfo(copyData); FileClose(fileDesc); } /* * SendRegularFile reads data from the given file, and sends these data to * stdout using the standard copy protocol. After all file data are sent, the * function ends the copy protocol and closes the file. */ void SendRegularFile(const char *filename) { const uint32 fileBufferSize = 32768; /* 32 KB */ const int fileFlags = (O_RDONLY | PG_BINARY); const int fileMode = 0; /* we currently do not check if the caller has permissions for this file */ File fileDesc = FileOpenForTransmitPerm(filename, fileFlags, fileMode); FileCompat fileCompat = FileCompatFromFileStart(fileDesc); /* * We read file's contents into buffers of 32 KB. This buffer size is twice * as large as Hadoop's default buffer size, and may later be configurable. */ StringInfo fileBuffer = makeStringInfo(); enlargeStringInfo(fileBuffer, fileBufferSize); SendCopyOutStart(); int readBytes = FileReadCompat(&fileCompat, fileBuffer->data, fileBufferSize, PG_WAIT_IO); while (readBytes > 0) { fileBuffer->len = readBytes; SendCopyData(fileBuffer); resetStringInfo(fileBuffer); readBytes = FileReadCompat(&fileCompat, fileBuffer->data, fileBufferSize, PG_WAIT_IO); } SendCopyDone(); FreeStringInfo(fileBuffer); FileClose(fileDesc); } /* Helper function that deallocates string info object. */ static void FreeStringInfo(StringInfo stringInfo) { resetStringInfo(stringInfo); pfree(stringInfo->data); pfree(stringInfo); } /* * Open a file with FileOpenForTransmitPerm() and pass default file mode for * the fileMode parameter. */ File FileOpenForTransmit(const char *filename, int fileFlags) { return FileOpenForTransmitPerm(filename, fileFlags, pg_file_create_mode); } /* * FileOpenForTransmitPerm opens file with the given filename and flags. On * success, the function returns the internal file handle for the opened file. * On failure the function errors out. */ File FileOpenForTransmitPerm(const char *filename, int fileFlags, int fileMode) { struct stat fileStat; int statOK = stat(filename, &fileStat); if (statOK >= 0) { if (S_ISDIR(fileStat.st_mode)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is a directory", filename))); } } File fileDesc = PathNameOpenFilePerm((char *) filename, fileFlags, fileMode); if (fileDesc < 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", filename))); } return fileDesc; } /* * SendCopyInStart sends the start copy in message to initiate receiving data * from stdin. The frontend should now send copy data. */ static void SendCopyInStart(void) { StringInfoData copyInStart = { NULL, 0, 0, 0 }; const char copyFormat = 1; /* binary copy format */ pq_beginmessage(©InStart, 'G'); pq_sendbyte(©InStart, copyFormat); pq_sendint(©InStart, 0, 2); pq_endmessage(©InStart); /* flush here to ensure that FE knows it can send data */ int flushed = pq_flush(); if (flushed != 0) { ereport(WARNING, (errmsg("could not flush copy start data"))); } } /* * SendCopyOutStart sends the start copy out message to initiate sending data to * stdout. After this message, the backend will continue by sending copy data. */ static void SendCopyOutStart(void) { StringInfoData copyOutStart = { NULL, 0, 0, 0 }; const char copyFormat = 1; /* binary copy format */ pq_beginmessage(©OutStart, 'H'); pq_sendbyte(©OutStart, copyFormat); pq_sendint(©OutStart, 0, 2); pq_endmessage(©OutStart); } /* Sends the copy-complete message. */ static void SendCopyDone(void) { StringInfoData copyDone = { NULL, 0, 0, 0 }; pq_beginmessage(©Done, 'c'); pq_endmessage(©Done); /* flush here to signal to FE that we are done */ int flushed = pq_flush(); if (flushed != 0) { ereport(WARNING, (errmsg("could not flush copy start data"))); } } /* Sends the copy data message to stdout. */ static void SendCopyData(StringInfo fileBuffer) { StringInfoData copyData = { NULL, 0, 0, 0 }; pq_beginmessage(©Data, 'd'); pq_sendbytes(©Data, fileBuffer->data, fileBuffer->len); pq_endmessage(©Data); } /* * ReceiveCopyData receives one copy data message from stdin, and writes this * message's contents into the given argument. The function then checks if the * copy protocol has been completed, and if it has, the function returns true. * If not, the function returns false indicating there are more data to read. * If the received message does not conform to the copy protocol, the function * mirrors copy.c's error behavior. */ static bool ReceiveCopyData(StringInfo copyData) { bool copyDone = true; const int unlimitedSize = PQ_LARGE_MESSAGE_LIMIT; HOLD_CANCEL_INTERRUPTS(); pq_startmsgread(); int messageType = pq_getbyte(); if (messageType == EOF) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("unexpected EOF on client connection"))); } /* consume the rest of message before checking for message type */ int messageCopied = pq_getmessage(copyData, unlimitedSize); if (messageCopied == EOF) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("unexpected EOF on client connection"))); } RESUME_CANCEL_INTERRUPTS(); switch (messageType) { case 'd': /* CopyData */ { copyDone = false; break; } case 'c': /* CopyDone */ { copyDone = true; break; } case 'f': /* CopyFail */ { ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), errmsg("COPY data failed: %s", pq_getmsgstring(copyData)))); break; } case 'H': /* Flush */ case 'S': /* Sync */ { /* * Ignore Flush/Sync for the convenience of client libraries (such * as libpq) that may send those without noticing that the command * they just sent was COPY. */ copyDone = false; break; } default: { ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("unexpected message type 0x%02X during COPY data", messageType))); break; } } return copyDone; } ================================================ FILE: src/backend/distributed/executor/tuple_destination.c ================================================ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "access/htup_details.h" #include "distributed/multi_server_executor.h" #include "distributed/subplan_execution.h" #include "distributed/tuple_destination.h" /* * TupleStoreTupleDestination is internal representation of a TupleDestination * which forwards tuples to a tuple store. */ typedef struct TupleStoreTupleDestination { TupleDestination pub; /* destination of tuples */ Tuplestorestate *tupleStore; /* how does tuples look like? */ TupleDesc tupleDesc; } TupleStoreTupleDestination; /* * TupleDestDestReceiver is internal representation of a DestReceiver which * forards tuples to a tuple destination. */ typedef struct TupleDestDestReceiver { DestReceiver pub; TupleDestination *tupleDest; /* parameters to pass to tupleDest->putTuple() */ Task *task; int placementIndex; } TupleDestDestReceiver; /* forward declarations for local functions */ static void TupleStoreTupleDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize); static void EnsureIntermediateSizeLimitNotExceeded(TupleDestinationStats * tupleDestinationStats); static TupleDesc TupleStoreTupleDestTupleDescForQuery(TupleDestination *self, int queryNumber); static void TupleDestNonePutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize); static TupleDesc TupleDestNoneTupleDescForQuery(TupleDestination *self, int queryNumber); static void TupleDestDestReceiverStartup(DestReceiver *copyDest, int operation, TupleDesc inputTupleDesc); static bool TupleDestDestReceiverReceive(TupleTableSlot *slot, DestReceiver *copyDest); static void TupleDestDestReceiverShutdown(DestReceiver *destReceiver); static void TupleDestDestReceiverDestroy(DestReceiver *destReceiver); /* * CreateTupleStoreTupleDest creates a TupleDestination which forwards tuples to * a tupleStore. */ TupleDestination * CreateTupleStoreTupleDest(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { TupleStoreTupleDestination *tupleStoreTupleDest = palloc0( sizeof(TupleStoreTupleDestination)); tupleStoreTupleDest->tupleStore = tupleStore; tupleStoreTupleDest->tupleDesc = tupleDescriptor; tupleStoreTupleDest->pub.putTuple = TupleStoreTupleDestPutTuple; tupleStoreTupleDest->pub.tupleDescForQuery = TupleStoreTupleDestTupleDescForQuery; TupleDestination *tupleDestination = &tupleStoreTupleDest->pub; tupleDestination->tupleDestinationStats = (TupleDestinationStats *) palloc0(sizeof(TupleDestinationStats)); return (TupleDestination *) tupleStoreTupleDest; } /* * TupleStoreTupleDestPutTuple implements TupleDestination->putTuple for * TupleStoreTupleDestination. */ static void TupleStoreTupleDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize) { TupleStoreTupleDestination *tupleDest = (TupleStoreTupleDestination *) self; /* * Remote execution sets tupleLibpqSize, however it is 0 for local execution. We prefer * to use tupleLibpqSize for the remote execution because that reflects the exact data * transfer size over the network. For local execution, we rely on the size of the * tuple. */ uint64 tupleSize = tupleLibpqSize; if (tupleSize == 0) { tupleSize = heapTuple->t_len; } /* * Enfoce citus.max_intermediate_result_size for subPlans if * the caller requested. */ TupleDestinationStats *tupleDestinationStats = self->tupleDestinationStats; if (SubPlanLevel > 0 && tupleDestinationStats != NULL) { tupleDestinationStats->totalIntermediateResultSize += tupleSize; EnsureIntermediateSizeLimitNotExceeded(tupleDestinationStats); } /* do the actual work */ tuplestore_puttuple(tupleDest->tupleStore, heapTuple); /* we record tuples received over network */ task->totalReceivedTupleData += tupleLibpqSize; } /* * EnsureIntermediateSizeLimitNotExceeded is a helper function for checking the current * state of the tupleDestinationStats and throws error if necessary. */ static void EnsureIntermediateSizeLimitNotExceeded(TupleDestinationStats *tupleDestinationStats) { if (!tupleDestinationStats) { /* unexpected, still prefer defensive approach */ return; } /* * We only care about subPlans. Also, if user disabled, no need to * check further. */ if (SubPlanLevel == 0 || MaxIntermediateResult < 0) { return; } uint64 maxIntermediateResultInBytes = MaxIntermediateResult * 1024L; if (tupleDestinationStats->totalIntermediateResultSize < maxIntermediateResultInBytes) { /* * We have not reached the size limit that the user requested, so * nothing to do for now. */ return; } ereport(ERROR, (errmsg("the intermediate result size exceeds " "citus.max_intermediate_result_size (currently %d kB)", MaxIntermediateResult), errdetail("Citus restricts the size of intermediate " "results of complex subqueries and CTEs to " "avoid accidentally pulling large result sets " "into once place."), errhint("To run the current query, set " "citus.max_intermediate_result_size to a higher" " value or -1 to disable."))); } /* * TupleStoreTupleDestTupleDescForQuery implements TupleDestination->TupleDescForQuery * for TupleStoreTupleDestination. */ static TupleDesc TupleStoreTupleDestTupleDescForQuery(TupleDestination *self, int queryNumber) { Assert(queryNumber == 0); TupleStoreTupleDestination *tupleDest = (TupleStoreTupleDestination *) self; return tupleDest->tupleDesc; } /* * CreateTupleDestNone creates a tuple destination which ignores the tuples. */ TupleDestination * CreateTupleDestNone(void) { TupleDestination *tupleDest = palloc0( sizeof(TupleDestination)); tupleDest->putTuple = TupleDestNonePutTuple; tupleDest->tupleDescForQuery = TupleDestNoneTupleDescForQuery; return (TupleDestination *) tupleDest; } /* * TupleDestNonePutTuple implements TupleDestination->putTuple for * no-op tuple destination. */ static void TupleDestNonePutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize) { /* nothing to do */ } /* * TupleDestNoneTupleDescForQuery implements TupleDestination->TupleDescForQuery * for no-op tuple destination. */ static TupleDesc TupleDestNoneTupleDescForQuery(TupleDestination *self, int queryNumber) { return NULL; } /* * CreateTupleDestDestReceiver creates a dest receiver which forwards tuples * to a tuple destination. */ DestReceiver * CreateTupleDestDestReceiver(TupleDestination *tupleDest, Task *task, int placementIndex) { TupleDestDestReceiver *destReceiver = palloc0(sizeof(TupleDestDestReceiver)); destReceiver->pub.rStartup = TupleDestDestReceiverStartup; destReceiver->pub.receiveSlot = TupleDestDestReceiverReceive; destReceiver->pub.rShutdown = TupleDestDestReceiverShutdown; destReceiver->pub.rDestroy = TupleDestDestReceiverDestroy; destReceiver->tupleDest = tupleDest; destReceiver->task = task; destReceiver->placementIndex = placementIndex; return (DestReceiver *) destReceiver; } /* * TupleDestDestReceiverStartup implements DestReceiver->rStartup for * TupleDestDestReceiver. */ static void TupleDestDestReceiverStartup(DestReceiver *destReceiver, int operation, TupleDesc inputTupleDesc) { /* nothing to do */ } /* * TupleDestDestReceiverReceive implements DestReceiver->receiveSlot for * TupleDestDestReceiver. */ static bool TupleDestDestReceiverReceive(TupleTableSlot *slot, DestReceiver *destReceiver) { TupleDestDestReceiver *tupleDestReceiver = (TupleDestDestReceiver *) destReceiver; TupleDestination *tupleDest = tupleDestReceiver->tupleDest; Task *task = tupleDestReceiver->task; int placementIndex = tupleDestReceiver->placementIndex; /* * DestReceiver doesn't support multiple result sets with different shapes. */ Assert(task->queryCount == 1); int queryNumber = 0; HeapTuple heapTuple = ExecFetchSlotHeapTuple(slot, true, NULL); uint64 tupleLibpqSize = 0; tupleDest->putTuple(tupleDest, task, placementIndex, queryNumber, heapTuple, tupleLibpqSize); return true; } /* * TupleDestDestReceiverShutdown implements DestReceiver->rShutdown for * TupleDestDestReceiver. */ static void TupleDestDestReceiverShutdown(DestReceiver *destReceiver) { /* nothing to do */ } /* * TupleDestDestReceiverDestroy implements DestReceiver->rDestroy for * TupleDestDestReceiver. */ static void TupleDestDestReceiverDestroy(DestReceiver *destReceiver) { /* nothing to do */ } ================================================ FILE: src/backend/distributed/metadata/dependency.c ================================================ /*------------------------------------------------------------------------- * * dependency.c * Functions to reason about distributed objects and their dependencies * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/sysattr.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid_d.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension_d.h" #include "catalog/pg_foreign_data_wrapper_d.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc_d.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_rewrite_d.h" #include "catalog/pg_shdepend.h" #include "catalog/pg_type.h" #include "commands/extension.h" #include "common/hashfn.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_depended_object.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/version_compat.h" /* * ObjectAddressCollector keeps track of collected ObjectAddresses. This can be used * together with RecurseObjectDependencies. * * We keep three different datastructures for the following reasons * - A List ordered by insert/collect order * - A Set to quickly O(1) check if an ObjectAddress has already been collected * - A set to check which objects are already visited */ typedef struct ObjectAddressCollector { List *dependencyList; HTAB *dependencySet; HTAB *visitedObjects; } ObjectAddressCollector; /* * DependencyMode distinguishes the data stored in DependencyDefinition. For details see * DependencyDefinition's inline comments in the data union. */ typedef enum DependencyMode { DependencyObjectAddress, DependencyPgDepend, DependencyPgShDepend } DependencyMode; typedef struct DependencyDefinition { /* describe how the dependency data is stored in the data field */ DependencyMode mode; /* * Dependencies can be found in different ways and therefore stored differently on the * definition. */ union { /* * pg_depend is used for dependencies found in the database local pg_depend table. * The entry is copied while scanning the table. The record can be inspected * during the chasing algorithm to follow dependencies of different classes, or * based on dependency type. */ FormData_pg_depend pg_depend; /* * pg_shdepend is used for dependencies found in the global pg_shdepend table. * The entry is copied while scanning the table. The record can be inspected * during the chasing algorithm to follow dependencies of different classes, or * based on dependency type. */ FormData_pg_shdepend pg_shdepend; /* * address is used for dependencies that are artificially added during the * chasing. Since they are added by citus code we assume the dependency needs to * be chased anyway, of course it will only actually be chased if the object is a * supported object by citus */ ObjectAddress address; } data; } DependencyDefinition; /* * ViewDependencyNode represents a view (or possibly a table) in a dependency graph of * views. */ typedef struct ViewDependencyNode { Oid id; int remainingDependencyCount; List *dependingNodes; }ViewDependencyNode; static List * GetRelationSequenceDependencyList(Oid relationId); static List * GetRelationFunctionDependencyList(Oid relationId); static List * GetRelationTriggerFunctionDependencyList(Oid relationId); static List * GetPublicationRelationsDependencyList(Oid relationId); static List * GetRelationStatsSchemaDependencyList(Oid relationId); static List * GetRelationIndicesDependencyList(Oid relationId); static DependencyDefinition * CreateObjectAddressDependencyDef(Oid classId, Oid objectId); static List * GetTypeConstraintDependencyDefinition(Oid typeId); static List * CreateObjectAddressDependencyDefList(Oid classId, List *objectIdList); static ObjectAddress DependencyDefinitionObjectAddress(DependencyDefinition *definition); static DeferredErrorMessage * DeferErrorIfHasUnsupportedDependency(const ObjectAddress * objectAddress); /* forward declarations for functions to interact with the ObjectAddressCollector */ static void InitObjectAddressCollector(ObjectAddressCollector *collector); static void CollectObjectAddress(ObjectAddressCollector *collector, const ObjectAddress *address); static bool IsObjectAddressCollected(ObjectAddress findAddress, ObjectAddressCollector *collector); static ObjectAddress * GetUndistributableDependency(const ObjectAddress *objectAddress); static bool ObjectAddressHasExtensionDependency(const ObjectAddress *target, ObjectAddress *extensionAddress, int extensionDependency); static void MarkObjectVisited(ObjectAddressCollector *collector, ObjectAddress target); static bool TargetObjectVisited(ObjectAddressCollector *collector, ObjectAddress target); typedef List *(*expandFn)(ObjectAddressCollector *collector, ObjectAddress target); typedef bool (*followFn)(ObjectAddressCollector *collector, DependencyDefinition *definition); typedef void (*applyFn)(ObjectAddressCollector *collector, DependencyDefinition *definition); /* forward declaration of functions that recurse pg_depend */ static void RecurseObjectDependencies(ObjectAddress target, expandFn expand, followFn follow, applyFn apply, ObjectAddressCollector *collector); static List * DependencyDefinitionFromPgDepend(ObjectAddress target); static List * DependencyDefinitionFromPgShDepend(ObjectAddress target); static bool FollowAllSupportedDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition); static bool FollowNewSupportedDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition); static bool FollowAllDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition); static bool FollowExtAndInternalDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition); static void ApplyAddToDependencyList(ObjectAddressCollector *collector, DependencyDefinition *definition); static void ApplyAddCitusDependedObjectsToDependencyList(ObjectAddressCollector * collector, DependencyDefinition * definition); static List * GetViewRuleReferenceDependencyList(Oid relationId); static List * ExpandCitusSupportedTypes(ObjectAddressCollector *collector, ObjectAddress target); static List * ExpandForPgVanilla(ObjectAddressCollector *collector, ObjectAddress target); static List * GetDependentRoleIdsFDW(Oid FDWOid); static List * ExpandRolesToGroups(Oid roleid); static ViewDependencyNode * BuildViewDependencyGraph(Oid relationId, HTAB *nodeMap); static bool IsObjectAddressOwnedByExtension(const ObjectAddress *target, ObjectAddress *extensionAddress); static bool ErrorOrWarnIfObjectHasUnsupportedDependency(const ObjectAddress *objectAddress); /* * GetUniqueDependenciesList takes a list of object addresses and returns a new list * of ObjectAddesses whose elements are unique. */ List * GetUniqueDependenciesList(List *objectAddressesList) { ObjectAddressCollector objectAddressCollector = { 0 }; InitObjectAddressCollector(&objectAddressCollector); ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, objectAddressesList) { if (IsObjectAddressCollected(*objectAddress, &objectAddressCollector)) { /* skip objects that are already collected */ continue; } CollectObjectAddress(&objectAddressCollector, objectAddress); } return objectAddressCollector.dependencyList; } /* * GetDependenciesForObject returns a list of ObjectAddesses to be created in order * before the target object could safely be created on a worker. Some of the object might * already be created on a worker. It should be created in an idempotent way. */ List * GetDependenciesForObject(const ObjectAddress *target) { ObjectAddressCollector collector = { 0 }; InitObjectAddressCollector(&collector); RecurseObjectDependencies(*target, &ExpandCitusSupportedTypes, &FollowNewSupportedDependencies, &ApplyAddToDependencyList, &collector); return collector.dependencyList; } /* * GetAllSupportedDependenciesForObject returns a list of all the ObjectAddresses to be * created in order before the target object could safely be created on a worker, if all * dependent objects are distributable. As a caller, you probably need to use * GetDependenciesForObject() which eliminates already distributed objects from the returned * list. * * Some of the object might already be created on a worker. It should be created * in an idempotent way. */ List * GetAllSupportedDependenciesForObject(const ObjectAddress *target) { ObjectAddressCollector collector = { 0 }; InitObjectAddressCollector(&collector); RecurseObjectDependencies(*target, &ExpandCitusSupportedTypes, &FollowAllSupportedDependencies, &ApplyAddToDependencyList, &collector); return collector.dependencyList; } /* * GetAllDependenciesForObject returns a list of all the dependent objects of the given * object irrespective of whether the dependent object is supported by Citus or not, if * the object can be found as dependency with RecurseObjectDependencies and * ExpandCitusSupportedTypes. * * This function will be used to provide meaningful error messages if any dependent * object for a given object is not supported. If you want to create dependencies for * an object, you probably need to use GetDependenciesForObject(). */ List * GetAllDependenciesForObject(const ObjectAddress *target) { ObjectAddressCollector collector = { 0 }; InitObjectAddressCollector(&collector); RecurseObjectDependencies(*target, &ExpandCitusSupportedTypes, &FollowAllDependencies, &ApplyAddToDependencyList, &collector); return collector.dependencyList; } /* * GetAllCitusDependedDependenciesForObject returns all the dependencies * which are owned by citus extension for the target. */ List * GetAllCitusDependedDependenciesForObject(const ObjectAddress *target) { ObjectAddressCollector collector = { 0 }; InitObjectAddressCollector(&collector); RecurseObjectDependencies(*target, &ExpandForPgVanilla, &FollowExtAndInternalDependencies, &ApplyAddCitusDependedObjectsToDependencyList, &collector); return collector.dependencyList; } /* * OrderObjectAddressListInDependencyOrder given a list of ObjectAddresses return a new * list of the same ObjectAddresses ordered on dependency order where dependencies * precedes the corresponding object in the list. * * The algortihm traveses pg_depend in a depth first order starting at the first object in * the provided list. By traversing depth first it will put the first dependency at the * head of the list with dependencies depending on them later. * * If the object is already in the list it is skipped for traversal. This happens when an * object was already added to the target list before it occurred in the input list. */ List * OrderObjectAddressListInDependencyOrder(List *objectAddressList) { ObjectAddressCollector collector = { 0 }; InitObjectAddressCollector(&collector); ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, objectAddressList) { if (IsObjectAddressCollected(*objectAddress, &collector)) { /* skip objects that are already ordered */ continue; } RecurseObjectDependencies(*objectAddress, &ExpandCitusSupportedTypes, &FollowAllSupportedDependencies, &ApplyAddToDependencyList, &collector); CollectObjectAddress(&collector, objectAddress); } return collector.dependencyList; } /* * RecurseObjectDependencies recursively visits all dependencies of an object. It sources * the dependencies from pg_depend and pg_shdepend while 'expanding' the list via an * optional `expand` function. * * Starting from the target ObjectAddress. For every dependency found the `follow` * function will be called. When `follow` returns true it will recursively visit the * dependencies for that object. * * Visiting will happen in depth first order, which is useful to create or sorted lists of * dependencies to create. * * For all dependencies that should be visited the apply function will be called. This * function is designed to be the mutating function for the context being passed. Although * nothing prevents the follow function to also mutate the context. * * - follow will be called on the way down, so the invocation order is top to bottom of * the dependency tree * - apply is called on the way back, so the invocation order is bottom to top. Apply is * not called for entries for which follow has returned false. */ static void RecurseObjectDependencies(ObjectAddress target, expandFn expand, followFn follow, applyFn apply, ObjectAddressCollector *collector) { if (TargetObjectVisited(collector, target)) { /* prevent infinite loops due to circular dependencies */ return; } MarkObjectVisited(collector, target); /* lookup both pg_depend and pg_shdepend for dependencies */ List *pgDependDefinitions = DependencyDefinitionFromPgDepend(target); List *pgShDependDefinitions = DependencyDefinitionFromPgShDepend(target); List *dependenyDefinitionList = list_concat(pgDependDefinitions, pgShDependDefinitions); /* concat expanded entries if applicable */ if (expand != NULL) { List *expandedEntries = expand(collector, target); dependenyDefinitionList = list_concat(dependenyDefinitionList, expandedEntries); } /* iterate all entries and recurse depth first */ DependencyDefinition *dependencyDefinition = NULL; foreach_declared_ptr(dependencyDefinition, dependenyDefinitionList) { if (follow == NULL || !follow(collector, dependencyDefinition)) { /* skip all pg_depend entries the user didn't want to follow */ continue; } /* * recurse depth first, this makes sure we call apply for the deepest dependency * first. */ ObjectAddress address = DependencyDefinitionObjectAddress(dependencyDefinition); RecurseObjectDependencies(address, expand, follow, apply, collector); /* now apply changes for current entry */ if (apply != NULL) { apply(collector, dependencyDefinition); } } } /* * DependencyDefinitionFromPgDepend loads all pg_depend records describing the * dependencies of target. */ static List * DependencyDefinitionFromPgDepend(ObjectAddress target) { ScanKeyData key[2]; HeapTuple depTup = NULL; List *dependenyDefinitionList = NIL; /* * iterate the actual pg_depend catalog */ Relation depRel = table_open(DependRelationId, AccessShareLock); /* scan pg_depend for classid = $1 AND objid = $2 using pg_depend_depender_index */ ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target.classId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target.objectId)); SysScanDesc depScan = systable_beginscan(depRel, DependDependerIndexId, true, NULL, 2, key); while (HeapTupleIsValid(depTup = systable_getnext(depScan))) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); DependencyDefinition *dependency = palloc0(sizeof(DependencyDefinition)); /* keep track of all pg_depend records as dependency definitions */ dependency->mode = DependencyPgDepend; dependency->data.pg_depend = *pg_depend; dependenyDefinitionList = lappend(dependenyDefinitionList, dependency); } systable_endscan(depScan); relation_close(depRel, AccessShareLock); return dependenyDefinitionList; } /* * DependencyDefinitionFromPgShDepend loads all pg_shdepend records describing the * dependencies of target. */ static List * DependencyDefinitionFromPgShDepend(ObjectAddress target) { ScanKeyData key[3]; HeapTuple depTup = NULL; List *dependenyDefinitionList = NIL; /* * iterate the actual pg_shdepend catalog */ Relation shdepRel = table_open(SharedDependRelationId, AccessShareLock); /* * Scan pg_shdepend for dbid = $1 AND classid = $2 AND objid = $3 using * pg_shdepend_depender_index * * where $1 is decided as follows: * - shared dependencies $1 = InvalidOid * - other dependencies $1 = MyDatabaseId * This is consistent with postgres' static classIdGetDbId function */ Oid dbid = InvalidOid; if (!IsSharedRelation(target.classId)) { dbid = MyDatabaseId; } ScanKeyInit(&key[0], Anum_pg_shdepend_dbid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(dbid)); ScanKeyInit(&key[1], Anum_pg_shdepend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target.classId)); ScanKeyInit(&key[2], Anum_pg_shdepend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target.objectId)); SysScanDesc shdepScan = systable_beginscan(shdepRel, SharedDependDependerIndexId, true, NULL, 3, key); while (HeapTupleIsValid(depTup = systable_getnext(shdepScan))) { Form_pg_shdepend pg_shdepend = (Form_pg_shdepend) GETSTRUCT(depTup); DependencyDefinition *dependency = palloc0(sizeof(DependencyDefinition)); /* keep track of all pg_shdepend records as dependency definitions */ dependency->mode = DependencyPgShDepend; dependency->data.pg_shdepend = *pg_shdepend; dependenyDefinitionList = lappend(dependenyDefinitionList, dependency); } systable_endscan(shdepScan); relation_close(shdepRel, AccessShareLock); return dependenyDefinitionList; } /* * InitObjectAddressCollector takes a pointer to an already allocated (possibly stack) * ObjectAddressCollector struct. It makes sure this struct is ready to be used for object * collection. * * If an already initialized collector is passed the collector will be cleared from its * contents to be reused. */ static void InitObjectAddressCollector(ObjectAddressCollector *collector) { assert_valid_hash_key3(ObjectAddress, classId, objectId, objectSubId); collector->dependencySet = CreateSimpleHashSetWithName(ObjectAddress, "dependency set"); collector->dependencyList = NULL; collector->visitedObjects = CreateSimpleHashSetWithName(ObjectAddress, "visited object set"); } /* * TargetObjectVisited returns true if the input target has been visited while * traversing pg_depend. */ static bool TargetObjectVisited(ObjectAddressCollector *collector, ObjectAddress target) { bool found = false; /* find in set */ hash_search(collector->visitedObjects, &target, HASH_FIND, &found); return found; } /* * MarkObjectVisited marks the object as visited during the traversal of * pg_depend. */ static void MarkObjectVisited(ObjectAddressCollector *collector, ObjectAddress target) { bool found = false; /* add to set */ ObjectAddress *address = (ObjectAddress *) hash_search(collector->visitedObjects, &target, HASH_ENTER, &found); if (!found) { /* copy object address in */ *address = target; } } /* * CollectObjectAddress adds an ObjectAddress to the collector. */ static void CollectObjectAddress(ObjectAddressCollector *collector, const ObjectAddress *collect) { bool found = false; /* add to set */ ObjectAddress *address = (ObjectAddress *) hash_search(collector->dependencySet, collect, HASH_ENTER, &found); if (!found) { /* copy object address in */ *address = *collect; } /* add to list*/ collector->dependencyList = lappend(collector->dependencyList, address); } /* * IsObjectAddressCollected is a helper function that can check if an ObjectAddress is * already in a (unsorted) list of ObjectAddresses */ static bool IsObjectAddressCollected(ObjectAddress findAddress, ObjectAddressCollector *collector) { bool found = false; /* add to set */ hash_search(collector->dependencySet, &findAddress, HASH_FIND, &found); return found; } /* * SupportedDependencyByCitus returns whether citus has support to distribute the object * addressed. */ bool SupportedDependencyByCitus(const ObjectAddress *address) { if (!EnableMetadataSync) { /* * If the user has disabled object propagation we need to fall back to the legacy * behaviour in which we only support schema creation */ switch (getObjectClass(address)) { case OCLASS_SCHEMA: { return !isTempNamespace(address->objectId); } default: { return false; } } /* should be unreachable */ Assert(false); } /* * looking at the type of a object to see if we know how to create the object on the * workers. */ switch (getObjectClass(address)) { case OCLASS_AM: { /* * Only support access methods if they came from extensions * During the dependency resolution it will cascade into the extension and * distributed that one instead of the Access Method. Now access methods can * be configured on tables on the workers. */ return IsObjectAddressOwnedByExtension(address, NULL); } case OCLASS_CONSTRAINT: { /* * Constraints are only supported when on domain types. Other constraints have * their typid set to InvalidOid. */ return OidIsValid(get_constraint_typid(address->objectId)); } case OCLASS_COLLATION: { return true; } case OCLASS_SCHEMA: { return !isTempNamespace(address->objectId); } case OCLASS_PROC: { return true; } case OCLASS_DATABASE: { return true; } case OCLASS_FOREIGN_SERVER: { return true; } case OCLASS_ROLE: { /* if it is a reserved role do not propagate */ if (IsReservedName(GetUserNameFromId(address->objectId, false))) { return false; } return true; } case OCLASS_EXTENSION: { return true; } case OCLASS_PUBLICATION: { return true; } case OCLASS_TSCONFIG: { return true; } case OCLASS_TSDICT: { return true; } case OCLASS_TYPE: { switch (get_typtype(address->objectId)) { case TYPTYPE_ENUM: case TYPTYPE_COMPOSITE: case TYPTYPE_DOMAIN: { return true; } case TYPTYPE_BASE: { /* * array types should be followed but not created, as they get created * by the original type. */ return type_is_array(address->objectId); } default: { /* type not supported */ return false; } } /* * should be unreachable, break here is to make sure the function has a path * without return, instead of falling through to the next block */ break; } case OCLASS_CLASS: { char relKind = get_rel_relkind(address->objectId); /* * composite types have a reference to a relation of composite type, we need * to follow those to get the dependencies of type fields. * * As we also handle tables as objects as well, follow dependencies * for tables. */ if (relKind == RELKIND_COMPOSITE_TYPE || relKind == RELKIND_RELATION || relKind == RELKIND_PARTITIONED_TABLE || relKind == RELKIND_FOREIGN_TABLE || relKind == RELKIND_SEQUENCE || relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX || relKind == RELKIND_VIEW) { return true; } return false; } default: { /* unsupported type */ return false; } } } /* * ErrorOrWarnIfObjectHasUnsupportedDependency returns false without throwing any message if * object doesn't have any unsupported dependency, else throws a message with proper level * (except the cluster doesn't have any node) and return true. */ static bool ErrorOrWarnIfObjectHasUnsupportedDependency(const ObjectAddress *objectAddress) { DeferredErrorMessage *errMsg = DeferErrorIfHasUnsupportedDependency(objectAddress); if (errMsg != NULL) { /* * Don't need to give any messages if there is no worker nodes in * the cluster as user's experience won't be affected on the single node even * if the object won't be distributed. */ if (!HasAnyNodes()) { return true; } /* * Since Citus drops and recreates some object while converting a table type * giving a DEBUG1 message is enough if the process in table type conversion * function call */ if (InTableTypeConversionFunctionCall) { RaiseDeferredError(errMsg, DEBUG1); } /* * If the view is object distributed, we should provide an error to not have * different definition of object on coordinator and worker nodes. If the object * is not distributed yet, we can create it locally to not affect user's local * usage experience. */ else if (IsAnyObjectDistributed(list_make1((ObjectAddress *) objectAddress))) { RaiseDeferredError(errMsg, ERROR); } else { if (EnableUnsupportedFeatureMessages) { RaiseDeferredError(errMsg, WARNING); } } return true; } return false; } /* * ErrorOrWarnIfAnyObjectHasUnsupportedDependency iteratively calls * ErrorOrWarnIfObjectHasUnsupportedDependency for given addresses. */ bool ErrorOrWarnIfAnyObjectHasUnsupportedDependency(List *objectAddresses) { ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, objectAddresses) { if (ErrorOrWarnIfObjectHasUnsupportedDependency(objectAddress)) { return true; } } return false; } /* * DeferErrorIfHasUnsupportedDependency returns deferred error message if the given * object has any undistributable dependency. */ static DeferredErrorMessage * DeferErrorIfHasUnsupportedDependency(const ObjectAddress *objectAddress) { ObjectAddress *undistributableDependency = GetUndistributableDependency( objectAddress); if (undistributableDependency == NULL) { return NULL; } StringInfo errorInfo = makeStringInfo(); StringInfo detailInfo = makeStringInfo(); char *objectDescription = getObjectDescription(objectAddress, false); char *dependencyDescription = getObjectDescription(undistributableDependency, false); /* * We expect callers to interpret the error returned from this function * as a warning if the object itself is just being created. In that case, * we expect them to report below error detail as well to indicate that * object itself will not be propagated but will still be created locally. * * Otherwise, callers are expected to throw the error returned from this * function as a hard one by ignoring the detail part. */ if (!IsAnyObjectDistributed(list_make1((ObjectAddress *) objectAddress))) { appendStringInfo(detailInfo, "\"%s\" will be created only locally", objectDescription); } if (SupportedDependencyByCitus(undistributableDependency)) { StringInfo hintInfo = makeStringInfo(); appendStringInfo(errorInfo, "\"%s\" has dependency to \"%s\" that is not in " "Citus' metadata", objectDescription, dependencyDescription); if (IsAnyObjectDistributed(list_make1((ObjectAddress *) objectAddress))) { appendStringInfo(hintInfo, "Distribute \"%s\" first to modify \"%s\" on worker nodes", dependencyDescription, objectDescription); } else { appendStringInfo(hintInfo, "Distribute \"%s\" first to distribute \"%s\"", dependencyDescription, objectDescription); } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorInfo->data, detailInfo->data, hintInfo->data); } appendStringInfo(errorInfo, "\"%s\" has dependency on unsupported " "object \"%s\"", objectDescription, dependencyDescription); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorInfo->data, detailInfo->data, NULL); } /* * DeferErrorIfAnyObjectHasUnsupportedDependency iteratively calls * DeferErrorIfHasUnsupportedDependency for given addresses. */ DeferredErrorMessage * DeferErrorIfAnyObjectHasUnsupportedDependency(const List *objectAddresses) { DeferredErrorMessage *deferredErrorMessage = NULL; ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, objectAddresses) { deferredErrorMessage = DeferErrorIfHasUnsupportedDependency(objectAddress); if (deferredErrorMessage) { return deferredErrorMessage; } } return NULL; } /* * GetUndistributableDependency checks whether object has any non-distributable * dependency. If any one found, it will be returned. */ static ObjectAddress * GetUndistributableDependency(const ObjectAddress *objectAddress) { List *dependencies = GetAllDependenciesForObject(objectAddress); ObjectAddress *dependency = NULL; /* * Users can disable metadata sync by their own risk. If it is disabled, Citus * doesn't propagate dependencies. So, if it is disabled, there is no undistributable * dependency. */ if (!EnableMetadataSync) { return NULL; } foreach_declared_ptr(dependency, dependencies) { /* * Objects with the id smaller than FirstNormalObjectId should be created within * initdb. Citus needs to have such objects as distributed, so we can not add * such check to dependency resolution logic. Though, Citus shouldn't error * out if such dependency is not supported. So, skip them here. */ if (dependency->objectId < FirstNormalObjectId) { continue; } /* * If object is distributed already, ignore it. */ if (IsAnyObjectDistributed(list_make1(dependency))) { continue; } /* * If the dependency is not supported with Citus, return the dependency. */ if (!SupportedDependencyByCitus(dependency)) { /* * Since we do not yet support distributed TS TEMPLATE and AM objects, we skip * dependency checks for text search templates. The user is expected to * manually create the TS TEMPLATE and AM objects. */ if (getObjectClass(dependency) != OCLASS_TSTEMPLATE && getObjectClass(dependency) != OCLASS_AM) { return dependency; } } if (getObjectClass(dependency) == OCLASS_CLASS) { char relKind = get_rel_relkind(dependency->objectId); if (relKind == RELKIND_SEQUENCE || relKind == RELKIND_COMPOSITE_TYPE || relKind == RELKIND_VIEW) { /* citus knows how to auto-distribute these dependencies */ continue; } else if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { /* * Indexes are only qualified for distributed objects for dependency * tracking purposes, so we can ignore those. */ continue; } else { /* * Citus doesn't know how to auto-distribute the rest of the RELKINDs * via dependency resolution */ return dependency; } } } return NULL; } /* * IsTableOwnedByExtension returns whether the table with the given relation ID is * owned by an extension. */ bool IsTableOwnedByExtension(Oid relationId) { ObjectAddress tableAddress = { 0 }; ObjectAddressSet(tableAddress, RelationRelationId, relationId); return IsObjectAddressOwnedByExtension(&tableAddress, NULL); } /* * ObjectAddressDependsOnExtension returns whether or not the object depends * on an extension. It is assumed that "an object having a dependency of type * DEPENDENCY_AUTO_EXTENSION to an extension" depends on that extension. */ bool ObjectAddressDependsOnExtension(const ObjectAddress *target) { return ObjectAddressHasExtensionDependency(target, NULL, DEPENDENCY_AUTO_EXTENSION); } /* * IsObjectAddressOwnedByExtension returns whether or not the object is owned by an * extension. It is assumed that an object having a dependency on an extension is created * by that extension and therefore owned by that extension. * * If extensionAddress is not set to a NULL pointer the function will write the extension * address this function depends on into this location. */ static bool IsObjectAddressOwnedByExtension(const ObjectAddress *target, ObjectAddress *extensionAddress) { return ObjectAddressHasExtensionDependency(target, extensionAddress, DEPENDENCY_EXTENSION); } /* * ObjectAddressHasExtensionDependency is a helper function that returns true if * given object has a dependency record (of type DEPENDENCY_EXTENSION or * DEPENDENCY_AUTO_EXTENSION) for an extension. * * If extensionAddress is not set to a NULL pointer the function will write the * extension address this function depends on into this location. */ static bool ObjectAddressHasExtensionDependency(const ObjectAddress *target, ObjectAddress *extensionAddress, int extensionDependency) { Assert(extensionDependency == DEPENDENCY_EXTENSION || extensionDependency == DEPENDENCY_AUTO_EXTENSION); ScanKeyData key[2]; HeapTuple depTup = NULL; bool result = false; Relation depRel = table_open(DependRelationId, AccessShareLock); /* scan pg_depend for classid = $1 AND objid = $2 using pg_depend_depender_index */ ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target->classId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(target->objectId)); SysScanDesc depScan = systable_beginscan(depRel, DependDependerIndexId, true, NULL, 2, key); while (HeapTupleIsValid(depTup = systable_getnext(depScan))) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); if (pg_depend->deptype == extensionDependency) { result = true; if (extensionAddress != NULL) { ObjectAddressSubSet(*extensionAddress, pg_depend->refclassid, pg_depend->refobjid, pg_depend->refobjsubid); } break; } } systable_endscan(depScan); table_close(depRel, AccessShareLock); return result; } /* * IsAnyObjectAddressOwnedByExtension iteratively calls IsObjectAddressOwnedByExtension * for given addresses to determine if any address is owned by an extension. */ bool IsAnyObjectAddressOwnedByExtension(const List *targets, ObjectAddress *extensionAddress) { ObjectAddress *target = NULL; foreach_declared_ptr(target, targets) { if (IsObjectAddressOwnedByExtension(target, extensionAddress)) { return true; } } return false; } /* * FirstExtensionWithSchema returns the first extension address whose schema is the same * as given schema. If no extension depends on the schema, it returns NULL. * i.e. decide if given schema is an extension schema as in * `CREATE EXTENSION [WITH] SCHEMA ;` */ ObjectAddress * FirstExtensionWithSchema(Oid schemaId) { ObjectAddress *extensionAddress = NULL; Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyData entry[1]; ScanKeyInit(&entry[0], Anum_pg_extension_extnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(schemaId)); SysScanDesc scan = systable_beginscan(relation, InvalidOid, false, NULL, 1, entry); HeapTuple extensionTuple = systable_getnext(scan); if (HeapTupleIsValid(extensionTuple)) { int extensionIdIndex = Anum_pg_extension_oid; TupleDesc tupleDescriptor = RelationGetDescr(relation); bool isNull = false; Datum extensionIdDatum = heap_getattr(extensionTuple, extensionIdIndex, tupleDescriptor, &isNull); Oid extensionId = DatumGetObjectId(extensionIdDatum); extensionAddress = palloc0(sizeof(ObjectAddress)); extensionAddress->objectId = extensionId; extensionAddress->classId = ExtensionRelationId; extensionAddress->objectSubId = 0; } systable_endscan(scan); table_close(relation, AccessShareLock); return extensionAddress; } /* * IsObjectAddressOwnedByCitus returns true if the given object address * is owned by the citus or citus_columnar extensions. */ bool IsObjectAddressOwnedByCitus(const ObjectAddress *objectAddress) { Oid citusId = get_extension_oid("citus", true); Oid citusColumnarId = get_extension_oid("citus_columnar", true); /* return false because we could not find any citus extension */ if (!OidIsValid(citusId) && !OidIsValid(citusColumnarId)) { return false; } ObjectAddress extObjectAddress = InvalidObjectAddress; bool ownedByExt = IsObjectAddressOwnedByExtension(objectAddress, &extObjectAddress); if (!ownedByExt) { return false; } bool ownedByCitus = OidIsValid(citusId) && extObjectAddress.objectId == citusId; bool ownedByCitusColumnar = OidIsValid(citusColumnarId) && extObjectAddress.objectId == citusColumnarId; return ownedByCitus || ownedByCitusColumnar; } /* * FollowNewSupportedDependencies applies filters on pg_depend entries to follow all * objects which should be distributed before the root object can safely be created. */ static bool FollowNewSupportedDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition) { if (definition->mode == DependencyPgDepend) { /* * For dependencies found in pg_depend: * * Follow only normal and extension dependencies. The latter is used to reach the * extensions, the objects that directly depend on the extension are eliminated * during the "apply" phase. * * Other dependencies are internal dependencies and managed by postgres. */ if (definition->data.pg_depend.deptype != DEPENDENCY_NORMAL && definition->data.pg_depend.deptype != DEPENDENCY_EXTENSION) { return false; } } /* rest of the tests are to see if we want to follow the actual dependency */ ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * If the object is already in our dependency list we do not have to follow any * further */ if (IsObjectAddressCollected(address, collector)) { return false; } /* * If the object is already distributed it is not a `new` object that needs to be * distributed before we create a dependent object */ ObjectAddress *copyAddress = palloc0(sizeof(ObjectAddress)); *copyAddress = address; if (IsAnyObjectDistributed(list_make1(copyAddress))) { return false; } /* * We can only distribute dependencies that citus knows how to distribute. * * But we don't want to bail out if the object is owned by extension, because * Citus can create the extension. */ if (!SupportedDependencyByCitus(&address) && !IsObjectAddressOwnedByExtension(&address, NULL)) { return false; } if (CitusExtensionObject(&address)) { /* following citus extension could complicate role management */ return false; } return true; } /* * FollowAllSupportedDependencies applies filters on pg_depend entries to follow the * dependency tree of objects in depth first order. We will visit all supported objects. * This is used to sort a list of dependencies in dependency order. */ static bool FollowAllSupportedDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition) { if (definition->mode == DependencyPgDepend) { /* * For dependencies found in pg_depend: * * Follow only normal and extension dependencies. The latter is used to reach the * extensions, the objects that directly depend on the extension are eliminated * during the "apply" phase. * * Other dependencies are internal dependencies and managed by postgres. */ if (definition->data.pg_depend.deptype != DEPENDENCY_NORMAL && definition->data.pg_depend.deptype != DEPENDENCY_EXTENSION) { return false; } } /* rest of the tests are to see if we want to follow the actual dependency */ ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * If the object is already in our dependency list we do not have to follow any * further */ if (IsObjectAddressCollected(address, collector)) { return false; } /* * We can only distribute dependencies that citus knows how to distribute. * * But we don't want to bail out if the object is owned by extension, because * Citus can create the extension. */ if (!SupportedDependencyByCitus(&address) && !IsObjectAddressOwnedByExtension(&address, NULL)) { return false; } if (CitusExtensionObject(&address)) { /* following citus extension could complicate role management */ return false; } return true; } /* * FollowAllDependencies applies filters on pg_depend entries to follow the dependency * tree of objects in depth first order. We will visit all objects irrespective of it is * supported by Citus or not. */ static bool FollowAllDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition) { if (definition->mode == DependencyPgDepend) { /* * For dependencies found in pg_depend: * * Follow only normal and extension dependencies. The latter is used to reach the * extensions, the objects that directly depend on the extension are eliminated * during the "apply" phase. * * Other dependencies are internal dependencies and managed by postgres. */ if (definition->data.pg_depend.deptype != DEPENDENCY_NORMAL && definition->data.pg_depend.deptype != DEPENDENCY_EXTENSION) { return false; } } /* rest of the tests are to see if we want to follow the actual dependency */ ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * If the object is already in our dependency list we do not have to follow any * further */ if (IsObjectAddressCollected(address, collector)) { return false; } if (CitusExtensionObject(&address)) { /* following citus extension could complicate role management */ return false; } return true; } /* * FollowExtAndInternalDependencies applies filters on pg_depend entries to follow * the dependency tree of objects in depth first order. We will visit all objects * irrespective of it is supported by Citus or not and it is internal or not. */ static bool FollowExtAndInternalDependencies(ObjectAddressCollector *collector, DependencyDefinition *definition) { ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * If the object is already in our dependency list we do not have to follow any * further */ if (IsObjectAddressCollected(address, collector)) { return false; } if (CitusExtensionObject(&address)) { /* * We do not need to follow citus extension because the purpose * of our walk is to find if an object is owned by citus. */ return false; } return true; } /* * ApplyAddToDependencyList is an apply function for RecurseObjectDependencies that will * collect all the ObjectAddresses for pg_depend entries to the context, except it is * extension owned one. * * The context here is assumed to be a (ObjectAddressCollector *) to the location where * all ObjectAddresses will be collected. */ static void ApplyAddToDependencyList(ObjectAddressCollector *collector, DependencyDefinition *definition) { ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * Objects owned by an extension are assumed to be created on the workers by creating * the extension in the cluster, we we don't want explicitly create them. * * Since we do need to capture the extension as a dependency we are following the * object instead of breaking the traversal there. */ if (IsObjectAddressOwnedByExtension(&address, NULL)) { return; } CollectObjectAddress(collector, &address); } /* * ApplyAddCitusDependedObjectsToDependencyList is an apply function for * RecurseObjectDependencies that will collect all the ObjectAddresses for * pg_depend entries to the context if it is citus extension owned one. * * The context here is assumed to be a (ObjectAddressCollector *) to the location where * all ObjectAddresses will be collected. */ static void ApplyAddCitusDependedObjectsToDependencyList(ObjectAddressCollector *collector, DependencyDefinition *definition) { ObjectAddress address = DependencyDefinitionObjectAddress(definition); /* * We only collect the object if it is owned by citus extension. */ if (IsObjectAddressOwnedByCitus(&address)) { CollectObjectAddress(collector, &address); } } /* * ExpandCitusSupportedTypes base on supported types by citus we might want to expand * the list of objects to visit in pg_depend. * * An example where we want to expand is for types. Their dependencies are not captured * with an entry in pg_depend from their object address, but by the object address of the * relation describing the type. */ static List * ExpandCitusSupportedTypes(ObjectAddressCollector *collector, ObjectAddress target) { List *result = NIL; switch (target.classId) { case AuthIdRelationId: { /* * Roles are members of other roles. These relations are not recorded directly * but can be deduced from pg_auth_members */ return ExpandRolesToGroups(target.objectId); } case ExtensionRelationId: { /* * FDWs get propagated along with the extensions they belong to. * In case there are GRANTed privileges on FDWs to roles, those * GRANT statements will be propagated to. In order to make sure * that those GRANT statements work, the privileged roles should * exist on the worker nodes. Hence, here we find these dependent * roles and add them as dependencies. */ Oid extensionId = target.objectId; List *FDWOids = GetDependentFDWsToExtension(extensionId); Oid FDWOid = InvalidOid; foreach_declared_oid(FDWOid, FDWOids) { List *dependentRoleIds = GetDependentRoleIdsFDW(FDWOid); List *dependencies = CreateObjectAddressDependencyDefList(AuthIdRelationId, dependentRoleIds); result = list_concat(result, dependencies); } break; } case TypeRelationId: { switch (get_typtype(target.objectId)) { /* * types depending on other types are not captured in pg_depend, instead * they are described with their dependencies by the relation that * describes the composite type. */ case TYPTYPE_COMPOSITE: { Oid typeRelationId = get_typ_typrelid(target.objectId); DependencyDefinition *dependency = CreateObjectAddressDependencyDef(RelationRelationId, typeRelationId); result = lappend(result, dependency); break; } /* * Domains can have constraints associated with them. Constraints themself * can depend on things like functions. To support the propagation of * these functions we will add the constraints to the list of objects to * be created. */ case TYPTYPE_DOMAIN: { List *dependencies = GetTypeConstraintDependencyDefinition(target.objectId); result = list_concat(result, dependencies); break; } } /* * array types don't have a normal dependency on their element type, instead * their dependency is an internal one. We can't follow interal dependencies * as that would cause a cyclic dependency on others, instead we expand here * to follow the dependency on the element type. */ if (type_is_array(target.objectId)) { Oid typeId = get_element_type(target.objectId); DependencyDefinition *dependency = CreateObjectAddressDependencyDef(TypeRelationId, typeId); result = lappend(result, dependency); } break; } case RelationRelationId: { /* * Triggers both depend to the relations and to the functions they * execute. Also, pg_depend records dependencies from triggers to the * functions but not from relations to their triggers. Given above two, * we directly expand depencies for the relations to trigger functions. * That way, we won't attempt to create the trigger as a dependency of * the relation, which would fail as the relation itself is not created * yet when ensuring dependencies. */ Oid relationId = target.objectId; List *triggerFunctionDepencyList = GetRelationTriggerFunctionDependencyList(relationId); result = list_concat(result, triggerFunctionDepencyList); /* * Statistics' both depend to the relations and to the schemas they belong * to. Also, pg_depend records dependencies from statistics to their schemas * but not from relations to their statistics' schemas. Given above two, * we directly expand dependencies for the relations to schemas of * statistics. */ List *statisticsSchemaDependencyList = GetRelationStatsSchemaDependencyList(relationId); result = list_concat(result, statisticsSchemaDependencyList); /* * Get the dependent sequences for tables (both as serial columns and * columns have nextval with existing sequences) and expand dependency list * with them. */ List *sequenceDependencyList = GetRelationSequenceDependencyList(relationId); result = list_concat(result, sequenceDependencyList); /* * Get the dependent functions for tables as columns has default values * and contraints, then expand dependency list with them. */ List *functionDependencyList = GetRelationFunctionDependencyList(relationId); result = list_concat(result, functionDependencyList); /* * Tables could have indexes. Indexes themself could have dependencies that * need to be propagated. eg. TEXT SEARCH CONFIGURATIONS. Here we add the * addresses of all indices to the list of objects to vist, as to make sure we * create all objects required by the indices before we create the table * including indices. */ List *indexDependencyList = GetRelationIndicesDependencyList(relationId); result = list_concat(result, indexDependencyList); /* * Get the dependencies of the rule for the given view. PG keeps internal * dependency between view and rule. As it is stated on the PG doc, if * there is an internal dependency, dependencies of the dependent object * behave much like they were dependencies of the referenced object. * * We need to expand dependencies by including dependencies of the rule * internally dependent to the view. PG doesn't keep any dependencies * from view to any object, but it keeps an internal dependency to the * rule and that rule has dependencies to other objects. */ char relKind = get_rel_relkind(relationId); if (relKind == RELKIND_VIEW || relKind == RELKIND_MATVIEW) { List *ruleRefDepList = GetViewRuleReferenceDependencyList(relationId); result = list_concat(result, ruleRefDepList); } break; } case PublicationRelationId: { Oid publicationId = target.objectId; /* * Publications do not depend directly on relations, because dropping * the relation will only remove it from the publications. However, * we add a dependency to ensure the relation is created first when * adding a node. */ List *relationDependencyList = GetPublicationRelationsDependencyList(publicationId); result = list_concat(result, relationDependencyList); /* * As of PostgreSQL 15, the same applies to schemas. */ List *schemaIdList = GetPublicationSchemas(publicationId); List *schemaDependencyList = CreateObjectAddressDependencyDefList(NamespaceRelationId, schemaIdList); result = list_concat(result, schemaDependencyList); break; } default: { /* no expansion for unsupported types */ break; } } return result; } /* * ExpandForPgVanilla only expands only comosite types because other types * will find their dependencies in pg_depend. The method should only be called by * is_citus_depended_object udf. */ static List * ExpandForPgVanilla(ObjectAddressCollector *collector, ObjectAddress target) { /* should only be called if GUC is enabled */ Assert(HideCitusDependentObjects == true); List *result = NIL; if (target.classId == TypeRelationId && get_typtype(target.objectId) == TYPTYPE_COMPOSITE) { /* * types depending on other types are not captured in pg_depend, instead * they are described with their dependencies by the relation that * describes the composite type. */ Oid typeRelationId = get_typ_typrelid(target.objectId); DependencyDefinition *dependency = CreateObjectAddressDependencyDef(RelationRelationId, typeRelationId); result = lappend(result, dependency); } return result; } /* * GetDependentRoleIdsFDW returns a list of role oids that has privileges on the * FDW with the given object id. */ static List * GetDependentRoleIdsFDW(Oid FDWOid) { List *roleIds = NIL; Acl *aclEntry = GetPrivilegesForFDW(FDWOid); if (aclEntry == NULL) { return NIL; } AclItem *privileges = ACL_DAT(aclEntry); int numberOfPrivsGranted = ACL_NUM(aclEntry); for (int i = 0; i < numberOfPrivsGranted; i++) { roleIds = lappend_oid(roleIds, privileges[i].ai_grantee); } return roleIds; } /* * ExpandRolesToGroups returns a list of object addresses pointing to roles that roleid * depends on. */ static List * ExpandRolesToGroups(Oid roleid) { Relation pgAuthMembers = table_open(AuthMemRelationId, AccessShareLock); HeapTuple tuple = NULL; ScanKeyData scanKey[1]; const int scanKeyCount = 1; /* scan pg_auth_members for member = $1 via index pg_auth_members_member_role_index */ ScanKeyInit(&scanKey[0], Anum_pg_auth_members_member, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(roleid)); SysScanDesc scanDescriptor = systable_beginscan(pgAuthMembers, AuthMemMemRoleIndexId, true, NULL, scanKeyCount, scanKey); List *roles = NIL; while ((tuple = systable_getnext(scanDescriptor)) != NULL) { Form_pg_auth_members membership = (Form_pg_auth_members) GETSTRUCT(tuple); DependencyDefinition *definition = palloc0(sizeof(DependencyDefinition)); definition->mode = DependencyObjectAddress; ObjectAddressSet(definition->data.address, AuthIdRelationId, membership->roleid); roles = lappend(roles, definition); } systable_endscan(scanDescriptor); table_close(pgAuthMembers, AccessShareLock); return roles; } /* * GetViewRuleReferenceDependencyList returns the dependencies of the view's * internal rule dependencies. */ static List * GetViewRuleReferenceDependencyList(Oid viewId) { List *dependencyTupleList = GetPgDependTuplesForDependingObjects(RelationRelationId, viewId); List *nonInternalDependenciesOfDependingRules = NIL; HeapTuple depTup = NULL; foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); /* * Dependencies of the internal rule dependency should be handled as the dependency * of referenced view object. * * PG doesn't keep dependency relation between views and dependent objects directly * but it keeps an internal dependency relation between the view and the rule, then * keeps the dependent objects of the view as non-internal dependencies of the * internally dependent rule object. */ if (pg_depend->deptype == DEPENDENCY_INTERNAL && pg_depend->classid == RewriteRelationId) { ObjectAddress ruleAddress = { 0 }; ObjectAddressSet(ruleAddress, RewriteRelationId, pg_depend->objid); /* Expand results with the noninternal dependencies of it */ List *ruleDependencies = DependencyDefinitionFromPgDepend(ruleAddress); DependencyDefinition *dependencyDef = NULL; foreach_declared_ptr(dependencyDef, ruleDependencies) { /* * Follow all dependencies of the internally dependent rule dependencies * except it is an internal dependency of view itself. */ if (dependencyDef->data.pg_depend.deptype == DEPENDENCY_INTERNAL || (dependencyDef->data.pg_depend.refclassid == RelationRelationId && dependencyDef->data.pg_depend.refobjid == viewId)) { continue; } nonInternalDependenciesOfDependingRules = lappend(nonInternalDependenciesOfDependingRules, dependencyDef); } } } return nonInternalDependenciesOfDependingRules; } /* * GetRelationSequenceDependencyList returns the sequence dependency definition * list for the given relation. */ static List * GetRelationSequenceDependencyList(Oid relationId) { List *seqInfoList = NIL; GetDependentSequencesWithRelation(relationId, &seqInfoList, 0, DEPENDENCY_AUTO); List *seqIdList = NIL; SequenceInfo *seqInfo = NULL; foreach_declared_ptr(seqInfo, seqInfoList) { seqIdList = lappend_oid(seqIdList, seqInfo->sequenceOid); } List *sequenceDependencyDefList = CreateObjectAddressDependencyDefList(RelationRelationId, seqIdList); return sequenceDependencyDefList; } /* * GetRelationFunctionDependencyList returns the function dependency definition * list for the given relation. */ static List * GetRelationFunctionDependencyList(Oid relationId) { List *dependentFunctionOids = GetDependentFunctionsWithRelation(relationId); List *functionDependencyDefList = CreateObjectAddressDependencyDefList(ProcedureRelationId, dependentFunctionOids); return functionDependencyDefList; } /* * GetRelationStatsSchemaDependencyList returns a list of DependencyDefinition * objects for the schemas that statistics' of the relation with relationId depends. */ static List * GetRelationStatsSchemaDependencyList(Oid relationId) { List *schemaIds = GetExplicitStatisticsSchemaIdList(relationId); return CreateObjectAddressDependencyDefList(NamespaceRelationId, schemaIds); } /* * CollectIndexOids implements PGIndexProcessor to create a list of all index oids */ static void CollectIndexOids(Form_pg_index formPgIndex, List **oids, int flags) { *oids = lappend_oid(*oids, formPgIndex->indexrelid); } /* * GetRelationIndicesDependencyList creates a list of ObjectAddressDependencies for the * indexes on a given relation. */ static List * GetRelationIndicesDependencyList(Oid relationId) { List *indexIds = ExecuteFunctionOnEachTableIndex(relationId, CollectIndexOids, 0); return CreateObjectAddressDependencyDefList(RelationRelationId, indexIds); } /* * GetRelationTriggerFunctionDependencyList returns a list of DependencyDefinition * objects for the functions that triggers of the relation with relationId depends. */ static List * GetRelationTriggerFunctionDependencyList(Oid relationId) { List *dependencyList = NIL; List *triggerIdList = GetExplicitTriggerIdList(relationId); Oid triggerId = InvalidOid; foreach_declared_oid(triggerId, triggerIdList) { Oid functionId = GetTriggerFunctionId(triggerId); DependencyDefinition *dependency = CreateObjectAddressDependencyDef(ProcedureRelationId, functionId); dependencyList = lappend(dependencyList, dependency); } return dependencyList; } /* * GetPublicationRelationsDependencyList creates a list of ObjectAddressDependencies for * a publication on the Citus relations it contains. This helps make sure we distribute * Citus tables before local tables. */ static List * GetPublicationRelationsDependencyList(Oid publicationId) { List *allRelationIds = GetPublicationRelations(publicationId, PUBLICATION_PART_ROOT); List *citusRelationIds = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, allRelationIds) { if (!IsCitusTable(relationId)) { continue; } citusRelationIds = lappend_oid(citusRelationIds, relationId); } return CreateObjectAddressDependencyDefList(RelationRelationId, citusRelationIds); } /* * GetTypeConstraintDependencyDefinition creates a list of constraint dependency * definitions for a given type */ static List * GetTypeConstraintDependencyDefinition(Oid typeId) { /* lookup and look all constraints to add them to the CreateDomainStmt */ Relation conRel = table_open(ConstraintRelationId, AccessShareLock); /* Look for CHECK Constraints on this domain */ ScanKeyData key[1]; ScanKeyInit(&key[0], Anum_pg_constraint_contypid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(typeId)); SysScanDesc scan = systable_beginscan(conRel, ConstraintTypidIndexId, true, NULL, 1, key); List *dependencies = NIL; HeapTuple conTup = NULL; while (HeapTupleIsValid(conTup = systable_getnext(scan))) { Form_pg_constraint c = (Form_pg_constraint) GETSTRUCT(conTup); if (c->contype != CONSTRAINT_CHECK) { /* Ignore non-CHECK constraints, shouldn't be any */ continue; } dependencies = lappend(dependencies, CreateObjectAddressDependencyDef( ConstraintRelationId, c->oid)); } systable_endscan(scan); table_close(conRel, NoLock); return dependencies; } /* * CreateObjectAddressDependencyDef returns DependencyDefinition object that * stores the ObjectAddress for the database object identified by classId and * objectId. */ static DependencyDefinition * CreateObjectAddressDependencyDef(Oid classId, Oid objectId) { DependencyDefinition *dependency = palloc0(sizeof(DependencyDefinition)); dependency->mode = DependencyObjectAddress; ObjectAddressSet(dependency->data.address, classId, objectId); return dependency; } /* * CreateObjectAddressDependencyDefList is a wrapper function for * CreateObjectAddressDependencyDef to operate on a list of relation oids, * instead of a single oid. */ static List * CreateObjectAddressDependencyDefList(Oid classId, List *objectIdList) { List *dependencyList = NIL; Oid objectId = InvalidOid; foreach_declared_oid(objectId, objectIdList) { DependencyDefinition *dependency = CreateObjectAddressDependencyDef(classId, objectId); dependencyList = lappend(dependencyList, dependency); } return dependencyList; } /* * DependencyDefinitionObjectAddress returns the object address of the dependency defined * by the dependency definition, irregardless what the source of the definition is */ static ObjectAddress DependencyDefinitionObjectAddress(DependencyDefinition *definition) { switch (definition->mode) { case DependencyObjectAddress: { return definition->data.address; } case DependencyPgDepend: { ObjectAddress address = { 0 }; ObjectAddressSet(address, definition->data.pg_depend.refclassid, definition->data.pg_depend.refobjid); return address; } case DependencyPgShDepend: { ObjectAddress address = { 0 }; ObjectAddressSet(address, definition->data.pg_shdepend.refclassid, definition->data.pg_shdepend.refobjid); return address; } } ereport(ERROR, (errmsg("unsupported dependency definition mode"))); } /* * BuildViewDependencyGraph gets a relation (or a view) and builds a dependency graph for the * depending views. */ static ViewDependencyNode * BuildViewDependencyGraph(Oid relationId, HTAB *nodeMap) { bool found = false; ViewDependencyNode *node = (ViewDependencyNode *) hash_search(nodeMap, &relationId, HASH_ENTER, &found); if (found) { return node; } node->id = relationId; node->remainingDependencyCount = 0; node->dependingNodes = NIL; Oid targetObjectClassId = RelationRelationId; Oid targetObjectId = relationId; List *dependencyTupleList = GetPgDependTuplesForDependingObjects(targetObjectClassId, targetObjectId); HeapTuple depTup = NULL; foreach_declared_ptr(depTup, dependencyTupleList) { Form_pg_depend pg_depend = (Form_pg_depend) GETSTRUCT(depTup); Oid dependingView = GetDependingView(pg_depend); if (dependingView != InvalidOid) { ViewDependencyNode *dependingNode = BuildViewDependencyGraph(dependingView, nodeMap); node->dependingNodes = lappend(node->dependingNodes, dependingNode); dependingNode->remainingDependencyCount++; } } return node; } /* * GetPgDependTuplesForDependingObjects scans pg_depend for given object and * returns a list of heap tuples for the objects depending on it. */ List * GetPgDependTuplesForDependingObjects(Oid targetObjectClassId, Oid targetObjectId) { List *dependencyTupleList = NIL; Relation pgDepend = table_open(DependRelationId, AccessShareLock); ScanKeyData key[2]; int scanKeyCount = 2; ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(targetObjectClassId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(targetObjectId)); bool useIndex = true; SysScanDesc depScan = systable_beginscan(pgDepend, DependReferenceIndexId, useIndex, NULL, scanKeyCount, key); HeapTuple dependencyTuple = NULL; while (HeapTupleIsValid(dependencyTuple = systable_getnext(depScan))) { /* copy the tuple first */ dependencyTuple = heap_copytuple(dependencyTuple); dependencyTupleList = lappend(dependencyTupleList, dependencyTuple); } systable_endscan(depScan); relation_close(pgDepend, NoLock); return dependencyTupleList; } /* * GetDependingViews takes a relation id, finds the views that depend on the relation * and returns list of the oids of those views. It recurses on the pg_depend table to * find the views that recursively depend on the table. * * The returned views will have the correct order for creating them, from the point of * dependencies between. */ List * GetDependingViews(Oid relationId) { HTAB *nodeMap = CreateSimpleHashWithName(Oid, ViewDependencyNode, "view dependency map (oid)"); ViewDependencyNode *tableNode = BuildViewDependencyGraph(relationId, nodeMap); List *dependingViews = NIL; List *nodeQueue = list_make1(tableNode); ViewDependencyNode *node = NULL; foreach_ptr_append(node, nodeQueue) { ViewDependencyNode *dependingNode = NULL; foreach_declared_ptr(dependingNode, node->dependingNodes) { ObjectAddress relationAddress = { 0 }; ObjectAddressSet(relationAddress, RelationRelationId, dependingNode->id); /* * This function does not catch views with circular dependencies, * because of the remaining dependency count check below. * Here we check if the view has a circular dependency or not. * If yes, we error out with a message that tells the user that * Citus does not handle circular dependencies. */ DeferredErrorMessage *depError = DeferErrorIfCircularDependencyExists(&relationAddress); if (depError != NULL) { RaiseDeferredError(depError, ERROR); } dependingNode->remainingDependencyCount--; if (dependingNode->remainingDependencyCount == 0) { nodeQueue = lappend(nodeQueue, dependingNode); dependingViews = lappend_oid(dependingViews, dependingNode->id); } } } return dependingViews; } /* * GetDependingView gets a row of pg_depend and returns the oid of the view that is depended. * If the depended object is not a rewrite object, the object to rewrite is not a view or it * is the same view with the depending one InvalidOid is returned. */ Oid GetDependingView(Form_pg_depend pg_depend) { if (pg_depend->classid != RewriteRelationId) { return InvalidOid; } Relation rewriteRel = table_open(RewriteRelationId, AccessShareLock); ScanKeyData rkey[1]; ScanKeyInit(&rkey[0], Anum_pg_rewrite_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(pg_depend->objid)); SysScanDesc rscan = systable_beginscan(rewriteRel, RewriteOidIndexId, true, NULL, 1, rkey); HeapTuple rewriteTup = systable_getnext(rscan); if (!HeapTupleIsValid(rewriteTup)) { /* * This function already verified that objid's classid is * RewriteRelationId, so it should exists. But be on the * safe side. */ ereport(ERROR, (errmsg("catalog lookup failed for view %u", pg_depend->objid))); } Form_pg_rewrite pg_rewrite = (Form_pg_rewrite) GETSTRUCT(rewriteTup); bool isView = get_rel_relkind(pg_rewrite->ev_class) == RELKIND_VIEW; bool isMatView = get_rel_relkind(pg_rewrite->ev_class) == RELKIND_MATVIEW; bool isDifferentThanRef = pg_rewrite->ev_class != pg_depend->refobjid; Oid dependingView = InvalidOid; if ((isView || isMatView) && isDifferentThanRef) { dependingView = pg_rewrite->ev_class; } systable_endscan(rscan); relation_close(rewriteRel, AccessShareLock); return dependingView; } ================================================ FILE: src/backend/distributed/metadata/distobject.c ================================================ /*------------------------------------------------------------------------- * * distobject.c * Functions to interact with distributed objects by their ObjectAddress * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/objectaddress.h" #include "catalog/pg_database.h" #include "catalog/pg_extension_d.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "commands/extension.h" #include "executor/spi.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" #include "parser/parse_type.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/rel.h" #include "citus_version.h" #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/remote_commands.h" #include "distributed/version_compat.h" #include "distributed/worker_transaction.h" static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress, char *objectName); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, Datum *paramValues); static bool IsObjectDistributed(const ObjectAddress *address); PG_FUNCTION_INFO_V1(mark_object_distributed); PG_FUNCTION_INFO_V1(citus_unmark_object_distributed); PG_FUNCTION_INFO_V1(master_unmark_object_distributed); /* * mark_object_distributed adds an object to pg_dist_object * in all of the nodes, for the connections to the other nodes this function * uses the user passed. */ Datum mark_object_distributed(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); Oid classId = PG_GETARG_OID(0); text *objectNameText = PG_GETARG_TEXT_P(1); char *objectName = text_to_cstring(objectNameText); Oid objectId = PG_GETARG_OID(2); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*objectAddress, classId, objectId); text *connectionUserText = PG_GETARG_TEXT_P(3); char *connectionUser = text_to_cstring(connectionUserText); /* * This function is called when a query is run from a Citus non-main database. * We need to insert into local pg_dist_object over a connection to make sure * 2PC still works. */ bool useConnectionForLocalQuery = true; MarkObjectDistributedWithName(objectAddress, objectName, useConnectionForLocalQuery, connectionUser); PG_RETURN_VOID(); } /* * citus_unmark_object_distributed(classid oid, objid oid, objsubid int,checkobjectexistence bool) * * Removes the entry for an object address from pg_dist_object. If checkobjectexistence is true, * throws an error if the object still exists. */ Datum citus_unmark_object_distributed(PG_FUNCTION_ARGS) { Oid classid = PG_GETARG_OID(0); Oid objid = PG_GETARG_OID(1); int32 objsubid = PG_GETARG_INT32(2); /* * SQL function master_unmark_object_distributed doesn't expect the * 4th argument but SQL function citus_unmark_object_distributed does * so as checkobjectexistence argument. For this reason, we try to * get the 4th argument only if this C function is called with 4 * arguments. */ bool checkObjectExistence = true; if (PG_NARGS() == 4) { checkObjectExistence = PG_GETARG_BOOL(3); } ObjectAddress address = { 0 }; ObjectAddressSubSet(address, classid, objid, objsubid); if (!IsObjectDistributed(&address)) { /* if the object is not distributed there is no need to unmark it */ PG_RETURN_VOID(); } if (checkObjectExistence && ObjectExists(&address)) { ereport(ERROR, (errmsg("object still exists"), errdetail("the %s \"%s\" still exists", getObjectTypeDescription(&address, /* missingOk: */ false), getObjectIdentity(&address, /* missingOk: */ false)), errhint("drop the object via a DROP command"))); } UnmarkObjectDistributed(&address); PG_RETURN_VOID(); } /* * master_unmark_object_distributed is a wrapper function for old UDF name. */ Datum master_unmark_object_distributed(PG_FUNCTION_ARGS) { return citus_unmark_object_distributed(fcinfo); } /* * ObjectExists checks if an object given by its object address exists * * This is done by opening the catalog for the object and search the catalog for the * objects' oid. If we can find a tuple the object is existing. If no tuple is found, or * we don't have the information to find the tuple by its oid we assume the object does * not exist. */ bool ObjectExists(const ObjectAddress *address) { if (address == NULL) { return false; } if (is_objectclass_supported(address->classId)) { Relation catalog = table_open(address->classId, AccessShareLock); HeapTuple objtup = get_catalog_object_by_oid(catalog, get_object_attnum_oid( address->classId), address->objectId); table_close(catalog, AccessShareLock); if (objtup != NULL) { return true; } return false; } return false; } /* * MarkObjectDistributed marks an object as a distributed object. Marking is done * by adding appropriate entries to citus.pg_dist_object and also marking the object * as distributed by opening a connection using current user to all remote nodes * with metadata if object propagation is on. * * This function should be used if the user creating the given object. If you want * to mark dependent objects as distributed check MarkObjectDistributedViaSuperUser. */ void MarkObjectDistributed(const ObjectAddress *distAddress) { bool useConnectionForLocalQuery = false; MarkObjectDistributedWithName(distAddress, "", useConnectionForLocalQuery, CurrentUserName()); } /* * MarkObjectDistributedWithName marks an object as a distributed object. * Same as MarkObjectDistributed but this function also allows passing an objectName * that is used in case the object does not exists for the current transaction. */ void MarkObjectDistributedWithName(const ObjectAddress *distAddress, char *objectName, bool useConnectionForLocalQuery, char *connectionUser) { if (!CitusHasBeenLoaded()) { elog(ERROR, "Cannot mark object distributed because Citus has not been loaded."); } /* * When a query is run from a Citus non-main database we need to insert into pg_dist_object * over a connection to make sure 2PC still works. */ if (useConnectionForLocalQuery) { StringInfo insertQuery = makeStringInfo(); appendStringInfo(insertQuery, "INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid)" "VALUES (%d, %d, %d) ON CONFLICT DO NOTHING", distAddress->classId, distAddress->objectId, distAddress->objectSubId); SendCommandToWorker(LocalHostName, PostPortNumber, insertQuery->data); } else { MarkObjectDistributedLocally(distAddress); } if (EnableMetadataSync) { char *workerPgDistObjectUpdateCommand = CreatePgDistObjectEntryCommand(distAddress, objectName); SendCommandToRemoteMetadataNodesParams(workerPgDistObjectUpdateCommand, connectionUser, 0, NULL, NULL); } } /* * MarkObjectDistributedViaSuperUser marks an object as a distributed object. Marking * is done by adding appropriate entries to citus.pg_dist_object and also marking the * object as distributed by opening a connection using super user to all remote nodes * with metadata if object propagation is on. * * This function should be used to mark dependent object as distributed. If you want * to mark the object you are creating please check MarkObjectDistributed. */ void MarkObjectDistributedViaSuperUser(const ObjectAddress *distAddress) { MarkObjectDistributedLocally(distAddress); if (EnableMetadataSync) { char *workerPgDistObjectUpdateCommand = CreatePgDistObjectEntryCommand(distAddress, ""); SendCommandToRemoteNodesWithMetadataViaSuperUser(workerPgDistObjectUpdateCommand); } } /* * MarkObjectDistributedLocally marks an object as a distributed object by citus. * Marking is done by adding appropriate entries to citus.pg_dist_object. * * This function should never be called alone, MarkObjectDistributed() or * MarkObjectDistributedViaSuperUser() should be called. */ void MarkObjectDistributedLocally(const ObjectAddress *distAddress) { int paramCount = 3; Oid paramTypes[3] = { OIDOID, OIDOID, INT4OID }; Datum paramValues[3] = { ObjectIdGetDatum(distAddress->classId), ObjectIdGetDatum(distAddress->objectId), Int32GetDatum(distAddress->objectSubId) }; char *insertQuery = "INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid) " "VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"; int spiStatus = ExecuteCommandAsSuperuser(insertQuery, paramCount, paramTypes, paramValues); if (spiStatus < 0) { ereport(ERROR, (errmsg("failed to insert object into citus.pg_dist_object"))); } } /* * ShouldMarkRelationDistributed is a helper function that * decides whether the input relation should be marked as distributed. */ bool ShouldMarkRelationDistributed(Oid relationId) { if (!EnableMetadataSync) { /* * Just in case anything goes wrong, we should still be able * to continue to the version upgrade. */ return false; } ObjectAddress *relationAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*relationAddress, RelationRelationId, relationId); bool pgObject = (relationId < FirstNormalObjectId); bool isObjectSupported = SupportedDependencyByCitus(relationAddress); bool ownedByExtension = IsTableOwnedByExtension(relationId); bool alreadyDistributed = IsObjectDistributed(relationAddress); bool hasUnsupportedDependency = DeferErrorIfAnyObjectHasUnsupportedDependency( list_make1(relationAddress)) != NULL; bool hasCircularDependency = DeferErrorIfCircularDependencyExists(relationAddress) != NULL; /* * pgObject: Citus never marks pg objects as distributed * isObjectSupported: Citus does not support propagation of some objects * ownedByExtension: let extensions manage its own objects * alreadyDistributed: most likely via earlier versions * hasUnsupportedDependency: Citus doesn't know how to distribute its dependencies * hasCircularDependency: Citus cannot handle circular dependencies */ if (pgObject || !isObjectSupported || ownedByExtension || alreadyDistributed || hasUnsupportedDependency || hasCircularDependency) { return false; } return true; } /* * CreatePgDistObjectEntryCommand creates command to insert pg_dist_object tuple * for the given object address. */ static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress, char *objectName) { /* create a list by adding the address of value to not to have warning */ List *objectAddressList = list_make1((ObjectAddress *) objectAddress); /* names also require a list so we create a nested list here */ List *objectNameList = list_make1(list_make1((char *) objectName)); List *distArgumetIndexList = list_make1_int(INVALID_DISTRIBUTION_ARGUMENT_INDEX); List *colocationIdList = list_make1_int(INVALID_COLOCATION_ID); List *forceDelegationList = list_make1_int(NO_FORCE_PUSHDOWN); char *workerPgDistObjectUpdateCommand = MarkObjectsDistributedCreateCommand(objectAddressList, objectNameList, distArgumetIndexList, colocationIdList, forceDelegationList); return workerPgDistObjectUpdateCommand; } /* * CitusExtensionObject returns true if the objectAddress represents * the Citus extension. */ bool CitusExtensionObject(const ObjectAddress *objectAddress) { if (objectAddress->classId != ExtensionRelationId) { return false; } char *extensionName = get_extension_name(objectAddress->objectId); if (extensionName != NULL && strncasecmp(extensionName, "citus", NAMEDATALEN) == 0) { return true; } return false; } /* * ExecuteCommandAsSuperuser executes a command via SPI as superuser. Using this * function (and in general SPI/SQL with superuser) should be avoided as much as * possible. This is to prevent any user to exploit the superuser access via * triggers. */ static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, Datum *paramValues) { Oid savedUserId = InvalidOid; int savedSecurityContext = 0; int spiConnected = SPI_connect(); if (spiConnected != SPI_OK_CONNECT) { ereport(ERROR, (errmsg("could not connect to SPI manager"))); } /* make sure we have write access */ GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); int spiStatus = SPI_execute_with_args(query, paramCount, paramTypes, paramValues, NULL, false, 0); SetUserIdAndSecContext(savedUserId, savedSecurityContext); int spiFinished = SPI_finish(); if (spiFinished != SPI_OK_FINISH) { ereport(ERROR, (errmsg("could not disconnect from SPI manager"))); } return spiStatus; } /* * UnmarkNodeWideObjectsDistributed deletes pg_dist_object records * for all distributed objects in given Drop stmt node. * * Today we only expect DropRoleStmt and DropdbStmt to get here. */ void UnmarkNodeWideObjectsDistributed(Node *node) { if (IsA(node, DropRoleStmt)) { DropRoleStmt *stmt = castNode(DropRoleStmt, node); List *allDropRoles = stmt->roles; List *distributedDropRoles = FilterDistributedRoles(allDropRoles); if (list_length(distributedDropRoles) > 0) { UnmarkRolesDistributed(distributedDropRoles); } } else if (IsA(node, DropdbStmt)) { DropdbStmt *stmt = castNode(DropdbStmt, node); char *dbName = stmt->dbname; Oid dbOid = get_database_oid(dbName, stmt->missing_ok); ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, dbOid); if (IsAnyObjectDistributed(list_make1(dbObjectAddress))) { UnmarkObjectDistributed(dbObjectAddress); } } } /* * UnmarkObjectDistributed removes the entry from pg_dist_object that marks this object as * distributed. This will prevent updates to that object to be propagated to the worker. */ void UnmarkObjectDistributed(const ObjectAddress *address) { int paramCount = 3; Oid paramTypes[3] = { OIDOID, OIDOID, INT4OID }; Datum paramValues[3] = { ObjectIdGetDatum(address->classId), ObjectIdGetDatum(address->objectId), Int32GetDatum(address->objectSubId) }; char *deleteQuery = "DELETE FROM pg_catalog.pg_dist_object WHERE classid = $1 AND " "objid = $2 AND objsubid = $3"; int spiStatus = ExecuteCommandAsSuperuser(deleteQuery, paramCount, paramTypes, paramValues); if (spiStatus < 0) { ereport(ERROR, (errmsg("failed to delete object from citus.pg_dist_object"))); } } /* * IsObjectDistributed returns if the object addressed is already distributed in the * cluster. This performs a local indexed lookup in pg_dist_object. */ static bool IsObjectDistributed(const ObjectAddress *address) { ScanKeyData key[3]; bool result = false; Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); /* scan pg_dist_object for classid = $1 AND objid = $2 AND objsubid = $3 via index */ ScanKeyInit(&key[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(address->classId)); ScanKeyInit(&key[1], Anum_pg_dist_object_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(address->objectId)); ScanKeyInit(&key[2], Anum_pg_dist_object_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(address->objectSubId)); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, DistObjectPrimaryKeyIndexId(), true, NULL, 3, key); HeapTuple pgDistObjectTup = systable_getnext(pgDistObjectScan); if (HeapTupleIsValid(pgDistObjectTup)) { result = true; } systable_endscan(pgDistObjectScan); relation_close(pgDistObjectRel, AccessShareLock); return result; } /* * IsAnyObjectDistributed iteratively calls IsObjectDistributed for given addresses to * determine if any object is distributed. */ bool IsAnyObjectDistributed(const List *addresses) { ObjectAddress *address = NULL; foreach_declared_ptr(address, addresses) { if (IsObjectDistributed(address)) { return true; } } return false; } /* * IsAnyParentObjectDistributed - true if at least one of the * given addresses is distributed. If an address has a non-zero * objectSubId, it checks the parent object (the object with * the same classId and objid, but with objectSubId = 0). For * example, a column address will check the table address. * If the address has a zero objectSubId, it checks the address * itself. */ bool IsAnyParentObjectDistributed(const List *addresses) { bool isDistributed = false; ListCell *lc = NULL; foreach(lc, addresses) { ObjectAddress *address = (ObjectAddress *) lfirst(lc); int32 savedObjectSubId = address->objectSubId; address->objectSubId = 0; isDistributed = IsObjectDistributed(address); address->objectSubId = savedObjectSubId; if (isDistributed) { break; } } return isDistributed; } /* * GetDistributedObjectAddressList returns a list of ObjectAddresses that contains all * distributed objects as marked in pg_dist_object */ List * GetDistributedObjectAddressList(void) { HeapTuple pgDistObjectTup = NULL; List *objectAddressList = NIL; Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, InvalidOid, false, NULL, 0, NULL); while (HeapTupleIsValid(pgDistObjectTup = systable_getnext(pgDistObjectScan))) { Form_pg_dist_object pg_dist_object = (Form_pg_dist_object) GETSTRUCT(pgDistObjectTup); ObjectAddress *objectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSubSet(*objectAddress, pg_dist_object->classid, pg_dist_object->objid, pg_dist_object->objsubid); objectAddressList = lappend(objectAddressList, objectAddress); } systable_endscan(pgDistObjectScan); relation_close(pgDistObjectRel, AccessShareLock); return objectAddressList; } /* * GetRoleSpecObjectForUser creates a RoleSpec object for the given roleOid. */ RoleSpec * GetRoleSpecObjectForUser(Oid roleOid) { RoleSpec *roleSpec = makeNode(RoleSpec); roleSpec->roletype = OidIsValid(roleOid) ? ROLESPEC_CSTRING : ROLESPEC_PUBLIC; roleSpec->rolename = OidIsValid(roleOid) ? GetUserNameFromId(roleOid, false) : NULL; roleSpec->location = -1; return roleSpec; } /* * UpdateDistributedObjectColocationId gets an old and a new colocationId * and updates the colocationId of all tuples in citus.pg_dist_object which * have the old colocationId to the new colocationId. */ void UpdateDistributedObjectColocationId(uint32 oldColocationId, uint32 newColocationId) { const bool indexOK = false; ScanKeyData scanKey[1]; Relation pgDistObjectRel = table_open(DistObjectRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistObjectRel); /* scan pg_dist_object for colocationId equal to old colocationId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_object_colocationid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(oldColocationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistObjectRel, InvalidOid, indexOK, NULL, 1, scanKey); HeapTuple heapTuple; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc0(tupleDescriptor->natts * sizeof(bool)); replace[Anum_pg_dist_object_colocationid - 1] = true; /* update the colocationId to the new one */ values[Anum_pg_dist_object_colocationid - 1] = UInt32GetDatum(newColocationId); isnull[Anum_pg_dist_object_colocationid - 1] = false; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistObjectRel, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistObjectRelationId()); pfree(values); pfree(isnull); pfree(replace); } systable_endscan(scanDescriptor); table_close(pgDistObjectRel, NoLock); CommandCounterIncrement(); } /* * DistributedFunctionList returns the list of ObjectAddress-es of all the * distributed functions found in pg_dist_object */ List * DistributedFunctionList(void) { List *distributedFunctionList = NIL; ScanKeyData key[1]; Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); /* scan pg_dist_object for classid = ProcedureRelationId via index */ ScanKeyInit(&key[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ProcedureRelationId)); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, DistObjectPrimaryKeyIndexId(), true, NULL, 1, key); HeapTuple pgDistObjectTup = NULL; while (HeapTupleIsValid(pgDistObjectTup = systable_getnext(pgDistObjectScan))) { Form_pg_dist_object pg_dist_object = (Form_pg_dist_object) GETSTRUCT(pgDistObjectTup); ObjectAddress *functionAddress = palloc0(sizeof(ObjectAddress)); functionAddress->classId = ProcedureRelationId; functionAddress->objectId = pg_dist_object->objid; functionAddress->objectSubId = pg_dist_object->objsubid; distributedFunctionList = lappend(distributedFunctionList, functionAddress); } systable_endscan(pgDistObjectScan); relation_close(pgDistObjectRel, AccessShareLock); return distributedFunctionList; } /* * DistributedSequenceList returns the list of ObjectAddress-es of all the * distributed sequences found in pg_dist_object */ List * DistributedSequenceList(void) { List *distributedSequenceList = NIL; ScanKeyData key[1]; Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); /* scan pg_dist_object for classid = RelationRelationId via index */ ScanKeyInit(&key[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, DistObjectPrimaryKeyIndexId(), true, NULL, 1, key); HeapTuple pgDistObjectTup = NULL; while (HeapTupleIsValid(pgDistObjectTup = systable_getnext(pgDistObjectScan))) { Form_pg_dist_object pg_dist_object = (Form_pg_dist_object) GETSTRUCT(pgDistObjectTup); if (get_rel_relkind(pg_dist_object->objid) == RELKIND_SEQUENCE) { ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); sequenceAddress->classId = RelationRelationId; sequenceAddress->objectId = pg_dist_object->objid; sequenceAddress->objectSubId = pg_dist_object->objsubid; distributedSequenceList = lappend(distributedSequenceList, sequenceAddress); } } systable_endscan(pgDistObjectScan); relation_close(pgDistObjectRel, AccessShareLock); return distributedSequenceList; } /* * GetForceDelegationAttrIndexInPgDistObject returns attrnum for force_delegation attr. * * force_delegation attr was added to table pg_dist_object using alter operation after * the version where Citus started supporting downgrades, and it's only column that we've * introduced to pg_dist_object since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_pg_dist_object and when this happens, then we know that attrnum force_delegation is * not Anum_pg_dist_object_force_delegation anymore but tupleDesc->natts - 1. */ int GetForceDelegationAttrIndexInPgDistObject(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_object ? (Anum_pg_dist_object_force_delegation - 1) : tupleDesc->natts - 1; } ================================================ FILE: src/backend/distributed/metadata/metadata_cache.c ================================================ /*------------------------------------------------------------------------- * * metadata_cache.c * Distributed table metadata cache * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "stdint.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_enum.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "commands/extension.h" #include "commands/trigger.h" #include "common/hashfn.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "nodes/memnodes.h" #include "nodes/pg_list.h" #include "parser/parse_func.h" #include "parser/parse_type.h" #include "storage/lmgr.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/catcache.h" #include "utils/datum.h" #include "utils/elog.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/inval.h" #include "utils/jsonb.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/palloc.h" #include "utils/rel.h" #include "utils/relmapper.h" #include "utils/resowner.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "citus_version.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/citus_depended_object.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/foreign_key_relationship.h" #include "distributed/function_utils.h" #include "distributed/listutils.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_local_group.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_node_metadata.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_placement.h" #include "distributed/pg_dist_shard.h" #include "distributed/remote_commands.h" #include "distributed/shardinterval_utils.h" #include "distributed/shared_library_init.h" #include "distributed/utils/array_type.h" #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" /* user configuration */ int ReadFromSecondaries = USE_SECONDARY_NODES_NEVER; /* * CitusTableCacheEntrySlot is entry type for DistTableCacheHash, * entry data outlives slot on invalidation, so requires indirection. */ typedef struct CitusTableCacheEntrySlot { /* lookup key - must be first. A pg_class.oid oid. */ Oid relationId; /* Citus table metadata (NULL for local tables) */ CitusTableCacheEntry *citusTableMetadata; /* * If isValid is false, we need to recheck whether the relation ID * belongs to a Citus or not. */ bool isValid; } CitusTableCacheEntrySlot; /* * ShardIdCacheEntry is the entry type for ShardIdCacheHash. * * This should never be used outside of this file. Use ShardInterval instead. */ typedef struct ShardIdCacheEntry { /* hash key, needs to be first */ uint64 shardId; /* pointer to the table entry to which this shard currently belongs */ CitusTableCacheEntry *tableEntry; /* index of the shard interval in the sortedShardIntervalArray of the table entry */ int shardIndex; } ShardIdCacheEntry; /* * ExtensionCreatedState is used to track if citus extension has been created * using CREATE EXTENSION command. * UNKNOWN : MetadataCache is invalid. State is UNKNOWN. * CREATED : Citus is created. * NOTCREATED : Citus is not created. */ typedef enum ExtensionCreatedState { UNKNOWN = 0, CREATED = 1, NOTCREATED = 2, } ExtensionCreatedState; /* * State which should be cleared upon DROP EXTENSION. When the configuration * changes, e.g. because extension is dropped, these summarily get set to 0. */ typedef struct MetadataCacheData { ExtensionCreatedState extensionCreatedState; Oid distShardRelationId; Oid distPlacementRelationId; Oid distBackgroundJobRelationId; Oid distBackgroundJobPKeyIndexId; Oid distBackgroundJobJobIdSequenceId; Oid distBackgroundTaskRelationId; Oid distBackgroundTaskPKeyIndexId; Oid distBackgroundTaskJobIdTaskIdIndexId; Oid distBackgroundTaskStatusTaskIdIndexId; Oid distBackgroundTaskTaskIdSequenceId; Oid distBackgroundTaskDependRelationId; Oid distBackgroundTaskDependTaskIdIndexId; Oid distBackgroundTaskDependDependsOnIndexId; Oid citusJobStatusScheduledId; Oid citusJobStatusRunningId; Oid citusJobStatusCancellingId; Oid citusJobStatusFinishedId; Oid citusJobStatusCancelledId; Oid citusJobStatusFailedId; Oid citusJobStatusFailingId; Oid citusTaskStatusBlockedId; Oid citusTaskStatusRunnableId; Oid citusTaskStatusRunningId; Oid citusTaskStatusDoneId; Oid citusTaskStatusErrorId; Oid citusTaskStatusUnscheduledId; Oid citusTaskStatusCancelledId; Oid citusTaskStatusCancellingId; Oid distRebalanceStrategyRelationId; Oid distNodeRelationId; Oid distNodeNodeIdIndexId; Oid distLocalGroupRelationId; Oid distObjectRelationId; Oid distObjectPrimaryKeyIndexId; Oid distCleanupRelationId; Oid distCleanupPrimaryKeyIndexId; Oid distColocationRelationId; Oid distColocationConfigurationIndexId; Oid distPartitionRelationId; Oid distTenantSchemaRelationId; Oid distPartitionLogicalRelidIndexId; Oid distPartitionColocationidIndexId; Oid distShardLogicalRelidIndexId; Oid distShardShardidIndexId; Oid distPlacementShardidIndexId; Oid distPlacementPlacementidIndexId; Oid distColocationidIndexId; Oid distPlacementGroupidIndexId; Oid distTransactionRelationId; Oid distTransactionGroupIndexId; Oid distTenantSchemaPrimaryKeyIndexId; Oid distTenantSchemaUniqueColocationIdIndexId; Oid citusCatalogNamespaceId; Oid copyFormatTypeId; Oid readIntermediateResultFuncId; Oid readIntermediateResultArrayFuncId; Oid extraDataContainerFuncId; Oid workerHashFunctionId; Oid anyValueFunctionId; Oid textSendAsJsonbFunctionId; Oid textoutFunctionId; Oid extensionOwner; Oid binaryCopyFormatId; Oid textCopyFormatId; Oid primaryNodeRoleId; Oid secondaryNodeRoleId; Oid unavailableNodeRoleId; Oid pgTableIsVisibleFuncId; Oid citusTableIsVisibleFuncId; Oid distAuthinfoRelationId; Oid distAuthinfoIndexId; Oid distPoolinfoRelationId; Oid distPoolinfoIndexId; Oid relationIsAKnownShardFuncId; Oid jsonbExtractPathFuncId; Oid jsonbExtractPathTextFuncId; Oid CitusDependentObjectFuncId; Oid distClockLogicalSequenceId; bool databaseNameValid; char databaseName[NAMEDATALEN]; } MetadataCacheData; static MetadataCacheData MetadataCache; /* Citus extension version variables */ bool EnableVersionChecks = true; /* version checks are enabled */ static bool citusVersionKnownCompatible = false; /* Variable to determine if we are in the process of creating citus */ static int CreateCitusTransactionLevel = 0; /* Hash table for informations about each partition */ static HTAB *DistTableCacheHash = NULL; static List *DistTableCacheExpired = NIL; /* Hash table for informations about each shard */ static HTAB *ShardIdCacheHash = NULL; static MemoryContext MetadataCacheMemoryContext = NULL; /* Hash table for information about each object */ static HTAB *DistObjectCacheHash = NULL; /* Hash table for informations about worker nodes */ static HTAB *WorkerNodeHash = NULL; static WorkerNode **WorkerNodeArray = NULL; static int WorkerNodeCount = 0; static bool workerNodeHashValid = false; /* default value is -1, for coordinator it's 0 and for worker nodes > 0 */ static int32 LocalGroupId = -1; /* default value is -1, increases with every node starting from 1 */ static int32 LocalNodeId = -1; /* built first time through in InitializeDistCache */ static ScanKeyData DistPartitionScanKey[1]; static ScanKeyData DistShardScanKey[1]; static ScanKeyData DistObjectScanKey[3]; /* local function forward declarations */ static HeapTuple PgDistPartitionTupleViaCatalog(Oid relationId); static ShardIdCacheEntry * LookupShardIdCacheEntry(int64 shardId, bool missingOk); static CitusTableCacheEntry * BuildCitusTableCacheEntry(Oid relationId); static void BuildCachedShardList(CitusTableCacheEntry *cacheEntry); static void PrepareWorkerNodeCache(void); static bool CheckInstalledVersion(int elevel); static char * AvailableExtensionVersion(void); static char * InstalledExtensionVersion(void); static bool CitusHasBeenLoadedInternal(void); static void InitializeCaches(void); static void InitializeDistCache(void); static void InitializeDistObjectCache(void); static void InitializeWorkerNodeCache(void); static void RegisterForeignKeyGraphCacheCallbacks(void); static void RegisterWorkerNodeCacheCallbacks(void); static void RegisterLocalGroupIdCacheCallbacks(void); static void RegisterAuthinfoCacheCallbacks(void); static void RegisterCitusTableCacheEntryReleaseCallbacks(void); static void ResetCitusTableCacheEntry(CitusTableCacheEntry *cacheEntry); static void RemoveStaleShardIdCacheEntries(CitusTableCacheEntry *tableEntry); static void CreateDistTableCache(void); static void CreateShardIdCache(void); static void CreateDistObjectCache(void); static void InvalidateForeignRelationGraphCacheCallback(Datum argument, Oid relationId); static void InvalidateNodeRelationCacheCallback(Datum argument, Oid relationId); static void InvalidateLocalGroupIdRelationCacheCallback(Datum argument, Oid relationId); static void InvalidateConnParamsCacheCallback(Datum argument, Oid relationId); static void CitusTableCacheEntryReleaseCallback(ResourceReleasePhase phase, bool isCommit, bool isTopLevel, void *arg); static HeapTuple LookupDistPartitionTuple(Relation pgDistPartition, Oid relationId); static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, Oid *columnTypeId, int32 *columnTypeMod, Oid *intervalTypeId, int32 *intervalTypeMod); static void CachedNamespaceLookup(const char *nspname, Oid *cachedOid); static void CachedRelationLookup(const char *relationName, Oid *cachedOid); static void CachedRelationLookupExtended(const char *relationName, Oid *cachedOid, bool missing_ok); static void CachedRelationNamespaceLookup(const char *relationName, Oid relnamespace, Oid *cachedOid); static void CachedRelationNamespaceLookupExtended(const char *relationName, Oid renamespace, Oid *cachedOid, bool missing_ok); static ShardPlacement * ResolveGroupShardPlacement(GroupShardPlacement * groupShardPlacement, CitusTableCacheEntry *tableEntry, int shardIndex); static Oid LookupEnumValueId(Oid typeId, char *valueName); static void InvalidateCitusTableCacheEntrySlot(CitusTableCacheEntrySlot *cacheSlot); static void InvalidateDistTableCache(void); static void InvalidateDistObjectCache(void); static bool InitializeTableCacheEntry(int64 shardId, bool missingOk); static bool IsCitusTableTypeInternal(char partitionMethod, char replicationModel, uint32 colocationId, CitusTableType tableType); static bool RefreshTableCacheEntryIfInvalid(ShardIdCacheEntry *shardEntry, bool missingOk); static Oid DistAuthinfoRelationId(void); static Oid DistAuthinfoIndexId(void); static Oid DistPoolinfoRelationId(void); static Oid DistPoolinfoIndexId(void); static int ParseVersionComponent(const char *version, char **endPtr); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_dist_partition_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_partition_cache_invalidate); PG_FUNCTION_INFO_V1(citus_dist_shard_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_shard_cache_invalidate); PG_FUNCTION_INFO_V1(citus_dist_placement_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_placement_cache_invalidate); PG_FUNCTION_INFO_V1(citus_dist_node_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_node_cache_invalidate); PG_FUNCTION_INFO_V1(citus_dist_local_group_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_local_group_cache_invalidate); PG_FUNCTION_INFO_V1(citus_conninfo_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_authinfo_cache_invalidate); PG_FUNCTION_INFO_V1(citus_dist_object_cache_invalidate); PG_FUNCTION_INFO_V1(master_dist_object_cache_invalidate); PG_FUNCTION_INFO_V1(role_exists); PG_FUNCTION_INFO_V1(authinfo_valid); PG_FUNCTION_INFO_V1(poolinfo_valid); /* * EnsureModificationsCanRun checks if the current node is in recovery mode or * citus.use_secondary_nodes is 'always'. If either is true the function errors out. */ void EnsureModificationsCanRun(void) { if (RecoveryInProgress() && !WritableStandbyCoordinator) { ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("writing to worker nodes is not currently allowed"), errdetail("the database is read-only"))); } if (ReadFromSecondaries == USE_SECONDARY_NODES_ALWAYS) { ereport(ERROR, (errmsg("writing to worker nodes is not currently allowed"), errdetail("citus.use_secondary_nodes is set to 'always'"))); } } /* * EnsureModificationsCanRunOnRelation first calls into EnsureModificationsCanRun() and * then does one more additional check. The additional check is to give a proper error * message if any relation that is modified is replicated, as replicated tables use * 2PC and 2PC cannot happen when recovery is in progress. */ void EnsureModificationsCanRunOnRelation(Oid relationId) { EnsureModificationsCanRun(); if (!OidIsValid(relationId) || !IsCitusTable(relationId)) { /* we are not interested in PG tables */ return; } bool modifiedTableReplicated = IsCitusTableType(relationId, REFERENCE_TABLE) || !SingleReplicatedTable(relationId); if (!IsCoordinator() && !AllowModificationsFromWorkersToReplicatedTables && modifiedTableReplicated) { ereport(ERROR, (errmsg("modifications via the worker nodes are not " "allowed for replicated tables such as reference " "tables or hash distributed tables with replication " "factor greater than 1."), errhint("All modifications to replicated tables should " "happen via the coordinator unless " "citus.allow_modifications_from_workers_to_replicated_tables " " = true."), errdetail("Allowing modifications from the worker nodes " "requires extra locking which might decrease " "the throughput."))); } /* * Even if user allows writes from standby, we should not allow for * replicated tables as they require 2PC. And, 2PC needs to write a log * record on the coordinator. */ if (!(RecoveryInProgress() && WritableStandbyCoordinator)) { return; } if (modifiedTableReplicated) { ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("writing to worker nodes is not currently " "allowed for replicated tables such as reference " "tables or hash distributed tables with replication " "factor greater than 1."), errhint("All modifications to replicated tables " "happen via 2PC, and 2PC requires the " "database to be in a writable state."), errdetail("the database is read-only"))); } } /* * IsCitusTableType returns true if the given table with relationId * belongs to a citus table that matches the given table type. If cache * entry already exists, prefer using IsCitusTableTypeCacheEntry to avoid * an extra lookup. */ bool IsCitusTableType(Oid relationId, CitusTableType tableType) { CitusTableCacheEntry *tableEntry = LookupCitusTableCacheEntry(relationId); /* we are not interested in postgres tables */ if (tableEntry == NULL) { return false; } return IsCitusTableTypeCacheEntry(tableEntry, tableType); } /* * GetCitusTableType is a helper function that returns the CitusTableType * for the given relationId. * Note that a single table can be qualified as multiple CitusTableType, such * as hash distributed tables are both HASH_DISTRIBUTED and DISTRIBUTED_TABLE. * This function returns the base type for a given table. * * If the table is not a Citus table, ANY_CITUS_TABLE_TYPE is returned. */ CitusTableType GetCitusTableType(CitusTableCacheEntry *tableEntry) { /* we do not expect local tables here */ Assert(tableEntry != NULL); if (IsCitusTableTypeCacheEntry(tableEntry, HASH_DISTRIBUTED)) { return HASH_DISTRIBUTED; } else if (IsCitusTableTypeCacheEntry(tableEntry, SINGLE_SHARD_DISTRIBUTED)) { return SINGLE_SHARD_DISTRIBUTED; } else if (IsCitusTableTypeCacheEntry(tableEntry, REFERENCE_TABLE)) { return REFERENCE_TABLE; } else if (IsCitusTableTypeCacheEntry(tableEntry, CITUS_LOCAL_TABLE)) { return CITUS_LOCAL_TABLE; } else if (IsCitusTableTypeCacheEntry(tableEntry, APPEND_DISTRIBUTED)) { return APPEND_DISTRIBUTED; } else if (IsCitusTableTypeCacheEntry(tableEntry, RANGE_DISTRIBUTED)) { return RANGE_DISTRIBUTED; } else { return ANY_CITUS_TABLE_TYPE; } } /* * IsCitusTableTypeCacheEntry returns true if the given table cache entry * belongs to a citus table that matches the given table type. */ bool IsCitusTableTypeCacheEntry(CitusTableCacheEntry *tableEntry, CitusTableType tableType) { return IsCitusTableTypeInternal(tableEntry->partitionMethod, tableEntry->replicationModel, tableEntry->colocationId, tableType); } /* * IsFirstShard returns true if the given shardId is the first shard. */ bool IsFirstShard(CitusTableCacheEntry *tableEntry, uint64 shardId) { if (tableEntry == NULL || tableEntry->sortedShardIntervalArray == NULL) { return false; } if (tableEntry->sortedShardIntervalArray[0]->shardId == INVALID_SHARD_ID) { return false; } if (shardId == tableEntry->sortedShardIntervalArray[0]->shardId) { return true; } else { return false; } } /* * HasDistributionKey returns true if given Citus table has a distribution key. */ bool HasDistributionKey(Oid relationId) { CitusTableCacheEntry *tableEntry = LookupCitusTableCacheEntry(relationId); if (tableEntry == NULL) { ereport(ERROR, (errmsg("relation with oid %u is not a Citus table", relationId))); } return HasDistributionKeyCacheEntry(tableEntry); } /* * HasDistributionKeyCacheEntry returns true if given cache entry identifies a * Citus table that has a distribution key. */ bool HasDistributionKeyCacheEntry(CitusTableCacheEntry *tableEntry) { return tableEntry->partitionMethod != DISTRIBUTE_BY_NONE; } /* * IsCitusTableTypeInternal returns true if the given table entry belongs to * the given table type group. For definition of table types, see CitusTableType. */ static bool IsCitusTableTypeInternal(char partitionMethod, char replicationModel, uint32 colocationId, CitusTableType tableType) { switch (tableType) { case HASH_DISTRIBUTED: { return partitionMethod == DISTRIBUTE_BY_HASH; } case APPEND_DISTRIBUTED: { return partitionMethod == DISTRIBUTE_BY_APPEND; } case RANGE_DISTRIBUTED: { return partitionMethod == DISTRIBUTE_BY_RANGE; } case SINGLE_SHARD_DISTRIBUTED: { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel != REPLICATION_MODEL_2PC && colocationId != INVALID_COLOCATION_ID; } case DISTRIBUTED_TABLE: { return partitionMethod == DISTRIBUTE_BY_HASH || partitionMethod == DISTRIBUTE_BY_RANGE || partitionMethod == DISTRIBUTE_BY_APPEND || (partitionMethod == DISTRIBUTE_BY_NONE && replicationModel != REPLICATION_MODEL_2PC && colocationId != INVALID_COLOCATION_ID); } case STRICTLY_PARTITIONED_DISTRIBUTED_TABLE: { return partitionMethod == DISTRIBUTE_BY_HASH || partitionMethod == DISTRIBUTE_BY_RANGE; } case REFERENCE_TABLE: { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel == REPLICATION_MODEL_2PC; } case CITUS_LOCAL_TABLE: { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel != REPLICATION_MODEL_2PC && colocationId == INVALID_COLOCATION_ID; } case ANY_CITUS_TABLE_TYPE: { return true; } default: { ereport(ERROR, (errmsg("Unknown table type %d", tableType))); } } return false; } /* * GetTableTypeName returns string representation of the table type. */ char * GetTableTypeName(Oid tableId) { if (!IsCitusTable(tableId)) { return "regular table"; } CitusTableCacheEntry *tableCacheEntry = GetCitusTableCacheEntry(tableId); if (IsCitusTableTypeCacheEntry(tableCacheEntry, HASH_DISTRIBUTED)) { return "distributed table"; } else if (IsCitusTableTypeCacheEntry(tableCacheEntry, REFERENCE_TABLE)) { return "reference table"; } else if (IsCitusTableTypeCacheEntry(tableCacheEntry, CITUS_LOCAL_TABLE)) { return "citus local table"; } else { return "unknown table"; } } /* * IsCitusTable returns whether relationId is a distributed relation or * not. */ bool IsCitusTable(Oid relationId) { /* * PostgreSQL's OID generator assigns user operation OIDs starting * from FirstNormalObjectId. This means no user object can have * an OID lower than FirstNormalObjectId. Therefore, if the * relationId is less than FirstNormalObjectId * (i.e. in PostgreSQL's reserved range), we can immediately * return false, since such objects cannot be Citus tables. */ if (relationId < FirstNormalObjectId) { return false; } return LookupCitusTableCacheEntry(relationId) != NULL; } /* * IsCitusTableRangeVar returns whether the table named in the given * rangeVar is a Citus table. */ bool IsCitusTableRangeVar(RangeVar *rangeVar, LOCKMODE lockMode, bool missingOK) { Oid relationId = RangeVarGetRelid(rangeVar, lockMode, missingOK); return IsCitusTable(relationId); } /* * IsCitusTableViaCatalog returns whether the given relation is a * distributed table or not. * * It does so by searching pg_dist_partition, explicitly bypassing caches, * because this function is designed to be used in cases where accessing * metadata tables is not safe. * * NB: Currently this still hardcodes pg_dist_partition logicalrelid column * offset and the corresponding index. If we ever come close to changing * that, we'll have to work a bit harder. */ bool IsCitusTableViaCatalog(Oid relationId) { HeapTuple partitionTuple = PgDistPartitionTupleViaCatalog(relationId); bool heapTupleIsValid = HeapTupleIsValid(partitionTuple); if (heapTupleIsValid) { heap_freetuple(partitionTuple); } return heapTupleIsValid; } /* * PartitionMethodViaCatalog gets a relationId and returns the partition * method column from pg_dist_partition via reading from catalog. */ char PartitionMethodViaCatalog(Oid relationId) { HeapTuple partitionTuple = PgDistPartitionTupleViaCatalog(relationId); if (!HeapTupleIsValid(partitionTuple)) { return DISTRIBUTE_BY_INVALID; } Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(partitionTuple, tupleDescriptor, datumArray, isNullArray); if (isNullArray[Anum_pg_dist_partition_partmethod - 1]) { /* partition method cannot be NULL, still let's make sure */ heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return DISTRIBUTE_BY_INVALID; } Datum partitionMethodDatum = datumArray[Anum_pg_dist_partition_partmethod - 1]; char partitionMethodChar = DatumGetChar(partitionMethodDatum); heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return partitionMethodChar; } /* * PartitionColumnViaCatalog gets a relationId and returns the partition * key column from pg_dist_partition via reading from catalog. */ Var * PartitionColumnViaCatalog(Oid relationId) { HeapTuple partitionTuple = PgDistPartitionTupleViaCatalog(relationId); if (!HeapTupleIsValid(partitionTuple)) { return NULL; } Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(partitionTuple, tupleDescriptor, datumArray, isNullArray); if (isNullArray[Anum_pg_dist_partition_partkey - 1]) { /* partition key cannot be NULL, still let's make sure */ heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return NULL; } Datum partitionKeyDatum = datumArray[Anum_pg_dist_partition_partkey - 1]; char *partitionKeyString = TextDatumGetCString(partitionKeyDatum); /* convert the string to a Node and ensure it is a Var */ Node *partitionNode = stringToNode(partitionKeyString); Assert(IsA(partitionNode, Var)); Var *partitionColumn = (Var *) partitionNode; heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return partitionColumn; } /* * ColocationIdViaCatalog gets a relationId and returns the colocation * id column from pg_dist_partition via reading from catalog. */ uint32 ColocationIdViaCatalog(Oid relationId) { HeapTuple partitionTuple = PgDistPartitionTupleViaCatalog(relationId); if (!HeapTupleIsValid(partitionTuple)) { return INVALID_COLOCATION_ID; } Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(partitionTuple, tupleDescriptor, datumArray, isNullArray); if (isNullArray[Anum_pg_dist_partition_colocationid - 1]) { /* colocation id cannot be NULL, still let's make sure */ heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return INVALID_COLOCATION_ID; } Datum colocationIdDatum = datumArray[Anum_pg_dist_partition_colocationid - 1]; uint32 colocationId = DatumGetUInt32(colocationIdDatum); heap_freetuple(partitionTuple); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); return colocationId; } /* * PgDistPartitionTupleViaCatalog is a helper function that searches * pg_dist_partition for the given relationId. The caller is responsible * for ensuring that the returned heap tuple is valid before accessing * its fields. */ static HeapTuple PgDistPartitionTupleViaCatalog(Oid relationId) { const int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = true; Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple partitionTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(partitionTuple)) { /* callers should have the tuple in their memory contexts */ partitionTuple = heap_copytuple(partitionTuple); } systable_endscan(scanDescriptor); table_close(pgDistPartition, AccessShareLock); return partitionTuple; } /* * IsReferenceTableByDistParams returns true if given partitionMethod and * replicationModel would identify a reference table. */ bool IsReferenceTableByDistParams(char partitionMethod, char replicationModel) { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel == REPLICATION_MODEL_2PC; } /* * IsCitusLocalTableByDistParams returns true if given partitionMethod, * replicationModel and colocationId would identify a citus local table. */ bool IsCitusLocalTableByDistParams(char partitionMethod, char replicationModel, uint32 colocationId) { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel != REPLICATION_MODEL_2PC && colocationId == INVALID_COLOCATION_ID; } /* * IsSingleShardTableByDistParams returns true if given partitionMethod, * replicationModel and colocationId would identify a single-shard distributed * table that has a null shard key. */ bool IsSingleShardTableByDistParams(char partitionMethod, char replicationModel, uint32 colocationId) { return partitionMethod == DISTRIBUTE_BY_NONE && replicationModel != REPLICATION_MODEL_2PC && colocationId != INVALID_COLOCATION_ID; } /* * CitusTableList returns a list that includes all the valid distributed table * cache entries. */ List * CitusTableList(void) { List *distributedTableList = NIL; Assert(CitusHasBeenLoaded() && CheckCitusVersion(WARNING)); /* first, we need to iterate over pg_dist_partition */ List *citusTableIdList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid relationId = InvalidOid; foreach_declared_oid(relationId, citusTableIdList) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); distributedTableList = lappend(distributedTableList, cacheEntry); } return distributedTableList; } /* * LoadShardInterval returns the, cached, metadata about a shard. * * The return value is a copy of the cached ShardInterval struct and may * therefore be modified and/or freed. */ ShardInterval * LoadShardInterval(uint64 shardId) { bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; int shardIndex = shardIdEntry->shardIndex; /* the offset better be in a valid range */ Assert(shardIndex < tableEntry->shardIntervalArrayLength); ShardInterval *sourceShardInterval = tableEntry->sortedShardIntervalArray[shardIndex]; /* copy value to return */ ShardInterval *shardInterval = CopyShardInterval(sourceShardInterval); return shardInterval; } /* * ShardExists returns whether given shard exists or not. It fails if missingOk is false * and shard is not found. */ bool ShardExists(uint64 shardId) { bool missingOk = true; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); if (!shardIdEntry) { return false; } return true; } /* * RelationIdOfShard returns the relationId of the given shardId. */ Oid RelationIdForShard(uint64 shardId) { bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; return tableEntry->relationId; } /* * ReferenceTableShardId returns true if the given shardId belongs to * a reference table. */ bool ReferenceTableShardId(uint64 shardId) { bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; return IsCitusTableTypeCacheEntry(tableEntry, REFERENCE_TABLE); } /* * DistributedTableShardId returns true if the given shardId belongs to * a distributed table. */ bool DistributedTableShardId(uint64 shardId) { if (shardId == INVALID_SHARD_ID) { return false; } bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; return IsCitusTableTypeCacheEntry(tableEntry, DISTRIBUTED_TABLE); } /* * LoadGroupShardPlacement returns the cached shard placement metadata * * The return value is a copy of the cached GroupShardPlacement struct and may * therefore be modified and/or freed. */ GroupShardPlacement * LoadGroupShardPlacement(uint64 shardId, uint64 placementId) { bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; int shardIndex = shardIdEntry->shardIndex; /* the offset better be in a valid range */ Assert(shardIndex < tableEntry->shardIntervalArrayLength); GroupShardPlacement *placementArray = tableEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = tableEntry->arrayOfPlacementArrayLengths[shardIndex]; for (int i = 0; i < numberOfPlacements; i++) { if (placementArray[i].placementId == placementId) { GroupShardPlacement *shardPlacement = CitusMakeNode(GroupShardPlacement); *shardPlacement = placementArray[i]; return shardPlacement; } } ereport(ERROR, (errmsg("could not find valid entry for shard placement " UINT64_FORMAT, placementId))); } /* * LoadShardPlacement returns a shard placement for the primary node. */ ShardPlacement * LoadShardPlacement(uint64 shardId, uint64 placementId) { bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; int shardIndex = shardIdEntry->shardIndex; GroupShardPlacement *groupPlacement = LoadGroupShardPlacement(shardId, placementId); ShardPlacement *nodePlacement = ResolveGroupShardPlacement(groupPlacement, tableEntry, shardIndex); return nodePlacement; } /* * ShardPlacementOnGroupIncludingOrphanedPlacements returns the shard placement * for the given shard on the given group, or returns NULL if no placement for * the shard exists on the group. * * NOTE: This can return inactive or orphaned placements. */ ShardPlacement * ShardPlacementOnGroupIncludingOrphanedPlacements(int32 groupId, uint64 shardId) { ShardPlacement *placementOnNode = NULL; bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; int shardIndex = shardIdEntry->shardIndex; GroupShardPlacement *placementArray = tableEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = tableEntry->arrayOfPlacementArrayLengths[shardIndex]; for (int placementIndex = 0; placementIndex < numberOfPlacements; placementIndex++) { GroupShardPlacement *placement = &placementArray[placementIndex]; if (placement->groupId == groupId) { placementOnNode = ResolveGroupShardPlacement(placement, tableEntry, shardIndex); break; } } return placementOnNode; } /* * ActiveShardPlacementOnGroup returns the active shard placement for the * given shard on the given group, or returns NULL if no active placement for * the shard exists on the group. */ ShardPlacement * ActiveShardPlacementOnGroup(int32 groupId, uint64 shardId) { ShardPlacement *placement = ShardPlacementOnGroupIncludingOrphanedPlacements(groupId, shardId); if (placement == NULL) { return NULL; } return placement; } /* * ResolveGroupShardPlacement takes a GroupShardPlacement and adds additional data to it, * such as the node we should consider it to be on. */ static ShardPlacement * ResolveGroupShardPlacement(GroupShardPlacement *groupShardPlacement, CitusTableCacheEntry *tableEntry, int shardIndex) { ShardInterval *shardInterval = tableEntry->sortedShardIntervalArray[shardIndex]; ShardPlacement *shardPlacement = CitusMakeNode(ShardPlacement); int32 groupId = groupShardPlacement->groupId; WorkerNode *workerNode = LookupNodeForGroup(groupId); /* copy everything into shardPlacement but preserve the header */ CitusNode header = shardPlacement->type; GroupShardPlacement *shardPlacementAsGroupPlacement = (GroupShardPlacement *) shardPlacement; *shardPlacementAsGroupPlacement = *groupShardPlacement; shardPlacement->type = header; SetPlacementNodeMetadata(shardPlacement, workerNode); /* fill in remaining fields */ Assert(tableEntry->partitionMethod != 0); shardPlacement->partitionMethod = tableEntry->partitionMethod; shardPlacement->colocationGroupId = tableEntry->colocationId; if (tableEntry->partitionMethod == DISTRIBUTE_BY_HASH) { Assert(shardInterval->minValueExists); Assert(shardInterval->valueTypeId == INT4OID); /* * Use the lower boundary of the interval's range to identify * it for colocation purposes. That remains meaningful even if * a concurrent session splits a shard. */ shardPlacement->representativeValue = DatumGetInt32(shardInterval->minValue); } else { shardPlacement->representativeValue = 0; } return shardPlacement; } /* * HasAnyNodes returns whether there are any nodes in pg_dist_node. */ bool HasAnyNodes(void) { PrepareWorkerNodeCache(); return WorkerNodeCount > 0; } /* * LookupNodeByNodeId returns a worker node by nodeId or NULL if the node * cannot be found. */ WorkerNode * LookupNodeByNodeId(uint32 nodeId) { PrepareWorkerNodeCache(); for (int workerNodeIndex = 0; workerNodeIndex < WorkerNodeCount; workerNodeIndex++) { WorkerNode *workerNode = WorkerNodeArray[workerNodeIndex]; if (workerNode->nodeId == nodeId) { WorkerNode *workerNodeCopy = palloc0(sizeof(WorkerNode)); *workerNodeCopy = *workerNode; return workerNodeCopy; } } return NULL; } /* * LookupNodeByNodeIdOrError returns a worker node by nodeId or errors out if the * node cannot be found. */ WorkerNode * LookupNodeByNodeIdOrError(uint32 nodeId) { WorkerNode *node = LookupNodeByNodeId(nodeId); if (node == NULL) { ereport(ERROR, (errmsg("node %d could not be found", nodeId))); } return node; } /* * LookupNodeForGroup searches the WorkerNodeHash for a worker which is a member of the * given group and also readable (a primary if we're reading from primaries, a secondary * if we're reading from secondaries). If such a node does not exist it emits an * appropriate error message. */ WorkerNode * LookupNodeForGroup(int32 groupId) { bool foundAnyNodes = false; PrepareWorkerNodeCache(); for (int workerNodeIndex = 0; workerNodeIndex < WorkerNodeCount; workerNodeIndex++) { WorkerNode *workerNode = WorkerNodeArray[workerNodeIndex]; int32 workerNodeGroupId = workerNode->groupId; if (workerNodeGroupId != groupId) { continue; } foundAnyNodes = true; if (NodeIsReadable(workerNode)) { return workerNode; } } if (!foundAnyNodes) { ereport(ERROR, (errmsg("there is a shard placement in node group %d but " "there are no nodes in that group", groupId))); } switch (ReadFromSecondaries) { case USE_SECONDARY_NODES_NEVER: { ereport(ERROR, (errmsg("node group %d does not have a primary node", groupId))); break; } case USE_SECONDARY_NODES_ALWAYS: { ereport(ERROR, (errmsg("node group %d does not have a secondary node", groupId))); break; } default: { ereport(FATAL, (errmsg("unrecognized value for use_secondary_nodes"))); } } } /* * ShardPlacementList returns the list of placements for the given shard from * the cache. * * The returned list is deep copied from the cache and thus can be modified * and pfree()d freely. */ List * ShardPlacementList(uint64 shardId) { List *placementList = NIL; bool missingOk = false; ShardIdCacheEntry *shardIdEntry = LookupShardIdCacheEntry(shardId, missingOk); CitusTableCacheEntry *tableEntry = shardIdEntry->tableEntry; int shardIndex = shardIdEntry->shardIndex; /* the offset better be in a valid range */ Assert(shardIndex < tableEntry->shardIntervalArrayLength); GroupShardPlacement *placementArray = tableEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = tableEntry->arrayOfPlacementArrayLengths[shardIndex]; for (int i = 0; i < numberOfPlacements; i++) { GroupShardPlacement *groupShardPlacement = &placementArray[i]; ShardPlacement *shardPlacement = ResolveGroupShardPlacement(groupShardPlacement, tableEntry, shardIndex); placementList = lappend(placementList, shardPlacement); } /* if no shard placements are found, warn the user */ if (numberOfPlacements == 0) { ereport(WARNING, (errmsg("could not find any shard placements for shardId " UINT64_FORMAT, shardId))); } return placementList; } /* * InitializeTableCacheEntry initializes a shard in cache. A possible reason * for not finding an entry in the cache is that the distributed table's cache * entry hasn't been accessed yet. Thus look up the distributed table, and * build the cache entry. Afterwards we know that the shard has to be in the * cache if it exists. If the shard does *not* exist, this function errors * (because LookupShardRelationFromCatalog errors out). * * If missingOk is true and the shard cannot be found, the function returns false. */ static bool InitializeTableCacheEntry(int64 shardId, bool missingOk) { Oid relationId = LookupShardRelationFromCatalog(shardId, missingOk); if (!OidIsValid(relationId)) { Assert(missingOk); return false; } /* trigger building the cache for the shard id */ GetCitusTableCacheEntry(relationId); /* lgtm[cpp/return-value-ignored] */ return true; } /* * RefreshTableCacheEntryIfInvalid checks if the cache entry is still valid and * refreshes it in cache when it's not. It returns true if it refreshed the * entry in the cache and false if it didn't. */ static bool RefreshTableCacheEntryIfInvalid(ShardIdCacheEntry *shardEntry, bool missingOk) { /* * We might have some concurrent metadata changes. In order to get the changes, * we first need to accept the cache invalidation messages. */ AcceptInvalidationMessages(); if (shardEntry->tableEntry->isValid) { return false; } Oid oldRelationId = shardEntry->tableEntry->relationId; Oid currentRelationId = LookupShardRelationFromCatalog(shardEntry->shardId, missingOk); /* * The relation OID to which the shard belongs could have changed, * most notably when the extension is dropped and a shard ID is * reused. Reload the cache entries for both old and new relation * ID and then look up the shard entry again. */ LookupCitusTableCacheEntry(oldRelationId); LookupCitusTableCacheEntry(currentRelationId); return true; } /* * LookupShardCacheEntry returns the cache entry belonging to a shard. * It errors out if that shard is unknown and missingOk is false. Else, * it will return a NULL cache entry. */ static ShardIdCacheEntry * LookupShardIdCacheEntry(int64 shardId, bool missingOk) { bool foundInCache = false; bool recheck = false; Assert(CitusHasBeenLoaded() && CheckCitusVersion(WARNING)); InitializeCaches(); ShardIdCacheEntry *shardEntry = hash_search(ShardIdCacheHash, &shardId, HASH_FIND, &foundInCache); if (!foundInCache) { if (!InitializeTableCacheEntry(shardId, missingOk)) { return NULL; } recheck = true; } else { recheck = RefreshTableCacheEntryIfInvalid(shardEntry, missingOk); } /* * If we (re-)loaded the table cache, re-search the shard cache - the * shard index might have changed. If we still can't find the entry, it * can't exist. */ if (recheck) { shardEntry = hash_search(ShardIdCacheHash, &shardId, HASH_FIND, &foundInCache); if (!foundInCache) { int eflag = (missingOk) ? DEBUG1 : ERROR; ereport(eflag, (errmsg("could not find valid entry for shard " UINT64_FORMAT, shardId))); } } return shardEntry; } /* * GetCitusTableCacheEntry looks up a pg_dist_partition entry for a * relation. * * Errors out if no relation matching the criteria could be found. */ CitusTableCacheEntry * GetCitusTableCacheEntry(Oid distributedRelationId) { CitusTableCacheEntry *cacheEntry = LookupCitusTableCacheEntry(distributedRelationId); if (cacheEntry) { return cacheEntry; } else { char *relationName = get_rel_name(distributedRelationId); if (relationName == NULL) { ereport(ERROR, (errmsg("relation with OID %u does not exist", distributedRelationId))); } else { ereport(ERROR, (errmsg("relation %s is not distributed", relationName))); } } } /* * GetCitusTableCacheEntry returns the distributed table metadata for the * passed relationId. For efficiency it caches lookups. This function returns * NULL if the relation isn't a distributed table. */ CitusTableCacheEntry * LookupCitusTableCacheEntry(Oid relationId) { bool foundInCache = false; void *hashKey = (void *) &relationId; /* * Can't be a distributed relation if the extension hasn't been loaded * yet. As we can't do lookups in nonexistent tables, directly return NULL * here. */ if (!CitusHasBeenLoaded()) { return NULL; } InitializeCaches(); /* * If the version is not known to be compatible, perform thorough check, * unless such checks are disabled. */ if (!citusVersionKnownCompatible && EnableVersionChecks) { bool isCitusTable = IsCitusTableViaCatalog(relationId); int reportLevel = DEBUG1; /* * If there's a version-mismatch, and we're dealing with a distributed * table, we have to error out as we can't return a valid entry. We * want to check compatibility in the non-distributed case as well, so * future lookups can use the cache if compatible. */ if (isCitusTable) { reportLevel = ERROR; } if (!CheckCitusVersion(reportLevel)) { /* incompatible, can't access cache, so return before doing so */ return NULL; } } /* * We might have some concurrent metadata changes. In order to get the changes, * we first need to accept the cache invalidation messages. */ AcceptInvalidationMessages(); CitusTableCacheEntrySlot *cacheSlot = hash_search(DistTableCacheHash, hashKey, HASH_ENTER, &foundInCache); /* return valid matches */ if (foundInCache) { if (cacheSlot->isValid) { return cacheSlot->citusTableMetadata; } else { /* * An invalidation was received or we encountered an OOM while building * the cache entry. We need to rebuild it. */ if (cacheSlot->citusTableMetadata) { /* * The CitusTableCacheEntry might still be in use. We therefore do * not reset it until the end of the transaction. */ MemoryContext oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); DistTableCacheExpired = lappend(DistTableCacheExpired, cacheSlot->citusTableMetadata); MemoryContextSwitchTo(oldContext); } } } /* zero out entry, but not the key part */ memset(((char *) cacheSlot) + sizeof(Oid), 0, sizeof(CitusTableCacheEntrySlot) - sizeof(Oid)); /* * We disable interrupts while creating the cache entry because loading * shard metadata can take a while, and if statement_timeout is too low, * this will get canceled on each call and we won't be able to run any * queries on the table. */ HOLD_INTERRUPTS(); cacheSlot->citusTableMetadata = BuildCitusTableCacheEntry(relationId); /* * Mark it as valid only after building the full entry, such that any * error that happened during the build would trigger a rebuild. */ cacheSlot->isValid = true; RESUME_INTERRUPTS(); return cacheSlot->citusTableMetadata; } /* * LookupDistObjectCacheEntry returns the distributed table metadata for the * passed relationId. For efficiency it caches lookups. */ DistObjectCacheEntry * LookupDistObjectCacheEntry(Oid classid, Oid objid, int32 objsubid) { bool foundInCache = false; DistObjectCacheEntryKey hashKey; ScanKeyData pgDistObjectKey[3]; memset(&hashKey, 0, sizeof(DistObjectCacheEntryKey)); hashKey.classid = classid; hashKey.objid = objid; hashKey.objsubid = objsubid; /* * Can't be a distributed relation if the extension hasn't been loaded * yet. As we can't do lookups in nonexistent tables, directly return NULL * here. */ if (!CitusHasBeenLoaded()) { return NULL; } InitializeCaches(); DistObjectCacheEntry *cacheEntry = hash_search(DistObjectCacheHash, &hashKey, HASH_ENTER, &foundInCache); /* return valid matches */ if (foundInCache) { /* * We might have some concurrent metadata changes. In order to get the changes, * we first need to accept the cache invalidation messages. */ AcceptInvalidationMessages(); if (cacheEntry->isValid) { return cacheEntry; } /* * This is where we'd free the old entry's out of band data if it had any. * Right now we don't have anything to free. */ } /* zero out entry, but not the key part */ memset(((char *) cacheEntry), 0, sizeof(DistObjectCacheEntry)); cacheEntry->key.classid = classid; cacheEntry->key.objid = objid; cacheEntry->key.objsubid = objsubid; Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); TupleDesc pgDistObjectTupleDesc = RelationGetDescr(pgDistObjectRel); ScanKeyInit(&pgDistObjectKey[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(classid)); ScanKeyInit(&pgDistObjectKey[1], Anum_pg_dist_object_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(objid)); ScanKeyInit(&pgDistObjectKey[2], Anum_pg_dist_object_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(objsubid)); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, DistObjectPrimaryKeyIndexId(), true, NULL, 3, pgDistObjectKey); HeapTuple pgDistObjectTup = systable_getnext(pgDistObjectScan); if (HeapTupleIsValid(pgDistObjectTup)) { Datum *datumArray = palloc(pgDistObjectTupleDesc->natts * sizeof(Datum)); bool *isNullArray = palloc(pgDistObjectTupleDesc->natts * sizeof(bool)); int forseDelegationIndex = GetForceDelegationAttrIndexInPgDistObject(pgDistObjectTupleDesc); heap_deform_tuple(pgDistObjectTup, pgDistObjectTupleDesc, datumArray, isNullArray); cacheEntry->isValid = true; cacheEntry->isDistributed = true; cacheEntry->distributionArgIndex = DatumGetInt32(datumArray[Anum_pg_dist_object_distribution_argument_index - 1]); cacheEntry->colocationId = DatumGetInt32(datumArray[Anum_pg_dist_object_colocationid - 1]); cacheEntry->forceDelegation = DatumGetBool(datumArray[forseDelegationIndex]); pfree(datumArray); pfree(isNullArray); } else { cacheEntry->isValid = true; cacheEntry->isDistributed = false; } systable_endscan(pgDistObjectScan); relation_close(pgDistObjectRel, AccessShareLock); return cacheEntry; } /* * BuildCitusTableCacheEntry is a helper routine for * LookupCitusTableCacheEntry() for building the cache contents. * This function returns NULL if the relation isn't a distributed table. */ static CitusTableCacheEntry * BuildCitusTableCacheEntry(Oid relationId) { Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); HeapTuple distPartitionTuple = LookupDistPartitionTuple(pgDistPartition, relationId); if (distPartitionTuple == NULL) { /* not a distributed table, done */ table_close(pgDistPartition, NoLock); return NULL; } MemoryContext oldContext = NULL; TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(distPartitionTuple, tupleDescriptor, datumArray, isNullArray); CitusTableCacheEntry *cacheEntry = MemoryContextAllocZero(MetadataCacheMemoryContext, sizeof(CitusTableCacheEntry)); cacheEntry->relationId = relationId; cacheEntry->partitionMethod = datumArray[Anum_pg_dist_partition_partmethod - 1]; Datum partitionKeyDatum = datumArray[Anum_pg_dist_partition_partkey - 1]; bool partitionKeyIsNull = isNullArray[Anum_pg_dist_partition_partkey - 1]; /* note that for reference tables partitionKeyisNull is true */ if (!partitionKeyIsNull) { oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); /* get the string representation of the partition column Var */ cacheEntry->partitionKeyString = TextDatumGetCString(partitionKeyDatum); /* convert the string to a Node and ensure it is a Var */ Node *partitionNode = stringToNode(cacheEntry->partitionKeyString); Assert(IsA(partitionNode, Var)); cacheEntry->partitionColumn = (Var *) partitionNode; MemoryContextSwitchTo(oldContext); } else { cacheEntry->partitionKeyString = NULL; } cacheEntry->colocationId = datumArray[Anum_pg_dist_partition_colocationid - 1]; if (isNullArray[Anum_pg_dist_partition_colocationid - 1]) { cacheEntry->colocationId = INVALID_COLOCATION_ID; } Datum replicationModelDatum = datumArray[Anum_pg_dist_partition_repmodel - 1]; if (isNullArray[Anum_pg_dist_partition_repmodel - 1]) { /* * repmodel is NOT NULL but before ALTER EXTENSION citus UPGRADE the column * doesn't exist */ cacheEntry->replicationModel = 'c'; } else { cacheEntry->replicationModel = DatumGetChar(replicationModelDatum); } if (isNullArray[GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor)]) { /* * We don't expect this to happen, but set it to false (the default value) * to not break if anything goes wrong. */ cacheEntry->autoConverted = false; } else { cacheEntry->autoConverted = DatumGetBool( datumArray[GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor)]); } heap_freetuple(distPartitionTuple); BuildCachedShardList(cacheEntry); /* we only need hash functions for hash distributed tables */ if (cacheEntry->partitionMethod == DISTRIBUTE_BY_HASH) { Var *partitionColumn = cacheEntry->partitionColumn; TypeCacheEntry *typeEntry = lookup_type_cache(partitionColumn->vartype, TYPECACHE_HASH_PROC_FINFO); FmgrInfo *hashFunction = MemoryContextAllocZero(MetadataCacheMemoryContext, sizeof(FmgrInfo)); fmgr_info_copy(hashFunction, &(typeEntry->hash_proc_finfo), MetadataCacheMemoryContext); cacheEntry->hashFunction = hashFunction; /* check the shard distribution for hash partitioned tables */ cacheEntry->hasUniformHashDistribution = HasUniformHashDistribution(cacheEntry->sortedShardIntervalArray, cacheEntry->shardIntervalArrayLength); } else { cacheEntry->hashFunction = NULL; } oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); cacheEntry->referencedRelationsViaForeignKey = ReferencedRelationIdList( cacheEntry->relationId); cacheEntry->referencingRelationsViaForeignKey = ReferencingRelationIdList( cacheEntry->relationId); MemoryContextSwitchTo(oldContext); table_close(pgDistPartition, NoLock); pfree(datumArray); pfree(isNullArray); cacheEntry->isValid = true; return cacheEntry; } /* * BuildCachedShardList() is a helper routine for BuildCitusTableCacheEntry() * building up the list of shards in a distributed relation. */ static void BuildCachedShardList(CitusTableCacheEntry *cacheEntry) { ShardInterval **shardIntervalArray = NULL; ShardInterval **sortedShardIntervalArray = NULL; FmgrInfo *shardIntervalCompareFunction = NULL; FmgrInfo *shardColumnCompareFunction = NULL; Oid columnTypeId = InvalidOid; int32 columnTypeMod = -1; Oid intervalTypeId = InvalidOid; int32 intervalTypeMod = -1; GetPartitionTypeInputInfo(cacheEntry->partitionKeyString, cacheEntry->partitionMethod, &columnTypeId, &columnTypeMod, &intervalTypeId, &intervalTypeMod); List *distShardTupleList = LookupDistShardTuples(cacheEntry->relationId); int shardIntervalArrayLength = list_length(distShardTupleList); if (shardIntervalArrayLength > 0) { Relation distShardRelation = table_open(DistShardRelationId(), AccessShareLock); TupleDesc distShardTupleDesc = RelationGetDescr(distShardRelation); int arrayIndex = 0; shardIntervalArray = MemoryContextAllocZero(MetadataCacheMemoryContext, shardIntervalArrayLength * sizeof(ShardInterval *)); cacheEntry->arrayOfPlacementArrays = MemoryContextAllocZero(MetadataCacheMemoryContext, shardIntervalArrayLength * sizeof(GroupShardPlacement *)); cacheEntry->arrayOfPlacementArrayLengths = MemoryContextAllocZero(MetadataCacheMemoryContext, shardIntervalArrayLength * sizeof(int)); HeapTuple shardTuple = NULL; foreach_declared_ptr(shardTuple, distShardTupleList) { ShardInterval *shardInterval = TupleToShardInterval(shardTuple, distShardTupleDesc, intervalTypeId, intervalTypeMod); MemoryContext oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); shardIntervalArray[arrayIndex] = CopyShardInterval(shardInterval); MemoryContextSwitchTo(oldContext); heap_freetuple(shardTuple); arrayIndex++; } table_close(distShardRelation, AccessShareLock); } /* look up value comparison function */ if (columnTypeId != InvalidOid) { /* allocate the comparison function in the cache context */ MemoryContext oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); shardColumnCompareFunction = GetFunctionInfo(columnTypeId, BTREE_AM_OID, BTORDER_PROC); MemoryContextSwitchTo(oldContext); } else { shardColumnCompareFunction = NULL; } /* look up interval comparison function */ if (intervalTypeId != InvalidOid) { /* allocate the comparison function in the cache context */ MemoryContext oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); shardIntervalCompareFunction = GetFunctionInfo(intervalTypeId, BTREE_AM_OID, BTORDER_PROC); MemoryContextSwitchTo(oldContext); } else { shardIntervalCompareFunction = NULL; } /* reference tables has a single shard which is not initialized */ if (cacheEntry->partitionMethod == DISTRIBUTE_BY_NONE) { cacheEntry->hasUninitializedShardInterval = true; cacheEntry->hasOverlappingShardInterval = true; /* * Note that during create_reference_table() call, * the reference table do not have any shards. */ if (shardIntervalArrayLength > 1) { char *relationName = get_rel_name(cacheEntry->relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("reference table \"%s\" has more than 1 shard", relationName))); } /* since there is a zero or one shard, it is already sorted */ sortedShardIntervalArray = shardIntervalArray; } else { /* sort the interval array */ sortedShardIntervalArray = SortShardIntervalArray(shardIntervalArray, shardIntervalArrayLength, cacheEntry->partitionColumn-> varcollid, shardIntervalCompareFunction); /* check if there exists any shard intervals with no min/max values */ cacheEntry->hasUninitializedShardInterval = HasUninitializedShardInterval(sortedShardIntervalArray, shardIntervalArrayLength); if (!cacheEntry->hasUninitializedShardInterval) { cacheEntry->hasOverlappingShardInterval = HasOverlappingShardInterval(sortedShardIntervalArray, shardIntervalArrayLength, cacheEntry->partitionColumn->varcollid, shardIntervalCompareFunction); } else { cacheEntry->hasOverlappingShardInterval = true; } ErrorIfInconsistentShardIntervals(cacheEntry); } cacheEntry->sortedShardIntervalArray = sortedShardIntervalArray; cacheEntry->shardIntervalArrayLength = 0; /* maintain shardId->(table,ShardInterval) cache */ for (int shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++) { ShardInterval *shardInterval = sortedShardIntervalArray[shardIndex]; int64 shardId = shardInterval->shardId; int placementOffset = 0; /* * Enable quick lookups of this shard ID by adding it to ShardIdCacheHash * or overwriting the previous values. */ ShardIdCacheEntry *shardIdCacheEntry = hash_search(ShardIdCacheHash, &shardId, HASH_ENTER, NULL); shardIdCacheEntry->tableEntry = cacheEntry; shardIdCacheEntry->shardIndex = shardIndex; /* * We should increment this only after we are sure this hasn't already * been assigned to any other relations. ResetCitusTableCacheEntry() * depends on this. */ cacheEntry->shardIntervalArrayLength++; /* build list of shard placements */ List *placementList = BuildShardPlacementList(shardId); int numberOfPlacements = list_length(placementList); /* and copy that list into the cache entry */ MemoryContext oldContext = MemoryContextSwitchTo(MetadataCacheMemoryContext); GroupShardPlacement *placementArray = palloc0(numberOfPlacements * sizeof(GroupShardPlacement)); GroupShardPlacement *srcPlacement = NULL; foreach_declared_ptr(srcPlacement, placementList) { placementArray[placementOffset] = *srcPlacement; placementOffset++; } MemoryContextSwitchTo(oldContext); cacheEntry->arrayOfPlacementArrays[shardIndex] = placementArray; cacheEntry->arrayOfPlacementArrayLengths[shardIndex] = numberOfPlacements; /* store the shard index in the ShardInterval */ shardInterval->shardIndex = shardIndex; } cacheEntry->shardColumnCompareFunction = shardColumnCompareFunction; cacheEntry->shardIntervalCompareFunction = shardIntervalCompareFunction; } /* * ErrorIfInconsistentShardIntervals checks if shard intervals are consistent with * our expectations. */ void ErrorIfInconsistentShardIntervals(CitusTableCacheEntry *cacheEntry) { /* * If table is hash-partitioned and has shards, there never should be any * uninitalized shards. Historically we've not prevented that for range * partitioned tables, but it might be a good idea to start doing so. */ if (cacheEntry->partitionMethod == DISTRIBUTE_BY_HASH && cacheEntry->hasUninitializedShardInterval) { ereport(ERROR, (errmsg("hash partitioned table has uninitialized shards"))); } if (cacheEntry->partitionMethod == DISTRIBUTE_BY_HASH && cacheEntry->hasOverlappingShardInterval) { ereport(ERROR, (errmsg("hash partitioned table has overlapping shards"))); } } /* * HasUniformHashDistribution determines whether the given list of sorted shards * has a uniform hash distribution, as produced by master_create_worker_shards for * hash partitioned tables. */ bool HasUniformHashDistribution(ShardInterval **shardIntervalArray, int shardIntervalArrayLength) { /* if there are no shards, there is no uniform distribution */ if (shardIntervalArrayLength == 0) { return false; } /* calculate the hash token increment */ uint64 hashTokenIncrement = HASH_TOKEN_COUNT / shardIntervalArrayLength; for (int shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++) { ShardInterval *shardInterval = shardIntervalArray[shardIndex]; int32 shardMinHashToken = PG_INT32_MIN + (shardIndex * hashTokenIncrement); int32 shardMaxHashToken = shardMinHashToken + (hashTokenIncrement - 1); if (shardIndex == (shardIntervalArrayLength - 1)) { shardMaxHashToken = PG_INT32_MAX; } if (DatumGetInt32(shardInterval->minValue) != shardMinHashToken || DatumGetInt32(shardInterval->maxValue) != shardMaxHashToken) { return false; } } return true; } /* * HasUninitializedShardInterval returns true if all the elements of the * sortedShardIntervalArray has min/max values. Callers of the function must * ensure that input shard interval array is sorted on shardminvalue and uninitialized * shard intervals are at the end of the array. */ bool HasUninitializedShardInterval(ShardInterval **sortedShardIntervalArray, int shardCount) { bool hasUninitializedShardInterval = false; if (shardCount == 0) { return hasUninitializedShardInterval; } Assert(sortedShardIntervalArray != NULL); /* * Since the shard interval array is sorted, and uninitialized ones stored * in the end of the array, checking the last element is enough. */ ShardInterval *lastShardInterval = sortedShardIntervalArray[shardCount - 1]; if (!lastShardInterval->minValueExists || !lastShardInterval->maxValueExists) { hasUninitializedShardInterval = true; } return hasUninitializedShardInterval; } /* * HasOverlappingShardInterval determines whether the given list of sorted * shards has overlapping ranges. */ bool HasOverlappingShardInterval(ShardInterval **shardIntervalArray, int shardIntervalArrayLength, Oid shardIntervalCollation, FmgrInfo *shardIntervalSortCompareFunction) { Datum comparisonDatum = 0; int comparisonResult = 0; /* zero/a single shard can't overlap */ if (shardIntervalArrayLength < 2) { return false; } ShardInterval *lastShardInterval = shardIntervalArray[0]; for (int shardIndex = 1; shardIndex < shardIntervalArrayLength; shardIndex++) { ShardInterval *curShardInterval = shardIntervalArray[shardIndex]; /* only called if !hasUninitializedShardInterval */ Assert(lastShardInterval->minValueExists && lastShardInterval->maxValueExists); Assert(curShardInterval->minValueExists && curShardInterval->maxValueExists); comparisonDatum = FunctionCall2Coll(shardIntervalSortCompareFunction, shardIntervalCollation, lastShardInterval->maxValue, curShardInterval->minValue); comparisonResult = DatumGetInt32(comparisonDatum); if (comparisonResult >= 0) { return true; } lastShardInterval = curShardInterval; } return false; } /* * CitusHasBeenLoaded returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. The result is cached as this is called very frequently. */ bool CitusHasBeenLoaded(void) { /* * We do not use Citus hooks during CREATE/ALTER EXTENSION citus * since the objects used by the C code might be not be there yet. */ if (creating_extension) { Oid citusExtensionOid = get_extension_oid("citus", true); if (CurrentExtensionObject == citusExtensionOid) { return false; } } /* * If extensionCreatedState is UNKNOWN, query pg_extension for Citus * and cache the result. Otherwise return the value extensionCreatedState * indicates. */ if (MetadataCache.extensionCreatedState == UNKNOWN) { bool extensionCreated = CitusHasBeenLoadedInternal(); if (extensionCreated) { /* * Loaded Citus for the first time in this session, or first time after * CREATE/ALTER EXTENSION citus. Do some initialisation. */ /* * Make sure the maintenance daemon is running if it was not already. */ StartupCitusBackend(); /* * This needs to be initialized so we can receive foreign relation graph * invalidation messages in InvalidateForeignRelationGraphCacheCallback(). * See the comments of InvalidateForeignKeyGraph for more context. */ DistColocationRelationId(); MetadataCache.extensionCreatedState = CREATED; } else { MetadataCache.extensionCreatedState = NOTCREATED; } } return (MetadataCache.extensionCreatedState == CREATED) ? true : false; } /* * CitusHasBeenLoadedInternal returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. */ static bool CitusHasBeenLoadedInternal(void) { if (IsBinaryUpgrade) { /* never use Citus logic during pg_upgrade */ return false; } Oid citusExtensionOid = get_extension_oid("citus", true); if (citusExtensionOid == InvalidOid) { /* Citus extension does not exist yet */ return false; } /* citus extension exists and has been created */ return true; } /* * GetCitusCreationLevel returns the level of the transaction creating citus */ int GetCitusCreationLevel(void) { return CreateCitusTransactionLevel; } /* * Sets the value of CreateCitusTransactionLevel based on int received which represents the * nesting level of the transaction that created the Citus extension */ void SetCreateCitusTransactionLevel(int val) { CreateCitusTransactionLevel = val; } /* * CheckCitusVersion checks whether there is a version mismatch between the * available version and the loaded version or between the installed version * and the loaded version. Returns true if compatible, false otherwise. * * As a side effect, this function also sets citusVersionKnownCompatible global * variable to true which reduces version check cost of next calls. */ bool CheckCitusVersion(int elevel) { if (citusVersionKnownCompatible || !CitusHasBeenLoaded() || !EnableVersionChecks) { return true; } if (CheckAvailableVersion(elevel) && CheckInstalledVersion(elevel)) { citusVersionKnownCompatible = true; return true; } else { return false; } } /* * CheckAvailableVersion compares CITUS_EXTENSIONVERSION and the currently * available version from the citus.control file. If they are not compatible, * this function logs an error with the specified elevel and returns false, * otherwise it returns true. */ bool CheckAvailableVersion(int elevel) { if (!EnableVersionChecks) { return true; } char *availableVersion = AvailableExtensionVersion(); if (!MajorVersionsCompatible(availableVersion, CITUS_EXTENSIONVERSION)) { ereport(elevel, (errmsg("loaded Citus library version differs from latest " "available extension version"), errdetail("Loaded library requires %s, but the latest control " "file specifies %s.", CITUS_MAJORVERSION, availableVersion), errhint("Restart the database to load the latest Citus " "library."))); return false; } return true; } /* * CheckInstalledVersion compares CITUS_EXTENSIONVERSION and the * extension's current version from the pg_extension catalog table. If they * are not compatible, this function logs an error with the specified elevel, * otherwise it returns true. */ static bool CheckInstalledVersion(int elevel) { Assert(CitusHasBeenLoaded()); Assert(EnableVersionChecks); char *installedVersion = InstalledExtensionVersion(); if (!MinorVersionsCompatibleRelaxed(installedVersion, CITUS_EXTENSIONVERSION)) { ereport(elevel, (errmsg("loaded Citus library version differs from installed " "extension version"), errdetail("Loaded library requires %s, but the installed " "extension version is %s.", CITUS_MAJORVERSION, installedVersion), errhint("Run ALTER EXTENSION citus UPDATE and try again."))); return false; } return true; } /* * InstalledAndAvailableVersionsSame compares extension's available version and * its current version from the pg_extension catalog table. If they are not same * returns false, otherwise returns true. */ bool InstalledAndAvailableVersionsSame() { char *installedVersion = InstalledExtensionVersion(); char *availableVersion = AvailableExtensionVersion(); if (strncmp(installedVersion, availableVersion, NAMEDATALEN) == 0) { return true; } return false; } /* * MajorVersionsCompatible checks whether both versions are compatible. They * are if major and minor version numbers match, the schema version is * ignored. Returns true if compatible, false otherwise. */ bool MajorVersionsCompatible(char *leftVersion, char *rightVersion) { const char schemaVersionSeparator = '-'; char *leftSeperatorPosition = strchr(leftVersion, schemaVersionSeparator); char *rightSeperatorPosition = strchr(rightVersion, schemaVersionSeparator); int leftComparisionLimit = 0; int rightComparisionLimit = 0; if (leftSeperatorPosition != NULL) { leftComparisionLimit = leftSeperatorPosition - leftVersion; } else { leftComparisionLimit = strlen(leftVersion); } if (rightSeperatorPosition != NULL) { rightComparisionLimit = rightSeperatorPosition - rightVersion; } else { rightComparisionLimit = strlen(leftVersion); } /* we can error out early if hypens are not in the same position */ if (leftComparisionLimit != rightComparisionLimit) { return false; } return strncmp(leftVersion, rightVersion, leftComparisionLimit) == 0; } /* * ParseVersionComponent parses the integer at the current position and * advances endPtr past the parsed digits to the next character. */ static int ParseVersionComponent(const char *version, char **endPtr) { errno = 0; long int val = strtol(version, endPtr, 10); if (errno == ERANGE || val > INT_MAX || val < INT_MIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("Invalid integer in version string"))); } return (int) val; } /* * MinorVersionsCompatibleRelaxed checks if two versions have the same major * version and their minor versions differ by at most 1. The schema version * (after '-') is ignored. Returns true if compatible, false otherwise. * * Version format expected: "major.minor-schema" (e.g., "13.1-2") */ bool MinorVersionsCompatibleRelaxed(char *leftVersion, char *rightVersion) { char *leftSep; char *rightSep; int leftMajor = ParseVersionComponent(leftVersion, &leftSep); int rightMajor = ParseVersionComponent(rightVersion, &rightSep); if (leftMajor != rightMajor) { return false; } int leftMinor = (*leftSep == '.') ? ParseVersionComponent(leftSep + 1, &leftSep) : 0; int rightMinor = (*rightSep == '.') ? ParseVersionComponent(rightSep + 1, &rightSep) : 0; int diff = leftMinor - rightMinor; return diff >= -1 && diff <= 1; } /* * AvailableExtensionVersion returns the Citus version from citus.control file. It also * saves the result, thus consecutive calls to CitusExtensionAvailableVersion will * not read the citus.control file again. */ static char * AvailableExtensionVersion(void) { LOCAL_FCINFO(fcinfo, 0); FmgrInfo flinfo; bool goForward = true; bool doCopy = false; char *availableExtensionVersion; InitializeCaches(); EState *estate = CreateExecutorState(); ReturnSetInfo *extensionsResultSet = makeNode(ReturnSetInfo); extensionsResultSet->econtext = GetPerTupleExprContext(estate); extensionsResultSet->allowedModes = SFRM_Materialize; fmgr_info(F_PG_AVAILABLE_EXTENSIONS, &flinfo); InitFunctionCallInfoData(*fcinfo, &flinfo, 0, InvalidOid, NULL, (Node *) extensionsResultSet); /* pg_available_extensions returns result set containing all available extensions */ (*pg_available_extensions)(fcinfo); TupleTableSlot *tupleTableSlot = MakeSingleTupleTableSlot( extensionsResultSet->setDesc, &TTSOpsMinimalTuple); bool hasTuple = tuplestore_gettupleslot(extensionsResultSet->setResult, goForward, doCopy, tupleTableSlot); while (hasTuple) { bool isNull = false; Datum extensionNameDatum = slot_getattr(tupleTableSlot, 1, &isNull); char *extensionName = NameStr(*DatumGetName(extensionNameDatum)); if (strcmp(extensionName, "citus") == 0) { Datum availableVersion = slot_getattr(tupleTableSlot, 2, &isNull); /* we will cache the result of citus version to prevent catalog access */ MemoryContext oldMemoryContext = MemoryContextSwitchTo( MetadataCacheMemoryContext); availableExtensionVersion = text_to_cstring(DatumGetTextPP(availableVersion)); MemoryContextSwitchTo(oldMemoryContext); ExecClearTuple(tupleTableSlot); ExecDropSingleTupleTableSlot(tupleTableSlot); return availableExtensionVersion; } ExecClearTuple(tupleTableSlot); hasTuple = tuplestore_gettupleslot(extensionsResultSet->setResult, goForward, doCopy, tupleTableSlot); } ExecDropSingleTupleTableSlot(tupleTableSlot); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not found"))); return NULL; /* keep compiler happy */ } /* * InstalledExtensionVersion returns the Citus version in PostgreSQL pg_extension table. */ static char * InstalledExtensionVersion(void) { ScanKeyData entry[1]; char *installedExtensionVersion = NULL; InitializeCaches(); Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum("citus")); SysScanDesc scandesc = systable_beginscan(relation, ExtensionNameIndexId, true, NULL, 1, entry); HeapTuple extensionTuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(extensionTuple)) { int extensionIndex = Anum_pg_extension_extversion; TupleDesc tupleDescriptor = RelationGetDescr(relation); bool isNull = false; Datum installedVersion = heap_getattr(extensionTuple, extensionIndex, tupleDescriptor, &isNull); if (isNull) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension version is null"))); } /* we will cache the result of citus version to prevent catalog access */ MemoryContext oldMemoryContext = MemoryContextSwitchTo( MetadataCacheMemoryContext); installedExtensionVersion = text_to_cstring(DatumGetTextPP(installedVersion)); MemoryContextSwitchTo(oldMemoryContext); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension is not loaded"))); } systable_endscan(scandesc); table_close(relation, AccessShareLock); return installedExtensionVersion; } /* return oid of pg_dist_shard relation */ Oid DistShardRelationId(void) { CachedRelationLookup("pg_dist_shard", &MetadataCache.distShardRelationId); return MetadataCache.distShardRelationId; } /* return oid of pg_dist_placement relation */ Oid DistPlacementRelationId(void) { CachedRelationLookup("pg_dist_placement", &MetadataCache.distPlacementRelationId); return MetadataCache.distPlacementRelationId; } /* return oid of pg_dist_node relation */ Oid DistNodeRelationId(void) { CachedRelationLookup("pg_dist_node", &MetadataCache.distNodeRelationId); return MetadataCache.distNodeRelationId; } /* return oid of pg_dist_node's primary key index */ Oid DistNodeNodeIdIndexId(void) { CachedRelationLookup("pg_dist_node_pkey", &MetadataCache.distNodeNodeIdIndexId); return MetadataCache.distNodeNodeIdIndexId; } /* return oid of pg_dist_local_group relation */ Oid DistLocalGroupIdRelationId(void) { CachedRelationLookup("pg_dist_local_group", &MetadataCache.distLocalGroupRelationId); return MetadataCache.distLocalGroupRelationId; } Oid DistBackgroundJobRelationId(void) { CachedRelationLookup("pg_dist_background_job", &MetadataCache.distBackgroundJobRelationId); return MetadataCache.distBackgroundJobRelationId; } Oid DistBackgroundJobPKeyIndexId(void) { CachedRelationLookup("pg_dist_background_job_pkey", &MetadataCache.distBackgroundJobPKeyIndexId); return MetadataCache.distBackgroundJobPKeyIndexId; } Oid DistBackgroundJobJobIdSequenceId(void) { CachedRelationLookup("pg_dist_background_job_job_id_seq", &MetadataCache.distBackgroundJobJobIdSequenceId); return MetadataCache.distBackgroundJobJobIdSequenceId; } Oid DistBackgroundTaskRelationId(void) { CachedRelationLookup("pg_dist_background_task", &MetadataCache.distBackgroundTaskRelationId); return MetadataCache.distBackgroundTaskRelationId; } Oid DistBackgroundTaskPKeyIndexId(void) { CachedRelationLookup("pg_dist_background_task_pkey", &MetadataCache.distBackgroundTaskPKeyIndexId); return MetadataCache.distBackgroundTaskPKeyIndexId; } Oid DistBackgroundTaskJobIdTaskIdIndexId(void) { CachedRelationLookup("pg_dist_background_task_job_id_task_id", &MetadataCache.distBackgroundTaskJobIdTaskIdIndexId); return MetadataCache.distBackgroundTaskJobIdTaskIdIndexId; } Oid DistBackgroundTaskStatusTaskIdIndexId(void) { CachedRelationLookup("pg_dist_background_task_status_task_id_index", &MetadataCache.distBackgroundTaskStatusTaskIdIndexId); return MetadataCache.distBackgroundTaskStatusTaskIdIndexId; } Oid DistBackgroundTaskTaskIdSequenceId(void) { CachedRelationLookup("pg_dist_background_task_task_id_seq", &MetadataCache.distBackgroundTaskTaskIdSequenceId); return MetadataCache.distBackgroundTaskTaskIdSequenceId; } Oid DistClockLogicalSequenceId(void) { CachedRelationLookup("pg_dist_clock_logical_seq", &MetadataCache.distClockLogicalSequenceId); return MetadataCache.distClockLogicalSequenceId; } Oid DistBackgroundTaskDependRelationId(void) { CachedRelationLookup("pg_dist_background_task_depend", &MetadataCache.distBackgroundTaskDependRelationId); return MetadataCache.distBackgroundTaskDependRelationId; } Oid DistBackgroundTaskDependTaskIdIndexId(void) { CachedRelationLookup("pg_dist_background_task_depend_task_id", &MetadataCache.distBackgroundTaskDependTaskIdIndexId); return MetadataCache.distBackgroundTaskDependTaskIdIndexId; } Oid DistBackgroundTaskDependDependsOnIndexId(void) { CachedRelationLookup("pg_dist_background_task_depend_depends_on", &MetadataCache.distBackgroundTaskDependDependsOnIndexId); return MetadataCache.distBackgroundTaskDependDependsOnIndexId; } /* return oid of pg_dist_rebalance_strategy relation */ Oid DistRebalanceStrategyRelationId(void) { CachedRelationLookup("pg_dist_rebalance_strategy", &MetadataCache.distRebalanceStrategyRelationId); return MetadataCache.distRebalanceStrategyRelationId; } /* return the oid of citus namespace */ Oid CitusCatalogNamespaceId(void) { CachedNamespaceLookup("citus", &MetadataCache.citusCatalogNamespaceId); return MetadataCache.citusCatalogNamespaceId; } /* return oid of pg_dist_object relation */ Oid DistObjectRelationId(void) { /* * In older versions pg_dist_object was living in the `citus` namespace, With Citus 11 * this has been moved to pg_dist_catalog. * * During upgrades it could therefore be that we simply need to look in the old * catalog. Since we expect to find it most of the time in the pg_catalog schema from * now on we will start there. * * even after the table has been moved, the oid's stay the same, so we don't have to * invalidate the cache after a move * * Note: during testing we also up/downgrade the extension, and sometimes interact * with the database when the schema and the binary are not in sync. Hance we always * allow the catalog to be missing on our first lookup. The error message might * therefore become misleading as it will complain about citus.pg_dist_object not * being found when called too early. */ CachedRelationLookupExtended("pg_dist_object", &MetadataCache.distObjectRelationId, true); if (!OidIsValid(MetadataCache.distObjectRelationId)) { /* * We can only ever reach here while we are creating/altering our extension before * the table is moved to pg_catalog. */ CachedRelationNamespaceLookupExtended("pg_dist_object", CitusCatalogNamespaceId(), &MetadataCache.distObjectRelationId, false); } return MetadataCache.distObjectRelationId; } /* return oid of pg_dist_object_pkey */ Oid DistObjectPrimaryKeyIndexId(void) { /* * In older versions pg_dist_object was living in the `citus` namespace, With Citus 11 * this has been moved to pg_dist_catalog. * * During upgrades it could therefore be that we simply need to look in the old * catalog. Since we expect to find it most of the time in the pg_catalog schema from * now on we will start there. * * even after the table has been moved, the oid's stay the same, so we don't have to * invalidate the cache after a move * * Note: during testing we also up/downgrade the extension, and sometimes interact * with the database when the schema and the binary are not in sync. Hance we always * allow the catalog to be missing on our first lookup. The error message might * therefore become misleading as it will complain about citus.pg_dist_object not * being found when called too early. */ CachedRelationLookupExtended("pg_dist_object_pkey", &MetadataCache.distObjectPrimaryKeyIndexId, true); if (!OidIsValid(MetadataCache.distObjectPrimaryKeyIndexId)) { /* * We can only ever reach here while we are creating/altering our extension before * the table is moved to pg_catalog. */ CachedRelationNamespaceLookupExtended("pg_dist_object_pkey", CitusCatalogNamespaceId(), &MetadataCache.distObjectPrimaryKeyIndexId, false); } return MetadataCache.distObjectPrimaryKeyIndexId; } /* return oid of pg_dist_cleanup relation */ Oid DistCleanupRelationId(void) { CachedRelationLookup("pg_dist_cleanup", &MetadataCache.distCleanupRelationId); return MetadataCache.distCleanupRelationId; } /* return oid of pg_dist_cleanup primary key index */ Oid DistCleanupPrimaryKeyIndexId(void) { CachedRelationLookup("pg_dist_cleanup_pkey", &MetadataCache.distCleanupPrimaryKeyIndexId); return MetadataCache.distCleanupPrimaryKeyIndexId; } /* return oid of pg_dist_colocation relation */ Oid DistColocationRelationId(void) { CachedRelationLookup("pg_dist_colocation", &MetadataCache.distColocationRelationId); return MetadataCache.distColocationRelationId; } /* return oid of pg_dist_colocation_configuration_index index */ Oid DistColocationConfigurationIndexId(void) { CachedRelationLookup("pg_dist_colocation_configuration_index", &MetadataCache.distColocationConfigurationIndexId); return MetadataCache.distColocationConfigurationIndexId; } /* return oid of pg_dist_schema relation */ Oid DistTenantSchemaRelationId(void) { CachedRelationLookup("pg_dist_schema", &MetadataCache.distTenantSchemaRelationId); return MetadataCache.distTenantSchemaRelationId; } /* return oid of pg_dist_schema_pkey index */ Oid DistTenantSchemaPrimaryKeyIndexId(void) { CachedRelationLookup("pg_dist_schema_pkey", &MetadataCache.distTenantSchemaPrimaryKeyIndexId); return MetadataCache.distTenantSchemaPrimaryKeyIndexId; } /* return oid of pg_dist_schema_unique_colocationid_index index */ Oid DistTenantSchemaUniqueColocationIdIndexId(void) { CachedRelationLookup("pg_dist_schema_unique_colocationid_index", &MetadataCache.distTenantSchemaUniqueColocationIdIndexId); return MetadataCache.distTenantSchemaUniqueColocationIdIndexId; } /* return oid of pg_dist_partition relation */ Oid DistPartitionRelationId(void) { CachedRelationLookup("pg_dist_partition", &MetadataCache.distPartitionRelationId); return MetadataCache.distPartitionRelationId; } /* return oid of pg_dist_partition_logical_relid_index index */ Oid DistPartitionLogicalRelidIndexId(void) { CachedRelationLookup("pg_dist_partition_logical_relid_index", &MetadataCache.distPartitionLogicalRelidIndexId); return MetadataCache.distPartitionLogicalRelidIndexId; } /* return oid of pg_dist_partition_colocationid_index index */ Oid DistPartitionColocationidIndexId(void) { CachedRelationLookup("pg_dist_partition_colocationid_index", &MetadataCache.distPartitionColocationidIndexId); return MetadataCache.distPartitionColocationidIndexId; } /* return oid of pg_dist_shard_logical_relid_index index */ Oid DistShardLogicalRelidIndexId(void) { CachedRelationLookup("pg_dist_shard_logical_relid_index", &MetadataCache.distShardLogicalRelidIndexId); return MetadataCache.distShardLogicalRelidIndexId; } /* return oid of pg_dist_shard_shardid_index index */ Oid DistShardShardidIndexId(void) { CachedRelationLookup("pg_dist_shard_shardid_index", &MetadataCache.distShardShardidIndexId); return MetadataCache.distShardShardidIndexId; } /* return oid of pg_dist_placement_shardid_index */ Oid DistPlacementShardidIndexId(void) { CachedRelationLookup("pg_dist_placement_shardid_index", &MetadataCache.distPlacementShardidIndexId); return MetadataCache.distPlacementShardidIndexId; } /* return oid of pg_dist_placement_placementid_index */ Oid DistPlacementPlacementidIndexId(void) { CachedRelationLookup("pg_dist_placement_placementid_index", &MetadataCache.distPlacementPlacementidIndexId); return MetadataCache.distPlacementPlacementidIndexId; } /* return oid of pg_dist_colocation_pkey */ Oid DistColocationIndexId(void) { CachedRelationLookup("pg_dist_colocation_pkey", &MetadataCache.distColocationidIndexId); return MetadataCache.distColocationidIndexId; } /* return oid of pg_dist_transaction relation */ Oid DistTransactionRelationId(void) { CachedRelationLookup("pg_dist_transaction", &MetadataCache.distTransactionRelationId); return MetadataCache.distTransactionRelationId; } /* return oid of pg_dist_transaction_group_index */ Oid DistTransactionGroupIndexId(void) { CachedRelationLookup("pg_dist_transaction_group_index", &MetadataCache.distTransactionGroupIndexId); return MetadataCache.distTransactionGroupIndexId; } /* return oid of pg_dist_placement_groupid_index */ Oid DistPlacementGroupidIndexId(void) { CachedRelationLookup("pg_dist_placement_groupid_index", &MetadataCache.distPlacementGroupidIndexId); return MetadataCache.distPlacementGroupidIndexId; } /* return oid of pg_dist_authinfo relation */ static Oid DistAuthinfoRelationId(void) { CachedRelationLookup("pg_dist_authinfo", &MetadataCache.distAuthinfoRelationId); return MetadataCache.distAuthinfoRelationId; } /* return oid of pg_dist_authinfo identification index */ static Oid DistAuthinfoIndexId(void) { CachedRelationLookup("pg_dist_authinfo_identification_index", &MetadataCache.distAuthinfoIndexId); return MetadataCache.distAuthinfoIndexId; } /* return oid of pg_dist_poolinfo relation */ static Oid DistPoolinfoRelationId(void) { CachedRelationLookup("pg_dist_poolinfo", &MetadataCache.distPoolinfoRelationId); return MetadataCache.distPoolinfoRelationId; } /* return oid of pg_dist_poolinfo primary key index */ static Oid DistPoolinfoIndexId(void) { CachedRelationLookup("pg_dist_poolinfo_pkey", &MetadataCache.distPoolinfoIndexId); return MetadataCache.distPoolinfoIndexId; } /* return oid of the read_intermediate_result(text,citus_copy_format) function */ Oid CitusReadIntermediateResultFuncId(void) { if (MetadataCache.readIntermediateResultFuncId == InvalidOid) { List *functionNameList = list_make2(makeString("pg_catalog"), makeString("read_intermediate_result")); Oid copyFormatTypeOid = CitusCopyFormatTypeId(); Oid paramOids[2] = { TEXTOID, copyFormatTypeOid }; bool missingOK = false; MetadataCache.readIntermediateResultFuncId = LookupFuncName(functionNameList, 2, paramOids, missingOK); } return MetadataCache.readIntermediateResultFuncId; } /* return oid of the read_intermediate_results(text[],citus_copy_format) function */ Oid CitusReadIntermediateResultArrayFuncId(void) { if (MetadataCache.readIntermediateResultArrayFuncId == InvalidOid) { List *functionNameList = list_make2(makeString("pg_catalog"), makeString("read_intermediate_results")); Oid copyFormatTypeOid = CitusCopyFormatTypeId(); Oid paramOids[2] = { TEXTARRAYOID, copyFormatTypeOid }; bool missingOK = false; MetadataCache.readIntermediateResultArrayFuncId = LookupFuncName(functionNameList, 2, paramOids, missingOK); } return MetadataCache.readIntermediateResultArrayFuncId; } /* return oid of the citus.copy_format enum type */ Oid CitusCopyFormatTypeId(void) { if (MetadataCache.copyFormatTypeId == InvalidOid) { char *typeName = "citus_copy_format"; MetadataCache.copyFormatTypeId = GetSysCacheOid2(TYPENAMENSP, Anum_pg_enum_oid, PointerGetDatum(typeName), PG_CATALOG_NAMESPACE); } return MetadataCache.copyFormatTypeId; } /* return oid of the 'binary' citus_copy_format enum value */ Oid BinaryCopyFormatId(void) { if (MetadataCache.binaryCopyFormatId == InvalidOid) { Oid copyFormatTypeId = CitusCopyFormatTypeId(); MetadataCache.binaryCopyFormatId = LookupEnumValueId(copyFormatTypeId, "binary"); } return MetadataCache.binaryCopyFormatId; } /* return oid of the 'text' citus_copy_format enum value */ Oid TextCopyFormatId(void) { if (MetadataCache.textCopyFormatId == InvalidOid) { Oid copyFormatTypeId = CitusCopyFormatTypeId(); MetadataCache.textCopyFormatId = LookupEnumValueId(copyFormatTypeId, "text"); } return MetadataCache.textCopyFormatId; } /* return oid of the citus_extradata_container(internal) function */ Oid CitusExtraDataContainerFuncId(void) { List *nameList = NIL; Oid paramOids[1] = { INTERNALOID }; if (MetadataCache.extraDataContainerFuncId == InvalidOid) { nameList = list_make2(makeString("pg_catalog"), makeString("citus_extradata_container")); MetadataCache.extraDataContainerFuncId = LookupFuncName(nameList, 1, paramOids, false); } return MetadataCache.extraDataContainerFuncId; } /* return oid of the any_value aggregate function */ Oid CitusAnyValueFunctionId(void) { if (MetadataCache.anyValueFunctionId == InvalidOid) { const int argCount = 1; MetadataCache.anyValueFunctionId = FunctionOid("pg_catalog", "any_value", argCount); } return MetadataCache.anyValueFunctionId; } /* return oid of the citus_text_send_as_jsonb(text) function */ Oid CitusTextSendAsJsonbFunctionId(void) { if (MetadataCache.textSendAsJsonbFunctionId == InvalidOid) { List *nameList = list_make2(makeString("pg_catalog"), makeString("citus_text_send_as_jsonb")); Oid paramOids[1] = { TEXTOID }; MetadataCache.textSendAsJsonbFunctionId = LookupFuncName(nameList, 1, paramOids, false); } return MetadataCache.textSendAsJsonbFunctionId; } /* return oid of the textout(text) function */ Oid TextOutFunctionId(void) { if (MetadataCache.textoutFunctionId == InvalidOid) { List *nameList = list_make2(makeString("pg_catalog"), makeString("textout")); Oid paramOids[1] = { TEXTOID }; MetadataCache.textoutFunctionId = LookupFuncName(nameList, 1, paramOids, false); } return MetadataCache.textoutFunctionId; } /* * RelationIsAKnownShardFuncId returns oid of the relation_is_a_known_shard function. */ Oid RelationIsAKnownShardFuncId(void) { if (MetadataCache.relationIsAKnownShardFuncId == InvalidOid) { const int argCount = 1; MetadataCache.relationIsAKnownShardFuncId = FunctionOid("pg_catalog", "relation_is_a_known_shard", argCount); } return MetadataCache.relationIsAKnownShardFuncId; } /* * JsonbExtractPathFuncId returns oid of the jsonb_extract_path function. */ Oid JsonbExtractPathFuncId(void) { if (MetadataCache.jsonbExtractPathFuncId == InvalidOid) { const int argCount = 2; MetadataCache.jsonbExtractPathFuncId = FunctionOid("pg_catalog", "jsonb_extract_path", argCount); } return MetadataCache.jsonbExtractPathFuncId; } /* * JsonbExtractPathTextFuncId returns oid of the jsonb_extract_path_text function. */ Oid JsonbExtractPathTextFuncId(void) { if (MetadataCache.jsonbExtractPathTextFuncId == InvalidOid) { const int argCount = 2; MetadataCache.jsonbExtractPathTextFuncId = FunctionOid("pg_catalog", "jsonb_extract_path_text", argCount); } return MetadataCache.jsonbExtractPathTextFuncId; } /* * CitusDependentObjectFuncId returns oid of the is_citus_depended_object function. */ Oid CitusDependentObjectFuncId(void) { if (!HideCitusDependentObjects) { ereport(ERROR, (errmsg( "is_citus_depended_object can only be used while running the regression tests"))); } if (MetadataCache.CitusDependentObjectFuncId == InvalidOid) { const int argCount = 2; MetadataCache.CitusDependentObjectFuncId = FunctionOid("pg_catalog", "is_citus_depended_object", argCount); } return MetadataCache.CitusDependentObjectFuncId; } /* * CurrentDatabaseName gets the name of the current database and caches * the result. * * Given that the database name cannot be changed when there is at least * one session connected to it, we do not need to implement any invalidation * mechanism. */ const char * CurrentDatabaseName(void) { if (!MetadataCache.databaseNameValid) { char *databaseName = get_database_name(MyDatabaseId); if (databaseName == NULL) { ereport(ERROR, (errmsg("database that is connected to does not exist"))); } strlcpy(MetadataCache.databaseName, databaseName, NAMEDATALEN); MetadataCache.databaseNameValid = true; } return MetadataCache.databaseName; } /* * CitusExtensionOwner() returns the owner of the 'citus' extension. That user * is, amongst others, used to perform actions a normal user might not be * allowed to perform. */ extern Oid CitusExtensionOwner(void) { ScanKeyData entry[1]; Form_pg_extension extensionForm = NULL; if (MetadataCache.extensionOwner != InvalidOid) { return MetadataCache.extensionOwner; } Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum("citus")); SysScanDesc scandesc = systable_beginscan(relation, ExtensionNameIndexId, true, NULL, 1, entry); HeapTuple extensionTuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(extensionTuple)) { extensionForm = (Form_pg_extension) GETSTRUCT(extensionTuple); /* * For some operations Citus requires superuser permissions; we use * the extension owner for that. The extension owner is guaranteed to * be a superuser (otherwise C functions can't be created), but it'd * be possible to change the owner. So check that this still a * superuser. */ if (!superuser_arg(extensionForm->extowner)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension needs to be owned by superuser"))); } MetadataCache.extensionOwner = extensionForm->extowner; Assert(OidIsValid(MetadataCache.extensionOwner)); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus extension not loaded"))); } systable_endscan(scandesc); table_close(relation, AccessShareLock); return MetadataCache.extensionOwner; } /* * CitusExtensionOwnerName returns the name of the owner of the extension. */ char * CitusExtensionOwnerName(void) { Oid superUserId = CitusExtensionOwner(); return GetUserNameFromId(superUserId, false); } /* return the username of the currently active role */ char * CurrentUserName(void) { Oid userId = GetUserId(); return GetUserNameFromId(userId, false); } /* * LookupTypeOid returns the Oid of the "{schemaNameSting}.{typeNameString}" type, or * InvalidOid if it does not exist. */ Oid LookupTypeOid(char *schemaNameSting, char *typeNameString) { String *schemaName = makeString(schemaNameSting); String *typeName = makeString(typeNameString); List *qualifiedName = list_make2(schemaName, typeName); TypeName *enumTypeName = makeTypeNameFromNameList(qualifiedName); /* typenameTypeId but instead of raising an error return InvalidOid */ Type tup = LookupTypeName(NULL, enumTypeName, NULL, false); if (tup == NULL) { return InvalidOid; } Oid nodeRoleTypId = ((Form_pg_type) GETSTRUCT(tup))->oid; ReleaseSysCache(tup); return nodeRoleTypId; } /* * LookupStringEnumValueId returns the Oid of the value in "pg_catalog.{enumName}" * which matches the provided valueName, or InvalidOid if the enum doesn't exist yet. */ static Oid LookupStringEnumValueId(char *enumName, char *valueName) { Oid enumTypeId = LookupTypeOid("pg_catalog", enumName); if (enumTypeId == InvalidOid) { return InvalidOid; } else { Oid valueId = LookupEnumValueId(enumTypeId, valueName); return valueId; } } /* * LookupEnumValueId looks up the OID of an enum value. */ static Oid LookupEnumValueId(Oid typeId, char *valueName) { Datum typeIdDatum = ObjectIdGetDatum(typeId); Datum valueDatum = CStringGetDatum(valueName); Datum valueIdDatum = DirectFunctionCall2(enum_in, valueDatum, typeIdDatum); Oid valueId = DatumGetObjectId(valueIdDatum); return valueId; } /* return the Oid of the 'primary' nodeRole enum value */ Oid PrimaryNodeRoleId(void) { if (!MetadataCache.primaryNodeRoleId) { MetadataCache.primaryNodeRoleId = LookupStringEnumValueId("noderole", "primary"); } return MetadataCache.primaryNodeRoleId; } /* return the Oid of the 'secodary' nodeRole enum value */ Oid SecondaryNodeRoleId(void) { if (!MetadataCache.secondaryNodeRoleId) { MetadataCache.secondaryNodeRoleId = LookupStringEnumValueId("noderole", "secondary"); } return MetadataCache.secondaryNodeRoleId; } /* return the Oid of the 'unavailable' nodeRole enum value */ Oid UnavailableNodeRoleId(void) { if (!MetadataCache.unavailableNodeRoleId) { MetadataCache.unavailableNodeRoleId = LookupStringEnumValueId("noderole", "unavailable"); } return MetadataCache.unavailableNodeRoleId; } Oid CitusJobStatusScheduledId(void) { if (!MetadataCache.citusJobStatusScheduledId) { MetadataCache.citusJobStatusScheduledId = LookupStringEnumValueId("citus_job_status", "scheduled"); } return MetadataCache.citusJobStatusScheduledId; } Oid CitusJobStatusRunningId(void) { if (!MetadataCache.citusJobStatusRunningId) { MetadataCache.citusJobStatusRunningId = LookupStringEnumValueId("citus_job_status", "running"); } return MetadataCache.citusJobStatusRunningId; } Oid CitusJobStatusCancellingId(void) { if (!MetadataCache.citusJobStatusCancellingId) { MetadataCache.citusJobStatusCancellingId = LookupStringEnumValueId("citus_job_status", "cancelling"); } return MetadataCache.citusJobStatusCancellingId; } Oid CitusJobStatusFinishedId(void) { if (!MetadataCache.citusJobStatusFinishedId) { MetadataCache.citusJobStatusFinishedId = LookupStringEnumValueId("citus_job_status", "finished"); } return MetadataCache.citusJobStatusFinishedId; } Oid CitusJobStatusCancelledId(void) { if (!MetadataCache.citusJobStatusCancelledId) { MetadataCache.citusJobStatusCancelledId = LookupStringEnumValueId("citus_job_status", "cancelled"); } return MetadataCache.citusJobStatusCancelledId; } Oid CitusJobStatusFailedId(void) { if (!MetadataCache.citusJobStatusFailedId) { MetadataCache.citusJobStatusFailedId = LookupStringEnumValueId("citus_job_status", "failed"); } return MetadataCache.citusJobStatusFailedId; } Oid CitusJobStatusFailingId(void) { if (!MetadataCache.citusJobStatusFailingId) { MetadataCache.citusJobStatusFailingId = LookupStringEnumValueId("citus_job_status", "failing"); } return MetadataCache.citusJobStatusFailingId; } Oid CitusTaskStatusBlockedId(void) { if (!MetadataCache.citusTaskStatusBlockedId) { MetadataCache.citusTaskStatusBlockedId = LookupStringEnumValueId("citus_task_status", "blocked"); } return MetadataCache.citusTaskStatusBlockedId; } Oid CitusTaskStatusCancelledId(void) { if (!MetadataCache.citusTaskStatusCancelledId) { MetadataCache.citusTaskStatusCancelledId = LookupStringEnumValueId("citus_task_status", "cancelled"); } return MetadataCache.citusTaskStatusCancelledId; } Oid CitusTaskStatusCancellingId(void) { if (!MetadataCache.citusTaskStatusCancellingId) { MetadataCache.citusTaskStatusCancellingId = LookupStringEnumValueId("citus_task_status", "cancelling"); } return MetadataCache.citusTaskStatusCancellingId; } Oid CitusTaskStatusRunnableId(void) { if (!MetadataCache.citusTaskStatusRunnableId) { MetadataCache.citusTaskStatusRunnableId = LookupStringEnumValueId("citus_task_status", "runnable"); } return MetadataCache.citusTaskStatusRunnableId; } Oid CitusTaskStatusRunningId(void) { if (!MetadataCache.citusTaskStatusRunningId) { MetadataCache.citusTaskStatusRunningId = LookupStringEnumValueId("citus_task_status", "running"); } return MetadataCache.citusTaskStatusRunningId; } Oid CitusTaskStatusDoneId(void) { if (!MetadataCache.citusTaskStatusDoneId) { MetadataCache.citusTaskStatusDoneId = LookupStringEnumValueId("citus_task_status", "done"); } return MetadataCache.citusTaskStatusDoneId; } Oid CitusTaskStatusErrorId(void) { if (!MetadataCache.citusTaskStatusErrorId) { MetadataCache.citusTaskStatusErrorId = LookupStringEnumValueId("citus_task_status", "error"); } return MetadataCache.citusTaskStatusErrorId; } Oid CitusTaskStatusUnscheduledId(void) { if (!MetadataCache.citusTaskStatusUnscheduledId) { MetadataCache.citusTaskStatusUnscheduledId = LookupStringEnumValueId("citus_task_status", "unscheduled"); } return MetadataCache.citusTaskStatusUnscheduledId; } /* * citus_dist_partition_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_partition are changed * on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_partition_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TriggerData *triggerData = (TriggerData *) fcinfo->context; Oid oldLogicalRelationId = InvalidOid; Oid newLogicalRelationId = InvalidOid; if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } if (RelationGetRelid(triggerData->tg_relation) != DistPartitionRelationId()) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("triggered on incorrect relation"))); } HeapTuple newTuple = triggerData->tg_newtuple; HeapTuple oldTuple = triggerData->tg_trigtuple; /* collect logicalrelid for OLD and NEW tuple */ if (oldTuple != NULL) { Form_pg_dist_partition distPart = (Form_pg_dist_partition) GETSTRUCT(oldTuple); oldLogicalRelationId = distPart->logicalrelid; } if (newTuple != NULL) { Form_pg_dist_partition distPart = (Form_pg_dist_partition) GETSTRUCT(newTuple); newLogicalRelationId = distPart->logicalrelid; } /* * Invalidate relcache for the relevant relation(s). In theory * logicalrelid should never change, but it doesn't hurt to be * paranoid. */ if (oldLogicalRelationId != InvalidOid && oldLogicalRelationId != newLogicalRelationId) { CitusInvalidateRelcacheByRelid(oldLogicalRelationId); } if (newLogicalRelationId != InvalidOid) { CitusInvalidateRelcacheByRelid(newLogicalRelationId); } PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_partition_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_partition_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_partition_cache_invalidate(fcinfo); } /* * citus_dist_shard_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_shard are changed * on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_shard_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TriggerData *triggerData = (TriggerData *) fcinfo->context; Oid oldLogicalRelationId = InvalidOid; Oid newLogicalRelationId = InvalidOid; if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } if (RelationGetRelid(triggerData->tg_relation) != DistShardRelationId()) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("triggered on incorrect relation"))); } HeapTuple newTuple = triggerData->tg_newtuple; HeapTuple oldTuple = triggerData->tg_trigtuple; /* collect logicalrelid for OLD and NEW tuple */ if (oldTuple != NULL) { Form_pg_dist_shard distShard = (Form_pg_dist_shard) GETSTRUCT(oldTuple); oldLogicalRelationId = distShard->logicalrelid; } if (newTuple != NULL) { Form_pg_dist_shard distShard = (Form_pg_dist_shard) GETSTRUCT(newTuple); newLogicalRelationId = distShard->logicalrelid; } /* * Invalidate relcache for the relevant relation(s). In theory * logicalrelid should never change, but it doesn't hurt to be * paranoid. */ if (oldLogicalRelationId != InvalidOid && oldLogicalRelationId != newLogicalRelationId) { CitusInvalidateRelcacheByRelid(oldLogicalRelationId); } if (newLogicalRelationId != InvalidOid) { CitusInvalidateRelcacheByRelid(newLogicalRelationId); } PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_shard_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_shard_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_shard_cache_invalidate(fcinfo); } /* * citus_dist_placement_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_placement are * changed on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_placement_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TriggerData *triggerData = (TriggerData *) fcinfo->context; Oid oldShardId = InvalidOid; Oid newShardId = InvalidOid; if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } /* * Before 7.0-2 this trigger is on pg_dist_shard_placement, * ignore trigger in this scenario. */ Oid pgDistShardPlacementId = get_relname_relid("pg_dist_shard_placement", PG_CATALOG_NAMESPACE); if (RelationGetRelid(triggerData->tg_relation) == pgDistShardPlacementId) { PG_RETURN_DATUM(PointerGetDatum(NULL)); } if (RelationGetRelid(triggerData->tg_relation) != DistPlacementRelationId()) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("triggered on incorrect relation"))); } HeapTuple newTuple = triggerData->tg_newtuple; HeapTuple oldTuple = triggerData->tg_trigtuple; /* collect shardid for OLD and NEW tuple */ if (oldTuple != NULL) { Form_pg_dist_placement distPlacement = (Form_pg_dist_placement) GETSTRUCT(oldTuple); oldShardId = distPlacement->shardid; } if (newTuple != NULL) { Form_pg_dist_placement distPlacement = (Form_pg_dist_placement) GETSTRUCT(newTuple); newShardId = distPlacement->shardid; } /* * Invalidate relcache for the relevant relation(s). In theory shardId * should never change, but it doesn't hurt to be paranoid. */ if (oldShardId != InvalidOid && oldShardId != newShardId) { CitusInvalidateRelcacheByShardId(oldShardId); } if (newShardId != InvalidOid) { CitusInvalidateRelcacheByShardId(newShardId); } PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_placement_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_placement_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_placement_cache_invalidate(fcinfo); } /* * citus_dist_node_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_node are changed * on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_node_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } CitusInvalidateRelcacheByRelid(DistNodeRelationId()); PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_node_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_node_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_node_cache_invalidate(fcinfo); } /* * citus_conninfo_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_authinfo are changed * on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_conninfo_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } CitusInvalidateRelcacheByRelid(DistAuthinfoRelationId()); PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_authinfo_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_authinfo_cache_invalidate(PG_FUNCTION_ARGS) { return citus_conninfo_cache_invalidate(fcinfo); } /* * citus_dist_local_group_cache_invalidate is a trigger function that performs * relcache invalidations when the contents of pg_dist_local_group are changed * on the SQL level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_local_group_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } CitusInvalidateRelcacheByRelid(DistLocalGroupIdRelationId()); PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_local_group_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_local_group_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_local_group_cache_invalidate(fcinfo); } /* * citus_dist_object_cache_invalidate is a trigger function that performs relcache * invalidation when the contents of pg_dist_object are changed on the SQL * level. * * NB: We decided there is little point in checking permissions here, there * are much easier ways to waste CPU than causing cache invalidations. */ Datum citus_dist_object_cache_invalidate(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (!CALLED_AS_TRIGGER(fcinfo)) { ereport(ERROR, (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED), errmsg("must be called as trigger"))); } CitusInvalidateRelcacheByRelid(DistObjectRelationId()); PG_RETURN_DATUM(PointerGetDatum(NULL)); } /* * master_dist_object_cache_invalidate is a wrapper function for old UDF name. */ Datum master_dist_object_cache_invalidate(PG_FUNCTION_ARGS) { return citus_dist_object_cache_invalidate(fcinfo); } /* * InitializeCaches() registers invalidation handlers for metadata_cache.c's * caches. */ static void InitializeCaches(void) { static bool performedInitialization = false; if (!performedInitialization) { MetadataCacheMemoryContext = NULL; /* * If either of dist table cache or shard cache * allocation and initializations fail due to an exception * that is caused by OOM or any other reason, * we reset the flag, and delete the shard cache memory * context to reclaim partially allocated memory. * * Command will continue to fail since we re-throw the exception. */ PG_TRY(); { /* set first, to avoid recursion dangers */ performedInitialization = true; /* make sure we've initialized CacheMemoryContext */ if (CacheMemoryContext == NULL) { CreateCacheMemoryContext(); } MetadataCacheMemoryContext = AllocSetContextCreate( CacheMemoryContext, "MetadataCacheMemoryContext", ALLOCSET_DEFAULT_SIZES); InitializeDistCache(); RegisterForeignKeyGraphCacheCallbacks(); RegisterWorkerNodeCacheCallbacks(); RegisterLocalGroupIdCacheCallbacks(); RegisterAuthinfoCacheCallbacks(); RegisterCitusTableCacheEntryReleaseCallbacks(); } PG_CATCH(); { performedInitialization = false; if (MetadataCacheMemoryContext != NULL) { MemoryContextDelete(MetadataCacheMemoryContext); } MetadataCacheMemoryContext = NULL; DistTableCacheHash = NULL; DistTableCacheExpired = NIL; ShardIdCacheHash = NULL; PG_RE_THROW(); } PG_END_TRY(); } } /* initialize the infrastructure for the metadata cache */ static void InitializeDistCache(void) { /* build initial scan keys, copied for every relation scan */ memset(&DistPartitionScanKey, 0, sizeof(DistPartitionScanKey)); fmgr_info_cxt(F_OIDEQ, &DistPartitionScanKey[0].sk_func, MetadataCacheMemoryContext); DistPartitionScanKey[0].sk_strategy = BTEqualStrategyNumber; DistPartitionScanKey[0].sk_subtype = InvalidOid; DistPartitionScanKey[0].sk_collation = InvalidOid; DistPartitionScanKey[0].sk_attno = Anum_pg_dist_partition_logicalrelid; memset(&DistShardScanKey, 0, sizeof(DistShardScanKey)); fmgr_info_cxt(F_OIDEQ, &DistShardScanKey[0].sk_func, MetadataCacheMemoryContext); DistShardScanKey[0].sk_strategy = BTEqualStrategyNumber; DistShardScanKey[0].sk_subtype = InvalidOid; DistShardScanKey[0].sk_collation = InvalidOid; DistShardScanKey[0].sk_attno = Anum_pg_dist_shard_logicalrelid; CreateDistTableCache(); CreateShardIdCache(); InitializeDistObjectCache(); } static void InitializeDistObjectCache(void) { /* build initial scan keys, copied for every relation scan */ memset(&DistObjectScanKey, 0, sizeof(DistObjectScanKey)); fmgr_info_cxt(F_OIDEQ, &DistObjectScanKey[0].sk_func, MetadataCacheMemoryContext); DistObjectScanKey[0].sk_strategy = BTEqualStrategyNumber; DistObjectScanKey[0].sk_subtype = InvalidOid; DistObjectScanKey[0].sk_collation = InvalidOid; DistObjectScanKey[0].sk_attno = Anum_pg_dist_object_classid; fmgr_info_cxt(F_OIDEQ, &DistObjectScanKey[1].sk_func, MetadataCacheMemoryContext); DistObjectScanKey[1].sk_strategy = BTEqualStrategyNumber; DistObjectScanKey[1].sk_subtype = InvalidOid; DistObjectScanKey[1].sk_collation = InvalidOid; DistObjectScanKey[1].sk_attno = Anum_pg_dist_object_objid; fmgr_info_cxt(F_INT4EQ, &DistObjectScanKey[2].sk_func, MetadataCacheMemoryContext); DistObjectScanKey[2].sk_strategy = BTEqualStrategyNumber; DistObjectScanKey[2].sk_subtype = InvalidOid; DistObjectScanKey[2].sk_collation = InvalidOid; DistObjectScanKey[2].sk_attno = Anum_pg_dist_object_objsubid; CreateDistObjectCache(); } /* * GetWorkerNodeHash returns the worker node data as a hash with the nodename and * nodeport as a key. * * The hash is returned from the cache, if the cache is not (yet) valid, it is first * rebuilt. */ HTAB * GetWorkerNodeHash(void) { PrepareWorkerNodeCache(); return WorkerNodeHash; } /* * PrepareWorkerNodeCache makes sure the worker node data from pg_dist_node is cached, * if it is not already cached. */ static void PrepareWorkerNodeCache(void) { InitializeCaches(); /* ensure relevant callbacks are registered */ /* * Simulate a SELECT from pg_dist_node, ensure pg_dist_node doesn't change while our * caller is using WorkerNodeHash. */ LockRelationOid(DistNodeRelationId(), AccessShareLock); /* * We might have some concurrent metadata changes. In order to get the changes, * we first need to accept the cache invalidation messages. */ AcceptInvalidationMessages(); if (!workerNodeHashValid) { InitializeWorkerNodeCache(); workerNodeHashValid = true; } } /* * InitializeWorkerNodeCache initialize the infrastructure for the worker node cache. * The function reads the worker nodes from the metadata table, adds them to the hash and * finally registers an invalidation callback. */ static void InitializeWorkerNodeCache(void) { HASHCTL info; long maxTableSize = (long) MaxWorkerNodesTracked; bool includeNodesFromOtherClusters = false; int workerNodeIndex = 0; InitializeCaches(); /* * Create the hash that holds the worker nodes. The key is the combination of * nodename and nodeport, instead of the unique nodeid because worker nodes are * searched by the nodename and nodeport in every physical plan creation. */ memset(&info, 0, sizeof(info)); info.keysize = sizeof(uint32) + WORKER_LENGTH + sizeof(uint32); info.entrysize = sizeof(WorkerNode); info.hcxt = MetadataCacheMemoryContext; info.hash = WorkerNodeHashCode; info.match = WorkerNodeCompare; int hashFlags = HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE; HTAB *newWorkerNodeHash = hash_create("Worker Node Hash", maxTableSize, &info, hashFlags); /* read the list from pg_dist_node */ List *workerNodeList = ReadDistNode(includeNodesFromOtherClusters); int newWorkerNodeCount = list_length(workerNodeList); WorkerNode **newWorkerNodeArray = MemoryContextAlloc(MetadataCacheMemoryContext, sizeof(WorkerNode *) * newWorkerNodeCount); /* iterate over the worker node list */ WorkerNode *currentNode = NULL; foreach_declared_ptr(currentNode, workerNodeList) { bool handleFound = false; /* search for the worker node in the hash, and then insert the values */ void *hashKey = (void *) currentNode; WorkerNode *workerNode = (WorkerNode *) hash_search(newWorkerNodeHash, hashKey, HASH_ENTER, &handleFound); /* fill the newly allocated workerNode in the cache */ strlcpy(workerNode->workerName, currentNode->workerName, WORKER_LENGTH); workerNode->workerPort = currentNode->workerPort; workerNode->groupId = currentNode->groupId; workerNode->nodeId = currentNode->nodeId; strlcpy(workerNode->workerRack, currentNode->workerRack, WORKER_LENGTH); workerNode->hasMetadata = currentNode->hasMetadata; workerNode->metadataSynced = currentNode->metadataSynced; workerNode->isActive = currentNode->isActive; workerNode->nodeRole = currentNode->nodeRole; workerNode->shouldHaveShards = currentNode->shouldHaveShards; workerNode->nodeprimarynodeid = currentNode->nodeprimarynodeid; workerNode->nodeisclone = currentNode->nodeisclone; strlcpy(workerNode->nodeCluster, currentNode->nodeCluster, NAMEDATALEN); newWorkerNodeArray[workerNodeIndex++] = workerNode; if (handleFound) { ereport(WARNING, (errmsg("multiple lines for worker node: \"%s:%u\"", workerNode->workerName, workerNode->workerPort))); } /* we do not need the currentNode anymore */ pfree(currentNode); } /* now, safe to destroy the old hash */ hash_destroy(WorkerNodeHash); if (WorkerNodeArray != NULL) { pfree(WorkerNodeArray); } WorkerNodeCount = newWorkerNodeCount; WorkerNodeArray = newWorkerNodeArray; WorkerNodeHash = newWorkerNodeHash; } /* * RegisterForeignKeyGraphCacheCallbacks registers callbacks required for * the foreign key graph cache. */ static void RegisterForeignKeyGraphCacheCallbacks(void) { /* Watch for invalidation events. */ CacheRegisterRelcacheCallback(InvalidateForeignRelationGraphCacheCallback, (Datum) 0); } /* * RegisterWorkerNodeCacheCallbacks registers the callbacks required for the * worker node cache. It's separate from InitializeWorkerNodeCache so the * callback can be registered early, before the metadata tables exist. */ static void RegisterWorkerNodeCacheCallbacks(void) { /* Watch for invalidation events. */ CacheRegisterRelcacheCallback(InvalidateNodeRelationCacheCallback, (Datum) 0); } /* * RegisterCitusTableCacheEntryReleaseCallbacks registers callbacks to release * cache entries. Data should be locked by callers to avoid staleness. */ static void RegisterCitusTableCacheEntryReleaseCallbacks(void) { RegisterResourceReleaseCallback(CitusTableCacheEntryReleaseCallback, NULL); } /* * GetLocalGroupId returns the group identifier of the local node. The function * assumes that pg_dist_local_group has exactly one row and has at least one * column. Otherwise, the function errors out. */ int32 GetLocalGroupId(void) { ScanKeyData scanKey[1]; int scanKeyCount = 0; int32 groupId = 0; InitializeCaches(); /* * Already set the group id, no need to read the heap again. */ if (LocalGroupId != -1) { return LocalGroupId; } Oid localGroupTableOid = DistLocalGroupIdRelationId(); if (localGroupTableOid == InvalidOid) { return 0; } Relation pgDistLocalGroupId = table_open(localGroupTableOid, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistLocalGroupId, InvalidOid, false, NULL, scanKeyCount, scanKey); TupleDesc tupleDescriptor = RelationGetDescr(pgDistLocalGroupId); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { bool isNull = false; Datum groupIdDatum = heap_getattr(heapTuple, Anum_pg_dist_local_groupid, tupleDescriptor, &isNull); groupId = DatumGetInt32(groupIdDatum); /* set the local cache variable */ LocalGroupId = groupId; } else { /* * Upgrade is happening. When upgrading postgres, pg_dist_local_group is * temporarily empty before citus_finish_pg_upgrade() finishes execution. */ groupId = GROUP_ID_UPGRADING; } systable_endscan(scanDescriptor); table_close(pgDistLocalGroupId, AccessShareLock); return groupId; } /* * GetNodeId returns the node identifier of the local node. */ int32 GetLocalNodeId(void) { InitializeCaches(); /* * Already set the node id, no need to read the heap again. */ if (LocalNodeId != -1) { return LocalNodeId; } uint32 nodeId = -1; int32 localGroupId = GetLocalGroupId(); bool includeNodesFromOtherClusters = false; List *workerNodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if (workerNode->groupId == localGroupId && workerNode->isActive) { nodeId = workerNode->nodeId; break; } } /* * nodeId is -1 if we cannot find an active node whose group id is * localGroupId in pg_dist_node. */ if (nodeId == -1) { elog(DEBUG4, "there is no active node with group id '%d' on pg_dist_node", localGroupId); /* * This is expected if the coordinator is not added to the metadata. * We'll return GLOBAL_PID_NODE_ID_FOR_NODES_NOT_IN_METADATA for this case and * for all cases so views can function almost normally */ nodeId = GLOBAL_PID_NODE_ID_FOR_NODES_NOT_IN_METADATA; } LocalNodeId = nodeId; return nodeId; } /* * RegisterLocalGroupIdCacheCallbacks registers the callbacks required to * maintain LocalGroupId at a consistent value. It's separate from * GetLocalGroupId so the callback can be registered early, before metadata * tables exist. */ static void RegisterLocalGroupIdCacheCallbacks(void) { /* Watch for invalidation events. */ CacheRegisterRelcacheCallback(InvalidateLocalGroupIdRelationCacheCallback, (Datum) 0); } /* * RegisterAuthinfoCacheCallbacks registers the callbacks required to * maintain cached connection parameters at fresh values. */ static void RegisterAuthinfoCacheCallbacks(void) { /* Watch for invalidation events. */ CacheRegisterRelcacheCallback(InvalidateConnParamsCacheCallback, (Datum) 0); } /* * ResetCitusTableCacheEntry frees any out-of-band memory used by a cache entry, * but does not free the entry itself. */ static void ResetCitusTableCacheEntry(CitusTableCacheEntry *cacheEntry) { if (cacheEntry->partitionKeyString != NULL) { pfree(cacheEntry->partitionKeyString); cacheEntry->partitionKeyString = NULL; } if (cacheEntry->shardIntervalCompareFunction != NULL) { pfree(cacheEntry->shardIntervalCompareFunction); cacheEntry->shardIntervalCompareFunction = NULL; } if (cacheEntry->hashFunction) { pfree(cacheEntry->hashFunction); cacheEntry->hashFunction = NULL; } if (cacheEntry->partitionColumn != NULL) { pfree(cacheEntry->partitionColumn); cacheEntry->partitionColumn = NULL; } if (cacheEntry->shardIntervalArrayLength == 0) { return; } /* clean up ShardIdCacheHash */ RemoveStaleShardIdCacheEntries(cacheEntry); for (int shardIndex = 0; shardIndex < cacheEntry->shardIntervalArrayLength; shardIndex++) { ShardInterval *shardInterval = cacheEntry->sortedShardIntervalArray[shardIndex]; GroupShardPlacement *placementArray = cacheEntry->arrayOfPlacementArrays[shardIndex]; bool valueByVal = shardInterval->valueByVal; /* delete the shard's placements */ if (placementArray != NULL) { pfree(placementArray); } /* delete data pointed to by ShardInterval */ if (!valueByVal) { if (shardInterval->minValueExists) { pfree(DatumGetPointer(shardInterval->minValue)); } if (shardInterval->maxValueExists) { pfree(DatumGetPointer(shardInterval->maxValue)); } } /* and finally the ShardInterval itself */ pfree(shardInterval); } if (cacheEntry->sortedShardIntervalArray) { pfree(cacheEntry->sortedShardIntervalArray); cacheEntry->sortedShardIntervalArray = NULL; } if (cacheEntry->arrayOfPlacementArrayLengths) { pfree(cacheEntry->arrayOfPlacementArrayLengths); cacheEntry->arrayOfPlacementArrayLengths = NULL; } if (cacheEntry->arrayOfPlacementArrays) { pfree(cacheEntry->arrayOfPlacementArrays); cacheEntry->arrayOfPlacementArrays = NULL; } if (cacheEntry->referencedRelationsViaForeignKey) { list_free(cacheEntry->referencedRelationsViaForeignKey); cacheEntry->referencedRelationsViaForeignKey = NIL; } if (cacheEntry->referencingRelationsViaForeignKey) { list_free(cacheEntry->referencingRelationsViaForeignKey); cacheEntry->referencingRelationsViaForeignKey = NIL; } cacheEntry->shardIntervalArrayLength = 0; cacheEntry->hasUninitializedShardInterval = false; cacheEntry->hasUniformHashDistribution = false; cacheEntry->hasOverlappingShardInterval = false; cacheEntry->autoConverted = false; pfree(cacheEntry); } /* * RemoveStaleShardIdCacheEntries removes all shard ID cache entries belonging to the * given table entry. If the shard ID belongs to a different (newer) table entry, * we leave it in place. */ static void RemoveStaleShardIdCacheEntries(CitusTableCacheEntry *invalidatedTableEntry) { int shardIndex = 0; int shardCount = invalidatedTableEntry->shardIntervalArrayLength; for (shardIndex = 0; shardIndex < shardCount; shardIndex++) { ShardInterval *shardInterval = invalidatedTableEntry->sortedShardIntervalArray[shardIndex]; int64 shardId = shardInterval->shardId; bool foundInCache = false; ShardIdCacheEntry *shardIdCacheEntry = hash_search(ShardIdCacheHash, &shardId, HASH_FIND, &foundInCache); if (foundInCache && shardIdCacheEntry->tableEntry == invalidatedTableEntry) { hash_search(ShardIdCacheHash, &shardId, HASH_REMOVE, &foundInCache); } } } /* * InvalidateForeignRelationGraphCacheCallback invalidates the foreign key relation * graph and entire distributed cache entries. */ static void InvalidateForeignRelationGraphCacheCallback(Datum argument, Oid relationId) { if (relationId == MetadataCache.distColocationRelationId) { SetForeignConstraintRelationshipGraphInvalid(); InvalidateDistTableCache(); } } /* * InvalidateForeignKeyGraph is used to invalidate the cached foreign key * graph (see ForeignKeyRelationGraph @ utils/foreign_key_relationship.c). * * To invalidate the foreign key graph, we hack around relcache invalidation * callbacks. Given that there is no metadata table associated with the foreign * key graph cache, we use pg_dist_colocation, which is never invalidated for * other purposes. * * We acknowledge that it is not a very intuitive way of implementing this cache * invalidation, but, seems acceptable for now. If this becomes problematic, we * could try using a magic oid where we're sure that no relation would ever use * that oid. */ void InvalidateForeignKeyGraph(void) { if (!CitusHasBeenLoaded()) { /* * We should not try to invalidate foreign key graph * if citus is not loaded. */ return; } CitusInvalidateRelcacheByRelid(DistColocationRelationId()); /* bump command counter to force invalidation to take effect */ CommandCounterIncrement(); } /* * InvalidateDistRelationCacheCallback flushes cache entries when a relation * is updated (or flushes the entire cache). */ void InvalidateDistRelationCacheCallback(Datum argument, Oid relationId) { /* invalidate either entire cache or a specific entry */ if (relationId == InvalidOid) { InvalidateDistTableCache(); InvalidateDistObjectCache(); InvalidateMetadataSystemCache(); } else { void *hashKey = (void *) &relationId; bool foundInCache = false; if (DistTableCacheHash == NULL) { return; } CitusTableCacheEntrySlot *cacheSlot = hash_search(DistTableCacheHash, hashKey, HASH_FIND, &foundInCache); if (foundInCache) { InvalidateCitusTableCacheEntrySlot(cacheSlot); } /* * if pg_dist_partition relcache is invalidated for some reason, * invalidate the MetadataCache. It is likely an overkill to invalidate * the entire cache here. But until a better fix, we keep it this way * for postgres regression tests that includes * REINDEX SCHEMA CONCURRENTLY pg_catalog * command. */ if (relationId == MetadataCache.distPartitionRelationId) { InvalidateMetadataSystemCache(); } if (relationId == MetadataCache.distObjectRelationId) { InvalidateDistObjectCache(); } } } /* * InvalidateCitusTableCacheEntrySlot marks a CitusTableCacheEntrySlot as invalid, * meaning it needs to be rebuilt and the citusTableMetadata (if any) should be * released. */ static void InvalidateCitusTableCacheEntrySlot(CitusTableCacheEntrySlot *cacheSlot) { /* recheck whether this is a distributed table */ cacheSlot->isValid = false; if (cacheSlot->citusTableMetadata != NULL) { /* reload the metadata */ cacheSlot->citusTableMetadata->isValid = false; /* clean up ShardIdCacheHash */ RemoveStaleShardIdCacheEntries(cacheSlot->citusTableMetadata); } } /* * InvalidateDistTableCache marks all DistTableCacheHash entries invalid. */ static void InvalidateDistTableCache(void) { CitusTableCacheEntrySlot *cacheSlot = NULL; HASH_SEQ_STATUS status; if (DistTableCacheHash == NULL) { return; } hash_seq_init(&status, DistTableCacheHash); while ((cacheSlot = (CitusTableCacheEntrySlot *) hash_seq_search(&status)) != NULL) { InvalidateCitusTableCacheEntrySlot(cacheSlot); } } /* * InvalidateDistObjectCache marks all DistObjectCacheHash entries invalid. */ static void InvalidateDistObjectCache(void) { DistObjectCacheEntry *cacheEntry = NULL; HASH_SEQ_STATUS status; if (DistObjectCacheHash == NULL) { return; } hash_seq_init(&status, DistObjectCacheHash); while ((cacheEntry = (DistObjectCacheEntry *) hash_seq_search(&status)) != NULL) { cacheEntry->isValid = false; } } /* * FlushDistTableCache flushes the entire distributed relation cache, frees * all entries, and recreates the cache. */ void FlushDistTableCache(void) { CitusTableCacheEntrySlot *cacheSlot = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, DistTableCacheHash); while ((cacheSlot = (CitusTableCacheEntrySlot *) hash_seq_search(&status)) != NULL) { ResetCitusTableCacheEntry(cacheSlot->citusTableMetadata); } hash_destroy(DistTableCacheHash); hash_destroy(ShardIdCacheHash); CreateDistTableCache(); CreateShardIdCache(); } /* CreateDistTableCache initializes the per-table hash table */ static void CreateDistTableCache(void) { HASHCTL info; MemSet(&info, 0, sizeof(info)); info.keysize = sizeof(Oid); info.entrysize = sizeof(CitusTableCacheEntrySlot); info.hash = tag_hash; info.hcxt = MetadataCacheMemoryContext; DistTableCacheHash = hash_create("Distributed Relation Cache", 32, &info, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); } /* CreateShardIdCache initializes the shard ID mapping */ static void CreateShardIdCache(void) { HASHCTL info; MemSet(&info, 0, sizeof(info)); info.keysize = sizeof(int64); info.entrysize = sizeof(ShardIdCacheEntry); info.hash = tag_hash; info.hcxt = MetadataCacheMemoryContext; ShardIdCacheHash = hash_create("Shard Id Cache", 128, &info, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); } /* CreateDistObjectCache initializes the per-object hash table */ static void CreateDistObjectCache(void) { HASHCTL info; MemSet(&info, 0, sizeof(info)); info.keysize = sizeof(DistObjectCacheEntryKey); info.entrysize = sizeof(DistObjectCacheEntry); info.hash = tag_hash; info.hcxt = MetadataCacheMemoryContext; DistObjectCacheHash = hash_create("Distributed Object Cache", 32, &info, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); } /* * InvalidateMetadataSystemCache resets all the cached OIDs and the extensionCreatedState * flag and invalidates the worker node, ConnParams, and local group ID caches. */ void InvalidateMetadataSystemCache(void) { InvalidateConnParamsHashEntries(); memset(&MetadataCache, 0, sizeof(MetadataCache)); workerNodeHashValid = false; LocalGroupId = -1; LocalNodeId = -1; } /* * AllCitusTableIds returns all citus table ids. */ List * AllCitusTableIds(void) { return CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); } /* * CitusTableTypeIdList function scans pg_dist_partition and returns a * list of OID's for the tables matching given citusTableType. * To create the list, it performs sequential scan. Since it is not expected * that this function will be called frequently, it is OK not to use index * scan. If this function becomes performance bottleneck, it is possible to * modify this function to perform index scan. */ List * CitusTableTypeIdList(CitusTableType citusTableType) { ScanKeyData scanKey[1]; int scanKeyCount = 0; List *relationIdList = NIL; Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, InvalidOid, false, NULL, scanKeyCount, scanKey); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); HeapTuple heapTuple = systable_getnext(scanDescriptor); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); while (HeapTupleIsValid(heapTuple)) { memset(datumArray, 0, tupleDescriptor->natts * sizeof(Datum)); memset(isNullArray, 0, tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); Datum partMethodDatum = datumArray[Anum_pg_dist_partition_partmethod - 1]; Datum replicationModelDatum = datumArray[Anum_pg_dist_partition_repmodel - 1]; Datum colocationIdDatum = datumArray[Anum_pg_dist_partition_colocationid - 1]; char partitionMethod = DatumGetChar(partMethodDatum); char replicationModel = DatumGetChar(replicationModelDatum); uint32 colocationId = DatumGetUInt32(colocationIdDatum); if (IsCitusTableTypeInternal(partitionMethod, replicationModel, colocationId, citusTableType)) { Datum relationIdDatum = datumArray[Anum_pg_dist_partition_logicalrelid - 1]; Oid relationId = DatumGetObjectId(relationIdDatum); relationIdList = lappend_oid(relationIdList, relationId); } heapTuple = systable_getnext(scanDescriptor); } pfree(datumArray); pfree(isNullArray); systable_endscan(scanDescriptor); table_close(pgDistPartition, AccessShareLock); return relationIdList; } /* * InvalidateNodeRelationCacheCallback destroys the WorkerNodeHash when * any change happens on pg_dist_node table. It also set WorkerNodeHash to * NULL, which allows consequent accesses to the hash read from the * pg_dist_node from scratch. */ static void InvalidateNodeRelationCacheCallback(Datum argument, Oid relationId) { if (relationId == InvalidOid || relationId == MetadataCache.distNodeRelationId) { workerNodeHashValid = false; LocalNodeId = -1; } } /* * InvalidateLocalGroupIdRelationCacheCallback sets the LocalGroupId to * the default value. */ static void InvalidateLocalGroupIdRelationCacheCallback(Datum argument, Oid relationId) { /* when invalidation happens simply set the LocalGroupId to the default value */ if (relationId == InvalidOid || relationId == MetadataCache.distLocalGroupRelationId) { LocalGroupId = -1; } } /* * InvalidateConnParamsCacheCallback sets isValid flag to false for all entries * in ConnParamsHash, a cache used during connection establishment. */ static void InvalidateConnParamsCacheCallback(Datum argument, Oid relationId) { if (relationId == MetadataCache.distAuthinfoRelationId || relationId == MetadataCache.distPoolinfoRelationId || relationId == InvalidOid) { ConnParamsHashEntry *entry = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, ConnParamsHash); while ((entry = (ConnParamsHashEntry *) hash_seq_search(&status)) != NULL) { entry->isValid = false; } } } /* * CitusTableCacheFlushInvalidatedEntries frees invalidated cache entries. * Invalidated entries aren't freed immediately as callers expect their lifetime * to extend beyond that scope. */ void CitusTableCacheFlushInvalidatedEntries() { if (DistTableCacheHash != NULL && DistTableCacheExpired != NIL) { CitusTableCacheEntry *cacheEntry = NULL; foreach_declared_ptr(cacheEntry, DistTableCacheExpired) { ResetCitusTableCacheEntry(cacheEntry); } list_free(DistTableCacheExpired); DistTableCacheExpired = NIL; } } /* * CitusTableCacheEntryReleaseCallback frees invalidated cache entries. */ static void CitusTableCacheEntryReleaseCallback(ResourceReleasePhase phase, bool isCommit, bool isTopLevel, void *arg) { if (isTopLevel && phase == RESOURCE_RELEASE_LOCKS) { CitusTableCacheFlushInvalidatedEntries(); } } /* * LookupDistPartitionTuple searches pg_dist_partition for relationId's entry * and returns that or, if no matching entry was found, NULL. */ static HeapTuple LookupDistPartitionTuple(Relation pgDistPartition, Oid relationId) { HeapTuple distPartitionTuple = NULL; ScanKeyData scanKey[1]; /* copy scankey to local copy, it will be modified during the scan */ scanKey[0] = DistPartitionScanKey[0]; /* set scan arguments */ scanKey[0].sk_argument = ObjectIdGetDatum(relationId); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), true, NULL, 1, scanKey); HeapTuple currentPartitionTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(currentPartitionTuple)) { distPartitionTuple = heap_copytuple(currentPartitionTuple); } systable_endscan(scanDescriptor); return distPartitionTuple; } /* * LookupDistShardTuples returns a list of all dist_shard tuples for the * specified relation. */ List * LookupDistShardTuples(Oid relationId) { List *distShardTupleList = NIL; ScanKeyData scanKey[1]; Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); /* copy scankey to local copy, it will be modified during the scan */ scanKey[0] = DistShardScanKey[0]; /* set scan arguments */ scanKey[0].sk_argument = ObjectIdGetDatum(relationId); SysScanDesc scanDescriptor = systable_beginscan(pgDistShard, DistShardLogicalRelidIndexId(), true, NULL, 1, scanKey); HeapTuple currentShardTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(currentShardTuple)) { HeapTuple shardTupleCopy = heap_copytuple(currentShardTuple); distShardTupleList = lappend(distShardTupleList, shardTupleCopy); currentShardTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgDistShard, AccessShareLock); return distShardTupleList; } /* * LookupShardRelationFromCatalog returns the logical relation oid a shard belongs to. * * Errors out if the shardId does not exist and missingOk is false. * Returns InvalidOid if the shardId does not exist and missingOk is true. */ Oid LookupShardRelationFromCatalog(int64 shardId, bool missingOk) { ScanKeyData scanKey[1]; int scanKeyCount = 1; Form_pg_dist_shard shardForm = NULL; Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); Oid relationId = InvalidOid; ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistShard, DistShardShardidIndexId(), true, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple) && !missingOk) { ereport(ERROR, (errmsg("could not find valid entry for shard " UINT64_FORMAT, shardId))); } if (!HeapTupleIsValid(heapTuple)) { relationId = InvalidOid; } else { shardForm = (Form_pg_dist_shard) GETSTRUCT(heapTuple); relationId = shardForm->logicalrelid; } systable_endscan(scanDescriptor); table_close(pgDistShard, NoLock); return relationId; } /* * GetPartitionTypeInputInfo populates output parameters with the interval type * identifier and modifier for the specified partition key/method combination. */ static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, Oid *columnTypeId, int32 *columnTypeMod, Oid *intervalTypeId, int32 *intervalTypeMod) { *columnTypeId = InvalidOid; *columnTypeMod = -1; *intervalTypeId = InvalidOid; *intervalTypeMod = -1; switch (partitionMethod) { case DISTRIBUTE_BY_APPEND: case DISTRIBUTE_BY_RANGE: case DISTRIBUTE_BY_HASH: { Node *partitionNode = stringToNode(partitionKeyString); Var *partitionColumn = (Var *) partitionNode; Assert(IsA(partitionNode, Var)); GetIntervalTypeInfo(partitionMethod, partitionColumn, intervalTypeId, intervalTypeMod); *columnTypeId = partitionColumn->vartype; *columnTypeMod = partitionColumn->vartypmod; break; } case DISTRIBUTE_BY_NONE: { break; } default: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported table partition type: %c", partitionMethod))); } } } /* * GetIntervalTypeInfo gets type id and type mod of the min/max values * of shard intervals for a distributed table with given partition method * and partition column. */ void GetIntervalTypeInfo(char partitionMethod, Var *partitionColumn, Oid *intervalTypeId, int32 *intervalTypeMod) { *intervalTypeId = InvalidOid; *intervalTypeMod = -1; switch (partitionMethod) { case DISTRIBUTE_BY_APPEND: case DISTRIBUTE_BY_RANGE: { /* we need a valid partition column Var in this case */ if (partitionColumn == NULL) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unexpected partition column value: null"), errdetail("Please report this to the Citus core team."))); } *intervalTypeId = partitionColumn->vartype; *intervalTypeMod = partitionColumn->vartypmod; break; } case DISTRIBUTE_BY_HASH: { *intervalTypeId = INT4OID; break; } default: { break; } } } /* * TupleToShardInterval transforms the specified dist_shard tuple into a new * ShardInterval using the provided descriptor and partition type information. */ ShardInterval * TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid intervalTypeId, int32 intervalTypeMod) { Datum datumArray[Natts_pg_dist_shard]; bool isNullArray[Natts_pg_dist_shard]; /* * We use heap_deform_tuple() instead of heap_getattr() to expand tuple * to contain missing values when ALTER TABLE ADD COLUMN happens. */ heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); ShardInterval *shardInterval = DeformedDistShardTupleToShardInterval(datumArray, isNullArray, intervalTypeId, intervalTypeMod); return shardInterval; } /* * DeformedDistShardTupleToShardInterval transforms the specified deformed * pg_dist_shard tuple into a new ShardInterval. */ ShardInterval * DeformedDistShardTupleToShardInterval(Datum *datumArray, bool *isNullArray, Oid intervalTypeId, int32 intervalTypeMod) { Oid inputFunctionId = InvalidOid; Oid typeIoParam = InvalidOid; Datum minValue = 0; Datum maxValue = 0; bool minValueExists = false; bool maxValueExists = false; int16 intervalTypeLen = 0; bool intervalByVal = false; char intervalAlign = '0'; char intervalDelim = '0'; Oid relationId = DatumGetObjectId(datumArray[Anum_pg_dist_shard_logicalrelid - 1]); int64 shardId = DatumGetInt64(datumArray[Anum_pg_dist_shard_shardid - 1]); char storageType = DatumGetChar(datumArray[Anum_pg_dist_shard_shardstorage - 1]); Datum minValueTextDatum = datumArray[Anum_pg_dist_shard_shardminvalue - 1]; Datum maxValueTextDatum = datumArray[Anum_pg_dist_shard_shardmaxvalue - 1]; bool minValueNull = isNullArray[Anum_pg_dist_shard_shardminvalue - 1]; bool maxValueNull = isNullArray[Anum_pg_dist_shard_shardmaxvalue - 1]; if (!minValueNull && !maxValueNull) { char *minValueString = TextDatumGetCString(minValueTextDatum); char *maxValueString = TextDatumGetCString(maxValueTextDatum); /* TODO: move this up the call stack to avoid per-tuple invocation? */ get_type_io_data(intervalTypeId, IOFunc_input, &intervalTypeLen, &intervalByVal, &intervalAlign, &intervalDelim, &typeIoParam, &inputFunctionId); /* finally convert min/max values to their actual types */ minValue = OidInputFunctionCall(inputFunctionId, minValueString, typeIoParam, intervalTypeMod); maxValue = OidInputFunctionCall(inputFunctionId, maxValueString, typeIoParam, intervalTypeMod); minValueExists = true; maxValueExists = true; } ShardInterval *shardInterval = CitusMakeNode(ShardInterval); shardInterval->relationId = relationId; shardInterval->storageType = storageType; shardInterval->valueTypeId = intervalTypeId; shardInterval->valueTypeLen = intervalTypeLen; shardInterval->valueByVal = intervalByVal; shardInterval->minValueExists = minValueExists; shardInterval->maxValueExists = maxValueExists; shardInterval->minValue = minValue; shardInterval->maxValue = maxValue; shardInterval->shardId = shardId; return shardInterval; } /* * CachedNamespaceLookup performs a cached lookup for the namespace (schema), with the * result cached in cachedOid. */ static void CachedNamespaceLookup(const char *nspname, Oid *cachedOid) { /* force callbacks to be registered, so we always get notified upon changes */ InitializeCaches(); if (*cachedOid == InvalidOid) { *cachedOid = get_namespace_oid(nspname, true); if (*cachedOid == InvalidOid) { ereport(ERROR, (errmsg( "cache lookup failed for namespace %s, called too early?", nspname))); } } } /* * CachedRelationLookup performs a cached lookup for the relation * relationName, with the result cached in *cachedOid. */ static void CachedRelationLookup(const char *relationName, Oid *cachedOid) { CachedRelationNamespaceLookup(relationName, PG_CATALOG_NAMESPACE, cachedOid); } /* * CachedRelationLookupExtended performs a cached lookup for the relation * relationName, with the result cached in *cachedOid. Will _not_ throw an error when * missing_ok is set to true. */ static void CachedRelationLookupExtended(const char *relationName, Oid *cachedOid, bool missing_ok) { CachedRelationNamespaceLookupExtended(relationName, PG_CATALOG_NAMESPACE, cachedOid, missing_ok); } static void CachedRelationNamespaceLookup(const char *relationName, Oid relnamespace, Oid *cachedOid) { CachedRelationNamespaceLookupExtended(relationName, relnamespace, cachedOid, false); } static void CachedRelationNamespaceLookupExtended(const char *relationName, Oid relnamespace, Oid *cachedOid, bool missing_ok) { /* force callbacks to be registered, so we always get notified upon changes */ InitializeCaches(); if (*cachedOid == InvalidOid) { *cachedOid = get_relname_relid(relationName, relnamespace); if (*cachedOid == InvalidOid && !missing_ok) { ereport(ERROR, (errmsg( "cache lookup failed for %s, called too early?", relationName))); } } } /* * RelationExists returns whether a relation with the given OID exists. */ bool RelationExists(Oid relationId) { HeapTuple relTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relationId)); bool relationExists = HeapTupleIsValid(relTuple); if (relationExists) { ReleaseSysCache(relTuple); } return relationExists; } /* * Register a relcache invalidation for a non-shared relation. * * We ignore the case that there's no corresponding pg_class entry - that * happens if we register a relcache invalidation (e.g. for a * pg_dist_partition deletion) after the relation has been dropped. That's ok, * because in those cases we're guaranteed to already have registered an * invalidation for the target relation. */ void CitusInvalidateRelcacheByRelid(Oid relationId) { HeapTuple classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relationId)); if (HeapTupleIsValid(classTuple)) { CacheInvalidateRelcacheByTuple(classTuple); ReleaseSysCache(classTuple); } } /* * Register a relcache invalidation for the distributed relation associated * with the shard. */ void CitusInvalidateRelcacheByShardId(int64 shardId) { ScanKeyData scanKey[1]; int scanKeyCount = 1; Form_pg_dist_shard shardForm = NULL; Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); /* * Load shard, to find the associated relation id. Can't use * LoadShardInterval directly because that'd fail if the shard doesn't * exist anymore, which we can't have. Also lower overhead is desirable * here. */ ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistShard, DistShardShardidIndexId(), true, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { shardForm = (Form_pg_dist_shard) GETSTRUCT(heapTuple); CitusInvalidateRelcacheByRelid(shardForm->logicalrelid); } else { /* * Couldn't find associated relation. That can primarily happen in two cases: * * 1) A placement row is inserted before the shard row. That's fine, * since we don't need invalidations via placements in that case. * * 2) The shard has been deleted, but some placements were * unreachable, and the user is manually deleting the rows. Not * much point in WARNING or ERRORing in that case either, there's * nothing to invalidate. * * Hence we just emit a DEBUG5 message. */ ereport(DEBUG5, (errmsg( "could not find distributed relation to invalidate for " "shard "INT64_FORMAT, shardId))); } systable_endscan(scanDescriptor); table_close(pgDistShard, NoLock); /* bump command counter, to force invalidation to take effect */ CommandCounterIncrement(); } /* * DistNodeMetadata returns the single metadata jsonb object stored in * pg_dist_node_metadata. */ Datum DistNodeMetadata(void) { Datum metadata = 0; ScanKeyData scanKey[1]; const int scanKeyCount = 0; Oid metadataTableOid = get_relname_relid("pg_dist_node_metadata", PG_CATALOG_NAMESPACE); if (metadataTableOid == InvalidOid) { ereport(ERROR, (errmsg("pg_dist_node_metadata was not found"))); } Relation pgDistNodeMetadata = table_open(metadataTableOid, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistNodeMetadata, InvalidOid, false, NULL, scanKeyCount, scanKey); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNodeMetadata); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { bool isNull = false; metadata = heap_getattr(heapTuple, Anum_pg_dist_node_metadata_metadata, tupleDescriptor, &isNull); Assert(!isNull); } else { ereport(ERROR, (errmsg( "could not find any entries in pg_dist_metadata"))); } /* * Copy the jsonb result before closing the table * since that memory can be freed. */ metadata = JsonbPGetDatum(DatumGetJsonbPCopy(metadata)); systable_endscan(scanDescriptor); table_close(pgDistNodeMetadata, AccessShareLock); return metadata; } /* * role_exists is a check constraint which ensures that roles referenced in the * pg_dist_authinfo catalog actually exist (at least at the time of insertion). */ Datum role_exists(PG_FUNCTION_ARGS) { Name roleName = PG_GETARG_NAME(0); bool roleExists = SearchSysCacheExists1(AUTHNAME, NameGetDatum(roleName)); PG_RETURN_BOOL(roleExists); } /* * GetPoolinfoViaCatalog searches the pg_dist_poolinfo table for a row matching * the provided nodeId and returns the poolinfo field of this row if found. * Otherwise, this function returns NULL. */ char * GetPoolinfoViaCatalog(int32 nodeId) { ScanKeyData scanKey[1]; const int scanKeyCount = 1; const AttrNumber nodeIdIdx = 1, poolinfoIdx = 2; Relation pgDistPoolinfo = table_open(DistPoolinfoRelationId(), AccessShareLock); bool indexOK = true; char *poolinfo = NULL; /* set scan arguments */ ScanKeyInit(&scanKey[0], nodeIdIdx, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(nodeId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPoolinfo, DistPoolinfoIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistPoolinfo); bool isNull = false; Datum poolinfoDatum = heap_getattr(heapTuple, poolinfoIdx, tupleDescriptor, &isNull); Assert(!isNull); poolinfo = TextDatumGetCString(poolinfoDatum); } systable_endscan(scanDescriptor); table_close(pgDistPoolinfo, AccessShareLock); return poolinfo; } /* * GetAuthinfoViaCatalog searches pg_dist_authinfo for a row matching a pro- * vided role and node id. Three types of rules are currently permitted: those * matching a specific node (non-zero nodeid), those matching all nodes (a * nodeid of zero), and those denoting a loopback connection (nodeid of -1). * Rolename must always be specified. If both types of rules exist for a given * user/host, the more specific (host-specific) rule wins. This means that when * both a zero and non-zero row exist for a given rolename, the non-zero row * has precedence. * * In short, this function will return a rule matching nodeId, or if that's * absent the rule for 0, or if that's absent, an empty string. Callers can * just use the returned authinfo and know the precedence has been honored. */ char * GetAuthinfoViaCatalog(const char *roleName, int64 nodeId) { char *authinfo = ""; Datum nodeIdDatumArray[2] = { Int32GetDatum(nodeId), Int32GetDatum(WILDCARD_NODE_ID) }; ArrayType *nodeIdArrayType = DatumArrayToArrayType(nodeIdDatumArray, lengthof(nodeIdDatumArray), INT4OID); ScanKeyData scanKey[2]; const AttrNumber nodeIdIdx = 1, roleIdx = 2, authinfoIdx = 3; /* * Our index's definition ensures correct precedence for positive nodeIds, * but when handling a negative value we need to traverse backwards to keep * the invariant that the zero rule has lowest precedence. */ ScanDirection direction = (nodeId < 0) ? BackwardScanDirection : ForwardScanDirection; if (ReindexIsProcessingIndex(DistAuthinfoIndexId())) { ereport(ERROR, (errmsg("authinfo is being reindexed; try again"))); } memset(&scanKey, 0, sizeof(scanKey)); /* first column in index is rolename, need exact match there ... */ ScanKeyInit(&scanKey[0], roleIdx, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(roleName)); /* second column is nodeId, match against array of nodeid and zero (any node) ... */ ScanKeyInit(&scanKey[1], nodeIdIdx, BTEqualStrategyNumber, F_INT4EQ, PointerGetDatum(nodeIdArrayType)); scanKey[1].sk_flags |= SK_SEARCHARRAY; /* * It's important that we traverse the index in order: we need to ensure * that rules with nodeid 0 are encountered last. We'll use the first tuple * we find. This ordering defines the precedence order of authinfo rules. */ Relation pgDistAuthinfo = table_open(DistAuthinfoRelationId(), AccessShareLock); Relation pgDistAuthinfoIdx = index_open(DistAuthinfoIndexId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan_ordered(pgDistAuthinfo, pgDistAuthinfoIdx, NULL, lengthof(scanKey), scanKey); /* first tuple represents highest-precedence rule for this node */ HeapTuple authinfoTuple = systable_getnext_ordered(scanDescriptor, direction); if (HeapTupleIsValid(authinfoTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistAuthinfo); bool isNull = false; Datum authinfoDatum = heap_getattr(authinfoTuple, authinfoIdx, tupleDescriptor, &isNull); Assert(!isNull); authinfo = TextDatumGetCString(authinfoDatum); } systable_endscan_ordered(scanDescriptor); index_close(pgDistAuthinfoIdx, AccessShareLock); table_close(pgDistAuthinfo, AccessShareLock); return authinfo; } /* * authinfo_valid is a check constraint to verify that an inserted authinfo row * uses only permitted libpq parameters. */ Datum authinfo_valid(PG_FUNCTION_ARGS) { char *authinfo = TextDatumGetCString(PG_GETARG_DATUM(0)); /* this array _must_ be kept in an order usable by bsearch */ const char *allowList[] = { "password", "sslcert", "sslkey" }; bool authinfoValid = CheckConninfo(authinfo, allowList, lengthof(allowList), NULL); PG_RETURN_BOOL(authinfoValid); } /* * poolinfo_valid is a check constraint to verify that an inserted poolinfo row * uses only permitted libpq parameters. */ Datum poolinfo_valid(PG_FUNCTION_ARGS) { char *poolinfo = TextDatumGetCString(PG_GETARG_DATUM(0)); /* this array _must_ be kept in an order usable by bsearch */ const char *allowList[] = { "dbname", "host", "port" }; bool poolinfoValid = CheckConninfo(poolinfo, allowList, lengthof(allowList), NULL); PG_RETURN_BOOL(poolinfoValid); } ================================================ FILE: src/backend/distributed/metadata/metadata_sync.c ================================================ /*------------------------------------------------------------------------- * * metadata_sync.c * * Routines for synchronizing metadata to all workers. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/pg_am.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_database.h" #include "catalog/pg_database_d.h" #include "catalog/pg_depend.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/async.h" #include "commands/dbcommands.h" #include "executor/spi.h" #include "foreign/foreign.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" #include "parser/parse_type.h" #include "postmaster/bgworker.h" #include "postmaster/postmaster.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "distributed/argutils.h" #include "distributed/backend_data.h" #include "distributed/background_worker_utils.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/maintenanced.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_colocation.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_schema.h" #include "distributed/pg_dist_shard.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/remote_transaction.h" #include "distributed/resource_lock.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/utils/array_type.h" #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* managed via a GUC */ char *EnableManualMetadataChangesForUser = ""; int MetadataSyncTransMode = METADATA_SYNC_TRANSACTIONAL; static void EnsureObjectMetadataIsSane(int distributionArgumentIndex, int colocationId); static List * GetFunctionDependenciesForObjects(ObjectAddress *objectAddress); static char * SchemaOwnerName(Oid objectId); static bool HasMetadataWorkers(void); static void CreateShellTableOnWorkers(Oid relationId); static void CreateTableMetadataOnWorkers(Oid relationId); static void CreateDependingViewsOnWorkers(Oid relationId); static void AddTableToPublications(Oid relationId); static NodeMetadataSyncResult SyncNodeMetadataToNodesOptional(void); static bool ShouldSyncTableMetadataInternal(bool hashDistributed, bool citusTableWithNoDistKey); static bool SyncNodeMetadataSnapshotToNode(WorkerNode *workerNode, bool raiseOnError); static void DropMetadataSnapshotOnNode(WorkerNode *workerNode); static char * CreateSequenceDependencyCommand(Oid relationId, Oid sequenceId, char *columnName); static GrantStmt * GenerateGrantStmtForRights(ObjectType objectType, Oid roleOid, Oid objectId, char *permission, bool withGrantOption); static List * GetObjectsForGrantStmt(ObjectType objectType, Oid objectId); static AccessPriv * GetAccessPrivObjectForGrantStmt(char *permission); static List * GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem); static List * GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem); static List * GenerateGrantOnFunctionQueriesFromAclItem(Oid schemaOid, AclItem *aclItem); static List * GrantOnSequenceDDLCommands(Oid sequenceOid); static List * GenerateGrantOnSequenceQueriesFromAclItem(Oid sequenceOid, AclItem *aclItem); static char * GenerateSetRoleQuery(Oid roleOid); static void MetadataSyncSigTermHandler(SIGNAL_ARGS); static void MetadataSyncSigAlrmHandler(SIGNAL_ARGS); static bool ShouldSkipMetadataChecks(void); static void EnsurePartitionMetadataIsSane(Oid relationId, char distributionMethod, int colocationId, char replicationModel, Var *distributionKey); static void EnsureCitusInitiatedOperation(void); static void EnsureShardMetadataIsSane(Oid relationId, int64 shardId, char storageType, text *shardMinValue, text *shardMaxValue); static void EnsureShardPlacementMetadataIsSane(Oid relationId, int64 shardId, int64 placementId, int64 shardLength, int32 groupId); static char * ColocationGroupCreateCommand(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation); static char * ColocationGroupDeleteCommand(uint32 colocationId); static char * RemoteSchemaIdExpressionById(Oid schemaId); static char * RemoteSchemaIdExpressionByName(char *schemaName); static char * GetRemoteTypeName(Oid typeId); static char * GetRemoteTypeNamespace(Oid typeId); static char * RemoteCollationIdExpression(Oid colocationId); static char * RemoteTableIdExpression(Oid relationId); PG_FUNCTION_INFO_V1(start_metadata_sync_to_all_nodes); PG_FUNCTION_INFO_V1(start_metadata_sync_to_node); PG_FUNCTION_INFO_V1(stop_metadata_sync_to_node); PG_FUNCTION_INFO_V1(worker_record_sequence_dependency); /* * Functions to modify metadata. Normally modifying metadata requires * superuser. However, these functions can be called with superusers * or regular users as long as the regular user owns the input object. */ PG_FUNCTION_INFO_V1(citus_internal_add_partition_metadata); PG_FUNCTION_INFO_V1(citus_internal_delete_partition_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_shard_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_placement_metadata); PG_FUNCTION_INFO_V1(citus_internal_delete_placement_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_placement_metadata_legacy); PG_FUNCTION_INFO_V1(citus_internal_update_placement_metadata); PG_FUNCTION_INFO_V1(citus_internal_delete_shard_metadata); PG_FUNCTION_INFO_V1(citus_internal_update_relation_colocation); PG_FUNCTION_INFO_V1(citus_internal_add_object_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_colocation_metadata); PG_FUNCTION_INFO_V1(citus_internal_delete_colocation_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_tenant_schema); PG_FUNCTION_INFO_V1(citus_internal_delete_tenant_schema); PG_FUNCTION_INFO_V1(citus_internal_update_none_dist_table_metadata); PG_FUNCTION_INFO_V1(citus_internal_database_command); static bool got_SIGTERM = false; static bool got_SIGALRM = false; #define METADATA_SYNC_APP_NAME "Citus Metadata Sync Daemon" /* * start_metadata_sync_to_node function sets hasmetadata column of the given * node to true, and then activate node without replicating reference tables. */ Datum start_metadata_sync_to_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); EnsureSuperUser(); EnsureCoordinator(); char *nodeNameString = text_to_cstring(nodeName); WorkerNode *workerNode = ModifiableWorkerNode(nodeNameString, nodePort); /* * Create MetadataSyncContext which is used throughout nodes' activation. * It contains activated nodes, bare connections if the mode is nontransactional, * and a memory context for allocation. */ bool collectCommands = false; bool nodesAddedInSameTransaction = false; MetadataSyncContext *context = CreateMetadataSyncContext(list_make1(workerNode), collectCommands, nodesAddedInSameTransaction); ActivateNodeList(context); TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * start_metadata_sync_to_all_nodes function sets hasmetadata column of * all the primary worker nodes to true, and then activate nodes without * replicating reference tables. */ Datum start_metadata_sync_to_all_nodes(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); List *nodeList = ActivePrimaryNonCoordinatorNodeList(RowShareLock); /* * Create MetadataSyncContext which is used throughout nodes' activation. * It contains activated nodes, bare connections if the mode is nontransactional, * and a memory context for allocation. */ bool collectCommands = false; bool nodesAddedInSameTransaction = false; MetadataSyncContext *context = CreateMetadataSyncContext(nodeList, collectCommands, nodesAddedInSameTransaction); ActivateNodeList(context); TransactionModifiedNodeMetadata = true; PG_RETURN_BOOL(true); } /* * SyncCitusTableMetadata syncs citus table metadata to worker nodes with metadata. * Our definition of metadata includes the shell table and its inter relations with * other shell tables, corresponding pg_dist_object, pg_dist_partiton, pg_dist_shard * and pg_dist_shard placement entries. This function also propagates the views that * depend on the given relation, to the metadata workers, and adds the relation to * the appropriate publications. */ void SyncCitusTableMetadata(Oid relationId) { CreateShellTableOnWorkers(relationId); CreateTableMetadataOnWorkers(relationId); CreateInterTableRelationshipOfRelationOnWorkers(relationId); if (!IsTableOwnedByExtension(relationId)) { ObjectAddress relationAddress = { 0 }; ObjectAddressSet(relationAddress, RelationRelationId, relationId); MarkObjectDistributed(&relationAddress); } CreateDependingViewsOnWorkers(relationId); AddTableToPublications(relationId); } /* * CreateDependingViewsOnWorkers takes a relationId and creates the views that depend on * that relation on workers with metadata. Propagated views are marked as distributed. */ static void CreateDependingViewsOnWorkers(Oid relationId) { List *views = GetDependingViews(relationId); if (list_length(views) < 1) { /* no view to propagate */ return; } SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); Oid viewOid = InvalidOid; foreach_declared_oid(viewOid, views) { if (!ShouldMarkRelationDistributed(viewOid)) { continue; } ObjectAddress *viewAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*viewAddress, RelationRelationId, viewOid); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(viewAddress)); char *createViewCommand = CreateViewDDLCommand(viewOid); char *alterViewOwnerCommand = AlterViewOwnerCommand(viewOid); SendCommandToWorkersWithMetadata(createViewCommand); SendCommandToWorkersWithMetadata(alterViewOwnerCommand); MarkObjectDistributed(viewAddress); } SendCommandToWorkersWithMetadata(ENABLE_DDL_PROPAGATION); } /* * AddTableToPublications adds the table to a publication on workers with metadata. */ static void AddTableToPublications(Oid relationId) { List *publicationIds = GetRelationPublications(relationId); if (publicationIds == NIL) { return; } Oid publicationId = InvalidOid; SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); foreach_declared_oid(publicationId, publicationIds) { ObjectAddress *publicationAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*publicationAddress, PublicationRelationId, publicationId); List *addresses = list_make1(publicationAddress); if (!ShouldPropagateAnyObject(addresses)) { /* skip non-distributed publications */ continue; } /* ensure schemas exist */ EnsureAllObjectDependenciesExistOnAllNodes(addresses); bool isAdd = true; char *alterPublicationCommand = GetAlterPublicationTableDDLCommand(publicationId, relationId, isAdd); /* send ALTER PUBLICATION .. ADD to workers with metadata */ SendCommandToWorkersWithMetadata(alterPublicationCommand); } SendCommandToWorkersWithMetadata(ENABLE_DDL_PROPAGATION); } /* * EnsureSequentialModeMetadataOperations makes sure that the current transaction is * already in sequential mode, or can still safely be put in sequential mode, * it errors if that is not possible. The error contains information for the user to * retry the transaction with sequential mode set from the beginning. * * Metadata objects (e.g., distributed table on the workers) exists only 1 instance of * the type used by potentially multiple other shards/connections. To make sure all * shards/connections in the transaction can interact with the metadata needs to be * visible on all connections used by the transaction, meaning we can only use 1 * connection per node. */ void EnsureSequentialModeMetadataOperations(void) { if (!IsTransactionBlock()) { /* we do not need to switch to sequential mode if we are not in a transaction */ return; } if (ParallelQueryExecutedInTransaction()) { ereport(ERROR, (errmsg( "cannot execute metadata syncing operation because there was a " "parallel operation on a distributed table in the " "transaction"), errdetail("When modifying metadata, Citus needs to " "perform all operations over a single connection per " "node to ensure consistency."), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail("Metadata synced or stopped syncing. To make " "sure subsequent commands see the metadata correctly " "we need to make sure to use only one connection for " "all future commands"))); SetLocalMultiShardModifyModeToSequential(); } /* * stop_metadata_sync_to_node function sets the hasmetadata column of the specified node * to false in pg_dist_node table, thus indicating that the specified worker node does not * receive DDL changes anymore and cannot be used for issuing queries. */ Datum stop_metadata_sync_to_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); EnsureSuperUser(); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); bool clearMetadata = PG_GETARG_BOOL(2); char *nodeNameString = text_to_cstring(nodeName); LockRelationOid(DistNodeRelationId(), ExclusiveLock); WorkerNode *workerNode = FindWorkerNodeAnyCluster(nodeNameString, nodePort); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("node (%s,%d) does not exist", nodeNameString, nodePort))); } if (NodeIsCoordinator(workerNode)) { ereport(NOTICE, (errmsg("node (%s,%d) is the coordinator and should have " "metadata, skipping stopping the metadata sync", nodeNameString, nodePort))); PG_RETURN_VOID(); } if (clearMetadata) { if (NodeIsPrimary(workerNode)) { ereport(NOTICE, (errmsg("dropping metadata on the node (%s,%d)", nodeNameString, nodePort))); DropMetadataSnapshotOnNode(workerNode); } else { /* * If this is a secondary node we can't actually clear metadata from it, * we assume the primary node is cleared. */ ereport(NOTICE, (errmsg("(%s,%d) is a secondary node: to clear the metadata," " you should clear metadata from the primary node", nodeNameString, nodePort))); } } workerNode = SetWorkerColumn(workerNode, Anum_pg_dist_node_hasmetadata, BoolGetDatum( false)); workerNode = SetWorkerColumn(workerNode, Anum_pg_dist_node_metadatasynced, BoolGetDatum(false)); TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * ClusterHasKnownMetadataWorkers returns true if the node executing the function * knows at least one worker with metadata. We do it * (a) by checking the node that executes the function is a worker with metadata * (b) the coordinator knows at least one worker with metadata. */ bool ClusterHasKnownMetadataWorkers() { return !IsCoordinator() || HasMetadataWorkers(); } /* * ShouldSyncUserCommandForObject checks if the user command should be synced to the * worker nodes for the given object. */ bool ShouldSyncUserCommandForObject(ObjectAddress objectAddress) { if (objectAddress.classId == RelationRelationId) { Oid relOid = objectAddress.objectId; return ShouldSyncTableMetadata(relOid) || ShouldSyncSequenceMetadata(relOid) || get_rel_relkind(relOid) == RELKIND_VIEW; } return false; } /* * ShouldSyncTableMetadata checks if the metadata of a distributed table should be * propagated to metadata workers, i.e. the table is a hash distributed table or * a Citus table that doesn't have shard key. */ bool ShouldSyncTableMetadata(Oid relationId) { if (!EnableMetadataSync || !OidIsValid(relationId) || !IsCitusTable(relationId)) { return false; } CitusTableCacheEntry *tableEntry = GetCitusTableCacheEntry(relationId); bool hashDistributed = IsCitusTableTypeCacheEntry(tableEntry, HASH_DISTRIBUTED); bool citusTableWithNoDistKey = !HasDistributionKeyCacheEntry(tableEntry); return ShouldSyncTableMetadataInternal(hashDistributed, citusTableWithNoDistKey); } /* * ShouldSyncTableMetadataViaCatalog checks if the metadata of a Citus table should * be propagated to metadata workers, i.e. the table is an MX table or Citus table * that doesn't have shard key. * Tables with streaming replication model (which means RF=1) and hash distribution are * considered as MX tables. * * ShouldSyncTableMetadataViaCatalog does not use the CitusTableCache and instead reads * from catalog tables directly. */ bool ShouldSyncTableMetadataViaCatalog(Oid relationId) { if (!OidIsValid(relationId) || !IsCitusTableViaCatalog(relationId)) { return false; } char partitionMethod = PartitionMethodViaCatalog(relationId); bool hashDistributed = partitionMethod == DISTRIBUTE_BY_HASH; bool citusTableWithNoDistKey = partitionMethod == DISTRIBUTE_BY_NONE; return ShouldSyncTableMetadataInternal(hashDistributed, citusTableWithNoDistKey); } /* * FetchRelationIdFromPgPartitionHeapTuple returns relation id from given heap tuple. */ Oid FetchRelationIdFromPgPartitionHeapTuple(HeapTuple heapTuple, TupleDesc tupleDesc) { Assert(heapTuple->t_tableOid == DistPartitionRelationId()); Datum *datumArray = (Datum *) palloc(tupleDesc->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDesc->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDesc, datumArray, isNullArray); Datum relationIdDatum = datumArray[Anum_pg_dist_partition_logicalrelid - 1]; Oid relationId = DatumGetObjectId(relationIdDatum); pfree(datumArray); pfree(isNullArray); return relationId; } /* * ShouldSyncTableMetadataInternal decides whether we should sync the metadata for a table * based on whether it is a hash distributed table, or a citus table with no distribution * key. * * This function is here to make sure that ShouldSyncTableMetadata and * ShouldSyncTableMetadataViaCatalog behaves the same way. */ static bool ShouldSyncTableMetadataInternal(bool hashDistributed, bool citusTableWithNoDistKey) { return hashDistributed || citusTableWithNoDistKey; } /* * ShouldSyncSequenceMetadata checks if the metadata of a sequence should be * propagated to metadata workers, i.e. the sequence is marked as distributed */ bool ShouldSyncSequenceMetadata(Oid relationId) { if (!OidIsValid(relationId) || !(get_rel_relkind(relationId) == RELKIND_SEQUENCE)) { return false; } ObjectAddress *sequenceAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*sequenceAddress, RelationRelationId, relationId); return IsAnyObjectDistributed(list_make1(sequenceAddress)); } /* * SyncMetadataSnapshotToNode does the following: * SyncNodeMetadataSnapshotToNode does the following: * 1. Sets the localGroupId on the worker so the worker knows which tuple in * pg_dist_node represents itself. * 2. Recreates the node metadata on the given worker. * If raiseOnError is true, it errors out if synchronization fails. */ static bool SyncNodeMetadataSnapshotToNode(WorkerNode *workerNode, bool raiseOnError) { char *currentUser = CurrentUserName(); /* generate and add the local group id's update query */ char *localGroupIdUpdateCommand = LocalGroupIdUpdateCommand(workerNode->groupId); /* generate the queries which drop the node metadata */ List *dropMetadataCommandList = NodeMetadataDropCommands(); /* generate the queries which create the node metadata from scratch */ List *createMetadataCommandList = NodeMetadataCreateCommands(); List *recreateMetadataSnapshotCommandList = list_make1(localGroupIdUpdateCommand); recreateMetadataSnapshotCommandList = list_concat(recreateMetadataSnapshotCommandList, dropMetadataCommandList); recreateMetadataSnapshotCommandList = list_concat(recreateMetadataSnapshotCommandList, createMetadataCommandList); /* * Send the snapshot recreation commands in a single remote transaction and * if requested, error out in any kind of failure. Note that it is not * required to send createMetadataSnapshotCommandList in the same transaction * that we send nodeDeleteCommand and nodeInsertCommand commands below. */ if (raiseOnError) { SendMetadataCommandListToWorkerListInCoordinatedTransaction(list_make1( workerNode), currentUser, recreateMetadataSnapshotCommandList); return true; } else { bool success = SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction( workerNode->workerName, workerNode->workerPort, currentUser, recreateMetadataSnapshotCommandList); return success; } } /* * DropMetadataSnapshotOnNode creates the queries which drop the metadata and sends them * to the worker given as parameter. */ static void DropMetadataSnapshotOnNode(WorkerNode *workerNode) { EnsureSequentialModeMetadataOperations(); char *userName = CurrentUserName(); /* * Detach partitions, break dependencies between sequences and table then * remove shell tables first. */ bool singleTransaction = true; List *dropMetadataCommandList = DetachPartitionCommandList(); dropMetadataCommandList = lappend(dropMetadataCommandList, BREAK_ALL_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND); dropMetadataCommandList = lappend(dropMetadataCommandList, WorkerDropAllShellTablesCommand(singleTransaction)); dropMetadataCommandList = list_concat(dropMetadataCommandList, NodeMetadataDropCommands()); dropMetadataCommandList = lappend(dropMetadataCommandList, LocalGroupIdUpdateCommand(0)); /* remove all dist table and object/table related metadata afterwards */ dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_PARTITIONS); dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_SHARDS); dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_PLACEMENTS); dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_DISTRIBUTED_OBJECTS); dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_COLOCATION); Assert(superuser()); SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction( workerNode->workerName, workerNode->workerPort, userName, dropMetadataCommandList); } /* * NodeMetadataCreateCommands returns list of queries that are * required to create the current metadata snapshot of the node that the * function is called. The metadata snapshot commands includes the * following queries: * * (i) Query that populates pg_dist_node table */ List * NodeMetadataCreateCommands(void) { List *metadataSnapshotCommandList = NIL; bool includeNodesFromOtherClusters = true; List *workerNodeList = ReadDistNode(includeNodesFromOtherClusters); /* make sure we have deterministic output for our tests */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); /* generate insert command for pg_dist_node table */ char *nodeListInsertCommand = NodeListInsertCommand(workerNodeList); metadataSnapshotCommandList = lappend(metadataSnapshotCommandList, nodeListInsertCommand); return metadataSnapshotCommandList; } /* * CitusTableMetadataCreateCommandList returns the set of commands necessary to * create the given distributed table metadata on a worker. */ List * CitusTableMetadataCreateCommandList(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *commandList = NIL; /* command to insert pg_dist_partition entry */ char *metadataCommand = DistributionCreateCommand(cacheEntry); commandList = lappend(commandList, metadataCommand); /* commands to insert pg_dist_shard & pg_dist_placement entries */ List *shardIntervalList = LoadShardIntervalList(relationId); List *shardMetadataInsertCommandList = ShardListInsertCommand(shardIntervalList); commandList = list_concat(commandList, shardMetadataInsertCommandList); return commandList; } /* * NodeMetadataDropCommands returns list of queries that are required to * drop all the metadata of the node that are not related to clustered tables. * The drop metadata snapshot commands includes the following queries: * * (i) Queries that delete all the rows from pg_dist_node table */ List * NodeMetadataDropCommands(void) { List *dropSnapshotCommandList = NIL; dropSnapshotCommandList = lappend(dropSnapshotCommandList, DELETE_ALL_NODES); return dropSnapshotCommandList; } /* * NodeListInsertCommand generates a single multi-row INSERT command that can be * executed to insert the nodes that are in workerNodeList to pg_dist_node table. */ char * NodeListInsertCommand(List *workerNodeList) { StringInfo nodeListInsertCommand = makeStringInfo(); int workerCount = list_length(workerNodeList); int processedWorkerNodeCount = 0; Oid primaryRole = PrimaryNodeRoleId(); /* if there are no workers, return NULL */ if (workerCount == 0) { return nodeListInsertCommand->data; } if (primaryRole == InvalidOid) { ereport(ERROR, (errmsg("bad metadata, noderole does not exist"), errdetail("you should never see this, please submit " "a bug report"), errhint("run ALTER EXTENSION citus UPDATE and try again"))); } /* generate the query without any values yet */ appendStringInfo(nodeListInsertCommand, "INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, " "noderack, hasmetadata, metadatasynced, isactive, noderole, " "nodecluster, shouldhaveshards, nodeisclone, nodeprimarynodeid) VALUES "); /* iterate over the worker nodes, add the values */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { char *hasMetadataString = workerNode->hasMetadata ? "TRUE" : "FALSE"; char *metadataSyncedString = workerNode->metadataSynced ? "TRUE" : "FALSE"; char *isActiveString = workerNode->isActive ? "TRUE" : "FALSE"; char *shouldHaveShards = workerNode->shouldHaveShards ? "TRUE" : "FALSE"; char *nodeiscloneString = workerNode->nodeisclone ? "TRUE" : "FALSE"; Datum nodeRoleOidDatum = ObjectIdGetDatum(workerNode->nodeRole); Datum nodeRoleStringDatum = DirectFunctionCall1(enum_out, nodeRoleOidDatum); char *nodeRoleString = DatumGetCString(nodeRoleStringDatum); appendStringInfo(nodeListInsertCommand, "(%d, %d, %s, %d, %s, %s, %s, %s, '%s'::noderole, %s, %s, %s, %d)", workerNode->nodeId, workerNode->groupId, quote_literal_cstr(workerNode->workerName), workerNode->workerPort, quote_literal_cstr(workerNode->workerRack), hasMetadataString, metadataSyncedString, isActiveString, nodeRoleString, quote_literal_cstr(workerNode->nodeCluster), shouldHaveShards, nodeiscloneString, workerNode->nodeprimarynodeid); processedWorkerNodeCount++; if (processedWorkerNodeCount != workerCount) { appendStringInfo(nodeListInsertCommand, ","); } } return nodeListInsertCommand->data; } /* * NodeListIdempotentInsertCommand generates an idempotent multi-row INSERT command that * can be executed to insert the nodes that are in workerNodeList to pg_dist_node table. * It would insert new nodes or replace current nodes with new nodes if nodename-nodeport * pairs already exist. */ char * NodeListIdempotentInsertCommand(List *workerNodeList) { StringInfo nodeInsertIdempotentCommand = makeStringInfo(); char *nodeInsertStr = NodeListInsertCommand(workerNodeList); appendStringInfoString(nodeInsertIdempotentCommand, nodeInsertStr); char *onConflictStr = " ON CONFLICT ON CONSTRAINT pg_dist_node_nodename_nodeport_key " "DO UPDATE SET nodeid = EXCLUDED.nodeid, " "groupid = EXCLUDED.groupid, " "nodename = EXCLUDED.nodename, " "nodeport = EXCLUDED.nodeport, " "noderack = EXCLUDED.noderack, " "hasmetadata = EXCLUDED.hasmetadata, " "isactive = EXCLUDED.isactive, " "noderole = EXCLUDED.noderole, " "nodecluster = EXCLUDED.nodecluster, " "metadatasynced = EXCLUDED.metadatasynced, " "shouldhaveshards = EXCLUDED.shouldhaveshards, " "nodeisclone = EXCLUDED.nodeisclone, " "nodeprimarynodeid = EXCLUDED.nodeprimarynodeid"; appendStringInfoString(nodeInsertIdempotentCommand, onConflictStr); return nodeInsertIdempotentCommand->data; } /* * MarkObjectsDistributedCreateCommand generates a command that can be executed to * insert or update the provided objects into pg_dist_object on a worker node. */ char * MarkObjectsDistributedCreateCommand(List *addresses, List *namesArg, List *distributionArgumentIndexes, List *colocationIds, List *forceDelegations) { StringInfo insertDistributedObjectsCommand = makeStringInfo(); Assert(list_length(addresses) == list_length(distributionArgumentIndexes)); Assert(list_length(distributionArgumentIndexes) == list_length(colocationIds)); appendStringInfo(insertDistributedObjectsCommand, "WITH distributed_object_data(typetext, objnames, " "objargs, distargumentindex, colocationid, force_delegation) AS (VALUES "); bool isFirstObject = true; for (int currentObjectCounter = 0; currentObjectCounter < list_length(addresses); currentObjectCounter++) { ObjectAddress *address = list_nth(addresses, currentObjectCounter); int distributionArgumentIndex = list_nth_int(distributionArgumentIndexes, currentObjectCounter); int colocationId = list_nth_int(colocationIds, currentObjectCounter); int forceDelegation = list_nth_int(forceDelegations, currentObjectCounter); List *names = NIL; List *args = NIL; char *objectType = NULL; if (IsMainDBCommand) { /* * When we try to distribute an object that's being created in a non Citus * main database, we cannot find the name, since the object is not visible * in Citus main database. * Because of that we need to pass the name to this function. */ names = list_nth(namesArg, currentObjectCounter); bool missingOk = false; objectType = getObjectTypeDescription(address, missingOk); } else { objectType = getObjectTypeDescription(address, false); getObjectIdentityParts(address, &names, &args, IsMainDBCommand); } if (!isFirstObject) { appendStringInfo(insertDistributedObjectsCommand, ", "); } isFirstObject = false; appendStringInfo(insertDistributedObjectsCommand, "(%s, ARRAY[", quote_literal_cstr(objectType)); char *name = NULL; bool firstInNameLoop = true; foreach_declared_ptr(name, names) { if (!firstInNameLoop) { appendStringInfo(insertDistributedObjectsCommand, ", "); } firstInNameLoop = false; appendStringInfoString(insertDistributedObjectsCommand, quote_literal_cstr(name)); } appendStringInfo(insertDistributedObjectsCommand, "]::text[], ARRAY["); char *arg; bool firstInArgLoop = true; foreach_declared_ptr(arg, args) { if (!firstInArgLoop) { appendStringInfo(insertDistributedObjectsCommand, ", "); } firstInArgLoop = false; appendStringInfoString(insertDistributedObjectsCommand, quote_literal_cstr(arg)); } appendStringInfo(insertDistributedObjectsCommand, "]::text[], "); appendStringInfo(insertDistributedObjectsCommand, "%d, ", distributionArgumentIndex); appendStringInfo(insertDistributedObjectsCommand, "%d, ", colocationId); appendStringInfo(insertDistributedObjectsCommand, "%s)", forceDelegation ? "true" : "false"); } appendStringInfo(insertDistributedObjectsCommand, ") "); appendStringInfo(insertDistributedObjectsCommand, "SELECT citus_internal.add_object_metadata(" "typetext, objnames, objargs, distargumentindex::int, colocationid::int, force_delegation::bool) " "FROM distributed_object_data;"); return insertDistributedObjectsCommand->data; } /* * citus_internal_add_object_metadata is an internal UDF to * add a row to pg_dist_object. */ Datum citus_internal_add_object_metadata(PG_FUNCTION_ARGS) { char *textType = TextDatumGetCString(PG_GETARG_DATUM(0)); ArrayType *nameArray = PG_GETARG_ARRAYTYPE_P(1); ArrayType *argsArray = PG_GETARG_ARRAYTYPE_P(2); int distributionArgumentIndex = PG_GETARG_INT32(3); int colocationId = PG_GETARG_INT32(4); bool forceDelegation = PG_GETARG_INT32(5); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); /* * Ensure given distributionArgumentIndex and colocationId values are * sane. Since we check sanity of object related parameters within * PgGetObjectAddress below, we are not checking them here. */ EnsureObjectMetadataIsSane(distributionArgumentIndex, colocationId); } /* * We check the acl/ownership while getting the object address. That * funtion also checks the sanity of given textType, nameArray and * argsArray parameters */ ObjectAddress objectAddress = PgGetObjectAddress(textType, nameArray, argsArray); /* First, disable propagation off to not to cause infinite propagation */ bool prevDependencyCreationValue = EnableMetadataSync; SetLocalEnableMetadataSync(false); MarkObjectDistributed(&objectAddress); if (distributionArgumentIndex != INVALID_DISTRIBUTION_ARGUMENT_INDEX || colocationId != INVALID_COLOCATION_ID) { int *distributionArgumentIndexAddress = distributionArgumentIndex == INVALID_DISTRIBUTION_ARGUMENT_INDEX ? NULL : &distributionArgumentIndex; int *colocationIdAddress = colocationId == INVALID_COLOCATION_ID ? NULL : &colocationId; bool *forceDelegationAddress = forceDelegation == false ? NULL : &forceDelegation; UpdateFunctionDistributionInfo(&objectAddress, distributionArgumentIndexAddress, colocationIdAddress, forceDelegationAddress); } SetLocalEnableMetadataSync(prevDependencyCreationValue); PG_RETURN_VOID(); } /* * EnsureObjectMetadataIsSane checks whether the distribution argument index and * colocation id metadata params for distributed object is sane. You can look * PgGetObjectAddress to find checks related to object sanity. */ static void EnsureObjectMetadataIsSane(int distributionArgumentIndex, int colocationId) { if (distributionArgumentIndex != INVALID_DISTRIBUTION_ARGUMENT_INDEX) { if (distributionArgumentIndex < 0 || distributionArgumentIndex > FUNC_MAX_ARGS) { ereport(ERROR, errmsg("distribution_argument_index must be between" " 0 and %d", FUNC_MAX_ARGS)); } } if (colocationId != INVALID_COLOCATION_ID) { if (colocationId < 0) { ereport(ERROR, errmsg("colocationId must be a positive number")); } } } /* * DistributionCreateCommands generates a commands that can be * executed to replicate the metadata for a Citus table. */ char * DistributionCreateCommand(CitusTableCacheEntry *cacheEntry) { StringInfo insertDistributionCommand = makeStringInfo(); Oid relationId = cacheEntry->relationId; char distributionMethod = cacheEntry->partitionMethod; char *qualifiedRelationName = generate_qualified_relation_name(relationId); uint32 colocationId = cacheEntry->colocationId; char replicationModel = cacheEntry->replicationModel; StringInfo tablePartitionKeyNameString = makeStringInfo(); if (!HasDistributionKeyCacheEntry(cacheEntry)) { appendStringInfo(tablePartitionKeyNameString, "NULL"); } else { char *partitionKeyColumnName = ColumnToColumnName(relationId, (Node *) cacheEntry->partitionColumn); appendStringInfo(tablePartitionKeyNameString, "%s", quote_literal_cstr(partitionKeyColumnName)); } appendStringInfo(insertDistributionCommand, "SELECT citus_internal.add_partition_metadata " "(%s::regclass, '%c', %s, %d, '%c')", quote_literal_cstr(qualifiedRelationName), distributionMethod, tablePartitionKeyNameString->data, colocationId, replicationModel); return insertDistributionCommand->data; } /* * DistributionDeleteCommand generates a command that can be executed * to drop a distributed table and its metadata on a remote node. */ char * DistributionDeleteCommand(const char *schemaName, const char *tableName) { StringInfo deleteDistributionCommand = makeStringInfo(); char *distributedRelationName = quote_qualified_identifier(schemaName, tableName); appendStringInfo(deleteDistributionCommand, "SELECT worker_drop_distributed_table(%s)", quote_literal_cstr(distributedRelationName)); return deleteDistributionCommand->data; } /* * DistributionDeleteMetadataCommand returns a query to delete pg_dist_partition * metadata from a worker node for a given table. */ char * DistributionDeleteMetadataCommand(Oid relationId) { StringInfo deleteCommand = makeStringInfo(); char *qualifiedRelationName = generate_qualified_relation_name(relationId); appendStringInfo(deleteCommand, "SELECT citus_internal.delete_partition_metadata(%s)", quote_literal_cstr(qualifiedRelationName)); return deleteCommand->data; } /* * TableOwnerResetCommand generates a commands that can be executed * to reset the table owner. */ char * TableOwnerResetCommand(Oid relationId) { StringInfo ownerResetCommand = makeStringInfo(); char *qualifiedRelationName = generate_qualified_relation_name(relationId); char *tableOwnerName = TableOwner(relationId); appendStringInfo(ownerResetCommand, "ALTER TABLE %s OWNER TO %s", qualifiedRelationName, quote_identifier(tableOwnerName)); return ownerResetCommand->data; } /* * ShardListInsertCommand generates a single command that can be * executed to replicate shard and shard placement metadata for the * given shard intervals. The function assumes that each shard has a * single placement, and asserts this information. */ List * ShardListInsertCommand(List *shardIntervalList) { List *commandList = NIL; int shardCount = list_length(shardIntervalList); /* if there are no shards, return empty list */ if (shardCount == 0) { return commandList; } /* add placements to insertPlacementCommand */ StringInfo insertPlacementCommand = makeStringInfo(); appendStringInfo(insertPlacementCommand, "WITH placement_data(shardid, " "shardlength, groupid, placementid) AS (VALUES "); ShardInterval *shardInterval = NULL; bool firstPlacementProcessed = false; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; List *shardPlacementList = ActiveShardPlacementList(shardId); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacementList) { if (firstPlacementProcessed) { /* * As long as this is not the first placement of the first shard, * append the comma. */ appendStringInfo(insertPlacementCommand, ", "); } firstPlacementProcessed = true; appendStringInfo(insertPlacementCommand, "(%ld, %ld, %d, %ld)", shardId, placement->shardLength, placement->groupId, placement->placementId); } } appendStringInfo(insertPlacementCommand, ") "); appendStringInfo(insertPlacementCommand, "SELECT citus_internal.add_placement_metadata(" "shardid, shardlength, groupid, placementid) " "FROM placement_data;"); /* now add shards to insertShardCommand */ StringInfo insertShardCommand = makeStringInfo(); appendStringInfo(insertShardCommand, "WITH shard_data(relationname, shardid, storagetype, " "shardminvalue, shardmaxvalue) AS (VALUES "); foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; Oid distributedRelationId = shardInterval->relationId; char *qualifiedRelationName = generate_qualified_relation_name( distributedRelationId); StringInfo minHashToken = makeStringInfo(); StringInfo maxHashToken = makeStringInfo(); if (shardInterval->minValueExists) { appendStringInfo(minHashToken, "'%d'", DatumGetInt32( shardInterval->minValue)); } else { appendStringInfo(minHashToken, "NULL"); } if (shardInterval->maxValueExists) { appendStringInfo(maxHashToken, "'%d'", DatumGetInt32( shardInterval->maxValue)); } else { appendStringInfo(maxHashToken, "NULL"); } appendStringInfo(insertShardCommand, "(%s::regclass, %ld, '%c'::\"char\", %s, %s)", quote_literal_cstr(qualifiedRelationName), shardId, shardInterval->storageType, minHashToken->data, maxHashToken->data); if (llast(shardIntervalList) != shardInterval) { appendStringInfo(insertShardCommand, ", "); } } appendStringInfo(insertShardCommand, ") "); appendStringInfo(insertShardCommand, "SELECT citus_internal.add_shard_metadata(relationname, shardid, " "storagetype, shardminvalue, shardmaxvalue) " "FROM shard_data;"); /* * There are no active placements for the table, so do not create the * command as it'd lead to syntax error. * * This is normally not an expected situation, however the current * implementation of citus_disable_node allows to disable nodes with * the only active placements. So, for example a single shard/placement * distributed table on a disabled node might trigger zero placement * case. * * TODO: remove this check once citus_disable_node errors out for * the above scenario. */ if (firstPlacementProcessed) { /* first insert shards, than the placements */ commandList = lappend(commandList, insertShardCommand->data); commandList = lappend(commandList, insertPlacementCommand->data); } return commandList; } /* * ShardListDeleteCommand generates a command list that can be executed to delete * shard and shard placement metadata for the given shard. */ List * ShardDeleteCommandList(ShardInterval *shardInterval) { uint64 shardId = shardInterval->shardId; StringInfo deleteShardCommand = makeStringInfo(); appendStringInfo(deleteShardCommand, "SELECT citus_internal.delete_shard_metadata(%ld);", shardId); return list_make1(deleteShardCommand->data); } /* * NodeDeleteCommand generate a command that can be * executed to delete the metadata for a worker node. */ char * NodeDeleteCommand(uint32 nodeId) { StringInfo nodeDeleteCommand = makeStringInfo(); appendStringInfo(nodeDeleteCommand, "DELETE FROM pg_dist_node " "WHERE nodeid = %u", nodeId); return nodeDeleteCommand->data; } /* * NodeStateUpdateCommand generates a command that can be executed to update * isactive column of a node in pg_dist_node table. */ char * NodeStateUpdateCommand(uint32 nodeId, bool isActive) { StringInfo nodeStateUpdateCommand = makeStringInfo(); char *isActiveString = isActive ? "TRUE" : "FALSE"; appendStringInfo(nodeStateUpdateCommand, "UPDATE pg_dist_node SET isactive = %s " "WHERE nodeid = %u", isActiveString, nodeId); return nodeStateUpdateCommand->data; } /* * ShouldHaveShardsUpdateCommand generates a command that can be executed to * update the shouldhaveshards column of a node in pg_dist_node table. */ char * ShouldHaveShardsUpdateCommand(uint32 nodeId, bool shouldHaveShards) { StringInfo nodeStateUpdateCommand = makeStringInfo(); char *shouldHaveShardsString = shouldHaveShards ? "TRUE" : "FALSE"; appendStringInfo(nodeStateUpdateCommand, "UPDATE pg_catalog.pg_dist_node SET shouldhaveshards = %s " "WHERE nodeid = %u", shouldHaveShardsString, nodeId); return nodeStateUpdateCommand->data; } /* * ColocationIdUpdateCommand creates the SQL command to change the colocationId * of the table with the given name to the given colocationId in pg_dist_partition * table. */ char * ColocationIdUpdateCommand(Oid relationId, uint32 colocationId) { StringInfo command = makeStringInfo(); char *qualifiedRelationName = generate_qualified_relation_name(relationId); appendStringInfo(command, "SELECT citus_internal.update_relation_colocation(%s::regclass, %d)", quote_literal_cstr(qualifiedRelationName), colocationId); return command->data; } /* * PlacementUpsertCommand creates a SQL command for upserting a pg_dist_placment * entry with the given properties. In the case of a conflict on placementId, the command * updates all properties (excluding the placementId) with the given ones. */ char * PlacementUpsertCommand(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId) { StringInfo command = makeStringInfo(); appendStringInfo(command, UPSERT_PLACEMENT, shardId, shardLength, groupId, placementId); return command->data; } /* * LocalGroupIdUpdateCommand creates the SQL command required to set the local group id * of a worker and returns the command in a string. */ char * LocalGroupIdUpdateCommand(int32 groupId) { StringInfo updateCommand = makeStringInfo(); appendStringInfo(updateCommand, "UPDATE pg_dist_local_group SET groupid = %d", groupId); return updateCommand->data; } /* * DDLCommandsForSequence returns the DDL commands needs to be run to create the * sequence and alter the owner to the given owner name. */ List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName) { List *sequenceDDLList = NIL; char *sequenceDef = pg_get_sequencedef_string(sequenceOid); char *escapedSequenceDef = quote_literal_cstr(sequenceDef); StringInfo wrappedSequenceDef = makeStringInfo(); StringInfo sequenceGrantStmt = makeStringInfo(); char *sequenceName = generate_qualified_relation_name(sequenceOid); Form_pg_sequence sequenceData = pg_get_sequencedef(sequenceOid); Oid sequenceTypeOid = sequenceData->seqtypid; char *typeName = format_type_be(sequenceTypeOid); /* create schema if needed */ appendStringInfo(wrappedSequenceDef, WORKER_APPLY_SEQUENCE_COMMAND, escapedSequenceDef, quote_literal_cstr(typeName)); appendStringInfo(sequenceGrantStmt, "ALTER SEQUENCE %s OWNER TO %s", sequenceName, quote_identifier(ownerName)); sequenceDDLList = lappend(sequenceDDLList, wrappedSequenceDef->data); sequenceDDLList = lappend(sequenceDDLList, sequenceGrantStmt->data); sequenceDDLList = list_concat(sequenceDDLList, GrantOnSequenceDDLCommands( sequenceOid)); return sequenceDDLList; } /* * GetAttributeTypeOid returns the OID of the type of the attribute of * provided relationId that has the provided attnum */ Oid GetAttributeTypeOid(Oid relationId, AttrNumber attnum) { Oid resultOid = InvalidOid; ScanKeyData key[2]; /* Grab an appropriate lock on the pg_attribute relation */ Relation attrel = table_open(AttributeRelationId, AccessShareLock); /* Use the index to scan only system attributes of the target relation */ ScanKeyInit(&key[0], Anum_pg_attribute_attrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); ScanKeyInit(&key[1], Anum_pg_attribute_attnum, BTLessEqualStrategyNumber, F_INT2LE, Int16GetDatum(attnum)); SysScanDesc scan = systable_beginscan(attrel, AttributeRelidNumIndexId, true, NULL, 2, key); HeapTuple attributeTuple; while (HeapTupleIsValid(attributeTuple = systable_getnext(scan))) { Form_pg_attribute att = (Form_pg_attribute) GETSTRUCT(attributeTuple); resultOid = att->atttypid; } systable_endscan(scan); table_close(attrel, AccessShareLock); return resultOid; } /* * GetDependentSequencesWithRelation appends the attnum and id of sequences that * have direct (owned sequences) or indirect dependency with the given relationId, * to the lists passed as NIL initially. * For both cases, we use the intermediate AttrDefault object from pg_depend. * If attnum is specified, we only return the sequences related to that * attribute of the relationId. * See DependencyType for the possible values of depType. * We use DEPENDENCY_INTERNAL for sequences created by identity column. * DEPENDENCY_AUTO for regular sequences. */ void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, AttrNumber attnum, char depType) { Assert(*seqInfoList == NIL); List *attrdefResult = NIL; List *attrdefAttnumResult = NIL; ScanKeyData key[3]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); if (attnum) { ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); } SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if (deprec->classid == AttrDefaultRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && deprec->deptype == depType) { /* * We are going to generate corresponding SequenceInfo * in the following loop. */ attrdefResult = lappend_oid(attrdefResult, deprec->objid); attrdefAttnumResult = lappend_int(attrdefAttnumResult, deprec->refobjsubid); } else if (deprec->deptype == depType && deprec->refobjsubid != 0 && deprec->classid == RelationRelationId && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { SequenceInfo *seqInfo = (SequenceInfo *) palloc(sizeof(SequenceInfo)); seqInfo->sequenceOid = deprec->objid; seqInfo->attributeNumber = deprec->refobjsubid; seqInfo->isNextValDefault = false; *seqInfoList = lappend(*seqInfoList, seqInfo); } } systable_endscan(scan); table_close(depRel, AccessShareLock); AttrNumber attrdefAttnum = InvalidAttrNumber; Oid attrdefOid = InvalidOid; forboth_int_oid(attrdefAttnum, attrdefAttnumResult, attrdefOid, attrdefResult) { List *sequencesFromAttrDef = GetSequencesFromAttrDef(attrdefOid); /* to simplify and eliminate cases like "DEFAULT nextval('..') - nextval('..')" */ if (list_length(sequencesFromAttrDef) > 1) { ereport(ERROR, (errmsg( "More than one sequence in a column default" " is not supported for distribution " "or for adding local tables to metadata"))); } if (list_length(sequencesFromAttrDef) == 1) { SequenceInfo *seqInfo = (SequenceInfo *) palloc(sizeof(SequenceInfo)); seqInfo->sequenceOid = linitial_oid(sequencesFromAttrDef); seqInfo->attributeNumber = attrdefAttnum; seqInfo->isNextValDefault = true; *seqInfoList = lappend(*seqInfoList, seqInfo); } } } /* * GetDependentDependentRelationsWithSequence returns a list of oids of * relations that have have a dependency on the given sequence. * There are three types of dependencies: * 1. direct auto (owned sequences), created using SERIAL or BIGSERIAL * 2. indirect auto (through an AttrDef), created using DEFAULT nextval('..') * 3. internal, created using GENERATED ALWAYS AS IDENTITY * * Depending on the passed deptype, we return the relations that have the * given type(s): * - DEPENDENCY_AUTO returns both 1 and 2 * - DEPENDENCY_INTERNAL returns 3 * * The returned list can contain duplicates, as the same relation can have * multiple dependencies on the sequence. */ List * GetDependentRelationsWithSequence(Oid sequenceOid, char depType) { List *relations = NIL; ScanKeyData key[2]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(sequenceOid)); SysScanDesc scan = systable_beginscan(depRel, DependDependerIndexId, true, NULL, lengthof(key), key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if ( deprec->refclassid == RelationRelationId && deprec->refobjsubid != 0 && deprec->deptype == depType) { relations = lappend_oid(relations, deprec->refobjid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); if (depType == DEPENDENCY_AUTO) { Oid attrDefOid; List *attrDefOids = GetAttrDefsFromSequence(sequenceOid); foreach_declared_oid(attrDefOid, attrDefOids) { ObjectAddress columnAddress = GetAttrDefaultColumnAddress(attrDefOid); relations = lappend_oid(relations, columnAddress.objectId); } } return relations; } /* * GetSequencesFromAttrDef returns a list of sequence OIDs that have * dependency with the given attrdefOid in pg_depend */ List * GetSequencesFromAttrDef(Oid attrdefOid) { List *sequencesResult = NIL; ScanKeyData key[2]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(AttrDefaultRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(attrdefOid)); SysScanDesc scan = systable_beginscan(depRel, DependDependerIndexId, true, NULL, 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if (deprec->refclassid == RelationRelationId && deprec->deptype == DEPENDENCY_NORMAL && get_rel_relkind(deprec->refobjid) == RELKIND_SEQUENCE) { sequencesResult = lappend_oid(sequencesResult, deprec->refobjid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return sequencesResult; } /* * GetAttrDefsFromSequence returns a list of attrdef OIDs that have * a dependency on the given sequence */ List * GetAttrDefsFromSequence(Oid seqOid) { List *attrDefsResult = NIL; ScanKeyData key[2]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(seqOid)); SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, lengthof(key), key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if (deprec->classid == AttrDefaultRelationId && deprec->deptype == DEPENDENCY_NORMAL) { attrDefsResult = lappend_oid(attrDefsResult, deprec->objid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return attrDefsResult; } /* * GetDependentFunctionsWithRelation returns the dependent functions for the * given relation id. */ List * GetDependentFunctionsWithRelation(Oid relationId) { List *referencingObjects = NIL; List *functionOids = NIL; ScanKeyData key[2]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* * objsubid is nonzero only for table columns and zero for anything else. * Since we are trying to find a dependency from the column of a table to * function we've added deprec->refobjsubid != 0 check. * * We are following DEPENDENCY_AUTO for dependencies via column and * DEPENDENCY_NORMAL anything else. Since only procedure dependencies * for those dependencies will be obtained in GetFunctionDependenciesForObjects * following both dependency types are not harmful. */ if ((deprec->refobjsubid != 0 && deprec->deptype == DEPENDENCY_AUTO) || deprec->deptype == DEPENDENCY_NORMAL) { ObjectAddress *refAddress = palloc(sizeof(ObjectAddress)); ObjectAddressSubSet(*refAddress, deprec->classid, deprec->objid, deprec->objsubid); referencingObjects = lappend(referencingObjects, refAddress); } } systable_endscan(scan); table_close(depRel, AccessShareLock); ObjectAddress *referencingObject = NULL; foreach_declared_ptr(referencingObject, referencingObjects) { functionOids = list_concat(functionOids, GetFunctionDependenciesForObjects(referencingObject)); } return functionOids; } /* * GetFunctionDependenciesForObjects returns a list of function OIDs that have * dependency with the given object */ static List * GetFunctionDependenciesForObjects(ObjectAddress *objectAddress) { List *functionOids = NIL; ScanKeyData key[3]; HeapTuple tup; Relation depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(objectAddress->classId)); ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(objectAddress->objectId)); ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(objectAddress->objectSubId)); SysScanDesc scan = systable_beginscan(depRel, DependDependerIndexId, true, NULL, 3, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); if (deprec->refclassid == ProcedureRelationId) { functionOids = lappend_oid(functionOids, deprec->refobjid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return functionOids; } /* * SequenceDependencyCommandList generates commands to record the dependency * of sequences on tables on the worker. This dependency does not exist by * default since the sequences and table are created separately, but it is * necessary to ensure that the sequence is dropped when the table is * dropped. */ List * SequenceDependencyCommandList(Oid relationId) { List *sequenceCommandList = NIL; List *columnNameList = NIL; List *sequenceIdList = NIL; ExtractDefaultColumnsAndOwnedSequences(relationId, &columnNameList, &sequenceIdList); char *columnName = NULL; Oid sequenceId = InvalidOid; forboth_ptr_oid(columnName, columnNameList, sequenceId, sequenceIdList) { if (!OidIsValid(sequenceId)) { /* * ExtractDefaultColumnsAndOwnedSequences returns entries for all columns, * but with 0 sequence ID unless there is default nextval(..). */ continue; } char *sequenceDependencyCommand = CreateSequenceDependencyCommand(relationId, sequenceId, columnName); sequenceCommandList = lappend(sequenceCommandList, makeTableDDLCommandString( sequenceDependencyCommand)); } return sequenceCommandList; } /* * IdentitySequenceDependencyCommandList generate a command to execute * a UDF (WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES) on workers to modify the identity * columns min/max values to produce unique values on workers. */ List * IdentitySequenceDependencyCommandList(Oid targetRelationId) { List *commandList = NIL; Relation relation = relation_open(targetRelationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); bool tableHasIdentityColumn = false; for (int attributeIndex = 0; attributeIndex < tupleDescriptor->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, attributeIndex); if (attributeForm->attidentity) { tableHasIdentityColumn = true; break; } } relation_close(relation, NoLock); if (tableHasIdentityColumn) { StringInfo stringInfo = makeStringInfo(); char *tableName = generate_qualified_relation_name(targetRelationId); appendStringInfo(stringInfo, WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES, quote_literal_cstr(tableName)); commandList = lappend(commandList, makeTableDDLCommandString( stringInfo->data)); } return commandList; } /* * CreateSequenceDependencyCommand generates a query string for calling * worker_record_sequence_dependency on the worker to recreate a sequence->table * dependency. */ static char * CreateSequenceDependencyCommand(Oid relationId, Oid sequenceId, char *columnName) { char *relationName = generate_qualified_relation_name(relationId); char *sequenceName = generate_qualified_relation_name(sequenceId); StringInfo sequenceDependencyCommand = makeStringInfo(); appendStringInfo(sequenceDependencyCommand, "SELECT pg_catalog.worker_record_sequence_dependency" "(%s::regclass,%s::regclass,%s)", quote_literal_cstr(sequenceName), quote_literal_cstr(relationName), quote_literal_cstr(columnName)); return sequenceDependencyCommand->data; } /* * worker_record_sequence_dependency records the fact that the sequence depends on * the table in pg_depend, such that it will be automatically dropped. */ Datum worker_record_sequence_dependency(PG_FUNCTION_ARGS) { Oid sequenceOid = PG_GETARG_OID(0); Oid relationOid = PG_GETARG_OID(1); Name columnName = PG_GETARG_NAME(2); const char *columnNameStr = NameStr(*columnName); /* lookup column definition */ HeapTuple columnTuple = SearchSysCacheAttName(relationOid, columnNameStr); if (!HeapTupleIsValid(columnTuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", columnNameStr))); } Form_pg_attribute columnForm = (Form_pg_attribute) GETSTRUCT(columnTuple); if (columnForm->attnum <= 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create dependency on system column \"%s\"", columnNameStr))); } ObjectAddress sequenceAddr = { .classId = RelationRelationId, .objectId = sequenceOid, .objectSubId = 0 }; ObjectAddress relationAddr = { .classId = RelationRelationId, .objectId = relationOid, .objectSubId = columnForm->attnum }; EnsureTableOwner(sequenceOid); EnsureTableOwner(relationOid); /* dependency from sequence to table */ recordDependencyOn(&sequenceAddr, &relationAddr, DEPENDENCY_AUTO); ReleaseSysCache(columnTuple); PG_RETURN_VOID(); } /* * CreateSchemaDDLCommand returns a "CREATE SCHEMA..." SQL string for creating the given * schema if not exists and with proper authorization. */ char * CreateSchemaDDLCommand(Oid schemaId) { char *schemaName = get_namespace_name(schemaId); StringInfo schemaNameDef = makeStringInfo(); const char *quotedSchemaName = quote_identifier(schemaName); const char *ownerName = quote_identifier(SchemaOwnerName(schemaId)); appendStringInfo(schemaNameDef, CREATE_SCHEMA_COMMAND, quotedSchemaName, ownerName); return schemaNameDef->data; } /* * GrantOnSchemaDDLCommands creates a list of ddl command for replicating the permissions * of roles on schemas. */ List * GrantOnSchemaDDLCommands(Oid schemaOid) { HeapTuple schemaTuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(schemaOid)); bool isNull = true; Datum aclDatum = SysCacheGetAttr(NAMESPACEOID, schemaTuple, Anum_pg_namespace_nspacl, &isNull); if (isNull) { ReleaseSysCache(schemaTuple); return NIL; } Acl *acl = DatumGetAclPCopy(aclDatum); AclItem *aclDat = ACL_DAT(acl); int aclNum = ACL_NUM(acl); List *commands = NIL; ReleaseSysCache(schemaTuple); for (int i = 0; i < aclNum; i++) { commands = list_concat(commands, GenerateGrantOnSchemaQueriesFromAclItem( schemaOid, &aclDat[i])); } return commands; } /* * GenerateGrantOnSchemaQueryFromACLItem generates a query string for replicating a users permissions * on a schema. */ List * GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem) { AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_SCHEMA; AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_SCHEMA; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_USAGE) || (permissions & ACL_USAGE)); Assert(!(grants & ACL_CREATE) || (permissions & ACL_CREATE)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); if (permissions & ACL_USAGE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_SCHEMA, granteeOid, schemaOid, "USAGE", grants & ACL_USAGE)); queries = lappend(queries, query); } if (permissions & ACL_CREATE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_SCHEMA, granteeOid, schemaOid, "CREATE", grants & ACL_CREATE)); queries = lappend(queries, query); } queries = lappend(queries, "RESET ROLE"); return queries; } /* * GrantOnDatabaseDDLCommands creates a list of ddl command for replicating the permissions * of roles on databases. */ List * GrantOnDatabaseDDLCommands(Oid databaseOid) { HeapTuple databaseTuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(databaseOid)); bool isNull = true; Datum aclDatum = SysCacheGetAttr(DATABASEOID, databaseTuple, Anum_pg_database_datacl, &isNull); if (isNull) { ReleaseSysCache(databaseTuple); return NIL; } Acl *acl = DatumGetAclPCopy(aclDatum); AclItem *aclDat = ACL_DAT(acl); int aclNum = ACL_NUM(acl); List *commands = NIL; ReleaseSysCache(databaseTuple); for (int i = 0; i < aclNum; i++) { commands = list_concat(commands, GenerateGrantOnDatabaseFromAclItem( databaseOid, &aclDat[i])); } return commands; } /* * GenerateGrantOnDatabaseFromAclItem generates a query string for replicating a users permissions * on a database. */ List * GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem) { AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_DATABASE; AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_DATABASE; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_CONNECT) || (permissions & ACL_CONNECT)); Assert(!(grants & ACL_CREATE) || (permissions & ACL_CREATE)); Assert(!(grants & ACL_CREATE_TEMP) || (permissions & ACL_CREATE_TEMP)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); if (permissions & ACL_CONNECT) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_DATABASE, granteeOid, databaseOid, "CONNECT", grants & ACL_CONNECT)); queries = lappend(queries, query); } if (permissions & ACL_CREATE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_DATABASE, granteeOid, databaseOid, "CREATE", grants & ACL_CREATE)); queries = lappend(queries, query); } if (permissions & ACL_CREATE_TEMP) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_DATABASE, granteeOid, databaseOid, "TEMPORARY", grants & ACL_CREATE_TEMP)); queries = lappend(queries, query); } queries = lappend(queries, "RESET ROLE"); return queries; } /* * GenerateGrantStmtForRights is the function for creating GrantStmt's for all * types of objects that are supported. It takes parameters to fill a GrantStmt's * fields and returns the GrantStmt. * The field `objects` of GrantStmt doesn't have a common structure for all types. * Make sure you have added your object type to GetObjectsForGrantStmt. */ static GrantStmt * GenerateGrantStmtForRights(ObjectType objectType, Oid roleOid, Oid objectId, char *permission, bool withGrantOption) { GrantStmt *stmt = makeNode(GrantStmt); stmt->is_grant = true; stmt->targtype = ACL_TARGET_OBJECT; stmt->objtype = objectType; stmt->objects = GetObjectsForGrantStmt(objectType, objectId); stmt->privileges = list_make1(GetAccessPrivObjectForGrantStmt(permission)); stmt->grantees = list_make1(GetRoleSpecObjectForUser(roleOid)); stmt->grant_option = withGrantOption; return stmt; } /* * GetObjectsForGrantStmt takes an object type and object id and returns the 'objects' * field to be used when creating GrantStmt. We have only one object here (the one with * the oid = objectId) but we pass it into the GrantStmt as a list with one element, * as GrantStmt->objects field is actually a list. */ static List * GetObjectsForGrantStmt(ObjectType objectType, Oid objectId) { switch (objectType) { /* supported object types */ case OBJECT_SCHEMA: { return list_make1(makeString(get_namespace_name(objectId))); } /* enterprise supported object types */ case OBJECT_FUNCTION: case OBJECT_AGGREGATE: case OBJECT_PROCEDURE: { ObjectWithArgs *owa = ObjectWithArgsFromOid(objectId); return list_make1(owa); } case OBJECT_FDW: { ForeignDataWrapper *fdw = GetForeignDataWrapper(objectId); return list_make1(makeString(fdw->fdwname)); } case OBJECT_FOREIGN_SERVER: { ForeignServer *server = GetForeignServer(objectId); return list_make1(makeString(server->servername)); } case OBJECT_SEQUENCE: { Oid namespaceOid = get_rel_namespace(objectId); RangeVar *sequence = makeRangeVar(get_namespace_name(namespaceOid), get_rel_name(objectId), -1); return list_make1(sequence); } case OBJECT_DATABASE: { return list_make1(makeString(get_database_name(objectId))); } default: { elog(ERROR, "unsupported object type for GRANT"); } } return NIL; } /* * GrantOnFunctionDDLCommands creates a list of ddl command for replicating the permissions * of roles on distributed functions. */ List * GrantOnFunctionDDLCommands(Oid functionOid) { HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); bool isNull = true; Datum aclDatum = SysCacheGetAttr(PROCOID, proctup, Anum_pg_proc_proacl, &isNull); if (isNull) { ReleaseSysCache(proctup); return NIL; } Acl *acl = DatumGetAclPCopy(aclDatum); AclItem *aclDat = ACL_DAT(acl); int aclNum = ACL_NUM(acl); List *commands = NIL; ReleaseSysCache(proctup); for (int i = 0; i < aclNum; i++) { commands = list_concat(commands, GenerateGrantOnFunctionQueriesFromAclItem( functionOid, &aclDat[i])); } return commands; } /* * GrantOnForeignServerDDLCommands creates a list of ddl command for replicating the * permissions of roles on distributed foreign servers. */ List * GrantOnForeignServerDDLCommands(Oid serverId) { HeapTuple servertup = SearchSysCache1(FOREIGNSERVEROID, ObjectIdGetDatum(serverId)); bool isNull = true; Datum aclDatum = SysCacheGetAttr(FOREIGNSERVEROID, servertup, Anum_pg_foreign_server_srvacl, &isNull); if (isNull) { ReleaseSysCache(servertup); return NIL; } Acl *aclEntry = DatumGetAclPCopy(aclDatum); AclItem *privileges = ACL_DAT(aclEntry); int numberOfPrivsGranted = ACL_NUM(aclEntry); List *commands = NIL; ReleaseSysCache(servertup); for (int i = 0; i < numberOfPrivsGranted; i++) { commands = list_concat(commands, GenerateGrantOnForeignServerQueriesFromAclItem( serverId, &privileges[i])); } return commands; } /* * GenerateGrantOnForeignServerQueriesFromAclItem generates a query string for * replicating a users permissions on a foreign server. */ List * GenerateGrantOnForeignServerQueriesFromAclItem(Oid serverId, AclItem *aclItem) { /* privileges to be granted */ AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_FOREIGN_SERVER; /* WITH GRANT OPTION clause */ AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_FOREIGN_SERVER; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_USAGE) || (permissions & ACL_USAGE)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; /* switch to the role which had granted acl */ queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); /* generate the GRANT stmt that will be executed by the grantor role */ if (permissions & ACL_USAGE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_FOREIGN_SERVER, granteeOid, serverId, "USAGE", grants & ACL_USAGE)); queries = lappend(queries, query); } /* reset the role back */ queries = lappend(queries, "RESET ROLE"); return queries; } /* * GenerateGrantOnFunctionQueryFromACLItem generates a query string for replicating a users permissions * on a distributed function. */ List * GenerateGrantOnFunctionQueriesFromAclItem(Oid functionOid, AclItem *aclItem) { AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_FUNCTION; AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_FUNCTION; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_EXECUTE) || (permissions & ACL_EXECUTE)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); if (permissions & ACL_EXECUTE) { char prokind = get_func_prokind(functionOid); ObjectType objectType; if (prokind == PROKIND_FUNCTION) { objectType = OBJECT_FUNCTION; } else if (prokind == PROKIND_PROCEDURE) { objectType = OBJECT_PROCEDURE; } else if (prokind == PROKIND_AGGREGATE) { objectType = OBJECT_AGGREGATE; } else { ereport(ERROR, (errmsg("unsupported prokind"), errdetail("GRANT commands on procedures are propagated only " "for procedures, functions, and aggregates."))); } char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( objectType, granteeOid, functionOid, "EXECUTE", grants & ACL_EXECUTE)); queries = lappend(queries, query); } queries = lappend(queries, "RESET ROLE"); return queries; } /* * GenerateGrantOnFDWQueriesFromAclItem generates a query string for * replicating a users permissions on a foreign data wrapper. */ List * GenerateGrantOnFDWQueriesFromAclItem(Oid FDWId, AclItem *aclItem) { /* privileges to be granted */ AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_FDW; /* WITH GRANT OPTION clause */ AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_FDW; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_USAGE) || (permissions & ACL_USAGE)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; /* switch to the role which had granted acl */ queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); /* generate the GRANT stmt that will be executed by the grantor role */ if (permissions & ACL_USAGE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_FDW, granteeOid, FDWId, "USAGE", grants & ACL_USAGE)); queries = lappend(queries, query); } /* reset the role back */ queries = lappend(queries, "RESET ROLE"); return queries; } /* * GetAccessPrivObjectForGrantStmt creates an AccessPriv object for the given permission. * It will be used when creating GrantStmt objects. */ static AccessPriv * GetAccessPrivObjectForGrantStmt(char *permission) { AccessPriv *accessPriv = makeNode(AccessPriv); accessPriv->priv_name = pstrdup(permission); accessPriv->cols = NULL; return accessPriv; } /* * GrantOnSequenceDDLCommands creates a list of ddl command for replicating the permissions * of roles on distributed sequences. */ static List * GrantOnSequenceDDLCommands(Oid sequenceOid) { HeapTuple seqtup = SearchSysCache1(RELOID, ObjectIdGetDatum(sequenceOid)); bool isNull = false; Datum aclDatum = SysCacheGetAttr(RELOID, seqtup, Anum_pg_class_relacl, &isNull); if (isNull) { ReleaseSysCache(seqtup); return NIL; } Acl *acl = DatumGetAclPCopy(aclDatum); AclItem *aclDat = ACL_DAT(acl); int aclNum = ACL_NUM(acl); List *commands = NIL; ReleaseSysCache(seqtup); for (int i = 0; i < aclNum; i++) { commands = list_concat(commands, GenerateGrantOnSequenceQueriesFromAclItem( sequenceOid, &aclDat[i])); } return commands; } /* * GenerateGrantOnSequenceQueriesFromAclItem generates a query string for replicating a users permissions * on a distributed sequence. */ static List * GenerateGrantOnSequenceQueriesFromAclItem(Oid sequenceOid, AclItem *aclItem) { AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_SEQUENCE; AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_SEQUENCE; /* * seems unlikely but we check if there is a grant option in the list without the actual permission */ Assert(!(grants & ACL_USAGE) || (permissions & ACL_USAGE)); Assert(!(grants & ACL_SELECT) || (permissions & ACL_SELECT)); Assert(!(grants & ACL_UPDATE) || (permissions & ACL_UPDATE)); Oid granteeOid = aclItem->ai_grantee; List *queries = NIL; queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); if (permissions & ACL_USAGE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_SEQUENCE, granteeOid, sequenceOid, "USAGE", grants & ACL_USAGE)); queries = lappend(queries, query); } if (permissions & ACL_SELECT) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_SEQUENCE, granteeOid, sequenceOid, "SELECT", grants & ACL_SELECT)); queries = lappend(queries, query); } if (permissions & ACL_UPDATE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( OBJECT_SEQUENCE, granteeOid, sequenceOid, "UPDATE", grants & ACL_UPDATE)); queries = lappend(queries, query); } queries = lappend(queries, "RESET ROLE"); return queries; } /* * SetLocalEnableMetadataSync sets the enable_metadata_sync locally */ void SetLocalEnableMetadataSync(bool state) { set_config_option("citus.enable_metadata_sync", state == true ? "on" : "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } static char * GenerateSetRoleQuery(Oid roleOid) { StringInfo buf = makeStringInfo(); appendStringInfo(buf, "SET ROLE %s", quote_identifier(GetUserNameFromId(roleOid, false))); return buf->data; } /* * TruncateTriggerCreateCommand creates a SQL query calling worker_create_truncate_trigger * function, which creates the truncate trigger on the worker. */ TableDDLCommand * TruncateTriggerCreateCommand(Oid relationId) { StringInfo triggerCreateCommand = makeStringInfo(); char *tableName = generate_qualified_relation_name(relationId); appendStringInfo(triggerCreateCommand, "SELECT worker_create_truncate_trigger(%s)", quote_literal_cstr(tableName)); TableDDLCommand *triggerDDLCommand = makeTableDDLCommandString( triggerCreateCommand->data); return triggerDDLCommand; } /* * SchemaOwnerName returns the name of the owner of the specified schema. */ static char * SchemaOwnerName(Oid objectId) { Oid ownerId = InvalidOid; HeapTuple tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(objectId)); if (HeapTupleIsValid(tuple)) { ownerId = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; } else { ownerId = GetUserId(); } char *ownerName = GetUserNameFromId(ownerId, false); ReleaseSysCache(tuple); return ownerName; } /* * HasMetadataWorkers returns true if any of the workers in the cluster has its * hasmetadata column set to true, which happens when start_metadata_sync_to_node * command is run. */ static bool HasMetadataWorkers(void) { List *workerNodeList = ActiveReadableNonCoordinatorNodeList(); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if (workerNode->hasMetadata) { return true; } } return false; } /* * CreateInterTableRelationshipOfRelationOnWorkers create inter table relationship * for the the given relation id on each worker node with metadata. */ void CreateInterTableRelationshipOfRelationOnWorkers(Oid relationId) { /* if the table is owned by an extension we don't create */ bool tableOwnedByExtension = IsTableOwnedByExtension(relationId); if (tableOwnedByExtension) { return; } List *commandList = InterTableRelationshipOfRelationCommandList(relationId); /* prevent recursive propagation */ SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); const char *command = NULL; foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } } /* * InterTableRelationshipOfRelationCommandList returns the command list to create * inter table relationship for the given relation. */ List * InterTableRelationshipOfRelationCommandList(Oid relationId) { /* commands to create foreign key constraints */ List *commandList = GetReferencingForeignConstaintCommands(relationId); /* commands to create partitioning hierarchy */ if (PartitionTable(relationId)) { char *alterTableAttachPartitionCommands = GenerateAlterTableAttachPartitionCommand(relationId); commandList = lappend(commandList, alterTableAttachPartitionCommands); } return commandList; } /* * CreateShellTableOnWorkers creates the shell table on each worker node with metadata * including sequence dependency and truncate triggers. */ static void CreateShellTableOnWorkers(Oid relationId) { if (IsTableOwnedByExtension(relationId)) { return; } List *commandList = list_make1(DISABLE_DDL_PROPAGATION); IncludeSequenceDefaults includeSequenceDefaults = WORKER_NEXTVAL_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = INCLUDE_IDENTITY; bool creatingShellTableOnRemoteNode = true; List *tableDDLCommands = GetFullTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, creatingShellTableOnRemoteNode); TableDDLCommand *tableDDLCommand = NULL; foreach_declared_ptr(tableDDLCommand, tableDDLCommands) { Assert(CitusIsA(tableDDLCommand, TableDDLCommand)); commandList = lappend(commandList, GetTableDDLCommand(tableDDLCommand)); } const char *command = NULL; foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } } /* * CreateTableMetadataOnWorkers creates the list of commands needed to create the * metadata of the given distributed table and sends these commands to all metadata * workers i.e. workers with hasmetadata=true. Before sending the commands, in order * to prevent recursive propagation, DDL propagation on workers are disabled with a * `SET citus.enable_ddl_propagation TO off;` command. */ static void CreateTableMetadataOnWorkers(Oid relationId) { List *commandList = CitusTableMetadataCreateCommandList(relationId); /* prevent recursive propagation */ SendCommandToWorkersWithMetadata(DISABLE_DDL_PROPAGATION); /* send the commands one by one */ const char *command = NULL; foreach_declared_ptr(command, commandList) { SendCommandToWorkersWithMetadata(command); } } /* * DetachPartitionCommandList returns list of DETACH commands to detach partitions * of all distributed tables. This function is used for detaching partitions in MX * workers before DROPping distributed partitioned tables in them. Thus, we are * disabling DDL propagation to the beginning of the commands (we are also enabling * DDL propagation at the end of command list to swtich back to original state). As * an extra step, if there are no partitions to DETACH, this function simply returns * empty list to not disable/enable DDL propagation for nothing. */ List * DetachPartitionCommandList(void) { List *detachPartitionCommandList = NIL; List *distributedTableList = CitusTableList(); /* we iterate over all distributed partitioned tables and DETACH their partitions */ CitusTableCacheEntry *cacheEntry = NULL; foreach_declared_ptr(cacheEntry, distributedTableList) { if (!PartitionedTable(cacheEntry->relationId)) { continue; } List *partitionList = PartitionList(cacheEntry->relationId); List *detachCommands = GenerateDetachPartitionCommandRelationIdList(partitionList); detachPartitionCommandList = list_concat(detachPartitionCommandList, detachCommands); } if (list_length(detachPartitionCommandList) == 0) { return NIL; } detachPartitionCommandList = lcons(DISABLE_DDL_PROPAGATION, detachPartitionCommandList); /* * We probably do not need this but as an extra precaution, we are enabling * DDL propagation to switch back to original state. */ detachPartitionCommandList = lappend(detachPartitionCommandList, ENABLE_DDL_PROPAGATION); return detachPartitionCommandList; } /* * SyncNodeMetadataToNodesOptional tries recreating the metadata * snapshot in the metadata workers that are out of sync. * Returns the result of synchronization. * * This function must be called within coordinated transaction * since updates on the pg_dist_node metadata must be rollbacked if anything * goes wrong. */ static NodeMetadataSyncResult SyncNodeMetadataToNodesOptional(void) { NodeMetadataSyncResult result = NODE_METADATA_SYNC_SUCCESS; if (!IsCoordinator()) { return NODE_METADATA_SYNC_SUCCESS; } /* * Request a RowExclusiveLock so we don't run concurrently with other * functions updating pg_dist_node, but allow concurrency with functions * which are just reading from pg_dist_node. */ if (!ConditionalLockRelationOid(DistNodeRelationId(), RowExclusiveLock)) { return NODE_METADATA_SYNC_FAILED_LOCK; } List *syncedWorkerList = NIL; List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata && !workerNode->metadataSynced) { bool raiseInterrupts = false; if (!SyncNodeMetadataSnapshotToNode(workerNode, raiseInterrupts)) { ereport(WARNING, (errmsg("failed to sync metadata to %s:%d", workerNode->workerName, workerNode->workerPort))); result = NODE_METADATA_SYNC_FAILED_SYNC; } else { /* we add successfully synced nodes to set metadatasynced column later */ syncedWorkerList = lappend(syncedWorkerList, workerNode); } } } foreach_declared_ptr(workerNode, syncedWorkerList) { SetWorkerColumnOptional(workerNode, Anum_pg_dist_node_metadatasynced, BoolGetDatum(true)); /* we fetch the same node again to check if it's synced or not */ WorkerNode *nodeUpdated = FindWorkerNode(workerNode->workerName, workerNode->workerPort); if (!nodeUpdated->metadataSynced) { /* set the result to FAILED to trigger the sync again */ result = NODE_METADATA_SYNC_FAILED_SYNC; } } return result; } /* * SyncNodeMetadataToNodes recreates the node metadata snapshot in all the * metadata workers. * * This function runs within a coordinated transaction since updates on * the pg_dist_node metadata must be rollbacked if anything * goes wrong. */ void SyncNodeMetadataToNodes(void) { EnsureCoordinator(); /* * Request a RowExclusiveLock so we don't run concurrently with other * functions updating pg_dist_node, but allow concurrency with functions * which are just reading from pg_dist_node. */ if (!ConditionalLockRelationOid(DistNodeRelationId(), RowExclusiveLock)) { ereport(ERROR, (errmsg("cannot sync metadata because a concurrent " "metadata syncing operation is in progress"))); } List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata) { SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_metadatasynced, BoolGetDatum(true)); bool raiseOnError = true; SyncNodeMetadataSnapshotToNode(workerNode, raiseOnError); } } } /* * SyncNodeMetadataToNodesMain is the main function for syncing node metadata to * MX nodes. It retries until success and then exits. */ void SyncNodeMetadataToNodesMain(Datum main_arg) { Oid databaseOid = DatumGetObjectId(main_arg); /* extension owner is passed via bgw_extra */ Oid extensionOwner = InvalidOid; memcpy_s(&extensionOwner, sizeof(extensionOwner), MyBgworkerEntry->bgw_extra, sizeof(Oid)); pqsignal(SIGTERM, MetadataSyncSigTermHandler); pqsignal(SIGALRM, MetadataSyncSigAlrmHandler); BackgroundWorkerUnblockSignals(); /* connect to database, after that we can actually access catalogs */ BackgroundWorkerInitializeConnectionByOid(databaseOid, extensionOwner, 0); /* make worker recognizable in pg_stat_activity */ pgstat_report_appname(METADATA_SYNC_APP_NAME); bool syncedAllNodes = false; while (!syncedAllNodes) { InvalidateMetadataSystemCache(); StartTransactionCommand(); /* * Some functions in ruleutils.c, which we use to get the DDL for * metadata propagation, require an active snapshot. */ PushActiveSnapshot(GetTransactionSnapshot()); if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping metadata sync"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { UseCoordinatedTransaction(); NodeMetadataSyncResult result = SyncNodeMetadataToNodesOptional(); syncedAllNodes = (result == NODE_METADATA_SYNC_SUCCESS); /* we use LISTEN/NOTIFY to wait for metadata syncing in tests */ if (result != NODE_METADATA_SYNC_FAILED_LOCK) { Async_Notify(METADATA_SYNC_CHANNEL, NULL); } } PopActiveSnapshot(); CommitTransactionCommand(); if (syncedAllNodes) { break; } /* * If backend is cancelled (e.g. bacause of distributed deadlock), * CHECK_FOR_INTERRUPTS() will raise a cancellation error which will * result in exit(1). */ CHECK_FOR_INTERRUPTS(); /* * SIGTERM is used for when maintenance daemon tries to clean-up * metadata sync daemons spawned by terminated maintenance daemons. */ if (got_SIGTERM) { exit(0); } /* * SIGALRM is used for testing purposes and it simulates an error in metadata * sync daemon. */ if (got_SIGALRM) { elog(ERROR, "Error in metadata sync daemon"); } pg_usleep(MetadataSyncRetryInterval * 1000); } } /* * MetadataSyncSigTermHandler set a flag to request termination of metadata * sync daemon. */ static void MetadataSyncSigTermHandler(SIGNAL_ARGS) { int save_errno = errno; got_SIGTERM = true; if (MyProc != NULL) { SetLatch(&MyProc->procLatch); } errno = save_errno; } /* * MetadataSyncSigAlrmHandler set a flag to request error at metadata * sync daemon. This is used for testing purposes. */ static void MetadataSyncSigAlrmHandler(SIGNAL_ARGS) { int save_errno = errno; got_SIGALRM = true; if (MyProc != NULL) { SetLatch(&MyProc->procLatch); } errno = save_errno; } /* * SpawnSyncNodeMetadataToNodes starts a background worker which runs node metadata * sync. On success it returns workers' handle. Otherwise it returns NULL. */ BackgroundWorkerHandle * SpawnSyncNodeMetadataToNodes(Oid database, Oid extensionOwner) { char workerName[BGW_MAXLEN]; SafeSnprintf(workerName, BGW_MAXLEN, "Citus Metadata Sync: %u/%u", database, extensionOwner); CitusBackgroundWorkerConfig config = { .workerName = workerName, .functionName = "SyncNodeMetadataToNodesMain", .mainArg = ObjectIdGetDatum(MyDatabaseId), .extensionOwner = extensionOwner, .needsNotification = true, .waitForStartup = false, .restartTime = CITUS_BGW_NEVER_RESTART, .startTime = CITUS_BGW_DEFAULT_START_TIME, .workerType = NULL, /* use default */ .extraData = NULL, .extraDataSize = 0 }; return RegisterCitusBackgroundWorker(&config); } /* * SignalMetadataSyncDaemon signals metadata sync daemons belonging to * the given database. */ void SignalMetadataSyncDaemon(Oid database, int sig) { int backendCount = pgstat_fetch_stat_numbackends(); for (int backend = 1; backend <= backendCount; backend++) { LocalPgBackendStatus *localBeEntry = pgstat_get_local_beentry_by_index(backend); if (!localBeEntry) { continue; } PgBackendStatus *beStatus = &localBeEntry->backendStatus; if (beStatus->st_databaseid == database && strncmp(beStatus->st_appname, METADATA_SYNC_APP_NAME, BGW_MAXLEN) == 0) { kill(beStatus->st_procpid, sig); } } } /* * ShouldInitiateMetadataSync returns if metadata sync daemon should be initiated. * It sets lockFailure to true if pg_dist_node lock couldn't be acquired for the * check. */ bool ShouldInitiateMetadataSync(bool *lockFailure) { if (!IsCoordinator()) { *lockFailure = false; return false; } Oid distNodeOid = DistNodeRelationId(); if (!ConditionalLockRelationOid(distNodeOid, AccessShareLock)) { *lockFailure = true; return false; } bool shouldSyncMetadata = false; List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata && !workerNode->metadataSynced) { shouldSyncMetadata = true; break; } } UnlockRelationOid(distNodeOid, AccessShareLock); *lockFailure = false; return shouldSyncMetadata; } /* * citus_internal_add_partition_metadata is an internal UDF to * add a row to pg_dist_partition. */ Datum citus_internal_add_partition_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "relation"); Oid relationId = PG_GETARG_OID(0); PG_ENSURE_ARGNOTNULL(1, "distribution method"); char distributionMethod = PG_GETARG_CHAR(1); PG_ENSURE_ARGNOTNULL(3, "Colocation ID"); int colocationId = PG_GETARG_INT32(3); PG_ENSURE_ARGNOTNULL(4, "replication model"); char replicationModel = PG_GETARG_CHAR(4); text *distributionColumnText = NULL; char *distributionColumnString = NULL; Var *distributionColumnVar = NULL; /* this flag is only valid for citus local tables, so set it to false */ bool autoConverted = false; /* only owner of the table (or superuser) is allowed to add the Citus metadata */ EnsureTableOwner(relationId); /* we want to serialize all the metadata changes to this table */ LockRelationOid(relationId, ShareUpdateExclusiveLock); if (!PG_ARGISNULL(2)) { distributionColumnText = PG_GETARG_TEXT_P(2); distributionColumnString = text_to_cstring(distributionColumnText); distributionColumnVar = BuildDistributionKeyFromColumnName(relationId, distributionColumnString, AccessShareLock); Assert(distributionColumnVar != NULL); } if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); if (distributionMethod == DISTRIBUTE_BY_NONE && distributionColumnVar != NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Reference or local tables cannot have " "distribution columns"))); } else if (distributionMethod != DISTRIBUTE_BY_NONE && distributionColumnVar == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Distribution column cannot be NULL for " "relation \"%s\"", get_rel_name(relationId)))); } /* * Even if the table owner is a malicious user and the partition * metadata is not sane, the user can only affect its own tables. * Given that the user is owner of the table, we should allow. */ EnsurePartitionMetadataIsSane(relationId, distributionMethod, colocationId, replicationModel, distributionColumnVar); } InsertIntoPgDistPartition(relationId, distributionMethod, distributionColumnVar, colocationId, replicationModel, autoConverted); PG_RETURN_VOID(); } /* * EnsurePartitionMetadataIsSane ensures that the input values are safe * for inserting into pg_dist_partition metadata. */ static void EnsurePartitionMetadataIsSane(Oid relationId, char distributionMethod, int colocationId, char replicationModel, Var *distributionColumnVar) { if (!(distributionMethod == DISTRIBUTE_BY_HASH || distributionMethod == DISTRIBUTE_BY_NONE)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Metadata syncing is only allowed for hash, reference " "and local tables:%c", distributionMethod))); } if (colocationId < INVALID_COLOCATION_ID) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Metadata syncing is only allowed for valid " "colocation id values."))); } else if (colocationId != INVALID_COLOCATION_ID && distributionMethod == DISTRIBUTE_BY_HASH) { int count = 1; List *targetColocatedTableList = ColocationGroupTableList(colocationId, count); /* * If we have any colocated hash tables, ensure if they share the * same distribution key properties. */ if (list_length(targetColocatedTableList) >= 1) { Oid targetRelationId = linitial_oid(targetColocatedTableList); EnsureColumnTypeEquality(relationId, targetRelationId, distributionColumnVar, DistPartitionKeyOrError(targetRelationId)); } } if (!(replicationModel == REPLICATION_MODEL_2PC || replicationModel == REPLICATION_MODEL_STREAMING || replicationModel == REPLICATION_MODEL_COORDINATOR)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Metadata syncing is only allowed for " "known replication models."))); } if (distributionMethod == DISTRIBUTE_BY_NONE && !(replicationModel == REPLICATION_MODEL_STREAMING || replicationModel == REPLICATION_MODEL_2PC)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Local or references tables can only have '%c' or '%c' " "as the replication model.", REPLICATION_MODEL_STREAMING, REPLICATION_MODEL_2PC))); } } /* * citus_internal_delete_partition_metadata is an internal UDF to * delete a row in pg_dist_partition. */ Datum citus_internal_delete_partition_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "relation"); Oid relationId = PG_GETARG_OID(0); /* only owner of the table (or superuser) is allowed to add the Citus metadata */ EnsureTableOwner(relationId); /* we want to serialize all the metadata changes to this table */ LockRelationOid(relationId, ShareUpdateExclusiveLock); if (!ShouldSkipMetadataChecks()) { EnsureCitusInitiatedOperation(); } DeletePartitionRow(relationId); PG_RETURN_VOID(); } /* * citus_internal_add_shard_metadata is an internal UDF to * add a row to pg_dist_shard. */ Datum citus_internal_add_shard_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "relation"); Oid relationId = PG_GETARG_OID(0); PG_ENSURE_ARGNOTNULL(1, "shard id"); int64 shardId = PG_GETARG_INT64(1); PG_ENSURE_ARGNOTNULL(2, "storage type"); char storageType = PG_GETARG_CHAR(2); text *shardMinValue = NULL; if (!PG_ARGISNULL(3)) { shardMinValue = PG_GETARG_TEXT_P(3); } text *shardMaxValue = NULL; if (!PG_ARGISNULL(4)) { shardMaxValue = PG_GETARG_TEXT_P(4); } /* only owner of the table (or superuser) is allowed to add the Citus metadata */ EnsureTableOwner(relationId); /* we want to serialize all the metadata changes to this table */ LockRelationOid(relationId, ShareUpdateExclusiveLock); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); /* * Even if the table owner is a malicious user and the shard metadata is * not sane, the user can only affect its own tables. Given that the * user is owner of the table, we should allow. */ EnsureShardMetadataIsSane(relationId, shardId, storageType, shardMinValue, shardMaxValue); } InsertShardRow(relationId, shardId, storageType, shardMinValue, shardMaxValue); PG_RETURN_VOID(); } /* * EnsureCitusInitiatedOperation is a helper function which ensures that * the execution is initiated by Citus. */ static void EnsureCitusInitiatedOperation(void) { if (!(IsCitusInternalBackend() || IsRebalancerInternalBackend())) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("This is an internal Citus function can only be " "used in a distributed transaction"))); } } /* * EnsureShardMetadataIsSane ensures that the input values are safe * for inserting into pg_dist_shard metadata. */ static void EnsureShardMetadataIsSane(Oid relationId, int64 shardId, char storageType, text *shardMinValue, text *shardMaxValue) { if (shardId <= INVALID_SHARD_ID) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Invalid shard id: %ld", shardId))); } if (!(storageType == SHARD_STORAGE_TABLE || storageType == SHARD_STORAGE_FOREIGN)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Invalid shard storage type: %c", storageType))); } char partitionMethod = PartitionMethodViaCatalog(relationId); if (partitionMethod == DISTRIBUTE_BY_INVALID) { /* connection from the coordinator operating on a shard */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("The relation \"%s\" does not have a valid " "entry in pg_dist_partition.", get_rel_name(relationId)))); } else if (!(partitionMethod == DISTRIBUTE_BY_HASH || partitionMethod == DISTRIBUTE_BY_NONE)) { /* connection from the coordinator operating on a shard */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Metadata syncing is only allowed for hash, " "reference and local tables: %c", partitionMethod))); } List *distShardTupleList = LookupDistShardTuples(relationId); if (partitionMethod == DISTRIBUTE_BY_NONE) { if (shardMinValue != NULL || shardMaxValue != NULL) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shards of reference or local table \"%s\" should " "have NULL shard ranges", relationName))); } else if (list_length(distShardTupleList) != 0) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("relation \"%s\" has already at least one shard, " "adding more is not allowed", relationName))); } } else if (partitionMethod == DISTRIBUTE_BY_HASH) { if (shardMinValue == NULL || shardMaxValue == NULL) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shards of has distributed table \"%s\" " "cannot have NULL shard ranges", relationName))); } char *shardMinValueString = text_to_cstring(shardMinValue); char *shardMaxValueString = text_to_cstring(shardMaxValue); /* pg_strtoint32 does the syntax and out of bound checks for us */ int32 shardMinValueInt = pg_strtoint32(shardMinValueString); int32 shardMaxValueInt = pg_strtoint32(shardMaxValueString); if (shardMinValueInt > shardMaxValueInt) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("shardMinValue=%d is greater than " "shardMaxValue=%d for table \"%s\", which is " "not allowed", shardMinValueInt, shardMaxValueInt, get_rel_name(relationId)))); } /* * We are only dealing with hash distributed tables, that's why we * can hard code data type and typemod. */ const int intervalTypeId = INT4OID; const int intervalTypeMod = -1; Relation distShardRelation = table_open(DistShardRelationId(), AccessShareLock); TupleDesc distShardTupleDesc = RelationGetDescr(distShardRelation); FmgrInfo *shardIntervalCompareFunction = GetFunctionInfo(intervalTypeId, BTREE_AM_OID, BTORDER_PROC); HeapTuple shardTuple = NULL; foreach_declared_ptr(shardTuple, distShardTupleList) { ShardInterval *shardInterval = TupleToShardInterval(shardTuple, distShardTupleDesc, intervalTypeId, intervalTypeMod); Datum firstMin = Int32GetDatum(shardMinValueInt); Datum firstMax = Int32GetDatum(shardMaxValueInt); Datum secondMin = shardInterval->minValue; Datum secondMax = shardInterval->maxValue; Oid collationId = InvalidOid; /* * This is an unexpected case as we are reading the metadata, which has * already been verified for being not NULL. Still, lets be extra * cautious to avoid any crashes. */ if (!shardInterval->minValueExists || !shardInterval->maxValueExists) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shards of has distributed table \"%s\" " "cannot have NULL shard ranges", relationName))); } if (ShardIntervalsOverlapWithParams(firstMin, firstMax, secondMin, secondMax, shardIntervalCompareFunction, collationId)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shard intervals overlap for table \"%s\": " "%ld and %ld", get_rel_name(relationId), shardId, shardInterval->shardId))); } } table_close(distShardRelation, NoLock); } } /* * citus_internal_add_placement_metadata is an internal UDF to * add a row to pg_dist_placement. */ Datum citus_internal_add_placement_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 shardId = PG_GETARG_INT64(0); int64 shardLength = PG_GETARG_INT64(1); int32 groupId = PG_GETARG_INT32(2); int64 placementId = PG_GETARG_INT64(3); citus_internal_add_placement_metadata_internal(shardId, shardLength, groupId, placementId); PG_RETURN_VOID(); } /* * citus_internal_add_placement_metadata is an internal UDF to * delete a row from pg_dist_placement. */ Datum citus_internal_delete_placement_metadata(PG_FUNCTION_ARGS) { PG_ENSURE_ARGNOTNULL(0, "placement_id"); int64 placementId = PG_GETARG_INT64(0); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); } DeleteShardPlacementRow(placementId); PG_RETURN_VOID(); } /* * citus_internal_add_placement_metadata_legacy is the old function that will be dropped. */ Datum citus_internal_add_placement_metadata_legacy(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 shardId = PG_GETARG_INT64(0); int64 shardLength = PG_GETARG_INT64(2); int32 groupId = PG_GETARG_INT32(3); int64 placementId = PG_GETARG_INT64(4); citus_internal_add_placement_metadata_internal(shardId, shardLength, groupId, placementId); PG_RETURN_VOID(); } /* * citus_internal_add_placement_metadata_internal is the internal function * too insert a row into pg_dist_placement */ void citus_internal_add_placement_metadata_internal(int64 shardId, int64 shardLength, int32 groupId, int64 placementId) { bool missingOk = false; Oid relationId = LookupShardRelationFromCatalog(shardId, missingOk); /* only owner of the table is allowed to modify the metadata */ EnsureTableOwner(relationId); /* we want to serialize all the metadata changes to this table */ LockRelationOid(relationId, ShareUpdateExclusiveLock); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); /* * Even if the table owner is a malicious user, as long as the shard placements * fit into basic requirements of Citus metadata, the user can only affect its * own tables. Given that the user is owner of the table, we should allow. */ EnsureShardPlacementMetadataIsSane(relationId, shardId, placementId, shardLength, groupId); } InsertShardPlacementRow(shardId, placementId, shardLength, groupId); } /* * EnsureShardPlacementMetadataIsSane ensures if the input parameters for * the shard placement metadata is sane. */ static void EnsureShardPlacementMetadataIsSane(Oid relationId, int64 shardId, int64 placementId, int64 shardLength, int32 groupId) { /* we have just read the metadata, so we are sure that the shard exists */ Assert(ShardExists(shardId)); if (placementId <= INVALID_PLACEMENT_ID) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shard placement has invalid placement id " "(%ld) for shard(%ld)", placementId, shardId))); } bool nodeIsInMetadata = false; WorkerNode *workerNode = PrimaryNodeForGroup(groupId, &nodeIsInMetadata); if (!workerNode) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Node with group id %d for shard placement " "%ld does not exist", groupId, shardId))); } } /* * ShouldSkipMetadataChecks returns true if the current user is allowed to * make any */ static bool ShouldSkipMetadataChecks(void) { if (strcmp(EnableManualMetadataChangesForUser, "") != 0) { /* * EnableManualMetadataChangesForUser is a GUC which * can be changed by a super user. We use this GUC as * a safety belt in case the current metadata checks are * too restrictive and the operator can allow users to skip * the checks. */ /* * Make sure that the user exists, and print it to prevent any * optimization skipping the get_role_oid call. */ bool missingOK = false; Oid allowedUserId = get_role_oid(EnableManualMetadataChangesForUser, missingOK); if (allowedUserId == GetUserId()) { return true; } } return false; } /* * citus_internal_update_placement_metadata is an internal UDF to * update a row in pg_dist_placement. */ Datum citus_internal_update_placement_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 shardId = PG_GETARG_INT64(0); int32 sourceGroupId = PG_GETARG_INT32(1); int32 targetGroupId = PG_GETARG_INT32(2); ShardPlacement *placement = NULL; if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); if (!ShardExists(shardId)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shard id does not exists: %ld", shardId))); } bool missingOk = false; EnsureShardOwner(shardId, missingOk); /* * This function ensures that the source group exists hence we * call it from this code-block. */ placement = ActiveShardPlacementOnGroup(sourceGroupId, shardId); bool nodeIsInMetadata = false; WorkerNode *workerNode = PrimaryNodeForGroup(targetGroupId, &nodeIsInMetadata); if (!workerNode) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Node with group id %d for shard placement " "%ld does not exist", targetGroupId, shardId))); } } else { placement = ActiveShardPlacementOnGroup(sourceGroupId, shardId); } /* * Updating pg_dist_placement ensures that the node with targetGroupId * exists and this is the only placement on that group. */ if (placement == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Active placement for shard %ld is not " "found on group:%d", shardId, targetGroupId))); } UpdatePlacementGroupId(placement->placementId, targetGroupId); PG_RETURN_VOID(); } /* * citus_internal_delete_shard_metadata is an internal UDF to * delete a row in pg_dist_shard and corresponding placement rows * from pg_dist_shard_placement. */ Datum citus_internal_delete_shard_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 shardId = PG_GETARG_INT64(0); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); if (!ShardExists(shardId)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Shard id does not exists: %ld", shardId))); } bool missingOk = false; EnsureShardOwner(shardId, missingOk); } List *shardPlacementList = ShardPlacementList(shardId); ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, shardPlacementList) { DeleteShardPlacementRow(shardPlacement->placementId); } DeleteShardRow(shardId); PG_RETURN_VOID(); } /* * citus_internal_update_relation_colocation is an internal UDF to * delete a row in pg_dist_shard and corresponding placement rows * from pg_dist_shard_placement. */ Datum citus_internal_update_relation_colocation(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); uint32 targetColocationId = PG_GETARG_UINT32(1); EnsureTableOwner(relationId); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); /* ensure that the table is in pg_dist_partition */ char partitionMethod = PartitionMethodViaCatalog(relationId); if (partitionMethod == DISTRIBUTE_BY_INVALID) { /* connection from the coordinator operating on a shard */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("The relation \"%s\" does not have a valid " "entry in pg_dist_partition.", get_rel_name(relationId)))); } else if (!IsCitusTableType(relationId, HASH_DISTRIBUTED) && !IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { /* connection from the coordinator operating on a shard */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Updating colocation ids are only allowed for hash " "and single shard distributed tables: %c", partitionMethod))); } int count = 1; List *targetColocatedTableList = ColocationGroupTableList(targetColocationId, count); if (list_length(targetColocatedTableList) == 0) { /* the table is colocated with none, so nothing to check */ } else { Oid targetRelationId = linitial_oid(targetColocatedTableList); ErrorIfShardPlacementsNotColocated(relationId, targetRelationId); CheckReplicationModel(relationId, targetRelationId); CheckDistributionColumnType(relationId, targetRelationId); } } bool localOnly = true; UpdateRelationColocationGroup(relationId, targetColocationId, localOnly); PG_RETURN_VOID(); } /* * citus_internal_add_colocation_metadata is an internal UDF to * add a row to pg_dist_colocation. */ Datum citus_internal_add_colocation_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); int colocationId = PG_GETARG_INT32(0); int shardCount = PG_GETARG_INT32(1); int replicationFactor = PG_GETARG_INT32(2); Oid distributionColumnType = PG_GETARG_INT32(3); Oid distributionColumnCollation = PG_GETARG_INT32(4); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); } InsertColocationGroupLocally(colocationId, shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); PG_RETURN_VOID(); } /* * citus_internal_delete_colocation_metadata is an internal UDF to * delte row from pg_dist_colocation. */ Datum citus_internal_delete_colocation_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); int colocationId = PG_GETARG_INT32(0); if (!ShouldSkipMetadataChecks()) { /* this UDF is not allowed allowed for executing as a separate command */ EnsureCitusInitiatedOperation(); } DeleteColocationGroupLocally(colocationId); PG_RETURN_VOID(); } /* * citus_internal_add_tenant_schema is an internal UDF to * call InsertTenantSchemaLocally on a remote node. * * None of the parameters are allowed to be NULL. To set the colocation * id to NULL in metadata, use INVALID_COLOCATION_ID. */ Datum citus_internal_add_tenant_schema(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "schema_id"); Oid schemaId = PG_GETARG_OID(0); PG_ENSURE_ARGNOTNULL(1, "colocation_id"); uint32 colocationId = PG_GETARG_INT32(1); InsertTenantSchemaLocally(schemaId, colocationId); PG_RETURN_VOID(); } /* * citus_internal_delete_tenant_schema is an internal UDF to * call DeleteTenantSchemaLocally on a remote node. * * The schemaId parameter is not allowed to be NULL. Morever, input schema is * expected to be dropped already because this function is called from Citus * drop hook and only used to clean up metadata after the schema is dropped. */ Datum citus_internal_delete_tenant_schema(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "schema_id"); Oid schemaId = PG_GETARG_OID(0); DeleteTenantSchemaLocally(schemaId); PG_RETURN_VOID(); } /* * citus_internal_update_none_dist_table_metadata is an internal UDF to * update a row in pg_dist_partition that belongs to given none-distributed * table. */ Datum citus_internal_update_none_dist_table_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "relation_id"); Oid relationId = PG_GETARG_OID(0); PG_ENSURE_ARGNOTNULL(1, "replication_model"); char replicationModel = PG_GETARG_CHAR(1); PG_ENSURE_ARGNOTNULL(2, "colocation_id"); uint32 colocationId = PG_GETARG_INT32(2); PG_ENSURE_ARGNOTNULL(3, "auto_converted"); bool autoConverted = PG_GETARG_BOOL(3); if (!ShouldSkipMetadataChecks()) { EnsureCitusInitiatedOperation(); } UpdateNoneDistTableMetadata(relationId, replicationModel, colocationId, autoConverted); PG_RETURN_VOID(); } /* * citus_internal_database_command is an internal UDF to * create a database in an idempotent maner without * transaction block restrictions. */ Datum citus_internal_database_command(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (!ShouldSkipMetadataChecks()) { EnsureCitusInitiatedOperation(); } PG_ENSURE_ARGNOTNULL(0, "command"); text *commandText = PG_GETARG_TEXT_P(0); char *command = text_to_cstring(commandText); Node *parseTree = ParseTreeNode(command); int saveNestLevel = NewGUCNestLevel(); set_config_option("citus.enable_ddl_propagation", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); set_config_option("citus.enable_create_database_propagation", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); /* * createdb() uses ParseState to report the error position for the * input command and the position is reported to be 0 when it's provided as NULL. * We're okay with that because we don't expect this UDF to be called with an incorrect * DDL command. */ ParseState *pstate = NULL; if (IsA(parseTree, CreatedbStmt)) { CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); bool missingOk = true; Oid databaseOid = get_database_oid(stmt->dbname, missingOk); if (!OidIsValid(databaseOid)) { createdb(pstate, (CreatedbStmt *) parseTree); } } else { ereport(ERROR, (errmsg("citus_internal.database_command() can only be used " "for CREATE DATABASE command by Citus."))); } /* rollback GUCs to the state before this session */ AtEOXact_GUC(true, saveNestLevel); PG_RETURN_VOID(); } /* * SyncNewColocationGroup synchronizes a new pg_dist_colocation entry to a worker. */ void SyncNewColocationGroupToNodes(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation) { char *command = ColocationGroupCreateCommand(colocationId, shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); /* * We require superuser for all pg_dist_colocation operations because we have * no reasonable way of restricting access. */ SendCommandToWorkersWithMetadataViaSuperUser(command); } /* * ColocationGroupCreateCommand returns a command for creating a colocation group. */ static char * ColocationGroupCreateCommand(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation) { StringInfo insertColocationCommand = makeStringInfo(); /* * Get type name and schema separately to defer type resolution. * This approach matches how SendColocationMetadataCommands handles types. */ char *typeName = GetRemoteTypeName(distributionColumnType); char *typeSchemaName = GetRemoteTypeNamespace(distributionColumnType); appendStringInfo(insertColocationCommand, "WITH colocation_data(" "colocationid, shardcount, replicationfactor, " "typeschema, typename, collationid) " "AS (VALUES (%d, %d, %d, ", colocationId, shardCount, replicationFactor); if (typeSchemaName != NULL && typeName != NULL) { /* Use quote_identifier so the schema name can be cast to regnamespace */ appendStringInfo(insertColocationCommand, "%s, %s, ", quote_literal_cstr(quote_identifier(typeSchemaName)), quote_literal_cstr(typeName)); } else if (typeName != NULL) { appendStringInfo(insertColocationCommand, "NULL, %s, ", quote_literal_cstr(typeName)); } else { appendStringInfo(insertColocationCommand, "NULL, NULL, "); } appendStringInfo(insertColocationCommand, "%s)) " "SELECT citus_internal.add_colocation_metadata(" "colocationid, shardcount, replicationfactor, " "coalesce(t.oid, 0), collationid) " "FROM colocation_data " "LEFT JOIN pg_type t ON (" "typename = t.typname " "AND (typeschema IS NULL OR " "t.typnamespace = " "(SELECT oid FROM pg_namespace WHERE nspname = typeschema)))", RemoteCollationIdExpression(distributionColumnCollation)); return insertColocationCommand->data; } /* * GetRemoteTypeName returns the unqualified name of a type. * Returns NULL for InvalidOid. */ static char * GetRemoteTypeName(Oid typeId) { if (typeId == InvalidOid) { return NULL; } HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeId)); if (!HeapTupleIsValid(typeTuple)) { return NULL; } Form_pg_type typeForm = (Form_pg_type) GETSTRUCT(typeTuple); char *typeName = pstrdup(NameStr(typeForm->typname)); ReleaseSysCache(typeTuple); return typeName; } /* * GetRemoteTypeNamespace returns the schema name of a type. * Returns NULL for InvalidOid or types in pg_catalog. */ static char * GetRemoteTypeNamespace(Oid typeId) { if (typeId == InvalidOid) { return NULL; } HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(typeId)); if (!HeapTupleIsValid(typeTuple)) { return NULL; } Form_pg_type typeForm = (Form_pg_type) GETSTRUCT(typeTuple); Oid typeNamespace = typeForm->typnamespace; ReleaseSysCache(typeTuple); /* Don't include schema for pg_catalog types for backward compatibility */ if (typeNamespace == PG_CATALOG_NAMESPACE) { return NULL; } return get_namespace_name(typeNamespace); } /* * RemoteCollationIdExpression returns an expression in text form that can * be used to obtain the OID of a collation on a different node when included * in a query string. */ static char * RemoteCollationIdExpression(Oid colocationId) { /* by default, use 0 (InvalidOid) */ char *expression = "0"; if (colocationId != InvalidOid) { Datum collationIdDatum = ObjectIdGetDatum(colocationId); HeapTuple collationTuple = SearchSysCache1(COLLOID, collationIdDatum); if (HeapTupleIsValid(collationTuple)) { Form_pg_collation collationform = (Form_pg_collation) GETSTRUCT(collationTuple); char *collationName = NameStr(collationform->collname); char *collationSchemaName = get_namespace_name(collationform->collnamespace); char *qualifiedCollationName = quote_qualified_identifier(collationSchemaName, collationName); StringInfo regcollationExpression = makeStringInfo(); appendStringInfo(regcollationExpression, "%s::regcollation", quote_literal_cstr(qualifiedCollationName)); expression = regcollationExpression->data; } ReleaseSysCache(collationTuple); } return expression; } /* * SyncDeleteColocationGroupToNodes deletes a pg_dist_colocation record from workers. */ void SyncDeleteColocationGroupToNodes(uint32 colocationId) { char *command = ColocationGroupDeleteCommand(colocationId); /* * We require superuser for all pg_dist_colocation operations because we have * no reasonable way of restricting access. */ SendCommandToWorkersWithMetadataViaSuperUser(command); } /* * ColocationGroupDeleteCommand returns a command for deleting a colocation group. */ static char * ColocationGroupDeleteCommand(uint32 colocationId) { StringInfo deleteColocationCommand = makeStringInfo(); appendStringInfo(deleteColocationCommand, "SELECT citus_internal.delete_colocation_metadata(%d)", colocationId); return deleteColocationCommand->data; } /* * TenantSchemaInsertCommand returns a command to call * citus_internal_add_tenant_schema(). */ char * TenantSchemaInsertCommand(Oid schemaId, uint32 colocationId) { StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.add_tenant_schema(%s, %u)", RemoteSchemaIdExpressionById(schemaId), colocationId); return command->data; } /* * TenantSchemaDeleteCommand returns a command to call * citus_internal_delete_tenant_schema(). */ char * TenantSchemaDeleteCommand(char *schemaName) { StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.delete_tenant_schema(%s)", RemoteSchemaIdExpressionByName(schemaName)); return command->data; } /* * UpdateNoneDistTableMetadataCommand returns a command to call * citus_internal_update_none_dist_table_metadata(). */ char * UpdateNoneDistTableMetadataCommand(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted) { StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.update_none_dist_table_metadata(%s, '%c', %u, %s)", RemoteTableIdExpression(relationId), replicationModel, colocationId, autoConverted ? "true" : "false"); return command->data; } /* * AddPlacementMetadataCommand returns a command to call * citus_internal_add_placement_metadata(). */ char * AddPlacementMetadataCommand(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId) { StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.add_placement_metadata(%ld, %ld, %d, %ld)", shardId, shardLength, groupId, placementId); return command->data; } /* * DeletePlacementMetadataCommand returns a command to call * citus_internal_delete_placement_metadata(). */ char * DeletePlacementMetadataCommand(uint64 placementId) { StringInfo command = makeStringInfo(); appendStringInfo(command, "SELECT citus_internal.delete_placement_metadata(%ld)", placementId); return command->data; } /* * RemoteSchemaIdExpressionById returns an expression in text form that * can be used to obtain the OID of the schema with given schema id on a * different node when included in a query string. */ static char * RemoteSchemaIdExpressionById(Oid schemaId) { char *schemaName = get_namespace_name(schemaId); if (schemaName == NULL) { ereport(ERROR, (errmsg("schema with OID %u does not exist", schemaId))); } return RemoteSchemaIdExpressionByName(schemaName); } /* * RemoteSchemaIdExpressionByName returns an expression in text form that * can be used to obtain the OID of the schema with given schema name on a * different node when included in a query string. */ static char * RemoteSchemaIdExpressionByName(char *schemaName) { StringInfo regnamespaceExpr = makeStringInfo(); appendStringInfo(regnamespaceExpr, "%s::regnamespace", quote_literal_cstr(quote_identifier(schemaName))); return regnamespaceExpr->data; } /* * RemoteTableIdExpression returns an expression in text form that * can be used to obtain the OID of given table on a different node * when included in a query string. */ static char * RemoteTableIdExpression(Oid relationId) { StringInfo regclassExpr = makeStringInfo(); appendStringInfo(regclassExpr, "%s::regclass", quote_literal_cstr(generate_qualified_relation_name(relationId))); return regclassExpr->data; } /* * SetMetadataSyncNodesFromNodeList sets list of nodes that needs to be metadata * synced among given node list into metadataSyncContext. */ void SetMetadataSyncNodesFromNodeList(MetadataSyncContext *context, List *nodeList) { /* sync is disabled, then no nodes to sync */ if (!EnableMetadataSync) { return; } List *activatedWorkerNodeList = NIL; WorkerNode *node = NULL; foreach_declared_ptr(node, nodeList) { if (NodeIsPrimary(node)) { /* warn if we have coordinator in nodelist */ if (NodeIsCoordinator(node)) { ereport(NOTICE, (errmsg("%s:%d is the coordinator and already contains " "metadata, skipping syncing the metadata", node->workerName, node->workerPort))); continue; } activatedWorkerNodeList = lappend(activatedWorkerNodeList, node); } } context->activatedWorkerNodeList = activatedWorkerNodeList; } /* * EstablishAndSetMetadataSyncBareConnections establishes and sets * connections used throughout nontransactional metadata sync. */ void EstablishAndSetMetadataSyncBareConnections(MetadataSyncContext *context) { Assert(MetadataSyncTransMode == METADATA_SYNC_NON_TRANSACTIONAL); int connectionFlags = REQUIRE_METADATA_CONNECTION; /* establish bare connections to activated worker nodes */ List *bareConnectionList = NIL; WorkerNode *node = NULL; foreach_declared_ptr(node, context->activatedWorkerNodeList) { MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, node->workerName, node->workerPort, CurrentUserName(), NULL); Assert(connection != NULL); ForceConnectionCloseAtTransactionEnd(connection); bareConnectionList = lappend(bareConnectionList, connection); } context->activatedWorkerBareConnections = bareConnectionList; } /* * CreateMetadataSyncContext creates a context which contains worker connections * and a MemoryContext to be used throughout the metadata sync. * * If we collect commands, connections will not be established as caller's intent * is to collect sync commands. * * If the nodes are newly added before activation, we would not try to unset * metadatasynced in separate transaction during nontransactional metadatasync. */ MetadataSyncContext * CreateMetadataSyncContext(List *nodeList, bool collectCommands, bool nodesAddedInSameTransaction) { /* should be alive during local transaction during the sync */ MemoryContext context = AllocSetContextCreate(TopTransactionContext, "metadata_sync_context", ALLOCSET_DEFAULT_SIZES); MetadataSyncContext *metadataSyncContext = (MetadataSyncContext *) palloc0( sizeof(MetadataSyncContext)); metadataSyncContext->context = context; metadataSyncContext->transactionMode = MetadataSyncTransMode; metadataSyncContext->collectCommands = collectCommands; metadataSyncContext->collectedCommands = NIL; metadataSyncContext->nodesAddedInSameTransaction = nodesAddedInSameTransaction; /* filter the nodes that needs to be activated from given node list */ SetMetadataSyncNodesFromNodeList(metadataSyncContext, nodeList); /* * establish connections only for nontransactional mode to prevent connection * open-close for each command */ if (!collectCommands && MetadataSyncTransMode == METADATA_SYNC_NON_TRANSACTIONAL) { EstablishAndSetMetadataSyncBareConnections(metadataSyncContext); } /* use 2PC coordinated transactions if we operate in transactional mode */ if (MetadataSyncTransMode == METADATA_SYNC_TRANSACTIONAL) { Use2PCForCoordinatedTransaction(); } return metadataSyncContext; } /* * ResetMetadataSyncMemoryContext resets memory context inside metadataSyncContext, if * we are not collecting commands. */ void ResetMetadataSyncMemoryContext(MetadataSyncContext *context) { if (!MetadataSyncCollectsCommands(context)) { MemoryContextReset(context->context); } } /* * MetadataSyncCollectsCommands returns whether context is used for collecting * commands instead of sending them to workers. */ bool MetadataSyncCollectsCommands(MetadataSyncContext *context) { return context->collectCommands; } /* * SendOrCollectCommandListToActivatedNodes sends the commands to the activated nodes with * bare connections inside metadatacontext or via coordinated connections. * Note that when context only collects commands, we add commands into the context * without sending the commands. */ void SendOrCollectCommandListToActivatedNodes(MetadataSyncContext *context, List *commands) { /* do nothing if no commands */ if (commands == NIL) { return; } /* * do not send any command to workers if we collect commands. * Collect commands into metadataSyncContext's collected command * list. */ if (MetadataSyncCollectsCommands(context)) { context->collectedCommands = list_concat(context->collectedCommands, commands); return; } /* send commands to new workers, the current user should be a superuser */ Assert(superuser()); if (context->transactionMode == METADATA_SYNC_TRANSACTIONAL) { List *workerNodes = context->activatedWorkerNodeList; SendMetadataCommandListToWorkerListInCoordinatedTransaction(workerNodes, CurrentUserName(), commands); } else if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL) { List *workerConnections = context->activatedWorkerBareConnections; SendCommandListToWorkerListWithBareConnections(workerConnections, commands); } else { pg_unreachable(); } } /* * SendOrCollectCommandListToMetadataNodes sends the commands to the metadata nodes with * bare connections inside metadatacontext or via coordinated connections. * Note that when context only collects commands, we add commands into the context * without sending the commands. */ void SendOrCollectCommandListToMetadataNodes(MetadataSyncContext *context, List *commands) { /* * do not send any command to workers if we collect commands. * Collect commands into metadataSyncContext's collected command * list. */ if (MetadataSyncCollectsCommands(context)) { context->collectedCommands = list_concat(context->collectedCommands, commands); return; } /* send commands to new workers, the current user should be a superuser */ Assert(superuser()); if (context->transactionMode == METADATA_SYNC_TRANSACTIONAL) { List *metadataNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); SendMetadataCommandListToWorkerListInCoordinatedTransaction(metadataNodes, CurrentUserName(), commands); } else if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL) { SendBareCommandListToMetadataWorkers(commands); } else { pg_unreachable(); } } /* * SendOrCollectCommandListToSingleNode sends the commands to the specific worker * indexed by nodeIdx with bare connection inside metadatacontext or via coordinated * connection. Note that when context only collects commands, we add commands into * the context without sending the commands. */ void SendOrCollectCommandListToSingleNode(MetadataSyncContext *context, List *commands, int nodeIdx) { /* * Do not send any command to workers if we collect commands. * Collect commands into metadataSyncContext's collected command * list. */ if (MetadataSyncCollectsCommands(context)) { context->collectedCommands = list_concat(context->collectedCommands, commands); return; } /* send commands to new workers, the current user should be a superuser */ Assert(superuser()); if (context->transactionMode == METADATA_SYNC_TRANSACTIONAL) { List *workerNodes = context->activatedWorkerNodeList; Assert(nodeIdx < list_length(workerNodes)); WorkerNode *node = list_nth(workerNodes, nodeIdx); SendMetadataCommandListToWorkerListInCoordinatedTransaction(list_make1(node), CurrentUserName(), commands); } else if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL) { List *workerConnections = context->activatedWorkerBareConnections; Assert(nodeIdx < list_length(workerConnections)); MultiConnection *workerConnection = list_nth(workerConnections, nodeIdx); List *connectionList = list_make1(workerConnection); SendCommandListToWorkerListWithBareConnections(connectionList, commands); } else { pg_unreachable(); } } /* * WorkerDropAllShellTablesCommand returns command required to drop shell tables * from workers. When singleTransaction is false, we create transaction per shell * table. Otherwise, we drop all shell tables within single transaction. */ char * WorkerDropAllShellTablesCommand(bool singleTransaction) { char *singleTransactionString = (singleTransaction) ? "true" : "false"; StringInfo removeAllShellTablesCommand = makeStringInfo(); appendStringInfo(removeAllShellTablesCommand, WORKER_DROP_ALL_SHELL_TABLES, singleTransactionString); return removeAllShellTablesCommand->data; } /* * WorkerDropSequenceDependencyCommand returns command to drop sequence dependencies for * given table. */ char * WorkerDropSequenceDependencyCommand(Oid relationId) { char *qualifiedTableName = generate_qualified_relation_name(relationId); StringInfo breakSequenceDepCommand = makeStringInfo(); appendStringInfo(breakSequenceDepCommand, BREAK_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND, quote_literal_cstr(qualifiedTableName)); return breakSequenceDepCommand->data; } /* * PropagateNodeWideObjectsCommandList is called during node activation to * propagate any object that should be propagated for every node. These are * generally not linked to any distributed object but change system wide behaviour. */ static List * PropagateNodeWideObjectsCommandList(void) { /* collect all commands */ List *ddlCommands = NIL; if (EnableAlterRoleSetPropagation) { /* * Get commands for database and postgres wide settings. Since these settings are not * linked to any role that can be distributed we need to distribute them seperately */ List *alterRoleSetCommands = GenerateAlterRoleSetCommandForRole(InvalidOid); ddlCommands = list_concat(ddlCommands, alterRoleSetCommands); } return ddlCommands; } /* * SyncDistributedObjects sync the distributed objects to the nodes in metadataSyncContext * with transactional or nontransactional mode according to transactionMode inside * metadataSyncContext. * * Transactions should be ordered like below: * - Nodewide objects (only roles for now), * - Deletion of sequence and shell tables and metadata entries * - All dependencies (e.g., types, schemas, sequences) and all shell distributed * table and their pg_dist_xx metadata entries * - Inter relation between those shell tables * * Note that we do not create the distributed dependencies on the coordinator * since all the dependencies should be present in the coordinator already. */ void SyncDistributedObjects(MetadataSyncContext *context) { if (context->activatedWorkerNodeList == NIL) { return; } EnsureSequentialModeMetadataOperations(); Assert(ShouldPropagate()); /* Send systemwide objects, only roles for now */ SendNodeWideObjectsSyncCommands(context); /* * Break dependencies between sequences-shell tables, then remove shell tables, * and metadata tables respectively. * We should delete shell tables before metadata entries as we look inside * pg_dist_partition to figure out shell tables. */ SendShellTableDeletionCommands(context); SendMetadataDeletionCommands(context); /* * Commands to insert pg_dist_colocation entries. * Replicating dist objects and their metadata depends on this step. */ SendColocationMetadataCommands(context); /* * Replicate all objects of the pg_dist_object to the remote node and * create metadata entries for Citus tables (pg_dist_shard, pg_dist_shard_placement, * pg_dist_partition, pg_dist_object). */ SendDependencyCreationCommands(context); SendDistTableMetadataCommands(context); SendDistObjectCommands(context); /* * Commands to insert pg_dist_schema entries. * * Need to be done after syncing distributed objects because the schemas * need to exist on the worker. */ SendTenantSchemaMetadataCommands(context); /* * After creating each table, handle the inter table relationship between * those tables. */ SendInterTableRelationshipCommands(context); } /* * SendNodeWideObjectsSyncCommands sends systemwide objects to workers with * transactional or nontransactional mode according to transactionMode inside * metadataSyncContext. */ void SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) { /* propagate node wide objects. It includes only roles for now. */ List *commandList = PropagateNodeWideObjectsCommandList(); if (commandList == NIL) { return; } commandList = lcons(DISABLE_DDL_PROPAGATION, commandList); commandList = lappend(commandList, ENABLE_DDL_PROPAGATION); SendOrCollectCommandListToActivatedNodes(context, commandList); } /* * SendShellTableDeletionCommands sends sequence, and shell table deletion * commands to workers with transactional or nontransactional mode according to * transactionMode inside metadataSyncContext. */ void SendShellTableDeletionCommands(MetadataSyncContext *context) { /* break all sequence deps for citus tables */ char *breakSeqDepsCommand = BREAK_ALL_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND; SendOrCollectCommandListToActivatedNodes(context, list_make1(breakSeqDepsCommand)); /* remove shell tables */ bool singleTransaction = (context->transactionMode == METADATA_SYNC_TRANSACTIONAL); char *dropShellTablesCommand = WorkerDropAllShellTablesCommand(singleTransaction); SendOrCollectCommandListToActivatedNodes(context, list_make1(dropShellTablesCommand)); } /* * SendMetadataDeletionCommands sends metadata entry deletion commands to workers * with transactional or nontransactional mode according to transactionMode inside * metadataSyncContext. */ void SendMetadataDeletionCommands(MetadataSyncContext *context) { /* remove pg_dist_partition entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_PARTITIONS)); /* remove pg_dist_shard entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_SHARDS)); /* remove pg_dist_placement entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_PLACEMENTS)); /* remove pg_dist_object entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_DISTRIBUTED_OBJECTS)); /* remove pg_dist_colocation entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_COLOCATION)); /* remove pg_dist_schema entries */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DELETE_ALL_TENANT_SCHEMAS)); } /* * SendColocationMetadataCommands sends colocation metadata with transactional or * nontransactional mode according to transactionMode inside metadataSyncContext. */ void SendColocationMetadataCommands(MetadataSyncContext *context) { ScanKeyData scanKey[1]; int scanKeyCount = 0; Relation relation = table_open(DistColocationRelationId(), AccessShareLock); SysScanDesc scanDesc = systable_beginscan(relation, InvalidOid, false, NULL, scanKeyCount, scanKey); MemoryContext oldContext = MemoryContextSwitchTo(context->context); HeapTuple nextTuple = NULL; while (true) { ResetMetadataSyncMemoryContext(context); nextTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(nextTuple)) { break; } StringInfo colocationGroupCreateCommand = makeStringInfo(); appendStringInfo(colocationGroupCreateCommand, "WITH colocation_group_data (colocationid, shardcount, " "replicationfactor, distributioncolumntypeschema, " "distributioncolumntypename, " "distributioncolumncollationname, " "distributioncolumncollationschema) AS (VALUES "); Form_pg_dist_colocation colocationForm = (Form_pg_dist_colocation) GETSTRUCT(nextTuple); /* * Get the type name and schema separately to defer type resolution. * This is necessary when the type (e.g., a domain) is defined in a * non-public schema that may not exist on the worker yet. */ char *typeName = GetRemoteTypeName(colocationForm->distributioncolumntype); char *typeSchemaName = GetRemoteTypeNamespace(colocationForm->distributioncolumntype); appendStringInfo(colocationGroupCreateCommand, "(%d, %d, %d, ", colocationForm->colocationid, colocationForm->shardcount, colocationForm->replicationfactor); /* Add type schema and name */ if (typeSchemaName != NULL && typeName != NULL) { /* Use quote_identifier so the schema name can be cast to regnamespace */ appendStringInfo(colocationGroupCreateCommand, "%s, %s, ", quote_literal_cstr(quote_identifier(typeSchemaName)), quote_literal_cstr(typeName)); } else if (typeName != NULL) { /* Type is in pg_catalog or no schema qualifier needed */ appendStringInfo(colocationGroupCreateCommand, "NULL, %s, ", quote_literal_cstr(typeName)); } else { /* InvalidOid or unknown type */ appendStringInfo(colocationGroupCreateCommand, "NULL, NULL, "); } /* * For collations, include the names in the VALUES section and then * join with pg_collation. */ Oid distributionColumCollation = colocationForm->distributioncolumncollation; if (distributionColumCollation != InvalidOid) { Datum collationIdDatum = ObjectIdGetDatum(distributionColumCollation); HeapTuple collationTuple = SearchSysCache1(COLLOID, collationIdDatum); if (HeapTupleIsValid(collationTuple)) { Form_pg_collation collationform = (Form_pg_collation) GETSTRUCT(collationTuple); char *collationName = NameStr(collationform->collname); char *collationSchemaName = get_namespace_name(collationform->collnamespace); appendStringInfo(colocationGroupCreateCommand, "%s, %s)", quote_literal_cstr(collationName), quote_literal_cstr(collationSchemaName)); ReleaseSysCache(collationTuple); } else { appendStringInfo(colocationGroupCreateCommand, "NULL, NULL)"); } } else { appendStringInfo(colocationGroupCreateCommand, "NULL, NULL)"); } /* * Use LEFT JOIN with pg_type to resolve the type OID at runtime. * This defers type resolution until execution on the worker, allowing * the type and its schema to be created first by dependency commands. */ appendStringInfo(colocationGroupCreateCommand, ") SELECT citus_internal.add_colocation_metadata(" "colocationid, shardcount, replicationfactor, " "coalesce(t.oid, 0), coalesce(c.oid, 0)) " "FROM colocation_group_data d " "LEFT JOIN pg_type t ON (" "d.distributioncolumntypename = t.typname " "AND (d.distributioncolumntypeschema IS NULL OR " "t.typnamespace = (SELECT oid FROM pg_namespace WHERE " "nspname = d.distributioncolumntypeschema))) " "LEFT JOIN pg_collation c " "ON (d.distributioncolumncollationname = c.collname " "AND c.collnamespace = (SELECT oid FROM pg_namespace WHERE " "nspname = d.distributioncolumncollationschema))"); List *commandList = list_make1(colocationGroupCreateCommand->data); SendOrCollectCommandListToActivatedNodes(context, commandList); } MemoryContextSwitchTo(oldContext); systable_endscan(scanDesc); table_close(relation, AccessShareLock); } /* * SendTenantSchemaMetadataCommands sends tenant schema metadata entries with * transactional or nontransactional mode according to transactionMode inside * metadataSyncContext. */ void SendTenantSchemaMetadataCommands(MetadataSyncContext *context) { ScanKeyData scanKey[1]; int scanKeyCount = 0; Relation pgDistTenantSchema = table_open(DistTenantSchemaRelationId(), AccessShareLock); SysScanDesc scanDesc = systable_beginscan(pgDistTenantSchema, InvalidOid, false, NULL, scanKeyCount, scanKey); MemoryContext oldContext = MemoryContextSwitchTo(context->context); HeapTuple heapTuple = NULL; while (true) { ResetMetadataSyncMemoryContext(context); heapTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(heapTuple)) { break; } Form_pg_dist_schema tenantSchemaForm = (Form_pg_dist_schema) GETSTRUCT(heapTuple); StringInfo insertTenantSchemaCommand = makeStringInfo(); appendStringInfo(insertTenantSchemaCommand, "SELECT citus_internal.add_tenant_schema(%s, %u)", RemoteSchemaIdExpressionById(tenantSchemaForm->schemaid), tenantSchemaForm->colocationid); List *commandList = list_make1(insertTenantSchemaCommand->data); SendOrCollectCommandListToActivatedNodes(context, commandList); } MemoryContextSwitchTo(oldContext); systable_endscan(scanDesc); table_close(pgDistTenantSchema, AccessShareLock); } /* * SendDependencyCreationCommands sends dependency creation commands to workers * with transactional or nontransactional mode according to transactionMode * inside metadataSyncContext. */ void SendDependencyCreationCommands(MetadataSyncContext *context) { /* disable ddl propagation */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DISABLE_DDL_PROPAGATION)); MemoryContext oldContext = MemoryContextSwitchTo(context->context); /* collect all dependencies in creation order and get their ddl commands */ List *dependencies = GetDistributedObjectAddressList(); /* * Depending on changes in the environment, such as the enable_metadata_sync guc * there might be objects in the distributed object address list that should currently * not be propagated by citus as they are 'not supported'. */ dependencies = FilterObjectAddressListByPredicate(dependencies, &SupportedDependencyByCitus); dependencies = OrderObjectAddressListInDependencyOrder(dependencies); /* * We need to create a subcontext as we reset the context after each dependency * creation but we want to preserve all dependency objects at metadataSyncContext. */ MemoryContext commandsContext = AllocSetContextCreate(context->context, "dependency commands context", ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(commandsContext); ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { if (!MetadataSyncCollectsCommands(context)) { MemoryContextReset(commandsContext); } if (IsAnyObjectAddressOwnedByExtension(list_make1(dependency), NULL)) { /* * We expect extension-owned objects to be created as a result * of the extension being created. */ continue; } /* dependency creation commands */ List *ddlCommands = GetAllDependencyCreateDDLCommands(list_make1(dependency)); SendOrCollectCommandListToActivatedNodes(context, ddlCommands); } MemoryContextSwitchTo(oldContext); if (!MetadataSyncCollectsCommands(context)) { MemoryContextDelete(commandsContext); } ResetMetadataSyncMemoryContext(context); /* enable ddl propagation */ SendOrCollectCommandListToActivatedNodes(context, list_make1(ENABLE_DDL_PROPAGATION)); } /* * SendDistTableMetadataCommands sends commands related to pg_dist_shard and, * pg_dist_shard_placement entries to workers with transactional or nontransactional * mode according to transactionMode inside metadataSyncContext. */ void SendDistTableMetadataCommands(MetadataSyncContext *context) { ScanKeyData scanKey[1]; int scanKeyCount = 0; Relation relation = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(relation); SysScanDesc scanDesc = systable_beginscan(relation, InvalidOid, false, NULL, scanKeyCount, scanKey); MemoryContext oldContext = MemoryContextSwitchTo(context->context); HeapTuple nextTuple = NULL; while (true) { ResetMetadataSyncMemoryContext(context); nextTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(nextTuple)) { break; } /* * Create Citus table metadata commands (pg_dist_shard, pg_dist_shard_placement, * pg_dist_partition). Only Citus tables have shard metadata. */ Oid relationId = FetchRelationIdFromPgPartitionHeapTuple(nextTuple, tupleDesc); if (!ShouldSyncTableMetadata(relationId)) { continue; } List *commandList = CitusTableMetadataCreateCommandList(relationId); SendOrCollectCommandListToActivatedNodes(context, commandList); } MemoryContextSwitchTo(oldContext); systable_endscan(scanDesc); table_close(relation, AccessShareLock); } /* * SendDistObjectCommands sends commands related to pg_dist_object entries to * workers with transactional or nontransactional mode according to transactionMode * inside metadataSyncContext. */ void SendDistObjectCommands(MetadataSyncContext *context) { ScanKeyData scanKey[1]; int scanKeyCount = 0; Relation relation = table_open(DistObjectRelationId(), AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(relation); SysScanDesc scanDesc = systable_beginscan(relation, InvalidOid, false, NULL, scanKeyCount, scanKey); MemoryContext oldContext = MemoryContextSwitchTo(context->context); HeapTuple nextTuple = NULL; while (true) { ResetMetadataSyncMemoryContext(context); nextTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(nextTuple)) { break; } Form_pg_dist_object pg_dist_object = (Form_pg_dist_object) GETSTRUCT(nextTuple); ObjectAddress *address = palloc(sizeof(ObjectAddress)); ObjectAddressSubSet(*address, pg_dist_object->classid, pg_dist_object->objid, pg_dist_object->objsubid); bool distributionArgumentIndexIsNull = false; Datum distributionArgumentIndexDatum = heap_getattr(nextTuple, Anum_pg_dist_object_distribution_argument_index, tupleDesc, &distributionArgumentIndexIsNull); int32 distributionArgumentIndex = DatumGetInt32(distributionArgumentIndexDatum); bool colocationIdIsNull = false; Datum colocationIdDatum = heap_getattr(nextTuple, Anum_pg_dist_object_colocationid, tupleDesc, &colocationIdIsNull); int32 colocationId = DatumGetInt32(colocationIdDatum); bool forceDelegationIsNull = false; Datum forceDelegationDatum = heap_getattr(nextTuple, GetForceDelegationAttrIndexInPgDistObject(tupleDesc) + 1, tupleDesc, &forceDelegationIsNull); bool forceDelegation = DatumGetBool(forceDelegationDatum); if (distributionArgumentIndexIsNull) { distributionArgumentIndex = INVALID_DISTRIBUTION_ARGUMENT_INDEX; } if (colocationIdIsNull) { colocationId = INVALID_COLOCATION_ID; } if (forceDelegationIsNull) { forceDelegation = NO_FORCE_PUSHDOWN; } char *workerMetadataUpdateCommand = MarkObjectsDistributedCreateCommand(list_make1(address), NIL, list_make1_int(distributionArgumentIndex), list_make1_int(colocationId), list_make1_int(forceDelegation)); SendOrCollectCommandListToActivatedNodes(context, list_make1(workerMetadataUpdateCommand)); } MemoryContextSwitchTo(oldContext); systable_endscan(scanDesc); relation_close(relation, NoLock); } /* * SendInterTableRelationshipCommands sends inter-table relationship commands * (e.g. constraints, attach partitions) to workers with transactional or * nontransactional mode per inter table relationship according to transactionMode * inside metadataSyncContext. */ void SendInterTableRelationshipCommands(MetadataSyncContext *context) { /* disable ddl propagation */ SendOrCollectCommandListToActivatedNodes(context, list_make1(DISABLE_DDL_PROPAGATION)); ScanKeyData scanKey[1]; int scanKeyCount = 0; Relation relation = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(relation); SysScanDesc scanDesc = systable_beginscan(relation, InvalidOid, false, NULL, scanKeyCount, scanKey); MemoryContext oldContext = MemoryContextSwitchTo(context->context); HeapTuple nextTuple = NULL; while (true) { ResetMetadataSyncMemoryContext(context); nextTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(nextTuple)) { break; } Oid relationId = FetchRelationIdFromPgPartitionHeapTuple(nextTuple, tupleDesc); if (!ShouldSyncTableMetadata(relationId)) { continue; } /* * Skip foreign key and partition creation when the Citus table is * owned by an extension. */ if (IsTableOwnedByExtension(relationId)) { continue; } List *commandList = InterTableRelationshipOfRelationCommandList(relationId); SendOrCollectCommandListToActivatedNodes(context, commandList); } MemoryContextSwitchTo(oldContext); systable_endscan(scanDesc); table_close(relation, AccessShareLock); /* enable ddl propagation */ SendOrCollectCommandListToActivatedNodes(context, list_make1(ENABLE_DDL_PROPAGATION)); } ================================================ FILE: src/backend/distributed/metadata/metadata_utility.c ================================================ /*------------------------------------------------------------------------- * * metadata_utility.c * Routines for reading and modifying master node's metadata. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/pg_authid.h" #include "catalog/pg_constraint.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc_d.h" #include "catalog/pg_type.h" #include "commands/extension.h" #include "commands/sequence.h" #include "nodes/makefuncs.h" #include "parser/scansup.h" #include "storage/lmgr.h" #include "storage/procarray.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/background_jobs.h" #include "distributed/citus_nodes.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_background_job.h" #include "distributed/pg_dist_background_task.h" #include "distributed/pg_dist_backrgound_task_depend.h" #include "distributed/pg_dist_colocation.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_placement.h" #include "distributed/pg_dist_shard.h" #include "distributed/reference_table_utils.h" #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_rebalancer.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #define DISK_SPACE_FIELDS 2 /* Local functions forward declarations */ static uint64 * AllocateUint64(uint64 value); static void RecordDistributedRelationDependencies(Oid distributedRelationId); static GroupShardPlacement * TupleToGroupShardPlacement(TupleDesc tupleDesc, HeapTuple heapTuple); static bool DistributedRelationSize(Oid relationId, SizeQueryType sizeQueryType, bool failOnError, uint64 *relationSize); static bool DistributedRelationSizeOnWorker(WorkerNode *workerNode, Oid relationId, SizeQueryType sizeQueryType, bool failOnError, uint64 *relationSize); static List * ShardIntervalsOnWorkerGroup(WorkerNode *workerNode, Oid relationId); static char * GenerateShardIdNameValuesForShardList(List *shardIntervalList, bool firstValue); static char * GenerateSizeQueryForRelationNameList(List *quotedShardNames, char *sizeFunction); static char * GetWorkerPartitionedSizeUDFNameBySizeQueryType(SizeQueryType sizeQueryType); static char * GetSizeQueryBySizeQueryType(SizeQueryType sizeQueryType); static char * GenerateAllShardStatisticsQueryForNode(WorkerNode *workerNode, List *citusTableIds); static List * GenerateShardStatisticsQueryList(List *workerNodeList, List *citusTableIds); static void ErrorIfNotSuitableToGetSize(Oid relationId); static List * OpenConnectionToNodes(List *workerNodeList); static void ReceiveShardIdAndSizeResults(List *connectionList, Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); static void AppendShardIdNameValues(StringInfo selectQuery, ShardInterval *shardInterval); static HeapTuple CreateDiskSpaceTuple(TupleDesc tupleDesc, uint64 availableBytes, uint64 totalBytes); static bool GetLocalDiskSpaceStats(uint64 *availableBytes, uint64 *totalBytes); static BackgroundTask * DeformBackgroundTaskHeapTuple(TupleDesc tupleDescriptor, HeapTuple taskTuple); static bool SetFieldValue(int attno, Datum values[], bool isnull[], bool replace[], Datum newValue); static bool SetFieldText(int attno, Datum values[], bool isnull[], bool replace[], const char *newValue); static bool SetFieldNull(int attno, Datum values[], bool isnull[], bool replace[]); #define InitFieldValue(attno, values, isnull, initValue) \ (void) SetFieldValue((attno), (values), (isnull), NULL, (initValue)) #define InitFieldText(attno, values, isnull, initValue) \ (void) SetFieldText((attno), (values), (isnull), NULL, (initValue)) #define InitFieldNull(attno, values, isnull) \ (void) SetFieldNull((attno), (values), (isnull), NULL) /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_local_disk_space_stats); PG_FUNCTION_INFO_V1(citus_table_size); PG_FUNCTION_INFO_V1(citus_total_relation_size); PG_FUNCTION_INFO_V1(citus_relation_size); PG_FUNCTION_INFO_V1(citus_shard_sizes); /* * CreateDiskSpaceTuple creates a tuple that is used as the return value * for citus_local_disk_space_stats. */ static HeapTuple CreateDiskSpaceTuple(TupleDesc tupleDescriptor, uint64 availableBytes, uint64 totalBytes) { Datum values[DISK_SPACE_FIELDS]; bool isNulls[DISK_SPACE_FIELDS]; /* form heap tuple for remote disk space statistics */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[0] = UInt64GetDatum(availableBytes); values[1] = UInt64GetDatum(totalBytes); HeapTuple diskSpaceTuple = heap_form_tuple(tupleDescriptor, values, isNulls); return diskSpaceTuple; } /* * citus_local_disk_space_stats returns total disk space and available disk * space for the disk that contains PGDATA. */ Datum citus_local_disk_space_stats(PG_FUNCTION_ARGS) { uint64 availableBytes = 0; uint64 totalBytes = 0; if (!GetLocalDiskSpaceStats(&availableBytes, &totalBytes)) { ereport(WARNING, (errmsg("could not get disk space"))); } TupleDesc tupleDescriptor = NULL; TypeFuncClass resultTypeClass = get_call_result_type(fcinfo, NULL, &tupleDescriptor); if (resultTypeClass != TYPEFUNC_COMPOSITE) { ereport(ERROR, (errmsg("return type must be a row type"))); } HeapTuple diskSpaceTuple = CreateDiskSpaceTuple(tupleDescriptor, availableBytes, totalBytes); PG_RETURN_DATUM(HeapTupleGetDatum(diskSpaceTuple)); } /* * GetLocalDiskSpaceStats returns total and available disk space for the disk containing * PGDATA (not considering tablespaces, quota). */ static bool GetLocalDiskSpaceStats(uint64 *availableBytes, uint64 *totalBytes) { struct statvfs buffer; if (statvfs(DataDir, &buffer) != 0) { return false; } /* * f_bfree: number of free blocks * f_frsize: fragment size, same as f_bsize usually * f_blocks: Size of fs in f_frsize units */ *availableBytes = buffer.f_bfree * buffer.f_frsize; *totalBytes = buffer.f_blocks * buffer.f_frsize; return true; } /* * GetNodeDiskSpaceStatsForConnection fetches the disk space statistics for the node * that is on the given connection, or returns false if unsuccessful. */ bool GetNodeDiskSpaceStatsForConnection(MultiConnection *connection, uint64 *availableBytes, uint64 *totalBytes) { PGresult *result = NULL; char *sizeQuery = "SELECT available_disk_size, total_disk_size " "FROM pg_catalog.citus_local_disk_space_stats()"; int queryResult = ExecuteOptionalRemoteCommand(connection, sizeQuery, &result); if (queryResult != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) != 1) { ereport(WARNING, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot get the disk space statistics for node %s:%d", connection->hostname, connection->port))); PQclear(result); ForgetResults(connection); return false; } char *availableBytesString = PQgetvalue(result, 0, 0); char *totalBytesString = PQgetvalue(result, 0, 1); *availableBytes = SafeStringToUint64(availableBytesString); *totalBytes = SafeStringToUint64(totalBytesString); PQclear(result); ForgetResults(connection); return true; } /* * citus_shard_sizes returns all shard ids and their sizes. */ Datum citus_shard_sizes(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); List *allCitusTableIds = AllCitusTableIds(); /* we don't need a distributed transaction here */ bool useDistributedTransaction = false; List *connectionList = SendShardStatisticsQueriesInParallel(allCitusTableIds, useDistributedTransaction); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); ReceiveShardIdAndSizeResults(connectionList, tupleStore, tupleDescriptor); PG_RETURN_VOID(); } /* * citus_total_relation_size accepts a distributed table name and returns a distributed table * and its indexes' total relation size. */ Datum citus_total_relation_size(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); bool failOnError = PG_GETARG_BOOL(1); SizeQueryType sizeQueryType = TOTAL_RELATION_SIZE; uint64 relationSize = 0; if (!DistributedRelationSize(relationId, sizeQueryType, failOnError, &relationSize)) { Assert(!failOnError); PG_RETURN_NULL(); } PG_RETURN_INT64(relationSize); } /* * citus_table_size accepts a distributed table name and returns a distributed table's total * relation size. */ Datum citus_table_size(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); bool failOnError = true; SizeQueryType sizeQueryType = TABLE_SIZE; uint64 relationSize = 0; /* We do not check if relation is really a table, like PostgreSQL is doing. */ if (!DistributedRelationSize(relationId, sizeQueryType, failOnError, &relationSize)) { Assert(!failOnError); PG_RETURN_NULL(); } PG_RETURN_INT64(relationSize); } /* * citus_relation_size accept a distributed relation name and returns a relation's 'main' * fork's size. * * Input relation is allowed to be an index on a distributed table too. */ Datum citus_relation_size(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); bool failOnError = true; SizeQueryType sizeQueryType = RELATION_SIZE; uint64 relationSize = 0; if (!DistributedRelationSize(relationId, sizeQueryType, failOnError, &relationSize)) { Assert(!failOnError); PG_RETURN_NULL(); } PG_RETURN_INT64(relationSize); } /* * SendShardStatisticsQueriesInParallel generates query lists for obtaining shard * statistics and then sends the commands in parallel by opening connections * to available nodes. It returns the connection list. */ List * SendShardStatisticsQueriesInParallel(List *citusTableIds, bool useDistributedTransaction) { List *workerNodeList = ActivePrimaryNodeList(NoLock); List *shardSizesQueryList = GenerateShardStatisticsQueryList(workerNodeList, citusTableIds); List *connectionList = OpenConnectionToNodes(workerNodeList); FinishConnectionListEstablishment(connectionList); if (useDistributedTransaction) { /* * For now, in the case we want to include shard min and max values, we also * want to update the entries in pg_dist_placement and pg_dist_shard with the * latest statistics. In order to detect distributed deadlocks, we assign a * distributed transaction ID to the current transaction */ UseCoordinatedTransaction(); } /* send commands in parallel */ for (int i = 0; i < list_length(connectionList); i++) { MultiConnection *connection = (MultiConnection *) list_nth(connectionList, i); char *shardSizesQuery = (char *) list_nth(shardSizesQueryList, i); if (useDistributedTransaction) { /* run the size query in a distributed transaction */ RemoteTransactionBeginIfNecessary(connection); } int querySent = SendRemoteCommand(connection, shardSizesQuery); if (querySent == 0) { ReportConnectionError(connection, WARNING); } } return connectionList; } /* * OpenConnectionToNodes opens a single connection per node * for the given workerNodeList. */ static List * OpenConnectionToNodes(List *workerNodeList) { List *connectionList = NIL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int connectionFlags = 0; MultiConnection *connection = StartNodeConnection(connectionFlags, nodeName, nodePort); connectionList = lappend(connectionList, connection); } return connectionList; } /* * GenerateShardStatisticsQueryList generates a query per node that will return: * shard_id, shard_name, shard_size for all shard placements on the node */ static List * GenerateShardStatisticsQueryList(List *workerNodeList, List *citusTableIds) { List *shardStatisticsQueryList = NIL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { char *shardStatisticsQuery = GenerateAllShardStatisticsQueryForNode(workerNode, citusTableIds); shardStatisticsQueryList = lappend(shardStatisticsQueryList, shardStatisticsQuery); } return shardStatisticsQueryList; } /* * ReceiveShardIdAndSizeResults receives shard id and size results from the given * connection list. */ static void ReceiveShardIdAndSizeResults(List *connectionList, Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; Datum values[SHARD_SIZES_COLUMN_COUNT]; bool isNulls[SHARD_SIZES_COLUMN_COUNT]; if (PQstatus(connection->pgConn) != CONNECTION_OK) { continue; } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, WARNING); continue; } int64 rowCount = PQntuples(result); int64 colCount = PQnfields(result); /* Although it is not expected */ if (colCount != SHARD_SIZES_COLUMN_COUNT) { ereport(WARNING, (errmsg("unexpected number of columns from " "citus_shard_sizes"))); continue; } for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); /* format is [0] shard id, [1] size */ values[0] = ParseIntField(result, rowIndex, 0); values[1] = ParseIntField(result, rowIndex, 1); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } PQclear(result); ForgetResults(connection); } } /* * DistributedRelationSize is helper function for each kind of citus size * functions. It first checks whether the relation is a distributed table or an * index belonging to a distributed table and size query can be run on it. * Connection to each node has to be established to get the size of the * relation. * Input relation is allowed to be an index on a distributed table too. */ static bool DistributedRelationSize(Oid relationId, SizeQueryType sizeQueryType, bool failOnError, uint64 *relationSize) { int logLevel = WARNING; if (failOnError) { logLevel = ERROR; } uint64 sumOfSizes = 0; if (XactModificationLevel == XACT_MODIFICATION_DATA) { ereport(logLevel, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("citus size functions cannot be called in transaction " "blocks which contain multi-shard data " "modifications"))); return false; } Relation relation = try_relation_open(relationId, AccessShareLock); if (relation == NULL) { ereport(logLevel, (errmsg("could not compute relation size: relation does not exist"))); return false; } ErrorIfNotSuitableToGetSize(relationId); table_close(relation, AccessShareLock); List *workerNodeList = ActiveReadableNodeList(); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { uint64 relationSizeOnNode = 0; bool gotSize = DistributedRelationSizeOnWorker(workerNode, relationId, sizeQueryType, failOnError, &relationSizeOnNode); if (!gotSize) { return false; } sumOfSizes += relationSizeOnNode; } *relationSize = sumOfSizes; return true; } /* * DistributedRelationSizeOnWorker gets the workerNode and relationId to calculate * size of that relation on the given workerNode by summing up the size of each * shard placement. * Input relation is allowed to be an index on a distributed table too. */ static bool DistributedRelationSizeOnWorker(WorkerNode *workerNode, Oid relationId, SizeQueryType sizeQueryType, bool failOnError, uint64 *relationSize) { int logLevel = WARNING; if (failOnError) { logLevel = ERROR; } char *workerNodeName = workerNode->workerName; uint32 workerNodePort = workerNode->workerPort; uint32 connectionFlag = 0; PGresult *result = NULL; /* if the relation is an index, update relationId and define indexId */ Oid indexId = InvalidOid; Oid relKind = get_rel_relkind(relationId); if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { indexId = relationId; bool missingOk = false; relationId = IndexGetRelation(indexId, missingOk); } List *shardIntervalsOnNode = ShardIntervalsOnWorkerGroup(workerNode, relationId); /* * We pass false here, because if we optimize this, we would include child tables. * But citus size functions shouldn't include them, like PG. */ bool optimizePartitionCalculations = false; StringInfo relationSizeQuery = GenerateSizeQueryOnMultiplePlacements( shardIntervalsOnNode, indexId, sizeQueryType, optimizePartitionCalculations); MultiConnection *connection = GetNodeConnection(connectionFlag, workerNodeName, workerNodePort); int queryResult = ExecuteOptionalRemoteCommand(connection, relationSizeQuery->data, &result); if (queryResult != 0) { ereport(logLevel, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("could not connect to %s:%d to get size of " "relation \"%s\"", workerNodeName, workerNodePort, get_rel_name(relationId)))); return false; } List *sizeList = ReadFirstColumnAsText(result); if (list_length(sizeList) != 1) { PQclear(result); ClearResults(connection, failOnError); ereport(logLevel, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse size of relation \"%s\" from %s:%d", get_rel_name(relationId), workerNodeName, workerNodePort))); return false; } StringInfo relationSizeStringInfo = (StringInfo) linitial(sizeList); char *relationSizeString = relationSizeStringInfo->data; if (strlen(relationSizeString) > 0) { *relationSize = SafeStringToUint64(relationSizeString); } else { /* * This means the shard is moved or dropped while citus_total_relation_size is * being executed. For this case we get an empty string as table size. * We can take that as zero to prevent any unnecessary errors. */ *relationSize = 0; } PQclear(result); ClearResults(connection, failOnError); return true; } /* * GroupShardPlacementsForTableOnGroup accepts a relationId and a group and returns a list * of GroupShardPlacement's representing all of the placements for the table which reside * on the group. */ List * GroupShardPlacementsForTableOnGroup(Oid relationId, int32 groupId) { CitusTableCacheEntry *distTableCacheEntry = GetCitusTableCacheEntry(relationId); List *resultList = NIL; int shardIntervalArrayLength = distTableCacheEntry->shardIntervalArrayLength; for (int shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++) { GroupShardPlacement *placementArray = distTableCacheEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = distTableCacheEntry->arrayOfPlacementArrayLengths[shardIndex]; for (int placementIndex = 0; placementIndex < numberOfPlacements; placementIndex++) { if (placementArray[placementIndex].groupId == groupId) { GroupShardPlacement *placement = palloc0(sizeof(GroupShardPlacement)); *placement = placementArray[placementIndex]; resultList = lappend(resultList, placement); } } } return resultList; } /* * ShardIntervalsOnWorkerGroup accepts a WorkerNode and returns a list of the shard * intervals of the given table which are placed on the group the node is a part of. */ static List * ShardIntervalsOnWorkerGroup(WorkerNode *workerNode, Oid relationId) { CitusTableCacheEntry *distTableCacheEntry = GetCitusTableCacheEntry(relationId); List *shardIntervalList = NIL; int shardIntervalArrayLength = distTableCacheEntry->shardIntervalArrayLength; for (int shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++) { GroupShardPlacement *placementArray = distTableCacheEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = distTableCacheEntry->arrayOfPlacementArrayLengths[shardIndex]; for (int placementIndex = 0; placementIndex < numberOfPlacements; placementIndex++) { GroupShardPlacement *placement = &placementArray[placementIndex]; if (placement->groupId == workerNode->groupId) { ShardInterval *cachedShardInterval = distTableCacheEntry->sortedShardIntervalArray[shardIndex]; ShardInterval *shardInterval = CopyShardInterval(cachedShardInterval); shardIntervalList = lappend(shardIntervalList, shardInterval); } } } return shardIntervalList; } /* * GenerateSizeQueryOnMultiplePlacements generates a select size query to get * size of multiple relations. Note that, different size functions supported by PG * are also supported by this function changing the size query type given as the * last parameter to function. Depending on the sizeQueryType enum parameter, the * generated query will call one of the functions: pg_relation_size, * pg_total_relation_size, pg_table_size and cstore_table_size. * This function uses UDFs named worker_partitioned_*_size for partitioned tables, * if the parameter optimizePartitionCalculations is true. The UDF to be called is * determined by the parameter sizeQueryType. * * indexId is provided if we're interested in the size of an index, not the whole * table. */ StringInfo GenerateSizeQueryOnMultiplePlacements(List *shardIntervalList, Oid indexId, SizeQueryType sizeQueryType, bool optimizePartitionCalculations) { StringInfo selectQuery = makeStringInfo(); List *partitionedShardNames = NIL; List *nonPartitionedShardNames = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { if (optimizePartitionCalculations && PartitionTable(shardInterval->relationId)) { /* * Skip child tables of a partitioned table as they are already counted in * worker_partitioned_*_size UDFs, if optimizePartitionCalculations is true. * We don't expect this case to happen, since we don't send the child tables * to this function. Because they are all eliminated in * ColocatedNonPartitionShardIntervalList. Therefore we can't cover here with * a test currently. This is added for possible future usages. */ continue; } /* we need to build the shard relation name, being an index or table */ Oid objectId = OidIsValid(indexId) ? indexId : shardInterval->relationId; uint64 shardId = shardInterval->shardId; Oid schemaId = get_rel_namespace(objectId); char *schemaName = get_namespace_name(schemaId); char *shardName = get_rel_name(objectId); AppendShardIdToName(&shardName, shardId); char *shardQualifiedName = quote_qualified_identifier(schemaName, shardName); char *quotedShardName = quote_literal_cstr(shardQualifiedName); /* for partitioned tables, we will call worker_partitioned_... size functions */ if (optimizePartitionCalculations && PartitionedTable(shardInterval->relationId)) { partitionedShardNames = lappend(partitionedShardNames, quotedShardName); } /* for non-partitioned tables, we will use Postgres' size functions */ else { nonPartitionedShardNames = lappend(nonPartitionedShardNames, quotedShardName); } } /* SELECT SUM(worker_partitioned_...) FROM VALUES (...) */ char *subqueryForPartitionedShards = GenerateSizeQueryForRelationNameList(partitionedShardNames, GetWorkerPartitionedSizeUDFNameBySizeQueryType (sizeQueryType)); /* SELECT SUM(pg_..._size) FROM VALUES (...) */ char *subqueryForNonPartitionedShards = GenerateSizeQueryForRelationNameList(nonPartitionedShardNames, GetSizeQueryBySizeQueryType(sizeQueryType)); appendStringInfo(selectQuery, "SELECT (%s) + (%s);", subqueryForPartitionedShards, subqueryForNonPartitionedShards); elog(DEBUG4, "Size Query: %s", selectQuery->data); return selectQuery; } /* * GenerateSizeQueryForPartitionedShards generates and returns a query with a template: * SELECT SUM( (relid) ) FROM (VALUES (), (), ...) as q(relid) */ static char * GenerateSizeQueryForRelationNameList(List *quotedShardNames, char *sizeFunction) { if (list_length(quotedShardNames) == 0) { return "SELECT 0"; } StringInfo selectQuery = makeStringInfo(); appendStringInfo(selectQuery, "SELECT SUM("); appendStringInfo(selectQuery, sizeFunction, "relid"); appendStringInfo(selectQuery, ") FROM (VALUES "); bool addComma = false; char *quotedShardName = NULL; foreach_declared_ptr(quotedShardName, quotedShardNames) { if (addComma) { appendStringInfoString(selectQuery, ", "); } addComma = true; appendStringInfo(selectQuery, "(%s)", quotedShardName); } appendStringInfoString(selectQuery, ") as q(relid)"); return selectQuery->data; } /* * GetWorkerPartitionedSizeUDFNameBySizeQueryType returns the corresponding worker * partitioned size query for given query type. * Errors out for an invalid query type. * Currently this function is only called with the type TOTAL_RELATION_SIZE. * The others are added for possible future usages. Since they are not used anywhere, * currently we can't cover them with tests. */ static char * GetWorkerPartitionedSizeUDFNameBySizeQueryType(SizeQueryType sizeQueryType) { switch (sizeQueryType) { case RELATION_SIZE: { return WORKER_PARTITIONED_RELATION_SIZE_FUNCTION; } case TOTAL_RELATION_SIZE: { return WORKER_PARTITIONED_RELATION_TOTAL_SIZE_FUNCTION; } case TABLE_SIZE: { return WORKER_PARTITIONED_TABLE_SIZE_FUNCTION; } default: { elog(ERROR, "Size query type couldn't be found."); } } } /* * GetSizeQueryBySizeQueryType returns the corresponding size query for given query type. * Errors out for an invalid query type. */ static char * GetSizeQueryBySizeQueryType(SizeQueryType sizeQueryType) { switch (sizeQueryType) { case RELATION_SIZE: { return PG_RELATION_SIZE_FUNCTION; } case TOTAL_RELATION_SIZE: { return PG_TOTAL_RELATION_SIZE_FUNCTION; } case TABLE_SIZE: { return PG_TABLE_SIZE_FUNCTION; } default: { elog(ERROR, "Size query type couldn't be found."); } } } /* * GenerateAllShardStatisticsQueryForNode generates a query that returns: * shard_id, shard_name, shard_size for all shard placements on the node */ static char * GenerateAllShardStatisticsQueryForNode(WorkerNode *workerNode, List *citusTableIds) { StringInfo allShardStatisticsQuery = makeStringInfo(); bool insertedValues = false; appendStringInfoString(allShardStatisticsQuery, "SELECT shard_id, "); appendStringInfo(allShardStatisticsQuery, PG_TOTAL_RELATION_SIZE_FUNCTION, "table_name"); appendStringInfoString(allShardStatisticsQuery, " FROM (VALUES "); Oid relationId = InvalidOid; foreach_declared_oid(relationId, citusTableIds) { /* * Ensure the table still exists by trying to acquire a lock on it * If function returns NULL, it means the table doesn't exist * hence we should skip */ Relation relation = try_relation_open(relationId, AccessShareLock); if (relation != NULL) { List *shardIntervalsOnNode = ShardIntervalsOnWorkerGroup(workerNode, relationId); if (list_length(shardIntervalsOnNode) == 0) { relation_close(relation, AccessShareLock); continue; } char *shardIdNameValues = GenerateShardIdNameValuesForShardList(shardIntervalsOnNode, !insertedValues); insertedValues = true; appendStringInfoString(allShardStatisticsQuery, shardIdNameValues); relation_close(relation, AccessShareLock); } } if (!insertedValues) { return "SELECT 0 AS shard_id, '' AS table_name LIMIT 0"; } appendStringInfoString(allShardStatisticsQuery, ") t(shard_id, table_name) " "WHERE to_regclass(table_name) IS NOT NULL"); return allShardStatisticsQuery->data; } /* * GenerateShardIdNameValuesForShardList generates a list of (shard_id, shard_name) values * for all shards in the list */ static char * GenerateShardIdNameValuesForShardList(List *shardIntervalList, bool firstValue) { StringInfo selectQuery = makeStringInfo(); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { if (!firstValue) { appendStringInfoString(selectQuery, ", "); } firstValue = false; AppendShardIdNameValues(selectQuery, shardInterval); } return selectQuery->data; } /* * AppendShardIdNameValues appends (shard_id, shard_name) for shard */ static void AppendShardIdNameValues(StringInfo selectQuery, ShardInterval *shardInterval) { uint64 shardId = shardInterval->shardId; Oid schemaId = get_rel_namespace(shardInterval->relationId); char *schemaName = get_namespace_name(schemaId); char *shardName = get_rel_name(shardInterval->relationId); AppendShardIdToName(&shardName, shardId); char *shardQualifiedName = quote_qualified_identifier(schemaName, shardName); char *quotedShardName = quote_literal_cstr(shardQualifiedName); appendStringInfo(selectQuery, "(" UINT64_FORMAT ", %s)", shardId, quotedShardName); } /* * ErrorIfNotSuitableToGetSize determines whether the relation is suitable to find * its' size with internal functions. */ static void ErrorIfNotSuitableToGetSize(Oid relationId) { if (!IsCitusTable(relationId)) { Oid relKind = get_rel_relkind(relationId); if (relKind != RELKIND_INDEX && relKind != RELKIND_PARTITIONED_INDEX) { char *relationName = get_rel_name(relationId); char *escapedRelationName = quote_literal_cstr(relationName); ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg( "cannot calculate the size because relation %s " "is not distributed", escapedRelationName))); } bool missingOk = false; Oid indexId = relationId; relationId = IndexGetRelation(relationId, missingOk); if (!IsCitusTable(relationId)) { char *tableName = get_rel_name(relationId); char *escapedTableName = quote_literal_cstr(tableName); char *indexName = get_rel_name(indexId); char *escapedIndexName = quote_literal_cstr(indexName); ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg( "cannot calculate the size because table %s for " "index %s is not distributed", escapedTableName, escapedIndexName))); } } } /* * CompareShardPlacementsByWorker compares two shard placements by their * worker node name and port. */ int CompareShardPlacementsByWorker(const void *leftElement, const void *rightElement) { const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); int nodeNameCmp = strncmp(leftPlacement->nodeName, rightPlacement->nodeName, WORKER_LENGTH); if (nodeNameCmp != 0) { return nodeNameCmp; } else if (leftPlacement->nodePort > rightPlacement->nodePort) { return 1; } else if (leftPlacement->nodePort < rightPlacement->nodePort) { return -1; } return 0; } /* * CompareShardPlacementsByGroupId compares two shard placements by their * group id. */ int CompareShardPlacementsByGroupId(const void *leftElement, const void *rightElement) { const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); if (leftPlacement->groupId > rightPlacement->groupId) { return 1; } else if (leftPlacement->groupId < rightPlacement->groupId) { return -1; } else { return 0; } } /* * TableShardReplicationFactor returns the current replication factor of the * given relation by looking into shard placements. It errors out if there * are different number of shard placements for different shards. It also * errors out if the table does not have any shards. */ uint32 TableShardReplicationFactor(Oid relationId) { uint32 replicationCount = 0; List *shardIntervalList = LoadShardIntervalList(relationId); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; List *shardPlacementList = ShardPlacementListSortedByWorker(shardId); uint32 shardPlacementCount = list_length(shardPlacementList); /* * Get the replication count of the first shard in the list, and error * out if there is a shard with different replication count. */ if (replicationCount == 0) { replicationCount = shardPlacementCount; } else if (replicationCount != shardPlacementCount) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot find the replication factor of the " "table %s", relationName), errdetail("The shard " UINT64_FORMAT " has different shards replication counts from " "other shards.", shardId))); } } /* error out if the table does not have any shards */ if (replicationCount == 0) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot find the replication factor of the " "table %s", relationName), errdetail("The table %s does not have any shards.", relationName))); } return replicationCount; } /* * LoadShardIntervalList returns a list of shard intervals related for a given * distributed table. The function returns an empty list if no shards can be * found for the given relation. * Since LoadShardIntervalList relies on sortedShardIntervalArray, it returns * a shard interval list whose elements are sorted on shardminvalue. Shard intervals * with uninitialized shard min/max values are placed in the end of the list. */ List * LoadShardIntervalList(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *shardList = NIL; for (int i = 0; i < cacheEntry->shardIntervalArrayLength; i++) { ShardInterval *newShardInterval = CopyShardInterval(cacheEntry->sortedShardIntervalArray[i]); shardList = lappend(shardList, newShardInterval); } return shardList; } /* * LoadUnsortedShardIntervalListViaCatalog returns a list of shard intervals related for a * given distributed table. The function returns an empty list if no shards can be found * for the given relation. * * This function does not use CitusTableCache and instead reads from catalog tables * directly. */ List * LoadUnsortedShardIntervalListViaCatalog(Oid relationId) { List *shardIntervalList = NIL; List *distShardTuples = LookupDistShardTuples(relationId); Relation distShardRelation = table_open(DistShardRelationId(), AccessShareLock); TupleDesc distShardTupleDesc = RelationGetDescr(distShardRelation); Oid intervalTypeId = InvalidOid; int32 intervalTypeMod = -1; char partitionMethod = PartitionMethodViaCatalog(relationId); Var *partitionColumn = PartitionColumnViaCatalog(relationId); GetIntervalTypeInfo(partitionMethod, partitionColumn, &intervalTypeId, &intervalTypeMod); HeapTuple distShardTuple = NULL; foreach_declared_ptr(distShardTuple, distShardTuples) { ShardInterval *interval = TupleToShardInterval(distShardTuple, distShardTupleDesc, intervalTypeId, intervalTypeMod); shardIntervalList = lappend(shardIntervalList, interval); } table_close(distShardRelation, AccessShareLock); return shardIntervalList; } /* * LoadShardIntervalWithLongestShardName is a utility function that returns * the shard interaval with the largest shardId for the given relationId. Note * that largest shardId implies longest shard name. */ ShardInterval * LoadShardIntervalWithLongestShardName(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); int shardIntervalCount = cacheEntry->shardIntervalArrayLength; int maxShardIndex = shardIntervalCount - 1; uint64 largestShardId = INVALID_SHARD_ID; for (int shardIndex = 0; shardIndex <= maxShardIndex; ++shardIndex) { ShardInterval *currentShardInterval = cacheEntry->sortedShardIntervalArray[shardIndex]; if (largestShardId < currentShardInterval->shardId) { largestShardId = currentShardInterval->shardId; } } return LoadShardInterval(largestShardId); } /* * ShardIntervalCount returns number of shard intervals for a given distributed table. * The function returns 0 if no shards can be found for the given relation id. */ int ShardIntervalCount(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); return cacheEntry->shardIntervalArrayLength; } /* * LoadShardList reads list of shards for given relationId from pg_dist_shard, * and returns the list of found shardIds. * Since LoadShardList relies on sortedShardIntervalArray, it returns a shard * list whose elements are sorted on shardminvalue. Shards with uninitialized * shard min/max values are placed in the end of the list. */ List * LoadShardList(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *shardList = NIL; for (int i = 0; i < cacheEntry->shardIntervalArrayLength; i++) { ShardInterval *currentShardInterval = cacheEntry->sortedShardIntervalArray[i]; uint64 *shardIdPointer = AllocateUint64(currentShardInterval->shardId); shardList = lappend(shardList, shardIdPointer); } return shardList; } /* Allocates eight bytes, and copies given value's contents those bytes. */ static uint64 * AllocateUint64(uint64 value) { uint64 *allocatedValue = (uint64 *) palloc0(sizeof(uint64)); Assert(sizeof(uint64) >= 8); (*allocatedValue) = value; return allocatedValue; } /* * CopyShardInterval creates a copy of the specified source ShardInterval. */ ShardInterval * CopyShardInterval(ShardInterval *srcInterval) { ShardInterval *destInterval = palloc0(sizeof(ShardInterval)); destInterval->type = srcInterval->type; destInterval->relationId = srcInterval->relationId; destInterval->storageType = srcInterval->storageType; destInterval->valueTypeId = srcInterval->valueTypeId; destInterval->valueTypeLen = srcInterval->valueTypeLen; destInterval->valueByVal = srcInterval->valueByVal; destInterval->minValueExists = srcInterval->minValueExists; destInterval->maxValueExists = srcInterval->maxValueExists; destInterval->shardId = srcInterval->shardId; destInterval->shardIndex = srcInterval->shardIndex; destInterval->minValue = 0; if (destInterval->minValueExists) { destInterval->minValue = datumCopy(srcInterval->minValue, srcInterval->valueByVal, srcInterval->valueTypeLen); } destInterval->maxValue = 0; if (destInterval->maxValueExists) { destInterval->maxValue = datumCopy(srcInterval->maxValue, srcInterval->valueByVal, srcInterval->valueTypeLen); } return destInterval; } /* * ShardLength finds shard placements for the given shardId, extracts the length * of an active shard, and returns the shard's length. This function errors * out if we cannot find any active shard placements for the given shardId. */ uint64 ShardLength(uint64 shardId) { uint64 shardLength = 0; List *shardPlacementList = ActiveShardPlacementList(shardId); if (shardPlacementList == NIL) { ereport(ERROR, (errmsg("could not find length of shard " UINT64_FORMAT, shardId), errdetail("Could not find any shard placements for the shard."))); } else { ShardPlacement *shardPlacement = (ShardPlacement *) linitial(shardPlacementList); shardLength = shardPlacement->shardLength; } return shardLength; } /* * NodeGroupHasShardPlacements returns whether any active shards are placed on the group */ bool NodeGroupHasShardPlacements(int32 groupId) { const int scanKeyCount = 1; const bool indexOK = false; ScanKeyData scanKey[1]; Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); SysScanDesc scanDescriptor = systable_beginscan(pgPlacement, DistPlacementGroupidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); bool hasActivePlacements = HeapTupleIsValid(heapTuple); systable_endscan(scanDescriptor); table_close(pgPlacement, NoLock); return hasActivePlacements; } /* * IsActiveShardPlacement checks if the shard placement is labelled as * active, and that it is placed in an active worker. * Expects shard worker to not be NULL. */ bool IsActiveShardPlacement(ShardPlacement *shardPlacement) { WorkerNode *workerNode = FindWorkerNode(shardPlacement->nodeName, shardPlacement->nodePort); if (!workerNode) { ereport(ERROR, (errmsg("There is a shard placement on node %s:%d but " "could not find the node.", shardPlacement->nodeName, shardPlacement->nodePort))); } return workerNode->isActive; } /* * IsRemoteShardPlacement returns true if the shard placement is on a remote * node. */ bool IsRemoteShardPlacement(ShardPlacement *shardPlacement) { return shardPlacement->groupId != GetLocalGroupId(); } /* * IsPlacementOnWorkerNode checks if the shard placement is for to the given * workenode. */ bool IsPlacementOnWorkerNode(ShardPlacement *placement, WorkerNode *workerNode) { if (strncmp(workerNode->workerName, placement->nodeName, WORKER_LENGTH) != 0) { return false; } return workerNode->workerPort == placement->nodePort; } /* * FilterShardPlacementList filters a list of shard placements based on a filter. * Keep only the shard for which the filter function returns true. */ List * FilterShardPlacementList(List *shardPlacementList, bool (*filter)(ShardPlacement *)) { List *filteredShardPlacementList = NIL; ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, shardPlacementList) { if (filter(shardPlacement)) { filteredShardPlacementList = lappend(filteredShardPlacementList, shardPlacement); } } return filteredShardPlacementList; } /* * FilterActiveShardPlacementListByNode filters a list of active shard placements based on given nodeName and nodePort. */ List * FilterActiveShardPlacementListByNode(List *shardPlacementList, WorkerNode *workerNode) { List *activeShardPlacementList = FilterShardPlacementList(shardPlacementList, IsActiveShardPlacement); List *filteredShardPlacementList = NIL; ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, activeShardPlacementList) { if (IsPlacementOnWorkerNode(shardPlacement, workerNode)) { filteredShardPlacementList = lappend(filteredShardPlacementList, shardPlacement); } } return filteredShardPlacementList; } /* * ActiveShardPlacementListOnGroup returns a list of active shard placements * that are sitting on group with groupId for given shardId. */ List * ActiveShardPlacementListOnGroup(uint64 shardId, int32 groupId) { List *activeShardPlacementListOnGroup = NIL; List *activePlacementList = ActiveShardPlacementList(shardId); ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, activePlacementList) { if (shardPlacement->groupId == groupId) { activeShardPlacementListOnGroup = lappend(activeShardPlacementListOnGroup, shardPlacement); } } return activeShardPlacementListOnGroup; } /* * ActiveShardPlacementList finds shard placements for the given shardId from * system catalogs, chooses placements that are in active state, and returns * these shard placements in a new list. */ List * ActiveShardPlacementList(uint64 shardId) { List *shardPlacementList = ShardPlacementList(shardId); List *activePlacementList = FilterShardPlacementList(shardPlacementList, IsActiveShardPlacement); return SortList(activePlacementList, CompareShardPlacementsByWorker); } /* * ShardPlacementListSortedByWorker returns shard placements sorted by worker port. */ List * ShardPlacementListSortedByWorker(uint64 shardId) { List *shardPlacementList = ShardPlacementList(shardId); return SortList(shardPlacementList, CompareShardPlacementsByWorker); } /* * ActiveShardPlacement finds a shard placement for the given shardId from * system catalog, chooses a placement that is in active state and returns * that shard placement. If this function cannot find a healthy shard placement * and missingOk is set to false it errors out. */ ShardPlacement * ActiveShardPlacement(uint64 shardId, bool missingOk) { List *activePlacementList = ActiveShardPlacementList(shardId); ShardPlacement *shardPlacement = NULL; if (list_length(activePlacementList) == 0) { if (!missingOk) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not find any healthy placement for shard " UINT64_FORMAT, shardId))); } return shardPlacement; } shardPlacement = (ShardPlacement *) linitial(activePlacementList); return shardPlacement; } /* * ActiveShardPlacementWorkerNode returns the worker node of the first placement of * a shard. */ WorkerNode * ActiveShardPlacementWorkerNode(uint64 shardId) { bool missingOK = false; List *sourcePlacementList = ActiveShardPlacementList(shardId); Assert(sourcePlacementList->length == 1); ShardPlacement *sourceShardPlacement = linitial(sourcePlacementList); WorkerNode *sourceShardToCopyNode = FindNodeWithNodeId(sourceShardPlacement->nodeId, missingOK); return sourceShardToCopyNode; } /* * BuildShardPlacementList finds shard placements for the given shardId from * system catalogs, converts these placements to their in-memory * representation, and returns the converted shard placements in a new list. * * This probably only should be called from metadata_cache.c. Resides here * because it shares code with other routines in this file. */ List * BuildShardPlacementList(int64 shardId) { List *shardPlacementList = NIL; ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); SysScanDesc scanDescriptor = systable_beginscan(pgPlacement, DistPlacementShardidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgPlacement); GroupShardPlacement *placement = TupleToGroupShardPlacement(tupleDescriptor, heapTuple); shardPlacementList = lappend(shardPlacementList, placement); heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgPlacement, NoLock); return shardPlacementList; } /* * BuildShardPlacementListForGroup finds shard placements for the given groupId * from system catalogs, converts these placements to their in-memory * representation, and returns the converted shard placements in a new list. */ List * AllShardPlacementsOnNodeGroup(int32 groupId) { List *shardPlacementList = NIL; ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); SysScanDesc scanDescriptor = systable_beginscan(pgPlacement, DistPlacementGroupidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgPlacement); GroupShardPlacement *placement = TupleToGroupShardPlacement(tupleDescriptor, heapTuple); shardPlacementList = lappend(shardPlacementList, placement); heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgPlacement, NoLock); return shardPlacementList; } /* * TupleToGroupShardPlacement takes in a heap tuple from pg_dist_placement, * and converts this tuple to in-memory struct. The function assumes the * caller already has locks on the tuple, and doesn't perform any locking. */ static GroupShardPlacement * TupleToGroupShardPlacement(TupleDesc tupleDescriptor, HeapTuple heapTuple) { bool isNullArray[Natts_pg_dist_placement]; Datum datumArray[Natts_pg_dist_placement]; if (HeapTupleHeaderGetNatts(heapTuple->t_data) != Natts_pg_dist_placement || HeapTupleHasNulls(heapTuple)) { ereport(ERROR, (errmsg("unexpected null in pg_dist_placement tuple"))); } /* * We use heap_deform_tuple() instead of heap_getattr() to expand tuple * to contain missing values when ALTER TABLE ADD COLUMN happens. */ heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); GroupShardPlacement *shardPlacement = CitusMakeNode(GroupShardPlacement); shardPlacement->placementId = DatumGetInt64( datumArray[Anum_pg_dist_placement_placementid - 1]); shardPlacement->shardId = DatumGetInt64( datumArray[Anum_pg_dist_placement_shardid - 1]); shardPlacement->shardLength = DatumGetInt64( datumArray[Anum_pg_dist_placement_shardlength - 1]); shardPlacement->groupId = DatumGetInt32( datumArray[Anum_pg_dist_placement_groupid - 1]); return shardPlacement; } /* * LookupTaskPlacementHostAndPort sets the nodename and nodeport for the given task placement * with a lookup. */ void LookupTaskPlacementHostAndPort(ShardPlacement *taskPlacement, char **nodeName, int *nodePort) { if (IsDummyPlacement(taskPlacement)) { /* * If we create a dummy placement for the local node, it is possible * that the entry doesn't exist in pg_dist_node, hence a lookup will fail. * In that case we want to use the dummy placements values. */ *nodeName = taskPlacement->nodeName; *nodePort = taskPlacement->nodePort; } else { /* * We want to lookup the node information again since it is possible that * there were changes in pg_dist_node and we will get those invalidations * in LookupNodeForGroup. */ WorkerNode *workerNode = LookupNodeForGroup(taskPlacement->groupId); *nodeName = workerNode->workerName; *nodePort = workerNode->workerPort; } } /* * IsDummyPlacement returns true if the given placement is a dummy placement. */ bool IsDummyPlacement(ShardPlacement *taskPlacement) { return taskPlacement->nodeId == LOCAL_NODE_ID; } /* * InsertShardRow opens the shard system catalog, and inserts a new row with the * given values into that system catalog. Note that we allow the user to pass in * null min/max values in case they are creating an empty shard. */ void InsertShardRow(Oid relationId, uint64 shardId, char storageType, text *shardMinValue, text *shardMaxValue) { Datum values[Natts_pg_dist_shard]; bool isNulls[Natts_pg_dist_shard]; /* form new shard tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[Anum_pg_dist_shard_logicalrelid - 1] = ObjectIdGetDatum(relationId); values[Anum_pg_dist_shard_shardid - 1] = Int64GetDatum(shardId); values[Anum_pg_dist_shard_shardstorage - 1] = CharGetDatum(storageType); /* dropped shardalias column must also be set; it is still part of the tuple */ isNulls[Anum_pg_dist_shard_shardalias_DROPPED - 1] = true; /* check if shard min/max values are null */ if (shardMinValue != NULL && shardMaxValue != NULL) { values[Anum_pg_dist_shard_shardminvalue - 1] = PointerGetDatum(shardMinValue); values[Anum_pg_dist_shard_shardmaxvalue - 1] = PointerGetDatum(shardMaxValue); } else { isNulls[Anum_pg_dist_shard_shardminvalue - 1] = true; isNulls[Anum_pg_dist_shard_shardmaxvalue - 1] = true; } /* open shard relation and insert new tuple */ Relation pgDistShard = table_open(DistShardRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistShard); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); CatalogTupleInsert(pgDistShard, heapTuple); /* invalidate previous cache entry and close relation */ CitusInvalidateRelcacheByRelid(relationId); CommandCounterIncrement(); table_close(pgDistShard, NoLock); } /* * InsertShardPlacementRowGlobally inserts shard placement that has given * parameters into pg_dist_placement globally. */ ShardPlacement * InsertShardPlacementRowGlobally(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId) { InsertShardPlacementRow(shardId, placementId, shardLength, groupId); char *insertPlacementCommand = AddPlacementMetadataCommand(shardId, placementId, shardLength, groupId); SendCommandToWorkersWithMetadata(insertPlacementCommand); return LoadShardPlacement(shardId, placementId); } /* * InsertShardPlacementRow opens the shard placement system catalog, and inserts * a new row with the given values into that system catalog. If placementId is * INVALID_PLACEMENT_ID, a new placement id will be assigned.Then, returns the * placement id of the added shard placement. */ uint64 InsertShardPlacementRow(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId) { Datum values[Natts_pg_dist_placement]; bool isNulls[Natts_pg_dist_placement]; /* form new shard placement tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); if (placementId == INVALID_PLACEMENT_ID) { placementId = master_get_new_placementid(NULL); } values[Anum_pg_dist_placement_placementid - 1] = Int64GetDatum(placementId); values[Anum_pg_dist_placement_shardid - 1] = Int64GetDatum(shardId); values[Anum_pg_dist_placement_shardstate - 1] = Int32GetDatum(1); values[Anum_pg_dist_placement_shardlength - 1] = Int64GetDatum(shardLength); values[Anum_pg_dist_placement_groupid - 1] = Int32GetDatum(groupId); /* open shard placement relation and insert new tuple */ Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); CatalogTupleInsert(pgDistPlacement, heapTuple); CitusInvalidateRelcacheByShardId(shardId); CommandCounterIncrement(); table_close(pgDistPlacement, NoLock); return placementId; } /* * InsertIntoPgDistPartition inserts a new tuple into pg_dist_partition. */ void InsertIntoPgDistPartition(Oid relationId, char distributionMethod, Var *distributionColumn, uint32 colocationId, char replicationModel, bool autoConverted) { char *distributionColumnString = NULL; /* open system catalog and insert new tuple */ Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *newValues = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *newNulls = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); /* form new tuple for pg_dist_partition */ newValues[Anum_pg_dist_partition_logicalrelid - 1] = ObjectIdGetDatum(relationId); newValues[Anum_pg_dist_partition_partmethod - 1] = CharGetDatum(distributionMethod); newValues[Anum_pg_dist_partition_colocationid - 1] = UInt32GetDatum(colocationId); newValues[Anum_pg_dist_partition_repmodel - 1] = CharGetDatum(replicationModel); newValues[GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor)] = BoolGetDatum(autoConverted); /* set partkey column to NULL for reference tables */ if (distributionMethod != DISTRIBUTE_BY_NONE) { distributionColumnString = nodeToString((Node *) distributionColumn); newValues[Anum_pg_dist_partition_partkey - 1] = CStringGetTextDatum(distributionColumnString); } else { newValues[Anum_pg_dist_partition_partkey - 1] = PointerGetDatum(NULL); newNulls[Anum_pg_dist_partition_partkey - 1] = true; } HeapTuple newTuple = heap_form_tuple(tupleDescriptor, newValues, newNulls); /* finally insert tuple, build index entries & register cache invalidation */ CatalogTupleInsert(pgDistPartition, newTuple); CitusInvalidateRelcacheByRelid(relationId); RecordDistributedRelationDependencies(relationId); CommandCounterIncrement(); table_close(pgDistPartition, NoLock); pfree(newValues); pfree(newNulls); } /* * RecordDistributedRelationDependencies creates the dependency entries * necessary for a distributed relation in addition to the preexisting ones * for a normal relation. * * We create one dependency from the (now distributed) relation to the citus * extension to prevent the extension from being dropped while distributed * tables exist. Furthermore a dependency from pg_dist_partition's * distribution clause to the underlying columns is created, but it's marked * as being owned by the relation itself. That means the entire table can be * dropped, but the column itself can't. Neither can the type of the * distribution column be changed (c.f. ATExecAlterColumnType). */ static void RecordDistributedRelationDependencies(Oid distributedRelationId) { ObjectAddress relationAddr = { 0, 0, 0 }; ObjectAddress citusExtensionAddr = { 0, 0, 0 }; relationAddr.classId = RelationRelationId; relationAddr.objectId = distributedRelationId; relationAddr.objectSubId = 0; citusExtensionAddr.classId = ExtensionRelationId; citusExtensionAddr.objectId = get_extension_oid("citus", false); citusExtensionAddr.objectSubId = 0; /* dependency from table entry to extension */ recordDependencyOn(&relationAddr, &citusExtensionAddr, DEPENDENCY_NORMAL); } /* * DeletePartitionRow removes the row from pg_dist_partition where the logicalrelid * field equals to distributedRelationId. Then, the function invalidates the * metadata cache. */ void DeletePartitionRow(Oid distributedRelationId) { ScanKeyData scanKey[1]; int scanKeyCount = 1; Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distributedRelationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, InvalidOid, false, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for partition %d", distributedRelationId))); } simple_heap_delete(pgDistPartition, &heapTuple->t_self); systable_endscan(scanDescriptor); /* invalidate the cache */ CitusInvalidateRelcacheByRelid(distributedRelationId); /* increment the counter so that next command can see the row */ CommandCounterIncrement(); table_close(pgDistPartition, NoLock); } /* * DeleteShardRow opens the shard system catalog, finds the unique row that has * the given shardId, and deletes this row. */ void DeleteShardRow(uint64 shardId) { ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgDistShard = table_open(DistShardRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistShard, DistShardShardidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for shard " UINT64_FORMAT, shardId))); } Form_pg_dist_shard pgDistShardForm = (Form_pg_dist_shard) GETSTRUCT(heapTuple); Oid distributedRelationId = pgDistShardForm->logicalrelid; simple_heap_delete(pgDistShard, &heapTuple->t_self); systable_endscan(scanDescriptor); /* invalidate previous cache entry */ CitusInvalidateRelcacheByRelid(distributedRelationId); CommandCounterIncrement(); table_close(pgDistShard, NoLock); } /* * DeleteShardPlacementRowGlobally deletes shard placement that has given * parameters from pg_dist_placement globally. */ void DeleteShardPlacementRowGlobally(uint64 placementId) { DeleteShardPlacementRow(placementId); char *deletePlacementCommand = DeletePlacementMetadataCommand(placementId); SendCommandToWorkersWithMetadata(deletePlacementCommand); } /* * DeleteShardPlacementRow opens the shard placement system catalog, finds the placement * with the given placementId, and deletes it. */ void DeleteShardPlacementRow(uint64 placementId) { const int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = true; bool isNull = false; Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_placementid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(placementId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPlacement, DistPlacementPlacementidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (heapTuple == NULL) { ereport(ERROR, (errmsg("could not find valid entry for shard placement " INT64_FORMAT, placementId))); } uint64 shardId = heap_getattr(heapTuple, Anum_pg_dist_placement_shardid, tupleDescriptor, &isNull); if (HeapTupleHeaderGetNatts(heapTuple->t_data) != Natts_pg_dist_placement || HeapTupleHasNulls(heapTuple)) { ereport(ERROR, (errmsg("unexpected null in pg_dist_placement tuple"))); } simple_heap_delete(pgDistPlacement, &heapTuple->t_self); systable_endscan(scanDescriptor); CitusInvalidateRelcacheByShardId(shardId); CommandCounterIncrement(); table_close(pgDistPlacement, NoLock); } /* * UpdatePlacementGroupId sets the groupId for the placement identified * by placementId. */ void UpdatePlacementGroupId(uint64 placementId, int groupId) { ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; bool colIsNull = false; Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_placementid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(placementId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPlacement, DistPlacementPlacementidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for shard placement " UINT64_FORMAT, placementId))); } values[Anum_pg_dist_placement_groupid - 1] = Int32GetDatum(groupId); isnull[Anum_pg_dist_placement_groupid - 1] = false; replace[Anum_pg_dist_placement_groupid - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistPlacement, &heapTuple->t_self, heapTuple); uint64 shardId = DatumGetInt64(heap_getattr(heapTuple, Anum_pg_dist_placement_shardid, tupleDescriptor, &colIsNull)); Assert(!colIsNull); CitusInvalidateRelcacheByShardId(shardId); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistPlacement, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * UpdatePgDistPartitionAutoConverted sets the autoConverted for the partition identified * by citusTableId. */ void UpdatePgDistPartitionAutoConverted(Oid citusTableId, bool autoConverted) { ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(citusTableId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for citus table with oid: %u", citusTableId))); } int autoconvertedindex = GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor); values[autoconvertedindex] = BoolGetDatum(autoConverted); isnull[autoconvertedindex] = false; replace[autoconvertedindex] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(citusTableId); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistPartition, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * UpdateDistributionColumnGlobally sets the distribution column and colocation ID * for a table in pg_dist_partition on all nodes */ void UpdateDistributionColumnGlobally(Oid relationId, char distributionMethod, Var *distributionColumn, int colocationId) { UpdateDistributionColumn(relationId, distributionMethod, distributionColumn, colocationId); if (ShouldSyncTableMetadata(relationId)) { /* we use delete+insert because syncing uses specialized RPCs */ char *deleteMetadataCommand = DistributionDeleteMetadataCommand(relationId); SendCommandToWorkersWithMetadata(deleteMetadataCommand); /* pick up the new metadata (updated above) */ CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); char *insertMetadataCommand = DistributionCreateCommand(cacheEntry); SendCommandToWorkersWithMetadata(insertMetadataCommand); } } /* * UpdateDistributionColumn sets the distribution column and colocation ID for a table * in pg_dist_partition. */ void UpdateDistributionColumn(Oid relationId, char distributionMethod, Var *distributionColumn, int colocationId) { ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for citus table with oid: %u", relationId))); } replace[Anum_pg_dist_partition_partmethod - 1] = true; values[Anum_pg_dist_partition_partmethod - 1] = CharGetDatum(distributionMethod); isnull[Anum_pg_dist_partition_partmethod - 1] = false; replace[Anum_pg_dist_partition_colocationid - 1] = true; values[Anum_pg_dist_partition_colocationid - 1] = UInt32GetDatum(colocationId); isnull[Anum_pg_dist_partition_colocationid - 1] = false; int autoconvertedindex = GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor); replace[autoconvertedindex] = true; values[autoconvertedindex] = BoolGetDatum(false); isnull[autoconvertedindex] = false; char *distributionColumnString = nodeToString((Node *) distributionColumn); replace[Anum_pg_dist_partition_partkey - 1] = true; values[Anum_pg_dist_partition_partkey - 1] = CStringGetTextDatum(distributionColumnString); isnull[Anum_pg_dist_partition_partkey - 1] = false; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(relationId); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistPartition, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * UpdateNoneDistTableMetadataGlobally globally updates pg_dist_partition for * given none-distributed table. */ void UpdateNoneDistTableMetadataGlobally(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted) { UpdateNoneDistTableMetadata(relationId, replicationModel, colocationId, autoConverted); if (ShouldSyncTableMetadata(relationId)) { char *metadataCommand = UpdateNoneDistTableMetadataCommand(relationId, replicationModel, colocationId, autoConverted); SendCommandToWorkersWithMetadata(metadataCommand); } } /* * UpdateNoneDistTableMetadata locally updates pg_dist_partition for given * none-distributed table. */ void UpdateNoneDistTableMetadata(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted) { if (HasDistributionKey(relationId)) { ereport(ERROR, (errmsg("cannot update metadata for a distributed " "table that has a distribution column"))); } ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for Citus table with oid: %u", relationId))); } values[Anum_pg_dist_partition_colocationid - 1] = UInt32GetDatum(colocationId); isnull[Anum_pg_dist_partition_colocationid - 1] = false; replace[Anum_pg_dist_partition_colocationid - 1] = true; values[Anum_pg_dist_partition_repmodel - 1] = CharGetDatum(replicationModel); isnull[Anum_pg_dist_partition_repmodel - 1] = false; replace[Anum_pg_dist_partition_repmodel - 1] = true; int autoconvertedindex = GetAutoConvertedAttrIndexInPgDistPartition(tupleDescriptor); values[autoconvertedindex] = BoolGetDatum(autoConverted); isnull[autoconvertedindex] = false; replace[autoconvertedindex] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(relationId); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistPartition, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * Check that the current user has `mode` permissions on relationId. * If not, also check relationId's attributes with `mask`, error out * privileges are not defined. * ACL mask is used because we assume that user has enough privilege * to distribute a table when either ACL_INSERT on the TABLE or * ACL_INSERT on ALL attributes. * In other situations, having a single attribute privilege is enough. * Superusers always have such permissions. */ void EnsureTablePermissions(Oid relationId, AclMode mode, AclMaskHow mask) { AclResult aclresult = pg_class_aclcheck(relationId, GetUserId(), mode); if (aclresult == ACLCHECK_OK) { return; } /* * Also check the attributes: for example "GRANT ALL(a)" has no table level * right but user is still allowed to lock table as needed. PostgreSQL will * still enforce ACL later so it's safe. */ aclresult = pg_attribute_aclcheck_all(relationId, GetUserId(), mode, mask); if (aclresult != ACLCHECK_OK) { aclcheck_error(aclresult, OBJECT_TABLE, get_rel_name(relationId)); } } /* * Check that the current user has owner rights to relationId, error out if * not. Superusers are regarded as owners. */ void EnsureTableOwner(Oid relationId) { if (!object_ownercheck(RelationRelationId, relationId, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_TABLE, get_rel_name(relationId)); } } /* * Check that the current user has owner rights to schemaId, error out if * not. Superusers are regarded as owners. */ void EnsureSchemaOwner(Oid schemaId) { if (!object_ownercheck(NamespaceRelationId, schemaId, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_SCHEMA, get_namespace_name(schemaId)); } } /* * Check that the current user has owner rights to functionId, error out if * not. Superusers are regarded as owners. Functions and procedures are * treated equally. */ void EnsureFunctionOwner(Oid functionId) { if (!object_ownercheck(ProcedureRelationId, functionId, GetUserId())) { aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FUNCTION, get_func_name(functionId)); } } /* * EnsureHashDistributedTable error out if the given relation is not a hash distributed table * with the given message. */ void EnsureHashDistributedTable(Oid relationId) { if (!IsCitusTableType(relationId, HASH_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s should be a " "hash distributed table", get_rel_name(relationId)))); } } /* * EnsureHashOrSingleShardDistributedTable error out if the given relation is not a * hash or single shard distributed table with the given message. */ void EnsureHashOrSingleShardDistributedTable(Oid relationId) { if (!IsCitusTableType(relationId, HASH_DISTRIBUTED) && !IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relation %s should be a " "hash or single shard distributed table", get_rel_name(relationId)))); } } /* * EnsureSuperUser check that the current user is a superuser and errors out if not. */ void EnsureSuperUser(void) { if (!superuser()) { ereport(ERROR, (errmsg("operation is not allowed"), errhint("Run the command with a superuser."))); } } Oid TableOwnerOid(Oid relationId) { HeapTuple tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relationId)); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation with OID %u does not exist", relationId))); } Oid userId = ((Form_pg_class) GETSTRUCT(tuple))->relowner; ReleaseSysCache(tuple); return userId; } /* * Return a table's owner as a string. */ char * TableOwner(Oid relationId) { return GetUserNameFromId(TableOwnerOid(relationId), false); } /* * IsForeignTable takes a relation id and returns true if it's a foreign table. * Returns false otherwise. */ bool IsForeignTable(Oid relationId) { return get_rel_relkind(relationId) == RELKIND_FOREIGN_TABLE; } /* * HasRunnableBackgroundTask looks in the catalog if there are any tasks that can be run. * For a task to be able to run the following conditions apply: * - Task is in Running state. This could happen when a Background Tasks Queue Monitor * had crashed or is otherwise restarted. To recover from such a failure tasks in * Running state are deemed Runnable. * - Task is in Runnable state with either _no_ value set in not_before, or a value that * has currently passed. If the not_before field is set to a time in the future the * task is currently not ready to be started. */ bool HasRunnableBackgroundTask() { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), AccessShareLock); /* find any job in states listed here */ BackgroundTaskStatus taskStatus[] = { BACKGROUND_TASK_STATUS_RUNNABLE, BACKGROUND_TASK_STATUS_RUNNING }; bool hasScheduledTask = false; for (int i = 0; !hasScheduledTask && i < lengthof(taskStatus); i++) { const int scanKeyCount = 1; ScanKeyData scanKey[1] = { 0 }; const bool indexOK = true; /* pg_dist_background_task.status == taskStatus[i] */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_status, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(BackgroundTaskStatusOid(taskStatus[i]))); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskStatusTaskIdIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple taskTuple = NULL; while (HeapTupleIsValid(taskTuple = systable_getnext(scanDescriptor))) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); BackgroundTask *task = DeformBackgroundTaskHeapTuple(tupleDescriptor, taskTuple); if (task->not_before && *(task->not_before) > GetCurrentTimestamp()) { continue; } hasScheduledTask = true; break; } systable_endscan(scanDescriptor); } table_close(pgDistBackgroundTasks, NoLock); return hasScheduledTask; } /* * BackgroundJobStatusByOid returns the C enum representation of a BackgroundJobsStatus * based on the Oid of the SQL enum value. */ BackgroundJobStatus BackgroundJobStatusByOid(Oid enumOid) { if (enumOid == CitusJobStatusScheduledId()) { return BACKGROUND_JOB_STATUS_SCHEDULED; } else if (enumOid == CitusJobStatusRunningId()) { return BACKGROUND_JOB_STATUS_RUNNING; } else if (enumOid == CitusJobStatusFinishedId()) { return BACKGROUND_JOB_STATUS_FINISHED; } else if (enumOid == CitusJobStatusCancelledId()) { return BACKGROUND_JOB_STATUS_CANCELLED; } else if (enumOid == CitusJobStatusFailingId()) { return BACKGROUND_JOB_STATUS_FAILING; } else if (enumOid == CitusJobStatusFailedId()) { return BACKGROUND_JOB_STATUS_FAILED; } else if (enumOid == CitusJobStatusCancellingId()) { return BACKGROUND_JOB_STATUS_CANCELLING; } elog(ERROR, "unknown enum value for citus_job_status"); } /* * BackgroundTaskStatusByOid returns the C enum representation of a BackgroundTaskStatus * based on the Oid of the SQL enum value. */ BackgroundTaskStatus BackgroundTaskStatusByOid(Oid enumOid) { if (enumOid == CitusTaskStatusDoneId()) { return BACKGROUND_TASK_STATUS_DONE; } else if (enumOid == CitusTaskStatusRunnableId()) { return BACKGROUND_TASK_STATUS_RUNNABLE; } else if (enumOid == CitusTaskStatusRunningId()) { return BACKGROUND_TASK_STATUS_RUNNING; } else if (enumOid == CitusTaskStatusErrorId()) { return BACKGROUND_TASK_STATUS_ERROR; } else if (enumOid == CitusTaskStatusUnscheduledId()) { return BACKGROUND_TASK_STATUS_UNSCHEDULED; } else if (enumOid == CitusTaskStatusBlockedId()) { return BACKGROUND_TASK_STATUS_BLOCKED; } else if (enumOid == CitusTaskStatusCancelledId()) { return BACKGROUND_TASK_STATUS_CANCELLED; } else if (enumOid == CitusTaskStatusCancellingId()) { return BACKGROUND_TASK_STATUS_CANCELLING; } ereport(ERROR, (errmsg("unknown enum value for citus_task_status"))); } /* * IsBackgroundJobStatusTerminal is a predicate returning if the BackgroundJobStatus * passed is a terminal state of the Background Job state machine. * * For a Job to be in it's terminal state, all tasks from that job should also be in their * terminal state. */ bool IsBackgroundJobStatusTerminal(BackgroundJobStatus status) { switch (status) { case BACKGROUND_JOB_STATUS_CANCELLED: case BACKGROUND_JOB_STATUS_FAILED: case BACKGROUND_JOB_STATUS_FINISHED: { return true; } case BACKGROUND_JOB_STATUS_CANCELLING: case BACKGROUND_JOB_STATUS_FAILING: case BACKGROUND_JOB_STATUS_RUNNING: case BACKGROUND_JOB_STATUS_SCHEDULED: { return false; } /* no default to make sure we explicitly add every state here */ } elog(ERROR, "unknown BackgroundJobStatus"); } /* * IsBackgroundTaskStatusTerminal is a predicate returning if the BackgroundTaskStatus * passed is a terminal state of the Background Task state machine. */ bool IsBackgroundTaskStatusTerminal(BackgroundTaskStatus status) { switch (status) { case BACKGROUND_TASK_STATUS_CANCELLED: case BACKGROUND_TASK_STATUS_DONE: case BACKGROUND_TASK_STATUS_ERROR: case BACKGROUND_TASK_STATUS_UNSCHEDULED: { return true; } case BACKGROUND_TASK_STATUS_BLOCKED: case BACKGROUND_TASK_STATUS_CANCELLING: case BACKGROUND_TASK_STATUS_RUNNABLE: case BACKGROUND_TASK_STATUS_RUNNING: { return false; } /* no default to make sure we explicitly add every state here */ } elog(ERROR, "unknown BackgroundTaskStatus"); } /* * BackgroundJobStatusOid returns the Oid corresponding to SQL enum value corresponding to * the BackgroundJobStatus. */ Oid BackgroundJobStatusOid(BackgroundJobStatus status) { switch (status) { case BACKGROUND_JOB_STATUS_SCHEDULED: { return CitusJobStatusScheduledId(); } case BACKGROUND_JOB_STATUS_RUNNING: { return CitusJobStatusRunningId(); } case BACKGROUND_JOB_STATUS_CANCELLING: { return CitusJobStatusCancellingId(); } case BACKGROUND_JOB_STATUS_FINISHED: { return CitusJobStatusFinishedId(); } case BACKGROUND_JOB_STATUS_CANCELLED: { return CitusJobStatusCancelledId(); } case BACKGROUND_JOB_STATUS_FAILING: { return CitusJobStatusFailingId(); } case BACKGROUND_JOB_STATUS_FAILED: { return CitusJobStatusFailedId(); } } elog(ERROR, "unknown BackgroundJobStatus"); } /* * BackgroundTaskStatusOid returns the Oid corresponding to SQL enum value corresponding to * the BackgroundTaskStatus. */ Oid BackgroundTaskStatusOid(BackgroundTaskStatus status) { switch (status) { case BACKGROUND_TASK_STATUS_BLOCKED: { return CitusTaskStatusBlockedId(); } case BACKGROUND_TASK_STATUS_RUNNABLE: { return CitusTaskStatusRunnableId(); } case BACKGROUND_TASK_STATUS_RUNNING: { return CitusTaskStatusRunningId(); } case BACKGROUND_TASK_STATUS_DONE: { return CitusTaskStatusDoneId(); } case BACKGROUND_TASK_STATUS_ERROR: { return CitusTaskStatusErrorId(); } case BACKGROUND_TASK_STATUS_UNSCHEDULED: { return CitusTaskStatusUnscheduledId(); } case BACKGROUND_TASK_STATUS_CANCELLED: { return CitusTaskStatusCancelledId(); } case BACKGROUND_TASK_STATUS_CANCELLING: { return CitusTaskStatusCancellingId(); } } elog(ERROR, "unknown BackgroundTaskStatus"); return InvalidOid; } /* * GetNextBackgroundJobsJobId reads and increments the SQL sequence associated with the * background job's job_id. After incrementing the counter it returns the counter back to * the caller. * * The return value is typically used to insert new jobs into the catalog. */ static int64 GetNextBackgroundJobsJobId(void) { return DatumGetInt64(nextval_internal(DistBackgroundJobJobIdSequenceId(), false)); } /* * GetNextBackgroundTaskTaskId reads and increments the SQL sequence associated with the * background job tasks' task_id. After incrementing the counter it returns the counter * back to the caller. * * The return value is typically used to insert new tasks into the catalog. */ static int64 GetNextBackgroundTaskTaskId(void) { return DatumGetInt64(nextval_internal(DistBackgroundTaskTaskIdSequenceId(), false)); } /* * HasNonTerminalJobOfType returns true if there is a job of a given type that is not in * its terminal state. * * Some jobs would want a single instance to be able to run at once. Before submitting a * new job if could see if there is a job of their type already executing. * * If a job is found the options jobIdOut is populated with the jobId. */ bool HasNonTerminalJobOfType(const char *jobType, int64 *jobIdOut) { Relation pgDistBackgroundJob = table_open(DistBackgroundJobRelationId(), AccessShareLock); /* find any job in states listed here */ BackgroundJobStatus jobStatus[] = { BACKGROUND_JOB_STATUS_RUNNING, BACKGROUND_JOB_STATUS_CANCELLING, BACKGROUND_JOB_STATUS_FAILING, BACKGROUND_JOB_STATUS_SCHEDULED }; NameData jobTypeName = { 0 }; namestrcpy(&jobTypeName, jobType); bool foundJob = false; for (int i = 0; !foundJob && i < lengthof(jobStatus); i++) { ScanKeyData scanKey[2] = { 0 }; const bool indexOK = true; /* pg_dist_background_job.status == jobStatus[i] */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_job_state, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(BackgroundJobStatusOid(jobStatus[i]))); /* pg_dist_background_job.job_type == jobType */ ScanKeyInit(&scanKey[1], Anum_pg_dist_background_job_job_type, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&jobTypeName)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundJob, InvalidOid, /* TODO use an actual index here */ indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple taskTuple = NULL; if (HeapTupleIsValid(taskTuple = systable_getnext(scanDescriptor))) { foundJob = true; if (jobIdOut) { Datum values[Natts_pg_dist_background_job] = { 0 }; bool isnull[Natts_pg_dist_background_job] = { 0 }; TupleDesc tupleDesc = RelationGetDescr(pgDistBackgroundJob); heap_deform_tuple(taskTuple, tupleDesc, values, isnull); *jobIdOut = DatumGetInt64(values[Anum_pg_dist_background_job_job_id - 1]); } } systable_endscan(scanDescriptor); } table_close(pgDistBackgroundJob, NoLock); return foundJob; } /* * CreateBackgroundJob is a helper function to insert a new Background Job into Citus' * catalog. After inserting the new job's metadataa into the catalog it returns the job_id * assigned to the new job. This is typically used to associate new tasks with the newly * created job. */ int64 CreateBackgroundJob(const char *jobType, const char *description) { Relation pgDistBackgroundJobs = table_open(DistBackgroundJobRelationId(), RowExclusiveLock); /* insert new job */ Datum values[Natts_pg_dist_background_job] = { 0 }; bool isnull[Natts_pg_dist_background_job] = { 0 }; NameData jobTypeName = { 0 }; memset(isnull, true, sizeof(isnull)); int64 jobId = GetNextBackgroundJobsJobId(); InitFieldValue(Anum_pg_dist_background_job_job_id, values, isnull, Int64GetDatum(jobId)); InitFieldValue(Anum_pg_dist_background_job_state, values, isnull, ObjectIdGetDatum(CitusJobStatusScheduledId())); if (jobType) { namestrcpy(&jobTypeName, jobType); InitFieldValue(Anum_pg_dist_background_job_job_type, values, isnull, NameGetDatum(&jobTypeName)); } if (description) { InitFieldText(Anum_pg_dist_background_job_description, values, isnull, description); } HeapTuple newTuple = heap_form_tuple(RelationGetDescr(pgDistBackgroundJobs), values, isnull); CatalogTupleInsert(pgDistBackgroundJobs, newTuple); CommandCounterIncrement(); table_close(pgDistBackgroundJobs, NoLock); return jobId; } /* * ScheduleBackgroundTask creates a new background task to be executed in the background. * * The new task is associated with an existing job based on it's id. * * Optionally the new task can depend on separate tasks associated with the same job. When * a new task is created with dependencies on previous tasks we assume this task is * blocked on its depending tasks. */ BackgroundTask * ScheduleBackgroundTask(int64 jobId, Oid owner, char *command, int dependingTaskCount, int64 dependingTaskIds[], int nodesInvolvedCount, int32 nodesInvolved[]) { BackgroundTask *task = NULL; Relation pgDistBackgroundJob = table_open(DistBackgroundJobRelationId(), ExclusiveLock); Relation pgDistBackgroundTask = table_open(DistBackgroundTaskRelationId(), ExclusiveLock); Relation pgDistbackgroundTasksDepend = NULL; if (dependingTaskCount > 0) { pgDistbackgroundTasksDepend = table_open(DistBackgroundTaskDependRelationId(), ExclusiveLock); } /* 1. verify job exist */ { ScanKeyData scanKey[1] = { 0 }; bool indexOK = true; /* pg_dist_background_job.job_id == $jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_job_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundJob, DistBackgroundJobPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple jobTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(jobTuple)) { ereport(ERROR, (errmsg("job for newly created task does not exist."))); } systable_endscan(scanDescriptor); } /* 2. insert new task */ { TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTask); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *nulls = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); int64 taskId = GetNextBackgroundTaskTaskId(); values[Anum_pg_dist_background_task_job_id - 1] = Int64GetDatum(jobId); nulls[Anum_pg_dist_background_task_job_id - 1] = false; values[Anum_pg_dist_background_task_task_id - 1] = Int64GetDatum(taskId); nulls[Anum_pg_dist_background_task_task_id - 1] = false; values[Anum_pg_dist_background_task_owner - 1] = ObjectIdGetDatum(owner); nulls[Anum_pg_dist_background_task_owner - 1] = false; Oid statusOid = InvalidOid; if (dependingTaskCount == 0) { statusOid = CitusTaskStatusRunnableId(); } else { statusOid = CitusTaskStatusBlockedId(); } values[Anum_pg_dist_background_task_status - 1] = ObjectIdGetDatum(statusOid); nulls[Anum_pg_dist_background_task_status - 1] = false; values[Anum_pg_dist_background_task_command - 1] = CStringGetTextDatum(command); nulls[Anum_pg_dist_background_task_command - 1] = false; values[Anum_pg_dist_background_task_message - 1] = CStringGetTextDatum(""); nulls[Anum_pg_dist_background_task_message - 1] = false; int nodesInvolvedIndex = GetNodesInvolvedAttrIndexInPgDistBackgroundTask(tupleDescriptor); values[nodesInvolvedIndex] = IntArrayToDatum(nodesInvolvedCount, nodesInvolved); nulls[nodesInvolvedIndex] = (nodesInvolvedCount == 0); HeapTuple newTuple = heap_form_tuple(tupleDescriptor, values, nulls); CatalogTupleInsert(pgDistBackgroundTask, newTuple); pfree(values); pfree(nulls); task = palloc0(sizeof(BackgroundTask)); task->taskid = taskId; task->status = BACKGROUND_TASK_STATUS_RUNNABLE; task->command = pstrdup(command); } /* 3. insert dependencies into catalog */ { for (int i = 0; i < dependingTaskCount; i++) { /* 3.1 after verifying the task exists for this job */ { ScanKeyData scanKey[2] = { 0 }; bool indexOK = true; /* pg_dist_background_task.job_id == $jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); /* pg_dist_background_task.task_id == $taskId */ ScanKeyInit(&scanKey[1], Anum_pg_dist_background_task_task_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(dependingTaskIds[i])); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTask, DistBackgroundTaskJobIdTaskIdIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple taskTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(taskTuple)) { ereport(ERROR, (errmsg("depending task for newly scheduled task does " "not exist"))); } systable_endscan(scanDescriptor); } Assert(pgDistbackgroundTasksDepend != NULL); Datum values[Natts_pg_dist_background_task_depend] = { 0 }; bool nulls[Natts_pg_dist_background_task_depend] = { 0 }; memset(nulls, true, sizeof(nulls)); values[Anum_pg_dist_background_task_depend_job_id - 1] = Int64GetDatum(jobId); nulls[Anum_pg_dist_background_task_depend_job_id - 1] = false; values[Anum_pg_dist_background_task_depend_task_id - 1] = Int64GetDatum(task->taskid); nulls[Anum_pg_dist_background_task_depend_task_id - 1] = false; values[Anum_pg_dist_background_task_depend_depends_on - 1] = Int64GetDatum(dependingTaskIds[i]); nulls[Anum_pg_dist_background_task_depend_depends_on - 1] = false; HeapTuple newTuple = heap_form_tuple( RelationGetDescr(pgDistbackgroundTasksDepend), values, nulls); CatalogTupleInsert(pgDistbackgroundTasksDepend, newTuple); } } if (pgDistbackgroundTasksDepend) { table_close(pgDistbackgroundTasksDepend, NoLock); } table_close(pgDistBackgroundTask, NoLock); table_close(pgDistBackgroundJob, NoLock); CommandCounterIncrement(); return task; } /* * ResetRunningBackgroundTasks finds all tasks currently in Running state and resets their * state back to runnable. * * While marking running tasks as runnable we check if the task might still be locked and * the pid is managed by our current postmaster. If both are the case we terminate the * backend. This will make sure that if a task was still running after a monitor crash or * restart we stop the executor before we start a new one. * * Any pid associated with the running tasks will be cleared back to the NULL value. */ void ResetRunningBackgroundTasks(void) { const int scanKeyCount = 1; ScanKeyData scanKey[1]; const bool indexOK = true; Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), ExclusiveLock); /* pg_dist_background_task.status == 'running' */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_status, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(CitusTaskStatusRunningId())); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskStatusTaskIdIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple taskTuple = NULL; List *taskIdsToWait = NIL; while (HeapTupleIsValid(taskTuple = systable_getnext(scanDescriptor))) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(taskTuple, tupleDescriptor, values, isnull); values[Anum_pg_dist_background_task_status - 1] = ObjectIdGetDatum(CitusTaskStatusRunnableId()); isnull[Anum_pg_dist_background_task_status - 1] = false; replace[Anum_pg_dist_background_task_status - 1] = true; /* if there is a pid we need to signal the backend to stop */ if (!isnull[Anum_pg_dist_background_task_pid - 1]) { /* * Before signalling the pid we check if the task lock is held, otherwise we * might cancel an arbitrary postgres backend */ int64 taskId = DatumGetInt64(values[Anum_pg_dist_background_task_task_id - 1]); /* No need to release lock, will get unlocked once our changes commit */ LOCKTAG locktag = { 0 }; SET_LOCKTAG_BACKGROUND_TASK(locktag, taskId); const bool sessionLock = false; const bool dontWait = true; LockAcquireResult locked = LockAcquire(&locktag, AccessExclusiveLock, sessionLock, dontWait); if (locked == LOCKACQUIRE_NOT_AVAIL) { /* * There is still an executor holding the lock, needs a SIGTERM. */ Datum pidDatum = values[Anum_pg_dist_background_task_pid - 1]; const Datum timeoutDatum = Int64GetDatum(0); Datum signalSuccessDatum = DirectFunctionCall2(pg_terminate_backend, pidDatum, timeoutDatum); bool signalSuccess = DatumGetBool(signalSuccessDatum); if (!signalSuccess) { /* * We run this backend as superuser, any failure will probably cause * long delays waiting on the task lock before we can commit. */ ereport(WARNING, (errmsg("could not send signal to process %d: %m", DatumGetInt32(pidDatum)), errdetail("failing to signal an old executor could cause " "delays starting the background task monitor"))); } /* * Since we didn't already acquire the lock here we need to wait on this * lock before committing the change to the catalog. However, we first * want to signal all backends before waiting on the lock, hence we keep a * list for later */ int64 *taskIdTarget = palloc0(sizeof(int64)); *taskIdTarget = taskId; taskIdsToWait = lappend(taskIdsToWait, taskIdTarget); } } values[Anum_pg_dist_background_task_pid - 1] = InvalidOid; isnull[Anum_pg_dist_background_task_pid - 1] = true; replace[Anum_pg_dist_background_task_pid - 1] = true; taskTuple = heap_modify_tuple(taskTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistBackgroundTasks, &taskTuple->t_self, taskTuple); pfree(values); pfree(isnull); pfree(replace); } if (list_length(taskIdsToWait) > 0) { ereport(LOG, (errmsg("waiting till all tasks release their lock before " "continuing with the background task monitor"))); /* there are tasks that need to release their lock before we can continue */ int64 *taskId = NULL; foreach_declared_ptr(taskId, taskIdsToWait) { LOCKTAG locktag = { 0 }; SET_LOCKTAG_BACKGROUND_TASK(locktag, *taskId); const bool sessionLock = false; const bool dontWait = false; (void) LockAcquire(&locktag, AccessExclusiveLock, sessionLock, dontWait); } } CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasks, NoLock); } /* * DeformBackgroundJobHeapTuple pareses a HeapTuple from pg_dist_background_job into its * inmemory representation. This can be used while scanning a heap to quickly get access * to all fields of a Job. */ static BackgroundJob * DeformBackgroundJobHeapTuple(TupleDesc tupleDescriptor, HeapTuple jobTuple) { Datum values[Natts_pg_dist_background_job] = { 0 }; bool nulls[Natts_pg_dist_background_job] = { 0 }; heap_deform_tuple(jobTuple, tupleDescriptor, values, nulls); BackgroundJob *job = palloc0(sizeof(BackgroundJob)); job->jobid = DatumGetInt64(values[Anum_pg_dist_background_job_job_id - 1]); job->state = BackgroundJobStatusByOid( DatumGetObjectId(values[Anum_pg_dist_background_job_state - 1])); if (!nulls[Anum_pg_dist_background_job_job_type - 1]) { Name jobTypeName = DatumGetName(values[Anum_pg_dist_background_job_job_type - 1]); job->jobType = pstrdup(NameStr(*jobTypeName)); } if (!nulls[Anum_pg_dist_background_job_description - 1]) { job->description = text_to_cstring( DatumGetTextP(values[Anum_pg_dist_background_job_description - 1])); } if (!nulls[Anum_pg_dist_background_job_started_at - 1]) { TimestampTz startedAt = DatumGetTimestampTz(values[Anum_pg_dist_background_job_started_at - 1]); SET_NULLABLE_FIELD(job, started_at, startedAt); } if (!nulls[Anum_pg_dist_background_job_finished_at - 1]) { TimestampTz finishedAt = DatumGetTimestampTz(values[Anum_pg_dist_background_job_finished_at - 1]); SET_NULLABLE_FIELD(job, finished_at, finishedAt); } return job; } /* * DeformBackgroundTaskHeapTuple parses a HeapTuple from pg_dist_background_task into its * inmemory representation. This can be used while scanning a heap to quickly get access * to all fields of a Task. */ static BackgroundTask * DeformBackgroundTaskHeapTuple(TupleDesc tupleDescriptor, HeapTuple taskTuple) { Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *nulls = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(taskTuple, tupleDescriptor, values, nulls); BackgroundTask *task = palloc0(sizeof(BackgroundTask)); task->jobid = DatumGetInt64(values[Anum_pg_dist_background_task_job_id - 1]); task->taskid = DatumGetInt64(values[Anum_pg_dist_background_task_task_id - 1]); task->owner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner - 1]); if (!nulls[Anum_pg_dist_background_task_pid - 1]) { int32 pid = DatumGetInt32(values[Anum_pg_dist_background_task_pid - 1]); SET_NULLABLE_FIELD(task, pid, pid); } task->status = BackgroundTaskStatusByOid( DatumGetObjectId(values[Anum_pg_dist_background_task_status - 1])); task->command = text_to_cstring( DatumGetTextP(values[Anum_pg_dist_background_task_command - 1])); if (!nulls[Anum_pg_dist_background_task_retry_count - 1]) { int32 retryCount = DatumGetInt32(values[Anum_pg_dist_background_task_retry_count - 1]); SET_NULLABLE_FIELD(task, retry_count, retryCount); } if (!nulls[Anum_pg_dist_background_task_not_before - 1]) { TimestampTz notBefore = DatumGetTimestampTz(values[Anum_pg_dist_background_task_not_before - 1]); SET_NULLABLE_FIELD(task, not_before, notBefore); } if (!nulls[Anum_pg_dist_background_task_message - 1]) { task->message = TextDatumGetCString(values[Anum_pg_dist_background_task_message - 1]); } int nodesInvolvedIndex = GetNodesInvolvedAttrIndexInPgDistBackgroundTask(tupleDescriptor); if (!nulls[nodesInvolvedIndex]) { ArrayType *nodesInvolvedArrayObject = DatumGetArrayTypeP(values[nodesInvolvedIndex]); task->nodesInvolved = IntegerArrayTypeToList(nodesInvolvedArrayObject); } pfree(values); pfree(nulls); return task; } /* * BackgroundTaskHasUmnetDependencies checks if a task from the given job has any unmet * dependencies. An unmet dependency is a Task that the task in question depends on and * has not reached its Done state. */ static bool BackgroundTaskHasUmnetDependencies(int64 jobId, int64 taskId) { bool hasUnmetDependency = false; Relation pgDistBackgroundTasksDepend = table_open(DistBackgroundTaskDependRelationId(), AccessShareLock); ScanKeyData scanKey[2] = { 0 }; bool indexOK = true; /* pg_catalog.pg_dist_background_task_depend.job_id = jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_depend_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); /* pg_catalog.pg_dist_background_task_depend.task_id = $taskId */ ScanKeyInit(&scanKey[1], Anum_pg_dist_background_task_depend_task_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(taskId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasksDepend, DistBackgroundTaskDependTaskIdIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple dependTuple = NULL; while (HeapTupleIsValid(dependTuple = systable_getnext(scanDescriptor))) { Form_pg_dist_background_task_depend depends = (Form_pg_dist_background_task_depend) GETSTRUCT(dependTuple); BackgroundTask *dependingJob = GetBackgroundTaskByTaskId(depends->depends_on); /* * Only when the status of all depending jobs is done we clear this job and say * that is has no unmet dependencies. */ if (dependingJob->status == BACKGROUND_TASK_STATUS_DONE) { continue; } hasUnmetDependency = true; break; } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasksDepend, AccessShareLock); return hasUnmetDependency; } /* * BackgroundTaskReadyToRun checks if a task is ready to run. This consists of two checks * - the task has no unmet dependencies * - the task either has no not_before value set, or the not_before time has passed. * * Due to the costs of checking we check them in reverse order, but conceptually they * should be thought of in the above order. */ static bool BackgroundTaskReadyToRun(BackgroundTask *task) { if (task->not_before) { if (*(task->not_before) > GetCurrentTimestamp()) { /* task should not yet be run */ return false; } } if (BackgroundTaskHasUmnetDependencies(task->jobid, task->taskid)) { return false; } return true; } /* * GetRunnableBackgroundTask returns the first candidate for a task to be run. When a task * is returned it has been checked for all the preconditions to hold. * * That means, if there is no task returned the background worker should close and let the * maintenance daemon start a new background tasks queue monitor once task become * available. */ BackgroundTask * GetRunnableBackgroundTask(void) { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), ExclusiveLock); BackgroundTaskStatus taskStatus[] = { BACKGROUND_TASK_STATUS_RUNNABLE }; BackgroundTask *task = NULL; for (int i = 0; !task && i < sizeof(taskStatus) / sizeof(taskStatus[0]); i++) { const int scanKeyCount = 1; ScanKeyData scanKey[1] = { 0 }; const bool indexOK = true; /* pg_dist_background_task.status == taskStatus[i] */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_status, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum( BackgroundTaskStatusOid(taskStatus[i]))); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskStatusTaskIdIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple taskTuple = NULL; TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); while (HeapTupleIsValid(taskTuple = systable_getnext(scanDescriptor))) { task = DeformBackgroundTaskHeapTuple(tupleDescriptor, taskTuple); if (BackgroundTaskReadyToRun(task) && IncrementParallelTaskCountForNodesInvolved(task)) { /* found task, close table and return */ break; } task = NULL; } systable_endscan(scanDescriptor); } table_close(pgDistBackgroundTasks, NoLock); return task; } /* * GetBackgroundJobByJobId loads a BackgroundJob from the catalog into memory. Return's a * null pointer if no job exist with the given JobId. */ BackgroundJob * GetBackgroundJobByJobId(int64 jobId) { ScanKeyData scanKey[1] = { 0 }; bool indexOK = true; Relation pgDistBackgroundJobs = table_open(DistBackgroundJobRelationId(), AccessShareLock); /* pg_dist_background_task.job_id == $jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_job_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundJobs, DistBackgroundJobPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple taskTuple = systable_getnext(scanDescriptor); BackgroundJob *job = NULL; if (HeapTupleIsValid(taskTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundJobs); job = DeformBackgroundJobHeapTuple(tupleDescriptor, taskTuple); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundJobs, AccessShareLock); return job; } /* * GetBackgroundTaskByTaskId loads a BackgroundTask from the catalog into memory. Return's * a null pointer if no job exist with the given JobId and TaskId. */ BackgroundTask * GetBackgroundTaskByTaskId(int64 taskId) { ScanKeyData scanKey[1] = { 0 }; bool indexOK = true; Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), AccessShareLock); /* pg_dist_background_task.task_id == $taskId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_task_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(taskId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple taskTuple = systable_getnext(scanDescriptor); BackgroundTask *task = NULL; if (HeapTupleIsValid(taskTuple)) { TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); task = DeformBackgroundTaskHeapTuple(tupleDescriptor, taskTuple); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasks, AccessShareLock); return task; } typedef struct JobTaskStatusCounts { int blocked; int runnable; int running; int done; int error; int unscheduled; int cancelled; int cancelling; } JobTaskStatusCounts; /* * JobTasksStatusCount scans all tasks associated with the provided job and count's the * number of tasks that are tracked in each state. Effectively grouping and counting the * tasks by their state. */ static JobTaskStatusCounts JobTasksStatusCount(int64 jobId) { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); ScanKeyData scanKey[1] = { 0 }; const bool indexOK = true; /* WHERE job_id = $task->jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskJobIdTaskIdIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); JobTaskStatusCounts counts = { 0 }; HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, values, isnull); Oid statusOid = DatumGetObjectId(values[Anum_pg_dist_background_task_status - 1]); BackgroundTaskStatus status = BackgroundTaskStatusByOid(statusOid); pfree(values); pfree(isnull); switch (status) { case BACKGROUND_TASK_STATUS_BLOCKED: { counts.blocked++; break; } case BACKGROUND_TASK_STATUS_RUNNABLE: { counts.runnable++; break; } case BACKGROUND_TASK_STATUS_RUNNING: { counts.running++; break; } case BACKGROUND_TASK_STATUS_DONE: { counts.done++; break; } case BACKGROUND_TASK_STATUS_ERROR: { counts.error++; break; } case BACKGROUND_TASK_STATUS_UNSCHEDULED: { counts.unscheduled++; break; } case BACKGROUND_TASK_STATUS_CANCELLED: { counts.cancelled++; break; } case BACKGROUND_TASK_STATUS_CANCELLING: { counts.cancelling++; break; } default: { elog(ERROR, "unknown state in pg_dist_background_task"); } } } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasks, NoLock); return counts; } /* * SetFieldValue populates values, isnull, replace according to the newValue passed, * returning if the value has been updated or not. The replace argument can be omitted if * we are simply initializing a field. * * suggested use would be: * bool updated = false; * updated |= SetFieldValue(Anum_...._...., isnull, replace, values, newValue); * updated |= SetFieldText(Anum_...._...., isnull, replace, values, "hello world"); * updated |= SetFieldNull(Anum_...._...., isnull, replace, values); * * Only if updated is set in the end the tuple has to be updated in the catalog. */ static bool SetFieldValue(int attno, Datum values[], bool isnull[], bool replace[], Datum newValue) { int idx = attno - 1; bool updated = false; if (!isnull[idx] && newValue == values[idx]) { return updated; } values[idx] = newValue; isnull[idx] = false; updated = true; if (replace) { replace[idx] = true; } return updated; } /* * SetFieldText populates values, isnull, replace according to the newValue passed, * returning if the value has been updated or not. The replace argument can be omitted if * we are simply initializing a field. * * suggested use would be: * bool updated = false; * updated |= SetFieldValue(Anum_...._...., isnull, replace, values, newValue); * updated |= SetFieldText(Anum_...._...., isnull, replace, values, "hello world"); * updated |= SetFieldNull(Anum_...._...., isnull, replace, values); * * Only if updated is set in the end the tuple has to be updated in the catalog. */ static bool SetFieldText(int attno, Datum values[], bool isnull[], bool replace[], const char *newValue) { int idx = attno - 1; bool updated = false; if (!isnull[idx]) { char *oldText = TextDatumGetCString(values[idx]); if (strcmp(oldText, newValue) == 0) { return updated; } } values[idx] = CStringGetTextDatum(newValue); isnull[idx] = false; updated = true; if (replace) { replace[idx] = true; } return updated; } /* * SetFieldNull populates values, isnull and replace according to a null value, * returning if the value has been updated or not. The replace argument can be omitted if * we are simply initializing a field. * * suggested use would be: * bool updated = false; * updated |= SetFieldValue(Anum_...._...., isnull, replace, values, newValue); * updated |= SetFieldText(Anum_...._...., isnull, replace, values, "hello world"); * updated |= SetFieldNull(Anum_...._...., isnull, replace, values); * * Only if updated is set in the end the tuple has to be updated in the catalog. */ static bool SetFieldNull(int attno, Datum values[], bool isnull[], bool replace[]) { int idx = attno - 1; bool updated = false; if (isnull[idx]) { return updated; } isnull[idx] = true; values[idx] = InvalidOid; updated = true; if (replace) { replace[idx] = true; } return updated; } /* * UpdateBackgroundJob updates the job's metadata based on the most recent status of all * its associated tasks. * * Since the state of a job is a function of the state of all associated tasks this * function projects the tasks states into the job's state. * * When Citus makes a change to any of the tasks associated with the job it should call * this function to correctly project the task updates onto the jobs metadata. */ void UpdateBackgroundJob(int64 jobId) { JobTaskStatusCounts counts = JobTasksStatusCount(jobId); BackgroundJobStatus status = BACKGROUND_JOB_STATUS_RUNNING; if (counts.cancelling > 0) { status = BACKGROUND_JOB_STATUS_CANCELLING; } else if (counts.cancelled > 0) { status = BACKGROUND_JOB_STATUS_CANCELLED; } else if (counts.blocked + counts.runnable + counts.running + counts.error + counts.unscheduled == 0) { /* all tasks are done, job is finished */ status = BACKGROUND_JOB_STATUS_FINISHED; } else if (counts.error + counts.unscheduled > 0) { /* we are either failing, or failed */ if (counts.blocked + counts.runnable + counts.running > 0) { /* failing, as there are still tasks to be run */ status = BACKGROUND_JOB_STATUS_FAILING; } else { status = BACKGROUND_JOB_STATUS_FAILED; } } else if (counts.blocked + counts.runnable + counts.running > 0) { status = BACKGROUND_JOB_STATUS_RUNNING; } else { return; } Relation pgDistBackgroundJobs = table_open(DistBackgroundJobRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundJobs); ScanKeyData scanKey[1] = { 0 }; const bool indexOK = true; /* WHERE job_id = $task->jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_job_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundJobs, DistBackgroundJobPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find background jobs entry for job_id: " UINT64_FORMAT, jobId))); } Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, values, isnull); bool updated = false; Oid stateOid = BackgroundJobStatusOid(status); updated |= SetFieldValue(Anum_pg_dist_background_job_state, values, isnull, replace, ObjectIdGetDatum(stateOid)); if (status == BACKGROUND_JOB_STATUS_RUNNING) { if (isnull[Anum_pg_dist_background_job_started_at - 1]) { /* first time status has been updated and was running, updating started_at */ TimestampTz startedAt = GetCurrentTimestamp(); updated |= SetFieldValue(Anum_pg_dist_background_job_started_at, values, isnull, replace, TimestampTzGetDatum(startedAt)); } } if (IsBackgroundJobStatusTerminal(status)) { if (isnull[Anum_pg_dist_background_job_finished_at - 1]) { /* didn't have a finished at time just yet, updating to now */ TimestampTz finishedAt = GetCurrentTimestamp(); updated |= SetFieldValue(Anum_pg_dist_background_job_finished_at, values, isnull, replace, TimestampTzGetDatum(finishedAt)); } } if (updated) { heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistBackgroundJobs, &heapTuple->t_self, heapTuple); CommandCounterIncrement(); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundJobs, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * UpdateBackgroundTask updates the catalog entry for the passed task, preventing an * actual update when the inmemory representation is the same as the one stored in the * catalog. */ void UpdateBackgroundTask(BackgroundTask *task) { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); ScanKeyData scanKey[1] = { 0 }; const bool indexOK = true; /* WHERE task_id = $task->taskid */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_task_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(task->taskid)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find background task entry for :" "job_id/task_id: " UINT64_FORMAT "/" UINT64_FORMAT, task->jobid, task->taskid))); } Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, values, isnull); bool updated = false; updated |= SetFieldValue(Anum_pg_dist_background_task_owner, values, isnull, replace, task->owner); if (task->pid) { updated |= SetFieldValue(Anum_pg_dist_background_task_pid, values, isnull, replace, Int32GetDatum(*task->pid)); } else { updated |= SetFieldNull(Anum_pg_dist_background_task_pid, values, isnull, replace); } Oid statusOid = ObjectIdGetDatum(BackgroundTaskStatusOid(task->status)); updated |= SetFieldValue(Anum_pg_dist_background_task_status, values, isnull, replace, statusOid); if (task->retry_count) { updated |= SetFieldValue(Anum_pg_dist_background_task_retry_count, values, isnull, replace, Int32GetDatum(*task->retry_count)); } else { updated |= SetFieldNull(Anum_pg_dist_background_task_retry_count, values, isnull, replace); } if (task->not_before) { updated |= SetFieldValue(Anum_pg_dist_background_task_not_before, values, isnull, replace, TimestampTzGetDatum(*task->not_before)); } else { updated |= SetFieldNull(Anum_pg_dist_background_task_not_before, values, isnull, replace); } if (task->message) { updated |= SetFieldText(Anum_pg_dist_background_task_message, values, isnull, replace, task->message); } else { updated |= SetFieldNull(Anum_pg_dist_background_task_message, values, isnull, replace); } if (updated) { heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistBackgroundTasks, &heapTuple->t_self, heapTuple); CommandCounterIncrement(); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasks, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * GetDependantTasks returns a list of taskId's containing all tasks depending on the task * passed via its arguments. * * Becasue tasks are int64 we allocate and return a List of int64 pointers. */ static List * GetDependantTasks(int64 jobId, int64 taskId) { Relation pgDistBackgroundTasksDepends = table_open(DistBackgroundTaskDependRelationId(), RowExclusiveLock); ScanKeyData scanKey[2] = { 0 }; const bool indexOK = true; /* pg_dist_background_task_depend.job_id = $jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_depend_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobId)); /* pg_dist_background_task_depend.depends_on = $taskId */ ScanKeyInit(&scanKey[1], Anum_pg_dist_background_task_depend_depends_on, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(taskId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasksDepends, DistBackgroundTaskDependDependsOnIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); List *dependantTasks = NIL; HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Form_pg_dist_background_task_depend depend = (Form_pg_dist_background_task_depend) GETSTRUCT(heapTuple); int64 *dTaskId = palloc0(sizeof(int64)); *dTaskId = depend->task_id; dependantTasks = lappend(dependantTasks, dTaskId); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasksDepends, NoLock); return dependantTasks; } /* * CancelTasksForJob cancels all tasks associated with a job that are not currently * running and are not already in their terminal state. Canceling these tasks consist of * updating the status of the task in the catalog. * * For all other tasks, namely the ones that are currently running, it returns the list of * Pid's of the tasks running. These backends should be signalled for cancellation. * * Since we are either signalling or changing the status of a task we perform appropriate * permission checks. This currently includes the exact same checks pg_cancel_backend * would perform. */ List * CancelTasksForJob(int64 jobid) { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), ExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); ScanKeyData scanKey[1] = { 0 }; /* WHERE jobId = $jobid */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(jobid)); const bool indexOK = true; SysScanDesc scanDescriptor = systable_beginscan( pgDistBackgroundTasks, DistBackgroundTaskJobIdTaskIdIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); List *runningTaskPids = NIL; HeapTuple taskTuple = NULL; while (HeapTupleIsValid(taskTuple = systable_getnext(scanDescriptor))) { Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *nulls = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(taskTuple, tupleDescriptor, values, nulls); Oid statusOid = DatumGetObjectId(values[Anum_pg_dist_background_task_status - 1]); BackgroundTaskStatus status = BackgroundTaskStatusByOid(statusOid); if (IsBackgroundTaskStatusTerminal(status)) { continue; } /* make sure the current user has the rights to cancel this task */ Oid taskOwner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner - 1]); if (superuser_arg(taskOwner) && !superuser()) { /* must be a superuser to cancel tasks owned by superuser */ ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be a superuser to cancel superuser tasks"))); } else if (!has_privs_of_role(GetUserId(), taskOwner) && !has_privs_of_role(GetUserId(), ROLE_PG_SIGNAL_BACKEND)) { /* user doesn't have the permissions to cancel this job */ ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be a member of the role whose task is being " "canceled or member of pg_signal_backend"))); } BackgroundTaskStatus newStatus = BACKGROUND_TASK_STATUS_CANCELLED; if (status == BACKGROUND_TASK_STATUS_RUNNING) { if (!nulls[Anum_pg_dist_background_task_pid - 1]) { int32 pid = DatumGetInt32(values[Anum_pg_dist_background_task_pid - 1]); runningTaskPids = lappend_int(runningTaskPids, pid); newStatus = BACKGROUND_TASK_STATUS_CANCELLING; } } /* update task to new status */ nulls[Anum_pg_dist_background_task_status - 1] = false; values[Anum_pg_dist_background_task_status - 1] = ObjectIdGetDatum( BackgroundTaskStatusOid(newStatus)); replace[Anum_pg_dist_background_task_status - 1] = true; taskTuple = heap_modify_tuple(taskTuple, tupleDescriptor, values, nulls, replace); CatalogTupleUpdate(pgDistBackgroundTasks, &taskTuple->t_self, taskTuple); pfree(values); pfree(nulls); pfree(replace); } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasks, NoLock); CommandCounterIncrement(); return runningTaskPids; } /* * UnscheduleDependentTasks follows the dependency tree of the provided task recursively * to unschedule any task depending on the current task. * * This is useful to unschedule any task that can never run because it will never satisfy * the unmet dependency constraint. */ void UnscheduleDependentTasks(BackgroundTask *task) { Relation pgDistBackgroundTasks = table_open(DistBackgroundTaskRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistBackgroundTasks); List *dependantTasks = GetDependantTasks(task->jobid, task->taskid); while (list_length(dependantTasks) > 0) { /* pop last item from stack */ int64 cTaskId = *(int64 *) llast(dependantTasks); dependantTasks = list_delete_last(dependantTasks); /* push new dependant tasks on to stack */ dependantTasks = list_concat(dependantTasks, GetDependantTasks(task->jobid, cTaskId)); /* unschedule current task */ { ScanKeyData scanKey[1] = { 0 }; /* WHERE taskId = dependentTask->taskId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_task_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(cTaskId)); const bool indexOK = true; SysScanDesc scanDescriptor = systable_beginscan(pgDistBackgroundTasks, DistBackgroundTaskPKeyIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find background task entry for " "task_id: " UINT64_FORMAT, cTaskId))); } Datum *values = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); values[Anum_pg_dist_background_task_status - 1] = ObjectIdGetDatum(CitusTaskStatusUnscheduledId()); isnull[Anum_pg_dist_background_task_status - 1] = false; replace[Anum_pg_dist_background_task_status - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistBackgroundTasks, &heapTuple->t_self, heapTuple); systable_endscan(scanDescriptor); pfree(values); pfree(isnull); pfree(replace); } } CommandCounterIncrement(); table_close(pgDistBackgroundTasks, NoLock); } /* * UnblockDependingBackgroundTasks unblocks any depending task that now satisfies the * constraaint that it doesn't have unmet dependencies anymore. For this to be done we * will find all tasks depending on the current task. Per found task we check if it has * any unmet dependencies. If no tasks are found that would block the execution of this * task we transition the task to Runnable state. */ void UnblockDependingBackgroundTasks(BackgroundTask *task) { Relation pgDistBackgroundTasksDepend = table_open(DistBackgroundTaskDependRelationId(), RowExclusiveLock); ScanKeyData scanKey[2] = { 0 }; /* WHERE jobId = $jobId */ ScanKeyInit(&scanKey[0], Anum_pg_dist_background_task_depend_job_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(task->jobid)); /* WHERE depends_on = $taskId */ ScanKeyInit(&scanKey[1], Anum_pg_dist_background_task_depend_depends_on, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(task->taskid)); const bool indexOK = true; SysScanDesc scanDescriptor = systable_beginscan( pgDistBackgroundTasksDepend, DistBackgroundTaskDependDependsOnIndexId(), indexOK, NULL, lengthof(scanKey), scanKey); HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { Form_pg_dist_background_task_depend depend = (Form_pg_dist_background_task_depend) GETSTRUCT(heapTuple); if (!BackgroundTaskHasUmnetDependencies(task->jobid, depend->task_id)) { /* * The task does not have any unmet dependencies anymore and should become * runnable */ BackgroundTask *unblockedTask = GetBackgroundTaskByTaskId(depend->task_id); if (unblockedTask->status == BACKGROUND_TASK_STATUS_CANCELLED) { continue; } Assert(unblockedTask->status == BACKGROUND_TASK_STATUS_BLOCKED); unblockedTask->status = BACKGROUND_TASK_STATUS_RUNNABLE; UpdateBackgroundTask(unblockedTask); } } systable_endscan(scanDescriptor); table_close(pgDistBackgroundTasksDepend, NoLock); } /* * GetAutoConvertedAttrIndexInPgDistPartition returns attrnum for autoconverted attr. * * autoconverted attr was added to table pg_dist_partition using alter operation after * the version where Citus started supporting downgrades, and it's only column that we've * introduced to pg_dist_partition since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_pg_dist_partition and when this happens, then we know that attrnum autoconverted is * not Anum_pg_dist_partition_autoconverted anymore but tupleDesc->natts - 1. */ int GetAutoConvertedAttrIndexInPgDistPartition(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_partition ? (Anum_pg_dist_partition_autoconverted - 1) : tupleDesc->natts - 1; } /* * GetNodesInvolvedAttrIndexInPgDistBackgroundTask returns attrnum for nodes_involved attr. * * nodes_involved attr was added to table pg_dist_background_task using alter operation after * the version where Citus started supporting downgrades, and it's only column that we've * introduced to pg_dist_background_task since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_pg_dist_background_task and when this happens, then we know that attrnum nodes_involved is * not Anum_pg_dist_background_task_nodes_involved anymore but tupleDesc->natts - 1. */ int GetNodesInvolvedAttrIndexInPgDistBackgroundTask(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_background_task ? (Anum_pg_dist_background_task_nodes_involved - 1) : tupleDesc->natts - 1; } ================================================ FILE: src/backend/distributed/metadata/node_metadata.c ================================================ /* * node_metadata.c * Functions that operate on pg_dist_node * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/tupmacs.h" #include "access/xact.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "commands/sequence.h" #include "executor/spi.h" #include "lib/stringinfo.h" #include "postmaster/postmaster.h" #include "storage/bufmgr.h" #include "storage/fd.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/plancache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "distributed/citus_acquire_lock.h" #include "distributed/citus_safe_lib.h" #include "distributed/clonenode_utils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/maintenanced.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_node_metadata.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/shared_connection_stats.h" #include "distributed/string_utils.h" #include "distributed/transaction_recovery.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" #define INVALID_GROUP_ID -1 /* default group size */ int GroupSize = 1; /* config variable managed via guc.c */ char *CurrentCluster = "default"; /* did current transaction modify pg_dist_node? */ bool TransactionModifiedNodeMetadata = false; bool EnableMetadataSync = true; typedef struct NodeMetadata { int32 groupId; char *nodeRack; bool hasMetadata; bool metadataSynced; bool isActive; Oid nodeRole; bool shouldHaveShards; uint32 nodeprimarynodeid; bool nodeisclone; char *nodeCluster; } NodeMetadata; /* local function forward declarations */ static void RemoveNodeFromCluster(char *nodeName, int32 nodePort); static void ErrorIfNodeContainsNonRemovablePlacements(WorkerNode *workerNode); static bool PlacementHasActivePlacementOnAnotherGroup(GroupShardPlacement *sourcePlacement); static int AddNodeMetadata(char *nodeName, int32 nodePort, NodeMetadata *nodeMetadata, bool *nodeAlreadyExists, bool localOnly); static int AddNodeMetadataViaMetadataContext(char *nodeName, int32 nodePort, NodeMetadata *nodeMetadata, bool *nodeAlreadyExists); static HeapTuple GetNodeTuple(const char *nodeName, int32 nodePort); static HeapTuple GetNodeByNodeId(int32 nodeId); static int32 GetNextGroupId(void); static int GetNextNodeId(void); static void InsertPlaceholderCoordinatorRecord(void); static void InsertNodeRow(int nodeid, char *nodename, int32 nodeport, NodeMetadata *nodeMetadata); static void DeleteNodeRow(char *nodename, int32 nodeport); static void BlockDistributedQueriesOnMetadataNodes(void); static WorkerNode * TupleToWorkerNode(Relation pgDistNode, TupleDesc tupleDescriptor, HeapTuple heapTuple); static bool NodeIsLocal(WorkerNode *worker); static void SetLockTimeoutLocally(int32 lock_cooldown); static void UpdateNodeLocation(int32 nodeId, char *newNodeName, int32 newNodePort, bool localOnly); static int GetNodePrimaryNodeIdAttrIndexInPgDistNode(TupleDesc tupleDesc); static int GetNodeIsCloneAttrIndexInPgDistNode(TupleDesc tupleDesc); static bool UnsetMetadataSyncedForAllWorkers(void); static char * GetMetadataSyncCommandToSetNodeColumn(WorkerNode *workerNode, int columnIndex, Datum value); static char * NodeHasmetadataUpdateCommand(uint32 nodeId, bool hasMetadata); static char * NodeMetadataSyncedUpdateCommand(uint32 nodeId, bool metadataSynced); static void ErrorIfCoordinatorMetadataSetFalse(WorkerNode *workerNode, Datum value, char *field); static WorkerNode * SetShouldHaveShards(WorkerNode *workerNode, bool shouldHaveShards); static void ErrorIfAnyNodeNotExist(List *nodeList); static void UpdateLocalGroupIdsViaMetadataContext(MetadataSyncContext *context); static void SendDeletionCommandsForReplicatedTablePlacements(MetadataSyncContext *context) ; static void SyncNodeMetadata(MetadataSyncContext *context); static void SetNodeStateViaMetadataContext(MetadataSyncContext *context, WorkerNode *workerNode, Datum value); static void MarkNodesNotSyncedInLoopBackConnection(MetadataSyncContext *context, pid_t parentSessionPid); static void EnsureParentSessionHasExclusiveLockOnPgDistNode(pid_t parentSessionPid); static void SetNodeMetadata(MetadataSyncContext *context, bool localOnly); static void EnsureTransactionalMetadataSyncMode(void); static BackgroundWorkerHandle * CheckBackgroundWorkerToObtainLocks(int32 lock_cooldown); static BackgroundWorkerHandle * LockPlacementsWithBackgroundWorkersInPrimaryNode( WorkerNode *workerNode, bool force, int32 lock_cooldown); static int32 CitusAddCloneNode(WorkerNode *primaryWorkerNode, char *cloneHostname, int32 clonePort); static void RemoveCloneNode(WorkerNode *cloneNode); /* Function definitions go here */ /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(citus_set_coordinator_host); PG_FUNCTION_INFO_V1(citus_add_node); PG_FUNCTION_INFO_V1(master_add_node); PG_FUNCTION_INFO_V1(citus_add_inactive_node); PG_FUNCTION_INFO_V1(master_add_inactive_node); PG_FUNCTION_INFO_V1(citus_add_secondary_node); PG_FUNCTION_INFO_V1(master_add_secondary_node); PG_FUNCTION_INFO_V1(citus_set_node_property); PG_FUNCTION_INFO_V1(master_set_node_property); PG_FUNCTION_INFO_V1(citus_remove_node); PG_FUNCTION_INFO_V1(master_remove_node); PG_FUNCTION_INFO_V1(citus_disable_node); PG_FUNCTION_INFO_V1(master_disable_node); PG_FUNCTION_INFO_V1(citus_activate_node); PG_FUNCTION_INFO_V1(master_activate_node); PG_FUNCTION_INFO_V1(citus_update_node); PG_FUNCTION_INFO_V1(citus_pause_node_within_txn); PG_FUNCTION_INFO_V1(master_update_node); PG_FUNCTION_INFO_V1(get_shard_id_for_distribution_column); PG_FUNCTION_INFO_V1(citus_nodename_for_nodeid); PG_FUNCTION_INFO_V1(citus_nodeport_for_nodeid); PG_FUNCTION_INFO_V1(citus_coordinator_nodeid); PG_FUNCTION_INFO_V1(citus_is_coordinator); PG_FUNCTION_INFO_V1(citus_internal_mark_node_not_synced); PG_FUNCTION_INFO_V1(citus_is_primary_node); PG_FUNCTION_INFO_V1(citus_add_clone_node); PG_FUNCTION_INFO_V1(citus_add_clone_node_with_nodeid); PG_FUNCTION_INFO_V1(citus_remove_clone_node); PG_FUNCTION_INFO_V1(citus_remove_clone_node_with_nodeid); /* * DefaultNodeMetadata creates a NodeMetadata struct with the fields set to * sane defaults, e.g. nodeRack = WORKER_DEFAULT_RACK. */ static NodeMetadata DefaultNodeMetadata() { NodeMetadata nodeMetadata; /* ensure uninitialized padding doesn't escape the function */ memset_struct_0(nodeMetadata); nodeMetadata.nodeRack = WORKER_DEFAULT_RACK; nodeMetadata.shouldHaveShards = true; nodeMetadata.groupId = INVALID_GROUP_ID; nodeMetadata.nodeisclone = false; nodeMetadata.nodeprimarynodeid = 0; /* 0 typically means InvalidNodeId */ return nodeMetadata; } /* * citus_set_coordinator_host configures the hostname and port through which worker * nodes can connect to the coordinator. */ Datum citus_set_coordinator_host(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeNameString = text_to_cstring(nodeName); NodeMetadata nodeMetadata = DefaultNodeMetadata(); nodeMetadata.groupId = 0; nodeMetadata.shouldHaveShards = false; nodeMetadata.nodeRole = PG_GETARG_OID(2); Name nodeClusterName = PG_GETARG_NAME(3); nodeMetadata.nodeCluster = NameStr(*nodeClusterName); /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (nodeMetadata.nodeRole == SecondaryNodeRoleId()) { EnsureTransactionalMetadataSyncMode(); } /* prevent concurrent modification */ LockRelationOid(DistNodeRelationId(), RowExclusiveLock); bool isCoordinatorInMetadata = false; WorkerNode *coordinatorNode = PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &isCoordinatorInMetadata); if (!isCoordinatorInMetadata) { bool nodeAlreadyExists = false; bool localOnly = false; /* add the coordinator to pg_dist_node if it was not already added */ AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata, &nodeAlreadyExists, localOnly); /* we just checked */ Assert(!nodeAlreadyExists); } else { /* * since AddNodeMetadata takes an exclusive lock on pg_dist_node, we * do not need to worry about concurrent changes (e.g. deletion) and * can proceed to update immediately. */ bool localOnly = false; UpdateNodeLocation(coordinatorNode->nodeId, nodeNameString, nodePort, localOnly); /* clear cached plans that have the old host/port */ ResetPlanCache(); } TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * EnsureTransactionalMetadataSyncMode ensures metadata sync mode is transactional. */ static void EnsureTransactionalMetadataSyncMode(void) { if (MetadataSyncTransMode == METADATA_SYNC_NON_TRANSACTIONAL) { ereport(ERROR, (errmsg("this operation cannot be completed in nontransactional " "metadata sync mode"), errhint("SET citus.metadata_sync_mode to 'transactional'"))); } } /* * citus_add_node function adds a new node to the cluster and returns its id. It also * replicates all reference tables to the new node. */ Datum citus_add_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeNameString = text_to_cstring(nodeName); NodeMetadata nodeMetadata = DefaultNodeMetadata(); bool nodeAlreadyExists = false; nodeMetadata.groupId = PG_GETARG_INT32(2); /* * During tests this function is called before nodeRole and nodeCluster have been * created. */ if (PG_NARGS() == 3) { nodeMetadata.nodeRole = InvalidOid; nodeMetadata.nodeCluster = "default"; } else { Name nodeClusterName = PG_GETARG_NAME(4); nodeMetadata.nodeCluster = NameStr(*nodeClusterName); nodeMetadata.nodeRole = PG_GETARG_OID(3); } if (nodeMetadata.groupId == COORDINATOR_GROUP_ID) { /* by default, we add the coordinator without shards */ nodeMetadata.shouldHaveShards = false; } /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (nodeMetadata.nodeRole == SecondaryNodeRoleId()) { EnsureTransactionalMetadataSyncMode(); } if (MetadataSyncTransMode == METADATA_SYNC_NON_TRANSACTIONAL && IsMultiStatementTransaction()) { /* * prevent inside transaction block as we use bare connections which can * lead deadlock */ ereport(ERROR, (errmsg("do not add node in transaction block " "when the sync mode is nontransactional"), errhint("add the node after SET citus.metadata_sync_mode " "TO 'transactional'"))); } int nodeId = AddNodeMetadataViaMetadataContext(nodeNameString, nodePort, &nodeMetadata, &nodeAlreadyExists); TransactionModifiedNodeMetadata = true; PG_RETURN_INT32(nodeId); } /* * master_add_node is a wrapper function for old UDF name. */ Datum master_add_node(PG_FUNCTION_ARGS) { return citus_add_node(fcinfo); } /* * citus_add_inactive_node function adds a new node to the cluster as inactive node * and returns id of the newly added node. It does not replicate reference * tables to the new node, it only adds new node to the pg_dist_node table. */ Datum citus_add_inactive_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeNameString = text_to_cstring(nodeName); Name nodeClusterName = PG_GETARG_NAME(4); NodeMetadata nodeMetadata = DefaultNodeMetadata(); bool nodeAlreadyExists = false; nodeMetadata.groupId = PG_GETARG_INT32(2); nodeMetadata.nodeRole = PG_GETARG_OID(3); nodeMetadata.nodeCluster = NameStr(*nodeClusterName); if (nodeMetadata.groupId == COORDINATOR_GROUP_ID) { ereport(ERROR, (errmsg("coordinator node cannot be added as inactive node"))); } /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (nodeMetadata.nodeRole == SecondaryNodeRoleId()) { EnsureTransactionalMetadataSyncMode(); } bool localOnly = false; int nodeId = AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata, &nodeAlreadyExists, localOnly); TransactionModifiedNodeMetadata = true; PG_RETURN_INT32(nodeId); } /* * master_add_inactive_node is a wrapper function for old UDF name. */ Datum master_add_inactive_node(PG_FUNCTION_ARGS) { return citus_add_inactive_node(fcinfo); } /* * citus_add_secondary_node adds a new secondary node to the cluster. It accepts as * arguments the primary node it should share a group with. */ Datum citus_add_secondary_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeName = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeNameString = text_to_cstring(nodeName); text *primaryName = PG_GETARG_TEXT_P(2); int32 primaryPort = PG_GETARG_INT32(3); char *primaryNameString = text_to_cstring(primaryName); Name nodeClusterName = PG_GETARG_NAME(4); NodeMetadata nodeMetadata = DefaultNodeMetadata(); bool nodeAlreadyExists = false; nodeMetadata.groupId = GroupForNode(primaryNameString, primaryPort); nodeMetadata.nodeCluster = NameStr(*nodeClusterName); nodeMetadata.nodeRole = SecondaryNodeRoleId(); nodeMetadata.isActive = true; /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ EnsureTransactionalMetadataSyncMode(); bool localOnly = false; int nodeId = AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata, &nodeAlreadyExists, localOnly); TransactionModifiedNodeMetadata = true; PG_RETURN_INT32(nodeId); } /* * master_add_secondary_node is a wrapper function for old UDF name. */ Datum master_add_secondary_node(PG_FUNCTION_ARGS) { return citus_add_secondary_node(fcinfo); } /* * citus_remove_node function removes the provided node from the pg_dist_node table of * the master node and all nodes with metadata. * The call to the citus_remove_node should be done by the super user and the specified * node should not have any active placements. * This function also deletes all reference table placements belong to the given node from * pg_dist_placement, but it does not drop actual placement at the node. In the case of * re-adding the node, citus_add_node first drops and re-creates the reference tables. */ Datum citus_remove_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); RemoveNodeFromCluster(text_to_cstring(nodeNameText), nodePort); TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * master_remove_node is a wrapper function for old UDF name. */ Datum master_remove_node(PG_FUNCTION_ARGS) { return citus_remove_node(fcinfo); } /* * citus_disable_node function sets isactive value of the provided node as inactive at * coordinator and all nodes with metadata regardless of the node having an active shard * placement. * * The call to the citus_disable_node must be done by the super user. * * This function also deletes all reference table placements belong to the given node * from pg_dist_placement, but it does not drop actual placement at the node. In the case * of re-activating the node, citus_add_node first drops and re-creates the reference * tables. */ Datum citus_disable_node(PG_FUNCTION_ARGS) { text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); bool synchronousDisableNode = 1; Assert(PG_NARGS() == 2 || PG_NARGS() == 3); if (PG_NARGS() == 3) { synchronousDisableNode = PG_GETARG_BOOL(2); } char *nodeName = text_to_cstring(nodeNameText); WorkerNode *workerNode = ModifiableWorkerNode(nodeName, nodePort); /* there is no concept of invalid coordinator */ bool isActive = false; ErrorIfCoordinatorMetadataSetFalse(workerNode, BoolGetDatum(isActive), "isactive"); /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (NodeIsSecondary(workerNode)) { EnsureTransactionalMetadataSyncMode(); } WorkerNode *firstWorkerNode = GetFirstPrimaryWorkerNode(); bool disablingFirstNode = (firstWorkerNode && firstWorkerNode->nodeId == workerNode->nodeId); if (disablingFirstNode && !synchronousDisableNode) { /* * We sync metadata async and optionally in the background worker, * it would mean that some nodes might get the updates while other * not. And, if the node metadata that is changing is the first * worker node, the problem gets nasty. We serialize modifications * to replicated tables by acquiring locks on the first worker node. * * If some nodes get the metadata changes and some do not, they'd be * acquiring the locks on different nodes. Hence, having the * possibility of diverged shard placements for the same shard. * * To prevent that, we currently do not allow disabling the first * worker node unless it is explicitly opted synchronous. */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("disabling the first worker node in the " "metadata is not allowed"), errhint("You can force disabling node, SELECT " "citus_disable_node('%s', %d, " "synchronous:=true);", workerNode->workerName, nodePort), errdetail("Citus uses the first worker node in the " "metadata for certain internal operations when " "replicated tables are modified. Synchronous mode " "ensures that all nodes have the same view of the " "first worker node, which is used for certain " "locking operations."))); } /* * First, locally mark the node as inactive. We'll later trigger background * worker to sync the metadata changes to the relevant nodes. */ workerNode = SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_isactive, BoolGetDatum(isActive)); if (NodeIsPrimary(workerNode)) { /* * We do not allow disabling nodes if it contains any * primary placement that is the "only" active placement * for any given shard. */ ErrorIfNodeContainsNonRemovablePlacements(workerNode); } TransactionModifiedNodeMetadata = true; if (synchronousDisableNode) { /* * The user might pick between sync vs async options. * - Pros for the sync option: * (a) the changes become visible on the cluster immediately * (b) even if the first worker node is disabled, there is no * risk of divergence of the placements of replicated shards * - Cons for the sync options: * (a) Does not work within 2PC transaction (e.g., BEGIN; * citus_disable_node(); PREPARE TRANSACTION ...); * (b) If there are multiple node failures (e.g., one another node * than the current node being disabled), the sync option would * fail because it'd try to sync the metadata changes to a node * that is not up and running. */ if (firstWorkerNode && firstWorkerNode->nodeId == workerNode->nodeId) { /* * We cannot let any modification query on a replicated table to run * concurrently with citus_disable_node() on the first worker node. If * we let that, some worker nodes might calculate FirstWorkerNode() * different than others. See LockShardListResourcesOnFirstWorker() * for the details. */ BlockDistributedQueriesOnMetadataNodes(); } SyncNodeMetadataToNodes(); } else if (UnsetMetadataSyncedForAllWorkers()) { /* * We have not propagated the node metadata changes yet, make sure that all the * active nodes get the metadata updates. We defer this operation to the * background worker to make it possible disabling nodes when multiple nodes * are down. * * Note that the active placements reside on the active nodes. Hence, when * Citus finds active placements, it filters out the placements that are on * the disabled nodes. That's why, we don't have to change/sync placement * metadata at this point. Instead, we defer that to citus_activate_node() * where we expect all nodes up and running. */ TriggerNodeMetadataSyncOnCommit(); } PG_RETURN_VOID(); } /* * BlockDistributedQueriesOnMetadataNodes blocks all the modification queries on * all nodes. Hence, should be used with caution. */ static void BlockDistributedQueriesOnMetadataNodes(void) { /* first, block on the coordinator */ LockRelationOid(DistNodeRelationId(), ExclusiveLock); /* * Note that we might re-design this lock to be more granular than * pg_dist_node, scoping only for modifications on the replicated * tables. However, we currently do not have any such mechanism and * given that citus_disable_node() runs instantly, it seems acceptable * to block reads (or modifications on non-replicated tables) for * a while. */ /* only superuser can disable node */ Assert(superuser()); SendCommandToWorkersWithMetadata( "LOCK TABLE pg_catalog.pg_dist_node IN EXCLUSIVE MODE;"); } /* * master_disable_node is a wrapper function for old UDF name. */ Datum master_disable_node(PG_FUNCTION_ARGS) { return citus_disable_node(fcinfo); } /* * citus_set_node_property sets a property of the node */ Datum citus_set_node_property(PG_FUNCTION_ARGS) { text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); text *propertyText = PG_GETARG_TEXT_P(2); bool value = PG_GETARG_BOOL(3); WorkerNode *workerNode = ModifiableWorkerNode(text_to_cstring(nodeNameText), nodePort); /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (NodeIsSecondary(workerNode)) { EnsureTransactionalMetadataSyncMode(); } if (strcmp(text_to_cstring(propertyText), "shouldhaveshards") == 0) { SetShouldHaveShards(workerNode, value); } else { ereport(ERROR, (errmsg( "only the 'shouldhaveshards' property can be set using this function"))); } TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * master_set_node_property is a wrapper function for old UDF name. */ Datum master_set_node_property(PG_FUNCTION_ARGS) { return citus_set_node_property(fcinfo); } /* * ModifiableWorkerNode gets the requested WorkerNode and also gets locks * required for modifying it. This fails if the node does not exist. */ WorkerNode * ModifiableWorkerNode(const char *nodeName, int32 nodePort) { CheckCitusVersion(ERROR); EnsureCoordinator(); /* take an exclusive lock on pg_dist_node to serialize pg_dist_node changes */ LockRelationOid(DistNodeRelationId(), ExclusiveLock); WorkerNode *workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort); if (workerNode == NULL) { ereport(ERROR, (errmsg("node at \"%s:%u\" does not exist", nodeName, nodePort))); } return workerNode; } /* * citus_activate_node UDF activates the given node. It sets the node's isactive * value to active and replicates all reference tables to that node. */ Datum citus_activate_node(PG_FUNCTION_ARGS) { text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeNameString = text_to_cstring(nodeNameText); WorkerNode *workerNode = ModifiableWorkerNode(nodeNameString, nodePort); /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (NodeIsSecondary(workerNode)) { EnsureTransactionalMetadataSyncMode(); } /* * Create MetadataSyncContext which is used throughout nodes' activation. * It contains activated nodes, bare connections if the mode is nontransactional, * and a memory context for allocation. */ bool collectCommands = false; bool nodesAddedInSameTransaction = false; MetadataSyncContext *context = CreateMetadataSyncContext(list_make1(workerNode), collectCommands, nodesAddedInSameTransaction); ActivateNodeList(context); TransactionModifiedNodeMetadata = true; PG_RETURN_INT32(workerNode->nodeId); } /* * master_activate_node is a wrapper function for old UDF name. */ Datum master_activate_node(PG_FUNCTION_ARGS) { return citus_activate_node(fcinfo); } /* * GroupForNode returns the group which a given node belongs to. * * It only works if the requested node is a part of CurrentCluster. */ uint32 GroupForNode(char *nodeName, int nodePort) { WorkerNode *workerNode = FindWorkerNode(nodeName, nodePort); if (workerNode == NULL) { ereport(ERROR, (errmsg("node at \"%s:%u\" does not exist", nodeName, nodePort))); } return workerNode->groupId; } /* * NodeIsPrimaryAndRemote returns whether the argument represents the remote * primary node. */ bool NodeIsPrimaryAndRemote(WorkerNode *worker) { return NodeIsPrimary(worker) && !NodeIsLocal(worker); } /* * NodeIsPrimary returns whether the argument represents a primary node. */ bool NodeIsPrimary(WorkerNode *worker) { Oid primaryRole = PrimaryNodeRoleId(); /* if nodeRole does not yet exist, all nodes are primary nodes */ if (primaryRole == InvalidOid) { return true; } return worker->nodeRole == primaryRole; } /* * NodeIsLocal returns whether the argument represents the local node. */ static bool NodeIsLocal(WorkerNode *worker) { return worker->groupId == GetLocalGroupId(); } /* * NodeIsSecondary returns whether the argument represents a secondary node. */ bool NodeIsSecondary(WorkerNode *worker) { Oid secondaryRole = SecondaryNodeRoleId(); /* if nodeRole does not yet exist, all nodes are primary nodes */ if (secondaryRole == InvalidOid) { return false; } return worker->nodeRole == secondaryRole; } /* * NodeIsReadable returns whether we're allowed to send SELECT queries to this * node. */ bool NodeIsReadable(WorkerNode *workerNode) { if (ReadFromSecondaries == USE_SECONDARY_NODES_NEVER && NodeIsPrimary(workerNode)) { return true; } if (ReadFromSecondaries == USE_SECONDARY_NODES_ALWAYS && NodeIsSecondary(workerNode)) { return true; } return false; } /* * PrimaryNodeForGroup returns the (unique) primary in the specified group. * * If there are any nodes in the requested group and groupContainsNodes is not NULL * it will set the bool groupContainsNodes references to true. */ WorkerNode * PrimaryNodeForGroup(int32 groupId, bool *groupContainsNodes) { WorkerNode *workerNode = NULL; HASH_SEQ_STATUS status; HTAB *workerNodeHash = GetWorkerNodeHash(); hash_seq_init(&status, workerNodeHash); while ((workerNode = hash_seq_search(&status)) != NULL) { int32 workerNodeGroupId = workerNode->groupId; if (workerNodeGroupId != groupId) { continue; } if (groupContainsNodes != NULL) { *groupContainsNodes = true; } if (NodeIsPrimary(workerNode)) { hash_seq_term(&status); return workerNode; } } return NULL; } /* * MarkNodesNotSyncedInLoopBackConnection unsets metadatasynced flag in separate * connection to localhost by calling the udf `citus_internal_mark_node_not_synced`. */ static void MarkNodesNotSyncedInLoopBackConnection(MetadataSyncContext *context, pid_t parentSessionPid) { Assert(context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL); Assert(!MetadataSyncCollectsCommands(context)); /* * Set metadatasynced to false for all activated nodes to mark the nodes as not synced * in case nontransactional metadata sync fails before we activate the nodes inside * metadataSyncContext. * We set metadatasynced to false at coordinator to mark the nodes as not synced. But we * do not set isactive and hasmetadata flags to false as we still want to route queries * to the nodes if their isactive flag is true and propagate DDL to the nodes if possible. * * NOTES: * 1) We use separate connection to localhost as we would rollback the local * transaction in case of failure. * 2) Operator should handle problems at workers if any. Wworkers probably fail * due to improper metadata when a query hits. Or DDL might fail due to desynced * nodes. (when hasmetadata = true, metadatasynced = false) * In those cases, proper metadata sync for the workers should be done.) */ /* * Because we try to unset metadatasynced flag with a separate transaction, * we could not find the new node if the node is added in the current local * transaction. But, hopefully, we do not need to unset metadatasynced for * the new node as local transaction would rollback in case of a failure. */ if (context->nodesAddedInSameTransaction) { return; } if (context->activatedWorkerNodeList == NIL) { return; } int connectionFlag = FORCE_NEW_CONNECTION; MultiConnection *connection = GetNodeConnection(connectionFlag, LocalHostName, PostPortNumber); List *commandList = NIL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, context->activatedWorkerNodeList) { /* * We need to prevent self deadlock when we access pg_dist_node using separate * connection to localhost. To achieve this, we check if the caller session's * pid holds the Exclusive lock on pg_dist_node. After ensuring that (we are * called from parent session which holds the Exclusive lock), we can safely * update node metadata by acquiring the relaxed lock. */ StringInfo metadatasyncCommand = makeStringInfo(); appendStringInfo(metadatasyncCommand, CITUS_INTERNAL_MARK_NODE_NOT_SYNCED, parentSessionPid, workerNode->nodeId); commandList = lappend(commandList, metadatasyncCommand->data); } SendCommandListToWorkerOutsideTransactionWithConnection(connection, commandList); CloseConnection(connection); } /* * SetNodeMetadata sets isactive, metadatasynced and hasmetadata flags locally * and, if required, remotely. */ static void SetNodeMetadata(MetadataSyncContext *context, bool localOnly) { /* do not execute local transaction if we collect commands */ if (!MetadataSyncCollectsCommands(context)) { List *updatedActivatedNodeList = NIL; WorkerNode *node = NULL; foreach_declared_ptr(node, context->activatedWorkerNodeList) { node = SetWorkerColumnLocalOnly(node, Anum_pg_dist_node_isactive, BoolGetDatum(true)); node = SetWorkerColumnLocalOnly(node, Anum_pg_dist_node_metadatasynced, BoolGetDatum(true)); node = SetWorkerColumnLocalOnly(node, Anum_pg_dist_node_hasmetadata, BoolGetDatum(true)); updatedActivatedNodeList = lappend(updatedActivatedNodeList, node); } /* reset activated nodes inside metadataSyncContext afer local update */ SetMetadataSyncNodesFromNodeList(context, updatedActivatedNodeList); } if (!localOnly && EnableMetadataSync) { WorkerNode *node = NULL; foreach_declared_ptr(node, context->activatedWorkerNodeList) { SetNodeStateViaMetadataContext(context, node, BoolGetDatum(true)); } } } /* * ActivateNodeList does some sanity checks and acquire Exclusive lock on pg_dist_node, * and then activates the nodes inside given metadataSyncContext. * * The function operates in 3 different modes according to transactionMode inside * metadataSyncContext. * * 1. MetadataSyncCollectsCommands(context): * Only collect commands instead of sending them to workers, * 2. context.transactionMode == METADATA_SYNC_TRANSACTIONAL: * Send all commands using coordinated transaction, * 3. context.transactionMode == METADATA_SYNC_NON_TRANSACTIONAL: * Send all commands using bare (no transaction block) connections. */ void ActivateNodeList(MetadataSyncContext *context) { if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL && IsMultiStatementTransaction()) { /* * prevent inside transaction block as we use bare connections which can * lead deadlock */ ereport(ERROR, (errmsg("do not sync metadata in transaction block " "when the sync mode is nontransactional"), errhint("resync after SET citus.metadata_sync_mode " "TO 'transactional'"))); } /* * We currently require the object propagation to happen via superuser, * see #5139. While activating a node, we sync both metadata and object * propagation. * * In order to have a fully transactional semantics with add/activate * node operations, we require superuser. Note that for creating * non-owned objects, we already require a superuser connection. * By ensuring the current user to be a superuser, we can guarantee * to send all commands within the same remote transaction. */ EnsureSuperUser(); /* * Take an exclusive lock on pg_dist_node to serialize pg_dist_node * changes. */ LockRelationOid(DistNodeRelationId(), ExclusiveLock); /* * Error if there is concurrent change to node table before acquiring * the lock */ ErrorIfAnyNodeNotExist(context->activatedWorkerNodeList); /* * we need to unset metadatasynced flag to false at coordinator in separate * transaction only at nontransactional sync mode and if we do not collect * commands. * * We make sure we set the flag to false at the start of nontransactional * metadata sync to mark those nodes are not synced in case of a failure in * the middle of the sync. */ if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL && !MetadataSyncCollectsCommands(context)) { MarkNodesNotSyncedInLoopBackConnection(context, MyProcPid); } /* * Delete existing reference and replicated table placements on the * given groupId if the group has been disabled earlier (e.g., isActive * set to false). */ SendDeletionCommandsForReplicatedTablePlacements(context); /* * SetNodeMetadata sets isactive, metadatasynced and hasmetadata flags * locally for following reasons: * * 1) Set isactive to true locally so that we can find activated nodes amongst * active workers, * 2) Do not fail just because the current metadata is not synced. (see * ErrorIfAnyMetadataNodeOutOfSync), * 3) To propagate activated nodes nodemetadata correctly. * * We are going to sync the metadata anyway in this transaction, set * isactive, metadatasynced, and hasmetadata to true locally. * The changes would rollback in case of failure. */ bool localOnly = true; SetNodeMetadata(context, localOnly); /* * Update local group ids so that upcoming transactions can see its effect. * Object dependency logic requires to have updated local group id. */ UpdateLocalGroupIdsViaMetadataContext(context); /* * Sync node metadata so that placement insertion does not fail due to * EnsureShardPlacementMetadataIsSane. */ SyncNodeMetadata(context); /* * Sync all dependencies and distributed objects with their pg_dist_xx tables to * metadata nodes inside metadataSyncContext. Depends on node metadata. */ SyncDistributedObjects(context); /* * Let all nodes to be active and synced after all operations succeeded. * we make sure that the metadata sync is idempotent and safe overall with multiple * other transactions, if nontransactional mode is used. * * We already took Exclusive lock on node metadata, which prevents modification * on node metadata on coordinator. The step will rollback, in case of a failure, * to the state where metadatasynced=false. */ localOnly = false; SetNodeMetadata(context, localOnly); } /* * ActivateCloneNodeAsPrimary sets the given worker node as primary and active * in the pg_dist_node catalog and make the clone node as first class citizen. */ void ActivateCloneNodeAsPrimary(WorkerNode *workerNode) { Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); TupleDesc copiedTupleDescriptor = CreateTupleDescCopy(tupleDescriptor); table_close(pgDistNode, AccessShareLock); /* * Set the node as primary and active. */ SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_noderole, ObjectIdGetDatum(PrimaryNodeRoleId())); SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_isactive, BoolGetDatum(true)); SetWorkerColumnLocalOnly(workerNode, GetNodeIsCloneAttrIndexInPgDistNode(copiedTupleDescriptor) + 1, BoolGetDatum(false)); SetWorkerColumnLocalOnly(workerNode, GetNodePrimaryNodeIdAttrIndexInPgDistNode( copiedTupleDescriptor) + 1, Int32GetDatum(0)); SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_hasmetadata, BoolGetDatum(true)); SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_metadatasynced, BoolGetDatum(true)); SetWorkerColumnLocalOnly(workerNode, Anum_pg_dist_node_shouldhaveshards, BoolGetDatum(true)); } /* * Acquires shard metadata locks on all shards residing in the given worker node * * TODO: This function is not compatible with query from any node feature. * To ensure proper behavior, it is essential to acquire locks on placements across all nodes * rather than limiting it to just the coordinator (or the specific node from which this function is called) */ void LockShardsInWorkerPlacementList(WorkerNode *workerNode, LOCKMODE lockMode) { List *placementList = AllShardPlacementsOnNodeGroup(workerNode->groupId); LockShardsInPlacementListMetadata(placementList, lockMode); } /* * This function is used to start a background worker to kill backends holding conflicting * locks with this backend. It returns NULL if the background worker could not be started. */ BackgroundWorkerHandle * CheckBackgroundWorkerToObtainLocks(int32 lock_cooldown) { BackgroundWorkerHandle *handle = StartLockAcquireHelperBackgroundWorker(MyProcPid, lock_cooldown) ; if (!handle) { /* * We failed to start a background worker, which probably means that we exceeded * max_worker_processes, and this is unlikely to be resolved by retrying. We do not want * to repeatedly throw an error because if citus_update_node is called to complete a * failover then finishing is the only way to bring the cluster back up. Therefore we * give up on killing other backends and simply wait for the lock. We do set * lock_timeout to lock_cooldown, because we don't want to wait forever to get a lock. */ SetLockTimeoutLocally(lock_cooldown); ereport(WARNING, (errmsg( "could not start background worker to kill backends with conflicting" " locks to force the update. Degrading to acquiring locks " "with a lock time out."), errhint( "Increasing max_worker_processes might help."))); } return handle; } /* * This function is used to lock shards in a primary node. * If force is true, we start a background worker to kill backends holding * conflicting locks with this backend. * * If the node is a primary node we block reads and writes. * * This lock has two purposes: * * - Ensure buggy code in Citus doesn't cause failures when the * nodename/nodeport of a node changes mid-query * * - Provide fencing during failover, after this function returns all * connections will use the new node location. * * Drawback: * * - This function blocks until all previous queries have finished. This * means that long-running queries will prevent failover. * * In case of node failure said long-running queries will fail in the end * anyway as they will be unable to commit successfully on the failed * machine. To cause quick failure of these queries use force => true * during the invocation of citus_update_node to terminate conflicting * backends proactively. * * It might be worth blocking reads to a secondary for the same reasons, * though we currently only query secondaries on follower clusters * where these locks will have no effect. */ BackgroundWorkerHandle * LockPlacementsWithBackgroundWorkersInPrimaryNode(WorkerNode *workerNode, bool force, int32 lock_cooldown) { BackgroundWorkerHandle *handle = NULL; if (NodeIsPrimary(workerNode)) { if (force) { handle = CheckBackgroundWorkerToObtainLocks(lock_cooldown); } LockShardsInWorkerPlacementList(workerNode, AccessExclusiveLock); } return handle; } /* * citus_update_node moves the requested node to a different nodename and nodeport. It * locks to ensure no queries are running concurrently; and is intended for customers who * are running their own failover solution. */ Datum citus_update_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int32 nodeId = PG_GETARG_INT32(0); text *newNodeName = PG_GETARG_TEXT_P(1); int32 newNodePort = PG_GETARG_INT32(2); /* * force is used when an update needs to happen regardless of conflicting locks. This * feature is important to force the update during a failover due to failure, eg. by * a high-availability system such as pg_auto_failover. The strategy is to start a * background worker that actively cancels backends holding conflicting locks with * this backend. * * Defaults to false */ bool force = PG_GETARG_BOOL(3); int32 lock_cooldown = PG_GETARG_INT32(4); char *newNodeNameString = text_to_cstring(newNodeName); WorkerNode *workerNodeWithSameAddress = FindWorkerNodeAnyCluster(newNodeNameString, newNodePort); if (workerNodeWithSameAddress != NULL) { /* a node with the given hostname and port already exists in the metadata */ if (workerNodeWithSameAddress->nodeId == nodeId) { /* it's the node itself, meaning this is a noop update */ PG_RETURN_VOID(); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("there is already another node with the specified " "hostname and port"))); } } WorkerNode *workerNode = FindNodeAnyClusterByNodeId(nodeId); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("node %u not found", nodeId))); } /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (NodeIsSecondary(workerNode)) { EnsureTransactionalMetadataSyncMode(); } BackgroundWorkerHandle *handle = LockPlacementsWithBackgroundWorkersInPrimaryNode( workerNode, force, lock_cooldown); /* * if we have planned statements such as prepared statements, we should clear the cache so that * the planned cache doesn't return the old nodename/nodepost. */ ResetPlanCache(); bool localOnly = true; UpdateNodeLocation(nodeId, newNodeNameString, newNodePort, localOnly); /* we should be able to find the new node from the metadata */ workerNode = FindWorkerNodeAnyCluster(newNodeNameString, newNodePort); Assert(workerNode->nodeId == nodeId); /* * Propagate the updated pg_dist_node entry to all metadata workers. * citus-ha uses citus_update_node() in a prepared transaction, and * we don't support coordinated prepared transactions, so we cannot * propagate the changes to the worker nodes here. Instead we mark * all metadata nodes as not-synced and ask maintenanced to do the * propagation. * * It is possible that maintenance daemon does the first resync too * early, but that's fine, since this will start a retry loop with * 5 second intervals until sync is complete. */ if (UnsetMetadataSyncedForAllWorkers()) { TriggerNodeMetadataSyncOnCommit(); } if (handle != NULL) { /* * this will be called on memory context cleanup as well, if the worker has been * terminated already this will be a noop */ TerminateBackgroundWorker(handle); } TransactionModifiedNodeMetadata = true; PG_RETURN_VOID(); } /* * This function is designed to obtain locks for all the shards in a worker placement list. * Once the transaction is committed, the acquired locks will be automatically released. * Therefore, it is essential to invoke this function within a transaction. * This function proves beneficial when there is a need to temporarily disable writes to a specific node within a transaction. */ Datum citus_pause_node_within_txn(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int32 nodeId = PG_GETARG_INT32(0); bool force = PG_GETARG_BOOL(1); int32 lock_cooldown = PG_GETARG_INT32(2); WorkerNode *workerNode = FindNodeAnyClusterByNodeId(nodeId); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("node %u not found", nodeId))); } LockPlacementsWithBackgroundWorkersInPrimaryNode(workerNode, force, lock_cooldown); PG_RETURN_VOID(); } /* * master_update_node is a wrapper function for old UDF name. */ Datum master_update_node(PG_FUNCTION_ARGS) { return citus_update_node(fcinfo); } /* * citus_add_clone_node adds a new node as a clone of an existing primary node. */ Datum citus_add_clone_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); text *cloneHostnameText = PG_GETARG_TEXT_P(0); int32 clonePort = PG_GETARG_INT32(1); text *primaryHostnameText = PG_GETARG_TEXT_P(2); int32 primaryPort = PG_GETARG_INT32(3); char *cloneHostname = text_to_cstring(cloneHostnameText); char *primaryHostname = text_to_cstring(primaryHostnameText); WorkerNode *primaryWorker = FindWorkerNodeAnyCluster(primaryHostname, primaryPort); if (primaryWorker == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("primary node %s:%d not found in pg_dist_node", primaryHostname, primaryPort))); } int32 cloneNodeId = CitusAddCloneNode(primaryWorker, cloneHostname, clonePort); PG_RETURN_INT32(cloneNodeId); } /* * citus_add_clone_node_with_nodeid adds a new node as a clone of an existing primary node * using the primary node's ID. It records the clone's hostname, port, and links it to the * primary node's ID. * * This function is useful when you already know the primary node's ID and want to add a clone * without needing to look it up by hostname and port. */ Datum citus_add_clone_node_with_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); text *cloneHostnameText = PG_GETARG_TEXT_P(0); int32 clonePort = PG_GETARG_INT32(1); int32 primaryNodeId = PG_GETARG_INT32(2); char *cloneHostname = text_to_cstring(cloneHostnameText); bool missingOk = false; WorkerNode *primaryWorkerNode = FindNodeWithNodeId(primaryNodeId, missingOk); if (primaryWorkerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("primary node with ID %d does not exist", primaryNodeId))); } int32 cloneNodeId = CitusAddCloneNode(primaryWorkerNode, cloneHostname, clonePort); PG_RETURN_INT32(cloneNodeId); } /* * CitusAddCloneNode function adds a new node as a clone of an existing primary node. * It records the clone's hostname, port, and links it to the primary node's ID. * The clone is initially marked as inactive and not having shards. */ static int32 CitusAddCloneNode(WorkerNode *primaryWorkerNode, char *cloneHostname, int32 clonePort) { Assert(primaryWorkerNode != NULL); /* Future-proofing: Ideally, a primary node should not itself be a clone. * This check might be more relevant once replica promotion logic exists. * For now, pg_dist_node.nodeisclone defaults to false for existing nodes. */ if (primaryWorkerNode->nodeisclone) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "primary node %s:%d is itself a clone and cannot have clones", primaryWorkerNode->workerName, primaryWorkerNode-> workerPort))); } if (!primaryWorkerNode->shouldHaveShards) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "primary node %s:%d does not have shards, node without shards cannot have clones", primaryWorkerNode->workerName, primaryWorkerNode-> workerPort))); } WorkerNode *existingCloneNode = FindWorkerNodeAnyCluster(cloneHostname, clonePort); if (existingCloneNode != NULL) { /* * Idempotency check: If the node already exists, is it already correctly * registered as a clone for THIS primary? */ if (existingCloneNode->nodeisclone && existingCloneNode->nodeprimarynodeid == primaryWorkerNode->nodeId) { ereport(NOTICE, (errmsg( "node %s:%d is already registered as a clone for primary %s:%d (nodeid %d)", cloneHostname, clonePort, primaryWorkerNode->workerName, primaryWorkerNode-> workerPort, primaryWorkerNode->nodeId))); PG_RETURN_INT32(existingCloneNode->nodeId); } else { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "a different node %s:%d (nodeid %d) already exists or is a clone for a different primary", cloneHostname, clonePort, existingCloneNode->nodeId))); } } EnsureValidStreamingReplica(primaryWorkerNode, cloneHostname, clonePort); char *operation = "add"; EnsureValidCloneMode(primaryWorkerNode, cloneHostname, clonePort, operation); NodeMetadata nodeMetadata = DefaultNodeMetadata(); nodeMetadata.nodeisclone = true; nodeMetadata.nodeprimarynodeid = primaryWorkerNode->nodeId; nodeMetadata.isActive = false; /* Replicas start as inactive */ nodeMetadata.shouldHaveShards = false; /* Replicas do not directly own primary shards */ nodeMetadata.groupId = INVALID_GROUP_ID; /* Replicas get a new group ID and do not belong to any existing group */ nodeMetadata.nodeRole = UnavailableNodeRoleId(); /* The node role is set to 'unavailable' */ nodeMetadata.nodeCluster = primaryWorkerNode->nodeCluster; /* Same cluster as primary */ /* Other fields like hasMetadata, metadataSynced will take defaults from DefaultNodeMetadata * (typically true, true for hasMetadata and metadataSynced if it's a new node, * or might need adjustment based on replica strategy) * For now, let's assume DefaultNodeMetadata provides suitable defaults for these * or they will be set by AddNodeMetadata/ActivateNodeList if needed. * Specifically, hasMetadata is often true, and metadataSynced true after activation. * Since this replica is inactive, metadata sync status might be less critical initially. */ bool nodeAlreadyExists = false; bool localOnly = false; /* Propagate change to other workers with metadata */ /* * AddNodeMetadata will take an ExclusiveLock on pg_dist_node. * It also checks again if the node already exists after acquiring the lock. */ int cloneNodeId = AddNodeMetadata(cloneHostname, clonePort, &nodeMetadata, &nodeAlreadyExists, localOnly); if (nodeAlreadyExists) { /* This case should ideally be caught by the FindWorkerNodeAnyCluster check above, * but AddNodeMetadata does its own check after locking. * If it already exists and is correctly configured, we might have returned NOTICE above. * If it exists but is NOT correctly configured as our replica, an ERROR would be more appropriate. * AddNodeMetadata returns the existing node's ID if it finds one. * We need to ensure it is the *correct* replica. */ WorkerNode *fetchedExistingNode = FindNodeAnyClusterByNodeId(cloneNodeId); if (fetchedExistingNode != NULL && fetchedExistingNode->nodeisclone && fetchedExistingNode->nodeprimarynodeid == primaryWorkerNode->nodeId) { ereport(NOTICE, (errmsg( "node %s:%d was already correctly registered as a clone for primary %s:%d (nodeid %d)", cloneHostname, clonePort, primaryWorkerNode->workerName, primaryWorkerNode-> workerPort, primaryWorkerNode->nodeId))); /* Intentional fall-through to return cloneNodeId */ } else { /* This state is less expected if our initial check passed or errored. */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "node %s:%d already exists but is not correctly configured as a clone for primary %s:%d", cloneHostname, clonePort, primaryWorkerNode->workerName, primaryWorkerNode->workerPort))); } } TransactionModifiedNodeMetadata = true; /* * Note: Clones added this way are inactive. * A separate UDF citus_promote_clone_and_rebalance * would be needed to activate them. */ return cloneNodeId; } /* * citus_remove_clone_node removes an inactive streaming clone node from Citus metadata. */ Datum citus_remove_clone_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); char *nodeName = text_to_cstring(nodeNameText); WorkerNode *workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("node \"%s:%d\" does not exist", nodeName, nodePort))); } RemoveCloneNode(workerNode); PG_RETURN_VOID(); } /* * citus_remove_clone_node_with_nodeid removes an inactive clone node from Citus metadata * using the node's ID. */ Datum citus_remove_clone_node_with_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); uint32 replicaNodeId = PG_GETARG_INT32(0); WorkerNode *replicaNode = FindNodeAnyClusterByNodeId(replicaNodeId); if (replicaNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Clone node with ID %d does not exist", replicaNodeId))); } RemoveCloneNode(replicaNode); PG_RETURN_VOID(); } static void RemoveCloneNode(WorkerNode *cloneNode) { Assert(cloneNode != NULL); if (!cloneNode->nodeisclone) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Node %s:%d (ID %d) is not a clone node. " "Use citus_remove_node() to remove primary or already promoted nodes.", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); } if (cloneNode->isActive) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Clone node %s:%d (ID %d) is marked as active and cannot be removed with this function. " "This might indicate a promoted clone. Consider using citus_remove_node() if you are sure, " "or ensure it's properly deactivated if it's an unpromoted clone in an unexpected state.", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); } /* * All checks passed, proceed with removal. * RemoveNodeFromCluster handles locking, catalog changes, connection closing, and metadata sync. */ ereport(NOTICE, (errmsg("Removing inactive clone node %s:%d (ID %d)", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); RemoveNodeFromCluster(cloneNode->workerName, cloneNode->workerPort); /* RemoveNodeFromCluster might set this, but setting it here ensures it's marked for this UDF's transaction. */ TransactionModifiedNodeMetadata = true; } /* * SetLockTimeoutLocally sets the lock_timeout to the given value. * This setting is local. */ static void SetLockTimeoutLocally(int32 lockCooldown) { set_config_option("lock_timeout", ConvertIntToString(lockCooldown), (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } static void UpdateNodeLocation(int32 nodeId, char *newNodeName, int32 newNodePort, bool localOnly) { const bool indexOK = true; Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); ScanKeyData scanKey[1]; Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodeid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(nodeId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistNode, DistNodeNodeIdIndexId(), indexOK, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for node \"%s:%d\"", newNodeName, newNodePort))); } values[Anum_pg_dist_node_nodeport - 1] = Int32GetDatum(newNodePort); isnull[Anum_pg_dist_node_nodeport - 1] = false; replace[Anum_pg_dist_node_nodeport - 1] = true; values[Anum_pg_dist_node_nodename - 1] = CStringGetTextDatum(newNodeName); isnull[Anum_pg_dist_node_nodename - 1] = false; replace[Anum_pg_dist_node_nodename - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistNode, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistNodeRelationId()); CommandCounterIncrement(); if (!localOnly && EnableMetadataSync) { WorkerNode *updatedNode = FindWorkerNodeAnyCluster(newNodeName, newNodePort); Assert(updatedNode->nodeId == nodeId); /* send the delete command to all primary nodes with metadata */ char *nodeDeleteCommand = NodeDeleteCommand(updatedNode->nodeId); SendCommandToWorkersWithMetadata(nodeDeleteCommand); /* send the insert command to all primary nodes with metadata */ char *nodeInsertCommand = NodeListInsertCommand(list_make1(updatedNode)); SendCommandToWorkersWithMetadata(nodeInsertCommand); } systable_endscan(scanDescriptor); table_close(pgDistNode, NoLock); pfree(values); pfree(isnull); pfree(replace); } /* * get_shard_id_for_distribution_column function takes a distributed table name and a * distribution value then returns shard id of the shard which belongs to given table and * contains given value. This function only works for hash distributed tables. */ Datum get_shard_id_for_distribution_column(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ShardInterval *shardInterval = NULL; /* * To have optional parameter as NULL, we defined this UDF as not strict, therefore * we need to check all parameters for NULL values. */ if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("relation cannot be NULL"))); } Oid relationId = PG_GETARG_OID(0); EnsureTablePermissions(relationId, ACL_SELECT, ACLMASK_ANY); if (!IsCitusTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("relation is not distributed"))); } if (!HasDistributionKey(relationId)) { List *shardIntervalList = LoadShardIntervalList(relationId); if (shardIntervalList == NIL) { PG_RETURN_INT64(0); } shardInterval = (ShardInterval *) linitial(shardIntervalList); } else if (IsCitusTableType(relationId, HASH_DISTRIBUTED) || IsCitusTableType(relationId, RANGE_DISTRIBUTED)) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); /* if given table is not reference table, distributionValue cannot be NULL */ if (PG_ARGISNULL(1)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("distribution value cannot be NULL for tables other " "than reference tables."))); } Datum inputDatum = PG_GETARG_DATUM(1); Oid inputDataType = get_fn_expr_argtype(fcinfo->flinfo, 1); char *distributionValueString = DatumToString(inputDatum, inputDataType); Var *distributionColumn = DistPartitionKeyOrError(relationId); Oid distributionDataType = distributionColumn->vartype; Datum distributionValueDatum = StringToDatum(distributionValueString, distributionDataType); shardInterval = FindShardInterval(distributionValueDatum, cacheEntry); } else { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("finding shard id of given distribution value is only " "supported for hash partitioned tables, range partitioned " "tables and reference tables."))); } if (shardInterval != NULL) { PG_RETURN_INT64(shardInterval->shardId); } PG_RETURN_INT64(0); } /* * citus_nodename_for_nodeid returns the node name for the node with given node id */ Datum citus_nodename_for_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int nodeId = PG_GETARG_INT32(0); WorkerNode *node = FindNodeAnyClusterByNodeId(nodeId); if (node == NULL) { PG_RETURN_NULL(); } PG_RETURN_TEXT_P(cstring_to_text(node->workerName)); } /* * citus_nodeport_for_nodeid returns the node port for the node with given node id */ Datum citus_nodeport_for_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int nodeId = PG_GETARG_INT32(0); WorkerNode *node = FindNodeAnyClusterByNodeId(nodeId); if (node == NULL) { PG_RETURN_NULL(); } PG_RETURN_INT32(node->workerPort); } /* * citus_coordinator_nodeid returns the node id of the coordinator node */ Datum citus_coordinator_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int coordinatorNodeId = FindCoordinatorNodeId(); if (coordinatorNodeId == -1) { PG_RETURN_INT32(0); } PG_RETURN_INT32(coordinatorNodeId); } /* * citus_is_coordinator returns whether the current node is a coordinator. * We consider the node a coordinator if its group ID is 0 and it has * pg_dist_node entries (only group ID 0 could indicate a worker without * metadata). */ Datum citus_is_coordinator(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); bool isCoordinator = false; if (GetLocalGroupId() == COORDINATOR_GROUP_ID && ActiveReadableNodeCount() > 0) { isCoordinator = true; } PG_RETURN_BOOL(isCoordinator); } /* * citus_is_primary_node returns whether the current node is a primary for * a given group_id. We consider the node a primary if it has * pg_dist_node entries marked as primary */ Datum citus_is_primary_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int32 groupId = GetLocalGroupId(); WorkerNode *workerNode = PrimaryNodeForGroup(groupId, NULL); if (workerNode == NULL) { ereport(WARNING, (errmsg("could not find the current node in pg_dist_node"), errdetail("If this is the coordinator node, consider adding it " "into the metadata by using citus_set_coordinator_host() " "UDF. Otherwise, if you're going to use this node as a " "worker node for a new cluster, make sure to add this " "node into the metadata from the coordinator by using " "citus_add_node() UDF."))); PG_RETURN_NULL(); } bool isPrimary = workerNode->nodeId == GetLocalNodeId(); PG_RETURN_BOOL(isPrimary); } /* * EnsureParentSessionHasExclusiveLockOnPgDistNode ensures given session id * holds Exclusive lock on pg_dist_node. */ static void EnsureParentSessionHasExclusiveLockOnPgDistNode(pid_t parentSessionPid) { StringInfo checkIfParentLockCommandStr = makeStringInfo(); int spiConnectionResult = SPI_connect(); if (spiConnectionResult != SPI_OK_CONNECT) { ereport(ERROR, (errmsg("could not connect to SPI manager"))); } char *checkIfParentLockCommand = "SELECT pid FROM pg_locks WHERE " "pid = %d AND database = %d AND relation = %d AND " "mode = 'ExclusiveLock' AND granted = TRUE"; appendStringInfo(checkIfParentLockCommandStr, checkIfParentLockCommand, parentSessionPid, MyDatabaseId, DistNodeRelationId()); bool readOnly = true; int spiQueryResult = SPI_execute(checkIfParentLockCommandStr->data, readOnly, 0); if (spiQueryResult != SPI_OK_SELECT) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", checkIfParentLockCommandStr->data))); } bool parentHasExclusiveLock = SPI_processed > 0; SPI_finish(); if (!parentHasExclusiveLock) { ereport(ERROR, (errmsg("lock is not held by the caller. Unexpected caller " "for citus_internal.mark_node_not_synced"))); } } /* * citus_internal_mark_node_not_synced unsets metadatasynced flag in separate connection * to localhost. Should only be called by `MarkNodesNotSyncedInLoopBackConnection`. * See it for details. */ Datum citus_internal_mark_node_not_synced(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* only called by superuser */ EnsureSuperUser(); pid_t parentSessionPid = PG_GETARG_INT32(0); /* fetch node by id */ int nodeId = PG_GETARG_INT32(1); HeapTuple heapTuple = GetNodeByNodeId(nodeId); /* ensure that parent session holds Exclusive lock to pg_dist_node */ EnsureParentSessionHasExclusiveLockOnPgDistNode(parentSessionPid); /* * We made sure parent session holds the ExclusiveLock, so we can unset * metadatasynced for the node safely with the relaxed lock here. */ Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc0(tupleDescriptor->natts * sizeof(bool)); values[Anum_pg_dist_node_metadatasynced - 1] = DatumGetBool(false); isnull[Anum_pg_dist_node_metadatasynced - 1] = false; replace[Anum_pg_dist_node_metadatasynced - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistNode, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistNodeRelationId()); CommandCounterIncrement(); table_close(pgDistNode, NoLock); pfree(values); pfree(isnull); pfree(replace); PG_RETURN_VOID(); } /* * FindWorkerNode searches over the worker nodes and returns the workerNode * if it already exists. Else, the function returns NULL. * * NOTE: A special case that this handles is when nodeName and nodePort are set * to LocalHostName and PostPortNumber. In that case we return the primary node * for the local group. */ WorkerNode * FindWorkerNode(const char *nodeName, int32 nodePort) { HTAB *workerNodeHash = GetWorkerNodeHash(); bool handleFound = false; WorkerNode *searchedNode = (WorkerNode *) palloc0(sizeof(WorkerNode)); strlcpy(searchedNode->workerName, nodeName, WORKER_LENGTH); searchedNode->workerPort = nodePort; void *hashKey = (void *) searchedNode; WorkerNode *cachedWorkerNode = (WorkerNode *) hash_search(workerNodeHash, hashKey, HASH_FIND, &handleFound); if (handleFound) { WorkerNode *workerNode = (WorkerNode *) palloc(sizeof(WorkerNode)); *workerNode = *cachedWorkerNode; return workerNode; } if (strcmp(LocalHostName, nodeName) == 0 && nodePort == PostPortNumber) { return PrimaryNodeForGroup(GetLocalGroupId(), NULL); } return NULL; } /* * FindWorkerNode searches over the worker nodes and returns the workerNode * if it exists otherwise it errors out. */ WorkerNode * FindWorkerNodeOrError(const char *nodeName, int32 nodePort) { WorkerNode *node = FindWorkerNode(nodeName, nodePort); if (node == NULL) { ereport(ERROR, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("node %s:%d not found", nodeName, nodePort))); } return node; } /* * FindWorkerNodeAnyCluster returns the workerNode no matter which cluster it is a part * of. FindWorkerNodes, like almost every other function, acts as if nodes in other * clusters do not exist. */ WorkerNode * FindWorkerNodeAnyCluster(const char *nodeName, int32 nodePort) { WorkerNode *workerNode = NULL; Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = GetNodeTuple(nodeName, nodePort); if (heapTuple != NULL) { workerNode = TupleToWorkerNode(pgDistNode, tupleDescriptor, heapTuple); } table_close(pgDistNode, NoLock); return workerNode; } /* * FindNodeAnyClusterByNodeId searches pg_dist_node and returns the node with * the nodeId. If the node can't be found returns NULL. */ WorkerNode * FindNodeAnyClusterByNodeId(uint32 nodeId) { bool includeNodesFromOtherClusters = true; List *nodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *node = NULL; foreach_declared_ptr(node, nodeList) { if (node->nodeId == nodeId) { return node; } } return NULL; } /* * FindNodeWithNodeId searches pg_dist_node and returns the node with the nodeId. * If the node cannot be found this functions errors. */ WorkerNode * FindNodeWithNodeId(int nodeId, bool missingOk) { List *nodeList = ActiveReadableNodeList(); WorkerNode *node = NULL; foreach_declared_ptr(node, nodeList) { if (node->nodeId == nodeId) { return node; } } /* there isn't any node with nodeId in pg_dist_node */ if (!missingOk) { elog(ERROR, "node with node id %d could not be found", nodeId); } return NULL; } /* * FindCoordinatorNodeId returns the node id of the coordinator node */ int FindCoordinatorNodeId() { bool includeNodesFromOtherClusters = false; List *nodeList = ReadDistNode(includeNodesFromOtherClusters); WorkerNode *node = NULL; foreach_declared_ptr(node, nodeList) { if (NodeIsCoordinator(node)) { return node->nodeId; } } return -1; } /* * ReadDistNode iterates over pg_dist_node table, converts each row * into its memory representation (i.e., WorkerNode) and adds them into * a list. Lastly, the list is returned to the caller. * * It skips nodes which are not in the current clusters unless requested to do otherwise * by includeNodesFromOtherClusters. */ List * ReadDistNode(bool includeNodesFromOtherClusters) { ScanKeyData scanKey[1]; int scanKeyCount = 0; List *workerNodeList = NIL; Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistNode, InvalidOid, false, NULL, scanKeyCount, scanKey); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { WorkerNode *workerNode = TupleToWorkerNode(pgDistNode, tupleDescriptor, heapTuple) ; if (includeNodesFromOtherClusters || strncmp(workerNode->nodeCluster, CurrentCluster, WORKER_LENGTH) == 0) { /* the coordinator acts as if it never sees nodes not in its cluster */ workerNodeList = lappend(workerNodeList, workerNode); } heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgDistNode, NoLock); return workerNodeList; } /* * RemoveNodeFromCluster removes the provided node from the pg_dist_node table of * the master node and all nodes with metadata. * The call to the master_remove_node should be done by the super user. If there are * active shard placements on the node; the function errors out. * This function also deletes all reference table placements belong to the given node from * pg_dist_placement, but it does not drop actual placement at the node. It also * modifies replication factor of the colocation group of reference tables, so that * replication factor will be equal to worker count. */ static void RemoveNodeFromCluster(char *nodeName, int32 nodePort) { WorkerNode *workerNode = ModifiableWorkerNode(nodeName, nodePort); /* * We do not allow metadata operations on secondary nodes in nontransactional * sync mode. */ if (NodeIsSecondary(workerNode)) { EnsureTransactionalMetadataSyncMode(); } if (NodeIsPrimary(workerNode)) { ErrorIfNodeContainsNonRemovablePlacements(workerNode); /* * Delete reference table placements so they are not taken into account * for the check if there are placements after this. */ bool localOnly = false; DeleteAllReplicatedTablePlacementsFromNodeGroup(workerNode->groupId, localOnly); /* * Secondary nodes are read-only, never 2PC is used. * Hence, no items can be inserted to pg_dist_transaction * for secondary nodes. */ DeleteWorkerTransactions(workerNode); } DeleteNodeRow(workerNode->workerName, nodePort); /* make sure we don't have any lingering session lifespan connections */ CloseNodeConnectionsAfterTransaction(workerNode->workerName, nodePort); if (EnableMetadataSync) { char *nodeDeleteCommand = NodeDeleteCommand(workerNode->nodeId); SendCommandToWorkersWithMetadata(nodeDeleteCommand); } } /* * ErrorIfNodeContainsNonRemovablePlacements throws an error if the input node * contains at least one placement on the node that is the last active * placement. */ static void ErrorIfNodeContainsNonRemovablePlacements(WorkerNode *workerNode) { int32 groupId = workerNode->groupId; List *shardPlacements = AllShardPlacementsOnNodeGroup(groupId); /* sort the list to prevent regression tests getting flaky */ shardPlacements = SortList(shardPlacements, CompareGroupShardPlacements); GroupShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacements) { if (!PlacementHasActivePlacementOnAnotherGroup(placement)) { Oid relationId = RelationIdForShard(placement->shardId); char *qualifiedRelationName = generate_qualified_relation_name(relationId); ereport(ERROR, (errmsg("cannot remove or disable the node " "%s:%d because because it contains " "the only shard placement for " "shard " UINT64_FORMAT, workerNode->workerName, workerNode->workerPort, placement->shardId), errdetail("One of the table(s) that prevents the operation " "complete successfully is %s", qualifiedRelationName), errhint("To proceed, either drop the tables or use " "undistribute_table() function to convert " "them to local tables"))); } } } /* * PlacementHasActivePlacementOnAnotherGroup returns true if there is at least * one more active placement of the input sourcePlacement on another group. */ static bool PlacementHasActivePlacementOnAnotherGroup(GroupShardPlacement *sourcePlacement) { uint64 shardId = sourcePlacement->shardId; List *activePlacementList = ActiveShardPlacementList(shardId); bool foundActivePlacementOnAnotherGroup = false; ShardPlacement *activePlacement = NULL; foreach_declared_ptr(activePlacement, activePlacementList) { if (activePlacement->groupId != sourcePlacement->groupId) { foundActivePlacementOnAnotherGroup = true; break; } } return foundActivePlacementOnAnotherGroup; } /* CountPrimariesWithMetadata returns the number of primary nodes which have metadata. */ uint32 CountPrimariesWithMetadata(void) { uint32 primariesWithMetadata = 0; WorkerNode *workerNode = NULL; HASH_SEQ_STATUS status; HTAB *workerNodeHash = GetWorkerNodeHash(); hash_seq_init(&status, workerNodeHash); while ((workerNode = hash_seq_search(&status)) != NULL) { if (workerNode->hasMetadata && NodeIsPrimary(workerNode)) { primariesWithMetadata++; } } return primariesWithMetadata; } /* * AddNodeMetadata checks the given node information and adds the specified node to the * pg_dist_node table of the master and workers with metadata. * If the node already exists, the function returns the id of the node. * If not, the following procedure is followed while adding a node: If the groupId is not * explicitly given by the user, the function picks the group that the new node should * be in with respect to GroupSize. Then, the new node is inserted into the local * pg_dist_node as well as the nodes with hasmetadata=true if localOnly is false. */ static int AddNodeMetadata(char *nodeName, int32 nodePort, NodeMetadata *nodeMetadata, bool *nodeAlreadyExists, bool localOnly) { EnsureCoordinator(); *nodeAlreadyExists = false; WorkerNode *workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort); if (workerNode != NULL) { /* return early without holding locks when the node already exists */ *nodeAlreadyExists = true; return workerNode->nodeId; } /* * We are going to change pg_dist_node, prevent any concurrent reads that * are not tolerant to concurrent node addition by taking an exclusive * lock (conflicts with all but AccessShareLock). * * We may want to relax or have more fine-grained locking in the future * to allow users to add multiple nodes concurrently. */ LockRelationOid(DistNodeRelationId(), ExclusiveLock); /* recheck in case 2 node additions pass the first check concurrently */ workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort); if (workerNode != NULL) { *nodeAlreadyExists = true; return workerNode->nodeId; } if (nodeMetadata->groupId != COORDINATOR_GROUP_ID && strcmp(nodeName, "localhost") != 0) { /* * User tries to add a worker with a non-localhost address. If the coordinator * is added with "localhost" as well, the worker won't be able to connect. */ bool isCoordinatorInMetadata = false; WorkerNode *coordinatorNode = PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &isCoordinatorInMetadata); if (isCoordinatorInMetadata && strcmp(coordinatorNode->workerName, "localhost") == 0) { ereport(ERROR, (errmsg("cannot add a worker node when the coordinator " "hostname is set to localhost"), errdetail("Worker nodes need to be able to connect to the " "coordinator to transfer data."), errhint("Use SELECT citus_set_coordinator_host('') " "to configure the coordinator hostname"))); } } /* * When adding the first worker when the coordinator has shard placements, * print a notice on how to drain the coordinator. */ if (nodeMetadata->groupId != COORDINATOR_GROUP_ID && CoordinatorAddedAsWorkerNode() && ActivePrimaryNonCoordinatorNodeCount() == 0 && NodeGroupHasShardPlacements(COORDINATOR_GROUP_ID)) { WorkerNode *coordinator = CoordinatorNodeIfAddedAsWorkerOrError(); ereport(NOTICE, (errmsg("shards are still on the coordinator after adding the " "new node"), errhint("Use SELECT rebalance_table_shards(); to balance " "shards data between workers and coordinator or " "SELECT citus_drain_node(%s,%d); to permanently " "move shards away from the coordinator.", quote_literal_cstr(coordinator->workerName), coordinator->workerPort))); } /* user lets Citus to decide on the group that the newly added node should be in */ if (nodeMetadata->groupId == INVALID_GROUP_ID) { nodeMetadata->groupId = GetNextGroupId(); } if (nodeMetadata->groupId == COORDINATOR_GROUP_ID) { /* * Coordinator has always the authoritative metadata, reflect this * fact in the pg_dist_node. */ nodeMetadata->hasMetadata = true; nodeMetadata->metadataSynced = true; /* * There is no concept of "inactive" coordinator, so hard code it. */ nodeMetadata->isActive = true; } /* if nodeRole hasn't been added yet there's a constraint for one-node-per-group */ if (nodeMetadata->nodeRole != InvalidOid && nodeMetadata->nodeRole == PrimaryNodeRoleId()) { WorkerNode *existingPrimaryNode = PrimaryNodeForGroup(nodeMetadata->groupId, NULL); if (existingPrimaryNode != NULL) { ereport(ERROR, (errmsg("group %d already has a primary node", nodeMetadata->groupId))); } } if (nodeMetadata->nodeRole == PrimaryNodeRoleId()) { if (strncmp(nodeMetadata->nodeCluster, WORKER_DEFAULT_CLUSTER, WORKER_LENGTH) != 0) { ereport(ERROR, (errmsg("primaries must be added to the default cluster"))); } } /* generate the new node id from the sequence */ int nextNodeIdInt = GetNextNodeId(); InsertNodeRow(nextNodeIdInt, nodeName, nodePort, nodeMetadata); workerNode = FindWorkerNodeAnyCluster(nodeName, nodePort); if (EnableMetadataSync && !localOnly) { /* send the delete command to all primary nodes with metadata */ char *nodeDeleteCommand = NodeDeleteCommand(workerNode->nodeId); SendCommandToWorkersWithMetadata(nodeDeleteCommand); /* finally prepare the insert command and send it to all primary nodes */ uint32 primariesWithMetadata = CountPrimariesWithMetadata(); if (primariesWithMetadata != 0) { List *workerNodeList = list_make1(workerNode); char *nodeInsertCommand = NodeListInsertCommand(workerNodeList); SendCommandToWorkersWithMetadata(nodeInsertCommand); } } return workerNode->nodeId; } /* * AddNodeMetadataViaMetadataContext does the same thing as AddNodeMetadata but * make use of metadata sync context to send commands to workers to support both * transactional and nontransactional sync modes. */ static int AddNodeMetadataViaMetadataContext(char *nodeName, int32 nodePort, NodeMetadata *nodeMetadata, bool *nodeAlreadyExists) { bool localOnly = true; int nodeId = AddNodeMetadata(nodeName, nodePort, nodeMetadata, nodeAlreadyExists, localOnly); /* do nothing as the node already exists */ if (*nodeAlreadyExists) { return nodeId; } /* * Create metadata sync context that is used throughout node addition * and activation if necessary. */ WorkerNode *node = ModifiableWorkerNode(nodeName, nodePort); /* we should always set active flag to true if we call citus_add_node */ node = SetWorkerColumnLocalOnly(node, Anum_pg_dist_node_isactive, DatumGetBool(true)); /* * After adding new node, if the node did not already exist, we will activate * the node. * If the worker is not marked as a coordinator, check that * the node is not trying to add itself */ if (node != NULL && node->groupId != COORDINATOR_GROUP_ID && node->nodeRole != SecondaryNodeRoleId() && IsWorkerTheCurrentNode(node)) { ereport(ERROR, (errmsg("Node cannot add itself as a worker."), errhint( "Add the node as a coordinator by using: " "SELECT citus_set_coordinator_host('%s', %d);", node->workerName, node->workerPort))); } List *nodeList = list_make1(node); bool collectCommands = false; bool nodesAddedInSameTransaction = true; MetadataSyncContext *context = CreateMetadataSyncContext(nodeList, collectCommands, nodesAddedInSameTransaction); if (EnableMetadataSync) { /* send the delete command to all primary nodes with metadata */ char *nodeDeleteCommand = NodeDeleteCommand(node->nodeId); SendOrCollectCommandListToMetadataNodes(context, list_make1(nodeDeleteCommand)); /* finally prepare the insert command and send it to all primary nodes */ uint32 primariesWithMetadata = CountPrimariesWithMetadata(); if (primariesWithMetadata != 0) { char *nodeInsertCommand = NULL; if (context->transactionMode == METADATA_SYNC_TRANSACTIONAL) { nodeInsertCommand = NodeListInsertCommand(nodeList); } else if (context->transactionMode == METADATA_SYNC_NON_TRANSACTIONAL) { /* * We need to ensure node insertion is idempotent in nontransactional * sync mode. */ nodeInsertCommand = NodeListIdempotentInsertCommand(nodeList); } Assert(nodeInsertCommand != NULL); SendOrCollectCommandListToMetadataNodes(context, list_make1(nodeInsertCommand)); } } ActivateNodeList(context); return nodeId; } /* * SetWorkerColumn function sets the column with the specified index * on the worker in pg_dist_node, by calling SetWorkerColumnLocalOnly. * It also sends the same command for node update to other metadata nodes. * If anything fails during the transaction, we rollback it. * Returns the new worker node after the modification. */ WorkerNode * SetWorkerColumn(WorkerNode *workerNode, int columnIndex, Datum value) { workerNode = SetWorkerColumnLocalOnly(workerNode, columnIndex, value); if (EnableMetadataSync) { char *metadataSyncCommand = GetMetadataSyncCommandToSetNodeColumn(workerNode, columnIndex, value); SendCommandToWorkersWithMetadata(metadataSyncCommand); } return workerNode; } /* * SetNodeStateViaMetadataContext sets or unsets isactive, metadatasynced, and hasmetadata * flags via metadataSyncContext. */ static void SetNodeStateViaMetadataContext(MetadataSyncContext *context, WorkerNode *workerNode, Datum value) { char *isActiveCommand = GetMetadataSyncCommandToSetNodeColumn(workerNode, Anum_pg_dist_node_isactive, value); char *metadatasyncedCommand = GetMetadataSyncCommandToSetNodeColumn(workerNode, Anum_pg_dist_node_metadatasynced, value); char *hasmetadataCommand = GetMetadataSyncCommandToSetNodeColumn(workerNode, Anum_pg_dist_node_hasmetadata, value); List *commandList = list_make3(isActiveCommand, metadatasyncedCommand, hasmetadataCommand); SendOrCollectCommandListToMetadataNodes(context, commandList); } /* * SetWorkerColumnOptional function sets the column with the specified index * on the worker in pg_dist_node, by calling SetWorkerColumnLocalOnly. * It also sends the same command optionally for node update to other metadata nodes, * meaning that failures are ignored. Returns the new worker node after the modification. */ WorkerNode * SetWorkerColumnOptional(WorkerNode *workerNode, int columnIndex, Datum value) { char *metadataSyncCommand = GetMetadataSyncCommandToSetNodeColumn(workerNode, columnIndex, value); List *workerNodeList = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, ShareLock); /* open connections in parallel */ WorkerNode *worker = NULL; foreach_declared_ptr(worker, workerNodeList) { bool success = SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction( worker->workerName, worker->workerPort, CurrentUserName(), list_make1(metadataSyncCommand)); if (!success) { /* metadata out of sync, mark the worker as not synced */ ereport(WARNING, (errmsg("Updating the metadata of the node %s:%d " "is failed on node %s:%d. " "Metadata on %s:%d is marked as out of sync.", workerNode->workerName, workerNode->workerPort, worker->workerName, worker->workerPort, worker->workerName, worker->workerPort))); SetWorkerColumnLocalOnly(worker, Anum_pg_dist_node_metadatasynced, BoolGetDatum(false)); } else if (workerNode->nodeId == worker->nodeId) { /* * If this is the node we want to update and it is updated succesfully, * then we can safely update the flag on the coordinator as well. */ SetWorkerColumnLocalOnly(workerNode, columnIndex, value); } } return FindWorkerNode(workerNode->workerName, workerNode->workerPort); } /* * SetWorkerColumnLocalOnly function sets the column with the specified index * (see pg_dist_node.h) on the worker in pg_dist_node. * It returns the new worker node after the modification. */ WorkerNode * SetWorkerColumnLocalOnly(WorkerNode *workerNode, int columnIndex, Datum value) { Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = GetNodeTuple(workerNode->workerName, workerNode->workerPort); Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc0(tupleDescriptor->natts * sizeof(bool)); if (heapTuple == NULL) { ereport(ERROR, (errmsg("could not find valid entry for node \"%s:%d\"", workerNode->workerName, workerNode->workerPort))); } values[columnIndex - 1] = value; isnull[columnIndex - 1] = false; replace[columnIndex - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdate(pgDistNode, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(DistNodeRelationId()); CommandCounterIncrement(); WorkerNode *newWorkerNode = TupleToWorkerNode(pgDistNode, tupleDescriptor, heapTuple); table_close(pgDistNode, NoLock); pfree(values); pfree(isnull); pfree(replace); return newWorkerNode; } /* * GetMetadataSyncCommandToSetNodeColumn checks if the given workerNode and value is * valid or not. Then it returns the necessary metadata sync command as a string. */ static char * GetMetadataSyncCommandToSetNodeColumn(WorkerNode *workerNode, int columnIndex, Datum value) { char *metadataSyncCommand = NULL; switch (columnIndex) { case Anum_pg_dist_node_hasmetadata: { ErrorIfCoordinatorMetadataSetFalse(workerNode, value, "hasmetadata"); metadataSyncCommand = NodeHasmetadataUpdateCommand(workerNode->nodeId, DatumGetBool(value)); break; } case Anum_pg_dist_node_isactive: { ErrorIfCoordinatorMetadataSetFalse(workerNode, value, "isactive"); metadataSyncCommand = NodeStateUpdateCommand(workerNode->nodeId, DatumGetBool(value)); break; } case Anum_pg_dist_node_shouldhaveshards: { metadataSyncCommand = ShouldHaveShardsUpdateCommand(workerNode->nodeId, DatumGetBool(value)); break; } case Anum_pg_dist_node_metadatasynced: { ErrorIfCoordinatorMetadataSetFalse(workerNode, value, "metadatasynced"); metadataSyncCommand = NodeMetadataSyncedUpdateCommand(workerNode->nodeId, DatumGetBool(value)); break; } default: { ereport(ERROR, (errmsg("could not find valid entry for node \"%s:%d\"", workerNode->workerName, workerNode->workerPort))); } } return metadataSyncCommand; } /* * NodeHasmetadataUpdateCommand generates and returns a SQL UPDATE command * that updates the hasmetada column of pg_dist_node, for the given nodeid. */ static char * NodeHasmetadataUpdateCommand(uint32 nodeId, bool hasMetadata) { StringInfo updateCommand = makeStringInfo(); char *hasMetadataString = hasMetadata ? "TRUE" : "FALSE"; appendStringInfo(updateCommand, "UPDATE pg_dist_node SET hasmetadata = %s " "WHERE nodeid = %u", hasMetadataString, nodeId); return updateCommand->data; } /* * NodeMetadataSyncedUpdateCommand generates and returns a SQL UPDATE command * that updates the metadataSynced column of pg_dist_node, for the given nodeid. */ static char * NodeMetadataSyncedUpdateCommand(uint32 nodeId, bool metadataSynced) { StringInfo updateCommand = makeStringInfo(); char *hasMetadataString = metadataSynced ? "TRUE" : "FALSE"; appendStringInfo(updateCommand, "UPDATE pg_dist_node SET metadatasynced = %s " "WHERE nodeid = %u", hasMetadataString, nodeId); return updateCommand->data; } /* * ErrorIfCoordinatorMetadataSetFalse throws an error if the input node * is the coordinator and the value is false. */ static void ErrorIfCoordinatorMetadataSetFalse(WorkerNode *workerNode, Datum value, char *field) { bool valueBool = DatumGetBool(value); if (!valueBool && workerNode->groupId == COORDINATOR_GROUP_ID) { ereport(ERROR, (errmsg("cannot change \"%s\" field of the " "coordinator node", field))); } } /* * SetShouldHaveShards function sets the shouldhaveshards column of the * specified worker in pg_dist_node. also propagates this to other metadata nodes. * It returns the new worker node after the modification. */ static WorkerNode * SetShouldHaveShards(WorkerNode *workerNode, bool shouldHaveShards) { return SetWorkerColumn(workerNode, Anum_pg_dist_node_shouldhaveshards, BoolGetDatum( shouldHaveShards)); } /* * GetNodeTuple function returns the heap tuple of given nodeName and nodePort. If the * node is not found this function returns NULL. * * This function may return worker nodes from other clusters. */ static HeapTuple GetNodeTuple(const char *nodeName, int32 nodePort) { Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); const int scanKeyCount = 2; const bool indexOK = false; ScanKeyData scanKey[2]; HeapTuple nodeTuple = NULL; ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodename, BTEqualStrategyNumber, F_TEXTEQ, CStringGetTextDatum(nodeName)); ScanKeyInit(&scanKey[1], Anum_pg_dist_node_nodeport, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(nodePort)); SysScanDesc scanDescriptor = systable_beginscan(pgDistNode, InvalidOid, indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { nodeTuple = heap_copytuple(heapTuple); } systable_endscan(scanDescriptor); table_close(pgDistNode, NoLock); return nodeTuple; } /* * GetNodeByNodeId returns the heap tuple for given node id by looking up catalog. */ static HeapTuple GetNodeByNodeId(int32 nodeId) { Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); const int scanKeyCount = 1; const bool indexOK = false; ScanKeyData scanKey[1]; HeapTuple nodeTuple = NULL; ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodeid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(nodeId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistNode, InvalidOid, indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { nodeTuple = heap_copytuple(heapTuple); } else { ereport(ERROR, (errmsg("could not find valid entry for node id %d", nodeId))); } systable_endscan(scanDescriptor); table_close(pgDistNode, NoLock); return nodeTuple; } /* * GetNextGroupId allocates and returns a unique groupId for the group * to be created. This allocation occurs both in shared memory and in write * ahead logs; writing to logs avoids the risk of having groupId collisions. * * Please note that the caller is still responsible for finalizing node data * and the groupId with the master node. Further note that this function relies * on an internal sequence created in initdb to generate unique identifiers. */ int32 GetNextGroupId() { text *sequenceName = cstring_to_text(GROUPID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); Oid savedUserId = InvalidOid; int savedSecurityContext = 0; GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); /* generate new and unique shardId from sequence */ Datum groupIdDatum = DirectFunctionCall1(nextval_oid, sequenceIdDatum); SetUserIdAndSecContext(savedUserId, savedSecurityContext); int32 groupId = DatumGetInt32(groupIdDatum); return groupId; } /* * GetNextNodeId allocates and returns a unique nodeId for the node * to be added. This allocation occurs both in shared memory and in write * ahead logs; writing to logs avoids the risk of having nodeId collisions. * * Please note that the caller is still responsible for finalizing node data * and the nodeId with the master node. Further note that this function relies * on an internal sequence created in initdb to generate unique identifiers. */ int GetNextNodeId() { text *sequenceName = cstring_to_text(NODEID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); Oid savedUserId = InvalidOid; int savedSecurityContext = 0; GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); /* generate new and unique shardId from sequence */ Datum nextNodeIdDatum = DirectFunctionCall1(nextval_oid, sequenceIdDatum); SetUserIdAndSecContext(savedUserId, savedSecurityContext); int nextNodeId = DatumGetUInt32(nextNodeIdDatum); return nextNodeId; } /* * EnsureCoordinator checks if the current node is the coordinator. If it does not, * the function errors out. */ void EnsureCoordinator(void) { int32 localGroupId = GetLocalGroupId(); if (localGroupId != 0) { ereport(ERROR, (errmsg("operation is not allowed on this node"), errhint("Connect to the coordinator and run it again."))); } } /* * EnsurePropagationToCoordinator checks whether the coordinator is added to the * metadata if we're not on the coordinator. * * Given that metadata syncing skips syncing metadata to the coordinator, we need * too make sure that the coordinator is added to the metadata before propagating * a command from a worker. For this reason, today we use this only for the commands * that we support propagating from workers. */ void EnsurePropagationToCoordinator(void) { if (!IsCoordinator()) { EnsureCoordinatorIsInMetadata(); } } /* * EnsureCoordinatorIsInMetadata checks whether the coordinator is added to the * metadata, which is required for many operations. */ void EnsureCoordinatorIsInMetadata(void) { bool isCoordinatorInMetadata = false; PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &isCoordinatorInMetadata); if (isCoordinatorInMetadata) { return; } /* be more descriptive when we're not on coordinator */ if (IsCoordinator()) { ereport(ERROR, (errmsg("coordinator is not added to the metadata"), errhint("Use SELECT citus_set_coordinator_host('') " "to configure the coordinator hostname"))); } else { ereport(ERROR, (errmsg("coordinator is not added to the metadata"), errhint("Use SELECT citus_set_coordinator_host('') " "on coordinator to configure the coordinator hostname"))); } } /* * InsertCoordinatorIfClusterEmpty can be used to ensure Citus tables can be * created even on a node that has just performed CREATE EXTENSION citus; */ void InsertCoordinatorIfClusterEmpty(void) { /* prevent concurrent node additions */ Relation pgDistNode = table_open(DistNodeRelationId(), RowShareLock); if (!HasAnyNodes()) { /* * create_distributed_table being called for the first time and there are * no pg_dist_node records. Add a record for the coordinator. */ InsertPlaceholderCoordinatorRecord(); } /* * We release the lock, if InsertPlaceholderCoordinatorRecord was called * we already have a strong (RowExclusive) lock. */ table_close(pgDistNode, RowShareLock); } /* * InsertPlaceholderCoordinatorRecord inserts a placeholder record for the coordinator * to be able to create distributed tables on a single node. */ static void InsertPlaceholderCoordinatorRecord(void) { NodeMetadata nodeMetadata = DefaultNodeMetadata(); nodeMetadata.groupId = 0; nodeMetadata.shouldHaveShards = true; nodeMetadata.nodeRole = PrimaryNodeRoleId(); nodeMetadata.nodeCluster = "default"; bool nodeAlreadyExists = false; bool localOnly = false; /* as long as there is a single node, localhost should be ok */ AddNodeMetadata(LocalHostName, PostPortNumber, &nodeMetadata, &nodeAlreadyExists, localOnly); } /* * InsertNodeRow opens the node system catalog, and inserts a new row with the * given values into that system catalog. * * NOTE: If you call this function you probably need to have taken a * ShareRowExclusiveLock then checked that you're not adding a second primary to * an existing group. If you don't it's possible for the metadata to become inconsistent. */ static void InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, NodeMetadata *nodeMetadata) { Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); Datum *values = palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isNulls = palloc0(tupleDescriptor->natts * sizeof(bool)); Datum nodeClusterStringDatum = CStringGetDatum(nodeMetadata->nodeCluster); Datum nodeClusterNameDatum = DirectFunctionCall1(namein, nodeClusterStringDatum); values[Anum_pg_dist_node_nodeid - 1] = UInt32GetDatum(nodeid); values[Anum_pg_dist_node_groupid - 1] = Int32GetDatum(nodeMetadata->groupId); values[Anum_pg_dist_node_nodename - 1] = CStringGetTextDatum(nodeName); values[Anum_pg_dist_node_nodeport - 1] = UInt32GetDatum(nodePort); values[Anum_pg_dist_node_noderack - 1] = CStringGetTextDatum(nodeMetadata->nodeRack); values[Anum_pg_dist_node_hasmetadata - 1] = BoolGetDatum(nodeMetadata->hasMetadata); values[Anum_pg_dist_node_metadatasynced - 1] = BoolGetDatum( nodeMetadata->metadataSynced); values[Anum_pg_dist_node_isactive - 1] = BoolGetDatum(nodeMetadata->isActive); values[Anum_pg_dist_node_noderole - 1] = ObjectIdGetDatum(nodeMetadata->nodeRole); values[Anum_pg_dist_node_nodecluster - 1] = nodeClusterNameDatum; values[Anum_pg_dist_node_shouldhaveshards - 1] = BoolGetDatum( nodeMetadata->shouldHaveShards); values[GetNodeIsCloneAttrIndexInPgDistNode(tupleDescriptor)] = BoolGetDatum(nodeMetadata->nodeisclone); values[GetNodePrimaryNodeIdAttrIndexInPgDistNode(tupleDescriptor)] = Int32GetDatum(nodeMetadata->nodeprimarynodeid); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); PushActiveSnapshot(GetTransactionSnapshot()); CatalogTupleInsert(pgDistNode, heapTuple); PopActiveSnapshot(); CitusInvalidateRelcacheByRelid(DistNodeRelationId()); /* increment the counter so that next command can see the row */ CommandCounterIncrement(); /* close relation */ table_close(pgDistNode, NoLock); pfree(values); pfree(isNulls); } /* * DeleteNodeRow removes the requested row from pg_dist_node table if it exists. */ static void DeleteNodeRow(char *nodeName, int32 nodePort) { const int scanKeyCount = 2; bool indexOK = false; ScanKeyData scanKey[2]; Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); /* * simple_heap_delete() expects that the caller has at least an * AccessShareLock on primary key index. * * XXX: This does not seem required, do we really need to acquire this lock? * Postgres doesn't acquire such locks on indexes before deleting catalog tuples. * Linking here the reasons we added this lock acquirement: * https://github.com/citusdata/citus/pull/2851#discussion_r306569462 * https://github.com/citusdata/citus/pull/2855#discussion_r313628554 * https://github.com/citusdata/citus/issues/1890 */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+ adds a bool “deferrable_ok” parameter */ Relation replicaIndex = index_open(RelationGetPrimaryKeyIndex(pgDistNode, false), AccessShareLock); #else Relation replicaIndex = index_open(RelationGetPrimaryKeyIndex(pgDistNode), AccessShareLock); #endif ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodename, BTEqualStrategyNumber, F_TEXTEQ, CStringGetTextDatum(nodeName)); ScanKeyInit(&scanKey[1], Anum_pg_dist_node_nodeport, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(nodePort)); SysScanDesc heapScan = systable_beginscan(pgDistNode, InvalidOid, indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(heapScan); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find valid entry for node \"%s:%d\"", nodeName, nodePort))); } simple_heap_delete(pgDistNode, &(heapTuple->t_self)); systable_endscan(heapScan); /* ensure future commands don't use the node we just removed */ CitusInvalidateRelcacheByRelid(DistNodeRelationId()); /* increment the counter so that next command won't see the row */ CommandCounterIncrement(); table_close(replicaIndex, AccessShareLock); table_close(pgDistNode, NoLock); } /* * TupleToWorkerNode takes in a heap tuple from pg_dist_node, and * converts this tuple to an equivalent struct in memory. The function assumes * the caller already has locks on the tuple, and doesn't perform any locking. */ static WorkerNode * TupleToWorkerNode(Relation pgDistNode, TupleDesc tupleDescriptor, HeapTuple heapTuple) { /* we add remove columns from pg_dist_node during extension upgrade and * and downgrads. Now the issue here is PostgreSQL never reuses the old * attnum. Dropped columns leave “holes” (attributes with attisdropped = true), * and a re-added column with the same name gets a new attnum at the end. So * we cannot use the deined Natts_pg_dist_node to allocate memory and also * we need to cater for the holes when fetching the column values */ int nAtts = tupleDescriptor->natts; Datum *datumArray = palloc0(sizeof(Datum) * nAtts); bool *isNullArray = palloc0(sizeof(bool) * nAtts); /* * We use heap_deform_tuple() instead of heap_getattr() to expand tuple * to contain missing values when ALTER TABLE ADD COLUMN happens. */ heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); char *nodeName = TextDatumGetCString(datumArray[Anum_pg_dist_node_nodename - 1]); char *nodeRack = TextDatumGetCString(datumArray[Anum_pg_dist_node_noderack - 1]); WorkerNode *workerNode = (WorkerNode *) palloc0(sizeof(WorkerNode)); workerNode->nodeId = DatumGetUInt32(datumArray[Anum_pg_dist_node_nodeid - 1]); workerNode->workerPort = DatumGetUInt32(datumArray[Anum_pg_dist_node_nodeport - 1]); workerNode->groupId = DatumGetInt32(datumArray[Anum_pg_dist_node_groupid - 1]); strlcpy(workerNode->workerName, nodeName, WORKER_LENGTH); strlcpy(workerNode->workerRack, nodeRack, WORKER_LENGTH); workerNode->hasMetadata = DatumGetBool(datumArray[Anum_pg_dist_node_hasmetadata - 1]); workerNode->metadataSynced = DatumGetBool(datumArray[Anum_pg_dist_node_metadatasynced - 1]); workerNode->isActive = DatumGetBool(datumArray[Anum_pg_dist_node_isactive - 1]); workerNode->nodeRole = DatumGetObjectId(datumArray[Anum_pg_dist_node_noderole - 1]); workerNode->shouldHaveShards = DatumGetBool( datumArray[Anum_pg_dist_node_shouldhaveshards - 1]); /* * nodecluster, nodeisclone and nodeprimarynodeid columns can be missing. In case * of extension creation/upgrade, master_initialize_node_metadata function is * called before the nodecluster column is added to pg_dist_node table. */ if (!isNullArray[Anum_pg_dist_node_nodecluster - 1]) { Name nodeClusterName = DatumGetName(datumArray[Anum_pg_dist_node_nodecluster - 1]); char *nodeClusterString = NameStr(*nodeClusterName); strlcpy(workerNode->nodeCluster, nodeClusterString, NAMEDATALEN); } int nodeIsCloneIdx = GetNodeIsCloneAttrIndexInPgDistNode(tupleDescriptor); int nodePrimaryNodeIdIdx = GetNodePrimaryNodeIdAttrIndexInPgDistNode(tupleDescriptor); if (!isNullArray[nodeIsCloneIdx]) { workerNode->nodeisclone = DatumGetBool(datumArray[nodeIsCloneIdx]); } if (!isNullArray[nodePrimaryNodeIdIdx]) { workerNode->nodeprimarynodeid = DatumGetInt32(datumArray[nodePrimaryNodeIdIdx]); } pfree(datumArray); pfree(isNullArray); return workerNode; } /* * GetNodePrimaryNodeIdAttrIndexInPgDistNode returns attrnum for nodeprimarynodeid attr. * * nodeprimarynodeid attr was added to table pg_dist_node using alter operation * after the version where Citus started supporting downgrades, and it's one of * the two columns that we've introduced to pg_dist_node since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_pg_dist_node and when this happens, then we know that attrnum * nodeprimarynodeid is not Anum_pg_dist_node_nodeprimarynodeid anymore but * tupleDesc->natts - 1. */ static int GetNodePrimaryNodeIdAttrIndexInPgDistNode(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_node ? (Anum_pg_dist_node_nodeprimarynodeid - 1) : tupleDesc->natts - 1; } /* * GetNodeIsCloneAttrIndexInPgDistNode returns attrnum for nodeisclone attr. * * Like, GetNodePrimaryNodeIdAttrIndexInPgDistNode(), performs a similar * calculation for nodeisclone attribute because this is column added to * pg_dist_node after we started supporting downgrades. * * Only difference with the mentioned function is that we know * the attrnum for nodeisclone is not Anum_pg_dist_node_nodeisclone anymore * but tupleDesc->natts - 2 because we added these columns consecutively * and we first add nodeisclone attribute and then nodeprimarynodeid attribute. */ static int GetNodeIsCloneAttrIndexInPgDistNode(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_node ? (Anum_pg_dist_node_nodeisclone - 1) : tupleDesc->natts - 2; } /* * StringToDatum transforms a string representation into a Datum. */ Datum StringToDatum(char *inputString, Oid dataType) { Oid typIoFunc = InvalidOid; Oid typIoParam = InvalidOid; int32 typeModifier = -1; getTypeInputInfo(dataType, &typIoFunc, &typIoParam); getBaseTypeAndTypmod(dataType, &typeModifier); Datum datum = OidInputFunctionCall(typIoFunc, inputString, typIoParam, typeModifier); return datum; } /* * DatumToString returns the string representation of the given datum. */ char * DatumToString(Datum datum, Oid dataType) { Oid typIoFunc = InvalidOid; bool typIsVarlena = false; getTypeOutputInfo(dataType, &typIoFunc, &typIsVarlena); char *outputString = OidOutputFunctionCall(typIoFunc, datum); return outputString; } /* * UnsetMetadataSyncedForAllWorkers sets the metadatasynced column of all metadata * worker nodes to false. It returns true if it updated at least a node. */ static bool UnsetMetadataSyncedForAllWorkers(void) { bool updatedAtLeastOne = false; ScanKeyData scanKey[3]; int scanKeyCount = 3; bool indexOK = false; /* * Concurrent citus_update_node() calls might iterate and try to update * pg_dist_node in different orders. To protect against deadlock, we * get an exclusive lock here. */ Relation relation = table_open(DistNodeRelationId(), ExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); ScanKeyInit(&scanKey[0], Anum_pg_dist_node_hasmetadata, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(true)); ScanKeyInit(&scanKey[1], Anum_pg_dist_node_metadatasynced, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(true)); /* coordinator always has the up to date metadata */ ScanKeyInit(&scanKey[2], Anum_pg_dist_node_groupid, BTGreaterStrategyNumber, F_INT4GT, Int32GetDatum(COORDINATOR_GROUP_ID)); CatalogIndexState indstate = CatalogOpenIndexes(relation); SysScanDesc scanDescriptor = systable_beginscan(relation, InvalidOid, indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { updatedAtLeastOne = true; } Datum *values = palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isnull = palloc(tupleDescriptor->natts * sizeof(bool)); bool *replace = palloc(tupleDescriptor->natts * sizeof(bool)); while (HeapTupleIsValid(heapTuple)) { memset(values, 0, tupleDescriptor->natts * sizeof(Datum)); memset(isnull, 0, tupleDescriptor->natts * sizeof(bool)); memset(replace, 0, tupleDescriptor->natts * sizeof(bool)); values[Anum_pg_dist_node_metadatasynced - 1] = BoolGetDatum(false); replace[Anum_pg_dist_node_metadatasynced - 1] = true; HeapTuple newHeapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace); CatalogTupleUpdateWithInfo(relation, &newHeapTuple->t_self, newHeapTuple, indstate); CommandCounterIncrement(); heap_freetuple(newHeapTuple); heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); CatalogCloseIndexes(indstate); table_close(relation, NoLock); pfree(values); pfree(isnull); pfree(replace); return updatedAtLeastOne; } /* * ErrorIfAnyNodeNotExist errors if any node in given list not found. */ static void ErrorIfAnyNodeNotExist(List *nodeList) { WorkerNode *node = NULL; foreach_declared_ptr(node, nodeList) { /* * First, locally mark the node is active, if everything goes well, * we are going to sync this information to all the metadata nodes. */ WorkerNode *workerNode = FindWorkerNodeAnyCluster(node->workerName, node->workerPort); if (workerNode == NULL) { ereport(ERROR, (errmsg("node at \"%s:%u\" does not exist", node->workerName, node->workerPort))); } } } /* * UpdateLocalGroupIdsViaMetadataContext updates local group ids for given list * of nodes with transactional or nontransactional mode according to transactionMode * inside metadataSyncContext. */ static void UpdateLocalGroupIdsViaMetadataContext(MetadataSyncContext *context) { int activatedPrimaryCount = list_length(context->activatedWorkerNodeList); int nodeIdx = 0; for (nodeIdx = 0; nodeIdx < activatedPrimaryCount; nodeIdx++) { WorkerNode *node = list_nth(context->activatedWorkerNodeList, nodeIdx); List *commandList = list_make1(LocalGroupIdUpdateCommand(node->groupId)); /* send commands to new workers, the current user should be a superuser */ Assert(superuser()); SendOrCollectCommandListToSingleNode(context, commandList, nodeIdx); } } /* * SendDeletionCommandsForReplicatedTablePlacements sends commands to delete replicated * placement for the metadata nodes with transactional or nontransactional mode according * to transactionMode inside metadataSyncContext. */ static void SendDeletionCommandsForReplicatedTablePlacements(MetadataSyncContext *context) { WorkerNode *node = NULL; foreach_declared_ptr(node, context->activatedWorkerNodeList) { if (!node->isActive) { bool localOnly = false; int32 groupId = node->groupId; DeleteAllReplicatedTablePlacementsFromNodeGroupViaMetadataContext(context, groupId, localOnly); } } } /* * SyncNodeMetadata syncs node metadata with transactional or nontransactional * mode according to transactionMode inside metadataSyncContext. */ static void SyncNodeMetadata(MetadataSyncContext *context) { CheckCitusVersion(ERROR); if (!EnableMetadataSync) { return; } /* * Do not fail when we call this method from activate_node_snapshot * from workers. */ if (!MetadataSyncCollectsCommands(context)) { EnsureCoordinator(); } EnsureModificationsCanRun(); EnsureSequentialModeMetadataOperations(); LockRelationOid(DistNodeRelationId(), ExclusiveLock); /* generate the queries which drop the node metadata */ List *dropMetadataCommandList = NodeMetadataDropCommands(); /* generate the queries which create the node metadata from scratch */ List *createMetadataCommandList = NodeMetadataCreateCommands(); List *recreateNodeSnapshotCommandList = dropMetadataCommandList; recreateNodeSnapshotCommandList = list_concat(recreateNodeSnapshotCommandList, createMetadataCommandList); /* * We should have already added node metadata to metadata workers. Sync node * metadata just for activated workers. */ SendOrCollectCommandListToActivatedNodes(context, recreateNodeSnapshotCommandList); } ================================================ FILE: src/backend/distributed/metadata/pg_get_object_address_16_17_18.c ================================================ /*------------------------------------------------------------------------- * * pg_get_object_address_16_17_18.c * * Copied functions from Postgres pg_get_object_address with acl/owner check. * Since we need to use intermediate data types Relation and Node from * the pg_get_object_address, we've copied that function from PG code and * added required owner/acl checks for our own purposes. * * We need to make sure that function works with future PG versions. Update * the function name according to supported PG versions as well. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/objectaddress.h" #include "catalog/pg_type.h" #include "mb/pg_wchar.h" #include "nodes/value.h" #include "parser/parse_type.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/varlena.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/version_compat.h" static void ErrorIfCurrentUserCanNotDistributeObject(char *textType, ObjectType type, ObjectAddress *addr, Node *node, Relation *relation); static List * textarray_to_strvaluelist(ArrayType *arr); /* * PgGetObjectAddress gets the object address. This function is mostly copied from * pg_get_object_address of the PG code. We need to copy that function to use * intermediate data types Relation and Node to check acl or ownership. * * Codes added by Citus are tagged with CITUS CODE BEGIN/END. */ ObjectAddress PgGetObjectAddress(char *ttype, ArrayType *namearr, ArrayType *argsarr) { List *name = NIL; TypeName *typename = NULL; List *args = NIL; Node *objnode = NULL; Relation relation; /* Decode object type, raise error if unknown */ int itype = read_objtype_from_string(ttype); if (itype < 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unsupported object type \"%s\"", ttype))); } ObjectType type = (ObjectType) itype; /* * Convert the text array to the representation appropriate for the given * object type. Most use a simple string Values list, but there are some * exceptions. */ if (type == OBJECT_TYPE || type == OBJECT_DOMAIN || type == OBJECT_CAST || type == OBJECT_TRANSFORM || type == OBJECT_DOMCONSTRAINT) { Datum *elems; bool *nulls; int nelems; deconstruct_array(namearr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); if (nelems != 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be exactly %d", 1))); } if (nulls[0]) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name or argument lists may not contain nulls"))); } typename = typeStringToTypeName(TextDatumGetCString(elems[0]), NULL); } else if (type == OBJECT_LARGEOBJECT) { Datum *elems; bool *nulls; int nelems; deconstruct_array(namearr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); if (nelems != 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be exactly %d", 1))); } if (nulls[0]) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("large object OID may not be null"))); } objnode = (Node *) makeFloat(TextDatumGetCString(elems[0])); } else { name = textarray_to_strvaluelist(namearr); if (list_length(name) < 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be at least %d", 1))); } } /* * If args are given, decode them according to the object type. */ if (type == OBJECT_AGGREGATE || type == OBJECT_FUNCTION || type == OBJECT_PROCEDURE || type == OBJECT_ROUTINE || type == OBJECT_OPERATOR || type == OBJECT_CAST || type == OBJECT_AMOP || type == OBJECT_AMPROC) { /* in these cases, the args list must be of TypeName */ Datum *elems; bool *nulls; int nelems; int i; deconstruct_array(argsarr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); args = NIL; for (i = 0; i < nelems; i++) { if (nulls[i]) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name or argument lists may not contain nulls"))); } args = lappend(args, typeStringToTypeName(TextDatumGetCString(elems[i]), NULL)); } } else { /* For all other object types, use string Values */ args = textarray_to_strvaluelist(argsarr); } /* * get_object_address is pretty sensitive to the length of its input * lists; check that they're what it wants. */ switch (type) { case OBJECT_DOMCONSTRAINT: case OBJECT_CAST: case OBJECT_USER_MAPPING: case OBJECT_PUBLICATION_REL: case OBJECT_DEFACL: case OBJECT_TRANSFORM: { if (list_length(args) != 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument list length must be exactly %d", 1))); } break; } case OBJECT_OPFAMILY: case OBJECT_OPCLASS: { if (list_length(name) < 2) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be at least %d", 2))); } break; } case OBJECT_AMOP: case OBJECT_AMPROC: { if (list_length(name) < 3) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be at least %d", 3))); } if (list_length(args) != 2) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument list length must be exactly %d", 2))); } break; } case OBJECT_OPERATOR: { if (list_length(args) != 2) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("argument list length must be exactly %d", 2))); } break; } default: { break; } } /* * Now build the Node type that get_object_address() expects for the given * type. */ switch (type) { case OBJECT_TABLE: case OBJECT_SEQUENCE: case OBJECT_VIEW: case OBJECT_MATVIEW: case OBJECT_INDEX: case OBJECT_FOREIGN_TABLE: case OBJECT_COLUMN: case OBJECT_ATTRIBUTE: case OBJECT_COLLATION: case OBJECT_CONVERSION: case OBJECT_STATISTIC_EXT: case OBJECT_TSPARSER: case OBJECT_TSDICTIONARY: case OBJECT_TSTEMPLATE: case OBJECT_TSCONFIGURATION: case OBJECT_DEFAULT: case OBJECT_POLICY: case OBJECT_RULE: case OBJECT_TRIGGER: case OBJECT_TABCONSTRAINT: case OBJECT_OPCLASS: case OBJECT_OPFAMILY: { objnode = (Node *) name; break; } case OBJECT_ACCESS_METHOD: case OBJECT_DATABASE: case OBJECT_EVENT_TRIGGER: case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: case OBJECT_LANGUAGE: case OBJECT_PARAMETER_ACL: case OBJECT_PUBLICATION: case OBJECT_ROLE: case OBJECT_SCHEMA: case OBJECT_SUBSCRIPTION: case OBJECT_TABLESPACE: { if (list_length(name) != 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name list length must be exactly %d", 1))); } objnode = linitial(name); break; } case OBJECT_TYPE: case OBJECT_DOMAIN: { objnode = (Node *) typename; break; } case OBJECT_CAST: case OBJECT_DOMCONSTRAINT: case OBJECT_TRANSFORM: { objnode = (Node *) list_make2(typename, linitial(args)); break; } case OBJECT_PUBLICATION_REL: { objnode = (Node *) list_make2(name, linitial(args)); break; } case OBJECT_PUBLICATION_NAMESPACE: case OBJECT_USER_MAPPING: { objnode = (Node *) list_make2(linitial(name), linitial(args)); break; } case OBJECT_DEFACL: { objnode = (Node *) lcons(linitial(args), name); break; } case OBJECT_AMOP: case OBJECT_AMPROC: { objnode = (Node *) list_make2(name, args); break; } case OBJECT_FUNCTION: case OBJECT_PROCEDURE: case OBJECT_ROUTINE: case OBJECT_AGGREGATE: case OBJECT_OPERATOR: { ObjectWithArgs *owa = makeNode(ObjectWithArgs); owa->objname = name; owa->objargs = args; objnode = (Node *) owa; break; } case OBJECT_LARGEOBJECT: { /* already handled above */ break; } /* no default, to let compiler warn about missing case */ } if (objnode == NULL) { elog(ERROR, "unrecognized object type: %d", type); } ObjectAddress addr = get_object_address(type, objnode, &relation, AccessShareLock, false); /* CITUS CODE BEGIN */ ErrorIfCurrentUserCanNotDistributeObject(ttype, type, &addr, objnode, &relation); /* CITUS CODE END */ /* We don't need the relcache entry, thank you very much */ if (relation) { relation_close(relation, AccessShareLock); } /* CITUS CODE BEGIN */ return addr; /* CITUS CODE END */ } /* * ErrorIfCurrentUserCanNotDistributeObject checks whether current user can * distribute object, if not errors out. */ static void ErrorIfCurrentUserCanNotDistributeObject(char *textType, ObjectType type, ObjectAddress *addr, Node *node, Relation *relation) { Oid userId = GetUserId(); if (!SupportedDependencyByCitus(addr)) { ereport(ERROR, (errmsg("%s object can not be distributed by Citus", textType), errdetail("Object type id is %d", type))); } switch (type) { case OBJECT_SCHEMA: case OBJECT_DATABASE: case OBJECT_FUNCTION: case OBJECT_PROCEDURE: case OBJECT_AGGREGATE: case OBJECT_TSCONFIGURATION: case OBJECT_TSDICTIONARY: case OBJECT_TYPE: case OBJECT_FOREIGN_SERVER: case OBJECT_SEQUENCE: case OBJECT_FOREIGN_TABLE: case OBJECT_TABLE: case OBJECT_EXTENSION: case OBJECT_COLLATION: case OBJECT_VIEW: case OBJECT_ROLE: case OBJECT_PUBLICATION: { check_object_ownership(userId, type, *addr, node, *relation); break; } default: { ereport(ERROR, (errmsg("%d object type is not supported within " "object propagation", type))); break; } } } /* * Copied from PG code. * * Convert an array of TEXT into a List of string Values, as emitted by the * parser, which is what get_object_address uses as input. */ static List * textarray_to_strvaluelist(ArrayType *arr) { Datum *elems; bool *nulls; int nelems; List *list = NIL; int i; deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); for (i = 0; i < nelems; i++) { if (nulls[i]) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("name or argument lists may not contain nulls"))); } list = lappend(list, makeString(TextDatumGetCString(elems[i]))); } return list; } ================================================ FILE: src/backend/distributed/operations/citus_create_restore_point.c ================================================ /*------------------------------------------------------------------------- * * citus_create_restore_point.c * * UDF for creating a consistent restore point across all nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "access/xlog.h" #include "access/xlog_internal.h" #include "catalog/pg_type.h" #include "nodes/pg_list.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/remote_commands.h" #define CREATE_RESTORE_POINT_COMMAND "SELECT pg_catalog.pg_create_restore_point($1::text)" /* * BLOCK_DISTRIBUTED_WRITES_COMMAND acquires ExclusiveLock on: * 1. pg_dist_transaction - blocks 2PC commit decisions * 2. pg_dist_partition - blocks DDL operations on distributed tables * * This ensures both DML (via 2PC) and DDL are blocked on metadata nodes. */ #define BLOCK_DISTRIBUTED_WRITES_COMMAND \ "LOCK TABLE pg_catalog.pg_dist_transaction IN EXCLUSIVE MODE; " \ "LOCK TABLE pg_catalog.pg_dist_partition IN EXCLUSIVE MODE" /* local functions forward declarations */ static List * OpenConnectionsToAllWorkerNodes(LOCKMODE lockMode); static void BlockDistributedTransactions(void); static void CreateRemoteRestorePoints(char *restoreName, List *connectionList); static void BlockDistributedTransactionsOnAllMetadataNodes(List *connectionList); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_create_restore_point); /* * citus_create_restore_point creates a cluster-consistent restore point * across all nodes in the Citus cluster. * * In coordinator-only mode, this function blocks new distributed writes * at the coordinator and creates restore points on all worker nodes. * * In MX mode (multi-writer), this function blocks both DML and DDL * operations on all metadata nodes by acquiring ExclusiveLock on: * - pg_dist_transaction: blocks 2PC commit decisions (DML) * - pg_dist_partition: blocks DDL on distributed tables * * This prevents new distributed transactions from recording commit decisions * and blocks schema changes, ensuring all restore points represent the same * consistent cluster state. * * The function returns the LSN of the restore point on the coordinator, * maintaining backward compatibility with the original implementation. * * Key insight: We do NOT need to drain in-flight transactions. The commit * decision in Citus 2PC happens when LogTransactionRecord() writes to * pg_dist_transaction, which occurs BEFORE the writer's local commit. * By blocking writes to pg_dist_transaction, we prevent commit decisions * from being made. Transactions that have already recorded their commit * decision will complete normally, while those that haven't will * be blocked. This creates a clean cut point for consistency. */ Datum citus_create_restore_point(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); EnsureCoordinator(); text *restoreNameText = PG_GETARG_TEXT_P(0); if (RecoveryInProgress()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), (errmsg("recovery is in progress"), errhint("WAL control functions cannot be executed during recovery.")))); } if (!XLogIsNeeded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("WAL level not sufficient for creating a restore point"), errhint("wal_level must be set to \"replica\" or \"logical\" at server " "start."))); } char *restoreNameString = text_to_cstring(restoreNameText); if (strlen(restoreNameString) >= MAXFNAMELEN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("value too long for restore point (maximum %d characters)", MAXFNAMELEN - 1))); } /* * establish connections to all nodes before taking any locks * ShareLock prevents new nodes being added, rendering connectionList incomplete */ List *connectionList = OpenConnectionsToAllWorkerNodes(ShareLock); XLogRecPtr localRestorePoint = InvalidXLogRecPtr; PG_TRY(); { /* * Send a BEGIN to bust through pgbouncer. We won't actually commit since * that takes time. Instead we just close the connections and roll back, * which doesn't undo pg_create_restore_point. */ RemoteTransactionListBegin(connectionList); /* DANGER: finish as quickly as possible after this */ BlockDistributedTransactions(); BlockDistributedTransactionsOnAllMetadataNodes(connectionList); /* do local restore point first to bail out early if something goes wrong */ localRestorePoint = XLogRestorePoint(restoreNameString); /* run pg_create_restore_point on all nodes */ CreateRemoteRestorePoints(restoreNameString, connectionList); /* close connections to all nodes and * all locks gets released as part of the transaction rollback */ MultiConnection *conn = NULL; foreach_declared_ptr(conn, connectionList) { ForgetResults(conn); CloseConnection(conn); } connectionList = NIL; } PG_CATCH(); { /* * On error, ensure we clean up connections and release locks. * Rolling back the metadata node transactions releases the * ExclusiveLocks on pg_dist_transaction cluster-wide. */ MultiConnection *conn = NULL; foreach_declared_ptr(conn, connectionList) { ForgetResults(conn); CloseConnection(conn); } connectionList = NIL; PG_RE_THROW(); } PG_END_TRY(); PG_RETURN_LSN(localRestorePoint); } /* * OpenConnectionsToAllNodes opens connections to all nodes and returns the list * of connections. */ static List * OpenConnectionsToAllWorkerNodes(LOCKMODE lockMode) { List *connectionList = NIL; int connectionFlags = FORCE_NEW_CONNECTION; List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(lockMode); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { MultiConnection *connection = StartNodeConnection(connectionFlags, workerNode->workerName, workerNode->workerPort); MarkRemoteTransactionCritical(connection); connectionList = lappend(connectionList, connection); } FinishConnectionListEstablishment(connectionList); return connectionList; } /* * BlockDistributedTransactions blocks distributed transactions that use 2PC * and changes to pg_dist_node (e.g. node addition) and pg_dist_partition * (table creation). */ static void BlockDistributedTransactions(void) { LockRelationOid(DistNodeRelationId(), ExclusiveLock); LockRelationOid(DistPartitionRelationId(), ExclusiveLock); LockRelationOid(DistTransactionRelationId(), ExclusiveLock); } /* * BlockDistributedTransactionsOnAllMetadataNodes blocks distributed transactions * on all metadata nodes by executing pg_lock_table remotely. * * This is the MX-mode equivalent of BlockDistributedTransactions(), extended * to all nodes capable of initiating distributed transactions. We must hold * these locks across the cluster to prevent commit decisions from being made * on any node. * * The function expects that connections are already in a transaction block * (BEGIN has been sent). The locks will be held until the transaction is * rolled back or committed. */ static void BlockDistributedTransactionsOnAllMetadataNodes(List *connectionList) { /* * Send LOCK TABLE commands to all metadata nodes in parallel. We use * standard SQL LOCK TABLE syntax to acquire ExclusiveLock on catalog * tables, mirroring what BlockDistributedTransactions() does on the * coordinator via LockRelationOid(). * * The BLOCK_DISTRIBUTED_WRITES_COMMAND acquires: * 1. ExclusiveLock on pg_dist_transaction (blocks 2PC commit decisions) * 2. ExclusiveLock on pg_dist_partition (blocks DDL on distributed tables) * * Note: Unlike the local coordinator lock which also locks pg_dist_node, * we don't lock pg_dist_node on remote nodes because node management * operations (adding/removing nodes) are still coordinator-only. * * These locks naturally serialize concurrent restore point operations * cluster-wide, so no additional advisory lock is needed. */ /* Build list of remote metadata node connections */ List *metadataConnectionList = NIL; MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { WorkerNode *workerNode = FindWorkerNode(connection->hostname, connection->port); bool isRemoteMetadataNode = workerNode != NULL && NodeIsPrimaryAndRemote(workerNode); if (isRemoteMetadataNode) { metadataConnectionList = lappend(metadataConnectionList, connection); } } /* Send lock commands in parallel to all remote metadata nodes */ foreach_declared_ptr(connection, metadataConnectionList) { /* * We could use ExecuteCriticalRemoteCommand instead, but it would * not allow us to execute the commands in parallel. So for sake of * performance, we use SendRemoteCommand and send lock commands in parallel * to all metadata nodes, and later wait for all lock acquisitions to complete. */ int querySent = SendRemoteCommand(connection, BLOCK_DISTRIBUTED_WRITES_COMMAND); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } /* * Wait for all lock acquisitions to complete. If any node fails to * acquire locks (e.g., due to a conflicting lock), this will error out. */ foreach_declared_ptr(connection, metadataConnectionList) { PGresult *result = GetRemoteCommandResult(connection, true); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ForgetResults(connection); } } /* * CreateRemoteRestorePoints creates a restore point via each of the * connections in the list in parallel. */ static void CreateRemoteRestorePoints(char *restoreName, List *connectionList) { int parameterCount = 1; Oid parameterTypes[1] = { TEXTOID }; const char *parameterValues[1] = { restoreName }; MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommandParams(connection, CREATE_RESTORE_POINT_COMMAND, parameterCount, parameterTypes, parameterValues, false); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } foreach_declared_ptr(connection, connectionList) { PGresult *result = GetRemoteCommandResult(connection, true); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ForgetResults(connection); } } ================================================ FILE: src/backend/distributed/operations/citus_split_shard_by_split_points.c ================================================ /*------------------------------------------------------------------------- * * citus_split_shard_by_split_points.c * * This file contains functions to split a shard. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/shard_split.h" #include "distributed/shardinterval_utils.h" #include "distributed/utils/array_type.h" #include "distributed/utils/distribution_column_map.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(citus_split_shard_by_split_points); /* * citus_split_shard_by_split_points(shard_id bigint, split_points text[], node_ids integer[], shard_transfer_mode citus.shard_transfer_mode) * Split source shard into multiple shards using the given split points. * 'shard_id' is the id of source shard to split. * 'split_points' is an array that represents the split points. * 'node_ids' is an array that represents the placement node ids of the new shards. * 'shard_transfer_mode citus.shard_transfer_mode' is the transfer mode for split. */ Datum citus_split_shard_by_split_points(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); uint64 shardIdToSplit = DatumGetUInt64(PG_GETARG_DATUM(0)); ArrayType *splitPointsArrayObject = PG_GETARG_ARRAYTYPE_P(1); List *shardSplitPointsList = TextArrayTypeToIntegerList(splitPointsArrayObject); ArrayType *nodeIdsArrayObject = PG_GETARG_ARRAYTYPE_P(2); List *nodeIdsForPlacementList = IntegerArrayTypeToList(nodeIdsArrayObject); Oid shardTransferModeOid = PG_GETARG_OID(3); SplitMode shardSplitMode = LookupSplitMode(shardTransferModeOid); DistributionColumnMap *distributionColumnOverrides = NULL; List *sourceColocatedShardIntervalList = NIL; SplitShard( shardSplitMode, SHARD_SPLIT_API, shardIdToSplit, shardSplitPointsList, nodeIdsForPlacementList, distributionColumnOverrides, sourceColocatedShardIntervalList, INVALID_COLOCATION_ID); PG_RETURN_VOID(); } /* * LookupSplitMode maps the oids of citus.shard_transfer_mode to SplitMode enum. */ SplitMode LookupSplitMode(Oid shardTransferModeOid) { SplitMode shardSplitMode = BLOCKING_SPLIT; Datum enumLabelDatum = DirectFunctionCall1(enum_out, shardTransferModeOid); char *enumLabel = DatumGetCString(enumLabelDatum); /* Extend with other modes as we support them */ if (strncmp(enumLabel, "block_writes", NAMEDATALEN) == 0) { shardSplitMode = BLOCKING_SPLIT; } else if (strncmp(enumLabel, "force_logical", NAMEDATALEN) == 0) { shardSplitMode = NON_BLOCKING_SPLIT; } else if (strncmp(enumLabel, "auto", NAMEDATALEN) == 0) { shardSplitMode = AUTO_SPLIT; } else { /* We will not get here as postgres will validate the enum value. */ ereport(ERROR, (errmsg( "Invalid shard tranfer mode: '%s'. Expected split mode is 'block_writes/auto/force_logical'.", enumLabel))); } return shardSplitMode; } ================================================ FILE: src/backend/distributed/operations/citus_tools.c ================================================ /*------------------------------------------------------------------------- * * citus_tools.c * UDF to run multi shard/worker queries * * This file contains functions to run commands on other worker/shards. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/htup_details.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "utils/builtins.h" #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/metadata_cache.h" #include "distributed/multi_server_executor.h" #include "distributed/remote_commands.h" #include "distributed/utils/array_type.h" #include "distributed/utils/function.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" PG_FUNCTION_INFO_V1(master_run_on_worker); static int ParseCommandParameters(FunctionCallInfo fcinfo, StringInfo **nodeNameArray, int **nodePortsArray, StringInfo **commandStringArray, bool *parallel); static void ExecuteCommandsInParallelAndStoreResults(StringInfo *nodeNameArray, int *nodePortArray, StringInfo *commandStringArray, bool *statusArray, StringInfo *resultStringArray, int commandCount); static bool GetConnectionStatusAndResult(MultiConnection *connection, bool *resultStatus, StringInfo queryResultString); static void ExecuteCommandsAndStoreResults(StringInfo *nodeNameArray, int *nodePortArray, StringInfo *commandStringArray, bool *statusArray, StringInfo *resultStringArray, int commandCount); static bool ExecuteOptionalSingleResultCommand(MultiConnection *connection, char *queryString, StringInfo queryResultString); static Tuplestorestate * CreateTupleStore(TupleDesc tupleDescriptor, StringInfo *nodeNameArray, int *nodePortArray, bool *statusArray, StringInfo *resultArray, int commandCount); /* * master_run_on_worker executes queries/commands to run on specified worker and * returns success status and query/command result. Expected input is 3 arrays * containing node names, node ports, and query strings, and boolean flag to specify * parallel execution. The function then returns node_name, node_port, success, * result tuples upon completion of the query. The same user credentials are used * to connect to remote nodes. */ Datum master_run_on_worker(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; bool parallelExecution = false; StringInfo *nodeNameArray = NULL; int *nodePortArray = NULL; StringInfo *commandStringArray = NULL; /* check to see if caller supports us returning a tuplestore */ if (!rsinfo || !(rsinfo->allowedModes & SFRM_Materialize)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("materialize mode required, but it is not " "allowed in this context"))); } int commandCount = ParseCommandParameters(fcinfo, &nodeNameArray, &nodePortArray, &commandStringArray, ¶llelExecution); MemoryContext per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; MemoryContext oldcontext = MemoryContextSwitchTo(per_query_ctx); /* get the requested return tuple description */ TupleDesc tupleDescriptor = CreateTupleDescCopy(rsinfo->expectedDesc); /* * Check to make sure we have correct tuple descriptor */ if (tupleDescriptor->natts != 4 || TupleDescAttr(tupleDescriptor, 0)->atttypid != TEXTOID || TupleDescAttr(tupleDescriptor, 1)->atttypid != INT4OID || TupleDescAttr(tupleDescriptor, 2)->atttypid != BOOLOID || TupleDescAttr(tupleDescriptor, 3)->atttypid != TEXTOID) { ereport(ERROR, (errcode(ERRCODE_INVALID_COLUMN_DEFINITION), errmsg("query-specified return tuple and " "function return type are not compatible"))); } /* * prepare storage for status and result values. * commandCount is based on user input however, it is the length of list * instead of a user given integer, hence this should be safe here in terms * of memory allocation. */ bool *statusArray = palloc0(commandCount * sizeof(bool)); StringInfo *resultArray = palloc0(commandCount * sizeof(StringInfo)); for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { resultArray[commandIndex] = makeStringInfo(); } if (parallelExecution) { ExecuteCommandsInParallelAndStoreResults(nodeNameArray, nodePortArray, commandStringArray, statusArray, resultArray, commandCount); } else { ExecuteCommandsAndStoreResults(nodeNameArray, nodePortArray, commandStringArray, statusArray, resultArray, commandCount); } /* let the caller know we're sending back a tuplestore */ rsinfo->returnMode = SFRM_Materialize; Tuplestorestate *tupleStore = CreateTupleStore(tupleDescriptor, nodeNameArray, nodePortArray, statusArray, resultArray, commandCount); rsinfo->setResult = tupleStore; rsinfo->setDesc = tupleDescriptor; MemoryContextSwitchTo(oldcontext); PG_RETURN_VOID(); } /* ParseCommandParameters reads call parameters and fills in data structures */ static int ParseCommandParameters(FunctionCallInfo fcinfo, StringInfo **nodeNameArray, int **nodePortsArray, StringInfo **commandStringArray, bool *parallel) { ArrayType *nodeNameArrayObject = PG_GETARG_ARRAYTYPE_P(0); ArrayType *nodePortArrayObject = PG_GETARG_ARRAYTYPE_P(1); ArrayType *commandStringArrayObject = PG_GETARG_ARRAYTYPE_P(2); bool parallelExecution = PG_GETARG_BOOL(3); int nodeNameCount = ArrayObjectCount(nodeNameArrayObject); int nodePortCount = ArrayObjectCount(nodePortArrayObject); int commandStringCount = ArrayObjectCount(commandStringArrayObject); Datum *nodeNameDatumArray = DeconstructArrayObject(nodeNameArrayObject); Datum *nodePortDatumArray = DeconstructArrayObject(nodePortArrayObject); Datum *commandStringDatumArray = DeconstructArrayObject(commandStringArrayObject); if (nodeNameCount != nodePortCount || nodeNameCount != commandStringCount) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("expected same number of node name, port, and query string"))); } StringInfo *nodeNames = palloc0(nodeNameCount * sizeof(StringInfo)); int *nodePorts = palloc0(nodeNameCount * sizeof(int)); StringInfo *commandStrings = palloc0(nodeNameCount * sizeof(StringInfo)); for (int index = 0; index < nodeNameCount; index++) { text *nodeNameText = DatumGetTextP(nodeNameDatumArray[index]); char *nodeName = text_to_cstring(nodeNameText); int32 nodePort = DatumGetInt32(nodePortDatumArray[index]); text *commandText = DatumGetTextP(commandStringDatumArray[index]); char *commandString = text_to_cstring(commandText); nodeNames[index] = makeStringInfo(); commandStrings[index] = makeStringInfo(); appendStringInfo(nodeNames[index], "%s", nodeName); nodePorts[index] = nodePort; appendStringInfo(commandStrings[index], "%s", commandString); } *nodeNameArray = nodeNames; *nodePortsArray = nodePorts; *commandStringArray = commandStrings; *parallel = parallelExecution; return nodeNameCount; } /* * ExecuteCommandsInParallelAndStoreResults connects to each node specified in * nodeNameArray and nodePortArray, and executes command in commandStringArray * in parallel fashion. Execution success status and result is reported for * each command in statusArray and resultStringArray. Each array contains * commandCount items. */ static void ExecuteCommandsInParallelAndStoreResults(StringInfo *nodeNameArray, int *nodePortArray, StringInfo *commandStringArray, bool *statusArray, StringInfo *resultStringArray, int commandCount) { MultiConnection **connectionArray = palloc0(commandCount * sizeof(MultiConnection *)); int finishedCount = 0; /* start connections asynchronously */ for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { char *nodeName = nodeNameArray[commandIndex]->data; int nodePort = nodePortArray[commandIndex]; int connectionFlags = FORCE_NEW_CONNECTION; connectionArray[commandIndex] = StartNodeConnection(connectionFlags, nodeName, nodePort); } /* establish connections */ for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { MultiConnection *connection = connectionArray[commandIndex]; StringInfo queryResultString = resultStringArray[commandIndex]; char *nodeName = nodeNameArray[commandIndex]->data; int nodePort = nodePortArray[commandIndex]; FinishConnectionEstablishment(connection); /* check whether connection attempt was successful */ if (PQstatus(connection->pgConn) != CONNECTION_OK) { appendStringInfo(queryResultString, "failed to connect to %s:%d", nodeName, nodePort); statusArray[commandIndex] = false; CloseConnection(connection); connectionArray[commandIndex] = NULL; finishedCount++; continue; } /* set the application_name to avoid nested execution checks */ int querySent = SendRemoteCommand(connection, psprintf( "SET application_name TO '%s%ld'", CITUS_RUN_COMMAND_APPLICATION_NAME_PREFIX, GetGlobalPID())); if (querySent == 0) { StoreErrorMessage(connection, queryResultString); statusArray[commandIndex] = false; CloseConnection(connection); connectionArray[commandIndex] = NULL; finishedCount++; continue; } statusArray[commandIndex] = true; } /* send queries at once */ for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { MultiConnection *connection = connectionArray[commandIndex]; if (connection == NULL) { continue; } bool raiseInterrupts = true; PGresult *queryResult = GetRemoteCommandResult(connection, raiseInterrupts); /* write the result value or error message to queryResultString */ StringInfo queryResultString = resultStringArray[commandIndex]; bool success = EvaluateSingleQueryResult(connection, queryResult, queryResultString); if (!success) { statusArray[commandIndex] = false; CloseConnection(connection); connectionArray[commandIndex] = NULL; finishedCount++; continue; } /* clear results for the next command */ PQclear(queryResult); bool raiseErrors = false; ClearResults(connection, raiseErrors); /* we only care about the SET application_name result on failure */ resetStringInfo(queryResultString); } /* send queries at once */ for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { MultiConnection *connection = connectionArray[commandIndex]; char *queryString = commandStringArray[commandIndex]->data; StringInfo queryResultString = resultStringArray[commandIndex]; /* * If we don't have a connection, nothing to send, error string should already * been filled. */ if (connection == NULL) { continue; } int querySent = SendRemoteCommand(connection, queryString); if (querySent == 0) { StoreErrorMessage(connection, queryResultString); statusArray[commandIndex] = false; CloseConnection(connection); connectionArray[commandIndex] = NULL; finishedCount++; } } /* check for query results */ while (finishedCount < commandCount) { for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { MultiConnection *connection = connectionArray[commandIndex]; StringInfo queryResultString = resultStringArray[commandIndex]; bool success = false; if (connection == NULL) { continue; } bool queryFinished = GetConnectionStatusAndResult(connection, &success, queryResultString); if (queryFinished) { finishedCount++; statusArray[commandIndex] = success; connectionArray[commandIndex] = NULL; CloseConnection(connection); } } CHECK_FOR_INTERRUPTS(); if (finishedCount < commandCount) { long sleepIntervalPerCycle = RemoteTaskCheckInterval * 1000L; pg_usleep(sleepIntervalPerCycle); } } pfree(connectionArray); } /* * GetConnectionStatusAndResult checks the active connection and returns true if * query execution is finished (either success or fail). * Query success/fail in resultStatus, and query result in queryResultString are * reported upon completion of the query. */ static bool GetConnectionStatusAndResult(MultiConnection *connection, bool *resultStatus, StringInfo queryResultString) { bool finished = true; ConnStatusType connectionStatus = PQstatus(connection->pgConn); *resultStatus = false; resetStringInfo(queryResultString); if (connectionStatus == CONNECTION_BAD) { appendStringInfo(queryResultString, "connection lost"); return finished; } int consumeInput = PQconsumeInput(connection->pgConn); if (consumeInput == 0) { appendStringInfo(queryResultString, "query result unavailable"); return finished; } /* check later if busy */ if (PQisBusy(connection->pgConn) != 0) { finished = false; return finished; } /* query result is available at this point */ PGresult *queryResult = PQgetResult(connection->pgConn); bool success = EvaluateSingleQueryResult(connection, queryResult, queryResultString); PQclear(queryResult); *resultStatus = success; finished = true; return true; } /* * ExecuteCommandsAndStoreResults connects to each node specified in * nodeNameArray and nodePortArray, and executes command in commandStringArray * in sequential order. Execution success status and result is reported for * each command in statusArray and resultStringArray. Each array contains * commandCount items. */ static void ExecuteCommandsAndStoreResults(StringInfo *nodeNameArray, int *nodePortArray, StringInfo *commandStringArray, bool *statusArray, StringInfo *resultStringArray, int commandCount) { for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { CHECK_FOR_INTERRUPTS(); char *nodeName = nodeNameArray[commandIndex]->data; int32 nodePort = nodePortArray[commandIndex]; char *queryString = commandStringArray[commandIndex]->data; StringInfo queryResultString = resultStringArray[commandIndex]; int connectionFlags = FORCE_NEW_CONNECTION; MultiConnection *connection = GetNodeConnection(connectionFlags, nodeName, nodePort); /* set the application_name to avoid nested execution checks */ bool success = ExecuteOptionalSingleResultCommand( connection, psprintf( "SET application_name TO '%s%ld'", CITUS_RUN_COMMAND_APPLICATION_NAME_PREFIX, GetGlobalPID()), queryResultString ); if (!success) { statusArray[commandIndex] = false; CloseConnection(connection); continue; } /* we only care about the SET application_name result on failure */ resetStringInfo(queryResultString); /* send the actual query string */ success = ExecuteOptionalSingleResultCommand(connection, queryString, queryResultString); statusArray[commandIndex] = success; CloseConnection(connection); } } /* * ExecuteOptionalSingleResultCommand executes a query at specified remote node using * the calling user's credentials. The function returns the query status * (success/failure), and query result. The query is expected to return a single * target containing zero or one rows. */ static bool ExecuteOptionalSingleResultCommand(MultiConnection *connection, char *queryString, StringInfo queryResultString) { if (PQstatus(connection->pgConn) != CONNECTION_OK) { appendStringInfo(queryResultString, "failed to connect to %s:%d", connection->hostname, connection->port); return false; } if (!SendRemoteCommand(connection, queryString)) { appendStringInfo(queryResultString, "failed to send query to %s:%d", connection->hostname, connection->port); return false; } bool raiseInterrupts = true; PGresult *queryResult = GetRemoteCommandResult(connection, raiseInterrupts); /* write the result value or error message to queryResultString */ bool success = EvaluateSingleQueryResult(connection, queryResult, queryResultString); /* clear result and close the connection */ PQclear(queryResult); bool raiseErrors = false; ClearResults(connection, raiseErrors); return success; } /* CreateTupleStore prepares result tuples from individual query results */ static Tuplestorestate * CreateTupleStore(TupleDesc tupleDescriptor, StringInfo *nodeNameArray, int *nodePortArray, bool *statusArray, StringInfo *resultArray, int commandCount) { Tuplestorestate *tupleStore = tuplestore_begin_heap(true, false, work_mem); bool nulls[4] = { false, false, false, false }; for (int commandIndex = 0; commandIndex < commandCount; commandIndex++) { Datum values[4]; StringInfo nodeNameString = nodeNameArray[commandIndex]; StringInfo resultString = resultArray[commandIndex]; text *nodeNameText = cstring_to_text_with_len(nodeNameString->data, nodeNameString->len); text *resultText = cstring_to_text_with_len(resultString->data, resultString->len); values[0] = PointerGetDatum(nodeNameText); values[1] = Int32GetDatum(nodePortArray[commandIndex]); values[2] = BoolGetDatum(statusArray[commandIndex]); values[3] = PointerGetDatum(resultText); HeapTuple tuple = heap_form_tuple(tupleDescriptor, values, nulls); tuplestore_puttuple(tupleStore, tuple); heap_freetuple(tuple); pfree(nodeNameText); pfree(resultText); } return tupleStore; } ================================================ FILE: src/backend/distributed/operations/create_shards.c ================================================ /*------------------------------------------------------------------------- * * create_shards.c * * This file contains functions to distribute a table by creating shards for it * across a set of worker nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include #include #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "port.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "postmaster/postmaster.h" #include "storage/fd.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(master_create_worker_shards); /* * master_create_worker_shards is a deprecated UDF that was used to * create shards for a hash-distributed table. */ Datum master_create_worker_shards(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("master_create_worker_shards has been deprecated"))); } /* * CreateShardsWithRoundRobinPolicy creates empty shards for the given table * based on the specified number of initial shards. The function first updates * metadata on the coordinator node to make this shard (and its placements) * visible. Note that the function assumes the table is hash partitioned and * calculates the min/max hash token ranges for each shard, giving them an equal * split of the hash space. Finally, function creates empty shard placements on * worker nodes. */ void CreateShardsWithRoundRobinPolicy(Oid distributedTableId, int32 shardCount, int32 replicationFactor, bool useExclusiveConnections) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); List *insertedShardPlacements = NIL; List *insertedShardIds = NIL; /* make sure table is hash partitioned */ CheckHashPartitionedTable(distributedTableId); /* * In contrast to append/range partitioned tables it makes more sense to * require ownership privileges - shards for hash-partitioned tables are * only created once, not continually during ingest as for the other * partitioning types. */ EnsureTableOwner(distributedTableId); /* we plan to add shards: get an exclusive lock on relation oid */ LockRelationOid(distributedTableId, ExclusiveLock); /* validate that shards haven't already been created for this table */ List *existingShardList = LoadShardList(distributedTableId); if (existingShardList != NIL) { char *tableName = get_rel_name(distributedTableId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("table \"%s\" has already had shards created for it", tableName))); } /* make sure that at least one shard is specified */ if (shardCount <= 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("shard_count must be positive"))); } /* make sure that at least one replica is specified */ if (replicationFactor <= 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("replication_factor must be positive"))); } /* make sure that RF=1 if the table is streaming replicated */ if (cacheEntry->replicationModel == REPLICATION_MODEL_STREAMING && replicationFactor > 1) { char *relationName = get_rel_name(cacheEntry->relationId); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("using replication factor %d with the streaming " "replication model is not supported", replicationFactor), errdetail("The table %s is marked as streaming replicated and " "the shard replication factor of streaming replicated " "tables must be 1.", relationName), errhint("Use replication factor 1."))); } /* calculate the split of the hash space */ uint64 hashTokenIncrement = HASH_TOKEN_COUNT / shardCount; /* don't allow concurrent node list changes that require an exclusive lock */ LockRelationOid(DistNodeRelationId(), RowShareLock); /* load and sort the worker node list for deterministic placement */ List *workerNodeList = DistributedTablePlacementNodeList(NoLock); workerNodeList = SortList(workerNodeList, CompareWorkerNodes); int32 workerNodeCount = list_length(workerNodeList); if (replicationFactor > workerNodeCount) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("replication_factor (%d) exceeds number of worker nodes " "(%d)", replicationFactor, workerNodeCount), errhint("Add more worker nodes or try again with a lower " "replication factor."))); } /* set shard storage type according to relation type */ char shardStorageType = ShardStorageType(distributedTableId); for (int64 shardIndex = 0; shardIndex < shardCount; shardIndex++) { uint32 roundRobinNodeIndex = shardIndex % workerNodeCount; /* initialize the hash token space for this shard */ int32 shardMinHashToken = PG_INT32_MIN + (shardIndex * hashTokenIncrement); int32 shardMaxHashToken = shardMinHashToken + (hashTokenIncrement - 1); uint64 *shardIdPtr = (uint64 *) palloc0(sizeof(uint64)); *shardIdPtr = GetNextShardId(); insertedShardIds = lappend(insertedShardIds, shardIdPtr); /* if we are at the last shard, make sure the max token value is INT_MAX */ if (shardIndex == (shardCount - 1)) { shardMaxHashToken = PG_INT32_MAX; } /* insert the shard metadata row along with its min/max values */ text *minHashTokenText = IntegerToText(shardMinHashToken); text *maxHashTokenText = IntegerToText(shardMaxHashToken); InsertShardRow(distributedTableId, *shardIdPtr, shardStorageType, minHashTokenText, maxHashTokenText); InsertShardPlacementRows(distributedTableId, *shardIdPtr, workerNodeList, roundRobinNodeIndex, replicationFactor); } /* * load shard placements for the shard at once after all placement insertions * finished. This prevents MetadataCache from rebuilding unnecessarily after * each placement insertion. */ uint64 *shardIdPtr; foreach_declared_ptr(shardIdPtr, insertedShardIds) { List *placementsForShard = ShardPlacementList(*shardIdPtr); insertedShardPlacements = list_concat(insertedShardPlacements, placementsForShard); } CreateShardsOnWorkers(distributedTableId, insertedShardPlacements, useExclusiveConnections); } /* * CreateColocatedShards creates shards for the target relation colocated with * the source relation. */ void CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool useExclusiveConnections) { List *insertedShardPlacements = NIL; List *insertedShardIds = NIL; CitusTableCacheEntry *targetCacheEntry = GetCitusTableCacheEntry(targetRelationId); Assert(targetCacheEntry->partitionMethod == DISTRIBUTE_BY_HASH || targetCacheEntry->partitionMethod == DISTRIBUTE_BY_NONE); /* * In contrast to append/range partitioned tables it makes more sense to * require ownership privileges - shards for hash-partitioned tables are * only created once, not continually during ingest as for the other * partitioning types. */ EnsureTableOwner(targetRelationId); /* we plan to add shards: get an exclusive lock on target relation oid */ LockRelationOid(targetRelationId, ExclusiveLock); /* we don't want source table to get dropped before we colocate with it */ LockRelationOid(sourceRelationId, AccessShareLock); /* prevent placement changes of the source relation until we colocate with them */ List *sourceShardIntervalList = LoadShardIntervalList(sourceRelationId); LockShardListMetadata(sourceShardIntervalList, ShareLock); /* validate that shards haven't already been created for this table */ List *existingShardList = LoadShardList(targetRelationId); if (existingShardList != NIL) { char *targetRelationName = get_rel_name(targetRelationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("table \"%s\" has already had shards created for it", targetRelationName))); } char targetShardStorageType = ShardStorageType(targetRelationId); ShardInterval *sourceShardInterval = NULL; foreach_declared_ptr(sourceShardInterval, sourceShardIntervalList) { uint64 sourceShardId = sourceShardInterval->shardId; uint64 *newShardIdPtr = (uint64 *) palloc0(sizeof(uint64)); *newShardIdPtr = GetNextShardId(); insertedShardIds = lappend(insertedShardIds, newShardIdPtr); text *shardMinValueText = NULL; text *shardMaxValueText = NULL; if (targetCacheEntry->partitionMethod == DISTRIBUTE_BY_NONE) { Assert(list_length(sourceShardIntervalList) == 1); } else { int32 shardMinValue = DatumGetInt32(sourceShardInterval->minValue); int32 shardMaxValue = DatumGetInt32(sourceShardInterval->maxValue); shardMinValueText = IntegerToText(shardMinValue); shardMaxValueText = IntegerToText(shardMaxValue); } List *sourceShardPlacementList = ShardPlacementListSortedByWorker( sourceShardId); InsertShardRow(targetRelationId, *newShardIdPtr, targetShardStorageType, shardMinValueText, shardMaxValueText); ShardPlacement *sourcePlacement = NULL; foreach_declared_ptr(sourcePlacement, sourceShardPlacementList) { int32 groupId = sourcePlacement->groupId; const uint64 shardSize = 0; InsertShardPlacementRow(*newShardIdPtr, INVALID_PLACEMENT_ID, shardSize, groupId); } } /* * load shard placements for the shard at once after all placement insertions * finished. This prevents MetadataCache from rebuilding unnecessarily after * each placement insertion. */ uint64 *shardIdPtr; foreach_declared_ptr(shardIdPtr, insertedShardIds) { List *placementsForShard = ShardPlacementList(*shardIdPtr); insertedShardPlacements = list_concat(insertedShardPlacements, placementsForShard); } CreateShardsOnWorkers(targetRelationId, insertedShardPlacements, useExclusiveConnections); } /* * CreateReferenceTableShard creates a single shard for the given * distributedTableId. The created shard does not have min/max values. * Also, the shard is replicated to the all active nodes in the cluster. */ void CreateReferenceTableShard(Oid distributedTableId) { int workerStartIndex = 0; text *shardMinValue = NULL; text *shardMaxValue = NULL; bool useExclusiveConnection = false; /* * In contrast to append/range partitioned tables it makes more sense to * require ownership privileges - shards for reference tables are * only created once, not continually during ingest as for the other * partitioning types such as append and range. */ EnsureTableOwner(distributedTableId); /* we plan to add shards: get an exclusive lock on relation oid */ LockRelationOid(distributedTableId, ExclusiveLock); /* set shard storage type according to relation type */ char shardStorageType = ShardStorageType(distributedTableId); /* validate that shards haven't already been created for this table */ List *existingShardList = LoadShardList(distributedTableId); if (existingShardList != NIL) { char *tableName = get_rel_name(distributedTableId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("table \"%s\" has already had shards created for it", tableName))); } /* * load and sort the worker node list for deterministic placements * create_reference_table has already acquired pg_dist_node lock */ List *nodeList = ReferenceTablePlacementNodeList(ShareLock); nodeList = SortList(nodeList, CompareWorkerNodes); int replicationFactor = list_length(nodeList); /* get the next shard id */ uint64 shardId = GetNextShardId(); InsertShardRow(distributedTableId, shardId, shardStorageType, shardMinValue, shardMaxValue); InsertShardPlacementRows(distributedTableId, shardId, nodeList, workerStartIndex, replicationFactor); /* * load shard placements for the shard at once after all placement insertions * finished. This prevents MetadataCache from rebuilding unnecessarily after * each placement insertion. */ List *insertedShardPlacements = ShardPlacementList(shardId); CreateShardsOnWorkers(distributedTableId, insertedShardPlacements, useExclusiveConnection); } /* * CreateSingleShardTableShardWithRoundRobinPolicy creates a single * shard for the given distributedTableId. The created shard does not * have min/max values. Unlike CreateReferenceTableShard, the shard is * _not_ replicated to all nodes but would have a single placement like * Citus local tables. * * However, this placement doesn't necessarily need to be placed on * coordinator. This is determined based on modulo of the colocation * id that given table has been associated to. */ void CreateSingleShardTableShardWithRoundRobinPolicy(Oid relationId, uint32 colocationId) { EnsureTableOwner(relationId); /* we plan to add shards: get an exclusive lock on relation oid */ LockRelationOid(relationId, ExclusiveLock); /* * Load and sort the worker node list for deterministic placement. * * Also take a RowShareLock on pg_dist_node to disallow concurrent * node list changes that require an exclusive lock. */ List *workerNodeList = DistributedTablePlacementNodeList(RowShareLock); workerNodeList = SortList(workerNodeList, CompareWorkerNodes); int roundRobinNodeIdx = EmptySingleShardTableColocationDecideNodeId(colocationId); char shardStorageType = ShardStorageType(relationId); text *minHashTokenText = NULL; text *maxHashTokenText = NULL; uint64 shardId = GetNextShardId(); InsertShardRow(relationId, shardId, shardStorageType, minHashTokenText, maxHashTokenText); int replicationFactor = 1; InsertShardPlacementRows(relationId, shardId, workerNodeList, roundRobinNodeIdx, replicationFactor); /* * load shard placements for the shard at once after all placement insertions * finished. This prevents MetadataCache from rebuilding unnecessarily after * each placement insertion. */ List *insertedShardPlacements = ShardPlacementList(shardId); /* * We don't need to force using exclusive connections because we're anyway * creating a single shard. */ bool useExclusiveConnection = false; CreateShardsOnWorkers(relationId, insertedShardPlacements, useExclusiveConnection); } /* * EmptySingleShardTableColocationDecideNodeId returns index of the node * that first shard to be created in given "single-shard table colocation * group" should be placed on. * * This is determined by modulo of the colocation id by the length of the * list returned by DistributedTablePlacementNodeList(). */ int EmptySingleShardTableColocationDecideNodeId(uint32 colocationId) { List *workerNodeList = DistributedTablePlacementNodeList(RowShareLock); int32 workerNodeCount = list_length(workerNodeList); if (workerNodeCount == 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("couldn't find any worker nodes"), errhint("Add more worker nodes"))); } return colocationId % workerNodeCount; } /* * CheckHashPartitionedTable looks up the partition information for the given * tableId and checks if the table is hash partitioned. If not, the function * throws an error. */ void CheckHashPartitionedTable(Oid distributedTableId) { char partitionType = PartitionMethod(distributedTableId); if (partitionType != DISTRIBUTE_BY_HASH) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported table partition type: %c", partitionType))); } } /* Helper function to convert an integer value to a text type */ text * IntegerToText(int32 value) { StringInfo valueString = makeStringInfo(); appendStringInfo(valueString, "%d", value); text *valueText = cstring_to_text(valueString->data); return valueText; } ================================================ FILE: src/backend/distributed/operations/delete_protocol.c ================================================ /*------------------------------------------------------------------------- * * delete_protocol.c * * Routine for deleting shards in the distributed cluster. This function takes * in a delete command and deletes a shard if and only if all rows in the shard * satisfy the conditions in the delete command. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "port.h" #include "access/xact.h" #include "catalog/namespace.h" #include "commands/dbcommands.h" #include "lib/stringinfo.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "tcop/tcopprot.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/varlena.h" #include "pg_version_constants.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/placement_connection.h" #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/shard_cleaner.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* Local functions forward declarations */ static int DropShards(Oid relationId, char *schemaName, char *relationName, List *deletableShardIntervalList, bool dropShardsMetadataOnly); static List * DropTaskList(Oid relationId, char *schemaName, char *relationName, List *deletableShardIntervalList); static void ExecuteDropShardPlacementCommandRemotely(ShardPlacement *shardPlacement, const char *shardRelationName, const char * dropShardPlacementCommand); static char * CreateDropShardPlacementCommand(const char *schemaName, const char *shardRelationName, char storageType); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_apply_delete_command); PG_FUNCTION_INFO_V1(citus_drop_all_shards); PG_FUNCTION_INFO_V1(master_drop_all_shards); PG_FUNCTION_INFO_V1(master_drop_sequences); /* * master_apply_delete_command is a deprecated function for dropping shards * in an append-distributed tables. */ Datum master_apply_delete_command(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("master_apply_delete_command has been deprecated"))); } /* * citus_drop_all_shards attempts to drop all shards for a given relation. * This function can be called even if the table has already been dropped. * In that case, the schema name and relation name arguments are used to * determine that table name. Otherwise, the relation ID is used and the * other arguments are ignored. */ Datum citus_drop_all_shards(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *schemaNameText = PG_GETARG_TEXT_P(1); text *relationNameText = PG_GETARG_TEXT_P(2); bool dropShardsMetadataOnly = PG_GETARG_BOOL(3); char *schemaName = text_to_cstring(schemaNameText); char *relationName = text_to_cstring(relationNameText); /* * The SQL_DROP trigger calls this function even for tables that are * not distributed. In that case, silently ignore and return -1. */ if (!IsCitusTableViaCatalog(relationId) || !EnableDDLPropagation) { PG_RETURN_INT32(-1); } EnsureCoordinator(); CheckTableSchemaNameForDrop(relationId, &schemaName, &relationName); /* * citus_drop_all_shards is typically called from the DROP TABLE trigger, * but could be called by a user directly. Make sure we have an * AccessExclusiveLock to prevent any other commands from running on this table * concurrently. */ LockRelationOid(relationId, AccessExclusiveLock); List *shardIntervalList = LoadUnsortedShardIntervalListViaCatalog(relationId); int droppedShardCount = DropShards(relationId, schemaName, relationName, shardIntervalList, dropShardsMetadataOnly); PG_RETURN_INT32(droppedShardCount); } /* * master_drop_all_shards is a wrapper function for old UDF name. */ Datum master_drop_all_shards(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); text *schemaNameText = PG_GETARG_TEXT_P(1); text *relationNameText = PG_GETARG_TEXT_P(2); bool dropShardsMetadataOnly = false; LOCAL_FCINFO(local_fcinfo, 4); InitFunctionCallInfoData(*local_fcinfo, NULL, 4, InvalidOid, NULL, NULL); local_fcinfo->args[0].value = ObjectIdGetDatum(relationId); local_fcinfo->args[0].isnull = false; local_fcinfo->args[1].value = PointerGetDatum(schemaNameText); local_fcinfo->args[1].isnull = false; local_fcinfo->args[2].value = PointerGetDatum(relationNameText); local_fcinfo->args[2].isnull = false; local_fcinfo->args[3].value = BoolGetDatum(dropShardsMetadataOnly); local_fcinfo->args[3].isnull = false; return citus_drop_all_shards(local_fcinfo); } /* * master_drop_sequences was previously used to drop sequences on workers * when using metadata syncing. * * It may still be called when dropping objects during CREATE EXTENSION, * hence the function remains in place. */ Datum master_drop_sequences(PG_FUNCTION_ARGS) { PG_RETURN_VOID(); } /* * CheckTableSchemaNameForDrop errors out if the current user does not * have permission to un-distribute the given relation, taking into * account that it may be called from the drop trigger. If the table exists, * the function rewrites the given table and schema name. */ void CheckTableSchemaNameForDrop(Oid relationId, char **schemaName, char **tableName) { char *tempTableName = get_rel_name(relationId); if (tempTableName != NULL) { /* ensure proper values are used if the table exists */ Oid schemaId = get_rel_namespace(relationId); (*schemaName) = get_namespace_name(schemaId); (*tableName) = tempTableName; EnsureTableOwner(relationId); } } /* * DropShards drops all given shards in a relation. The id, name and schema * for the relation are explicitly provided, since this function may be * called when the table is already dropped. * * We mark shard placements that we couldn't drop as to be deleted later, but * we do delete the shard metadadata. * * If dropShardsMetadataOnly is true, then we don't send remote commands to drop the shards: * we only remove pg_dist_placement and pg_dist_shard rows. */ static int DropShards(Oid relationId, char *schemaName, char *relationName, List *deletableShardIntervalList, bool dropShardsMetadataOnly) { Assert(OidIsValid(relationId)); Assert(schemaName != NULL); Assert(relationName != NULL); UseCoordinatedTransaction(); /* * We will use below variable across this function to decide if we can * use local execution */ int32 localGroupId = GetLocalGroupId(); /* DROP table commands are currently only supported from the coordinator */ Assert(localGroupId == COORDINATOR_GROUP_ID); Use2PCForCoordinatedTransaction(); List *dropTaskList = DropTaskList(relationId, schemaName, relationName, deletableShardIntervalList); bool shouldExecuteTasksLocally = ShouldExecuteTasksLocally(dropTaskList); Task *task = NULL; foreach_declared_ptr(task, dropTaskList) { uint64 shardId = task->anchorShardId; ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, task->taskPlacementList) { uint64 shardPlacementId = shardPlacement->placementId; int32 shardPlacementGroupId = shardPlacement->groupId; bool isLocalShardPlacement = (shardPlacementGroupId == localGroupId); /* * If this variable is true, that means the active DROP SCHEMA/DATABASE ... CASCADE * will drop the shard. If we try to drop it over another connection, we will * get into a distributed deadlock. Hence, if this variable is true we should just * delete the shard placement metadata and skip dropping the shard for now. */ bool skipIfDropSchemaOrDBInProgress = isLocalShardPlacement && DropSchemaOrDBInProgress() && localGroupId == COORDINATOR_GROUP_ID; /* * We want to send commands to drop shards when both * skipIfDropSchemaOrDBInProgress and dropShardsMetadataOnly are false. */ bool applyRemoteShardsDrop = !skipIfDropSchemaOrDBInProgress && !dropShardsMetadataOnly; if (applyRemoteShardsDrop) { /* * If it is a local placement of a distributed table or a reference table, * then execute the DROP command locally. */ if (isLocalShardPlacement && shouldExecuteTasksLocally) { List *singleTaskList = list_make1(task); ExecuteLocalUtilityTaskList(singleTaskList); } else { /* * Either it was not a local placement or we could not use * local execution even if it was a local placement. * If it is the second case, then it is possibly because in * current transaction, some commands or queries connected * to local group as well. * * Regardless of the node is a remote node or the current node, * try to open a new connection (or use an existing one) to * connect to that node to drop the shard placement over that * remote connection. */ const char *dropShardPlacementCommand = TaskQueryString(task); ExecuteDropShardPlacementCommandRemotely(shardPlacement, relationName, dropShardPlacementCommand); if (isLocalShardPlacement) { SetLocalExecutionStatus(LOCAL_EXECUTION_DISABLED); } } } DeleteShardPlacementRow(shardPlacementId); } /* * Now that we deleted all placements of the shard (or their metadata), * delete the shard metadata as well. */ DeleteShardRow(shardId); } int droppedShardCount = list_length(deletableShardIntervalList); return droppedShardCount; } /* * DropTaskList returns a list of tasks to execute a DROP command on shard * placements of distributed table. This is handled separately from other * DDL commands because we handle it via the DROP trigger, which is called * whenever a drop cascades. */ static List * DropTaskList(Oid relationId, char *schemaName, char *relationName, List *deletableShardIntervalList) { /* resulting task list */ List *taskList = NIL; /* enumerate the tasks when putting them to the taskList */ int taskId = 1; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, deletableShardIntervalList) { Assert(shardInterval->relationId == relationId); uint64 shardId = shardInterval->shardId; char storageType = shardInterval->storageType; char *shardRelationName = pstrdup(relationName); /* build shard relation name */ AppendShardIdToName(&shardRelationName, shardId); char *dropShardPlacementCommand = CreateDropShardPlacementCommand(schemaName, shardRelationName, storageType); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryString(task, dropShardPlacementCommand); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ShardPlacementList(shardId); taskList = lappend(taskList, task); } return taskList; } /* * ExecuteDropShardPlacementCommandRemotely executes the given DROP shard command * via remote critical connection. */ static void ExecuteDropShardPlacementCommandRemotely(ShardPlacement *shardPlacement, const char *relationName, const char *dropShardPlacementCommand) { Assert(shardPlacement != NULL); Assert(relationName != NULL); Assert(dropShardPlacementCommand != NULL); uint32 connectionFlags = FOR_DDL; MultiConnection *connection = GetPlacementConnection(connectionFlags, shardPlacement, NULL); /* * This code-path doesn't support optional connections, so we don't expect * NULL connections. */ Assert(connection != NULL); RemoteTransactionBeginIfNecessary(connection); if (PQstatus(connection->pgConn) != CONNECTION_OK) { char *workerName = shardPlacement->nodeName; uint32 workerPort = shardPlacement->nodePort; /* build shard relation name */ uint64 shardId = shardPlacement->shardId; char *shardRelationName = pstrdup(relationName); AppendShardIdToName(&shardRelationName, shardId); ereport(WARNING, (errmsg("could not connect to shard \"%s\" on node " "\"%s:%u\"", shardRelationName, workerName, workerPort), errdetail("Marking this shard placement for " "deletion"))); InsertCleanupOnSuccessRecordInCurrentTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, shardRelationName, shardPlacement->groupId); return; } MarkRemoteTransactionCritical(connection); ExecuteCriticalRemoteCommand(connection, dropShardPlacementCommand); } /* * CreateDropShardPlacementCommand function builds the DROP command to drop * the given shard relation by qualifying it with schema name according to * shard relation's storage type. */ static char * CreateDropShardPlacementCommand(const char *schemaName, const char *shardRelationName, char storageType) { Assert(schemaName != NULL); Assert(shardRelationName != NULL); StringInfo workerDropQuery = makeStringInfo(); const char *quotedShardName = quote_qualified_identifier(schemaName, shardRelationName); /* build workerDropQuery according to shard storage type */ if (storageType == SHARD_STORAGE_TABLE) { appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, quotedShardName); } else if (storageType == SHARD_STORAGE_FOREIGN) { appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, quotedShardName); } else { /* no other storage type is expected here */ Assert(false); } return workerDropQuery->data; } ================================================ FILE: src/backend/distributed/operations/health_check.c ================================================ /*------------------------------------------------------------------------- * * health_check.c * * UDFs to run health check operations by coordinating simple queries to test connectivity * between connection pairs in the cluster. * * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/builtins.h" #include "distributed/argutils.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" /* simple query to run on workers to check connectivity */ #define CONNECTIVITY_CHECK_QUERY "SELECT 1" #define CONNECTIVITY_CHECK_COLUMNS 5 PG_FUNCTION_INFO_V1(citus_check_connection_to_node); PG_FUNCTION_INFO_V1(citus_check_cluster_node_health); static bool CheckConnectionToNode(char *nodeName, uint32 nodePort); static void StoreAllConnectivityChecks(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); static char * GetConnectivityCheckCommand(const char *nodeName, const uint32 nodePort); /* * citus_check_connection_to_node sends a simple query from a worker node to another * node, and returns success status. */ Datum citus_check_connection_to_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); char *nodeName = PG_GETARG_TEXT_TO_CSTRING(0); uint32 nodePort = PG_GETARG_UINT32(1); bool success = CheckConnectionToNode(nodeName, nodePort); PG_RETURN_BOOL(success); } /* * CheckConnectionToNode sends a simple query to a node and returns success status */ static bool CheckConnectionToNode(char *nodeName, uint32 nodePort) { int connectionFlags = 0; MultiConnection *connection = GetNodeConnection(connectionFlags, nodeName, nodePort); int responseStatus = ExecuteOptionalRemoteCommand(connection, CONNECTIVITY_CHECK_QUERY, NULL); return responseStatus == RESPONSE_OKAY; } /* * citus_check_cluster_node_health UDF performs connectivity checks from all the nodes to * all the nodes, and report success status */ Datum citus_check_cluster_node_health(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); StoreAllConnectivityChecks(tupleStore, tupleDescriptor); PG_RETURN_VOID(); } /* * StoreAllConnectivityChecks performs connectivity checks from all the nodes to all the * nodes, and report success status. * * Algorithm is: * for sourceNode in activeReadableNodeList: * c = connectToNode(sourceNode) * for targetNode in activeReadableNodeList: * result = c.execute("SELECT citus_check_connection_to_node(targetNode.name, targetNode.port") * emit sourceNode.name, sourceNode.port, targetNode.name, targetNode.port, result * * -- result -> true -> connection attempt from source to target succeeded * -- result -> false -> connection attempt from source to target failed * -- result -> NULL -> connection attempt from the current node to source node failed */ static void StoreAllConnectivityChecks(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { Datum values[CONNECTIVITY_CHECK_COLUMNS]; bool isNulls[CONNECTIVITY_CHECK_COLUMNS]; /* * Get all the readable node list so that we will check connectivity to followers in * the cluster as well. */ List *workerNodeList = ActiveReadableNodeList(); /* we want to check for connectivity in a deterministic order */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); /* * We iterate over the workerNodeList twice, for source and target worker nodes. This * operation is safe for foreach_declared_ptr macro, as long as we use different variables for * each iteration. */ WorkerNode *sourceWorkerNode = NULL; foreach_declared_ptr(sourceWorkerNode, workerNodeList) { const char *sourceNodeName = sourceWorkerNode->workerName; const int sourceNodePort = sourceWorkerNode->workerPort; int32 connectionFlags = 0; /* open a connection to the source node using the synchronous api */ MultiConnection *connectionToSourceNode = GetNodeConnection(connectionFlags, sourceNodeName, sourceNodePort); /* the second iteration over workerNodeList for the target worker nodes. */ WorkerNode *targetWorkerNode = NULL; foreach_declared_ptr(targetWorkerNode, workerNodeList) { const char *targetNodeName = targetWorkerNode->workerName; const int targetNodePort = targetWorkerNode->workerPort; char *connectivityCheckCommandToTargetNode = GetConnectivityCheckCommand(targetNodeName, targetNodePort); PGresult *result = NULL; int executionResult = ExecuteOptionalRemoteCommand(connectionToSourceNode, connectivityCheckCommandToTargetNode, &result); /* get ready for the next tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[0] = PointerGetDatum(cstring_to_text(sourceNodeName)); values[1] = Int32GetDatum(sourceNodePort); values[2] = PointerGetDatum(cstring_to_text(targetNodeName)); values[3] = Int32GetDatum(targetNodePort); /* * If we could not send the query or the result was not ok, set success field * to NULL. This may indicate connection errors to a worker node, however that * node can potentially connect to other nodes. * * Therefore, we mark the success as NULL to indicate that the connectivity * status is unknown. */ if (executionResult != RESPONSE_OKAY) { isNulls[4] = true; } else { int rowIndex = 0; int columnIndex = 0; values[4] = BoolGetDatum(ParseBoolField(result, rowIndex, columnIndex)); } tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); PQclear(result); ForgetResults(connectionToSourceNode); } } } /* * GetConnectivityCheckCommand returns the command to check connections to a node */ static char * GetConnectivityCheckCommand(const char *nodeName, const uint32 nodePort) { StringInfo connectivityCheckCommand = makeStringInfo(); appendStringInfo(connectivityCheckCommand, "SELECT citus_check_connection_to_node('%s', %d)", nodeName, nodePort); return connectivityCheckCommand->data; } ================================================ FILE: src/backend/distributed/operations/isolate_shards.c ================================================ /*------------------------------------------------------------------------- * * split_shards.c * * This file contains functions to split a shard according to a given * distribution column value. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "fmgr.h" #include "libpq-fe.h" #include "catalog/pg_class.h" #include "nodes/pg_list.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/typcache.h" #include "distributed/colocation_utils.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_split.h" #include "distributed/utils/distribution_column_map.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(isolate_tenant_to_new_shard); PG_FUNCTION_INFO_V1(worker_hash); /* * isolate_tenant_to_new_shard isolates a tenant to its own shard by spliting * the current matching shard. */ Datum isolate_tenant_to_new_shard(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid relationId = PG_GETARG_OID(0); Datum inputDatum = PG_GETARG_DATUM(1); text *cascadeOptionText = PG_GETARG_TEXT_P(2); Oid shardTransferModeOid = PG_GETARG_OID(3); EnsureTableOwner(relationId); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); char partitionMethod = cacheEntry->partitionMethod; if (partitionMethod != DISTRIBUTE_BY_HASH) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot isolate tenant because tenant isolation " "is only support for hash distributed tables"))); } List *colocatedTableList = ColocatedTableList(relationId); int colocatedTableCount = list_length(colocatedTableList); Oid inputDataType = get_fn_expr_argtype(fcinfo->flinfo, 1); char *tenantIdString = DatumToString(inputDatum, inputDataType); char *cascadeOptionString = text_to_cstring(cascadeOptionText); if (pg_strncasecmp(cascadeOptionString, "CASCADE", NAMEDATALEN) != 0 && colocatedTableCount > 1) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot isolate tenant because \"%s\" has colocated " "tables", relationName), errhint("Use CASCADE option to isolate tenants for the " "colocated tables too. Example usage: " "isolate_tenant_to_new_shard('%s', '%s', 'CASCADE')", relationName, tenantIdString))); } EnsureReferenceTablesExistOnAllNodes(); Var *distributionColumn = DistPartitionKey(relationId); /* earlier we checked that the table was hash partitioned, so there should be a distribution column */ Assert(distributionColumn != NULL); Oid distributionColumnType = distributionColumn->vartype; Datum tenantIdDatum = StringToDatum(tenantIdString, distributionColumnType); ShardInterval *sourceShard = FindShardInterval(tenantIdDatum, cacheEntry); if (sourceShard == NULL) { ereport(ERROR, (errmsg("tenant does not have a shard"))); } int shardMinValue = DatumGetInt32(sourceShard->minValue); int shardMaxValue = DatumGetInt32(sourceShard->maxValue); if (shardMinValue == shardMaxValue) { char *tableName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), (errmsg("table %s has already been isolated for the given value", quote_identifier(tableName))))); } List *sourcePlacementList = ActiveShardPlacementList(sourceShard->shardId); if (list_length(sourcePlacementList) > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot isolate tenants when using shard replication"))); } ShardPlacement *sourceShardPlacement = linitial(sourcePlacementList); /* get hash function name */ FmgrInfo *hashFunction = cacheEntry->hashFunction; /* get hashed value of the distribution value */ Datum hashedValueDatum = FunctionCall1Coll(hashFunction, cacheEntry->partitionColumn->varcollid, tenantIdDatum); int hashedValue = DatumGetInt32(hashedValueDatum); List *shardSplitPointsList = NIL; /* * If the hash value lies at one of the boundaries, we only have a single * split point. */ if (hashedValue == shardMinValue) { shardSplitPointsList = list_make1_int(hashedValue); } else if (hashedValue == shardMaxValue) { shardSplitPointsList = list_make1_int(hashedValue - 1); } else { shardSplitPointsList = list_make2_int(hashedValue - 1, hashedValue); } /* we currently place the isolated hash value into the same node */ int sourceNodeId = sourceShardPlacement->nodeId; List *nodeIdsForPlacementList = list_make2_int(sourceNodeId, sourceNodeId); if (list_length(shardSplitPointsList) > 1) { nodeIdsForPlacementList = lappend_int(nodeIdsForPlacementList, sourceNodeId); } DistributionColumnMap *distributionColumnOverrides = NULL; List *sourceColocatedShardIntervalList = NIL; SplitMode splitMode = LookupSplitMode(shardTransferModeOid); SplitShard(splitMode, ISOLATE_TENANT_TO_NEW_SHARD, sourceShard->shardId, shardSplitPointsList, nodeIdsForPlacementList, distributionColumnOverrides, sourceColocatedShardIntervalList, INVALID_COLOCATION_ID); cacheEntry = GetCitusTableCacheEntry(relationId); ShardInterval *newShard = FindShardInterval(tenantIdDatum, cacheEntry); PG_RETURN_INT64(newShard->shardId); } /* * worker_hash returns the hashed value of the given value. */ Datum worker_hash(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Datum valueDatum = PG_GETARG_DATUM(0); /* figure out hash function from the data type */ Oid valueDataType = get_fn_expr_argtype(fcinfo->flinfo, 0); TypeCacheEntry *typeEntry = lookup_type_cache(valueDataType, TYPECACHE_HASH_PROC_FINFO); if (typeEntry->hash_proc_finfo.fn_oid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot find a hash function for the input type"), errhint("Cast input to a data type with a hash function."))); } FmgrInfo *hashFunction = palloc0(sizeof(FmgrInfo)); fmgr_info_copy(hashFunction, &(typeEntry->hash_proc_finfo), CurrentMemoryContext); /* calculate hash value */ Datum hashedValueDatum = FunctionCall1Coll(hashFunction, PG_GET_COLLATION(), valueDatum); PG_RETURN_INT32(hashedValueDatum); } ================================================ FILE: src/backend/distributed/operations/modify_multiple_shards.c ================================================ /*------------------------------------------------------------------------- * * modify_multiple_shards.c * UDF to run multi shard update/delete queries * * This file contains master_modify_multiple_shards function, which takes a update * or delete query and runs it worker shards of the distributed table. The distributed * modify operation can be done within a distributed transaction. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "catalog/pg_class.h" #include "commands/dbcommands.h" #include "commands/event_trigger.h" #include "nodes/makefuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/resource_lock.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" PG_FUNCTION_INFO_V1(master_modify_multiple_shards); /* * master_modify_multiple_shards takes in a DELETE or UPDATE query string and * executes it. This is mainly provided for backwards compatibility, users * should use regular UPDATE and DELETE commands. */ Datum master_modify_multiple_shards(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *queryText = PG_GETARG_TEXT_P(0); char *queryString = text_to_cstring(queryText); RawStmt *rawStmt = (RawStmt *) ParseTreeRawStmt(queryString); Node *queryTreeNode = rawStmt->stmt; if (!IsA(queryTreeNode, DeleteStmt) && !IsA(queryTreeNode, UpdateStmt)) { ereport(ERROR, (errmsg("query \"%s\" is not a delete or update " "statement", queryString))); } ereport(WARNING, (errmsg("master_modify_multiple_shards is deprecated and will be " "removed in a future release."), errhint("Run the command directly"))); ExecuteQueryStringIntoDestReceiver(queryString, NULL, None_Receiver); PG_RETURN_INT32(0); } ================================================ FILE: src/backend/distributed/operations/node_promotion.c ================================================ #include "postgres.h" #include "utils/fmgrprotos.h" #include "utils/pg_lsn.h" #include "distributed/argutils.h" #include "distributed/clonenode_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/remote_commands.h" #include "distributed/shard_rebalancer.h" static void BlockAllWritesToWorkerNode(WorkerNode *workerNode); static bool GetNodeIsInRecoveryStatus(WorkerNode *workerNode); static void PromoteCloneNode(WorkerNode *cloneWorkerNode); static void EnsureSingleNodePromotion(WorkerNode *primaryNode); PG_FUNCTION_INFO_V1(citus_promote_clone_and_rebalance); /* * citus_promote_clone_and_rebalance promotes an inactive clone node to become * the new primary node, replacing its original primary node. * * This function performs the following steps: * 1. Validates that the clone node exists and is properly configured * 2. Ensures the clone is inactive and has a valid primary node reference * 3. Blocks all writes to the primary node to prevent data divergence * 4. Waits for the clone to catch up with the primary's WAL position * 5. Promotes the clone node to become a standalone primary * 6. Updates metadata to mark the clone as active and primary * 7. Rebalances shards between the old primary and new primary * 8. Returns information about the promotion and any shard movements * * Arguments: * - clone_nodeid: The node ID of the clone to promote * - catchUpTimeoutSeconds: Maximum time to wait for clone to catch up (default: 300) * * The function ensures data consistency by blocking writes during the promotion * process and verifying replication lag before proceeding. */ Datum citus_promote_clone_and_rebalance(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* Ensure superuser and coordinator */ EnsureSuperUser(); EnsureCoordinator(); /* Get clone_nodeid argument */ int32 cloneNodeIdArg = PG_GETARG_INT32(0); /* Get catchUpTimeoutSeconds argument with default value of 300 */ int32 catchUpTimeoutSeconds = PG_ARGISNULL(2) ? 300 : PG_GETARG_INT32(2); /* Lock pg_dist_node to prevent concurrent modifications during this operation */ LockRelationOid(DistNodeRelationId(), RowExclusiveLock); WorkerNode *cloneNode = FindNodeAnyClusterByNodeId(cloneNodeIdArg); if (cloneNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Clone node with ID %d not found.", cloneNodeIdArg))); } if (!cloneNode->nodeisclone || cloneNode->nodeprimarynodeid == 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Node %s:%d (ID %d) is not a valid clone or its primary node ID is not set.", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); } if (cloneNode->isActive) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Clone node %s:%d (ID %d) is already active and cannot be promoted.", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); } WorkerNode *primaryNode = FindNodeAnyClusterByNodeId(cloneNode->nodeprimarynodeid); if (primaryNode == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Primary node with ID %d (for clone %s:%d) not found.", cloneNode->nodeprimarynodeid, cloneNode->workerName, cloneNode->workerPort))); } if (primaryNode->nodeisclone) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Primary node %s:%d (ID %d) is itself a clone.", primaryNode->workerName, primaryNode->workerPort, primaryNode->nodeId))); } if (!primaryNode->isActive) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Primary node %s:%d (ID %d) is not active.", primaryNode->workerName, primaryNode->workerPort, primaryNode->nodeId))); } /* Ensure the primary node is related to the clone node */ if (primaryNode->nodeId != cloneNode->nodeprimarynodeid) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Clone node %s:%d (ID %d) is not a clone of the primary node %s:%d (ID %d).", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId, primaryNode->workerName, primaryNode->workerPort, primaryNode->nodeId))); } EnsureSingleNodePromotion(primaryNode); ereport(NOTICE, (errmsg( "Starting promotion process for clone node %s:%d (ID %d), original primary %s:%d (ID %d)", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId, primaryNode->workerName, primaryNode->workerPort, primaryNode ->nodeId))); /* Step 0: Check if clone is replica of provided primary node and is not synchronous */ char *operation = "promote"; EnsureValidCloneMode(primaryNode, cloneNode->workerName, cloneNode->workerPort, operation); /* Step 1: Block Writes on Original Primary's Shards */ ereport(NOTICE, (errmsg( "Blocking writes on shards of original primary node %s:%d (group %d)", primaryNode->workerName, primaryNode->workerPort, primaryNode ->groupId))); BlockAllWritesToWorkerNode(primaryNode); /* Step 2: Wait for Clone to Catch Up */ ereport(NOTICE, (errmsg( "Waiting for clone %s:%d to catch up with primary %s:%d (timeout: %d seconds)", cloneNode->workerName, cloneNode->workerPort, primaryNode->workerName, primaryNode->workerPort, catchUpTimeoutSeconds))); bool caughtUp = false; const int sleepIntervalSeconds = 5; int elapsedTimeSeconds = 0; while (elapsedTimeSeconds < catchUpTimeoutSeconds) { uint64 repLag = GetReplicationLag(primaryNode, cloneNode); if (repLag <= 0) { caughtUp = true; break; } pg_usleep(sleepIntervalSeconds * 1000000L); elapsedTimeSeconds += sleepIntervalSeconds; } if (!caughtUp) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Clone %s:%d failed to catch up with primary %s:%d within %d seconds.", cloneNode->workerName, cloneNode->workerPort, primaryNode->workerName, primaryNode->workerPort, catchUpTimeoutSeconds))); } ereport(NOTICE, (errmsg("Clone %s:%d is now caught up with primary %s:%d.", cloneNode->workerName, cloneNode->workerPort, primaryNode->workerName, primaryNode->workerPort))); /* Step 3: PostgreSQL Clone Promotion */ ereport(NOTICE, (errmsg("Attempting to promote clone %s:%d via pg_promote().", cloneNode->workerName, cloneNode->workerPort))); PromoteCloneNode(cloneNode); /* Step 4: Update Clone Metadata in pg_dist_node on Coordinator */ ereport(NOTICE, (errmsg("Updating metadata for promoted clone %s:%d (ID %d)", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); ActivateCloneNodeAsPrimary(cloneNode); /* We need to sync metadata changes to all nodes before rebalancing shards * since the rebalancing algorithm depends on the latest metadata. */ SyncNodeMetadataToNodes(); /* Step 5: Split Shards Between Primary and Clone */ SplitShardsBetweenPrimaryAndClone(primaryNode, cloneNode, PG_GETARG_NAME_OR_NULL(1)) ; TransactionModifiedNodeMetadata = true; /* Inform Citus about metadata change */ TriggerNodeMetadataSyncOnCommit(); /* Ensure changes are propagated */ ereport(NOTICE, (errmsg( "Clone node %s:%d (ID %d) metadata updated. It is now a primary", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); /* Step 6: Unblock Writes (should be handled by transaction commit) */ ereport(NOTICE, (errmsg( "Clone node %s:%d (ID %d) successfully registered as a worker node", cloneNode->workerName, cloneNode->workerPort, cloneNode-> nodeId))); PG_RETURN_VOID(); } /* * PromoteCloneNode promotes a clone node to a primary node using PostgreSQL's * pg_promote() function. * * This function performs the following steps: * 1. Connects to the clone node * 2. Executes pg_promote(wait := true) to promote the clone to primary * 3. Reconnects to verify the promotion was successful * 4. Checks if the node is still in recovery mode (which would indicate failure) * * The function throws an ERROR if: * - Connection to the clone node fails * - The pg_promote() command fails * - The clone is still in recovery mode after promotion attempt * * On success, it logs a NOTICE message confirming the promotion. * * Note: This function assumes the clone has already been validated for promotion * (e.g., replication lag is acceptable, clone is not synchronous, etc.) */ static void PromoteCloneNode(WorkerNode *cloneWorkerNode) { /* Step 1: Connect to the clone node */ int connectionFlag = 0; MultiConnection *cloneConnection = GetNodeConnection(connectionFlag, cloneWorkerNode->workerName, cloneWorkerNode->workerPort); if (PQstatus(cloneConnection->pgConn) != CONNECTION_OK) { ReportConnectionError(cloneConnection, ERROR); } /* Step 2: Execute pg_promote() to promote the clone to primary */ const char *promoteQuery = "SELECT pg_promote(wait := true);"; int resultCode = SendRemoteCommand(cloneConnection, promoteQuery); if (resultCode == 0) { ReportConnectionError(cloneConnection, ERROR); } ForgetResults(cloneConnection); CloseConnection(cloneConnection); /* Step 3: Reconnect and verify the promotion was successful */ if (GetNodeIsInRecoveryStatus(cloneWorkerNode)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Failed to promote clone %s:%d (ID %d). It is still in recovery.", cloneWorkerNode->workerName, cloneWorkerNode->workerPort, cloneWorkerNode->nodeId))); } else { ereport(NOTICE, (errmsg( "Clone node %s:%d (ID %d) has been successfully promoted.", cloneWorkerNode->workerName, cloneWorkerNode->workerPort, cloneWorkerNode->nodeId))); } } static void BlockAllWritesToWorkerNode(WorkerNode *workerNode) { ereport(NOTICE, (errmsg("Blocking all writes to worker node %s:%d (ID %d)", workerNode->workerName, workerNode->workerPort, workerNode-> nodeId))); LockShardsInWorkerPlacementList(workerNode, AccessExclusiveLock); } /* * GetNodeIsInRecoveryStatus checks if a PostgreSQL node is currently in recovery mode. * * This function connects to the specified worker node and executes pg_is_in_recovery() * to determine if the node is still acting as a replica (in recovery) or has been * promoted to a primary (not in recovery). * * Arguments: * - workerNode: The WorkerNode to check recovery status for * * Returns: * - true if the node is in recovery mode (acting as a replica) * - false if the node is not in recovery mode (acting as a primary) * * The function will ERROR if: * - Cannot establish connection to the node * - The remote query fails * - The query result cannot be parsed * * This is used after promoting a clone node to verify that the * promotion was successful and the node is no longer in recovery mode. */ static bool GetNodeIsInRecoveryStatus(WorkerNode *workerNode) { int connectionFlag = 0; MultiConnection *nodeConnection = GetNodeConnection(connectionFlag, workerNode->workerName, workerNode->workerPort); if (PQstatus(nodeConnection->pgConn) != CONNECTION_OK) { ReportConnectionError(nodeConnection, ERROR); } const char *recoveryQuery = "SELECT pg_is_in_recovery();"; int resultCode = SendRemoteCommand(nodeConnection, recoveryQuery); if (resultCode == 0) { ReportConnectionError(nodeConnection, ERROR); } PGresult *result = GetRemoteCommandResult(nodeConnection, true); if (!IsResponseOK(result)) { ReportResultError(nodeConnection, result, ERROR); } List *recoveryStatusList = ReadFirstColumnAsText(result); if (list_length(recoveryStatusList) != 1) { PQclear(result); ClearResults(nodeConnection, true); CloseConnection(nodeConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse recovery status result from %s:%d", workerNode->workerName, workerNode->workerPort))); } StringInfo recoveryStatusInfo = (StringInfo) linitial(recoveryStatusList); bool isInRecovery = (strcmp(recoveryStatusInfo->data, "t") == 0) || (strcmp( recoveryStatusInfo ->data, "true") == 0) ; PQclear(result); ForgetResults(nodeConnection); CloseConnection(nodeConnection); return isInRecovery; } /* * EnsureSingleNodePromotion ensures that only one node promotion operation * can proceed at a time by acquiring necessary locks and checking for * conflicting operations. * * This function performs the following safety checks: * 1. Verifies no rebalance operations are currently running, as they would * conflict with the shard redistribution that occurs during promotion * 2. Acquires exclusive placement colocation locks on all shards residing * on the primary node's group to prevent concurrent shard operations * * The locks are acquired in shard ID order to prevent deadlocks when * multiple operations attempt to lock the same set of shards. * * Arguments: * - primaryNode: The primary node whose shards need to be locked * * Throws ERROR if: * - A rebalance operation is already running * - Unable to acquire necessary locks */ static void EnsureSingleNodePromotion(WorkerNode *primaryNode) { /* Error out if some rebalancer is running */ int64 jobId = 0; if (HasNonTerminalJobOfType("rebalance", &jobId)) { ereport(ERROR, ( errmsg("A rebalance operation is already running as job %ld", jobId), errdetail("A rebalance was already scheduled as background job"), errhint("To monitor progress, run: SELECT * FROM " "citus_rebalance_status();"))); } List *placementList = AllShardPlacementsOnNodeGroup(primaryNode->groupId); /* lock shards in order of shard id to prevent deadlock */ placementList = SortList(placementList, CompareShardPlacementsByShardId); GroupShardPlacement *placement = NULL; foreach_declared_ptr(placement, placementList) { int64 shardId = placement->shardId; ShardInterval *shardInterval = LoadShardInterval(shardId); Oid distributedTableId = shardInterval->relationId; AcquirePlacementColocationLock(distributedTableId, ExclusiveLock, "promote clone") ; } } ================================================ FILE: src/backend/distributed/operations/node_protocol.c ================================================ /*------------------------------------------------------------------------- * * node_protocol.c * Routines for requesting information from the master node for creating or * updating shards. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" #include "access/attnum.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/stratnum.h" #include "access/sysattr.h" #include "access/tupdesc.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_index.h" #include "catalog/pg_namespace.h" #include "catalog/pg_seclabel.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "utils/relcache.h" #include "utils/ruleutils.h" #include "utils/varlena.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/namespace_utils.h" #include "distributed/pg_dist_shard.h" #include "distributed/shared_library_init.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" /* Shard related configuration */ int ShardCount = 32; int ShardReplicationFactor = 1; /* desired replication factor for shards */ int NextShardId = 0; int NextPlacementId = 0; static void GatherIndexAndConstraintDefinitionListExcludingReplicaIdentity(Form_pg_index indexForm, List ** indexDDLEventList, int indexFlags); static Datum WorkerNodeGetDatum(WorkerNode *workerNode, TupleDesc tupleDescriptor); static char * CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, const ColumnarOptions *options); static char * GetTableDDLCommandColumnar(void *context); static TableDDLCommand * ColumnarGetTableOptionsDDL(Oid relationId); static List * CreateSecurityLabelCommands(Oid relationId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_get_table_metadata); PG_FUNCTION_INFO_V1(master_get_table_ddl_events); PG_FUNCTION_INFO_V1(master_get_new_shardid); PG_FUNCTION_INFO_V1(master_get_new_placementid); PG_FUNCTION_INFO_V1(master_get_active_worker_nodes); PG_FUNCTION_INFO_V1(citus_get_active_worker_nodes); PG_FUNCTION_INFO_V1(master_get_round_robin_candidate_nodes); PG_FUNCTION_INFO_V1(master_stage_shard_row); PG_FUNCTION_INFO_V1(master_stage_shard_placement_row); /* * master_get_table_metadata is a deprecated UDF. */ Datum master_get_table_metadata(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("master_get_table_metadata is deprecated"))); } /* * master_get_table_ddl_events takes in a relation name, and returns the set of * DDL commands needed to reconstruct the relation. The returned DDL commands * are similar in flavor to schema definitions that pgdump returns. The function * errors if given relation does not exist. */ Datum master_get_table_ddl_events(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); FuncCallContext *functionContext = NULL; ListCell *tableDDLEventCell = NULL; /* * On the very first call to this function, we first use the given relation * name to get to the relation. We then recreate the list of DDL statements * issued for this relation, and save the first statement's position in the * function context. */ if (SRF_IS_FIRSTCALL()) { text *relationName = PG_GETARG_TEXT_P(0); Oid relationId = ResolveRelationId(relationName, false); IncludeSequenceDefaults includeSequenceDefaults = NEXTVAL_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = INCLUDE_IDENTITY; /* create a function context for cross-call persistence */ functionContext = SRF_FIRSTCALL_INIT(); /* switch to memory context appropriate for multiple function calls */ MemoryContext oldContext = MemoryContextSwitchTo( functionContext->multi_call_memory_ctx); /* allocate DDL statements, and then save position in DDL statements */ bool creatingShellTableOnRemoteNode = false; List *tableDDLEventList = GetFullTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, creatingShellTableOnRemoteNode); tableDDLEventCell = list_head(tableDDLEventList); ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); wrapper->list = tableDDLEventList; wrapper->listCell = tableDDLEventCell; functionContext->user_fctx = wrapper; MemoryContextSwitchTo(oldContext); } /* * On every call to this function, we get the current position in the * statement list. We then iterate to the next position in the list and * return the current statement, if we have not yet reached the end of * list. */ functionContext = SRF_PERCALL_SETUP(); ListCellAndListWrapper *wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { TableDDLCommand *ddlStatement = (TableDDLCommand *) lfirst(wrapper->listCell); Assert(CitusIsA(ddlStatement, TableDDLCommand)); text *ddlStatementText = cstring_to_text(GetTableDDLCommand(ddlStatement)); wrapper->listCell = lnext(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(ddlStatementText)); } else { SRF_RETURN_DONE(functionContext); } } /* * master_get_new_shardid is a user facing wrapper function around GetNextShardId() * which allocates and returns a unique shardId for the shard to be created. * * NB: This can be called by any user; for now we have decided that that's * ok. We might want to restrict this to users part of a specific role or such * at some later point. */ Datum master_get_new_shardid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); uint64 shardId = GetNextShardId(); Datum shardIdDatum = Int64GetDatum(shardId); PG_RETURN_DATUM(shardIdDatum); } /* * GetNextShardId allocates and returns a unique shardId for the shard to be * created. This allocation occurs both in shared memory and in write ahead * logs; writing to logs avoids the risk of having shardId collisions. * * Please note that the caller is still responsible for finalizing shard data * and the shardId with the master node. */ uint64 GetNextShardId() { Oid savedUserId = InvalidOid; int savedSecurityContext = 0; uint64 shardId = 0; /* * In regression tests, we would like to generate shard IDs consistently * even if the tests run in parallel. Instead of the sequence, we can use * the next_shard_id GUC to specify which shard ID the current session should * generate next. The GUC is automatically increased by 1 every time a new * shard ID is generated. */ if (NextShardId > 0) { shardId = NextShardId; NextShardId += 1; return shardId; } text *sequenceName = cstring_to_text(SHARDID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); /* generate new and unique shardId from sequence */ Datum shardIdDatum = DirectFunctionCall1(nextval_oid, sequenceIdDatum); SetUserIdAndSecContext(savedUserId, savedSecurityContext); shardId = DatumGetInt64(shardIdDatum); return shardId; } /* * master_get_new_placementid is a user facing wrapper function around * GetNextPlacementId() which allocates and returns a unique placement id for the * placement to be created. * * NB: This can be called by any user; for now we have decided that that's * ok. We might want to restrict this to users part of a specific role or such * at some later point. */ Datum master_get_new_placementid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); uint64 placementId = GetNextPlacementId(); Datum placementIdDatum = Int64GetDatum(placementId); PG_RETURN_DATUM(placementIdDatum); } /* * GetNextPlacementId allocates and returns a unique placementId for * the placement to be created. This allocation occurs both in shared memory * and in write ahead logs; writing to logs avoids the risk of having placementId * collisions. * * NB: This can be called by any user; for now we have decided that that's * ok. We might want to restrict this to users part of a specific role or such * at some later point. */ uint64 GetNextPlacementId(void) { Oid savedUserId = InvalidOid; int savedSecurityContext = 0; uint64 placementId = 0; /* * In regression tests, we would like to generate placement IDs consistently * even if the tests run in parallel. Instead of the sequence, we can use * the next_placement_id GUC to specify which shard ID the current session * should generate next. The GUC is automatically increased by 1 every time * a new placement ID is generated. */ if (NextPlacementId > 0) { placementId = NextPlacementId; NextPlacementId += 1; return placementId; } text *sequenceName = cstring_to_text(PLACEMENTID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); /* generate new and unique placement id from sequence */ Datum placementIdDatum = DirectFunctionCall1(nextval_oid, sequenceIdDatum); SetUserIdAndSecContext(savedUserId, savedSecurityContext); placementId = DatumGetInt64(placementIdDatum); return placementId; } /* * master_get_round_robin_candidate_nodes is a stub UDF to make pg_upgrade * work flawlessly while upgrading servers from 6.1. This implementation * will be removed after the UDF dropped on the sql side properly. */ Datum master_get_round_robin_candidate_nodes(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("this function is deprecated and no longer is used"))); } /* * master_stage_shard_row is a stub UDF to make pg_upgrade * work flawlessly while upgrading servers from 6.1. This implementation * will be removed after the UDF dropped on the sql side properly. */ Datum master_stage_shard_row(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("this function is deprecated and no longer is used"))); } /* * master_stage_shard_placement_row is a stub UDF to make pg_upgrade * work flawlessly while upgrading servers from 6.1. This implementation * will be removed after the UDF dropped on the sql side properly. */ Datum master_stage_shard_placement_row(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("this function is deprecated and no longer is used"))); } /* * citus_get_active_worker_nodes returns a set of active worker host names and * port numbers in deterministic order. Currently we assume that all worker * nodes in pg_dist_node are active. */ Datum citus_get_active_worker_nodes(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); FuncCallContext *functionContext = NULL; uint32 workerNodeCount = 0; if (SRF_IS_FIRSTCALL()) { /* create a function context for cross-call persistence */ functionContext = SRF_FIRSTCALL_INIT(); /* switch to memory context appropriate for multiple function calls */ MemoryContext oldContext = MemoryContextSwitchTo( functionContext->multi_call_memory_ctx); List *workerNodeList = ActiveReadableNonCoordinatorNodeList(); workerNodeCount = (uint32) list_length(workerNodeList); functionContext->user_fctx = workerNodeList; functionContext->max_calls = workerNodeCount; /* * This tuple descriptor must match the output parameters declared for * the function in pg_proc. */ TupleDesc tupleDescriptor = CreateTemplateTupleDesc(WORKER_NODE_FIELDS); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 1, "node_name", TEXTOID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 2, "node_port", INT8OID, -1, 0); functionContext->tuple_desc = BlessTupleDesc(tupleDescriptor); MemoryContextSwitchTo(oldContext); } functionContext = SRF_PERCALL_SETUP(); uint32 workerNodeIndex = functionContext->call_cntr; workerNodeCount = functionContext->max_calls; if (workerNodeIndex < workerNodeCount) { List *workerNodeList = functionContext->user_fctx; WorkerNode *workerNode = list_nth(workerNodeList, workerNodeIndex); Datum workerNodeDatum = WorkerNodeGetDatum(workerNode, functionContext->tuple_desc); SRF_RETURN_NEXT(functionContext, workerNodeDatum); } else { SRF_RETURN_DONE(functionContext); } } /* * master_get_active_worker_nodes is a wrapper function for old UDF name. */ Datum master_get_active_worker_nodes(PG_FUNCTION_ARGS) { return citus_get_active_worker_nodes(fcinfo); } /* Finds the relationId from a potentially qualified relation name. */ Oid ResolveRelationId(text *relationName, bool missingOk) { /* resolve relationId from passed in schema and relation name */ List *relationNameList = textToQualifiedNameList(relationName); RangeVar *relation = makeRangeVarFromNameList(relationNameList); Oid relationId = RangeVarGetRelid(relation, NoLock, missingOk); return relationId; } /* * GetFullTableCreationCommands takes in a relationId, includeSequenceDefaults, * and returns the list of DDL commands needed to reconstruct the relation. * When includeSequenceDefaults is NEXTVAL_SEQUENCE_DEFAULTS, the function also creates * DEFAULT clauses for columns getting their default values from a sequence. * When it's WORKER_NEXTVAL_SEQUENCE_DEFAULTS, the function creates the DEFAULT * clause using worker_nextval('sequence') and not nextval('sequence') * These DDL commands are all palloced; and include the table's schema * definition, optional column storage and statistics definitions, and index * constraint and trigger definitions. * When IncludeIdentities is NO_IDENTITY, the function does not include identity column * specifications. When it's INCLUDE_IDENTITY it creates GENERATED .. AS IDENTIY clauses. */ List * GetFullTableCreationCommands(Oid relationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, bool creatingShellTableOnRemoteNode) { List *tableDDLEventList = NIL; List *preLoadCreationCommandList = GetPreLoadTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, NULL); tableDDLEventList = list_concat(tableDDLEventList, preLoadCreationCommandList); List *postLoadCreationCommandList = GetPostLoadTableCreationCommands(relationId, true, true); if (creatingShellTableOnRemoteNode) { /* * While creating shell tables, we need to associate dependencies between * sequences and the relation. We also need to add truncate trigger for it * if it is not the foreign table. */ List *sequenceDependencyCommandList = SequenceDependencyCommandList(relationId); tableDDLEventList = list_concat(tableDDLEventList, sequenceDependencyCommandList); if (!IsForeignTable(relationId)) { TableDDLCommand *truncateTriggerCommand = TruncateTriggerCreateCommand( relationId); tableDDLEventList = lappend(tableDDLEventList, truncateTriggerCommand); } /* * For identity column sequences, we only need to modify * their min/max values to produce unique values on the worker nodes. */ List *identitySequenceDependencyCommandList = IdentitySequenceDependencyCommandList(relationId); tableDDLEventList = list_concat(tableDDLEventList, identitySequenceDependencyCommandList); } tableDDLEventList = list_concat(tableDDLEventList, postLoadCreationCommandList); return tableDDLEventList; } /* * GetPostLoadTableCreationCommands takes in a relationId and returns the list * of DDL commands that should be applied after loading the data. */ List * GetPostLoadTableCreationCommands(Oid relationId, bool includeIndexes, bool includeReplicaIdentity) { List *tableDDLEventList = NIL; /* * Include all the commands (e.g., create index, set index clustered * and set index statistics) regarding the indexes. Note that * running all these commands in parallel might fail as the * latter two depends on the first one. So, the caller should * execute the commands sequentially. */ int indexFlags = INCLUDE_INDEX_ALL_STATEMENTS; if (includeIndexes && includeReplicaIdentity) { List *indexAndConstraintCommandList = GetTableIndexAndConstraintCommands(relationId, indexFlags); tableDDLEventList = list_concat(tableDDLEventList, indexAndConstraintCommandList); } else if (includeIndexes && !includeReplicaIdentity) { /* * Do not include the indexes/constraints that backs * replica identity, if any. */ List *indexAndConstraintCommandList = GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(relationId, indexFlags); tableDDLEventList = list_concat(tableDDLEventList, indexAndConstraintCommandList); } if (includeReplicaIdentity) { List *replicaIdentityEvents = GetTableReplicaIdentityCommand(relationId); tableDDLEventList = list_concat(tableDDLEventList, replicaIdentityEvents); } List *triggerCommands = GetExplicitTriggerCommandList(relationId); tableDDLEventList = list_concat(tableDDLEventList, triggerCommands); List *statisticsCommands = GetExplicitStatisticsCommandList(relationId); tableDDLEventList = list_concat(tableDDLEventList, statisticsCommands); return tableDDLEventList; } /* * GetTableReplicaIdentityCommand returns the list of DDL commands to * (re)define the replica identity choice for a given table. */ List * GetTableReplicaIdentityCommand(Oid relationId) { List *replicaIdentityCreateCommandList = NIL; /* * We skip non-relations because postgres does not support * ALTER TABLE .. REPLICA IDENTITY on non-relations. */ char relationKind = get_rel_relkind(relationId); if (relationKind != RELKIND_RELATION) { return NIL; } char *replicaIdentityCreateCommand = pg_get_replica_identity_command(relationId); if (replicaIdentityCreateCommand) { replicaIdentityCreateCommandList = lappend( replicaIdentityCreateCommandList, makeTableDDLCommandString(replicaIdentityCreateCommand)); } return replicaIdentityCreateCommandList; } /* * GetPreLoadTableCreationCommands takes in a relationId, and returns the list of DDL * commands needed to reconstruct the relation, excluding indexes and constraints, * to facilitate faster data load. */ List * GetPreLoadTableCreationCommands(Oid relationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, char *accessMethod) { List *tableDDLEventList = NIL; int saveNestLevel = PushEmptySearchPath(); /* fetch table schema and column option definitions */ char *tableSchemaDef = pg_get_tableschemadef_string(relationId, includeSequenceDefaults, includeIdentityDefaults, accessMethod); char *tableColumnOptionsDef = pg_get_tablecolumnoptionsdef_string(relationId); tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString( tableSchemaDef)); if (tableColumnOptionsDef != NULL) { tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString( tableColumnOptionsDef)); } /* add columnar options for cstore tables */ if (accessMethod == NULL && extern_IsColumnarTableAmTable(relationId)) { TableDDLCommand *cstoreOptionsDDL = ColumnarGetTableOptionsDDL(relationId); if (cstoreOptionsDDL != NULL) { tableDDLEventList = lappend(tableDDLEventList, cstoreOptionsDDL); } } List *tableACLList = pg_get_table_grants(relationId); if (tableACLList != NIL) { char *tableACLCommand = NULL; foreach_declared_ptr(tableACLCommand, tableACLList) { tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString(tableACLCommand)); } } char *tableOwnerDef = TableOwnerResetCommand(relationId); if (tableOwnerDef != NULL) { tableDDLEventList = lappend(tableDDLEventList, makeTableDDLCommandString( tableOwnerDef)); } List *tableRowLevelSecurityCommands = GetTableRowLevelSecurityCommands(relationId); tableDDLEventList = list_concat(tableDDLEventList, tableRowLevelSecurityCommands); List *policyCommands = CreatePolicyCommands(relationId); tableDDLEventList = list_concat(tableDDLEventList, policyCommands); List *securityLabelCommands = CreateSecurityLabelCommands(relationId); tableDDLEventList = list_concat(tableDDLEventList, securityLabelCommands); /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); return tableDDLEventList; } /* * GetTableIndexAndConstraintCommands returns the list of DDL commands to * (re)create indexes and constraints for a given table. */ List * GetTableIndexAndConstraintCommands(Oid relationId, int indexFlags) { return ExecuteFunctionOnEachTableIndex(relationId, GatherIndexAndConstraintDefinitionList, indexFlags); } /* * GetTableIndexAndConstraintCommands returns the list of DDL commands to * (re)create indexes and constraints for a given table. */ List * GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(Oid relationId, int indexFlags) { return ExecuteFunctionOnEachTableIndex(relationId, GatherIndexAndConstraintDefinitionListExcludingReplicaIdentity, indexFlags); } /* * GatherIndexAndConstraintDefinitionListExcludingReplicaIdentity is a wrapper around * GatherIndexAndConstraintDefinitionList(), which only excludes the indexes or * constraints that back the replica identity. */ static void GatherIndexAndConstraintDefinitionListExcludingReplicaIdentity(Form_pg_index indexForm, List **indexDDLEventList, int indexFlags) { Oid relationId = indexForm->indrelid; Relation relation = table_open(relationId, AccessShareLock); Oid replicaIdentityIndex = GetRelationIdentityOrPK(relation); if (replicaIdentityIndex == indexForm->indexrelid) { /* this index is backing the replica identity, so skip */ table_close(relation, NoLock); return; } GatherIndexAndConstraintDefinitionList(indexForm, indexDDLEventList, indexFlags); table_close(relation, NoLock); } /* * Get replica identity index or if it is not defined a primary key. * * If neither is defined, returns InvalidOid. * * Inspired from postgres/src/backend/replication/logical/worker.c */ Oid GetRelationIdentityOrPK(Relation rel) { Oid idxoid = RelationGetReplicaIndex(rel); if (!OidIsValid(idxoid)) { /* Determine the index OID of the primary key (PG18 adds a second parameter) */ #if PG_VERSION_NUM >= PG_VERSION_18 idxoid = RelationGetPrimaryKeyIndex(rel, false /* deferred_ok */); #else idxoid = RelationGetPrimaryKeyIndex(rel); #endif } return idxoid; } /* * GatherIndexAndConstraintDefinitionList adds the DDL command for the given index. */ void GatherIndexAndConstraintDefinitionList(Form_pg_index indexForm, List **indexDDLEventList, int indexFlags) { /* generate fully-qualified names */ int saveNestLevel = PushEmptySearchPath(); Oid indexId = indexForm->indexrelid; bool indexImpliedByConstraint = IndexImpliedByAConstraint(indexForm); /* get the corresponding constraint or index statement */ if (indexImpliedByConstraint) { if (indexFlags & INCLUDE_CREATE_CONSTRAINT_STATEMENTS) { Oid constraintId = get_index_constraint(indexId); Assert(constraintId != InvalidOid); /* include constraints backed by indexes only when explicitly asked */ char *statementDef = pg_get_constraintdef_command(constraintId); *indexDDLEventList = lappend(*indexDDLEventList, makeTableDDLCommandString(statementDef)); } } else if (indexFlags & INCLUDE_CREATE_INDEX_STATEMENTS) { /* * Include indexes that are not backing constraints only when * explicitly asked. */ char *statementDef = pg_get_indexdef_string(indexId); *indexDDLEventList = lappend(*indexDDLEventList, makeTableDDLCommandString(statementDef)); } /* if table is clustered on this index, append definition to the list */ if ((indexFlags & INCLUDE_INDEX_CLUSTERED_STATEMENTS) && indexForm->indisclustered) { char *clusteredDef = pg_get_indexclusterdef_string(indexId); Assert(clusteredDef != NULL); *indexDDLEventList = lappend(*indexDDLEventList, makeTableDDLCommandString( clusteredDef)); } /* we need alter index commands for altered targets on expression indexes */ if (indexFlags & INCLUDE_INDEX_STATISTICS_STATEMENTTS) { List *alterIndexStatisticsCommands = GetAlterIndexStatisticsCommands(indexId); *indexDDLEventList = list_concat(*indexDDLEventList, alterIndexStatisticsCommands); } /* revert back to original search_path */ PopEmptySearchPath(saveNestLevel); } /* * GetTableRowLevelSecurityCommands takes in a relationId, and returns the list of * commands needed to reconstruct the row level security policy. */ List * GetTableRowLevelSecurityCommands(Oid relationId) { List *rowLevelSecurityCommandList = NIL; List *rowLevelSecurityEnableCommands = pg_get_row_level_security_commands(relationId); char *rowLevelSecurityCommand = NULL; foreach_declared_ptr(rowLevelSecurityCommand, rowLevelSecurityEnableCommands) { rowLevelSecurityCommandList = lappend( rowLevelSecurityCommandList, makeTableDDLCommandString(rowLevelSecurityCommand)); } return rowLevelSecurityCommandList; } /* * CreateSecurityLabelCommands - return the SECURITY LABEL commands on * the table identified by relationId. It is used by GetPreLoadTableCreationCommands() * to reconstruct the security labels on the table and its columns. */ static List * CreateSecurityLabelCommands(Oid relationId) { List *securityLabelCommands = NIL; if (!RegularTable(relationId)) /* should be an Assert ? */ { return securityLabelCommands; } Relation pg_seclabel = table_open(SecLabelRelationId, AccessShareLock); ScanKeyData skey[1]; ScanKeyInit(&skey[0], Anum_pg_seclabel_objoid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scan = systable_beginscan(pg_seclabel, SecLabelObjectIndexId, true, NULL, 1, &skey[0]); HeapTuple tuple = NULL; List *table_name = NIL; Relation relation = NULL; TupleDesc tupleDescriptor = NULL; List *securityLabelStmts = NULL; ListCell *lc; while (HeapTupleIsValid(tuple = systable_getnext(scan))) { SecLabelStmt *secLabelStmt = makeNode(SecLabelStmt); if (relation == NULL) { relation = relation_open(relationId, AccessShareLock); if (!RelationIsVisible(relationId)) { char *nsname = get_namespace_name(RelationGetNamespace(relation)); table_name = lappend(table_name, makeString(nsname)); } char *relname = get_rel_name(relationId); table_name = lappend(table_name, makeString(relname)); } Datum datumArray[Natts_pg_seclabel]; bool isNullArray[Natts_pg_seclabel]; heap_deform_tuple(tuple, RelationGetDescr(pg_seclabel), datumArray, isNullArray); int subObjectId = DatumGetInt32( datumArray[Anum_pg_seclabel_objsubid - 1]); secLabelStmt->provider = TextDatumGetCString( datumArray[Anum_pg_seclabel_provider - 1]); secLabelStmt->label = TextDatumGetCString( datumArray[Anum_pg_seclabel_label - 1]); if (subObjectId > 0) { /* Its a column; construct the name */ secLabelStmt->objtype = OBJECT_COLUMN; List *col_name = list_copy(table_name); if (tupleDescriptor == NULL) { tupleDescriptor = RelationGetDescr(relation); } Form_pg_attribute attrForm = TupleDescAttr(tupleDescriptor, subObjectId - 1); char *attributeName = NameStr(attrForm->attname); col_name = lappend(col_name, makeString(attributeName)); secLabelStmt->object = (Node *) col_name; } else { Assert(subObjectId == 0); secLabelStmt->objtype = OBJECT_TABLE; secLabelStmt->object = (Node *) table_name; } securityLabelStmts = lappend(securityLabelStmts, secLabelStmt); } foreach(lc, securityLabelStmts) { Node *stmt = (Node *) lfirst(lc); char *secLabelStmtString = DeparseTreeNode(stmt); TableDDLCommand *secLabelCommand = makeTableDDLCommandString(secLabelStmtString); securityLabelCommands = lappend(securityLabelCommands, secLabelCommand); } systable_endscan(scan); table_close(pg_seclabel, AccessShareLock); if (relation != NULL) { relation_close(relation, AccessShareLock); } return securityLabelCommands; } /* * IndexImpliedByAConstraint is a helper function to be used while scanning * pg_index. It returns true if the index identified by the given indexForm is * implied by a constraint. Note that caller is responsible for passing a valid * indexFrom, which means an alive heap tuple which is of form Form_pg_index. */ bool IndexImpliedByAConstraint(Form_pg_index indexForm) { Assert(indexForm != NULL); bool indexImpliedByConstraint = false; /* * A primary key index is always created by a constraint statement. * A unique key index or exclusion index is created by a constraint * if and only if the index has a corresponding constraint entry in * pg_depend. Any other index form is never associated with a constraint. */ if (indexForm->indisprimary) { indexImpliedByConstraint = true; } else if (indexForm->indisunique || indexForm->indisexclusion) { Oid constraintId = get_index_constraint(indexForm->indexrelid); indexImpliedByConstraint = OidIsValid(constraintId); } return indexImpliedByConstraint; } /* * ShardStorageType returns the shard storage type according to relation type. */ char ShardStorageType(Oid relationId) { char shardStorageType = 0; char relationType = get_rel_relkind(relationId); if (RegularTable(relationId)) { shardStorageType = SHARD_STORAGE_TABLE; } else if (relationType == RELKIND_FOREIGN_TABLE) { shardStorageType = SHARD_STORAGE_FOREIGN; } else { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unexpected relation type: %c", relationType))); } return shardStorageType; } /* * IsCoordinator function returns true if this node is identified as the * schema/coordinator/master node of the cluster. */ bool IsCoordinator(void) { return (GetLocalGroupId() == COORDINATOR_GROUP_ID); } /* * WorkerNodeGetDatum converts the worker node passed to it into its datum * representation. To do this, the function first creates the heap tuple from * the worker node name and port. Then, the function converts the heap tuple * into a datum and returns it. */ static Datum WorkerNodeGetDatum(WorkerNode *workerNode, TupleDesc tupleDescriptor) { Datum values[WORKER_NODE_FIELDS]; bool isNulls[WORKER_NODE_FIELDS]; memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[0] = CStringGetTextDatum(workerNode->workerName); values[1] = Int64GetDatum((int64) workerNode->workerPort); HeapTuple workerNodeTuple = heap_form_tuple(tupleDescriptor, values, isNulls); Datum workerNodeDatum = HeapTupleGetDatum(workerNodeTuple); return workerNodeDatum; } /* * DistributedTableReplicationIsEnabled returns true if distributed table shards * are replicated according to ShardReplicationFactor. */ bool DistributedTableReplicationIsEnabled() { return (ShardReplicationFactor > 1); } /* * makeTableDDLCommandString creates a TableDDLCommand based on a constant string. If the * TableDDLCommand is turned into a sharded table command the constant will be wrapped in * worker_apply_shard_ddl_command with the target shardId. If the command applies to an * un-sharded table (eg. mx) the command is applied as is. */ TableDDLCommand * makeTableDDLCommandString(char *commandStr) { TableDDLCommand *command = CitusMakeNode(TableDDLCommand); command->type = TABLE_DDL_COMMAND_STRING; command->commandStr = commandStr; return command; } /* * makeTableDDLCommandString creates an implementation of TableDDLCommand that creates the * final sql command based on function pointers being passed. */ TableDDLCommand * makeTableDDLCommandFunction(TableDDLFunction function, TableDDLShardedFunction shardedFunction, void *context) { TableDDLCommand *command = CitusMakeNode(TableDDLCommand); /* * Function pointers are called later without verifying them not being NULL. Guard * developers from making a mistake with them directly when they could be made. */ Assert(function != NULL); Assert(shardedFunction != NULL); command->type = TABLE_DDL_COMMAND_FUNCTION; command->function.function = function; command->function.shardedFunction = shardedFunction; command->function.context = context; return command; } /* * GetShardedTableDDLCommandString is the internal function for TableDDLCommand objects * created with makeTableDDLCommandString. */ static char * GetShardedTableDDLCommandString(TableDDLCommand *command, uint64 shardId, char *schemaName) { StringInfoData buf = { 0 }; initStringInfo(&buf); Assert(command->type == TABLE_DDL_COMMAND_STRING); char *escapedDDLCommand = quote_literal_cstr(command->commandStr); if (schemaName != NULL && strcmp(schemaName, "public") != 0) { char *escapedSchemaName = quote_literal_cstr(schemaName); appendStringInfo(&buf, WORKER_APPLY_SHARD_DDL_COMMAND, shardId, escapedSchemaName, escapedDDLCommand); } else { appendStringInfo(&buf, WORKER_APPLY_SHARD_DDL_COMMAND_WITHOUT_SCHEMA, shardId, escapedDDLCommand); } return buf.data; } /* * GetTableDDLCommandString is the internal function for TableDDLCommand objects created * with makeTableDDLCommandString to return the non-sharded version of the ddl command. */ static char * GetTableDDLCommandString(TableDDLCommand *command) { Assert(command->type == TABLE_DDL_COMMAND_STRING); return command->commandStr; } /* * GetShardedTableDDLCommand returns the ddl command expressed by this TableDDLCommand * where all applicable names are transformed into the names for a shard identified by * shardId * * schemaName is deprecated but used for TableDDLCommandString. All other implementations * will need to rely solely on the shardId. */ char * GetShardedTableDDLCommand(TableDDLCommand *command, uint64 shardId, char *schemaName) { switch (command->type) { case TABLE_DDL_COMMAND_STRING: { return GetShardedTableDDLCommandString(command, shardId, schemaName); } case TABLE_DDL_COMMAND_FUNCTION: { return command->function.shardedFunction(shardId, command->function.context); } } /* unreachable: compiler should warn/error when not all cases are covered above */ ereport(ERROR, (errmsg("unsupported TableDDLCommand: %d", command->type))); } /* * GetTableDDLCommand returns the ddl command expressed by this TableDDLCommand where all * table names are targeting the base table, not any shards. */ char * GetTableDDLCommand(TableDDLCommand *command) { switch (command->type) { case TABLE_DDL_COMMAND_STRING: { return GetTableDDLCommandString(command); } case TABLE_DDL_COMMAND_FUNCTION: { return command->function.function(command->function.context); } } /* unreachable: compiler should warn/error when not all cases are covered above */ ereport(ERROR, (errmsg("unsupported TableDDLCommand: %d", command->type))); } /* * CitusCreateAlterColumnarTableSet generates a portable */ static char * CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, const ColumnarOptions *options) { StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, "ALTER TABLE %s SET (" "columnar.chunk_group_row_limit = %d, " "columnar.stripe_row_limit = %lu, " "columnar.compression_level = %d, " "columnar.compression = %s);", qualifiedRelationName, options->chunkRowCount, options->stripeRowCount, options->compressionLevel, quote_literal_cstr(extern_CompressionTypeStr( options->compressionType))); return buf.data; } /* * GetTableDDLCommandColumnar is an internal function used to turn a * ColumnarTableDDLContext stored on the context of a TableDDLCommandFunction into a sql * command that will be executed against a table. The resulting command will set the * options of the table to the same options as the relation on the coordinator. */ static char * GetTableDDLCommandColumnar(void *context) { ColumnarTableDDLContext *tableDDLContext = (ColumnarTableDDLContext *) context; char *qualifiedShardName = quote_qualified_identifier(tableDDLContext->schemaName, tableDDLContext->relationName); return CitusCreateAlterColumnarTableSet(qualifiedShardName, &tableDDLContext->options); } /* * GetShardedTableDDLCommandColumnar is an internal function used to turn a * ColumnarTableDDLContext stored on the context of a TableDDLCommandFunction into a sql * command that will be executed against a shard. The resulting command will set the * options of the shard to the same options as the relation the shard is based on. */ char * GetShardedTableDDLCommandColumnar(uint64 shardId, void *context) { ColumnarTableDDLContext *tableDDLContext = (ColumnarTableDDLContext *) context; /* * AppendShardId is destructive of the original cahr *, given we want to serialize * more than once we copy it before appending the shard id. */ char *relationName = pstrdup(tableDDLContext->relationName); AppendShardIdToName(&relationName, shardId); char *qualifiedShardName = quote_qualified_identifier(tableDDLContext->schemaName, relationName); return CitusCreateAlterColumnarTableSet(qualifiedShardName, &tableDDLContext->options); } /* * ColumnarGetCustomTableOptionsDDL returns a TableDDLCommand representing a command that * will apply the passed columnar options to the relation identified by relationId on a * new table or shard. */ TableDDLCommand * ColumnarGetCustomTableOptionsDDL(char *schemaName, char *relationName, ColumnarOptions options) { ColumnarTableDDLContext *context = (ColumnarTableDDLContext *) palloc0( sizeof(ColumnarTableDDLContext)); /* build the context */ context->schemaName = schemaName; context->relationName = relationName; context->options = options; /* create TableDDLCommand based on the context build above */ return makeTableDDLCommandFunction( GetTableDDLCommandColumnar, GetShardedTableDDLCommandColumnar, context); } /* * ColumnarGetTableOptionsDDL returns a TableDDLCommand representing a command that will * apply the columnar options currently applicable to the relation identified by * relationId on a new table or shard. */ static TableDDLCommand * ColumnarGetTableOptionsDDL(Oid relationId) { Oid namespaceId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(namespaceId); char *relationName = get_rel_name(relationId); ColumnarOptions options = { 0 }; extern_ReadColumnarOptions(relationId, &options); return ColumnarGetCustomTableOptionsDDL(schemaName, relationName, options); } ================================================ FILE: src/backend/distributed/operations/partitioning.c ================================================ /*------------------------------------------------------------------------- * * partitioning.c * Functions for dealing with partitioned tables. * * Copyright (c) Microsoft Corporation. All rights reserved. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "funcapi.h" #include "access/htup.h" #include "access/htup_details.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(time_partition_range); /* * time_partition_range returns the lower and upper bound of partition * key values for the partition of a time-partitioned table. */ Datum time_partition_range(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); /* create tuple descriptor for return value */ TupleDesc metadataDescriptor = NULL; TypeFuncClass resultTypeClass = get_call_result_type(fcinfo, NULL, &metadataDescriptor); if (resultTypeClass != TYPEFUNC_COMPOSITE) { ereport(ERROR, (errmsg("return type must be a row type"))); } /* get the pg_class record */ HeapTuple tuple = SearchSysCache1(RELOID, relationId); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errmsg("relation with OID %u does not exist", relationId))); } /* get the pg_class record */ bool isNull = false; Datum partitionBoundDatum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relpartbound, &isNull); if (isNull) { ereport(ERROR, (errmsg("relation \"%s\" is not a partition", get_rel_name(relationId)))); } PartitionBoundSpec *partitionBoundSpec = (PartitionBoundSpec *) stringToNode(TextDatumGetCString(partitionBoundDatum)); if (!IsA(partitionBoundSpec, PartitionBoundSpec)) { ereport(ERROR, (errmsg("expected PartitionBoundSpec"))); } if (partitionBoundSpec->strategy != PARTITION_STRATEGY_RANGE) { ereport(ERROR, (errmsg("relation \"%s\" is not a range partition", get_rel_name(relationId)), errdetail("time_partition_range can only be used for " "partitions of range-partitioned tables with a single " "partition column"))); } Datum values[2]; bool isNulls[2]; memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); if (partitionBoundSpec->is_default) { /* return NULL for default partition */ isNulls[0] = true; isNulls[1] = true; } else { if (list_length(partitionBoundSpec->lowerdatums) != 1 || list_length(partitionBoundSpec->upperdatums) != 1) { ereport(ERROR, (errmsg("relation \"%s\" is a partition with multiple " "partition columns", get_rel_name(relationId)), errdetail("time_partition_range can only be used for " "partitions of range-partitioned tables with a " "single partition column"))); } PartitionRangeDatum *lowerBoundDatum = castNode(PartitionRangeDatum, linitial(partitionBoundSpec->lowerdatums)); PartitionRangeDatum *upperBoundDatum = castNode(PartitionRangeDatum, linitial(partitionBoundSpec->upperdatums)); Const *lowerConst = castNode(Const, lowerBoundDatum->value); Const *upperConst = castNode(Const, upperBoundDatum->value); char *lowerConstStr = DatumToString(lowerConst->constvalue, lowerConst->consttype); char *upperConstStr = DatumToString(upperConst->constvalue, upperConst->consttype); values[0] = CStringGetTextDatum(lowerConstStr); values[1] = CStringGetTextDatum(upperConstStr); } HeapTuple metadataTuple = heap_form_tuple(metadataDescriptor, values, isNulls); Datum metadataDatum = HeapTupleGetDatum(metadataTuple); ReleaseSysCache(tuple); PG_RETURN_DATUM(metadataDatum); } ================================================ FILE: src/backend/distributed/operations/replicate_none_dist_table_shard.c ================================================ /*------------------------------------------------------------------------- * * replicate_none_dist_table_shard.c * Routines to replicate shard of none-distributed table to * a remote node. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "nodes/pg_list.h" #include "distributed/adaptive_executor.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/replicate_none_dist_table_shard.h" #include "distributed/shard_utils.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" static void CreateForeignKeysFromReferenceTablesOnShards(Oid noneDistTableId); static Oid ForeignConstraintGetReferencingTableId(const char *queryString); static void EnsureNoneDistTableWithCoordinatorPlacement(Oid noneDistTableId); static void SetLocalEnableManualChangesToShard(bool state); /* * NoneDistTableReplicateCoordinatorPlacement replicates local (presumably * coordinator) shard placement of given none-distributed table to given * target nodes and inserts records for new placements into pg_dist_placement. */ void NoneDistTableReplicateCoordinatorPlacement(Oid noneDistTableId, List *targetNodeList) { EnsureCoordinator(); EnsureNoneDistTableWithCoordinatorPlacement(noneDistTableId); /* * We don't expect callers try to replicate the shard to remote nodes * if some of the remote nodes have a placement for the shard already. */ int64 shardId = GetFirstShardId(noneDistTableId); List *remoteShardPlacementList = FilterShardPlacementList(ActiveShardPlacementList(shardId), IsRemoteShardPlacement); if (list_length(remoteShardPlacementList) > 0) { ereport(ERROR, (errmsg("table already has a remote shard placement"))); } uint64 shardLength = ShardLength(shardId); /* insert new placements to pg_dist_placement */ List *insertedPlacementList = NIL; WorkerNode *targetNode = NULL; foreach_declared_ptr(targetNode, targetNodeList) { ShardPlacement *shardPlacement = InsertShardPlacementRowGlobally(shardId, GetNextPlacementId(), shardLength, targetNode->groupId); /* and save the placement for shard creation on workers */ insertedPlacementList = lappend(insertedPlacementList, shardPlacement); } /* create new placements */ bool useExclusiveConnection = false; CreateShardsOnWorkers(noneDistTableId, insertedPlacementList, useExclusiveConnection); /* fetch coordinator placement before deleting it */ Oid localPlacementTableId = GetTableLocalShardOid(noneDistTableId, shardId); ShardPlacement *coordinatorPlacement = linitial(ActiveShardPlacementListOnGroup(shardId, COORDINATOR_GROUP_ID)); /* * CreateForeignKeysFromReferenceTablesOnShards and CopyFromLocalTableIntoDistTable * need to ignore the local placement, hence we temporarily delete it before * calling them. */ DeleteShardPlacementRowGlobally(coordinatorPlacement->placementId); /* and copy data from local placement to new placements */ CopyFromLocalTableIntoDistTable( localPlacementTableId, noneDistTableId ); /* * CreateShardsOnWorkers only creates the foreign keys where given relation * is the referencing one, so we need to create the foreign keys where given * relation is the referenced one as well. We're only interested in the cases * where the referencing relation is a reference table because the other * possible table types --i.e., Citus local tables atm-- cannot have placements * on remote nodes. * * Note that we need to create the foreign keys where given relation is the * referenced one after copying the data so that constraint checks can pass. */ CreateForeignKeysFromReferenceTablesOnShards(noneDistTableId); /* using the same placement id, re-insert the deleted placement */ InsertShardPlacementRowGlobally(shardId, coordinatorPlacement->placementId, shardLength, COORDINATOR_GROUP_ID); } /* * NoneDistTableDeleteCoordinatorPlacement deletes pg_dist_placement record for * local (presumably coordinator) shard placement of given none-distributed table. */ void NoneDistTableDeleteCoordinatorPlacement(Oid noneDistTableId) { EnsureCoordinator(); EnsureNoneDistTableWithCoordinatorPlacement(noneDistTableId); int64 shardId = GetFirstShardId(noneDistTableId); /* we've already verified that table has a coordinator placement */ ShardPlacement *coordinatorPlacement = linitial(ActiveShardPlacementListOnGroup(shardId, COORDINATOR_GROUP_ID)); /* remove the old placement from metadata of local node, i.e., coordinator */ DeleteShardPlacementRowGlobally(coordinatorPlacement->placementId); } /* * NoneDistTableDropCoordinatorPlacementTable drops local (presumably coordinator) * shard placement table of given none-distributed table. */ void NoneDistTableDropCoordinatorPlacementTable(Oid noneDistTableId) { EnsureCoordinator(); if (HasDistributionKey(noneDistTableId)) { ereport(ERROR, (errmsg("table is not a none-distributed table"))); } /* * We undistribute Citus local tables that are not chained with any reference * tables via foreign keys at the end of the utility hook. * Here we temporarily set the related GUC to off to disable the logic for * internally executed DDL's that might invoke this mechanism unnecessarily. * * We also temporarily disable citus.enable_manual_changes_to_shards GUC to * allow given command to modify shard. Note that we disable it only for * local session because changes made to shards are allowed for Citus internal * backends anyway. */ int saveNestLevel = NewGUCNestLevel(); SetLocalEnableLocalReferenceForeignKeys(false); SetLocalEnableManualChangesToShard(true); StringInfo dropShardCommand = makeStringInfo(); int64 shardId = GetFirstShardId(noneDistTableId); ShardInterval *shardInterval = LoadShardInterval(shardId); appendStringInfo(dropShardCommand, DROP_REGULAR_TABLE_COMMAND, ConstructQualifiedShardName(shardInterval)); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = INVALID_TASK_ID; task->taskType = DDL_TASK; task->replicationModel = REPLICATION_MODEL_INVALID; SetTaskQueryString(task, dropShardCommand->data); ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(targetPlacement, CoordinatorNodeIfAddedAsWorkerOrError()); task->taskPlacementList = list_make1(targetPlacement); bool localExecutionSupported = true; ExecuteUtilityTaskList(list_make1(task), localExecutionSupported); AtEOXact_GUC(true, saveNestLevel); } /* * CreateForeignKeysFromReferenceTablesOnShards creates foreign keys on shards * where given none-distributed table is the referenced table and the referencing * one is a reference table. */ static void CreateForeignKeysFromReferenceTablesOnShards(Oid noneDistTableId) { EnsureCoordinator(); if (HasDistributionKey(noneDistTableId)) { ereport(ERROR, (errmsg("table is not a none-distributed table"))); } List *ddlCommandList = GetForeignConstraintFromOtherReferenceTablesCommands(noneDistTableId); if (list_length(ddlCommandList) == 0) { return; } List *taskList = NIL; char *command = NULL; foreach_declared_ptr(command, ddlCommandList) { List *commandTaskList = InterShardDDLTaskList( ForeignConstraintGetReferencingTableId(command), noneDistTableId, command ); taskList = list_concat(taskList, commandTaskList); } if (list_length(taskList) == 0) { return; } bool localExecutionSupported = true; ExecuteUtilityTaskList(taskList, localExecutionSupported); } /* * ForeignConstraintGetReferencedTableId parses given foreign constraint command and * extracts refenrencing table id from it. */ static Oid ForeignConstraintGetReferencingTableId(const char *queryString) { Node *queryNode = ParseTreeNode(queryString); if (!IsA(queryNode, AlterTableStmt)) { ereport(ERROR, (errmsg("command is not an ALTER TABLE statement"))); } AlterTableStmt *foreignConstraintStmt = (AlterTableStmt *) queryNode; if (list_length(foreignConstraintStmt->cmds) != 1) { ereport(ERROR, (errmsg("command does not contain a single command"))); } AlterTableCmd *command = (AlterTableCmd *) linitial(foreignConstraintStmt->cmds); if (command->subtype == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; if (constraint && constraint->contype == CONSTR_FOREIGN) { bool missingOk = false; return RangeVarGetRelid(foreignConstraintStmt->relation, NoLock, missingOk); } } ereport(ERROR, (errmsg("command does not contain a foreign constraint"))); } /* * EnsureNoneDistTableWithCoordinatorPlacement throws an error if given * table is not a none-distributed that has a coordinator placement. */ static void EnsureNoneDistTableWithCoordinatorPlacement(Oid noneDistTableId) { if (HasDistributionKey(noneDistTableId)) { ereport(ERROR, (errmsg("table is not a none-distributed table"))); } int64 shardId = GetFirstShardId(noneDistTableId); if (!ActiveShardPlacementListOnGroup(shardId, COORDINATOR_GROUP_ID)) { ereport(ERROR, (errmsg("table does not have a coordinator placement"))); } } /* * SetLocalEnableManualChangesToShard locally enables * citus.enable_manual_changes_to_shards GUC. */ static void SetLocalEnableManualChangesToShard(bool state) { set_config_option("citus.enable_manual_changes_to_shards", state ? "on" : "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } ================================================ FILE: src/backend/distributed/operations/shard_cleaner.c ================================================ /*------------------------------------------------------------------------- * * shard_cleaner.c * This implements the background process that cleans shards and resources * that are left around. * * Copyright (c) 2018, Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/xact.h" #include "catalog/namespace.h" #include "commands/dbcommands.h" #include "commands/sequence.h" #include "nodes/makefuncs.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "distributed/citus_safe_lib.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/pg_dist_cleanup.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/worker_transaction.h" #define REPLICATION_SLOT_CATALOG_TABLE_NAME "pg_replication_slots" #define STR_ERRCODE_OBJECT_IN_USE "55006" #define STR_ERRCODE_UNDEFINED_OBJECT "42704" /* GUC configuration for shard cleaner */ int NextOperationId = 0; int NextCleanupRecordId = 0; /* Data structure for cleanup operation */ /* * CleanupRecord represents a record from pg_dist_cleanup. */ typedef struct CleanupRecord { /* unique identifier of the record */ uint64 recordId; /* identifier of the operation that generated the record */ OperationId operationId; /* type of the object (e.g. shard) */ CleanupObject objectType; /* fully qualified name of the object */ char *objectName; /* node group ID on which the object is located */ int nodeGroupId; /* cleanup policy that determines when object is cleaned */ CleanupPolicy policy; } CleanupRecord; /* operation ID set by RegisterOperationNeedingCleanup */ OperationId CurrentOperationId = INVALID_OPERATION_ID; /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(citus_cleanup_orphaned_shards); PG_FUNCTION_INFO_V1(citus_cleanup_orphaned_resources); PG_FUNCTION_INFO_V1(isolation_cleanup_orphaned_resources); static bool TryDropResourceByCleanupRecordOutsideTransaction(CleanupRecord *record, char *nodeName, int nodePort); static bool TryDropShardOutsideTransaction(char *qualifiedTableName, char *nodeName, int nodePort); static bool TryDropSubscriptionOutsideTransaction(char *subscriptionName, char *nodeName, int nodePort); static bool TryDropPublicationOutsideTransaction(char *publicationName, char *nodeName, int nodePort); static bool TryDropReplicationSlotOutsideTransaction(char *replicationSlotName, char *nodeName, int nodePort); static bool TryDropUserOutsideTransaction(char *username, char *nodeName, int nodePort); static bool TryDropDatabaseOutsideTransaction(char *databaseName, char *nodeName, int nodePort); static CleanupRecord * GetCleanupRecordByNameAndType(char *objectName, CleanupObject type); /* Functions for cleanup infrastructure */ static CleanupRecord * TupleToCleanupRecord(HeapTuple heapTuple, TupleDesc tupleDescriptor); static OperationId GetNextOperationId(void); static uint64 GetNextCleanupRecordId(void); static void LockOperationId(OperationId operationId); static bool TryLockOperationId(OperationId operationId); static void DeleteCleanupRecordByRecordId(uint64 recordId); static void DeleteCleanupRecordByRecordIdOutsideTransaction(uint64 recordId); static bool CleanupRecordExists(uint64 recordId); static List * ListCleanupRecords(void); static List * ListCleanupRecordsForCurrentOperation(void); static int DropOrphanedResourcesForCleanup(void); static int CompareCleanupRecordsByObjectType(const void *leftElement, const void *rightElement); /* * citus_cleanup_orphaned_shards is noop. * Use citus_cleanup_orphaned_resources instead. */ Datum citus_cleanup_orphaned_shards(PG_FUNCTION_ARGS) { ereport(WARNING, (errmsg("citus_cleanup_orphaned_shards is deprecated. " "Use citus_cleanup_orphaned_resources instead"))); PG_RETURN_VOID(); } /* * citus_cleanup_orphaned_resources implements a user-facing UDF to delete * orphaned resources that are present in the system. These resources are * orphaned by previous actions that either failed or marked the resources * for deferred cleanup. * * The function takes no arguments and runs on co-ordinator. It cannot be run in a * transaction, because holding the locks it takes for a long time is not good. * While the locks are held, it is impossible for the background daemon to * perform concurrent cleanup. */ Datum citus_cleanup_orphaned_resources(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PreventInTransactionBlock(true, "citus_cleanup_orphaned_resources"); int droppedCount = DropOrphanedResourcesForCleanup(); if (droppedCount > 0) { ereport(NOTICE, (errmsg("cleaned up %d orphaned resources", droppedCount))); } PG_RETURN_VOID(); } /* * isolation_cleanup_orphaned_resources implements a test UDF that's the same as * citus_cleanup_orphaned_resources. The only difference is that this command can * be run in transactions, this is needed to test this function in isolation tests * since commands are automatically run in transactions there. */ Datum isolation_cleanup_orphaned_resources(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); DropOrphanedResourcesForCleanup(); PG_RETURN_VOID(); } /* * DropOrphanedResourcesInSeparateTransaction cleans up orphaned resources by * connecting to localhost. */ void DropOrphanedResourcesInSeparateTransaction(void) { int connectionFlag = FORCE_NEW_CONNECTION; MultiConnection *connection = GetNodeConnection(connectionFlag, LocalHostName, PostPortNumber); ExecuteCriticalRemoteCommand(connection, "CALL citus_cleanup_orphaned_resources()"); CloseConnection(connection); } /* * TryDropOrphanedResources is a wrapper around DropOrphanedResourcesForCleanup * that catches any errors to make it safe to use in the maintenance daemon. * * If dropping any of the resources failed this function returns -1, otherwise it * returns the number of dropped resources. */ int TryDropOrphanedResources() { int droppedResourceCount = 0; MemoryContext savedContext = CurrentMemoryContext; /* * Start a subtransaction so we can rollback database's state to it in case * of error. */ BeginInternalSubTransaction(NULL); PG_TRY(); { droppedResourceCount = DropOrphanedResourcesForCleanup(); /* * Releasing a subtransaction doesn't free its memory context, since the * data it contains will be needed at upper commit. See the comments for * AtSubCommit_Memory() at postgres/src/backend/access/transam/xact.c. */ ReleaseCurrentSubTransaction(); } PG_CATCH(); { MemoryContextSwitchTo(savedContext); ErrorData *edata = CopyErrorData(); FlushErrorState(); RollbackAndReleaseCurrentSubTransaction(); /* rethrow as WARNING */ edata->elevel = WARNING; ThrowErrorData(edata); } PG_END_TRY(); return droppedResourceCount; } /* * DropOrphanedResourcesForCleanup removes resources that were marked for cleanup by operation. * It does so by trying to take an exclusive lock on the resources. If the lock cannot be * obtained it skips the resource and continues with others. * The resource that has been skipped will be removed at a later iteration when there are no * locks held anymore. */ static int DropOrphanedResourcesForCleanup() { List *cleanupRecordList = ListCleanupRecords(); /* * We sort the records before cleaning up by their types, because of dependencies. * For example, a subscription might depend on a publication. */ cleanupRecordList = SortList(cleanupRecordList, CompareCleanupRecordsByObjectType); int removedResourceCountForCleanup = 0; int failedResourceCountForCleanup = 0; CleanupRecord *record = NULL; foreach_declared_ptr(record, cleanupRecordList) { if (!PrimaryNodeForGroup(record->nodeGroupId, NULL)) { continue; } /* Advisory locks are reentrant */ if (!TryLockOperationId(record->operationId)) { /* operation that the cleanup record is part of is still running */ continue; } char *resourceName = record->objectName; WorkerNode *workerNode = LookupNodeForGroup(record->nodeGroupId); /* * Now that we have the lock, check if record exists. * The operation could have completed successfully just after we called * ListCleanupRecords in which case the record will be now gone. */ if (!CleanupRecordExists(record->recordId)) { continue; } if (TryDropResourceByCleanupRecordOutsideTransaction(record, workerNode->workerName, workerNode->workerPort)) { if (record->policy == CLEANUP_DEFERRED_ON_SUCCESS) { ereport(LOG, (errmsg("deferred drop of orphaned resource %s on %s:%d " "completed", resourceName, workerNode->workerName, workerNode->workerPort))); } else { ereport(LOG, (errmsg("cleaned up orphaned resource %s on %s:%d which " "was left behind after a failed operation", resourceName, workerNode->workerName, workerNode->workerPort))); } /* delete the cleanup record */ DeleteCleanupRecordByRecordId(record->recordId); removedResourceCountForCleanup++; } else { /* * We log failures at the end, since they occur repeatedly * for a large number of objects. */ failedResourceCountForCleanup++; } } if (failedResourceCountForCleanup > 0) { ereport(WARNING, (errmsg("failed to clean up %d orphaned resources out of %d", failedResourceCountForCleanup, list_length(cleanupRecordList)))); } return removedResourceCountForCleanup; } /* * RegisterOperationNeedingCleanup is be called by an operation to register * for cleanup. */ OperationId RegisterOperationNeedingCleanup(void) { CurrentOperationId = GetNextOperationId(); LockOperationId(CurrentOperationId); return CurrentOperationId; } /* * FinalizeOperationNeedingCleanupOnSuccess is be called by an operation to signal * completion with success. This will trigger cleanup of appropriate resources. */ void FinalizeOperationNeedingCleanupOnSuccess(const char *operationName) { /* We must have a valid OperationId. Any operation requring cleanup * will call RegisterOperationNeedingCleanup. */ Assert(CurrentOperationId != INVALID_OPERATION_ID); List *currentOperationRecordList = ListCleanupRecordsForCurrentOperation(); /* * We sort the records before cleaning up by their types, because of dependencies. * For example, a subscription might depend on a publication. */ currentOperationRecordList = SortList(currentOperationRecordList, CompareCleanupRecordsByObjectType); int failedShardCountOnComplete = 0; CleanupRecord *record = NULL; foreach_declared_ptr(record, currentOperationRecordList) { if (record->policy == CLEANUP_ALWAYS) { WorkerNode *workerNode = LookupNodeForGroup(record->nodeGroupId); /* * For all resources of CurrentOperationId that are marked as 'CLEANUP_ALWAYS' * drop resource and cleanup records. */ if (TryDropResourceByCleanupRecordOutsideTransaction(record, workerNode->workerName, workerNode->workerPort)) { /* * Delete cleanup records outside transaction as: * The resources are marked as 'CLEANUP_ALWAYS' and should be cleaned no matter * the operation succeeded or failed. */ DeleteCleanupRecordByRecordIdOutsideTransaction(record->recordId); } else if (record->objectType == CLEANUP_OBJECT_SHARD_PLACEMENT) { /* * We log failures at the end, since they occur repeatedly * for a large number of objects. */ failedShardCountOnComplete++; } } else if (record->policy == CLEANUP_ON_FAILURE) { /* Delete cleanup records (and not the actual resource) in same transaction as: * The resources are marked as 'CLEANUP_ON_FAILURE' and we are approaching a successful * completion of the operation. However, we cannot guarentee that operation will succeed * so we tie the Delete with parent transaction. */ DeleteCleanupRecordByRecordId(record->recordId); } } if (failedShardCountOnComplete > 0) { ereport(WARNING, (errmsg( "failed to clean up %d orphaned shards out of %d after " "a %s operation completed", failedShardCountOnComplete, list_length(currentOperationRecordList), operationName))); } } /* * CompareRecordsByObjectType is a comparison function for sort * cleanup records by their object type. */ static int CompareCleanupRecordsByObjectType(const void *leftElement, const void *rightElement) { CleanupRecord *leftRecord = *((CleanupRecord **) leftElement); CleanupRecord *rightRecord = *((CleanupRecord **) rightElement); /* we compare 64-bit integers, instead of casting their difference to int */ if (leftRecord->objectType > rightRecord->objectType) { return 1; } else if (leftRecord->objectType < rightRecord->objectType) { return -1; } return 0; } /* * InsertCleanupOnSuccessRecordInCurrentTransaction inserts a new pg_dist_cleanup entry * as part of the current transaction. This is primarily useful for deferred drop scenarios, * since these records would roll back in case of operation failure. And for the same reason, * always sets the policy type to CLEANUP_DEFERRED_ON_SUCCESS. */ void InsertCleanupOnSuccessRecordInCurrentTransaction(CleanupObject objectType, char *objectName, int nodeGroupId) { /* We must have a valid OperationId. Any operation requring cleanup * will call RegisterOperationNeedingCleanup. */ Assert(CurrentOperationId != INVALID_OPERATION_ID); Datum values[Natts_pg_dist_cleanup]; bool isNulls[Natts_pg_dist_cleanup]; /* form new shard tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); uint64 recordId = GetNextCleanupRecordId(); OperationId operationId = CurrentOperationId; values[Anum_pg_dist_cleanup_record_id - 1] = UInt64GetDatum(recordId); values[Anum_pg_dist_cleanup_operation_id - 1] = UInt64GetDatum(operationId); values[Anum_pg_dist_cleanup_object_type - 1] = Int32GetDatum(objectType); values[Anum_pg_dist_cleanup_object_name - 1] = CStringGetTextDatum(objectName); values[Anum_pg_dist_cleanup_node_group_id - 1] = Int32GetDatum(nodeGroupId); values[Anum_pg_dist_cleanup_policy_type - 1] = Int32GetDatum(CLEANUP_DEFERRED_ON_SUCCESS); /* open cleanup relation and insert new tuple */ Oid relationId = DistCleanupRelationId(); Relation pgDistCleanup = table_open(relationId, RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistCleanup); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); CatalogTupleInsert(pgDistCleanup, heapTuple); CommandCounterIncrement(); table_close(pgDistCleanup, NoLock); } /* * InsertCleanupRecordOutsideTransaction inserts a new pg_dist_cleanup entry in a * separate transaction to ensure the record persists after rollback. We should * delete these records if the operation completes successfully. * * This is used in scenarios where we need to cleanup resources on operation * completion (CLEANUP_ALWAYS) or on failure (CLEANUP_ON_FAILURE). */ void InsertCleanupRecordOutsideTransaction(CleanupObject objectType, char *objectName, int nodeGroupId, CleanupPolicy policy) { /* We must have a valid OperationId. Any operation requring cleanup * will call RegisterOperationNeedingCleanup. */ Assert(CurrentOperationId != INVALID_OPERATION_ID); /* assert the circumstance noted in function comment */ Assert(policy == CLEANUP_ALWAYS || policy == CLEANUP_ON_FAILURE); StringInfo sequenceName = makeStringInfo(); appendStringInfo(sequenceName, "%s.%s", PG_CATALOG, CLEANUPRECORDID_SEQUENCE_NAME); StringInfo command = makeStringInfo(); appendStringInfo(command, "INSERT INTO %s.%s " " (record_id, operation_id, object_type, object_name, node_group_id, policy_type) " " VALUES ( nextval('%s'), " UINT64_FORMAT ", %d, %s, %d, %d)", PG_CATALOG, PG_DIST_CLEANUP, sequenceName->data, CurrentOperationId, objectType, quote_literal_cstr(objectName), nodeGroupId, policy); MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(CitusExtensionOwnerName()); SendCommandListToWorkerOutsideTransactionWithConnection(connection, list_make1(command->data)); } /* * DeleteCleanupRecordByRecordIdOutsideTransaction deletes a cleanup record by record id. */ static void DeleteCleanupRecordByRecordIdOutsideTransaction(uint64 recordId) { StringInfo command = makeStringInfo(); appendStringInfo(command, "DELETE FROM %s.%s " "WHERE record_id = %lu", PG_CATALOG, PG_DIST_CLEANUP, recordId); MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction( CitusExtensionOwnerName()); SendCommandListToWorkerOutsideTransactionWithConnection(connection, list_make1(command->data)); } /* * TryDropResourceByCleanupRecordOutsideTransaction tries to drop the given resource * and returns true on success. */ static bool TryDropResourceByCleanupRecordOutsideTransaction(CleanupRecord *record, char *nodeName, int nodePort) { switch (record->objectType) { case CLEANUP_OBJECT_SHARD_PLACEMENT: { return TryDropShardOutsideTransaction(record->objectName, nodeName, nodePort); } case CLEANUP_OBJECT_SUBSCRIPTION: { return TryDropSubscriptionOutsideTransaction(record->objectName, nodeName, nodePort); } case CLEANUP_OBJECT_PUBLICATION: { return TryDropPublicationOutsideTransaction(record->objectName, nodeName, nodePort); } case CLEANUP_OBJECT_REPLICATION_SLOT: { return TryDropReplicationSlotOutsideTransaction(record->objectName, nodeName, nodePort); } case CLEANUP_OBJECT_USER: { return TryDropUserOutsideTransaction(record->objectName, nodeName, nodePort); } case CLEANUP_OBJECT_DATABASE: { return TryDropDatabaseOutsideTransaction(record->objectName, nodeName, nodePort); } default: { ereport(WARNING, (errmsg( "Invalid object type %d on failed operation cleanup", record->objectType))); return false; } } return false; } /* * TryDropShardOutsideTransaction tries to drop the given shard placement and returns * true on success. */ static bool TryDropShardOutsideTransaction(char *qualifiedTableName, char *nodeName, int nodePort) { /* prepare sql query to execute to drop the shard */ StringInfo dropQuery = makeStringInfo(); appendStringInfo(dropQuery, DROP_REGULAR_TABLE_COMMAND, qualifiedTableName); /* * We set a lock_timeout here so that if there are running queries on the * shards we won't get blocked more than 1s and fail. * * The lock timeout also avoids getting stuck in a distributed deadlock, which * can occur because we might be holding pg_dist_placement locks while also * taking locks on the shard placements, and this code interrupts the * distributed deadlock detector. */ List *dropCommandList = list_make2("SET LOCAL lock_timeout TO '1s'", dropQuery->data); /* remove the shard from the node */ int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CurrentUserName(), NULL); bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection( workerConnection, dropCommandList); return success; } /* * TryDropSubscriptionOutsideTransaction drops subscription with the given name on the * subscriber node if it exists. Note that this doesn't drop the replication slot on the * publisher node. The reason is that sometimes this is not possible. To known * cases where this is not possible are: * 1. Due to the node with the replication slot being down. * 2. Due to a deadlock when the replication is on the same node as the * subscription, which is the case for shard splits to the local node. * * So instead of directly dropping the subscription, including the attached * replication slot, the subscription is first disconnected from the * replication slot before dropping it. The replication slot itself should be * dropped using DropReplicationSlot on the source connection. */ static bool TryDropSubscriptionOutsideTransaction(char *subscriptionName, char *nodeName, int nodePort) { int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), NULL); RemoteTransactionBegin(connection); if (ExecuteOptionalRemoteCommand(connection, "SET LOCAL lock_timeout TO '1s'", NULL) != 0) { RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } int querySent = SendRemoteCommand( connection, psprintf("ALTER SUBSCRIPTION %s DISABLE", quote_identifier(subscriptionName))); if (querySent == 0) { ReportConnectionError(connection, WARNING); RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { char *errorcode = PQresultErrorField(result, PG_DIAG_SQLSTATE); if (errorcode != NULL && strcmp(errorcode, STR_ERRCODE_UNDEFINED_OBJECT) == 0) { /* * The subscription doesn't exist, so we can return right away. * This DropSubscription call is effectively a no-op. */ PQclear(result); ForgetResults(connection); RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return true; } else { ReportResultError(connection, result, WARNING); PQclear(result); ForgetResults(connection); RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } } PQclear(result); ForgetResults(connection); RemoteTransactionCommit(connection); ResetRemoteTransaction(connection); StringInfo alterQuery = makeStringInfo(); appendStringInfo(alterQuery, "ALTER SUBSCRIPTION %s SET (slot_name = NONE)", quote_identifier(subscriptionName)); StringInfo dropQuery = makeStringInfo(); appendStringInfo(dropQuery, "DROP SUBSCRIPTION %s", quote_identifier(subscriptionName)); List *dropCommandList = list_make3("SET LOCAL lock_timeout TO '1s'", alterQuery->data, dropQuery->data); bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection( connection, dropCommandList); return success; } /* * TryDropPublicationOutsideTransaction drops the publication with the given name if it * exists. */ static bool TryDropPublicationOutsideTransaction(char *publicationName, char *nodeName, int nodePort) { int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), NULL); StringInfo dropQuery = makeStringInfo(); appendStringInfo(dropQuery, "DROP PUBLICATION IF EXISTS %s", quote_identifier(publicationName)); List *dropCommandList = list_make2("SET LOCAL lock_timeout TO '1s'", dropQuery->data); bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection( connection, dropCommandList); return success; } /* * TryDropReplicationSlotOutsideTransaction drops the replication slot with the given * name if it exists. */ static bool TryDropReplicationSlotOutsideTransaction(char *replicationSlotName, char *nodeName, int nodePort) { int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), NULL); RemoteTransactionBegin(connection); if (ExecuteOptionalRemoteCommand(connection, "SET LOCAL lock_timeout TO '1s'", NULL) != 0) { RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } int querySent = SendRemoteCommand( connection, psprintf( "select pg_drop_replication_slot(slot_name) from " REPLICATION_SLOT_CATALOG_TABLE_NAME " where slot_name = %s", quote_literal_cstr(replicationSlotName)) ); if (querySent == 0) { ReportConnectionError(connection, WARNING); RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (IsResponseOK(result)) { PQclear(result); ForgetResults(connection); RemoteTransactionCommit(connection); ResetRemoteTransaction(connection); return true; } char *errorcode = PQresultErrorField(result, PG_DIAG_SQLSTATE); if (errorcode != NULL && strcmp(errorcode, STR_ERRCODE_OBJECT_IN_USE) != 0) { /* throw a warning unless object is in use */ ReportResultError(connection, result, WARNING); } PQclear(result); ForgetResults(connection); RemoteTransactionAbort(connection); ResetRemoteTransaction(connection); return false; } /* * TryDropUserOutsideTransaction drops the user with the given name if it exists. */ static bool TryDropUserOutsideTransaction(char *username, char *nodeName, int nodePort) { int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), NULL); /* * The DROP USER command should not propagate, so we temporarily disable * DDL propagation. */ bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection( connection, list_make3( "SET LOCAL lock_timeout TO '1s'", "SET LOCAL citus.enable_ddl_propagation TO OFF;", psprintf("DROP USER IF EXISTS %s;", quote_identifier(username)))); return success; } /* * TryDropDatabaseOutsideTransaction drops the database with the given name * if it exists. */ static bool TryDropDatabaseOutsideTransaction(char *databaseName, char *nodeName, int nodePort) { int connectionFlags = (OUTSIDE_TRANSACTION | FORCE_NEW_CONNECTION); MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), NULL); if (PQstatus(connection->pgConn) != CONNECTION_OK) { return false; } /* * We want to disable DDL propagation and set lock_timeout before issuing * the DROP DATABASE command but we cannot do so in a way that's scoped * to the DROP DATABASE command. This is because, we cannot use a * transaction block for the DROP DATABASE command. * * For this reason, to avoid leaking the lock_timeout and DDL propagation * settings to future commands, we force the connection to close at the end * of the transaction. */ ForceConnectionCloseAtTransactionEnd(connection); /* * The DROP DATABASE command should not propagate, so we disable DDL * propagation. */ List *commandList = list_make3( "SET lock_timeout TO '1s'", "SET citus.enable_ddl_propagation TO OFF;", psprintf("DROP DATABASE IF EXISTS %s;", quote_identifier(databaseName)) ); bool executeCommand = true; const char *commandString = NULL; foreach_declared_ptr(commandString, commandList) { /* * Cannot use SendOptionalCommandListToWorkerOutsideTransactionWithConnection() * because we don't want to open a transaction block on remote nodes as DROP * DATABASE commands cannot be run inside a transaction block. */ if (ExecuteOptionalRemoteCommand( connection, commandString, NULL) != RESPONSE_OKAY) { executeCommand = false; break; } } CloseConnection(connection); return executeCommand; } /* * ErrorIfCleanupRecordForShardExists errors out if a cleanup record for the given * shard name exists. */ void ErrorIfCleanupRecordForShardExists(char *shardName) { CleanupRecord *record = GetCleanupRecordByNameAndType(shardName, CLEANUP_OBJECT_SHARD_PLACEMENT); if (record == NULL) { return; } ereport(ERROR, (errmsg("shard move failed as the orphaned shard %s leftover " "from the previous move could not be cleaned up", record->objectName))); } /* * GetNextOperationId allocates and returns a unique operationId for an operation * requiring potential cleanup. This allocation occurs both in shared memory and * in write ahead logs; writing to logs avoids the risk of having operationId collisions. */ static OperationId GetNextOperationId() { OperationId operationdId = INVALID_OPERATION_ID; /* * In regression tests, we would like to generate operation IDs consistently * even if the tests run in parallel. Instead of the sequence, we can use * the next_operation_id GUC to specify which operation ID the current session should * generate next. The GUC is automatically increased by 1 every time a new * operation ID is generated. */ if (NextOperationId > 0) { operationdId = NextOperationId; NextOperationId += 1; return operationdId; } /* Generate sequence using a subtransaction. else we can hold replication slot creation for operations */ StringInfo sequenceName = makeStringInfo(); appendStringInfo(sequenceName, "%s.%s", PG_CATALOG, OPERATIONID_SEQUENCE_NAME); StringInfo nextValueCommand = makeStringInfo(); appendStringInfo(nextValueCommand, "SELECT nextval(%s);", quote_literal_cstr(sequenceName->data)); MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction( CitusExtensionOwnerName()); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data, &result); if (queryResult != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) != 1 || PQnfields(result) != 1) { ReportResultError(connection, result, ERROR); } operationdId = SafeStringToUint64(PQgetvalue(result, 0, 0 /* nodeId column*/)); PQclear(result); ForgetResults(connection); return operationdId; } /* * ListCleanupRecords lists all the current cleanup records. */ static List * ListCleanupRecords(void) { Relation pgDistCleanup = table_open(DistCleanupRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistCleanup); List *recordList = NIL; int scanKeyCount = 0; bool indexOK = false; SysScanDesc scanDescriptor = systable_beginscan(pgDistCleanup, InvalidOid, indexOK, NULL, scanKeyCount, NULL); HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { CleanupRecord *record = TupleToCleanupRecord(heapTuple, tupleDescriptor); recordList = lappend(recordList, record); } systable_endscan(scanDescriptor); table_close(pgDistCleanup, NoLock); return recordList; } /* * ListCleanupRecordsForCurrentOperation lists all the cleanup records for * current operation. */ static List * ListCleanupRecordsForCurrentOperation(void) { /* We must have a valid OperationId. Any operation requring cleanup * will call RegisterOperationNeedingCleanup. */ Assert(CurrentOperationId != INVALID_OPERATION_ID); Relation pgDistCleanup = table_open(DistCleanupRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistCleanup); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_dist_cleanup_operation_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(CurrentOperationId)); int scanKeyCount = 1; Oid scanIndexId = InvalidOid; bool useIndex = false; SysScanDesc scanDescriptor = systable_beginscan(pgDistCleanup, scanIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = NULL; List *recordList = NIL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { CleanupRecord *record = TupleToCleanupRecord(heapTuple, tupleDescriptor); recordList = lappend(recordList, record); } systable_endscan(scanDescriptor); table_close(pgDistCleanup, NoLock); return recordList; } /* * GetCleanupRecordByNameAndType returns the cleanup record with given name and type, * if any, returns NULL otherwise. */ static CleanupRecord * GetCleanupRecordByNameAndType(char *objectName, CleanupObject type) { CleanupRecord *objectFound = NULL; Relation pgDistCleanup = table_open(DistCleanupRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistCleanup); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_dist_cleanup_object_type, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(type)); int scanKeyCount = 1; Oid scanIndexId = InvalidOid; bool useIndex = false; SysScanDesc scanDescriptor = systable_beginscan(pgDistCleanup, scanIndexId, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = NULL; while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { CleanupRecord *record = TupleToCleanupRecord(heapTuple, tupleDescriptor); if (strcmp(record->objectName, objectName) == 0) { objectFound = record; break; } } systable_endscan(scanDescriptor); table_close(pgDistCleanup, NoLock); return objectFound; } /* * TupleToCleanupRecord converts a pg_dist_cleanup record tuple into a CleanupRecord struct. */ static CleanupRecord * TupleToCleanupRecord(HeapTuple heapTuple, TupleDesc tupleDescriptor) { Datum datumArray[Natts_pg_dist_cleanup]; bool isNullArray[Natts_pg_dist_cleanup]; heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); CleanupRecord *record = palloc0(sizeof(CleanupRecord)); record->recordId = DatumGetUInt64(datumArray[Anum_pg_dist_cleanup_record_id - 1]); record->operationId = DatumGetUInt64(datumArray[Anum_pg_dist_cleanup_operation_id - 1]); record->objectType = DatumGetInt32(datumArray[Anum_pg_dist_cleanup_object_type - 1]); record->objectName = TextDatumGetCString(datumArray[Anum_pg_dist_cleanup_object_name - 1]); record->nodeGroupId = DatumGetInt32(datumArray[Anum_pg_dist_cleanup_node_group_id - 1]); record->policy = DatumGetInt32(datumArray[Anum_pg_dist_cleanup_policy_type - 1]); return record; } /* * CleanupRecordExists returns whether a cleanup record with the given * record ID exists in pg_dist_cleanup. */ static bool CleanupRecordExists(uint64 recordId) { Relation pgDistCleanup = table_open(DistCleanupRelationId(), AccessShareLock); const int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = true; ScanKeyInit(&scanKey[0], Anum_pg_dist_cleanup_record_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(recordId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistCleanup, DistCleanupPrimaryKeyIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); bool recordExists = HeapTupleIsValid(heapTuple); systable_endscan(scanDescriptor); CommandCounterIncrement(); table_close(pgDistCleanup, NoLock); return recordExists; } /* * DeleteCleanupRecordByRecordId deletes a single pg_dist_cleanup entry. */ static void DeleteCleanupRecordByRecordId(uint64 recordId) { Relation pgDistCleanup = table_open(DistCleanupRelationId(), RowExclusiveLock); const int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = true; ScanKeyInit(&scanKey[0], Anum_pg_dist_cleanup_record_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(recordId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistCleanup, DistCleanupPrimaryKeyIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (heapTuple == NULL) { ereport(ERROR, (errmsg("could not find cleanup record " UINT64_FORMAT, recordId))); } simple_heap_delete(pgDistCleanup, &heapTuple->t_self); systable_endscan(scanDescriptor); CommandCounterIncrement(); table_close(pgDistCleanup, NoLock); } /* * GetNextCleanupRecordId allocates and returns a unique recordid for a cleanup entry. * This allocation occurs both in shared memory and * in write ahead logs; writing to logs avoids the risk of having operationId collisions. */ static uint64 GetNextCleanupRecordId(void) { uint64 recordId = INVALID_CLEANUP_RECORD_ID; /* * In regression tests, we would like to generate record IDs consistently * even if the tests run in parallel. Instead of the sequence, we can use * the next_record_id GUC to specify which recordid ID the current session should * generate next. The GUC is automatically increased by 1 every time a new * record ID is generated. */ if (NextCleanupRecordId > 0) { recordId = NextCleanupRecordId; NextCleanupRecordId += 1; return recordId; } RangeVar *sequenceName = makeRangeVar(PG_CATALOG, CLEANUPRECORDID_SEQUENCE_NAME, -1); bool missingOK = false; Oid sequenceId = RangeVarGetRelid(sequenceName, NoLock, missingOK); bool checkPermissions = false; return nextval_internal(sequenceId, checkPermissions); } /* * LockOperationId takes an exclusive lock to ensure that only one process * can cleanup operationId resources at the same time. */ static void LockOperationId(OperationId operationId) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SET_LOCKTAG_CLEANUP_OPERATION_ID(tag, operationId); (void) LockAcquire(&tag, ExclusiveLock, sessionLock, dontWait); } /* * TryLockOperationId takes an exclusive lock (with dontWait = true) to ensure that * only one process can cleanup operationId resources at the same time. */ static bool TryLockOperationId(OperationId operationId) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = true; SET_LOCKTAG_CLEANUP_OPERATION_ID(tag, operationId); LockAcquireResult lockResult = LockAcquire(&tag, ExclusiveLock, sessionLock, dontWait); return (lockResult != LOCKACQUIRE_NOT_AVAIL); } ================================================ FILE: src/backend/distributed/operations/shard_rebalancer.c ================================================ /*------------------------------------------------------------------------- * * shard_rebalancer.c * * Function definitions for the shard rebalancer tool. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/genam.h" #include "access/htup_details.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/dbcommands.h" #include "commands/sequence.h" #include "common/hashfn.h" #include "postmaster/postmaster.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/guc_tables.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "pg_version_constants.h" #include "distributed/argutils.h" #include "distributed/background_jobs.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/enterprise.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_logical_replication.h" #include "distributed/multi_progress.h" #include "distributed/multi_server_executor.h" #include "distributed/pg_dist_rebalance_strategy.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_transfer.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/worker_protocol.h" /* RebalanceOptions are the options used to control the rebalance algorithm */ typedef struct RebalanceOptions { List *relationIdList; float4 threshold; int32 maxShardMoves; ArrayType *excludedShardArray; bool drainOnly; float4 improvementThreshold; Form_pg_dist_rebalance_strategy rebalanceStrategy; const char *operationName; WorkerNode *workerNode; List *involvedWorkerNodeList; } RebalanceOptions; typedef struct SplitPrimaryCloneShards { /* * primaryShardPlacementList contains the placements that * should stay on primary worker node. */ List *primaryShardIdList; /* * cloneShardPlacementList contains the placements that should stay on * clone worker node. */ List *cloneShardIdList; } SplitPrimaryCloneShards; static SplitPrimaryCloneShards * GetPrimaryCloneSplitRebalanceSteps(RebalanceOptions *options, WorkerNode *cloneNode); /* * RebalanceState is used to keep the internal state of the rebalance * algorithm in one place. */ typedef struct RebalanceState { /* * placementsHash contains the current state of all shard placements, it * is initialized from pg_dist_placement and is then modified based on the * found shard moves. */ HTAB *placementsHash; /* * placementUpdateList contains all of the updates that have been done to * reach the current state of placementsHash. */ List *placementUpdateList; RebalancePlanFunctions *functions; /* * fillStateListDesc contains all NodeFillStates ordered from full nodes to * empty nodes. */ List *fillStateListDesc; /* * fillStateListAsc contains all NodeFillStates ordered from empty nodes to * full nodes. */ List *fillStateListAsc; /* * disallowedPlacementList contains all placements that currently exist, * but are not allowed according to the shardAllowedOnNode function. */ List *disallowedPlacementList; /* * totalCost is the cost of all the shards in the cluster added together. */ float4 totalCost; /* * totalCapacity is the capacity of all the nodes in the cluster added * together. */ float4 totalCapacity; /* * ignoredMoves is the number of moves that were ignored. This is used to * limit the amount of loglines we send. */ int64 ignoredMoves; } RebalanceState; /* RebalanceContext stores the context for the function callbacks */ typedef struct RebalanceContext { FmgrInfo shardCostUDF; FmgrInfo nodeCapacityUDF; FmgrInfo shardAllowedOnNodeUDF; } RebalanceContext; /* WorkerHashKey contains hostname and port to be used as a key in a hash */ typedef struct WorkerHashKey { char hostname[MAX_NODE_LENGTH]; int port; } WorkerHashKey; /* WorkerShardIds represents a set of shardIds grouped by worker */ typedef struct WorkerShardIds { WorkerHashKey worker; /* This is a uint64 hashset representing the shard ids for a specific worker */ HTAB *shardIds; } WorkerShardIds; /* ShardStatistics contains statistics about a shard */ typedef struct ShardStatistics { uint64 shardId; /* The shard its size in bytes. */ uint64 totalSize; XLogRecPtr shardLSN; } ShardStatistics; /* * WorkerShardStatistics represents a set of statistics about shards, * grouped by worker. */ typedef struct WorkerShardStatistics { WorkerHashKey worker; XLogRecPtr workerLSN; /* * Statistics for each shard on this worker: * key: shardId * value: ShardStatistics */ HTAB *statistics; } WorkerShardStatistics; /* * ShardMoveDependencyHashEntry contains the taskId which any new shard * move task within the corresponding colocation group * must take a dependency on */ typedef struct ShardMoveDependencyInfo { int64 key; int64 taskId; } ShardMoveDependencyInfo; /* * ShardMoveSourceNodeHashEntry keeps track of the source nodes * of the moves. */ typedef struct ShardMoveSourceNodeHashEntry { /* this is the key */ int32 node_id; List *taskIds; } ShardMoveSourceNodeHashEntry; /* * ShardMoveDependencies keeps track of all needed dependencies * between shard moves. */ typedef struct ShardMoveDependencies { HTAB *colocationDependencies; HTAB *nodeDependencies; bool parallelTransferColocatedShards; } ShardMoveDependencies; char *VariablesToBePassedToNewConnections = NULL; /* static declarations for main logic */ static int ShardActivePlacementCount(HTAB *activePlacementsHash, uint64 shardId, List *activeWorkerNodeList); static void UpdateShardPlacement(PlacementUpdateEvent *placementUpdateEvent, List *responsiveNodeList, Oid shardReplicationModeOid); /* static declarations for main logic's utility functions */ static HTAB * ShardPlacementsListToHash(List *shardPlacementList); static bool PlacementsHashFind(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode); static void PlacementsHashEnter(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode); static void PlacementsHashRemove(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode); static int PlacementsHashCompare(const void *lhsKey, const void *rhsKey, Size keySize); static uint32 PlacementsHashHashCode(const void *key, Size keySize); static bool WorkerNodeListContains(List *workerNodeList, const char *workerName, uint32 workerPort); static void UpdateColocatedShardPlacementProgress(uint64 shardId, char *sourceName, int sourcePort, uint64 progress); static NodeFillState * FindFillStateForPlacement(RebalanceState *state, ShardPlacement *placement); static RebalanceState * InitRebalanceState(List *workerNodeList, List *shardPlacementList, RebalancePlanFunctions *functions); static void MoveShardsAwayFromDisallowedNodes(RebalanceState *state); static bool FindAndMoveShardCost(float4 utilizationLowerBound, float4 utilizationUpperBound, float4 improvementThreshold, RebalanceState *state); static NodeFillState * FindAllowedTargetFillState(RebalanceState *state, uint64 shardId); static void MoveShardCost(NodeFillState *sourceFillState, NodeFillState *targetFillState, ShardCost *shardCost, RebalanceState *state); static int CompareNodeFillStateAsc(const void *void1, const void *void2); static int CompareNodeFillStateDesc(const void *void1, const void *void2); static int CompareShardCostAsc(const void *void1, const void *void2); static int CompareShardCostDesc(const void *void1, const void *void2); static int CompareDisallowedPlacementAsc(const void *void1, const void *void2); static int CompareDisallowedPlacementDesc(const void *void1, const void *void2); static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *context); static float4 NodeCapacity(WorkerNode *workerNode, void *context); static ShardCost GetShardCost(uint64 shardId, void *context); static List * NonColocatedDistRelationIdList(void); static void RebalanceTableShards(RebalanceOptions *options, Oid shardReplicationModeOid); static int64 RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationModeOid, bool ParallelTransferReferenceTables, bool ParallelTransferColocatedShards); static void AcquireRebalanceColocationLock(Oid relationId, const char *operationName); static void ExecutePlacementUpdates(List *placementUpdateList, Oid shardReplicationModeOid, char *noticeOperation); static float4 CalculateUtilization(float4 totalCost, float4 capacity); static Form_pg_dist_rebalance_strategy GetRebalanceStrategy(Name name); static void EnsureShardCostUDF(Oid functionOid); static void EnsureNodeCapacityUDF(Oid functionOid); static void EnsureShardAllowedOnNodeUDF(Oid functionOid); static HTAB * BuildWorkerShardStatisticsHash(PlacementUpdateEventProgress *steps, int stepCount); static HTAB * GetShardStatistics(MultiConnection *connection, HTAB *shardIds); static HTAB * GetMovedShardIdsByWorker(PlacementUpdateEventProgress *steps, int stepCount, bool fromSource); static uint64 WorkerShardSize(HTAB *workerShardStatistics, char *workerName, int workerPort, uint64 shardId); static XLogRecPtr WorkerShardLSN(HTAB *workerShardStatisticsHash, char *workerName, int workerPort, uint64 shardId); static XLogRecPtr WorkerLSN(HTAB *workerShardStatisticsHash, char *workerName, int workerPort); static void AddToWorkerShardIdSet(HTAB *shardsByWorker, char *workerName, int workerPort, uint64 shardId); static HTAB * BuildShardSizesHash(ProgressMonitorData *monitor, HTAB *shardStatistics); static void ErrorOnConcurrentRebalance(RebalanceOptions *); static List * GetSetCommandListForNewConnections(void); static int64 GetColocationId(PlacementUpdateEvent *move); static ShardMoveDependencies InitializeShardMoveDependencies(bool ParallelTransferColocatedShards); static int64 * GenerateTaskMoveDependencyList(PlacementUpdateEvent *move, int64 colocationId, int64 *refTablesDepTaskIds, int refTablesDepTaskIdsCount, ShardMoveDependencies shardMoveDependencies, int *nDepends); static void UpdateShardMoveDependencies(PlacementUpdateEvent *move, uint64 colocationId, int64 taskId, ShardMoveDependencies shardMoveDependencies); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(rebalance_table_shards); PG_FUNCTION_INFO_V1(replicate_table_shards); PG_FUNCTION_INFO_V1(get_rebalance_table_shards_plan); PG_FUNCTION_INFO_V1(get_rebalance_progress); PG_FUNCTION_INFO_V1(citus_drain_node); PG_FUNCTION_INFO_V1(master_drain_node); PG_FUNCTION_INFO_V1(citus_shard_cost_by_disk_size); PG_FUNCTION_INFO_V1(citus_validate_rebalance_strategy_functions); PG_FUNCTION_INFO_V1(pg_dist_rebalance_strategy_enterprise_check); PG_FUNCTION_INFO_V1(citus_rebalance_start); PG_FUNCTION_INFO_V1(citus_rebalance_stop); PG_FUNCTION_INFO_V1(citus_rebalance_wait); PG_FUNCTION_INFO_V1(get_snapshot_based_node_split_plan); bool RunningUnderCitusTestSuite = false; int MaxRebalancerLoggedIgnoredMoves = 5; int RebalancerByDiskSizeBaseCost = 100 * 1024 * 1024; bool PropagateSessionSettingsForLoopbackConnection = false; static const char *PlacementUpdateTypeNames[] = { [PLACEMENT_UPDATE_INVALID_FIRST] = "unknown", [PLACEMENT_UPDATE_MOVE] = "move", [PLACEMENT_UPDATE_COPY] = "copy", }; static const char *PlacementUpdateStatusNames[] = { [PLACEMENT_UPDATE_STATUS_NOT_STARTED_YET] = "Not Started Yet", [PLACEMENT_UPDATE_STATUS_SETTING_UP] = "Setting Up", [PLACEMENT_UPDATE_STATUS_COPYING_DATA] = "Copying Data", [PLACEMENT_UPDATE_STATUS_CATCHING_UP] = "Catching Up", [PLACEMENT_UPDATE_STATUS_CREATING_CONSTRAINTS] = "Creating Constraints", [PLACEMENT_UPDATE_STATUS_FINAL_CATCH_UP] = "Final Catchup", [PLACEMENT_UPDATE_STATUS_CREATING_FOREIGN_KEYS] = "Creating Foreign Keys", [PLACEMENT_UPDATE_STATUS_COMPLETING] = "Completing", [PLACEMENT_UPDATE_STATUS_COMPLETED] = "Completed", }; #ifdef USE_ASSERT_CHECKING /* * Check that all the invariants of the state hold. */ static void CheckRebalanceStateInvariants(const RebalanceState *state) { NodeFillState *fillState = NULL; NodeFillState *prevFillState = NULL; int fillStateIndex = 0; int fillStateLength = list_length(state->fillStateListAsc); Assert(state != NULL); Assert(list_length(state->fillStateListAsc) == list_length(state->fillStateListDesc)); foreach_declared_ptr(fillState, state->fillStateListAsc) { float4 totalCost = 0; ShardCost *shardCost = NULL; ShardCost *prevShardCost = NULL; if (prevFillState != NULL) { /* Check that the previous fill state is more empty than this one */ bool higherUtilization = fillState->utilization > prevFillState->utilization; bool sameUtilization = fillState->utilization == prevFillState->utilization; bool lowerOrSameCapacity = fillState->capacity <= prevFillState->capacity; Assert(higherUtilization || (sameUtilization && lowerOrSameCapacity)); } /* Check that fillStateListDesc is the reversed version of fillStateListAsc */ Assert(list_nth(state->fillStateListDesc, fillStateLength - fillStateIndex - 1) == fillState); foreach_declared_ptr(shardCost, fillState->shardCostListDesc) { if (prevShardCost != NULL) { /* Check that shard costs are sorted in descending order */ Assert(shardCost->cost <= prevShardCost->cost); } totalCost += shardCost->cost; prevShardCost = shardCost; } /* Check that utilization field is up to date. */ Assert(fillState->utilization == CalculateUtilization(fillState->totalCost, fillState->capacity)); /* lgtm[cpp/equality-on-floats] */ /* * Check that fillState->totalCost is within 0.1% difference of * sum(fillState->shardCostListDesc->cost) * We cannot compare exactly, because these numbers are floats and * fillState->totalCost is modified by doing + and - on it. So instead * we check that the numbers are roughly the same. */ float4 absoluteDifferenceBetweenTotalCosts = fabsf(fillState->totalCost - totalCost); float4 maximumAbsoluteValueOfTotalCosts = fmaxf(fabsf(fillState->totalCost), fabsf(totalCost)); Assert(absoluteDifferenceBetweenTotalCosts <= maximumAbsoluteValueOfTotalCosts / 1000); prevFillState = fillState; fillStateIndex++; } } #else #define CheckRebalanceStateInvariants(l) ((void) 0) #endif /* USE_ASSERT_CHECKING */ /* * BigIntArrayDatumContains checks if the array contains the given number. */ static bool BigIntArrayDatumContains(Datum *array, int arrayLength, uint64 toFind) { for (int i = 0; i < arrayLength; i++) { if (DatumGetInt64(array[i]) == toFind) { return true; } } return false; } /* * FullShardPlacementList returns a List containing all the shard placements of * a specific table (excluding the excludedShardArray) */ static List * FullShardPlacementList(Oid relationId, ArrayType *excludedShardArray) { List *shardPlacementList = NIL; CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry(relationId); int shardIntervalArrayLength = citusTableCacheEntry->shardIntervalArrayLength; int excludedShardIdCount = ArrayObjectCount(excludedShardArray); Datum *excludedShardArrayDatum = DeconstructArrayObject(excludedShardArray); for (int shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++) { ShardInterval *shardInterval = citusTableCacheEntry->sortedShardIntervalArray[shardIndex]; GroupShardPlacement *placementArray = citusTableCacheEntry->arrayOfPlacementArrays[shardIndex]; int numberOfPlacements = citusTableCacheEntry->arrayOfPlacementArrayLengths[shardIndex]; if (BigIntArrayDatumContains(excludedShardArrayDatum, excludedShardIdCount, shardInterval->shardId)) { continue; } for (int placementIndex = 0; placementIndex < numberOfPlacements; placementIndex++) { GroupShardPlacement *groupPlacement = &placementArray[placementIndex]; WorkerNode *worker = LookupNodeForGroup(groupPlacement->groupId); ShardPlacement *placement = CitusMakeNode(ShardPlacement); placement->shardId = groupPlacement->shardId; placement->shardLength = groupPlacement->shardLength; placement->nodeId = worker->nodeId; placement->nodeName = pstrdup(worker->workerName); placement->nodePort = worker->workerPort; placement->placementId = groupPlacement->placementId; shardPlacementList = lappend(shardPlacementList, placement); } } return SortList(shardPlacementList, CompareShardPlacements); } /* * SortedActiveWorkers returns all the active workers like * ActiveReadableNodeList, but sorted. */ static List * SortedActiveWorkers() { List *activeWorkerList = ActiveReadableNodeList(); return SortList(activeWorkerList, CompareWorkerNodes); } /* * GetRebalanceSteps returns a List of PlacementUpdateEvents that are needed to * rebalance a list of tables. */ static List * GetRebalanceSteps(RebalanceOptions *options) { EnsureShardCostUDF(options->rebalanceStrategy->shardCostFunction); EnsureNodeCapacityUDF(options->rebalanceStrategy->nodeCapacityFunction); EnsureShardAllowedOnNodeUDF(options->rebalanceStrategy->shardAllowedOnNodeFunction); RebalanceContext context; memset(&context, 0, sizeof(RebalanceContext)); fmgr_info(options->rebalanceStrategy->shardCostFunction, &context.shardCostUDF); fmgr_info(options->rebalanceStrategy->nodeCapacityFunction, &context.nodeCapacityUDF); fmgr_info(options->rebalanceStrategy->shardAllowedOnNodeFunction, &context.shardAllowedOnNodeUDF); RebalancePlanFunctions rebalancePlanFunctions = { .shardAllowedOnNode = ShardAllowedOnNode, .nodeCapacity = NodeCapacity, .shardCost = GetShardCost, .context = &context, }; if (options->involvedWorkerNodeList == NULL) { /* * If the user did not specify a list of worker nodes, we use all the * active worker nodes. */ options->involvedWorkerNodeList = SortedActiveWorkers(); } /* sort the lists to make the function more deterministic */ List *activeWorkerList = options->involvedWorkerNodeList; /*SortedActiveWorkers(); */ int shardAllowedNodeCount = 0; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, activeWorkerList) { if (workerNode->shouldHaveShards) { shardAllowedNodeCount++; } } if (shardAllowedNodeCount < ShardReplicationFactor) { ereport(ERROR, (errmsg("Shard replication factor (%d) cannot be greater than " "number of nodes with should_have_shards=true (%d).", ShardReplicationFactor, shardAllowedNodeCount))); } List *activeShardPlacementListList = NIL; List *unbalancedShards = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, options->relationIdList) { List *shardPlacementList = FullShardPlacementList(relationId, options->excludedShardArray); List *activeShardPlacementListForRelation = FilterShardPlacementList(shardPlacementList, IsActiveShardPlacement); if (options->workerNode != NULL) { activeShardPlacementListForRelation = FilterActiveShardPlacementListByNode( shardPlacementList, options->workerNode); } if (list_length(activeShardPlacementListForRelation) >= shardAllowedNodeCount) { activeShardPlacementListList = lappend(activeShardPlacementListList, activeShardPlacementListForRelation); } else { /* * If the number of shard groups are less than the number of worker nodes, * at least one of the worker nodes will remain empty. For such cases, * we consider those shard groups as a colocation group and try to * distribute them across the cluster. */ unbalancedShards = list_concat(unbalancedShards, activeShardPlacementListForRelation); } } if (list_length(unbalancedShards) > 0) { activeShardPlacementListList = lappend(activeShardPlacementListList, unbalancedShards); } if (options->threshold < options->rebalanceStrategy->minimumThreshold) { ereport(WARNING, (errmsg( "the given threshold is lower than the minimum " "threshold allowed by the rebalance strategy, " "using the minimum allowed threshold instead" ), errdetail("Using threshold of %.2f", options->rebalanceStrategy->minimumThreshold ) )); options->threshold = options->rebalanceStrategy->minimumThreshold; } return RebalancePlacementUpdates(activeWorkerList, activeShardPlacementListList, options->threshold, options->maxShardMoves, options->drainOnly, options->improvementThreshold, &rebalancePlanFunctions); } /* * ShardAllowedOnNode determines if shard is allowed on a specific worker node. */ static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *voidContext) { if (!workerNode->shouldHaveShards) { return false; } RebalanceContext *context = voidContext; Datum allowed = FunctionCall2(&context->shardAllowedOnNodeUDF, shardId, workerNode->nodeId); return DatumGetBool(allowed); } /* * NodeCapacity returns the relative capacity of a node. A node with capacity 2 * can contain twice as many shards as a node with capacity 1. The actual * capacity can be a number grounded in reality, like the disk size, number of * cores, but it doesn't have to be. */ static float4 NodeCapacity(WorkerNode *workerNode, void *voidContext) { if (!workerNode->shouldHaveShards) { return 0; } RebalanceContext *context = voidContext; Datum capacity = FunctionCall1(&context->nodeCapacityUDF, workerNode->nodeId); return DatumGetFloat4(capacity); } /* * GetShardCost returns the cost of the given shard. A shard with cost 2 will * be weighted as heavily as two shards with cost 1. This cost number can be a * number grounded in reality, like the shard size on disk, but it doesn't have * to be. */ static ShardCost GetShardCost(uint64 shardId, void *voidContext) { ShardCost shardCost = { 0 }; shardCost.shardId = shardId; RebalanceContext *context = voidContext; Datum shardCostDatum = FunctionCall1(&context->shardCostUDF, UInt64GetDatum(shardId)); shardCost.cost = DatumGetFloat4(shardCostDatum); return shardCost; } /* * citus_shard_cost_by_disk_size gets the cost for a shard based on the disk * size of the shard on a worker. The worker to check the disk size is * determined by choosing the first active placement for the shard. The disk * size is calculated using pg_total_relation_size, so it includes indexes. * * SQL signature: * citus_shard_cost_by_disk_size(shardid bigint) returns float4 */ Datum citus_shard_cost_by_disk_size(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 shardId = PG_GETARG_INT64(0); bool missingOk = false; ShardPlacement *shardPlacement = ActiveShardPlacement(shardId, missingOk); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CostByDiscSizeContext", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedNonPartitionShardIntervalList(shardInterval); uint64 colocationSizeInBytes = ShardListSizeInBytes(colocatedShardList, shardPlacement->nodeName, shardPlacement->nodePort); MemoryContextSwitchTo(oldContext); MemoryContextReset(localContext); colocationSizeInBytes += RebalancerByDiskSizeBaseCost; if (colocationSizeInBytes <= 0) { PG_RETURN_FLOAT4(1); } PG_RETURN_FLOAT4(colocationSizeInBytes); } /* * GetColocatedRebalanceSteps takes a List of PlacementUpdateEvents and creates * a new List of containing those and all the updates for colocated shards. */ static List * GetColocatedRebalanceSteps(List *placementUpdateList) { ListCell *placementUpdateCell = NULL; List *colocatedUpdateList = NIL; foreach(placementUpdateCell, placementUpdateList) { PlacementUpdateEvent *placementUpdate = lfirst(placementUpdateCell); ShardInterval *shardInterval = LoadShardInterval(placementUpdate->shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); ListCell *colocatedShardCell = NULL; foreach(colocatedShardCell, colocatedShardList) { ShardInterval *colocatedShard = lfirst(colocatedShardCell); PlacementUpdateEvent *colocatedUpdate = palloc0(sizeof(PlacementUpdateEvent)); colocatedUpdate->shardId = colocatedShard->shardId; colocatedUpdate->sourceNode = placementUpdate->sourceNode; colocatedUpdate->targetNode = placementUpdate->targetNode; colocatedUpdate->updateType = placementUpdate->updateType; colocatedUpdateList = lappend(colocatedUpdateList, colocatedUpdate); } } return colocatedUpdateList; } /* * AcquireRelationColocationLock tries to acquire a lock for * rebalance/replication. If this is it not possible it fails * instantly because this means another rebalance/replication * is currently happening. This would really mess up planning. */ static void AcquireRebalanceColocationLock(Oid relationId, const char *operationName) { uint32 lockId = relationId; LOCKTAG tag; CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry(relationId); if (citusTableCacheEntry->colocationId != INVALID_COLOCATION_ID) { lockId = citusTableCacheEntry->colocationId; } SET_LOCKTAG_REBALANCE_COLOCATION(tag, (int64) lockId); LockAcquireResult lockAcquired = LockAcquire(&tag, ExclusiveLock, false, true); if (!lockAcquired) { ereport(ERROR, (errmsg("could not acquire the lock required to %s %s", operationName, generate_qualified_relation_name(relationId)), errdetail("It means that either a concurrent shard move " "or shard copy is happening."), errhint("Make sure that the concurrent operation has " "finished and re-run the command"))); } } /* * AcquirePlacementColocationLock tries to acquire a lock for * rebalance/replication while moving/copying the placement. If this * is it not possible it fails instantly because this means * another move/copy is currently happening. This would really mess up planning. */ void AcquirePlacementColocationLock(Oid relationId, int lockMode, const char *operationName) { uint32 lockId = relationId; LOCKTAG tag; CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry(relationId); if (citusTableCacheEntry->colocationId != INVALID_COLOCATION_ID) { lockId = citusTableCacheEntry->colocationId; } SET_LOCKTAG_REBALANCE_PLACEMENT_COLOCATION(tag, (int64) lockId); LockAcquireResult lockAcquired = LockAcquire(&tag, lockMode, false, true); if (!lockAcquired) { ereport(ERROR, (errmsg("could not acquire the lock required to %s %s", operationName, generate_qualified_relation_name(relationId)), errdetail("It means that either a concurrent shard move " "or colocated distributed table creation is " "happening."), errhint("Make sure that the concurrent operation has " "finished and re-run the command"))); } } /* * GetResponsiveWorkerList returns a List of workers that respond to new * connection requests. */ static List * GetResponsiveWorkerList() { List *activeWorkerList = ActiveReadableNodeList(); ListCell *activeWorkerCell = NULL; List *responsiveWorkerList = NIL; foreach(activeWorkerCell, activeWorkerList) { WorkerNode *worker = lfirst(activeWorkerCell); int connectionFlag = FORCE_NEW_CONNECTION; MultiConnection *connection = GetNodeConnection(connectionFlag, worker->workerName, worker->workerPort); if (connection != NULL && connection->pgConn != NULL) { if (PQstatus(connection->pgConn) == CONNECTION_OK) { responsiveWorkerList = lappend(responsiveWorkerList, worker); } CloseConnection(connection); } } return responsiveWorkerList; } /* * ExecutePlacementUpdates copies or moves a shard placement by calling the * corresponding functions in Citus in a separate subtransaction for each * update. */ static void ExecutePlacementUpdates(List *placementUpdateList, Oid shardReplicationModeOid, char *noticeOperation) { List *responsiveWorkerList = GetResponsiveWorkerList(); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "ExecutePlacementLoopContext", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); ListCell *placementUpdateCell = NULL; DropOrphanedResourcesInSeparateTransaction(); foreach(placementUpdateCell, placementUpdateList) { PlacementUpdateEvent *placementUpdate = lfirst(placementUpdateCell); ereport(NOTICE, (errmsg( "%s shard %lu from %s:%u to %s:%u ...", noticeOperation, placementUpdate->shardId, placementUpdate->sourceNode->workerName, placementUpdate->sourceNode->workerPort, placementUpdate->targetNode->workerName, placementUpdate->targetNode->workerPort ))); UpdateShardPlacement(placementUpdate, responsiveWorkerList, shardReplicationModeOid); MemoryContextReset(localContext); } MemoryContextSwitchTo(oldContext); } /* * SetupRebalanceMonitor initializes the dynamic shared memory required for storing the * progress information of a rebalance process. The function takes a List of * PlacementUpdateEvents for all shards that will be moved (including colocated * ones) and the relation id of the target table. The dynamic shared memory * portion consists of a RebalanceMonitorHeader and multiple * PlacementUpdateEventProgress, one for each planned shard placement move. The * dsm_handle of the created segment is saved in the progress of the current backend so * that it can be read by external agents such as get_rebalance_progress function by * calling pg_stat_get_progress_info UDF. Since currently only VACUUM commands are * officially allowed as the command type, we describe ourselves as a VACUUM command and * in order to distinguish a rebalancer progress from regular VACUUM progresses, we put * a magic number to the first progress field as an indicator. Finally we return the * dsm handle so that it can be used for updating the progress and cleaning things up. */ void SetupRebalanceMonitor(List *placementUpdateList, Oid relationId, uint64 initialProgressState, PlacementUpdateStatus initialStatus) { List *colocatedUpdateList = GetColocatedRebalanceSteps(placementUpdateList); ListCell *colocatedUpdateCell = NULL; dsm_handle dsmHandle; ProgressMonitorData *monitor = CreateProgressMonitor( list_length(colocatedUpdateList), sizeof(PlacementUpdateEventProgress), &dsmHandle); PlacementUpdateEventProgress *rebalanceSteps = ProgressMonitorSteps(monitor); int32 eventIndex = 0; foreach(colocatedUpdateCell, colocatedUpdateList) { PlacementUpdateEvent *colocatedUpdate = lfirst(colocatedUpdateCell); PlacementUpdateEventProgress *event = rebalanceSteps + eventIndex; strlcpy(event->sourceName, colocatedUpdate->sourceNode->workerName, 255); strlcpy(event->targetName, colocatedUpdate->targetNode->workerName, 255); event->shardId = colocatedUpdate->shardId; event->sourcePort = colocatedUpdate->sourceNode->workerPort; event->targetPort = colocatedUpdate->targetNode->workerPort; event->updateType = colocatedUpdate->updateType; pg_atomic_init_u64(&event->updateStatus, initialStatus); pg_atomic_init_u64(&event->progress, initialProgressState); eventIndex++; } RegisterProgressMonitor(REBALANCE_ACTIVITY_MAGIC_NUMBER, relationId, dsmHandle); } /* * rebalance_table_shards rebalances the shards across the workers. * * SQL signature: * * rebalance_table_shards( * relation regclass, * threshold float4, * max_shard_moves int, * excluded_shard_list bigint[], * shard_transfer_mode citus.shard_transfer_mode, * drain_only boolean, * rebalance_strategy name * ) RETURNS VOID */ Datum rebalance_table_shards(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); List *relationIdList = NIL; if (!PG_ARGISNULL(0)) { Oid relationId = PG_GETARG_OID(0); ErrorIfMoveUnsupportedTableType(relationId); relationIdList = list_make1_oid(relationId); } else { /* * Note that we don't need to do any checks to error out for * citus local tables here as NonColocatedDistRelationIdList * already doesn't return non-distributed tables. */ relationIdList = NonColocatedDistRelationIdList(); } PG_ENSURE_ARGNOTNULL(2, "max_shard_moves"); PG_ENSURE_ARGNOTNULL(3, "excluded_shard_list"); PG_ENSURE_ARGNOTNULL(4, "shard_transfer_mode"); PG_ENSURE_ARGNOTNULL(5, "drain_only"); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy( PG_GETARG_NAME_OR_NULL(6)); RebalanceOptions options = { .relationIdList = relationIdList, .threshold = PG_GETARG_FLOAT4_OR_DEFAULT(1, strategy->defaultThreshold), .maxShardMoves = PG_GETARG_INT32(2), .excludedShardArray = PG_GETARG_ARRAYTYPE_P(3), .drainOnly = PG_GETARG_BOOL(5), .rebalanceStrategy = strategy, .involvedWorkerNodeList = NULL, .improvementThreshold = strategy->improvementThreshold, }; Oid shardTransferModeOid = PG_GETARG_OID(4); RebalanceTableShards(&options, shardTransferModeOid); PG_RETURN_VOID(); } /* * citus_rebalance_start rebalances the shards across the workers. * * SQL signature: * * citus_rebalance_start( * rebalance_strategy name DEFAULT NULL, * drain_only boolean DEFAULT false, * shard_transfer_mode citus.shard_transfer_mode default 'auto' * ) RETURNS VOID */ Datum citus_rebalance_start(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); List *relationIdList = NonColocatedDistRelationIdList(); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy(PG_GETARG_NAME_OR_NULL(0)); PG_ENSURE_ARGNOTNULL(1, "drain_only"); bool drainOnly = PG_GETARG_BOOL(1); PG_ENSURE_ARGNOTNULL(2, "shard_transfer_mode"); Oid shardTransferModeOid = PG_GETARG_OID(2); PG_ENSURE_ARGNOTNULL(3, "parallel_transfer_reference_tables"); bool ParallelTransferReferenceTables = PG_GETARG_BOOL(3); PG_ENSURE_ARGNOTNULL(4, "parallel_transfer_colocated_shards"); bool ParallelTransferColocatedShards = PG_GETARG_BOOL(4); RebalanceOptions options = { .relationIdList = relationIdList, .threshold = strategy->defaultThreshold, .maxShardMoves = 10000000, .excludedShardArray = construct_empty_array(INT4OID), .drainOnly = drainOnly, .rebalanceStrategy = strategy, .improvementThreshold = strategy->improvementThreshold, }; int jobId = RebalanceTableShardsBackground(&options, shardTransferModeOid, ParallelTransferReferenceTables, ParallelTransferColocatedShards); if (jobId == 0) { PG_RETURN_NULL(); } PG_RETURN_INT64(jobId); } /* * citus_rebalance_stop stops any ongoing background rebalance that is executing. * Raises an error when there is no backgound rebalance ongoing at the moment. */ Datum citus_rebalance_stop(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 jobId = 0; if (!HasNonTerminalJobOfType("rebalance", &jobId)) { ereport(ERROR, (errmsg("no ongoing rebalance that can be stopped"))); } DirectFunctionCall1(citus_job_cancel, Int64GetDatum(jobId)); PG_RETURN_VOID(); } /* * citus_rebalance_wait waits till an ongoing background rebalance has finished execution. * A warning will be displayed if no rebalance is ongoing. */ Datum citus_rebalance_wait(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 jobId = 0; if (!HasNonTerminalJobOfType("rebalance", &jobId)) { ereport(WARNING, (errmsg("no ongoing rebalance that can be waited on"))); PG_RETURN_VOID(); } citus_job_wait_internal(jobId, NULL); PG_RETURN_VOID(); } /* * GetRebalanceStrategy returns the rebalance strategy from * pg_dist_rebalance_strategy matching the given name. If name is NULL it * returns the default rebalance strategy from pg_dist_rebalance_strategy. */ static Form_pg_dist_rebalance_strategy GetRebalanceStrategy(Name name) { Relation pgDistRebalanceStrategy = table_open(DistRebalanceStrategyRelationId(), AccessShareLock); const int scanKeyCount = 1; ScanKeyData scanKey[1]; if (name == NULL) { /* WHERE default_strategy=true */ ScanKeyInit(&scanKey[0], Anum_pg_dist_rebalance_strategy_default_strategy, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(true)); } else { /* WHERE name=$name */ ScanKeyInit(&scanKey[0], Anum_pg_dist_rebalance_strategy_name, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(name)); } SysScanDesc scanDescriptor = systable_beginscan(pgDistRebalanceStrategy, InvalidOid, false, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { if (name == NULL) { ereport(ERROR, (errmsg( "no rebalance_strategy was provided, but there is also no default strategy set"))); } ereport(ERROR, (errmsg("could not find rebalance strategy with name %s", (char *) name))); } Form_pg_dist_rebalance_strategy strategy = (Form_pg_dist_rebalance_strategy) GETSTRUCT(heapTuple); Form_pg_dist_rebalance_strategy strategy_copy = palloc0(sizeof(FormData_pg_dist_rebalance_strategy)); /* Copy data over by dereferencing */ *strategy_copy = *strategy; systable_endscan(scanDescriptor); table_close(pgDistRebalanceStrategy, NoLock); return strategy_copy; } /* * citus_drain_node drains a node by setting shouldhaveshards to false and * running the rebalancer after in drain_only mode. */ Datum citus_drain_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_ENSURE_ARGNOTNULL(0, "nodename"); PG_ENSURE_ARGNOTNULL(1, "nodeport"); PG_ENSURE_ARGNOTNULL(2, "shard_transfer_mode"); text *nodeNameText = PG_GETARG_TEXT_P(0); int32 nodePort = PG_GETARG_INT32(1); Oid shardTransferModeOid = PG_GETARG_OID(2); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy( PG_GETARG_NAME_OR_NULL(3)); RebalanceOptions options = { .relationIdList = NonColocatedDistRelationIdList(), .threshold = strategy->defaultThreshold, .maxShardMoves = 0, .excludedShardArray = construct_empty_array(INT4OID), .drainOnly = true, .rebalanceStrategy = strategy, }; char *nodeName = text_to_cstring(nodeNameText); options.workerNode = FindWorkerNodeOrError(nodeName, nodePort); /* * This is done in a separate session. This way it's not undone if the * draining fails midway through. */ ExecuteRebalancerCommandInSeparateTransaction(psprintf( "SELECT master_set_node_property(%s, %i, 'shouldhaveshards', false)", quote_literal_cstr(nodeName), nodePort)); RebalanceTableShards(&options, shardTransferModeOid); PG_RETURN_VOID(); } /* * replicate_table_shards replicates under-replicated shards of the specified * table. */ Datum replicate_table_shards(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); uint32 shardReplicationFactor = PG_GETARG_INT32(1); int32 maxShardCopies = PG_GETARG_INT32(2); ArrayType *excludedShardArray = PG_GETARG_ARRAYTYPE_P(3); Oid shardReplicationModeOid = PG_GETARG_OID(4); if (IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errmsg("cannot replicate single shard tables' shards"))); } char transferMode = LookupShardTransferMode(shardReplicationModeOid); EnsureReferenceTablesExistOnAllNodesExtended(transferMode); AcquireRebalanceColocationLock(relationId, "replicate"); List *activeWorkerList = SortedActiveWorkers(); List *shardPlacementList = FullShardPlacementList(relationId, excludedShardArray); List *activeShardPlacementList = FilterShardPlacementList(shardPlacementList, IsActiveShardPlacement); List *placementUpdateList = ReplicationPlacementUpdates(activeWorkerList, activeShardPlacementList, shardReplicationFactor); placementUpdateList = list_truncate(placementUpdateList, maxShardCopies); ExecutePlacementUpdates(placementUpdateList, shardReplicationModeOid, "Copying"); PG_RETURN_VOID(); } /* * master_drain_node is a wrapper function for old UDF name. */ Datum master_drain_node(PG_FUNCTION_ARGS) { return citus_drain_node(fcinfo); } /* * get_rebalance_table_shards_plan function calculates the shard move steps * required for the rebalance operations including the ones for colocated * tables. * * SQL signature: * * get_rebalance_table_shards_plan( * relation regclass, * threshold float4, * max_shard_moves int, * excluded_shard_list bigint[], * drain_only boolean, * rebalance_strategy name * ) */ Datum get_rebalance_table_shards_plan(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); List *relationIdList = NIL; if (!PG_ARGISNULL(0)) { Oid relationId = PG_GETARG_OID(0); ErrorIfMoveUnsupportedTableType(relationId); relationIdList = list_make1_oid(relationId); } else { /* * Note that we don't need to do any checks to error out for * citus local tables here as NonColocatedDistRelationIdList * already doesn't return non-distributed tables. */ relationIdList = NonColocatedDistRelationIdList(); } PG_ENSURE_ARGNOTNULL(2, "max_shard_moves"); PG_ENSURE_ARGNOTNULL(3, "excluded_shard_list"); PG_ENSURE_ARGNOTNULL(4, "drain_only"); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy( PG_GETARG_NAME_OR_NULL(5)); RebalanceOptions options = { .relationIdList = relationIdList, .threshold = PG_GETARG_FLOAT4_OR_DEFAULT(1, strategy->defaultThreshold), .maxShardMoves = PG_GETARG_INT32(2), .excludedShardArray = PG_GETARG_ARRAYTYPE_P(3), .drainOnly = PG_GETARG_BOOL(4), .rebalanceStrategy = strategy, .improvementThreshold = PG_GETARG_FLOAT4_OR_DEFAULT( 6, strategy->improvementThreshold), }; List *placementUpdateList = GetRebalanceSteps(&options); List *colocatedUpdateList = GetColocatedRebalanceSteps(placementUpdateList); ListCell *colocatedUpdateCell = NULL; TupleDesc tupdesc; Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); foreach(colocatedUpdateCell, colocatedUpdateList) { PlacementUpdateEvent *colocatedUpdate = lfirst(colocatedUpdateCell); Datum values[7]; bool nulls[7]; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = ObjectIdGetDatum(RelationIdForShard(colocatedUpdate->shardId)); values[1] = UInt64GetDatum(colocatedUpdate->shardId); values[2] = UInt64GetDatum(ShardLength(colocatedUpdate->shardId)); values[3] = PointerGetDatum(cstring_to_text( colocatedUpdate->sourceNode->workerName)); values[4] = UInt32GetDatum(colocatedUpdate->sourceNode->workerPort); values[5] = PointerGetDatum(cstring_to_text( colocatedUpdate->targetNode->workerName)); values[6] = UInt32GetDatum(colocatedUpdate->targetNode->workerPort); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } return (Datum) 0; } /* * get_rebalance_progress collects information about the ongoing rebalance operations and * returns the concatenated list of steps involved in the operations, along with their * progress information. Currently the progress field can take 4 integer values * (-1: error, 0: waiting, 1: moving, 2: moved). The progress field is of type bigint * because we may implement a more granular, byte-level progress as a future improvement. */ Datum get_rebalance_progress(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); List *segmentList = NIL; TupleDesc tupdesc; Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); /* get the addresses of all current rebalance monitors */ List *rebalanceMonitorList = ProgressMonitorList(REBALANCE_ACTIVITY_MAGIC_NUMBER, &segmentList); ProgressMonitorData *monitor = NULL; foreach_declared_ptr(monitor, rebalanceMonitorList) { PlacementUpdateEventProgress *placementUpdateEvents = ProgressMonitorSteps( monitor); HTAB *shardStatistics = BuildWorkerShardStatisticsHash(placementUpdateEvents, monitor->stepCount); HTAB *shardSizes = BuildShardSizesHash(monitor, shardStatistics); for (int eventIndex = 0; eventIndex < monitor->stepCount; eventIndex++) { PlacementUpdateEventProgress *step = placementUpdateEvents + eventIndex; uint64 shardId = step->shardId; ShardInterval *shardInterval = LoadShardInterval(shardId); uint64 sourceSize = WorkerShardSize(shardStatistics, step->sourceName, step->sourcePort, shardId); uint64 targetSize = WorkerShardSize(shardStatistics, step->targetName, step->targetPort, shardId); XLogRecPtr sourceLSN = WorkerLSN(shardStatistics, step->sourceName, step->sourcePort); XLogRecPtr targetLSN = WorkerShardLSN(shardStatistics, step->targetName, step->targetPort, shardId); uint64 shardSize = 0; ShardStatistics *shardSizesStat = hash_search(shardSizes, &shardId, HASH_FIND, NULL); if (shardSizesStat) { shardSize = shardSizesStat->totalSize; } Datum values[15]; bool nulls[15]; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = monitor->processId; values[1] = ObjectIdGetDatum(shardInterval->relationId); values[2] = UInt64GetDatum(shardId); values[3] = UInt64GetDatum(shardSize); values[4] = PointerGetDatum(cstring_to_text(step->sourceName)); values[5] = UInt32GetDatum(step->sourcePort); values[6] = PointerGetDatum(cstring_to_text(step->targetName)); values[7] = UInt32GetDatum(step->targetPort); values[8] = UInt64GetDatum(pg_atomic_read_u64(&step->progress)); values[9] = UInt64GetDatum(sourceSize); values[10] = UInt64GetDatum(targetSize); values[11] = PointerGetDatum( cstring_to_text(PlacementUpdateTypeNames[step->updateType])); values[12] = LSNGetDatum(sourceLSN); if (sourceLSN == InvalidXLogRecPtr) { nulls[12] = true; } values[13] = LSNGetDatum(targetLSN); if (targetLSN == InvalidXLogRecPtr) { nulls[13] = true; } values[14] = PointerGetDatum(cstring_to_text( PlacementUpdateStatusNames[ pg_atomic_read_u64( &step->updateStatus)])); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } } DetachFromDSMSegments(segmentList); return (Datum) 0; } /* * BuildShardSizesHash creates a hash that maps a shardid to its full size * within the cluster. It does this by using the rebalance progress monitor * state to find the node the shard is currently on. It then looks up the shard * size in the shardStatistics hashmap for this node. */ static HTAB * BuildShardSizesHash(ProgressMonitorData *monitor, HTAB *shardStatistics) { HASHCTL info = { .keysize = sizeof(uint64), .entrysize = sizeof(ShardStatistics), .hcxt = CurrentMemoryContext }; HTAB *shardSizes = hash_create( "ShardSizeHash", 32, &info, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); PlacementUpdateEventProgress *placementUpdateEvents = ProgressMonitorSteps(monitor); for (int eventIndex = 0; eventIndex < monitor->stepCount; eventIndex++) { PlacementUpdateEventProgress *step = placementUpdateEvents + eventIndex; uint64 shardId = step->shardId; uint64 shardSize = 0; uint64 backupShardSize = 0; uint64 progress = pg_atomic_read_u64(&step->progress); uint64 sourceSize = WorkerShardSize(shardStatistics, step->sourceName, step->sourcePort, shardId); uint64 targetSize = WorkerShardSize(shardStatistics, step->targetName, step->targetPort, shardId); if (progress == REBALANCE_PROGRESS_WAITING || progress == REBALANCE_PROGRESS_MOVING) { /* * If we are not done with the move, the correct shard size is the * size on the source. */ shardSize = sourceSize; backupShardSize = targetSize; } else if (progress == REBALANCE_PROGRESS_MOVED) { /* * If we are done with the move, the correct shard size is the size * on the target */ shardSize = targetSize; backupShardSize = sourceSize; } if (shardSize == 0) { if (backupShardSize == 0) { /* * We don't have any useful shard size. This can happen when a * shard is moved multiple times and it is not present on * either of these nodes. Probably the shard is on a worker * related to another event. In the weird case that this shard * is on the nodes and actually is size 0, we will have no * entry in the hashmap. When fetching from it we always * default to 0 if no entry is found, so that's fine. */ continue; } /* * Because of the way we fetch shard sizes they are from a slightly * earlier moment than the progress state we just read from shared * memory. Usually this is no problem, but there exist some race * conditions where this matters. For example, for very quick moves * it is possible that even though a step is now reported as MOVED, * when we read the shard sizes the move had not even started yet. * This in turn can mean that the target size is 0 while the source * size is not. We try to handle such rare edge cases by falling * back on the other shard size if that one is not 0. */ shardSize = backupShardSize; } ShardStatistics *currentWorkerStatistics = hash_search(shardSizes, &shardId, HASH_ENTER, NULL); currentWorkerStatistics->totalSize = shardSize; } return shardSizes; } /* * WorkerShardSize returns the size of a shard in bytes on a worker, based on * the workerShardStatisticsHash. */ static uint64 WorkerShardSize(HTAB *workerShardStatisticsHash, char *workerName, int workerPort, uint64 shardId) { WorkerHashKey workerKey = { 0 }; strlcpy(workerKey.hostname, workerName, MAX_NODE_LENGTH); workerKey.port = workerPort; WorkerShardStatistics *workerStats = hash_search(workerShardStatisticsHash, &workerKey, HASH_FIND, NULL); if (!workerStats) { return 0; } ShardStatistics *shardStats = hash_search(workerStats->statistics, &shardId, HASH_FIND, NULL); if (!shardStats) { return 0; } return shardStats->totalSize; } /* * WorkerShardLSN returns the LSN of a shard on a worker, based on * the workerShardStatisticsHash. If there is no LSN data in the * statistics object, returns InvalidXLogRecPtr. */ static XLogRecPtr WorkerShardLSN(HTAB *workerShardStatisticsHash, char *workerName, int workerPort, uint64 shardId) { WorkerHashKey workerKey = { 0 }; strlcpy(workerKey.hostname, workerName, MAX_NODE_LENGTH); workerKey.port = workerPort; WorkerShardStatistics *workerStats = hash_search(workerShardStatisticsHash, &workerKey, HASH_FIND, NULL); if (!workerStats) { return InvalidXLogRecPtr; } ShardStatistics *shardStats = hash_search(workerStats->statistics, &shardId, HASH_FIND, NULL); if (!shardStats) { return InvalidXLogRecPtr; } return shardStats->shardLSN; } /* * WorkerLSN returns the LSN of a worker, based on the workerShardStatisticsHash. * If there is no LSN data in the statistics object, returns InvalidXLogRecPtr. */ static XLogRecPtr WorkerLSN(HTAB *workerShardStatisticsHash, char *workerName, int workerPort) { WorkerHashKey workerKey = { 0 }; strlcpy(workerKey.hostname, workerName, MAX_NODE_LENGTH); workerKey.port = workerPort; WorkerShardStatistics *workerStats = hash_search(workerShardStatisticsHash, &workerKey, HASH_FIND, NULL); if (!workerStats) { return InvalidXLogRecPtr; } return workerStats->workerLSN; } /* * BuildWorkerShardStatisticsHash returns a shard id -> shard statistics hash containing * sizes of shards on the source node and destination node. */ static HTAB * BuildWorkerShardStatisticsHash(PlacementUpdateEventProgress *steps, int stepCount) { HTAB *shardsByWorker = GetMovedShardIdsByWorker(steps, stepCount, true); HASHCTL info = { .keysize = sizeof(WorkerHashKey), .entrysize = sizeof(WorkerShardStatistics), .hcxt = CurrentMemoryContext }; HTAB *workerShardStatistics = hash_create("WorkerShardStatistics", 32, &info, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); WorkerShardIds *entry = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, shardsByWorker); while ((entry = hash_seq_search(&status)) != NULL) { int connectionFlags = 0; MultiConnection *connection = GetNodeConnection(connectionFlags, entry->worker.hostname, entry->worker.port); HTAB *statistics = GetShardStatistics(connection, entry->shardIds); WorkerHashKey workerKey = { 0 }; strlcpy(workerKey.hostname, entry->worker.hostname, MAX_NODE_LENGTH); workerKey.port = entry->worker.port; WorkerShardStatistics *moveStat = hash_search(workerShardStatistics, &entry->worker, HASH_ENTER, NULL); moveStat->statistics = statistics; moveStat->workerLSN = GetRemoteLogPosition(connection); } return workerShardStatistics; } /* * GetShardStatistics fetches the statics for the given shard ids over the * given connection. It returns a hashmap where the keys are the shard ids and * the values are the statistics. */ static HTAB * GetShardStatistics(MultiConnection *connection, HTAB *shardIds) { StringInfo query = makeStringInfo(); appendStringInfoString( query, "WITH shard_names (shard_id, schema_name, table_name) AS ((VALUES "); bool isFirst = true; uint64 *shardIdPtr = NULL; HASH_SEQ_STATUS status; hash_seq_init(&status, shardIds); while ((shardIdPtr = hash_seq_search(&status)) != NULL) { uint64 shardId = *shardIdPtr; ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; char *shardName = get_rel_name(relationId); AppendShardIdToName(&shardName, shardId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); if (!isFirst) { appendStringInfo(query, ", "); } appendStringInfo(query, "(" UINT64_FORMAT ",%s,%s)", shardId, quote_literal_cstr(schemaName), quote_literal_cstr(shardName)); isFirst = false; } appendStringInfoString(query, "))"); appendStringInfoString( query, " SELECT shard_id, coalesce(pg_total_relation_size(tables.relid),0), tables.lsn" /* for each shard in shardIds */ " FROM shard_names" /* check if its name can be found in pg_class, if so return size */ " LEFT JOIN" " (SELECT c.oid AS relid, c.relname, n.nspname, ss.latest_end_lsn AS lsn" " FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " " LEFT JOIN pg_subscription_rel sr ON sr.srrelid = c.oid " " LEFT JOIN pg_stat_subscription ss ON sr.srsubid = ss.subid) tables" " ON tables.relname = shard_names.table_name AND" " tables.nspname = shard_names.schema_name "); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(connection, query->data, &result); if (queryResult != RESPONSE_OKAY) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot get the size because of a connection error"))); } int rowCount = PQntuples(result); int colCount = PQnfields(result); /* This is not expected to ever happen, but we check just to be sure */ if (colCount < 2) { ereport(ERROR, (errmsg("unexpected number of columns returned by: %s", query->data))); } HASHCTL info = { .keysize = sizeof(uint64), .entrysize = sizeof(ShardStatistics), .hcxt = CurrentMemoryContext }; HTAB *shardStatistics = hash_create("ShardStatisticsHash", 32, &info, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { char *shardIdString = PQgetvalue(result, rowIndex, 0); uint64 shardId = strtou64(shardIdString, NULL, 10); char *sizeString = PQgetvalue(result, rowIndex, 1); uint64 totalSize = strtou64(sizeString, NULL, 10); ShardStatistics *statistics = hash_search(shardStatistics, &shardId, HASH_ENTER, NULL); statistics->totalSize = totalSize; if (PQgetisnull(result, rowIndex, 2)) { statistics->shardLSN = InvalidXLogRecPtr; } else { char *LSNString = PQgetvalue(result, rowIndex, 2); Datum LSNDatum = DirectFunctionCall1(pg_lsn_in, CStringGetDatum(LSNString)); statistics->shardLSN = DatumGetLSN(LSNDatum); } } PQclear(result); bool raiseErrors = true; ClearResults(connection, raiseErrors); return shardStatistics; } /* * GetMovedShardIdsByWorker groups the shard ids in the provided steps by * worker. It returns a hashmap that contains a set of these shard ids. */ static HTAB * GetMovedShardIdsByWorker(PlacementUpdateEventProgress *steps, int stepCount, bool fromSource) { HASHCTL info = { .keysize = sizeof(WorkerHashKey), .entrysize = sizeof(WorkerShardIds), .hcxt = CurrentMemoryContext }; HTAB *shardsByWorker = hash_create("GetRebalanceStepsByWorker", 32, &info, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); for (int stepIndex = 0; stepIndex < stepCount; stepIndex++) { PlacementUpdateEventProgress *step = &(steps[stepIndex]); AddToWorkerShardIdSet(shardsByWorker, step->sourceName, step->sourcePort, step->shardId); if (pg_atomic_read_u64(&step->progress) == REBALANCE_PROGRESS_WAITING) { /* * shard move has not started so we don't need target stats for * this shard */ continue; } AddToWorkerShardIdSet(shardsByWorker, step->targetName, step->targetPort, step->shardId); } return shardsByWorker; } /* * AddToWorkerShardIdSet adds the shard id to the shard id set for the * specified worker in the shardsByWorker hashmap. */ static void AddToWorkerShardIdSet(HTAB *shardsByWorker, char *workerName, int workerPort, uint64 shardId) { WorkerHashKey workerKey = { 0 }; strlcpy(workerKey.hostname, workerName, MAX_NODE_LENGTH); workerKey.port = workerPort; bool isFound = false; WorkerShardIds *workerShardIds = hash_search(shardsByWorker, &workerKey, HASH_ENTER, &isFound); if (!isFound) { HASHCTL info = { .keysize = sizeof(uint64), .entrysize = sizeof(uint64), .hcxt = CurrentMemoryContext }; workerShardIds->shardIds = hash_create( "WorkerShardIdsSet", 32, &info, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); } hash_search(workerShardIds->shardIds, &shardId, HASH_ENTER, NULL); } /* * NonColocatedDistRelationIdList returns a list of distributed table oids, one * for each existing colocation group. */ static List * NonColocatedDistRelationIdList(void) { List *relationIdList = NIL; List *allCitusTablesList = CitusTableTypeIdList(ANY_CITUS_TABLE_TYPE); Oid tableId = InvalidOid; /* allocate sufficient capacity for O(1) expected look-up time */ int capacity = (int) (list_length(allCitusTablesList) / 0.75) + 1; int flags = HASH_ELEM | HASH_CONTEXT | HASH_BLOBS; HASHCTL info = { .keysize = sizeof(Oid), .entrysize = sizeof(Oid), .hcxt = CurrentMemoryContext }; HTAB *alreadySelectedColocationIds = hash_create("RebalanceColocationIdSet", capacity, &info, flags); foreach_declared_oid(tableId, allCitusTablesList) { bool foundInSet = false; CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry( tableId); if (!IsCitusTableTypeCacheEntry(citusTableCacheEntry, DISTRIBUTED_TABLE)) { /* * We're only interested in distributed tables, should ignore * reference tables and citus local tables. */ continue; } if (citusTableCacheEntry->colocationId != INVALID_COLOCATION_ID) { hash_search(alreadySelectedColocationIds, &citusTableCacheEntry->colocationId, HASH_ENTER, &foundInSet); if (foundInSet) { continue; } } relationIdList = lappend_oid(relationIdList, tableId); } return relationIdList; } /* * RebalanceTableShards rebalances the shards for the relations inside the * relationIdList across the different workers. */ static void RebalanceTableShards(RebalanceOptions *options, Oid shardReplicationModeOid) { char transferMode = LookupShardTransferMode(shardReplicationModeOid); if (list_length(options->relationIdList) == 0) { EnsureReferenceTablesExistOnAllNodesExtended(transferMode); return; } char *operationName = "rebalance"; if (options->drainOnly) { operationName = "move"; } options->operationName = operationName; ErrorOnConcurrentRebalance(options); List *placementUpdateList = GetRebalanceSteps(options); if (transferMode == TRANSFER_MODE_AUTOMATIC) { /* * If the shard transfer mode is set to auto, we should check beforehand * if we are able to use logical replication to transfer shards or not. * We throw an error if any of the tables do not have a replica identity, which * is required for logical replication to replicate UPDATE and DELETE commands. */ PlacementUpdateEvent *placementUpdate = NULL; foreach_declared_ptr(placementUpdate, placementUpdateList) { Oid relationId = RelationIdForShard(placementUpdate->shardId); List *colocatedTableList = ColocatedTableList(relationId); VerifyTablesHaveReplicaIdentity(colocatedTableList); } } EnsureReferenceTablesExistOnAllNodesExtended(transferMode); if (list_length(placementUpdateList) == 0) { return; } /* * This uses the first relationId from the list, it's only used for display * purposes so it does not really matter which to show */ SetupRebalanceMonitor(placementUpdateList, linitial_oid(options->relationIdList), REBALANCE_PROGRESS_WAITING, PLACEMENT_UPDATE_STATUS_NOT_STARTED_YET); ExecutePlacementUpdates(placementUpdateList, shardReplicationModeOid, "Moving"); FinalizeCurrentProgressMonitor(); } /* * ErrorOnConcurrentRebalance raises an error with extra information when there is already * a rebalance running. */ static void ErrorOnConcurrentRebalance(RebalanceOptions *options) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, options->relationIdList) { /* this provides the legacy error when the lock can't be acquired */ AcquireRebalanceColocationLock(relationId, options->operationName); } int64 jobId = 0; if (HasNonTerminalJobOfType("rebalance", &jobId)) { ereport(ERROR, ( errmsg("A rebalance is already running as job %ld", jobId), errdetail("A rebalance was already scheduled as background job"), errhint("To monitor progress, run: SELECT * FROM " "citus_rebalance_status();"))); } } /* * GetColocationId function returns the colocationId of the shard in a PlacementUpdateEvent. */ static int64 GetColocationId(PlacementUpdateEvent *move) { ShardInterval *shardInterval = LoadShardInterval(move->shardId); CitusTableCacheEntry *citusTableCacheEntry = GetCitusTableCacheEntry( shardInterval->relationId); return citusTableCacheEntry->colocationId; } /* * InitializeShardMoveDependencies function creates the hash maps that we use to track * the latest moves so that subsequent moves with the same properties must take a dependency * on them. There are two hash maps. One is for tracking the latest move scheduled in a * given colocation group and the other one is for tracking source nodes of all moves. */ static ShardMoveDependencies InitializeShardMoveDependencies(bool ParallelTransferColocatedShards) { ShardMoveDependencies shardMoveDependencies; shardMoveDependencies.colocationDependencies = CreateSimpleHashWithNameAndSize(int64, ShardMoveDependencyInfo, "colocationDependencyHashMap", 6); shardMoveDependencies.nodeDependencies = CreateSimpleHashWithNameAndSize(int32, ShardMoveSourceNodeHashEntry, "nodeDependencyHashMap", 6); shardMoveDependencies.parallelTransferColocatedShards = ParallelTransferColocatedShards; return shardMoveDependencies; } /* * GenerateTaskMoveDependencyList creates and returns a List of taskIds that * the move must take a dependency on, given the shard move dependencies as input. */ static int64 * GenerateTaskMoveDependencyList(PlacementUpdateEvent *move, int64 colocationId, int64 *refTablesDepTaskIds, int refTablesDepTaskIdsCount, ShardMoveDependencies shardMoveDependencies, int *nDepends) { HTAB *dependsList = CreateSimpleHashSetWithNameAndSize(int64, "shardMoveDependencyList", 0); bool found; if (!shardMoveDependencies.parallelTransferColocatedShards) { /* Check if there exists a move in the same colocation group scheduled earlier. */ ShardMoveDependencyInfo *shardMoveDependencyInfo = hash_search( shardMoveDependencies.colocationDependencies, &colocationId, HASH_ENTER, & found); if (found) { hash_search(dependsList, &shardMoveDependencyInfo->taskId, HASH_ENTER, NULL); } } /* * Check if there exists moves scheduled earlier whose source node * overlaps with the current move's target node. * The earlier/first move might make space for the later/second move. * So we could run out of disk space (or at least overload the node) * if we move the second shard to it before the first one is moved away. */ ShardMoveSourceNodeHashEntry *shardMoveSourceNodeHashEntry = hash_search( shardMoveDependencies.nodeDependencies, &move->targetNode->nodeId, HASH_FIND, &found); if (found) { int64 *taskId = NULL; foreach_declared_ptr(taskId, shardMoveSourceNodeHashEntry->taskIds) { hash_search(dependsList, taskId, HASH_ENTER, NULL); } } *nDepends = hash_get_num_entries(dependsList); if (*nDepends == 0) { /* * shard copy can only start after finishing copy of reference table shards * so each shard task will have a dependency on the task that indicates the * copy complete of reference tables */ while (refTablesDepTaskIdsCount > 0) { int64 refTableTaskId = *refTablesDepTaskIds; hash_search(dependsList, &refTableTaskId, HASH_ENTER, NULL); refTablesDepTaskIds++; refTablesDepTaskIdsCount--; } } *nDepends = hash_get_num_entries(dependsList); int64 *dependsArray = NULL; if (*nDepends > 0) { HASH_SEQ_STATUS seq; dependsArray = palloc((*nDepends) * sizeof(int64)); hash_seq_init(&seq, dependsList); int i = 0; int64 *dependsTaskId; while ((dependsTaskId = (int64 *) hash_seq_search(&seq)) != NULL) { dependsArray[i++] = *dependsTaskId; } } return dependsArray; } /* * UpdateShardMoveDependencies function updates the dependency maps with the latest move's taskId. */ static void UpdateShardMoveDependencies(PlacementUpdateEvent *move, uint64 colocationId, int64 taskId, ShardMoveDependencies shardMoveDependencies) { if (!shardMoveDependencies.parallelTransferColocatedShards) { ShardMoveDependencyInfo *shardMoveDependencyInfo = hash_search( shardMoveDependencies.colocationDependencies, &colocationId, HASH_ENTER, NULL); shardMoveDependencyInfo->taskId = taskId; } bool found; ShardMoveSourceNodeHashEntry *shardMoveSourceNodeHashEntry = hash_search( shardMoveDependencies.nodeDependencies, &move->sourceNode->nodeId, HASH_ENTER, &found); if (!found) { shardMoveSourceNodeHashEntry->taskIds = NIL; } int64 *newTaskId = palloc0(sizeof(int64)); *newTaskId = taskId; shardMoveSourceNodeHashEntry->taskIds = lappend( shardMoveSourceNodeHashEntry->taskIds, newTaskId); } /* * RebalanceTableShardsBackground rebalances the shards for the relations * inside the relationIdList across the different workers. It does so using our * background job+task infrastructure. */ static int64 RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationModeOid, bool ParallelTransferReferenceTables, bool ParallelTransferColocatedShards) { if (list_length(options->relationIdList) == 0) { ereport(NOTICE, (errmsg("No tables to rebalance"))); return 0; } char *operationName = "rebalance"; if (options->drainOnly) { operationName = "move"; } options->operationName = operationName; ErrorOnConcurrentRebalance(options); const char shardTransferMode = LookupShardTransferMode(shardReplicationModeOid); List *colocatedTableList = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, options->relationIdList) { colocatedTableList = list_concat(colocatedTableList, ColocatedTableList(relationId)); } Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { EnsureTableOwner(colocatedTableId); } List *placementUpdateList = GetRebalanceSteps(options); if (list_length(placementUpdateList) == 0) { ereport(NOTICE, (errmsg("No moves available for rebalancing"))); return 0; } if (shardTransferMode == TRANSFER_MODE_AUTOMATIC) { /* * If the shard transfer mode is set to auto, we should check beforehand * if we are able to use logical replication to transfer shards or not. * We throw an error if any of the tables do not have a replica identity, which * is required for logical replication to replicate UPDATE and DELETE commands. */ PlacementUpdateEvent *placementUpdate = NULL; foreach_declared_ptr(placementUpdate, placementUpdateList) { relationId = RelationIdForShard(placementUpdate->shardId); List *colocatedTables = ColocatedTableList(relationId); VerifyTablesHaveReplicaIdentity(colocatedTables); } } DropOrphanedResourcesInSeparateTransaction(); /* find the name of the shard transfer mode to interpolate in the scheduled command */ Datum shardTranferModeLabelDatum = DirectFunctionCall1(enum_out, shardReplicationModeOid); char *shardTranferModeLabel = DatumGetCString(shardTranferModeLabelDatum); /* schedule planned moves */ int64 jobId = CreateBackgroundJob("rebalance", "Rebalance all colocation groups"); /* buffer used to construct the sql command for the tasks */ StringInfoData buf = { 0 }; initStringInfo(&buf); List *referenceTableIdList = NIL; int64 *refTablesDepTaskIds = NULL; int refTablesDepTaskIdsCount = 0; if (HasNodesWithMissingReferenceTables(&referenceTableIdList)) { if (shardTransferMode == TRANSFER_MODE_AUTOMATIC) { VerifyTablesHaveReplicaIdentity(referenceTableIdList); } /* * Reference tables need to be copied to (newly-added) nodes, this needs to be the * first task before we can move any other table. */ if (ParallelTransferReferenceTables) { refTablesDepTaskIds = ScheduleTasksToParallelCopyReferenceTablesOnAllMissingNodes( jobId, shardTransferMode, &refTablesDepTaskIdsCount); ereport(DEBUG2, (errmsg("%d dependent copy reference table tasks for job %ld", refTablesDepTaskIdsCount, jobId), errdetail("Rebalance scheduled as background job"), errhint("To monitor progress, run: SELECT * FROM " "citus_rebalance_status();"))); } else { /* Move all reference tables as single task. Classical way */ appendStringInfo(&buf, "SELECT pg_catalog.replicate_reference_tables(%s)", quote_literal_cstr(shardTranferModeLabel)); int32 nodesInvolved[] = { 0 }; /* replicate_reference_tables permissions require superuser */ Oid superUserId = CitusExtensionOwner(); BackgroundTask *task = ScheduleBackgroundTask(jobId, superUserId, buf.data, 0, NULL, 0, nodesInvolved); refTablesDepTaskIds = palloc0(sizeof(int64)); refTablesDepTaskIds[0] = task->taskid; refTablesDepTaskIdsCount = 1; } } PlacementUpdateEvent *move = NULL; ShardMoveDependencies shardMoveDependencies = InitializeShardMoveDependencies(ParallelTransferColocatedShards); foreach_declared_ptr(move, placementUpdateList) { resetStringInfo(&buf); appendStringInfo(&buf, "SELECT pg_catalog.citus_move_shard_placement(%ld,%u,%u,%s)", move->shardId, move->sourceNode->nodeId, move->targetNode->nodeId, quote_literal_cstr(shardTranferModeLabel)); int64 colocationId = GetColocationId(move); int nDepends = 0; int64 *dependsArray = GenerateTaskMoveDependencyList(move, colocationId, refTablesDepTaskIds, refTablesDepTaskIdsCount, shardMoveDependencies, &nDepends); int32 nodesInvolved[2] = { 0 }; nodesInvolved[0] = move->sourceNode->nodeId; nodesInvolved[1] = move->targetNode->nodeId; BackgroundTask *task = ScheduleBackgroundTask(jobId, GetUserId(), buf.data, nDepends, dependsArray, 2, nodesInvolved); UpdateShardMoveDependencies(move, colocationId, task->taskid, shardMoveDependencies); } ereport(NOTICE, (errmsg("Scheduled %d moves as job %ld", list_length(placementUpdateList), jobId), errdetail("Rebalance scheduled as background job"), errhint("To monitor progress, run: SELECT * FROM " "citus_rebalance_status();"))); return jobId; } /* * UpdateShardPlacement copies or moves a shard placement by calling * the corresponding functions in Citus in a subtransaction. */ static void UpdateShardPlacement(PlacementUpdateEvent *placementUpdateEvent, List *responsiveNodeList, Oid shardReplicationModeOid) { PlacementUpdateType updateType = placementUpdateEvent->updateType; uint64 shardId = placementUpdateEvent->shardId; WorkerNode *sourceNode = placementUpdateEvent->sourceNode; WorkerNode *targetNode = placementUpdateEvent->targetNode; Datum shardTranferModeLabelDatum = DirectFunctionCall1(enum_out, shardReplicationModeOid); char *shardTranferModeLabel = DatumGetCString(shardTranferModeLabelDatum); StringInfo placementUpdateCommand = makeStringInfo(); /* if target node is not responsive, don't continue */ bool targetResponsive = WorkerNodeListContains(responsiveNodeList, targetNode->workerName, targetNode->workerPort); if (!targetResponsive) { ereport(ERROR, (errmsg("target node %s:%d is not responsive", targetNode->workerName, targetNode->workerPort))); } /* if source node is not responsive, don't continue */ bool sourceResponsive = WorkerNodeListContains(responsiveNodeList, sourceNode->workerName, sourceNode->workerPort); if (!sourceResponsive) { ereport(ERROR, (errmsg("source node %s:%d is not responsive", sourceNode->workerName, sourceNode->workerPort))); } if (updateType == PLACEMENT_UPDATE_MOVE) { appendStringInfo(placementUpdateCommand, "SELECT pg_catalog.citus_move_shard_placement(%ld,%u,%u,%s)", shardId, sourceNode->nodeId, targetNode->nodeId, quote_literal_cstr(shardTranferModeLabel)); } else if (updateType == PLACEMENT_UPDATE_COPY) { appendStringInfo(placementUpdateCommand, "SELECT pg_catalog.citus_copy_shard_placement(%ld,%u,%u,%s)", shardId, sourceNode->nodeId, targetNode->nodeId, quote_literal_cstr(shardTranferModeLabel)); } else { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only moving or copying shards is supported"))); } UpdateColocatedShardPlacementProgress(shardId, sourceNode->workerName, sourceNode->workerPort, REBALANCE_PROGRESS_MOVING); /* * In case of failure, we throw an error such that rebalance_table_shards * fails early. */ ExecuteRebalancerCommandInSeparateTransaction(placementUpdateCommand->data); UpdateColocatedShardPlacementProgress(shardId, sourceNode->workerName, sourceNode->workerPort, REBALANCE_PROGRESS_MOVED); } /* * ExecuteRebalancerCommandInSeparateTransaction runs a command in a separate * transaction that is commited right away. This is useful for things that you * don't want to rollback when the current transaction is rolled back. * Set true to 'useExclusiveTransactionBlock' to initiate a BEGIN and COMMIT statements. */ void ExecuteRebalancerCommandInSeparateTransaction(char *command) { int connectionFlag = FORCE_NEW_CONNECTION; MultiConnection *connection = GetNodeConnection(connectionFlag, LocalHostName, PostPortNumber); List *commandList = NIL; commandList = lappend(commandList, psprintf( "SET LOCAL application_name TO '%s%ld'", CITUS_REBALANCER_APPLICATION_NAME_PREFIX, GetGlobalPID())); if (PropagateSessionSettingsForLoopbackConnection) { List *setCommands = GetSetCommandListForNewConnections(); char *setCommand = NULL; foreach_declared_ptr(setCommand, setCommands) { commandList = lappend(commandList, setCommand); } } commandList = lappend(commandList, command); SendCommandListToWorkerOutsideTransactionWithConnection(connection, commandList); CloseConnection(connection); } /* * GetSetCommandListForNewConnections returns a list of SET statements to * be executed in new connections to worker nodes. */ static List * GetSetCommandListForNewConnections(void) { List *commandList = NIL; int gucCount = 0; struct config_generic **guc_vars = get_guc_variables(&gucCount); for (int gucIndex = 0; gucIndex < gucCount; gucIndex++) { struct config_generic *var = (struct config_generic *) guc_vars[gucIndex]; if (var->source == PGC_S_SESSION && IsSettingSafeToPropagate(var->name)) { const char *variableValue = GetConfigOption(var->name, true, true); commandList = lappend(commandList, psprintf("SET LOCAL %s TO '%s';", var->name, variableValue)); } } return commandList; } /* * RebalancePlacementUpdates returns a list of placement updates which makes the * cluster balanced. We move shards to these nodes until all nodes become utilized. * We consider a node under-utilized if it has less than floor((1.0 - threshold) * * placementCountAverage) shard placements. In each iteration we choose the node * with maximum number of shard placements as the source, and we choose the node * with minimum number of shard placements as the target. Then we choose a shard * which is placed in the source node but not in the target node as the shard to * move. * * The activeShardPlacementListList argument contains a list of lists of active shard * placements. Each of these lists are balanced independently. This is used to * make sure different colocation groups are balanced separately, so each list * contains the placements of a colocation group. */ List * RebalancePlacementUpdates(List *workerNodeList, List *activeShardPlacementListList, double threshold, int32 maxShardMoves, bool drainOnly, float4 improvementThreshold, RebalancePlanFunctions *functions) { List *rebalanceStates = NIL; RebalanceState *state = NULL; List *shardPlacementList = NIL; List *placementUpdateList = NIL; foreach_declared_ptr(shardPlacementList, activeShardPlacementListList) { state = InitRebalanceState(workerNodeList, shardPlacementList, functions); rebalanceStates = lappend(rebalanceStates, state); } foreach_declared_ptr(state, rebalanceStates) { state->placementUpdateList = placementUpdateList; MoveShardsAwayFromDisallowedNodes(state); placementUpdateList = state->placementUpdateList; } if (!drainOnly) { foreach_declared_ptr(state, rebalanceStates) { state->placementUpdateList = placementUpdateList; /* calculate lower bound for placement count */ float4 averageUtilization = (state->totalCost / state->totalCapacity); float4 utilizationLowerBound = ((1.0 - threshold) * averageUtilization); float4 utilizationUpperBound = ((1.0 + threshold) * averageUtilization); bool moreMovesAvailable = true; while (list_length(state->placementUpdateList) < maxShardMoves && moreMovesAvailable) { moreMovesAvailable = FindAndMoveShardCost( utilizationLowerBound, utilizationUpperBound, improvementThreshold, state); } placementUpdateList = state->placementUpdateList; if (moreMovesAvailable) { ereport(NOTICE, (errmsg( "Stopped searching before we were out of moves. " "Please rerun the rebalancer after it's finished " "for a more optimal placement."))); break; } } } foreach_declared_ptr(state, rebalanceStates) { hash_destroy(state->placementsHash); } int64 ignoredMoves = 0; foreach_declared_ptr(state, rebalanceStates) { ignoredMoves += state->ignoredMoves; } if (ignoredMoves > 0) { if (MaxRebalancerLoggedIgnoredMoves == -1 || ignoredMoves <= MaxRebalancerLoggedIgnoredMoves) { ereport(NOTICE, ( errmsg( "Ignored %ld moves, all of which are shown in notices above", ignoredMoves ), errhint( "If you do want these moves to happen, try changing improvement_threshold to a lower value than what it is now (%g).", improvementThreshold) )); } else { ereport(NOTICE, ( errmsg( "Ignored %ld moves, %d of which are shown in notices above", ignoredMoves, MaxRebalancerLoggedIgnoredMoves ), errhint( "If you do want these moves to happen, try changing improvement_threshold to a lower value than what it is now (%g).", improvementThreshold) )); } } return placementUpdateList; } /* * InitRebalanceState sets up a RebalanceState for it's arguments. The * RebalanceState contains the information needed to calculate shard moves. */ static RebalanceState * InitRebalanceState(List *workerNodeList, List *shardPlacementList, RebalancePlanFunctions *functions) { ShardPlacement *placement = NULL; HASH_SEQ_STATUS status; WorkerNode *workerNode = NULL; RebalanceState *state = palloc0(sizeof(RebalanceState)); state->functions = functions; state->placementsHash = ShardPlacementsListToHash(shardPlacementList); /* create empty fill state for all of the worker nodes */ foreach_declared_ptr(workerNode, workerNodeList) { NodeFillState *fillState = palloc0(sizeof(NodeFillState)); fillState->node = workerNode; fillState->capacity = functions->nodeCapacity(workerNode, functions->context); /* * Set the utilization here although the totalCost is not set yet. This * is needed to set the utilization to INFINITY when the capacity is 0. */ fillState->utilization = CalculateUtilization(fillState->totalCost, fillState->capacity); state->fillStateListAsc = lappend(state->fillStateListAsc, fillState); state->fillStateListDesc = lappend(state->fillStateListDesc, fillState); state->totalCapacity += fillState->capacity; } /* Fill the fill states for all of the worker nodes based on the placements */ foreach_htab(placement, &status, state->placementsHash) { ShardCost *shardCost = palloc0(sizeof(ShardCost)); NodeFillState *fillState = FindFillStateForPlacement(state, placement); Assert(fillState != NULL); *shardCost = functions->shardCost(placement->shardId, functions->context); fillState->totalCost += shardCost->cost; fillState->utilization = CalculateUtilization(fillState->totalCost, fillState->capacity); fillState->shardCostListDesc = lappend(fillState->shardCostListDesc, shardCost); fillState->shardCostListDesc = SortList(fillState->shardCostListDesc, CompareShardCostDesc); state->totalCost += shardCost->cost; if (!functions->shardAllowedOnNode(placement->shardId, fillState->node, functions->context)) { DisallowedPlacement *disallowed = palloc0(sizeof(DisallowedPlacement)); disallowed->shardCost = shardCost; disallowed->fillState = fillState; state->disallowedPlacementList = lappend(state->disallowedPlacementList, disallowed); } } foreach_htab_cleanup(placement, &status); state->fillStateListAsc = SortList(state->fillStateListAsc, CompareNodeFillStateAsc); state->fillStateListDesc = SortList(state->fillStateListDesc, CompareNodeFillStateDesc); CheckRebalanceStateInvariants(state); return state; } /* * CalculateUtilization returns INFINITY when capacity is 0 and * totalCost/capacity otherwise. */ static float4 CalculateUtilization(float4 totalCost, float4 capacity) { if (capacity <= 0) { return INFINITY; } return totalCost / capacity; } /* * FindFillStateForPlacement finds the fillState for the workernode that * matches the placement. */ static NodeFillState * FindFillStateForPlacement(RebalanceState *state, ShardPlacement *placement) { NodeFillState *fillState = NULL; /* Find the correct fill state to add the placement to and do that */ foreach_declared_ptr(fillState, state->fillStateListAsc) { if (IsPlacementOnWorkerNode(placement, fillState->node)) { return fillState; } } return NULL; } /* * CompareNodeFillStateAsc can be used to sort fill states from empty to full. */ static int CompareNodeFillStateAsc(const void *void1, const void *void2) { const NodeFillState *a = *((const NodeFillState **) void1); const NodeFillState *b = *((const NodeFillState **) void2); if (a->utilization < b->utilization) { return -1; } if (a->utilization > b->utilization) { return 1; } /* * If utilization prefer nodes with more capacity, since utilization will * grow slower on those */ if (a->capacity > b->capacity) { return -1; } if (a->capacity < b->capacity) { return 1; } /* Finally differentiate by node id */ if (a->node->nodeId < b->node->nodeId) { return -1; } return a->node->nodeId > b->node->nodeId; } /* * CompareNodeFillStateDesc can be used to sort fill states from full to empty. */ static int CompareNodeFillStateDesc(const void *a, const void *b) { return -CompareNodeFillStateAsc(a, b); } /* * CompareShardCostAsc can be used to sort shard costs from low cost to high * cost. */ static int CompareShardCostAsc(const void *void1, const void *void2) { const ShardCost *a = *((const ShardCost **) void1); const ShardCost *b = *((const ShardCost **) void2); if (a->cost < b->cost) { return -1; } if (a->cost > b->cost) { return 1; } /* make compare function (more) stable for tests */ if (a->shardId > b->shardId) { return -1; } return a->shardId < b->shardId; } /* * CompareShardCostDesc can be used to sort shard costs from high cost to low * cost. */ static int CompareShardCostDesc(const void *a, const void *b) { return -CompareShardCostAsc(a, b); } /* * MoveShardsAwayFromDisallowedNodes returns a list of placement updates that * move any shards that are not allowed on their current node to a node that * they are allowed on. */ static void MoveShardsAwayFromDisallowedNodes(RebalanceState *state) { DisallowedPlacement *disallowedPlacement = NULL; state->disallowedPlacementList = SortList(state->disallowedPlacementList, CompareDisallowedPlacementDesc); /* Move shards off of nodes they are not allowed on */ foreach_declared_ptr(disallowedPlacement, state->disallowedPlacementList) { NodeFillState *targetFillState = FindAllowedTargetFillState( state, disallowedPlacement->shardCost->shardId); if (targetFillState == NULL) { ereport(WARNING, (errmsg( "Not allowed to move shard " UINT64_FORMAT " anywhere from %s:%d", disallowedPlacement->shardCost->shardId, disallowedPlacement->fillState->node->workerName, disallowedPlacement->fillState->node->workerPort ))); continue; } MoveShardCost(disallowedPlacement->fillState, targetFillState, disallowedPlacement->shardCost, state); } } /* * CompareDisallowedPlacementAsc can be used to sort disallowed placements from * low cost to high cost. */ static int CompareDisallowedPlacementAsc(const void *void1, const void *void2) { const DisallowedPlacement *a = *((const DisallowedPlacement **) void1); const DisallowedPlacement *b = *((const DisallowedPlacement **) void2); return CompareShardCostAsc(&(a->shardCost), &(b->shardCost)); } /* * CompareDisallowedPlacementDesc can be used to sort disallowed placements from * high cost to low cost. */ static int CompareDisallowedPlacementDesc(const void *a, const void *b) { return -CompareDisallowedPlacementAsc(a, b); } /* * FindAllowedTargetFillState finds the first fill state in fillStateListAsc * where the shard can be moved to. */ static NodeFillState * FindAllowedTargetFillState(RebalanceState *state, uint64 shardId) { NodeFillState *targetFillState = NULL; foreach_declared_ptr(targetFillState, state->fillStateListAsc) { bool hasShard = PlacementsHashFind( state->placementsHash, shardId, targetFillState->node); if (!hasShard && state->functions->shardAllowedOnNode( shardId, targetFillState->node, state->functions->context)) { bool targetHasShard = PlacementsHashFind(state->placementsHash, shardId, targetFillState->node); /* skip if the shard is already placed on the target node */ if (!targetHasShard) { return targetFillState; } } } return NULL; } /* * MoveShardCost moves a shardcost from the source to the target fill states * and updates the RebalanceState accordingly. What it does in detail is: * 1. add a placement update to state->placementUpdateList * 2. update state->placementsHash * 3. update totalcost, utilization and shardCostListDesc in source and target * 4. resort state->fillStateListAsc/Desc */ static void MoveShardCost(NodeFillState *sourceFillState, NodeFillState *targetFillState, ShardCost *shardCost, RebalanceState *state) { uint64 shardIdToMove = shardCost->shardId; /* construct the placement update */ PlacementUpdateEvent *placementUpdateEvent = palloc0(sizeof(PlacementUpdateEvent)); placementUpdateEvent->updateType = PLACEMENT_UPDATE_MOVE; placementUpdateEvent->shardId = shardIdToMove; placementUpdateEvent->sourceNode = sourceFillState->node; placementUpdateEvent->targetNode = targetFillState->node; /* record the placement update */ state->placementUpdateList = lappend(state->placementUpdateList, placementUpdateEvent); /* update the placements hash and the node shard lists */ PlacementsHashRemove(state->placementsHash, shardIdToMove, sourceFillState->node); PlacementsHashEnter(state->placementsHash, shardIdToMove, targetFillState->node); sourceFillState->totalCost -= shardCost->cost; sourceFillState->utilization = CalculateUtilization(sourceFillState->totalCost, sourceFillState->capacity); sourceFillState->shardCostListDesc = list_delete_ptr( sourceFillState->shardCostListDesc, shardCost); targetFillState->totalCost += shardCost->cost; targetFillState->utilization = CalculateUtilization(targetFillState->totalCost, targetFillState->capacity); targetFillState->shardCostListDesc = lappend(targetFillState->shardCostListDesc, shardCost); targetFillState->shardCostListDesc = SortList(targetFillState->shardCostListDesc, CompareShardCostDesc); state->fillStateListAsc = SortList(state->fillStateListAsc, CompareNodeFillStateAsc); state->fillStateListDesc = SortList(state->fillStateListDesc, CompareNodeFillStateDesc); CheckRebalanceStateInvariants(state); } /* * FindAndMoveShardCost is the main rebalancing algorithm. This takes the * current state and returns a list with a new move appended that improves the * balance of shards. The algorithm is greedy and will use the first new move * that improves the balance. It finds nodes by trying to move a shard from the * most utilized node (highest utilization) to the emptiest node (lowest * utilization). If no moves are possible it will try the second emptiest node * until it tried all of them. Then it wil try the second fullest node. If it * was able to find a move it will return true and false if it couldn't. * * This algorithm won't necessarily result in the best possible balance. Getting * the best balance is an NP problem, so it's not feasible to go for the best * balance. This algorithm was chosen because of the following reasons: * 1. Literature research showed that similar problems would get within 2X of * the optimal balance with a greedy algoritm. * 2. Every move will always improve the balance. So if the user stops a * rebalance midway through, they will never be in a worse situation than * before. * 3. It's pretty easy to reason about. * 4. It's simple to implement. * * utilizationLowerBound and utilizationUpperBound are used to indicate what * the target utilization range of all nodes is. If they are within this range, * then balance is good enough. If all nodes are in this range then the cluster * is considered balanced and no more moves are done. This is mostly useful for * the by_disk_size rebalance strategy. If we wouldn't have this then the * rebalancer could become flappy in certain cases. * * improvementThreshold is a threshold that can be used to ignore moves when * they only improve the balance a little relative to the cost of the shard. * Again this is mostly useful for the by_disk_size rebalance strategy. * Without this threshold the rebalancer would move a shard of 1TB when this * move only improves the cluster by 10GB. */ static bool FindAndMoveShardCost(float4 utilizationLowerBound, float4 utilizationUpperBound, float4 improvementThreshold, RebalanceState *state) { NodeFillState *sourceFillState = NULL; NodeFillState *targetFillState = NULL; /* * find a source node for the move, starting at the node with the highest * utilization */ foreach_declared_ptr(sourceFillState, state->fillStateListDesc) { /* Don't move shards away from nodes that are already too empty, we're * done searching */ if (sourceFillState->utilization <= utilizationLowerBound) { return false; } /* find a target node for the move, starting at the node with the * lowest utilization */ foreach_declared_ptr(targetFillState, state->fillStateListAsc) { ShardCost *shardCost = NULL; /* Don't add more shards to nodes that are already at the upper * bound. We should try the next source node now because further * target nodes will also be above the upper bound */ if (targetFillState->utilization >= utilizationUpperBound) { break; } /* Don't move a shard between nodes that both have decent * utilization. We should try the next source node now because * further target nodes will also have have decent utilization */ if (targetFillState->utilization >= utilizationLowerBound && sourceFillState->utilization <= utilizationUpperBound) { break; } /* find a shardcost that can be moved between between nodes that * makes the cost distribution more equal */ foreach_declared_ptr(shardCost, sourceFillState->shardCostListDesc) { bool targetHasShard = PlacementsHashFind(state->placementsHash, shardCost->shardId, targetFillState->node); float4 newTargetTotalCost = targetFillState->totalCost + shardCost->cost; float4 newTargetUtilization = CalculateUtilization( newTargetTotalCost, targetFillState->capacity); float4 newSourceTotalCost = sourceFillState->totalCost - shardCost->cost; float4 newSourceUtilization = CalculateUtilization( newSourceTotalCost, sourceFillState->capacity); /* Skip shards that already are on the node */ if (targetHasShard) { continue; } /* Skip shards that already are not allowed on the node */ if (!state->functions->shardAllowedOnNode(shardCost->shardId, targetFillState->node, state->functions->context)) { continue; } /* * If the target is still less utilized than the source, then * this is clearly a good move. And if they are equally * utilized too. */ if (newTargetUtilization <= newSourceUtilization) { MoveShardCost(sourceFillState, targetFillState, shardCost, state); return true; } /* * The target is now more utilized than the source. So we need * to determine if the move is a net positive for the overall * cost distribution. This means that the new highest * utilization of source and target is lower than the previous * highest, or the highest utilization is the same, but the * lowest increased. */ if (newTargetUtilization > sourceFillState->utilization) { continue; } if (newTargetUtilization == sourceFillState->utilization && newSourceUtilization <= targetFillState->utilization ) /* lgtm[cpp/equality-on-floats] */ { /* * this can trigger when capacity of the nodes is not the * same. Example (also a test): * - node with capacity 3 * - node with capacity 1 * - 3 shards with cost 1 * Best distribution would be 2 shards on node with * capacity 3 and one on node with capacity 1 */ continue; } /* * fmaxf and fminf here are only needed for cases when nodes * have different capacities. If they are the same, then both * arguments are equal. */ float4 utilizationImprovement = fmaxf( sourceFillState->utilization - newTargetUtilization, newSourceUtilization - targetFillState->utilization ); float4 utilizationAddedByShard = fminf( newTargetUtilization - targetFillState->utilization, sourceFillState->utilization - newSourceUtilization ); /* * If the shard causes a lot of utilization, but the * improvement which is gained by moving it is small, then we * ignore the move. Probably there are other shards that are * better candidates, and in any case it's probably not worth * the effort to move the this shard. * * One of the main cases this tries to avoid is the rebalancer * moving a very large shard with the "by_disk_size" strategy * when that only gives a small benefit in data distribution. */ float4 normalizedUtilizationImprovement = utilizationImprovement / utilizationAddedByShard; if (normalizedUtilizationImprovement < improvementThreshold) { state->ignoredMoves++; if (MaxRebalancerLoggedIgnoredMoves == -1 || state->ignoredMoves <= MaxRebalancerLoggedIgnoredMoves) { ereport(NOTICE, ( errmsg( "Ignoring move of shard %ld from %s:%d to %s:%d, because the move only brings a small improvement relative to the shard its size", shardCost->shardId, sourceFillState->node->workerName, sourceFillState->node->workerPort, targetFillState->node->workerName, targetFillState->node->workerPort ), errdetail( "The balance improvement of %g is lower than the improvement_threshold of %g", normalizedUtilizationImprovement, improvementThreshold ) )); } continue; } MoveShardCost(sourceFillState, targetFillState, shardCost, state); return true; } } } return false; } /* * ReplicationPlacementUpdates returns a list of placement updates which * replicates shard placements that need re-replication. To do this, the * function loops over the active shard placements, and for each shard placement * which needs to be re-replicated, it chooses an active worker node with * smallest number of shards as the target node. */ List * ReplicationPlacementUpdates(List *workerNodeList, List *activeShardPlacementList, int shardReplicationFactor) { List *placementUpdateList = NIL; ListCell *shardPlacementCell = NULL; uint32 workerNodeIndex = 0; HTAB *placementsHash = ShardPlacementsListToHash(activeShardPlacementList); uint32 workerNodeCount = list_length(workerNodeList); /* get number of shards per node */ uint32 *shardCountArray = palloc0(workerNodeCount * sizeof(uint32)); foreach(shardPlacementCell, activeShardPlacementList) { ShardPlacement *placement = lfirst(shardPlacementCell); for (workerNodeIndex = 0; workerNodeIndex < workerNodeCount; workerNodeIndex++) { WorkerNode *node = list_nth(workerNodeList, workerNodeIndex); if (strncmp(node->workerName, placement->nodeName, WORKER_LENGTH) == 0 && node->workerPort == placement->nodePort) { shardCountArray[workerNodeIndex]++; break; } } } foreach(shardPlacementCell, activeShardPlacementList) { WorkerNode *sourceNode = NULL; WorkerNode *targetNode = NULL; uint32 targetNodeShardCount = UINT_MAX; uint32 targetNodeIndex = 0; ShardPlacement *placement = (ShardPlacement *) lfirst(shardPlacementCell); uint64 shardId = placement->shardId; /* skip the shard placement if it has enough replications */ int activePlacementCount = ShardActivePlacementCount(placementsHash, shardId, workerNodeList); if (activePlacementCount >= shardReplicationFactor) { continue; } /* * We can copy the shard from any active worker node that contains the * shard. */ for (workerNodeIndex = 0; workerNodeIndex < workerNodeCount; workerNodeIndex++) { WorkerNode *workerNode = list_nth(workerNodeList, workerNodeIndex); bool placementExists = PlacementsHashFind(placementsHash, shardId, workerNode); if (placementExists) { sourceNode = workerNode; break; } } /* * If we couldn't find any worker node which contains the shard, then * all copies of the shard are list and we should error out. */ if (sourceNode == NULL) { ereport(ERROR, (errmsg("could not find a source for shard " UINT64_FORMAT, shardId))); } /* * We can copy the shard to any worker node that doesn't contain the shard. * Among such worker nodes, we choose the worker node with minimum shard * count as the target. */ for (workerNodeIndex = 0; workerNodeIndex < workerNodeCount; workerNodeIndex++) { WorkerNode *workerNode = list_nth(workerNodeList, workerNodeIndex); if (!NodeCanHaveDistTablePlacements(workerNode)) { /* never replicate placements to nodes that should not have placements */ continue; } /* skip this node if it already contains the shard */ bool placementExists = PlacementsHashFind(placementsHash, shardId, workerNode); if (placementExists) { continue; } /* compare and change the target node */ if (shardCountArray[workerNodeIndex] < targetNodeShardCount) { targetNode = workerNode; targetNodeShardCount = shardCountArray[workerNodeIndex]; targetNodeIndex = workerNodeIndex; } } /* * If there is no worker node which doesn't contain the shard, then the * shard replication factor is greater than number of worker nodes, and * we should error out. */ if (targetNode == NULL) { ereport(ERROR, (errmsg("could not find a target for shard " UINT64_FORMAT, shardId))); } /* construct the placement update */ PlacementUpdateEvent *placementUpdateEvent = palloc0( sizeof(PlacementUpdateEvent)); placementUpdateEvent->updateType = PLACEMENT_UPDATE_COPY; placementUpdateEvent->shardId = shardId; placementUpdateEvent->sourceNode = sourceNode; placementUpdateEvent->targetNode = targetNode; /* record the placement update */ placementUpdateList = lappend(placementUpdateList, placementUpdateEvent); /* update the placements hash and the shard count array */ PlacementsHashEnter(placementsHash, shardId, targetNode); shardCountArray[targetNodeIndex]++; } hash_destroy(placementsHash); return placementUpdateList; } /* * ShardActivePlacementCount returns the number of active placements for the * given shard which are placed at the active worker nodes. */ static int ShardActivePlacementCount(HTAB *activePlacementsHash, uint64 shardId, List *activeWorkerNodeList) { int shardActivePlacementCount = 0; ListCell *workerNodeCell = NULL; foreach(workerNodeCell, activeWorkerNodeList) { WorkerNode *workerNode = lfirst(workerNodeCell); bool placementExists = PlacementsHashFind(activePlacementsHash, shardId, workerNode); if (placementExists) { shardActivePlacementCount++; } } return shardActivePlacementCount; } /* * ShardPlacementsListToHash creates and returns a hash set from a shard * placement list. */ static HTAB * ShardPlacementsListToHash(List *shardPlacementList) { ListCell *shardPlacementCell = NULL; HASHCTL info; int shardPlacementCount = list_length(shardPlacementList); memset(&info, 0, sizeof(info)); info.keysize = sizeof(ShardPlacement); info.entrysize = sizeof(ShardPlacement); info.hash = PlacementsHashHashCode; info.match = PlacementsHashCompare; info.hcxt = CurrentMemoryContext; int hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); HTAB *shardPlacementsHash = hash_create("ActivePlacements Hash", shardPlacementCount, &info, hashFlags); foreach(shardPlacementCell, shardPlacementList) { ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell); void *hashKey = (void *) shardPlacement; hash_search(shardPlacementsHash, hashKey, HASH_ENTER, NULL); } return shardPlacementsHash; } /* * PlacementsHashFind returns true if there exists a shard placement with the * given workerNode and shard id in the given placements hash, otherwise it * returns false. */ static bool PlacementsHashFind(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode) { bool placementFound = false; ShardPlacement shardPlacement; memset(&shardPlacement, 0, sizeof(shardPlacement)); shardPlacement.shardId = shardId; shardPlacement.nodeName = workerNode->workerName; shardPlacement.nodePort = workerNode->workerPort; void *hashKey = (void *) (&shardPlacement); hash_search(placementsHash, hashKey, HASH_FIND, &placementFound); return placementFound; } /* * PlacementsHashEnter enters a shard placement for the given worker node and * shard id to the given placements hash. */ static void PlacementsHashEnter(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode) { ShardPlacement shardPlacement; memset(&shardPlacement, 0, sizeof(shardPlacement)); shardPlacement.shardId = shardId; shardPlacement.nodeName = workerNode->workerName; shardPlacement.nodePort = workerNode->workerPort; void *hashKey = (void *) (&shardPlacement); hash_search(placementsHash, hashKey, HASH_ENTER, NULL); } /* * PlacementsHashRemove removes the shard placement for the given worker node and * shard id from the given placements hash. */ static void PlacementsHashRemove(HTAB *placementsHash, uint64 shardId, WorkerNode *workerNode) { ShardPlacement shardPlacement; memset(&shardPlacement, 0, sizeof(shardPlacement)); shardPlacement.shardId = shardId; shardPlacement.nodeName = workerNode->workerName; shardPlacement.nodePort = workerNode->workerPort; void *hashKey = (void *) (&shardPlacement); hash_search(placementsHash, hashKey, HASH_REMOVE, NULL); } /* * PlacementsHashCompare compares two shard placements using shard id, node name, * and node port number. */ static int PlacementsHashCompare(const void *lhsKey, const void *rhsKey, Size keySize) { const ShardPlacement *placementLhs = (const ShardPlacement *) lhsKey; const ShardPlacement *placementRhs = (const ShardPlacement *) rhsKey; int shardIdCompare = 0; /* first, compare by shard id */ if (placementLhs->shardId < placementRhs->shardId) { shardIdCompare = -1; } else if (placementLhs->shardId > placementRhs->shardId) { shardIdCompare = 1; } if (shardIdCompare != 0) { return shardIdCompare; } /* then, compare by node name */ int nodeNameCompare = strncmp(placementLhs->nodeName, placementRhs->nodeName, WORKER_LENGTH); if (nodeNameCompare != 0) { return nodeNameCompare; } /* finally, compare by node port */ int nodePortCompare = placementLhs->nodePort - placementRhs->nodePort; return nodePortCompare; } /* * PlacementsHashHashCode computes the hash code for a shard placement from the * placement's shard id, node name, and node port number. */ static uint32 PlacementsHashHashCode(const void *key, Size keySize) { const ShardPlacement *placement = (const ShardPlacement *) key; const uint64 *shardId = &(placement->shardId); const char *nodeName = placement->nodeName; const uint32 *nodePort = &(placement->nodePort); /* standard hash function outlined in Effective Java, Item 8 */ uint32 result = 17; result = 37 * result + tag_hash(shardId, sizeof(uint64)); result = 37 * result + string_hash(nodeName, WORKER_LENGTH); result = 37 * result + tag_hash(nodePort, sizeof(uint32)); return result; } /* WorkerNodeListContains checks if the worker node exists in the given list. */ static bool WorkerNodeListContains(List *workerNodeList, const char *workerName, uint32 workerPort) { bool workerNodeListContains = false; ListCell *workerNodeCell = NULL; foreach(workerNodeCell, workerNodeList) { WorkerNode *workerNode = (WorkerNode *) lfirst(workerNodeCell); if ((strncmp(workerNode->workerName, workerName, WORKER_LENGTH) == 0) && (workerNode->workerPort == workerPort)) { workerNodeListContains = true; break; } } return workerNodeListContains; } /* * UpdateColocatedShardPlacementProgress updates the progress of the given placement, * along with its colocated placements, to the given state. */ static void UpdateColocatedShardPlacementProgress(uint64 shardId, char *sourceName, int sourcePort, uint64 progress) { ProgressMonitorData *header = GetCurrentProgressMonitor(); if (header != NULL) { PlacementUpdateEventProgress *steps = ProgressMonitorSteps(header); ListCell *colocatedShardIntervalCell = NULL; ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardIntervalList = ColocatedShardIntervalList(shardInterval); for (int moveIndex = 0; moveIndex < header->stepCount; moveIndex++) { PlacementUpdateEventProgress *step = steps + moveIndex; uint64 currentShardId = step->shardId; bool colocatedShard = false; foreach(colocatedShardIntervalCell, colocatedShardIntervalList) { ShardInterval *candidateShard = lfirst(colocatedShardIntervalCell); if (candidateShard->shardId == currentShardId) { colocatedShard = true; break; } } if (colocatedShard && strcmp(step->sourceName, sourceName) == 0 && step->sourcePort == sourcePort) { pg_atomic_write_u64(&step->progress, progress); } } } } /* * pg_dist_rebalance_strategy_enterprise_check is a now removed function, but * to avoid issues during upgrades a C stub is kept. */ Datum pg_dist_rebalance_strategy_enterprise_check(PG_FUNCTION_ARGS) { PG_RETURN_VOID(); } /* * citus_validate_rebalance_strategy_functions checks all the functions for * their correct signature. * * SQL signature: * * citus_validate_rebalance_strategy_functions( * shard_cost_function regproc, * node_capacity_function regproc, * shard_allowed_on_node_function regproc, * ) RETURNS VOID */ Datum citus_validate_rebalance_strategy_functions(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureShardCostUDF(PG_GETARG_OID(0)); EnsureNodeCapacityUDF(PG_GETARG_OID(1)); EnsureShardAllowedOnNodeUDF(PG_GETARG_OID(2)); PG_RETURN_VOID(); } /* * EnsureShardCostUDF checks that the UDF matching the oid has the correct * signature to be used as a ShardCost function. The expected signature is: * * shard_cost(shardid bigint) returns float4 */ static void EnsureShardCostUDF(Oid functionOid) { HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { ereport(ERROR, (errmsg("cache lookup failed for shard_cost_function with oid %u", functionOid))); } Form_pg_proc procForm = (Form_pg_proc) GETSTRUCT(proctup); char *name = NameStr(procForm->proname); if (procForm->pronargs != 1) { ereport(ERROR, (errmsg("signature for shard_cost_function is incorrect"), errdetail( "number of arguments of %s should be 1, not %i", name, procForm->pronargs))); } if (procForm->proargtypes.values[0] != INT8OID) { ereport(ERROR, (errmsg("signature for shard_cost_function is incorrect"), errdetail( "argument type of %s should be bigint", name))); } if (procForm->prorettype != FLOAT4OID) { ereport(ERROR, (errmsg("signature for shard_cost_function is incorrect"), errdetail("return type of %s should be real", name))); } ReleaseSysCache(proctup); } /* * SplitShardsBetweenPrimaryAndClone splits the shards in shardPlacementList * between the primary and clone nodes, adding them to the respective lists. */ void SplitShardsBetweenPrimaryAndClone(WorkerNode *primaryNode, WorkerNode *cloneNode, Name strategyName) { CheckCitusVersion(ERROR); List *relationIdList = NonColocatedDistRelationIdList(); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy(strategyName);/* We use default strategy for now */ RebalanceOptions options = { .relationIdList = relationIdList, .threshold = 0, /* Threshold is not strictly needed for two nodes */ .maxShardMoves = -1, /* No limit on moves between these two nodes */ .excludedShardArray = construct_empty_array(INT8OID), .drainOnly = false, /* Not a drain operation */ .rebalanceStrategy = strategy, .improvementThreshold = 0, /* Consider all beneficial moves */ .workerNode = primaryNode /* indicate Primary node as a source node */ }; SplitPrimaryCloneShards *splitShards = GetPrimaryCloneSplitRebalanceSteps(&options , cloneNode); AdjustShardsForPrimaryCloneNodeSplit(primaryNode, cloneNode, splitShards->primaryShardIdList, splitShards-> cloneShardIdList); } /* * GetPrimaryCloneSplitRebalanceSteps returns a List of PlacementUpdateEvents that are needed to * rebalance a list of tables. */ static SplitPrimaryCloneShards * GetPrimaryCloneSplitRebalanceSteps(RebalanceOptions *options, WorkerNode *cloneNode) { WorkerNode *sourceNode = options->workerNode; WorkerNode *targetNode = cloneNode; /* Initialize rebalance plan functions and context */ EnsureShardCostUDF(options->rebalanceStrategy->shardCostFunction); EnsureNodeCapacityUDF(options->rebalanceStrategy->nodeCapacityFunction); EnsureShardAllowedOnNodeUDF(options->rebalanceStrategy->shardAllowedOnNodeFunction); RebalanceContext context; memset(&context, 0, sizeof(RebalanceContext)); fmgr_info(options->rebalanceStrategy->shardCostFunction, &context.shardCostUDF); fmgr_info(options->rebalanceStrategy->nodeCapacityFunction, &context.nodeCapacityUDF); fmgr_info(options->rebalanceStrategy->shardAllowedOnNodeFunction, &context.shardAllowedOnNodeUDF); RebalancePlanFunctions rebalancePlanFunctions = { .shardAllowedOnNode = ShardAllowedOnNode, .nodeCapacity = NodeCapacity, .shardCost = GetShardCost, .context = &context, }; /* * Collect all active shard placements on the source node for the given relations. * Unlike the main rebalancer, we build a single list of all relevant source placements * across all specified relations (or all relations if none specified). */ List *allSourcePlacements = NIL; Oid relationIdItr = InvalidOid; foreach_declared_oid(relationIdItr, options->relationIdList) { List *shardPlacementList = FullShardPlacementList(relationIdItr, options->excludedShardArray); List *activeShardPlacementsForRelation = FilterShardPlacementList(shardPlacementList, IsActiveShardPlacement); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, activeShardPlacementsForRelation) { if (placement->nodeId == sourceNode->nodeId) { /* Ensure we don't add duplicate shardId if it's somehow listed under multiple relations */ bool alreadyAdded = false; ShardPlacement *existingPlacement = NULL; foreach_declared_ptr(existingPlacement, allSourcePlacements) { if (existingPlacement->shardId == placement->shardId) { alreadyAdded = true; break; } } if (!alreadyAdded) { allSourcePlacements = lappend(allSourcePlacements, placement); } } } } List *activeWorkerList = list_make2(options->workerNode, cloneNode); SplitPrimaryCloneShards *splitShards = palloc0(sizeof(SplitPrimaryCloneShards)); splitShards->primaryShardIdList = NIL; splitShards->cloneShardIdList = NIL; if (list_length(allSourcePlacements) > 0) { /* * Initialize RebalanceState considering only the source node's shards * and the two active workers (source and target). */ RebalanceState *state = InitRebalanceState(activeWorkerList, allSourcePlacements, &rebalancePlanFunctions); NodeFillState *sourceFillState = NULL; NodeFillState *targetFillState = NULL; ListCell *fsc = NULL; /* Identify the fill states for our specific source and target nodes */ foreach(fsc, state->fillStateListAsc) /* Could be fillStateListDesc too, order doesn't matter here */ { NodeFillState *fs = (NodeFillState *) lfirst(fsc); if (fs->node->nodeId == sourceNode->nodeId) { sourceFillState = fs; } else if (fs->node->nodeId == targetNode->nodeId) { targetFillState = fs; } } if (sourceFillState != NULL && targetFillState != NULL) { /* * The goal is to move roughly half the total cost from source to target. * The target node is assumed to be empty or its existing load is not * considered for this specific two-node balancing plan's shard distribution. * We calculate costs based *only* on the shards currently on the source node. */ /* * The core idea is to simulate the balancing process between these two nodes. * We have all shards on sourceFillState. TargetFillState is initially empty (in terms of these specific shards). * We want to move shards from source to target until their costs are as balanced as possible. */ float4 sourceCurrentCost = sourceFillState->totalCost; float4 targetCurrentCost = 0; /* Representing cost on target from these source shards */ /* Sort shards on source node by cost (descending). This is a common heuristic. */ sourceFillState->shardCostListDesc = SortList(sourceFillState-> shardCostListDesc, CompareShardCostDesc); List *potentialMoves = NIL; ListCell *lc_shardcost = NULL; /* * Iterate through each shard on the source node. For each shard, decide if moving it * to the target node would improve the balance (or is necessary to reach balance). * A simple greedy approach: move shard if target node's current cost is less than source's. */ foreach(lc_shardcost, sourceFillState->shardCostListDesc) { ShardCost *shardToConsider = (ShardCost *) lfirst(lc_shardcost); /* * If moving this shard makes the target less loaded than the source would become, * or if target is simply less loaded currently, consider the move. * More accurately, we move if target's cost + shard's cost < source's cost - shard's cost (approximately) * or if target is significantly emptier. * The condition (targetCurrentCost < sourceCurrentCost - shardToConsider->cost) is a greedy choice. * A better check: would moving this shard reduce the difference in costs? * Current difference: abs(sourceCurrentCost - targetCurrentCost) * Difference after move: abs((sourceCurrentCost - shardToConsider->cost) - (targetCurrentCost + shardToConsider->cost)) * Move if new difference is smaller. */ float4 costOfShard = shardToConsider->cost; float4 diffBefore = fabsf(sourceCurrentCost - targetCurrentCost); float4 diffAfter = fabsf((sourceCurrentCost - costOfShard) - ( targetCurrentCost + costOfShard)); if (diffAfter < diffBefore) { PlacementUpdateEvent *update = palloc0(sizeof(PlacementUpdateEvent)); update->shardId = shardToConsider->shardId; update->sourceNode = sourceNode; update->targetNode = targetNode; update->updateType = PLACEMENT_UPDATE_MOVE; potentialMoves = lappend(potentialMoves, update); splitShards->cloneShardIdList = lappend_int(splitShards-> cloneShardIdList, shardToConsider->shardId ); /* Update simulated costs for the next iteration */ sourceCurrentCost -= costOfShard; targetCurrentCost += costOfShard; } else { splitShards->primaryShardIdList = lappend_int(splitShards-> primaryShardIdList, shardToConsider->shardId ); } } } /* RebalanceState is in memory context, will be cleaned up */ } return splitShards; } /* * Snapshot-based node split plan outputs the shard placement plan * for primary and replica based node split * * SQL signature: * get_snapshot_based_node_split_plan( * primary_node_name text, * primary_node_port integer, * replica_node_name text, * replica_node_port integer, * rebalance_strategy name DEFAULT NULL * */ Datum get_snapshot_based_node_split_plan(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *primaryNodeNameText = PG_GETARG_TEXT_P(0); int32 primaryNodePort = PG_GETARG_INT32(1); text *cloneNodeNameText = PG_GETARG_TEXT_P(2); int32 cloneNodePort = PG_GETARG_INT32(3); char *primaryNodeName = text_to_cstring(primaryNodeNameText); char *cloneNodeName = text_to_cstring(cloneNodeNameText); WorkerNode *primaryNode = FindWorkerNodeOrError(primaryNodeName, primaryNodePort); WorkerNode *cloneNode = FindWorkerNodeOrError(cloneNodeName, cloneNodePort); if (!cloneNode->nodeisclone || cloneNode->nodeprimarynodeid == 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Node %s:%d (ID %d) is not a valid clone or its primary node ID is not set.", cloneNode->workerName, cloneNode->workerPort, cloneNode->nodeId))); } if (primaryNode->nodeisclone) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Primary node %s:%d (ID %d) is itself a replica.", primaryNode->workerName, primaryNode->workerPort, primaryNode->nodeId))); } /* Ensure the primary node is related to the replica node */ if (primaryNode->nodeId != cloneNode->nodeprimarynodeid) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Clone node %s:%d (ID %d) is not a clone of the primary node %s:%d (ID %d).", cloneNode->workerName, cloneNode->workerPort, cloneNode->nodeId, primaryNode->workerName, primaryNode->workerPort, primaryNode->nodeId))); } List *relationIdList = NonColocatedDistRelationIdList(); Form_pg_dist_rebalance_strategy strategy = GetRebalanceStrategy( PG_GETARG_NAME_OR_NULL(4)); RebalanceOptions options = { .relationIdList = relationIdList, .threshold = 0, /* Threshold is not strictly needed for two nodes */ .maxShardMoves = -1, /* No limit on moves between these two nodes */ .excludedShardArray = construct_empty_array(INT8OID), .drainOnly = false, /* Not a drain operation */ .rebalanceStrategy = strategy, .improvementThreshold = 0, /* Consider all beneficial moves */ .workerNode = primaryNode /* indicate Primary node as a source node */ }; SplitPrimaryCloneShards *splitShards = GetPrimaryCloneSplitRebalanceSteps( &options, cloneNode); int shardId = 0; TupleDesc tupdesc; Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); Datum values[4]; bool nulls[4]; foreach_declared_int(shardId, splitShards->primaryShardIdList) { ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); ListCell *colocatedShardCell = NULL; foreach(colocatedShardCell, colocatedShardList) { ShardInterval *colocatedShard = lfirst(colocatedShardCell); int colocatedShardId = colocatedShard->shardId; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = ObjectIdGetDatum(RelationIdForShard(colocatedShardId)); values[1] = UInt64GetDatum(colocatedShardId); values[2] = UInt64GetDatum(ShardLength(colocatedShardId)); values[3] = PointerGetDatum(cstring_to_text("Primary Node")); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } } foreach_declared_int(shardId, splitShards->cloneShardIdList) { ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); ListCell *colocatedShardCell = NULL; foreach(colocatedShardCell, colocatedShardList) { ShardInterval *colocatedShard = lfirst(colocatedShardCell); int colocatedShardId = colocatedShard->shardId; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = ObjectIdGetDatum(RelationIdForShard(colocatedShardId)); values[1] = UInt64GetDatum(colocatedShardId); values[2] = UInt64GetDatum(ShardLength(colocatedShardId)); values[3] = PointerGetDatum(cstring_to_text("Clone Node")); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } } return (Datum) 0; } /* * EnsureNodeCapacityUDF checks that the UDF matching the oid has the correct * signature to be used as a NodeCapacity function. The expected signature is: * * node_capacity(nodeid int) returns float4 */ static void EnsureNodeCapacityUDF(Oid functionOid) { HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { ereport(ERROR, (errmsg( "cache lookup failed for node_capacity_function with oid %u", functionOid))); } Form_pg_proc procForm = (Form_pg_proc) GETSTRUCT(proctup); char *name = NameStr(procForm->proname); if (procForm->pronargs != 1) { ereport(ERROR, (errmsg("signature for node_capacity_function is incorrect"), errdetail( "number of arguments of %s should be 1, not %i", name, procForm->pronargs))); } if (procForm->proargtypes.values[0] != INT4OID) { ereport(ERROR, (errmsg("signature for node_capacity_function is incorrect"), errdetail("argument type of %s should be int", name))); } if (procForm->prorettype != FLOAT4OID) { ereport(ERROR, (errmsg("signature for node_capacity_function is incorrect"), errdetail("return type of %s should be real", name))); } ReleaseSysCache(proctup); } /* * EnsureShardAllowedOnNodeUDF checks that the UDF matching the oid has the correct * signature to be used as a NodeCapacity function. The expected signature is: * * shard_allowed_on_node(shardid bigint, nodeid int) returns boolean */ static void EnsureShardAllowedOnNodeUDF(Oid functionOid) { HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); if (!HeapTupleIsValid(proctup)) { ereport(ERROR, (errmsg( "cache lookup failed for shard_allowed_on_node_function with oid %u", functionOid))); } Form_pg_proc procForm = (Form_pg_proc) GETSTRUCT(proctup); char *name = NameStr(procForm->proname); if (procForm->pronargs != 2) { ereport(ERROR, (errmsg( "signature for shard_allowed_on_node_function is incorrect"), errdetail( "number of arguments of %s should be 2, not %i", name, procForm->pronargs))); } if (procForm->proargtypes.values[0] != INT8OID) { ereport(ERROR, (errmsg( "signature for shard_allowed_on_node_function is incorrect"), errdetail( "type of first argument of %s should be bigint", name))); } if (procForm->proargtypes.values[1] != INT4OID) { ereport(ERROR, (errmsg( "signature for shard_allowed_on_node_function is incorrect"), errdetail( "type of second argument of %s should be int", name))); } if (procForm->prorettype != BOOLOID) { ereport(ERROR, (errmsg( "signature for shard_allowed_on_node_function is incorrect"), errdetail( "return type of %s should be boolean", name))); } ReleaseSysCache(proctup); } ================================================ FILE: src/backend/distributed/operations/shard_split.c ================================================ /*------------------------------------------------------------------------- * * shard_split.c * * Function definitions for the shard split. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "postmaster/postmaster.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/adaptive_executor.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/hash_helpers.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_shard.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_split.h" #include "distributed/shard_transfer.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_logical_replication.h" #include "distributed/shared_library_init.h" #include "distributed/utils/array_type.h" #include "distributed/utils/distribution_column_map.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" /* * Entry for map that tracks ShardInterval -> Placement Node * created by split workflow. */ typedef struct ShardCreatedByWorkflowEntry { ShardInterval *shardIntervalKey; WorkerNode *workerNodeValue; } ShardCreatedByWorkflowEntry; /* * Entry for map that trackes dummy shards. * Key: node + owner * Value: List of dummy shards for that node + owner */ typedef struct GroupedDummyShards { NodeAndOwner key; List *shardIntervals; } GroupedDummyShards; /* Function declarations */ static void ErrorIfCannotSplitShard(SplitOperation splitOperation, ShardInterval *sourceShard); static void ErrorIfCannotSplitShardExtended(SplitOperation splitOperation, ShardInterval *shardIntervalToSplit, List *shardSplitPointsList, List *nodeIdsForPlacementList); static bool CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *workerNode); static void ErrorIfModificationAndSplitInTheSameTransaction(SplitOperation splitOperation); static void CreateSplitShardsForShardGroup(List *shardGroupSplitIntervalListList, List *workersForPlacementList); static void CreateDummyShardsForShardGroup(HTAB *mapOfPlacementToDummyShardList, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, WorkerNode *sourceWorkerNode, List *workersForPlacementList); static HTAB * CreateWorkerForPlacementSet(List *workersForPlacementList); static void CreateAuxiliaryStructuresForShardGroup(List *shardGroupSplitIntervalListList, List *workersForPlacementList, bool includeReplicaIdentity); static void CreateReplicaIdentitiesForDummyShards(HTAB *mapOfPlacementToDummyShardList); static void CreateObjectOnPlacement(List *objectCreationCommandList, WorkerNode *workerNode); static List * CreateSplitIntervalsForShardGroup(List *sourceColocatedShardList, List *splitPointsForShard); static void CreateSplitIntervalsForShard(ShardInterval *sourceShard, List *splitPointsForShard, List **shardSplitChildrenIntervalList); static void BlockingShardSplit(SplitOperation splitOperation, uint64 splitWorkflowId, List *sourceColocatedShardIntervalList, List *shardSplitPointsList, List *workersForPlacementList, DistributionColumnMap *distributionColumnOverrides); static void NonBlockingShardSplit(SplitOperation splitOperation, uint64 splitWorkflowId, List *sourceColocatedShardIntervalList, List *shardSplitPointsList, List *workersForPlacementList, DistributionColumnMap *distributionColumnOverrides, uint32 targetColocationId); static void DoSplitCopy(WorkerNode *sourceShardNode, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *workersForPlacementList, char *snapShotName, DistributionColumnMap *distributionColumnOverrides); static StringInfo CreateSplitCopyCommand(ShardInterval *sourceShardSplitInterval, char *distributionColumnName, List *splitChildrenShardIntervalList, List *workersForPlacementList); static Task * CreateSplitCopyTask(StringInfo splitCopyUdfCommand, char *snapshotName, int taskId, uint64 jobId); static void UpdateDistributionColumnsForShardGroup(List *colocatedShardList, DistributionColumnMap *distCols, char distributionMethod, int shardCount, uint32 colocationId); static void InsertSplitChildrenShardMetadata(List *shardGroupSplitIntervalListList, List *workersForPlacementList); static void CreatePartitioningHierarchyForBlockingSplit(List * shardGroupSplitIntervalListList, List *workersForPlacementList); static void CreateForeignKeyConstraints(List *shardGroupSplitIntervalListList, List *workersForPlacementList); static Task * CreateTaskForDDLCommandList(List *ddlCommandList, WorkerNode *workerNode); static StringInfo CreateSplitShardReplicationSetupUDF(List * sourceColocatedShardIntervalList, List * shardGroupSplitIntervalListList, List *destinationWorkerNodesList, DistributionColumnMap * distributionColumnOverrides); static List * ParseReplicationSlotInfoFromResult(PGresult *result); static List * ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *destinationWorkerNodesList, DistributionColumnMap * distributionColumnOverrides); static void ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection); static void AddDummyShardEntryInMap(HTAB *mapOfPlacementToDummyShardList, uint32 targetNodeId, ShardInterval *shardInterval); static uint64 GetNextShardIdForSplitChild(void); static void AcquireNonblockingSplitLock(Oid relationId); static List * GetWorkerNodesFromWorkerIds(List *nodeIdsForPlacementList); static void DropShardListMetadata(List *shardIntervalList); /* Customize error message strings based on operation type */ static const char *const SplitOperationName[] = { [SHARD_SPLIT_API] = "split", [ISOLATE_TENANT_TO_NEW_SHARD] = "isolate", [CREATE_DISTRIBUTED_TABLE] = "create" }; static const char *const SplitOperationAPIName[] = { [SHARD_SPLIT_API] = "citus_split_shard_by_split_points", [ISOLATE_TENANT_TO_NEW_SHARD] = "isolate_tenant_to_new_shard", [CREATE_DISTRIBUTED_TABLE] = "create_distributed_table_concurrently" }; static const char *const SplitTargetName[] = { [SHARD_SPLIT_API] = "shard", [ISOLATE_TENANT_TO_NEW_SHARD] = "tenant", [CREATE_DISTRIBUTED_TABLE] = "distributed table" }; /* Function definitions */ /* * ErrorIfCannotSplitShard checks relation kind and invalid shards. It errors * out if we are not able to split the given shard. */ static void ErrorIfCannotSplitShard(SplitOperation splitOperation, ShardInterval *sourceShard) { Oid relationId = sourceShard->relationId; ListCell *colocatedTableCell = NULL; /* checks for table ownership and foreign tables */ List *colocatedTableList = ColocatedTableList(relationId); foreach(colocatedTableCell, colocatedTableList) { Oid colocatedTableId = lfirst_oid(colocatedTableCell); /* check that user has owner rights in all co-located tables */ EnsureTableOwner(colocatedTableId); char relationKind = get_rel_relkind(colocatedTableId); if (relationKind == RELKIND_FOREIGN_TABLE) { char *relationName = get_rel_name(colocatedTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot %s %s because \"%s\" is a " "foreign table", SplitOperationName[splitOperation], SplitTargetName[splitOperation], relationName), errdetail("Splitting shards backed by foreign tables " "is not supported."))); } } } /* * Extended checks before we decide to split the shard. * When all consumers (Example : ISOLATE_TENANT_TO_NEW_SHARD) directly call 'SplitShard' API, * this method will be merged with 'ErrorIfCannotSplitShard' above. */ static void ErrorIfCannotSplitShardExtended(SplitOperation splitOperation, ShardInterval *shardIntervalToSplit, List *shardSplitPointsList, List *nodeIdsForPlacementList) { /* we should not perform checks for create distributed table operation */ if (splitOperation == CREATE_DISTRIBUTED_TABLE) { return; } CitusTableCacheEntry *cachedTableEntry = GetCitusTableCacheEntry( shardIntervalToSplit->relationId); /* Perform checks common to both blocking and non-blocking Split API here. */ if (!IsCitusTableTypeCacheEntry(cachedTableEntry, HASH_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Cannot %s %s as operation " "is only supported for hash distributed tables.", SplitOperationName[splitOperation], SplitTargetName[splitOperation]))); } uint32 relationReplicationFactor = TableShardReplicationFactor( shardIntervalToSplit->relationId); if (relationReplicationFactor > 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "Operation %s not supported for %s as replication factor '%u' " "is greater than 1.", SplitOperationName[splitOperation], SplitTargetName[splitOperation], relationReplicationFactor))); } int splitPointsCount = list_length(shardSplitPointsList); int nodeIdsCount = list_length(nodeIdsForPlacementList); int shardsCount = splitPointsCount + 1; if (nodeIdsCount != shardsCount) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( "Number of worker node ids should be one greater split points. " "NodeId count is '%d' and SplitPoint count is '%d'.", nodeIdsCount, splitPointsCount))); } if (shardsCount > MAX_SHARD_COUNT) { ereport(ERROR, (errmsg( "Resulting shard count '%d' with split is greater than max shard count '%d' limit.", shardsCount, MAX_SHARD_COUNT))); } Assert(shardIntervalToSplit->minValueExists); Assert(shardIntervalToSplit->maxValueExists); /* We already verified table is Hash Distributed. We know (minValue, maxValue) are integers. */ int32 minValue = DatumGetInt32(shardIntervalToSplit->minValue); int32 maxValue = DatumGetInt32(shardIntervalToSplit->maxValue); /* Fail if Shard Interval cannot be split anymore i.e (min, max) range overlap. */ if (minValue == maxValue) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( "Cannot split shard id \"%lu\" as min/max range are equal: ('%d', '%d').", shardIntervalToSplit->shardId, minValue, maxValue))); } NullableDatum lastShardSplitPoint = { 0, true /*isnull*/ }; Datum shardSplitPoint; foreach_declared_int(shardSplitPoint, shardSplitPointsList) { int32 shardSplitPointValue = DatumGetInt32(shardSplitPoint); /* * 1) All Split points should lie within the shard interval range. * 2) Given our split points inclusive, you cannot specify the max value in a range as a split point. * Example: Shard 81060002 range is from (0,1073741823). '1073741823' as split point is invalid. * '1073741822' is correct and will split shard to: (0, 1073741822) and (1073741823, 1073741823). */ if (shardSplitPointValue < minValue || shardSplitPointValue > maxValue) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( "Split point %d is outside the min/max range(%d, %d) for shard id %lu.", shardSplitPointValue, DatumGetInt32(shardIntervalToSplit->minValue), DatumGetInt32(shardIntervalToSplit->maxValue), shardIntervalToSplit->shardId))); } else if (maxValue == shardSplitPointValue) { int32 validSplitPoint = shardIntervalToSplit->maxValue - 1; ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( "Invalid split point %d, as split points should be inclusive. Please use %d instead.", maxValue, validSplitPoint))); } /* Split points should be in strictly increasing order */ int32 lastShardSplitPointValue = DatumGetInt32(lastShardSplitPoint.value); if (!lastShardSplitPoint.isnull && shardSplitPointValue <= lastShardSplitPointValue) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( "Invalid Split Points '%d' followed by '%d'. " "All split points should be strictly increasing.", lastShardSplitPointValue, shardSplitPointValue))); } lastShardSplitPoint = (NullableDatum) { shardSplitPoint, false }; } } /* * ErrorIfModificationAndSplitInTheSameTransaction will error if we detect split operation * in the same transaction which has modification before. */ static void ErrorIfModificationAndSplitInTheSameTransaction(SplitOperation splitOperation) { if (XactModificationLevel > XACT_MODIFICATION_NONE) { ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot %s %s after other modifications " "in the same transaction.", SplitOperationName[splitOperation], SplitTargetName[splitOperation]))); } } /* * ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction will error if we detect multiple * nonblocking shard movements/splits in the same transaction. */ void ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction(void) { if (PlacementMovedUsingLogicalReplicationInTX) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("multiple shard movements/splits via logical " "replication in the same transaction is currently " "not supported"))); } } /* * GetWorkerNodesFromWorkerIds returns list of worker nodes given a list * of worker ids. It will error if any node id is invalid. */ static List * GetWorkerNodesFromWorkerIds(List *nodeIdsForPlacementList) { List *workersForPlacementList = NIL; int32 nodeId; foreach_declared_int(nodeId, nodeIdsForPlacementList) { uint32 nodeIdValue = (uint32) nodeId; WorkerNode *workerNode = LookupNodeByNodeId(nodeIdValue); /* NodeId in Citus are unsigned and range from [1, 4294967296]. */ if (nodeIdValue < 1 || workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("Invalid Node Id '%u'.", nodeIdValue))); } workersForPlacementList = lappend(workersForPlacementList, (void *) workerNode); } return workersForPlacementList; } /* * SplitShard API to split a given shard (or shard group) based on specified split points * to a set of destination nodes. * 'splitMode' : Mode of split operation. * 'splitOperation' : Customer operation that triggered split. * 'shardInterval' : Source shard interval to be split. * 'shardSplitPointsList' : Split Points list for the source 'shardInterval'. * 'nodeIdsForPlacementList' : Placement list corresponding to split children. * 'distributionColumnOverrides': Maps relation IDs to distribution columns. * If not specified, the distribution column is read * from the metadata. * 'colocatedShardIntervalList' : Shard interval list for colocation group. (only used for * create_distributed_table_concurrently). * 'targetColocationId' : Specifies the colocation ID (only used for * create_distributed_table_concurrently). */ void SplitShard(SplitMode splitMode, SplitOperation splitOperation, uint64 shardIdToSplit, List *shardSplitPointsList, List *nodeIdsForPlacementList, DistributionColumnMap *distributionColumnOverrides, List *colocatedShardIntervalList, uint32 targetColocationId) { const char *operationName = SplitOperationAPIName[splitOperation]; ErrorIfModificationAndSplitInTheSameTransaction(splitOperation); ShardInterval *shardIntervalToSplit = LoadShardInterval(shardIdToSplit); List *colocatedTableList = ColocatedTableList(shardIntervalToSplit->relationId); if (splitMode == AUTO_SPLIT) { VerifyTablesHaveReplicaIdentity(colocatedTableList); } /* Acquire global lock to prevent concurrent split on the same colocation group or relation */ Oid relationId = RelationIdForShard(shardIdToSplit); AcquirePlacementColocationLock(relationId, ExclusiveLock, "split"); /* sort the tables to avoid deadlocks */ colocatedTableList = SortList(colocatedTableList, CompareOids); Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { /* * Block concurrent DDL / TRUNCATE commands on the relation. Similarly, * block concurrent citus_move_shard_placement() / isolate_tenant_to_new_shard() * on any shard of the same relation. */ LockRelationOid(colocatedTableId, ShareUpdateExclusiveLock); } ErrorIfCannotSplitShard(splitOperation, shardIntervalToSplit); ErrorIfCannotSplitShardExtended( splitOperation, shardIntervalToSplit, shardSplitPointsList, nodeIdsForPlacementList); List *workersForPlacementList = GetWorkerNodesFromWorkerIds(nodeIdsForPlacementList); ErrorIfNotAllNodesHaveReferenceTableReplicas(workersForPlacementList); List *sourceColocatedShardIntervalList = NIL; if (colocatedShardIntervalList == NIL) { sourceColocatedShardIntervalList = ColocatedShardIntervalList( shardIntervalToSplit); } else { sourceColocatedShardIntervalList = colocatedShardIntervalList; } DropOrphanedResourcesInSeparateTransaction(); /* use the user-specified shard ID as the split workflow ID */ uint64 splitWorkflowId = shardIntervalToSplit->shardId; /* Start operation to prepare for generating cleanup records */ RegisterOperationNeedingCleanup(); if (splitMode == BLOCKING_SPLIT) { ereport(LOG, (errmsg("performing blocking %s ", operationName))); BlockingShardSplit( splitOperation, splitWorkflowId, sourceColocatedShardIntervalList, shardSplitPointsList, workersForPlacementList, distributionColumnOverrides); } else { ereport(LOG, (errmsg("performing non-blocking %s ", operationName))); NonBlockingShardSplit( splitOperation, splitWorkflowId, sourceColocatedShardIntervalList, shardSplitPointsList, workersForPlacementList, distributionColumnOverrides, targetColocationId); PlacementMovedUsingLogicalReplicationInTX = true; } /* * Drop temporary objects that were marked as CLEANUP_ALWAYS. */ FinalizeOperationNeedingCleanupOnSuccess(operationName); } /* * SplitShard API to split a given shard (or shard group) in blocking fashion * based on specified split points to a set of destination nodes. * splitOperation : Customer operation that triggered split. * splitWorkflowId : Number used to identify split workflow in names. * sourceColocatedShardIntervalList : Source shard group to be split. * shardSplitPointsList : Split Points list for the source 'shardInterval'. * workersForPlacementList : Placement list corresponding to split children. */ static void BlockingShardSplit(SplitOperation splitOperation, uint64 splitWorkflowId, List *sourceColocatedShardIntervalList, List *shardSplitPointsList, List *workersForPlacementList, DistributionColumnMap *distributionColumnOverrides) { const char *operationName = SplitOperationAPIName[splitOperation]; BlockWritesToShardList(sourceColocatedShardIntervalList); /* First create shard interval metadata for split children */ List *shardGroupSplitIntervalListList = CreateSplitIntervalsForShardGroup( sourceColocatedShardIntervalList, shardSplitPointsList); /* Only single placement allowed (already validated RelationReplicationFactor = 1) */ ShardInterval *firstShard = linitial(sourceColocatedShardIntervalList); WorkerNode *sourceShardNode = ActiveShardPlacementWorkerNode(firstShard->shardId); ereport(LOG, (errmsg("creating child shards for %s", operationName))); /* Physically create split children. */ CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList, workersForPlacementList); ereport(LOG, (errmsg("performing copy for %s", operationName))); /* For Blocking split, copy isn't snapshotted */ char *snapshotName = NULL; ConflictWithIsolationTestingBeforeCopy(); DoSplitCopy(sourceShardNode, sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, workersForPlacementList, snapshotName, distributionColumnOverrides); ConflictWithIsolationTestingAfterCopy(); ereport(LOG, (errmsg( "creating auxillary structures (indexes, stats, replicaindentities, triggers) for %s", operationName))); /* Create auxiliary structures (indexes, stats, replicaindentities, triggers) */ CreateAuxiliaryStructuresForShardGroup(shardGroupSplitIntervalListList, workersForPlacementList, true /* includeReplicaIdentity*/); /* * Up to this point, we performed various subtransactions that may * require additional clean-up in case of failure. The remaining operations * going forward are part of the same distributed transaction. */ /* * Delete old shards metadata and mark the shards as to be deferred drop. * Have to do that before creating the new shard metadata, * because there's cross-checks preventing inconsistent metadata * (like overlapping shards). */ ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s", operationName))); InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList); DropShardListMetadata(sourceColocatedShardIntervalList); /* Insert new shard and placement metdata */ InsertSplitChildrenShardMetadata(shardGroupSplitIntervalListList, workersForPlacementList); /* create partitioning hierarchy, if any */ CreatePartitioningHierarchyForBlockingSplit( shardGroupSplitIntervalListList, workersForPlacementList); ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s", operationName))); /* * Create foreign keys if exists after the metadata changes happening in * InsertSplitChildrenShardMetadata() because the foreign * key creation depends on the new metadata. */ CreateForeignKeyConstraints(shardGroupSplitIntervalListList, workersForPlacementList); CitusInvalidateRelcacheByRelid(DistShardRelationId()); } /* Check if a relation with given name already exists on the worker node */ static bool CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *workerNode) { char *schemaName = get_namespace_name( get_rel_namespace(shardInterval->relationId)); char *shardName = get_rel_name(shardInterval->relationId); AppendShardIdToName(&shardName, shardInterval->shardId); StringInfo checkShardExistsQuery = makeStringInfo(); /* * We pass schemaName and shardName without quote_identifier, since * they are used as strings here. */ appendStringInfo(checkShardExistsQuery, "SELECT EXISTS (SELECT FROM pg_catalog.pg_tables WHERE schemaname = %s AND tablename = %s);", quote_literal_cstr(schemaName), quote_literal_cstr(shardName)); int connectionFlags = 0; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags, workerNode->workerName, workerNode->workerPort, CitusExtensionOwnerName(), get_database_name( MyDatabaseId)); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(connection, checkShardExistsQuery->data, &result); if (queryResult != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) != 1) { ReportResultError(connection, result, ERROR); } char *existsString = PQgetvalue(result, 0, 0); bool tableExists = strcmp(existsString, "t") == 0; PQclear(result); ForgetResults(connection); return tableExists; } /* Create ShardGroup split children on a list of corresponding workers. */ static void CreateSplitShardsForShardGroup(List *shardGroupSplitIntervalListList, List *workersForPlacementList) { /* * Iterate over all the shards in the shard group. */ List *shardIntervalList = NIL; foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; /* * Iterate on split shards DDL command list for a given shard * and create them on corresponding workerPlacementNode. */ forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { /* Populate list of commands necessary to create shard interval on destination */ List *splitShardCreationCommandList = GetPreLoadTableCreationCommands( shardInterval->relationId, false, /* includeSequenceDefaults */ false, /* includeIdentityDefaults */ NULL /* auto add columnar options for cstore tables */); splitShardCreationCommandList = WorkerApplyShardDDLCommandList( splitShardCreationCommandList, shardInterval->shardId); /* Log resource for cleanup in case of failure only. * Before we log a record, do a best effort check to see if a shard with same name exists. * This is because, it will cause shard creation to fail and we will end up cleaning the * old shard. We don't want that. */ bool relationExists = CheckIfRelationWithSameNameExists(shardInterval, workerPlacementNode); if (relationExists) { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("relation %s already exists on worker %s:%d", ConstructQualifiedShardName(shardInterval), workerPlacementNode->workerName, workerPlacementNode->workerPort))); } InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, ConstructQualifiedShardName( shardInterval), workerPlacementNode->groupId, CLEANUP_ON_FAILURE); /* Create new split child shard on the specified placement list */ CreateObjectOnPlacement(splitShardCreationCommandList, workerPlacementNode); } } } /* Create a DDL task with corresponding task list on given worker node */ static Task * CreateTaskForDDLCommandList(List *ddlCommandList, WorkerNode *workerNode) { Task *ddlTask = CitusMakeNode(Task); ddlTask->taskType = DDL_TASK; ddlTask->replicationModel = REPLICATION_MODEL_INVALID; SetTaskQueryStringList(ddlTask, ddlCommandList); ShardPlacement *taskPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(taskPlacement, workerNode); ddlTask->taskPlacementList = list_make1(taskPlacement); return ddlTask; } /* Create ShardGroup auxiliary structures (indexes, stats, replicaindentities, triggers) * on a list of corresponding workers. */ static void CreateAuxiliaryStructuresForShardGroup(List *shardGroupSplitIntervalListList, List *workersForPlacementList, bool includeReplicaIdentity) { List *shardIntervalList = NIL; List *ddlTaskExecList = NIL; /* * Iterate over all the shards in the shard group. */ foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; /* * Iterate on split shard interval list for given shard and create tasks * for every single split shard in a shard group. */ forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { List *ddlCommandList = GetPostLoadTableCreationCommands( shardInterval->relationId, true /* includeIndexes */, includeReplicaIdentity); ddlCommandList = WorkerApplyShardDDLCommandList( ddlCommandList, shardInterval->shardId); /* * A task is expected to be instantiated with a non-null 'ddlCommandList'. * The list can be empty, if no auxiliary structures are present. */ if (ddlCommandList != NULL) { Task *ddlTask = CreateTaskForDDLCommandList(ddlCommandList, workerPlacementNode); ddlTaskExecList = lappend(ddlTaskExecList, ddlTask); } } } ExecuteTaskListOutsideTransaction( ROW_MODIFY_NONE, ddlTaskExecList, MaxAdaptiveExecutorPoolSize, NULL /* jobIdList (ignored by API impl.) */); } /* * Perform Split Copy from source shard(s) to split children. * 'sourceShardNode' : Source shard worker node. * 'sourceColocatedShardIntervalList' : List of source shard intervals from shard group. * 'shardGroupSplitIntervalListList' : List of shard intervals for split children. * 'workersForPlacementList' : List of workers for split children placement. */ static void DoSplitCopy(WorkerNode *sourceShardNode, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *destinationWorkerNodesList, char *snapShotName, DistributionColumnMap *distributionColumnOverrides) { ShardInterval *sourceShardIntervalToCopy = NULL; List *splitShardIntervalList = NIL; int taskId = 0; List *splitCopyTaskList = NIL; forboth_ptr(sourceShardIntervalToCopy, sourceColocatedShardIntervalList, splitShardIntervalList, shardGroupSplitIntervalListList) { /* * Skip copying data for partitioned tables, because they contain no * data themselves. Their partitions do contain data, but those are * different colocated shards that will be copied seperately. */ if (PartitionedTable(sourceShardIntervalToCopy->relationId)) { continue; } Oid relationId = sourceShardIntervalToCopy->relationId; Var *distributionColumn = GetDistributionColumnWithOverrides(relationId, distributionColumnOverrides); Assert(distributionColumn != NULL); bool missingOK = false; char *distributionColumnName = get_attname(relationId, distributionColumn->varattno, missingOK); StringInfo splitCopyUdfCommand = CreateSplitCopyCommand( sourceShardIntervalToCopy, distributionColumnName, splitShardIntervalList, destinationWorkerNodesList); /* Create copy task. Snapshot name is required for nonblocking splits */ Task *splitCopyTask = CreateSplitCopyTask(splitCopyUdfCommand, snapShotName, taskId, sourceShardIntervalToCopy->shardId); ShardPlacement *taskPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(taskPlacement, sourceShardNode); splitCopyTask->taskPlacementList = list_make1(taskPlacement); splitCopyTaskList = lappend(splitCopyTaskList, splitCopyTask); taskId++; } ExecuteTaskListOutsideTransaction(ROW_MODIFY_NONE, splitCopyTaskList, MaxAdaptiveExecutorPoolSize, NULL /* jobIdList (ignored by API impl.) */); } /* * Create Copy command for a given shard source shard to be copied to corresponding split children. * 'sourceShardSplitInterval' : Source shard interval to be copied. * 'splitChildrenShardINnerIntervalList' : List of shard intervals for split children. * 'destinationWorkerNodesList' : List of workers for split children placement. * Here is an example of a 2 way split copy : * SELECT * from worker_split_copy( * 81060000, -- source shard id to split copy * ARRAY[ * -- split copy info for split children 1 * ROW(81060015, -- destination shard id * -2147483648, -- split range begin * 1073741823, --split range end * 10 -- worker node id)::pg_catalog.split_copy_info, * -- split copy info for split children 2 * ROW(81060016, --destination shard id * 1073741824, --split range begin * 2147483647, --split range end * 11 -- workef node id)::pg_catalog.split_copy_info * ] * ); */ static StringInfo CreateSplitCopyCommand(ShardInterval *sourceShardSplitInterval, char *distributionColumnName, List *splitChildrenShardIntervalList, List *destinationWorkerNodesList) { StringInfo splitCopyInfoArray = makeStringInfo(); appendStringInfo(splitCopyInfoArray, "ARRAY["); ShardInterval *splitChildShardInterval = NULL; bool addComma = false; WorkerNode *destinationWorkerNode = NULL; forboth_ptr(splitChildShardInterval, splitChildrenShardIntervalList, destinationWorkerNode, destinationWorkerNodesList) { if (addComma) { appendStringInfo(splitCopyInfoArray, ","); } StringInfo splitCopyInfoRow = makeStringInfo(); appendStringInfo(splitCopyInfoRow, "ROW(%lu, %d, %d, %u)::pg_catalog.split_copy_info", splitChildShardInterval->shardId, DatumGetInt32(splitChildShardInterval->minValue), DatumGetInt32(splitChildShardInterval->maxValue), destinationWorkerNode->nodeId); appendStringInfo(splitCopyInfoArray, "%s", splitCopyInfoRow->data); addComma = true; } appendStringInfo(splitCopyInfoArray, "]"); StringInfo splitCopyUdf = makeStringInfo(); appendStringInfo(splitCopyUdf, "SELECT pg_catalog.worker_split_copy(%lu, %s, %s);", sourceShardSplitInterval->shardId, quote_literal_cstr(distributionColumnName), splitCopyInfoArray->data); return splitCopyUdf; } /* * CreateSplitCopyTask creates a task for copying data. * In the case of Non-blocking split, snapshotted copy task is created with given 'snapshotName'. * 'snapshotName' is NULL for Blocking split. */ static Task * CreateSplitCopyTask(StringInfo splitCopyUdfCommand, char *snapshotName, int taskId, uint64 jobId) { List *ddlCommandList = NIL; StringInfo beginTransaction = makeStringInfo(); appendStringInfo(beginTransaction, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); ddlCommandList = lappend(ddlCommandList, beginTransaction->data); /* Set snapshot for non-blocking shard split. */ if (snapshotName != NULL) { StringInfo snapShotString = makeStringInfo(); appendStringInfo(snapShotString, "SET TRANSACTION SNAPSHOT %s;", quote_literal_cstr( snapshotName)); ddlCommandList = lappend(ddlCommandList, snapShotString->data); } ddlCommandList = lappend(ddlCommandList, splitCopyUdfCommand->data); StringInfo commitCommand = makeStringInfo(); appendStringInfo(commitCommand, "COMMIT;"); ddlCommandList = lappend(ddlCommandList, commitCommand->data); Task *splitCopyTask = CitusMakeNode(Task); splitCopyTask->jobId = jobId; splitCopyTask->taskId = taskId; splitCopyTask->taskType = READ_TASK; splitCopyTask->replicationModel = REPLICATION_MODEL_INVALID; SetTaskQueryStringList(splitCopyTask, ddlCommandList); return splitCopyTask; } /* * Create an object on a worker node. */ static void CreateObjectOnPlacement(List *objectCreationCommandList, WorkerNode *workerPlacementNode) { MultiConnection *connection = GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION, workerPlacementNode->workerName, workerPlacementNode->workerPort, NULL, NULL); SendCommandListToWorkerOutsideTransactionWithConnection(connection, objectCreationCommandList); } /* * Create split children intervals for a shardgroup given list of split points. * Example: * 'sourceColocatedShardIntervalList': Colocated shard S1[-2147483648, 2147483647] & S2[-2147483648, 2147483647] * 'splitPointsForShard': [0] (2 way split) * 'shardGroupSplitIntervalListList': * [ * [ S1_1(-2147483648, 0), S1_2(1, 2147483647) ], // Split Interval List for S1. * [ S2_1(-2147483648, 0), S2_2(1, 2147483647) ] // Split Interval List for S2. * ] */ static List * CreateSplitIntervalsForShardGroup(List *sourceColocatedShardIntervalList, List *splitPointsForShard) { List *shardGroupSplitIntervalListList = NIL; ShardInterval *shardToSplitInterval = NULL; foreach_declared_ptr(shardToSplitInterval, sourceColocatedShardIntervalList) { List *shardSplitIntervalList = NIL; CreateSplitIntervalsForShard(shardToSplitInterval, splitPointsForShard, &shardSplitIntervalList); shardGroupSplitIntervalListList = lappend(shardGroupSplitIntervalListList, shardSplitIntervalList); } return shardGroupSplitIntervalListList; } /* * Create split children intervals given a sourceshard and a list of split points. * Example: SourceShard is range [0, 100] and SplitPoints are (15, 30) will give us: * [(0, 15) (16, 30) (31, 100)] */ static void CreateSplitIntervalsForShard(ShardInterval *sourceShard, List *splitPointsForShard, List **shardSplitChildrenIntervalList) { /* For 'N' split points, we will have N+1 shard intervals created. */ int shardIntervalCount = list_length(splitPointsForShard) + 1; ListCell *splitPointCell = list_head(splitPointsForShard); int32 splitParentMaxValue = DatumGetInt32(sourceShard->maxValue); int32 currentSplitChildMinValue = DatumGetInt32(sourceShard->minValue); /* if we are splitting a Citus local table, assume whole shard range */ if (!sourceShard->maxValueExists) { splitParentMaxValue = PG_INT32_MAX; } if (!sourceShard->minValueExists) { currentSplitChildMinValue = PG_INT32_MIN; } for (int index = 0; index < shardIntervalCount; index++) { ShardInterval *splitChildShardInterval = CopyShardInterval(sourceShard); splitChildShardInterval->shardIndex = -1; splitChildShardInterval->shardId = GetNextShardIdForSplitChild(); splitChildShardInterval->minValueExists = true; splitChildShardInterval->minValue = currentSplitChildMinValue; splitChildShardInterval->maxValueExists = true; /* Length of splitPointsForShard is one less than 'shardIntervalCount' and we need to account */ /* for 'splitPointCell' being NULL for last iteration. */ if (splitPointCell) { splitChildShardInterval->maxValue = DatumGetInt32((Datum) lfirst( splitPointCell)); splitPointCell = lnext(splitPointsForShard, splitPointCell); } else { splitChildShardInterval->maxValue = splitParentMaxValue; } currentSplitChildMinValue = splitChildShardInterval->maxValue + 1; *shardSplitChildrenIntervalList = lappend(*shardSplitChildrenIntervalList, splitChildShardInterval); } } /* * UpdateDistributionColumnsForShardGroup globally updates the pg_dist_partition metadata * for each relation that has a shard in colocatedShardList. * * This is used primarily for Citus local -> distributed table conversion * in create_distributed_table_concurrently. * * It would be nicer to keep this separate from shard split, but we need to do the * update at exactly the right point in the shard split process, namely after * replication slot creation and before inserting shard metadata, which itself * needs to happen before foreign key creation (mainly because the foreign key * functions depend on metadata). */ static void UpdateDistributionColumnsForShardGroup(List *colocatedShardList, DistributionColumnMap *distributionColumnMap, char distributionMethod, int shardCount, uint32 colocationId) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, colocatedShardList) { Oid relationId = shardInterval->relationId; Var *distributionColumn = GetDistributionColumnFromMap(distributionColumnMap, relationId); /* we should have an entry for every relation ID in the colocation group */ Assert(distributionColumn != NULL); if (colocationId == INVALID_COLOCATION_ID) { /* * Getting here with an invalid co-location ID means that no * appropriate co-location group exists yet. */ colocationId = CreateColocationGroup(shardCount, ShardReplicationFactor, distributionColumn->vartype, distributionColumn->varcollid); } UpdateDistributionColumnGlobally(relationId, distributionMethod, distributionColumn, colocationId); } } /* * Insert new shard and placement metadata. * Sync the Metadata with all nodes if enabled. */ static void InsertSplitChildrenShardMetadata(List *shardGroupSplitIntervalListList, List *workersForPlacementList) { List *shardIntervalList = NIL; List *syncedShardList = NIL; /* * Iterate over all the shards in the shard group. */ foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { /* * Iterate on split shards list for a given shard and insert metadata. */ ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { InsertShardRow( shardInterval->relationId, shardInterval->shardId, shardInterval->storageType, IntegerToText(DatumGetInt32(shardInterval->minValue)), IntegerToText(DatumGetInt32(shardInterval->maxValue))); InsertShardPlacementRow( shardInterval->shardId, INVALID_PLACEMENT_ID, /* triggers generation of new id */ 0, /* shard length (zero for HashDistributed Table) */ workerPlacementNode->groupId); if (ShouldSyncTableMetadata(shardInterval->relationId)) { syncedShardList = lappend(syncedShardList, shardInterval); } } } /* send commands to synced nodes one by one */ List *splitOffShardMetadataCommandList = ShardListInsertCommand(syncedShardList); char *command = NULL; foreach_declared_ptr(command, splitOffShardMetadataCommandList) { SendCommandToWorkersWithMetadata(command); } } /* * CreatePartitioningHierarchy creates the partitioning * hierarchy between the shardList, if any. */ static void CreatePartitioningHierarchyForBlockingSplit(List *shardGroupSplitIntervalListList, List *workersForPlacementList) { /* Create partition heirarchy between shards */ List *shardIntervalList = NIL; /* * Iterate over all the shards in the shard group. */ foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; /* * Iterate on split shards list for a given shard and create constraints. */ forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { if (PartitionTable(shardInterval->relationId)) { char *attachPartitionCommand = GenerateAttachShardPartitionCommand(shardInterval); SendCommandToWorker( workerPlacementNode->workerName, workerPlacementNode->workerPort, attachPartitionCommand); } } } } /* * Create foreign key constraints on the split children shards. */ static void CreateForeignKeyConstraints(List *shardGroupSplitIntervalListList, List *workersForPlacementList) { /* Create constraints between shards */ List *shardIntervalList = NIL; /* * Iterate over all the shards in the shard group. */ foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; /* * Iterate on split shards list for a given shard and create constraints. */ forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { List *shardForeignConstraintCommandList = NIL; List *referenceTableForeignConstraintList = NIL; CopyShardForeignConstraintCommandListGrouped( shardInterval, &shardForeignConstraintCommandList, &referenceTableForeignConstraintList); List *constraintCommandList = NIL; constraintCommandList = list_concat(constraintCommandList, shardForeignConstraintCommandList); constraintCommandList = list_concat(constraintCommandList, referenceTableForeignConstraintList); char *constraintCommand = NULL; foreach_declared_ptr(constraintCommand, constraintCommandList) { SendCommandToWorker( workerPlacementNode->workerName, workerPlacementNode->workerPort, constraintCommand); } } } } /* * DropShardListMetadata drops shard metadata from both the coordinator and * mx nodes. */ static void DropShardListMetadata(List *shardIntervalList) { ListCell *shardIntervalCell = NULL; foreach(shardIntervalCell, shardIntervalList) { ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell); ListCell *shardPlacementCell = NULL; Oid relationId = shardInterval->relationId; uint64 oldShardId = shardInterval->shardId; /* delete metadata from synced nodes */ if (ShouldSyncTableMetadata(relationId)) { ListCell *commandCell = NULL; /* send the commands one by one (calls citus_internal.delete_shard_metadata internally) */ List *shardMetadataDeleteCommandList = ShardDeleteCommandList(shardInterval); foreach(commandCell, shardMetadataDeleteCommandList) { char *command = (char *) lfirst(commandCell); SendCommandToWorkersWithMetadata(command); } } /* delete shard placements */ List *shardPlacementList = ActiveShardPlacementList(oldShardId); foreach(shardPlacementCell, shardPlacementList) { ShardPlacement *placement = (ShardPlacement *) lfirst(shardPlacementCell); DeleteShardPlacementRow(placement->placementId); } /* delete shard row */ DeleteShardRow(oldShardId); } } /* * AcquireNonblockingSplitLock does not allow concurrent nonblocking splits, because we share memory and * replication slots. */ static void AcquireNonblockingSplitLock(Oid relationId) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = true; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_NONBLOCKING_SPLIT); LockAcquireResult lockAcquired = LockAcquire(&tag, ExclusiveLock, sessionLock, dontWait); if (!lockAcquired) { ereport(ERROR, (errmsg("could not acquire the lock required to split " "concurrently %s.", generate_qualified_relation_name( relationId)), errdetail("It means that either a concurrent shard move " "or distributed table creation is happening."), errhint("Make sure that the concurrent operation has " "finished and re-run the command"))); } } /* * SplitShard API to split a given shard (or shard group) in non-blocking fashion * based on specified split points to a set of destination nodes. * splitOperation : Customer operation that triggered split. * splitWorkflowId : Number used to identify split workflow in names. * sourceColocatedShardIntervalList : Source shard group to be split. * shardSplitPointsList : Split Points list for the source 'shardInterval'. * workersForPlacementList : Placement list corresponding to split children. * distributionColumnList : Maps relation IDs to distribution columns. * If not specified, the distribution column is read * from the metadata. * targetColocationId : Specifies the colocation ID (only used for * create_distributed_table_concurrently). */ void NonBlockingShardSplit(SplitOperation splitOperation, uint64 splitWorkflowId, List *sourceColocatedShardIntervalList, List *shardSplitPointsList, List *workersForPlacementList, DistributionColumnMap *distributionColumnOverrides, uint32 targetColocationId) { const char *operationName = SplitOperationAPIName[splitOperation]; ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction(); char *superUser = CitusExtensionOwnerName(); char *databaseName = get_database_name(MyDatabaseId); /* First create shard interval metadata for split children */ List *shardGroupSplitIntervalListList = CreateSplitIntervalsForShardGroup( sourceColocatedShardIntervalList, shardSplitPointsList); ShardInterval *firstShard = linitial(sourceColocatedShardIntervalList); /* Acquire global lock to prevent concurrent nonblocking splits */ AcquireNonblockingSplitLock(firstShard->relationId); WorkerNode *sourceShardToCopyNode = ActiveShardPlacementWorkerNode(firstShard->shardId); /* Create hashmap to group shards for publication-subscription management */ HTAB *publicationInfoHash = CreateShardSplitInfoMapForPublication( sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, workersForPlacementList); int connectionFlags = FORCE_NEW_CONNECTION; MultiConnection *sourceConnection = GetNodeUserDatabaseConnection( connectionFlags, sourceShardToCopyNode->workerName, sourceShardToCopyNode->workerPort, superUser, databaseName); ClaimConnectionExclusively(sourceConnection); MultiConnection *sourceReplicationConnection = GetReplicationConnection(sourceShardToCopyNode->workerName, sourceShardToCopyNode->workerPort); /* Non-Blocking shard split workflow starts here */ ereport(LOG, (errmsg("creating child shards for %s", operationName))); /* 1) Physically create split children. */ CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList, workersForPlacementList); /* * 2) Create dummy shards due to PG logical replication constraints. * Refer to the comment section of 'CreateDummyShardsForShardGroup' for indepth * information. */ HTAB *mapOfPlacementToDummyShardList = CreateSimpleHash(NodeAndOwner, GroupedShardSplitInfos); CreateDummyShardsForShardGroup( mapOfPlacementToDummyShardList, sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, sourceShardToCopyNode, workersForPlacementList); /* * 3) Create replica identities on dummy shards. This needs to be done * before the subscriptions are created. Otherwise the subscription * creation will get stuck waiting for the publication to send a * replica identity. Since we never actually write data into these * dummy shards there's no point in creating these indexes after the * initial COPY phase, like we do for the replica identities on the * target shards. */ CreateReplicaIdentitiesForDummyShards(mapOfPlacementToDummyShardList); ereport(LOG, (errmsg( "creating replication artifacts (publications, replication slots, subscriptions for %s", operationName))); /* 4) Create Publications. */ CreatePublications(sourceConnection, publicationInfoHash); /* 5) Execute 'worker_split_shard_replication_setup UDF */ List *replicationSlotInfoList = ExecuteSplitShardReplicationSetupUDF( sourceShardToCopyNode, sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, workersForPlacementList, distributionColumnOverrides); /* * Subscriber flow starts from here. * Populate 'ShardSplitSubscriberMetadata' for subscription management. */ List *logicalRepTargetList = PopulateShardSplitSubscriptionsMetadataList( publicationInfoHash, replicationSlotInfoList, shardGroupSplitIntervalListList, workersForPlacementList); HTAB *groupedLogicalRepTargetsHash = CreateGroupedLogicalRepTargetsHash( logicalRepTargetList); /* Create connections to the target nodes */ CreateGroupedLogicalRepTargetsConnections( groupedLogicalRepTargetsHash, superUser, databaseName); char *logicalRepDecoderPlugin = "citus"; /* * 6) Create replication slots and keep track of their snapshot. */ char *snapshot = CreateReplicationSlots( sourceConnection, sourceReplicationConnection, logicalRepTargetList, logicalRepDecoderPlugin); /* * 7) Create subscriptions. This isn't strictly needed yet at this * stage, but this way we error out quickly if it fails. */ CreateSubscriptions( sourceConnection, databaseName, logicalRepTargetList); /* * We have to create the primary key (or any other replica identity) * before the update/delete operations that are queued will be * replicated. Because if the replica identity does not exist on the * target, the replication would fail. * * So the latest possible moment we could do this is right after the * initial data COPY, but before enabling the susbcriptions. It might * seem like a good idea to it after the initial data COPY, since * it's generally the rule that it's cheaper to build an index at once * than to create it incrementally. This general rule, is why we create * all the regular indexes as late during the move as possible. * * But as it turns out in practice it's not as clear cut, and we saw a * speed degradation in the time it takes to move shards when doing the * replica identity creation after the initial COPY. So, instead we * keep it before the COPY. */ CreateReplicaIdentities(logicalRepTargetList); ereport(LOG, (errmsg("performing copy for %s", operationName))); /* 8) Do snapshotted Copy */ DoSplitCopy(sourceShardToCopyNode, sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, workersForPlacementList, snapshot, distributionColumnOverrides); ereport(LOG, (errmsg("replicating changes for %s", operationName))); /* * 9) Logically replicate all the changes and do most of the table DDL, * like index and foreign key creation. */ bool skipInterShardRelationshipCreation = false; CompleteNonBlockingShardTransfer(sourceColocatedShardIntervalList, sourceConnection, publicationInfoHash, logicalRepTargetList, groupedLogicalRepTargetsHash, SHARD_SPLIT, skipInterShardRelationshipCreation); /* * 10) Delete old shards metadata and mark the shards as to be deferred drop. * Have to do that before creating the new shard metadata, * because there's cross-checks preventing inconsistent metadata * (like overlapping shards). */ ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s", operationName))); InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList); DropShardListMetadata(sourceColocatedShardIntervalList); /* * 11) In case of create_distributed_table_concurrently, which converts * a Citus local table to a distributed table, update the distributed * table metadata now. * * We would rather have this be outside of the scope of NonBlockingShardSplit, * but we cannot make metadata changes before replication slot creation, and * we cannot create the replication slot before creating new shards and * corresponding publications, because the decoder uses a catalog snapshot * from the time of the slot creation, which means it would not be able to see * the shards or publications when replication starts if it was created before. * * We also cannot easily move metadata changes to be after this function, * because CreateForeignKeyConstraints relies on accurate metadata and * we also want to perform the clean-up logic in PG_CATCH in case of * failure. * * Hence, this appears to be the only suitable spot for updating * pg_dist_partition and pg_dist_colocation. */ if (splitOperation == CREATE_DISTRIBUTED_TABLE) { /* we currently only use split for hash-distributed tables */ char distributionMethod = DISTRIBUTE_BY_HASH; int shardCount = list_length(shardSplitPointsList) + 1; UpdateDistributionColumnsForShardGroup(sourceColocatedShardIntervalList, distributionColumnOverrides, distributionMethod, shardCount, targetColocationId); } /* 12) Insert new shard and placement metdata */ InsertSplitChildrenShardMetadata(shardGroupSplitIntervalListList, workersForPlacementList); /* 13) create partitioning hierarchy, if any, this needs to be done * after the metadata is correct, because it fails for some * uninvestigated reason otherwise. */ CreatePartitioningHierarchy(logicalRepTargetList); ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s", operationName))); /* * 14) Create foreign keys if exists after the metadata changes happening in * InsertSplitChildrenShardMetadata() because the foreign * key creation depends on the new metadata. */ CreateUncheckedForeignKeyConstraints(logicalRepTargetList); /* * 15) Release shared memory allocated by worker_split_shard_replication_setup udf * at source node. */ ExecuteSplitShardReleaseSharedMemory(sourceConnection); /* 16) Close source connection */ CloseConnection(sourceConnection); /* 17) Close all subscriber connections */ CloseGroupedLogicalRepTargetsConnections(groupedLogicalRepTargetsHash); /* 18) Close connection of template replication slot */ CloseConnection(sourceReplicationConnection); } /* * Given we are using PG logical replication infrastructure there are some constraints * that need to met around matching table names in source and target nodes: * The restrictions in context of split are: * Constraint 1: Dummy source shard(s) from shard group must exist on all destination nodes. * Constraint 2: Dummy target shards from shard group must exist on source node. * Example : * Shard1[1-200] is co-located with Shard2[1-200] in Worker0. * We are splitting 2-way to worker0 (same node) and worker1 (different node). * * Non-Dummy shards (expected from Split): * In Worker0 --> Shard1_1 and Shard2_1. * In Worker1 --> Shard1_2 and Shard2_2. * * Dummy shards: * From constraint 1, we need to create: Dummy Shard1 and Shard2 in Worker0. Dummy Shard1 and Shard2 in Worker1 * Note 1 : Given there is an overlap of source and destination in Worker0, Shard1 and Shard2 need not be created. * Be very careful here, dropping Shard1, Shard2 with customer data to create dummy Shard1, Shard2 on worker0 is catastrophic. * * From constraint 2, we need to create: Dummy Shard1_1, Shard2_1, Shard1_2 and Shard2_2 in Worker0. * Note 2 : Given there is an overlap of source and destination in Worker0, Shard1_1 and Shard2_1 need not be created. */ static void CreateDummyShardsForShardGroup(HTAB *mapOfPlacementToDummyShardList, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, WorkerNode *sourceWorkerNode, List *workersForPlacementList) { /* * Statisfy Constraint 1: Create dummy source shard(s) on all destination nodes. * If source node is also in desintation, skip dummy shard creation(see Note 1 from function description). * We are guarenteed to have a single active placement for source shard. This is enforced earlier by ErrorIfCannotSplitShardExtended. */ /* List 'workersForPlacementList' can have duplicates. We need all unique destination nodes. */ HTAB *workersForPlacementSet = CreateWorkerForPlacementSet(workersForPlacementList); HASH_SEQ_STATUS status; hash_seq_init(&status, workersForPlacementSet); WorkerNode *workerPlacementNode = NULL; while ((workerPlacementNode = (WorkerNode *) hash_seq_search(&status)) != NULL) { if (workerPlacementNode->nodeId == sourceWorkerNode->nodeId) { continue; } ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, sourceColocatedShardIntervalList) { /* Populate list of commands necessary to create shard interval on destination */ List *splitShardCreationCommandList = GetPreLoadTableCreationCommands( shardInterval->relationId, false, /* includeSequenceDefaults */ false, /* includeIdentityDefaults */ NULL /* auto add columnar options for cstore tables */); splitShardCreationCommandList = WorkerApplyShardDDLCommandList( splitShardCreationCommandList, shardInterval->shardId); /* Log resource for cleanup in case of failure only. * Before we log a record, do a best effort check to see if a shard with same name exists. * This is because, it will cause shard creation to fail and we will end up cleaning the * old shard. We don't want that. */ bool relationExists = CheckIfRelationWithSameNameExists(shardInterval, workerPlacementNode); if (relationExists) { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("relation %s already exists on worker %s:%d", ConstructQualifiedShardName(shardInterval), workerPlacementNode->workerName, workerPlacementNode->workerPort))); } /* Log shard in pg_dist_cleanup. Given dummy shards are transient resources, * we want to cleanup irrespective of operation success or failure. */ InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, ConstructQualifiedShardName( shardInterval), workerPlacementNode->groupId, CLEANUP_ALWAYS); /* Create dummy source shard on the specified placement list */ CreateObjectOnPlacement(splitShardCreationCommandList, workerPlacementNode); /* Add dummy source shard entry created for placement node in map */ AddDummyShardEntryInMap(mapOfPlacementToDummyShardList, workerPlacementNode->nodeId, shardInterval); } } /* * Statisfy Constraint 2: Create dummy target shards from shard group on source node. * If the target shard was created on source node as placement, skip it (See Note 2 from function description). */ List *shardIntervalList = NULL; foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; workerPlacementNode = NULL; forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { if (workerPlacementNode->nodeId == sourceWorkerNode->nodeId) { continue; } List *splitShardCreationCommandList = GetPreLoadTableCreationCommands( shardInterval->relationId, false, /* includeSequenceDefaults */ false, /* includeIdentityDefaults */ NULL /* auto add columnar options for cstore tables */); splitShardCreationCommandList = WorkerApplyShardDDLCommandList( splitShardCreationCommandList, shardInterval->shardId); /* Log resource for cleanup in case of failure only. * Before we log a record, do a best effort check to see if a shard with same name exists. * This is because, it will cause shard creation to fail and we will end up cleaning the * old shard. We don't want that. */ bool relationExists = CheckIfRelationWithSameNameExists(shardInterval, sourceWorkerNode); if (relationExists) { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("relation %s already exists on worker %s:%d", ConstructQualifiedShardName(shardInterval), sourceWorkerNode->workerName, sourceWorkerNode->workerPort))); } /* Log shard in pg_dist_cleanup. Given dummy shards are transient resources, * we want to cleanup irrespective of operation success or failure. */ InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, ConstructQualifiedShardName( shardInterval), sourceWorkerNode->groupId, CLEANUP_ALWAYS); /* Create dummy split child shard on source worker node */ CreateObjectOnPlacement(splitShardCreationCommandList, sourceWorkerNode); /* Add dummy split child shard entry created on source node */ AddDummyShardEntryInMap(mapOfPlacementToDummyShardList, sourceWorkerNode->nodeId, shardInterval); } } } /* * CreateWorkerForPlacementSet returns a set with unique worker nodes. */ static HTAB * CreateWorkerForPlacementSet(List *workersForPlacementList) { HASHCTL info = { 0 }; info.keysize = sizeof(WorkerNode); info.hash = WorkerNodeHashCode; info.match = WorkerNodeCompare; /* we don't have value field as it's a set */ info.entrysize = info.keysize; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE); HTAB *workerForPlacementSet = hash_create("worker placement set", 32, &info, hashFlags); WorkerNode *workerForPlacement = NULL; foreach_declared_ptr(workerForPlacement, workersForPlacementList) { void *hashKey = (void *) workerForPlacement; hash_search(workerForPlacementSet, hashKey, HASH_ENTER, NULL); } return workerForPlacementSet; } /* * ExecuteSplitShardReplicationSetupUDF executes * 'worker_split_shard_replication_setup' UDF on source shard node * and returns list of ReplicationSlotInfo. */ static List * ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode, List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *destinationWorkerNodesList, DistributionColumnMap *distributionColumnOverrides) { StringInfo splitShardReplicationUDF = CreateSplitShardReplicationSetupUDF( sourceColocatedShardIntervalList, shardGroupSplitIntervalListList, destinationWorkerNodesList, distributionColumnOverrides); /* Force a new connection to execute the UDF */ int connectionFlags = 0; MultiConnection *sourceConnection = GetNodeUserDatabaseConnection(connectionFlags, sourceWorkerNode-> workerName, sourceWorkerNode-> workerPort, CitusExtensionOwnerName(), get_database_name( MyDatabaseId)); ClaimConnectionExclusively(sourceConnection); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(sourceConnection, splitShardReplicationUDF->data, &result); /* * Result should contain atleast one tuple. The information returned is * set of tuples where each tuple is formatted as: * . */ if (queryResult != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) < 1 || PQnfields(result) != 3) { PQclear(result); ForgetResults(sourceConnection); CloseConnection(sourceConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "Failed to run worker_split_shard_replication_setup UDF. " "It should successfully execute for splitting a shard in " "a non-blocking way. Please retry."))); } /* Get replication slot information */ List *replicationSlotInfoList = ParseReplicationSlotInfoFromResult(result); PQclear(result); ForgetResults(sourceConnection); CloseConnection(sourceConnection); return replicationSlotInfoList; } /* * ExecuteSplitShardReleaseSharedMemory releases dynamic shared memory * at source node. * As a part of non-blocking split workflow, worker_split_shard_replication_setup allocates * shared memory to store split information. This has to be released after split completes(or fails). */ static void ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection) { StringInfo splitShardReleaseMemoryUDF = makeStringInfo(); appendStringInfo(splitShardReleaseMemoryUDF, "SELECT pg_catalog.worker_split_shard_release_dsm();"); ExecuteCriticalRemoteCommand(sourceConnection, splitShardReleaseMemoryUDF->data); } /* * CreateSplitShardReplicationSetupUDF creates and returns * parameterized 'worker_split_shard_replication_setup' UDF command. * * 'sourceShardSplitIntervalList' : Source shard interval to split. * 'shardGroupSplitIntervalListList' : List of shard intervals for split children.. * 'destinationWorkerNodesList' : List of workers for split children placement. * * For example consider below input values: * sourceColocatedShardIntervalList : [sourceShardInterval] * shardGroupSplitIntervalListList : [] * destinationWorkerNodesList : [worker1, worker2] * * SELECT * FROM worker_split_shard_replication_setup( * Array[ * ROW(sourceShardId, childFirstShardId, childFirstMinRange, childFirstMaxRange, worker1)::citus.split_shard_info, * ROW(sourceShardId, childSecondShardId, childSecondMinRange, childSecondMaxRange, worker2)::citus.split_shard_info * ], CurrentOperationId); */ StringInfo CreateSplitShardReplicationSetupUDF(List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *destinationWorkerNodesList, DistributionColumnMap *distributionColumnOverrides) { StringInfo splitChildrenRows = makeStringInfo(); ShardInterval *sourceShardIntervalToCopy = NULL; List *splitChildShardIntervalList = NULL; bool addComma = false; forboth_ptr(sourceShardIntervalToCopy, sourceColocatedShardIntervalList, splitChildShardIntervalList, shardGroupSplitIntervalListList) { int64 sourceShardId = sourceShardIntervalToCopy->shardId; Oid relationId = sourceShardIntervalToCopy->relationId; Var *distributionColumn = GetDistributionColumnWithOverrides(relationId, distributionColumnOverrides); bool missingOK = false; char *distributionColumnName = get_attname(relationId, distributionColumn->varattno, missingOK); ShardInterval *splitChildShardInterval = NULL; WorkerNode *destinationWorkerNode = NULL; forboth_ptr(splitChildShardInterval, splitChildShardIntervalList, destinationWorkerNode, destinationWorkerNodesList) { if (addComma) { appendStringInfo(splitChildrenRows, ","); } StringInfo minValueString = makeStringInfo(); appendStringInfo(minValueString, "%d", DatumGetInt32( splitChildShardInterval->minValue)); StringInfo maxValueString = makeStringInfo(); appendStringInfo(maxValueString, "%d", DatumGetInt32( splitChildShardInterval->maxValue)); appendStringInfo(splitChildrenRows, "ROW(%lu, %s, %lu, %s, %s, %u)::pg_catalog.split_shard_info", sourceShardId, quote_literal_cstr(distributionColumnName), splitChildShardInterval->shardId, quote_literal_cstr(minValueString->data), quote_literal_cstr(maxValueString->data), destinationWorkerNode->nodeId); addComma = true; } } StringInfo splitShardReplicationUDF = makeStringInfo(); appendStringInfo(splitShardReplicationUDF, "SELECT * FROM pg_catalog.worker_split_shard_replication_setup(" "ARRAY[%s], %lu);", splitChildrenRows->data, CurrentOperationId); return splitShardReplicationUDF; } /* * ParseReplicationSlotInfoFromResult parses custom datatype 'replication_slot_info'. * 'replication_slot_info' is a tuple with below format: * */ static List * ParseReplicationSlotInfoFromResult(PGresult *result) { int64 rowCount = PQntuples(result); List *replicationSlotInfoList = NIL; for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { ReplicationSlotInfo *replicationSlot = (ReplicationSlotInfo *) palloc0( sizeof(ReplicationSlotInfo)); char *targeNodeIdString = PQgetvalue(result, rowIndex, 0 /* nodeId column*/); replicationSlot->targetNodeId = strtoul(targeNodeIdString, NULL, 10); bool missingOk = false; replicationSlot->tableOwnerId = get_role_oid( PQgetvalue(result, rowIndex, 1 /* table owner name column */), missingOk); /* Replication slot name */ replicationSlot->name = pstrdup(PQgetvalue(result, rowIndex, 2 /* slot name column */)); replicationSlotInfoList = lappend(replicationSlotInfoList, replicationSlot); } return replicationSlotInfoList; } /* * AddDummyShardEntryInMap adds shard entry into hash map to keep track * of dummy shards that are created. These shards are cleanedup after split completes. * * This is a cautious measure to keep track of dummy shards created for constraints * of logical replication. We cautiously delete only the dummy shards added in the DummyShardHashMap. */ static void AddDummyShardEntryInMap(HTAB *mapOfPlacementToDummyShardList, uint32 targetNodeId, ShardInterval *shardInterval) { NodeAndOwner key; key.nodeId = targetNodeId; key.tableOwnerId = TableOwnerOid(shardInterval->relationId); bool found = false; GroupedDummyShards *nodeMappingEntry = (GroupedDummyShards *) hash_search(mapOfPlacementToDummyShardList, &key, HASH_ENTER, &found); if (!found) { nodeMappingEntry->shardIntervals = NIL; } nodeMappingEntry->shardIntervals = lappend(nodeMappingEntry->shardIntervals, shardInterval); } /* * CreateReplicaIdentitiesForDummyShards creates replica indentities for split * dummy shards. */ static void CreateReplicaIdentitiesForDummyShards(HTAB *mapOfDummyShardToPlacement) { /* Create Replica Identities for dummy shards */ HASH_SEQ_STATUS status; hash_seq_init(&status, mapOfDummyShardToPlacement); GroupedDummyShards *entry = NULL; while ((entry = (GroupedDummyShards *) hash_seq_search(&status)) != NULL) { uint32 nodeId = entry->key.nodeId; WorkerNode *shardToBeDroppedNode = FindNodeWithNodeId(nodeId, false /* missingOk */); List *dummyShardIntervalList = entry->shardIntervals; CreateReplicaIdentitiesOnNode(dummyShardIntervalList, shardToBeDroppedNode->workerName, shardToBeDroppedNode->workerPort); } } /* * GetNextShardIdForSplitChild returns shard id to be used for split child. * The function connects to the local node through a new connection and gets the next * sequence. This prevents self deadlock when 'CREATE_REPLICATION_SLOT' is executed * as a part of nonblocking split workflow. */ static uint64 GetNextShardIdForSplitChild() { uint64 shardId = 0; /* * In regression tests, we would like to generate shard IDs consistently * even if the tests run in parallel. Instead of the sequence, we can use * the next_shard_id GUC to specify which shard ID the current session should * generate next. The GUC is automatically increased by 1 every time a new * shard ID is generated. */ if (NextShardId > 0) { shardId = NextShardId; NextShardId += 1; return shardId; } StringInfo nextValueCommand = makeStringInfo(); appendStringInfo(nextValueCommand, "SELECT nextval(%s);", quote_literal_cstr( "pg_catalog.pg_dist_shardid_seq")); MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction( CitusExtensionOwnerName()); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data, &result); if (queryResult != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) != 1 || PQnfields(result) != 1) { PQclear(result); ForgetResults(connection); CloseConnection(connection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "Could not generate next shard id while executing shard splits."))); } shardId = SafeStringToUint64(PQgetvalue(result, 0, 0 /* nodeId column*/)); PQclear(result); ForgetResults(connection); return shardId; } ================================================ FILE: src/backend/distributed/operations/shard_transfer.c ================================================ /*------------------------------------------------------------------------- * * shard_transfer.c * * This file contains functions to transfer shards between nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "fmgr.h" #include "miscadmin.h" #include "access/htup_details.h" #include "catalog/pg_class.h" #include "catalog/pg_enum.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "utils/rel.h" #include "utils/syscache.h" #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_replication.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_progress.h" #include "distributed/reference_table_utils.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_split.h" #include "distributed/shard_transfer.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* local type declarations */ /* * ShardInterval along with to be executed * DDL command list. */ typedef struct ShardCommandList { ShardInterval *shardInterval; List *ddlCommandList; } ShardCommandList; static const char *ShardTransferTypeNames[] = { [SHARD_TRANSFER_INVALID_FIRST] = "unknown", [SHARD_TRANSFER_MOVE] = "move", [SHARD_TRANSFER_COPY] = "copy", }; static const char *ShardTransferTypeNamesCapitalized[] = { [SHARD_TRANSFER_INVALID_FIRST] = "unknown", [SHARD_TRANSFER_MOVE] = "Move", [SHARD_TRANSFER_COPY] = "Copy", }; static const char *ShardTransferTypeNamesContinuous[] = { [SHARD_TRANSFER_INVALID_FIRST] = "unknown", [SHARD_TRANSFER_MOVE] = "Moving", [SHARD_TRANSFER_COPY] = "Copying", }; static const char *ShardTransferTypeFunctionNames[] = { [SHARD_TRANSFER_INVALID_FIRST] = "unknown", [SHARD_TRANSFER_MOVE] = "citus_move_shard_placement", [SHARD_TRANSFER_COPY] = "citus_copy_shard_placement", }; /* local function forward declarations */ static bool CanUseLogicalReplication(Oid relationId, char shardReplicationMode); static void ErrorIfTableCannotBeReplicated(Oid relationId); static void ErrorIfTargetNodeIsNotSafeForTransfer(const char *targetNodeName, int targetNodePort, ShardTransferType transferType); static void ErrorIfSameNode(char *sourceNodeName, int sourceNodePort, char *targetNodeName, int targetNodePort, const char *operationName); static void CopyShardTables(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, bool useLogicalReplication, const char *operationName, uint32 optionFlags); static void CopyShardTablesViaLogicalReplication(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, uint32 optionFlags); static void CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, uint32 optionFlags); static void EnsureShardCanBeCopied(int64 shardId, const char *sourceNodeName, int32 sourceNodePort, const char *targetNodeName, int32 targetNodePort); static List * RecreateTableDDLCommandList(Oid relationId); static void EnsureTableListOwner(List *tableIdList); static void ErrorIfReplicatingDistributedTableWithFKeys(List *tableIdList); static void DropShardPlacementsFromMetadata(List *shardList, char *nodeName, int32 nodePort); static void UpdateColocatedShardPlacementMetadataOnWorkers(int64 shardId, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort); static bool IsShardListOnNode(List *colocatedShardList, char *targetNodeName, uint32 targetPort); static void SetupRebalanceMonitorForShardTransfer(uint64 shardId, Oid distributedTableId, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType); static void CheckSpaceConstraints(MultiConnection *connection, uint64 colocationSizeInBytes); static void EnsureAllShardsCanBeCopied(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort); static void EnsureEnoughDiskSpaceForShardMove(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType); static bool TransferAlreadyCompleted(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType); static void LockColocatedRelationsForMove(List *colocatedTableList); static void ErrorIfForeignTableForShardTransfer(List *colocatedTableList, ShardTransferType transferType); static List * RecreateShardDDLCommandList(ShardInterval *shardInterval, const char *sourceNodeName, int32 sourceNodePort); static List * PostLoadShardCreationCommandList(ShardInterval *shardInterval, const char *sourceNodeName, int32 sourceNodePort); static ShardCommandList * CreateShardCommandList(ShardInterval *shardInterval, List *ddlCommandList); static char * CreateShardCopyCommand(ShardInterval *shard, WorkerNode *targetNode); static void AcquireShardPlacementLock(uint64_t shardId, int lockMode, Oid relationId, const char *operationName); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(citus_copy_shard_placement); PG_FUNCTION_INFO_V1(citus_copy_shard_placement_with_nodeid); PG_FUNCTION_INFO_V1(master_copy_shard_placement); PG_FUNCTION_INFO_V1(citus_move_shard_placement); PG_FUNCTION_INFO_V1(citus_move_shard_placement_with_nodeid); PG_FUNCTION_INFO_V1(master_move_shard_placement); PG_FUNCTION_INFO_V1(citus_internal_copy_single_shard_placement); double DesiredPercentFreeAfterMove = 10; bool CheckAvailableSpaceBeforeMove = true; /* * citus_copy_shard_placement implements a user-facing UDF to copy a placement * from a source node to a target node, including all co-located placements. */ Datum citus_copy_shard_placement(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 shardId = PG_GETARG_INT64(0); text *sourceNodeNameText = PG_GETARG_TEXT_P(1); int32 sourceNodePort = PG_GETARG_INT32(2); text *targetNodeNameText = PG_GETARG_TEXT_P(3); int32 targetNodePort = PG_GETARG_INT32(4); Oid shardReplicationModeOid = PG_GETARG_OID(5); char *sourceNodeName = text_to_cstring(sourceNodeNameText); char *targetNodeName = text_to_cstring(targetNodeNameText); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); TransferShards(shardId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, shardReplicationMode, SHARD_TRANSFER_COPY, 0); PG_RETURN_VOID(); } /* * citus_copy_shard_placement_with_nodeid implements a user-facing UDF to copy a placement * from a source node to a target node, including all co-located placements. */ Datum citus_copy_shard_placement_with_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 shardId = PG_GETARG_INT64(0); uint32 sourceNodeId = PG_GETARG_INT32(1); uint32 targetNodeId = PG_GETARG_INT32(2); Oid shardReplicationModeOid = PG_GETARG_OID(3); bool missingOk = false; WorkerNode *sourceNode = FindNodeWithNodeId(sourceNodeId, missingOk); WorkerNode *targetNode = FindNodeWithNodeId(targetNodeId, missingOk); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); TransferShards(shardId, sourceNode->workerName, sourceNode->workerPort, targetNode->workerName, targetNode->workerPort, shardReplicationMode, SHARD_TRANSFER_COPY, 0); PG_RETURN_VOID(); } /* * master_copy_shard_placement is a wrapper function for old UDF name. */ Datum master_copy_shard_placement(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 shardId = PG_GETARG_INT64(0); text *sourceNodeNameText = PG_GETARG_TEXT_P(1); int32 sourceNodePort = PG_GETARG_INT32(2); text *targetNodeNameText = PG_GETARG_TEXT_P(3); int32 targetNodePort = PG_GETARG_INT32(4); bool doRepair = PG_GETARG_BOOL(5); Oid shardReplicationModeOid = PG_GETARG_OID(6); char *sourceNodeName = text_to_cstring(sourceNodeNameText); char *targetNodeName = text_to_cstring(targetNodeNameText); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); if (doRepair) { ereport(WARNING, (errmsg("do_repair argument is deprecated"))); } TransferShards(shardId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, shardReplicationMode, SHARD_TRANSFER_COPY, 0); PG_RETURN_VOID(); } /* * citus_internal_copy_single_shard_placement is an internal function that * copies a single shard placement from a source node to a target node. * It has two main differences from citus_copy_shard_placement: * 1. it copies only a single shard placement, not all colocated shards * 2. It allows to defer the constraints creation and this same function * can be used to create the constraints later. * * The primary use case for this function is to transfer the shards of * reference tables. Since all reference tables are colocated together, * and each reference table has only one shard, this function can be used * to transfer the shards of reference tables in parallel. * Furthermore, the reference tables could have relations with * other reference tables, so we need to ensure that their constraints * are also transferred after copying the shards to the target node. * For this reason, we allow the caller to defer the constraints creation. * * This function is not supposed to be called by the user directly. */ Datum citus_internal_copy_single_shard_placement(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 shardId = PG_GETARG_INT64(0); uint32 sourceNodeId = PG_GETARG_INT32(1); uint32 targetNodeId = PG_GETARG_INT32(2); uint32 flags = PG_GETARG_INT32(3); Oid shardReplicationModeOid = PG_GETARG_OID(4); bool missingOk = false; WorkerNode *sourceNode = FindNodeWithNodeId(sourceNodeId, missingOk); WorkerNode *targetNode = FindNodeWithNodeId(targetNodeId, missingOk); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); /* * This is an internal function that is used by the rebalancer. * It is not supposed to be called by the user directly. */ if (!IsRebalancerInternalBackend()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("This is an internal Citus function that can only" " be used by a rebalancer task"))); } TransferShards(shardId, sourceNode->workerName, sourceNode->workerPort, targetNode->workerName, targetNode->workerPort, shardReplicationMode, SHARD_TRANSFER_COPY, flags); PG_RETURN_VOID(); } /* * citus_move_shard_placement moves given shard (and its co-located shards) from one * node to the other node. To accomplish this it entirely recreates the table structure * before copying all data. * * After that, there are two different paths. First one is blocking shard move in the * sense that during shard move all modifications are paused to the shard. The second * one relies on logical replication meaning that the writes blocked only for a very * short duration almost only when the metadata is actually being updated. * * After successful move operation, shards in the source node gets deleted. If the move * fails at any point, this function throws an error, leaving the cluster without doing * any changes in source node or target node. */ Datum citus_move_shard_placement(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); List *referenceTableIdList = NIL; if (HasNodesWithMissingReferenceTables(&referenceTableIdList)) { ereport(ERROR, (errmsg("there are missing reference tables on some nodes"), errhint("Copy reference tables first with " "replicate_reference_tables() or use " "citus_rebalance_start() that will do it automatically." ))); } int64 shardId = PG_GETARG_INT64(0); char *sourceNodeName = text_to_cstring(PG_GETARG_TEXT_P(1)); int32 sourceNodePort = PG_GETARG_INT32(2); char *targetNodeName = text_to_cstring(PG_GETARG_TEXT_P(3)); int32 targetNodePort = PG_GETARG_INT32(4); Oid shardReplicationModeOid = PG_GETARG_OID(5); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); TransferShards(shardId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, shardReplicationMode, SHARD_TRANSFER_MOVE, 0); PG_RETURN_VOID(); } /* * citus_move_shard_placement_with_nodeid does the same as citus_move_shard_placement, * but accepts node ids as parameters, instead of hostname and port. */ Datum citus_move_shard_placement_with_nodeid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 shardId = PG_GETARG_INT64(0); uint32 sourceNodeId = PG_GETARG_INT32(1); uint32 targetNodeId = PG_GETARG_INT32(2); Oid shardReplicationModeOid = PG_GETARG_OID(3); bool missingOk = false; WorkerNode *sourceNode = FindNodeWithNodeId(sourceNodeId, missingOk); WorkerNode *targetNode = FindNodeWithNodeId(targetNodeId, missingOk); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); TransferShards(shardId, sourceNode->workerName, sourceNode->workerPort, targetNode->workerName, targetNode->workerPort, shardReplicationMode, SHARD_TRANSFER_MOVE, 0); PG_RETURN_VOID(); } /* * AcquireShardPlacementLock tries to acquire a lock on the shardid * while moving/copying the shard placement. If this * is it not possible it fails instantly because this means * another move/copy on same shard is currently happening. */ static void AcquireShardPlacementLock(uint64_t shardId, int lockMode, Oid relationId, const char *operationName) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = true; SET_LOCKTAG_SHARD_MOVE(tag, shardId); LockAcquireResult lockAcquired = LockAcquire(&tag, lockMode, sessionLock, dontWait); if (!lockAcquired) { ereport(ERROR, (errmsg("could not acquire the lock required to %s %s", operationName, generate_qualified_relation_name(relationId)), errdetail("It means that either a concurrent shard move " "or colocated distributed table creation is " "happening."), errhint("Make sure that the concurrent operation has " "finished and re-run the command"))); } } /* * TransferShards is responsible for handling shard transfers. * * The optionFlags parameter controls the transfer behavior: * * - By default, shard colocation groups are treated as a single unit. This works * well for distributed tables, since they can contain multiple colocated shards * on the same node, and shard transfers can still be parallelized at the group level. * * - Reference tables are different: every reference table belongs to the same * colocation group but has only a single shard. To parallelize reference table * transfers, we must bypass the colocation group. The * SHARD_TRANSFER_SINGLE_SHARD_ONLY flag enables this behavior by transferring * only the specific shardId passed into the function, ignoring colocated shards. * * - Reference tables may also define foreign key relationships with each other. * Since we cannot create those relationships until all shards have been moved, * the SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS flag is used to defer their * creation until shard transfer completes. * * - After shards are transferred, the SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY * flag is used to create the foreign key relationships for already-transferred * reference tables. * * Currently, optionFlags are only used to customize reference table transfers. * For distributed tables, optionFlags should always be set to 0. * passing 0 as optionFlags means that the default behavior will be used for * all aspects of the shard transfer. That is to consider all colocated shards * as a single unit and return after creating the necessary relationships. */ void TransferShards(int64 shardId, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, char shardReplicationMode, ShardTransferType transferType, uint32 optionFlags) { /* strings to be used in log messages */ const char *operationName = ShardTransferTypeNames[transferType]; const char *operationNameCapitalized = ShardTransferTypeNamesCapitalized[transferType]; const char *operationFunctionName = ShardTransferTypeFunctionNames[transferType]; /* cannot transfer shard to the same node */ ErrorIfSameNode(sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, operationName); ShardInterval *shardInterval = LoadShardInterval(shardId); Oid distributedTableId = shardInterval->relationId; /* error if unsupported shard transfer */ if (transferType == SHARD_TRANSFER_MOVE) { ErrorIfMoveUnsupportedTableType(distributedTableId); } else if (transferType == SHARD_TRANSFER_COPY) { ErrorIfTableCannotBeReplicated(distributedTableId); EnsureNoModificationsHaveBeenDone(); } ErrorIfTargetNodeIsNotSafeForTransfer(targetNodeName, targetNodePort, transferType); AcquirePlacementColocationLock(distributedTableId, RowExclusiveLock, operationName); List *colocatedTableList; List *colocatedShardList; /* * If SHARD_TRANSFER_SINGLE_SHARD_ONLY is set, we only transfer a single shard * specified by shardId. Otherwise, we transfer all colocated shards. */ bool isSingleShardOnly = optionFlags & SHARD_TRANSFER_SINGLE_SHARD_ONLY; if (isSingleShardOnly) { colocatedTableList = list_make1_oid(distributedTableId); colocatedShardList = list_make1(shardInterval); } else { colocatedTableList = ColocatedTableList(distributedTableId); colocatedShardList = ColocatedShardIntervalList(shardInterval); } EnsureTableListOwner(colocatedTableList); if (transferType == SHARD_TRANSFER_MOVE) { /* * Block concurrent DDL / TRUNCATE commands on the relation. while, * allow concurrent citus_move_shard_placement() on the shards of * the same relation. */ LockColocatedRelationsForMove(colocatedTableList); } ErrorIfForeignTableForShardTransfer(colocatedTableList, transferType); if (transferType == SHARD_TRANSFER_COPY) { ErrorIfReplicatingDistributedTableWithFKeys(colocatedTableList); } /* * We sort shardIntervalList so that lock operations will not cause any * deadlocks. But we do not need to do that if the list contain only one * shard. */ if (!isSingleShardOnly) { colocatedShardList = SortList(colocatedShardList, CompareShardIntervalsById); } /* We have pretty much covered the concurrent rebalance operations * and we want to allow concurrent moves within the same colocation group. * but at the same time we want to block the concurrent moves on the same shard * placement. So we lock the shard moves before starting the transfer. */ foreach_declared_ptr(shardInterval, colocatedShardList) { int64 shardIdToLock = shardInterval->shardId; AcquireShardPlacementLock(shardIdToLock, ExclusiveLock, distributedTableId, operationName); } bool transferAlreadyCompleted = TransferAlreadyCompleted(colocatedShardList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, transferType); /* * If we just need to create the shard relationships,We don't need to do anything * else other than calling CopyShardTables with SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY * flag. */ bool createRelationshipsOnly = optionFlags & SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY; if (createRelationshipsOnly) { if (!transferAlreadyCompleted) { /* * if the transfer is not completed, and we are here just to create * the relationships, we can return right away */ ereport(WARNING, (errmsg("shard is not present on node %s:%d", targetNodeName, targetNodePort), errdetail("%s may have not completed.", operationNameCapitalized))); return; } CopyShardTables(colocatedShardList, sourceNodeName, sourceNodePort, targetNodeName , targetNodePort, (shardReplicationMode == TRANSFER_MODE_FORCE_LOGICAL), operationFunctionName, optionFlags); /* We don't need to do anything else, just return */ return; } if (transferAlreadyCompleted) { /* if the transfer is already completed, we can return right away */ ereport(WARNING, (errmsg("shard is already present on node %s:%d", targetNodeName, targetNodePort), errdetail("%s may have already completed.", operationNameCapitalized))); return; } EnsureAllShardsCanBeCopied(colocatedShardList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort); if (shardReplicationMode == TRANSFER_MODE_AUTOMATIC) { VerifyTablesHaveReplicaIdentity(colocatedTableList); } EnsureEnoughDiskSpaceForShardMove(colocatedShardList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, transferType); SetupRebalanceMonitorForShardTransfer(shardId, distributedTableId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, transferType); UpdatePlacementUpdateStatusForShardIntervalList( colocatedShardList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_SETTING_UP); /* * At this point of the shard moves, we don't need to block the writes to * shards when logical replication is used. */ bool useLogicalReplication = CanUseLogicalReplication(distributedTableId, shardReplicationMode); if (!useLogicalReplication) { BlockWritesToShardList(colocatedShardList); } else if (transferType == SHARD_TRANSFER_MOVE) { /* * We prevent multiple shard moves in a transaction that use logical * replication. That's because the first call opens a transaction block * on the worker to drop the old shard placement and replication slot * creation waits for pending transactions to finish, which will not * happen ever. In other words, we prevent a self-deadlock if both * source shard placements are on the same node. */ if (PlacementMovedUsingLogicalReplicationInTX) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("moving multiple shard placements via logical " "replication in the same transaction is currently " "not supported"), errhint("If you wish to move multiple shard placements " "in a single transaction set the shard_transfer_mode " "to 'block_writes'."))); } PlacementMovedUsingLogicalReplicationInTX = true; } if (transferType == SHARD_TRANSFER_COPY && !IsCitusTableType(distributedTableId, REFERENCE_TABLE)) { /* * When copying a shard to a new node, we should first ensure that reference * tables are present such that joins work immediately after copying the shard. * When copying a reference table, we are probably trying to achieve just that. * * Since this a long-running operation we do this after the error checks, but * before taking metadata locks. */ EnsureReferenceTablesExistOnAllNodesExtended(shardReplicationMode); } DropOrphanedResourcesInSeparateTransaction(); ShardInterval *colocatedShard = NULL; foreach_declared_ptr(colocatedShard, colocatedShardList) { /* * This is to prevent any race condition possibility among the shard moves. * We don't allow the move to happen if the shard we are going to move has an * orphaned placement somewhere that is not cleanup up yet. */ char *qualifiedShardName = ConstructQualifiedShardName(colocatedShard); ErrorIfCleanupRecordForShardExists(qualifiedShardName); } CopyShardTables(colocatedShardList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, useLogicalReplication, operationFunctionName, optionFlags); if (transferType == SHARD_TRANSFER_MOVE) { /* delete old shards metadata and mark the shards as to be deferred drop */ int32 sourceGroupId = GroupForNode(sourceNodeName, sourceNodePort); InsertCleanupRecordsForShardPlacementsOnNode(colocatedShardList, sourceGroupId); } /* * Finally insert the placements to pg_dist_placement and sync it to the * metadata workers. */ colocatedShard = NULL; foreach_declared_ptr(colocatedShard, colocatedShardList) { uint64 colocatedShardId = colocatedShard->shardId; uint32 groupId = GroupForNode(targetNodeName, targetNodePort); uint64 placementId = GetNextPlacementId(); InsertShardPlacementRow(colocatedShardId, placementId, ShardLength(colocatedShardId), groupId); if (transferType == SHARD_TRANSFER_COPY && ShouldSyncTableMetadata(colocatedShard->relationId)) { char *placementCommand = PlacementUpsertCommand(colocatedShardId, placementId, 0, groupId); SendCommandToWorkersWithMetadata(placementCommand); } } if (transferType == SHARD_TRANSFER_MOVE) { /* * Since this is move operation, we remove the placements from the metadata * for the source node after copy. */ DropShardPlacementsFromMetadata(colocatedShardList, sourceNodeName, sourceNodePort); UpdateColocatedShardPlacementMetadataOnWorkers(shardId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort); } UpdatePlacementUpdateStatusForShardIntervalList( colocatedShardList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_COMPLETED); FinalizeCurrentProgressMonitor(); } /* * AdjustShardsForPrimaryCloneNodeSplit is called when a primary-clone node split * occurs. It adjusts the shard placements between the primary and clone nodes based * on the provided shard lists. Since the clone is an exact replica of the primary * but the metadata is not aware of this replication, this function updates the * metadata to reflect the new shard distribution. * * The function handles three types of shards: * * 1. Shards moving to clone node (cloneShardList): * - Updates shard placement metadata to move placements from primary to clone * - No data movement is needed since the clone already has the data * - Adds cleanup records to remove the shard data from primary at transaction commit * * 2. Shards staying on primary node (primaryShardList): * - Metadata already correctly reflects these shards on primary * - Adds cleanup records to remove the shard data from clone node * * 3. Reference tables: * - Inserts new placement records on the clone node * - Data is already present on clone, so only metadata update is needed * * This function does not perform any actual data movement; it only updates the * shard placement metadata and schedules cleanup operations for later execution. */ void AdjustShardsForPrimaryCloneNodeSplit(WorkerNode *primaryNode, WorkerNode *cloneNode, List *primaryShardList, List *cloneShardList) { /* Input validation */ if (primaryNode == NULL || cloneNode == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("primary or clone worker node is NULL"))); } if (primaryNode->nodeId == cloneNode->nodeId) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("primary and clone nodes must be different"))); } ereport(NOTICE, (errmsg( "adjusting shard placements for primary %s:%d and clone %s:%d", primaryNode->workerName, primaryNode->workerPort, cloneNode->workerName, cloneNode->workerPort))); RegisterOperationNeedingCleanup(); /* * Process shards that will stay on the primary node. * For these shards, we need to remove their data from the clone node * since the metadata already correctly reflects them on primary. */ uint64 shardId = 0; uint32 primaryGroupId = GroupForNode(primaryNode->workerName, primaryNode->workerPort) ; uint32 cloneGroupId = GroupForNode(cloneNode->workerName, cloneNode->workerPort); ereport(NOTICE, (errmsg("processing %d shards for primary node GroupID %d", list_length(primaryShardList), primaryGroupId))); /* * For each shard staying on primary, insert cleanup records to remove * the shard data from the clone node. The metadata already correctly * reflects these shards on primary, so no metadata changes are needed. */ foreach_declared_int(shardId, primaryShardList) { ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); char *qualifiedShardName = ConstructQualifiedShardName(shardInterval); ereport(LOG, (errmsg( "inserting DELETE shard record for shard %s from clone node GroupID %d", qualifiedShardName, cloneGroupId))); InsertCleanupRecordsForShardPlacementsOnNode(colocatedShardList, cloneGroupId); } /* * Process shards that will move to the clone node. * For these shards, we need to: * 1. Update metadata to move placements from primary to clone * 2. Remove the shard data from primary (via cleanup records) * 3. No data movement needed since clone already has the data */ ereport(NOTICE, (errmsg("processing %d shards for clone node GroupID %d", list_length( cloneShardList), cloneGroupId))); foreach_declared_int(shardId, cloneShardList) { ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); /* * Create new shard placement records on the clone node for all * colocated shards. This moves the shard placements from primary * to clone in the metadata. */ foreach_declared_ptr(shardInterval, colocatedShardList) { uint64 colocatedShardId = shardInterval->shardId; uint64 placementId = GetNextPlacementId(); InsertShardPlacementRow(colocatedShardId, placementId, ShardLength(colocatedShardId), cloneGroupId); } /* * Update the metadata on worker nodes to reflect the new shard * placement distribution between primary and clone nodes. */ UpdateColocatedShardPlacementMetadataOnWorkers(shardId, primaryNode->workerName, primaryNode->workerPort, cloneNode->workerName, cloneNode->workerPort); /* * Remove the shard placement records from primary node metadata * since these shards are now served from the clone node. */ DropShardPlacementsFromMetadata(colocatedShardList, primaryNode->workerName, primaryNode->workerPort); char *qualifiedShardName = ConstructQualifiedShardName(shardInterval); ereport(LOG, (errmsg( "inserting DELETE shard record for shard %s from primary node GroupID %d", qualifiedShardName, primaryGroupId))); /* * Insert cleanup records to remove the shard data from primary node * at transaction commit. This frees up space on the primary node * since the data is now served from the clone node. */ InsertCleanupRecordsForShardPlacementsOnNode(colocatedShardList, primaryGroupId); } /* * Handle reference tables - these need to be available on both * primary and clone nodes. Since the clone already has the data, * we just need to insert placement records for the clone node. */ int colocationId = GetReferenceTableColocationId(); if (colocationId == INVALID_COLOCATION_ID) { /* we have no reference table yet. */ return; } ShardInterval *shardInterval = NULL; List *referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE); Oid referenceTableId = linitial_oid(referenceTableIdList); List *shardIntervalList = LoadShardIntervalList(referenceTableId); foreach_declared_ptr(shardInterval, shardIntervalList) { List *colocatedShardList = ColocatedShardIntervalList(shardInterval); ShardInterval *colocatedShardInterval = NULL; /* * For each reference table shard, create placement records on the * clone node. The data is already present on the clone, so we only * need to update the metadata to make the clone aware of these shards. */ foreach_declared_ptr(colocatedShardInterval, colocatedShardList) { uint64 colocatedShardId = colocatedShardInterval->shardId; /* * Insert shard placement record for the clone node and * propagate the metadata change to worker nodes. */ uint64 placementId = GetNextPlacementId(); InsertShardPlacementRow(colocatedShardId, placementId, ShardLength(colocatedShardId), cloneGroupId); char *placementCommand = PlacementUpsertCommand(colocatedShardId, placementId, 0, cloneGroupId); SendCommandToWorkersWithMetadata(placementCommand); } } ereport(NOTICE, (errmsg( "shard placement adjustment complete for primary %s:%d and clone %s:%d", primaryNode->workerName, primaryNode->workerPort, cloneNode->workerName, cloneNode->workerPort))); } /* * Insert deferred cleanup records. * The shards will be dropped by background cleaner later. */ void InsertDeferredDropCleanupRecordsForShards(List *shardIntervalList) { ListCell *shardIntervalCell = NULL; foreach(shardIntervalCell, shardIntervalList) { ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell); ListCell *shardPlacementCell = NULL; uint64 oldShardId = shardInterval->shardId; /* mark for deferred drop */ List *shardPlacementList = ActiveShardPlacementList(oldShardId); foreach(shardPlacementCell, shardPlacementList) { ShardPlacement *placement = (ShardPlacement *) lfirst(shardPlacementCell); /* get shard name */ char *qualifiedShardName = ConstructQualifiedShardName(shardInterval); /* Log shard in pg_dist_cleanup. * Parent shards are to be dropped only on sucess after split workflow is complete, * so mark the policy as 'CLEANUP_DEFERRED_ON_SUCCESS'. * We also log cleanup record in the current transaction. If the current transaction rolls back, * we do not generate a record at all. */ InsertCleanupOnSuccessRecordInCurrentTransaction( CLEANUP_OBJECT_SHARD_PLACEMENT, qualifiedShardName, placement->groupId); } } } /* * InsertCleanupRecordsForShardPlacementsOnNode inserts deferred cleanup records. * The shards will be dropped by background cleaner later. * This function does this only for the placements on the given node. */ void InsertCleanupRecordsForShardPlacementsOnNode(List *shardIntervalList, int32 groupId) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { /* get shard name */ char *qualifiedShardName = ConstructQualifiedShardName(shardInterval); /* Log shard in pg_dist_cleanup. * Parent shards are to be dropped only on sucess after split workflow is complete, * so mark the policy as 'CLEANUP_DEFERRED_ON_SUCCESS'. * We also log cleanup record in the current transaction. If the current transaction rolls back, * we do not generate a record at all. */ InsertCleanupOnSuccessRecordInCurrentTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, qualifiedShardName, groupId); } } /* * IsShardListOnNode determines whether a co-located shard list has * active placements on a given node. */ static bool IsShardListOnNode(List *colocatedShardList, char *targetNodeName, uint32 targetNodePort) { WorkerNode *workerNode = FindWorkerNode(targetNodeName, targetNodePort); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Moving shards to a non-existing node is not supported"))); } /* * We exhaustively search all co-located shards */ ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, colocatedShardList) { uint64 shardId = shardInterval->shardId; List *placementList = ActiveShardPlacementListOnGroup(shardId, workerNode->groupId); if (placementList == NIL) { return false; } } return true; } /* * LockColocatedRelationsForMove takes a list of relations, locks all of them * using ShareLock */ static void LockColocatedRelationsForMove(List *colocatedTableList) { Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { LockRelationOid(colocatedTableId, RowExclusiveLock); } } /* * ErrorIfForeignTableForShardTransfer takes a list of relations, errors out if * there's a foreign table in the list. */ static void ErrorIfForeignTableForShardTransfer(List *colocatedTableList, ShardTransferType transferType) { Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { if (IsForeignTable(colocatedTableId)) { char *relationName = get_rel_name(colocatedTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot %s shard", ShardTransferTypeNames[transferType]), errdetail("Table %s is a foreign table. " "%s shards backed by foreign tables is " "not supported.", relationName, ShardTransferTypeNamesContinuous[transferType]))); } } } /* * EnsureAllShardsCanBeCopied is a wrapper around EnsureShardCanBeCopied. */ static void EnsureAllShardsCanBeCopied(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort) { ShardInterval *colocatedShard = NULL; foreach_declared_ptr(colocatedShard, colocatedShardList) { uint64 colocatedShardId = colocatedShard->shardId; /* * To transfer shard, there should be healthy placement in source node and no * placement in the target node. */ EnsureShardCanBeCopied(colocatedShardId, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort); } } /* * EnsureEnoughDiskSpaceForShardMove checks that there is enough space for * shard moves of the given colocated shard list from source node to target node. * It tries to clean up old shard placements to ensure there is enough space. */ static void EnsureEnoughDiskSpaceForShardMove(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType) { if (!CheckAvailableSpaceBeforeMove || transferType != SHARD_TRANSFER_MOVE) { return; } uint64 colocationSizeInBytes = ShardListSizeInBytes(colocatedShardList, sourceNodeName, sourceNodePort); uint32 connectionFlag = 0; MultiConnection *connection = GetNodeConnection(connectionFlag, targetNodeName, targetNodePort); CheckSpaceConstraints(connection, colocationSizeInBytes); } /* * TransferAlreadyCompleted returns true if the given shard transfer is already done. * Returns false otherwise. */ static bool TransferAlreadyCompleted(List *colocatedShardList, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType) { if (transferType == SHARD_TRANSFER_MOVE && IsShardListOnNode(colocatedShardList, targetNodeName, targetNodePort) && !IsShardListOnNode(colocatedShardList, sourceNodeName, sourceNodePort)) { return true; } if (transferType == SHARD_TRANSFER_COPY && IsShardListOnNode(colocatedShardList, targetNodeName, targetNodePort) && IsShardListOnNode(colocatedShardList, sourceNodeName, sourceNodePort)) { return true; } return false; } /* * ShardListSizeInBytes returns the size in bytes of a set of shard tables. */ uint64 ShardListSizeInBytes(List *shardList, char *workerNodeName, uint32 workerNodePort) { uint32 connectionFlag = 0; /* we skip child tables of a partitioned table if this boolean variable is true */ bool optimizePartitionCalculations = true; /* we're interested in whole table, not a particular index */ Oid indexId = InvalidOid; StringInfo tableSizeQuery = GenerateSizeQueryOnMultiplePlacements(shardList, indexId, TOTAL_RELATION_SIZE, optimizePartitionCalculations); MultiConnection *connection = GetNodeConnection(connectionFlag, workerNodeName, workerNodePort); PGresult *result = NULL; int queryResult = ExecuteOptionalRemoteCommand(connection, tableSizeQuery->data, &result); if (queryResult != RESPONSE_OKAY) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot get the size because of a connection error"))); } List *sizeList = ReadFirstColumnAsText(result); if (list_length(sizeList) != 1) { ereport(ERROR, (errmsg( "received wrong number of rows from worker, expected 1 received %d", list_length(sizeList)))); } StringInfo totalSizeStringInfo = (StringInfo) linitial(sizeList); char *totalSizeString = totalSizeStringInfo->data; uint64 totalSize = SafeStringToUint64(totalSizeString); PQclear(result); ForgetResults(connection); return totalSize; } /* * SetupRebalanceMonitorForShardTransfer prepares the parameters and * calls SetupRebalanceMonitor, unless the current transfer is a move * initiated by the rebalancer. * See comments on SetupRebalanceMonitor */ static void SetupRebalanceMonitorForShardTransfer(uint64 shardId, Oid distributedTableId, char *sourceNodeName, uint32 sourceNodePort, char *targetNodeName, uint32 targetNodePort, ShardTransferType transferType) { if (transferType == SHARD_TRANSFER_MOVE && IsRebalancerInternalBackend()) { /* * We want to be able to track progress of shard moves using * get_rebalancer_progress. If this move is initiated by the rebalancer, * then the rebalancer call has already set up the shared memory that is * used to do that, so we should return here. * But if citus_move_shard_placement is called directly by the user * (or through any other mechanism), then the shared memory is not * set up yet. In that case we do it here. */ return; } WorkerNode *sourceNode = FindWorkerNode(sourceNodeName, sourceNodePort); WorkerNode *targetNode = FindWorkerNode(targetNodeName, targetNodePort); PlacementUpdateEvent *placementUpdateEvent = palloc0( sizeof(PlacementUpdateEvent)); placementUpdateEvent->updateType = transferType == SHARD_TRANSFER_COPY ? PLACEMENT_UPDATE_COPY : PLACEMENT_UPDATE_MOVE; placementUpdateEvent->shardId = shardId; placementUpdateEvent->sourceNode = sourceNode; placementUpdateEvent->targetNode = targetNode; SetupRebalanceMonitor(list_make1(placementUpdateEvent), distributedTableId, REBALANCE_PROGRESS_MOVING, PLACEMENT_UPDATE_STATUS_SETTING_UP); } /* * CheckSpaceConstraints checks there is enough space to place the colocation * on the node that the connection is connected to. */ static void CheckSpaceConstraints(MultiConnection *connection, uint64 colocationSizeInBytes) { uint64 diskAvailableInBytes = 0; uint64 diskSizeInBytes = 0; bool success = GetNodeDiskSpaceStatsForConnection(connection, &diskAvailableInBytes, &diskSizeInBytes); if (!success) { ereport(ERROR, (errmsg("Could not fetch disk stats for node: %s-%d", connection->hostname, connection->port))); } uint64 diskAvailableInBytesAfterShardMove = 0; if (diskAvailableInBytes < colocationSizeInBytes) { /* * even though the space will be less than "0", we set it to 0 for convenience. */ diskAvailableInBytes = 0; } else { diskAvailableInBytesAfterShardMove = diskAvailableInBytes - colocationSizeInBytes; } uint64 desiredNewDiskAvailableInBytes = diskSizeInBytes * (DesiredPercentFreeAfterMove / 100); if (diskAvailableInBytesAfterShardMove < desiredNewDiskAvailableInBytes) { ereport(ERROR, (errmsg("not enough empty space on node if the shard is moved, " "actual available space after move will be %ld bytes, " "desired available space after move is %ld bytes, " "estimated size increase on node after move is %ld bytes.", diskAvailableInBytesAfterShardMove, desiredNewDiskAvailableInBytes, colocationSizeInBytes), errhint( "consider lowering citus.desired_percent_disk_available_after_move."))); } } /* * ErrorIfTargetNodeIsNotSafeForTransfer throws error if the target node is not * eligible for shard transfers. */ static void ErrorIfTargetNodeIsNotSafeForTransfer(const char *targetNodeName, int targetNodePort, ShardTransferType transferType) { WorkerNode *workerNode = FindWorkerNode(targetNodeName, targetNodePort); if (workerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s shards to a non-existing node is not supported", ShardTransferTypeNamesContinuous[transferType]), errhint( "Add the target node via SELECT citus_add_node('%s', %d);", targetNodeName, targetNodePort))); } if (!workerNode->isActive) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s shards to a non-active node is not supported", ShardTransferTypeNamesContinuous[transferType]), errhint( "Activate the target node via SELECT citus_activate_node('%s', %d);", targetNodeName, targetNodePort))); } if (transferType == SHARD_TRANSFER_MOVE && !workerNode->shouldHaveShards) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Moving shards to a node that shouldn't have a shard is " "not supported"), errhint("Allow shards on the target node via " "SELECT * FROM citus_set_node_property('%s', %d, 'shouldhaveshards', true);", targetNodeName, targetNodePort))); } if (!NodeIsPrimary(workerNode)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s shards to a secondary (e.g., replica) node is " "not supported", ShardTransferTypeNamesContinuous[transferType]))); } } /* * ErrorIfSameNode throws an error if the two host:port combinations * are the same. */ static void ErrorIfSameNode(char *sourceNodeName, int sourceNodePort, char *targetNodeName, int targetNodePort, const char *operationName) { if (strncmp(sourceNodeName, targetNodeName, MAX_NODE_LENGTH) == 0 && sourceNodePort == targetNodePort) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot %s shard to the same node", operationName))); } } /* * master_move_shard_placement is a wrapper around citus_move_shard_placement. */ Datum master_move_shard_placement(PG_FUNCTION_ARGS) { return citus_move_shard_placement(fcinfo); } /* * ErrorIfMoveUnsupportedTableType is a helper function for rebalance_table_shards * and citus_move_shard_placement udf's to error out if relation with relationId * is not a distributed table. */ void ErrorIfMoveUnsupportedTableType(Oid relationId) { if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { return; } char *qualifiedRelationName = generate_qualified_relation_name(relationId); if (!IsCitusTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table %s is a regular postgres table, you can " "only move shards of a citus table", qualifiedRelationName))); } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table %s is a local table, moving shard of " "a local table added to metadata is currently " "not supported", qualifiedRelationName))); } else if (IsCitusTableType(relationId, REFERENCE_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table %s is a reference table, moving shard of " "a reference table is not supported", qualifiedRelationName))); } } /* * VerifyTablesHaveReplicaIdentity throws an error if any of the tables * do not have a replica identity, which is required for logical replication * to replicate UPDATE and DELETE commands. */ void VerifyTablesHaveReplicaIdentity(List *colocatedTableList) { ListCell *colocatedTableCell = NULL; foreach(colocatedTableCell, colocatedTableList) { Oid colocatedTableId = lfirst_oid(colocatedTableCell); if (!RelationCanPublishAllModifications(colocatedTableId)) { char *colocatedRelationName = get_rel_name(colocatedTableId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot use logical replication to transfer shards of " "the relation %s since it doesn't have a REPLICA " "IDENTITY or PRIMARY KEY", colocatedRelationName), errdetail("UPDATE and DELETE commands on the shard will " "error out during logical replication unless " "there is a REPLICA IDENTITY or PRIMARY KEY."), errhint("If you wish to continue without a replica " "identity set the shard_transfer_mode to " "'force_logical' or 'block_writes'."))); } } } /* * RelationCanPublishAllModifications returns true if the relation is safe to publish * all modification while being replicated via logical replication. */ bool RelationCanPublishAllModifications(Oid relationId) { Relation relation = RelationIdGetRelation(relationId); bool canPublish = false; if (relation == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not open relation with OID %u", relationId))); } /* if relation has replica identity we are always good */ if (relation->rd_rel->relreplident == REPLICA_IDENTITY_FULL || OidIsValid(RelationGetReplicaIndex(relation))) { canPublish = true; } /* partitioned tables do not contain any data themselves, can always replicate */ if (PartitionedTable(relationId)) { canPublish = true; } RelationClose(relation); return canPublish; } /* * BlockWritesToShardList blocks writes to all shards in the given shard * list. The function assumes that all the shards in the list are colocated. */ void BlockWritesToShardList(List *shardList) { ShardInterval *shard = NULL; foreach_declared_ptr(shard, shardList) { /* * We need to lock the referenced reference table metadata to avoid * asynchronous shard copy in case of cascading DML operations. */ LockReferencedReferenceShardDistributionMetadata(shard->shardId, ExclusiveLock); LockShardDistributionMetadata(shard->shardId, ExclusiveLock); } /* following code relies on the list to have at least one shard */ if (list_length(shardList) == 0) { return; } /* * Since the function assumes that the input shards are colocated, * calculating shouldSyncMetadata for a single table is sufficient. */ ShardInterval *firstShardInterval = (ShardInterval *) linitial(shardList); Oid firstDistributedTableId = firstShardInterval->relationId; bool shouldSyncMetadata = ShouldSyncTableMetadata(firstDistributedTableId); if (shouldSyncMetadata || !IsCoordinator()) { /* * Even if users disable metadata sync, we cannot allow them not to * acquire the remote locks. Hence, we have !IsCoordinator() check. */ LockShardListMetadataOnWorkers(ExclusiveLock, shardList); } } /* * CanUseLogicalReplication returns true if the given table can be logically replicated. */ static bool CanUseLogicalReplication(Oid relationId, char shardReplicationMode) { if (shardReplicationMode == TRANSFER_MODE_BLOCK_WRITES) { /* user explicitly chose not to use logical replication */ return false; } /* * Logical replication doesn't support replicating foreign tables and views. */ if (!RegularTable(relationId)) { ereport(LOG, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Cannot use logical replication for " "shard move since the relation %s is not " "a regular relation", get_rel_name(relationId)))); return false; } /* Logical replication doesn't support inherited tables */ if (IsParentTable(relationId)) { ereport(LOG, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Cannot use logical replication for " "shard move since the relation %s is an " "inherited relation", get_rel_name(relationId)))); return false; } return true; } /* * ErrorIfTableCannotBeReplicated function errors out if the given table is not suitable * for its shard being replicated. There are 2 cases in which shard replication is not * allowed: * * 1) MX tables, since RF=1 is a must MX tables * 2) Reference tables, since the shard should already exist in all workers */ static void ErrorIfTableCannotBeReplicated(Oid relationId) { /* * Note that ShouldSyncTableMetadata() returns true for both MX tables * and reference tables. */ bool shouldSyncMetadata = ShouldSyncTableMetadata(relationId); if (!shouldSyncMetadata) { return; } CitusTableCacheEntry *tableEntry = GetCitusTableCacheEntry(relationId); char *relationName = get_rel_name(relationId); if (IsCitusTableTypeCacheEntry(tableEntry, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), (errmsg("Table %s is a local table. Replicating " "shard of a local table added to metadata " "currently is not supported", quote_literal_cstr(relationName))))); } /* * ShouldSyncTableMetadata() returns true also for reference table, * we don't want to error in that case since reference tables aren't * automatically replicated to active nodes with no shards, and * master_copy_shard_placement() can be used to create placements in * such nodes. */ if (tableEntry->replicationModel == REPLICATION_MODEL_STREAMING) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), (errmsg("Table %s is streaming replicated. Shards " "of streaming replicated tables cannot " "be copied", quote_literal_cstr(relationName))))); } } /* * LookupShardTransferMode maps the oids of citus.shard_transfer_mode enum * values to a char. */ char LookupShardTransferMode(Oid shardReplicationModeOid) { char shardReplicationMode = 0; Datum enumLabelDatum = DirectFunctionCall1(enum_out, shardReplicationModeOid); char *enumLabel = DatumGetCString(enumLabelDatum); if (strncmp(enumLabel, "auto", NAMEDATALEN) == 0) { shardReplicationMode = TRANSFER_MODE_AUTOMATIC; } else if (strncmp(enumLabel, "force_logical", NAMEDATALEN) == 0) { shardReplicationMode = TRANSFER_MODE_FORCE_LOGICAL; } else if (strncmp(enumLabel, "block_writes", NAMEDATALEN) == 0) { shardReplicationMode = TRANSFER_MODE_BLOCK_WRITES; } else { ereport(ERROR, (errmsg("invalid label for enum: %s", enumLabel))); } return shardReplicationMode; } /* * EnsureTableListOwner ensures current user owns given tables. Superusers * are regarded as owners. */ static void EnsureTableListOwner(List *tableIdList) { Oid tableId = InvalidOid; foreach_declared_oid(tableId, tableIdList) { EnsureTableOwner(tableId); } } /* * ErrorIfReplicatingDistributedTableWithFKeys errors out if given tables are not * suitable for replication. */ static void ErrorIfReplicatingDistributedTableWithFKeys(List *tableIdList) { Oid tableId = InvalidOid; foreach_declared_oid(tableId, tableIdList) { List *foreignConstraintCommandList = GetReferencingForeignConstaintCommands(tableId); if (foreignConstraintCommandList != NIL && IsCitusTableType(tableId, DISTRIBUTED_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot replicate shards with foreign keys"))); } } } /* * CopyShardTables copies a shard along with its co-located shards from a source * node to target node. It does not make any checks about state of the shards. * It is caller's responsibility to make those checks if they are necessary. */ static void CopyShardTables(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, bool useLogicalReplication, const char *operationName, uint32 optionFlags) { if (list_length(shardIntervalList) < 1) { return; } /* Start operation to prepare for generating cleanup records */ RegisterOperationNeedingCleanup(); bool createRelationshipsOnly = optionFlags & SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY; /* * If we're just going to create relationships only always use * CopyShardTablesViaBlockWrites. */ if (useLogicalReplication && !createRelationshipsOnly) { CopyShardTablesViaLogicalReplication(shardIntervalList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, optionFlags); } else { CopyShardTablesViaBlockWrites(shardIntervalList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, optionFlags); } /* * Drop temporary objects that were marked as CLEANUP_ALWAYS. */ FinalizeOperationNeedingCleanupOnSuccess(operationName); } /* * CopyShardTablesViaLogicalReplication copies a shard along with its co-located shards * from a source node to target node via logical replication. */ static void CopyShardTablesViaLogicalReplication(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, uint32 optionFlags) { MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CopyShardTablesViaLogicalReplication", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); /* * Iterate through the colocated shards and create them on the * target node. We do not create the indexes yet. */ ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { Oid relationId = shardInterval->relationId; uint64 shardId = shardInterval->shardId; List *tableRecreationCommandList = RecreateTableDDLCommandList(relationId); tableRecreationCommandList = WorkerApplyShardDDLCommandList(tableRecreationCommandList, shardId); char *tableOwner = TableOwner(shardInterval->relationId); /* drop the shard we created on the target, in case of failure */ InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, ConstructQualifiedShardName(shardInterval), GroupForNode(targetNodeName, targetNodePort), CLEANUP_ON_FAILURE); SendCommandListToWorkerOutsideTransaction(targetNodeName, targetNodePort, tableOwner, tableRecreationCommandList); MemoryContextReset(localContext); } MemoryContextSwitchTo(oldContext); bool skipRelationshipCreation = (optionFlags & SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS); /* data copy is done seperately when logical replication is used */ LogicallyReplicateShards(shardIntervalList, sourceNodeName, sourceNodePort, targetNodeName, targetNodePort, skipRelationshipCreation); } /* * CreateShardCommandList creates a struct for shard interval * along with DDL commands to be executed. */ static ShardCommandList * CreateShardCommandList(ShardInterval *shardInterval, List *ddlCommandList) { ShardCommandList *shardCommandList = palloc0( sizeof(ShardCommandList)); shardCommandList->shardInterval = shardInterval; shardCommandList->ddlCommandList = ddlCommandList; return shardCommandList; } /* * CopyShardTablesViaBlockWrites copies a shard along with its co-located shards * from a source node to target node via COPY command. While the command is in * progress, the modifications on the source node is blocked. */ static void CopyShardTablesViaBlockWrites(List *shardIntervalList, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, uint32 optionFlags) { MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CopyShardTablesViaBlockWrites", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); WorkerNode *sourceNode = FindWorkerNode(sourceNodeName, sourceNodePort); WorkerNode *targetNode = FindWorkerNode(targetNodeName, targetNodePort); ShardInterval *shardInterval = NULL; bool createRelationshipsOnly = optionFlags & SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY; /* * If we’re only asked to create the relationships, the shards are already * present and populated on the node. Skip the table‑setup and data‑loading * steps and proceed straight to creating the relationships. */ if (!createRelationshipsOnly) { /* iterate through the colocated shards and copy each */ foreach_declared_ptr(shardInterval, shardIntervalList) { /* * For each shard we first create the shard table in a separate * transaction and then we copy the data and create the indexes in a * second separate transaction. The reason we don't do both in a single * transaction is so we can see the size of the new shard growing * during the copy when we run get_rebalance_progress in another * session. If we wouldn't split these two phases up, then the table * wouldn't be visible in the session that get_rebalance_progress uses. * So get_rebalance_progress would always report its size as 0. */ List *ddlCommandList = RecreateShardDDLCommandList(shardInterval, sourceNodeName, sourceNodePort); char *tableOwner = TableOwner(shardInterval->relationId); /* drop the shard we created on the target, in case of failure */ InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SHARD_PLACEMENT, ConstructQualifiedShardName( shardInterval), GroupForNode(targetNodeName, targetNodePort), CLEANUP_ON_FAILURE); SendCommandListToWorkerOutsideTransaction(targetNodeName, targetNodePort, tableOwner, ddlCommandList); } UpdatePlacementUpdateStatusForShardIntervalList( shardIntervalList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_COPYING_DATA); ConflictWithIsolationTestingBeforeCopy(); CopyShardsToNode(sourceNode, targetNode, shardIntervalList, NULL); ConflictWithIsolationTestingAfterCopy(); UpdatePlacementUpdateStatusForShardIntervalList( shardIntervalList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_CREATING_CONSTRAINTS); foreach_declared_ptr(shardInterval, shardIntervalList) { List *ddlCommandList = PostLoadShardCreationCommandList(shardInterval, sourceNodeName, sourceNodePort); char *tableOwner = TableOwner(shardInterval->relationId); SendCommandListToWorkerOutsideTransaction(targetNodeName, targetNodePort, tableOwner, ddlCommandList); MemoryContextReset(localContext); } } /* * Skip creating shard relationships if the caller has requested that they * not be created. */ bool skipRelationshipCreation = (optionFlags & SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS); if (!skipRelationshipCreation) { /* * Once all shards are copied, we can recreate relationships between shards. * Create DDL commands to Attach child tables to their parents in a partitioning hierarchy. */ List *shardIntervalWithDDCommandsList = NIL; foreach_declared_ptr(shardInterval, shardIntervalList) { if (PartitionTable(shardInterval->relationId)) { char *attachPartitionCommand = GenerateAttachShardPartitionCommand(shardInterval); ShardCommandList *shardCommandList = CreateShardCommandList( shardInterval, list_make1(attachPartitionCommand)); shardIntervalWithDDCommandsList = lappend(shardIntervalWithDDCommandsList, shardCommandList); } } UpdatePlacementUpdateStatusForShardIntervalList( shardIntervalList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_CREATING_FOREIGN_KEYS); /* * Iterate through the colocated shards and create DDL commamnds * to create the foreign constraints. */ foreach_declared_ptr(shardInterval, shardIntervalList) { List *shardForeignConstraintCommandList = NIL; List *referenceTableForeignConstraintList = NIL; CopyShardForeignConstraintCommandListGrouped(shardInterval, & shardForeignConstraintCommandList, & referenceTableForeignConstraintList); ShardCommandList *shardCommandList = CreateShardCommandList( shardInterval, list_concat(shardForeignConstraintCommandList, referenceTableForeignConstraintList)); shardIntervalWithDDCommandsList = lappend(shardIntervalWithDDCommandsList, shardCommandList); } /* Now execute the Partitioning & Foreign constraints creation commads. */ ShardCommandList *shardCommandList = NULL; foreach_declared_ptr(shardCommandList, shardIntervalWithDDCommandsList) { char *tableOwner = TableOwner(shardCommandList->shardInterval->relationId); SendCommandListToWorkerOutsideTransaction(targetNodeName, targetNodePort, tableOwner, shardCommandList->ddlCommandList); } UpdatePlacementUpdateStatusForShardIntervalList( shardIntervalList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_COMPLETING); } MemoryContextReset(localContext); MemoryContextSwitchTo(oldContext); } /* * CopyShardsToNode copies the list of shards from the source to the target. * When snapshotName is not NULL it will do the COPY using this snapshot name. */ void CopyShardsToNode(WorkerNode *sourceNode, WorkerNode *targetNode, List *shardIntervalList, char *snapshotName) { int taskId = 0; List *copyTaskList = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { /* * Skip copying data for partitioned tables, because they contain no * data themselves. Their partitions do contain data, but those are * different colocated shards that will be copied seperately. */ if (PartitionedTable(shardInterval->relationId)) { continue; } List *ddlCommandList = NIL; /* * This uses repeatable read because we want to read the table in * the state exactly as it was when the snapshot was created. This * is needed when using this code for the initial data copy when * using logical replication. The logical replication catchup might * fail otherwise, because some of the updates that it needs to do * have already been applied on the target. */ StringInfo beginTransaction = makeStringInfo(); appendStringInfo(beginTransaction, "BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;"); ddlCommandList = lappend(ddlCommandList, beginTransaction->data); /* Set snapshot for non-blocking shard split. */ if (snapshotName != NULL) { StringInfo snapShotString = makeStringInfo(); appendStringInfo(snapShotString, "SET TRANSACTION SNAPSHOT %s;", quote_literal_cstr( snapshotName)); ddlCommandList = lappend(ddlCommandList, snapShotString->data); } char *copyCommand = CreateShardCopyCommand( shardInterval, targetNode); ddlCommandList = lappend(ddlCommandList, copyCommand); StringInfo commitCommand = makeStringInfo(); appendStringInfo(commitCommand, "COMMIT;"); ddlCommandList = lappend(ddlCommandList, commitCommand->data); Task *task = CitusMakeNode(Task); task->jobId = shardInterval->shardId; task->taskId = taskId; task->taskType = READ_TASK; task->replicationModel = REPLICATION_MODEL_INVALID; SetTaskQueryStringList(task, ddlCommandList); ShardPlacement *taskPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(taskPlacement, sourceNode); task->taskPlacementList = list_make1(taskPlacement); copyTaskList = lappend(copyTaskList, task); taskId++; } ExecuteTaskListOutsideTransaction(ROW_MODIFY_NONE, copyTaskList, MaxAdaptiveExecutorPoolSize, NULL /* jobIdList (ignored by API impl.) */); } /* * CreateShardCopyCommand constructs the command to copy a shard to another * worker node. This command needs to be run on the node wher you want to copy * the shard from. */ static char * CreateShardCopyCommand(ShardInterval *shard, WorkerNode *targetNode) { char *shardName = ConstructQualifiedShardName(shard); StringInfo query = makeStringInfo(); appendStringInfo(query, "SELECT pg_catalog.worker_copy_table_to_node(%s::regclass, %u);", quote_literal_cstr(shardName), targetNode->nodeId); return query->data; } /* * EnsureShardCanBeCopied checks if the given shard has a healthy placement in the source * node and no placements in the target node. */ static void EnsureShardCanBeCopied(int64 shardId, const char *sourceNodeName, int32 sourceNodePort, const char *targetNodeName, int32 targetNodePort) { List *shardPlacementList = ShardPlacementList(shardId); /* error if the source shard placement does not exist */ SearchShardPlacementInListOrError(shardPlacementList, sourceNodeName, sourceNodePort); ShardPlacement *targetPlacement = SearchShardPlacementInList(shardPlacementList, targetNodeName, targetNodePort); if (targetPlacement != NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "shard " INT64_FORMAT " already exists in the target node", shardId))); } /* * Make sure the relation exists. In some cases the relation is actually dropped but * the metadata remains, such as dropping table while citus.enable_ddl_propagation * is set to off. */ ShardInterval *shardInterval = LoadShardInterval(shardId); Oid distributedTableId = shardInterval->relationId; EnsureRelationExists(distributedTableId); } /* * SearchShardPlacementInList searches a provided list for a shard placement with the * specified node name and port. This function returns NULL if no such * placement exists in the provided list. */ ShardPlacement * SearchShardPlacementInList(List *shardPlacementList, const char *nodeName, uint32 nodePort) { ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, shardPlacementList) { if (strncmp(nodeName, shardPlacement->nodeName, MAX_NODE_LENGTH) == 0 && nodePort == shardPlacement->nodePort) { return shardPlacement; } } return NULL; } /* * SearchShardPlacementInListOrError searches a provided list for a shard * placement with the specified node name and port. This function throws an * error if no such placement exists in the provided list. * * This is a separate function (instead of using missingOk), so static analysis * reasons about NULL returns correctly. */ ShardPlacement * SearchShardPlacementInListOrError(List *shardPlacementList, const char *nodeName, uint32 nodePort) { ShardPlacement *placement = SearchShardPlacementInList(shardPlacementList, nodeName, nodePort); if (placement == NULL) { ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("could not find placement matching \"%s:%d\"", nodeName, nodePort), errhint("Confirm the placement still exists and try again."))); } return placement; } /* * RecreateShardDDLCommandList generates a command list to recreate a shard, * but without any data init and without the post-load table creation commands. */ static List * RecreateShardDDLCommandList(ShardInterval *shardInterval, const char *sourceNodeName, int32 sourceNodePort) { int64 shardId = shardInterval->shardId; Oid relationId = shardInterval->relationId; List *tableRecreationCommandList = RecreateTableDDLCommandList(relationId); return WorkerApplyShardDDLCommandList(tableRecreationCommandList, shardId); } /* * PostLoadShardCreationCommandList generates a command list to finalize the * creation of a shard after the data has been loaded. This creates stuff like * the indexes on the table. */ static List * PostLoadShardCreationCommandList(ShardInterval *shardInterval, const char *sourceNodeName, int32 sourceNodePort) { int64 shardId = shardInterval->shardId; Oid relationId = shardInterval->relationId; bool includeReplicaIdentity = true; List *indexCommandList = GetPostLoadTableCreationCommands(relationId, true, includeReplicaIdentity); return WorkerApplyShardDDLCommandList(indexCommandList, shardId); } /* * CopyShardForeignConstraintCommandList generates command list to create foreign * constraints existing in source shard after copying it to the other node. */ List * CopyShardForeignConstraintCommandList(ShardInterval *shardInterval) { List *colocatedShardForeignConstraintCommandList = NIL; List *referenceTableForeignConstraintList = NIL; CopyShardForeignConstraintCommandListGrouped( shardInterval, &colocatedShardForeignConstraintCommandList, &referenceTableForeignConstraintList); return list_concat(colocatedShardForeignConstraintCommandList, referenceTableForeignConstraintList); } /* * CopyShardForeignConstraintCommandListGrouped generates command lists * to create foreign constraints existing in source shard after copying it to other * node in separate groups for foreign constraints in between hash distributed tables * and from a hash distributed to reference tables. */ void CopyShardForeignConstraintCommandListGrouped(ShardInterval *shardInterval, List ** colocatedShardForeignConstraintCommandList, List **referenceTableForeignConstraintList) { Oid relationId = shardInterval->relationId; Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *escapedSchemaName = quote_literal_cstr(schemaName); int shardIndex = 0; List *commandList = GetReferencingForeignConstaintCommands(relationId); /* we will only use shardIndex if there is a foreign constraint */ if (commandList != NIL) { shardIndex = ShardIndex(shardInterval); } *colocatedShardForeignConstraintCommandList = NIL; *referenceTableForeignConstraintList = NIL; const char *command = NULL; foreach_declared_ptr(command, commandList) { char *escapedCommand = quote_literal_cstr(command); uint64 referencedShardId = INVALID_SHARD_ID; bool colocatedForeignKey = false; StringInfo applyForeignConstraintCommand = makeStringInfo(); /* we need to parse the foreign constraint command to get referenced table id */ Oid referencedRelationId = ForeignConstraintGetReferencedTableId(command); if (referencedRelationId == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot create foreign key constraint"), errdetail("Referenced relation cannot be found."))); } Oid referencedSchemaId = get_rel_namespace(referencedRelationId); char *referencedSchemaName = get_namespace_name(referencedSchemaId); char *escapedReferencedSchemaName = quote_literal_cstr(referencedSchemaName); if (relationId == referencedRelationId) { referencedShardId = shardInterval->shardId; } else if (IsCitusTableType(referencedRelationId, REFERENCE_TABLE)) { referencedShardId = GetFirstShardId(referencedRelationId); } else if (IsCitusTableType(referencedRelationId, CITUS_LOCAL_TABLE)) { /* * Only reference tables and citus local tables can have foreign * keys to citus local tables but we already do not allow copying * citus local table shards and we don't try to replicate citus * local table shards. So, the referencing table must be a reference * table in this context. */ Assert(IsCitusTableType(relationId, REFERENCE_TABLE)); /* * We don't set foreign keys from reference tables to citus local * tables in worker shard placements of reference tables because * we don't have the shard placement for citus local table in worker * nodes. */ continue; } else { referencedShardId = ColocatedShardIdInRelation(referencedRelationId, shardIndex); colocatedForeignKey = true; } appendStringInfo(applyForeignConstraintCommand, WORKER_APPLY_INTER_SHARD_DDL_COMMAND, shardInterval->shardId, escapedSchemaName, referencedShardId, escapedReferencedSchemaName, escapedCommand); if (colocatedForeignKey) { *colocatedShardForeignConstraintCommandList = lappend( *colocatedShardForeignConstraintCommandList, applyForeignConstraintCommand->data); } else { *referenceTableForeignConstraintList = lappend( *referenceTableForeignConstraintList, applyForeignConstraintCommand->data); } } } /* * GetFirstShardId is a helper function which returns the first * shardId of the given distributed relation. The function doesn't * sort the shardIds, so it is mostly useful for reference tables. */ uint64 GetFirstShardId(Oid relationId) { List *shardList = LoadShardList(relationId); uint64 *shardIdPointer = (uint64 *) linitial(shardList); return (*shardIdPointer); } /* * ConstuctQualifiedShardName creates the fully qualified name string of the * given shard in ._ format. */ char * ConstructQualifiedShardName(ShardInterval *shardInterval) { Oid schemaId = get_rel_namespace(shardInterval->relationId); char *schemaName = get_namespace_name(schemaId); char *tableName = get_rel_name(shardInterval->relationId); char *shardName = pstrdup(tableName); AppendShardIdToName(&shardName, shardInterval->shardId); shardName = quote_qualified_identifier(schemaName, shardName); return shardName; } /* * RecreateTableDDLCommandList returns a list of DDL statements similar to that * returned by GetTableCreationCommands except that the list begins with a "DROP TABLE" * or "DROP FOREIGN TABLE" statement to facilitate idempotent recreation of a placement. */ static List * RecreateTableDDLCommandList(Oid relationId) { const char *qualifiedRelationName = generate_qualified_relation_name(relationId); StringInfo dropCommand = makeStringInfo(); IncludeSequenceDefaults includeSequenceDefaults = NO_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = NO_IDENTITY; /* build appropriate DROP command based on relation kind */ if (RegularTable(relationId)) { appendStringInfo(dropCommand, DROP_REGULAR_TABLE_COMMAND, qualifiedRelationName); } else if (IsForeignTable(relationId)) { appendStringInfo(dropCommand, DROP_FOREIGN_TABLE_COMMAND, qualifiedRelationName); } else { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("target is not a regular, foreign or partitioned " "table"))); } List *dropCommandList = list_make1(makeTableDDLCommandString(dropCommand->data)); List *createCommandList = GetPreLoadTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, NULL); List *recreateCommandList = list_concat(dropCommandList, createCommandList); return recreateCommandList; } /* * DropShardPlacementsFromMetadata drops the shard placement metadata for * the shard placements of given shard interval list from pg_dist_placement. */ static void DropShardPlacementsFromMetadata(List *shardList, char *nodeName, int32 nodePort) { ShardInterval *shardInverval = NULL; foreach_declared_ptr(shardInverval, shardList) { uint64 shardId = shardInverval->shardId; List *shardPlacementList = ShardPlacementList(shardId); ShardPlacement *placement = SearchShardPlacementInListOrError(shardPlacementList, nodeName, nodePort); DeleteShardPlacementRow(placement->placementId); } } /* * UpdateColocatedShardPlacementMetadataOnWorkers updates the metadata about the * placements of the given shard and its colocated shards by changing the nodename and * nodeport of the shards from the source nodename/port to target nodename/port. * * Note that the function does nothing if the given shard belongs to a non-mx table. */ static void UpdateColocatedShardPlacementMetadataOnWorkers(int64 shardId, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort) { ShardInterval *shardInterval = LoadShardInterval(shardId); ListCell *colocatedShardCell = NULL; bool shouldSyncMetadata = ShouldSyncTableMetadata(shardInterval->relationId); if (!shouldSyncMetadata) { return; } uint32 sourceGroupId = GroupForNode(sourceNodeName, sourceNodePort); uint32 targetGroupId = GroupForNode(targetNodeName, targetNodePort); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); /* iterate through the colocated shards and copy each */ foreach(colocatedShardCell, colocatedShardList) { ShardInterval *colocatedShard = (ShardInterval *) lfirst(colocatedShardCell); StringInfo updateCommand = makeStringInfo(); appendStringInfo(updateCommand, "SELECT citus_internal.update_placement_metadata(%ld, %d, %d)", colocatedShard->shardId, sourceGroupId, targetGroupId); SendCommandToWorkersWithMetadata(updateCommand->data); } } /* * WorkerApplyShardDDLCommandList wraps all DDL commands in ddlCommandList * in a call to worker_apply_shard_ddl_command to apply the DDL command to * the shard specified by shardId. */ List * WorkerApplyShardDDLCommandList(List *ddlCommandList, int64 shardId) { List *applyDDLCommandList = NIL; TableDDLCommand *ddlCommand = NULL; foreach_declared_ptr(ddlCommand, ddlCommandList) { Assert(CitusIsA(ddlCommand, TableDDLCommand)); char *applyDDLCommand = GetShardedTableDDLCommand(ddlCommand, shardId, NULL); applyDDLCommandList = lappend(applyDDLCommandList, applyDDLCommand); } return applyDDLCommandList; } /* * UpdatePlacementUpdateStatusForShardIntervalList updates the status field for shards * in the given shardInterval list. */ void UpdatePlacementUpdateStatusForShardIntervalList(List *shardIntervalList, char *sourceName, int sourcePort, PlacementUpdateStatus status) { List *segmentList = NIL; List *rebalanceMonitorList = NULL; if (!HasProgressMonitor()) { rebalanceMonitorList = ProgressMonitorList(REBALANCE_ACTIVITY_MAGIC_NUMBER, &segmentList); } else { rebalanceMonitorList = list_make1(GetCurrentProgressMonitor()); } ProgressMonitorData *monitor = NULL; foreach_declared_ptr(monitor, rebalanceMonitorList) { PlacementUpdateEventProgress *steps = ProgressMonitorSteps(monitor); for (int moveIndex = 0; moveIndex < monitor->stepCount; moveIndex++) { PlacementUpdateEventProgress *step = steps + moveIndex; uint64 currentShardId = step->shardId; bool foundInList = false; ShardInterval *candidateShard = NULL; foreach_declared_ptr(candidateShard, shardIntervalList) { if (candidateShard->shardId == currentShardId) { foundInList = true; break; } } if (foundInList && strcmp(step->sourceName, sourceName) == 0 && step->sourcePort == sourcePort) { pg_atomic_write_u64(&step->updateStatus, status); } } } DetachFromDSMSegments(segmentList); } ================================================ FILE: src/backend/distributed/operations/stage_protocol.c ================================================ /*------------------------------------------------------------------------- * * stage_protocol.c * * Routines for staging PostgreSQL table data as shards into the distributed * cluster. These user-defined functions are similar to the psql-side \stage * command, but also differ from them in that users stage data from tables and * not files, and that they can also append to existing shards. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/partition.h" #include "commands/tablecmds.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/foreign_key_relationship.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/placement_connection.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" /* Local functions forward declarations */ static List * RelationShardListForShardCreate(ShardInterval *shardInterval); static bool WorkerShardStats(ShardPlacement *placement, Oid relationId, const char *shardName, uint64 *shardSize); static void UpdateTableStatistics(Oid relationId); static void ReceiveAndUpdateShardsSizes(List *connectionList); static void UpdateShardSize(uint64 shardId, ShardInterval *shardInterval, Oid relationId, List *shardPlacementList, uint64 shardSize); static bool ProcessShardStatisticsRow(PGresult *result, int64 rowIndex, uint64 *shardId, uint64 *shardSize); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_create_empty_shard); PG_FUNCTION_INFO_V1(master_append_table_to_shard); PG_FUNCTION_INFO_V1(citus_update_shard_statistics); PG_FUNCTION_INFO_V1(master_update_shard_statistics); PG_FUNCTION_INFO_V1(citus_update_table_statistics); /* * master_create_empty_shard creates an empty shard for the given distributed * table. The function first updates metadata on the coordinator node to make * this shard visible. Then it creates empty shard on worker node and added * shard placement row to metadata table. */ Datum master_create_empty_shard(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *relationNameText = PG_GETARG_TEXT_P(0); char *relationName = text_to_cstring(relationNameText); uint32 attemptableNodeCount = 0; ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); uint32 candidateNodeIndex = 0; List *candidateNodeList = NIL; text *nullMinValue = NULL; text *nullMaxValue = NULL; char storageType = SHARD_STORAGE_TABLE; Oid relationId = ResolveRelationId(relationNameText, false); EnsureTablePermissions(relationId, ACL_INSERT, ACLMASK_ALL); CheckDistributedTable(relationId); /* * distributed tables might have dependencies on different objects, since we create * shards for a distributed table via multiple sessions these objects will be created * via their own connection and committed immediately so they become visible to all * sessions creating shards. */ ObjectAddressSet(*tableAddress, RelationRelationId, relationId); EnsureAllObjectDependenciesExistOnAllNodes(list_make1(tableAddress)); EnsureReferenceTablesExistOnAllNodes(); /* don't allow the table to be dropped */ LockRelationOid(relationId, AccessShareLock); /* don't allow concurrent node list changes that require an exclusive lock */ LockRelationOid(DistNodeRelationId(), RowShareLock); /* set the storage type of foreign tables to 'f' */ if (IsForeignTable(relationId)) { storageType = SHARD_STORAGE_FOREIGN; } if (IsCitusTableType(relationId, HASH_DISTRIBUTED)) { ereport(ERROR, (errmsg("relation \"%s\" is a hash partitioned table", relationName), errdetail("We currently don't support creating shards " "on hash-partitioned tables"))); } else if (IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errmsg("relation \"%s\" is a single shard table", relationName), errdetail("We currently don't support creating shards " "on single shard tables"))); } else if (IsCitusTableType(relationId, REFERENCE_TABLE)) { ereport(ERROR, (errmsg("relation \"%s\" is a reference table", relationName), errdetail("We currently don't support creating shards " "on reference tables"))); } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("relation \"%s\" is a local table", relationName), errdetail("We currently don't support creating shards " "on local tables"))); } /* generate new and unique shardId from sequence */ uint64 shardId = GetNextShardId(); /* if enough live groups, add an extra candidate node as backup */ List *workerNodeList = DistributedTablePlacementNodeList(NoLock); if (list_length(workerNodeList) > ShardReplicationFactor) { attemptableNodeCount = ShardReplicationFactor + 1; } else { attemptableNodeCount = ShardReplicationFactor; } /* first retrieve a list of random nodes for shard placements */ while (candidateNodeIndex < attemptableNodeCount) { WorkerNode *candidateNode = WorkerGetRoundRobinCandidateNode(workerNodeList, shardId, candidateNodeIndex); if (candidateNode == NULL) { ereport(ERROR, (errmsg("could only find %u of %u possible nodes", candidateNodeIndex, attemptableNodeCount))); } candidateNodeList = lappend(candidateNodeList, candidateNode); candidateNodeIndex++; } InsertShardRow(relationId, shardId, storageType, nullMinValue, nullMaxValue); CreateAppendDistributedShardPlacements(relationId, shardId, candidateNodeList, ShardReplicationFactor); PG_RETURN_INT64(shardId); } /* * master_append_table_to_shard is a deprecated function for appending data * to a shard in an append-distributed table. */ Datum master_append_table_to_shard(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("master_append_table_to_shard has been deprecated"))); } /* * citus_update_shard_statistics updates metadata (shard size and shard min/max * values) of the given shard and returns the updated shard size. */ Datum citus_update_shard_statistics(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int64 shardId = PG_GETARG_INT64(0); uint64 shardSize = UpdateShardStatistics(shardId); PG_RETURN_INT64(shardSize); } /* * citus_update_table_statistics updates metadata (shard size and shard min/max * values) of the shards of the given table */ Datum citus_update_table_statistics(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid distributedTableId = PG_GETARG_OID(0); /* * Ensure the table still exists by trying to acquire a lock on it * If function returns NULL, it means the table doesn't exist * hence we should skip */ Relation relation = try_relation_open(distributedTableId, AccessShareLock); if (relation != NULL) { UpdateTableStatistics(distributedTableId); /* * We release the lock here since citus_update_table_statistics * is usually used in the following command: * SELECT citus_update_table_statistics(logicalrelid) FROM pg_dist_partition; * In this way we avoid holding the locks on distributed tables for a long time: * If we close the relation with NoLock, the locks on the distributed tables will * be held until the above command is finished (all distributed tables are updated). */ relation_close(relation, AccessShareLock); } else { ereport(NOTICE, (errmsg("relation with OID %u does not exist, skipping", distributedTableId))); } PG_RETURN_VOID(); } /* * master_update_shard_statistics is a wrapper function for old UDF name. */ Datum master_update_shard_statistics(PG_FUNCTION_ARGS) { return citus_update_shard_statistics(fcinfo); } /* * CheckDistributedTable checks if the given relationId corresponds to a * distributed table. If it does not, the function errors out. */ void CheckDistributedTable(Oid relationId) { char *relationName = get_rel_name(relationId); /* check that the relationId belongs to a table */ EnsureRelationKindSupported(relationId); if (!IsCitusTable(relationId)) { ereport(ERROR, (errmsg("relation \"%s\" is not a distributed table", relationName))); } } /* * CreateAppendDistributedShardPlacements creates shards for append distributed * tables on worker nodes. After successfully creating shard on the worker, * shard placement rows are added to the metadata. */ void CreateAppendDistributedShardPlacements(Oid relationId, int64 shardId, List *workerNodeList, int replicationFactor) { int attemptCount = replicationFactor; int workerNodeCount = list_length(workerNodeList); int placementsCreated = 0; IncludeSequenceDefaults includeSequenceDefaults = NO_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = NO_IDENTITY; bool creatingShellTableOnRemoteNode = false; List *ddlCommandList = GetFullTableCreationCommands(relationId, includeSequenceDefaults, includeIdentityDefaults, creatingShellTableOnRemoteNode); uint32 connectionFlag = FOR_DDL; char *relationOwner = TableOwner(relationId); /* if we have enough nodes, add an extra placement attempt for backup */ if (workerNodeCount > replicationFactor) { attemptCount++; } for (int attemptNumber = 0; attemptNumber < attemptCount; attemptNumber++) { int workerNodeIndex = attemptNumber % workerNodeCount; WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList, workerNodeIndex); if (NodeIsCoordinator(workerNode)) { ereport(NOTICE, (errmsg( "Creating placements for the append partitioned tables on the coordinator is not supported, skipping coordinator ..."))); continue; } uint32 nodeGroupId = workerNode->groupId; char *nodeName = workerNode->workerName; uint32 nodePort = workerNode->workerPort; const uint64 shardSize = 0; MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag, nodeName, nodePort, relationOwner, NULL); if (PQstatus(connection->pgConn) != CONNECTION_OK) { ereport(WARNING, (errmsg("could not connect to node \"%s:%u\"", nodeName, nodePort))); continue; } List *commandList = WorkerCreateShardCommandList(relationId, shardId, ddlCommandList); ExecuteCriticalRemoteCommandList(connection, commandList); InsertShardPlacementRow(shardId, INVALID_PLACEMENT_ID, shardSize, nodeGroupId); placementsCreated++; if (placementsCreated >= replicationFactor) { break; } } /* check if we created enough shard replicas */ if (placementsCreated < replicationFactor) { ereport(ERROR, (errmsg("could only create %u of %u of required shard replicas", placementsCreated, replicationFactor))); } } /* * InsertShardPlacementRows inserts shard placements to the metadata table on * the coordinator node. */ void InsertShardPlacementRows(Oid relationId, int64 shardId, List *workerNodeList, int workerStartIndex, int replicationFactor) { int workerNodeCount = list_length(workerNodeList); for (int placementIndex = 0; placementIndex < replicationFactor; placementIndex++) { int workerNodeIndex = (workerStartIndex + placementIndex) % workerNodeCount; WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList, workerNodeIndex); uint32 nodeGroupId = workerNode->groupId; const uint64 shardSize = 0; InsertShardPlacementRow(shardId, INVALID_PLACEMENT_ID, shardSize, nodeGroupId); } } /* * CreateShardsOnWorkers creates shards on worker nodes given the shard placements * as a parameter. The function creates the shards via the executor. This means * that it can adopt the number of connections required to create the shards. */ void CreateShardsOnWorkers(Oid distributedRelationId, List *shardPlacements, bool useExclusiveConnection) { IncludeSequenceDefaults includeSequenceDefaults = NO_SEQUENCE_DEFAULTS; IncludeIdentities includeIdentityDefaults = NO_IDENTITY; bool creatingShellTableOnRemoteNode = false; List *ddlCommandList = GetFullTableCreationCommands(distributedRelationId, includeSequenceDefaults, includeIdentityDefaults, creatingShellTableOnRemoteNode); int taskId = 1; List *taskList = NIL; int poolSize = 1; ShardPlacement *shardPlacement = NULL; foreach_declared_ptr(shardPlacement, shardPlacements) { uint64 shardId = shardPlacement->shardId; ShardInterval *shardInterval = LoadShardInterval(shardId); List *relationShardList = RelationShardListForShardCreate(shardInterval); List *commandList = WorkerCreateShardCommandList(distributedRelationId, shardId, ddlCommandList); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryStringList(task, commandList); task->replicationModel = REPLICATION_MODEL_INVALID; task->dependentTaskList = NIL; task->anchorShardId = shardId; task->relationShardList = relationShardList; task->taskPlacementList = list_make1(shardPlacement); taskList = lappend(taskList, task); } if (useExclusiveConnection) { /* * When the table has local data, we force max parallelization so data * copy is done efficiently. We also prefer to use max parallelization * when we're inside a transaction block because the user might execute * compute heavy commands (e.g., load data or create index) later in the * transaction block. */ SetLocalForceMaxQueryParallelization(); /* * TODO: After we fix adaptive executor to record parallel access for * ForceMaxQueryParallelization, we should remove this. This is just * to force adaptive executor to record parallel access to relations. * * Adaptive executor uses poolSize to decide if it should record parallel * access to relations or not, and it ignores ForceMaxQueryParallelization * because of some complications in TRUNCATE. */ poolSize = MaxAdaptiveExecutorPoolSize; } bool localExecutionSupported = true; ExecuteUtilityTaskListExtended(taskList, poolSize, localExecutionSupported); } /* * RelationShardListForShardCreate gets a shard interval and returns the placement * accesses that would happen when a placement of the shard interval is created. */ static List * RelationShardListForShardCreate(ShardInterval *shardInterval) { Oid relationId = shardInterval->relationId; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *referencedRelationList = cacheEntry->referencedRelationsViaForeignKey; List *referencingRelationList = cacheEntry->referencingRelationsViaForeignKey; int shardIndex = -1; /* list_concat_*() modifies the first arg, so make a copy first */ List *allForeignKeyRelations = list_copy(referencedRelationList); allForeignKeyRelations = list_concat_unique_oid(allForeignKeyRelations, referencingRelationList); /* record the placement access of the shard itself */ RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = relationId; relationShard->shardId = shardInterval->shardId; List *relationShardList = list_make1(relationShard); if ((IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, SINGLE_SHARD_DISTRIBUTED)) && cacheEntry->colocationId != INVALID_COLOCATION_ID) { shardIndex = ShardIndex(shardInterval); } /* all foregin key constraint relations */ Oid fkeyRelationid = InvalidOid; foreach_declared_oid(fkeyRelationid, allForeignKeyRelations) { uint64 fkeyShardId = INVALID_SHARD_ID; if (!IsCitusTable(fkeyRelationid)) { /* we're not interested in local tables */ continue; } if (IsCitusTableType(fkeyRelationid, REFERENCE_TABLE)) { fkeyShardId = GetFirstShardId(fkeyRelationid); } else if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) && IsCitusTableType(fkeyRelationid, HASH_DISTRIBUTED)) { /* hash distributed tables should be colocated to have fkey */ Assert(TableColocationId(fkeyRelationid) == cacheEntry->colocationId); fkeyShardId = ColocatedShardIdInRelation(fkeyRelationid, shardIndex); } else { /* * We currently do not support foreign keys from/to local tables or * non-colocated tables when creating shards. Also note that shard * creation via shard moves doesn't happen in a transaction block, * so not relevant here. */ continue; } RelationShard *fkeyRelationShard = CitusMakeNode(RelationShard); fkeyRelationShard->relationId = fkeyRelationid; fkeyRelationShard->shardId = fkeyShardId; relationShardList = lappend(relationShardList, fkeyRelationShard); } /* if partitioned table, make sure to record the parent table */ if (PartitionTable(relationId)) { RelationShard *parentRelationShard = CitusMakeNode(RelationShard); /* partitioned tables are always co-located */ Assert(shardIndex != -1); parentRelationShard->relationId = PartitionParentOid(relationId); parentRelationShard->shardId = ColocatedShardIdInRelation(parentRelationShard->relationId, shardIndex); relationShardList = lappend(relationShardList, parentRelationShard); } return relationShardList; } /* * WorkerCreateShardCommandList returns a list of DDL commands for the given * shardId to create the shard on the worker node. */ List * WorkerCreateShardCommandList(Oid relationId, uint64 shardId, List *ddlCommandList) { List *commandList = NIL; Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); TableDDLCommand *ddlCommand = NULL; foreach_declared_ptr(ddlCommand, ddlCommandList) { Assert(CitusIsA(ddlCommand, TableDDLCommand)); char *applyDDLCommand = GetShardedTableDDLCommand(ddlCommand, shardId, schemaName); commandList = lappend(commandList, applyDDLCommand); } ShardInterval *shardInterval = LoadShardInterval(shardId); commandList = list_concat( commandList, CopyShardForeignConstraintCommandList(shardInterval) ); /* * If the shard is created for a partition, send the command to create the * partitioning hierarcy on the shard. */ if (PartitionTable(relationId)) { char *attachPartitionCommand = GenerateAttachShardPartitionCommand(shardInterval); commandList = lappend(commandList, attachPartitionCommand); } return commandList; } /* * UpdateShardStatistics updates metadata (shard size and shard min/max values) * of the given shard and returns the updated shard size. */ uint64 UpdateShardStatistics(int64 shardId) { ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; bool statsOK = false; uint64 shardSize = 0; /* Build shard qualified name. */ char *shardName = get_rel_name(relationId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); AppendShardIdToName(&shardName, shardId); char *shardQualifiedName = quote_qualified_identifier(schemaName, shardName); List *shardPlacementList = ActiveShardPlacementList(shardId); /* get shard's statistics from a shard placement */ ShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacementList) { statsOK = WorkerShardStats(placement, relationId, shardQualifiedName, &shardSize); if (statsOK) { break; } } /* * If for some reason we appended data to a shard, but failed to retrieve * statistics we just WARN here to avoid losing shard-state updates. Note * that this means we will return 0 as the shard fill-factor, and this shard * also won't be pruned as the statistics will be empty. If the failure was * transient, a subsequent append call will fetch the correct statistics. */ if (!statsOK) { ereport(WARNING, (errmsg("could not get statistics for shard %s", shardQualifiedName), errdetail("Setting shard statistics to NULL"))); } UpdateShardSize(shardId, shardInterval, relationId, shardPlacementList, shardSize); return shardSize; } /* * UpdateTableStatistics updates metadata (shard size and shard min/max values) * of the shards of the given table. Follows a similar logic to citus_shard_sizes function. */ static void UpdateTableStatistics(Oid relationId) { List *citusTableIds = NIL; citusTableIds = lappend_oid(citusTableIds, relationId); /* we want to use a distributed transaction here to detect distributed deadlocks */ bool useDistributedTransaction = true; List *connectionList = SendShardStatisticsQueriesInParallel(citusTableIds, useDistributedTransaction); ReceiveAndUpdateShardsSizes(connectionList); } /* * ReceiveAndUpdateShardsSizes receives shard id and size * results from the given connection list, and updates * respective entries in pg_dist_placement. */ static void ReceiveAndUpdateShardsSizes(List *connectionList) { /* * From the connection list, we will not get all the shards, but * all the placements. We use a hash table to remember already visited shard ids * since we update all the different placements of a shard id at once. */ HTAB *alreadyVisitedShardPlacements = CreateSimpleHashSetWithName(Oid, "oid visited hash set"); MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { if (PQstatus(connection->pgConn) != CONNECTION_OK) { continue; } bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, WARNING); continue; } int64 rowCount = PQntuples(result); int64 colCount = PQnfields(result); /* Although it is not expected */ if (colCount != SHARD_SIZES_COLUMN_COUNT) { ereport(WARNING, (errmsg("unexpected number of columns from " "citus_update_table_statistics"))); continue; } for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { uint64 shardId = 0; uint64 shardSize = 0; if (!ProcessShardStatisticsRow(result, rowIndex, &shardId, &shardSize)) { /* this row has no valid shard statistics */ continue; } if (OidVisited(alreadyVisitedShardPlacements, shardId)) { /* We have already updated this placement list */ continue; } VisitOid(alreadyVisitedShardPlacements, shardId); ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; List *shardPlacementList = ActiveShardPlacementList(shardId); UpdateShardSize(shardId, shardInterval, relationId, shardPlacementList, shardSize); } PQclear(result); ForgetResults(connection); } hash_destroy(alreadyVisitedShardPlacements); } /* * ProcessShardStatisticsRow processes a row of shard statistics of the input PGresult * - it returns true if this row belongs to a valid shard * - it returns false if this row has no valid shard statistics (shardId = INVALID_SHARD_ID) * * Input tuples are assumed to be of the form: * (shard_id bigint, shard_name text, shard_size bigint) */ static bool ProcessShardStatisticsRow(PGresult *result, int64 rowIndex, uint64 *shardId, uint64 *shardSize) { *shardId = ParseIntField(result, rowIndex, 0); /* check for the dummy entries we put so that UNION ALL wouldn't complain */ if (*shardId == INVALID_SHARD_ID) { /* this row has no valid shard statistics */ return false; } *shardSize = ParseIntField(result, rowIndex, 1); return true; } /* * UpdateShardSize updates the shardlength (shard size) of the given * shard and its placements in pg_dist_placement. */ static void UpdateShardSize(uint64 shardId, ShardInterval *shardInterval, Oid relationId, List *shardPlacementList, uint64 shardSize) { ShardPlacement *placement = NULL; /* update metadata for each shard placement */ foreach_declared_ptr(placement, shardPlacementList) { uint64 placementId = placement->placementId; int32 groupId = placement->groupId; DeleteShardPlacementRow(placementId); InsertShardPlacementRow(shardId, placementId, shardSize, groupId); } } /* * WorkerShardStats queries the worker node, and retrieves shard statistics that * we assume have changed after new table data have been appended to the shard. */ static bool WorkerShardStats(ShardPlacement *placement, Oid relationId, const char *shardName, uint64 *shardSize) { StringInfo tableSizeQuery = makeStringInfo(); PGresult *queryResult = NULL; char *tableSizeStringEnd = NULL; int connectionFlags = 0; MultiConnection *connection = GetPlacementConnection(connectionFlags, placement, NULL); /* * This code-path doesn't support optional connections, so we don't expect * NULL connections. */ Assert(connection != NULL); *shardSize = 0; char *quotedShardName = quote_literal_cstr(shardName); appendStringInfo(tableSizeQuery, SHARD_TABLE_SIZE_QUERY, quotedShardName); int executeCommand = ExecuteOptionalRemoteCommand(connection, tableSizeQuery->data, &queryResult); if (executeCommand != 0) { return false; } char *tableSizeString = PQgetvalue(queryResult, 0, 0); if (tableSizeString == NULL) { PQclear(queryResult); ForgetResults(connection); return false; } errno = 0; uint64 tableSize = strtou64(tableSizeString, &tableSizeStringEnd, 0); if (errno != 0 || (*tableSizeStringEnd) != '\0') { PQclear(queryResult); ForgetResults(connection); return false; } *shardSize = tableSize; PQclear(queryResult); ForgetResults(connection); return true; } /* * ForeignConstraintGetReferencedTableId parses given foreign constraint query and * extracts referenced table id from it. */ Oid ForeignConstraintGetReferencedTableId(const char *queryString) { Node *queryNode = ParseTreeNode(queryString); AlterTableStmt *foreignConstraintStmt = (AlterTableStmt *) queryNode; AlterTableCmd *command = (AlterTableCmd *) linitial(foreignConstraintStmt->cmds); if (command->subtype == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { RangeVar *referencedTable = constraint->pktable; return RangeVarGetRelid(referencedTable, NoLock, foreignConstraintStmt->missing_ok); } } return InvalidOid; } ================================================ FILE: src/backend/distributed/operations/worker_copy_table_to_node_udf.c ================================================ /*------------------------------------------------------------------------- * * worker_copy_table_to_node_udf.c * * This file implements the worker_copy_table_to_node UDF. This UDF can be * used to copy the data in a shard (or other table) from one worker node to * another. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "executor/executor.h" /* for CreateExecutorState(), FreeExecutorState(), CreateExprContext(), etc. */ #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/priority.h" #include "distributed/worker_shard_copy.h" PG_FUNCTION_INFO_V1(worker_copy_table_to_node); /* * worker_copy_table_to_node copies a shard from this worker to another worker * * SQL signature: * * worker_copy_table_to_node( * source_table regclass, * target_node_id integer * ) RETURNS VOID */ Datum worker_copy_table_to_node(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); uint32_t targetNodeId = PG_GETARG_INT32(1); if (IsCitusTable(relationId)) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table %s is a Citus table, only copies of " "shards or regular postgres tables are supported", qualifiedRelationName))); } Oid schemaOid = get_rel_namespace(relationId); char *relationSchemaName = get_namespace_name(schemaOid); char *relationName = get_rel_name(relationId); char *relationQualifiedName = quote_qualified_identifier( relationSchemaName, relationName); EState *executor = CreateExecutorState(); DestReceiver *destReceiver = CreateShardCopyDestReceiver( executor, list_make2(relationSchemaName, relationName), targetNodeId); StringInfo selectShardQueryForCopy = makeStringInfo(); /* * Even though we do COPY(SELECT ...) all the columns, we can't just do SELECT * because we need to not COPY generated colums. */ const char *columnList = CopyableColumnNamesFromRelationName(relationSchemaName, relationName); appendStringInfo(selectShardQueryForCopy, "SELECT %s FROM %s;", columnList, relationQualifiedName); ParamListInfo params = NULL; ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params, destReceiver); FreeExecutorState(executor); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/operations/worker_node_manager.c ================================================ /*------------------------------------------------------------------------- * * worker_node_manager.c * Routines for reading worker nodes from membership file, and allocating * candidate nodes for shard placement. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "common/ip.h" #include "libpq/hba.h" #include "libpq/libpq-be.h" #include "postmaster/postmaster.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/shmem.h" #include "utils/guc.h" #include "utils/hsearch.h" #include "utils/memutils.h" #include "distributed/coordinator_protocol.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/worker_manager.h" /* Config variables managed via guc.c */ char *WorkerListFileName; int MaxWorkerNodesTracked = 2048; /* determines worker node hash table size */ /* Local functions forward declarations */ static bool NodeIsPrimaryWorker(WorkerNode *node); static bool NodeIsReadableWorker(WorkerNode *node); /* ------------------------------------------------------------ * Worker node selection functions follow * ------------------------------------------------------------ */ /* * WorkerGetRoundRobinCandidateNode takes in a list of worker nodes and returns * a candidate worker node from that list. To select this node, this function * uses the round-robin policy. An ideal round-robin implementation requires * keeping shared state for shard placements; and we instead approximate our * implementation by relying on the ever-increasing shardId. So, the first * worker node selected will be the node at the (shardId MOD worker node count) * index and the remaining candidate nodes will be the next nodes in the list. * * Note that the function returns null if the worker membership list does not * contain enough nodes to place all replicas. */ WorkerNode * WorkerGetRoundRobinCandidateNode(List *workerNodeList, uint64 shardId, uint32 placementIndex) { uint32 workerNodeCount = list_length(workerNodeList); WorkerNode *candidateNode = NULL; if (placementIndex < workerNodeCount) { uint32 candidateNodeIndex = (shardId + placementIndex) % workerNodeCount; candidateNode = (WorkerNode *) list_nth(workerNodeList, candidateNodeIndex); } return candidateNode; } /* * ActivePrimaryNonCoordinatorNodeCount returns the number of groups with a primary in the cluster. * This method excludes coordinator even if it is added as a worker to cluster. */ uint32 ActivePrimaryNonCoordinatorNodeCount(void) { List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(NoLock); uint32 liveWorkerCount = list_length(workerNodeList); return liveWorkerCount; } /* * ActiveReadableNodeCount returns the number of nodes in the cluster. */ uint32 ActiveReadableNodeCount(void) { List *nodeList = ActiveReadableNodeList(); return list_length(nodeList); } /* * NodeIsCoordinator returns true if the given node represents the coordinator. */ bool NodeIsCoordinator(WorkerNode *node) { return node->groupId == COORDINATOR_GROUP_ID; } /* * ActiveNodeListFilterFunc returns a list of all active nodes that checkFunction * returns true for. * lockMode specifies which lock to use on pg_dist_node, this is necessary when * the caller wouldn't want nodes to be added concurrent to their use of this list */ static List * FilterActiveNodeListFunc(LOCKMODE lockMode, bool (*checkFunction)(WorkerNode *)) { List *workerNodeList = NIL; WorkerNode *workerNode = NULL; HASH_SEQ_STATUS status; Assert(checkFunction != NULL); if (lockMode != NoLock) { LockRelationOid(DistNodeRelationId(), lockMode); } HTAB *workerNodeHash = GetWorkerNodeHash(); hash_seq_init(&status, workerNodeHash); while ((workerNode = hash_seq_search(&status)) != NULL) { if (workerNode->isActive && checkFunction(workerNode)) { WorkerNode *workerNodeCopy = palloc0(sizeof(WorkerNode)); *workerNodeCopy = *workerNode; workerNodeList = lappend(workerNodeList, workerNodeCopy); } } return workerNodeList; } /* * ActivePrimaryNonCoordinatorNodeList returns a list of all active primary worker nodes * in workerNodeHash. lockMode specifies which lock to use on pg_dist_node, * this is necessary when the caller wouldn't want nodes to be added concurrent * to their use of this list. * This method excludes coordinator even if it is added as a worker to cluster. */ List * ActivePrimaryNonCoordinatorNodeList(LOCKMODE lockMode) { EnsureModificationsCanRun(); return FilterActiveNodeListFunc(lockMode, NodeIsPrimaryWorker); } /* * ActivePrimaryNodeList returns a list of all active primary nodes in * workerNodeHash. */ List * ActivePrimaryNodeList(LOCKMODE lockMode) { EnsureModificationsCanRun(); return FilterActiveNodeListFunc(lockMode, NodeIsPrimary); } /* * ActivePrimaryRemoteNodeList returns a list of all active primary nodes in * workerNodeHash except the local one. */ List * ActivePrimaryRemoteNodeList(LOCKMODE lockMode) { EnsureModificationsCanRun(); return FilterActiveNodeListFunc(lockMode, NodeIsPrimaryAndRemote); } /* * NodeIsPrimaryWorker returns true if the node is a primary worker node. */ static bool NodeIsPrimaryWorker(WorkerNode *node) { return !NodeIsCoordinator(node) && NodeIsPrimary(node); } /* * CoordinatorAddedAsWorkerNode returns true if coordinator is added to the * pg_dist_node. */ bool CoordinatorAddedAsWorkerNode() { bool groupContainsNodes = false; PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &groupContainsNodes); return groupContainsNodes; } /* * ReferenceTablePlacementNodeList returns the set of nodes that should have * reference table placements. This includes all primaries, including the * coordinator if known. */ List * ReferenceTablePlacementNodeList(LOCKMODE lockMode) { EnsureModificationsCanRun(); return FilterActiveNodeListFunc(lockMode, NodeIsPrimary); } /* * CoordinatorNodeIfAddedAsWorkerOrError returns the WorkerNode object for * coordinator node if it is added to pg_dist_node, otherwise errors out. * Also, as CoordinatorAddedAsWorkerNode acquires AccessShareLock on pg_dist_node * and doesn't release it, callers can safely assume coordinator won't be * removed from metadata until the end of transaction when this function * returns coordinator node. */ WorkerNode * CoordinatorNodeIfAddedAsWorkerOrError() { ErrorIfCoordinatorNotAddedAsWorkerNode(); WorkerNode *coordinatorNode = LookupNodeForGroup(COORDINATOR_GROUP_ID); WorkerNode *coordinatorNodeCopy = palloc0(sizeof(WorkerNode)); *coordinatorNodeCopy = *coordinatorNode; return coordinatorNodeCopy; } /* * ErrorIfCoordinatorNotAddedAsWorkerNode errors out if coordinator is not added * to metadata. */ void ErrorIfCoordinatorNotAddedAsWorkerNode() { if (CoordinatorAddedAsWorkerNode()) { return; } ereport(ERROR, (errmsg("operation is not allowed when coordinator " "is not added into metadata"), errhint("Use \"SELECT citus_set_coordinator_host('" "', '')\" to configure the " "coordinator hostname and port"))); } /* * DistributedTablePlacementNodeList returns a list of all active, primary * worker nodes that can store new data, i.e shouldstoreshards is 'true' */ List * DistributedTablePlacementNodeList(LOCKMODE lockMode) { EnsureModificationsCanRun(); return FilterActiveNodeListFunc(lockMode, NodeCanHaveDistTablePlacements); } /* * NodeCanHaveDistTablePlacements returns true if the given node can have * shards of a distributed table. */ bool NodeCanHaveDistTablePlacements(WorkerNode *node) { if (!NodeIsPrimary(node)) { return false; } return node->shouldHaveShards; } /* * ActiveReadableNonCoordinatorNodeList returns a list of all nodes in workerNodeHash * that are readable nodes This method excludes coordinator. */ List * ActiveReadableNonCoordinatorNodeList(void) { return FilterActiveNodeListFunc(NoLock, NodeIsReadableWorker); } /* * ActiveReadableNodeList returns a list of all nodes in workerNodeHash * that are readable workers. * This method includes coordinator if it is added as a worker to the cluster. */ List * ActiveReadableNodeList(void) { return FilterActiveNodeListFunc(NoLock, NodeIsReadable); } /* * NodeIsReadableWorker returns true if the given node is a readable worker node. */ static bool NodeIsReadableWorker(WorkerNode *node) { return !NodeIsCoordinator(node) && NodeIsReadable(node); } /* * CompareWorkerNodes compares two pointers to worker nodes using the exact * same logic employed by WorkerNodeCompare. */ int CompareWorkerNodes(const void *leftElement, const void *rightElement) { const void *leftWorker = *((const void **) leftElement); const void *rightWorker = *((const void **) rightElement); Size ignoredKeySize = 0; int compare = WorkerNodeCompare(leftWorker, rightWorker, ignoredKeySize); return compare; } /* * WorkerNodeCompare compares two worker nodes by their host name and port * number. Two nodes that only differ by their rack locations are considered to * be equal to each other. */ int WorkerNodeCompare(const void *lhsKey, const void *rhsKey, Size keySize) { const WorkerNode *workerLhs = (const WorkerNode *) lhsKey; const WorkerNode *workerRhs = (const WorkerNode *) rhsKey; return NodeNamePortCompare(workerLhs->workerName, workerRhs->workerName, workerLhs->workerPort, workerRhs->workerPort); } /* * WorkerNodeHashCode computes the hash code for a worker node from the node's * host name and port number. Nodes that only differ by their rack locations * hash to the same value. */ uint32 WorkerNodeHashCode(const void *key, Size keySize) { const WorkerNode *worker = (const WorkerNode *) key; const char *workerName = worker->workerName; const uint32 *workerPort = &(worker->workerPort); /* standard hash function outlined in Effective Java, Item 8 */ uint32 result = 17; result = 37 * result + string_hash(workerName, WORKER_LENGTH); result = 37 * result + tag_hash(workerPort, sizeof(uint32)); return result; } /* * NodeNamePortCompare implements the common logic for comparing two nodes * with their given nodeNames and ports. * * This function is useful for ensuring consistency of sort operations between * different representations of nodes in the cluster such as WorkerNode and * WorkerPool. */ int NodeNamePortCompare(const char *workerLhsName, const char *workerRhsName, int workerLhsPort, int workerRhsPort) { int nameCompare = strncmp(workerLhsName, workerRhsName, WORKER_LENGTH); if (nameCompare != 0) { return nameCompare; } int portCompare = workerLhsPort - workerRhsPort; return portCompare; } /* * GetFirstPrimaryWorkerNode returns the primary worker node with the * lowest rank based on CompareWorkerNodes. * * The ranking is arbitrary, but needs to be kept consistent with IsFirstWorkerNode. */ WorkerNode * GetFirstPrimaryWorkerNode(void) { List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(RowShareLock); WorkerNode *firstWorkerNode = NULL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if (firstWorkerNode == NULL || CompareWorkerNodes(&workerNode, &firstWorkerNode) < 0) { firstWorkerNode = workerNode; } } return firstWorkerNode; } ================================================ FILE: src/backend/distributed/operations/worker_shard_copy.c ================================================ /*------------------------------------------------------------------------- * * worker_shard_copy.c * Functions for copying a shard to destination. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "commands/copy.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" #include "distributed/relation_utils.h" #include "distributed/remote_commands.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_shard_copy.h" /* * LocalCopyBuffer is used in copy callback to return the copied rows. * The reason this is a global variable is that we cannot pass an additional * argument to the copy callback. */ static StringInfo LocalCopyBuffer; typedef struct ShardCopyDestReceiver { /* public DestReceiver interface */ DestReceiver pub; /* Destination Relation Name */ List *destinationShardFullyQualifiedName; /* descriptor of the tuples that are sent to the worker */ TupleDesc tupleDescriptor; /* state on how to copy out data types */ CopyOutState copyOutState; FmgrInfo *columnOutputFunctions; /* number of tuples sent */ int64 tuplesSent; /* destination node id */ uint32_t destinationNodeId; /* local copy if destination shard in same node */ bool useLocalCopy; /* EState for per-tuple memory allocation */ EState *executorState; /* * Connection for destination shard (NULL if useLocalCopy is true) */ MultiConnection *connection; } ShardCopyDestReceiver; static bool ShardCopyDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); static void ShardCopyDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static void ShardCopyDestReceiverShutdown(DestReceiver *destReceiver); static void ShardCopyDestReceiverDestroy(DestReceiver *destReceiver); static bool CanUseLocalCopy(uint32_t destinationNodeId); static StringInfo ConstructShardCopyStatement(List *destinationShardFullyQualifiedName, bool useBinaryFormat, TupleDesc tupleDesc); static void WriteLocalTuple(TupleTableSlot *slot, ShardCopyDestReceiver *copyDest); static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead); static void LocalCopyToShard(ShardCopyDestReceiver *copyDest, CopyOutState localCopyOutState); static void ConnectToRemoteAndStartCopy(ShardCopyDestReceiver *copyDest); static bool CanUseLocalCopy(uint32_t destinationNodeId) { /* If destination node is same as source, use local copy */ return GetLocalNodeId() == (int32) destinationNodeId; } /* Connect to node with source shard and trigger copy start. */ static void ConnectToRemoteAndStartCopy(ShardCopyDestReceiver *copyDest) { int connectionFlags = OUTSIDE_TRANSACTION; char *currentUser = CurrentUserName(); WorkerNode *workerNode = FindNodeWithNodeId(copyDest->destinationNodeId, false /* missingOk */); copyDest->connection = GetNodeUserDatabaseConnection(connectionFlags, workerNode->workerName, workerNode->workerPort, currentUser, NULL /* database (current) */); ClaimConnectionExclusively(copyDest->connection); RemoteTransactionBeginIfNecessary(copyDest->connection); SetupReplicationOriginRemoteSession(copyDest->connection); StringInfo copyStatement = ConstructShardCopyStatement( copyDest->destinationShardFullyQualifiedName, copyDest->copyOutState->binary, copyDest->tupleDescriptor); if (!SendRemoteCommand(copyDest->connection, copyStatement->data)) { ReportConnectionError(copyDest->connection, ERROR); } PGresult *result = GetRemoteCommandResult(copyDest->connection, true /* raiseInterrupts */); if (PQresultStatus(result) != PGRES_COPY_IN) { ReportResultError(copyDest->connection, result, ERROR); } PQclear(result); } /* * CreateShardCopyDestReceiver creates a DestReceiver that copies into * a destinationShardFullyQualifiedName on destinationNodeId. */ DestReceiver * CreateShardCopyDestReceiver(EState *executorState, List *destinationShardFullyQualifiedName, uint32_t destinationNodeId) { ShardCopyDestReceiver *copyDest = (ShardCopyDestReceiver *) palloc0( sizeof(ShardCopyDestReceiver)); /* set up the DestReceiver function pointers */ copyDest->pub.receiveSlot = ShardCopyDestReceiverReceive; copyDest->pub.rStartup = ShardCopyDestReceiverStartup; copyDest->pub.rShutdown = ShardCopyDestReceiverShutdown; copyDest->pub.rDestroy = ShardCopyDestReceiverDestroy; copyDest->pub.mydest = DestCopyOut; copyDest->executorState = executorState; copyDest->destinationNodeId = destinationNodeId; copyDest->destinationShardFullyQualifiedName = destinationShardFullyQualifiedName; copyDest->tuplesSent = 0; copyDest->connection = NULL; copyDest->useLocalCopy = CanUseLocalCopy(destinationNodeId); return (DestReceiver *) copyDest; } /* * ShardCopyDestReceiverReceive implements the receiveSlot function of * ShardCopyDestReceiver. It takes a TupleTableSlot and sends the contents to * the appropriate destination node. */ static bool ShardCopyDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) { ShardCopyDestReceiver *copyDest = (ShardCopyDestReceiver *) dest; /* * Switch to a per-tuple memory memory context. When used in * context of Split Copy, this is a no-op as switch is already done. */ EState *executorState = copyDest->executorState; MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState); MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); /* If remote copy, connect lazily and initiate copy */ if (copyDest->tuplesSent == 0 && (!copyDest->useLocalCopy)) { ConnectToRemoteAndStartCopy(copyDest); } slot_getallattrs(slot); Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; CopyOutState copyOutState = copyDest->copyOutState; if (copyDest->useLocalCopy) { /* Setup replication origin session for local copy*/ WriteLocalTuple(slot, copyDest); if (copyOutState->fe_msgbuf->len > LocalCopyFlushThresholdByte) { LocalCopyToShard(copyDest, copyOutState); } } else { resetStringInfo(copyOutState->fe_msgbuf); if (copyDest->copyOutState->binary && copyDest->tuplesSent == 0) { AppendCopyBinaryHeaders(copyDest->copyOutState); } AppendCopyRowData(columnValues, columnNulls, copyDest->tupleDescriptor, copyOutState, copyDest->columnOutputFunctions, NULL /* columnCoercionPaths */); if (!PutRemoteCopyData(copyDest->connection, copyOutState->fe_msgbuf->data, copyOutState->fe_msgbuf->len)) { char *destinationShardSchemaName = linitial( copyDest->destinationShardFullyQualifiedName); char *destinationShardRelationName = lsecond( copyDest->destinationShardFullyQualifiedName); char *errorMessage = PQerrorMessage(copyDest->connection->pgConn); ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("Failed to COPY to shard %s.%s : %s,", destinationShardSchemaName, destinationShardRelationName, errorMessage), errdetail("failed to send %d bytes %s on node %u", copyOutState->fe_msgbuf->len, copyOutState->fe_msgbuf->data, copyDest->destinationNodeId))); } } MemoryContextSwitchTo(oldContext); ResetPerTupleExprContext(executorState); copyDest->tuplesSent++; return true; } /* * ShardCopyDestReceiverStartup implements the rStartup interface of ShardCopyDestReceiver. */ static void ShardCopyDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor) { ShardCopyDestReceiver *copyDest = (ShardCopyDestReceiver *) dest; copyDest->tupleDescriptor = inputTupleDescriptor; copyDest->tuplesSent = 0; const char *delimiterCharacter = "\t"; const char *nullPrintCharacter = "\\N"; /* define how tuples will be serialised */ CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->binary = EnableBinaryProtocol && CanUseBinaryCopyFormat( inputTupleDescriptor); copyOutState->null_print = (char *) nullPrintCharacter; copyOutState->null_print_client = (char *) nullPrintCharacter; copyOutState->fe_msgbuf = makeStringInfo(); copyOutState->delim = (char *) delimiterCharacter; copyOutState->rowcontext = GetPerTupleMemoryContext(copyDest->executorState); copyDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor, copyOutState->binary); copyDest->copyOutState = copyOutState; if (copyDest->useLocalCopy) { /* Setup replication origin session for local copy*/ SetupReplicationOriginLocalSession(); } } /* * ShardCopyDestReceiverShutdown implements the rShutdown interface of * ShardCopyDestReceiver. It ends all open COPY operations, copying any pending * data in buffer. */ static void ShardCopyDestReceiverShutdown(DestReceiver *dest) { ShardCopyDestReceiver *copyDest = (ShardCopyDestReceiver *) dest; if (copyDest->useLocalCopy) { if (copyDest->copyOutState != NULL && copyDest->copyOutState->fe_msgbuf->len > 0) { /* end the COPY input */ LocalCopyToShard(copyDest, copyDest->copyOutState); } } else if (copyDest->connection != NULL) { resetStringInfo(copyDest->copyOutState->fe_msgbuf); if (copyDest->copyOutState->binary) { AppendCopyBinaryFooters(copyDest->copyOutState); } /* end the COPY input */ if (!PutRemoteCopyEnd(copyDest->connection, NULL /* errormsg */)) { char *destinationShardSchemaName = linitial( copyDest->destinationShardFullyQualifiedName); char *destinationShardRelationName = lsecond( copyDest->destinationShardFullyQualifiedName); ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("Failed to COPY to destination shard %s.%s", destinationShardSchemaName, destinationShardRelationName), errdetail("failed to send %d bytes %s on node %u", copyDest->copyOutState->fe_msgbuf->len, copyDest->copyOutState->fe_msgbuf->data, copyDest->destinationNodeId))); } /* check whether there were any COPY errors */ PGresult *result = GetRemoteCommandResult(copyDest->connection, true /* raiseInterrupts */); if (PQresultStatus(result) != PGRES_COMMAND_OK) { ReportCopyError(copyDest->connection, result); } PQclear(result); ForgetResults(copyDest->connection); ResetReplicationOriginRemoteSession(copyDest->connection); CloseConnection(copyDest->connection); } } /* * ShardCopyDestReceiverDestroy frees the DestReceiver. */ static void ShardCopyDestReceiverDestroy(DestReceiver *dest) { ShardCopyDestReceiver *copyDest = (ShardCopyDestReceiver *) dest; if (copyDest->useLocalCopy) { ResetReplicationOriginLocalSession(); } if (copyDest->copyOutState) { pfree(copyDest->copyOutState); } if (copyDest->columnOutputFunctions) { pfree(copyDest->columnOutputFunctions); } pfree(copyDest); } /* * CopyableColumnNamesFromTupleDesc function creates and returns a comma seperated column names string to be used in COPY * and SELECT statements when copying a table. The COPY and SELECT statements should filter out the GENERATED columns since COPY * statement fails to handle them. Iterating over the attributes of the table we also need to skip the dropped columns. */ const char * CopyableColumnNamesFromTupleDesc(TupleDesc tupDesc) { StringInfo columnList = makeStringInfo(); bool firstInList = true; for (int i = 0; i < tupDesc->natts; i++) { Form_pg_attribute att = TupleDescAttr(tupDesc, i); if (att->attgenerated || att->attisdropped) { continue; } if (!firstInList) { appendStringInfo(columnList, ","); } firstInList = false; appendStringInfo(columnList, "%s", quote_identifier(NameStr(att->attname))); } return columnList->data; } /* * CopyableColumnNamesFromRelationName function is a wrapper for CopyableColumnNamesFromTupleDesc. */ const char * CopyableColumnNamesFromRelationName(const char *schemaName, const char *relationName) { Oid namespaceOid = get_namespace_oid(schemaName, true); Oid relationId = get_relname_relid(relationName, namespaceOid); Relation relation = relation_open(relationId, AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(relation); const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc); relation_close(relation, NoLock); return columnList; } /* * ConstructShardCopyStatement constructs the text of a COPY statement * for copying into a result table */ static StringInfo ConstructShardCopyStatement(List *destinationShardFullyQualifiedName, bool useBinaryFormat, TupleDesc tupleDesc) { char *destinationShardSchemaName = linitial(destinationShardFullyQualifiedName); char *destinationShardRelationName = lsecond(destinationShardFullyQualifiedName); StringInfo command = makeStringInfo(); const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc); appendStringInfo(command, "COPY %s.%s (%s) FROM STDIN", quote_identifier(destinationShardSchemaName), quote_identifier( destinationShardRelationName), columnList); if (useBinaryFormat) { appendStringInfo(command, " WITH (format binary);"); } else { appendStringInfo(command, ";"); } return command; } /* Write Tuple to Local Shard. */ static void WriteLocalTuple(TupleTableSlot *slot, ShardCopyDestReceiver *copyDest) { CopyOutState localCopyOutState = copyDest->copyOutState; /* * Since we are doing a local copy, the following statements should * use local execution to see the changes */ SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED); bool isBinaryCopy = localCopyOutState->binary; bool shouldAddBinaryHeaders = (isBinaryCopy && localCopyOutState->fe_msgbuf->len == 0); if (shouldAddBinaryHeaders) { AppendCopyBinaryHeaders(localCopyOutState); } Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; FmgrInfo *columnOutputFunctions = copyDest->columnOutputFunctions; AppendCopyRowData(columnValues, columnNulls, copyDest->tupleDescriptor, localCopyOutState, columnOutputFunctions, NULL /* columnCoercionPaths */); } /* * LocalCopyToShard performs local copy for the given destination shard. */ static void LocalCopyToShard(ShardCopyDestReceiver *copyDest, CopyOutState localCopyOutState) { bool isBinaryCopy = localCopyOutState->binary; if (isBinaryCopy) { AppendCopyBinaryFooters(localCopyOutState); } /* * Set the buffer as a global variable to allow ReadFromLocalBufferCallback * to read from it. We cannot pass additional arguments to * ReadFromLocalBufferCallback. */ LocalCopyBuffer = localCopyOutState->fe_msgbuf; char *destinationShardSchemaName = linitial( copyDest->destinationShardFullyQualifiedName); char *destinationShardRelationName = lsecond( copyDest->destinationShardFullyQualifiedName); Oid destinationSchemaOid = get_namespace_oid(destinationShardSchemaName, false /* missing_ok */); Oid destinationShardOid = get_relname_relid(destinationShardRelationName, destinationSchemaOid); DefElem *binaryFormatOption = NULL; if (isBinaryCopy) { binaryFormatOption = makeDefElem("format", (Node *) makeString("binary"), -1); } Relation shard = table_open(destinationShardOid, RowExclusiveLock); ParseState *pState = make_parsestate(NULL /* parentParseState */); (void) addRangeTableEntryForRelation(pState, shard, AccessShareLock, NULL /* alias */, false /* inh */, false /* inFromCl */); List *options = (isBinaryCopy) ? list_make1(binaryFormatOption) : NULL; CopyFromState cstate = BeginCopyFrom(pState, shard, NULL /* whereClause */, NULL /* fileName */, false /* is_program */, ReadFromLocalBufferCallback, NULL /* attlist (NULL is all columns) */, options); CopyFrom(cstate); EndCopyFrom(cstate); resetStringInfo(localCopyOutState->fe_msgbuf); table_close(shard, NoLock); free_parsestate(pState); } /* * ReadFromLocalBufferCallback is the copy callback. * It always tries to copy maxRead bytes. */ static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead) { int bytesRead = 0; int avail = LocalCopyBuffer->len - LocalCopyBuffer->cursor; int bytesToRead = Min(avail, maxRead); if (bytesToRead > 0) { memcpy_s(outBuf, bytesToRead, &LocalCopyBuffer->data[LocalCopyBuffer->cursor], bytesToRead); } bytesRead += bytesToRead; LocalCopyBuffer->cursor += bytesToRead; return bytesRead; } ================================================ FILE: src/backend/distributed/operations/worker_split_copy_udf.c ================================================ /*------------------------------------------------------------------------- * * worker_split_copy_udf.c * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/distribution_column.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/multi_executor.h" #include "distributed/utils/array_type.h" #include "distributed/worker_shard_copy.h" PG_FUNCTION_INFO_V1(worker_split_copy); typedef struct SplitCopyInfo { uint64 destinationShardId; /* destination shard id */ Datum destinationShardMinHashValue; /* min hash value of destination shard */ Datum destinationShardMaxHashValue; /* max hash value of destination shard */ uint32_t destinationShardNodeId; /* node where split child shard is to be placed */ } SplitCopyInfo; static void ParseSplitCopyInfoDatum(Datum splitCopyInfoDatum, SplitCopyInfo **splitCopyInfo); static DestReceiver ** CreateShardCopyDestReceivers(EState *estate, ShardInterval * shardIntervalToSplitCopy, List *splitCopyInfoList); static DestReceiver * CreatePartitionedSplitCopyDestReceiver(EState *executor, ShardInterval * shardIntervalToSplitCopy, char *partitionColumnName, List *splitCopyInfoList); static void BuildMinMaxRangeArrays(List *splitCopyInfoList, ArrayType **minValueArray, ArrayType **maxValueArray); static char * TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName, char *sourceShardToCopyPrefix, char *sourceShardToCopyQualifiedName, List *splitCopyInfoList); /* * worker_split_copy(source_shard_id bigint, splitCopyInfo pg_catalog.split_copy_info[]) * UDF to split copy shard to list of destination shards. * 'source_shard_id' : Source ShardId to split copy. * 'splitCopyInfos' : Array of Split Copy Info (destination_shard's id, min/max ranges and node_id) */ Datum worker_split_copy(PG_FUNCTION_ARGS) { uint64 shardIdToSplitCopy = DatumGetUInt64(PG_GETARG_DATUM(0)); ShardInterval *shardIntervalToSplitCopy = LoadShardInterval(shardIdToSplitCopy); text *partitionColumnText = PG_GETARG_TEXT_P(1); char *partitionColumnName = text_to_cstring(partitionColumnText); ArrayType *splitCopyInfoArrayObject = PG_GETARG_ARRAYTYPE_P(2); bool arrayHasNull = ARR_HASNULL(splitCopyInfoArrayObject); if (arrayHasNull) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("pg_catalog.split_copy_info array " "cannot contain null values"))); } const int slice_ndim = 0; ArrayMetaState *mState = NULL; ArrayIterator copyInfo_iterator = array_create_iterator(splitCopyInfoArrayObject, slice_ndim, mState); Datum copyInfoDatum = 0; bool isnull = false; List *splitCopyInfoList = NIL; while (array_iterate(copyInfo_iterator, ©InfoDatum, &isnull)) { SplitCopyInfo *splitCopyInfo = NULL; ParseSplitCopyInfoDatum(copyInfoDatum, &splitCopyInfo); splitCopyInfoList = lappend(splitCopyInfoList, splitCopyInfo); } EState *executor = CreateExecutorState(); DestReceiver *splitCopyDestReceiver = CreatePartitionedSplitCopyDestReceiver(executor, shardIntervalToSplitCopy, partitionColumnName, splitCopyInfoList); Oid sourceShardToCopySchemaOId = get_rel_namespace( shardIntervalToSplitCopy->relationId); char *sourceShardToCopySchemaName = get_namespace_name(sourceShardToCopySchemaOId); char *sourceShardPrefix = get_rel_name(shardIntervalToSplitCopy->relationId); char *sourceShardToCopyName = pstrdup(sourceShardPrefix); AppendShardIdToName(&sourceShardToCopyName, shardIdToSplitCopy); char *sourceShardToCopyQualifiedName = quote_qualified_identifier( sourceShardToCopySchemaName, sourceShardToCopyName); ereport(LOG, (errmsg("%s", TraceWorkerSplitCopyUdf(sourceShardToCopySchemaName, sourceShardPrefix, sourceShardToCopyQualifiedName, splitCopyInfoList)))); StringInfo selectShardQueryForCopy = makeStringInfo(); const char *columnList = CopyableColumnNamesFromRelationName( sourceShardToCopySchemaName, sourceShardToCopyName); appendStringInfo(selectShardQueryForCopy, "SELECT %s FROM %s;", columnList, sourceShardToCopyQualifiedName); ParamListInfo params = NULL; ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params, (DestReceiver *) splitCopyDestReceiver); FreeExecutorState(executor); PG_RETURN_VOID(); } /* Trace split copy udf */ static char * TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName, char *sourceShardToCopyPrefix, char *sourceShardToCopyQualifiedName, List *splitCopyInfoList) { StringInfo splitCopyTrace = makeStringInfo(); appendStringInfo(splitCopyTrace, "performing copy from shard %s to [", sourceShardToCopyQualifiedName); /* split copy always has at least two destinations */ int index = 1; int splitWayCount = list_length(splitCopyInfoList); SplitCopyInfo *splitCopyInfo = NULL; foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { char *shardNameCopy = pstrdup(sourceShardToCopyPrefix); AppendShardIdToName(&shardNameCopy, splitCopyInfo->destinationShardId); char *shardNameCopyQualifiedName = quote_qualified_identifier( sourceShardToCopySchemaName, shardNameCopy); appendStringInfo(splitCopyTrace, "%s (nodeId: %u)", shardNameCopyQualifiedName, splitCopyInfo->destinationShardNodeId); pfree(shardNameCopy); if (index < splitWayCount) { appendStringInfo(splitCopyTrace, ", "); } index++; } appendStringInfo(splitCopyTrace, "]"); return splitCopyTrace->data; } /* Parse a single SplitCopyInfo Tuple */ static void ParseSplitCopyInfoDatum(Datum splitCopyInfoDatum, SplitCopyInfo **splitCopyInfo) { HeapTupleHeader dataTuple = DatumGetHeapTupleHeader(splitCopyInfoDatum); SplitCopyInfo *copyInfo = palloc0(sizeof(SplitCopyInfo)); bool isnull = false; Datum destinationShardIdDatum = GetAttributeByName(dataTuple, "destination_shard_id", &isnull); if (isnull) { ereport(ERROR, (errmsg( "destination_shard_id for pg_catalog.split_copy_info cannot be null."))); } copyInfo->destinationShardId = DatumGetUInt64(destinationShardIdDatum); Datum minValueDatum = GetAttributeByName(dataTuple, "destination_shard_min_value", &isnull); if (isnull) { ereport(ERROR, (errmsg( "destination_shard_min_value for pg_catalog.split_copy_info cannot be null."))); } copyInfo->destinationShardMinHashValue = minValueDatum; Datum maxValueDatum = GetAttributeByName(dataTuple, "destination_shard_max_value", &isnull); if (isnull) { ereport(ERROR, (errmsg( "destination_shard_max_value for pg_catalog.split_copy_info cannot be null."))); } copyInfo->destinationShardMaxHashValue = maxValueDatum; Datum nodeIdDatum = GetAttributeByName(dataTuple, "destination_shard_node_id", &isnull); if (isnull) { ereport(ERROR, (errmsg( "destination_shard_node_id for pg_catalog.split_copy_info cannot be null."))); } copyInfo->destinationShardNodeId = DatumGetInt32(nodeIdDatum); *splitCopyInfo = copyInfo; } /* Build 'min/max' hash range arrays for PartitionedResultDestReceiver */ static void BuildMinMaxRangeArrays(List *splitCopyInfoList, ArrayType **minValueArray, ArrayType **maxValueArray) { int partitionCount = list_length(splitCopyInfoList); Datum *minValues = palloc0(partitionCount * sizeof(Datum)); bool *minValueNulls = palloc0(partitionCount * sizeof(bool)); Datum *maxValues = palloc0(partitionCount * sizeof(Datum)); bool *maxValueNulls = palloc0(partitionCount * sizeof(bool)); SplitCopyInfo *splitCopyInfo = NULL; int index = 0; foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { minValues[index] = splitCopyInfo->destinationShardMinHashValue; maxValues[index] = splitCopyInfo->destinationShardMaxHashValue; /* Caller enforces that min/max values will be not-null */ minValueNulls[index] = false; maxValueNulls[index] = false; index++; } *minValueArray = CreateArrayFromDatums(minValues, minValueNulls, partitionCount, TEXTOID); *maxValueArray = CreateArrayFromDatums(maxValues, maxValueNulls, partitionCount, TEXTOID); } /* * Create underlying ShardCopyDestReceivers for PartitionedResultDestReceiver * Each ShardCopyDestReceivers will be responsible for copying tuples from source shard, * that fall under its min/max range, to specified destination shard. */ static DestReceiver ** CreateShardCopyDestReceivers(EState *estate, ShardInterval *shardIntervalToSplitCopy, List *splitCopyInfoList) { DestReceiver **shardCopyDests = palloc0(splitCopyInfoList->length * sizeof(DestReceiver *)); SplitCopyInfo *splitCopyInfo = NULL; int index = 0; char *sourceShardNamePrefix = get_rel_name(shardIntervalToSplitCopy->relationId); foreach_declared_ptr(splitCopyInfo, splitCopyInfoList) { Oid destinationShardSchemaOid = get_rel_namespace( shardIntervalToSplitCopy->relationId); char *destinationShardSchemaName = get_namespace_name(destinationShardSchemaOid); char *destinationShardNameCopy = pstrdup(sourceShardNamePrefix); AppendShardIdToName(&destinationShardNameCopy, splitCopyInfo->destinationShardId); DestReceiver *shardCopyDest = CreateShardCopyDestReceiver( estate, list_make2(destinationShardSchemaName, destinationShardNameCopy), splitCopyInfo->destinationShardNodeId); shardCopyDests[index] = shardCopyDest; index++; } return shardCopyDests; } /* Create PartitionedSplitCopyDestReceiver along with underlying ShardCopyDestReceivers */ static DestReceiver * CreatePartitionedSplitCopyDestReceiver(EState *estate, ShardInterval *shardIntervalToSplitCopy, char *partitionColumnName, List *splitCopyInfoList) { /* Create underlying ShardCopyDestReceivers */ DestReceiver **shardCopyDestReceivers = CreateShardCopyDestReceivers( estate, shardIntervalToSplitCopy, splitCopyInfoList); /* construct an artificial CitusTableCacheEntry for routing tuples to appropriate ShardCopyReceiver */ ArrayType *minValuesArray = NULL; ArrayType *maxValuesArray = NULL; BuildMinMaxRangeArrays(splitCopyInfoList, &minValuesArray, &maxValuesArray); /* we currently only support hash-distribution */ char partitionMethod = DISTRIBUTE_BY_HASH; /* synthetically build the partition column by looking at shard columns */ uint64 shardId = shardIntervalToSplitCopy->shardId; bool missingOK = false; Oid shardRelationId = LookupShardRelationFromCatalog(shardId, missingOK); Var *partitionColumn = BuildDistributionKeyFromColumnName(shardRelationId, partitionColumnName, AccessShareLock); CitusTableCacheEntry *shardSearchInfo = QueryTupleShardSearchInfo(minValuesArray, maxValuesArray, partitionMethod, partitionColumn); /* Construct PartitionedResultDestReceiver from cache and underlying ShardCopyDestReceivers */ int partitionColumnIndex = partitionColumn->varattno - 1; int partitionCount = splitCopyInfoList->length; DestReceiver *splitCopyDestReceiver = CreatePartitionedResultDestReceiver( partitionColumnIndex, partitionCount, shardSearchInfo, shardCopyDestReceivers, true /* lazyStartup */, false /* allowNullPartitionColumnValues */); return splitCopyDestReceiver; } ================================================ FILE: src/backend/distributed/operations/worker_split_shard_release_dsm_udf.c ================================================ /*------------------------------------------------------------------------- * * worker_split_shard_release_dsm.c * This file contains functions to release dynamic shared memory segment * allocated during split workflow. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_shared_memory.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(worker_split_shard_release_dsm); Datum worker_split_shard_release_dsm(PG_FUNCTION_ARGS) { ReleaseSharedMemoryOfShardSplitInfo(); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c ================================================ /*------------------------------------------------------------------------- * * worker_split_shard_replication_setup_udf.c * This file contains functions to setup information about list of shards * that are being split. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" #include "distributed/distribution_column.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/remote_commands.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_utils.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_logical_replication.h" #include "distributed/shardsplit_shared_memory.h" #include "distributed/tuplestore.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(worker_split_shard_replication_setup); static HTAB *ShardInfoHashMap = NULL; /* Function declarations */ static void ParseShardSplitInfoFromDatum(Datum shardSplitInfoDatum, uint64 *sourceShardId, char **partitionColumnName, uint64 *childShardId, int32 *minValue, int32 *maxValue, int32 *nodeId); static ShardSplitInfo * CreateShardSplitInfo(uint64 sourceShardIdToSplit, char *partitionColumnName, uint64 desSplitChildShardId, int32 minValue, int32 maxValue, int32 nodeId); static void AddShardSplitInfoEntryForNodeInMap(ShardSplitInfo *shardSplitInfo); static void PopulateShardSplitInfoInSM(ShardSplitInfoSMHeader *shardSplitInfoSMHeader, OperationId operationId); static void ReturnReplicationSlotInfo(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor, OperationId operationId); /* * worker_split_shard_replication_setup UDF creates in-memory data structures * to store the meta information about the shard undergoing split and new split * children along with their placements. This info is required during the catch up * phase of logical replication. * This meta information is stored in a shared memory segment and accessed * by logical decoding plugin. * * Split information is given by user as an Array of custom data type 'pg_catalog.split_shard_info'. * (worker_split_shard_replication_setup(pg_catalog.split_shard_info[])) * * Fields of custom data type 'pg_catalog.split_shard_info': * source_shard_id - id of the shard that is undergoing a split * * distribution_column - Distribution column name * * child_shard_id - id of shard that stores a specific range of values * belonging to sourceShardId(parent) * * shard_min_value - Lower bound(inclusive) of hash value which childShard stores * * shard_max_value - Upper bound(inclusive) of hash value which childShard stores * * node_id - Node where the childShardId is located * * The function parses the data and builds routing map with key for each distinct * pair. Multiple shards can be placed on the same destination node. * Source and destination nodes can be same too. * * There is a 1-1 mapping between a (table owner, node) and replication slot. One replication * slot takes care of replicating changes for all shards belonging to the same owner on a particular node. * * During the replication phase, WAL senders will attach to the shared memory * populated by current UDF. It routes the tuple from the source shard to the appropriate destination * shard for which the respective slot is responsible. */ Datum worker_split_shard_replication_setup(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { ereport(ERROR, (errmsg("split_shard_info array cannot be NULL"))); } ArrayType *shardInfoArrayObject = PG_GETARG_ARRAYTYPE_P(0); if (array_contains_nulls(shardInfoArrayObject)) { ereport(ERROR, (errmsg("Unexpectedly shard info array contains a null value"))); } OperationId operationId = DatumGetUInt64(PG_GETARG_DATUM(1)); /* SetupMap */ ShardInfoHashMap = CreateSimpleHash(NodeAndOwner, GroupedShardSplitInfos); int shardSplitInfoCount = 0; ArrayIterator shardInfo_iterator = array_create_iterator(shardInfoArrayObject, 0, NULL); Datum shardInfoDatum = 0; bool isnull = false; while (array_iterate(shardInfo_iterator, &shardInfoDatum, &isnull)) { uint64 sourceShardId = 0; char *partitionColumnName = NULL; uint64 childShardId = 0; int32 minValue = 0; int32 maxValue = 0; int32 nodeId = 0; ParseShardSplitInfoFromDatum(shardInfoDatum, &sourceShardId, &partitionColumnName, &childShardId, &minValue, &maxValue, &nodeId); ShardSplitInfo *shardSplitInfo = CreateShardSplitInfo( sourceShardId, partitionColumnName, childShardId, minValue, maxValue, nodeId); AddShardSplitInfoEntryForNodeInMap(shardSplitInfo); shardSplitInfoCount++; } dsm_handle dsmHandle; ShardSplitInfoSMHeader *splitShardInfoSMHeader = CreateSharedMemoryForShardSplitInfo(shardSplitInfoCount, &dsmHandle); PopulateShardSplitInfoInSM(splitShardInfoSMHeader, operationId); /* store handle in statically allocated shared memory*/ StoreShardSplitSharedMemoryHandle(dsmHandle); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); ReturnReplicationSlotInfo(tupleStore, tupleDescriptor, operationId); PG_RETURN_VOID(); } /* * CreateShardSplitInfo function constructs ShardSplitInfo data structure * with appropriate OIs' for source and destination relation. * * sourceShardIdToSplit - Existing shardId which has a valid entry in cache and catalogue * partitionColumnName - Name of column to use for partitioning * desSplitChildShardId - New split child shard which doesn't have an entry in metacache yet * minValue - Minimum hash value for desSplitChildShardId * maxValue - Maximum hash value for desSplitChildShardId * nodeId - NodeId where * However we can use shard ID and construct qualified shardName. */ ShardSplitInfo * CreateShardSplitInfo(uint64 sourceShardIdToSplit, char *partitionColumnName, uint64 desSplitChildShardId, int32 minValue, int32 maxValue, int32 nodeId) { ShardInterval *shardIntervalToSplit = LoadShardInterval(sourceShardIdToSplit); /* If metadata is not synced, we cannot proceed further as split work flow assumes * metadata to be synced on worker node hosting source shard to split. */ if (shardIntervalToSplit == NULL) { ereport(ERROR, errmsg( "Could not find metadata corresponding to source shard id: %ld. " "Split workflow assumes metadata to be synced across " "worker nodes hosting source shards.", sourceShardIdToSplit)); } /* Oid of distributed table */ Oid citusTableOid = shardIntervalToSplit->relationId; Oid sourceShardToSplitOid = GetTableLocalShardOid(citusTableOid, sourceShardIdToSplit); /* Oid of dummy table at the source */ Oid destSplitChildShardOid = GetTableLocalShardOid(citusTableOid, desSplitChildShardId); if (citusTableOid == InvalidOid || sourceShardToSplitOid == InvalidOid || destSplitChildShardOid == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Invalid citusTableOid:%u, " "sourceShardToSplitOid:%u, " "destSplitChildShardOid:%u ", citusTableOid, sourceShardToSplitOid, destSplitChildShardOid))); } /* determine the partition column in the tuple descriptor */ Var *partitionColumn = BuildDistributionKeyFromColumnName(sourceShardToSplitOid, partitionColumnName, AccessShareLock); if (partitionColumn == NULL) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Invalid Partition Column"))); } int partitionColumnIndex = partitionColumn->varattno - 1; ShardSplitInfo *shardSplitInfo = palloc0(sizeof(ShardSplitInfo)); shardSplitInfo->distributedTableOid = citusTableOid; shardSplitInfo->partitionColumnIndex = partitionColumnIndex; shardSplitInfo->sourceShardOid = sourceShardToSplitOid; shardSplitInfo->splitChildShardOid = destSplitChildShardOid; shardSplitInfo->shardMinValue = minValue; shardSplitInfo->shardMaxValue = maxValue; shardSplitInfo->nodeId = nodeId; shardSplitInfo->sourceShardId = sourceShardIdToSplit; shardSplitInfo->splitChildShardId = desSplitChildShardId; return shardSplitInfo; } /* * AddShardSplitInfoEntryForNodeInMap function adds ShardSplitInfo entry * to the hash map. The key is nodeId on which the new shard is to be placed. */ static void AddShardSplitInfoEntryForNodeInMap(ShardSplitInfo *shardSplitInfo) { NodeAndOwner key; key.nodeId = shardSplitInfo->nodeId; key.tableOwnerId = TableOwnerOid(shardSplitInfo->distributedTableOid); bool found = false; GroupedShardSplitInfos *groupedInfos = (GroupedShardSplitInfos *) hash_search(ShardInfoHashMap, &key, HASH_ENTER, &found); if (!found) { groupedInfos->shardSplitInfoList = NIL; } groupedInfos->shardSplitInfoList = lappend(groupedInfos->shardSplitInfoList, (ShardSplitInfo *) shardSplitInfo); } /* * PopulateShardSplitInfoInSM function copies information from the hash map * into shared memory segment. This information is consumed by the WAL sender * process during logical replication. * * shardSplitInfoSMHeader - Shared memory header */ static void PopulateShardSplitInfoInSM(ShardSplitInfoSMHeader *shardSplitInfoSMHeader, OperationId operationId) { HASH_SEQ_STATUS status; hash_seq_init(&status, ShardInfoHashMap); GroupedShardSplitInfos *entry = NULL; int splitInfoIndex = 0; while ((entry = (GroupedShardSplitInfos *) hash_seq_search(&status)) != NULL) { uint32_t nodeId = entry->key.nodeId; uint32_t tableOwnerId = entry->key.tableOwnerId; char *derivedSlotName = ReplicationSlotNameForNodeAndOwnerForOperation(SHARD_SPLIT, nodeId, tableOwnerId, operationId); List *shardSplitInfoList = entry->shardSplitInfoList; ShardSplitInfo *splitShardInfo = NULL; foreach_declared_ptr(splitShardInfo, shardSplitInfoList) { shardSplitInfoSMHeader->splitInfoArray[splitInfoIndex] = *splitShardInfo; strcpy_s(shardSplitInfoSMHeader->splitInfoArray[splitInfoIndex].slotName, NAMEDATALEN, derivedSlotName); splitInfoIndex++; } } } /* * ParseShardSplitInfoFromDatum deserializes individual fields of 'pg_catalog.split_shard_info' * datatype. */ static void ParseShardSplitInfoFromDatum(Datum shardSplitInfoDatum, uint64 *sourceShardId, char **partitionColumnName, uint64 *childShardId, int32 *minValue, int32 *maxValue, int32 *nodeId) { HeapTupleHeader dataTuple = DatumGetHeapTupleHeader(shardSplitInfoDatum); bool isnull = false; Datum sourceShardIdDatum = GetAttributeByName(dataTuple, "source_shard_id", &isnull); if (isnull) { ereport(ERROR, (errmsg("source_shard_id for split_shard_info can't be null"))); } *sourceShardId = DatumGetUInt64(sourceShardIdDatum); Datum partitionColumnDatum = GetAttributeByName(dataTuple, "distribution_column", &isnull); if (isnull) { ereport(ERROR, (errmsg( "distribution_column for split_shard_info can't be null"))); } *partitionColumnName = TextDatumGetCString(partitionColumnDatum); Datum childShardIdDatum = GetAttributeByName(dataTuple, "child_shard_id", &isnull); if (isnull) { ereport(ERROR, (errmsg("child_shard_id for split_shard_info can't be null"))); } *childShardId = DatumGetUInt64(childShardIdDatum); Datum minValueDatum = GetAttributeByName(dataTuple, "shard_min_value", &isnull); if (isnull) { ereport(ERROR, (errmsg("shard_min_value for split_shard_info can't be null"))); } char *shardMinValueString = text_to_cstring(DatumGetTextP(minValueDatum)); *minValue = SafeStringToInt32(shardMinValueString); Datum maxValueDatum = GetAttributeByName(dataTuple, "shard_max_value", &isnull); if (isnull) { ereport(ERROR, (errmsg("shard_max_value for split_shard_info can't be null"))); } char *shardMaxValueString = text_to_cstring(DatumGetTextP(maxValueDatum)); *maxValue = SafeStringToInt32(shardMaxValueString); Datum nodeIdDatum = GetAttributeByName(dataTuple, "node_id", &isnull); if (isnull) { ereport(ERROR, (errmsg("node_id for split_shard_info can't be null"))); } *nodeId = DatumGetInt32(nodeIdDatum); } /* * ReturnReplicationSlotInfo writes 'pg_catalog.replication_slot_info' * records to tuplestore. * This information is used by the coordinator to create replication slots as a * part of non-blocking split workflow. */ static void ReturnReplicationSlotInfo(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor, OperationId operationId) { HASH_SEQ_STATUS status; hash_seq_init(&status, ShardInfoHashMap); GroupedShardSplitInfos *entry = NULL; while ((entry = (GroupedShardSplitInfos *) hash_seq_search(&status)) != NULL) { Datum values[3]; bool nulls[3]; memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); values[0] = Int32GetDatum(entry->key.nodeId); char *tableOwnerName = GetUserNameFromId(entry->key.tableOwnerId, false); values[1] = CStringGetTextDatum(tableOwnerName); char *slotName = ReplicationSlotNameForNodeAndOwnerForOperation(SHARD_SPLIT, entry->key.nodeId, entry->key.tableOwnerId, operationId); values[2] = CStringGetTextDatum(slotName); tuplestore_putvalues(tupleStore, tupleDescriptor, values, nulls); } } ================================================ FILE: src/backend/distributed/planner/README.md ================================================ # Distributed Query Planner The distributed query planner is entered through the `distributed_planner` function in `distributed_planner.c`. This is the hook that Postgres calls instead of `standard_planner`. If the input query is trivial (e.g., no joins, no subqueries/ctes, single table and single shard), we create a very simple `PlannedStmt`. If the query is not trivial, call `standard_planner` to build a `PlannedStmt`. For queries containing a distributed table or reference table, we then proceed with distributed planning, which overwrites the `planTree` in the `PlannedStmt`. Distributed planning (`CreateDistributedPlan`) tries several different methods to plan the query: 1. Fast-path router planner, proceed if the query prunes down to a single shard of a single table 2. Router planner, proceed if the query prunes down to a single set of co-located shards 3. Modification planning, proceed if the query is a DML command and all joins are co-located 4. Recursive planning, find CTEs and subqueries that cannot be pushed down and go back to 1 5. Logical planner, constructs a multi-relational algebra tree to find a distributed execution plan ## Fast-path router planner By examining the query tree, if we can decide that the query hits only a single shard of a single table, we can skip calling `standard_planner()`. Later on the execution, we simply fetch the filter on the distribution key and do the pruning. As the name reveals, this can be considered as a sub-item of Router planner described below. The only difference is that fast-path planner doesn't rely on `standard_planner()` for collecting restriction information. ## Router planner During the call to `standard_planner`, Postgres calls a hook named `multi_relation_restriction_hook`. We use this hook to determine explicit and implicit filters on (occurrences of) distributed tables. We apply shard pruning to all tables using the filters in `PlanRouterQuery`. If all tables prune down to a single shard and all those shards are on the same node, then the query is router plannable meaning it can be fully executed by one of the worker nodes. The router planner preserves the original query tree, but the query does need to be rewritten to have shard names instead of table names. We cannot simply put the shard names into the query tree, because it actually contains relation OIDs. If the deparsing logic resolves those OIDs, it would throw an error since the shards do not exist on the coordinator. Instead, we replace the table entries with a fake SQL function call to `citus_extradata_container` and encode the original table ID and the shard ID into the parameter of the function. The deparsing functions recognise the fake function call and convert it into a shard name. ## Recursive planning CTEs and subqueries that cannot be pushed down (checked using `DeferErrorIfCannotPushdownSubquery`) and do not contain references to the outer query are planned by recursively calling the `planner` function with the subquery as the parse tree. Because the planner is called from the top, any type of query that is supported by Postgres or Citus can be a valid subquery. The resulting plan is added to the `subPlanList` in the `DistributedPlan` and the subquery is replaced by a call to `read_intermediate_result` with a particular subplan name. At execution time, all of the plans in the `subPlanList` are executed and their output is sent to all the workers and written into an intermediate result with the subplan name (see `subplan_execution.c`). In the remainder of the planner, calls to `read_intermediate_result` are treated in the same way as reference tables, which means they can be joined by any column. ## Logical planner The logical planner constructs a multi-relational algebra tree from the query with operators such as `MultiTable`, `MultiProject`, `MultiJoin` and `MultiCollect`. It first picks a strategy for handling joins in `MultiLogicalPlanCreate` (pushdown planning, or join order planning) and then builds a `MultiNode` tree based on the original query tree. In the initial `MultiNode` tree, each `MultiTable` is wrapped in `MultiCollect`, which effectively means collect the entire table in one place. The `MultiNode` tree is passed to the logical optimizer which transforms the tree into one that requires less network traffic by pushing down operators. Finally, the physical planner transforms the `MultiNode` tree into a `DistributedPlan` which contains the queries to execute on shards and can be passed to the executor. ### Pushdown planning During the call to `standard_planner`, Postgres calls a hook named `multi_relation_restriction_hook`. We use this hook to determine whether all (occurrences of) distributed tables are joined on their respective distribution columns. When this is the case, we can be somewhat agnostic to the structure of subqueries and other joins. In that case, we treat the whole join tree as a single `MultiTable` and deparse this part of the query as is during physical planning. Pushing down a subquery is only possible when the subquery can be answered without a merge step (checked using `DeferErrorIfCannotPushdownSubquery`). However, you may notice that these subqueries are already replaced by `read_intermediate_result` calls during recursive planning. Only subqueries that have references to the outer query remain at this stage would pass through recursive planning and fail the check. ### Join order planning The join order planner is applied to the join tree in the original query and generates all possible pair-wise join orders. If a pair-wise join is not on the distribution column, it requires re-partitioning. The join order that requires the fewest re-partition steps is converted into a tree of `MultiJoin` nodes, prior to running the logical optimizer. The optimizer leaves the `MultiJoin` tree in tact, but pushes down select and collect operators below the `MultiJoin` nodes. ### Logical optimizer The logical optimizer uses commutativity rules to push project and select operators down below the `MultiCollect` nodes. Everything above the `MultiCollect` operator will be is executed on the coordinator and everything below on the workers. Additionally, the optimizer uses distributivity rules to push down operators below the `MultiJoin` nodes, such that filters and projections are applied prior to joins. This is primarily relevant for re-partition joins which first try to reduce the data by applying selections and projections, and then re-partitioning the result. A number of SQL clauses like aggregates, GROUP BY, ORDER BY, LIMIT can only be pushed down below the `MultiCollect` under certain conditions. All these clauses are bundled together in a `MultiExtendedOpNode`. After the basic transformation, the `MultiExtendedOpNode`s are directly above the `MultiCollect` nodes. They are then split into a coordinator and a worker part and the worker part is pushed down below the `MultiCollect`. ### Physical planner This section needs to be expanded. ## Modification planning In terms of modification planning, we distinguish between several cases: 1. DML planning (`CreateModifyPlan`) 1.a. UPDATE/DELETE planning 1.b. INSERT planning 2. INSERT...SELECT planning (`CreateInsertSelectPlan`) ### UPDATE/DELETE planning UPDATE and DELETE commands are handled by the router planner (`PlanRouterQuery`), but when tables prune to multiple shards we do not fall back to other planners, but instead proceed to generate a task for each shard, as long as all subqueries can be pushed down. We can do this because UPDATE and DELETE never have a meaningful merge step on the coordinator, other than concatening RETURNING rows. When there are CTEs or subqueries that cannot be pushed down in the UPDATE/DELETE, we continue with recursive planning and try again. If recursive planning cannot resolve the issue (e.g. due to a correlated subquery), then the command will error out. ### INSERT planning Distributed planning for INSERT commands is relatively complicated because of multi-row INSERT. Each row in a multi-row INSERT may go to different shard and therefore we need to construct a different query for each shard. The logic for this is primarily in `BuildRoutesForInsert`, which builds the set of rows for each shard. Rather than construct a full query tree for each shard, we put each set of rows in the `rowValuesLists` of the `Task` and replace the VALUES section of the INSERT just before deparsing in `UpdateTaskQueryString`. One additional complication for INSERTs is that it is very common to have a function call (such as `now()` or `nextval(..)`) in the position of the distribution column. In that case building the task list is deferred until the functions have been evaluated in the executor. ### INSERT ... SELECT query planning Citus supports `INSERT ... SELECT` queries either by pushing down the whole query to the worker nodes or pulling the `SELECT` part to the coordinator. #### INSERT ... SELECT - by pushing down If `INSERT ... SELECT` query can be planned by pushing down it to the worker nodes, Citus selects to choose that logic first. Query is planned separately for each shard in the target table. Do so by replacing the partitioning qual parameter using the shard's actual boundary values to create modify task for each shard. Then, shard pruning is performed to decide on to which shards query will be pushed down. Finally, checks if the target shardInterval has exactly same placements with the select task's available anchor placements. #### INSERT...SELECT - via the coordinator If the query can not be pushed down to the worker nodes, two different approaches can be followed depending on whether ON CONFLICT or RETURNING clauses are used. * If `ON CONFLICT` or `RETURNING` are not used, Citus uses `COPY` command to handle such queries. After planning the `SELECT` part of the `INSERT ... SELECT` query, including subqueries and CTEs, it executes the plan and send results back to the DestReceiver which is created using the target table info. * Since `COPY` command supports neither `ON CONFLICT` nor `RETURNING` clauses, Citus perform `INSERT ... SELECT` queries with `ON CONFLICT` or `RETURNING` clause in two phases. First, Citus plans the `SELECT` part of the query, executes the plan and saves results to the intermediate table which is colocated with target table of the `INSERT ... SELECT` query. Then, `INSERT ... SELECT` query is directly run on the worker node using the intermediate table as the source table. ================================================ FILE: src/backend/distributed/planner/combine_query_planner.c ================================================ /*------------------------------------------------------------------------- * * combine_query_planner.c * Routines for planning the combine query that runs on the coordinator * to combine results from the workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/planner.h" #include "rewrite/rewriteManip.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/combine_query_planner.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" static List * RemoteScanTargetList(List *workerTargetList); static PlannedStmt * BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetList, CustomScan *remoteScan); static Plan * CitusCustomScanPathPlan(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *best_path, List *tlist, List *clauses, List *custom_plans); bool ReplaceCitusExtraDataContainer = false; CustomScan *ReplaceCitusExtraDataContainerWithCustomScan = NULL; /* * CitusCustomScanPathMethods defines the methods for a custom path we insert into the * planner during the planning of the query part that will be executed on the node * coordinating the query. */ static CustomPathMethods CitusCustomScanPathMethods = { .CustomName = "CitusCustomScanPath", .PlanCustomPath = CitusCustomScanPathPlan, }; /* * PlanCombineQuery takes in a distributed plan and a custom scan node which * wraps remote part of the plan. This function finds the combine query structure * in the multi plan, and builds the final select plan to execute on the tuples * returned by remote scan on the coordinator node. Note that this select * plan is executed after result files are retrieved from worker nodes and * filled into the tuple store inside provided custom scan. */ PlannedStmt * PlanCombineQuery(DistributedPlan *distributedPlan, CustomScan *remoteScan) { Query *combineQuery = distributedPlan->combineQuery; Job *workerJob = distributedPlan->workerJob; List *workerTargetList = workerJob->jobQuery->targetList; List *remoteScanTargetList = RemoteScanTargetList(workerTargetList); return BuildSelectStatementViaStdPlanner(combineQuery, remoteScanTargetList, remoteScan); } /* * RemoteScanTargetList uses the given worker target list's expressions, and creates * a target list for the remote scan on the coordinator node. */ static List * RemoteScanTargetList(List *workerTargetList) { List *remoteScanTargetList = NIL; const Index tableId = 1; AttrNumber columnId = 1; ListCell *workerTargetCell = NULL; foreach(workerTargetCell, workerTargetList) { TargetEntry *workerTargetEntry = (TargetEntry *) lfirst(workerTargetCell); if (workerTargetEntry->resjunk) { continue; } Var *remoteScanColumn = makeVarFromTargetEntry(tableId, workerTargetEntry); remoteScanColumn->varattno = columnId; remoteScanColumn->varattnosyn = columnId; columnId++; if (remoteScanColumn->vartype == RECORDOID || remoteScanColumn->vartype == RECORDARRAYOID) { remoteScanColumn->vartypmod = BlessRecordExpression(workerTargetEntry->expr); } /* * The remote scan target entry has two pieces to it. The first piece is the * target entry's expression, which we set to the newly created column. * The second piece is sort and group clauses that we implicitly copy * from the worker target entry. Note that any changes to worker target * entry's sort and group clauses will *break* us here. */ TargetEntry *remoteScanTargetEntry = flatCopyTargetEntry(workerTargetEntry); remoteScanTargetEntry->expr = (Expr *) remoteScanColumn; remoteScanTargetList = lappend(remoteScanTargetList, remoteScanTargetEntry); } return remoteScanTargetList; } /* * CreateCitusCustomScanPath creates a custom path node that will return the CustomScan if * the path ends up in the best_path during postgres planning. We use this function during * the set relation hook of postgres during the planning of the query part that will be * executed on the query coordinating node. */ Path * CreateCitusCustomScanPath(PlannerInfo *root, RelOptInfo *relOptInfo, Index restrictionIndex, RangeTblEntry *rte, CustomScan *remoteScan) { CitusCustomScanPath *path = (CitusCustomScanPath *) newNode( sizeof(CitusCustomScanPath), T_CustomPath); path->custom_path.methods = &CitusCustomScanPathMethods; path->custom_path.path.pathtype = T_CustomScan; path->custom_path.path.pathtarget = relOptInfo->reltarget; path->custom_path.path.parent = relOptInfo; /* necessary to avoid extra Result node in PG15 */ path->custom_path.flags = CUSTOMPATH_SUPPORT_PROJECTION; /* * The 100k rows we put on the cost of the path is kind of arbitrary and could be * improved in accuracy to produce better plans. * * 100k on the row estimate causes the postgres planner to behave very much like the * old citus planner in the plans it produces. Namely the old planner had hardcoded * the use of Hash Aggregates for most of the operations, unless a postgres guc was * set that would disallow hash aggregates to be used. * * Ideally we would be able to provide estimates close to postgres' estimates on the * workers to let the standard planner choose an optimal solution for the combineQuery. */ path->custom_path.path.rows = 100000; path->remoteScan = remoteScan; return (Path *) path; } /* * CitusCustomScanPathPlan is called for the CitusCustomScanPath node in the best_path * after the postgres planner has evaluated all possible paths. * * This function returns a Plan node, more specifically the CustomScan Plan node that has * the ability to execute the distributed part of the query. * * When this function is called there is an extra list of clauses passed in that might not * already have been applied to the plan. We add these clauses to the quals this node will * execute. The quals are evaluated before returning the tuples scanned from the workers * to the plan above ours to make sure they do not end up in the final result. */ static Plan * CitusCustomScanPathPlan(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) { CitusCustomScanPath *citusPath = (CitusCustomScanPath *) best_path; /* * Columns could have been pruned from the target list by the standard planner. * A situation in which this might happen is a CASE that is proven to be always the * same causing the other column to become useless; * CASE WHEN ... <> NULL * THEN ... * ELSE ... * END * Since nothing is equal to NULL it will always end up in the else branch. The final * target list the planenr needs from our node is passed in as tlist. By placing that * as the target list on our scan the internal rows will be projected to this one. */ citusPath->remoteScan->scan.plan.targetlist = tlist; /* * The custom_scan_tlist contains target entries for to the "output" of the call * to citus_extradata_container, which is actually replaced by a CustomScan. * The target entries are initialized with varno 1 (see RemoteScanTargetList), since * it's currently the only relation in the join tree of the combineQuery. * * If the citus_extradata_container function call is not the first relation to * appear in the flattened rtable for the entire plan, then varno is now pointing * to the wrong relation and needs to be updated. * * Example: * When the combineQuery field of the DistributedPlan is * INSERT INTO local SELECT .. FROM citus_extradata_container. * In that case the varno of citusdata_extradata_container should be 3, because * it is preceded range table entries for "local" and the subquery. */ if (rel->relid != 1) { TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, citusPath->remoteScan->custom_scan_tlist) { /* we created this list, so we know it only contains Var */ Assert(IsA(targetEntry->expr, Var)); Var *var = (Var *) targetEntry->expr; var->varno = rel->relid; } } /* clauses might have been added by the planner, need to add them to our scan */ RestrictInfo *restrictInfo = NULL; List **quals = &citusPath->remoteScan->scan.plan.qual; foreach_declared_ptr(restrictInfo, clauses) { *quals = lappend(*quals, restrictInfo->clause); } return (Plan *) citusPath->remoteScan; } /* * BuildSelectStatementViaStdPlanner creates a PlannedStmt where it combines the * combineQuery and the remoteScan. It utilizes the standard_planner from postgres to * create a plan based on the combineQuery. */ static PlannedStmt * BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetList, CustomScan *remoteScan) { /* * the standard planner will scribble on the target list. Since it is essential to not * change the custom_scan_tlist we copy the target list before adding them to any. * The remoteScanTargetList is used in the end to extract the column names to be added to * the alias we will create for the CustomScan, (expressed as the * citus_extradata_container function call in the combineQuery). */ remoteScan->custom_scan_tlist = copyObject(remoteScanTargetList); remoteScan->scan.plan.targetlist = copyObject(remoteScanTargetList); /* * We will overwrite the alias of the rangetable which describes the custom scan. * Ideally we would have set the correct column names and alias on the range table in * the combine query already when we inserted the extra data container. This could be * improved in the future. */ /* find the rangetable entry for the extradata container and overwrite its alias */ RangeTblEntry *extradataContainerRTE = NULL; FindCitusExtradataContainerRTE((Node *) combineQuery, &extradataContainerRTE); if (extradataContainerRTE != NULL) { /* extract column names from the remoteScanTargetList */ List *columnNameList = NIL; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, remoteScanTargetList) { columnNameList = lappend(columnNameList, makeString(targetEntry->resname)); } extradataContainerRTE->eref = makeAlias("remote_scan", columnNameList); } /* * Print the combine query at debug level 4. Since serializing the query is relatively * cpu intensive we only perform that if we are actually logging DEBUG4. */ const int logCombineQueryLevel = DEBUG4; if (IsLoggableLevel(logCombineQueryLevel)) { StringInfo queryString = makeStringInfo(); pg_get_query_def(combineQuery, queryString); elog(logCombineQueryLevel, "combine query: %s", queryString->data); } PlannedStmt *standardStmt = NULL; PG_TRY(); { /* This code should not be re-entrant, we check via asserts below */ Assert(ReplaceCitusExtraDataContainer == false); Assert(ReplaceCitusExtraDataContainerWithCustomScan == NULL); ReplaceCitusExtraDataContainer = true; ReplaceCitusExtraDataContainerWithCustomScan = remoteScan; standardStmt = standard_planner(combineQuery, NULL, 0, NULL); ReplaceCitusExtraDataContainer = false; ReplaceCitusExtraDataContainerWithCustomScan = NULL; } PG_CATCH(); { ReplaceCitusExtraDataContainer = false; ReplaceCitusExtraDataContainerWithCustomScan = NULL; PG_RE_THROW(); } PG_END_TRY(); Assert(standardStmt != NULL); return standardStmt; } /* * Finds the rangetable entry in the query that refers to the citus_extradata_container * and stores the pointer in result. */ bool FindCitusExtradataContainerRTE(Node *node, RangeTblEntry **result) { if (node == NULL) { return false; } if (IsA(node, RangeTblEntry)) { RangeTblEntry *rangeTblEntry = castNode(RangeTblEntry, node); if (rangeTblEntry->rtekind == RTE_FUNCTION && list_length(rangeTblEntry->functions) == 1) { RangeTblFunction *rangeTblFunction = (RangeTblFunction *) linitial( rangeTblEntry->functions); if (!IsA(rangeTblFunction->funcexpr, FuncExpr)) { return false; } FuncExpr *funcExpr = castNode(FuncExpr, rangeTblFunction->funcexpr); if (funcExpr->funcid == CitusExtraDataContainerFuncId()) { *result = rangeTblEntry; return true; } } /* query_tree_walker descends into RTEs */ return false; } else if (IsA(node, Query)) { const int flags = QTW_EXAMINE_RTES_BEFORE; return query_tree_walker((Query *) node, FindCitusExtradataContainerRTE, result, flags); } return expression_tree_walker(node, FindCitusExtradataContainerRTE, result); } ================================================ FILE: src/backend/distributed/planner/cte_inline.c ================================================ /*------------------------------------------------------------------------- * * cte_inline.c * For multi-shard queries, Citus can only recursively plan CTEs. Instead, * with the functions defined in this file, the certain CTEs can be inlined * as subqueries in the query tree. In that case, more optimal distributed * planning, the query pushdown planning, kicks in and the CTEs can actually * be pushed down as long as it is safe to pushdown as a subquery. * * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "rewrite/rewriteManip.h" #include "pg_version_compat.h" #include "pg_version_constants.h" #include "distributed/cte_inline.h" typedef struct inline_cte_walker_context { const char *ctename; /* name and relative level of target CTE */ int levelsup; int refcount; /* number of remaining references */ Query *ctequery; /* query to substitute */ List *aliascolnames; /* citus addition to Postgres' inline_cte_walker_context */ } inline_cte_walker_context; /* copy & paste from Postgres source, moved into a function for readability */ static bool PostgreSQLCTEInlineCondition(CommonTableExpr *cte, CmdType cmdType); /* the following utility functions are copy & paste from PostgreSQL code */ static void inline_cte(Query *mainQuery, CommonTableExpr *cte); static bool inline_cte_walker(Node *node, inline_cte_walker_context *context); static bool contain_dml(Node *node); static bool contain_dml_walker(Node *node, void *context); /* the following utility functions are related to Citus' logic */ static bool RecursivelyInlineCteWalker(Node *node, void *context); static void InlineCTEsInQueryTree(Query *query); static bool QueryTreeContainsInlinableCteWalker(Node *node, void *context); /* * RecursivelyInlineCtesInQueryTree gets a query and recursively traverses the * tree from top to bottom. On each level, the CTEs that are eligable for * inlining are inlined as subqueries. This is useful in distributed planning * because Citus' sub(query) planning logic superior to CTE planning, where CTEs * are always recursively planned, which might produce very slow executions. */ void RecursivelyInlineCtesInQueryTree(Query *query) { InlineCTEsInQueryTree(query); query_tree_walker(query, RecursivelyInlineCteWalker, NULL, 0); } /* * RecursivelyInlineCteWalker recursively finds all the Query nodes and * recursively inline eligable ctes. */ static bool RecursivelyInlineCteWalker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; InlineCTEsInQueryTree(query); query_tree_walker(query, RecursivelyInlineCteWalker, NULL, 0); /* we're done, no need to recurse anymore for this query */ return false; } return expression_tree_walker(node, RecursivelyInlineCteWalker, context); } /* * InlineCTEsInQueryTree gets a query tree and tries to inline CTEs as subqueries * in the query tree. * * Most of the code is coming from PostgreSQL's CTE inlining logic, there are very * few additions that Citus added, which are already commented in the code. */ void InlineCTEsInQueryTree(Query *query) { ListCell *cteCell = NULL; /* iterate on the copy of the list because we'll be modifying query->cteList */ List *copyOfCteList = list_copy(query->cteList); foreach(cteCell, copyOfCteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(cteCell); /* * First, make sure that Postgres is OK to inline the CTE. Later, check for * distributed query planning constraints that might prevent inlining. */ if (PostgreSQLCTEInlineCondition(cte, query->commandType)) { elog(DEBUG1, "CTE %s is going to be inlined via " "distributed planning", cte->ctename); /* do the hard work of cte inlining */ inline_cte(query, cte); /* clean-up the necessary fields for distributed planning */ cte->cterefcount = 0; query->cteList = list_delete_ptr(query->cteList, cte); } } } /* * QueryTreeContainsInlinableCTE recursively traverses the queryTree, and returns true * if any of the (sub)queries in the queryTree contains at least one CTE. */ bool QueryTreeContainsInlinableCTE(Query *queryTree) { return QueryTreeContainsInlinableCteWalker((Node *) queryTree, NULL); } /* * QueryTreeContainsInlinableCteWalker walks over the node, and returns true if any of * the (sub)queries in the node contains at least one CTE. */ static bool QueryTreeContainsInlinableCteWalker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; ListCell *cteCell = NULL; foreach(cteCell, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(cteCell); if (PostgreSQLCTEInlineCondition(cte, query->commandType)) { /* * Return true even if we can find a single CTE that is * eligable for inlining. */ return true; } } return query_tree_walker(query, QueryTreeContainsInlinableCteWalker, NULL, 0); } return expression_tree_walker(node, QueryTreeContainsInlinableCteWalker, NULL); } /* * PostgreSQLCTEInlineCondition returns true if the CTE is considered * safe to inline by Postgres. */ static bool PostgreSQLCTEInlineCondition(CommonTableExpr *cte, CmdType cmdType) { /* * Consider inlining the CTE (creating RTE_SUBQUERY RTE(s)) instead of * implementing it as a separately-planned CTE. * * We cannot inline if any of these conditions hold: * * 1. The user said not to (the CTEMaterializeAlways option). * * 2. The CTE is recursive. * * 3. The CTE has side-effects; this includes either not being a plain * SELECT, or containing volatile functions. Inlining might change * the side-effects, which would be bad. * * Otherwise, we have an option whether to inline or not. That should * always be a win if there's just a single reference, but if the CTE * is multiply-referenced then it's unclear: inlining adds duplicate * computations, but the ability to absorb restrictions from the outer * query level could outweigh that. We do not have nearly enough * information at this point to tell whether that's true, so we let * the user express a preference. Our default behavior is to inline * only singly-referenced CTEs, but a CTE marked CTEMaterializeNever * will be inlined even if multiply referenced. */ if ( (cte->ctematerialized == CTEMaterializeNever || (cte->ctematerialized == CTEMaterializeDefault && cte->cterefcount == 1)) && !cte->cterecursive && cmdType == CMD_SELECT && !contain_dml(cte->ctequery) && !contain_volatile_functions(cte->ctequery)) { return true; } return false; } /* *INDENT-OFF* */ /* * inline_cte: convert RTE_CTE references to given CTE into RTE_SUBQUERYs */ static void inline_cte(Query *mainQuery, CommonTableExpr *cte) { struct inline_cte_walker_context context; context.ctename = cte->ctename; /* Start at levelsup = -1 because we'll immediately increment it */ context.levelsup = -1; context.refcount = cte->cterefcount; context.ctequery = castNode(Query, cte->ctequery); context.aliascolnames = cte->aliascolnames; (void) inline_cte_walker((Node *) mainQuery, &context); /* Assert we replaced all references */ Assert(context.refcount == 0); } /* * See PostgreSQL's source code at src/backend/optimizer/plan/subselect.c. */ static bool inline_cte_walker(Node *node, inline_cte_walker_context *context) { if (node == NULL) return false; if (IsA(node, Query)) { Query *query = (Query *) node; context->levelsup++; (void) query_tree_walker(query, inline_cte_walker, context, QTW_EXAMINE_RTES_AFTER); context->levelsup--; return false; } else if (IsA(node, RangeTblEntry)) { RangeTblEntry *rte = (RangeTblEntry *) node; if (rte->rtekind == RTE_CTE && strcmp(rte->ctename, context->ctename) == 0 && rte->ctelevelsup == context->levelsup) { /* * Found a reference to replace. Generate a copy of the CTE query * with appropriate level adjustment for outer references (e.g., * to other CTEs). */ Query *newquery = copyObject(context->ctequery); if (context->levelsup > 0) IncrementVarSublevelsUp((Node *) newquery, context->levelsup, 1); /* * Convert the RTE_CTE RTE into a RTE_SUBQUERY. * * Historically, a FOR UPDATE clause has been treated as extending * into views and subqueries, but not into CTEs. We preserve this * distinction by not trying to push rowmarks into the new * subquery. */ rte->rtekind = RTE_SUBQUERY; rte->subquery = newquery; rte->security_barrier = false; List *columnAliasList = context->aliascolnames; int columnAliasCount = list_length(columnAliasList); int columnIndex = 1; for (; columnIndex < list_length(rte->subquery->targetList) + 1; ++columnIndex) { /* * Rename the column only if a column alias is defined. * Notice that column alias count could be less than actual * column count. We only use provided aliases and keep the * original column names if no alias is defined. */ if (columnAliasCount >= columnIndex) { String *columnAlias = (String *) list_nth(columnAliasList, columnIndex - 1); Assert(IsA(columnAlias, String)); TargetEntry *targetEntry = list_nth(rte->subquery->targetList, columnIndex - 1); Assert(IsA(columnAlias, String)); targetEntry->resname = strVal(columnAlias); } } /* Zero out CTE-specific fields */ rte->ctename = NULL; rte->ctelevelsup = 0; rte->self_reference = false; rte->coltypes = NIL; rte->coltypmods = NIL; rte->colcollations = NIL; /* Count the number of replacements we've done */ context->refcount--; } return false; } return expression_tree_walker(node, inline_cte_walker, context); } /* * contain_dml: is any subquery not a plain SELECT? * * We reject SELECT FOR UPDATE/SHARE as well as INSERT etc. */ static bool contain_dml(Node *node) { return contain_dml_walker(node, NULL); } static bool contain_dml_walker(Node *node, void *context) { if (node == NULL) return false; if (IsA(node, Query)) { Query *query = (Query *) node; if (query->commandType != CMD_SELECT || query->rowMarks != NIL) return true; return query_tree_walker(query, contain_dml_walker, context, 0); } return expression_tree_walker(node, contain_dml_walker, context); } /* *INDENT-ON* */ ================================================ FILE: src/backend/distributed/planner/deparse_shard_query.c ================================================ /*------------------------------------------------------------------------- * * deparse_shard_query.c * * This file contains functions for deparsing shard queries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "access/heapam.h" #include "access/htup_details.h" #include "catalog/pg_constraint.h" #include "catalog/pg_namespace.h" #include "catalog/pg_operator.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "parser/parsetree.h" #include "storage/lock.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_ruleutils.h" #include "distributed/combine_query_planner.h" #include "distributed/deparse_shard_query.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/shard_utils.h" #include "distributed/stats/stat_tenants.h" #include "distributed/version_compat.h" static void UpdateTaskQueryString(Query *query, Task *task); static RelationShard * FindRelationShard(Oid inputRelationId, List *relationShardList); static void ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte); static bool ShouldLazyDeparseQuery(Task *task); static char * DeparseTaskQuery(Task *task, Query *query); /* * RebuildQueryStrings deparses the job query for each task to * include execution-time changes such as function evaluation. */ void RebuildQueryStrings(Job *workerJob) { Query *originalQuery = workerJob->jobQuery; List *taskList = workerJob->taskList; Task *task = NULL; bool isSingleTask = list_length(taskList) == 1; if (originalQuery->commandType == CMD_INSERT) { AddInsertAliasIfNeeded(originalQuery); } foreach_declared_ptr(task, taskList) { Query *query = originalQuery; /* * Copy the query if there are multiple tasks. If there is a single * task, we scribble on the original query to avoid the copying * overhead. */ if (!isSingleTask) { query = copyObject(originalQuery); } if (UpdateOrDeleteOrMergeQuery(query)) { List *relationShardList = task->relationShardList; /* * For UPDATE and DELETE queries, we may have subqueries and joins, so * we use relation shard list to update shard names and call * pg_get_query_def() directly. */ UpdateRelationToShardNames((Node *) query, relationShardList); } else if (query->commandType == CMD_INSERT && task->modifyWithSubquery) { /* for INSERT..SELECT, adjust shard names in SELECT part */ List *relationShardList = task->relationShardList; ShardInterval *shardInterval = LoadShardInterval(task->anchorShardId); RangeTblEntry *copiedInsertRte = ExtractResultRelationRTEOrError(query); RangeTblEntry *copiedSubqueryRte = ExtractSelectRangeTableEntry(query); Query *copiedSubquery = copiedSubqueryRte->subquery; /* there are no restrictions to add for reference and citus local tables */ if (IsCitusTableType(shardInterval->relationId, DISTRIBUTED_TABLE)) { AddPartitionKeyNotNullFilterToSelect(copiedSubquery); } ReorderInsertSelectTargetLists(query, copiedInsertRte, copiedSubqueryRte); UpdateRelationToShardNames((Node *) copiedSubquery, relationShardList); } if (query->commandType == CMD_INSERT) { RangeTblEntry *modifiedRelationRTE = linitial(originalQuery->rtable); /* * We store the modified relaiton ID in the task so we can lazily call * deparse_shard_query when the string is needed */ task->anchorDistributedTableId = modifiedRelationRTE->relid; /* * For multi-row inserts, we modify the VALUES before storing the * query in the task. */ RangeTblEntry *valuesRTE = ExtractDistributedInsertValuesRTE(query); if (valuesRTE != NULL) { Assert(valuesRTE->rtekind == RTE_VALUES); Assert(task->rowValuesLists != NULL); valuesRTE->values_lists = task->rowValuesLists; } } bool isQueryObjectOrText = GetTaskQueryType(task) == TASK_QUERY_TEXT || GetTaskQueryType(task) == TASK_QUERY_OBJECT; ereport(DEBUG4, (errmsg("query before rebuilding: %s", !isQueryObjectOrText ? "(null)" : TaskQueryString(task)))); task->partitionKeyValue = workerJob->partitionKeyValue; SetJobColocationId(workerJob); task->colocationId = workerJob->colocationId; UpdateTaskQueryString(query, task); /* * If parameters were resolved in the job query, then they are now also * resolved in the query string. */ task->parametersInQueryStringResolved = workerJob->parametersInJobQueryResolved; ereport(DEBUG4, (errmsg("query after rebuilding: %s", TaskQueryString(task)))); } } /* * AddInsertAliasIfNeeded adds an alias in UPSERTs and multi-row INSERTs to avoid * deparsing issues (e.g. RETURNING might reference the original table name, * which has been replaced by a shard name). */ void AddInsertAliasIfNeeded(Query *query) { Assert(query->commandType == CMD_INSERT); if (query->onConflict == NULL && ExtractDistributedInsertValuesRTE(query) == NULL) { /* simple single-row insert does not need an alias */ return; } RangeTblEntry *rangeTableEntry = linitial(query->rtable); if (rangeTableEntry->alias != NULL) { /* INSERT already has an alias */ return; } Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); rangeTableEntry->alias = alias; } /* * UpdateTaskQueryString updates the query string stored within the provided * Task. If the Task has row values from a multi-row INSERT, those are injected * into the provided query before deparse occurs (the query's full VALUES list * will be restored before this function returns). */ static void UpdateTaskQueryString(Query *query, Task *task) { SetTaskQueryIfShouldLazyDeparse(task, query); } /* * CreateQualsForShardInterval creates the necessary qual conditions over the * given attnum and rtindex for the given shard interval. */ Node * CreateQualsForShardInterval(RelationShard *relationShard, int attnum, int rtindex) { uint64 shardId = relationShard->shardId; Oid relationId = relationShard->relationId; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); Var *partitionColumnVar = cacheEntry->partitionColumn; /* * Add constraints for the relation identified by rtindex, specifically on its column at attnum. * Create a Var node representing this column, which will be used to compare against the bounds * from the partition column of shard interval. */ Var *outerTablePartitionColumnVar = makeVar( rtindex, attnum, partitionColumnVar->vartype, partitionColumnVar->vartypmod, partitionColumnVar->varcollid, 0); bool isFirstShard = IsFirstShard(cacheEntry, shardId); /* load the interval for the shard and create constant nodes for the upper/lower bounds */ ShardInterval *shardInterval = LoadShardInterval(shardId); Const *constNodeLowerBound = makeConst(INT4OID, -1, InvalidOid, sizeof(int32), shardInterval->minValue, false, true); Const *constNodeUpperBound = makeConst(INT4OID, -1, InvalidOid, sizeof(int32), shardInterval->maxValue, false, true); Const *constNodeZero = makeConst(INT4OID, -1, InvalidOid, sizeof(int32), Int32GetDatum(0), false, true); /* create a function expression node for the hash partition column */ FuncExpr *hashFunction = makeNode(FuncExpr); hashFunction->funcid = cacheEntry->hashFunction->fn_oid; hashFunction->args = list_make1(outerTablePartitionColumnVar); hashFunction->funcresulttype = get_func_rettype(cacheEntry->hashFunction->fn_oid); hashFunction->funcretset = false; /* create a function expression for the lower bound of the shard interval */ Oid resultTypeOid = get_func_rettype( cacheEntry->shardIntervalCompareFunction->fn_oid); FuncExpr *lowerBoundFuncExpr = makeNode(FuncExpr); lowerBoundFuncExpr->funcid = cacheEntry->shardIntervalCompareFunction->fn_oid; lowerBoundFuncExpr->args = list_make2((Node *) constNodeLowerBound, (Node *) hashFunction); lowerBoundFuncExpr->funcresulttype = resultTypeOid; lowerBoundFuncExpr->funcretset = false; Oid lessThan = GetSysCacheOid(OPERNAMENSP, Anum_pg_operator_oid, CStringGetDatum("<"), resultTypeOid, resultTypeOid, ObjectIdGetDatum( PG_CATALOG_NAMESPACE)); /* * Finally, check if the comparison result is less than 0, i.e., * shardInterval->minValue < hash(partitionColumn) * See SearchCachedShardInterval for the behavior at the boundaries. */ Expr *lowerBoundExpr = make_opclause(lessThan, BOOLOID, false, (Expr *) lowerBoundFuncExpr, (Expr *) constNodeZero, InvalidOid, InvalidOid); /* create a function expression for the upper bound of the shard interval */ FuncExpr *upperBoundFuncExpr = makeNode(FuncExpr); upperBoundFuncExpr->funcid = cacheEntry->shardIntervalCompareFunction->fn_oid; upperBoundFuncExpr->args = list_make2((Node *) hashFunction, (Expr *) constNodeUpperBound); upperBoundFuncExpr->funcresulttype = resultTypeOid; upperBoundFuncExpr->funcretset = false; Oid lessThanOrEqualTo = GetSysCacheOid(OPERNAMENSP, Anum_pg_operator_oid, CStringGetDatum("<="), resultTypeOid, resultTypeOid, ObjectIdGetDatum(PG_CATALOG_NAMESPACE)); /* * Finally, check if the comparison result is less than or equal to 0, i.e., * hash(partitionColumn) <= shardInterval->maxValue * See SearchCachedShardInterval for the behavior at the boundaries. */ Expr *upperBoundExpr = make_opclause(lessThanOrEqualTo, BOOLOID, false, (Expr *) upperBoundFuncExpr, (Expr *) constNodeZero, InvalidOid, InvalidOid); /* create a node for both upper and lower bound */ Node *shardIntervalBoundQuals = make_and_qual((Node *) lowerBoundExpr, (Node *) upperBoundExpr); /* * Add a null test for the partition column for the first shard. * This is because we need to include the null values in exactly one of the shard queries. * The null test is added as an OR clause to the existing AND clause. */ if (isFirstShard) { /* null test for the first shard */ NullTest *nullTest = makeNode(NullTest); nullTest->nulltesttype = IS_NULL; /* Check for IS NULL */ nullTest->arg = (Expr *) outerTablePartitionColumnVar; /* The variable to check */ nullTest->argisrow = false; shardIntervalBoundQuals = (Node *) make_orclause(list_make2(nullTest, shardIntervalBoundQuals)); } return shardIntervalBoundQuals; } /* * UpdateWhereClauseToPushdownRecurringOuterJoinWalker walks over the query tree and * updates the WHERE clause for outer joins satisfying feasibility conditions. */ bool UpdateWhereClauseToPushdownRecurringOuterJoinWalker(Node *node, List *relationShardList) { if (node == NULL) { return false; } if (IsA(node, Query)) { UpdateWhereClauseToPushdownRecurringOuterJoin((Query *) node, relationShardList); return query_tree_walker((Query *) node, UpdateWhereClauseToPushdownRecurringOuterJoinWalker, relationShardList, QTW_EXAMINE_RTES_BEFORE); } if (!IsA(node, RangeTblEntry)) { return expression_tree_walker(node, UpdateWhereClauseToPushdownRecurringOuterJoinWalker, relationShardList); } return false; } /* * UpdateWhereClauseToPushdownRecurringOuterJoin * * Inject shard interval predicates into the query WHERE clause for certain * outer joins to make the join semantically correct when distributed. * * Why this is needed: * When an inner side of an OUTER JOIN is a distributed table that has been * routed to a single shard, we cannot simply replace the RTE with the shard * name and rely on implicit pruning: the preserved (outer) side could still * produce rows whose join keys would hash to other shards. To keep results * consistent with the global execution semantics we restrict the preserved * (outer) side to only those partition key values that would route to the * chosen shard (plus NULLs, which are assigned to exactly one shard). * * What the function does: * 1. Iterate over the top-level jointree->fromlist. * 2. For each JoinExpr call CanPushdownRecurringOuterJoinExtended() which: * - Verifies shape / join type is eligible. * - Returns: * outerRtIndex : RT index whose column we will constrain, * outerRte / innerRte, * attnum : attribute number (partition column) on outer side. * This is compared to partition column of innerRte. * 3. Find the RelationShard for the inner distributed table (innerRte->relid) * in relationShardList; skip if absent (no fixed shard chosen). * 4. Build the shard qualification with CreateQualsForShardInterval(): * (minValue < hash(partcol) AND hash(partcol) <= maxValue) * and, for the first shard only, OR (partcol IS NULL). * The Var refers to (outerRtIndex, attnum) so the restriction applies to * the preserved outer input. * 5. AND the new quals into jointree->quals (creating it if NULL). * * The function does not return anything, it modifies the query in place. */ void UpdateWhereClauseToPushdownRecurringOuterJoin(Query *query, List *relationShardList) { if (query == NULL) { return; } FromExpr *fromExpr = query->jointree; if (fromExpr == NULL || fromExpr->fromlist == NIL) { return; } ListCell *fromExprCell; foreach(fromExprCell, fromExpr->fromlist) { Node *fromItem = (Node *) lfirst(fromExprCell); if (!IsA(fromItem, JoinExpr)) { continue; } JoinExpr *joinExpr = (JoinExpr *) fromItem; /* * We will check if we need to add constraints to the WHERE clause. */ RangeTblEntry *innerRte = NULL; RangeTblEntry *outerRte = NULL; int outerRtIndex = -1; int attnum; if (!CanPushdownRecurringOuterJoinExtended(joinExpr, query, &outerRtIndex, &outerRte, &innerRte, &attnum)) { continue; } if (attnum == InvalidAttrNumber) { continue; } ereport(DEBUG5, (errmsg( "Distributed table from the inner part of the outer join: %s.", innerRte->eref->aliasname))); RelationShard *relationShard = FindRelationShard(innerRte->relid, relationShardList); if (relationShard == NULL || relationShard->shardId == INVALID_SHARD_ID) { continue; } Node *shardIntervalBoundQuals = CreateQualsForShardInterval(relationShard, attnum, outerRtIndex); if (fromExpr->quals == NULL) { fromExpr->quals = (Node *) shardIntervalBoundQuals; } else { fromExpr->quals = make_and_qual(fromExpr->quals, shardIntervalBoundQuals); } } } /* * UpdateRelationToShardNames walks over the query tree and appends shard ids to * relations. It uses unique identity value to establish connection between a * shard and the range table entry. If the range table id is not given a * identity, than the relation is not referenced from the query, no connection * could be found between a shard and this relation. Therefore relation is replaced * by set of NULL values so that the query would work at worker without any problems. * */ bool UpdateRelationToShardNames(Node *node, List *relationShardList) { uint64 shardId = INVALID_SHARD_ID; if (node == NULL) { return false; } /* want to look at all RTEs, even in subqueries, CTEs and such */ if (IsA(node, Query)) { return query_tree_walker((Query *) node, UpdateRelationToShardNames, relationShardList, QTW_EXAMINE_RTES_BEFORE); } if (!IsA(node, RangeTblEntry)) { return expression_tree_walker(node, UpdateRelationToShardNames, relationShardList); } RangeTblEntry *newRte = (RangeTblEntry *) node; if (newRte->rtekind == RTE_FUNCTION) { newRte = NULL; if (!FindCitusExtradataContainerRTE(node, &newRte)) { /* only update function rtes containing citus_extradata_container */ return false; } } else if (newRte->rtekind != RTE_RELATION) { return false; } if (!IsCitusTable(newRte->relid)) { /* leave local tables as is */ return false; } RelationShard *relationShard = FindRelationShard(newRte->relid, relationShardList); bool replaceRteWithNullValues = relationShard == NULL || relationShard->shardId == INVALID_SHARD_ID; if (replaceRteWithNullValues) { ConvertRteToSubqueryWithEmptyResult(newRte); return false; } shardId = relationShard->shardId; Oid relationId = relationShard->relationId; char *relationName = get_rel_name(relationId); AppendShardIdToName(&relationName, shardId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); ModifyRangeTblExtraData(newRte, CITUS_RTE_SHARD, schemaName, relationName, NIL); return false; } /* * FindRelationShard finds the RelationShard for shard relation with * given Oid if exists in given relationShardList. Otherwise, returns NULL. */ static RelationShard * FindRelationShard(Oid inputRelationId, List *relationShardList) { RelationShard *relationShard = NULL; /* * Search for the restrictions associated with the RTE. There better be * some, otherwise this query wouldn't be eligible as a router query. * FIXME: We should probably use a hashtable here, to do efficient lookup. */ foreach_declared_ptr(relationShard, relationShardList) { if (inputRelationId == relationShard->relationId) { return relationShard; } } return NULL; } /* * ConvertRteToSubqueryWithEmptyResult converts given relation RTE into * subquery RTE that returns no results. */ static void ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte) { Relation relation = table_open(rte->relid, NoLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); int columnCount = tupleDescriptor->natts; List *targetList = NIL; for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { FormData_pg_attribute *attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); if (attributeForm->attisdropped) { continue; } StringInfo resname = makeStringInfo(); Const *constValue = makeNullConst(attributeForm->atttypid, attributeForm->atttypmod, attributeForm->attcollation); appendStringInfo(resname, "%s", attributeForm->attname.data); TargetEntry *targetEntry = makeNode(TargetEntry); targetEntry->expr = (Expr *) constValue; targetEntry->resno = columnIndex; targetEntry->resname = resname->data; targetList = lappend(targetList, targetEntry); } table_close(relation, NoLock); FromExpr *joinTree = makeNode(FromExpr); joinTree->quals = makeBoolConst(false, false); Query *subquery = makeNode(Query); subquery->commandType = CMD_SELECT; subquery->querySource = QSRC_ORIGINAL; subquery->canSetTag = true; subquery->targetList = targetList; subquery->jointree = joinTree; rte->rtekind = RTE_SUBQUERY; /* no permission checking for this RTE */ rte->perminfoindex = 0; rte->subquery = subquery; rte->alias = copyObject(rte->eref); } /* * ShouldLazyDeparseQuery returns true if we should lazily deparse the query * when adding it to the task. Right now it simply checks if any shards on the * local node can be used for the task. */ static bool ShouldLazyDeparseQuery(Task *task) { return TaskAccessesLocalNode(task); } /* * SetTaskQueryIfShouldLazyDeparse attaches the query to the task so that it can be used during * execution. If local execution can possibly take place it sets task->jobQueryReferenceForLazyDeparsing. * If not it deparses the query and sets queryStringLazy, to avoid blowing the * size of the task unnecesarily. */ void SetTaskQueryIfShouldLazyDeparse(Task *task, Query *query) { if (ShouldLazyDeparseQuery(task)) { task->taskQuery.queryType = TASK_QUERY_OBJECT; task->taskQuery.data.jobQueryReferenceForLazyDeparsing = query; task->queryCount = 1; return; } SetTaskQueryString(task, AnnotateQuery(DeparseTaskQuery(task, query), task->partitionKeyValue, task->colocationId)); } /* * SetTaskQueryString attaches the query string to the task so that it can be * used during execution. It also unsets jobQueryReferenceForLazyDeparsing to be sure * these are kept in sync. */ void SetTaskQueryString(Task *task, char *queryString) { if (queryString == NULL) { task->taskQuery.queryType = TASK_QUERY_NULL; task->queryCount = 0; } else { task->taskQuery.queryType = TASK_QUERY_TEXT; task->taskQuery.data.queryStringLazy = queryString; task->queryCount = 1; } } /* * SetTaskQueryStringList sets the queryStringList of the given task. */ void SetTaskQueryStringList(Task *task, List *queryStringList) { Assert(queryStringList != NIL); task->taskQuery.queryType = TASK_QUERY_TEXT_LIST; task->taskQuery.data.queryStringList = queryStringList; task->queryCount = list_length(queryStringList); } void SetTaskQueryPlan(Task *task, Query *query, PlannedStmt *localPlan) { Assert(localPlan != NULL); task->taskQuery.queryType = TASK_QUERY_LOCAL_PLAN; task->taskQuery.data.localCompiled = (LocalCompilation *) palloc0( sizeof(LocalCompilation)); task->taskQuery.data.localCompiled->query = query; task->taskQuery.data.localCompiled->plan = localPlan; task->queryCount = 1; } PlannedStmt * TaskQueryLocalPlan(Task *task) { Assert(task->taskQuery.queryType == TASK_QUERY_LOCAL_PLAN); return task->taskQuery.data.localCompiled->plan; } /* * DeparseTaskQuery is a general way of deparsing a query based on a task. */ static char * DeparseTaskQuery(Task *task, Query *query) { StringInfo queryString = makeStringInfo(); if (query->commandType == CMD_INSERT) { /* * For INSERT queries we cannot use pg_get_query_def. Mainly because we * cannot run UpdateRelationToShardNames on an INSERT query. This is * because the PG deparsing logic fails when trying to insert into a * RTE_FUNCTION (which is what will happen if you call * UpdateRelationToShardNames). */ deparse_shard_query(query, task->anchorDistributedTableId, task->anchorShardId, queryString); } else { pg_get_query_def(query, queryString); } return queryString->data; } /* * GetTaskQueryType returns the type of the task query. */ int GetTaskQueryType(Task *task) { return task->taskQuery.queryType; } /* * TaskQueryStringAtIndex returns query at given index among the possibly * multiple queries that a task can have. */ char * TaskQueryStringAtIndex(Task *task, int index) { Assert(index < task->queryCount); int taskQueryType = GetTaskQueryType(task); if (taskQueryType == TASK_QUERY_TEXT_LIST) { return list_nth(task->taskQuery.data.queryStringList, index); } return TaskQueryString(task); } /* * TaskQueryString generates task query string text if missing. * * For performance reasons, the queryString is generated lazily. For example * for local queries it is usually not needed to generate it, so this way we * can skip the expensive deparsing+parsing. */ char * TaskQueryString(Task *task) { int taskQueryType = GetTaskQueryType(task); if (taskQueryType == TASK_QUERY_NULL) { /* if task query type is TASK_QUERY_NULL then the data will be NULL, * this is unexpected state */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unexpected task query state: task query type is null"), errdetail("Please report this to the Citus core team."))); } else if (taskQueryType == TASK_QUERY_TEXT_LIST) { return StringJoin(task->taskQuery.data.queryStringList, ';'); } else if (taskQueryType == TASK_QUERY_TEXT) { return task->taskQuery.data.queryStringLazy; } else if (taskQueryType == TASK_QUERY_LOCAL_PLAN) { Query *query = task->taskQuery.data.localCompiled->query; Assert(query != NULL); /* * Use the query of the local compilation to generate the * query string. For local compiled tasks, the query is retained * for this purpose, which may be EXPLAIN ANALYZing the task, or * command logging. Generating the query string on the fly is * acceptable because the plan of the local compilation is used * for query execution. */ MemoryContext previousContext = MemoryContextSwitchTo(GetMemoryChunkContext( query)); UpdateRelationToShardNames((Node *) query, task->relationShardList); MemoryContextSwitchTo(previousContext); return AnnotateQuery(DeparseTaskQuery(task, query), task->partitionKeyValue, task->colocationId); } Query *jobQueryReferenceForLazyDeparsing = task->taskQuery.data.jobQueryReferenceForLazyDeparsing; /* * At this point task query type should be TASK_QUERY_OBJECT. */ Assert(task->taskQuery.queryType == TASK_QUERY_OBJECT && jobQueryReferenceForLazyDeparsing != NULL); /* * Switch to the memory context of task->jobQueryReferenceForLazyDeparsing before generating the query * string. This way the query string is not freed in between multiple * executions of a prepared statement. Except when UpdateTaskQueryString is * used to set task->jobQueryReferenceForLazyDeparsing, in that case it is freed but it will be set to * NULL on the next execution of the query because UpdateTaskQueryString * does that. */ MemoryContext previousContext = MemoryContextSwitchTo(GetMemoryChunkContext( jobQueryReferenceForLazyDeparsing)); char *queryString = DeparseTaskQuery(task, jobQueryReferenceForLazyDeparsing); MemoryContextSwitchTo(previousContext); SetTaskQueryString(task, queryString); return task->taskQuery.data.queryStringLazy; } ================================================ FILE: src/backend/distributed/planner/distributed_planner.c ================================================ /*------------------------------------------------------------------------- * * distributed_planner.c * General Citus planner code. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/pg_class.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/plancat.h" #include "optimizer/planmain.h" #include "optimizer/planner.h" #include "parser/parse_relation.h" #include "parser/parse_type.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/citus_depended_object.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/combine_query_planner.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/cte_inline.h" #include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_planner.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/merge_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/shard_utils.h" #include "distributed/shardinterval_utils.h" #include "distributed/stats/stat_tenants.h" #include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" static List *plannerRestrictionContextList = NIL; int MultiTaskQueryLogLevel = CITUS_LOG_LEVEL_OFF; /* multi-task query log level */ static uint64 NextPlanId = 1; /* keep track of planner call stack levels */ int PlannerLevel = 0; static bool ListContainsDistributedTableRTE(List *rangeTableList, bool *maybeHasForeignDistributedTable); static bool PlanContainsDistributedSubPlanRTE(List *subPlanList); static PlannedStmt * CreateDistributedPlannedStmt(DistributedPlanningContext * planContext); static PlannedStmt * InlineCtesAndCreateDistributedPlannedStmt(uint64 planId, DistributedPlanningContext *planContext); static PlannedStmt * TryCreateDistributedPlannedStmt(PlannedStmt *localPlan, Query *originalQuery, Query *query, ParamListInfo boundParams, PlannerRestrictionContext * plannerRestrictionContext); static DeferredErrorMessage * DeferErrorIfPartitionTableNotSingleReplicated(Oid relationId); static int AssignRTEIdentities(List *rangeTableList, int rteIdCounter); static void AssignRTEIdentity(RangeTblEntry *rangeTableEntry, int rteIdentifier); static void AdjustPartitioningForDistributedPlanning(List *rangeTableList, bool setPartitionedTablesInherited); static bool RTEWentThroughAdjustPartitioning(RangeTblEntry *rangeTableEntry); static PlannedStmt * FinalizeNonRouterPlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan, CustomScan *customScan); static PlannedStmt * FinalizeRouterPlan(PlannedStmt *localPlan, CustomScan *customScan); static AppendRelInfo * FindTargetAppendRelInfo(PlannerInfo *root, int relationRteIndex); static List * makeTargetListFromCustomScanList(List *custom_scan_tlist); static List * makeCustomScanTargetlistFromExistingTargetList(List *existingTargetlist); static int32 BlessRecordExpressionList(List *exprs); static void CheckNodeIsDumpable(Node *node); static Node * CheckNodeCopyAndSerialization(Node *node); static void AdjustReadIntermediateResultCost(RangeTblEntry *rangeTableEntry, RelOptInfo *relOptInfo); static void AdjustReadIntermediateResultArrayCost(RangeTblEntry *rangeTableEntry, RelOptInfo *relOptInfo); static void AdjustReadIntermediateResultsCostInternal(RelOptInfo *relOptInfo, List *columnTypes, int resultIdCount, Datum *resultIds, Const *resultFormatConst); static List * OuterPlanParamsList(PlannerInfo *root); static List * CopyPlanParamList(List *originalPlanParamList); static void CreateAndPushPlannerRestrictionContext(DistributedPlanningContext * planContext, FastPathRestrictionContext * fastPathContext); static PlannerRestrictionContext * CurrentPlannerRestrictionContext(void); static void PopPlannerRestrictionContext(void); static void ResetPlannerRestrictionContext(PlannerRestrictionContext * plannerRestrictionContext); static PlannedStmt * PlanFastPathDistributedStmt(DistributedPlanningContext *planContext); static PlannedStmt * PlanDistributedStmt(DistributedPlanningContext *planContext, int rteIdCounter); static RTEListProperties * GetRTEListProperties(List *rangeTableList); static List * TranslatedVars(PlannerInfo *root, int relationIndex); static void WarnIfListHasForeignDistributedTable(List *rangeTableList); static RouterPlanType GetRouterPlanType(Query *query, Query *originalQuery, bool hasUnresolvedParams); static void ConcatenateRTablesAndPerminfos(PlannedStmt *mainPlan, PlannedStmt *concatPlan); static bool CheckPostPlanDistribution(DistributedPlanningContext *planContext, bool isDistributedQuery, List *rangeTableList); #if PG_VERSION_NUM >= PG_VERSION_18 static int DisableSelfJoinElimination(void); #endif /* Distributed planner hook */ PlannedStmt * distributed_planner(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) { bool needsDistributedPlanning = false; bool fastPathRouterQuery = false; FastPathRestrictionContext fastPathContext = { 0 }; #if PG_VERSION_NUM >= PG_VERSION_18 int saveNestLevel = -1; #endif List *rangeTableList = ExtractRangeTableEntryList(parse); if (cursorOptions & CURSOR_OPT_FORCE_DISTRIBUTED) { /* this cursor flag could only be set when Citus has been loaded */ Assert(CitusHasBeenLoaded()); /* * We cannot have merge command for this path as well because * there cannot be recursively planned merge command. */ Assert(!IsMergeQuery(parse)); needsDistributedPlanning = true; } else if (CitusHasBeenLoaded()) { bool maybeHasForeignDistributedTable = false; needsDistributedPlanning = ListContainsDistributedTableRTE(rangeTableList, &maybeHasForeignDistributedTable); if (needsDistributedPlanning) { fastPathRouterQuery = FastPathRouterQuery(parse, &fastPathContext); if (maybeHasForeignDistributedTable) { WarnIfListHasForeignDistributedTable(rangeTableList); } } } int rteIdCounter = 1; DistributedPlanningContext planContext = { .query = parse, .cursorOptions = cursorOptions, .boundParams = boundParams, }; if (needsDistributedPlanning) { /* * standard_planner scribbles on its input, but for deparsing we need the * unmodified form. Before copying we call AssignRTEIdentities to be able * to match RTEs in the rewritten query tree with those in the original * tree. */ rteIdCounter = AssignRTEIdentities(rangeTableList, rteIdCounter); planContext.originalQuery = copyObject(parse); if (!fastPathRouterQuery) { /* * When there are partitioned tables (not applicable to fast path), * pretend that they are regular tables to avoid unnecessary work * in standard_planner. */ bool setPartitionedTablesInherited = false; AdjustPartitioningForDistributedPlanning(rangeTableList, setPartitionedTablesInherited); #if PG_VERSION_NUM >= PG_VERSION_18 saveNestLevel = DisableSelfJoinElimination(); #endif } } /* * Make sure that we hide shard names on the Citus MX worker nodes. See comments in * HideShardsFromSomeApplications() for the details. */ HideShardsFromSomeApplications(parse); /* * If GUC is set, we prevent queries, which contain pg meta relations, from * showing any citus dependent object. The flag is expected to be set only before * postgres vanilla tests. */ HideCitusDependentObjectsOnQueriesOfPgMetaTables((Node *) parse, NULL); /* create a restriction context and put it at the end of our plan context's context list */ CreateAndPushPlannerRestrictionContext(&planContext, &fastPathContext); /* * We keep track of how many times we've recursed into the planner, primarily * to detect whether we are in a function call. We need to make sure that the * PlannerLevel is decremented exactly once at the end of the next PG_TRY * block, both in the happy case and when an error occurs. */ PlannerLevel++; PlannedStmt *result = NULL; PG_TRY(); { if (fastPathRouterQuery) { result = PlanFastPathDistributedStmt(&planContext); } else { /* * Call into standard_planner because the Citus planner relies on both the * restriction information per table and parse tree transformations made by * postgres' planner. */ planContext.plan = standard_planner(planContext.query, NULL, planContext.cursorOptions, planContext.boundParams); #if PG_VERSION_NUM >= PG_VERSION_18 if (needsDistributedPlanning) { Assert(saveNestLevel > 0); AtEOXact_GUC(true, saveNestLevel); } /* Pop the plan context from the current restriction context */ planContext.plannerRestrictionContext->planContext = NULL; #endif needsDistributedPlanning = CheckPostPlanDistribution(&planContext, needsDistributedPlanning, rangeTableList); if (needsDistributedPlanning) { result = PlanDistributedStmt(&planContext, rteIdCounter); } else if ((result = TryToDelegateFunctionCall(&planContext)) == NULL) { result = planContext.plan; } } } PG_CATCH(); { PopPlannerRestrictionContext(); PlannerLevel--; PG_RE_THROW(); } PG_END_TRY(); PlannerLevel--; /* remove the context from the context list */ PopPlannerRestrictionContext(); /* * In some cases, for example; parameterized SQL functions, we may miss that * there is a need for distributed planning. Such cases only become clear after * standard_planner performs some modifications on parse tree. In such cases * we will simply error out. */ if (!needsDistributedPlanning && NeedsDistributedPlanning(parse)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning on this " "query because parameterized queries for SQL " "functions referencing distributed tables are " "not supported"), errhint("Consider using PL/pgSQL functions instead."))); } /* * We annotate the query for tenant statisisics. */ AttributeQueryIfAnnotated(query_string, parse->commandType); return result; } /* * ExtractRangeTableEntryList is a wrapper around ExtractRangeTableEntryWalker. * The function traverses the input query and returns all the range table * entries that are in the query tree. */ List * ExtractRangeTableEntryList(Query *query) { List *rteList = NIL; ExtractRangeTableEntryWalker((Node *) query, &rteList); return rteList; } /* * NeedsDistributedPlanning returns true if the Citus extension is loaded and * the query contains a distributed table. * * This function allows queries containing local tables to pass through the * distributed planner. How to handle local tables is a decision that should * be made within the planner */ bool NeedsDistributedPlanning(Query *query) { if (!CitusHasBeenLoaded()) { return false; } CmdType commandType = query->commandType; if (commandType != CMD_SELECT && commandType != CMD_INSERT && commandType != CMD_UPDATE && commandType != CMD_DELETE) { return false; } List *allRTEs = ExtractRangeTableEntryList(query); return ListContainsDistributedTableRTE(allRTEs, NULL); } /* * ListContainsDistributedTableRTE gets a list of range table entries * and returns true if there is at least one distributed relation range * table entry in the list. The boolean maybeHasForeignDistributedTable * variable is set to true if the list contains a foreign table. */ static bool ListContainsDistributedTableRTE(List *rangeTableList, bool *maybeHasForeignDistributedTable) { ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->rtekind != RTE_RELATION) { continue; } if (HideCitusDependentObjects && IsolationIsSerializable() && IsPgLocksTable( rangeTableEntry)) { /* * Postgres tidscan.sql test fails if we do not filter pg_locks table because * test results, which show taken locks in serializable isolation mode, * fails by showing extra lock taken by IsCitusTable below. */ continue; } if (IsCitusTable(rangeTableEntry->relid)) { if (maybeHasForeignDistributedTable != NULL && IsForeignTable(rangeTableEntry->relid)) { *maybeHasForeignDistributedTable = true; } return true; } } return false; } /* * PlanContainsDistributedSubPlanRTE checks whether any of the subplans in the given * subPlanList is a Read Intermediate Result function scan. * * It is used by the check after standard_planner() to determine whether the plan * still requires distributed planning; in addition to checking the range table for * distributed tables, we also need to check whether there are any subplans that * read intermediate results, which indicates a distributed subplan and therefore * that distributed planning is required. */ static bool PlanContainsDistributedSubPlanRTE(List *subPlanList) { ListCell *subPlanCell = NULL; foreach(subPlanCell, subPlanList) { Node *planRoot = (Node *) lfirst(subPlanCell); if (!IsA(planRoot, FunctionScan)) { continue; } List *functionList = ((FunctionScan *) planRoot)->functions; if (functionList == NIL) { continue; } RangeTblFunction *rangeTblfunction = (RangeTblFunction *) linitial(functionList); if (IsReadIntermediateResultFunction(rangeTblfunction->funcexpr)) { return true; } } return false; } /* * AssignRTEIdentities function modifies query tree by adding RTE identities to the * RTE_RELATIONs. * * Please note that, we want to avoid modifying query tree as much as possible * because if PostgreSQL changes the way it uses modified fields, that may break * our logic. * * Returns the next id. This can be used to call on a rangeTableList that may've * been partially assigned. Should be set to 1 initially. */ static int AssignRTEIdentities(List *rangeTableList, int rteIdCounter) { ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); /* * To be able to track individual RTEs through PostgreSQL's query * planning, we need to be able to figure out whether an RTE is * actually a copy of another, rather than a different one. We * simply number the RTEs starting from 1. * * Note that we're only interested in RTE_RELATIONs and thus assigning * identifiers to those RTEs only. */ if (rangeTableEntry->rtekind == RTE_RELATION && rangeTableEntry->values_lists == NIL) { AssignRTEIdentity(rangeTableEntry, rteIdCounter++); } } return rteIdCounter; } /* * AdjustPartitioningForDistributedPlanning function modifies query tree by * changing inh flag and relkind of partitioned tables. We want Postgres to * treat partitioned tables as regular relations (i.e. we do not want to * expand them to their partitions) since it breaks Citus planning in different * ways. We let anything related to partitioning happen on the shards. * * Please note that, we want to avoid modifying query tree as much as possible * because if PostgreSQL changes the way it uses modified fields, that may break * our logic. */ static void AdjustPartitioningForDistributedPlanning(List *rangeTableList, bool setPartitionedTablesInherited) { ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); /* * We want Postgres to behave partitioned tables as regular relations * (i.e. we do not want to expand them to their partitions). To do this * we set each partitioned table's inh flag to appropriate * value before and after dropping to the standart_planner. */ if (rangeTableEntry->rtekind == RTE_RELATION && PartitionedTable(rangeTableEntry->relid)) { rangeTableEntry->inh = setPartitionedTablesInherited; if (setPartitionedTablesInherited) { rangeTableEntry->relkind = RELKIND_PARTITIONED_TABLE; } else { rangeTableEntry->relkind = RELKIND_RELATION; } } } } /* * RTEWentThroughAdjustPartitioning returns true if the given rangetableentry * has been modified through AdjustPartitioningForDistributedPlanning * function, false otherwise. */ static bool RTEWentThroughAdjustPartitioning(RangeTblEntry *rangeTableEntry) { return (rangeTableEntry->rtekind == RTE_RELATION && PartitionedTable(rangeTableEntry->relid) && rangeTableEntry->inh == false); } /* * AssignRTEIdentity assigns the given rteIdentifier to the given range table * entry. * * To be able to track RTEs through postgres' query planning, which copies and * duplicate, and modifies them, we sometimes need to figure out whether two * RTEs are copies of the same original RTE. For that we, hackishly, use a * field normally unused in RTE_RELATION RTEs. * * The assigned identifier better be unique within a plantree. */ static void AssignRTEIdentity(RangeTblEntry *rangeTableEntry, int rteIdentifier) { Assert(rangeTableEntry->rtekind == RTE_RELATION); rangeTableEntry->values_lists = list_make2_int(rteIdentifier, rangeTableEntry->inh); } /* GetRTEIdentity returns the identity assigned with AssignRTEIdentity. */ int GetRTEIdentity(RangeTblEntry *rte) { Assert(rte->rtekind == RTE_RELATION); /* * Since SQL functions might be in-lined by standard_planner, * we might miss assigning an RTE identity for RangeTblEntries * related to SQL functions. We already have checks in other * places to throw an error for SQL functions but they are not * sufficient due to function in-lining; so here we capture such * cases and throw an error here. */ if (list_length(rte->values_lists) != 2) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning on this " "query because parameterized queries for SQL " "functions referencing distributed tables are " "not supported"), errhint("Consider using PL/pgSQL functions instead."))); } Assert(IsA(rte->values_lists, IntList)); return linitial_int(rte->values_lists); } /* * GetOriginalInh gets the original value of the inheritance flag set by * AssignRTEIdentity. The planner resets this flag in the rewritten query, * but we need it during deparsing. */ bool GetOriginalInh(RangeTblEntry *rte) { return lsecond_int(rte->values_lists); } /* * GetQueryLockMode returns the necessary lock mode to be acquired for the * given query. (See comment written in RangeTblEntry->rellockmode) */ LOCKMODE GetQueryLockMode(Query *query) { if (IsModifyCommand(query)) { return RowExclusiveLock; } else if (query->hasForUpdate) { return RowShareLock; } else { return AccessShareLock; } } /* * IsModifyCommand returns true if the query performs modifications, false * otherwise. */ bool IsModifyCommand(Query *query) { CmdType commandType = query->commandType; if (commandType == CMD_INSERT || commandType == CMD_UPDATE || commandType == CMD_DELETE || commandType == CMD_MERGE) { return true; } return false; } /* * IsMultiTaskPlan returns true if job contains multiple tasks. */ bool IsMultiTaskPlan(DistributedPlan *distributedPlan) { Job *workerJob = distributedPlan->workerJob; if (workerJob != NULL && list_length(workerJob->taskList) > 1) { return true; } return false; } /* * PlanFastPathDistributedStmt creates a distributed planned statement using * the FastPathPlanner. */ static PlannedStmt * PlanFastPathDistributedStmt(DistributedPlanningContext *planContext) { FastPathRestrictionContext *fastPathContext = planContext->plannerRestrictionContext->fastPathRestrictionContext; Assert(fastPathContext != NULL); Assert(fastPathContext->fastPathRouterQuery); FastPathPreprocessParseTree(planContext->query); if (!fastPathContext->delayFastPathPlanning) { planContext->plan = FastPathPlanner(planContext->originalQuery, planContext->query, planContext->boundParams); } return CreateDistributedPlannedStmt(planContext); } /* * PlanDistributedStmt creates a distributed planned statement using the PG * planner. */ static PlannedStmt * PlanDistributedStmt(DistributedPlanningContext *planContext, int rteIdCounter) { /* may've inlined new relation rtes */ List *rangeTableList = ExtractRangeTableEntryList(planContext->query); rteIdCounter = AssignRTEIdentities(rangeTableList, rteIdCounter); PlannedStmt *result = CreateDistributedPlannedStmt(planContext); bool setPartitionedTablesInherited = true; AdjustPartitioningForDistributedPlanning(rangeTableList, setPartitionedTablesInherited); return result; } /* * DissuadePlannerFromUsingPlan try dissuade planner when planning a plan that * potentially failed due to unresolved prepared statement parameters. */ void DissuadePlannerFromUsingPlan(PlannedStmt *plan) { /* * Arbitrarily high cost, but low enough that it can be added up * without overflowing by choose_custom_plan(). */ Assert(plan != NULL); plan->planTree->total_cost = FLT_MAX / 100000000; } /* * CreateDistributedPlannedStmt encapsulates the logic needed to transform a particular * query into a distributed plan that is encapsulated by a PlannedStmt. */ static PlannedStmt * CreateDistributedPlannedStmt(DistributedPlanningContext *planContext) { uint64 planId = NextPlanId++; bool hasUnresolvedParams = false; PlannedStmt *resultPlan = NULL; if (QueryTreeContainsInlinableCTE(planContext->originalQuery)) { /* * Inlining CTEs as subqueries in the query can avoid recursively * planning some (or all) of the CTEs. In other words, the inlined * CTEs could become part of query pushdown planning, which is much * more efficient than recursively planning. So, first try distributed * planning on the inlined CTEs in the query tree. * * We also should fallback to distributed planning with non-inlined CTEs * if the distributed planning fails with inlined CTEs, because recursively * planning CTEs can provide full SQL coverage, although it might be slow. */ resultPlan = InlineCtesAndCreateDistributedPlannedStmt(planId, planContext); if (resultPlan != NULL) { return resultPlan; } } if (HasUnresolvedExternParamsWalker((Node *) planContext->originalQuery, planContext->boundParams)) { hasUnresolvedParams = true; } bool allowRecursivePlanning = true; DistributedPlan *distributedPlan = CreateDistributedPlan(planId, allowRecursivePlanning, planContext->originalQuery, planContext->query, planContext->boundParams, hasUnresolvedParams, planContext->plannerRestrictionContext); /* * If no plan was generated, prepare a generic error to be emitted. * Normally this error message will never returned to the user, as it's * usually due to unresolved prepared statement parameters - in that case * the logic below will force a custom plan (i.e. with parameters bound to * specific values) to be generated. But sql (not plpgsql) functions * unfortunately don't go through a codepath supporting custom plans - so * we still need to have an error prepared. */ if (!distributedPlan) { /* currently always should have a more specific error otherwise */ Assert(hasUnresolvedParams); distributedPlan = CitusMakeNode(DistributedPlan); distributedPlan->planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "could not create distributed plan", "Possibly this is caused by the use of parameters in SQL " "functions, which is not supported in Citus.", "Consider using PL/pgSQL functions instead."); } /* * Error out if none of the planners resulted in a usable plan, unless the * error was possibly triggered by missing parameters. In that case we'll * not error out here, but instead rely on postgres' custom plan logic. * Postgres re-plans prepared statements the first five executions * (i.e. it produces custom plans), after that the cost of a generic plan * is compared with the average custom plan cost. We support otherwise * unsupported prepared statement parameters by assigning an exorbitant * cost to the unsupported query. That'll lead to the custom plan being * chosen. But for that to be possible we can't error out here, as * otherwise that logic is never reached. */ if (distributedPlan->planningError && !hasUnresolvedParams) { RaiseDeferredError(distributedPlan->planningError, ERROR); } CheckAndBuildDelayedFastPathPlan(planContext, distributedPlan); /* remember the plan's identifier for identifying subplans */ distributedPlan->planId = planId; /* create final plan by combining local plan with distributed plan */ resultPlan = FinalizePlan(planContext->plan, distributedPlan); /* * As explained above, force planning costs to be unrealistically high if * query planning failed (possibly) due to prepared statement parameters or * if it is planned as a multi shard modify query. */ if ((distributedPlan->planningError || (UpdateOrDeleteOrMergeQuery(planContext->originalQuery) && IsMultiTaskPlan( distributedPlan))) && hasUnresolvedParams) { DissuadePlannerFromUsingPlan(resultPlan); } return resultPlan; } /* * InlineCtesAndCreateDistributedPlannedStmt gets all the parameters required * for creating a distributed planned statement. The function is primarily a * wrapper on top of CreateDistributedPlannedStmt(), by first inlining the * CTEs and calling CreateDistributedPlannedStmt() in PG_TRY() block. The * function returns NULL if the planning fails on the query where eligable * CTEs are inlined. */ static PlannedStmt * InlineCtesAndCreateDistributedPlannedStmt(uint64 planId, DistributedPlanningContext *planContext) { /* * We'll inline the CTEs and try distributed planning, preserve the original * query in case the planning fails and we fallback to recursive planning of * CTEs. */ Query *copyOfOriginalQuery = copyObject(planContext->originalQuery); RecursivelyInlineCtesInQueryTree(copyOfOriginalQuery); /* after inlining, we shouldn't have any inlinable CTEs */ Assert(!QueryTreeContainsInlinableCTE(copyOfOriginalQuery)); /* simply recurse into CreateDistributedPlannedStmt() in a PG_TRY() block */ PlannedStmt *result = TryCreateDistributedPlannedStmt(planContext->plan, copyOfOriginalQuery, planContext->query, planContext->boundParams, planContext-> plannerRestrictionContext); return result; } /* * TryCreateDistributedPlannedStmt is a wrapper around CreateDistributedPlannedStmt, simply * calling it in PG_TRY()/PG_CATCH() block. The function returns a PlannedStmt if the input * query can be planned by Citus. If not, the function returns NULL and generates a DEBUG4 * message with the reason for the failure. */ static PlannedStmt * TryCreateDistributedPlannedStmt(PlannedStmt *localPlan, Query *originalQuery, Query *query, ParamListInfo boundParams, PlannerRestrictionContext *plannerRestrictionContext) { MemoryContext savedContext = CurrentMemoryContext; PlannedStmt *result = NULL; DistributedPlanningContext *planContext = palloc0(sizeof(DistributedPlanningContext)); planContext->plan = localPlan; planContext->boundParams = boundParams; planContext->originalQuery = originalQuery; planContext->query = query; planContext->plannerRestrictionContext = plannerRestrictionContext; PG_TRY(); { result = CreateDistributedPlannedStmt(planContext); } PG_CATCH(); { MemoryContextSwitchTo(savedContext); ErrorData *edata = CopyErrorData(); FlushErrorState(); /* don't try to intercept PANIC or FATAL, let those breeze past us */ if (edata->elevel != ERROR) { PG_RE_THROW(); } ereport(DEBUG4, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Planning after CTEs inlined failed with " "\nmessage: %s\ndetail: %s\nhint: %s", edata->message ? edata->message : "", edata->detail ? edata->detail : "", edata->hint ? edata->hint : ""))); /* leave the error handling system */ FreeErrorData(edata); result = NULL; } PG_END_TRY(); return result; } /* * GetRouterPlanType checks the parse tree to return appropriate plan type. */ static RouterPlanType GetRouterPlanType(Query *query, Query *originalQuery, bool hasUnresolvedParams) { if (!IsModifyCommand(originalQuery)) { return SELECT_QUERY; } Oid targetRelationId = ModifyQueryResultRelationId(query); EnsureModificationsCanRunOnRelation(targetRelationId); EnsurePartitionTableNotReplicated(targetRelationId); /* Check the type of modification being done */ if (InsertSelectIntoCitusTable(originalQuery)) { if (hasUnresolvedParams) { return REPLAN_WITH_BOUND_PARAMETERS; } return INSERT_SELECT_INTO_CITUS_TABLE; } else if (InsertSelectIntoLocalTable(originalQuery)) { if (hasUnresolvedParams) { return REPLAN_WITH_BOUND_PARAMETERS; } return INSERT_SELECT_INTO_LOCAL_TABLE; } else if (IsMergeQuery(originalQuery)) { if (hasUnresolvedParams) { return REPLAN_WITH_BOUND_PARAMETERS; } return MERGE_QUERY; } else { return DML_QUERY; } } /* * CreateDistributedPlan generates a distributed plan for a query. * It goes through 3 steps: * * 1. Try router planner * 2. Generate subplans for CTEs and complex subqueries * - If any, go back to step 1 by calling itself recursively * 3. Logical planner */ DistributedPlan * CreateDistributedPlan(uint64 planId, bool allowRecursivePlanning, Query *originalQuery, Query *query, ParamListInfo boundParams, bool hasUnresolvedParams, PlannerRestrictionContext *plannerRestrictionContext) { DistributedPlan *distributedPlan = NULL; bool hasCtes = originalQuery->cteList != NIL; /* Step 1: Try router planner */ RouterPlanType routerPlan = GetRouterPlanType(query, originalQuery, hasUnresolvedParams); switch (routerPlan) { case INSERT_SELECT_INTO_CITUS_TABLE: { distributedPlan = CreateInsertSelectPlan(planId, originalQuery, plannerRestrictionContext, boundParams); break; } case INSERT_SELECT_INTO_LOCAL_TABLE: { distributedPlan = CreateInsertSelectIntoLocalTablePlan(planId, originalQuery, boundParams, hasUnresolvedParams, plannerRestrictionContext); break; } case DML_QUERY: { /* modifications are always routed through the same planner/executor */ distributedPlan = CreateModifyPlan(originalQuery, query, plannerRestrictionContext); break; } case MERGE_QUERY: { distributedPlan = CreateMergePlan(planId, originalQuery, query, plannerRestrictionContext, boundParams); break; } case REPLAN_WITH_BOUND_PARAMETERS: { /* * Unresolved parameters can cause performance regressions in * INSERT...SELECT when the partition column is a parameter * because we don't perform any additional pruning in the executor. */ return NULL; } case SELECT_QUERY: { /* * For select queries we, if router executor is enabled, first try to * plan the query as a router query. If not supported, otherwise try * the full blown plan/optimize/physical planning process needed to * produce distributed query plans. */ distributedPlan = CreateRouterPlan(originalQuery, query, plannerRestrictionContext); break; } } /* the functions above always return a plan, possibly with an error */ Assert(distributedPlan); if (distributedPlan->planningError == NULL) { return distributedPlan; } else { RaiseDeferredError(distributedPlan->planningError, DEBUG2); } if (hasUnresolvedParams) { /* * There are parameters that don't have a value in boundParams. * * The remainder of the planning logic cannot handle unbound * parameters. We return a NULL plan, which will have an * extremely high cost, such that postgres will replan with * bound parameters. */ return NULL; } /* force evaluation of bound params */ boundParams = copyParamList(boundParams); /* * If there are parameters that do have a value in boundParams, replace * them in the original query. This allows us to more easily cut the * query into pieces (during recursive planning) or deparse parts of * the query (during subquery pushdown planning). */ originalQuery = (Query *) ResolveExternalParams((Node *) originalQuery, boundParams); Assert(originalQuery != NULL); /* Step 2: Generate subplans for CTEs and complex subqueries */ /* * Plan subqueries and CTEs that cannot be pushed down by recursively * calling the planner and return the resulting plans to subPlanList. * Note that GenerateSubplansForSubqueriesAndCTEs will reset perminfoindexes * for some RTEs in originalQuery->rtable list, while not changing * originalQuery->rteperminfos. That's fine because we will go through * standard_planner again, which will adjust things accordingly in * set_plan_references>add_rtes_to_flat_rtable>add_rte_to_flat_rtable. */ List *subPlanList = GenerateSubplansForSubqueriesAndCTEs(planId, originalQuery, plannerRestrictionContext, routerPlan); /* * If subqueries were recursively planned then we need to replan the query * to get the new planner restriction context and apply planner transformations. * * We could simplify this code if the logical planner was capable of dealing * with an original query. In that case, we would only have to filter the * planner restriction context. * * Note that we check both for subplans and whether the query had CTEs * prior to calling GenerateSubplansForSubqueriesAndCTEs. If none of * the CTEs are referenced then there are no subplans, but we still want * to retry the router planner. */ if (list_length(subPlanList) > 0 || hasCtes) { /* * recursive planner should handle all the tree from bottom to * top at single pass. i.e. It should have already recursively planned all * required parts in its first pass. Hence, we expect allowRecursivePlanning * to be true. Otherwise, this means we have bug at recursive planner, * which needs to be handled. We add a check here and return error. */ if (!allowRecursivePlanning) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("recursive complex joins are only supported " "when all distributed tables are co-located and " "joined on their distribution columns"))); } Query *newQuery = copyObject(originalQuery); bool setPartitionedTablesInherited = false; PlannerRestrictionContext *currentPlannerRestrictionContext = CurrentPlannerRestrictionContext(); /* reset the current planner restrictions context */ ResetPlannerRestrictionContext(currentPlannerRestrictionContext); /* * We force standard_planner to treat partitioned tables as regular tables * by clearing the inh flag on RTEs. We already did this at the start of * distributed_planner, but on a copy of the original query, so we need * to do it again here. */ AdjustPartitioningForDistributedPlanning(ExtractRangeTableEntryList(newQuery), setPartitionedTablesInherited); /* * Some relations may have been removed from the query, but we can skip * AssignRTEIdentities since we currently do not rely on RTE identities * being contiguous. */ standard_planner(newQuery, NULL, 0, boundParams); /* overwrite the old transformed query with the new transformed query */ *query = *newQuery; /* * recurse into CreateDistributedPlan with subqueries/CTEs replaced. * We only allow recursive planning once, which should have already done all * the necessary transformations. So, we do not allow recursive planning once again. */ allowRecursivePlanning = false; distributedPlan = CreateDistributedPlan(planId, allowRecursivePlanning, originalQuery, query, NULL, false, plannerRestrictionContext); /* distributedPlan cannot be null since hasUnresolvedParams argument was false */ Assert(distributedPlan != NULL); distributedPlan->subPlanList = subPlanList; return distributedPlan; } /* * DML command returns a planning error, even after recursive planning. The * logical planner cannot handle DML commands so return the plan with the * error. */ if (IsModifyCommand(originalQuery)) { return distributedPlan; } /* * CTEs are stripped from the original query by RecursivelyPlanSubqueriesAndCTEs. * If we get here and there are still CTEs that means that none of the CTEs are * referenced. We therefore also strip the CTEs from the rewritten query. */ query->cteList = NIL; Assert(originalQuery->cteList == NIL); /* Step 3: Try Logical planner */ MultiTreeRoot *logicalPlan = MultiLogicalPlanCreate(originalQuery, query, plannerRestrictionContext); MultiLogicalPlanOptimize(logicalPlan); /* * This check is here to make it likely that all node types used in * Citus are dumpable. Explain can dump logical and physical plans * using the extended outfuncs infrastructure, but it's infeasible to * test most plans. MultiQueryContainerNode always serializes the * physical plan, so there's no need to check that separately */ CheckNodeIsDumpable((Node *) logicalPlan); /* Create the physical plan */ distributedPlan = CreatePhysicalDistributedPlan(logicalPlan, plannerRestrictionContext); /* distributed plan currently should always succeed or error out */ Assert(distributedPlan && distributedPlan->planningError == NULL); return distributedPlan; } /* * EnsurePartitionTableNotReplicated errors out if the input relation is * a partition table and the table has a replication factor greater than * one. * * If the table is not a partition or replication factor is 1, the function * becomes a no-op. */ void EnsurePartitionTableNotReplicated(Oid relationId) { DeferredErrorMessage *deferredError = DeferErrorIfPartitionTableNotSingleReplicated(relationId); if (deferredError != NULL) { RaiseDeferredError(deferredError, ERROR); } } /* * DeferErrorIfPartitionTableNotSingleReplicated defers error if the input relation * is a partition table with replication factor > 1. Otherwise, the function returns * NULL. */ static DeferredErrorMessage * DeferErrorIfPartitionTableNotSingleReplicated(Oid relationId) { if (PartitionTableNoLock(relationId) && !SingleReplicatedTable(relationId)) { Oid parentOid = PartitionParentOid(relationId); char *parentRelationTest = get_rel_name(parentOid); StringInfo errorHint = makeStringInfo(); appendStringInfo(errorHint, "Run the query on the parent table " "\"%s\" instead.", parentRelationTest); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "modifications on partitions when replication " "factor is greater than 1 is not supported", NULL, errorHint->data); } return NULL; } /* * ResolveExternalParams replaces the external parameters that appears * in the query with the corresponding entries in the boundParams. * * Note that this function is inspired by eval_const_expr() on Postgres. * We cannot use that function because it requires access to PlannerInfo. */ Node * ResolveExternalParams(Node *inputNode, ParamListInfo boundParams) { /* consider resolving external parameters only when boundParams exists */ if (!boundParams) { return inputNode; } if (inputNode == NULL) { return NULL; } if (IsA(inputNode, Param)) { Param *paramToProcess = (Param *) inputNode; int numberOfParameters = boundParams->numParams; int parameterId = paramToProcess->paramid; int16 typeLength = 0; bool typeByValue = false; Datum constValue = 0; if (paramToProcess->paramkind != PARAM_EXTERN) { return inputNode; } if (parameterId < 0) { return inputNode; } /* parameterId starts from 1 */ int parameterIndex = parameterId - 1; if (parameterIndex >= numberOfParameters) { return inputNode; } ParamExternData *correspondingParameterData = &boundParams->params[parameterIndex]; if (!(correspondingParameterData->pflags & PARAM_FLAG_CONST)) { return inputNode; } get_typlenbyval(paramToProcess->paramtype, &typeLength, &typeByValue); bool paramIsNull = correspondingParameterData->isnull; if (paramIsNull) { constValue = 0; } else if (typeByValue) { constValue = correspondingParameterData->value; } else { /* * Out of paranoia ensure that datum lives long enough, * although bind params currently should always live * long enough. */ constValue = datumCopy(correspondingParameterData->value, typeByValue, typeLength); } return (Node *) makeConst(paramToProcess->paramtype, paramToProcess->paramtypmod, paramToProcess->paramcollid, typeLength, constValue, paramIsNull, typeByValue); } else if (IsA(inputNode, Query)) { return (Node *) query_tree_mutator((Query *) inputNode, ResolveExternalParams, boundParams, 0); } return expression_tree_mutator(inputNode, ResolveExternalParams, boundParams); } /* * GetDistributedPlan returns the associated DistributedPlan for a CustomScan. * * Callers should only read from the returned data structure, since it may be * the plan of a prepared statement and may therefore be reused. */ DistributedPlan * GetDistributedPlan(CustomScan *customScan) { Assert(list_length(customScan->custom_private) == 1); Node *node = (Node *) linitial(customScan->custom_private); Assert(CitusIsA(node, DistributedPlan)); CheckNodeCopyAndSerialization(node); DistributedPlan *distributedPlan = (DistributedPlan *) node; return distributedPlan; } /* * FinalizePlan combines local plan with distributed plan and creates a plan * which can be run by the PostgreSQL executor. */ PlannedStmt * FinalizePlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan) { PlannedStmt *finalPlan = NULL; CustomScan *customScan = makeNode(CustomScan); MultiExecutorType executorType = MULTI_EXECUTOR_INVALID_FIRST; /* this field is used in JobExecutorType */ distributedPlan->relationIdList = localPlan->relationOids; if (!distributedPlan->planningError) { executorType = JobExecutorType(distributedPlan); } switch (executorType) { case MULTI_EXECUTOR_ADAPTIVE: { customScan->methods = &AdaptiveExecutorCustomScanMethods; break; } case MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT: { customScan->methods = &NonPushableInsertSelectCustomScanMethods; break; } case MULTI_EXECUTOR_NON_PUSHABLE_MERGE_QUERY: { customScan->methods = &NonPushableMergeCommandCustomScanMethods; break; } default: { customScan->methods = &DelayedErrorCustomScanMethods; break; } } if (IsMultiTaskPlan(distributedPlan)) { /* if it is not a single task executable plan, inform user according to the log level */ if (MultiTaskQueryLogLevel != CITUS_LOG_LEVEL_OFF) { ereport(MultiTaskQueryLogLevel, (errmsg( "multi-task query about to be executed"), errhint( "Queries are split to multiple tasks " "if they have to be split into several" " queries on the workers."))); } } distributedPlan->queryId = localPlan->queryId; Node *distributedPlanData = (Node *) distributedPlan; customScan->custom_private = list_make1(distributedPlanData); /* necessary to avoid extra Result node in PG15 */ customScan->flags = CUSTOMPATH_SUPPORT_BACKWARD_SCAN | CUSTOMPATH_SUPPORT_PROJECTION; /* * Fast path queries cannot have any subplans by definition, so skip * expensive traversals. */ if (!distributedPlan->fastPathRouterPlan) { /* * Record subplans used by distributed plan to make intermediate result * pruning easier. * * We do this before finalizing the plan, because the combineQuery is * rewritten by standard_planner in FinalizeNonRouterPlan. */ distributedPlan->usedSubPlanNodeList = FindSubPlanUsages(distributedPlan); } if (distributedPlan->combineQuery) { finalPlan = FinalizeNonRouterPlan(localPlan, distributedPlan, customScan); } else { finalPlan = FinalizeRouterPlan(localPlan, customScan); } return finalPlan; } /* * FinalizeNonRouterPlan gets the distributed custom scan plan, and creates the * final master select plan on the top of this distributed plan for adaptive executor. */ static PlannedStmt * FinalizeNonRouterPlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan, CustomScan *customScan) { PlannedStmt *finalPlan = PlanCombineQuery(distributedPlan, customScan); finalPlan->queryId = localPlan->queryId; finalPlan->utilityStmt = localPlan->utilityStmt; /* add original range table list for access permission checks */ ConcatenateRTablesAndPerminfos(finalPlan, localPlan); return finalPlan; } static void ConcatenateRTablesAndPerminfos(PlannedStmt *mainPlan, PlannedStmt *concatPlan) { mainPlan->rtable = list_concat(mainPlan->rtable, concatPlan->rtable); /* * concatPlan's range table list is concatenated to mainPlan's range table list * therefore all the perminfoindexes should be updated to their value * PLUS the highest perminfoindex in mainPlan's perminfos, which is exactly * the list length. */ int mainPlan_highest_perminfoindex = list_length(mainPlan->permInfos); ListCell *lc; foreach(lc, concatPlan->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); if (rte->perminfoindex != 0) { rte->perminfoindex = rte->perminfoindex + mainPlan_highest_perminfoindex; } } /* finally, concatenate perminfos as well */ mainPlan->permInfos = list_concat(mainPlan->permInfos, concatPlan->permInfos); } /* * FinalizeRouterPlan gets a CustomScan node which already wrapped distributed * part of a router plan and sets it as the direct child of the router plan * because we don't run any query on master node for router executable queries. * Here, we also rebuild the column list to read from the remote scan. */ static PlannedStmt * FinalizeRouterPlan(PlannedStmt *localPlan, CustomScan *customScan) { List *columnNameList = NIL; customScan->custom_scan_tlist = makeCustomScanTargetlistFromExistingTargetList(localPlan->planTree->targetlist); customScan->scan.plan.targetlist = makeTargetListFromCustomScanList(customScan->custom_scan_tlist); /* extract the column names from the final targetlist*/ TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, customScan->scan.plan.targetlist) { String *columnName = makeString(targetEntry->resname); columnNameList = lappend(columnNameList, columnName); } PlannedStmt *routerPlan = makeNode(PlannedStmt); routerPlan->planTree = (Plan *) customScan; RangeTblEntry *remoteScanRangeTableEntry = RemoteScanRangeTableEntry(columnNameList); routerPlan->rtable = list_make1(remoteScanRangeTableEntry); /* add original range table list for access permission checks */ ConcatenateRTablesAndPerminfos(routerPlan, localPlan); routerPlan->canSetTag = true; routerPlan->relationOids = NIL; routerPlan->queryId = localPlan->queryId; routerPlan->utilityStmt = localPlan->utilityStmt; routerPlan->commandType = localPlan->commandType; routerPlan->hasReturning = localPlan->hasReturning; return routerPlan; } /* * makeCustomScanTargetlistFromExistingTargetList rebuilds the targetlist from the remote * query into a list that can be used as the custom_scan_tlist for our Citus Custom Scan. */ static List * makeCustomScanTargetlistFromExistingTargetList(List *existingTargetlist) { List *custom_scan_tlist = NIL; /* we will have custom scan range table entry as the first one in the list */ const int customScanRangeTableIndex = 1; /* build a targetlist to read from the custom scan output */ TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, existingTargetlist) { Assert(IsA(targetEntry, TargetEntry)); /* * This is unlikely to be hit because we would not need resjunk stuff * at the toplevel of a router query - all things needing it have been * pushed down. */ if (targetEntry->resjunk) { continue; } /* build target entry pointing to remote scan range table entry */ Var *newVar = makeVarFromTargetEntry(customScanRangeTableIndex, targetEntry); if (newVar->vartype == RECORDOID || newVar->vartype == RECORDARRAYOID) { /* * Add the anonymous composite type to the type cache and store * the key in vartypmod. Eventually this makes its way into the * TupleDesc used by the executor, which uses it to parse the * query results from the workers in BuildTupleFromCStrings. */ newVar->vartypmod = BlessRecordExpression(targetEntry->expr); } TargetEntry *newTargetEntry = flatCopyTargetEntry(targetEntry); newTargetEntry->expr = (Expr *) newVar; custom_scan_tlist = lappend(custom_scan_tlist, newTargetEntry); } return custom_scan_tlist; } /* * makeTargetListFromCustomScanList based on a custom_scan_tlist create the target list to * use on the Citus Custom Scan Node. The targetlist differs from the custom_scan_tlist in * a way that the expressions in the targetlist all are references to the index (resno) in * the custom_scan_tlist in their varattno while the varno is replaced with INDEX_VAR * instead of the range table entry index. */ static List * makeTargetListFromCustomScanList(List *custom_scan_tlist) { List *targetList = NIL; TargetEntry *targetEntry = NULL; int resno = 1; foreach_declared_ptr(targetEntry, custom_scan_tlist) { /* * INDEX_VAR is used to reference back to the TargetEntry in custom_scan_tlist by * its resno (index) */ Var *newVar = makeVarFromTargetEntry(INDEX_VAR, targetEntry); TargetEntry *newTargetEntry = makeTargetEntry((Expr *) newVar, resno, targetEntry->resname, targetEntry->resjunk); targetList = lappend(targetList, newTargetEntry); resno++; } return targetList; } /* * BlessRecordExpression ensures we can parse an anonymous composite type on the * target list of a query that is sent to the worker. * * We cannot normally parse record types coming from the workers unless we * "bless" the tuple descriptor, which adds a transient type to the type cache * and assigns it a type mod value, which is the key in the type cache. */ int32 BlessRecordExpression(Expr *expr) { int32 typeMod = -1; if (IsA(expr, FuncExpr) || IsA(expr, OpExpr)) { /* * Handle functions that return records on the target * list, e.g. SELECT function_call(1,2); */ Oid resultTypeId = InvalidOid; TupleDesc resultTupleDesc = NULL; /* get_expr_result_type blesses the tuple descriptor */ TypeFuncClass typeClass = get_expr_result_type((Node *) expr, &resultTypeId, &resultTupleDesc); if (typeClass == TYPEFUNC_COMPOSITE) { typeMod = resultTupleDesc->tdtypmod; } } else if (IsA(expr, RowExpr)) { /* * Handle row expressions, e.g. SELECT (1,2); */ RowExpr *rowExpr = (RowExpr *) expr; ListCell *argCell = NULL; int currentResno = 1; TupleDesc rowTupleDesc = CreateTemplateTupleDesc(list_length(rowExpr->args)); foreach(argCell, rowExpr->args) { Node *rowArg = (Node *) lfirst(argCell); Oid rowArgTypeId = exprType(rowArg); int rowArgTypeMod = exprTypmod(rowArg); if (rowArgTypeId == RECORDOID || rowArgTypeId == RECORDARRAYOID) { /* ensure nested rows are blessed as well */ rowArgTypeMod = BlessRecordExpression((Expr *) rowArg); } TupleDescInitEntry(rowTupleDesc, currentResno, NULL, rowArgTypeId, rowArgTypeMod, 0); TupleDescInitEntryCollation(rowTupleDesc, currentResno, exprCollation(rowArg)); currentResno++; } BlessTupleDesc(rowTupleDesc); typeMod = rowTupleDesc->tdtypmod; } /* * Record aggregates need blessed typmods to parse worker results. * For PG18, AGG_MATCH_RECORD allows MIN/MAX on composite types. * * Limitation: For multi-argument aggregates returning RECORD, we only * bless the first argument's type. This works for MIN/MAX (single-argument) * but may not handle custom multi-argument aggregates correctly. */ else if (IsA(expr, Aggref)) { Aggref *aggref = (Aggref *) expr; if (aggref->aggtype == RECORDOID && list_length(aggref->args) > 0) { if (list_length(aggref->args) > 1) { ereport(DEBUG2, (errmsg( "blessing record aggregate with %d arguments, using first", list_length(aggref->args)))); } TargetEntry *argTle = (TargetEntry *) linitial(aggref->args); Oid argTypeId = exprType((Node *) argTle->expr); int32 argTypeMod = exprTypmod((Node *) argTle->expr); if (argTypeId == RECORDOID) { argTypeMod = BlessRecordExpression((Expr *) argTle->expr); } /* Use a RECORD TupleDesc derived from a named rowtype argument. */ if (type_is_rowtype(argTypeId)) { TupleDesc argTupleDesc = lookup_rowtype_tupdesc_copy(argTypeId, argTypeMod); argTupleDesc->tdtypeid = RECORDOID; argTupleDesc->tdtypmod = -1; BlessTupleDesc(argTupleDesc); typeMod = argTupleDesc->tdtypmod; } /* * If argTypeId is not a rowtype, we leave typeMod as -1. * This should not happen in practice since AGG_MATCH_RECORD * only matches rowtypes, but it's safe to fall through. */ } } else if (IsA(expr, ArrayExpr)) { /* * Handle row array expressions, e.g. SELECT ARRAY[(1,2)]; * Postgres allows ARRAY[(1,2),(1,2,3)]. We do not. */ ArrayExpr *arrayExpr = (ArrayExpr *) expr; typeMod = BlessRecordExpressionList(arrayExpr->elements); } else if (IsA(expr, NullIfExpr)) { NullIfExpr *nullIfExpr = (NullIfExpr *) expr; typeMod = BlessRecordExpressionList(nullIfExpr->args); } else if (IsA(expr, MinMaxExpr)) { MinMaxExpr *minMaxExpr = (MinMaxExpr *) expr; typeMod = BlessRecordExpressionList(minMaxExpr->args); } else if (IsA(expr, CoalesceExpr)) { CoalesceExpr *coalesceExpr = (CoalesceExpr *) expr; typeMod = BlessRecordExpressionList(coalesceExpr->args); } else if (IsA(expr, CaseExpr)) { CaseExpr *caseExpr = (CaseExpr *) expr; List *results = NIL; ListCell *whenCell = NULL; foreach(whenCell, caseExpr->args) { CaseWhen *whenArg = (CaseWhen *) lfirst(whenCell); results = lappend(results, whenArg->result); } if (caseExpr->defresult != NULL) { results = lappend(results, caseExpr->defresult); } typeMod = BlessRecordExpressionList(results); } return typeMod; } /* * BlessRecordExpressionList maps BlessRecordExpression over a list. * Returns typmod of all expressions, or -1 if they are not all the same. * Ignores expressions with a typmod of -1. */ static int32 BlessRecordExpressionList(List *exprs) { int32 finalTypeMod = -1; ListCell *exprCell = NULL; foreach(exprCell, exprs) { Node *exprArg = (Node *) lfirst(exprCell); int32 exprTypeMod = BlessRecordExpression((Expr *) exprArg); if (exprTypeMod == -1) { continue; } else if (finalTypeMod == -1) { finalTypeMod = exprTypeMod; } else if (finalTypeMod != exprTypeMod) { return -1; } } return finalTypeMod; } /* * RemoteScanRangeTableEntry creates a range table entry from given column name * list to represent a remote scan. */ RangeTblEntry * RemoteScanRangeTableEntry(List *columnNameList) { RangeTblEntry *remoteScanRangeTableEntry = makeNode(RangeTblEntry); /* we use RTE_VALUES for custom scan because we can't look up relation */ remoteScanRangeTableEntry->rtekind = RTE_VALUES; remoteScanRangeTableEntry->eref = makeAlias("remote_scan", columnNameList); remoteScanRangeTableEntry->inh = false; remoteScanRangeTableEntry->inFromCl = true; return remoteScanRangeTableEntry; } /* * CheckNodeIsDumpable checks that the passed node can be dumped using * nodeToString(). As this checks is expensive, it's only active when * assertions are enabled. */ static void CheckNodeIsDumpable(Node *node) { #ifdef USE_ASSERT_CHECKING char *out = nodeToString(node); pfree(out); #endif } /* * CheckNodeCopyAndSerialization checks copy/dump/read functions * for nodes and returns copy of the input. * * It is only active when assertions are enabled, otherwise it returns * the input directly. We use this to confirm that our serialization * and copy logic produces the correct plan during regression tests. * * It does not check string equality on node dumps due to differences * in some Postgres types. */ static Node * CheckNodeCopyAndSerialization(Node *node) { #ifdef USE_ASSERT_CHECKING char *out = nodeToString(node); Node *nodeCopy = copyObject(node); char *outCopy = nodeToString(nodeCopy); pfree(out); pfree(outCopy); return nodeCopy; #else return node; #endif } /* * multi_join_restriction_hook is a hook called by postgresql standard planner * to notify us about various planning information regarding joins. We use * it to learn about the joining column. */ void multi_join_restriction_hook(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra) { if (bms_is_empty(innerrel->relids) || bms_is_empty(outerrel->relids)) { /* * We do not expect empty relids. Still, ignoring such JoinRestriction is * preferable for two reasons: * 1. This might be a query that doesn't rely on JoinRestrictions at all (e.g., * local query). * 2. We cannot process them when they are empty (and likely to segfault if * we allow as-is). */ ereport(DEBUG1, (errmsg("Join restriction information is NULL"))); } /* * Use a memory context that's guaranteed to live long enough, could be * called in a more shortly lived one (e.g. with GEQO). */ PlannerRestrictionContext *plannerRestrictionContext = CurrentPlannerRestrictionContext(); MemoryContext restrictionsMemoryContext = plannerRestrictionContext->memoryContext; MemoryContext oldMemoryContext = MemoryContextSwitchTo(restrictionsMemoryContext); JoinRestrictionContext *joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; Assert(joinRestrictionContext != NULL); JoinRestriction *joinRestriction = palloc0(sizeof(JoinRestriction)); joinRestriction->joinType = jointype; joinRestriction->plannerInfo = root; /* * We create a copy of restrictInfoList and relids because with geqo they may * be created in a memory context which will be deleted when we still need it, * thus we create a copy of it in our memory context. */ joinRestriction->joinRestrictInfoList = copyObject(extra->restrictlist); joinRestriction->innerrelRelids = bms_copy(innerrel->relids); joinRestriction->outerrelRelids = bms_copy(outerrel->relids); joinRestrictionContext->joinRestrictionList = lappend(joinRestrictionContext->joinRestrictionList, joinRestriction); /* * Keep track if we received any semi joins here. If we didn't we can * later safely convert any semi joins in the rewritten query to inner * joins. */ joinRestrictionContext->hasSemiJoin = joinRestrictionContext->hasSemiJoin || extra->sjinfo->jointype == JOIN_SEMI; joinRestrictionContext->hasOuterJoin = joinRestrictionContext->hasOuterJoin || IS_OUTER_JOIN(extra->sjinfo->jointype); MemoryContextSwitchTo(oldMemoryContext); } /* * multi_relation_restriction_hook is a hook called by postgresql standard planner * to notify us about various planning information regarding a relation. We use * it to retrieve restrictions on relations. */ void multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo, Index restrictionIndex, RangeTblEntry *rte) { CitusTableCacheEntry *cacheEntry = NULL; if (ReplaceCitusExtraDataContainer && IsCitusExtraDataContainerRelation(rte)) { /* * We got here by planning the query part that needs to be executed on the query * coordinator node. * We have verified the occurrence of the citus_extra_datacontainer function * encoding the remote scan we plan to execute here. We will replace all paths * with a path describing our custom scan. */ Path *path = CreateCitusCustomScanPath(root, relOptInfo, restrictionIndex, rte, ReplaceCitusExtraDataContainerWithCustomScan); /* replace all paths with our custom scan and recalculate cheapest */ relOptInfo->pathlist = list_make1(path); set_cheapest(relOptInfo); return; } AdjustReadIntermediateResultCost(rte, relOptInfo); AdjustReadIntermediateResultArrayCost(rte, relOptInfo); if (rte->rtekind != RTE_RELATION) { return; } /* * Use a memory context that's guaranteed to live long enough, could be * called in a more shortly lived one (e.g. with GEQO). */ PlannerRestrictionContext *plannerRestrictionContext = CurrentPlannerRestrictionContext(); MemoryContext restrictionsMemoryContext = plannerRestrictionContext->memoryContext; MemoryContext oldMemoryContext = MemoryContextSwitchTo(restrictionsMemoryContext); bool isCitusTable = IsCitusTable(rte->relid); RelationRestriction *relationRestriction = palloc0(sizeof(RelationRestriction)); relationRestriction->index = restrictionIndex; relationRestriction->relationId = rte->relid; relationRestriction->rte = rte; relationRestriction->relOptInfo = relOptInfo; relationRestriction->citusTable = isCitusTable; relationRestriction->plannerInfo = root; /* see comments on GetVarFromAssignedParam() */ relationRestriction->outerPlanParamsList = OuterPlanParamsList(root); relationRestriction->translatedVars = TranslatedVars(root, relationRestriction->index); RelationRestrictionContext *relationRestrictionContext = plannerRestrictionContext->relationRestrictionContext; /* * We're also keeping track of whether all participant * tables are reference tables. */ if (isCitusTable) { cacheEntry = GetCitusTableCacheEntry(rte->relid); relationRestrictionContext->allReferenceTables &= IsCitusTableTypeCacheEntry(cacheEntry, REFERENCE_TABLE); } relationRestrictionContext->relationRestrictionList = lappend(relationRestrictionContext->relationRestrictionList, relationRestriction); MemoryContextSwitchTo(oldMemoryContext); #if PG_VERSION_NUM >= PG_VERSION_18 if (root->query_level == 1 && plannerRestrictionContext->planContext != NULL) { /* We're at the top query with a distributed context; see if Postgres * has changed the query tree we passed to it in distributed_planner(). * This check was necessitated by PG commit 1e4351a, becuase in it the * planner modfies a copy of the passed in query tree with the consequence * that changes are not reflected back to the caller of standard_planner(). */ Query *query = plannerRestrictionContext->planContext->query; if (root->parse != query) { /* * The Postgres planner has reconstructed the query tree, so the query * tree our distributed context passed in (to standard_planner() is * updated to track the new query tree. */ ereport(DEBUG4, (errmsg( "Detected query reconstruction by Postgres planner, updating " "planContext to track it"))); plannerRestrictionContext->planContext->query = root->parse; } } #endif } /* * multi_get_relation_info_hook modifies the relation's indexlist * if necessary, to avoid a crash in PG16 caused by our * Citus function AdjustPartitioningForDistributedPlanning(). * * AdjustPartitioningForDistributedPlanning() is a hack that we use * to prevent Postgres' standard_planner() to expand all the partitions * for the distributed planning when a distributed partitioned table * is queried. It is required for both correctness and performance * reasons. Although we can eliminate the use of the function for * the correctness (e.g., make sure that rest of the planner can handle * partitions), it's performance implication is hard to avoid. Certain * planning logic of Citus (such as router or query pushdown) relies * heavily on the relationRestrictionList. If * AdjustPartitioningForDistributedPlanning() is removed, all the * partitions show up in the relationRestrictionList, causing high * planning times for such queries. */ void multi_get_relation_info_hook(PlannerInfo *root, Oid relationObjectId, bool inhparent, RelOptInfo *rel) { if (!CitusHasBeenLoaded()) { return; } Index varno = rel->relid; RangeTblEntry *rangeTableEntry = planner_rt_fetch(varno, root); if (RTEWentThroughAdjustPartitioning(rangeTableEntry)) { ListCell *lc = NULL; foreach(lc, rel->indexlist) { IndexOptInfo *indexOptInfo = (IndexOptInfo *) lfirst(lc); if (get_rel_relkind(indexOptInfo->indexoid) == RELKIND_PARTITIONED_INDEX) { /* * Normally, we should not need this. However, the combination of * Postgres commit 3c569049b7b502bb4952483d19ce622ff0af5fd6 and * Citus function AdjustPartitioningForDistributedPlanning() * forces us to do this. The commit expects partitioned indexes * to belong to relations with "inh" flag set properly. Whereas, the * function overrides "inh" flag. To avoid a crash, * we go over the list of indexinfos and remove all partitioned indexes. * Partitioned indexes were ignored pre PG16 anyway, we are essentially * not breaking any logic. */ rel->indexlist = foreach_delete_current(rel->indexlist, lc); } } } } /* * TranslatedVars deep copies the translated vars for the given relation index * if there is any append rel list. */ static List * TranslatedVars(PlannerInfo *root, int relationIndex) { List *translatedVars = NIL; if (root->append_rel_list != NIL) { AppendRelInfo *targetAppendRelInfo = FindTargetAppendRelInfo(root, relationIndex); if (targetAppendRelInfo != NULL) { /* postgres deletes translated_vars, hence we deep copy them here */ Node *targetNode = NULL; foreach_declared_ptr(targetNode, targetAppendRelInfo->translated_vars) { translatedVars = lappend(translatedVars, copyObject(targetNode)); } } } return translatedVars; } /* * FindTargetAppendRelInfo finds the target append rel info for the given * relation rte index. */ static AppendRelInfo * FindTargetAppendRelInfo(PlannerInfo *root, int relationRteIndex) { AppendRelInfo *appendRelInfo = NULL; /* iterate on the queries that are part of UNION ALL subselects */ foreach_declared_ptr(appendRelInfo, root->append_rel_list) { /* * We're only interested in the child rel that is equal to the * relation we're investigating. Here we don't need to find the offset * because postgres adds an offset to child_relid and parent_relid after * calling multi_relation_restriction_hook. */ if (appendRelInfo->child_relid == relationRteIndex) { return appendRelInfo; } } return NULL; } /* * AdjustReadIntermediateResultCost adjusts the row count and total cost * of a read_intermediate_result call based on the file size. */ static void AdjustReadIntermediateResultCost(RangeTblEntry *rangeTableEntry, RelOptInfo *relOptInfo) { if (rangeTableEntry->rtekind != RTE_FUNCTION || list_length(rangeTableEntry->functions) != 1) { /* avoid more expensive checks below for non-functions */ return; } if (!CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG5)) { /* read_intermediate_result may not exist */ return; } if (!ContainsReadIntermediateResultFunction((Node *) rangeTableEntry->functions)) { return; } RangeTblFunction *rangeTableFunction = (RangeTblFunction *) linitial( rangeTableEntry->functions); FuncExpr *funcExpression = (FuncExpr *) rangeTableFunction->funcexpr; Const *resultIdConst = (Const *) linitial(funcExpression->args); if (!IsA(resultIdConst, Const)) { /* not sure how to interpret non-const */ return; } Datum resultIdDatum = resultIdConst->constvalue; Const *resultFormatConst = (Const *) lsecond(funcExpression->args); if (!IsA(resultFormatConst, Const)) { /* not sure how to interpret non-const */ return; } AdjustReadIntermediateResultsCostInternal(relOptInfo, rangeTableFunction->funccoltypes, 1, &resultIdDatum, resultFormatConst); } /* * AdjustReadIntermediateResultArrayCost adjusts the row count and total cost * of a read_intermediate_results(resultIds, format) call based on the file size. */ static void AdjustReadIntermediateResultArrayCost(RangeTblEntry *rangeTableEntry, RelOptInfo *relOptInfo) { Datum *resultIdArray = NULL; int resultIdCount = 0; if (rangeTableEntry->rtekind != RTE_FUNCTION || list_length(rangeTableEntry->functions) != 1) { /* avoid more expensive checks below for non-functions */ return; } if (!CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG5)) { /* read_intermediate_result may not exist */ return; } if (!ContainsReadIntermediateResultArrayFunction((Node *) rangeTableEntry->functions)) { return; } RangeTblFunction *rangeTableFunction = (RangeTblFunction *) linitial(rangeTableEntry->functions); FuncExpr *funcExpression = (FuncExpr *) rangeTableFunction->funcexpr; Const *resultIdConst = (Const *) linitial(funcExpression->args); if (!IsA(resultIdConst, Const)) { /* not sure how to interpret non-const */ return; } Datum resultIdArrayDatum = resultIdConst->constvalue; deconstruct_array(DatumGetArrayTypeP(resultIdArrayDatum), TEXTOID, -1, false, 'i', &resultIdArray, NULL, &resultIdCount); Const *resultFormatConst = (Const *) lsecond(funcExpression->args); if (!IsA(resultFormatConst, Const)) { /* not sure how to interpret non-const */ return; } AdjustReadIntermediateResultsCostInternal(relOptInfo, rangeTableFunction->funccoltypes, resultIdCount, resultIdArray, resultFormatConst); } /* * AdjustReadIntermediateResultsCostInternal adjusts the row count and total cost * of reading intermediate results based on file sizes. */ static void AdjustReadIntermediateResultsCostInternal(RelOptInfo *relOptInfo, List *columnTypes, int resultIdCount, Datum *resultIds, Const *resultFormatConst) { PathTarget *reltarget = relOptInfo->reltarget; List *pathList = relOptInfo->pathlist; double rowCost = 0.; double rowSizeEstimate = 0; double rowCountEstimate = 0.; double ioCost = 0.; QualCost funcCost = { 0., 0. }; int64 totalResultSize = 0; ListCell *typeCell = NULL; Datum resultFormatDatum = resultFormatConst->constvalue; Oid resultFormatId = DatumGetObjectId(resultFormatDatum); bool binaryFormat = (resultFormatId == BinaryCopyFormatId()); for (int index = 0; index < resultIdCount; index++) { char *resultId = TextDatumGetCString(resultIds[index]); int64 resultSize = IntermediateResultSize(resultId); if (resultSize < 0) { /* result does not exist, will probably error out later on */ return; } if (binaryFormat) { /* subtract 11-byte signature + 8 byte header + 2-byte footer */ totalResultSize -= 21; } totalResultSize += resultSize; } /* start with the cost of evaluating quals */ rowCost += relOptInfo->baserestrictcost.per_tuple; /* postgres' estimate for the width of the rows */ rowSizeEstimate += reltarget->width; /* add 2 bytes for column count (binary) or line separator (text) */ rowSizeEstimate += 2; foreach(typeCell, columnTypes) { Oid columnTypeId = lfirst_oid(typeCell); Oid inputFunctionId = InvalidOid; Oid typeIOParam = InvalidOid; if (binaryFormat) { getTypeBinaryInputInfo(columnTypeId, &inputFunctionId, &typeIOParam); /* binary format: 4 bytes for field size */ rowSizeEstimate += 4; } else { getTypeInputInfo(columnTypeId, &inputFunctionId, &typeIOParam); /* text format: 1 byte for tab separator */ rowSizeEstimate += 1; } /* add the cost of parsing a column */ add_function_cost(NULL, inputFunctionId, NULL, &funcCost); } rowCost += funcCost.per_tuple; /* estimate the number of rows based on the file size and estimated row size */ rowCountEstimate = Max(1, (double) totalResultSize / rowSizeEstimate); /* cost of reading the data */ ioCost = seq_page_cost * totalResultSize / BLCKSZ; Assert(pathList != NIL); /* tell the planner about the cost and row count of the function */ Path *path = (Path *) linitial(pathList); path->rows = rowCountEstimate; path->total_cost = rowCountEstimate * rowCost + ioCost; path->startup_cost = funcCost.startup + relOptInfo->baserestrictcost.startup; } /* * OuterPlanParamsList creates a list of RootPlanParams for outer nodes of the * given root. The first item in the list corresponds to parent_root, and the * last item corresponds to the outer most node. */ static List * OuterPlanParamsList(PlannerInfo *root) { List *planParamsList = NIL; for (PlannerInfo *outerNodeRoot = root->parent_root; outerNodeRoot != NULL; outerNodeRoot = outerNodeRoot->parent_root) { RootPlanParams *rootPlanParams = palloc0(sizeof(RootPlanParams)); rootPlanParams->root = outerNodeRoot; /* * TODO: In SearchPlannerParamList() we are only interested in Var plan * params, consider copying just them here. */ rootPlanParams->plan_params = CopyPlanParamList(outerNodeRoot->plan_params); planParamsList = lappend(planParamsList, rootPlanParams); } return planParamsList; } /* * CopyPlanParamList deep copies the input PlannerParamItem list and returns the newly * allocated list. * Note that we cannot use copyObject() function directly since there is no support for * copying PlannerParamItem structs. */ static List * CopyPlanParamList(List *originalPlanParamList) { ListCell *planParamCell = NULL; List *copiedPlanParamList = NIL; foreach(planParamCell, originalPlanParamList) { PlannerParamItem *originalParamItem = lfirst(planParamCell); PlannerParamItem *copiedParamItem = makeNode(PlannerParamItem); copiedParamItem->paramId = originalParamItem->paramId; copiedParamItem->item = copyObject(originalParamItem->item); copiedPlanParamList = lappend(copiedPlanParamList, copiedParamItem); } return copiedPlanParamList; } /* * CreateAndPushPlannerRestrictionContext creates a new planner restriction * context with an empty relation restriction context and an empty join and * a copy of the given fast path restriction context (if present). Finally, * the planner restriction context is inserted to the beginning of the * global plannerRestrictionContextList and, in PG18+, given a reference to * its distributed plan context. */ static void CreateAndPushPlannerRestrictionContext(DistributedPlanningContext *planContext, FastPathRestrictionContext * fastPathRestrictionContext) { PlannerRestrictionContext *plannerRestrictionContext = palloc0(sizeof(PlannerRestrictionContext)); plannerRestrictionContext->relationRestrictionContext = palloc0(sizeof(RelationRestrictionContext)); plannerRestrictionContext->joinRestrictionContext = palloc0(sizeof(JoinRestrictionContext)); plannerRestrictionContext->fastPathRestrictionContext = palloc0(sizeof(FastPathRestrictionContext)); if (fastPathRestrictionContext != NULL) { /* copy the given fast path restriction context */ FastPathRestrictionContext *plannersFastPathCtx = plannerRestrictionContext->fastPathRestrictionContext; plannersFastPathCtx->fastPathRouterQuery = fastPathRestrictionContext->fastPathRouterQuery; plannersFastPathCtx->distributionKeyValue = fastPathRestrictionContext->distributionKeyValue; plannersFastPathCtx->distributionKeyHasParam = fastPathRestrictionContext->distributionKeyHasParam; plannersFastPathCtx->delayFastPathPlanning = fastPathRestrictionContext->delayFastPathPlanning; } plannerRestrictionContext->memoryContext = CurrentMemoryContext; /* we'll apply logical AND as we add tables */ plannerRestrictionContext->relationRestrictionContext->allReferenceTables = true; plannerRestrictionContextList = lcons(plannerRestrictionContext, plannerRestrictionContextList); planContext->plannerRestrictionContext = plannerRestrictionContext; #if PG_VERSION_NUM >= PG_VERSION_18 plannerRestrictionContext->planContext = planContext; #endif } /* * TranslatedVarsForRteIdentity gets an rteIdentity and returns the * translatedVars that belong to the range table relation. If no * translatedVars found, the function returns NIL; */ List * TranslatedVarsForRteIdentity(int rteIdentity) { PlannerRestrictionContext *currentPlannerRestrictionContext = CurrentPlannerRestrictionContext(); List *relationRestrictionList = currentPlannerRestrictionContext->relationRestrictionContext-> relationRestrictionList; RelationRestriction *relationRestriction = NULL; foreach_declared_ptr(relationRestriction, relationRestrictionList) { if (GetRTEIdentity(relationRestriction->rte) == rteIdentity) { return relationRestriction->translatedVars; } } return NIL; } /* * CurrentRestrictionContext returns the most recently added * PlannerRestrictionContext from the plannerRestrictionContextList list. */ static PlannerRestrictionContext * CurrentPlannerRestrictionContext(void) { Assert(plannerRestrictionContextList != NIL); PlannerRestrictionContext *plannerRestrictionContext = (PlannerRestrictionContext *) linitial(plannerRestrictionContextList); if (plannerRestrictionContext == NULL) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("planner restriction context stack was empty"), errdetail("Please report this to the Citus core team."))); } return plannerRestrictionContext; } /* * PopPlannerRestrictionContext removes the most recently added restriction contexts from * the planner restriction context list. The function assumes the list is not empty. */ static void PopPlannerRestrictionContext(void) { #if PG_VERSION_NUM >= PG_VERSION_18 /* * PG18+: Clear the restriction context's planContext pointer; this is done * by distributed_planner() when popping the context, but in case of error * during standard_planner() we want to clean up here also. */ PlannerRestrictionContext *plannerRestrictionContext = (PlannerRestrictionContext *) linitial(plannerRestrictionContextList); plannerRestrictionContext->planContext = NULL; #endif plannerRestrictionContextList = list_delete_first(plannerRestrictionContextList); } /* * ResetPlannerRestrictionContext resets the element of the given planner * restriction context. */ static void ResetPlannerRestrictionContext(PlannerRestrictionContext *plannerRestrictionContext) { plannerRestrictionContext->relationRestrictionContext = palloc0(sizeof(RelationRestrictionContext)); plannerRestrictionContext->joinRestrictionContext = palloc0(sizeof(JoinRestrictionContext)); plannerRestrictionContext->fastPathRestrictionContext = palloc0(sizeof(FastPathRestrictionContext)); /* we'll apply logical AND as we add tables */ plannerRestrictionContext->relationRestrictionContext->allReferenceTables = true; } /* * HasUnresolvedExternParamsWalker returns true if the passed in expression * has external parameters that are not contained in boundParams, false * otherwise. */ bool HasUnresolvedExternParamsWalker(Node *expression, ParamListInfo boundParams) { if (expression == NULL) { return false; } if (IsA(expression, Param)) { Param *param = (Param *) expression; int paramId = param->paramid; /* only care about user supplied parameters */ if (param->paramkind != PARAM_EXTERN) { return false; } /* check whether parameter is available (and valid) */ if (boundParams && paramId > 0 && paramId <= boundParams->numParams) { Oid paramType = InvalidOid; /* give hook a chance in case parameter is dynamic */ if (boundParams->paramFetch != NULL) { ParamExternData externParamPlaceholder; paramType = (*boundParams->paramFetch)(boundParams, paramId, false, &externParamPlaceholder)->ptype; } else { paramType = boundParams->params[paramId - 1].ptype; } if (OidIsValid(paramType)) { return false; } } return true; } /* keep traversing */ if (IsA(expression, Query)) { return query_tree_walker((Query *) expression, HasUnresolvedExternParamsWalker, boundParams, 0); } else { return expression_tree_walker(expression, HasUnresolvedExternParamsWalker, boundParams); } } /* * ContainsSingleShardTable returns true if given query contains reference * to a single-shard table. */ bool ContainsSingleShardTable(Query *query) { RTEListProperties *rteListProperties = GetRTEListPropertiesForQuery(query); return rteListProperties->hasSingleShardDistTable; } /* * GetRTEListPropertiesForQuery is a wrapper around GetRTEListProperties that * returns RTEListProperties for the rte list retrieved from query. */ RTEListProperties * GetRTEListPropertiesForQuery(Query *query) { List *rteList = ExtractRangeTableEntryList(query); return GetRTEListProperties(rteList); } /* * GetRTEListProperties returns RTEListProperties struct processing the given * rangeTableList. */ static RTEListProperties * GetRTEListProperties(List *rangeTableList) { RTEListProperties *rteListProperties = palloc0(sizeof(RTEListProperties)); RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (rangeTableEntry->rtekind != RTE_RELATION) { continue; } else if (rangeTableEntry->relkind == RELKIND_VIEW) { /* * Skip over views, distributed tables within (regular) views are * already in rangeTableList. */ continue; } if (rangeTableEntry->relkind == RELKIND_MATVIEW) { /* * Record materialized views as they are similar to postgres local tables * but it is nice to record them separately. * * Regular tables, partitioned tables or foreign tables can be a local or * distributed tables and we can qualify them accurately. * * For regular views, we don't care because their definitions are already * in the same query tree and we can detect what is inside the view definition. * * For materialized views, they are just local tables in the queries. But, when * REFRESH MATERIALIZED VIEW is used, they behave similar to regular views, adds * the view definition to the query. Hence, it is useful to record it seperately * and let the callers decide on what to do. */ rteListProperties->hasMaterializedView = true; continue; } Oid relationId = rangeTableEntry->relid; CitusTableCacheEntry *cacheEntry = LookupCitusTableCacheEntry(relationId); if (!cacheEntry) { rteListProperties->hasPostgresLocalTable = true; } else if (IsCitusTableTypeCacheEntry(cacheEntry, REFERENCE_TABLE)) { rteListProperties->hasReferenceTable = true; } else if (IsCitusTableTypeCacheEntry(cacheEntry, CITUS_LOCAL_TABLE)) { rteListProperties->hasCitusLocalTable = true; } else if (IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE)) { rteListProperties->hasDistributedTable = true; if (!HasDistributionKeyCacheEntry(cacheEntry)) { rteListProperties->hasSingleShardDistTable = true; } else { rteListProperties->hasDistTableWithShardKey = true; } } else { /* it's not expected, but let's do a bug catch here */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("encountered with an unexpected citus " "table type while processing range table " "entries of query"))); } } rteListProperties->hasCitusTable = (rteListProperties->hasDistributedTable || rteListProperties->hasReferenceTable || rteListProperties->hasCitusLocalTable); return rteListProperties; } /* * WarnIfListHasForeignDistributedTable iterates the given list and logs a WARNING * if the given relation is a distributed foreign table. * We do that because now we only support Citus Local Tables for foreign tables. */ static void WarnIfListHasForeignDistributedTable(List *rangeTableList) { static bool DistributedForeignTableWarningPrompted = false; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (DistributedForeignTableWarningPrompted) { return; } Oid relationId = rangeTableEntry->relid; if (IsForeignTable(relationId) && IsCitusTable(relationId) && !IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { DistributedForeignTableWarningPrompted = true; ereport(WARNING, (errmsg( "support for distributed foreign tables are deprecated, " "please use Citus managed local tables"), (errdetail( "Foreign tables can be added to metadata using UDF: " "citus_add_local_table_to_metadata()")))); } } } static bool CheckPostPlanDistribution(DistributedPlanningContext *planContext, bool isDistributedQuery, List *rangeTableList) { if (isDistributedQuery) { Query *origQuery = planContext->originalQuery; Query *plannedQuery = planContext->query; Node *origQuals = origQuery->jointree->quals; Node *plannedQuals = plannedQuery->jointree->quals; #if PG_VERSION_NUM >= PG_VERSION_17 if (IsMergeQuery(origQuery)) { origQuals = origQuery->mergeJoinCondition; plannedQuals = plannedQuery->mergeJoinCondition; } #endif /* * If the WHERE quals have been eliminated by the Postgres planner, possibly * by an OR clause that was simplified to TRUE, we need to check if the * planned query still requires distributed planning. */ if (origQuals != NULL && plannedQuals == NULL) { /* First check if the plan has a distributed table */ bool planHasDistribution = ListContainsDistributedTableRTE( planContext->plan->rtable, NULL); /* ..or a distributed subplan */ planHasDistribution = planHasDistribution || PlanContainsDistributedSubPlanRTE( planContext->plan->subplans); /* * The plan has a distributed relation, so we know for sure that * the query requires distributed planning. */ if (planHasDistribution) { return true; } /* * Otherwise, if the query has less range table entries after Postgres, * planning, we should re-evaluate the distribution of the query. Postgres * may have optimized away all citus tables, per issues 7782, 7783. */ List *rtesPostPlan = ExtractRangeTableEntryList(plannedQuery); if (list_length(rtesPostPlan) < list_length(rangeTableList)) { bool hasDistTable = ListContainsDistributedTableRTE( rtesPostPlan, NULL); if (hasDistTable != isDistributedQuery) { ereport(DEBUG4, (errmsg( "Plan has flipped from distributed to local " "after Postgres planning, updating distributed to %u", hasDistTable))); isDistributedQuery = hasDistTable; } } } } return isDistributedQuery; } #if PG_VERSION_NUM >= PG_VERSION_18 /* * DisableSelfJoinElimination is used to prevent self join elimination * during distributed query planning to ensure shard queries are correctly * generated. PG18's self join elimination (fc069a3a6) changes the Query * in a way that can cause problems for queries with a mix of Citus and * Postgres tables. Self join elimination is allowed on Postgres tables * only so queries involving shards get the benefit of it. */ static int DisableSelfJoinElimination(void) { int NestLevel = NewGUCNestLevel(); set_config_option("enable_self_join_elimination", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); return NestLevel; } #endif ================================================ FILE: src/backend/distributed/planner/extended_op_node_utils.c ================================================ /*------------------------------------------------------------------------- * * extended_op_node_utils.c implements the logic for building the necessary * information that is shared among both the worker and master extended * op nodes. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "pg_version_constants.h" #include "distributed/extended_op_node_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/pg_dist_partition.h" static bool GroupedByPartitionColumn(MultiNode *node, MultiExtendedOp *opNode); static bool ExtendedOpNodeContainsRepartitionSubquery(MultiExtendedOp *originalOpNode); static bool HasNonPartitionColumnDistinctAgg(List *targetEntryList, Node *havingQual, List *tableNodeList); static bool PartitionColumnInTableList(Var *column, List *tableNodeList); static bool ShouldPullDistinctColumn(bool repartitionSubquery, bool groupedByDisjointPartitionColumn, bool hasNonPartitionColumnDistinctAgg, bool onlyPushableWindowFunctions); static bool CanPushDownGroupingAndHaving(bool pullUpIntermediateRows, bool groupedByDisjointPartitionColumn, bool hasWindowFuncs, bool onlyPushableWindowFunctions); /* * BuildExtendedOpNodeProperties is a helper function that simply builds * the necessary information for processing the extended op node. The return * value should be used in a read-only manner. */ ExtendedOpNodeProperties BuildExtendedOpNodeProperties(MultiExtendedOp *extendedOpNode, bool hasNonDistributableAggregates) { ExtendedOpNodeProperties extendedOpNodeProperties; List *tableNodeList = FindNodesOfType((MultiNode *) extendedOpNode, T_MultiTable); bool groupedByDisjointPartitionColumn = GroupedByPartitionColumn((MultiNode *) extendedOpNode, extendedOpNode); bool pullUpIntermediateRows = !groupedByDisjointPartitionColumn && hasNonDistributableAggregates; bool repartitionSubquery = ExtendedOpNodeContainsRepartitionSubquery(extendedOpNode); List *targetList = extendedOpNode->targetList; Node *havingQual = extendedOpNode->havingQual; bool hasNonPartitionColumnDistinctAgg = HasNonPartitionColumnDistinctAgg(targetList, havingQual, tableNodeList); bool pushDownGroupingAndHaving = CanPushDownGroupingAndHaving(pullUpIntermediateRows, groupedByDisjointPartitionColumn, extendedOpNode->hasWindowFuncs, extendedOpNode->onlyPushableWindowFunctions); bool pullDistinctColumns = ShouldPullDistinctColumn(repartitionSubquery, groupedByDisjointPartitionColumn, hasNonPartitionColumnDistinctAgg, extendedOpNode->onlyPushableWindowFunctions); extendedOpNodeProperties.hasGroupBy = extendedOpNode->groupClauseList != NIL; extendedOpNodeProperties.hasAggregate = TargetListHasAggregates(targetList); extendedOpNodeProperties.groupedByDisjointPartitionColumn = groupedByDisjointPartitionColumn; extendedOpNodeProperties.repartitionSubquery = repartitionSubquery; extendedOpNodeProperties.hasNonPartitionColumnDistinctAgg = hasNonPartitionColumnDistinctAgg; extendedOpNodeProperties.pullDistinctColumns = pullDistinctColumns; extendedOpNodeProperties.pullUpIntermediateRows = pullUpIntermediateRows; extendedOpNodeProperties.hasWindowFuncs = extendedOpNode->hasWindowFuncs; extendedOpNodeProperties.onlyPushableWindowFunctions = extendedOpNode->onlyPushableWindowFunctions; extendedOpNodeProperties.pushDownGroupingAndHaving = pushDownGroupingAndHaving; return extendedOpNodeProperties; } /* * GroupedByPartitionColumn returns true if a GROUP BY in the opNode contains * the partition column of the underlying relation, which is determined by * searching the MultiNode tree for a MultiTable and MultiPartition with * a matching column. * * When there is a re-partition join, the search terminates at the * MultiPartition node. Hence we can push down the GROUP BY if the join * column is in the GROUP BY. */ static bool GroupedByPartitionColumn(MultiNode *node, MultiExtendedOp *opNode) { if (node == NULL) { return false; } if (CitusIsA(node, MultiTable)) { MultiTable *tableNode = (MultiTable *) node; Oid relationId = tableNode->relationId; if (relationId == SUBQUERY_RELATION_ID) { /* ignore subqueries for now */ return false; } else if (relationId != SUBQUERY_PUSHDOWN_RELATION_ID) { if (!IsCitusTableType(relationId, STRICTLY_PARTITIONED_DISTRIBUTED_TABLE)) { /* only range- and hash-distributed tables are strictly partitioned */ return false; } } if (GroupedByColumn(opNode->groupClauseList, opNode->targetList, tableNode->partitionColumn)) { /* this node is partitioned by a column in the GROUP BY */ return true; } } else if (CitusIsA(node, MultiPartition)) { MultiPartition *partitionNode = (MultiPartition *) node; if (GroupedByColumn(opNode->groupClauseList, opNode->targetList, partitionNode->partitionColumn)) { /* this node is partitioned by a column in the GROUP BY */ return true; } } else if (UnaryOperator(node)) { MultiNode *childNode = ((MultiUnaryNode *) node)->childNode; if (GroupedByPartitionColumn(childNode, opNode)) { /* a child node is partitioned by a column in the GROUP BY */ return true; } } else if (BinaryOperator(node)) { MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; MultiNode *rightChildNode = ((MultiBinaryNode *) node)->rightChildNode; if (GroupedByPartitionColumn(leftChildNode, opNode) || GroupedByPartitionColumn(rightChildNode, opNode)) { /* a child node is partitioned by a column in the GROUP BY */ return true; } } return false; } /* * ExtendedOpNodeContainsRepartitionSubquery is a utility function that * returns true if the extended op node contains a re-partition subquery. */ static bool ExtendedOpNodeContainsRepartitionSubquery(MultiExtendedOp *originalOpNode) { MultiNode *parentNode = ParentNode((MultiNode *) originalOpNode); MultiNode *childNode = ChildNode((MultiUnaryNode *) originalOpNode); if (CitusIsA(parentNode, MultiTable) && CitusIsA(childNode, MultiCollect)) { return true; } return false; } /* * HasNonPartitionColumnDistinctAgg returns true if target entry or having qualifier * has non-partition column reference in aggregate (distinct) definition. Note that, * it only checks aggs subfield of Aggref, it does not check FILTER or SORT clauses. * Having any non-column reference like operator expression, function call, or const * is considered as a non-partition column. Even if the expression contains partition column * like (column + 1), it needs to be evaluated at coordinator, since we can't reliably verify * the distinctness of the expression result like (column % 5) or (column + column). */ static bool HasNonPartitionColumnDistinctAgg(List *targetEntryList, Node *havingQual, List *tableNodeList) { List *targetVarList = pull_var_clause((Node *) targetEntryList, PVC_INCLUDE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS); /* having clause can't have window functions, no need to recurse for that */ List *havingVarList = pull_var_clause((Node *) havingQual, PVC_INCLUDE_AGGREGATES); List *aggregateCheckList = list_concat(targetVarList, havingVarList); ListCell *aggregateCheckCell = NULL; foreach(aggregateCheckCell, aggregateCheckList) { Node *targetNode = lfirst(aggregateCheckCell); ListCell *varCell = NULL; bool isPartitionColumn = false; if (!IsA(targetNode, Aggref)) { continue; } Aggref *targetAgg = castNode(Aggref, targetNode); if (targetAgg->aggdistinct == NIL) { continue; } /* * We are dealing with a more complex count distinct, it needs to be * evaluated at coordinator level. */ if (list_length(targetAgg->args) > 1 || list_length(targetAgg->aggdistinct) > 1) { return true; } TargetEntry *firstTargetEntry = linitial_node(TargetEntry, targetAgg->args); Node *firstTargetExprNode = strip_implicit_coercions( (Node *) firstTargetEntry->expr); if (!IsA(firstTargetExprNode, Var)) { return true; } List *varList = pull_var_clause_default((Node *) targetAgg->args); foreach(varCell, varList) { Node *targetVar = (Node *) lfirst(varCell); Assert(IsA(targetVar, Var)); isPartitionColumn = PartitionColumnInTableList((Var *) targetVar, tableNodeList); if (!isPartitionColumn) { return true; } } } return false; } /* * PartitionColumnInTableList returns true if provided column is a partition * column from provided table node list. It also returns false if a column is * partition column of an append distributed table. */ static bool PartitionColumnInTableList(Var *column, List *tableNodeList) { ListCell *tableNodeCell = NULL; foreach(tableNodeCell, tableNodeList) { MultiTable *tableNode = lfirst(tableNodeCell); Var *partitionColumn = tableNode->partitionColumn; if (partitionColumn != NULL && partitionColumn->varno == column->varno && partitionColumn->varattno == column->varattno) { if (!IsCitusTableType(tableNode->relationId, APPEND_DISTRIBUTED)) { return true; } } } return false; } /* * ShouldPullDistinctColumn returns true if distinct aggregate should pull * individual columns from worker to coordinator and evaluate aggregate operation * on the coordinator. * * Pull cases are: * - repartition subqueries * - query has count distinct on a non-partition column on at least one target * - count distinct is on a non-partition column and query is not * grouped on partition column */ static bool ShouldPullDistinctColumn(bool repartitionSubquery, bool groupedByDisjointPartitionColumn, bool hasNonPartitionColumnDistinctAgg, bool onlyPushableWindowFunctions) { if (repartitionSubquery) { return true; } /* don't pull distinct columns when it can be pushed down */ if (onlyPushableWindowFunctions && groupedByDisjointPartitionColumn) { return false; } else if (hasNonPartitionColumnDistinctAgg) { return true; } return false; } /* * CanPushDownGroupingAndHaving returns whether GROUP BY & HAVING should be * pushed down to worker. */ static bool CanPushDownGroupingAndHaving(bool pullUpIntermediateRows, bool groupedByDisjointPartitionColumn, bool hasWindowFuncs, bool onlyPushableWindowFunctions) { /* don't push down if we're pulling up */ if (pullUpIntermediateRows) { return false; } /* * If grouped by a partition column we can push down the having qualifier. * * When a query with subquery is provided, we can't determine if * groupedByDisjointPartitionColumn, therefore we also check if there is a * window function too. If there is a window function we would know that it * is safe to push down (i.e. it is partitioned on distribution column, and * if there is a group by, it contains distribution column). */ return groupedByDisjointPartitionColumn || (hasWindowFuncs && onlyPushableWindowFunctions); } ================================================ FILE: src/backend/distributed/planner/fast_path_router_planner.c ================================================ /*------------------------------------------------------------------------- * * fast_path_router_planner.c * * Planning logic for fast path router planner queries. In this context, * we define "Fast Path Planning" as trivial queries where Citus * can skip relying on the standard_planner() and handle all the planning. * * For router planner, standard_planner() is mostly important to generate * the necessary restriction information. Later, the restriction information * generated by the standard_planner is used to decide whether all the shards * that a distributed query touches reside on a single worker node. However, * standard_planner() does a lot of extra things such as cost estimation and * execution path generations which are completely unnecessary in the context * of distributed planning. * * There are certain types of queries where Citus could skip relying on * standard_planner() to generate the restriction information. For queries * in the following format, Citus does not need any information that the * standard_planner() generates: * SELECT ... FROM single_table WHERE distribution_key = X; or * DELETE FROM single_table WHERE distribution_key = X; or * UPDATE single_table SET value_1 = value_2 + 1 WHERE distribution_key = X; * * Note that the queries might not be as simple as the above such that * GROUP BY, WINDOW FUNCIONS, ORDER BY or HAVING etc. are all acceptable. The * only rule is that the query is on a single distributed (or reference) table * and there is a "distribution_key = X;" in the WHERE clause. With that, we * could use to decide the shard that a distributed query touches reside on * a worker node. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "optimizer/optimizer.h" #include "tcop/pquery.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/distributed_planner.h" #include "distributed/insert_select_planner.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_physical_planner.h" /* only to use some utility functions */ #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" bool EnableFastPathRouterPlanner = true; bool EnableLocalFastPathQueryOptimization = true; static bool ColumnAppearsMultipleTimes(Node *quals, Var *distributionKey); static bool DistKeyInSimpleOpExpression(Expr *clause, Var *distColumn, Node **distributionKeyValue); static bool ConjunctionContainsColumnFilter(Node *node, Var *column, Node **distributionKeyValue); /* * FastPathPreprocessParseTree is used to apply transformations on the parse tree * that are expected by the Postgres planner. This is called on both delayed FastPath * and non-delayed FastPath queries. */ void FastPathPreprocessParseTree(Query *parse) { /* * Citus planner relies on some of the transformations on constant * evaluation on the parse tree. */ parse->targetList = (List *) eval_const_expressions(NULL, (Node *) parse->targetList); parse->jointree->quals = (Node *) eval_const_expressions(NULL, (Node *) parse->jointree->quals); } /* * FastPathPlanner is intended to be used instead of standard_planner() for trivial * queries defined by FastPathRouterQuery(). * * The basic idea is that we need a very little of what standard_planner() does for * the trivial queries. So skip calling standard_planner() to save CPU cycles. * */ PlannedStmt * FastPathPlanner(Query *originalQuery, Query *parse, ParamListInfo boundParams) { PlannedStmt *result = GeneratePlaceHolderPlannedStmt(originalQuery); return result; } /* * GeneratePlaceHolderPlannedStmt creates a planned statement which contains * a sequential scan on the relation that is accessed by the input query. * The returned PlannedStmt is not proper (e.g., set_plan_references() is * not called on the plan or the quals are not set), so should not be * passed to the executor directly. This is only useful to have a * placeholder PlannedStmt where target list is properly set. Note that * this is what router executor relies on. * * This function makes the assumption (and the assertion) that * the input query is in the form defined by FastPathRouterQuery(). */ PlannedStmt * GeneratePlaceHolderPlannedStmt(Query *parse) { PlannedStmt *result = makeNode(PlannedStmt); SeqScan *scanNode = makeNode(SeqScan); Plan *plan = &(scanNode->scan.plan); FastPathRestrictionContext fprCtxt PG_USED_FOR_ASSERTS_ONLY = { 0 }; Assert(FastPathRouterQuery(parse, &fprCtxt)); /* there is only a single relation rte */ scanNode->scan.scanrelid = 1; plan->targetlist = copyObject(FetchStatementTargetList((Node *) parse)); plan->qual = NULL; plan->lefttree = NULL; plan->righttree = NULL; plan->plan_node_id = 1; /* rtable is used for access permission checks */ result->commandType = parse->commandType; result->queryId = parse->queryId; result->stmt_len = parse->stmt_len; result->rtable = copyObject(parse->rtable); result->permInfos = copyObject(parse->rteperminfos); result->planTree = (Plan *) plan; result->hasReturning = (parse->returningList != NIL); Oid relationId = ExtractFirstCitusTableId(parse); result->relationOids = list_make1_oid(relationId); return result; } /* * InitializeFastPathContext - helper function to initialize a FastPath * restriction context with the details that the FastPath code path needs. */ static void InitializeFastPathContext(FastPathRestrictionContext *fastPathContext, Node *distributionKeyValue, bool canAvoidDeparse, Query *query) { Assert(fastPathContext != NULL); Assert(!fastPathContext->fastPathRouterQuery); Assert(!fastPathContext->delayFastPathPlanning); /* * We're looking at a fast path query, so we can fill the * fastPathContext with relevant details. */ fastPathContext->fastPathRouterQuery = true; if (distributionKeyValue == NULL) { /* nothing to record */ } else if (IsA(distributionKeyValue, Const)) { fastPathContext->distributionKeyValue = (Const *) distributionKeyValue; } else if (IsA(distributionKeyValue, Param)) { fastPathContext->distributionKeyHasParam = true; } /* * If local execution and the fast path optimization to * avoid deparse are enabled, and it is safe to do local * execution.. */ if (EnableLocalFastPathQueryOptimization && EnableLocalExecution && GetCurrentLocalExecutionStatus() != LOCAL_EXECUTION_DISABLED) { /* * .. we can delay fast path planning until we know whether * or not the shard is local. Make a final check for volatile * functions in the query tree to determine if we should delay * the fast path planning. */ fastPathContext->delayFastPathPlanning = canAvoidDeparse && !FindNodeMatchingCheckFunction( (Node *) query, CitusIsVolatileFunction); } } /* * FastPathRouterQuery gets a query and returns true if the query is eligible for * being a fast path router query. It also fills the given fastPathContext with * details about the query such as the distribution key value (if available), * whether the distribution key is a parameter, and the range table entry for the * table being queried. * The requirements for the fast path query can be listed below: * * - SELECT/UPDATE/DELETE query without CTES, sublinks-subqueries, set operations * - The query should touch only a single hash distributed or reference table * - The distribution with equality operator should be in the WHERE clause * and it should be ANDed with any other filters. Also, the distribution * key should only exist once in the WHERE clause. So basically, * SELECT ... FROM dist_table WHERE dist_key = X * If the filter is a const, distributionKeyValue is set * - All INSERT statements (including multi-row INSERTs) as long as the commands * don't have any sublinks/CTEs etc * - */ bool FastPathRouterQuery(Query *query, FastPathRestrictionContext *fastPathContext) { if (!EnableFastPathRouterPlanner) { return false; } if (IsMergeQuery(query)) { /* MERGE command is not a fast path query */ return false; } /* * We want to deal with only very simple queries. Some of the * checks might be too restrictive, still we prefer this way. */ if (query->cteList != NIL || query->hasSubLinks || query->setOperations != NULL || query->hasTargetSRFs || query->hasModifyingCTE) { return false; } if (CheckInsertSelectQuery(query)) { /* we don't support INSERT..SELECT in the fast-path */ return false; } else if (query->commandType == CMD_INSERT) { /* we don't need to do any further checks, all INSERTs are fast-path */ InitializeFastPathContext(fastPathContext, NULL, true, query); return true; } int numFromRels = list_length(query->rtable); /* make sure that there is only one range table in FROM clause */ if ((numFromRels != 1) #if PG_VERSION_NUM >= PG_VERSION_18 /* with a PG18+ twist for GROUP rte - if present make sure there's two range tables */ && (!query->hasGroupRTE || numFromRels != 2) #endif ) { return false; } RangeTblEntry *rangeTableEntry = (RangeTblEntry *) linitial(query->rtable); if (rangeTableEntry->rtekind != RTE_RELATION) { return false; } /* we don't want to deal with append/range distributed tables */ Oid distributedTableId = rangeTableEntry->relid; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); if (IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, APPEND_DISTRIBUTED)) { return false; } bool isFastPath = false; bool canAvoidDeparse = false; Node *distributionKeyValue = NULL; /* * If the table doesn't have a distribution column, we don't need to * check anything further. */ Var *distributionKey = PartitionColumn(distributedTableId, 1); if (!distributionKey) { /* * Local execution may avoid a deparse on single shard distributed tables or * citus local tables. We don't yet support reference tables in this code-path * because modifications on reference tables are complicated to support here. */ canAvoidDeparse = IsCitusTableTypeCacheEntry(cacheEntry, SINGLE_SHARD_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, CITUS_LOCAL_TABLE); isFastPath = true; } else { FromExpr *joinTree = query->jointree; Node *quals = NULL; canAvoidDeparse = IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE); if (joinTree == NULL || (joinTree->quals == NULL && canAvoidDeparse)) { /* no quals, not a fast path query */ return false; } quals = joinTree->quals; if (quals != NULL && IsA(quals, List)) { quals = (Node *) make_ands_explicit((List *) quals); } /* * Distribution column must be used in a simple equality match check and it must be * place at top level conjunction operator. In simple words, we should have * WHERE dist_key = VALUE [AND ....]; * * We're also not allowing any other appearances of the distribution key in the quals. * * Overall the logic might sound fuzzy since it involves two individual checks: * (a) Check for top level AND operator with one side being "dist_key = const" * (b) Only allow single appearance of "dist_key" in the quals * * This is to simplify both of the individual checks and omit various edge cases * that might arise with multiple distribution keys in the quals. */ isFastPath = (ConjunctionContainsColumnFilter(quals, distributionKey, &distributionKeyValue) && !ColumnAppearsMultipleTimes(quals, distributionKey)); } if (isFastPath) { InitializeFastPathContext(fastPathContext, distributionKeyValue, canAvoidDeparse, query); } return isFastPath; } /* * ColumnAppearsMultipleTimes returns true if the given input * appears more than once in the quals. */ static bool ColumnAppearsMultipleTimes(Node *quals, Var *distributionKey) { ListCell *varClauseCell = NULL; int partitionColumnReferenceCount = 0; /* make sure partition column is used only once in the quals */ List *varClauseList = pull_var_clause_default(quals); foreach(varClauseCell, varClauseList) { Var *column = (Var *) lfirst(varClauseCell); if (equal(column, distributionKey)) { partitionColumnReferenceCount++; if (partitionColumnReferenceCount > 1) { return true; } } } return false; } /* * ConjunctionContainsColumnFilter returns true if the query contains an exact * match (equal) expression on the provided column. The function returns true only * if the match expression has an AND relation with the rest of the expression tree. * * If the conjuction contains column filter which is const, distributionKeyValue is set. */ static bool ConjunctionContainsColumnFilter(Node *node, Var *column, Node **distributionKeyValue) { if (node == NULL) { return false; } if (IsA(node, OpExpr)) { OpExpr *opExpr = (OpExpr *) node; bool distKeyInSimpleOpExpression = DistKeyInSimpleOpExpression((Expr *) opExpr, column, distributionKeyValue); if (!distKeyInSimpleOpExpression) { return false; } return OperatorImplementsEquality(opExpr->opno); } else if (IsA(node, BoolExpr)) { BoolExpr *boolExpr = (BoolExpr *) node; List *argumentList = boolExpr->args; ListCell *argumentCell = NULL; /* * We do not descend into boolean expressions other than AND. * If the column filter appears in an OR clause, we do not * consider it even if it is logically the same as a single value * comparison (e.g. ` = OR false`) */ if (boolExpr->boolop != AND_EXPR) { return false; } foreach(argumentCell, argumentList) { Node *argumentNode = (Node *) lfirst(argumentCell); if (ConjunctionContainsColumnFilter(argumentNode, column, distributionKeyValue)) { return true; } } } return false; } /* * DistKeyInSimpleOpExpression checks whether given expression is a simple operator * expression with either (dist_key = param) or (dist_key = const). Note that the * operands could be in the reverse order as well. * * When a const is found, distributionKeyValue is set. */ static bool DistKeyInSimpleOpExpression(Expr *clause, Var *distColumn, Node **distributionKeyValue) { Param *paramClause = NULL; Const *constantClause = NULL; Var *columnInExpr = NULL; Node *leftOperand; Node *rightOperand; if (!BinaryOpExpression(clause, &leftOperand, &rightOperand)) { return false; } if (IsA(rightOperand, Param) && IsA(leftOperand, Var)) { paramClause = (Param *) rightOperand; columnInExpr = (Var *) leftOperand; } else if (IsA(leftOperand, Param) && IsA(rightOperand, Var)) { paramClause = (Param *) leftOperand; columnInExpr = (Var *) rightOperand; } else if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) { constantClause = (Const *) rightOperand; columnInExpr = (Var *) leftOperand; } else if (IsA(leftOperand, Const) && IsA(rightOperand, Var)) { constantClause = (Const *) leftOperand; columnInExpr = (Var *) rightOperand; } else { return false; } if (paramClause && paramClause->paramkind != PARAM_EXTERN) { /* we can only handle param_externs */ return false; } else if (constantClause && constantClause->constisnull) { /* we can only handle non-null constants */ return false; } /* at this point we should have the columnInExpr */ Assert(columnInExpr); bool distColumnExists = equal(distColumn, columnInExpr); if (distColumnExists && constantClause != NULL && distColumn->vartype == constantClause->consttype && *distributionKeyValue == NULL) { /* if the vartypes do not match, let shard pruning handle it later */ *distributionKeyValue = (Node *) copyObject(constantClause); } else if (paramClause != NULL) { *distributionKeyValue = (Node *) copyObject(paramClause); } return distColumnExists; } ================================================ FILE: src/backend/distributed/planner/function_call_delegation.c ================================================ /*------------------------------------------------------------------------- * * function_call_delegation.c * Planning logic for delegating a function call to a worker when the * function was distributed with a distribution argument and the worker * has metadata. * * Copyright (c), Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" #include "parser/parse_coerce.h" #include "parser/parsetree.h" #include "tcop/dest.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/function_call_delegation.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/recursive_planning.h" #include "distributed/remote_commands.h" #include "distributed/shard_pruning.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" struct ParamWalkerContext { bool hasParam; ParamKind paramKind; }; extern AllowedDistributionColumn AllowedDistributionColumnValue; static bool contain_param_walker(Node *node, void *context); static void CheckDelegatedFunctionExecution(DistObjectCacheEntry *procedure, FuncExpr *funcExpr); static bool IsQuerySimple(Query *query); static FuncExpr * FunctionInFromClause(List *fromlist, Query *query); static void EnableInForceDelegatedFuncExecution(Const *distArgument, uint32 colocationId); /* global variable keeping track of whether we are in a delegated function call */ bool InTopLevelDelegatedFunctionCall = false; /* global variable keeping track of whether we are in a delegated function call */ bool InDelegatedFunctionCall = false; /* * contain_param_walker scans node for Param nodes. * Ignore the return value, instead check context afterwards. * * context is a struct ParamWalkerContext*. * hasParam is set to true if we find a Param node. * paramKind is set to the paramkind of the Param node if any found. * paramKind is set to PARAM_EXEC if both PARAM_EXEC & PARAM_EXTERN are found. * * By time we walk, Param nodes are either PARAM_EXTERN or PARAM_EXEC. */ static bool contain_param_walker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Param)) { Param *paramNode = (Param *) node; struct ParamWalkerContext *pwcontext = (struct ParamWalkerContext *) context; pwcontext->hasParam = true; pwcontext->paramKind = paramNode->paramkind; return paramNode->paramkind == PARAM_EXEC; } else { return expression_tree_walker((Node *) node, contain_param_walker, context); } } /* * TryToDelegateFunctionCall calls a function on the worker if possible. * We only support delegating the SELECT func(...) form for distributed * functions colocated by distributed tables, and not more complicated * forms involving multiple function calls, FROM clauses, WHERE clauses, * ... Those complex forms are handled in the coordinator. */ PlannedStmt * TryToDelegateFunctionCall(DistributedPlanningContext *planContext) { ShardPlacement *placement = NULL; struct ParamWalkerContext walkerParamContext = { 0 }; bool inTransactionBlock = false; if (!CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG4)) { /* Citus is not ready to determine whether function is distributed */ return NULL; } int32 localGroupId = GetLocalGroupId(); if (localGroupId == GROUP_ID_UPGRADING) { /* do not delegate while upgrading */ return NULL; } if (planContext->query == NULL) { /* no query (mostly here to be defensive) */ return NULL; } if (planContext->query->commandType != CMD_SELECT) { /* not a SELECT */ return NULL; } FromExpr *joinTree = planContext->query->jointree; if (joinTree == NULL) { /* no join tree (mostly here to be defensive) */ return NULL; } if (joinTree->quals != NULL) { /* query has a WHERE section */ return NULL; } FuncExpr *fromFuncExpr = NULL; if (joinTree->fromlist != NIL) { if (list_length(joinTree->fromlist) != 1) { /* e.g. SELECT ... FROM rel1, rel2. */ Assert(list_length(joinTree->fromlist) > 1); return NULL; } /* * In the planning phase empty FROMs are represented with an RTE_RESULT. * When we arrive here, standard_planner has already been called which calls * replace_empty_jointree() which replaces empty fromlist with a list of * single RTE_RESULT RangleTableRef node. */ RangeTblRef *reference = linitial(joinTree->fromlist); if (IsA(reference, RangeTblRef)) { RangeTblEntry *rtentry = rt_fetch(reference->rtindex, planContext->query->rtable); if (rtentry->rtekind == RTE_FUNCTION) { /* * Look for a function in the FROM clause. */ fromFuncExpr = FunctionInFromClause(joinTree->fromlist, planContext->query); } else if (rtentry->rtekind != RTE_RESULT) { /* e.g. SELECT f() FROM rel */ ereport(DEBUG4, (errmsg("FromList item is not empty"))); return NULL; } } else { /* * e.g. IsA(reference, JoinExpr). This is explicit join expressions * like INNER JOIN, NATURAL JOIN, ... */ return NULL; } } FuncExpr *targetFuncExpr = NULL; List *targetList = planContext->query->targetList; int targetListLen = list_length(targetList); if (targetListLen == 1) { TargetEntry *targetEntry = (TargetEntry *) linitial(targetList); if (IsA(targetEntry->expr, FuncExpr)) { /* function from the SELECT clause e.g. SELECT fn() FROM */ targetFuncExpr = (FuncExpr *) targetEntry->expr; } } /* * Look for one of: * SELECT fn(...); * SELECT ... FROM fn(...); */ FuncExpr *funcExpr = NULL; if (targetFuncExpr != NULL) { if (fromFuncExpr != NULL) { /* query is of the form: SELECT fn() FROM fn() */ return NULL; } /* query is of the form: SELECT fn(); */ funcExpr = targetFuncExpr; } else if (fromFuncExpr != NULL) { /* query is of the form: SELECT ... FROM fn(); */ funcExpr = fromFuncExpr; } else { /* query does not have a function call in SELECT or FROM */ return NULL; } DistObjectCacheEntry *procedure = LookupDistObjectCacheEntry(ProcedureRelationId, funcExpr->funcid, 0); if (procedure == NULL || !procedure->isDistributed) { /* not a distributed function call */ return NULL; } else { ereport(DEBUG4, (errmsg("function is distributed"))); } if (IsCitusInternalBackend()) { bool isFunctionForceDelegated = procedure->forceDelegation; /* * We are planning a call to a distributed function within a Citus backend, * that means that this is the delegated call. If the function is forcefully * delegated, capture the distribution argument. */ if (isFunctionForceDelegated) { CheckDelegatedFunctionExecution(procedure, funcExpr); } /* Are we planning the top function call? */ if (ExecutorLevel == 0 && PlannerLevel == 1) { /* * InTopLevelDelegatedFunctionCall flag grants the levy * to do remote tasks from a delegated function. */ if (!isFunctionForceDelegated) { /* * we are planning a regular delegated call, we * are allowed to do remote execution. */ InTopLevelDelegatedFunctionCall = true; } else if (!IsMultiStatementTransaction()) { /* * we are planning a force-delegated call, we * are allowed to do remote execution if there * is no explicit BEGIN-END transaction. */ InTopLevelDelegatedFunctionCall = true; } } return NULL; } /* * Cannot delegate functions for INSERT ... SELECT func(), since they require * coordinated transactions. */ if (PlanningInsertSelect()) { ereport(DEBUG1, (errmsg("not pushing down function calls in INSERT ... SELECT"))); return NULL; } /* dissuade the planner from trying a generic plan with parameters */ (void) expression_tree_walker((Node *) funcExpr->args, contain_param_walker, &walkerParamContext); if (walkerParamContext.hasParam) { if (walkerParamContext.paramKind == PARAM_EXTERN) { /* Don't log a message, we should end up here again without a parameter */ DissuadePlannerFromUsingPlan(planContext->plan); } else { ereport(DEBUG1, (errmsg("arguments in a distributed function must " "not contain subqueries"))); } return NULL; } if (IsMultiStatementTransaction()) { if (!procedure->forceDelegation) { /* cannot delegate function calls in a multi-statement transaction */ ereport(DEBUG4, (errmsg("not pushing down function calls in " "a multi-statement transaction"))); return NULL; } else { Node *partitionValueNode = (Node *) list_nth(funcExpr->args, procedure->distributionArgIndex); if (!IsA(partitionValueNode, Const)) { ereport(DEBUG1, (errmsg("distribution argument value must be a " "constant when using force_delegation flag"))); return NULL; } /* * If the expression is simple, such as, SELECT function() or PEFORM function() * in PL/PgSQL code, PL engine does a simple expression evaluation which can't * interpret the Citus CustomScan Node. * Note: Function from FROM clause is not simple, so it's ok to pushdown. */ if ((MaybeExecutingUDF() || DoBlockLevel > 0) && IsQuerySimple(planContext->query) && !fromFuncExpr) { ereport(DEBUG1, (errmsg("Skipping pushdown of function " "from a PL/PgSQL simple expression"))); return NULL; } /* * When is flag is on, delegate the function call in a multi-statement * transaction but with restrictions. */ ereport(DEBUG1, (errmsg("pushing down function call in " "a multi-statement transaction"))); inTransactionBlock = true; } } if (contain_volatile_functions((Node *) funcExpr->args)) { ereport(DEBUG1, (errmsg("arguments in a distributed function must " "be constant expressions"))); return NULL; } Oid colocatedRelationId = ColocatedTableId(procedure->colocationId); if (colocatedRelationId == InvalidOid) { ereport(DEBUG4, (errmsg("function does not have co-located tables"))); return NULL; } /* * This can be called in queries like SELECT ... WHERE EXISTS(SELECT func()), or other * forms of CTEs or subqueries. We don't push-down in those cases. */ if (GeneratingSubplans()) { ereport(DEBUG1, (errmsg( "not pushing down function calls in CTEs or Subqueries"))); return NULL; } CitusTableCacheEntry *distTable = GetCitusTableCacheEntry(colocatedRelationId); if (IsCitusTableType(colocatedRelationId, REFERENCE_TABLE)) { placement = ShardPlacementForFunctionColocatedWithReferenceTable(distTable); } else if (IsCitusTableType(colocatedRelationId, SINGLE_SHARD_DISTRIBUTED)) { placement = ShardPlacementForFunctionColocatedWithSingleShardTable(distTable); } else { placement = ShardPlacementForFunctionColocatedWithDistTable(procedure, funcExpr->args, distTable-> partitionColumn, distTable, planContext->plan); } /* return if we could not find a placement */ if (placement == NULL) { return NULL; } WorkerNode *workerNode = FindWorkerNode(placement->nodeName, placement->nodePort); if (workerNode == NULL || !workerNode->hasMetadata || !workerNode->metadataSynced) { ereport(DEBUG1, (errmsg("the worker node does not have metadata"))); return NULL; } else if (workerNode->groupId == GetLocalGroupId()) { /* If the force_pushdown flag is set, capture the distribution argument */ if (procedure->forceDelegation) { CheckDelegatedFunctionExecution(procedure, funcExpr); } /* * Two reasons for this: * (a) It would lead to infinite recursion as the node would * keep pushing down the procedure as it gets * (b) It doesn't have any value to pushdown as we are already * on the node itself */ ereport(DEBUG1, (errmsg("not pushing down function to the same node"))); return NULL; } ereport(DEBUG1, (errmsg("pushing down the function call"))); Task *task = CitusMakeNode(Task); /* * In a multi-statement block the function should be part of the sorrounding * transaction, at this time, not knowing the operations in the function, it * is safe to assume that it's a write task. * * TODO: We should compile the function to see the internals of the function * and find if this has read-only tasks, does it involve doing a remote task * or queries involving non-distribution column, etc. */ if (inTransactionBlock) { task->taskType = MODIFY_TASK; } else { task->taskType = READ_TASK; } task->taskPlacementList = list_make1(placement); SetTaskQueryIfShouldLazyDeparse(task, planContext->query); task->anchorShardId = placement->shardId; task->replicationModel = distTable->replicationModel; Job *job = CitusMakeNode(Job); job->jobId = UniqueJobId(); job->jobQuery = planContext->query; job->taskList = list_make1(task); DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); distributedPlan->workerJob = job; distributedPlan->combineQuery = NULL; distributedPlan->expectResults = true; /* worker will take care of any necessary locking, treat query as read-only */ distributedPlan->modLevel = ROW_MODIFY_READONLY; return FinalizePlan(planContext->plan, distributedPlan); } /* * ShardPlacementForFunctionColocatedWithDistTable decides on a placement * for delegating a procedure call that accesses a distributed table. */ ShardPlacement * ShardPlacementForFunctionColocatedWithDistTable(DistObjectCacheEntry *procedure, List *argumentList, Var *partitionColumn, CitusTableCacheEntry *cacheEntry, PlannedStmt *plan) { if (procedure->distributionArgIndex < 0 || procedure->distributionArgIndex >= list_length(argumentList)) { ereport(DEBUG1, (errmsg("cannot push down invalid distribution_argument_index"))); return NULL; } Node *partitionValueNode = (Node *) list_nth(argumentList, procedure->distributionArgIndex); partitionValueNode = strip_implicit_coercions(partitionValueNode); if (IsA(partitionValueNode, Param)) { Param *partitionParam = (Param *) partitionValueNode; if (partitionParam->paramkind == PARAM_EXTERN) { /* * Don't log a message, we should end up here again without a * parameter. * Note that "plan" can be null, for example when a CALL statement * is prepared. */ if (plan) { DissuadePlannerFromUsingPlan(plan); } return NULL; } } if (!IsA(partitionValueNode, Const)) { ereport(DEBUG1, (errmsg("distribution argument value must be a constant"))); return NULL; } Const *partitionValue = (Const *) partitionValueNode; if (partitionValue->consttype != partitionColumn->vartype) { bool missingOk = false; partitionValue = TransformPartitionRestrictionValue(partitionColumn, partitionValue, missingOk); } Datum partitionValueDatum = partitionValue->constvalue; ShardInterval *shardInterval = FindShardInterval(partitionValueDatum, cacheEntry); if (shardInterval == NULL) { ereport(DEBUG1, (errmsg("cannot push down call, failed to find shard interval"))); return NULL; } List *placementList = ActiveShardPlacementList(shardInterval->shardId); if (list_length(placementList) != 1) { /* punt on this for now */ ereport(DEBUG1, (errmsg( "cannot push down function call for replicated distributed tables"))); return NULL; } return linitial(placementList); } /* * ShardPlacementForFunctionColocatedWithSingleShardTable decides on a placement * for delegating a function call that reads from a single shard table. */ ShardPlacement * ShardPlacementForFunctionColocatedWithSingleShardTable(CitusTableCacheEntry *cacheEntry) { const ShardInterval *shardInterval = cacheEntry->sortedShardIntervalArray[0]; if (shardInterval == NULL) { ereport(DEBUG1, (errmsg("cannot push down call, failed to find shard interval"))); return NULL; } List *placementList = ActiveShardPlacementList(shardInterval->shardId); if (list_length(placementList) != 1) { /* punt on this for now */ ereport(DEBUG1, (errmsg( "cannot push down function call for replicated distributed tables"))); return NULL; } return (ShardPlacement *) linitial(placementList); } /* * ShardPlacementForFunctionColocatedWithReferenceTable decides on a placement for delegating * a function call that reads from a reference table. * * If citus.task_assignment_policy is set to round-robin, we assign a different placement * on consecutive runs. Otherwise the function returns the first placement available. */ ShardPlacement * ShardPlacementForFunctionColocatedWithReferenceTable(CitusTableCacheEntry *cacheEntry) { const ShardInterval *shardInterval = cacheEntry->sortedShardIntervalArray[0]; const uint64 referenceTableShardId = shardInterval->shardId; /* Get the list of active shard placements ordered by the groupid */ List *placementList = ActiveShardPlacementList(referenceTableShardId); placementList = SortList(placementList, CompareShardPlacementsByGroupId); /* do not try to delegate to coordinator even if it is in metadata */ placementList = RemoveCoordinatorPlacementIfNotSingleNode(placementList); if (TaskAssignmentPolicy == TASK_ASSIGNMENT_ROUND_ROBIN) { /* reorder the placement list */ placementList = RoundRobinReorder(placementList); } return (ShardPlacement *) linitial(placementList); } /* * Checks to see if the procedure is being executed on a worker after delegated * by the coordinator. If the flag forceDelegation is set, capture the distribution * argument value, to be used by the planner to make sure that function uses only * the colocated shards of the distribution argument. */ void CheckDelegatedFunctionExecution(DistObjectCacheEntry *procedure, FuncExpr *funcExpr) { Assert(procedure->forceDelegation); /* * On the coordinator PartiallyEvaluateExpression() descends into an * expression tree to evaluate expressions that can be resolved to a * constant. Expressions containing a Var are skipped, since the value * of the Var is not known on the coordinator. */ Node *partitionValueNode = (Node *) list_nth(funcExpr->args, procedure->distributionArgIndex); Assert(partitionValueNode); partitionValueNode = strip_implicit_coercions(partitionValueNode); if (IsA(partitionValueNode, Param)) { Param *partitionParam = (Param *) partitionValueNode; if (partitionParam->paramkind == PARAM_EXTERN) { /* we should end up here again without a parameter */ return; } } if (IsA(partitionValueNode, Const)) { Const *partitionValueConst = (Const *) partitionValueNode; ereport(DEBUG1, (errmsg("Pushdown argument: %s", pretty_format_node_dump( nodeToString(partitionValueNode))))); EnableInForceDelegatedFuncExecution(partitionValueConst, procedure->colocationId); } } /* * Function returns true if the query is simple enough to skip the full executor * It checks only for expressions in the query clauses, and not WHERE and FROM * lists. */ static bool IsQuerySimple(Query *query) { if (query->hasAggs || query->hasWindowFuncs || query->hasTargetSRFs || query->hasSubLinks || query->cteList || query->groupClause || query->groupingSets || query->havingQual || query->windowClause || query->distinctClause || query->sortClause || query->limitOffset || query->limitCount || query->setOperations) { return false; } return true; } /* * Look for a function in the FROM clause. */ static FuncExpr * FunctionInFromClause(List *fromlist, Query *query) { if (list_length(fromlist) != 1) { /* We are looking for a single function */ return NULL; } RangeTblRef *reference = linitial(fromlist); if (!IsA(reference, RangeTblRef)) { /* Skip if there is no RTE */ return NULL; } RangeTblEntry *rtentry = rt_fetch(reference->rtindex, query->rtable); if (rtentry->rtekind != RTE_FUNCTION) { return NULL; } if (list_length(rtentry->functions) != 1) { /* Skip if RTE isn't a single FuncExpr */ return NULL; } RangeTblFunction *rtfunc = (RangeTblFunction *) linitial(rtentry->functions); if (!IsA(rtfunc->funcexpr, FuncExpr)) { /* Skip if RTE isn't a simple FuncExpr */ return NULL; } return (FuncExpr *) rtfunc->funcexpr; } /* * Sets a flag to true indicating that the current node is executing a delegated * function call, using forceDelegation, within a distributed transaction issued * by the coordinator. Also, saves the distribution argument. */ static void EnableInForceDelegatedFuncExecution(Const *distArgument, uint32 colocationId) { /* * If the distribution key is already set, the key is fixed until * the force-delegation function returns. All nested force-delegation * functions must use the same key. */ if (AllowedDistributionColumnValue.isActive) { return; } /* * The saved distribution argument need to persist through the life * of the query, both during the planning (where we save) and execution * (where we compare) */ MemoryContext oldcontext = MemoryContextSwitchTo(TopTransactionContext); ereport(DEBUG1, errmsg("Saving Distribution Argument: %s:%d", pretty_format_node_dump(nodeToString(distArgument)), colocationId)); AllowedDistributionColumnValue.distributionColumnValue = copyObject(distArgument); AllowedDistributionColumnValue.colocationId = colocationId; AllowedDistributionColumnValue.executorLevel = ExecutorLevel; AllowedDistributionColumnValue.isActive = true; MemoryContextSwitchTo(oldcontext); } /* * Within a 2PC, when a function is delegated to a remote node, we pin * the distribution argument as the shard key for all the SQL in the * function's block. The restriction is imposed to not to access other * nodes from the current node and violate the transactional integrity of * the 2PC. Reset the distribution argument value once the function ends. */ void CheckAndResetAllowedShardKeyValueIfNeeded(void) { /* * If no distribution argument is pinned or the pinned argument was * set by a nested-executor from upper level, nothing to reset. */ if (!AllowedDistributionColumnValue.isActive || ExecutorLevel > AllowedDistributionColumnValue.executorLevel) { return; } Assert(ExecutorLevel == AllowedDistributionColumnValue.executorLevel); pfree(AllowedDistributionColumnValue.distributionColumnValue); AllowedDistributionColumnValue.isActive = false; AllowedDistributionColumnValue.executorLevel = 0; } /* * Function returns true if the current shard key in the adaptive executor * matches the saved distribution argument of a force_delegation function. */ bool IsShardKeyValueAllowed(Const *shardKey, uint32 colocationId) { Assert(AllowedDistributionColumnValue.isActive); Assert(ExecutorLevel > AllowedDistributionColumnValue.executorLevel); ereport(DEBUG4, errmsg( "Comparing saved:%s with Shard key: %s colocationid:%d:%d", pretty_format_node_dump( nodeToString(AllowedDistributionColumnValue. distributionColumnValue)), pretty_format_node_dump(nodeToString(shardKey)), AllowedDistributionColumnValue.colocationId, colocationId)); return (equal(AllowedDistributionColumnValue.distributionColumnValue, shardKey) && (AllowedDistributionColumnValue.colocationId == colocationId)); } ================================================ FILE: src/backend/distributed/planner/insert_select_planner.c ================================================ /*------------------------------------------------------------------------- * * insert_select_planner.c * * Planning logic for INSERT..SELECT. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_class.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_coerce.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/errormessage.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/query_pushdown_planning.h" #include "distributed/recursive_planning.h" #include "distributed/repartition_executor.h" #include "distributed/resource_lock.h" #include "distributed/version_compat.h" static void PrepareInsertSelectForCitusPlanner(Query *insertSelectQuery); static DistributedPlan * CreateInsertSelectPlanInternal(uint64 planId, Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext, ParamListInfo boundParams); static DistributedPlan * CreateDistributedInsertSelectPlan(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); static bool InsertSelectHasRouterSelect(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); static Task * RouterModifyTaskForShardInterval(Query *originalQuery, CitusTableCacheEntry * targetTableCacheEntry, ShardInterval *shardInterval, PlannerRestrictionContext * plannerRestrictionContext, uint32 taskIdIndex, bool allRelationsJoinedOnPartitionKey, DeferredErrorMessage **routerPlannerError); static Query * CreateCombineQueryForRouterPlan(DistributedPlan *distPlan); static List * CreateTargetListForCombineQuery(List *targetList); static DeferredErrorMessage * DistributedInsertSelectSupported(Query *queryTree, RangeTblEntry *insertRte, RangeTblEntry *subqueryRte, bool allReferenceTables, bool routerSelect, PlannerRestrictionContext * plannerRestrictionContext); static DeferredErrorMessage * InsertPartitionColumnMatchesSelect(Query *query, RangeTblEntry *insertRte, RangeTblEntry * subqueryRte, Oid * selectPartitionColumnTableId); static DistributedPlan * CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse, ParamListInfo boundParams); static DeferredErrorMessage * NonPushableInsertSelectSupported(Query *insertSelectQuery); static void RelabelTargetEntryList(List *selectTargetList, List *insertTargetList); static List * AddInsertSelectCasts(List *insertTargetList, List *selectTargetList, Oid targetRelationId); static Expr * CastExpr(Expr *expr, Oid sourceType, Oid targetType, Oid targetCollation, int targetTypeMod); static Oid GetNextvalReturnTypeCatalog(void); static void AppendCastedEntry(TargetEntry *insertEntry, TargetEntry *selectEntry, Oid castFromType, Oid targetType, Oid collation, int32 typmod, int targetEntryIndex, List **projectedEntries, List **nonProjectedEntries); static void SetTargetEntryName(TargetEntry *tle, const char *format, int index); static void ResetTargetEntryResno(List *targetList); static void ProcessEntryPair(TargetEntry *insertEntry, TargetEntry *selectEntry, Form_pg_attribute attr, int targetEntryIndex, List **projectedEntries, List **nonProjectedEntries); /* depth of current insert/select planner. */ static int insertSelectPlannerLevel = 0; /* * InsertSelectIntoCitusTable returns true when the input query is an * INSERT INTO ... SELECT kind of query and the target is a citus * table. * * Note that the input query should be the original parsetree of * the query (i.e., not passed trough the standard planner). */ bool InsertSelectIntoCitusTable(Query *query) { bool insertSelectQuery = CheckInsertSelectQuery(query); if (insertSelectQuery) { RangeTblEntry *insertRte = ExtractResultRelationRTE(query); if (IsCitusTable(insertRte->relid)) { return true; } } return false; } /* * InsertSelectIntoLocalTable checks whether INSERT INTO ... SELECT inserts * into local table. Note that query must be a sample of INSERT INTO ... SELECT * type of query. */ bool InsertSelectIntoLocalTable(Query *query) { bool insertSelectQuery = CheckInsertSelectQuery(query); if (insertSelectQuery) { RangeTblEntry *insertRte = ExtractResultRelationRTE(query); if (!IsCitusTable(insertRte->relid)) { return true; } } return false; } /* * CheckInsertSelectQuery returns true when the input query is an INSERT INTO * ... SELECT kind of query. * * This function is inspired from getInsertSelectQuery() on * rewrite/rewriteManip.c. */ bool CheckInsertSelectQuery(Query *query) { CmdType commandType = query->commandType; if (commandType != CMD_INSERT) { return false; } if (query->jointree == NULL || !IsA(query->jointree, FromExpr)) { return false; } List *fromList = query->jointree->fromlist; if (list_length(fromList) != 1) { return false; } RangeTblRef *rangeTableReference = linitial(fromList); if (!IsA(rangeTableReference, RangeTblRef)) { return false; } RangeTblEntry *subqueryRte = rt_fetch(rangeTableReference->rtindex, query->rtable); if (subqueryRte->rtekind != RTE_SUBQUERY) { return false; } /* ensure that there is a query */ Assert(IsA(subqueryRte->subquery, Query)); return true; } /* * CoordinatorInsertSelectExecScan is a wrapper around * CoordinatorInsertSelectExecScanInternal which also properly increments * or decrements insertSelectExecutorLevel. */ DistributedPlan * CreateInsertSelectPlan(uint64 planId, Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, ParamListInfo boundParams) { DistributedPlan *result = NULL; insertSelectPlannerLevel++; PG_TRY(); { result = CreateInsertSelectPlanInternal(planId, originalQuery, plannerRestrictionContext, boundParams); } PG_CATCH(); { insertSelectPlannerLevel--; PG_RE_THROW(); } PG_END_TRY(); insertSelectPlannerLevel--; return result; } /* * CreateInsertSelectPlan tries to create a distributed plan for an * INSERT INTO distributed_table SELECT ... query by push down the * command to the workers and if that is not possible it creates a * plan for evaluating the SELECT on the coordinator. */ static DistributedPlan * CreateInsertSelectPlanInternal(uint64 planId, Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, ParamListInfo boundParams) { DeferredErrorMessage *deferredError = ErrorIfOnConflictNotSupported(originalQuery); if (deferredError != NULL) { /* raising the error as there is no possible solution for the unsupported on conflict statements */ RaiseDeferredError(deferredError, ERROR); } DistributedPlan *distributedPlan = CreateDistributedInsertSelectPlan(originalQuery, plannerRestrictionContext); if (distributedPlan->planningError != NULL) { RaiseDeferredError(distributedPlan->planningError, DEBUG1); /* * If INSERT..SELECT cannot be distributed, pull to coordinator or use * repartitioning. */ distributedPlan = CreateNonPushableInsertSelectPlan(planId, originalQuery, boundParams); } return distributedPlan; } /* * CreateDistributedInsertSelectPlan creates a DistributedPlan for distributed * INSERT ... SELECT queries which could consist of multiple tasks. * * The function never returns NULL, it errors out if cannot create the DistributedPlan. */ static DistributedPlan * CreateDistributedInsertSelectPlan(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { List *sqlTaskList = NIL; uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint64 jobId = INVALID_JOB_ID; DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); RangeTblEntry *insertRte = ExtractResultRelationRTEOrError(originalQuery); RangeTblEntry *subqueryRte = ExtractSelectRangeTableEntry(originalQuery); Oid targetRelationId = insertRte->relid; CitusTableCacheEntry *targetCacheEntry = GetCitusTableCacheEntry(targetRelationId); int shardCount = targetCacheEntry->shardIntervalArrayLength; RelationRestrictionContext *relationRestrictionContext = plannerRestrictionContext->relationRestrictionContext; bool allReferenceTables = relationRestrictionContext->allReferenceTables; bool routerSelect = InsertSelectHasRouterSelect(copyObject(originalQuery), plannerRestrictionContext); distributedPlan->modLevel = RowModifyLevelForQuery(originalQuery); /* * Error semantics for INSERT ... SELECT queries are different than regular * modify queries. Thus, handle separately. */ distributedPlan->planningError = DistributedInsertSelectSupported(originalQuery, insertRte, subqueryRte, allReferenceTables, routerSelect, plannerRestrictionContext); if (distributedPlan->planningError) { return distributedPlan; } /* * if the query goes to a single node ("router" in Citus' parlance), * we don't need to go through AllDistributionKeysInQueryAreEqual checks. * * For PG16+, this is required as some of the outer JOINs are converted to * "ON(true)" and filters are pushed down to the table scans. As * AllDistributionKeysInQueryAreEqual rely on JOIN filters, it will fail to * detect the router case. However, we can still detect it by checking if * the query is a router query as the router query checks the filters on * the tables. */ bool allDistributionKeysInQueryAreEqual = routerSelect || AllDistributionKeysInQueryAreEqual(originalQuery, plannerRestrictionContext); /* * Plan select query for each shard in the target table. Do so by replacing the * partitioning qual parameter added in distributed_planner() using the current shard's * actual boundary values. Also, add the current shard's boundary values to the * top level subquery to ensure that even if the partitioning qual is not distributed * to all the tables, we never run the queries on the shards that don't match with * the current shard boundaries. Finally, perform the normal shard pruning to * decide on whether to push the query to the current shard or not. */ for (int shardOffset = 0; shardOffset < shardCount; shardOffset++) { ShardInterval *targetShardInterval = targetCacheEntry->sortedShardIntervalArray[shardOffset]; Task *modifyTask = RouterModifyTaskForShardInterval(originalQuery, targetCacheEntry, targetShardInterval, plannerRestrictionContext, taskIdIndex, allDistributionKeysInQueryAreEqual, &distributedPlan-> planningError); if (distributedPlan->planningError != NULL) { return distributedPlan; } /* add the task if it could be created */ if (modifyTask != NULL) { modifyTask->modifyWithSubquery = true; sqlTaskList = lappend(sqlTaskList, modifyTask); } taskIdIndex++; } /* Create the worker job */ Job *workerJob = CitusMakeNode(Job); workerJob->taskList = sqlTaskList; workerJob->subqueryPushdown = false; workerJob->dependentJobList = NIL; workerJob->jobId = jobId; workerJob->jobQuery = originalQuery; workerJob->requiresCoordinatorEvaluation = RequiresCoordinatorEvaluation(originalQuery); /* and finally the multi plan */ distributedPlan->workerJob = workerJob; distributedPlan->combineQuery = NULL; distributedPlan->expectResults = originalQuery->returningList != NIL; distributedPlan->targetRelationId = targetRelationId; return distributedPlan; } /* * InsertSelectHasRouterSelect is a helper function that returns true of the SELECT * part of the INSERT .. SELECT query is a router query. */ static bool InsertSelectHasRouterSelect(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { RangeTblEntry *subqueryRte = ExtractSelectRangeTableEntry(originalQuery); DistributedPlan *distributedPlan = CreateRouterPlan(subqueryRte->subquery, subqueryRte->subquery, plannerRestrictionContext); return distributedPlan->planningError == NULL; } /* * CreateInsertSelectIntoLocalTablePlan creates the plan for INSERT .. SELECT queries * where the selected table is distributed and the inserted table is not. * * To create the plan, this function first creates a distributed plan for the SELECT * part. Then puts it as a subquery to the original (non-distributed) INSERT query as * a subquery. Finally, it puts this INSERT query, which now has a distributed SELECT * subquery, in the combineQuery. * * If the SELECT query is a router query, whose distributed plan does not have a * combineQuery, this function also creates a dummy combineQuery for that. */ DistributedPlan * CreateInsertSelectIntoLocalTablePlan(uint64 planId, Query *insertSelectQuery, ParamListInfo boundParams, bool hasUnresolvedParams, PlannerRestrictionContext *plannerRestrictionContext) { PrepareInsertSelectForCitusPlanner(insertSelectQuery); /* get the SELECT query (may have changed after PrepareInsertSelectForCitusPlanner) */ RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery); Query *selectQuery = selectRte->subquery; bool allowRecursivePlanning = true; DistributedPlan *distPlan = CreateDistributedPlan(planId, allowRecursivePlanning, selectQuery, copyObject(selectQuery), boundParams, hasUnresolvedParams, plannerRestrictionContext); /* * We don't expect distPlan to be NULL here because hasUnresolvedParams is * already checked before this function and CreateDistributedPlan only returns * NULL when there are unresolved parameters. */ Assert(distPlan != NULL); if (distPlan->planningError) { return distPlan; } if (distPlan->combineQuery == NULL) { /* * For router queries, we construct a synthetic master query that simply passes * on the results of the remote tasks, which we can then use as the select in * the INSERT .. SELECT. */ distPlan->combineQuery = CreateCombineQueryForRouterPlan( distPlan); } /* * combineQuery of a distributed select is for combining the results from * worker nodes on the coordinator node. Putting it as a subquery to the * INSERT query, causes the INSERT query to insert the combined select value * from the workers. And making the resulting insert query the combineQuery * let's us execute this insert command. * * So this operation makes the master query insert the result of the * distributed select instead of returning it. */ selectRte->subquery = distPlan->combineQuery; distPlan->combineQuery = insertSelectQuery; return distPlan; } /* * PrepareInsertSelectForCitusPlanner prepares an INSERT..SELECT query tree * that was passed to the planner for use by Citus. * * First, it rebuilds the target lists of the INSERT and the SELECT * to be in the same order, which is not guaranteed in the parse tree. * * Second, some of the constants in the target list will have type * "unknown", which would confuse the Citus planner. To address that, * we add casts to SELECT target list entries whose type does not correspond * to the destination. This also helps us feed the output directly into * a COPY stream for INSERT..SELECT via coordinator. * * In case of UNION or other set operations, the SELECT does not have a * clearly defined target list, so we first wrap the UNION in a subquery. * UNION queries do not have the "unknown" type problem. * * Finally, if the INSERT has CTEs, we move those CTEs into the SELECT, * such that we can plan the SELECT as an independent query. To ensure * the ctelevelsup for CTE RTE's remain the same, we wrap the SELECT into * a subquery, unless we already did so in case of a UNION. */ static void PrepareInsertSelectForCitusPlanner(Query *insertSelectQuery) { RangeTblEntry *insertRte = ExtractResultRelationRTEOrError(insertSelectQuery); RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery); Oid targetRelationId = insertRte->relid; bool isWrapped = false; /* * PG18 is stricter about GroupRTE/GroupVar. For INSERT … SELECT with a GROUP BY, * flatten the SELECT’s targetList and havingQual so Vars point to base RTEs and * avoid Unrecognized range table id. */ FlattenGroupExprs(selectRte->subquery); if (selectRte->subquery->setOperations != NULL) { /* * Prepare UNION query for reordering and adding casts by * wrapping it in a subquery to have a single target list. */ selectRte->subquery = WrapSubquery(selectRte->subquery); isWrapped = true; } /* this is required for correct deparsing of the query */ ReorderInsertSelectTargetLists(insertSelectQuery, insertRte, selectRte); /* * Cast types of insert target list and select projection list to * match the column types of the target relation. */ selectRte->subquery->targetList = AddInsertSelectCasts(insertSelectQuery->targetList, copyObject(selectRte->subquery->targetList), targetRelationId); if (list_length(insertSelectQuery->cteList) > 0) { if (!isWrapped) { /* * By wrapping the SELECT in a subquery, we can avoid adjusting * ctelevelsup in RTE's that point to the CTEs. */ selectRte->subquery = WrapSubquery(selectRte->subquery); } /* copy CTEs from the INSERT ... SELECT statement into outer SELECT */ selectRte->subquery->cteList = copyObject(insertSelectQuery->cteList); selectRte->subquery->hasModifyingCTE = insertSelectQuery->hasModifyingCTE; insertSelectQuery->cteList = NIL; } } /* * CreateCombineQueryForRouterPlan is used for creating a dummy combineQuery * for a router plan, since router plans normally don't have one. */ static Query * CreateCombineQueryForRouterPlan(DistributedPlan *distPlan) { const Index insertTableId = 1; List *tableIdList = list_make1(makeInteger(insertTableId)); Job *dependentJob = distPlan->workerJob; List *dependentTargetList = dependentJob->jobQuery->targetList; /* compute column names for the derived table */ uint32 columnCount = (uint32) list_length(dependentTargetList); List *columnNameList = DerivedColumnNameList(columnCount, dependentJob->jobId); List *funcColumnNames = NIL; List *funcColumnTypes = NIL; List *funcColumnTypeMods = NIL; List *funcCollations = NIL; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, dependentTargetList) { Node *expr = (Node *) targetEntry->expr; char *name = targetEntry->resname; if (name == NULL) { name = pstrdup("unnamed"); } funcColumnNames = lappend(funcColumnNames, makeString(name)); funcColumnTypes = lappend_oid(funcColumnTypes, exprType(expr)); funcColumnTypeMods = lappend_int(funcColumnTypeMods, exprTypmod(expr)); funcCollations = lappend_oid(funcCollations, exprCollation(expr)); } RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(NULL, columnNameList, tableIdList, funcColumnNames, funcColumnTypes, funcColumnTypeMods, funcCollations); List *targetList = CreateTargetListForCombineQuery(dependentTargetList); RangeTblRef *rangeTableRef = makeNode(RangeTblRef); rangeTableRef->rtindex = 1; FromExpr *joinTree = makeNode(FromExpr); joinTree->quals = NULL; joinTree->fromlist = list_make1(rangeTableRef); Query *combineQuery = makeNode(Query); combineQuery->commandType = CMD_SELECT; combineQuery->querySource = QSRC_ORIGINAL; combineQuery->canSetTag = true; combineQuery->rtable = list_make1(rangeTableEntry); /* * This part of the code is more of a sanity check for readability, * it doesn't really do anything. * We know that Only relation RTEs and subquery RTEs that were once relation * RTEs (views) have their perminfoindex set. (see ExecCheckPermissions function) * DerivedRangeTableEntry sets the rtekind to RTE_FUNCTION * Hence we should have no perminfos here. */ Assert(rangeTableEntry->rtekind == RTE_FUNCTION && rangeTableEntry->perminfoindex == 0); combineQuery->rteperminfos = NIL; combineQuery->targetList = targetList; combineQuery->jointree = joinTree; return combineQuery; } /* * CreateTargetListForCombineQuery is used for creating a target list for * master query. */ static List * CreateTargetListForCombineQuery(List *targetList) { List *newTargetEntryList = NIL; const uint32 masterTableId = 1; int columnId = 1; /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; foreach_declared_ptr(originalTargetEntry, targetList) { TargetEntry *newTargetEntry = flatCopyTargetEntry(originalTargetEntry); Var *column = makeVarFromTargetEntry(masterTableId, originalTargetEntry); column->varattno = columnId; column->varattnosyn = columnId; columnId++; if (column->vartype == RECORDOID || column->vartype == RECORDARRAYOID) { column->vartypmod = BlessRecordExpression(originalTargetEntry->expr); } Expr *newExpression = (Expr *) column; newTargetEntry->expr = newExpression; newTargetEntryList = lappend(newTargetEntryList, newTargetEntry); } return newTargetEntryList; } /* * DistributedInsertSelectSupported returns NULL if the INSERT ... SELECT query * is supported, or a description why not. */ static DeferredErrorMessage * DistributedInsertSelectSupported(Query *queryTree, RangeTblEntry *insertRte, RangeTblEntry *subqueryRte, bool allReferenceTables, bool routerSelect, PlannerRestrictionContext *plannerRestrictionContext) { Oid selectPartitionColumnTableId = InvalidOid; Oid targetRelationId = insertRte->relid; ListCell *rangeTableCell = NULL; /* we only do this check for INSERT ... SELECT queries */ Assert(InsertSelectIntoCitusTable(queryTree)); Query *subquery = subqueryRte->subquery; if (!NeedsDistributedPlanning(subquery)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "distributed INSERT ... SELECT can only select from " "distributed tables", NULL, NULL); } RTEListProperties *subqueryRteListProperties = GetRTEListPropertiesForQuery(subquery); if (subqueryRteListProperties->hasDistributedTable && (subqueryRteListProperties->hasCitusLocalTable || subqueryRteListProperties->hasPostgresLocalTable)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "distributed INSERT ... SELECT cannot select from " "distributed tables and local tables at the same time", NULL, NULL); } if (subqueryRteListProperties->hasDistributedTable && IsCitusTableType(targetRelationId, CITUS_LOCAL_TABLE)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "distributed INSERT ... SELECT cannot insert into a " "local table that is added to metadata", NULL, NULL); } /* * In some cases, it might be possible to allow postgres local tables * in distributed insert select. However, we want to behave consistent * on all cases including Citus MX, and let insert select via coordinator * to kick-in. */ if (subqueryRteListProperties->hasPostgresLocalTable) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "distributed INSERT ... SELECT cannot select from " "a local table", NULL, NULL); return NULL; } /* we do not expect to see a view in modify target */ foreach(rangeTableCell, queryTree->rtable) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->rtekind == RTE_RELATION && rangeTableEntry->relkind == RELKIND_VIEW) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot insert into view over distributed table", NULL, NULL); } } if (FindNodeMatchingCheckFunction((Node *) queryTree, CitusIsVolatileFunction)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "volatile functions are not allowed in distributed " "INSERT ... SELECT queries", NULL, NULL); } DeferredErrorMessage *error = NULL; /* * We can skip SQL support related checks for router queries as * they are safe to route with any SQL. */ if (!routerSelect) { /* first apply toplevel pushdown checks to SELECT query */ error = DeferErrorIfUnsupportedSubqueryPushdown(subquery, plannerRestrictionContext, true); if (error) { return error; } /* then apply subquery pushdown checks to SELECT query */ error = DeferErrorIfCannotPushdownSubquery(subquery, false); if (error) { return error; } } if (IsCitusTableType(targetRelationId, CITUS_LOCAL_TABLE)) { /* * If we're inserting into a citus local table, it is ok because we've * checked the non-existence of distributed tables in the subquery. */ } else if (IsCitusTableType(targetRelationId, REFERENCE_TABLE)) { /* * If we're inserting into a reference table, all participating tables * should be reference tables as well. */ if (!allReferenceTables) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "only reference tables may be queried when targeting " "a reference table with distributed INSERT ... SELECT", NULL, NULL); } } else { /* * Note that we've already checked the non-existence of Postgres * tables in the subquery. */ if (subqueryRteListProperties->hasCitusLocalTable || subqueryRteListProperties->hasMaterializedView) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "distributed INSERT ... SELECT cannot select from " "a local relation when inserting into a distributed " "table", NULL, NULL); } if (HasDistributionKey(targetRelationId)) { /* ensure that INSERT's partition column comes from SELECT's partition column */ error = InsertPartitionColumnMatchesSelect(queryTree, insertRte, subqueryRte, &selectPartitionColumnTableId); if (error) { return error; } } } /* All tables in source list and target table should be colocated. */ List *distributedRelationIdList = DistributedRelationIdList(subquery); distributedRelationIdList = lappend_oid(distributedRelationIdList, targetRelationId); if (!AllDistributedRelationsInListColocated(distributedRelationIdList)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "INSERT target relation and all source relations of the " "SELECT must be colocated in distributed INSERT ... SELECT", NULL, NULL); } return NULL; } /* * RouterModifyTaskForShardInterval creates a modify task by * replacing the partitioning qual parameter added in distributed_planner() * with the shardInterval's boundary value. Then perform the normal * shard pruning on the subquery. Finally, checks if the target shardInterval * has exactly same placements with the select task's available anchor * placements. * * The function errors out if the subquery is not router select query (i.e., * subqueries with non equi-joins.). */ static Task * RouterModifyTaskForShardInterval(Query *originalQuery, CitusTableCacheEntry *targetTableCacheEntry, ShardInterval *shardInterval, PlannerRestrictionContext *plannerRestrictionContext, uint32 taskIdIndex, bool safeToPushdownSubquery, DeferredErrorMessage **routerPlannerError) { Query *copiedQuery = copyObject(originalQuery); RangeTblEntry *copiedInsertRte = ExtractResultRelationRTEOrError(copiedQuery); RangeTblEntry *copiedSubqueryRte = ExtractSelectRangeTableEntry(copiedQuery); Query *copiedSubquery = (Query *) copiedSubqueryRte->subquery; uint64 shardId = shardInterval->shardId; Oid distributedTableId = shardInterval->relationId; PlannerRestrictionContext *copyOfPlannerRestrictionContext = palloc0( sizeof(PlannerRestrictionContext)); StringInfo queryString = makeStringInfo(); ListCell *restrictionCell = NULL; List *selectPlacementList = NIL; uint64 selectAnchorShardId = INVALID_SHARD_ID; List *relationShardList = NIL; List *prunedShardIntervalListList = NIL; uint64 jobId = INVALID_JOB_ID; bool allReferenceTables = plannerRestrictionContext->relationRestrictionContext->allReferenceTables; List *shardOpExpressions = NIL; RestrictInfo *shardRestrictionList = NULL; bool multiShardModifyQuery = false; List *relationRestrictionList = NIL; copyOfPlannerRestrictionContext->relationRestrictionContext = CopyRelationRestrictionContext( plannerRestrictionContext->relationRestrictionContext); copyOfPlannerRestrictionContext->joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; copyOfPlannerRestrictionContext->fastPathRestrictionContext = plannerRestrictionContext->fastPathRestrictionContext; relationRestrictionList = copyOfPlannerRestrictionContext->relationRestrictionContext-> relationRestrictionList; /* grab shared metadata lock to stop concurrent placement additions */ LockShardDistributionMetadata(shardId, ShareLock); /* * Replace the partitioning qual parameter value in all baserestrictinfos. * Note that this has to be done on a copy, as the walker modifies in place. */ foreach(restrictionCell, relationRestrictionList) { RelationRestriction *restriction = lfirst(restrictionCell); List *originalBaseRestrictInfo = restriction->relOptInfo->baserestrictinfo; List *extendedBaseRestrictInfo = originalBaseRestrictInfo; Index rteIndex = restriction->index; if (!safeToPushdownSubquery || allReferenceTables) { continue; } shardOpExpressions = ShardIntervalOpExpressions(shardInterval, rteIndex); /* means it is a reference table and do not add any shard interval information */ if (shardOpExpressions == NIL) { continue; } shardRestrictionList = make_simple_restrictinfo(restriction->plannerInfo, (Expr *) shardOpExpressions); extendedBaseRestrictInfo = lappend(extendedBaseRestrictInfo, shardRestrictionList); restriction->relOptInfo->baserestrictinfo = extendedBaseRestrictInfo; } /* * We also need to add shard interval range to the subquery in case * the partition qual not distributed all tables such as some * subqueries in WHERE clause. * * Note that we need to add the ranges before the shard pruning to * prevent shard pruning logic (i.e, namely UpdateRelationNames()) * modifies range table entries, which makes hard to add the quals. */ RTEListProperties *subqueryRteListProperties = GetRTEListPropertiesForQuery( copiedSubquery); if (subqueryRteListProperties->hasDistTableWithShardKey) { AddPartitionKeyNotNullFilterToSelect(copiedSubquery); } /* mark that we don't want the router planner to generate dummy hosts/queries */ bool replacePrunedQueryWithDummy = false; /* * Use router planner to decide on whether we can push down the query or not. * If we can, we also rely on the side-effects that all RTEs have been updated * to point to the relevant nodes and selectPlacementList is determined. */ DeferredErrorMessage *planningError = PlanRouterQuery(copiedSubquery, copyOfPlannerRestrictionContext, &selectPlacementList, &selectAnchorShardId, &relationShardList, &prunedShardIntervalListList, replacePrunedQueryWithDummy, &multiShardModifyQuery, NULL, NULL); Assert(!multiShardModifyQuery); if (planningError) { *routerPlannerError = planningError; return NULL; } /* ensure that we do not send queries where select is pruned away completely */ if (list_length(selectPlacementList) == 0) { ereport(DEBUG2, (errmsg("Skipping target shard interval " UINT64_FORMAT " since SELECT query for it pruned away", shardId))); return NULL; } /* get the placements for insert target shard and its intersection with select */ List *insertShardPlacementList = ActiveShardPlacementList(shardId); List *intersectedPlacementList = IntersectPlacementList(insertShardPlacementList, selectPlacementList); /* * If insert target does not have exactly the same placements with the select, * we sholdn't run the query. */ if (list_length(insertShardPlacementList) != list_length(intersectedPlacementList)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning for the given " "modification"), errdetail("Insert query cannot be executed on all placements " "for shard " UINT64_FORMAT "", shardId))); } /* this is required for correct deparsing of the query */ ReorderInsertSelectTargetLists(copiedQuery, copiedInsertRte, copiedSubqueryRte); /* setting an alias simplifies deparsing of RETURNING */ if (copiedInsertRte->alias == NULL) { Alias *alias = makeAlias(CITUS_TABLE_ALIAS, NIL); copiedInsertRte->alias = alias; } /* and generate the full query string */ deparse_shard_query(copiedQuery, distributedTableId, shardInterval->shardId, queryString); ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data))); Task *modifyTask = CreateBasicTask(jobId, taskIdIndex, MODIFY_TASK, queryString->data); modifyTask->dependentTaskList = NULL; modifyTask->anchorShardId = shardId; modifyTask->taskPlacementList = insertShardPlacementList; modifyTask->relationShardList = relationShardList; modifyTask->replicationModel = targetTableCacheEntry->replicationModel; modifyTask->isLocalTableModification = false; return modifyTask; } /* * ReorderInsertSelectTargetLists reorders the target lists of INSERT/SELECT * query which is required for deparsing purposes. The reordered query is returned. * * The necessity for this function comes from the fact that ruleutils.c is not supposed * to be used on "rewritten" queries (i.e. ones that have been passed through * QueryRewrite()). Query rewriting is the process in which views and such are expanded, * and, INSERT/UPDATE targetlists are reordered to match the physical order, * defaults etc. For the details of reordeing, see transformInsertRow() and * rewriteTargetListIU(). */ Query * ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte, RangeTblEntry *subqueryRte) { ListCell *insertTargetEntryCell; List *newSubqueryTargetlist = NIL; List *newInsertTargetlist = NIL; List *columnNameList = NIL; int resno = 1; Index selectTableId = 2; int targetEntryIndex = 0; Query *subquery = subqueryRte->subquery; Oid insertRelationId = insertRte->relid; /* * We implement the following algorithm for the reoderding: * - Iterate over the INSERT target list entries * - If the target entry includes a Var, find the corresponding * SELECT target entry on the original query and update resno * - If the target entry does not include a Var (i.e., defaults * or constants), create new target entry and add that to * SELECT target list * - Create a new INSERT target entry with respect to the new * SELECT target entry created. */ foreach(insertTargetEntryCell, originalQuery->targetList) { TargetEntry *oldInsertTargetEntry = lfirst(insertTargetEntryCell); TargetEntry *newSubqueryTargetEntry = NULL; AttrNumber originalAttrNo = get_attnum(insertRelationId, oldInsertTargetEntry->resname); /* we need to explore the underlying expression */ Node *expr = strip_implicit_coercions((Node *) oldInsertTargetEntry->expr); /* see transformInsertRow() for the details */ if (IsA(expr, SubscriptingRef) || IsA(expr, FieldStore)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg( "cannot plan distributed INSERT INTO ... SELECT query"), errhint("Do not use array references and field stores " "on the INSERT target list."))); } /* * It is safe to pull Var clause and ignore the coercions since that * are already going to be added on the workers implicitly. */ List *targetVarList = pull_var_clause((Node *) oldInsertTargetEntry->expr, PVC_RECURSE_AGGREGATES); int targetVarCount = list_length(targetVarList); /* a single INSERT target entry cannot have more than one Var */ Assert(targetVarCount <= 1); if (targetVarCount == 1) { Var *oldInsertVar = (Var *) linitial(targetVarList); TargetEntry *oldSubqueryTle = list_nth(subquery->targetList, oldInsertVar->varattno - 1); newSubqueryTargetEntry = copyObject(oldSubqueryTle); newSubqueryTargetEntry->resno = resno; newSubqueryTargetlist = lappend(newSubqueryTargetlist, newSubqueryTargetEntry); } else { newSubqueryTargetEntry = makeTargetEntry(oldInsertTargetEntry->expr, resno, oldInsertTargetEntry->resname, oldInsertTargetEntry->resjunk); newSubqueryTargetlist = lappend(newSubqueryTargetlist, newSubqueryTargetEntry); } String *columnName = makeString(newSubqueryTargetEntry->resname); columnNameList = lappend(columnNameList, columnName); /* * The newly created select target entry cannot be a junk entry since junk * entries are not in the final target list and we're processing the * final target list entries. */ Assert(!newSubqueryTargetEntry->resjunk); Var *newInsertVar = makeVar(selectTableId, resno, exprType((Node *) newSubqueryTargetEntry->expr), exprTypmod((Node *) newSubqueryTargetEntry->expr), exprCollation((Node *) newSubqueryTargetEntry->expr), 0); TargetEntry *newInsertTargetEntry = makeTargetEntry( (Expr *) newInsertVar, originalAttrNo, oldInsertTargetEntry->resname, oldInsertTargetEntry->resjunk); newInsertTargetlist = lappend(newInsertTargetlist, newInsertTargetEntry); resno++; } /* * if there are any remaining target list entries (i.e., GROUP BY column not on the * target list of subquery), update the remaining resnos. */ int subqueryTargetLength = list_length(subquery->targetList); for (; targetEntryIndex < subqueryTargetLength; ++targetEntryIndex) { TargetEntry *oldSubqueryTle = list_nth(subquery->targetList, targetEntryIndex); /* * Skip non-junk entries since we've already processed them above and this * loop only is intended for junk entries. */ if (!oldSubqueryTle->resjunk) { continue; } TargetEntry *newSubqueryTargetEntry = copyObject(oldSubqueryTle); newSubqueryTargetEntry->resno = resno; newSubqueryTargetlist = lappend(newSubqueryTargetlist, newSubqueryTargetEntry); resno++; } originalQuery->targetList = newInsertTargetlist; subquery->targetList = newSubqueryTargetlist; subqueryRte->eref->colnames = columnNameList; return NULL; } /* * InsertPartitionColumnMatchesSelect returns NULL the partition column in the * table targeted by INSERTed matches with the any of the SELECTed table's * partition column. Returns the error description if there's no match. * * On return without error (i.e., if partition columns match), the function * also sets selectPartitionColumnTableId. */ static DeferredErrorMessage * InsertPartitionColumnMatchesSelect(Query *query, RangeTblEntry *insertRte, RangeTblEntry *subqueryRte, Oid *selectPartitionColumnTableId) { ListCell *targetEntryCell = NULL; uint32 rangeTableId = 1; Oid insertRelationId = insertRte->relid; Var *insertPartitionColumn = PartitionColumn(insertRelationId, rangeTableId); Query *subquery = subqueryRte->subquery; bool targetTableHasPartitionColumn = false; foreach(targetEntryCell, query->targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); List *insertTargetEntryColumnList = pull_var_clause_default((Node *) targetEntry); Var *subqueryPartitionColumn = NULL; /* * We only consider target entries that include a single column. Note that this * is slightly different than directly checking the whether the targetEntry->expr * is a var since the var could be wrapped into an implicit/explicit casting. * * Also note that we skip the target entry if it does not contain a Var, which * corresponds to columns with DEFAULT values on the target list. */ if (list_length(insertTargetEntryColumnList) != 1) { continue; } Var *insertVar = (Var *) linitial(insertTargetEntryColumnList); AttrNumber originalAttrNo = targetEntry->resno; /* skip processing of target table non-partition columns */ if (originalAttrNo != insertPartitionColumn->varattno) { continue; } /* INSERT query includes the partition column */ targetTableHasPartitionColumn = true; TargetEntry *subqueryTargetEntry = list_nth(subquery->targetList, insertVar->varattno - 1); Expr *selectTargetExpr = subqueryTargetEntry->expr; RangeTblEntry *subqueryPartitionColumnRelationIdRTE = NULL; List *parentQueryList = list_make2(query, subquery); bool skipOuterVars = false; FindReferencedTableColumn(selectTargetExpr, parentQueryList, subquery, &subqueryPartitionColumn, &subqueryPartitionColumnRelationIdRTE, skipOuterVars); Oid subqueryPartitionColumnRelationId = subqueryPartitionColumnRelationIdRTE ? subqueryPartitionColumnRelationIdRTE-> relid : InvalidOid; /* * Corresponding (i.e., in the same ordinal position as the target table's * partition column) select target entry does not directly belong a table. * Evaluate its expression type and error out properly. */ if (subqueryPartitionColumnRelationId == InvalidOid) { char *errorDetailTemplate = "Subquery contains %s in the " "same position as the target table's " "partition column."; char *exprDescription = ""; switch (selectTargetExpr->type) { case T_Const: { exprDescription = "a constant value"; break; } case T_OpExpr: { exprDescription = "an operator"; break; } case T_FuncExpr: { FuncExpr *subqueryFunctionExpr = (FuncExpr *) selectTargetExpr; switch (subqueryFunctionExpr->funcformat) { case COERCE_EXPLICIT_CALL: { exprDescription = "a function call"; break; } case COERCE_EXPLICIT_CAST: { exprDescription = "an explicit cast"; break; } case COERCE_IMPLICIT_CAST: { exprDescription = "an implicit cast"; break; } default: { exprDescription = "a function call"; break; } } break; } case T_Aggref: { exprDescription = "an aggregation"; break; } case T_CaseExpr: { exprDescription = "a case expression"; break; } case T_CoalesceExpr: { exprDescription = "a coalesce expression"; break; } case T_RowExpr: { exprDescription = "a row expression"; break; } case T_MinMaxExpr: { exprDescription = "a min/max expression"; break; } case T_CoerceViaIO: { exprDescription = "an explicit coercion"; break; } default: { exprDescription = "an expression that is not a simple column reference"; break; } } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed INSERT INTO ... SELECT " "because the partition columns in the source table " "and subquery do not match", psprintf(errorDetailTemplate, exprDescription), "Ensure the target table's partition column has a " "corresponding simple column reference to a distributed " "table's partition column in the subquery."); } /* * Insert target expression could only be non-var if the select target * entry does not have the same type (i.e., target column requires casting). */ if (!IsA(targetEntry->expr, Var)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed INSERT INTO ... SELECT " "because the partition columns in the source table " "and subquery do not match", "The data type of the target table's partition column " "should exactly match the data type of the " "corresponding simple column reference in the subquery.", NULL); } /* finally, check that the select target column is a partition column */ if (!IsPartitionColumn(selectTargetExpr, subquery, skipOuterVars)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed INSERT INTO ... SELECT " "because the partition columns in the source table " "and subquery do not match", "The target table's partition column should correspond " "to a partition column in the subquery.", NULL); } /* finally, check that the select target column is a partition column */ /* we can set the select relation id */ *selectPartitionColumnTableId = subqueryPartitionColumnRelationId; break; } if (!targetTableHasPartitionColumn) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed INSERT INTO ... SELECT " "because the partition columns in the source table " "and subquery do not match", "the query doesn't include the target table's " "partition column", NULL); } return NULL; } /* * CreateNonPushableInsertSelectPlan creates a query plan for a SELECT into a * distributed table. The query plan can also be executed on a worker in MX. */ static DistributedPlan * CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse, ParamListInfo boundParams) { Query *insertSelectQuery = copyObject(parse); DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); distributedPlan->modLevel = RowModifyLevelForQuery(insertSelectQuery); distributedPlan->planningError = NonPushableInsertSelectSupported(insertSelectQuery); if (distributedPlan->planningError != NULL) { return distributedPlan; } PrepareInsertSelectForCitusPlanner(insertSelectQuery); /* get the SELECT query (may have changed after PrepareInsertSelectForCitusPlanner) */ RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery); Query *selectQuery = selectRte->subquery; /* * Later we might need to call WrapTaskListForProjection(), which requires * that select target list has unique names, otherwise the outer query * cannot select columns unambiguously. So we relabel select columns to * match target columns. */ List *insertTargetList = insertSelectQuery->targetList; RelabelTargetEntryList(selectQuery->targetList, insertTargetList); /* * Make a copy of the select query, since following code scribbles it * but we need to keep the original for EXPLAIN. */ Query *selectQueryCopy = copyObject(selectQuery); /* plan the subquery, this may be another distributed query */ int cursorOptions = CURSOR_OPT_PARALLEL_OK; PlannedStmt *selectPlan = pg_plan_query(selectQueryCopy, NULL, cursorOptions, boundParams); /* decide whether we can repartition the results */ RangeTblEntry *insertRte = ExtractResultRelationRTEOrError(insertSelectQuery); Oid targetRelationId = insertRte->relid; bool repartitioned = IsRedistributablePlan(selectPlan->planTree) && IsSupportedRedistributionTarget(targetRelationId); /* * It's not possible to generate a distributed plan for a SELECT * having more than one tasks if it references a single-shard table. * * For this reason, right now we don't expect an INSERT .. SELECT * query to go through the repartitioned INSERT .. SELECT logic if the * SELECT query references a single-shard table. */ Assert(!repartitioned || !ContainsSingleShardTable(selectQueryCopy)); distributedPlan->modifyQueryViaCoordinatorOrRepartition = insertSelectQuery; distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition = selectPlan; distributedPlan->modifyWithSelectMethod = repartitioned ? MODIFY_WITH_SELECT_REPARTITION : MODIFY_WITH_SELECT_VIA_COORDINATOR; distributedPlan->expectResults = insertSelectQuery->returningList != NIL; distributedPlan->intermediateResultIdPrefix = InsertSelectResultIdPrefix(planId); distributedPlan->targetRelationId = targetRelationId; return distributedPlan; } /* * NonPushableInsertSelectSupported returns an error if executing an * INSERT ... SELECT command by pulling results of the SELECT to the coordinator * or with repartitioning is unsupported because it needs to generate sequence * values or insert into an append-distributed table. */ static DeferredErrorMessage * NonPushableInsertSelectSupported(Query *insertSelectQuery) { DeferredErrorMessage *deferredError = ErrorIfOnConflictNotSupported( insertSelectQuery); if (deferredError) { return deferredError; } RangeTblEntry *insertRte = ExtractResultRelationRTE(insertSelectQuery); if (IsCitusTableType(insertRte->relid, APPEND_DISTRIBUTED)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "INSERT ... SELECT into an append-distributed table is " "not supported", NULL, NULL); } return NULL; } /* * InsertSelectResultPrefix returns the prefix to use for intermediate * results of an INSERT ... SELECT via the coordinator that runs in two * phases in order to do RETURNING or ON CONFLICT. */ char * InsertSelectResultIdPrefix(uint64 planId) { StringInfo resultIdPrefix = makeStringInfo(); appendStringInfo(resultIdPrefix, "insert_select_" UINT64_FORMAT, planId); return resultIdPrefix->data; } /* * Return true if the expression tree can change value within a single scan * (i.e. the planner must treat it as VOLATILE). * We just delegate to PostgreSQL’s helper. */ static inline bool expr_is_volatile(Node *node) { /* contain_volatile_functions() also returns true for set-returning * volatile functions and for nextval()/currval(). */ return contain_volatile_functions(node); } /* * WrapSubquery * * Build a wrapper query: * * SELECT * FROM ( ) * citus_insert_select_subquery * * Purpose: * - Preserve column numbering while lifting volatile expressions to the coordinator. * - Volatile (non-deterministic) expressions not used in GROUP BY / ORDER BY * are lifted to the outer SELECT to ensure they are evaluated only once. * - Stable/immutable expressions or volatile ones required by GROUP BY / ORDER BY * stay in the subquery and are accessed via Vars in the outer SELECT. */ Query * WrapSubquery(Query *subquery) { /* * 1. Build the wrapper skeleton: SELECT ... FROM (subquery) alias */ ParseState *pstate = make_parsestate(NULL); Query *outerQuery = makeNode(Query); outerQuery->commandType = CMD_SELECT; Alias *alias = makeAlias("citus_insert_select_subquery", NIL); RangeTblEntry *rte_subq = RangeTableEntryFromNSItem( addRangeTableEntryForSubquery(pstate, subquery, /* still points to original subquery */ alias, false, /* not LATERAL */ true)); /* in FROM clause */ outerQuery->rtable = list_make1(rte_subq); /* Ensure RTE_SUBQUERY has proper permission handling */ Assert(rte_subq->rtekind == RTE_SUBQUERY && rte_subq->perminfoindex == 0); outerQuery->rteperminfos = NIL; RangeTblRef *rtref = makeNode(RangeTblRef); rtref->rtindex = 1; /* Only one RTE, so index is 1 */ outerQuery->jointree = makeFromExpr(list_make1(rtref), NULL); /* * 2. Create new target lists for inner (worker) and outer (coordinator) */ List *newInnerTL = NIL; List *newOuterTL = NIL; int nextResno = 1; TargetEntry *te = NULL; foreach_declared_ptr(te, subquery->targetList) { if (te->resjunk) { /* Keep resjunk entries only in subquery (not in outer query) */ newInnerTL = lappend(newInnerTL, te); continue; } bool isVolatile = expr_is_volatile((Node *) te->expr); bool usedInSort = (te->ressortgroupref != 0); if (isVolatile && !usedInSort) { /* * Lift volatile expression to outer query so it's evaluated once. * In inner query, place a NULL of the same type to preserve column position. */ TargetEntry *outerTE = makeTargetEntry(copyObject(te->expr), list_length(newOuterTL) + 1, te->resname, false); newOuterTL = lappend(newOuterTL, outerTE); Const *nullConst = makeNullConst(exprType((Node *) te->expr), exprTypmod((Node *) te->expr), exprCollation((Node *) te->expr)); TargetEntry *placeholder = makeTargetEntry((Expr *) nullConst, nextResno++, /* preserve column position */ te->resname, false); /* visible, not resjunk */ newInnerTL = lappend(newInnerTL, placeholder); } else { /* * Either: * - expression is stable or immutable, or * - volatile but needed for sorting or grouping * * In both cases, keep it in subquery and reference it using a Var. */ TargetEntry *innerTE = te; /* reuse original node */ innerTE->resno = nextResno++; newInnerTL = lappend(newInnerTL, innerTE); Var *v = makeVar(/* subquery reference index is 1 */ rtref->rtindex, /* same as 1, but self‑documenting */ innerTE->resno, exprType((Node *) innerTE->expr), exprTypmod((Node *) innerTE->expr), exprCollation((Node *) innerTE->expr), 0); TargetEntry *outerTE = makeTargetEntry((Expr *) v, list_length(newOuterTL) + 1, innerTE->resname, false); newOuterTL = lappend(newOuterTL, outerTE); } } /* * 3. Assign target lists and return the wrapper query */ subquery->targetList = newInnerTL; outerQuery->targetList = newOuterTL; return outerQuery; } /* * RelabelTargetEntryList relabels select target list to have matching names with * insert target list. */ static void RelabelTargetEntryList(List *selectTargetList, List *insertTargetList) { TargetEntry *selectTargetEntry = NULL; TargetEntry *insertTargetEntry = NULL; forboth_ptr(selectTargetEntry, selectTargetList, insertTargetEntry, insertTargetList) { selectTargetEntry->resname = insertTargetEntry->resname; } } /* * AddInsertSelectCasts ensures that the columns in the given target lists * have the same type as the corresponding columns of the target relation. * It adds casts when necessary. * * Returns the updated selectTargetList. */ static List * AddInsertSelectCasts(List *insertTargetList, List *selectTargetList, Oid targetRelationId) { List *projectedEntries = NIL; List *nonProjectedEntries = NIL; /* * ReorderInsertSelectTargetLists() ensures that the first few columns of the * SELECT query match the insert targets. It might also include additional * items (for GROUP BY, etc.), so the insertTargetList is shorter. */ Assert(list_length(insertTargetList) <= list_length(selectTargetList)); Relation distributedRelation = table_open(targetRelationId, RowExclusiveLock); TupleDesc destTupleDescriptor = RelationGetDescr(distributedRelation); int targetEntryIndex = 0; TargetEntry *insertEntry = NULL; TargetEntry *selectEntry = NULL; forboth_ptr(insertEntry, insertTargetList, selectEntry, selectTargetList) { /* * Retrieve the target attribute corresponding to the insert entry. * The attribute is located at (resno - 1) in the tuple descriptor. */ Form_pg_attribute attr = TupleDescAttr(destTupleDescriptor, insertEntry->resno - 1); ProcessEntryPair(insertEntry, selectEntry, attr, targetEntryIndex, &projectedEntries, &nonProjectedEntries); targetEntryIndex++; } /* Append any additional non-projected entries from selectTargetList */ for (int entryIndex = list_length(insertTargetList); entryIndex < list_length(selectTargetList); entryIndex++) { nonProjectedEntries = lappend(nonProjectedEntries, list_nth(selectTargetList, entryIndex)); } /* Concatenate projected and non-projected entries and reset resno numbering */ selectTargetList = list_concat(projectedEntries, nonProjectedEntries); ResetTargetEntryResno(selectTargetList); table_close(distributedRelation, NoLock); return selectTargetList; } /* * Processes a single pair of insert and select target entries. * It compares the source and target types and appends either the * original select entry or a casted version to the appropriate list. */ static void ProcessEntryPair(TargetEntry *insertEntry, TargetEntry *selectEntry, Form_pg_attribute attr, int targetEntryIndex, List **projectedEntries, List **nonProjectedEntries) { Oid effectiveSourceType = exprType((Node *) selectEntry->expr); Oid targetType = attr->atttypid; /* * If the select expression is a NextValueExpr, use its actual return type. * * NextValueExpr represents a call to the nextval() function, which is used to * obtain the next value from a sequence—commonly for populating auto-increment * columns. In many cases, nextval() returns an INT8 (bigint), but the actual * return type may differ depending on database configuration or custom implementations. * * Since the target column might have a different type (e.g., INT4), we need to * obtain the real return type of nextval() to ensure that any type coercion is applied * correctly. This is done by calling GetNextvalReturnTypeCatalog(), which looks up the * function in the catalog and returns its return type. The effectiveSourceType is then * set to this value, ensuring that subsequent comparisons and casts use the correct type. */ if (IsA(selectEntry->expr, NextValueExpr)) { effectiveSourceType = GetNextvalReturnTypeCatalog(); } if (effectiveSourceType != targetType) { AppendCastedEntry(insertEntry, selectEntry, effectiveSourceType, targetType, attr->attcollation, attr->atttypmod, targetEntryIndex, projectedEntries, nonProjectedEntries); } else { /* Types match, no cast needed */ *projectedEntries = lappend(*projectedEntries, selectEntry); } } /* * Resets the resno field for each target entry in the list so that * they are numbered sequentially. */ static void ResetTargetEntryResno(List *targetList) { int entryResNo = 1; ListCell *lc = NULL; foreach(lc, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc); tle->resno = entryResNo++; } } /* * Looks up the nextval(regclass) function in pg_proc, returning its actual * rettype. In a standard build, that will be INT8OID, but this is more robust. */ static Oid GetNextvalReturnTypeCatalog(void) { Oid argTypes[1] = { REGCLASSOID }; List *nameList = list_make1(makeString("nextval")); /* Look up the nextval(regclass) function */ Oid nextvalFuncOid = LookupFuncName(nameList, 1, argTypes, false); if (!OidIsValid(nextvalFuncOid)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not find function nextval(regclass)"))); } /* Retrieve and validate the return type of the nextval function */ Oid nextvalReturnType = get_func_rettype(nextvalFuncOid); if (!OidIsValid(nextvalReturnType)) { elog(ERROR, "could not determine return type of nextval(regclass)"); } return nextvalReturnType; } /** * Modifies the given insert entry to match the target column's type and typmod, * then creates and appends a new target entry containing a casted expression * to the projected list. If the original select entry is used by ORDER BY or GROUP BY, * it is marked as junk to avoid ambiguity. */ static void AppendCastedEntry(TargetEntry *insertEntry, TargetEntry *selectEntry, Oid castFromType, Oid targetType, Oid collation, int32 typmod, int targetEntryIndex, List **projectedEntries, List **nonProjectedEntries) { /* Update the insert entry's Var to match the target column's type, typmod, and collation */ Assert(IsA(insertEntry->expr, Var)); { Var *insertVar = (Var *) insertEntry->expr; insertVar->vartype = targetType; insertVar->vartypmod = typmod; insertVar->varcollid = collation; } /* Create a new TargetEntry with the casted expression */ TargetEntry *coercedEntry = copyObject(selectEntry); coercedEntry->expr = CastExpr((Expr *) selectEntry->expr, castFromType, targetType, collation, typmod); coercedEntry->ressortgroupref = 0; /* Assign a unique name to the coerced entry */ SetTargetEntryName(coercedEntry, "auto_coerced_by_citus_%d", targetEntryIndex); *projectedEntries = lappend(*projectedEntries, coercedEntry); /* If the original select entry is referenced in ORDER BY or GROUP BY, * mark it as junk and rename it to avoid ambiguity. */ if (selectEntry->ressortgroupref != 0) { selectEntry->resjunk = true; SetTargetEntryName(selectEntry, "discarded_target_item_%d", targetEntryIndex); *nonProjectedEntries = lappend(*nonProjectedEntries, selectEntry); } } /* * CastExpr returns an expression which casts the given expr from sourceType to * the given targetType. */ static Expr * CastExpr(Expr *expr, Oid sourceType, Oid targetType, Oid targetCollation, int targetTypeMod) { Oid coercionFuncId = InvalidOid; CoercionPathType coercionType = find_coercion_pathway(targetType, sourceType, COERCION_EXPLICIT, &coercionFuncId); if (coercionType == COERCION_PATH_FUNC) { FuncExpr *coerceExpr = makeNode(FuncExpr); coerceExpr->funcid = coercionFuncId; coerceExpr->args = list_make1(copyObject(expr)); coerceExpr->funccollid = targetCollation; coerceExpr->funcresulttype = targetType; return (Expr *) coerceExpr; } else if (coercionType == COERCION_PATH_RELABELTYPE) { RelabelType *coerceExpr = makeNode(RelabelType); coerceExpr->arg = copyObject(expr); coerceExpr->resulttype = targetType; coerceExpr->resulttypmod = targetTypeMod; coerceExpr->resultcollid = targetCollation; coerceExpr->relabelformat = COERCE_IMPLICIT_CAST; coerceExpr->location = -1; return (Expr *) coerceExpr; } else if (coercionType == COERCION_PATH_ARRAYCOERCE) { Oid sourceBaseType = get_base_element_type(sourceType); Oid targetBaseType = get_base_element_type(targetType); CaseTestExpr *elemExpr = makeNode(CaseTestExpr); elemExpr->collation = targetCollation; elemExpr->typeId = sourceBaseType; elemExpr->typeMod = -1; Expr *elemCastExpr = CastExpr((Expr *) elemExpr, sourceBaseType, targetBaseType, targetCollation, targetTypeMod); ArrayCoerceExpr *coerceExpr = makeNode(ArrayCoerceExpr); coerceExpr->arg = copyObject(expr); coerceExpr->elemexpr = elemCastExpr; coerceExpr->resultcollid = targetCollation; coerceExpr->resulttype = targetType; coerceExpr->resulttypmod = targetTypeMod; coerceExpr->location = -1; coerceExpr->coerceformat = COERCE_IMPLICIT_CAST; return (Expr *) coerceExpr; } else if (coercionType == COERCION_PATH_COERCEVIAIO) { CoerceViaIO *coerceExpr = makeNode(CoerceViaIO); coerceExpr->arg = (Expr *) copyObject(expr); coerceExpr->resulttype = targetType; coerceExpr->resultcollid = targetCollation; coerceExpr->coerceformat = COERCE_IMPLICIT_CAST; coerceExpr->location = -1; return (Expr *) coerceExpr; } else { ereport(ERROR, (errmsg("could not find a conversion path from type %d to %d", sourceType, targetType))); } return NULL; /* keep compiler happy */ } /* Helper function to set the target entry name using a formatted string */ static void SetTargetEntryName(TargetEntry *tle, const char *format, int index) { StringInfo resnameString = makeStringInfo(); appendStringInfo(resnameString, format, index); tle->resname = resnameString->data; } /* PlanningInsertSelect returns true if we are planning an INSERT ...SELECT query */ bool PlanningInsertSelect(void) { return insertSelectPlannerLevel > 0; } ================================================ FILE: src/backend/distributed/planner/intermediate_result_pruning.c ================================================ /*------------------------------------------------------------------------- * * intermediate_result_pruning.c * Functions for pruning intermediate result broadcasting. * * We only send intermediate results of subqueries and CTEs to worker nodes * that use them in the remainder of the distributed plan to avoid unnecessary * network traffic. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/hashfn.h" #include "utils/builtins.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_ruleutils.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/query_utils.h" #include "distributed/worker_manager.h" /* controlled via GUC, used mostly for testing */ bool LogIntermediateResults = false; static List * FindSubPlansUsedInNode(Node *node, SubPlanAccessType accessType); static void AppendAllAccessedWorkerNodes(IntermediateResultsHashEntry *entry, DistributedPlan *distributedPlan, int nodeCount); static void AppendAllWorkerNodes(IntermediateResultsHashEntry *entry); static List * FindAllRemoteWorkerNodesUsingSubplan(IntermediateResultsHashEntry *entry); static List * RemoveLocalNodeFromWorkerList(List *workerNodeList); static void LogIntermediateResultMulticastSummary(IntermediateResultsHashEntry *entry, List *workerNodeList); /* * FindSubPlanUsages finds the subplans used in the master query and the * job query and returns them as a combined list of UsedDistributedSubPlan * structs. * * The list may contain duplicates if the subplan is referenced multiple * times. */ List * FindSubPlanUsages(DistributedPlan *plan) { List *localSubPlans = NIL; List *remoteSubPlans = NIL; if (plan->combineQuery != NULL) { localSubPlans = FindSubPlansUsedInNode((Node *) plan->combineQuery, SUBPLAN_ACCESS_LOCAL); } if (plan->workerJob != NULL) { /* * Mark the subplans as needed on remote side. Note that this decision is * revisited on execution, when the query only consists of intermediate * results. */ remoteSubPlans = FindSubPlansUsedInNode((Node *) plan->workerJob->jobQuery, SUBPLAN_ACCESS_REMOTE); } if (plan->modifyQueryViaCoordinatorOrRepartition != NULL) { /* INSERT..SELECT plans currently do not have a workerJob */ Assert(plan->workerJob == NULL); /* * The SELECT in an INSERT..SELECT is not fully planned yet and we cannot * perform pruning. We therefore require all subplans used in the * INSERT..SELECT to be available all nodes. */ remoteSubPlans = FindSubPlansUsedInNode((Node *) plan->modifyQueryViaCoordinatorOrRepartition, SUBPLAN_ACCESS_ANYWHERE); } /* merge the used subplans */ return list_concat(localSubPlans, remoteSubPlans); } /* * FindSubPlansUsedInPlan finds all the subplans used by the plan by traversing * the input node. */ static List * FindSubPlansUsedInNode(Node *node, SubPlanAccessType accessType) { List *rangeTableList = NIL; ListCell *rangeTableCell = NULL; List *usedSubPlanList = NIL; ExtractRangeTableEntryWalker(node, &rangeTableList); foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = lfirst(rangeTableCell); if (rangeTableEntry->rtekind == RTE_FUNCTION) { char *resultId = FindIntermediateResultIdIfExists(rangeTableEntry); if (resultId == NULL) { continue; } /* * Use a Value to be able to use list_append_unique and store * the result ID in the DistributedPlan. */ UsedDistributedSubPlan *usedPlan = CitusMakeNode(UsedDistributedSubPlan); usedPlan->subPlanId = pstrdup(resultId); usedPlan->accessType = accessType; usedSubPlanList = lappend(usedSubPlanList, usedPlan); } } return usedSubPlanList; } /* * RecordSubplanExecutionsOnNodes iterates over the usedSubPlanNodeList, * and for each entry, record the workerNodes that are accessed by * the distributed plan. * * Later, we'll use this information while we broadcast the intermediate * results to the worker nodes. The idea is that the intermediate result * should only be broadcasted to the worker nodes that are accessed by * the distributedPlan(s) that the subPlan is used in. * * Finally, the function recursively descends into the actual subplans * of the input distributedPlan as well. */ void RecordSubplanExecutionsOnNodes(HTAB *intermediateResultsHash, DistributedPlan *distributedPlan) { List *usedSubPlanNodeList = distributedPlan->usedSubPlanNodeList; List *subPlanList = distributedPlan->subPlanList; ListCell *subPlanCell = NULL; int nodeCount = list_length(ActiveReadableNodeList()); foreach(subPlanCell, usedSubPlanNodeList) { UsedDistributedSubPlan *usedPlan = lfirst(subPlanCell); char *resultId = usedPlan->subPlanId; IntermediateResultsHashEntry *entry = SearchIntermediateResult( intermediateResultsHash, resultId); /* * There is no need to traverse the subplan if the intermediate result * will be written to a local file and sent to all nodes. Note that the * remaining subplans in the distributed plan should still be traversed. */ if (list_length(entry->nodeIdList) == nodeCount && entry->writeLocalFile) { elog(DEBUG4, "Subplan %s is used in all workers", resultId); continue; } if (usedPlan->accessType == SUBPLAN_ACCESS_LOCAL) { /* subPlan needs to be written locally as the planner decided */ entry->writeLocalFile = true; } else if (usedPlan->accessType == SUBPLAN_ACCESS_REMOTE) { /* * traverse the plan and add find all worker nodes * * If we have reference tables in the distributed plan, all the * workers will be in the node list. We can improve intermediate result * pruning by deciding which reference table shard will be accessed earlier. */ AppendAllAccessedWorkerNodes(entry, distributedPlan, nodeCount); elog(DEBUG4, "Subplan %s is used in %lu", resultId, distributedPlan->planId); } else if (usedPlan->accessType == SUBPLAN_ACCESS_ANYWHERE) { /* subplan is needed on all nodes */ entry->writeLocalFile = true; AppendAllWorkerNodes(entry); } } /* descend into the subPlans */ foreach(subPlanCell, subPlanList) { DistributedSubPlan *subPlan = (DistributedSubPlan *) lfirst(subPlanCell); CustomScan *customScan = FetchCitusCustomScanIfExists(subPlan->plan->planTree); if (customScan) { DistributedPlan *distributedPlanOfSubPlan = GetDistributedPlan(customScan); RecordSubplanExecutionsOnNodes(intermediateResultsHash, distributedPlanOfSubPlan); } } } /* * AppendAllAccessedWorkerNodes iterates over all the tasks in a distributed plan * to updates the list of worker nodes that can be accessed when this plan is * executed in entry. Depending on the plan, the function may give the decision for * writing the results locally. * * If there are multiple placements of a Shard, all of them are considered and * all the workers with placements are appended to the list. This effectively * means that if there is a reference table access in the distributed plan, all * the workers will be in the resulting list. */ static void AppendAllAccessedWorkerNodes(IntermediateResultsHashEntry *entry, DistributedPlan *distributedPlan, int nodeCount) { List *taskList = distributedPlan->workerJob->taskList; ListCell *taskCell = NULL; foreach(taskCell, taskList) { Task *task = lfirst(taskCell); ListCell *placementCell = NULL; foreach(placementCell, task->taskPlacementList) { ShardPlacement *placement = lfirst(placementCell); if (placement->nodeId == LOCAL_NODE_ID) { entry->writeLocalFile = true; continue; } entry->nodeIdList = list_append_unique_int(entry->nodeIdList, placement->nodeId); /* early return if all the workers are accessed */ if (list_length(entry->nodeIdList) == nodeCount && entry->writeLocalFile) { return; } } } } /* * AppendAllWorkerNodes appends all node IDs of readable worker nodes to the * nodeIdList, meaning the corresponding intermediate result should be sent * to all readable nodes. */ static void AppendAllWorkerNodes(IntermediateResultsHashEntry *entry) { List *workerNodeList = ActiveReadableNodeList(); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { entry->nodeIdList = list_append_unique_int(entry->nodeIdList, workerNode->nodeId); } } /* * MakeIntermediateResultHTAB is a helper method that creates a Hash Table that * stores information on the intermediate result. */ HTAB * MakeIntermediateResultHTAB() { HASHCTL info = { 0 }; int initialNumberOfElements = 16; info.keysize = NAMEDATALEN; info.entrysize = sizeof(IntermediateResultsHashEntry); info.hash = string_hash; info.hcxt = CurrentMemoryContext; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); HTAB *intermediateResultsHash = hash_create("Intermediate results hash", initialNumberOfElements, &info, hashFlags); return intermediateResultsHash; } /* * FindAllWorkerNodesUsingSubplan creates a list of worker nodes that * may need to access subplan results. The function also sets writeToLocalFile * flag if the result should also need be written locally. */ List * FindAllWorkerNodesUsingSubplan(HTAB *intermediateResultsHash, char *resultId) { IntermediateResultsHashEntry *entry = SearchIntermediateResult(intermediateResultsHash, resultId); List *remoteWorkerNodes = FindAllRemoteWorkerNodesUsingSubplan(entry); /* * Don't include the current worker if the result will be written to local * file as this would be very inefficient and potentially leading race * conditions while tring to write the same file twice. */ if (entry->writeLocalFile) { remoteWorkerNodes = RemoveLocalNodeFromWorkerList(remoteWorkerNodes); } LogIntermediateResultMulticastSummary(entry, remoteWorkerNodes); return remoteWorkerNodes; } /* * FindAllRemoteWorkerNodesUsingSubplan goes over the nodeIdList of the * intermediate result entry, and returns a list of workerNodes that the * entry should be multi-casted to. The aim of the function is to filter * out nodes with LOCAL_NODE_ID. */ static List * FindAllRemoteWorkerNodesUsingSubplan(IntermediateResultsHashEntry *entry) { List *workerNodeList = NIL; ListCell *nodeIdCell = NULL; foreach(nodeIdCell, entry->nodeIdList) { uint32 nodeId = lfirst_int(nodeIdCell); WorkerNode *workerNode = LookupNodeByNodeId(nodeId); if (workerNode != NULL) { workerNodeList = lappend(workerNodeList, workerNode); } } return workerNodeList; } /* * RemoveLocalNodeFromWorkerList goes over the input workerNode list and * removes the worker node with the local group id, and returns a new list. */ static List * RemoveLocalNodeFromWorkerList(List *workerNodeList) { int32 localGroupId = GetLocalGroupId(); ListCell *workerNodeCell = NULL; foreach(workerNodeCell, workerNodeList) { WorkerNode *workerNode = (WorkerNode *) lfirst(workerNodeCell); if (workerNode->groupId == localGroupId) { return list_delete_cell(workerNodeList, workerNodeCell); } } return workerNodeList; } /* * LogIntermediateResultMulticastSummary is a utility function to DEBUG output * the decisions given on which intermediate result should be sent to which node. * * For details, see the function comments. */ static void LogIntermediateResultMulticastSummary(IntermediateResultsHashEntry *entry, List *workerNodeList) { char *resultId = entry->key; /* * Log a summary of decisions made for intermediate result multicast. By default * we log at level DEBUG4. When the user has set citus.log_intermediate_results * we change the log level to DEBUG1. This is mostly useful in regression tests * where we specifically want to debug this decisions, but not all DEBUG4 messages. */ int logLevel = DEBUG4; if (LogIntermediateResults) { logLevel = DEBUG1; } if (IsLoggableLevel(logLevel)) { if (entry->writeLocalFile) { elog(logLevel, "Subplan %s will be written to local file", resultId); } WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { elog(logLevel, "Subplan %s will be sent to %s:%d", resultId, workerNode->workerName, workerNode->workerPort); } } } /* * SearchIntermediateResult searches through intermediateResultsHash for a given * intermediate result id. * * If an entry is not found, creates a new entry with sane defaults. */ IntermediateResultsHashEntry * SearchIntermediateResult(HTAB *intermediateResultsHash, char *resultId) { bool found = false; IntermediateResultsHashEntry *entry = hash_search(intermediateResultsHash, resultId, HASH_ENTER, &found); /* use sane defaults */ if (!found) { entry->nodeIdList = NIL; entry->writeLocalFile = false; } return entry; } ================================================ FILE: src/backend/distributed/planner/local_distributed_join_planner.c ================================================ /*------------------------------------------------------------------------- * * local_distributed_join_planner.c * * This file contains functions to convert convert local-distributed * tables to subqueries so that they can be planned by the router planner. * * * The current algorithm checks if there is any table in the `jointree` that * should be converted, if so it creates conversion candidates. * With conversion candidates, it will convert either a distributed table or a local table to a * subquery until it is plannable by router planner. It will choose a distributed table if we * expect it to return few rows, such as a constant equality filter on a unique column. * * ```sql * -- assuming dist.a is a unique column, this will convert distributed table * SELECT * FROM dist join local ON(a) where dist.a = 5; * ``` * * If the uniqueness is defined on multiple columns such as `dist.a, dist.b` * then distributed table will only be chosen if there is a constant equality in all of the columns such as: * * ```sql * SELECT * FROM dist join local ON(a) where dist.a = 5 AND dist.b =10; -- this will choose distributed table * SELECT * FROM dist join local ON(a) where dist.a = 5 AND dist.b >10; -- this won't since no equality on dist.b * SELECT * FROM dist join local ON(a) where dist.a = 5; -- this won't since no equality on dist.b * ``` * * The algorithm will also not favor distributed tables if there exists a * distributed table which is expected to return many rows, because in that * case we will already plan local tables hence there is no point in converting some distributed tables. * * ```sql * -- here only the local table will be chosen * SELECT * FROM dist_without_unique JOIN dist_with_unique USING(a) join local USING (a); * ``` * * this also makes the algorithm consistent. * * The algorithm can understand `OR` and `AND` expressions in the filters. * * There is a GUC called `local_table_join_policy` consisting of 4 modes: * `none`: don't do any conversion * `prefer-local`: prefer converting local tables if there is * `prefer-distributed`: prefer converting distributed tables if there is * `auto`: use the above mechanism to decide (constant equality on unique column) * * `auto` mode is the default. * * While converting to a subquery, we use a trick to avoid unnecessary network bandwidth, * if there are columns that are not required in a table that will be converted to a subquery, We do: * * ```sql * SELECT t.a, NULL, NULL (SELECT a FROM table) t * ``` * * instead of * * ```sql * SELECT a, NULL, NULL FROM table * ``` * * There are NULLs in the query because we currently don't have an easy way to update the Vars * that reference the non-required ones and we don't want to break the postgres query. * * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "catalog/pg_class.h" #include "catalog/pg_index.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/listutils.h" #include "distributed/local_distributed_join_planner.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/query_colocation_checker.h" #include "distributed/query_pushdown_planning.h" #include "distributed/recursive_planning.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/shard_pruning.h" #include "distributed/version_compat.h" #define INVALID_RTE_IDENTITY -1 /* * Managed via a GUC */ int LocalTableJoinPolicy = LOCAL_JOIN_POLICY_AUTO; /* * RangeTableEntryDetails contains some information about * a range table entry so that we don't need to calculate * them over and over. */ typedef struct RangeTableEntryDetails { RangeTblEntry *rangeTableEntry; List *requiredAttributeNumbers; bool hasConstantFilterOnUniqueColumn; RTEPermissionInfo *perminfo; } RangeTableEntryDetails; /* * ConversionCandidates contains candidates that could * be converted to a subquery. This is used as a convenience to * first generate all the candidates and then choose which ones to convert. */ typedef struct ConversionCandidates { List *distributedTableList; /* reference or distributed table */ List *localTableList; /* local or citus local table */ }ConversionCandidates; /* * IndexColumns contains the column numbers for an index. * For example if there is an index on (a, b) then it will contain * their column numbers (1,2). */ typedef struct IndexColumns { List *indexColumnNos; }IndexColumns; /* * ConversionChoice represents which conversion group * to convert to a subquery. Currently we either convert all * local tables, or distributed tables. */ typedef enum ConversionChoice { CONVERT_LOCAL_TABLES = 1, CONVERT_DISTRIBUTED_TABLES = 2 }ConversionChoice; static bool HasConstantFilterOnUniqueColumn(RangeTblEntry *rangeTableEntry, RelationRestriction *relationRestriction); static ConversionCandidates * CreateConversionCandidates(PlannerRestrictionContext * plannerRestrictionContext, List *rangeTableList, int resultRTEIdentity, List *rteperminfos); static void AppendUniqueIndexColumnsToList(Form_pg_index indexForm, List **uniqueIndexes, int flags); static ConversionChoice GetConversionChoice(ConversionCandidates * conversionCandidates, PlannerRestrictionContext * plannerRestrictionContext); static bool AllRangeTableEntriesHaveUniqueIndex(List *rangeTableEntryDetailsList); static bool FirstIsSuperSetOfSecond(List *firstIntList, List *secondIntList); static void ConvertRTEsToSubquery(List *rangeTableEntryDetailsList, RecursivePlanningContext *context); static int ResultRTEIdentity(Query *query); static List * RTEListToConvert(ConversionCandidates *conversionCandidates, ConversionChoice conversionChoice); /* * RecursivelyPlanLocalTableJoins gets a query and the planner * restrictions. As long as the query is not plannable by router planner, * it converts either a local or distributed table to a subquery. */ void RecursivelyPlanLocalTableJoins(Query *query, RecursivePlanningContext *context) { PlannerRestrictionContext *plannerRestrictionContext = GetPlannerRestrictionContext(context); List *rangeTableList = query->rtable; List *rteperminfos = query->rteperminfos; int resultRTEIdentity = ResultRTEIdentity(query); ConversionCandidates *conversionCandidates = CreateConversionCandidates(plannerRestrictionContext, rangeTableList, resultRTEIdentity, rteperminfos); ConversionChoice conversionChoise = GetConversionChoice(conversionCandidates, plannerRestrictionContext); List *rteListToConvert = RTEListToConvert(conversionCandidates, conversionChoise); ConvertRTEsToSubquery(rteListToConvert, context); } /* * ResultRTEIdentity returns the result RTE's identity if it exists, * otherwise it returns INVALID_RTE_INDENTITY */ static int ResultRTEIdentity(Query *query) { int resultRTEIdentity = INVALID_RTE_IDENTITY; if (IsModifyCommand(query)) { RangeTblEntry *resultRTE = ExtractResultRelationRTEOrError(query); resultRTEIdentity = GetRTEIdentity(resultRTE); } return resultRTEIdentity; } /* * RTEListToConvert to converts returns a list of RTEs that should * be converted to a subquery. */ static List * RTEListToConvert(ConversionCandidates *conversionCandidates, ConversionChoice conversionChoice) { List *rtesToConvert = NIL; if (conversionChoice == CONVERT_LOCAL_TABLES) { rtesToConvert = list_concat(rtesToConvert, conversionCandidates->localTableList); } else { rtesToConvert = list_concat(rtesToConvert, conversionCandidates->distributedTableList); } return rtesToConvert; } /* * GetConversionChoice returns the conversion choice considering the local table * join policy. */ static ConversionChoice GetConversionChoice(ConversionCandidates *conversionCandidates, PlannerRestrictionContext *plannerRestrictionContext) { RangeTableEntryDetails *localRTECandidate = NULL; RangeTableEntryDetails *distributedRTECandidate = NULL; if (list_length(conversionCandidates->localTableList) > 0) { localRTECandidate = linitial(conversionCandidates->localTableList); } if (list_length(conversionCandidates->distributedTableList) > 0) { distributedRTECandidate = linitial(conversionCandidates->distributedTableList); } if (LocalTableJoinPolicy == LOCAL_JOIN_POLICY_PREFER_LOCAL) { return localRTECandidate ? CONVERT_LOCAL_TABLES : CONVERT_DISTRIBUTED_TABLES; } else if (LocalTableJoinPolicy == LOCAL_JOIN_POLICY_PREFER_DISTRIBUTED) { return distributedRTECandidate ? CONVERT_DISTRIBUTED_TABLES : CONVERT_LOCAL_TABLES; } else { /* * We want to convert distributed tables only if all the distributed tables * have a constant filter on a unique index, otherwise we would be redundantly * converting a distributed table as we will convert all the other local tables. */ bool allRangeTableEntriesHaveUniqueIndex = AllRangeTableEntriesHaveUniqueIndex( conversionCandidates->distributedTableList); if (allRangeTableEntriesHaveUniqueIndex) { return distributedRTECandidate ? CONVERT_DISTRIBUTED_TABLES : CONVERT_LOCAL_TABLES; } else { return localRTECandidate ? CONVERT_LOCAL_TABLES : CONVERT_DISTRIBUTED_TABLES; } } } /* * ConvertRTEsToSubquery converts all the given range table entries * to a subquery. */ static void ConvertRTEsToSubquery(List *rangeTableEntryDetailsList, RecursivePlanningContext *context) { RangeTableEntryDetails *rangeTableEntryDetails = NULL; foreach_declared_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) { RangeTblEntry *rangeTableEntry = rangeTableEntryDetails->rangeTableEntry; List *requiredAttributeNumbers = rangeTableEntryDetails->requiredAttributeNumbers; ReplaceRTERelationWithRteSubquery(rangeTableEntry, requiredAttributeNumbers, context, rangeTableEntryDetails->perminfo); } } /* * AllRangeTableEntriesHaveUniqueIndex returns true if all of the RTE's in the given * list have a unique index. */ static bool AllRangeTableEntriesHaveUniqueIndex(List *rangeTableEntryDetailsList) { RangeTableEntryDetails *rangeTableEntryDetails = NULL; foreach_declared_ptr(rangeTableEntryDetails, rangeTableEntryDetailsList) { if (!rangeTableEntryDetails->hasConstantFilterOnUniqueColumn) { return false; } } return true; } /* * ShouldConvertLocalTableJoinsToSubqueries returns true if we should * convert local-dist table joins to subqueries. */ bool ShouldConvertLocalTableJoinsToSubqueries(List *rangeTableList) { if (LocalTableJoinPolicy == LOCAL_JOIN_POLICY_NEVER) { /* user doesn't want Citus to enable local table joins */ return false; } if (!ContainsLocalTableDistributedTableJoin(rangeTableList)) { return false; } return true; } /* * HasConstantFilterOnUniqueColumn returns true if the given rangeTableEntry has a constant * filter on a unique column. */ static bool HasConstantFilterOnUniqueColumn(RangeTblEntry *rangeTableEntry, RelationRestriction *relationRestriction) { if (rangeTableEntry == NULL || relationRestriction == NULL) { /* * Postgres might not pass relationRestriction info with hooks if * the table doesn't contribute to the result, and in that case * relationRestriction will be NULL. Ideally it doesn't make sense * to recursively plan such tables but for the time being we don't * add any special logic for these tables as it might introduce bugs. */ return false; } bool joinOnFalse = JoinConditionIsOnFalse(relationRestriction->relOptInfo->joininfo); if (joinOnFalse) { /* If there is a WHERE FALSE, we consider it as a constant filter. */ return true; } List *baseRestrictionList = relationRestriction->relOptInfo->baserestrictinfo; List *restrictClauseList = get_all_actual_clauses(baseRestrictionList); List *rteEqualityColumnsNos = FetchEqualityAttrNumsForRTE((Node *) restrictClauseList); List *uniqueIndexColumnsList = ExecuteFunctionOnEachTableIndex(rangeTableEntry->relid, AppendUniqueIndexColumnsToList, INCLUDE_INDEX_ALL_STATEMENTS); IndexColumns *indexColumns = NULL; foreach_declared_ptr(indexColumns, uniqueIndexColumnsList) { List *uniqueIndexColumnNos = indexColumns->indexColumnNos; if (FirstIsSuperSetOfSecond(rteEqualityColumnsNos, uniqueIndexColumnNos)) { return true; } } return false; } /* * FirstIsSuperSetOfSecond returns true if the first int List * contains every element of the second int List. */ static bool FirstIsSuperSetOfSecond(List *firstIntList, List *secondIntList) { int curInt = 0; foreach_declared_int(curInt, secondIntList) { if (!list_member_int(firstIntList, curInt)) { return false; } } return true; } /* * AppendUniqueIndexColumnsToList adds the given index's column numbers if it is a * unique index. */ static void AppendUniqueIndexColumnsToList(Form_pg_index indexForm, List **uniqueIndexGroups, int flags) { if (indexForm->indisunique || indexForm->indisprimary) { IndexColumns *indexColumns = palloc0(sizeof(IndexColumns)); List *uniqueIndexes = NIL; for (int i = 0; i < indexForm->indkey.dim1; i++) { uniqueIndexes = list_append_unique_int(uniqueIndexes, indexForm->indkey.values[i]); } if (list_length(uniqueIndexes) == 0) { return; } indexColumns->indexColumnNos = uniqueIndexes; *uniqueIndexGroups = lappend(*uniqueIndexGroups, indexColumns); } } /* * RequiredAttrNumbersForRelation returns the required attribute numbers for * the input RTE relation in order for the planning to succeed. * * The function could be optimized by not adding the columns that only appear * WHERE clause as a filter (e.g., not a join clause). */ List * RequiredAttrNumbersForRelation(RangeTblEntry *rangeTableEntry, PlannerRestrictionContext *plannerRestrictionContext) { RelationRestriction *relationRestriction = RelationRestrictionForRelation(rangeTableEntry, plannerRestrictionContext); if (relationRestriction == NULL) { return NIL; } PlannerInfo *plannerInfo = relationRestriction->plannerInfo; int rteIndex = relationRestriction->index; /* * Here we used the query from plannerInfo because it has the optimizations * so that it doesn't have unnecessary columns. The original query doesn't have * some of these optimizations hence if we use it here, we don't get the * 'required' attributes. */ Query *queryToProcess = plannerInfo->parse; return RequiredAttrNumbersForRelationInternal(queryToProcess, rteIndex); } /* * RequiredAttrNumbersForRelationInternal returns the required attribute numbers * for the input range-table-index in the query parameter. */ List * RequiredAttrNumbersForRelationInternal(Query *queryToProcess, int rteIndex) { List *allVarsInQuery = pull_vars_of_level((Node *) queryToProcess, 0); List *requiredAttrNumbers = NIL; Var *var = NULL; foreach_declared_ptr(var, allVarsInQuery) { if (var->varno == rteIndex) { requiredAttrNumbers = list_append_unique_int(requiredAttrNumbers, var->varattno); } } return requiredAttrNumbers; } /* * CreateConversionCandidates creates the conversion candidates that might * be converted to a subquery so that citus planners can work. */ static ConversionCandidates * CreateConversionCandidates(PlannerRestrictionContext *plannerRestrictionContext, List *rangeTableList, int resultRTEIdentity, List *rteperminfos) { ConversionCandidates *conversionCandidates = palloc0(sizeof(ConversionCandidates)); RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { /* we're only interested in tables */ if (!IsRecursivelyPlannableRelation(rangeTableEntry)) { continue; } int rteIdentity = GetRTEIdentity(rangeTableEntry); /* result relation cannot converted to a subquery */ if (resultRTEIdentity == rteIdentity) { continue; } RelationRestriction *relationRestriction = RelationRestrictionForRelation(rangeTableEntry, plannerRestrictionContext); RangeTableEntryDetails *rangeTableEntryDetails = palloc0(sizeof(RangeTableEntryDetails)); rangeTableEntryDetails->rangeTableEntry = rangeTableEntry; rangeTableEntryDetails->requiredAttributeNumbers = RequiredAttrNumbersForRelation(rangeTableEntry, plannerRestrictionContext); rangeTableEntryDetails->hasConstantFilterOnUniqueColumn = HasConstantFilterOnUniqueColumn(rangeTableEntry, relationRestriction); rangeTableEntryDetails->perminfo = NULL; if (rangeTableEntry->perminfoindex) { rangeTableEntryDetails->perminfo = getRTEPermissionInfo(rteperminfos, rangeTableEntry); } bool referenceOrDistributedTable = IsCitusTableType(rangeTableEntry->relid, REFERENCE_TABLE) || IsCitusTableType(rangeTableEntry->relid, DISTRIBUTED_TABLE); if (referenceOrDistributedTable) { conversionCandidates->distributedTableList = lappend(conversionCandidates->distributedTableList, rangeTableEntryDetails); } else { conversionCandidates->localTableList = lappend(conversionCandidates->localTableList, rangeTableEntryDetails); } } return conversionCandidates; } ================================================ FILE: src/backend/distributed/planner/local_plan_cache.c ================================================ /*------------------------------------------------------------------------- * * local_plan_cache.c * * Local plan cache related functions * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparse_shard_query.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/local_plan_cache.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/version_compat.h" static Query * GetLocalShardQueryForCache(Query *jobQuery, Task *task, ParamListInfo paramListInfo); static char * DeparseLocalShardQuery(Query *jobQuery, List *relationShardList, Oid anchorDistributedTableId, int64 anchorShardId); static int ExtractParameterTypesForParamListInfo(ParamListInfo originalParamListInfo, Oid **parameterTypes); /* * CacheLocalPlanForShardQuery replaces the relation OIDs in the job query * with shard relation OIDs and then plans the query and caches the result * in the originalDistributedPlan (which may be preserved across executions). */ void CacheLocalPlanForShardQuery(Task *task, DistributedPlan *originalDistributedPlan, ParamListInfo paramListInfo) { PlannedStmt *localPlan = GetCachedLocalPlan(task, originalDistributedPlan); if (localPlan != NULL) { /* we already have a local plan */ return; } if (list_length(task->relationShardList) == 0) { /* zero shard plan, no need to cache */ return; } /* * All memory allocations should happen in the plan's context * since we'll cache the local plan there. */ MemoryContext oldContext = MemoryContextSwitchTo(GetMemoryChunkContext(originalDistributedPlan)); /* * We prefer to use jobQuery (over task->query) because we don't want any * functions/params to have been evaluated in the cached plan. */ Query *jobQuery = copyObject(originalDistributedPlan->workerJob->jobQuery); Query *localShardQuery = GetLocalShardQueryForCache(jobQuery, task, paramListInfo); LOCKMODE lockMode = GetQueryLockMode(localShardQuery); /* fast path queries can only have a single RTE by definition */ RangeTblEntry *rangeTableEntry = (RangeTblEntry *) linitial(localShardQuery->rtable); /* * If the shard has been created in this transction, we wouldn't see the relationId * for it, so do not cache. */ if (rangeTableEntry->relid == InvalidOid) { pfree(jobQuery); pfree(localShardQuery); MemoryContextSwitchTo(oldContext); return; } LockRelationOid(rangeTableEntry->relid, lockMode); LocalPlannedStatement *localPlannedStatement = CitusMakeNode(LocalPlannedStatement); localPlan = planner(localShardQuery, NULL, 0, NULL); localPlannedStatement->localPlan = localPlan; localPlannedStatement->shardId = task->anchorShardId; localPlannedStatement->localGroupId = GetLocalGroupId(); originalDistributedPlan->workerJob->localPlannedStatements = lappend(originalDistributedPlan->workerJob->localPlannedStatements, localPlannedStatement); MemoryContextSwitchTo(oldContext); } /* * GetLocalShardQueryForCache is a helper function which generates * the local shard query based on the jobQuery. The function should * not be used for generic purposes, it is specialized for local cached * queries. * * It is not guaranteed to have consistent attribute numbers on the shards * and on the shell (e.g., distributed/reference tables) due to DROP COLUMN * commands. * * To avoid any edge cases due to such discrepancies, we first deparse the * jobQuery with the tables replaced to shards, and parse the query string * back. This is normally a very expensive operation, however we only do it * once per cached local plan, which is acceptable. */ static Query * GetLocalShardQueryForCache(Query *jobQuery, Task *task, ParamListInfo orig_paramListInfo) { char *shardQueryString = DeparseLocalShardQuery(jobQuery, task->relationShardList, task->anchorDistributedTableId, task->anchorShardId); ereport(DEBUG5, (errmsg("Local shard query that is going to be cached: %s", shardQueryString))); Oid *parameterTypes = NULL; int numberOfParameters = ExtractParameterTypesForParamListInfo(orig_paramListInfo, ¶meterTypes); Query *localShardQuery = ParseQueryString(shardQueryString, parameterTypes, numberOfParameters); return localShardQuery; } /* * DeparseLocalShardQuery is a helper function to deparse given jobQuery for the shard(s) * identified by the relationShardList, anchorDistributedTableId and anchorShardId. * * For the details and comparison with TaskQueryString(), see the comments in the function. */ static char * DeparseLocalShardQuery(Query *jobQuery, List *relationShardList, Oid anchorDistributedTableId, int64 anchorShardId) { StringInfo queryString = makeStringInfo(); /* * We imitate what TaskQueryString() does, but we cannot rely on that function * as the parameters might have been already resolved on the QueryTree in the * task. Instead, we operate on the jobQuery where are sure that the * coordination evaluation has not happened. * * Local shard queries are only applicable for local cached query execution. * In the local cached query execution mode, we can use a query structure * (or query string) with unevaluated expressions as we allow function calls * to be evaluated when the query on the shard is executed (e.g., do no have * coordinator evaluation, instead let Postgres executor evaluate values). * * Additionally, we can allow them to be evaluated again because they are stable, * and we do not cache plans / use unevaluated query strings for queries containing * volatile functions. */ if (jobQuery->commandType == CMD_INSERT) { /* * We currently do not support INSERT .. SELECT here. To support INSERT..SELECT * queries, we should update the relation names to shard names in the SELECT * clause (e.g., UpdateRelationToShardNames()). */ Assert(!CheckInsertSelectQuery(jobQuery)); AddInsertAliasIfNeeded(jobQuery); /* * For INSERT queries we cannot use pg_get_query_def. Mainly because we * cannot run UpdateRelationToShardNames on an INSERT query. This is * because the PG deparsing logic fails when trying to insert into a * RTE_FUNCTION (which is what will happen if you call * UpdateRelationToShardNames). */ deparse_shard_query(jobQuery, anchorDistributedTableId, anchorShardId, queryString); } else { UpdateRelationToShardNames((Node *) jobQuery, relationShardList); pg_get_query_def(jobQuery, queryString); } return queryString->data; } /* * ExtractParameterTypesForParamListInfo is a helper function which helps to * extract the parameter types of the given ParamListInfo via the second * parameter of the function. * * The function also returns the number of parameters. If no parameter exists, * the function returns 0. */ static int ExtractParameterTypesForParamListInfo(ParamListInfo originalParamListInfo, Oid **parameterTypes) { *parameterTypes = NULL; int numberOfParameters = 0; if (originalParamListInfo != NULL) { const char **parameterValues = NULL; ParamListInfo paramListInfo = copyParamList(originalParamListInfo); ExtractParametersForLocalExecution(paramListInfo, parameterTypes, ¶meterValues); numberOfParameters = paramListInfo->numParams; } return numberOfParameters; } /* * GetCachedLocalPlan is a helper function which return the cached * plan in the distributedPlan for the given task if exists. * * Otherwise, the function returns NULL. */ PlannedStmt * GetCachedLocalPlan(Task *task, DistributedPlan *distributedPlan) { if (distributedPlan == NULL || distributedPlan->workerJob == NULL) { return NULL; } if (list_length(distributedPlan->workerJob->taskList) != 1) { /* we only support plan caching for single shard queries */ return NULL; } List *cachedPlanList = distributedPlan->workerJob->localPlannedStatements; LocalPlannedStatement *localPlannedStatement = NULL; int32 localGroupId = GetLocalGroupId(); foreach_declared_ptr(localPlannedStatement, cachedPlanList) { if (localPlannedStatement->shardId == task->anchorShardId && localPlannedStatement->localGroupId == localGroupId) { /* already have a cached plan, no need to continue */ return localPlannedStatement->localPlan; } } return NULL; } /* * IsLocalPlanCachingSupported returns whether (part of) the task can be planned * and executed locally and whether caching is supported (single shard, no volatile * functions). */ bool IsLocalPlanCachingSupported(Job *currentJob, DistributedPlan *originalDistributedPlan) { if (originalDistributedPlan->numberOfTimesExecuted < 1) { /* * Only cache if a plan is being reused (via a prepared statement). */ return false; } if (!currentJob->deferredPruning) { /* * When not using deferred pruning we may have already replaced distributed * table RTEs with citus_extradata_container RTEs to pass the shard ID to the * deparser. In that case, we cannot pass the query tree directly to the * planner. * * If desired, we can relax this check by improving the implementation of * CacheLocalPlanForShardQuery to translate citus_extradata_container * to a shard relation OID. */ return false; } List *taskList = currentJob->taskList; if (list_length(taskList) != 1) { /* we only support plan caching for single shard queries */ return false; } Task *task = linitial(taskList); if (!TaskAccessesLocalNode(task)) { /* not a local task */ return false; } if (!EnableLocalExecution) { /* user requested not to use local execution */ return false; } if (GetCurrentLocalExecutionStatus() == LOCAL_EXECUTION_DISABLED) { /* transaction already connected to localhost */ return false; } Query *originalJobQuery = originalDistributedPlan->workerJob->jobQuery; if (contain_volatile_functions((Node *) originalJobQuery)) { /* * We do not cache plans with volatile functions in the query. * * The reason we care about volatile functions is primarily that we * already executed them in ExecuteCoordinatorEvaluableExpressions * and since we're falling back to the original query tree here we would * execute them again if we execute the plan. */ return false; } return true; } ================================================ FILE: src/backend/distributed/planner/merge_planner.c ================================================ /*------------------------------------------------------------------------- * * merge_planner.c * * This file contains functions to help plan MERGE queries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "tcop/tcopprot.h" #include "utils/lsyscache.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_custom_scan.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/local_distributed_join_planner.h" #include "distributed/merge_planner.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_node_metadata.h" #include "distributed/query_colocation_checker.h" #include "distributed/query_pushdown_planning.h" #include "distributed/repartition_executor.h" #include "distributed/shard_pruning.h" #include "distributed/shared_library_init.h" static int SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, CitusTableCacheEntry *targetRelation); static int FindTargetListEntryWithVarExprAttno(List *targetList, AttrNumber varattno); static Var * ValidateAndReturnVarIfSupported(Node *entryExpr); static DeferredErrorMessage * DeferErrorIfTargetHasFalseClause(Oid targetRelationId, PlannerRestrictionContext * plannerRestrictionContext); static void ErrorIfMergeQueryQualAndTargetListNotSupported(Oid targetRelationId, Query *originalQuery); static void ErrorIfMergeNotSupported(Query *query, Oid targetRelationId, List *rangeTableList); static void ErrorIfMergeHasUnsupportedTables(Oid targetRelationId, List *rangeTableList); static bool IsDistributionColumnInMergeSource(Expr *columnExpression, Query *query, bool skipOuterVars); static DeferredErrorMessage * DeferErrorIfRoutableMergeNotSupported(Query *query, List *rangeTableList, PlannerRestrictionContext * plannerRestrictionContext, Oid targetRelationId); static bool MergeSourceHasRouterSelect(Query *query, PlannerRestrictionContext * plannerRestrictionContext); static DeferredErrorMessage * MergeQualAndTargetListFunctionsSupported(Oid resultRelationId, Query *query, Node *quals, List *targetList, CmdType commandType); static DistributedPlan * CreateRouterMergePlan(Oid targetRelationId, Query *originalQuery, Query *query, List *rangeTableList, PlannerRestrictionContext * plannerRestrictionContext); static void ErrorIfRepartitionMergeNotSupported(Oid targetRelationId, Query *mergeQuery, Query *sourceQuery); static void ConvertSourceRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, PlannerRestrictionContext * plannerRestrictionContext); static void ConvertSubqueryRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte); static void ConvertCteRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte); static void ConvertRelationRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, PlannerRestrictionContext * plannerRestrictionContext); static void ErrorIfUnsupportedCTEs(Query *query); static void ContainsUnsupportedCTEs(Query *query); static bool MergeQueryCTEWalker(Node *node, void *context); static DistributedPlan * CreateNonPushableMergePlan(Oid targetRelationId, uint64 planId, Query *originalQuery, Query *query, PlannerRestrictionContext * plannerRestrictionContext, ParamListInfo boundParams); static char * MergeCommandResultIdPrefix(uint64 planId); static void ErrorIfMergeHasReturningList(Query *query); static Node * GetMergeJoinCondition(Query *mergeQuery); /* * CreateMergePlan * 1) Check for conditions that are not supported in MERGE command. * 2) Try to create a pushable plan * - Check for conditions suitable for a routable plan, if not found, * raise deferred error * 3) Try to create repartition and redistribution plan * - Check for conditions that prevent repartition strategy, if found, * raise an exception and quit. */ DistributedPlan * CreateMergePlan(uint64 planId, Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext, ParamListInfo boundParams) { Oid targetRelationId = ModifyQueryResultRelationId(originalQuery); /* * Step 1: Look for definitive error conditions applicable to both Routable * and Repartition strategies. */ List *rangeTableList = ExtractRangeTableEntryList(originalQuery); ErrorIfMergeNotSupported(originalQuery, targetRelationId, rangeTableList); /* Step 2: Try pushable merge plan */ DistributedPlan *distributedPlan = CreateRouterMergePlan(targetRelationId, originalQuery, query, rangeTableList, plannerRestrictionContext); /* Step 3: If the routing plan failed, try for repartition strategy */ if (distributedPlan->planningError != NULL) { RaiseDeferredError(distributedPlan->planningError, DEBUG1); /* If MERGE is not routable, try repartitioning */ distributedPlan = CreateNonPushableMergePlan(targetRelationId, planId, originalQuery, query, plannerRestrictionContext, boundParams); } return distributedPlan; } /* * GetMergeJoinTree constructs and returns the jointree for a MERGE query. */ FromExpr * GetMergeJoinTree(Query *mergeQuery) { FromExpr *mergeJointree = NULL; #if PG_VERSION_NUM >= PG_VERSION_17 /* * In Postgres 17, the query tree has a specific field for the merge condition. * For deriving the WhereClauseList from the merge condition, we construct a dummy * jointree with an empty fromlist. This works because the fromlist of a merge query * join tree consists of range table references only, and range table references are * disregarded by the WhereClauseList() walker. * Relevant PG17 commit: 0294df2f1 */ mergeJointree = makeFromExpr(NIL, mergeQuery->mergeJoinCondition); #else mergeJointree = mergeQuery->jointree; #endif return mergeJointree; } /* * GetMergeJoinCondition returns the quals of the ON condition */ static Node * GetMergeJoinCondition(Query *mergeQuery) { Node *joinCondition = NULL; #if PG_VERSION_NUM >= PG_VERSION_17 joinCondition = (Node *) mergeQuery->mergeJoinCondition; #else joinCondition = (Node *) mergeQuery->jointree->quals; #endif return joinCondition; } /* * CreateRouterMergePlan attempts to create a pushable plan for the given MERGE * SQL statement. If the planning fails, the ->planningError is set to a description * of the failure. */ static DistributedPlan * CreateRouterMergePlan(Oid targetRelationId, Query *originalQuery, Query *query, List *rangeTableList, PlannerRestrictionContext *plannerRestrictionContext) { DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); Assert(originalQuery->commandType == CMD_MERGE); Assert(OidIsValid(targetRelationId)); distributedPlan->planningError = DeferErrorIfRoutableMergeNotSupported(originalQuery, rangeTableList, plannerRestrictionContext, targetRelationId); if (distributedPlan->planningError != NULL) { return distributedPlan; } Job *job = RouterJob(originalQuery, plannerRestrictionContext, &distributedPlan->planningError); if (distributedPlan->planningError != NULL) { return distributedPlan; } ereport(DEBUG1, (errmsg("Creating MERGE router plan"))); distributedPlan->workerJob = job; distributedPlan->targetRelationId = targetRelationId; distributedPlan->modLevel = RowModifyLevelForQuery(query); /* There is no coordinator query for MERGE */ distributedPlan->combineQuery = NULL; /* MERGE doesn't support RETURNING clause */ distributedPlan->expectResults = false; distributedPlan->fastPathRouterPlan = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; return distributedPlan; } /* * CreateNonPushableMergePlan comes into effect if the router planning fails * and incorporates two planning strategies. * * ExecuteSourceAtWorkerAndRepartition(): Plan the source query independently, * execute the results into intermediate files, and repartition the files to * co-locate them with the merge-target table. Subsequently, compile a final * merge query on the target table using the intermediate results as the data * source. * * ExecuteSourceAtCoordAndRedistribution(): Execute the plan that requires * evaluation at the coordinator, run the query on the coordinator, and * redistribute the resulting rows to ensure colocation with the target shards. * Direct the MERGE SQL operation to the worker nodes' target shards, using the * intermediate files colocated with the data as the data source. */ static DistributedPlan * CreateNonPushableMergePlan(Oid targetRelationId, uint64 planId, Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext, ParamListInfo boundParams) { Query *mergeQuery = copyObject(originalQuery); RangeTblEntry *sourceRte = ExtractMergeSourceRangeTableEntry(mergeQuery, false); DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); ereport(DEBUG1, (errmsg("Creating MERGE repartition plan"))); ConvertSourceRTEIntoSubquery(mergeQuery, sourceRte, plannerRestrictionContext); Query *sourceQuery = sourceRte->subquery; ErrorIfRepartitionMergeNotSupported(targetRelationId, mergeQuery, sourceQuery); CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(targetRelationId); if (IsCitusTableType(targetRelation->relationId, SINGLE_SHARD_DISTRIBUTED)) { /* * if target table is SINGLE_SHARD_DISTRIBUTED let's set this to invalid -1 * so later in execution phase we don't rely on this value and try to find single shard of target instead. */ distributedPlan->sourceResultRepartitionColumnIndex = -1; } else { /* * Get the index of the column in the source query that will be utilized * to repartition the source rows, ensuring colocation with the target */ distributedPlan->sourceResultRepartitionColumnIndex = SourceResultPartitionColumnIndex(mergeQuery, sourceQuery->targetList, targetRelation); } /* * Make a copy of the source query, since following code scribbles it * but we need to keep the original for EXPLAIN. */ Query *sourceQueryCopy = copyObject(sourceQuery); /* plan the subquery, this may be another distributed query */ int cursorOptions = CURSOR_OPT_PARALLEL_OK; PlannedStmt *sourceRowsPlan = pg_plan_query(sourceQueryCopy, NULL, cursorOptions, boundParams); bool isRepartitionAllowed = IsRedistributablePlan(sourceRowsPlan->planTree) && IsSupportedRedistributionTarget(targetRelationId); /* If plan is distributed, no work at the coordinator */ if (isRepartitionAllowed) { distributedPlan->modifyWithSelectMethod = MODIFY_WITH_SELECT_REPARTITION; } else { distributedPlan->modifyWithSelectMethod = MODIFY_WITH_SELECT_VIA_COORDINATOR; } /* There is no coordinator query for MERGE */ distributedPlan->combineQuery = NULL; /* MERGE doesn't support RETURNING clause */ distributedPlan->expectResults = false; distributedPlan->modLevel = RowModifyLevelForQuery(mergeQuery); distributedPlan->targetRelationId = targetRelationId; distributedPlan->intermediateResultIdPrefix = MergeCommandResultIdPrefix(planId); distributedPlan->modifyQueryViaCoordinatorOrRepartition = mergeQuery; distributedPlan->selectPlanForModifyViaCoordinatorOrRepartition = sourceRowsPlan; distributedPlan->fastPathRouterPlan = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; return distributedPlan; } /* * ContainsUnsupportedCTEs checks the CTE if it's modifying or recursive CTE, if true, * raises an exception. */ static void ContainsUnsupportedCTEs(Query *query) { if (query->hasModifyingCTE) { ereport(ERROR, (errmsg("CTEs with modifying actions are not yet " "supported in MERGE"))); } if (query->hasRecursive) { ereport(ERROR, (errmsg("Recursive CTEs are not yet " "supported in MERGE"))); } } /* * MergeQueryCTEWalker descends into the MERGE query to check for any subqueries */ static bool MergeQueryCTEWalker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; ContainsUnsupportedCTEs(query); query_tree_walker(query, MergeQueryCTEWalker, NULL, 0); /* we're done, no need to recurse anymore for this query */ return false; } return expression_tree_walker(node, MergeQueryCTEWalker, context); } /* * ErrorIfUnsupportedCTEs checks for unsupported CTEs, such as, modifying and recursive */ static void ErrorIfUnsupportedCTEs(Query *query) { ContainsUnsupportedCTEs(query); query_tree_walker(query, MergeQueryCTEWalker, NULL, 0); } /* * ErrorIfMergeHasUnsupportedTables checks if all the tables(target, source or any CTE * present) in the MERGE command are local i.e. a combination of Citus local and Non-Citus * tables (regular Postgres tables), or distributed tables with some restrictions * raises an exception for all other combinations. */ static void ErrorIfMergeHasUnsupportedTables(Oid targetRelationId, List *rangeTableList) { RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { Oid relationId = rangeTableEntry->relid; switch (rangeTableEntry->rtekind) { case RTE_RELATION: { /* Check the relation type */ break; } case RTE_SUBQUERY: case RTE_FUNCTION: case RTE_TABLEFUNC: case RTE_VALUES: case RTE_JOIN: case RTE_CTE: #if PG_VERSION_NUM >= PG_VERSION_18 case RTE_GROUP: #endif { /* Skip them as base table(s) will be checked */ continue; } /* * RTE_NAMEDTUPLESTORE is typically used in ephmeral named relations, * such as, trigger data; until we find a genuine use case, raise an * exception. * RTE_RESULT is a node added by the planner and we shouldn't * encounter it in the parse tree. */ case RTE_NAMEDTUPLESTORE: case RTE_RESULT: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("MERGE command is not supported with " "Tuplestores and results"))); break; } default: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "MERGE command: Unrecognized range table entry(%d) ", rangeTableEntry->rtekind))); } } /* RTE Relation can be of various types, check them now */ switch (rangeTableEntry->relkind) { /* skip the regular views as they are replaced with subqueries */ case RELKIND_VIEW: { continue; } case RELKIND_MATVIEW: case RELKIND_FOREIGN_TABLE: { /* These two cases as a target is not allowed */ if (relationId == targetRelationId) { /* Usually we don't reach this exception as the Postgres parser catches it */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("MERGE command is not allowed on " "relation type(relkind:%c)", rangeTableEntry->relkind))); } break; } case RELKIND_RELATION: case RELKIND_PARTITIONED_TABLE: { /* Check for citus/postgres table types */ Assert(OidIsValid(relationId)); break; } default: { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Unexpected table type(relkind:%c) " "in MERGE command", rangeTableEntry->relkind))); } } /* * Check for unsupported distributed tables */ if (extern_IsColumnarTableAmTable(relationId) && relationId == targetRelationId) { /* Columnar tables are not supported */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Columnar table as target is " "not allowed in MERGE command"))); } else if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { /* Append/Range distributed tables are not supported */ if (IsCitusTableType(relationId, APPEND_DISTRIBUTED) || IsCitusTableType(relationId, RANGE_DISTRIBUTED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("For MERGE command, append/range distribution " "table is not supported yet"))); } } else if (IsCitusTableType(relationId, REFERENCE_TABLE) && relationId == targetRelationId) { /* Reference table as a target is not allowed */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Reference table as target is " "not allowed in MERGE command"))); } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { /* * All the tables are local/reference, supported as long as * coordinator is in the metadata. */ if (FindCoordinatorNodeId() == -1) { ereport(ERROR, (errmsg("Coordinator node is not in " "the metadata"), errhint("To ensure that the distributed planner " "planner the Citus table, please consider " "configuring a coordinator node"))); } } } } /* * IsPartitionColumnInMerge returns true if the given column is a partition column. * The function uses FindReferencedTableColumn to find the original relation * id and column that the column expression refers to. It then checks whether * that column is a partition column of the relation. * * Also, the function returns always false for reference tables given that * reference tables do not have partition column. * * If skipOuterVars is true, then it doesn't process the outervars. */ bool IsDistributionColumnInMergeSource(Expr *columnExpression, Query *query, bool skipOuterVars) { bool isDistributionColumn = false; Var *column = NULL; RangeTblEntry *relationRTE = NULL; /* ParentQueryList is same as the original query for MERGE */ FindReferencedTableColumn(columnExpression, list_make1(query), query, &column, &relationRTE, skipOuterVars); Oid relationId = relationRTE ? relationRTE->relid : InvalidOid; if (relationId != InvalidOid && column != NULL) { Var *distributionColumn = DistPartitionKey(relationId); /* not all distributed tables have partition column */ if (distributionColumn != NULL && column->varattno == distributionColumn->varattno) { isDistributionColumn = true; } } return isDistributionColumn; } /* * MergeQualAndTargetListFunctionsSupported Checks WHEN/ON clause actions to see what functions * are allowed, if we are updating distribution column, etc. */ static DeferredErrorMessage * MergeQualAndTargetListFunctionsSupported(Oid resultRelationId, Query *query, Node *quals, List *targetList, CmdType commandType) { uint32 targetRangeTableIndex = query->resultRelation; FromExpr *joinTree = GetMergeJoinTree(query); Var *distributionColumn = NULL; if (IsCitusTable(resultRelationId) && HasDistributionKey(resultRelationId)) { distributionColumn = PartitionColumn(resultRelationId, targetRangeTableIndex); } ListCell *targetEntryCell = NULL; bool hasVarArgument = false; /* A STABLE function is passed a Var argument */ bool hasBadCoalesce = false; /* CASE/COALESCE passed a mutable function */ foreach(targetEntryCell, targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); bool targetEntryDistributionColumn = false; AttrNumber targetColumnAttrNumber = InvalidAttrNumber; if (distributionColumn) { if (commandType == CMD_UPDATE) { /* * Note that it is not possible to give an alias to * UPDATE table SET ... */ if (targetEntry->resname) { targetColumnAttrNumber = get_attnum(resultRelationId, targetEntry->resname); if (targetColumnAttrNumber == distributionColumn->varattno) { targetEntryDistributionColumn = true; } } } } /* * joinTree->quals, retrieved by GetMergeJoinTree() - either from * mergeJoinCondition (PG >= 17) or jointree->quals (PG < 17), * only contains the quals that present in "ON (..)" clause. Action * quals that can be specified for each specific action, as in * "WHEN AND THEN "", are * saved into "qual" field of the corresponding action's entry in * mergeActionList, see * https://github.com/postgres/postgres/blob/e6da68a6e1d60a037b63a9c9ed36e5ef0a996769/src/backend/parser/parse_merge.c#L285-L293. * * For this reason, even if TargetEntryChangesValue() could prove that * an action's quals ensure that the action cannot change the distribution * key, this is not the case as we don't provide action quals to * TargetEntryChangesValue(), but just joinTree, which only contains * the "ON (..)" clause quals. */ if (targetEntryDistributionColumn && TargetEntryChangesValue(targetEntry, distributionColumn, joinTree)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "updating the distribution column is not " "allowed in MERGE actions", NULL, NULL); } if (FindNodeMatchingCheckFunction((Node *) targetEntry->expr, CitusIsVolatileFunction)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in MERGE actions on distributed " "tables must not be VOLATILE", NULL, NULL); } if (MasterIrreducibleExpression((Node *) targetEntry->expr, &hasVarArgument, &hasBadCoalesce)) { Assert(hasVarArgument || hasBadCoalesce); } if (FindNodeMatchingCheckFunction((Node *) targetEntry->expr, NodeIsFieldStore)) { /* DELETE cannot do field indirection already */ Assert(commandType == CMD_UPDATE || commandType == CMD_INSERT); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "inserting or modifying composite type fields is not " "supported", NULL, "Use the column name to insert or update the composite " "type as a single value"); } } /* * Check the condition, convert list of expressions into expression tree for further processing */ if (quals) { if (IsA(quals, List)) { quals = (Node *) make_ands_explicit((List *) quals); } if (FindNodeMatchingCheckFunction((Node *) quals, CitusIsVolatileFunction)) { StringInfo errorMessage = makeStringInfo(); appendStringInfo(errorMessage, "functions used in the %s clause of MERGE " "queries on distributed tables must not be VOLATILE", (commandType == CMD_MERGE) ? "ON" : "WHEN"); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorMessage->data, NULL, NULL); } else if (MasterIrreducibleExpression(quals, &hasVarArgument, &hasBadCoalesce)) { Assert(hasVarArgument || hasBadCoalesce); } } if (hasVarArgument) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "STABLE functions used in MERGE queries " "cannot be called with column references", NULL, NULL); } if (hasBadCoalesce) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "non-IMMUTABLE functions are not allowed in CASE or " "COALESCE statements", NULL, NULL); } if (quals != NULL && nodeTag(quals) == T_CurrentOfExpr) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot run MERGE actions with cursors", NULL, NULL); } return NULL; } /* * RepartitionMergeSupported checks if certain conditions cannot accommodate the * strategy of repartition and redistribution of source rows, the routine will verify * them and subsequently raises an exception. */ static void ErrorIfRepartitionMergeNotSupported(Oid targetRelationId, Query *mergeQuery, Query *sourceQuery) { if (!IsCitusTableType(targetRelationId, DISTRIBUTED_TABLE)) { ereport(ERROR, (errmsg("MERGE involving repartition of rows " "is supported only if the target is distributed"))); } RTEListProperties *queryRteListProperties = GetRTEListPropertiesForQuery(mergeQuery); if (queryRteListProperties->hasPostgresLocalTable) { ereport(ERROR, (errmsg("MERGE INTO an distributed table from " "Postgres table is not yet supported"))); } queryRteListProperties = GetRTEListPropertiesForQuery(sourceQuery); if (!queryRteListProperties->hasCitusTable) { ereport(ERROR, (errmsg("To MERGE into a distributed table, source must " "be Citus table(s)"))); } /* * Sub-queries and CTEs are not allowed in actions and ON clause */ Node *joinCondition = GetMergeJoinCondition(mergeQuery); if (FindNodeMatchingCheckFunction(joinCondition, IsNodeSubquery)) { ereport(ERROR, (errmsg("Sub-queries and CTEs are not allowed in ON clause for MERGE " "with repartitioning"), errhint("Consider making the source and target colocated " "and joined on the distribution column to make it a " "routable query"))); } MergeAction *action = NULL; foreach_declared_ptr(action, mergeQuery->mergeActionList) { if (FindNodeMatchingCheckFunction((Node *) action, IsNodeSubquery)) { ereport(ERROR, (errmsg("Sub-queries and CTEs are not allowed in actions for MERGE " "with repartitioning"), errhint("Consider making the source and target colocated " "and joined on the distribution column to make it a " "routable query"))); } } } /* * ConvertCteRTEIntoSubquery takes a RTE_CTE and converts it into a RTE_SUBQUERY. */ static void ConvertCteRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte) { CommonTableExpr *sourceCte = NULL; CommonTableExpr *candidateCte = NULL; List *cteList = NIL; /* * Presently, CTEs are only permitted within the USING clause, and thus, * we search for the corresponding one */ foreach_declared_ptr(candidateCte, mergeQuery->cteList) { if (strcmp(candidateCte->ctename, sourceRte->ctename) == 0) { /* The source CTE that will be converted to a subquery */ sourceCte = candidateCte; } else { /* * Save any other CTEs that are referenced, either directly * or indirectly, in the source CTE. */ cteList = lappend(cteList, candidateCte); } } Assert(sourceCte); Query *cteQuery = (Query *) copyObject(sourceCte->ctequery); sourceRte->rtekind = RTE_SUBQUERY; /* sanity check - sourceRte was RTE_CTE previously so it should have no perminfo */ Assert(sourceRte->perminfoindex == 0); /* * As we are delinking the CTE from main query, we have to walk through the * tree and decrement the ctelevelsup, but by wrapping a subquery, we avoid * adjusting the ctelevelsup in RTE's */ sourceRte->subquery = WrapSubquery(cteQuery); /* Copy the rest of the CTEs(if any) and remove them from main query */ sourceRte->subquery->cteList = copyObject(cteList); mergeQuery->cteList = NIL; /* Zero out CTE-specific fields */ sourceRte->security_barrier = false; sourceRte->ctename = NULL; sourceRte->ctelevelsup = 0; sourceRte->self_reference = false; sourceRte->coltypes = NIL; sourceRte->coltypmods = NIL; sourceRte->colcollations = NIL; } /* * ConvertRelationRTEIntoSubquery takes a RTE_RELATION and converts it into a RTE_SUBQUERY, * which is basically a SELECT * FROM the relation. */ static void ConvertRelationRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, PlannerRestrictionContext *plannerRestrictionContext) { Query *sourceResultsQuery = makeNode(Query); RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); List *requiredAttributes = NIL; RelationRestriction *relationRestriction = RelationRestrictionForRelation(sourceRte, plannerRestrictionContext); if (relationRestriction) { requiredAttributes = RequiredAttrNumbersForRelationInternal(mergeQuery, relationRestriction->index); } sourceResultsQuery->commandType = CMD_SELECT; /* we copy the input rteRelation to preserve the rteIdentity */ RangeTblEntry *newRangeTableEntry = copyObject(sourceRte); sourceResultsQuery->rtable = list_make1(newRangeTableEntry); sourceResultsQuery->rteperminfos = NIL; if (sourceRte->perminfoindex) { /* create permission info for newRangeTableEntry */ RTEPermissionInfo *perminfo = getRTEPermissionInfo(mergeQuery->rteperminfos, sourceRte); /* update the sourceResultsQuery's rteperminfos accordingly */ newRangeTableEntry->perminfoindex = 1; sourceResultsQuery->rteperminfos = list_make1(perminfo); } /* set the FROM expression to the subquery */ newRangeTableRef->rtindex = SINGLE_RTE_INDEX; sourceResultsQuery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); sourceResultsQuery->targetList = CreateFilteredTargetListForRelation(sourceRte->relid, requiredAttributes); List *restrictionList = GetRestrictInfoListForRelation(sourceRte, plannerRestrictionContext); List *copyRestrictionList = copyObject(restrictionList); Expr *andedBoundExpressions = make_ands_explicit(copyRestrictionList); sourceResultsQuery->jointree->quals = (Node *) andedBoundExpressions; /* * Originally the quals were pointing to the RTE and its varno * was pointing to its index in rtable. However now we converted the RTE * to a subquery and the quals should be pointing to that subquery, which * is the only RTE in its rtable, hence we update the varnos so that they * point to the subquery RTE. * Originally: rtable: [rte1, current_rte, rte3...] * Now: rtable: [rte1, subquery[current_rte], rte3...] --subquery[current_rte] refers to its rtable. */ Node *quals = sourceResultsQuery->jointree->quals; UpdateVarNosInNode(quals, SINGLE_RTE_INDEX); /* replace the function with the constructed subquery */ sourceRte->rtekind = RTE_SUBQUERY; sourceRte->perminfoindex = 0; sourceRte->subquery = sourceResultsQuery; sourceRte->inh = false; } /* * ConvertSubqueryRTEIntoSubquery takes a RTE_SUBQUERY and wraps it into a new * subquery, which eliminates any resjunk columns and adjusts the CTE levelsup. * In addition, if the subquery happens to be a SET operation, such as, * (SELECT * from a UNION SELECT * FROM b), it reorders, adds casts and * prepares a single taget list */ static void ConvertSubqueryRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte) { sourceRte->subquery = WrapSubquery(sourceRte->subquery); if (list_length(mergeQuery->cteList) > 0) { /* copy CTEs from the MERGE ... INTO statement into source subquery */ sourceRte->subquery->cteList = copyObject(mergeQuery->cteList); sourceRte->subquery->hasModifyingCTE = mergeQuery->hasModifyingCTE; mergeQuery->cteList = NIL; } } /* * ConvertSourceRTEIntoSubquery converts MERGE's source RTE into a subquery, * whose result rows are repartitioned during runtime. */ static void ConvertSourceRTEIntoSubquery(Query *mergeQuery, RangeTblEntry *sourceRte, PlannerRestrictionContext *plannerRestrictionContext) { switch (sourceRte->rtekind) { case RTE_SUBQUERY: { ConvertSubqueryRTEIntoSubquery(mergeQuery, sourceRte); return; } case RTE_RELATION: { ConvertRelationRTEIntoSubquery(mergeQuery, sourceRte, plannerRestrictionContext); return; } case RTE_CTE: { ConvertCteRTEIntoSubquery(mergeQuery, sourceRte); return; } default: { ereport(ERROR, (errmsg("Currently, Citus only supports " "table, subquery, and CTEs as " "valid sources for the MERGE " "operation"))); } } } /* * ErrorIfMergeHasReturningList raises an exception if the MERGE has * a RETURNING clause, as we don't support this yet for Citus tables * Relevant PG17 commit: c649fa24a */ static void ErrorIfMergeHasReturningList(Query *query) { if (query->returningList) { ereport(ERROR, (errmsg("MERGE with RETURNING is not yet supported " "for Citus tables"))); } } /* * ErrorIfMergeNotSupported Checks for conditions that are not supported in either * the routable or repartition strategies. It checks for * - MERGE with a RETURNING clause * - Supported table types and their combinations * - Check the target lists and quals of both the query and merge actions * - Supported CTEs */ static void ErrorIfMergeNotSupported(Query *query, Oid targetRelationId, List *rangeTableList) { ErrorIfMergeHasReturningList(query); ErrorIfMergeHasUnsupportedTables(targetRelationId, rangeTableList); ErrorIfMergeQueryQualAndTargetListNotSupported(targetRelationId, query); ErrorIfUnsupportedCTEs(query); } /* * DeferErrorIfTargetHasFalseClause checks for the presence of a false clause in the * target relation and throws an exception if found. Router planner prunes all the shards * for relations with such clauses, resulting in no task generation for the job. However, * in the case of a MERGE query, tasks still need to be generated for the shards of the * source relation. */ static DeferredErrorMessage * DeferErrorIfTargetHasFalseClause(Oid targetRelationId, PlannerRestrictionContext *plannerRestrictionContext) { ListCell *restrictionCell = NULL; foreach( restrictionCell, plannerRestrictionContext->relationRestrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(restrictionCell); Oid relationId = relationRestriction->relationId; /* Check only for target relation */ if (relationId != targetRelationId) { continue; } List *baseRestrictionList = relationRestriction->relOptInfo->baserestrictinfo; List *restrictClauseList = get_all_actual_clauses(baseRestrictionList); if (ContainsFalseClause(restrictClauseList) || JoinConditionIsOnFalse(relationRestriction->relOptInfo->joininfo)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Routing query is not possible with " "no shards for target", NULL, NULL); } } return NULL; } /* * DeferErrorIfRoutableMergeNotSupported Checks for conditions that prevent pushable planning, if * found, raises a deferred error, which then continues to try repartitioning strategy. */ static DeferredErrorMessage * DeferErrorIfRoutableMergeNotSupported(Query *query, List *rangeTableList, PlannerRestrictionContext * plannerRestrictionContext, Oid targetRelationId) { List *distTablesList = NIL; List *refTablesList = NIL; List *localTablesList = NIL; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { Oid relationId = rangeTableEntry->relid; if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { distTablesList = lappend(distTablesList, rangeTableEntry); } else if (IsCitusTableType(relationId, REFERENCE_TABLE)) { refTablesList = lappend(refTablesList, rangeTableEntry); } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { localTablesList = lappend(localTablesList, rangeTableEntry); } } if (list_length(distTablesList) > 0 && list_length(refTablesList) > 0) { ereport(DEBUG1, (errmsg( "A mix of distributed and reference table, try repartitioning"))); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "A mix of distributed and reference table, " "routable query is not possible", NULL, NULL); } if (list_length(distTablesList) > 0 && list_length(localTablesList) > 0) { ereport(DEBUG1, (errmsg("A mix of distributed and local table, " "try repartitioning"))); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "A mix of distributed and citus-local table, " "routable query is not possible", NULL, NULL); } /* * If all tables are either local or reference tables, no need to proceed further down * as the below checks are applicable for distributed tables only */ if (list_length(distTablesList) == 0) { return NULL; } /* Only one distributed table is involved in the MERGE */ if (list_length(distTablesList) == 1) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "There is only one distributed table, merge is not " "pushable, try repartitioning", NULL, NULL); } /* Ensure all distributed tables are indeed co-located */ if (!AllDistributedRelationsInRTEListColocated(distTablesList)) { ereport(DEBUG1, (errmsg("Distributed tables are not co-located, try " "repartitioning"))); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "For MERGE command, all the distributed tables " "must be colocated", NULL, NULL); } DeferredErrorMessage *deferredError = NULL; /* * if the query goes to a single node ("router" in Citus' parlance), * we don't need to go through certain SQL support and colocation checks. * * For PG16+, this is required as some of the outer JOINs are converted to * "ON(true)" and filters are pushed down to the table scans. As * DeferErrorIfUnsupportedSubqueryPushdown rely on JOIN filters, it will fail to * detect the router case. However, we can still detect it by checking if * the query is a router query as the router query checks the filters on * the tables. */ if (!MergeSourceHasRouterSelect(query, plannerRestrictionContext)) { deferredError = DeferErrorIfUnsupportedSubqueryPushdown(query, plannerRestrictionContext, true); if (deferredError) { ereport(DEBUG1, (errmsg("Sub-query is not pushable, try repartitioning"))); return deferredError; } if (HasDangerousJoinUsing(query->rtable, (Node *) query->jointree)) { ereport(DEBUG1, (errmsg( "Query has ambigious joins, merge is not pushable, try repartitioning"))); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "a join with USING causes an internal naming " "conflict, use ON instead", NULL, NULL); } } deferredError = DeferErrorIfTargetHasFalseClause(targetRelationId, plannerRestrictionContext); if (deferredError) { ereport(DEBUG1, (errmsg("Target relation has a filter of the " "form: false (AND ..), which results " "in empty shards, but we still need " "to evaluate NOT-MATCHED clause, try " "repartitioning"))); return deferredError; } /* * If execution has reached this point, it indicates that the query can be delegated to the worker. * However, before proceeding with this delegation, we need to confirm that the user is utilizing * the distribution column of the source table in the Insert variable. * If this is not the case, we should refrain from pushing down the query. * This is just a deffered error which will be handle by caller. */ Var *insertVar = FetchAndValidateInsertVarIfExists(targetRelationId, query); if (insertVar && !IsDistributionColumnInMergeSource((Expr *) insertVar, query, true)) { ereport(DEBUG1, (errmsg( "MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied"))); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "MERGE INSERT must use the source table distribution column value for push down to workers. Otherwise, repartitioning will be applied", NULL, NULL); } return NULL; } /* * MergeSourceHasRouterSelect is a helper function that returns true of the source * part of the merge query is a router query. */ static bool MergeSourceHasRouterSelect(Query *query, PlannerRestrictionContext *plannerRestrictionContext) { Query *copiedQuery = copyObject(query); RangeTblEntry *mergeSourceRte = ExtractMergeSourceRangeTableEntry(copiedQuery, true); if (mergeSourceRte == NULL) { /* * We might potentially support this case in the future, but for now, * we don't support MERGE with JOIN in the source. */ return false; } ConvertSourceRTEIntoSubquery(copiedQuery, mergeSourceRte, plannerRestrictionContext); Query *sourceQuery = mergeSourceRte->subquery; DistributedPlan *distributedPlan = CreateRouterPlan(sourceQuery, sourceQuery, plannerRestrictionContext); return distributedPlan->planningError == NULL; } /* * ErrorIfMergeQueryQualAndTargetListNotSupported does check for a MERGE command in the query, if it finds * one, it will verify the below criteria * - Distributed tables co-location requirements * - Checks target-lists and functions-in-quals in TargetlistAndFunctionsSupported */ static void ErrorIfMergeQueryQualAndTargetListNotSupported(Oid targetRelationId, Query *originalQuery) { /* * TODO: For now, we are adding an exception where any volatile or stable * functions are not allowed in the MERGE query, but this will become too * restrictive as this will prevent many useful and simple cases, such as, * INSERT VALUES(ts::timestamp), bigserial column inserts etc. But without * this restriction, we have a potential danger of some of the function(s) * getting executed at the worker which will result in incorrect behavior. */ if (contain_mutable_functions((Node *) originalQuery)) { ereport(ERROR, (errmsg("non-IMMUTABLE functions are not yet " "supported in MERGE sql with distributed tables"))); } Node *joinCondition = GetMergeJoinCondition(originalQuery); DeferredErrorMessage *deferredError = MergeQualAndTargetListFunctionsSupported( targetRelationId, originalQuery, joinCondition, originalQuery->targetList, originalQuery->commandType); if (deferredError) { RaiseDeferredError(deferredError, ERROR); } /* * MERGE is a special case where we have multiple modify statements * within itself. Check each INSERT/UPDATE/DELETE individually. */ MergeAction *action = NULL; foreach_declared_ptr(action, originalQuery->mergeActionList) { Assert(originalQuery->returningList == NULL); deferredError = MergeQualAndTargetListFunctionsSupported(targetRelationId, originalQuery, action->qual, action->targetList, action->commandType); if (deferredError) { /* MERGE's unsupported scenario, raise the exception */ RaiseDeferredError(deferredError, ERROR); } } } /* * MergeCommandResultIdPrefix returns the prefix to use for intermediate results of * an MERGE INTO ... USING source-query results via the coordinator. */ static char * MergeCommandResultIdPrefix(uint64 planId) { StringInfo resultIdPrefix = makeStringInfo(); appendStringInfo(resultIdPrefix, "merge_into_" UINT64_FORMAT, planId); return resultIdPrefix->data; } /* * ValidateAndReturnVarIfSupported Checks for valid expressions of type Var, and * returns the Var if it finds one, for everything else, raises an exception. */ static Var * ValidateAndReturnVarIfSupported(Node *entryExpr) { if (!IsA(entryExpr, Var)) { ereport(ERROR, (errmsg("MERGE INSERT is using unsupported expression type " "for distribution column"), errdetail("Inserting arbitrary values that don't correspond " "to the joined column values can lead to unpredictable " "outcomes where rows are incorrectly distributed " "among different shards"))); } /* Found a Var inserting into target's distribution column */ return (Var *) entryExpr; } /* * SourceResultPartitionColumnIndex collects all Join conditions from the * ON clause and verifies if there is a join, either left or right, with * the distribution column of the given target. Once a match is found, it * returns the index of that match in the source's target list. */ static int SourceResultPartitionColumnIndex(Query *mergeQuery, List *sourceTargetList, CitusTableCacheEntry *targetRelation) { List *mergeJoinConditionList = WhereClauseList(GetMergeJoinTree(mergeQuery)); Var *targetColumn = targetRelation->partitionColumn; Var *sourceRepartitionVar = NULL; bool foundTypeMismatch = false; OpExpr *validJoinClause = SinglePartitionJoinClause(list_make1(targetColumn), mergeJoinConditionList, &foundTypeMismatch); if (!validJoinClause) { if (foundTypeMismatch) { ereport(ERROR, (errmsg("In the MERGE ON clause, there is a datatype mismatch " "between target's distribution " "column and the expression originating from the source."), errdetail( "If the types are different, Citus uses different hash " "functions for the two column types, which might " "lead to incorrect repartitioning of the result data"))); } ereport(ERROR, (errmsg("The required join operation is missing between " "the target's distribution column and any " "expression originating from the source. The " "issue may arise from a non-equi-join."), errdetail("Without a equi-join condition on the target's " "distribution column, the source rows " "cannot be efficiently redistributed, and " "the NOT-MATCHED condition cannot be evaluated " "unambiguously. This can result in incorrect or " "unexpected results when attempting to merge " "tables in a distributed setting"))); } /* both are verified in SinglePartitionJoinClause to not be NULL, assert is to guard */ Var *leftColumn = LeftColumnOrNULL(validJoinClause); Var *rightColumn = RightColumnOrNULL(validJoinClause); Assert(leftColumn != NULL); Assert(rightColumn != NULL); if (equal(targetColumn, leftColumn)) { sourceRepartitionVar = rightColumn; } else if (equal(targetColumn, rightColumn)) { sourceRepartitionVar = leftColumn; } /* Either we find an insert-action or it's not relevant for certain class of tables */ Var *insertVar = FetchAndValidateInsertVarIfExists(targetRelation->relationId, mergeQuery); if (insertVar) { /* INSERT action, must choose joining column for inserted value */ bool joinedOnInsertColumn = JoinOnColumns(list_make1(targetColumn), insertVar, mergeJoinConditionList); if (joinedOnInsertColumn) { sourceRepartitionVar = insertVar; } else { ereport(ERROR, (errmsg("MERGE INSERT must use the " "source's joining column for " "target's distribution column"))); } } Assert(sourceRepartitionVar); int sourceResultRepartitionColumnIndex = FindTargetListEntryWithVarExprAttno(sourceTargetList, sourceRepartitionVar->varattno); if (sourceResultRepartitionColumnIndex == -1) { ereport(ERROR, (errmsg("Unexpected column index of the source list"))); } else { ereport(DEBUG1, (errmsg("Using column - index:%d from the source list " "to redistribute", sourceResultRepartitionColumnIndex))); } return sourceResultRepartitionColumnIndex; } /* * ExtractMergeSourceRangeTableEntry returns the range table entry of source * table or source query in USING clause. */ RangeTblEntry * ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk) { Assert(IsMergeQuery(query)); List *fromList = query->jointree->fromlist; /* * We should have only one RTE(MergeStmt->sourceRelation) in the from-list * unless Postgres community changes the representation of merge. */ if (list_length(fromList) != 1) { ereport(ERROR, (errmsg("Unexpected source list in MERGE sql USING clause"))); } RangeTblRef *reference = linitial(fromList); /* * The planner sometimes generates JoinExprs internally; these can * have rtindex = 0 if there are no join alias variables referencing * such joins. */ if (reference->rtindex == 0) { if (!joinSourceOk) { ereport(ERROR, (errmsg("Source is not an explicit query"), errhint("Source query is a Join expression, " "try converting into a query as SELECT * " "FROM (..Join..)"))); } return NULL; } Assert(reference->rtindex >= 1); RangeTblEntry *subqueryRte = rt_fetch(reference->rtindex, query->rtable); return subqueryRte; } /* * FetchAndValidateInsertVarIfExists checks to see if MERGE is inserting a * value into the target which is not from the source table, if so, it * raises an exception. The return value is the Var that's being inserted * into the target's distribution column, If no INSERT action exist, it * simply returns a NULL. * Note: Inserting random values other than the joined column values will * result in unexpected behaviour of rows ending up in incorrect shards, to * prevent such mishaps, we disallow such inserts here. */ Var * FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query) { Assert(IsMergeQuery(query)); if (!IsCitusTableType(targetRelationId, DISTRIBUTED_TABLE)) { return NULL; } if (!HasDistributionKey(targetRelationId)) { return NULL; } bool foundDistributionColumn = false; MergeAction *action = NULL; uint32 targetRangeTableIndex = query->resultRelation; foreach_declared_ptr(action, query->mergeActionList) { /* Skip MATCHED clause as INSERTS are not allowed in it */ if (matched_compat(action)) { continue; } /* NOT MATCHED can have either INSERT, DO NOTHING or UPDATE(PG17) */ if (action->commandType == CMD_NOTHING || action->commandType == CMD_UPDATE) { return NULL; } if (action->targetList == NIL) { /* INSERT DEFAULT VALUES is not allowed */ ereport(ERROR, (errmsg("cannot perform MERGE INSERT with DEFAULTS"), errdetail("Inserting arbitrary values that don't correspond " "to the joined column values can lead to " "unpredictable outcomes where rows are " "incorrectly distributed among different " "shards"))); } Assert(action->commandType == CMD_INSERT); Var *targetDistributionKey = PartitionColumn(targetRelationId, targetRangeTableIndex); TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, action->targetList) { AttrNumber originalAttrNo = targetEntry->resno; /* skip processing of target table non-distribution columns */ if (originalAttrNo != targetDistributionKey->varattno) { continue; } foundDistributionColumn = true; Node *insertExpr = strip_implicit_coercions((Node *) copyObject(targetEntry->expr)); return ValidateAndReturnVarIfSupported(insertExpr); } if (!foundDistributionColumn) { ereport(ERROR, (errmsg("MERGE INSERT must have distribution column as value"))); } } return NULL; } /* * FindTargetListEntryWithVarExprAttno finds the index of the target * entry whose expr is a Var that points to input varattno. * * If no such target entry is found, it returns -1. */ static int FindTargetListEntryWithVarExprAttno(List *targetList, AttrNumber varattno) { int targetEntryIndex = 0; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, targetList) { if (IsA(targetEntry->expr, Var) && ((Var *) targetEntry->expr)->varattno == varattno) { return targetEntryIndex; } targetEntryIndex++; } return -1; } /* * IsLocalTableModification returns true if the table modified is a Postgres table. * We do not support recursive planning for MERGE yet, so we could have a join * between local and Citus tables. Only allow local tables when it is the target table. */ bool IsLocalTableModification(Oid targetRelationId, Query *query, uint64 shardId, RTEListProperties *rteProperties) { /* No-op for SELECT command */ if (!IsModifyCommand(query)) { return false; } /* For MERGE, we have to check only the target relation */ if (IsMergeQuery(query) && !IsCitusTable(targetRelationId)) { /* Postgres table */ return true; } if (shardId == INVALID_SHARD_ID && ContainsOnlyLocalOrReferenceTables(rteProperties)) { return true; } return false; } ================================================ FILE: src/backend/distributed/planner/multi_explain.c ================================================ /*------------------------------------------------------------------------- * * multi_explain.c * Citus explain support. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "commands/copy.h" #include "commands/createas.h" #include "commands/dbcommands.h" #include "commands/explain.h" #include "commands/tablecmds.h" #include "executor/tstoreReceiver.h" #include "lib/stringinfo.h" #include "nodes/nodeFuncs.h" #include "nodes/plannodes.h" #include "nodes/primnodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" #include "optimizer/planner.h" #include "parser/analyze.h" #include "portability/instr_time.h" #include "rewrite/rewriteHandler.h" #include "tcop/dest.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/snapmgr.h" #include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_18 #include "commands/explain_dr.h" /* CreateExplainSerializeDestReceiver() */ #include "commands/explain_format.h" #endif #include "distributed/citus_depended_object.h" #include "distributed/citus_nodefuncs.h" #include "distributed/combine_query_planner.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_planner.h" #include "distributed/executor_util.h" #include "distributed/insert_select_executor.h" #include "distributed/insert_select_planner.h" #include "distributed/jsonbutils.h" #include "distributed/listutils.h" #include "distributed/merge_planner.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/placement_connection.h" #include "distributed/recursive_planning.h" #include "distributed/remote_commands.h" #include "distributed/subplan_execution.h" #include "distributed/tuple_destination.h" #include "distributed/tuplestore.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* Config variables that enable printing distributed query plans */ bool ExplainDistributedQueries = true; bool ExplainAllTasks = false; int ExplainAnalyzeSortMethod = EXPLAIN_ANALYZE_SORT_BY_TIME; extern MemoryContext SubPlanExplainAnalyzeContext; /* * If enabled, EXPLAIN ANALYZE output & other statistics of last worker task * are saved in following variables. */ static char *SavedExplainPlan = NULL; static double SavedExecutionDurationMillisec = 0.0; static double SavedExplainPlanNtuples = 0; static double SavedExplainPlanNloops = 0; extern SubPlanExplainOutputData *SubPlanExplainOutput; uint8 TotalExplainOutputCapacity = 0; uint8 NumTasksOutput = 0; /* struct to save explain flags */ typedef struct { bool verbose; bool costs; bool buffers; bool wal; bool timing; bool summary; #if PG_VERSION_NUM >= PG_VERSION_17 bool memory; ExplainSerializeOption serialize; #endif ExplainFormat format; } ExplainOptions; /* EXPLAIN flags of current distributed explain */ #if PG_VERSION_NUM >= PG_VERSION_17 static ExplainOptions CurrentDistributedQueryExplainOptions = { 0, 0, 0, 0, 0, 0, 0, EXPLAIN_SERIALIZE_NONE, EXPLAIN_FORMAT_TEXT }; #else static ExplainOptions CurrentDistributedQueryExplainOptions = { 0, 0, 0, 0, 0, 0, EXPLAIN_FORMAT_TEXT }; #endif /* Result for a single remote EXPLAIN command */ typedef struct RemoteExplainPlan { int placementIndex; List *explainOutputList; } RemoteExplainPlan; /* * ExplainAnalyzeDestination is internal representation of a TupleDestination * which collects EXPLAIN ANALYZE output after the main query is run. */ typedef struct ExplainAnalyzeDestination { TupleDestination pub; Task *originalTask; TupleDestination *originalTaskDestination; TupleDesc lastSavedExplainAnalyzeTupDesc; } ExplainAnalyzeDestination; #if PG_VERSION_NUM >= PG_VERSION_17 && PG_VERSION_NUM < PG_VERSION_18 /* copied from explain.c */ /* Instrumentation data for SERIALIZE option */ typedef struct SerializeMetrics { uint64 bytesSent; /* # of bytes serialized */ instr_time timeSpent; /* time spent serializing */ BufferUsage bufferUsage; /* buffers accessed during serialization */ } SerializeMetrics; /* copied from explain.c */ static void ExplainIndentText(ExplainState *es); static SerializeMetrics GetSerializationMetrics(DestReceiver *dest); /* * DestReceiver functions for SERIALIZE option * * A DestReceiver for query tuples, that serializes passed rows into RowData * messages while measuring the resources expended and total serialized size, * while never sending the data to the client. This allows measuring the * overhead of deTOASTing and datatype out/sendfuncs, which are not otherwise * exercisable without actually hitting the network. * * copied from explain.c */ typedef struct SerializeDestReceiver { DestReceiver pub; ExplainState *es; /* this EXPLAIN statement's ExplainState */ int8 format; /* text or binary, like pq wire protocol */ TupleDesc attrinfo; /* the output tuple desc */ int nattrs; /* current number of columns */ FmgrInfo *finfos; /* precomputed call info for output fns */ MemoryContext tmpcontext; /* per-row temporary memory context */ StringInfoData buf; /* buffer to hold the constructed message */ SerializeMetrics metrics; /* collected metrics */ } SerializeDestReceiver; #endif #if PG_VERSION_NUM >= PG_VERSION_17 /* * Various places within need to convert bytes to kilobytes. Round these up * to the next whole kilobyte. * copied from explain.c */ #define BYTES_TO_KILOBYTES(b) (((b) + 1023) / 1024) /* copied from explain.c */ static bool peek_buffer_usage(ExplainState *es, const BufferUsage *usage); static void show_buffer_usage(ExplainState *es, const BufferUsage *usage); static void show_memory_counters(ExplainState *es, const MemoryContextCounters *mem_counters); static void ExplainPrintSerialize(ExplainState *es, SerializeMetrics *metrics); #endif /* Explain functions for distributed queries */ static void ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es); static void ExplainJob(CitusScanState *scanState, Job *job, ExplainState *es, ParamListInfo params); static void ExplainMapMergeJob(MapMergeJob *mapMergeJob, ExplainState *es); static void ExplainTaskList(CitusScanState *scanState, List *taskList, ExplainState *es, ParamListInfo params); static RemoteExplainPlan * RemoteExplain(Task *task, ExplainState *es, ParamListInfo params); static RemoteExplainPlan * GetSavedRemoteExplain(Task *task, ExplainState *es); static RemoteExplainPlan * FetchRemoteExplainFromWorkers(Task *task, ExplainState *es, ParamListInfo params); static void ExplainTask(CitusScanState *scanState, Task *task, int placementIndex, List *explainOutputList, ExplainState *es); static void ExplainTaskPlacement(ShardPlacement *taskPlacement, List *explainOutputList, ExplainState *es); static StringInfo BuildRemoteExplainQuery(char *queryString, ExplainState *es); static const char * ExplainFormatStr(ExplainFormat format); #if PG_VERSION_NUM >= PG_VERSION_17 static const char * ExplainSerializeStr(ExplainSerializeOption serializeOption); #endif static void ExplainWorkerPlan(PlannedStmt *plannedStmt, DistributedSubPlan *subPlan, DestReceiver *dest, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, #if PG_VERSION_NUM >= PG_VERSION_17 const BufferUsage *bufusage, const MemoryContextCounters *mem_counters, #endif double *executionDurationMillisec, double *executionTuples, double *executionLoops); static ExplainFormat ExtractFieldExplainFormat(Datum jsonbDoc, const char *fieldName, ExplainFormat defaultValue); #if PG_VERSION_NUM >= PG_VERSION_17 static ExplainSerializeOption ExtractFieldExplainSerialize(Datum jsonbDoc, const char *fieldName, ExplainSerializeOption defaultValue); #endif static TupleDestination * CreateExplainAnlyzeDestination(Task *task, TupleDestination *taskDest); static void ExplainAnalyzeDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize); static TupleDesc ExplainAnalyzeDestTupleDescForQuery(TupleDestination *self, int queryNumber); static char * WrapQueryForExplainAnalyze(const char *queryString, TupleDesc tupleDesc, ParamListInfo params); static char * FetchPlanQueryForExplainAnalyze(const char *queryString, ParamListInfo params); static char * ParameterResolutionSubquery(ParamListInfo params); static List * SplitString(const char *str, char delimiter, int maxLength); /* Static Explain functions copied from explain.c */ static void ExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv); static double elapsed_time(instr_time *starttime); static void ExplainPropertyBytes(const char *qlabel, int64 bytes, ExplainState *es); static uint64 TaskReceivedTupleData(Task *task); static bool ShowReceivedTupleData(CitusScanState *scanState, ExplainState *es); static bool PlanStateAnalyzeWalker(PlanState *planState, void *ctx); static void ExtractAnalyzeStats(DistributedSubPlan *subPlan, PlanState *planState); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(worker_last_saved_explain_analyze); PG_FUNCTION_INFO_V1(worker_save_query_explain_analyze); /* * CitusExplainScan is a custom scan explain callback function which is used to * print explain information of a Citus plan which includes both combine query and * distributed plan. */ void CitusExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es) { if (es->generic) { ereport(ERROR, (errmsg( "EXPLAIN GENERIC_PLAN is currently not supported for Citus tables"))); } CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *distributedPlan = scanState->distributedPlan; EState *executorState = ScanStateGetExecutorState(scanState); ParamListInfo params = executorState->es_param_list_info; if (!ExplainDistributedQueries) { ExplainPropertyBool("citus.explain_distributed_queries", false, es); return; } ExplainOpenGroup("Distributed Query", "Distributed Query", true, es); /* * ExplainOnePlan function of postgres might be called in this codepath. * It requires an ActiveSnapshot being set. Make sure to make ActiveSnapshot available before calling into * Citus Explain functions. */ PushActiveSnapshot(executorState->es_snapshot); if (distributedPlan->subPlanList != NIL) { ExplainSubPlans(distributedPlan, es); } ExplainJob(scanState, distributedPlan->workerJob, es, params); PopActiveSnapshot(); ExplainCloseGroup("Distributed Query", "Distributed Query", true, es); } /* * NonPushableInsertSelectExplainScan is a custom scan explain callback function * which is used to print explain information of a Citus plan for an INSERT INTO * distributed_table SELECT ... query that is evaluated on the coordinator or * uses repartitioning. */ void NonPushableInsertSelectExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es) { CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *distributedPlan = scanState->distributedPlan; Query *insertSelectQuery = distributedPlan->modifyQueryViaCoordinatorOrRepartition; RangeTblEntry *selectRte = ExtractSelectRangeTableEntry(insertSelectQuery); /* * Create a copy because ExplainOneQuery can modify the query, and later * executions of prepared statements might require it. See * https://github.com/citusdata/citus/issues/3947 for what can happen. */ Query *queryCopy = copyObject(selectRte->subquery); bool repartition = distributedPlan->modifyWithSelectMethod == MODIFY_WITH_SELECT_REPARTITION; if (es->analyze) { ereport(ERROR, (errmsg("EXPLAIN ANALYZE is currently not supported for INSERT " "... SELECT commands %s", repartition ? "with repartitioning" : "via coordinator"))); } if (repartition) { ExplainPropertyText("INSERT/SELECT method", "repartition", es); } else { ExplainPropertyText("INSERT/SELECT method", "pull to coordinator", es); } ExplainOpenGroup("Select Query", "Select Query", false, es); /* explain the inner SELECT query */ IntoClause *into = NULL; ParamListInfo params = NULL; /* * With PG14, we need to provide a string here, * for now we put an empty string, which is valid according to postgres. */ char *queryString = pstrdup(""); ExplainOneQuery(queryCopy, 0, into, es, queryString, params, NULL); ExplainCloseGroup("Select Query", "Select Query", false, es); } /* * NonPushableMergeSqlExplainScan is a custom scan explain callback function * which is used to print explain information of a Citus plan for MERGE INTO * distributed_table USING (source query/table), where source can be any query * whose results are repartitioned to colocated with the target table. */ void NonPushableMergeCommandExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es) { CitusScanState *scanState = (CitusScanState *) node; DistributedPlan *distributedPlan = scanState->distributedPlan; Query *mergeQuery = distributedPlan->modifyQueryViaCoordinatorOrRepartition; RangeTblEntry *sourceRte = ExtractMergeSourceRangeTableEntry(mergeQuery, false); /* * Create a copy because ExplainOneQuery can modify the query, and later * executions of prepared statements might require it. See * https://github.com/citusdata/citus/issues/3947 for what can happen. */ Query *sourceQueryCopy = copyObject(sourceRte->subquery); bool repartition = distributedPlan->modifyWithSelectMethod == MODIFY_WITH_SELECT_REPARTITION; if (es->analyze) { ereport(ERROR, (errmsg("EXPLAIN ANALYZE is currently not supported for " "MERGE INTO ... commands with repartitioning"))); } Oid targetRelationId = ModifyQueryResultRelationId(mergeQuery); StringInfo mergeMethodMessage = makeStringInfo(); appendStringInfo(mergeMethodMessage, "MERGE INTO %s method", get_rel_name(targetRelationId)); if (repartition) { ExplainPropertyText(mergeMethodMessage->data, "repartition", es); } else { ExplainPropertyText(mergeMethodMessage->data, "pull to coordinator", es); } ExplainOpenGroup("Source Query", "Source Query", false, es); /* explain the MERGE source query */ IntoClause *into = NULL; ParamListInfo params = NULL; /* * With PG14, we need to provide a string here, for now we put an empty * string, which is valid according to postgres. */ char *queryString = pstrdup(""); ExplainOneQuery(sourceQueryCopy, 0, into, es, queryString, params, NULL); ExplainCloseGroup("Source Query", "Source Query", false, es); } /* * ExtractAnalyzeStats parses the EXPLAIN ANALYZE output of the pre-executed * subplans and injects the parsed statistics into queryDesc->planstate->instrument. */ static void ExtractAnalyzeStats(DistributedSubPlan *subPlan, PlanState *planState) { if (!planState) { return; } Instrumentation *instr = planState->instrument; if (!IsA(planState, CustomScanState)) { instr->ntuples = subPlan->ntuples; instr->nloops = 1; /* subplan nodes are executed only once */ return; } Assert(IsA(planState, CustomScanState)); if (subPlan->numTasksOutput <= 0) { return; } ListCell *lc; int tasksOutput = 0; double tasksNtuples = 0; double tasksNloops = 0; memset(instr, 0, sizeof(Instrumentation)); DistributedPlan *newdistributedPlan = ((CitusScanState *) planState)->distributedPlan; /* * Inject the earlier executed results—extracted from the workers' EXPLAIN output— * into the newly created tasks. */ foreach(lc, newdistributedPlan->workerJob->taskList) { Task *task = (Task *) lfirst(lc); uint32 taskId = task->taskId; if (tasksOutput > subPlan->numTasksOutput) { break; } if (!subPlan->totalExplainOutput[taskId].explainOutput) { continue; } /* * Now feed the earlier saved output, which will be used * by RemoteExplain() when printing tasks */ MemoryContext taskContext = GetMemoryChunkContext(task); task->totalReceivedTupleData = subPlan->totalExplainOutput[taskId].totalReceivedTupleData; task->fetchedExplainAnalyzeExecutionDuration = subPlan->totalExplainOutput[taskId].executionDuration; task->fetchedExplainAnalyzePlan = MemoryContextStrdup(taskContext, subPlan->totalExplainOutput[taskId].explainOutput); tasksNtuples += subPlan->totalExplainOutput[taskId].executionNtuples; tasksNloops = subPlan->totalExplainOutput[taskId].executionNloops; subPlan->totalExplainOutput[taskId].explainOutput = NULL; tasksOutput++; } instr->ntuples = tasksNtuples; instr->nloops = tasksNloops; } /* * ExplainSubPlans generates EXPLAIN output for subplans for CTEs * and complex subqueries. Because the planning for these queries * is done along with the top-level plan, we cannot determine the * planning time and set it to 0. */ static void ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) { ListCell *subPlanCell = NULL; uint64 planId = distributedPlan->planId; ExplainOpenGroup("Subplans", "Subplans", false, es); foreach(subPlanCell, distributedPlan->subPlanList) { DistributedSubPlan *subPlan = (DistributedSubPlan *) lfirst(subPlanCell); PlannedStmt *plan = subPlan->plan; ParamListInfo params = NULL; /* * With PG14, we need to provide a string here, * for now we put an empty string, which is valid according to postgres. */ char *queryString = pstrdup(""); instr_time planduration; BufferUsage bufusage_start, bufusage; #if PG_VERSION_NUM >= PG_VERSION_17 MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; if (es->memory) { /* copy paste from postgres code */ planner_ctx = AllocSetContextCreate(CurrentMemoryContext, "explain analyze planner context", ALLOCSET_DEFAULT_SIZES); saved_ctx = MemoryContextSwitchTo(planner_ctx); } #endif if (es->buffers) { bufusage_start = pgBufferUsage; } if (es->format == EXPLAIN_FORMAT_TEXT) { char *resultId = GenerateResultId(planId, subPlan->subPlanId); appendStringInfoSpaces(es->str, es->indent * 2); appendStringInfo(es->str, "-> Distributed Subplan %s\n", resultId); es->indent += 3; } ExplainOpenGroup("Subplan", NULL, true, es); if (es->analyze) { if (es->timing) { ExplainPropertyFloat("Subplan Duration", "ms", subPlan->durationMillisecs, 2, es); } ExplainPropertyBytes("Intermediate Data Size", subPlan->bytesSentPerWorker, es); StringInfo destination = makeStringInfo(); if (subPlan->remoteWorkerCount && subPlan->writeLocalFile) { appendStringInfo(destination, "Send to %d nodes, write locally", subPlan->remoteWorkerCount); } else if (subPlan->writeLocalFile) { appendStringInfoString(destination, "Write locally"); } else { appendStringInfo(destination, "Send to %d nodes", subPlan->remoteWorkerCount); } ExplainPropertyText("Result destination", destination->data, es); } INSTR_TIME_SET_ZERO(planduration); /* calc differences of buffer counters. */ if (es->buffers) { memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } ExplainOpenGroup("PlannedStmt", "PlannedStmt", false, es); DestReceiver *dest = None_Receiver; /* No query execution */ double executionDurationMillisec = 0.0; double executionTuples = 0; double executionLoops = 0; /* Capture memory stats on PG17+ */ #if PG_VERSION_NUM >= PG_VERSION_17 if (es->memory) { MemoryContextSwitchTo(saved_ctx); MemoryContextMemConsumed(planner_ctx, &mem_counters); } /* Execute EXPLAIN without ANALYZE */ ExplainWorkerPlan(plan, subPlan, dest, es, queryString, params, NULL, &planduration, (es->buffers ? &bufusage : NULL), (es->memory ? &mem_counters : NULL), &executionDurationMillisec, &executionTuples, &executionLoops); #else /* Execute EXPLAIN without ANALYZE */ ExplainWorkerPlan(plan, subPlan, dest, es, queryString, params, NULL, &planduration, &executionDurationMillisec, &executionTuples, &executionLoops); #endif ExplainCloseGroup("PlannedStmt", "PlannedStmt", false, es); ExplainCloseGroup("Subplan", NULL, true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { es->indent -= 3; } } ExplainCloseGroup("Subplans", "Subplans", false, es); } /* * ExplainPropertyBytes formats bytes in a human readable way by using * pg_size_pretty. */ static void ExplainPropertyBytes(const char *qlabel, int64 bytes, ExplainState *es) { Datum textDatum = DirectFunctionCall1(pg_size_pretty, Int64GetDatum(bytes)); ExplainPropertyText(qlabel, TextDatumGetCString(textDatum), es); } /* * ShowReceivedTupleData returns true if explain should show received data. * This is only the case when using EXPLAIN ANALYZE on queries that return * rows. */ static bool ShowReceivedTupleData(CitusScanState *scanState, ExplainState *es) { TupleDesc tupDesc = ScanStateGetTupleDescriptor(scanState); return es->analyze && tupDesc != NULL && tupDesc->natts > 0; } /* * ExplainJob shows the EXPLAIN output for a Job in the physical plan of * a distributed query by showing the remote EXPLAIN for the first task, * or all tasks if citus.explain_all_tasks is on. */ static void ExplainJob(CitusScanState *scanState, Job *job, ExplainState *es, ParamListInfo params) { List *dependentJobList = job->dependentJobList; int dependentJobCount = list_length(dependentJobList); ListCell *dependentJobCell = NULL; List *taskList = job->taskList; int taskCount = list_length(taskList); ExplainOpenGroup("Job", "Job", true, es); ExplainPropertyInteger("Task Count", NULL, taskCount, es); if (ShowReceivedTupleData(scanState, es)) { Task *task = NULL; uint64 totalReceivedTupleDataForAllTasks = 0; foreach_declared_ptr(task, taskList) { totalReceivedTupleDataForAllTasks += TaskReceivedTupleData(task); } ExplainPropertyBytes("Tuple data received from nodes", totalReceivedTupleDataForAllTasks, es); } if (dependentJobCount > 0) { ExplainPropertyText("Tasks Shown", "None, not supported for re-partition " "queries", es); } else if (ExplainAllTasks || taskCount <= 1) { ExplainPropertyText("Tasks Shown", "All", es); } else { StringInfo tasksShownText = makeStringInfo(); appendStringInfo(tasksShownText, "One of %d", taskCount); ExplainPropertyText("Tasks Shown", tasksShownText->data, es); } /* * We cannot fetch EXPLAIN plans for jobs that have dependencies, since the * intermediate tables have not been created. */ if (dependentJobCount == 0) { ExplainOpenGroup("Tasks", "Tasks", false, es); ExplainTaskList(scanState, taskList, es, params); ExplainCloseGroup("Tasks", "Tasks", false, es); } else { ExplainOpenGroup("Dependent Jobs", "Dependent Jobs", false, es); /* show explain output for dependent jobs, if any */ foreach(dependentJobCell, dependentJobList) { Job *dependentJob = (Job *) lfirst(dependentJobCell); if (CitusIsA(dependentJob, MapMergeJob)) { ExplainMapMergeJob((MapMergeJob *) dependentJob, es); } } ExplainCloseGroup("Dependent Jobs", "Dependent Jobs", false, es); } ExplainCloseGroup("Job", "Job", true, es); } /* * TaskReceivedTupleData returns the amount of data that was received by the * coordinator for the task. If it's a RETURNING DML task the value stored in * totalReceivedTupleData is not correct yet because it only counts the bytes for * one placement. */ static uint64 TaskReceivedTupleData(Task *task) { if (task->taskType == MODIFY_TASK) { return task->totalReceivedTupleData * list_length(task->taskPlacementList); } return task->totalReceivedTupleData; } /* * ExplainMapMergeJob shows a very basic EXPLAIN plan for a MapMergeJob. It does * not yet show the EXPLAIN plan for the individual tasks, because this requires * specific logic for getting the query (which is wrapped in a UDF), and the * queries may use intermediate tables that have not been created. */ static void ExplainMapMergeJob(MapMergeJob *mapMergeJob, ExplainState *es) { List *dependentJobList = mapMergeJob->job.dependentJobList; int dependentJobCount = list_length(dependentJobList); ListCell *dependentJobCell = NULL; int mapTaskCount = list_length(mapMergeJob->mapTaskList); int mergeTaskCount = list_length(mapMergeJob->mergeTaskList); if (es->format == EXPLAIN_FORMAT_TEXT) { appendStringInfoSpaces(es->str, es->indent * 2); appendStringInfo(es->str, "-> MapMergeJob\n"); es->indent += 3; } ExplainOpenGroup("MapMergeJob", NULL, true, es); ExplainPropertyInteger("Map Task Count", NULL, mapTaskCount, es); ExplainPropertyInteger("Merge Task Count", NULL, mergeTaskCount, es); if (dependentJobCount > 0) { ExplainOpenGroup("Dependent Jobs", "Dependent Jobs", false, es); foreach(dependentJobCell, dependentJobList) { Job *dependentJob = (Job *) lfirst(dependentJobCell); if (CitusIsA(dependentJob, MapMergeJob)) { ExplainMapMergeJob((MapMergeJob *) dependentJob, es); } } ExplainCloseGroup("Dependent Jobs", "Dependent Jobs", false, es); } ExplainCloseGroup("MapMergeJob", NULL, true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { es->indent -= 3; } } /* * CompareTasksByFetchedExplainAnalyzeDuration is a helper function to compare two tasks by their execution duration. */ static int CompareTasksByFetchedExplainAnalyzeDuration(const void *leftElement, const void *rightElement) { const Task *leftTask = *((const Task **) leftElement); const Task *rightTask = *((const Task **) rightElement); double leftTaskExecutionDuration = leftTask->fetchedExplainAnalyzeExecutionDuration; double rightTaskExecutionDuration = rightTask->fetchedExplainAnalyzeExecutionDuration; double diff = leftTaskExecutionDuration - rightTaskExecutionDuration; if (diff > 0) { return -1; } else if (diff < 0) { return 1; } return 0; } /* * ExplainTaskList shows the remote EXPLAIN and execution time for the first task * in taskList, or all tasks if citus.explain_all_tasks is on. */ static void ExplainTaskList(CitusScanState *scanState, List *taskList, ExplainState *es, ParamListInfo params) { List *remoteExplainList = NIL; /* if tasks are executed, we sort them by time; unless we are on a test env */ if (es->analyze && ExplainAnalyzeSortMethod == EXPLAIN_ANALYZE_SORT_BY_TIME) { /* sort by execution duration only in case of ANALYZE */ taskList = SortList(taskList, CompareTasksByFetchedExplainAnalyzeDuration); } else { /* make sure that the output is consistent */ taskList = SortList(taskList, CompareTasksByTaskId); } Task *task = NULL; foreach_declared_ptr(task, taskList) { RemoteExplainPlan *remoteExplain = RemoteExplain(task, es, params); remoteExplainList = lappend(remoteExplainList, remoteExplain); if (!ExplainAllTasks) { break; } } RemoteExplainPlan *remoteExplain = NULL; forboth_ptr(task, taskList, remoteExplain, remoteExplainList) { ExplainTask(scanState, task, remoteExplain->placementIndex, remoteExplain->explainOutputList, es); } } /* * RemoteExplain fetches the remote EXPLAIN output for a single task. */ static RemoteExplainPlan * RemoteExplain(Task *task, ExplainState *es, ParamListInfo params) { /* * For EXPLAIN EXECUTE we still use the old method, so task->fetchedExplainAnalyzePlan * can be NULL for some cases of es->analyze == true. */ if (es->analyze && task->fetchedExplainAnalyzePlan) { return GetSavedRemoteExplain(task, es); } else { return FetchRemoteExplainFromWorkers(task, es, params); } } /* * GetSavedRemoteExplain creates a remote EXPLAIN output from information saved * in task. */ static RemoteExplainPlan * GetSavedRemoteExplain(Task *task, ExplainState *es) { RemoteExplainPlan *remotePlan = (RemoteExplainPlan *) palloc0( sizeof(RemoteExplainPlan)); /* * Similar to postgres' ExplainQuery(), we split by newline only for * text format. */ if (es->format == EXPLAIN_FORMAT_TEXT) { /* * We limit the size of EXPLAIN plans to RSIZE_MAX_MEM (256MB). */ remotePlan->explainOutputList = SplitString(task->fetchedExplainAnalyzePlan, '\n', RSIZE_MAX_MEM); } else { StringInfo explainAnalyzeString = makeStringInfo(); appendStringInfoString(explainAnalyzeString, task->fetchedExplainAnalyzePlan); remotePlan->explainOutputList = list_make1(explainAnalyzeString); } remotePlan->placementIndex = task->fetchedExplainAnalyzePlacementIndex; return remotePlan; } /* * FetchRemoteExplainFromWorkers fetches the remote EXPLAIN output for a single * task by querying it from worker nodes. It tries each shard placement until * one succeeds or all failed. */ static RemoteExplainPlan * FetchRemoteExplainFromWorkers(Task *task, ExplainState *es, ParamListInfo params) { List *taskPlacementList = task->taskPlacementList; int placementCount = list_length(taskPlacementList); RemoteExplainPlan *remotePlan = (RemoteExplainPlan *) palloc0( sizeof(RemoteExplainPlan)); StringInfo explainQuery = BuildRemoteExplainQuery(TaskQueryString(task), es); /* * Use a coordinated transaction to ensure that we open a transaction block * such that we can set a savepoint. */ UseCoordinatedTransaction(); for (int placementIndex = 0; placementIndex < placementCount; placementIndex++) { ShardPlacement *taskPlacement = list_nth(taskPlacementList, placementIndex); int connectionFlags = 0; remotePlan->placementIndex = placementIndex; MultiConnection *connection = GetPlacementConnection(connectionFlags, taskPlacement, NULL); /* * This code-path doesn't support optional connections, so we don't expect * NULL connections. */ Assert(connection != NULL); /* try other placements if we fail to connect this one */ if (PQstatus(connection->pgConn) != CONNECTION_OK) { continue; } RemoteTransactionBeginIfNecessary(connection); /* * Start a savepoint for the explain query. After running the explain * query, we will rollback to this savepoint. This saves us from side * effects of EXPLAIN ANALYZE on DML queries. */ ExecuteCriticalRemoteCommand(connection, "SAVEPOINT citus_explain_savepoint"); /* run explain query */ int numParams = params ? params->numParams : 0; Oid *paramTypes = NULL; const char **paramValues = NULL; PGresult *queryResult = NULL; if (params) { ExtractParametersFromParamList(params, ¶mTypes, ¶mValues, false); } int sendStatus = SendRemoteCommandParams(connection, explainQuery->data, numParams, paramTypes, paramValues, false); if (sendStatus != 0) { queryResult = GetRemoteCommandResult(connection, false); if (!IsResponseOK(queryResult)) { PQclear(queryResult); ForgetResults(connection); continue; } } /* read explain query results */ remotePlan->explainOutputList = ReadFirstColumnAsText(queryResult); PQclear(queryResult); ForgetResults(connection); /* rollback to the savepoint */ ExecuteCriticalRemoteCommand(connection, "ROLLBACK TO SAVEPOINT citus_explain_savepoint"); if (remotePlan->explainOutputList != NIL) { break; } } return remotePlan; } /* * ExplainTask shows the EXPLAIN output for an single task. The output has been * fetched from the placement at index placementIndex. If explainOutputList is NIL, * then the EXPLAIN output could not be fetched from any placement. */ static void ExplainTask(CitusScanState *scanState, Task *task, int placementIndex, List *explainOutputList, ExplainState *es) { ExplainOpenGroup("Task", NULL, true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { appendStringInfoSpaces(es->str, es->indent * 2); appendStringInfo(es->str, "-> Task\n"); es->indent += 3; } if (es->verbose) { const char *queryText = TaskQueryString(task); ExplainPropertyText("Query", queryText, es); } if (ShowReceivedTupleData(scanState, es)) { ExplainPropertyBytes("Tuple data received from node", TaskReceivedTupleData(task), es); } if (explainOutputList != NIL) { List *taskPlacementList = task->taskPlacementList; ShardPlacement *taskPlacement = list_nth(taskPlacementList, placementIndex); ExplainTaskPlacement(taskPlacement, explainOutputList, es); } else { ExplainPropertyText("Error", "Could not get remote plan.", es); } ExplainCloseGroup("Task", NULL, true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { es->indent -= 3; } } /* * ExplainTaskPlacement shows the EXPLAIN output for an individual task placement. * It corrects the indentation of the remote explain output to match the local * output. */ static void ExplainTaskPlacement(ShardPlacement *taskPlacement, List *explainOutputList, ExplainState *es) { int savedIndentation = es->indent; StringInfo nodeAddress = makeStringInfo(); char *nodeName = taskPlacement->nodeName; uint32 nodePort = taskPlacement->nodePort; const char *nodeDatabase = CurrentDatabaseName(); ListCell *explainOutputCell = NULL; int rowIndex = 0; appendStringInfo(nodeAddress, "host=%s port=%d dbname=%s", nodeName, nodePort, nodeDatabase); ExplainPropertyText("Node", nodeAddress->data, es); ExplainOpenGroup("Remote Plan", "Remote Plan", false, es); if (es->format == EXPLAIN_FORMAT_JSON || es->format == EXPLAIN_FORMAT_YAML) { /* prevent appending the remote EXPLAIN on the same line */ appendStringInfoChar(es->str, '\n'); } foreach(explainOutputCell, explainOutputList) { StringInfo rowString = (StringInfo) lfirst(explainOutputCell); int rowLength = strlen(rowString->data); char *lineStart = rowString->data; /* parse the lines in the remote EXPLAIN for proper indentation */ while (lineStart < rowString->data + rowLength) { /* find the end-of-line */ char *lineEnd = strchr(lineStart, '\n'); if (lineEnd == NULL) { /* no end-of-line, use end of row string instead */ lineEnd = rowString->data + rowLength; } /* convert line to a separate string */ *lineEnd = '\0'; /* indentation that is applied to all lines */ appendStringInfoSpaces(es->str, es->indent * 2); if (es->format == EXPLAIN_FORMAT_TEXT && rowIndex == 0) { /* indent the first line of the remote plan with an arrow */ appendStringInfoString(es->str, "-> "); es->indent += 2; } /* show line in the output */ appendStringInfo(es->str, "%s\n", lineStart); /* continue at the start of the next line */ lineStart = lineEnd + 1; } rowIndex++; } ExplainCloseGroup("Remote Plan", "Remote Plan", false, es); if (es->format == EXPLAIN_FORMAT_TEXT) { es->indent = savedIndentation; } } /* * BuildRemoteExplainQuery returns an EXPLAIN query string * to run on a worker node which explicitly contains all * the options in the explain state. */ static StringInfo BuildRemoteExplainQuery(char *queryString, ExplainState *es) { StringInfo explainQuery = makeStringInfo(); const char *formatStr = ExplainFormatStr(es->format); #if PG_VERSION_NUM >= PG_VERSION_17 const char *serializeStr = ExplainSerializeStr(es->serialize); #endif appendStringInfo(explainQuery, "EXPLAIN (ANALYZE %s, VERBOSE %s, " "COSTS %s, BUFFERS %s, WAL %s, " "TIMING %s, SUMMARY %s, " #if PG_VERSION_NUM >= PG_VERSION_17 "MEMORY %s, SERIALIZE %s, " #endif "FORMAT %s) %s", es->analyze ? "TRUE" : "FALSE", es->verbose ? "TRUE" : "FALSE", es->costs ? "TRUE" : "FALSE", es->buffers ? "TRUE" : "FALSE", es->wal ? "TRUE" : "FALSE", es->timing ? "TRUE" : "FALSE", es->summary ? "TRUE" : "FALSE", #if PG_VERSION_NUM >= PG_VERSION_17 es->memory ? "TRUE" : "FALSE", serializeStr, #endif formatStr, queryString); return explainQuery; } /* * ExplainFormatStr converts the given explain format to string. */ static const char * ExplainFormatStr(ExplainFormat format) { switch (format) { case EXPLAIN_FORMAT_XML: { return "XML"; } case EXPLAIN_FORMAT_JSON: { return "JSON"; } case EXPLAIN_FORMAT_YAML: { return "YAML"; } default: { return "TEXT"; } } } #if PG_VERSION_NUM >= PG_VERSION_17 /* * ExplainSerializeStr converts the given explain serialize option to string. */ static const char * ExplainSerializeStr(ExplainSerializeOption serializeOption) { switch (serializeOption) { case EXPLAIN_SERIALIZE_NONE: { return "none"; } case EXPLAIN_SERIALIZE_TEXT: { return "text"; } case EXPLAIN_SERIALIZE_BINARY: { return "binary"; } default: { return "none"; } } } #endif /* * worker_last_saved_explain_analyze returns the last saved EXPLAIN ANALYZE output of * a worker task query. It returns NULL if nothing has been saved yet. */ Datum worker_last_saved_explain_analyze(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); if (SavedExplainPlan != NULL) { int columnCount = tupleDescriptor->natts; if (columnCount != 4) { ereport(ERROR, (errmsg("expected 4 output columns in definition of " "worker_last_saved_explain_analyze, but got %d", columnCount))); } bool columnNulls[4] = { false }; Datum columnValues[4] = { CStringGetTextDatum(SavedExplainPlan), Float8GetDatum(SavedExecutionDurationMillisec), Float8GetDatum(SavedExplainPlanNtuples), Float8GetDatum(SavedExplainPlanNloops) }; tuplestore_putvalues(tupleStore, tupleDescriptor, columnValues, columnNulls); } PG_RETURN_DATUM(0); } /* * worker_save_query_explain_analyze executes and returns results of query while * saving its EXPLAIN ANALYZE to be fetched later. */ Datum worker_save_query_explain_analyze(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *queryText = PG_GETARG_TEXT_P(0); char *queryString = text_to_cstring(queryText); double executionDurationMillisec = 0.0; double executionTuples = 0; double executionLoops = 0; Datum explainOptions = PG_GETARG_DATUM(1); ExplainState *es = NewExplainState(); es->analyze = true; /* use the same defaults as NewExplainState() for following options */ es->buffers = ExtractFieldBoolean(explainOptions, "buffers", es->buffers); es->wal = ExtractFieldBoolean(explainOptions, "wal", es->wal); es->costs = ExtractFieldBoolean(explainOptions, "costs", es->costs); es->summary = ExtractFieldBoolean(explainOptions, "summary", es->summary); es->verbose = ExtractFieldBoolean(explainOptions, "verbose", es->verbose); es->timing = ExtractFieldBoolean(explainOptions, "timing", es->timing); es->format = ExtractFieldExplainFormat(explainOptions, "format", es->format); #if PG_VERSION_NUM >= PG_VERSION_17 es->memory = ExtractFieldBoolean(explainOptions, "memory", es->memory); es->serialize = ExtractFieldExplainSerialize(explainOptions, "serialize", es->serialize); #endif TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); DestReceiver *tupleStoreDest = CreateTuplestoreDestReceiver(); SetTuplestoreDestReceiverParams(tupleStoreDest, tupleStore, CurrentMemoryContext, false, NULL, NULL); List *parseTreeList = pg_parse_query(queryString); if (list_length(parseTreeList) != 1) { ereport(ERROR, (errmsg("cannot EXPLAIN ANALYZE multiple queries"))); } RawStmt *parseTree = linitial(parseTreeList); ParamListInfo boundParams = ExecutorBoundParams(); int numParams = boundParams ? boundParams->numParams : 0; Oid *paramTypes = NULL; const char **paramValues = NULL; if (boundParams != NULL) { ExtractParametersFromParamList(boundParams, ¶mTypes, ¶mValues, false); } /* resolve OIDs of unknown (user-defined) types */ Query *analyzedQuery = parse_analyze_varparams(parseTree, queryString, ¶mTypes, &numParams, NULL); /* pg_rewrite_query is a wrapper around QueryRewrite with some debugging logic */ List *queryList = pg_rewrite_query(analyzedQuery); if (list_length(queryList) != 1) { ereport(ERROR, (errmsg("cannot EXPLAIN ANALYZE a query rewritten " "into multiple queries"))); } Query *query = linitial(queryList); ExplainBeginOutput(es); /* plan query and record planning stats */ instr_time planStart; instr_time planDuration; #if PG_VERSION_NUM >= PG_VERSION_17 BufferUsage bufusage_start, bufusage; MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; if (es->memory) { /* * Create a new memory context to measure planner's memory consumption * accurately. Note that if the planner were to be modified to use a * different memory context type, here we would be changing that to * AllocSet, which might be undesirable. However, we don't have a way * to create a context of the same type as another, so we pray and * hope that this is OK. * * copied from explain.c */ planner_ctx = AllocSetContextCreate(CurrentMemoryContext, "explain analyze planner context", ALLOCSET_DEFAULT_SIZES); saved_ctx = MemoryContextSwitchTo(planner_ctx); } if (es->buffers) { bufusage_start = pgBufferUsage; } #endif INSTR_TIME_SET_CURRENT(planStart); PlannedStmt *plan = pg_plan_query(query, NULL, CURSOR_OPT_PARALLEL_OK, NULL); INSTR_TIME_SET_CURRENT(planDuration); INSTR_TIME_SUBTRACT(planDuration, planStart); #if PG_VERSION_NUM >= PG_VERSION_17 if (es->memory) { MemoryContextSwitchTo(saved_ctx); MemoryContextMemConsumed(planner_ctx, &mem_counters); } /* calc differences of buffer counters. */ if (es->buffers) { memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } /* do the actual EXPLAIN ANALYZE */ ExplainWorkerPlan(plan, NULL, tupleStoreDest, es, queryString, boundParams, NULL, &planDuration, (es->buffers ? &bufusage : NULL), (es->memory ? &mem_counters : NULL), &executionDurationMillisec, &executionTuples, &executionLoops); #else /* do the actual EXPLAIN ANALYZE */ ExplainWorkerPlan(plan, NULL, tupleStoreDest, es, queryString, boundParams, NULL, &planDuration, &executionDurationMillisec, &executionTuples, &executionLoops); #endif ExplainEndOutput(es); /* save EXPLAIN ANALYZE result to be fetched later */ MemoryContext oldContext = MemoryContextSwitchTo(TopTransactionContext); FreeSavedExplainPlan(); SavedExplainPlan = pstrdup(es->str->data); SavedExecutionDurationMillisec = executionDurationMillisec; SavedExplainPlanNtuples = executionTuples; SavedExplainPlanNloops = executionLoops; MemoryContextSwitchTo(oldContext); PG_RETURN_DATUM(0); } /* * FreeSavedExplainPlan frees allocated saved explain plan if any. */ void FreeSavedExplainPlan(void) { if (SavedExplainPlan) { pfree(SavedExplainPlan); SavedExplainPlan = NULL; } } /* * ExtractFieldExplainFormat gets value of fieldName from jsonbDoc, or returns * defaultValue if it doesn't exist. */ static ExplainFormat ExtractFieldExplainFormat(Datum jsonbDoc, const char *fieldName, ExplainFormat defaultValue) { Datum jsonbDatum = 0; bool found = ExtractFieldJsonbDatum(jsonbDoc, fieldName, &jsonbDatum); if (!found) { return defaultValue; } const char *formatStr = DatumGetCString(DirectFunctionCall1(jsonb_out, jsonbDatum)); if (pg_strcasecmp(formatStr, "\"text\"") == 0) { return EXPLAIN_FORMAT_TEXT; } else if (pg_strcasecmp(formatStr, "\"xml\"") == 0) { return EXPLAIN_FORMAT_XML; } else if (pg_strcasecmp(formatStr, "\"yaml\"") == 0) { return EXPLAIN_FORMAT_YAML; } else if (pg_strcasecmp(formatStr, "\"json\"") == 0) { return EXPLAIN_FORMAT_JSON; } ereport(ERROR, (errmsg("Invalid explain analyze format: %s", formatStr))); return 0; } #if PG_VERSION_NUM >= PG_VERSION_17 /* * ExtractFieldExplainSerialize gets value of fieldName from jsonbDoc, or returns * defaultValue if it doesn't exist. */ static ExplainSerializeOption ExtractFieldExplainSerialize(Datum jsonbDoc, const char *fieldName, ExplainSerializeOption defaultValue) { Datum jsonbDatum = 0; bool found = ExtractFieldJsonbDatum(jsonbDoc, fieldName, &jsonbDatum); if (!found) { return defaultValue; } const char *serializeStr = DatumGetCString(DirectFunctionCall1(jsonb_out, jsonbDatum)); if (pg_strcasecmp(serializeStr, "\"none\"") == 0) { return EXPLAIN_SERIALIZE_NONE; } else if (pg_strcasecmp(serializeStr, "\"off\"") == 0) { return EXPLAIN_SERIALIZE_NONE; } else if (pg_strcasecmp(serializeStr, "\"text\"") == 0) { return EXPLAIN_SERIALIZE_TEXT; } else if (pg_strcasecmp(serializeStr, "\"binary\"") == 0) { return EXPLAIN_SERIALIZE_BINARY; } ereport(ERROR, (errmsg("Invalid explain analyze serialize: %s", serializeStr))); return 0; } #endif /* * CitusExplainOneQuery is the executor hook that is called when * postgres wants to explain a query. */ void CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv) { /* save the flags of current EXPLAIN command */ CurrentDistributedQueryExplainOptions.costs = es->costs; CurrentDistributedQueryExplainOptions.buffers = es->buffers; CurrentDistributedQueryExplainOptions.wal = es->wal; CurrentDistributedQueryExplainOptions.verbose = es->verbose; CurrentDistributedQueryExplainOptions.summary = es->summary; CurrentDistributedQueryExplainOptions.timing = es->timing; CurrentDistributedQueryExplainOptions.format = es->format; #if PG_VERSION_NUM >= PG_VERSION_17 CurrentDistributedQueryExplainOptions.memory = es->memory; CurrentDistributedQueryExplainOptions.serialize = es->serialize; #endif /* rest is copied from ExplainOneQuery() */ instr_time planstart, planduration; BufferUsage bufusage_start, bufusage; #if PG_VERSION_NUM >= PG_VERSION_17 MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; if (es->memory) { /* copy paste from postgres code */ planner_ctx = AllocSetContextCreate(CurrentMemoryContext, "explain analyze planner context", ALLOCSET_DEFAULT_SIZES); saved_ctx = MemoryContextSwitchTo(planner_ctx); } #endif if (es->buffers) { bufusage_start = pgBufferUsage; } INSTR_TIME_SET_CURRENT(planstart); /* * We should not hide any objects while explaining some query to not break * postgres vanilla tests. * * The filter 'is_citus_depended_object' is added to explain result * and causes some tests to fail if HideCitusDependentObjects is true. * Therefore, we disable HideCitusDependentObjects until the current transaction * ends. * * We do not use security quals because a postgres vanilla test fails * with a change of order for its result. */ SetLocalHideCitusDependentObjectsDisabledWhenAlreadyEnabled(); /* plan the query */ PlannedStmt *plan = pg_plan_query(query, NULL, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); /* calc differences of buffer counters. */ if (es->buffers) { memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } /* capture memory stats on PG17+ */ #if PG_VERSION_NUM >= PG_VERSION_17 if (es->memory) { MemoryContextSwitchTo(saved_ctx); MemoryContextMemConsumed(planner_ctx, &mem_counters); } #endif #if PG_VERSION_NUM >= PG_VERSION_17 /* PostgreSQL 17 signature (9 args: includes mem_counters) */ ExplainOnePlan( plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), (es->memory ? &mem_counters : NULL) ); #else ExplainOnePlan( plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL) ); #endif } /* * CreateExplainAnlyzeDestination creates a destination suitable for collecting * explain analyze output from workers. */ static TupleDestination * CreateExplainAnlyzeDestination(Task *task, TupleDestination *taskDest) { ExplainAnalyzeDestination *tupleDestination = palloc0( sizeof(ExplainAnalyzeDestination)); tupleDestination->originalTask = task; tupleDestination->originalTaskDestination = taskDest; TupleDesc lastSavedExplainAnalyzeTupDesc = CreateTemplateTupleDesc(4); TupleDescInitEntry(lastSavedExplainAnalyzeTupDesc, 1, "explain analyze", TEXTOID, 0, 0); TupleDescInitEntry(lastSavedExplainAnalyzeTupDesc, 2, "duration", FLOAT8OID, 0, 0); TupleDescInitEntry(lastSavedExplainAnalyzeTupDesc, 3, "ntuples", FLOAT8OID, 0, 0); TupleDescInitEntry(lastSavedExplainAnalyzeTupDesc, 4, "nloops", FLOAT8OID, 0, 0); tupleDestination->lastSavedExplainAnalyzeTupDesc = lastSavedExplainAnalyzeTupDesc; tupleDestination->pub.putTuple = ExplainAnalyzeDestPutTuple; tupleDestination->pub.tupleDescForQuery = ExplainAnalyzeDestTupleDescForQuery; return (TupleDestination *) tupleDestination; } /* * EnsureExplainOutputCapacity is to ensure capacity for new entries. Input * parameter requiredSize is minimum number of elements needed. */ static void EnsureExplainOutputCapacity(int requiredSize) { if (requiredSize < TotalExplainOutputCapacity) { return; } int newCapacity = (TotalExplainOutputCapacity == 0) ? 32 : TotalExplainOutputCapacity * 2; while (newCapacity <= requiredSize) { newCapacity *= 2; } if (SubPlanExplainOutput == NULL) { SubPlanExplainOutput = (SubPlanExplainOutputData *) MemoryContextAllocZero( SubPlanExplainAnalyzeContext, newCapacity * sizeof(SubPlanExplainOutputData)); } else { /* Use repalloc and manually zero the new memory */ int oldSize = TotalExplainOutputCapacity * sizeof(SubPlanExplainOutputData); int newSize = newCapacity * sizeof(SubPlanExplainOutputData); SubPlanExplainOutput = (SubPlanExplainOutputData *) repalloc(SubPlanExplainOutput, newSize); /* Zero out the newly allocated memory */ MemSet((char *) SubPlanExplainOutput + oldSize, 0, newSize - oldSize); } TotalExplainOutputCapacity = newCapacity; } /* * ExplainAnalyzeDestPutTuple implements TupleDestination->putTuple * for ExplainAnalyzeDestination. */ static void ExplainAnalyzeDestPutTuple(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple heapTuple, uint64 tupleLibpqSize) { uint32 taskId = task->taskId; ExplainAnalyzeDestination *tupleDestination = (ExplainAnalyzeDestination *) self; if (queryNumber == 0) { TupleDestination *originalTupDest = tupleDestination->originalTaskDestination; originalTupDest->putTuple(originalTupDest, task, placementIndex, 0, heapTuple, tupleLibpqSize); tupleDestination->originalTask->totalReceivedTupleData += tupleLibpqSize; if (SubPlanExplainAnalyzeContext) { EnsureExplainOutputCapacity(taskId + 1); SubPlanExplainOutput[taskId].totalReceivedTupleData = tupleDestination->originalTask->totalReceivedTupleData; } } else if (queryNumber == 1) { bool isNull = false; TupleDesc tupDesc = tupleDestination->lastSavedExplainAnalyzeTupDesc; Datum explainAnalyze = heap_getattr(heapTuple, 1, tupDesc, &isNull); if (isNull) { ereport(WARNING, (errmsg( "received null explain analyze output from worker"))); return; } Datum executionDuration = heap_getattr(heapTuple, 2, tupDesc, &isNull); Datum executionTuples = heap_getattr(heapTuple, 3, tupDesc, &isNull); Datum executionLoops = heap_getattr(heapTuple, 4, tupDesc, &isNull); if (isNull) { ereport(WARNING, (errmsg("received null execution time from worker"))); return; } char *fetchedExplainAnalyzePlan = TextDatumGetCString(explainAnalyze); double fetchedExplainAnalyzeExecutionDuration = DatumGetFloat8(executionDuration); double fetchedExplainAnalyzeTuples = DatumGetFloat8(executionTuples); double fetchedExplainAnalyzeLoops = DatumGetFloat8(executionLoops); /* * Allocate fetchedExplainAnalyzePlan in the same context as the Task, since we are * currently in execution context and a Task can span multiple executions. * * Although we won't reuse the same value in a future execution, but we have * calls to CheckNodeCopyAndSerialization() which asserts copy functions of the task * work as expected, which will try to copy this value in a future execution. * * Why don't we just allocate this field in executor context and reset it before * the next execution? Because when an error is raised we can skip pretty much most * of the meaningful places that we can insert the reset. * * TODO: Take all EXPLAIN ANALYZE related fields out of Task and store them in a * Task to ExplainAnalyzePrivate mapping in multi_explain.c, so we don't need to * do these hacky memory context management tricks. */ MemoryContext taskContext = GetMemoryChunkContext(tupleDestination->originalTask); tupleDestination->originalTask->fetchedExplainAnalyzePlan = MemoryContextStrdup(taskContext, fetchedExplainAnalyzePlan); tupleDestination->originalTask->fetchedExplainAnalyzePlacementIndex = placementIndex; tupleDestination->originalTask->fetchedExplainAnalyzeExecutionDuration = fetchedExplainAnalyzeExecutionDuration; /* We should build tupleDestination in subPlan similar to the above */ if (SubPlanExplainAnalyzeContext) { EnsureExplainOutputCapacity(taskId + 1); SubPlanExplainOutput[taskId].explainOutput = MemoryContextStrdup(SubPlanExplainAnalyzeContext, fetchedExplainAnalyzePlan); SubPlanExplainOutput[taskId].executionDuration = fetchedExplainAnalyzeExecutionDuration; SubPlanExplainOutput[taskId].executionNtuples = fetchedExplainAnalyzeTuples; SubPlanExplainOutput[taskId].executionNloops = fetchedExplainAnalyzeLoops; NumTasksOutput++; } } else { ereport(ERROR, (errmsg("cannot get EXPLAIN ANALYZE of multiple queries"), errdetail("while receiving tuples for query %d", queryNumber))); } } /* * ResetExplainAnalyzeData reset fields in Task that are used by multi_explain.c */ void ResetExplainAnalyzeData(List *taskList) { Task *task = NULL; foreach_declared_ptr(task, taskList) { if (task->fetchedExplainAnalyzePlan != NULL) { pfree(task->fetchedExplainAnalyzePlan); } task->totalReceivedTupleData = 0; task->fetchedExplainAnalyzePlacementIndex = 0; task->fetchedExplainAnalyzePlan = NULL; } } /* * ExplainAnalyzeDestTupleDescForQuery implements TupleDestination->tupleDescForQuery * for ExplainAnalyzeDestination. */ static TupleDesc ExplainAnalyzeDestTupleDescForQuery(TupleDestination *self, int queryNumber) { ExplainAnalyzeDestination *tupleDestination = (ExplainAnalyzeDestination *) self; if (queryNumber == 0) { TupleDestination *originalTupDest = tupleDestination->originalTaskDestination; return originalTupDest->tupleDescForQuery(originalTupDest, 0); } else if (queryNumber == 1) { return tupleDestination->lastSavedExplainAnalyzeTupDesc; } ereport(ERROR, (errmsg("cannot get EXPLAIN ANALYZE of multiple queries"), errdetail("while requesting for tuple descriptor of query %d", queryNumber))); return NULL; } /* * RequestedForExplainAnalyze returns true if we should get the EXPLAIN ANALYZE * output for the given custom scan node. */ bool RequestedForExplainAnalyze(CitusScanState *node) { /* * When running a distributed plan—either the root plan or a subplan’s * distributed fragment—we need to know if we’re under EXPLAIN ANALYZE. * Subplans can’t receive the EXPLAIN ANALYZE flag directly, so we use * SubPlanExplainAnalyzeContext as a flag to indicate that context. */ return (node->customScanState.ss.ps.state->es_instrument != 0) || (SubPlanLevel > 0 && SubPlanExplainAnalyzeContext); } /* * ExplainAnalyzeTaskList returns a task list suitable for explain analyze. After executing * these tasks, fetchedExplainAnalyzePlan of originalTaskList should be populated. */ List * ExplainAnalyzeTaskList(List *originalTaskList, TupleDestination *defaultTupleDest, TupleDesc tupleDesc, ParamListInfo params) { List *explainAnalyzeTaskList = NIL; Task *originalTask = NULL; foreach_declared_ptr(originalTask, originalTaskList) { if (originalTask->queryCount != 1) { ereport(ERROR, (errmsg("cannot get EXPLAIN ANALYZE of multiple queries"))); } Task *explainAnalyzeTask = copyObject(originalTask); const char *queryString = TaskQueryString(explainAnalyzeTask); ParamListInfo taskParams = params; /* * We will not send parameters if they have already been resolved in the query * string. */ if (explainAnalyzeTask->parametersInQueryStringResolved) { taskParams = NULL; } char *wrappedQuery = WrapQueryForExplainAnalyze(queryString, tupleDesc, taskParams); char *fetchQuery = FetchPlanQueryForExplainAnalyze(queryString, taskParams); SetTaskQueryStringList(explainAnalyzeTask, list_make2(wrappedQuery, fetchQuery)); TupleDestination *originalTaskDest = originalTask->tupleDest ? originalTask->tupleDest : defaultTupleDest; explainAnalyzeTask->tupleDest = CreateExplainAnlyzeDestination(originalTask, originalTaskDest); explainAnalyzeTaskList = lappend(explainAnalyzeTaskList, explainAnalyzeTask); } return explainAnalyzeTaskList; } /* * WrapQueryForExplainAnalyze wraps a query into a worker_save_query_explain_analyze() * call so we can fetch its explain analyze after its execution. */ static char * WrapQueryForExplainAnalyze(const char *queryString, TupleDesc tupleDesc, ParamListInfo params) { StringInfo columnDef = makeStringInfo(); for (int columnIndex = 0; columnIndex < tupleDesc->natts; columnIndex++) { if (columnIndex != 0) { appendStringInfoString(columnDef, ", "); } Form_pg_attribute attr = TupleDescAttr(tupleDesc, columnIndex); char *attrType = format_type_extended(attr->atttypid, attr->atttypmod, FORMAT_TYPE_TYPEMOD_GIVEN | FORMAT_TYPE_FORCE_QUALIFY); appendStringInfo(columnDef, "field_%d %s", columnIndex, attrType); } /* * column definition cannot be empty, so create a dummy column definition for * queries with no results. */ if (tupleDesc->natts == 0) { appendStringInfo(columnDef, "dummy_field int"); } StringInfo explainOptions = makeStringInfo(); appendStringInfo(explainOptions, "{\"verbose\": %s, \"costs\": %s, \"buffers\": %s, \"wal\": %s, " #if PG_VERSION_NUM >= PG_VERSION_17 "\"memory\": %s, \"serialize\": \"%s\", " #endif "\"timing\": %s, \"summary\": %s, \"format\": \"%s\"}", CurrentDistributedQueryExplainOptions.verbose ? "true" : "false", CurrentDistributedQueryExplainOptions.costs ? "true" : "false", CurrentDistributedQueryExplainOptions.buffers ? "true" : "false", CurrentDistributedQueryExplainOptions.wal ? "true" : "false", #if PG_VERSION_NUM >= PG_VERSION_17 CurrentDistributedQueryExplainOptions.memory ? "true" : "false", ExplainSerializeStr(CurrentDistributedQueryExplainOptions.serialize), #endif CurrentDistributedQueryExplainOptions.timing ? "true" : "false", CurrentDistributedQueryExplainOptions.summary ? "true" : "false", ExplainFormatStr(CurrentDistributedQueryExplainOptions.format)); StringInfo wrappedQuery = makeStringInfo(); /* * We do not include dummy column if original query didn't return any columns. * Otherwise, number of columns that original query returned wouldn't match * number of columns returned by worker_save_query_explain_analyze. */ char *workerSaveQueryFetchCols = (tupleDesc->natts == 0) ? "" : "*"; if (params != NULL) { /* * Add a dummy CTE to ensure all parameters are referenced, such that their * types can be resolved. */ appendStringInfo(wrappedQuery, "WITH unused AS (%s) ", ParameterResolutionSubquery(params)); } appendStringInfo(wrappedQuery, "SELECT %s FROM worker_save_query_explain_analyze(%s, %s) AS (%s)", workerSaveQueryFetchCols, quote_literal_cstr(queryString), quote_literal_cstr(explainOptions->data), columnDef->data); return wrappedQuery->data; } /* * FetchPlanQueryForExplainAnalyze generates a query to fetch the plan saved * by worker_save_query_explain_analyze from the worker. */ static char * FetchPlanQueryForExplainAnalyze(const char *queryString, ParamListInfo params) { StringInfo fetchQuery = makeStringInfo(); if (params != NULL) { /* * Add a dummy CTE to ensure all parameters are referenced, such that their * types can be resolved. */ appendStringInfo(fetchQuery, "WITH unused AS (%s) ", ParameterResolutionSubquery(params)); } appendStringInfoString(fetchQuery, "SELECT explain_analyze_output, execution_duration, " "execution_ntuples, execution_nloops " "FROM worker_last_saved_explain_analyze()"); return fetchQuery->data; } /* * ParameterResolutionSubquery generates a subquery that returns all parameters * in params with explicit casts to their type names. This can be used in cases * where we use custom type parameters that are not directly referenced. */ static char * ParameterResolutionSubquery(ParamListInfo params) { StringInfo paramsQuery = makeStringInfo(); appendStringInfo(paramsQuery, "SELECT"); for (int paramIndex = 0; paramIndex < params->numParams; paramIndex++) { ParamExternData *param = ¶ms->params[paramIndex]; char *typeName = format_type_extended(param->ptype, -1, FORMAT_TYPE_FORCE_QUALIFY); appendStringInfo(paramsQuery, "%s $%d::%s", paramIndex > 0 ? "," : "", paramIndex + 1, typeName); } return paramsQuery->data; } /* * SplitString splits the given string by the given delimiter. * * Why not use strtok_s()? Its signature and semantics are difficult to understand. * * Why not use strchr() (similar to do_text_output_multiline)? Although not banned, * it isn't safe if by any chance str is not null-terminated. */ static List * SplitString(const char *str, char delimiter, int maxLength) { size_t len = strnlen(str, maxLength); if (len == 0) { return NIL; } List *tokenList = NIL; StringInfo token = makeStringInfo(); for (size_t index = 0; index < len; index++) { if (str[index] == delimiter) { tokenList = lappend(tokenList, token); token = makeStringInfo(); } else { appendStringInfoChar(token, str[index]); } } /* append last token */ tokenList = lappend(tokenList, token); return tokenList; } /* below are private functions copied from explain.c */ /* *INDENT-OFF* */ /* * ExplainOneQuery - * print out the execution plan for one Query * * "into" is NULL unless we are explaining the contents of a CreateTableAsStmt. */ static void ExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv) { /* if an advisor plugin is present, let it manage things */ if (ExplainOneQuery_hook) { (*ExplainOneQuery_hook) (query, cursorOptions, into, es, queryString, params, queryEnv); } else { instr_time planstart, planduration; BufferUsage bufusage_start, bufusage; #if PG_VERSION_NUM >= PG_VERSION_17 MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; if (es->memory) { /* copy paste from postgres code */ planner_ctx = AllocSetContextCreate(CurrentMemoryContext, "explain analyze planner context", ALLOCSET_DEFAULT_SIZES); saved_ctx = MemoryContextSwitchTo(planner_ctx); } #endif if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ PlannedStmt *plan = pg_plan_query(query, NULL, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); /* calc differences of buffer counters. */ if (es->buffers) { memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); } /* 1) Capture memory counters on PG17+ only once: */ #if PG_VERSION_NUM >= PG_VERSION_17 if (es->memory) { MemoryContextSwitchTo(saved_ctx); MemoryContextMemConsumed(planner_ctx, &mem_counters); } #endif #if PG_VERSION_NUM >= PG_VERSION_17 ExplainOnePlan( plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), (es->memory ? &mem_counters: NULL) ); #else ExplainOnePlan( plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL) ); #endif } } /* * PlanStateAnalyzeWalker Tree walker callback that visits each PlanState node in the * plan tree and extracts analyze statistics from CustomScanState tasks using * ExtractAnalyzeStats. Always returns false to recurse into all children. */ static bool PlanStateAnalyzeWalker(PlanState *planState, void *ctx) { DistributedSubPlan *subplan = (DistributedSubPlan *) ctx; ExtractAnalyzeStats(subplan, planState); return false; } /* * ExplainWorkerPlan produces explain output into es. If es->analyze, it also executes * the given plannedStmt and sends the results to dest. It puts total time to execute in * executionDurationMillisec. * * This is based on postgres' ExplainOnePlan(). We couldn't use an IntoClause to store results * into tupleStore, so we had to copy the same functionality with some minor changes. * * Keeping the formatting to make comparing with the ExplainOnePlan() easier. * * TODO: Send a PR to postgres to change ExplainOnePlan's API to use a more generic result * destination. */ static void ExplainWorkerPlan(PlannedStmt *plannedstmt, DistributedSubPlan *subPlan, DestReceiver *dest, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, #if PG_VERSION_NUM >= PG_VERSION_17 const BufferUsage *bufusage, const MemoryContextCounters *mem_counters, #endif double *executionDurationMillisec, double *executionTuples, double *executionLoops) { QueryDesc *queryDesc; instr_time starttime; double totaltime = 0; int eflags; int instrument_option = 0; /* Sub-plan already executed; skipping execution */ bool executeQuery = (es->analyze && !subPlan); bool executeSubplan = (es->analyze && subPlan); Assert(plannedstmt->commandType != CMD_UTILITY); if (es->analyze && es->timing) instrument_option |= INSTRUMENT_TIMER; else if (es->analyze) instrument_option |= INSTRUMENT_ROWS; if (es->buffers) instrument_option |= INSTRUMENT_BUFFERS; if (es->wal) instrument_option |= INSTRUMENT_WAL; /* * We always collect timing for the entire statement, even when node-level * timing is off, so we don't look at es->timing here. (We could skip * this if !es->summary, but it's hardly worth the complication.) */ INSTR_TIME_SET_CURRENT(starttime); /* * Use a snapshot with an updated command ID to ensure this query sees * results of any previously executed queries. */ PushCopiedSnapshot(GetActiveSnapshot()); UpdateActiveSnapshotCommandId(); /* Create a QueryDesc for the query */ queryDesc = CreateQueryDesc( plannedstmt, /* PlannedStmt *plannedstmt */ queryString, /* const char *sourceText */ GetActiveSnapshot(), /* Snapshot snapshot */ InvalidSnapshot, /* Snapshot crosscheck_snapshot */ dest, /* DestReceiver *dest */ params, /* ParamListInfo params */ queryEnv, /* QueryEnvironment *queryEnv */ instrument_option /* int instrument_options */ ); /* Select execution options */ if (executeQuery) eflags = 0; /* default run-to-completion flags */ else eflags = EXEC_FLAG_EXPLAIN_ONLY; /* call ExecutorStart to prepare the plan for execution */ ExecutorStart(queryDesc, eflags); /* Execute the plan for statistics if asked for */ if (executeQuery) { ScanDirection dir = ForwardScanDirection; /* run the plan */ /* run the plan: count = 0 (all rows) */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+ dropped the “execute_once” boolean */ ExecutorRun(queryDesc, dir, 0L); #else /* PG 17- still expect the 4th ‘once’ argument */ ExecutorRun(queryDesc, dir, 0L, true); #endif /* run cleanup too */ ExecutorFinish(queryDesc); /* We can't run ExecutorEnd 'till we're done printing the stats... */ totaltime += elapsed_time(&starttime); } ExplainOpenGroup("Query", NULL, true, es); if (executeSubplan) { ExtractAnalyzeStats(subPlan, queryDesc->planstate); planstate_tree_walker(queryDesc->planstate, PlanStateAnalyzeWalker, (void *) subPlan); } /* Create textual dump of plan tree */ ExplainPrintPlan(es, queryDesc); #if PG_VERSION_NUM >= PG_VERSION_17 /* Show buffer and/or memory usage in planning */ if (peek_buffer_usage(es, bufusage) || mem_counters) { ExplainOpenGroup("Planning", "Planning", true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { ExplainIndentText(es); appendStringInfoString(es->str, "Planning:\n"); es->indent++; } if (bufusage) show_buffer_usage(es, bufusage); if (mem_counters) show_memory_counters(es, mem_counters); if (es->format == EXPLAIN_FORMAT_TEXT) es->indent--; ExplainCloseGroup("Planning", "Planning", true, es); } #endif if (es->summary && planduration) { double plantime = INSTR_TIME_GET_DOUBLE(*planduration); ExplainPropertyFloat("Planning Time", "ms", 1000.0 * plantime, 3, es); } /* Print info about runtime of triggers */ if (es->analyze) ExplainPrintTriggers(es, queryDesc); /* * Print info about JITing. Tied to es->costs because we don't want to * display this in regression tests, as it'd cause output differences * depending on build options. Might want to separate that out from COSTS * at a later stage. */ if (es->costs) ExplainPrintJITSummary(es, queryDesc); #if PG_VERSION_NUM >= PG_VERSION_17 if (es->serialize != EXPLAIN_SERIALIZE_NONE) { /* the SERIALIZE option requires its own tuple receiver */ DestReceiver *dest_serialize = CreateExplainSerializeDestReceiver(es); /* grab serialization metrics before we destroy the DestReceiver */ SerializeMetrics serializeMetrics = GetSerializationMetrics(dest_serialize); /* call the DestReceiver's destroy method even during explain */ dest_serialize->rDestroy(dest_serialize); /* Print info about serialization of output */ ExplainPrintSerialize(es, &serializeMetrics); } #endif /* * Close down the query and free resources. Include time for this in the * total execution time (although it should be pretty minimal). */ INSTR_TIME_SET_CURRENT(starttime); if (executeQuery) { Instrumentation *instr = queryDesc->planstate->instrument; *executionTuples = instr->ntuples; *executionLoops = instr->nloops; } ExecutorEnd(queryDesc); FreeQueryDesc(queryDesc); PopActiveSnapshot(); /* We need a CCI just in case query expanded to multiple plans */ if (executeQuery) CommandCounterIncrement(); totaltime += elapsed_time(&starttime); /* * We only report execution time if we actually ran the query (that is, * the user specified ANALYZE), and if summary reporting is enabled (the * user can set SUMMARY OFF to not have the timing information included in * the output). By default, ANALYZE sets SUMMARY to true. */ if (es->summary && es->analyze) ExplainPropertyFloat("Execution Time", "ms", 1000.0 * totaltime, 3, es); *executionDurationMillisec = totaltime * 1000; ExplainCloseGroup("Query", NULL, true, es); } /* * Compute elapsed time in seconds since given timestamp. * * Copied from explain.c. */ static double elapsed_time(instr_time *starttime) { instr_time endtime; INSTR_TIME_SET_CURRENT(endtime); INSTR_TIME_SUBTRACT(endtime, *starttime); return INSTR_TIME_GET_DOUBLE(endtime); } #if PG_VERSION_NUM >= PG_VERSION_17 && PG_VERSION_NUM < PG_VERSION_18 /* * Indent a text-format line. * * We indent by two spaces per indentation level. However, when emitting * data for a parallel worker there might already be data on the current line * (cf. ExplainOpenWorker); in that case, don't indent any more. * * Copied from explain.c. */ static void ExplainIndentText(ExplainState *es) { Assert(es->format == EXPLAIN_FORMAT_TEXT); if (es->str->len == 0 || es->str->data[es->str->len - 1] == '\n') appendStringInfoSpaces(es->str, es->indent * 2); } /* * GetSerializationMetrics - collect metrics * * We have to be careful here since the receiver could be an IntoRel * receiver if the subject statement is CREATE TABLE AS. In that * case, return all-zeroes stats. * * Copied from explain.c. */ static SerializeMetrics GetSerializationMetrics(DestReceiver *dest) { SerializeMetrics empty; if (dest->mydest == DestExplainSerialize) return ((SerializeDestReceiver *) dest)->metrics; memset(&empty, 0, sizeof(SerializeMetrics)); INSTR_TIME_SET_ZERO(empty.timeSpent); return empty; } #endif #if PG_VERSION_NUM >= PG_VERSION_17 /* * Return whether show_buffer_usage would have anything to print, if given * the same 'usage' data. Note that when the format is anything other than * text, we print even if the counters are all zeroes. * * Copied from explain.c. */ static bool peek_buffer_usage(ExplainState *es, const BufferUsage *usage) { bool has_shared; bool has_local; bool has_temp; bool has_shared_timing; bool has_local_timing; bool has_temp_timing; if (usage == NULL) return false; if (es->format != EXPLAIN_FORMAT_TEXT) return true; has_shared = (usage->shared_blks_hit > 0 || usage->shared_blks_read > 0 || usage->shared_blks_dirtied > 0 || usage->shared_blks_written > 0); has_local = (usage->local_blks_hit > 0 || usage->local_blks_read > 0 || usage->local_blks_dirtied > 0 || usage->local_blks_written > 0); has_temp = (usage->temp_blks_read > 0 || usage->temp_blks_written > 0); has_shared_timing = (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)); has_local_timing = (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->local_blk_write_time)); has_temp_timing = (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)); return has_shared || has_local || has_temp || has_shared_timing || has_local_timing || has_temp_timing; } /* * Show buffer usage details. This better be sync with peek_buffer_usage. * * Copied from explain.c. */ static void show_buffer_usage(ExplainState *es, const BufferUsage *usage) { if (es->format == EXPLAIN_FORMAT_TEXT) { bool has_shared = (usage->shared_blks_hit > 0 || usage->shared_blks_read > 0 || usage->shared_blks_dirtied > 0 || usage->shared_blks_written > 0); bool has_local = (usage->local_blks_hit > 0 || usage->local_blks_read > 0 || usage->local_blks_dirtied > 0 || usage->local_blks_written > 0); bool has_temp = (usage->temp_blks_read > 0 || usage->temp_blks_written > 0); bool has_shared_timing = (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)); bool has_local_timing = (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->local_blk_write_time)); bool has_temp_timing = (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time) || !INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)); /* Show only positive counter values. */ if (has_shared || has_local || has_temp) { ExplainIndentText(es); appendStringInfoString(es->str, "Buffers:"); if (has_shared) { appendStringInfoString(es->str, " shared"); if (usage->shared_blks_hit > 0) appendStringInfo(es->str, " hit=%lld", (long long) usage->shared_blks_hit); if (usage->shared_blks_read > 0) appendStringInfo(es->str, " read=%lld", (long long) usage->shared_blks_read); if (usage->shared_blks_dirtied > 0) appendStringInfo(es->str, " dirtied=%lld", (long long) usage->shared_blks_dirtied); if (usage->shared_blks_written > 0) appendStringInfo(es->str, " written=%lld", (long long) usage->shared_blks_written); if (has_local || has_temp) appendStringInfoChar(es->str, ','); } if (has_local) { appendStringInfoString(es->str, " local"); if (usage->local_blks_hit > 0) appendStringInfo(es->str, " hit=%lld", (long long) usage->local_blks_hit); if (usage->local_blks_read > 0) appendStringInfo(es->str, " read=%lld", (long long) usage->local_blks_read); if (usage->local_blks_dirtied > 0) appendStringInfo(es->str, " dirtied=%lld", (long long) usage->local_blks_dirtied); if (usage->local_blks_written > 0) appendStringInfo(es->str, " written=%lld", (long long) usage->local_blks_written); if (has_temp) appendStringInfoChar(es->str, ','); } if (has_temp) { appendStringInfoString(es->str, " temp"); if (usage->temp_blks_read > 0) appendStringInfo(es->str, " read=%lld", (long long) usage->temp_blks_read); if (usage->temp_blks_written > 0) appendStringInfo(es->str, " written=%lld", (long long) usage->temp_blks_written); } appendStringInfoChar(es->str, '\n'); } /* As above, show only positive counter values. */ if (has_shared_timing || has_local_timing || has_temp_timing) { ExplainIndentText(es); appendStringInfoString(es->str, "I/O Timings:"); if (has_shared_timing) { appendStringInfoString(es->str, " shared"); if (!INSTR_TIME_IS_ZERO(usage->shared_blk_read_time)) appendStringInfo(es->str, " read=%0.3f", INSTR_TIME_GET_MILLISEC(usage->shared_blk_read_time)); if (!INSTR_TIME_IS_ZERO(usage->shared_blk_write_time)) appendStringInfo(es->str, " write=%0.3f", INSTR_TIME_GET_MILLISEC(usage->shared_blk_write_time)); if (has_local_timing || has_temp_timing) appendStringInfoChar(es->str, ','); } if (has_local_timing) { appendStringInfoString(es->str, " local"); if (!INSTR_TIME_IS_ZERO(usage->local_blk_read_time)) appendStringInfo(es->str, " read=%0.3f", INSTR_TIME_GET_MILLISEC(usage->local_blk_read_time)); if (!INSTR_TIME_IS_ZERO(usage->local_blk_write_time)) appendStringInfo(es->str, " write=%0.3f", INSTR_TIME_GET_MILLISEC(usage->local_blk_write_time)); if (has_temp_timing) appendStringInfoChar(es->str, ','); } if (has_temp_timing) { appendStringInfoString(es->str, " temp"); if (!INSTR_TIME_IS_ZERO(usage->temp_blk_read_time)) appendStringInfo(es->str, " read=%0.3f", INSTR_TIME_GET_MILLISEC(usage->temp_blk_read_time)); if (!INSTR_TIME_IS_ZERO(usage->temp_blk_write_time)) appendStringInfo(es->str, " write=%0.3f", INSTR_TIME_GET_MILLISEC(usage->temp_blk_write_time)); } appendStringInfoChar(es->str, '\n'); } } else { ExplainPropertyInteger("Shared Hit Blocks", NULL, usage->shared_blks_hit, es); ExplainPropertyInteger("Shared Read Blocks", NULL, usage->shared_blks_read, es); ExplainPropertyInteger("Shared Dirtied Blocks", NULL, usage->shared_blks_dirtied, es); ExplainPropertyInteger("Shared Written Blocks", NULL, usage->shared_blks_written, es); ExplainPropertyInteger("Local Hit Blocks", NULL, usage->local_blks_hit, es); ExplainPropertyInteger("Local Read Blocks", NULL, usage->local_blks_read, es); ExplainPropertyInteger("Local Dirtied Blocks", NULL, usage->local_blks_dirtied, es); ExplainPropertyInteger("Local Written Blocks", NULL, usage->local_blks_written, es); ExplainPropertyInteger("Temp Read Blocks", NULL, usage->temp_blks_read, es); ExplainPropertyInteger("Temp Written Blocks", NULL, usage->temp_blks_written, es); if (track_io_timing) { ExplainPropertyFloat("Shared I/O Read Time", "ms", INSTR_TIME_GET_MILLISEC(usage->shared_blk_read_time), 3, es); ExplainPropertyFloat("Shared I/O Write Time", "ms", INSTR_TIME_GET_MILLISEC(usage->shared_blk_write_time), 3, es); ExplainPropertyFloat("Local I/O Read Time", "ms", INSTR_TIME_GET_MILLISEC(usage->local_blk_read_time), 3, es); ExplainPropertyFloat("Local I/O Write Time", "ms", INSTR_TIME_GET_MILLISEC(usage->local_blk_write_time), 3, es); ExplainPropertyFloat("Temp I/O Read Time", "ms", INSTR_TIME_GET_MILLISEC(usage->temp_blk_read_time), 3, es); ExplainPropertyFloat("Temp I/O Write Time", "ms", INSTR_TIME_GET_MILLISEC(usage->temp_blk_write_time), 3, es); } } } /* * Show memory usage details. * * Copied from explain.c. */ static void show_memory_counters(ExplainState *es, const MemoryContextCounters *mem_counters) { int64 memUsedkB = BYTES_TO_KILOBYTES(mem_counters->totalspace - mem_counters->freespace); int64 memAllocatedkB = BYTES_TO_KILOBYTES(mem_counters->totalspace); if (es->format == EXPLAIN_FORMAT_TEXT) { ExplainIndentText(es); appendStringInfo(es->str, "Memory: used=" INT64_FORMAT "kB allocated=" INT64_FORMAT "kB", memUsedkB, memAllocatedkB); appendStringInfoChar(es->str, '\n'); } else { ExplainPropertyInteger("Memory Used", "kB", memUsedkB, es); ExplainPropertyInteger("Memory Allocated", "kB", memAllocatedkB, es); } } /* * ExplainPrintSerialize - * Append information about query output volume to es->str. * * Copied from explain.c. */ static void ExplainPrintSerialize(ExplainState *es, SerializeMetrics *metrics) { const char *format; /* We shouldn't get called for EXPLAIN_SERIALIZE_NONE */ if (es->serialize == EXPLAIN_SERIALIZE_TEXT) format = "text"; else { Assert(es->serialize == EXPLAIN_SERIALIZE_BINARY); format = "binary"; } ExplainOpenGroup("Serialization", "Serialization", true, es); if (es->format == EXPLAIN_FORMAT_TEXT) { ExplainIndentText(es); if (es->timing) appendStringInfo(es->str, "Serialization: time=%.3f ms output=" UINT64_FORMAT "kB format=%s\n", 1000.0 * INSTR_TIME_GET_DOUBLE(metrics->timeSpent), BYTES_TO_KILOBYTES(metrics->bytesSent), format); else appendStringInfo(es->str, "Serialization: output=" UINT64_FORMAT "kB format=%s\n", BYTES_TO_KILOBYTES(metrics->bytesSent), format); if (es->buffers && peek_buffer_usage(es, &metrics->bufferUsage)) { es->indent++; show_buffer_usage(es, &metrics->bufferUsage); es->indent--; } } else { if (es->timing) ExplainPropertyFloat("Time", "ms", 1000.0 * INSTR_TIME_GET_DOUBLE(metrics->timeSpent), 3, es); ExplainPropertyInteger("Output Volume", "kB", BYTES_TO_KILOBYTES(metrics->bytesSent), es); ExplainPropertyText("Format", format, es); if (es->buffers) show_buffer_usage(es, &metrics->bufferUsage); } ExplainCloseGroup("Serialization", "Serialization", true, es); } #endif ================================================ FILE: src/backend/distributed/planner/multi_join_order.c ================================================ /*------------------------------------------------------------------------- * * multi_join_order.c * * Routines for constructing the join order list using a rule-based approach. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "lib/stringinfo.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/worker_protocol.h" /* Config variables managed via guc.c */ bool LogMultiJoinOrder = false; /* print join order as a debugging aid */ bool EnableSingleHashRepartitioning = false; /* Function pointer type definition for join rule evaluation functions */ typedef JoinOrderNode *(*RuleEvalFunction) (JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static char *RuleNameArray[JOIN_RULE_LAST] = { 0 }; /* ordered join rule names */ static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ static bool JoinExprListWalker(Node *node, List **joinList); static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex); static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClauseList); static List * BestJoinOrder(List *candidateJoinOrders); static List * FewestOfJoinRuleType(List *candidateJoinOrders, JoinRuleType ruleType); static uint32 JoinRuleTypeCount(List *joinOrder, JoinRuleType ruleTypeToCount); static List * LatestLargeDataTransfer(List *candidateJoinOrders); static void PrintJoinOrderList(List *joinOrder); static uint32 LargeDataTransferLocation(List *joinOrder); static List * TableEntryListDifference(List *lhsTableList, List *rhsTableList); /* Local functions forward declarations for join evaluations */ static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *joinClauseList, JoinType joinType); static List * RangeTableIdList(List *tableList); static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType); static char * JoinRuleName(JoinRuleType ruleType); static JoinOrderNode * ReferenceJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * CartesianProductReferenceJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * LocalJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType joinRuleType, List *partitionColumnList, char partitionMethod, TableEntry *anchorTable); /* * JoinExprList flattens the JoinExpr nodes in the FROM expression and translate implicit * joins to inner joins. This function does not consider (right-)nested joins. */ List * JoinExprList(FromExpr *fromExpr) { List *joinList = NIL; List *fromList = fromExpr->fromlist; ListCell *fromCell = NULL; foreach(fromCell, fromList) { Node *nextNode = (Node *) lfirst(fromCell); if (joinList != NIL) { /* multiple nodes in from clause, add an explicit join between them */ int nextRangeTableIndex = 0; /* find the left most range table in this node */ ExtractLeftMostRangeTableIndex((Node *) fromExpr, &nextRangeTableIndex); RangeTblRef *nextRangeTableRef = makeNode(RangeTblRef); nextRangeTableRef->rtindex = nextRangeTableIndex; /* join the previous node with nextRangeTableRef */ JoinExpr *newJoinExpr = makeNode(JoinExpr); newJoinExpr->jointype = JOIN_INNER; newJoinExpr->rarg = (Node *) nextRangeTableRef; newJoinExpr->quals = NULL; joinList = lappend(joinList, newJoinExpr); } JoinExprListWalker(nextNode, &joinList); } return joinList; } /* * JoinExprListWalker the JoinExpr nodes in a join tree in the order in which joins are * to be executed. If there are no joins then no elements are added to joinList. */ static bool JoinExprListWalker(Node *node, List **joinList) { bool walkerResult = false; if (node == NULL) { return false; } if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; walkerResult = JoinExprListWalker(joinExpr->larg, joinList); (*joinList) = lappend(*joinList, joinExpr); } else { walkerResult = expression_tree_walker(node, JoinExprListWalker, joinList); } return walkerResult; } /* * ExtractLeftMostRangeTableIndex extracts the range table index of the left-most * leaf in a join tree. */ static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex) { bool walkerResult = false; Assert(node != NULL); if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; walkerResult = ExtractLeftMostRangeTableIndex(joinExpr->larg, rangeTableIndex); } else if (IsA(node, RangeTblRef)) { RangeTblRef *rangeTableRef = (RangeTblRef *) node; *rangeTableIndex = rangeTableRef->rtindex; walkerResult = true; } else { walkerResult = expression_tree_walker(node, ExtractLeftMostRangeTableIndex, rangeTableIndex); } return walkerResult; } /* * JoinOnColumns determines whether two columns are joined by a given join clause list. */ bool JoinOnColumns(List *currentPartitionColumnList, Var *candidateColumn, List *joinClauseList) { if (candidateColumn == NULL || list_length(currentPartitionColumnList) == 0) { /* * LocalJoin can only be happening if we have both a current column and a target * column, otherwise we are not joining two local tables */ return false; } Var *currentColumn = NULL; foreach_declared_ptr(currentColumn, currentPartitionColumnList) { Node *joinClause = NULL; foreach_declared_ptr(joinClause, joinClauseList) { if (!NodeIsEqualsOpExpr(joinClause)) { continue; } OpExpr *joinClauseOpExpr = castNode(OpExpr, joinClause); Var *leftColumn = LeftColumnOrNULL(joinClauseOpExpr); Var *rightColumn = RightColumnOrNULL(joinClauseOpExpr); /* * Check if both join columns and both partition key columns match, since the * current and candidate column's can't be NULL we know they won't match if either * of the columns resolved to NULL above. */ if (equal(leftColumn, currentColumn) && equal(rightColumn, candidateColumn)) { return true; } if (equal(leftColumn, candidateColumn) && equal(rightColumn, currentColumn)) { return true; } } } return false; } /* * NodeIsEqualsOpExpr checks if the node is an OpExpr, where the operator * matches OperatorImplementsEquality. */ bool NodeIsEqualsOpExpr(Node *node) { if (!IsA(node, OpExpr)) { return false; } OpExpr *opExpr = castNode(OpExpr, node); return OperatorImplementsEquality(opExpr->opno); } /* * JoinOrderList calculates the best join order and join rules that apply given * the list of tables and join clauses. First, the function generates a set of * candidate join orders, each with a different table as its first table. Then, * the function chooses among these candidates the join order that transfers the * least amount of data across the network, and returns this join order. */ List * JoinOrderList(List *tableEntryList, List *joinClauseList) { List *candidateJoinOrderList = NIL; ListCell *tableEntryCell = NULL; foreach(tableEntryCell, tableEntryList) { TableEntry *startingTable = (TableEntry *) lfirst(tableEntryCell); /* each candidate join order starts with a different table */ List *candidateJoinOrder = JoinOrderForTable(startingTable, tableEntryList, joinClauseList); if (candidateJoinOrder != NULL) { candidateJoinOrderList = lappend(candidateJoinOrderList, candidateJoinOrder); } } if (list_length(candidateJoinOrderList) == 0) { /* there are no plans that we can create, time to error */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("complex joins are only supported when all distributed " "tables are joined on their distribution columns with " "equal operator"))); } List *bestJoinOrder = BestJoinOrder(candidateJoinOrderList); /* if logging is enabled, print join order */ if (LogMultiJoinOrder) { PrintJoinOrderList(bestJoinOrder); } return bestJoinOrder; } /* * JoinOrderForTable creates a join order whose first element is the given first * table. To determine each subsequent element in the join order, the function * then chooses the table that has the lowest ranking join rule, and with which * it can join the table to the previous table in the join order. The function * repeats this until it determines all elements in the join order list, and * returns this list. */ static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClauseList) { JoinRuleType firstJoinRule = JOIN_RULE_INVALID_FIRST; int joinedTableCount = 1; int totalTableCount = list_length(tableEntryList); /* create join node for the first table */ Oid firstRelationId = firstTable->relationId; uint32 firstTableId = firstTable->rangeTableId; Var *firstPartitionColumn = PartitionColumn(firstRelationId, firstTableId); char firstPartitionMethod = PartitionMethod(firstRelationId); JoinOrderNode *firstJoinNode = MakeJoinOrderNode(firstTable, firstJoinRule, list_make1(firstPartitionColumn), firstPartitionMethod, firstTable); /* add first node to the join order */ List *joinOrderList = list_make1(firstJoinNode); List *joinedTableList = list_make1(firstTable); JoinOrderNode *currentJoinNode = firstJoinNode; /* loop until we join all remaining tables */ while (joinedTableCount < totalTableCount) { ListCell *pendingTableCell = NULL; JoinOrderNode *nextJoinNode = NULL; JoinRuleType nextJoinRuleType = JOIN_RULE_LAST; List *pendingTableList = TableEntryListDifference(tableEntryList, joinedTableList); /* * Iterate over all pending tables, and find the next best table to * join. The best table is the one whose join rule requires the least * amount of data transfer. */ foreach(pendingTableCell, pendingTableList) { TableEntry *pendingTable = (TableEntry *) lfirst(pendingTableCell); JoinType joinType = JOIN_INNER; /* evaluate all join rules for this pending table */ JoinOrderNode *pendingJoinNode = EvaluateJoinRules(joinedTableList, currentJoinNode, pendingTable, joinClauseList, joinType); if (pendingJoinNode == NULL) { /* no join order could be generated, we try our next pending table */ continue; } /* if this rule is better than previous ones, keep it */ JoinRuleType pendingJoinRuleType = pendingJoinNode->joinRuleType; if (pendingJoinRuleType < nextJoinRuleType) { nextJoinNode = pendingJoinNode; nextJoinRuleType = pendingJoinRuleType; } } if (nextJoinNode == NULL) { /* * There is no next join node found, this will repeat indefinitely hence we * bail and let JoinOrderList try a new initial table */ return NULL; } Assert(nextJoinNode != NULL); TableEntry *nextJoinedTable = nextJoinNode->tableEntry; /* add next node to the join order */ joinOrderList = lappend(joinOrderList, nextJoinNode); joinedTableList = lappend(joinedTableList, nextJoinedTable); currentJoinNode = nextJoinNode; joinedTableCount++; } return joinOrderList; } /* * BestJoinOrder takes in a list of candidate join orders, and determines the * best join order among these candidates. The function uses two heuristics for * this. First, the function chooses join orders that have the fewest number of * join operators that cause large data transfers. Second, the function chooses * join orders where large data transfers occur later in the execution. */ static List * BestJoinOrder(List *candidateJoinOrders) { uint32 highestValidIndex = JOIN_RULE_LAST - 1; uint32 candidateCount PG_USED_FOR_ASSERTS_ONLY = 0; /* * We start with the highest ranking rule type (cartesian product), and walk * over these rules in reverse order. For each rule type, we then keep join * orders that only contain the fewest number of join rules of that type. * * For example, the algorithm chooses join orders like the following: * (a) The algorithm prefers join orders with 2 cartesian products (CP) to * those that have 3 or more, if there isn't a join order with fewer CPs. * (b) Assuming that all join orders have the same number of CPs, the * algorithm prefers join orders with 2 dual partitions (DP) to those that * have 3 or more, if there isn't a join order with fewer DPs; and so * forth. */ for (uint32 ruleTypeIndex = highestValidIndex; ruleTypeIndex > 0; ruleTypeIndex--) { JoinRuleType ruleType = (JoinRuleType) ruleTypeIndex; candidateJoinOrders = FewestOfJoinRuleType(candidateJoinOrders, ruleType); } /* * If there is a tie, we pick candidate join orders where large data * transfers happen at later stages of query execution. This results in more * data being filtered via joins, selections, and projections earlier on. */ candidateJoinOrders = LatestLargeDataTransfer(candidateJoinOrders); /* we should have at least one join order left after optimizations */ candidateCount = list_length(candidateJoinOrders); Assert(candidateCount > 0); /* * If there still is a tie, we pick the join order whose relation appeared * earliest in the query's range table entry list. */ List *bestJoinOrder = (List *) linitial(candidateJoinOrders); return bestJoinOrder; } /* * FewestOfJoinRuleType finds join orders that have the fewest number of times * the given join rule occurs in the candidate join orders, and filters all * other join orders. For example, if four candidate join orders have a join * rule appearing 3, 5, 3, and 6 times, only two join orders that have the join * rule appearing 3 times will be returned. */ static List * FewestOfJoinRuleType(List *candidateJoinOrders, JoinRuleType ruleType) { List *fewestJoinOrders = NULL; uint32 fewestRuleCount = INT_MAX; ListCell *joinOrderCell = NULL; foreach(joinOrderCell, candidateJoinOrders) { List *joinOrder = (List *) lfirst(joinOrderCell); uint32 ruleTypeCount = JoinRuleTypeCount(joinOrder, ruleType); if (ruleTypeCount == fewestRuleCount) { fewestJoinOrders = lappend(fewestJoinOrders, joinOrder); } else if (ruleTypeCount < fewestRuleCount) { fewestJoinOrders = list_make1(joinOrder); fewestRuleCount = ruleTypeCount; } } return fewestJoinOrders; } /* Counts the number of times the given join rule occurs in the join order. */ static uint32 JoinRuleTypeCount(List *joinOrder, JoinRuleType ruleTypeToCount) { uint32 ruleTypeCount = 0; ListCell *joinOrderNodeCell = NULL; foreach(joinOrderNodeCell, joinOrder) { JoinOrderNode *joinOrderNode = (JoinOrderNode *) lfirst(joinOrderNodeCell); JoinRuleType ruleType = joinOrderNode->joinRuleType; if (ruleType == ruleTypeToCount) { ruleTypeCount++; } } return ruleTypeCount; } /* * LatestLargeDataTransfer finds and returns join orders where a large data * transfer join rule occurs as late as possible in the join order. Late large * data transfers result in more data being filtered before data gets shuffled * in the network. */ static List * LatestLargeDataTransfer(List *candidateJoinOrders) { List *latestJoinOrders = NIL; uint32 latestJoinLocation = 0; ListCell *joinOrderCell = NULL; foreach(joinOrderCell, candidateJoinOrders) { List *joinOrder = (List *) lfirst(joinOrderCell); uint32 joinRuleLocation = LargeDataTransferLocation(joinOrder); if (joinRuleLocation == latestJoinLocation) { latestJoinOrders = lappend(latestJoinOrders, joinOrder); } else if (joinRuleLocation > latestJoinLocation) { latestJoinOrders = list_make1(joinOrder); latestJoinLocation = joinRuleLocation; } } return latestJoinOrders; } /* * LargeDataTransferLocation finds the first location of a large data transfer * join rule, and returns that location. If the join order does not have any * large data transfer rules, the function returns one location past the end of * the join order list. */ static uint32 LargeDataTransferLocation(List *joinOrder) { uint32 joinRuleLocation = 0; ListCell *joinOrderNodeCell = NULL; foreach(joinOrderNodeCell, joinOrder) { JoinOrderNode *joinOrderNode = (JoinOrderNode *) lfirst(joinOrderNodeCell); JoinRuleType joinRuleType = joinOrderNode->joinRuleType; /* we consider the following join rules to cause large data transfers */ if (joinRuleType == SINGLE_HASH_PARTITION_JOIN || joinRuleType == SINGLE_RANGE_PARTITION_JOIN || joinRuleType == DUAL_PARTITION_JOIN || joinRuleType == CARTESIAN_PRODUCT) { break; } joinRuleLocation++; } return joinRuleLocation; } /* Prints the join order list and join rules for debugging purposes. */ static void PrintJoinOrderList(List *joinOrder) { StringInfo printBuffer = makeStringInfo(); ListCell *joinOrderNodeCell = NULL; bool firstJoinNode = true; foreach(joinOrderNodeCell, joinOrder) { JoinOrderNode *joinOrderNode = (JoinOrderNode *) lfirst(joinOrderNodeCell); Oid relationId = joinOrderNode->tableEntry->relationId; char *relationName = get_rel_name(relationId); if (firstJoinNode) { appendStringInfo(printBuffer, "[ \"%s\" ]", relationName); firstJoinNode = false; } else { JoinRuleType ruleType = (JoinRuleType) joinOrderNode->joinRuleType; char *ruleName = JoinRuleName(ruleType); appendStringInfo(printBuffer, "[ %s ", ruleName); appendStringInfo(printBuffer, "\"%s\" ]", relationName); } } ereport(LOG, (errmsg("join order: %s", printBuffer->data))); } /* * TableEntryListDifference returns a list containing table entries that are in * the left-hand side table list, but not in the right-hand side table list. */ static List * TableEntryListDifference(List *lhsTableList, List *rhsTableList) { List *tableListDifference = NIL; ListCell *lhsTableCell = NULL; foreach(lhsTableCell, lhsTableList) { TableEntry *lhsTableEntry = (TableEntry *) lfirst(lhsTableCell); ListCell *rhsTableCell = NULL; bool lhsTableEntryExists = false; foreach(rhsTableCell, rhsTableList) { TableEntry *rhsTableEntry = (TableEntry *) lfirst(rhsTableCell); if ((lhsTableEntry->relationId == rhsTableEntry->relationId) && (lhsTableEntry->rangeTableId == rhsTableEntry->rangeTableId)) { lhsTableEntryExists = true; } } if (!lhsTableEntryExists) { tableListDifference = lappend(tableListDifference, lhsTableEntry); } } return tableListDifference; } /* * EvaluateJoinRules takes in a list of already joined tables and a candidate * next table, evaluates different join rules between the two tables, and finds * the best join rule that applies. The function returns the applicable join * order node which includes the join rule and the partition information. */ static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *joinClauseList, JoinType joinType) { JoinOrderNode *nextJoinNode = NULL; uint32 lowestValidIndex = JOIN_RULE_INVALID_FIRST + 1; uint32 highestValidIndex = JOIN_RULE_LAST - 1; /* * We first find all applicable join clauses between already joined tables * and the candidate table. */ List *joinedTableIdList = RangeTableIdList(joinedTableList); uint32 candidateTableId = candidateTable->rangeTableId; List *applicableJoinClauses = ApplicableJoinClauses(joinedTableIdList, candidateTableId, joinClauseList); /* we then evaluate all join rules in order */ for (uint32 ruleIndex = lowestValidIndex; ruleIndex <= highestValidIndex; ruleIndex++) { JoinRuleType ruleType = (JoinRuleType) ruleIndex; RuleEvalFunction ruleEvalFunction = JoinRuleEvalFunction(ruleType); nextJoinNode = (*ruleEvalFunction)(currentJoinNode, candidateTable, applicableJoinClauses, joinType); /* break after finding the first join rule that applies */ if (nextJoinNode != NULL) { break; } } if (nextJoinNode == NULL) { return NULL; } Assert(nextJoinNode != NULL); nextJoinNode->joinType = joinType; nextJoinNode->joinClauseList = applicableJoinClauses; return nextJoinNode; } /* Extracts range table identifiers from the given table list, and returns them. */ static List * RangeTableIdList(List *tableList) { List *rangeTableIdList = NIL; ListCell *tableCell = NULL; foreach(tableCell, tableList) { TableEntry *tableEntry = (TableEntry *) lfirst(tableCell); uint32 rangeTableId = tableEntry->rangeTableId; rangeTableIdList = lappend_int(rangeTableIdList, rangeTableId); } return rangeTableIdList; } /* * JoinRuleEvalFunction returns a function pointer for the rule evaluation * function; this rule evaluation function corresponds to the given rule type. * The function also initializes the rule evaluation function array in a static * code block, if the array has not been initialized. */ static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType) { static bool ruleEvalFunctionsInitialized = false; if (!ruleEvalFunctionsInitialized) { RuleEvalFunctionArray[REFERENCE_JOIN] = &ReferenceJoin; RuleEvalFunctionArray[LOCAL_PARTITION_JOIN] = &LocalJoin; RuleEvalFunctionArray[SINGLE_RANGE_PARTITION_JOIN] = &SinglePartitionJoin; RuleEvalFunctionArray[SINGLE_HASH_PARTITION_JOIN] = &SinglePartitionJoin; RuleEvalFunctionArray[DUAL_PARTITION_JOIN] = &DualPartitionJoin; RuleEvalFunctionArray[CARTESIAN_PRODUCT_REFERENCE_JOIN] = &CartesianProductReferenceJoin; RuleEvalFunctionArray[CARTESIAN_PRODUCT] = &CartesianProduct; ruleEvalFunctionsInitialized = true; } RuleEvalFunction ruleEvalFunction = RuleEvalFunctionArray[ruleType]; Assert(ruleEvalFunction != NULL); return ruleEvalFunction; } /* Returns a string name for the given join rule type. */ static char * JoinRuleName(JoinRuleType ruleType) { static bool ruleNamesInitialized = false; if (!ruleNamesInitialized) { /* use strdup() to be independent of memory contexts */ RuleNameArray[REFERENCE_JOIN] = strdup("reference join"); RuleNameArray[LOCAL_PARTITION_JOIN] = strdup("local partition join"); RuleNameArray[SINGLE_HASH_PARTITION_JOIN] = strdup("single hash partition join"); RuleNameArray[SINGLE_RANGE_PARTITION_JOIN] = strdup("single range partition join"); RuleNameArray[DUAL_PARTITION_JOIN] = strdup("dual partition join"); RuleNameArray[CARTESIAN_PRODUCT_REFERENCE_JOIN] = strdup( "cartesian product reference join"); RuleNameArray[CARTESIAN_PRODUCT] = strdup("cartesian product"); ruleNamesInitialized = true; } char *ruleName = RuleNameArray[ruleType]; Assert(ruleName != NULL); return ruleName; } /* * ReferenceJoin evaluates if the candidate table is a reference table for inner, * left and anti join. For right join, current join node must be represented by * a reference table. For full join, both of them must be a reference table. */ static JoinOrderNode * ReferenceJoin(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { int applicableJoinCount = list_length(applicableJoinClauses); if (applicableJoinCount <= 0) { return NULL; } bool leftIsReferenceTable = IsCitusTableType( currentJoinNode->tableEntry->relationId, REFERENCE_TABLE); bool rightIsReferenceTable = IsCitusTableType(candidateTable->relationId, REFERENCE_TABLE); if (!IsSupportedReferenceJoin(joinType, leftIsReferenceTable, rightIsReferenceTable)) { return NULL; } return MakeJoinOrderNode(candidateTable, REFERENCE_JOIN, currentJoinNode->partitionColumnList, currentJoinNode->partitionMethod, currentJoinNode->anchorTable); } /* * IsSupportedReferenceJoin checks if with this join type we can safely do a simple join * on the reference table on all the workers. */ bool IsSupportedReferenceJoin(JoinType joinType, bool leftIsReferenceTable, bool rightIsReferenceTable) { if ((joinType == JOIN_INNER || joinType == JOIN_LEFT || joinType == JOIN_ANTI) && rightIsReferenceTable) { return true; } else if ((joinType == JOIN_RIGHT) && leftIsReferenceTable) { return true; } else if (joinType == JOIN_FULL && leftIsReferenceTable && rightIsReferenceTable) { return true; } return false; } /* * ReferenceJoin evaluates if the candidate table is a reference table for inner, * left and anti join. For right join, current join node must be represented by * a reference table. For full join, both of them must be a reference table. */ static JoinOrderNode * CartesianProductReferenceJoin(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { bool leftIsReferenceTable = IsCitusTableType( currentJoinNode->tableEntry->relationId, REFERENCE_TABLE); bool rightIsReferenceTable = IsCitusTableType(candidateTable->relationId, REFERENCE_TABLE); if (!IsSupportedReferenceJoin(joinType, leftIsReferenceTable, rightIsReferenceTable)) { return NULL; } return MakeJoinOrderNode(candidateTable, CARTESIAN_PRODUCT_REFERENCE_JOIN, currentJoinNode->partitionColumnList, currentJoinNode->partitionMethod, currentJoinNode->anchorTable); } /* * LocalJoin takes the current partition key column and the candidate table's * partition key column and the partition method for each table. The function * then evaluates if tables in the join order and the candidate table can be * joined locally, without any data transfers. If they can, the function returns * a join order node for a local join. Otherwise, the function returns null. * * Anchor table is used to decide whether the JoinOrderNode can be joined * locally with the candidate table. That table is updated by each join type * applied over JoinOrderNode. Note that, we lost the anchor table after * dual partitioning and cartesian product. */ static JoinOrderNode * LocalJoin(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { Oid relationId = candidateTable->relationId; uint32 tableId = candidateTable->rangeTableId; Var *candidatePartitionColumn = PartitionColumn(relationId, tableId); List *currentPartitionColumnList = currentJoinNode->partitionColumnList; char candidatePartitionMethod = PartitionMethod(relationId); char currentPartitionMethod = currentJoinNode->partitionMethod; TableEntry *currentAnchorTable = currentJoinNode->anchorTable; /* * If we previously dual-hash re-partitioned the tables for a join or made cartesian * product, there is no anchor table anymore. In that case we don't allow local join. */ if (currentAnchorTable == NULL) { return NULL; } /* the partition method should be the same for a local join */ if (currentPartitionMethod != candidatePartitionMethod) { return NULL; } bool joinOnPartitionColumns = JoinOnColumns(currentPartitionColumnList, candidatePartitionColumn, applicableJoinClauses); if (!joinOnPartitionColumns) { return NULL; } /* shard interval lists must have 1-1 matching for local joins */ bool coPartitionedTables = CoPartitionedTables(currentAnchorTable->relationId, relationId); if (!coPartitionedTables) { return NULL; } /* * Since we are applying a local join to the candidate table we need to keep track of * the partition column of the candidate table on the MultiJoinNode. This will allow * subsequent joins colocated with this candidate table to correctly be recognized as * a local join as well. */ currentPartitionColumnList = list_append_unique(currentPartitionColumnList, candidatePartitionColumn); JoinOrderNode *nextJoinNode = MakeJoinOrderNode(candidateTable, LOCAL_PARTITION_JOIN, currentPartitionColumnList, currentPartitionMethod, currentAnchorTable); return nextJoinNode; } /* * SinglePartitionJoin takes the current and the candidate table's partition keys * and methods. The function then evaluates if either "tables in the join order" * or the candidate table is already partitioned on a join column. If they are, * the function returns a join order node with the already partitioned column as * the next partition key. Otherwise, the function returns null. */ static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { List *currentPartitionColumnList = currentJoinNode->partitionColumnList; char currentPartitionMethod = currentJoinNode->partitionMethod; TableEntry *currentAnchorTable = currentJoinNode->anchorTable; JoinRuleType currentJoinRuleType = currentJoinNode->joinRuleType; Oid relationId = candidateTable->relationId; uint32 tableId = candidateTable->rangeTableId; Var *candidatePartitionColumn = PartitionColumn(relationId, tableId); char candidatePartitionMethod = PartitionMethod(relationId); /* outer joins are not supported yet */ if (IS_OUTER_JOIN(joinType)) { return NULL; } /* * If we previously dual-hash re-partitioned the tables for a join or made * cartesian product, we currently don't allow a single-repartition join. */ if (currentJoinRuleType == DUAL_PARTITION_JOIN || currentJoinRuleType == CARTESIAN_PRODUCT) { return NULL; } OpExpr *joinClause = SinglePartitionJoinClause(currentPartitionColumnList, applicableJoinClauses, NULL); if (joinClause != NULL) { if (currentPartitionMethod == DISTRIBUTE_BY_HASH) { /* * Single hash repartitioning may perform worse than dual hash * repartitioning. Thus, we control it via a guc. */ if (!EnableSingleHashRepartitioning) { return NULL; } return MakeJoinOrderNode(candidateTable, SINGLE_HASH_PARTITION_JOIN, currentPartitionColumnList, currentPartitionMethod, currentAnchorTable); } else if (candidatePartitionMethod == DISTRIBUTE_BY_RANGE) { return MakeJoinOrderNode(candidateTable, SINGLE_RANGE_PARTITION_JOIN, currentPartitionColumnList, currentPartitionMethod, currentAnchorTable); } } /* evaluate re-partitioning the current table only if the rule didn't apply above */ if (candidatePartitionMethod != DISTRIBUTE_BY_NONE) { /* * Create a new unique list (set) with the partition column of the candidate table * to check if a single repartition join will work for this table. When it works * the set is retained on the MultiJoinNode for later local join verification. */ List *candidatePartitionColumnList = list_make1(candidatePartitionColumn); joinClause = SinglePartitionJoinClause(candidatePartitionColumnList, applicableJoinClauses, NULL); if (joinClause != NULL) { if (candidatePartitionMethod == DISTRIBUTE_BY_HASH) { /* * Single hash repartitioning may perform worse than dual hash * repartitioning. Thus, we control it via a guc. */ if (!EnableSingleHashRepartitioning) { return NULL; } return MakeJoinOrderNode(candidateTable, SINGLE_HASH_PARTITION_JOIN, candidatePartitionColumnList, candidatePartitionMethod, candidateTable); } else if (currentPartitionMethod == DISTRIBUTE_BY_RANGE) { return MakeJoinOrderNode(candidateTable, SINGLE_RANGE_PARTITION_JOIN, candidatePartitionColumnList, candidatePartitionMethod, candidateTable); } } } return NULL; } /* * SinglePartitionJoinClause walks over the applicable join clause list, and * finds an applicable join clause for the given partition column. If no such * clause exists, the function returns NULL. */ OpExpr * SinglePartitionJoinClause(List *partitionColumnList, List *applicableJoinClauses, bool *foundTypeMismatch) { if (foundTypeMismatch) { *foundTypeMismatch = false; } if (list_length(partitionColumnList) == 0) { return NULL; } Var *partitionColumn = NULL; foreach_declared_ptr(partitionColumn, partitionColumnList) { Node *applicableJoinClause = NULL; foreach_declared_ptr(applicableJoinClause, applicableJoinClauses) { if (!NodeIsEqualsOpExpr(applicableJoinClause)) { continue; } OpExpr *applicableJoinOpExpr = castNode(OpExpr, applicableJoinClause); Var *leftColumn = LeftColumnOrNULL(applicableJoinOpExpr); Var *rightColumn = RightColumnOrNULL(applicableJoinOpExpr); if (leftColumn == NULL || rightColumn == NULL) { /* not a simple partition column join */ continue; } /* * We first check if partition column matches either of the join columns * and if it does, we then check if the join column types match. If the * types are different, we will use different hash functions for the two * column types, and will incorrectly repartition the data. */ if (equal(leftColumn, partitionColumn) || equal(rightColumn, partitionColumn)) { if (leftColumn->vartype == rightColumn->vartype) { return applicableJoinOpExpr; } else { ereport(DEBUG1, (errmsg("single partition column types do not " "match"))); if (foundTypeMismatch) { *foundTypeMismatch = true; } } } } } return NULL; } /* * DualPartitionJoin evaluates if a join clause exists between "tables in the * join order" and the candidate table. If such a clause exists, both tables can * be repartitioned on the join column; and the function returns a join order * node with the join column as the next partition key. Otherwise, the function * returns null. */ static JoinOrderNode * DualPartitionJoin(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { OpExpr *joinClause = DualPartitionJoinClause(applicableJoinClauses); if (joinClause) { /* because of the dual partition, anchor table and partition column get lost */ return MakeJoinOrderNode(candidateTable, DUAL_PARTITION_JOIN, NIL, REDISTRIBUTE_BY_HASH, NULL); } return NULL; } /* * DualPartitionJoinClause walks over the applicable join clause list, and finds * an applicable join clause for dual re-partitioning. If no such clause exists, * the function returns NULL. */ OpExpr * DualPartitionJoinClause(List *applicableJoinClauses) { Node *applicableJoinClause = NULL; foreach_declared_ptr(applicableJoinClause, applicableJoinClauses) { if (!NodeIsEqualsOpExpr(applicableJoinClause)) { continue; } OpExpr *applicableJoinOpExpr = castNode(OpExpr, applicableJoinClause); Var *leftColumn = LeftColumnOrNULL(applicableJoinOpExpr); Var *rightColumn = RightColumnOrNULL(applicableJoinOpExpr); if (leftColumn == NULL || rightColumn == NULL) { continue; } /* we only need to check that the join column types match */ if (leftColumn->vartype == rightColumn->vartype) { return applicableJoinOpExpr; } else { ereport(DEBUG1, (errmsg("dual partition column types do not match"))); } } return NULL; } /* * CartesianProduct always evaluates to true since all tables can be combined * using a cartesian product operator. This function acts as a catch-all rule, * in case none of the join rules apply. */ static JoinOrderNode * CartesianProduct(JoinOrderNode *currentJoinNode, TableEntry *candidateTable, List *applicableJoinClauses, JoinType joinType) { if (list_length(applicableJoinClauses) == 0) { /* Because of the cartesian product, anchor table information got lost */ return MakeJoinOrderNode(candidateTable, CARTESIAN_PRODUCT, currentJoinNode->partitionColumnList, currentJoinNode->partitionMethod, NULL); } return NULL; } /* Constructs and returns a join-order node with the given arguments */ JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType joinRuleType, List *partitionColumnList, char partitionMethod, TableEntry *anchorTable) { JoinOrderNode *joinOrderNode = palloc0(sizeof(JoinOrderNode)); joinOrderNode->tableEntry = tableEntry; joinOrderNode->joinRuleType = joinRuleType; joinOrderNode->joinType = JOIN_INNER; joinOrderNode->partitionColumnList = partitionColumnList; joinOrderNode->partitionMethod = partitionMethod; joinOrderNode->joinClauseList = NIL; joinOrderNode->anchorTable = anchorTable; return joinOrderNode; } /* * IsApplicableJoinClause tests if the current joinClause is applicable to the join at * hand. * * Given a list of left hand tables and a candidate right hand table the join clause is * valid if atleast 1 column is from the right hand table AND all columns can be found * in either the list of tables on the left *or* in the right hand table. */ bool IsApplicableJoinClause(List *leftTableIdList, uint32 rightTableId, Node *joinClause) { List *varList = pull_var_clause_default(joinClause); Var *var = NULL; bool joinContainsRightTable = false; foreach_declared_ptr(var, varList) { uint32 columnTableId = var->varno; if (rightTableId == columnTableId) { joinContainsRightTable = true; } else if (!list_member_int(leftTableIdList, columnTableId)) { /* * We couldn't find this column either on the right hand side (first if * statement), nor in the list on the left. This join clause involves a table * not yet available during the candidate join. */ return false; } } /* * All columns referenced in this clause are available during this join, now the join * is applicable if we found our candidate table as well */ return joinContainsRightTable; } /* * ApplicableJoinClauses finds all join clauses that apply between the given * left table list and the right table, and returns these found join clauses. */ List * ApplicableJoinClauses(List *leftTableIdList, uint32 rightTableId, List *joinClauseList) { List *applicableJoinClauses = NIL; /* make sure joinClauseList contains only join clauses */ joinClauseList = JoinClauseList(joinClauseList); Node *joinClause = NULL; foreach_declared_ptr(joinClause, joinClauseList) { if (IsApplicableJoinClause(leftTableIdList, rightTableId, joinClause)) { applicableJoinClauses = lappend(applicableJoinClauses, joinClause); } } return applicableJoinClauses; } /* * Returns the left column only when directly referenced in the given join clause, * otherwise NULL is returned. */ Var * LeftColumnOrNULL(OpExpr *joinClause) { List *argumentList = joinClause->args; Node *leftArgument = (Node *) linitial(argumentList); leftArgument = strip_implicit_coercions(leftArgument); if (!IsA(leftArgument, Var)) { return NULL; } return castNode(Var, leftArgument); } /* * Returns the right column only when directly referenced in the given join clause, * otherwise NULL is returned. * */ Var * RightColumnOrNULL(OpExpr *joinClause) { List *argumentList = joinClause->args; Node *rightArgument = (Node *) lsecond(argumentList); rightArgument = strip_implicit_coercions(rightArgument); if (!IsA(rightArgument, Var)) { return NULL; } return castNode(Var, rightArgument); } /* * PartitionColumn builds the partition column for the given relation, and sets * the partition column's range table references to the given table identifier. * * Note that reference tables do not have partition column. Thus, this function * returns NULL when called for reference tables. */ Var * PartitionColumn(Oid relationId, uint32 rangeTableId) { Var *partitionKey = DistPartitionKey(relationId); Var *partitionColumn = NULL; /* short circuit for reference tables */ if (partitionKey == NULL) { return partitionColumn; } partitionColumn = partitionKey; partitionColumn->varno = rangeTableId; partitionColumn->varnosyn = rangeTableId; return partitionColumn; } /* * DistPartitionKey returns the partition key column for the given relation. Note * that in the context of distributed join and query planning, the callers of * this function *must* set the partition key column's range table reference * (varno) to match the table's location in the query range table list. * * Note that reference tables do not have partition column. Thus, this function * returns NULL when called for reference tables. */ Var * DistPartitionKey(Oid relationId) { CitusTableCacheEntry *partitionEntry = GetCitusTableCacheEntry(relationId); /* non-distributed tables do not have partition column */ if (!HasDistributionKeyCacheEntry(partitionEntry)) { return NULL; } return copyObject(partitionEntry->partitionColumn); } /* * DistPartitionKeyOrError is the same as DistPartitionKey but errors out instead * of returning NULL if this is called with a relationId of a reference table. */ Var * DistPartitionKeyOrError(Oid relationId) { Var *partitionKey = DistPartitionKey(relationId); if (partitionKey == NULL) { ereport(ERROR, (errmsg( "no distribution column found for relation %d", relationId))); } return partitionKey; } /* Returns the partition method for the given relation. */ char PartitionMethod(Oid relationId) { /* errors out if not a distributed table */ CitusTableCacheEntry *partitionEntry = GetCitusTableCacheEntry(relationId); char partitionMethod = partitionEntry->partitionMethod; return partitionMethod; } /* Returns the replication model for the given relation. */ char TableReplicationModel(Oid relationId) { /* errors out if not a distributed table */ CitusTableCacheEntry *partitionEntry = GetCitusTableCacheEntry(relationId); char replicationModel = partitionEntry->replicationModel; return replicationModel; } ================================================ FILE: src/backend/distributed/planner/multi_logical_optimizer.c ================================================ /*------------------------------------------------------------------------- * * multi_logical_optimizer.c * Routines for optimizing logical plan trees based on multi-relational * algebra. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/nbtree.h" #include "catalog/indexing.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/extension.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/tlist.h" #include "parser/parse_agg.h" #include "parser/parse_coerce.h" #include "parser/parse_oper.h" #include "parser/parsetree.h" #include "rewrite/rewriteManip.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/errormessage.h" #include "distributed/extended_op_node_utils.h" #include "distributed/function_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/query_pushdown_planning.h" #include "distributed/string_utils.h" #include "distributed/tdigest_extension.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* Config variable managed via guc.c */ int LimitClauseRowFetchCount = -1; /* number of rows to fetch from each task */ double CountDistinctErrorRate = 0.0; /* precision of count(distinct) approximate */ int CoordinatorAggregationStrategy = COORDINATOR_AGGREGATION_ROW_GATHER; bool AllowAggregateWorkerCombineOnInternalTypes = true; /* Constant used throughout file */ static const uint32 masterTableId = 1; /* first range table reference on the master node */ typedef struct MasterAggregateWalkerContext { const ExtendedOpNodeProperties *extendedOpNodeProperties; AttrNumber columnId; } MasterAggregateWalkerContext; typedef struct WorkerAggregateWalkerContext { const ExtendedOpNodeProperties *extendedOpNodeProperties; List *expressionList; bool createGroupByClause; } WorkerAggregateWalkerContext; /* * QueryTargetList encapsulates the necessary fields to form * worker query's target list. */ typedef struct QueryTargetList { List *targetEntryList; /* the list of target entries */ AttrNumber targetProjectionNumber; /* the index of the last entry */ } QueryTargetList; /* * QueryGroupClause encapsulates the necessary fields to form * worker query's group by clause. */ typedef struct QueryGroupClause { List *groupClauseList; /* the list of group clause entries */ Index *nextSortGroupRefIndex; /* pointer to the index of the largest sort group reference index */ } QueryGroupClause; /* * QueryDistinctClause encapsulates the necessary fields to form * worker query's DISTINCT/DISTINCT ON parts. */ typedef struct QueryDistinctClause { List *workerDistinctClause; /* the list of distinct clause entries */ bool workerHasDistinctOn; } QueryDistinctClause; /* * QueryWindowClause encapsulates the necessary fields to form * worker query's window clause. */ typedef struct QueryWindowClause { List *workerWindowClauseList; /* the list of window clause entries */ bool hasWindowFunctions; Index *nextSortGroupRefIndex; /* see QueryGroupClause */ } QueryWindowClause; /* * QueryOrderByLimit encapsulates the necessary fields to form * worker query's order by and limit clauses. Note that we don't * keep track of limit offset clause since it is incorporated * into the limit clause during the processing. */ typedef struct QueryOrderByLimit { Node *workerLimitCount; List *workerSortClauseList; Index *nextSortGroupRefIndex; /* see QueryGroupClause */ } QueryOrderByLimit; /* * LimitPushdownable tells us how a limit can be pushed down. * See WorkerLimitCount for details. */ typedef enum LimitPushdownable { LIMIT_CANNOT_PUSHDOWN, LIMIT_CAN_PUSHDOWN, LIMIT_CAN_APPROXIMATE, } LimitPushdownable; /* * OrderByLimitReference a structure that is used commonly while * processing sort and limit clauses. */ typedef struct OrderByLimitReference { bool groupedByDisjointPartitionColumn; bool onlyPushableWindowFunctions; bool groupClauseIsEmpty; bool sortClauseIsEmpty; bool hasOrderByAggregate; bool canApproximate; bool hasDistinctOn; } OrderByLimitReference; /* Local functions forward declarations */ static MultiSelect * AndSelectNode(MultiSelect *selectNode); static MultiSelect * OrSelectNode(MultiSelect *selectNode); static List * OrSelectClauseList(List *selectClauseList); static void PushDownNodeLoop(MultiUnaryNode *currentNode); static void PullUpCollectLoop(MultiCollect *collectNode); static void AddressProjectSpecialConditions(MultiProject *projectNode); static PushDownStatus CanPushDown(MultiUnaryNode *parentNode); static PullUpStatus CanPullUp(MultiUnaryNode *childNode); static PushDownStatus Commutative(MultiUnaryNode *parentNode, MultiUnaryNode *childNode); static PushDownStatus Distributive(MultiUnaryNode *parentNode, MultiBinaryNode *childNode); static PullUpStatus Factorizable(MultiBinaryNode *parentNode, MultiUnaryNode *childNode); static List * SelectClauseTableIdList(List *selectClauseList); static MultiUnaryNode * GenerateLeftNode(MultiUnaryNode *currentNode, MultiBinaryNode *binaryNode); static MultiUnaryNode * GenerateRightNode(MultiUnaryNode *currentNode, MultiBinaryNode *binaryNode); static MultiUnaryNode * GenerateNode(MultiUnaryNode *currentNode, MultiNode *childNode); static List * TableIdListColumns(List *tableIdList, List *columnList); static List * TableIdListSelectClauses(List *tableIdList, List *selectClauseList); static void PushDownBelowUnaryChild(MultiUnaryNode *currentNode, MultiUnaryNode *childNode); static void PlaceUnaryNodeChild(MultiUnaryNode *unaryNode, MultiUnaryNode *childNode); static void PlaceBinaryNodeLeftChild(MultiBinaryNode *binaryNode, MultiUnaryNode *newLeftChildNode); static void PlaceBinaryNodeRightChild(MultiBinaryNode *binaryNode, MultiUnaryNode *newRightChildNode); static void RemoveUnaryNode(MultiUnaryNode *unaryNode); static void PullUpUnaryNode(MultiUnaryNode *unaryNode); static void ParentSetNewChild(MultiNode *parentNode, MultiNode *oldChildNode, MultiNode *newChildNode); /* Local functions forward declarations for aggregate expressions */ static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode, MultiExtendedOp *masterNode, MultiExtendedOp *workerNode); static void TransformSubqueryNode(MultiTable *subqueryNode, bool subqueryHasNonDistributableAggregates); static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode, ExtendedOpNodeProperties * extendedOpNodeProperties); static Node * MasterAggregateMutator(Node *originalNode, MasterAggregateWalkerContext *walkerContext); static Expr * MasterAggregateExpression(Aggref *originalAggregate, MasterAggregateWalkerContext *walkerContext); static Expr * MasterAverageExpression(Oid sumAggregateType, Oid countAggregateType, AttrNumber *columnId); static Expr * AddTypeConversion(Node *originalAggregate, Node *newExpression); static MultiExtendedOp * WorkerExtendedOpNode(MultiExtendedOp *originalOpNode, ExtendedOpNodeProperties * extendedOpNodeProperties); static void ProcessTargetListForWorkerQuery(List *targetEntryList, ExtendedOpNodeProperties * extendedOpNodeProperties, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause); static void ProcessHavingClauseForWorkerQuery(Node *havingQual, ExtendedOpNodeProperties * extendedOpNodeProperties, Node **workerHavingQual, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause); static void ProcessDistinctClauseForWorkerQuery(List *distinctClause, bool hasDistinctOn, List *groupClauseList, bool queryHasAggregates, QueryDistinctClause *queryDistinctClause, bool *distinctPreventsLimitPushdown); static void ProcessWindowFunctionsForWorkerQuery(List *windowClauseList, List *originalTargetEntryList, QueryWindowClause *queryWindowClause, QueryTargetList *queryTargetList); static void ProcessWindowFunctionPullUpForWorkerQuery(List *windowClause, QueryTargetList *queryTargetList); static void ProcessLimitOrderByForWorkerQuery(OrderByLimitReference orderByLimitReference, Node *originalLimitCount, Node *limitOffset, List *sortClauseList, List *groupClauseList, List *originalTargetList, QueryOrderByLimit *queryOrderByLimit, QueryTargetList *queryTargetList); static OrderByLimitReference BuildOrderByLimitReference(bool hasDistinctOn, bool groupedByDisjointPartitionColumn, bool onlyPushableWindowFunctions, List *groupClause, List *sortClauseList, List *targetList); static void ExpandWorkerTargetEntry(List *expressionList, TargetEntry *originalTargetEntry, bool addToGroupByClause, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause); static Index GetNextSortGroupRef(List *targetEntryList); static TargetEntry * GenerateWorkerTargetEntry(TargetEntry *targetEntry, Expr *workerExpression, AttrNumber targetProjectionNumber); static void AppendTargetEntryToGroupClause(TargetEntry *targetEntry, QueryGroupClause *queryGroupClause); static bool WorkerAggregateWalker(Node *node, WorkerAggregateWalkerContext *walkerContext); static List * WorkerAggregateExpressionList(Aggref *originalAggregate, WorkerAggregateWalkerContext * walkerContextry); static AggregateType GetAggregateType(Aggref *aggregatExpression); static Oid AggregateArgumentType(Aggref *aggregate); static Expr * FirstAggregateArgument(Aggref *aggregate); static bool AggregateEnabledCustom(Aggref *aggregateExpression); static Oid CitusFunctionOidWithSignature(char *functionName, int numargs, Oid *argtypes); static Oid WorkerPartialAggOid(void); static Oid WorkerBinaryPartialAggOid(void); static Oid CoordBinaryCombineAggOid(void); static bool IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm); static Oid CoordCombineAggOid(void); static Oid AggregateFunctionOid(const char *functionName, Oid inputType); static Oid TypeOid(Oid schemaId, const char *typeName); static SortGroupClause * CreateSortGroupClause(Var *column); /* Local functions forward declarations for count(distinct) approximations */ static const char * CountDistinctHashFunctionName(Oid argumentType); static int CountDistinctStorageSize(double approximationErrorRate); static Const * MakeIntegerConstInt64(int64 integerValue); static Const * MakeIntegerConst(int32 integerValue); /* Local functions forward declarations for aggregate expression checks */ static bool HasNonDistributableAggregates(MultiNode *logicalPlanNode); static bool CanPushDownExpression(Node *expression, const ExtendedOpNodeProperties * extendedOpNodeProperties); static DeferredErrorMessage * DeferErrorIfHasNonDistributableAggregates(MultiNode * logicalPlanNode); static DeferredErrorMessage * DeferErrorIfUnsupportedArrayAggregate(Aggref * arrayAggregateExpression); static DeferredErrorMessage * DeferErrorIfUnsupportedJsonAggregate(AggregateType type, Aggref * aggregateExpression); static DeferredErrorMessage * DeferErrorIfUnsupportedAggregateDistinct(Aggref * aggregateExpression, MultiNode * logicalPlanNode); static Var * AggregateDistinctColumn(Aggref *aggregateExpression); static bool TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode, Var *distinctColumn, AggregateType aggregateType); /* Local functions forward declarations for limit clauses */ static Node * WorkerLimitCount(Node *limitCount, Node *limitOffset, OrderByLimitReference orderByLimitReference); static List * WorkerSortClauseList(Node *limitCount, List *groupClauseList, List *sortClauseList, OrderByLimitReference orderByLimitReference); static bool CanPushDownLimitApproximate(List *sortClauseList, List *targetList); static bool HasOrderByAggregate(List *sortClauseList, List *targetList); static bool HasOrderByNonCommutativeAggregate(List *sortClauseList, List *targetList); static bool HasOrderByComplexExpression(List *sortClauseList, List *targetList); static bool HasOrderByHllType(List *sortClauseList, List *targetList); static bool ShouldProcessDistinctOrderAndLimitForWorker(ExtendedOpNodeProperties * extendedOpNodeProperties, bool pushingDownOriginalGrouping, Node *havingQual); static bool IsIndexInRange(const List *list, int index); /* * MultiLogicalPlanOptimize applies multi-relational algebra optimizations on * the given logical plan tree. Specifically, the function applies four set of * optimizations in a particular order. * * First, the function splits the search node into two nodes that contain And * and Or clauses, and pushes down the node that contains And clauses. Second, * the function pushes down the project node; this node either contains columns * to return to the user, or aggregate expressions used by the aggregate node. * Third, the function pulls up the collect operators in the tree. Fourth, the * function finds the extended operator node, and splits this node into master * and worker extended operator nodes. */ void MultiLogicalPlanOptimize(MultiTreeRoot *multiLogicalPlan) { MultiNode *logicalPlanNode = (MultiNode *) multiLogicalPlan; bool hasNonDistributableAggregates = HasNonDistributableAggregates( logicalPlanNode); List *extendedOpNodeList = FindNodesOfType(logicalPlanNode, T_MultiExtendedOp); MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) linitial(extendedOpNodeList); ExtendedOpNodeProperties extendedOpNodeProperties = BuildExtendedOpNodeProperties( extendedOpNode, hasNonDistributableAggregates); if (!extendedOpNodeProperties.groupedByDisjointPartitionColumn && !extendedOpNodeProperties.pullUpIntermediateRows) { DeferredErrorMessage *aggregatePushdownError = DeferErrorIfHasNonDistributableAggregates(logicalPlanNode); if (aggregatePushdownError != NULL) { if (CoordinatorAggregationStrategy == COORDINATOR_AGGREGATION_DISABLED) { RaiseDeferredError(aggregatePushdownError, ERROR); } else { extendedOpNodeProperties.pullUpIntermediateRows = true; extendedOpNodeProperties.pushDownGroupingAndHaving = false; } } } /* * If a select node exists, we use the idempower property to split the node * into two nodes that contain And and Or clauses. If both And and Or nodes * exist, we modify the tree in place to swap the original select node with * And and Or nodes. We then push down the And select node if it exists. */ List *selectNodeList = FindNodesOfType(logicalPlanNode, T_MultiSelect); if (selectNodeList != NIL) { MultiSelect *selectNode = (MultiSelect *) linitial(selectNodeList); MultiSelect *andSelectNode = AndSelectNode(selectNode); MultiSelect *orSelectNode = OrSelectNode(selectNode); if (andSelectNode != NULL && orSelectNode != NULL) { MultiNode *parentNode = ParentNode((MultiNode *) selectNode); MultiNode *childNode = ChildNode((MultiUnaryNode *) selectNode); Assert(UnaryOperator(parentNode)); SetChild((MultiUnaryNode *) parentNode, (MultiNode *) orSelectNode); SetChild((MultiUnaryNode *) orSelectNode, (MultiNode *) andSelectNode); SetChild((MultiUnaryNode *) andSelectNode, (MultiNode *) childNode); } else if (andSelectNode != NULL && orSelectNode == NULL) { andSelectNode = selectNode; /* no need to modify the tree */ } if (andSelectNode != NULL) { PushDownNodeLoop((MultiUnaryNode *) andSelectNode); } } /* push down the multi project node */ List *projectNodeList = FindNodesOfType(logicalPlanNode, T_MultiProject); MultiProject *projectNode = (MultiProject *) linitial(projectNodeList); PushDownNodeLoop((MultiUnaryNode *) projectNode); /* pull up collect nodes and merge duplicate collects */ List *collectNodeList = FindNodesOfType(logicalPlanNode, T_MultiCollect); MultiCollect *collectNode = NULL; foreach_declared_ptr(collectNode, collectNodeList) { PullUpCollectLoop(collectNode); } /* * We split the extended operator node into its equivalent master and worker * operator nodes; and if the extended operator has aggregates, we transform * aggregate functions accordingly for the master and worker operator nodes. * If we can push down the limit clause, we also add limit count and sort * clause list to the worker operator node. We then push the worker operator * node below the collect node. */ MultiExtendedOp *masterExtendedOpNode = MasterExtendedOpNode(extendedOpNode, &extendedOpNodeProperties); MultiExtendedOp *workerExtendedOpNode = WorkerExtendedOpNode(extendedOpNode, &extendedOpNodeProperties); ApplyExtendedOpNodes(extendedOpNode, masterExtendedOpNode, workerExtendedOpNode); List *tableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); MultiTable *tableNode = NULL; foreach_declared_ptr(tableNode, tableNodeList) { if (tableNode->relationId == SUBQUERY_RELATION_ID) { DeferredErrorMessage *error = DeferErrorIfHasNonDistributableAggregates((MultiNode *) tableNode); bool subqueryHasNonDistributableAggregates = false; if (error != NULL) { if (CoordinatorAggregationStrategy == COORDINATOR_AGGREGATION_DISABLED) { RaiseDeferredError(error, ERROR); } else { subqueryHasNonDistributableAggregates = true; } } TransformSubqueryNode(tableNode, subqueryHasNonDistributableAggregates); } } /* * When enabled, count(distinct) approximation uses hll as the intermediate * data type. We currently have a mismatch between hll target entry and sort * clause's sortop oid, so we can't push an order by on the hll data type to * the worker node. We check that here and error out if necessary. */ bool hasOrderByHllType = HasOrderByHllType(workerExtendedOpNode->sortClauseList, workerExtendedOpNode->targetList); if (hasOrderByHllType) { ereport(ERROR, (errmsg("cannot approximate count(distinct) and order by it"), errhint("You might need to disable approximations for either " "count(distinct) or limit through configuration."))); } if (TargetListContainsSubquery(masterExtendedOpNode->targetList)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot push down subquery on the target list"), errdetail("Subqueries in the SELECT part of the query can only " "be pushed down if they happen before aggregates and " "window functions"))); } } /* * AndSelectNode looks for AND clauses in the given select node. If they exist, * the function returns these clauses in a new node. Otherwise, the function * returns null. */ static MultiSelect * AndSelectNode(MultiSelect *selectNode) { MultiSelect *andSelectNode = NULL; List *selectClauseList = selectNode->selectClauseList; List *orSelectClauseList = OrSelectClauseList(selectClauseList); /* AND clauses are select clauses that are not OR clauses */ List *andSelectClauseList = list_difference(selectClauseList, orSelectClauseList); if (andSelectClauseList != NIL) { andSelectNode = CitusMakeNode(MultiSelect); andSelectNode->selectClauseList = andSelectClauseList; } return andSelectNode; } /* * OrSelectNode looks for OR clauses in the given select node. If they exist, * the function returns these clauses in a new node. Otherwise, the function * returns null. */ static MultiSelect * OrSelectNode(MultiSelect *selectNode) { MultiSelect *orSelectNode = NULL; List *selectClauseList = selectNode->selectClauseList; List *orSelectClauseList = OrSelectClauseList(selectClauseList); if (orSelectClauseList != NIL) { orSelectNode = CitusMakeNode(MultiSelect); orSelectNode->selectClauseList = orSelectClauseList; } return orSelectNode; } /* * OrSelectClauseList walks over the select clause list, and returns all clauses * that have OR expressions in them. */ static List * OrSelectClauseList(List *selectClauseList) { List *orSelectClauseList = NIL; Node *selectClause = NULL; foreach_declared_ptr(selectClause, selectClauseList) { bool orClause = is_orclause(selectClause); if (orClause) { orSelectClauseList = lappend(orSelectClauseList, selectClause); } } return orSelectClauseList; } /* * PushDownNodeLoop pushes down the current node as far down the plan tree as * possible. For this, the function first addresses any special conditions that * may apply on the current node. Then, the function pushes down the current * node if its child node is unary. If the child is binary, the function splits * the current node into two nodes by applying generation rules, and recurses * into itself to push down these two nodes. */ static void PushDownNodeLoop(MultiUnaryNode *currentNode) { MultiUnaryNode *projectNodeGenerated = NULL; MultiUnaryNode *leftNodeGenerated = NULL; MultiUnaryNode *rightNodeGenerated = NULL; PushDownStatus pushDownStatus = CanPushDown(currentNode); while (pushDownStatus == PUSH_DOWN_VALID || pushDownStatus == PUSH_DOWN_SPECIAL_CONDITIONS) { MultiNode *childNode = currentNode->childNode; bool unaryChild = UnaryOperator(childNode); bool binaryChild = BinaryOperator(childNode); /* * We first check if we can use the idempower property to split the * project node. We split at a partition node as it captures the * minimal set of columns needed from a partition job. After the split * we break from the loop and recursively call pushdown for the * generated project node. */ MultiNode *parentNode = ParentNode((MultiNode *) currentNode); CitusNodeTag currentNodeType = CitusNodeTag(currentNode); CitusNodeTag parentNodeType = CitusNodeTag(parentNode); if (currentNodeType == T_MultiProject && parentNodeType == T_MultiPartition) { projectNodeGenerated = GenerateNode(currentNode, childNode); PlaceUnaryNodeChild(currentNode, projectNodeGenerated); break; } /* address any special conditions before we can perform the pushdown */ if (pushDownStatus == PUSH_DOWN_SPECIAL_CONDITIONS) { MultiProject *projectNode = (MultiProject *) currentNode; Assert(currentNodeType == T_MultiProject); AddressProjectSpecialConditions(projectNode); } if (unaryChild) { MultiUnaryNode *unaryChildNode = (MultiUnaryNode *) childNode; PushDownBelowUnaryChild(currentNode, unaryChildNode); } else if (binaryChild) { MultiBinaryNode *binaryChildNode = (MultiBinaryNode *) childNode; leftNodeGenerated = GenerateLeftNode(currentNode, binaryChildNode); rightNodeGenerated = GenerateRightNode(currentNode, binaryChildNode); /* push down the generated nodes below the binary child node */ PlaceBinaryNodeLeftChild(binaryChildNode, leftNodeGenerated); PlaceBinaryNodeRightChild(binaryChildNode, rightNodeGenerated); /* * Remove the current node, and break out of the push down loop for * the current node. Then, recurse into the push down function for * the newly generated nodes. */ RemoveUnaryNode(currentNode); break; } pushDownStatus = CanPushDown(currentNode); } /* recursively perform pushdown of any nodes generated in the loop */ if (projectNodeGenerated != NULL) { PushDownNodeLoop(projectNodeGenerated); } if (leftNodeGenerated != NULL) { PushDownNodeLoop(leftNodeGenerated); } if (rightNodeGenerated != NULL) { PushDownNodeLoop(rightNodeGenerated); } } /* * PullUpCollectLoop pulls up the collect node as far up as possible in the plan * tree. The function also merges two collect nodes that are direct descendants * of each other by removing the given collect node from the tree. */ static void PullUpCollectLoop(MultiCollect *collectNode) { MultiUnaryNode *currentNode = (MultiUnaryNode *) collectNode; PullUpStatus pullUpStatus = CanPullUp(currentNode); while (pullUpStatus == PULL_UP_VALID) { PullUpUnaryNode(currentNode); pullUpStatus = CanPullUp(currentNode); } /* * After pulling up the collect node, if we find that our child node is also * a collect, we merge the two collect nodes together by removing this node. */ MultiNode *childNode = currentNode->childNode; if (CitusIsA(childNode, MultiCollect)) { RemoveUnaryNode(currentNode); } } /* * AddressProjectSpecialConditions adds columns to the project node if necessary * to make the node commutative and distributive with its child node. For this, * the function checks for any special conditions between the project and child * node, and determines the child node columns to add for the special conditions * to apply. The function then adds these columns to the project node. */ static void AddressProjectSpecialConditions(MultiProject *projectNode) { MultiNode *childNode = ChildNode((MultiUnaryNode *) projectNode); CitusNodeTag childNodeTag = CitusNodeTag(childNode); List *childColumnList = NIL; /* * We check if we need to include any child columns in the project node to * address the following special conditions. * * SNC1: project node must include child node's projected columns, or * SNC2: project node must include child node's partition column, or * SNC3: project node must include child node's selection columns, or * NSC1: project node must include child node's join columns. */ if (childNodeTag == T_MultiProject) { MultiProject *projectChildNode = (MultiProject *) childNode; List *projectColumnList = projectChildNode->columnList; childColumnList = copyObject(projectColumnList); } else if (childNodeTag == T_MultiPartition) { MultiPartition *partitionNode = (MultiPartition *) childNode; Var *partitionColumn = partitionNode->partitionColumn; List *partitionColumnList = list_make1(partitionColumn); childColumnList = copyObject(partitionColumnList); } else if (childNodeTag == T_MultiSelect) { MultiSelect *selectNode = (MultiSelect *) childNode; Node *selectClauseList = (Node *) selectNode->selectClauseList; List *selectList = pull_var_clause_default(selectClauseList); childColumnList = copyObject(selectList); } else if (childNodeTag == T_MultiJoin) { MultiJoin *joinNode = (MultiJoin *) childNode; Node *joinClauseList = (Node *) joinNode->joinClauseList; List *joinList = pull_var_clause_default(joinClauseList); childColumnList = copyObject(joinList); } /* * If we need to include any child columns, then find the columns that are * not already in the project column list, and add them. */ if (childColumnList != NIL) { List *projectColumnList = projectNode->columnList; List *newColumnList = list_concat_unique(projectColumnList, childColumnList); projectNode->columnList = newColumnList; } } /* * CanPushDown determines if a particular node can be moved below its child. The * criteria for pushing down a node is determined by multi-relational algebra's * rules for commutativity and distributivity. */ static PushDownStatus CanPushDown(MultiUnaryNode *parentNode) { PushDownStatus pushDownStatus = PUSH_DOWN_INVALID_FIRST; MultiNode *childNode = parentNode->childNode; bool unaryChild = UnaryOperator(childNode); bool binaryChild = BinaryOperator(childNode); if (unaryChild) { pushDownStatus = Commutative(parentNode, (MultiUnaryNode *) childNode); } else if (binaryChild) { pushDownStatus = Distributive(parentNode, (MultiBinaryNode *) childNode); } Assert(pushDownStatus != PUSH_DOWN_INVALID_FIRST); return pushDownStatus; } /* * CanPullUp determines if a particular node can be moved above its parent. The * criteria for pulling up a node is determined by multi-relational algebra's * rules for commutativity and factorizability. */ static PullUpStatus CanPullUp(MultiUnaryNode *childNode) { PullUpStatus pullUpStatus = PULL_UP_INVALID_FIRST; MultiNode *parentNode = ParentNode((MultiNode *) childNode); bool unaryParent = UnaryOperator(parentNode); bool binaryParent = BinaryOperator(parentNode); if (unaryParent) { /* * Evaluate if parent can be pushed down below the child node, since it * is equivalent to pulling up the child above its parent. */ PushDownStatus parentPushDownStatus = Commutative((MultiUnaryNode *) parentNode, childNode); if (parentPushDownStatus == PUSH_DOWN_VALID) { pullUpStatus = PULL_UP_VALID; } else { pullUpStatus = PULL_UP_NOT_VALID; } } else if (binaryParent) { pullUpStatus = Factorizable((MultiBinaryNode *) parentNode, childNode); } Assert(pullUpStatus != PULL_UP_INVALID_FIRST); return pullUpStatus; } /* * Commutative returns a status which denotes whether the given parent node can * be pushed down below its child node using the commutative property. */ static PushDownStatus Commutative(MultiUnaryNode *parentNode, MultiUnaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* we cannot be commutative with non-query operators */ if (childNodeTag == T_MultiTreeRoot || childNodeTag == T_MultiTable) { return PUSH_DOWN_NOT_VALID; } /* first check for commutative operators and no special conditions */ if ((parentNodeTag == T_MultiPartition && childNodeTag == T_MultiProject) || (parentNodeTag == T_MultiPartition && childNodeTag == T_MultiPartition) || (parentNodeTag == T_MultiPartition && childNodeTag == T_MultiSelect)) { pushDownStatus = PUSH_DOWN_VALID; } if ((parentNodeTag == T_MultiCollect && childNodeTag == T_MultiProject) || (parentNodeTag == T_MultiCollect && childNodeTag == T_MultiCollect) || (parentNodeTag == T_MultiCollect && childNodeTag == T_MultiSelect)) { pushDownStatus = PUSH_DOWN_VALID; } if (parentNodeTag == T_MultiSelect) { pushDownStatus = PUSH_DOWN_VALID; } if (parentNodeTag == T_MultiProject && childNodeTag == T_MultiCollect) { pushDownStatus = PUSH_DOWN_VALID; } /* * The project node is commutative with the below operators given that * its special conditions apply. */ if ((parentNodeTag == T_MultiProject && childNodeTag == T_MultiProject) || (parentNodeTag == T_MultiProject && childNodeTag == T_MultiPartition) || (parentNodeTag == T_MultiProject && childNodeTag == T_MultiSelect) || (parentNodeTag == T_MultiProject && childNodeTag == T_MultiJoin)) { pushDownStatus = PUSH_DOWN_SPECIAL_CONDITIONS; } return pushDownStatus; } /* * Distributive returns a status which denotes whether the given parent node can * be pushed down below its binary child node using the distributive property. */ static PushDownStatus Distributive(MultiUnaryNode *parentNode, MultiBinaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* special condition checks for partition operator are not implemented */ Assert(parentNodeTag != T_MultiPartition); /* * The project node is distributive with the join operator given that its * special conditions apply. */ if (parentNodeTag == T_MultiProject) { pushDownStatus = PUSH_DOWN_SPECIAL_CONDITIONS; } /* collect node is distributive without special conditions */ if ((parentNodeTag == T_MultiCollect && childNodeTag == T_MultiJoin) || (parentNodeTag == T_MultiCollect && childNodeTag == T_MultiCartesianProduct)) { pushDownStatus = PUSH_DOWN_VALID; } /* * The select node is distributive with a binary operator if all tables in * the select clauses are output by the binary child. The select clauses are * individually AND'd; and therefore this check is sufficient to implement * the NSC3 special condition in multi-relational algebra. */ if ((parentNodeTag == T_MultiSelect && childNodeTag == T_MultiJoin) || (parentNodeTag == T_MultiSelect && childNodeTag == T_MultiCartesianProduct)) { MultiSelect *selectNode = (MultiSelect *) parentNode; List *selectClauseList = selectNode->selectClauseList; List *selectTableIdList = SelectClauseTableIdList(selectClauseList); List *childTableIdList = OutputTableIdList((MultiNode *) childNode); /* find tables that are in select clause list, but not in child list */ List *diffList = list_difference_int(selectTableIdList, childTableIdList); if (diffList == NIL) { pushDownStatus = PUSH_DOWN_VALID; } } return pushDownStatus; } /* * Factorizable returns a status which denotes whether the given unary child * node can be pulled up above its binary parent node using the factorizability * property. The function currently performs this check only for collect node * types; other node types have generation rules that are not yet implemented. */ static PullUpStatus Factorizable(MultiBinaryNode *parentNode, MultiUnaryNode *childNode) { PullUpStatus pullUpStatus = PULL_UP_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* * The following nodes are factorizable with their parents, but we don't * have their generation rules implemented. We therefore assert here. */ Assert(childNodeTag != T_MultiProject); Assert(childNodeTag != T_MultiPartition); Assert(childNodeTag != T_MultiSelect); if ((childNodeTag == T_MultiCollect && parentNodeTag == T_MultiJoin) || (childNodeTag == T_MultiCollect && parentNodeTag == T_MultiCartesianProduct)) { pullUpStatus = PULL_UP_VALID; } return pullUpStatus; } /* * SelectClauseTableIdList finds the (range) table identifier for each select * clause in the given list, and returns these identifiers in a new list. */ static List * SelectClauseTableIdList(List *selectClauseList) { List *tableIdList = NIL; Node *selectClause = NULL; foreach_declared_ptr(selectClause, selectClauseList) { List *selectColumnList = pull_var_clause_default(selectClause); if (list_length(selectColumnList) == 0) { /* filter is a constant, e.g. false or 1=0 */ continue; } Var *selectColumn = (Var *) linitial(selectColumnList); int selectColumnTableId = (int) selectColumn->varno; tableIdList = lappend_int(tableIdList, selectColumnTableId); } return tableIdList; } /* * GenerateLeftNode splits the current node over the binary node by applying the * generation rule for distributivity in multi-relational algebra. After the * split, the function returns the left node. */ static MultiUnaryNode * GenerateLeftNode(MultiUnaryNode *currentNode, MultiBinaryNode *binaryNode) { MultiNode *leftChildNode = binaryNode->leftChildNode; MultiUnaryNode *leftNodeGenerated = GenerateNode(currentNode, leftChildNode); return leftNodeGenerated; } /* * GenerateRightNode splits the current node over the binary node by applying * the generation rule for distributivity in multi-relational algebra. After the * split, the function returns the right node. */ static MultiUnaryNode * GenerateRightNode(MultiUnaryNode *currentNode, MultiBinaryNode *binaryNode) { MultiNode *rightChildNode = binaryNode->rightChildNode; MultiUnaryNode *rightNodeGenerated = GenerateNode(currentNode, rightChildNode); return rightNodeGenerated; } /* * GenerateNode determines the current node's type, and applies the relevant * generation node for that node type. If the current node is a project node, * the function creates a new project node with attributes that only have the * child subtree's tables. Else if the current node is a select node, the * function creates a new select node with select clauses that only belong to * the tables output by the child node's subtree. */ static MultiUnaryNode * GenerateNode(MultiUnaryNode *currentNode, MultiNode *childNode) { MultiUnaryNode *generatedNode = NULL; CitusNodeTag currentNodeType = CitusNodeTag(currentNode); List *tableIdList = OutputTableIdList(childNode); if (currentNodeType == T_MultiProject) { MultiProject *projectNode = (MultiProject *) currentNode; List *columnList = copyObject(projectNode->columnList); List *newColumnList = TableIdListColumns(tableIdList, columnList); if (newColumnList != NIL) { MultiProject *newProjectNode = CitusMakeNode(MultiProject); newProjectNode->columnList = newColumnList; generatedNode = (MultiUnaryNode *) newProjectNode; } } else if (currentNodeType == T_MultiSelect) { MultiSelect *selectNode = (MultiSelect *) currentNode; List *selectClauseList = copyObject(selectNode->selectClauseList); List *newSelectClauseList = TableIdListSelectClauses(tableIdList, selectClauseList); if (newSelectClauseList != NIL) { MultiSelect *newSelectNode = CitusMakeNode(MultiSelect); newSelectNode->selectClauseList = newSelectClauseList; generatedNode = (MultiUnaryNode *) newSelectNode; } } return generatedNode; } /* * TableIdListColumns walks over the given column list, finds columns belonging * to the given table id list, and returns the found columns in a new list. */ static List * TableIdListColumns(List *tableIdList, List *columnList) { List *tableColumnList = NIL; Var *column = NULL; foreach_declared_ptr(column, columnList) { int columnTableId = (int) column->varno; bool tableListMember = list_member_int(tableIdList, columnTableId); if (tableListMember) { tableColumnList = lappend(tableColumnList, column); } } return tableColumnList; } /* * TableIdListSelectClauses walks over the given select clause list, finds the * select clauses whose column references belong to the given table list, and * returns the found clauses in a new list. */ static List * TableIdListSelectClauses(List *tableIdList, List *selectClauseList) { List *tableSelectClauseList = NIL; Node *selectClause = NULL; foreach_declared_ptr(selectClause, selectClauseList) { List *selectColumnList = pull_var_clause_default(selectClause); if (list_length(selectColumnList) == 0) { /* filter is a constant, e.g. false or 1=0, always include it */ tableSelectClauseList = lappend(tableSelectClauseList, selectClause); } else { Var *selectColumn = (Var *) linitial(selectColumnList); int selectClauseTableId = (int) selectColumn->varno; bool tableIdListMember = list_member_int(tableIdList, selectClauseTableId); if (tableIdListMember) { tableSelectClauseList = lappend(tableSelectClauseList, selectClause); } } } return tableSelectClauseList; } /* Pushes down the current node below its unary child node. */ static void PushDownBelowUnaryChild(MultiUnaryNode *currentNode, MultiUnaryNode *childNode) { MultiNode *parentNode = ParentNode((MultiNode *) currentNode); MultiNode *childChildNode = ChildNode(childNode); /* current node's parent now points to the child node */ ParentSetNewChild(parentNode, (MultiNode *) currentNode, (MultiNode *) childNode); /* current node's child becomes its parent */ SetChild(childNode, (MultiNode *) currentNode); /* current node points to the child node's child */ SetChild(currentNode, childChildNode); } /* * PlaceUnaryNodeChild inserts the new node as a child node under the given * unary node. The function also places the previous child node under the new * child node. */ static void PlaceUnaryNodeChild(MultiUnaryNode *unaryNode, MultiUnaryNode *newChildNode) { MultiNode *oldChildNode = ChildNode(unaryNode); SetChild(unaryNode, (MultiNode *) newChildNode); SetChild(newChildNode, oldChildNode); } /* * PlaceBinaryNodeLeftChild inserts the new left child as the binary node's left * child. The function also places the previous left child below the new child * node. */ static void PlaceBinaryNodeLeftChild(MultiBinaryNode *binaryNode, MultiUnaryNode *newLeftChildNode) { if (newLeftChildNode == NULL) { return; } SetChild(newLeftChildNode, binaryNode->leftChildNode); SetLeftChild(binaryNode, (MultiNode *) newLeftChildNode); } /* * PlaceBinaryNodeRightChild inserts the new right child as the binary node's * right child. The function also places the previous right child below the new * child node. */ static void PlaceBinaryNodeRightChild(MultiBinaryNode *binaryNode, MultiUnaryNode *newRightChildNode) { if (newRightChildNode == NULL) { return; } SetChild(newRightChildNode, binaryNode->rightChildNode); SetRightChild(binaryNode, (MultiNode *) newRightChildNode); } /* Removes the given unary node from the logical plan, and frees the node. */ static void RemoveUnaryNode(MultiUnaryNode *unaryNode) { MultiNode *parentNode = ParentNode((MultiNode *) unaryNode); MultiNode *childNode = ChildNode(unaryNode); /* set parent to directly point to unary node's child */ ParentSetNewChild(parentNode, (MultiNode *) unaryNode, childNode); pfree(unaryNode); } /* Pulls up the given current node above its parent node. */ static void PullUpUnaryNode(MultiUnaryNode *unaryNode) { MultiNode *parentNode = ParentNode((MultiNode *) unaryNode); bool unaryParent = UnaryOperator(parentNode); bool binaryParent = BinaryOperator(parentNode); if (unaryParent) { /* pulling up a node is the same as pushing down the node's unary parent */ MultiUnaryNode *unaryParentNode = (MultiUnaryNode *) parentNode; PushDownBelowUnaryChild(unaryParentNode, unaryNode); } else if (binaryParent) { MultiBinaryNode *binaryParentNode = (MultiBinaryNode *) parentNode; MultiNode *parentParentNode = ParentNode((MultiNode *) binaryParentNode); MultiNode *childNode = unaryNode->childNode; /* make the parent node point to the unary node's child node */ if (binaryParentNode->leftChildNode == ((MultiNode *) unaryNode)) { SetLeftChild(binaryParentNode, childNode); } else { SetRightChild(binaryParentNode, childNode); } /* make the parent parent node point to the unary node */ ParentSetNewChild(parentParentNode, parentNode, (MultiNode *) unaryNode); /* make the unary node point to the (old) parent node */ SetChild(unaryNode, parentNode); } } /* * ParentSetNewChild takes in the given parent node, and replaces the parent's * old child node with the new child node. The function needs the old child node * in case the parent is a binary node and the function needs to determine which * side of the parent node the new child node needs to go to. */ static void ParentSetNewChild(MultiNode *parentNode, MultiNode *oldChildNode, MultiNode *newChildNode) { bool unaryParent = UnaryOperator(parentNode); bool binaryParent = BinaryOperator(parentNode); if (unaryParent) { MultiUnaryNode *unaryParentNode = (MultiUnaryNode *) parentNode; SetChild(unaryParentNode, newChildNode); } else if (binaryParent) { MultiBinaryNode *binaryParentNode = (MultiBinaryNode *) parentNode; /* determine which side of the parent the old child is on */ if (binaryParentNode->leftChildNode == oldChildNode) { SetLeftChild(binaryParentNode, newChildNode); } else { SetRightChild(binaryParentNode, newChildNode); } } } /* * ApplyExtendedOpNodes replaces the original extended operator node with the * master and worker extended operator nodes. The function then pushes down the * worker node below the original node's child node. Note that for the push down * to apply, the original node's child must be a collect node. */ static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode, MultiExtendedOp *masterNode, MultiExtendedOp *workerNode) { MultiNode *parentNode = ParentNode((MultiNode *) originalNode); MultiNode *collectNode = ChildNode((MultiUnaryNode *) originalNode); MultiNode *collectChildNode = ChildNode((MultiUnaryNode *) collectNode); /* original node's child must be a collect node */ Assert(CitusIsA(collectNode, MultiCollect)); Assert(UnaryOperator(parentNode)); /* swap the original aggregate node with the master extended node */ SetChild((MultiUnaryNode *) parentNode, (MultiNode *) masterNode); SetChild((MultiUnaryNode *) masterNode, (MultiNode *) collectNode); /* add the worker extended node below the collect node */ SetChild((MultiUnaryNode *) collectNode, (MultiNode *) workerNode); SetChild((MultiUnaryNode *) workerNode, (MultiNode *) collectChildNode); /* clean up the original extended operator node */ pfree(originalNode); } /* * TransformSubqueryNode splits the extended operator node under subquery * multi table node into its equivalent master and worker operator nodes, and * we transform aggregate functions accordingly for the master and worker * operator nodes. We create a partition node based on the first group by * column of the extended operator node and set it as the child of the master * operator node. */ static void TransformSubqueryNode(MultiTable *subqueryNode, bool subqueryHasNonDistributableAggregates) { if (CoordinatorAggregationStrategy != COORDINATOR_AGGREGATION_DISABLED && HasNonDistributableAggregates((MultiNode *) subqueryNode)) { subqueryHasNonDistributableAggregates = true; } MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) ChildNode((MultiUnaryNode *) subqueryNode); MultiNode *collectNode = ChildNode((MultiUnaryNode *) extendedOpNode); MultiNode *collectChildNode = ChildNode((MultiUnaryNode *) collectNode); ExtendedOpNodeProperties extendedOpNodeProperties = BuildExtendedOpNodeProperties(extendedOpNode, subqueryHasNonDistributableAggregates); MultiExtendedOp *masterExtendedOpNode = MasterExtendedOpNode(extendedOpNode, &extendedOpNodeProperties); MultiExtendedOp *workerExtendedOpNode = WorkerExtendedOpNode(extendedOpNode, &extendedOpNodeProperties); List *groupClauseList = extendedOpNode->groupClauseList; List *targetEntryList = extendedOpNode->targetList; List *groupTargetEntryList = GroupTargetEntryList(groupClauseList, targetEntryList); TargetEntry *groupByTargetEntry = (TargetEntry *) linitial(groupTargetEntryList); Expr *groupByExpression = groupByTargetEntry->expr; MultiPartition *partitionNode = CitusMakeNode(MultiPartition); /* * If group by is on a function expression, then we create a new column from * function expression result type. Because later while creating partition * tasks, we expect a column type to partition intermediate results. * Note that we will only need partition type. So we set column type to * result type of the function expression, and set other fields of column to * default values. */ if (IsA(groupByExpression, Var)) { partitionNode->partitionColumn = (Var *) groupByExpression; } else if (IsA(groupByExpression, FuncExpr)) { FuncExpr *functionExpression = (FuncExpr *) groupByExpression; Index tableId = 0; AttrNumber columnAttributeNumber = InvalidAttrNumber; Oid columnType = functionExpression->funcresulttype; int32 columnTypeMod = -1; Oid columnCollationOid = InvalidOid; Index columnLevelSup = 0; Var *partitionColumn = makeVar(tableId, columnAttributeNumber, columnType, columnTypeMod, columnCollationOid, columnLevelSup); partitionNode->partitionColumn = partitionColumn; } else { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot run this subquery"), errdetail("Currently only columns and function expressions " "are allowed in group by expression of subqueries"))); } SetChild((MultiUnaryNode *) subqueryNode, (MultiNode *) masterExtendedOpNode); SetChild((MultiUnaryNode *) masterExtendedOpNode, (MultiNode *) partitionNode); SetChild((MultiUnaryNode *) partitionNode, (MultiNode *) collectNode); SetChild((MultiUnaryNode *) collectNode, (MultiNode *) workerExtendedOpNode); SetChild((MultiUnaryNode *) workerExtendedOpNode, (MultiNode *) collectChildNode); } /* * MasterExtendedOpNode creates the master extended operator node from the given * target entries. The function walks over these target entries; and for entries * with aggregates in them, this function calls the aggregate expression mutator * function. * * Note that the function logically depends on the worker extended operator node * function. If the target entry does not contain aggregate functions, we assume * all work is done on the worker side, and create a column that references the * worker nodes' results. */ static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode, ExtendedOpNodeProperties *extendedOpNodeProperties) { List *targetEntryList = originalOpNode->targetList; List *newTargetEntryList = NIL; List *newGroupClauseList = NIL; Node *originalHavingQual = originalOpNode->havingQual; Node *newHavingQual = NULL; MasterAggregateWalkerContext walkerContext = { .extendedOpNodeProperties = extendedOpNodeProperties, .columnId = 1, }; /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; foreach_declared_ptr(originalTargetEntry, targetEntryList) { TargetEntry *newTargetEntry = flatCopyTargetEntry(originalTargetEntry); Expr *originalExpression = originalTargetEntry->expr; Expr *newExpression = NULL; if (CanPushDownExpression((Node *) originalExpression, extendedOpNodeProperties)) { /* * The expression was entirely pushed down to worker. * We simply make it reference the output generated by worker nodes. */ Var *column = makeVarFromTargetEntry(masterTableId, originalTargetEntry); column->varattno = walkerContext.columnId; column->varattnosyn = walkerContext.columnId; walkerContext.columnId++; if (column->vartype == RECORDOID || column->vartype == RECORDARRAYOID) { column->vartypmod = BlessRecordExpression(originalTargetEntry->expr); } newExpression = (Expr *) column; } else { Node *newNode = MasterAggregateMutator((Node *) originalExpression, &walkerContext); newExpression = (Expr *) newNode; } newTargetEntry->expr = newExpression; newTargetEntryList = lappend(newTargetEntryList, newTargetEntry); } if (!extendedOpNodeProperties->pushDownGroupingAndHaving) { /* * Not pushing down GROUP BY, need to regroup on coordinator * and apply having on the coordinator. */ newGroupClauseList = originalOpNode->groupClauseList; if (originalHavingQual != NULL) { newHavingQual = MasterAggregateMutator(originalHavingQual, &walkerContext); if (IsA(newHavingQual, List)) { /* * unflatten having qual to allow standard planner to work when transforming * the master query to a plan */ newHavingQual = (Node *) make_ands_explicit( castNode(List, newHavingQual)); } } } MultiExtendedOp *masterExtendedOpNode = CitusMakeNode(MultiExtendedOp); masterExtendedOpNode->targetList = newTargetEntryList; masterExtendedOpNode->groupClauseList = newGroupClauseList; masterExtendedOpNode->sortClauseList = originalOpNode->sortClauseList; masterExtendedOpNode->distinctClause = originalOpNode->distinctClause; masterExtendedOpNode->hasDistinctOn = originalOpNode->hasDistinctOn; masterExtendedOpNode->limitCount = originalOpNode->limitCount; masterExtendedOpNode->limitOffset = originalOpNode->limitOffset; masterExtendedOpNode->limitOption = originalOpNode->limitOption; masterExtendedOpNode->havingQual = newHavingQual; if (!extendedOpNodeProperties->onlyPushableWindowFunctions) { masterExtendedOpNode->hasWindowFuncs = originalOpNode->hasWindowFuncs; masterExtendedOpNode->windowClause = originalOpNode->windowClause; masterExtendedOpNode->onlyPushableWindowFunctions = false; } return masterExtendedOpNode; } /* * MasterAggregateMutator walks over the original target entry expression, and * creates the new expression tree to execute on the master node. The function * transforms aggregates, and copies columns; and recurses into the expression * mutator function for all other expression types. * * Please note that the recursive mutator function traverses the expression tree * in depth first order. For this function to set attribute numbers correctly, * WorkerAggregateWalker() *must* walk over the expression tree in the same * depth first order. */ static Node * MasterAggregateMutator(Node *originalNode, MasterAggregateWalkerContext *walkerContext) { Node *newNode = NULL; if (originalNode == NULL) { return NULL; } if (IsA(originalNode, Aggref)) { Aggref *originalAggregate = (Aggref *) originalNode; if (CanPushDownExpression(originalNode, walkerContext->extendedOpNodeProperties)) { /* * The expression was entirely pushed down to worker. * We simply make it reference the output generated by worker nodes. */ Var *column = makeVar(masterTableId, walkerContext->columnId, originalAggregate->aggtype, -1, originalAggregate->aggcollid, 0); walkerContext->columnId++; if (column->vartype == RECORDOID || column->vartype == RECORDARRAYOID) { column->vartypmod = BlessRecordExpression((Expr *) originalNode); } newNode = (Node *) column; } else { Expr *newExpression = MasterAggregateExpression(originalAggregate, walkerContext); newNode = (Node *) newExpression; } } else if (IsA(originalNode, Var)) { Var *origColumn = (Var *) originalNode; Var *newColumn = makeVar(masterTableId, walkerContext->columnId, origColumn->vartype, origColumn->vartypmod, origColumn->varcollid, origColumn->varlevelsup); walkerContext->columnId++; newNode = (Node *) newColumn; } else { newNode = expression_tree_mutator(originalNode, MasterAggregateMutator, (void *) walkerContext); } return newNode; } /* * MasterAggregateExpression creates the master aggregate expression using the * original aggregate and aggregate's type information. This function handles * the average, count, array_agg, hll and topn aggregates separately due to * differences in these aggregate functions' transformations. * * Note that this function has implicit knowledge of the transformations applied * for worker nodes on the original aggregate. The function uses this implicit * knowledge to create the appropriate master function with correct data types. */ static Expr * MasterAggregateExpression(Aggref *originalAggregate, MasterAggregateWalkerContext *walkerContext) { const Index columnLevelsUp = 0; /* normal column */ const AttrNumber argumentId = 1; /* our aggregates have single arguments */ AggregateType aggregateType = GetAggregateType(originalAggregate); Expr *newMasterExpression = NULL; if (walkerContext->extendedOpNodeProperties->pullUpIntermediateRows) { Aggref *aggregate = (Aggref *) copyObject(originalAggregate); TargetEntry *targetEntry; foreach_declared_ptr(targetEntry, aggregate->args) { targetEntry->expr = (Expr *) makeVar(masterTableId, walkerContext->columnId, exprType((Node *) targetEntry->expr), exprTypmod((Node *) targetEntry->expr), exprCollation((Node *) targetEntry->expr), columnLevelsUp); walkerContext->columnId++; } aggregate->aggdirectargs = NIL; Expr *directarg; foreach_declared_ptr(directarg, originalAggregate->aggdirectargs) { /* * Need to replace nodes that contain any Vars with Vars referring * to the related column of the result set returned for the worker * aggregation. * * When there are no Vars, then the expression can be fully evaluated * on the coordinator, so we skip it here. This is not just an * optimization, but the result of the expression might require * calling the final function of the aggregate, and doing so when * there are no input rows (i.e.: with an empty tuple slot) is not * desirable for the node-executor methods. */ if (pull_var_clause_default((Node *) directarg) != NIL) { Var *var = makeVar(masterTableId, walkerContext->columnId, exprType((Node *) directarg), exprTypmod((Node *) directarg), exprCollation((Node *) directarg), columnLevelsUp); aggregate->aggdirectargs = lappend(aggregate->aggdirectargs, var); walkerContext->columnId++; } else { aggregate->aggdirectargs = lappend(aggregate->aggdirectargs, directarg); } } if (aggregate->aggfilter) { aggregate->aggfilter = (Expr *) makeVar(masterTableId, walkerContext->columnId, BOOLOID, -1, InvalidOid, columnLevelsUp); walkerContext->columnId++; } newMasterExpression = (Expr *) aggregate; } else if (aggregateType == AGGREGATE_COUNT && originalAggregate->aggdistinct && CountDistinctErrorRate == DISABLE_DISTINCT_APPROXIMATION && walkerContext->extendedOpNodeProperties->pullDistinctColumns) { Aggref *aggregate = (Aggref *) copyObject(originalAggregate); List *varList = pull_var_clause_default((Node *) aggregate); List *uniqueVarList = NIL; int startColumnCount = walkerContext->columnId; /* determine unique vars that were placed in target list by worker */ Var *column = NULL; foreach_declared_ptr(column, varList) { uniqueVarList = list_append_unique(uniqueVarList, copyObject(column)); } /* * Go over each var inside aggregate and update their varattno's according to * worker query target entry column index. */ Var *columnToUpdate = NULL; foreach_declared_ptr(columnToUpdate, varList) { int columnIndex = 0; Var *currentVar = NULL; foreach_declared_ptr(currentVar, uniqueVarList) { if (equal(columnToUpdate, currentVar)) { break; } columnIndex++; } columnToUpdate->varno = masterTableId; columnToUpdate->varnosyn = masterTableId; columnToUpdate->varattno = startColumnCount + columnIndex; columnToUpdate->varattnosyn = startColumnCount + columnIndex; } /* we added that many columns */ walkerContext->columnId += list_length(uniqueVarList); newMasterExpression = (Expr *) aggregate; } else if (aggregateType == AGGREGATE_COUNT && originalAggregate->aggdistinct && CountDistinctErrorRate != DISABLE_DISTINCT_APPROXIMATION) { /* * If enabled, we check for count(distinct) approximations before count * distincts. For this, we first compute hll_add_agg(hll_hash(column)) on * worker nodes, and get hll values. We then gather hlls on the master * node, and compute hll_cardinality(hll_union_agg(hll)). */ const int argCount = 1; const int defaultTypeMod = -1; /* extract schema name of hll */ Oid hllId = get_extension_oid(HLL_EXTENSION_NAME, false); Oid hllSchemaOid = get_extension_schema(hllId); const char *hllSchemaName = get_namespace_name(hllSchemaOid); Oid unionFunctionId = FunctionOid(hllSchemaName, HLL_UNION_AGGREGATE_NAME, argCount); Oid cardinalityFunctionId = FunctionOid(hllSchemaName, HLL_CARDINALITY_FUNC_NAME, argCount); Oid cardinalityReturnType = get_func_rettype(cardinalityFunctionId); Oid hllType = TypeOid(hllSchemaOid, HLL_TYPE_NAME); Oid hllTypeCollationId = get_typcollation(hllType); Var *hllColumn = makeVar(masterTableId, walkerContext->columnId, hllType, defaultTypeMod, hllTypeCollationId, columnLevelsUp); walkerContext->columnId++; TargetEntry *hllTargetEntry = makeTargetEntry((Expr *) hllColumn, argumentId, NULL, false); Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = hllType; unionAggregate->args = list_make1(hllTargetEntry); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make1_oid(unionAggregate->aggtype); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; FuncExpr *cardinalityExpression = makeNode(FuncExpr); cardinalityExpression->funcid = cardinalityFunctionId; cardinalityExpression->funcresulttype = cardinalityReturnType; cardinalityExpression->args = list_make1(unionAggregate); newMasterExpression = (Expr *) cardinalityExpression; } else if (aggregateType == AGGREGATE_AVERAGE) { /* * If the original aggregate is an average, we first compute sum(colum) * and count(column) on worker nodes. Then, we compute (sum(sum(column)) * / sum(count(column))) on the master node. */ const char *sumAggregateName = AggregateNames[AGGREGATE_SUM]; const char *countAggregateName = AggregateNames[AGGREGATE_COUNT]; Oid argumentType = AggregateArgumentType(originalAggregate); Oid sumFunctionId = AggregateFunctionOid(sumAggregateName, argumentType); Oid countFunctionId = AggregateFunctionOid(countAggregateName, ANYOID); /* calculate the aggregate types that worker nodes are going to return */ Oid workerSumReturnType = get_func_rettype(sumFunctionId); Oid workerCountReturnType = get_func_rettype(countFunctionId); /* create the expression sum(sum(column) / sum(count(column))) */ newMasterExpression = MasterAverageExpression(workerSumReturnType, workerCountReturnType, &(walkerContext->columnId)); } else if (aggregateType == AGGREGATE_COUNT) { /* * Count aggregates are handled in two steps. First, worker nodes report * their count results. Then, the master node sums up these results. */ /* worker aggregate and original aggregate have the same return type */ Oid workerReturnType = exprType((Node *) originalAggregate); int32 workerReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid workerCollationId = exprCollation((Node *) originalAggregate); const char *sumAggregateName = AggregateNames[AGGREGATE_SUM]; Oid sumFunctionId = AggregateFunctionOid(sumAggregateName, workerReturnType); Oid masterReturnType = get_func_rettype(sumFunctionId); Aggref *newMasterAggregate = copyObject(originalAggregate); newMasterAggregate->aggstar = false; newMasterAggregate->aggdistinct = NULL; newMasterAggregate->aggfnoid = sumFunctionId; newMasterAggregate->aggtype = masterReturnType; newMasterAggregate->aggfilter = NULL; newMasterAggregate->aggtranstype = InvalidOid; newMasterAggregate->aggargtypes = list_make1_oid(newMasterAggregate->aggtype); newMasterAggregate->aggsplit = AGGSPLIT_SIMPLE; Var *column = makeVar(masterTableId, walkerContext->columnId, workerReturnType, workerReturnTypeMod, workerCollationId, columnLevelsUp); walkerContext->columnId++; /* aggref expects its arguments to be wrapped in target entries */ TargetEntry *columnTargetEntry = makeTargetEntry((Expr *) column, argumentId, NULL, false); newMasterAggregate->args = list_make1(columnTargetEntry); /* cast numeric sum result to bigint (count's return type) */ CoerceViaIO *coerceExpr = makeNode(CoerceViaIO); coerceExpr->arg = (Expr *) newMasterAggregate; coerceExpr->resulttype = INT8OID; coerceExpr->resultcollid = InvalidOid; coerceExpr->coerceformat = COERCE_IMPLICIT_CAST; coerceExpr->location = -1; /* convert NULL to 0 in case of no rows */ Const *zeroConst = MakeIntegerConstInt64(0); List *coalesceArgs = list_make2(coerceExpr, zeroConst); CoalesceExpr *coalesceExpr = makeNode(CoalesceExpr); coalesceExpr->coalescetype = INT8OID; coalesceExpr->coalescecollid = InvalidOid; coalesceExpr->args = coalesceArgs; coalesceExpr->location = -1; newMasterExpression = (Expr *) coalesceExpr; } else if (aggregateType == AGGREGATE_ARRAY_AGG || aggregateType == AGGREGATE_JSONB_AGG || aggregateType == AGGREGATE_JSONB_OBJECT_AGG || aggregateType == AGGREGATE_JSON_AGG || aggregateType == AGGREGATE_JSON_OBJECT_AGG) { /* * Array and json aggregates are handled in two steps. First, we compute * array_agg() or json aggregate on the worker nodes. Then, we gather * the arrays or jsons on the master and compute the array_cat_agg() * or jsonb_cat_agg() aggregate on them to get the final array or json. */ const char *catAggregateName = NULL; Oid catInputType = InvalidOid; /* worker aggregate and original aggregate have same return type */ Oid workerReturnType = exprType((Node *) originalAggregate); int32 workerReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid workerCollationId = exprCollation((Node *) originalAggregate); /* assert that we do not support array or json aggregation with * distinct or order by */ Assert(!originalAggregate->aggorder); Assert(!originalAggregate->aggdistinct); if (aggregateType == AGGREGATE_ARRAY_AGG) { /* array_cat_agg() takes anyarray as input */ catAggregateName = ARRAY_CAT_AGGREGATE_NAME; catInputType = ANYCOMPATIBLEARRAYOID; } else if (aggregateType == AGGREGATE_JSONB_AGG || aggregateType == AGGREGATE_JSONB_OBJECT_AGG) { /* jsonb_cat_agg() takes jsonb as input */ catAggregateName = JSONB_CAT_AGGREGATE_NAME; catInputType = JSONBOID; } else { /* json_cat_agg() takes json as input */ catAggregateName = JSON_CAT_AGGREGATE_NAME; catInputType = JSONOID; } Assert(catAggregateName != NULL); Assert(catInputType != InvalidOid); Oid aggregateFunctionId = AggregateFunctionOid(catAggregateName, catInputType); /* create argument for the array_cat_agg() or jsonb_cat_agg() aggregate */ Var *column = makeVar(masterTableId, walkerContext->columnId, workerReturnType, workerReturnTypeMod, workerCollationId, columnLevelsUp); TargetEntry *catAggArgument = makeTargetEntry((Expr *) column, argumentId, NULL, false); walkerContext->columnId++; /* construct the master array_cat_agg() or jsonb_cat_agg() expression */ Aggref *newMasterAggregate = copyObject(originalAggregate); newMasterAggregate->aggfnoid = aggregateFunctionId; newMasterAggregate->args = list_make1(catAggArgument); newMasterAggregate->aggfilter = NULL; newMasterAggregate->aggtranstype = InvalidOid; if (aggregateType == AGGREGATE_ARRAY_AGG) { /* * Postgres expects the type of the array here such as INT4ARRAYOID. * Hence we set it to workerReturnType. If we set this to * ANYCOMPATIBLEARRAYOID then we will get the following error: * "argument declared anycompatiblearray is not an array but type anycompatiblearray" */ newMasterAggregate->aggargtypes = list_make1_oid(workerReturnType); } else { newMasterAggregate->aggargtypes = list_make1_oid(ANYARRAYOID); } newMasterAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) newMasterAggregate; } else if (aggregateType == AGGREGATE_HLL_ADD || aggregateType == AGGREGATE_HLL_UNION) { /* * If hll aggregates are called, we simply create the hll_union_aggregate * to apply in the master after running the original aggregate in * workers. */ Oid hllType = exprType((Node *) originalAggregate); Oid unionFunctionId = AggregateFunctionOid(HLL_UNION_AGGREGATE_NAME, hllType); int32 hllReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid hllTypeCollationId = exprCollation((Node *) originalAggregate); Var *hllColumn = makeVar(masterTableId, walkerContext->columnId, hllType, hllReturnTypeMod, hllTypeCollationId, columnLevelsUp); walkerContext->columnId++; TargetEntry *hllTargetEntry = makeTargetEntry((Expr *) hllColumn, argumentId, NULL, false); Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = hllType; unionAggregate->args = list_make1(hllTargetEntry); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make1_oid(hllType); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) unionAggregate; } else if (aggregateType == AGGREGATE_TOPN_UNION_AGG || aggregateType == AGGREGATE_TOPN_ADD_AGG) { /* * Top-N aggregates are handled in two steps. First, we compute * topn_add_agg() or topn_union_agg() aggregates on the worker nodes. * Then, we gather the Top-Ns on the master and take the union of all * to get the final topn. */ /* worker aggregate and original aggregate have same return type */ Oid topnType = exprType((Node *) originalAggregate); Oid unionFunctionId = AggregateFunctionOid(TOPN_UNION_AGGREGATE_NAME, topnType); int32 topnReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid topnTypeCollationId = exprCollation((Node *) originalAggregate); /* create argument for the topn_union_agg() aggregate */ Var *topnColumn = makeVar(masterTableId, walkerContext->columnId, topnType, topnReturnTypeMod, topnTypeCollationId, columnLevelsUp); walkerContext->columnId++; TargetEntry *topNTargetEntry = makeTargetEntry((Expr *) topnColumn, argumentId, NULL, false); /* construct the master topn_union_agg() expression */ Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = topnType; unionAggregate->args = list_make1(topNTargetEntry); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make1_oid(topnType); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) unionAggregate; } else if (aggregateType == AGGREGATE_TDIGEST_COMBINE || aggregateType == AGGREGATE_TDIGEST_ADD_DOUBLE) { /* tdigest of column */ Oid tdigestType = TDigestExtensionTypeOid(); /* tdigest type */ Oid unionFunctionId = TDigestExtensionAggTDigest1(); int32 tdigestReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid tdigestTypeCollationId = exprCollation((Node *) originalAggregate); /* create first argument for tdigest_precentile(tdigest, double) */ Var *tdigestColumn = makeVar(masterTableId, walkerContext->columnId, tdigestType, tdigestReturnTypeMod, tdigestTypeCollationId, columnLevelsUp); TargetEntry *tdigestTargetEntry = makeTargetEntry((Expr *) tdigestColumn, argumentId, NULL, false); walkerContext->columnId++; /* construct the master tdigest(tdigest) expression */ Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = originalAggregate->aggtype; unionAggregate->args = list_make1(tdigestTargetEntry); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make1_oid(tdigestType); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) unionAggregate; } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLEARRAY || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLEARRAY) { /* tdigest of column */ Oid tdigestType = TDigestExtensionTypeOid(); Oid unionFunctionId = InvalidOid; if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLE) { unionFunctionId = TDigestExtensionAggTDigestPercentile2(); } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLEARRAY) { unionFunctionId = TDigestExtensionAggTDigestPercentile2a(); } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLE) { unionFunctionId = TDigestExtensionAggTDigestPercentileOf2(); } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLEARRAY) { unionFunctionId = TDigestExtensionAggTDigestPercentileOf2a(); } Assert(OidIsValid(unionFunctionId)); int32 tdigestReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid tdigestTypeCollationId = exprCollation((Node *) originalAggregate); /* create first argument for tdigest_precentile(tdigest, double) */ Var *tdigestColumn = makeVar(masterTableId, walkerContext->columnId, tdigestType, tdigestReturnTypeMod, tdigestTypeCollationId, columnLevelsUp); TargetEntry *tdigestTargetEntry = makeTargetEntry((Expr *) tdigestColumn, argumentId, NULL, false); walkerContext->columnId++; /* construct the master tdigest_precentile(tdigest, double) expression */ Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = originalAggregate->aggtype; unionAggregate->args = list_make2( tdigestTargetEntry, list_nth(originalAggregate->args, 2)); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make2_oid( tdigestType, list_nth_oid(originalAggregate->aggargtypes, 2)); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) unionAggregate; } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLEARRAY || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLEARRAY) { /* tdigest of column */ Oid tdigestType = TDigestExtensionTypeOid(); /* These functions already will combine the tdigest arguments returned */ Oid unionFunctionId = originalAggregate->aggfnoid; int32 tdigestReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid tdigestTypeCollationId = exprCollation((Node *) originalAggregate); /* create first argument for tdigest_precentile(tdigest, double) */ Var *tdigestColumn = makeVar(masterTableId, walkerContext->columnId, tdigestType, tdigestReturnTypeMod, tdigestTypeCollationId, columnLevelsUp); TargetEntry *tdigestTargetEntry = makeTargetEntry((Expr *) tdigestColumn, argumentId, NULL, false); walkerContext->columnId++; /* construct the master tdigest_precentile(tdigest, double) expression */ Aggref *unionAggregate = makeNode(Aggref); unionAggregate->aggfnoid = unionFunctionId; unionAggregate->aggtype = originalAggregate->aggtype; unionAggregate->args = list_make2( tdigestTargetEntry, list_nth(originalAggregate->args, 1)); unionAggregate->aggkind = AGGKIND_NORMAL; unionAggregate->aggfilter = NULL; unionAggregate->aggtranstype = InvalidOid; unionAggregate->aggargtypes = list_make2_oid( tdigestType, list_nth_oid(originalAggregate->aggargtypes, 1)); unionAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) unionAggregate; } else if (aggregateType == AGGREGATE_CUSTOM_COMBINE) { HeapTuple aggTuple = SearchSysCache1(AGGFNOID, ObjectIdGetDatum(originalAggregate->aggfnoid)); Form_pg_aggregate aggform; Oid combine; bool useBinaryCoordinatorCombine = false; if (!HeapTupleIsValid(aggTuple)) { elog(ERROR, "citus cache lookup failed for aggregate %u", originalAggregate->aggfnoid); return NULL; } else { aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryCoordinatorCombine = aggform->aggtranstype != InvalidOid && IsAggTransTypeBinarySerializable(aggform); ReleaseSysCache(aggTuple); } if (combine != InvalidOid) { Oid coordCombineId = useBinaryCoordinatorCombine ? CoordBinaryCombineAggOid() : CoordCombineAggOid(); Oid workerReturnType = useBinaryCoordinatorCombine ? BYTEAOID : CSTRINGOID; int32 workerReturnTypeMod = -1; Oid workerCollationId = InvalidOid; Oid resultType = exprType((Node *) originalAggregate); Const *aggOidParam = makeConst(OIDOID, -1, InvalidOid, sizeof(Oid), ObjectIdGetDatum(originalAggregate->aggfnoid), false, true); Var *column = makeVar(masterTableId, walkerContext->columnId, workerReturnType, workerReturnTypeMod, workerCollationId, columnLevelsUp); walkerContext->columnId++; Const *nullTag = makeNullConst(resultType, -1, InvalidOid); List *aggArguments = list_make3(makeTargetEntry((Expr *) aggOidParam, 1, NULL, false), makeTargetEntry((Expr *) column, 2, NULL, false), makeTargetEntry((Expr *) nullTag, 3, NULL, false)); /* coord_combine_agg(agg, workercol) */ Aggref *newMasterAggregate = makeNode(Aggref); newMasterAggregate->aggfnoid = coordCombineId; newMasterAggregate->aggtype = originalAggregate->aggtype; newMasterAggregate->args = aggArguments; newMasterAggregate->aggkind = AGGKIND_NORMAL; newMasterAggregate->aggfilter = NULL; newMasterAggregate->aggtranstype = INTERNALOID; newMasterAggregate->aggargtypes = list_make3_oid(OIDOID, workerReturnType, resultType); newMasterAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) newMasterAggregate; } else { elog(ERROR, "Aggregate lacks COMBINEFUNC"); } } else { /* * All other aggregates are handled as they are. These include sum, min, * and max. */ /* worker aggregate and original aggregate have the same return type */ Oid workerReturnType = exprType((Node *) originalAggregate); int32 workerReturnTypeMod = exprTypmod((Node *) originalAggregate); Oid workerCollationId = exprCollation((Node *) originalAggregate); const char *aggregateName = AggregateNames[aggregateType]; Oid aggregateFunctionId = AggregateFunctionOid(aggregateName, workerReturnType); Oid masterReturnType = get_func_rettype(aggregateFunctionId); Aggref *newMasterAggregate = copyObject(originalAggregate); newMasterAggregate->aggdistinct = NULL; newMasterAggregate->aggfnoid = aggregateFunctionId; newMasterAggregate->aggtype = masterReturnType; newMasterAggregate->aggfilter = NULL; /* * Polymorphic aggregates determine their actual return type based on * their argument type, so replace it with the worker return type. */ if (IsPolymorphicTypeFamily1(masterReturnType)) { newMasterAggregate->aggtype = workerReturnType; Expr *firstArg = FirstAggregateArgument(originalAggregate); newMasterAggregate->aggcollid = exprCollation((Node *) firstArg); } Var *column = makeVar(masterTableId, walkerContext->columnId, workerReturnType, workerReturnTypeMod, workerCollationId, columnLevelsUp); walkerContext->columnId++; /* aggref expects its arguments to be wrapped in target entries */ TargetEntry *columnTargetEntry = makeTargetEntry((Expr *) column, argumentId, NULL, false); newMasterAggregate->args = list_make1(columnTargetEntry); newMasterExpression = (Expr *) newMasterAggregate; } /* * Aggregate functions could have changed the return type. If so, we wrap * the new expression with a conversion function to make it have the same * type as the original aggregate. We need this since functions like sorting * and grouping have already been chosen based on the original type. */ Expr *typeConvertedExpression = AddTypeConversion((Node *) originalAggregate, (Node *) newMasterExpression); if (typeConvertedExpression != NULL) { newMasterExpression = typeConvertedExpression; } return newMasterExpression; } /* * MasterAverageExpression creates an expression of the form (sum(column1) / * sum(column2)), where column1 is the sum of the original value, and column2 is * the count of that value. This expression allows us to evaluate the average * function over distributed data. */ static Expr * MasterAverageExpression(Oid sumAggregateType, Oid countAggregateType, AttrNumber *columnId) { const char *sumAggregateName = AggregateNames[AGGREGATE_SUM]; const int32 defaultTypeMod = -1; const Index defaultLevelsUp = 0; const AttrNumber argumentId = 1; Oid sumTypeCollationId = get_typcollation(sumAggregateType); Oid countTypeCollationId = get_typcollation(countAggregateType); /* create the first argument for sum(column1) */ Var *firstColumn = makeVar(masterTableId, (*columnId), sumAggregateType, defaultTypeMod, sumTypeCollationId, defaultLevelsUp); TargetEntry *firstTargetEntry = makeTargetEntry((Expr *) firstColumn, argumentId, NULL, false); (*columnId)++; Aggref *firstSum = makeNode(Aggref); firstSum->aggfnoid = AggregateFunctionOid(sumAggregateName, sumAggregateType); firstSum->aggtype = get_func_rettype(firstSum->aggfnoid); firstSum->args = list_make1(firstTargetEntry); firstSum->aggkind = AGGKIND_NORMAL; firstSum->aggtranstype = InvalidOid; firstSum->aggargtypes = list_make1_oid(firstSum->aggtype); firstSum->aggsplit = AGGSPLIT_SIMPLE; /* create the second argument for sum(column2) */ Var *secondColumn = makeVar(masterTableId, (*columnId), countAggregateType, defaultTypeMod, countTypeCollationId, defaultLevelsUp); TargetEntry *secondTargetEntry = makeTargetEntry((Expr *) secondColumn, argumentId, NULL, false); (*columnId)++; Aggref *secondSum = makeNode(Aggref); secondSum->aggfnoid = AggregateFunctionOid(sumAggregateName, countAggregateType); secondSum->aggtype = get_func_rettype(secondSum->aggfnoid); secondSum->args = list_make1(secondTargetEntry); secondSum->aggkind = AGGKIND_NORMAL; secondSum->aggtranstype = InvalidOid; secondSum->aggargtypes = list_make1_oid(firstSum->aggtype); secondSum->aggsplit = AGGSPLIT_SIMPLE; /* * Build the division operator between these two aggregates. This function * will convert the types of the aggregates if necessary. */ List *operatorNameList = list_make1(makeString(DIVISION_OPER_NAME)); Expr *opExpr = make_op(NULL, operatorNameList, (Node *) firstSum, (Node *) secondSum, NULL, -1); return opExpr; } /* * AddTypeConversion checks if the given expressions generate the same types. If * they don't, the function adds a type conversion function on top of the new * expression to have it generate the same type as the original aggregate. */ static Expr * AddTypeConversion(Node *originalAggregate, Node *newExpression) { Oid newTypeId = exprType(newExpression); Oid originalTypeId = exprType(originalAggregate); int32 originalTypeMod = exprTypmod(originalAggregate); /* nothing to do if the two types are the same */ if (originalTypeId == newTypeId) { return NULL; } /* otherwise, add a type conversion function */ Node *typeConvertedExpression = coerce_to_target_type(NULL, newExpression, newTypeId, originalTypeId, originalTypeMod, COERCION_EXPLICIT, COERCE_EXPLICIT_CAST, -1); Assert(typeConvertedExpression != NULL); return (Expr *) typeConvertedExpression; } /* * WorkerExtendedOpNode creates the worker extended operator node from the given * originalOpNode and extendedOpNodeProperties. * * For the details of the processing see the comments of the functions that * are called from this function. */ static MultiExtendedOp * WorkerExtendedOpNode(MultiExtendedOp *originalOpNode, ExtendedOpNodeProperties *extendedOpNodeProperties) { bool distinctPreventsLimitPushdown = false; QueryTargetList queryTargetList; QueryGroupClause queryGroupClause; QueryDistinctClause queryDistinctClause; QueryWindowClause queryWindowClause; QueryOrderByLimit queryOrderByLimit; Node *queryHavingQual = NULL; List *originalTargetEntryList = originalOpNode->targetList; List *originalGroupClauseList = originalOpNode->groupClauseList; List *originalSortClauseList = originalOpNode->sortClauseList; Node *originalHavingQual = originalOpNode->havingQual; Node *originalLimitCount = originalOpNode->limitCount; Node *originalLimitOffset = originalOpNode->limitOffset; List *originalWindowClause = originalOpNode->windowClause; List *originalDistinctClause = originalOpNode->distinctClause; bool hasDistinctOn = originalOpNode->hasDistinctOn; int originalGroupClauseLength = list_length(originalGroupClauseList); /* initialize to default values */ memset(&queryTargetList, 0, sizeof(queryTargetList)); memset(&queryGroupClause, 0, sizeof(queryGroupClause)); memset(&queryDistinctClause, 0, sizeof(queryDistinctClause)); memset(&queryWindowClause, 0, sizeof(queryWindowClause)); memset(&queryOrderByLimit, 0, sizeof(queryOrderByLimit)); /* calculate the next sort group index based on the original target list */ Index nextSortGroupRefIndex = GetNextSortGroupRef(originalTargetEntryList); /* targetProjectionNumber starts from 1 */ queryTargetList.targetProjectionNumber = 1; if (!extendedOpNodeProperties->pullUpIntermediateRows) { queryGroupClause.groupClauseList = copyObject(originalGroupClauseList); } else { queryGroupClause.groupClauseList = NIL; } /* * For the purpose of this variable, not pushing down when there are no groups * is pushing down the original grouping, ie the worker's GROUP BY matches * the master's GROUP BY. */ bool pushingDownOriginalGrouping = list_length(queryGroupClause.groupClauseList) == originalGroupClauseLength; /* * nextSortGroupRefIndex is used by group by, window and order by clauses. * Thus, we pass a reference to a single nextSortGroupRefIndex and expect * it modified separately while processing those parts of the query. */ queryGroupClause.nextSortGroupRefIndex = &nextSortGroupRefIndex; queryWindowClause.nextSortGroupRefIndex = &nextSortGroupRefIndex; queryOrderByLimit.nextSortGroupRefIndex = &nextSortGroupRefIndex; /* process each part of the query in order to generate the worker query's parts */ ProcessTargetListForWorkerQuery(originalTargetEntryList, extendedOpNodeProperties, &queryTargetList, &queryGroupClause); ProcessHavingClauseForWorkerQuery(originalHavingQual, extendedOpNodeProperties, &queryHavingQual, &queryTargetList, &queryGroupClause); /* * Planner optimizations may leave window clauses with hasWindowFuncs as false. * Ignore window clauses in that case. */ if (extendedOpNodeProperties->hasWindowFuncs) { if (extendedOpNodeProperties->onlyPushableWindowFunctions) { ProcessWindowFunctionsForWorkerQuery(originalWindowClause, originalTargetEntryList, &queryWindowClause, &queryTargetList); } else { ProcessWindowFunctionPullUpForWorkerQuery(originalWindowClause, &queryTargetList); } } if (ShouldProcessDistinctOrderAndLimitForWorker(extendedOpNodeProperties, pushingDownOriginalGrouping, originalHavingQual)) { bool queryHasAggregates = TargetListHasAggregates(originalTargetEntryList); ProcessDistinctClauseForWorkerQuery(originalDistinctClause, hasDistinctOn, queryGroupClause.groupClauseList, queryHasAggregates, &queryDistinctClause, &distinctPreventsLimitPushdown); /* * Order by and limit clauses are relevant to each other, and processing * them together makes it handy for us. * * The other parts of the query might have already prohibited pushing down * LIMIT and ORDER BY clauses as described below: * (1) Creating a new group by clause during aggregate mutation, or * (2) Distinct clause is not pushed down */ bool groupByExtended = list_length(queryGroupClause.groupClauseList) > originalGroupClauseLength; if (pushingDownOriginalGrouping && !groupByExtended && !distinctPreventsLimitPushdown) { /* both sort and limit clauses rely on similar information */ OrderByLimitReference limitOrderByReference = BuildOrderByLimitReference(hasDistinctOn, extendedOpNodeProperties-> groupedByDisjointPartitionColumn, extendedOpNodeProperties-> onlyPushableWindowFunctions, originalGroupClauseList, originalSortClauseList, originalTargetEntryList); ProcessLimitOrderByForWorkerQuery(limitOrderByReference, originalLimitCount, originalLimitOffset, originalSortClauseList, originalGroupClauseList, originalTargetEntryList, &queryOrderByLimit, &queryTargetList); } } /* finally, fill the extended op node with the data we gathered */ MultiExtendedOp *workerExtendedOpNode = CitusMakeNode(MultiExtendedOp); workerExtendedOpNode->targetList = queryTargetList.targetEntryList; workerExtendedOpNode->groupClauseList = queryGroupClause.groupClauseList; workerExtendedOpNode->havingQual = queryHavingQual; workerExtendedOpNode->hasDistinctOn = queryDistinctClause.workerHasDistinctOn; workerExtendedOpNode->distinctClause = queryDistinctClause.workerDistinctClause; workerExtendedOpNode->hasWindowFuncs = queryWindowClause.hasWindowFunctions; workerExtendedOpNode->windowClause = queryWindowClause.workerWindowClauseList; workerExtendedOpNode->sortClauseList = queryOrderByLimit.workerSortClauseList; workerExtendedOpNode->limitCount = queryOrderByLimit.workerLimitCount; /* * If the limitCount cannot be pushed down it will be NULL, so the deparser will * ignore the limitOption. */ workerExtendedOpNode->limitOption = originalOpNode->limitOption; return workerExtendedOpNode; } /* * ProcessTargetListForWorkerQuery gets the inputs and modifies the outputs * such that the worker query's target list and group by clauses are extended * for the given inputs. * * The function walks over the input targetEntryList. For the entries * with aggregates in them, it calls the recursive aggregate walker function to * create aggregates for the worker nodes. For example, the avg() is sent to * the worker with two expressions count() and sum(). Thus, a single target entry * might end up with multiple expressions in the worker query. * * The function doesn't change the aggregates in the window functions and sends them * as-is. The reason is that Citus only supports pushing down window functions when * this is safe to do. * * The function also handles count distinct operator if it is used in repartition * subqueries or on non-partition columns (e.g., cannot be pushed down). Each * column in count distinct aggregate is added to target list, and group by * list of worker extended operator. This approach guarantees the distinctness * in the worker queries. * * inputs: targetEntryList, extendedOpNodeProperties * outputs: queryTargetList, queryGroupClause */ static void ProcessTargetListForWorkerQuery(List *targetEntryList, ExtendedOpNodeProperties *extendedOpNodeProperties, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause) { WorkerAggregateWalkerContext workerAggContext = { .extendedOpNodeProperties = extendedOpNodeProperties, }; /* iterate over original target entries */ TargetEntry *originalTargetEntry = NULL; foreach_declared_ptr(originalTargetEntry, targetEntryList) { Expr *originalExpression = originalTargetEntry->expr; List *newExpressionList = NIL; /* reset walker context */ workerAggContext.expressionList = NIL; workerAggContext.createGroupByClause = false; /* * If we can push down the expression we copy the expression to the targetlist of the worker query. * Otherwise the expression is processed to be combined on the coordinator. */ if (CanPushDownExpression((Node *) originalExpression, extendedOpNodeProperties)) { newExpressionList = list_make1(originalExpression); } else { WorkerAggregateWalker((Node *) originalExpression, &workerAggContext); newExpressionList = workerAggContext.expressionList; } ExpandWorkerTargetEntry(newExpressionList, originalTargetEntry, workerAggContext.createGroupByClause, queryTargetList, queryGroupClause); } } /* * ProcessHavingClauseForWorkerQuery gets the inputs and modifies the outputs * such that the worker query's target list and group by clauses are extended * based on the inputs. * * The rule is that Citus always applies the HAVING clause on the * coordinator. Thus, it pulls the necessary data from the workers. Also, when the * having clause is safe to pushdown to the workers, workerHavingQual is set to * be the original having clause. * * inputs: originalHavingQual, extendedOpNodeProperties * outputs: workerHavingQual, queryTargetList, queryGroupClause */ static void ProcessHavingClauseForWorkerQuery(Node *originalHavingQual, ExtendedOpNodeProperties *extendedOpNodeProperties, Node **workerHavingQual, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause) { *workerHavingQual = NULL; if (originalHavingQual == NULL) { return; } if (extendedOpNodeProperties->pushDownGroupingAndHaving) { /* * We converted the having expression to a list in subquery pushdown * planner. However, this query cannot be parsed as it is in the worker. * We should convert this back to being explicit for worker query * so that it can be parsed when it hits the standard planner in worker. */ if (IsA(originalHavingQual, List)) { *workerHavingQual = (Node *) make_ands_explicit((List *) originalHavingQual); } else { *workerHavingQual = originalHavingQual; } } else { /* * If the GROUP BY or PARTITION BY is not on the distribution column * then we need to combine the aggregates in the HAVING across shards. */ WorkerAggregateWalkerContext workerAggContext = { .extendedOpNodeProperties = extendedOpNodeProperties, }; WorkerAggregateWalker(originalHavingQual, &workerAggContext); List *newExpressionList = workerAggContext.expressionList; TargetEntry *targetEntry = NULL; ExpandWorkerTargetEntry(newExpressionList, targetEntry, workerAggContext.createGroupByClause, queryTargetList, queryGroupClause); } } /* * ProcessDistinctClauseForWorkerQuery gets the inputs and modifies the outputs * such that worker query's DISTINCT and DISTINCT ON clauses are set accordingly. * Note the function may or may not decide to pushdown the DISTINCT and DISTINCT * on clauses based on the inputs. * * See the detailed comments in the function for the rules of pushing down DISTINCT * and DISTINCT ON clauses to the worker queries. * * The function also sets distinctPreventsLimitPushdown. As the name reveals, * distinct could prevent pushing down LIMIT clauses later in the planning. * For the details, see the comments in the function. * * inputs: distinctClause, hasDistinctOn, groupClauseList, queryHasAggregates * outputs: queryDistinctClause, distinctPreventsLimitPushdown * */ static void ProcessDistinctClauseForWorkerQuery(List *distinctClause, bool hasDistinctOn, List *groupClauseList, bool queryHasAggregates, QueryDistinctClause *queryDistinctClause, bool *distinctPreventsLimitPushdown) { *distinctPreventsLimitPushdown = false; if (distinctClause == NIL) { return; } bool distinctClauseSupersetofGroupClause = false; if (groupClauseList == NIL || IsGroupBySubsetOfDistinct(groupClauseList, distinctClause)) { distinctClauseSupersetofGroupClause = true; } else { distinctClauseSupersetofGroupClause = false; /* * GROUP BY being a subset of DISTINCT guarantees the * distinctness on the workers. Otherwise, pushing down * LIMIT might cause missing the necessary data from * the worker query */ *distinctPreventsLimitPushdown = true; } /* * Distinct is pushed down to worker query only if the query does not * contain an aggregate in which master processing might be required to * complete the final result before distinct operation. We also prevent * distinct pushdown if distinct clause is missing some entries that * group by clause has. */ bool shouldPushdownDistinct = !queryHasAggregates && distinctClauseSupersetofGroupClause; if (shouldPushdownDistinct) { queryDistinctClause->workerDistinctClause = distinctClause; queryDistinctClause->workerHasDistinctOn = hasDistinctOn; } } /* * ProcessWindowFunctionsForWorkerQuery gets the inputs and modifies the outputs such * that worker query's workerWindowClauseList is set when the window clauses are safe to * pushdown. * * Note that even though Citus only pushes down the window functions, it may need to * modify the target list of the worker query when the window function refers to * an avg(). The reason is that any aggregate which is also referred by other * target entries would be mutated by Citus. Thus, we add a copy of the same aggregate * to the worker target list to make sure that the window function refers to the * non-mutated aggregate. * * inputs: windowClauseList, originalTargetEntryList * outputs: queryWindowClause, queryTargetList * */ static void ProcessWindowFunctionsForWorkerQuery(List *windowClauseList, List *originalTargetEntryList, QueryWindowClause *queryWindowClause, QueryTargetList *queryTargetList) { if (windowClauseList == NIL) { return; } queryWindowClause->workerWindowClauseList = windowClauseList; queryWindowClause->hasWindowFunctions = true; } /* ProcessWindowFunctionPullUpForWorkerQuery pulls up inputs for window functions */ static void ProcessWindowFunctionPullUpForWorkerQuery(List *windowClause, QueryTargetList *queryTargetList) { if (windowClause != NIL) { List *columnList = pull_var_clause_default((Node *) windowClause); Expr *newExpression = NULL; foreach_declared_ptr(newExpression, columnList) { TargetEntry *newTargetEntry = makeNode(TargetEntry); newTargetEntry->expr = newExpression; newTargetEntry->resname = WorkerColumnName(queryTargetList->targetProjectionNumber); /* force resjunk to false as we may need this on the master */ newTargetEntry->resjunk = false; newTargetEntry->resno = queryTargetList->targetProjectionNumber; queryTargetList->targetEntryList = lappend(queryTargetList->targetEntryList, newTargetEntry); queryTargetList->targetProjectionNumber++; } } } /* * ProcessLimitOrderByForWorkerQuery gets the inputs and modifies the outputs * such that worker query's LIMIT and ORDER BY clauses are set accordingly. * Adding entries to ORDER BY might trigger adding new entries to newTargetEntryList. * See GenerateNewTargetEntriesForSortClauses() for the details. * * For the decisions on whether and how to pushdown LIMIT and ORDER BY are documented * in the functions that are called from this function. * * inputs: sortLimitReference, originalLimitCount, limitOffset, * sortClauseList, groupClauseList, originalTargetList * outputs: queryOrderByLimit, queryTargetList */ static void ProcessLimitOrderByForWorkerQuery(OrderByLimitReference orderByLimitReference, Node *originalLimitCount, Node *limitOffset, List *sortClauseList, List *groupClauseList, List *originalTargetList, QueryOrderByLimit *queryOrderByLimit, QueryTargetList *queryTargetList) { queryOrderByLimit->workerLimitCount = WorkerLimitCount(originalLimitCount, limitOffset, orderByLimitReference); queryOrderByLimit->workerSortClauseList = WorkerSortClauseList(originalLimitCount, groupClauseList, sortClauseList, orderByLimitReference); } /* * BuildOrderByLimitReference is a helper function that simply builds * the necessary information for processing the limit and order by. * The return value should be used in a read-only manner. */ static OrderByLimitReference BuildOrderByLimitReference(bool hasDistinctOn, bool groupedByDisjointPartitionColumn, bool onlyPushableWindowFunctions, List *groupClause, List *sortClauseList, List *targetList) { OrderByLimitReference limitOrderByReference; limitOrderByReference.groupedByDisjointPartitionColumn = groupedByDisjointPartitionColumn; limitOrderByReference.onlyPushableWindowFunctions = onlyPushableWindowFunctions; limitOrderByReference.hasDistinctOn = hasDistinctOn; limitOrderByReference.groupClauseIsEmpty = (groupClause == NIL); limitOrderByReference.sortClauseIsEmpty = (sortClauseList == NIL); limitOrderByReference.canApproximate = CanPushDownLimitApproximate(sortClauseList, targetList); limitOrderByReference.hasOrderByAggregate = HasOrderByAggregate(sortClauseList, targetList); return limitOrderByReference; } /* * TargetListHasAggregates returns true if any of the elements in the * target list contain aggregates that are not inside the window functions. * This function should not be called if window functions are being pulled up. */ bool TargetListHasAggregates(List *targetEntryList) { TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, targetEntryList) { Expr *targetExpr = targetEntry->expr; bool hasAggregates = contain_aggs_of_level((Node *) targetExpr, 0); bool hasWindowFunction = contain_window_function((Node *) targetExpr); /* * If the expression uses aggregates inside window function contain agg * clause still returns true. We want to make sure it is not a part of * window function before we proceed. */ if (hasAggregates && !hasWindowFunction) { return true; } } return false; } /* * ExpandWorkerTargetEntry is a utility function which processes the * expressions that are intended to be added to the worker target list. * * In summary, the function gets a list of expressions, converts them to target * entries and updates all the necessary fields such that the expression is correctly * added to the worker query's target list. * * Inputs: * - expressionList: The list of expressions that should be added to the worker query's * target list. * - originalTargetEntry: Target entry that the expressionList generated for. NULL * if the expressionList is not generated from any target entry. * - addToGroupByClause: True if the expressionList should also be added to the * worker query's GROUP BY clause. */ static void ExpandWorkerTargetEntry(List *expressionList, TargetEntry *originalTargetEntry, bool addToGroupByClause, QueryTargetList *queryTargetList, QueryGroupClause *queryGroupClause) { /* now create target entries for each new expression */ Expr *newExpression = NULL; foreach_declared_ptr(newExpression, expressionList) { /* generate and add the new target entry to the target list */ TargetEntry *newTargetEntry = GenerateWorkerTargetEntry(originalTargetEntry, newExpression, queryTargetList->targetProjectionNumber); queryTargetList->targetProjectionNumber++; queryTargetList->targetEntryList = lappend(queryTargetList->targetEntryList, newTargetEntry); /* * Detect new targets of type Var and add it to group clause list. * This case is expected only if the target entry has aggregates and * it is inside a repartitioned subquery. We create group by entry * for each Var in target list. This code does not check if this * Var was already in the target list or in group by clauses. */ if (IsA(newExpression, Var) && addToGroupByClause) { AppendTargetEntryToGroupClause(newTargetEntry, queryGroupClause); } } } /* * GetNextSortGroupRef gets a target list entry and returns * the next ressortgroupref that should be used based on the * input target list. */ static Index GetNextSortGroupRef(List *targetEntryList) { Index nextSortGroupRefIndex = 0; /* find max of sort group ref index */ TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, targetEntryList) { if (targetEntry->ressortgroupref > nextSortGroupRefIndex) { nextSortGroupRefIndex = targetEntry->ressortgroupref; } } /* next group ref index starts from max group ref index + 1 */ nextSortGroupRefIndex++; return nextSortGroupRefIndex; } /* * GenerateWorkerTargetEntry is a simple utility function which gets a * target entry, an expression and a targetProjectionNumber. * * The function returns a newly allocated target entry which can be added * to the worker's target list. */ static TargetEntry * GenerateWorkerTargetEntry(TargetEntry *targetEntry, Expr *workerExpression, AttrNumber targetProjectionNumber) { TargetEntry *newTargetEntry = NULL; /* * If a target entry is already provided, use a copy of * it because some of the callers rely on resorigtbl and * resorigcol. */ if (targetEntry) { newTargetEntry = flatCopyTargetEntry(targetEntry); } else { newTargetEntry = makeNode(TargetEntry); } if (newTargetEntry->resname == NULL) { newTargetEntry->resname = WorkerColumnName(targetProjectionNumber); } /* we can't generate a target entry without an expression */ Assert(workerExpression != NULL); /* force resjunk to false as we may need this on the master */ newTargetEntry->expr = workerExpression; newTargetEntry->resjunk = false; newTargetEntry->resno = targetProjectionNumber; return newTargetEntry; } /* * AppendTargetEntryToGroupClause gets a target entry, pointer to group list * and the ressortgroupref index. * * The function modifies all of the three input such that the target entry is * appended to the group clause and the index is incremented by one. */ static void AppendTargetEntryToGroupClause(TargetEntry *targetEntry, QueryGroupClause *queryGroupClause) { Expr *targetExpr PG_USED_FOR_ASSERTS_ONLY = targetEntry->expr; /* we currently only support appending Var target entries */ Assert(IsA(targetExpr, Var)); Var *targetColumn = (Var *) targetEntry->expr; SortGroupClause *groupByClause = CreateSortGroupClause(targetColumn); /* the target entry should have an index */ targetEntry->ressortgroupref = *queryGroupClause->nextSortGroupRefIndex; /* the group by clause entry should point to the correct index in the target list */ groupByClause->tleSortGroupRef = *queryGroupClause->nextSortGroupRefIndex; /* update the group by list and the index's value */ queryGroupClause->groupClauseList = lappend(queryGroupClause->groupClauseList, groupByClause); (*queryGroupClause->nextSortGroupRefIndex)++; } /* * WorkerAggregateWalker walks over the original target entry expression, and * creates the list of expression trees (potentially more than one) to execute * on the worker nodes. The function creates new expressions for aggregates and * columns; and recurses into expression_tree_walker() for all other expression * types. */ static bool WorkerAggregateWalker(Node *node, WorkerAggregateWalkerContext *walkerContext) { bool walkerResult = false; if (node == NULL) { return false; } if (IsA(node, Aggref)) { if (CanPushDownExpression(node, walkerContext->extendedOpNodeProperties)) { walkerContext->expressionList = lappend(walkerContext->expressionList, node); } else { Aggref *originalAggregate = (Aggref *) node; List *workerAggregateList = WorkerAggregateExpressionList(originalAggregate, walkerContext); walkerContext->expressionList = list_concat(walkerContext->expressionList, workerAggregateList); } } else if (IsA(node, Var)) { Var *originalColumn = (Var *) node; walkerContext->expressionList = lappend(walkerContext->expressionList, originalColumn); } else { walkerResult = expression_tree_walker(node, WorkerAggregateWalker, (void *) walkerContext); } return walkerResult; } /* * WorkerAggregateExpressionList takes in the original aggregate function, and * determines the transformed aggregate functions to execute on worker nodes. * The function then returns these aggregates in a list. It also creates * group by clauses for newly added targets to be placed in the extended operator * node. */ static List * WorkerAggregateExpressionList(Aggref *originalAggregate, WorkerAggregateWalkerContext *walkerContext) { List *workerAggregateList = NIL; if (walkerContext->extendedOpNodeProperties->pullUpIntermediateRows) { TargetEntry *targetEntry; foreach_declared_ptr(targetEntry, originalAggregate->args) { workerAggregateList = lappend(workerAggregateList, targetEntry->expr); } Expr *directarg; foreach_declared_ptr(directarg, originalAggregate->aggdirectargs) { /* * The worker aggregation should execute any node that contains any * Var nodes and return the result in the targetlist, so that the * combine query can then fetch the result via remote scan; see * MasterAggregateExpression. */ if (pull_var_clause_default((Node *) directarg) != NIL) { workerAggregateList = lappend(workerAggregateList, directarg); } } if (originalAggregate->aggfilter) { workerAggregateList = lappend(workerAggregateList, originalAggregate->aggfilter); } return workerAggregateList; } AggregateType aggregateType = GetAggregateType(originalAggregate); if (aggregateType == AGGREGATE_COUNT && originalAggregate->aggdistinct && CountDistinctErrorRate == DISABLE_DISTINCT_APPROXIMATION && walkerContext->extendedOpNodeProperties->pullDistinctColumns) { Aggref *aggregate = (Aggref *) copyObject(originalAggregate); List *columnList = pull_var_clause_default((Node *) aggregate); Var *column = NULL; foreach_declared_ptr(column, columnList) { workerAggregateList = list_append_unique(workerAggregateList, column); } walkerContext->createGroupByClause = true; } else if (aggregateType == AGGREGATE_COUNT && originalAggregate->aggdistinct && CountDistinctErrorRate != DISABLE_DISTINCT_APPROXIMATION) { /* * If the original aggregate is a count(distinct) approximation, we want * to compute hll_add_agg(hll_hash(var), storageSize) on worker nodes. */ const AttrNumber firstArgumentId = 1; const AttrNumber secondArgumentId = 2; const int hashArgumentCount = 2; const int addArgumentCount = 2; /* init hll_hash() related variables */ Oid argumentType = AggregateArgumentType(originalAggregate); TargetEntry *argument = (TargetEntry *) linitial(originalAggregate->args); Expr *argumentExpression = copyObject(argument->expr); /* extract schema name of hll */ Oid hllId = get_extension_oid(HLL_EXTENSION_NAME, false); Oid hllSchemaOid = get_extension_schema(hllId); const char *hllSchemaName = get_namespace_name(hllSchemaOid); const char *hashFunctionName = CountDistinctHashFunctionName(argumentType); Oid hashFunctionId = FunctionOid(hllSchemaName, hashFunctionName, hashArgumentCount); Oid hashFunctionReturnType = get_func_rettype(hashFunctionId); /* init hll_add_agg() related variables */ Oid addFunctionId = FunctionOid(hllSchemaName, HLL_ADD_AGGREGATE_NAME, addArgumentCount); Oid hllType = TypeOid(hllSchemaOid, HLL_TYPE_NAME); int logOfStorageSize = CountDistinctStorageSize(CountDistinctErrorRate); Const *logOfStorageSizeConst = MakeIntegerConst(logOfStorageSize); /* construct hll_hash() expression */ FuncExpr *hashFunction = makeNode(FuncExpr); hashFunction->funcid = hashFunctionId; hashFunction->funcresulttype = hashFunctionReturnType; hashFunction->args = list_make1(argumentExpression); /* construct hll_add_agg() expression */ TargetEntry *hashedColumnArgument = makeTargetEntry((Expr *) hashFunction, firstArgumentId, NULL, false); TargetEntry *storageSizeArgument = makeTargetEntry((Expr *) logOfStorageSizeConst, secondArgumentId, NULL, false); List *addAggregateArgumentList = list_make2(hashedColumnArgument, storageSizeArgument); Aggref *addAggregateFunction = makeNode(Aggref); addAggregateFunction->aggfnoid = addFunctionId; addAggregateFunction->aggtype = hllType; addAggregateFunction->args = addAggregateArgumentList; addAggregateFunction->aggkind = AGGKIND_NORMAL; addAggregateFunction->aggfilter = (Expr *) copyObject( originalAggregate->aggfilter); workerAggregateList = lappend(workerAggregateList, addAggregateFunction); } else if (aggregateType == AGGREGATE_AVERAGE) { /* * If the original aggregate is an average, we want to compute sum(var) * and count(var) on worker nodes. */ Aggref *sumAggregate = copyObject(originalAggregate); Aggref *countAggregate = copyObject(originalAggregate); /* extract function names for sum and count */ const char *sumAggregateName = AggregateNames[AGGREGATE_SUM]; const char *countAggregateName = AggregateNames[AGGREGATE_COUNT]; /* * Find the type of the expression over which we execute the aggregate. * We then need to find the right sum function for that type. */ Oid argumentType = AggregateArgumentType(originalAggregate); /* find function implementing sum over the original type */ sumAggregate->aggfnoid = AggregateFunctionOid(sumAggregateName, argumentType); sumAggregate->aggtype = get_func_rettype(sumAggregate->aggfnoid); sumAggregate->aggtranstype = InvalidOid; sumAggregate->aggargtypes = list_make1_oid(argumentType); sumAggregate->aggsplit = AGGSPLIT_SIMPLE; /* count has any input type */ countAggregate->aggfnoid = AggregateFunctionOid(countAggregateName, ANYOID); countAggregate->aggtype = get_func_rettype(countAggregate->aggfnoid); countAggregate->aggtranstype = InvalidOid; countAggregate->aggargtypes = list_make1_oid(argumentType); countAggregate->aggsplit = AGGSPLIT_SIMPLE; workerAggregateList = lappend(workerAggregateList, sumAggregate); workerAggregateList = lappend(workerAggregateList, countAggregate); } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLEARRAY || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLEARRAY) { /* * The original query has an aggregate in the form of either * - tdigest_percentile(column, compression, quantile) * - tdigest_percentile(column, compression, quantile[]) * - tdigest_percentile_of(column, compression, value) * - tdigest_percentile_of(column, compression, value[]) * * We are creating the worker part of this query by creating a * - tdigest(column, compression) * * One could see we are passing argument 0 and argument 1 from the original query * in here. This corresponds with the list_nth calls in the args and aggargstypes * list construction. The tdigest function and type are read from the catalog. */ Aggref *newWorkerAggregate = copyObject(originalAggregate); newWorkerAggregate->aggfnoid = TDigestExtensionAggTDigest2(); newWorkerAggregate->aggtype = TDigestExtensionTypeOid(); newWorkerAggregate->args = list_make2( list_nth(newWorkerAggregate->args, 0), list_nth(newWorkerAggregate->args, 1)); newWorkerAggregate->aggkind = AGGKIND_NORMAL; newWorkerAggregate->aggtranstype = InvalidOid; newWorkerAggregate->aggargtypes = list_make2_oid( list_nth_oid(newWorkerAggregate->aggargtypes, 0), list_nth_oid(newWorkerAggregate->aggargtypes, 1)); newWorkerAggregate->aggsplit = AGGSPLIT_SIMPLE; workerAggregateList = lappend(workerAggregateList, newWorkerAggregate); } else if (aggregateType == AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLEARRAY || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLE || aggregateType == AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLEARRAY) { /* * The original query has an aggregate in the form of either * - tdigest_percentile(tdigest, quantile) * - tdigest_percentile(tdigest, quantile[]) * - tdigest_percentile_of(tdigest, value) * - tdigest_percentile_of(tdigest, value[]) * * We are creating the worker part of this query by creating a * - tdigest(tdigest) * * One could see we are passing argument 0 from the original query in here. This * corresponds with the list_nth calls in the args and aggargstypes list * construction. The tdigest function and type are read from the catalog. */ Aggref *newWorkerAggregate = copyObject(originalAggregate); newWorkerAggregate->aggfnoid = TDigestExtensionAggTDigest1(); newWorkerAggregate->aggtype = TDigestExtensionTypeOid(); newWorkerAggregate->args = list_make1(list_nth(newWorkerAggregate->args, 0)); newWorkerAggregate->aggkind = AGGKIND_NORMAL; newWorkerAggregate->aggtranstype = InvalidOid; newWorkerAggregate->aggargtypes = list_make1_oid( list_nth_oid(newWorkerAggregate->aggargtypes, 0)); newWorkerAggregate->aggsplit = AGGSPLIT_SIMPLE; workerAggregateList = lappend(workerAggregateList, newWorkerAggregate); } else if (aggregateType == AGGREGATE_CUSTOM_COMBINE) { HeapTuple aggTuple = SearchSysCache1(AGGFNOID, ObjectIdGetDatum(originalAggregate->aggfnoid)); Form_pg_aggregate aggform; Oid combine; bool useBinaryWorkerAggregate = false; if (!HeapTupleIsValid(aggTuple)) { elog(ERROR, "citus cache lookup failed for aggregate %u", originalAggregate->aggfnoid); return NULL; } else { aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryWorkerAggregate = (OidIsValid(aggform->aggtranstype) && IsAggTransTypeBinarySerializable(aggform)); ReleaseSysCache(aggTuple); } if (combine != InvalidOid) { Const *aggOidParam = makeConst(REGPROCEDUREOID, -1, InvalidOid, sizeof(Oid), ObjectIdGetDatum(originalAggregate->aggfnoid), false, true); List *newWorkerAggregateArgs = list_make1(makeTargetEntry((Expr *) aggOidParam, 1, NULL, false)); if (list_length(originalAggregate->args) == 1) { /* * Single argument case, append 'arg' to worker_partial_agg(agg, arg). * We don't wrap single argument in a row expression because * it has performance implications to unwrap arguments on each * SFUNC invocation. */ TargetEntry *newArg = copyObject((TargetEntry *) linitial(originalAggregate->args)); newArg->resno++; newWorkerAggregateArgs = lappend(newWorkerAggregateArgs, newArg); } else { /* * Aggregation on workers assumes a single aggregation parameter. * To still be able to handle multiple parameters, we combine * parameters into a single row expression, i.e., append 'ROW(...args)' * to worker_partial_agg(agg, ROW(...args)). */ RowExpr *rowExpr = makeNode(RowExpr); rowExpr->row_typeid = RECORDOID; rowExpr->row_format = COERCE_EXPLICIT_CALL; rowExpr->location = -1; rowExpr->colnames = NIL; TargetEntry *arg = NULL; foreach_declared_ptr(arg, originalAggregate->args) { rowExpr->args = lappend(rowExpr->args, copyObject(arg->expr)); } newWorkerAggregateArgs = lappend(newWorkerAggregateArgs, makeTargetEntry((Expr *) rowExpr, 2, NULL, false)); } /* worker_partial_agg(agg, arg) or worker_partial_agg(agg, ROW(...args)) */ Aggref *newWorkerAggregate = copyObject(originalAggregate); if (useBinaryWorkerAggregate) { newWorkerAggregate->aggfnoid = WorkerBinaryPartialAggOid(); newWorkerAggregate->aggtype = BYTEAOID; } else { newWorkerAggregate->aggfnoid = WorkerPartialAggOid(); newWorkerAggregate->aggtype = CSTRINGOID; } newWorkerAggregate->args = newWorkerAggregateArgs; newWorkerAggregate->aggkind = AGGKIND_NORMAL; newWorkerAggregate->aggtranstype = INTERNALOID; newWorkerAggregate->aggargtypes = lcons_oid(OIDOID, newWorkerAggregate->aggargtypes); newWorkerAggregate->aggsplit = AGGSPLIT_SIMPLE; workerAggregateList = list_make1(newWorkerAggregate); } else { elog(ERROR, "Aggregate lacks COMBINEFUNC"); } } else { /* * All other aggregates are sent as they are to the worker nodes. */ Aggref *workerAggregate = copyObject(originalAggregate); workerAggregateList = lappend(workerAggregateList, workerAggregate); } return workerAggregateList; } /* * GetAggregateType scans pg_catalog.pg_proc for the given aggregate oid, and * finds the aggregate's name. The function then matches the aggregate's name to * previously stored strings, and returns the appropriate aggregate type. */ static AggregateType GetAggregateType(Aggref *aggregateExpression) { Oid aggFunctionId = aggregateExpression->aggfnoid; /* custom aggregates with combine func take precedence over name-based logic */ if (aggFunctionId >= FirstNormalObjectId && AggregateEnabledCustom(aggregateExpression)) { return AGGREGATE_CUSTOM_COMBINE; } /* look up the function name */ char *aggregateProcName = get_func_name(aggFunctionId); if (aggregateProcName == NULL) { ereport(ERROR, (errmsg("citus cache lookup failed for function %u", aggFunctionId))); } uint32 aggregateCount = lengthof(AggregateNames); for (uint32 aggregateIndex = 1; aggregateIndex < aggregateCount; aggregateIndex++) { const char *aggregateName = AggregateNames[aggregateIndex]; if (strncmp(aggregateName, aggregateProcName, NAMEDATALEN) == 0) { return aggregateIndex; } } /* * All functions from github.com/tvondra/tdigest start with the "tdigest" prefix. * Since it requires lookups of function names in a schema we would like to only * perform these checks if there is some chance it will actually result in a positive * hit. */ if (StringStartsWith(aggregateProcName, "tdigest")) { if (aggFunctionId == TDigestExtensionAggTDigest1()) { return AGGREGATE_TDIGEST_COMBINE; } if (aggFunctionId == TDigestExtensionAggTDigest2()) { return AGGREGATE_TDIGEST_ADD_DOUBLE; } if (aggFunctionId == TDigestExtensionAggTDigestPercentile3()) { return AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLE; } if (aggFunctionId == TDigestExtensionAggTDigestPercentile3a()) { return AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLEARRAY; } if (aggFunctionId == TDigestExtensionAggTDigestPercentile2()) { return AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLE; } if (aggFunctionId == TDigestExtensionAggTDigestPercentile2a()) { return AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLEARRAY; } if (aggFunctionId == TDigestExtensionAggTDigestPercentileOf3()) { return AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLE; } if (aggFunctionId == TDigestExtensionAggTDigestPercentileOf3a()) { return AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLEARRAY; } if (aggFunctionId == TDigestExtensionAggTDigestPercentileOf2()) { return AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLE; } if (aggFunctionId == TDigestExtensionAggTDigestPercentileOf2a()) { return AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLEARRAY; } } /* handle any remaining built-in aggregates with a suitable combinefn */ if (AggregateEnabledCustom(aggregateExpression)) { return AGGREGATE_CUSTOM_COMBINE; } if (CoordinatorAggregationStrategy == COORDINATOR_AGGREGATION_DISABLED) { ereport(ERROR, (errmsg("unsupported aggregate function %s", aggregateProcName))); } else { return AGGREGATE_CUSTOM_ROW_GATHER; } } /* Extracts the type of the argument over which the aggregate is operating. */ static Oid AggregateArgumentType(Aggref *aggregate) { List *argumentList = aggregate->args; TargetEntry *argument = (TargetEntry *) linitial(argumentList); Oid returnTypeId = exprType((Node *) argument->expr); /* Here we currently support aggregates with only one argument; assert that. */ Assert(list_length(argumentList) == 1); return returnTypeId; } /* * FirstAggregateArgument returns the first argument of the aggregate. */ static Expr * FirstAggregateArgument(Aggref *aggregate) { List *argumentList = aggregate->args; Assert(list_length(argumentList) >= 1); TargetEntry *argument = (TargetEntry *) linitial(argumentList); return argument->expr; } /* * AggregateEnabledCustom returns whether given aggregate can be * distributed across workers using worker_partial_agg & coord_combine_agg. */ static bool AggregateEnabledCustom(Aggref *aggregateExpression) { if (aggregateExpression->aggorder != NIL || list_length(aggregateExpression->args) == 0) { return false; } Oid aggregateOid = aggregateExpression->aggfnoid; HeapTuple aggTuple = SearchSysCache1(AGGFNOID, aggregateOid); if (!HeapTupleIsValid(aggTuple)) { elog(ERROR, "citus cache lookup failed."); } Form_pg_aggregate aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); if (aggform->aggcombinefn == InvalidOid) { ReleaseSysCache(aggTuple); return false; } HeapTuple typeTuple = SearchSysCache1(TYPEOID, aggform->aggtranstype); if (!HeapTupleIsValid(typeTuple)) { elog(ERROR, "citus cache lookup failed."); } Form_pg_type typeform = (Form_pg_type) GETSTRUCT(typeTuple); bool supportsSafeCombine = typeform->typtype != TYPTYPE_PSEUDO; if (AllowAggregateWorkerCombineOnInternalTypes && typeform->oid == INTERNALOID && !supportsSafeCombine) { /* check if the type supports a SERIALFUNC/DESERIALFUNC - if it does * then we can leverage that for safe transfer of the state across the wire. */ if (aggform->aggserialfn != InvalidOid && aggform->aggdeserialfn != InvalidOid) { supportsSafeCombine = true; } } ReleaseSysCache(aggTuple); ReleaseSysCache(typeTuple); return supportsSafeCombine; } /* * AggregateArgMatchLevel and AggregateArgumentMatchLevel() * * Citus needs to resolve an aggregate function OID by (name, argument type) * when planning distributed aggregates. In the multi-shard path we run the * aggregate on each shard (worker tasks) and then build a coordinator-side * “combine” aggregate over the per-shard results. To construct that master * aggregate expression, we must find the correct underlying Postgres aggregate * implementation (OID). * * Postgres defines many aggregates using polymorphic pseudo-types rather than * concrete types. For example, min/max are defined for: * - anyarray (e.g., int[], text[]) * - anyenum (e.g., a user-defined enum type) * - anyelement (e.g., int4, text, numeric) * - record (e.g., a named composite/row type) * so an “exact type only” lookup can miss the right candidate and fail with * "no matching oid for function". * * AggregateArgMatchLevel is a ranking of how well a candidate aggregate * declaration matches the input type. AggregateArgumentMatchLevel() computes * that rank for a pair of types: * * declaredArgType: the aggregate's declared argument type taken from a * pg_proc candidate (e.g., ANYARRAYOID, ANYELEMENTOID, RECORDOID, or a * concrete type OID). * * inputType: the actual argument type of the user query expression for which * we are resolving the aggregate (e.g., INT4OID for int, the array type OID * for int[], or a rowtype OID for a composite type column). * * The OID resolution logic scans candidate aggregates and selects the best * match (highest rank), preferring: * 1) AGG_MATCH_EXACT: * declaredArgType == inputType * Example: min(int4) with inputType = INT4OID. * * 2) AGG_MATCH_ARRAY_POLY: * declaredArgType is ANYARRAY and inputType is an * array type. * Example: min(int[]) matches min(anyarray). * * 3) AGG_MATCH_GENERAL_POLY: * declaredArgType is ANYELEMENT/ANYENUM and is compatible * with inputType. * Example: min(mood_enum) matches min(anyenum), or min(text) matches a * polymorphic min(anyelement). * * 4) AGG_MATCH_RECORD: * declaredArgType is RECORD and inputType is a rowtype/composite. * Example: min(product_rating) matches min(record). * * This makes aggregate OID resolution robust across PG versions and additional * polymorphic signatures introduced in PG18 (notably for min/max). */ typedef enum AggregateArgMatchLevel { AGG_MATCH_NONE = 0, AGG_MATCH_RECORD = 1, AGG_MATCH_GENERAL_POLY = 2, AGG_MATCH_ARRAY_POLY = 3, AGG_MATCH_EXACT = 4 } AggregateArgMatchLevel; static AggregateArgMatchLevel AggregateArgumentMatchLevel(Oid declaredArgType, Oid inputType) { if (declaredArgType == inputType) { return AGG_MATCH_EXACT; } bool inputIsArray = (inputType == ANYARRAYOID) || type_is_array(inputType); bool inputIsEnum = (inputType == ANYENUMOID) || type_is_enum(inputType); switch (declaredArgType) { case ANYARRAYOID: { return inputIsArray ? AGG_MATCH_ARRAY_POLY : AGG_MATCH_NONE; } case ANYELEMENTOID: { return AGG_MATCH_GENERAL_POLY; } case ANYENUMOID: { return inputIsEnum ? AGG_MATCH_GENERAL_POLY : AGG_MATCH_NONE; } case RECORDOID: { return type_is_rowtype(inputType) ? AGG_MATCH_RECORD : AGG_MATCH_NONE; } default: { return AGG_MATCH_NONE; } } } /* * AggregateFunctionOid performs a reverse lookup on aggregate function name, * and returns the corresponding aggregate function oid for the given function * name and input type. */ static Oid AggregateFunctionOid(const char *functionName, Oid inputType) { Oid functionOid = InvalidOid; AggregateArgMatchLevel bestMatch = AGG_MATCH_NONE; ScanKeyData scanKey[1]; int scanKeyCount = 1; Relation procRelation = table_open(ProcedureRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_proc_proname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(functionName)); SysScanDesc scanDescriptor = systable_beginscan(procRelation, ProcedureNameArgsNspIndexId, true, NULL, scanKeyCount, scanKey); /* loop until we find the right function */ HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_proc procForm = (Form_pg_proc) GETSTRUCT(heapTuple); int argumentCount = procForm->pronargs; if (argumentCount == 1) { Oid declaredArgType = procForm->proargtypes.values[0]; AggregateArgMatchLevel matchLevel = AggregateArgumentMatchLevel(declaredArgType, inputType); if (matchLevel > bestMatch) { bestMatch = matchLevel; functionOid = procForm->oid; if (bestMatch == AGG_MATCH_EXACT) { break; } } } Assert(argumentCount <= 1); heapTuple = systable_getnext(scanDescriptor); } if (functionOid == InvalidOid) { ereport(ERROR, (errmsg("no matching oid for function: %s", functionName))); } systable_endscan(scanDescriptor); table_close(procRelation, AccessShareLock); return functionOid; } /* * CitusFunctionOidWithSignature looks up a function with given input types. * Looks in pg_catalog schema, as this function's sole purpose is * support aggregate lookup. */ static Oid CitusFunctionOidWithSignature(char *functionName, int numargs, Oid *argtypes) { List *aggregateName = list_make2(makeString("pg_catalog"), makeString(functionName)); FuncCandidateList clist = FuncnameGetCandidates(aggregateName, numargs, NIL, false, false, false, true); for (; clist; clist = clist->next) { if (memcmp(clist->args, argtypes, numargs * sizeof(Oid)) == 0) { return clist->oid; } } ereport(ERROR, (errmsg("no matching oid for function: %s", functionName))); return InvalidOid; } /* * WorkerPartialAggOid looks up oid of pg_catalog.worker_partial_agg */ static Oid WorkerPartialAggOid() { Oid argtypes[] = { OIDOID, ANYELEMENTOID, }; return CitusFunctionOidWithSignature(WORKER_PARTIAL_AGGREGATE_NAME, 2, argtypes); } /* * CoordCombineAggOid looks up oid of pg_catalog.coord_combine_agg */ static Oid CoordCombineAggOid() { Oid argtypes[] = { OIDOID, CSTRINGOID, ANYELEMENTOID, }; return CitusFunctionOidWithSignature(COORD_COMBINE_AGGREGATE_NAME, 3, argtypes); } /* * WorkerBinaryPartialAggOid looks up oid of pg_catalog.worker_binary_partial_agg */ static Oid WorkerBinaryPartialAggOid() { Oid argtypes[] = { OIDOID, ANYELEMENTOID, }; return CitusFunctionOidWithSignature(WORKER_BINARY_PARTIAL_AGGREGATE_NAME, 2, argtypes ); } /* * CoordBinaryCombineAggOid looks up oid of pg_catalog.coord_binary_combine_agg */ static Oid CoordBinaryCombineAggOid() { Oid argtypes[] = { OIDOID, BYTEAOID, ANYELEMENTOID, }; return CitusFunctionOidWithSignature(COORD_BINARY_COMBINE_AGGREGATE_NAME, 3, argtypes) ; } /* * TypeOid looks for a type that has the given name and schema, and returns the * corresponding type's oid. */ static Oid TypeOid(Oid schemaId, const char *typeName) { Oid typeOid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, PointerGetDatum(typeName), ObjectIdGetDatum(schemaId)); return typeOid; } static bool IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm) { Oid transitionType = aggForm->aggtranstype; if (AllowAggregateWorkerCombineOnInternalTypes && transitionType == INTERNALOID) { /* For aggregates with internal transition types, we apply the binary serialization * check on the output value of the SERIALFUNC. If a serialfunc exists, Postgres * requires that the serialfunc return a bytea - which will be binary serializable */ return (aggForm->aggserialfn != InvalidOid); } HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(transitionType)); if (!HeapTupleIsValid(typeTuple)) { elog(ERROR, "citus cache lookup failed for transition type %u", transitionType); } Form_pg_type typeForm = (Form_pg_type) GETSTRUCT(typeTuple); bool isBinaryCoercible = typeForm->typsend != InvalidOid && typeForm->typreceive != InvalidOid; ReleaseSysCache(typeTuple); return isBinaryCoercible; } /* * CreateSortGroupClause creates SortGroupClause for a given column Var. * The caller should set tleSortGroupRef field and respective * TargetEntry->ressortgroupref fields to appropriate SortGroupRefIndex. */ static SortGroupClause * CreateSortGroupClause(Var *column) { Oid lessThanOperator = InvalidOid; Oid equalsOperator = InvalidOid; bool hashable = false; SortGroupClause *groupByClause = makeNode(SortGroupClause); get_sort_group_operators(column->vartype, true, true, true, &lessThanOperator, &equalsOperator, NULL, &hashable); groupByClause->eqop = equalsOperator; groupByClause->hashable = hashable; groupByClause->nulls_first = false; groupByClause->sortop = lessThanOperator; return groupByClause; } /* * CountDistinctHashFunctionName resolves the hll_hash function name to use for * the given input type, and returns this function name. */ static const char * CountDistinctHashFunctionName(Oid argumentType) { /* resolve hash function name based on input argument type */ switch (argumentType) { case INT4OID: { return HLL_HASH_INTEGER_FUNC_NAME; } case INT8OID: { return HLL_HASH_BIGINT_FUNC_NAME; } case TEXTOID: case BPCHAROID: case VARCHAROID: { return HLL_HASH_TEXT_FUNC_NAME; } default: { return HLL_HASH_ANY_FUNC_NAME; } } } /* * CountDistinctStorageSize takes in the desired precision for count distinct * approximations, and returns the log-base-2 of storage space needed for the * HyperLogLog algorithm. */ static int CountDistinctStorageSize(double approximationErrorRate) { double desiredStorageSize = pow((1.04 / approximationErrorRate), 2); double logOfDesiredStorageSize = log(desiredStorageSize) / log(2); /* keep log2(storage size) inside allowed range */ int logOfStorageSize = (int) rint(logOfDesiredStorageSize); if (logOfStorageSize < 4) { logOfStorageSize = 4; } else if (logOfStorageSize > 17) { logOfStorageSize = 17; } return logOfStorageSize; } /* Makes an integer constant node from the given value, and returns that node. */ static Const * MakeIntegerConst(int32 integerValue) { const int typeCollationId = get_typcollation(INT4OID); const int16 typeLength = get_typlen(INT4OID); const int32 typeModifier = -1; const bool typeIsNull = false; const bool typePassByValue = true; Datum integerDatum = Int32GetDatum(integerValue); Const *integerConst = makeConst(INT4OID, typeModifier, typeCollationId, typeLength, integerDatum, typeIsNull, typePassByValue); return integerConst; } /* Makes a 64-bit integer constant node from the given value, and returns that node. */ static Const * MakeIntegerConstInt64(int64 integerValue) { const int typeCollationId = get_typcollation(INT8OID); const int16 typeLength = get_typlen(INT8OID); const int32 typeModifier = -1; const bool typeIsNull = false; const bool typePassByValue = true; Datum integer64Datum = Int64GetDatum(integerValue); Const *integer64Const = makeConst(INT8OID, typeModifier, typeCollationId, typeLength, integer64Datum, typeIsNull, typePassByValue); return integer64Const; } /* * HasNonDistributableAggregates checks for if any aggregates cannot be pushed down. * This only checks with GetAggregateType. DeferErrorIfHasNonDistributableAggregates * performs further checks which should be done if aggregates are not being pushed down. */ static bool HasNonDistributableAggregates(MultiNode *logicalPlanNode) { if (CoordinatorAggregationStrategy == COORDINATOR_AGGREGATION_DISABLED) { return false; } List *opNodeList = FindNodesOfType(logicalPlanNode, T_MultiExtendedOp); MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) linitial(opNodeList); List *targetList = extendedOpNode->targetList; Node *havingQual = extendedOpNode->havingQual; /* * PVC_REJECT_PLACEHOLDERS is implicit if PVC_INCLUDE_PLACEHOLDERS isn't * specified. */ List *expressionList = pull_var_clause((Node *) targetList, PVC_INCLUDE_AGGREGATES | PVC_INCLUDE_WINDOWFUNCS); expressionList = list_concat(expressionList, pull_var_clause(havingQual, PVC_INCLUDE_AGGREGATES)); Node *expression = NULL; foreach_declared_ptr(expression, expressionList) { /* only consider aggregate expressions */ if (!IsA(expression, Aggref)) { continue; } AggregateType aggregateType = GetAggregateType((Aggref *) expression); Assert(aggregateType != AGGREGATE_INVALID_FIRST); if (aggregateType == AGGREGATE_CUSTOM_ROW_GATHER) { return true; } } return false; } /* * CanPushDownExpression returns whether the expression can be pushed down to workers. */ static bool CanPushDownExpression(Node *expression, const ExtendedOpNodeProperties *extendedOpNodeProperties) { if (contain_nextval_expression_walker(expression, NULL)) { /* nextval can only be evaluated on the coordinator */ return false; } bool hasAggregate = contain_aggs_of_level(expression, 0); bool hasWindowFunction = contain_window_function(expression); if (!hasAggregate && !hasWindowFunction) { /* * If the query has the form SELECT expression, agg(..) FROM table; * then expression should be evaluated on the coordinator. * * Other than the efficiency part of this, we could also crash if * we pushed down the expression to the workers. When pushing down * expressions to workers we create a Var reference to the worker * tuples. If the result from worker is empty, but we need to have * at least a row in coordinator result, postgres will crash when * trying to evaluate the Var. * * For details, see https://github.com/citusdata/citus/pull/3961 */ if (!extendedOpNodeProperties->hasAggregate || extendedOpNodeProperties->hasGroupBy) { return true; } } /* aggregates inside pushed down window functions can be pushed down */ bool hasPushableWindowFunction = hasWindowFunction && extendedOpNodeProperties->onlyPushableWindowFunctions; if (hasPushableWindowFunction) { return true; } if (extendedOpNodeProperties->pushDownGroupingAndHaving && !hasWindowFunction) { return true; } if (hasAggregate && !hasWindowFunction && extendedOpNodeProperties->groupedByDisjointPartitionColumn) { return true; } return false; } /* * DeferErrorIfHasNonDistributableAggregates extracts aggregate expressions from * the logical plan, walks over them and uses helper functions to check if we * can transform these aggregate expressions and push them down to worker nodes. */ static DeferredErrorMessage * DeferErrorIfHasNonDistributableAggregates(MultiNode *logicalPlanNode) { DeferredErrorMessage *error = NULL; List *opNodeList = FindNodesOfType(logicalPlanNode, T_MultiExtendedOp); MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) linitial(opNodeList); List *targetList = extendedOpNode->targetList; Node *havingQual = extendedOpNode->havingQual; /* * PVC_REJECT_PLACEHOLDERS is implicit if PVC_INCLUDE_PLACEHOLDERS isn't * specified. */ List *expressionList = pull_var_clause((Node *) targetList, PVC_INCLUDE_AGGREGATES | PVC_INCLUDE_WINDOWFUNCS); expressionList = list_concat(expressionList, pull_var_clause(havingQual, PVC_INCLUDE_AGGREGATES)); Node *expression = NULL; foreach_declared_ptr(expression, expressionList) { /* only consider aggregate expressions */ if (!IsA(expression, Aggref)) { continue; } /* GetAggregateType errors out on unsupported aggregate types */ Aggref *aggregateExpression = (Aggref *) expression; AggregateType aggregateType = GetAggregateType(aggregateExpression); Assert(aggregateType != AGGREGATE_INVALID_FIRST); /* * Check that we can transform the current aggregate expression. These * functions error out on unsupported array_agg and aggregate (distinct) * clauses. */ if (aggregateType == AGGREGATE_ARRAY_AGG) { error = DeferErrorIfUnsupportedArrayAggregate(aggregateExpression); } else if (aggregateType == AGGREGATE_JSONB_AGG || aggregateType == AGGREGATE_JSON_AGG) { error = DeferErrorIfUnsupportedJsonAggregate(aggregateType, aggregateExpression); } else if (aggregateType == AGGREGATE_JSONB_OBJECT_AGG || aggregateType == AGGREGATE_JSON_OBJECT_AGG) { error = DeferErrorIfUnsupportedJsonAggregate(aggregateType, aggregateExpression); } else if (aggregateExpression->aggdistinct) { error = DeferErrorIfUnsupportedAggregateDistinct(aggregateExpression, logicalPlanNode); } if (error != NULL) { return error; } } return NULL; } /* * DeferErrorIfUnsupportedArrayAggregate checks if we can transform the array aggregate * expression and push it down to the worker node. If we cannot transform the * aggregate, this function errors. */ static DeferredErrorMessage * DeferErrorIfUnsupportedArrayAggregate(Aggref *arrayAggregateExpression) { /* if array_agg has order by, we error out */ if (arrayAggregateExpression->aggorder) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "array_agg with order by is unsupported", NULL, NULL); } /* if array_agg has distinct, we error out */ if (arrayAggregateExpression->aggdistinct) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "array_agg (distinct) is unsupported", NULL, NULL); } return NULL; } /* * DeferErrorIfUnsupportedJsonAggregate checks if we can transform the json * aggregate expression and push it down to the worker node. If we cannot * transform the aggregate, this function errors. */ static DeferredErrorMessage * DeferErrorIfUnsupportedJsonAggregate(AggregateType type, Aggref *aggregateExpression) { /* if json aggregate has order by, we error out */ if (aggregateExpression->aggdistinct || aggregateExpression->aggorder) { StringInfoData errorDetail; initStringInfo(&errorDetail); const char *name = AggregateNames[type]; appendStringInfoString(&errorDetail, name); if (aggregateExpression->aggorder) { appendStringInfoString(&errorDetail, " with order by is unsupported"); } else { appendStringInfoString(&errorDetail, " (distinct) is unsupported"); } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorDetail.data, NULL, NULL); } return NULL; } /* * DeferErrorIfUnsupportedAggregateDistinct checks if we can transform the aggregate * (distinct expression) and push it down to the worker node. It handles count * (distinct) separately to check if we can use distinct approximations. If we * cannot transform the aggregate, this function errors. */ static DeferredErrorMessage * DeferErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, MultiNode *logicalPlanNode) { const char *errorDetail = NULL; bool distinctSupported = true; AggregateType aggregateType = GetAggregateType(aggregateExpression); /* If we're aggregating on coordinator, this becomes simple. */ if (aggregateType == AGGREGATE_CUSTOM_ROW_GATHER) { return NULL; } /* * We partially support count(distinct) in subqueries, other distinct aggregates in * subqueries are not supported yet. */ if (aggregateType == AGGREGATE_COUNT) { Node *aggregateArgument = (Node *) linitial(aggregateExpression->args); List *columnList = pull_var_clause_default(aggregateArgument); Var *column = NULL; foreach_declared_ptr(column, columnList) { if (column->varattno <= 0) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot compute count (distinct)", "Non-column references are not supported yet", NULL); } } } else { List *multiTableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); MultiTable *multiTable = NULL; foreach_declared_ptr(multiTable, multiTableNodeList) { if (multiTable->relationId == SUBQUERY_RELATION_ID || multiTable->relationId == SUBQUERY_PUSHDOWN_RELATION_ID) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot compute aggregate (distinct)", "Only count(distinct) aggregate is " "supported in subqueries", NULL); } } } /* if we have a count(distinct), and distinct approximation is enabled */ if (aggregateType == AGGREGATE_COUNT && CountDistinctErrorRate != DISABLE_DISTINCT_APPROXIMATION) { bool missingOK = true; Oid distinctExtensionId = get_extension_oid(HLL_EXTENSION_NAME, missingOK); /* if extension for distinct approximation is loaded, we are good */ if (distinctExtensionId != InvalidOid) { return NULL; } else { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot compute count (distinct) approximation", NULL, "You need to have the hll extension loaded."); } } if (aggregateType == AGGREGATE_COUNT) { List *aggregateVarList = pull_var_clause_default((Node *) aggregateExpression); if (aggregateVarList == NIL) { distinctSupported = false; errorDetail = "aggregate (distinct) with no columns is unsupported"; } } List *repartitionNodeList = FindNodesOfType(logicalPlanNode, T_MultiPartition); if (repartitionNodeList != NIL) { distinctSupported = false; errorDetail = "aggregate (distinct) with table repartitioning is unsupported"; } List *tableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); List *extendedOpNodeList = FindNodesOfType(logicalPlanNode, T_MultiExtendedOp); MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) linitial(extendedOpNodeList); Var *distinctColumn = AggregateDistinctColumn(aggregateExpression); if (distinctSupported) { if (distinctColumn == NULL) { /* * If the query has a single table, and table is grouped by partition * column, then we support count distincts even distinct column can * not be identified. */ distinctSupported = TablePartitioningSupportsDistinct(tableNodeList, extendedOpNode, distinctColumn, aggregateType); if (!distinctSupported) { errorDetail = "aggregate (distinct) on complex expressions is" " unsupported"; } } else if (aggregateType != AGGREGATE_COUNT) { bool supports = TablePartitioningSupportsDistinct(tableNodeList, extendedOpNode, distinctColumn, aggregateType); if (!supports) { distinctSupported = false; errorDetail = "table partitioning is unsuitable for aggregate (distinct)"; } } } /* if current aggregate expression isn't supported, error out */ if (!distinctSupported) { const char *errorHint = NULL; if (aggregateType == AGGREGATE_COUNT) { errorHint = "You can load the hll extension from contrib " "packages and enable distinct approximations."; } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot compute aggregate (distinct)", errorDetail, errorHint); } return NULL; } /* * AggregateDistinctColumn checks if the given aggregate expression's distinct * clause is on a single column. If it is, the function finds and returns that * column. Otherwise, the function returns null. * The function expects to find a single column here, no FieldSelect or other * expressions are accepted as a column. */ static Var * AggregateDistinctColumn(Aggref *aggregateExpression) { /* only consider aggregates with distincts */ if (!aggregateExpression->aggdistinct) { return NULL; } int aggregateArgumentCount = list_length(aggregateExpression->args); if (aggregateArgumentCount != 1) { return NULL; } TargetEntry *aggregateTargetEntry = (TargetEntry *) linitial( aggregateExpression->args); if (!IsA(aggregateTargetEntry->expr, Var)) { return NULL; } Var *aggregateColumn = (Var *) aggregateTargetEntry->expr; return aggregateColumn; } /* * TablePartitioningSupportsDistinct walks over all tables in the given list and * checks that each table's partitioning method is suitable for pushing down an * aggregate (distinct) expression to worker nodes. For this, the function needs * to check that task results do not overlap with one another on the distinct * column. */ static bool TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode, Var *distinctColumn, AggregateType aggregateType) { bool distinctSupported = true; MultiTable *tableNode = NULL; foreach_declared_ptr(tableNode, tableNodeList) { Oid relationId = tableNode->relationId; bool tableDistinctSupported = false; if (relationId == SUBQUERY_RELATION_ID || relationId == SUBQUERY_PUSHDOWN_RELATION_ID) { return true; } /* if table has one shard, task results don't overlap */ List *shardList = LoadShardList(relationId); if (list_length(shardList) == 1) { continue; } /* * We need to check that task results don't overlap. We can only do this * if table is range partitioned. */ if (IsCitusTableType(relationId, RANGE_DISTRIBUTED) || IsCitusTableType(relationId, HASH_DISTRIBUTED)) { Var *tablePartitionColumn = tableNode->partitionColumn; if (aggregateType == AGGREGATE_COUNT) { tableDistinctSupported = true; } /* if distinct is on table partition column, we can push it down */ if (distinctColumn != NULL && tablePartitionColumn->varno == distinctColumn->varno && tablePartitionColumn->varattno == distinctColumn->varattno) { tableDistinctSupported = true; } /* if results are grouped by partition column, we can push down */ bool groupedByPartitionColumn = GroupedByColumn(opNode->groupClauseList, opNode->targetList, tablePartitionColumn); if (groupedByPartitionColumn) { tableDistinctSupported = true; } } if (!tableDistinctSupported) { distinctSupported = false; break; } } return distinctSupported; } /* * GroupedByColumn walks over group clauses in the given list, and checks if any * of the group clauses is on the given column. */ bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column) { bool groupedByColumn = false; if (column == NULL) { return false; } SortGroupClause *groupClause = NULL; foreach_declared_ptr(groupClause, groupClauseList) { TargetEntry *groupTargetEntry = get_sortgroupclause_tle(groupClause, targetList); Expr *groupExpression = (Expr *) groupTargetEntry->expr; if (IsA(groupExpression, Var)) { Var *groupColumn = (Var *) groupExpression; if (groupColumn->varno == column->varno && groupColumn->varattno == column->varattno) { groupedByColumn = true; break; } } } return groupedByColumn; } /* * SubqueryMultiTableList extracts multi tables in the given logical plan tree * and returns subquery multi tables in a new list. */ List * SubqueryMultiTableList(MultiNode *multiNode) { List *subqueryMultiTableList = NIL; List *multiTableNodeList = FindNodesOfType(multiNode, T_MultiTable); MultiTable *multiTable = NULL; foreach_declared_ptr(multiTable, multiTableNodeList) { Query *subquery = multiTable->subquery; if (subquery != NULL) { subqueryMultiTableList = lappend(subqueryMultiTableList, multiTable); } } return subqueryMultiTableList; } /* * GroupTargetEntryList walks over group clauses in the given list, finds * matching target entries and return them in a new list. */ List * GroupTargetEntryList(List *groupClauseList, List *targetEntryList) { List *groupTargetEntryList = NIL; SortGroupClause *groupClause = NULL; foreach_declared_ptr(groupClause, groupClauseList) { TargetEntry *groupTargetEntry = get_sortgroupclause_tle(groupClause, targetEntryList); groupTargetEntryList = lappend(groupTargetEntryList, groupTargetEntry); } return groupTargetEntryList; } /* * IsPartitionColumn returns true if the given column is a partition column. * The function uses FindReferencedTableColumn to find the original relation * id and column that the column expression refers to. It then checks whether * that column is a partition column of the relation. * * Also, the function returns always false for reference tables given that * reference tables do not have partition column. The function does not * support queries with CTEs, it would return false if columnExpression * refers to a column returned by a CTE. * * If skipOuterVars is true, then it doesn't process the outervars. */ bool IsPartitionColumn(Expr *columnExpression, Query *query, bool skipOuterVars) { bool isPartitionColumn = false; Var *column = NULL; RangeTblEntry *relationRTE = NULL; FindReferencedTableColumn(columnExpression, NIL, query, &column, &relationRTE, skipOuterVars); Oid relationId = relationRTE ? relationRTE->relid : InvalidOid; if (relationId != InvalidOid && column != NULL) { Var *partitionColumn = DistPartitionKey(relationId); /* not all distributed tables have partition column */ if (partitionColumn != NULL && column->varattno == partitionColumn->varattno) { isPartitionColumn = true; } } return isPartitionColumn; } /* * FindReferencedTableColumn recursively traverses query tree to find actual relation * id, and column that columnExpression refers to. If columnExpression is a * non-relational or computed/derived expression, the function returns NULL for * rte and NULL for column. The caller should provide parent query list from * top of the tree to this particular Query's parent. This argument is used to look * into CTEs that may be present in the query. * * If skipOuterVars is true, then it doesn't check vars coming from outer queries. * We probably don't need this skipOuterVars check but we wanted to be on the safe side * and used it only in UNION path, we can separately work on verifying that it doesn't break * anything existing. */ void FindReferencedTableColumn(Expr *columnExpression, List *parentQueryList, Query *query, Var **column, RangeTblEntry **rteContainingReferencedColumn, bool skipOuterVars) { Var *candidateColumn = NULL; Expr *strippedColumnExpression = (Expr *) strip_implicit_coercions( (Node *) columnExpression); *rteContainingReferencedColumn = NULL; *column = NULL; if (IsA(strippedColumnExpression, Var)) { candidateColumn = (Var *) strippedColumnExpression; } else if (IsA(strippedColumnExpression, FieldSelect)) { FieldSelect *compositeField = (FieldSelect *) strippedColumnExpression; Expr *fieldExpression = compositeField->arg; if (IsA(fieldExpression, Var)) { candidateColumn = (Var *) fieldExpression; } } if (candidateColumn == NULL) { return; } /* Walk up varlevelsup as many times as needed */ while (candidateColumn->varlevelsup > 0) { /* Caller asked us to ignore any outer Vars → just bail out */ if (skipOuterVars) { return; } /* Locate the parent query that owns this Var */ int parentIdx = list_length(parentQueryList) - candidateColumn->varlevelsup; if (!IsIndexInRange(parentQueryList, parentIdx)) { return; /* malformed tree */ } /* Work on a fresh copy of the Var with varlevelsup reset */ candidateColumn = copyObject(candidateColumn); candidateColumn->varlevelsup = 0; /* * Make a *completely private* copy of parentQueryList for the * next recursion step. We copy the whole list and then truncate * so every recursive branch owns its own list cells. */ List *newParent = list_copy(parentQueryList); /* duplicates every cell */ newParent = list_truncate(newParent, parentIdx); query = list_nth(parentQueryList, parentIdx); parentQueryList = newParent; /* hand private copy down */ /* Loop again if still pointing to an outer level */ } if (candidateColumn->varattno == InvalidAttrNumber) { /* * varattno can be 0 in case of SELECT table FROM table, but that Var * definitely does not correspond to a specific column. */ return; } List *rangetableList = query->rtable; int rangeTableEntryIndex = candidateColumn->varno - 1; RangeTblEntry *rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_RELATION) { *rteContainingReferencedColumn = rangeTableEntry; *column = candidateColumn; } else if (rangeTableEntry->rtekind == RTE_SUBQUERY) { Query *subquery = rangeTableEntry->subquery; List *targetEntryList = subquery->targetList; AttrNumber targetEntryIndex = candidateColumn->varattno - 1; TargetEntry *subqueryTargetEntry = list_nth(targetEntryList, targetEntryIndex); Expr *subColumnExpression = subqueryTargetEntry->expr; /* append current query to parent query list */ parentQueryList = lappend(parentQueryList, query); FindReferencedTableColumn(subColumnExpression, parentQueryList, subquery, column, rteContainingReferencedColumn, skipOuterVars); } else if (rangeTableEntry->rtekind == RTE_JOIN) { List *joinColumnList = rangeTableEntry->joinaliasvars; AttrNumber joinColumnIndex = candidateColumn->varattno - 1; Expr *joinColumn = list_nth(joinColumnList, joinColumnIndex); /* parent query list stays the same since still in the same query boundary */ FindReferencedTableColumn(joinColumn, parentQueryList, query, column, rteContainingReferencedColumn, skipOuterVars); } #if PG_VERSION_NUM >= PG_VERSION_18 else if (rangeTableEntry->rtekind == RTE_GROUP) { /* * PG 18: synthetic GROUP RTE. Each groupexprs item corresponds to the * columns produced by the grouping step, in the *same ordinal order* as * the Vars that reference them. */ List *groupexprs = rangeTableEntry->groupexprs; AttrNumber groupIndex = candidateColumn->varattno - 1; /* this must always hold unless upstream Postgres mis-constructed the RTE_GROUP */ Assert(groupIndex >= 0 && groupIndex < list_length(groupexprs)); Expr *groupExpr = (Expr *) list_nth(groupexprs, groupIndex); /* Recurse on the underlying expression (stay in the same query) */ FindReferencedTableColumn(groupExpr, parentQueryList, query, column, rteContainingReferencedColumn, skipOuterVars); } #endif /* PG_VERSION_NUM >= 180000 */ else if (rangeTableEntry->rtekind == RTE_CTE) { /* * Resolve through a CTE even when skipOuterVars == false. * Maintain the invariant that each recursion level owns a private, * correctly-bounded copy of parentQueryList. */ int cteParentListIndex = list_length(parentQueryList) - rangeTableEntry->ctelevelsup - 1; Query *cteParentQuery = NULL; List *cteList = NIL; CommonTableExpr *cte = NULL; /* * This should have been an error case, not marking it as error at the * moment due to usage from IsPartitionColumn. Callers of that function * do not have access to parent query list. */ if (IsIndexInRange(parentQueryList, cteParentListIndex)) { cteParentQuery = list_nth(parentQueryList, cteParentListIndex); cteList = cteParentQuery->cteList; } CommonTableExpr *candidateCte = NULL; foreach_declared_ptr(candidateCte, cteList) { if (strcmp(candidateCte->ctename, rangeTableEntry->ctename) == 0) { cte = candidateCte; break; } } if (cte != NULL) { Query *cteQuery = (Query *) cte->ctequery; AttrNumber targetEntryIndex = candidateColumn->varattno - 1; if (targetEntryIndex >= 0 && targetEntryIndex < list_length(cteQuery->targetList)) { TargetEntry *targetEntry = list_nth(cteQuery->targetList, targetEntryIndex); /* Build a private, bounded parentQueryList before recursing into the CTE. * Invariant: list is [top … current], owned by this call (no aliasing). * For RTE_CTE: * owner_idx = list_length(parentQueryList) - rangeTableEntry->ctelevelsup - 1; * newParent = lappend(list_truncate(list_copy(parentQueryList), owner_idx + 1), query); * Example (Q0 owns CTE; we’re in Q2 via nested subquery): * parent=[Q0,Q1,Q2], ctelevelsup=2 ⇒ owner_idx=0 ⇒ newParent=[Q0,Q2]. * Keeps outer-Var level math correct without mutating the caller’s list. */ List *newParent = list_copy(parentQueryList); newParent = list_truncate(newParent, cteParentListIndex + 1); newParent = lappend(newParent, query); FindReferencedTableColumn(targetEntry->expr, newParent, cteQuery, column, rteContainingReferencedColumn, skipOuterVars); } } } } /* * IsIndexInRange returns true if the given index is within the * range of the given list. */ static bool IsIndexInRange(const List *list, int index) { return index >= 0 && index < list_length(list); } /* * ExtractQueryWalker walks over a query, and finds all queries in the query * tree and returns these queries. Note that the function also recurses into * the subqueries in WHERE clause. */ bool ExtractQueryWalker(Node *node, List **queryList) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; (*queryList) = lappend(*queryList, query); return query_tree_walker(query, ExtractQueryWalker, queryList, 0); } return expression_tree_walker(node, ExtractQueryWalker, queryList); } /* * WorkerLimitCount checks if the given input contains a valid limit node, and * if that node can be pushed down. For this, the function checks if this limit * count or a meaningful approximation of it can be pushed down to worker nodes. * If they can, the function returns the limit count. * * The limit push-down decision tree is as follows: * group by? * 1/ \0 * group by partition column? (exact pd) * 0/ \1 * order by? (exact pd) * 1/ \0 * has order by agg? (no pd) * 1/ \0 * can approximate? (exact pd) * 1/ \0 * (approx pd) (no pd) * * When an offset is present, the offset value is added to limit because for a query * with LIMIT x OFFSET y, (x+y) records should be pulled from the workers. * * If no limit is present or can be pushed down, then WorkerLimitCount * returns null. */ static Node * WorkerLimitCount(Node *limitCount, Node *limitOffset, OrderByLimitReference orderByLimitReference) { Node *workerLimitNode = NULL; LimitPushdownable canPushDownLimit = LIMIT_CANNOT_PUSHDOWN; if (limitCount == NULL) { /* no limit node to push down */ return NULL; } if (!IsA(limitCount, Const)) { /* * We only push down constant LIMIT clauses to make sure we get back * the minimum number of rows. */ return NULL; } if (limitOffset != NULL && !IsA(limitOffset, Const)) { /* * If OFFSET is not a constant then we cannot calculate the LIMIT to * push down. */ return NULL; } /* * If window functions are computed on coordinator, we cannot push down LIMIT. * If we don't have group by clauses, or we have group by partition column, * or if we have order by clauses without aggregates, we can push down the * original limit. Else if we have order by clauses with commutative aggregates, * we can push down approximate limits. */ if (!orderByLimitReference.onlyPushableWindowFunctions) { canPushDownLimit = LIMIT_CANNOT_PUSHDOWN; } else if (orderByLimitReference.groupClauseIsEmpty || orderByLimitReference.groupedByDisjointPartitionColumn) { canPushDownLimit = LIMIT_CAN_PUSHDOWN; } else if (orderByLimitReference.sortClauseIsEmpty) { canPushDownLimit = LIMIT_CANNOT_PUSHDOWN; } else if (!orderByLimitReference.hasOrderByAggregate) { canPushDownLimit = LIMIT_CAN_PUSHDOWN; } else if (orderByLimitReference.canApproximate) { canPushDownLimit = LIMIT_CAN_APPROXIMATE; } /* create the workerLimitNode according to the decisions above */ if (canPushDownLimit == LIMIT_CAN_PUSHDOWN) { workerLimitNode = (Node *) copyObject(limitCount); } else if (canPushDownLimit == LIMIT_CAN_APPROXIMATE) { Const *workerLimitConst = (Const *) copyObject(limitCount); int64 workerLimitCount = (int64) LimitClauseRowFetchCount; workerLimitConst->constvalue = Int64GetDatum(workerLimitCount); workerLimitNode = (Node *) workerLimitConst; } /* * If offset clause is present and limit can be pushed down (whether exactly or * approximately), add the offset value to limit on workers */ if (workerLimitNode != NULL && limitOffset != NULL) { Const *workerLimitConst = (Const *) workerLimitNode; /* Only update the worker limit if the const is not null.*/ if (!workerLimitConst->constisnull) { Const *workerOffsetConst = (Const *) limitOffset; int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); /* If the offset is null, it defaults to 0 when cast to int64. */ int64 workerOffsetCount = DatumGetInt64(workerOffsetConst->constvalue); workerLimitCount = workerLimitCount + workerOffsetCount; workerLimitNode = (Node *) MakeIntegerConstInt64(workerLimitCount); } } /* display debug message on limit push down */ if (workerLimitNode != NULL) { Const *workerLimitConst = (Const *) workerLimitNode; if (!workerLimitConst->constisnull) { int64 workerLimitCount = DatumGetInt64(workerLimitConst->constvalue); ereport(DEBUG1, (errmsg("push down of limit count: " INT64_FORMAT, workerLimitCount))); } else { ereport(DEBUG1, (errmsg("push down of limit count: ALL"))); } } return workerLimitNode; } /* * WorkerSortClauseList first checks if the given input contains a limit * or hasDistinctOn that can be pushed down. If it does, the function then * checks if we need to add any sorting and grouping clauses to the sort list we * push down for the limit. If we do, the function adds these clauses and * returns them. Otherwise, the function returns null. */ static List * WorkerSortClauseList(Node *limitCount, List *groupClauseList, List *sortClauseList, OrderByLimitReference orderByLimitReference) { List *workerSortClauseList = NIL; /* if no limit node and no hasDistinctOn, no need to push down sort clauses */ if (limitCount == NULL && !orderByLimitReference.hasDistinctOn) { return NIL; } /* If window functions are computed on coordinator, we cannot push down sorting. */ if (!orderByLimitReference.onlyPushableWindowFunctions) { return NIL; } sortClauseList = copyObject(sortClauseList); /* * If we are pushing down the limit, push down any order by clauses. Also if * we are pushing down the limit because the order by clauses don't have any * aggregates, add group by clauses to the order by list. We do this because * rows that belong to the same grouping may appear in different "offsets" * in different task results. By ordering on the group by clause, we ensure * that query results are consistent. */ if (orderByLimitReference.groupClauseIsEmpty || orderByLimitReference.groupedByDisjointPartitionColumn) { workerSortClauseList = sortClauseList; } else if (sortClauseList != NIL) { bool orderByNonAggregates = !orderByLimitReference.hasOrderByAggregate; bool canApproximate = orderByLimitReference.canApproximate; if (orderByNonAggregates) { workerSortClauseList = sortClauseList; workerSortClauseList = list_concat(workerSortClauseList, groupClauseList); } else if (canApproximate) { workerSortClauseList = sortClauseList; } } return workerSortClauseList; } /* * CanPushDownLimitApproximate checks if we can push down the limit clause to * the worker nodes, and get approximate and meaningful results. We can do this * only when: (1) the user has enabled the limit approximation and (2) the query * has order by clauses that are commutative. */ static bool CanPushDownLimitApproximate(List *sortClauseList, List *targetList) { bool canApproximate = false; /* user hasn't enabled the limit approximation */ if (LimitClauseRowFetchCount == DISABLE_LIMIT_APPROXIMATION) { return false; } if (sortClauseList != NIL) { bool orderByNonCommutativeAggregate = HasOrderByNonCommutativeAggregate(sortClauseList, targetList); bool orderByComplex = HasOrderByComplexExpression(sortClauseList, targetList); if (!orderByNonCommutativeAggregate && !orderByComplex) { canApproximate = true; } } return canApproximate; } /* * HasOrderByAggregate walks over the given order by clauses, and checks if we * have an order by an aggregate function. If we do, the function returns true. */ static bool HasOrderByAggregate(List *sortClauseList, List *targetList) { bool hasOrderByAggregate = false; SortGroupClause *sortClause = NULL; foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); bool containsAggregate = contain_aggs_of_level(sortExpression, 0); if (containsAggregate) { hasOrderByAggregate = true; break; } } return hasOrderByAggregate; } /* * HasOrderByNonCommutativeAggregate walks over the given order by clauses, * and checks if we have an order by an aggregate which is not commutative. */ static bool HasOrderByNonCommutativeAggregate(List *sortClauseList, List *targetList) { bool hasOrderByNonCommutativeAggregate = false; SortGroupClause *sortClause = NULL; foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); /* if sort expression is an aggregate, check its type */ if (IsA(sortExpression, Aggref)) { Aggref *aggregate = (Aggref *) sortExpression; AggregateType aggregateType = GetAggregateType(aggregate); if (aggregateType != AGGREGATE_MIN && aggregateType != AGGREGATE_MAX && aggregateType != AGGREGATE_SUM && aggregateType != AGGREGATE_COUNT && aggregateType != AGGREGATE_BIT_AND && aggregateType != AGGREGATE_BIT_OR && aggregateType != AGGREGATE_EVERY && aggregateType != AGGREGATE_ANY_VALUE) { hasOrderByNonCommutativeAggregate = true; break; } } } return hasOrderByNonCommutativeAggregate; } /* * HasOrderByComplexExpression walks over the given order by clauses, and checks * if we have a nested expression that contains an aggregate function within it. * If we do, the function returns true. */ static bool HasOrderByComplexExpression(List *sortClauseList, List *targetList) { bool hasOrderByComplexExpression = false; SortGroupClause *sortClause = NULL; foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); /* simple aggregate functions are ok */ if (IsA(sortExpression, Aggref)) { continue; } bool nestedAggregate = contain_aggs_of_level(sortExpression, 0); if (nestedAggregate) { hasOrderByComplexExpression = true; break; } } return hasOrderByComplexExpression; } /* * HasOrderByHllType walks over the given order by clauses, and checks if any of * those clauses operate on hll data type. If they do, the function returns true. */ static bool HasOrderByHllType(List *sortClauseList, List *targetList) { bool hasOrderByHllType = false; /* check whether HLL is loaded */ Oid hllId = get_extension_oid(HLL_EXTENSION_NAME, true); if (!OidIsValid(hllId)) { return hasOrderByHllType; } Oid hllSchemaOid = get_extension_schema(hllId); Oid hllTypeId = TypeOid(hllSchemaOid, HLL_TYPE_NAME); SortGroupClause *sortClause = NULL; foreach_declared_ptr(sortClause, sortClauseList) { Node *sortExpression = get_sortgroupclause_expr(sortClause, targetList); Oid sortColumnTypeId = exprType(sortExpression); if (sortColumnTypeId == hllTypeId) { hasOrderByHllType = true; break; } } return hasOrderByHllType; } /* * ShouldProcessDistinctOrderAndLimitForWorker returns whether * ProcessDistinctClauseForWorkerQuery should be called. If not, * neither should ProcessLimitOrderByForWorkerQuery. */ static bool ShouldProcessDistinctOrderAndLimitForWorker(ExtendedOpNodeProperties * extendedOpNodeProperties, bool pushingDownOriginalGrouping, Node *havingQual) { if (extendedOpNodeProperties->pullUpIntermediateRows) { return false; } /* window functions must be evaluated beforehand */ if (!extendedOpNodeProperties->onlyPushableWindowFunctions) { return false; } if (extendedOpNodeProperties->pushDownGroupingAndHaving) { return true; } /* If the same GROUP BY is being pushed down and there's no HAVING, * then the push down logic will be able to handle this scenario. */ if (pushingDownOriginalGrouping && havingQual == NULL) { return true; } return false; } /* * WorkerColumnName returns a palloc'd string for being the resname of a TargetEntry. */ char * WorkerColumnName(AttrNumber resno) { StringInfoData name = { 0 }; initStringInfo(&name); appendStringInfo(&name, WORKER_COLUMN_FORMAT, resno); return name.data; } /* * IsGroupBySubsetOfDistinct checks whether each clause in group clauses also * exists in the distinct clauses. Note that, empty group clause is not a subset * of distinct clause. */ bool IsGroupBySubsetOfDistinct(List *groupClauses, List *distinctClauses) { /* There must be a group clause */ if (list_length(groupClauses) == 0) { return false; } SortGroupClause *groupClause = NULL; foreach_declared_ptr(groupClause, groupClauses) { bool isFound = false; SortGroupClause *distinctClause = NULL; foreach_declared_ptr(distinctClause, distinctClauses) { if (groupClause->tleSortGroupRef == distinctClause->tleSortGroupRef) { isFound = true; break; } } /* * If we can't find any member of group clause in the distinct clause, * that means group clause is not a subset of distinct clause. */ if (!isFound) { return false; } } return true; } ================================================ FILE: src/backend/distributed/planner/multi_logical_planner.c ================================================ /*------------------------------------------------------------------------- * * multi_logical_planner.c * * Routines for constructing a logical plan tree from the given Query tree * structure. This new logical plan is based on multi-relational algebra rules. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/heapam.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_class.h" #include "commands/defrem.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/prep.h" #include "optimizer/tlist.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_18 typedef OpIndexInterpretation OpBtreeInterpretation; #endif #include "distributed/citus_clauses.h" #include "distributed/colocation_utils.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* Struct to differentiate different qualifier types in an expression tree walker */ typedef struct QualifierWalkerContext { List *baseQualifierList; List *outerJoinQualifierList; } QualifierWalkerContext; /* Function pointer type definition for apply join rule functions */ typedef MultiNode *(*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); typedef bool (*CheckNodeFunc)(Node *); static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ static FieldSelect * CompositeFieldRecursive(Expr *expression, Query *query); static Oid NodeTryGetRteRelid(Node *node); static bool FullCompositeFieldList(List *compositeFieldList); static bool HasUnsupportedJoinWalker(Node *node, void *context); static bool ErrorHintRequired(const char *errorHint, Query *queryTree); static bool HasComplexRangeTableType(Query *queryTree); static bool IsReadIntermediateResultArrayFunction(Node *node); static bool IsCitusExtraDataContainerFunc(Node *node); static bool IsFunctionWithOid(Node *node, Oid funcOid); static bool IsGroupingFunc(Node *node); static bool ExtractFromExpressionWalker(Node *node, QualifierWalkerContext *walkerContext); static List * MultiTableNodeList(List *tableEntryList, List *rangeTableList); static List * AddMultiCollectNodes(List *tableNodeList); static MultiNode * MultiJoinTree(List *joinOrderList, List *collectTableList, List *joinClauseList); static MultiCollect * CollectNodeForTable(List *collectTableList, uint32 rangeTableId); static MultiSelect * MultiSelectNode(List *whereClauseList); static bool IsSelectClause(Node *clause); /* Local functions forward declarations for applying joins */ static MultiNode * ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, List *partitionColumnList, JoinType joinType, List *joinClauseList); static RuleApplyFunction JoinRuleApplyFunction(JoinRuleType ruleType); static MultiNode * ApplyReferenceJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); static MultiNode * ApplyLocalJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); static MultiNode * ApplySingleRangePartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses); static MultiNode * ApplySingleHashPartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses); static MultiJoin * ApplySinglePartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); static MultiNode * ApplyDualPartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); static MultiNode * ApplyCartesianProductReferenceJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); static MultiNode * ApplyCartesianProduct(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *joinClauses); /* * MultiLogicalPlanCreate takes in both the original query and its corresponding modified * query tree yield by the standard planner. It uses helper functions to create logical * plan and adds a root node to top of it. The original query is only used for subquery * pushdown planning. * * We also pass queryTree and plannerRestrictionContext to the planner. They * are primarily used to decide whether the subquery is safe to pushdown. * If not, it helps to produce meaningful error messages for subquery * pushdown planning. */ MultiTreeRoot * MultiLogicalPlanCreate(Query *originalQuery, Query *queryTree, PlannerRestrictionContext *plannerRestrictionContext) { MultiNode *multiQueryNode = NULL; if (ShouldUseSubqueryPushDown(originalQuery, queryTree, plannerRestrictionContext)) { multiQueryNode = SubqueryMultiNodeTree(originalQuery, queryTree, plannerRestrictionContext); } else { multiQueryNode = MultiNodeTree(queryTree); } /* add a root node to serve as the permanent handle to the tree */ MultiTreeRoot *rootNode = CitusMakeNode(MultiTreeRoot); SetChild((MultiUnaryNode *) rootNode, multiQueryNode); return rootNode; } /* * FindNodeMatchingCheckFunction finds a node for which the checker function returns true. * * To call this function directly with an RTE, use: * range_table_walker(rte, FindNodeMatchingCheckFunction, checker, QTW_EXAMINE_RTES_BEFORE) */ bool FindNodeMatchingCheckFunction(Node *node, CheckNodeFunc checker) { if (node == NULL) { return false; } if (checker(node)) { return true; } if (IsA(node, RangeTblEntry)) { /* query_tree_walker descends into RTEs */ return false; } else if (IsA(node, Query)) { return query_tree_walker((Query *) node, FindNodeMatchingCheckFunction, checker, QTW_EXAMINE_RTES_BEFORE); } return expression_tree_walker(node, FindNodeMatchingCheckFunction, checker); } /* * TargetListOnPartitionColumn checks if at least one target list entry is on * partition column. */ bool TargetListOnPartitionColumn(Query *query, List *targetEntryList) { bool targetListOnPartitionColumn = false; List *compositeFieldList = NIL; ListCell *targetEntryCell = NULL; foreach(targetEntryCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Expr *targetExpression = targetEntry->expr; bool skipOuterVars = true; bool isPartitionColumn = IsPartitionColumn(targetExpression, query, skipOuterVars); Var *column = NULL; RangeTblEntry *rte = NULL; FindReferencedTableColumn(targetExpression, NIL, query, &column, &rte, skipOuterVars); Oid relationId = rte ? rte->relid : InvalidOid; /* * If the expression belongs to a non-distributed table continue searching for * other partition keys. */ if (IsCitusTable(relationId) && !HasDistributionKey(relationId)) { continue; } /* append-distributed tables do not have a strict partition column */ if (IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { continue; } if (isPartitionColumn) { FieldSelect *compositeField = CompositeFieldRecursive(targetExpression, query); if (compositeField) { compositeFieldList = lappend(compositeFieldList, compositeField); } else { targetListOnPartitionColumn = true; break; } } } /* check composite fields */ if (!targetListOnPartitionColumn) { bool fullCompositeFieldList = FullCompositeFieldList(compositeFieldList); if (fullCompositeFieldList) { targetListOnPartitionColumn = true; } } /* * We could still behave as if the target list is on partition column if * range table entries don't contain a distributed table. */ if (!targetListOnPartitionColumn) { if (!FindNodeMatchingCheckFunctionInRangeTableList(query->rtable, IsTableWithDistKeyRTE)) { targetListOnPartitionColumn = true; } } return targetListOnPartitionColumn; } /* * FindNodeMatchingCheckFunctionInRangeTableList finds a node for which the checker * function returns true. * * FindNodeMatchingCheckFunctionInRangeTableList relies on * FindNodeMatchingCheckFunction() but only considers the range table entries. */ bool FindNodeMatchingCheckFunctionInRangeTableList(List *rtable, CheckNodeFunc checker) { int rtWalkFlags = QTW_EXAMINE_RTES_BEFORE; #if PG_VERSION_NUM >= PG_VERSION_18 /* * PG18+: Do not descend into GROUP BY expressions subqueries, they * have already been visited as recursive planning is depth-first. */ rtWalkFlags |= QTW_IGNORE_GROUPEXPRS; #endif return range_table_walker(rtable, FindNodeMatchingCheckFunction, checker, rtWalkFlags); } /* * NodeTryGetRteRelid returns the relid of the given RTE_RELATION RangeTableEntry. * Returns InvalidOid if any of these assumptions fail for given node. */ static Oid NodeTryGetRteRelid(Node *node) { if (node == NULL) { return InvalidOid; } if (!IsA(node, RangeTblEntry)) { return InvalidOid; } RangeTblEntry *rangeTableEntry = (RangeTblEntry *) node; if (rangeTableEntry->rtekind != RTE_RELATION) { return InvalidOid; } return rangeTableEntry->relid; } /* * IsCitusTableRTE gets a node and returns true if the node is a * range table relation entry that points to a distributed relation. */ bool IsCitusTableRTE(Node *node) { Oid relationId = NodeTryGetRteRelid(node); return relationId != InvalidOid && IsCitusTable(relationId); } /* * IsDistributedOrReferenceTableRTE returns true if the given node * is eeither a distributed(hash/range/append) or reference table. */ bool IsDistributedOrReferenceTableRTE(Node *node) { Oid relationId = NodeTryGetRteRelid(node); if (!OidIsValid(relationId)) { return false; } return IsCitusTableType(relationId, DISTRIBUTED_TABLE) || IsCitusTableType(relationId, REFERENCE_TABLE); } /* * IsDistributedTableRTE gets a node and returns true if the node * is a range table relation entry that points to a distributed relation, * returning false still if the relation is a reference table. */ bool IsDistributedTableRTE(Node *node) { Oid relationId = NodeTryGetRteRelid(node); return relationId != InvalidOid && IsCitusTableType(relationId, DISTRIBUTED_TABLE); } /* * IsReferenceTableRTE gets a node and returns true if the node * is a range table relation entry that points to a reference table. */ bool IsReferenceTableRTE(Node *node) { Oid relationId = NodeTryGetRteRelid(node); return relationId != InvalidOid && IsCitusTableType(relationId, REFERENCE_TABLE); } /* * IsTableWithDistKeyRTE gets a node and returns true if the node * is a range table relation entry that points to a distributed table * that has a distribution column. */ bool IsTableWithDistKeyRTE(Node *node) { Oid relationId = NodeTryGetRteRelid(node); return relationId != InvalidOid && IsCitusTable(relationId) && HasDistributionKey(relationId); } /* * FullCompositeFieldList gets a composite field list, and checks if all fields * of composite type are used in the list. */ static bool FullCompositeFieldList(List *compositeFieldList) { bool fullCompositeFieldList = true; bool *compositeFieldArray = NULL; uint32 compositeFieldCount = 0; ListCell *fieldSelectCell = NULL; foreach(fieldSelectCell, compositeFieldList) { FieldSelect *fieldSelect = (FieldSelect *) lfirst(fieldSelectCell); Expr *fieldExpression = fieldSelect->arg; if (!IsA(fieldExpression, Var)) { continue; } if (compositeFieldArray == NULL) { Var *compositeColumn = (Var *) fieldExpression; Oid compositeTypeId = compositeColumn->vartype; Oid compositeRelationId = get_typ_typrelid(compositeTypeId); /* get composite type attribute count */ Relation relation = relation_open(compositeRelationId, AccessShareLock); compositeFieldCount = relation->rd_att->natts; compositeFieldArray = palloc0(compositeFieldCount * sizeof(bool)); relation_close(relation, AccessShareLock); for (uint32 compositeFieldIndex = 0; compositeFieldIndex < compositeFieldCount; compositeFieldIndex++) { compositeFieldArray[compositeFieldIndex] = false; } } uint32 compositeFieldIndex = fieldSelect->fieldnum - 1; compositeFieldArray[compositeFieldIndex] = true; } for (uint32 fieldIndex = 0; fieldIndex < compositeFieldCount; fieldIndex++) { if (!compositeFieldArray[fieldIndex]) { fullCompositeFieldList = false; } } if (compositeFieldCount == 0) { fullCompositeFieldList = false; } return fullCompositeFieldList; } /* * CompositeFieldRecursive recursively finds composite field in the query tree * referred by given expression. If expression does not refer to a composite * field, then it returns NULL. * * If expression is a field select we directly return composite field. If it is * a column is referenced from a subquery, then we recursively check that subquery * until we reach the source of that column, and find composite field. If this * column is referenced from join range table entry, then we resolve which join * column it refers and recursively use this column with the same query. */ static FieldSelect * CompositeFieldRecursive(Expr *expression, Query *query) { FieldSelect *compositeField = NULL; List *rangetableList = query->rtable; Var *candidateColumn = NULL; if (IsA(expression, FieldSelect)) { compositeField = (FieldSelect *) expression; return compositeField; } if (IsA(expression, Var)) { candidateColumn = (Var *) expression; } else { return NULL; } Index rangeTableEntryIndex = candidateColumn->varno - 1; RangeTblEntry *rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_SUBQUERY) { Query *subquery = rangeTableEntry->subquery; List *targetEntryList = subquery->targetList; AttrNumber targetEntryIndex = candidateColumn->varattno - 1; TargetEntry *subqueryTargetEntry = list_nth(targetEntryList, targetEntryIndex); Expr *subqueryExpression = subqueryTargetEntry->expr; compositeField = CompositeFieldRecursive(subqueryExpression, subquery); } else if (rangeTableEntry->rtekind == RTE_JOIN) { List *joinColumnList = rangeTableEntry->joinaliasvars; AttrNumber joinColumnIndex = candidateColumn->varattno - 1; Expr *joinColumn = list_nth(joinColumnList, joinColumnIndex); compositeField = CompositeFieldRecursive(joinColumn, query); } return compositeField; } /* * SubqueryEntryList finds the subquery nodes in the range table entry list, and * builds a list of subquery range table entries from these subquery nodes. Range * table entry list also includes subqueries which are pulled up. We don't want * to add pulled up subqueries to list, so we walk over join tree indexes and * check range table entries referenced in the join tree. */ List * SubqueryEntryList(Query *queryTree) { List *rangeTableList = queryTree->rtable; List *subqueryEntryList = NIL; List *joinTreeTableIndexList = NIL; ListCell *joinTreeTableIndexCell = NULL; /* * Extract all range table indexes from the join tree. Note that here we * only walk over range table entries at this level and do not recurse into * subqueries. */ ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* * Join tree's range table index starts from 1 in the query tree. But, * list indexes start from 0. */ int joinTreeTableIndex = lfirst_int(joinTreeTableIndexCell); int rangeTableListIndex = joinTreeTableIndex - 1; RangeTblEntry *rangeTableEntry = (RangeTblEntry *) list_nth(rangeTableList, rangeTableListIndex); if (rangeTableEntry->rtekind == RTE_SUBQUERY) { subqueryEntryList = lappend(subqueryEntryList, rangeTableEntry); } } return subqueryEntryList; } /* * MultiNodeTree takes in a parsed query tree and uses that tree to construct a * logical plan. This plan is based on multi-relational algebra. This function * creates the logical plan in several steps. * * First, the function checks if there is a subquery. If there is a subquery * it recursively creates nested multi trees. If this query has a subquery, the * function does not create any join trees and jumps to last step. * * If there is no subquery, the function calculates the join order using tables * in the query and join clauses between the tables. Second, the function * starts building the logical plan from the bottom-up, and begins with the table * and collect nodes. Third, the function builds the join tree using the join * order information and table nodes. * * In the last step, the function adds the select, project, aggregate, sort, * group, and limit nodes if they appear in the original query tree. */ MultiNode * MultiNodeTree(Query *queryTree) { List *rangeTableList = queryTree->rtable; List *targetEntryList = queryTree->targetList; List *joinClauseList = NIL; List *joinOrderList = NIL; List *tableEntryList = NIL; List *tableNodeList = NIL; List *collectTableList = NIL; MultiNode *joinTreeNode = NULL; MultiNode *currentTopNode = NULL; /* verify we can perform distributed planning on this query */ DeferredErrorMessage *unsupportedQueryError = DeferErrorIfQueryNotSupported( queryTree); if (unsupportedQueryError != NULL) { RaiseDeferredError(unsupportedQueryError, ERROR); } /* extract where clause qualifiers and verify we can plan for them */ List *whereClauseList = WhereClauseList(queryTree->jointree); unsupportedQueryError = DeferErrorIfUnsupportedClause(whereClauseList); if (unsupportedQueryError) { RaiseDeferredErrorInternal(unsupportedQueryError, ERROR); } /* * If we have a subquery, build a multi table node for the subquery and * add a collect node on top of the multi table node. */ List *subqueryEntryList = SubqueryEntryList(queryTree); if (subqueryEntryList != NIL) { MultiCollect *subqueryCollectNode = CitusMakeNode(MultiCollect); ListCell *columnCell = NULL; /* we only support single subquery in the entry list */ Assert(list_length(subqueryEntryList) == 1); RangeTblEntry *subqueryRangeTableEntry = (RangeTblEntry *) linitial( subqueryEntryList); Query *subqueryTree = subqueryRangeTableEntry->subquery; /* ensure if subquery satisfies preconditions */ Assert(DeferErrorIfUnsupportedSubqueryRepartition(subqueryTree) == NULL); MultiTable *subqueryNode = CitusMakeNode(MultiTable); subqueryNode->relationId = SUBQUERY_RELATION_ID; subqueryNode->rangeTableId = SUBQUERY_RANGE_TABLE_ID; subqueryNode->partitionColumn = NULL; subqueryNode->alias = NULL; subqueryNode->referenceNames = NULL; /* * We disregard pulled subqueries. This changes order of range table list. * We do not allow subquery joins, so we will have only one range table * entry in range table list after dropping pulled subquery. For this * reason, here we are updating columns in the most outer query for where * clause list and target list accordingly. */ Assert(list_length(subqueryEntryList) == 1); List *whereClauseColumnList = pull_var_clause_default((Node *) whereClauseList); List *targetListColumnList = pull_var_clause_default((Node *) targetEntryList); List *columnList = list_concat(whereClauseColumnList, targetListColumnList); foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); column->varno = 1; } /* recursively create child nested multitree */ MultiNode *subqueryExtendedNode = MultiNodeTree(subqueryTree); SetChild((MultiUnaryNode *) subqueryCollectNode, (MultiNode *) subqueryNode); SetChild((MultiUnaryNode *) subqueryNode, subqueryExtendedNode); currentTopNode = (MultiNode *) subqueryCollectNode; } else { /* * We calculate the join order using the list of tables in the query and * the join clauses between them. Note that this function owns the table * entry list's memory, and JoinOrderList() shallow copies the list's * elements. */ joinClauseList = JoinClauseList(whereClauseList); tableEntryList = UsedTableEntryList(queryTree); /* build the list of multi table nodes */ tableNodeList = MultiTableNodeList(tableEntryList, rangeTableList); /* add collect nodes on top of the multi table nodes */ collectTableList = AddMultiCollectNodes(tableNodeList); /* find best join order for commutative inner joins */ joinOrderList = JoinOrderList(tableEntryList, joinClauseList); /* build join tree using the join order and collected tables */ joinTreeNode = MultiJoinTree(joinOrderList, collectTableList, joinClauseList); currentTopNode = joinTreeNode; } Assert(currentTopNode != NULL); /* build select node if the query has selection criteria */ MultiSelect *selectNode = MultiSelectNode(whereClauseList); if (selectNode != NULL) { SetChild((MultiUnaryNode *) selectNode, currentTopNode); currentTopNode = (MultiNode *) selectNode; } /* build project node for the columns to project */ MultiProject *projectNode = MultiProjectNode(targetEntryList); SetChild((MultiUnaryNode *) projectNode, currentTopNode); currentTopNode = (MultiNode *) projectNode; /* * We build the extended operator node to capture aggregate functions, group * clauses, sort clauses, limit/offset clauses, and expressions. We need to * distinguish between aggregates and expressions; and we address this later * in the logical optimizer. */ MultiExtendedOp *extendedOpNode = MultiExtendedOpNode(queryTree, queryTree); SetChild((MultiUnaryNode *) extendedOpNode, currentTopNode); currentTopNode = (MultiNode *) extendedOpNode; return currentTopNode; } /* * ContainsReadIntermediateResultFunction determines whether an expression tree * contains a call to the read_intermediate_result function. */ bool ContainsReadIntermediateResultFunction(Node *node) { return FindNodeMatchingCheckFunction(node, IsReadIntermediateResultFunction); } /* * ContainsReadIntermediateResultArrayFunction determines whether an expression * tree contains a call to the read_intermediate_results(result_ids, format) * function. */ bool ContainsReadIntermediateResultArrayFunction(Node *node) { return FindNodeMatchingCheckFunction(node, IsReadIntermediateResultArrayFunction); } /* * IsReadIntermediateResultFunction determines whether a given node is a function call * to the read_intermediate_result function. */ bool IsReadIntermediateResultFunction(Node *node) { return IsFunctionWithOid(node, CitusReadIntermediateResultFuncId()); } /* * IsReadIntermediateResultArrayFunction determines whether a given node is a * function call to the read_intermediate_results(result_ids, format) function. */ static bool IsReadIntermediateResultArrayFunction(Node *node) { return IsFunctionWithOid(node, CitusReadIntermediateResultArrayFuncId()); } /* * IsCitusExtraDataContainerRelation determines whether a range table entry contains a * call to the citus_extradata_container function. */ bool IsCitusExtraDataContainerRelation(RangeTblEntry *rte) { if (rte->rtekind != RTE_FUNCTION || list_length(rte->functions) != 1) { /* avoid more expensive checks below for non-functions */ return false; } if (!CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG5)) { return false; } return FindNodeMatchingCheckFunction((Node *) rte->functions, IsCitusExtraDataContainerFunc); } /* * IsCitusExtraDataContainerFunc determines whether a given node is a function call * to the citus_extradata_container function. */ static bool IsCitusExtraDataContainerFunc(Node *node) { return IsFunctionWithOid(node, CitusExtraDataContainerFuncId()); } /* * IsFunctionWithOid determines whether a given node is a function call * to the read_intermediate_result function. */ static bool IsFunctionWithOid(Node *node, Oid funcOid) { if (IsA(node, FuncExpr)) { FuncExpr *funcExpr = (FuncExpr *) node; if (funcExpr->funcid == funcOid) { return true; } } return false; } /* * IsGroupingFunc returns whether node is a GroupingFunc. */ static bool IsGroupingFunc(Node *node) { return IsA(node, GroupingFunc); } /* * FindIntermediateResultIdIfExists extracts the id of the intermediate result * if the given RTE contains a read_intermediate_results function, NULL otherwise */ char * FindIntermediateResultIdIfExists(RangeTblEntry *rte) { char *resultId = NULL; Assert(rte->rtekind == RTE_FUNCTION); List *functionList = rte->functions; RangeTblFunction *rangeTblfunction = (RangeTblFunction *) linitial(functionList); FuncExpr *funcExpr = (FuncExpr *) rangeTblfunction->funcexpr; if (IsReadIntermediateResultFunction((Node *) funcExpr)) { Const *resultIdConst = linitial(funcExpr->args); if (!resultIdConst->constisnull) { resultId = TextDatumGetCString(resultIdConst->constvalue); } } return resultId; } /* * ErrorIfQueryNotSupported checks that we can perform distributed planning for * the given query. The checks in this function will be removed as we support * more functionality in our distributed planning. */ DeferredErrorMessage * DeferErrorIfQueryNotSupported(Query *queryTree) { char *errorMessage = NULL; bool preconditionsSatisfied = true; const char *errorHint = NULL; const char *joinHint = "Consider joining tables on partition column and have " "equal filter on joining columns."; const char *filterHint = "Consider using an equality filter on the distributed " "table's partition column."; if (queryTree->setOperations) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with UNION, INTERSECT, or " "EXCEPT"; errorHint = filterHint; } if (queryTree->hasRecursive) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with RECURSIVE"; errorHint = filterHint; } if (queryTree->cteList) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with common table expressions"; errorHint = filterHint; } if (queryTree->hasForUpdate) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with FOR UPDATE/SHARE commands"; errorHint = filterHint; } if (queryTree->groupingSets) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with GROUPING SETS, CUBE, " "or ROLLUP"; errorHint = filterHint; } if (FindNodeMatchingCheckFunction((Node *) queryTree, IsGroupingFunc)) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with GROUPING"; errorHint = filterHint; } bool hasUnsupportedJoin = HasUnsupportedJoinWalker((Node *) queryTree->jointree, NULL); if (hasUnsupportedJoin) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with join types other than " "INNER or OUTER JOINS"; errorHint = joinHint; } bool hasComplexRangeTableType = HasComplexRangeTableType(queryTree); if (hasComplexRangeTableType) { preconditionsSatisfied = false; errorMessage = "could not run distributed query with complex table expressions"; errorHint = filterHint; } if (FindNodeMatchingCheckFunction((Node *) queryTree->limitCount, IsNodeSubquery)) { preconditionsSatisfied = false; errorMessage = "subquery in LIMIT is not supported in multi-shard queries"; } if (FindNodeMatchingCheckFunction((Node *) queryTree->limitOffset, IsNodeSubquery)) { preconditionsSatisfied = false; errorMessage = "subquery in OFFSET is not supported in multi-shard queries"; } RTEListProperties *queryRteListProperties = GetRTEListPropertiesForQuery(queryTree); if (queryRteListProperties->hasCitusLocalTable || queryRteListProperties->hasPostgresLocalTable) { preconditionsSatisfied = false; errorMessage = "direct joins between distributed and local tables are " "not supported"; errorHint = LOCAL_TABLE_SUBQUERY_CTE_HINT; } /* finally check and error out if not satisfied */ if (!preconditionsSatisfied) { bool showHint = ErrorHintRequired(errorHint, queryTree); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorMessage, NULL, showHint ? errorHint : NULL); } return NULL; } /* * HasUnsupportedJoinWalker returns tree if the query contains an unsupported * join type. We currently support inner, left, right, full and anti joins. * Semi joins are not supported. A full description of these join types is * included in nodes/nodes.h. */ static bool HasUnsupportedJoinWalker(Node *node, void *context) { bool hasUnsupportedJoin = false; if (node == NULL) { return false; } if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; JoinType joinType = joinExpr->jointype; bool outerJoin = IS_OUTER_JOIN(joinType); if (!outerJoin && joinType != JOIN_INNER && joinType != JOIN_SEMI) { hasUnsupportedJoin = true; } } if (!hasUnsupportedJoin) { hasUnsupportedJoin = expression_tree_walker(node, HasUnsupportedJoinWalker, NULL); } return hasUnsupportedJoin; } /* * ErrorHintRequired returns true if error hint shold be displayed with the * query error message. Error hint is valid only for queries involving reference * and hash partitioned tables. If more than one hash distributed table is * present we display the hint only if the tables are colocated. If the query * only has reference table(s), then it is handled by router planner. */ static bool ErrorHintRequired(const char *errorHint, Query *queryTree) { List *distributedRelationIdList = DistributedRelationIdList(queryTree); ListCell *relationIdCell = NULL; List *colocationIdList = NIL; if (errorHint == NULL) { return false; } foreach(relationIdCell, distributedRelationIdList) { Oid relationId = lfirst_oid(relationIdCell); if (IsCitusTableType(relationId, REFERENCE_TABLE)) { continue; } else if (IsCitusTableType(relationId, HASH_DISTRIBUTED) || IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { int colocationId = TableColocationId(relationId); colocationIdList = list_append_unique_int(colocationIdList, colocationId); } else { return false; } } /* do not display the hint if there are more than one colocation group */ if (list_length(colocationIdList) > 1) { return false; } return true; } /* * DeferErrorIfUnsupportedSubqueryRepartition checks that we can perform distributed planning for * the given subquery. If not, a deferred error is returned. The function recursively * does this check to all lower levels of the subquery. */ DeferredErrorMessage * DeferErrorIfUnsupportedSubqueryRepartition(Query *subqueryTree) { char *errorDetail = NULL; bool preconditionsSatisfied = true; List *joinTreeTableIndexList = NIL; if (!subqueryTree->hasAggs) { preconditionsSatisfied = false; errorDetail = "Subqueries without aggregates are not supported yet"; } if (subqueryTree->groupClause == NIL) { preconditionsSatisfied = false; errorDetail = "Subqueries without group by clause are not supported yet"; } if (subqueryTree->sortClause != NULL) { preconditionsSatisfied = false; errorDetail = "Subqueries with order by clause are not supported yet"; } if (subqueryTree->limitCount != NULL) { preconditionsSatisfied = false; errorDetail = "Subqueries with limit are not supported yet"; } if (subqueryTree->limitOffset != NULL) { preconditionsSatisfied = false; errorDetail = "Subqueries with offset are not supported yet"; } if (subqueryTree->hasSubLinks) { preconditionsSatisfied = false; errorDetail = "Subqueries other than from-clause subqueries are unsupported"; } /* finally check and return error if conditions are not satisfied */ if (!preconditionsSatisfied) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed planning on this query", errorDetail, NULL); } /* * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ ExtractRangeTableIndexWalker((Node *) subqueryTree->jointree, &joinTreeTableIndexList); Assert(list_length(joinTreeTableIndexList) == 1); /* continue with the inner subquery */ int rangeTableIndex = linitial_int(joinTreeTableIndexList); RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableIndex, subqueryTree->rtable); if (rangeTableEntry->rtekind == RTE_RELATION) { return NULL; } Assert(rangeTableEntry->rtekind == RTE_SUBQUERY); Query *innerSubquery = rangeTableEntry->subquery; /* recursively continue to the inner subqueries */ return DeferErrorIfUnsupportedSubqueryRepartition(innerSubquery); } /* * HasComplexRangeTableType checks if the given query tree contains any complex * range table types. For this, the function walks over all range tables in the * join tree, and checks if they correspond to simple relations or subqueries. * If they don't, the function assumes the query has complex range tables. */ static bool HasComplexRangeTableType(Query *queryTree) { List *rangeTableList = queryTree->rtable; List *joinTreeTableIndexList = NIL; ListCell *joinTreeTableIndexCell = NULL; bool hasComplexRangeTableType = false; /* * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* * Join tree's range table index starts from 1 in the query tree. But, * list indexes start from 0. */ int joinTreeTableIndex = lfirst_int(joinTreeTableIndexCell); int rangeTableListIndex = joinTreeTableIndex - 1; RangeTblEntry *rangeTableEntry = (RangeTblEntry *) list_nth(rangeTableList, rangeTableListIndex); /* * Check if the range table in the join tree is a simple relation or a * subquery or a function. Note that RTE_FUNCTIONs are handled via (sub)query * pushdown. */ if (rangeTableEntry->rtekind != RTE_RELATION && rangeTableEntry->rtekind != RTE_SUBQUERY && rangeTableEntry->rtekind != RTE_FUNCTION && rangeTableEntry->rtekind != RTE_VALUES && !IsJsonTableRTE(rangeTableEntry)) { hasComplexRangeTableType = true; } /* * Check if the subquery range table entry includes children inheritance. * * Note that PostgreSQL flattens out simple union all queries into an * append relation, sets "inh" field of RangeTblEntry to true and deletes * set operations. Here we check this for subqueries. */ if (rangeTableEntry->rtekind == RTE_SUBQUERY && rangeTableEntry->inh) { hasComplexRangeTableType = true; } } return hasComplexRangeTableType; } /* * WhereClauseList walks over the FROM expression in the query tree, and builds * a list of all clauses from the expression tree. The function checks for both * implicitly and explicitly defined clauses, but only selects INNER join * explicit clauses, and skips any outer-join clauses. Explicit clauses are * expressed as "SELECT ... FROM R1 INNER JOIN R2 ON R1.A = R2.A". Implicit * joins differ in that they live in the WHERE clause, and are expressed as * "SELECT ... FROM ... WHERE R1.a = R2.a". */ List * WhereClauseList(FromExpr *fromExpr) { FromExpr *fromExprCopy = copyObject(fromExpr); QualifierWalkerContext *walkerContext = palloc0(sizeof(QualifierWalkerContext)); ExtractFromExpressionWalker((Node *) fromExprCopy, walkerContext); List *whereClauseList = walkerContext->baseQualifierList; return whereClauseList; } /* * QualifierList walks over the FROM expression in the query tree, and builds * a list of all qualifiers from the expression tree. The function checks for * both implicitly and explicitly defined qualifiers. Note that this function * is very similar to WhereClauseList(), but QualifierList() also includes * outer-join clauses. */ List * QualifierList(FromExpr *fromExpr) { FromExpr *fromExprCopy = copyObject(fromExpr); QualifierWalkerContext *walkerContext = palloc0(sizeof(QualifierWalkerContext)); List *qualifierList = NIL; ExtractFromExpressionWalker((Node *) fromExprCopy, walkerContext); qualifierList = list_concat(qualifierList, walkerContext->baseQualifierList); qualifierList = list_concat(qualifierList, walkerContext->outerJoinQualifierList); return qualifierList; } /* * DeferErrorIfUnsupportedClause walks over the given list of clauses, and * checks that we can recognize all the clauses. This function ensures that * we do not drop an unsupported clause type on the floor, and thus prevents * erroneous results. * * Returns a deferred error, caller is responsible for raising the error. */ DeferredErrorMessage * DeferErrorIfUnsupportedClause(List *clauseList) { ListCell *clauseCell = NULL; foreach(clauseCell, clauseList) { Node *clause = (Node *) lfirst(clauseCell); if (!(IsSelectClause(clause) || IsJoinClause(clause) || is_orclause(clause))) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "unsupported clause type", NULL, NULL); } } return NULL; } /* * JoinClauseList finds the join clauses from the given where clause expression * list, and returns them. The function does not iterate into nested OR clauses * and relies on find_duplicate_ors() in the optimizer to pull up factorizable * OR clauses. */ List * JoinClauseList(List *whereClauseList) { List *joinClauseList = NIL; ListCell *whereClauseCell = NULL; foreach(whereClauseCell, whereClauseList) { Node *whereClause = (Node *) lfirst(whereClauseCell); if (IsJoinClause(whereClause)) { joinClauseList = lappend(joinClauseList, whereClause); } } return joinClauseList; } /* * ExtractFromExpressionWalker walks over a FROM expression, and finds all * implicit and explicit qualifiers in the expression. The function looks at * join and from expression nodes to find qualifiers, and returns these * qualifiers. * * Note that we don't want outer join clauses in regular outer join planning, * but we need outer join clauses in subquery pushdown prerequisite checks. * Therefore, outer join qualifiers are returned in a different list than other * qualifiers inside the given walker context. For this reason, we return two * qualifier lists. * * Note that we check if the qualifier node in join and from expression nodes * is a list node. If it is not a list node which is the case for subqueries, * then we run eval_const_expressions(), canonicalize_qual() and make_ands_implicit() * on the qualifier node and get a list of flattened implicitly AND'ed qualifier * list. Actually in the planer phase of PostgreSQL these functions also run on * subqueries but differently from the outermost query, they are run on a copy * of parse tree and changes do not get persisted as modifications to the original * query tree. * * Also this function adds SubLinks to the baseQualifierList when they appear on * the query's WHERE clause. The callers of the function should consider processing * Sublinks as well. */ static bool ExtractFromExpressionWalker(Node *node, QualifierWalkerContext *walkerContext) { if (node == NULL) { return false; } /* * Get qualifier lists of join and from expression nodes. Note that in the * case of subqueries, PostgreSQL can skip simplifying, flattening and * making ANDs implicit. If qualifiers node is not a list, then we run these * preprocess routines on qualifiers node. */ if (IsA(node, JoinExpr)) { List *joinQualifierList = NIL; JoinExpr *joinExpression = (JoinExpr *) node; Node *joinQualifiersNode = joinExpression->quals; JoinType joinType = joinExpression->jointype; if (joinQualifiersNode != NULL) { if (IsA(joinQualifiersNode, List)) { joinQualifierList = (List *) joinQualifiersNode; } else { /* this part of code only run for subqueries */ Node *joinClause = eval_const_expressions(NULL, joinQualifiersNode); joinClause = (Node *) canonicalize_qual((Expr *) joinClause, false); joinQualifierList = make_ands_implicit((Expr *) joinClause); } } /* return outer join clauses in a separate list */ if (joinType == JOIN_INNER || joinType == JOIN_SEMI) { walkerContext->baseQualifierList = list_concat(walkerContext->baseQualifierList, joinQualifierList); } else if (IS_OUTER_JOIN(joinType)) { walkerContext->outerJoinQualifierList = list_concat(walkerContext->outerJoinQualifierList, joinQualifierList); } } else if (IsA(node, FromExpr)) { List *fromQualifierList = NIL; FromExpr *fromExpression = (FromExpr *) node; Node *fromQualifiersNode = fromExpression->quals; if (fromQualifiersNode != NULL) { if (IsA(fromQualifiersNode, List)) { fromQualifierList = (List *) fromQualifiersNode; } else { /* this part of code only run for subqueries */ Node *fromClause = eval_const_expressions(NULL, fromQualifiersNode); fromClause = (Node *) canonicalize_qual((Expr *) fromClause, false); fromQualifierList = make_ands_implicit((Expr *) fromClause); } walkerContext->baseQualifierList = list_concat(walkerContext->baseQualifierList, fromQualifierList); } } bool walkerResult = expression_tree_walker(node, ExtractFromExpressionWalker, (void *) walkerContext); return walkerResult; } /* * IsJoinClause determines if the given node is a join clause according to our * criteria. Our criteria defines a join clause as an equi join operator between * two columns that belong to two different tables. */ bool IsJoinClause(Node *clause) { Var *var = NULL; /* * take all column references from the clause, if we find 2 column references from a * different relation we assume this is a join clause */ List *varList = pull_var_clause_default(clause); if (list_length(varList) <= 0) { /* no column references in query, not describing a join */ return false; } Var *initialVar = castNode(Var, linitial(varList)); foreach_declared_ptr(var, varList) { if (var->varno != initialVar->varno) { /* * this column reference comes from a different relation, hence describing a * join */ return true; } } /* all column references were to the same relation, no join */ return false; } /* * TableEntryList finds the regular relation nodes in the range table entry * list, and builds a list of table entries from these regular relation nodes. */ List * TableEntryList(List *rangeTableList) { List *tableEntryList = NIL; ListCell *rangeTableCell = NULL; uint32 tableId = 1; /* range table indices start at 1 */ foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->rtekind == RTE_RELATION) { TableEntry *tableEntry = (TableEntry *) palloc0(sizeof(TableEntry)); tableEntry->relationId = rangeTableEntry->relid; tableEntry->rangeTableId = tableId; tableEntryList = lappend(tableEntryList, tableEntry); } /* * Increment tableId regardless so that table entry's tableId remains * congruent with column's range table reference (varno). */ tableId++; } return tableEntryList; } /* * UsedTableEntryList returns list of relation range table entries * that are referenced within the query. Unused entries due to query * flattening or re-rewriting are ignored. */ List * UsedTableEntryList(Query *query) { List *tableEntryList = NIL; List *rangeTableList = query->rtable; List *joinTreeTableIndexList = NIL; ListCell *joinTreeTableIndexCell = NULL; ExtractRangeTableIndexWalker((Node *) query->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { int joinTreeTableIndex = lfirst_int(joinTreeTableIndexCell); RangeTblEntry *rangeTableEntry = rt_fetch(joinTreeTableIndex, rangeTableList); if (rangeTableEntry->rtekind == RTE_RELATION) { TableEntry *tableEntry = (TableEntry *) palloc0(sizeof(TableEntry)); tableEntry->relationId = rangeTableEntry->relid; tableEntry->rangeTableId = joinTreeTableIndex; tableEntryList = lappend(tableEntryList, tableEntry); } } return tableEntryList; } /* * MultiTableNodeList builds a list of MultiTable nodes from the given table * entry list. A multi table node represents one entry from the range table * list. These entries may belong to the same physical relation in the case of * self-joins. */ static List * MultiTableNodeList(List *tableEntryList, List *rangeTableList) { List *tableNodeList = NIL; ListCell *tableEntryCell = NULL; foreach(tableEntryCell, tableEntryList) { TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); Oid relationId = tableEntry->relationId; uint32 rangeTableId = tableEntry->rangeTableId; Var *partitionColumn = PartitionColumn(relationId, rangeTableId); RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableId, rangeTableList); MultiTable *tableNode = CitusMakeNode(MultiTable); tableNode->subquery = NULL; tableNode->relationId = relationId; tableNode->rangeTableId = rangeTableId; tableNode->partitionColumn = partitionColumn; tableNode->alias = rangeTableEntry->alias; tableNode->referenceNames = rangeTableEntry->eref; tableNode->includePartitions = GetOriginalInh(rangeTableEntry); tableNode->tablesample = rangeTableEntry->tablesample; tableNodeList = lappend(tableNodeList, tableNode); } return tableNodeList; } /* Adds a MultiCollect node on top of each MultiTable node in the given list. */ static List * AddMultiCollectNodes(List *tableNodeList) { List *collectTableList = NIL; ListCell *tableNodeCell = NULL; foreach(tableNodeCell, tableNodeList) { MultiTable *tableNode = (MultiTable *) lfirst(tableNodeCell); MultiCollect *collectNode = CitusMakeNode(MultiCollect); SetChild((MultiUnaryNode *) collectNode, (MultiNode *) tableNode); collectTableList = lappend(collectTableList, collectNode); } return collectTableList; } /* * MultiJoinTree takes in the join order information and the list of tables, and * builds a join tree by applying the corresponding join rules. The function * builds a left deep tree, as expressed by the join order list. * * The function starts by setting the first table as the top node in the join * tree. Then, the function iterates over the list of tables, and builds a new * join node between the top of the join tree and the next table in the list. * At each iteration, the function sets the top of the join tree to the newly * built list. This results in a left deep join tree, and the function returns * this tree after every table in the list has been joined. */ static MultiNode * MultiJoinTree(List *joinOrderList, List *collectTableList, List *joinWhereClauseList) { MultiNode *currentTopNode = NULL; ListCell *joinOrderCell = NULL; bool firstJoinNode = true; foreach(joinOrderCell, joinOrderList) { JoinOrderNode *joinOrderNode = (JoinOrderNode *) lfirst(joinOrderCell); uint32 joinTableId = joinOrderNode->tableEntry->rangeTableId; MultiCollect *collectNode = CollectNodeForTable(collectTableList, joinTableId); if (firstJoinNode) { currentTopNode = (MultiNode *) collectNode; firstJoinNode = false; } else { JoinRuleType joinRuleType = joinOrderNode->joinRuleType; JoinType joinType = joinOrderNode->joinType; List *partitionColumnList = joinOrderNode->partitionColumnList; List *joinClauseList = joinOrderNode->joinClauseList; /* * Build a join node between the top of our join tree and the next * table in the join order. */ MultiNode *newJoinNode = ApplyJoinRule(currentTopNode, (MultiNode *) collectNode, joinRuleType, partitionColumnList, joinType, joinClauseList); /* the new join node becomes the top of our join tree */ currentTopNode = newJoinNode; } } /* current top node points to the entire left deep join tree */ return currentTopNode; } /* * CollectNodeForTable finds the MultiCollect node whose MultiTable node has the * given range table identifier. Note that this function expects each collect * node in the given list to have one table node as its child. */ static MultiCollect * CollectNodeForTable(List *collectTableList, uint32 rangeTableId) { MultiCollect *collectNodeForTable = NULL; ListCell *collectTableCell = NULL; foreach(collectTableCell, collectTableList) { MultiCollect *collectNode = (MultiCollect *) lfirst(collectTableCell); List *tableIdList = OutputTableIdList((MultiNode *) collectNode); uint32 tableId = (uint32) linitial_int(tableIdList); Assert(list_length(tableIdList) == 1); if (tableId == rangeTableId) { collectNodeForTable = collectNode; break; } } Assert(collectNodeForTable != NULL); return collectNodeForTable; } /* * MultiSelectNode extracts the select clauses from the given where clause list, * and builds a MultiSelect node from these clauses. If the expression tree does * not have any select clauses, the function return null. */ static MultiSelect * MultiSelectNode(List *whereClauseList) { List *selectClauseList = NIL; MultiSelect *selectNode = NULL; ListCell *whereClauseCell = NULL; foreach(whereClauseCell, whereClauseList) { Node *whereClause = (Node *) lfirst(whereClauseCell); if (IsSelectClause(whereClause)) { selectClauseList = lappend(selectClauseList, whereClause); } } if (list_length(selectClauseList) > 0) { selectNode = CitusMakeNode(MultiSelect); selectNode->selectClauseList = selectClauseList; } return selectNode; } /* * IsSelectClause determines if the given node is a select clause according to * our criteria. Our criteria defines a select clause as an expression that has * zero or more columns belonging to only one table. The function assumes that * no sublinks exists in the clause. */ static bool IsSelectClause(Node *clause) { ListCell *columnCell = NULL; bool isSelectClause = true; /* extract columns from the clause */ List *columnList = pull_var_clause_default(clause); if (list_length(columnList) == 0) { return true; } /* get first column's tableId */ Var *firstColumn = (Var *) linitial(columnList); Index firstColumnTableId = firstColumn->varno; /* check if all columns are from the same table */ foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); if (column->varno != firstColumnTableId) { isSelectClause = false; } } return isSelectClause; } /* * MultiProjectNode builds the project node using the target entry information * from the query tree. The project node only encapsulates projected columns, * and does not include aggregates, group clauses, or project expressions. */ MultiProject * MultiProjectNode(List *targetEntryList) { List *uniqueColumnList = NIL; ListCell *columnCell = NULL; /* extract the list of columns and remove any duplicates */ List *columnList = pull_var_clause_default((Node *) targetEntryList); foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); uniqueColumnList = list_append_unique(uniqueColumnList, column); } /* create project node with list of columns to project */ MultiProject *projectNode = CitusMakeNode(MultiProject); projectNode->columnList = uniqueColumnList; return projectNode; } /* Builds the extended operator node using fields from the given query tree. */ MultiExtendedOp * MultiExtendedOpNode(Query *queryTree, Query *originalQuery) { MultiExtendedOp *extendedOpNode = CitusMakeNode(MultiExtendedOp); extendedOpNode->targetList = queryTree->targetList; extendedOpNode->groupClauseList = queryTree->groupClause; extendedOpNode->sortClauseList = queryTree->sortClause; extendedOpNode->limitCount = queryTree->limitCount; extendedOpNode->limitOffset = queryTree->limitOffset; extendedOpNode->limitOption = queryTree->limitOption; extendedOpNode->havingQual = queryTree->havingQual; extendedOpNode->distinctClause = queryTree->distinctClause; extendedOpNode->hasDistinctOn = queryTree->hasDistinctOn; extendedOpNode->hasWindowFuncs = queryTree->hasWindowFuncs; extendedOpNode->windowClause = queryTree->windowClause; extendedOpNode->onlyPushableWindowFunctions = !queryTree->hasWindowFuncs || SafeToPushdownWindowFunction(originalQuery, NULL); return extendedOpNode; } /* Helper function to return the parent node of the given node. */ MultiNode * ParentNode(MultiNode *multiNode) { MultiNode *parentNode = multiNode->parentNode; return parentNode; } /* Helper function to return the child of the given unary node. */ MultiNode * ChildNode(MultiUnaryNode *multiNode) { MultiNode *childNode = multiNode->childNode; return childNode; } /* Helper function to return the grand child of the given unary node. */ MultiNode * GrandChildNode(MultiUnaryNode *multiNode) { MultiNode *childNode = ChildNode(multiNode); MultiNode *grandChildNode = ChildNode((MultiUnaryNode *) childNode); return grandChildNode; } /* Sets the given child node as a child of the given unary parent node. */ void SetChild(MultiUnaryNode *parent, MultiNode *child) { parent->childNode = child; child->parentNode = (MultiNode *) parent; } /* Sets the given child node as a left child of the given parent node. */ void SetLeftChild(MultiBinaryNode *parent, MultiNode *leftChild) { parent->leftChildNode = leftChild; leftChild->parentNode = (MultiNode *) parent; } /* Sets the given child node as a right child of the given parent node. */ void SetRightChild(MultiBinaryNode *parent, MultiNode *rightChild) { parent->rightChildNode = rightChild; rightChild->parentNode = (MultiNode *) parent; } /* Returns true if the given node is a unary operator. */ bool UnaryOperator(MultiNode *node) { bool unaryOperator = false; if (CitusIsA(node, MultiTreeRoot) || CitusIsA(node, MultiTable) || CitusIsA(node, MultiCollect) || CitusIsA(node, MultiSelect) || CitusIsA(node, MultiProject) || CitusIsA(node, MultiPartition) || CitusIsA(node, MultiExtendedOp)) { unaryOperator = true; } return unaryOperator; } /* Returns true if the given node is a binary operator. */ bool BinaryOperator(MultiNode *node) { bool binaryOperator = false; if (CitusIsA(node, MultiJoin) || CitusIsA(node, MultiCartesianProduct)) { binaryOperator = true; } return binaryOperator; } /* * OutputTableIdList finds all table identifiers that are output by the given * multi node, and returns these identifiers in a new list. */ List * OutputTableIdList(MultiNode *multiNode) { List *tableIdList = NIL; List *tableNodeList = FindNodesOfType(multiNode, T_MultiTable); ListCell *tableNodeCell = NULL; foreach(tableNodeCell, tableNodeList) { MultiTable *tableNode = (MultiTable *) lfirst(tableNodeCell); int tableId = (int) tableNode->rangeTableId; if (tableId != SUBQUERY_RANGE_TABLE_ID) { tableIdList = lappend_int(tableIdList, tableId); } } return tableIdList; } /* * FindNodesOfType takes in a given logical plan tree, and recursively traverses * the tree in preorder. The function finds all nodes of requested type during * the traversal, and returns them in a list. */ List * FindNodesOfType(MultiNode *node, int type) { List *nodeList = NIL; /* terminal condition for recursion */ if (node == NULL) { return NIL; } /* current node has expected node type */ int nodeType = CitusNodeTag(node); if (nodeType == type) { nodeList = lappend(nodeList, node); } if (UnaryOperator(node)) { MultiNode *childNode = ((MultiUnaryNode *) node)->childNode; List *childNodeList = FindNodesOfType(childNode, type); nodeList = list_concat(nodeList, childNodeList); } else if (BinaryOperator(node)) { MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; MultiNode *rightChildNode = ((MultiBinaryNode *) node)->rightChildNode; List *leftChildNodeList = FindNodesOfType(leftChildNode, type); List *rightChildNodeList = FindNodesOfType(rightChildNode, type); nodeList = list_concat(nodeList, leftChildNodeList); nodeList = list_concat(nodeList, rightChildNodeList); } return nodeList; } /* * pull_var_clause_default calls pull_var_clause with the most commonly used * arguments for distributed planning. */ List * pull_var_clause_default(Node *node) { /* * PVC_REJECT_PLACEHOLDERS is implicit if PVC_INCLUDE_PLACEHOLDERS * isn't specified. */ List *columnList = pull_var_clause(node, PVC_RECURSE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS); return columnList; } /* * ApplyJoinRule finds the join rule application function that corresponds to * the given join rule, and calls this function to create a new join node that * joins the left and right nodes together. */ static MultiNode * ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, List *partitionColumnList, JoinType joinType, List *joinClauseList) { List *leftTableIdList = OutputTableIdList(leftNode); List *rightTableIdList = OutputTableIdList(rightNode); int rightTableIdCount PG_USED_FOR_ASSERTS_ONLY = 0; rightTableIdCount = list_length(rightTableIdList); Assert(rightTableIdCount == 1); /* find applicable join clauses between the left and right data sources */ uint32 rightTableId = (uint32) linitial_int(rightTableIdList); List *applicableJoinClauses = ApplicableJoinClauses(leftTableIdList, rightTableId, joinClauseList); /* call the join rule application function to create the new join node */ RuleApplyFunction ruleApplyFunction = JoinRuleApplyFunction(ruleType); MultiNode *multiNode = (*ruleApplyFunction)(leftNode, rightNode, partitionColumnList, joinType, applicableJoinClauses); if (joinType != JOIN_INNER && CitusIsA(multiNode, MultiJoin)) { MultiJoin *joinNode = (MultiJoin *) multiNode; /* preserve non-join clauses for OUTER joins */ joinNode->joinClauseList = list_copy(joinClauseList); } return multiNode; } /* * JoinRuleApplyFunction returns a function pointer for the rule application * function; this rule application function corresponds to the given rule type. * This function also initializes the rule application function array in a * static code block, if the array has not been initialized. */ static RuleApplyFunction JoinRuleApplyFunction(JoinRuleType ruleType) { static bool ruleApplyFunctionInitialized = false; if (!ruleApplyFunctionInitialized) { RuleApplyFunctionArray[REFERENCE_JOIN] = &ApplyReferenceJoin; RuleApplyFunctionArray[LOCAL_PARTITION_JOIN] = &ApplyLocalJoin; RuleApplyFunctionArray[SINGLE_HASH_PARTITION_JOIN] = &ApplySingleHashPartitionJoin; RuleApplyFunctionArray[SINGLE_RANGE_PARTITION_JOIN] = &ApplySingleRangePartitionJoin; RuleApplyFunctionArray[DUAL_PARTITION_JOIN] = &ApplyDualPartitionJoin; RuleApplyFunctionArray[CARTESIAN_PRODUCT_REFERENCE_JOIN] = &ApplyCartesianProductReferenceJoin; RuleApplyFunctionArray[CARTESIAN_PRODUCT] = &ApplyCartesianProduct; ruleApplyFunctionInitialized = true; } RuleApplyFunction ruleApplyFunction = RuleApplyFunctionArray[ruleType]; Assert(ruleApplyFunction != NULL); return ruleApplyFunction; } /* * ApplyBroadcastJoin creates a new MultiJoin node that joins the left and the * right node. The new node uses the broadcast join rule to perform the join. */ static MultiNode * ApplyReferenceJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiJoin *joinNode = CitusMakeNode(MultiJoin); joinNode->joinRuleType = REFERENCE_JOIN; joinNode->joinType = joinType; joinNode->joinClauseList = applicableJoinClauses; SetLeftChild((MultiBinaryNode *) joinNode, leftNode); SetRightChild((MultiBinaryNode *) joinNode, rightNode); return (MultiNode *) joinNode; } /* * ApplyCartesianProductReferenceJoin creates a new MultiJoin node that joins * the left and the right node. The new node uses the broadcast join rule to * perform the join. */ static MultiNode * ApplyCartesianProductReferenceJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiJoin *joinNode = CitusMakeNode(MultiJoin); joinNode->joinRuleType = CARTESIAN_PRODUCT_REFERENCE_JOIN; joinNode->joinType = joinType; joinNode->joinClauseList = applicableJoinClauses; SetLeftChild((MultiBinaryNode *) joinNode, leftNode); SetRightChild((MultiBinaryNode *) joinNode, rightNode); return (MultiNode *) joinNode; } /* * ApplyLocalJoin creates a new MultiJoin node that joins the left and the right * node. The new node uses the local join rule to perform the join. */ static MultiNode * ApplyLocalJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiJoin *joinNode = CitusMakeNode(MultiJoin); joinNode->joinRuleType = LOCAL_PARTITION_JOIN; joinNode->joinType = joinType; joinNode->joinClauseList = applicableJoinClauses; SetLeftChild((MultiBinaryNode *) joinNode, leftNode); SetRightChild((MultiBinaryNode *) joinNode, rightNode); return (MultiNode *) joinNode; } /* * ApplySingleRangePartitionJoin is a wrapper around ApplySinglePartitionJoin() * which sets the joinRuleType properly. */ static MultiNode * ApplySingleRangePartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiJoin *joinNode = ApplySinglePartitionJoin(leftNode, rightNode, partitionColumnList, joinType, applicableJoinClauses); joinNode->joinRuleType = SINGLE_RANGE_PARTITION_JOIN; return (MultiNode *) joinNode; } /* * ApplySingleHashPartitionJoin is a wrapper around ApplySinglePartitionJoin() * which sets the joinRuleType properly. */ static MultiNode * ApplySingleHashPartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiJoin *joinNode = ApplySinglePartitionJoin(leftNode, rightNode, partitionColumnList, joinType, applicableJoinClauses); joinNode->joinRuleType = SINGLE_HASH_PARTITION_JOIN; return (MultiNode *) joinNode; } /* * ApplySinglePartitionJoin creates a new MultiJoin node that joins the left and * right node. The function also adds a MultiPartition node on top of the node * (left or right) that is not partitioned on the join column. */ static MultiJoin * ApplySinglePartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { Var *partitionColumn = linitial(partitionColumnList); uint32 partitionTableId = partitionColumn->varno; /* create all operator structures up front */ MultiJoin *joinNode = CitusMakeNode(MultiJoin); MultiCollect *collectNode = CitusMakeNode(MultiCollect); MultiPartition *partitionNode = CitusMakeNode(MultiPartition); /* * We first find the appropriate join clause. Then, we compare the partition * column against the join clause's columns. If one of the columns matches, * we introduce a (re-)partition operator for the other column. */ OpExpr *joinClause = SinglePartitionJoinClause(partitionColumnList, applicableJoinClauses, NULL); Assert(joinClause != NULL); /* both are verified in SinglePartitionJoinClause to not be NULL, assert is to guard */ Var *leftColumn = LeftColumnOrNULL(joinClause); Var *rightColumn = RightColumnOrNULL(joinClause); Assert(leftColumn != NULL); Assert(rightColumn != NULL); if (equal(partitionColumn, leftColumn)) { partitionNode->partitionColumn = rightColumn; partitionNode->splitPointTableId = partitionTableId; } else if (equal(partitionColumn, rightColumn)) { partitionNode->partitionColumn = leftColumn; partitionNode->splitPointTableId = partitionTableId; } /* determine the node the partition operator goes on top of */ List *rightTableIdList = OutputTableIdList(rightNode); uint32 rightTableId = (uint32) linitial_int(rightTableIdList); Assert(list_length(rightTableIdList) == 1); /* * If the right child node is partitioned on the partition key column, we * add the partition operator on the left child node; and vice versa. Then, * we add a collect operator on top of the partition operator, and always * make sure that we have at most one relation on the right-hand side. */ if (partitionTableId == rightTableId) { SetChild((MultiUnaryNode *) partitionNode, leftNode); SetChild((MultiUnaryNode *) collectNode, (MultiNode *) partitionNode); SetLeftChild((MultiBinaryNode *) joinNode, (MultiNode *) collectNode); SetRightChild((MultiBinaryNode *) joinNode, rightNode); } else { SetChild((MultiUnaryNode *) partitionNode, rightNode); SetChild((MultiUnaryNode *) collectNode, (MultiNode *) partitionNode); SetLeftChild((MultiBinaryNode *) joinNode, leftNode); SetRightChild((MultiBinaryNode *) joinNode, (MultiNode *) collectNode); } /* finally set join operator fields */ joinNode->joinType = joinType; joinNode->joinClauseList = applicableJoinClauses; return joinNode; } /* * ApplyDualPartitionJoin creates a new MultiJoin node that joins the left and * right node. The function also adds two MultiPartition operators on top of * both nodes to repartition these nodes' data on the join clause columns. */ static MultiNode * ApplyDualPartitionJoin(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { /* find the appropriate join clause */ OpExpr *joinClause = DualPartitionJoinClause(applicableJoinClauses); Assert(joinClause != NULL); /* both are verified in DualPartitionJoinClause to not be NULL, assert is to guard */ Var *leftColumn = LeftColumnOrNULL(joinClause); Var *rightColumn = RightColumnOrNULL(joinClause); Assert(leftColumn != NULL); Assert(rightColumn != NULL); List *rightTableIdList = OutputTableIdList(rightNode); uint32 rightTableId = (uint32) linitial_int(rightTableIdList); Assert(list_length(rightTableIdList) == 1); MultiPartition *leftPartitionNode = CitusMakeNode(MultiPartition); MultiPartition *rightPartitionNode = CitusMakeNode(MultiPartition); /* find the partition node each join clause column belongs to */ if (leftColumn->varno == rightTableId) { leftPartitionNode->partitionColumn = rightColumn; rightPartitionNode->partitionColumn = leftColumn; } else { leftPartitionNode->partitionColumn = leftColumn; rightPartitionNode->partitionColumn = rightColumn; } /* add partition operators on top of left and right nodes */ SetChild((MultiUnaryNode *) leftPartitionNode, leftNode); SetChild((MultiUnaryNode *) rightPartitionNode, rightNode); /* add collect operators on top of the two partition operators */ MultiCollect *leftCollectNode = CitusMakeNode(MultiCollect); MultiCollect *rightCollectNode = CitusMakeNode(MultiCollect); SetChild((MultiUnaryNode *) leftCollectNode, (MultiNode *) leftPartitionNode); SetChild((MultiUnaryNode *) rightCollectNode, (MultiNode *) rightPartitionNode); /* add join operator on top of the two collect operators */ MultiJoin *joinNode = CitusMakeNode(MultiJoin); joinNode->joinRuleType = DUAL_PARTITION_JOIN; joinNode->joinType = joinType; joinNode->joinClauseList = applicableJoinClauses; SetLeftChild((MultiBinaryNode *) joinNode, (MultiNode *) leftCollectNode); SetRightChild((MultiBinaryNode *) joinNode, (MultiNode *) rightCollectNode); return (MultiNode *) joinNode; } /* Creates a cartesian product node that joins the left and the right node. */ static MultiNode * ApplyCartesianProduct(MultiNode *leftNode, MultiNode *rightNode, List *partitionColumnList, JoinType joinType, List *applicableJoinClauses) { MultiCartesianProduct *cartesianNode = CitusMakeNode(MultiCartesianProduct); SetLeftChild((MultiBinaryNode *) cartesianNode, leftNode); SetRightChild((MultiBinaryNode *) cartesianNode, rightNode); return (MultiNode *) cartesianNode; } /* * OperatorImplementsEquality returns true if the given opno represents an * equality operator. The function retrieves btree interpretation list for this * opno and check if BTEqualStrategyNumber strategy is present. */ bool OperatorImplementsEquality(Oid opno) { bool equalityOperator = false; List *btreeIntepretationList = get_op_btree_interpretation(opno); ListCell *btreeInterpretationCell = NULL; foreach(btreeInterpretationCell, btreeIntepretationList) { OpBtreeInterpretation *btreeIntepretation = (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); #if PG_VERSION_NUM >= PG_VERSION_18 if (btreeIntepretation->cmptype == BTEqualStrategyNumber) #else if (btreeIntepretation->strategy == BTEqualStrategyNumber) #endif { equalityOperator = true; break; } } return equalityOperator; } ================================================ FILE: src/backend/distributed/planner/multi_physical_planner.c ================================================ /*------------------------------------------------------------------------- * * multi_physical_planner.c * Routines for creating physical plans from given multi-relational algebra * trees. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/hash.h" #include "access/heapam.h" #include "access/nbtree.h" #include "access/skey.h" #include "access/xlog.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/defrem.h" #include "commands/sequence.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "nodes/print.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_relation.h" #include "parser/parse_type.h" #include "parser/parsetree.h" #include "rewrite/rewriteManip.h" #include "utils/builtins.h" #include "utils/catcache.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" #include "distributed/string_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" /* RepartitionJoinBucketCountPerNode determines bucket amount during repartitions */ int RepartitionJoinBucketCountPerNode = 4; /* Policy to use when assigning tasks to worker nodes */ int TaskAssignmentPolicy = TASK_ASSIGNMENT_GREEDY; bool EnableUniqueJobIds = true; /* * OperatorCache is used for caching operator identifiers for given typeId, * accessMethodId and strategyNumber. It is initialized to empty list as * there are no items in the cache. */ static List *OperatorCache = NIL; /* context passed down in AddAnyValueAggregates mutator */ typedef struct AddAnyValueAggregatesContext { /* SortGroupClauses corresponding to the GROUP BY clause */ List *groupClauseList; /* TargetEntry's to which the GROUP BY clauses refer */ List *groupByTargetEntryList; /* * haveNonVarGrouping is true if there are expressions in the * GROUP BY target entries. We use this as an optimisation to * skip expensive checks when possible. */ bool haveNonVarGrouping; } AddAnyValueAggregatesContext; /* Local functions forward declarations for job creation */ static Job * BuildJobTree(MultiTreeRoot *multiTree); static MultiNode * LeftMostNode(MultiTreeRoot *multiTree); static Oid RangePartitionJoinBaseRelationId(MultiJoin *joinNode); static MultiTable * FindTableNode(MultiNode *multiNode, int rangeTableId); static Query * BuildJobQuery(MultiNode *multiNode, List *dependentJobList); static List * BaseRangeTableList(MultiNode *multiNode); static List * QueryTargetList(MultiNode *multiNode); static List * TargetEntryList(List *expressionList); static Node * AddAnyValueAggregates(Node *node, AddAnyValueAggregatesContext *context); static List * QueryGroupClauseList(MultiNode *multiNode); static List * QuerySelectClauseList(MultiNode *multiNode); static List * QueryFromList(List *rangeTableList); static Node * QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableList); static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, Oid leftRelId, Oid rightRelId, List *leftColumnVars, List *rightColumnVars); static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList); static int ExtractRangeTableId(Node *node); static void ExtractColumns(RangeTblEntry *callingRTE, int rangeTableId, List **columnNames, List **columnVars); static RangeTblEntry * ConstructCallingRTE(RangeTblEntry *rangeTableEntry, List *dependentJobList); static Query * BuildSubqueryJobQuery(MultiNode *multiNode); static void UpdateAllColumnAttributes(Node *columnContainer, List *rangeTableList, List *dependentJobList); static void UpdateColumnAttributes(Var *column, List *rangeTableList, List *dependentJobList); static Index NewTableId(Index originalTableId, List *rangeTableList); static AttrNumber NewColumnId(Index originalTableId, AttrNumber originalColumnId, RangeTblEntry *newRangeTableEntry, List *dependentJobList); static Job * JobForRangeTable(List *jobList, RangeTblEntry *rangeTableEntry); static Job * JobForTableIdList(List *jobList, List *searchedTableIdList); static List * ChildNodeList(MultiNode *multiNode); static Job * BuildJob(Query *jobQuery, List *dependentJobList); static MapMergeJob * BuildMapMergeJob(Query *jobQuery, List *dependentJobList, Var *partitionKey, PartitionType partitionType, Oid baseRelationId, BoundaryNodeJobType boundaryNodeJobType); static uint32 HashPartitionCount(void); /* Local functions forward declarations for task list creation and helper functions */ static Job * BuildJobTreeTaskList(Job *jobTree, PlannerRestrictionContext *plannerRestrictionContext); static bool IsInnerTableOfOuterJoin(RelationRestriction *relationRestriction, Bitmapset *distributedTables, bool *outerPartHasDistributedTable); static void ErrorIfUnsupportedShardDistribution(Query *query); static Task * QueryPushdownTaskCreate(Query *originalQuery, int shardIndex, RelationRestrictionContext *restrictionContext, uint32 taskId, TaskType taskType, bool modifyRequiresCoordinatorEvaluation, bool updateQualsForOuterJoin, DeferredErrorMessage **planningError); static List * SqlTaskList(Job *job); static bool DependsOnHashPartitionJob(Job *job); static uint32 AnchorRangeTableId(List *rangeTableList); static List * BaseRangeTableIdList(List *rangeTableList); static List * AnchorRangeTableIdList(List *rangeTableList, List *baseRangeTableIdList); static void AdjustColumnOldAttributes(List *expressionList); static List * RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, List *dependentJobList); static OperatorCacheEntry * LookupOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber); static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber); static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery, List *dependentJobList); static JoinSequenceNode * JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *dependentJobList); static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependentJobList); static void CheckJoinBetweenColumns(OpExpr *joinClause); static List * FindRangeTableFragmentsList(List *rangeTableFragmentsList, int taskId); static bool JoinPrunable(RangeTableFragment *leftFragment, RangeTableFragment *rightFragment); static ShardInterval * FragmentInterval(RangeTableFragment *fragment); static StringInfo FragmentIntervalString(ShardInterval *fragmentInterval); static List * DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList); static List * BuildRelationShardList(List *rangeTableList, List *fragmentList); static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList); static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment); static List * FetchTaskResultNameList(List *mapOutputFetchTaskList); static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId); static List * PruneSqlTaskDependencies(List *sqlTaskList); static List * AssignTaskList(List *sqlTaskList); static bool HasMergeTaskDependencies(List *sqlTaskList); static List * GreedyAssignTaskList(List *taskList); static Task * GreedyAssignTask(WorkerNode *workerNode, List *taskList, List *activeShardPlacementLists); static List * ReorderAndAssignTaskList(List *taskList, ReorderFunction reorderFunction); static int CompareTasksByShardId(const void *leftElement, const void *rightElement); static List * ActiveShardPlacementLists(List *taskList); static List * LeftRotateList(List *list, uint32 rotateCount); static List * FindDependentMergeTaskList(Task *sqlTask); static List * AssignDualHashTaskList(List *taskList); static void AssignDataFetchDependencies(List *taskList); static uint32 TaskListHighestTaskId(List *taskList); static List * MapTaskList(MapMergeJob *mapMergeJob, List *filterTaskList); static StringInfo CreateMapQueryString(MapMergeJob *mapMergeJob, Task *filterTask, uint32 partitionColumnIndex, bool useBinaryFormat); static char * PartitionResultNamePrefix(uint64 jobId, int32 taskId); static char * PartitionResultName(uint64 jobId, uint32 taskId, uint32 partitionId); static ShardInterval ** RangeIntervalArrayWithNullBucket(ShardInterval **intervalArray, int intervalCount); static List * MergeTaskList(MapMergeJob *mapMergeJob, List *mapTaskList, uint32 taskIdIndex); static List * FetchEqualityAttrNumsForRTEOpExpr(OpExpr *opExpr); static List * FetchEqualityAttrNumsForRTEBoolExpr(BoolExpr *boolExpr); static List * FetchEqualityAttrNumsForList(List *nodeList); static int PartitionColumnIndex(Var *targetVar, List *targetList); static List * GetColumnOriginalIndexes(Oid relationId); static bool QueryTreeHasImproperForDeparseNodes(Node *inputNode, void *context); static Node * AdjustImproperForDeparseNodes(Node *inputNode, void *context); static bool IsImproperForDeparseRelabelTypeNode(Node *inputNode); static bool IsImproperForDeparseCoerceViaIONode(Node *inputNode); static CollateExpr * RelabelTypeToCollateExpr(RelabelType *relabelType); /* * CreatePhysicalDistributedPlan is the entry point for physical plan generation. The * function builds the physical plan; this plan includes the list of tasks to be * executed on worker nodes, and the final query to run on the master node. */ DistributedPlan * CreatePhysicalDistributedPlan(MultiTreeRoot *multiTree, PlannerRestrictionContext *plannerRestrictionContext) { /* build the worker job tree and check that we only have one job in the tree */ Job *workerJob = BuildJobTree(multiTree); /* create the tree of executable tasks for the worker job */ workerJob = BuildJobTreeTaskList(workerJob, plannerRestrictionContext); /* build the final merge query to execute on the master */ List *masterDependentJobList = list_make1(workerJob); Query *combineQuery = BuildJobQuery((MultiNode *) multiTree, masterDependentJobList); DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); distributedPlan->workerJob = workerJob; distributedPlan->combineQuery = combineQuery; distributedPlan->modLevel = ROW_MODIFY_READONLY; distributedPlan->expectResults = true; return distributedPlan; } /* * ModifyLocalTableJob returns true if the given task contains * a modification of local table. */ bool ModifyLocalTableJob(Job *job) { if (job == NULL) { return false; } List *taskList = job->taskList; if (list_length(taskList) != 1) { return false; } Task *singleTask = (Task *) linitial(taskList); return singleTask->isLocalTableModification; } /* * BuildJobTree builds the physical job tree from the given logical plan tree. * The function walks over the logical plan from the bottom up, finds boundaries * for jobs, and creates the query structure for each job. The function also * sets dependencies between jobs, and then returns the top level worker job. */ static Job * BuildJobTree(MultiTreeRoot *multiTree) { /* start building the tree from the deepest left node */ MultiNode *leftMostNode = LeftMostNode(multiTree); MultiNode *currentNode = leftMostNode; MultiNode *parentNode = ParentNode(currentNode); List *loopDependentJobList = NIL; Job *topLevelJob = NULL; while (parentNode != NULL) { CitusNodeTag currentNodeType = CitusNodeTag(currentNode); CitusNodeTag parentNodeType = CitusNodeTag(parentNode); BoundaryNodeJobType boundaryNodeJobType = JOB_INVALID_FIRST; /* we first check if this node forms the boundary for a remote job */ if (currentNodeType == T_MultiJoin) { MultiJoin *joinNode = (MultiJoin *) currentNode; if (joinNode->joinRuleType == SINGLE_HASH_PARTITION_JOIN || joinNode->joinRuleType == SINGLE_RANGE_PARTITION_JOIN || joinNode->joinRuleType == DUAL_PARTITION_JOIN) { boundaryNodeJobType = JOIN_MAP_MERGE_JOB; } } else if (currentNodeType == T_MultiCollect && parentNodeType != T_MultiPartition) { boundaryNodeJobType = TOP_LEVEL_WORKER_JOB; } /* * If this node is at the boundary for a repartition or top level worker * job, we build the corresponding job(s) and set their dependencies. */ if (boundaryNodeJobType == JOIN_MAP_MERGE_JOB) { MultiJoin *joinNode = (MultiJoin *) currentNode; MultiNode *leftChildNode = joinNode->binaryNode.leftChildNode; MultiNode *rightChildNode = joinNode->binaryNode.rightChildNode; PartitionType partitionType = PARTITION_INVALID_FIRST; Oid baseRelationId = InvalidOid; if (joinNode->joinRuleType == SINGLE_RANGE_PARTITION_JOIN) { partitionType = RANGE_PARTITION_TYPE; baseRelationId = RangePartitionJoinBaseRelationId(joinNode); } else if (joinNode->joinRuleType == SINGLE_HASH_PARTITION_JOIN) { partitionType = SINGLE_HASH_PARTITION_TYPE; baseRelationId = RangePartitionJoinBaseRelationId(joinNode); } else if (joinNode->joinRuleType == DUAL_PARTITION_JOIN) { partitionType = DUAL_HASH_PARTITION_TYPE; } if (CitusIsA(leftChildNode, MultiPartition)) { MultiPartition *partitionNode = (MultiPartition *) leftChildNode; MultiNode *queryNode = GrandChildNode((MultiUnaryNode *) partitionNode); Var *partitionKey = partitionNode->partitionColumn; /* build query and partition job */ List *dependentJobList = list_copy(loopDependentJobList); Query *jobQuery = BuildJobQuery(queryNode, dependentJobList); MapMergeJob *mapMergeJob = BuildMapMergeJob(jobQuery, dependentJobList, partitionKey, partitionType, baseRelationId, JOIN_MAP_MERGE_JOB); /* reset dependent job list */ loopDependentJobList = NIL; loopDependentJobList = list_make1(mapMergeJob); } if (CitusIsA(rightChildNode, MultiPartition)) { MultiPartition *partitionNode = (MultiPartition *) rightChildNode; MultiNode *queryNode = GrandChildNode((MultiUnaryNode *) partitionNode); Var *partitionKey = partitionNode->partitionColumn; /* * The right query and right partition job do not depend on any * jobs since our logical plan tree is left deep. */ Query *jobQuery = BuildJobQuery(queryNode, NIL); MapMergeJob *mapMergeJob = BuildMapMergeJob(jobQuery, NIL, partitionKey, partitionType, baseRelationId, JOIN_MAP_MERGE_JOB); /* append to the dependent job list for on-going dependencies */ loopDependentJobList = lappend(loopDependentJobList, mapMergeJob); } } else if (boundaryNodeJobType == TOP_LEVEL_WORKER_JOB) { MultiNode *childNode = ChildNode((MultiUnaryNode *) currentNode); List *dependentJobList = list_copy(loopDependentJobList); bool subqueryPushdown = false; List *subqueryMultiTableList = SubqueryMultiTableList(childNode); int subqueryCount = list_length(subqueryMultiTableList); if (subqueryCount > 0) { subqueryPushdown = true; } /* * Build top level query. If subquery pushdown is set, we use * sligthly different version of BuildJobQuery(). They are similar * but we don't need some parts of BuildJobQuery() for subquery * pushdown such as updating column attributes etc. */ if (subqueryPushdown) { Query *topLevelQuery = BuildSubqueryJobQuery(childNode); topLevelJob = BuildJob(topLevelQuery, dependentJobList); topLevelJob->subqueryPushdown = true; } else { Query *topLevelQuery = BuildJobQuery(childNode, dependentJobList); topLevelJob = BuildJob(topLevelQuery, dependentJobList); } } /* walk up the tree */ currentNode = parentNode; parentNode = ParentNode(currentNode); } return topLevelJob; } /* * LeftMostNode finds the deepest left node in the left-deep logical plan tree. * We build the physical plan by traversing the logical plan from the bottom up; * and this function helps us find the bottom of the logical tree. */ static MultiNode * LeftMostNode(MultiTreeRoot *multiTree) { MultiNode *currentNode = (MultiNode *) multiTree; MultiNode *leftChildNode = ChildNode((MultiUnaryNode *) multiTree); while (leftChildNode != NULL) { currentNode = leftChildNode; if (UnaryOperator(currentNode)) { leftChildNode = ChildNode((MultiUnaryNode *) currentNode); } else if (BinaryOperator(currentNode)) { MultiBinaryNode *binaryNode = (MultiBinaryNode *) currentNode; leftChildNode = binaryNode->leftChildNode; } } return currentNode; } /* * RangePartitionJoinBaseRelationId finds partition node from join node, and * returns base relation id of this node. Note that this function assumes that * given join node is range partition join type. */ static Oid RangePartitionJoinBaseRelationId(MultiJoin *joinNode) { MultiPartition *partitionNode = NULL; MultiNode *leftChildNode = joinNode->binaryNode.leftChildNode; MultiNode *rightChildNode = joinNode->binaryNode.rightChildNode; if (CitusIsA(leftChildNode, MultiPartition)) { partitionNode = (MultiPartition *) leftChildNode; } else if (CitusIsA(rightChildNode, MultiPartition)) { partitionNode = (MultiPartition *) rightChildNode; } else { Assert(false); } Index baseTableId = partitionNode->splitPointTableId; MultiTable *baseTable = FindTableNode((MultiNode *) joinNode, baseTableId); Oid baseRelationId = baseTable->relationId; return baseRelationId; } /* * FindTableNode walks over the given logical plan tree, and returns the table * node that corresponds to the given range tableId. */ static MultiTable * FindTableNode(MultiNode *multiNode, int rangeTableId) { MultiTable *foundTableNode = NULL; List *tableNodeList = FindNodesOfType(multiNode, T_MultiTable); ListCell *tableNodeCell = NULL; foreach(tableNodeCell, tableNodeList) { MultiTable *tableNode = (MultiTable *) lfirst(tableNodeCell); if (tableNode->rangeTableId == rangeTableId) { foundTableNode = tableNode; break; } } Assert(foundTableNode != NULL); return foundTableNode; } /* * BuildJobQuery traverses the given logical plan tree, determines the job that * corresponds to this part of the tree, and builds the query structure for that * particular job. The function assumes that jobs this particular job depends on * have already been built, as their output is needed to build the query. */ static Query * BuildJobQuery(MultiNode *multiNode, List *dependentJobList) { bool updateColumnAttributes = false; List *targetList = NIL; List *sortClauseList = NIL; Node *limitCount = NULL; Node *limitOffset = NULL; LimitOption limitOption = LIMIT_OPTION_COUNT; Node *havingQual = NULL; bool hasDistinctOn = false; List *distinctClause = NIL; bool isRepartitionJoin = false; bool hasWindowFuncs = false; List *windowClause = NIL; /* we start building jobs from below the collect node */ Assert(!CitusIsA(multiNode, MultiCollect)); /* * First check if we are building a master/worker query. If we are building * a worker query, we update the column attributes for target entries, select * and join columns. Because if underlying query includes repartition joins, * then we create multiple queries from a join. In this case, range table lists * and column lists are subject to change. * * Note that we don't do this for master queries, as column attributes for * master target entries are already set during the master/worker split. */ MultiNode *parentNode = ParentNode(multiNode); if (parentNode != NULL) { updateColumnAttributes = true; } /* * If we are building this query on a repartitioned subquery job then we * don't need to update column attributes. */ if (dependentJobList != NIL) { Job *job = (Job *) linitial(dependentJobList); if (CitusIsA(job, MapMergeJob)) { isRepartitionJoin = true; } } /* * If we have an extended operator, then we copy the operator's target list. * Otherwise, we use the target list based on the MultiProject node at this * level in the query tree. */ List *extendedOpNodeList = FindNodesOfType(multiNode, T_MultiExtendedOp); if (extendedOpNodeList != NIL) { MultiExtendedOp *extendedOp = (MultiExtendedOp *) linitial(extendedOpNodeList); targetList = copyObject(extendedOp->targetList); distinctClause = extendedOp->distinctClause; hasDistinctOn = extendedOp->hasDistinctOn; hasWindowFuncs = extendedOp->hasWindowFuncs; windowClause = extendedOp->windowClause; } else { targetList = QueryTargetList(multiNode); } /* build the join tree and the range table list */ List *rangeTableList = BaseRangeTableList(multiNode); Node *joinRoot = QueryJoinTree(multiNode, dependentJobList, &rangeTableList); /* update the column attributes for target entries */ if (updateColumnAttributes) { UpdateAllColumnAttributes((Node *) targetList, rangeTableList, dependentJobList); } /* extract limit count/offset and sort clauses */ if (extendedOpNodeList != NIL) { MultiExtendedOp *extendedOp = (MultiExtendedOp *) linitial(extendedOpNodeList); limitCount = extendedOp->limitCount; limitOffset = extendedOp->limitOffset; limitOption = extendedOp->limitOption; sortClauseList = extendedOp->sortClauseList; havingQual = extendedOp->havingQual; } /* build group clauses */ List *groupClauseList = QueryGroupClauseList(multiNode); /* build the where clause list using select predicates */ List *selectClauseList = QuerySelectClauseList(multiNode); /* set correct column attributes for select and having clauses */ if (updateColumnAttributes) { UpdateAllColumnAttributes((Node *) selectClauseList, rangeTableList, dependentJobList); UpdateAllColumnAttributes(havingQual, rangeTableList, dependentJobList); } /* * Group by on primary key allows all columns to appear in the target * list, but after re-partitioning we will be querying an intermediate * table that does not have the primary key. We therefore wrap all the * columns that do not appear in the GROUP BY in an any_value aggregate. */ if (groupClauseList != NIL && isRepartitionJoin) { targetList = (List *) WrapUngroupedVarsInAnyValueAggregate( (Node *) targetList, groupClauseList, targetList, true); havingQual = WrapUngroupedVarsInAnyValueAggregate( (Node *) havingQual, groupClauseList, targetList, false); } /* * Build the From/Where construct. We keep the where-clause list implicitly * AND'd, since both partition and join pruning depends on the clauses being * expressed as a list. */ FromExpr *joinTree = makeNode(FromExpr); joinTree->quals = (Node *) list_copy(selectClauseList); joinTree->fromlist = list_make1(joinRoot); /* build the query structure for this job */ Query *jobQuery = makeNode(Query); jobQuery->commandType = CMD_SELECT; jobQuery->querySource = QSRC_ORIGINAL; jobQuery->canSetTag = true; jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; jobQuery->limitCount = limitCount; jobQuery->limitOption = limitOption; jobQuery->havingQual = havingQual; jobQuery->hasAggs = contain_aggs_of_level((Node *) targetList, 0) || contain_aggs_of_level((Node *) havingQual, 0); jobQuery->distinctClause = distinctClause; jobQuery->hasDistinctOn = hasDistinctOn; jobQuery->windowClause = windowClause; jobQuery->hasWindowFuncs = hasWindowFuncs; jobQuery->hasSubLinks = checkExprHasSubLink((Node *) jobQuery); Assert(jobQuery->hasWindowFuncs == contain_window_function((Node *) jobQuery)); return jobQuery; } /* * BaseRangeTableList returns the list of range table entries for base tables in * the query. These base tables stand in contrast to derived tables generated by * repartition jobs. Note that this function only considers base tables relevant * to the current query, and does not visit nodes under the collect node. */ static List * BaseRangeTableList(MultiNode *multiNode) { List *baseRangeTableList = NIL; List *pendingNodeList = list_make1(multiNode); while (pendingNodeList != NIL) { MultiNode *currMultiNode = (MultiNode *) linitial(pendingNodeList); CitusNodeTag nodeType = CitusNodeTag(currMultiNode); pendingNodeList = list_delete_first(pendingNodeList); if (nodeType == T_MultiTable) { /* * We represent subqueries as MultiTables, and so for base table * entries we skip the subquery ones. */ MultiTable *multiTable = (MultiTable *) currMultiNode; if (multiTable->relationId != SUBQUERY_RELATION_ID && multiTable->relationId != SUBQUERY_PUSHDOWN_RELATION_ID) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->inFromCl = true; rangeTableEntry->eref = multiTable->referenceNames; rangeTableEntry->alias = multiTable->alias; rangeTableEntry->relid = multiTable->relationId; rangeTableEntry->inh = multiTable->includePartitions; rangeTableEntry->tablesample = multiTable->tablesample; SetRangeTblExtraData(rangeTableEntry, CITUS_RTE_RELATION, NULL, NULL, list_make1_int(multiTable->rangeTableId), NIL, NIL, NIL, NIL); baseRangeTableList = lappend(baseRangeTableList, rangeTableEntry); } } /* do not visit nodes that belong to remote queries */ if (nodeType != T_MultiCollect) { List *childNodeList = ChildNodeList(currMultiNode); pendingNodeList = list_concat(pendingNodeList, childNodeList); } } return baseRangeTableList; } /* * DerivedRangeTableEntry builds a range table entry for the derived table. This * derived table either represents the output of a repartition job; or the data * on worker nodes in case of the master node query. */ RangeTblEntry * DerivedRangeTableEntry(MultiNode *multiNode, List *columnList, List *tableIdList, List *funcColumnNames, List *funcColumnTypes, List *funcColumnTypeMods, List *funcCollations) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->inFromCl = true; rangeTableEntry->eref = makeNode(Alias); rangeTableEntry->eref->colnames = columnList; SetRangeTblExtraData(rangeTableEntry, CITUS_RTE_REMOTE_QUERY, NULL, NULL, tableIdList, funcColumnNames, funcColumnTypes, funcColumnTypeMods, funcCollations); return rangeTableEntry; } /* * DerivedColumnNameList builds a column name list for derived (intermediate) * tables. These column names are then used when building the create stament * query string for derived tables. */ List * DerivedColumnNameList(uint32 columnCount, uint64 generatingJobId) { List *columnNameList = NIL; for (uint32 columnIndex = 0; columnIndex < columnCount; columnIndex++) { StringInfo columnName = makeStringInfo(); appendStringInfo(columnName, "intermediate_column_"); appendStringInfo(columnName, UINT64_FORMAT "_", generatingJobId); appendStringInfo(columnName, "%u", columnIndex); String *columnValue = makeString(columnName->data); columnNameList = lappend(columnNameList, columnValue); } return columnNameList; } /* * QueryTargetList returns the target entry list for the projected columns * needed to evaluate the operators above the given multiNode. To do this, * the function retrieves a list of all MultiProject nodes below the given * node and picks the columns from the top-most MultiProject node, as this * will be the minimal list of columns needed. Note that this function relies * on a pre-order traversal of the operator tree by the function FindNodesOfType. */ static List * QueryTargetList(MultiNode *multiNode) { List *projectNodeList = FindNodesOfType(multiNode, T_MultiProject); if (list_length(projectNodeList) == 0) { /* * The physical planner assumes that all worker queries would have * target list entries based on the fact that at least the column * on the JOINs have to be on the target list. However, there is * an exception to that if there is a cartesian product join and * there is no additional target list entries belong to one side * of the JOIN. Once we support cartesian product join, we should * remove this error. */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning on this query"), errdetail("Cartesian products are currently unsupported"))); } MultiProject *topProjectNode = (MultiProject *) linitial(projectNodeList); List *columnList = topProjectNode->columnList; List *queryTargetList = TargetEntryList(columnList); Assert(queryTargetList != NIL); return queryTargetList; } /* * TargetEntryList creates a target entry for each expression in the given list, * and returns the newly created target entries in a list. */ static List * TargetEntryList(List *expressionList) { List *targetEntryList = NIL; ListCell *expressionCell = NULL; foreach(expressionCell, expressionList) { Expr *expression = (Expr *) lfirst(expressionCell); int columnNumber = list_length(targetEntryList) + 1; StringInfo columnName = makeStringInfo(); appendStringInfo(columnName, "column%d", columnNumber); TargetEntry *targetEntry = makeTargetEntry(expression, columnNumber, columnName->data, false); targetEntryList = lappend(targetEntryList, targetEntry); } return targetEntryList; } /* * WrapUngroupedVarsInAnyValueAggregate finds Var nodes in the expression * that do not refer to any GROUP BY column and wraps them in an any_value * aggregate. These columns are allowed when the GROUP BY is on a primary * key of a relation, but not if we wrap the relation in a subquery. * However, since we still know the value is unique, any_value gives the * right result. */ Node * WrapUngroupedVarsInAnyValueAggregate(Node *expression, List *groupClauseList, List *targetList, bool checkExpressionEquality) { if (expression == NULL) { return NULL; } AddAnyValueAggregatesContext context; context.groupClauseList = groupClauseList; context.groupByTargetEntryList = GroupTargetEntryList(groupClauseList, targetList); context.haveNonVarGrouping = false; if (checkExpressionEquality) { /* * If the GROUP BY contains non-Var expressions, we need to do an expensive * subexpression equality check. */ TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, context.groupByTargetEntryList) { if (!IsA(targetEntry->expr, Var)) { context.haveNonVarGrouping = true; break; } } } /* put the result in the same memory context */ MemoryContext nodeContext = GetMemoryChunkContext(expression); MemoryContext oldContext = MemoryContextSwitchTo(nodeContext); Node *result = expression_tree_mutator(expression, AddAnyValueAggregates, &context); MemoryContextSwitchTo(oldContext); return result; } /* * AddAnyValueAggregates wraps all vars that do not appear in the GROUP BY * clause or are inside an aggregate function in an any_value aggregate * function. This is needed because postgres allows columns that are not * in the GROUP BY to appear on the target list as long as the primary key * of the table is in the GROUP BY, but we sometimes wrap the join tree * in a subquery in which case the primary key information is lost. * * This function copies parts of the node tree, but may contain references * to the original node tree. * * The implementation is derived from / inspired by * check_ungrouped_columns_walker. */ static Node * AddAnyValueAggregates(Node *node, AddAnyValueAggregatesContext *context) { if (node == NULL) { return node; } if (IsA(node, Aggref) || IsA(node, GroupingFunc)) { /* any column is allowed to appear in an aggregate or grouping */ return node; } else if (IsA(node, Var)) { Var *var = (Var *) node; /* * Check whether this Var appears in the GROUP BY. */ TargetEntry *groupByTargetEntry = NULL; foreach_declared_ptr(groupByTargetEntry, context->groupByTargetEntryList) { if (!IsA(groupByTargetEntry->expr, Var)) { continue; } Var *groupByVar = (Var *) groupByTargetEntry->expr; /* we should only be doing this at the top level of the query */ Assert(groupByVar->varlevelsup == 0); if (var->varno == groupByVar->varno && var->varattno == groupByVar->varattno) { /* this Var is in the GROUP BY, do not wrap it */ return node; } } /* * We have found a Var that does not appear in the GROUP BY. * Wrap it in an any_value aggregate. */ Aggref *agg = makeNode(Aggref); agg->aggfnoid = CitusAnyValueFunctionId(); agg->aggtype = var->vartype; agg->args = list_make1(makeTargetEntry((Expr *) var, 1, NULL, false)); agg->aggkind = AGGKIND_NORMAL; agg->aggtranstype = InvalidOid; agg->aggargtypes = list_make1_oid(var->vartype); agg->aggsplit = AGGSPLIT_SIMPLE; agg->aggcollid = exprCollation((Node *) var); return (Node *) agg; } else if (context->haveNonVarGrouping) { /* * The GROUP BY contains at least one expression. Check whether the * current expression is equal to one of the GROUP BY expressions. * Otherwise, continue to descend into subexpressions. */ TargetEntry *groupByTargetEntry = NULL; foreach_declared_ptr(groupByTargetEntry, context->groupByTargetEntryList) { if (equal(node, groupByTargetEntry->expr)) { /* do not descend into mutator, all Vars are safe */ return node; } } } return expression_tree_mutator(node, AddAnyValueAggregates, context); } /* * QueryGroupClauseList extracts the group clause list from the logical plan. If * no grouping clauses exist, the function returns an empty list. */ static List * QueryGroupClauseList(MultiNode *multiNode) { List *groupClauseList = NIL; List *pendingNodeList = list_make1(multiNode); while (pendingNodeList != NIL) { MultiNode *currMultiNode = (MultiNode *) linitial(pendingNodeList); CitusNodeTag nodeType = CitusNodeTag(currMultiNode); pendingNodeList = list_delete_first(pendingNodeList); /* extract the group clause list from the extended operator */ if (nodeType == T_MultiExtendedOp) { MultiExtendedOp *extendedOpNode = (MultiExtendedOp *) currMultiNode; groupClauseList = extendedOpNode->groupClauseList; } /* add children only if this node isn't a multi collect and multi table */ if (nodeType != T_MultiCollect && nodeType != T_MultiTable) { List *childNodeList = ChildNodeList(currMultiNode); pendingNodeList = list_concat(pendingNodeList, childNodeList); } } return groupClauseList; } /* * QuerySelectClauseList traverses the given logical plan tree, and extracts all * select clauses from the select nodes. Note that this function does not walk * below a collect node; the clauses below the collect node apply to a remote * query, and they would have been captured by the remote job we depend upon. */ static List * QuerySelectClauseList(MultiNode *multiNode) { List *selectClauseList = NIL; List *pendingNodeList = list_make1(multiNode); while (pendingNodeList != NIL) { MultiNode *currMultiNode = (MultiNode *) linitial(pendingNodeList); CitusNodeTag nodeType = CitusNodeTag(currMultiNode); pendingNodeList = list_delete_first(pendingNodeList); /* extract select clauses from the multi select node */ if (nodeType == T_MultiSelect) { MultiSelect *selectNode = (MultiSelect *) currMultiNode; List *clauseList = copyObject(selectNode->selectClauseList); selectClauseList = list_concat(selectClauseList, clauseList); } /* add children only if this node isn't a multi collect */ if (nodeType != T_MultiCollect) { List *childNodeList = ChildNodeList(currMultiNode); pendingNodeList = list_concat(pendingNodeList, childNodeList); } } return selectClauseList; } /* * Create a tree of JoinExpr and RangeTblRef nodes for the job query from * a given multiNode. If the tree contains MultiCollect or MultiJoin nodes, * add corresponding entries to the range table list. We need to construct * the entries at the same time as the tree to know the appropriate rtindex. */ static Node * QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableList) { CitusNodeTag nodeType = CitusNodeTag(multiNode); switch (nodeType) { case T_MultiJoin: { MultiJoin *joinNode = (MultiJoin *) multiNode; MultiBinaryNode *binaryNode = (MultiBinaryNode *) multiNode; ListCell *columnCell = NULL; JoinExpr *joinExpr = makeNode(JoinExpr); joinExpr->jointype = joinNode->joinType; joinExpr->isNatural = false; joinExpr->larg = QueryJoinTree(binaryNode->leftChildNode, dependentJobList, rangeTableList); joinExpr->rarg = QueryJoinTree(binaryNode->rightChildNode, dependentJobList, rangeTableList); joinExpr->usingClause = NIL; joinExpr->alias = NULL; joinExpr->rtindex = list_length(*rangeTableList) + 1; /* * PostgreSQL's optimizer may mark left joins as anti-joins, when there * is a right-hand-join-key-is-null restriction, but there is no logic * in ruleutils to deparse anti-joins, so we cannot construct a task * query containing anti-joins. We therefore translate anti-joins back * into left-joins. At some point, we may also want to use different * join pruning logic for anti-joins. * * This approach would not work for anti-joins introduced via NOT EXISTS * sublinks, but currently such queries are prevented by error checks in * the logical planner. */ if (joinExpr->jointype == JOIN_ANTI) { joinExpr->jointype = JOIN_LEFT; } /* fix the column attributes in ON (...) clauses */ List *columnList = pull_var_clause_default((Node *) joinNode->joinClauseList); foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); UpdateColumnAttributes(column, *rangeTableList, dependentJobList); /* adjust our column old attributes for partition pruning to work */ column->varnosyn = column->varno; column->varattnosyn = column->varattno; } /* make AND clauses explicit after fixing them */ joinExpr->quals = (Node *) make_ands_explicit(joinNode->joinClauseList); RangeTblEntry *rangeTableEntry = JoinRangeTableEntry(joinExpr, dependentJobList, *rangeTableList); *rangeTableList = lappend(*rangeTableList, rangeTableEntry); return (Node *) joinExpr; } case T_MultiTable: { MultiTable *rangeTableNode = (MultiTable *) multiNode; MultiUnaryNode *unaryNode = (MultiUnaryNode *) multiNode; if (unaryNode->childNode != NULL) { /* MultiTable is actually a subquery, return the query tree below */ Node *childNode = QueryJoinTree(unaryNode->childNode, dependentJobList, rangeTableList); return childNode; } else { RangeTblRef *rangeTableRef = makeNode(RangeTblRef); uint32 rangeTableId = rangeTableNode->rangeTableId; rangeTableRef->rtindex = NewTableId(rangeTableId, *rangeTableList); return (Node *) rangeTableRef; } } case T_MultiCollect: { List *tableIdList = OutputTableIdList(multiNode); Job *dependentJob = JobForTableIdList(dependentJobList, tableIdList); List *dependentTargetList = dependentJob->jobQuery->targetList; /* compute column names for the derived table */ uint32 columnCount = (uint32) list_length(dependentTargetList); List *columnNameList = DerivedColumnNameList(columnCount, dependentJob->jobId); List *funcColumnNames = NIL; List *funcColumnTypes = NIL; List *funcColumnTypeMods = NIL; List *funcCollations = NIL; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, dependentTargetList) { Node *expr = (Node *) targetEntry->expr; char *name = targetEntry->resname; if (name == NULL) { name = pstrdup("unnamed"); } funcColumnNames = lappend(funcColumnNames, makeString(name)); funcColumnTypes = lappend_oid(funcColumnTypes, exprType(expr)); funcColumnTypeMods = lappend_int(funcColumnTypeMods, exprTypmod(expr)); funcCollations = lappend_oid(funcCollations, exprCollation(expr)); } RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, columnNameList, tableIdList, funcColumnNames, funcColumnTypes, funcColumnTypeMods, funcCollations); RangeTblRef *rangeTableRef = makeNode(RangeTblRef); rangeTableRef->rtindex = list_length(*rangeTableList) + 1; *rangeTableList = lappend(*rangeTableList, rangeTableEntry); return (Node *) rangeTableRef; } case T_MultiCartesianProduct: { MultiBinaryNode *binaryNode = (MultiBinaryNode *) multiNode; JoinExpr *joinExpr = makeNode(JoinExpr); joinExpr->jointype = JOIN_INNER; joinExpr->isNatural = false; joinExpr->larg = QueryJoinTree(binaryNode->leftChildNode, dependentJobList, rangeTableList); joinExpr->rarg = QueryJoinTree(binaryNode->rightChildNode, dependentJobList, rangeTableList); joinExpr->usingClause = NIL; joinExpr->alias = NULL; joinExpr->quals = NULL; joinExpr->rtindex = list_length(*rangeTableList) + 1; RangeTblEntry *rangeTableEntry = JoinRangeTableEntry(joinExpr, dependentJobList, *rangeTableList); *rangeTableList = lappend(*rangeTableList, rangeTableEntry); return (Node *) joinExpr; } case T_MultiTreeRoot: case T_MultiSelect: case T_MultiProject: case T_MultiExtendedOp: case T_MultiPartition: { MultiUnaryNode *unaryNode = (MultiUnaryNode *) multiNode; Assert(UnaryOperator(multiNode)); Node *childNode = QueryJoinTree(unaryNode->childNode, dependentJobList, rangeTableList); return childNode; } default: { ereport(ERROR, (errmsg("unrecognized multi-node type: %d", nodeType))); } } } /* * JoinRangeTableEntry builds a range table entry for a fully initialized JoinExpr node. * The column names and vars are determined using expandRTE, analogous to * transformFromClauseItem. */ static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); List *leftColumnNames = NIL; List *leftColumnVars = NIL; List *joinedColumnNames = NIL; List *joinedColumnVars = NIL; int leftRangeTableId = ExtractRangeTableId(joinExpr->larg); RangeTblEntry *leftRTE = rt_fetch(leftRangeTableId, rangeTableList); List *rightColumnNames = NIL; List *rightColumnVars = NIL; int rightRangeTableId = ExtractRangeTableId(joinExpr->rarg); RangeTblEntry *rightRTE = rt_fetch(rightRangeTableId, rangeTableList); rangeTableEntry->rtekind = RTE_JOIN; rangeTableEntry->relid = InvalidOid; rangeTableEntry->inFromCl = true; rangeTableEntry->alias = joinExpr->alias; rangeTableEntry->jointype = joinExpr->jointype; rangeTableEntry->subquery = NULL; rangeTableEntry->eref = makeAlias("unnamed_join", NIL); RangeTblEntry *leftCallingRTE = ConstructCallingRTE(leftRTE, dependentJobList); RangeTblEntry *rightCallingRte = ConstructCallingRTE(rightRTE, dependentJobList); ExtractColumns(leftCallingRTE, leftRangeTableId, &leftColumnNames, &leftColumnVars); ExtractColumns(rightCallingRte, rightRangeTableId, &rightColumnNames, &rightColumnVars); Oid leftRelId = leftCallingRTE->relid; Oid rightRelId = rightCallingRte->relid; joinedColumnNames = list_concat(joinedColumnNames, leftColumnNames); joinedColumnNames = list_concat(joinedColumnNames, rightColumnNames); joinedColumnVars = list_concat(joinedColumnVars, leftColumnVars); joinedColumnVars = list_concat(joinedColumnVars, rightColumnVars); rangeTableEntry->eref->colnames = joinedColumnNames; rangeTableEntry->joinaliasvars = joinedColumnVars; SetJoinRelatedColumnsCompat(rangeTableEntry, leftRelId, rightRelId, leftColumnVars, rightColumnVars); return rangeTableEntry; } /* * SetJoinRelatedColumnsCompat sets join related fields on the given range table entry. * Currently it sets joinleftcols/joinrightcols which are introduced with postgres 13. * For more info see postgres commit: 9ce77d75c5ab094637cc4a446296dc3be6e3c221 */ static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, Oid leftRelId, Oid rightRelId, List *leftColumnVars, List *rightColumnVars) { /* We don't have any merged columns so set it to 0 */ rangeTableEntry->joinmergedcols = 0; if (OidIsValid(leftRelId)) { rangeTableEntry->joinleftcols = GetColumnOriginalIndexes(leftRelId); } else { int leftColsSize = list_length(leftColumnVars); rangeTableEntry->joinleftcols = GeneratePositiveIntSequenceList(leftColsSize); } if (OidIsValid(rightRelId)) { rangeTableEntry->joinrightcols = GetColumnOriginalIndexes(rightRelId); } else { int rightColsSize = list_length(rightColumnVars); rangeTableEntry->joinrightcols = GeneratePositiveIntSequenceList(rightColsSize); } } /* * GetColumnOriginalIndexes gets the original indexes of columns by taking column drops into account. */ static List * GetColumnOriginalIndexes(Oid relationId) { List *originalIndexes = NIL; Relation relation = table_open(relationId, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); for (int columnIndex = 0; columnIndex < tupleDescriptor->natts; columnIndex++) { Form_pg_attribute currentColumn = TupleDescAttr(tupleDescriptor, columnIndex); if (currentColumn->attisdropped) { continue; } originalIndexes = lappend_int(originalIndexes, columnIndex + 1); } table_close(relation, NoLock); return originalIndexes; } /* * ExtractRangeTableId gets the range table id from a node that could * either be a JoinExpr or RangeTblRef. */ static int ExtractRangeTableId(Node *node) { int rangeTableId = 0; if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; rangeTableId = joinExpr->rtindex; } else if (IsA(node, RangeTblRef)) { RangeTblRef *rangeTableRef = (RangeTblRef *) node; rangeTableId = rangeTableRef->rtindex; } Assert(rangeTableId > 0); return rangeTableId; } /* * ExtractColumns gets a list of column names and vars for a given range * table entry using expandRTE. */ static void ExtractColumns(RangeTblEntry *callingRTE, int rangeTableId, List **columnNames, List **columnVars) { int subLevelsUp = 0; int location = -1; bool includeDroppedColumns = false; #if PG_VERSION_NUM >= PG_VERSION_18 expandRTE(callingRTE, rangeTableId, subLevelsUp, VAR_RETURNING_DEFAULT, /* new argument on PG 18+ */ location, includeDroppedColumns, columnNames, columnVars); #else expandRTE(callingRTE, rangeTableId, subLevelsUp, location, includeDroppedColumns, columnNames, columnVars); #endif } /* * ConstructCallingRTE constructs a calling RTE from the given range table entry and * dependentJobList in case of repartition joins. Since the range table entries in a job * query are mocked RTE_FUNCTION entries, this construction is needed to form an RTE * that expandRTE can handle. */ static RangeTblEntry * ConstructCallingRTE(RangeTblEntry *rangeTableEntry, List *dependentJobList) { RangeTblEntry *callingRTE = NULL; CitusRTEKind rangeTableKind = GetRangeTblKind(rangeTableEntry); if (rangeTableKind == CITUS_RTE_JOIN) { /* * For joins, we can call expandRTE directly. */ callingRTE = rangeTableEntry; } else if (rangeTableKind == CITUS_RTE_RELATION) { /* * For distributed tables, we construct a regular table RTE to call * expandRTE, which will extract columns from the distributed table * schema. */ callingRTE = makeNode(RangeTblEntry); callingRTE->rtekind = RTE_RELATION; callingRTE->eref = rangeTableEntry->eref; callingRTE->relid = rangeTableEntry->relid; callingRTE->inh = rangeTableEntry->inh; } else if (rangeTableKind == CITUS_RTE_REMOTE_QUERY) { Job *dependentJob = JobForRangeTable(dependentJobList, rangeTableEntry); Query *jobQuery = dependentJob->jobQuery; /* * For re-partition jobs, we construct a subquery RTE to call expandRTE, * which will extract the columns from the target list of the job query. */ callingRTE = makeNode(RangeTblEntry); callingRTE->rtekind = RTE_SUBQUERY; callingRTE->eref = rangeTableEntry->eref; callingRTE->subquery = jobQuery; } else { ereport(ERROR, (errmsg("unsupported Citus RTE kind: %d", rangeTableKind))); } return callingRTE; } /* * QueryFromList creates the from list construct that is used for building the * query's join tree. The function creates the from list by making a range table * reference for each entry in the given range table list. */ static List * QueryFromList(List *rangeTableList) { List *fromList = NIL; int rangeTableCount = list_length(rangeTableList); for (Index rangeTableIndex = 1; rangeTableIndex <= rangeTableCount; rangeTableIndex++) { RangeTblRef *rangeTableReference = makeNode(RangeTblRef); rangeTableReference->rtindex = rangeTableIndex; fromList = lappend(fromList, rangeTableReference); } return fromList; } /* * BuildSubqueryJobQuery traverses the given logical plan tree, finds MultiTable * which represents the subquery. It builds the query structure by adding this * subquery as it is to range table list of the query. * * Such as if user runs a query like this; * * SELECT avg(id) FROM ( * SELECT ... FROM () * ) * * then this function will build this worker query as keeping subquery as it is; * * SELECT sum(id), count(id) FROM ( * SELECT ... FROM () * ) */ static Query * BuildSubqueryJobQuery(MultiNode *multiNode) { List *targetList = NIL; List *sortClauseList = NIL; Node *havingQual = NULL; Node *limitCount = NULL; Node *limitOffset = NULL; bool hasAggregates = false; List *distinctClause = NIL; bool hasDistinctOn = false; bool hasWindowFuncs = false; List *windowClause = NIL; /* we start building jobs from below the collect node */ Assert(!CitusIsA(multiNode, MultiCollect)); List *subqueryMultiTableList = SubqueryMultiTableList(multiNode); Assert(list_length(subqueryMultiTableList) == 1); MultiTable *multiTable = (MultiTable *) linitial(subqueryMultiTableList); Query *subquery = multiTable->subquery; /* build subquery range table list */ RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->rtekind = RTE_SUBQUERY; rangeTableEntry->inFromCl = true; rangeTableEntry->eref = multiTable->referenceNames; rangeTableEntry->alias = multiTable->alias; rangeTableEntry->subquery = subquery; List *rangeTableList = list_make1(rangeTableEntry); /* * If we have an extended operator, then we copy the operator's target list. * Otherwise, we use the target list based on the MultiProject node at this * level in the query tree. */ List *extendedOpNodeList = FindNodesOfType(multiNode, T_MultiExtendedOp); if (extendedOpNodeList != NIL) { MultiExtendedOp *extendedOp = (MultiExtendedOp *) linitial(extendedOpNodeList); targetList = copyObject(extendedOp->targetList); } else { targetList = QueryTargetList(multiNode); } /* extract limit count/offset, sort and having clauses */ if (extendedOpNodeList != NIL) { MultiExtendedOp *extendedOp = (MultiExtendedOp *) linitial(extendedOpNodeList); limitCount = extendedOp->limitCount; limitOffset = extendedOp->limitOffset; sortClauseList = extendedOp->sortClauseList; havingQual = extendedOp->havingQual; distinctClause = extendedOp->distinctClause; hasDistinctOn = extendedOp->hasDistinctOn; hasWindowFuncs = extendedOp->hasWindowFuncs; windowClause = extendedOp->windowClause; } /* build group clauses */ List *groupClauseList = QueryGroupClauseList(multiNode); /* build the where clause list using select predicates */ List *whereClauseList = QuerySelectClauseList(multiNode); if (contain_aggs_of_level((Node *) targetList, 0) || contain_aggs_of_level((Node *) havingQual, 0)) { hasAggregates = true; } /* distinct is not sent to worker query if there are top level aggregates */ if (hasAggregates) { hasDistinctOn = false; distinctClause = NIL; } /* * Build the From/Where construct. We keep the where-clause list implicitly * AND'd, since both partition and join pruning depends on the clauses being * expressed as a list. */ FromExpr *joinTree = makeNode(FromExpr); joinTree->quals = (Node *) whereClauseList; joinTree->fromlist = QueryFromList(rangeTableList); /* build the query structure for this job */ Query *jobQuery = makeNode(Query); jobQuery->commandType = CMD_SELECT; jobQuery->querySource = QSRC_ORIGINAL; jobQuery->canSetTag = true; jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; jobQuery->limitCount = limitCount; jobQuery->havingQual = havingQual; jobQuery->hasAggs = hasAggregates; jobQuery->hasDistinctOn = hasDistinctOn; jobQuery->distinctClause = distinctClause; jobQuery->hasWindowFuncs = hasWindowFuncs; jobQuery->windowClause = windowClause; jobQuery->hasSubLinks = checkExprHasSubLink((Node *) jobQuery); Assert(jobQuery->hasWindowFuncs == contain_window_function((Node *) jobQuery)); return jobQuery; } /* * UpdateAllColumnAttributes extracts column references from provided columnContainer * and calls UpdateColumnAttributes to updates the column's range table reference (varno) and * column attribute number for the range table (varattno). */ static void UpdateAllColumnAttributes(Node *columnContainer, List *rangeTableList, List *dependentJobList) { ListCell *columnCell = NULL; List *columnList = pull_var_clause_default(columnContainer); foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); UpdateColumnAttributes(column, rangeTableList, dependentJobList); } } /* * UpdateColumnAttributes updates the column's range table reference (varno) and * column attribute number for the range table (varattno). The function uses the * newly built range table list to update the given column's attributes. */ static void UpdateColumnAttributes(Var *column, List *rangeTableList, List *dependentJobList) { Index originalTableId = column->varnosyn; AttrNumber originalColumnId = column->varattnosyn; /* find the new table identifier */ Index newTableId = NewTableId(originalTableId, rangeTableList); AttrNumber newColumnId = originalColumnId; /* if this is a derived table, find the new column identifier */ RangeTblEntry *newRangeTableEntry = rt_fetch(newTableId, rangeTableList); if (GetRangeTblKind(newRangeTableEntry) == CITUS_RTE_REMOTE_QUERY) { newColumnId = NewColumnId(originalTableId, originalColumnId, newRangeTableEntry, dependentJobList); } column->varno = newTableId; column->varattno = newColumnId; } /* * NewTableId determines the new tableId for the query that is currently being * built. In this query, the original tableId represents the order of the table * in the initial parse tree. When queries involve repartitioning, we re-order * tables; and the new tableId corresponds to this new table order. */ static Index NewTableId(Index originalTableId, List *rangeTableList) { Index rangeTableIndex = 1; ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); List *originalTableIdList = NIL; ExtractRangeTblExtraData(rangeTableEntry, NULL, NULL, NULL, &originalTableIdList); bool listMember = list_member_int(originalTableIdList, originalTableId); if (listMember) { return rangeTableIndex; } rangeTableIndex++; } ereport(ERROR, (errmsg("Unrecognized range table id %d", (int) originalTableId))); return 0; } /* * NewColumnId determines the new columnId for the query that is currently being * built. In this query, the original columnId corresponds to the column in base * tables. When the current query is a partition job and generates intermediate * tables, the columns have a different order and the new columnId corresponds * to this order. Please note that this function assumes columnIds for dependent * jobs have already been updated. */ static AttrNumber NewColumnId(Index originalTableId, AttrNumber originalColumnId, RangeTblEntry *newRangeTableEntry, List *dependentJobList) { AttrNumber newColumnId = 1; AttrNumber columnIndex = 1; Job *dependentJob = JobForRangeTable(dependentJobList, newRangeTableEntry); List *targetEntryList = dependentJob->jobQuery->targetList; ListCell *targetEntryCell = NULL; foreach(targetEntryCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Expr *expression = targetEntry->expr; Var *column = (Var *) expression; Assert(IsA(expression, Var)); /* * Check against the *old* values for this column, as the new values * would have been updated already. */ if (column->varnosyn == originalTableId && column->varattnosyn == originalColumnId) { newColumnId = columnIndex; break; } columnIndex++; } return newColumnId; } /* * JobForRangeTable returns the job that corresponds to the given range table * entry. The function walks over jobs in the given job list, and compares each * job's table list against the given range table entry's table list. When two * table lists match, the function returns the matching job. Note that we call * this function in practice when we need to determine which one of the jobs we * depend upon corresponds to given range table entry. */ static Job * JobForRangeTable(List *jobList, RangeTblEntry *rangeTableEntry) { List *searchedTableIdList = NIL; CitusRTEKind rangeTableKind; ExtractRangeTblExtraData(rangeTableEntry, &rangeTableKind, NULL, NULL, &searchedTableIdList); Assert(rangeTableKind == CITUS_RTE_REMOTE_QUERY); Job *searchedJob = JobForTableIdList(jobList, searchedTableIdList); return searchedJob; } /* * JobForTableIdList returns the job that corresponds to the given * tableIdList. The function walks over jobs in the given job list, and * compares each job's table list against the given table list. When the * two table lists match, the function returns the matching job. */ static Job * JobForTableIdList(List *jobList, List *searchedTableIdList) { Job *searchedJob = NULL; ListCell *jobCell = NULL; foreach(jobCell, jobList) { Job *job = (Job *) lfirst(jobCell); List *jobRangeTableList = job->jobQuery->rtable; List *jobTableIdList = NIL; ListCell *jobRangeTableCell = NULL; foreach(jobRangeTableCell, jobRangeTableList) { RangeTblEntry *jobRangeTable = (RangeTblEntry *) lfirst(jobRangeTableCell); List *tableIdList = NIL; ExtractRangeTblExtraData(jobRangeTable, NULL, NULL, NULL, &tableIdList); /* copy the list since list_concat is destructive */ tableIdList = list_copy(tableIdList); jobTableIdList = list_concat(jobTableIdList, tableIdList); } /* * Check if the searched range table's tableIds and the current job's * tableIds are the same. */ List *lhsDiff = list_difference_int(jobTableIdList, searchedTableIdList); List *rhsDiff = list_difference_int(searchedTableIdList, jobTableIdList); if (lhsDiff == NIL && rhsDiff == NIL) { searchedJob = job; break; } } Assert(searchedJob != NULL); return searchedJob; } /* Returns the list of children for the given multi node. */ static List * ChildNodeList(MultiNode *multiNode) { List *childNodeList = NIL; bool isUnaryNode = UnaryOperator(multiNode); bool isBinaryNode = BinaryOperator(multiNode); /* relation table nodes don't have any children */ if (CitusIsA(multiNode, MultiTable)) { MultiTable *multiTable = (MultiTable *) multiNode; if (multiTable->relationId != SUBQUERY_RELATION_ID) { return NIL; } } if (isUnaryNode) { MultiUnaryNode *unaryNode = (MultiUnaryNode *) multiNode; childNodeList = list_make1(unaryNode->childNode); } else if (isBinaryNode) { MultiBinaryNode *binaryNode = (MultiBinaryNode *) multiNode; childNodeList = list_make2(binaryNode->leftChildNode, binaryNode->rightChildNode); } return childNodeList; } /* * UniqueJobId allocates and returns a unique jobId for the job to be executed. * * The resulting job ID is built up as: * <16-bit group ID><24-bit process ID><1-bit secondary flag><23-bit local counter> * * When citus.enable_unique_job_ids is off then only the local counter is * included to get repeatable results. */ uint64 UniqueJobId(void) { static uint32 jobIdCounter = 0; uint64 jobId = 0; uint64 processId = 0; uint64 localGroupId = 0; jobIdCounter++; if (EnableUniqueJobIds) { /* * Add the local group id information to the jobId to * prevent concurrent jobs on different groups to conflict. */ localGroupId = GetLocalGroupId() & 0xFF; jobId = jobId | (localGroupId << 48); /* * Add the current process ID to distinguish jobs by this * backends from jobs started by other backends. Process * IDs can have at most 24-bits on platforms supported by * Citus. */ processId = MyProcPid & 0xFFFFFF; jobId = jobId | (processId << 24); /* * Add an extra bit for secondaries to distinguish their * jobs from primaries. */ if (RecoveryInProgress()) { jobId = jobId | (1 << 23); } } /* * Use the remaining 23 bits to distinguish jobs by the * same backend. */ uint64 jobIdNumber = jobIdCounter & 0x1FFFFFF; jobId = jobId | jobIdNumber; return jobId; } /* Builds a job from the given job query and dependent job list. */ static Job * BuildJob(Query *jobQuery, List *dependentJobList) { Job *job = CitusMakeNode(Job); job->jobId = UniqueJobId(); job->jobQuery = jobQuery; job->dependentJobList = dependentJobList; job->requiresCoordinatorEvaluation = false; return job; } /* * BuildMapMergeJob builds a MapMerge job from the given query and dependent job * list. The function then copies and updates the logical plan's partition * column, and uses the join rule type to determine the physical repartitioning * method to apply. */ static MapMergeJob * BuildMapMergeJob(Query *jobQuery, List *dependentJobList, Var *partitionKey, PartitionType partitionType, Oid baseRelationId, BoundaryNodeJobType boundaryNodeJobType) { List *rangeTableList = jobQuery->rtable; Var *partitionColumn = copyObject(partitionKey); /* update the logical partition key's table and column identifiers */ UpdateColumnAttributes(partitionColumn, rangeTableList, dependentJobList); MapMergeJob *mapMergeJob = CitusMakeNode(MapMergeJob); mapMergeJob->job.jobId = UniqueJobId(); mapMergeJob->job.jobQuery = jobQuery; mapMergeJob->job.dependentJobList = dependentJobList; mapMergeJob->partitionColumn = partitionColumn; mapMergeJob->sortedShardIntervalArrayLength = 0; /* * We assume dual partition join defaults to hash partitioning, and single * partition join defaults to range partitioning. In practice, the join type * should have no impact on the physical repartitioning (hash/range) method. * If join type is not set, this means this job represents a subquery, and * uses hash partitioning. */ if (partitionType == DUAL_HASH_PARTITION_TYPE) { uint32 partitionCount = HashPartitionCount(); mapMergeJob->partitionType = DUAL_HASH_PARTITION_TYPE; mapMergeJob->partitionCount = partitionCount; } else if (partitionType == SINGLE_HASH_PARTITION_TYPE || partitionType == RANGE_PARTITION_TYPE) { CitusTableCacheEntry *cache = GetCitusTableCacheEntry(baseRelationId); int shardCount = cache->shardIntervalArrayLength; ShardInterval **cachedSortedShardIntervalArray = cache->sortedShardIntervalArray; bool hasUninitializedShardInterval = cache->hasUninitializedShardInterval; ShardInterval **sortedShardIntervalArray = palloc0(sizeof(ShardInterval) * shardCount); for (int shardIndex = 0; shardIndex < shardCount; shardIndex++) { sortedShardIntervalArray[shardIndex] = CopyShardInterval(cachedSortedShardIntervalArray[shardIndex]); } if (hasUninitializedShardInterval) { ereport(ERROR, (errmsg("cannot range repartition shard with " "missing min/max values"))); } mapMergeJob->partitionType = partitionType; mapMergeJob->partitionCount = (uint32) shardCount; mapMergeJob->sortedShardIntervalArray = sortedShardIntervalArray; mapMergeJob->sortedShardIntervalArrayLength = shardCount; } return mapMergeJob; } /* * HashPartitionCount returns the number of partition files we create for a hash * partition task. The function follows Hadoop's method for picking the number * of reduce tasks: 0.95 or 1.75 * node count * max reduces per node. We choose * the lower constant 0.95 so that all tasks can start immediately, but round it * to 1.0 so that we have a smooth number of partition tasks. */ static uint32 HashPartitionCount(void) { uint32 groupCount = list_length(ActiveReadableNodeList()); double maxReduceTasksPerNode = RepartitionJoinBucketCountPerNode; uint32 partitionCount = (uint32) rint(groupCount * maxReduceTasksPerNode); return partitionCount; } /* ------------------------------------------------------------ * Functions that relate to building and assigning tasks follow * ------------------------------------------------------------ */ /* * BuildJobTreeTaskList takes in the given job tree and walks over jobs in this * tree bottom up. The function then creates tasks for each job in the tree, * sets dependencies between tasks and their downstream dependencies and assigns * tasks to worker nodes. */ static Job * BuildJobTreeTaskList(Job *jobTree, PlannerRestrictionContext *plannerRestrictionContext) { List *flattenedJobList = NIL; /* * We traverse the job tree in preorder, and append each visited job to our * flattened list. This way, each job in our list appears before the jobs it * depends on. */ List *jobStack = list_make1(jobTree); while (jobStack != NIL) { Job *job = (Job *) llast(jobStack); flattenedJobList = lappend(flattenedJobList, job); /* pop top element and push its children to the stack */ jobStack = list_delete_ptr(jobStack, job); jobStack = list_union_ptr(jobStack, job->dependentJobList); } /* * We walk the job list in reverse order to visit jobs bottom up. This way, * we can create dependencies between tasks bottom up, and assign them to * worker nodes accordingly. */ uint32 flattenedJobCount = (int32) list_length(flattenedJobList); for (int32 jobIndex = (flattenedJobCount - 1); jobIndex >= 0; jobIndex--) { Job *job = (Job *) list_nth(flattenedJobList, jobIndex); List *sqlTaskList = NIL; ListCell *assignedSqlTaskCell = NULL; /* create sql tasks for the job, and prune redundant data fetch tasks */ if (job->subqueryPushdown) { bool isMultiShardQuery = false; List *prunedRelationShardList = TargetShardIntervalsForRestrictInfo(plannerRestrictionContext-> relationRestrictionContext, &isMultiShardQuery, NULL); DeferredErrorMessage *deferredErrorMessage = NULL; sqlTaskList = QueryPushdownSqlTaskList(job->jobQuery, job->jobId, plannerRestrictionContext-> relationRestrictionContext, prunedRelationShardList, READ_TASK, false, &deferredErrorMessage); if (deferredErrorMessage != NULL) { RaiseDeferredErrorInternal(deferredErrorMessage, ERROR); } } else { sqlTaskList = SqlTaskList(job); } sqlTaskList = PruneSqlTaskDependencies(sqlTaskList); /* * We first assign sql and merge tasks to worker nodes. Next, we assign * sql tasks' data fetch dependencies. */ List *assignedSqlTaskList = AssignTaskList(sqlTaskList); AssignDataFetchDependencies(assignedSqlTaskList); /* if the parameters has not been resolved, record it */ job->parametersInJobQueryResolved = !HasUnresolvedExternParamsWalker((Node *) job->jobQuery, NULL); /* * Make final adjustments for the assigned tasks. * * First, update SELECT tasks' parameters resolved field. * * Second, assign merge task's data fetch dependencies. */ foreach(assignedSqlTaskCell, assignedSqlTaskList) { Task *assignedSqlTask = (Task *) lfirst(assignedSqlTaskCell); /* we don't support parameters in the physical planner */ if (assignedSqlTask->taskType == READ_TASK) { assignedSqlTask->parametersInQueryStringResolved = job->parametersInJobQueryResolved; } List *assignedMergeTaskList = FindDependentMergeTaskList(assignedSqlTask); AssignDataFetchDependencies(assignedMergeTaskList); } /* * If we have a MapMerge job, the map tasks in this job wrap around the * SQL tasks and their assignments. */ if (CitusIsA(job, MapMergeJob)) { MapMergeJob *mapMergeJob = (MapMergeJob *) job; uint32 taskIdIndex = TaskListHighestTaskId(assignedSqlTaskList) + 1; List *mapTaskList = MapTaskList(mapMergeJob, assignedSqlTaskList); List *mergeTaskList = MergeTaskList(mapMergeJob, mapTaskList, taskIdIndex); mapMergeJob->mapTaskList = mapTaskList; mapMergeJob->mergeTaskList = mergeTaskList; } else { job->taskList = assignedSqlTaskList; } } return jobTree; } /* * QueryPushdownSqlTaskList creates a list of SQL tasks to execute the given subquery * pushdown job. For this, it is being checked whether the query is router * plannable per target shard interval. For those router plannable worker * queries, we create a SQL task and append the task to the task list that is going * to be executed. */ List * QueryPushdownSqlTaskList(Query *query, uint64 jobId, RelationRestrictionContext *relationRestrictionContext, List *prunedRelationShardList, TaskType taskType, bool modifyRequiresCoordinatorEvaluation, DeferredErrorMessage **planningError) { List *sqlTaskList = NIL; uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ int minShardOffset = INT_MAX; int prevShardCount = 0; Bitmapset *taskRequiredForShardIndex = NULL; Bitmapset *distributedTableIndex = NULL; /* error if shards are not co-partitioned */ ErrorIfUnsupportedShardDistribution(query); if (list_length(relationRestrictionContext->relationRestrictionList) == 0) { *planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot handle complex subqueries when the " "router executor is disabled", NULL, NULL); return NIL; } RelationRestriction *relationRestriction = NULL; List *prunedShardList = NULL; /* First loop, gather the indexes of distributed tables * this is required to decide whether we can skip shards * from inner tables of outer joins */ foreach_declared_ptr(relationRestriction, relationRestrictionContext->relationRestrictionList) { Oid relationId = relationRestriction->relationId; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (!HasDistributionKeyCacheEntry(cacheEntry)) { continue; } /* we expect distributed tables to have the same shard count */ if (prevShardCount > 0 && prevShardCount != cacheEntry->shardIntervalArrayLength) { *planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "shard counts of co-located tables do not " "match", NULL, NULL); return NIL; } prevShardCount = cacheEntry->shardIntervalArrayLength; distributedTableIndex = bms_add_member(distributedTableIndex, relationRestriction->index); } /* In the second loop, populate taskRequiredForShardIndex */ bool updateQualsForOuterJoin = false; bool outerPartHasDistributedTable = false; bool noDistTables = bms_is_empty(distributedTableIndex); bool hasRefOrSchemaShardedTable = false; forboth_ptr(prunedShardList, prunedRelationShardList, relationRestriction, relationRestrictionContext->relationRestrictionList) { Oid relationId = relationRestriction->relationId; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (!HasDistributionKeyCacheEntry(cacheEntry)) { if (noDistTables && !hasRefOrSchemaShardedTable) { /* * Before continuing, check if we're looking at a reference or schema- * sharded table. If so, and it is the first such table we've seen, we * add a task for shard index 0; all reference and schema sharded tables * have shard index 0 so we can hard-code the value rather than looking at * the shardIndex in pruned shard list, as is done further on down for * distributed tables. * * Note that this only needs to be done once, regardless of how many * reference or schema sharded tables there are; they all have the * same shard index (0), and will require just one task. * * Also note that this is only done if there are no distributed tables * involved; the relevant shard indexes will get added, and furthermore * we don't want to incorrectly add shard index 0 if for example a left * outer join between a reference table and a distributed table also has * a restriction that prunes out shard index 0 of the distributed table. */ CitusTableType currentTableType = GetCitusTableType(cacheEntry); hasRefOrSchemaShardedTable = currentTableType == REFERENCE_TABLE || currentTableType == SINGLE_SHARD_DISTRIBUTED; if (hasRefOrSchemaShardedTable) { taskRequiredForShardIndex = bms_add_member(taskRequiredForShardIndex, 0); minShardOffset = 0; } } continue; } /* * For left joins we don't care about the shards pruned for the right hand side. * If the right hand side would prune to a smaller set we should still send it to * all tables of the left hand side. However if the right hand side is bigger than * the left hand side we don't have to send the query to any shard that is not * matching anything on the left hand side. * * Instead we will simply skip any RelationRestriction if it is an OUTER join, * the table is part of the non-outer side of the join and the outer side has a * distributed table. */ if (IsInnerTableOfOuterJoin(relationRestriction, distributedTableIndex, &outerPartHasDistributedTable)) { if (outerPartHasDistributedTable) { /* we can skip the shards from this relation restriction */ continue; } else { /* The outer part does not include distributed tables, we can not skip shards. * Also, we will possibly update the quals of the outer relation for recurring join push down, mark here. */ updateQualsForOuterJoin = true; } } ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, prunedShardList) { int shardIndex = shardInterval->shardIndex; taskRequiredForShardIndex = bms_add_member(taskRequiredForShardIndex, shardIndex); minShardOffset = Min(minShardOffset, shardIndex); } } /* * We might fail to find outer joins from the relationRestrictionContext * when the original query has CTEs. In order to ensure that we always mark * the outer joins correctly and compute additional quals when necessary, * check the task query as well. */ if (!updateQualsForOuterJoin && FindNodeMatchingCheckFunction((Node *) query, IsOuterJoinExpr)) { /* * We have an outer join, so assume "might" need to update quals. * See the usage of this flag in QueryPushdownTaskCreate(). */ updateQualsForOuterJoin = true; } /* * We keep track of minShardOffset to skip over a potentially big amount of pruned * shards. However, we need to start at minShardOffset - 1 to make sure we don't * miss to first/min shard recorder as bms_next_member will return the first member * added after shardOffset. Meaning minShardOffset would be the first member we * expect. * * We don't have to keep track of maxShardOffset as the bitmapset will only have been * allocated till the last shard we have added. Therefore, the iterator will quickly * identify the end of the bitmapset. */ int shardOffset = minShardOffset - 1; while ((shardOffset = bms_next_member(taskRequiredForShardIndex, shardOffset)) >= 0) { Task *subqueryTask = QueryPushdownTaskCreate(query, shardOffset, relationRestrictionContext, taskIdIndex, taskType, modifyRequiresCoordinatorEvaluation, updateQualsForOuterJoin, planningError); if (*planningError != NULL) { return NIL; } subqueryTask->jobId = jobId; sqlTaskList = lappend(sqlTaskList, subqueryTask); ++taskIdIndex; } /* If we detected a reference or schema sharded table then there * should be no distributed tables involved and exactly one task. */ Assert(!hasRefOrSchemaShardedTable || (noDistTables && list_length(sqlTaskList) == 1)); /* If it is a modify task with multiple tables */ if (taskType == MODIFY_TASK && list_length( relationRestrictionContext->relationRestrictionList) > 1) { ListCell *taskCell = NULL; foreach(taskCell, sqlTaskList) { Task *task = (Task *) lfirst(taskCell); task->modifyWithSubquery = true; } } return sqlTaskList; } /* * IsInnerTableOfOuterJoin tests based on the join information envoded in a * RelationRestriction if the table accessed for this relation is * a) in an outer join * b) on the inner part of said join * * The function also sets outerPartHasDistributedTable if the outer part * of the corresponding join has a distributed table. */ static bool IsInnerTableOfOuterJoin(RelationRestriction *relationRestriction, Bitmapset *distributedTables, bool *outerPartHasDistributedTable) { RestrictInfo *joinInfo = NULL; foreach_declared_ptr(joinInfo, relationRestriction->relOptInfo->joininfo) { if (joinInfo->outer_relids == NULL) { /* not an outer join */ continue; } /* * This join restriction info describes an outer join, we need to figure out if * our table is in the non outer part of this join. If that is the case this is a * non outer table of an outer join. */ bool isInOuter = bms_is_member(relationRestriction->relOptInfo->relid, joinInfo->outer_relids); if (!isInOuter) { /* this table is joined in the inner part of an outer join */ /* set if the outer part has a distributed relation */ *outerPartHasDistributedTable = bms_overlap(joinInfo->outer_relids, distributedTables); /* this is an inner table of an outer join */ return true; } } /* we have not found any join clause that satisfies both requirements */ return false; } /* * ErrorIfUnsupportedShardDistribution gets list of relations in the given query * and checks if two conditions below hold for them, otherwise it errors out. * a. Every relation is distributed by range or hash. This means shards are * disjoint based on the partition column. * b. All relations have 1-to-1 shard partitioning between them. This means * shard count for every relation is same and for every shard in a relation * there is exactly one shard in other relations with same min/max values. */ static void ErrorIfUnsupportedShardDistribution(Query *query) { Oid firstTableRelationId = InvalidOid; List *relationIdList = DistributedRelationIdList(query); List *nonReferenceRelations = NIL; ListCell *relationIdCell = NULL; uint32 relationIndex = 0; uint32 rangeDistributedRelationCount = 0; uint32 hashDistOrSingleShardRelCount = 0; uint32 appendDistributedRelationCount = 0; foreach(relationIdCell, relationIdList) { Oid relationId = lfirst_oid(relationIdCell); if (IsCitusTableType(relationId, RANGE_DISTRIBUTED)) { rangeDistributedRelationCount++; nonReferenceRelations = lappend_oid(nonReferenceRelations, relationId); } else if (IsCitusTableType(relationId, HASH_DISTRIBUTED) || IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { hashDistOrSingleShardRelCount++; nonReferenceRelations = lappend_oid(nonReferenceRelations, relationId); } else if (IsCitusTable(relationId) && !HasDistributionKey(relationId)) { /* do not need to handle non-distributed tables */ continue; } else { appendDistributedRelationCount++; } } if ((rangeDistributedRelationCount > 0) && (hashDistOrSingleShardRelCount > 0)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot push down this subquery"), errdetail("A query including both range and hash " "partitioned relations are unsupported"))); } else if ((rangeDistributedRelationCount > 0) && (appendDistributedRelationCount > 0)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot push down this subquery"), errdetail("A query including both range and append " "partitioned relations are unsupported"))); } else if ((appendDistributedRelationCount > 0) && (hashDistOrSingleShardRelCount > 0)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot push down this subquery"), errdetail("A query including both append and hash " "partitioned relations are unsupported"))); } foreach(relationIdCell, nonReferenceRelations) { Oid relationId = lfirst_oid(relationIdCell); Oid currentRelationId = relationId; /* get shard list of first relation and continue for the next relation */ if (relationIndex == 0) { firstTableRelationId = relationId; relationIndex++; continue; } /* check if this table has 1-1 shard partitioning with first table */ bool coPartitionedTables = CoPartitionedTables(firstTableRelationId, currentRelationId); if (!coPartitionedTables) { char *firstRelName = get_rel_name(firstTableRelationId); char *currentRelName = get_rel_name(currentRelationId); int compareResult = strcmp(firstRelName, currentRelName); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot push down this subquery"), errdetail("%s and %s are not colocated", (compareResult > 0 ? currentRelName : firstRelName), (compareResult > 0 ? firstRelName : currentRelName)))); } } } /* * SubqueryTaskCreate creates a sql task by replacing the target * shardInterval's boundary value. */ static Task * QueryPushdownTaskCreate(Query *originalQuery, int shardIndex, RelationRestrictionContext *restrictionContext, uint32 taskId, TaskType taskType, bool modifyRequiresCoordinatorEvaluation, bool updateQualsForOuterJoin, DeferredErrorMessage **planningError) { Query *taskQuery = copyObject(originalQuery); StringInfo queryString = makeStringInfo(); ListCell *restrictionCell = NULL; List *taskShardList = NIL; List *relationShardList = NIL; uint64 jobId = INVALID_JOB_ID; uint64 anchorShardId = INVALID_SHARD_ID; bool modifyWithSubselect = false; RangeTblEntry *resultRangeTable = NULL; Oid resultRelationOid = InvalidOid; /* * If it is a modify query with sub-select, we need to set result relation shard's id * as anchor shard id. */ if (UpdateOrDeleteOrMergeQuery(originalQuery)) { resultRangeTable = rt_fetch(originalQuery->resultRelation, originalQuery->rtable); resultRelationOid = resultRangeTable->relid; modifyWithSubselect = true; } /* * Find the relevant shard out of each relation for this task. */ foreach(restrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(restrictionCell); Oid relationId = relationRestriction->relationId; ShardInterval *shardInterval = NULL; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (!HasDistributionKeyCacheEntry(cacheEntry)) { /* non-distributed tables have only one shard */ shardInterval = cacheEntry->sortedShardIntervalArray[0]; /* use as anchor shard only if we couldn't find any yet */ if (anchorShardId == INVALID_SHARD_ID) { anchorShardId = shardInterval->shardId; } } else if (UpdateOrDeleteOrMergeQuery(originalQuery)) { shardInterval = cacheEntry->sortedShardIntervalArray[shardIndex]; if (!modifyWithSubselect || relationId == resultRelationOid) { /* for UPDATE/DELETE the shard in the result relation becomes the anchor shard */ anchorShardId = shardInterval->shardId; } } else { /* for SELECT we pick an arbitrary shard as the anchor shard */ shardInterval = cacheEntry->sortedShardIntervalArray[shardIndex]; anchorShardId = shardInterval->shardId; } ShardInterval *copiedShardInterval = CopyShardInterval(shardInterval); taskShardList = lappend(taskShardList, list_make1(copiedShardInterval)); RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = copiedShardInterval->relationId; relationShard->shardId = copiedShardInterval->shardId; relationShardList = lappend(relationShardList, relationShard); } Assert(anchorShardId != INVALID_SHARD_ID); List *taskPlacementList = PlacementsForWorkersContainingAllShards(taskShardList); if (list_length(taskPlacementList) == 0) { *planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot find a worker that has active placements for all " "shards in the query", NULL, NULL); return NULL; } /* * Augment the relations in the query with the shard IDs. */ UpdateRelationToShardNames((Node *) taskQuery, relationShardList); /* * Ands are made implicit during shard pruning, as predicate comparison and * refutation depend on it being so. We need to make them explicit again so * that the query string is generated as (...) AND (...) as opposed to * (...), (...). */ if (taskQuery->jointree->quals != NULL && IsA(taskQuery->jointree->quals, List)) { taskQuery->jointree->quals = (Node *) make_ands_explicit( (List *) taskQuery->jointree->quals); } if (updateQualsForOuterJoin) { /* * QueryPushdownSqlTaskList() might set this when it detects an outer join, * even if the outer join is not surely known to be happening between a * recurring and a distributed rel. However, it's still safe to call * UpdateWhereClauseToPushdownRecurringOuterJoinWalker() here as it only * acts on the where clause if the join is happening between a * recurring and a distributed rel. */ UpdateWhereClauseToPushdownRecurringOuterJoinWalker((Node *) taskQuery, relationShardList); } Task *subqueryTask = CreateBasicTask(jobId, taskId, taskType, NULL); if ((taskType == MODIFY_TASK && !modifyRequiresCoordinatorEvaluation) || taskType == READ_TASK) { pg_get_query_def(taskQuery, queryString); ereport(DEBUG4, (errmsg("distributed statement: %s", queryString->data))); SetTaskQueryString(subqueryTask, queryString->data); } subqueryTask->dependentTaskList = NULL; subqueryTask->anchorShardId = anchorShardId; subqueryTask->taskPlacementList = taskPlacementList; subqueryTask->relationShardList = relationShardList; return subqueryTask; } /* * CoPartitionedTables checks if given two distributed tables are co-located. */ bool CoPartitionedTables(Oid firstRelationId, Oid secondRelationId) { CitusTableCacheEntry *firstTableCache = GetCitusTableCacheEntry(firstRelationId); CitusTableCacheEntry *secondTableCache = GetCitusTableCacheEntry(secondRelationId); if (firstTableCache->partitionMethod == DISTRIBUTE_BY_APPEND || secondTableCache->partitionMethod == DISTRIBUTE_BY_APPEND) { /* * Append-distributed tables can have overlapping shards. Therefore they are * never co-partitioned, not even with themselves. */ return false; } /* * Check if the tables have the same colocation ID - if so, we know * they're colocated. */ if (firstTableCache->colocationId != INVALID_COLOCATION_ID && firstTableCache->colocationId == secondTableCache->colocationId) { return true; } if (firstRelationId == secondRelationId) { /* * Even without an explicit co-location ID, non-append tables can be considered * co-located with themselves. */ return true; } return false; } /* * SqlTaskList creates a list of SQL tasks to execute the given job. For this, * the function walks over each range table in the job's range table list, gets * each range table's table fragments, and prunes unneeded table fragments. The * function then joins table fragments from different range tables, and creates * all fragment combinations. For each created combination, the function builds * a SQL task, and appends this task to a task list. */ static List * SqlTaskList(Job *job) { List *sqlTaskList = NIL; uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint64 jobId = job->jobId; bool anchorRangeTableBasedAssignment = false; uint32 anchorRangeTableId = 0; Query *jobQuery = job->jobQuery; List *rangeTableList = jobQuery->rtable; List *whereClauseList = (List *) jobQuery->jointree->quals; List *dependentJobList = job->dependentJobList; /* * If we don't depend on a hash partition, then we determine the largest * table around which we build our queries. This reduces data fetching. */ bool dependsOnHashPartitionJob = DependsOnHashPartitionJob(job); if (!dependsOnHashPartitionJob) { anchorRangeTableBasedAssignment = true; anchorRangeTableId = AnchorRangeTableId(rangeTableList); Assert(anchorRangeTableId != 0); Assert(anchorRangeTableId <= list_length(rangeTableList)); } /* adjust our column old attributes for partition pruning to work */ AdjustColumnOldAttributes(whereClauseList); AdjustColumnOldAttributes(jobQuery->targetList); /* * Ands are made implicit during shard pruning, as predicate comparison and * refutation depend on it being so. We need to make them explicit again so * that the query string is generated as (...) AND (...) as opposed to * (...), (...). */ Node *whereClauseTree = (Node *) make_ands_explicit( (List *) jobQuery->jointree->quals); jobQuery->jointree->quals = whereClauseTree; /* * For each range table, we first get a list of their shards or merge tasks. * We also apply partition pruning based on the selection criteria. If all * range table fragments are pruned away, we return an empty task list. */ List *rangeTableFragmentsList = RangeTableFragmentsList(rangeTableList, whereClauseList, dependentJobList); if (rangeTableFragmentsList == NIL) { return NIL; } /* * We then generate fragment combinations according to how range tables join * with each other (and apply join pruning). Each fragment combination then * represents one SQL task's dependencies. */ List *fragmentCombinationList = FragmentCombinationList(rangeTableFragmentsList, jobQuery, dependentJobList); /* * Adjust RelabelType and CoerceViaIO nodes that are improper for deparsing. * We first check if there are any such nodes by using a query tree walker. * The reason is that a query tree mutator will create a deep copy of all * the query sublinks, and we don't want to do that unless necessary, as it * would be inefficient. */ if (QueryTreeHasImproperForDeparseNodes((Node *) jobQuery, NULL)) { jobQuery = (Query *) AdjustImproperForDeparseNodes((Node *) jobQuery, NULL); } ListCell *fragmentCombinationCell = NULL; foreach(fragmentCombinationCell, fragmentCombinationList) { List *fragmentCombination = (List *) lfirst(fragmentCombinationCell); /* create tasks to fetch fragments required for the sql task */ List *dataFetchTaskList = DataFetchTaskList(jobId, taskIdIndex, fragmentCombination); int32 dataFetchTaskCount = list_length(dataFetchTaskList); taskIdIndex += dataFetchTaskCount; /* update range table entries with fragment aliases (in place) */ Query *taskQuery = copyObject(jobQuery); List *fragmentRangeTableList = taskQuery->rtable; UpdateRangeTableAlias(fragmentRangeTableList, fragmentCombination); /* transform the updated task query to a SQL query string */ StringInfo sqlQueryString = makeStringInfo(); pg_get_query_def(taskQuery, sqlQueryString); Task *sqlTask = CreateBasicTask(jobId, taskIdIndex, READ_TASK, sqlQueryString->data); sqlTask->dependentTaskList = dataFetchTaskList; sqlTask->relationShardList = BuildRelationShardList(fragmentRangeTableList, fragmentCombination); /* log the query string we generated */ ereport(DEBUG4, (errmsg("generated sql query for task %d", sqlTask->taskId), errdetail("query string: \"%s\"", sqlQueryString->data))); sqlTask->anchorShardId = INVALID_SHARD_ID; if (anchorRangeTableBasedAssignment) { sqlTask->anchorShardId = AnchorShardId(fragmentCombination, anchorRangeTableId); } taskIdIndex++; sqlTaskList = lappend(sqlTaskList, sqlTask); } return sqlTaskList; } /* * RelabelTypeToCollateExpr converts RelabelType's into CollationExpr's. * With that, we will be able to pushdown COLLATE's. */ static CollateExpr * RelabelTypeToCollateExpr(RelabelType *relabelType) { Assert(OidIsValid(relabelType->resultcollid)); CollateExpr *collateExpr = makeNode(CollateExpr); collateExpr->arg = relabelType->arg; collateExpr->collOid = relabelType->resultcollid; collateExpr->location = relabelType->location; return collateExpr; } /* * DependsOnHashPartitionJob checks if the given job depends on a hash * partitioning job. */ static bool DependsOnHashPartitionJob(Job *job) { bool dependsOnHashPartitionJob = false; List *dependentJobList = job->dependentJobList; uint32 dependentJobCount = (uint32) list_length(dependentJobList); if (dependentJobCount > 0) { Job *dependentJob = (Job *) linitial(dependentJobList); if (CitusIsA(dependentJob, MapMergeJob)) { MapMergeJob *mapMergeJob = (MapMergeJob *) dependentJob; if (mapMergeJob->partitionType == DUAL_HASH_PARTITION_TYPE) { dependsOnHashPartitionJob = true; } } } return dependsOnHashPartitionJob; } /* * AnchorRangeTableId determines the table around which we build our queries, * and returns this table's range table id. We refer to this table as the anchor * table, and make sure that the anchor table's shards are moved or cached only * when absolutely necessary. */ static uint32 AnchorRangeTableId(List *rangeTableList) { uint32 anchorRangeTableId = 0; uint64 maxTableSize = 0; /* * We first filter anything but ordinary tables. Then, we pick the table(s) * with the most number of shards as our anchor table. If multiple tables * have the most number of shards, we have a draw. */ List *baseTableIdList = BaseRangeTableIdList(rangeTableList); List *anchorTableRTIList = AnchorRangeTableIdList(rangeTableList, baseTableIdList); ListCell *anchorTableIdCell = NULL; int anchorTableIdCount = list_length(anchorTableRTIList); Assert(anchorTableIdCount > 0); if (anchorTableIdCount == 1) { anchorRangeTableId = (uint32) linitial_int(anchorTableRTIList); return anchorRangeTableId; } /* * If more than one table has the most number of shards, we break the draw * by comparing table sizes and picking the table with the largest size. */ foreach(anchorTableIdCell, anchorTableRTIList) { uint32 anchorTableId = (uint32) lfirst_int(anchorTableIdCell); RangeTblEntry *tableEntry = rt_fetch(anchorTableId, rangeTableList); uint64 tableSize = 0; List *shardList = LoadShardList(tableEntry->relid); ListCell *shardCell = NULL; foreach(shardCell, shardList) { uint64 *shardIdPointer = (uint64 *) lfirst(shardCell); uint64 shardId = (*shardIdPointer); uint64 shardSize = ShardLength(shardId); tableSize += shardSize; } if (tableSize > maxTableSize) { maxTableSize = tableSize; anchorRangeTableId = anchorTableId; } } if (anchorRangeTableId == 0) { /* all tables have the same shard count and size 0, pick the first */ anchorRangeTableId = (uint32) linitial_int(anchorTableRTIList); } return anchorRangeTableId; } /* * BaseRangeTableIdList walks over range tables in the given range table list, * finds range tables that correspond to base (non-repartitioned) tables, and * returns these range tables' identifiers in a new list. */ static List * BaseRangeTableIdList(List *rangeTableList) { List *baseRangeTableIdList = NIL; uint32 rangeTableId = 1; ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (GetRangeTblKind(rangeTableEntry) == CITUS_RTE_RELATION) { baseRangeTableIdList = lappend_int(baseRangeTableIdList, rangeTableId); } rangeTableId++; } return baseRangeTableIdList; } /* * AnchorRangeTableIdList finds ordinary table(s) with the most number of shards * and returns the corresponding range table id(s) in a list. */ static List * AnchorRangeTableIdList(List *rangeTableList, List *baseRangeTableIdList) { List *anchorTableRTIList = NIL; uint32 maxShardCount = 0; ListCell *baseRangeTableIdCell = NULL; uint32 baseRangeTableCount = list_length(baseRangeTableIdList); if (baseRangeTableCount == 1) { return baseRangeTableIdList; } uint32 referenceTableRTI = 0; foreach(baseRangeTableIdCell, baseRangeTableIdList) { uint32 baseRangeTableId = (uint32) lfirst_int(baseRangeTableIdCell); RangeTblEntry *tableEntry = rt_fetch(baseRangeTableId, rangeTableList); Oid citusTableId = tableEntry->relid; if (IsCitusTableType(citusTableId, REFERENCE_TABLE)) { referenceTableRTI = baseRangeTableId; continue; } List *shardList = LoadShardList(citusTableId); uint32 shardCount = (uint32) list_length(shardList); if (shardCount > maxShardCount) { anchorTableRTIList = list_make1_int(baseRangeTableId); maxShardCount = shardCount; } else if (shardCount == maxShardCount) { anchorTableRTIList = lappend_int(anchorTableRTIList, baseRangeTableId); } } /* * We favor distributed tables over reference tables as anchor tables. But * in case we cannot find any distributed tables, we let reference table to be * anchor table. For now, we cannot see a query that might require this, but we * want to be backward compatiable. */ if (list_length(anchorTableRTIList) == 0) { return referenceTableRTI > 0 ? list_make1_int(referenceTableRTI) : NIL; } return anchorTableRTIList; } /* * AdjustColumnOldAttributes adjust the old tableId (varnosyn) and old columnId * (varattnosyn), and sets them equal to the new values. We need this adjustment * for partition pruning where we compare these columns with partition columns * loaded from system catalogs. Since columns loaded from system catalogs always * have the same old and new values, we also need to adjust column values here. */ static void AdjustColumnOldAttributes(List *expressionList) { List *columnList = pull_var_clause_default((Node *) expressionList); ListCell *columnCell = NULL; foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); column->varnosyn = column->varno; column->varattnosyn = column->varattno; } } /* * RangeTableFragmentsList walks over range tables in the given range table list * and for each table, the function creates a list of its fragments. A fragment * in this list represents either a regular shard or a merge task. Once a list * for each range table is constructed, the function applies partition pruning * using the given where clause list. Then, the function appends the fragment * list for each range table to a list of lists, and returns this list of lists. */ static List * RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, List *dependentJobList) { List *rangeTableFragmentsList = NIL; uint32 rangeTableIndex = 0; const uint32 fragmentSize = sizeof(RangeTableFragment); ListCell *rangeTableCell = NULL; foreach(rangeTableCell, rangeTableList) { uint32 tableId = rangeTableIndex + 1; /* tableId starts from 1 */ RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); CitusRTEKind rangeTableKind = GetRangeTblKind(rangeTableEntry); if (rangeTableKind == CITUS_RTE_RELATION) { Oid relationId = rangeTableEntry->relid; ListCell *shardIntervalCell = NULL; List *shardFragmentList = NIL; List *prunedShardIntervalList = PruneShards(relationId, tableId, whereClauseList, NULL); /* * If we prune all shards for one table, query results will be empty. * We can therefore return NIL for the task list here. */ if (prunedShardIntervalList == NIL) { return NIL; } foreach(shardIntervalCell, prunedShardIntervalList) { ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell); RangeTableFragment *shardFragment = palloc0(fragmentSize); shardFragment->fragmentReference = shardInterval; shardFragment->fragmentType = CITUS_RTE_RELATION; shardFragment->rangeTableId = tableId; shardFragmentList = lappend(shardFragmentList, shardFragment); } rangeTableFragmentsList = lappend(rangeTableFragmentsList, shardFragmentList); } else if (rangeTableKind == CITUS_RTE_REMOTE_QUERY) { List *mergeTaskFragmentList = NIL; ListCell *mergeTaskCell = NULL; Job *dependentJob = JobForRangeTable(dependentJobList, rangeTableEntry); Assert(CitusIsA(dependentJob, MapMergeJob)); MapMergeJob *dependentMapMergeJob = (MapMergeJob *) dependentJob; List *mergeTaskList = dependentMapMergeJob->mergeTaskList; /* if there are no tasks for the dependent job, just return NIL */ if (mergeTaskList == NIL) { return NIL; } foreach(mergeTaskCell, mergeTaskList) { Task *mergeTask = (Task *) lfirst(mergeTaskCell); RangeTableFragment *mergeTaskFragment = palloc0(fragmentSize); mergeTaskFragment->fragmentReference = mergeTask; mergeTaskFragment->fragmentType = CITUS_RTE_REMOTE_QUERY; mergeTaskFragment->rangeTableId = tableId; mergeTaskFragmentList = lappend(mergeTaskFragmentList, mergeTaskFragment); } rangeTableFragmentsList = lappend(rangeTableFragmentsList, mergeTaskFragmentList); } rangeTableIndex++; } return rangeTableFragmentsList; } /* * BuildBaseConstraint builds and returns a base constraint. This constraint * implements an expression in the form of (column <= max && column >= min), * where column is the partition key, and min and max values represent a shard's * min and max values. These shard values are filled in after the constraint is * built. */ Node * BuildBaseConstraint(Var *column) { /* Build these expressions with only one argument for now */ OpExpr *lessThanExpr = MakeOpExpression(column, BTLessEqualStrategyNumber); OpExpr *greaterThanExpr = MakeOpExpression(column, BTGreaterEqualStrategyNumber); /* Build base constaint as an and of two qual conditions */ Node *baseConstraint = make_and_qual((Node *) lessThanExpr, (Node *) greaterThanExpr); return baseConstraint; } /* * MakeOpExpressionExtended builds an operator expression node that's of * the form "Var Expr", where, Expr must either be a Const or a Var * (*1). * * This operator expression implements the operator clause as defined by * the variable and the strategy number. */ OpExpr * MakeOpExpressionExtended(Var *leftVar, Expr *rightArg, int16 strategyNumber) { /* * Other types of expressions are probably also fine to be used, but * none of the callers need support for them for now, so we haven't * tested them (*1). */ Assert(IsA(rightArg, Const) || IsA(rightArg, Var)); Oid typeId = leftVar->vartype; Oid collationId = leftVar->varcollid; Oid accessMethodId = BTREE_AM_OID; OperatorCacheEntry *operatorCacheEntry = LookupOperatorByType(typeId, accessMethodId, strategyNumber); Oid operatorId = operatorCacheEntry->operatorId; Oid operatorClassInputType = operatorCacheEntry->operatorClassInputType; char typeType = operatorCacheEntry->typeType; /* * Relabel variable if input type of default operator class is not equal to * the variable type. Note that we don't relabel the variable if the default * operator class variable type is a pseudo-type. */ if (operatorClassInputType != typeId && typeType != TYPTYPE_PSEUDO) { leftVar = (Var *) makeRelabelType((Expr *) leftVar, operatorClassInputType, -1, collationId, COERCE_IMPLICIT_CAST); } /* Now make the expression with the given variable and a null constant */ OpExpr *expression = (OpExpr *) make_opclause(operatorId, InvalidOid, /* no result type yet */ false, /* no return set */ (Expr *) leftVar, rightArg, InvalidOid, collationId); /* Set implementing function id and result type */ expression->opfuncid = get_opcode(operatorId); expression->opresulttype = get_func_rettype(expression->opfuncid); return expression; } /* * MakeOpExpression is a wrapper around MakeOpExpressionExtended * that creates a null constant of the appropriate type for right * hand side operator class input type. As a result, it builds an * operator expression node that's of the form "Var NULL". */ OpExpr * MakeOpExpression(Var *leftVar, int16 strategyNumber) { Oid typeId = leftVar->vartype; Oid typeModId = leftVar->vartypmod; Oid collationId = leftVar->varcollid; Oid accessMethodId = BTREE_AM_OID; OperatorCacheEntry *operatorCacheEntry = LookupOperatorByType(typeId, accessMethodId, strategyNumber); Oid operatorClassInputType = operatorCacheEntry->operatorClassInputType; Const *constantValue = makeNullConst(operatorClassInputType, typeModId, collationId); return MakeOpExpressionExtended(leftVar, (Expr *) constantValue, strategyNumber); } /* * LookupOperatorByType is a wrapper around GetOperatorByType(), * operatorClassInputType() and get_typtype() functions that uses a cache to avoid * multiple lookups of operators and its related fields within a single session by * their types, access methods and strategy numbers. * LookupOperatorByType function errors out if it cannot find corresponding * default operator class with the given parameters on the system catalogs. */ static OperatorCacheEntry * LookupOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber) { OperatorCacheEntry *matchingCacheEntry = NULL; ListCell *cacheEntryCell = NULL; /* search the cache */ foreach(cacheEntryCell, OperatorCache) { OperatorCacheEntry *cacheEntry = lfirst(cacheEntryCell); if ((cacheEntry->typeId == typeId) && (cacheEntry->accessMethodId == accessMethodId) && (cacheEntry->strategyNumber == strategyNumber)) { matchingCacheEntry = cacheEntry; break; } } /* if not found in the cache, call GetOperatorByType and put the result in cache */ if (matchingCacheEntry == NULL) { Oid operatorClassId = GetDefaultOpClass(typeId, accessMethodId); if (operatorClassId == InvalidOid) { /* if operatorId is invalid, error out */ ereport(ERROR, (errmsg("cannot find default operator class for type:%d," " access method: %d", typeId, accessMethodId))); } /* fill the other fields to the cache */ Oid operatorId = GetOperatorByType(typeId, accessMethodId, strategyNumber); Oid operatorClassInputType = get_opclass_input_type(operatorClassId); char typeType = get_typtype(operatorClassInputType); /* make sure we've initialized CacheMemoryContext */ if (CacheMemoryContext == NULL) { CreateCacheMemoryContext(); } MemoryContext oldContext = MemoryContextSwitchTo(CacheMemoryContext); matchingCacheEntry = palloc0(sizeof(OperatorCacheEntry)); matchingCacheEntry->typeId = typeId; matchingCacheEntry->accessMethodId = accessMethodId; matchingCacheEntry->strategyNumber = strategyNumber; matchingCacheEntry->operatorId = operatorId; matchingCacheEntry->operatorClassInputType = operatorClassInputType; matchingCacheEntry->typeType = typeType; OperatorCache = lappend(OperatorCache, matchingCacheEntry); MemoryContextSwitchTo(oldContext); } return matchingCacheEntry; } /* * GetOperatorByType returns the operator oid for the given type, access method, * and strategy number. */ static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber) { /* Get default operator class from pg_opclass */ Oid operatorClassId = GetDefaultOpClass(typeId, accessMethodId); Oid operatorFamily = get_opclass_family(operatorClassId); Oid operatorClassInputType = get_opclass_input_type(operatorClassId); /* Lookup for the operator with the desired input type in the family */ Oid operatorId = get_opfamily_member(operatorFamily, operatorClassInputType, operatorClassInputType, strategyNumber); return operatorId; } /* * BinaryOpExpression checks that a given expression is a binary operator. If * this is the case it returns true and sets leftOperand and rightOperand to * the left and right hand side of the operator. left/rightOperand will be * stripped of implicit coercions by strip_implicit_coercions. */ bool BinaryOpExpression(Expr *clause, Node **leftOperand, Node **rightOperand) { if (!is_opclause(clause) || list_length(((OpExpr *) clause)->args) != 2) { if (leftOperand != NULL) { *leftOperand = NULL; } if (rightOperand != NULL) { *rightOperand = NULL; } return false; } if (leftOperand != NULL) { *leftOperand = get_leftop(clause); Assert(*leftOperand != NULL); *leftOperand = strip_implicit_coercions(*leftOperand); } if (rightOperand != NULL) { *rightOperand = get_rightop(clause); Assert(*rightOperand != NULL); *rightOperand = strip_implicit_coercions(*rightOperand); } return true; } /* * MakeInt4Column creates a column of int4 type with invalid table id and max * attribute number. */ Var * MakeInt4Column() { Index tableId = 0; AttrNumber columnAttributeNumber = RESERVED_HASHED_COLUMN_ID; Oid columnType = INT4OID; int32 columnTypeMod = -1; Oid columnCollationOid = InvalidOid; Index columnLevelSup = 0; Var *int4Column = makeVar(tableId, columnAttributeNumber, columnType, columnTypeMod, columnCollationOid, columnLevelSup); return int4Column; } /* Updates the base constraint with the given min/max values. */ void UpdateConstraint(Node *baseConstraint, ShardInterval *shardInterval) { BoolExpr *andExpr = (BoolExpr *) baseConstraint; Node *lessThanExpr = (Node *) linitial(andExpr->args); Node *greaterThanExpr = (Node *) lsecond(andExpr->args); Node *minNode = get_rightop((Expr *) greaterThanExpr); /* right op */ Node *maxNode = get_rightop((Expr *) lessThanExpr); /* right op */ Assert(shardInterval != NULL); Assert(shardInterval->minValueExists); Assert(shardInterval->maxValueExists); Assert(minNode != NULL); Assert(maxNode != NULL); Assert(IsA(minNode, Const)); Assert(IsA(maxNode, Const)); Const *minConstant = (Const *) minNode; Const *maxConstant = (Const *) maxNode; minConstant->constvalue = datumCopy(shardInterval->minValue, shardInterval->valueByVal, shardInterval->valueTypeLen); maxConstant->constvalue = datumCopy(shardInterval->maxValue, shardInterval->valueByVal, shardInterval->valueTypeLen); minConstant->constisnull = false; maxConstant->constisnull = false; } /* * FragmentCombinationList first builds an ordered sequence of range tables that * join together. The function then iteratively adds fragments from each joined * range table, and forms fragment combinations (lists) that cover all tables. * While doing so, the function also performs join pruning to remove unnecessary * fragment pairs. Last, the function adds each fragment combination (list) to a * list, and returns this list. */ static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery, List *dependentJobList) { List *fragmentCombinationList = NIL; List *fragmentCombinationQueue = NIL; List *emptyList = NIL; /* find a sequence that joins the range tables in the list */ JoinSequenceNode *joinSequenceArray = JoinSequenceArray(rangeTableFragmentsList, jobQuery, dependentJobList); /* * We use breadth-first search with pruning to create fragment combinations. * For this, we first queue the root node (an empty combination), and then * start traversing our search space. */ fragmentCombinationQueue = lappend(fragmentCombinationQueue, emptyList); while (fragmentCombinationQueue != NIL) { ListCell *tableFragmentCell = NULL; int32 joiningTableSequenceIndex = -1; /* pop first element from the fragment queue */ List *fragmentCombination = linitial(fragmentCombinationQueue); fragmentCombinationQueue = list_delete_first(fragmentCombinationQueue); /* * If this combination covered all range tables in a join sequence, add * this combination to our result set. */ int32 joinSequenceIndex = list_length(fragmentCombination); int32 rangeTableCount = list_length(rangeTableFragmentsList); if (joinSequenceIndex == rangeTableCount) { fragmentCombinationList = lappend(fragmentCombinationList, fragmentCombination); continue; } /* find the next range table to add to our search space */ uint32 tableId = joinSequenceArray[joinSequenceIndex].rangeTableId; List *tableFragments = FindRangeTableFragmentsList(rangeTableFragmentsList, tableId); /* resolve sequence index for the previous range table we join against */ int32 joiningTableId = joinSequenceArray[joinSequenceIndex].joiningRangeTableId; if (joiningTableId != NON_PRUNABLE_JOIN) { for (int32 sequenceIndex = 0; sequenceIndex < rangeTableCount; sequenceIndex++) { JoinSequenceNode *joinSequenceNode = &joinSequenceArray[sequenceIndex]; if (joinSequenceNode->rangeTableId == joiningTableId) { joiningTableSequenceIndex = sequenceIndex; break; } } Assert(joiningTableSequenceIndex != -1); } /* * We walk over each range table fragment, and check if we can prune out * this fragment joining with the existing fragment combination. If we * can't prune away, we create a new fragment combination and add it to * our search space. */ foreach(tableFragmentCell, tableFragments) { RangeTableFragment *tableFragment = lfirst(tableFragmentCell); bool joinPrunable = false; if (joiningTableId != NON_PRUNABLE_JOIN) { RangeTableFragment *joiningTableFragment = list_nth(fragmentCombination, joiningTableSequenceIndex); joinPrunable = JoinPrunable(joiningTableFragment, tableFragment); } /* if join can't be pruned, extend fragment combination and search */ if (!joinPrunable) { List *newFragmentCombination = list_copy(fragmentCombination); newFragmentCombination = lappend(newFragmentCombination, tableFragment); fragmentCombinationQueue = lappend(fragmentCombinationQueue, newFragmentCombination); } } } return fragmentCombinationList; } /* * NodeIsRangeTblRefReferenceTable checks if the node is a RangeTblRef that * points to a reference table in the rangeTableList. */ static bool NodeIsRangeTblRefReferenceTable(Node *node, List *rangeTableList) { if (!IsA(node, RangeTblRef)) { return false; } RangeTblRef *tableRef = castNode(RangeTblRef, node); RangeTblEntry *rangeTableEntry = rt_fetch(tableRef->rtindex, rangeTableList); CitusRTEKind rangeTableType = GetRangeTblKind(rangeTableEntry); if (rangeTableType != CITUS_RTE_RELATION) { return false; } return IsCitusTableType(rangeTableEntry->relid, REFERENCE_TABLE); } /* * FetchEqualityAttrNumsForRTE fetches the attribute numbers from quals * which have an equality operator */ List * FetchEqualityAttrNumsForRTE(Node *node) { if (node == NULL) { return NIL; } if (IsA(node, List)) { return FetchEqualityAttrNumsForList((List *) node); } else if (IsA(node, OpExpr)) { return FetchEqualityAttrNumsForRTEOpExpr((OpExpr *) node); } else if (IsA(node, BoolExpr)) { return FetchEqualityAttrNumsForRTEBoolExpr((BoolExpr *) node); } return NIL; } /* * FetchEqualityAttrNumsForList fetches the attribute numbers of expression * of the form "= constant" from the given node list. */ static List * FetchEqualityAttrNumsForList(List *nodeList) { List *attributeNums = NIL; Node *node = NULL; bool hasAtLeastOneEquality = false; foreach_declared_ptr(node, nodeList) { List *fetchedEqualityAttrNums = FetchEqualityAttrNumsForRTE(node); hasAtLeastOneEquality |= list_length(fetchedEqualityAttrNums) > 0; attributeNums = list_concat(attributeNums, fetchedEqualityAttrNums); } /* * the given list is in the form of AND'ed expressions * hence if we have one equality then it is enough. * E.g: dist.a = 5 AND dist.a > 10 */ if (hasAtLeastOneEquality) { return attributeNums; } return NIL; } /* * FetchEqualityAttrNumsForRTEOpExpr fetches the attribute numbers of expression * of the form "= constant" from the given opExpr. */ static List * FetchEqualityAttrNumsForRTEOpExpr(OpExpr *opExpr) { if (!OperatorImplementsEquality(opExpr->opno)) { return NIL; } List *attributeNums = NIL; Var *var = NULL; if (VarConstOpExprClause(opExpr, &var, NULL)) { attributeNums = lappend_int(attributeNums, var->varattno); } return attributeNums; } /* * FetchEqualityAttrNumsForRTEBoolExpr fetches the attribute numbers of expression * of the form "= constant" from the given boolExpr. */ static List * FetchEqualityAttrNumsForRTEBoolExpr(BoolExpr *boolExpr) { if (boolExpr->boolop != AND_EXPR && boolExpr->boolop != OR_EXPR) { return NIL; } List *attributeNums = NIL; bool hasEquality = true; Node *arg = NULL; foreach_declared_ptr(arg, boolExpr->args) { List *attributeNumsInSubExpression = FetchEqualityAttrNumsForRTE(arg); if (boolExpr->boolop == AND_EXPR) { hasEquality |= list_length(attributeNumsInSubExpression) > 0; } else if (boolExpr->boolop == OR_EXPR) { hasEquality &= list_length(attributeNumsInSubExpression) > 0; } attributeNums = list_concat(attributeNums, attributeNumsInSubExpression); } if (hasEquality) { return attributeNums; } return NIL; } /* * JoinSequenceArray walks over the join nodes in the job query and constructs a join * sequence containing an entry for each joined table. The function then returns an * array of join sequence nodes, in which each node contains the id of a table in the * range table list and the id of a preceding table with which it is joined, if any. */ static JoinSequenceNode * JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *dependentJobList) { List *rangeTableList = jobQuery->rtable; uint32 rangeTableCount = (uint32) list_length(rangeTableList); uint32 sequenceNodeSize = sizeof(JoinSequenceNode); uint32 joinedTableCount = 0; ListCell *joinExprCell = NULL; uint32 firstRangeTableId = 1; JoinSequenceNode *joinSequenceArray = palloc0(rangeTableCount * sequenceNodeSize); List *joinExprList = JoinExprList(jobQuery->jointree); /* pick first range table as starting table for the join sequence */ if (list_length(joinExprList) > 0) { JoinExpr *firstExpr = (JoinExpr *) linitial(joinExprList); RangeTblRef *leftTableRef = (RangeTblRef *) firstExpr->larg; firstRangeTableId = leftTableRef->rtindex; } else { /* when there are no joins, the join sequence contains a node for the table */ firstRangeTableId = 1; } joinSequenceArray[joinedTableCount].rangeTableId = firstRangeTableId; joinSequenceArray[joinedTableCount].joiningRangeTableId = NON_PRUNABLE_JOIN; joinedTableCount++; foreach(joinExprCell, joinExprList) { JoinExpr *joinExpr = (JoinExpr *) lfirst(joinExprCell); RangeTblRef *rightTableRef = castNode(RangeTblRef, joinExpr->rarg); uint32 nextRangeTableId = rightTableRef->rtindex; Index existingRangeTableId = 0; bool applyJoinPruning = false; List *nextJoinClauseList = make_ands_implicit((Expr *) joinExpr->quals); bool leftIsReferenceTable = NodeIsRangeTblRefReferenceTable(joinExpr->larg, rangeTableList); bool rightIsReferenceTable = NodeIsRangeTblRefReferenceTable(joinExpr->rarg, rangeTableList); bool isReferenceJoin = IsSupportedReferenceJoin(joinExpr->jointype, leftIsReferenceTable, rightIsReferenceTable); /* * If next join clause list is empty, the user tried a cartesian product * between tables. We don't support this functionality for non * reference joins, and error out. */ if (nextJoinClauseList == NIL && !isReferenceJoin) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform distributed planning on this query"), errdetail("Cartesian products are currently unsupported"))); } /* * We now determine if we can apply join pruning between existing range * tables and this new one. */ Node *nextJoinClause = NULL; foreach_declared_ptr(nextJoinClause, nextJoinClauseList) { if (!NodeIsEqualsOpExpr(nextJoinClause)) { continue; } OpExpr *nextJoinClauseOpExpr = castNode(OpExpr, nextJoinClause); if (!IsJoinClause((Node *) nextJoinClauseOpExpr)) { continue; } Var *leftColumn = LeftColumnOrNULL(nextJoinClauseOpExpr); Var *rightColumn = RightColumnOrNULL(nextJoinClauseOpExpr); if (leftColumn == NULL || rightColumn == NULL) { continue; } Index leftRangeTableId = leftColumn->varno; Index rightRangeTableId = rightColumn->varno; /* * We have a table from the existing join list joining with the next * table. First resolve the existing table's range table id. */ if (leftRangeTableId == nextRangeTableId) { existingRangeTableId = rightRangeTableId; } else { existingRangeTableId = leftRangeTableId; } /* * Then, we check if we can apply join pruning between the existing * range table and this new one. For this, columns need to have the * same type and be the partition column for their respective tables. */ if (leftColumn->vartype != rightColumn->vartype) { continue; } bool leftPartitioned = PartitionedOnColumn(leftColumn, rangeTableList, dependentJobList); bool rightPartitioned = PartitionedOnColumn(rightColumn, rangeTableList, dependentJobList); if (leftPartitioned && rightPartitioned) { /* make sure this join clause references only simple columns */ CheckJoinBetweenColumns(nextJoinClauseOpExpr); applyJoinPruning = true; break; } } /* set next joining range table's info in the join sequence */ JoinSequenceNode *nextJoinSequenceNode = &joinSequenceArray[joinedTableCount]; if (applyJoinPruning) { nextJoinSequenceNode->rangeTableId = nextRangeTableId; nextJoinSequenceNode->joiningRangeTableId = (int32) existingRangeTableId; } else { nextJoinSequenceNode->rangeTableId = nextRangeTableId; nextJoinSequenceNode->joiningRangeTableId = NON_PRUNABLE_JOIN; } joinedTableCount++; } return joinSequenceArray; } /* * PartitionedOnColumn finds the given column's range table entry, and checks if * that range table is partitioned on the given column. Note that since reference * tables do not have partition columns, the function returns false when the distributed * relation is a reference table. */ static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependentJobList) { bool partitionedOnColumn = false; Index rangeTableId = column->varno; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableId, rangeTableList); CitusRTEKind rangeTableType = GetRangeTblKind(rangeTableEntry); if (rangeTableType == CITUS_RTE_RELATION) { Oid relationId = rangeTableEntry->relid; Var *partitionColumn = PartitionColumn(relationId, rangeTableId); /* non-distributed tables do not have partition columns */ if (IsCitusTable(relationId) && !HasDistributionKey(relationId)) { return false; } if (partitionColumn->varattno == column->varattno) { partitionedOnColumn = true; } } else if (rangeTableType == CITUS_RTE_REMOTE_QUERY) { Job *job = JobForRangeTable(dependentJobList, rangeTableEntry); MapMergeJob *mapMergeJob = (MapMergeJob *) job; /* * The column's current attribute number is it's location in the target * list for the table represented by the remote query. We retrieve this * value from the target list to compare against the partition column * as stored in the job. */ List *targetEntryList = job->jobQuery->targetList; int32 columnIndex = column->varattno - 1; Assert(columnIndex >= 0); Assert(columnIndex < list_length(targetEntryList)); TargetEntry *targetEntry = (TargetEntry *) list_nth(targetEntryList, columnIndex); Var *remoteRelationColumn = (Var *) targetEntry->expr; Assert(IsA(remoteRelationColumn, Var)); /* retrieve the partition column for the job */ Var *partitionColumn = mapMergeJob->partitionColumn; if (partitionColumn->varattno == remoteRelationColumn->varattno) { partitionedOnColumn = true; } } return partitionedOnColumn; } /* Checks that the join clause references only simple columns. */ static void CheckJoinBetweenColumns(OpExpr *joinClause) { List *argumentList = joinClause->args; Node *leftArgument = (Node *) linitial(argumentList); Node *rightArgument = (Node *) lsecond(argumentList); Node *strippedLeftArgument = strip_implicit_coercions(leftArgument); Node *strippedRightArgument = strip_implicit_coercions(rightArgument); NodeTag leftArgumentType = nodeTag(strippedLeftArgument); NodeTag rightArgumentType = nodeTag(strippedRightArgument); if (leftArgumentType != T_Var || rightArgumentType != T_Var) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform local joins that involve expressions"), errdetail("local joins can be performed between columns only"))); } } /* * FindRangeTableFragmentsList walks over the given list of range table fragments * and, returns the one with the given table id. */ static List * FindRangeTableFragmentsList(List *rangeTableFragmentsList, int tableId) { List *foundTableFragments = NIL; ListCell *rangeTableFragmentsCell = NULL; foreach(rangeTableFragmentsCell, rangeTableFragmentsList) { List *tableFragments = (List *) lfirst(rangeTableFragmentsCell); if (tableFragments != NIL) { RangeTableFragment *tableFragment = (RangeTableFragment *) linitial(tableFragments); if (tableFragment->rangeTableId == tableId) { foundTableFragments = tableFragments; break; } } } return foundTableFragments; } /* * JoinPrunable checks if a join between the given left and right fragments can * be pruned away, without performing the actual join. To do this, the function * checks if we have a hash repartition join. If we do, the function determines * pruning based on partitionIds. Else if we have a merge repartition join, the * function checks if the two fragments have disjoint intervals. */ static bool JoinPrunable(RangeTableFragment *leftFragment, RangeTableFragment *rightFragment) { /* * If both range tables are remote queries, we then have a hash repartition * join. In that case, we can just prune away this join if left and right * hand side fragments have the same partitionId. */ if (leftFragment->fragmentType == CITUS_RTE_REMOTE_QUERY && rightFragment->fragmentType == CITUS_RTE_REMOTE_QUERY) { Task *leftMergeTask = (Task *) leftFragment->fragmentReference; Task *rightMergeTask = (Task *) rightFragment->fragmentReference; if (leftMergeTask->partitionId != rightMergeTask->partitionId) { ereport(DEBUG2, (errmsg("join prunable for task partitionId %u and %u", leftMergeTask->partitionId, rightMergeTask->partitionId))); return true; } else { return false; } } /* * We have a single (re)partition join. We now get shard intervals for both * fragments, and then check if these intervals overlap. */ ShardInterval *leftFragmentInterval = FragmentInterval(leftFragment); ShardInterval *rightFragmentInterval = FragmentInterval(rightFragment); bool overlap = ShardIntervalsOverlap(leftFragmentInterval, rightFragmentInterval); if (!overlap) { if (IsLoggableLevel(DEBUG2)) { StringInfo leftString = FragmentIntervalString(leftFragmentInterval); StringInfo rightString = FragmentIntervalString(rightFragmentInterval); ereport(DEBUG2, (errmsg("join prunable for intervals %s and %s", leftString->data, rightString->data))); } return true; } return false; } /* * FragmentInterval takes the given fragment, and determines the range of data * covered by this fragment. The function then returns this range (interval). */ static ShardInterval * FragmentInterval(RangeTableFragment *fragment) { ShardInterval *fragmentInterval = NULL; if (fragment->fragmentType == CITUS_RTE_RELATION) { Assert(CitusIsA(fragment->fragmentReference, ShardInterval)); fragmentInterval = (ShardInterval *) fragment->fragmentReference; } else if (fragment->fragmentType == CITUS_RTE_REMOTE_QUERY) { Assert(CitusIsA(fragment->fragmentReference, Task)); Task *mergeTask = (Task *) fragment->fragmentReference; fragmentInterval = mergeTask->shardInterval; } return fragmentInterval; } /* Checks if the given shard intervals have overlapping ranges. */ bool ShardIntervalsOverlap(ShardInterval *firstInterval, ShardInterval *secondInterval) { CitusTableCacheEntry *intervalRelation = GetCitusTableCacheEntry(firstInterval->relationId); Assert(IsCitusTableTypeCacheEntry(intervalRelation, DISTRIBUTED_TABLE)); if (!(firstInterval->minValueExists && firstInterval->maxValueExists && secondInterval->minValueExists && secondInterval->maxValueExists)) { return true; } Datum firstMin = firstInterval->minValue; Datum firstMax = firstInterval->maxValue; Datum secondMin = secondInterval->minValue; Datum secondMax = secondInterval->maxValue; FmgrInfo *comparisonFunction = intervalRelation->shardIntervalCompareFunction; Oid collation = intervalRelation->partitionColumn->varcollid; return ShardIntervalsOverlapWithParams(firstMin, firstMax, secondMin, secondMax, comparisonFunction, collation); } /* * ShardIntervalsOverlapWithParams is a helper function which compares the input * shard min/max values, and returns true if the shards overlap. * The caller is responsible to ensure the input shard min/max values are not NULL. */ bool ShardIntervalsOverlapWithParams(Datum firstMin, Datum firstMax, Datum secondMin, Datum secondMax, FmgrInfo *comparisonFunction, Oid collation) { /* * We need to have min/max values for both intervals first. Then, we assume * two intervals i1 = [min1, max1] and i2 = [min2, max2] do not overlap if * (max1 < min2) or (max2 < min1). For details, please see the explanation * on overlapping intervals at http://www.rgrjr.com/emacs/overlap.html. */ Datum firstDatum = FunctionCall2Coll(comparisonFunction, collation, firstMax, secondMin); Datum secondDatum = FunctionCall2Coll(comparisonFunction, collation, secondMax, firstMin); int firstComparison = DatumGetInt32(firstDatum); int secondComparison = DatumGetInt32(secondDatum); if (firstComparison < 0 || secondComparison < 0) { return false; } return true; } /* * FragmentIntervalString takes the given fragment interval, and converts this * interval into its string representation for use in debug messages. */ static StringInfo FragmentIntervalString(ShardInterval *fragmentInterval) { Oid typeId = fragmentInterval->valueTypeId; Oid outputFunctionId = InvalidOid; bool typeVariableLength = false; Assert(fragmentInterval->minValueExists); Assert(fragmentInterval->maxValueExists); FmgrInfo *outputFunction = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); getTypeOutputInfo(typeId, &outputFunctionId, &typeVariableLength); fmgr_info(outputFunctionId, outputFunction); char *minValueString = OutputFunctionCall(outputFunction, fragmentInterval->minValue); char *maxValueString = OutputFunctionCall(outputFunction, fragmentInterval->maxValue); StringInfo fragmentIntervalString = makeStringInfo(); appendStringInfo(fragmentIntervalString, "[%s,%s]", minValueString, maxValueString); return fragmentIntervalString; } /* * DataFetchTaskList builds a merge fetch task for every remote query result * in the given fragment list, appends these merge fetch tasks into a list, * and returns this list. */ static List * DataFetchTaskList(uint64 jobId, uint32 taskIdIndex, List *fragmentList) { List *dataFetchTaskList = NIL; ListCell *fragmentCell = NULL; foreach(fragmentCell, fragmentList) { RangeTableFragment *fragment = (RangeTableFragment *) lfirst(fragmentCell); if (fragment->fragmentType == CITUS_RTE_REMOTE_QUERY) { Task *mergeTask = (Task *) fragment->fragmentReference; char *undefinedQueryString = NULL; /* create merge fetch task and have it depend on the merge task */ Task *mergeFetchTask = CreateBasicTask(jobId, taskIdIndex, MERGE_FETCH_TASK, undefinedQueryString); mergeFetchTask->dependentTaskList = list_make1(mergeTask); dataFetchTaskList = lappend(dataFetchTaskList, mergeFetchTask); taskIdIndex++; } } return dataFetchTaskList; } /* * CreateBasicTask creates a task, initializes fields that are common to each task, * and returns the created task. */ Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType, char *queryString) { Task *task = CitusMakeNode(Task); task->jobId = jobId; task->taskId = taskId; task->taskType = taskType; task->replicationModel = REPLICATION_MODEL_INVALID; SetTaskQueryString(task, queryString); return task; } /* * BuildRelationShardList builds a list of RelationShard pairs for a task. * This represents the mapping of range table entries to shard IDs for a * task for the purposes of locking, deparsing, and connection management. */ static List * BuildRelationShardList(List *rangeTableList, List *fragmentList) { List *relationShardList = NIL; ListCell *fragmentCell = NULL; foreach(fragmentCell, fragmentList) { RangeTableFragment *fragment = (RangeTableFragment *) lfirst(fragmentCell); Index rangeTableId = fragment->rangeTableId; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableId, rangeTableList); CitusRTEKind fragmentType = fragment->fragmentType; if (fragmentType == CITUS_RTE_RELATION) { ShardInterval *shardInterval = (ShardInterval *) fragment->fragmentReference; RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = rangeTableEntry->relid; relationShard->shardId = shardInterval->shardId; relationShardList = lappend(relationShardList, relationShard); } } return relationShardList; } /* * UpdateRangeTableAlias walks over each fragment in the given fragment list, * and creates an alias that represents the fragment name to be used in the * query. The function then updates the corresponding range table entry with * this alias. */ static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList) { ListCell *fragmentCell = NULL; foreach(fragmentCell, fragmentList) { RangeTableFragment *fragment = (RangeTableFragment *) lfirst(fragmentCell); Index rangeTableId = fragment->rangeTableId; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableId, rangeTableList); Alias *fragmentAlias = FragmentAlias(rangeTableEntry, fragment); rangeTableEntry->alias = fragmentAlias; } } /* * FragmentAlias creates an alias structure that captures the table fragment's * name on the worker node. Each fragment represents either a regular shard, or * a merge task. */ static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment) { char *aliasName = NULL; char *schemaName = NULL; char *fragmentName = NULL; CitusRTEKind fragmentType = fragment->fragmentType; if (fragmentType == CITUS_RTE_RELATION) { ShardInterval *shardInterval = (ShardInterval *) fragment->fragmentReference; uint64 shardId = shardInterval->shardId; Oid relationId = rangeTableEntry->relid; char *relationName = get_rel_name(relationId); Oid schemaId = get_rel_namespace(relationId); schemaName = get_namespace_name(schemaId); aliasName = relationName; /* * Set shard name in alias to _. */ fragmentName = pstrdup(relationName); AppendShardIdToName(&fragmentName, shardId); } else if (fragmentType == CITUS_RTE_REMOTE_QUERY) { Task *mergeTask = (Task *) fragment->fragmentReference; List *mapOutputFetchTaskList = mergeTask->dependentTaskList; List *resultNameList = FetchTaskResultNameList(mapOutputFetchTaskList); List *mapJobTargetList = mergeTask->mapJobTargetList; /* determine whether all types have binary input/output functions */ bool useBinaryFormat = CanUseBinaryCopyFormatForTargetList(mapJobTargetList); /* generate the query on the intermediate result */ Query *fragmentSetQuery = BuildReadIntermediateResultsArrayQuery(mapJobTargetList, NIL, resultNameList, useBinaryFormat); /* we only really care about the function RTE */ RangeTblEntry *readIntermediateResultsRTE = linitial(fragmentSetQuery->rtable); /* crudely override the fragment RTE */ *rangeTableEntry = *readIntermediateResultsRTE; return rangeTableEntry->alias; } /* * We need to set the aliasname to relation name, as pg_get_query_def() uses * the relation name to disambiguate column names from different tables. */ Alias *alias = rangeTableEntry->alias; if (alias == NULL) { alias = makeNode(Alias); alias->aliasname = aliasName; } ModifyRangeTblExtraData(rangeTableEntry, CITUS_RTE_SHARD, schemaName, fragmentName, NIL); return alias; } /* * FetchTaskResultNameList builds a list of result names that reflect * the output of map-fetch tasks. */ static List * FetchTaskResultNameList(List *mapOutputFetchTaskList) { List *resultNameList = NIL; Task *mapOutputFetchTask = NULL; foreach_declared_ptr(mapOutputFetchTask, mapOutputFetchTaskList) { Task *mapTask = linitial(mapOutputFetchTask->dependentTaskList); int partitionId = mapOutputFetchTask->partitionId; char *resultName = PartitionResultName(mapTask->jobId, mapTask->taskId, partitionId); resultNameList = lappend(resultNameList, resultName); } return resultNameList; } /* * AnchorShardId walks over each fragment in the given fragment list, finds the * fragment that corresponds to the given anchor range tableId, and returns this * fragment's shard identifier. Note that the given tableId must correspond to a * base relation. */ static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId) { uint64 anchorShardId = INVALID_SHARD_ID; ListCell *fragmentCell = NULL; foreach(fragmentCell, fragmentList) { RangeTableFragment *fragment = (RangeTableFragment *) lfirst(fragmentCell); if (fragment->rangeTableId == anchorRangeTableId) { Assert(fragment->fragmentType == CITUS_RTE_RELATION); Assert(CitusIsA(fragment->fragmentReference, ShardInterval)); ShardInterval *shardInterval = (ShardInterval *) fragment->fragmentReference; anchorShardId = shardInterval->shardId; break; } } Assert(anchorShardId != INVALID_SHARD_ID); return anchorShardId; } /* * PruneSqlTaskDependencies iterates over each sql task from the given sql task * list, and prunes away merge-fetch tasks, as the task assignment algorithm * ensures co-location of these tasks. */ static List * PruneSqlTaskDependencies(List *sqlTaskList) { ListCell *sqlTaskCell = NULL; foreach(sqlTaskCell, sqlTaskList) { Task *sqlTask = (Task *) lfirst(sqlTaskCell); List *dependentTaskList = sqlTask->dependentTaskList; List *prunedDependendTaskList = NIL; ListCell *dependentTaskCell = NULL; foreach(dependentTaskCell, dependentTaskList) { Task *dataFetchTask = (Task *) lfirst(dependentTaskCell); /* * If we have a merge fetch task, our task assignment algorithm makes * sure that the sql task is colocated with the anchor shard / merge * task. We can therefore prune out this data fetch task. */ if (dataFetchTask->taskType == MERGE_FETCH_TASK) { List *mergeFetchDependencyList = dataFetchTask->dependentTaskList; Assert(list_length(mergeFetchDependencyList) == 1); Task *mergeTaskReference = (Task *) linitial(mergeFetchDependencyList); prunedDependendTaskList = lappend(prunedDependendTaskList, mergeTaskReference); ereport(DEBUG2, (errmsg("pruning merge fetch taskId %d", dataFetchTask->taskId), errdetail("Creating dependency on merge taskId %d", mergeTaskReference->taskId))); } } sqlTask->dependentTaskList = prunedDependendTaskList; } return sqlTaskList; } /* * MapTaskList creates a list of map tasks for the given MapMerge job. For this, * the function walks over each filter task (sql task) in the given filter task * list, and wraps this task with a map function call. The map function call * repartitions the filter task's output according to MapMerge job's parameters. */ static List * MapTaskList(MapMergeJob *mapMergeJob, List *filterTaskList) { List *mapTaskList = NIL; Query *filterQuery = mapMergeJob->job.jobQuery; ListCell *filterTaskCell = NULL; Var *partitionColumn = mapMergeJob->partitionColumn; uint32 partitionColumnResNo = 0; List *groupClauseList = filterQuery->groupClause; if (groupClauseList != NIL) { List *targetEntryList = filterQuery->targetList; List *groupTargetEntryList = GroupTargetEntryList(groupClauseList, targetEntryList); TargetEntry *groupByTargetEntry = (TargetEntry *) linitial(groupTargetEntryList); partitionColumnResNo = groupByTargetEntry->resno; } else { partitionColumnResNo = PartitionColumnIndex(partitionColumn, filterQuery->targetList); } /* determine whether all types have binary input/output functions */ bool useBinaryFormat = CanUseBinaryCopyFormatForTargetList(filterQuery->targetList); foreach(filterTaskCell, filterTaskList) { Task *filterTask = (Task *) lfirst(filterTaskCell); StringInfo mapQueryString = CreateMapQueryString(mapMergeJob, filterTask, partitionColumnResNo, useBinaryFormat); /* convert filter query task into map task */ Task *mapTask = filterTask; SetTaskQueryString(mapTask, mapQueryString->data); mapTask->taskType = MAP_TASK; /* * We do not support fail-over in case of map tasks, since we would also * have to fail over the corresponding merge tasks. We therefore truncate * the list down to the first element. */ mapTask->taskPlacementList = list_truncate(mapTask->taskPlacementList, 1); mapTaskList = lappend(mapTaskList, mapTask); } return mapTaskList; } /* * PartitionColumnIndex finds the index of the given target var. */ static int PartitionColumnIndex(Var *targetVar, List *targetList) { TargetEntry *targetEntry = NULL; int resNo = 1; foreach_declared_ptr(targetEntry, targetList) { if (IsA(targetEntry->expr, Var)) { Var *candidateVar = (Var *) targetEntry->expr; if (candidateVar->varattno == targetVar->varattno && candidateVar->varno == targetVar->varno) { return resNo; } resNo++; } } ereport(ERROR, (errmsg("unexpected state: %d varno %d varattno couldn't be found", targetVar->varno, targetVar->varattno))); return resNo; } /* * CreateMapQueryString creates and returns the map query string for the given filterTask. */ static StringInfo CreateMapQueryString(MapMergeJob *mapMergeJob, Task *filterTask, uint32 partitionColumnIndex, bool useBinaryFormat) { uint64 jobId = filterTask->jobId; uint32 taskId = filterTask->taskId; char *resultNamePrefix = PartitionResultNamePrefix(jobId, taskId); /* wrap repartition query string around filter query string */ StringInfo mapQueryString = makeStringInfo(); char *filterQueryString = TaskQueryString(filterTask); PartitionType partitionType = mapMergeJob->partitionType; Var *partitionColumn = mapMergeJob->partitionColumn; Oid partitionColumnType = partitionColumn->vartype; ShardInterval **intervalArray = mapMergeJob->sortedShardIntervalArray; uint32 intervalCount = mapMergeJob->partitionCount; if (partitionType == DUAL_HASH_PARTITION_TYPE) { partitionColumnType = INT4OID; intervalArray = GenerateSyntheticShardIntervalArray(intervalCount); } else if (partitionType == SINGLE_HASH_PARTITION_TYPE) { partitionColumnType = INT4OID; } else if (partitionType == RANGE_PARTITION_TYPE) { /* add a partition for NULL values at index 0 */ intervalArray = RangeIntervalArrayWithNullBucket(intervalArray, intervalCount); intervalCount++; } Oid intervalTypeOutFunc = InvalidOid; bool intervalTypeVarlena = false; ArrayType *minValueArray = NULL; ArrayType *maxValueArray = NULL; getTypeOutputInfo(partitionColumnType, &intervalTypeOutFunc, &intervalTypeVarlena); ShardMinMaxValueArrays(intervalArray, intervalCount, intervalTypeOutFunc, &minValueArray, &maxValueArray); StringInfo minValuesString = ArrayObjectToString(minValueArray, TEXTOID, InvalidOid); StringInfo maxValuesString = ArrayObjectToString(maxValueArray, TEXTOID, InvalidOid); char *partitionMethodString = partitionType == RANGE_PARTITION_TYPE ? "range" : "hash"; /* * Non-partition columns can easily contain NULL values, so we allow NULL * values in the column by which we re-partition. They will end up in the * first partition. */ bool allowNullPartitionColumnValue = true; /* * We currently generate empty results for each partition and fetch all of them. */ bool generateEmptyResults = true; appendStringInfo(mapQueryString, "SELECT partition_index" ", %s || '_' || partition_index::text " ", rows_written " "FROM pg_catalog.worker_partition_query_result" "(%s,%s,%d,%s,%s,%s,%s,%s,%s) WHERE rows_written > 0", quote_literal_cstr(resultNamePrefix), quote_literal_cstr(resultNamePrefix), quote_literal_cstr(filterQueryString), partitionColumnIndex - 1, quote_literal_cstr(partitionMethodString), minValuesString->data, maxValuesString->data, useBinaryFormat ? "true" : "false", allowNullPartitionColumnValue ? "true" : "false", generateEmptyResults ? "true" : "false"); return mapQueryString; } /* * PartitionResultNamePrefix returns the prefix we use for worker_partition_query_result * results. Each result will have a _ suffix. */ static char * PartitionResultNamePrefix(uint64 jobId, int32 taskId) { StringInfo resultNamePrefix = makeStringInfo(); appendStringInfo(resultNamePrefix, "repartition_" UINT64_FORMAT "_%u", jobId, taskId); return resultNamePrefix->data; } /* * PartitionResultName returns the name of a worker_partition_query_result result for * a specific partition. */ static char * PartitionResultName(uint64 jobId, uint32 taskId, uint32 partitionId) { StringInfo resultName = makeStringInfo(); char *resultNamePrefix = PartitionResultNamePrefix(jobId, taskId); appendStringInfo(resultName, "%s_%d", resultNamePrefix, partitionId); return resultName->data; } /* * GenerateSyntheticShardIntervalArray returns a shard interval pointer array * which has a uniform hash distribution for the given input partitionCount. * * The function only fills the min/max values of shard the intervals. Thus, should * not be used for general purpose operations. */ ShardInterval ** GenerateSyntheticShardIntervalArray(int partitionCount) { ShardInterval **shardIntervalArray = palloc0(partitionCount * sizeof(ShardInterval *)); uint64 hashTokenIncrement = HASH_TOKEN_COUNT / partitionCount; for (int shardIndex = 0; shardIndex < partitionCount; ++shardIndex) { ShardInterval *shardInterval = CitusMakeNode(ShardInterval); /* calculate the split of the hash space */ int32 shardMinHashToken = PG_INT32_MIN + (shardIndex * hashTokenIncrement); int32 shardMaxHashToken = shardMinHashToken + (hashTokenIncrement - 1); /* extend the last range to cover the full range of integers */ if (shardIndex == (partitionCount - 1)) { shardMaxHashToken = PG_INT32_MAX; } shardInterval->relationId = InvalidOid; shardInterval->minValueExists = true; shardInterval->minValue = Int32GetDatum(shardMinHashToken); shardInterval->maxValueExists = true; shardInterval->maxValue = Int32GetDatum(shardMaxHashToken); shardInterval->shardId = INVALID_SHARD_ID; shardInterval->valueTypeId = INT4OID; shardIntervalArray[shardIndex] = shardInterval; } return shardIntervalArray; } /* * RangeIntervalArrayWithNullBucket prepends an additional bucket for NULL values * to intervalArray and returns the result. * * When we support NULL values in (range-partitioned) shards, we will need to revise * this logic, since there may already be an interval for NULL values. */ static ShardInterval ** RangeIntervalArrayWithNullBucket(ShardInterval **intervalArray, int intervalCount) { int fullIntervalCount = intervalCount + 1; ShardInterval **fullIntervalArray = palloc0(fullIntervalCount * sizeof(ShardInterval *)); fullIntervalArray[0] = CitusMakeNode(ShardInterval); fullIntervalArray[0]->minValueExists = true; fullIntervalArray[0]->maxValueExists = true; fullIntervalArray[0]->valueTypeId = intervalArray[0]->valueTypeId; for (int intervalIndex = 1; intervalIndex < fullIntervalCount; intervalIndex++) { fullIntervalArray[intervalIndex] = intervalArray[intervalIndex - 1]; } return fullIntervalArray; } /* * Determine RowModifyLevel required for given query */ RowModifyLevel RowModifyLevelForQuery(Query *query) { CmdType commandType = query->commandType; if (commandType == CMD_SELECT) { if (query->hasModifyingCTE) { /* skip checking for INSERT as those CTEs are recursively planned */ CommonTableExpr *cte = NULL; foreach_declared_ptr(cte, query->cteList) { Query *cteQuery = (Query *) cte->ctequery; if (cteQuery->commandType == CMD_UPDATE || cteQuery->commandType == CMD_DELETE) { return ROW_MODIFY_NONCOMMUTATIVE; } } } return ROW_MODIFY_READONLY; } if (commandType == CMD_INSERT) { if (query->onConflict == NULL) { return ROW_MODIFY_COMMUTATIVE; } else { return ROW_MODIFY_NONCOMMUTATIVE; } } if (commandType == CMD_UPDATE || commandType == CMD_DELETE || commandType == CMD_MERGE) { return ROW_MODIFY_NONCOMMUTATIVE; } return ROW_MODIFY_NONE; } /* * ArrayObjectToString converts an SQL object to its string representation. */ StringInfo ArrayObjectToString(ArrayType *arrayObject, Oid columnType, int32 columnTypeMod) { Datum arrayDatum = PointerGetDatum(arrayObject); Oid outputFunctionId = InvalidOid; bool typeVariableLength = false; Oid arrayOutType = get_array_type(columnType); if (arrayOutType == InvalidOid) { char *columnTypeName = format_type_be(columnType); ereport(ERROR, (errmsg("cannot range repartition table on column type %s", columnTypeName))); } FmgrInfo *arrayOutFunction = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); getTypeOutputInfo(arrayOutType, &outputFunctionId, &typeVariableLength); fmgr_info(outputFunctionId, arrayOutFunction); char *arrayOutputText = OutputFunctionCall(arrayOutFunction, arrayDatum); char *arrayOutputEscapedText = quote_literal_cstr(arrayOutputText); /* add an explicit cast to array's string representation */ char *arrayOutTypeName = format_type_be(arrayOutType); StringInfo arrayString = makeStringInfo(); appendStringInfo(arrayString, "%s::%s", arrayOutputEscapedText, arrayOutTypeName); return arrayString; } /* * MergeTaskList creates a list of merge tasks for the given MapMerge job. While * doing this, the function also establishes dependencies between each merge * task and its downstream map task dependencies by creating "map fetch" tasks. */ static List * MergeTaskList(MapMergeJob *mapMergeJob, List *mapTaskList, uint32 taskIdIndex) { List *mergeTaskList = NIL; uint64 jobId = mapMergeJob->job.jobId; uint32 partitionCount = mapMergeJob->partitionCount; /* build column name and column type arrays (table schema) */ Query *filterQuery = mapMergeJob->job.jobQuery; List *targetEntryList = filterQuery->targetList; /* if all map tasks were pruned away, return NIL for merge tasks */ if (mapTaskList == NIL) { return NIL; } /* * XXX: We currently ignore the 0th partition bucket that range partitioning * generates. This bucket holds all values less than the minimum value or * NULLs, both of which we can currently ignore. However, when we support * range re-partitioned OUTER joins, we will need these rows for the * relation whose rows are retained in the OUTER join. */ uint32 initialPartitionId = 0; if (mapMergeJob->partitionType == RANGE_PARTITION_TYPE) { initialPartitionId = 1; partitionCount = partitionCount + 1; } else if (mapMergeJob->partitionType == SINGLE_HASH_PARTITION_TYPE) { initialPartitionId = 0; } /* build merge tasks and their associated "map output fetch" tasks */ for (uint32 partitionId = initialPartitionId; partitionId < partitionCount; partitionId++) { List *mapOutputFetchTaskList = NIL; ListCell *mapTaskCell = NULL; uint32 mergeTaskId = taskIdIndex; /* create logical merge task (not executed, but useful for bookkeeping) */ Task *mergeTask = CreateBasicTask(jobId, mergeTaskId, MERGE_TASK, ""); mergeTask->partitionId = partitionId; taskIdIndex++; /* create tasks to fetch map outputs to this merge task */ foreach(mapTaskCell, mapTaskList) { Task *mapTask = (Task *) lfirst(mapTaskCell); /* find the node name/port for map task's execution */ List *mapTaskPlacementList = mapTask->taskPlacementList; ShardPlacement *mapTaskPlacement = linitial(mapTaskPlacementList); char *partitionResultName = PartitionResultName(jobId, mapTask->taskId, partitionId); /* we currently only fetch a single fragment at a time */ DistributedResultFragment singleFragmentTransfer; singleFragmentTransfer.resultId = partitionResultName; singleFragmentTransfer.nodeId = mapTaskPlacement->nodeId; singleFragmentTransfer.rowCount = 0; singleFragmentTransfer.targetShardId = INVALID_SHARD_ID; singleFragmentTransfer.targetShardIndex = partitionId; NodeToNodeFragmentsTransfer fragmentsTransfer; fragmentsTransfer.nodes.sourceNodeId = mapTaskPlacement->nodeId; /* * Target node is not yet decided, and not necessary for * QueryStringForFragmentsTransfer. */ fragmentsTransfer.nodes.targetNodeId = -1; fragmentsTransfer.fragmentList = list_make1(&singleFragmentTransfer); char *fetchQueryString = QueryStringForFragmentsTransfer(&fragmentsTransfer); Task *mapOutputFetchTask = CreateBasicTask(jobId, taskIdIndex, MAP_OUTPUT_FETCH_TASK, fetchQueryString); mapOutputFetchTask->partitionId = partitionId; mapOutputFetchTask->upstreamTaskId = mergeTaskId; mapOutputFetchTask->dependentTaskList = list_make1(mapTask); taskIdIndex++; mapOutputFetchTaskList = lappend(mapOutputFetchTaskList, mapOutputFetchTask); } /* merge task depends on completion of fetch tasks */ mergeTask->dependentTaskList = mapOutputFetchTaskList; mergeTask->mapJobTargetList = targetEntryList; /* if single repartitioned, each merge task represents an interval */ if (mapMergeJob->partitionType == RANGE_PARTITION_TYPE) { int32 mergeTaskIntervalId = partitionId - 1; ShardInterval **mergeTaskIntervals = mapMergeJob->sortedShardIntervalArray; Assert(mergeTaskIntervalId >= 0); mergeTask->shardInterval = mergeTaskIntervals[mergeTaskIntervalId]; } else if (mapMergeJob->partitionType == SINGLE_HASH_PARTITION_TYPE) { int32 mergeTaskIntervalId = partitionId; ShardInterval **mergeTaskIntervals = mapMergeJob->sortedShardIntervalArray; Assert(mergeTaskIntervalId >= 0); mergeTask->shardInterval = mergeTaskIntervals[mergeTaskIntervalId]; } mergeTaskList = lappend(mergeTaskList, mergeTask); } return mergeTaskList; } /* * AssignTaskList assigns locations to given tasks based on dependencies between * tasks and configured task assignment policies. The function also handles the * case where multiple SQL tasks depend on the same merge task, and makes sure * that this group of multiple SQL tasks and the merge task are assigned to the * same location. */ static List * AssignTaskList(List *sqlTaskList) { List *assignedSqlTaskList = NIL; bool hasAnchorShardId = false; ListCell *sqlTaskCell = NULL; List *primarySqlTaskList = NIL; ListCell *primarySqlTaskCell = NULL; ListCell *constrainedSqlTaskCell = NULL; /* no tasks to assign */ if (sqlTaskList == NIL) { return NIL; } Task *firstSqlTask = (Task *) linitial(sqlTaskList); if (firstSqlTask->anchorShardId != INVALID_SHARD_ID) { hasAnchorShardId = true; } /* * If these SQL tasks don't depend on any merge tasks, we can assign each * one independently of the other. We therefore go ahead and assign these * SQL tasks using the "anchor shard based" assignment algorithms. */ bool hasMergeTaskDependencies = HasMergeTaskDependencies(sqlTaskList); if (!hasMergeTaskDependencies) { Assert(hasAnchorShardId); assignedSqlTaskList = AssignAnchorShardTaskList(sqlTaskList); return assignedSqlTaskList; } /* * SQL tasks can depend on merge tasks in one of two ways: (1) each SQL task * depends on merge task(s) that no other SQL task depends upon, (2) several * SQL tasks depend on the same merge task(s) and all need to be assigned to * the same worker node. To handle the second case, we first pick a primary * SQL task among those that depend on the same merge task, and assign it. */ foreach(sqlTaskCell, sqlTaskList) { Task *sqlTask = (Task *) lfirst(sqlTaskCell); List *mergeTaskList = FindDependentMergeTaskList(sqlTask); Task *firstMergeTask = (Task *) linitial(mergeTaskList); if (!firstMergeTask->assignmentConstrained) { firstMergeTask->assignmentConstrained = true; primarySqlTaskList = lappend(primarySqlTaskList, sqlTask); } } if (hasAnchorShardId) { primarySqlTaskList = AssignAnchorShardTaskList(primarySqlTaskList); } else { primarySqlTaskList = AssignDualHashTaskList(primarySqlTaskList); } /* propagate SQL task assignments to the merge tasks we depend upon */ foreach(primarySqlTaskCell, primarySqlTaskList) { Task *sqlTask = (Task *) lfirst(primarySqlTaskCell); List *mergeTaskList = FindDependentMergeTaskList(sqlTask); ListCell *mergeTaskCell = NULL; foreach(mergeTaskCell, mergeTaskList) { Task *mergeTask = (Task *) lfirst(mergeTaskCell); Assert(mergeTask->taskPlacementList == NIL); mergeTask->taskPlacementList = list_copy(sqlTask->taskPlacementList); } assignedSqlTaskList = lappend(assignedSqlTaskList, sqlTask); } /* * If we had a set of SQL tasks depending on the same merge task, we only * assigned one SQL task from that set. We call the assigned SQL task the * primary, and note that the remaining SQL tasks are constrained by the * primary's task assignment. We propagate the primary's task assignment in * each set to the remaining (constrained) tasks. */ List *constrainedSqlTaskList = TaskListDifference(sqlTaskList, primarySqlTaskList); foreach(constrainedSqlTaskCell, constrainedSqlTaskList) { Task *sqlTask = (Task *) lfirst(constrainedSqlTaskCell); List *mergeTaskList = FindDependentMergeTaskList(sqlTask); List *mergeTaskPlacementList = NIL; ListCell *mergeTaskCell = NULL; foreach(mergeTaskCell, mergeTaskList) { Task *mergeTask = (Task *) lfirst(mergeTaskCell); /* * If we have more than one merge task, both of them should have the * same task placement list. */ mergeTaskPlacementList = mergeTask->taskPlacementList; Assert(mergeTaskPlacementList != NIL); ereport(DEBUG3, (errmsg("propagating assignment from merge task %d " "to constrained sql task %d", mergeTask->taskId, sqlTask->taskId))); } sqlTask->taskPlacementList = list_copy(mergeTaskPlacementList); assignedSqlTaskList = lappend(assignedSqlTaskList, sqlTask); } return assignedSqlTaskList; } /* * HasMergeTaskDependencies checks if sql tasks in the given sql task list have * any dependencies on merge tasks. If they do, the function returns true. */ static bool HasMergeTaskDependencies(List *sqlTaskList) { bool hasMergeTaskDependencies = false; Task *sqlTask = (Task *) linitial(sqlTaskList); List *dependentTaskList = sqlTask->dependentTaskList; ListCell *dependentTaskCell = NULL; foreach(dependentTaskCell, dependentTaskList) { Task *dependentTask = (Task *) lfirst(dependentTaskCell); if (dependentTask->taskType == MERGE_TASK) { hasMergeTaskDependencies = true; break; } } return hasMergeTaskDependencies; } /* Return true if two tasks are equal, false otherwise. */ bool TasksEqual(const Task *a, const Task *b) { Assert(CitusIsA(a, Task)); Assert(CitusIsA(b, Task)); if (a->taskType != b->taskType) { return false; } if (a->jobId != b->jobId) { return false; } if (a->taskId != b->taskId) { return false; } return true; } /* Is the passed in Task a member of the list. */ bool TaskListMember(const List *taskList, const Task *task) { const ListCell *taskCell = NULL; foreach(taskCell, taskList) { if (TasksEqual((Task *) lfirst(taskCell), task)) { return true; } } return false; } /* * TaskListDifference returns a list that contains all the tasks in taskList1 * that are not in taskList2. The returned list is freshly allocated via * palloc(), but the cells themselves point to the same objects as the cells * of the input lists. */ List * TaskListDifference(const List *list1, const List *list2) { const ListCell *taskCell = NULL; List *resultList = NIL; if (list2 == NIL) { return list_copy(list1); } foreach(taskCell, list1) { if (!TaskListMember(list2, lfirst(taskCell))) { resultList = lappend(resultList, lfirst(taskCell)); } } return resultList; } /* * AssignAnchorShardTaskList assigns locations to the given tasks based on the * configured task assignment policy. The distributed executor later sends these * tasks to their assigned locations for remote execution. */ List * AssignAnchorShardTaskList(List *taskList) { List *assignedTaskList = NIL; /* choose task assignment policy based on config value */ if (TaskAssignmentPolicy == TASK_ASSIGNMENT_GREEDY) { assignedTaskList = GreedyAssignTaskList(taskList); } else if (TaskAssignmentPolicy == TASK_ASSIGNMENT_FIRST_REPLICA) { assignedTaskList = FirstReplicaAssignTaskList(taskList); } else if (TaskAssignmentPolicy == TASK_ASSIGNMENT_ROUND_ROBIN) { assignedTaskList = RoundRobinAssignTaskList(taskList); } Assert(assignedTaskList != NIL); return assignedTaskList; } /* * GreedyAssignTaskList uses a greedy algorithm similar to Hadoop's, and assigns * locations to the given tasks. The ideal assignment algorithm balances three * properties: (a) determinism, (b) even load distribution, and (c) consistency * across similar task lists. To maintain these properties, the algorithm sorts * all its input lists. */ static List * GreedyAssignTaskList(List *taskList) { List *assignedTaskList = NIL; uint32 assignedTaskCount = 0; uint32 taskCount = list_length(taskList); /* get the worker node list and sort the list */ List *workerNodeList = ActiveReadableNodeList(); workerNodeList = SortList(workerNodeList, CompareWorkerNodes); /* * We first sort tasks by their anchor shard id. We then walk over each task * in the sorted list, get the task's anchor shard id, and look up the shard * placements (locations) for this shard id. Next, we sort the placements by * their insertion time, and append them to a new list. */ taskList = SortList(taskList, CompareTasksByShardId); List *activeShardPlacementLists = ActiveShardPlacementLists(taskList); while (assignedTaskCount < taskCount) { ListCell *workerNodeCell = NULL; uint32 loopStartTaskCount = assignedTaskCount; /* walk over each node and check if we can assign a task to it */ foreach(workerNodeCell, workerNodeList) { WorkerNode *workerNode = (WorkerNode *) lfirst(workerNodeCell); Task *assignedTask = GreedyAssignTask(workerNode, taskList, activeShardPlacementLists); if (assignedTask != NULL) { assignedTaskList = lappend(assignedTaskList, assignedTask); assignedTaskCount++; } } /* if we could not assign any new tasks, avoid looping forever */ if (assignedTaskCount == loopStartTaskCount) { uint32 remainingTaskCount = taskCount - assignedTaskCount; ereport(ERROR, (errmsg("failed to assign %u task(s) to worker nodes", remainingTaskCount))); } } return assignedTaskList; } /* * GreedyAssignTask tries to assign a task to the given worker node. To do this, * the function walks over tasks' anchor shard ids, and finds the first set of * nodes the shards were replicated to. If any of these replica nodes and the * given worker node match, the corresponding task is assigned to that node. If * not, the function goes on to search the second set of replicas and so forth. * * Note that this function has side-effects; when the function assigns a new * task, it overwrites the corresponding task list pointer. */ static Task * GreedyAssignTask(WorkerNode *workerNode, List *taskList, List *activeShardPlacementLists) { Task *assignedTask = NULL; List *taskPlacementList = NIL; ShardPlacement *primaryPlacement = NULL; uint32 rotatePlacementListBy = 0; uint32 replicaIndex = 0; uint32 replicaCount = ShardReplicationFactor; const char *workerName = workerNode->workerName; const uint32 workerPort = workerNode->workerPort; while ((assignedTask == NULL) && (replicaIndex < replicaCount)) { /* walk over all tasks and try to assign one */ ListCell *taskCell = NULL; ListCell *placementListCell = NULL; forboth(taskCell, taskList, placementListCell, activeShardPlacementLists) { Task *task = (Task *) lfirst(taskCell); List *placementList = (List *) lfirst(placementListCell); /* check if we already assigned this task */ if (task == NULL) { continue; } /* check if we have enough replicas */ uint32 placementCount = list_length(placementList); if (placementCount <= replicaIndex) { continue; } ShardPlacement *placement = (ShardPlacement *) list_nth(placementList, replicaIndex); if ((strncmp(placement->nodeName, workerName, WORKER_LENGTH) == 0) && (placement->nodePort == workerPort)) { /* we found a task to assign to the given worker node */ assignedTask = task; taskPlacementList = placementList; rotatePlacementListBy = replicaIndex; /* overwrite task list to signal that this task is assigned */ SetListCellPtr(taskCell, NULL); break; } } /* go over the next set of shard replica placements */ replicaIndex++; } /* if we found a task placement list, rotate and assign task placements */ if (assignedTask != NULL) { taskPlacementList = LeftRotateList(taskPlacementList, rotatePlacementListBy); assignedTask->taskPlacementList = taskPlacementList; primaryPlacement = (ShardPlacement *) linitial(assignedTask->taskPlacementList); ereport(DEBUG3, (errmsg("assigned task %u to node %s:%u", assignedTask->taskId, primaryPlacement->nodeName, primaryPlacement->nodePort))); } return assignedTask; } /* * FirstReplicaAssignTaskList assigns locations to the given tasks simply by * looking at placements for a given shard. A particular task's assignments are * then ordered by the insertion order of the relevant placements rows. In other * words, a task for a specific shard is simply assigned to the first replica * for that shard. This algorithm is extremely simple and intended for use when * a customer has placed shards carefully and wants strong guarantees about * which shards will be used by what nodes (i.e. for stronger memory residency * guarantees). */ List * FirstReplicaAssignTaskList(List *taskList) { /* No additional reordering need take place for this algorithm */ ReorderFunction reorderFunction = NULL; taskList = ReorderAndAssignTaskList(taskList, reorderFunction); return taskList; } /* * RoundRobinAssignTaskList uses a round-robin algorithm to assign locations to * the given tasks. An ideal round-robin implementation requires keeping shared * state for task assignments; and we instead approximate our implementation by * relying on the sequentially increasing jobId. For each task, we mod its jobId * by the number of active shard placements, and ensure that we rotate between * these placements across subsequent queries. */ List * RoundRobinAssignTaskList(List *taskList) { taskList = ReorderAndAssignTaskList(taskList, RoundRobinReorder); return taskList; } /* * RoundRobinReorder implements the core of the round-robin assignment policy. * It takes a placement list and rotates a copy of it based on the latest stable * transaction id provided by PostgreSQL. * * We prefer to use transactionId as the seed for the rotation to use the replicas * in the same worker node within the same transaction. This becomes more important * when we're reading from (the same or multiple) reference tables within a * transaction. With this approach, we can prevent reads to expand the worker nodes * that participate in a distributed transaction. * * Note that we prefer PostgreSQL's transactionId over distributed transactionId that * Citus generates since the distributed transactionId is generated during the execution * where as task-assignment happens duing the planning. */ List * RoundRobinReorder(List *placementList) { TransactionId transactionId = GetMyProcLocalTransactionId(); uint32 activePlacementCount = list_length(placementList); uint32 roundRobinIndex = (transactionId % activePlacementCount); placementList = LeftRotateList(placementList, roundRobinIndex); return placementList; } /* * ReorderAndAssignTaskList finds the placements for a task based on its anchor * shard id and then sorts them by insertion time. If reorderFunction is given, * it is used to reorder the placements list in a custom fashion (for instance, * by rotation or shuffling). Returns the task list with placements assigned. */ static List * ReorderAndAssignTaskList(List *taskList, ReorderFunction reorderFunction) { List *assignedTaskList = NIL; ListCell *taskCell = NULL; ListCell *placementListCell = NULL; uint32 unAssignedTaskCount = 0; if (taskList == NIL) { return NIL; } /* * We first sort tasks by their anchor shard id. We then sort placements for * each anchor shard by the placement's insertion time. Note that we sort * these lists just to make our policy more deterministic. */ taskList = SortList(taskList, CompareTasksByShardId); List *activeShardPlacementLists = ActiveShardPlacementLists(taskList); forboth(taskCell, taskList, placementListCell, activeShardPlacementLists) { Task *task = (Task *) lfirst(taskCell); List *placementList = (List *) lfirst(placementListCell); /* inactive placements are already filtered out */ uint32 activePlacementCount = list_length(placementList); if (activePlacementCount > 0) { if (reorderFunction != NULL) { placementList = reorderFunction(placementList); } task->taskPlacementList = placementList; ShardPlacement *primaryPlacement = (ShardPlacement *) linitial( task->taskPlacementList); ereport(DEBUG3, (errmsg("assigned task %u to node %s:%u", task->taskId, primaryPlacement->nodeName, primaryPlacement->nodePort))); assignedTaskList = lappend(assignedTaskList, task); } else { unAssignedTaskCount++; } } /* if we have unassigned tasks, error out */ if (unAssignedTaskCount > 0) { ereport(ERROR, (errmsg("failed to assign %u task(s) to worker nodes", unAssignedTaskCount))); } return assignedTaskList; } /* Helper function to compare two tasks by their anchor shardId. */ static int CompareTasksByShardId(const void *leftElement, const void *rightElement) { const Task *leftTask = *((const Task **) leftElement); const Task *rightTask = *((const Task **) rightElement); uint64 leftShardId = leftTask->anchorShardId; uint64 rightShardId = rightTask->anchorShardId; /* we compare 64-bit integers, instead of casting their difference to int */ if (leftShardId > rightShardId) { return 1; } else if (leftShardId < rightShardId) { return -1; } else { return 0; } } /* * ActiveShardPlacementLists finds the active shard placement list for each task in * the given task list, sorts each shard placement list by shard creation time, * and adds the sorted placement list into a new list of lists. The function also * ensures a one-to-one mapping between each placement list in the new list of * lists and each task in the given task list. */ static List * ActiveShardPlacementLists(List *taskList) { List *shardPlacementLists = NIL; ListCell *taskCell = NULL; foreach(taskCell, taskList) { Task *task = (Task *) lfirst(taskCell); uint64 anchorShardId = task->anchorShardId; List *activeShardPlacementList = ActiveShardPlacementList(anchorShardId); if (activeShardPlacementList == NIL) { ereport(ERROR, (errmsg("no active placements were found for shard " UINT64_FORMAT, anchorShardId))); } /* sort shard placements by their creation time */ activeShardPlacementList = SortList(activeShardPlacementList, CompareShardPlacements); shardPlacementLists = lappend(shardPlacementLists, activeShardPlacementList); } return shardPlacementLists; } /* * CompareShardPlacements compares two shard placements by placement id. */ int CompareShardPlacements(const void *leftElement, const void *rightElement) { const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); uint64 leftPlacementId = leftPlacement->placementId; uint64 rightPlacementId = rightPlacement->placementId; if (leftPlacementId < rightPlacementId) { return -1; } else if (leftPlacementId > rightPlacementId) { return 1; } else { return 0; } } /* * CompareGroupShardPlacements compares two group shard placements by placement id. */ int CompareGroupShardPlacements(const void *leftElement, const void *rightElement) { const GroupShardPlacement *leftPlacement = *((const GroupShardPlacement **) leftElement); const GroupShardPlacement *rightPlacement = *((const GroupShardPlacement **) rightElement); uint64 leftPlacementId = leftPlacement->placementId; uint64 rightPlacementId = rightPlacement->placementId; if (leftPlacementId < rightPlacementId) { return -1; } else if (leftPlacementId > rightPlacementId) { return 1; } else { return 0; } } /* * LeftRotateList returns a copy of the given list that has been cyclically * shifted to the left by the given rotation count. For this, the function * repeatedly moves the list's first element to the end of the list, and * then returns the newly rotated list. */ static List * LeftRotateList(List *list, uint32 rotateCount) { List *rotatedList = list_copy(list); for (uint32 rotateIndex = 0; rotateIndex < rotateCount; rotateIndex++) { void *firstElement = linitial(rotatedList); rotatedList = list_delete_first(rotatedList); rotatedList = lappend(rotatedList, firstElement); } return rotatedList; } /* * FindDependentMergeTaskList walks over the given task's dependent task list, * finds the merge tasks in the list, and returns those found tasks in a new * list. */ static List * FindDependentMergeTaskList(Task *sqlTask) { List *dependentMergeTaskList = NIL; List *dependentTaskList = sqlTask->dependentTaskList; ListCell *dependentTaskCell = NULL; foreach(dependentTaskCell, dependentTaskList) { Task *dependentTask = (Task *) lfirst(dependentTaskCell); if (dependentTask->taskType == MERGE_TASK) { dependentMergeTaskList = lappend(dependentMergeTaskList, dependentTask); } } return dependentMergeTaskList; } /* * AssignDualHashTaskList uses a round-robin algorithm to assign locations to * tasks; these tasks don't have any anchor shards and instead operate on (hash * repartitioned) merged tables. */ static List * AssignDualHashTaskList(List *taskList) { List *assignedTaskList = NIL; ListCell *taskCell = NULL; Task *firstTask = (Task *) linitial(taskList); uint64 jobId = firstTask->jobId; uint32 assignedTaskIndex = 0; /* * We start assigning tasks at an index determined by the jobId. This way, * if subsequent jobs have a small number of tasks, we won't allocate the * tasks to the same worker repeatedly. */ List *workerNodeList = ActiveReadableNodeList(); uint32 workerNodeCount = (uint32) list_length(workerNodeList); uint32 beginningNodeIndex = jobId % workerNodeCount; /* sort worker node list and task list for deterministic results */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); taskList = SortList(taskList, CompareTasksByTaskId); foreach(taskCell, taskList) { Task *task = (Task *) lfirst(taskCell); List *taskPlacementList = NIL; for (uint32 replicaIndex = 0; replicaIndex < ShardReplicationFactor; replicaIndex++) { uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + replicaIndex; uint32 assignmentIndex = assignmentOffset % workerNodeCount; WorkerNode *workerNode = list_nth(workerNodeList, assignmentIndex); ShardPlacement *taskPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(taskPlacement, workerNode); taskPlacementList = lappend(taskPlacementList, taskPlacement); } task->taskPlacementList = taskPlacementList; ShardPlacement *primaryPlacement = (ShardPlacement *) linitial( task->taskPlacementList); ereport(DEBUG3, (errmsg("assigned task %u to node %s:%u", task->taskId, primaryPlacement->nodeName, primaryPlacement->nodePort))); assignedTaskList = lappend(assignedTaskList, task); assignedTaskIndex++; } return assignedTaskList; } /* * SetPlacementNodeMetadata sets nodename, nodeport, nodeid and groupid for the placement. */ void SetPlacementNodeMetadata(ShardPlacement *placement, WorkerNode *workerNode) { placement->nodeName = pstrdup(workerNode->workerName); placement->nodePort = workerNode->workerPort; placement->nodeId = workerNode->nodeId; placement->groupId = workerNode->groupId; } /* * CompareTasksByTaskId is a helper function to compare two tasks by their taskId. */ int CompareTasksByTaskId(const void *leftElement, const void *rightElement) { const Task *leftTask = *((const Task **) leftElement); const Task *rightTask = *((const Task **) rightElement); uint32 leftTaskId = leftTask->taskId; uint32 rightTaskId = rightTask->taskId; int taskIdDiff = leftTaskId - rightTaskId; return taskIdDiff; } /* * AssignDataFetchDependencies walks over tasks in the given sql or merge task * list. The function then propagates worker node assignments from each sql or * merge task to the task's data fetch dependencies. */ static void AssignDataFetchDependencies(List *taskList) { ListCell *taskCell = NULL; foreach(taskCell, taskList) { Task *task = (Task *) lfirst(taskCell); List *dependentTaskList = task->dependentTaskList; ListCell *dependentTaskCell = NULL; Assert(task->taskPlacementList != NIL); Assert(task->taskType == READ_TASK || task->taskType == MERGE_TASK); foreach(dependentTaskCell, dependentTaskList) { Task *dependentTask = (Task *) lfirst(dependentTaskCell); if (dependentTask->taskType == MAP_OUTPUT_FETCH_TASK) { dependentTask->taskPlacementList = task->taskPlacementList; } } } } /* * TaskListHighestTaskId walks over tasks in the given task list, finds the task * that has the largest taskId, and returns that taskId. * * Note: This function assumes that the dependent taskId's are set before the * taskId's for the given task list. */ static uint32 TaskListHighestTaskId(List *taskList) { uint32 highestTaskId = 0; ListCell *taskCell = NULL; foreach(taskCell, taskList) { Task *task = (Task *) lfirst(taskCell); if (task->taskId > highestTaskId) { highestTaskId = task->taskId; } } return highestTaskId; } /* * QueryTreeHasImproperForDeparseNodes walks over the node, * and returns true if there are RelabelType or * CoerceViaIONodes which are improper for deparse */ static bool QueryTreeHasImproperForDeparseNodes(Node *inputNode, void *context) { if (inputNode == NULL) { return false; } else if (IsImproperForDeparseRelabelTypeNode(inputNode) || IsImproperForDeparseCoerceViaIONode(inputNode)) { return true; } else if (IsA(inputNode, Query)) { return query_tree_walker((Query *) inputNode, QueryTreeHasImproperForDeparseNodes, NULL, 0); } return expression_tree_walker(inputNode, QueryTreeHasImproperForDeparseNodes, NULL); } /* * AdjustImproperForDeparseNodes takes an input rewritten query and modifies * nodes which, after going through our planner, pose a problem when * deparsing. So far we have two such type of Nodes that may pose problems: * RelabelType and CoerceIO nodes. * Details will be written in comments in the corresponding if conditions. */ static Node * AdjustImproperForDeparseNodes(Node *inputNode, void *context) { if (inputNode == NULL) { return NULL; } if (IsImproperForDeparseRelabelTypeNode(inputNode)) { /* * The planner converts CollateExpr to RelabelType * and here we convert back. */ return (Node *) RelabelTypeToCollateExpr((RelabelType *) inputNode); } else if (IsImproperForDeparseCoerceViaIONode(inputNode)) { /* * The planner converts some ::text/::varchar casts to ::cstring * and here we convert back to text because cstring is a pseudotype * and it cannot be casted to most resulttypes */ CoerceViaIO *iocoerce = (CoerceViaIO *) inputNode; Node *arg = (Node *) iocoerce->arg; Const *cstringToText = (Const *) arg; cstringToText->consttype = TEXTOID; cstringToText->constlen = -1; Type textType = typeidType(TEXTOID); char *constvalue = NULL; if (!cstringToText->constisnull) { constvalue = DatumGetCString(cstringToText->constvalue); } cstringToText->constvalue = stringTypeDatum(textType, constvalue, cstringToText->consttypmod); ReleaseSysCache(textType); return inputNode; } else if (IsA(inputNode, Query)) { return (Node *) query_tree_mutator((Query *) inputNode, AdjustImproperForDeparseNodes, NULL, QTW_DONT_COPY_QUERY); } return expression_tree_mutator(inputNode, AdjustImproperForDeparseNodes, NULL); } /* * Checks if the given node is of Relabel type which is improper for deparsing * The planner converts some CollateExpr to RelabelType nodes, and we need * to find these nodes. They would be improperly deparsed without the * "COLLATE" expression. */ static bool IsImproperForDeparseRelabelTypeNode(Node *inputNode) { return (IsA(inputNode, RelabelType) && OidIsValid(((RelabelType *) inputNode)->resultcollid) && ((RelabelType *) inputNode)->resultcollid != DEFAULT_COLLATION_OID); } /* * Checks if the given node is of CoerceViaIO type which is improper for deparsing * The planner converts some ::text/::varchar casts to ::cstring, and we need * to find these nodes. They would be improperly deparsed with "cstring" which cannot * be casted to most resulttypes. */ static bool IsImproperForDeparseCoerceViaIONode(Node *inputNode) { return (IsA(inputNode, CoerceViaIO) && IsA(((CoerceViaIO *) inputNode)->arg, Const) && ((Const *) ((CoerceViaIO *) inputNode)->arg)->consttype == CSTRINGOID); } ================================================ FILE: src/backend/distributed/planner/multi_router_planner.c ================================================ /*------------------------------------------------------------------------- * * multi_router_planner.c * * This file contains functions to plan multiple shard queries without any * aggregation step including distributed table modifications. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "access/stratnum.h" #include "access/tupdesc.h" #include "access/tupdesc_details.h" #include "access/xact.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "executor/execdesc.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/joininfo.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/planmain.h" #include "optimizer/planner.h" #include "optimizer/restrictinfo.h" #include "parser/parse_oper.h" #include "parser/parsetree.h" #include "postmaster/postmaster.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/distribution_column.h" #include "distributed/errormessage.h" #include "distributed/executor_util.h" #include "distributed/insert_select_planner.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/merge_planner.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/relay_utility.h" #include "distributed/resource_lock.h" #include "distributed/shard_pruning.h" #include "distributed/shard_utils.h" #include "distributed/shardinterval_utils.h" /* intermediate value for INSERT processing */ typedef struct InsertValues { Expr *partitionValueExpr; /* partition value provided in INSERT row */ List *rowValues; /* full values list of INSERT row, possibly NIL */ int64 shardId; /* target shard for this row, possibly invalid */ Index listIndex; /* index to make our sorting stable */ } InsertValues; /* * A ModifyRoute encapsulates the information needed to route modifications * to the appropriate shard. For a single-shard modification, only one route * is needed, but in the case of e.g. a multi-row INSERT, lists of these values * will help divide the rows by their destination shards, permitting later * shard-and-row-specific extension of the original SQL. */ typedef struct ModifyRoute { int64 shardId; /* identifier of target shard */ List *rowValuesLists; /* for multi-row INSERTs, list of rows to be inserted */ } ModifyRoute; typedef struct WalkerState { bool containsVar; bool varArgument; bool badCoalesce; } WalkerState; bool EnableRouterExecution = true; bool EnableNonColocatedRouterQueryPushdown = false; /* planner functions forward declarations */ static void CreateSingleTaskRouterSelectPlan(DistributedPlan *distributedPlan, Query *originalQuery, Query *query, PlannerRestrictionContext * plannerRestrictionContext); static bool IsTidColumn(Node *node); static DeferredErrorMessage * ModifyPartialQuerySupported(Query *queryTree, bool multiShardQuery, Oid *distributedTableId); static DeferredErrorMessage * MultiShardUpdateDeleteSupported(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); static DeferredErrorMessage * SingleShardUpdateDeleteSupported(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); static bool MasterIrreducibleExpressionWalker(Node *expression, WalkerState *state); static bool MasterIrreducibleExpressionFunctionChecker(Oid func_id, void *context); static Job * RouterInsertJob(Query *originalQuery); static void ErrorIfNoShardsExist(CitusTableCacheEntry *cacheEntry); static DeferredErrorMessage * DeferErrorIfModifyView(Query *queryTree); static Job * CreateJob(Query *query); static Task * CreateTask(TaskType taskType); static bool RelationPrunesToMultipleShards(List *relationShardList); static void NormalizeMultiRowInsertTargetList(Query *query); static void AppendNextDummyColReference(Alias *expendedReferenceNames); static String * MakeDummyColumnString(int dummyColumnId); static List * BuildRoutesForInsert(Query *query, DeferredErrorMessage **planningError); static List * GroupInsertValuesByShardId(List *insertValuesList); static List * ExtractInsertValuesList(Query *query, Var *partitionColumn); static DeferredErrorMessage * DeferErrorIfUnsupportedRouterPlannableSelectQuery(Query * query); static DeferredErrorMessage * ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree); static DeferredErrorMessage * ErrorIfQueryHasCTEWithSearchClause(Query *queryTree); static bool ContainsSearchClauseWalker(Node *node, void *context); static bool SelectsFromDistributedTable(List *rangeTableList, Query *query); static bool AllShardsColocated(List *relationShardList); static ShardPlacement * CreateDummyPlacement(bool hasLocalRelation); static ShardPlacement * CreateLocalDummyPlacement(); static int CompareInsertValuesByShardId(const void *leftElement, const void *rightElement); static List * SingleShardTaskList(Query *query, uint64 jobId, List *relationShardList, List *placementList, uint64 shardId, bool parametersInQueryResolved, bool isLocalTableModification, Const *partitionKeyValue, int colocationId, bool delayedFastPath); static bool RowLocksOnRelations(Node *node, List **rtiLockList); static void ReorderTaskPlacementsByTaskAssignmentPolicy(Job *job, TaskAssignmentPolicyType taskAssignmentPolicy, List *placementList); static bool ModifiesLocalTableWithRemoteCitusLocalTable(List *rangeTableList); static DeferredErrorMessage * DeferErrorIfUnsupportedLocalTableJoin(List *rangeTableList); static bool IsLocallyAccessibleCitusLocalTable(Oid relationId); static bool ConvertToQueryOnShard(Query *query, Oid relationID, Oid shardRelationId); /* * CreateRouterPlan attempts to create a router executor plan for the given * SELECT statement. ->planningError is set if planning fails. */ DistributedPlan * CreateRouterPlan(Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext) { DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); distributedPlan->planningError = DeferErrorIfUnsupportedRouterPlannableSelectQuery( query); if (distributedPlan->planningError == NULL) { CreateSingleTaskRouterSelectPlan(distributedPlan, originalQuery, query, plannerRestrictionContext); } distributedPlan->fastPathRouterPlan = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; return distributedPlan; } /* * CreateModifyPlan attempts to create a plan for the given modification * statement. If planning fails ->planningError is set to a description of * the failure. */ DistributedPlan * CreateModifyPlan(Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext) { Job *job = NULL; DistributedPlan *distributedPlan = CitusMakeNode(DistributedPlan); bool multiShardQuery = false; Assert(originalQuery->commandType != CMD_SELECT); distributedPlan->modLevel = RowModifyLevelForQuery(query); distributedPlan->planningError = ModifyQuerySupported(query, originalQuery, multiShardQuery, plannerRestrictionContext); if (distributedPlan->planningError != NULL) { return distributedPlan; } if (UpdateOrDeleteOrMergeQuery(query)) { job = RouterJob(originalQuery, plannerRestrictionContext, &distributedPlan->planningError); } else { job = RouterInsertJob(originalQuery); } if (distributedPlan->planningError != NULL) { return distributedPlan; } ereport(DEBUG2, (errmsg("Creating router plan"))); distributedPlan->workerJob = job; distributedPlan->combineQuery = NULL; distributedPlan->expectResults = originalQuery->returningList != NIL; distributedPlan->targetRelationId = ResultRelationOidForQuery(query); distributedPlan->fastPathRouterPlan = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; return distributedPlan; } /* * CreateSingleTaskRouterSelectPlan creates a physical plan for given SELECT query. * The returned plan is a router task that returns query results from a single worker. * If not router plannable, the returned plan's planningError describes the problem. */ static void CreateSingleTaskRouterSelectPlan(DistributedPlan *distributedPlan, Query *originalQuery, Query *query, PlannerRestrictionContext *plannerRestrictionContext) { Assert(query->commandType == CMD_SELECT); distributedPlan->modLevel = RowModifyLevelForQuery(query); Job *job = RouterJob(originalQuery, plannerRestrictionContext, &distributedPlan->planningError); if (distributedPlan->planningError != NULL) { /* query cannot be handled by this planner */ return; } ereport(DEBUG2, (errmsg("Creating router plan"))); distributedPlan->workerJob = job; distributedPlan->combineQuery = NULL; distributedPlan->expectResults = true; } /* * ShardIntervalOpExpressions returns a list of OpExprs with exactly two * items in it. The list consists of shard interval ranges with partition columns * such as (partitionColumn >= shardMinValue) and (partitionColumn <= shardMaxValue). * * The function returns hashed columns generated by MakeInt4Column() for the hash * partitioned tables in place of partition columns. * * The function returns NIL if shard interval does not belong to a hash, * range and append distributed tables. * * NB: If you update this, also look at PrunableExpressionsWalker(). */ List * ShardIntervalOpExpressions(ShardInterval *shardInterval, Index rteIndex) { Oid relationId = shardInterval->relationId; Var *partitionColumn = NULL; if (IsCitusTableType(relationId, HASH_DISTRIBUTED)) { partitionColumn = MakeInt4Column(); } else if (IsCitusTableType(relationId, RANGE_DISTRIBUTED) || IsCitusTableType( relationId, APPEND_DISTRIBUTED)) { Assert(rteIndex > 0); partitionColumn = PartitionColumn(relationId, rteIndex); } else { /* do not add any shard range interval for reference tables */ return NIL; } /* build the base expression for constraint */ Node *baseConstraint = BuildBaseConstraint(partitionColumn); /* walk over shard list and check if shards can be pruned */ if (shardInterval->minValueExists && shardInterval->maxValueExists) { UpdateConstraint(baseConstraint, shardInterval); } return list_make1(baseConstraint); } /* * AddPartitionKeyNotNullFilterToSelect adds the following filters to a subquery: * * partitionColumn IS NOT NULL * * The function expects and asserts that subquery's target list contains a partition * column value. Thus, this function should never be called with reference tables. */ void AddPartitionKeyNotNullFilterToSelect(Query *subqery) { List *targetList = subqery->targetList; ListCell *targetEntryCell = NULL; Var *targetPartitionColumnVar = NULL; /* iterate through the target entries */ foreach(targetEntryCell, targetList) { TargetEntry *targetEntry = lfirst(targetEntryCell); bool skipOuterVars = true; if (IsPartitionColumn(targetEntry->expr, subqery, skipOuterVars) && IsA(targetEntry->expr, Var)) { targetPartitionColumnVar = (Var *) targetEntry->expr; break; } } /* we should have found target partition column */ Assert(targetPartitionColumnVar != NULL); #if PG_VERSION_NUM >= PG_VERSION_18 if (subqery->hasGroupRTE) { /* if the partition column is a grouped column, we need to flatten it * to ensure query deparsing works correctly. We choose to do this here * instead of in ruletils.c because we want to keep the flattening logic * close to the NOT NULL filter injection. */ RangeTblEntry *partitionRTE = rt_fetch(targetPartitionColumnVar->varno, subqery->rtable); if (partitionRTE->rtekind == RTE_GROUP) { targetPartitionColumnVar = (Var *) flatten_group_exprs(NULL, subqery, (Node *) targetPartitionColumnVar); } } #endif /* create expression for partition_column IS NOT NULL */ NullTest *nullTest = makeNode(NullTest); nullTest->nulltesttype = IS_NOT_NULL; nullTest->arg = (Expr *) targetPartitionColumnVar; nullTest->argisrow = false; /* finally add the quals */ if (subqery->jointree->quals == NULL) { subqery->jointree->quals = (Node *) nullTest; } else { subqery->jointree->quals = make_and_qual(subqery->jointree->quals, (Node *) nullTest); } } /* * ExtractSourceResultRangeTableEntry Generic wrapper for modification commands that * utilizes results as input, based on an source query. */ RangeTblEntry * ExtractSourceResultRangeTableEntry(Query *query) { if (IsMergeQuery(query)) { return ExtractMergeSourceRangeTableEntry(query, false); } else if (CheckInsertSelectQuery(query)) { return ExtractSelectRangeTableEntry(query); } return NULL; } /* * ExtractSelectRangeTableEntry returns the range table entry of the subquery. * Note that the function expects and asserts that the input query be * an INSERT...SELECT query. */ RangeTblEntry * ExtractSelectRangeTableEntry(Query *query) { Assert(InsertSelectIntoCitusTable(query) || InsertSelectIntoLocalTable(query)); /* * Since we already asserted InsertSelectIntoCitusTable() it is safe to access * both lists */ List *fromList = query->jointree->fromlist; RangeTblRef *reference = linitial(fromList); RangeTblEntry *subqueryRte = rt_fetch(reference->rtindex, query->rtable); return subqueryRte; } /* * ModifyQueryResultRelationId returns the result relation's Oid * for the given modification query. * * The function errors out if the input query is not a * modify query (e.g., INSERT, UPDATE, DELETE or MERGE). So, this * function is not expected to be called on SELECT queries. */ Oid ModifyQueryResultRelationId(Query *query) { /* only modify queries have result relations */ if (!IsModifyCommand(query)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("input query is not a modification query"))); } RangeTblEntry *resultRte = ExtractResultRelationRTE(query); Assert(OidIsValid(resultRte->relid)); return resultRte->relid; } /* * ResultRelationOidForQuery returns the OID of the relation this is modified * by a given query. */ Oid ResultRelationOidForQuery(Query *query) { RangeTblEntry *resultRTE = rt_fetch(query->resultRelation, query->rtable); return resultRTE->relid; } /* * ExtractResultRelationRTE returns the table's resultRelation range table * entry. This returns NULL when there's no resultRelation, such as in a SELECT * query. */ RangeTblEntry * ExtractResultRelationRTE(Query *query) { if (query->resultRelation > 0) { return rt_fetch(query->resultRelation, query->rtable); } return NULL; } /* * ExtractResultRelationRTEOrError returns the table's resultRelation range table * entry and errors out if there's no result relation at all, e.g. like in a * SELECT query. * * This is a separate function (instead of using missingOk), so static analysis * reasons about NULL returns correctly. */ RangeTblEntry * ExtractResultRelationRTEOrError(Query *query) { RangeTblEntry *relation = ExtractResultRelationRTE(query); if (relation == NULL) { ereport(ERROR, (errmsg("no result relation could be found for the query"), errhint("is this a SELECT query?"))); } return relation; } /* * IsTidColumn gets a node and returns true if the node is a Var type of TID. */ static bool IsTidColumn(Node *node) { if (IsA(node, Var)) { Var *column = (Var *) node; if (column->vartype == TIDOID) { return true; } } return false; } /* * TargetlistAndFunctionsSupported implements a subset of what ModifyPartialQuerySupported * checks, that subset being checking what functions are allowed, if we are * updating distribution column, etc. * Note: This subset of checks are repeated for each MERGE modify action. */ DeferredErrorMessage * TargetlistAndFunctionsSupported(Oid resultRelationId, FromExpr *joinTree, Node *quals, List *targetList, CmdType commandType, List *returningList) { uint32 rangeTableId = 1; Var *partitionColumn = NULL; if (IsCitusTable(resultRelationId)) { partitionColumn = PartitionColumn(resultRelationId, rangeTableId); } bool hasVarArgument = false; /* A STABLE function is passed a Var argument */ bool hasBadCoalesce = false; /* CASE/COALESCE passed a mutable function */ ListCell *targetEntryCell = NULL; foreach(targetEntryCell, targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); /* skip resjunk entries: UPDATE adds some for ctid, etc. */ if (targetEntry->resjunk) { continue; } bool targetEntryPartitionColumn = false; AttrNumber targetColumnAttrNumber = InvalidAttrNumber; /* reference tables do not have partition column */ if (partitionColumn == NULL) { targetEntryPartitionColumn = false; } else { if (commandType == CMD_UPDATE) { /* * Note that it is not possible to give an alias to * UPDATE table SET ... */ if (targetEntry->resname) { targetColumnAttrNumber = get_attnum(resultRelationId, targetEntry->resname); if (targetColumnAttrNumber == partitionColumn->varattno) { targetEntryPartitionColumn = true; } } } } if (commandType == CMD_UPDATE && FindNodeMatchingCheckFunction((Node *) targetEntry->expr, CitusIsVolatileFunction)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in UPDATE queries on distributed " "tables must not be VOLATILE", NULL, NULL); } if (commandType == CMD_UPDATE && targetEntryPartitionColumn && TargetEntryChangesValue(targetEntry, partitionColumn, joinTree)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "modifying the partition value of rows is not " "allowed", NULL, NULL); } if (commandType == CMD_UPDATE && MasterIrreducibleExpression((Node *) targetEntry->expr, &hasVarArgument, &hasBadCoalesce)) { Assert(hasVarArgument || hasBadCoalesce); } if (FindNodeMatchingCheckFunction((Node *) targetEntry->expr, NodeIsFieldStore)) { /* DELETE cannot do field indirection already */ Assert(commandType == CMD_UPDATE || commandType == CMD_INSERT); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "inserting or modifying composite type fields is not " "supported", NULL, "Use the column name to insert or update the composite " "type as a single value"); } } if (joinTree != NULL) { if (FindNodeMatchingCheckFunction((Node *) quals, CitusIsVolatileFunction)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in the WHERE/ON/WHEN clause of modification " "queries on distributed tables must not be VOLATILE", NULL, NULL); } else if (MasterIrreducibleExpression(quals, &hasVarArgument, &hasBadCoalesce)) { Assert(hasVarArgument || hasBadCoalesce); } } if (hasVarArgument) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "STABLE functions used in UPDATE queries " "cannot be called with column references", NULL, NULL); } if (hasBadCoalesce) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "non-IMMUTABLE functions are not allowed in CASE or " "COALESCE statements", NULL, NULL); } if (contain_mutable_functions((Node *) returningList)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "non-IMMUTABLE functions are not allowed in the " "RETURNING clause", NULL, NULL); } if (quals != NULL && nodeTag(quals) == T_CurrentOfExpr) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot run DML queries with cursors", NULL, NULL); } return NULL; } /* * ModifyPartialQuerySupported implements a subset of what ModifyQuerySupported checks, * that subset being what's necessary to check modifying CTEs for. */ static DeferredErrorMessage * ModifyPartialQuerySupported(Query *queryTree, bool multiShardQuery, Oid *distributedTableIdOutput) { DeferredErrorMessage *deferredError = DeferErrorIfModifyView(queryTree); if (deferredError != NULL) { return deferredError; } CmdType commandType = queryTree->commandType; deferredError = DeferErrorIfUnsupportedLocalTableJoin(queryTree->rtable); if (deferredError != NULL) { return deferredError; } /* * Reject subqueries which are in SELECT or WHERE clause. * Queries which include subqueries in FROM clauses are rejected below. */ if (queryTree->hasSubLinks == true) { /* we support subqueries for INSERTs only via INSERT INTO ... SELECT */ if (!UpdateOrDeleteOrMergeQuery(queryTree)) { Assert(queryTree->commandType == CMD_INSERT); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "subqueries are not supported within INSERT queries", NULL, "Try rewriting your queries with 'INSERT " "INTO ... SELECT' syntax."); } } /* reject queries which include CommonTableExpr which aren't routable */ if (queryTree->cteList != NIL) { ListCell *cteCell = NULL; /* CTEs still not supported for INSERTs. */ if (queryTree->commandType == CMD_INSERT) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner doesn't support common table expressions with INSERT queries.", NULL, NULL); } foreach(cteCell, queryTree->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(cteCell); Query *cteQuery = (Query *) cte->ctequery; if (cteQuery->commandType != CMD_SELECT) { /* Modifying CTEs still not supported for multi shard queries. */ if (multiShardQuery) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner doesn't support non-select common table expressions with multi shard queries.", NULL, NULL); } /* Modifying CTEs exclude both INSERT CTEs & INSERT queries. */ else if (cteQuery->commandType == CMD_INSERT) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner doesn't support INSERT common table expressions.", NULL, NULL); } } if (cteQuery->hasForUpdate && FindNodeMatchingCheckFunctionInRangeTableList(cteQuery->rtable, IsReferenceTableRTE)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner doesn't support SELECT FOR UPDATE" " in common table expressions involving reference tables.", NULL, NULL); } if (FindNodeMatchingCheckFunction((Node *) cteQuery, CitusIsVolatileFunction)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner doesn't support VOLATILE functions" " in common table expressions.", NULL, NULL); } if (cteQuery->commandType == CMD_SELECT) { DeferredErrorMessage *cteError = DeferErrorIfUnsupportedRouterPlannableSelectQuery(cteQuery); if (cteError) { return cteError; } } } } Oid resultRelationId = ModifyQueryResultRelationId(queryTree); *distributedTableIdOutput = resultRelationId; commandType = queryTree->commandType; if (commandType == CMD_INSERT || commandType == CMD_UPDATE || commandType == CMD_DELETE) { deferredError = TargetlistAndFunctionsSupported(resultRelationId, queryTree->jointree, queryTree->jointree->quals, queryTree->targetList, commandType, queryTree->returningList); if (deferredError) { return deferredError; } } deferredError = ErrorIfOnConflictNotSupported(queryTree); if (deferredError != NULL) { return deferredError; } /* set it for caller to use when we don't return any errors */ *distributedTableIdOutput = resultRelationId; return NULL; } /* * DeferErrorIfUnsupportedLocalTableJoin returns an error message * if there is an unsupported join in the given range table list. */ static DeferredErrorMessage * DeferErrorIfUnsupportedLocalTableJoin(List *rangeTableList) { if (ModifiesLocalTableWithRemoteCitusLocalTable(rangeTableList)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Modifying local tables with remote local tables is " "not supported.", NULL, "Consider wrapping remote local table to a CTE, " "or subquery"); } return NULL; } /* * ModifiesLocalTableWithRemoteCitusLocalTable returns true if a local * table is modified with a remote citus local table. This could be a case with * MX structure. */ static bool ModifiesLocalTableWithRemoteCitusLocalTable(List *rangeTableList) { bool containsLocalResultRelation = false; bool containsRemoteCitusLocalTable = false; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (!IsRecursivelyPlannableRelation(rangeTableEntry)) { continue; } if (IsCitusTableType(rangeTableEntry->relid, CITUS_LOCAL_TABLE)) { if (!IsLocallyAccessibleCitusLocalTable(rangeTableEntry->relid)) { containsRemoteCitusLocalTable = true; } } else if (!IsCitusTable(rangeTableEntry->relid)) { containsLocalResultRelation = true; } } return containsLocalResultRelation && containsRemoteCitusLocalTable; } /* * IsLocallyAccessibleCitusLocalTable returns true if the given table * is a citus local table that can be accessed using local execution. */ static bool IsLocallyAccessibleCitusLocalTable(Oid relationId) { if (!IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { return false; } List *shardIntervalList = LoadShardIntervalList(relationId); /* * Citus local tables should always have exactly one shard, but we have * this check for safety. */ if (list_length(shardIntervalList) != 1) { return false; } ShardInterval *shardInterval = linitial(shardIntervalList); uint64 shardId = shardInterval->shardId; ShardPlacement *localShardPlacement = ActiveShardPlacementOnGroup(GetLocalGroupId(), shardId); return localShardPlacement != NULL; } /* * NodeIsFieldStore returns true if given Node is a FieldStore object. */ bool NodeIsFieldStore(Node *node) { return node && IsA(node, FieldStore); } /* * ModifyQuerySupported returns NULL if the query only contains supported * features, otherwise it returns an error description. * Note that we need both the original query and the modified one because * different checks need different versions. In particular, we cannot * perform the ContainsReadIntermediateResultFunction check on the * rewritten query because it may have been replaced by a subplan, * while some of the checks for setting the partition column value rely * on the rewritten query. */ DeferredErrorMessage * ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuery, PlannerRestrictionContext *plannerRestrictionContext) { Oid distributedTableId = InvalidOid; DeferredErrorMessage *error = ModifyPartialQuerySupported(queryTree, multiShardQuery, &distributedTableId); if (error) { return error; } List *rangeTableList = NIL; CmdType commandType = queryTree->commandType; bool fastPathRouterQuery = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; /* * Here, we check if a recursively planned query tries to modify * rows based on the ctid column. This is a bad idea because ctid of * the rows could be changed before the modification part of * the query is executed. * * We can exclude fast path queries since they cannot have intermediate * results by definition. */ if (!fastPathRouterQuery && ContainsReadIntermediateResultFunction((Node *) originalQuery)) { bool hasTidColumn = FindNodeMatchingCheckFunction( (Node *) originalQuery->jointree, IsTidColumn); if (hasTidColumn) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed planning for the given " "modification", "Recursively planned distributed modifications " "with ctid on where clause are not supported.", NULL); } } /* * Extract range table entries for queries that are not fast path. We can skip fast * path queries because their definition is a single RTE entry, which is a relation, * so the following check doesn't apply for fast-path queries. */ if (!fastPathRouterQuery) { ExtractRangeTableEntryWalker((Node *) originalQuery, &rangeTableList); } bool containsLocalTableDistributedTableJoin = ContainsLocalTableDistributedTableJoin(queryTree->rtable); RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (rangeTableEntry->rtekind == RTE_RELATION) { /* we do not expect to see a view in modify query */ if (rangeTableEntry->relkind == RELKIND_VIEW) { /* * we already check if modify is run on a view in DeferErrorIfModifyView * function call. In addition, since Postgres replaced views in FROM * clause with subqueries, encountering with a view should not be a problem here. */ } else if (rangeTableEntry->relkind == RELKIND_MATVIEW) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "materialized views in " "modify queries are not supported", NULL, NULL); } /* for other kinds of relations, check if it's distributed */ else { if (IsRelationLocalTableOrMatView(rangeTableEntry->relid) && containsLocalTableDistributedTableJoin) { StringInfo errorMessage = makeStringInfo(); char *relationName = get_rel_name(rangeTableEntry->relid); if (IsCitusTable(rangeTableEntry->relid)) { appendStringInfo(errorMessage, "local table %s cannot be joined with these distributed tables", relationName); } else { appendStringInfo(errorMessage, "relation %s is not distributed", relationName); } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorMessage->data, NULL, NULL); } } } else if (rangeTableEntry->rtekind == RTE_VALUES || rangeTableEntry->rtekind == RTE_RESULT ) { /* do nothing, this type is supported */ } else { char *rangeTableEntryErrorDetail = NULL; /* * We support UPDATE, DELETE and MERGE with subqueries and joins unless * they are multi shard queries. */ if (UpdateOrDeleteOrMergeQuery(queryTree)) { continue; } /* * Error out for rangeTableEntries that we do not support. * We do not explicitly specify "in FROM clause" in the error detail * for the features that we do not support at all (SUBQUERY, JOIN). */ if (rangeTableEntry->rtekind == RTE_SUBQUERY) { StringInfo errorHint = makeStringInfo(); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry( distributedTableId); char *partitionColumnName = ColumnToColumnName(distributedTableId, (Node *) cacheEntry->partitionColumn); appendStringInfo(errorHint, "Consider using an equality filter on " "partition column \"%s\" to target a single shard.", partitionColumnName); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "subqueries are not " "supported in modifications across multiple shards", errorHint->data, NULL); } else if (rangeTableEntry->rtekind == RTE_JOIN) { rangeTableEntryErrorDetail = "Joins are not supported in distributed" " modifications."; } else if (rangeTableEntry->rtekind == RTE_FUNCTION) { rangeTableEntryErrorDetail = "Functions must not appear in the FROM" " clause of a distributed modifications."; } else if (rangeTableEntry->rtekind == RTE_CTE) { rangeTableEntryErrorDetail = "Common table expressions are not supported" " in distributed modifications."; } else { rangeTableEntryErrorDetail = "Unrecognized range table entry."; } return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot perform distributed planning for the given " "modifications", rangeTableEntryErrorDetail, NULL); } } if (commandType != CMD_INSERT) { DeferredErrorMessage *errorMessage = NULL; if (multiShardQuery) { errorMessage = MultiShardUpdateDeleteSupported( originalQuery, plannerRestrictionContext); } else { errorMessage = SingleShardUpdateDeleteSupported(originalQuery, plannerRestrictionContext); } if (errorMessage != NULL) { return errorMessage; } } DeferredErrorMessage *CTEWithSearchClauseError = ErrorIfQueryHasCTEWithSearchClause(originalQuery); if (CTEWithSearchClauseError != NULL) { return CTEWithSearchClauseError; } return NULL; } /* * Modify statements on simple updetable views are not supported yet. * Actually, we need the original query (the query before postgres * pg_rewrite_query) to detect if the view sitting in rtable is to * be updated or just to be used in FROM clause. * Hence, tracing the postgres source code, we deduced that postgres * puts the relation to be modified to the first entry of rtable. * If first element of the range table list is a simple updatable * view and this view is not coming from FROM clause (inFromCl = False), * then update is run "on" that view. */ static DeferredErrorMessage * DeferErrorIfModifyView(Query *queryTree) { if (queryTree->rtable != NIL) { RangeTblEntry *firstRangeTableElement = (RangeTblEntry *) linitial( queryTree->rtable); if (firstRangeTableElement->rtekind == RTE_RELATION && firstRangeTableElement->relkind == RELKIND_VIEW && firstRangeTableElement->inFromCl == false) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot modify views when the query contains citus tables", NULL, NULL); } } return NULL; } /* * ErrorIfOnConflictNotSupprted returns an error if an INSERT query has an * unsupported ON CONFLICT clause. In particular, changing the partition * column value or using volatile functions is not allowed. */ DeferredErrorMessage * ErrorIfOnConflictNotSupported(Query *queryTree) { uint32 rangeTableId = 1; ListCell *setTargetCell = NULL; bool specifiesPartitionValue = false; CmdType commandType = queryTree->commandType; if (commandType != CMD_INSERT || queryTree->onConflict == NULL) { return NULL; } Oid distributedTableId = ExtractFirstCitusTableId(queryTree); Var *partitionColumn = PartitionColumn(distributedTableId, rangeTableId); List *onConflictSet = queryTree->onConflict->onConflictSet; Node *arbiterWhere = queryTree->onConflict->arbiterWhere; Node *onConflictWhere = queryTree->onConflict->onConflictWhere; /* * onConflictSet is expanded via expand_targetlist() on the standard planner. * This ends up adding all the columns to the onConflictSet even if the user * does not explicitly state the columns in the query. * * The following loop simply allows "DO UPDATE SET part_col = table.part_col" * types of elements in the target list, which are added by expand_targetlist(). * Any other attempt to update partition column value is forbidden. */ foreach(setTargetCell, onConflictSet) { TargetEntry *setTargetEntry = (TargetEntry *) lfirst(setTargetCell); bool setTargetEntryPartitionColumn = false; /* reference tables do not have partition column */ if (partitionColumn == NULL) { setTargetEntryPartitionColumn = false; } else { Oid resultRelationId = ModifyQueryResultRelationId(queryTree); AttrNumber targetColumnAttrNumber = InvalidAttrNumber; if (setTargetEntry->resname) { targetColumnAttrNumber = get_attnum(resultRelationId, setTargetEntry->resname); if (targetColumnAttrNumber == partitionColumn->varattno) { setTargetEntryPartitionColumn = true; } } } if (setTargetEntryPartitionColumn) { Expr *setExpr = setTargetEntry->expr; if (IsA(setExpr, Var) && ((Var *) setExpr)->varattno == partitionColumn->varattno) { specifiesPartitionValue = false; } else { specifiesPartitionValue = true; } } else { /* * Similarly, allow "DO UPDATE SET col_1 = table.col_1" types of * target list elements. Note that, the following check allows * "DO UPDATE SET col_1 = table.col_2", which is not harmful. */ if (IsA(setTargetEntry->expr, Var)) { continue; } else if (contain_mutable_functions((Node *) setTargetEntry->expr)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in the DO UPDATE SET clause of " "INSERTs on distributed tables must be marked " "IMMUTABLE", NULL, NULL); } } } /* error if either arbiter or on conflict WHERE contains a mutable function */ if (contain_mutable_functions((Node *) arbiterWhere) || contain_mutable_functions((Node *) onConflictWhere)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in the WHERE clause of the " "ON CONFLICT clause of INSERTs on distributed " "tables must be marked IMMUTABLE", NULL, NULL); } if (specifiesPartitionValue) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "modifying the partition value of rows is not " "allowed", NULL, NULL); } return NULL; } /* * MultiShardUpdateDeleteSupported returns the error message if the update/delete is * not pushdownable, otherwise it returns NULL. */ static DeferredErrorMessage * MultiShardUpdateDeleteSupported(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { DeferredErrorMessage *errorMessage = NULL; RangeTblEntry *resultRangeTable = ExtractResultRelationRTE(originalQuery); Oid resultRelationOid = resultRangeTable->relid; if (HasDangerousJoinUsing(originalQuery->rtable, (Node *) originalQuery->jointree)) { errorMessage = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "a join with USING causes an internal naming conflict, use " "ON instead", NULL, NULL); } else if (FindNodeMatchingCheckFunction((Node *) originalQuery, CitusIsVolatileFunction)) { errorMessage = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in UPDATE queries on distributed " "tables must not be VOLATILE", NULL, NULL); } else if (IsCitusTableType(resultRelationOid, REFERENCE_TABLE)) { errorMessage = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "only reference tables may be queried when targeting " "a reference table with multi shard UPDATE/DELETE queries " "with multiple tables ", NULL, NULL); } else { errorMessage = DeferErrorIfUnsupportedSubqueryPushdown( originalQuery, plannerRestrictionContext, true); } return errorMessage; } /* * SingleShardUpdateDeleteSupported returns the error message if the update/delete query is * not routable, otherwise it returns NULL. */ static DeferredErrorMessage * SingleShardUpdateDeleteSupported(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { DeferredErrorMessage *errorMessage = NULL; /* * We currently do not support volatile functions in update/delete statements because * the function evaluation logic does not know how to distinguish volatile functions * (that need to be evaluated per row) from stable functions (that need to be evaluated per query), * and it is also not safe to push the volatile functions down on replicated tables. */ if (FindNodeMatchingCheckFunction((Node *) originalQuery, CitusIsVolatileFunction)) { errorMessage = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "functions used in UPDATE queries on distributed " "tables must not be VOLATILE", NULL, NULL); } return errorMessage; } /* * HasDangerousJoinUsing search jointree for unnamed JOIN USING. Check the * implementation of has_dangerous_join_using in ruleutils. */ bool HasDangerousJoinUsing(List *rtableList, Node *joinTreeNode) { if (IsA(joinTreeNode, RangeTblRef)) { /* nothing to do here */ } else if (IsA(joinTreeNode, FromExpr)) { FromExpr *fromExpr = (FromExpr *) joinTreeNode; ListCell *listCell; foreach(listCell, fromExpr->fromlist) { if (HasDangerousJoinUsing(rtableList, (Node *) lfirst(listCell))) { return true; } } } else if (IsA(joinTreeNode, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) joinTreeNode; /* Is it an unnamed JOIN with USING? */ if (joinExpr->alias == NULL && joinExpr->usingClause) { /* * Yes, so check each join alias var to see if any of them are not * simple references to underlying columns. If so, we have a * dangerous situation and must pick unique aliases. */ RangeTblEntry *joinRTE = rt_fetch(joinExpr->rtindex, rtableList); ListCell *listCell; foreach(listCell, joinRTE->joinaliasvars) { Var *aliasVar = (Var *) lfirst(listCell); if (aliasVar != NULL && !IsA(aliasVar, Var)) { return true; } } } /* Nope, but inspect children */ if (HasDangerousJoinUsing(rtableList, joinExpr->larg)) { return true; } if (HasDangerousJoinUsing(rtableList, joinExpr->rarg)) { return true; } } else { elog(ERROR, "unrecognized node type: %d", (int) nodeTag(joinTreeNode)); } return false; } /* * UpdateOrDeleteOrMergeQuery checks if the given query is an UPDATE or DELETE or * MERGE command. If it is, it returns true otherwise it returns false. */ bool UpdateOrDeleteOrMergeQuery(Query *query) { return (query->commandType == CMD_UPDATE || query->commandType == CMD_DELETE || query->commandType == CMD_MERGE); } /* * IsMergeQuery checks if the given query is a MERGE SQL command. */ bool IsMergeQuery(Query *query) { return (query->commandType == CMD_MERGE); } /* * If the expression contains STABLE functions which accept any parameters derived from a * Var returns true and sets varArgument. * * If the expression contains a CASE or COALESCE which invoke non-IMMUTABLE functions * returns true and sets badCoalesce. * * Assumes the expression contains no VOLATILE functions. * * Var's are allowed, but only if they are passed solely to IMMUTABLE functions * * We special-case CASE/COALESCE because those are evaluated lazily. We could evaluate * CASE/COALESCE expressions which don't reference Vars, or partially evaluate some * which do, but for now we just error out. That makes both the code and user-education * easier. */ bool MasterIrreducibleExpression(Node *expression, bool *varArgument, bool *badCoalesce) { WalkerState data; data.containsVar = data.varArgument = data.badCoalesce = false; bool result = MasterIrreducibleExpressionWalker(expression, &data); *varArgument |= data.varArgument; *badCoalesce |= data.badCoalesce; return result; } static bool MasterIrreducibleExpressionWalker(Node *expression, WalkerState *state) { char volatileFlag = 0; WalkerState childState = { false, false, false }; bool containsDisallowedFunction = false; bool hasVolatileFunction PG_USED_FOR_ASSERTS_ONLY = false; if (expression == NULL) { return false; } if (IsA(expression, CoalesceExpr)) { CoalesceExpr *expr = (CoalesceExpr *) expression; if (contain_mutable_functions((Node *) (expr->args))) { state->badCoalesce = true; return true; } else { /* * There's no need to recurse. Since there are no STABLE functions * varArgument will never be set. */ return false; } } if (IsA(expression, CaseExpr)) { if (contain_mutable_functions(expression)) { state->badCoalesce = true; return true; } return false; } if (IsA(expression, Var)) { state->containsVar = true; return false; } /* * In order for statement replication to give us consistent results it's important * that we either disallow or evaluate on the coordinator anything which has a * volatility category above IMMUTABLE. Newer versions of postgres might add node * types which should be checked in this function. * * Look through contain_mutable_functions_walker or future PG's equivalent for new * node types before bumping this version number to fix compilation; e.g. for any * PostgreSQL after 9.5, see check_functions_in_node. Review * MasterIrreducibleExpressionFunctionChecker for any changes in volatility * permissibility ordering. * * Once you've added them to this check, make sure you also evaluate them in the * executor! */ hasVolatileFunction = check_functions_in_node(expression, MasterIrreducibleExpressionFunctionChecker, &volatileFlag); /* the caller should have already checked for this */ Assert(!hasVolatileFunction); Assert(volatileFlag != PROVOLATILE_VOLATILE); if (volatileFlag == PROVOLATILE_STABLE) { containsDisallowedFunction = expression_tree_walker(expression, MasterIrreducibleExpressionWalker, &childState); if (childState.containsVar) { state->varArgument = true; } state->badCoalesce |= childState.badCoalesce; state->varArgument |= childState.varArgument; return (containsDisallowedFunction || childState.containsVar); } /* keep traversing */ return expression_tree_walker(expression, MasterIrreducibleExpressionWalker, state); } /* * MasterIrreducibleExpressionFunctionChecker returns true if a provided function * oid corresponds to a volatile function. It also updates provided context if * the current volatility flag is more permissive than the provided one. It is * only called from check_functions_in_node as checker function. */ static bool MasterIrreducibleExpressionFunctionChecker(Oid func_id, void *context) { char volatileFlag = func_volatile(func_id); char *volatileContext = (char *) context; if (volatileFlag == PROVOLATILE_VOLATILE || *volatileContext == PROVOLATILE_VOLATILE) { *volatileContext = PROVOLATILE_VOLATILE; } else if (volatileFlag == PROVOLATILE_STABLE || *volatileContext == PROVOLATILE_STABLE) { *volatileContext = PROVOLATILE_STABLE; } else { *volatileContext = PROVOLATILE_IMMUTABLE; } return (volatileFlag == PROVOLATILE_VOLATILE); } /* * TargetEntryChangesValue determines whether the given target entry may * change the value given a column and a join tree. * * The function assumes that the "targetEntry" references given "column" * Var via its "resname" and is used as part of a modify query. This means * that, for example, for an update query, the input "targetEntry" constructs * the following assignment operation as part of the SET clause: * "col_a = expr_a ", where, "col_a" refers to input "column" Var (via * "resname") as per the assumption written above. And we want to understand * if "expr_a" (which is pointed to by targetEntry->expr) refers directly to * the "column" Var, or "expr_a" is a value that is implied to be equal * to "column" Var by the qualifiers of the join tree. If so, we know that * the value of "col_a" effectively cannot be changed by this assignment * operation. */ bool TargetEntryChangesValue(TargetEntry *targetEntry, Var *column, FromExpr *joinTree) { bool isColumnValueChanged = true; Expr *setExpr = targetEntry->expr; if (IsA(setExpr, Var)) { Var *newValue = (Var *) setExpr; if (column->varno == newValue->varno && column->varattno == newValue->varattno) { /* * Target entry is of the form "SET col_a = foo.col_b", * where foo also points to the same range table entry * and col_a and col_b are the same. So, effectively * they're literally referring to the same column. */ isColumnValueChanged = false; } else { List *restrictClauseList = WhereClauseList(joinTree); OpExpr *equalityExpr = MakeOpExpressionExtended(column, (Expr *) newValue, BTEqualStrategyNumber); bool predicateIsImplied = predicate_implied_by(list_make1(equalityExpr), restrictClauseList, false); if (predicateIsImplied) { /* * Target entry is of the form * "SET col_a = foo.col_b WHERE col_a = foo.col_b (AND (...))", * where foo points to a different relation or it points * to the same relation but col_a is not the same column as col_b. */ isColumnValueChanged = false; } } } else if (IsA(setExpr, Const)) { Const *newValue = (Const *) setExpr; List *restrictClauseList = WhereClauseList(joinTree); OpExpr *equalityExpr = MakeOpExpression(column, BTEqualStrategyNumber); Node *rightOp = get_rightop((Expr *) equalityExpr); Assert(rightOp != NULL); Assert(IsA(rightOp, Const)); Const *rightConst = (Const *) rightOp; rightConst->constvalue = newValue->constvalue; rightConst->constisnull = newValue->constisnull; rightConst->constbyval = newValue->constbyval; bool predicateIsImplied = predicate_implied_by(list_make1(equalityExpr), restrictClauseList, false); if (predicateIsImplied) { /* * Target entry is of the form * "SET col_a = const_a WHERE col_a = const_a (AND (...))". */ isColumnValueChanged = false; } } return isColumnValueChanged; } /* * RouterInsertJob builds a Job to represent an insertion performed by the provided * query. For inserts we always defer shard pruning and generating the task list to * the executor. */ static Job * RouterInsertJob(Query *originalQuery) { Assert(originalQuery->commandType == CMD_INSERT); bool isMultiRowInsert = IsMultiRowInsert(originalQuery); if (isMultiRowInsert) { /* add default expressions to RTE_VALUES in multi-row INSERTs */ NormalizeMultiRowInsertTargetList(originalQuery); } Job *job = CreateJob(originalQuery); job->requiresCoordinatorEvaluation = RequiresCoordinatorEvaluation(originalQuery); job->deferredPruning = true; job->partitionKeyValue = ExtractInsertPartitionKeyValue(originalQuery); return job; } /* * CreateJob returns a new Job for the given query. */ static Job * CreateJob(Query *query) { Job *job = CitusMakeNode(Job); job->jobId = UniqueJobId(); job->jobQuery = query; job->taskList = NIL; job->dependentJobList = NIL; job->subqueryPushdown = false; job->requiresCoordinatorEvaluation = false; job->deferredPruning = false; return job; } /* * ErrorIfNoShardsExist throws an error if the given table has no shards. */ static void ErrorIfNoShardsExist(CitusTableCacheEntry *cacheEntry) { int shardCount = cacheEntry->shardIntervalArrayLength; if (shardCount == 0) { Oid distributedTableId = cacheEntry->relationId; char *relationName = get_rel_name(distributedTableId); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find any shards"), errdetail("No shards exist for distributed table \"%s\".", relationName), errhint("Run master_create_worker_shards to create shards " "and try again."))); } } /* * RouterInsertTaskList generates a list of tasks for performing an INSERT on * a distributed table via the router executor. */ List * RouterInsertTaskList(Query *query, bool parametersInQueryResolved, DeferredErrorMessage **planningError) { List *insertTaskList = NIL; Oid distributedTableId = ExtractFirstCitusTableId(query); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); ErrorIfNoShardsExist(cacheEntry); Assert(query->commandType == CMD_INSERT); List *modifyRouteList = BuildRoutesForInsert(query, planningError); if (*planningError != NULL) { return NIL; } ModifyRoute *modifyRoute = NULL; foreach_declared_ptr(modifyRoute, modifyRouteList) { Task *modifyTask = CreateTask(MODIFY_TASK); modifyTask->anchorShardId = modifyRoute->shardId; modifyTask->replicationModel = cacheEntry->replicationModel; modifyTask->rowValuesLists = modifyRoute->rowValuesLists; RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->shardId = modifyRoute->shardId; relationShard->relationId = distributedTableId; modifyTask->relationShardList = list_make1(relationShard); modifyTask->taskPlacementList = ActiveShardPlacementList( modifyRoute->shardId); modifyTask->parametersInQueryStringResolved = parametersInQueryResolved; insertTaskList = lappend(insertTaskList, modifyTask); } return insertTaskList; } /* * CreateTask returns a new Task with the given type. */ static Task * CreateTask(TaskType taskType) { Task *task = CitusMakeNode(Task); task->taskType = taskType; task->jobId = INVALID_JOB_ID; task->taskId = INVALID_TASK_ID; SetTaskQueryString(task, NULL); task->anchorShardId = INVALID_SHARD_ID; task->taskPlacementList = NIL; task->dependentTaskList = NIL; task->partitionId = 0; task->upstreamTaskId = INVALID_TASK_ID; task->shardInterval = NULL; task->assignmentConstrained = false; task->replicationModel = REPLICATION_MODEL_INVALID; task->relationRowLockList = NIL; task->modifyWithSubquery = false; task->partiallyLocalOrRemote = false; task->relationShardList = NIL; return task; } /* * ExtractFirstCitusTableId takes a given query, and finds the relationId * for the first distributed table in that query. If the function cannot find a * distributed table, it returns InvalidOid. * * We only use this function for modifications and fast path queries, which * should have the first distributed table in the top-level rtable. */ Oid ExtractFirstCitusTableId(Query *query) { List *rangeTableList = query->rtable; ListCell *rangeTableCell = NULL; Oid distributedTableId = InvalidOid; foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (IsCitusTable(rangeTableEntry->relid)) { distributedTableId = rangeTableEntry->relid; break; } } return distributedTableId; } /* * RouterJob builds a Job to represent a single shard select/update/delete and * multiple shard update/delete queries. */ Job * RouterJob(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, DeferredErrorMessage **planningError) { uint64 shardId = INVALID_SHARD_ID; List *placementList = NIL; List *relationShardList = NIL; List *prunedShardIntervalListList = NIL; bool isMultiShardModifyQuery = false; Const *partitionKeyValue = NULL; /* router planner should create task even if it doesn't hit a shard at all */ bool replacePrunedQueryWithDummy = true; bool isLocalTableModification = false; /* check if this query requires coordinator evaluation */ bool requiresCoordinatorEvaluation = RequiresCoordinatorEvaluation(originalQuery); FastPathRestrictionContext *fastPathRestrictionContext = plannerRestrictionContext->fastPathRestrictionContext; /* * We prefer to defer shard pruning/task generation to the * execution when the parameter on the distribution key * cannot be resolved. */ if (fastPathRestrictionContext->fastPathRouterQuery && fastPathRestrictionContext->distributionKeyHasParam) { Job *job = CreateJob(originalQuery); job->deferredPruning = true; ereport(DEBUG2, (errmsg("Deferred pruning for a fast-path router " "query"))); return job; } else { (*planningError) = PlanRouterQuery(originalQuery, plannerRestrictionContext, &placementList, &shardId, &relationShardList, &prunedShardIntervalListList, replacePrunedQueryWithDummy, &isMultiShardModifyQuery, &partitionKeyValue, &isLocalTableModification); } if (*planningError) { return NULL; } Job *job = CreateJob(originalQuery); job->partitionKeyValue = partitionKeyValue; if (originalQuery->resultRelation > 0) { RangeTblEntry *updateOrDeleteOrMergeRTE = ExtractResultRelationRTE(originalQuery); if (updateOrDeleteOrMergeRTE->rtekind == RTE_SUBQUERY) { /* * Not generating tasks for MERGE target relation might * result in incorrect behavior as source rows with NOT * MATCHED clause might qualify for insertion. */ if (IsMergeQuery(originalQuery)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Merge command is currently " "unsupported with filters that " "prunes down to zero shards"), errhint("Avoid `WHERE false` clause or " "any equivalent filters that " "could prune down to zero shards"))); } else { /* * If all of the shards are pruned, we replace the * relation RTE into subquery RTE that returns no * results. However, this is not useful for UPDATE * and DELETE queries. Therefore, if we detect a * UPDATE or DELETE RTE with subquery type, we just * set task list to empty and return the job. */ job->taskList = NIL; return job; } } } if (isMultiShardModifyQuery) { job->taskList = QueryPushdownSqlTaskList(originalQuery, job->jobId, plannerRestrictionContext-> relationRestrictionContext, prunedShardIntervalListList, MODIFY_TASK, requiresCoordinatorEvaluation, planningError); if (*planningError) { return NULL; } } else { GenerateSingleShardRouterTaskList(job, relationShardList, placementList, shardId, isLocalTableModification, fastPathRestrictionContext-> delayFastPathPlanning); } job->requiresCoordinatorEvaluation = requiresCoordinatorEvaluation; return job; } /* * CheckAttributesMatch checks if the attributes of the Citus table and the shard * table match. * * It is used to ensure that the shard table has the same schema as the Citus * table before replacing the Citus table OID with the shard table OID in the * parse tree we (Citus planner) recieved from Postgres. */ static bool CheckAttributesMatch(Oid citusTableId, Oid shardTableId) { bool same_schema = false; Relation citusRelation = RelationIdGetRelation(citusTableId); Relation shardRelation = RelationIdGetRelation(shardTableId); if (RelationIsValid(citusRelation) && RelationIsValid(shardRelation)) { TupleDesc citusTupDesc = citusRelation->rd_att; TupleDesc shardTupDesc = shardRelation->rd_att; if (citusTupDesc->natts == shardTupDesc->natts) { /* * Do an attribute-by-attribute comparison. This is borrowed from * the Postgres function equalTupleDescs(), which we cannot use * because the citus table and shard table have different composite * types. */ same_schema = true; for (int i = 0; i < citusTupDesc->natts && same_schema; i++) { Form_pg_attribute attr1 = TupleDescAttr(citusTupDesc, i); Form_pg_attribute attr2 = TupleDescAttr(shardTupDesc, i); if (strcmp(NameStr(attr1->attname), NameStr(attr2->attname)) != 0) { same_schema = false; } if (attr1->atttypid != attr2->atttypid) { same_schema = false; } if (attr1->atttypmod != attr2->atttypmod) { same_schema = false; } if (attr1->attcollation != attr2->attcollation) { same_schema = false; } /* Record types derived from tables could have dropped fields. */ if (attr1->attisdropped != attr2->attisdropped) { same_schema = false; } } } } RelationClose(citusRelation); RelationClose(shardRelation); return same_schema; } /* * CheckAndBuildDelayedFastPathPlan() - if the query being planned is a fast * path query, not marked for deferred pruning and the placement for the task * is not a dummy placement then if the placement is local to this node we can * take a shortcut of replacing the OID of the citus table with the OID of the * shard in the query tree and plan that directly, instead of deparsing the * parse tree to a SQL query on the shard and parsing and planning that in * the local executor. Instead, the local executor can use the plan created * here. */ void CheckAndBuildDelayedFastPathPlan(DistributedPlanningContext *planContext, DistributedPlan *plan) { FastPathRestrictionContext *fastPathContext = planContext->plannerRestrictionContext->fastPathRestrictionContext; if (!fastPathContext->delayFastPathPlanning) { return; } Job *job = plan->workerJob; Assert(job != NULL); if (job->deferredPruning) { /* Execution time pruning => don't know which shard at this point */ planContext->plan = FastPathPlanner(planContext->originalQuery, planContext->query, planContext->boundParams); return; } List *tasks = job->taskList; Assert(list_length(tasks) == 1); Task *task = (Task *) linitial(tasks); List *placements = task->taskPlacementList; int32 localGroupId = GetLocalGroupId(); ShardPlacement *primaryPlacement = (ShardPlacement *) linitial(placements); bool isLocalExecution = !IsDummyPlacement(primaryPlacement) && (primaryPlacement->groupId == localGroupId); bool canBuildLocalPlan = true; if (isLocalExecution) { List *relationShards = task->relationShardList; Assert(list_length(relationShards) == 1); RelationShard *relationShard = (RelationShard *) linitial(relationShards); Assert(relationShard->shardId == primaryPlacement->shardId); /* * Today FastPathRouterQuery() doesn't set delayFastPathPlanning to true for * reference tables. We should be looking at 1 placement, or their replication * factor. */ Assert(list_length(placements) == 1 || list_length(placements) == TableShardReplicationFactor(relationShard->relationId)); canBuildLocalPlan = ConvertToQueryOnShard(planContext->query, relationShard->relationId, relationShard->shardId); if (canBuildLocalPlan) { /* Plan the query with the new shard relation id */ planContext->plan = standard_planner(planContext->query, NULL, planContext->cursorOptions, planContext->boundParams); SetTaskQueryPlan(task, job->jobQuery, planContext->plan); ereport(DEBUG2, (errmsg( "Fast-path router query: created local execution plan " "to avoid deparse and compile of shard query"))); return; } } /* * Either the shard is not local to this node, or it was not safe to replace * the OIDs in the parse tree; in any case we fall back to generating the shard * query and compiling that. */ Assert(!isLocalExecution || (isLocalExecution && !canBuildLocalPlan)); /* Fall back to fast path planner and generating SQL query on the shard */ planContext->plan = FastPathPlanner(planContext->originalQuery, planContext->query, planContext->boundParams); UpdateRelationToShardNames((Node *) job->jobQuery, task->relationShardList); SetTaskQueryIfShouldLazyDeparse(task, job->jobQuery); } /* * ConvertToQueryOnShard() converts the given query on a citus table (identified by * citusTableOid) to a query on a shard (identified by shardId). * * The function assumes that the query is a "fast path" query - it has only one * RangeTblEntry and one RTEPermissionInfo. * * It acquires the same lock on the shard that was acquired on the citus table * by the Postgres parser. It checks that the attribute numbers and metadata of * the shard table and citus table are identical - otherwise it is not safe * to proceed with this shortcut. Assuming the attributes do match, the actual * conversion involves changing the target list entries that reference the * citus table's oid to reference the shard's relation id instead. Finally, * it changes the RangeTblEntry's relid to the shard's relation id and (PG16+) * changes the RTEPermissionInfo's relid to the shard's relation id also. * At this point the Query is ready for the postgres planner. */ static bool ConvertToQueryOnShard(Query *query, Oid citusTableOid, Oid shardId) { Assert(list_length(query->rtable) == 1 #if PG_VERSION_NUM >= PG_VERSION_18 || (list_length(query->rtable) == 2 && query->hasGroupRTE) #endif ); RangeTblEntry *citusTableRte = (RangeTblEntry *) linitial(query->rtable); Assert(citusTableRte->relid == citusTableOid); const char *citusTableName = get_rel_name(citusTableOid); Assert(citusTableName != NULL); /* construct shard relation name */ char *shardRelationName = pstrdup(citusTableName); AppendShardIdToName(&shardRelationName, shardId); /* construct the schema name */ char *schemaName = get_namespace_name(get_rel_namespace(citusTableOid)); /* now construct a range variable for the shard */ RangeVar shardRangeVar = { .relname = shardRelationName, .schemaname = schemaName, .inh = citusTableRte->inh, .relpersistence = RELPERSISTENCE_PERMANENT, }; /* Must apply the same lock to the shard that was applied to the citus table */ Oid shardRelationId = RangeVarGetRelidExtended(&shardRangeVar, citusTableRte->rellockmode, 0, NULL, NULL); /* Verify that the attributes of citus table and shard table match */ if (!CheckAttributesMatch(citusTableOid, shardRelationId)) { /* There is a difference between the attributes of the citus * table and the shard table. This can happen if there is a DROP * COLUMN on the citus table. In this case, we cannot * convert the query to a shard query, so clean up and return. */ UnlockRelationOid(shardRelationId, citusTableRte->rellockmode); ereport(DEBUG2, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "Router planner fast path cannot modify parse tree for local execution: shard table \"%s.%s\" does not match the " "distributed table \"%s.%s\"", schemaName, shardRelationName, schemaName, citusTableName))); pfree(shardRelationName); pfree(schemaName); return false; } /* Change the target list entries that reference the original citus table's relation id */ ListCell *lc = NULL; foreach(lc, query->targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(lc); if (targetEntry->resorigtbl == citusTableOid) { targetEntry->resorigtbl = shardRelationId; } } /* Change the range table entry's oid to that of the shard's */ Assert(shardRelationId != InvalidOid); citusTableRte->relid = shardRelationId; /* Change the range table permission oid to that of the shard's (PG16+) */ Assert(list_length(query->rteperminfos) == 1); RTEPermissionInfo *rtePermInfo = (RTEPermissionInfo *) linitial(query->rteperminfos); rtePermInfo->relid = shardRelationId; return true; } /* * GenerateSingleShardRouterTaskList is a wrapper around other corresponding task * list generation functions specific to single shard selects and modifications. * * The function updates the input job's taskList in-place. */ void GenerateSingleShardRouterTaskList(Job *job, List *relationShardList, List *placementList, uint64 shardId, bool isLocalTableModification, bool delayedFastPath) { Query *originalQuery = job->jobQuery; if (originalQuery->commandType == CMD_SELECT) { SetJobColocationId(job); job->taskList = SingleShardTaskList(originalQuery, job->jobId, relationShardList, placementList, shardId, job->parametersInJobQueryResolved, isLocalTableModification, job->partitionKeyValue, job->colocationId, delayedFastPath); /* * Queries to reference tables, or distributed tables with multiple replica's have * their task placements reordered according to the configured * task_assignment_policy. This is only applicable to select queries as the modify * queries will _always_ be executed on all placements. * * We also ignore queries that are targeting only intermediate results (e.g., no * valid anchorShardId). */ if (shardId != INVALID_SHARD_ID) { ReorderTaskPlacementsByTaskAssignmentPolicy(job, TaskAssignmentPolicy, placementList); } } else if (shardId == INVALID_SHARD_ID && !isLocalTableModification) { /* modification that prunes to 0 shards */ job->taskList = NIL; } else { SetJobColocationId(job); job->taskList = SingleShardTaskList(originalQuery, job->jobId, relationShardList, placementList, shardId, job->parametersInJobQueryResolved, isLocalTableModification, job->partitionKeyValue, job->colocationId, delayedFastPath); } } /* * ReorderTaskPlacementsByTaskAssignmentPolicy applies selective reordering for supported * TaskAssignmentPolicyTypes. * * Supported Types * - TASK_ASSIGNMENT_ROUND_ROBIN round robin schedule queries among placements * * By default it does not reorder the task list, implying a first-replica strategy. */ static void ReorderTaskPlacementsByTaskAssignmentPolicy(Job *job, TaskAssignmentPolicyType taskAssignmentPolicy, List *placementList) { if (taskAssignmentPolicy == TASK_ASSIGNMENT_ROUND_ROBIN) { /* * We hit a single shard on router plans, and there should be only * one task in the task list */ Assert(list_length(job->taskList) == 1); Task *task = (Task *) linitial(job->taskList); /* * For round-robin SELECT queries, we don't want to include the coordinator * because the user is trying to distributed the load across nodes via * round-robin policy. Otherwise, the local execution would prioritize * executing the local tasks and especially for reference tables on the * coordinator this would prevent load balancing across nodes. * * For other worker nodes in Citus MX, we let the local execution to kick-in * even for round-robin policy, that's because we expect the clients to evenly * connect to the worker nodes. */ Assert(ReadOnlyTask(task->taskType)); placementList = RemoveCoordinatorPlacementIfNotSingleNode(placementList); /* reorder the placement list */ List *reorderedPlacementList = RoundRobinReorder(placementList); task->taskPlacementList = reorderedPlacementList; ShardPlacement *primaryPlacement = (ShardPlacement *) linitial( reorderedPlacementList); ereport(DEBUG3, (errmsg("assigned task %u to node %s:%u", task->taskId, primaryPlacement->nodeName, primaryPlacement->nodePort))); } } /* * RemoveCoordinatorPlacementIfNotSingleNode gets a task placement list and returns the list * by removing the placement belonging to the coordinator (if any). * * If the list has a single element or no placements on the coordinator, the list * returned is unmodified. */ List * RemoveCoordinatorPlacementIfNotSingleNode(List *placementList) { ListCell *placementCell = NULL; if (list_length(placementList) < 2) { return placementList; } foreach(placementCell, placementList) { ShardPlacement *placement = (ShardPlacement *) lfirst(placementCell); if (placement->groupId == COORDINATOR_GROUP_ID) { return list_delete_ptr(placementList, placement); } } return placementList; } /* * SingleShardTaskList generates a task for single shard query * and returns it as a list. */ static List * SingleShardTaskList(Query *query, uint64 jobId, List *relationShardList, List *placementList, uint64 shardId, bool parametersInQueryResolved, bool isLocalTableModification, Const *partitionKeyValue, int colocationId, bool delayedFastPath) { TaskType taskType = READ_TASK; char replicationModel = 0; if (query->commandType != CMD_SELECT) { List *rangeTableList = NIL; ExtractRangeTableEntryWalker((Node *) query, &rangeTableList); RangeTblEntry *updateOrDeleteRTE = ExtractResultRelationRTE(query); Assert(updateOrDeleteRTE != NULL); CitusTableCacheEntry *modificationTableCacheEntry = NULL; if (IsCitusTable(updateOrDeleteRTE->relid)) { modificationTableCacheEntry = GetCitusTableCacheEntry( updateOrDeleteRTE->relid); } if (IsCitusTableType(updateOrDeleteRTE->relid, REFERENCE_TABLE) && SelectsFromDistributedTable(rangeTableList, query)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot perform select on a distributed table " "and modify a reference table"))); } taskType = MODIFY_TASK; if (modificationTableCacheEntry) { replicationModel = modificationTableCacheEntry->replicationModel; } } if (taskType == READ_TASK && query->hasModifyingCTE) { /* assume ErrorIfQueryHasUnroutableModifyingCTE checked query already */ CommonTableExpr *cte = NULL; foreach_declared_ptr(cte, query->cteList) { Query *cteQuery = (Query *) cte->ctequery; if (cteQuery->commandType != CMD_SELECT) { RangeTblEntry *updateOrDeleteRTE = ExtractResultRelationRTE(cteQuery); CitusTableCacheEntry *modificationTableCacheEntry = GetCitusTableCacheEntry( updateOrDeleteRTE->relid); taskType = MODIFY_TASK; replicationModel = modificationTableCacheEntry->replicationModel; break; } } } Task *task = CreateTask(taskType); task->isLocalTableModification = isLocalTableModification; List *relationRowLockList = NIL; RowLocksOnRelations((Node *) query, &relationRowLockList); /* * For performance reasons, we skip generating the queryString. For local * execution this is not needed, so we wait until the executor determines * that the query cannot be executed locally. */ task->taskPlacementList = placementList; task->partitionKeyValue = partitionKeyValue; task->colocationId = colocationId; if (!delayedFastPath) { SetTaskQueryIfShouldLazyDeparse(task, query); } task->anchorShardId = shardId; task->jobId = jobId; task->relationShardList = relationShardList; task->relationRowLockList = relationRowLockList; task->replicationModel = replicationModel; task->parametersInQueryStringResolved = parametersInQueryResolved; return list_make1(task); } /* * RowLocksOnRelations forms the list for range table IDs and corresponding * row lock modes. */ static bool RowLocksOnRelations(Node *node, List **relationRowLockList) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; ListCell *rowMarkCell = NULL; foreach(rowMarkCell, query->rowMarks) { RowMarkClause *rowMarkClause = (RowMarkClause *) lfirst(rowMarkCell); RangeTblEntry *rangeTable = rt_fetch(rowMarkClause->rti, query->rtable); Oid relationId = rangeTable->relid; if (IsCitusTable(relationId)) { RelationRowLock *relationRowLock = CitusMakeNode(RelationRowLock); relationRowLock->relationId = relationId; relationRowLock->rowLockStrength = rowMarkClause->strength; *relationRowLockList = lappend(*relationRowLockList, relationRowLock); } } return query_tree_walker(query, RowLocksOnRelations, relationRowLockList, 0); } else { return expression_tree_walker(node, RowLocksOnRelations, relationRowLockList); } } /* * SelectsFromDistributedTable checks if there is a select on a distributed * table by looking into range table entries. */ static bool SelectsFromDistributedTable(List *rangeTableList, Query *query) { ListCell *rangeTableCell = NULL; RangeTblEntry *resultRangeTableEntry = NULL; if (query->resultRelation > 0) { resultRangeTableEntry = ExtractResultRelationRTE(query); } foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->relid == InvalidOid) { continue; } if (rangeTableEntry->relkind == RELKIND_VIEW || rangeTableEntry->relkind == RELKIND_MATVIEW) { /* * Skip over views, which would error out in GetCitusTableCacheEntry. * Distributed tables within (regular) views are already in rangeTableList. */ continue; } CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry( rangeTableEntry->relid); if (IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE) && (resultRangeTableEntry == NULL || resultRangeTableEntry->relid != rangeTableEntry->relid)) { return true; } } return false; } /* * PlanRouterQuery runs router pruning logic for SELECT, UPDATE, DELETE, and * MERGE queries. If there are shards present and query is routable, all RTEs * have been updated to point to the relevant shards in the originalQuery. Also, * placementList is filled with the list of worker nodes that has all the * required shard placements for the query execution. anchorShardId is set to * the first pruned shardId of the given query. Finally, relationShardList is * filled with the list of relation-to-shard mappings for the query. * * If the given query is not routable, it fills planningError with the related * DeferredErrorMessage. The caller can check this error message to see if query * is routable or not. * * Note: If the query prunes down to 0 shards due to filters (e.g. WHERE false), * or the query has only read_intermediate_result calls (no relations left after * recursively planning CTEs and subqueries), then it will be assigned to an * arbitrary worker node in a round-robin fashion. * * Relations that prune down to 0 shards are replaced by subqueries returning * 0 values in UpdateRelationToShardNames. */ DeferredErrorMessage * PlanRouterQuery(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, List **placementList, uint64 *anchorShardId, List **relationShardList, List **prunedShardIntervalListList, bool replacePrunedQueryWithDummy, bool *multiShardModifyQuery, Const **partitionValueConst, bool *isLocalTableModification) { bool isMultiShardQuery = false; DeferredErrorMessage *planningError = NULL; bool shardsPresent = false; CmdType commandType = originalQuery->commandType; Oid targetRelationId = InvalidOid; bool fastPathRouterQuery = plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery; *placementList = NIL; /* * When FastPathRouterQuery() returns true, we know that standard_planner() has * not been called. Thus, restriction information is not avaliable and we do the * shard pruning based on the distribution column in the quals of the query. */ if (fastPathRouterQuery) { Const *distributionKeyValue = plannerRestrictionContext->fastPathRestrictionContext->distributionKeyValue; List *shardIntervalList = TargetShardIntervalForFastPathQuery(originalQuery, &isMultiShardQuery, distributionKeyValue, partitionValueConst); Assert(!isMultiShardQuery); *prunedShardIntervalListList = shardIntervalList; ereport(DEBUG2, (errmsg("Distributed planning for a fast-path router " "query"))); } else { *prunedShardIntervalListList = TargetShardIntervalsForRestrictInfo(plannerRestrictionContext-> relationRestrictionContext, &isMultiShardQuery, partitionValueConst); } if (isMultiShardQuery) { /* * If multiShardQuery is true and it is a type of SELECT query, then * return deferred error. We do not support multi-shard SELECT queries * with this code path. */ if (commandType == CMD_SELECT) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner cannot handle multi-shard select queries", NULL, NULL); } Assert(UpdateOrDeleteOrMergeQuery(originalQuery)); if (!IsMergeQuery(originalQuery)) { planningError = ModifyQuerySupported(originalQuery, originalQuery, isMultiShardQuery, plannerRestrictionContext); } if (planningError != NULL) { return planningError; } else { *multiShardModifyQuery = true; return planningError; } } *relationShardList = RelationShardListForShardIntervalList(*prunedShardIntervalListList, &shardsPresent); if (!EnableNonColocatedRouterQueryPushdown && !AllShardsColocated(*relationShardList)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "router planner does not support queries that " "reference non-colocated distributed tables", NULL, NULL); } if (!shardsPresent && !replacePrunedQueryWithDummy) { /* * For INSERT ... SELECT, this query could be still a valid for some other target * shard intervals. Thus, we should return empty list if there aren't any matching * workers, so that the caller can decide what to do with this task. */ return NULL; } /* * We bail out if there are RTEs that prune multiple shards above, but * there can also be multiple RTEs that reference the same relation. */ if (RelationPrunesToMultipleShards(*relationShardList)) { planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot run command which targets " "multiple shards", NULL, NULL); return planningError; } /* we need anchor shard id for select queries with router planner */ uint64 shardId = GetAnchorShardId(*prunedShardIntervalListList); /* both Postgres tables and materialized tables are locally avaliable */ RTEListProperties *rteProperties = GetRTEListPropertiesForQuery(originalQuery); if (isLocalTableModification) { *isLocalTableModification = IsLocalTableModification(targetRelationId, originalQuery, shardId, rteProperties); } bool hasPostgresLocalRelation = rteProperties->hasPostgresLocalTable || rteProperties->hasMaterializedView; List *taskPlacementList = CreateTaskPlacementListForShardIntervals(*prunedShardIntervalListList, shardsPresent, replacePrunedQueryWithDummy, hasPostgresLocalRelation); if (taskPlacementList == NIL) { planningError = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "found no worker with all shard placements", NULL, NULL); return planningError; } /* * If this is an UPDATE or DELETE query which requires coordinator evaluation, * don't try update shard names, and postpone that to execution phase. Also, if * this is a delayed fast path query, we don't update the shard names * either, as the shard names will be updated in the fast path query planner. */ bool isUpdateOrDelete = UpdateOrDeleteOrMergeQuery(originalQuery); bool delayedFastPath = plannerRestrictionContext->fastPathRestrictionContext->delayFastPathPlanning; if (!(isUpdateOrDelete && RequiresCoordinatorEvaluation(originalQuery)) && !delayedFastPath) { UpdateRelationToShardNames((Node *) originalQuery, *relationShardList); } *multiShardModifyQuery = false; *placementList = taskPlacementList; *anchorShardId = shardId; return planningError; } /* * AllShardsColocated returns true if all the shards in the given relationShardList * have colocated tables and are on the same shard index. */ static bool AllShardsColocated(List *relationShardList) { RelationShard *relationShard = NULL; int shardIndex = -1; int colocationId = -1; CitusTableType tableType = ANY_CITUS_TABLE_TYPE; foreach_declared_ptr(relationShard, relationShardList) { Oid relationId = relationShard->relationId; uint64 shardId = relationShard->shardId; if (shardId == INVALID_SHARD_ID) { /* intermediate results are always colocated, so ignore */ continue; } CitusTableCacheEntry *tableEntry = LookupCitusTableCacheEntry(relationId); if (tableEntry == NULL) { /* local tables never colocated */ return false; } CitusTableType currentTableType = GetCitusTableType(tableEntry); if (currentTableType == REFERENCE_TABLE) { /* * Reference tables are always colocated so it is * safe to skip them. */ continue; } else if (IsCitusTableTypeCacheEntry(tableEntry, DISTRIBUTED_TABLE)) { if (tableType == ANY_CITUS_TABLE_TYPE) { tableType = currentTableType; } else if (tableType != currentTableType) { /* * We cannot qualify different types of distributed tables * as colocated. */ return false; } if (currentTableType == RANGE_DISTRIBUTED || currentTableType == APPEND_DISTRIBUTED) { /* we do not have further strict colocation checks */ continue; } } int currentColocationId = TableColocationId(relationId); if (colocationId == -1) { colocationId = currentColocationId; } else if (colocationId != currentColocationId) { return false; } int currentIndex = ShardIndex(LoadShardInterval(shardId)); if (shardIndex == -1) { shardIndex = currentIndex; } else if (shardIndex != currentIndex) { return false; } } return true; } /* * ContainsOnlyLocalOrReferenceTables returns true if there are no distributed * tables in the query. In other words, the query might reference only local * tables and/or reference tables, but no fully distributed tables. */ bool ContainsOnlyLocalOrReferenceTables(RTEListProperties *rteProperties) { /* If hasDistributedTable is false, then all tables are either local or reference. */ return !rteProperties->hasDistributedTable; } /* * CreateTaskPlacementListForShardIntervals returns a list of shard placements * on which it can access all shards in shardIntervalListList, which contains * a list of shards for each relation in the query. * * If the query contains a local table then hasLocalRelation should be set to * true. In that case, CreateTaskPlacementListForShardIntervals only returns * a placement for the local node or an empty list if the shards cannot be * accessed locally. * * If generateDummyPlacement is true and there are no shards that need to be * accessed to answer the query (shardsPresent is false), then a single * placement is returned that is either local or follows a round-robin policy. * A typical example is a router query that only reads an intermediate result. * This will happen on the coordinator, unless the user wants to balance the * load by setting the citus.task_assignment_policy. */ List * CreateTaskPlacementListForShardIntervals(List *shardIntervalListList, bool shardsPresent, bool generateDummyPlacement, bool hasLocalRelation) { List *placementList = NIL; if (shardsPresent) { /* * Determine the workers that have all shard placements, if any. */ List *shardPlacementList = PlacementsForWorkersContainingAllShards(shardIntervalListList); if (hasLocalRelation) { ShardPlacement *taskPlacement = NULL; /* * If there is a local table, we only allow the local placement to * be used. If there is none, we disallow the query. */ foreach_declared_ptr(taskPlacement, shardPlacementList) { if (taskPlacement->groupId == GetLocalGroupId()) { placementList = lappend(placementList, taskPlacement); } } } else { placementList = shardPlacementList; } } else if (generateDummyPlacement) { ShardPlacement *dummyPlacement = CreateDummyPlacement(hasLocalRelation); placementList = list_make1(dummyPlacement); } return placementList; } /* * CreateLocalDummyPlacement creates a dummy placement for the local node that * can be used for queries that don't involve any shards. The typical examples * are: * (a) queries that consist of only intermediate results * (b) queries that hit zero shards (... WHERE false;) */ static ShardPlacement * CreateLocalDummyPlacement() { ShardPlacement *dummyPlacement = CitusMakeNode(ShardPlacement); dummyPlacement->nodeId = LOCAL_NODE_ID; dummyPlacement->nodeName = LocalHostName; dummyPlacement->nodePort = PostPortNumber; dummyPlacement->groupId = GetLocalGroupId(); return dummyPlacement; } /* * CreateDummyPlacement creates a dummy placement that can be used for queries * that don't involve any shards. The typical examples are: * (a) queries that consist of only intermediate results * (b) queries that hit zero shards (... WHERE false;) * * If round robin policy is set, the placement could be on any node in pg_dist_node. * Else, the local node is set for the placement. * * Queries can also involve local tables. In that case we always use the local * node. */ static ShardPlacement * CreateDummyPlacement(bool hasLocalRelation) { static uint32 zeroShardQueryRoundRobin = 0; if (TaskAssignmentPolicy != TASK_ASSIGNMENT_ROUND_ROBIN || hasLocalRelation) { return CreateLocalDummyPlacement(); } List *workerNodeList = ActiveReadableNonCoordinatorNodeList(); if (workerNodeList == NIL) { /* * We want to round-robin over the workers, but there are no workers. * To make sure the query can still succeed we fall back to returning * a local dummy placement. */ return CreateLocalDummyPlacement(); } int workerNodeCount = list_length(workerNodeList); int workerNodeIndex = zeroShardQueryRoundRobin % workerNodeCount; WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList, workerNodeIndex); ShardPlacement *dummyPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(dummyPlacement, workerNode); zeroShardQueryRoundRobin++; return dummyPlacement; } /* * RelationShardListForShardIntervalList is a utility function which gets a list of * shardInterval, and returns a list of RelationShard. */ List * RelationShardListForShardIntervalList(List *shardIntervalList, bool *shardsPresent) { List *relationShardList = NIL; ListCell *shardIntervalListCell = NULL; foreach(shardIntervalListCell, shardIntervalList) { List *prunedShardIntervalList = (List *) lfirst(shardIntervalListCell); /* no shard is present or all shards are pruned out case will be handled later */ if (prunedShardIntervalList == NIL) { continue; } *shardsPresent = true; ListCell *shardIntervalCell = NULL; foreach(shardIntervalCell, prunedShardIntervalList) { ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell); RelationShard *relationShard = CitusMakeNode(RelationShard); relationShard->relationId = shardInterval->relationId; relationShard->shardId = shardInterval->shardId; relationShardList = lappend(relationShardList, relationShard); } } return relationShardList; } /* * GetAnchorShardId returns the anchor shard id given relation shard list. * The desired anchor shard is found as follows: * * - Return the first distributed table shard id in the relationShardList if * there is any. * - Return a random reference table shard id if all the shards belong to * reference tables * - Return INVALID_SHARD_ID on empty lists */ uint64 GetAnchorShardId(List *prunedShardIntervalListList) { ListCell *prunedShardIntervalListCell = NULL; uint64 referenceShardId = INVALID_SHARD_ID; foreach(prunedShardIntervalListCell, prunedShardIntervalListList) { List *prunedShardIntervalList = (List *) lfirst(prunedShardIntervalListCell); /* no shard is present or all shards are pruned out case will be handled later */ if (prunedShardIntervalList == NIL) { continue; } ShardInterval *shardInterval = linitial(prunedShardIntervalList); if (ReferenceTableShardId(shardInterval->shardId)) { referenceShardId = shardInterval->shardId; } else { return shardInterval->shardId; } } return referenceShardId; } /* * TargetShardIntervalForFastPathQuery gets a query which is in * the form defined by FastPathRouterQuery() and returns exactly * one list of one shard interval (see FastPathRouterQuery() * for the detail). * * If the caller requested the distributionKey value that this function * yields, set outputPartitionValueConst. */ List * TargetShardIntervalForFastPathQuery(Query *query, bool *isMultiShardQuery, Const *inputDistributionKeyValue, Const **outputPartitionValueConst) { Oid relationId = ExtractFirstCitusTableId(query); if (!HasDistributionKey(relationId)) { /* we don't need to do shard pruning for single shard tables */ return list_make1(LoadShardIntervalList(relationId)); } if (inputDistributionKeyValue && !inputDistributionKeyValue->constisnull) { CitusTableCacheEntry *cache = GetCitusTableCacheEntry(relationId); Var *distributionKey = cache->partitionColumn; /* * We currently don't allow implicitly coerced values to be handled by fast- * path planner. Still, let's be defensive for any future changes. */ if (inputDistributionKeyValue->consttype != distributionKey->vartype) { bool missingOk = false; inputDistributionKeyValue = TransformPartitionRestrictionValue(distributionKey, inputDistributionKeyValue, missingOk); } ShardInterval *cachedShardInterval = FindShardInterval(inputDistributionKeyValue->constvalue, cache); if (cachedShardInterval == NULL) { ereport(ERROR, (errmsg("could not find shardinterval to which to send " "the query"))); } if (outputPartitionValueConst != NULL) { /* set the outgoing partition column value if requested */ *outputPartitionValueConst = inputDistributionKeyValue; } ShardInterval *shardInterval = CopyShardInterval(cachedShardInterval); List *shardIntervalList = list_make1(shardInterval); return list_make1(shardIntervalList); } Node *quals = query->jointree->quals; int relationIndex = 1; /* * We couldn't do the shard pruning based on inputDistributionKeyValue as it might * be passed as NULL. Still, we can search the quals for distribution key. */ Const *distributionKeyValueInQuals = NULL; List *prunedShardIntervalList = PruneShards(relationId, relationIndex, make_ands_implicit((Expr *) quals), &distributionKeyValueInQuals); if (!distributionKeyValueInQuals || distributionKeyValueInQuals->constisnull) { /* * If the distribution key equals to NULL, we prefer to treat it as a zero shard * query as it cannot return any rows. */ return NIL; } /* we're only expecting single shard from a single table */ if (list_length(prunedShardIntervalList) > 1) { *isMultiShardQuery = true; } else if (list_length(prunedShardIntervalList) == 1 && outputPartitionValueConst != NULL) { /* set the outgoing partition column value if requested */ *outputPartitionValueConst = distributionKeyValueInQuals; } return list_make1(prunedShardIntervalList); } /* * TargetShardIntervalsForRestrictInfo performs shard pruning for all referenced * relations in the relation restriction context and returns list of shards per * relation. Shard pruning is done based on provided restriction context per relation. * The function sets multiShardQuery to true if any of the relations pruned down to * more than one active shard. It also records pruned shard intervals in relation * restriction context to be used later on. Some queries may have contradiction * clauses like 'and false' or 'and 1=0', such queries are treated as if all of * the shards of joining relations are pruned out. */ List * TargetShardIntervalsForRestrictInfo(RelationRestrictionContext *restrictionContext, bool *multiShardQuery, Const **partitionValueConst) { List *prunedShardIntervalListList = NIL; ListCell *restrictionCell = NULL; bool multiplePartitionValuesExist = false; Const *queryPartitionValueConst = NULL; Assert(restrictionContext != NULL); foreach(restrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(restrictionCell); Oid relationId = relationRestriction->relationId; if (!IsCitusTable(relationId)) { /* ignore local tables for shard pruning purposes */ continue; } Index tableId = relationRestriction->index; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); int shardCount = cacheEntry->shardIntervalArrayLength; List *baseRestrictionList = relationRestriction->relOptInfo->baserestrictinfo; List *restrictClauseList = get_all_actual_clauses(baseRestrictionList); List *prunedShardIntervalList = NIL; /* * Queries may have contradiction clauses like 'false', or '1=0' in * their filters. Such queries would have pseudo constant 'false' * inside relOptInfo->joininfo list. We treat such cases as if all * shards of the table are pruned out. */ bool joinFalseQuery = JoinConditionIsOnFalse( relationRestriction->relOptInfo->joininfo); if (!joinFalseQuery && shardCount > 0) { Const *restrictionPartitionValueConst = NULL; prunedShardIntervalList = PruneShards(relationId, tableId, restrictClauseList, &restrictionPartitionValueConst); if (list_length(prunedShardIntervalList) > 1) { (*multiShardQuery) = true; } if (restrictionPartitionValueConst != NULL && queryPartitionValueConst == NULL) { queryPartitionValueConst = restrictionPartitionValueConst; } else if (restrictionPartitionValueConst != NULL && !equal(queryPartitionValueConst, restrictionPartitionValueConst)) { multiplePartitionValuesExist = true; } } prunedShardIntervalListList = lappend(prunedShardIntervalListList, prunedShardIntervalList); } /* * Different restrictions might have different partition columns. * We report partition column value if there is only one. */ if (multiplePartitionValuesExist) { queryPartitionValueConst = NULL; } /* set the outgoing partition column value if requested */ if (partitionValueConst != NULL) { *partitionValueConst = queryPartitionValueConst; } return prunedShardIntervalListList; } /* * JoinConditionIsOnFalse returns true for queries that * have contradiction clauses like 'false', or '1=0' in * their filters. Such queries would have pseudo constant 'false' * inside joininfo list. */ bool JoinConditionIsOnFalse(List *joinInfoList) { List *pseudoJoinRestrictionList = extract_actual_clauses(joinInfoList, true); bool joinFalseQuery = ContainsFalseClause(pseudoJoinRestrictionList); return joinFalseQuery; } /* * RelationPrunesToMultipleShards returns true if the given list of * relation-to-shard mappings contains at least two mappings with * the same relation, but different shards. */ static bool RelationPrunesToMultipleShards(List *relationShardList) { ListCell *relationShardCell = NULL; RelationShard *previousRelationShard = NULL; relationShardList = SortList(relationShardList, CompareRelationShards); foreach(relationShardCell, relationShardList) { RelationShard *relationShard = (RelationShard *) lfirst(relationShardCell); if (previousRelationShard != NULL && relationShard->relationId == previousRelationShard->relationId && relationShard->shardId != previousRelationShard->shardId) { return true; } previousRelationShard = relationShard; } return false; } /* * PlacementsForWorkersContainingAllShards returns list of shard placements for workers * that contain all shard intervals in the given list of shard interval lists. */ List * PlacementsForWorkersContainingAllShards(List *shardIntervalListList) { bool firstShard = true; List *currentPlacementList = NIL; List *shardIntervalList = NIL; foreach_declared_ptr(shardIntervalList, shardIntervalListList) { if (shardIntervalList == NIL) { continue; } Assert(list_length(shardIntervalList) == 1); ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); uint64 shardId = shardInterval->shardId; /* retrieve all active shard placements for this shard */ List *newPlacementList = ActiveShardPlacementList(shardId); if (firstShard) { firstShard = false; currentPlacementList = newPlacementList; } else { /* keep placements that still exists for this shard */ currentPlacementList = IntersectPlacementList(currentPlacementList, newPlacementList); } /* * Bail out if placement list becomes empty. This means there is no worker * containing all shards referenced by the query, hence we can not forward * this query directly to any worker. */ if (currentPlacementList == NIL) { break; } } return currentPlacementList; } /* * BuildRoutesForInsert returns a list of ModifyRoute objects for an INSERT * query or an empty list if the partition column value is defined as an ex- * pression that still needs to be evaluated. If any partition column value * falls within 0 or multiple (overlapping) shards, the planning error is set. * * Multi-row INSERTs are handled by grouping their rows by target shard. These * groups are returned in ascending order by shard id, ready for later deparse * to shard-specific SQL. */ static List * BuildRoutesForInsert(Query *query, DeferredErrorMessage **planningError) { Oid distributedTableId = ExtractFirstCitusTableId(query); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); List *modifyRouteList = NIL; ListCell *insertValuesCell = NULL; Assert(query->commandType == CMD_INSERT); /* tables that don't have distribution column can only have one shard */ if (!HasDistributionKeyCacheEntry(cacheEntry)) { List *shardIntervalList = LoadShardIntervalList(distributedTableId); int shardCount = list_length(shardIntervalList); if (shardCount != 1) { if (IsCitusTableTypeCacheEntry(cacheEntry, REFERENCE_TABLE)) { ereport(ERROR, (errmsg("reference table cannot have %d shards", shardCount))); } else if (IsCitusTableTypeCacheEntry(cacheEntry, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("local table cannot have %d shards", shardCount))); } else if (IsCitusTableTypeCacheEntry(cacheEntry, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errmsg("distributed tables having a null shard key " "cannot have %d shards", shardCount))); } } ShardInterval *shardInterval = linitial(shardIntervalList); ModifyRoute *modifyRoute = palloc(sizeof(ModifyRoute)); modifyRoute->shardId = shardInterval->shardId; RangeTblEntry *valuesRTE = ExtractDistributedInsertValuesRTE(query); if (valuesRTE != NULL) { /* add the values list for a multi-row INSERT */ modifyRoute->rowValuesLists = valuesRTE->values_lists; } else { modifyRoute->rowValuesLists = NIL; } modifyRouteList = lappend(modifyRouteList, modifyRoute); return modifyRouteList; } Var *partitionColumn = cacheEntry->partitionColumn; /* get full list of insert values and iterate over them to prune */ List *insertValuesList = ExtractInsertValuesList(query, partitionColumn); foreach(insertValuesCell, insertValuesList) { InsertValues *insertValues = (InsertValues *) lfirst(insertValuesCell); List *prunedShardIntervalList = NIL; Node *partitionValueExpr = (Node *) insertValues->partitionValueExpr; /* * We only support constant partition values at this point. Sometimes * they are wrappend in an implicit coercion though. Most notably * FuncExpr coercions for casts created with CREATE CAST ... WITH * FUNCTION .. AS IMPLICIT. To support this first we strip them here. * Then we do the coercion manually below using * TransformPartitionRestrictionValue, if the types are not the same. * * NOTE: eval_const_expressions below would do some of these removals * too, but it's unclear if it would do all of them. It is possible * that there are no cases where this strip_implicit_coercions call is * really necessary at all, but currently that's hard to rule out. * So to be on the safe side we call strip_implicit_coercions too, to * be sure we support as much as possible. */ partitionValueExpr = strip_implicit_coercions(partitionValueExpr); /* * By evaluating constant expressions an expression such as 2 + 4 * will become const 6. That way we can use them as a partition column * value. Normally the planner evaluates constant expressions, but we * may be working on the original query tree here. So we do it here * explicitely before checking that the partition value is a const. * * NOTE: We do not use expression_planner here, since all it does * apart from calling eval_const_expressions is call fix_opfuncids. * This is not needed here, since it's a no-op for T_Const nodes and we * error out below in all other cases. */ partitionValueExpr = eval_const_expressions(NULL, partitionValueExpr); if (!IsA(partitionValueExpr, Const)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("failed to evaluate partition key in insert"), errhint("try using constant values for partition column"))); } Const *partitionValueConst = (Const *) partitionValueExpr; if (partitionValueConst->constisnull) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("cannot perform an INSERT with NULL in the partition " "column"))); } /* actually do the coercions that we skipped before, if fails throw an * error */ if (partitionValueConst->consttype != partitionColumn->vartype) { bool missingOk = false; partitionValueConst = TransformPartitionRestrictionValue(partitionColumn, partitionValueConst, missingOk); } if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED)) { Datum partitionValue = partitionValueConst->constvalue; ShardInterval *shardInterval = FindShardInterval(partitionValue, cacheEntry); if (shardInterval != NULL) { prunedShardIntervalList = list_make1(shardInterval); } } else { Index tableId = 1; OpExpr *equalityExpr = MakeOpExpression(partitionColumn, BTEqualStrategyNumber); Node *rightOp = get_rightop((Expr *) equalityExpr); Assert(rightOp != NULL); Assert(IsA(rightOp, Const)); Const *rightConst = (Const *) rightOp; rightConst->constvalue = partitionValueConst->constvalue; rightConst->constisnull = partitionValueConst->constisnull; rightConst->constbyval = partitionValueConst->constbyval; List *restrictClauseList = list_make1(equalityExpr); prunedShardIntervalList = PruneShards(distributedTableId, tableId, restrictClauseList, NULL); } int prunedShardIntervalCount = list_length(prunedShardIntervalList); if (prunedShardIntervalCount != 1) { char *partitionKeyString = cacheEntry->partitionKeyString; char *partitionColumnName = ColumnToColumnName(distributedTableId, stringToNode(partitionKeyString)); StringInfo errorMessage = makeStringInfo(); StringInfo errorHint = makeStringInfo(); const char *targetCountType = NULL; if (prunedShardIntervalCount == 0) { targetCountType = "no"; } else { targetCountType = "multiple"; } if (prunedShardIntervalCount == 0) { appendStringInfo(errorHint, "Make sure you have created a shard which " "can receive this partition column value."); } else { appendStringInfo(errorHint, "Make sure the value for partition column " "\"%s\" falls into a single shard.", partitionColumnName); } appendStringInfo(errorMessage, "cannot run INSERT command which targets %s " "shards", targetCountType); (*planningError) = DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorMessage->data, NULL, errorHint->data); return NIL; } ShardInterval *targetShard = (ShardInterval *) linitial(prunedShardIntervalList); insertValues->shardId = targetShard->shardId; } modifyRouteList = GroupInsertValuesByShardId(insertValuesList); return modifyRouteList; } /* * IsMultiRowInsert returns whether the given query is a multi-row INSERT. * * It does this by determining whether the query is an INSERT that has an * RTE_VALUES. Single-row INSERTs will have their RTE_VALUES optimised away * in transformInsertStmt, and instead use the target list. */ bool IsMultiRowInsert(Query *query) { return ExtractDistributedInsertValuesRTE(query) != NULL; } /* * ExtractDistributedInsertValuesRTE does precisely that. If the provided * query is not an INSERT, or if the INSERT does not have a VALUES RTE * (i.e. it is not a multi-row INSERT), this function returns NULL. * If all those conditions are met, an RTE representing the multiple values * of a multi-row INSERT is returned. */ RangeTblEntry * ExtractDistributedInsertValuesRTE(Query *query) { ListCell *rteCell = NULL; if (query->commandType != CMD_INSERT) { return NULL; } foreach(rteCell, query->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(rteCell); if (rte->rtekind == RTE_VALUES) { return rte; } } return NULL; } /* * NormalizeMultiRowInsertTargetList ensures all elements of multi-row INSERT target * lists are Vars. In multi-row INSERTs, most target list entries contain a Var * expression pointing to a position within the values_lists field of a VALUES * RTE, but non-NULL default columns are handled differently. Instead of adding * the default expression to each row, a single expression encoding the DEFAULT * appears in the target list. For consistency, we move these expressions into * values lists and replace them with an appropriately constructed Var. */ static void NormalizeMultiRowInsertTargetList(Query *query) { ListCell *valuesListCell = NULL; ListCell *targetEntryCell = NULL; int targetEntryNo = 0; RangeTblEntry *valuesRTE = ExtractDistributedInsertValuesRTE(query); if (valuesRTE == NULL) { return; } foreach(valuesListCell, valuesRTE->values_lists) { List *valuesList = (List *) lfirst(valuesListCell); Expr **valuesArray = (Expr **) PointerArrayFromList(valuesList); List *expandedValuesList = NIL; foreach(targetEntryCell, query->targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Expr *targetExpr = targetEntry->expr; if (IsA(targetExpr, Var)) { /* expression from the VALUES section */ Var *targetListVar = (Var *) targetExpr; targetExpr = valuesArray[targetListVar->varattno - 1]; } else { /* copy the column's default expression */ targetExpr = copyObject(targetExpr); } expandedValuesList = lappend(expandedValuesList, targetExpr); } SetListCellPtr(valuesListCell, (void *) expandedValuesList); } /* reset coltypes, coltypmods, colcollations and rebuild them below */ valuesRTE->coltypes = NIL; valuesRTE->coltypmods = NIL; valuesRTE->colcollations = NIL; foreach(targetEntryCell, query->targetList) { TargetEntry *targetEntry = lfirst(targetEntryCell); Node *targetExprNode = (Node *) targetEntry->expr; /* RTE_VALUES comes 2nd, after destination table */ Index valuesVarno = 2; targetEntryNo++; Oid targetType = exprType(targetExprNode); int32 targetTypmod = exprTypmod(targetExprNode); Oid targetColl = exprCollation(targetExprNode); valuesRTE->coltypes = lappend_oid(valuesRTE->coltypes, targetType); valuesRTE->coltypmods = lappend_int(valuesRTE->coltypmods, targetTypmod); valuesRTE->colcollations = lappend_oid(valuesRTE->colcollations, targetColl); if (IsA(targetExprNode, Var)) { Var *targetVar = (Var *) targetExprNode; targetVar->varattno = targetEntryNo; continue; } /* replace the original expression with a Var referencing values_lists */ Var *syntheticVar = makeVar(valuesVarno, targetEntryNo, targetType, targetTypmod, targetColl, 0); targetEntry->expr = (Expr *) syntheticVar; /* * Postgres appends a dummy column reference into valuesRTE->eref->colnames * list in addRangeTableEntryForValues for each column specified in VALUES * clause. Now that we replaced DEFAULT column with a synthetic Var, we also * need to add a dummy column reference for that column. */ AppendNextDummyColReference(valuesRTE->eref); } } /* * AppendNextDummyColReference appends a new dummy column reference to colnames * list of given Alias object. */ static void AppendNextDummyColReference(Alias *expendedReferenceNames) { int existingColReferences = list_length(expendedReferenceNames->colnames); int nextColReferenceId = existingColReferences + 1; String *missingColumnString = MakeDummyColumnString(nextColReferenceId); expendedReferenceNames->colnames = lappend(expendedReferenceNames->colnames, missingColumnString); } /* * MakeDummyColumnString returns a String (Value) object by appending given * integer to end of the "column" string. */ static String * MakeDummyColumnString(int dummyColumnId) { StringInfo dummyColumnStringInfo = makeStringInfo(); appendStringInfo(dummyColumnStringInfo, "column%d", dummyColumnId); String *dummyColumnString = makeString(dummyColumnStringInfo->data); return dummyColumnString; } /* * IntersectPlacementList performs placement pruning based on matching on * nodeName:nodePort fields of shard placement data. We start pruning from all * placements of the first relation's shard. Then for each relation's shard, we * compute intersection of the new shards placement with existing placement list. * This operation could have been done using other methods, but since we do not * expect very high replication factor, iterating over a list and making string * comparisons should be sufficient. */ List * IntersectPlacementList(List *lhsPlacementList, List *rhsPlacementList) { ListCell *lhsPlacementCell = NULL; List *placementList = NIL; /* Keep existing placement in the list if it is also present in new placement list */ foreach(lhsPlacementCell, lhsPlacementList) { ShardPlacement *lhsPlacement = (ShardPlacement *) lfirst(lhsPlacementCell); ListCell *rhsPlacementCell = NULL; foreach(rhsPlacementCell, rhsPlacementList) { ShardPlacement *rhsPlacement = (ShardPlacement *) lfirst(rhsPlacementCell); if (rhsPlacement->nodePort == lhsPlacement->nodePort && strncmp(rhsPlacement->nodeName, lhsPlacement->nodeName, WORKER_LENGTH) == 0) { placementList = lappend(placementList, rhsPlacement); /* * We don't need to add the same placement over and over again. This * could happen if both placements of a shard appear on the same node. */ break; } } } return placementList; } /* * GroupInsertValuesByShardId takes care of grouping the rows from a multi-row * INSERT by target shard. At this point, all pruning has taken place and we * need only to build sets of rows for each destination. This is done by a * simple sort (by shard identifier) and gather step. The sort has the side- * effect of getting things in ascending order to avoid unnecessary deadlocks * during Task execution. */ static List * GroupInsertValuesByShardId(List *insertValuesList) { ModifyRoute *route = NULL; ListCell *insertValuesCell = NULL; List *modifyRouteList = NIL; insertValuesList = SortList(insertValuesList, CompareInsertValuesByShardId); foreach(insertValuesCell, insertValuesList) { InsertValues *insertValues = (InsertValues *) lfirst(insertValuesCell); int64 shardId = insertValues->shardId; bool foundSameShardId = false; if (route != NULL) { if (route->shardId == shardId) { foundSameShardId = true; } else { /* new shard id seen; current aggregation done; add to list */ modifyRouteList = lappend(modifyRouteList, route); } } if (foundSameShardId) { /* * Our current value has the same shard id as our aggregate object, * so append the rowValues. */ route->rowValuesLists = lappend(route->rowValuesLists, insertValues->rowValues); } else { /* we encountered a new shard id; build a new aggregate object */ route = (ModifyRoute *) palloc(sizeof(ModifyRoute)); route->shardId = insertValues->shardId; route->rowValuesLists = list_make1(insertValues->rowValues); } } /* left holding one final aggregate object; add to list */ modifyRouteList = lappend(modifyRouteList, route); return modifyRouteList; } /* * ExtractInsertValuesList extracts the partition column value for an INSERT * command and returns it within an InsertValues struct. For single-row INSERTs * this is simply a value extracted from the target list, but multi-row INSERTs * will generate a List of InsertValues, each with full row values in addition * to the partition value. If a partition value is NULL or missing altogether, * this function errors. */ static List * ExtractInsertValuesList(Query *query, Var *partitionColumn) { List *insertValuesList = NIL; TargetEntry *targetEntry = get_tle_by_resno(query->targetList, partitionColumn->varattno); if (targetEntry == NULL) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("cannot perform an INSERT without a partition column " "value"))); } /* * We've got a multi-row INSERT. PostgreSQL internally represents such * commands by linking Vars in the target list to lists of values within * a special VALUES range table entry. By extracting the right positional * expression from each list within that RTE, we will extract the partition * values for each row within the multi-row INSERT. */ if (IsA(targetEntry->expr, Var)) { Var *partitionVar = (Var *) targetEntry->expr; ListCell *valuesListCell = NULL; Index ivIndex = 0; RangeTblEntry *referencedRTE = rt_fetch(partitionVar->varno, query->rtable); foreach(valuesListCell, referencedRTE->values_lists) { InsertValues *insertValues = (InsertValues *) palloc(sizeof(InsertValues)); insertValues->rowValues = (List *) lfirst(valuesListCell); insertValues->partitionValueExpr = list_nth(insertValues->rowValues, (partitionVar->varattno - 1)); insertValues->shardId = INVALID_SHARD_ID; insertValues->listIndex = ivIndex; insertValuesList = lappend(insertValuesList, insertValues); ivIndex++; } } /* nothing's been found yet; this is a simple single-row INSERT */ if (insertValuesList == NIL) { InsertValues *insertValues = (InsertValues *) palloc(sizeof(InsertValues)); insertValues->rowValues = NIL; insertValues->partitionValueExpr = targetEntry->expr; insertValues->shardId = INVALID_SHARD_ID; insertValuesList = lappend(insertValuesList, insertValues); } return insertValuesList; } /* * ExtractInsertPartitionKeyValue extracts the partition column value * from an INSERT query. If the expression in the partition column is * non-constant or it is a multi-row INSERT with multiple different partition * column values, the function returns NULL. */ Const * ExtractInsertPartitionKeyValue(Query *query) { Oid distributedTableId = ExtractFirstCitusTableId(query); uint32 rangeTableId = 1; Const *singlePartitionValueConst = NULL; if (!HasDistributionKey(distributedTableId)) { return NULL; } Var *partitionColumn = PartitionColumn(distributedTableId, rangeTableId); TargetEntry *targetEntry = get_tle_by_resno(query->targetList, partitionColumn->varattno); if (targetEntry == NULL) { /* partition column value not specified */ return NULL; } Node *targetExpression = strip_implicit_coercions((Node *) targetEntry->expr); /* * Multi-row INSERTs have a Var in the target list that points to * an RTE_VALUES. */ if (IsA(targetExpression, Var)) { Var *partitionVar = (Var *) targetExpression; ListCell *valuesListCell = NULL; RangeTblEntry *referencedRTE = rt_fetch(partitionVar->varno, query->rtable); foreach(valuesListCell, referencedRTE->values_lists) { List *rowValues = (List *) lfirst(valuesListCell); Node *partitionValueNode = list_nth(rowValues, partitionVar->varattno - 1); Expr *partitionValueExpr = (Expr *) strip_implicit_coercions( partitionValueNode); if (!IsA(partitionValueExpr, Const)) { /* non-constant value in the partition column */ singlePartitionValueConst = NULL; break; } Const *partitionValueConst = (Const *) partitionValueExpr; if (singlePartitionValueConst == NULL) { /* first row has a constant in the partition column, looks promising! */ singlePartitionValueConst = partitionValueConst; } else if (!equal(partitionValueConst, singlePartitionValueConst)) { /* multiple different values in the partition column, too bad */ singlePartitionValueConst = NULL; break; } else { /* another row with the same partition column value! */ } } } else if (IsA(targetExpression, Const)) { /* single-row INSERT with a constant partition column value */ singlePartitionValueConst = (Const *) targetExpression; } else { /* single-row INSERT with a non-constant partition column value */ singlePartitionValueConst = NULL; } if (singlePartitionValueConst != NULL) { singlePartitionValueConst = copyObject(singlePartitionValueConst); } return singlePartitionValueConst; } /* * DeferErrorIfUnsupportedRouterPlannableSelectQuery checks if given query is router plannable, * SELECT query, setting distributedPlan->planningError if not. * The query is router plannable if it is a modify query, or if it is a select * query issued on a hash partitioned distributed table. Router plannable checks * for select queries can be turned off by setting citus.enable_router_execution * flag to false. */ static DeferredErrorMessage * DeferErrorIfUnsupportedRouterPlannableSelectQuery(Query *query) { List *rangeTableRelationList = NIL; ListCell *rangeTableRelationCell = NULL; if (query->commandType != CMD_SELECT) { return DeferredError(ERRCODE_ASSERT_FAILURE, "Only SELECT query types are supported in this path", NULL, NULL); } if (!EnableRouterExecution) { return DeferredError(ERRCODE_SUCCESSFUL_COMPLETION, "Router planner not enabled.", NULL, NULL); } bool hasPostgresOrCitusLocalTable = false; bool hasDistributedTable = false; bool hasReferenceTable = false; List *distributedRelationList = NIL; ExtractRangeTableRelationWalker((Node *) query, &rangeTableRelationList); foreach(rangeTableRelationCell, rangeTableRelationList) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(rangeTableRelationCell); if (rte->rtekind == RTE_RELATION) { Oid distributedTableId = rte->relid; /* local tables are allowed if there are no distributed tables */ if (!IsCitusTable(distributedTableId)) { hasPostgresOrCitusLocalTable = true; continue; } else if (IsCitusTableType(distributedTableId, REFERENCE_TABLE)) { hasReferenceTable = true; continue; } else if (IsCitusTableType(distributedTableId, CITUS_LOCAL_TABLE)) { hasPostgresOrCitusLocalTable = true; elog(DEBUG4, "Router planner finds a local table added to metadata"); continue; } if (IsCitusTableType(distributedTableId, APPEND_DISTRIBUTED)) { return DeferredError( ERRCODE_FEATURE_NOT_SUPPORTED, "Router planner does not support append-partitioned tables.", NULL, NULL); } if (IsCitusTableType(distributedTableId, DISTRIBUTED_TABLE)) { hasDistributedTable = true; distributedRelationList = lappend_oid(distributedRelationList, distributedTableId); } /* * Currently, we don't support tables with replication factor > 1, * except reference tables with SELECT ... FOR UPDATE queries. It is * also not supported from MX nodes. */ if (query->hasForUpdate) { uint32 tableReplicationFactor = TableShardReplicationFactor( distributedTableId); if (tableReplicationFactor > 1 && IsCitusTableType(distributedTableId, DISTRIBUTED_TABLE)) { return DeferredError( ERRCODE_FEATURE_NOT_SUPPORTED, "SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables.", NULL, NULL); } } } } /* * We want to make sure nextval happens on the coordinator / the current * node, since the user may have certain expectations around the values * produced by the sequence. We therefore cannot push down the nextval * call as part of a router query. * * We let queries with nextval in the target list fall through to * the logical planner, which will ensure that the nextval is called * in the combine query on the coordinator. * * If there are no distributed or reference tables in the query, * then the query will anyway happen on the coordinator, so we can * allow nextval. */ if (contain_nextval_expression_walker((Node *) query->targetList, NULL) && (hasDistributedTable || hasReferenceTable)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Sequences cannot be used in router queries", NULL, NULL); } /* local tables are not allowed if there are distributed tables */ if (hasPostgresOrCitusLocalTable && hasDistributedTable) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Local tables cannot be used in distributed queries.", NULL, NULL); } DeferredErrorMessage *CTEWithSearchClauseError = ErrorIfQueryHasCTEWithSearchClause(query); if (CTEWithSearchClauseError != NULL) { return CTEWithSearchClauseError; } return ErrorIfQueryHasUnroutableModifyingCTE(query); } /* * Copy a RelationRestrictionContext. Note that several subfields are copied * shallowly, for lack of copyObject support. * * Note that CopyRelationRestrictionContext copies the following fields per relation * context: index, relationId, distributedRelation, rte, relOptInfo->baserestrictinfo * and relOptInfo->joininfo. Also, the function shallowly copies plannerInfo and * prunedShardIntervalList which are read-only. All other parts of the relOptInfo * is also shallowly copied. */ RelationRestrictionContext * CopyRelationRestrictionContext(RelationRestrictionContext *oldContext) { RelationRestrictionContext *newContext = (RelationRestrictionContext *) palloc(sizeof(RelationRestrictionContext)); ListCell *relationRestrictionCell = NULL; newContext->allReferenceTables = oldContext->allReferenceTables; newContext->relationRestrictionList = NIL; foreach(relationRestrictionCell, oldContext->relationRestrictionList) { RelationRestriction *oldRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); RelationRestriction *newRestriction = (RelationRestriction *) palloc0(sizeof(RelationRestriction)); newRestriction->index = oldRestriction->index; newRestriction->relationId = oldRestriction->relationId; newRestriction->citusTable = oldRestriction->citusTable; newRestriction->rte = copyObject(oldRestriction->rte); /* can't be copied, we copy (flatly) a RelOptInfo, and then decouple baserestrictinfo */ newRestriction->relOptInfo = palloc(sizeof(RelOptInfo)); *newRestriction->relOptInfo = *oldRestriction->relOptInfo; newRestriction->relOptInfo->baserestrictinfo = copyObject(oldRestriction->relOptInfo->baserestrictinfo); newRestriction->relOptInfo->joininfo = copyObject(oldRestriction->relOptInfo->joininfo); /* not copyable, but readonly */ newRestriction->plannerInfo = oldRestriction->plannerInfo; newContext->relationRestrictionList = lappend(newContext->relationRestrictionList, newRestriction); } return newContext; } /* * ErrorIfQueryHasUnroutableModifyingCTE checks if the query contains modifying common table * expressions and errors out if it does. */ static DeferredErrorMessage * ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree) { Assert(queryTree->commandType == CMD_SELECT); if (!queryTree->hasModifyingCTE) { return NULL; } /* we can't route conflicting replication models */ char replicationModel = 0; CommonTableExpr *cte = NULL; foreach_declared_ptr(cte, queryTree->cteList) { Query *cteQuery = (Query *) cte->ctequery; /* * Here we only check for command type of top level query. Normally there can be * nested CTE, however PostgreSQL dictates that data-modifying statements must * be at top level of CTE. Therefore it is OK to just check for top level. * Similarly, we do not need to check for subqueries. */ if (cteQuery->commandType != CMD_SELECT && cteQuery->commandType != CMD_UPDATE && cteQuery->commandType != CMD_DELETE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "only SELECT, UPDATE, or DELETE common table expressions " "may be router planned", NULL, NULL); } if (cteQuery->commandType != CMD_SELECT) { Oid distributedTableId = InvalidOid; DeferredErrorMessage *cteError = ModifyPartialQuerySupported(cteQuery, false, &distributedTableId); if (cteError) { return cteError; } CitusTableCacheEntry *modificationTableCacheEntry = GetCitusTableCacheEntry(distributedTableId); if (!IsCitusTableTypeCacheEntry(modificationTableCacheEntry, DISTRIBUTED_TABLE)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot router plan modification of a non-distributed table", NULL, NULL); } if (replicationModel && modificationTableCacheEntry->replicationModel != replicationModel) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot route mixed replication models", NULL, NULL); } replicationModel = modificationTableCacheEntry->replicationModel; } } /* everything OK */ return NULL; } /* * ErrorIfQueryHasCTEWithSearchClause checks if the query contains any common table * expressions with search clause and errors out if it does. */ static DeferredErrorMessage * ErrorIfQueryHasCTEWithSearchClause(Query *queryTree) { if (ContainsSearchClauseWalker((Node *) queryTree, NULL)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "CTEs with search clauses are not supported", NULL, NULL); } return NULL; } /* * ContainsSearchClauseWalker walks over the node and finds if there are any * CommonTableExprs with search clause */ static bool ContainsSearchClauseWalker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, CommonTableExpr)) { if (((CommonTableExpr *) node)->search_clause != NULL) { return true; } } if (IsA(node, Query)) { return query_tree_walker((Query *) node, ContainsSearchClauseWalker, NULL, 0); } return expression_tree_walker(node, ContainsSearchClauseWalker, NULL); } /* * get_all_actual_clauses * * Returns a list containing the bare clauses from 'restrictinfo_list'. * * This loses the distinction between regular and pseudoconstant clauses, * so be careful what you use it for. */ List * get_all_actual_clauses(List *restrictinfo_list) { List *result = NIL; ListCell *l; foreach(l, restrictinfo_list) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(l); Assert(IsA(rinfo, RestrictInfo)); result = lappend(result, rinfo->clause); } return result; } /* * CompareInsertValuesByShardId does what it says in the name. Used for sorting * InsertValues objects by their shard. */ static int CompareInsertValuesByShardId(const void *leftElement, const void *rightElement) { InsertValues *leftValue = *((InsertValues **) leftElement); InsertValues *rightValue = *((InsertValues **) rightElement); int64 leftShardId = leftValue->shardId; int64 rightShardId = rightValue->shardId; Index leftIndex = leftValue->listIndex; Index rightIndex = rightValue->listIndex; if (leftShardId > rightShardId) { return 1; } else if (leftShardId < rightShardId) { return -1; } else { /* shard identifiers are the same, list index is secondary sort key */ if (leftIndex > rightIndex) { return 1; } else if (leftIndex < rightIndex) { return -1; } else { return 0; } } } ================================================ FILE: src/backend/distributed/planner/query_colocation_checker.c ================================================ /*------------------------------------------------------------------------- * * query_colocation_checker.c implements the logic for determining * whether any subqueries in a given query are co-located (e.g., * distribution keys of the relations inside subqueries are equal). * * The main logic behind non colocated subquery joins is that we pick * an anchor range table entry and check for distribution key equality * of any other subqueries in the given query. If for a given subquery, * we cannot find distribution key equality with the anchor rte, we * recursively plan that subquery. * * We also used a hacky solution for picking relations as the anchor range * table entries. The hack is that we wrap them into a subquery. This is only * necessary since some of the attribute equivalence checks are based on * queries rather than range table entries. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/relation.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "utils/rel.h" #include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_logical_planner.h" /* only to access utility functions */ #include "distributed/pg_dist_partition.h" #include "distributed/query_colocation_checker.h" #include "distributed/relation_restriction_equivalence.h" static RangeTblEntry * AnchorRte(Query *subquery); static List * UnionRelationRestrictionLists(List *firstRelationList, List *secondRelationList); static List * CreateDummyTargetList(Oid relationId, List *requiredAttributes); static TargetEntry * CreateTargetEntryForColumn(Form_pg_attribute attributeTuple, Index rteIndex, int attributeNumber, int resno); static TargetEntry * CreateTargetEntryForNullCol(Form_pg_attribute attributeTuple, int resno); static TargetEntry * CreateUnusedTargetEntry(int resno); /* * CreateColocatedJoinChecker is a helper function that simply calculates * a ColocatedJoinChecker with the given input and returns it. */ ColocatedJoinChecker CreateColocatedJoinChecker(Query *subquery, PlannerRestrictionContext *restrictionContext) { ColocatedJoinChecker colocatedJoinChecker = { 0 }; Query *anchorSubquery = NULL; /* we couldn't pick an anchor subquery, no need to continue */ RangeTblEntry *anchorRangeTblEntry = AnchorRte(subquery); if (anchorRangeTblEntry == NULL) { colocatedJoinChecker.anchorRelationRestrictionList = NIL; return colocatedJoinChecker; } if (anchorRangeTblEntry->rtekind == RTE_RELATION) { /* * If we get a relation as our anchor, wrap into a subquery. The only * reason that we wrap the relation into a subquery is that some of the utility * functions (i.e., FilterPlannerRestrictionForQuery()) rely on queries * not relations. */ RTEPermissionInfo *perminfo = NULL; if (anchorRangeTblEntry->perminfoindex) { perminfo = getRTEPermissionInfo(subquery->rteperminfos, anchorRangeTblEntry); } anchorSubquery = WrapRteRelationIntoSubquery(anchorRangeTblEntry, NIL, perminfo); } else if (anchorRangeTblEntry->rtekind == RTE_SUBQUERY) { anchorSubquery = anchorRangeTblEntry->subquery; } else { /* we don't expect any other RTE type here */ pg_unreachable(); } PlannerRestrictionContext *anchorPlannerRestrictionContext = FilterPlannerRestrictionForQuery(restrictionContext, anchorSubquery); RelationRestrictionContext *anchorRelationRestrictionContext = anchorPlannerRestrictionContext->relationRestrictionContext; List *anchorRestrictionEquivalences = GenerateAllAttributeEquivalences(anchorPlannerRestrictionContext); /* fill the non colocated planning context */ colocatedJoinChecker.subquery = subquery; colocatedJoinChecker.subqueryPlannerRestriction = restrictionContext; colocatedJoinChecker.anchorRelationRestrictionList = anchorRelationRestrictionContext->relationRestrictionList; colocatedJoinChecker.anchorAttributeEquivalences = anchorRestrictionEquivalences; return colocatedJoinChecker; } /* * AnchorRte gets a query and searches for a relation or a subquery within * the join tree of the query such that we can use it as our anchor range * table entry during our non colocated subquery planning. * * The function returns NULL if it cannot find a proper range table entry for our * purposes. See the function for the details. */ static RangeTblEntry * AnchorRte(Query *subquery) { FromExpr *joinTree = subquery->jointree; Relids joinRelIds = get_relids_in_jointree((Node *) joinTree, false, false); int currentRTEIndex = -1; RangeTblEntry *anchorRangeTblEntry = NULL; /* * Pick a random anchor relation or subquery (i.e., the first) for now. We * might consider picking a better rte as the anchor. For example, we could * iterate on the joinRelIds, and check which rteIndex has more distribution * key equiality with rteIndexes. For the time being, the current primitive * approach helps us in many cases. */ while ((currentRTEIndex = bms_next_member(joinRelIds, currentRTEIndex)) >= 0) { RangeTblEntry *currentRte = rt_fetch(currentRTEIndex, subquery->rtable); /* * We always prefer distributed relations if we can find any. The * reason is that Citus is currently able to recursively plan * subqueries, but not relations. * * For the subqueries, make sure that the subquery contains at least one * distributed table and doesn't have a set operation. * * TODO: The set operation restriction might sound weird, but, the restriction * equivalence generation functions ignore set operations. We should * integrate the logic in SafeToPushdownUnionSubquery() to * GenerateAllAttributeEquivalences() such that the latter becomes aware of * the set operations. */ if (anchorRangeTblEntry == NULL && currentRte->rtekind == RTE_SUBQUERY && FindNodeMatchingCheckFunction((Node *) currentRte->subquery, IsDistributedTableRTE) && currentRte->subquery->setOperations == NULL && !ContainsUnionSubquery(currentRte->subquery)) { /* found a subquery, keep it if we cannot find a relation */ anchorRangeTblEntry = currentRte; } else if (currentRte->rtekind == RTE_RELATION) { Oid relationId = currentRte->relid; if (!IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { /* * We're not interested in non distributed relations. */ continue; } anchorRangeTblEntry = currentRte; break; } } return anchorRangeTblEntry; } /* * SubqueryColocated returns true if the input subquery has a distribution * key equality with the anchor subquery. In other words, we refer the * distribution key equality of relations as "colocation" in this context. */ bool SubqueryColocated(Query *subquery, ColocatedJoinChecker *checker) { List *anchorRelationRestrictionList = checker->anchorRelationRestrictionList; List *anchorAttributeEquivalences = checker->anchorAttributeEquivalences; PlannerRestrictionContext *restrictionContext = checker->subqueryPlannerRestriction; PlannerRestrictionContext *filteredPlannerContext = FilterPlannerRestrictionForQuery(restrictionContext, subquery); List *filteredRestrictionList = filteredPlannerContext->relationRestrictionContext->relationRestrictionList; /* * There are no relations in the input subquery, such as a subquery * that consist of only intermediate results or without FROM * clause or subquery in WHERE clause anded with FALSE. * * Note that for the subquery in WHERE clause, the input original * subquery (a.k.a., which didn't go through standard_planner()) may * contain distributed relations, but postgres is smart enough to * not generate the restriction information. That's the reason for * not asserting non-existence of distributed relations. */ if (list_length(filteredRestrictionList) == 0) { return true; } /* * We merge the relation restrictions of the input subquery and the anchor * restrictions to form a temporary relation restriction context. The aim of * forming this temporary context is to check whether the context contains * distribution key equality or not. */ List *unionedRelationRestrictionList = UnionRelationRestrictionLists(anchorRelationRestrictionList, filteredRestrictionList); /* * We already have the attributeEquivalences, thus, only need to prepare * the planner restrictions with unioned relations for our purpose of * distribution key equality. Note that we don't need to calculate the * join restrictions, we're already relying on the attributeEquivalences * provided by the context. */ RelationRestrictionContext *unionedRelationRestrictionContext = palloc0( sizeof(RelationRestrictionContext)); unionedRelationRestrictionContext->relationRestrictionList = unionedRelationRestrictionList; PlannerRestrictionContext *unionedPlannerRestrictionContext = palloc0( sizeof(PlannerRestrictionContext)); unionedPlannerRestrictionContext->relationRestrictionContext = unionedRelationRestrictionContext; if (!RestrictionEquivalenceForPartitionKeysViaEquivalences( unionedPlannerRestrictionContext, anchorAttributeEquivalences)) { return false; } return true; } /* * WrapRteRelationIntoSubquery wraps the given relation range table entry * in a newly constructed "(SELECT * FROM table_name as anchor_relation)" query. * * Note that the query returned by this function does not contain any filters or * projections. The returned query should be used cautiosly and it is mostly * designed for generating a stub query. */ Query * WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, List *requiredAttributes, RTEPermissionInfo *perminfo) { Query *subquery = makeNode(Query); RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); subquery->commandType = CMD_SELECT; /* we copy the input rteRelation to preserve the rteIdentity */ RangeTblEntry *newRangeTableEntry = copyObject(rteRelation); subquery->rtable = list_make1(newRangeTableEntry); if (perminfo) { newRangeTableEntry->perminfoindex = 1; subquery->rteperminfos = list_make1(perminfo); } /* set the FROM expression to the subquery */ newRangeTableRef = makeNode(RangeTblRef); newRangeTableRef->rtindex = SINGLE_RTE_INDEX; subquery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); subquery->targetList = CreateFilteredTargetListForRelation(rteRelation->relid, requiredAttributes); if (list_length(subquery->targetList) == 0) { /* * in case there is no required column, we assign one dummy NULL target entry * to the subquery targetList so that it has at least one target. * (targetlist should have at least one element) */ subquery->targetList = CreateDummyTargetList(rteRelation->relid, requiredAttributes); } return subquery; } /* * CreateAllTargetListForRelation creates a target list which contains all the columns * of the given relation. If the column is not in required columns, then it is added * as a NULL column. */ List * CreateAllTargetListForRelation(Oid relationId, List *requiredAttributes) { Relation relation = relation_open(relationId, AccessShareLock); int numberOfAttributes = RelationGetNumberOfAttributes(relation); List *targetList = NIL; int varAttrNo = 1; for (int attrNum = 1; attrNum <= numberOfAttributes; attrNum++) { Form_pg_attribute attributeTuple = TupleDescAttr(relation->rd_att, attrNum - 1); int resNo = attrNum; if (attributeTuple->attisdropped) { /* * For dropped columns, we generate a dummy null column because * varattno in relation and subquery are different things, however if * we put the NULL columns to the subquery for the dropped columns, * they will point to the same variable. */ TargetEntry *nullTargetEntry = CreateUnusedTargetEntry(resNo); targetList = lappend(targetList, nullTargetEntry); continue; } if (!list_member_int(requiredAttributes, attrNum)) { TargetEntry *nullTargetEntry = CreateTargetEntryForNullCol(attributeTuple, resNo); targetList = lappend(targetList, nullTargetEntry); } else { TargetEntry *targetEntry = CreateTargetEntryForColumn(attributeTuple, SINGLE_RTE_INDEX, varAttrNo++, resNo); targetList = lappend(targetList, targetEntry); } } relation_close(relation, NoLock); return targetList; } /* * CreateFilteredTargetListForRelation creates a target list which contains * only the required columns of the given relation. If there is not required * columns then a dummy NULL column is put as the only entry. */ List * CreateFilteredTargetListForRelation(Oid relationId, List *requiredAttributes) { Relation relation = relation_open(relationId, AccessShareLock); int numberOfAttributes = RelationGetNumberOfAttributes(relation); List *targetList = NIL; int resultNo = 1; for (int attrNum = 1; attrNum <= numberOfAttributes; attrNum++) { Form_pg_attribute attributeTuple = TupleDescAttr(relation->rd_att, attrNum - 1); if (list_member_int(requiredAttributes, attrNum)) { /* In the subquery with only required attribute numbers, the result no * corresponds to the ordinal index of it in targetList. */ TargetEntry *targetEntry = CreateTargetEntryForColumn(attributeTuple, SINGLE_RTE_INDEX, attrNum, resultNo++); targetList = lappend(targetList, targetEntry); } } relation_close(relation, NoLock); return targetList; } /* * CreateDummyTargetList creates a target list which contains only a * NULL entry. */ static List * CreateDummyTargetList(Oid relationId, List *requiredAttributes) { int resno = 1; TargetEntry *dummyTargetEntry = CreateUnusedTargetEntry(resno); return list_make1(dummyTargetEntry); } /* * CreateTargetEntryForColumn creates a target entry for the given * column. */ static TargetEntry * CreateTargetEntryForColumn(Form_pg_attribute attributeTuple, Index rteIndex, int attributeNumber, int resno) { Var *targetColumn = makeVar(rteIndex, attributeNumber, attributeTuple->atttypid, attributeTuple->atttypmod, attributeTuple->attcollation, 0); TargetEntry *targetEntry = makeTargetEntry((Expr *) targetColumn, resno, pstrdup(attributeTuple->attname.data), false); return targetEntry; } /* * CreateTargetEntryForNullCol creates a target entry that has a NULL expression. */ static TargetEntry * CreateTargetEntryForNullCol(Form_pg_attribute attributeTuple, int resno) { Expr *nullExpr = (Expr *) makeNullConst(attributeTuple->atttypid, attributeTuple->atttypmod, attributeTuple->attcollation); char *resName = attributeTuple->attname.data; TargetEntry *targetEntry = makeTargetEntry(nullExpr, resno, pstrdup(resName), false); return targetEntry; } /* * CreateUnusedTargetEntry creates a dummy target entry which is not used * in postgres query. */ static TargetEntry * CreateUnusedTargetEntry(int resno) { StringInfo colname = makeStringInfo(); appendStringInfo(colname, "dummy-%d", resno); Expr *nullExpr = (Expr *) makeNullConst(INT4OID, 0, InvalidOid); TargetEntry *targetEntry = makeTargetEntry(nullExpr, resno, colname->data, false); return targetEntry; } /* * UnionRelationRestrictionLists merges two relation restriction lists * and returns a newly allocated list. The merged relation restriction * list doesn't contain any duplicate elements. */ static List * UnionRelationRestrictionLists(List *firstRelationList, List *secondRelationList) { List *unionedRelationRestrictionList = NULL; ListCell *relationRestrictionCell = NULL; Relids rteIdentities = NULL; /* list_concat destructively modifies the first list, thus copy it */ firstRelationList = list_copy(firstRelationList); List *allRestrictionList = list_concat(firstRelationList, secondRelationList); foreach(relationRestrictionCell, allRestrictionList) { RelationRestriction *restriction = (RelationRestriction *) lfirst(relationRestrictionCell); int rteIdentity = GetRTEIdentity(restriction->rte); /* already have the same rte, skip */ if (bms_is_member(rteIdentity, rteIdentities)) { continue; } unionedRelationRestrictionList = lappend(unionedRelationRestrictionList, restriction); rteIdentities = bms_add_member(rteIdentities, rteIdentity); } RelationRestrictionContext *unionedRestrictionContext = palloc0( sizeof(RelationRestrictionContext)); unionedRestrictionContext->relationRestrictionList = unionedRelationRestrictionList; return unionedRelationRestrictionList; } ================================================ FILE: src/backend/distributed/planner/query_pushdown_planning.c ================================================ /*------------------------------------------------------------------------- * * query_pushdown_planning.c * * Routines for creating pushdown plans for queries. Both select and modify * queries can be planned using query pushdown logic passing the checks given * in this file. * * Checks are controlled to understand whether the query can be sent to worker * nodes by simply adding shard_id to table names and getting the correct result * from them. That means, all the required data is present on the workers. * * For select queries, Citus try to use query pushdown planner if it has a * subquery or function RTEs. For modify queries, Citus try to use query pushdown * planner if the query accesses multiple tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "parser/parsetree.h" #include "pg_version_constants.h" #include "distributed/citus_clauses.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/version_compat.h" #define INVALID_RELID -1 /* * RecurringTuplesType is used to distinguish different types of expressions * that always produce the same set of tuples when a shard is queried. We make * this distinction to produce relevant error messages when recurring tuples * are used in a way that would give incorrect results. */ typedef enum RecurringTuplesType { RECURRING_TUPLES_INVALID = 0, RECURRING_TUPLES_REFERENCE_TABLE, RECURRING_TUPLES_FUNCTION, RECURRING_TUPLES_EMPTY_JOIN_TREE, RECURRING_TUPLES_RESULT_FUNCTION, RECURRING_TUPLES_VALUES, RECURRING_TUPLES_JSON_TABLE } RecurringTuplesType; /* * RelidsReferenceWalkerContext is used to find Vars in a (sub)query that * refer to certain relids from the upper query. */ typedef struct RelidsReferenceWalkerContext { int level; Relids relids; int foundRelid; } RelidsReferenceWalkerContext; /* Config variable managed via guc.c */ bool SubqueryPushdown = false; /* is subquery pushdown enabled */ int ValuesMaterializationThreshold = 100; /* Local functions forward declarations */ static bool JoinTreeContainsSubqueryWalker(Node *joinTreeNode, void *context); static bool IsFunctionOrValuesRTE(Node *node); static bool WindowPartitionOnDistributionColumn(Query *query); static DeferredErrorMessage * DeferErrorIfFromClauseRecurs(Query *queryTree); static RecurringTuplesType FromClauseRecurringTupleType(Query *queryTree); static DeferredErrorMessage * DeferredErrorIfUnsupportedRecurringTuplesJoin( PlannerRestrictionContext *plannerRestrictionContext, bool plannerPhase); static DeferredErrorMessage * DeferErrorIfUnsupportedTableCombination(Query *queryTree); static DeferredErrorMessage * DeferErrorIfSubqueryRequiresMerge(Query *subqueryTree, bool lateral, char *referencedThing); static bool ExtractSetOperationStatementWalker(Node *node, List **setOperationList); static RecurringTuplesType FetchFirstRecurType(PlannerInfo *plannerInfo, Relids relids); static bool ContainsRecurringRTE(RangeTblEntry *rangeTableEntry, RecurringTuplesType *recurType); static bool ContainsRecurringRangeTable(List *rangeTable, RecurringTuplesType *recurType); static bool HasRecurringTuples(Node *node, RecurringTuplesType *recurType); static MultiNode * SubqueryPushdownMultiNodeTree(Query *queryTree); static MultiTable * MultiSubqueryPushdownTable(Query *subquery); static List * CreateSubqueryTargetListAndAdjustVars(List *columnList); static AttrNumber FindResnoForVarInTargetList(List *targetList, int varno, int varattno); static bool RelationInfoContainsOnlyRecurringTuples(PlannerInfo *plannerInfo, Relids relids); static char * RecurringTypeDescription(RecurringTuplesType recurType); static DeferredErrorMessage * DeferredErrorIfUnsupportedLateralSubquery(PlannerInfo * plannerInfo, Relids recurringRelIds, Relids nonRecurringRelIds); static bool ContainsLateralSubquery(PlannerInfo *plannerInfo); static Var * PartitionColumnForPushedDownSubquery(Query *query); static bool ContainsReferencesToRelids(Query *query, Relids relids, int *foundRelid); static bool ContainsReferencesToRelidsWalker(Node *node, RelidsReferenceWalkerContext *context); /* * ShouldUseSubqueryPushDown determines whether it's desirable to use * subquery pushdown to plan the query based on the original and * rewritten query. */ bool ShouldUseSubqueryPushDown(Query *originalQuery, Query *rewrittenQuery, PlannerRestrictionContext *plannerRestrictionContext) { /* * We check the existence of subqueries in FROM clause on the modified query * given that if postgres already flattened the subqueries, MultiNodeTree() * can plan corresponding distributed plan. */ if (JoinTreeContainsSubquery(rewrittenQuery)) { return true; } /* * We check the existence of subqueries in WHERE and HAVING clause on the * modified query. In some cases subqueries in the original query are * converted into inner joins and in those cases MultiNodeTree() can plan * the rewritten plan. */ if (WhereOrHavingClauseContainsSubquery(rewrittenQuery)) { return true; } /* * We check the existence of subqueries in the SELECT clause on the modified * query. */ if (TargetListContainsSubquery(rewrittenQuery->targetList)) { return true; } /* * We check if postgres planned any semi joins, MultiNodeTree doesn't * support these so we fail. Postgres is able to replace some IN/ANY * subqueries with semi joins and then replace those with inner joins (ones * where the subquery returns unique results). This allows MultiNodeTree to * execute these subqueries (because they are converted to inner joins). * However, even in that case the rewrittenQuery still contains join nodes * with jointype JOIN_SEMI because Postgres doesn't actually update these. * The way we find out instead if it actually planned semi joins, is by * checking the joins that were sent to multi_join_restriction_hook. If no * joins of type JOIN_SEMI are sent it is safe to convert all JOIN_SEMI * nodes to JOIN_INNER nodes (which is what is done in MultiNodeTree). */ JoinRestrictionContext *joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; if (joinRestrictionContext->hasSemiJoin) { return true; } /* * We process function and VALUES RTEs as subqueries, since the join order planner * does not know how to handle them. */ if (FindNodeMatchingCheckFunction((Node *) originalQuery, IsFunctionOrValuesRTE)) { return true; } /* * We handle outer joins as subqueries, since the join order planner * does not know how to handle them. */ if (FindNodeMatchingCheckFunction((Node *) originalQuery->jointree, IsOuterJoinExpr)) { return true; } /* * Original query may not have an outer join while rewritten query does. * We should push down in this case. * An example of this is https://github.com/citusdata/citus/issues/2739 * where postgres pulls-up the outer-join in the subquery. */ if (FindNodeMatchingCheckFunction((Node *) rewrittenQuery->jointree, IsOuterJoinExpr)) { return true; } /* * Some unsupported join clauses in logical planner * may be supported by subquery pushdown planner. */ List *qualifierList = QualifierList(rewrittenQuery->jointree); if (DeferErrorIfUnsupportedClause(qualifierList) != NULL) { return true; } /* check if the query has a window function and it is safe to pushdown */ if (originalQuery->hasWindowFuncs && SafeToPushdownWindowFunction(originalQuery, NULL)) { return true; } return false; } /* * JoinTreeContainsSubquery returns true if the input query contains any subqueries * in the join tree (e.g., FROM clause). */ bool JoinTreeContainsSubquery(Query *query) { FromExpr *joinTree = query->jointree; if (!joinTree) { return false; } return JoinTreeContainsSubqueryWalker((Node *) joinTree, query); } /* * HasEmptyJoinTree returns whether the query selects from anything. */ bool HasEmptyJoinTree(Query *query) { if (query->rtable == NIL) { return true; } else if (list_length(query->rtable) == 1) { RangeTblEntry *rte = (RangeTblEntry *) linitial(query->rtable); if (rte->rtekind == RTE_RESULT) { return true; } } return false; } /* * JoinTreeContainsSubqueryWalker returns true if the input joinTreeNode * references to a subquery. Otherwise, recurses into the expression. */ static bool JoinTreeContainsSubqueryWalker(Node *joinTreeNode, void *context) { if (joinTreeNode == NULL) { return false; } if (IsA(joinTreeNode, RangeTblRef)) { Query *query = (Query *) context; RangeTblRef *rangeTableRef = (RangeTblRef *) joinTreeNode; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableRef->rtindex, query->rtable); if (rangeTableEntry->rtekind == RTE_SUBQUERY) { return true; } return false; } return expression_tree_walker(joinTreeNode, JoinTreeContainsSubqueryWalker, context); } /* * WhereOrHavingClauseContainsSubquery returns true if the input query contains * any subqueries in the WHERE or HAVING clause. */ bool WhereOrHavingClauseContainsSubquery(Query *query) { if (FindNodeMatchingCheckFunction(query->havingQual, IsNodeSubquery)) { return true; } if (!query->jointree) { return false; } /* * We search the whole jointree here, not just the quals. The reason for * this is that the fromlist can contain other FromExpr nodes again or * JoinExpr nodes that also have quals. If that's the case we need to check * those as well if they contain andy subqueries. */ return FindNodeMatchingCheckFunction((Node *) query->jointree, IsNodeSubquery); } /* * TargetList returns true if the input query contains * any subqueries in the WHERE clause. */ bool TargetListContainsSubquery(List *targetList) { bool hasSubquery = FindNodeMatchingCheckFunction((Node *) targetList, IsNodeSubquery); return hasSubquery; } /* * IsFunctionRTE determines whether the given node is a function RTE. */ static bool IsFunctionOrValuesRTE(Node *node) { if (IsA(node, RangeTblEntry)) { RangeTblEntry *rangeTblEntry = (RangeTblEntry *) node; if (rangeTblEntry->rtekind == RTE_FUNCTION || rangeTblEntry->rtekind == RTE_VALUES || IsJsonTableRTE(rangeTblEntry)) { return true; } } return false; } /* * IsNodeSubquery returns true if the given node is a Query or SubPlan or a * Param node with paramkind PARAM_EXEC. * * The check for SubPlan is needed when this is used on a already rewritten * query. Such a query has SubPlan nodes instead of SubLink nodes (which * contain a Query node). * The check for PARAM_EXEC is needed because some very simple subqueries like * (select 1) are converted to init plans in the rewritten query. In this case * the only thing left in the query tree is a Param node with type PARAM_EXEC. */ bool IsNodeSubquery(Node *node) { if (node == NULL) { return false; } if (IsA(node, Query) || IsA(node, SubPlan)) { return true; } if (!IsA(node, Param)) { return false; } return ((Param *) node)->paramkind == PARAM_EXEC; } /* * IsOuterJoinExpr returns whether the given node is an outer join expression. */ bool IsOuterJoinExpr(Node *node) { bool isOuterJoin = false; if (node == NULL) { return false; } if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; JoinType joinType = joinExpr->jointype; if (IS_OUTER_JOIN(joinType)) { isOuterJoin = true; } } return isOuterJoin; } /* * SafeToPushdownWindowFunction checks if the query with window function is supported. * Returns the result accordingly and modifies errorDetail if non null. */ bool SafeToPushdownWindowFunction(Query *query, StringInfo *errorDetail) { ListCell *windowClauseCell = NULL; List *windowClauseList = query->windowClause; /* * We need to check each window clause separately if there is a partition by clause * and if it is partitioned on the distribution column. */ foreach(windowClauseCell, windowClauseList) { WindowClause *windowClause = lfirst(windowClauseCell); if (!windowClause->partitionClause) { if (errorDetail) { *errorDetail = makeStringInfo(); appendStringInfoString(*errorDetail, "Window functions without PARTITION BY on distribution " "column is currently unsupported"); } return false; } } if (!WindowPartitionOnDistributionColumn(query)) { if (errorDetail) { *errorDetail = makeStringInfo(); appendStringInfoString(*errorDetail, "Window functions with PARTITION BY list missing distribution " "column is currently unsupported"); } return false; } return true; } /* * WindowPartitionOnDistributionColumn checks if the given subquery has one * or more window functions and at least one of them is not partitioned by * distribution column. The function returns false if your window function does not * have a partition by clause or it does not include the distribution column. * * Please note that if the query does not have a window function, the function * returns true. */ static bool WindowPartitionOnDistributionColumn(Query *query) { List *windowClauseList = query->windowClause; ListCell *windowClauseCell = NULL; foreach(windowClauseCell, windowClauseList) { WindowClause *windowClause = lfirst(windowClauseCell); List *partitionClauseList = windowClause->partitionClause; List *targetEntryList = query->targetList; List *groupTargetEntryList = GroupTargetEntryList(partitionClauseList, targetEntryList); bool partitionOnDistributionColumn = TargetListOnPartitionColumn(query, groupTargetEntryList); if (!partitionOnDistributionColumn) { return false; } } return true; } /* * SubqueryMultiNodeTree gets the query objects and returns logical plan * for subqueries. * * We currently have two different code paths for creating logic plan for subqueries: * (i) subquery pushdown * (ii) single relation repartition subquery * * In order to create the logical plan, we follow the algorithm below: * - If subquery pushdown planner can plan the query * - We're done, we create the multi plan tree and return * - Else * - If the query is not eligible for single table repartition subquery planning * - Throw the error that the subquery pushdown planner generated * - If it is eligible for single table repartition subquery planning * - Check for the errors for single table repartition subquery planning * - If no errors found, we're done. Create the multi plan and return * - If found errors, throw it */ MultiNode * SubqueryMultiNodeTree(Query *originalQuery, Query *queryTree, PlannerRestrictionContext *plannerRestrictionContext) { /* * This is a generic error check that applies to both subquery pushdown * and single table repartition subquery. */ DeferredErrorMessage *unsupportedQueryError = DeferErrorIfQueryNotSupported( originalQuery); if (unsupportedQueryError != NULL) { RaiseDeferredError(unsupportedQueryError, ERROR); } /* * We reach here at the third step of the planning, thus we already checked for pushed down * feasibility of recurring outer joins, at this step the unsupported outer join check should * only generate an error when there is a lateral subquery. */ DeferredErrorMessage *subqueryPushdownError = DeferErrorIfUnsupportedSubqueryPushdown( originalQuery, plannerRestrictionContext, false); if (subqueryPushdownError != NULL) { RaiseDeferredError(subqueryPushdownError, ERROR); } MultiNode *multiQueryNode = SubqueryPushdownMultiNodeTree(originalQuery); Assert(multiQueryNode != NULL); return multiQueryNode; } /* * DeferErrorIfContainsUnsupportedSubqueryPushdown iterates on the query's subquery * entry list and uses helper functions to check if we can push down subquery * to worker nodes. These helper functions returns a deferred error if we * cannot push down the subquery. */ DeferredErrorMessage * DeferErrorIfUnsupportedSubqueryPushdown(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext, bool plannerPhase) { bool outerMostQueryHasLimit = false; ListCell *subqueryCell = NULL; List *subqueryList = NIL; if (originalQuery->limitCount != NULL) { outerMostQueryHasLimit = true; } /* * We're checking two things here: * (i) If the query contains a top level union, ensure that all leaves * return the partition key at the same position * (ii) Else, check whether all relations joined on the partition key or not */ if (ContainsUnionSubquery(originalQuery)) { if (!SafeToPushdownUnionSubquery(originalQuery, plannerRestrictionContext)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot pushdown the subquery since not all subqueries " "in the UNION have the partition column in the same " "position", "Each leaf query of the UNION should return the " "partition column in the same position and all joins " "must be on the partition column", NULL); } } else if (!RestrictionEquivalenceForPartitionKeys(plannerRestrictionContext)) { StringInfo errorMessage = makeStringInfo(); bool isMergeCmd = IsMergeQuery(originalQuery); appendStringInfo(errorMessage, "%s" "only supported when all distributed tables are " "co-located and joined on their distribution columns", isMergeCmd ? "MERGE command is " : "complex joins are "); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errorMessage->data, NULL, NULL); } /* we shouldn't allow reference tables in the FROM clause when the query has sublinks */ DeferredErrorMessage *error = DeferErrorIfFromClauseRecurs(originalQuery); if (error) { return error; } error = DeferredErrorIfUnsupportedRecurringTuplesJoin(plannerRestrictionContext, plannerPhase); if (error) { return error; } /* * We first extract all the queries that appear in the original query. Later, * we delete the original query given that error rules does not apply to the * top level query. For instance, we could support any LIMIT/ORDER BY on the * top level query. */ ExtractQueryWalker((Node *) originalQuery, &subqueryList); subqueryList = list_delete(subqueryList, originalQuery); /* iterate on the subquery list and error out accordingly */ foreach(subqueryCell, subqueryList) { Query *subquery = lfirst(subqueryCell); error = DeferErrorIfCannotPushdownSubquery(subquery, outerMostQueryHasLimit); if (error) { return error; } } return NULL; } /* * DeferErrorIfFromClauseRecurs returns a deferred error if the * given query is not suitable for subquery pushdown. * * While planning sublinks, we rely on Postgres in the sense that it converts some of * sublinks into joins. * * In some cases, sublinks are pulled up and converted into outer joins. Those cases * are already handled with RecursivelyPlanRecurringTupleOuterJoinWalker() or thrown * an error for in DeferredErrorIfUnsupportedRecurringTuplesJoin(). * * If the sublinks are not pulled up, we should still error out in if the expression * in the FROM clause would recur for every shard in a subquery on the WHERE clause. * * Otherwise, the result would include duplicate rows. */ static DeferredErrorMessage * DeferErrorIfFromClauseRecurs(Query *queryTree) { if (!queryTree->hasSubLinks) { return NULL; } RecurringTuplesType recurType = FromClauseRecurringTupleType(queryTree); if (recurType == RECURRING_TUPLES_REFERENCE_TABLE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains a reference table", NULL, NULL); } else if (recurType == RECURRING_TUPLES_FUNCTION) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains a set returning function", NULL, NULL); } else if (recurType == RECURRING_TUPLES_RESULT_FUNCTION) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains a CTE or subquery", NULL, NULL); } else if (recurType == RECURRING_TUPLES_EMPTY_JOIN_TREE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains a subquery without FROM", NULL, NULL); } else if (recurType == RECURRING_TUPLES_VALUES) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains VALUES", NULL, NULL); } else if (recurType == RECURRING_TUPLES_JSON_TABLE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "correlated subqueries are not supported when " "the FROM clause contains JSON_TABLE", NULL, NULL); } /* * We get here when there is neither a distributed table, nor recurring tuples. * That usually means that there isn't a FROM at all (only sublinks), this * implies that queryTree is recurring, but whether this is a problem depends * on outer queries, not on queryTree itself. */ return NULL; } /* * FromClauseRecurringTupleType returns tuple recurrence information * in query result based on range table entries in from clause. * * Returned information is used to prepare appropriate deferred error * message for subquery pushdown checks. */ static RecurringTuplesType FromClauseRecurringTupleType(Query *queryTree) { RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; if (HasEmptyJoinTree(queryTree)) { return RECURRING_TUPLES_EMPTY_JOIN_TREE; } if (FindNodeMatchingCheckFunctionInRangeTableList(queryTree->rtable, IsDistributedTableRTE)) { /* * There is a distributed table somewhere in the FROM clause. * * In the typical case this means that the query does not recur, * but there are two exceptions: * * - outer joins such as reference_table LEFT JOIN distributed_table * - FROM reference_table WHERE .. (SELECT .. FROM distributed_table) .. * * However, we check all subqueries and joins separately, so we would * find such conditions in other calls. */ return RECURRING_TUPLES_INVALID; } /* * Try to figure out which type of recurring tuples we have to produce a * relevant error message. If there are several we'll pick the first one. */ ContainsRecurringRangeTable(queryTree->rtable, &recurType); return recurType; } /* * DeferredErrorIfUnsupportedRecurringTuplesJoin returns a DeferredError if * there exists a join between a recurring rel (such as reference tables * and intermediate_results) and a non-recurring rel (such as distributed tables * and subqueries that we can push-down to worker nodes) when plannerPhase is * true, so that we try to recursively plan these joins. * During recursive planning phase, we either replace those with recursive plans * or leave them if it is safe to push-down. * During the logical planning phase (plannerPhase is false), we only check if * such queries have lateral subqueries. */ static DeferredErrorMessage * DeferredErrorIfUnsupportedRecurringTuplesJoin(PlannerRestrictionContext * plannerRestrictionContext, bool plannerPhase) { List *joinRestrictionList = plannerRestrictionContext->joinRestrictionContext->joinRestrictionList; ListCell *joinRestrictionCell = NULL; RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; foreach(joinRestrictionCell, joinRestrictionList) { JoinRestriction *joinRestriction = (JoinRestriction *) lfirst( joinRestrictionCell); JoinType joinType = joinRestriction->joinType; PlannerInfo *plannerInfo = joinRestriction->plannerInfo; Relids innerrelRelids = joinRestriction->innerrelRelids; Relids outerrelRelids = joinRestriction->outerrelRelids; /* * This loop aims to determine whether this join is between a recurring * rel and a non-recurring rel, and if so, whether it can yield an incorrect * result set due to recurring tuples. * * For outer joins, this can only happen if it's a lateral outer join * where the inner distributed subquery references the recurring outer * rel. This because, such outer joins should not appear here because * the recursive planner (RecursivelyPlanRecurringTupleOuterJoinWalker) * should have already planned the non-recurring side if it wasn't a * lateral join. For this reason, if the outer join is between a recurring * rel --on the outer side-- and a non-recurring rel --on the other side--, * we throw an error assuming that it's a lateral outer join. * Also note that; in the context of outer joins, we only check left outer * and full outer joins because PostgreSQL converts right joins to left * joins before passing them through "set_join_pathlist_hook"s. * * For semi / anti joins, we anyway throw an error when the inner * side is a distributed subquery that references a recurring outer rel * (in the FROM clause) thanks to DeferErrorIfFromClauseRecurs. And when * the inner side is a recurring rel and the outer side a non-recurring * one, then the non-recurring side can't reference the recurring side * anyway. * * For those reasons, here we perform below lateral join checks only for * outer (except anti) / inner joins but not for anti / semi joins. */ if (joinType == JOIN_LEFT) { if (RelationInfoContainsOnlyRecurringTuples(plannerInfo, innerrelRelids)) { /* inner side only contains recurring rels */ continue; } if (RelationInfoContainsOnlyRecurringTuples(plannerInfo, outerrelRelids)) { if (plannerPhase) { /* * We have not yet tried to recursively plan this join, we should * defer an error. */ recurType = FetchFirstRecurType(plannerInfo, outerrelRelids); break; } /* * Inner side contains distributed rels but the outer side only * contains recurring rels, might be an unsupported lateral outer * join. * Note that plannerInfo->hasLateralRTEs is not always set to * true, so here we check rtes, see ContainsLateralSubquery for details. */ if (ContainsLateralSubquery(plannerInfo)) { recurType = FetchFirstRecurType(plannerInfo, outerrelRelids); break; } } } else if (joinType == JOIN_FULL) { bool innerContainOnlyRecurring = RelationInfoContainsOnlyRecurringTuples(plannerInfo, innerrelRelids); bool outerContainOnlyRecurring = RelationInfoContainsOnlyRecurringTuples(plannerInfo, outerrelRelids); if (innerContainOnlyRecurring && !outerContainOnlyRecurring) { /* * Right side contains distributed rels but the left side only * contains recurring rels, must be an unsupported lateral outer * join. */ recurType = FetchFirstRecurType(plannerInfo, innerrelRelids); break; } if (!innerContainOnlyRecurring && outerContainOnlyRecurring) { /* * Left side contains distributed rels but the right side only * contains recurring rels, must be an unsupported lateral outer * join. */ recurType = FetchFirstRecurType(plannerInfo, outerrelRelids); break; } } else if (joinType == JOIN_INNER && plannerInfo->hasLateralRTEs) { /* * Sometimes we cannot push down INNER JOINS when they have only * recurring tuples on one side and a lateral on the other side. * See comment on DeferredErrorIfUnsupportedLateralSubquery for * details. * * When planning inner joins, postgres can move RTEs from left to * right and from right to left. So we don't know on which side the * lateral join wil appear. Thus we try to find a side of the join * that only contains recurring tuples. And then we check the other * side to see if it contains an unsupported lateral join. * */ if (RelationInfoContainsOnlyRecurringTuples(plannerInfo, innerrelRelids)) { DeferredErrorMessage *deferredError = DeferredErrorIfUnsupportedLateralSubquery(plannerInfo, innerrelRelids, outerrelRelids); if (deferredError) { return deferredError; } } else if (RelationInfoContainsOnlyRecurringTuples(plannerInfo, outerrelRelids)) { /* * This branch uses "else if" instead of "if", because if both * sides contain only recurring tuples there will never be an * unsupported lateral subquery. */ DeferredErrorMessage *deferredError = DeferredErrorIfUnsupportedLateralSubquery(plannerInfo, outerrelRelids, innerrelRelids); if (deferredError) { return deferredError; } } } } if (recurType != RECURRING_TUPLES_INVALID) { char *errmsg = psprintf("cannot perform a lateral outer join when " "a distributed subquery references %s", RecurringTypeDescription(recurType)); return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, errmsg, NULL, NULL); } return NULL; } /* * CanPushdownSubquery checks if we can push down the given * subquery to worker nodes. */ bool CanPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit) { return DeferErrorIfCannotPushdownSubquery(subqueryTree, outerMostQueryHasLimit) == NULL; } /* * DeferErrorIfCannotPushdownSubquery checks if we can push down the given * subquery to worker nodes. If we cannot push down the subquery, this function * returns a deferred error. * * We can push down a subquery if it follows rules below: * a. If there is an aggregate, it must be grouped on partition column. * b. If there is a join, it must be between two regular tables or two subqueries. * We don't support join between a regular table and a subquery. And columns on * the join condition must be partition columns. * c. If there is a distinct clause, it must be on the partition column. * * This function is very similar to DeferErrorIfQueryNotSupported() in logical * planner, but we don't reuse it, because differently for subqueries we support * a subset of distinct, union and left joins. * * Note that this list of checks is not exhaustive, there can be some cases * which we let subquery to run but returned results would be wrong. Such as if * a subquery has a group by on another subquery which includes order by with * limit, we let this query to run, but results could be wrong depending on the * features of underlying tables. */ DeferredErrorMessage * DeferErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit) { bool preconditionsSatisfied = true; char *errorDetail = NULL; DeferredErrorMessage *deferredError = DeferErrorIfUnsupportedTableCombination( subqueryTree); if (deferredError) { return deferredError; } if (HasEmptyJoinTree(subqueryTree) && contain_mutable_functions((Node *) subqueryTree->targetList)) { preconditionsSatisfied = false; errorDetail = "Subqueries without a FROM clause can only contain immutable " "functions"; } /* * Correlated subqueries are effectively functions that are repeatedly called * for the values of the vars that point to the outer query. We can liberally * push down SQL features within such a function, as long as co-located join * checks are applied. */ if (!ContainsReferencesToOuterQuery(subqueryTree)) { deferredError = DeferErrorIfSubqueryRequiresMerge(subqueryTree, false, "another query"); if (deferredError) { return deferredError; } } /* * Limit is partially supported when SubqueryPushdown is set. * The outermost query must have a limit clause. */ if (subqueryTree->limitCount && SubqueryPushdown && !outerMostQueryHasLimit) { preconditionsSatisfied = false; errorDetail = "Limit in subquery without limit in the outermost query is " "unsupported"; } if (subqueryTree->setOperations) { deferredError = DeferErrorIfUnsupportedUnionQuery(subqueryTree); if (deferredError) { return deferredError; } } if (subqueryTree->hasRecursive) { preconditionsSatisfied = false; errorDetail = "Recursive queries are currently unsupported"; } if (subqueryTree->cteList) { preconditionsSatisfied = false; errorDetail = "Common Table Expressions are currently unsupported"; } if (subqueryTree->hasForUpdate) { preconditionsSatisfied = false; errorDetail = "For Update/Share commands are currently unsupported"; } /* grouping sets are not allowed in subqueries*/ if (subqueryTree->groupingSets) { preconditionsSatisfied = false; errorDetail = "could not run distributed query with GROUPING SETS, CUBE, " "or ROLLUP"; } deferredError = DeferErrorIfFromClauseRecurs(subqueryTree); if (deferredError) { return deferredError; } /* finally check and return deferred if not satisfied */ if (!preconditionsSatisfied) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", errorDetail, NULL); } return NULL; } /* * FlattenGroupExprs flattens the GROUP BY expressions in the query tree * by replacing VAR nodes referencing the GROUP range table with the actual * GROUP BY expression. This is used by Citus planning to ensure correctness * when analysing and building the distributed plan. */ void FlattenGroupExprs(Query *queryTree) { #if PG_VERSION_NUM >= PG_VERSION_18 if (queryTree->hasGroupRTE) { queryTree->targetList = (List *) flatten_group_exprs(NULL, queryTree, (Node *) queryTree->targetList); queryTree->havingQual = flatten_group_exprs(NULL, queryTree, queryTree->havingQual); } #endif } /* * DeferErrorIfSubqueryRequiresMerge returns a deferred error if the subquery * requires a merge step on the coordinator (e.g. limit, group by non-distribution * column, etc.). */ static DeferredErrorMessage * DeferErrorIfSubqueryRequiresMerge(Query *subqueryTree, bool lateral, char *referencedThing) { bool preconditionsSatisfied = true; char *errorDetail = NULL; char *lateralString = lateral ? "lateral " : ""; if (subqueryTree->limitOffset) { preconditionsSatisfied = false; errorDetail = psprintf("Offset clause is currently unsupported when a %ssubquery " "references a column from %s", lateralString, referencedThing); } /* limit is not supported when SubqueryPushdown is not set */ if (subqueryTree->limitCount && !SubqueryPushdown) { preconditionsSatisfied = false; errorDetail = psprintf("Limit clause is currently unsupported when a " "%ssubquery references a column from %s", lateralString, referencedThing); } /* group clause list must include partition column */ if (subqueryTree->groupClause) { List *groupClauseList = subqueryTree->groupClause; List *targetEntryList = subqueryTree->targetList; List *groupTargetEntryList = GroupTargetEntryList(groupClauseList, targetEntryList); bool groupOnPartitionColumn = TargetListOnPartitionColumn(subqueryTree, groupTargetEntryList); if (!groupOnPartitionColumn) { preconditionsSatisfied = false; errorDetail = psprintf("Group by list without partition column is currently " "unsupported when a %ssubquery references a column " "from %s", lateralString, referencedThing); } } /* we don't support aggregates without group by */ if (subqueryTree->hasAggs && (subqueryTree->groupClause == NULL)) { preconditionsSatisfied = false; errorDetail = psprintf("Aggregates without group by are currently unsupported " "when a %ssubquery references a column from %s", lateralString, referencedThing); } /* having clause without group by on partition column is not supported */ if (subqueryTree->havingQual && (subqueryTree->groupClause == NULL)) { preconditionsSatisfied = false; errorDetail = psprintf("Having qual without group by on partition column is " "currently unsupported when a %ssubquery references " "a column from %s", lateralString, referencedThing); } /* * We support window functions when the window function * is partitioned on distribution column. */ StringInfo errorInfo = NULL; if (subqueryTree->hasWindowFuncs && !SafeToPushdownWindowFunction(subqueryTree, &errorInfo)) { errorDetail = (char *) errorInfo->data; preconditionsSatisfied = false; } /* distinct clause list must include partition column */ if (subqueryTree->distinctClause) { List *distinctClauseList = subqueryTree->distinctClause; List *targetEntryList = subqueryTree->targetList; List *distinctTargetEntryList = GroupTargetEntryList(distinctClauseList, targetEntryList); bool distinctOnPartitionColumn = TargetListOnPartitionColumn(subqueryTree, distinctTargetEntryList); if (!distinctOnPartitionColumn) { preconditionsSatisfied = false; errorDetail = "Distinct on columns without partition column is " "currently unsupported"; } } /* finally check and return deferred if not satisfied */ if (!preconditionsSatisfied) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", errorDetail, NULL); } return NULL; } /* * DeferErrorIfUnsupportedTableCombination checks if the given query tree contains any * unsupported range table combinations. For this, the function walks over all * range tables in the join tree, and checks if they correspond to simple relations * or subqueries. It also checks if there is a join between a regular table and * a subquery and if join is on more than two range table entries. If any error is found, * a deferred error is returned. Else, NULL is returned. */ static DeferredErrorMessage * DeferErrorIfUnsupportedTableCombination(Query *queryTree) { List *rangeTableList = queryTree->rtable; List *joinTreeTableIndexList = NIL; int joinTreeTableIndex = 0; bool unsupportedTableCombination = false; char *errorDetail = NULL; /* * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach_declared_int(joinTreeTableIndex, joinTreeTableIndexList) { /* * Join tree's range table index starts from 1 in the query tree. But, * list indexes start from 0. */ int rangeTableListIndex = joinTreeTableIndex - 1; RangeTblEntry *rangeTableEntry = (RangeTblEntry *) list_nth(rangeTableList, rangeTableListIndex); /* * Check if the range table in the join tree is a simple relation, a * subquery, or immutable function. */ if (rangeTableEntry->rtekind == RTE_RELATION || rangeTableEntry->rtekind == RTE_SUBQUERY || rangeTableEntry->rtekind == RTE_RESULT || IsJsonTableRTE(rangeTableEntry)) { /* accepted */ } else if (rangeTableEntry->rtekind == RTE_VALUES) { /* * When GUC is set to -1, we disable materialization, when set to 0, * we materialize everything. Other values are compared against the * length of the values_lists. */ int valuesRowCount = list_length(rangeTableEntry->values_lists); if (ValuesMaterializationThreshold >= 0 && valuesRowCount > ValuesMaterializationThreshold) { unsupportedTableCombination = true; errorDetail = "VALUES has more than " "\"citus.values_materialization_threshold\" " "entries, so it is materialized"; } else if (contain_mutable_functions((Node *) rangeTableEntry->values_lists)) { /* VALUES should not contain mutable functions */ unsupportedTableCombination = true; errorDetail = "Only immutable functions can be used in VALUES"; } } else if (rangeTableEntry->rtekind == RTE_FUNCTION) { List *functionList = rangeTableEntry->functions; if (list_length(functionList) == 1 && ContainsReadIntermediateResultFunction(linitial(functionList))) { /* * The read_intermediate_result function is volatile, but we know * it has the same result across all nodes and can therefore treat * it as a reference table. */ } else if (contain_mutable_functions((Node *) functionList)) { unsupportedTableCombination = true; errorDetail = "Only immutable functions can be used as a table " "expressions in a multi-shard query"; } else { /* immutable function RTEs are treated as reference tables */ } } else if (rangeTableEntry->rtekind == RTE_CTE) { unsupportedTableCombination = true; errorDetail = "CTEs in subqueries are currently unsupported"; break; } else { unsupportedTableCombination = true; errorDetail = "Table expressions other than relations, subqueries, " "and immutable functions are currently unsupported"; break; } } /* finally check and error out if not satisfied */ if (unsupportedTableCombination) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", errorDetail, NULL); } return NULL; } /* * DeferErrorIfUnsupportedUnionQuery is a helper function for ErrorIfCannotPushdownSubquery(). * The function also errors out for set operations INTERSECT and EXCEPT. */ DeferredErrorMessage * DeferErrorIfUnsupportedUnionQuery(Query *subqueryTree) { List *setOperationStatementList = NIL; ListCell *setOperationStatmentCell = NULL; RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; ExtractSetOperationStatementWalker((Node *) subqueryTree->setOperations, &setOperationStatementList); foreach(setOperationStatmentCell, setOperationStatementList) { SetOperationStmt *setOperation = (SetOperationStmt *) lfirst(setOperationStatmentCell); Node *leftArg = setOperation->larg; Node *rightArg = setOperation->rarg; int leftArgRTI = 0; int rightArgRTI = 0; if (setOperation->op != SETOP_UNION) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "Intersect and Except are currently unsupported", NULL); } if (IsA(leftArg, RangeTblRef)) { leftArgRTI = ((RangeTblRef *) leftArg)->rtindex; Query *leftArgSubquery = rt_fetch(leftArgRTI, subqueryTree->rtable)->subquery; recurType = FromClauseRecurringTupleType(leftArgSubquery); if (recurType != RECURRING_TUPLES_INVALID) { break; } } if (IsA(rightArg, RangeTblRef)) { rightArgRTI = ((RangeTblRef *) rightArg)->rtindex; Query *rightArgSubquery = rt_fetch(rightArgRTI, subqueryTree->rtable)->subquery; recurType = FromClauseRecurringTupleType(rightArgSubquery); if (recurType != RECURRING_TUPLES_INVALID) { break; } } } if (recurType == RECURRING_TUPLES_REFERENCE_TABLE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "Reference tables are not supported with union operator", NULL); } else if (recurType == RECURRING_TUPLES_FUNCTION) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "Table functions are not supported with union operator", NULL); } else if (recurType == RECURRING_TUPLES_EMPTY_JOIN_TREE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "Subqueries without a FROM clause are not supported with " "union operator", NULL); } else if (recurType == RECURRING_TUPLES_RESULT_FUNCTION) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "Complex subqueries and CTEs are not supported within a " "UNION", NULL); } else if (recurType == RECURRING_TUPLES_VALUES) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "VALUES is not supported within a " "UNION", NULL); } else if (recurType == RECURRING_TUPLES_JSON_TABLE) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "cannot push down this subquery", "JSON_TABLE is not supported within a " "UNION", NULL); } return NULL; } /* * ExtractSetOperationStatementWalker walks over a set operations statment, * and finds all set operations in the tree. */ static bool ExtractSetOperationStatementWalker(Node *node, List **setOperationList) { if (node == NULL) { return false; } if (IsA(node, SetOperationStmt)) { SetOperationStmt *setOperation = (SetOperationStmt *) node; (*setOperationList) = lappend(*setOperationList, setOperation); } bool walkerResult = expression_tree_walker(node, ExtractSetOperationStatementWalker, setOperationList); return walkerResult; } /* * RelationInfoContainsOnlyRecurringTuples returns false if any of the relations in * a RelOptInfo is not recurring. */ static bool RelationInfoContainsOnlyRecurringTuples(PlannerInfo *plannerInfo, Relids relids) { int relationId = -1; while ((relationId = bms_next_member(relids, relationId)) >= 0) { /* outer join RTE check in PG16 */ if (IsRelOptOuterJoin(plannerInfo, relationId)) { continue; } RangeTblEntry *rangeTableEntry = plannerInfo->simple_rte_array[relationId]; if (FindNodeMatchingCheckFunctionInRangeTableList(list_make1(rangeTableEntry), IsDistributedTableRTE)) { /* we already found a distributed table, no need to check further */ return false; } /* * If there are no distributed tables, there should be at least * one recurring rte. */ RecurringTuplesType recurType PG_USED_FOR_ASSERTS_ONLY; Assert(ContainsRecurringRTE(rangeTableEntry, &recurType)); } return true; } /* * RecurringTypeDescription returns a discriptive string for the given * recurType. This string can be used in error messages to help the users * understand why a query cannot be planned. */ static char * RecurringTypeDescription(RecurringTuplesType recurType) { switch (recurType) { case RECURRING_TUPLES_REFERENCE_TABLE: { return "a reference table"; } case RECURRING_TUPLES_FUNCTION: { return "a table function"; } case RECURRING_TUPLES_EMPTY_JOIN_TREE: { return "a subquery without FROM"; } case RECURRING_TUPLES_RESULT_FUNCTION: { return "complex subqueries, CTEs or local tables"; } case RECURRING_TUPLES_VALUES: { return "a VALUES clause"; } case RECURRING_TUPLES_JSON_TABLE: { return "a JSON_TABLE"; } case RECURRING_TUPLES_INVALID: { /* * This branch should never be hit, but it's here just in case it * happens. */ return "an unknown recurring tuple"; } } /* * This should never be hit, but is needed to fix compiler warnings. */ return "an unknown recurring tuple"; } /* * ContainsReferencesToRelids determines whether the given query contains * any references that point to columns of the given relids. The given relids * should be from exactly one query level above the given query. * * If the function returns true, then foundRelid is set to the first relid that * was referenced. * * There are some queries where it cannot easily be determined if the relids * are used, e.g because the query contains placeholder vars. In those cases * this function returns true, because it's better to error out than to return * wrong results. But in these cases foundRelid is set to INVALID_RELID. */ static bool ContainsReferencesToRelids(Query *query, Relids relids, int *foundRelid) { RelidsReferenceWalkerContext context = { 0 }; context.level = 1; context.relids = relids; context.foundRelid = INVALID_RELID; int flags = 0; if (query_tree_walker(query, ContainsReferencesToRelidsWalker, &context, flags)) { *foundRelid = context.foundRelid; return true; } return false; } /* * ContainsReferencesToRelidsWalker determines whether the given query * contains any Vars that reference the relids in the context. * * ContainsReferencesToRelidsWalker recursively descends into subqueries * and increases the level by 1 before recursing. */ static bool ContainsReferencesToRelidsWalker(Node *node, RelidsReferenceWalkerContext *context) { if (node == NULL) { return false; } if (IsA(node, Var)) { Var *var = (Var *) node; if (var->varlevelsup == context->level && bms_is_member(var->varno, context->relids)) { context->foundRelid = var->varno; return true; } return false; } else if (IsA(node, Aggref)) { if (((Aggref *) node)->agglevelsup > context->level) { /* * TODO: Only return true when aggref points to an aggregate that * uses vars from a recurring tuple. */ return true; } } else if (IsA(node, GroupingFunc)) { if (((GroupingFunc *) node)->agglevelsup > context->level) { /* * TODO: Only return true when groupingfunc points to a grouping * func that uses vars from a recurring tuple. */ return true; } return false; } else if (IsA(node, PlaceHolderVar)) { if (((PlaceHolderVar *) node)->phlevelsup > context->level) { /* * TODO: Only return true when aggref points to a placeholdervar * that uses vars from a recurring tuple. */ return true; } } else if (IsA(node, Query)) { Query *query = (Query *) node; int flags = 0; context->level += 1; bool found = query_tree_walker(query, ContainsReferencesToRelidsWalker, context, flags); context->level -= 1; return found; } return expression_tree_walker(node, ContainsReferencesToRelidsWalker, context); } /* * DeferredErrorIfUnsupportedLateralSubquery returns true if * notFullyRecurringRelids contains a lateral subquery that we do not support. * * If there is an inner join with a lateral subquery we cannot * push it down when the following properties all hold: * 1. The lateral subquery contains some non recurring tuples * 2. The lateral subquery references a recurring tuple from * outside of the subquery (recurringRelids) * 3. The lateral subquery requires a merge step (e.g. a LIMIT) * 4. The reference to the recurring tuple should be something else than an * equality check on the distribution column, e.g. equality on a non * distribution column. * * Property number four is considered both hard to detect and * probably not used very often, so we only check for 1, 2 and 3. */ static DeferredErrorMessage * DeferredErrorIfUnsupportedLateralSubquery(PlannerInfo *plannerInfo, Relids recurringRelids, Relids notFullyRecurringRelids) { int relationId = -1; while ((relationId = bms_next_member(notFullyRecurringRelids, relationId)) >= 0) { RangeTblEntry *rangeTableEntry = plannerInfo->simple_rte_array[relationId]; if (!rangeTableEntry->lateral) { continue; } /* TODO: What about others kinds? */ if (rangeTableEntry->rtekind == RTE_SUBQUERY) { /* property number 1, contains non-recurring tuples */ if (!FindNodeMatchingCheckFunctionInRangeTableList( list_make1(rangeTableEntry), IsDistributedTableRTE)) { continue; } /* property number 2, references recurring tuple */ int recurringRelid = INVALID_RELID; if (!ContainsReferencesToRelids(rangeTableEntry->subquery, recurringRelids, &recurringRelid)) { continue; } char *recurTypeDescription = "an aggregate, grouping func or placeholder var coming from the outer query"; if (recurringRelid != INVALID_RELID) { RangeTblEntry *recurringRangeTableEntry = plannerInfo->simple_rte_array[recurringRelid]; RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; ContainsRecurringRTE(recurringRangeTableEntry, &recurType); recurTypeDescription = RecurringTypeDescription(recurType); /* * Add the alias for all recuring tuples where it is useful to * see them. We don't add it for VALUES and intermediate * results, because there the aliases are currently hardcoded * strings anyway. */ if (recurType != RECURRING_TUPLES_VALUES && recurType != RECURRING_TUPLES_RESULT_FUNCTION && recurType != RECURRING_TUPLES_JSON_TABLE) { recurTypeDescription = psprintf("%s (%s)", recurTypeDescription, recurringRangeTableEntry->eref-> aliasname); } } /* property number 3, has a merge step */ DeferredErrorMessage *deferredError = DeferErrorIfSubqueryRequiresMerge( rangeTableEntry->subquery, true, recurTypeDescription); if (deferredError) { return deferredError; } } } return NULL; } /* * ContainsLateralSubquery checks if the given plannerInfo contains any * lateral subqueries in its rtable. If it does, it returns true, otherwise false. */ static bool ContainsLateralSubquery(PlannerInfo *plannerInfo) { ListCell *lc; foreach(lc, plannerInfo->parse->rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); /* We are only interested in subqueries that are lateral */ if (rte->lateral && rte->rtekind == RTE_SUBQUERY) { return true; } } return false; } /* * FetchFirstRecurType checks whether the relationInfo * contains any recurring table expression, namely a reference table, * or immutable function. If found, FetchFirstRecurType * returns true. * * Note that since relation ids of relationInfo indexes to the range * table entry list of planner info, planner info is also passed. */ static RecurringTuplesType FetchFirstRecurType(PlannerInfo *plannerInfo, Relids relids) { RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; int relationId = -1; while ((relationId = bms_next_member(relids, relationId)) >= 0) { RangeTblEntry *rangeTableEntry = plannerInfo->simple_rte_array[relationId]; /* relationInfo has this range table entry */ if (ContainsRecurringRTE(rangeTableEntry, &recurType)) { return recurType; } } return recurType; } /* * ContainsRecurringRTE returns whether the range table entry contains * any entry that generates the same set of tuples when repeating it in * a query on different shards. */ static bool ContainsRecurringRTE(RangeTblEntry *rangeTableEntry, RecurringTuplesType *recurType) { return ContainsRecurringRangeTable(list_make1(rangeTableEntry), recurType); } /* * ContainsRecurringRangeTable returns whether the range table list contains * any entry that generates the same set of tuples when repeating it in * a query on different shards. */ static bool ContainsRecurringRangeTable(List *rangeTable, RecurringTuplesType *recurType) { return range_table_walker(rangeTable, HasRecurringTuples, recurType, QTW_EXAMINE_RTES_BEFORE); } /* * IsJsonTableRTE checks whether the RTE refers to a JSON_TABLE * table function, which was introduced in PostgreSQL 17. */ bool IsJsonTableRTE(RangeTblEntry *rte) { #if PG_VERSION_NUM >= PG_VERSION_17 if (rte == NULL) { return false; } return (rte->rtekind == RTE_TABLEFUNC && rte->tablefunc->functype == TFT_JSON_TABLE); #endif return false; } /* * HasRecurringTuples returns whether any part of the expression will generate * the same set of tuples in every query on shards when executing a distributed * query. */ static bool HasRecurringTuples(Node *node, RecurringTuplesType *recurType) { if (node == NULL) { return false; } if (IsA(node, RangeTblEntry)) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) node; if (rangeTableEntry->rtekind == RTE_RELATION) { Oid relationId = rangeTableEntry->relid; if (IsCitusTableType(relationId, REFERENCE_TABLE)) { *recurType = RECURRING_TUPLES_REFERENCE_TABLE; /* * Tuples from reference tables will recur in every query on shards * that includes it. */ return true; } } else if (rangeTableEntry->rtekind == RTE_FUNCTION) { List *functionList = rangeTableEntry->functions; if (list_length(functionList) == 1 && ContainsReadIntermediateResultFunction((Node *) functionList)) { *recurType = RECURRING_TUPLES_RESULT_FUNCTION; } else { *recurType = RECURRING_TUPLES_FUNCTION; } /* * Tuples from functions will recur in every query on shards that includes * it. */ return true; } else if (rangeTableEntry->rtekind == RTE_RESULT) { *recurType = RECURRING_TUPLES_EMPTY_JOIN_TREE; return true; } else if (rangeTableEntry->rtekind == RTE_VALUES) { *recurType = RECURRING_TUPLES_VALUES; return true; } else if (IsJsonTableRTE(rangeTableEntry)) { *recurType = RECURRING_TUPLES_JSON_TABLE; return true; } return false; } else if (IsA(node, Query)) { Query *query = (Query *) node; if (HasEmptyJoinTree(query)) { *recurType = RECURRING_TUPLES_EMPTY_JOIN_TREE; /* * Queries with empty join trees will recur in every query on shards * that includes it. */ return true; } return query_tree_walker((Query *) node, HasRecurringTuples, recurType, QTW_EXAMINE_RTES_BEFORE); } return expression_tree_walker(node, HasRecurringTuples, recurType); } /* * SubqueryPushdownMultiNodeTree creates logical plan for subquery pushdown logic. * Note that this logic will be changed in next iterations, so we decoupled it * from other parts of code although it causes some code duplication. * * Current subquery pushdown support in MultiTree logic requires a single range * table entry in the top most from clause. Therefore we inject a synthetic * query derived from the top level query and make it the only range table * entry for the top level query. This way we can push down any subquery joins * down to workers without invoking join order planner. */ static MultiNode * SubqueryPushdownMultiNodeTree(Query *originalQuery) { Query *queryTree = copyObject(originalQuery); /* * PG18+ need to flatten GROUP BY expressions to ensure correct processing * later on, such as identification of partition columns in GROUP BY. */ FlattenGroupExprs(queryTree); List *targetEntryList = queryTree->targetList; MultiCollect *subqueryCollectNode = CitusMakeNode(MultiCollect); /* verify we can perform distributed planning on this query */ DeferredErrorMessage *unsupportedQueryError = DeferErrorIfQueryNotSupported( queryTree); if (unsupportedQueryError != NULL) { RaiseDeferredError(unsupportedQueryError, ERROR); } /* * We would be creating a new Query and pushing down top level query's * contents down to it. Join and filter clauses in higher level query would * be transferred to lower query. Therefore after this function we would * only have a single range table entry in the top level query. We need to * create a target list entry in lower query for each column reference in * upper level query's target list and having clauses. Any column reference * in the upper query will be updated to have varno=1, and varattno= * of matching target entry in pushed down query. * Consider query * SELECT s1.a, sum(s2.c) * FROM (some subquery) s1, (some subquery) s2 * WHERE s1.a = s2.a * GROUP BY s1.a * HAVING avg(s2.b); * * We want to prepare a multi tree to avoid subquery joins at top level, * therefore above query is converted to an equivalent * SELECT worker_column_0, sum(worker_column_1) * FROM ( * SELECT * s1.a AS worker_column_0, * s2.c AS worker_column_1, * s2.b AS worker_column_2 * FROM (some subquery) s1, (some subquery) s2 * WHERE s1.a = s2.a) worker_subquery * GROUP BY worker_column_0 * HAVING avg(worker_column_2); * After this conversion MultiTree is created as follows * * MultiExtendedOpNode( * targetList : worker_column_0, sum(worker_column_1) * groupBy : worker_column_0 * having : avg(worker_column_2)) * --->MultiProject (worker_column_0, worker_column_1, worker_column_2) * --->---> MultiTable (subquery : worker_subquery) * * Master and worker queries will be created out of this MultiTree at later stages. */ /* * columnList contains all columns returned by subquery. Subquery target * entry list, subquery range table entry's column name list are derived from * columnList. Columns mentioned in multiProject node and multiExtendedOp * node are indexed with their respective position in columnList. */ List *targetColumnList = pull_vars_of_level((Node *) targetEntryList, 0); List *havingClauseColumnList = pull_var_clause_default(queryTree->havingQual); List *columnList = list_concat(targetColumnList, havingClauseColumnList); /* create a target entry for each unique column */ List *subqueryTargetEntryList = CreateSubqueryTargetListAndAdjustVars(columnList); /* new query only has target entries, join tree, and rtable*/ Query *pushedDownQuery = makeNode(Query); pushedDownQuery->commandType = queryTree->commandType; pushedDownQuery->targetList = subqueryTargetEntryList; pushedDownQuery->jointree = copyObject(queryTree->jointree); pushedDownQuery->rtable = copyObject(queryTree->rtable); pushedDownQuery->rteperminfos = copyObject(queryTree->rteperminfos); pushedDownQuery->setOperations = copyObject(queryTree->setOperations); pushedDownQuery->querySource = queryTree->querySource; pushedDownQuery->hasSubLinks = queryTree->hasSubLinks; #if PG_VERSION_NUM >= PG_VERSION_18 pushedDownQuery->hasGroupRTE = queryTree->hasGroupRTE; #endif MultiTable *subqueryNode = MultiSubqueryPushdownTable(pushedDownQuery); SetChild((MultiUnaryNode *) subqueryCollectNode, (MultiNode *) subqueryNode); MultiNode *currentTopNode = (MultiNode *) subqueryCollectNode; /* build project node for the columns to project */ MultiProject *projectNode = MultiProjectNode(targetEntryList); SetChild((MultiUnaryNode *) projectNode, currentTopNode); currentTopNode = (MultiNode *) projectNode; /* * We build the extended operator node to capture aggregate functions, group * clauses, sort clauses, limit/offset clauses, and expressions. We need to * distinguish between aggregates and expressions; and we address this later * in the logical optimizer. */ MultiExtendedOp *extendedOpNode = MultiExtendedOpNode(queryTree, originalQuery); /* * Postgres standard planner converts having qual node to a list of and * clauses and expects havingQual to be of type List when executing the * query later. This function is called on an original query, therefore * havingQual has not been converted yet. Perform conversion here. */ if (extendedOpNode->havingQual != NULL && !IsA(extendedOpNode->havingQual, List)) { extendedOpNode->havingQual = (Node *) make_ands_implicit((Expr *) extendedOpNode->havingQual); } /* * Group by on primary key allows all columns to appear in the target * list, but once we wrap the join tree into a subquery the GROUP BY * will no longer directly refer to the primary key and referencing * columns that are not in the GROUP BY would result in an error. To * prevent that we wrap all the columns that do not appear in the * GROUP BY in an any_value aggregate. */ if (extendedOpNode->groupClauseList != NIL) { extendedOpNode->targetList = (List *) WrapUngroupedVarsInAnyValueAggregate( (Node *) extendedOpNode->targetList, extendedOpNode->groupClauseList, extendedOpNode->targetList, true); extendedOpNode->havingQual = WrapUngroupedVarsInAnyValueAggregate( (Node *) extendedOpNode->havingQual, extendedOpNode->groupClauseList, extendedOpNode->targetList, false); } /* * Postgres standard planner evaluates expressions in the LIMIT/OFFSET clauses. * Since we're using original query here, we should manually evaluate the * expression on the LIMIT and OFFSET clauses. Note that logical optimizer * expects those clauses to be already evaluated. */ extendedOpNode->limitCount = PartiallyEvaluateExpression(extendedOpNode->limitCount, NULL); extendedOpNode->limitOffset = PartiallyEvaluateExpression(extendedOpNode->limitOffset, NULL); SetChild((MultiUnaryNode *) extendedOpNode, currentTopNode); currentTopNode = (MultiNode *) extendedOpNode; return currentTopNode; } /* * CreateSubqueryTargetListAndAdjustVars creates a target entry for each unique * column in the column list, adjusts the columns to point into the subquery target * list and returns the new subquery target list. */ static List * CreateSubqueryTargetListAndAdjustVars(List *columnList) { Var *column = NULL; List *subqueryTargetEntryList = NIL; foreach_declared_ptr(column, columnList) { /* * To avoid adding the same column multiple times, we first check whether there * is already a target entry containing a Var with the given varno and varattno. */ AttrNumber resNo = FindResnoForVarInTargetList(subqueryTargetEntryList, column->varno, column->varattno); if (resNo == InvalidAttrNumber) { /* Var is not yet on the target list, create a new entry */ resNo = list_length(subqueryTargetEntryList) + 1; /* * The join tree in the subquery is an exact duplicate of the original * query. Hence, we can make a copy of the original Var. However, if the * original Var was in a sublink it would be pointing up whereas now it * will be placed directly on the target list. Hence we reset the * varlevelsup. */ Var *subqueryTargetListVar = (Var *) copyObject(column); subqueryTargetListVar->varlevelsup = 0; TargetEntry *newTargetEntry = makeNode(TargetEntry); newTargetEntry->expr = (Expr *) subqueryTargetListVar; newTargetEntry->resname = WorkerColumnName(resNo); newTargetEntry->resjunk = false; newTargetEntry->resno = resNo; subqueryTargetEntryList = lappend(subqueryTargetEntryList, newTargetEntry); } /* * Change the original column reference to point to the target list * entry in the subquery. There is only 1 subquery, so the varno is 1. */ column->varno = 1; column->varattno = resNo; /* * 1 subquery means there is one range table entry so with Postgres 16+ we need * to ensure that column's varnullingrels - the set of join rels that can null * the var - is empty. Otherwise, when given the query, the Postgres planner * may attempt to access a non-existent range table and segfault, as in #7787. */ column->varnullingrels = NULL; } return subqueryTargetEntryList; } /* * FindResnoForVarInTargetList finds a Var on a target list that has the given varno * (range table entry number) and varattno (column number) and returns the resno * of the target list entry. */ static AttrNumber FindResnoForVarInTargetList(List *targetList, int varno, int varattno) { TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, targetList) { if (!IsA(targetEntry->expr, Var)) { continue; } Var *targetEntryVar = (Var *) targetEntry->expr; if (targetEntryVar->varno == varno && targetEntryVar->varattno == varattno) { return targetEntry->resno; } } return InvalidAttrNumber; } /* * MultiSubqueryPushdownTable creates a MultiTable from the given subquery, * populates column list and returns the multitable. */ static MultiTable * MultiSubqueryPushdownTable(Query *subquery) { StringInfo rteName = makeStringInfo(); List *columnNamesList = NIL; ListCell *targetEntryCell = NULL; appendStringInfo(rteName, "worker_subquery"); foreach(targetEntryCell, subquery->targetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); columnNamesList = lappend(columnNamesList, makeString(targetEntry->resname)); } MultiTable *subqueryTableNode = CitusMakeNode(MultiTable); subqueryTableNode->subquery = subquery; subqueryTableNode->relationId = SUBQUERY_PUSHDOWN_RELATION_ID; subqueryTableNode->rangeTableId = SUBQUERY_RANGE_TABLE_ID; subqueryTableNode->partitionColumn = PartitionColumnForPushedDownSubquery(subquery); subqueryTableNode->alias = makeNode(Alias); subqueryTableNode->alias->aliasname = rteName->data; subqueryTableNode->referenceNames = makeNode(Alias); subqueryTableNode->referenceNames->aliasname = rteName->data; subqueryTableNode->referenceNames->colnames = columnNamesList; return subqueryTableNode; } /* * PartitionColumnForPushedDownSubquery finds the partition column on the target * list of a pushed down subquery. */ static Var * PartitionColumnForPushedDownSubquery(Query *query) { List *targetEntryList = query->targetList; TargetEntry *targetEntry = NULL; foreach_declared_ptr(targetEntry, targetEntryList) { if (targetEntry->resjunk) { continue; } Expr *targetExpression = targetEntry->expr; if (IsA(targetExpression, Var)) { bool skipOuterVars = true; bool isPartitionColumn = IsPartitionColumn(targetExpression, query, skipOuterVars); if (isPartitionColumn) { Var *partitionColumn = copyObject((Var *) targetExpression); /* the pushed down subquery is the only range table entry */ partitionColumn->varno = 1; /* point the var to the position in the subquery target list */ partitionColumn->varattno = targetEntry->resno; return partitionColumn; } } } return NULL; } ================================================ FILE: src/backend/distributed/planner/recursive_planning.c ================================================ /*------------------------------------------------------------------------- * * recursive_planning.c * * Logic for calling the postgres planner recursively for CTEs and * non-pushdownable subqueries in distributed queries. * * PostgreSQL with Citus can execute 4 types of queries: * * - Postgres queries on local tables and functions. * * These queries can use all SQL features, but they may not reference * distributed tables. * * - Router queries that can be executed on a single by node by replacing * table names with shard names. * * These queries can use nearly all SQL features, but only if they have * a single-valued filter on the distribution column. * * - Multi-shard queries that can be executed by performing a task for each * shard in a distributed table and performing a merge step. * * These queries have limited SQL support. They may only include * subqueries if the subquery can be executed on each shard by replacing * table names with shard names and concatenating the result. * * These queries have very limited SQL support and only support basic * inner joins and subqueries without joins. * * To work around the limitations of these planners, we recursively call * the planner for CTEs and unsupported subqueries to obtain a list of * subplans. * * During execution, each subplan is executed separately through the method * that is appropriate for that query. The results are written to temporary * files on the workers. In the original query, the CTEs and subqueries are * replaced by mini-subqueries that read from the temporary files. * * This allows almost all SQL to be directly or indirectly supported, * because if all subqueries that contain distributed tables have been * replaced then what remains is a router query which can use nearly all * SQL features. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "catalog/pg_class.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "utils/builtins.h" #include "utils/guc.h" #include "utils/lsyscache.h" #include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/combine_query_planner.h" #include "distributed/commands/multi_copy.h" #include "distributed/distributed_planner.h" #include "distributed/distribution_column.h" #include "distributed/errormessage.h" #include "distributed/listutils.h" #include "distributed/local_distributed_join_planner.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/query_colocation_checker.h" #include "distributed/query_pushdown_planning.h" #include "distributed/query_utils.h" #include "distributed/recursive_planning.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/shard_pruning.h" #include "distributed/version_compat.h" bool EnableRecurringOuterJoinPushdown = true; bool EnableOuterJoinsWithPseudoconstantQualsPrePG17 = false; /* * RecursivePlanningContext is used to recursively plan subqueries * and CTEs, pull results to the coordinator, and push it back into * the workers. */ struct RecursivePlanningContextInternal { int level; uint64 planId; bool allDistributionKeysInQueryAreEqual; /* used for some optimizations */ List *subPlanList; PlannerRestrictionContext *plannerRestrictionContext; bool restrictionEquivalenceCheck; bool forceRecursivelyPlanRecurringOuterJoins; }; /* track depth of current recursive planner query */ static int recursivePlanningDepth = 0; /* * CteReferenceWalkerContext is used to collect CTE references in * CteReferenceListWalker. */ typedef struct CteReferenceWalkerContext { int level; List *cteReferenceList; } CteReferenceWalkerContext; /* * VarLevelsUpWalkerContext is used to find Vars in a (sub)query that * refer to upper levels and therefore cannot be planned separately. */ typedef struct VarLevelsUpWalkerContext { int level; } VarLevelsUpWalkerContext; /* local function forward declarations */ static DeferredErrorMessage * RecursivelyPlanSubqueriesAndCTEs(Query *query, RecursivePlanningContext * context); static bool ShouldRecursivelyPlanNonColocatedSubqueries(Query *subquery, RecursivePlanningContext * context); static bool ContainsSubquery(Query *query); static bool ShouldRecursivelyPlanOuterJoins(Query *query, RecursivePlanningContext *context); static void RecursivelyPlanNonColocatedSubqueries(Query *subquery, RecursivePlanningContext *context); static void RecursivelyPlanNonColocatedJoinWalker(Node *joinNode, ColocatedJoinChecker * colocatedJoinChecker, RecursivePlanningContext * recursivePlanningContext); static void RecursivelyPlanNonColocatedSubqueriesInWhere(Query *query, ColocatedJoinChecker * colocatedJoinChecker, RecursivePlanningContext * recursivePlanningContext); static bool RecursivelyPlanRecurringTupleOuterJoinWalker(Node *node, Query *query, RecursivePlanningContext * context, bool chainedJoin); static void RecursivelyPlanDistributedJoinNode(Node *node, Query *query, RecursivePlanningContext *context); static bool IsRTERefRecurring(RangeTblRef *rangeTableRef, Query *query); static List * SublinkListFromWhere(Query *originalQuery); static bool ExtractSublinkWalker(Node *node, List **sublinkList); static bool ShouldRecursivelyPlanSublinks(Query *query); static bool RecursivelyPlanAllSubqueries(Node *node, RecursivePlanningContext *planningContext); static DeferredErrorMessage * RecursivelyPlanCTEs(Query *query, RecursivePlanningContext *context); static bool RecursivelyPlanSubqueryWalker(Node *node, RecursivePlanningContext *context); static bool ShouldRecursivelyPlanSubquery(Query *subquery, RecursivePlanningContext *context); static bool AllDistributionKeysInSubqueryAreEqual(Query *subquery, PlannerRestrictionContext * restrictionContext); static bool ShouldRecursivelyPlanSetOperation(Query *query, RecursivePlanningContext *context); static bool RecursivelyPlanSubquery(Query *subquery, RecursivePlanningContext *planningContext); static void RecursivelyPlanSetOperations(Query *query, Node *node, RecursivePlanningContext *context); static bool IsLocalTableRteOrMatView(Node *node); static DistributedSubPlan * CreateDistributedSubPlan(uint32 subPlanId, Query *subPlanQuery); static bool CteReferenceListWalker(Node *node, CteReferenceWalkerContext *context); static bool ContainsReferencesToOuterQueryWalker(Node *node, VarLevelsUpWalkerContext *context); static bool NodeContainsSubqueryReferencingOuterQuery(Node *node); static void WrapFunctionsInSubqueries(Query *query); static void TransformFunctionRTE(RangeTblEntry *rangeTblEntry); static bool ShouldTransformRTE(RangeTblEntry *rangeTableEntry); static Query * BuildReadIntermediateResultsQuery(List *targetEntryList, List *columnAliasList, Const *resultIdConst, Oid functionOid, bool useBinaryCopyFormat); static Query * CreateOuterSubquery(RangeTblEntry *rangeTableEntry, List *outerSubqueryTargetList); static List * GenerateRequiredColNamesFromTargetList(List *targetList); static char * GetRelationNameAndAliasName(RangeTblEntry *rangeTablentry); static bool CanPushdownRecurringOuterJoinOnOuterRTE(RangeTblEntry *rte); static bool CanPushdownRecurringOuterJoinOnInnerVar(Var *innervar, RangeTblEntry *rte); static bool CanPushdownRecurringOuterJoin(JoinExpr *joinExpr, Query *query); #if PG_VERSION_NUM < PG_VERSION_17 static bool hasPseudoconstantQuals(RelationRestrictionContext * relationRestrictionContext); #endif /* * GenerateSubplansForSubqueriesAndCTEs is a wrapper around RecursivelyPlanSubqueriesAndCTEs. * The function returns the subplans if necessary. For the details of when/how subplans are * generated, see RecursivelyPlanSubqueriesAndCTEs(). * * Note that the input originalQuery query is modified if any subplans are generated. */ List * GenerateSubplansForSubqueriesAndCTEs(uint64 planId, Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, RouterPlanType routerPlan) { RecursivePlanningContext context; recursivePlanningDepth++; /* * Plan subqueries and CTEs that cannot be pushed down by recursively * calling the planner and add the resulting plans to subPlanList. */ context.level = 0; context.planId = planId; context.subPlanList = NIL; context.plannerRestrictionContext = plannerRestrictionContext; context.forceRecursivelyPlanRecurringOuterJoins = false; /* * Force recursive planning of recurring outer joins for these queries * since the planning error from the previous step is generated prior to * the actual planning attempt. */ if (routerPlan == DML_QUERY) { context.forceRecursivelyPlanRecurringOuterJoins = true; } /* * Calculating the distribution key equality upfront is a trade-off for us. * * When the originalQuery contains the distribution key equality, we'd be * able to skip further checks for each lower level subqueries (i.e., if the * all query contains distribution key equality, each subquery also contains * distribution key equality.) * * When the originalQuery doesn't contain the distribution key equality, * calculating this wouldn't help us at all, we should individually check * each each subquery and subquery joins among subqueries. */ context.allDistributionKeysInQueryAreEqual = AllDistributionKeysInQueryAreEqual(originalQuery, plannerRestrictionContext); DeferredErrorMessage *error = RecursivelyPlanSubqueriesAndCTEs(originalQuery, &context); if (error != NULL) { recursivePlanningDepth--; RaiseDeferredError(error, ERROR); } if (context.subPlanList && IsLoggableLevel(DEBUG1)) { StringInfo subPlanString = makeStringInfo(); pg_get_query_def(originalQuery, subPlanString); ereport(DEBUG1, (errmsg( "Plan " UINT64_FORMAT " query after replacing subqueries and CTEs: %s", planId, subPlanString->data))); } recursivePlanningDepth--; return context.subPlanList; } /* * RecursivelyPlanSubqueriesAndCTEs finds subqueries and CTEs that cannot be pushed down to * workers directly and instead plans them by recursively calling the planner and * adding the subplan to subPlanList. * * Subplans are executed prior to the distributed plan and the results are written * to temporary files on workers. * * CTE references are replaced by a subquery on the read_intermediate_result * function, which reads from the temporary file. * * If recursive planning results in an error then the error is returned. Otherwise, the * subplans will be added to subPlanList. */ static DeferredErrorMessage * RecursivelyPlanSubqueriesAndCTEs(Query *query, RecursivePlanningContext *context) { DeferredErrorMessage *error = RecursivelyPlanCTEs(query, context); if (error != NULL) { return error; } if (SubqueryPushdown) { /* * When the subquery_pushdown flag is enabled we make some hacks * to push down subqueries with LIMIT. Recursive planning would * valiantly do the right thing and try to recursively plan the * inner subqueries, but we don't really want it to because those * subqueries might not be supported and would be much slower. * * Instead, we skip recursive planning altogether when * subquery_pushdown is enabled. */ return NULL; } /* make sure function calls in joins are executed in the coordinator */ WrapFunctionsInSubqueries(query); /* descend into subqueries */ query_tree_walker(query, RecursivelyPlanSubqueryWalker, context, 0); /* * At this point, all CTEs, leaf subqueries containing local tables and * non-pushdownable subqueries have been replaced. We now check for * combinations of subqueries that cannot be pushed down (e.g. * UNION ). * * This code also runs for the top-level query, which allows us to support * top-level set operations. */ if (ShouldRecursivelyPlanSetOperation(query, context)) { RecursivelyPlanSetOperations(query, (Node *) query->setOperations, context); } if (query->havingQual != NULL) { if (NodeContainsSubqueryReferencingOuterQuery(query->havingQual)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Subqueries in HAVING cannot refer to outer query", NULL, NULL); } RecursivelyPlanAllSubqueries(query->havingQual, context); } /* * If the query doesn't have distribution key equality, * recursively plan some of its subqueries. */ if (ShouldRecursivelyPlanNonColocatedSubqueries(query, context)) { RecursivelyPlanNonColocatedSubqueries(query, context); } if (ShouldConvertLocalTableJoinsToSubqueries(query->rtable)) { /* * Logical planner cannot handle "local_table" [OUTER] JOIN "dist_table", or * a query with local table/citus local table and subquery. We convert local/citus local * tables to a subquery until they can be planned. */ RecursivelyPlanLocalTableJoins(query, context); } /* * Similarly, logical planner cannot handle outer joins when the outer rel * is recurring, such as " LEFT JOIN ". In that case, * we convert distributed table into a subquery and recursively plan inner * side of the outer join. That way, inner rel gets converted into an intermediate * result and logical planner can handle the new query since it's of the from * " LEFT JOIN ". */ if (ShouldRecursivelyPlanOuterJoins(query, context)) { RecursivelyPlanRecurringTupleOuterJoinWalker((Node *) query->jointree, query, context, false); } /* * If the FROM clause is recurring (does not contain a distributed table), * then we cannot have any distributed tables appearing in subqueries in * the SELECT and WHERE clauses. * * We do the sublink conversations at the end of the recursive planning * because earlier steps might have transformed the query into a * shape that needs recursively planning the sublinks. */ if (ShouldRecursivelyPlanSublinks(query)) { /* replace all subqueries in the WHERE clause */ if (query->jointree && query->jointree->quals) { RecursivelyPlanAllSubqueries((Node *) query->jointree->quals, context); } /* replace all subqueries in the SELECT clause */ RecursivelyPlanAllSubqueries((Node *) query->targetList, context); } return NULL; } /* * GetPlannerRestrictionContext returns the planner restriction context * from the given context. */ PlannerRestrictionContext * GetPlannerRestrictionContext(RecursivePlanningContext *recursivePlanningContext) { return recursivePlanningContext->plannerRestrictionContext; } /* * ShouldRecursivelyPlanNonColocatedSubqueries returns true if the input query contains joins * that are not on the distribution key. * * * Note that at the point that this function is called, we've already recursively planned all * the leaf subqueries. Thus, we're actually checking whether the joins among the subqueries * on the distribution key or not. */ static bool ShouldRecursivelyPlanNonColocatedSubqueries(Query *subquery, RecursivePlanningContext *context) { /* * If the input query already contains the equality, simply return since it is not * possible to find any non colocated subqueries. */ if (context->allDistributionKeysInQueryAreEqual) { return false; } /* * This check helps us in two ways: * (i) We're not targeting queries that don't include subqueries at all, * they should go through regular planning. * (ii) Lower level subqueries are already recursively planned, so we should * only bother non-colocated subquery joins, which only happens when * there are subqueries. */ if (!ContainsSubquery(subquery)) { return false; } /* direct joins with local tables are not supported by any of Citus planners */ if (FindNodeMatchingCheckFunctionInRangeTableList(subquery->rtable, IsLocalTableRteOrMatView)) { return false; } /* * Finally, check whether this subquery contains distribution key equality or not. */ if (!AllDistributionKeysInSubqueryAreEqual(subquery, context->plannerRestrictionContext)) { return true; } return false; } /* * ContainsSubquery returns true if the input query contains any subqueries * in the FROM or WHERE clauses. */ static bool ContainsSubquery(Query *query) { return JoinTreeContainsSubquery(query) || WhereOrHavingClauseContainsSubquery(query); } /* * ShouldRecursivelyPlanOuterJoins returns true if the JoinRestrictionContext * that given RecursivePlanningContext holds implies that the query has outer * join(s) that might need to be recursively planned. */ static bool ShouldRecursivelyPlanOuterJoins(Query *query, RecursivePlanningContext *context) { if (!context || !context->plannerRestrictionContext || !context->plannerRestrictionContext->joinRestrictionContext) { ereport(ERROR, (errmsg("unexpectedly got NULL pointer in recursive " "planning context"))); } bool hasOuterJoin = context->plannerRestrictionContext->joinRestrictionContext->hasOuterJoin; #if PG_VERSION_NUM < PG_VERSION_17 if (!EnableOuterJoinsWithPseudoconstantQualsPrePG17 && !hasOuterJoin) { /* * PG16 commit 695f5deb7902865901eb2d50a70523af655c3a00 * disallows replacing joins with scans in queries with pseudoconstant quals. * This commit prevents the set_join_pathlist_hook from being called * if any of the join restrictions is a pseudo-constant. * So in these cases, citus has no info on the join, never sees that the query * has an outer join, and ends up producing an incorrect plan. * PG17 fixes this by commit 9e9931d2bf40e2fea447d779c2e133c2c1256ef3 * Therefore, we take this extra measure here for PG versions less than 17. * hasOuterJoin can never be true when set_join_pathlist_hook is absent. */ if (hasPseudoconstantQuals( context->plannerRestrictionContext->relationRestrictionContext) && FindNodeMatchingCheckFunction((Node *) query->jointree, IsOuterJoinExpr)) { ereport(ERROR, (errmsg("Distributed queries with outer joins and " "pseudoconstant quals are not supported in PG16."), errdetail( "PG16 disallows replacing joins with scans when the" " query has pseudoconstant quals"), errhint("Consider upgrading your PG version to PG17+"))); } } #endif return hasOuterJoin; } /* * RecursivelyPlanNonColocatedSubqueries gets a query which includes one or more * other subqueries that are not joined on their distribution keys. The function * tries to recursively plan some of the subqueries to make the input query * executable by Citus. * * The function picks an anchor subquery and iterates on the remaining subqueries. * Whenever it finds a non colocated subquery with the anchor subquery, the function * decides to recursively plan the non colocated subquery. * * The function first handles subqueries in FROM clause (i.e., jointree->fromlist) and then * subqueries in WHERE clause (i.e., jointree->quals). * * The function does not treat outer joins seperately. Thus, we might end up with * a query where the function decides to recursively plan an outer side of an outer * join (i.e., LEFT side of LEFT JOIN). For simplicity, we chose to do so and handle * outer joins with a seperate pass on the join tree. */ static void RecursivelyPlanNonColocatedSubqueries(Query *subquery, RecursivePlanningContext *context) { FromExpr *joinTree = subquery->jointree; /* create the context for the non colocated subquery planning */ PlannerRestrictionContext *restrictionContext = context->plannerRestrictionContext; ColocatedJoinChecker colocatedJoinChecker = CreateColocatedJoinChecker(subquery, restrictionContext); /* * Although this is a rare case, we weren't able to pick an anchor * range table entry, so we cannot continue. */ if (colocatedJoinChecker.anchorRelationRestrictionList == NIL) { return; } /* handle from clause subqueries first */ RecursivelyPlanNonColocatedJoinWalker((Node *) joinTree, &colocatedJoinChecker, context); /* handle subqueries in WHERE clause */ RecursivelyPlanNonColocatedSubqueriesInWhere(subquery, &colocatedJoinChecker, context); } /* * RecursivelyPlanNonColocatedJoinWalker gets a join node and walks over it to find * subqueries that live under the node. * * When a subquery found, it's checked whether the subquery is colocated with the * anchor subquery specified in the nonColocatedJoinContext. If not, * the subquery is recursively planned. */ static void RecursivelyPlanNonColocatedJoinWalker(Node *joinNode, ColocatedJoinChecker *colocatedJoinChecker, RecursivePlanningContext *recursivePlanningContext) { if (joinNode == NULL) { return; } else if (IsA(joinNode, FromExpr)) { FromExpr *fromExpr = (FromExpr *) joinNode; ListCell *fromExprCell; /* * For each element of the from list, check whether the element is * colocated with the anchor subquery by recursing until we * find the subqueries. */ foreach(fromExprCell, fromExpr->fromlist) { Node *fromElement = (Node *) lfirst(fromExprCell); RecursivelyPlanNonColocatedJoinWalker(fromElement, colocatedJoinChecker, recursivePlanningContext); } } else if (IsA(joinNode, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) joinNode; /* recurse into the left subtree */ RecursivelyPlanNonColocatedJoinWalker(joinExpr->larg, colocatedJoinChecker, recursivePlanningContext); /* recurse into the right subtree */ RecursivelyPlanNonColocatedJoinWalker(joinExpr->rarg, colocatedJoinChecker, recursivePlanningContext); } else if (IsA(joinNode, RangeTblRef)) { int rangeTableIndex = ((RangeTblRef *) joinNode)->rtindex; List *rangeTableList = colocatedJoinChecker->subquery->rtable; RangeTblEntry *rte = rt_fetch(rangeTableIndex, rangeTableList); /* we're only interested in subqueries for now */ if (rte->rtekind != RTE_SUBQUERY) { return; } /* * If the subquery is not colocated with the anchor subquery, * recursively plan it. */ Query *subquery = rte->subquery; if (!SubqueryColocated(subquery, colocatedJoinChecker)) { RecursivelyPlanSubquery(subquery, recursivePlanningContext); } } else { pg_unreachable(); } } /* * RecursivelyPlanNonColocatedSubqueriesInWhere gets a query and walks over its * sublinks to find subqueries that live in WHERE clause. * * When a subquery found, it's checked whether the subquery is colocated with the * anchor subquery specified in the nonColocatedJoinContext. If not, * the subquery is recursively planned. */ static void RecursivelyPlanNonColocatedSubqueriesInWhere(Query *query, ColocatedJoinChecker *colocatedJoinChecker, RecursivePlanningContext * recursivePlanningContext) { List *sublinkList = SublinkListFromWhere(query); ListCell *sublinkCell = NULL; foreach(sublinkCell, sublinkList) { SubLink *sublink = (SubLink *) lfirst(sublinkCell); Query *subselect = (Query *) sublink->subselect; /* subselect is probably never NULL, but anyway lets keep the check */ if (subselect == NULL) { continue; } if (!SubqueryColocated(subselect, colocatedJoinChecker)) { RecursivelyPlanSubquery(subselect, recursivePlanningContext); } } } /* * RecursivelyPlanRecurringTupleOuterJoinWalker descends into a join tree and * recursively plans all non-recurring (i.e., distributed) rels that that * participate in an outer join expression together with a recurring rel, * such as in " LEFT JOIN ", i.e., * where the recurring rel causes returning recurring tuples from the worker * nodes. * * Returns true if given node is recurring. * * See RecursivelyPlanDistributedJoinNode() function for the explanation on * what does it mean for a node to be "recurring" or "distributed". */ static bool RecursivelyPlanRecurringTupleOuterJoinWalker(Node *node, Query *query, RecursivePlanningContext * recursivePlanningContext, bool chainedJoin) { if (node == NULL) { return false; } else if (IsA(node, FromExpr)) { FromExpr *fromExpr = (FromExpr *) node; ListCell *fromExprCell; /* search for join trees in each FROM element */ foreach(fromExprCell, fromExpr->fromlist) { Node *fromElement = (Node *) lfirst(fromExprCell); RecursivelyPlanRecurringTupleOuterJoinWalker(fromElement, query, recursivePlanningContext, false); } /* * Can only appear during the top-level call and top-level callers * are not interested in the return value. Even more, we can't tell * whether a FromExpr is recurring or not. */ return false; } else if (IsA(node, JoinExpr)) { JoinExpr *joinExpr = (JoinExpr *) node; Node *leftNode = joinExpr->larg; Node *rightNode = joinExpr->rarg; /* * There may be recursively plannable outer joins deeper in the join tree. * * We first handle the sub join trees and then the top level one since the * top level join expression might not require recursive planning after * handling the sub join trees. */ bool leftNodeRecurs = RecursivelyPlanRecurringTupleOuterJoinWalker(leftNode, query, recursivePlanningContext, true); bool rightNodeRecurs = RecursivelyPlanRecurringTupleOuterJoinWalker(rightNode, query, recursivePlanningContext, true); switch (joinExpr->jointype) { case JOIN_LEFT: { /* left join */ if (leftNodeRecurs && !rightNodeRecurs) { if (recursivePlanningContext->forceRecursivelyPlanRecurringOuterJoins || chainedJoin || !CanPushdownRecurringOuterJoin(joinExpr, query)) { ereport(DEBUG1, (errmsg("recursively planning right side of " "the left join since the outer side " "is a recurring rel"))); RecursivelyPlanDistributedJoinNode(rightNode, query, recursivePlanningContext); } else { ereport(DEBUG3, (errmsg( "a push down safe left join with recurring left side"))); leftNodeRecurs = false; /* left node will be pushed down */ } } /* * A LEFT JOIN is recurring if the lhs is recurring. * Note that we might have converted the rhs into a recurring * one too if the lhs is recurring, but this anyway has no * effects when deciding whether a LEFT JOIN is recurring. */ return leftNodeRecurs; } case JOIN_RIGHT: { /* right join */ if (!leftNodeRecurs && rightNodeRecurs) { if (recursivePlanningContext->forceRecursivelyPlanRecurringOuterJoins || chainedJoin || !CanPushdownRecurringOuterJoin(joinExpr, query)) { ereport(DEBUG1, (errmsg("recursively planning left side of " "the right join since the outer side " "is a recurring rel"))); RecursivelyPlanDistributedJoinNode(leftNode, query, recursivePlanningContext); } else { ereport(DEBUG3, (errmsg( "a push down safe right join with recurring left side"))); rightNodeRecurs = false; /* right node will be pushed down */ } } /* * Similar to LEFT JOINs, a RIGHT JOIN is recurring if the rhs * is recurring. */ return rightNodeRecurs; } case JOIN_FULL: { /* * full join * full join */ if (leftNodeRecurs && !rightNodeRecurs) { ereport(DEBUG1, (errmsg("recursively planning right side of " "the full join since the other side " "is a recurring rel"))); RecursivelyPlanDistributedJoinNode(rightNode, query, recursivePlanningContext); } else if (!leftNodeRecurs && rightNodeRecurs) { ereport(DEBUG1, (errmsg("recursively planning left side of " "the full join since the other side " "is a recurring rel"))); RecursivelyPlanDistributedJoinNode(leftNode, query, recursivePlanningContext); } /* * An OUTER JOIN is recurring if any sides of the join is * recurring. As in other outer join types, it doesn't matter * whether the other side was / became recurring or not. */ return leftNodeRecurs || rightNodeRecurs; } case JOIN_INNER: { /* * We don't need to recursively plan non-outer joins and we * already descended into sub join trees to handle outer joins * buried in them. */ return leftNodeRecurs && rightNodeRecurs; } default: { ereport(ERROR, (errmsg("got unexpected join type (%d) when recursively " "planning a join", joinExpr->jointype))); } } } else if (IsA(node, RangeTblRef)) { return IsRTERefRecurring((RangeTblRef *) node, query); } else { ereport(ERROR, errmsg("got unexpected node type (%d) when recursively " "planning a join", nodeTag(node))); } } /* * RecursivelyPlanDistributedJoinNode is a helper function for * RecursivelyPlanRecurringTupleOuterJoinWalker that recursively plans given * distributed node that is known to be inner side of an outer join. * * Fails to do so if the distributed join node references the recurring one. * In that case, we don't throw an error here but instead we let * DeferredErrorIfUnsupportedRecurringTuplesJoin to so for a better error * message. * * We call a node "distributed" if it points to a distributed table or a * more complex object (i.e., a join tree or a subquery) that can be pushed * down to the worker nodes directly. For a join, this means that it's either * an INNER join where any side of it is a distributed table / a distributed * sub join tree, or an OUTER join where the outer side is a distributed table * / a distributed sub join tree. */ static void RecursivelyPlanDistributedJoinNode(Node *node, Query *query, RecursivePlanningContext *recursivePlanningContext) { if (IsA(node, JoinExpr)) { /* * This, for example, means that RecursivelyPlanRecurringTupleOuterJoinWalker * needs to plan inner side, i.e., " INNER JOIN ", * of the following join: * LEFT JOIN ( JOIN ) * * XXX: Ideally, we should handle such a sub join tree by moving * it into a subquery "as a whole" but this implies that we need to * rebuild the rtable and re-point all the Vars to the new rtable * indexes, so we've not implemented that yet. * * Instead, we recursively plan all the distributed tables in that * sub join tree. This is much more inefficient than the other * approach (since we lose the opportunity to push-down the whole * sub join tree into the workers) but is easier to implement. */ RecursivelyPlanDistributedJoinNode(((JoinExpr *) node)->larg, query, recursivePlanningContext); RecursivelyPlanDistributedJoinNode(((JoinExpr *) node)->rarg, query, recursivePlanningContext); return; } if (!IsA(node, RangeTblRef)) { ereport(ERROR, (errmsg("unexpected join node type (%d)", nodeTag(node)))); } RangeTblRef *rangeTableRef = (RangeTblRef *) node; if (IsRTERefRecurring(rangeTableRef, query)) { /* * Not the top-level callers but RecursivelyPlanDistributedJoinNode * might call itself for recurring nodes and need to skip them. */ return; } RangeTblEntry *distributedRte = rt_fetch(rangeTableRef->rtindex, query->rtable); if (distributedRte->rtekind == RTE_RELATION) { ereport(DEBUG1, (errmsg("recursively planning distributed relation %s " "since it is part of a distributed join node " "that is outer joined with a recurring rel", GetRelationNameAndAliasName(distributedRte)))); PlannerRestrictionContext *restrictionContext = GetPlannerRestrictionContext(recursivePlanningContext); List *requiredAttributes = RequiredAttrNumbersForRelation(distributedRte, restrictionContext); RTEPermissionInfo *perminfo = NULL; if (distributedRte->perminfoindex) { perminfo = getRTEPermissionInfo(query->rteperminfos, distributedRte); } ReplaceRTERelationWithRteSubquery(distributedRte, requiredAttributes, recursivePlanningContext, perminfo); } else if (distributedRte->rtekind == RTE_SUBQUERY) { /* * We don't try logging the subquery here because RecursivelyPlanSubquery * will anyway do so if the query doesn't reference the outer query. */ ereport(DEBUG1, (errmsg("recursively planning the distributed subquery " "since it is part of a distributed join node " "that is outer joined with a recurring rel"))); bool recursivelyPlanned = RecursivelyPlanSubquery(distributedRte->subquery, recursivePlanningContext); if (!recursivelyPlanned) { /* * RecursivelyPlanSubquery fails to plan a subquery only if it * contains references to the outer query. This means that, we can't * plan such outer joins (like ) * if it's a LATERAL join where the distributed side is a subquery that * references the outer side, as in, * * SELECT * FROM reference * LEFT JOIN LATERAL * (SELECT * FROM distributed WHERE reference.b > distributed.b) q * USING (a); */ Assert(ContainsReferencesToOuterQuery(distributedRte->subquery)); } } else { /* * We don't expect RecursivelyPlanRecurringTupleOuterJoinWalker to try recursively * plan such an RTE. */ ereport(ERROR, errmsg("got unexpected RTE type (%d) when recursively " "planning a join", distributedRte->rtekind)); } } /* * IsRTERefRecurring returns true if given rte reference points to a recurring * rte. * * If an rte points to a table, then we call it recurring if the table is not * a distributed table. Otherwise, e.g., if it points a query, then we call it * recurring if none of the rtes that belongs to the query point to a distributed * table. * * Note that it's safe to assume a subquery is not recurring if we have a rte reference * to a distributed table somewhere in the query tree. For example, considering * the subquery (q) of the the following query: * SELECT * FROM ref LEFT JOIN (SELECT * FROM ref LEFT dist) q, * one might think that it's not appropriate to call IsRTERefRecurring for subquery * (q). However, this is already not the case because this function is called * in the context of recursive planning and hence any query that contains * rtes pointing to distributed tables and that cannot be pushed down to worker * nodes should've been recursively planned already. This is because, the recursive * planner processes the queries in bottom-up fashion. For this reason, the subquery * in the example should've already be converted to the following before we check * the rte reference that points to the subquery (q): * SELECT * FROM ref LEFT JOIN (SELECT * FROM ref LEFT (SELECT * FROM read_intermediate_result()) dist_1) * That way, we wouldn't incorrectly say that (SELECT * FROM ref LEFT dist) is a * distributed subquery (due to having a reference to a distributed table). */ static bool IsRTERefRecurring(RangeTblRef *rangeTableRef, Query *query) { int rangeTableIndex = rangeTableRef->rtindex; List *rangeTableList = query->rtable; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableIndex, rangeTableList); return !FindNodeMatchingCheckFunctionInRangeTableList(list_make1(rangeTableEntry), IsDistributedTableRTE); } /* * SublinkListFromWhere finds the subquery nodes in the where clause of the given query. Note * that the function should be called on the original query given that postgres * standard_planner() may convert the subqueries in WHERE clause to joins. */ static List * SublinkListFromWhere(Query *originalQuery) { FromExpr *joinTree = originalQuery->jointree; List *sublinkList = NIL; if (!joinTree) { return NIL; } Node *queryQuals = joinTree->quals; ExtractSublinkWalker(queryQuals, &sublinkList); return sublinkList; } /* * ExtractSublinkWalker walks over a quals node, and finds all sublinks * in that node. */ static bool ExtractSublinkWalker(Node *node, List **sublinkList) { bool walkerResult = false; if (node == NULL) { return false; } if (IsA(node, SubLink)) { (*sublinkList) = lappend(*sublinkList, node); } else { walkerResult = expression_tree_walker(node, ExtractSublinkWalker, sublinkList); } return walkerResult; } /* * ShouldRecursivelyPlanSublinks returns true if the query has a recurring * FROM clause. */ static bool ShouldRecursivelyPlanSublinks(Query *query) { bool hasDistributedTable = (FindNodeMatchingCheckFunctionInRangeTableList( query->rtable, IsDistributedTableRTE)); return !hasDistributedTable; } /* * RecursivelyPlanAllSubqueries descends into an expression tree and recursively * plans all subqueries that contain at least one distributed table. The recursive * planning starts from the top of the input query. */ static bool RecursivelyPlanAllSubqueries(Node *node, RecursivePlanningContext *planningContext) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; if (FindNodeMatchingCheckFunctionInRangeTableList(query->rtable, IsCitusTableRTE)) { RecursivelyPlanSubquery(query, planningContext); } return false; } return expression_tree_walker(node, RecursivelyPlanAllSubqueries, planningContext); } /* * RecursivelyPlanCTEs plans all CTEs in the query by recursively calling the planner * The resulting plan is added to planningContext->subPlanList and CTE references * are replaced by subqueries that call read_intermediate_result, which reads the * intermediate result of the CTE after it is executed. * * Recursive and modifying CTEs are not yet supported and return an error. */ static DeferredErrorMessage * RecursivelyPlanCTEs(Query *query, RecursivePlanningContext *planningContext) { ListCell *cteCell = NULL; CteReferenceWalkerContext context = { -1, NIL }; if (query->cteList == NIL) { /* no CTEs, nothing to do */ return NULL; } if (query->hasRecursive) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "recursive CTEs are only supported when they " "contain a filter on the distribution column", NULL, NULL); } /* get all RTE_CTEs that point to CTEs from cteList */ CteReferenceListWalker((Node *) query, &context); foreach(cteCell, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *) lfirst(cteCell); char *cteName = cte->ctename; Query *subquery = (Query *) cte->ctequery; uint64 planId = planningContext->planId; List *cteTargetList = NIL; ListCell *rteCell = NULL; int replacedCtesCount = 0; if (ContainsReferencesToOuterQuery(subquery)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "CTEs that refer to other subqueries are not " "supported in multi-shard queries", NULL, NULL); } if (cte->cterefcount == 0 && subquery->commandType == CMD_SELECT) { /* * SELECT CTEs that aren't referenced aren't executed in postgres. * We don't need to generate a subplan for it and can take the rest * of this iteration off. */ continue; } uint32 subPlanId = list_length(planningContext->subPlanList) + 1; if (IsLoggableLevel(DEBUG1)) { StringInfo subPlanString = makeStringInfo(); pg_get_query_def(subquery, subPlanString); ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT "_%u for CTE %s: %s", planId, subPlanId, cteName, subPlanString->data))); } /* build a sub plan for the CTE */ DistributedSubPlan *subPlan = CreateDistributedSubPlan(subPlanId, subquery); planningContext->subPlanList = lappend(planningContext->subPlanList, subPlan); /* build the result_id parameter for the call to read_intermediate_result */ char *resultId = GenerateResultId(planId, subPlanId); if (subquery->returningList) { /* modifying CTE with returning */ cteTargetList = subquery->returningList; } else { /* regular SELECT CTE */ cteTargetList = subquery->targetList; } /* replace references to the CTE with a subquery that reads results */ Query *resultQuery = BuildSubPlanResultQuery(cteTargetList, cte->aliascolnames, resultId); foreach(rteCell, context.cteReferenceList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rteCell); if (rangeTableEntry->rtekind != RTE_CTE) { /* * This RTE pointed to a preceding CTE that was already replaced * by a subplan. */ continue; } if (strncmp(rangeTableEntry->ctename, cteName, NAMEDATALEN) == 0) { /* change the RTE_CTE into an RTE_SUBQUERY */ rangeTableEntry->rtekind = RTE_SUBQUERY; rangeTableEntry->ctename = NULL; rangeTableEntry->ctelevelsup = 0; if (replacedCtesCount == 0) { /* * Replace the first CTE reference with the result query directly. */ rangeTableEntry->subquery = resultQuery; } else { /* * Replace subsequent CTE references with a copy of the result * query. */ rangeTableEntry->subquery = copyObject(resultQuery); } replacedCtesCount++; } } Assert(cte->cterefcount == replacedCtesCount); } /* * All CTEs are now executed through subplans and RTE_CTEs pointing * to the CTE list have been replaced with subqueries. We can now * clear the cteList. */ query->cteList = NIL; return NULL; } /* * RecursivelyPlanSubqueryWalker recursively finds all the Query nodes and * recursively plans if necessary. */ static bool RecursivelyPlanSubqueryWalker(Node *node, RecursivePlanningContext *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; context->level += 1; /* * First, make sure any subqueries and CTEs within this subquery * are recursively planned if necessary. */ DeferredErrorMessage *error = RecursivelyPlanSubqueriesAndCTEs(query, context); if (error != NULL) { RaiseDeferredError(error, ERROR); } context->level -= 1; /* * Recursively plan this subquery if it cannot be pushed down and is * eligible for recursive planning. */ if (ShouldRecursivelyPlanSubquery(query, context)) { RecursivelyPlanSubquery(query, context); } /* we're done, no need to recurse anymore for this query */ return false; } return expression_tree_walker(node, RecursivelyPlanSubqueryWalker, context); } /* * ShouldRecursivelyPlanSubquery decides whether the input subquery should be recursively * planned or not. * * For the details, see the cases in the function. */ static bool ShouldRecursivelyPlanSubquery(Query *subquery, RecursivePlanningContext *context) { if (FindNodeMatchingCheckFunctionInRangeTableList(subquery->rtable, IsLocalTableRteOrMatView)) { /* * Postgres can always plan queries that don't require distributed planning. * Note that we need to check this first, otherwise the calls to the many other * Citus planner functions would error our due to local relations. * * TODO: We could only successfully create distributed plans with local tables * when the local tables are on the leaf queries and the upper level queries * do not contain any other local tables. */ } else if (CanPushdownSubquery(subquery, false)) { /* * We should do one more check for the distribution key equality. * * If the input query to the planner doesn't contain distribution key equality, * we should further check whether this individual subquery contains or not. * * If all relations are not joined on their distribution keys for the given * subquery, we cannot push push it down and therefore we should try to * recursively plan it. */ if (!context->allDistributionKeysInQueryAreEqual && !AllDistributionKeysInSubqueryAreEqual(subquery, context->plannerRestrictionContext)) { return true; } /* * Citus can pushdown this subquery, no need to recursively * plan which is much more expensive than pushdown. */ return false; } return true; } /* * AllDistributionKeysInSubqueryAreEqual is a wrapper function * for AllDistributionKeysInQueryAreEqual(). Here, we filter the * planner restrictions for the given subquery and do the restriction * equality checks on the filtered restriction. */ static bool AllDistributionKeysInSubqueryAreEqual(Query *subquery, PlannerRestrictionContext *restrictionContext) { /* we don't support distribution eq. checks for CTEs yet */ if (subquery->cteList != NIL) { return false; } PlannerRestrictionContext *filteredRestrictionContext = FilterPlannerRestrictionForQuery(restrictionContext, subquery); bool allDistributionKeysInSubqueryAreEqual = AllDistributionKeysInQueryAreEqual(subquery, filteredRestrictionContext); if (!allDistributionKeysInSubqueryAreEqual) { return false; } return true; } /* * ShouldRecursivelyPlanSetOperation determines whether the leaf queries of a * set operations tree need to be recursively planned in order to support the * query as a whole. */ static bool ShouldRecursivelyPlanSetOperation(Query *query, RecursivePlanningContext *context) { SetOperationStmt *setOperations = (SetOperationStmt *) query->setOperations; if (setOperations == NULL) { return false; } if (context->level == 0) { /* * We cannot push down top-level set operation. Recursively plan the * leaf nodes such that it becomes a router query. */ return true; } if (setOperations->op != SETOP_UNION) { /* * We can only push down UNION operaionts, plan other set operations * recursively. */ return true; } if (DeferErrorIfUnsupportedUnionQuery(query) != NULL) { /* * If at least one leaf query in the union is recurring, then all * leaf nodes need to be recurring. */ return true; } PlannerRestrictionContext *filteredRestrictionContext = FilterPlannerRestrictionForQuery(context->plannerRestrictionContext, query); if (!SafeToPushdownUnionSubquery(query, filteredRestrictionContext)) { /* * The distribution column is not in the same place in all sides * of the union, meaning we cannot determine distribution column * equivalence. Recursive planning is necessary. */ return true; } return false; } /* * RecursivelyPlanSetOperations descends into a tree of set operations * (e.g. UNION, INTERSECTS) and recursively plans all leaf nodes that * contain distributed tables. */ static void RecursivelyPlanSetOperations(Query *query, Node *node, RecursivePlanningContext *context) { if (IsA(node, SetOperationStmt)) { SetOperationStmt *setOperations = (SetOperationStmt *) node; RecursivelyPlanSetOperations(query, setOperations->larg, context); RecursivelyPlanSetOperations(query, setOperations->rarg, context); } else if (IsA(node, RangeTblRef)) { RangeTblRef *rangeTableRef = (RangeTblRef *) node; RangeTblEntry *rangeTableEntry = rt_fetch(rangeTableRef->rtindex, query->rtable); Query *subquery = rangeTableEntry->subquery; if (rangeTableEntry->rtekind == RTE_SUBQUERY && FindNodeMatchingCheckFunction((Node *) subquery, IsDistributedTableRTE)) { RecursivelyPlanSubquery(subquery, context); } } else { ereport(ERROR, (errmsg("unexpected node type (%d) while " "expecting set operations or " "range table references", nodeTag(node)))); } } /* * IsLocalTableRteOrMatView gets a node and returns true if the node is a range * table entry that points to a postgres local or citus local table or to a * materialized view. */ static bool IsLocalTableRteOrMatView(Node *node) { if (node == NULL) { return false; } if (!IsA(node, RangeTblEntry)) { return false; } RangeTblEntry *rangeTableEntry = (RangeTblEntry *) node; if (rangeTableEntry->rtekind != RTE_RELATION) { return false; } if (rangeTableEntry->relkind == RELKIND_VIEW) { return false; } Oid relationId = rangeTableEntry->relid; return IsRelationLocalTableOrMatView(relationId); } /* * IsRelationLocalTableOrMatView returns true if the given relation * is a citus local, local, or materialized view. */ bool IsRelationLocalTableOrMatView(Oid relationId) { if (!IsCitusTable(relationId)) { /* postgres local table or a materialized view */ return true; } else if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE)) { return true; } /* no local table found */ return false; } /* * RecursivelyPlanSubquery recursively plans a query, replaces it with a * result query and returns the subplan. * * Before we recursively plan the given subquery, we should ensure * that the subquery doesn't contain any references to the outer * queries (i.e., such queries cannot be separately planned). In * that case, the function doesn't recursively plan the input query * and immediately returns. Later, the planner decides on what to do * with the query. */ static bool RecursivelyPlanSubquery(Query *subquery, RecursivePlanningContext *planningContext) { uint64 planId = planningContext->planId; Query *debugQuery = NULL; if (ContainsReferencesToOuterQuery(subquery)) { elog(DEBUG2, "skipping recursive planning for the subquery since it " "contains references to outer queries"); return false; } /* * Subquery will go through the standard planner, thus to properly deparse it * we keep its copy: debugQuery. */ if (IsLoggableLevel(DEBUG1)) { debugQuery = copyObject(subquery); } /* * Create the subplan and append it to the list in the planning context. */ int subPlanId = list_length(planningContext->subPlanList) + 1; DistributedSubPlan *subPlan = CreateDistributedSubPlan(subPlanId, subquery); planningContext->subPlanList = lappend(planningContext->subPlanList, subPlan); /* build the result_id parameter for the call to read_intermediate_result */ char *resultId = GenerateResultId(planId, subPlanId); /* * BuildSubPlanResultQuery() can optionally use provided column aliases. * We do not need to send additional alias list for subqueries. */ Query *resultQuery = BuildSubPlanResultQuery(subquery->targetList, NIL, resultId); if (IsLoggableLevel(DEBUG1)) { StringInfo subqueryString = makeStringInfo(); pg_get_query_def(debugQuery, subqueryString); ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT "_%u for subquery %s", planId, subPlanId, subqueryString->data))); } /* finally update the input subquery to point the result query */ *subquery = *resultQuery; return true; } /* * CreateDistributedSubPlan creates a distributed subplan by recursively calling * the planner from the top, which may either generate a local plan or another * distributed plan, which can itself contain subplans. */ static DistributedSubPlan * CreateDistributedSubPlan(uint32 subPlanId, Query *subPlanQuery) { int cursorOptions = 0; if (ContainsReadIntermediateResultFunction((Node *) subPlanQuery)) { /* * Make sure we go through distributed planning if there are * read_intermediate_result calls, even if there are no distributed * tables in the query anymore. * * We cannot perform this check in the planner itself, since that * would also cause the workers to attempt distributed planning. */ cursorOptions |= CURSOR_OPT_FORCE_DISTRIBUTED; } DistributedSubPlan *subPlan = CitusMakeNode(DistributedSubPlan); subPlan->plan = planner(subPlanQuery, NULL, cursorOptions, NULL); subPlan->subPlanId = subPlanId; return subPlan; } /* * CteReferenceListWalker finds all references to CTEs in the top level of a query * and adds them to context->cteReferenceList. */ static bool CteReferenceListWalker(Node *node, CteReferenceWalkerContext *context) { if (node == NULL) { return false; } if (IsA(node, RangeTblEntry)) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) node; if (rangeTableEntry->rtekind == RTE_CTE && rangeTableEntry->ctelevelsup == context->level) { context->cteReferenceList = lappend(context->cteReferenceList, rangeTableEntry); } /* caller will descend into range table entry */ return false; } else if (IsA(node, Query)) { Query *query = (Query *) node; context->level += 1; query_tree_walker(query, CteReferenceListWalker, context, QTW_EXAMINE_RTES_BEFORE); context->level -= 1; return false; } else { return expression_tree_walker(node, CteReferenceListWalker, context); } } /* * ContainsReferencesToOuterQuery determines whether the given query contains * anything that points outside of the query itself. Such queries cannot be * planned recursively. */ bool ContainsReferencesToOuterQuery(Query *query) { VarLevelsUpWalkerContext context = { 0 }; int flags = 0; return query_tree_walker(query, ContainsReferencesToOuterQueryWalker, &context, flags); } /* * ContainsReferencesToOuterQueryWalker determines whether the given query * contains any Vars that point more than context->level levels up. * * ContainsReferencesToOuterQueryWalker recursively descends into subqueries * and increases the level by 1 before recursing. */ static bool ContainsReferencesToOuterQueryWalker(Node *node, VarLevelsUpWalkerContext *context) { if (node == NULL) { return false; } if (IsA(node, Var)) { if (((Var *) node)->varlevelsup > context->level) { return true; } return false; } else if (IsA(node, Aggref)) { if (((Aggref *) node)->agglevelsup > context->level) { return true; } } else if (IsA(node, GroupingFunc)) { if (((GroupingFunc *) node)->agglevelsup > context->level) { return true; } return false; } else if (IsA(node, PlaceHolderVar)) { if (((PlaceHolderVar *) node)->phlevelsup > context->level) { return true; } } else if (IsA(node, Query)) { Query *query = (Query *) node; int flags = 0; context->level += 1; bool found = query_tree_walker(query, ContainsReferencesToOuterQueryWalker, context, flags); context->level -= 1; return found; } return expression_tree_walker(node, ContainsReferencesToOuterQueryWalker, context); } /* * NodeContainsSubqueryReferencingOuterQuery determines whether the given node * contains anything that points outside of the query itself. */ static bool NodeContainsSubqueryReferencingOuterQuery(Node *node) { List *sublinks = NIL; ExtractSublinkWalker(node, &sublinks); SubLink *sublink; foreach_declared_ptr(sublink, sublinks) { if (ContainsReferencesToOuterQuery(castNode(Query, sublink->subselect))) { return true; } } return false; } /* * ReplaceRTERelationWithRteSubquery replaces the input rte relation target entry * with a subquery. The function also pushes down the filters to the subquery. * * It then recursively plans the subquery. This subquery is wrapped with another subquery * as a trick to reduce network cost, because we currently don't have an easy way to * skip generating NULL's for non-required columns, and if we create (SELECT a, NULL, NULL FROM table) * then this will be sent over network and NULL's also occupy some space. Instead of this we generate: * (SELECT t.a, NULL, NULL FROM (SELECT a FROM table) t). The inner subquery will be recursively planned * but the outer part will not be yet it will still have the NULL columns so that the query is correct. */ void ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, List *requiredAttrNumbers, RecursivePlanningContext *context, RTEPermissionInfo *perminfo) { Query *subquery = WrapRteRelationIntoSubquery(rangeTableEntry, requiredAttrNumbers, perminfo); List *outerQueryTargetList = CreateAllTargetListForRelation(rangeTableEntry->relid, requiredAttrNumbers); List *restrictionList = GetRestrictInfoListForRelation(rangeTableEntry, context->plannerRestrictionContext); List *copyRestrictionList = copyObject(restrictionList); Expr *andedBoundExpressions = make_ands_explicit(copyRestrictionList); subquery->jointree->quals = (Node *) andedBoundExpressions; /* * Originally the quals were pointing to the RTE and its varno * was pointing to its index in rtable. However now we converted the RTE * to a subquery and the quals should be pointing to that subquery, which * is the only RTE in its rtable, hence we update the varnos so that they * point to the subquery RTE. * Originally: rtable: [rte1, current_rte, rte3...] * Now: rtable: [rte1, subquery[current_rte], rte3...] --subquery[current_rte] refers to its rtable. */ Node *quals = subquery->jointree->quals; UpdateVarNosInNode(quals, SINGLE_RTE_INDEX); /* replace the function with the constructed subquery */ rangeTableEntry->rtekind = RTE_SUBQUERY; rangeTableEntry->perminfoindex = 0; rangeTableEntry->subquery = subquery; /* * If the relation is inherited, it'll still be inherited as * we've copied it earlier. This is to prevent the newly created * subquery being treated as inherited. */ rangeTableEntry->inh = false; if (IsLoggableLevel(DEBUG1)) { char *relationAndAliasName = GetRelationNameAndAliasName(rangeTableEntry); ereport(DEBUG1, (errmsg("Wrapping relation %s to a subquery", relationAndAliasName))); } /* as we created the subquery, now forcefully recursively plan it */ bool recursivelyPlanned = RecursivelyPlanSubquery(subquery, context); if (!recursivelyPlanned) { ereport(ERROR, (errmsg( "unexpected state: query should have been recursively planned"))); } Query *outerSubquery = CreateOuterSubquery(rangeTableEntry, outerQueryTargetList); rangeTableEntry->subquery = outerSubquery; } /* * GetRelationNameAndAliasName returns the relname + alias name if * alias name exists otherwise only the relname is returned. */ static char * GetRelationNameAndAliasName(RangeTblEntry *rangeTableEntry) { StringInfo str = makeStringInfo(); appendStringInfo(str, "\"%s\"", get_rel_name(rangeTableEntry->relid)); char *aliasName = NULL; if (rangeTableEntry->alias) { aliasName = rangeTableEntry->alias->aliasname; } if (aliasName) { appendStringInfo(str, " \"%s\"", aliasName); } return str->data; } /* * CreateOuterSubquery creates outer subquery which contains * the given range table entry in its rtable. */ static Query * CreateOuterSubquery(RangeTblEntry *rangeTableEntry, List *outerSubqueryTargetList) { List *innerSubqueryColNames = GenerateRequiredColNamesFromTargetList( outerSubqueryTargetList); Query *outerSubquery = makeNode(Query); outerSubquery->commandType = CMD_SELECT; /* we copy the input rteRelation to preserve the rteIdentity */ RangeTblEntry *innerSubqueryRTE = copyObject(rangeTableEntry); innerSubqueryRTE->eref->colnames = innerSubqueryColNames; outerSubquery->rtable = list_make1(innerSubqueryRTE); /* sanity check */ Assert(innerSubqueryRTE->rtekind == RTE_SUBQUERY && innerSubqueryRTE->perminfoindex == 0); outerSubquery->rteperminfos = NIL; /* set the FROM expression to the subquery */ RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); newRangeTableRef->rtindex = 1; outerSubquery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); outerSubquery->targetList = outerSubqueryTargetList; return outerSubquery; } /* * GenerateRequiredColNamesFromTargetList generates the required colnames * from the given target list. */ static List * GenerateRequiredColNamesFromTargetList(List *targetList) { TargetEntry *entry = NULL; List *innerSubqueryColNames = NIL; foreach_declared_ptr(entry, targetList) { if (IsA(entry->expr, Var)) { /* * column names of the inner subquery should only contain the * required columns, as in if we choose 'b' from ('a','b') colnames * should be 'a' not ('a','b') */ innerSubqueryColNames = lappend(innerSubqueryColNames, makeString( entry->resname)); } } return innerSubqueryColNames; } /* * UpdateVarNosInNode iterates the Vars in the * given node and updates the varno's as the newVarNo. */ void UpdateVarNosInNode(Node *node, Index newVarNo) { List *varList = pull_var_clause(node, PVC_RECURSE_AGGREGATES | PVC_RECURSE_PLACEHOLDERS); Var *var = NULL; foreach_declared_ptr(var, varList) { var->varno = newVarNo; } } /* * IsRecursivelyPlannableRelation returns true if the given range table entry * is a relation type that can be converted to a subquery. */ bool IsRecursivelyPlannableRelation(RangeTblEntry *rangeTableEntry) { if (rangeTableEntry->rtekind != RTE_RELATION) { return false; } return rangeTableEntry->relkind == RELKIND_PARTITIONED_TABLE || rangeTableEntry->relkind == RELKIND_RELATION || rangeTableEntry->relkind == RELKIND_MATVIEW || rangeTableEntry->relkind == RELKIND_FOREIGN_TABLE; } /* * ContainsLocalTableDistributedTableJoin returns true if the input range table list * contains a direct join between local RTE and an RTE that contains a distributed * or reference table. */ bool ContainsLocalTableDistributedTableJoin(List *rangeTableList) { bool containsLocalTable = false; bool containsDistributedTable = false; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, rangeTableList) { if (FindNodeMatchingCheckFunctionInRangeTableList(list_make1(rangeTableEntry), IsDistributedOrReferenceTableRTE)) { containsDistributedTable = true; } else if (IsRecursivelyPlannableRelation(rangeTableEntry) && IsLocalTableRteOrMatView((Node *) rangeTableEntry)) { /* we consider citus local tables as local table */ containsLocalTable = true; } } return containsLocalTable && containsDistributedTable; } /* * WrapFunctionsInSubqueries iterates over all the immediate Range Table Entries * of a query and wraps the functions inside (SELECT * FROM fnc() f) * subqueries, so that those functions will be executed on the coordinator if * necessary. * * We wrap all the functions that are used in joins except the ones that are * laterally joined or have WITH ORDINALITY clauses. * */ static void WrapFunctionsInSubqueries(Query *query) { List *rangeTableList = query->rtable; ListCell *rangeTableCell = NULL; /* * If we have only one function call in a query without any joins, we can * easily decide where to execute it. * * If there are some subqueries and/or functions that are joined with a * function, it is not trivial to decide whether we should run this * function in the coordinator or in workers and therefore we may need to * wrap some of those functions in subqueries. * * If we have only one RTE, we leave the parsed query tree as it is. This * also makes sure we do not wrap an already wrapped function call * because we know that there will always be 1 RTE in a wrapped function. * */ if (list_length(rangeTableList) < 2) { return; } /* iterate over all RTEs and wrap them if necessary */ foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (ShouldTransformRTE(rangeTableEntry)) { TransformFunctionRTE(rangeTableEntry); } } } /* * TransformFunctionRTE wraps a given function RangeTableEntry * inside a (SELECT * from function() f) subquery. * * The said RangeTableEntry is modified and now points to the new subquery. * */ static void TransformFunctionRTE(RangeTblEntry *rangeTblEntry) { Query *subquery = makeNode(Query); RangeTblRef *newRangeTableRef = makeNode(RangeTblRef); Var *targetColumn = NULL; TargetEntry *targetEntry = NULL; AttrNumber targetColumnIndex = 0; RangeTblFunction *rangeTblFunction = linitial(rangeTblEntry->functions); subquery->commandType = CMD_SELECT; /* copy the input rangeTblEntry to prevent cycles */ RangeTblEntry *newRangeTableEntry = copyObject(rangeTblEntry); /* set the FROM expression to the subquery */ subquery->rtable = list_make1(newRangeTableEntry); /* sanity check */ Assert(newRangeTableEntry->rtekind == RTE_FUNCTION && newRangeTableEntry->perminfoindex == 0); subquery->rteperminfos = NIL; newRangeTableRef->rtindex = 1; subquery->jointree = makeFromExpr(list_make1(newRangeTableRef), NULL); /* Determine the result type of the function. * * If function return type is not composite or rowtype can't be determined, * tupleDesc is set to null here */ TupleDesc tupleDesc = (TupleDesc) get_expr_result_tupdesc(rangeTblFunction->funcexpr, true); /* * If tupleDesc is not null, we iterate over all the attributes and * create targetEntries * */ if (tupleDesc) { /* * A sample function join that end up here: * * CREATE FUNCTION f(..) RETURNS TABLE(c1 int, c2 text) AS .. ; * SELECT .. FROM table JOIN f(..) ON ( .. ) ; * * We will iterate over Tuple Description attributes. i.e (c1 int, c2 text) */ if (tupleDesc->natts > MaxAttrNumber) { ereport(ERROR, (errmsg("bad number of tuple descriptor attributes"))); } AttrNumber natts = tupleDesc->natts; for (targetColumnIndex = 0; targetColumnIndex < natts; targetColumnIndex++) { FormData_pg_attribute *attribute = TupleDescAttr(tupleDesc, targetColumnIndex); Oid columnType = attribute->atttypid; char *columnName = attribute->attname.data; /* * The indexing of attributes and TupleDesc and varattno differ * * varattno=0 corresponds to whole row * varattno=1 corresponds to first column that is stored in tupDesc->attrs[0] * * That's why we need to add one to the targetColumnIndex * */ targetColumn = makeVar(1, targetColumnIndex + 1, columnType, -1, InvalidOid, 0); targetEntry = makeTargetEntry((Expr *) targetColumn, targetColumnIndex + 1, columnName, false); subquery->targetList = lappend(subquery->targetList, targetEntry); } } /* * If tupleDesc is NULL we have 2 different cases: * * 1. The function returns a record but the attributes can not be * determined just by looking at the function definition. In this case the * column names and types must be defined explicitly in the query * * 2. The function returns a non-composite type (e.g. int, text, jsonb ..) * */ else { /* create target entries for all columns returned by the function */ ListCell *functionColumnName = NULL; List *functionColumnNames = rangeTblEntry->eref->colnames; foreach(functionColumnName, functionColumnNames) { char *columnName = strVal(lfirst(functionColumnName)); Oid columnType = InvalidOid; /* * If the function returns a set of records, the query needs * to explicitly name column names and types * * Use explicitly defined types in the query if they are * available * */ if (list_length(rangeTblFunction->funccoltypes) > 0) { /* * A sample function join that end up here: * * CREATE FUNCTION get_set_of_records() RETURNS SETOF RECORD AS * $cmd$ * SELECT x, x+1 FROM generate_series(0,4) f(x) * $cmd$ * LANGUAGE SQL; * * SELECT * * FROM table1 JOIN get_set_of_records() AS t2(x int, y int) * ON (id = x); * * Note that the function definition does not have column * names and types. Therefore the user needs to explicitly * state them in the query * */ columnType = list_nth_oid(rangeTblFunction->funccoltypes, targetColumnIndex); } /* use the types in the function definition otherwise */ else { /* * Only functions returning simple types end up here. * A sample function: * * CREATE FUNCTION add(integer, integer) RETURNS integer AS * 'SELECT $1 + $2;' * LANGUAGE SQL; * SELECT * FROM table JOIN add(3,5) sum ON ( .. ) ; * */ FuncExpr *funcExpr = (FuncExpr *) rangeTblFunction->funcexpr; columnType = funcExpr->funcresulttype; } /* Note that the column k is associated with varattno/resno of k+1 */ targetColumn = makeVar(1, targetColumnIndex + 1, columnType, -1, InvalidOid, 0); targetEntry = makeTargetEntry((Expr *) targetColumn, targetColumnIndex + 1, columnName, false); subquery->targetList = lappend(subquery->targetList, targetEntry); targetColumnIndex++; } } /* replace the function with the constructed subquery */ rangeTblEntry->rtekind = RTE_SUBQUERY; rangeTblEntry->subquery = subquery; } /* * ShouldTransformRTE determines whether a given RTE should bne wrapped in a * subquery. * * Not all functions should be wrapped in a subquery for now. As we support more * functions to be used in joins, the constraints here will be relaxed. * */ static bool ShouldTransformRTE(RangeTblEntry *rangeTableEntry) { /* * We should wrap only function rtes that are not LATERAL and * without WITH ORDINALITY clause */ if (rangeTableEntry->rtekind != RTE_FUNCTION || rangeTableEntry->lateral || rangeTableEntry->funcordinality) { return false; } return true; } /* * BuildSubPlanResultQuery returns a query of the form: * * SELECT * * FROM * read_intermediate_result('', ') * AS res (); * * The caller can optionally supply a columnAliasList, which is useful for * CTEs that have column aliases. * * If any of the types in the target list cannot be used in the binary copy format, * then the copy format 'text' is used, otherwise 'binary' is used. */ Query * BuildSubPlanResultQuery(List *targetEntryList, List *columnAliasList, char *resultId) { Oid functionOid = CitusReadIntermediateResultFuncId(); bool useBinaryCopyFormat = CanUseBinaryCopyFormatForTargetList(targetEntryList); Const *resultIdConst = makeNode(Const); resultIdConst->consttype = TEXTOID; resultIdConst->consttypmod = -1; resultIdConst->constlen = -1; resultIdConst->constvalue = CStringGetTextDatum(resultId); resultIdConst->constbyval = false; resultIdConst->constisnull = false; resultIdConst->location = -1; return BuildReadIntermediateResultsQuery(targetEntryList, columnAliasList, resultIdConst, functionOid, useBinaryCopyFormat); } /* * BuildReadIntermediateResultsArrayQuery returns a query of the form: * * SELECT * * FROM * read_intermediate_results(ARRAY['', ...]::text[], ') * AS res (); * * The caller can optionally supply a columnAliasList, which is useful for * CTEs that have column aliases. * * If useBinaryCopyFormat is true, then 'binary' format is used. Otherwise, * 'text' format is used. */ Query * BuildReadIntermediateResultsArrayQuery(List *targetEntryList, List *columnAliasList, List *resultIdList, bool useBinaryCopyFormat) { Oid functionOid = CitusReadIntermediateResultArrayFuncId(); Const *resultIdConst = makeNode(Const); resultIdConst->consttype = TEXTARRAYOID; resultIdConst->consttypmod = -1; resultIdConst->constlen = -1; resultIdConst->constvalue = PointerGetDatum(strlist_to_textarray(resultIdList)); resultIdConst->constbyval = false; resultIdConst->constisnull = false; resultIdConst->location = -1; return BuildReadIntermediateResultsQuery(targetEntryList, columnAliasList, resultIdConst, functionOid, useBinaryCopyFormat); } /* * For the given target list, build an empty relation with the same target list. * For example, if the target list is (a, b, c), and resultId is "empty", then * it returns a Query object for this SQL: * SELECT a, b, c FROM (VALUES (NULL, NULL, NULL)) AS empty(a, b, c) WHERE false; */ Query * BuildEmptyResultQuery(List *targetEntryList, char *resultId) { List *targetList = NIL; ListCell *targetEntryCell = NULL; List *colTypes = NIL; List *colTypMods = NIL; List *colCollations = NIL; List *colNames = NIL; List *valueConsts = NIL; List *valueTargetList = NIL; List *valueColNames = NIL; int targetIndex = 1; /* build the target list and column lists needed */ foreach(targetEntryCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Node *targetExpr = (Node *) targetEntry->expr; char *columnName = targetEntry->resname; Oid columnType = exprType(targetExpr); Oid columnTypMod = exprTypmod(targetExpr); Oid columnCollation = exprCollation(targetExpr); if (targetEntry->resjunk) { continue; } Var *tgtVar = makeVar(1, targetIndex, columnType, columnTypMod, columnCollation, 0); TargetEntry *tgtEntry = makeTargetEntry((Expr *) tgtVar, targetIndex, columnName, false); Const *valueConst = makeConst(columnType, columnTypMod, columnCollation, 0, (Datum) 0, true, false); StringInfoData *columnString = makeStringInfo(); appendStringInfo(columnString, "column%d", targetIndex); TargetEntry *valueTgtEntry = makeTargetEntry((Expr *) tgtVar, targetIndex, columnString->data, false); valueConsts = lappend(valueConsts, valueConst); valueTargetList = lappend(valueTargetList, valueTgtEntry); valueColNames = lappend(valueColNames, makeString(columnString->data)); colNames = lappend(colNames, makeString(columnName)); colTypes = lappend_oid(colTypes, columnType); colTypMods = lappend_oid(colTypMods, columnTypMod); colCollations = lappend_oid(colCollations, columnCollation); targetList = lappend(targetList, tgtEntry); targetIndex++; } /* Build a RangeTable Entry for the VALUES relation */ RangeTblEntry *valuesRangeTable = makeNode(RangeTblEntry); valuesRangeTable->rtekind = RTE_VALUES; valuesRangeTable->values_lists = list_make1(valueConsts); valuesRangeTable->colcollations = colCollations; valuesRangeTable->coltypes = colTypes; valuesRangeTable->coltypmods = colTypMods; valuesRangeTable->alias = NULL; valuesRangeTable->eref = makeAlias("*VALUES*", valueColNames); valuesRangeTable->inFromCl = true; RangeTblRef *valuesRTRef = makeNode(RangeTblRef); valuesRTRef->rtindex = 1; FromExpr *valuesJoinTree = makeNode(FromExpr); valuesJoinTree->fromlist = list_make1(valuesRTRef); /* build the VALUES query */ Query *valuesQuery = makeNode(Query); valuesQuery->canSetTag = true; valuesQuery->commandType = CMD_SELECT; valuesQuery->rtable = list_make1(valuesRangeTable); valuesQuery->rteperminfos = NIL; valuesQuery->jointree = valuesJoinTree; valuesQuery->targetList = valueTargetList; /* build the relation selecting from the VALUES */ RangeTblEntry *emptyRangeTable = makeNode(RangeTblEntry); emptyRangeTable->rtekind = RTE_SUBQUERY; emptyRangeTable->subquery = valuesQuery; emptyRangeTable->alias = makeAlias(resultId, colNames); emptyRangeTable->eref = emptyRangeTable->alias; emptyRangeTable->inFromCl = true; /* build the SELECT query */ Query *resultQuery = makeNode(Query); resultQuery->commandType = CMD_SELECT; resultQuery->canSetTag = true; resultQuery->rtable = list_make1(emptyRangeTable); resultQuery->rteperminfos = NIL; RangeTblRef *rangeTableRef = makeNode(RangeTblRef); rangeTableRef->rtindex = 1; /* insert a FALSE qual to ensure 0 rows returned */ FromExpr *joinTree = makeNode(FromExpr); joinTree->fromlist = list_make1(rangeTableRef); joinTree->quals = makeBoolConst(false, false); resultQuery->jointree = joinTree; resultQuery->targetList = targetList; return resultQuery; } /* * BuildReadIntermediateResultsQuery is the common code for generating * queries to read from result files. It is used by * BuildReadIntermediateResultsArrayQuery and BuildSubPlanResultQuery. */ static Query * BuildReadIntermediateResultsQuery(List *targetEntryList, List *columnAliasList, Const *resultIdConst, Oid functionOid, bool useBinaryCopyFormat) { List *funcColNames = NIL; List *funcColTypes = NIL; List *funcColTypMods = NIL; List *funcColCollations = NIL; ListCell *targetEntryCell = NULL; List *targetList = NIL; int columnNumber = 1; Oid copyFormatId = BinaryCopyFormatId(); int columnAliasCount = list_length(columnAliasList); /* build the target list and column definition list */ foreach(targetEntryCell, targetEntryList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Node *targetExpr = (Node *) targetEntry->expr; char *columnName = targetEntry->resname; Oid columnType = exprType(targetExpr); Oid columnTypMod = exprTypmod(targetExpr); Oid columnCollation = exprCollation(targetExpr); if (targetEntry->resjunk) { continue; } funcColNames = lappend(funcColNames, makeString(columnName)); funcColTypes = lappend_int(funcColTypes, columnType); funcColTypMods = lappend_int(funcColTypMods, columnTypMod); funcColCollations = lappend_int(funcColCollations, columnCollation); Var *functionColumnVar = makeNode(Var); functionColumnVar->varno = 1; functionColumnVar->varattno = columnNumber; functionColumnVar->vartype = columnType; functionColumnVar->vartypmod = columnTypMod; functionColumnVar->varcollid = columnCollation; functionColumnVar->varlevelsup = 0; functionColumnVar->varnosyn = 1; functionColumnVar->varattnosyn = columnNumber; functionColumnVar->location = -1; TargetEntry *newTargetEntry = makeNode(TargetEntry); newTargetEntry->expr = (Expr *) functionColumnVar; newTargetEntry->resno = columnNumber; /* * Rename the column only if a column alias is defined. * Notice that column alias count could be less than actual * column count. We only use provided aliases and keep the * original column names if no alias is defined. */ if (columnAliasCount >= columnNumber) { String *columnAlias = (String *) list_nth(columnAliasList, columnNumber - 1); Assert(IsA(columnAlias, String)); newTargetEntry->resname = strVal(columnAlias); } else { newTargetEntry->resname = columnName; } newTargetEntry->resjunk = false; targetList = lappend(targetList, newTargetEntry); columnNumber++; } /* build the citus_copy_format parameter for the call to read_intermediate_result */ if (!useBinaryCopyFormat) { copyFormatId = TextCopyFormatId(); } Const *resultFormatConst = makeNode(Const); resultFormatConst->consttype = CitusCopyFormatTypeId(); resultFormatConst->consttypmod = -1; resultFormatConst->constlen = 4; resultFormatConst->constvalue = ObjectIdGetDatum(copyFormatId); resultFormatConst->constbyval = true; resultFormatConst->constisnull = false; resultFormatConst->location = -1; /* build the call to read_intermediate_result */ FuncExpr *funcExpr = makeNode(FuncExpr); funcExpr->funcid = functionOid; funcExpr->funcretset = true; funcExpr->funcvariadic = false; funcExpr->funcformat = 0; funcExpr->funccollid = 0; funcExpr->inputcollid = 0; funcExpr->location = -1; funcExpr->args = list_make2(resultIdConst, resultFormatConst); /* build the RTE for the call to read_intermediate_result */ RangeTblFunction *rangeTableFunction = makeNode(RangeTblFunction); rangeTableFunction->funccolcount = list_length(funcColNames); rangeTableFunction->funccolnames = funcColNames; rangeTableFunction->funccoltypes = funcColTypes; rangeTableFunction->funccoltypmods = funcColTypMods; rangeTableFunction->funccolcollations = funcColCollations; rangeTableFunction->funcparams = NULL; rangeTableFunction->funcexpr = (Node *) funcExpr; Alias *funcAlias = makeNode(Alias); funcAlias->aliasname = "intermediate_result"; funcAlias->colnames = funcColNames; RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->rtekind = RTE_FUNCTION; rangeTableEntry->functions = list_make1(rangeTableFunction); rangeTableEntry->inFromCl = true; rangeTableEntry->eref = funcAlias; /* build the join tree using the read_intermediate_result RTE */ RangeTblRef *rangeTableRef = makeNode(RangeTblRef); rangeTableRef->rtindex = 1; FromExpr *joinTree = makeNode(FromExpr); joinTree->fromlist = list_make1(rangeTableRef); /* build the SELECT query */ Query *resultQuery = makeNode(Query); resultQuery->commandType = CMD_SELECT; resultQuery->rtable = list_make1(rangeTableEntry); resultQuery->rteperminfos = NIL; resultQuery->jointree = joinTree; resultQuery->targetList = targetList; return resultQuery; } /* * GenerateResultId generates the result ID that is used to identify an intermediate * result of the subplan with the given plan ID and subplan ID. */ char * GenerateResultId(uint64 planId, uint32 subPlanId) { StringInfo resultId = makeStringInfo(); appendStringInfo(resultId, UINT64_FORMAT "_%u", planId, subPlanId); return resultId->data; } /* * GeneratingSubplans returns true if we are currently in the process of * generating subplans. */ bool GeneratingSubplans(void) { return recursivePlanningDepth > 0; } #if PG_VERSION_NUM < PG_VERSION_17 /* * hasPseudoconstantQuals returns true if any of the planner infos in the * relation restriction list of the input relation restriction context * has a pseudoconstant qual */ static bool hasPseudoconstantQuals(RelationRestrictionContext *relationRestrictionContext) { ListCell *objectCell = NULL; foreach(objectCell, relationRestrictionContext->relationRestrictionList) { if (((RelationRestriction *) lfirst( objectCell))->plannerInfo->hasPseudoConstantQuals) { return true; } } return false; } #endif /* * CanPushdownRecurringOuterJoinOnOuterRTE returns true if the given range table entry * is safe for pushdown when it is the outer relation of a outer join when the * inner relation is not recurring. * Currently, we only allow reference tables. */ static bool CanPushdownRecurringOuterJoinOnOuterRTE(RangeTblEntry *rte) { if (IsCitusTable(rte->relid) && IsCitusTableType(rte->relid, REFERENCE_TABLE)) { return true; } else { ereport(DEBUG5, (errmsg("RTE type %d is not safe for pushdown", rte->rtekind))); return false; } } /* * ResolveBaseVarFromSubquery recursively resolves a Var from a subquery target list to * the base Var and RTE */ bool ResolveBaseVarFromSubquery(Var *var, Query *query, Var **baseVar, RangeTblEntry **baseRte) { TargetEntry *tle = get_tle_by_resno(query->targetList, var->varattno); if (!tle || !IsA(tle->expr, Var)) { return false; } Var *tleVar = (Var *) tle->expr; RangeTblEntry *rte = rt_fetch(tleVar->varno, query->rtable); if (rte == NULL) { return false; } if (rte->rtekind == RTE_RELATION || rte->rtekind == RTE_FUNCTION) { *baseVar = tleVar; *baseRte = rte; return true; } else if (rte->rtekind == RTE_SUBQUERY) { /* Prevent overflow, and allow query cancellation */ check_stack_depth(); CHECK_FOR_INTERRUPTS(); return ResolveBaseVarFromSubquery(tleVar, rte->subquery, baseVar, baseRte); } return false; } /* * CanPushdownRecurringOuterJoinOnInnerVar checks if the inner variable * from a join qual for a join pushdown. It returns true if it is valid, * it is the partition column and hash distributed, otherwise it returns false. */ static bool CanPushdownRecurringOuterJoinOnInnerVar(Var *innerVar, RangeTblEntry *rte) { if (!innerVar || !rte) { return false; } if (innerVar->varattno == InvalidAttrNumber) { return false; } CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(rte->relid); if (!cacheEntry || GetCitusTableType(cacheEntry) != HASH_DISTRIBUTED) { return false; } /* Check if the inner variable is part of the distribution column */ if (cacheEntry->partitionColumn && innerVar->varattno == cacheEntry->partitionColumn->varattno) { return true; } return false; } /* * JoinTreeContainsLateral checks if the given node contains a lateral * join. It returns true if it does, otherwise false. * * It recursively traverses the join tree and checks each RangeTblRef and JoinExpr * for lateral joins. */ static bool JoinTreeContainsLateral(Node *node, List *rtable) { if (node == NULL) { return false; } /* Prevent overflow, and allow query cancellation */ check_stack_depth(); CHECK_FOR_INTERRUPTS(); if (IsA(node, RangeTblRef)) { RangeTblEntry *rte = rt_fetch(((RangeTblRef *) node)->rtindex, rtable); if (rte == NULL) { return false; } if (rte->lateral) { return true; } if (rte->rtekind == RTE_SUBQUERY) { if (rte->subquery) { return JoinTreeContainsLateral((Node *) rte->subquery->jointree, rte->subquery->rtable); } } return false; } else if (IsA(node, JoinExpr)) { JoinExpr *join = (JoinExpr *) node; return JoinTreeContainsLateral(join->larg, rtable) || JoinTreeContainsLateral(join->rarg, rtable); } else if (IsA(node, FromExpr)) { FromExpr *fromExpr = (FromExpr *) node; ListCell *lc = NULL; foreach(lc, fromExpr->fromlist) { if (JoinTreeContainsLateral((Node *) lfirst(lc), rtable)) { return true; } } } return false; } /* * CanPushdownRecurringOuterJoinExtended checks if the given join expression * is an outer join between recurring rel -on outer part- and a distributed * rel -on the inner side- and if it is feasible to push down the join. If feasible, * it computes the outer relation's range table index, the outer relation's * range table entry, the inner (distributed) relation's range table entry, and the * attribute number of the partition column in the outer relation. */ bool CanPushdownRecurringOuterJoinExtended(JoinExpr *joinExpr, Query *query, int *outerRtIndex, RangeTblEntry **outerRte, RangeTblEntry **distRte, int *attnum) { if (!EnableRecurringOuterJoinPushdown) { return false; } if (!IS_OUTER_JOIN(joinExpr->jointype)) { return false; } if (joinExpr->jointype != JOIN_LEFT && joinExpr->jointype != JOIN_RIGHT) { return false; } /* Push down for chained joins is not supported in this path. */ if (IsA(joinExpr->rarg, JoinExpr) || IsA(joinExpr->larg, JoinExpr)) { ereport(DEBUG5, (errmsg( "One side is a join expression, pushdown is not supported in this path."))); return false; } /* Push down for joins with fromExpr on one side is not supported in this path. */ if (!IsA(joinExpr->larg, RangeTblRef) || !IsA(joinExpr->rarg, RangeTblRef)) { ereport(DEBUG5, (errmsg( "One side is not a RangeTblRef, pushdown is not supported in this path."))); return false; } if (joinExpr->jointype == JOIN_LEFT) { *outerRtIndex = (((RangeTblRef *) joinExpr->larg)->rtindex); } else /* JOIN_RIGHT */ { *outerRtIndex = (((RangeTblRef *) joinExpr->rarg)->rtindex); } *outerRte = rt_fetch(*outerRtIndex, query->rtable); if (!CanPushdownRecurringOuterJoinOnOuterRTE(*outerRte)) { return false; } /* For now if we see any lateral join in the join tree, we return false. * This check can be improved to support the cases where the lateral reference * does not cause an error in the final planner checks. */ if (JoinTreeContainsLateral(joinExpr->rarg, query->rtable) || JoinTreeContainsLateral( joinExpr->larg, query->rtable)) { ereport(DEBUG5, (errmsg("Lateral join is not supported for pushdown " "in this path."))); return false; } /* Check if the join is performed on the distribution column */ List *joinClauseList = make_ands_implicit((Expr *) joinExpr->quals); if (joinClauseList == NIL) { return false; } Node *joinClause = NULL; foreach_declared_ptr(joinClause, joinClauseList) { if (!NodeIsEqualsOpExpr(joinClause)) { continue; } OpExpr *joinClauseExpr = castNode(OpExpr, joinClause); Var *leftColumn = LeftColumnOrNULL(joinClauseExpr); Var *rightColumn = RightColumnOrNULL(joinClauseExpr); if (leftColumn == NULL || rightColumn == NULL) { continue; } RangeTblEntry *rte; Var *innerVar; if (leftColumn->varno == *outerRtIndex) { /* left column is the outer table of the comparison, get right */ rte = rt_fetch(rightColumn->varno, query->rtable); innerVar = rightColumn; /* additional constraints will be introduced on outer relation variable */ *attnum = leftColumn->varattno; } else if (rightColumn->varno == *outerRtIndex) { /* right column is the outer table of the comparison, get left*/ rte = rt_fetch(leftColumn->varno, query->rtable); innerVar = leftColumn; /* additional constraints will be introduced on outer relation variable */ *attnum = rightColumn->varattno; } else { continue; } /* the simple case, the inner table itself a Citus table */ if (rte && IsCitusTable(rte->relid)) { if (CanPushdownRecurringOuterJoinOnInnerVar(innerVar, rte)) { *distRte = rte; return true; } } /* the inner table is a subquery, extract the base relation referred in the qual */ else if (rte && rte->rtekind == RTE_SUBQUERY) { Var *baseVar = NULL; RangeTblEntry *baseRte = NULL; if (ResolveBaseVarFromSubquery(innerVar, rte->subquery, &baseVar, &baseRte)) { if (baseRte && IsCitusTable(baseRte->relid)) { if (CanPushdownRecurringOuterJoinOnInnerVar(baseVar, baseRte)) { *distRte = baseRte; return true; } } } } } return false; } /* * CanPushdownRecurringOuterJoin initializes input variables to call * CanPushdownRecurringOuterJoinExtended. * See CanPushdownRecurringOuterJoinExtended for more details. */ bool CanPushdownRecurringOuterJoin(JoinExpr *joinExpr, Query *query) { int outerRtIndex; RangeTblEntry *outerRte = NULL; RangeTblEntry *innerRte = NULL; int attnum; return CanPushdownRecurringOuterJoinExtended(joinExpr, query, &outerRtIndex, &outerRte, &innerRte, &attnum); } ================================================ FILE: src/backend/distributed/planner/relation_restriction_equivalence.c ================================================ /* * relation_restriction_equivalence.c * * This file contains functions helper functions for planning * queries with colocated tables and subqueries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "parser/parsetree.h" #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/query_utils.h" #include "distributed/relation_restriction_equivalence.h" #include "distributed/shard_pruning.h" static uint32 AttributeEquivalenceId = 1; /* * AttributeEquivalenceClass * * Whenever we find an equality clause A = B, where both A and B originates from * relation attributes (i.e., not random expressions), we create an * AttributeEquivalenceClass to record this knowledge. If we later find another * equivalence B = C, we create another AttributeEquivalenceClass. Finally, we can * apply transitivity rules and generate a new AttributeEquivalenceClass which includes * A, B and C. * * Note that equality among the members are identified by the varattno and rteIdentity. */ typedef struct AttributeEquivalenceClass { uint32 equivalenceId; List *equivalentAttributes; Index unionQueryPartitionKeyIndex; } AttributeEquivalenceClass; typedef struct FindQueryContainingRteIdentityContext { int targetRTEIdentity; Query *query; }FindQueryContainingRteIdentityContext; /* * AttributeEquivalenceClassMember - one member expression of an * AttributeEquivalenceClass. The important thing to consider is that * the class member contains "rteIndentity" field. Note that each RTE_RELATION * is assigned a unique rteIdentity in AssignRTEIdentities() function. * * "varno" and "varattno" is directly used from a Var clause that is being added * to the attribute equivalence. Since we only use this class for relations, the member * also includes the relation id field. */ typedef struct AttributeEquivalenceClassMember { Oid relationId; int rteIdentity; Index varno; AttrNumber varattno; } AttributeEquivalenceClassMember; /* * ECGroupByExpr * Helper structure to group EquivalenceClasses by their non-Var expressions. */ typedef struct ECGroupByExpr { Node *strippedExpr; /* The canonical non-Var expression (stripped) */ List *ecsWithThisExpr; /* List of EquivalenceClass* sharing this expr */ List *varsInTheseECs; /* Cached list of Var* from all these ECs */ } ECGroupByExpr; static bool ContextContainsLocalRelation(RelationRestrictionContext *restrictionContext); static bool ContextContainsAppendRelation(RelationRestrictionContext *restrictionContext); static int RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo); static Var * FindUnionAllVar(PlannerInfo *root, List *translatedVars, Oid relationOid, Index relationRteIndex, Index *partitionKeyIndex); static bool ContainsMultipleDistributedRelations(PlannerRestrictionContext * plannerRestrictionContext); static List * GenerateAttributeEquivalencesForRelationRestrictions( RelationRestrictionContext *restrictionContext); static List * MergeEquivalenceClassesWithSameFunctions(RelationRestrictionContext * restrictionContext); static AttributeEquivalenceClass * AttributeEquivalenceClassForEquivalenceClass( EquivalenceClass *plannerEqClass, RelationRestriction *relationRestriction); static void AddToAttributeEquivalenceClass(AttributeEquivalenceClass * attributeEquivalenceClass, PlannerInfo *root, Var *varToBeAdded); static void AddRteSubqueryToAttributeEquivalenceClass(AttributeEquivalenceClass * attributeEquivalenceClass, RangeTblEntry * rangeTableEntry, PlannerInfo *root, Var *varToBeAdded); static Query * GetTargetSubquery(PlannerInfo *root, RangeTblEntry *rangeTableEntry, Var *varToBeAdded); static void AddUnionAllSetOperationsToAttributeEquivalenceClass( AttributeEquivalenceClass * attributeEquivalenceClass, PlannerInfo *root, Var *varToBeAdded); static void AddUnionSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass * attributeEquivalenceClass, PlannerInfo *root, SetOperationStmt * setOperation, Var *varToBeAdded); static void AddRteRelationToAttributeEquivalenceClass(AttributeEquivalenceClass * attrEquivalenceClass, RangeTblEntry *rangeTableEntry, Var *varToBeAdded); static Var * GetVarFromAssignedParam(List *outerPlanParamsList, Param *plannerParam, PlannerInfo **rootContainingVar); static Var * SearchPlannerParamList(List *plannerParamList, Param *plannerParam); static List * GenerateAttributeEquivalencesForJoinRestrictions(JoinRestrictionContext *joinRestrictionContext); static bool AttributeClassContainsAttributeClassMember(AttributeEquivalenceClassMember * inputMember, AttributeEquivalenceClass * attributeEquivalenceClass); static List * AddAttributeClassToAttributeClassList(List *attributeEquivalenceList, AttributeEquivalenceClass * attributeEquivalence); static bool AttributeEquivalencesAreEqual(AttributeEquivalenceClass * firstAttributeEquivalence, AttributeEquivalenceClass * secondAttributeEquivalence); static AttributeEquivalenceClass * GenerateCommonEquivalence(List * attributeEquivalenceList, RelationRestrictionContext * relationRestrictionContext); static AttributeEquivalenceClass * GenerateEquivalenceClassForRelationRestriction( RelationRestrictionContext * relationRestrictionContext); static void ListConcatUniqueAttributeClassMemberLists(AttributeEquivalenceClass * firstClass, AttributeEquivalenceClass * secondClass); static Var * PartitionKeyForRTEIdentityInQuery(Query *query, int targetRTEIndex, Index *partitionKeyIndex); static bool AllDistributedRelationsInRestrictionContextColocated( RelationRestrictionContext * restrictionContext); static bool IsNotSafeRestrictionToRecursivelyPlan(Node *node); static bool HasPlaceHolderVar(Node *node); static JoinRestrictionContext * FilterJoinRestrictionContext(JoinRestrictionContext * joinRestrictionContext, Relids queryRteIdentities); static bool RangeTableArrayContainsAnyRTEIdentities(RangeTblEntry **rangeTableEntries, int rangeTableArrayLength, Relids queryRteIdentities); static Relids QueryRteIdentities(Query *queryTree); static Query * FindQueryContainingRTEIdentity(Query *mainQuery, int rteIndex); static bool FindQueryContainingRTEIdentityInternal(Node *node, FindQueryContainingRteIdentityContext * context); static int ParentCountPriorToAppendRel(List *appendRelList, AppendRelInfo *appendRelInfo); static bool PartitionColumnSelectedForOuterJoin(Query *query, RelationRestrictionContext * restrictionContext, JoinRestrictionContext * joinRestrictionContext); static bool PartitionColumnIsInTargetList(Query *query, JoinRestriction *joinRestriction, RelationRestrictionContext *restrictionContext); /* * AllDistributionKeysInQueryAreEqual returns true if either * (i) there exists join in the query and all relations joined on their * partition keys * (ii) there exists only union set operations and all relations has * partition keys in the same ordinal position in the query */ bool AllDistributionKeysInQueryAreEqual(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { /* we don't support distribution key equality checks for CTEs yet */ if (originalQuery->cteList != NIL) { return false; } /* we don't support distribution key equality checks for local tables */ RelationRestrictionContext *restrictionContext = plannerRestrictionContext->relationRestrictionContext; if (ContextContainsLocalRelation(restrictionContext)) { return false; } bool restrictionEquivalenceForPartitionKeys = RestrictionEquivalenceForPartitionKeys(plannerRestrictionContext); if (restrictionEquivalenceForPartitionKeys) { return true; } if (originalQuery->setOperations || ContainsUnionSubquery(originalQuery)) { return SafeToPushdownUnionSubquery(originalQuery, plannerRestrictionContext); } return false; } /* * ContextContainsLocalRelation determines whether the given * RelationRestrictionContext contains any local tables. */ static bool ContextContainsLocalRelation(RelationRestrictionContext *restrictionContext) { ListCell *relationRestrictionCell = NULL; foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = lfirst(relationRestrictionCell); if (!relationRestriction->citusTable) { return true; } } return false; } /* * ContextContainsAppendRelation determines whether the given * RelationRestrictionContext contains any append-distributed tables. */ static bool ContextContainsAppendRelation(RelationRestrictionContext *restrictionContext) { ListCell *relationRestrictionCell = NULL; foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = lfirst(relationRestrictionCell); if (IsCitusTableType(relationRestriction->relationId, APPEND_DISTRIBUTED)) { return true; } } return false; } /* * SafeToPushdownUnionSubquery returns true if all the relations are returns * partition keys in the same ordinal position and there is no reference table * exists. * * Note that the function expects (and asserts) the input query to be a top * level union query defined by TopLevelUnionQuery(). * * Lastly, the function fails to produce correct output if the target lists contains * multiple partition keys on the target list such as the following: * * select count(*) from ( * select user_id, user_id from users_table * union * select 2, user_id from users_table) u; * * For the above query, although the second item in the target list make this query * safe to push down, the function would fail to return true. */ bool SafeToPushdownUnionSubquery(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext) { RelationRestrictionContext *restrictionContext = plannerRestrictionContext->relationRestrictionContext; JoinRestrictionContext *joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; AttributeEquivalenceClass *attributeEquivalence = palloc0(sizeof(AttributeEquivalenceClass)); ListCell *relationRestrictionCell = NULL; attributeEquivalence->equivalenceId = AttributeEquivalenceId++; /* * Ensure that the partition column is in the same place across all * leaf queries in the UNION and construct an equivalence class for * these columns. */ foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = lfirst(relationRestrictionCell); Index partitionKeyIndex = InvalidAttrNumber; PlannerInfo *relationPlannerRoot = relationRestriction->plannerInfo; int targetRTEIndex = GetRTEIdentity(relationRestriction->rte); Var *varToBeAdded = PartitionKeyForRTEIdentityInQuery(originalQuery, targetRTEIndex, &partitionKeyIndex); /* union does not have partition key in the target list */ if (partitionKeyIndex == 0) { continue; } /* * This should never happen but to be on the safe side, we have this */ if (relationPlannerRoot->simple_rel_array_size < relationRestriction->index) { continue; } /* * We update the varno because we use the original parse tree for finding the * var. However the rest of the code relies on a query tree that might be different * than the original parse tree because of postgres optimizations. * That's why we update the varno to reflect the rteIndex in the modified query tree. */ varToBeAdded->varno = relationRestriction->index; /* * The current relation does not have its partition key in the target list. */ if (partitionKeyIndex == InvalidAttrNumber) { continue; } /* * We find the first relations partition key index in the target list. Later, * we check whether all the relations have partition keys in the * same position. */ if (attributeEquivalence->unionQueryPartitionKeyIndex == InvalidAttrNumber) { attributeEquivalence->unionQueryPartitionKeyIndex = partitionKeyIndex; } else if (attributeEquivalence->unionQueryPartitionKeyIndex != partitionKeyIndex) { continue; } Assert(varToBeAdded != NULL); AddToAttributeEquivalenceClass(attributeEquivalence, relationPlannerRoot, varToBeAdded); } /* * For queries of the form: * (SELECT ... FROM a JOIN b ...) UNION (SELECT .. FROM c JOIN d ... ) * * we determine whether all relations are joined on the partition column * by adding the equivalence classes that can be inferred from joins. */ List *relationRestrictionAttributeEquivalenceList = GenerateAttributeEquivalencesForRelationRestrictions(restrictionContext); List *joinRestrictionAttributeEquivalenceList = GenerateAttributeEquivalencesForJoinRestrictions(joinRestrictionContext); List *allAttributeEquivalenceList = list_concat(relationRestrictionAttributeEquivalenceList, joinRestrictionAttributeEquivalenceList); allAttributeEquivalenceList = lappend(allAttributeEquivalenceList, attributeEquivalence); if (!EquivalenceListContainsRelationsEquality(allAttributeEquivalenceList, restrictionContext)) { /* cannot confirm equality for all distribution colums */ return false; } if (!AllDistributedRelationsInRestrictionContextColocated(restrictionContext)) { /* distribution columns are equal, but tables are not co-located */ return false; } if (!PartitionColumnSelectedForOuterJoin(originalQuery, restrictionContext, joinRestrictionContext)) { /* outer join does not select partition column of outer relation */ return false; } return true; } /* * PartitionColumnSelectedForOuterJoin checks whether the partition column of * the outer relation is selected in the target list of the query. * * If there is no outer join, it returns true. */ static bool PartitionColumnSelectedForOuterJoin(Query *query, RelationRestrictionContext *restrictionContext, JoinRestrictionContext *joinRestrictionContext) { ListCell *joinRestrictionCell; foreach(joinRestrictionCell, joinRestrictionContext->joinRestrictionList) { JoinRestriction *joinRestriction = (JoinRestriction *) lfirst( joinRestrictionCell); /* Restriction context includes alternative plans, sufficient to check for left joins.*/ if (joinRestriction->joinType != JOIN_LEFT) { continue; } if (!PartitionColumnIsInTargetList(query, joinRestriction, restrictionContext)) { /* outer join does not select partition column of outer relation */ return false; } } return true; } /* * PartitionColumnIsInTargetList checks whether the partition column of * the given relation is included in the target list of the query. */ static bool PartitionColumnIsInTargetList(Query *query, JoinRestriction *joinRestriction, RelationRestrictionContext *restrictionContext) { Relids relids = joinRestriction->outerrelRelids; int relationId = -1; Index partitionKeyIndex = InvalidAttrNumber; while ((relationId = bms_next_member(relids, relationId)) >= 0) { RangeTblEntry *rte = joinRestriction->plannerInfo->simple_rte_array[relationId]; if (rte->rtekind != RTE_RELATION) { /* skip if it is not a relation */ continue; } int targetRTEIndex = GetRTEIdentity(rte); PartitionKeyForRTEIdentityInQuery(query, targetRTEIndex, &partitionKeyIndex); if (partitionKeyIndex == 0) { /* partition key is not in the target list */ return false; } } return true; } /* * RangeTableOffsetCompat returns the range table offset(in glob->finalrtable) for the appendRelInfo. */ static int RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo) { int parentCount = ParentCountPriorToAppendRel(root->append_rel_list, appendRelInfo); int skipParentCount = parentCount - 1; int i = 1; for (; i < root->simple_rel_array_size; i++) { RangeTblEntry *rte = root->simple_rte_array[i]; if (rte->inh) { /* * We skip the previous parents because we want to find the offset * for the given append rel info. */ if (skipParentCount > 0) { skipParentCount--; continue; } break; } } int indexInRtable = (i - 1); /* * Postgres adds the global rte array size to parent_relid as an offset. * Here we do the reverse operation: Commit on postgres side: * 6ef77cf46e81f45716ec981cb08781d426181378 */ int parentRelIndex = appendRelInfo->parent_relid - 1; return parentRelIndex - indexInRtable; } /* * FindUnionAllVar finds the variable used in union all for the side that has * relationRteIndex as its index and the same varattno as the partition key of * the given relation with relationOid. */ static Var * FindUnionAllVar(PlannerInfo *root, List *translatedVars, Oid relationOid, Index relationRteIndex, Index *partitionKeyIndex) { if (!IsCitusTableType(relationOid, STRICTLY_PARTITIONED_DISTRIBUTED_TABLE)) { /* we only care about hash and range partitioned tables */ *partitionKeyIndex = 0; return NULL; } Var *relationPartitionKey = DistPartitionKeyOrError(relationOid); AttrNumber childAttrNumber = 0; *partitionKeyIndex = 0; ListCell *translatedVarCell; foreach(translatedVarCell, translatedVars) { Node *targetNode = (Node *) lfirst(translatedVarCell); childAttrNumber++; if (!IsA(targetNode, Var)) { continue; } Var *targetVar = (Var *) lfirst(translatedVarCell); if (targetVar->varno == relationRteIndex && targetVar->varattno == relationPartitionKey->varattno) { *partitionKeyIndex = childAttrNumber; return targetVar; } } return NULL; } /* * RestrictionEquivalenceForPartitionKeys aims to deduce whether each of the RTE_RELATION * is joined with at least one another RTE_RELATION on their partition keys. If each * RTE_RELATION follows the above rule, we can conclude that all RTE_RELATIONs are * joined on their partition keys. * * Before doing the expensive equality checks, we do a cheaper check to understand * whether there are more than one distributed relations. Otherwise, we exit early. * * The function returns true if all relations are joined on their partition keys. * Otherwise, the function returns false. We ignore reference tables at all since * they don't have partition keys. * * In order to do that, we invented a new equivalence class namely: * AttributeEquivalenceClass. In very simple words, a AttributeEquivalenceClass is * identified by an unique id and consists of a list of AttributeEquivalenceMembers. * * Each AttributeEquivalenceMember is designed to identify attributes uniquely within the * whole query. The necessity of this arise since varno attributes are defined within * a single level of a query. Instead, here we want to identify each RTE_RELATION uniquely * and try to find equality among each RTE_RELATION's partition key. * * Each equality among RTE_RELATION is saved using an AttributeEquivalenceClass where * each member attribute is identified by a AttributeEquivalenceMember. In the final * step, we try generate a common attribute equivalence class that holds as much as * AttributeEquivalenceMembers whose attributes are a partition keys. * * RestrictionEquivalenceForPartitionKeys uses both relation restrictions and join restrictions * to find as much as information that Postgres planner provides to extensions. For the * details of the usage, please see GenerateAttributeEquivalencesForRelationRestrictions() * and GenerateAttributeEquivalencesForJoinRestrictions(). */ bool RestrictionEquivalenceForPartitionKeys(PlannerRestrictionContext *restrictionContext) { if (ContextContainsLocalRelation(restrictionContext->relationRestrictionContext)) { return false; } else if (!ContainsMultipleDistributedRelations(restrictionContext)) { /* there is a single distributed relation, no need to continue */ return true; } else if (ContextContainsAppendRelation( restrictionContext->relationRestrictionContext)) { /* we never consider append-distributed tables co-located */ return false; } List *attributeEquivalenceList = GenerateAllAttributeEquivalences(restrictionContext); return RestrictionEquivalenceForPartitionKeysViaEquivalences( restrictionContext, attributeEquivalenceList); } /* * RestrictionEquivalenceForPartitionKeysViaEquivalences follows the same rules * with RestrictionEquivalenceForPartitionKeys(). The only difference is that * this function allows passing pre-computed attribute equivalences along with * the planner restriction context. */ bool RestrictionEquivalenceForPartitionKeysViaEquivalences(PlannerRestrictionContext * plannerRestrictionContext, List *allAttributeEquivalenceList) { RelationRestrictionContext *restrictionContext = plannerRestrictionContext->relationRestrictionContext; /* there is a single distributed relation, no need to continue */ if (!ContainsMultipleDistributedRelations(plannerRestrictionContext)) { return true; } return EquivalenceListContainsRelationsEquality(allAttributeEquivalenceList, restrictionContext); } /* * ContainsMultipleDistributedRelations returns true if the input planner * restriction context contains more than one distributed relation. */ static bool ContainsMultipleDistributedRelations(PlannerRestrictionContext * plannerRestrictionContext) { RelationRestrictionContext *restrictionContext = plannerRestrictionContext->relationRestrictionContext; uint32 distributedRelationCount = UniqueRelationCount(restrictionContext, DISTRIBUTED_TABLE); /* * If the query includes a single relation which is not a reference table, * we should not check the partition column equality. * Consider two example cases: * (i) The query includes only a single colocated relation * (ii) A colocated relation is joined with a (or multiple) reference * table(s) where colocated relation is not joined on the partition key * * For the above two cases, we don't need to execute the partition column equality * algorithm. The reason is that the essence of this function is to ensure that the * tasks that are going to be created should not need data from other tasks. In both * cases mentioned above, the necessary data per task would be on available. */ if (distributedRelationCount <= 1) { return false; } return true; } /* * GenerateAllAttributeEquivalences gets the planner restriction context and returns * the list of all attribute equivalences based on both join restrictions and relation * restrictions. */ List * GenerateAllAttributeEquivalences(PlannerRestrictionContext *plannerRestrictionContext) { RelationRestrictionContext *relationRestrictionContext = plannerRestrictionContext->relationRestrictionContext; JoinRestrictionContext *joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; /* reset the equivalence id counter per call to prevent overflows */ AttributeEquivalenceId = 1; List *relationRestrictionAttributeEquivalenceList = GenerateAttributeEquivalencesForRelationRestrictions(relationRestrictionContext); List *joinRestrictionAttributeEquivalenceList = GenerateAttributeEquivalencesForJoinRestrictions(joinRestrictionContext); List *allAttributeEquivalenceList = list_concat( relationRestrictionAttributeEquivalenceList, joinRestrictionAttributeEquivalenceList); return allAttributeEquivalenceList; } /* * UniqueRelationCount iterates over the relations and returns the * unique relation count. We use RTEIdentity as the identifiers, so if * the same relation appears twice in the restrictionContext, we count * it as a single item. */ uint32 UniqueRelationCount(RelationRestrictionContext *restrictionContext, CitusTableType tableType) { ListCell *relationRestrictionCell = NULL; List *rteIdentityList = NIL; foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); Oid relationId = relationRestriction->relationId; CitusTableCacheEntry *cacheEntry = LookupCitusTableCacheEntry(relationId); if (cacheEntry == NULL) { /* we don't expect non-distributed tables, still be no harm to skip */ continue; } if (IsCitusTableTypeCacheEntry(cacheEntry, tableType)) { int rteIdentity = GetRTEIdentity(relationRestriction->rte); rteIdentityList = list_append_unique_int(rteIdentityList, rteIdentity); } } return list_length(rteIdentityList); } /* * EquivalenceListContainsRelationsEquality gets a list of attributed equivalence * list and a relation restriction context. The function first generates a common * equivalence class out of the attributeEquivalenceList. Later, the function checks * whether all the relations exists in the common equivalence class. * */ bool EquivalenceListContainsRelationsEquality(List *attributeEquivalenceList, RelationRestrictionContext *restrictionContext) { ListCell *commonEqClassCell = NULL; ListCell *relationRestrictionCell = NULL; Relids commonRteIdentities = NULL; /* * In general we're trying to expand existing the equivalence classes to find a * common equivalence class. The main goal is to test whether this main class * contains all partition keys of the existing relations. */ AttributeEquivalenceClass *commonEquivalenceClass = GenerateCommonEquivalence( attributeEquivalenceList, restrictionContext); /* add the rte indexes of relations to a bitmap */ foreach(commonEqClassCell, commonEquivalenceClass->equivalentAttributes) { AttributeEquivalenceClassMember *classMember = (AttributeEquivalenceClassMember *) lfirst(commonEqClassCell); int rteIdentity = classMember->rteIdentity; commonRteIdentities = bms_add_member(commonRteIdentities, rteIdentity); } /* check whether all relations exists in the main restriction list */ foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); int rteIdentity = GetRTEIdentity(relationRestriction->rte); /* we shouldn't check for the equality of non-distributed tables */ if (IsCitusTable(relationRestriction->relationId) && !HasDistributionKey(relationRestriction->relationId)) { continue; } if (!bms_is_member(rteIdentity, commonRteIdentities)) { return false; } } return true; } /* * GenerateAttributeEquivalencesForRelationRestrictions gets a relation restriction * context and returns a list of AttributeEquivalenceClass. * * The algorithm followed can be summarized as below: * * - Per relation restriction * - Per plannerInfo's eq_class * - Create an AttributeEquivalenceClass * - Add all Vars that appear in the plannerInfo's * eq_class to the AttributeEquivalenceClass * - While doing that, consider LATERAL vars as well. * See GetVarFromAssignedParam() for the details. Note * that we're using parentPlannerInfo while adding the * LATERAL vars given that we rely on that plannerInfo. * */ static List * GenerateAttributeEquivalencesForRelationRestrictions(RelationRestrictionContext *restrictionContext) { List *attributeEquivalenceList = NIL; ListCell *relationRestrictionCell = NULL; bool foundRLSPattern = false; if (restrictionContext == NULL) { return attributeEquivalenceList; } /* * First pass: Process equivalence classes using the original algorithm. * This builds the standard attribute equivalence list. * * Skip RLS pattern detection entirely if the query doesn't * use Row Level Security. The hasRowSecurity flag is checked from the query's * parse tree when any table has RLS policies active. This allows us to skip * both the pattern detection loop AND the expensive merge pass for non-RLS * queries (common case). * * For RLS queries, detect patterns efficiently. We only need * to find one EC with both Var + non-Var members to justify the merge pass. * Once found, skip further pattern checks and focus on building equivalences. */ bool skipRLSProcessing = true; foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); skipRLSProcessing = !relationRestriction->plannerInfo->parse->hasRowSecurity; List *equivalenceClasses = relationRestriction->plannerInfo->eq_classes; ListCell *equivalenceClassCell = NULL; foreach(equivalenceClassCell, equivalenceClasses) { EquivalenceClass *plannerEqClass = (EquivalenceClass *) lfirst(equivalenceClassCell); /* * RLS pattern = EC with both Var and non-Var (function) members. * Finding even one such pattern means we need the merge pass. */ if (!skipRLSProcessing && !foundRLSPattern) { bool hasVar = false; bool hasNonVar = false; ListCell *memberCell = NULL; foreach(memberCell, plannerEqClass->ec_members) { EquivalenceMember *member = (EquivalenceMember *) lfirst(memberCell); Node *expr = strip_implicit_coercions((Node *) member->em_expr); if (IsA(expr, Var)) { hasVar = true; } else if (member->em_is_const && !IsA(expr, Param) && !IsA(expr, Const)) { /* * Found a pseudoconstant expression (no Vars) that's not a * Param or Const - this is the RLS function pattern. */ hasNonVar = true; } /* Early exit: If we've found both, we have the pattern */ if (hasVar && hasNonVar) { foundRLSPattern = true; break; } } } AttributeEquivalenceClass *attributeEquivalence = AttributeEquivalenceClassForEquivalenceClass(plannerEqClass, relationRestriction); attributeEquivalenceList = AddAttributeClassToAttributeClassList(attributeEquivalenceList, attributeEquivalence); } } /* * Second pass: Handle RLS-specific case where PostgreSQL splits join conditions * across multiple EquivalenceClasses due to volatile functions in RLS policies. * * When RLS policies use volatile functions (e.g., current_setting()), PostgreSQL * creates separate EquivalenceClasses that both contain the same volatile function: * EC1: [table_a.tenant_id, current_setting(...)] * EC2: [table_b.tenant_id, current_setting(...)] * * We need to recognize that these should be merged to detect that tables are * joined on their distribution columns: [table_a.tenant_id, table_b.tenant_id] */ if (foundRLSPattern) { List *rlsMergedList = MergeEquivalenceClassesWithSameFunctions( restrictionContext); /* Append any newly created merged classes to the original list */ attributeEquivalenceList = list_concat(attributeEquivalenceList, rlsMergedList); } return attributeEquivalenceList; } /* * MergeEquivalenceClassesWithSameFunctions scans equivalence classes * looking for RLS-specific patterns where volatile functions cause PostgreSQL to * split what should be a single join condition across multiple EquivalenceClasses. * * This function specifically targets the pattern: * EC1: [table_a.col, COERCEVIAIO(func(...))] * EC2: [table_b.col, COERCEVIAIO(func(...))] * * Where the underlying function calls are identical after stripping implicit coercions * (e.g., both resolve to current_setting('session.current_tenant_id')). * * PostgreSQL wraps RLS policy expressions in COERCEVIAIO nodes to handle type * conversions (e.g., text → UUID). We strip these to compare the actual function calls. * * Returns a list of newly created merged AttributeEquivalenceClasses. Each merged * class contains the Var members from pairs of EquivalenceClasses that share identical * non-Var expressions. For example, if EC1 contains [table_a.tenant_id, func()] and * EC2 contains [table_b.tenant_id, func()], the returned list will include a new * AttributeEquivalenceClass with [table_a.tenant_id, table_b.tenant_id]. Only classes * with 2+ members are returned (indicating an actual join between tables). */ static List * MergeEquivalenceClassesWithSameFunctions(RelationRestrictionContext *restrictionContext) { List *newlyMergedClasses = NIL; List *ecGroupList = NIL; /* List of ECGroupByExpr* */ ListCell *relationRestrictionCell = NULL; if (restrictionContext == NULL) { return NIL; } /* * Phase 1: Collect candidate ECs and group them by their non-Var expressions. * * Strategy: For each EC, extract and strip all non-Var expressions, then * find or create a group for each unique expression. This gives us direct * access to all ECs sharing the same expression. */ foreach(relationRestrictionCell, restrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); List *equivalenceClasses = relationRestriction->plannerInfo->eq_classes; ListCell *equivalenceClassCell = NULL; foreach(equivalenceClassCell, equivalenceClasses) { EquivalenceClass *ec = (EquivalenceClass *) lfirst(equivalenceClassCell); bool hasVar = false; List *nonVarExprs = NIL; ListCell *memberCell = NULL; /* * Single pass through EC members: collect Vars and non-Var expressions. * Strip coercions once and cache the results. */ foreach(memberCell, ec->ec_members) { EquivalenceMember *member = (EquivalenceMember *) lfirst(memberCell); Node *expr = strip_implicit_coercions((Node *) member->em_expr); if (IsA(expr, Var)) { hasVar = true; } else if (member->em_is_const && !IsA(expr, Param) && !IsA(expr, Const)) { /* * Found a pseudoconstant expression (no Vars) - potential RLS function. * After stripping, this is typically a FUNCEXPR like * current_setting('session.current_tenant_id'). */ nonVarExprs = lappend(nonVarExprs, expr); } } /* Only process ECs with both Var and non-Var members (RLS pattern) */ if (!hasVar || nonVarExprs == NIL) { continue; } /* * For each non-Var expression in this EC, find or create a group. * Multiple ECs with the same expression will be grouped together. */ ListCell *exprCell = NULL; foreach(exprCell, nonVarExprs) { Node *strippedExpr = (Node *) lfirst(exprCell); ECGroupByExpr *matchingGroup = NULL; ListCell *groupCell = NULL; /* Search for existing group with this expression */ foreach(groupCell, ecGroupList) { ECGroupByExpr *group = (ECGroupByExpr *) lfirst(groupCell); if (equal(group->strippedExpr, strippedExpr)) { matchingGroup = group; break; } } /* Create new group if this is the first EC with this expression */ if (matchingGroup == NULL) { matchingGroup = palloc0(sizeof(ECGroupByExpr)); matchingGroup->strippedExpr = strippedExpr; matchingGroup->ecsWithThisExpr = NIL; matchingGroup->varsInTheseECs = NIL; ecGroupList = lappend(ecGroupList, matchingGroup); } /* Add this EC to the group (avoid duplicates) */ if (!list_member_ptr(matchingGroup->ecsWithThisExpr, ec)) { matchingGroup->ecsWithThisExpr = lappend(matchingGroup->ecsWithThisExpr, ec); } } } } /* * Phase 2: For each group with 2+ ECs, extract all Vars and create a merged * AttributeEquivalenceClass. This is where we detect the join pattern. * * Idea here is that if multiple ECs share the same non-Var expression (e.g., RLS * function), then all Vars in those ECs are implicitly equal to each other. */ ListCell *groupCell = NULL; foreach(groupCell, ecGroupList) { ECGroupByExpr *group = (ECGroupByExpr *) lfirst(groupCell); /* Skip groups with only one EC - no join to detect */ if (list_length(group->ecsWithThisExpr) < 2) { continue; } /* * Extract all Vars from all ECs in this group. * These Vars are implicitly equal via the shared expression. */ ListCell *ecCell = NULL; foreach(ecCell, group->ecsWithThisExpr) { EquivalenceClass *ec = (EquivalenceClass *) lfirst(ecCell); ListCell *memberCell = NULL; foreach(memberCell, ec->ec_members) { EquivalenceMember *member = (EquivalenceMember *) lfirst(memberCell); Node *expr = strip_implicit_coercions((Node *) member->em_expr); if (IsA(expr, Var)) { /* Cache this Var for later processing */ group->varsInTheseECs = lappend(group->varsInTheseECs, expr); } } } /* Need at least 2 Vars from different tables to represent a join */ if (list_length(group->varsInTheseECs) < 2) { continue; } /* * Create the merged AttributeEquivalenceClass. */ AttributeEquivalenceClass *mergedClass = palloc0(sizeof(AttributeEquivalenceClass)); mergedClass->equivalenceId = AttributeEquivalenceId++; mergedClass->equivalentAttributes = NIL; /* * Match each Var to its RelationRestriction by comparing varno to * the restriction's index field (which is the RTE index). */ ListCell *varCell = NULL; foreach(varCell, group->varsInTheseECs) { Var *var = (Var *) lfirst(varCell); ListCell *relResCell = NULL; bool foundMatch = false; /* * Find the RelationRestriction that corresponds to this Var. * The index field contains the RTE index (varno) of the relation. */ foreach(relResCell, restrictionContext->relationRestrictionList) { RelationRestriction *relRestriction = (RelationRestriction *) lfirst(relResCell); /* Direct match: varno equals the restriction's index */ if (var->varno == relRestriction->index) { /* * Process this Var through AddToAttributeEquivalenceClass. * This handles subqueries, UNION ALL, LATERAL joins, etc. */ AddToAttributeEquivalenceClass(mergedClass, relRestriction->plannerInfo, var); foundMatch = true; break; } } /* * If we didn't find a matching restriction, this Var might be from * a context not tracked in our restriction list (e.g., subquery). * We skip it as we only care about Vars from distributed tables. */ if (!foundMatch) { elog(DEBUG2, "Skipping Var with varno=%d in RLS merge - " "no matching RelationRestriction found", var->varno); } } /* Only emit if we successfully merged attributes from multiple sources */ if (list_length(mergedClass->equivalentAttributes) >= 2) { newlyMergedClasses = lappend(newlyMergedClasses, mergedClass); } } return newlyMergedClasses; } /* * AttributeEquivalenceClassForEquivalenceClass is a helper function for * GenerateAttributeEquivalencesForRelationRestrictions. The function takes an * EquivalenceClass and the relation restriction that the equivalence class * belongs to. The function returns an AttributeEquivalenceClass that is composed * of ec_members that are simple Var references. * * The function also takes case of LATERAL joins by simply replacing the PARAM_EXEC * with the corresponding expression. */ static AttributeEquivalenceClass * AttributeEquivalenceClassForEquivalenceClass(EquivalenceClass *plannerEqClass, RelationRestriction *relationRestriction) { AttributeEquivalenceClass *attributeEquivalence = palloc0(sizeof(AttributeEquivalenceClass)); ListCell *equivilanceMemberCell = NULL; PlannerInfo *plannerInfo = relationRestriction->plannerInfo; attributeEquivalence->equivalenceId = AttributeEquivalenceId++; foreach(equivilanceMemberCell, plannerEqClass->ec_members) { EquivalenceMember *equivalenceMember = (EquivalenceMember *) lfirst(equivilanceMemberCell); Node *equivalenceNode = strip_implicit_coercions( (Node *) equivalenceMember->em_expr); Expr *strippedEquivalenceExpr = (Expr *) equivalenceNode; Var *expressionVar = NULL; if (IsA(strippedEquivalenceExpr, Param)) { PlannerInfo *outerNodeRoot = NULL; Param *equivalenceParam = (Param *) strippedEquivalenceExpr; expressionVar = GetVarFromAssignedParam(relationRestriction->outerPlanParamsList, equivalenceParam, &outerNodeRoot); if (expressionVar) { AddToAttributeEquivalenceClass(attributeEquivalence, outerNodeRoot, expressionVar); } } else if (IsA(strippedEquivalenceExpr, Var)) { expressionVar = (Var *) strippedEquivalenceExpr; AddToAttributeEquivalenceClass(attributeEquivalence, plannerInfo, expressionVar); } } return attributeEquivalence; } /* * GetVarFromAssignedParam returns the Var that is assigned to the given * plannerParam if its kind is PARAM_EXEC. * * If the paramkind is not equal to PARAM_EXEC the function returns NULL. Similarly, * if there is no Var corresponding to the given param is, the function returns NULL. * * Rationale behind this function: * * While iterating through the equivalence classes of RTE_RELATIONs, we * observe that there are PARAM type of equivalence member expressions for * the RTE_RELATIONs which actually belong to lateral vars from the other query * levels. * * We're also keeping track of the RTE_RELATION's outer nodes' * plan_params lists which is expected to hold the parameters that are required * for its lower level queries as it is documented: * * plan_params contains the expressions that this query level needs to * make available to a lower query level that is currently being planned. * * This function is a helper function to iterate through the outer node's query's * plan_params and looks for the param that the equivalence member has. The * comparison is done via the "paramid" field. Finally, if the found parameter's * item is a Var, we conclude that Postgres standard_planner replaced the Var * with the Param on assign_param_for_var() function * @src/backend/optimizer/plan/subselect.c. */ static Var * GetVarFromAssignedParam(List *outerPlanParamsList, Param *plannerParam, PlannerInfo **rootContainingVar) { Var *assignedVar = NULL; ListCell *rootPlanParamsCell = NULL; Assert(plannerParam != NULL); /* we're only interested in parameters that Postgres added for execution */ if (plannerParam->paramkind != PARAM_EXEC) { return NULL; } foreach(rootPlanParamsCell, outerPlanParamsList) { RootPlanParams *outerPlanParams = lfirst(rootPlanParamsCell); assignedVar = SearchPlannerParamList(outerPlanParams->plan_params, plannerParam); if (assignedVar != NULL) { *rootContainingVar = outerPlanParams->root; break; } } #if PG_VERSION_NUM >= PG_VERSION_18 /* * In PG18+, the dereferenced PARAM node could be a GroupVar if the * query has a GROUP BY. In that case, we need to make an extra * hop to get the underlying Var from the grouping expressions. */ if (assignedVar != NULL) { Query *parse = (*rootContainingVar)->parse; if (parse->hasGroupRTE) { RangeTblEntry *rte = rt_fetch(assignedVar->varno, parse->rtable); if (rte->rtekind == RTE_GROUP) { Assert(assignedVar->varattno >= 1 && assignedVar->varattno <= list_length(rte->groupexprs)); Node *groupVar = list_nth(rte->groupexprs, assignedVar->varattno - 1); if (IsA(groupVar, Var)) { assignedVar = (Var *) groupVar; } else { /* todo: handle PlaceHolderVar case if needed */ ereport(DEBUG2, (errmsg( "GroupVar maps to non-Var group expr; bailing out"))); assignedVar = NULL; } } } } #endif return assignedVar; } /* * SearchPlannerParamList searches in plannerParamList and returns the Var that * corresponds to the given plannerParam. If there is no Var corresponding to the * given param is, the function returns NULL. */ static Var * SearchPlannerParamList(List *plannerParamList, Param *plannerParam) { Var *assignedVar = NULL; ListCell *plannerParameterCell = NULL; foreach(plannerParameterCell, plannerParamList) { PlannerParamItem *plannerParamItem = (PlannerParamItem *) lfirst(plannerParameterCell); if (plannerParamItem->paramId != plannerParam->paramid) { continue; } /* TODO: Should we consider PlaceHolderVar? */ if (!IsA(plannerParamItem->item, Var)) { continue; } assignedVar = (Var *) plannerParamItem->item; break; } return assignedVar; } /* * GenerateCommonEquivalence gets a list of unrelated AttributeEquiavalenceClass * whose all members are partition keys. * * With the equivalence classes, the function follows the algorithm * outlined below: * * - Add the first equivalence class to the common equivalence class * - Then, iterate on the remaining equivalence classes * - If any of the members equal to the common equivalence class * add all the members of the equivalence class to the common * class * - Start the iteration from the beginning. The reason is that * in case any of the classes we've passed is equivalent to the * newly added one. To optimize the algorithm, we utilze the * equivalence class ids and skip the ones that are already added. * - Finally, return the common equivalence class. */ static AttributeEquivalenceClass * GenerateCommonEquivalence(List *attributeEquivalenceList, RelationRestrictionContext *relationRestrictionContext) { Bitmapset *addedEquivalenceIds = NULL; uint32 equivalenceListSize = list_length(attributeEquivalenceList); uint32 equivalenceClassIndex = 0; AttributeEquivalenceClass *commonEquivalenceClass = palloc0( sizeof(AttributeEquivalenceClass)); commonEquivalenceClass->equivalenceId = 0; /* * We seed the common equivalence class with a the first distributed * table since we always want the input distributed relations to be * on the common class. */ AttributeEquivalenceClass *firstEquivalenceClass = GenerateEquivalenceClassForRelationRestriction(relationRestrictionContext); /* we skip the calculation if there are not enough information */ if (equivalenceListSize < 1 || firstEquivalenceClass == NULL) { return commonEquivalenceClass; } commonEquivalenceClass->equivalentAttributes = firstEquivalenceClass->equivalentAttributes; addedEquivalenceIds = bms_add_member(addedEquivalenceIds, firstEquivalenceClass->equivalenceId); while (equivalenceClassIndex < equivalenceListSize) { ListCell *equivalenceMemberCell = NULL; bool restartLoop = false; AttributeEquivalenceClass *currentEquivalenceClass = list_nth( attributeEquivalenceList, equivalenceClassIndex); /* * This is an optimization. If we already added the same equivalence class, * we could skip it since we've already added all the relevant equivalence * members. */ if (bms_is_member(currentEquivalenceClass->equivalenceId, addedEquivalenceIds)) { equivalenceClassIndex++; continue; } foreach(equivalenceMemberCell, currentEquivalenceClass->equivalentAttributes) { AttributeEquivalenceClassMember *attributeEquialanceMember = (AttributeEquivalenceClassMember *) lfirst(equivalenceMemberCell); if (AttributeClassContainsAttributeClassMember(attributeEquialanceMember, commonEquivalenceClass)) { ListConcatUniqueAttributeClassMemberLists(commonEquivalenceClass, currentEquivalenceClass); addedEquivalenceIds = bms_add_member(addedEquivalenceIds, currentEquivalenceClass-> equivalenceId); /* * It seems inefficient to start from the beginning. * But, we should somehow restart from the beginning to test that * whether the already skipped ones are equal or not. */ restartLoop = true; break; } } if (restartLoop) { equivalenceClassIndex = 0; } else { ++equivalenceClassIndex; } } return commonEquivalenceClass; } /* * GenerateEquivalenceClassForRelationRestriction generates an AttributeEquivalenceClass * with a single AttributeEquivalenceClassMember. */ static AttributeEquivalenceClass * GenerateEquivalenceClassForRelationRestriction(RelationRestrictionContext * relationRestrictionContext) { ListCell *relationRestrictionCell = NULL; AttributeEquivalenceClassMember *eqMember = NULL; AttributeEquivalenceClass *eqClassForRelation = NULL; foreach(relationRestrictionCell, relationRestrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); Var *relationPartitionKey = DistPartitionKey(relationRestriction->relationId); if (relationPartitionKey) { eqClassForRelation = palloc0(sizeof(AttributeEquivalenceClass)); eqMember = palloc0(sizeof(AttributeEquivalenceClassMember)); eqMember->relationId = relationRestriction->relationId; eqMember->rteIdentity = GetRTEIdentity(relationRestriction->rte); eqMember->varno = relationRestriction->index; eqMember->varattno = relationPartitionKey->varattno; eqClassForRelation->equivalentAttributes = lappend(eqClassForRelation->equivalentAttributes, eqMember); break; } } return eqClassForRelation; } /* * ListConcatUniqueAttributeClassMemberLists gets two attribute equivalence classes. It * basically concatenates attribute equivalence member lists uniquely and updates the * firstClass' member list with the list. * * Basically, the function iterates over the secondClass' member list and checks whether * it already exists in the firstClass' member list. If not, the member is added to the * firstClass. */ static void ListConcatUniqueAttributeClassMemberLists(AttributeEquivalenceClass *firstClass, AttributeEquivalenceClass *secondClass) { ListCell *equivalenceClassMemberCell = NULL; List *equivalenceMemberList = secondClass->equivalentAttributes; foreach(equivalenceClassMemberCell, equivalenceMemberList) { AttributeEquivalenceClassMember *newEqMember = (AttributeEquivalenceClassMember *) lfirst(equivalenceClassMemberCell); if (AttributeClassContainsAttributeClassMember(newEqMember, firstClass)) { continue; } firstClass->equivalentAttributes = lappend(firstClass->equivalentAttributes, newEqMember); } } /* * GenerateAttributeEquivalencesForJoinRestrictions gets a join restriction * context and returns a list of AttrributeEquivalenceClass. * * The algorithm followed can be summarized as below: * * - Per join restriction * - Per RestrictInfo of the join restriction * - Check whether the join restriction is in the form of (Var1 = Var2) * - Create an AttributeEquivalenceClass * - Add both Var1 and Var2 to the AttributeEquivalenceClass */ static List * GenerateAttributeEquivalencesForJoinRestrictions(JoinRestrictionContext * joinRestrictionContext) { List *attributeEquivalenceList = NIL; ListCell *joinRestrictionCell = NULL; if (joinRestrictionContext == NULL) { return attributeEquivalenceList; } foreach(joinRestrictionCell, joinRestrictionContext->joinRestrictionList) { JoinRestriction *joinRestriction = (JoinRestriction *) lfirst(joinRestrictionCell); ListCell *restrictionInfoList = NULL; foreach(restrictionInfoList, joinRestriction->joinRestrictInfoList) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(restrictionInfoList); Expr *restrictionClause = rinfo->clause; if (!IsA(restrictionClause, OpExpr)) { continue; } OpExpr *restrictionOpExpr = (OpExpr *) restrictionClause; if (list_length(restrictionOpExpr->args) != 2) { continue; } if (!OperatorImplementsEquality(restrictionOpExpr->opno)) { continue; } Node *leftNode = linitial(restrictionOpExpr->args); Node *rightNode = lsecond(restrictionOpExpr->args); /* we also don't want implicit coercions */ Expr *strippedLeftExpr = (Expr *) strip_implicit_coercions((Node *) leftNode); Expr *strippedRightExpr = (Expr *) strip_implicit_coercions( (Node *) rightNode); if (!(IsA(strippedLeftExpr, Var) && IsA(strippedRightExpr, Var))) { continue; } Var *leftVar = (Var *) strippedLeftExpr; Var *rightVar = (Var *) strippedRightExpr; AttributeEquivalenceClass *attributeEquivalence = palloc0( sizeof(AttributeEquivalenceClass)); attributeEquivalence->equivalenceId = AttributeEquivalenceId++; AddToAttributeEquivalenceClass(attributeEquivalence, joinRestriction->plannerInfo, leftVar); AddToAttributeEquivalenceClass(attributeEquivalence, joinRestriction->plannerInfo, rightVar); attributeEquivalenceList = AddAttributeClassToAttributeClassList(attributeEquivalenceList, attributeEquivalence); } } return attributeEquivalenceList; } /* * AddToAttributeEquivalenceClass is a key function for building the attribute * equivalences. The function gets a plannerInfo, var and attribute equivalence * class. It searches for the RTE_RELATION(s) that the input var belongs to and * adds the found Var(s) to the input attribute equivalence class. * * Note that the input var could come from a subquery (i.e., not directly from an * RTE_RELATION). That's the reason we recursively call the function until the * RTE_RELATION found. * * The algorithm could be summarized as follows: * * - If the RTE that corresponds to a relation * - Generate an AttributeEquivalenceMember and add to the input * AttributeEquivalenceClass * - If the RTE that corresponds to a subquery * - If the RTE that corresponds to a UNION ALL subquery * - Iterate on each of the appendRels (i.e., each of the UNION ALL query) * - Recursively add all children of the set operation's * corresponding target entries * - If the corresponding subquery entry is a UNION set operation * - Recursively add all children of the set operation's * corresponding target entries * - If the corresponding subquery is a regular subquery (i.e., No set operations) * - Recursively try to add the corresponding target entry to the * equivalence class */ static void AddToAttributeEquivalenceClass(AttributeEquivalenceClass *attributeEquivalenceClass, PlannerInfo *root, Var *varToBeAdded) { /* punt if it's a whole-row var rather than a plain column reference */ if (varToBeAdded->varattno == InvalidAttrNumber) { return; } /* we also don't want to process ctid, tableoid etc */ if (varToBeAdded->varattno < InvalidAttrNumber) { return; } /* outer join checks in PG16 */ if (IsRelOptOuterJoin(root, varToBeAdded->varno)) { return; } RangeTblEntry *rangeTableEntry = root->simple_rte_array[varToBeAdded->varno]; if (rangeTableEntry->rtekind == RTE_RELATION) { AddRteRelationToAttributeEquivalenceClass(attributeEquivalenceClass, rangeTableEntry, varToBeAdded); } else if (rangeTableEntry->rtekind == RTE_SUBQUERY) { AddRteSubqueryToAttributeEquivalenceClass(attributeEquivalenceClass, rangeTableEntry, root, varToBeAdded); } } /* * AddRteSubqueryToAttributeEquivalenceClass adds the given var to the given * attribute equivalence class. * * The main algorithm is outlined in AddToAttributeEquivalenceClass(). */ static void AddRteSubqueryToAttributeEquivalenceClass(AttributeEquivalenceClass *attributeEquivalenceClass, RangeTblEntry *rangeTableEntry, PlannerInfo *root, Var *varToBeAdded) { RelOptInfo *baseRelOptInfo = find_base_rel(root, varToBeAdded->varno); Query *targetSubquery = GetTargetSubquery(root, rangeTableEntry, varToBeAdded); /* * We might not always get the subquery because the subquery might be a * referencing to RELOPT_DEADREL such that the corresponding join is * removed via join_is_removable(). * * Returning here implies that PostgreSQL doesn't need to plan the * subquery because it doesn't contribute to the query result at all. * Since the relations in the subquery does not appear in the query * plan as well, Citus would simply ignore the subquery and treat that * as a safe-to-pushdown subquery. */ if (targetSubquery == NULL) { return; } TargetEntry *subqueryTargetEntry = get_tle_by_resno(targetSubquery->targetList, varToBeAdded->varattno); /* if we fail to find corresponding target entry, do not proceed */ if (subqueryTargetEntry == NULL || subqueryTargetEntry->resjunk) { return; } /* we're only interested in Vars */ if (!IsA(subqueryTargetEntry->expr, Var)) { return; } varToBeAdded = (Var *) subqueryTargetEntry->expr; /* * "inh" flag is set either when inheritance or "UNION ALL" exists in the * subquery. Here we're only interested in the "UNION ALL" case. * * Else, we check one more thing: Does the subquery contain a "UNION" query. * If so, we recursively traverse all "UNION" tree and add the corresponding * target list elements to the attribute equivalence. * * Finally, if it is a regular subquery (i.e., does not contain UNION or UNION ALL), * we simply recurse to find the corresponding RTE_RELATION to add to the * equivalence class. * * Note that we're treating "UNION" and "UNION ALL" clauses differently given * that postgres planner process/plans them separately. */ if (rangeTableEntry->inh) { AddUnionAllSetOperationsToAttributeEquivalenceClass(attributeEquivalenceClass, root, varToBeAdded); } else if (targetSubquery->setOperations) { AddUnionSetOperationsToAttributeEquivalenceClass(attributeEquivalenceClass, baseRelOptInfo->subroot, (SetOperationStmt *) targetSubquery->setOperations, varToBeAdded); } else if (varToBeAdded && IsA(varToBeAdded, Var) && varToBeAdded->varlevelsup == 0) { AddToAttributeEquivalenceClass(attributeEquivalenceClass, baseRelOptInfo->subroot, varToBeAdded); } } /* * GetTargetSubquery returns the corresponding subquery for the given planner root, * range table entry and the var. * * The aim of this function is to simplify extracting the subquery in case of "UNION ALL" * queries. */ static Query * GetTargetSubquery(PlannerInfo *root, RangeTblEntry *rangeTableEntry, Var *varToBeAdded) { Query *targetSubquery = NULL; /* * For subqueries other than "UNION ALL", find the corresponding targetSubquery. See * the details of how we process subqueries in the below comments. */ if (!rangeTableEntry->inh) { RelOptInfo *baseRelOptInfo = find_base_rel(root, varToBeAdded->varno); /* If the targetSubquery was not planned, we have to punt */ if (baseRelOptInfo->subroot == NULL) { return NULL; } Assert(IsA(baseRelOptInfo->subroot, PlannerInfo)); targetSubquery = baseRelOptInfo->subroot->parse; Assert(IsA(targetSubquery, Query)); } else { targetSubquery = rangeTableEntry->subquery; } return targetSubquery; } /* * IsRelOptOuterJoin returns true if the RelOpt referenced * by varNo is an outer join, false otherwise. */ bool IsRelOptOuterJoin(PlannerInfo *root, int varNo) { if (root->simple_rel_array_size <= varNo) { return true; } RelOptInfo *rel = root->simple_rel_array[varNo]; if (rel == NULL) { /* must be an outer join */ return true; } return false; } /* * AddUnionAllSetOperationsToAttributeEquivalenceClass recursively iterates on all the * append rels, sets the varno's accordingly and adds the * var the given equivalence class. */ static void AddUnionAllSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass * attributeEquivalenceClass, PlannerInfo *root, Var *varToBeAdded) { List *appendRelList = root->append_rel_list; ListCell *appendRelCell = NULL; /* iterate on the queries that are part of UNION ALL subqueries */ foreach(appendRelCell, appendRelList) { AppendRelInfo *appendRelInfo = (AppendRelInfo *) lfirst(appendRelCell); /* * We're only interested in UNION ALL clauses and parent_reloid is invalid * only for UNION ALL (i.e., equals to a legitimate Oid for inheritance) */ if (appendRelInfo->parent_reloid != InvalidOid) { continue; } int rtoffset = RangeTableOffsetCompat(root, appendRelInfo); int childRelId = appendRelInfo->child_relid - rtoffset; if (root->simple_rel_array_size <= childRelId) { /* we prefer to return over an Assert or error to be defensive */ return; } RangeTblEntry *rte = root->simple_rte_array[childRelId]; if (rte->inh) { /* * This code-path may require improvements. If a leaf of a UNION ALL * (e.g., an entry in appendRelList) itself is another UNION ALL * (e.g., rte->inh = true), the logic here might get into an infinite * recursion. * * The downside of "continue" here is that certain UNION ALL queries * that are safe to pushdown may not be pushed down. */ continue; } else if (rte->rtekind == RTE_RELATION) { Index partitionKeyIndex = 0; List *translatedVars = TranslatedVarsForRteIdentity(GetRTEIdentity(rte)); Var *varToBeAddedOnUnionAllSubquery = FindUnionAllVar(root, translatedVars, rte->relid, childRelId, &partitionKeyIndex); if (partitionKeyIndex == 0) { /* no partition key on the target list */ continue; } if (attributeEquivalenceClass->unionQueryPartitionKeyIndex == 0) { /* the first partition key index we found */ attributeEquivalenceClass->unionQueryPartitionKeyIndex = partitionKeyIndex; } else if (attributeEquivalenceClass->unionQueryPartitionKeyIndex != partitionKeyIndex) { /* * Partition keys on the leaves of the UNION ALL queries on * different ordinal positions. We cannot pushdown, so skip. */ continue; } if (varToBeAddedOnUnionAllSubquery != NULL) { AddToAttributeEquivalenceClass(attributeEquivalenceClass, root, varToBeAddedOnUnionAllSubquery); } } else { /* set the varno accordingly for this specific child */ varToBeAdded->varno = childRelId; AddToAttributeEquivalenceClass(attributeEquivalenceClass, root, varToBeAdded); } } } /* * ParentCountPriorToAppendRel returns the number of parents that come before * the given append rel info. */ static int ParentCountPriorToAppendRel(List *appendRelList, AppendRelInfo *targetAppendRelInfo) { int targetParentIndex = targetAppendRelInfo->parent_relid; Bitmapset *parent_ids = NULL; AppendRelInfo *appendRelInfo = NULL; foreach_declared_ptr(appendRelInfo, appendRelList) { int curParentIndex = appendRelInfo->parent_relid; if (curParentIndex <= targetParentIndex) { parent_ids = bms_add_member(parent_ids, curParentIndex); } } return bms_num_members(parent_ids); } /* * AddUnionSetOperationsToAttributeEquivalenceClass recursively iterates on all the * setOperations and adds each corresponding target entry to the given equivalence * class. * * Although the function silently accepts INTERSECT and EXPECT set operations, they are * rejected later in the planning. We prefer this behavior to provide better error * messages. */ static void AddUnionSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass * attributeEquivalenceClass, PlannerInfo *root, SetOperationStmt *setOperation, Var *varToBeAdded) { List *rangeTableIndexList = NIL; ListCell *rangeTableIndexCell = NULL; ExtractRangeTableIndexWalker((Node *) setOperation, &rangeTableIndexList); foreach(rangeTableIndexCell, rangeTableIndexList) { int rangeTableIndex = lfirst_int(rangeTableIndexCell); varToBeAdded->varno = rangeTableIndex; AddToAttributeEquivalenceClass(attributeEquivalenceClass, root, varToBeAdded); } } /* * AddRteRelationToAttributeEquivalenceClass adds the given var to the given equivalence * class using the rteIdentity provided by the rangeTableEntry. Note that * rteIdentities are only assigned to RTE_RELATIONs and this function asserts * the input rte to be an RTE_RELATION. */ static void AddRteRelationToAttributeEquivalenceClass(AttributeEquivalenceClass * attrEquivalenceClass, RangeTblEntry *rangeTableEntry, Var *varToBeAdded) { Oid relationId = rangeTableEntry->relid; /* we don't consider local tables in the equality on columns */ if (!IsCitusTable(relationId)) { return; } Var *relationPartitionKey = DistPartitionKey(relationId); Assert(rangeTableEntry->rtekind == RTE_RELATION); /* * we only calculate the equivalence of distributed tables. * This leads to certain shortcomings in the query planning when reference * tables and/or intermediate results are involved in the query. For example, * the following query patterns could actually be pushed-down in a single iteration * "(intermediate_res INNER JOIN dist dist1) INNER JOIN dist dist2 " or * "(ref INNER JOIN dist dist1) JOIN dist dist2" * * However, if there are no explicit join conditions between distributed tables, * the planner cannot deduce the equivalence between the distributed tables. * * Instead, we should be able to track all the equivalences between range table * entries, and expand distributed table equivalences that happens via * reference table/intermediate results */ if (relationPartitionKey == NULL) { return; } /* we're only interested in distribution columns */ if (relationPartitionKey->varattno != varToBeAdded->varattno) { return; } AttributeEquivalenceClassMember *attributeEqMember = palloc0( sizeof(AttributeEquivalenceClassMember)); attributeEqMember->varattno = varToBeAdded->varattno; attributeEqMember->varno = varToBeAdded->varno; attributeEqMember->rteIdentity = GetRTEIdentity(rangeTableEntry); attributeEqMember->relationId = rangeTableEntry->relid; attrEquivalenceClass->equivalentAttributes = lappend(attrEquivalenceClass->equivalentAttributes, attributeEqMember); } /* * AttributeClassContainsAttributeClassMember returns true if it the input class member * is already exists in the attributeEquivalenceClass. An equality is identified by the * varattno and rteIdentity. */ static bool AttributeClassContainsAttributeClassMember(AttributeEquivalenceClassMember *inputMember, AttributeEquivalenceClass * attributeEquivalenceClass) { ListCell *classCell = NULL; foreach(classCell, attributeEquivalenceClass->equivalentAttributes) { AttributeEquivalenceClassMember *memberOfClass = (AttributeEquivalenceClassMember *) lfirst(classCell); if (memberOfClass->rteIdentity == inputMember->rteIdentity && memberOfClass->varattno == inputMember->varattno) { return true; } } return false; } /* * AddAttributeClassToAttributeClassList checks for certain properties of the * input attributeEquivalence before adding it to the attributeEquivalenceList. * * Firstly, the function skips adding NULL attributeEquivalence to the list. * Secondly, since an attribute equivalence class with a single member does * not contribute to our purposes, we skip such classed adding to the list. * Finally, we don't want to add an equivalence class whose exact equivalent * already exists in the list. */ static List * AddAttributeClassToAttributeClassList(List *attributeEquivalenceList, AttributeEquivalenceClass *attributeEquivalence) { ListCell *attributeEquivalenceCell = NULL; if (attributeEquivalence == NULL) { return attributeEquivalenceList; } /* * Note that in some cases we allow having equivalentAttributes with zero or * one elements. For the details, see AddToAttributeEquivalenceClass(). */ List *equivalentAttributes = attributeEquivalence->equivalentAttributes; if (list_length(equivalentAttributes) < 2) { return attributeEquivalenceList; } /* we don't want to add an attributeEquivalence which already exists */ foreach(attributeEquivalenceCell, attributeEquivalenceList) { AttributeEquivalenceClass *currentAttributeEquivalence = (AttributeEquivalenceClass *) lfirst(attributeEquivalenceCell); if (AttributeEquivalencesAreEqual(currentAttributeEquivalence, attributeEquivalence)) { return attributeEquivalenceList; } } attributeEquivalenceList = lappend(attributeEquivalenceList, attributeEquivalence); return attributeEquivalenceList; } /* * AttributeEquivalencesAreEqual returns true if both input attribute equivalence * classes contains exactly the same members. */ static bool AttributeEquivalencesAreEqual(AttributeEquivalenceClass *firstAttributeEquivalence, AttributeEquivalenceClass *secondAttributeEquivalence) { List *firstEquivalenceMemberList = firstAttributeEquivalence->equivalentAttributes; List *secondEquivalenceMemberList = secondAttributeEquivalence->equivalentAttributes; ListCell *firstAttributeEquivalenceCell = NULL; ListCell *secondAttributeEquivalenceCell = NULL; if (list_length(firstEquivalenceMemberList) != list_length( secondEquivalenceMemberList)) { return false; } foreach(firstAttributeEquivalenceCell, firstEquivalenceMemberList) { AttributeEquivalenceClassMember *firstEqMember = (AttributeEquivalenceClassMember *) lfirst(firstAttributeEquivalenceCell); bool foundAnEquivalentMember = false; foreach(secondAttributeEquivalenceCell, secondEquivalenceMemberList) { AttributeEquivalenceClassMember *secondEqMember = (AttributeEquivalenceClassMember *) lfirst( secondAttributeEquivalenceCell); if (firstEqMember->rteIdentity == secondEqMember->rteIdentity && firstEqMember->varattno == secondEqMember->varattno) { foundAnEquivalentMember = true; break; } } /* we couldn't find an equivalent member */ if (!foundAnEquivalentMember) { return false; } } return true; } /* * ContainsUnionSubquery gets a queryTree and returns true if the query * contains * - a subquery with UNION set operation * - no joins above the UNION set operation in the query tree * * Note that the function allows top level unions being wrapped into aggregations * queries and/or simple projection queries that only selects some fields from * the lower level queries. * * If there exists joins before the set operations, the function returns false. * Similarly, if the query does not contain any union set operations, the * function returns false. */ bool ContainsUnionSubquery(Query *queryTree) { List *rangeTableList = queryTree->rtable; List *joinTreeTableIndexList = NIL; ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); uint32 joiningRangeTableCount = list_length(joinTreeTableIndexList); /* don't allow joins on top of unions */ if (joiningRangeTableCount > 1) { return false; } /* subquery without FROM */ if (joiningRangeTableCount == 0) { return false; } Index subqueryRteIndex = linitial_int(joinTreeTableIndexList); RangeTblEntry *rangeTableEntry = rt_fetch(subqueryRteIndex, rangeTableList); if (rangeTableEntry->rtekind != RTE_SUBQUERY) { return false; } Query *subqueryTree = rangeTableEntry->subquery; Node *setOperations = subqueryTree->setOperations; if (setOperations != NULL) { SetOperationStmt *setOperationStatement = (SetOperationStmt *) setOperations; /* * Note that the set operation tree is traversed elsewhere for ensuring * that we only support UNIONs. */ if (setOperationStatement->op != SETOP_UNION) { return false; } return true; } return ContainsUnionSubquery(subqueryTree); } /* * PartitionKeyForRTEIdentityInQuery finds the partition key var(if exists), * in the given original query for the rte that has targetRTEIndex. */ static Var * PartitionKeyForRTEIdentityInQuery(Query *originalQuery, int targetRTEIndex, Index *partitionKeyIndex) { Query *originalQueryContainingRTEIdentity = FindQueryContainingRTEIdentity(originalQuery, targetRTEIndex); if (!originalQueryContainingRTEIdentity) { /* * We should always find the query but we have this check for sanity. * This check makes sure that if there is a bug while finding the query, * we don't get a crash etc. and the only downside will be we might be recursively * planning a query that could be pushed down. */ return NULL; } /* * This approach fails to detect when * the top level query might have the column indexes in different order: * explain * SELECT count(*) FROM * ( * SELECT user_id,value_2 FROM events_table * UNION * SELECT value_2, user_id FROM (SELECT user_id, value_2, random() FROM events_table) as foo * ) foobar; * So we hit https://github.com/citusdata/citus/issues/5093. */ List *relationTargetList = originalQueryContainingRTEIdentity->targetList; ListCell *targetEntryCell = NULL; Index partitionKeyTargetAttrIndex = 0; foreach(targetEntryCell, relationTargetList) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); Expr *targetExpression = targetEntry->expr; partitionKeyTargetAttrIndex++; bool skipOuterVars = false; if (!targetEntry->resjunk && IsA(targetExpression, Var) && IsPartitionColumn(targetExpression, originalQueryContainingRTEIdentity, skipOuterVars)) { Var *targetColumn = (Var *) targetExpression; /* * We find the referenced table column to support distribution * columns that are correlated. */ RangeTblEntry *rteContainingPartitionKey = NULL; FindReferencedTableColumn(targetExpression, NIL, originalQueryContainingRTEIdentity, &targetColumn, &rteContainingPartitionKey, skipOuterVars); if (rteContainingPartitionKey->rtekind == RTE_RELATION && GetRTEIdentity(rteContainingPartitionKey) == targetRTEIndex) { *partitionKeyIndex = partitionKeyTargetAttrIndex; return (Var *) copyObject(targetColumn); } } } return NULL; } /* * FindQueryContainingRTEIdentity finds the query/subquery that has an RTE * with rteIndex in its rtable. */ static Query * FindQueryContainingRTEIdentity(Query *query, int rteIndex) { FindQueryContainingRteIdentityContext *findRteIdentityContext = palloc0(sizeof(FindQueryContainingRteIdentityContext)); findRteIdentityContext->targetRTEIdentity = rteIndex; FindQueryContainingRTEIdentityInternal((Node *) query, findRteIdentityContext); return findRteIdentityContext->query; } /* * FindQueryContainingRTEIdentityInternal walks on the given node to find a query * which has an RTE that has a given rteIdentity. */ static bool FindQueryContainingRTEIdentityInternal(Node *node, FindQueryContainingRteIdentityContext *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; Query *parentQuery = context->query; context->query = query; if (query_tree_walker(query, FindQueryContainingRTEIdentityInternal, context, QTW_EXAMINE_RTES_BEFORE)) { return true; } context->query = parentQuery; return false; } if (!IsA(node, RangeTblEntry)) { return expression_tree_walker(node, FindQueryContainingRTEIdentityInternal, context); } RangeTblEntry *rte = (RangeTblEntry *) node; if (rte->rtekind == RTE_RELATION) { if (GetRTEIdentity(rte) == context->targetRTEIdentity) { return true; } } return false; } /* * AllDistributedRelationsInRestrictionContextColocated determines whether all of the * distributed relations in the given relation restrictions list are co-located. */ static bool AllDistributedRelationsInRestrictionContextColocated(RelationRestrictionContext * restrictionContext) { RelationRestriction *relationRestriction = NULL; List *relationIdList = NIL; /* check whether all relations exists in the main restriction list */ foreach_declared_ptr(relationRestriction, restrictionContext->relationRestrictionList) { relationIdList = lappend_oid(relationIdList, relationRestriction->relationId); } return AllDistributedRelationsInListColocated(relationIdList); } /* * AllDistributedRelationsInRTEListColocated determines whether all of the * distributed relations in the given RangeTableEntry list are co-located. */ bool AllDistributedRelationsInRTEListColocated(List *rangeTableEntryList) { RangeTblEntry *rangeTableEntry = NULL; List *relationIdList = NIL; foreach_declared_ptr(rangeTableEntry, rangeTableEntryList) { relationIdList = lappend_oid(relationIdList, rangeTableEntry->relid); } return AllDistributedRelationsInListColocated(relationIdList); } /* * AllDistributedRelationsInListColocated determines whether all of the * distributed relations in the given list are co-located. */ bool AllDistributedRelationsInListColocated(List *relationList) { int initialColocationId = INVALID_COLOCATION_ID; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationList) { if (!IsCitusTable(relationId)) { /* not interested in Postgres tables */ continue; } if (!IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { /* not interested in non-distributed tables */ continue; } if (IsCitusTableType(relationId, APPEND_DISTRIBUTED)) { /* * If we got to this point, it means there are multiple distributed * relations and at least one of them is append-distributed. Since * we do not consider append-distributed tables to be co-located, * we can immediately return false. */ return false; } int colocationId = TableColocationId(relationId); if (initialColocationId == INVALID_COLOCATION_ID) { initialColocationId = colocationId; } else if (colocationId != initialColocationId) { return false; } } return true; } /* * RelationIdList returns list of unique relation ids in query tree. */ List * DistributedRelationIdList(Query *query) { List *rangeTableList = NIL; List *relationIdList = NIL; ListCell *tableEntryCell = NULL; ExtractRangeTableRelationWalker((Node *) query, &rangeTableList); List *tableEntryList = TableEntryList(rangeTableList); foreach(tableEntryCell, tableEntryList) { TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); Oid relationId = tableEntry->relationId; if (!IsCitusTable(relationId)) { continue; } relationIdList = list_append_unique_oid(relationIdList, relationId); } return relationIdList; } /* * FilterPlannerRestrictionForQuery gets a planner restriction context and * set of rte identities. It returns the restrictions that that appear * in the queryRteIdentities and returns a newly allocated * PlannerRestrictionContext. The function also sets all the other fields of * the PlannerRestrictionContext with respect to the filtered restrictions. */ PlannerRestrictionContext * FilterPlannerRestrictionForQuery(PlannerRestrictionContext *plannerRestrictionContext, Query *query) { Relids queryRteIdentities = QueryRteIdentities(query); RelationRestrictionContext *relationRestrictionContext = plannerRestrictionContext->relationRestrictionContext; JoinRestrictionContext *joinRestrictionContext = plannerRestrictionContext->joinRestrictionContext; RelationRestrictionContext *filteredRelationRestrictionContext = FilterRelationRestrictionContext(relationRestrictionContext, queryRteIdentities); JoinRestrictionContext *filtererdJoinRestrictionContext = FilterJoinRestrictionContext(joinRestrictionContext, queryRteIdentities); /* allocate the filtered planner restriction context and set all the fields */ PlannerRestrictionContext *filteredPlannerRestrictionContext = palloc0( sizeof(PlannerRestrictionContext)); filteredPlannerRestrictionContext->fastPathRestrictionContext = palloc0( sizeof(FastPathRestrictionContext)); filteredPlannerRestrictionContext->memoryContext = plannerRestrictionContext->memoryContext; int totalRelationCount = UniqueRelationCount( filteredRelationRestrictionContext, ANY_CITUS_TABLE_TYPE); int referenceRelationCount = UniqueRelationCount( filteredRelationRestrictionContext, REFERENCE_TABLE); filteredRelationRestrictionContext->allReferenceTables = (totalRelationCount == referenceRelationCount); /* finally set the relation and join restriction contexts */ filteredPlannerRestrictionContext->relationRestrictionContext = filteredRelationRestrictionContext; filteredPlannerRestrictionContext->joinRestrictionContext = filtererdJoinRestrictionContext; return filteredPlannerRestrictionContext; } /* * GetRestrictInfoListForRelation gets a range table entry and planner * restriction context. The function returns a list of expressions that * appear in the restriction context for only the given relation. And, * all the varnos are set to 1. */ List * GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, PlannerRestrictionContext *plannerRestrictionContext) { RelationRestriction *relationRestriction = RelationRestrictionForRelation(rangeTblEntry, plannerRestrictionContext); if (relationRestriction == NULL) { return NIL; } RelOptInfo *relOptInfo = relationRestriction->relOptInfo; List *baseRestrictInfo = relOptInfo->baserestrictinfo; bool joinConditionIsOnFalse = JoinConditionIsOnFalse(relOptInfo->joininfo); if (joinConditionIsOnFalse) { /* found WHERE false, no need to continue, we just return a false clause */ bool value = false; bool isNull = false; Node *falseClause = makeBoolConst(value, isNull); return list_make1(falseClause); } List *restrictExprList = NIL; RestrictInfo *restrictInfo = NULL; foreach_declared_ptr(restrictInfo, baseRestrictInfo) { Expr *restrictionClause = restrictInfo->clause; /* * we cannot process some restriction clauses because they are not * safe to recursively plan. */ if (FindNodeMatchingCheckFunction((Node *) restrictionClause, IsNotSafeRestrictionToRecursivelyPlan)) { continue; } /* * If the restriction involves multiple tables, we cannot add it to * input relation's expression list. */ Relids varnos = pull_varnos(relationRestriction->plannerInfo, (Node *) restrictionClause); if (bms_num_members(varnos) != 1) { continue; } /* * PlaceHolderVar is not relevant to be processed inside a restriction clause. * Otherwise, pull_var_clause_default would throw error. PG would create * the restriction to physical Var that PlaceHolderVar points anyway, so it is * safe to skip this restriction. */ if (FindNodeMatchingCheckFunction((Node *) restrictionClause, HasPlaceHolderVar)) { continue; } /* * We're going to add this restriction expression to a subquery * which consists of only one relation in its jointree. Thus, * simply set the varnos accordingly. */ Expr *copyOfRestrictClause = (Expr *) copyObject((Node *) restrictionClause); List *varClauses = pull_var_clause_default((Node *) copyOfRestrictClause); Var *column = NULL; foreach_declared_ptr(column, varClauses) { column->varno = SINGLE_RTE_INDEX; column->varnosyn = SINGLE_RTE_INDEX; } restrictExprList = lappend(restrictExprList, copyOfRestrictClause); } return restrictExprList; } /* * RelationRestrictionForRelation gets the relation restriction for the given * range table entry. */ RelationRestriction * RelationRestrictionForRelation(RangeTblEntry *rangeTableEntry, PlannerRestrictionContext *plannerRestrictionContext) { int rteIdentity = GetRTEIdentity(rangeTableEntry); RelationRestrictionContext *relationRestrictionContext = plannerRestrictionContext->relationRestrictionContext; Relids queryRteIdentities = bms_make_singleton(rteIdentity); RelationRestrictionContext *filteredRelationRestrictionContext = FilterRelationRestrictionContext(relationRestrictionContext, queryRteIdentities); List *filteredRelationRestrictionList = filteredRelationRestrictionContext->relationRestrictionList; if (list_length(filteredRelationRestrictionList) < 1) { return NULL; } RelationRestriction *relationRestriction = (RelationRestriction *) linitial(filteredRelationRestrictionList); return relationRestriction; } /* * IsNotSafeRestrictionToRecursivelyPlan returns true if the given node * is not a safe restriction to be recursivelly planned. */ static bool IsNotSafeRestrictionToRecursivelyPlan(Node *node) { if (IsA(node, Param) || IsA(node, SubLink) || IsA(node, SubPlan) || IsA(node, AlternativeSubPlan)) { return true; } return false; } /* * HasPlaceHolderVar returns true if given node contains any PlaceHolderVar. */ static bool HasPlaceHolderVar(Node *node) { return IsA(node, PlaceHolderVar); } /* * FilterRelationRestrictionContext gets a relation restriction context and * set of rte identities. It returns the relation restrictions that that appear * in the queryRteIdentities and returns a newly allocated * RelationRestrictionContext. */ RelationRestrictionContext * FilterRelationRestrictionContext(RelationRestrictionContext *relationRestrictionContext, Relids queryRteIdentities) { RelationRestrictionContext *filteredRestrictionContext = palloc0(sizeof(RelationRestrictionContext)); ListCell *relationRestrictionCell = NULL; foreach(relationRestrictionCell, relationRestrictionContext->relationRestrictionList) { RelationRestriction *relationRestriction = (RelationRestriction *) lfirst(relationRestrictionCell); int rteIdentity = GetRTEIdentity(relationRestriction->rte); if (bms_is_member(rteIdentity, queryRteIdentities)) { filteredRestrictionContext->relationRestrictionList = lappend(filteredRestrictionContext->relationRestrictionList, relationRestriction); } } return filteredRestrictionContext; } /* * FilterJoinRestrictionContext gets a join restriction context and * set of rte identities. It returns the join restrictions that that appear * in the queryRteIdentities and returns a newly allocated * JoinRestrictionContext. * * Note that the join restriction is added to the return context as soon as * any range table entry that appear in the join belongs to queryRteIdentities. */ static JoinRestrictionContext * FilterJoinRestrictionContext(JoinRestrictionContext *joinRestrictionContext, Relids queryRteIdentities) { JoinRestrictionContext *filtererdJoinRestrictionContext = palloc0(sizeof(JoinRestrictionContext)); ListCell *joinRestrictionCell = NULL; foreach(joinRestrictionCell, joinRestrictionContext->joinRestrictionList) { JoinRestriction *joinRestriction = (JoinRestriction *) lfirst(joinRestrictionCell); RangeTblEntry **rangeTableEntries = joinRestriction->plannerInfo->simple_rte_array; int rangeTableArrayLength = joinRestriction->plannerInfo->simple_rel_array_size; if (RangeTableArrayContainsAnyRTEIdentities(rangeTableEntries, rangeTableArrayLength, queryRteIdentities)) { filtererdJoinRestrictionContext->joinRestrictionList = lappend( filtererdJoinRestrictionContext->joinRestrictionList, joinRestriction); } } /* * No need to re calculate has join fields as we are still operating on * the same query and as these values are calculated per-query basis. */ filtererdJoinRestrictionContext->hasSemiJoin = joinRestrictionContext->hasSemiJoin; filtererdJoinRestrictionContext->hasOuterJoin = joinRestrictionContext->hasOuterJoin; return filtererdJoinRestrictionContext; } /* * RangeTableArrayContainsAnyRTEIdentities returns true if any of the range table entries * in rangeTableEntries array is a range table relation specified in queryRteIdentities. */ static bool RangeTableArrayContainsAnyRTEIdentities(RangeTblEntry **rangeTableEntries, int rangeTableArrayLength, Relids queryRteIdentities) { /* simple_rte_array starts from 1, see plannerInfo struct */ for (int rteIndex = 1; rteIndex < rangeTableArrayLength; ++rteIndex) { RangeTblEntry *rangeTableEntry = rangeTableEntries[rteIndex]; List *rangeTableRelationList = NULL; ListCell *rteRelationCell = NULL; #if PG_VERSION_NUM >= PG_VERSION_18 /* * In PG18+, planner array simple_rte_array may contain NULL entries * for "dead relations". See PG commits 5f6f951 and e9a20e4 for details. */ if (rangeTableEntry == NULL) { continue; } #endif /* * Get list of all RTE_RELATIONs in the given range table entry * (i.e.,rangeTableEntry could be a subquery where we're interested * in relations). */ if (rangeTableEntry->rtekind == RTE_SUBQUERY) { ExtractRangeTableRelationWalker((Node *) rangeTableEntry->subquery, &rangeTableRelationList); } else if (rangeTableEntry->rtekind == RTE_RELATION) { ExtractRangeTableRelationWalker((Node *) rangeTableEntry, &rangeTableRelationList); } else { /* we currently do not accept any other RTE types here */ continue; } foreach(rteRelationCell, rangeTableRelationList) { RangeTblEntry *rteRelation = (RangeTblEntry *) lfirst(rteRelationCell); Assert(rteRelation->rtekind == RTE_RELATION); int rteIdentity = GetRTEIdentity(rteRelation); if (bms_is_member(rteIdentity, queryRteIdentities)) { return true; } } } return false; } /* * QueryRteIdentities gets a queryTree, find get all the rte identities assigned by * us. */ static Relids QueryRteIdentities(Query *queryTree) { List *rangeTableList = NULL; ListCell *rangeTableCell = NULL; Relids queryRteIdentities = NULL; /* extract range table entries for simple relations only */ ExtractRangeTableRelationWalker((Node *) queryTree, &rangeTableList); foreach(rangeTableCell, rangeTableList) { RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); /* we're only interested in relations */ Assert(rangeTableEntry->rtekind == RTE_RELATION); int rteIdentity = GetRTEIdentity(rangeTableEntry); queryRteIdentities = bms_add_member(queryRteIdentities, rteIdentity); } return queryRteIdentities; } ================================================ FILE: src/backend/distributed/planner/shard_pruning.c ================================================ /*------------------------------------------------------------------------- * * shard_pruning.c * Shard pruning related code. * * The goal of shard pruning is to find a minimal (super)set of shards that * need to be queried to find rows matching the expression in a query. * * In PruneShards we first make a compact representation of the given * query logical tree. This tree represents boolean operators and its * associated valid constraints (expression nodes) and whether boolean * operator has associated unknown constraints. This allows essentially * unknown constraints to be replaced by a simple placeholder flag. * * For example query: WHERE (hash_col IN (1,2)) AND (other_col=1 OR other_col=2) * Gets transformed by steps: * 1. AND(hash_col IN (1,2), OR(X, X)) * 2. AND(hash_col IN (1,2), OR(X)) * 3. AND(hash_col IN (1,2), X) * Where X represents any set of unrecognized unprunable constraint(s). * * Above allows the following pruning machinery to understand that * the target shard is determined solely by constraint: hash_col IN (1,2). * Here it does not matter what X is as its ANDed by a valid constraint. * Pruning machinery will fail, returning all shards, if it encounters * eg. OR(hash_col=1, X) as this condition does not limit the target shards. * * PruneShards secondly computes a simplified disjunctive normal form (DNF) * of the logical tree as a list of pruning instances. Each pruning instance * contains all AND-ed constraints on the partition column. An OR expression * will result in two or more new pruning instances being added for the * subexpressions. The "parent" instance is marked isPartial and ignored * during pruning. * * We use the distributive property for constraints of the form P AND (Q OR R) * to rewrite it to (P AND Q) OR (P AND R) by copying constraints from parent * to "child" pruning instances. However, we do not distribute nested * expressions. While (P OR Q) AND (R OR S) is logically equivalent to (P AND * R) OR (P AND S) OR (Q AND R) OR (Q AND S), in our implementation it becomes * P OR Q OR R OR S. This is acceptable since this will always result in a * superset of shards. If this proves to be a issue in practice, a more * complete algorithm could be implemented. * * We then evaluate each non-partial pruning instance in the disjunction * through the following, increasingly expensive, steps: * * 1) If there is a constant equality constraint on the partition column, and * no overlapping shards exist, find the shard interval in which the * constant falls * * 2) If there is a hash range constraint on the partition column, find the * shard interval matching the range * * 3) If there are range constraints (e.g. (a > 0 AND a < 10)) on the * partition column, find the shard intervals that overlap with the range * * 4) If there are overlapping shards, exhaustively search all shards that are * not excluded by constraints * * Finally, the union of the shards found by each pruning instance is * returned. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/planner.h" #include "parser/parse_coerce.h" #include "utils/arrayaccess.h" #include "utils/catcache.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/ruleutils.h" #include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_18 typedef OpIndexInterpretation OpBtreeInterpretation; #endif #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* * Tree node for compact representation of the given query logical tree. * Represent a single boolean operator node and its associated * valid constraints (expression nodes) and invalid constraint flag. */ typedef struct PruningTreeNode { /* Indicates is this AND/OR boolean operator */ BoolExprType boolop; /* Does this boolean operator have unknown/unprunable constraint(s) */ bool hasInvalidConstraints; /* List of recognized valid prunable constraints of this boolean opearator */ List *validConstraints; /* Child boolean producing operators. Parents are always different from their children */ List *childBooleanNodes; } PruningTreeNode; /* * Context used for expression_tree_walker */ typedef struct PruningTreeBuildContext { Var *partitionColumn; PruningTreeNode *current; } PruningTreeBuildContext; /* * A pruning instance is a set of ANDed constraints on a partition key. */ typedef struct PruningInstance { /* Does this instance contain any prunable expressions? */ bool hasValidConstraint; /* * This constraint never evaluates to true, i.e. pruning does not have to * be performed. */ bool evaluatesToFalse; /* * Constraints on the partition column value. If multiple values are * found the more restrictive one should be stored here. Even for * a hash-partitioned table, actual column-values are stored here, *not* * hashed values. */ Const *lessConsts; Const *lessEqualConsts; Const *equalConsts; Const *greaterEqualConsts; Const *greaterConsts; /* * Constraint using a pre-hashed column value. The constant will store the * hashed value, not the original value of the restriction. */ Const *hashedEqualConsts; /* * Has this PruningInstance been added to * ClauseWalkerContext->pruningInstances? This is not done immediately, * but the first time a constraint (independent of us being able to handle * that constraint) is found. */ bool addedToPruningInstances; /* * When OR clauses are found, the non-ORed part (think of a < 3 AND (a > 5 * OR a > 7)) of the expression is stored in one PruningInstance which is * then copied for the ORed expressions. The original is marked as * isPartial, to avoid being used for pruning. */ bool isPartial; } PruningInstance; /* * Partial instances that need to be finished building. This is used to * collect all ANDed restrictions, before looking into ORed expressions. */ typedef struct PendingPruningInstance { PruningInstance *instance; PruningTreeNode *continueAt; } PendingPruningInstance; typedef union \ { \ FunctionCallInfoBaseData fcinfo; \ /* ensure enough space for nargs args is available */ \ char fcinfo_data[SizeForFunctionCallInfo(2)]; \ } FunctionCall2InfoData; /* * We also ignore this warning in ./configure, but that's not always enough. * The flags that are used during compilation by ./configure are determined by * the compiler support it detects. This is usually GCC. This warning is only * present in clang. So it would normally be fine to not use it with GCC. The * problem is that clang is used to compile the JIT bitcode when postgres is * compiled with -with-llvm. So in the end both clang and GCC are used to * compile the project. * * So the flag is not provided on the command line, because ./configure notices * that GCC doesn't support it. But this warning persists when compiling the * bitcode. So that's why we ignore it here explicitly. */ #ifdef __clang__ #pragma clang diagnostic ignored "-Wgnu-variable-sized-type-not-at-end" #endif /* __clang__ */ /* * Data necessary to perform a single PruneShards(). */ typedef struct ClauseWalkerContext { Var *partitionColumn; char partitionMethod; /* ORed list of pruning targets */ List *pruningInstances; /* * Partially built PruningInstances, that need to be completed by doing a * separate PrunableExpressionsWalker() pass. */ List *pendingInstances; /* PruningInstance currently being built, all eligible constraints are added here */ PruningInstance *currentPruningInstance; /* * Information about function calls we need to perform. Re-using the same * FunctionCall2InfoData, instead of using FunctionCall2Coll, is often * cheaper. */ FunctionCall2InfoData compareValueFunctionCall; FunctionCall2InfoData compareIntervalFunctionCall; } ClauseWalkerContext; static bool BuildPruningTree(Node *node, PruningTreeBuildContext *context); static void SimplifyPruningTree(PruningTreeNode *node, PruningTreeNode *parent); static void PrunableExpressions(PruningTreeNode *node, ClauseWalkerContext *context); static void PrunableExpressionsWalker(PruningTreeNode *node, ClauseWalkerContext *context); static bool IsValidPartitionKeyRestriction(OpExpr *opClause); static void AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *varClause, Const *constantClause); static void AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, ScalarArrayOpExpr * arrayOperatorExpression); static bool SAORestrictions(ScalarArrayOpExpr *arrayOperatorExpression, Var *partitionColumn, List **requestedRestrictions); static void ErrorTypesDontMatch(Oid firstType, Oid firstCollId, Oid secondType, Oid secondCollId); static bool IsValidHashRestriction(OpExpr *opClause); static void AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *varClause, Const *constantClause); static void AddNewConjuction(ClauseWalkerContext *context, PruningTreeNode *node); static PruningInstance * CopyPartialPruningInstance(PruningInstance *sourceInstance); static List * ShardArrayToList(ShardInterval **shardArray, int length); static List * DeepCopyShardIntervalList(List *originalShardIntervalList); static int PerformValueCompare(FunctionCallInfo compareFunctionCall, Datum a, Datum b); static int PerformCompare(FunctionCallInfo compareFunctionCall); static List * PruneOne(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune); static List * PruneWithBoundaries(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune); static List * ExhaustivePrune(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune); static bool ExhaustivePruneOne(ShardInterval *curInterval, ClauseWalkerContext *context, PruningInstance *prune); static int UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, FunctionCallInfo compareFunction, bool includeMin); static int LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, FunctionCallInfo compareFunction, bool includeMax); static PruningTreeNode * CreatePruningNode(BoolExprType boolop); static OpExpr * SAORestrictionArrayEqualityOp(ScalarArrayOpExpr *arrayOperatorExpression, Var *partitionColumn); static void DebugLogNode(char *fmt, Node *node, List *deparseCtx); static void DebugLogPruningInstance(PruningInstance *pruning, List *deparseCtx); static int ConstraintCount(PruningTreeNode *node); /* * PruneShards returns all shards from a distributed table that cannot be * proven to be eliminated by whereClauseList. * * For non-distributed tables such as reference table, the function * simply returns the single shard that the table has. * * When there is a single = filter in the where * clause list, the constant is written to the partitionValueConst pointer. */ List * PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, Const **partitionValueConst) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); int shardCount = cacheEntry->shardIntervalArrayLength; char partitionMethod = cacheEntry->partitionMethod; ClauseWalkerContext context = { 0 }; ListCell *pruneCell; List *prunedList = NIL; bool foundRestriction = false; bool foundPartitionColumnValue = false; Const *singlePartitionValueConst = NULL; /* there are no shards to return */ if (shardCount == 0) { return NIL; } /* always return empty result if WHERE clause is of the form: false (AND ..) */ if (ContainsFalseClause(whereClauseList)) { return NIL; } /* short circuit for non-distributed tables such as reference table */ if (!HasDistributionKeyCacheEntry(cacheEntry)) { prunedList = ShardArrayToList(cacheEntry->sortedShardIntervalArray, cacheEntry->shardIntervalArrayLength); return DeepCopyShardIntervalList(prunedList); } context.partitionMethod = partitionMethod; context.partitionColumn = PartitionColumn(relationId, rangeTableId); context.currentPruningInstance = palloc0(sizeof(PruningInstance)); if (cacheEntry->shardIntervalCompareFunction) { /* initiate function call info once (allows comparators to cache metadata) */ InitFunctionCallInfoData(*(FunctionCallInfo) & context.compareIntervalFunctionCall, cacheEntry->shardIntervalCompareFunction, 2, cacheEntry->partitionColumn->varcollid, NULL, NULL); } else { ereport(ERROR, (errmsg("shard pruning not possible without " "a shard interval comparator"))); } if (cacheEntry->shardColumnCompareFunction) { /* initiate function call info once (allows comparators to cache metadata) */ InitFunctionCallInfoData(*(FunctionCallInfo) & context.compareValueFunctionCall, cacheEntry->shardColumnCompareFunction, 2, cacheEntry->partitionColumn->varcollid, NULL, NULL); } else { ereport(ERROR, (errmsg("shard pruning not possible without " "a partition column comparator"))); } PruningTreeNode *tree = CreatePruningNode(AND_EXPR); PruningTreeBuildContext treeBuildContext = { 0 }; treeBuildContext.current = tree; treeBuildContext.partitionColumn = PartitionColumn(relationId, rangeTableId); /* Build logical tree of prunable restrictions and invalid restrictions */ BuildPruningTree((Node *) whereClauseList, &treeBuildContext); /* Simplify logic tree of prunable restrictions */ SimplifyPruningTree(tree, NULL); /* Figure out what we can prune on */ PrunableExpressions(tree, &context); List *debugLoggedPruningInstances = NIL; /* * Prune using each of the PrunableInstances we found, and OR results * together. */ foreach(pruneCell, context.pruningInstances) { PruningInstance *prune = (PruningInstance *) lfirst(pruneCell); /* * If this is a partial instance, a fully built one has also been * added. Skip. */ if (prune->isPartial) { continue; } /* * If the current instance has no prunable expressions, we'll have to * return all shards. No point in continuing pruning in that case. */ if (!prune->hasValidConstraint) { foundRestriction = false; break; } if (context.partitionMethod == DISTRIBUTE_BY_HASH) { if (!prune->evaluatesToFalse && !prune->equalConsts && !prune->hashedEqualConsts) { /* if hash-partitioned and no equals constraints, return all shards */ foundRestriction = false; break; } else if (partitionValueConst != NULL && prune->equalConsts != NULL) { if (!foundPartitionColumnValue) { /* remember the partition column value */ singlePartitionValueConst = prune->equalConsts; foundPartitionColumnValue = true; } else if (singlePartitionValueConst == NULL) { /* already found multiple partition column values */ } else if (!equal(prune->equalConsts, singlePartitionValueConst)) { /* found multiple partition column values */ singlePartitionValueConst = NULL; } } } List *pruneOneList = PruneOne(cacheEntry, &context, prune); if (prunedList) { /* * We can use list_union_ptr, which is a lot faster than doing * comparing shards by value, because all the ShardIntervals are * guaranteed to be from * CitusTableCacheEntry->sortedShardIntervalArray (thus having the * same pointer values). */ prunedList = list_union_ptr(prunedList, pruneOneList); } else { prunedList = pruneOneList; } foundRestriction = true; if (IsLoggableLevel(DEBUG3) && pruneOneList) { debugLoggedPruningInstances = lappend(debugLoggedPruningInstances, prune); } } /* found no valid restriction, build list of all shards */ if (!foundRestriction) { prunedList = ShardArrayToList(cacheEntry->sortedShardIntervalArray, cacheEntry->shardIntervalArrayLength); } if (IsLoggableLevel(DEBUG3)) { char *relationName = get_rel_name(relationId); if (foundRestriction && debugLoggedPruningInstances != NIL) { List *deparseCtx = deparse_context_for("unknown", relationId); foreach(pruneCell, debugLoggedPruningInstances) { PruningInstance *prune = (PruningInstance *) lfirst(pruneCell); DebugLogPruningInstance(prune, deparseCtx); } } else { ereport(DEBUG3, (errmsg("no shard pruning constraints on %s found", relationName))); } ereport(DEBUG3, (errmsg("shard count after pruning for %s: %d", relationName, list_length(prunedList)))); } /* if requested, copy the partition value constant */ if (partitionValueConst != NULL) { if (singlePartitionValueConst != NULL) { *partitionValueConst = copyObject(singlePartitionValueConst); } else { *partitionValueConst = NULL; } } /* * Deep copy list, so it's independent of the CitusTableCacheEntry * contents. */ return DeepCopyShardIntervalList(prunedList); } /* * IsValidConditionNode checks whether node is a valid constraint for pruning. */ static bool IsValidConditionNode(Node *node, Var *partitionColumn) { if (IsA(node, OpExpr)) { OpExpr *opClause = (OpExpr *) node; Var *varClause = NULL; if (VarConstOpExprClause(opClause, &varClause, NULL)) { if (equal(varClause, partitionColumn)) { return IsValidPartitionKeyRestriction(opClause); } else if (varClause->varattno == RESERVED_HASHED_COLUMN_ID) { return IsValidHashRestriction(opClause); } } return false; } else if (IsA(node, ScalarArrayOpExpr)) { ScalarArrayOpExpr *arrayOperatorExpression = (ScalarArrayOpExpr *) node; return SAORestrictions(arrayOperatorExpression, partitionColumn, NULL); } else { return false; } } /* * BuildPruningTree builds a logical tree of constraints for pruning. */ static bool BuildPruningTree(Node *node, PruningTreeBuildContext *context) { if (node == NULL) { return false; } if (IsA(node, List)) { return expression_tree_walker(node, BuildPruningTree, context); } else if (IsA(node, BoolExpr)) { BoolExpr *boolExpr = (BoolExpr *) node; if (boolExpr->boolop == NOT_EXPR) { /* * With Var-Const conditions we should not encounter NOT_EXPR nodes. * Postgres standard planner applies De Morgan's laws to remove them. * We still encounter them with subqueries inside NOT, for example with: * WHERE id NOT IN (SELECT id FROM something). * We treat these as invalid constraints for pruning when we encounter them. */ context->current->hasInvalidConstraints = true; return false; } else if (context->current->boolop != boolExpr->boolop) { PruningTreeNode *child = CreatePruningNode(boolExpr->boolop); context->current->childBooleanNodes = lappend( context->current->childBooleanNodes, child); PruningTreeBuildContext newContext = { 0 }; newContext.partitionColumn = context->partitionColumn; newContext.current = child; return expression_tree_walker((Node *) boolExpr->args, BuildPruningTree, &newContext); } else { return expression_tree_walker(node, BuildPruningTree, context); } } else if (IsValidConditionNode(node, context->partitionColumn)) { context->current->validConstraints = lappend(context->current->validConstraints, node); return false; } else { context->current->hasInvalidConstraints = true; return false; } } /* * SimplifyPruningTree reduces logical tree of valid and invalid constraints for pruning. * The goal is to remove any node having just a single constraint associated with it. * This constraint is assigned to the parent logical node. * * For example 'AND(hash_col = 1, OR(X))' gets simplified to 'AND(hash_col = 1, X)', * where X is any unknown condition. */ static void SimplifyPruningTree(PruningTreeNode *node, PruningTreeNode *parent) { /* Copy list of children as its mutated inside the loop */ List *childBooleanNodes = list_copy(node->childBooleanNodes); ListCell *cell; foreach(cell, childBooleanNodes) { PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); SimplifyPruningTree(child, node); } if (!parent) { /* Root is always ANDed expressions */ Assert(node->boolop == AND_EXPR); return; } /* Boolean operator with single (recognized/unknown) constraint gets simplified */ if (ConstraintCount(node) <= 1) { Assert(node->childBooleanNodes == NIL); parent->validConstraints = list_concat(parent->validConstraints, node->validConstraints); parent->hasInvalidConstraints = parent->hasInvalidConstraints || node->hasInvalidConstraints; /* Remove current node from parent. Its constraint was assigned to the parent above */ parent->childBooleanNodes = list_delete_ptr(parent->childBooleanNodes, node); } } /* * ContainsFalseClause returns whether the flattened where clause list * contains false as a clause. */ bool ContainsFalseClause(List *whereClauseList) { bool containsFalseClause = false; ListCell *clauseCell = NULL; foreach(clauseCell, whereClauseList) { Node *clause = (Node *) lfirst(clauseCell); if (IsA(clause, Const)) { Const *constant = (Const *) clause; if (constant->consttype == BOOLOID && !DatumGetBool(constant->constvalue)) { containsFalseClause = true; break; } } } return containsFalseClause; } /* * PrunableExpressions builds a list of all prunable expressions in node, * storing them in context->pruningInstances. */ static void PrunableExpressions(PruningTreeNode *tree, ClauseWalkerContext *context) { /* * Build initial list of prunable expressions. As long as only, * implicitly or explicitly, ANDed expressions are found, this perform a * depth-first search. When an ORed expression is found, the current * PruningInstance is added to context->pruningInstances (once for each * ORed expression), then the tree-traversal is continued without * recursing. Once at the top-level again, we'll process all pending * expressions - that allows us to find all ANDed expressions, before * recursing into an ORed expression. */ PrunableExpressionsWalker(tree, context); /* * Process all pending instances. While processing, new ones might be * added to the list, so don't use foreach(). * * Check the places in PruningInstanceWalker that push onto * context->pendingInstances why construction of the PruningInstance might * be pending. * * We copy the partial PruningInstance, and continue adding information by * calling PrunableExpressionsWalker() on the copy, continuing at the * node stored in PendingPruningInstance->continueAt. */ while (context->pendingInstances != NIL) { PendingPruningInstance *instance = (PendingPruningInstance *) linitial(context->pendingInstances); PruningInstance *newPrune = CopyPartialPruningInstance(instance->instance); context->pendingInstances = list_delete_first(context->pendingInstances); context->currentPruningInstance = newPrune; PrunableExpressionsWalker(instance->continueAt, context); context->currentPruningInstance = NULL; } } /* * PrunableExpressionsWalker() is the main work horse for * PrunableExpressions(). */ static void PrunableExpressionsWalker(PruningTreeNode *node, ClauseWalkerContext *context) { ListCell *cell = NULL; if (node == NULL) { return; } if (node->boolop == OR_EXPR) { /* * "Queue" partial pruning instances. This is used to convert * expressions like (A AND (B OR C) AND D) into (A AND B AND D), * (A AND C AND D), with A, B, C, D being restrictions. When the * OR is encountered, a reference to the partially built * PruningInstance (containing A at this point), is added to * context->pendingInstances once for B and once for C. Once a * full tree-walk completed, PrunableExpressions() will complete * the pending instances, which'll now also know about restriction * D, by calling PrunableExpressionsWalker() once for B and once * for C. */ if (node->hasInvalidConstraints) { PruningTreeNode *child = CreatePruningNode(AND_EXPR); child->hasInvalidConstraints = true; AddNewConjuction(context, child); } foreach(cell, node->validConstraints) { Node *constraint = (Node *) lfirst(cell); PruningTreeNode *child = CreatePruningNode(AND_EXPR); child->validConstraints = list_make1(constraint); AddNewConjuction(context, child); } foreach(cell, node->childBooleanNodes) { PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); Assert(child->boolop == AND_EXPR); AddNewConjuction(context, child); } return; } Assert(node->boolop == AND_EXPR); foreach(cell, node->validConstraints) { Node *constraint = (Node *) lfirst(cell); if (IsA(constraint, OpExpr)) { OpExpr *opClause = (OpExpr *) constraint; PruningInstance *prune = context->currentPruningInstance; Var *varClause = NULL; Const *constantClause = NULL; if (!prune->addedToPruningInstances) { context->pruningInstances = lappend(context->pruningInstances, prune); prune->addedToPruningInstances = true; } if (VarConstOpExprClause(opClause, &varClause, &constantClause)) { if (equal(varClause, context->partitionColumn)) { /* * Found a restriction on the partition column itself. Update the * current constraint with the new information. */ AddPartitionKeyRestrictionToInstance(context, opClause, varClause, constantClause); } else if (varClause->varattno == RESERVED_HASHED_COLUMN_ID) { /* * Found restriction that directly specifies the boundaries of a * hashed column. */ AddHashRestrictionToInstance(context, opClause, varClause, constantClause); } else { /* We encounter here only valid constraints */ Assert(false); } } else { /* We encounter here only valid constraints */ Assert(false); } } else if (IsA(constraint, ScalarArrayOpExpr)) { ScalarArrayOpExpr *arrayOperatorExpression = (ScalarArrayOpExpr *) constraint; AddSAOPartitionKeyRestrictionToInstance(context, arrayOperatorExpression); } else { /* We encounter here only valid constraints */ Assert(false); } } if (node->hasInvalidConstraints) { PruningInstance *prune = context->currentPruningInstance; /* * Mark unknown expression as added, so we'll fail pruning if there's no ANDed * restrictions that we know how to deal with. */ if (!prune->addedToPruningInstances) { context->pruningInstances = lappend(context->pruningInstances, prune); prune->addedToPruningInstances = true; } } foreach(cell, node->childBooleanNodes) { PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); Assert(child->boolop == OR_EXPR); PrunableExpressionsWalker(child, context); } } /* * VarConstOpExprClause check whether an expression is a valid comparison of a Var to a Const. * Also obtaining the var with constant when valid. */ bool VarConstOpExprClause(OpExpr *opClause, Var **varClause, Const **constantClause) { Var *foundVarClause = NULL; Const *foundConstantClause = NULL; Node *leftOperand; Node *rightOperand; if (!BinaryOpExpression((Expr *) opClause, &leftOperand, &rightOperand)) { return false; } if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) { foundVarClause = (Var *) leftOperand; foundConstantClause = (Const *) rightOperand; } else if (IsA(leftOperand, Const) && IsA(rightOperand, Var)) { foundVarClause = (Var *) rightOperand; foundConstantClause = (Const *) leftOperand; } else { return false; } if (varClause) { *varClause = foundVarClause; } if (constantClause) { *constantClause = foundConstantClause; } return true; } /* * AddSAOPartitionKeyRestrictionToInstance adds partcol = arrayelem operator * restriction to the current pruning instance for each element of the array. These * restrictions are added to pruning instance to prune shards based on IN/=ANY * constraints. */ static void AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, ScalarArrayOpExpr *arrayOperatorExpression) { List *restrictions = NULL; bool validSAORestriction PG_USED_FOR_ASSERTS_ONLY = SAORestrictions(arrayOperatorExpression, context->partitionColumn, &restrictions); Assert(validSAORestriction); PruningTreeNode *node = CreatePruningNode(OR_EXPR); node->validConstraints = restrictions; AddNewConjuction(context, node); } /* * SAORestrictions checks whether an SAO constraint is valid. * Also obtains equality restrictions. */ static bool SAORestrictions(ScalarArrayOpExpr *arrayOperatorExpression, Var *partitionColumn, List **requestedRestrictions) { Node *leftOpExpression = linitial(arrayOperatorExpression->args); Node *strippedLeftOpExpression = strip_implicit_coercions(leftOpExpression); bool usingEqualityOperator = OperatorImplementsEquality( arrayOperatorExpression->opno); Expr *arrayArgument = (Expr *) lsecond(arrayOperatorExpression->args); /* checking for partcol = ANY(const, value, s); or partcol IN (const,b,c); */ if (usingEqualityOperator && strippedLeftOpExpression != NULL && equal(strippedLeftOpExpression, partitionColumn) && IsA(arrayArgument, Const)) { Const *arrayConst = (Const *) arrayArgument; int16 typlen = 0; bool typbyval = false; char typalign = '\0'; Datum arrayElement = 0; Datum inArray = arrayConst->constvalue; bool isNull = false; bool foundValid = false; /* check for the NULL right-hand expression*/ if (inArray == 0) { return false; } ArrayType *array = DatumGetArrayTypeP(arrayConst->constvalue); /* get the necessary information from array type to iterate over it */ Oid elementType = ARR_ELEMTYPE(array); get_typlenbyvalalign(elementType, &typlen, &typbyval, &typalign); /* Iterate over the righthand array of expression */ ArrayIterator arrayIterator = array_create_iterator(array, 0, NULL); while (array_iterate(arrayIterator, &arrayElement, &isNull)) { if (isNull) { /* * We can ignore IN (NULL) clauses because a value is never * equal to NULL. */ continue; } foundValid = true; if (requestedRestrictions) { Const *constElement = makeConst(elementType, -1, arrayConst->constcollid, typlen, arrayElement, isNull, typbyval); /* build partcol = arrayelem operator */ OpExpr *arrayEqualityOp = SAORestrictionArrayEqualityOp( arrayOperatorExpression, partitionColumn); arrayEqualityOp->args = list_make2(strippedLeftOpExpression, constElement); *requestedRestrictions = lappend(*requestedRestrictions, arrayEqualityOp); } else { break; } } return foundValid; } return false; } /* * AddNewConjuction adds the OpExpr to pending instance list of context * as conjunction as partial instance. */ static void AddNewConjuction(ClauseWalkerContext *context, PruningTreeNode *node) { PendingPruningInstance *instance = palloc0(sizeof(PendingPruningInstance)); instance->instance = context->currentPruningInstance; instance->continueAt = node; /* * Signal that this instance is not to be used for pruning on * its own. Once the pending instance is processed, it'll be * used. */ instance->instance->isPartial = true; context->pendingInstances = lappend(context->pendingInstances, instance); } /* * IsValidPartitionKeyRestriction check whether an operator clause is * a valid restriction for comparing to a partition column. */ static bool IsValidPartitionKeyRestriction(OpExpr *opClause) { ListCell *btreeInterpretationCell = NULL; bool matchedOp = false; List *btreeInterpretationList = get_op_btree_interpretation(opClause->opno); foreach(btreeInterpretationCell, btreeInterpretationList) { OpBtreeInterpretation *btreeInterpretation = (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); #if PG_VERSION_NUM >= PG_VERSION_18 if (btreeInterpretation->cmptype == ROWCOMPARE_NE) #else if (btreeInterpretation->strategy == ROWCOMPARE_NE) #endif { /* TODO: could add support for this, if we feel like it */ return false; } matchedOp = true; } return matchedOp; } /* * AddPartitionKeyRestrictionToInstance adds information about a PartitionKey * $op Const restriction to the current pruning instance. */ static void AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *partitionColumn, Const *constantClause) { PruningInstance *prune = context->currentPruningInstance; ListCell *btreeInterpretationCell = NULL; /* only have extra work to do if const isn't same type as partition column */ if (constantClause->consttype != partitionColumn->vartype) { /* we want our restriction value in terms of the type of the partition column */ constantClause = TransformPartitionRestrictionValue(partitionColumn, constantClause, true); if (constantClause == NULL) { /* couldn't coerce value, its invalid restriction */ return; } } if (constantClause->constisnull) { /* we cannot do pruning on NULL values */ return; } /* at this point, we'd better be able to pass binary Datums to comparison functions */ Assert(IsBinaryCoercible(constantClause->consttype, partitionColumn->vartype)); List *btreeInterpretationList = get_op_btree_interpretation(opClause->opno); foreach(btreeInterpretationCell, btreeInterpretationList) { OpBtreeInterpretation *btreeInterpretation = (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); #if PG_VERSION_NUM >= PG_VERSION_18 switch (btreeInterpretation->cmptype) #else switch (btreeInterpretation->strategy) #endif { case BTLessStrategyNumber: { if (!prune->lessConsts || PerformValueCompare((FunctionCallInfo) & context->compareValueFunctionCall, constantClause->constvalue, prune->lessConsts->constvalue) < 0) { prune->lessConsts = constantClause; } break; } case BTLessEqualStrategyNumber: { if (!prune->lessEqualConsts || PerformValueCompare((FunctionCallInfo) & context->compareValueFunctionCall, constantClause->constvalue, prune->lessEqualConsts->constvalue) < 0) { prune->lessEqualConsts = constantClause; } break; } case BTEqualStrategyNumber: { if (!prune->equalConsts) { prune->equalConsts = constantClause; } else if (PerformValueCompare((FunctionCallInfo) & context->compareValueFunctionCall, constantClause->constvalue, prune->equalConsts->constvalue) != 0) { /* key can't be equal to two values */ prune->evaluatesToFalse = true; } break; } case BTGreaterEqualStrategyNumber: { if (!prune->greaterEqualConsts || PerformValueCompare((FunctionCallInfo) & context->compareValueFunctionCall, constantClause->constvalue, prune->greaterEqualConsts->constvalue) > 0 ) { prune->greaterEqualConsts = constantClause; } break; } case BTGreaterStrategyNumber: { if (!prune->greaterConsts || PerformValueCompare((FunctionCallInfo) & context->compareValueFunctionCall, constantClause->constvalue, prune->greaterConsts->constvalue) > 0) { prune->greaterConsts = constantClause; } break; } default: { Assert(false); } } } prune->hasValidConstraint = true; } /* * TransformPartitionRestrictionValue works around how PostgreSQL sometimes * chooses to try to wrap our Var in a coercion rather than the Const. * To deal with this, we strip coercions from both and manually coerce * the Const into the type of our partition column. * It is conceivable that in some instances this may not be possible, * in those cases we will simply fail to prune partitions based on this clause. */ Const * TransformPartitionRestrictionValue(Var *partitionColumn, Const *restrictionValue, bool missingOk) { Node *transformedValue = coerce_to_target_type(NULL, (Node *) restrictionValue, restrictionValue->consttype, partitionColumn->vartype, partitionColumn->vartypmod, COERCION_ASSIGNMENT, COERCE_IMPLICIT_CAST, -1); /* if NULL, no implicit coercion is possible between the types */ if (transformedValue == NULL) { if (!missingOk) { ErrorTypesDontMatch(partitionColumn->vartype, partitionColumn->varcollid, restrictionValue->consttype, restrictionValue->constcollid); } return NULL; } /* if still not a constant, evaluate coercion */ if (!IsA(transformedValue, Const)) { transformedValue = (Node *) expression_planner((Expr *) transformedValue); } /* if still not a constant, no immutable coercion matched */ if (!IsA(transformedValue, Const)) { if (!missingOk) { ErrorTypesDontMatch(partitionColumn->vartype, partitionColumn->varcollid, restrictionValue->consttype, restrictionValue->constcollid); } return NULL; } return (Const *) transformedValue; } /* * ErrorTypesDontMatch throws an error explicitly printing the type names. */ static void ErrorTypesDontMatch(Oid firstType, Oid firstCollId, Oid secondType, Oid secondCollId) { Datum firstTypename = DirectFunctionCall1Coll(regtypeout, firstCollId, ObjectIdGetDatum(firstType)); Datum secondTypename = DirectFunctionCall1Coll(regtypeout, secondCollId, ObjectIdGetDatum(secondType)); ereport(ERROR, (errmsg("Cannot coerce %s to %s", DatumGetCString(secondTypename), DatumGetCString(firstTypename)))); } /* * IsValidHashRestriction checks whether an operator clause is a valid restriction for hashed column. */ static bool IsValidHashRestriction(OpExpr *opClause) { ListCell *btreeInterpretationCell = NULL; List *btreeInterpretationList = get_op_btree_interpretation(opClause->opno); foreach(btreeInterpretationCell, btreeInterpretationList) { OpBtreeInterpretation *btreeInterpretation = (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); #if PG_VERSION_NUM >= PG_VERSION_18 if (btreeInterpretation->cmptype == BTGreaterEqualStrategyNumber) #else if (btreeInterpretation->strategy == BTGreaterEqualStrategyNumber) #endif { return true; } } return false; } /* * AddHashRestrictionToInstance adds information about a * RESERVED_HASHED_COLUMN_ID = Const restriction to the current pruning * instance. */ static void AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *varClause, Const *constantClause) { /* be paranoid */ Assert(IsBinaryCoercible(constantClause->consttype, INT4OID)); Assert(IsValidHashRestriction(opClause)); /* * Ladidadida, dirty hackety hack. We only add such * constraints (in ShardIntervalOpExpressions()) to select a * shard based on its exact boundaries. For efficient binary * search it's better to simply use one representative value * to look up the shard. In practice, this is sufficient for * now. */ PruningInstance *prune = context->currentPruningInstance; Assert(!prune->hashedEqualConsts); prune->hashedEqualConsts = constantClause; prune->hasValidConstraint = true; } /* * CopyPartialPruningInstance copies a partial PruningInstance, so it can be * completed. */ static PruningInstance * CopyPartialPruningInstance(PruningInstance *sourceInstance) { PruningInstance *newInstance = palloc(sizeof(PruningInstance)); Assert(sourceInstance->isPartial); /* * To make the new PruningInstance useful for pruning, we have to reset it * being partial - if necessary it'll be marked so again by * PrunableExpressionsWalker(). */ *newInstance = *sourceInstance; newInstance->addedToPruningInstances = false; newInstance->isPartial = false; return newInstance; } /* * ShardArrayToList builds a list of out the array of ShardInterval*. */ static List * ShardArrayToList(ShardInterval **shardArray, int length) { List *shardIntervalList = NIL; for (int shardIndex = 0; shardIndex < length; shardIndex++) { ShardInterval *shardInterval = shardArray[shardIndex]; shardIntervalList = lappend(shardIntervalList, shardInterval); } return shardIntervalList; } /* * DeepCopyShardIntervalList copies originalShardIntervalList and the * contained ShardIntervals, into a new list. */ static List * DeepCopyShardIntervalList(List *originalShardIntervalList) { List *copiedShardIntervalList = NIL; ShardInterval *originalShardInterval = NULL; foreach_declared_ptr(originalShardInterval, originalShardIntervalList) { ShardInterval *copiedShardInterval = CopyShardInterval(originalShardInterval); copiedShardIntervalList = lappend(copiedShardIntervalList, copiedShardInterval); } return copiedShardIntervalList; } /* * PruneOne returns all shards in the table that match a single * PruningInstance. */ static List * PruneOne(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune) { ShardInterval *shardInterval = NULL; /* Well, if life always were this easy... */ if (prune->evaluatesToFalse) { return NIL; } /* * For an equal constraints, if there's no overlapping shards (always the * case for hash and range partitioning, sometimes for append), can * perform binary search for the right interval. That's usually the * fastest, so try that first. */ if (prune->equalConsts && !cacheEntry->hasOverlappingShardInterval) { shardInterval = FindShardInterval(prune->equalConsts->constvalue, cacheEntry); /* * If pruned down to nothing, we're done. Otherwise see if other * methods prune down further / to nothing. */ if (!shardInterval) { return NIL; } } /* * If the hash value we're looking for is known, we can search for the * interval directly. That's fast and should only ever be the case for a * hash-partitioned table. */ if (prune->hashedEqualConsts) { ShardInterval **sortedShardIntervalArray = cacheEntry->sortedShardIntervalArray; Assert(context->partitionMethod == DISTRIBUTE_BY_HASH); int shardIndex = FindShardIntervalIndex(prune->hashedEqualConsts->constvalue, cacheEntry); if (shardIndex == INVALID_SHARD_INDEX) { return NIL; } else if (shardInterval && sortedShardIntervalArray[shardIndex]->shardId != shardInterval->shardId) { /* * equalConst based pruning above yielded a different shard than * pruning based on pre-hashed equality. This is useful in case * of INSERT ... SELECT, where both can occur together (one via * join/colocation, the other via a plain equality restriction). */ return NIL; } else { return list_make1(sortedShardIntervalArray[shardIndex]); } } /* * If previous pruning method yielded a single shard, and the table is not * hash partitioned, attempt range based pruning to exclude it further. * * That's particularly important in particular for subquery pushdown, * where it's very common to have a user specified equality restriction, * and a range based restriction for shard boundaries, added by the * subquery machinery. */ if (shardInterval) { if (context->partitionMethod != DISTRIBUTE_BY_HASH && ExhaustivePruneOne(shardInterval, context, prune)) { return NIL; } else { /* no chance to prune further, return */ return list_make1(shardInterval); } } /* * Should never get here for hashing, we've filtered down to either zero * or one shard, and returned. */ Assert(context->partitionMethod != DISTRIBUTE_BY_HASH); /* * Next method: binary search with fuzzy boundaries. Can't trivially do so * if shards have overlapping boundaries. * * TODO: If we kept shard intervals separately sorted by both upper and * lower boundaries, this should be possible? */ if (!cacheEntry->hasOverlappingShardInterval && ( prune->greaterConsts || prune->greaterEqualConsts || prune->lessConsts || prune->lessEqualConsts)) { return PruneWithBoundaries(cacheEntry, context, prune); } /* * Brute force: Check each shard. */ return ExhaustivePrune(cacheEntry, context, prune); } /* * PerformCompare invokes comparator with prepared values, check for * unexpected NULL returns. */ static int PerformCompare(FunctionCallInfo compareFunctionCall) { Datum result = FunctionCallInvoke(compareFunctionCall); if (compareFunctionCall->isnull) { elog(ERROR, "function %u returned NULL", compareFunctionCall->flinfo->fn_oid); } return DatumGetInt32(result); } /* * PerformValueCompare invokes comparator with a/b, and checks for unexpected * NULL returns. */ static int PerformValueCompare(FunctionCallInfo compareFunctionCall, Datum a, Datum b) { fcSetArg(compareFunctionCall, 0, a); fcSetArg(compareFunctionCall, 1, b); return PerformCompare(compareFunctionCall); } /* * LowerShardBoundary returns the index of the first ShardInterval that's >= * (if includeMax) or > partitionColumnValue. */ static int LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, FunctionCallInfo compareFunction, bool includeMax) { int lowerBoundIndex = 0; int upperBoundIndex = shardCount; Assert(shardCount != 0); /* setup partitionColumnValue argument once */ fcSetArg(compareFunction, 0, partitionColumnValue); /* * Now we test partitionColumnValue used in where clause such as * partCol > partitionColumnValue (or partCol >= partitionColumnValue) * against four possibilities, these are: * 1) partitionColumnValue falls into a specific shard, such that: * partitionColumnValue >= shard[x].min, and * partitionColumnValue < shard[x].max (or partitionColumnValue <= shard[x].max). * 2) partitionColumnValue < shard[x].min for all the shards * 3) partitionColumnValue > shard[x].max for all the shards * 4) partitionColumnValue falls in between two shards, such that: * partitionColumnValue > shard[x].max and * partitionColumnValue < shard[x+1].min * * For 1), we find that shard in below loop using binary search and * return the index of it. For the others, see the end of this function. */ while (lowerBoundIndex < upperBoundIndex) { int middleIndex = lowerBoundIndex + ((upperBoundIndex - lowerBoundIndex) / 2); /* setup minValue as argument */ fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->minValue); /* execute cmp(partitionValue, lowerBound) */ int minValueComparison = PerformCompare(compareFunction); /* and evaluate results */ if (minValueComparison < 0) { /* value smaller than entire range */ upperBoundIndex = middleIndex; continue; } /* setup maxValue as argument */ fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->maxValue); /* execute cmp(partitionValue, upperBound) */ int maxValueComparison = PerformCompare(compareFunction); if ((maxValueComparison == 0 && !includeMax) || maxValueComparison > 0) { /* value bigger than entire range */ lowerBoundIndex = middleIndex + 1; continue; } /* partitionColumnValue falls into a specific shard, possibility 1) */ return middleIndex; } Assert(lowerBoundIndex == upperBoundIndex); /* * If we get here, none of the ShardIntervals exactly contain the value * (we'd have hit the return middleIndex; case otherwise). Figure out * whether there's possibly any interval containing a value that's bigger * than the partition key one. * * Also note that we initialized lowerBoundIndex with 0. Similarly, * we always set it to the index of the shard that we consider as our * lower boundary during binary search. */ if (lowerBoundIndex == shardCount) { /* * Since lowerBoundIndex is an inclusive index, being equal to shardCount * means all the shards have smaller values than partitionColumnValue, * which corresponds to possibility 3). * In that case, since we can't have a lower bound shard, we return * INVALID_SHARD_INDEX here. */ return INVALID_SHARD_INDEX; } /* * partitionColumnValue is either smaller than all the shards or falls in * between two shards, which corresponds to possibility 2) or 4). * Knowing that lowerBoundIndex is an inclusive index, we directly return * it as the index for the lower bound shard here. */ return lowerBoundIndex; } /* * UpperShardBoundary returns the index of the last ShardInterval that's <= * (if includeMin) or < partitionColumnValue. */ static int UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, FunctionCallInfo compareFunction, bool includeMin) { int lowerBoundIndex = 0; int upperBoundIndex = shardCount; Assert(shardCount != 0); /* setup partitionColumnValue argument once */ fcSetArg(compareFunction, 0, partitionColumnValue); /* * Now we test partitionColumnValue used in where clause such as * partCol < partitionColumnValue (or partCol <= partitionColumnValue) * against four possibilities, these are: * 1) partitionColumnValue falls into a specific shard, such that: * partitionColumnValue <= shard[x].max, and * partitionColumnValue > shard[x].min (or partitionColumnValue >= shard[x].min). * 2) partitionColumnValue > shard[x].max for all the shards * 3) partitionColumnValue < shard[x].min for all the shards * 4) partitionColumnValue falls in between two shards, such that: * partitionColumnValue > shard[x].max and * partitionColumnValue < shard[x+1].min * * For 1), we find that shard in below loop using binary search and * return the index of it. For the others, see the end of this function. */ while (lowerBoundIndex < upperBoundIndex) { int middleIndex = lowerBoundIndex + ((upperBoundIndex - lowerBoundIndex) / 2); /* setup minValue as argument */ fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->minValue); /* execute cmp(partitionValue, lowerBound) */ int minValueComparison = PerformCompare(compareFunction); /* and evaluate results */ if ((minValueComparison == 0 && !includeMin) || minValueComparison < 0) { /* value smaller than entire range */ upperBoundIndex = middleIndex; continue; } /* setup maxValue as argument */ fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->maxValue); /* execute cmp(partitionValue, upperBound) */ int maxValueComparison = PerformCompare(compareFunction); if (maxValueComparison > 0) { /* value bigger than entire range */ lowerBoundIndex = middleIndex + 1; continue; } /* partitionColumnValue falls into a specific shard, possibility 1) */ return middleIndex; } Assert(lowerBoundIndex == upperBoundIndex); /* * If we get here, none of the ShardIntervals exactly contain the value * (we'd have hit the return middleIndex; case otherwise). Figure out * whether there's possibly any interval containing a value that's smaller * than the partition key one. * * Also note that we initialized upperBoundIndex with shardCount. Similarly, * we always set it to the index of the next shard that we consider as our * upper boundary during binary search. */ if (upperBoundIndex == 0) { /* * Since upperBoundIndex is an exclusive index, being equal to 0 means * all the shards have greater values than partitionColumnValue, which * corresponds to possibility 3). * In that case, since we can't have an upper bound shard, we return * INVALID_SHARD_INDEX here. */ return INVALID_SHARD_INDEX; } /* * partitionColumnValue is either greater than all the shards or falls in * between two shards, which corresponds to possibility 2) or 4). * Knowing that upperBoundIndex is an exclusive index, we return the index * for the previous shard here. */ return upperBoundIndex - 1; } /* * PruneWithBoundaries searches for shards that match inequality constraints, * using binary search on both the upper and lower boundary, and returns a * list of surviving shards. */ static List * PruneWithBoundaries(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune) { List *remainingShardList = NIL; int shardCount = cacheEntry->shardIntervalArrayLength; ShardInterval **sortedShardIntervalArray = cacheEntry->sortedShardIntervalArray; bool hasLowerBound = false; bool hasUpperBound = false; Datum lowerBound = 0; Datum upperBound = 0; bool lowerBoundInclusive = false; bool upperBoundInclusive = false; int lowerBoundIdx = -1; int upperBoundIdx = -1; FunctionCallInfo compareFunctionCall = (FunctionCallInfo) & context->compareIntervalFunctionCall; if (prune->greaterEqualConsts) { lowerBound = prune->greaterEqualConsts->constvalue; lowerBoundInclusive = true; hasLowerBound = true; } if (prune->greaterConsts) { /* * Use the more restrictive one, if both greater and greaterEqual * constraints are specified. */ if (!hasLowerBound || PerformValueCompare(compareFunctionCall, prune->greaterConsts->constvalue, lowerBound) >= 0) { lowerBound = prune->greaterConsts->constvalue; lowerBoundInclusive = false; hasLowerBound = true; } } if (prune->lessEqualConsts) { upperBound = prune->lessEqualConsts->constvalue; upperBoundInclusive = true; hasUpperBound = true; } if (prune->lessConsts) { /* * Use the more restrictive one, if both less and lessEqual * constraints are specified. */ if (!hasUpperBound || PerformValueCompare(compareFunctionCall, prune->lessConsts->constvalue, upperBound) <= 0) { upperBound = prune->lessConsts->constvalue; upperBoundInclusive = false; hasUpperBound = true; } } Assert(hasLowerBound || hasUpperBound); /* find lower bound */ if (hasLowerBound) { lowerBoundIdx = LowerShardBoundary(lowerBound, sortedShardIntervalArray, shardCount, compareFunctionCall, lowerBoundInclusive); } else { lowerBoundIdx = 0; } /* find upper bound */ if (hasUpperBound) { upperBoundIdx = UpperShardBoundary(upperBound, sortedShardIntervalArray, shardCount, compareFunctionCall, upperBoundInclusive); } else { upperBoundIdx = shardCount - 1; } if (lowerBoundIdx == INVALID_SHARD_INDEX) { return NIL; } else if (upperBoundIdx == INVALID_SHARD_INDEX) { return NIL; } /* * Build list of all shards that are in the range of shards (possibly 0). */ for (int curIdx = lowerBoundIdx; curIdx <= upperBoundIdx; curIdx++) { remainingShardList = lappend(remainingShardList, sortedShardIntervalArray[curIdx]); } return remainingShardList; } /* * ExhaustivePrune returns a list of shards matching PruningInstances * constraints, by simply checking them for each individual shard. */ static List * ExhaustivePrune(CitusTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune) { List *remainingShardList = NIL; int shardCount = cacheEntry->shardIntervalArrayLength; ShardInterval **sortedShardIntervalArray = cacheEntry->sortedShardIntervalArray; for (int curIdx = 0; curIdx < shardCount; curIdx++) { ShardInterval *curInterval = sortedShardIntervalArray[curIdx]; if (!ExhaustivePruneOne(curInterval, context, prune)) { remainingShardList = lappend(remainingShardList, curInterval); } } return remainingShardList; } /* * ExhaustivePruneOne returns whether curInterval is pruned away. */ static bool ExhaustivePruneOne(ShardInterval *curInterval, ClauseWalkerContext *context, PruningInstance *prune) { FunctionCallInfo compareFunctionCall = (FunctionCallInfo) & context->compareIntervalFunctionCall; Datum compareWith = 0; /* NULL boundaries can't be compared to */ if (!curInterval->minValueExists || !curInterval->maxValueExists) { return false; } if (prune->equalConsts) { compareWith = prune->equalConsts->constvalue; if (PerformValueCompare(compareFunctionCall, compareWith, curInterval->minValue) < 0) { return true; } if (PerformValueCompare(compareFunctionCall, compareWith, curInterval->maxValue) > 0) { return true; } } if (prune->greaterEqualConsts) { compareWith = prune->greaterEqualConsts->constvalue; if (PerformValueCompare(compareFunctionCall, curInterval->maxValue, compareWith) < 0) { return true; } } if (prune->greaterConsts) { compareWith = prune->greaterConsts->constvalue; if (PerformValueCompare(compareFunctionCall, curInterval->maxValue, compareWith) <= 0) { return true; } } if (prune->lessEqualConsts) { compareWith = prune->lessEqualConsts->constvalue; if (PerformValueCompare(compareFunctionCall, curInterval->minValue, compareWith) > 0) { return true; } } if (prune->lessConsts) { compareWith = prune->lessConsts->constvalue; if (PerformValueCompare(compareFunctionCall, curInterval->minValue, compareWith) >= 0) { return true; } } return false; } /* * Helper for creating a node for pruning tree */ static PruningTreeNode * CreatePruningNode(BoolExprType boolop) { PruningTreeNode *node = palloc0(sizeof(PruningTreeNode)); node->boolop = boolop; node->childBooleanNodes = NIL; node->validConstraints = NIL; node->hasInvalidConstraints = false; return node; } /* * SAORestrictionArrayEqualityOp creates an equality operator * for a single element of a scalar array constraint. */ static OpExpr * SAORestrictionArrayEqualityOp(ScalarArrayOpExpr *arrayOperatorExpression, Var *partitionColumn) { OpExpr *arrayEqualityOp = makeNode(OpExpr); arrayEqualityOp->opno = arrayOperatorExpression->opno; arrayEqualityOp->opfuncid = arrayOperatorExpression->opfuncid; arrayEqualityOp->inputcollid = arrayOperatorExpression->inputcollid; arrayEqualityOp->opresulttype = get_func_rettype( arrayOperatorExpression->opfuncid); arrayEqualityOp->opcollid = partitionColumn->varcollid; arrayEqualityOp->location = -1; return arrayEqualityOp; } /* * DebugLogNode is a helper for logging expression nodes. */ static void DebugLogNode(char *fmt, Node *node, List *deparseCtx) { if (node != NULL) { char *deparsed = deparse_expression(node, deparseCtx, false, false); ereport(DEBUG3, (errmsg(fmt, deparsed))); } } /* * DebugLogPruningInstance is a helper for logging purning constraints. */ static void DebugLogPruningInstance(PruningInstance *pruning, List *deparseCtx) { DebugLogNode("constraint value: %s", (Node *) pruning->equalConsts, deparseCtx); DebugLogNode("constraint (lt) value: %s", \ (Node *) pruning->lessConsts, deparseCtx); DebugLogNode("constraint (lteq) value: %s", \ (Node *) pruning->lessEqualConsts, deparseCtx); DebugLogNode("constraint (gt) value: %s", \ (Node *) pruning->greaterConsts, deparseCtx); DebugLogNode("constraint (gteq) value: %s", (Node *) pruning->greaterEqualConsts, deparseCtx); } /* * ConstraintCount returns how many arguments this node is taking. */ static int ConstraintCount(PruningTreeNode *node) { return list_length(node->childBooleanNodes) + list_length(node->validConstraints) + (node->hasInvalidConstraints ? 1 : 0); } ================================================ FILE: src/backend/distributed/planner/tdigest_extension.c ================================================ /*------------------------------------------------------------------------- * * tdigest_extension.c * Helper functions to get access to tdigest specific data. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/htup_details.h" #include "catalog/pg_extension.h" #include "catalog/pg_type.h" #include "parser/parse_func.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "distributed/metadata_cache.h" #include "distributed/tdigest_extension.h" #include "distributed/version_compat.h" static Oid LookupTDigestFunction(const char *functionName, int argcount, Oid *argtypes); /* * TDigestExtensionSchema finds the schema the tdigest extension is installed in. The * function will return InvalidOid if the extension is not installed. */ Oid TDigestExtensionSchema() { ScanKeyData entry[1]; Form_pg_extension extensionForm = NULL; Oid tdigestExtensionSchema = InvalidOid; Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum("tdigest")); SysScanDesc scandesc = systable_beginscan(relation, ExtensionNameIndexId, true, NULL, 1, entry); HeapTuple extensionTuple = systable_getnext(scandesc); /* * We assume that there can be at most one matching tuple, if no tuple found the * extension is not installed. The value of InvalidOid will not be changed. */ if (HeapTupleIsValid(extensionTuple)) { extensionForm = (Form_pg_extension) GETSTRUCT(extensionTuple); tdigestExtensionSchema = extensionForm->extnamespace; Assert(OidIsValid(tdigestExtensionSchema)); } systable_endscan(scandesc); table_close(relation, AccessShareLock); return tdigestExtensionSchema; } /* * TDigestExtensionTypeOid performs a lookup for the Oid of the type representing the * tdigest as installed by the tdigest extension returns InvalidOid if the type cannot be * found. */ Oid TDigestExtensionTypeOid() { Oid tdigestSchemaOid = TDigestExtensionSchema(); if (!OidIsValid(tdigestSchemaOid)) { return InvalidOid; } char *namespaceName = get_namespace_name(tdigestSchemaOid); return LookupTypeOid(namespaceName, "tdigest"); } /* * LookupTDigestFunction is a helper function specifically to lookup functions in the * namespace/schema where the tdigest extension is installed. This makes the lookup of * following aggregate functions easier and less repetitive. */ static Oid LookupTDigestFunction(const char *functionName, int argcount, Oid *argtypes) { Oid tdigestSchemaOid = TDigestExtensionSchema(); if (!OidIsValid(tdigestSchemaOid)) { return InvalidOid; } char *namespaceName = get_namespace_name(tdigestSchemaOid); return LookupFuncName( list_make2(makeString(namespaceName), makeString(pstrdup(functionName))), argcount, argtypes, true); } /* * TDigestExtensionAggTDigest1 performs a lookup for the Oid of the tdigest aggregate; * tdigest(tdigest) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigest1() { return LookupTDigestFunction("tdigest", 1, (Oid[]) { TDigestExtensionTypeOid() }); } /* * TDigestExtensionAggTDigest2 performs a lookup for the Oid of the tdigest aggregate; * tdigest(value double precision, compression int) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigest2() { return LookupTDigestFunction("tdigest", 2, (Oid[]) { FLOAT8OID, INT4OID }); } /* * TDigestExtensionAggTDigestPercentile2 performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile(tdigest, double precision) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentile2() { return LookupTDigestFunction("tdigest_percentile", 2, (Oid[]) { TDigestExtensionTypeOid(), FLOAT8OID }); } /* * TDigestExtensionAggTDigestPercentile2a performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile(tdigest, double precision[]) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentile2a(void) { return LookupTDigestFunction("tdigest_percentile", 2, (Oid[]) { TDigestExtensionTypeOid(), FLOAT8ARRAYOID }); } /* * TDigestExtensionAggTDigestPercentile3 performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile(double precision, int, double precision) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentile3() { return LookupTDigestFunction("tdigest_percentile", 3, (Oid[]) { FLOAT8OID, INT4OID, FLOAT8OID }); } /* * TDigestExtensionAggTDigestPercentile3a performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile(double precision, int, double precision[]) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentile3a(void) { return LookupTDigestFunction("tdigest_percentile", 3, (Oid[]) { FLOAT8OID, INT4OID, FLOAT8ARRAYOID }); } /* * TDigestExtensionAggTDigestPercentileOf2 performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile_of(tdigest, double precision) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentileOf2() { return LookupTDigestFunction("tdigest_percentile_of", 2, (Oid[]) { TDigestExtensionTypeOid(), FLOAT8OID }); } /* * TDigestExtensionAggTDigestPercentileOf2a performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile_of(tdigest, double precision[]) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentileOf2a(void) { return LookupTDigestFunction("tdigest_percentile_of", 2, (Oid[]) { TDigestExtensionTypeOid(), FLOAT8ARRAYOID }); } /* * TDigestExtensionAggTDigestPercentileOf3 performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile_of(double precision, int, double precision) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentileOf3() { return LookupTDigestFunction("tdigest_percentile_of", 3, (Oid[]) { FLOAT8OID, INT4OID, FLOAT8OID }); } /* * TDigestExtensionAggTDigestPercentileOf3a performs a lookup for the Oid of the tdigest * aggregate; * tdigest_percentile_of(double precision, int, double precision[]) * * If the aggregate is not found InvalidOid is returned. */ Oid TDigestExtensionAggTDigestPercentileOf3a(void) { return LookupTDigestFunction("tdigest_percentile_of", 3, (Oid[]) { FLOAT8OID, INT4OID, FLOAT8ARRAYOID }); } ================================================ FILE: src/backend/distributed/progress/multi_progress.c ================================================ /*------------------------------------------------------------------------- * * multi_progress.c * Routines for tracking long-running jobs and seeing their progress. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "storage/dsm.h" #include "utils/builtins.h" #include "distributed/function_utils.h" #include "distributed/listutils.h" #include "distributed/multi_progress.h" #include "distributed/version_compat.h" /* dynamic shared memory handle of the current progress */ static uint64 currentProgressDSMHandle = DSM_HANDLE_INVALID; static ProgressMonitorData * MonitorDataFromDSMHandle(dsm_handle dsmHandle, dsm_segment **attachedSegment); /* * CreateProgressMonitor is used to create a place to store progress * information related to long running processes. The function creates a * dynamic shared memory segment consisting of a header regarding to the * process and an array of "steps" that the long running "operations" consists * of. After initializing the data in the array of steps, the shared memory * segment can be shared with other processes using RegisterProgressMonitor, by * giving it the value that's written to the dsmHandle argument. */ ProgressMonitorData * CreateProgressMonitor(int stepCount, Size stepSize, dsm_handle *dsmHandle) { if (stepSize <= 0 || stepCount <= 0) { ereport(ERROR, (errmsg("number of steps and size of each step should be " "positive values"))); } Size monitorSize = sizeof(ProgressMonitorData) + stepSize * stepCount; dsm_segment *dsmSegment = dsm_create(monitorSize, DSM_CREATE_NULL_IF_MAXSEGMENTS); if (dsmSegment == NULL) { ereport(WARNING, (errmsg("could not create a dynamic shared memory segment to " "keep track of progress of the current command"))); return NULL; } *dsmHandle = dsm_segment_handle(dsmSegment); ProgressMonitorData *monitor = MonitorDataFromDSMHandle(*dsmHandle, &dsmSegment); monitor->stepCount = stepCount; monitor->processId = MyProcPid; return monitor; } /* * RegisterProgressMonitor shares dsmHandle with other postgres process by * storing it in pg_stat_get_progress_info output, to be parsed by a * progress retrieval command later on. This behavior may cause unrelated (but * hopefully harmless) rows in pg_stat_progress_vacuum output. The caller of * this function should provide a magic number, a unique 64 bit unsigned * integer, to distinguish different types of commands. * * IMPORTANT: After registering the progress monitor, all modification to the * data should be done using concurrency safe operations (i.e. locks and * atomics) */ void RegisterProgressMonitor(uint64 progressTypeMagicNumber, Oid relationId, dsm_handle dsmHandle) { pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, relationId); pgstat_progress_update_param(1, dsmHandle); pgstat_progress_update_param(0, progressTypeMagicNumber); currentProgressDSMHandle = dsmHandle; } /* * GetCurrentProgressMonitor function returns the header and steps array related to the * current progress. A progress monitor should be created by calling * CreateProgressMonitor, before calling this function. */ ProgressMonitorData * GetCurrentProgressMonitor(void) { dsm_segment *dsmSegment = NULL; ProgressMonitorData *monitor = MonitorDataFromDSMHandle(currentProgressDSMHandle, &dsmSegment); return monitor; } /* * FinalizeCurrentProgressMonitor releases the dynamic memory segment of the current * progress monitoring data structure and removes the process from * pg_stat_get_progress_info() output. If there's no such dynamic memory * segment this is a no-op. */ void FinalizeCurrentProgressMonitor(void) { if (currentProgressDSMHandle == DSM_HANDLE_INVALID) { return; } dsm_segment *dsmSegment = dsm_find_mapping(currentProgressDSMHandle); if (dsmSegment != NULL) { dsm_detach(dsmSegment); } pgstat_progress_end_command(); currentProgressDSMHandle = DSM_HANDLE_INVALID; } /* * HasProgressMonitor returns true if there is a current progress monitor, * by checking the variable currentProgressDSMHandle. Returns false otherwise. */ bool HasProgressMonitor(void) { return currentProgressDSMHandle != DSM_HANDLE_INVALID; } /* * ProgressMonitorList returns the addresses of monitors of ongoing commands, associated * with the given identifier magic number. The function takes a pass in * pg_stat_get_progress_info output, filters the rows according to the given magic number, * and returns the list of addresses of dynamic shared memory segments. Notice that the * caller detach from the attached segments with a call to DetachFromDSMSegments function. */ List * ProgressMonitorList(uint64 commandTypeMagicNumber, List **attachedDSMSegments) { /* * The expected magic number should reside in the first progress field and the * actual segment handle in the second but the slot ordering is 1-indexed in the * tuple table slot and there are 3 other fields before the progress fields in the * pg_stat_get_progress_info output. */ const int magicNumberIndex = 0 + 1 + 3; const int dsmHandleIndex = 1 + 1 + 3; /* * Currently, Postgres' progress logging mechanism supports only the VACUUM * operations. Therefore, we identify ourselves as a VACUUM command but only fill * a couple of the available fields. Therefore the commands that use Citus' progress * monitoring API will appear in pg_stat_progress_vacuum output. */ text *commandTypeText = cstring_to_text("VACUUM"); Datum commandTypeDatum = PointerGetDatum(commandTypeText); List *monitorList = NIL; Oid getProgressInfoFunctionOid = FunctionOid("pg_catalog", "pg_stat_get_progress_info", 1); ReturnSetInfo *progressResultSet = FunctionCallGetTupleStore1( pg_stat_get_progress_info, getProgressInfoFunctionOid, commandTypeDatum); TupleTableSlot *tupleTableSlot = MakeSingleTupleTableSlot( progressResultSet->setDesc, &TTSOpsMinimalTuple); /* iterate over tuples in tuple store, and send them to destination */ for (;;) { bool isNull = false; bool nextTuple = tuplestore_gettupleslot(progressResultSet->setResult, true, false, tupleTableSlot); if (!nextTuple) { break; } Datum magicNumberDatum = slot_getattr(tupleTableSlot, magicNumberIndex, &isNull); uint64 magicNumber = DatumGetUInt64(magicNumberDatum); if (!isNull && magicNumber == commandTypeMagicNumber) { Datum dsmHandleDatum = slot_getattr(tupleTableSlot, dsmHandleIndex, &isNull); dsm_handle dsmHandle = DatumGetUInt64(dsmHandleDatum); dsm_segment *attachedSegment = NULL; ProgressMonitorData *monitor = MonitorDataFromDSMHandle(dsmHandle, &attachedSegment); if (monitor != NULL) { *attachedDSMSegments = lappend(*attachedDSMSegments, attachedSegment); monitorList = lappend(monitorList, monitor); } } ExecClearTuple(tupleTableSlot); } ExecDropSingleTupleTableSlot(tupleTableSlot); return monitorList; } /* * MonitorDataFromDSMHandle returns the progress monitoring data structure at the * given segment */ ProgressMonitorData * MonitorDataFromDSMHandle(dsm_handle dsmHandle, dsm_segment **attachedSegment) { dsm_segment *dsmSegment = dsm_find_mapping(dsmHandle); if (dsmSegment == NULL) { dsmSegment = dsm_attach(dsmHandle); } if (dsmSegment == NULL) { return NULL; } ProgressMonitorData *monitor = (ProgressMonitorData *) dsm_segment_address( dsmSegment); *attachedSegment = dsmSegment; return monitor; } /* * ProgressMonitorSteps returns a pointer to the array of steps that are stored * in a progress monitor. This is simply the data right after the header, so * this function is trivial. The main purpose of this function is to make the * intent clear to readers of the code. * * NOTE: The pointer this function returns is explicitly not stored in the * header, because the header is shared between processes. The absolute pointer * to the steps can have a different value between processes though, because * the same piece of shared memory often has a different address in different * processes. So we calculate this pointer over and over to make sure we use * the right value for each process. */ void * ProgressMonitorSteps(ProgressMonitorData *monitor) { return monitor + 1; } /* * DetachFromDSMSegments ensures that the process is detached from all of the segments in * the given list. */ void DetachFromDSMSegments(List *dsmSegmentList) { dsm_segment *dsmSegment = NULL; foreach_declared_ptr(dsmSegment, dsmSegmentList) { dsm_detach(dsmSegment); } } ================================================ FILE: src/backend/distributed/relay/relay_event_utility.c ================================================ /*------------------------------------------------------------------------- * * relay_event_utility.c * * Routines for handling DDL statements that relate to relay files. These * routines extend relation, index and constraint names in utility commands. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "c.h" #include "access/genam.h" #include "access/hash.h" #include "access/heapam.h" #include "access/htup.h" #include "access/htup_details.h" #include "access/skey.h" #include "access/stratnum.h" #include "catalog/indexing.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "nodes/value.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "utils/relcache.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/relay_utility.h" #include "distributed/version_compat.h" /* Local functions forward declarations */ static void RelayEventExtendConstraintAndIndexNames(AlterTableStmt *alterTableStmt, Constraint *constraint, uint64 shardId); static bool UpdateWholeRowColumnReferencesWalker(Node *node, uint64 *shardId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(shard_name); /* * RelayEventExtendNames extends relation names in the given parse tree for * certain utility commands. The function more specifically extends table and * index names in the parse tree by appending the given shardId; thereby * avoiding name collisions in the database among sharded tables. This function * has the side effect of extending relation names in the parse tree. */ void RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId) { /* we don't extend names in extension or schema commands */ NodeTag nodeType = nodeTag(parseTree); if (nodeType == T_CreateExtensionStmt || nodeType == T_CreateSchemaStmt || nodeType == T_CreateSeqStmt || nodeType == T_AlterSeqStmt || nodeType == T_CreateForeignServerStmt) { return; } switch (nodeType) { case T_AlterObjectSchemaStmt: { AlterObjectSchemaStmt *alterObjectSchemaStmt = (AlterObjectSchemaStmt *) parseTree; ObjectType objectType = alterObjectSchemaStmt->objectType; if (objectType == OBJECT_STATISTIC_EXT) { RangeVar *stat = makeRangeVarFromNameList( (List *) alterObjectSchemaStmt->object); /* append shard id */ AppendShardIdToName(&stat->relname, shardId); alterObjectSchemaStmt->object = (Node *) MakeNameListFromRangeVar(stat); } else { char **relationName = &(alterObjectSchemaStmt->relation->relname); char **relationSchemaName = &(alterObjectSchemaStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); /* append shardId to base relation name */ AppendShardIdToName(relationName, shardId); } break; } case T_AlterStatsStmt: { AlterStatsStmt *alterStatsStmt = (AlterStatsStmt *) parseTree; RangeVar *stat = makeRangeVarFromNameList(alterStatsStmt->defnames); AppendShardIdToName(&stat->relname, shardId); alterStatsStmt->defnames = MakeNameListFromRangeVar(stat); break; } case T_AlterTableStmt: { /* * We append shardId to the very end of table and index, constraint * and trigger names to avoid name collisions. */ AlterTableStmt *alterTableStmt = (AlterTableStmt *) parseTree; Oid relationId = InvalidOid; char **relationName = &(alterTableStmt->relation->relname); char **relationSchemaName = &(alterTableStmt->relation->schemaname); List *commandList = alterTableStmt->cmds; /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); /* append shardId to base relation name */ AppendShardIdToName(relationName, shardId); AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { if (command->subtype == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; RelayEventExtendConstraintAndIndexNames(alterTableStmt, constraint, shardId); } else if (command->subtype == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnDefinition->constraints) { RelayEventExtendConstraintAndIndexNames(alterTableStmt, constraint, shardId); } } else if (command->subtype == AT_DropConstraint || command->subtype == AT_ValidateConstraint) { char **constraintName = &(command->name); const bool constraintMissingOk = true; if (!OidIsValid(relationId)) { const bool rvMissingOk = false; relationId = RangeVarGetRelid(alterTableStmt->relation, AccessShareLock, rvMissingOk); } Oid constraintOid = get_relation_constraint_oid(relationId, command->name, constraintMissingOk); if (!OidIsValid(constraintOid)) { AppendShardIdToName(constraintName, shardId); } } else if (command->subtype == AT_ClusterOn) { char **indexName = &(command->name); AppendShardIdToName(indexName, shardId); } else if (command->subtype == AT_ReplicaIdentity) { ReplicaIdentityStmt *replicaIdentity = (ReplicaIdentityStmt *) command->def; if (replicaIdentity->identity_type == REPLICA_IDENTITY_INDEX) { char **indexName = &(replicaIdentity->name); AppendShardIdToName(indexName, shardId); } } else if (command->subtype == AT_EnableTrig || command->subtype == AT_DisableTrig || command->subtype == AT_EnableAlwaysTrig || command->subtype == AT_EnableReplicaTrig) { char **triggerName = &(command->name); AppendShardIdToName(triggerName, shardId); } } break; } case T_AlterOwnerStmt: { AlterOwnerStmt *alterOwnerStmt = castNode(AlterOwnerStmt, parseTree); /* we currently extend names in alter owner statements only for statistics */ Assert(alterOwnerStmt->objectType == OBJECT_STATISTIC_EXT); RangeVar *stat = makeRangeVarFromNameList((List *) alterOwnerStmt->object); AppendShardIdToName(&stat->relname, shardId); alterOwnerStmt->object = (Node *) MakeNameListFromRangeVar(stat); break; } case T_ClusterStmt: { ClusterStmt *clusterStmt = (ClusterStmt *) parseTree; /* we do not support clustering the entire database */ if (clusterStmt->relation == NULL) { ereport(ERROR, (errmsg("cannot extend name for multi-relation cluster"))); } char **relationName = &(clusterStmt->relation->relname); char **relationSchemaName = &(clusterStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); if (clusterStmt->indexname != NULL) { char **indexName = &(clusterStmt->indexname); AppendShardIdToName(indexName, shardId); } break; } case T_CreateForeignTableStmt: case T_CreateStmt: { CreateStmt *createStmt = (CreateStmt *) parseTree; char **relationName = &(createStmt->relation->relname); char **relationSchemaName = &(createStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); break; } case T_CreateTrigStmt: { CreateTrigStmt *createTriggerStmt = (CreateTrigStmt *) parseTree; CreateTriggerEventExtendNames(createTriggerStmt, schemaName, shardId); break; } case T_AlterObjectDependsStmt: { AlterObjectDependsStmt *alterTriggerDependsStmt = (AlterObjectDependsStmt *) parseTree; ObjectType objectType = alterTriggerDependsStmt->objectType; if (objectType == OBJECT_TRIGGER) { AlterTriggerDependsEventExtendNames(alterTriggerDependsStmt, schemaName, shardId); } else { ereport(WARNING, (errmsg("unsafe object type in alter object " "depends statement"), errdetail("Object type: %u", (uint32) objectType))); } break; } case T_DropStmt: { DropStmt *dropStmt = (DropStmt *) parseTree; ObjectType objectType = dropStmt->removeType; if (objectType == OBJECT_TABLE || objectType == OBJECT_INDEX || objectType == OBJECT_FOREIGN_TABLE || objectType == OBJECT_FOREIGN_SERVER) { String *relationSchemaNameValue = NULL; String *relationNameValue = NULL; uint32 dropCount = list_length(dropStmt->objects); if (dropCount > 1) { ereport(ERROR, (errmsg("cannot extend name for multiple drop objects"))); } /* * We now need to extend a single relation or index name. To be * able to do this extension, we need to extract the names' * addresses from the value objects they are stored in. Other- * wise, the repalloc called in AppendShardIdToName() will not * have the correct memory address for the name. */ List *relationNameList = (List *) linitial(dropStmt->objects); int relationNameListLength = list_length(relationNameList); switch (relationNameListLength) { case 1: { relationNameValue = linitial(relationNameList); break; } case 2: { relationSchemaNameValue = linitial(relationNameList); relationNameValue = lsecond(relationNameList); break; } case 3: { relationSchemaNameValue = lsecond(relationNameList); relationNameValue = lthird(relationNameList); break; } default: { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("improper relation name: \"%s\"", NameListToString(relationNameList)))); break; } } /* prefix with schema name if it is not added already */ if (relationSchemaNameValue == NULL) { String *schemaNameValue = makeString(pstrdup(schemaName)); relationNameList = lcons(schemaNameValue, relationNameList); } char **relationName = &(strVal(relationNameValue)); AppendShardIdToName(relationName, shardId); } else if (objectType == OBJECT_POLICY) { DropPolicyEventExtendNames(dropStmt, schemaName, shardId); } else if (objectType == OBJECT_TRIGGER) { DropTriggerEventExtendNames(dropStmt, schemaName, shardId); } else if (objectType == OBJECT_STATISTIC_EXT) { List *shardStatisticsList = NIL; List *objectNameList = NULL; foreach_declared_ptr(objectNameList, dropStmt->objects) { RangeVar *stat = makeRangeVarFromNameList(objectNameList); SetSchemaNameIfNotExist(&stat->schemaname, schemaName); AppendShardIdToName(&stat->relname, shardId); shardStatisticsList = lappend(shardStatisticsList, MakeNameListFromRangeVar(stat)); } dropStmt->objects = shardStatisticsList; } else { ereport(WARNING, (errmsg("unsafe object type in drop statement"), errdetail("Object type: %u", (uint32) objectType))); } break; } case T_GrantStmt: { GrantStmt *grantStmt = (GrantStmt *) parseTree; if (grantStmt->targtype == ACL_TARGET_OBJECT && grantStmt->objtype == OBJECT_TABLE) { RangeVar *relation = NULL; foreach_declared_ptr(relation, grantStmt->objects) { char **relationName = &(relation->relname); char **relationSchemaName = &(relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); } } break; } case T_CreatePolicyStmt: { CreatePolicyEventExtendNames((CreatePolicyStmt *) parseTree, schemaName, shardId); break; } case T_AlterPolicyStmt: { AlterPolicyEventExtendNames((AlterPolicyStmt *) parseTree, schemaName, shardId); break; } case T_IndexStmt: { IndexStmt *indexStmt = (IndexStmt *) parseTree; char **relationName = &(indexStmt->relation->relname); char **indexName = &(indexStmt->idxname); char **relationSchemaName = &(indexStmt->relation->schemaname); /* * Concurrent index statements cannot run within a transaction block. * Therefore, we do not support them. */ if (indexStmt->concurrent) { ereport(ERROR, (errmsg("cannot extend name for concurrent index"))); } /* * In the regular DDL execution code path (for non-sharded tables), * if the index statement results from a table creation command, the * indexName may be null. For sharded tables however, we intercept * that code path and explicitly set the index name. Therefore, the * index name in here cannot be null. */ if ((*indexName) == NULL) { ereport(ERROR, (errmsg("cannot extend name for null index name"))); } /* extend ColumnRef nodes in the IndexStmt with the shardId */ UpdateWholeRowColumnReferencesWalker((Node *) indexStmt->indexParams, &shardId); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, schemaName); AppendShardIdToName(relationName, shardId); AppendShardIdToName(indexName, shardId); break; } case T_ReindexStmt: { ReindexStmt *reindexStmt = (ReindexStmt *) parseTree; ReindexObjectType objectType = reindexStmt->kind; if (objectType == REINDEX_OBJECT_TABLE || objectType == REINDEX_OBJECT_INDEX) { char **objectName = &(reindexStmt->relation->relname); char **objectSchemaName = &(reindexStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(objectSchemaName, schemaName); AppendShardIdToName(objectName, shardId); } break; } case T_RenameStmt: { RenameStmt *renameStmt = (RenameStmt *) parseTree; ObjectType objectType = renameStmt->renameType; if (objectType == OBJECT_TABLE || objectType == OBJECT_INDEX || objectType == OBJECT_FOREIGN_TABLE) { char **oldRelationName = &(renameStmt->relation->relname); char **newRelationName = &(renameStmt->newname); char **objectSchemaName = &(renameStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(objectSchemaName, schemaName); AppendShardIdToName(oldRelationName, shardId); AppendShardIdToName(newRelationName, shardId); } else if (objectType == OBJECT_COLUMN) { char **relationName = &(renameStmt->relation->relname); char **objectSchemaName = &(renameStmt->relation->schemaname); /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(objectSchemaName, schemaName); AppendShardIdToName(relationName, shardId); } else if (objectType == OBJECT_TRIGGER) { AlterTriggerRenameEventExtendNames(renameStmt, schemaName, shardId); } else if (objectType == OBJECT_POLICY) { RenamePolicyEventExtendNames(renameStmt, schemaName, shardId); } else if (objectType == OBJECT_STATISTIC_EXT) { RangeVar *stat = makeRangeVarFromNameList((List *) renameStmt->object); AppendShardIdToName(&stat->relname, shardId); AppendShardIdToName(&renameStmt->newname, shardId); SetSchemaNameIfNotExist(&stat->schemaname, schemaName); renameStmt->object = (Node *) MakeNameListFromRangeVar(stat); } else { ereport(WARNING, (errmsg("unsafe object type in rename statement"), errdetail("Object type: %u", (uint32) objectType))); } break; } case T_CreateStatsStmt: { CreateStatsStmt *createStatsStmt = (CreateStatsStmt *) parseTree; /* because CREATE STATISTICS statements can only have one relation */ RangeVar *relation = linitial(createStatsStmt->relations); char **relationName = &(relation->relname); char **objectSchemaName = &(relation->schemaname); SetSchemaNameIfNotExist(objectSchemaName, schemaName); AppendShardIdToName(relationName, shardId); RangeVar *stat = makeRangeVarFromNameList(createStatsStmt->defnames); AppendShardIdToName(&stat->relname, shardId); createStatsStmt->defnames = MakeNameListFromRangeVar(stat); break; } case T_TruncateStmt: { /* * We currently do not support truncate statements. This is * primarily because truncates allow implicit modifications to * sequences through table column dependencies. As we have not * determined our dependency model for sequences, we error here. */ ereport(ERROR, (errmsg("cannot extend name for truncate statement"))); break; } case T_SecLabelStmt: { SecLabelStmt *secLabelStmt = (SecLabelStmt *) parseTree; /* Should be looking at a security label for a table or column */ if (secLabelStmt->objtype == OBJECT_TABLE || secLabelStmt->objtype == OBJECT_COLUMN) { List *qualified_name = (List *) secLabelStmt->object; String *table_name = NULL; switch (list_length(qualified_name)) { case 1: { table_name = castNode(String, linitial(qualified_name)); break; } case 2: case 3: { table_name = castNode(String, lsecond(qualified_name)); break; } default: { /* Unlikely, but just in case */ ereport(ERROR, (errmsg( "unhandled name type in security label; name is: \"%s\"", NameListToString(qualified_name)))); break; } } /* Now change the table name: -> */ char *relationName = strVal(table_name); AppendShardIdToName(&relationName, shardId); strVal(table_name) = relationName; } else { ereport(WARNING, (errmsg( "unsafe object type in security label statement"), errdetail("Object type: %u", (uint32) secLabelStmt->objtype))); } break; /* End of handling Security Label */ } default: { ereport(WARNING, (errmsg("unsafe statement type in name extension"), errdetail("Statement type: %u", (uint32) nodeType))); break; } } } /* * RelayEventExtendConstraintAndIndexNames extends the names of constraints * and indexes in given constraint with the shardId. */ static void RelayEventExtendConstraintAndIndexNames(AlterTableStmt *alterTableStmt, Constraint *constraint, uint64 shardId) { char **constraintName = &(constraint->conname); const bool missingOk = false; Oid relationId = RangeVarGetRelid(alterTableStmt->relation, AccessShareLock, missingOk); if (constraint->indexname) { char **indexName = &(constraint->indexname); AppendShardIdToName(indexName, shardId); } /* * Append shardId to constraint names if * - table is not partitioned or * - constraint is not a CHECK constraint * * We do not want to append shardId to partitioned table shards because * the names of constraints will be inherited, and the shardId will no * longer be valid for the child table. * * See MergeConstraintsIntoExisting function in Postgres that requires * inherited check constraints in child tables to have the same name * with those in parent tables. */ if (!PartitionedTable(relationId) || constraint->contype != CONSTR_CHECK) { /* * constraint->conname could be empty in the case of * ADD {PRIMARY KEY, UNIQUE} USING INDEX. * In this case, already extended index name will be used by postgres. */ if (constraint->conname != NULL) { AppendShardIdToName(constraintName, shardId); } } } /* * RelayEventExtendNamesForInterShardCommands extends relation names in the given parse * tree for certain utility commands. The function more specifically extends table, index * and constraint names in the parse tree by appending the given shardId; thereby * avoiding name collisions in the database among sharded tables. This function * has the side effect of extending relation names in the parse tree. */ void RelayEventExtendNamesForInterShardCommands(Node *parseTree, uint64 leftShardId, char *leftShardSchemaName, uint64 rightShardId, char *rightShardSchemaName) { NodeTag nodeType = nodeTag(parseTree); switch (nodeType) { case T_AlterTableStmt: { AlterTableStmt *alterTableStmt = (AlterTableStmt *) parseTree; List *commandList = alterTableStmt->cmds; AlterTableCmd *command = NULL; foreach_declared_ptr(command, commandList) { char **referencedTableName = NULL; char **relationSchemaName = NULL; if (command->subtype == AT_AddConstraint) { Constraint *constraint = (Constraint *) command->def; if (constraint->contype == CONSTR_FOREIGN) { referencedTableName = &(constraint->pktable->relname); relationSchemaName = &(constraint->pktable->schemaname); } } else if (command->subtype == AT_AddColumn) { ColumnDef *columnDefinition = (ColumnDef *) command->def; List *columnConstraints = columnDefinition->constraints; Constraint *constraint = NULL; foreach_declared_ptr(constraint, columnConstraints) { if (constraint->contype == CONSTR_FOREIGN) { referencedTableName = &(constraint->pktable->relname); relationSchemaName = &(constraint->pktable->schemaname); } } } else if (command->subtype == AT_AttachPartition || command->subtype == AT_DetachPartition) { PartitionCmd *partitionCommand = (PartitionCmd *) command->def; referencedTableName = &(partitionCommand->name->relname); relationSchemaName = &(partitionCommand->name->schemaname); } else { continue; } /* prefix with schema name if it is not added already */ SetSchemaNameIfNotExist(relationSchemaName, rightShardSchemaName); /* * We will not append shard id to left shard name. This will be * handled when we drop into RelayEventExtendNames. */ AppendShardIdToName(referencedTableName, rightShardId); } /* drop into RelayEventExtendNames for non-inter table commands */ RelayEventExtendNames(parseTree, leftShardSchemaName, leftShardId); break; } default: { ereport(WARNING, (errmsg("unsafe statement type in name extension"), errdetail("Statement type: %u", (uint32) nodeType))); break; } } } /* * UpdateWholeRowColumnReferencesWalker extends ColumnRef nodes that end with A_Star * with the given shardId. * * ColumnRefs that don't reference A_Star are not extended as catalog access isn't * allowed here and we don't otherwise have enough context to disambiguate a * field name that is identical to the table name. */ static bool UpdateWholeRowColumnReferencesWalker(Node *node, uint64 *shardId) { bool walkIsComplete = false; if (node == NULL) { return false; } if (IsA(node, IndexElem)) { IndexElem *indexElem = (IndexElem *) node; walkIsComplete = raw_expression_tree_walker(indexElem->expr, UpdateWholeRowColumnReferencesWalker, shardId); } else if (IsA(node, ColumnRef)) { ColumnRef *columnRef = (ColumnRef *) node; Node *lastField = llast(columnRef->fields); if (IsA(lastField, A_Star)) { /* * ColumnRef fields list ends with an A_Star, so we can blindly * extend the penultimate element with the shardId. */ int colrefFieldCount = list_length(columnRef->fields); String *relnameValue = list_nth(columnRef->fields, colrefFieldCount - 2); Assert(IsA(relnameValue, String)); AppendShardIdToName(&strVal(relnameValue), *shardId); } /* might be more than one ColumnRef to visit */ walkIsComplete = false; } else { walkIsComplete = raw_expression_tree_walker(node, UpdateWholeRowColumnReferencesWalker, shardId); } return walkIsComplete; } /* * SetSchemaNameIfNotExist function checks whether schemaName is set and if it is not set * it sets its value to given newSchemaName. */ void SetSchemaNameIfNotExist(char **schemaName, const char *newSchemaName) { if ((*schemaName) == NULL) { *schemaName = pstrdup(newSchemaName); } } /* * AppendShardIdToName appends shardId to the given name. The function takes in * the name's address in order to reallocate memory for the name in the same * memory context the name was originally created in. */ void AppendShardIdToName(char **name, uint64 shardId) { char extendedName[NAMEDATALEN]; int nameLength = strlen(*name); char shardIdAndSeparator[NAMEDATALEN]; uint32 longNameHash = 0; int multiByteClipLength = 0; if (nameLength >= NAMEDATALEN) { ereport(ERROR, (errcode(ERRCODE_NAME_TOO_LONG), errmsg("identifier must be less than %d characters", NAMEDATALEN))); } SafeSnprintf(shardIdAndSeparator, NAMEDATALEN, "%c" UINT64_FORMAT, SHARD_NAME_SEPARATOR, shardId); int shardIdAndSeparatorLength = strlen(shardIdAndSeparator); /* * If *name strlen is < (NAMEDATALEN - shardIdAndSeparatorLength), * it is safe merely to append the separator and shardId. */ if (nameLength < (NAMEDATALEN - shardIdAndSeparatorLength)) { SafeSnprintf(extendedName, NAMEDATALEN, "%s%s", (*name), shardIdAndSeparator); } /* * Otherwise, we need to truncate the name further to accommodate * a sufficient hash value. The resulting name will avoid collision * with other hashed names such that for any given schema with * 90 distinct object names that are long enough to require hashing * (typically 57-63 characters), the chance of a collision existing is: * * If randomly generated UTF8 names: * (1e-6) * (9.39323783788e-114) ~= (9.39e-120) * If random case-insensitive ASCII names (letter first, 37 useful characters): * (1e-6) * (2.80380202421e-74) ~= (2.8e-80) * If names sharing only N distinct 45- to 47-character prefixes: * (1e-6) * (1/N) = (1e-6/N) * 1e-7 for 10 distinct prefixes * 5e-8 for 20 distinct prefixes * * In practice, since shard IDs are globally unique, the risk of name collision * exists only amongst objects that pertain to a single distributed table * and are created for each shard: the table name and the names of any indexes * or index-backed constraints. Since there are typically less than five such * names, and almost never more than ten, the expected collision rate even in * the worst case (ten names share same 45- to 47-character prefix) is roughly * 1e-8: one in 100 million schemas will experience a name collision only if ALL * 100 million schemas present the worst-case scenario. */ else { longNameHash = hash_any((unsigned char *) (*name), nameLength); multiByteClipLength = pg_mbcliplen(*name, nameLength, (NAMEDATALEN - shardIdAndSeparatorLength - 10)); SafeSnprintf(extendedName, NAMEDATALEN, "%.*s%c%.8x%s", multiByteClipLength, (*name), SHARD_NAME_SEPARATOR, longNameHash, shardIdAndSeparator); } (*name) = (char *) repalloc((*name), NAMEDATALEN); int neededBytes = SafeSnprintf((*name), NAMEDATALEN, "%s", extendedName); if (neededBytes < 0) { ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory: %m"))); } else if (neededBytes >= NAMEDATALEN) { ereport(ERROR, (errcode(ERRCODE_STRING_DATA_RIGHT_TRUNCATION), errmsg("new name %s would be truncated at %d characters", extendedName, NAMEDATALEN))); } } /* * shard_name() provides a PG function interface to AppendShardNameToId above. * Returns the name of a shard as a quoted schema-qualified identifier. */ Datum shard_name(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); int64 shardId = PG_GETARG_INT64(1); bool skipQualifyPublic = PG_GETARG_BOOL(2); char *qualifiedName = NULL; if (shardId <= 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("shard_id cannot be zero or negative value"))); } if (!OidIsValid(relationId)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("object_name does not reference a valid relation"))); } char *relationName = get_rel_name(relationId); if (relationName == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("object_name does not reference a valid relation"))); } AppendShardIdToName(&relationName, shardId); Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); if (skipQualifyPublic && strncmp(schemaName, "public", NAMEDATALEN) == 0) { qualifiedName = (char *) quote_identifier(relationName); } else { qualifiedName = quote_qualified_identifier(schemaName, relationName); } PG_RETURN_TEXT_P(cstring_to_text(qualifiedName)); } ================================================ FILE: src/backend/distributed/replication/multi_logical_replication.c ================================================ /*------------------------------------------------------------------------- * * multi_logical_replication.c * * This file contains functions to use logical replication on the distributed * tables for moving/replicating shards. * * Copyright (c) 2017, Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" #include "catalog/pg_subscription_rel.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "nodes/bitmapset.h" #include "parser/scansup.h" #include "postmaster/interrupt.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/lock.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/fmgrprotos.h" #include "utils/formatting.h" #include "utils/guc.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/pg_lsn.h" #include "utils/rel.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_replication.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/priority.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_transfer.h" #include "distributed/version_compat.h" #define CURRENT_LOG_POSITION_COMMAND "SELECT pg_current_wal_lsn()" /* decimal representation of Adler-16 hash value of citus_shard_move_publication */ #define SHARD_MOVE_ADVISORY_LOCK_FIRST_KEY 44000 /* decimal representation of Adler-16 hash value of citus_shard_move_subscription */ #define SHARD_MOVE_ADVISORY_LOCK_SECOND_KEY 55152 static const char *publicationPrefix[] = { [SHARD_MOVE] = "citus_shard_move_publication_", [SHARD_SPLIT] = "citus_shard_split_publication_", }; static const char *replicationSlotPrefix[] = { [SHARD_MOVE] = "citus_shard_move_slot_", [SHARD_SPLIT] = "citus_shard_split_slot_", }; /* * IMPORTANT: All the subscription names should start with "citus_". Otherwise * our utility hook does not defend against non-superusers altering or dropping * them, which is important for security purposes. * * We should also keep these in sync with IsCitusShardTransferBackend(). */ static const char *subscriptionPrefix[] = { [SHARD_MOVE] = "citus_shard_move_subscription_", [SHARD_SPLIT] = "citus_shard_split_subscription_", }; static const char *subscriptionRolePrefix[] = { [SHARD_MOVE] = "citus_shard_move_subscription_role_", [SHARD_SPLIT] = "citus_shard_split_subscription_role_", }; /* GUC variable, defaults to 2 hours */ int LogicalReplicationTimeout = 2 * 60 * 60 * 1000; /* see the comment in master_move_shard_placement */ bool PlacementMovedUsingLogicalReplicationInTX = false; /* report in every 10 seconds */ static int logicalReplicationProgressReportTimeout = 10 * 1000; static List * PrepareReplicationSubscriptionList(List *shardList); static List * GetReplicaIdentityCommandListForShard(Oid relationId, uint64 shardId); static List * GetIndexCommandListForShardBackingReplicaIdentity(Oid relationId, uint64 shardId); static void CreatePostLogicalReplicationDataLoadObjects(List *logicalRepTargetList, LogicalRepType type, bool skipInterShardRelationships); static void ExecuteCreateIndexCommands(List *logicalRepTargetList); static void ExecuteCreateConstraintsBackedByIndexCommands(List *logicalRepTargetList); static List * ConvertNonExistingPlacementDDLCommandsToTasks(List *shardCommandList, char *targetNodeName, int targetNodePort); static void ExecuteClusterOnCommands(List *logicalRepTargetList); static void ExecuteCreateIndexStatisticsCommands(List *logicalRepTargetList); static void ExecuteRemainingPostLoadTableCommands(List *logicalRepTargetList); static char * escape_param_str(const char *str); static XLogRecPtr GetRemoteLSN(MultiConnection *connection, char *command); static void WaitForMiliseconds(long timeout); static XLogRecPtr GetSubscriptionPosition(GroupedLogicalRepTargets * groupedLogicalRepTargets); static HTAB * CreateShardMovePublicationInfoHash(WorkerNode *targetNode, List *shardIntervals); static List * CreateShardMoveLogicalRepTargetList(HTAB *publicationInfoHash, List *shardList); static void WaitForGroupedLogicalRepTargetsToCatchUp(XLogRecPtr sourcePosition, GroupedLogicalRepTargets * groupedLogicalRepTargets); /* * LogicallyReplicateShards replicates a list of shards from one node to another * using logical replication. Once replication is reasonably caught up, writes * are blocked and then the publication and subscription are dropped. * * The caller of the function should ensure that logical replication is applicable * for the given shards, source and target nodes. Also, the caller is responsible * for ensuring that the input shard list consists of co-located distributed tables * or a single shard. */ void LogicallyReplicateShards(List *shardList, char *sourceNodeName, int sourceNodePort, char *targetNodeName, int targetNodePort, bool skipInterShardRelationshipCreation) { char *superUser = CitusExtensionOwnerName(); char *databaseName = get_database_name(MyDatabaseId); int connectionFlags = FORCE_NEW_CONNECTION; List *replicationSubscriptionList = PrepareReplicationSubscriptionList(shardList); /* no shards to move */ if (list_length(replicationSubscriptionList) == 0) { return; } MultiConnection *sourceConnection = GetNodeUserDatabaseConnection(connectionFlags, sourceNodeName, sourceNodePort, superUser, databaseName); /* * Operations on publications and replication slots cannot run in a * transaction block. We claim the connections exclusively to ensure they * do not get used for metadata syncing, which does open a transaction * block. */ ClaimConnectionExclusively(sourceConnection); WorkerNode *sourceNode = FindWorkerNode(sourceNodeName, sourceNodePort); WorkerNode *targetNode = FindWorkerNode(targetNodeName, targetNodePort); HTAB *publicationInfoHash = CreateShardMovePublicationInfoHash( targetNode, replicationSubscriptionList); List *logicalRepTargetList = CreateShardMoveLogicalRepTargetList(publicationInfoHash, shardList); HTAB *groupedLogicalRepTargetsHash = CreateGroupedLogicalRepTargetsHash( logicalRepTargetList); CreateGroupedLogicalRepTargetsConnections(groupedLogicalRepTargetsHash, superUser, databaseName); MultiConnection *sourceReplicationConnection = GetReplicationConnection(sourceConnection->hostname, sourceConnection->port); /* set up the publication on the source and subscription on the target */ CreatePublications(sourceConnection, publicationInfoHash); char *snapshot = CreateReplicationSlots( sourceConnection, sourceReplicationConnection, logicalRepTargetList, "pgoutput"); CreateSubscriptions( sourceConnection, sourceConnection->database, logicalRepTargetList); /* only useful for isolation testing, see the function comment for the details */ ConflictWithIsolationTestingBeforeCopy(); /* * We have to create the primary key (or any other replica identity) * before the update/delete operations that are queued will be * replicated. Because if the replica identity does not exist on the * target, the replication would fail. * * So the latest possible moment we could do this is right after the * initial data COPY, but before enabling the susbcriptions. It might * seem like a good idea to it after the initial data COPY, since * it's generally the rule that it's cheaper to build an index at once * than to create it incrementally. This general rule, is why we create * all the regular indexes as late during the move as possible. * * But as it turns out in practice it's not as clear cut, and we saw a * speed degradation in the time it takes to move shards when doing the * replica identity creation after the initial COPY. So, instead we * keep it before the COPY. */ CreateReplicaIdentities(logicalRepTargetList); UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceNodeName, sourceNodePort, PLACEMENT_UPDATE_STATUS_COPYING_DATA); CopyShardsToNode(sourceNode, targetNode, shardList, snapshot); /* * We can close this connection now, because we're done copying the * data and thus don't need access to the snapshot anymore. The * replication slot will still be at the same LSN, because the * subscriptions have not been enabled yet. */ CloseConnection(sourceReplicationConnection); /* * Start the replication and copy all data */ CompleteNonBlockingShardTransfer(shardList, sourceConnection, publicationInfoHash, logicalRepTargetList, groupedLogicalRepTargetsHash, SHARD_MOVE, skipInterShardRelationshipCreation); /* * We use these connections exclusively for subscription management, * because otherwise subsequent metadata changes may inadvertedly use * these connections instead of the connections that were used to * grab locks in BlockWritesToShardList. */ CloseGroupedLogicalRepTargetsConnections(groupedLogicalRepTargetsHash); CloseConnection(sourceConnection); } /* * CreateGroupedLogicalRepTargetsHash creates a hashmap that groups the subscriptions * logicalRepTargetList by node. This is useful for cases where we want to * iterate the subscriptions by node, so we can batch certain operations, such * as checking subscription readiness. */ HTAB * CreateGroupedLogicalRepTargetsHash(List *logicalRepTargetList) { HTAB *logicalRepTargetsHash = CreateSimpleHash(uint32, GroupedLogicalRepTargets); LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { bool found = false; GroupedLogicalRepTargets *groupedLogicalRepTargets = (GroupedLogicalRepTargets *) hash_search( logicalRepTargetsHash, &target->replicationSlot->targetNodeId, HASH_ENTER, &found); if (!found) { groupedLogicalRepTargets->logicalRepTargetList = NIL; groupedLogicalRepTargets->superuserConnection = NULL; } groupedLogicalRepTargets->logicalRepTargetList = lappend(groupedLogicalRepTargets->logicalRepTargetList, target); } return logicalRepTargetsHash; } /* * CompleteNonBlockingShardTransfer uses logical replication to apply the changes * made on the source to the target. It also runs all DDL on the target shards * that need to be run after the data copy. * * For shard splits it skips the partition hierarchy and foreign key creation * though, since those need to happen after the metadata is updated. */ void CompleteNonBlockingShardTransfer(List *shardList, MultiConnection *sourceConnection, HTAB *publicationInfoHash, List *logicalRepTargetList, HTAB *groupedLogicalRepTargetsHash, LogicalRepType type, bool skipInterShardRelationshipCreation) { /* Start applying the changes from the replication slots to catch up. */ EnableSubscriptions(logicalRepTargetList); UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceConnection->hostname, sourceConnection->port, PLACEMENT_UPDATE_STATUS_CATCHING_UP); /* * Wait until all the subscriptions are caught up to changes that * happened after the initial COPY on the shards. */ WaitForAllSubscriptionsToCatchUp(sourceConnection, groupedLogicalRepTargetsHash); UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceConnection->hostname, sourceConnection->port, PLACEMENT_UPDATE_STATUS_CREATING_CONSTRAINTS); /* * Now lets create the post-load objects, such as the indexes, constraints * and partitioning hierarchy. Once they are done, wait until the replication * catches up again. So we don't block writes too long. */ CreatePostLogicalReplicationDataLoadObjects(logicalRepTargetList, type, skipInterShardRelationshipCreation); UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceConnection->hostname, sourceConnection->port, PLACEMENT_UPDATE_STATUS_FINAL_CATCH_UP); WaitForAllSubscriptionsToCatchUp(sourceConnection, groupedLogicalRepTargetsHash); /* only useful for isolation testing, see the function comment for the details */ ConflictWithIsolationTestingAfterCopy(); /* * We're almost done, we'll block the writes to the shards that we're * replicating and expect all the subscription to catch up quickly * afterwards. * * Notice that although shards in partitioned relation are excluded from * logical replication, they are still locked against modification, and * foreign constraints are created on them too. */ BlockWritesToShardList(shardList); WaitForAllSubscriptionsToCatchUp(sourceConnection, groupedLogicalRepTargetsHash); if (type != SHARD_SPLIT && !skipInterShardRelationshipCreation) { UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceConnection->hostname, sourceConnection->port, PLACEMENT_UPDATE_STATUS_CREATING_FOREIGN_KEYS); /* * We're creating the foreign constraints to reference tables after the * data is already replicated and all the necessary locks are acquired. * * We prefer to do it here because the placements of reference tables * are always valid, and any modification during the shard move would * cascade to the hash distributed tables' shards if we had created * the constraints earlier. The same is true for foreign keys between * tables owned by different users. */ CreateUncheckedForeignKeyConstraints(logicalRepTargetList); } UpdatePlacementUpdateStatusForShardIntervalList( shardList, sourceConnection->hostname, sourceConnection->port, PLACEMENT_UPDATE_STATUS_COMPLETING); } /* * CreateShardMovePublicationInfoHash creates hashmap of PublicationInfos for a * shard move. Even though we only support moving a shard to a single target * node, the resulting hashmap can have multiple PublicationInfos in it. * The reason for that is that we need a separate publication for each * distributed table owning user in the shard group. */ static HTAB * CreateShardMovePublicationInfoHash(WorkerNode *targetNode, List *shardIntervals) { HTAB *publicationInfoHash = CreateSimpleHash(NodeAndOwner, PublicationInfo); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervals) { NodeAndOwner key; key.nodeId = targetNode->nodeId; key.tableOwnerId = TableOwnerOid(shardInterval->relationId); bool found = false; PublicationInfo *publicationInfo = (PublicationInfo *) hash_search(publicationInfoHash, &key, HASH_ENTER, &found); if (!found) { publicationInfo->name = PublicationName(SHARD_MOVE, key.nodeId, key.tableOwnerId); publicationInfo->shardIntervals = NIL; } publicationInfo->shardIntervals = lappend(publicationInfo->shardIntervals, shardInterval); } return publicationInfoHash; } /* * CreateShardMoveLogicalRepTargetList creates the list containing all the * subscriptions that should be connected to the publications in the given * publicationHash. */ static List * CreateShardMoveLogicalRepTargetList(HTAB *publicationInfoHash, List *shardList) { List *logicalRepTargetList = NIL; HASH_SEQ_STATUS status; hash_seq_init(&status, publicationInfoHash); Oid nodeId = InvalidOid; PublicationInfo *publication = NULL; while ((publication = (PublicationInfo *) hash_seq_search(&status)) != NULL) { Oid ownerId = publication->key.tableOwnerId; nodeId = publication->key.nodeId; LogicalRepTarget *target = palloc0(sizeof(LogicalRepTarget)); target->subscriptionName = SubscriptionName(SHARD_MOVE, ownerId); target->tableOwnerId = ownerId; target->publication = publication; publication->target = target; target->newShards = NIL; target->subscriptionOwnerName = SubscriptionRoleName(SHARD_MOVE, ownerId); target->replicationSlot = palloc0(sizeof(ReplicationSlotInfo)); target->replicationSlot->name = ReplicationSlotNameForNodeAndOwnerForOperation(SHARD_MOVE, nodeId, ownerId, CurrentOperationId); target->replicationSlot->targetNodeId = nodeId; target->replicationSlot->tableOwnerId = ownerId; logicalRepTargetList = lappend(logicalRepTargetList, target); } ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardList) { NodeAndOwner key; key.nodeId = nodeId; key.tableOwnerId = TableOwnerOid(shardInterval->relationId); bool found = false; publication = (PublicationInfo *) hash_search( publicationInfoHash, &key, HASH_FIND, &found); if (!found) { ereport(ERROR, errmsg("Could not find publication matching a split")); } publication->target->newShards = lappend( publication->target->newShards, shardInterval); } return logicalRepTargetList; } /* * PrepareReplicationSubscriptionList returns list of shards to be logically * replicated from given shard list. This is needed because Postgres does not * allow logical replication on partitioned tables, therefore shards belonging * to a partitioned tables should be exluded from logical replication * subscription list. */ static List * PrepareReplicationSubscriptionList(List *shardList) { List *replicationSubscriptionList = NIL; ListCell *shardCell = NULL; foreach(shardCell, shardList) { ShardInterval *shardInterval = (ShardInterval *) lfirst(shardCell); if (!PartitionedTable(shardInterval->relationId)) { /* only add regular and child tables to subscription */ replicationSubscriptionList = lappend(replicationSubscriptionList, shardInterval); } } return replicationSubscriptionList; } /* * CreateReplicaIdentities creates replica identities for all the shards that * are part of the given subscriptions. */ void CreateReplicaIdentities(List *logicalRepTargetList) { LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { MultiConnection *superuserConnection = target->superuserConnection; CreateReplicaIdentitiesOnNode( target->newShards, superuserConnection->hostname, superuserConnection->port); } } /* * CreateReplicaIdentitiesOnNode gets a shardList and creates all the replica * identities on the shards in the given node. */ void CreateReplicaIdentitiesOnNode(List *shardList, char *nodeName, int32 nodePort) { MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateReplicaIdentitiesOnNode", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); ShardInterval *shardInterval; foreach_declared_ptr(shardInterval, shardList) { uint64 shardId = shardInterval->shardId; Oid relationId = shardInterval->relationId; List *backingIndexCommandList = GetIndexCommandListForShardBackingReplicaIdentity(relationId, shardId); List *replicaIdentityShardCommandList = GetReplicaIdentityCommandListForShard(relationId, shardId); List *commandList = list_concat(backingIndexCommandList, replicaIdentityShardCommandList); if (commandList != NIL) { ereport(DEBUG1, (errmsg("Creating replica identity for shard %ld on " "target node %s:%d", shardId, nodeName, nodePort))); SendCommandListToWorkerOutsideTransaction(nodeName, nodePort, TableOwner(relationId), commandList); } MemoryContextReset(localContext); } MemoryContextSwitchTo(oldContext); } /* * GetIndexCommandListForShardBackingReplicaIdentity returns all the create index * commands that are needed to create replica identity. If the table doesn't have * a replica identity, the function returns NIL. */ static List * GetIndexCommandListForShardBackingReplicaIdentity(Oid relationId, uint64 shardId) { List *commandList = NIL; Relation relation = table_open(relationId, AccessShareLock); Oid replicaIdentityIndex = GetRelationIdentityOrPK(relation); table_close(relation, NoLock); if (OidIsValid(replicaIdentityIndex)) { /* * The replica identity is backed by an index or primary key, * so get the index/pkey definition first. */ HeapTuple indexTuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(replicaIdentityIndex)); if (!HeapTupleIsValid(indexTuple)) { /* should not happen */ elog(ERROR, "cache lookup failed for index %u", replicaIdentityIndex); } Form_pg_index indexForm = ((Form_pg_index) GETSTRUCT(indexTuple)); List *indexCommandTableDDLList = NIL; int indexFlags = INCLUDE_INDEX_ALL_STATEMENTS; GatherIndexAndConstraintDefinitionList(indexForm, &indexCommandTableDDLList, indexFlags); List *indexCommandShardDDLList = WorkerApplyShardDDLCommandList(indexCommandTableDDLList, shardId); commandList = list_concat(commandList, indexCommandShardDDLList); ReleaseSysCache(indexTuple); } return commandList; } /* * GetReplicaIdentityCommandListForShard returns the create replica identity * command that are needed to create replica identity. If the table doesn't have * a replica identity, the function returns NIL. */ static List * GetReplicaIdentityCommandListForShard(Oid relationId, uint64 shardId) { List *replicaIdentityTableDDLCommand = GetTableReplicaIdentityCommand(relationId); List *replicaIdentityShardCommandList = WorkerApplyShardDDLCommandList(replicaIdentityTableDDLCommand, shardId); return replicaIdentityShardCommandList; } /* * CreatePostLogicalReplicationDataLoadObjects gets a shardList and creates all * the objects that can be created after the data is moved with logical replication. */ static void CreatePostLogicalReplicationDataLoadObjects(List *logicalRepTargetList, LogicalRepType type, bool skipInterShardRelationships) { /* * We create indexes in 4 steps. * - CREATE INDEX statements * - CREATE CONSTRAINT statements that are backed by * indexes (unique and exclude constraints) * - ALTER TABLE %s CLUSTER ON %s * - ALTER INDEX %s ALTER COLUMN %d SET STATISTICS %d * * On each step, we execute can execute commands in parallel. For example, * multiple indexes on the shard table or indexes for the colocated shards * can be created in parallel. However, the latter two steps, clustering the * table and setting the statistics of indexes, depends on the indexes being * created. That's why the execution is divided into four distinct stages. */ ExecuteCreateIndexCommands(logicalRepTargetList); ExecuteCreateConstraintsBackedByIndexCommands(logicalRepTargetList); ExecuteClusterOnCommands(logicalRepTargetList); ExecuteCreateIndexStatisticsCommands(logicalRepTargetList); /* * Once the indexes are created, there are few more objects like triggers and table * statistics that should be created after the data move. */ ExecuteRemainingPostLoadTableCommands(logicalRepTargetList); /* * Creating the partitioning hierarchy errors out in shard splits when */ if (type != SHARD_SPLIT && !skipInterShardRelationships) { /* create partitioning hierarchy, if any */ CreatePartitioningHierarchy(logicalRepTargetList); } } /* * ExecuteCreateIndexCommands gets a shardList and creates all the indexes * for the given shardList in the given target node. * * The execution is done in parallel, and throws an error if any of the * commands fail. */ static void ExecuteCreateIndexCommands(List *logicalRepTargetList) { List *taskList = NIL; LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; List *tableCreateIndexCommandList = GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(relationId, INCLUDE_CREATE_INDEX_STATEMENTS); List *shardCreateIndexCommandList = WorkerApplyShardDDLCommandList(tableCreateIndexCommandList, shardInterval->shardId); List *taskListForShard = ConvertNonExistingPlacementDDLCommandsToTasks( shardCreateIndexCommandList, target->superuserConnection->hostname, target->superuserConnection->port); taskList = list_concat(taskList, taskListForShard); } } /* * We are going to create indexes and constraints using the current user. That is * alright because an index/constraint always belongs to the owner of the table, * and Citus already ensures that the current user owns all the tables that are * moved. * * CREATE INDEX commands acquire ShareLock on a relation. So, it is * allowed to run multiple CREATE INDEX commands concurrently on a table * and across different tables (e.g., shards). */ ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(indexes)"))); ExecuteTaskListOutsideTransaction(ROW_MODIFY_NONE, taskList, MaxAdaptiveExecutorPoolSize, NIL); } /* * ExecuteCreateConstraintsBackedByIndexCommands gets a shardList and creates all the constraints * that are backed by indexes for the given shardList in the given target node. * * The execution is done in sequential mode, and throws an error if any of the * commands fail. */ static void ExecuteCreateConstraintsBackedByIndexCommands(List *logicalRepTargetList) { ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(constraints backed by indexes)"))); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateConstraintsBackedByIndexContext", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; List *tableCreateConstraintCommandList = GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(relationId, INCLUDE_CREATE_CONSTRAINT_STATEMENTS); if (tableCreateConstraintCommandList == NIL) { /* no constraints backed by indexes, skip */ MemoryContextReset(localContext); continue; } List *shardCreateConstraintCommandList = WorkerApplyShardDDLCommandList(tableCreateConstraintCommandList, shardInterval->shardId); char *tableOwner = TableOwner(shardInterval->relationId); SendCommandListToWorkerOutsideTransaction( target->superuserConnection->hostname, target->superuserConnection->port, tableOwner, shardCreateConstraintCommandList); MemoryContextReset(localContext); } } MemoryContextSwitchTo(oldContext); } /* * ConvertNonExistingShardDDLCommandsToTasks generates one task per input * element in shardCommandList. * * The generated tasks' placements do not exist (yet). We are generating * fake placements for the tasks. */ static List * ConvertNonExistingPlacementDDLCommandsToTasks(List *shardCommandList, char *targetNodeName, int targetNodePort) { WorkerNode *workerNode = FindWorkerNodeOrError(targetNodeName, targetNodePort); List *taskList = NIL; uint64 jobId = INVALID_JOB_ID; ListCell *commandCell = NULL; int taskId = 1; foreach(commandCell, shardCommandList) { char *command = (char *) lfirst(commandCell); Task *task = CreateBasicTask(jobId, taskId, DDL_TASK, command); /* this placement currently does not exist */ ShardPlacement *taskPlacement = CitusMakeNode(ShardPlacement); SetPlacementNodeMetadata(taskPlacement, workerNode); task->taskPlacementList = list_make1(taskPlacement); taskList = lappend(taskList, task); taskId++; } return taskList; } /* * ExecuteClusterOnCommands gets a shardList and creates all the CLUSTER ON commands * for the given shardList in the given target node. * * The execution is done in parallel, and in case of any failure, the transaction * is aborted. */ static void ExecuteClusterOnCommands(List *logicalRepTargetList) { List *taskList = NIL; LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; List *tableAlterTableClusterOnCommandList = GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(relationId, INCLUDE_INDEX_CLUSTERED_STATEMENTS); List *shardAlterTableClusterOnCommandList = WorkerApplyShardDDLCommandList(tableAlterTableClusterOnCommandList, shardInterval->shardId); List *taskListForShard = ConvertNonExistingPlacementDDLCommandsToTasks( shardAlterTableClusterOnCommandList, target->superuserConnection->hostname, target->superuserConnection->port); taskList = list_concat(taskList, taskListForShard); } } ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(CLUSTER ON)"))); ExecuteTaskListOutsideTransaction(ROW_MODIFY_NONE, taskList, MaxAdaptiveExecutorPoolSize, NIL); } /* * ExecuteCreateIndexStatisticsCommands gets a shardList and creates * all the statistics objects for the indexes in the given target node. * * The execution is done in sequentially, and in case of any failure, the transaction * is aborted. */ static void ExecuteCreateIndexStatisticsCommands(List *logicalRepTargetList) { ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(index statistics)"))); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateIndexStatisticsContext", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; List *tableAlterIndexSetStatisticsCommandList = GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(relationId, INCLUDE_INDEX_STATISTICS_STATEMENTTS); List *shardAlterIndexSetStatisticsCommandList = WorkerApplyShardDDLCommandList(tableAlterIndexSetStatisticsCommandList, shardInterval->shardId); if (shardAlterIndexSetStatisticsCommandList == NIL) { /* no index statistics exists, skip */ MemoryContextReset(localContext); continue; } /* * These remaining operations do not require significant resources, so no * need to create them in parallel. */ char *tableOwner = TableOwner(shardInterval->relationId); SendCommandListToWorkerOutsideTransaction( target->superuserConnection->hostname, target->superuserConnection->port, tableOwner, shardAlterIndexSetStatisticsCommandList); MemoryContextReset(localContext); } } MemoryContextSwitchTo(oldContext); } /* * ExecuteRemainingPostLoadTableCommands gets a shardList and creates * all the remaining post load objects other than the indexes * in the given target node. */ static void ExecuteRemainingPostLoadTableCommands(List *logicalRepTargetList) { ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(triggers and table statistics)" ))); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateTableStatisticsContext", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { Oid relationId = shardInterval->relationId; bool includeIndexes = false; bool includeReplicaIdentity = false; List *tablePostLoadTableCommandList = GetPostLoadTableCreationCommands(relationId, includeIndexes, includeReplicaIdentity); List *shardPostLoadTableCommandList = WorkerApplyShardDDLCommandList(tablePostLoadTableCommandList, shardInterval->shardId); if (shardPostLoadTableCommandList == NIL) { /* no index statistics exists, skip */ continue; } /* * These remaining operations do not require significant resources, so no * need to create them in parallel. */ char *tableOwner = TableOwner(shardInterval->relationId); SendCommandListToWorkerOutsideTransaction( target->superuserConnection->hostname, target->superuserConnection->port, tableOwner, shardPostLoadTableCommandList); MemoryContextReset(localContext); } } MemoryContextSwitchTo(oldContext); } /* * CreatePartitioningHierarchy gets a shardList and creates the partitioning * hierarchy between the shardList, if any, */ void CreatePartitioningHierarchy(List *logicalRepTargetList) { ereport(DEBUG1, (errmsg("Creating post logical replication objects " "(partitioning hierarchy)"))); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreatePartitioningHierarchy", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, target->newShards) { if (PartitionTable(shardInterval->relationId)) { char *attachPartitionCommand = GenerateAttachShardPartitionCommand(shardInterval); char *tableOwner = TableOwner(shardInterval->relationId); /* * Attaching partition may acquire conflicting locks when created in * parallel, so create them sequentially. Also attaching partition * is a quick operation, so it is fine to execute sequentially. */ MultiConnection *connection = GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION, target->superuserConnection->hostname, target->superuserConnection->port, tableOwner, NULL); ExecuteCriticalRemoteCommand(connection, attachPartitionCommand); MemoryContextReset(localContext); } } } MemoryContextSwitchTo(oldContext); } /* * CreateUncheckedForeignKeyConstraints is used to create the foreign * constraints on the logical replication target without checking that they are * actually valid. * * We skip the validation phase of foreign keys to after a shard * move/copy/split because the validation is pretty costly and given that the * source placements are already valid, the validation in the target nodes is * useless. */ void CreateUncheckedForeignKeyConstraints(List *logicalRepTargetList) { MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateKeyForeignConstraints", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); /* * Iterate over all the shards in the shard group. */ LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ShardInterval *shardInterval = NULL; /* * Iterate on split shards list for a given shard and create constraints. */ foreach_declared_ptr(shardInterval, target->newShards) { List *commandList = CopyShardForeignConstraintCommandList( shardInterval); commandList = list_concat( list_make1("SET LOCAL citus.skip_constraint_validation TO ON;"), commandList); SendCommandListToWorkerOutsideTransactionWithConnection( target->superuserConnection, commandList); MemoryContextReset(localContext); } } MemoryContextSwitchTo(oldContext); } /* * ConflictWithIsolationTestingBeforeCopy is only useful to test * get_rebalance_progress by pausing before doing the actual copy. This way we * can see the state of the tables at that point. This should not be called by * any code-path except for code paths to move and split shards(). * * Note that since the cost of calling this function is pretty low, we prefer * to use it in non-assert builds as well not to diverge in the behaviour. */ extern void ConflictWithIsolationTestingBeforeCopy(void) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; if (RunningUnderCitusTestSuite) { SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, SHARD_MOVE_ADVISORY_LOCK_SECOND_KEY, SHARD_MOVE_ADVISORY_LOCK_FIRST_KEY, 2); /* uses sharelock so concurrent moves don't conflict with eachother */ (void) LockAcquire(&tag, ShareLock, sessionLock, dontWait); } } /* * ConflictWithIsolationTestingAfterCopy is only useful for two types of tests. * 1. Testing the output of get_rebalance_progress after the copy is completed, * but before the move is completely finished. Because finishing the move * will clear the contents of get_rebalance_progress. * 2. To test that our non-blocking shard moves/splits actually don't block * writes. Since logically replicating shards does eventually block * modifications, it becomes tricky to use isolation tester to show * concurrent behaviour of online shard rebalancing and modification * queries. So, during logical replication we call this function at * the end of the catchup, right before blocking writes. * * Note that since the cost of calling this function is pretty low, we prefer * to use it in non-assert builds as well not to diverge in the behaviour. */ extern void ConflictWithIsolationTestingAfterCopy(void) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; if (RunningUnderCitusTestSuite) { SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, SHARD_MOVE_ADVISORY_LOCK_FIRST_KEY, SHARD_MOVE_ADVISORY_LOCK_SECOND_KEY, 2); /* uses sharelock so concurrent moves don't conflict with eachother */ (void) LockAcquire(&tag, ShareLock, sessionLock, dontWait); } } /* * PublicationName returns the name of the publication for the given node and * table owner. */ char * PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId) { return psprintf("%s%u_%u_%lu", publicationPrefix[type], nodeId, ownerId, CurrentOperationId); } /* * ReplicationSlotNameForNodeAndOwnerForOperation returns the name of the * replication slot for the given node, table owner and operation id. */ char * ReplicationSlotNameForNodeAndOwnerForOperation(LogicalRepType type, uint32_t nodeId, Oid ownerId, OperationId operationId) { StringInfo slotName = makeStringInfo(); appendStringInfo(slotName, "%s%u_%u_%lu", replicationSlotPrefix[type], nodeId, ownerId, operationId); if (slotName->len > NAMEDATALEN) { ereport(ERROR, (errmsg( "Replication Slot name:%s having length:%d is greater than maximum allowed length:%d", slotName->data, slotName->len, NAMEDATALEN))); } return slotName->data; } /* * SubscriptionName returns the name of the subscription for the given owner. */ char * SubscriptionName(LogicalRepType type, Oid ownerId) { return psprintf("%s%u_%lu", subscriptionPrefix[type], ownerId, CurrentOperationId); } /* * SubscriptionRoleName returns the name of the role used by the * subscription that subscribes to the tables of the given owner. */ char * SubscriptionRoleName(LogicalRepType type, Oid ownerId) { return psprintf("%s%u_%lu", subscriptionRolePrefix[type], ownerId, CurrentOperationId); } /* * GetQueryResultStringList expects a query that returns a single column of * strings. This query is executed on the connection and the function then * returns the results of the query in a List. */ List * GetQueryResultStringList(MultiConnection *connection, char *query) { bool raiseInterrupts = true; int querySent = SendRemoteCommand(connection, query); if (querySent == 0) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } int rowCount = PQntuples(result); int columnCount = PQnfields(result); if (columnCount != 1) { ereport(ERROR, (errmsg("unexpected number of columns returned while reading "))); } List *resultList = NIL; for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { int columnIndex = 0; StringInfo resultStringInfo = makeStringInfo(); char *resultString = PQgetvalue(result, rowIndex, columnIndex); /* we're using the stringinfo to copy the data into the current memory context */ appendStringInfoString(resultStringInfo, resultString); resultList = lappend(resultList, resultStringInfo->data); } PQclear(result); ForgetResults(connection); return resultList; } /* * CreatePublications creates a the publications defined in the * publicationInfoHash over the given connection. */ void CreatePublications(MultiConnection *connection, HTAB *publicationInfoHash) { HASH_SEQ_STATUS status; hash_seq_init(&status, publicationInfoHash); PublicationInfo *entry = NULL; while ((entry = (PublicationInfo *) hash_seq_search(&status)) != NULL) { StringInfo createPublicationCommand = makeStringInfo(); bool prefixWithComma = false; appendStringInfo(createPublicationCommand, "CREATE PUBLICATION %s FOR TABLE ", quote_identifier(entry->name)); ShardInterval *shard = NULL; foreach_declared_ptr(shard, entry->shardIntervals) { char *shardName = ConstructQualifiedShardName(shard); if (prefixWithComma) { appendStringInfoString(createPublicationCommand, ","); } appendStringInfoString(createPublicationCommand, shardName); prefixWithComma = true; } WorkerNode *worker = FindWorkerNode(connection->hostname, connection->port); InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_PUBLICATION, entry->name, worker->groupId, CLEANUP_ALWAYS); ExecuteCriticalRemoteCommand(connection, DISABLE_DDL_PROPAGATION); ExecuteCriticalRemoteCommand(connection, createPublicationCommand->data); ExecuteCriticalRemoteCommand(connection, ENABLE_DDL_PROPAGATION); pfree(createPublicationCommand->data); pfree(createPublicationCommand); } } /* * GetReplicationConnection opens a new replication connection to this node. * This connection can be used to send replication commands, such as * CREATE_REPLICATION_SLOT. */ MultiConnection * GetReplicationConnection(char *nodeName, int nodePort) { int connectionFlags = FORCE_NEW_CONNECTION; connectionFlags |= REQUIRE_REPLICATION_CONNECTION_PARAM; MultiConnection *connection = GetNodeUserDatabaseConnection( connectionFlags, nodeName, nodePort, CitusExtensionOwnerName(), get_database_name(MyDatabaseId)); /* * Replication connections are special and don't support all of SQL, so we * don't want it to be used for other purposes what we create it for. */ ClaimConnectionExclusively(connection); return connection; } /* * CreateReplicationSlot creates a replication slot with the given slot name * over the given connection. The given connection should be a replication * connection. This function returns the name of the snapshot that is used for * this replication slot. When using this snapshot name for other transactions * you need to keep the given replication connection open until you have used * the snapshot name. */ static char * CreateReplicationSlot(MultiConnection *connection, char *slotname, char *outputPlugin) { StringInfo createReplicationSlotCommand = makeStringInfo(); appendStringInfo(createReplicationSlotCommand, "CREATE_REPLICATION_SLOT %s LOGICAL %s EXPORT_SNAPSHOT;", quote_identifier(slotname), quote_identifier(outputPlugin)); PGresult *result = NULL; int response = ExecuteOptionalRemoteCommand(connection, createReplicationSlotCommand->data, &result); if (response != RESPONSE_OKAY || !IsResponseOK(result) || PQntuples(result) != 1) { ReportResultError(connection, result, ERROR); } /*'snapshot_name' is second column where index starts from zero. * We're using the pstrdup to copy the data into the current memory context */ char *snapShotName = pstrdup(PQgetvalue(result, 0, 2 /* columIndex */)); PQclear(result); ForgetResults(connection); return snapShotName; } /* * CreateReplicationSlots creates the replication slots that the subscriptions * in the logicalRepTargetList can use. * * This function returns the snapshot name of the replication slots that are * used by the subscription. When using this snapshot name for other * transactions you need to keep the given replication connection open until * you are finished using the snapshot. */ char * CreateReplicationSlots(MultiConnection *sourceConnection, MultiConnection *sourceReplicationConnection, List *logicalRepTargetList, char *outputPlugin) { ReplicationSlotInfo *firstReplicationSlot = NULL; char *snapshot = NULL; LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ReplicationSlotInfo *replicationSlot = target->replicationSlot; WorkerNode *worker = FindWorkerNode(sourceConnection->hostname, sourceConnection->port); InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_REPLICATION_SLOT, replicationSlot->name, worker->groupId, CLEANUP_ALWAYS); if (!firstReplicationSlot) { firstReplicationSlot = replicationSlot; snapshot = CreateReplicationSlot( sourceReplicationConnection, replicationSlot->name, outputPlugin ); } else { ExecuteCriticalRemoteCommand( sourceConnection, psprintf("SELECT pg_catalog.pg_copy_logical_replication_slot(%s, %s)", quote_literal_cstr(firstReplicationSlot->name), quote_literal_cstr(replicationSlot->name))); } } return snapshot; } /* * CreateSubscriptions creates the subscriptions according to their definition * in the logicalRepTargetList. The remote node(s) needs to have appropriate * pg_dist_authinfo rows for the superuser such that the apply process can * connect. Because the generated CREATE SUBSCRIPTION statements use the host * and port names directly (rather than looking up any relevant * pg_dist_poolinfo rows), all such connections remain direct and will not * route through any configured poolers. * * The subscriptions created by this function are created in the disabled * state. This is done so a data copy can be done manually afterwards. To * enable the subscriptions you can use EnableSubscriptions(). */ void CreateSubscriptions(MultiConnection *sourceConnection, char *databaseName, List *logicalRepTargetList) { LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { int ownerId = target->tableOwnerId; WorkerNode *worker = FindWorkerNode(target->superuserConnection->hostname, target->superuserConnection->port); /* * The CREATE USER command should not propagate, so we temporarily * disable DDL propagation. * * Subscription workers have SUPERUSER permissions. Hence we temporarily * create a user with SUPERUSER permissions and then alter it to NOSUPERUSER. * This prevents permission escalations. */ SendCommandListToWorkerOutsideTransactionWithConnection( target->superuserConnection, list_make2( "SET LOCAL citus.enable_ddl_propagation TO OFF;", psprintf( "CREATE USER %s SUPERUSER IN ROLE %s;", quote_identifier(target->subscriptionOwnerName), quote_identifier(GetUserNameFromId(ownerId, false)) ))); InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_USER, target->subscriptionOwnerName, worker->groupId, CLEANUP_ALWAYS); StringInfo conninfo = makeStringInfo(); appendStringInfo(conninfo, "host='%s' port=%d user='%s' dbname='%s' " "connect_timeout=20", escape_param_str(sourceConnection->hostname), sourceConnection->port, escape_param_str(sourceConnection->user), escape_param_str( databaseName)); if (CpuPriorityLogicalRepSender != CPU_PRIORITY_INHERIT && list_length(logicalRepTargetList) <= MaxHighPriorityBackgroundProcesess) { appendStringInfo(conninfo, " options='-c citus.cpu_priority=%d'", CpuPriorityLogicalRepSender); } StringInfo createSubscriptionCommand = makeStringInfo(); appendStringInfo(createSubscriptionCommand, "CREATE SUBSCRIPTION %s CONNECTION %s PUBLICATION %s " "WITH (citus_use_authinfo=true, create_slot=false, " /* * password_required specifies whether connections to the publisher * made as a result of this subscription must use password authentication. * However, this setting is ignored when the subscription is owned * by a superuser. * Given that this command is executed below with superuser * ExecuteCriticalRemoteCommand(target->superuserConnection, * createSubscriptionCommand->data); * We are safe to pass password_required as false because * it will be ignored anyway */ "copy_data=false, enabled=false, slot_name=%s, password_required=false", quote_identifier(target->subscriptionName), quote_literal_cstr(conninfo->data), quote_identifier(target->publication->name), quote_identifier(target->replicationSlot->name)); if (EnableBinaryProtocol) { appendStringInfoString(createSubscriptionCommand, ", binary=true)"); } else { appendStringInfoString(createSubscriptionCommand, ")"); } ExecuteCriticalRemoteCommand(target->superuserConnection, createSubscriptionCommand->data); pfree(createSubscriptionCommand->data); pfree(createSubscriptionCommand); InsertCleanupRecordOutsideTransaction(CLEANUP_OBJECT_SUBSCRIPTION, target->subscriptionName, worker->groupId, CLEANUP_ALWAYS); ExecuteCriticalRemoteCommand(target->superuserConnection, psprintf( "ALTER SUBSCRIPTION %s OWNER TO %s", quote_identifier(target->subscriptionName), quote_identifier(target->subscriptionOwnerName) )); /* * The ALTER ROLE command should not propagate, so we temporarily * disable DDL propagation. */ SendCommandListToWorkerOutsideTransactionWithConnection( target->superuserConnection, list_make2( "SET LOCAL citus.enable_ddl_propagation TO OFF;", psprintf( "ALTER ROLE %s NOSUPERUSER;", quote_identifier(target->subscriptionOwnerName) ))); } } /* * EnableSubscriptions enables all the the subscriptions in the * logicalRepTargetList. This means the replication slot will start to be read * and the catchup phase begins. */ void EnableSubscriptions(List *logicalRepTargetList) { LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { ExecuteCriticalRemoteCommand(target->superuserConnection, psprintf( "ALTER SUBSCRIPTION %s ENABLE", target->subscriptionName )); } } /* *INDENT-OFF* */ /* * Escaping libpq connect parameter strings. * * Replaces "'" with "\'" and "\" with "\\". * * Copied from dblink.c to escape libpq params */ static char * escape_param_str(const char *str) { StringInfoData buf; initStringInfo(&buf); for (const char *cp = str; *cp; cp++) { if (*cp == '\\' || *cp == '\'') appendStringInfoChar(&buf, '\\'); appendStringInfoChar(&buf, *cp); } return buf.data; } /* *INDENT-ON* */ /* * GetRemoteLogPosition gets the current WAL log position over the given connection. */ XLogRecPtr GetRemoteLogPosition(MultiConnection *connection) { return GetRemoteLSN(connection, CURRENT_LOG_POSITION_COMMAND); } /* * GetRemoteLSN executes a command that returns a single LSN over the given connection * and returns it as an XLogRecPtr (uint64). */ static XLogRecPtr GetRemoteLSN(MultiConnection *connection, char *command) { bool raiseInterrupts = false; XLogRecPtr remoteLogPosition = InvalidXLogRecPtr; int querySent = SendRemoteCommand(connection, command); if (querySent == 0) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } int rowCount = PQntuples(result); if (rowCount != 1) { PQclear(result); ForgetResults(connection); return InvalidXLogRecPtr; } int colCount = PQnfields(result); if (colCount != 1) { ereport(ERROR, (errmsg("unexpected number of columns returned by: %s", command))); } if (!PQgetisnull(result, 0, 0)) { char *resultString = PQgetvalue(result, 0, 0); Datum remoteLogPositionDatum = DirectFunctionCall1Coll(pg_lsn_in, InvalidOid, CStringGetDatum( resultString)); remoteLogPosition = DatumGetLSN(remoteLogPositionDatum); } PQclear(result); ForgetResults(connection); return remoteLogPosition; } /* * CreateGroupedLogicalRepTargetsConnections creates connections for all of the nodes * in the groupedLogicalRepTargetsHash. */ void CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash, char *user, char *databaseName) { int connectionFlags = FORCE_NEW_CONNECTION; HASH_SEQ_STATUS status; GroupedLogicalRepTargets *groupedLogicalRepTargets = NULL; foreach_htab(groupedLogicalRepTargets, &status, groupedLogicalRepTargetsHash) { WorkerNode *targetWorkerNode = FindNodeWithNodeId( groupedLogicalRepTargets->nodeId, false); MultiConnection *superuserConnection = GetNodeUserDatabaseConnection(connectionFlags, targetWorkerNode->workerName, targetWorkerNode->workerPort, user, databaseName); /* * Operations on subscriptions cannot run in a transaction block. We * claim the connections exclusively to ensure they do not get used for * metadata syncing, which does open a transaction block. */ ClaimConnectionExclusively(superuserConnection); groupedLogicalRepTargets->superuserConnection = superuserConnection; LogicalRepTarget *target = NULL; foreach_declared_ptr(target, groupedLogicalRepTargets->logicalRepTargetList) { target->superuserConnection = superuserConnection; } } } /* * CreateGroupedLogicalRepTargetsConnections closes the connections for all of the * nodes in the groupedLogicalRepTargetsHash. */ void CloseGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash) { HASH_SEQ_STATUS status; GroupedLogicalRepTargets *groupedLogicalRepTargets = NULL; foreach_htab(groupedLogicalRepTargets, &status, groupedLogicalRepTargetsHash) { CloseConnection(groupedLogicalRepTargets->superuserConnection); } } /* * SubscriptionNamesValueList returns a SQL value list containing the * subscription names from the logicalRepTargetList. This value list can * be used in a query by using the IN operator. */ static char * SubscriptionNamesValueList(List *logicalRepTargetList) { StringInfo subscriptionValueList = makeStringInfo(); appendStringInfoString(subscriptionValueList, "("); bool first = true; LogicalRepTarget *target = NULL; foreach_declared_ptr(target, logicalRepTargetList) { if (!first) { appendStringInfoString(subscriptionValueList, ","); } else { first = false; } appendStringInfoString(subscriptionValueList, quote_literal_cstr( target->subscriptionName)); } appendStringInfoString(subscriptionValueList, ")"); return subscriptionValueList->data; } /* * WaitForAllSubscriptionToCatchUp waits until the last LSN reported by the * subscription. * * The function errors if the target LSN doesn't increase within * LogicalReplicationErrorTimeout. The function also reports its progress in * every logicalReplicationProgressReportTimeout. */ void WaitForAllSubscriptionsToCatchUp(MultiConnection *sourceConnection, HTAB *groupedLogicalRepTargetsHash) { XLogRecPtr sourcePosition = GetRemoteLogPosition(sourceConnection); HASH_SEQ_STATUS status; GroupedLogicalRepTargets *groupedLogicalRepTargets = NULL; foreach_htab(groupedLogicalRepTargets, &status, groupedLogicalRepTargetsHash) { WaitForGroupedLogicalRepTargetsToCatchUp(sourcePosition, groupedLogicalRepTargets); } } /* * WaitForNodeSubscriptionToCatchUp waits until the last LSN reported by the * subscription. * * The function errors if the target LSN doesn't increase within * LogicalReplicationErrorTimeout. The function also reports its progress in * every logicalReplicationProgressReportTimeout. */ static void WaitForGroupedLogicalRepTargetsToCatchUp(XLogRecPtr sourcePosition, GroupedLogicalRepTargets * groupedLogicalRepTargets) { XLogRecPtr previousTargetPosition = 0; TimestampTz previousLSNIncrementTime = GetCurrentTimestamp(); /* report in the first iteration as well */ TimestampTz previousReportTime = 0; MultiConnection *superuserConnection = groupedLogicalRepTargets->superuserConnection; /* * We might be in the loop for a while. Since we don't need to preserve * any memory beyond this function, we can simply switch to a child context * and reset it on every iteration to make sure we don't slowly build up * a lot of memory. */ MemoryContext loopContext = AllocSetContextCreateInternal(CurrentMemoryContext, "WaitForShardSubscriptionToCatchUp", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); MemoryContext oldContext = MemoryContextSwitchTo(loopContext); while (true) { XLogRecPtr targetPosition = GetSubscriptionPosition(groupedLogicalRepTargets); if (targetPosition >= sourcePosition) { ereport(LOG, (errmsg( "The LSN of the target subscriptions on node %s:%d have " "caught up with the source LSN ", superuserConnection->hostname, superuserConnection->port))); break; } /* * The following logic ensures that the subsription continues to grow withing * LogicalReplicationErrorTimeout duration. Otherwise, we error out since we * suspect that there is a problem on the target. It also handles the progess * reporting. */ if (targetPosition > previousTargetPosition) { /* variable is only used for the log message */ uint64 previousTargetBeforeThisLoop = previousTargetPosition; previousTargetPosition = targetPosition; previousLSNIncrementTime = GetCurrentTimestamp(); if (TimestampDifferenceExceeds(previousReportTime, GetCurrentTimestamp(), logicalReplicationProgressReportTimeout)) { ereport(LOG, (errmsg("The LSN of the target subscriptions on node %s:%d " "has increased from %X/%X to %X/%X at %s where the " "source LSN is %X/%X ", superuserConnection->hostname, superuserConnection->port, LSN_FORMAT_ARGS(previousTargetBeforeThisLoop), LSN_FORMAT_ARGS(targetPosition), timestamptz_to_str(previousLSNIncrementTime), LSN_FORMAT_ARGS(sourcePosition)))); previousReportTime = GetCurrentTimestamp(); } } else { if (TimestampDifferenceExceeds(previousLSNIncrementTime, GetCurrentTimestamp(), LogicalReplicationTimeout)) { ereport(ERROR, (errmsg("The logical replication waiting timeout " "of %d msec is exceeded", LogicalReplicationTimeout), errdetail("The LSN on the target subscription hasn't " "caught up ready on the target node %s:%d", superuserConnection->hostname, superuserConnection->port), errhint( "There might have occurred problems on the target " "node. If not consider using higher values for " "citus.logical_replication_error_timeout"))); } } /* sleep for 1 seconds (1000 miliseconds) and try again */ WaitForMiliseconds(1000); MemoryContextReset(loopContext); } MemoryContextSwitchTo(oldContext); } /* * WaitForMiliseconds waits for given timeout and then checks for some * interrupts. */ static void WaitForMiliseconds(long timeout) { int latchFlags = WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH; /* wait until timeout, or until somebody wakes us up */ int rc = WaitLatch(MyLatch, latchFlags, timeout, PG_WAIT_EXTENSION); /* emergency bailout if postmaster has died */ if (rc & WL_POSTMASTER_DEATH) { proc_exit(1); } if (rc & WL_LATCH_SET) { ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } if (ConfigReloadPending) { ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); } } /* * GetSubscriptionPosition gets the minimum WAL log position of the * subscription given subscriptions: That is the WAL log position on the source * node up to which the subscription completed replication. */ static XLogRecPtr GetSubscriptionPosition(GroupedLogicalRepTargets *groupedLogicalRepTargets) { char *subscriptionValueList = SubscriptionNamesValueList( groupedLogicalRepTargets->logicalRepTargetList); return GetRemoteLSN(groupedLogicalRepTargets->superuserConnection, psprintf( "SELECT min(latest_end_lsn) FROM pg_stat_subscription " "WHERE subname IN %s", subscriptionValueList)); } ================================================ FILE: src/backend/distributed/shardsplit/shardsplit_decoder.c ================================================ /*------------------------------------------------------------------------- * * shardsplit_decoder.c * Logical Replication output plugin * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_namespace.h" #include "replication/logical.h" #include "utils/lsyscache.h" #include "utils/typcache.h" #include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_shared_memory.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); static LogicalDecodeChangeCB pgOutputPluginChangeCB; #define InvalidRepOriginId 0 static HTAB *SourceToDestinationShardMap = NULL; static bool replication_origin_filter_cb(LogicalDecodingContext *ctx, RepOriginId origin_id); /* Plugin callback */ static void shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change); /* Helper methods */ static int32_t GetHashValueForIncomingTuple(Relation sourceShardRelation, HeapTuple tuple, int partitionColumIndex, Oid distributedTableOid); static Oid FindTargetRelationOid(Relation sourceShardRelation, HeapTuple tuple, char *currentSlotName); static HeapTuple GetTupleForTargetSchema(HeapTuple sourceRelationTuple, TupleDesc sourceTupleDesc, TupleDesc targetTupleDesc); /* * Postgres uses 'pgoutput' as default plugin for logical replication. * We want to reuse Postgres pgoutput's functionality as much as possible. * Hence we load all the functions of this plugin and override as required. */ void _PG_output_plugin_init(OutputPluginCallbacks *cb) { LogicalOutputPluginInit plugin_init = (LogicalOutputPluginInit) (void *) load_external_function("pgoutput", "_PG_output_plugin_init", false, NULL); if (plugin_init == NULL) { elog(ERROR, "output plugins have to declare the _PG_output_plugin_init symbol"); } /* ask the output plugin to fill the callback struct */ plugin_init(cb); /* actual pgoutput callback will be called with the appropriate destination shard */ pgOutputPluginChangeCB = cb->change_cb; cb->change_cb = shard_split_change_cb; cb->filter_by_origin_cb = replication_origin_filter_cb; } /* * replication_origin_filter_cb call back function filters out publication of changes * originated from any other node other than the current node. This is * identified by the "origin_id" of the changes. The origin_id is set to * a non-zero value in the origin node as part of WAL replication for internal * operations like shard split/moves/create_distributed_table etc. */ static bool replication_origin_filter_cb(LogicalDecodingContext *ctx, RepOriginId origin_id) { return (origin_id != InvalidRepOriginId); } /* * shard_split_change_cb function emits the incoming tuple change * to the appropriate destination shard. */ static void shard_split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change) { /* * If Citus has not been loaded yet, pass the changes * through to the undrelying decoder plugin. */ if (!CitusHasBeenLoaded()) { pgOutputPluginChangeCB(ctx, txn, relation, change); return; } /* check if the relation is publishable.*/ if (!is_publishable_relation(relation)) { return; } char *replicationSlotName = ctx->slot->data.name.data; if (replicationSlotName == NULL) { elog(ERROR, "Replication slot name is NULL!"); return; } /* * Initialize SourceToDestinationShardMap if not already initialized. * This gets initialized during the replication of first message. */ if (SourceToDestinationShardMap == NULL) { SourceToDestinationShardMap = PopulateSourceToDestinationShardMapForSlot( replicationSlotName, TopMemoryContext); } Oid targetRelationOid = InvalidOid; #if PG_VERSION_NUM >= PG_VERSION_17 switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { HeapTuple newTuple = change->data.tp.newtuple; targetRelationOid = FindTargetRelationOid(relation, newTuple, replicationSlotName); break; } /* updating non-partition column value */ case REORDER_BUFFER_CHANGE_UPDATE: { HeapTuple newTuple = change->data.tp.newtuple; targetRelationOid = FindTargetRelationOid(relation, newTuple, replicationSlotName); break; } case REORDER_BUFFER_CHANGE_DELETE: { HeapTuple oldTuple = change->data.tp.oldtuple; targetRelationOid = FindTargetRelationOid(relation, oldTuple, replicationSlotName); break; } /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ default: { ereport(ERROR, errmsg( "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } } #else switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { HeapTuple newTuple = &(change->data.tp.newtuple->tuple); targetRelationOid = FindTargetRelationOid(relation, newTuple, replicationSlotName); break; } /* updating non-partition column value */ case REORDER_BUFFER_CHANGE_UPDATE: { HeapTuple newTuple = &(change->data.tp.newtuple->tuple); targetRelationOid = FindTargetRelationOid(relation, newTuple, replicationSlotName); break; } case REORDER_BUFFER_CHANGE_DELETE: { HeapTuple oldTuple = &(change->data.tp.oldtuple->tuple); targetRelationOid = FindTargetRelationOid(relation, oldTuple, replicationSlotName); break; } /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ default: { ereport(ERROR, errmsg( "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } } #endif /* Current replication slot is not responsible for handling the change */ if (targetRelationOid == InvalidOid) { return; } Relation targetRelation = RelationIdGetRelation(targetRelationOid); /* * If any columns from source relation have been dropped, then the tuple needs to * be formatted according to the target relation. */ TupleDesc sourceRelationDesc = RelationGetDescr(relation); TupleDesc targetRelationDesc = RelationGetDescr(targetRelation); if (sourceRelationDesc->natts > targetRelationDesc->natts) { #if PG_VERSION_NUM >= PG_VERSION_17 switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple = targetRelationNewTuple; break; } case REORDER_BUFFER_CHANGE_UPDATE: { HeapTuple sourceRelationNewTuple = change->data.tp.newtuple; HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple = targetRelationNewTuple; /* * Format oldtuple according to the target relation. If the column values of replica * identiy change, then the old tuple is non-null and needs to be formatted according * to the target relation schema. */ if (change->data.tp.oldtuple != NULL) { HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple = targetRelationOldTuple; } break; } case REORDER_BUFFER_CHANGE_DELETE: { HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple; HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple = targetRelationOldTuple; break; } /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ default: { ereport(ERROR, errmsg( "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } } #else switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple); HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple->tuple = *targetRelationNewTuple; break; } case REORDER_BUFFER_CHANGE_UPDATE: { HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple); HeapTuple targetRelationNewTuple = GetTupleForTargetSchema( sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.newtuple->tuple = *targetRelationNewTuple; /* * Format oldtuple according to the target relation. If the column values of replica * identiy change, then the old tuple is non-null and needs to be formatted according * to the target relation schema. */ if (change->data.tp.oldtuple != NULL) { HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple); HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple->tuple = *targetRelationOldTuple; } break; } case REORDER_BUFFER_CHANGE_DELETE: { HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple); HeapTuple targetRelationOldTuple = GetTupleForTargetSchema( sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc); change->data.tp.oldtuple->tuple = *targetRelationOldTuple; break; } /* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */ default: { ereport(ERROR, errmsg( "Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE", change->action)); } } #endif } pgOutputPluginChangeCB(ctx, txn, targetRelation, change); RelationClose(targetRelation); } /* * FindTargetRelationOid returns the destination relation Oid for the incoming * tuple. * sourceShardRelation - Relation on which a commit has happened. * tuple - changed tuple. * currentSlotName - Name of replication slot that is processing this update. */ static Oid FindTargetRelationOid(Relation sourceShardRelation, HeapTuple tuple, char *currentSlotName) { Oid targetRelationOid = InvalidOid; Oid sourceShardRelationOid = sourceShardRelation->rd_id; /* Get child shard list for source(parent) shard from hashmap*/ bool found = false; SourceToDestinationShardMapEntry *entry = (SourceToDestinationShardMapEntry *) hash_search( SourceToDestinationShardMap, &sourceShardRelationOid, HASH_FIND, &found); /* * Source shard Oid might not exist in the hash map. This can happen * in below cases: * 1) The commit can belong to any other table that is not under going split. * 2) The commit can be recursive in nature. When the source shard * receives a commit(a), the WAL sender processes this commit message. This * commit is applied to a child shard which is placed on the same node as a * part of replication. This in turn creates one more commit(b) which is recursive in nature. * Commit 'b' should be skipped as the source shard and destination for commit 'b' * are same and the commit has already been applied. */ if (!found) { return InvalidOid; } ShardSplitInfo *shardSplitInfo = (ShardSplitInfo *) lfirst(list_head( entry-> shardSplitInfoList)); int hashValue = GetHashValueForIncomingTuple(sourceShardRelation, tuple, shardSplitInfo->partitionColumnIndex, shardSplitInfo->distributedTableOid); shardSplitInfo = NULL; foreach_declared_ptr(shardSplitInfo, entry->shardSplitInfoList) { if (shardSplitInfo->shardMinValue <= hashValue && shardSplitInfo->shardMaxValue >= hashValue) { targetRelationOid = shardSplitInfo->splitChildShardOid; break; } } return targetRelationOid; } /* * GetHashValueForIncomingTuple returns the hash value of the partition * column for the incoming tuple. */ static int32_t GetHashValueForIncomingTuple(Relation sourceShardRelation, HeapTuple tuple, int partitionColumnIndex, Oid distributedTableOid) { TupleDesc relationTupleDes = RelationGetDescr(sourceShardRelation); Form_pg_attribute partitionColumn = TupleDescAttr(relationTupleDes, partitionColumnIndex); bool isNull = false; Datum partitionColumnValue = heap_getattr(tuple, partitionColumnIndex + 1, relationTupleDes, &isNull); TypeCacheEntry *typeEntry = lookup_type_cache(partitionColumn->atttypid, TYPECACHE_HASH_PROC_FINFO); /* get hashed value of the distribution value */ Datum hashedValueDatum = FunctionCall1Coll(&(typeEntry->hash_proc_finfo), typeEntry->typcollation, partitionColumnValue); return DatumGetInt32(hashedValueDatum); } /* * GetTupleForTargetSchema returns a tuple with the schema of the target relation. * If some columns within the source relations are dropped, we would have to reformat * the tuple to match the schema of the target relation. * * Consider the below scenario: * Session1 : Drop column followed by create_distributed_table_concurrently * Session2 : Concurrent insert workload * * The child shards created by create_distributed_table_concurrently will have less columns * than the source shard because some column were dropped. * The incoming tuple from session2 will have more columns as the writes * happened on source shard. But now the tuple needs to be applied on child shard. So we need to format * it according to child schema. */ static HeapTuple GetTupleForTargetSchema(HeapTuple sourceRelationTuple, TupleDesc sourceRelDesc, TupleDesc targetRelDesc) { /* Deform the tuple */ Datum *oldValues = (Datum *) palloc0(sourceRelDesc->natts * sizeof(Datum)); bool *oldNulls = (bool *) palloc0(sourceRelDesc->natts * sizeof(bool)); heap_deform_tuple(sourceRelationTuple, sourceRelDesc, oldValues, oldNulls); /* Create new tuple by skipping dropped columns */ int nextAttributeIndex = 0; Datum *newValues = (Datum *) palloc0(targetRelDesc->natts * sizeof(Datum)); bool *newNulls = (bool *) palloc0(targetRelDesc->natts * sizeof(bool)); for (int i = 0; i < sourceRelDesc->natts; i++) { if (TupleDescAttr(sourceRelDesc, i)->attisdropped) { continue; } newValues[nextAttributeIndex] = oldValues[i]; newNulls[nextAttributeIndex] = oldNulls[i]; nextAttributeIndex++; } HeapTuple targetRelationTuple = heap_form_tuple(targetRelDesc, newValues, newNulls); return targetRelationTuple; } ================================================ FILE: src/backend/distributed/shardsplit/shardsplit_logical_replication.c ================================================ /*------------------------------------------------------------------------- * * shardsplit_logical_replication.c * * Function definitions for logically replicating shard to split children. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "commands/dbcommands.h" #include "nodes/pg_list.h" #include "utils/builtins.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/priority.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_split.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_logical_replication.h" #include "distributed/shared_library_init.h" static HTAB *ShardInfoHashMapForPublications = NULL; /* function declarations */ static void AddPublishableShardEntryInMap(uint32 targetNodeId, ShardInterval *shardInterval, bool isChildShardInterval); static LogicalRepTarget * CreateLogicalRepTarget(Oid tableOwnerId, uint32 nodeId, List *replicationSlotInfoList); /* * CreateShardSplitInfoMapForPublication creates a hashmap that groups * shards for creating publications and subscriptions. * * While creating publications and subscriptions, apart from table owners, * placement of child shard matters too. To further understand this, please see * the following example: * * Shard1(on Worker1) is to be split in Shard2 and Shard3 on Worker2 and Worker3 respectively. * Lets assume the owner to be 'A'. The hashmap groups shard list in the following way. * * Map key * ======= ------ ------ * ------> |Shard2|-->|Shard1| * ------ ------ * * ------ ------ * ------> |Shard3|-->|Shard1| * ------ ------ * Shard1 is a dummy table that is to be created on Worker2 and Worker3. * Based on the above placement, we would need to create two publications on the source node. */ HTAB * CreateShardSplitInfoMapForPublication(List *sourceColocatedShardIntervalList, List *shardGroupSplitIntervalListList, List *destinationWorkerNodesList) { ShardInfoHashMapForPublications = CreateSimpleHash(NodeAndOwner, PublicationInfo); ShardInterval *sourceShardIntervalToCopy = NULL; List *splitChildShardIntervalList = NULL; forboth_ptr(sourceShardIntervalToCopy, sourceColocatedShardIntervalList, splitChildShardIntervalList, shardGroupSplitIntervalListList) { /* * Skipping partitioned table for logical replication. * Since PG13, logical replication is supported for partitioned tables. * However, we want to keep the behaviour consistent with shard moves. */ if (PartitionedTable(sourceShardIntervalToCopy->relationId)) { continue; } ShardInterval *splitChildShardInterval = NULL; WorkerNode *destinationWorkerNode = NULL; forboth_ptr(splitChildShardInterval, splitChildShardIntervalList, destinationWorkerNode, destinationWorkerNodesList) { uint32 destinationWorkerNodeId = destinationWorkerNode->nodeId; /* Add child shard for publication. * If a columnar shard is a part of publications, then writes on the shard fail. * In the case of local split, adding child shards to the publication * would prevent copying the initial data done through 'DoSplitCopy'. * Hence we avoid adding columnar child shards to publication. */ if (!extern_IsColumnarTableAmTable(splitChildShardInterval->relationId)) { AddPublishableShardEntryInMap(destinationWorkerNodeId, splitChildShardInterval, true /*isChildShardInterval*/); } /* Add parent shard if not already added */ AddPublishableShardEntryInMap(destinationWorkerNodeId, sourceShardIntervalToCopy, false /*isChildShardInterval*/); } } return ShardInfoHashMapForPublications; } /* * AddPublishableShardEntryInMap adds a shard interval in the list * of shards to be published. */ static void AddPublishableShardEntryInMap(uint32 targetNodeId, ShardInterval *shardInterval, bool isChildShardInterval) { NodeAndOwner key; key.nodeId = targetNodeId; key.tableOwnerId = TableOwnerOid(shardInterval->relationId); bool found = false; PublicationInfo *publicationInfo = (PublicationInfo *) hash_search(ShardInfoHashMapForPublications, &key, HASH_ENTER, &found); /* Create a new list for pair */ if (!found) { publicationInfo->shardIntervals = NIL; publicationInfo->name = PublicationName(SHARD_SPLIT, key.nodeId, key.tableOwnerId); } /* Add child shard interval */ if (isChildShardInterval) { publicationInfo->shardIntervals = lappend(publicationInfo->shardIntervals, shardInterval); /* We return from here as the child interval is only added once in the list */ return; } /* Check if parent is already added */ ShardInterval *existingShardInterval = NULL; foreach_declared_ptr(existingShardInterval, publicationInfo->shardIntervals) { if (existingShardInterval->shardId == shardInterval->shardId) { /* parent shard interval is already added hence return */ return; } } /* Add parent shard Interval */ publicationInfo->shardIntervals = lappend(publicationInfo->shardIntervals, shardInterval); } /* * PopulateShardSplitSubscriptionsMetadataList returns a list of 'LogicalRepTarget' * structure. * * shardSplitInfoHashMap - Shards are grouped by key. * For each key, we create a metadata structure. This facilitates easy * publication-subscription management. * * replicationSlotInfoList - List of replication slot info. */ List * PopulateShardSplitSubscriptionsMetadataList(HTAB *shardSplitInfoHashMap, List *replicationSlotInfoList, List *shardGroupSplitIntervalListList, List *workersForPlacementList) { HASH_SEQ_STATUS status; hash_seq_init(&status, shardSplitInfoHashMap); PublicationInfo *publication = NULL; List *logicalRepTargetList = NIL; while ((publication = (PublicationInfo *) hash_seq_search(&status)) != NULL) { uint32 nodeId = publication->key.nodeId; uint32 tableOwnerId = publication->key.tableOwnerId; LogicalRepTarget *target = CreateLogicalRepTarget(tableOwnerId, nodeId, replicationSlotInfoList); target->publication = publication; publication->target = target; logicalRepTargetList = lappend(logicalRepTargetList, target); } List *shardIntervalList = NIL; foreach_declared_ptr(shardIntervalList, shardGroupSplitIntervalListList) { ShardInterval *shardInterval = NULL; WorkerNode *workerPlacementNode = NULL; forboth_ptr(shardInterval, shardIntervalList, workerPlacementNode, workersForPlacementList) { NodeAndOwner key; key.nodeId = workerPlacementNode->nodeId; key.tableOwnerId = TableOwnerOid(shardInterval->relationId); bool found = false; publication = (PublicationInfo *) hash_search( ShardInfoHashMapForPublications, &key, HASH_FIND, &found); if (!found) { ereport(ERROR, errmsg("Could not find publication matching a split")); } publication->target->newShards = lappend( publication->target->newShards, shardInterval); } } return logicalRepTargetList; } /* * Creates a 'LogicalRepTarget' structure for given table owner, node id. * It scans the list of 'ReplicationSlotInfo' to identify the corresponding slot * to be used for given tableOwnerId and nodeId. */ static LogicalRepTarget * CreateLogicalRepTarget(Oid tableOwnerId, uint32 nodeId, List *replicationSlotInfoList) { LogicalRepTarget *target = palloc0(sizeof(LogicalRepTarget)); target->subscriptionName = SubscriptionName(SHARD_SPLIT, tableOwnerId); target->tableOwnerId = tableOwnerId; target->subscriptionOwnerName = SubscriptionRoleName(SHARD_SPLIT, tableOwnerId); target->superuserConnection = NULL; /* * Each 'ReplicationSlotInfo' belongs to a unique combination of node id and owner. * Traverse the slot list to identify the corresponding slot for given * table owner and node. */ ReplicationSlotInfo *replicationSlot = NULL; foreach_declared_ptr(replicationSlot, replicationSlotInfoList) { if (nodeId == replicationSlot->targetNodeId && tableOwnerId == replicationSlot->tableOwnerId) { target->replicationSlot = replicationSlot; break; } } if (!target->replicationSlot) { ereport(ERROR, errmsg( "Could not find replication slot matching a subscription %s", target->subscriptionName)); } return target; } ================================================ FILE: src/backend/distributed/shardsplit/shardsplit_shared_memory.c ================================================ /*------------------------------------------------------------------------- * * shardsplit_shared_memory.c * API's for creating and accessing shared memory segments to store * shard split information. 'setup_shard_replication' UDF creates the * shared memory, populates the contents and WAL sender processes are * the consumers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/hashfn.h" #include "storage/ipc.h" #include "utils/memutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/multi_logical_replication.h" #include "distributed/shardinterval_utils.h" #include "distributed/shardsplit_shared_memory.h" const char *SharedMemoryNameForHandleManagement = "Shared memory handle for shard split"; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* Function declarations */ static ShardSplitInfoSMHeader * AllocateSharedMemoryForShardSplitInfo(int shardSplitInfoCount, Size shardSplitInfoSize, dsm_handle * dsmHandle); static ShardSplitInfoSMHeader * GetShardSplitInfoSMHeaderFromDSMHandle(dsm_handle dsmHandle); static dsm_handle GetShardSplitSharedMemoryHandle(void); static void ShardSplitShmemInit(void); /* * GetShardSplitInfoSMHeaderFromDSMHandle returns the header of the shared memory * segment. It pins the mapping till lifetime of the backend process accessing it. */ static ShardSplitInfoSMHeader * GetShardSplitInfoSMHeaderFromDSMHandle(dsm_handle dsmHandle) { dsm_segment *dsmSegment = dsm_find_mapping(dsmHandle); if (dsmSegment == NULL) { dsmSegment = dsm_attach(dsmHandle); } if (dsmSegment == NULL) { ereport(ERROR, (errmsg("could not attach to dynamic shared memory segment " "corresponding to handle:%u", dsmHandle))); } /* * Detatching segment associated with resource owner with 'dsm_pin_mapping' call before the * resource owner releases, to avoid warning being logged and potential leaks. */ dsm_pin_mapping(dsmSegment); ShardSplitInfoSMHeader *header = (ShardSplitInfoSMHeader *) dsm_segment_address( dsmSegment); if (header == NULL) { ereport(ERROR, (errmsg("Could not get shared memory segment header " "corresponding to handle for split workflow:%u", dsmHandle))); } return header; } /* * GetShardSplitInfoSMHeader returns pointer to the header of shared memory segment. */ ShardSplitInfoSMHeader * GetShardSplitInfoSMHeader() { dsm_handle dsmHandle = GetShardSplitSharedMemoryHandle(); ShardSplitInfoSMHeader *shardSplitInfoSMHeader = GetShardSplitInfoSMHeaderFromDSMHandle(dsmHandle); return shardSplitInfoSMHeader; } /* * AllocateSharedMemoryForShardSplitInfo is used to allocate and store * information about the shard undergoing a split. The function allocates dynamic * shared memory segment consisting of a header and an array of ShardSplitInfo structure. * The contents of this shared memory segment are consumed by WAL sender process * during catch up phase of replication through logical decoding plugin. * * The shared memory segment exists till the catch up phase completes or the * postmaster shutsdown. */ static ShardSplitInfoSMHeader * AllocateSharedMemoryForShardSplitInfo(int shardSplitInfoCount, Size shardSplitInfoSize, dsm_handle *dsmHandle) { if (shardSplitInfoCount <= 0 || shardSplitInfoSize <= 0) { ereport(ERROR, (errmsg("shardSplitInfoCount and size of each step should be " "positive values"))); } Size totalSize = offsetof(ShardSplitInfoSMHeader, splitInfoArray) + (shardSplitInfoCount * shardSplitInfoSize); dsm_segment *dsmSegment = dsm_create(totalSize, DSM_CREATE_NULL_IF_MAXSEGMENTS); if (dsmSegment == NULL) { ereport(ERROR, (errmsg("could not create a dynamic shared memory segment to " "store shard split info"))); } *dsmHandle = dsm_segment_handle(dsmSegment); /* * Pin the segment till Postmaster shutsdown since we need this * segment even after the session ends for replication catchup phase. */ dsm_pin_segment(dsmSegment); ShardSplitInfoSMHeader *shardSplitInfoSMHeader = GetShardSplitInfoSMHeaderFromDSMHandle(*dsmHandle); shardSplitInfoSMHeader->count = shardSplitInfoCount; return shardSplitInfoSMHeader; } /* * CreateSharedMemoryForShardSplitInfo is a wrapper function which creates shared memory * for storing shard split infomation. The function returns pointer to the header of * shared memory segment. * * shardSplitInfoCount - number of 'ShardSplitInfo ' elements to be allocated * dsmHandle - handle of the allocated shared memory segment */ ShardSplitInfoSMHeader * CreateSharedMemoryForShardSplitInfo(int shardSplitInfoCount, dsm_handle *dsmHandle) { ShardSplitInfoSMHeader *shardSplitInfoSMHeader = AllocateSharedMemoryForShardSplitInfo(shardSplitInfoCount, sizeof(ShardSplitInfo), dsmHandle); return shardSplitInfoSMHeader; } /* * ReleaseSharedMemoryOfShardSplitInfo releases(unpins) the dynamic shared memory segment * allocated by 'worker_split_shard_replication_setup'. This shared memory was pinned * to Postmaster process and is valid till Postmaster shutsdown or * explicitly unpinned by calling 'dsm_unpin_segment'. */ void ReleaseSharedMemoryOfShardSplitInfo() { /* Get handle of dynamic shared memory segment*/ dsm_handle dsmHandle = GetShardSplitSharedMemoryHandle(); if (dsmHandle == DSM_HANDLE_INVALID) { return; } /* * Unpin the dynamic shared memory segment. 'dsm_pin_segment' was * called previously by 'AllocateSharedMemoryForShardSplitInfo'. */ dsm_unpin_segment(dsmHandle); /* * As dynamic shared memory is unpinned, store an invalid handle in static * shared memory used for handle management. */ StoreShardSplitSharedMemoryHandle(DSM_HANDLE_INVALID); } /* * InitializeShardSplitSMHandleManagement requests the necessary shared memory * from Postgres and sets up the shared memory startup hook. * This memory is used to store handle of other shared memories allocated during split workflow. */ void InitializeShardSplitSMHandleManagement(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = ShardSplitShmemInit; } static void ShardSplitShmemInit(void) { bool alreadyInitialized = false; ShardSplitShmemData *smData = ShmemInitStruct(SharedMemoryNameForHandleManagement, sizeof(ShardSplitShmemData), &alreadyInitialized); if (!alreadyInitialized) { char *trancheName = "Split Shard Setup Tranche"; NamedLWLockTranche *namedLockTranche = &smData->namedLockTranche; /* start by zeroing out all the memory */ memset(smData, 0, sizeof(ShardSplitShmemData)); namedLockTranche->trancheId = LWLockNewTrancheId(); LWLockRegisterTranche(namedLockTranche->trancheId, trancheName); LWLockInitialize(&smData->lock, namedLockTranche->trancheId); smData->dsmHandle = DSM_HANDLE_INVALID; } if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * StoreShardSplitSharedMemoryHandle stores a handle of shared memory * allocated and populated by 'worker_split_shard_replication_setup' UDF. * This handle is stored in a different statically allocated shared memory * segment with name 'Shared memory handle for shard split'. */ void StoreShardSplitSharedMemoryHandle(dsm_handle dsmHandle) { bool found = false; ShardSplitShmemData *smData = ShmemInitStruct(SharedMemoryNameForHandleManagement, sizeof(ShardSplitShmemData), &found); if (!found) { ereport(ERROR, errmsg( "Shared memory for handle management should have been initialized during boot")); } /* * We only support non concurrent split. However, it is fine to take a * lock and store the handle incase concurrent splits are introduced in future. */ LWLockAcquire(&smData->lock, LW_EXCLUSIVE); /* * In a normal situation, previously stored handle should have been invalidated * before the current function is called. * If this handle is still valid, it means cleanup of previous split shard * workflow failed. Log a waring and continue the current shard split operation. * Skip warning if new handle to be stored is invalid. We store invalid handle * when shared memory is released by calling worker_split_shard_release_dsm. */ if (smData->dsmHandle != DSM_HANDLE_INVALID && dsmHandle != DSM_HANDLE_INVALID) { ereport(WARNING, errmsg( "Previous split shard worflow was not successfully and could not complete the cleanup phase." " Continuing with the current split shard workflow.")); } /* Store the incoming handle */ smData->dsmHandle = dsmHandle; LWLockRelease(&smData->lock); } /* * GetShardSplitSharedMemoryHandle returns the handle of dynamic shared memory segment stored * by 'worker_split_shard_replication_setup' UDF. This handle is requested by WAL sender processes * during logical replication phase or during cleanup. */ dsm_handle GetShardSplitSharedMemoryHandle(void) { bool found = false; ShardSplitShmemData *smData = ShmemInitStruct(SharedMemoryNameForHandleManagement, sizeof(ShardSplitShmemData), &found); if (!found) { ereport(ERROR, errmsg( "Shared memory for handle management should have been initialized during boot")); } LWLockAcquire(&smData->lock, LW_SHARED); dsm_handle dsmHandle = smData->dsmHandle; LWLockRelease(&smData->lock); return dsmHandle; } /* * PopulateSourceToDestinationShardMapForSlot populates 'SourceToDestinationShard' hash map for a given slot. * Key of the map is Oid of source shard which is undergoing a split and value is a list of corresponding child shards. * To populate the map, the function traverses 'ShardSplitInfo' array stored within shared memory segment. */ HTAB * PopulateSourceToDestinationShardMapForSlot(char *slotName, MemoryContext cxt) { HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(Oid); info.entrysize = sizeof(SourceToDestinationShardMapEntry); info.hash = uint32_hash; info.hcxt = cxt; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION); HTAB *sourceShardToDesShardMap = hash_create("SourceToDestinationShardMap", 128, &info, hashFlags); MemoryContext oldContext = MemoryContextSwitchTo(cxt); ShardSplitInfoSMHeader *smHeader = GetShardSplitInfoSMHeader(); for (int index = 0; index < smHeader->count; index++) { if (strcmp(smHeader->splitInfoArray[index].slotName, slotName) == 0) { Oid sourceShardOid = smHeader->splitInfoArray[index].sourceShardOid; bool found = false; SourceToDestinationShardMapEntry *entry = (SourceToDestinationShardMapEntry *) hash_search( sourceShardToDesShardMap, &sourceShardOid, HASH_ENTER, &found); if (!found) { entry->shardSplitInfoList = NIL; entry->sourceShardKey = sourceShardOid; } ShardSplitInfo *shardSplitInfoForSlot = (ShardSplitInfo *) palloc0( sizeof(ShardSplitInfo)); *shardSplitInfoForSlot = smHeader->splitInfoArray[index]; entry->shardSplitInfoList = lappend(entry->shardSplitInfoList, (ShardSplitInfo *) shardSplitInfoForSlot); } } MemoryContextSwitchTo(oldContext); return sourceShardToDesShardMap; } ================================================ FILE: src/backend/distributed/shared_library_init.c ================================================ /*------------------------------------------------------------------------- * * shared_library_init.c * Functionality related to the initialization of the Citus extension. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include #include #include "postgres.h" /* necessary to get alloca on illumos */ #ifdef __sun #include #endif #include "fmgr.h" #include "miscadmin.h" #include "safe_lib.h" #include "catalog/objectaccess.h" #include "catalog/pg_authid.h" #include "catalog/pg_extension.h" #include "commands/explain.h" #include "commands/extension.h" #include "commands/seclabel.h" #include "common/string.h" #include "executor/executor.h" #include "libpq/auth.h" #include "optimizer/paths.h" #include "optimizer/plancat.h" #include "optimizer/planner.h" #include "port/atomics.h" #include "postmaster/postmaster.h" #include "replication/walsender.h" #include "storage/ipc.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/guc_tables.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "citus_version.h" #include "columnar/columnar.h" #include "distributed/adaptive_executor.h" #include "distributed/backend_data.h" #include "distributed/background_jobs.h" #include "distributed/causal_clock.h" #include "distributed/citus_depended_object.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_safe_lib.h" #include "distributed/combine_query_planner.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/cte_inline.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/local_distributed_join_planner.h" #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/log_utils.h" #include "distributed/maintenanced.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_logical_replication.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/pg_dist_partition.h" #include "distributed/placement_connection.h" #include "distributed/priority.h" #include "distributed/query_pushdown_planning.h" #include "distributed/recursive_planning.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/remote_transaction.h" #include "distributed/repartition_executor.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/resource_lock.h" #include "distributed/run_from_same_connection.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" #include "distributed/shard_transfer.h" #include "distributed/shardsplit_shared_memory.h" #include "distributed/shared_connection_stats.h" #include "distributed/shared_library_init.h" #include "distributed/stats/query_stats.h" #include "distributed/stats/stat_counters.h" #include "distributed/stats/stat_tenants.h" #include "distributed/subplan_execution.h" #include "distributed/time_constants.h" #include "distributed/transaction_management.h" #include "distributed/transaction_recovery.h" #include "distributed/utils/directory.h" #include "distributed/worker_log_messages.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" /* marks shared object as one loadable by the postgres version compiled against */ #if PG_VERSION_NUM >= PG_VERSION_18 PG_MODULE_MAGIC_EXT(.name = "citus", .version = "15.0devel"); #else PG_MODULE_MAGIC; #endif ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM = NULL; CompressionTypeStr_type extern_CompressionTypeStr = NULL; IsColumnarTableAmTable_type extern_IsColumnarTableAmTable = NULL; ReadColumnarOptions_type extern_ReadColumnarOptions = NULL; /* * Define "pass-through" functions so that a SQL function defined as one of * these symbols in the citus module can use the definition in the columnar * module. */ #define DEFINE_COLUMNAR_PASSTHROUGH_FUNC(funcname) \ static PGFunction CppConcat(extern_, funcname); \ PG_FUNCTION_INFO_V1(funcname); \ Datum funcname(PG_FUNCTION_ARGS) \ { \ return CppConcat(extern_, funcname)(fcinfo); \ } #define INIT_COLUMNAR_SYMBOL(typename, funcname) \ CppConcat(extern_, funcname) = \ (typename) (void *) lookup_external_function(handle, # funcname) #define CDC_DECODER_DYNAMIC_LIB_PATH "$libdir/citus_decoders:$libdir" DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_handler) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_set) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(alter_columnar_table_reset) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(upgrade_columnar_storage) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(downgrade_columnar_storage) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_relation_storageid) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_storage_info) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(columnar_store_memory_stats) DEFINE_COLUMNAR_PASSTHROUGH_FUNC(test_columnar_storage_write_new_page) #define DUMMY_REAL_TIME_EXECUTOR_ENUM_VALUE 9999999 static char *CitusVersion = CITUS_VERSION; static char *DeprecatedEmptyString = ""; static char *MitmfifoEmptyString = ""; static bool DeprecatedDeferShardDeleteOnMove = true; static bool DeprecatedDeferShardDeleteOnSplit = true; static bool DeprecatedReplicateReferenceTablesOnActivate = false; static bool DeprecatedEnableStatisticsCollection = false; /* deprecated GUC value that should not be used anywhere outside this file */ static int ReplicationModel = REPLICATION_MODEL_STREAMING; /* we override the application_name assign_hook and keep a pointer to the old one */ static GucStringAssignHook OldApplicationNameAssignHook = NULL; /* * Flag to indicate when ApplicationNameAssignHook becomes responsible for * updating the global pid. */ static bool FinishedStartupCitusBackend = false; static object_access_hook_type PrevObjectAccessHook = NULL; static shmem_request_hook_type prev_shmem_request_hook = NULL; void _PG_init(void); static void citus_shmem_request(void); static void CitusObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg); static void DoInitialCleanup(void); static void ResizeStackToMaximumDepth(void); static void multi_log_hook(ErrorData *edata); static bool IsSequenceOverflowError(ErrorData *edata); static void RegisterConnectionCleanup(void); static void RegisterSaveBackendStatsIntoSavedBackendStatsHash(void); static void RegisterExternalClientBackendCounterDecrement(void); static void CitusCleanupConnectionsAtExit(int code, Datum arg); static void SaveBackendStatsIntoSavedBackendStatsHashAtExit(int code, Datum arg); static void DecrementExternalClientBackendCounterAtExit(int code, Datum arg); static void CreateRequiredDirectories(void); static void RegisterCitusConfigVariables(void); static void OverridePostgresConfigProperties(void); static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra, GucSource source); static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source); static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source); static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source); static bool ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source); static void ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra); static void ApplicationNameAssignHook(const char *newval, void *extra); static void CpuPriorityAssignHook(int newval, void *extra); static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source); static void NodeConninfoGucAssignHook(const char *newval, void *extra); static const char * MaxSharedPoolSizeGucShowHook(void); static const char * LocalPoolSizeGucShowHook(void); static bool WarnIfLocalExecutionDisabled(bool *newval, void **extra, GucSource source); static void CitusAuthHook(Port *port, int status); static bool IsSuperuser(char *userName); static void AdjustDynamicLibraryPathForCdcDecoders(void); static void EnableChangeDataCaptureAssignHook(bool newval, void *extra); static ClientAuthentication_hook_type original_client_auth_hook = NULL; static emit_log_hook_type original_emit_log_hook = NULL; /* *INDENT-OFF* */ /* GUC enum definitions */ static const struct config_enum_entry propagate_set_commands_options[] = { {"none", PROPSETCMD_NONE, false}, {"local", PROPSETCMD_LOCAL, false}, {NULL, 0, false} }; static const struct config_enum_entry stat_statements_track_options[] = { { "none", STAT_STATEMENTS_TRACK_NONE, false }, { "all", STAT_STATEMENTS_TRACK_ALL, false }, { NULL, 0, false } }; static const struct config_enum_entry stat_tenants_track_options[] = { { "none", STAT_TENANTS_TRACK_NONE, false }, { "all", STAT_TENANTS_TRACK_ALL, false }, { NULL, 0, false } }; static const struct config_enum_entry task_assignment_policy_options[] = { { "greedy", TASK_ASSIGNMENT_GREEDY, false }, { "first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false }, { "round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false }, { NULL, 0, false } }; static const struct config_enum_entry replication_model_options[] = { { "statement", REPLICATION_MODEL_COORDINATOR, false }, { "streaming", REPLICATION_MODEL_STREAMING, false }, { NULL, 0, false } }; static const struct config_enum_entry task_executor_type_options[] = { { "adaptive", MULTI_EXECUTOR_ADAPTIVE, false }, { "real-time", DUMMY_REAL_TIME_EXECUTOR_ENUM_VALUE, false }, /* keep it for backward comp. */ { "task-tracker", MULTI_EXECUTOR_ADAPTIVE, false }, { NULL, 0, false } }; static const struct config_enum_entry use_secondary_nodes_options[] = { { "never", USE_SECONDARY_NODES_NEVER, false }, { "always", USE_SECONDARY_NODES_ALWAYS, false }, { NULL, 0, false } }; static const struct config_enum_entry coordinator_aggregation_options[] = { { "disabled", COORDINATOR_AGGREGATION_DISABLED, false }, { "row-gather", COORDINATOR_AGGREGATION_ROW_GATHER, false }, { NULL, 0, false } }; static const struct config_enum_entry log_level_options[] = { { "off", CITUS_LOG_LEVEL_OFF, false }, { "debug5", DEBUG5, false}, { "debug4", DEBUG4, false}, { "debug3", DEBUG3, false}, { "debug2", DEBUG2, false}, { "debug1", DEBUG1, false}, { "debug", DEBUG2, true}, { "log", LOG, false}, { "info", INFO, true}, { "notice", NOTICE, false}, { "warning", WARNING, false}, { "error", ERROR, false}, { NULL, 0, false} }; static const struct config_enum_entry local_table_join_policies[] = { { "never", LOCAL_JOIN_POLICY_NEVER, false}, { "prefer-local", LOCAL_JOIN_POLICY_PREFER_LOCAL, false}, { "prefer-distributed", LOCAL_JOIN_POLICY_PREFER_DISTRIBUTED, false}, { "auto", LOCAL_JOIN_POLICY_AUTO, false}, { NULL, 0, false} }; static const struct config_enum_entry multi_shard_modify_connection_options[] = { { "parallel", PARALLEL_CONNECTION, false }, { "sequential", SEQUENTIAL_CONNECTION, false }, { NULL, 0, false } }; static const struct config_enum_entry explain_analyze_sort_method_options[] = { { "execution-time", EXPLAIN_ANALYZE_SORT_BY_TIME, false }, { "taskId", EXPLAIN_ANALYZE_SORT_BY_TASK_ID, false }, { NULL, 0, false } }; static const struct config_enum_entry create_object_propagation_options[] = { {"deferred", CREATE_OBJECT_PROPAGATION_DEFERRED, false}, {"automatic", CREATE_OBJECT_PROPAGATION_AUTOMATIC, false}, {"immediate", CREATE_OBJECT_PROPAGATION_IMMEDIATE, false}, {NULL, 0, false} }; /* * This used to choose CPU priorities for GUCs. For most other integer options * we use the -1 value as inherit/default/unset. For CPU priorities this isn't * possible, because they can actually have negative values. So we need a value * outside of the range that's valid for priorities. But if this is only one * more or less than the valid values, this can also be quite confusing for * people that don't know the exact range of valid values. * * So, instead we opt for using an enum that contains all valid priority values * as strings, as well as the "inherit" string to indicate that the priority * value should not be changed. */ static const struct config_enum_entry cpu_priority_options[] = { { "inherit", CPU_PRIORITY_INHERIT, false }, { "-20", -20, false}, { "-19", -19, false}, { "-18", -18, false}, { "-17", -17, false}, { "-16", -16, false}, { "-15", -15, false}, { "-14", -14, false}, { "-13", -13, false}, { "-12", -12, false}, { "-11", -11, false}, { "-10", -10, false}, { "-9", -9, false}, { "-8", -8, false}, { "-7", -7, false}, { "-6", -6, false}, { "-5", -5, false}, { "-4", -4, false}, { "-3", -3, false}, { "-2", -2, false}, { "-1", -1, false}, { "0", 0, false}, { "1", 1, false}, { "2", 2, false}, { "3", 3, false}, { "4", 4, false}, { "5", 5, false}, { "6", 6, false}, { "7", 7, false}, { "8", 8, false}, { "9", 9, false}, { "10", 10, false}, { "11", 11, false}, { "12", 12, false}, { "13", 13, false}, { "14", 14, false}, { "15", 15, false}, { "16", 16, false}, { "17", 17, false}, { "18", 18, false}, { "19", 19, false}, { NULL, 0, false} }; static const struct config_enum_entry metadata_sync_mode_options[] = { { "transactional", METADATA_SYNC_TRANSACTIONAL, false }, { "nontransactional", METADATA_SYNC_NON_TRANSACTIONAL, false }, { NULL, 0, false } }; /* *INDENT-ON* */ /*----------------------------------------------------------------------* * On PG 18+ the hook signature changed; we wrap the old Citus handler * in a fresh function that matches the new typedef exactly. *----------------------------------------------------------------------*/ static void citus_executor_run_adapter(QueryDesc *queryDesc, ScanDirection direction, uint64 count #if PG_VERSION_NUM < PG_VERSION_18 , bool run_once #endif ) { /* PG18+ has no run_once flag */ CitusExecutorRun(queryDesc, direction, count, #if PG_VERSION_NUM >= PG_VERSION_18 true #else run_once #endif ); } /* shared library initialization function */ void _PG_init(void) { if (!process_shared_preload_libraries_in_progress) { ereport(ERROR, (errmsg("Citus can only be loaded via shared_preload_libraries"), errhint("Add citus to shared_preload_libraries configuration " "variable in postgresql.conf in master and workers. Note " "that citus should be at the beginning of " "shared_preload_libraries."))); } /* * Register contstraint_handler hooks of safestringlib first. This way * loading the extension will error out if one of these constraints are hit * during load. */ set_str_constraint_handler_s(ereport_constraint_handler); set_mem_constraint_handler_s(ereport_constraint_handler); /* * Perform checks before registering any hooks, to avoid erroring out in a * partial state. * * In many cases (e.g. planner and utility hook, to run inside * pg_stat_statements et. al.) we have to be loaded before other hooks * (thus as the innermost/last running hook) to be able to do our * duties. For simplicity insist that all hooks are previously unused. */ if (planner_hook != NULL || ProcessUtility_hook != NULL || ExecutorStart_hook != NULL || ExecutorRun_hook != NULL || ExplainOneQuery_hook != NULL) { ereport(ERROR, (errmsg("Citus has to be loaded first"), errhint("Place citus at the beginning of " "shared_preload_libraries."))); } ResizeStackToMaximumDepth(); /* * Extend the database directory structure before continuing with * initialization - one of the later steps might require them to exist. * If in a sub-process (windows / EXEC_BACKEND) this already has been * done. */ if (!IsUnderPostmaster) { CreateRequiredDirectories(); } InitConnParams(); /* * Register Citus configuration variables. Do so before intercepting * hooks or calling initialization functions, in case we want to do the * latter in a configuration dependent manner. */ RegisterCitusConfigVariables(); /* make our additional node types known */ RegisterNodes(); /* make our custom scan nodes known */ RegisterCitusCustomScanMethods(); /* intercept planner */ planner_hook = distributed_planner; /* register for planner hook */ set_rel_pathlist_hook = multi_relation_restriction_hook; get_relation_info_hook = multi_get_relation_info_hook; set_join_pathlist_hook = multi_join_restriction_hook; ExecutorStart_hook = CitusExecutorStart; ExecutorRun_hook = citus_executor_run_adapter; ExplainOneQuery_hook = CitusExplainOneQuery; prev_ExecutorEnd = ExecutorEnd_hook; ExecutorEnd_hook = CitusAttributeToEnd; /* register hook for error messages */ original_emit_log_hook = emit_log_hook; emit_log_hook = multi_log_hook; /* * Register hook for counting client backends that * are successfully authenticated. */ original_client_auth_hook = ClientAuthentication_hook; ClientAuthentication_hook = CitusAuthHook; prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = citus_shmem_request; InitializeMaintenanceDaemon(); InitializeMaintenanceDaemonForMainDb(); /* initialize coordinated transaction management */ InitializeTransactionManagement(); InitializeBackendManagement(); InitializeConnectionManagement(); InitPlacementConnectionManagement(); InitRelationAccessHash(); InitializeCitusQueryStats(); InitializeSharedConnectionStats(); InitializeLocallyReservedSharedConnections(); InitializeClusterClockMem(); /* * Adjust the Dynamic Library Path to prepend citus_decodes to the dynamic * library path. This is needed to make sure that the citus decoders are * loaded before the default decoders for CDC. */ if (EnableChangeDataCapture) { AdjustDynamicLibraryPathForCdcDecoders(); } /* initialize shard split shared memory handle management */ InitializeShardSplitSMHandleManagement(); InitializeMultiTenantMonitorSMHandleManagement(); InitializeStatCountersShmem(); /* enable modification of pg_catalog tables during pg_upgrade */ if (IsBinaryUpgrade) { SetConfigOption("allow_system_table_mods", "true", PGC_POSTMASTER, PGC_S_OVERRIDE); } /* * In postmasters execution of _PG_init, IsUnderPostmaster will be false and * we want to do the cleanup at that time only, otherwise there is a chance that * there will be parallel queries and we might do a cleanup for things that are * already in use. This is only needed in Windows. */ if (!IsUnderPostmaster) { DoInitialCleanup(); } PrevObjectAccessHook = object_access_hook; object_access_hook = CitusObjectAccessHook; /* ensure columnar module is loaded at the right time */ load_file(COLUMNAR_MODULE_NAME, false); /* * Register utility hook. This must be done after loading columnar, so * that the citus hook is called first, followed by the columnar hook, * followed by standard_ProcessUtility. That allows citus to distribute * ALTER TABLE commands before columnar strips out the columnar-specific * options. */ PrevProcessUtility = (ProcessUtility_hook != NULL) ? ProcessUtility_hook : standard_ProcessUtility; ProcessUtility_hook = citus_ProcessUtility; /* * Acquire symbols for columnar functions that citus calls. */ void *handle = NULL; /* use load_external_function() the first time to initialize the handle */ extern_ColumnarSupportsIndexAM = (ColumnarSupportsIndexAM_type) (void *) load_external_function(COLUMNAR_MODULE_NAME, "ColumnarSupportsIndexAM", true, &handle); CacheRegisterRelcacheCallback(InvalidateDistRelationCacheCallback, (Datum) 0); INIT_COLUMNAR_SYMBOL(CompressionTypeStr_type, CompressionTypeStr); INIT_COLUMNAR_SYMBOL(IsColumnarTableAmTable_type, IsColumnarTableAmTable); INIT_COLUMNAR_SYMBOL(ReadColumnarOptions_type, ReadColumnarOptions); /* initialize symbols for "pass-through" functions */ INIT_COLUMNAR_SYMBOL(PGFunction, columnar_handler); INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_set); INIT_COLUMNAR_SYMBOL(PGFunction, alter_columnar_table_reset); INIT_COLUMNAR_SYMBOL(PGFunction, upgrade_columnar_storage); INIT_COLUMNAR_SYMBOL(PGFunction, downgrade_columnar_storage); INIT_COLUMNAR_SYMBOL(PGFunction, columnar_relation_storageid); INIT_COLUMNAR_SYMBOL(PGFunction, columnar_storage_info); INIT_COLUMNAR_SYMBOL(PGFunction, columnar_store_memory_stats); INIT_COLUMNAR_SYMBOL(PGFunction, test_columnar_storage_write_new_page); /* * This part is only for SECURITY LABEL tests * mimicking what an actual security label provider would do */ if (RunningUnderCitusTestSuite) { register_label_provider("citus '!tests_label_provider", citus_test_object_relabel); } } /* * PrependCitusDecodersToDynamicLibrayPath prepends the $libdir/citus_decoders * to the dynamic library path. This is needed to make sure that the citus * decoders are loaded before the default decoders for CDC. */ static void AdjustDynamicLibraryPathForCdcDecoders(void) { if (strcmp(Dynamic_library_path, "$libdir") == 0) { SetConfigOption("dynamic_library_path", CDC_DECODER_DYNAMIC_LIB_PATH, PGC_POSTMASTER, PGC_S_OVERRIDE); } } /* * Requests any additional shared memory required for citus. */ static void citus_shmem_request(void) { if (prev_shmem_request_hook) { prev_shmem_request_hook(); } RequestAddinShmemSpace(BackendManagementShmemSize()); RequestAddinShmemSpace(SharedConnectionStatsShmemSize()); RequestAddinShmemSpace(MaintenanceDaemonShmemSize()); RequestAddinShmemSpace(CitusQueryStatsSharedMemSize()); RequestAddinShmemSpace(LogicalClockShmemSize()); RequestNamedLWLockTranche(STATS_SHARED_MEM_NAME, 1); RequestAddinShmemSpace(StatCountersShmemSize()); RequestNamedLWLockTranche(SAVED_BACKEND_STATS_HASH_LOCK_TRANCHE_NAME, 1); } /* * DoInitialCleanup does cleanup at start time. * Currently it: * - Removes intermediate result directories ( in case there are any leftovers) */ static void DoInitialCleanup(void) { CleanupJobCacheDirectory(); } /* * Stack size increase during high memory load may cause unexpected crashes. * With this alloca call, we are increasing stack size explicitly, so that if * it is not possible to increase stack size, we will get an OOM error instead * of a crash. * * This function is called on backend startup. The allocated memory will * automatically be released at the end of the function's scope. However, we'd * have already expanded the stack and it wouldn't shrink back. So, in a sense, * per backend we're securing max_stack_depth kB's of memory on the stack upfront. * * Not all the backends require max_stack_depth kB's on the stack, so we might end * up with unnecessary allocations. However, the default value is 2MB, which seems * an acceptable trade-off. Also, allocating memory upfront may perform better * under some circumstances. */ static void ResizeStackToMaximumDepth(void) { #ifndef WIN32 long max_stack_depth_bytes = max_stack_depth * 1024L; /* * Explanation of IGNORE-BANNED: * alloca is safe to use here since we limit the allocated size. We cannot * use malloc as a replacement, since we actually want to grow the stack * here. */ volatile char *stack_resizer = alloca(max_stack_depth_bytes); /* IGNORE-BANNED */ /* * Different architectures might have different directions while * growing the stack. So, touch both ends. */ stack_resizer[0] = 0; stack_resizer[max_stack_depth_bytes - 1] = 0; /* * Passing the address to external function also prevents the function * from being optimized away, and the debug elog can also help with * diagnosis if needed. */ elog(DEBUG5, "entry stack is at %p, increased to %p, the top and bottom values of " "the stack is %d and %d", &stack_resizer[0], &stack_resizer[max_stack_depth_bytes - 1], stack_resizer[max_stack_depth_bytes - 1], stack_resizer[0]); #endif } /* * multi_log_hook intercepts postgres log commands. We use this to override * postgres error messages when they're not specific enough for the users. */ static void multi_log_hook(ErrorData *edata) { /* * Show the user a meaningful error message when a backend is cancelled * by the distributed deadlock detection. Also reset the state for this, * since the next cancelation of the backend might have another reason. * * We also want to provide a useful hint for sequence overflow errors * because they're likely to be caused by the way Citus handles smallint/int * based sequences on worker nodes. Note that we add the hint without checking * whether we're on a worker node or the sequence was used on a distributed * table because catalog might not be available at this point. And given * that this hint might be shown for regular Postgres tables too, we inject * the hint only when EnableUnsupportedFeatureMessages is set to true. * Otherwise, vanilla tests would fail. */ bool clearState = true; if (edata->elevel == ERROR && edata->sqlerrcode == ERRCODE_QUERY_CANCELED && MyBackendGotCancelledDueToDeadlock(clearState)) { edata->sqlerrcode = ERRCODE_T_R_DEADLOCK_DETECTED; /* * This hook is called by EmitErrorReport() when emitting the ereport * either to frontend or to the server logs. And some callers of * EmitErrorReport() (e.g.: errfinish()) seems to assume that string * fields of given ErrorData object needs to be freed. For this reason, * we copy the message into heap here. */ edata->message = pstrdup("canceling the transaction since it was " "involved in a distributed deadlock"); } else if (EnableUnsupportedFeatureMessages && IsSequenceOverflowError(edata)) { edata->detail = pstrdup("nextval(sequence) calls in worker nodes " "are not supported for column defaults of " "type int or smallint"); edata->hint = pstrdup("If the command was issued from a worker node, " "try issuing it from the coordinator node " "instead."); } if (original_emit_log_hook) { original_emit_log_hook(edata); } } /* * IsSequenceOverflowError returns true if the given error is a sequence * overflow error. */ static bool IsSequenceOverflowError(ErrorData *edata) { static const char *sequenceOverflowedMsgPrefix = "nextval: reached maximum value of sequence"; static const int sequenceOverflowedMsgPrefixLen = 42; return edata->elevel == ERROR && edata->sqlerrcode == ERRCODE_SEQUENCE_GENERATOR_LIMIT_EXCEEDED && edata->message != NULL && strncmp(edata->message, sequenceOverflowedMsgPrefix, sequenceOverflowedMsgPrefixLen) == 0; } /* * StartupCitusBackend initializes per-backend infrastructure, and is called * the first time citus is used in a database. * * NB: All code here has to be able to cope with this routine being called * multiple times in the same backend. This will e.g. happen when the * extension is created, upgraded or dropped. Due to the way we detect the * extension being dropped this can also happen when autovacuum runs ANALYZE on * pg_dist_partition, see InvalidateDistRelationCacheCallback for details. */ void StartupCitusBackend(void) { InitializeMaintenanceDaemonBackend(); /* * For query backends this will be a no-op, because InitializeBackendData * is already called from the CitusAuthHook. But for background workers we * still need to initialize the backend data. */ InitializeBackendData(application_name); /* * If this is an external connection or a background workers this will * generate the global PID for this connection. For internal connections * this is a no-op, since InitializeBackendData will already have extracted * the gpid from the application_name. */ AssignGlobalPID(application_name); SetBackendDataDatabaseId(); RegisterConnectionCleanup(); RegisterSaveBackendStatsIntoSavedBackendStatsHash(); FinishedStartupCitusBackend = true; } /* * GetCurrentClientMinMessageLevelName returns the name of the * the GUC client_min_messages for its specified value. */ const char * GetClientMinMessageLevelNameForValue(int minMessageLevel) { struct config_enum record = { 0 }; record.options = log_level_options; const char *clientMinMessageLevelName = config_enum_lookup_by_value(&record, minMessageLevel); return clientMinMessageLevelName; } /* * RegisterConnectionCleanup cleans up any resources left at the end of the * session. We prefer to cleanup before shared memory exit to make sure that * this session properly releases anything hold in the shared memory. */ static void RegisterConnectionCleanup(void) { static bool registeredCleanup = false; if (registeredCleanup == false) { before_shmem_exit(CitusCleanupConnectionsAtExit, 0); registeredCleanup = true; } } /* * RegisterSaveBackendStatsIntoSavedBackendStatsHash registers the function * that saves the backend stats for the exited backends into the saved backend * stats hash. */ static void RegisterSaveBackendStatsIntoSavedBackendStatsHash(void) { static bool registeredSaveBackendStats = false; if (registeredSaveBackendStats == false) { before_shmem_exit(SaveBackendStatsIntoSavedBackendStatsHashAtExit, 0); registeredSaveBackendStats = true; } } /* * RegisterExternalClientBackendCounterDecrement is called when the backend terminates. * For all client backends, we register a callback that will undo */ static void RegisterExternalClientBackendCounterDecrement(void) { static bool registeredCleanup = false; if (registeredCleanup == false) { before_shmem_exit(DecrementExternalClientBackendCounterAtExit, 0); registeredCleanup = true; } } /* * CitusCleanupConnectionsAtExit is called before_shmem_exit() of the * backend for the purposes of any clean-up needed. */ static void CitusCleanupConnectionsAtExit(int code, Datum arg) { /* properly close all the cached connections */ ShutdownAllConnections(); /* * Make sure that we give the shared connections back to the shared * pool if any. This operation is a no-op if the reserved connections * are already given away. */ DeallocateReservedConnections(); /* we don't want any monitoring view/udf to show already exited backends */ SetActiveMyBackend(false); UnSetGlobalPID(); } /* * SaveBackendStatsIntoSavedBackendStatsHashAtExit is called before_shmem_exit() * of the backend for the purposes of saving the backend stats for the exited * backends into the saved backend stats hash. */ static void SaveBackendStatsIntoSavedBackendStatsHashAtExit(int code, Datum arg) { if (code) { /* don't try to save the stats during a crash */ return; } SaveBackendStatsIntoSavedBackendStatsHash(); } /* * DecrementExternalClientBackendCounterAtExit is called before_shmem_exit() of the * backend for the purposes decrementing */ static void DecrementExternalClientBackendCounterAtExit(int code, Datum arg) { DecrementExternalClientBackendCounter(); } /* * CreateRequiredDirectories - Create directories required for Citus to * function. * * These used to be created by initdb, but that's not possible anymore. */ static void CreateRequiredDirectories(void) { const char *subdir = ("base/" PG_JOB_CACHE_DIR); if (MakePGDirectory(subdir) != 0 && errno != EEXIST) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", subdir))); } } /* Register Citus configuration variables. */ static void RegisterCitusConfigVariables(void) { DefineCustomBoolVariable( "citus.all_modifications_commutative", gettext_noop("Bypasses commutativity checks when enabled"), NULL, &AllModificationsCommutative, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.allow_aggregate_worker_combine_on_internal_types", gettext_noop("Enables aggregate worker partial aggregates on aggregates that " "have internal type for the aggregate partial state storage."), gettext_noop( "This setting allows the use of pushdown of custom aggregates that have " "an STYPE that is internal. This is typically okay to do, but if a custom aggregate " "persists OID information or any node specific data into the state, this can cause " "weirdness when combining in the coordinator, so this is left as an option to turn off " "in those cases worker combine functions on internal types."), &AllowAggregateWorkerCombineOnInternalTypes, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.allow_modifications_from_workers_to_replicated_tables", gettext_noop("Enables modifications from workers to replicated " "tables such as reference tables or hash " "distributed tables with replication factor " "greater than 1."), gettext_noop("Allowing modifications from the worker nodes " "requires extra locking which might decrease " "the throughput. Disabling this GUC skips the " "extra locking and prevents modifications from " "worker nodes."), &AllowModificationsFromWorkersToReplicatedTables, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.allow_nested_distributed_execution", gettext_noop("Enables distributed execution within a task " "of another distributed execution."), gettext_noop("Nested distributed execution can happen when Citus " "pushes down a call to a user-defined function within " "a distributed query, and the function contains another " "distributed query. In this scenario, Citus makes no " "guarantess with regards to correctness and it is therefore " "disallowed by default. This setting can be used to allow " "nested distributed execution."), &AllowNestedDistributedExecution, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.allow_unsafe_constraints", gettext_noop("Enables unique constraints and exclusion constraints " "that do not include a distribution column."), gettext_noop("To enforce global uniqueness, Citus normally requires " "that unique constraints and exclusion constraints contain " "the distribution column. If the tuple does not include the " "distribution column, Citus cannot ensure that the same value " "is not present in another shard. However, in some cases the " "index creator knows that uniqueness within the shard implies " "global uniqueness (e.g. when indexing an expression derived " "from the distribution column) and adding the distribution column " "separately may not be desirable. This setting can then be used " "to disable the check."), &AllowUnsafeConstraints, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.allow_unsafe_locks_from_workers", gettext_noop("Enables acquiring a distributed lock from a worker " "when the coordinator is not in the metadata"), gettext_noop("Set to false by default. If set to true, enables " "acquiring a distributed lock from a worker " "when the coordinator is not in the metadata. " "This type of lock is unsafe because the worker will not be " "able to lock the coordinator; the coordinator will be able to " "intialize distributed operations on the resources locked " "by the worker. This can lead to concurrent operations from the " "coordinator and distributed deadlocks since the coordinator " "and the workers would not acquire locks across the same nodes " "in the same order."), &EnableAcquiringUnsafeLockFromWorkers, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.background_task_queue_interval", gettext_noop("Time to wait between checks for scheduled background tasks."), NULL, &BackgroundTaskQueueCheckInterval, 5000, -1, 7 * 24 * 3600 * 1000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.check_available_space_before_move", gettext_noop("When enabled will check free disk space before a shard move"), gettext_noop( "Free disk space will be checked when this setting is enabled before each shard move."), &CheckAvailableSpaceBeforeMove, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomStringVariable( "citus.cluster_name", gettext_noop("Which cluster this node is a part of"), NULL, &CurrentCluster, "default", PGC_SU_BACKEND, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.coordinator_aggregation_strategy", gettext_noop("Sets the strategy for when an aggregate cannot be pushed down. " "'row-gather' will pull up intermediate rows to the coordinator, " "while 'disabled' will error if coordinator aggregation is necessary"), NULL, &CoordinatorAggregationStrategy, COORDINATOR_AGGREGATION_ROW_GATHER, coordinator_aggregation_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.copy_switchover_threshold", gettext_noop("Sets the threshold for copy to be switched " "over per connection."), gettext_noop("Data size threshold to switch over the active placement for " "a connection. If this is too low, overhead of starting COPY " "commands will hurt the performance. If this is too high, " "buffered data will use lots of memory. 4MB is a good balance " "between memory usage and performance. Note that this is irrelevant " "in the common case where we open one connection per placement."), &CopySwitchOverThresholdBytes, 4 * 1024 * 1024, 1, INT_MAX, PGC_USERSET, GUC_UNIT_BYTE | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomRealVariable( "citus.count_distinct_error_rate", gettext_noop("Desired error rate when calculating count(distinct) " "approximates using the postgresql-hll extension. " "0.0 disables approximations for count(distinct); 1.0 " "provides no guarantees about the accuracy of results."), NULL, &CountDistinctErrorRate, 0.0, 0.0, 1.0, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); /* * This doesn't use cpu_priority_options on purpose, because we always need * to know the actual priority value so that `RESET citus.cpu_priority` * actually changes the priority back. */ DefineCustomIntVariable( "citus.cpu_priority", gettext_noop("Sets the CPU priority of the current backend."), gettext_noop("Lower numbers cause more favorable scheduling, so the " "queries that this backend runs will be able to use more " "CPU resources compared to queries from other backends. " "WARNING: Changing this setting can lead to a pnemomenom " "called 'priority inversion', due to locks being held " "between different backends. This means that processes " "might be scheduled in the exact oposite way of what you " "want, i.e. processes that you want scheduled a lot, are " "scheduled very little. So use this setting at your own " "risk."), &CpuPriority, GetOwnPriority(), -20, 19, PGC_SUSET, GUC_STANDARD, NULL, CpuPriorityAssignHook, NULL); DefineCustomEnumVariable( "citus.cpu_priority_for_logical_replication_senders", gettext_noop("Sets the CPU priority for backends that send logical " "replication changes to other nodes for online shard " "moves and splits."), gettext_noop("Lower numbers cause more favorable scheduling, so the " "backends used to do the shard move will get more CPU " "resources. 'inherit' is a special value and disables " "overriding the CPU priority for backends that send " "logical replication changes."), &CpuPriorityLogicalRepSender, CPU_PRIORITY_INHERIT, cpu_priority_options, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.create_object_propagation", gettext_noop("Controls the behavior of CREATE statements in transactions for " "supported objects"), gettext_noop("When creating new objects in transactions this setting is used to " "determine the behavior for propagating. When objects are created " "in a multi-statement transaction block Citus needs to switch to " "sequential mode (if not already) to make sure the objects are " "visible to later statements on shards. The switch to sequential is " "not always desired. By changing this behavior the user can trade " "off performance for full transactional consistency on the creation " "of new objects."), &CreateObjectPropagationMode, CREATE_OBJECT_PROPAGATION_IMMEDIATE, create_object_propagation_options, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.defer_drop_after_shard_move", gettext_noop("Deprecated, Citus always defers drop after shard move"), NULL, &DeprecatedDeferShardDeleteOnMove, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.defer_drop_after_shard_split", gettext_noop("Deprecated, Citus always defers drop after shard split"), NULL, &DeprecatedDeferShardDeleteOnSplit, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "citus.defer_shard_delete_interval", gettext_noop("Sets the time to wait between background deletion for shards."), gettext_noop("Shards that are marked for deferred deletion need to be deleted in " "the background at a later time. This is done at a regular interval " "configured here. The deletion is executed optimistically, it tries " "to take a lock on a shard to clean, if the lock can't be acquired " "the background worker moves on. When set to -1 this background " "process is skipped."), &DeferShardDeleteInterval, 15000, -1, 7 * 24 * 3600 * 1000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomRealVariable( "citus.desired_percent_disk_available_after_move", gettext_noop( "Sets how many percentage of free disk space should be after a shard move"), gettext_noop( "This setting controls how much free space should be available after a shard move. " "If the free disk space will be lower than this parameter, then shard move will result in " "an error."), &DesiredPercentFreeAfterMove, 10.0, 0.0, 100.0, PGC_SIGHUP, GUC_STANDARD, NULL, NULL, NULL); DefineCustomRealVariable( "citus.distributed_deadlock_detection_factor", gettext_noop("Sets the time to wait before checking for distributed " "deadlocks. Postgres' deadlock_timeout setting is " "multiplied with the value. If the value is set to -1, " "distributed deadlock detection is disabled."), NULL, &DistributedDeadlockDetectionTimeoutFactor, 2.0, -1.0, 1000.0, PGC_SIGHUP, GUC_STANDARD, ErrorIfNotASuitableDeadlockFactor, NULL, NULL); DefineCustomBoolVariable( "citus.enable_alter_database_owner", gettext_noop("Enables propagating ALTER DATABASE ... OWNER TO ... statements to " "workers"), NULL, &EnableAlterDatabaseOwner, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_alter_role_propagation", gettext_noop("Enables propagating ALTER ROLE statements to workers (excluding " "ALTER ROLE SET)"), NULL, &EnableAlterRolePropagation, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_alter_role_set_propagation", gettext_noop("Enables propagating ALTER ROLE SET statements to workers"), NULL, &EnableAlterRoleSetPropagation, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_binary_protocol", gettext_noop( "Enables communication between nodes using binary protocol when possible"), NULL, &EnableBinaryProtocol, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_change_data_capture", gettext_noop("Enables using replication origin tracking for change data capture"), NULL, &EnableChangeDataCapture, false, PGC_USERSET, GUC_STANDARD, NULL, EnableChangeDataCaptureAssignHook, NULL); DefineCustomBoolVariable( "citus.enable_cluster_clock", gettext_noop("When users explicitly call UDF citus_get_transaction_clock() " "and the flag is true, it returns the maximum " "clock among all nodes. All nodes move to the " "new clock. If clocks go bad for any reason, " "this serves as a safety valve."), NULL, &EnableClusterClock, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_cost_based_connection_establishment", gettext_noop("When enabled the connection establishment times " "and task execution times into account for deciding " "whether or not to establish new connections."), NULL, &EnableCostBasedConnectionEstablishment, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_create_database_propagation", gettext_noop("Enables propagating CREATE DATABASE " "and DROP DATABASE statements to workers."), NULL, &EnableCreateDatabasePropagation, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_create_role_propagation", gettext_noop("Enables propagating CREATE ROLE " "and DROP ROLE statements to workers"), NULL, &EnableCreateRolePropagation, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_create_type_propagation", gettext_noop("Enables propagating of CREATE TYPE statements to workers"), NULL, &EnableCreateTypePropagation, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_ddl_propagation", gettext_noop("Enables propagating DDL statements to worker shards"), NULL, &EnableDDLPropagation, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_deadlock_prevention", gettext_noop("Avoids deadlocks by preventing concurrent multi-shard commands"), gettext_noop("Multi-shard modifications such as UPDATE, DELETE, and " "INSERT...SELECT are typically executed in parallel. If multiple " "such commands run concurrently and affect the same rows, then " "they are likely to deadlock. When enabled, this flag prevents " "multi-shard modifications from running concurrently when they " "affect the same shards in order to prevent deadlocks."), &EnableDeadlockPrevention, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_fast_path_router_planner", gettext_noop("Enables fast path router planner"), NULL, &EnableFastPathRouterPlanner, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_local_execution", gettext_noop("Enables queries on shards that are local to the current node " "to be planned and executed locally."), NULL, &EnableLocalExecution, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_local_fast_path_query_optimization", gettext_noop("Enables the planner to avoid a query deparse and planning if " "the shard is local to the current node."), NULL, &EnableLocalFastPathQueryOptimization, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, WarnIfLocalExecutionDisabled, NULL, NULL); DefineCustomBoolVariable( "citus.enable_local_reference_table_foreign_keys", gettext_noop("Enables foreign keys from/to local tables"), gettext_noop("When enabled, foreign keys between local tables and reference " "tables supported."), &EnableLocalReferenceForeignKeys, true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_manual_changes_to_shards", gettext_noop("Enables dropping and truncating known shards."), gettext_noop("Set to false by default. If set to true, enables " "dropping and truncating shards on the coordinator " "(or the workers with metadata)"), &EnableManualChangesToShards, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomStringVariable( "citus.enable_manual_metadata_changes_for_user", gettext_noop("Enables some helper UDFs to modify metadata " "for the given user"), NULL, &EnableManualMetadataChangesForUser, "", PGC_SIGHUP, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_metadata_sync", gettext_noop("Enables object and metadata syncing."), NULL, &EnableMetadataSync, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_non_colocated_router_query_pushdown", gettext_noop("Enables router planner for the queries that reference " "non-colocated distributed tables."), gettext_noop("Normally, router planner planner is only enabled for " "the queries that reference colocated distributed tables " "because it is not guaranteed to have the target shards " "always on the same node, e.g., after rebalancing the " "shards. For this reason, while enabling this flag allows " "some degree of optimization for the queries that reference " "non-colocated distributed tables, it is not guaranteed " "that the same query will work after rebalancing the shards " "or altering the shard count of one of those distributed " "tables."), &EnableNonColocatedRouterQueryPushdown, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_outer_joins_with_pseudoconstant_quals_pre_pg17", gettext_noop("Enables running distributed queries with outer joins " "and pseudoconstant quals pre PG17."), gettext_noop("Set to false by default. If set to true, enables " "running distributed queries with outer joins and " "pseudoconstant quals, at user's own risk, because " "pre PG17, Citus doesn't have access to " "set_join_pathlist_hook, which doesn't guarantee correct" "query results. Note that in PG17+, this GUC has no effect" "and the user can run such queries"), &EnableOuterJoinsWithPseudoconstantQualsPrePG17, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_recurring_outer_join_pushdown", gettext_noop("Enables outer join pushdown for recurring relations."), gettext_noop("When enabled, Citus will try to push down outer joins " "between recurring and non-recurring relations to workers " "whenever feasible by introducing correctness constraints " "to the where clause of the query. Note that if this is " "disabled, or push down is not feasible, the result will " "be computed via recursive planning."), &EnableRecurringOuterJoinPushdown, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_repartition_joins", gettext_noop("Allows Citus to repartition data between nodes."), NULL, &EnableRepartitionJoins, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_repartitioned_insert_select", gettext_noop("Enables repartitioned INSERT/SELECTs"), NULL, &EnableRepartitionedInsertSelect, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_router_execution", gettext_noop("Enables router execution"), NULL, &EnableRouterExecution, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_schema_based_sharding", gettext_noop("Enables schema based sharding."), gettext_noop("The schemas created while this is ON will be automatically " "associated with individual colocation groups such that the " "tables created in those schemas will be automatically " "converted to colocated distributed tables without a shard " "key."), &EnableSchemaBasedSharding, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_single_hash_repartition_joins", gettext_noop("Enables single hash repartitioning between hash " "distributed tables"), NULL, &EnableSingleHashRepartitioning, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_stat_counters", gettext_noop("Enables the collection of statistic counters for Citus."), gettext_noop("When enabled, Citus maintains a set of statistic " "counters for the Citus extension. These statistics are " "available in the citus_stat_counters view and are " "lost on server shutdown and can be reset by executing " "the function citus_stat_counters_reset() on demand."), &EnableStatCounters, ENABLE_STAT_COUNTERS_DEFAULT, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_statistics_collection", gettext_noop("Deprecated."), NULL, &DeprecatedEnableStatisticsCollection, false, PGC_SIGHUP, GUC_SUPERUSER_ONLY, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_unique_job_ids", gettext_noop("Enables unique job IDs by prepending the local process ID and " "group ID. This should usually be enabled, but can be disabled " "for repeatable output in regression tests."), NULL, &EnableUniqueJobIds, true, PGC_USERSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_unsafe_triggers", gettext_noop("Enables arbitrary triggers on distributed tables which may cause " "visibility and deadlock issues. Use at your own risk."), NULL, &EnableUnsafeTriggers, false, PGC_USERSET, GUC_STANDARD | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_unsupported_feature_messages", gettext_noop("Controls showing of some citus related messages. It is intended to " "be used before vanilla tests to stop unwanted citus messages."), NULL, &EnableUnsupportedFeatureMessages, true, PGC_SUSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enable_version_checks", gettext_noop("Enables version checks during CREATE/ALTER EXTENSION commands"), NULL, &EnableVersionChecks, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enforce_foreign_key_restrictions", gettext_noop("Enforce restrictions while querying distributed/reference " "tables with foreign keys"), gettext_noop("When enabled, cascading modifications from reference tables " "to distributed tables are traced and acted accordingly " "to avoid creating distributed deadlocks and ensure correctness."), &EnforceForeignKeyRestrictions, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.enforce_object_restrictions_for_local_objects", gettext_noop( "Controls some restrictions for local objects."), NULL, &EnforceLocalObjectRestrictions, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.executor_slow_start_interval", gettext_noop("Time to wait between opening connections to the same worker node"), gettext_noop("When the individual tasks of a multi-shard query take very " "little time, they can often be finished over a single (often " "already cached) connection. To avoid redundantly opening " "additional connections, the executor waits between connection " "attempts for the configured number of milliseconds. At the end " "of the interval, it increases the number of connections it is " "allowed to open next time."), &ExecutorSlowStartInterval, 10, 0, INT_MAX, PGC_USERSET, GUC_UNIT_MS | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.explain_all_tasks", gettext_noop("Enables showing output for all tasks in Explain."), gettext_noop("The Explain command for distributed queries shows " "the remote plan for a single task by default. When " "this configuration entry is enabled, the plan for " "all tasks is shown, but the Explain takes longer."), &ExplainAllTasks, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.explain_analyze_sort_method", gettext_noop("Sets the sorting method for EXPLAIN ANALYZE queries."), gettext_noop("This parameter is intended for testing. It is developed " "to get consistent regression test outputs. When it is set " "to 'time', EXPLAIN ANALYZE output is sorted by execution " "duration on workers. When it is set to 'taskId', it is " "sorted by task id. By default, it is set to 'time'; but " "in regression tests, it's set to 'taskId' for consistency."), &ExplainAnalyzeSortMethod, EXPLAIN_ANALYZE_SORT_BY_TIME, explain_analyze_sort_method_options, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.explain_distributed_queries", gettext_noop("Enables Explain for distributed queries."), gettext_noop("When enabled, the Explain command shows remote and local " "plans when used with a distributed query. It is enabled " "by default, but can be disabled for regression tests."), &ExplainDistributedQueries, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.force_max_query_parallelization", gettext_noop("Open as many connections as possible to maximize query " "parallelization"), gettext_noop("When enabled, Citus will force the executor to use " "as many connections as possible while executing a " "parallel distributed query. If not enabled, the executor " "might choose to use less connections to optimize overall " "query execution throughput. Internally, setting this true " "will end up with using one connection per task."), &ForceMaxQueryParallelization, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.function_opens_transaction_block", gettext_noop("Open transaction blocks for function calls"), gettext_noop("When enabled, Citus will always send a BEGIN to workers when " "running distributed queres in a function. When disabled, the " "queries may be committed immediately after the statemnent " "completes. Disabling this flag is dangerous, it is only provided " "for backwards compatibility with pre-8.2 behaviour."), &FunctionOpensTransactionBlock, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomStringVariable( "citus.grep_remote_commands", gettext_noop( "Applies \"command\" like citus.grep_remote_commands, if returns " "true, the command is logged."), NULL, &GrepRemoteCommands, "", PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.hide_citus_dependent_objects", gettext_noop( "Hides some objects, which depends on citus extension, from pg meta class queries. " "It is intended to be used only before postgres vanilla tests to not break them."), NULL, &HideCitusDependentObjects, false, PGC_USERSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); /* * This was a GUC we added on Citus 11.0.1, and * replaced with another name on 11.0.2 via #5920. * However, as this GUC has been used in * citus_shard_indexes_on_worker-11.0.1 * script. So, it is not easy to completely get rid * of the GUC. Especially with PG 15+, Postgres verifies * existence of the GUCs that are used. So, without this * CREATE EXTENSION fails. */ DefineCustomStringVariable( "citus.hide_shards_from_app_name_prefixes", gettext_noop("Deprecated, use citus.show_shards_for_app_name_prefixes"), NULL, &DeprecatedEmptyString, "", PGC_SUSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.isolation_test_session_process_id", NULL, NULL, &IsolationTestSessionProcessID, -1, -1, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.isolation_test_session_remote_process_id", NULL, NULL, &IsolationTestSessionRemoteProcessID, -1, -1, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.limit_clause_row_fetch_count", gettext_noop("Number of rows to fetch per task for limit clause optimization."), gettext_noop("Select queries get partitioned and executed as smaller " "tasks. In some cases, select queries with limit clauses " "may need to fetch all rows from each task to generate " "results. In those cases, and where an approximation would " "produce meaningful results, this configuration value sets " "the number of rows to fetch from each task."), &LimitClauseRowFetchCount, -1, -1, INT_MAX, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.local_copy_flush_threshold", gettext_noop("Sets the threshold for local copy to be flushed."), NULL, &LocalCopyFlushThresholdByte, 512 * 1024, 1, INT_MAX, PGC_USERSET, GUC_UNIT_BYTE | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomStringVariable( "citus.local_hostname", gettext_noop("Sets the hostname when connecting back to itself."), gettext_noop("For some operations nodes, mostly the coordinator, connect back to " "itself. When configuring SSL certificates it sometimes is required " "to use a specific hostname to match the CN of the certificate when " "verify-full is used."), &LocalHostName, "localhost", PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.local_shared_pool_size", gettext_noop( "Sets the maximum number of connections allowed for the shards on the " "local node across all the backends from this node. Setting to -1 disables " "connections throttling. Setting to 0 makes it auto-adjust, meaning " "equal to the half of max_connections on the coordinator."), gettext_noop("As a rule of thumb, the value should be at most equal to the " "max_connections on the local node."), &LocalSharedPoolSize, 0, -1, INT_MAX, PGC_SIGHUP, GUC_SUPERUSER_ONLY, NULL, NULL, LocalPoolSizeGucShowHook); DefineCustomEnumVariable( "citus.local_table_join_policy", gettext_noop("defines the behaviour when a distributed table " "is joined with a local table"), gettext_noop( "There are 4 values available. The default, 'auto' will recursively plan " "distributed tables if there is a constant filter on a unique index. " "'prefer-local' will choose local tables if possible. " "'prefer-distributed' will choose distributed tables if possible. " "'never' will basically skip local table joins." ), &LocalTableJoinPolicy, LOCAL_JOIN_POLICY_AUTO, local_table_join_policies, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.log_distributed_deadlock_detection", gettext_noop("Log distributed deadlock detection related processing in " "the server log"), NULL, &LogDistributedDeadlockDetection, false, PGC_SIGHUP, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.log_intermediate_results", gettext_noop("Log intermediate results sent to other nodes"), NULL, &LogIntermediateResults, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.log_local_commands", gettext_noop("Log queries that are executed locally, can be overriden by " "citus.log_remote_commands"), NULL, &LogLocalCommands, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.log_multi_join_order", gettext_noop("Logs the distributed join order to the server log."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, we print the distributed join order."), &LogMultiJoinOrder, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.log_remote_commands", gettext_noop("Log queries sent to other nodes in the server log"), NULL, &LogRemoteCommands, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.logical_replication_timeout", gettext_noop("Sets the timeout to error out when logical replication is used"), gettext_noop("Citus uses logical replication when it moves/replicates shards. " "This setting determines when Citus gives up waiting for progress " "during logical replication and errors out."), &LogicalReplicationTimeout, 2 * 60 * 60 * 1000, 0, 7 * 24 * 3600 * 1000, PGC_SIGHUP, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE | GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_adaptive_executor_pool_size", gettext_noop("Sets the maximum number of connections per worker node used by " "the adaptive executor to execute a multi-shard command"), gettext_noop("The adaptive executor may open multiple connections per worker " "node when running multi-shard commands to parallelize the command " "across multiple cores on the worker. This setting specifies the " "maximum number of connections it will open. The number of " "connections is also bounded by the number of shards on the node. " "This setting can be used to reduce the memory usage of a query " "and allow a higher degree of concurrency when concurrent " "multi-shard queries open too many connections to a worker."), &MaxAdaptiveExecutorPoolSize, 16, 1, INT_MAX, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_background_task_executors", gettext_noop( "Sets the maximum number of parallel task executor workers for scheduled " "background tasks"), gettext_noop( "Controls the maximum number of parallel task executors the task monitor " "can create for scheduled background tasks. Note that the value is not effective " "if it is set a value higher than 'max_worker_processes' postgres parameter . It is " "also not guaranteed to have exactly specified number of parallel task executors " "because total background worker count is shared by all background workers. The value " "represents the possible maximum number of task executors."), &MaxBackgroundTaskExecutors, 1, 1, MAX_BG_TASK_EXECUTORS, PGC_SIGHUP, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_background_task_executors_per_node", gettext_noop( "Sets the maximum number of parallel background task executor workers " "for scheduled background tasks that involve a particular node"), NULL, &MaxBackgroundTaskExecutorsPerNode, 1, 1, 128, PGC_SIGHUP, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_cached_connection_lifetime", gettext_noop("Sets the maximum lifetime of cached connections to other nodes."), NULL, &MaxCachedConnectionLifetime, 10 * MS_PER_MINUTE, -1, INT_MAX, PGC_USERSET, GUC_UNIT_MS | GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_cached_conns_per_worker", gettext_noop("Sets the maximum number of connections to cache per worker."), gettext_noop("Each backend opens connections to the workers to query the " "shards. At the end of the transaction, the configurated number " "of connections is kept open to speed up subsequent commands. " "Increasing this value will reduce the latency of multi-shard " "queries, but increases overhead on the workers"), &MaxCachedConnectionsPerWorker, 1, 0, INT_MAX, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_client_connections", gettext_noop("Sets the maximum number of connections regular clients can make"), gettext_noop("To ensure that a Citus cluster has a sufficient number of " "connection slots to serve queries internally, it can be " "useful to reserve connection slots for Citus internal " "connections. When max_client_connections is set to a value " "below max_connections, the remaining connections are reserved " "for connections between Citus nodes. This does not affect " "superuser_reserved_connections. If set to -1, no connections " "are reserved."), &MaxClientConnections, -1, -1, MaxConnections, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_high_priority_background_processes", gettext_noop("Sets the maximum number of background processes " "that can have their CPU priority increased at the same " "time on a specific node."), gettext_noop("This setting is useful to make sure logical replication " "senders don't take over the CPU of the entire machine."), &MaxHighPriorityBackgroundProcesess, 2, 0, 10000, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_intermediate_result_size", gettext_noop("Sets the maximum size of the intermediate results in KB for " "CTEs and complex subqueries."), NULL, &MaxIntermediateResult, 1048576, -1, MAX_KILOBYTES, PGC_USERSET, GUC_UNIT_KB | GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_matview_size_to_auto_recreate", gettext_noop("Sets the maximum size of materialized views in MB to " "automatically distribute them."), NULL, &MaxMatViewSizeToAutoRecreate, 1024, -1, INT_MAX, PGC_USERSET, GUC_UNIT_MB | GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_rebalancer_logged_ignored_moves", gettext_noop("Sets the maximum number of ignored moves the rebalance logs"), NULL, &MaxRebalancerLoggedIgnoredMoves, 5, -1, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.max_shared_pool_size", gettext_noop("Sets the maximum number of connections allowed per worker node " "across all the backends from this node. Setting to -1 disables " "connections throttling. Setting to 0 makes it auto-adjust, meaning " "equal to max_connections on the coordinator."), gettext_noop("As a rule of thumb, the value should be at most equal to the " "max_connections on the remote nodes."), &MaxSharedPoolSize, 0, -1, INT_MAX, PGC_SIGHUP, GUC_SUPERUSER_ONLY, NULL, NULL, MaxSharedPoolSizeGucShowHook); DefineCustomIntVariable( "citus.max_worker_nodes_tracked", gettext_noop("Sets the maximum number of worker nodes that are tracked."), gettext_noop("Worker nodes' network locations, their membership and " "health status are tracked in a shared hash table on " "the master node. This configuration value limits the " "size of the hash table, and consequently the maximum " "number of worker nodes that can be tracked. " "Citus keeps some information about the worker nodes " "in the shared memory for certain optimizations. The " "optimizations are enforced up to this number of worker " "nodes. Any additional worker nodes may not benefit from " "the optimizations."), &MaxWorkerNodesTracked, 2048, 1024, INT_MAX, PGC_POSTMASTER, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.metadata_sync_interval", gettext_noop("Sets the time to wait between metadata syncs."), gettext_noop("metadata sync needs to run every so often " "to synchronize metadata to metadata nodes " "that are out of sync."), &MetadataSyncInterval, 60 * MS_PER_SECOND, 1, 7 * MS_PER_DAY, PGC_SIGHUP, GUC_UNIT_MS | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.metadata_sync_mode", gettext_noop("Sets transaction mode for metadata syncs."), gettext_noop("metadata sync can be run inside a single coordinated " "transaction or with multiple small transactions in " "idempotent way. By default we sync metadata in single " "coordinated transaction. When we hit memory problems " "at workers, we have alternative nontransactional mode " "where we send each command with separate transaction."), &MetadataSyncTransMode, METADATA_SYNC_TRANSACTIONAL, metadata_sync_mode_options, PGC_SUSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.metadata_sync_retry_interval", gettext_noop("Sets the interval to retry failed metadata syncs."), gettext_noop("metadata sync needs to run every so often " "to synchronize metadata to metadata nodes " "that are out of sync."), &MetadataSyncRetryInterval, 5 * MS_PER_SECOND, 1, 7 * MS_PER_DAY, PGC_SIGHUP, GUC_UNIT_MS | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); /* * Previously we setting this configuration parameter * in the fly for failure tests schedule. * However, PG15 doesn't allow that anymore: reserved prefixes * like "citus" cannot be used to set non-existing GUCs. * Relevant PG commit: 88103567cb8fa5be46dc9fac3e3b8774951a2be7 */ DefineCustomStringVariable( "citus.mitmfifo", gettext_noop("Sets the citus mitm fifo path for failure tests"), gettext_noop("This GUC is only used for testing."), &MitmfifoEmptyString, "", PGC_SUSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.multi_shard_modify_mode", gettext_noop("Sets the connection type for multi shard modify queries"), NULL, &MultiShardConnectionType, PARALLEL_CONNECTION, multi_shard_modify_connection_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.multi_task_query_log_level", gettext_noop("Sets the level of multi task query execution log messages"), NULL, &MultiTaskQueryLogLevel, CITUS_LOG_LEVEL_OFF, log_level_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.next_cleanup_record_id", gettext_noop("Set the next cleanup record ID to use in operation creation."), gettext_noop("Cleanup record IDs are normally generated using a sequence. If " "next_cleanup_record_id is set to a non-zero value, cleanup record IDs will " "instead be generated by incrementing from the value of " "this GUC and this will be reflected in the GUC. This is " "mainly useful to ensure consistent cleanup record IDs when running " "tests in parallel."), &NextCleanupRecordId, 0, 0, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.next_operation_id", gettext_noop("Set the next operation ID to use in operation creation."), gettext_noop("Operation IDs are normally generated using a sequence. If " "next_operation_id is set to a non-zero value, operation IDs will " "instead be generated by incrementing from the value of " "this GUC and this will be reflected in the GUC. This is " "mainly useful to ensure consistent operation IDs when running " "tests in parallel."), &NextOperationId, 0, 0, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.next_placement_id", gettext_noop("Set the next placement ID to use in placement creation."), gettext_noop("Placement IDs are normally generated using a sequence. If " "next_placement_id is set to a non-zero value, placement IDs will " "instead be generated by incrementing from the value of " "this GUC and this will be reflected in the GUC. This is " "mainly useful to ensure consistent placement IDs when running " "tests in parallel."), &NextPlacementId, 0, 0, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.next_shard_id", gettext_noop("Set the next shard ID to use in shard creation."), gettext_noop("Shard IDs are normally generated using a sequence. If " "next_shard_id is set to a non-zero value, shard IDs will " "instead be generated by incrementing from the value of " "this GUC and this will be reflected in the GUC. This is " "mainly useful to ensure consistent shard IDs when running " "tests in parallel."), &NextShardId, 0, 0, INT_MAX, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.node_connection_timeout", gettext_noop("Sets the maximum duration to connect to worker nodes."), NULL, &NodeConnectionTimeout, 30 * MS_PER_SECOND, 10 * MS, MS_PER_HOUR, PGC_USERSET, GUC_UNIT_MS | GUC_STANDARD, NULL, NULL, NULL); DefineCustomStringVariable( "citus.node_conninfo", gettext_noop("Sets parameters used for outbound connections."), NULL, &NodeConninfo, #ifdef USE_SSL "sslmode=require", #else "sslmode=prefer", #endif PGC_SIGHUP, GUC_SUPERUSER_ONLY, NodeConninfoGucCheckHook, NodeConninfoGucAssignHook, NULL); DefineCustomBoolVariable( "citus.override_table_visibility", gettext_noop("Enables replacing occurrrences of pg_catalog.pg_table_visible() " "with pg_catalog.citus_table_visible()"), gettext_noop("When enabled, shards on the Citus MX worker (data) nodes would be " "filtered out by many psql commands to provide better user " "experience."), &OverrideTableVisibility, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.prevent_incomplete_connection_establishment", gettext_noop("When enabled, the executor waits until all the connections " "are successfully established."), gettext_noop("Under some load, the executor may decide to establish some " "extra connections to further parallelize the execution. However, " "before the connection establishment is done, the execution might " "have already finished. When this GUC is set to true, the execution " "waits for such connections to be established."), &PreventIncompleteConnectionEstablishment, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.propagate_session_settings_for_loopback_connection", gettext_noop( "When enabled, rebalancer propagates all the allowed GUC settings to new connections."), NULL, &PropagateSessionSettingsForLoopbackConnection, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.propagate_set_commands", gettext_noop("Sets which SET commands are propagated to workers."), NULL, &PropagateSetCommands, PROPSETCMD_NONE, propagate_set_commands_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.rebalancer_by_disk_size_base_cost", gettext_noop( "When using the by_disk_size rebalance strategy each shard group " "will get this cost in bytes added to its actual disk size. This " "is used to avoid creating a bad balance when there's very little " "data in some of the shards. The assumption is that even empty " "shards have some cost, because of parallelism and because empty " "shard groups will likely grow in the future."), gettext_noop( "The main reason this is configurable, is so it can be lowered for Citus its regression tests."), &RebalancerByDiskSizeBaseCost, 100 * 1024 * 1024, 0, INT_MAX, PGC_USERSET, GUC_UNIT_BYTE | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.recover_2pc_interval", gettext_noop("Sets the time to wait between recovering 2PCs."), gettext_noop("2PC transaction recovery needs to run every so often " "to clean up records in pg_dist_transaction and " "potentially roll failed 2PCs forward. This setting " "determines how often recovery should run, " "use -1 to disable."), &Recover2PCInterval, 60 * MS_PER_SECOND, -1, 7 * MS_PER_DAY, PGC_SIGHUP, GUC_UNIT_MS | GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.remote_copy_flush_threshold", gettext_noop("Sets the threshold for remote copy to be flushed."), gettext_noop("When sending data over remote connections via the COPY protocol, " "bytes are first buffered internally by libpq. If the number of " "bytes buffered exceeds the threshold, Citus waits for all the " "bytes to flush."), &RemoteCopyFlushThreshold, 8 * 1024 * 1024, 0, INT_MAX, PGC_USERSET, GUC_UNIT_BYTE | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.remote_task_check_interval", gettext_noop("Sets the frequency at which we check job statuses."), gettext_noop("The master node assigns tasks to workers nodes, and " "then regularly checks with them about each task's " "progress. This configuration value sets the time " "interval between two consequent checks."), &RemoteTaskCheckInterval, 10, 1, INT_MAX, PGC_USERSET, GUC_UNIT_MS | GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.repartition_join_bucket_count_per_node", gettext_noop("Sets the bucket size for repartition joins per node"), gettext_noop("Repartition joins create buckets in each node and " "uses those to shuffle data around nodes. "), &RepartitionJoinBucketCountPerNode, 4, 1, INT_MAX, PGC_SIGHUP, GUC_STANDARD | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); /* deprecated setting */ DefineCustomBoolVariable( "citus.replicate_reference_tables_on_activate", NULL, NULL, &DeprecatedReplicateReferenceTablesOnActivate, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.replication_model", gettext_noop("Deprecated. Please use citus.shard_replication_factor instead"), gettext_noop( "Shard replication model is determined by the shard replication factor. " "'statement' replication is used only when the replication factor is one."), &ReplicationModel, REPLICATION_MODEL_STREAMING, replication_model_options, PGC_SUSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, WarnIfReplicationModelIsSet, NULL, NULL); DefineCustomBoolVariable( "citus.running_under_citus_test_suite", gettext_noop( "Only useful for testing purposes, when set to true, Citus does some " "tricks to implement useful isolation tests with rebalancing. It also " "registers a dummy label provider for SECURITY LABEL tests. Should " "never be set to true on production systems "), gettext_noop("for details of the tricks implemented, refer to the source code"), &RunningUnderCitusTestSuite, false, PGC_SUSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.select_opens_transaction_block", gettext_noop("Open transaction blocks for SELECT commands"), gettext_noop("When enabled, Citus will always send a BEGIN to workers when " "running a distributed SELECT in a transaction block (the " "default). When disabled, Citus will only send BEGIN before " "the first write or other operation that requires a distributed " "transaction, meaning the SELECT on the worker commits " "immediately, releasing any locks and apply any changes made " "through function calls even if the distributed transaction " "aborts."), &SelectOpensTransactionBlock, true, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.shard_count", gettext_noop("Sets the number of shards for a new hash-partitioned table " "created with create_distributed_table()."), NULL, &ShardCount, 32, 1, MAX_SHARD_COUNT, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.shard_replication_factor", gettext_noop("Sets the replication factor for shards."), gettext_noop("Shards are replicated across nodes according to this " "replication factor. Note that shards read this " "configuration value at sharded table creation time, " "and later reuse the initially read value."), &ShardReplicationFactor, 1, 1, MAX_SHARD_REPLICATION_FACTOR, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomStringVariable( "citus.show_shards_for_app_name_prefixes", gettext_noop("If application_name starts with one of these values, show shards"), gettext_noop("Citus places distributed tables and shards in the same schema. " "That can cause confusion when inspecting the list of tables on " "a node with shards. By default the shards are hidden from " "pg_class. This GUC can be used to show the shards to certain " "applications based on the application_name of the connection. " "The default is empty string, which hides shards from all " "applications. This behaviour can be overridden using the " "citus.override_table_visibility setting"), &ShowShardsForAppNamePrefixes, "", PGC_USERSET, GUC_STANDARD, ShowShardsForAppNamePrefixesCheckHook, ShowShardsForAppNamePrefixesAssignHook, NULL); DefineCustomBoolVariable( "citus.skip_advisory_lock_permission_checks", gettext_noop("Postgres would normally enforce some " "ownership checks while acquiring locks. " "When this setting is 'on', Citus skips " "ownership checks on internal advisory " "locks."), NULL, &SkipAdvisoryLockPermissionChecks, false, PGC_SUSET, GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.skip_constraint_validation", gettext_noop("Skip validation of constraints"), gettext_noop("Validating constraints is a costly operation which effects Citus' " "performance negatively. With this GUC set to true, we skip " "validating them. Constraint validation can be redundant for some " "cases. For instance, when moving a shard, which has already " "validated constraints at the source; we don't need to validate " "the constraints again at the destination."), &SkipConstraintValidation, false, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.skip_jsonb_validation_in_copy", gettext_noop("Skip validation of JSONB columns on the coordinator during COPY " "into a distributed table"), gettext_noop("Parsing large JSON objects may incur significant CPU overhead, " "which can lower COPY throughput. If this GUC is set (the default), " "JSON parsing is skipped on the coordinator, which means you cannot " "see the line number in case of malformed JSON, but throughput will " "be higher. This setting does not apply if the input format is " "binary."), &SkipJsonbValidationInCopy, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.sort_returning", gettext_noop("Sorts the RETURNING clause to get consistent test output"), gettext_noop("This feature is not intended for users. It is developed " "to get consistent regression test outputs. When enabled, " "the RETURNING clause returns the tuples sorted. The sort " "is done for all the entries, starting from the first one. " "Finally, the sorting is done in ASC order."), &SortReturning, false, PGC_SUSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); /* * It takes about 140 bytes of shared memory to store one row, therefore * this setting should be used responsibly. setting it to 10M will require * 1.4GB of shared memory. */ DefineCustomIntVariable( "citus.stat_statements_max", gettext_noop("Determines maximum number of statements tracked by " "citus_stat_statements."), NULL, &StatStatementsMax, 50000, 1000, 10000000, PGC_POSTMASTER, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomIntVariable( "citus.stat_statements_purge_interval", gettext_noop("Determines time interval in seconds for " "citus_stat_statements to purge expired entries."), NULL, &StatStatementsPurgeInterval, 10, -1, INT_MAX, PGC_SIGHUP, GUC_UNIT_MS | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.stat_statements_track", gettext_noop( "Enables/Disables the stats collection for citus_stat_statements."), gettext_noop("Enables the stats collection when set to 'all'. " "Disables when set to 'none'. Disabling can be useful for " "avoiding extra CPU cycles needed for the calculations."), &StatStatementsTrack, STAT_STATEMENTS_TRACK_NONE, stat_statements_track_options, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.stat_tenants_limit", gettext_noop("Number of tenants to be shown in citus_stat_tenants."), NULL, &StatTenantsLimit, 100, 1, 10000, PGC_POSTMASTER, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.stat_tenants_log_level", gettext_noop("Sets the level of citus_stat_tenants log messages"), NULL, &StatTenantsLogLevel, CITUS_LOG_LEVEL_OFF, log_level_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.stat_tenants_period", gettext_noop("Period in seconds to be used for calculating the tenant " "statistics in citus_stat_tenants."), NULL, &StatTenantsPeriod, 60, 1, 60 * 60 * 24, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.stat_tenants_track", gettext_noop("Enables/Disables the stats collection for citus_stat_tenants."), gettext_noop("Enables the stats collection when set to 'all'. " "Disables when set to 'none'. Disabling can be useful for " "avoiding extra CPU cycles needed for the calculations."), &StatTenantsTrack, STAT_TENANTS_TRACK_NONE, stat_tenants_track_options, PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomRealVariable( "citus.stat_tenants_untracked_sample_rate", gettext_noop("Sampling rate for new tenants in citus_stat_tenants."), NULL, &StatTenantsSampleRateForNewTenants, 1, 0, 1, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.subquery_pushdown", gettext_noop("Usage of this GUC is highly discouraged, please read the long " "description"), gettext_noop("When enabled, the planner skips many correctness checks " "for subqueries and pushes down the queries to shards as-is. " "It means that the queries are likely to return wrong results " "unless the user is absolutely sure that pushing down the " "subquery is safe. This GUC is maintained only for backward " "compatibility, no new users are supposed to use it. The planner " "is capable of pushing down as much computation as possible to the " "shards depending on the query."), &SubqueryPushdown, false, PGC_USERSET, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NoticeIfSubqueryPushdownEnabled, NULL, NULL); DefineCustomStringVariable( "citus.superuser", gettext_noop("Name of a superuser role to be used in Citus main database " "connections"), NULL, &SuperuserRole, "", PGC_SUSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.task_assignment_policy", gettext_noop("Sets the policy to use when assigning tasks to worker nodes."), gettext_noop("The master node assigns tasks to worker nodes based on shard " "locations. This configuration value specifies the policy to " "use when making these assignments. The greedy policy aims to " "evenly distribute tasks across worker nodes, first-replica just " "assigns tasks in the order shard placements were created, " "and the round-robin policy assigns tasks to worker nodes in " "a round-robin fashion."), &TaskAssignmentPolicy, TASK_ASSIGNMENT_GREEDY, task_assignment_policy_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.task_executor_type", gettext_noop("Sets the executor type to be used for distributed queries."), gettext_noop("The master node chooses between two different executor types " "when executing a distributed query.The adaptive executor is " "optimal for simple key-value lookup queries and queries that " "involve aggregations and/or co-located joins on multiple shards. "), &TaskExecutorType, MULTI_EXECUTOR_ADAPTIVE, task_executor_type_options, PGC_USERSET, GUC_STANDARD, WarnIfDeprecatedExecutorUsed, NULL, NULL); DefineCustomBoolVariable( "citus.use_citus_managed_tables", gettext_noop("Allows new local tables to be accessed on workers"), gettext_noop("Adds all newly created tables to Citus metadata by default, " "when enabled. Set to false by default."), &AddAllLocalTablesToMetadata, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.use_secondary_nodes", gettext_noop("Sets the policy to use when choosing nodes for SELECT queries."), NULL, &ReadFromSecondaries, USE_SECONDARY_NODES_NEVER, use_secondary_nodes_options, PGC_SU_BACKEND, GUC_STANDARD, NULL, NULL, NULL); DefineCustomIntVariable( "citus.values_materialization_threshold", gettext_noop("Sets the maximum number of rows allowed for pushing down " "VALUES clause in multi-shard queries. If the number of " "rows exceeds the threshold, the VALUES is materialized " "via pull-push execution. When set to -1, materialization " "is disabled. When set to 0, all VALUES are materialized."), gettext_noop("When the VALUES is pushed down (i.e., not materialized), " "the VALUES clause needs to be deparsed for every shard on " "the coordinator - and parsed on the workers. As this " "setting increased, the associated overhead is multiplied " "by the shard count. When materialized, the VALUES is " "deparsed and parsed once. The downside of materialization " "is that Postgres may choose a poor plan when joining " "the materialized result with tables."), &ValuesMaterializationThreshold, 100, -1, INT_MAX, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomStringVariable( "citus.version", gettext_noop("Shows the Citus library version"), NULL, &CitusVersion, CITUS_VERSION, PGC_INTERNAL, GUC_STANDARD, NULL, NULL, NULL); DefineCustomEnumVariable( "citus.worker_min_messages", gettext_noop("Log messages from workers only if their log level is at or above " "the configured level"), NULL, &WorkerMinMessages, NOTICE, log_level_options, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); DefineCustomBoolVariable( "citus.writable_standby_coordinator", gettext_noop("Enables simple DML via a streaming replica of the coordinator"), NULL, &WritableStandbyCoordinator, false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); /* warn about config items in the citus namespace that are not registered above */ EmitWarningsOnPlaceholders("citus"); OverridePostgresConfigProperties(); } /* * OverridePostgresConfigProperties overrides GUC properties where we want * custom behaviour. We should consider using Postgres function find_option * in this function once it is exported by Postgres in a later release. */ static void OverridePostgresConfigProperties(void) { int gucCount = 0; struct config_generic **guc_vars = get_guc_variables(&gucCount); for (int gucIndex = 0; gucIndex < gucCount; gucIndex++) { struct config_generic *var = (struct config_generic *) guc_vars[gucIndex]; if (strcmp(var->name, "application_name") == 0) { struct config_string *stringVar = (struct config_string *) var; OldApplicationNameAssignHook = stringVar->assign_hook; stringVar->assign_hook = ApplicationNameAssignHook; } /* * Turn on GUC_REPORT for search_path. GUC_REPORT provides that an S (Parameter Status) * packet is appended after the C (Command Complete) packet sent from the server * for SET command. S packet contains the new value of the parameter * if its value has been changed. */ if (strcmp(var->name, "search_path") == 0) { var->flags |= GUC_REPORT; } } } /* * We don't want to allow values less than 1.0. However, we define -1 as the value to disable * distributed deadlock checking. Here we enforce our special constraint. */ static bool ErrorIfNotASuitableDeadlockFactor(double *newval, void **extra, GucSource source) { if (*newval <= 1.0 && *newval != -1.0) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "citus.distributed_deadlock_detection_factor cannot be less than 1. " "To disable distributed deadlock detection set the value to -1."))); return false; } return true; } /* * WarnIfDeprecatedExecutorUsed prints a warning and sets the config value to * adaptive executor (a.k.a., ignores real-time executor). */ static bool WarnIfDeprecatedExecutorUsed(int *newval, void **extra, GucSource source) { if (*newval == DUMMY_REAL_TIME_EXECUTOR_ENUM_VALUE) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Ignoring the setting, real-time executor is " "deprecated"))); /* adaptive executor is superset of real-time, so switch to that */ *newval = MULTI_EXECUTOR_ADAPTIVE; } return true; } /* * WarnIfLocalExecutionDisabled is used to emit a warning message when * enabling citus.enable_local_fast_path_query_optimization if * citus.enable_local_execution was disabled. */ static bool WarnIfLocalExecutionDisabled(bool *newval, void **extra, GucSource source) { if (*newval == true && EnableLocalExecution == false) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "citus.enable_local_execution must be set in order for " "citus.enable_local_fast_path_query_optimization to be effective."))); } return true; } /* * NoticeIfSubqueryPushdownEnabled prints a notice when a user sets * citus.subquery_pushdown to ON. It doesn't print the notice if the * value is already true. */ static bool NoticeIfSubqueryPushdownEnabled(bool *newval, void **extra, GucSource source) { /* notice only when the value changes */ if (*newval == true && SubqueryPushdown == false) { ereport(NOTICE, (errcode(ERRCODE_WARNING_DEPRECATED_FEATURE), errmsg("Setting citus.subquery_pushdown flag is " "discouraged becuase it forces the planner " "to pushdown certain queries, skipping " "relevant correctness checks."), errdetail( "When enabled, the planner skips many correctness checks " "for subqueries and pushes down the queries to shards as-is. " "It means that the queries are likely to return wrong results " "unless the user is absolutely sure that pushing down the " "subquery is safe. This GUC is maintained only for backward " "compatibility, no new users are supposed to use it. The planner " "is capable of pushing down as much computation as possible to the " "shards depending on the query."))); } return true; } /* * WarnIfReplicationModelIsSet prints a warning when a user sets * citus.replication_model. */ static bool WarnIfReplicationModelIsSet(int *newval, void **extra, GucSource source) { /* print a warning only when user sets the guc */ if (source == PGC_S_SESSION) { ereport(NOTICE, (errcode(ERRCODE_WARNING_DEPRECATED_FEATURE), errmsg( "Setting citus.replication_model has no effect. Please use " "citus.shard_replication_factor instead."), errdetail( "Citus determines the replication model based on the " "replication factor and the replication models of the colocated " "shards. If a colocated table is present, the replication model " "is inherited. Otherwise 'streaming' replication is preferred if " "supported by the replication factor."))); } return true; } /* * ShowShardsForAppNamePrefixesCheckHook ensures that the * citus.show_shards_for_app_name_prefixes holds a valid list of application_name * values. */ static bool ShowShardsForAppNamePrefixesCheckHook(char **newval, void **extra, GucSource source) { List *prefixList = NIL; /* SplitGUCList scribbles on the input */ char *splitCopy = pstrdup(*newval); /* check whether we can split into a list of identifiers */ if (!SplitGUCList(splitCopy, ',', &prefixList)) { GUC_check_errdetail("not a valid list of identifiers"); return false; } char *appNamePrefix = NULL; foreach_declared_ptr(appNamePrefix, prefixList) { int prefixLength = strlen(appNamePrefix); if (prefixLength >= NAMEDATALEN) { GUC_check_errdetail("prefix %s is more than %d characters", appNamePrefix, NAMEDATALEN); return false; } char *prefixAscii = pstrdup(appNamePrefix); pg_clean_ascii(prefixAscii, 0); if (strcmp(prefixAscii, appNamePrefix) != 0) { GUC_check_errdetail("prefix %s in citus.show_shards_for_app_name_prefixes " "contains non-ascii characters", appNamePrefix); return false; } } return true; } /* * ShowShardsForAppNamePrefixesAssignHook ensures changes to * citus.show_shards_for_app_name_prefixes are reflected in the decision * whether or not to show shards. */ static void ShowShardsForAppNamePrefixesAssignHook(const char *newval, void *extra) { ResetHideShardsDecision(); } /* * ApplicationNameAssignHook is called whenever application_name changes * to allow us to reset our hide shards decision. */ static void ApplicationNameAssignHook(const char *newval, void *extra) { ResetHideShardsDecision(); DetermineCitusBackendType(newval); /* * We use StartupCitusBackend to initialize the global pid after catalogs * are available. After that happens this hook becomes responsible to update * the global pid on later application_name changes. So we set the * FinishedStartupCitusBackend flag in StartupCitusBackend to indicate when * this responsibility handoff has happened. * * Also note that when application_name changes, we don't actually need to * try re-assigning the global pid for external client backends and * background workers because application_name doesn't affect the global * pid for such backends - note that !IsExternalClientBackend() check covers * both types of backends. Plus, * trying to re-assign the global pid for such backends would unnecessarily * cause performing a catalog access when the cached local node id is * invalidated. However, accessing to the catalog tables is dangerous in * certain situations like when we're not in a transaction block. And for * the other types of backends, i.e., the Citus internal backends, we need * to re-assign the global pid when the application_name changes because for * such backends we simply extract the global pid inherited from the * originating backend from the application_name -that's specified by * originating backend when openning that connection- and this doesn't require * catalog access. * * Another solution to the catalog table acccess problem would be to update * global pid lazily, like we do for HideShards. But that's not possible * for the global pid, since it is stored in shared memory instead of in a * process-local global variable. So other processes might want to read it * before this process has updated it. So instead we try to set it as early * as reasonably possible, which is also why we extract global pids in the * AuthHook already (extracting doesn't require catalog access). */ if (FinishedStartupCitusBackend && !IsExternalClientBackend()) { AssignGlobalPID(newval); } OldApplicationNameAssignHook(newval, extra); } /* * NodeConninfoGucCheckHook ensures conninfo settings are in the expected form * and that the keywords of all non-null settings are on a allowlist devised to * keep users from setting options that may result in confusion. */ static bool NodeConninfoGucCheckHook(char **newval, void **extra, GucSource source) { /* this array _must_ be kept in an order usable by bsearch */ const char *allowedConninfoKeywords[] = { "connect_timeout", #if defined(ENABLE_GSS) && defined(ENABLE_SSPI) "gsslib", #endif "host", "keepalives", "keepalives_count", "keepalives_idle", "keepalives_interval", #if defined(ENABLE_GSS) || defined(ENABLE_SSPI) "krbsrvname", #endif "sslcert", "sslcompression", "sslcrl", "sslkey", #if PG_VERSION_NUM >= PG_VERSION_18 "sslkeylogfile", #endif "sslmode", #if PG_VERSION_NUM >= PG_VERSION_17 "sslnegotiation", #endif "sslrootcert", "tcp_user_timeout", }; char *errorMsg = NULL; bool conninfoValid = CheckConninfo(*newval, allowedConninfoKeywords, lengthof(allowedConninfoKeywords), &errorMsg); if (!conninfoValid) { GUC_check_errdetail("%s", errorMsg); } return conninfoValid; } /* * CpuPriorityAssignHook changes the priority of the current backend to match * the chosen value. */ static void CpuPriorityAssignHook(int newval, void *extra) { SetOwnPriority(newval); } /* * NodeConninfoGucAssignHook is the assignment hook for the node_conninfo GUC * variable. Though this GUC is a "string", we actually parse it as a non-URI * PQconninfo key/value setting, storing the resultant PQconninfoOption values * using the public functions in connection_configuration.c. */ static void NodeConninfoGucAssignHook(const char *newval, void *extra) { if (newval == NULL) { newval = ""; } if (strcmp(newval, NodeConninfo) == 0 && checkAtBootPassed) { /* It did not change, no need to do anything */ return; } checkAtBootPassed = true; PQconninfoOption *optionArray = PQconninfoParse(newval, NULL); if (optionArray == NULL) { ereport(FATAL, (errmsg("cannot parse node_conninfo value"), errdetail("The GUC check hook should prevent " "all malformed values."))); } ResetConnParams(); for (PQconninfoOption *option = optionArray; option->keyword != NULL; option++) { if (option->val == NULL || option->val[0] == '\0') { continue; } AddConnParam(option->keyword, option->val); } PQconninfoFree(optionArray); /* * Mark all connections for shutdown, since they have been opened using old * connection settings. This is mostly important when changing SSL * parameters, otherwise these would not be applied and connections could * be unencrypted when the user doesn't want that. */ CloseAllConnectionsAfterTransaction(); } /* * MaxSharedPoolSizeGucShowHook overrides the value that is shown to the * user when the default value has not been set. */ static const char * MaxSharedPoolSizeGucShowHook(void) { StringInfo newvalue = makeStringInfo(); if (MaxSharedPoolSize == 0) { appendStringInfo(newvalue, "%d", GetMaxSharedPoolSize()); } else { appendStringInfo(newvalue, "%d", MaxSharedPoolSize); } return (const char *) newvalue->data; } /* * LocalPoolSizeGucShowHook overrides the value that is shown to the * user when the default value has not been set. */ static const char * LocalPoolSizeGucShowHook(void) { StringInfo newvalue = makeStringInfo(); appendStringInfo(newvalue, "%d", GetLocalSharedPoolSize()); return (const char *) newvalue->data; } /* * CitusAuthHook is a callback for client authentication that Postgres provides. * Citus uses this hook to count the number of active backends. */ static void CitusAuthHook(Port *port, int status) { /* * We determine the backend type here because other calls in this hook rely * on it, both IsExternalClientBackend and InitializeBackendData. These * calls would normally initialize its value based on the application_name * global, but this global is not set yet at this point in the connection * initialization. So here we determine it based on the value from Port. */ DetermineCitusBackendType(port->application_name); /* external connections to not have a GPID immediately */ if (IsExternalClientBackend()) { /* * We raise the shared connection counter pre-emptively. As a result, we may * have scenarios in which a few simultaneous connection attempts prevent * each other from succeeding, but we avoid scenarios where we oversubscribe * the system. * * By also calling RegisterExternalClientBackendCounterDecrement here, we * immediately lower the counter if we throw a FATAL error below. The client * connection counter may temporarily exceed maxClientConnections in between. */ RegisterExternalClientBackendCounterDecrement(); uint32 externalClientCount = IncrementExternalClientBackendCounter(); /* * Limit non-superuser client connections if citus.max_client_connections * is set. */ if (MaxClientConnections >= 0 && !IsSuperuser(port->user_name) && externalClientCount > MaxClientConnections) { ereport(FATAL, (errcode(ERRCODE_TOO_MANY_CONNECTIONS), errmsg("remaining connection slots are reserved for " "non-replication superuser connections"), errdetail("the server is configured to accept up to %d " "regular client connections", MaxClientConnections))); } } /* * Right after this, but before we assign global pid, this backend might * get blocked by a DDL as that happens during parsing. * * That's why, we now initialize its backend data, with the gpid. * * We do this so that this backend gets the chance to show up in * citus_lock_waits. * * We cannot assign a new global PID yet here, because that would require * reading from catalogs, but that's not allowed this early in the * connection startup (because no database has been assigned yet). * * A second reason is for backends that never call StartupCitusBackend. For * those we already set the global PID in the backend data here to be able * to do blocked process detection on connections that are opened over a * replication connection. A replication connection backend will never call * StartupCitusBackend, which normally sets up the global PID. */ InitializeBackendData(port->application_name); IsMainDB = (strncmp(MainDb, "", NAMEDATALEN) == 0 || strncmp(MainDb, port->database_name, NAMEDATALEN) == 0); /* let other authentication hooks to kick in first */ if (original_client_auth_hook) { original_client_auth_hook(port, status); } } /* * IsSuperuser returns whether the role with the given name is superuser. If * the user doesn't exist, this simply returns false instead of throwing an * error. This is done to not leak information about users existing or not, in * some cases postgres is vague about this on purpose. So, by returning false * we let postgres return this possibly vague error message. */ static bool IsSuperuser(char *roleName) { if (roleName == NULL) { return false; } HeapTuple roleTuple = SearchSysCache1(AUTHNAME, CStringGetDatum(roleName)); if (!HeapTupleIsValid(roleTuple)) { return false; } Form_pg_authid rform = (Form_pg_authid) GETSTRUCT(roleTuple); bool isSuperuser = rform->rolsuper; ReleaseSysCache(roleTuple); return isSuperuser; } /* * CitusObjectAccessHook is called when an object is created. * * We currently use it to track CREATE EXTENSION citus; operations to make sure we * clear the metadata if the transaction is rolled back. */ static void CitusObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg) { if (PrevObjectAccessHook) { PrevObjectAccessHook(access, classId, objectId, subId, arg); } /* Checks if the access is post_create and that it's an extension id */ if (access == OAT_POST_CREATE && classId == ExtensionRelationId) { /* There's currently an engine bug that makes it difficult to check * the provided objectId with extension oid so we will set the value * regardless if it's citus being created */ SetCreateCitusTransactionLevel(GetCurrentTransactionNestLevel()); } } /* * EnableChangeDataCaptureAssignHook is called whenever the * citus.enable_change_data_capture setting is changed to dynamically * adjust the dynamic_library_path based on the new value. */ static void EnableChangeDataCaptureAssignHook(bool newval, void *extra) { if (newval) { /* CDC enabled: add citus_decoders to the path */ AdjustDynamicLibraryPathForCdcDecoders(); } } ================================================ FILE: src/backend/distributed/sql/cat_upgrades/add_clone_info_to_pg_dist_node.sql ================================================ -- Add replica information columns to pg_dist_node ALTER TABLE pg_catalog.pg_dist_node ADD COLUMN nodeisclone BOOLEAN NOT NULL DEFAULT FALSE; ALTER TABLE pg_catalog.pg_dist_node ADD COLUMN nodeprimarynodeid INT4 NOT NULL DEFAULT 0; -- Add a comment to the table and columns for clarity in \d output COMMENT ON COLUMN pg_catalog.pg_dist_node.nodeisclone IS 'Indicates if this node is a replica of another node.'; COMMENT ON COLUMN pg_catalog.pg_dist_node.nodeprimarynodeid IS 'If nodeisclone is true, this stores the nodeid of its primary node.'; ================================================ FILE: src/backend/distributed/sql/cat_upgrades/remove_clone_info_to_pg_dist_node.sql ================================================ -- Remove clone information columns to pg_dist_node ALTER TABLE pg_catalog.pg_dist_node DROP COLUMN IF EXISTS nodeisclone; ALTER TABLE pg_catalog.pg_dist_node DROP COLUMN IF EXISTS nodeprimarynodeid; ================================================ FILE: src/backend/distributed/sql/citus--10.0-1--10.0-2.sql ================================================ -- citus--10.0-1--10.0-2 --#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.0-1--10.0-2.sql" END IF; END; $check_columnar$; GRANT SELECT ON public.citus_tables TO public; ================================================ FILE: src/backend/distributed/sql/citus--10.0-2--10.0-3.sql ================================================ -- citus--10.0-2--10.0-3 #include "udfs/citus_update_table_statistics/10.0-3.sql" CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.master_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; CREATE OR REPLACE FUNCTION pg_catalog.citus_get_active_worker_nodes(OUT node_name text, OUT node_port bigint) RETURNS SETOF record LANGUAGE C STRICT ROWS 100 AS 'MODULE_PATHNAME', $$citus_get_active_worker_nodes$$; COMMENT ON FUNCTION pg_catalog.citus_get_active_worker_nodes() IS 'fetch set of active worker nodes'; ================================================ FILE: src/backend/distributed/sql/citus--10.0-3--10.0-4.sql ================================================ -- citus--10.0-3--10.0-4 -- This migration file aims to fix 2 issues with upgrades on clusters -- 1. a bug in public schema dependency for citus_tables view. -- -- Users who do not have public schema in their clusters were unable to upgrade -- to Citus 10.x due to the citus_tables view that used to be created in public -- schema #include "udfs/citus_tables/10.0-4.sql" -- 2. a bug in our PG upgrade functions -- -- Users who took the 9.5-2--10.0-1 upgrade path already have the fix, but users -- who took the 9.5-1--10.0-1 upgrade path do not. Hence, we repeat the CREATE OR -- REPLACE from the 9.5-2 definition for citus_prepare_pg_upgrade. #include "udfs/citus_prepare_pg_upgrade/9.5-2.sql" #include "udfs/citus_finish_pg_upgrade/10.0-4.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.0-4--10.1-1.sql ================================================ -- citus--10.0-4--10.1-1 -- add the current database to the distributed objects if not already in there. -- this is to reliably propagate some of the alter database commands that might be -- supported. INSERT INTO citus.pg_dist_object SELECT 'pg_catalog.pg_database'::regclass::oid AS oid, (SELECT oid FROM pg_database WHERE datname = current_database()) as objid, 0 as objsubid ON CONFLICT DO NOTHING; --#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.0-3--10.1-1.sql" END IF; END; $check_columnar$; #include "udfs/create_distributed_table/10.1-1.sql"; #include "udfs/worker_partitioned_relation_total_size/10.1-1.sql" #include "udfs/worker_partitioned_relation_size/10.1-1.sql" #include "udfs/worker_partitioned_table_size/10.1-1.sql" #include "udfs/citus_prepare_pg_upgrade/10.1-1.sql" #include "udfs/citus_finish_pg_upgrade/10.1-1.sql" #include "udfs/citus_local_disk_space_stats/10.1-1.sql" #include "udfs/get_rebalance_table_shards_plan/10.1-1.sql" #include "udfs/citus_add_rebalance_strategy/10.1-1.sql" ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ADD COLUMN improvement_threshold float4 NOT NULL default 0; UPDATE pg_catalog.pg_dist_rebalance_strategy SET improvement_threshold = 0.5 WHERE name = 'by_disk_size'; #include "udfs/get_rebalance_progress/10.1-1.sql" -- use streaming replication when replication factor = 1 WITH replicated_shards AS ( SELECT shardid FROM pg_dist_placement WHERE shardstate = 1 OR shardstate = 3 GROUP BY shardid HAVING count(*) <> 1 ), replicated_relations AS ( SELECT DISTINCT logicalrelid FROM pg_dist_shard JOIN replicated_shards USING (shardid) ) UPDATE pg_dist_partition SET repmodel = 's' WHERE repmodel = 'c' AND partmethod = 'h' AND logicalrelid NOT IN (SELECT * FROM replicated_relations); #include "udfs/citus_shards/10.1-1.sql" DROP TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger ON pg_catalog.pg_dist_rebalance_strategy; DROP FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check(); #include "udfs/citus_cleanup_orphaned_shards/10.1-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.1-1--10.2-1.sql ================================================ -- citus--10.1-1--10.2-1 -- bump version to 10.2-1 DROP FUNCTION IF EXISTS pg_catalog.stop_metadata_sync_to_node(text, integer); GRANT ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) TO PUBLIC; -- the same shard cannot have placements on different nodes ALTER TABLE pg_catalog.pg_dist_placement ADD CONSTRAINT placement_shardid_groupid_unique_index UNIQUE (shardid, groupid); #include "udfs/stop_metadata_sync_to_node/10.2-1.sql" --#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.1-1--10.2-1.sql" END IF; END; $check_columnar$; #include "udfs/citus_internal_add_partition_metadata/10.2-1.sql"; #include "udfs/citus_internal_add_shard_metadata/10.2-1.sql"; #include "udfs/citus_internal_add_placement_metadata/10.2-1.sql"; #include "udfs/citus_internal_update_placement_metadata/10.2-1.sql"; #include "udfs/citus_internal_delete_shard_metadata/10.2-1.sql"; #include "udfs/citus_internal_update_relation_colocation/10.2-1.sql"; #include "udfs/create_time_partitions/10.2-1.sql" #include "udfs/drop_old_time_partitions/10.2-1.sql" #include "udfs/get_missing_time_partition_ranges/10.2-1.sql" #include "udfs/worker_nextval/10.2-1.sql" DROP FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text); CREATE FUNCTION pg_catalog.citus_drop_all_shards(logicalrelid regclass, schema_name text, table_name text, drop_shards_metadata_only boolean default false) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_drop_all_shards$$; COMMENT ON FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text, boolean) IS 'drop all shards in a relation and update metadata'; #include "udfs/citus_drop_trigger/10.2-1.sql"; #include "udfs/citus_prepare_pg_upgrade/10.2-1.sql" #include "udfs/citus_finish_pg_upgrade/10.2-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.2-1--10.2-2.sql ================================================ -- citus--10.2-1--10.2-2 -- bump version to 10.2-2 --#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.2-1--10.2-2.sql" END IF; END; $check_columnar$; ================================================ FILE: src/backend/distributed/sql/citus--10.2-2--10.2-3.sql ================================================ -- citus--10.2-2--10.2-3 -- bump version to 10.2-3 --#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.2-2--10.2-3.sql" END IF; END; $check_columnar$; ================================================ FILE: src/backend/distributed/sql/citus--10.2-3--10.2-4.sql ================================================ -- citus--10.2-3--10.2-4 -- bump version to 10.2-4 DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.2-3--10.2-4.sql" END IF; END; $check_columnar$; #include "udfs/fix_partition_shard_index_names/10.2-4.sql" #include "udfs/fix_all_partition_shard_index_names/10.2-4.sql" #include "udfs/worker_fix_partition_shard_index_names/10.2-4.sql" #include "udfs/citus_finish_pg_upgrade/10.2-4.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.2-4--10.2-5.sql ================================================ #include "udfs/citus_finish_pg_upgrade/10.2-5.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.2-5--10.2-4.sql ================================================ #include "udfs/citus_finish_pg_upgrade/10.2-4.sql" ================================================ FILE: src/backend/distributed/sql/citus--10.2-5--11.0-1.sql ================================================ -- citus--10.2-5--11.0-1 -- bump version to 11.0-1 #include "udfs/citus_disable_node/11.0-1.sql" #include "udfs/create_distributed_function/11.0-1.sql" #include "udfs/citus_check_connection_to_node/11.0-1.sql" #include "udfs/citus_check_cluster_node_health/11.0-1.sql" #include "udfs/citus_shards_on_worker/11.0-1.sql" #include "udfs/citus_shard_indexes_on_worker/11.0-1.sql" #include "udfs/citus_internal_add_object_metadata/11.0-1.sql" #include "udfs/citus_internal_add_colocation_metadata/11.0-1.sql" #include "udfs/citus_internal_delete_colocation_metadata/11.0-1.sql" #include "udfs/citus_run_local_command/11.0-1.sql" #include "udfs/worker_drop_sequence_dependency/11.0-1.sql" #include "udfs/worker_drop_shell_table/11.0-1.sql" #include "udfs/get_all_active_transactions/11.0-1.sql" #include "udfs/get_global_active_transactions/11.0-1.sql" #include "udfs/citus_internal_local_blocked_processes/11.0-1.sql" #include "udfs/citus_internal_global_blocked_processes/11.0-1.sql" #include "udfs/run_command_on_all_nodes/11.0-1.sql" #include "udfs/citus_stat_activity/11.0-1.sql" #include "udfs/worker_create_or_replace_object/11.0-1.sql" #include "udfs/citus_isolation_test_session_is_blocked/11.0-1.sql" #include "udfs/citus_blocking_pids/11.0-1.sql" #include "udfs/citus_calculate_gpid/11.0-1.sql" #include "udfs/citus_backend_gpid/11.0-1.sql" DROP VIEW IF EXISTS pg_catalog.citus_lock_waits; DROP VIEW IF EXISTS pg_catalog.citus_dist_stat_activity; DROP VIEW IF EXISTS pg_catalog.citus_worker_stat_activity; DROP FUNCTION IF EXISTS pg_catalog.citus_dist_stat_activity(); DROP FUNCTION IF EXISTS pg_catalog.citus_worker_stat_activity(); #include "udfs/citus_dist_stat_activity/11.0-1.sql" -- a very simple helper function defined for citus_lock_waits CREATE OR REPLACE FUNCTION get_nodeid_for_groupid(groupIdInput int) RETURNS int AS $$ DECLARE returnNodeNodeId int := 0; begin SELECT nodeId into returnNodeNodeId FROM pg_dist_node WHERE groupid = groupIdInput and nodecluster = current_setting('citus.cluster_name'); RETURN returnNodeNodeId; end $$ LANGUAGE plpgsql; #include "udfs/citus_lock_waits/11.0-1.sql" #include "udfs/pg_cancel_backend/11.0-1.sql" #include "udfs/pg_terminate_backend/11.0-1.sql" #include "udfs/worker_partition_query_result/11.0-1.sql" DROP FUNCTION pg_catalog.master_apply_delete_command(text); DROP FUNCTION pg_catalog.master_get_table_metadata(text); DROP FUNCTION pg_catalog.master_append_table_to_shard(bigint, text, text, integer); -- all existing citus local tables are auto converted -- none of the other tables can have auto-converted as true ALTER TABLE pg_catalog.pg_dist_partition ADD COLUMN autoconverted boolean DEFAULT false; ALTER TABLE citus.pg_dist_object ADD COLUMN force_delegation bool DEFAULT NULL; UPDATE pg_catalog.pg_dist_partition SET autoconverted = TRUE WHERE partmethod = 'n' AND repmodel = 's'; REVOKE ALL ON FUNCTION start_metadata_sync_to_node(text, integer) FROM PUBLIC; REVOKE ALL ON FUNCTION stop_metadata_sync_to_node(text, integer,bool) FROM PUBLIC; DO LANGUAGE plpgsql $$ BEGIN IF EXISTS (SELECT 1 FROM pg_dist_shard where shardstorage = 'c') THEN RAISE EXCEPTION 'cstore_fdw tables are deprecated as of Citus 11.0' USING HINT = 'Install Citus 10.2 and convert your cstore_fdw tables to the columnar access method before upgrading further'; END IF; END; $$; -- Here we keep track of partitioned tables that exists before Citus 11 -- where we need to call fix_all_partition_shard_index_names() before -- metadata is synced. Note that after citus-11, we automatically -- adjust the indexes so we only need to fix existing indexes DO LANGUAGE plpgsql $$ DECLARE partitioned_table_exists bool :=false; BEGIN SELECT count(*) > 0 INTO partitioned_table_exists FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid WHERE c.relkind = 'p'; UPDATE pg_dist_node_metadata SET metadata=jsonb_set(metadata, '{partitioned_citus_table_exists_pre_11}', to_jsonb(partitioned_table_exists), true); END; $$; #include "udfs/citus_finalize_upgrade_to_citus11/11.0-1.sql" ALTER TABLE citus.pg_dist_object SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_object TO public; #include "udfs/citus_prepare_pg_upgrade/11.0-1.sql" #include "udfs/citus_finish_pg_upgrade/11.0-1.sql" #include "udfs/citus_nodename_for_nodeid/11.0-1.sql" #include "udfs/citus_nodeport_for_nodeid/11.0-1.sql" #include "udfs/citus_nodeid_for_gpid/11.0-1.sql" #include "udfs/citus_pid_for_gpid/11.0-1.sql" #include "udfs/citus_coordinator_nodeid/11.0-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.0-1--11.0-2.sql ================================================ #include "udfs/citus_shards_on_worker/11.0-2.sql" #include "udfs/citus_shard_indexes_on_worker/11.0-2.sql" #include "udfs/citus_is_coordinator/11.0-2.sql" #include "udfs/citus_disable_node/11.0-2.sql" #include "udfs/run_command_on_coordinator/11.0-2.sql" #include "udfs/start_metadata_sync_to_all_nodes/11.0-2.sql" #include "udfs/citus_finalize_upgrade_to_citus11/11.0-2.sql" #include "udfs/citus_finish_citus_upgrade/11.0-2.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.0-2--11.0-3.sql ================================================ #include "udfs/citus_finalize_upgrade_to_citus11/11.0-3.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.0-3--11.0-4.sql ================================================ #include "udfs/citus_finish_pg_upgrade/11.0-4.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.0-4--11.0-3.sql ================================================ #include "udfs/citus_finish_pg_upgrade/11.0-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.0-4--11.1-1.sql ================================================ #include "udfs/citus_locks/11.1-1.sql" #include "udfs/citus_tables/11.1-1.sql" #include "udfs/citus_shards/11.1-1.sql" #include "udfs/create_distributed_table_concurrently/11.1-1.sql" #include "udfs/citus_internal_delete_partition_metadata/11.1-1.sql" #include "udfs/citus_copy_shard_placement/11.1-1.sql" -- We should not introduce breaking sql changes to upgrade files after they are released. -- We did that for worker_fetch_foreign_file in v9.0.0 and worker_repartition_cleanup in v9.2.0. -- When we try to drop those udfs in that file, they were missing for some clients unexpectedly -- due to buggy changes in old upgrade scripts. For that case, the fix is to change DROP statements -- with DROP IF EXISTS for those 2 udfs in 11.0-4--11.1-1. -- Fixes an upgrade problem for worker_fetch_foreign_file when upgrade starts from 8.3 up to 11.1 -- Fixes an upgrade problem for worker_repartition_cleanup when upgrade starts from 9.1 up to 11.1 -- Refer the related PR https://github.com/citusdata/citus/pull/6441 for more information DROP FUNCTION IF EXISTS pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]); DROP FUNCTION IF EXISTS pg_catalog.worker_repartition_cleanup(bigint); DROP FUNCTION pg_catalog.worker_create_schema(bigint,text); DROP FUNCTION pg_catalog.worker_cleanup_job_schema_cache(); DROP FUNCTION pg_catalog.worker_fetch_partition_file(bigint, integer, integer, integer, text, integer); DROP FUNCTION pg_catalog.worker_hash_partition_table(bigint, integer, text, text, oid, anyarray); DROP FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[], text[]); DROP FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray); DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--11.0-3--11.1-1.sql" END IF; END; $check_columnar$; -- If upgrading citus, the columnar objects are already being a part of the -- citus extension, and must be detached so that they can be attached -- to the citus_columnar extension. DO $check_citus$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus' and p.proname = 'columnar_handler' ) THEN ALTER EXTENSION citus DROP SCHEMA columnar; ALTER EXTENSION citus DROP SCHEMA columnar_internal; ALTER EXTENSION citus DROP SEQUENCE columnar_internal.storageid_seq; -- columnar tables ALTER EXTENSION citus DROP TABLE columnar_internal.options; ALTER EXTENSION citus DROP TABLE columnar_internal.stripe; ALTER EXTENSION citus DROP TABLE columnar_internal.chunk_group; ALTER EXTENSION citus DROP TABLE columnar_internal.chunk; ALTER EXTENSION citus DROP FUNCTION columnar_internal.columnar_handler; ALTER EXTENSION citus DROP ACCESS METHOD columnar; ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_set; ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_reset; ALTER EXTENSION citus DROP FUNCTION columnar.get_storage_id; -- columnar view ALTER EXTENSION citus DROP VIEW columnar.storage; ALTER EXTENSION citus DROP VIEW columnar.options; ALTER EXTENSION citus DROP VIEW columnar.stripe; ALTER EXTENSION citus DROP VIEW columnar.chunk_group; ALTER EXTENSION citus DROP VIEW columnar.chunk; -- functions under citus_internal for columnar ALTER EXTENSION citus DROP FUNCTION citus_internal.upgrade_columnar_storage; ALTER EXTENSION citus DROP FUNCTION citus_internal.downgrade_columnar_storage; ALTER EXTENSION citus DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog; END IF; END $check_citus$; #include "udfs/citus_prepare_pg_upgrade/11.1-1.sql" #include "udfs/citus_finish_pg_upgrade/11.1-1.sql" DROP FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8); #include "udfs/get_all_active_transactions/11.1-1.sql" #include "udfs/citus_split_shard_by_split_points/11.1-1.sql" #include "udfs/worker_split_copy/11.1-1.sql" #include "udfs/worker_copy_table_to_node/11.1-1.sql" #include "udfs/worker_split_shard_replication_setup/11.1-1.sql" #include "udfs/citus_isolation_test_session_is_blocked/11.1-1.sql" #include "udfs/replicate_reference_tables/11.1-1.sql" #include "udfs/worker_split_shard_release_dsm/11.1-1.sql" DROP FUNCTION pg_catalog.isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text); #include "udfs/isolate_tenant_to_new_shard/11.1-1.sql" -- Table of records to: -- 1) Cleanup leftover resources after a failure -- 2) Deferred drop of old shard placements after a split. #include "udfs/citus_cleanup_orphaned_resources/11.1-1.sql" CREATE TABLE citus.pg_dist_cleanup ( record_id bigint primary key, operation_id bigint not null, object_type int not null, object_name text not null, node_group_id int not null, policy_type int not null ); ALTER TABLE citus.pg_dist_cleanup SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_cleanup TO public; -- Sequence used to generate operation Ids and record Ids in pg_dist_cleanup_record. CREATE SEQUENCE citus.pg_dist_operationid_seq; ALTER SEQUENCE citus.pg_dist_operationid_seq SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_operationid_seq TO public; CREATE SEQUENCE citus.pg_dist_cleanup_recordid_seq; ALTER SEQUENCE citus.pg_dist_cleanup_recordid_seq SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_cleanup_recordid_seq TO public; -- We recreate these two UDF from 11.0-1 on purpose, because we changed their -- old definition. By recreating it here upgrades also pick up the new changes. #include "udfs/pg_cancel_backend/11.0-1.sql" #include "udfs/pg_terminate_backend/11.0-1.sql" CREATE TYPE citus.citus_job_status AS ENUM ('scheduled', 'running', 'finished', 'cancelling', 'cancelled', 'failing', 'failed'); ALTER TYPE citus.citus_job_status SET SCHEMA pg_catalog; CREATE TABLE citus.pg_dist_background_job ( job_id bigserial NOT NULL, state pg_catalog.citus_job_status DEFAULT 'scheduled' NOT NULL, job_type name NOT NULL, description text NOT NULL, started_at timestamptz, finished_at timestamptz, CONSTRAINT pg_dist_background_job_pkey PRIMARY KEY (job_id) ); ALTER TABLE citus.pg_dist_background_job SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_background_job TO PUBLIC; GRANT SELECT ON pg_catalog.pg_dist_background_job_job_id_seq TO PUBLIC; CREATE TYPE citus.citus_task_status AS ENUM ('blocked', 'runnable', 'running', 'done', 'cancelling', 'error', 'unscheduled', 'cancelled'); ALTER TYPE citus.citus_task_status SET SCHEMA pg_catalog; CREATE TABLE citus.pg_dist_background_task( job_id bigint NOT NULL REFERENCES pg_catalog.pg_dist_background_job(job_id), task_id bigserial NOT NULL, owner regrole NOT NULL DEFAULT CURRENT_USER::regrole, pid integer, status pg_catalog.citus_task_status default 'runnable' NOT NULL, command text NOT NULL, retry_count integer, not_before timestamptz, -- can be null to indicate no delay for start of the task, will be set on failure to delay retries message text NOT NULL DEFAULT '', CONSTRAINT pg_dist_background_task_pkey PRIMARY KEY (task_id), CONSTRAINT pg_dist_background_task_job_id_task_id UNIQUE (job_id, task_id) -- required for FK's to enforce tasks only reference other tasks within the same job ); ALTER TABLE citus.pg_dist_background_task SET SCHEMA pg_catalog; CREATE INDEX pg_dist_background_task_status_task_id_index ON pg_catalog.pg_dist_background_task USING btree(status, task_id); GRANT SELECT ON pg_catalog.pg_dist_background_task TO PUBLIC; GRANT SELECT ON pg_catalog.pg_dist_background_task_task_id_seq TO PUBLIC; CREATE TABLE citus.pg_dist_background_task_depend( job_id bigint NOT NULL REFERENCES pg_catalog.pg_dist_background_job(job_id) ON DELETE CASCADE, task_id bigint NOT NULL, depends_on bigint NOT NULL, PRIMARY KEY (job_id, task_id, depends_on), FOREIGN KEY (job_id, task_id) REFERENCES pg_catalog.pg_dist_background_task (job_id, task_id) ON DELETE CASCADE, FOREIGN KEY (job_id, depends_on) REFERENCES pg_catalog.pg_dist_background_task (job_id, task_id) ON DELETE CASCADE ); ALTER TABLE citus.pg_dist_background_task_depend SET SCHEMA pg_catalog; CREATE INDEX pg_dist_background_task_depend_task_id ON pg_catalog.pg_dist_background_task_depend USING btree(job_id, task_id); CREATE INDEX pg_dist_background_task_depend_depends_on ON pg_catalog.pg_dist_background_task_depend USING btree(job_id, depends_on); GRANT SELECT ON pg_catalog.pg_dist_background_task_depend TO PUBLIC; #include "udfs/citus_job_wait/11.1-1.sql" #include "udfs/citus_job_cancel/11.1-1.sql" #include "udfs/citus_rebalance_start/11.1-1.sql" #include "udfs/citus_rebalance_stop/11.1-1.sql" #include "udfs/citus_rebalance_wait/11.1-1.sql" #include "udfs/get_rebalance_progress/11.1-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.1-1--11.2-1.sql ================================================ -- citus--11.1-1--11.2-1 DROP FUNCTION pg_catalog.worker_append_table_to_shard(text, text, text, integer); #include "udfs/get_rebalance_progress/11.2-1.sql" #include "udfs/citus_isolation_test_session_is_blocked/11.2-1.sql" #include "datatypes/citus_cluster_clock/11.2-1.sql" #include "udfs/citus_get_node_clock/11.2-1.sql" #include "udfs/citus_get_transaction_clock/11.2-1.sql" #include "udfs/citus_is_clock_after/11.2-1.sql" #include "udfs/citus_internal_adjust_local_clock_to_remote/11.2-1.sql" #include "udfs/citus_job_list/11.2-1.sql" #include "udfs/citus_job_status/11.2-1.sql" #include "udfs/citus_rebalance_status/11.2-1.sql" #include "udfs/worker_split_shard_replication_setup/11.2-1.sql" #include "udfs/citus_task_wait/11.2-1.sql" #include "udfs/citus_prepare_pg_upgrade/11.2-1.sql" #include "udfs/citus_finish_pg_upgrade/11.2-1.sql" #include "udfs/citus_copy_shard_placement/11.2-1.sql" #include "udfs/citus_move_shard_placement/11.2-1.sql" #include "udfs/citus_internal_add_placement_metadata/11.2-1.sql"; -- drop orphaned shards after inserting records for them into pg_dist_cleanup INSERT INTO pg_dist_cleanup SELECT nextval('pg_dist_cleanup_recordid_seq'), 0, 1, shard_name(sh.logicalrelid, sh.shardid) AS object_name, plc.groupid AS node_group_id, 0 FROM pg_dist_placement plc JOIN pg_dist_shard sh ON sh.shardid = plc.shardid WHERE plc.shardstate = 4; DELETE FROM pg_dist_placement WHERE shardstate = 4; ================================================ FILE: src/backend/distributed/sql/citus--11.2-1--11.2-2.sql ================================================ -- citus--11.2-1--11.2-2 #include "udfs/worker_adjust_identity_column_seq_ranges/11.2-2.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.2-2--11.3-1.sql ================================================ -- citus--11.2-1--11.3-1 #include "udfs/repl_origin_helper/11.3-1.sql" ALTER TABLE pg_catalog.pg_dist_authinfo REPLICA IDENTITY USING INDEX pg_dist_authinfo_identification_index; ALTER TABLE pg_catalog.pg_dist_partition REPLICA IDENTITY USING INDEX pg_dist_partition_logical_relid_index; ALTER TABLE pg_catalog.pg_dist_placement REPLICA IDENTITY USING INDEX pg_dist_placement_placementid_index; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy REPLICA IDENTITY USING INDEX pg_dist_rebalance_strategy_name_key; ALTER TABLE pg_catalog.pg_dist_shard REPLICA IDENTITY USING INDEX pg_dist_shard_shardid_index; ALTER TABLE pg_catalog.pg_dist_transaction REPLICA IDENTITY USING INDEX pg_dist_transaction_unique_constraint; #include "udfs/worker_drop_all_shell_tables/11.3-1.sql" #include "udfs/citus_internal_mark_node_not_synced/11.3-1.sql" #include "udfs/citus_stat_tenants_local/11.3-1.sql" #include "udfs/citus_stat_tenants/11.3-1.sql" #include "udfs/citus_stat_tenants_local_reset/11.3-1.sql" #include "udfs/citus_stat_tenants_reset/11.3-1.sql" -- we introduce nodes_involved, which will be used internally to -- limit the number of parallel tasks running per node ALTER TABLE pg_catalog.pg_dist_background_task ADD COLUMN nodes_involved int[] DEFAULT NULL; ================================================ FILE: src/backend/distributed/sql/citus--11.3-1--11.3-2.sql ================================================ DROP VIEW citus_shards; DROP VIEW IF EXISTS pg_catalog.citus_tables; DROP VIEW IF EXISTS public.citus_tables; DROP FUNCTION citus_shard_sizes; #include "udfs/citus_shard_sizes/11.3-2.sql" #include "udfs/citus_shards/11.3-2.sql" #include "udfs/citus_tables/11.3-2.sql" ================================================ FILE: src/backend/distributed/sql/citus--11.3-2--12.0-1.sql ================================================ -- citus--11.3-1--12.0-1 -- bump version to 12.0-1 CREATE TABLE citus.pg_dist_schema ( schemaid oid NOT NULL, colocationid int NOT NULL, CONSTRAINT pg_dist_schema_pkey PRIMARY KEY (schemaid), CONSTRAINT pg_dist_schema_unique_colocationid_index UNIQUE (colocationid) ); ALTER TABLE citus.pg_dist_schema SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_schema TO public; -- udfs used to modify pg_dist_schema on workers, to sync metadata #include "udfs/citus_internal_add_tenant_schema/12.0-1.sql" #include "udfs/citus_internal_delete_tenant_schema/12.0-1.sql" #include "udfs/citus_prepare_pg_upgrade/12.0-1.sql" #include "udfs/citus_finish_pg_upgrade/12.0-1.sql" -- udfs used to modify pg_dist_schema globally via drop trigger #include "udfs/citus_internal_unregister_tenant_schema_globally/12.0-1.sql" #include "udfs/citus_drop_trigger/12.0-1.sql" #include "udfs/citus_tables/12.0-1.sql" DROP VIEW citus_shards; #include "udfs/citus_shards/12.0-1.sql" #include "udfs/citus_schemas/12.0-1.sql" -- udfs used to include schema-based tenants in tenant monitoring #include "udfs/citus_stat_tenants_local/12.0-1.sql" -- udfs to convert a regular/tenant schema to a tenant/regular schema #include "udfs/citus_schema_distribute/12.0-1.sql" #include "udfs/citus_schema_undistribute/12.0-1.sql" #include "udfs/drop_old_time_partitions/12.0-1.sql" #include "udfs/get_missing_time_partition_ranges/12.0-1.sql" -- Update the default rebalance strategy to 'by_disk_size', but only if the -- default is currently 'by_shard_count' SELECT citus_set_default_rebalance_strategy(name) FROM pg_dist_rebalance_strategy WHERE name = 'by_disk_size' AND (SELECT default_strategy FROM pg_dist_rebalance_strategy WHERE name = 'by_shard_count'); ================================================ FILE: src/backend/distributed/sql/citus--12.0-1--12.1-1.sql ================================================ -- citus--12.0-1--12.1-1 -- bump version to 12.1-1 #include "udfs/citus_pause_node_within_txn/12.1-1.sql" #include "udfs/citus_prepare_pg_upgrade/12.1-1.sql" #include "udfs/citus_finish_pg_upgrade/12.1-1.sql" #include "udfs/citus_internal_update_none_dist_table_metadata/12.1-1.sql" #include "udfs/citus_internal_delete_placement_metadata/12.1-1.sql" #include "udfs/citus_schema_move/12.1-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--12.1-1--13.0-1.sql ================================================ -- citus--12.1-1--13.0-1.sql -- bump version to 13.0-1 #include "udfs/citus_prepare_pg_upgrade/13.0-1.sql" #include "udfs/create_time_partitions/13.0-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--13.0-1--13.1-1.sql ================================================ -- citus--13.0-1--13.1-1 -- bump version to 13.1-1 #include "udfs/citus_internal_database_command/13.1-1.sql" #include "udfs/citus_add_rebalance_strategy/13.1-1.sql" DROP FUNCTION pg_catalog.citus_unmark_object_distributed(oid, oid, int); #include "udfs/citus_unmark_object_distributed/13.1-1.sql" ALTER TABLE pg_catalog.pg_dist_transaction ADD COLUMN outer_xid xid8; #include "udfs/citus_internal_acquire_citus_advisory_object_class_lock/13.1-1.sql" GRANT USAGE ON SCHEMA citus_internal TO PUBLIC; REVOKE ALL ON FUNCTION citus_internal.find_groupid_for_node FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.pg_dist_node_trigger_func FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.pg_dist_rebalance_strategy_trigger_func FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.pg_dist_shard_placement_trigger_func FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.refresh_isolation_tester_prepared_statement FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.replace_isolation_tester_func FROM PUBLIC; REVOKE ALL ON FUNCTION citus_internal.restore_isolation_tester_func FROM PUBLIC; #include "udfs/citus_internal_add_colocation_metadata/13.1-1.sql" #include "udfs/citus_internal_add_object_metadata/13.1-1.sql" #include "udfs/citus_internal_add_partition_metadata/13.1-1.sql" #include "udfs/citus_internal_add_placement_metadata/13.1-1.sql" #include "udfs/citus_internal_add_shard_metadata/13.1-1.sql" #include "udfs/citus_internal_add_tenant_schema/13.1-1.sql" #include "udfs/citus_internal_adjust_local_clock_to_remote/13.1-1.sql" #include "udfs/citus_internal_delete_colocation_metadata/13.1-1.sql" #include "udfs/citus_internal_delete_partition_metadata/13.1-1.sql" #include "udfs/citus_internal_delete_placement_metadata/13.1-1.sql" #include "udfs/citus_internal_delete_shard_metadata/13.1-1.sql" #include "udfs/citus_internal_delete_tenant_schema/13.1-1.sql" #include "udfs/citus_internal_local_blocked_processes/13.1-1.sql" #include "udfs/citus_internal_global_blocked_processes/13.1-1.sql" #include "udfs/citus_blocking_pids/13.1-1.sql" #include "udfs/citus_isolation_test_session_is_blocked/13.1-1.sql" DROP VIEW IF EXISTS pg_catalog.citus_lock_waits; #include "udfs/citus_lock_waits/13.1-1.sql" #include "udfs/citus_internal_mark_node_not_synced/13.1-1.sql" #include "udfs/citus_internal_unregister_tenant_schema_globally/13.1-1.sql" #include "udfs/citus_drop_trigger/13.1-1.sql" #include "udfs/citus_internal_update_none_dist_table_metadata/13.1-1.sql" #include "udfs/citus_internal_update_placement_metadata/13.1-1.sql" #include "udfs/citus_internal_update_relation_colocation/13.1-1.sql" #include "udfs/repl_origin_helper/13.1-1.sql" #include "udfs/citus_finish_pg_upgrade/13.1-1.sql" #include "udfs/citus_is_primary_node/13.1-1.sql" #include "udfs/citus_stat_counters/13.1-1.sql" #include "udfs/citus_stat_counters_reset/13.1-1.sql" #include "udfs/citus_nodes/13.1-1.sql" -- Since shard_name/13.1-1.sql first drops the function and then creates it, we first -- need to drop citus_shards view since that view depends on this function. And immediately -- after creating the function, we recreate citus_shards view again. DROP VIEW pg_catalog.citus_shards; #include "udfs/shard_name/13.1-1.sql" #include "udfs/citus_shards/12.0-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--13.1-1--13.2-1.sql ================================================ -- citus--13.1-1--13.2-1 -- bump version to 13.2-1 #include "udfs/worker_last_saved_explain_analyze/13.2-1.sql" #include "cat_upgrades/add_clone_info_to_pg_dist_node.sql" #include "udfs/citus_add_clone_node/13.2-1.sql" #include "udfs/citus_remove_clone_node/13.2-1.sql" #include "udfs/citus_promote_clone_and_rebalance/13.2-1.sql" #include "udfs/get_snapshot_based_node_split_plan/13.2-1.sql" #include "udfs/citus_rebalance_start/13.2-1.sql" #include "udfs/citus_internal_copy_single_shard_placement/13.2-1.sql" #include "udfs/citus_finish_pg_upgrade/13.2-1.sql" #include "udfs/citus_stats/13.2-1.sql" DO $drop_leftover_old_columnar_objects$ BEGIN -- If old columnar exists, i.e., the columnar access method that we had before Citus 11.1, -- and we don't have any relations using the old columnar, then we want to drop the columnar -- objects. This is because, we don't want to automatically create the "citus_columnar" -- extension together with the "citus" extension anymore. And for the cases where we don't -- want to automatically create the "citus_columnar" extension, there is no point of keeping -- the columnar objects that we had before Citus 11.1 around. IF ( SELECT EXISTS ( SELECT 1 FROM pg_am WHERE -- looking for an access method whose name is "columnar" .. pg_am.amname = 'columnar' AND -- .. and there should *NOT* be such a dependency edge in pg_depend, where .. NOT EXISTS ( SELECT 1 FROM pg_depend WHERE -- .. the depender is columnar access method (2601 = access method class) .. pg_depend.classid = 2601 AND pg_depend.objid = pg_am.oid AND pg_depend.objsubid = 0 AND -- .. and the dependee is an extension (3079 = extension class) pg_depend.refclassid = 3079 AND pg_depend.refobjsubid = 0 LIMIT 1 ) AND -- .. and there should *NOT* be any relations using it NOT EXISTS ( SELECT 1 FROM pg_class WHERE pg_class.relam = pg_am.oid LIMIT 1 ) ) ) THEN -- Below we drop the columnar objects in such an order that the objects that depend on -- other objects are dropped first. DROP VIEW IF EXISTS columnar.options; DROP VIEW IF EXISTS columnar.stripe; DROP VIEW IF EXISTS columnar.chunk_group; DROP VIEW IF EXISTS columnar.chunk; DROP VIEW IF EXISTS columnar.storage; DROP ACCESS METHOD IF EXISTS columnar; DROP SEQUENCE IF EXISTS columnar_internal.storageid_seq; DROP TABLE IF EXISTS columnar_internal.options; DROP TABLE IF EXISTS columnar_internal.stripe; DROP TABLE IF EXISTS columnar_internal.chunk_group; DROP TABLE IF EXISTS columnar_internal.chunk; DROP FUNCTION IF EXISTS columnar_internal.columnar_handler; DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_set; DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_reset; DROP FUNCTION IF EXISTS columnar.get_storage_id; DROP FUNCTION IF EXISTS citus_internal.upgrade_columnar_storage; DROP FUNCTION IF EXISTS citus_internal.downgrade_columnar_storage; DROP FUNCTION IF EXISTS citus_internal.columnar_ensure_am_depends_catalog; DROP SCHEMA IF EXISTS columnar; DROP SCHEMA IF EXISTS columnar_internal; END IF; END $drop_leftover_old_columnar_objects$; ================================================ FILE: src/backend/distributed/sql/citus--13.2-1--14.0-1.sql ================================================ -- citus--13.2-1--14.0-1 -- bump version to 14.0-1 #include "udfs/citus_prepare_pg_upgrade/14.0-1.sql" #include "udfs/citus_finish_pg_upgrade/14.0-1.sql" #include "udfs/worker_binary_partial_agg_ffunc/14.0-1.sql" #include "udfs/coord_binary_combine_agg_sfunc/14.0-1.sql" #include "udfs/coord_binary_combine_agg_ffunc/14.0-1.sql" #include "udfs/worker_binary_partial_agg/14.0-1.sql" #include "udfs/coord_binary_combine_agg/14.0-1.sql" #include "udfs/fix_pre_citus14_colocation_group_collation_mismatches/14.0-1.sql" #include "udfs/citus_finish_citus_upgrade/14.0-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--14.0-1--15.0-1.sql ================================================ -- citus--14.0-1--15.0-1 -- bump version to 15.0-1 ================================================ FILE: src/backend/distributed/sql/citus--8.0-1--8.0-2.sql ================================================ -- citus--7.5-7--8.0-1 SET search_path = 'pg_catalog'; CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$relation_is_a_known_shard$$; COMMENT ON FUNCTION relation_is_a_known_shard(regclass) IS 'returns true if the given relation is a known shard'; CREATE OR REPLACE FUNCTION pg_catalog.citus_table_is_visible(oid) RETURNS bool LANGUAGE C STRICT STABLE PARALLEL SAFE AS 'MODULE_PATHNAME', $$citus_table_is_visible$$; COMMENT ON FUNCTION citus_table_is_visible(oid) IS 'wrapper on pg_table_is_visible, filtering out tables (and indexes) that are known to be shards'; -- this is the exact same query with what \d -- command produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) CREATE VIEW citus.citus_shards_on_worker AS SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','S','f','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; ALTER VIEW citus.citus_shards_on_worker SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shards_on_worker TO public; -- this is the exact same query with what \di -- command produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) CREATE VIEW citus.citus_shard_indexes_on_worker AS SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; ALTER VIEW citus.citus_shard_indexes_on_worker SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shard_indexes_on_worker TO public; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-1.sql ================================================ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION citus" to load this file. \quit CREATE SCHEMA citus; SET search_path = 'pg_catalog'; -- Enable SSL to encrypt all trafic by default -- create temporary UDF that has the power to change settings within postgres and drop it -- after ssl has been setup. CREATE FUNCTION citus_setup_ssl() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_setup_ssl$$; DO LANGUAGE plpgsql $$ BEGIN -- setup ssl when postgres is OpenSSL-enabled IF current_setting('ssl_ciphers') != 'none' THEN PERFORM citus_setup_ssl(); END IF; END; $$; DROP FUNCTION citus_setup_ssl(); -- Citus data types CREATE TYPE citus.distribution_type AS ENUM ( 'hash', 'range', 'append' ); -- Citus tables & corresponding indexes CREATE TABLE citus.pg_dist_partition( logicalrelid regclass NOT NULL, partmethod "char" NOT NULL, partkey text, colocationid integer DEFAULT 0 NOT NULL, repmodel "char" DEFAULT 'c' NOT NULL ); -- SELECT granted to PUBLIC in upgrade script CREATE UNIQUE INDEX pg_dist_partition_logical_relid_index ON citus.pg_dist_partition using btree(logicalrelid); ALTER TABLE citus.pg_dist_partition SET SCHEMA pg_catalog; CREATE INDEX pg_dist_partition_colocationid_index ON pg_catalog.pg_dist_partition using btree(colocationid); CREATE TABLE citus.pg_dist_shard( logicalrelid regclass NOT NULL, shardid int8 NOT NULL, shardstorage "char" NOT NULL, shardalias text, shardminvalue text, shardmaxvalue text ); -- ALTER-after-CREATE to keep table tuple layout consistent -- with earlier versions of Citus. ALTER TABLE citus.pg_dist_shard DROP shardalias; -- SELECT granted to PUBLIC in upgrade script CREATE UNIQUE INDEX pg_dist_shard_shardid_index ON citus.pg_dist_shard using btree(shardid); CREATE INDEX pg_dist_shard_logical_relid_index ON citus.pg_dist_shard using btree(logicalrelid); ALTER TABLE citus.pg_dist_shard SET SCHEMA pg_catalog; CREATE SEQUENCE citus.pg_dist_shard_placement_placementid_seq NO CYCLE; ALTER SEQUENCE citus.pg_dist_shard_placement_placementid_seq SET SCHEMA pg_catalog; CREATE TABLE citus.pg_dist_shard_placement( shardid int8 NOT NULL, shardstate int4 NOT NULL, shardlength int8 NOT NULL, nodename text NOT NULL, nodeport int8 NOT NULL, placementid bigint NOT NULL DEFAULT nextval('pg_catalog.pg_dist_shard_placement_placementid_seq') ); -- SELECT granted to PUBLIC in upgrade script CREATE UNIQUE INDEX pg_dist_shard_placement_placementid_index ON citus.pg_dist_shard_placement using btree(placementid); CREATE INDEX pg_dist_shard_placement_shardid_index ON citus.pg_dist_shard_placement using btree(shardid); CREATE INDEX pg_dist_shard_placement_nodeid_index ON citus.pg_dist_shard_placement using btree(nodename, nodeport); ALTER TABLE citus.pg_dist_shard_placement SET SCHEMA pg_catalog; -- Citus sequences -- Internal sequence to generate 64-bit shard ids. These identifiers are then -- used to identify shards in the distributed database. CREATE SEQUENCE citus.pg_dist_shardid_seq MINVALUE 102008 NO CYCLE; ALTER SEQUENCE citus.pg_dist_shardid_seq SET SCHEMA pg_catalog; -- Internal sequence to generate 32-bit jobIds. These identifiers are then -- used to identify jobs in the distributed database; and they wrap at 32-bits -- to allow for worker nodes to independently execute their distributed jobs. CREATE SEQUENCE citus.pg_dist_jobid_seq MINVALUE 2 -- first jobId reserved for clean up jobs MAXVALUE 4294967296; ALTER SEQUENCE citus.pg_dist_jobid_seq SET SCHEMA pg_catalog; -- Citus functions -- master_* functions CREATE FUNCTION master_get_table_metadata(relation_name text, OUT logical_relid oid, OUT part_storage_type "char", OUT part_method "char", OUT part_key text, OUT part_replica_count integer, OUT part_max_size bigint, OUT part_placement_policy integer) RETURNS record LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$master_get_table_metadata$$; COMMENT ON FUNCTION master_get_table_metadata(relation_name text) IS 'fetch metadata values for the table'; CREATE FUNCTION master_get_table_ddl_events(text) RETURNS SETOF text LANGUAGE C STRICT ROWS 100 AS 'MODULE_PATHNAME', $$master_get_table_ddl_events$$; COMMENT ON FUNCTION master_get_table_ddl_events(text) IS 'fetch set of ddl statements for the table'; CREATE FUNCTION master_get_new_shardid() RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_get_new_shardid$$; COMMENT ON FUNCTION master_get_new_shardid() IS 'fetch unique shardId'; CREATE FUNCTION master_create_empty_shard(text) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_create_empty_shard$$; COMMENT ON FUNCTION master_create_empty_shard(text) IS 'create an empty shard and shard placements for the table'; CREATE FUNCTION master_append_table_to_shard(bigint, text, text, integer) RETURNS real LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_append_table_to_shard$$; COMMENT ON FUNCTION master_append_table_to_shard(bigint, text, text, integer) IS 'append given table to all shard placements and update metadata'; CREATE FUNCTION master_drop_all_shards(logicalrelid regclass, schema_name text, table_name text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drop_all_shards$$; COMMENT ON FUNCTION master_drop_all_shards(regclass, text, text) IS 'drop all shards in a relation and update metadata'; CREATE FUNCTION master_apply_delete_command(text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_apply_delete_command$$; COMMENT ON FUNCTION master_apply_delete_command(text) IS 'drop shards matching delete criteria and update metadata'; CREATE FUNCTION master_get_active_worker_nodes(OUT node_name text, OUT node_port bigint) RETURNS SETOF record LANGUAGE C STRICT ROWS 100 AS 'MODULE_PATHNAME', $$master_get_active_worker_nodes$$; COMMENT ON FUNCTION master_get_active_worker_nodes() IS 'fetch set of active worker nodes'; CREATE FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, distribution_method citus.distribution_type) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_create_distributed_table$$; COMMENT ON FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, distribution_method citus.distribution_type) IS 'define the table distribution functions'; -- define shard creation function for hash-partitioned tables CREATE FUNCTION master_create_worker_shards(table_name text, shard_count integer, replication_factor integer DEFAULT 2) RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; -- task_tracker_* functions CREATE FUNCTION task_tracker_assign_task(bigint, integer, text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_assign_task$$; COMMENT ON FUNCTION task_tracker_assign_task(bigint, integer, text) IS 'assign a task to execute'; CREATE FUNCTION task_tracker_task_status(bigint, integer) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_task_status$$; COMMENT ON FUNCTION task_tracker_task_status(bigint, integer) IS 'check an assigned task''s execution status'; CREATE FUNCTION task_tracker_cleanup_job(bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_cleanup_job$$; COMMENT ON FUNCTION task_tracker_cleanup_job(bigint) IS 'clean up all tasks associated with a job'; -- worker_* functions CREATE FUNCTION worker_fetch_partition_file(bigint, integer, integer, integer, text, integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fetch_partition_file$$; COMMENT ON FUNCTION worker_fetch_partition_file(bigint, integer, integer, integer, text, integer) IS 'fetch partition file from remote node'; CREATE FUNCTION worker_range_partition_table(bigint, integer, text, text, oid, anyarray) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_range_partition_table$$; COMMENT ON FUNCTION worker_range_partition_table(bigint, integer, text, text, oid, anyarray) IS 'range partition query results'; CREATE FUNCTION worker_hash_partition_table(bigint, integer, text, text, oid, anyarray) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_hash_partition_table$$; COMMENT ON FUNCTION worker_hash_partition_table(bigint, integer, text, text, oid, anyarray) IS 'hash partition query results'; CREATE FUNCTION worker_merge_files_into_table(bigint, integer, text[], text[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_merge_files_into_table$$; COMMENT ON FUNCTION worker_merge_files_into_table(bigint, integer, text[], text[]) IS 'merge files into a table'; CREATE FUNCTION worker_merge_files_and_run_query(bigint, integer, text, text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_merge_files_and_run_query$$; COMMENT ON FUNCTION worker_merge_files_and_run_query(bigint, integer, text, text) IS 'merge files and run a reduce query on merged files'; CREATE FUNCTION worker_cleanup_job_schema_cache() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_cleanup_job_schema_cache$$; COMMENT ON FUNCTION worker_cleanup_job_schema_cache() IS 'cleanup all job schemas in current database'; CREATE FUNCTION worker_append_table_to_shard(text, text, text, integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_append_table_to_shard$$; COMMENT ON FUNCTION worker_append_table_to_shard(text, text, text, integer) IS 'append a regular table''s contents to the shard'; CREATE FUNCTION master_drop_sequences(sequence_names text[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drop_sequences$$; COMMENT ON FUNCTION master_drop_sequences(text[]) IS 'drop specified sequences from the cluster'; CREATE FUNCTION master_dist_partition_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_partition_cache_invalidate$$; COMMENT ON FUNCTION master_dist_partition_cache_invalidate() IS 'register relcache invalidation for changed rows'; CREATE FUNCTION master_dist_shard_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_shard_cache_invalidate$$; COMMENT ON FUNCTION master_dist_shard_cache_invalidate() IS 'register relcache invalidation for changed rows'; -- internal functions, not user accessible CREATE FUNCTION citus_extradata_container(INTERNAL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_extradata_container$$; COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; -- Citus triggers CREATE TRIGGER dist_partition_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON pg_catalog.pg_dist_partition FOR EACH ROW EXECUTE PROCEDURE master_dist_partition_cache_invalidate(); CREATE TRIGGER dist_shard_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON pg_catalog.pg_dist_shard FOR EACH ROW EXECUTE PROCEDURE master_dist_shard_cache_invalidate(); -- Citus aggregates DO $proc$ BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $$ CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; $$; ELSE EXECUTE $$ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; $$; END IF; END$proc$; GRANT SELECT ON pg_catalog.pg_dist_partition TO public; GRANT SELECT ON pg_catalog.pg_dist_shard TO public; GRANT SELECT ON pg_catalog.pg_dist_shard_placement TO public; -- empty, but required to update the extension version CREATE FUNCTION pg_catalog.master_modify_multiple_shards(text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_modify_multiple_shards$$; COMMENT ON FUNCTION master_modify_multiple_shards(text) IS 'push delete and update queries to shards'; CREATE FUNCTION pg_catalog.master_update_shard_statistics(shard_id bigint) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_update_shard_statistics$$; COMMENT ON FUNCTION master_update_shard_statistics(bigint) IS 'updates shard statistics and returns the updated shard size'; CREATE FUNCTION pg_catalog.worker_apply_shard_ddl_command(bigint, text, text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_apply_shard_ddl_command$$; COMMENT ON FUNCTION worker_apply_shard_ddl_command(bigint, text, text) IS 'extend ddl command with shardId and apply on database'; CREATE FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fetch_foreign_file$$; COMMENT ON FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]) IS 'fetch foreign file from remote node and apply file'; CREATE FUNCTION pg_catalog.worker_apply_shard_ddl_command(bigint, text) RETURNS void LANGUAGE sql AS $worker_apply_shard_ddl_command$ SELECT pg_catalog.worker_apply_shard_ddl_command($1, 'public', $2); $worker_apply_shard_ddl_command$; COMMENT ON FUNCTION worker_apply_shard_ddl_command(bigint, text) IS 'extend ddl command with shardId and apply on database'; CREATE FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$shard_name$$; COMMENT ON FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint) IS 'returns schema-qualified, shard-extended identifier of object name'; CREATE SEQUENCE citus.pg_dist_groupid_seq MINVALUE 1 MAXVALUE 4294967296; CREATE SEQUENCE citus.pg_dist_node_nodeid_seq MINVALUE 1 MAXVALUE 4294967296; ALTER SEQUENCE citus.pg_dist_groupid_seq SET SCHEMA pg_catalog; ALTER SEQUENCE citus.pg_dist_node_nodeid_seq SET SCHEMA pg_catalog; -- add pg_dist_node CREATE TABLE citus.pg_dist_node( nodeid int NOT NULL DEFAULT nextval('pg_dist_groupid_seq') PRIMARY KEY, groupid int NOT NULL DEFAULT nextval('pg_dist_node_nodeid_seq'), nodename text NOT NULL, nodeport int NOT NULL DEFAULT 5432, noderack text NOT NULL DEFAULT 'default', UNIQUE (nodename, nodeport) ); -- ALTER-after-CREATE to preserve table tuple layout ALTER TABLE citus.pg_dist_node ADD hasmetadata bool NOT NULL DEFAULT false, ADD isactive bool NOT NULL DEFAULT true; ALTER TABLE citus.pg_dist_node SET SCHEMA pg_catalog; CREATE FUNCTION master_dist_node_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_node_cache_invalidate$$; COMMENT ON FUNCTION master_dist_node_cache_invalidate() IS 'invalidate internal cache of nodes when pg_dist_nodes changes'; CREATE TRIGGER dist_node_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON pg_catalog.pg_dist_node FOR EACH ROW EXECUTE PROCEDURE master_dist_node_cache_invalidate(); CREATE FUNCTION master_remove_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_remove_node$$; COMMENT ON FUNCTION master_remove_node(nodename text, nodeport integer) IS 'remove node from the cluster'; CREATE FUNCTION pg_catalog.master_get_new_placementid() RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_get_new_placementid$$; COMMENT ON FUNCTION pg_catalog.master_get_new_placementid() IS 'fetch unique placementid'; CREATE FUNCTION pg_catalog.worker_drop_distributed_table(logicalrelid Oid) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_distributed_table$$; COMMENT ON FUNCTION pg_catalog.worker_drop_distributed_table(logicalrelid Oid) IS 'drop the clustered table and its reference from metadata tables'; CREATE FUNCTION pg_catalog.column_name_to_column(table_name regclass, column_name text) RETURNS text LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$column_name_to_column$$; COMMENT ON FUNCTION pg_catalog.column_name_to_column(table_name regclass, column_name text) IS 'convert a column name to its textual Var representation'; CREATE FUNCTION pg_catalog.get_colocated_table_array(regclass) RETURNS regclass[] AS 'citus' LANGUAGE C STRICT; CREATE TABLE citus.pg_dist_local_group( groupid int NOT NULL PRIMARY KEY) ; -- insert the default value for being the coordinator node INSERT INTO citus.pg_dist_local_group VALUES (0); ALTER TABLE citus.pg_dist_local_group SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_local_group TO public; CREATE TABLE citus.pg_dist_transaction ( groupid int NOT NULL, gid text NOT NULL ); CREATE INDEX pg_dist_transaction_group_index ON citus.pg_dist_transaction using btree(groupid); ALTER TABLE citus.pg_dist_transaction SET SCHEMA pg_catalog; ALTER TABLE pg_catalog.pg_dist_transaction ADD CONSTRAINT pg_dist_transaction_unique_constraint UNIQUE (groupid, gid); GRANT SELECT ON pg_catalog.pg_dist_transaction TO public; CREATE FUNCTION pg_catalog.recover_prepared_transactions() RETURNS int LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$recover_prepared_transactions$$; COMMENT ON FUNCTION pg_catalog.recover_prepared_transactions() IS 'recover prepared transactions started by this node'; CREATE SEQUENCE citus.pg_dist_colocationid_seq MINVALUE 1 MAXVALUE 4294967296; ALTER SEQUENCE citus.pg_dist_colocationid_seq SET SCHEMA pg_catalog; -- add pg_dist_colocation CREATE TABLE citus.pg_dist_colocation( colocationid int NOT NULL PRIMARY KEY, shardcount int NOT NULL, replicationfactor int NOT NULL, distributioncolumntype oid NOT NULL ); ALTER TABLE citus.pg_dist_colocation SET SCHEMA pg_catalog; CREATE INDEX pg_dist_colocation_configuration_index ON pg_dist_colocation USING btree(shardcount, replicationfactor, distributioncolumntype); CREATE FUNCTION create_reference_table(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$create_reference_table$$; COMMENT ON FUNCTION create_reference_table(table_name regclass) IS 'create a distributed reference table'; CREATE FUNCTION pg_catalog.worker_apply_inter_shard_ddl_command(referencing_shard bigint, referencing_schema_name text, referenced_shard bigint, referenced_schema_name text, command text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_apply_inter_shard_ddl_command$$; COMMENT ON FUNCTION pg_catalog.worker_apply_inter_shard_ddl_command(referencing_shard bigint, referencing_schema_name text, referenced_shard bigint, referenced_schema_name text, command text) IS 'executes inter shard ddl command'; CREATE FUNCTION pg_catalog.master_dist_placement_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_placement_cache_invalidate$$; COMMENT ON FUNCTION master_dist_placement_cache_invalidate() IS 'register relcache invalidation for changed placements'; CREATE TRIGGER dist_placement_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON pg_catalog.pg_dist_shard_placement FOR EACH ROW EXECUTE PROCEDURE master_dist_placement_cache_invalidate(); CREATE FUNCTION mark_tables_colocated(source_table_name regclass, target_table_names regclass[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$mark_tables_colocated$$; COMMENT ON FUNCTION mark_tables_colocated(source_table_name regclass, target_table_names regclass[]) IS 'mark target distributed tables as colocated with the source table'; CREATE FUNCTION start_metadata_sync_to_node(nodename text, nodeport integer) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$start_metadata_sync_to_node$$; COMMENT ON FUNCTION start_metadata_sync_to_node(nodename text, nodeport integer) IS 'sync metadata to node'; CREATE FUNCTION worker_create_truncate_trigger(table_name regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_truncate_trigger$$; COMMENT ON FUNCTION worker_create_truncate_trigger(tablename regclass) IS 'create truncate trigger for distributed table'; CREATE FUNCTION stop_metadata_sync_to_node(nodename text, nodeport integer) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$stop_metadata_sync_to_node$$; COMMENT ON FUNCTION stop_metadata_sync_to_node(nodename text, nodeport integer) IS 'stop metadata sync to node'; CREATE FUNCTION column_to_column_name(table_name regclass, column_var_text text) RETURNS text LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$column_to_column_name$$; COMMENT ON FUNCTION column_to_column_name(table_name regclass, column_var_text text) IS 'convert the textual Var representation to a column name'; CREATE FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$create_distributed_table$$; COMMENT ON FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text) IS 'creates a distributed table'; CREATE FUNCTION get_shard_id_for_distribution_column(table_name regclass, distribution_value "any" DEFAULT NULL) RETURNS bigint LANGUAGE C AS 'MODULE_PATHNAME', $$get_shard_id_for_distribution_column$$; COMMENT ON FUNCTION get_shard_id_for_distribution_column(table_name regclass, distribution_value "any") IS 'return shard id which belongs to given table and contains given value'; CREATE FUNCTION lock_shard_resources(lock_mode int, shard_id bigint[]) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$lock_shard_resources$$; COMMENT ON FUNCTION lock_shard_resources(lock_mode int, shard_id bigint[]) IS 'lock shard resource to serialise non-commutative writes'; CREATE FUNCTION lock_shard_metadata(lock_mode int, shard_id bigint[]) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$lock_shard_metadata$$; COMMENT ON FUNCTION lock_shard_metadata(lock_mode int, shard_id bigint[]) IS 'lock shard metadata to prevent writes during metadata changes'; CREATE FUNCTION master_drop_distributed_table_metadata(logicalrelid regclass, schema_name text, table_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drop_distributed_table_metadata$$; COMMENT ON FUNCTION master_drop_distributed_table_metadata(logicalrelid regclass, schema_name text, table_name text) IS 'delete metadata of the distributed table'; -- allow users to read catalog tables GRANT SELECT ON pg_catalog.pg_dist_node TO public; GRANT SELECT ON pg_catalog.pg_dist_colocation TO public; GRANT SELECT ON pg_catalog.pg_dist_colocationid_seq TO public; GRANT SELECT ON pg_catalog.pg_dist_groupid_seq TO public; GRANT SELECT ON pg_catalog.pg_dist_node_nodeid_seq TO public; GRANT SELECT ON pg_catalog.pg_dist_shard_placement_placementid_seq TO public; GRANT SELECT ON pg_catalog.pg_dist_shardid_seq TO public; GRANT SELECT ON pg_catalog.pg_dist_jobid_seq TO public; CREATE FUNCTION upgrade_to_reference_table(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$upgrade_to_reference_table$$; COMMENT ON FUNCTION upgrade_to_reference_table(table_name regclass) IS 'upgrades an existing broadcast table to a reference table'; CREATE FUNCTION master_disable_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_disable_node$$; COMMENT ON FUNCTION master_disable_node(nodename text, nodeport integer) IS 'removes node from the cluster temporarily'; -- functions for running commands on workers and shards CREATE FUNCTION pg_catalog.master_run_on_worker(worker_name text[], port integer[], command text[], parallel boolean, OUT node_name text, OUT node_port integer, OUT success boolean, OUT result text ) RETURNS SETOF record LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$master_run_on_worker$$; CREATE TYPE citus.colocation_placement_type AS ( shardid1 bigint, shardid2 bigint, nodename text, nodeport bigint ); -- -- distributed_tables_colocated returns true if given tables are co-located, false otherwise. -- The function checks shard definitions, matches shard placements for given tables. -- CREATE FUNCTION pg_catalog.distributed_tables_colocated(table1 regclass, table2 regclass) RETURNS bool LANGUAGE plpgsql AS $function$ DECLARE colocated_shard_count int; table1_shard_count int; table2_shard_count int; table1_placement_count int; table2_placement_count int; table1_placements citus.colocation_placement_type[]; table2_placements citus.colocation_placement_type[]; BEGIN SELECT count(*), (SELECT count(*) FROM pg_dist_shard a WHERE a.logicalrelid = table1), (SELECT count(*) FROM pg_dist_shard b WHERE b.logicalrelid = table2) INTO colocated_shard_count, table1_shard_count, table2_shard_count FROM pg_dist_shard tba JOIN pg_dist_shard tbb USING(shardminvalue, shardmaxvalue) WHERE tba.logicalrelid = table1 AND tbb.logicalrelid = table2; IF (table1_shard_count != table2_shard_count OR table1_shard_count != colocated_shard_count) THEN RETURN false; END IF; WITH colocated_shards AS ( SELECT tba.shardid as shardid1, tbb.shardid as shardid2 FROM pg_dist_shard tba JOIN pg_dist_shard tbb USING(shardminvalue, shardmaxvalue) WHERE tba.logicalrelid = table1 AND tbb.logicalrelid = table2), left_shard_placements AS ( SELECT cs.shardid1, cs.shardid2, sp.nodename, sp.nodeport FROM colocated_shards cs JOIN pg_dist_shard_placement sp ON (cs.shardid1 = sp.shardid) WHERE sp.shardstate = 1) SELECT array_agg( (lsp.shardid1, lsp.shardid2, lsp.nodename, lsp.nodeport)::citus.colocation_placement_type ORDER BY shardid1, shardid2, nodename, nodeport), count(distinct lsp.shardid1) FROM left_shard_placements lsp INTO table1_placements, table1_placement_count; WITH colocated_shards AS ( SELECT tba.shardid as shardid1, tbb.shardid as shardid2 FROM pg_dist_shard tba JOIN pg_dist_shard tbb USING(shardminvalue, shardmaxvalue) WHERE tba.logicalrelid = table1 AND tbb.logicalrelid = table2), right_shard_placements AS ( SELECT cs.shardid1, cs.shardid2, sp.nodename, sp.nodeport FROM colocated_shards cs LEFT JOIN pg_dist_shard_placement sp ON(cs.shardid2 = sp.shardid) WHERE sp.shardstate = 1) SELECT array_agg( (rsp.shardid1, rsp.shardid2, rsp.nodename, rsp.nodeport)::citus.colocation_placement_type ORDER BY shardid1, shardid2, nodename, nodeport), count(distinct rsp.shardid2) FROM right_shard_placements rsp INTO table2_placements, table2_placement_count; IF (table1_shard_count != table1_placement_count OR table1_placement_count != table2_placement_count) THEN RETURN false; END IF; IF (array_length(table1_placements, 1) != array_length(table2_placements, 1)) THEN RETURN false; END IF; FOR i IN 1..array_length(table1_placements,1) LOOP IF (table1_placements[i].nodename != table2_placements[i].nodename OR table1_placements[i].nodeport != table2_placements[i].nodeport) THEN RETURN false; END IF; END LOOP; RETURN true; END; $function$; CREATE FUNCTION pg_catalog.run_command_on_workers(command text, parallel bool default true, OUT nodename text, OUT nodeport int, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE workers text[]; ports int[]; commands text[]; BEGIN WITH citus_workers AS ( SELECT * FROM master_get_active_worker_nodes() ORDER BY node_name, node_port) SELECT array_agg(node_name), array_agg(node_port), array_agg(command) INTO workers, ports, commands FROM citus_workers; RETURN QUERY SELECT * FROM master_run_on_worker(workers, ports, commands, parallel); END; $function$; CREATE FUNCTION pg_catalog.run_command_on_placements(table_name regclass, command text, parallel bool default true, OUT nodename text, OUT nodeport int, OUT shardid bigint, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE workers text[]; ports int[]; shards bigint[]; commands text[]; BEGIN WITH citus_placements AS ( SELECT ds.logicalrelid::regclass AS tablename, ds.shardid AS shardid, shard_name(ds.logicalrelid, ds.shardid) AS shardname, dsp.nodename AS nodename, dsp.nodeport::int AS nodeport FROM pg_dist_shard ds JOIN pg_dist_shard_placement dsp USING (shardid) WHERE dsp.shardstate = 1 and ds.logicalrelid::regclass = table_name ORDER BY ds.logicalrelid, ds.shardid, dsp.nodename, dsp.nodeport) SELECT array_agg(cp.nodename), array_agg(cp.nodeport), array_agg(cp.shardid), array_agg(format(command, cp.shardname)) INTO workers, ports, shards, commands FROM citus_placements cp; RETURN QUERY SELECT r.node_name, r.node_port, shards[ordinality], r.success, r.result FROM master_run_on_worker(workers, ports, commands, parallel) WITH ORDINALITY r; END; $function$; CREATE FUNCTION pg_catalog.run_command_on_colocated_placements( table_name1 regclass, table_name2 regclass, command text, parallel bool default true, OUT nodename text, OUT nodeport int, OUT shardid1 bigint, OUT shardid2 bigint, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE workers text[]; ports int[]; shards1 bigint[]; shards2 bigint[]; commands text[]; BEGIN IF NOT (SELECT distributed_tables_colocated(table_name1, table_name2)) THEN RAISE EXCEPTION 'tables % and % are not co-located', table_name1, table_name2; END IF; WITH active_shard_placements AS ( SELECT ds.logicalrelid, ds.shardid AS shardid, shard_name(ds.logicalrelid, ds.shardid) AS shardname, ds.shardminvalue AS shardminvalue, ds.shardmaxvalue AS shardmaxvalue, dsp.nodename AS nodename, dsp.nodeport::int AS nodeport FROM pg_dist_shard ds JOIN pg_dist_shard_placement dsp USING (shardid) WHERE dsp.shardstate = 1 and (ds.logicalrelid::regclass = table_name1 or ds.logicalrelid::regclass = table_name2) ORDER BY ds.logicalrelid, ds.shardid, dsp.nodename, dsp.nodeport), citus_colocated_placements AS ( SELECT a.logicalrelid::regclass AS tablename1, a.shardid AS shardid1, shard_name(a.logicalrelid, a.shardid) AS shardname1, b.logicalrelid::regclass AS tablename2, b.shardid AS shardid2, shard_name(b.logicalrelid, b.shardid) AS shardname2, a.nodename AS nodename, a.nodeport::int AS nodeport FROM active_shard_placements a, active_shard_placements b WHERE a.shardminvalue = b.shardminvalue AND a.shardmaxvalue = b.shardmaxvalue AND a.logicalrelid != b.logicalrelid AND a.nodename = b.nodename AND a.nodeport = b.nodeport AND a.logicalrelid::regclass = table_name1 AND b.logicalrelid::regclass = table_name2 ORDER BY a.logicalrelid, a.shardid, nodename, nodeport) SELECT array_agg(cp.nodename), array_agg(cp.nodeport), array_agg(cp.shardid1), array_agg(cp.shardid2), array_agg(format(command, cp.shardname1, cp.shardname2)) INTO workers, ports, shards1, shards2, commands FROM citus_colocated_placements cp; RETURN QUERY SELECT r.node_name, r.node_port, shards1[ordinality], shards2[ordinality], r.success, r.result FROM master_run_on_worker(workers, ports, commands, parallel) WITH ORDINALITY r; END; $function$; CREATE FUNCTION pg_catalog.run_command_on_shards(table_name regclass, command text, parallel bool default true, OUT shardid bigint, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE workers text[]; ports int[]; shards bigint[]; commands text[]; shard_count int; BEGIN SELECT COUNT(*) INTO shard_count FROM pg_dist_shard WHERE logicalrelid = table_name; WITH citus_shards AS ( SELECT ds.logicalrelid::regclass AS tablename, ds.shardid AS shardid, shard_name(ds.logicalrelid, ds.shardid) AS shardname, array_agg(dsp.nodename) AS nodenames, array_agg(dsp.nodeport) AS nodeports FROM pg_dist_shard ds LEFT JOIN pg_dist_shard_placement dsp USING (shardid) WHERE dsp.shardstate = 1 and ds.logicalrelid::regclass = table_name GROUP BY ds.logicalrelid, ds.shardid ORDER BY ds.logicalrelid, ds.shardid) SELECT array_agg(cs.nodenames[1]), array_agg(cs.nodeports[1]), array_agg(cs.shardid), array_agg(format(command, cs.shardname)) INTO workers, ports, shards, commands FROM citus_shards cs; IF (shard_count != array_length(workers, 1)) THEN RAISE NOTICE 'some shards do not have active placements'; END IF; RETURN QUERY SELECT shards[ordinality], r.success, r.result FROM master_run_on_worker(workers, ports, commands, parallel) WITH ORDINALITY r; END; $function$; CREATE FUNCTION master_dist_local_group_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_local_group_cache_invalidate$$; COMMENT ON FUNCTION master_dist_local_group_cache_invalidate() IS 'register node cache invalidation for changed rows'; CREATE TRIGGER dist_local_group_cache_invalidate AFTER UPDATE ON pg_catalog.pg_dist_local_group FOR EACH ROW EXECUTE PROCEDURE master_dist_local_group_cache_invalidate(); CREATE FUNCTION worker_apply_sequence_command(text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_apply_sequence_command$$; COMMENT ON FUNCTION worker_apply_sequence_command(text) IS 'create a sequence which products globally unique values'; CREATE FUNCTION isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text DEFAULT '') RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$isolate_tenant_to_new_shard$$; COMMENT ON FUNCTION isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text) IS 'isolate a tenant to its own shard and return the new shard id'; CREATE FUNCTION worker_hash(value "any") RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_hash$$; COMMENT ON FUNCTION worker_hash(value "any") IS 'calculate hashed value and return it'; -- table size functions CREATE FUNCTION citus_table_size(logicalrelid regclass) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_table_size$$; COMMENT ON FUNCTION citus_table_size(logicalrelid regclass) IS 'get disk space used by the specified table, excluding indexes'; CREATE FUNCTION citus_relation_size(logicalrelid regclass) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_relation_size$$; COMMENT ON FUNCTION citus_relation_size(logicalrelid regclass) IS 'get disk space used by the ''main'' fork'; CREATE FUNCTION citus_total_relation_size(logicalrelid regclass) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_total_relation_size$$; COMMENT ON FUNCTION citus_total_relation_size(logicalrelid regclass) IS 'get total disk space used by the specified table'; CREATE FUNCTION pg_catalog.citus_truncate_trigger() RETURNS trigger LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_truncate_trigger$$; COMMENT ON FUNCTION pg_catalog.citus_truncate_trigger() IS 'trigger function called when truncating the distributed table'; -- introduce replication-agnostic pg_dist_placement table ALTER SEQUENCE pg_catalog.pg_dist_shard_placement_placementid_seq RENAME TO pg_dist_placement_placementid_seq; ALTER TABLE pg_catalog.pg_dist_shard_placement ALTER COLUMN placementid SET DEFAULT nextval('pg_catalog.pg_dist_placement_placementid_seq'); CREATE TABLE citus.pg_dist_placement ( placementid BIGINT NOT NULL default nextval('pg_dist_placement_placementid_seq'::regclass), shardid BIGINT NOT NULL, shardstate INT NOT NULL, shardlength BIGINT NOT NULL, groupid INT NOT NULL ); ALTER TABLE citus.pg_dist_placement SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_placement TO public; CREATE INDEX pg_dist_placement_groupid_index ON pg_dist_placement USING btree(groupid); CREATE INDEX pg_dist_placement_shardid_index ON pg_dist_placement USING btree(shardid); CREATE UNIQUE INDEX pg_dist_placement_placementid_index ON pg_dist_placement USING btree(placementid); CREATE OR REPLACE FUNCTION citus.find_groupid_for_node(text, int) RETURNS int AS $$ DECLARE groupid int := (SELECT groupid FROM pg_dist_node WHERE nodename = $1 AND nodeport = $2); BEGIN IF groupid IS NULL THEN RAISE EXCEPTION 'There is no node at "%:%"', $1, $2; ELSE RETURN groupid; END IF; END; $$ LANGUAGE plpgsql; INSERT INTO pg_catalog.pg_dist_placement SELECT placementid, shardid, shardstate, shardlength, citus.find_groupid_for_node(placement.nodename, placement.nodeport::int) AS groupid FROM pg_dist_shard_placement placement; DROP TRIGGER dist_placement_cache_invalidate ON pg_catalog.pg_dist_shard_placement; CREATE TRIGGER dist_placement_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON pg_catalog.pg_dist_placement FOR EACH ROW EXECUTE PROCEDURE master_dist_placement_cache_invalidate(); -- this should be removed when noderole is added but for now it ensures the below view -- returns the correct results and that placements unambiguously belong to a view ALTER TABLE pg_catalog.pg_dist_node ADD CONSTRAINT pg_dist_node_groupid_unique UNIQUE (groupid); DROP TABLE pg_dist_shard_placement; CREATE VIEW citus.pg_dist_shard_placement AS SELECT shardid, shardstate, shardlength, nodename, nodeport, placementid -- assumes there's only one node per group FROM pg_dist_placement placement INNER JOIN pg_dist_node node ON ( placement.groupid = node.groupid ); ALTER VIEW citus.pg_dist_shard_placement SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_shard_placement TO public; -- add some triggers which make it look like pg_dist_shard_placement is still a table ALTER VIEW pg_catalog.pg_dist_shard_placement ALTER placementid SET DEFAULT nextval('pg_dist_placement_placementid_seq'); CREATE OR REPLACE FUNCTION citus.pg_dist_shard_placement_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'DELETE') THEN DELETE FROM pg_dist_placement WHERE placementid = OLD.placementid; RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN UPDATE pg_dist_placement SET shardid = NEW.shardid, shardstate = NEW.shardstate, shardlength = NEW.shardlength, placementid = NEW.placementid, groupid = citus.find_groupid_for_node(NEW.nodename, NEW.nodeport) WHERE placementid = OLD.placementid; RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO pg_dist_placement (placementid, shardid, shardstate, shardlength, groupid) VALUES (NEW.placementid, NEW.shardid, NEW.shardstate, NEW.shardlength, citus.find_groupid_for_node(NEW.nodename, NEW.nodeport)); RETURN NEW; END IF; END; $$ LANGUAGE plpgsql; CREATE TRIGGER pg_dist_shard_placement_trigger INSTEAD OF INSERT OR UPDATE OR DELETE ON pg_dist_shard_placement FOR EACH ROW EXECUTE PROCEDURE citus.pg_dist_shard_placement_trigger_func(); CREATE TYPE pg_catalog.noderole AS ENUM ( 'primary', -- node is available and accepting writes 'secondary', -- node is available but only accepts reads 'unavailable' -- node is in recovery or otherwise not usable -- adding new values to a type inside of a transaction (such as during an ALTER EXTENSION -- citus UPDATE) isn't allowed in PG 9.6, and only allowed in PG10 if you don't use the -- new values inside of the same transaction. You might need to replace this type with a -- new one and then change the column type in pg_dist_node. There's a list of -- alternatives here: -- https://stackoverflow.com/questions/1771543/postgresql-updating-an-enum-type/41696273 ); ALTER TABLE pg_dist_node ADD COLUMN noderole noderole NOT NULL DEFAULT 'primary'; -- we're now allowed to have more than one node per group ALTER TABLE pg_catalog.pg_dist_node DROP CONSTRAINT pg_dist_node_groupid_unique; -- so make sure pg_dist_shard_placement only returns writable placements CREATE OR REPLACE VIEW pg_catalog.pg_dist_shard_placement AS SELECT shardid, shardstate, shardlength, nodename, nodeport, placementid FROM pg_dist_placement placement INNER JOIN pg_dist_node node ON ( placement.groupid = node.groupid AND node.noderole = 'primary' ); CREATE OR REPLACE FUNCTION citus.pg_dist_node_trigger_func() RETURNS TRIGGER AS $$ BEGIN -- AddNodeMetadata also takes out a ShareRowExclusiveLock LOCK TABLE pg_dist_node IN SHARE ROW EXCLUSIVE MODE; IF (TG_OP = 'INSERT') THEN IF NEW.noderole = 'primary' AND EXISTS (SELECT 1 FROM pg_dist_node WHERE groupid = NEW.groupid AND noderole = 'primary' AND nodeid <> NEW.nodeid) THEN RAISE EXCEPTION 'there cannot be two primary nodes in a group'; END IF; RETURN NEW; ELSIF (TG_OP = 'UPDATE') THEN IF NEW.noderole = 'primary' AND EXISTS (SELECT 1 FROM pg_dist_node WHERE groupid = NEW.groupid AND noderole = 'primary' AND nodeid <> NEW.nodeid) THEN RAISE EXCEPTION 'there cannot be two primary nodes in a group'; END IF; RETURN NEW; END IF; END; $$ LANGUAGE plpgsql; CREATE TRIGGER pg_dist_node_trigger BEFORE INSERT OR UPDATE ON pg_dist_node FOR EACH ROW EXECUTE PROCEDURE citus.pg_dist_node_trigger_func(); -- distributed deadlocks CREATE FUNCTION assign_distributed_transaction_id(initiator_node_identifier int4, transaction_number int8, transaction_stamp timestamptz) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$assign_distributed_transaction_id$$; COMMENT ON FUNCTION assign_distributed_transaction_id(initiator_node_identifier int4, transaction_number int8, transaction_stamp timestamptz) IS 'Only intended for internal use, users should not call this. The function sets the distributed transaction id'; CREATE OR REPLACE FUNCTION get_current_transaction_id(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$get_current_transaction_id$$; COMMENT ON FUNCTION get_current_transaction_id(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns the current backend data including distributed transaction id'; CREATE OR REPLACE FUNCTION get_all_active_transactions(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION get_all_active_transactions(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns distributed transaction ids of active distributed transactions'; CREATE OR REPLACE FUNCTION check_distributed_deadlocks() RETURNS BOOL LANGUAGE 'c' STRICT AS $$MODULE_PATHNAME$$, $$check_distributed_deadlocks$$; COMMENT ON FUNCTION check_distributed_deadlocks() IS 'does a distributed deadlock check, if a deadlock found cancels one of the participating backends and returns true '; CREATE FUNCTION pg_catalog.dump_local_wait_edges( OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$dump_local_wait_edges$$; COMMENT ON FUNCTION pg_catalog.dump_local_wait_edges() IS 'returns all local lock wait chains, that start from distributed transactions'; CREATE FUNCTION pg_catalog.dump_global_wait_edges( OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE 'c' STRICT AS $$MODULE_PATHNAME$$, $$dump_global_wait_edges$$; COMMENT ON FUNCTION pg_catalog.dump_global_wait_edges() IS 'returns a global list of blocked transactions originating from this node'; CREATE FUNCTION citus.replace_isolation_tester_func() RETURNS void AS $$ DECLARE version integer := current_setting('server_version_num'); BEGIN IF version >= 100000 THEN ALTER FUNCTION pg_catalog.pg_isolation_test_session_is_blocked(integer, integer[]) RENAME TO old_pg_isolation_test_session_is_blocked; ALTER FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(integer, integer[]) RENAME TO pg_isolation_test_session_is_blocked; ELSE ALTER FUNCTION pg_catalog.pg_blocking_pids(integer) RENAME TO old_pg_blocking_pids; ALTER FUNCTION pg_catalog.citus_blocking_pids(integer) RENAME TO pg_blocking_pids; END IF; END; $$ LANGUAGE plpgsql; CREATE FUNCTION citus.restore_isolation_tester_func() RETURNS void AS $$ DECLARE version integer := current_setting('server_version_num'); BEGIN IF version >= 100000 THEN ALTER FUNCTION pg_catalog.pg_isolation_test_session_is_blocked(integer, integer[]) RENAME TO citus_isolation_test_session_is_blocked; ALTER FUNCTION pg_catalog.old_pg_isolation_test_session_is_blocked(integer, integer[]) RENAME TO pg_isolation_test_session_is_blocked; ELSE ALTER FUNCTION pg_catalog.pg_blocking_pids(integer) RENAME TO citus_blocking_pids; ALTER FUNCTION pg_catalog.old_pg_blocking_pids(integer) RENAME TO pg_blocking_pids; END IF; END; $$ LANGUAGE plpgsql; CREATE FUNCTION citus.refresh_isolation_tester_prepared_statement() RETURNS void AS $$ BEGIN -- isolation creates a prepared statement using the old function before tests have a -- chance to call replace_isolation_tester_func. By calling that prepared statement -- with a different search_path we force a re-parse which picks up the new function SET search_path TO 'citus'; EXECUTE 'EXECUTE isolationtester_waiting (0)'; RESET search_path; END; $$ LANGUAGE plpgsql; CREATE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalTransactionNum int8; BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT transaction_number INTO mLocalTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; SELECT array_agg(process_id) INTO mRemoteBlockingPids FROM ( WITH activeTransactions AS ( SELECT process_id, transaction_number FROM get_all_active_transactions() ), blockingTransactions AS ( SELECT blocking_transaction_num AS txn_num FROM dump_global_wait_edges() WHERE waiting_transaction_num = mLocalTransactionNum ) SELECT activeTransactions.process_id FROM activeTransactions, blockingTransactions WHERE activeTransactions.transaction_number = blockingTransactions.txn_num ) AS sub; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; CREATE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedTransactionNum int8; BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT transaction_number INTO mBlockedTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; RETURN EXISTS ( SELECT 1 FROM dump_global_wait_edges() WHERE waiting_transaction_num = mBlockedTransactionNum ); END; $$ LANGUAGE plpgsql; ALTER TABLE pg_dist_node ADD COLUMN nodecluster name NOT NULL DEFAULT 'default'; ALTER TABLE pg_dist_node ADD CONSTRAINT primaries_are_only_allowed_in_the_default_cluster CHECK (NOT (nodecluster <> 'default' AND noderole = 'primary')); CREATE FUNCTION master_add_node(nodename text, nodeport integer, groupid integer default 0, noderole noderole default 'primary', nodecluster name default 'default', OUT nodeid integer, OUT groupid integer, OUT nodename text, OUT nodeport integer, OUT noderack text, OUT hasmetadata boolean, OUT isactive bool, OUT noderole noderole, OUT nodecluster name) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_node$$; COMMENT ON FUNCTION master_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; CREATE FUNCTION master_add_inactive_node(nodename text, nodeport integer, groupid integer default 0, noderole noderole default 'primary', nodecluster name default 'default', OUT nodeid integer, OUT groupid integer, OUT nodename text, OUT nodeport integer, OUT noderack text, OUT hasmetadata boolean, OUT isactive bool, OUT noderole noderole, OUT nodecluster name) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_add_inactive_node$$; COMMENT ON FUNCTION master_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; CREATE FUNCTION master_activate_node(nodename text, nodeport integer, OUT nodeid integer, OUT groupid integer, OUT nodename text, OUT nodeport integer, OUT noderack text, OUT hasmetadata boolean, OUT isactive bool, OUT noderole noderole, OUT nodecluster name) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_activate_node$$; COMMENT ON FUNCTION master_activate_node(nodename text, nodeport integer) IS 'activate a node which is in the cluster'; CREATE FUNCTION master_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name default 'default', OUT nodeid integer, OUT groupid integer, OUT nodename text, OUT nodeport integer, OUT noderack text, OUT hasmetadata boolean, OUT isactive bool, OUT noderole noderole, OUT nodecluster name) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_secondary_node$$; COMMENT ON FUNCTION master_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name) IS 'add a secondary node to the cluster'; CREATE FUNCTION master_update_node(node_id int, new_node_name text, new_node_port int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_update_node$$; COMMENT ON FUNCTION master_update_node(node_id int, new_node_name text, new_node_port int) IS 'change the location of a node'; -- shard statistics CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID AS $$ DECLARE colocated_tables regclass[]; BEGIN SELECT get_colocated_table_array(relation) INTO colocated_tables; PERFORM master_update_shard_statistics(shardid) FROM pg_dist_shard WHERE logicalrelid = ANY (colocated_tables); END; $$ LANGUAGE 'plpgsql'; COMMENT ON FUNCTION master_update_table_statistics(regclass) IS 'updates shard statistics of the given table and its colocated tables'; CREATE OR REPLACE FUNCTION get_colocated_shard_array(bigint) RETURNS BIGINT[] LANGUAGE C STRICT AS 'citus', $$get_colocated_shard_array$$; COMMENT ON FUNCTION get_colocated_shard_array(bigint) IS 'returns the array of colocated shards of the given shard'; -- distributed backups CREATE OR REPLACE FUNCTION pg_catalog.citus_create_restore_point(text) RETURNS pg_lsn LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_create_restore_point$$; COMMENT ON FUNCTION pg_catalog.citus_create_restore_point(text) IS 'temporarily block writes and create a named restore point on all nodes'; -- functions for giving node a unique identifier CREATE OR REPLACE FUNCTION pg_catalog.citus_version() RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_version$$; COMMENT ON FUNCTION pg_catalog.citus_version() IS 'Citus version string'; CREATE TABLE citus.pg_dist_node_metadata( metadata jsonb NOT NULL ); ALTER TABLE citus.pg_dist_node_metadata SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_node_metadata TO public; CREATE FUNCTION pg_catalog.citus_server_id() RETURNS uuid LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_server_id$$; COMMENT ON FUNCTION citus_server_id() IS 'generates a random UUID to be used as server identifier'; -- Insert the latest extension version into pg_dist_node_metadata -- for new installations. -- -- While users could technically upgrade to an intermediate version -- everything in Citus fails until it is upgraded to the latest version, -- so it seems safe to use the latest. INSERT INTO pg_dist_node_metadata SELECT jsonb_build_object('server_id', citus_server_id()::text, 'last_upgrade_version', default_version) FROM pg_available_extensions WHERE name = 'citus'; -- rebalancer functions CREATE TYPE citus.shard_transfer_mode AS ENUM ( 'auto', 'force_logical', 'block_writes' ); CREATE OR REPLACE FUNCTION master_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_move_shard_placement$$; COMMENT ON FUNCTION master_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a shard from a the source node to the destination node'; CREATE FUNCTION master_copy_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, do_repair bool DEFAULT true, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_copy_shard_placement$$; COMMENT ON FUNCTION master_copy_shard_placement(shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, do_repair bool, shard_transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; -- intermediate result functions CREATE OR REPLACE FUNCTION pg_catalog.create_intermediate_result(result_id text, query text) RETURNS bigint LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$create_intermediate_result$$; COMMENT ON FUNCTION pg_catalog.create_intermediate_result(text,text) IS 'execute a query and write its results to local result file'; CREATE OR REPLACE FUNCTION pg_catalog.broadcast_intermediate_result(result_id text, query text) RETURNS bigint LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$broadcast_intermediate_result$$; COMMENT ON FUNCTION pg_catalog.broadcast_intermediate_result(text,text) IS 'execute a query and write its results to an result file on all workers'; CREATE TYPE pg_catalog.citus_copy_format AS ENUM ('csv', 'binary', 'text'); CREATE OR REPLACE FUNCTION pg_catalog.read_intermediate_result(result_id text, format pg_catalog.citus_copy_format default 'csv') RETURNS SETOF record LANGUAGE C STRICT VOLATILE PARALLEL SAFE AS 'MODULE_PATHNAME', $$read_intermediate_result$$; COMMENT ON FUNCTION pg_catalog.read_intermediate_result(text,pg_catalog.citus_copy_format) IS 'read a file and return it as a set of records'; CREATE FUNCTION pg_catalog.citus_text_send_as_jsonb(text) RETURNS bytea LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_text_send_as_jsonb$$; -- Citus json aggregate helpers CREATE FUNCTION pg_catalog.citus_jsonb_concatenate(state jsonb, val jsonb) RETURNS jsonb LANGUAGE SQL AS $function$ SELECT CASE WHEN val IS NULL THEN state WHEN jsonb_typeof(state) = 'null' THEN val ELSE state || val END; $function$; CREATE FUNCTION pg_catalog.citus_jsonb_concatenate_final(state jsonb) RETURNS jsonb LANGUAGE SQL AS $function$ SELECT CASE WHEN jsonb_typeof(state) = 'null' THEN NULL ELSE state END; $function$; CREATE FUNCTION pg_catalog.citus_json_concatenate(state json, val json) RETURNS json LANGUAGE SQL AS $function$ SELECT CASE WHEN val IS NULL THEN state WHEN json_typeof(state) = 'null' THEN val WHEN json_typeof(state) = 'object' THEN (SELECT json_object_agg(key, value) FROM ( SELECT * FROM json_each(state) UNION ALL SELECT * FROM json_each(val) ) t) ELSE (SELECT json_agg(a) FROM ( SELECT json_array_elements(state) AS a UNION ALL SELECT json_array_elements(val) AS a ) t) END; $function$; CREATE FUNCTION pg_catalog.citus_json_concatenate_final(state json) RETURNS json LANGUAGE SQL AS $function$ SELECT CASE WHEN json_typeof(state) = 'null' THEN NULL ELSE state END; $function$; -- Citus json aggregates CREATE AGGREGATE pg_catalog.jsonb_cat_agg(jsonb) ( SFUNC = citus_jsonb_concatenate, FINALFUNC = citus_jsonb_concatenate_final, STYPE = jsonb, INITCOND = 'null' ); COMMENT ON AGGREGATE pg_catalog.jsonb_cat_agg(jsonb) IS 'concatenate input jsonbs into a single jsonb'; CREATE AGGREGATE pg_catalog.json_cat_agg(json) ( SFUNC = citus_json_concatenate, FINALFUNC = citus_json_concatenate_final, STYPE = json, INITCOND = 'null' ); COMMENT ON AGGREGATE pg_catalog.json_cat_agg(json) IS 'concatenate input jsons into a single json'; CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE v_obj record; sequence_names text[] := '{}'; table_colocation_id integer; propagate_drop boolean := false; BEGIN -- collect set of dropped sequences to drop on workers later SELECT array_agg(object_identity) INTO sequence_names FROM pg_event_trigger_dropped_objects() WHERE object_type = 'sequence'; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- drop all shards and the metadata PERFORM master_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_drop_distributed_table_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; IF cardinality(sequence_names) = 0 THEN RETURN; END IF; PERFORM master_drop_sequences(sequence_names); END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; CREATE EVENT TRIGGER citus_cascade_to_partition ON SQL_DROP EXECUTE PROCEDURE citus_drop_trigger(); -- pg_dist_authinfo CREATE FUNCTION pg_catalog.role_exists(name) RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$role_exists$$; COMMENT ON FUNCTION role_exists(name) IS 'returns whether a role exists'; CREATE FUNCTION pg_catalog.authinfo_valid(text) RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$authinfo_valid$$; COMMENT ON FUNCTION authinfo_valid(text) IS 'returns whether an authinfo is valid'; CREATE TABLE citus.pg_dist_authinfo ( nodeid integer NOT NULL, rolename name NOT NULL CONSTRAINT role_exists CHECK (role_exists(rolename)), authinfo text NOT NULL CONSTRAINT authinfo_valid CHECK (authinfo_valid(authinfo)) ); CREATE UNIQUE INDEX pg_dist_authinfo_identification_index ON citus.pg_dist_authinfo (rolename, nodeid DESC); ALTER TABLE citus.pg_dist_authinfo SET SCHEMA pg_catalog; REVOKE ALL ON pg_catalog.pg_dist_authinfo FROM PUBLIC; CREATE FUNCTION master_dist_authinfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$master_dist_authinfo_cache_invalidate$$; COMMENT ON FUNCTION master_dist_authinfo_cache_invalidate() IS 'register authinfo cache invalidation on any modifications'; CREATE FUNCTION task_tracker_conninfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$task_tracker_conninfo_cache_invalidate$$; COMMENT ON FUNCTION task_tracker_conninfo_cache_invalidate() IS 'invalidate task-tracker conninfo cache'; CREATE TRIGGER dist_authinfo_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_authinfo FOR EACH STATEMENT EXECUTE PROCEDURE master_dist_authinfo_cache_invalidate(); CREATE TRIGGER dist_authinfo_task_tracker_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_authinfo FOR EACH STATEMENT EXECUTE PROCEDURE task_tracker_conninfo_cache_invalidate(); -- citus_stat_statements CREATE FUNCTION pg_catalog.citus_query_stats(OUT queryid bigint, OUT userid oid, OUT dbid oid, OUT executor bigint, OUT partition_key text, OUT calls bigint) RETURNS SETOF record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_query_stats$$; CREATE FUNCTION pg_catalog.citus_stat_statements_reset() RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_stat_statements_reset$$; CREATE FUNCTION pg_catalog.citus_stat_statements(OUT queryid bigint, OUT userid oid, OUT dbid oid, OUT query text, OUT executor bigint, OUT partition_key text, OUT calls bigint) RETURNS SETOF record LANGUAGE plpgsql AS $citus_stat_statements$ BEGIN IF EXISTS ( SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements') THEN RETURN QUERY SELECT pss.queryid, pss.userid, pss.dbid, pss.query, cqs.executor, cqs.partition_key, cqs.calls FROM pg_stat_statements(true) pss JOIN citus_query_stats() cqs USING (queryid); ELSE RAISE EXCEPTION 'pg_stat_statements is not installed' USING HINT = 'install pg_stat_statements extension and try again'; END IF; END; $citus_stat_statements$; CREATE VIEW citus.citus_stat_statements as SELECT * FROM pg_catalog.citus_stat_statements(); ALTER VIEW citus.citus_stat_statements SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_statements TO public; CREATE FUNCTION pg_catalog.citus_executor_name(executor_type int) RETURNS TEXT LANGUAGE plpgsql AS $function$ BEGIN IF (executor_type = 1) THEN RETURN 'real-time'; ELSIF (executor_type = 2) THEN RETURN 'task-tracker'; ELSIF (executor_type = 3) THEN RETURN 'router'; ELSIF (executor_type = 4) THEN RETURN 'insert-select'; ELSE RETURN 'unknown'; END IF; END; $function$; DROP VIEW pg_catalog.citus_stat_statements; CREATE VIEW citus.citus_stat_statements AS SELECT queryid, userid, dbid, query, pg_catalog.citus_executor_name(executor::int) AS executor, partition_key, calls FROM pg_catalog.citus_stat_statements(); ALTER VIEW citus.citus_stat_statements SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_statements TO public; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_statements_reset() FROM PUBLIC; -- pg_dist_poolinfo CREATE FUNCTION pg_catalog.poolinfo_valid(text) RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$poolinfo_valid$$; COMMENT ON FUNCTION pg_catalog.poolinfo_valid(text) IS 'returns whether a poolinfo is valid'; CREATE TABLE citus.pg_dist_poolinfo ( nodeid integer PRIMARY KEY REFERENCES pg_dist_node(nodeid) ON DELETE CASCADE, poolinfo text NOT NULL CONSTRAINT poolinfo_valid CHECK (poolinfo_valid(poolinfo)) ); ALTER TABLE citus.pg_dist_poolinfo SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_poolinfo TO public; ALTER FUNCTION master_dist_authinfo_cache_invalidate() RENAME TO master_conninfo_cache_invalidate; CREATE TRIGGER dist_poolinfo_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_poolinfo FOR EACH STATEMENT EXECUTE PROCEDURE master_conninfo_cache_invalidate(); CREATE TRIGGER dist_poolinfo_task_tracker_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_poolinfo FOR EACH STATEMENT EXECUTE PROCEDURE task_tracker_conninfo_cache_invalidate(); RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-10--8.0-11.sql ================================================ -- citus--8.0-10--8.0-11 SET search_path = 'pg_catalog'; -- Deprecated functions DROP FUNCTION IF EXISTS worker_hash_partition_table(bigint,integer,text,text,oid,integer); DROP FUNCTION IF EXISTS worker_foreign_file_path(text); DROP FUNCTION IF EXISTS worker_find_block_local_path(bigint,text[]); DROP FUNCTION IF EXISTS worker_fetch_query_results_file(bigint,integer,integer,text,integer); DROP FUNCTION IF EXISTS master_drop_distributed_table_metadata(regclass,text,text); -- Testing functions REVOKE ALL ON FUNCTION citus_blocking_pids(integer) FROM PUBLIC; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; -- Maintenance function REVOKE ALL ON FUNCTION worker_cleanup_job_schema_cache() FROM PUBLIC; REVOKE ALL ON FUNCTION recover_prepared_transactions() FROM PUBLIC; REVOKE ALL ON FUNCTION check_distributed_deadlocks() FROM PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-11--8.0-12.sql ================================================ -- citus--8.0-11--8.0-12 SET search_path = 'pg_catalog'; CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_statements(OUT queryid bigint, OUT userid oid, OUT dbid oid, OUT query text, OUT executor bigint, OUT partition_key text, OUT calls bigint) RETURNS SETOF record LANGUAGE plpgsql AS $citus_stat_statements$ BEGIN IF EXISTS ( SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements') THEN RETURN QUERY SELECT pss.queryid, pss.userid, pss.dbid, pss.query, cqs.executor, cqs.partition_key, cqs.calls FROM pg_stat_statements(true) pss JOIN citus_query_stats() cqs USING (queryid, userid, dbid); ELSE RAISE EXCEPTION 'pg_stat_statements is not installed' USING HINT = 'install pg_stat_statements extension and try again'; END IF; END; $citus_stat_statements$; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-12--8.0-13.sql ================================================ -- citus--8.0-12--8.0-13 CREATE FUNCTION citus_check_defaults_for_sslmode() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_check_defaults_for_sslmode$$; DO LANGUAGE plpgsql $$ BEGIN -- Citus 8.1 and higher default to requiring SSL for all outgoing connections -- (specified by citus.node_conninfo). -- If it looks like we are about to enforce ssl for outgoing connections on a postgres -- installation that does not have ssl turned on we fall back to sslmode=prefer for -- outgoing connections. -- This will only be the case for upgrades from previous versions of Citus, on new -- installations we will have turned on ssl in an earlier stage of the extension -- creation. IF NOT current_setting('ssl')::boolean THEN PERFORM citus_check_defaults_for_sslmode(); END IF; END; $$; DROP FUNCTION citus_check_defaults_for_sslmode(); ================================================ FILE: src/backend/distributed/sql/citus--8.0-13--8.1-1.sql ================================================ -- citus--8.0-13--8.1-1.sql -- bump version to 8.1-1 ================================================ FILE: src/backend/distributed/sql/citus--8.0-2--8.0-3.sql ================================================ -- citus--8.0-2--8.0-3 SET search_path = 'pg_catalog'; CREATE FUNCTION master_remove_partition_metadata(logicalrelid regclass, schema_name text, table_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_remove_partition_metadata$$; COMMENT ON FUNCTION master_remove_partition_metadata(logicalrelid regclass, schema_name text, table_name text) IS 'deletes the partition metadata of a distributed table'; CREATE OR REPLACE FUNCTION master_remove_distributed_table_metadata_from_workers(logicalrelid regclass, schema_name text, table_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_remove_distributed_table_metadata_from_workers$$; COMMENT ON FUNCTION master_remove_distributed_table_metadata_from_workers(logicalrelid regclass, schema_name text, table_name text) IS 'drops the table and removes all the metadata belonging the distributed table in the worker nodes with metadata.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE v_obj record; sequence_names text[] := '{}'; table_colocation_id integer; propagate_drop boolean := false; BEGIN -- collect set of dropped sequences to drop on workers later SELECT array_agg(object_identity) INTO sequence_names FROM pg_event_trigger_dropped_objects() WHERE object_type = 'sequence'; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; IF cardinality(sequence_names) = 0 THEN RETURN; END IF; PERFORM master_drop_sequences(sequence_names); END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-3--8.0-4.sql ================================================ -- citus--8.0-3--8.0-4 SET search_path = 'pg_catalog'; CREATE OR REPLACE FUNCTION lock_relation_if_exists(table_name text, lock_mode text) RETURNS BOOL LANGUAGE C STRICT as 'MODULE_PATHNAME', $$lock_relation_if_exists$$; COMMENT ON FUNCTION lock_relation_if_exists(table_name text, lock_mode text) IS 'used internally to locks relation in the lock_mode if the relation exists without throwing errors; consider using LOCK * IN * MODE instead'; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-4--8.0-5.sql ================================================ -- citus--8.0-4--8.0-5.sql SET search_path = 'pg_catalog'; DROP FUNCTION IF EXISTS get_all_active_transactions(); CREATE OR REPLACE FUNCTION get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION get_all_active_transactions(OUT datid oid, OUT datname text, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns distributed transaction ids of active distributed transactions'; CREATE OR REPLACE FUNCTION citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_dist_stat_activity$$; COMMENT ON FUNCTION citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on distributed tables'; CREATE VIEW citus.citus_dist_stat_activity AS SELECT * FROM pg_catalog.citus_dist_stat_activity(); ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; CREATE OR REPLACE FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_worker_stat_activity$$; COMMENT ON FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on shards of distributed tables'; CREATE VIEW citus.citus_worker_stat_activity AS SELECT * FROM pg_catalog.citus_worker_stat_activity(); ALTER VIEW citus.citus_worker_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_worker_stat_activity TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-5--8.0-6.sql ================================================ -- citus--8.0-5--8.0-6 SET search_path = 'pg_catalog'; CREATE FUNCTION get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_global_active_transactions$$; COMMENT ON FUNCTION get_global_active_transactions(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns distributed transaction ids of active distributed transactions from each node of the cluster'; CREATE OR REPLACE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalTransactionNum int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT transaction_number FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT transaction_number INTO mLocalTransactionNum FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT transaction_number INTO mLocalTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; IF EXISTS (SELECT waiting_transaction_num FROM dump_global_wait_edges() WHERE waiting_transaction_num = mLocalTransactionNum) THEN SELECT array_agg(pBlockedPid) INTO mRemoteBlockingPids; END IF; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; #include "udfs/citus_isolation_test_session_is_blocked/8.0-6.sql" RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-6--8.0-7.sql ================================================ -- citus--8.0-6--8.0-7 SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH citus_dist_stat_activity AS ( SELECT * FROM citus_dist_stat_activity ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_node_id, waiting_transaction_num, blocking_node_id, blocking_transaction_num) * FROM dump_global_wait_edges() ), citus_dist_stat_activity_with_node_id AS ( SELECT citus_dist_stat_activity.*, (CASE citus_dist_stat_activity.master_query_host_name WHEN 'coordinator_host' THEN 0 ELSE pg_dist_node.nodeid END) as initiator_node_id FROM citus_dist_stat_activity LEFT JOIN pg_dist_node ON citus_dist_stat_activity.master_query_host_name = pg_dist_node.nodename AND citus_dist_stat_activity.master_query_host_port = pg_dist_node.nodeport ) SELECT waiting.pid AS waiting_pid, blocking.pid AS blocking_pid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.initiator_node_id AS waiting_node_id, blocking.initiator_node_id AS blocking_node_id, waiting.master_query_host_name AS waiting_node_name, blocking.master_query_host_name AS blocking_node_name, waiting.master_query_host_port AS waiting_node_port, blocking.master_query_host_port AS blocking_node_port FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_node_id waiting ON (unique_global_wait_edges.waiting_transaction_num = waiting.transaction_number AND unique_global_wait_edges.waiting_node_id = waiting.initiator_node_id) JOIN citus_dist_stat_activity_with_node_id blocking ON (unique_global_wait_edges.blocking_transaction_num = blocking.transaction_number AND unique_global_wait_edges.blocking_node_id = blocking.initiator_node_id); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-7--8.0-8.sql ================================================ -- citus--8.0-7--8.0-8 SET search_path = 'pg_catalog'; DROP FUNCTION IF EXISTS pg_catalog.worker_drop_distributed_table(logicalrelid Oid); CREATE FUNCTION worker_drop_distributed_table(table_name text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_distributed_table$$; COMMENT ON FUNCTION worker_drop_distributed_table(table_name text) IS 'drop the distributed table and its reference from metadata tables'; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-8--8.0-9.sql ================================================ -- citus--8.0-8--8.0-9 SET search_path = 'pg_catalog'; REVOKE ALL ON FUNCTION master_activate_node(text,int) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_node(text,int,int,noderole,name) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_secondary_node(text,int,text,int,name) FROM PUBLIC; REVOKE ALL ON FUNCTION master_disable_node(text,int) FROM PUBLIC; REVOKE ALL ON FUNCTION master_remove_node(text,int) FROM PUBLIC; REVOKE ALL ON FUNCTION master_update_node(int,text,int) FROM PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.0-9--8.0-10.sql ================================================ -- citus--8.0-9--8.0-10 SET search_path = 'pg_catalog'; CREATE FUNCTION worker_execute_sql_task(jobid bigint, taskid integer, query text, binary bool) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_execute_sql_task$$; COMMENT ON FUNCTION worker_execute_sql_task(bigint, integer, text, bool) IS 'execute a query and write the results to a task file'; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.1-1--8.2-1.sql ================================================ -- citus--8.1-1--8.2-1.sql -- bump version to 8.2-1 ================================================ FILE: src/backend/distributed/sql/citus--8.2-1--8.2-2.sql ================================================ -- citus--8.2-1--8.2-2.sql DROP FUNCTION IF EXISTS pg_catalog.create_insert_proxy_for_table(regclass,regclass); ================================================ FILE: src/backend/distributed/sql/citus--8.2-2--8.2-3.sql ================================================ -- citus--8.2-2--8.2-3 SET search_path = 'pg_catalog'; DROP FUNCTION master_update_node(node_id int, new_node_name text, new_node_port int); CREATE OR REPLACE FUNCTION master_update_node(node_id int, new_node_name text, new_node_port int, force bool DEFAULT false, lock_cooldown int DEFAULT 10000) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_update_node$$; COMMENT ON FUNCTION master_update_node(node_id int, new_node_name text, new_node_port int, force bool, lock_cooldown int) IS 'change the location of a node. when force => true it will wait lock_cooldown ms before killing competing locks'; REVOKE ALL ON FUNCTION master_update_node(int,text,int,bool,int) FROM PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--8.2-3--8.2-4.sql ================================================ -- citus--8.2-3--8.2-4 CREATE OR REPLACE FUNCTION pg_catalog.citus_executor_name(executor_type int) RETURNS text LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_executor_name$$; COMMENT ON FUNCTION pg_catalog.citus_executor_name(int) IS 'return the name of the external based for the value in citus_stat_statements() output'; ================================================ FILE: src/backend/distributed/sql/citus--8.2-4--8.3-1.sql ================================================ -- citus--8.2-4--8.3-1 -- bump version to 8.3-1 ================================================ FILE: src/backend/distributed/sql/citus--8.3-1--9.0-1.sql ================================================ -- citus--8.3-1--9.0-1 SET search_path = 'pg_catalog'; -- We swapped the groupid and nodeid sequences when creating pg_dist_node ALTER TABLE pg_dist_node ALTER COLUMN groupid SET DEFAULT nextval ('pg_dist_groupid_seq'); ALTER TABLE pg_dist_node ALTER COLUMN nodeid SET DEFAULT nextval('pg_dist_node_nodeid_seq'); CREATE SCHEMA IF NOT EXISTS citus_internal; -- move citus internal functions to citus_internal to make space in the citus schema for -- our public interface ALTER FUNCTION citus.find_groupid_for_node SET SCHEMA citus_internal; ALTER FUNCTION citus.pg_dist_node_trigger_func SET SCHEMA citus_internal; ALTER FUNCTION citus.pg_dist_shard_placement_trigger_func SET SCHEMA citus_internal; ALTER FUNCTION citus.refresh_isolation_tester_prepared_statement SET SCHEMA citus_internal; ALTER FUNCTION citus.replace_isolation_tester_func SET SCHEMA citus_internal; ALTER FUNCTION citus.restore_isolation_tester_func SET SCHEMA citus_internal; -- we can now safely grant usage on the citus schema to use types GRANT USAGE ON SCHEMA citus TO public; #include "udfs/pg_dist_shard_placement_trigger_func/9.0-1.sql" #include "udfs/worker_create_or_replace_object/9.0-1.sql" CREATE OR REPLACE FUNCTION pg_catalog.master_unmark_object_distributed(classid oid, objid oid, objsubid int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_unmark_object_distributed$$; COMMENT ON FUNCTION pg_catalog.master_unmark_object_distributed(classid oid, objid oid, objsubid int) IS 'remove an object address from citus.pg_dist_object once the object has been deleted'; CREATE TABLE citus.pg_dist_object ( -- fields used for composite primary key classid oid NOT NULL, objid oid NOT NULL, objsubid integer NOT NULL, -- fields used for upgrades type text DEFAULT NULL, object_names text[] DEFAULT NULL, object_args text[] DEFAULT NULL, -- fields that are only valid for distributed -- functions/procedures distribution_argument_index int, colocationid int, CONSTRAINT pg_dist_object_pkey PRIMARY KEY (classid, objid, objsubid) ); CREATE FUNCTION master_dist_object_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_object_cache_invalidate$$; COMMENT ON FUNCTION master_dist_object_cache_invalidate() IS 'register relcache invalidation for changed rows'; CREATE TRIGGER dist_object_cache_invalidate AFTER INSERT OR UPDATE OR DELETE ON citus.pg_dist_object FOR EACH ROW EXECUTE PROCEDURE master_dist_object_cache_invalidate(); #include "udfs/create_distributed_function/9.0-1.sql" #include "udfs/citus_drop_trigger/9.0-1.sql" #include "udfs/citus_prepare_pg_upgrade/9.0-1.sql" #include "udfs/citus_finish_pg_upgrade/9.0-1.sql" -- We truncate pg_dist_node during metadata syncing, but we do not want -- this to cascade to pg_dist_poolinfo, which is generally maintained -- by the operator. ALTER TABLE pg_dist_poolinfo DROP CONSTRAINT pg_dist_poolinfo_nodeid_fkey; -- if the rebalancer extension is still around, drop it before creating Citus functions DROP EXTENSION IF EXISTS shard_rebalancer; #include "udfs/get_rebalance_table_shards_plan/9.0-1.sql" #include "udfs/replicate_table_shards/9.0-1.sql" #include "udfs/rebalance_table_shards/9.0-1.sql" #include "udfs/get_rebalance_progress/9.0-1.sql" DROP FUNCTION master_add_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_node(nodename text, nodeport integer, groupid integer default 0, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_node$$; COMMENT ON FUNCTION master_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; DROP FUNCTION master_add_inactive_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_inactive_node(nodename text, nodeport integer, groupid integer default 0, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_add_inactive_node$$; COMMENT ON FUNCTION master_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; DROP FUNCTION master_activate_node(text, integer); CREATE FUNCTION master_activate_node(nodename text, nodeport integer) RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_activate_node$$; COMMENT ON FUNCTION master_activate_node(nodename text, nodeport integer) IS 'activate a node which is in the cluster'; DROP FUNCTION master_add_secondary_node(text, integer, text, integer, name); CREATE FUNCTION master_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_secondary_node$$; COMMENT ON FUNCTION master_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name) IS 'add a secondary node to the cluster'; REVOKE ALL ON FUNCTION master_activate_node(text,int) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_node(text,int,int,noderole,name) FROM PUBLIC; REVOKE ALL ON FUNCTION master_add_secondary_node(text,int,text,int,name) FROM PUBLIC; ALTER TABLE pg_dist_node ADD COLUMN metadatasynced BOOLEAN DEFAULT FALSE; COMMENT ON COLUMN pg_dist_node.metadatasynced IS 'indicates whether the node has the most recent metadata'; CREATE FUNCTION worker_apply_sequence_command(create_sequence_command text, sequence_type_id regtype DEFAULT 'bigint'::regtype) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_apply_sequence_command$$; COMMENT ON FUNCTION worker_apply_sequence_command(text,regtype) IS 'create a sequence which produces globally unique values'; #include "udfs/citus_isolation_test_session_is_blocked/9.0-1.sql" CREATE FUNCTION ensure_truncate_trigger_is_after() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- register triggers -- FOR table_name, trigger_name IN SELECT tgrelid::regclass, tgname FROM pg_dist_partition JOIN pg_trigger ON tgrelid=logicalrelid JOIN pg_class ON pg_class.oid=logicalrelid WHERE tgname LIKE 'truncate_trigger_%' AND tgfoid = 'citus_truncate_trigger'::regproc LOOP command := 'drop trigger ' || trigger_name || ' on ' || table_name; EXECUTE command; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; END; $$; SELECT ensure_truncate_trigger_is_after(); DROP FUNCTION ensure_truncate_trigger_is_after; -- This sequence is unused DROP SEQUENCE pg_catalog.pg_dist_jobid_seq; RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--9.0-1--9.0-2.sql ================================================ -- Using the citus schema is a bad idea since many environments use "citus" -- as the main user and the "citus" schema then sits in front of the -- search_path. REVOKE USAGE ON SCHEMA citus FROM public; -- redefine distributed_tables_colocated to avoid using citus schema #include "udfs/distributed_tables_colocated/9.0-2.sql" -- type was used in old version of distributed_tables_colocated DROP TYPE citus.colocation_placement_type; ================================================ FILE: src/backend/distributed/sql/citus--9.0-2--9.1-1.sql ================================================ ALTER TABLE pg_catalog.pg_dist_node ADD shouldhaveshards bool NOT NULL DEFAULT true; COMMENT ON COLUMN pg_catalog.pg_dist_node.shouldhaveshards IS 'indicates whether the node is eligible to contain data from distributed tables'; #include "udfs/master_set_node_property/9.1-1.sql" #include "udfs/master_drain_node/9.1-1.sql" #include "udfs/worker_create_schema/9.1-1.sql" #include "udfs/worker_repartition_cleanup/9.1-1.sql" #include "udfs/rebalance_table_shards/9.1-1.sql" #include "udfs/get_rebalance_table_shards_plan/9.1-1.sql" #include "udfs/master_add_node/9.1-1.sql" #include "udfs/master_add_inactive_node/9.1-1.sql" #include "udfs/alter_role_if_exists/9.1-1.sql" -- we don't maintain replication factor of reference tables anymore and just -- use -1 instead. UPDATE pg_dist_colocation SET replicationfactor = -1 WHERE distributioncolumntype = 0; #include "udfs/any_value/9.1-1.sql" -- drop function which was used for upgrading from 6.0 -- creation was removed from citus--7.0-1.sql DROP FUNCTION IF EXISTS pg_catalog.master_initialize_node_metadata; -- Support infrastructure for distributing aggregation CREATE FUNCTION pg_catalog.worker_partial_agg_sfunc(internal, oid, anyelement) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.worker_partial_agg_sfunc(internal, oid, anyelement) IS 'transition function for worker_partial_agg'; CREATE FUNCTION pg_catalog.worker_partial_agg_ffunc(internal) RETURNS cstring AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.worker_partial_agg_ffunc(internal) IS 'finalizer for worker_partial_agg'; CREATE FUNCTION pg_catalog.coord_combine_agg_sfunc(internal, oid, cstring, anyelement) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_combine_agg_sfunc(internal, oid, cstring, anyelement) IS 'transition function for coord_combine_agg'; CREATE FUNCTION pg_catalog.coord_combine_agg_ffunc(internal, oid, cstring, anyelement) RETURNS anyelement AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_combine_agg_ffunc(internal, oid, cstring, anyelement) IS 'finalizer for coord_combine_agg'; -- select worker_partial_agg(agg, ...) -- equivalent to -- select to_cstring(agg_without_ffunc(...)) CREATE AGGREGATE pg_catalog.worker_partial_agg(oid, anyelement) ( STYPE = internal, SFUNC = pg_catalog.worker_partial_agg_sfunc, FINALFUNC = pg_catalog.worker_partial_agg_ffunc ); COMMENT ON AGGREGATE pg_catalog.worker_partial_agg(oid, anyelement) IS 'support aggregate for implementing partial aggregation on workers'; -- select coord_combine_agg(agg, col) -- equivalent to -- select agg_ffunc(agg_combine(from_cstring(col))) CREATE AGGREGATE pg_catalog.coord_combine_agg(oid, cstring, anyelement) ( STYPE = internal, SFUNC = pg_catalog.coord_combine_agg_sfunc, FINALFUNC = pg_catalog.coord_combine_agg_ffunc, FINALFUNC_EXTRA ); COMMENT ON AGGREGATE pg_catalog.coord_combine_agg(oid, cstring, anyelement) IS 'support aggregate for implementing combining partial aggregate results from workers'; REVOKE ALL ON FUNCTION pg_catalog.worker_partial_agg_ffunc FROM PUBLIC; REVOKE ALL ON FUNCTION pg_catalog.worker_partial_agg_sfunc FROM PUBLIC; REVOKE ALL ON FUNCTION pg_catalog.coord_combine_agg_ffunc FROM PUBLIC; REVOKE ALL ON FUNCTION pg_catalog.coord_combine_agg_sfunc FROM PUBLIC; REVOKE ALL ON FUNCTION pg_catalog.worker_partial_agg FROM PUBLIC; REVOKE ALL ON FUNCTION pg_catalog.coord_combine_agg FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_partial_agg_ffunc TO PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_partial_agg_sfunc TO PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_combine_agg_ffunc TO PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_combine_agg_sfunc TO PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_partial_agg TO PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_combine_agg TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/citus--9.1-1--9.2-1.sql ================================================ #include "udfs/read_intermediate_results/9.2-1.sql" #include "udfs/fetch_intermediate_results/9.2-1.sql" #include "udfs/worker_partition_query_result/9.2-1.sql" ALTER TABLE pg_catalog.pg_dist_colocation ADD distributioncolumncollation oid; UPDATE pg_catalog.pg_dist_colocation dc SET distributioncolumncollation = t.typcollation FROM pg_catalog.pg_type t WHERE t.oid = dc.distributioncolumntype; UPDATE pg_catalog.pg_dist_colocation dc SET distributioncolumncollation = 0 WHERE distributioncolumncollation IS NULL; ALTER TABLE pg_catalog.pg_dist_colocation ALTER COLUMN distributioncolumncollation SET NOT NULL; DROP INDEX pg_dist_colocation_configuration_index; -- distributioncolumntype should be listed first so that this index can be used for looking up reference tables' colocation id CREATE INDEX pg_dist_colocation_configuration_index ON pg_dist_colocation USING btree(distributioncolumntype, shardcount, replicationfactor, distributioncolumncollation); CREATE TABLE citus.pg_dist_rebalance_strategy( name name NOT NULL, default_strategy boolean NOT NULL DEFAULT false, shard_cost_function regproc NOT NULL, node_capacity_function regproc NOT NULL, shard_allowed_on_node_function regproc NOT NULL, default_threshold float4 NOT NULL, minimum_threshold float4 NOT NULL DEFAULT 0, UNIQUE(name) ); ALTER TABLE citus.pg_dist_rebalance_strategy SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.pg_dist_rebalance_strategy TO public; #include "udfs/citus_validate_rebalance_strategy_functions/9.2-1.sql" #include "udfs/pg_dist_rebalance_strategy_trigger_func/9.2-1.sql" CREATE TRIGGER pg_dist_rebalance_strategy_validation_trigger BEFORE INSERT OR UPDATE ON pg_dist_rebalance_strategy FOR EACH ROW EXECUTE PROCEDURE citus_internal.pg_dist_rebalance_strategy_trigger_func(); #include "udfs/citus_add_rebalance_strategy/9.2-1.sql" #include "udfs/citus_set_default_rebalance_strategy/9.2-1.sql" #include "udfs/citus_shard_cost_1/9.2-1.sql" #include "udfs/citus_shard_cost_by_disk_size/9.2-1.sql" #include "udfs/citus_node_capacity_1/9.2-1.sql" #include "udfs/citus_shard_allowed_on_node_true/9.2-1.sql" INSERT INTO pg_catalog.pg_dist_rebalance_strategy( name, default_strategy, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold ) VALUES ( 'by_shard_count', true, 'citus_shard_cost_1', 'citus_node_capacity_1', 'citus_shard_allowed_on_node_true', 0, 0 ), ( 'by_disk_size', false, 'citus_shard_cost_by_disk_size', 'citus_node_capacity_1', 'citus_shard_allowed_on_node_true', 0.1, 0.01 ); CREATE FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check() RETURNS TRIGGER LANGUAGE C AS 'MODULE_PATHNAME'; CREATE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_dist_rebalance_strategy FOR EACH STATEMENT EXECUTE FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check(); #include "udfs/master_drain_node/9.2-1.sql" #include "udfs/rebalance_table_shards/9.2-1.sql" #include "udfs/get_rebalance_table_shards_plan/9.2-1.sql" #include "udfs/citus_prepare_pg_upgrade/9.2-1.sql" #include "udfs/citus_finish_pg_upgrade/9.2-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--9.2-1--9.2-2.sql ================================================ #include "udfs/worker_create_schema/9.2-2.sql" -- reserve UINT32_MAX (4294967295) for a special node ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq MAXVALUE 4294967294; ================================================ FILE: src/backend/distributed/sql/citus--9.2-2--9.2-4.sql ================================================ -- we've some issues with versioning, and we're fixing it by bumping version -- from 9.2-2 to 9.2-4 see #3673 for details ================================================ FILE: src/backend/distributed/sql/citus--9.2-4--9.3-2.sql ================================================ -- citus--9.2-4--9.3-2 -- bump version to 9.3-2 #include "udfs/citus_extradata_container/9.3-2.sql" #include "udfs/update_distributed_table_colocation/9.3-2.sql" #include "udfs/replicate_reference_tables/9.3-2.sql" #include "udfs/citus_remote_connection_stats/9.3-2.sql" #include "udfs/worker_create_or_alter_role/9.3-2.sql" #include "udfs/truncate_local_data_after_distributing_table/9.3-2.sql" -- add citus extension owner as a distributed object, if not already in there INSERT INTO citus.pg_dist_object SELECT (SELECT oid FROM pg_class WHERE relname = 'pg_authid') AS oid, (SELECT oid FROM pg_authid WHERE rolname = current_user) as objid, 0 as objsubid ON CONFLICT DO NOTHING; ================================================ FILE: src/backend/distributed/sql/citus--9.3-1--9.2-4.sql ================================================ -- citus--9.3-1--9.2-4 -- this is an unusual upgrade path, we are doing it because -- we have accidentally tagged master branch with v9.2-3 -- however master branch was already bumped to v9.3-1 -- with this file, we are undoing the catalog changes that -- have happened between 9.2-2 to 9.3-1, and making 9.2-4 -- as the release that we can -- undo the changes for citus_extradata_container that happened on citus 9.3 DROP FUNCTION IF EXISTS pg_catalog.citus_extradata_container(INTERNAL); CREATE FUNCTION pg_catalog.citus_extradata_container(INTERNAL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_extradata_container$$; COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; DROP FUNCTION IF EXISTS pg_catalog.update_distributed_table_colocation(regclass, text); ================================================ FILE: src/backend/distributed/sql/citus--9.3-2--9.4-1.sql ================================================ -- citus--9.3-2--9.4-1 -- bump version to 9.4-1 #include "udfs/worker_last_saved_explain_analyze/9.4-1.sql" #include "udfs/worker_save_query_explain_analyze/9.4-1.sql" ================================================ FILE: src/backend/distributed/sql/citus--9.4-1--9.4-2.sql ================================================ -- 9.4-1--9.4-2 was added later as a patch to fix a bug in our PG upgrade functions #include "udfs/citus_prepare_pg_upgrade/9.4-2.sql" #include "udfs/citus_finish_pg_upgrade/9.4-2.sql" ================================================ FILE: src/backend/distributed/sql/citus--9.4-1--9.5-1.sql ================================================ -- citus--9.4-1--9.5-1 -- bump version to 9.5-1 #include "udfs/undistribute_table/9.5-1.sql" #include "udfs/create_citus_local_table/9.5-1.sql" #include "udfs/citus_drop_trigger/9.5-1.sql" #include "udfs/worker_record_sequence_dependency/9.5-1.sql" #include "udfs/citus_finish_pg_upgrade/9.5-1.sql" #include "udfs/citus_prepare_pg_upgrade/9.5-1.sql" SET search_path = 'pg_catalog'; DROP FUNCTION task_tracker_assign_task(bigint, integer, text); DROP FUNCTION task_tracker_task_status(bigint, integer); DROP FUNCTION task_tracker_cleanup_job(bigint); DROP FUNCTION worker_merge_files_and_run_query(bigint, integer, text, text); DROP FUNCTION worker_execute_sql_task(bigint, integer, text, bool); DROP TRIGGER dist_authinfo_task_tracker_cache_invalidate ON pg_catalog.pg_dist_authinfo; DROP TRIGGER dist_poolinfo_task_tracker_cache_invalidate ON pg_catalog.pg_dist_poolinfo; DROP FUNCTION task_tracker_conninfo_cache_invalidate(); DROP FUNCTION master_drop_sequences(text[]); RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--9.4-2--9.4-1.sql ================================================ -- -- 9.4-1--9.4-2 was added later as a patch to fix a bug in our PG upgrade functions -- -- This script brings users who installed the patch released back to the 9.4-1 -- upgrade path. We do this via a semantical downgrade since there has already been -- introduced new changes in the schema from 9.4-1 to 9.5-1. To make sure we include all -- changes made during that version change we decide to use the existing upgrade path from -- our later introduced 9.4-2 version. -- ================================================ FILE: src/backend/distributed/sql/citus--9.4-2--9.4-3.sql ================================================ -- 9.4-2--9.4-3 was added later as a patch to improve master_update_table_statistics CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.master_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; ================================================ FILE: src/backend/distributed/sql/citus--9.4-3--9.4-2.sql ================================================ -- citus--9.4-3--9.4-2 -- This is a downgrade path that will revert the changes made in citus--9.4-2--9.4-3.sql -- 9.4-2--9.4-3 was added later as a patch to improve master_update_table_statistics. -- We have this downgrade script so that we can continue from the main upgrade path -- when upgrading to later versions. CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID AS $$ DECLARE colocated_tables regclass[]; BEGIN SELECT get_colocated_table_array(relation) INTO colocated_tables; PERFORM master_update_shard_statistics(shardid) FROM pg_dist_shard WHERE logicalrelid = ANY (colocated_tables); END; $$ LANGUAGE 'plpgsql'; COMMENT ON FUNCTION master_update_table_statistics(regclass) IS 'updates shard statistics of the given table and its colocated tables'; ================================================ FILE: src/backend/distributed/sql/citus--9.5-1--10.0-4.sql ================================================ -- citus--9.5-1--10.0-4 -- This migration file aims to fix the issues with upgrades on clusters without public schema. -- This file is created by the following command, and some more changes in a separate commit -- cat citus--9.5-1--10.0-1.sql citus--10.0-1--10.0-2.sql citus--10.0-2--10.0-3.sql > citus--9.5-1--10.0-4.sql -- copy of citus--9.5-1--10.0-1 DROP FUNCTION pg_catalog.upgrade_to_reference_table(regclass); DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass); #include "udfs/citus_total_relation_size/10.0-1.sql" #include "udfs/citus_finish_pg_upgrade/10.0-1.sql" #include "udfs/alter_distributed_table/10.0-1.sql" #include "udfs/alter_table_set_access_method/10.0-1.sql" #include "udfs/undistribute_table/10.0-1.sql" #include "udfs/create_citus_local_table/10.0-1.sql" #include "udfs/citus_set_coordinator_host/10.0-1.sql" #include "udfs/citus_add_node/10.0-1.sql" #include "udfs/citus_activate_node/10.0-1.sql" #include "udfs/citus_add_inactive_node/10.0-1.sql" #include "udfs/citus_add_secondary_node/10.0-1.sql" #include "udfs/citus_disable_node/10.0-1.sql" #include "udfs/citus_drain_node/10.0-1.sql" #include "udfs/citus_remove_node/10.0-1.sql" #include "udfs/citus_set_node_property/10.0-1.sql" #include "udfs/citus_unmark_object_distributed/10.0-1.sql" #include "udfs/citus_update_node/10.0-1.sql" #include "udfs/citus_update_shard_statistics/10.0-1.sql" #include "udfs/citus_update_table_statistics/10.0-1.sql" #include "udfs/citus_copy_shard_placement/10.0-1.sql" #include "udfs/citus_move_shard_placement/10.0-1.sql" #include "udfs/citus_drop_trigger/10.0-1.sql" #include "udfs/worker_change_sequence_dependency/10.0-1.sql" #include "udfs/remove_local_tables_from_metadata/10.0-1.sql" --#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--9.5-1--10.0-1.sql" END IF; END; $check_columnar$; #include "udfs/time_partition_range/10.0-1.sql" #include "udfs/time_partitions/10.0-1.sql" #include "udfs/alter_old_partitions_set_access_method/10.0-1.sql" ALTER FUNCTION pg_catalog.master_conninfo_cache_invalidate() RENAME TO citus_conninfo_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_local_group_cache_invalidate() RENAME TO citus_dist_local_group_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_node_cache_invalidate() RENAME TO citus_dist_node_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_object_cache_invalidate() RENAME TO citus_dist_object_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_partition_cache_invalidate() RENAME TO citus_dist_partition_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_placement_cache_invalidate() RENAME TO citus_dist_placement_cache_invalidate; ALTER FUNCTION pg_catalog.master_dist_shard_cache_invalidate() RENAME TO citus_dist_shard_cache_invalidate; #include "udfs/citus_conninfo_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_local_group_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_node_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_object_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_partition_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_placement_cache_invalidate/10.0-1.sql" #include "udfs/citus_dist_shard_cache_invalidate/10.0-1.sql" ALTER FUNCTION pg_catalog.master_drop_all_shards(regclass, text, text) RENAME TO citus_drop_all_shards; DROP FUNCTION pg_catalog.master_modify_multiple_shards(text); DROP FUNCTION pg_catalog.master_create_distributed_table(regclass, text, citus.distribution_type); DROP FUNCTION pg_catalog.master_create_worker_shards(text, integer, integer); DROP FUNCTION pg_catalog.mark_tables_colocated(regclass, regclass[]); #include "udfs/citus_shard_sizes/10.0-1.sql" #include "udfs/citus_shards/10.0-1.sql" #include "udfs/fix_pre_citus10_partitioned_table_constraint_names/10.0-1.sql" #include "udfs/worker_fix_pre_citus10_partitioned_table_constraint_names/10.0-1.sql" DROP FUNCTION pg_catalog.citus_dist_stat_activity CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_dist_stat_activity$$; COMMENT ON FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on distributed tables'; CREATE VIEW citus.citus_dist_stat_activity AS SELECT * FROM pg_catalog.citus_dist_stat_activity(); ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH citus_dist_stat_activity AS ( SELECT * FROM citus_dist_stat_activity ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_node_id, waiting_transaction_num, blocking_node_id, blocking_transaction_num) * FROM dump_global_wait_edges() ), citus_dist_stat_activity_with_node_id AS ( SELECT citus_dist_stat_activity.*, (CASE citus_dist_stat_activity.distributed_query_host_name WHEN 'coordinator_host' THEN 0 ELSE pg_dist_node.nodeid END) as initiator_node_id FROM citus_dist_stat_activity LEFT JOIN pg_dist_node ON citus_dist_stat_activity.distributed_query_host_name = pg_dist_node.nodename AND citus_dist_stat_activity.distributed_query_host_port = pg_dist_node.nodeport ) SELECT waiting.pid AS waiting_pid, blocking.pid AS blocking_pid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.initiator_node_id AS waiting_node_id, blocking.initiator_node_id AS blocking_node_id, waiting.distributed_query_host_name AS waiting_node_name, blocking.distributed_query_host_name AS blocking_node_name, waiting.distributed_query_host_port AS waiting_node_port, blocking.distributed_query_host_port AS blocking_node_port FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_node_id waiting ON (unique_global_wait_edges.waiting_transaction_num = waiting.transaction_number AND unique_global_wait_edges.waiting_node_id = waiting.initiator_node_id) JOIN citus_dist_stat_activity_with_node_id blocking ON (unique_global_wait_edges.blocking_transaction_num = blocking.transaction_number AND unique_global_wait_edges.blocking_node_id = blocking.initiator_node_id); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; DROP FUNCTION citus_worker_stat_activity CASCADE; CREATE OR REPLACE FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_worker_stat_activity$$; COMMENT ON FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on shards of distributed tables'; CREATE VIEW citus.citus_worker_stat_activity AS SELECT * FROM pg_catalog.citus_worker_stat_activity(); ALTER VIEW citus.citus_worker_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_worker_stat_activity TO PUBLIC; -- copy of citus--10.0-1--10.0-2 --#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql" DO $check_columnar$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid) INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid) WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler' ) THEN #include "../../columnar/sql/columnar--10.0-1--10.0-2.sql" END IF; END; $check_columnar$; -- copy of citus--10.0-2--10.0-3 #include "udfs/citus_update_table_statistics/10.0-3.sql" CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.master_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; CREATE OR REPLACE FUNCTION pg_catalog.citus_get_active_worker_nodes(OUT node_name text, OUT node_port bigint) RETURNS SETOF record LANGUAGE C STRICT ROWS 100 AS 'MODULE_PATHNAME', $$citus_get_active_worker_nodes$$; COMMENT ON FUNCTION pg_catalog.citus_get_active_worker_nodes() IS 'fetch set of active worker nodes'; -- copy of citus--10.0-3--10.0-4 -- This migration file aims to fix 2 issues with upgrades on clusters -- 1. a bug in public schema dependency for citus_tables view. -- -- Users who do not have public schema in their clusters were unable to upgrade -- to Citus 10.x due to the citus_tables view that used to be created in public -- schema #include "udfs/citus_tables/10.0-4.sql" -- 2. a bug in our PG upgrade functions -- -- Users who took the 9.5-2--10.0-1 upgrade path already have the fix, but users -- who took the 9.5-1--10.0-1 upgrade path do not. Hence, we repeat the CREATE OR -- REPLACE from the 9.5-2 definition for citus_prepare_pg_upgrade. #include "udfs/citus_prepare_pg_upgrade/9.5-2.sql" #include "udfs/citus_finish_pg_upgrade/10.0-4.sql" RESET search_path; ================================================ FILE: src/backend/distributed/sql/citus--9.5-1--9.5-2.sql ================================================ -- 9.5-1--9.5-2 was added later as a patch to fix a bug in our PG upgrade functions #include "udfs/citus_prepare_pg_upgrade/9.5-2.sql" #include "udfs/citus_finish_pg_upgrade/9.5-2.sql" ================================================ FILE: src/backend/distributed/sql/citus--9.5-2--9.5-1.sql ================================================ -- -- 9.5-1--9.5-2 was added later as a patch to fix a bug in our PG upgrade functions -- -- This script brings users who installed the patch released back to the 9.5-1 -- upgrade path. We do this via a semantical downgrade since there has already been -- introduced new changes in the schema from 9.5-1 to 10.0-1. To make sure we include all -- changes made during that version change we decide to use the existing upgrade path from -- our later introduced 9.5-1 version. -- ================================================ FILE: src/backend/distributed/sql/citus--9.5-2--9.5-3.sql ================================================ -- 9.5-2--9.5-3 was added later as a patch to improve master_update_table_statistics CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.master_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; ================================================ FILE: src/backend/distributed/sql/citus--9.5-3--9.5-2.sql ================================================ -- citus--9.5-3--9.5-2 -- This is a downgrade path that will revert the changes made in citus--9.5-2--9.5-3.sql -- 9.5-2--9.5-3 was added later as a patch to improve master_update_table_statistics. -- We have this downgrade script so that we can continue from the main upgrade path -- when upgrading to later versions. CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID AS $$ DECLARE colocated_tables regclass[]; BEGIN SELECT get_colocated_table_array(relation) INTO colocated_tables; PERFORM master_update_shard_statistics(shardid) FROM pg_dist_shard WHERE logicalrelid = ANY (colocated_tables); END; $$ LANGUAGE 'plpgsql'; COMMENT ON FUNCTION master_update_table_statistics(regclass) IS 'updates shard statistics of the given table and its colocated tables'; ================================================ FILE: src/backend/distributed/sql/datatypes/citus_cluster_clock/11.2-1.sql ================================================ -- -- cluster_clock base type is a combination of -- uint64 cluster clock logical timestamp at the commit -- uint32 cluster clock counter(ticks with in the logical clock) -- CREATE TYPE citus.cluster_clock; CREATE FUNCTION pg_catalog.cluster_clock_in(cstring) RETURNS citus.cluster_clock AS 'MODULE_PATHNAME',$$cluster_clock_in$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_out(citus.cluster_clock) RETURNS cstring AS 'MODULE_PATHNAME',$$cluster_clock_out$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_recv(internal) RETURNS citus.cluster_clock AS 'MODULE_PATHNAME',$$cluster_clock_recv$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_send(citus.cluster_clock) RETURNS bytea AS 'MODULE_PATHNAME',$$cluster_clock_send$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_logical(citus.cluster_clock) RETURNS bigint AS 'MODULE_PATHNAME',$$cluster_clock_logical$$ LANGUAGE C IMMUTABLE STRICT; CREATE TYPE citus.cluster_clock ( internallength = 12, -- specifies the size of the memory block required to hold the type uint64 + uint32 input = cluster_clock_in, output = cluster_clock_out, receive = cluster_clock_recv, send = cluster_clock_send ); ALTER TYPE citus.cluster_clock SET SCHEMA pg_catalog; COMMENT ON TYPE cluster_clock IS 'combination of (logical, counter): 42 bits + 22 bits'; -- -- Define the required operators -- CREATE FUNCTION pg_catalog.cluster_clock_lt(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_lt$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_le(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_le$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_eq(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_eq$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_ne(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_ne$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_ge(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_ge$$ LANGUAGE C IMMUTABLE STRICT; CREATE FUNCTION pg_catalog.cluster_clock_gt(cluster_clock, cluster_clock) RETURNS bool AS 'MODULE_PATHNAME',$$cluster_clock_gt$$ LANGUAGE C IMMUTABLE STRICT; CREATE OPERATOR < ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_lt, commutator = > , negator = >= , restrict = scalarltsel, join = scalarltjoinsel ); CREATE OPERATOR <= ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_le, commutator = >= , negator = > , restrict = scalarlesel, join = scalarlejoinsel ); CREATE OPERATOR = ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_eq, commutator = = , negator = <> , restrict = eqsel, join = eqjoinsel ); CREATE OPERATOR <> ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_ne, commutator = <> , negator = = , restrict = neqsel, join = neqjoinsel ); CREATE OPERATOR >= ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_ge, commutator = <= , negator = < , restrict = scalargesel, join = scalargejoinsel ); CREATE OPERATOR > ( leftarg = cluster_clock, rightarg = cluster_clock, procedure = cluster_clock_gt, commutator = < , negator = <= , restrict = scalargtsel, join = scalargtjoinsel ); -- Create the support function too CREATE FUNCTION pg_catalog.cluster_clock_cmp(cluster_clock, cluster_clock) RETURNS int4 AS 'MODULE_PATHNAME',$$cluster_clock_cmp$$ LANGUAGE C IMMUTABLE STRICT; -- Define operator class to be be used by an index for type cluster_clock. CREATE OPERATOR CLASS pg_catalog.cluster_clock_ops DEFAULT FOR TYPE cluster_clock USING btree AS OPERATOR 1 < , OPERATOR 2 <= , OPERATOR 3 = , OPERATOR 4 >= , OPERATOR 5 > , FUNCTION 1 cluster_clock_cmp(cluster_clock, cluster_clock); -- -- Create sequences for logical and counter fields of the type cluster_clock, to -- be used as a storage. -- CREATE SEQUENCE citus.pg_dist_clock_logical_seq START 1; ALTER SEQUENCE citus.pg_dist_clock_logical_seq SET SCHEMA pg_catalog; REVOKE UPDATE ON SEQUENCE pg_catalog.pg_dist_clock_logical_seq FROM public; ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.0-4--9.5-1.sql ================================================ -- citus--10.0-4--9.5-1 -- This migration file aims to fix the issues with upgrades on clusters without public schema. -- This file is created by the following command, and some more changes in a separate commit -- cat citus--10.0-3--10.0-2.sql citus--10.0-2--10.0-1.sql citus--10.0-1--9.5-1.sql > citus--10.0-4--9.5-1.sql -- copy of citus--10.0-4--10.0-3 -- -- 10.0-3--10.0-4 was added later as a patch to fix a bug in our PG upgrade functions -- -- The upgrade fixes a bug in citus_(prepare|finish)_pg_upgrade. Given the old versions of -- these functions contain a bug it is better to _not_ restore the old version and keep -- the patched version of the function. -- -- This is inline with the downgrade scripts for earlier versions of this patch -- -- copy of citus--10.0-3--10.0-2 -- this is a downgrade path that will revert the changes made in citus--10.0-2--10.0-3.sql DROP FUNCTION pg_catalog.citus_update_table_statistics(regclass); #include "../udfs/citus_update_table_statistics/10.0-1.sql" CREATE OR REPLACE FUNCTION master_update_table_statistics(relation regclass) RETURNS VOID AS $$ DECLARE colocated_tables regclass[]; BEGIN SELECT get_colocated_table_array(relation) INTO colocated_tables; PERFORM master_update_shard_statistics(shardid) FROM pg_dist_shard WHERE logicalrelid = ANY (colocated_tables); END; $$ LANGUAGE 'plpgsql'; COMMENT ON FUNCTION master_update_table_statistics(regclass) IS 'updates shard statistics of the given table and its colocated tables'; DROP FUNCTION pg_catalog.citus_get_active_worker_nodes(OUT text, OUT bigint); -- copy of citus--10.0-2--10.0-1.sql #include "../../../columnar/sql/downgrades/columnar--10.0-2--10.0-1.sql" -- copy of citus--10.0-1--9.5-1 -- In Citus 10.0, we added another internal udf (notify_constraint_dropped) -- to be called by citus_drop_trigger. Since this script is executed when -- downgrading Citus, we don't have notify_constraint_dropped in citus.so. -- For this reason, we first need to downgrade citus_drop_trigger so it doesn't -- call notify_constraint_dropped. -- To downgrade citus_drop_trigger, we first need to have the old version of -- citus_drop_all_shards as we renamed it in Citus 10.0. ALTER FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text) RENAME TO master_drop_all_shards; #include "../udfs/citus_drop_trigger/9.5-1.sql" -- Now we can safely drop notify_constraint_dropped as we downgraded citus_drop_trigger. DROP FUNCTION pg_catalog.notify_constraint_dropped(); #include "../udfs/citus_finish_pg_upgrade/9.5-1.sql" #include "../../../columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql" DROP VIEW IF EXISTS pg_catalog.citus_tables; DROP VIEW IF EXISTS public.citus_tables; DROP FUNCTION pg_catalog.alter_distributed_table(regclass, text, int, text, boolean); DROP FUNCTION pg_catalog.alter_table_set_access_method(regclass, text); DROP FUNCTION pg_catalog.citus_total_relation_size(regclass,boolean); DROP FUNCTION pg_catalog.undistribute_table(regclass,boolean); DROP FUNCTION pg_catalog.citus_add_local_table_to_metadata(regclass,boolean); DROP FUNCTION pg_catalog.citus_add_node(text, integer, integer, noderole, name); DROP FUNCTION pg_catalog.citus_activate_node(text, integer); DROP FUNCTION pg_catalog.citus_add_inactive_node(text, integer, integer, noderole, name); DROP FUNCTION pg_catalog.citus_add_secondary_node(text, integer, text, integer, name); DROP FUNCTION pg_catalog.citus_disable_node(text, integer); DROP FUNCTION pg_catalog.citus_drain_node(text, integer, citus.shard_transfer_mode, name); DROP FUNCTION pg_catalog.citus_remove_node(text, integer); DROP FUNCTION pg_catalog.citus_set_node_property(text, integer, text, boolean); DROP FUNCTION pg_catalog.citus_unmark_object_distributed(oid, oid, int); DROP FUNCTION pg_catalog.citus_update_node(int, text, int, bool, int); DROP FUNCTION pg_catalog.citus_update_shard_statistics(bigint); DROP FUNCTION pg_catalog.citus_update_table_statistics(regclass); DROP FUNCTION pg_catalog.citus_copy_shard_placement(bigint, text, integer, text, integer, bool, citus.shard_transfer_mode); DROP FUNCTION pg_catalog.citus_move_shard_placement(bigint, text, integer, text, integer, citus.shard_transfer_mode); ALTER FUNCTION pg_catalog.citus_conninfo_cache_invalidate() RENAME TO master_conninfo_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_local_group_cache_invalidate() RENAME TO master_dist_local_group_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_node_cache_invalidate() RENAME TO master_dist_node_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_object_cache_invalidate() RENAME TO master_dist_object_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_partition_cache_invalidate() RENAME TO master_dist_partition_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_placement_cache_invalidate() RENAME TO master_dist_placement_cache_invalidate; ALTER FUNCTION pg_catalog.citus_dist_shard_cache_invalidate() RENAME TO master_dist_shard_cache_invalidate; #include "../udfs/citus_conninfo_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_local_group_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_node_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_object_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_partition_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_placement_cache_invalidate/9.5-1.sql" #include "../udfs/citus_dist_shard_cache_invalidate/9.5-1.sql" DROP VIEW pg_catalog.time_partitions; DROP FUNCTION pg_catalog.time_partition_range(regclass); DROP PROCEDURE pg_catalog.alter_old_partitions_set_access_method(regclass,timestamptz,name); DROP FUNCTION pg_catalog.citus_set_coordinator_host(text,int,noderole,name); DROP FUNCTION pg_catalog.worker_change_sequence_dependency(regclass, regclass, regclass); CREATE FUNCTION pg_catalog.mark_tables_colocated(source_table_name regclass, target_table_names regclass[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$mark_tables_colocated$$; COMMENT ON FUNCTION pg_catalog.mark_tables_colocated(source_table_name regclass, target_table_names regclass[]) IS 'mark target distributed tables as colocated with the source table'; CREATE FUNCTION pg_catalog.master_modify_multiple_shards(text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_modify_multiple_shards$$; COMMENT ON FUNCTION master_modify_multiple_shards(text) IS 'push delete and update queries to shards'; CREATE FUNCTION pg_catalog.master_create_distributed_table(table_name regclass, distribution_column text, distribution_method citus.distribution_type) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_create_distributed_table$$; COMMENT ON FUNCTION pg_catalog.master_create_distributed_table(table_name regclass, distribution_column text, distribution_method citus.distribution_type) IS 'define the table distribution functions'; CREATE FUNCTION pg_catalog.master_create_worker_shards(table_name text, shard_count integer, replication_factor integer DEFAULT 2) RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; DROP FUNCTION pg_catalog.remove_local_tables_from_metadata(); #include "../udfs/citus_total_relation_size/7.0-1.sql" #include "../udfs/upgrade_to_reference_table/8.0-1.sql" #include "../udfs/undistribute_table/9.5-1.sql" #include "../udfs/create_citus_local_table/9.5-1.sql" DROP VIEW pg_catalog.citus_shards CASCADE; DROP FUNCTION pg_catalog.citus_shard_sizes(OUT table_name text, OUT size bigint); DROP FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(); DROP FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(regclass); DROP FUNCTION pg_catalog.worker_fix_pre_citus10_partitioned_table_constraint_names(regclass,bigint,text); DROP FUNCTION pg_catalog.citus_dist_stat_activity CASCADE; DROP FUNCTION pg_catalog.citus_worker_stat_activity CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_dist_stat_activity$$; COMMENT ON FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on distributed tables'; CREATE VIEW citus.citus_dist_stat_activity AS SELECT * FROM pg_catalog.citus_dist_stat_activity(); ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH citus_dist_stat_activity AS ( SELECT * FROM citus_dist_stat_activity ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_node_id, waiting_transaction_num, blocking_node_id, blocking_transaction_num) * FROM dump_global_wait_edges() ), citus_dist_stat_activity_with_node_id AS ( SELECT citus_dist_stat_activity.*, (CASE citus_dist_stat_activity.master_query_host_name WHEN 'coordinator_host' THEN 0 ELSE pg_dist_node.nodeid END) as initiator_node_id FROM citus_dist_stat_activity LEFT JOIN pg_dist_node ON citus_dist_stat_activity.master_query_host_name = pg_dist_node.nodename AND citus_dist_stat_activity.master_query_host_port = pg_dist_node.nodeport ) SELECT waiting.pid AS waiting_pid, blocking.pid AS blocking_pid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.initiator_node_id AS waiting_node_id, blocking.initiator_node_id AS blocking_node_id, waiting.master_query_host_name AS waiting_node_name, blocking.master_query_host_name AS blocking_node_name, waiting.master_query_host_port AS waiting_node_port, blocking.master_query_host_port AS blocking_node_port FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_node_id waiting ON (unique_global_wait_edges.waiting_transaction_num = waiting.transaction_number AND unique_global_wait_edges.waiting_node_id = waiting.initiator_node_id) JOIN citus_dist_stat_activity_with_node_id blocking ON (unique_global_wait_edges.blocking_transaction_num = blocking.transaction_number AND unique_global_wait_edges.blocking_node_id = blocking.initiator_node_id); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; CREATE OR REPLACE FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_worker_stat_activity$$; COMMENT ON FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT master_query_host_name text, OUT master_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on shards of distributed tables'; CREATE VIEW citus.citus_worker_stat_activity AS SELECT * FROM pg_catalog.citus_worker_stat_activity(); ALTER VIEW citus.citus_worker_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_worker_stat_activity TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.1-1--10.0-4.sql ================================================ -- citus--10.1-1--10.0-4 -- This migration file aims to fix the issues with upgrades on clusters without public schema. -- copy of citus--10.1-1--10.0-3 -- remove databases as distributed objects to prevent unknown object types being managed -- on older versions. DELETE FROM citus.pg_dist_object WHERE classid = 'pg_catalog.pg_database'::regclass::oid; #include "../../../columnar/sql/downgrades/columnar--10.1-1--10.0-3.sql" DROP FUNCTION pg_catalog.create_distributed_table(regclass, text, citus.distribution_type, text, int); CREATE FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$create_distributed_table$$; COMMENT ON FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text) IS 'creates a distributed table'; DROP FUNCTION pg_catalog.worker_partitioned_relation_total_size(regclass); DROP FUNCTION pg_catalog.worker_partitioned_relation_size(regclass); DROP FUNCTION pg_catalog.worker_partitioned_table_size(regclass); DROP FUNCTION pg_catalog.citus_local_disk_space_stats(); #include "../udfs/citus_prepare_pg_upgrade/9.5-1.sql" #include "../udfs/citus_finish_pg_upgrade/10.0-1.sql" #include "../udfs/get_rebalance_table_shards_plan/9.2-1.sql" -- the migration for citus_add_rebalance_strategy from 9.2-1 was the first one, -- so it doesn't have a DROP. This is why we DROP manually here. DROP FUNCTION pg_catalog.citus_add_rebalance_strategy; #include "../udfs/citus_add_rebalance_strategy/9.2-1.sql" ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DROP COLUMN improvement_threshold; -- the migration for get_rebalance_progress from 9.0-1 was the first one, -- so it doesn't have a DROP. This is why we DROP manually here. DROP FUNCTION pg_catalog.get_rebalance_progress; #include "../udfs/get_rebalance_progress/9.0-1.sql" CREATE OR REPLACE VIEW pg_catalog.citus_shards AS WITH shard_sizes AS (SELECT * FROM pg_catalog.citus_shard_sizes()) SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, (SELECT size FROM shard_sizes WHERE shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) = table_name OR 'public.' || shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) = table_name LIMIT 1) as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid ORDER BY pg_dist_shard.logicalrelid::text, shardid ; #include "../udfs/citus_finish_pg_upgrade/10.0-1.sql" CREATE FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check() RETURNS TRIGGER LANGUAGE C AS 'MODULE_PATHNAME'; CREATE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_dist_rebalance_strategy FOR EACH STATEMENT EXECUTE FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check(); DROP PROCEDURE pg_catalog.citus_cleanup_orphaned_shards(); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.2-1--10.1-1.sql ================================================ -- citus--10.2-1--10.1-1 #include "../../../columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql" DROP FUNCTION pg_catalog.stop_metadata_sync_to_node(text, integer, bool); CREATE FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$stop_metadata_sync_to_node$$; COMMENT ON FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer) IS 'stop metadata sync to node'; DROP FUNCTION pg_catalog.citus_internal_add_partition_metadata(regclass, "char", text, integer, "char"); DROP FUNCTION pg_catalog.citus_internal_add_shard_metadata(regclass, bigint, "char", text, text); DROP FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, integer, bigint, integer, bigint); DROP FUNCTION pg_catalog.citus_internal_update_placement_metadata(bigint, integer, integer); DROP FUNCTION pg_catalog.citus_internal_delete_shard_metadata(bigint); DROP FUNCTION pg_catalog.citus_internal_update_relation_colocation(oid, integer); DROP FUNCTION pg_catalog.create_time_partitions(regclass, interval, timestamp with time zone, timestamp with time zone); DROP FUNCTION pg_catalog.get_missing_time_partition_ranges(regclass, interval, timestamp with time zone, timestamp with time zone); DROP FUNCTION pg_catalog.worker_nextval(regclass); DROP PROCEDURE pg_catalog.drop_old_time_partitions(regclass, timestamptz); REVOKE ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) FROM PUBLIC; ALTER TABLE pg_catalog.pg_dist_placement DROP CONSTRAINT placement_shardid_groupid_unique_index; DROP FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text, boolean); CREATE FUNCTION pg_catalog.citus_drop_all_shards(logicalrelid regclass, schema_name text, table_name text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drop_all_shards$$; COMMENT ON FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text) IS 'drop all shards in a relation and update metadata'; #include "../udfs/citus_drop_trigger/10.0-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.2-2--10.2-1.sql ================================================ -- citus--10.2-2--10.2-1 #include "../../../columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.2-3--10.2-2.sql ================================================ -- citus--10.2-3--10.2-2 #include "../../../columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--10.2-4--10.2-3.sql ================================================ -- citus--10.2-4--10.2-3 DROP FUNCTION pg_catalog.fix_all_partition_shard_index_names(); DROP FUNCTION pg_catalog.fix_partition_shard_index_names(regclass); DROP FUNCTION pg_catalog.worker_fix_partition_shard_index_names(regclass, text, text); #include "../udfs/citus_finish_pg_upgrade/10.2-1.sql" -- This needs to be done after downgrading citus_finish_pg_upgrade. This is -- because citus_finish_pg_upgrade/10.2-4 depends on columnar_ensure_am_depends_catalog, -- which is dropped by columnar--10.2-4--10.2-3.sql #include "../../../columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.0-1--10.2-4.sql ================================================ -- citus--11.0-1--10.2-4 REVOKE SELECT ON pg_catalog.pg_dist_object FROM public; ALTER TABLE pg_catalog.pg_dist_object SET SCHEMA citus; DROP FUNCTION pg_catalog.create_distributed_function(regprocedure, text, text, bool); DROP FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean, boolean, boolean); #include "../udfs/worker_partition_query_result/9.2-1.sql" CREATE FUNCTION pg_catalog.master_apply_delete_command(text) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_apply_delete_command$$; COMMENT ON FUNCTION pg_catalog.master_apply_delete_command(text) IS 'drop shards matching delete criteria and update metadata'; CREATE FUNCTION pg_catalog.master_get_table_metadata( relation_name text, OUT logical_relid oid, OUT part_storage_type "char", OUT part_method "char", OUT part_key text, OUT part_replica_count integer, OUT part_max_size bigint, OUT part_placement_policy integer) RETURNS record LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$master_get_table_metadata$$; COMMENT ON FUNCTION master_get_table_metadata(relation_name text) IS 'fetch metadata values for the table'; ALTER TABLE pg_catalog.pg_dist_partition DROP COLUMN autoconverted; CREATE FUNCTION master_append_table_to_shard(bigint, text, text, integer) RETURNS real LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_append_table_to_shard$$; COMMENT ON FUNCTION master_append_table_to_shard(bigint, text, text, integer) IS 'append given table to all shard placements and update metadata'; GRANT ALL ON FUNCTION start_metadata_sync_to_node(text, integer) TO PUBLIC; GRANT ALL ON FUNCTION stop_metadata_sync_to_node(text, integer,bool) TO PUBLIC; DROP FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool); CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer) IS 'removes node from the cluster temporarily'; DROP FUNCTION pg_catalog.citus_check_connection_to_node (text, integer); DROP FUNCTION pg_catalog.citus_check_cluster_node_health (); DROP FUNCTION pg_catalog.citus_internal_add_object_metadata(text, text[], text[], integer, integer, boolean); DROP FUNCTION pg_catalog.citus_internal_add_colocation_metadata(int, int, int, regtype, oid); DROP FUNCTION pg_catalog.citus_internal_delete_colocation_metadata(int); DROP FUNCTION pg_catalog.citus_run_local_command(text); DROP FUNCTION pg_catalog.worker_drop_sequence_dependency(text); DROP FUNCTION pg_catalog.worker_drop_shell_table(table_name text); CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','S','f','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; DROP FUNCTION pg_catalog.citus_shards_on_worker(); DROP FUNCTION pg_catalog.citus_shard_indexes_on_worker(); #include "../udfs/create_distributed_function/9.0-1.sql" ALTER TABLE citus.pg_dist_object DROP COLUMN force_delegation; SET search_path = 'pg_catalog'; DROP FUNCTION IF EXISTS get_all_active_transactions(); CREATE OR REPLACE FUNCTION get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION get_all_active_transactions(OUT datid oid, OUT datname text, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns distributed transaction ids of active distributed transactions'; DROP FUNCTION IF EXISTS get_global_active_transactions(); CREATE FUNCTION get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_global_active_transactions$$; COMMENT ON FUNCTION get_global_active_transactions(OUT database_id oid, OUT process_id int, OUT initiator_node_identifier int4, OUT transaction_number int8, OUT transaction_stamp timestamptz) IS 'returns distributed transaction ids of active distributed transactions from each node of the cluster'; RESET search_path; DROP VIEW pg_catalog.citus_lock_waits; DROP FUNCTION citus_internal_local_blocked_processes; DROP FUNCTION citus_internal_global_blocked_processes; DROP VIEW IF EXISTS pg_catalog.citus_dist_stat_activity; DROP FUNCTION IF EXISTS pg_catalog.citus_dist_stat_activity; CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_dist_stat_activity$$; COMMENT ON FUNCTION pg_catalog.citus_dist_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on distributed tables'; CREATE VIEW citus.citus_dist_stat_activity AS SELECT * FROM pg_catalog.citus_dist_stat_activity(); ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; SET search_path = 'pg_catalog'; DROP VIEW IF EXISTS citus_worker_stat_activity; DROP FUNCTION IF EXISTS citus_worker_stat_activity; CREATE OR REPLACE FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_worker_stat_activity$$; COMMENT ON FUNCTION citus_worker_stat_activity(OUT query_hostname text, OUT query_hostport int, OUT distributed_query_host_name text, OUT distributed_query_host_port int, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT datid oid, OUT datname name, OUT pid int, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr INET, OUT client_hostname TEXT, OUT client_port int, OUT backend_start timestamptz, OUT xact_start timestamptz, OUT query_start timestamptz, OUT state_change timestamptz, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query text, OUT backend_type text) IS 'returns distributed transaction activity on shards of distributed tables'; DROP FUNCTION pg_catalog.worker_create_or_replace_object(text[]); #include "../udfs/worker_create_or_replace_object/9.0-1.sql" DROP FUNCTION IF EXISTS pg_catalog.pg_cancel_backend(bigint); DROP FUNCTION IF EXISTS pg_catalog.pg_terminate_backend(bigint, bigint); DROP FUNCTION pg_catalog.dump_local_wait_edges; CREATE FUNCTION pg_catalog.dump_local_wait_edges( OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$dump_local_wait_edges$$; COMMENT ON FUNCTION pg_catalog.dump_local_wait_edges() IS 'returns all local lock wait chains, that start from distributed transactions'; DROP FUNCTION pg_catalog.dump_global_wait_edges; CREATE FUNCTION pg_catalog.dump_global_wait_edges( OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE 'c' STRICT AS $$MODULE_PATHNAME$$, $$dump_global_wait_edges$$; COMMENT ON FUNCTION pg_catalog.dump_global_wait_edges() IS 'returns a global list of blocked transactions originating from this node'; DROP FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]); CREATE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedTransactionNum int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT transaction_number FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT transaction_number INTO mBlockedTransactionNum FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT transaction_number INTO mBlockedTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; RETURN EXISTS ( SELECT 1 FROM dump_global_wait_edges() WHERE waiting_transaction_num = mBlockedTransactionNum ) OR EXISTS ( -- Check on the workers if any logical replication job spawned by the -- current PID is blocked, by checking it's application name -- Query is heavily based on: https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT result FROM run_command_on_workers($two$ SELECT blocked_activity.application_name AS blocked_application FROM pg_catalog.pg_locks blocked_locks JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid JOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid AND blocking_locks.pid != blocked_locks.pid JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid WHERE NOT blocked_locks.GRANTED AND blocked_activity.application_name LIKE 'citus_shard_move_subscription_%' $two$) where result='citus_shard_move_subscription_' || pBlockedPid); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; DROP FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer); CREATE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalTransactionNum int8; BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT transaction_number INTO mLocalTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; SELECT array_agg(process_id) INTO mRemoteBlockingPids FROM ( WITH activeTransactions AS ( SELECT process_id, transaction_number FROM get_all_active_transactions() ), blockingTransactions AS ( SELECT blocking_transaction_num AS txn_num FROM dump_global_wait_edges() WHERE waiting_transaction_num = mLocalTransactionNum ) SELECT activeTransactions.process_id FROM activeTransactions, blockingTransactions WHERE activeTransactions.transaction_number = blockingTransactions.txn_num ) AS sub; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_blocking_pids(integer) FROM PUBLIC; CREATE VIEW citus.citus_worker_stat_activity AS SELECT * FROM pg_catalog.citus_worker_stat_activity(); ALTER VIEW citus.citus_worker_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_worker_stat_activity TO PUBLIC; -- we have to recreate this view because we drop citus_dist_stat_activity that this view depends CREATE VIEW citus.citus_lock_waits AS WITH citus_dist_stat_activity AS ( SELECT * FROM citus_dist_stat_activity ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_node_id, waiting_transaction_num, blocking_node_id, blocking_transaction_num) * FROM dump_global_wait_edges() ), citus_dist_stat_activity_with_node_id AS ( SELECT citus_dist_stat_activity.*, (CASE citus_dist_stat_activity.distributed_query_host_name WHEN 'coordinator_host' THEN 0 ELSE pg_dist_node.nodeid END) as initiator_node_id FROM citus_dist_stat_activity LEFT JOIN pg_dist_node ON citus_dist_stat_activity.distributed_query_host_name = pg_dist_node.nodename AND citus_dist_stat_activity.distributed_query_host_port = pg_dist_node.nodeport ) SELECT waiting.pid AS waiting_pid, blocking.pid AS blocking_pid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.initiator_node_id AS waiting_node_id, blocking.initiator_node_id AS blocking_node_id, waiting.distributed_query_host_name AS waiting_node_name, blocking.distributed_query_host_name AS blocking_node_name, waiting.distributed_query_host_port AS waiting_node_port, blocking.distributed_query_host_port AS blocking_node_port FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_node_id waiting ON (unique_global_wait_edges.waiting_transaction_num = waiting.transaction_number AND unique_global_wait_edges.waiting_node_id = waiting.initiator_node_id) JOIN citus_dist_stat_activity_with_node_id blocking ON (unique_global_wait_edges.blocking_transaction_num = blocking.transaction_number AND unique_global_wait_edges.blocking_node_id = blocking.initiator_node_id); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; DROP FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool); DROP FUNCTION pg_catalog.citus_calculate_gpid(integer,integer); DROP FUNCTION pg_catalog.citus_backend_gpid(); DROP FUNCTION get_nodeid_for_groupid(integer); RESET search_path; DROP VIEW pg_catalog.citus_stat_activity; DROP FUNCTION pg_catalog.citus_stat_activity; DROP FUNCTION pg_catalog.run_command_on_all_nodes; DROP FUNCTION pg_catalog.citus_nodename_for_nodeid(integer); DROP FUNCTION pg_catalog.citus_nodeport_for_nodeid(integer); DROP FUNCTION pg_catalog.citus_nodeid_for_gpid(bigint); DROP FUNCTION pg_catalog.citus_pid_for_gpid(bigint); DROP FUNCTION pg_catalog.citus_coordinator_nodeid(); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.0-2--11.0-1.sql ================================================ #include "../udfs/citus_shards_on_worker/11.0-1.sql" #include "../udfs/citus_shard_indexes_on_worker/11.0-1.sql" #include "../udfs/citus_finalize_upgrade_to_citus11/11.0-1.sql" DROP FUNCTION pg_catalog.citus_disable_node(text, integer, bool); CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool) IS 'removes node from the cluster temporarily'; REVOKE ALL ON FUNCTION pg_catalog.citus_disable_node(text,int, bool) FROM PUBLIC; DROP FUNCTION pg_catalog.citus_is_coordinator(); DROP FUNCTION pg_catalog.run_command_on_coordinator(text,boolean); DROP FUNCTION pg_catalog.start_metadata_sync_to_all_nodes(); DROP PROCEDURE pg_catalog.citus_finish_citus_upgrade(); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.0-3--11.0-2.sql ================================================ #include "../udfs/citus_finalize_upgrade_to_citus11/11.0-2.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.1-1--11.0-4.sql ================================================ CREATE FUNCTION pg_catalog.worker_create_schema(jobid bigint, username text) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_create_schema$function$; CREATE FUNCTION pg_catalog.worker_cleanup_job_schema_cache() RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_cleanup_job_schema_cache$function$; CREATE FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_fetch_foreign_file$function$; CREATE FUNCTION pg_catalog.worker_fetch_partition_file(bigint, integer, integer, integer, text, integer) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_fetch_partition_file$function$; CREATE FUNCTION pg_catalog.worker_hash_partition_table(bigint, integer, text, text, oid, anyarray) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_hash_partition_table$function$; CREATE FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[], text[]) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_merge_files_into_table$function$; CREATE FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_range_partition_table$function$; CREATE FUNCTION pg_catalog.worker_repartition_cleanup(bigint) RETURNS void LANGUAGE c STRICT AS 'MODULE_PATHNAME', $function$worker_repartition_cleanup$function$; -- add relations to citus ALTER EXTENSION citus ADD SCHEMA columnar; ALTER EXTENSION citus ADD SEQUENCE columnar.storageid_seq; ALTER EXTENSION citus ADD TABLE columnar.options; ALTER EXTENSION citus ADD TABLE columnar.stripe; ALTER EXTENSION citus ADD TABLE columnar.chunk_group; ALTER EXTENSION citus ADD TABLE columnar.chunk; ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler; ALTER EXTENSION citus ADD ACCESS METHOD columnar; ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set; ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset; ALTER EXTENSION citus ADD FUNCTION citus_internal.upgrade_columnar_storage; ALTER EXTENSION citus ADD FUNCTION citus_internal.downgrade_columnar_storage; ALTER EXTENSION citus ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog; DROP FUNCTION pg_catalog.citus_split_shard_by_split_points( shard_id bigint, split_points text[], node_ids integer[], shard_transfer_mode citus.shard_transfer_mode); DROP FUNCTION pg_catalog.worker_split_copy( source_shard_id bigint, distribution_column text, splitCopyInfos pg_catalog.split_copy_info[]); DROP TYPE pg_catalog.split_copy_info; DROP FUNCTION pg_catalog.worker_copy_table_to_node( source_table regclass, target_node_id integer); DROP FUNCTION pg_catalog.worker_split_shard_replication_setup( splitShardInfo pg_catalog.split_shard_info[]); DROP TYPE pg_catalog.split_shard_info; DROP TYPE pg_catalog.replication_slot_info; DROP FUNCTION pg_catalog.worker_split_shard_release_dsm(); DROP FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8); #include "../udfs/get_all_active_transactions/11.0-1.sql" DROP VIEW pg_catalog.citus_locks; DROP FUNCTION pg_catalog.citus_locks(); #include "../udfs/citus_tables/10.0-4.sql" #include "../udfs/citus_shards/10.1-1.sql" DROP FUNCTION pg_catalog.replicate_reference_tables(citus.shard_transfer_mode); #include "../udfs/replicate_reference_tables/9.3-2.sql" DROP FUNCTION pg_catalog.isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text, shard_transfer_mode citus.shard_transfer_mode); #include "../udfs/isolate_tenant_to_new_shard/8.0-1.sql" DROP FUNCTION pg_catalog.create_distributed_table_concurrently; DROP FUNCTION pg_catalog.citus_internal_delete_partition_metadata(regclass); -- Check if user has any cleanup records. -- If not, DROP pg_dist_cleanup and continue safely. -- Otherwise, raise an exception to stop the downgrade process. DO $$ DECLARE cleanup_record_count INTEGER; BEGIN SELECT COUNT(*) INTO cleanup_record_count FROM pg_dist_cleanup; IF cleanup_record_count = 0 THEN -- no cleanup records exist, can safely downgrade DROP TABLE pg_catalog.pg_dist_cleanup; ELSE RAISE EXCEPTION 'pg_dist_cleanup is introduced in Citus 11.1' USING HINT = 'To downgrade Citus to an older version, you should ' 'first cleanup all the orphaned resources and make sure ' 'pg_dist_cleanup is empty, by executing ' 'CALL citus_cleanup_orphaned_resources();'; END IF; END; $$ LANGUAGE plpgsql; DROP SEQUENCE pg_catalog.pg_dist_operationid_seq; DROP SEQUENCE pg_catalog.pg_dist_cleanup_recordid_seq; DROP PROCEDURE pg_catalog.citus_cleanup_orphaned_resources(); DROP FUNCTION pg_catalog.citus_rebalance_start(name, bool, citus.shard_transfer_mode); DROP FUNCTION pg_catalog.citus_rebalance_stop(); DROP FUNCTION pg_catalog.citus_rebalance_wait(); DROP FUNCTION pg_catalog.citus_job_cancel(bigint); DROP FUNCTION pg_catalog.citus_job_wait(bigint, pg_catalog.citus_job_status); DROP TABLE pg_catalog.pg_dist_background_task_depend; DROP TABLE pg_catalog.pg_dist_background_task; DROP TYPE pg_catalog.citus_task_status; DROP TABLE pg_catalog.pg_dist_background_job; DROP TYPE pg_catalog.citus_job_status; DROP FUNCTION pg_catalog.citus_copy_shard_placement; #include "../udfs/citus_copy_shard_placement/10.0-1.sql" #include "../udfs/get_rebalance_progress/10.1-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.2-1--11.1-1.sql ================================================ -- citus--11.2-1--11.1-1 #include "../udfs/get_rebalance_progress/11.1-1.sql" #include "../udfs/citus_isolation_test_session_is_blocked/11.1-1.sql" DROP FUNCTION pg_catalog.citus_get_node_clock(); DROP FUNCTION pg_catalog.citus_get_transaction_clock(); DROP FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(cluster_clock); DROP FUNCTION pg_catalog.citus_is_clock_after(cluster_clock, cluster_clock); DROP FUNCTION pg_catalog.citus_job_list(); DROP FUNCTION pg_catalog.citus_job_status(bigint,boolean); DROP FUNCTION pg_catalog.citus_rebalance_status(boolean); DROP FUNCTION pg_catalog.cluster_clock_logical(cluster_clock); DROP SEQUENCE pg_catalog.pg_dist_clock_logical_seq; DROP OPERATOR CLASS pg_catalog.cluster_clock_ops USING btree CASCADE; DROP OPERATOR FAMILY pg_catalog.cluster_clock_ops USING btree CASCADE; DROP TYPE pg_catalog.cluster_clock CASCADE; DROP FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[], bigint); DROP TYPE pg_catalog.replication_slot_info; DROP TYPE pg_catalog.split_shard_info; CREATE FUNCTION pg_catalog.worker_append_table_to_shard(text, text, text, integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_append_table_to_shard$$; COMMENT ON FUNCTION pg_catalog.worker_append_table_to_shard(text, text, text, integer) IS 'append a regular table''s contents to the shard'; #include "../udfs/worker_split_shard_replication_setup/11.1-1.sql" DROP FUNCTION pg_catalog.citus_task_wait(bigint, pg_catalog.citus_task_status); #include "../udfs/citus_prepare_pg_upgrade/11.1-1.sql" #include "../udfs/citus_finish_pg_upgrade/11.1-1.sql" DROP FUNCTION pg_catalog.citus_copy_shard_placement(bigint, integer, integer, citus.shard_transfer_mode); DROP FUNCTION pg_catalog.citus_move_shard_placement(bigint, integer, integer, citus.shard_transfer_mode); DROP FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, bigint, integer, bigint); #include "../udfs/citus_internal_add_placement_metadata/10.2-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.2-2--11.2-1.sql ================================================ -- citus--11.2-2--11.2-1 DROP FUNCTION IF EXISTS pg_catalog.worker_adjust_identity_column_seq_ranges(regclass); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.3-1--11.2-2.sql ================================================ -- citus--11.3-1--11.2-1 DROP FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking(); DROP FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking(); DROP FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active(); ALTER TABLE pg_catalog.pg_dist_authinfo REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_partition REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_placement REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_shard REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_transaction REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_authinfo REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_partition REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_placement REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_shard REPLICA IDENTITY NOTHING; ALTER TABLE pg_catalog.pg_dist_transaction REPLICA IDENTITY NOTHING; DROP PROCEDURE pg_catalog.worker_drop_all_shell_tables(bool); DROP FUNCTION pg_catalog.citus_internal_mark_node_not_synced(int, int); DROP VIEW pg_catalog.citus_stat_tenants_local; DROP FUNCTION pg_catalog.citus_stat_tenants_local(boolean); DROP VIEW pg_catalog.citus_stat_tenants; DROP FUNCTION pg_catalog.citus_stat_tenants(boolean); DROP FUNCTION pg_catalog.citus_stat_tenants_local_reset(); DROP FUNCTION pg_catalog.citus_stat_tenants_reset(); ALTER TABLE pg_catalog.pg_dist_background_task DROP COLUMN nodes_involved; ================================================ FILE: src/backend/distributed/sql/downgrades/citus--11.3-2--11.3-1.sql ================================================ DROP VIEW IF EXISTS public.citus_tables; DROP VIEW IF EXISTS pg_catalog.citus_tables; DROP VIEW pg_catalog.citus_shards; DROP FUNCTION pg_catalog.citus_shard_sizes; #include "../udfs/citus_shard_sizes/10.0-1.sql" -- citus_shards/11.1-1.sql tries to create citus_shards in pg_catalog but it is not allowed. -- Here we use citus_shards/10.0-1.sql to properly create the view in citus schema and -- then alter it to pg_catalog, so citus_shards/11.1-1.sql can REPLACE it without any errors. #include "../udfs/citus_shards/10.0-1.sql" #include "../udfs/citus_tables/11.1-1.sql" #include "../udfs/citus_shards/11.1-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-2.sql ================================================ -- citus--12.0-1--11.3-1 DO $$ BEGIN -- Throw an error if user has created any tenant schemas. IF EXISTS (SELECT 1 FROM pg_catalog.pg_dist_schema) THEN RAISE EXCEPTION 'cannot downgrade Citus because there are ' 'tenant schemas created.' USING HINT = 'To downgrade Citus to an older version, you should ' 'first issue SELECT citus.schema_tenant_unset("%s") ' 'for each tenant schema.'; END IF; -- Throw an error if user has any distributed tables without a shard key. IF EXISTS ( SELECT 1 FROM pg_dist_partition WHERE repmodel != 't' AND partmethod = 'n' AND colocationid != 0) THEN RAISE EXCEPTION 'cannot downgrade Citus because there are ' 'distributed tables without a shard key.' USING HINT = 'You can find the distributed tables without a shard ' 'key in the cluster by using the following query: ' '"SELECT * FROM citus_tables WHERE distribution_column ' '= '''' AND colocation_id > 0".', DETAIL = 'To downgrade Citus to an older version, you should ' 'first convert those tables to Postgres tables by ' 'executing SELECT undistribute_table("%s").'; END IF; END; $$ LANGUAGE plpgsql; DROP FUNCTION pg_catalog.citus_schema_distribute(regnamespace); DROP FUNCTION pg_catalog.citus_schema_undistribute(regnamespace); DROP FUNCTION pg_catalog.citus_internal_add_tenant_schema(Oid, int); #include "../udfs/citus_prepare_pg_upgrade/11.2-1.sql" #include "../udfs/citus_finish_pg_upgrade/11.2-1.sql" DROP FUNCTION pg_catalog.citus_internal_delete_tenant_schema(Oid); DROP FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(Oid, text); #include "../udfs/citus_drop_trigger/10.2-1.sql" -- citus_schemas might be created in either of the schemas DROP VIEW IF EXISTS public.citus_schemas; DROP VIEW IF EXISTS pg_catalog.citus_schemas; DROP VIEW IF EXISTS public.citus_tables; DROP VIEW IF EXISTS pg_catalog.citus_tables; DROP VIEW pg_catalog.citus_shards; #include "../udfs/citus_tables/11.3-2.sql" #include "../udfs/citus_shards/11.3-2.sql" DROP TABLE pg_catalog.pg_dist_schema; DROP VIEW pg_catalog.citus_stat_tenants_local; DROP FUNCTION pg_catalog.citus_stat_tenants_local_internal( BOOLEAN, OUT INT, OUT TEXT, OUT INT, OUT INT, OUT INT, OUT INT, OUT DOUBLE PRECISION, OUT DOUBLE PRECISION, OUT BIGINT); #include "../udfs/citus_stat_tenants_local/11.3-1.sql" #include "../udfs/drop_old_time_partitions/10.2-1.sql" #include "../udfs/get_missing_time_partition_ranges/10.2-1.sql" -- This explicitly does not reset the rebalance strategy to by_shard_count, -- because there's no way of knowing if the rebalance strategy before the -- upgrade was by_disk_size or by_shard_count. And even in previous versions -- by_disk_size is considered superior for quite some time. ================================================ FILE: src/backend/distributed/sql/downgrades/citus--12.1-1--12.0-1.sql ================================================ -- citus--12.1-1--12.0-1 DROP FUNCTION pg_catalog.citus_pause_node_within_txn(int,bool,int); -- we have modified the relevant upgrade script to include any_value changes -- we don't need to upgrade this downgrade path for any_value changes -- since if we are doing a Citus downgrade, not PG downgrade, then it would be no-op. DROP FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean ); DROP FUNCTION pg_catalog.citus_internal_delete_placement_metadata( placement_id bigint ); DROP FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode ); DROP FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_id integer, shard_transfer_mode citus.shard_transfer_mode ); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--13.0-1--12.1-1.sql ================================================ -- citus--13.0-1--12.1-1 -- this is an empty downgrade path since citus--12.1-1--13.0-1.sql is empty #include "../udfs/create_time_partitions/10.2-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--13.1-1--13.0-1.sql ================================================ -- citus--13.1-1--13.0-1 DROP FUNCTION citus_internal.database_command(text); DROP FUNCTION citus_internal.acquire_citus_advisory_object_class_lock(int, cstring); #include "../udfs/citus_add_rebalance_strategy/10.1-1.sql" DROP FUNCTION pg_catalog.citus_unmark_object_distributed(oid,oid,int,boolean); #include "../udfs/citus_unmark_object_distributed/10.0-1.sql" ALTER TABLE pg_catalog.pg_dist_transaction DROP COLUMN outer_xid; REVOKE USAGE ON SCHEMA citus_internal FROM PUBLIC; DROP FUNCTION pg_catalog.citus_is_primary_node(); DROP FUNCTION citus_internal.add_colocation_metadata(int, int, int, regtype, oid); DROP FUNCTION citus_internal.add_object_metadata(text, text[], text[], integer, integer, boolean); DROP FUNCTION citus_internal.add_partition_metadata(regclass, "char", text, integer, "char"); DROP FUNCTION citus_internal.add_placement_metadata(bigint, bigint, integer, bigint); DROP FUNCTION citus_internal.add_shard_metadata(regclass, bigint, "char", text, text); DROP FUNCTION citus_internal.add_tenant_schema(oid, integer); DROP FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock); DROP FUNCTION citus_internal.delete_colocation_metadata(int); DROP FUNCTION citus_internal.delete_partition_metadata(regclass); DROP FUNCTION citus_internal.delete_placement_metadata(bigint); DROP FUNCTION citus_internal.delete_shard_metadata(bigint); DROP FUNCTION citus_internal.delete_tenant_schema(oid); DROP FUNCTION citus_internal.local_blocked_processes(); #include "../udfs/citus_blocking_pids/11.0-1.sql" #include "../udfs/citus_isolation_test_session_is_blocked/11.1-1.sql" DROP VIEW IF EXISTS pg_catalog.citus_lock_waits; #include "../udfs/citus_lock_waits/11.0-1.sql" DROP FUNCTION citus_internal.global_blocked_processes(); DROP FUNCTION citus_internal.mark_node_not_synced(int, int); DROP FUNCTION citus_internal.unregister_tenant_schema_globally(oid, text); #include "../udfs/citus_drop_trigger/12.0-1.sql" DROP FUNCTION citus_internal.update_none_dist_table_metadata(oid, "char", bigint, boolean); DROP FUNCTION citus_internal.update_placement_metadata(bigint, integer, integer); DROP FUNCTION citus_internal.update_relation_colocation(oid, int); DROP FUNCTION citus_internal.start_replication_origin_tracking(); DROP FUNCTION citus_internal.stop_replication_origin_tracking(); DROP FUNCTION citus_internal.is_replication_origin_tracking_active(); #include "../udfs/citus_finish_pg_upgrade/12.1-1.sql" DROP VIEW pg_catalog.citus_stat_counters; DROP FUNCTION pg_catalog.citus_stat_counters(oid); DROP FUNCTION pg_catalog.citus_stat_counters_reset(oid); DROP VIEW IF EXISTS pg_catalog.citus_nodes; -- Definition of shard_name() prior to this release doesn't have a separate SQL file -- because it's quite an old UDF that its prior definition(s) was(were) squashed into -- citus--8.0-1.sql. For this reason, to downgrade it, here we directly execute its old -- definition instead of including it from such a separate file. -- -- And before dropping and creating the function, we also need to drop citus_shards view -- since it depends on it. And immediately after creating the function, we recreate -- citus_shards view again. DROP VIEW pg_catalog.citus_shards; DROP FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint, skip_qualify_public boolean); CREATE FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$shard_name$$; COMMENT ON FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint) IS 'returns schema-qualified, shard-extended identifier of object name'; #include "../udfs/citus_shards/12.0-1.sql" ================================================ FILE: src/backend/distributed/sql/downgrades/citus--13.2-1--13.1-1.sql ================================================ -- citus--13.2-1--13.1-1 -- downgrade version to 13.1-1 DROP FUNCTION IF EXISTS citus_internal.citus_internal_copy_single_shard_placement(bigint, integer, integer, integer, citus.shard_transfer_mode); DROP FUNCTION IF EXISTS pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode, boolean, boolean); #include "../udfs/citus_rebalance_start/11.1-1.sql" DROP FUNCTION IF EXISTS pg_catalog.worker_last_saved_explain_analyze(); #include "../udfs/worker_last_saved_explain_analyze/9.4-1.sql" DROP FUNCTION IF EXISTS pg_catalog.citus_add_clone_node(text, integer, text, integer); DROP FUNCTION IF EXISTS pg_catalog.citus_add_clone_node_with_nodeid(text, integer, integer); DROP FUNCTION IF EXISTS pg_catalog.citus_remove_clone_node(text, integer); DROP FUNCTION IF EXISTS pg_catalog.citus_remove_clone_node_with_nodeid(integer); DROP FUNCTION IF EXISTS pg_catalog.citus_promote_clone_and_rebalance(integer, name, integer); DROP FUNCTION IF EXISTS pg_catalog.get_snapshot_based_node_split_plan(text, integer, text, integer, name); #include "../cat_upgrades/remove_clone_info_to_pg_dist_node.sql" #include "../udfs/citus_finish_pg_upgrade/13.1-1.sql" -- Note that we intentionally don't add the old columnar objects back to the "citus" -- extension in this downgrade script, even if they were present in the older version. -- -- If the user wants to create "citus_columnar" extension later, "citus_columnar" -- will anyway properly create them at the scope of that extension. DROP VIEW IF EXISTS pg_catalog.citus_stats; ================================================ FILE: src/backend/distributed/sql/downgrades/citus--14.0-1--13.2-1.sql ================================================ -- citus--14.0-1--13.2-1 -- downgrade version to 13.2-1 #include "../udfs/citus_prepare_pg_upgrade/13.0-1.sql" #include "../udfs/citus_finish_pg_upgrade/13.2-1.sql" DROP AGGREGATE IF EXISTS pg_catalog.worker_binary_partial_agg(oid, anyelement); DROP AGGREGATE IF EXISTS pg_catalog.coord_binary_combine_agg(oid, bytea, anyelement); DROP FUNCTION IF EXISTS pg_catalog.worker_binary_partial_agg_ffunc(internal); DROP FUNCTION IF EXISTS pg_catalog.coord_binary_combine_agg_sfunc(internal, oid, bytea, anyelement); DROP FUNCTION IF EXISTS pg_catalog.coord_binary_combine_agg_ffunc(internal, oid, bytea, anyelement); #include "../udfs/citus_finish_citus_upgrade/11.0-2.sql" DROP FUNCTION IF EXISTS pg_catalog.fix_pre_citus14_colocation_group_collation_mismatches(); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--15.0-1--14.0-1.sql ================================================ -- citus--15.0-1--14.0-1 -- downgrade version to 14.0-1 ================================================ FILE: src/backend/distributed/sql/downgrades/citus--9.2-4--9.2-2.sql ================================================ -- citus--9.2-4--9.2-2 -- this is an empty downgrade path since citus--9.2-2--9.2-4.sql is empty ================================================ FILE: src/backend/distributed/sql/downgrades/citus--9.3-2--9.2-4.sql ================================================ -- citus--9.3-2--9.2-4 -- this is a downgrade path that will revert the changes made in citus--9.2-4--9.3-2.sql -- -- 9.3-2 added citus extension owner as a distributed object, if not already in there. -- However we can not really know if it was a distributed owner prior to 9.3-2. -- That's why we leave the record in place. -- Revert the return type to void DROP FUNCTION pg_catalog.citus_extradata_container(INTERNAL); CREATE FUNCTION pg_catalog.citus_extradata_container(INTERNAL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_extradata_container$$; COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; -- Remove newly introduced functions that are absent in earlier versions DROP FUNCTION pg_catalog.update_distributed_table_colocation(regclass, text); DROP FUNCTION pg_catalog.replicate_reference_tables(); DROP FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int); DROP FUNCTION pg_catalog.worker_create_or_alter_role( role_name text, create_role_utility_query text, alter_role_utility_query text); DROP FUNCTION pg_catalog.truncate_local_data_after_distributing_table( function_name regclass); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--9.4-1--9.3-2.sql ================================================ -- citus--9.4-1--9.3-2 -- this is a downgrade path that will revert the changes made in citus--9.3-2--9.4-1.sql DROP FUNCTION pg_catalog.worker_last_saved_explain_analyze(); DROP FUNCTION pg_catalog.worker_save_query_explain_analyze(query text, options jsonb); ================================================ FILE: src/backend/distributed/sql/downgrades/citus--9.5-1--9.4-1.sql ================================================ -- citus--9.5-1--9.4-1 SET search_path = 'pg_catalog'; #include "../udfs/citus_drop_trigger/9.0-1.sql" -- Check if user has any citus local tables. -- If not, DROP create_citus_local_table UDF and continue safely. -- Otherwise, raise an exception to stop the downgrade process. DO $$ DECLARE citus_local_table_count INTEGER; BEGIN SELECT COUNT(*) INTO citus_local_table_count FROM pg_dist_partition WHERE repmodel != 't' AND partmethod = 'n'; IF citus_local_table_count = 0 THEN -- no citus local tables exist, can safely downgrade DROP FUNCTION create_citus_local_table(table_name regclass); ELSE RAISE EXCEPTION 'citus local tables are introduced in Citus 9.5' USING HINT = 'To downgrade Citus to an older version, you should ' 'first convert each citus local table to a postgres ' 'table by executing SELECT undistribute_table("%s")'; END IF; END; $$ LANGUAGE plpgsql; DROP FUNCTION worker_record_sequence_dependency(regclass, regclass, name); -- task_tracker_* functions CREATE FUNCTION task_tracker_assign_task(bigint, integer, text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_assign_task$$; COMMENT ON FUNCTION task_tracker_assign_task(bigint, integer, text) IS 'assign a task to execute'; CREATE FUNCTION task_tracker_task_status(bigint, integer) RETURNS integer LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_task_status$$; COMMENT ON FUNCTION task_tracker_task_status(bigint, integer) IS 'check an assigned task''s execution status'; CREATE FUNCTION task_tracker_cleanup_job(bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$task_tracker_cleanup_job$$; COMMENT ON FUNCTION task_tracker_cleanup_job(bigint) IS 'clean up all tasks associated with a job'; CREATE FUNCTION worker_merge_files_and_run_query(bigint, integer, text, text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_merge_files_and_run_query$$; COMMENT ON FUNCTION worker_merge_files_and_run_query(bigint, integer, text, text) IS 'merge files and run a reduce query on merged files'; CREATE FUNCTION worker_execute_sql_task(jobid bigint, taskid integer, query text, binary bool) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_execute_sql_task$$; COMMENT ON FUNCTION worker_execute_sql_task(bigint, integer, text, bool) IS 'execute a query and write the results to a task file'; CREATE FUNCTION task_tracker_conninfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$task_tracker_conninfo_cache_invalidate$$; COMMENT ON FUNCTION task_tracker_conninfo_cache_invalidate() IS 'invalidate task-tracker conninfo cache'; CREATE TRIGGER dist_poolinfo_task_tracker_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_poolinfo FOR EACH STATEMENT EXECUTE PROCEDURE task_tracker_conninfo_cache_invalidate(); CREATE TRIGGER dist_authinfo_task_tracker_cache_invalidate AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON pg_catalog.pg_dist_authinfo FOR EACH STATEMENT EXECUTE PROCEDURE task_tracker_conninfo_cache_invalidate(); CREATE FUNCTION master_drop_sequences(sequence_names text[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drop_sequences$$; COMMENT ON FUNCTION master_drop_sequences(text[]) IS 'drop specified sequences from the cluster'; RESET search_path; DROP FUNCTION pg_catalog.undistribute_table(table_name regclass); ================================================ FILE: src/backend/distributed/sql/udfs/alter_distributed_table/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_distributed_table( table_name regclass, distribution_column text DEFAULT NULL, shard_count int DEFAULT NULL, colocate_with text DEFAULT NULL, cascade_to_colocated boolean DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$alter_distributed_table$$; COMMENT ON FUNCTION pg_catalog.alter_distributed_table( table_name regclass, distribution_column text, shard_count int, colocate_with text, cascade_to_colocated boolean) IS 'alters a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_distributed_table/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_distributed_table( table_name regclass, distribution_column text DEFAULT NULL, shard_count int DEFAULT NULL, colocate_with text DEFAULT NULL, cascade_to_colocated boolean DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$alter_distributed_table$$; COMMENT ON FUNCTION pg_catalog.alter_distributed_table( table_name regclass, distribution_column text, shard_count int, colocate_with text, cascade_to_colocated boolean) IS 'alters a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_old_partitions_set_access_method/10.0-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.alter_old_partitions_set_access_method( parent_table_name regclass, older_than timestamptz, new_access_method name) LANGUAGE plpgsql AS $$ DECLARE r record; BEGIN -- first check whether we can convert all the to_value's to timestamptz BEGIN PERFORM FROM pg_catalog.time_partitions WHERE parent_table = parent_table_name AND to_value IS NOT NULL AND to_value::timestamptz <= older_than AND access_method <> new_access_method; EXCEPTION WHEN invalid_datetime_format THEN RAISE 'partition column of % cannot be cast to a timestamptz', parent_table_name; END; -- now convert the partitions in separate transactions FOR r IN SELECT partition, from_value, to_value FROM pg_catalog.time_partitions WHERE parent_table = parent_table_name AND to_value IS NOT NULL AND to_value::timestamptz <= older_than AND access_method <> new_access_method ORDER BY to_value::timestamptz LOOP RAISE NOTICE 'converting % with start time % and end time %', r.partition, r.from_value, r.to_value; PERFORM pg_catalog.alter_table_set_access_method(r.partition, new_access_method); COMMIT; END LOOP; END; $$; COMMENT ON PROCEDURE pg_catalog.alter_old_partitions_set_access_method( parent_table_name regclass, older_than timestamptz, new_access_method name) IS 'convert old partitions of a time-partitioned table to a new access method'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_old_partitions_set_access_method/latest.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.alter_old_partitions_set_access_method( parent_table_name regclass, older_than timestamptz, new_access_method name) LANGUAGE plpgsql AS $$ DECLARE r record; BEGIN -- first check whether we can convert all the to_value's to timestamptz BEGIN PERFORM FROM pg_catalog.time_partitions WHERE parent_table = parent_table_name AND to_value IS NOT NULL AND to_value::timestamptz <= older_than AND access_method <> new_access_method; EXCEPTION WHEN invalid_datetime_format THEN RAISE 'partition column of % cannot be cast to a timestamptz', parent_table_name; END; -- now convert the partitions in separate transactions FOR r IN SELECT partition, from_value, to_value FROM pg_catalog.time_partitions WHERE parent_table = parent_table_name AND to_value IS NOT NULL AND to_value::timestamptz <= older_than AND access_method <> new_access_method ORDER BY to_value::timestamptz LOOP RAISE NOTICE 'converting % with start time % and end time %', r.partition, r.from_value, r.to_value; PERFORM pg_catalog.alter_table_set_access_method(r.partition, new_access_method); COMMIT; END LOOP; END; $$; COMMENT ON PROCEDURE pg_catalog.alter_old_partitions_set_access_method( parent_table_name regclass, older_than timestamptz, new_access_method name) IS 'convert old partitions of a time-partitioned table to a new access method'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_role_if_exists/9.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_role_if_exists( role_name text, utility_query text) RETURNS BOOL LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$alter_role_if_exists$$; COMMENT ON FUNCTION pg_catalog.alter_role_if_exists( role_name text, utility_query text) IS 'runs the utility query, if the role exists'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_role_if_exists/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_role_if_exists( role_name text, utility_query text) RETURNS BOOL LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$alter_role_if_exists$$; COMMENT ON FUNCTION pg_catalog.alter_role_if_exists( role_name text, utility_query text) IS 'runs the utility query, if the role exists'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_table_set_access_method/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_table_set_access_method( table_name regclass, access_method text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$alter_table_set_access_method$$; COMMENT ON FUNCTION pg_catalog.alter_table_set_access_method( table_name regclass, access_method text) IS 'alters a table''s access method'; ================================================ FILE: src/backend/distributed/sql/udfs/alter_table_set_access_method/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.alter_table_set_access_method( table_name regclass, access_method text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$alter_table_set_access_method$$; COMMENT ON FUNCTION pg_catalog.alter_table_set_access_method( table_name regclass, access_method text) IS 'alters a table''s access method'; ================================================ FILE: src/backend/distributed/sql/udfs/any_value/9.1-1.sql ================================================ DO $proc$ BEGIN -- PG16 has its own any_value, so only create it pre PG16. IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $$ CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $agg$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $agg$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; $$; END IF; END $proc$; ================================================ FILE: src/backend/distributed/sql/udfs/any_value/latest.sql ================================================ DO $proc$ BEGIN -- PG16 has its own any_value, so only create it pre PG16. IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $$ CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $agg$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $agg$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; $$; END IF; END $proc$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_activate_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_activate_node(nodename text, nodeport integer) RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_activate_node$$; COMMENT ON FUNCTION pg_catalog.citus_activate_node(nodename text, nodeport integer) IS 'activate a node which is in the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_activate_node(text, integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_activate_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_activate_node(nodename text, nodeport integer) RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_activate_node$$; COMMENT ON FUNCTION pg_catalog.citus_activate_node(nodename text, nodeport integer) IS 'activate a node which is in the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_activate_node(text, integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_clone_node/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_add_clone_node( replica_hostname text, replica_port integer, primary_hostname text, primary_port integer) RETURNS INTEGER LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_add_clone_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_clone_node(text, integer, text, integer) IS 'Adds a new node as a clone of an existing primary node. The clone is initially inactive. Returns the nodeid of the new clone node.'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_clone_node(text, int, text, int) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_add_clone_node_with_nodeid( replica_hostname text, replica_port integer, primary_nodeid integer) RETURNS INTEGER LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_add_clone_node_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_add_clone_node_with_nodeid(text, integer, integer) IS 'Adds a new node as a clone of an existing primary node using the primary node''s ID. The clone is initially inactive. Returns the nodeid of the new clone node.'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_clone_node_with_nodeid(text, int, int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_clone_node/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_add_clone_node( replica_hostname text, replica_port integer, primary_hostname text, primary_port integer) RETURNS INTEGER LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_add_clone_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_clone_node(text, integer, text, integer) IS 'Adds a new node as a clone of an existing primary node. The clone is initially inactive. Returns the nodeid of the new clone node.'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_clone_node(text, int, text, int) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_add_clone_node_with_nodeid( replica_hostname text, replica_port integer, primary_nodeid integer) RETURNS INTEGER LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_add_clone_node_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_add_clone_node_with_nodeid(text, integer, integer) IS 'Adds a new node as a clone of an existing primary node using the primary node''s ID. The clone is initially inactive. Returns the nodeid of the new clone node.'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_clone_node_with_nodeid(text, int, int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_inactive_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_inactive_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_add_inactive_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_inactive_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_inactive_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_add_inactive_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_rebalance_strategy/10.1-1.sql ================================================ DROP FUNCTION pg_catalog.citus_add_rebalance_strategy; CREATE OR REPLACE FUNCTION pg_catalog.citus_add_rebalance_strategy( name name, shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc, default_threshold float4, minimum_threshold float4 DEFAULT 0, improvement_threshold float4 DEFAULT 0 ) RETURNS VOID AS $$ INSERT INTO pg_catalog.pg_dist_rebalance_strategy( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold ) VALUES ( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold ); $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_add_rebalance_strategy(name,regproc,regproc,regproc,float4, float4, float4) IS 'adds a new rebalance strategy which can be used when rebalancing shards or draining nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_rebalance_strategy/13.1-1.sql ================================================ DROP FUNCTION pg_catalog.citus_add_rebalance_strategy; CREATE OR REPLACE FUNCTION pg_catalog.citus_add_rebalance_strategy( name name, shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc, default_threshold float4, minimum_threshold float4 DEFAULT 0, improvement_threshold float4 DEFAULT 0 ) RETURNS VOID AS $$ INSERT INTO pg_catalog.pg_dist_rebalance_strategy( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold, improvement_threshold ) VALUES ( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold, improvement_threshold ); $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_add_rebalance_strategy(name,regproc,regproc,regproc,float4, float4, float4) IS 'adds a new rebalance strategy which can be used when rebalancing shards or draining nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_rebalance_strategy/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_add_rebalance_strategy( name name, shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc, default_threshold float4, minimum_threshold float4 DEFAULT 0 ) RETURNS VOID AS $$ INSERT INTO pg_catalog.pg_dist_rebalance_strategy( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold ) VALUES ( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold ); $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_add_rebalance_strategy(name,regproc,regproc,regproc,float4, float4) IS 'adds a new rebalance strategy which can be used when rebalancing shards or draining nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_rebalance_strategy/latest.sql ================================================ DROP FUNCTION pg_catalog.citus_add_rebalance_strategy; CREATE OR REPLACE FUNCTION pg_catalog.citus_add_rebalance_strategy( name name, shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc, default_threshold float4, minimum_threshold float4 DEFAULT 0, improvement_threshold float4 DEFAULT 0 ) RETURNS VOID AS $$ INSERT INTO pg_catalog.pg_dist_rebalance_strategy( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold, improvement_threshold ) VALUES ( name, shard_cost_function, node_capacity_function, shard_allowed_on_node_function, default_threshold, minimum_threshold, improvement_threshold ); $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_add_rebalance_strategy(name,regproc,regproc,regproc,float4, float4, float4) IS 'adds a new rebalance strategy which can be used when rebalancing shards or draining nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_secondary_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_secondary_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name) IS 'add a secondary node to the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_secondary_node(text,int,text,int,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_add_secondary_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_secondary_node$$; COMMENT ON FUNCTION pg_catalog.citus_add_secondary_node(nodename text, nodeport integer, primaryname text, primaryport integer, nodecluster name) IS 'add a secondary node to the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_add_secondary_node(text,int,text,int,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_backend_gpid/11.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_backend_gpid() RETURNS BIGINT LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_backend_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_backend_gpid() IS 'returns gpid of the current backend'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_backend_gpid() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_backend_gpid/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_backend_gpid() RETURNS BIGINT LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_backend_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_backend_gpid() IS 'returns gpid of the current backend'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_backend_gpid() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_blocking_pids/11.0-1.sql ================================================ DROP FUNCTION pg_catalog.citus_blocking_pids; CREATE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalGlobalPid int8; BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT global_pid INTO mLocalGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; SELECT array_agg(global_pid) INTO mRemoteBlockingPids FROM ( WITH activeTransactions AS ( SELECT global_pid FROM get_all_active_transactions() ), blockingTransactions AS ( SELECT blocking_global_pid FROM citus_internal_global_blocked_processes() WHERE waiting_global_pid = mLocalGlobalPid ) SELECT activeTransactions.global_pid FROM activeTransactions, blockingTransactions WHERE activeTransactions.global_pid = blockingTransactions.blocking_global_pid ) AS sub; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_blocking_pids(integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_blocking_pids/13.1-1.sql ================================================ DROP FUNCTION pg_catalog.citus_blocking_pids; CREATE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalGlobalPid int8; BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT global_pid INTO mLocalGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; SELECT array_agg(global_pid) INTO mRemoteBlockingPids FROM ( WITH activeTransactions AS ( SELECT global_pid FROM get_all_active_transactions() ), blockingTransactions AS ( SELECT blocking_global_pid FROM citus_internal.global_blocked_processes() WHERE waiting_global_pid = mLocalGlobalPid ) SELECT activeTransactions.global_pid FROM activeTransactions, blockingTransactions WHERE activeTransactions.global_pid = blockingTransactions.blocking_global_pid ) AS sub; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_blocking_pids(integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_blocking_pids/latest.sql ================================================ DROP FUNCTION pg_catalog.citus_blocking_pids; CREATE FUNCTION pg_catalog.citus_blocking_pids(pBlockedPid integer) RETURNS int4[] AS $$ DECLARE mLocalBlockingPids int4[]; mRemoteBlockingPids int4[]; mLocalGlobalPid int8; BEGIN SELECT pg_catalog.old_pg_blocking_pids(pBlockedPid) INTO mLocalBlockingPids; IF (array_length(mLocalBlockingPids, 1) > 0) THEN RETURN mLocalBlockingPids; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. SELECT global_pid INTO mLocalGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; SELECT array_agg(global_pid) INTO mRemoteBlockingPids FROM ( WITH activeTransactions AS ( SELECT global_pid FROM get_all_active_transactions() ), blockingTransactions AS ( SELECT blocking_global_pid FROM citus_internal.global_blocked_processes() WHERE waiting_global_pid = mLocalGlobalPid ) SELECT activeTransactions.global_pid FROM activeTransactions, blockingTransactions WHERE activeTransactions.global_pid = blockingTransactions.blocking_global_pid ) AS sub; RETURN mRemoteBlockingPids; END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_blocking_pids(integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_calculate_gpid/11.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_calculate_gpid(nodeid integer, pid integer) RETURNS BIGINT LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_calculate_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_calculate_gpid(nodeid integer, pid integer) IS 'calculate gpid of a backend running on any node'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_calculate_gpid(integer, integer) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_calculate_gpid/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_calculate_gpid(nodeid integer, pid integer) RETURNS BIGINT LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_calculate_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_calculate_gpid(nodeid integer, pid integer) IS 'calculate gpid of a backend running on any node'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_calculate_gpid(integer, integer) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_check_cluster_node_health/11.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_check_cluster_node_health ( OUT from_nodename text, OUT from_nodeport int, OUT to_nodename text, OUT to_nodeport int, OUT result bool ) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_check_cluster_node_health$$; COMMENT ON FUNCTION pg_catalog.citus_check_cluster_node_health () IS 'checks connections between all nodes in the cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_check_cluster_node_health/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_check_cluster_node_health ( OUT from_nodename text, OUT from_nodeport int, OUT to_nodename text, OUT to_nodeport int, OUT result bool ) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_check_cluster_node_health$$; COMMENT ON FUNCTION pg_catalog.citus_check_cluster_node_health () IS 'checks connections between all nodes in the cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_check_connection_to_node/11.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_check_connection_to_node ( nodename text, nodeport integer DEFAULT 5432) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_check_connection_to_node$$; COMMENT ON FUNCTION pg_catalog.citus_check_connection_to_node ( nodename text, nodeport integer) IS 'checks connection to another node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_check_connection_to_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_check_connection_to_node ( nodename text, nodeport integer DEFAULT 5432) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_check_connection_to_node$$; COMMENT ON FUNCTION pg_catalog.citus_check_connection_to_node ( nodename text, nodeport integer) IS 'checks connection to another node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_cleanup_orphaned_resources/11.1-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_cleanup_orphaned_resources() LANGUAGE C AS 'citus', $$citus_cleanup_orphaned_resources$$; COMMENT ON PROCEDURE pg_catalog.citus_cleanup_orphaned_resources() IS 'cleanup orphaned resources'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_cleanup_orphaned_resources/latest.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_cleanup_orphaned_resources() LANGUAGE C AS 'citus', $$citus_cleanup_orphaned_resources$$; COMMENT ON PROCEDURE pg_catalog.citus_cleanup_orphaned_resources() IS 'cleanup orphaned resources'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_cleanup_orphaned_shards/10.1-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_cleanup_orphaned_shards() LANGUAGE C AS 'citus', $$citus_cleanup_orphaned_shards$$; COMMENT ON PROCEDURE pg_catalog.citus_cleanup_orphaned_shards() IS 'cleanup orphaned shards'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_cleanup_orphaned_shards/latest.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_cleanup_orphaned_shards() LANGUAGE C AS 'citus', $$citus_cleanup_orphaned_shards$$; COMMENT ON PROCEDURE pg_catalog.citus_cleanup_orphaned_shards() IS 'cleanup orphaned shards'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_conninfo_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_conninfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_conninfo_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_conninfo_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_conninfo_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_conninfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_authinfo_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_conninfo_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_conninfo_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_conninfo_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_conninfo_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_conninfo_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_coordinator_nodeid/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_coordinator_nodeid() RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_coordinator_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_coordinator_nodeid() IS 'returns node id of the coordinator node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_coordinator_nodeid/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_coordinator_nodeid() RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_coordinator_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_coordinator_nodeid() IS 'returns node id of the coordinator node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_copy_shard_placement/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, do_repair bool DEFAULT true, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement(shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, do_repair bool, shard_transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_copy_shard_placement/11.1-1.sql ================================================ DROP FUNCTION pg_catalog.citus_copy_shard_placement; CREATE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement(shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_copy_shard_placement/11.2-1.sql ================================================ -- citus_copy_shard_placement, but with nodeid CREATE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement(shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_copy_shard_placement/latest.sql ================================================ -- citus_copy_shard_placement, but with nodeid CREATE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_copy_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_copy_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_copy_shard_placement(shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'copy a shard from the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_disable_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer) IS 'removes node from the cluster temporarily'; REVOKE ALL ON FUNCTION pg_catalog.citus_disable_node(text,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_disable_node/11.0-1.sql ================================================ DROP FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer); CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool) IS 'removes node from the cluster temporarily'; REVOKE ALL ON FUNCTION pg_catalog.citus_disable_node(text,int, bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_disable_node/11.0-2.sql ================================================ DROP FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool); CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, synchronous bool default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, synchronous bool) IS 'removes node from the cluster temporarily'; REVOKE ALL ON FUNCTION pg_catalog.citus_disable_node(text,int, bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_disable_node/latest.sql ================================================ DROP FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, force bool); CREATE FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, synchronous bool default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_disable_node$$; COMMENT ON FUNCTION pg_catalog.citus_disable_node(nodename text, nodeport integer, synchronous bool) IS 'removes node from the cluster temporarily'; REVOKE ALL ON FUNCTION pg_catalog.citus_disable_node(text,int, bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_local_group_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_local_group_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_local_group_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_local_group_cache_invalidate() IS 'register node cache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_local_group_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_local_group_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_local_group_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_local_group_cache_invalidate() IS 'register node cache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_local_group_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_local_group_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_local_group_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_local_group_cache_invalidate() IS 'register node cache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_node_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_node_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_node_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_node_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_node_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_node_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_node_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_node_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_node_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_node_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_node_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_node_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_object_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_object_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_object_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_object_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_object_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_object_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_object_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_object_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_object_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_object_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_object_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_object_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_partition_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_partition_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$citus_dist_partition_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_partition_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_partition_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_partition_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$master_dist_partition_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_partition_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_partition_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_partition_cache_invalidate() RETURNS trigger LANGUAGE C AS 'citus', $$citus_dist_partition_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_partition_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_placement_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_placement_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_placement_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_placement_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_placement_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_placement_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_placement_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_placement_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_placement_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_placement_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_placement_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_placement_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_shard_cache_invalidate/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_shard_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_shard_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_shard_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_shard_cache_invalidate/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.master_dist_shard_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$master_dist_shard_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.master_dist_shard_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_shard_cache_invalidate/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_dist_shard_cache_invalidate() RETURNS trigger LANGUAGE C AS 'MODULE_PATHNAME', $$citus_dist_shard_cache_invalidate$$; COMMENT ON FUNCTION pg_catalog.citus_dist_shard_cache_invalidate() IS 'register relcache invalidation for changed rows'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_stat_activity/11.0-1.sql ================================================ DROP VIEW IF EXISTS pg_catalog.citus_dist_stat_activity; CREATE OR REPLACE VIEW citus.citus_dist_stat_activity AS SELECT * FROM citus_stat_activity WHERE is_worker_query = false; ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_dist_stat_activity/latest.sql ================================================ DROP VIEW IF EXISTS pg_catalog.citus_dist_stat_activity; CREATE OR REPLACE VIEW citus.citus_dist_stat_activity AS SELECT * FROM citus_stat_activity WHERE is_worker_query = false; ALTER VIEW citus.citus_dist_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_dist_stat_activity TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drain_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_drain_node( nodename text, nodeport integer, shard_transfer_mode citus.shard_transfer_mode default 'auto', rebalance_strategy name default NULL ) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$citus_drain_node$$; COMMENT ON FUNCTION pg_catalog.citus_drain_node(text,int,citus.shard_transfer_mode,name) IS 'mark a node to be drained of data and actually drain it as well'; REVOKE ALL ON FUNCTION pg_catalog.citus_drain_node(text,int,citus.shard_transfer_mode,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drain_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_drain_node( nodename text, nodeport integer, shard_transfer_mode citus.shard_transfer_mode default 'auto', rebalance_strategy name default NULL ) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$citus_drain_node$$; COMMENT ON FUNCTION pg_catalog.citus_drain_node(text,int,citus.shard_transfer_mode,name) IS 'mark a node to be drained of data and actually drain it as well'; REVOKE ALL ON FUNCTION pg_catalog.citus_drain_node(text,int,citus.shard_transfer_mode,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.notify_constraint_dropped() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$notify_constraint_dropped$$; CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE constraint_event_count INTEGER; v_obj record; sequence_names text[] := '{}'; table_colocation_id integer; propagate_drop boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; SELECT COUNT(*) INTO constraint_event_count FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table constraint'); IF constraint_event_count > 0 THEN -- Tell utility hook that a table constraint is dropped so we might -- need to undistribute some of the citus local tables that are not -- connected to any reference tables. PERFORM notify_constraint_dropped(); END IF; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE constraint_event_count INTEGER; v_obj record; dropped_table_is_a_partition boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); -- If both original and normal values are false, the dropped table was a partition -- that was dropped as a result of its parent being dropped -- NOTE: the other way around is not true: -- the table being a partition doesn't imply both original and normal values are false SELECT (v_obj.original = false AND v_obj.normal = false) INTO dropped_table_is_a_partition; -- The partition's shards will be dropped when dropping the parent's shards, so we can skip: -- i.e. we call citus_drop_all_shards with drop_shards_metadata_only parameter set to true IF dropped_table_is_a_partition THEN PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := true); ELSE PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false); END IF; PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; SELECT COUNT(*) INTO constraint_event_count FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table constraint'); IF constraint_event_count > 0 THEN -- Tell utility hook that a table constraint is dropped so we might -- need to undistribute some of the citus local tables that are not -- connected to any reference tables. PERFORM notify_constraint_dropped(); END IF; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE constraint_event_count INTEGER; v_obj record; dropped_table_is_a_partition boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); -- If both original and normal values are false, the dropped table was a partition -- that was dropped as a result of its parent being dropped -- NOTE: the other way around is not true: -- the table being a partition doesn't imply both original and normal values are false SELECT (v_obj.original = false AND v_obj.normal = false) INTO dropped_table_is_a_partition; -- The partition's shards will be dropped when dropping the parent's shards, so we can skip: -- i.e. we call citus_drop_all_shards with drop_shards_metadata_only parameter set to true IF dropped_table_is_a_partition THEN PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := true); ELSE PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false); END IF; PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP -- Remove entries from pg_catalog.pg_dist_schema for all dropped tenant schemas. -- Also delete the corresponding colocation group from pg_catalog.pg_dist_colocation. -- -- Although normally we automatically delete the colocation groups when they become empty, -- we don't do so for the colocation groups that are created for tenant schemas. For this -- reason, here we need to delete the colocation group when the tenant schema is dropped. IF v_obj.object_type = 'schema' AND EXISTS (SELECT 1 FROM pg_catalog.pg_dist_schema WHERE schemaid = v_obj.objid) THEN PERFORM pg_catalog.citus_internal_unregister_tenant_schema_globally(v_obj.objid, v_obj.object_name); END IF; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; SELECT COUNT(*) INTO constraint_event_count FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table constraint'); IF constraint_event_count > 0 THEN -- Tell utility hook that a table constraint is dropped so we might -- need to undistribute some of the citus local tables that are not -- connected to any reference tables. PERFORM notify_constraint_dropped(); END IF; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE constraint_event_count INTEGER; v_obj record; dropped_table_is_a_partition boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); -- If both original and normal values are false, the dropped table was a partition -- that was dropped as a result of its parent being dropped -- NOTE: the other way around is not true: -- the table being a partition doesn't imply both original and normal values are false SELECT (v_obj.original = false AND v_obj.normal = false) INTO dropped_table_is_a_partition; -- The partition's shards will be dropped when dropping the parent's shards, so we can skip: -- i.e. we call citus_drop_all_shards with drop_shards_metadata_only parameter set to true IF dropped_table_is_a_partition THEN PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := true); ELSE PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false); END IF; PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP -- Remove entries from pg_catalog.pg_dist_schema for all dropped tenant schemas. -- Also delete the corresponding colocation group from pg_catalog.pg_dist_colocation. -- -- Although normally we automatically delete the colocation groups when they become empty, -- we don't do so for the colocation groups that are created for tenant schemas. For this -- reason, here we need to delete the colocation group when the tenant schema is dropped. IF v_obj.object_type = 'schema' AND EXISTS (SELECT 1 FROM pg_catalog.pg_dist_schema WHERE schemaid = v_obj.objid) THEN PERFORM citus_internal.unregister_tenant_schema_globally(v_obj.objid, v_obj.object_name); END IF; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; SELECT COUNT(*) INTO constraint_event_count FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table constraint'); IF constraint_event_count > 0 THEN -- Tell utility hook that a table constraint is dropped so we might -- need to undistribute some of the citus local tables that are not -- connected to any reference tables. PERFORM notify_constraint_dropped(); END IF; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE v_obj record; sequence_names text[] := '{}'; table_colocation_id integer; propagate_drop boolean := false; BEGIN -- collect set of dropped sequences to drop on workers later SELECT array_agg(object_identity) INTO sequence_names FROM pg_event_trigger_dropped_objects() WHERE object_type = 'sequence'; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; IF cardinality(sequence_names) > 0 THEN PERFORM master_drop_sequences(sequence_names); END IF; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE v_obj record; sequence_names text[] := '{}'; table_colocation_id integer; propagate_drop boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name); PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_drop_trigger/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog AS $cdbdt$ DECLARE constraint_event_count INTEGER; v_obj record; dropped_table_is_a_partition boolean := false; BEGIN FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table', 'foreign table') LOOP -- first drop the table and metadata on the workers -- then drop all the shards on the workers -- finally remove the pg_dist_partition entry on the coordinator PERFORM master_remove_distributed_table_metadata_from_workers(v_obj.objid, v_obj.schema_name, v_obj.object_name); -- If both original and normal values are false, the dropped table was a partition -- that was dropped as a result of its parent being dropped -- NOTE: the other way around is not true: -- the table being a partition doesn't imply both original and normal values are false SELECT (v_obj.original = false AND v_obj.normal = false) INTO dropped_table_is_a_partition; -- The partition's shards will be dropped when dropping the parent's shards, so we can skip: -- i.e. we call citus_drop_all_shards with drop_shards_metadata_only parameter set to true IF dropped_table_is_a_partition THEN PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := true); ELSE PERFORM citus_drop_all_shards(v_obj.objid, v_obj.schema_name, v_obj.object_name, drop_shards_metadata_only := false); END IF; PERFORM master_remove_partition_metadata(v_obj.objid, v_obj.schema_name, v_obj.object_name); END LOOP; FOR v_obj IN SELECT * FROM pg_event_trigger_dropped_objects() LOOP -- Remove entries from pg_catalog.pg_dist_schema for all dropped tenant schemas. -- Also delete the corresponding colocation group from pg_catalog.pg_dist_colocation. -- -- Although normally we automatically delete the colocation groups when they become empty, -- we don't do so for the colocation groups that are created for tenant schemas. For this -- reason, here we need to delete the colocation group when the tenant schema is dropped. IF v_obj.object_type = 'schema' AND EXISTS (SELECT 1 FROM pg_catalog.pg_dist_schema WHERE schemaid = v_obj.objid) THEN PERFORM citus_internal.unregister_tenant_schema_globally(v_obj.objid, v_obj.object_name); END IF; -- remove entries from citus.pg_dist_object for all dropped root (objsubid = 0) objects PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid); END LOOP; SELECT COUNT(*) INTO constraint_event_count FROM pg_event_trigger_dropped_objects() WHERE object_type IN ('table constraint'); IF constraint_event_count > 0 THEN -- Tell utility hook that a table constraint is dropped so we might -- need to undistribute some of the citus local tables that are not -- connected to any reference tables. PERFORM notify_constraint_dropped(); END IF; END; $cdbdt$; COMMENT ON FUNCTION pg_catalog.citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_extradata_container/9.3-2.sql ================================================ -- we use the citus_extradata_container function as a range table entry in the query part -- executed on the coordinator. Now that we are letting this query be planned by the -- postgres planner we need to be able to pass column names and type information with this -- function. This requires the change of the prototype of the function and add a return -- type. Changing the return type of the function requires we drop the function first. DROP FUNCTION citus_extradata_container(INTERNAL); CREATE OR REPLACE FUNCTION citus_extradata_container(INTERNAL) RETURNS SETOF record LANGUAGE C AS 'MODULE_PATHNAME', $$citus_extradata_container$$; COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_extradata_container/latest.sql ================================================ -- we use the citus_extradata_container function as a range table entry in the query part -- executed on the coordinator. Now that we are letting this query be planned by the -- postgres planner we need to be able to pass column names and type information with this -- function. This requires the change of the prototype of the function and add a return -- type. Changing the return type of the function requires we drop the function first. DROP FUNCTION citus_extradata_container(INTERNAL); CREATE OR REPLACE FUNCTION citus_extradata_container(INTERNAL) RETURNS SETOF record LANGUAGE C AS 'MODULE_PATHNAME', $$citus_extradata_container$$; COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finalize_upgrade_to_citus11/11.0-1.sql ================================================ -- citus_finalize_upgrade_to_citus11() is a helper UDF ensures -- the upgrade to Citus 11 is finished successfully. Upgrade to -- Citus 11 requires all active primary worker nodes to get the -- metadata. And, this function's job is to sync the metadata to -- the nodes that does not already have -- once the function finishes without any errors and returns true -- the cluster is ready for running distributed queries from -- the worker nodes. When debug is enabled, the function provides -- more information to the user. CREATE OR REPLACE FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(enforce_version_check bool default true) RETURNS bool LANGUAGE plpgsql AS $$ BEGIN --------------------------------------------- -- This script consists of N stages -- Each step is documented, and if log level -- is reduced to DEBUG1, each step is logged -- as well --------------------------------------------- ------------------------------------------------------------------------------------------ -- STAGE 0: Ensure no concurrent node metadata changing operation happens while this -- script is running via acquiring a strong lock on the pg_dist_node ------------------------------------------------------------------------------------------ BEGIN LOCK TABLE pg_dist_node IN EXCLUSIVE MODE NOWAIT; EXCEPTION WHEN OTHERS THEN RAISE 'Another node metadata changing operation is in progress, try again.'; END; ------------------------------------------------------------------------------------------ -- STAGE 1: We want all the commands to run in the same transaction block. Without -- sequential mode, metadata syncing cannot be done in a transaction block along with -- other commands ------------------------------------------------------------------------------------------ SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; ------------------------------------------------------------------------------------------ -- STAGE 2: Ensure we have the prerequisites -- (a) only superuser can run this script -- (b) cannot be executed when enable_ddl_propagation is False -- (c) can only be executed from the coordinator ------------------------------------------------------------------------------------------ DECLARE is_superuser_running boolean := False; enable_ddl_prop boolean:= False; local_group_id int := 0; BEGIN SELECT rolsuper INTO is_superuser_running FROM pg_roles WHERE rolname = current_user; IF is_superuser_running IS NOT True THEN RAISE EXCEPTION 'This operation can only be initiated by superuser'; END IF; SELECT current_setting('citus.enable_ddl_propagation') INTO enable_ddl_prop; IF enable_ddl_prop IS NOT True THEN RAISE EXCEPTION 'This operation cannot be completed when citus.enable_ddl_propagation is False.'; END IF; SELECT groupid INTO local_group_id FROM pg_dist_local_group; IF local_group_id != 0 THEN RAISE EXCEPTION 'Operation is not allowed on this node. Connect to the coordinator and run it again.'; ELSE RAISE DEBUG 'We are on the coordinator, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 3: Ensure all primary nodes are active ------------------------------------------------------------------------------------------ DECLARE primary_disabled_worker_node_count int := 0; BEGIN SELECT count(*) INTO primary_disabled_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary' AND NOT isactive; IF primary_disabled_worker_node_count != 0 THEN RAISE EXCEPTION 'There are inactive primary worker nodes, you need to activate the nodes first.' 'Use SELECT citus_activate_node() to activate the disabled nodes'; ELSE RAISE DEBUG 'There are no disabled worker nodes, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 4: Ensure there is no connectivity issues in the cluster ------------------------------------------------------------------------------------------ DECLARE all_nodes_can_connect_to_each_other boolean := False; BEGIN SELECT bool_and(coalesce(result, false)) INTO all_nodes_can_connect_to_each_other FROM citus_check_cluster_node_health(); IF all_nodes_can_connect_to_each_other != True THEN RAISE EXCEPTION 'There are unhealth primary nodes, you need to ensure all ' 'nodes are up and running. Also, make sure that all nodes can connect ' 'to each other. Use SELECT * FROM citus_check_cluster_node_health(); ' 'to check the cluster health'; ELSE RAISE DEBUG 'Cluster is healthy, all nodes can connect to each other'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 5: Ensure all nodes are on the same version ------------------------------------------------------------------------------------------ DECLARE coordinator_version text := ''; worker_node_version text := ''; worker_node_version_count int := 0; BEGIN SELECT extversion INTO coordinator_version from pg_extension WHERE extname = 'citus'; -- first, check if all nodes have the same versions SELECT count(distinct result) INTO worker_node_version_count FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'''); IF enforce_version_check AND worker_node_version_count != 1 THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'some of the workers have different versions.'; ELSE RAISE DEBUG 'All worker nodes have the same Citus version'; END IF; -- second, check if all nodes have the same versions SELECT result INTO worker_node_version FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'';') GROUP BY result; IF enforce_version_check AND coordinator_version != worker_node_version THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'the coordinator has version % and the worker(s) has %', coordinator_version, worker_node_version; ELSE RAISE DEBUG 'All nodes have the same Citus version'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 6: Ensure all the partitioned tables have the proper naming structure -- As described on https://github.com/citusdata/citus/issues/4962 -- existing indexes on partitioned distributed tables can collide -- with the index names exists on the shards -- luckily, we know how to fix it. -- And, note that we should do this even if the cluster is a basic plan -- (e.g., single node Citus) such that when cluster scaled out, everything -- works as intended -- And, this should be done only ONCE for a cluster as it can be a pretty -- time consuming operation. Thus, even if the function is called multiple time, -- we keep track of it and do not re-execute this part if not needed. ------------------------------------------------------------------------------------------ DECLARE partitioned_table_exists_pre_11 boolean:=False; BEGIN -- we recorded if partitioned tables exists during upgrade to Citus 11 SELECT metadata->>'partitioned_citus_table_exists_pre_11' INTO partitioned_table_exists_pre_11 FROM pg_dist_node_metadata; IF partitioned_table_exists_pre_11 IS NOT NULL AND partitioned_table_exists_pre_11 THEN -- this might take long depending on the number of partitions and shards... RAISE NOTICE 'Preparing all the existing partitioned table indexes'; PERFORM pg_catalog.fix_all_partition_shard_index_names(); -- great, we are done with fixing the existing wrong index names -- so, lets remove this UPDATE pg_dist_node_metadata SET metadata=jsonb_delete(metadata, 'partitioned_citus_table_exists_pre_11'); ELSE RAISE DEBUG 'There are no partitioned tables that should be fixed'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 7: Return early if there are no primary worker nodes -- We don't strictly need this step, but it gives a nicer notice message ------------------------------------------------------------------------------------------ DECLARE primary_worker_node_count bigint :=0; BEGIN SELECT count(*) INTO primary_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary'; IF primary_worker_node_count = 0 THEN RAISE NOTICE 'There are no primary worker nodes, no need to sync metadata to any node'; RETURN true; ELSE RAISE DEBUG 'There are % primary worker nodes, continue to sync metadata', primary_worker_node_count; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 8: Do the actual metadata & object syncing to the worker nodes -- For the "already synced" metadata nodes, we do not strictly need to -- sync the objects & metadata, but there is no harm to do it anyway -- it'll only cost some execution time but makes sure that we have a -- a consistent metadata & objects across all the nodes ------------------------------------------------------------------------------------------ DECLARE BEGIN -- this might take long depending on the number of tables & objects ... RAISE NOTICE 'Preparing to sync the metadata to all nodes'; PERFORM start_metadata_sync_to_node(nodename,nodeport) FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary'; END; RETURN true; END; $$; COMMENT ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) IS 'finalizes upgrade to Citus'; REVOKE ALL ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finalize_upgrade_to_citus11/11.0-2.sql ================================================ -- citus_finalize_upgrade_to_citus11() is a helper UDF ensures -- the upgrade to Citus 11 is finished successfully. Upgrade to -- Citus 11 requires all active primary worker nodes to get the -- metadata. And, this function's job is to sync the metadata to -- the nodes that does not already have -- once the function finishes without any errors and returns true -- the cluster is ready for running distributed queries from -- the worker nodes. When debug is enabled, the function provides -- more information to the user. CREATE OR REPLACE FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(enforce_version_check bool default true) RETURNS bool LANGUAGE plpgsql AS $$ BEGIN --------------------------------------------- -- This script consists of N stages -- Each step is documented, and if log level -- is reduced to DEBUG1, each step is logged -- as well --------------------------------------------- ------------------------------------------------------------------------------------------ -- STAGE 0: Ensure no concurrent node metadata changing operation happens while this -- script is running via acquiring a strong lock on the pg_dist_node ------------------------------------------------------------------------------------------ BEGIN LOCK TABLE pg_dist_node IN EXCLUSIVE MODE NOWAIT; EXCEPTION WHEN OTHERS THEN RAISE 'Another node metadata changing operation is in progress, try again.'; END; ------------------------------------------------------------------------------------------ -- STAGE 1: We want all the commands to run in the same transaction block. Without -- sequential mode, metadata syncing cannot be done in a transaction block along with -- other commands ------------------------------------------------------------------------------------------ SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; ------------------------------------------------------------------------------------------ -- STAGE 2: Ensure we have the prerequisites -- (a) only superuser can run this script -- (b) cannot be executed when enable_ddl_propagation is False -- (c) can only be executed from the coordinator ------------------------------------------------------------------------------------------ DECLARE is_superuser_running boolean := False; enable_ddl_prop boolean:= False; local_group_id int := 0; BEGIN SELECT rolsuper INTO is_superuser_running FROM pg_roles WHERE rolname = current_user; IF is_superuser_running IS NOT True THEN RAISE EXCEPTION 'This operation can only be initiated by superuser'; END IF; SELECT current_setting('citus.enable_ddl_propagation') INTO enable_ddl_prop; IF enable_ddl_prop IS NOT True THEN RAISE EXCEPTION 'This operation cannot be completed when citus.enable_ddl_propagation is False.'; END IF; SELECT groupid INTO local_group_id FROM pg_dist_local_group; IF local_group_id != 0 THEN RAISE EXCEPTION 'Operation is not allowed on this node. Connect to the coordinator and run it again.'; ELSE RAISE DEBUG 'We are on the coordinator, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 3: Ensure all primary nodes are active ------------------------------------------------------------------------------------------ DECLARE primary_disabled_worker_node_count int := 0; BEGIN SELECT count(*) INTO primary_disabled_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary' AND NOT isactive; IF primary_disabled_worker_node_count != 0 THEN RAISE EXCEPTION 'There are inactive primary worker nodes, you need to activate the nodes first.' 'Use SELECT citus_activate_node() to activate the disabled nodes'; ELSE RAISE DEBUG 'There are no disabled worker nodes, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 4: Ensure there is no connectivity issues in the cluster ------------------------------------------------------------------------------------------ DECLARE all_nodes_can_connect_to_each_other boolean := False; BEGIN SELECT bool_and(coalesce(result, false)) INTO all_nodes_can_connect_to_each_other FROM citus_check_cluster_node_health(); IF all_nodes_can_connect_to_each_other != True THEN RAISE EXCEPTION 'There are unhealth primary nodes, you need to ensure all ' 'nodes are up and running. Also, make sure that all nodes can connect ' 'to each other. Use SELECT * FROM citus_check_cluster_node_health(); ' 'to check the cluster health'; ELSE RAISE DEBUG 'Cluster is healthy, all nodes can connect to each other'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 5: Ensure all nodes are on the same version ------------------------------------------------------------------------------------------ DECLARE coordinator_version text := ''; worker_node_version text := ''; worker_node_version_count int := 0; BEGIN SELECT extversion INTO coordinator_version from pg_extension WHERE extname = 'citus'; -- first, check if all nodes have the same versions SELECT count(distinct result) INTO worker_node_version_count FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'''); IF enforce_version_check AND worker_node_version_count != 1 THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'some of the workers have different versions.'; ELSE RAISE DEBUG 'All worker nodes have the same Citus version'; END IF; -- second, check if all nodes have the same versions SELECT result INTO worker_node_version FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'';') GROUP BY result; IF enforce_version_check AND coordinator_version != worker_node_version THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'the coordinator has version % and the worker(s) has %', coordinator_version, worker_node_version; ELSE RAISE DEBUG 'All nodes have the same Citus version'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 6: Ensure all the partitioned tables have the proper naming structure -- As described on https://github.com/citusdata/citus/issues/4962 -- existing indexes on partitioned distributed tables can collide -- with the index names exists on the shards -- luckily, we know how to fix it. -- And, note that we should do this even if the cluster is a basic plan -- (e.g., single node Citus) such that when cluster scaled out, everything -- works as intended -- And, this should be done only ONCE for a cluster as it can be a pretty -- time consuming operation. Thus, even if the function is called multiple time, -- we keep track of it and do not re-execute this part if not needed. ------------------------------------------------------------------------------------------ DECLARE partitioned_table_exists_pre_11 boolean:=False; BEGIN -- we recorded if partitioned tables exists during upgrade to Citus 11 SELECT metadata->>'partitioned_citus_table_exists_pre_11' INTO partitioned_table_exists_pre_11 FROM pg_dist_node_metadata; IF partitioned_table_exists_pre_11 IS NOT NULL AND partitioned_table_exists_pre_11 THEN -- this might take long depending on the number of partitions and shards... RAISE NOTICE 'Preparing all the existing partitioned table indexes'; PERFORM pg_catalog.fix_all_partition_shard_index_names(); -- great, we are done with fixing the existing wrong index names -- so, lets remove this UPDATE pg_dist_node_metadata SET metadata=jsonb_delete(metadata, 'partitioned_citus_table_exists_pre_11'); ELSE RAISE DEBUG 'There are no partitioned tables that should be fixed'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 7: Return early if there are no primary worker nodes -- We don't strictly need this step, but it gives a nicer notice message ------------------------------------------------------------------------------------------ DECLARE primary_worker_node_count bigint :=0; BEGIN SELECT count(*) INTO primary_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary'; IF primary_worker_node_count = 0 THEN RAISE NOTICE 'There are no primary worker nodes, no need to sync metadata to any node'; RETURN true; ELSE RAISE DEBUG 'There are % primary worker nodes, continue to sync metadata', primary_worker_node_count; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 8: Do the actual metadata & object syncing to the worker nodes -- For the "already synced" metadata nodes, we do not strictly need to -- sync the objects & metadata, but there is no harm to do it anyway -- it'll only cost some execution time but makes sure that we have a -- a consistent metadata & objects across all the nodes ------------------------------------------------------------------------------------------ DECLARE BEGIN -- this might take long depending on the number of tables & objects ... RAISE NOTICE 'Preparing to sync the metadata to all nodes'; PERFORM start_metadata_sync_to_all_nodes(); END; RETURN true; END; $$; COMMENT ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) IS 'finalizes upgrade to Citus'; REVOKE ALL ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finalize_upgrade_to_citus11/11.0-3.sql ================================================ -- citus_finalize_upgrade_to_citus11() is a helper UDF ensures -- the upgrade to Citus 11 is finished successfully. Upgrade to -- Citus 11 requires all active primary worker nodes to get the -- metadata. And, this function's job is to sync the metadata to -- the nodes that does not already have -- once the function finishes without any errors and returns true -- the cluster is ready for running distributed queries from -- the worker nodes. When debug is enabled, the function provides -- more information to the user. CREATE OR REPLACE FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(enforce_version_check bool default true) RETURNS bool LANGUAGE plpgsql AS $$ BEGIN --------------------------------------------- -- This script consists of N stages -- Each step is documented, and if log level -- is reduced to DEBUG1, each step is logged -- as well --------------------------------------------- ------------------------------------------------------------------------------------------ -- STAGE 0: Ensure no concurrent node metadata changing operation happens while this -- script is running via acquiring a strong lock on the pg_dist_node ------------------------------------------------------------------------------------------ BEGIN LOCK TABLE pg_dist_node IN EXCLUSIVE MODE NOWAIT; EXCEPTION WHEN OTHERS THEN RAISE 'Another node metadata changing operation is in progress, try again.'; END; ------------------------------------------------------------------------------------------ -- STAGE 1: We want all the commands to run in the same transaction block. Without -- sequential mode, metadata syncing cannot be done in a transaction block along with -- other commands ------------------------------------------------------------------------------------------ SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; ------------------------------------------------------------------------------------------ -- STAGE 2: Ensure we have the prerequisites -- (a) only superuser can run this script -- (b) cannot be executed when enable_ddl_propagation is False -- (c) can only be executed from the coordinator ------------------------------------------------------------------------------------------ DECLARE is_superuser_running boolean := False; enable_ddl_prop boolean:= False; local_group_id int := 0; BEGIN SELECT rolsuper INTO is_superuser_running FROM pg_roles WHERE rolname = current_user; IF is_superuser_running IS NOT True THEN RAISE EXCEPTION 'This operation can only be initiated by superuser'; END IF; SELECT current_setting('citus.enable_ddl_propagation') INTO enable_ddl_prop; IF enable_ddl_prop IS NOT True THEN RAISE EXCEPTION 'This operation cannot be completed when citus.enable_ddl_propagation is False.'; END IF; SELECT groupid INTO local_group_id FROM pg_dist_local_group; IF local_group_id != 0 THEN RAISE EXCEPTION 'Operation is not allowed on this node. Connect to the coordinator and run it again.'; ELSE RAISE DEBUG 'We are on the coordinator, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 3: Ensure all primary nodes are active ------------------------------------------------------------------------------------------ DECLARE primary_disabled_worker_node_count int := 0; BEGIN SELECT count(*) INTO primary_disabled_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary' AND NOT isactive; IF primary_disabled_worker_node_count != 0 THEN RAISE EXCEPTION 'There are inactive primary worker nodes, you need to activate the nodes first.' 'Use SELECT citus_activate_node() to activate the disabled nodes'; ELSE RAISE DEBUG 'There are no disabled worker nodes, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 4: Ensure there is no connectivity issues in the cluster ------------------------------------------------------------------------------------------ DECLARE all_nodes_can_connect_to_each_other boolean := False; BEGIN SELECT bool_and(coalesce(result, false)) INTO all_nodes_can_connect_to_each_other FROM citus_check_cluster_node_health(); IF all_nodes_can_connect_to_each_other != True THEN RAISE EXCEPTION 'There are unhealth primary nodes, you need to ensure all ' 'nodes are up and running. Also, make sure that all nodes can connect ' 'to each other. Use SELECT * FROM citus_check_cluster_node_health(); ' 'to check the cluster health'; ELSE RAISE DEBUG 'Cluster is healthy, all nodes can connect to each other'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 5: Ensure all nodes are on the same version ------------------------------------------------------------------------------------------ DECLARE coordinator_version text := ''; worker_node_version text := ''; worker_node_version_count int := 0; BEGIN SELECT extversion INTO coordinator_version from pg_extension WHERE extname = 'citus'; -- first, check if all nodes have the same versions SELECT count(distinct result) INTO worker_node_version_count FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'''); IF enforce_version_check AND worker_node_version_count = 0 THEN RAISE DEBUG 'There are no worker nodes'; ELSIF enforce_version_check AND worker_node_version_count != 1 THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'some of the workers have different versions.'; ELSE RAISE DEBUG 'All worker nodes have the same Citus version'; END IF; -- second, check if all nodes have the same versions SELECT result INTO worker_node_version FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'';') GROUP BY result; IF enforce_version_check AND coordinator_version != worker_node_version THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'the coordinator has version % and the worker(s) has %', coordinator_version, worker_node_version; ELSE RAISE DEBUG 'All nodes have the same Citus version'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 6: Ensure all the partitioned tables have the proper naming structure -- As described on https://github.com/citusdata/citus/issues/4962 -- existing indexes on partitioned distributed tables can collide -- with the index names exists on the shards -- luckily, we know how to fix it. -- And, note that we should do this even if the cluster is a basic plan -- (e.g., single node Citus) such that when cluster scaled out, everything -- works as intended -- And, this should be done only ONCE for a cluster as it can be a pretty -- time consuming operation. Thus, even if the function is called multiple time, -- we keep track of it and do not re-execute this part if not needed. ------------------------------------------------------------------------------------------ DECLARE partitioned_table_exists_pre_11 boolean:=False; BEGIN -- we recorded if partitioned tables exists during upgrade to Citus 11 SELECT metadata->>'partitioned_citus_table_exists_pre_11' INTO partitioned_table_exists_pre_11 FROM pg_dist_node_metadata; IF partitioned_table_exists_pre_11 IS NOT NULL AND partitioned_table_exists_pre_11 THEN -- this might take long depending on the number of partitions and shards... RAISE NOTICE 'Preparing all the existing partitioned table indexes'; PERFORM pg_catalog.fix_all_partition_shard_index_names(); -- great, we are done with fixing the existing wrong index names -- so, lets remove this UPDATE pg_dist_node_metadata SET metadata=jsonb_delete(metadata, 'partitioned_citus_table_exists_pre_11'); ELSE RAISE DEBUG 'There are no partitioned tables that should be fixed'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 7: Return early if there are no primary worker nodes -- We don't strictly need this step, but it gives a nicer notice message ------------------------------------------------------------------------------------------ DECLARE primary_worker_node_count bigint :=0; BEGIN SELECT count(*) INTO primary_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary'; IF primary_worker_node_count = 0 THEN RAISE NOTICE 'There are no primary worker nodes, no need to sync metadata to any node'; RETURN true; ELSE RAISE DEBUG 'There are % primary worker nodes, continue to sync metadata', primary_worker_node_count; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 8: Do the actual metadata & object syncing to the worker nodes -- For the "already synced" metadata nodes, we do not strictly need to -- sync the objects & metadata, but there is no harm to do it anyway -- it'll only cost some execution time but makes sure that we have a -- a consistent metadata & objects across all the nodes ------------------------------------------------------------------------------------------ DECLARE BEGIN -- this might take long depending on the number of tables & objects ... RAISE NOTICE 'Preparing to sync the metadata to all nodes'; PERFORM start_metadata_sync_to_all_nodes(); END; RETURN true; END; $$; COMMENT ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) IS 'finalizes upgrade to Citus'; REVOKE ALL ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finalize_upgrade_to_citus11/latest.sql ================================================ -- citus_finalize_upgrade_to_citus11() is a helper UDF ensures -- the upgrade to Citus 11 is finished successfully. Upgrade to -- Citus 11 requires all active primary worker nodes to get the -- metadata. And, this function's job is to sync the metadata to -- the nodes that does not already have -- once the function finishes without any errors and returns true -- the cluster is ready for running distributed queries from -- the worker nodes. When debug is enabled, the function provides -- more information to the user. CREATE OR REPLACE FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(enforce_version_check bool default true) RETURNS bool LANGUAGE plpgsql AS $$ BEGIN --------------------------------------------- -- This script consists of N stages -- Each step is documented, and if log level -- is reduced to DEBUG1, each step is logged -- as well --------------------------------------------- ------------------------------------------------------------------------------------------ -- STAGE 0: Ensure no concurrent node metadata changing operation happens while this -- script is running via acquiring a strong lock on the pg_dist_node ------------------------------------------------------------------------------------------ BEGIN LOCK TABLE pg_dist_node IN EXCLUSIVE MODE NOWAIT; EXCEPTION WHEN OTHERS THEN RAISE 'Another node metadata changing operation is in progress, try again.'; END; ------------------------------------------------------------------------------------------ -- STAGE 1: We want all the commands to run in the same transaction block. Without -- sequential mode, metadata syncing cannot be done in a transaction block along with -- other commands ------------------------------------------------------------------------------------------ SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; ------------------------------------------------------------------------------------------ -- STAGE 2: Ensure we have the prerequisites -- (a) only superuser can run this script -- (b) cannot be executed when enable_ddl_propagation is False -- (c) can only be executed from the coordinator ------------------------------------------------------------------------------------------ DECLARE is_superuser_running boolean := False; enable_ddl_prop boolean:= False; local_group_id int := 0; BEGIN SELECT rolsuper INTO is_superuser_running FROM pg_roles WHERE rolname = current_user; IF is_superuser_running IS NOT True THEN RAISE EXCEPTION 'This operation can only be initiated by superuser'; END IF; SELECT current_setting('citus.enable_ddl_propagation') INTO enable_ddl_prop; IF enable_ddl_prop IS NOT True THEN RAISE EXCEPTION 'This operation cannot be completed when citus.enable_ddl_propagation is False.'; END IF; SELECT groupid INTO local_group_id FROM pg_dist_local_group; IF local_group_id != 0 THEN RAISE EXCEPTION 'Operation is not allowed on this node. Connect to the coordinator and run it again.'; ELSE RAISE DEBUG 'We are on the coordinator, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 3: Ensure all primary nodes are active ------------------------------------------------------------------------------------------ DECLARE primary_disabled_worker_node_count int := 0; BEGIN SELECT count(*) INTO primary_disabled_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary' AND NOT isactive; IF primary_disabled_worker_node_count != 0 THEN RAISE EXCEPTION 'There are inactive primary worker nodes, you need to activate the nodes first.' 'Use SELECT citus_activate_node() to activate the disabled nodes'; ELSE RAISE DEBUG 'There are no disabled worker nodes, continue to sync metadata'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 4: Ensure there is no connectivity issues in the cluster ------------------------------------------------------------------------------------------ DECLARE all_nodes_can_connect_to_each_other boolean := False; BEGIN SELECT bool_and(coalesce(result, false)) INTO all_nodes_can_connect_to_each_other FROM citus_check_cluster_node_health(); IF all_nodes_can_connect_to_each_other != True THEN RAISE EXCEPTION 'There are unhealth primary nodes, you need to ensure all ' 'nodes are up and running. Also, make sure that all nodes can connect ' 'to each other. Use SELECT * FROM citus_check_cluster_node_health(); ' 'to check the cluster health'; ELSE RAISE DEBUG 'Cluster is healthy, all nodes can connect to each other'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 5: Ensure all nodes are on the same version ------------------------------------------------------------------------------------------ DECLARE coordinator_version text := ''; worker_node_version text := ''; worker_node_version_count int := 0; BEGIN SELECT extversion INTO coordinator_version from pg_extension WHERE extname = 'citus'; -- first, check if all nodes have the same versions SELECT count(distinct result) INTO worker_node_version_count FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'''); IF enforce_version_check AND worker_node_version_count = 0 THEN RAISE DEBUG 'There are no worker nodes'; ELSIF enforce_version_check AND worker_node_version_count != 1 THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'some of the workers have different versions.'; ELSE RAISE DEBUG 'All worker nodes have the same Citus version'; END IF; -- second, check if all nodes have the same versions SELECT result INTO worker_node_version FROM run_command_on_workers('SELECT extversion from pg_extension WHERE extname = ''citus'';') GROUP BY result; IF enforce_version_check AND coordinator_version != worker_node_version THEN RAISE EXCEPTION 'All nodes should have the same Citus version installed. Currently ' 'the coordinator has version % and the worker(s) has %', coordinator_version, worker_node_version; ELSE RAISE DEBUG 'All nodes have the same Citus version'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 6: Ensure all the partitioned tables have the proper naming structure -- As described on https://github.com/citusdata/citus/issues/4962 -- existing indexes on partitioned distributed tables can collide -- with the index names exists on the shards -- luckily, we know how to fix it. -- And, note that we should do this even if the cluster is a basic plan -- (e.g., single node Citus) such that when cluster scaled out, everything -- works as intended -- And, this should be done only ONCE for a cluster as it can be a pretty -- time consuming operation. Thus, even if the function is called multiple time, -- we keep track of it and do not re-execute this part if not needed. ------------------------------------------------------------------------------------------ DECLARE partitioned_table_exists_pre_11 boolean:=False; BEGIN -- we recorded if partitioned tables exists during upgrade to Citus 11 SELECT metadata->>'partitioned_citus_table_exists_pre_11' INTO partitioned_table_exists_pre_11 FROM pg_dist_node_metadata; IF partitioned_table_exists_pre_11 IS NOT NULL AND partitioned_table_exists_pre_11 THEN -- this might take long depending on the number of partitions and shards... RAISE NOTICE 'Preparing all the existing partitioned table indexes'; PERFORM pg_catalog.fix_all_partition_shard_index_names(); -- great, we are done with fixing the existing wrong index names -- so, lets remove this UPDATE pg_dist_node_metadata SET metadata=jsonb_delete(metadata, 'partitioned_citus_table_exists_pre_11'); ELSE RAISE DEBUG 'There are no partitioned tables that should be fixed'; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 7: Return early if there are no primary worker nodes -- We don't strictly need this step, but it gives a nicer notice message ------------------------------------------------------------------------------------------ DECLARE primary_worker_node_count bigint :=0; BEGIN SELECT count(*) INTO primary_worker_node_count FROM pg_dist_node WHERE groupid != 0 AND noderole = 'primary'; IF primary_worker_node_count = 0 THEN RAISE NOTICE 'There are no primary worker nodes, no need to sync metadata to any node'; RETURN true; ELSE RAISE DEBUG 'There are % primary worker nodes, continue to sync metadata', primary_worker_node_count; END IF; END; ------------------------------------------------------------------------------------------ -- STAGE 8: Do the actual metadata & object syncing to the worker nodes -- For the "already synced" metadata nodes, we do not strictly need to -- sync the objects & metadata, but there is no harm to do it anyway -- it'll only cost some execution time but makes sure that we have a -- a consistent metadata & objects across all the nodes ------------------------------------------------------------------------------------------ DECLARE BEGIN -- this might take long depending on the number of tables & objects ... RAISE NOTICE 'Preparing to sync the metadata to all nodes'; PERFORM start_metadata_sync_to_all_nodes(); END; RETURN true; END; $$; COMMENT ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) IS 'finalizes upgrade to Citus'; REVOKE ALL ON FUNCTION pg_catalog.citus_finalize_upgrade_to_citus11(bool) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_citus_upgrade/11.0-2.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_finish_citus_upgrade() LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE current_version_string text; last_upgrade_version_string text; last_upgrade_major_version int; last_upgrade_minor_version int; last_upgrade_sqlpatch_version int; performed_upgrade bool := false; BEGIN SELECT extversion INTO current_version_string FROM pg_extension WHERE extname = 'citus'; -- assume some arbitrarily old version when no last upgrade version is defined SELECT coalesce(metadata->>'last_upgrade_version', '8.0-1') INTO last_upgrade_version_string FROM pg_dist_node_metadata; SELECT r[1], r[2], r[3] FROM regexp_matches(last_upgrade_version_string,'([0-9]+)\.([0-9]+)-([0-9]+)','') r INTO last_upgrade_major_version, last_upgrade_minor_version, last_upgrade_sqlpatch_version; IF last_upgrade_major_version IS NULL OR last_upgrade_minor_version IS NULL OR last_upgrade_sqlpatch_version IS NULL THEN -- version string is not valid, use an arbitrarily old version number last_upgrade_major_version := 8; last_upgrade_minor_version := 0; last_upgrade_sqlpatch_version := 1; END IF; IF last_upgrade_major_version < 11 THEN PERFORM citus_finalize_upgrade_to_citus11(); performed_upgrade := true; END IF; -- add new upgrade steps here IF NOT performed_upgrade THEN RAISE NOTICE 'already at the latest distributed schema version (%)', last_upgrade_version_string; RETURN; END IF; UPDATE pg_dist_node_metadata SET metadata = jsonb_set(metadata, array['last_upgrade_version'], to_jsonb(current_version_string)); END; $cppu$; COMMENT ON PROCEDURE pg_catalog.citus_finish_citus_upgrade() IS 'after upgrading Citus on all nodes call this function to upgrade the distributed schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_citus_upgrade/14.0-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_finish_citus_upgrade() LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE current_version_string text; last_upgrade_version_string text; last_upgrade_major_version int; last_upgrade_minor_version int; last_upgrade_sqlpatch_version int; performed_upgrade bool := false; BEGIN SELECT extversion INTO current_version_string FROM pg_extension WHERE extname = 'citus'; -- assume some arbitrarily old version when no last upgrade version is defined SELECT coalesce(metadata->>'last_upgrade_version', '8.0-1') INTO last_upgrade_version_string FROM pg_dist_node_metadata; SELECT r[1], r[2], r[3] FROM regexp_matches(last_upgrade_version_string,'([0-9]+)\.([0-9]+)-([0-9]+)','') r INTO last_upgrade_major_version, last_upgrade_minor_version, last_upgrade_sqlpatch_version; IF last_upgrade_major_version IS NULL OR last_upgrade_minor_version IS NULL OR last_upgrade_sqlpatch_version IS NULL THEN -- version string is not valid, use an arbitrarily old version number last_upgrade_major_version := 8; last_upgrade_minor_version := 0; last_upgrade_sqlpatch_version := 1; END IF; IF last_upgrade_major_version < 11 THEN PERFORM citus_finalize_upgrade_to_citus11(); performed_upgrade := true; END IF; IF last_upgrade_major_version < 14 THEN PERFORM fix_pre_citus14_colocation_group_collation_mismatches(); performed_upgrade := true; END IF; -- add new upgrade steps here IF NOT performed_upgrade THEN RAISE NOTICE 'already at the latest distributed schema version (%)', last_upgrade_version_string; RETURN; END IF; UPDATE pg_dist_node_metadata SET metadata = jsonb_set(metadata, array['last_upgrade_version'], to_jsonb(current_version_string)); END; $cppu$; COMMENT ON PROCEDURE pg_catalog.citus_finish_citus_upgrade() IS 'after upgrading Citus on all nodes call this function to upgrade the distributed schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_citus_upgrade/latest.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.citus_finish_citus_upgrade() LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE current_version_string text; last_upgrade_version_string text; last_upgrade_major_version int; last_upgrade_minor_version int; last_upgrade_sqlpatch_version int; performed_upgrade bool := false; BEGIN SELECT extversion INTO current_version_string FROM pg_extension WHERE extname = 'citus'; -- assume some arbitrarily old version when no last upgrade version is defined SELECT coalesce(metadata->>'last_upgrade_version', '8.0-1') INTO last_upgrade_version_string FROM pg_dist_node_metadata; SELECT r[1], r[2], r[3] FROM regexp_matches(last_upgrade_version_string,'([0-9]+)\.([0-9]+)-([0-9]+)','') r INTO last_upgrade_major_version, last_upgrade_minor_version, last_upgrade_sqlpatch_version; IF last_upgrade_major_version IS NULL OR last_upgrade_minor_version IS NULL OR last_upgrade_sqlpatch_version IS NULL THEN -- version string is not valid, use an arbitrarily old version number last_upgrade_major_version := 8; last_upgrade_minor_version := 0; last_upgrade_sqlpatch_version := 1; END IF; IF last_upgrade_major_version < 11 THEN PERFORM citus_finalize_upgrade_to_citus11(); performed_upgrade := true; END IF; IF last_upgrade_major_version < 14 THEN PERFORM fix_pre_citus14_colocation_group_collation_mismatches(); performed_upgrade := true; END IF; -- add new upgrade steps here IF NOT performed_upgrade THEN RAISE NOTICE 'already at the latest distributed schema version (%)', last_upgrade_version_string; RETURN; END IF; UPDATE pg_dist_node_metadata SET metadata = jsonb_set(metadata, array['last_upgrade_version'], to_jsonb(current_version_string)); END; $cppu$; COMMENT ON PROCEDURE pg_catalog.citus_finish_citus_upgrade() IS 'after upgrading Citus on all nodes call this function to upgrade the distributed schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers -- DELETE/INSERT to avoid primary key violations WITH old_records AS ( DELETE FROM citus.pg_dist_object RETURNING type, object_names, object_args, distribution_argument_index, colocationid ) INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM old_records naming, pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; PERFORM citus_internal.columnar_ensure_objects_exist(); END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.0-4.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; PERFORM citus_internal.columnar_ensure_objects_exist(); END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; $cmd$; ELSE EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-4.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; $cmd$; ELSE EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM citus_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-5.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; $cmd$; ELSE EXECUTE $cmd$ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM citus_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM citus_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/11.0-4.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM citus_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/12.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- PG16 has its own any_value, so only create it pre PG16. -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; RESET citus.enable_ddl_propagation; -- -- Citus creates the any_value aggregate but because of a compatibility -- issue between pg15-pg16 -- any_value is created in PG16, we drop -- and create it during upgrade IF upgraded version is less than 16. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; $cmd$; END IF; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; -- if we are upgrading from PG14/PG15 to PG16+, -- we need to regenerate the partkeys because they will include varnullingrels as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_16_upgrade WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- PG16 has its own any_value, so only create it pre PG16. -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; RESET citus.enable_ddl_propagation; -- -- Citus creates the any_value aggregate but because of a compatibility -- issue between pg15-pg16 -- any_value is created in PG16, we drop -- and create it during upgrade IF upgraded version is less than 16. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; $cmd$; END IF; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; -- if we are upgrading from PG14/PG15 to PG16+, -- we need to regenerate the partkeys because they will include varnullingrels as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_16_upgrade WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; -- Temporarily disable trigger to check for validity of functions while -- inserting. The current contents of the table might be invalid if one of -- the functions was removed by the user without also removing the -- rebalance strategy. Obviously that's not great, but it should be no -- reason to fail the upgrade. ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- set dependencies for columnar table access method PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- PG16 has its own any_value, so only create it pre PG16. -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; RESET citus.enable_ddl_propagation; -- -- Citus creates the any_value aggregate but because of a compatibility -- issue between pg15-pg16 -- any_value is created in PG16, we drop -- and create it during upgrade IF upgraded version is less than 16. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; $cmd$; END IF; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; -- if we are upgrading from PG14/PG15 to PG16+, -- we need to regenerate the partkeys because they will include varnullingrels as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_16_upgrade WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; -- Temporarily disable trigger to check for validity of functions while -- inserting. The current contents of the table might be invalid if one of -- the functions was removed by the user without also removing the -- rebalance strategy. Obviously that's not great, but it should be no -- reason to fail the upgrade. ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- If citus_columnar extension exists, then perform the post PG-upgrade work for columnar as well. -- -- First look if pg_catalog.columnar_finish_pg_upgrade function exists as part of the citus_columnar -- extension. (We check whether it's part of the extension just for security reasons). If it does, then -- call it. If not, then look for columnar_internal.columnar_ensure_am_depends_catalog function and as -- part of the citus_columnar extension. If so, then call it. We alternatively check for the latter UDF -- just because pg_catalog.columnar_finish_pg_upgrade function is introduced in citus_columnar 13.2-1 -- and as of today all it does is to call columnar_internal.columnar_ensure_am_depends_catalog function. IF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if pg_catalog.columnar_finish_pg_upgrade function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'pg_catalog' AND pg_proc.proname = 'columnar_finish_pg_upgrade' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM pg_catalog.columnar_finish_pg_upgrade(); ELSIF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if columnar_internal.columnar_ensure_am_depends_catalog function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'columnar_internal' AND pg_proc.proname = 'columnar_ensure_am_depends_catalog' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); END IF; -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/14.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- PG16 has its own any_value, so only create it pre PG16. -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; RESET citus.enable_ddl_propagation; -- -- Citus creates the any_value aggregate but because of a compatibility -- issue between pg15-pg16 -- any_value is created in PG16, we drop -- and create it during upgrade IF upgraded version is less than 16. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; $cmd$; END IF; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; -- if we are upgrading from PG14/PG15 to PG16+, -- we need to regenerate the partkeys because they will include varnullingrels as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_16_upgrade WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; -- if we are upgrading to PG18+, -- we need to regenerate the partkeys because they will include varreturningtype as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_18_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_18_upgrade WHERE pg_dist_partkeys_pre_18_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_18_upgrade; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; -- Temporarily disable trigger to check for validity of functions while -- inserting. The current contents of the table might be invalid if one of -- the functions was removed by the user without also removing the -- rebalance strategy. Obviously that's not great, but it should be no -- reason to fail the upgrade. ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- If citus_columnar extension exists, then perform the post PG-upgrade work for columnar as well. -- -- First look if pg_catalog.columnar_finish_pg_upgrade function exists as part of the citus_columnar -- extension. (We check whether it's part of the extension just for security reasons). If it does, then -- call it. If not, then look for columnar_internal.columnar_ensure_am_depends_catalog function and as -- part of the citus_columnar extension. If so, then call it. We alternatively check for the latter UDF -- just because pg_catalog.columnar_finish_pg_upgrade function is introduced in citus_columnar 13.2-1 -- and as of today all it does is to call columnar_internal.columnar_ensure_am_depends_catalog function. IF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if pg_catalog.columnar_finish_pg_upgrade function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'pg_catalog' AND pg_proc.proname = 'columnar_finish_pg_upgrade' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM pg_catalog.columnar_finish_pg_upgrade(); ELSIF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if columnar_internal.columnar_ensure_am_depends_catalog function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'columnar_internal' AND pg_proc.proname = 'columnar_ensure_am_depends_catalog' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); END IF; -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers -- DELETE/INSERT to avoid primary key violations WITH old_records AS ( DELETE FROM citus.pg_dist_object RETURNING type, object_names, object_args, distribution_argument_index, colocationid ) INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM old_records naming, pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers -- DELETE/INSERT to avoid primary key violations WITH old_records AS ( DELETE FROM citus.pg_dist_object RETURNING type, object_names, object_args, distribution_argument_index, colocationid ) INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM old_records naming, pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/9.4-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers -- DELETE/INSERT to avoid primary key violations WITH old_records AS ( DELETE FROM citus.pg_dist_object RETURNING type, object_names, object_args, distribution_argument_index, colocationid ) INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM old_records naming, pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/9.5-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- restore pg_dist_object from the stable identifiers TRUNCATE citus.pg_dist_object; INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ DECLARE table_name regclass; command text; trigger_name text; BEGIN IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; ELSE EXECUTE $cmd$ SET citus.enable_ddl_propagation TO OFF; CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; RESET citus.enable_ddl_propagation; $cmd$; END IF; -- -- Citus creates the array_cat_agg but because of a compatibility -- issue between pg13-pg14, we drop and create it during upgrade. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. -- We are not using: -- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg -- because we don't have an easy way to check if the aggregate -- exists with anyarray type or anycompatiblearray type. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; -- PG16 has its own any_value, so only create it pre PG16. -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN EXECUTE $cmd$ -- disable propagation to prevent EnsureCoordinator errors -- the aggregate created here does not depend on Citus extension (yet) -- since we add the dependency with the next command SET citus.enable_ddl_propagation TO OFF; CREATE OR REPLACE FUNCTION pg_catalog.any_value_agg ( anyelement, anyelement ) RETURNS anyelement AS $$ SELECT CASE WHEN $1 IS NULL THEN $2 ELSE $1 END; $$ LANGUAGE SQL STABLE; CREATE AGGREGATE pg_catalog.any_value ( sfunc = pg_catalog.any_value_agg, combinefunc = pg_catalog.any_value_agg, basetype = anyelement, stype = anyelement ); COMMENT ON AGGREGATE pg_catalog.any_value(anyelement) IS 'Returns the value of any row in the group. It is mostly useful when you know there will be only 1 element.'; RESET citus.enable_ddl_propagation; -- -- Citus creates the any_value aggregate but because of a compatibility -- issue between pg15-pg16 -- any_value is created in PG16, we drop -- and create it during upgrade IF upgraded version is less than 16. -- And as Citus creates it, there needs to be a dependency to the -- Citus extension, so we create that dependency here. INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value_agg') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; INSERT INTO pg_depend SELECT 'pg_proc'::regclass::oid as classid, (SELECT oid FROM pg_proc WHERE proname = 'any_value') as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'e' as deptype; $cmd$; END IF; -- -- restore citus catalog tables -- INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; -- if we are upgrading from PG14/PG15 to PG16+, -- we need to regenerate the partkeys because they will include varnullingrels as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_16_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_16_upgrade WHERE pg_dist_partkeys_pre_16_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_16_upgrade; -- if we are upgrading to PG18+, -- we need to regenerate the partkeys because they will include varreturningtype as well. UPDATE pg_catalog.pg_dist_partition SET partkey = column_name_to_column(pg_dist_partkeys_pre_18_upgrade.logicalrelid, col_name) FROM public.pg_dist_partkeys_pre_18_upgrade WHERE pg_dist_partkeys_pre_18_upgrade.logicalrelid = pg_dist_partition.logicalrelid; DROP TABLE public.pg_dist_partkeys_pre_18_upgrade; INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; INSERT INTO pg_catalog.pg_dist_cleanup SELECT * FROM public.pg_dist_cleanup; INSERT INTO pg_catalog.pg_dist_schema SELECT schemaname::regnamespace, colocationid FROM public.pg_dist_schema; -- enterprise catalog tables INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; -- Temporarily disable trigger to check for validity of functions while -- inserting. The current contents of the table might be invalid if one of -- the functions was removed by the user without also removing the -- rebalance strategy. Obviously that's not great, but it should be no -- reason to fail the upgrade. ALTER TABLE pg_catalog.pg_dist_rebalance_strategy DISABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT name, default_strategy, shard_cost_function::regprocedure::regproc, node_capacity_function::regprocedure::regproc, shard_allowed_on_node_function::regprocedure::regproc, default_threshold, minimum_threshold, improvement_threshold FROM public.pg_dist_rebalance_strategy; ALTER TABLE pg_catalog.pg_dist_rebalance_strategy ENABLE TRIGGER pg_dist_rebalance_strategy_validation_trigger; -- -- drop backup tables -- DROP TABLE public.pg_dist_authinfo; DROP TABLE public.pg_dist_colocation; DROP TABLE public.pg_dist_local_group; DROP TABLE public.pg_dist_node; DROP TABLE public.pg_dist_node_metadata; DROP TABLE public.pg_dist_partition; DROP TABLE public.pg_dist_placement; DROP TABLE public.pg_dist_poolinfo; DROP TABLE public.pg_dist_shard; DROP TABLE public.pg_dist_transaction; DROP TABLE public.pg_dist_rebalance_strategy; DROP TABLE public.pg_dist_cleanup; DROP TABLE public.pg_dist_schema; -- -- reset sequences -- PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); PERFORM setval('pg_catalog.pg_dist_operationid_seq', (SELECT MAX(operation_id)+1 AS max_operation_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_cleanup_recordid_seq', (SELECT MAX(record_id)+1 AS max_record_id FROM pg_dist_cleanup), false); PERFORM setval('pg_catalog.pg_dist_clock_logical_seq', (SELECT last_value FROM public.pg_dist_clock_logical_seq), false); DROP TABLE public.pg_dist_clock_logical_seq; -- -- register triggers -- FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition JOIN pg_class ON (logicalrelid = oid) WHERE relkind <> 'f' LOOP trigger_name := 'truncate_trigger_' || table_name::oid; command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; EXECUTE command; command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); EXECUTE command; END LOOP; -- -- set dependencies -- INSERT INTO pg_depend SELECT 'pg_class'::regclass::oid as classid, p.logicalrelid::regclass::oid as objid, 0 as objsubid, 'pg_extension'::regclass::oid as refclassid, (select oid from pg_extension where extname = 'citus') as refobjid, 0 as refobjsubid , 'n' as deptype FROM pg_catalog.pg_dist_partition p; -- If citus_columnar extension exists, then perform the post PG-upgrade work for columnar as well. -- -- First look if pg_catalog.columnar_finish_pg_upgrade function exists as part of the citus_columnar -- extension. (We check whether it's part of the extension just for security reasons). If it does, then -- call it. If not, then look for columnar_internal.columnar_ensure_am_depends_catalog function and as -- part of the citus_columnar extension. If so, then call it. We alternatively check for the latter UDF -- just because pg_catalog.columnar_finish_pg_upgrade function is introduced in citus_columnar 13.2-1 -- and as of today all it does is to call columnar_internal.columnar_ensure_am_depends_catalog function. IF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if pg_catalog.columnar_finish_pg_upgrade function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'pg_catalog' AND pg_proc.proname = 'columnar_finish_pg_upgrade' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM pg_catalog.columnar_finish_pg_upgrade(); ELSIF EXISTS ( SELECT 1 FROM pg_depend JOIN pg_proc ON (pg_depend.objid = pg_proc.oid) JOIN pg_namespace ON (pg_proc.pronamespace = pg_namespace.oid) JOIN pg_extension ON (pg_depend.refobjid = pg_extension.oid) WHERE -- Looking if columnar_internal.columnar_ensure_am_depends_catalog function exists and -- if there is a dependency record from it (proc class = 1255) .. pg_depend.classid = 1255 AND pg_namespace.nspname = 'columnar_internal' AND pg_proc.proname = 'columnar_ensure_am_depends_catalog' AND -- .. to citus_columnar extension (3079 = extension class), if it exists. pg_depend.refclassid = 3079 AND pg_extension.extname = 'citus_columnar' ) THEN PERFORM columnar_internal.columnar_ensure_am_depends_catalog(); END IF; -- restore pg_dist_object from the stable identifiers TRUNCATE pg_catalog.pg_dist_object; INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) SELECT address.classid, address.objid, address.objsubid, naming.distribution_argument_index, naming.colocationid FROM public.pg_dist_object naming, pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; DROP TABLE public.pg_dist_object; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_get_node_clock/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_get_node_clock() RETURNS pg_catalog.cluster_clock LANGUAGE C VOLATILE PARALLEL UNSAFE STRICT AS 'MODULE_PATHNAME',$$citus_get_node_clock$$; COMMENT ON FUNCTION pg_catalog.citus_get_node_clock() IS 'Returns monotonically increasing timestamp with logical clock value as close to epoch value (in milli seconds) as possible, and a counter for ticks(maximum of 4 million) within the logical clock'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_get_node_clock/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_get_node_clock() RETURNS pg_catalog.cluster_clock LANGUAGE C VOLATILE PARALLEL UNSAFE STRICT AS 'MODULE_PATHNAME',$$citus_get_node_clock$$; COMMENT ON FUNCTION pg_catalog.citus_get_node_clock() IS 'Returns monotonically increasing timestamp with logical clock value as close to epoch value (in milli seconds) as possible, and a counter for ticks(maximum of 4 million) within the logical clock'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_get_transaction_clock/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_get_transaction_clock() RETURNS pg_catalog.cluster_clock LANGUAGE C VOLATILE PARALLEL UNSAFE STRICT AS 'MODULE_PATHNAME',$$citus_get_transaction_clock$$; COMMENT ON FUNCTION pg_catalog.citus_get_transaction_clock() IS 'Returns a transaction timestamp logical clock'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_get_transaction_clock/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_get_transaction_clock() RETURNS pg_catalog.cluster_clock LANGUAGE C VOLATILE PARALLEL UNSAFE STRICT AS 'MODULE_PATHNAME',$$citus_get_transaction_clock$$; COMMENT ON FUNCTION pg_catalog.citus_get_transaction_clock() IS 'Returns a transaction timestamp logical clock'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_acquire_citus_advisory_object_class_lock/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.acquire_citus_advisory_object_class_lock(objectClass int, qualifiedObjectName cstring) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_acquire_citus_advisory_object_class_lock$$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_acquire_citus_advisory_object_class_lock/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.acquire_citus_advisory_object_class_lock(objectClass int, qualifiedObjectName cstring) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_acquire_citus_advisory_object_class_lock$$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_colocation_metadata/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_colocation_metadata( colocation_id int, shard_count int, replication_factor int, distribution_column_type regtype, distribution_column_collation oid) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_colocation_metadata(int,int,int,regtype,oid) IS 'Inserts a co-location group into pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_colocation_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_colocation_metadata( colocation_id int, shard_count int, replication_factor int, distribution_column_type regtype, distribution_column_collation oid) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_colocation_metadata$$; COMMENT ON FUNCTION citus_internal.add_colocation_metadata(int,int,int,regtype,oid) IS 'Inserts a co-location group into pg_dist_colocation'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_colocation_metadata( colocation_id int, shard_count int, replication_factor int, distribution_column_type regtype, distribution_column_collation oid) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_colocation_metadata(int,int,int,regtype,oid) IS 'Inserts a co-location group into pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_colocation_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_colocation_metadata( colocation_id int, shard_count int, replication_factor int, distribution_column_type regtype, distribution_column_collation oid) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_colocation_metadata$$; COMMENT ON FUNCTION citus_internal.add_colocation_metadata(int,int,int,regtype,oid) IS 'Inserts a co-location group into pg_dist_colocation'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_colocation_metadata( colocation_id int, shard_count int, replication_factor int, distribution_column_type regtype, distribution_column_collation oid) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_colocation_metadata(int,int,int,regtype,oid) IS 'Inserts a co-location group into pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_object_metadata/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_object_metadata( typeText text, objNames text[], objArgs text[], distribution_argument_index int, colocationid int, force_delegation bool) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_object_metadata(text,text[],text[],int,int,bool) IS 'Inserts distributed object into pg_dist_object'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_object_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_object_metadata( typeText text, objNames text[], objArgs text[], distribution_argument_index int, colocationid int, force_delegation bool) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_object_metadata$$; COMMENT ON FUNCTION citus_internal.add_object_metadata(text,text[],text[],int,int,bool) IS 'Inserts distributed object into pg_dist_object'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_object_metadata( typeText text, objNames text[], objArgs text[], distribution_argument_index int, colocationid int, force_delegation bool) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_object_metadata(text,text[],text[],int,int,bool) IS 'Inserts distributed object into pg_dist_object'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_object_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_object_metadata( typeText text, objNames text[], objArgs text[], distribution_argument_index int, colocationid int, force_delegation bool) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_object_metadata$$; COMMENT ON FUNCTION citus_internal.add_object_metadata(text,text[],text[],int,int,bool) IS 'Inserts distributed object into pg_dist_object'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_object_metadata( typeText text, objNames text[], objArgs text[], distribution_argument_index int, colocationid int, force_delegation bool) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_object_metadata(text,text[],text[],int,int,bool) IS 'Inserts distributed object into pg_dist_object'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_partition_metadata/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_partition_metadata( relation_id regclass, distribution_method "char", distribution_column text, colocation_id integer, replication_model "char") RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_partition_metadata(regclass, "char", text, integer, "char") IS 'Inserts into pg_dist_partition with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_partition_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_partition_metadata( relation_id regclass, distribution_method "char", distribution_column text, colocation_id integer, replication_model "char") RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_internal_add_partition_metadata$$; COMMENT ON FUNCTION citus_internal.add_partition_metadata(regclass, "char", text, integer, "char") IS 'Inserts into pg_dist_partition with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_partition_metadata( relation_id regclass, distribution_method "char", distribution_column text, colocation_id integer, replication_model "char") RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_partition_metadata(regclass, "char", text, integer, "char") IS 'Inserts into pg_dist_partition with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_partition_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_partition_metadata( relation_id regclass, distribution_method "char", distribution_column text, colocation_id integer, replication_model "char") RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_internal_add_partition_metadata$$; COMMENT ON FUNCTION citus_internal.add_partition_metadata(regclass, "char", text, integer, "char") IS 'Inserts into pg_dist_partition with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_partition_metadata( relation_id regclass, distribution_method "char", distribution_column text, colocation_id integer, replication_model "char") RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_partition_metadata(regclass, "char", text, integer, "char") IS 'Inserts into pg_dist_partition with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_placement_metadata/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_state integer, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, integer, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_placement_metadata/11.2-1.sql ================================================ -- create a new function, without shardstate CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; -- replace the old one so it would call the old C function with shard_state CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_state integer, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata_legacy$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, integer, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_placement_metadata/13.1-1.sql ================================================ -- create a new function, without shardstate CREATE OR REPLACE FUNCTION citus_internal.add_placement_metadata( shard_id bigint, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata$$; COMMENT ON FUNCTION citus_internal.add_placement_metadata(bigint, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; -- create a new function, without shardstate CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; -- replace the old one so it would call the old C function with shard_state CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_state integer, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata_legacy$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, integer, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_placement_metadata/latest.sql ================================================ -- create a new function, without shardstate CREATE OR REPLACE FUNCTION citus_internal.add_placement_metadata( shard_id bigint, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata$$; COMMENT ON FUNCTION citus_internal.add_placement_metadata(bigint, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; -- create a new function, without shardstate CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; -- replace the old one so it would call the old C function with shard_state CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_placement_metadata( shard_id bigint, shard_state integer, shard_length bigint, group_id integer, placement_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_add_placement_metadata_legacy$$; COMMENT ON FUNCTION pg_catalog.citus_internal_add_placement_metadata(bigint, integer, bigint, integer, bigint) IS 'Inserts into pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_shard_metadata/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_shard_metadata( relation_id regclass, shard_id bigint, storage_type "char", shard_min_value text, shard_max_value text ) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_shard_metadata(regclass, bigint, "char", text, text) IS 'Inserts into pg_dist_shard with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_shard_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_shard_metadata( relation_id regclass, shard_id bigint, storage_type "char", shard_min_value text, shard_max_value text ) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_internal_add_shard_metadata$$; COMMENT ON FUNCTION citus_internal.add_shard_metadata(regclass, bigint, "char", text, text) IS 'Inserts into pg_dist_shard with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_shard_metadata( relation_id regclass, shard_id bigint, storage_type "char", shard_min_value text, shard_max_value text ) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_shard_metadata(regclass, bigint, "char", text, text) IS 'Inserts into pg_dist_shard with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_shard_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_shard_metadata( relation_id regclass, shard_id bigint, storage_type "char", shard_min_value text, shard_max_value text ) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$citus_internal_add_shard_metadata$$; COMMENT ON FUNCTION citus_internal.add_shard_metadata(regclass, bigint, "char", text, text) IS 'Inserts into pg_dist_shard with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_shard_metadata( relation_id regclass, shard_id bigint, storage_type "char", shard_min_value text, shard_max_value text ) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_shard_metadata(regclass, bigint, "char", text, text) IS 'Inserts into pg_dist_shard with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_tenant_schema/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_tenant_schema(schema_id Oid, colocation_id int) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_tenant_schema(Oid, int) IS 'insert given tenant schema into pg_dist_schema with given colocation id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_tenant_schema/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_tenant_schema(schema_id Oid, colocation_id int) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_add_tenant_schema$$; COMMENT ON FUNCTION citus_internal.add_tenant_schema(Oid, int) IS 'insert given tenant schema into pg_dist_schema with given colocation id'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_tenant_schema(schema_id Oid, colocation_id int) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_tenant_schema(Oid, int) IS 'insert given tenant schema into pg_dist_schema with given colocation id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_add_tenant_schema/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.add_tenant_schema(schema_id Oid, colocation_id int) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_add_tenant_schema$$; COMMENT ON FUNCTION citus_internal.add_tenant_schema(Oid, int) IS 'insert given tenant schema into pg_dist_schema with given colocation id'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_add_tenant_schema(schema_id Oid, colocation_id int) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_add_tenant_schema(Oid, int) IS 'insert given tenant schema into pg_dist_schema with given colocation id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_adjust_local_clock_to_remote/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) RETURNS void LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_internal_adjust_local_clock_to_remote$$; COMMENT ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) IS 'Internal UDF used to adjust the local clock to the maximum of nodes in the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_adjust_local_clock_to_remote/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) RETURNS void LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_internal_adjust_local_clock_to_remote$$; COMMENT ON FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) IS 'Internal UDF used to adjust the local clock to the maximum of nodes in the cluster'; REVOKE ALL ON FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) RETURNS void LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_internal_adjust_local_clock_to_remote$$; COMMENT ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) IS 'Internal UDF used to adjust the local clock to the maximum of nodes in the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_adjust_local_clock_to_remote/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) RETURNS void LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_internal_adjust_local_clock_to_remote$$; COMMENT ON FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) IS 'Internal UDF used to adjust the local clock to the maximum of nodes in the cluster'; REVOKE ALL ON FUNCTION citus_internal.adjust_local_clock_to_remote(pg_catalog.cluster_clock) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) RETURNS void LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME', $$citus_internal_adjust_local_clock_to_remote$$; COMMENT ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) IS 'Internal UDF used to adjust the local clock to the maximum of nodes in the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_internal_adjust_local_clock_to_remote(pg_catalog.cluster_clock) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_copy_single_shard_placement/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.citus_internal_copy_single_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, flags integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_copy_single_shard_placement$$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_copy_single_shard_placement/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.citus_internal_copy_single_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, flags integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_copy_single_shard_placement$$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_database_command/13.1-1.sql ================================================ -- -- citus_internal.database_command run given database command without transaction block restriction. CREATE OR REPLACE FUNCTION citus_internal.database_command(command text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION citus_internal.database_command(text) IS 'run a database command without transaction block restrictions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql ================================================ -- -- citus_internal.database_command run given database command without transaction block restriction. CREATE OR REPLACE FUNCTION citus_internal.database_command(command text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION citus_internal.database_command(text) IS 'run a database command without transaction block restrictions'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_colocation_metadata/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_colocation_metadata( colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_colocation_metadata(int) IS 'deletes a co-location group from pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_colocation_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_colocation_metadata( colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_colocation_metadata$$; COMMENT ON FUNCTION citus_internal.delete_colocation_metadata(int) IS 'deletes a co-location group from pg_dist_colocation'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_colocation_metadata( colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_colocation_metadata(int) IS 'deletes a co-location group from pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_colocation_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_colocation_metadata( colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_colocation_metadata$$; COMMENT ON FUNCTION citus_internal.delete_colocation_metadata(int) IS 'deletes a co-location group from pg_dist_colocation'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_colocation_metadata( colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_colocation_metadata(int) IS 'deletes a co-location group from pg_dist_colocation'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_partition_metadata/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_partition_metadata(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_partition_metadata(regclass) IS 'Deletes a row from pg_dist_partition with table ownership checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_partition_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_partition_metadata(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_partition_metadata$$; COMMENT ON FUNCTION citus_internal.delete_partition_metadata(regclass) IS 'Deletes a row from pg_dist_partition with table ownership checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_partition_metadata(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_partition_metadata(regclass) IS 'Deletes a row from pg_dist_partition with table ownership checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_partition_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_partition_metadata(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_partition_metadata$$; COMMENT ON FUNCTION citus_internal.delete_partition_metadata(regclass) IS 'Deletes a row from pg_dist_partition with table ownership checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_partition_metadata(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_partition_metadata(regclass) IS 'Deletes a row from pg_dist_partition with table ownership checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_placement_metadata/12.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_placement_metadata( placement_id bigint) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_placement_metadata(bigint) IS 'Delete placement with given id from pg_dist_placement metadata table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_placement_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_placement_metadata( placement_id bigint) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_placement_metadata$$; COMMENT ON FUNCTION citus_internal.delete_placement_metadata(bigint) IS 'Delete placement with given id from pg_dist_placement metadata table.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_placement_metadata( placement_id bigint) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_placement_metadata(bigint) IS 'Delete placement with given id from pg_dist_placement metadata table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_placement_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_placement_metadata( placement_id bigint) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_placement_metadata$$; COMMENT ON FUNCTION citus_internal.delete_placement_metadata(bigint) IS 'Delete placement with given id from pg_dist_placement metadata table.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_placement_metadata( placement_id bigint) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_placement_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_placement_metadata(bigint) IS 'Delete placement with given id from pg_dist_placement metadata table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_shard_metadata/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_shard_metadata(shard_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_shard_metadata(bigint) IS 'Deletes rows from pg_dist_shard and pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_shard_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_shard_metadata(shard_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_shard_metadata$$; COMMENT ON FUNCTION citus_internal.delete_shard_metadata(bigint) IS 'Deletes rows from pg_dist_shard and pg_dist_shard_placement with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_shard_metadata(shard_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_shard_metadata(bigint) IS 'Deletes rows from pg_dist_shard and pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_shard_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_shard_metadata(shard_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_delete_shard_metadata$$; COMMENT ON FUNCTION citus_internal.delete_shard_metadata(bigint) IS 'Deletes rows from pg_dist_shard and pg_dist_shard_placement with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_shard_metadata(shard_id bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_shard_metadata(bigint) IS 'Deletes rows from pg_dist_shard and pg_dist_shard_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_tenant_schema/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_tenant_schema(schema_id Oid) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_tenant_schema(Oid) IS 'delete given tenant schema from pg_dist_schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_tenant_schema/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_tenant_schema(schema_id Oid) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_tenant_schema$$; COMMENT ON FUNCTION citus_internal.delete_tenant_schema(Oid) IS 'delete given tenant schema from pg_dist_schema'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_tenant_schema(schema_id Oid) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_tenant_schema(Oid) IS 'delete given tenant schema from pg_dist_schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_delete_tenant_schema/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.delete_tenant_schema(schema_id Oid) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_delete_tenant_schema$$; COMMENT ON FUNCTION citus_internal.delete_tenant_schema(Oid) IS 'delete given tenant schema from pg_dist_schema'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_delete_tenant_schema(schema_id Oid) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_delete_tenant_schema(Oid) IS 'delete given tenant schema from pg_dist_schema'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_global_blocked_processes/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_global_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_global_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_global_blocked_processes() IS 'returns a global list of blocked backends originating from this node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_global_blocked_processes/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.global_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_global_blocked_processes$$; COMMENT ON FUNCTION citus_internal.global_blocked_processes() IS 'returns a global list of blocked backends originating from this node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_global_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_global_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_global_blocked_processes() IS 'returns a global list of blocked backends originating from this node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_global_blocked_processes/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.global_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_global_blocked_processes$$; COMMENT ON FUNCTION citus_internal.global_blocked_processes() IS 'returns a global list of blocked backends originating from this node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_global_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_global_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_global_blocked_processes() IS 'returns a global list of blocked backends originating from this node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_local_blocked_processes/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_local_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_local_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_local_blocked_processes() IS 'returns all local lock wait chains, that start from any citus backend'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_local_blocked_processes/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.local_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_local_blocked_processes$$; COMMENT ON FUNCTION citus_internal.local_blocked_processes() IS 'returns all local lock wait chains, that start from any citus backend'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_local_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_local_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_local_blocked_processes() IS 'returns all local lock wait chains, that start from any citus backend'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_local_blocked_processes/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.local_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_local_blocked_processes$$; COMMENT ON FUNCTION citus_internal.local_blocked_processes() IS 'returns all local lock wait chains, that start from any citus backend'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_local_blocked_processes( OUT waiting_global_pid int8, OUT waiting_pid int4, OUT waiting_node_id int4, OUT waiting_transaction_num int8, OUT waiting_transaction_stamp timestamptz, OUT blocking_global_pid int8, OUT blocking_pid int4, OUT blocking_node_id int4, OUT blocking_transaction_num int8, OUT blocking_transaction_stamp timestamptz, OUT blocking_transaction_waiting bool) RETURNS SETOF RECORD LANGUAGE C STRICT AS $$MODULE_PATHNAME$$, $$citus_internal_local_blocked_processes$$; COMMENT ON FUNCTION pg_catalog.citus_internal_local_blocked_processes() IS 'returns all local lock wait chains, that start from any citus backend'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_mark_node_not_synced/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_mark_node_not_synced(parent_pid int, nodeid int) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_mark_node_not_synced$$; COMMENT ON FUNCTION citus_internal_mark_node_not_synced(int, int) IS 'marks given node not synced by unsetting metadatasynced column at the start of the nontransactional sync.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_mark_node_not_synced/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.mark_node_not_synced(parent_pid int, nodeid int) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_mark_node_not_synced$$; COMMENT ON FUNCTION citus_internal.mark_node_not_synced(int, int) IS 'marks given node not synced by unsetting metadatasynced column at the start of the nontransactional sync.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_mark_node_not_synced(parent_pid int, nodeid int) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_mark_node_not_synced$$; COMMENT ON FUNCTION citus_internal_mark_node_not_synced(int, int) IS 'marks given node not synced by unsetting metadatasynced column at the start of the nontransactional sync.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_mark_node_not_synced/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.mark_node_not_synced(parent_pid int, nodeid int) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_mark_node_not_synced$$; COMMENT ON FUNCTION citus_internal.mark_node_not_synced(int, int) IS 'marks given node not synced by unsetting metadatasynced column at the start of the nontransactional sync.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_mark_node_not_synced(parent_pid int, nodeid int) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_mark_node_not_synced$$; COMMENT ON FUNCTION citus_internal_mark_node_not_synced(int, int) IS 'marks given node not synced by unsetting metadatasynced column at the start of the nontransactional sync.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_unregister_tenant_schema_globally/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) IS 'Delete a tenant schema and the corresponding colocation group from metadata tables.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_unregister_tenant_schema_globally/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.unregister_tenant_schema_globally(schema_id Oid, schema_name text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_unregister_tenant_schema_globally$$; COMMENT ON FUNCTION citus_internal.unregister_tenant_schema_globally(schema_id Oid, schema_name text) IS 'Delete a tenant schema and the corresponding colocation group from metadata tables.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) IS 'Delete a tenant schema and the corresponding colocation group from metadata tables.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_unregister_tenant_schema_globally/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.unregister_tenant_schema_globally(schema_id Oid, schema_name text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_unregister_tenant_schema_globally$$; COMMENT ON FUNCTION citus_internal.unregister_tenant_schema_globally(schema_id Oid, schema_name text) IS 'Delete a tenant schema and the corresponding colocation group from metadata tables.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_unregister_tenant_schema_globally(schema_id Oid, schema_name text) IS 'Delete a tenant schema and the corresponding colocation group from metadata tables.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_none_dist_table_metadata/12.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata(oid, "char", bigint, boolean) IS 'Update pg_dist_partition metadata table for given none-distributed table, to convert it to another type of none-distributed table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_none_dist_table_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_update_none_dist_table_metadata$$; COMMENT ON FUNCTION citus_internal.update_none_dist_table_metadata(oid, "char", bigint, boolean) IS 'Update pg_dist_partition metadata table for given none-distributed table, to convert it to another type of none-distributed table.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata(oid, "char", bigint, boolean) IS 'Update pg_dist_partition metadata table for given none-distributed table, to convert it to another type of none-distributed table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_none_dist_table_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_update_none_dist_table_metadata$$; COMMENT ON FUNCTION citus_internal.update_none_dist_table_metadata(oid, "char", bigint, boolean) IS 'Update pg_dist_partition metadata table for given none-distributed table, to convert it to another type of none-distributed table.'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata( relation_id oid, replication_model "char", colocation_id bigint, auto_converted boolean) RETURNS void LANGUAGE C VOLATILE AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_none_dist_table_metadata(oid, "char", bigint, boolean) IS 'Update pg_dist_partition metadata table for given none-distributed table, to convert it to another type of none-distributed table.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_placement_metadata/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_placement_metadata( shard_id bigint, source_group_id integer, target_group_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_placement_metadata(bigint, integer, integer) IS 'Updates into pg_dist_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_placement_metadata/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_placement_metadata( shard_id bigint, source_group_id integer, target_group_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_update_placement_metadata$$; COMMENT ON FUNCTION citus_internal.update_placement_metadata(bigint, integer, integer) IS 'Updates into pg_dist_placement with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_placement_metadata( shard_id bigint, source_group_id integer, target_group_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_placement_metadata(bigint, integer, integer) IS 'Updates into pg_dist_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_placement_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_placement_metadata( shard_id bigint, source_group_id integer, target_group_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_update_placement_metadata$$; COMMENT ON FUNCTION citus_internal.update_placement_metadata(bigint, integer, integer) IS 'Updates into pg_dist_placement with user checks'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_placement_metadata( shard_id bigint, source_group_id integer, target_group_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_placement_metadata(bigint, integer, integer) IS 'Updates into pg_dist_placement with user checks'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_relation_colocation/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_relation_colocation(relation_id Oid, target_colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_relation_colocation(oid, int) IS 'Updates colocationId field of pg_dist_partition for the relation_id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_relation_colocation/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_relation_colocation(relation_id Oid, target_colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_update_relation_colocation$$; COMMENT ON FUNCTION citus_internal.update_relation_colocation(oid, int) IS 'Updates colocationId field of pg_dist_partition for the relation_id'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_relation_colocation(relation_id Oid, target_colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_relation_colocation(oid, int) IS 'Updates colocationId field of pg_dist_partition for the relation_id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_internal_update_relation_colocation/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.update_relation_colocation(relation_id Oid, target_colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_update_relation_colocation$$; COMMENT ON FUNCTION citus_internal.update_relation_colocation(oid, int) IS 'Updates colocationId field of pg_dist_partition for the relation_id'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_update_relation_colocation(relation_id Oid, target_colocation_id int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME'; COMMENT ON FUNCTION pg_catalog.citus_internal_update_relation_colocation(oid, int) IS 'Updates colocationId field of pg_dist_partition for the relation_id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_clock_after/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_is_clock_after(clock_one pg_catalog.cluster_clock, clock_two pg_catalog.cluster_clock) RETURNS BOOL LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME',$$citus_is_clock_after$$; COMMENT ON FUNCTION pg_catalog.citus_is_clock_after(pg_catalog.cluster_clock, pg_catalog.cluster_clock) IS 'Accepts logical clock timestamps of two causally related events and returns true if the argument1 happened before argument2'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_clock_after/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_is_clock_after(clock_one pg_catalog.cluster_clock, clock_two pg_catalog.cluster_clock) RETURNS BOOL LANGUAGE C STABLE PARALLEL SAFE STRICT AS 'MODULE_PATHNAME',$$citus_is_clock_after$$; COMMENT ON FUNCTION pg_catalog.citus_is_clock_after(pg_catalog.cluster_clock, pg_catalog.cluster_clock) IS 'Accepts logical clock timestamps of two causally related events and returns true if the argument1 happened before argument2'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_coordinator/11.0-2.sql ================================================ CREATE FUNCTION pg_catalog.citus_is_coordinator() RETURNS bool LANGUAGE c STRICT AS 'MODULE_PATHNAME', $$citus_is_coordinator$$; COMMENT ON FUNCTION pg_catalog.citus_is_coordinator() IS 'returns whether the current node is a coordinator'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_coordinator/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_is_coordinator() RETURNS bool LANGUAGE c STRICT AS 'MODULE_PATHNAME', $$citus_is_coordinator$$; COMMENT ON FUNCTION pg_catalog.citus_is_coordinator() IS 'returns whether the current node is a coordinator'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_primary_node/13.1-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_is_primary_node() RETURNS bool LANGUAGE c STRICT AS 'MODULE_PATHNAME', $$citus_is_primary_node$$; COMMENT ON FUNCTION pg_catalog.citus_is_primary_node() IS 'returns whether the current node is the primary node in the group'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_is_primary_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_is_primary_node() RETURNS bool LANGUAGE c STRICT AS 'MODULE_PATHNAME', $$citus_is_primary_node$$; COMMENT ON FUNCTION pg_catalog.citus_is_primary_node() IS 'returns whether the current node is the primary node in the group'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedGlobalPid int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT 1 FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT global_pid INTO mBlockedGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; RETURN EXISTS ( SELECT 1 FROM citus_internal_global_blocked_processes() WHERE waiting_global_pid = mBlockedGlobalPid ) OR EXISTS ( -- Check on the workers if any logical replication job spawned by the -- current PID is blocked, by checking it's application name -- Query is heavily based on: https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT result FROM run_command_on_workers($two$ SELECT blocked_activity.application_name AS blocked_application FROM pg_catalog.pg_locks blocked_locks JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid JOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid AND blocking_locks.pid != blocked_locks.pid JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid WHERE NOT blocked_locks.GRANTED AND blocked_activity.application_name LIKE 'citus_shard_move_subscription_%' $two$) where result LIKE 'citus_shard_move_subscription_%_' || pBlockedPid); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedGlobalPid int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT 1 FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT global_pid INTO mBlockedGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; RETURN EXISTS ( SELECT 1 FROM citus_internal_global_blocked_processes() WHERE waiting_global_pid = mBlockedGlobalPid ); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedGlobalPid int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT 1 FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT global_pid INTO mBlockedGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; -- We convert the blocking_global_pid to a regular pid and only look at -- blocks caused by the interesting pids, or the workerProcessPid. If we -- don't do that we might find unrelated blocks caused by some random -- other processes that are not involved in this isolation test. Because we -- run our isolation tests on a single physical machine, the PID part of -- the GPID is known to be unique within the whole cluster. RETURN EXISTS ( SELECT 1 FROM citus_internal_global_blocked_processes() WHERE waiting_global_pid = mBlockedGlobalPid AND ( citus_pid_for_gpid(blocking_global_pid) in ( select * from unnest(pInterestingPids) ) OR citus_pid_for_gpid(blocking_global_pid) = workerProcessId ) ); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedGlobalPid int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT 1 FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT global_pid INTO mBlockedGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; -- We convert the blocking_global_pid to a regular pid and only look at -- blocks caused by the interesting pids, or the workerProcessPid. If we -- don't do that we might find unrelated blocks caused by some random -- other processes that are not involved in this isolation test. Because we -- run our isolation tests on a single physical machine, the PID part of -- the GPID is known to be unique within the whole cluster. RETURN EXISTS ( SELECT 1 FROM citus_internal.global_blocked_processes() WHERE waiting_global_pid = mBlockedGlobalPid AND ( citus_pid_for_gpid(blocking_global_pid) in ( select * from unnest(pInterestingPids) ) OR citus_pid_for_gpid(blocking_global_pid) = workerProcessId ) ); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/8.0-6.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedTransactionNum int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT transaction_number FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT transaction_number INTO mBlockedTransactionNum FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT transaction_number INTO mBlockedTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; RETURN EXISTS ( SELECT 1 FROM dump_global_wait_edges() WHERE waiting_transaction_num = mBlockedTransactionNum ); END; $$ LANGUAGE plpgsql; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedTransactionNum int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT transaction_number FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT transaction_number INTO mBlockedTransactionNum FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT transaction_number INTO mBlockedTransactionNum FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; RETURN EXISTS ( SELECT 1 FROM dump_global_wait_edges() WHERE waiting_transaction_num = mBlockedTransactionNum ) OR EXISTS ( -- Check on the workers if any logical replication job spawned by the -- current PID is blocked, by checking it's application name -- Query is heavily based on: https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT result FROM run_command_on_workers($two$ SELECT blocked_activity.application_name AS blocked_application FROM pg_catalog.pg_locks blocked_locks JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid JOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid AND blocking_locks.pid != blocked_locks.pid JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid WHERE NOT blocked_locks.GRANTED AND blocked_activity.application_name LIKE 'citus_shard_move_subscription_%' $two$) where result LIKE 'citus_shard_move_subscription_%_' || pBlockedPid); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[]) RETURNS boolean AS $$ DECLARE mBlockedGlobalPid int8; workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id'); coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id'); BEGIN IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN RETURN true; END IF; -- pg says we're not blocked locally; check whether we're blocked globally. -- Note that worker process may be blocked or waiting for a lock. So we need to -- get transaction number for both of them. Following IF provides the transaction -- number when the worker process waiting for other session. IF EXISTS (SELECT 1 FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions() WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId; ELSE -- Check whether transactions initiated from the coordinator get locked SELECT global_pid INTO mBlockedGlobalPid FROM get_all_active_transactions() WHERE process_id = pBlockedPid; END IF; -- We convert the blocking_global_pid to a regular pid and only look at -- blocks caused by the interesting pids, or the workerProcessPid. If we -- don't do that we might find unrelated blocks caused by some random -- other processes that are not involved in this isolation test. Because we -- run our isolation tests on a single physical machine, the PID part of -- the GPID is known to be unique within the whole cluster. RETURN EXISTS ( SELECT 1 FROM citus_internal.global_blocked_processes() WHERE waiting_global_pid = mBlockedGlobalPid AND ( citus_pid_for_gpid(blocking_global_pid) in ( select * from unnest(pInterestingPids) ) OR citus_pid_for_gpid(blocking_global_pid) = workerProcessId ) ); END; $$ LANGUAGE plpgsql; REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_cancel/11.1-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_job_cancel(jobid bigint) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_job_cancel$$; COMMENT ON FUNCTION pg_catalog.citus_job_cancel(jobid bigint) IS 'cancel a scheduled or running job and all of its tasks that didn''t finish yet'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_job_cancel(jobid bigint) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_cancel/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_job_cancel(jobid bigint) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$citus_job_cancel$$; COMMENT ON FUNCTION pg_catalog.citus_job_cancel(jobid bigint) IS 'cancel a scheduled or running job and all of its tasks that didn''t finish yet'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_job_cancel(jobid bigint) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_list/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_list () RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz ) LANGUAGE SQL AS $fn$ SELECT job_id, state, job_type, description, started_at, finished_at FROM pg_dist_background_job ORDER BY job_id $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_list/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_list () RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz ) LANGUAGE SQL AS $fn$ SELECT job_id, state, job_type, description, started_at, finished_at FROM pg_dist_background_job ORDER BY job_id $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_status/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( job_id bigint, raw boolean DEFAULT FALSE ) RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz, details jsonb ) LANGUAGE SQL STRICT AS $fn$ WITH rp AS MATERIALIZED ( SELECT sessionid, sum(source_shard_size) as source_shard_size, sum(target_shard_size) as target_shard_size, any_value(status) as status, any_value(sourcename) as sourcename, any_value(sourceport) as sourceport, any_value(targetname) as targetname, any_value(targetport) as targetport, max(source_lsn) as source_lsn, min(target_lsn) as target_lsn FROM get_rebalance_progress() GROUP BY sessionid ), task_state_occurence_counts AS ( SELECT t.status, count(task_id) FROM pg_dist_background_job j JOIN pg_dist_background_task t ON t.job_id = j.job_id WHERE j.job_id = $1 GROUP BY t.status ), running_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), 'phase', rp.status, 'size' , jsonb_build_object( 'source', rp.source_shard_size, 'target', rp.target_shard_size), 'hosts', jsonb_build_object( 'source', rp.sourcename || ':' || rp.sourceport, 'target', rp.targetname || ':' || rp.targetport), 'message', t.message, 'command', t.command, 'task_id', t.task_id ) || CASE WHEN ($2) THEN jsonb_build_object( 'size', jsonb_build_object( 'source', rp.source_shard_size, 'target', rp.target_shard_size), 'LSN', jsonb_build_object( 'source', rp.source_lsn, 'target', rp.target_lsn, 'lag', rp.source_lsn - rp.target_lsn)) ELSE jsonb_build_object( 'size', jsonb_build_object( 'source', pg_size_pretty(rp.source_shard_size), 'target', pg_size_pretty(rp.target_shard_size)), 'LSN', jsonb_build_object( 'source', rp.source_lsn, 'target', rp.target_lsn, 'lag', pg_size_pretty(rp.source_lsn - rp.target_lsn))) END) AS tasks FROM rp JOIN pg_dist_background_task t ON rp.sessionid = t.pid JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND t.status = 'running' ), errored_or_retried_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), 'message', t.message, 'command', t.command, 'task_id', t.task_id )) AS tasks FROM pg_dist_background_task t JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND NOT EXISTS (SELECT 1 FROM rp WHERE rp.sessionid = t.pid) AND (t.status = 'error' OR (t.status = 'runnable' AND t.retry_count > 0)) ) SELECT job_id, state, job_type, description, started_at, finished_at, jsonb_build_object( 'task_state_counts', (SELECT jsonb_object_agg(status, count) FROM task_state_occurence_counts), 'tasks', (COALESCE((SELECT tasks FROM running_task_details),'[]'::jsonb) || COALESCE((SELECT tasks FROM errored_or_retried_task_details),'[]'::jsonb))) AS details FROM pg_dist_background_job j WHERE j.job_id = $1 $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_status/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_job_status ( job_id bigint, raw boolean DEFAULT FALSE ) RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz, details jsonb ) LANGUAGE SQL STRICT AS $fn$ WITH rp AS MATERIALIZED ( SELECT sessionid, sum(source_shard_size) as source_shard_size, sum(target_shard_size) as target_shard_size, any_value(status) as status, any_value(sourcename) as sourcename, any_value(sourceport) as sourceport, any_value(targetname) as targetname, any_value(targetport) as targetport, max(source_lsn) as source_lsn, min(target_lsn) as target_lsn FROM get_rebalance_progress() GROUP BY sessionid ), task_state_occurence_counts AS ( SELECT t.status, count(task_id) FROM pg_dist_background_job j JOIN pg_dist_background_task t ON t.job_id = j.job_id WHERE j.job_id = $1 GROUP BY t.status ), running_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), 'phase', rp.status, 'size' , jsonb_build_object( 'source', rp.source_shard_size, 'target', rp.target_shard_size), 'hosts', jsonb_build_object( 'source', rp.sourcename || ':' || rp.sourceport, 'target', rp.targetname || ':' || rp.targetport), 'message', t.message, 'command', t.command, 'task_id', t.task_id ) || CASE WHEN ($2) THEN jsonb_build_object( 'size', jsonb_build_object( 'source', rp.source_shard_size, 'target', rp.target_shard_size), 'LSN', jsonb_build_object( 'source', rp.source_lsn, 'target', rp.target_lsn, 'lag', rp.source_lsn - rp.target_lsn)) ELSE jsonb_build_object( 'size', jsonb_build_object( 'source', pg_size_pretty(rp.source_shard_size), 'target', pg_size_pretty(rp.target_shard_size)), 'LSN', jsonb_build_object( 'source', rp.source_lsn, 'target', rp.target_lsn, 'lag', pg_size_pretty(rp.source_lsn - rp.target_lsn))) END) AS tasks FROM rp JOIN pg_dist_background_task t ON rp.sessionid = t.pid JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND t.status = 'running' ), errored_or_retried_task_details AS ( SELECT jsonb_agg(jsonb_build_object( 'state', t.status, 'retried', coalesce(t.retry_count,0), 'message', t.message, 'command', t.command, 'task_id', t.task_id )) AS tasks FROM pg_dist_background_task t JOIN pg_dist_background_job j ON t.job_id = j.job_id WHERE j.job_id = $1 AND NOT EXISTS (SELECT 1 FROM rp WHERE rp.sessionid = t.pid) AND (t.status = 'error' OR (t.status = 'runnable' AND t.retry_count > 0)) ) SELECT job_id, state, job_type, description, started_at, finished_at, jsonb_build_object( 'task_state_counts', (SELECT jsonb_object_agg(status, count) FROM task_state_occurence_counts), 'tasks', (COALESCE((SELECT tasks FROM running_task_details),'[]'::jsonb) || COALESCE((SELECT tasks FROM errored_or_retried_task_details),'[]'::jsonb))) AS details FROM pg_dist_background_job j WHERE j.job_id = $1 $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_wait/11.1-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME',$$citus_job_wait$$; COMMENT ON FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status) IS 'blocks till the job identified by jobid is at the specified status, or reached a terminal status. Only waits for terminal status when no desired_status was specified. The return value indicates if the desired status was reached or not. When no desired status was specified it will assume any terminal status was desired'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_job_wait/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME',$$citus_job_wait$$; COMMENT ON FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status) IS 'blocks till the job identified by jobid is at the specified status, or reached a terminal status. Only waits for terminal status when no desired_status was specified. The return value indicates if the desired status was reached or not. When no desired status was specified it will assume any terminal status was desired'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_job_wait(jobid bigint, desired_status pg_catalog.citus_job_status) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_local_disk_space_stats/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_local_disk_space_stats( OUT available_disk_size bigint, OUT total_disk_size bigint) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_local_disk_space_stats$$; COMMENT ON FUNCTION pg_catalog.citus_local_disk_space_stats() IS 'returns statistics on available disk space on the local node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_local_disk_space_stats/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_local_disk_space_stats( OUT available_disk_size bigint, OUT total_disk_size bigint) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_local_disk_space_stats$$; COMMENT ON FUNCTION pg_catalog.citus_local_disk_space_stats() IS 'returns statistics on available disk space on the local node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_lock_waits/11.0-1.sql ================================================ SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH unique_global_wait_edges_with_calculated_gpids AS ( SELECT -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL -- also for legacy reasons citus_internal_global_blocked_processes() returns groupId, we replace that with nodeIds case WHEN waiting_global_pid !=0 THEN waiting_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(waiting_node_id), waiting_pid) END waiting_global_pid, case WHEN blocking_global_pid !=0 THEN blocking_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(blocking_node_id), blocking_pid) END blocking_global_pid, -- citus_internal_global_blocked_processes returns groupId, we replace it here with actual -- nodeId to be consisten with the other views get_nodeid_for_groupid(blocking_node_id) as blocking_node_id, get_nodeid_for_groupid(waiting_node_id) as waiting_node_id, blocking_transaction_waiting FROM citus_internal_global_blocked_processes() ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_global_pid, blocking_global_pid) * FROM unique_global_wait_edges_with_calculated_gpids ), citus_dist_stat_activity_with_calculated_gpids AS ( -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL SELECT CASE WHEN global_pid != 0 THEN global_pid ELSE citus_calculate_gpid(nodeid, pid) END global_pid, nodeid, pid, query FROM citus_dist_stat_activity ) SELECT waiting.global_pid as waiting_gpid, blocking.global_pid as blocking_gpid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.nodeid AS waiting_nodeid, blocking.nodeid AS blocking_nodeid FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_calculated_gpids waiting ON (unique_global_wait_edges.waiting_global_pid = waiting.global_pid) JOIN citus_dist_stat_activity_with_calculated_gpids blocking ON (unique_global_wait_edges.blocking_global_pid = blocking.global_pid); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_lock_waits/13.1-1.sql ================================================ SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH unique_global_wait_edges_with_calculated_gpids AS ( SELECT -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL -- also for legacy reasons citus_internal.global_blocked_processes() returns groupId, we replace that with nodeIds case WHEN waiting_global_pid !=0 THEN waiting_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(waiting_node_id), waiting_pid) END waiting_global_pid, case WHEN blocking_global_pid !=0 THEN blocking_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(blocking_node_id), blocking_pid) END blocking_global_pid, -- citus_internal.global_blocked_processes returns groupId, we replace it here with actual -- nodeId to be consisten with the other views get_nodeid_for_groupid(blocking_node_id) as blocking_node_id, get_nodeid_for_groupid(waiting_node_id) as waiting_node_id, blocking_transaction_waiting FROM citus_internal.global_blocked_processes() ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_global_pid, blocking_global_pid) * FROM unique_global_wait_edges_with_calculated_gpids ), citus_dist_stat_activity_with_calculated_gpids AS ( -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL SELECT CASE WHEN global_pid != 0 THEN global_pid ELSE citus_calculate_gpid(nodeid, pid) END global_pid, nodeid, pid, query FROM citus_dist_stat_activity ) SELECT waiting.global_pid as waiting_gpid, blocking.global_pid as blocking_gpid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.nodeid AS waiting_nodeid, blocking.nodeid AS blocking_nodeid FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_calculated_gpids waiting ON (unique_global_wait_edges.waiting_global_pid = waiting.global_pid) JOIN citus_dist_stat_activity_with_calculated_gpids blocking ON (unique_global_wait_edges.blocking_global_pid = blocking.global_pid); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_lock_waits/latest.sql ================================================ SET search_path = 'pg_catalog'; CREATE VIEW citus.citus_lock_waits AS WITH unique_global_wait_edges_with_calculated_gpids AS ( SELECT -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL -- also for legacy reasons citus_internal.global_blocked_processes() returns groupId, we replace that with nodeIds case WHEN waiting_global_pid !=0 THEN waiting_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(waiting_node_id), waiting_pid) END waiting_global_pid, case WHEN blocking_global_pid !=0 THEN blocking_global_pid ELSE citus_calculate_gpid(get_nodeid_for_groupid(blocking_node_id), blocking_pid) END blocking_global_pid, -- citus_internal.global_blocked_processes returns groupId, we replace it here with actual -- nodeId to be consisten with the other views get_nodeid_for_groupid(blocking_node_id) as blocking_node_id, get_nodeid_for_groupid(waiting_node_id) as waiting_node_id, blocking_transaction_waiting FROM citus_internal.global_blocked_processes() ), unique_global_wait_edges AS ( SELECT DISTINCT ON(waiting_global_pid, blocking_global_pid) * FROM unique_global_wait_edges_with_calculated_gpids ), citus_dist_stat_activity_with_calculated_gpids AS ( -- if global_pid is NULL, it is most likely that a backend is blocked on a DDL SELECT CASE WHEN global_pid != 0 THEN global_pid ELSE citus_calculate_gpid(nodeid, pid) END global_pid, nodeid, pid, query FROM citus_dist_stat_activity ) SELECT waiting.global_pid as waiting_gpid, blocking.global_pid as blocking_gpid, waiting.query AS blocked_statement, blocking.query AS current_statement_in_blocking_process, waiting.nodeid AS waiting_nodeid, blocking.nodeid AS blocking_nodeid FROM unique_global_wait_edges JOIN citus_dist_stat_activity_with_calculated_gpids waiting ON (unique_global_wait_edges.waiting_global_pid = waiting.global_pid) JOIN citus_dist_stat_activity_with_calculated_gpids blocking ON (unique_global_wait_edges.blocking_global_pid = blocking.global_pid); ALTER VIEW citus.citus_lock_waits SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_lock_waits TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_locks/11.1-1.sql ================================================ -- citus_locks combines the pg_locks views from all nodes and adds global_pid, nodeid, and -- relation_name. The columns of citus_locks don't change based on the Postgres version, -- however the pg_locks's columns do. Postgres 14 added one more column to pg_locks -- (waitstart timestamptz). citus_locks has the most expansive column set, including the -- newly added column. If citus_locks is queried in a Postgres version where pg_locks -- doesn't have some columns, the values for those columns in citus_locks will be NULL CREATE OR REPLACE FUNCTION pg_catalog.citus_locks ( OUT global_pid bigint, OUT nodeid int, OUT locktype text, OUT database oid, OUT relation oid, OUT relation_name text, OUT page integer, OUT tuple smallint, OUT virtualxid text, OUT transactionid xid, OUT classid oid, OUT objid oid, OUT objsubid smallint, OUT virtualtransaction text, OUT pid integer, OUT mode text, OUT granted boolean, OUT fastpath boolean, OUT waitstart timestamp with time zone ) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_citus_locks_rows_as_jsonb.citus_locks_row_as_jsonb)::jsonb FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::jsonb)::jsonb || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::jsonb AS citus_locks_row_as_jsonb FROM run_command_on_all_nodes ( $$ SELECT coalesce(to_jsonb (array_agg(citus_locks_from_one_node.*)), '[{}]'::jsonb) FROM ( SELECT global_pid, pg_locks.relation::regclass::text AS relation_name, pg_locks.* FROM pg_locks LEFT JOIN get_all_active_transactions () ON process_id = pid) AS citus_locks_from_one_node; $$, parallel:= TRUE, give_warning_for_connection_errors:= TRUE) WHERE success = 't') AS all_citus_locks_rows_as_jsonb)) AS ( global_pid bigint, nodeid int, locktype text, database oid, relation oid, relation_name text, page integer, tuple smallint, virtualxid text, transactionid xid, classid oid, objid oid, objsubid smallint, virtualtransaction text, pid integer, mode text, granted boolean, fastpath boolean, waitstart timestamp with time zone ); END; $function$; CREATE OR REPLACE VIEW citus.citus_locks AS SELECT * FROM pg_catalog.citus_locks(); ALTER VIEW citus.citus_locks SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_locks TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_locks/latest.sql ================================================ -- citus_locks combines the pg_locks views from all nodes and adds global_pid, nodeid, and -- relation_name. The columns of citus_locks don't change based on the Postgres version, -- however the pg_locks's columns do. Postgres 14 added one more column to pg_locks -- (waitstart timestamptz). citus_locks has the most expansive column set, including the -- newly added column. If citus_locks is queried in a Postgres version where pg_locks -- doesn't have some columns, the values for those columns in citus_locks will be NULL CREATE OR REPLACE FUNCTION pg_catalog.citus_locks ( OUT global_pid bigint, OUT nodeid int, OUT locktype text, OUT database oid, OUT relation oid, OUT relation_name text, OUT page integer, OUT tuple smallint, OUT virtualxid text, OUT transactionid xid, OUT classid oid, OUT objid oid, OUT objsubid smallint, OUT virtualtransaction text, OUT pid integer, OUT mode text, OUT granted boolean, OUT fastpath boolean, OUT waitstart timestamp with time zone ) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_citus_locks_rows_as_jsonb.citus_locks_row_as_jsonb)::jsonb FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::jsonb)::jsonb || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::jsonb AS citus_locks_row_as_jsonb FROM run_command_on_all_nodes ( $$ SELECT coalesce(to_jsonb (array_agg(citus_locks_from_one_node.*)), '[{}]'::jsonb) FROM ( SELECT global_pid, pg_locks.relation::regclass::text AS relation_name, pg_locks.* FROM pg_locks LEFT JOIN get_all_active_transactions () ON process_id = pid) AS citus_locks_from_one_node; $$, parallel:= TRUE, give_warning_for_connection_errors:= TRUE) WHERE success = 't') AS all_citus_locks_rows_as_jsonb)) AS ( global_pid bigint, nodeid int, locktype text, database oid, relation oid, relation_name text, page integer, tuple smallint, virtualxid text, transactionid xid, classid oid, objid oid, objsubid smallint, virtualtransaction text, pid integer, mode text, granted boolean, fastpath boolean, waitstart timestamp with time zone ); END; $function$; CREATE OR REPLACE VIEW citus.citus_locks AS SELECT * FROM pg_catalog.citus_locks(); ALTER VIEW citus.citus_locks SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_locks TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_move_shard_placement/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_move_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a shard from a the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_move_shard_placement/11.2-1.sql ================================================ -- citus_move_shard_placement, but with nodeid CREATE FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_move_shard_placement_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode) IS 'move a shard from the source node to the destination node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_move_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a shard from a the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_move_shard_placement/latest.sql ================================================ -- citus_move_shard_placement, but with nodeid CREATE FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_move_shard_placement_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_id integer, target_node_id integer, transfer_mode citus.shard_transfer_mode) IS 'move a shard from the source node to the destination node'; CREATE OR REPLACE FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_move_shard_placement$$; COMMENT ON FUNCTION pg_catalog.citus_move_shard_placement( shard_id bigint, source_node_name text, source_node_port integer, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a shard from a the source node to the destination node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_node_capacity_1/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_node_capacity_1(int) RETURNS float4 AS $$ SELECT 1.0::float4 $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_node_capacity_1(int) IS 'a node capacity function for use by the rebalance algorithm that always returns 1'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_node_capacity_1/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_node_capacity_1(int) RETURNS float4 AS $$ SELECT 1.0::float4 $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_node_capacity_1(int) IS 'a node capacity function for use by the rebalance algorithm that always returns 1'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodeid_for_gpid/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodeid_for_gpid(global_pid bigint) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodeid_for_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_nodeid_for_gpid(global_pid bigint) IS 'returns node id for the global process with given global pid'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodeid_for_gpid/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodeid_for_gpid(global_pid bigint) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodeid_for_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_nodeid_for_gpid(global_pid bigint) IS 'returns node id for the global process with given global pid'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodename_for_nodeid/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodename_for_nodeid(nodeid integer) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodename_for_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_nodename_for_nodeid(nodeid integer) IS 'returns node name for the node with given node id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodename_for_nodeid/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodename_for_nodeid(nodeid integer) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodename_for_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_nodename_for_nodeid(nodeid integer) IS 'returns node name for the node with given node id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodeport_for_nodeid/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodeport_for_nodeid(nodeid integer) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodeport_for_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_nodeport_for_nodeid(nodeid integer) IS 'returns node port for the node with given node id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodeport_for_nodeid/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_nodeport_for_nodeid(nodeid integer) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_nodeport_for_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_nodeport_for_nodeid(nodeid integer) IS 'returns node port for the node with given node id'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodes/13.1-1.sql ================================================ SET search_path = 'pg_catalog'; DROP VIEW IF EXISTS pg_catalog.citus_nodes; CREATE OR REPLACE VIEW citus.citus_nodes AS SELECT nodename, nodeport, CASE WHEN groupid = 0 THEN 'coordinator' ELSE 'worker' END AS role, isactive AS active FROM pg_dist_node; ALTER VIEW citus.citus_nodes SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_nodes TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_nodes/latest.sql ================================================ SET search_path = 'pg_catalog'; DROP VIEW IF EXISTS pg_catalog.citus_nodes; CREATE OR REPLACE VIEW citus.citus_nodes AS SELECT nodename, nodeport, CASE WHEN groupid = 0 THEN 'coordinator' ELSE 'worker' END AS role, isactive AS active FROM pg_dist_node; ALTER VIEW citus.citus_nodes SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_nodes TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_pause_node_within_txn/12.1-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_pause_node_within_txn(node_id int, force bool DEFAULT false, lock_cooldown int DEFAULT 10000) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_pause_node_within_txn$$; COMMENT ON FUNCTION pg_catalog.citus_pause_node_within_txn(node_id int, force bool , lock_cooldown int ) IS 'pauses node with given id which leads to add lock in tables and prevent any queries to be executed on that node'; REVOKE ALL ON FUNCTION pg_catalog.citus_pause_node_within_txn(int,bool,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_pause_node_within_txn/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_pause_node_within_txn(node_id int, force bool DEFAULT false, lock_cooldown int DEFAULT 10000) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_pause_node_within_txn$$; COMMENT ON FUNCTION pg_catalog.citus_pause_node_within_txn(node_id int, force bool , lock_cooldown int ) IS 'pauses node with given id which leads to add lock in tables and prevent any queries to be executed on that node'; REVOKE ALL ON FUNCTION pg_catalog.citus_pause_node_within_txn(int,bool,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_pid_for_gpid/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_pid_for_gpid(global_pid bigint) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_pid_for_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_pid_for_gpid(global_pid bigint) IS 'returns process id for the global process with given global pid'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_pid_for_gpid/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_pid_for_gpid(global_pid bigint) RETURNS integer LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$citus_pid_for_gpid$$; COMMENT ON FUNCTION pg_catalog.citus_pid_for_gpid(global_pid bigint) IS 'returns process id for the global process with given global pid'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM citus.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM citus.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_schema; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/12.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- We should drop any_value because PG16 has its own any_value function -- We can remove this part when we drop support for PG16 DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_schema; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; -- if we are upgrading from PG14/PG15 to PG16+, -- we will need to regenerate the partkeys because they will include varnullingrels as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/13.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- We should drop any_value because PG16+ has its own any_value function -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); END IF; -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_schema; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; -- if we are upgrading from PG14/PG15 to PG16+, -- we will need to regenerate the partkeys because they will include varnullingrels as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/14.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- We should drop any_value because PG16+ has its own any_value function -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); END IF; -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_schema; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; -- if we are upgrading from PG14/PG15 to PG16+, -- we will need to regenerate the partkeys because they will include varnullingrels as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; -- similarly, if we are upgrading to PG18+, -- we will need to regenerate the partkeys because they will include varreturningtype as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_18_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varreturningtype%'; -- remove duplicates (we would only have duplicates if we are upgrading from pre-16 to PG18+) DELETE FROM public.pg_dist_partkeys_pre_18_upgrade USING public.pg_dist_partkeys_pre_16_upgrade p16 WHERE public.pg_dist_partkeys_pre_18_upgrade.logicalrelid = p16.logicalrelid AND public.pg_dist_partkeys_pre_18_upgrade.col_name = p16.col_name; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- store upgrade stable identifiers on pg_dist_object catalog UPDATE citus.pg_dist_object SET (type, object_names, object_args) = (SELECT * FROM pg_identify_object_as_address(classid, objid, objsubid)); END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog UPDATE citus.pg_dist_object SET (type, object_names, object_args) = (SELECT * FROM pg_identify_object_as_address(classid, objid, objsubid)); END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/9.4-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM citus.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog UPDATE citus.pg_dist_object SET (type, object_names, object_args) = (SELECT * FROM pg_identify_object_as_address(classid, objid, objsubid)); END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/9.5-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM citus.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog AS $cppu$ BEGIN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); -- -- We are dropping the aggregates because postgres 14 changed -- array_cat type from anyarray to anycompatiblearray. When -- upgrading to pg14, specifically when running pg_restore on -- array_cat_agg we would get an error. So we drop the aggregate -- and create the right one on citus_finish_pg_upgrade. DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- We should drop any_value because PG16+ has its own any_value function -- We can remove this part when we drop support for PG16 IF substring(current_Setting('server_version'), '\d+')::int < 16 THEN DELETE FROM pg_depend WHERE objid IN (SELECT oid FROM pg_proc WHERE proname = 'any_value' OR proname = 'any_value_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); DROP AGGREGATE IF EXISTS pg_catalog.any_value(anyelement); DROP FUNCTION IF EXISTS pg_catalog.any_value_agg(anyelement, anyelement); END IF; -- -- Drop existing backup tables -- DROP TABLE IF EXISTS public.pg_dist_partition; DROP TABLE IF EXISTS public.pg_dist_shard; DROP TABLE IF EXISTS public.pg_dist_placement; DROP TABLE IF EXISTS public.pg_dist_node_metadata; DROP TABLE IF EXISTS public.pg_dist_node; DROP TABLE IF EXISTS public.pg_dist_local_group; DROP TABLE IF EXISTS public.pg_dist_transaction; DROP TABLE IF EXISTS public.pg_dist_colocation; DROP TABLE IF EXISTS public.pg_dist_authinfo; DROP TABLE IF EXISTS public.pg_dist_poolinfo; DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; DROP TABLE IF EXISTS public.pg_dist_object; DROP TABLE IF EXISTS public.pg_dist_cleanup; DROP TABLE IF EXISTS public.pg_dist_schema; DROP TABLE IF EXISTS public.pg_dist_clock_logical_seq; -- -- backup citus catalog tables -- CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; CREATE TABLE public.pg_dist_cleanup AS SELECT * FROM pg_catalog.pg_dist_cleanup; -- save names of the tenant schemas instead of their oids because the oids might change after pg upgrade CREATE TABLE public.pg_dist_schema AS SELECT schemaid::regnamespace::text AS schemaname, colocationid FROM pg_catalog.pg_dist_schema; -- enterprise catalog tables CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; -- sequences CREATE TABLE public.pg_dist_clock_logical_seq AS SELECT last_value FROM pg_catalog.pg_dist_clock_logical_seq; CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT name, default_strategy, shard_cost_function::regprocedure::text, node_capacity_function::regprocedure::text, shard_allowed_on_node_function::regprocedure::text, default_threshold, minimum_threshold, improvement_threshold FROM pg_catalog.pg_dist_rebalance_strategy; -- store upgrade stable identifiers on pg_dist_object catalog CREATE TABLE public.pg_dist_object AS SELECT address.type, address.object_names, address.object_args, objects.distribution_argument_index, objects.colocationid FROM pg_catalog.pg_dist_object objects, pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; -- if we are upgrading from PG14/PG15 to PG16+, -- we will need to regenerate the partkeys because they will include varnullingrels as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_16_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varnullingrels%'; -- similarly, if we are upgrading to PG18+, -- we will need to regenerate the partkeys because they will include varreturningtype as well. -- so we save the partkeys as column names here CREATE TABLE IF NOT EXISTS public.pg_dist_partkeys_pre_18_upgrade AS SELECT logicalrelid, column_to_column_name(logicalrelid, partkey) as col_name FROM pg_catalog.pg_dist_partition WHERE partkey IS NOT NULL AND partkey NOT ILIKE '%varreturningtype%'; -- remove duplicates (we would only have duplicates if we are upgrading from pre-16 to PG18+) DELETE FROM public.pg_dist_partkeys_pre_18_upgrade USING public.pg_dist_partkeys_pre_16_upgrade p16 WHERE public.pg_dist_partkeys_pre_18_upgrade.logicalrelid = p16.logicalrelid AND public.pg_dist_partkeys_pre_18_upgrade.col_name = p16.col_name; END; $cppu$; COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_promote_clone_and_rebalance/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_promote_clone_and_rebalance( clone_nodeid integer, rebalance_strategy name DEFAULT NULL, catchup_timeout_seconds integer DEFAULT 300 ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_promote_clone_and_rebalance(integer, name, integer) IS 'Promotes a registered clone node to a primary, performs necessary metadata updates, and rebalances a portion of shards from its original primary to the newly promoted node. The catchUpTimeoutSeconds parameter controls how long to wait for the clone to catch up with the primary (default: 300 seconds).'; REVOKE ALL ON FUNCTION pg_catalog.citus_promote_clone_and_rebalance(integer, name, integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_promote_clone_and_rebalance/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_promote_clone_and_rebalance( clone_nodeid integer, rebalance_strategy name DEFAULT NULL, catchup_timeout_seconds integer DEFAULT 300 ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_promote_clone_and_rebalance(integer, name, integer) IS 'Promotes a registered clone node to a primary, performs necessary metadata updates, and rebalances a portion of shards from its original primary to the newly promoted node. The catchUpTimeoutSeconds parameter controls how long to wait for the clone to catch up with the primary (default: 300 seconds).'; REVOKE ALL ON FUNCTION pg_catalog.citus_promote_clone_and_rebalance(integer, name, integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_start/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_start( rebalance_strategy name DEFAULT NULL, drain_only boolean DEFAULT false, shard_transfer_mode citus.shard_transfer_mode default 'auto' ) RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode) IS 'rebalance the shards in the cluster in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_start/13.2-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode); CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_start( rebalance_strategy name DEFAULT NULL, drain_only boolean DEFAULT false, shard_transfer_mode citus.shard_transfer_mode default 'auto', parallel_transfer_reference_tables boolean DEFAULT false, parallel_transfer_colocated_shards boolean DEFAULT false ) RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode, boolean, boolean) IS 'rebalance the shards in the cluster in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode, boolean, boolean) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_start/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode); CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_start( rebalance_strategy name DEFAULT NULL, drain_only boolean DEFAULT false, shard_transfer_mode citus.shard_transfer_mode default 'auto', parallel_transfer_reference_tables boolean DEFAULT false, parallel_transfer_colocated_shards boolean DEFAULT false ) RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode, boolean, boolean) IS 'rebalance the shards in the cluster in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_start(name, boolean, citus.shard_transfer_mode, boolean, boolean) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_status/11.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_status ( raw boolean DEFAULT FALSE ) RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz, details jsonb ) LANGUAGE SQL STRICT AS $fn$ SELECT job_status.* FROM pg_dist_background_job j, citus_job_status (j.job_id, $1) job_status WHERE j.job_id IN ( SELECT job_id FROM pg_dist_background_job WHERE job_type = 'rebalance' ORDER BY job_id DESC LIMIT 1 ); $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_status/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_status ( raw boolean DEFAULT FALSE ) RETURNS TABLE ( job_id bigint, state pg_catalog.citus_job_status, job_type name, description text, started_at timestamptz, finished_at timestamptz, details jsonb ) LANGUAGE SQL STRICT AS $fn$ SELECT job_status.* FROM pg_dist_background_job j, citus_job_status (j.job_id, $1) job_status WHERE j.job_id IN ( SELECT job_id FROM pg_dist_background_job WHERE job_type = 'rebalance' ORDER BY job_id DESC LIMIT 1 ); $fn$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_stop/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_stop() RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_stop() IS 'stop a rebalance that is running in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_stop() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_stop/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_stop() RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_stop() IS 'stop a rebalance that is running in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_stop() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_wait/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_wait() RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_wait() IS 'wait on a running rebalance in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_wait() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_rebalance_wait/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_rebalance_wait() RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_rebalance_wait() IS 'wait on a running rebalance in the background'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_rebalance_wait() TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remote_connection_stats/9.3-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_remote_connection_stats$$; COMMENT ON FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) IS 'returns statistics about remote connections'; REVOKE ALL ON FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remote_connection_stats/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_remote_connection_stats$$; COMMENT ON FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) IS 'returns statistics about remote connections'; REVOKE ALL ON FUNCTION pg_catalog.citus_remote_connection_stats( OUT hostname text, OUT port int, OUT database_name text, OUT connection_count_to_node int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remove_clone_node/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_remove_clone_node( nodename text, nodeport integer ) RETURNS VOID LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_remove_clone_node$$; COMMENT ON FUNCTION pg_catalog.citus_remove_clone_node(text, integer) IS 'Removes an inactive streaming clone node from Citus metadata. Errors if the node is not found, not registered as a clone, or is currently marked active.'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_clone_node(text, integer) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid( nodeid integer ) RETURNS VOID LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_remove_clone_node_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid(integer) IS 'Removes an inactive streaming clone node from Citus metadata using its node ID. Errors if the node is not found, not registered as a clone, or is currently marked active.'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid(integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remove_clone_node/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_remove_clone_node( nodename text, nodeport integer ) RETURNS VOID LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_remove_clone_node$$; COMMENT ON FUNCTION pg_catalog.citus_remove_clone_node(text, integer) IS 'Removes an inactive streaming clone node from Citus metadata. Errors if the node is not found, not registered as a clone, or is currently marked active.'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_clone_node(text, integer) FROM PUBLIC; CREATE OR REPLACE FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid( nodeid integer ) RETURNS VOID LANGUAGE C VOLATILE STRICT AS 'MODULE_PATHNAME', $$citus_remove_clone_node_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid(integer) IS 'Removes an inactive streaming clone node from Citus metadata using its node ID. Errors if the node is not found, not registered as a clone, or is currently marked active.'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_clone_node_with_nodeid(integer) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remove_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_remove_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_remove_node$$; COMMENT ON FUNCTION pg_catalog.citus_remove_node(nodename text, nodeport integer) IS 'remove node from the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_node(text,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_remove_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_remove_node(nodename text, nodeport integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_remove_node$$; COMMENT ON FUNCTION pg_catalog.citus_remove_node(nodename text, nodeport integer) IS 'remove node from the cluster'; REVOKE ALL ON FUNCTION pg_catalog.citus_remove_node(text,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_run_local_command/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_run_local_command(command text) RETURNS void AS $$ BEGIN EXECUTE $1; END; $$ LANGUAGE PLPGSQL; COMMENT ON FUNCTION pg_catalog.citus_run_local_command(text) IS 'citus_run_local_command executes the input command'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_run_local_command/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_run_local_command(command text) RETURNS void AS $$ BEGIN EXECUTE $1; END; $$ LANGUAGE PLPGSQL; COMMENT ON FUNCTION pg_catalog.citus_run_local_command(text) IS 'citus_run_local_command executes the input command'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_distribute/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_distribute(schemaname regnamespace) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_distribute$$; COMMENT ON FUNCTION pg_catalog.citus_schema_distribute(schemaname regnamespace) IS 'distributes a schema, allowing it to move between nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_distribute/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_distribute(schemaname regnamespace) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_distribute$$; COMMENT ON FUNCTION pg_catalog.citus_schema_distribute(schemaname regnamespace) IS 'distributes a schema, allowing it to move between nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_move/12.1-1.sql ================================================ -- citus_schema_move, using target node name and node port CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_move$$; COMMENT ON FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a distributed schema to given node'; -- citus_schema_move, using target node id CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_id integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_move_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_id integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a distributed schema to given node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_move/latest.sql ================================================ -- citus_schema_move, using target node name and node port CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_move$$; COMMENT ON FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_name text, target_node_port integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a distributed schema to given node'; -- citus_schema_move, using target node id CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_id integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_move_with_nodeid$$; COMMENT ON FUNCTION pg_catalog.citus_schema_move( schema_id regnamespace, target_node_id integer, shard_transfer_mode citus.shard_transfer_mode) IS 'move a distributed schema to given node'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_undistribute/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_undistribute(schemaname regnamespace) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_undistribute$$; COMMENT ON FUNCTION pg_catalog.citus_schema_undistribute(schemaname regnamespace) IS 'reverts schema distribution, moving it back to the coordinator'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schema_undistribute/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_schema_undistribute(schemaname regnamespace) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_schema_undistribute$$; COMMENT ON FUNCTION pg_catalog.citus_schema_undistribute(schemaname regnamespace) IS 'reverts schema distribution, moving it back to the coordinator'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schemas/12.0-1.sql ================================================ DO $$ declare citus_schemas_create_query text; BEGIN citus_schemas_create_query=$CSCQ$ CREATE OR REPLACE VIEW %I.citus_schemas AS SELECT ts.schemaid::regnamespace AS schema_name, ts.colocationid AS colocation_id, CASE WHEN pg_catalog.has_schema_privilege(CURRENT_USER, ts.schemaid::regnamespace, 'USAGE') THEN pg_size_pretty(coalesce(schema_sizes.schema_size, 0)) ELSE NULL END AS schema_size, pg_get_userbyid(n.nspowner) AS schema_owner FROM pg_dist_schema ts JOIN pg_namespace n ON (ts.schemaid = n.oid) LEFT JOIN ( SELECT c.relnamespace::regnamespace schema_id, SUM(size) AS schema_size FROM citus_shard_sizes() css, pg_dist_shard ds, pg_class c WHERE css.shard_id = ds.shardid AND ds.logicalrelid = c.oid GROUP BY schema_id ) schema_sizes ON schema_sizes.schema_id = ts.schemaid ORDER BY schema_name; $CSCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_schemas_create_query, 'public'); REVOKE ALL ON public.citus_schemas FROM public; GRANT SELECT ON public.citus_schemas TO public; ELSE EXECUTE format(citus_schemas_create_query, 'citus'); ALTER VIEW citus.citus_schemas SET SCHEMA pg_catalog; REVOKE ALL ON pg_catalog.citus_schemas FROM public; GRANT SELECT ON pg_catalog.citus_schemas TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_schemas/latest.sql ================================================ DO $$ declare citus_schemas_create_query text; BEGIN citus_schemas_create_query=$CSCQ$ CREATE OR REPLACE VIEW %I.citus_schemas AS SELECT ts.schemaid::regnamespace AS schema_name, ts.colocationid AS colocation_id, CASE WHEN pg_catalog.has_schema_privilege(CURRENT_USER, ts.schemaid::regnamespace, 'USAGE') THEN pg_size_pretty(coalesce(schema_sizes.schema_size, 0)) ELSE NULL END AS schema_size, pg_get_userbyid(n.nspowner) AS schema_owner FROM pg_dist_schema ts JOIN pg_namespace n ON (ts.schemaid = n.oid) LEFT JOIN ( SELECT c.relnamespace::regnamespace schema_id, SUM(size) AS schema_size FROM citus_shard_sizes() css, pg_dist_shard ds, pg_class c WHERE css.shard_id = ds.shardid AND ds.logicalrelid = c.oid GROUP BY schema_id ) schema_sizes ON schema_sizes.schema_id = ts.schemaid ORDER BY schema_name; $CSCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_schemas_create_query, 'public'); REVOKE ALL ON public.citus_schemas FROM public; GRANT SELECT ON public.citus_schemas TO public; ELSE EXECUTE format(citus_schemas_create_query, 'citus'); ALTER VIEW citus.citus_schemas SET SCHEMA pg_catalog; REVOKE ALL ON pg_catalog.citus_schemas FROM public; GRANT SELECT ON pg_catalog.citus_schemas TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_coordinator_host/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_set_coordinator_host( host text, port integer default current_setting('port')::int, node_role noderole default 'primary', node_cluster name default 'default') RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_set_coordinator_host$$; COMMENT ON FUNCTION pg_catalog.citus_set_coordinator_host(text,integer,noderole,name) IS 'set the host and port of the coordinator'; REVOKE ALL ON FUNCTION pg_catalog.citus_set_coordinator_host(text,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_coordinator_host/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_set_coordinator_host( host text, port integer default current_setting('port')::int, node_role noderole default 'primary', node_cluster name default 'default') RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_set_coordinator_host$$; COMMENT ON FUNCTION pg_catalog.citus_set_coordinator_host(text,integer,noderole,name) IS 'set the host and port of the coordinator'; REVOKE ALL ON FUNCTION pg_catalog.citus_set_coordinator_host(text,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_default_rebalance_strategy/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_set_default_rebalance_strategy( name text ) RETURNS VOID STRICT AS $$ BEGIN LOCK TABLE pg_dist_rebalance_strategy IN SHARE ROW EXCLUSIVE MODE; IF NOT EXISTS (SELECT 1 FROM pg_dist_rebalance_strategy t WHERE t.name = $1) THEN RAISE EXCEPTION 'strategy with specified name does not exist'; END IF; UPDATE pg_dist_rebalance_strategy SET default_strategy = false WHERE default_strategy = true; UPDATE pg_dist_rebalance_strategy t SET default_strategy = true WHERE t.name = $1; END; $$ LANGUAGE plpgsql; COMMENT ON FUNCTION pg_catalog.citus_set_default_rebalance_strategy(text) IS 'changes the default rebalance strategy to the one with the specified name'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_default_rebalance_strategy/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_set_default_rebalance_strategy( name text ) RETURNS VOID STRICT AS $$ BEGIN LOCK TABLE pg_dist_rebalance_strategy IN SHARE ROW EXCLUSIVE MODE; IF NOT EXISTS (SELECT 1 FROM pg_dist_rebalance_strategy t WHERE t.name = $1) THEN RAISE EXCEPTION 'strategy with specified name does not exist'; END IF; UPDATE pg_dist_rebalance_strategy SET default_strategy = false WHERE default_strategy = true; UPDATE pg_dist_rebalance_strategy t SET default_strategy = true WHERE t.name = $1; END; $$ LANGUAGE plpgsql; COMMENT ON FUNCTION pg_catalog.citus_set_default_rebalance_strategy(text) IS 'changes the default rebalance strategy to the one with the specified name'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_node_property/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'citus_set_node_property'; COMMENT ON FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) IS 'set a property of a node in pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_set_node_property/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'citus_set_node_property'; COMMENT ON FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) IS 'set a property of a node in pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.citus_set_node_property( nodename text, nodeport integer, property text, value boolean) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_allowed_on_node_true/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_allowed_on_node_true(bigint, int) RETURNS boolean AS $$ SELECT true $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_shard_allowed_on_node_true(bigint,int) IS 'a shard_allowed_on_node_function for use by the rebalance algorithm that always returns true'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_allowed_on_node_true/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_allowed_on_node_true(bigint, int) RETURNS boolean AS $$ SELECT true $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_shard_allowed_on_node_true(bigint,int) IS 'a shard_allowed_on_node_function for use by the rebalance algorithm that always returns true'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_cost_1/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_cost_1(bigint) RETURNS float4 AS $$ SELECT 1.0::float4 $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_shard_cost_1(bigint) IS 'a shard cost function for use by the rebalance algorithm that always returns 1'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_cost_1/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_cost_1(bigint) RETURNS float4 AS $$ SELECT 1.0::float4 $$ LANGUAGE sql; COMMENT ON FUNCTION pg_catalog.citus_shard_cost_1(bigint) IS 'a shard cost function for use by the rebalance algorithm that always returns 1'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_cost_by_disk_size/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_cost_by_disk_size(bigint) RETURNS float4 AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_shard_cost_by_disk_size(bigint) IS 'a shard cost function for use by the rebalance algorithm that returns the disk size in bytes for the specified shard and the shards that are colocated with it'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_cost_by_disk_size/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_cost_by_disk_size(bigint) RETURNS float4 AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_shard_cost_by_disk_size(bigint) IS 'a shard cost function for use by the rebalance algorithm that returns the disk size in bytes for the specified shard and the shards that are colocated with it'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( OUT schema_name name, OUT index_name name, OUT table_type text, OUT owner_name name, OUT shard_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.hide_shards_from_app_name_prefixes = '' AS $$ BEGIN -- this is the query that \di produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS SELECT schema_name as "Schema", index_name as "Name", table_type as "Type", owner_name as "Owner", shard_name as "Table" FROM pg_catalog.citus_shard_indexes_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/11.0-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( OUT schema_name name, OUT index_name name, OUT table_type text, OUT owner_name name, OUT shard_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \di produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS SELECT schema_name as "Schema", index_name as "Name", table_type as "Type", owner_name as "Owner", shard_name as "Table" FROM pg_catalog.citus_shard_indexes_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_indexes_on_worker/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_indexes_on_worker( OUT schema_name name, OUT index_name name, OUT table_type text, OUT owner_name name, OUT shard_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \di produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shard_indexes_on_worker AS SELECT schema_name as "Schema", index_name as "Name", table_type as "Type", owner_name as "Owner", shard_name as "Table" FROM pg_catalog.citus_shard_indexes_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_sizes/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_shard_sizes(OUT table_name text, OUT size bigint) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_shard_sizes$$; COMMENT ON FUNCTION pg_catalog.citus_shard_sizes(OUT table_name text, OUT size bigint) IS 'returns shards sizes across citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_sizes/11.3-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_sizes(OUT shard_id int, OUT size bigint) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_shard_sizes$$; COMMENT ON FUNCTION pg_catalog.citus_shard_sizes(OUT shard_id int, OUT size bigint) IS 'returns shards sizes across citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shard_sizes/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shard_sizes(OUT shard_id int, OUT size bigint) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_shard_sizes$$; COMMENT ON FUNCTION pg_catalog.citus_shard_sizes(OUT shard_id int, OUT size bigint) IS 'returns shards sizes across citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/10.0-1.sql ================================================ CREATE OR REPLACE VIEW citus.citus_shards AS WITH shard_sizes AS (SELECT * FROM pg_catalog.citus_shard_sizes()) SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, (SELECT size FROM shard_sizes WHERE shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) = table_name OR 'public.' || shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) = table_name LIMIT 1) as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid ORDER BY pg_dist_shard.logicalrelid::text, shardid ; ALTER VIEW citus.citus_shards SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/10.1-1.sql ================================================ CREATE OR REPLACE VIEW pg_catalog.citus_shards AS SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, size as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid LEFT JOIN (SELECT (regexp_matches(table_name,'_(\d+)$'))[1]::int as shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes ON pg_dist_shard.shardid = shard_sizes.shard_id WHERE pg_dist_placement.shardstate = 1 ORDER BY pg_dist_shard.logicalrelid::text, shardid ; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/11.1-1.sql ================================================ CREATE OR REPLACE VIEW pg_catalog.citus_shards AS SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, size as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid LEFT JOIN (SELECT (regexp_matches(table_name,'_(\d+)$'))[1]::int as shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes ON pg_dist_shard.shardid = shard_sizes.shard_id WHERE pg_dist_placement.shardstate = 1 AND -- filter out tables owned by extensions pg_dist_partition.logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY pg_dist_shard.logicalrelid::text, shardid ; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/11.3-2.sql ================================================ CREATE OR REPLACE VIEW citus.citus_shards AS SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, size as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid LEFT JOIN (SELECT shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes ON pg_dist_shard.shardid = shard_sizes.shard_id WHERE pg_dist_placement.shardstate = 1 AND -- filter out tables owned by extensions pg_dist_partition.logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY pg_dist_shard.logicalrelid::text, shardid ; ALTER VIEW citus.citus_shards SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/12.0-1.sql ================================================ CREATE OR REPLACE VIEW citus.citus_shards AS SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN colocationid IN (SELECT colocationid FROM pg_dist_schema) THEN 'schema' WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' WHEN colocationid = 0 THEN 'local' ELSE 'distributed' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, size as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid LEFT JOIN (SELECT shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes ON pg_dist_shard.shardid = shard_sizes.shard_id WHERE pg_dist_placement.shardstate = 1 AND -- filter out tables owned by extensions pg_dist_partition.logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY pg_dist_shard.logicalrelid::text, shardid ; ALTER VIEW citus.citus_shards SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards/latest.sql ================================================ CREATE OR REPLACE VIEW citus.citus_shards AS SELECT pg_dist_shard.logicalrelid AS table_name, pg_dist_shard.shardid, shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, CASE WHEN colocationid IN (SELECT colocationid FROM pg_dist_schema) THEN 'schema' WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' WHEN colocationid = 0 THEN 'local' ELSE 'distributed' END AS citus_table_type, colocationid AS colocation_id, pg_dist_node.nodename, pg_dist_node.nodeport, size as shard_size FROM pg_dist_shard JOIN pg_dist_placement ON pg_dist_shard.shardid = pg_dist_placement.shardid JOIN pg_dist_node ON pg_dist_placement.groupid = pg_dist_node.groupid JOIN pg_dist_partition ON pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid LEFT JOIN (SELECT shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes ON pg_dist_shard.shardid = shard_sizes.shard_id WHERE pg_dist_placement.shardstate = 1 AND -- filter out tables owned by extensions pg_dist_partition.logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY pg_dist_shard.logicalrelid::text, shardid ; ALTER VIEW citus.citus_shards SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_shards TO public; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( OUT schema_name name, OUT shard_name name, OUT table_type text, OUT owner_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.hide_shards_from_app_name_prefixes = '' AS $$ BEGIN -- this is the query that \d produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','S','f','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS SELECT schema_name as "Schema", shard_name as "Name", table_type as "Type", owner_name as "Owner" FROM pg_catalog.citus_shards_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards_on_worker/11.0-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( OUT schema_name name, OUT shard_name name, OUT table_type text, OUT owner_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \d produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','S','f','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS SELECT schema_name as "Schema", shard_name as "Name", table_type as "Type", owner_name as "Owner" FROM pg_catalog.citus_shards_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_shards_on_worker/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_shards_on_worker( OUT schema_name name, OUT shard_name name, OUT table_type text, OUT owner_name name) RETURNS SETOF record LANGUAGE plpgsql SET citus.show_shards_for_app_name_prefixes = '*' AS $$ BEGIN -- this is the query that \d produces, except pg_table_is_visible -- is replaced with pg_catalog.relation_is_a_known_shard(c.oid) RETURN QUERY SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'table' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','S','f','') AND n.nspname <> 'pg_catalog' AND n.nspname <> 'information_schema' AND n.nspname !~ '^pg_toast' AND pg_catalog.relation_is_a_known_shard(c.oid) ORDER BY 1,2; END; $$; CREATE OR REPLACE VIEW pg_catalog.citus_shards_on_worker AS SELECT schema_name as "Schema", shard_name as "Name", table_type as "Type", owner_name as "Owner" FROM pg_catalog.citus_shards_on_worker() s; ================================================ FILE: src/backend/distributed/sql/udfs/citus_split_shard_by_split_points/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_split_shard_by_split_points( shard_id bigint, split_points text[], -- A 'nodeId' is a uint32 in CITUS [1, 4294967296] but postgres does not have unsigned type support. -- Use integer (consistent with other previously defined UDFs that take nodeId as integer) as for all practical purposes it is big enough. node_ids integer[], -- Three modes to be implemented: block_writes, force_logical and auto. -- The default mode is auto. shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_split_shard_by_split_points$$; COMMENT ON FUNCTION pg_catalog.citus_split_shard_by_split_points(shard_id bigint, split_points text[], nodeIds integer[], citus.shard_transfer_mode) IS 'split a shard using split mode.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_split_shard_by_split_points/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_split_shard_by_split_points( shard_id bigint, split_points text[], -- A 'nodeId' is a uint32 in CITUS [1, 4294967296] but postgres does not have unsigned type support. -- Use integer (consistent with other previously defined UDFs that take nodeId as integer) as for all practical purposes it is big enough. node_ids integer[], -- Three modes to be implemented: block_writes, force_logical and auto. -- The default mode is auto. shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_split_shard_by_split_points$$; COMMENT ON FUNCTION pg_catalog.citus_split_shard_by_split_points(shard_id bigint, split_points text[], nodeIds integer[], citus.shard_transfer_mode) IS 'split a shard using split mode.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_activity/11.0-1.sql ================================================ -- citus_stat_activity combines the pg_stat_activity views from all nodes and adds global_pid, nodeid and is_worker_query columns. -- The columns of citus_stat_activity don't change based on the Postgres version, however the pg_stat_activity's columns do. -- Both Postgres 13 and 14 added one more column to pg_stat_activity (leader_pid and query_id). -- citus_stat_activity has the most expansive column set, including the newly added columns. -- If citus_stat_activity is queried in a Postgres version where pg_stat_activity doesn't have some columns citus_stat_activity has -- the values for those columns will be NULL CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_activity(OUT global_pid bigint, OUT nodeid int, OUT is_worker_query boolean, OUT datid oid, OUT datname name, OUT pid integer, OUT leader_pid integer, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr inet, OUT client_hostname text, OUT client_port integer, OUT backend_start timestamp with time zone, OUT xact_start timestamp with time zone, OUT query_start timestamp with time zone, OUT state_change timestamp with time zone, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query_id bigint, OUT query text, OUT backend_type text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_csa_rows_as_jsonb.csa_row_as_jsonb)::JSONB FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::JSONB)::JSONB || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::JSONB AS csa_row_as_jsonb FROM run_command_on_all_nodes($$ SELECT coalesce(to_jsonb(array_agg(csa_from_one_node.*)), '[{}]'::JSONB) FROM ( SELECT global_pid, worker_query AS is_worker_query, pg_stat_activity.* FROM pg_stat_activity LEFT JOIN get_all_active_transactions() ON process_id = pid ) AS csa_from_one_node; $$, parallel:=true, give_warning_for_connection_errors:=true) WHERE success = 't' ) AS all_csa_rows_as_jsonb )) AS (global_pid bigint, nodeid int, is_worker_query boolean, datid oid, datname name, pid integer, leader_pid integer, usesysid oid, usename name, application_name text, client_addr inet, client_hostname text, client_port integer, backend_start timestamp with time zone, xact_start timestamp with time zone, query_start timestamp with time zone, state_change timestamp with time zone, wait_event_type text, wait_event text, state text, backend_xid xid, backend_xmin xid, query_id bigint, query text, backend_type text); END; $function$; CREATE OR REPLACE VIEW citus.citus_stat_activity AS SELECT * FROM pg_catalog.citus_stat_activity(); ALTER VIEW citus.citus_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_activity TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_activity/latest.sql ================================================ -- citus_stat_activity combines the pg_stat_activity views from all nodes and adds global_pid, nodeid and is_worker_query columns. -- The columns of citus_stat_activity don't change based on the Postgres version, however the pg_stat_activity's columns do. -- Both Postgres 13 and 14 added one more column to pg_stat_activity (leader_pid and query_id). -- citus_stat_activity has the most expansive column set, including the newly added columns. -- If citus_stat_activity is queried in a Postgres version where pg_stat_activity doesn't have some columns citus_stat_activity has -- the values for those columns will be NULL CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_activity(OUT global_pid bigint, OUT nodeid int, OUT is_worker_query boolean, OUT datid oid, OUT datname name, OUT pid integer, OUT leader_pid integer, OUT usesysid oid, OUT usename name, OUT application_name text, OUT client_addr inet, OUT client_hostname text, OUT client_port integer, OUT backend_start timestamp with time zone, OUT xact_start timestamp with time zone, OUT query_start timestamp with time zone, OUT state_change timestamp with time zone, OUT wait_event_type text, OUT wait_event text, OUT state text, OUT backend_xid xid, OUT backend_xmin xid, OUT query_id bigint, OUT query text, OUT backend_type text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_csa_rows_as_jsonb.csa_row_as_jsonb)::JSONB FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::JSONB)::JSONB || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::JSONB AS csa_row_as_jsonb FROM run_command_on_all_nodes($$ SELECT coalesce(to_jsonb(array_agg(csa_from_one_node.*)), '[{}]'::JSONB) FROM ( SELECT global_pid, worker_query AS is_worker_query, pg_stat_activity.* FROM pg_stat_activity LEFT JOIN get_all_active_transactions() ON process_id = pid ) AS csa_from_one_node; $$, parallel:=true, give_warning_for_connection_errors:=true) WHERE success = 't' ) AS all_csa_rows_as_jsonb )) AS (global_pid bigint, nodeid int, is_worker_query boolean, datid oid, datname name, pid integer, leader_pid integer, usesysid oid, usename name, application_name text, client_addr inet, client_hostname text, client_port integer, backend_start timestamp with time zone, xact_start timestamp with time zone, query_start timestamp with time zone, state_change timestamp with time zone, wait_event_type text, wait_event text, state text, backend_xid xid, backend_xmin xid, query_id bigint, query text, backend_type text); END; $function$; CREATE OR REPLACE VIEW citus.citus_stat_activity AS SELECT * FROM pg_catalog.citus_stat_activity(); ALTER VIEW citus.citus_stat_activity SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_activity TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_counters/13.1-1.sql ================================================ -- See the comments for the function in -- src/backend/distributed/stats/stat_counters.c for more details. CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters( database_id oid DEFAULT 0, -- must always be the first column or you should accordingly update -- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c OUT database_id oid, -- Following stat counter columns must be in the same order as the -- StatType enum defined in src/include/distributed/stats/stat_counters.h OUT connection_establishment_succeeded bigint, OUT connection_establishment_failed bigint, OUT connection_reused bigint, OUT query_execution_single_shard bigint, OUT query_execution_multi_shard bigint, -- must always be the last column or you should accordingly update -- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c OUT stats_reset timestamp with time zone ) RETURNS SETOF RECORD LANGUAGE C STRICT VOLATILE PARALLEL SAFE AS 'MODULE_PATHNAME', $$citus_stat_counters$$; COMMENT ON FUNCTION pg_catalog.citus_stat_counters(oid) IS 'Returns Citus stat counters for the given database OID, or for all databases if 0 is passed. Includes only databases with at least one connection since last restart, including dropped ones.'; -- returns the stat counters for all the databases in local node CREATE VIEW citus.citus_stat_counters AS SELECT pg_database.oid, pg_database.datname as name, -- We always COALESCE the counters to 0 because the LEFT JOIN -- will bring the databases that have never been connected to -- since the last restart with NULL counters, but we want to -- show them with 0 counters in the view. COALESCE(citus_stat_counters.connection_establishment_succeeded, 0) as connection_establishment_succeeded, COALESCE(citus_stat_counters.connection_establishment_failed, 0) as connection_establishment_failed, COALESCE(citus_stat_counters.connection_reused, 0) as connection_reused, COALESCE(citus_stat_counters.query_execution_single_shard, 0) as query_execution_single_shard, COALESCE(citus_stat_counters.query_execution_multi_shard, 0) as query_execution_multi_shard, citus_stat_counters.stats_reset FROM pg_catalog.pg_database LEFT JOIN (SELECT (pg_catalog.citus_stat_counters(0)).*) citus_stat_counters ON (oid = database_id); ALTER VIEW citus.citus_stat_counters SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_counters TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_counters/latest.sql ================================================ -- See the comments for the function in -- src/backend/distributed/stats/stat_counters.c for more details. CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters( database_id oid DEFAULT 0, -- must always be the first column or you should accordingly update -- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c OUT database_id oid, -- Following stat counter columns must be in the same order as the -- StatType enum defined in src/include/distributed/stats/stat_counters.h OUT connection_establishment_succeeded bigint, OUT connection_establishment_failed bigint, OUT connection_reused bigint, OUT query_execution_single_shard bigint, OUT query_execution_multi_shard bigint, -- must always be the last column or you should accordingly update -- StoreDatabaseStatsIntoTupStore() function in src/backend/distributed/stats/stat_counters.c OUT stats_reset timestamp with time zone ) RETURNS SETOF RECORD LANGUAGE C STRICT VOLATILE PARALLEL SAFE AS 'MODULE_PATHNAME', $$citus_stat_counters$$; COMMENT ON FUNCTION pg_catalog.citus_stat_counters(oid) IS 'Returns Citus stat counters for the given database OID, or for all databases if 0 is passed. Includes only databases with at least one connection since last restart, including dropped ones.'; -- returns the stat counters for all the databases in local node CREATE VIEW citus.citus_stat_counters AS SELECT pg_database.oid, pg_database.datname as name, -- We always COALESCE the counters to 0 because the LEFT JOIN -- will bring the databases that have never been connected to -- since the last restart with NULL counters, but we want to -- show them with 0 counters in the view. COALESCE(citus_stat_counters.connection_establishment_succeeded, 0) as connection_establishment_succeeded, COALESCE(citus_stat_counters.connection_establishment_failed, 0) as connection_establishment_failed, COALESCE(citus_stat_counters.connection_reused, 0) as connection_reused, COALESCE(citus_stat_counters.query_execution_single_shard, 0) as query_execution_single_shard, COALESCE(citus_stat_counters.query_execution_multi_shard, 0) as query_execution_multi_shard, citus_stat_counters.stats_reset FROM pg_catalog.pg_database LEFT JOIN (SELECT (pg_catalog.citus_stat_counters(0)).*) citus_stat_counters ON (oid = database_id); ALTER VIEW citus.citus_stat_counters SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stat_counters TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_counters_reset/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters_reset(database_oid oid DEFAULT 0) RETURNS VOID LANGUAGE C STRICT PARALLEL SAFE AS 'MODULE_PATHNAME', $$citus_stat_counters_reset$$; COMMENT ON FUNCTION pg_catalog.citus_stat_counters_reset(oid) IS 'Resets Citus stat counters for the given database OID or for the current database if nothing or 0 is provided.'; -- Rather than using explicit superuser() check in the function, we use -- the GRANT system to REVOKE access to it when creating the extension. -- Administrators can later change who can access it, or leave them as -- only available to superuser / database cluster owner, if they choose. REVOKE ALL ON FUNCTION pg_catalog.citus_stat_counters_reset(oid) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_counters_reset/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_counters_reset(database_oid oid DEFAULT 0) RETURNS VOID LANGUAGE C STRICT PARALLEL SAFE AS 'MODULE_PATHNAME', $$citus_stat_counters_reset$$; COMMENT ON FUNCTION pg_catalog.citus_stat_counters_reset(oid) IS 'Resets Citus stat counters for the given database OID or for the current database if nothing or 0 is provided.'; -- Rather than using explicit superuser() check in the function, we use -- the GRANT system to REVOKE access to it when creating the extension. -- Administrators can later change who can access it, or leave them as -- only available to superuser / database cluster owner, if they choose. REVOKE ALL ON FUNCTION pg_catalog.citus_stat_counters_reset(oid) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants/11.3-1.sql ================================================ -- cts in the query is an abbreviation for citus_stat_tenants CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants ( return_all_tenants BOOLEAN DEFAULT FALSE, OUT nodeid INT, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT ) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN IF array_position(enumvals, 'log') >= array_position(enumvals, setting) AND setting != 'off' FROM pg_settings WHERE name = 'citus.stat_tenants_log_level' THEN RAISE LOG 'Generating citus_stat_tenants'; END IF; RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_cst_rows_as_jsonb.cst_row_as_jsonb)::jsonb FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::jsonb)::jsonb || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::jsonb AS cst_row_as_jsonb FROM run_command_on_all_nodes ( $$ SELECT coalesce(to_jsonb (array_agg(cstl.*)), '[]'::jsonb) FROM citus_stat_tenants_local($$||return_all_tenants||$$) cstl; $$, parallel:= TRUE, give_warning_for_connection_errors:= TRUE) WHERE success = 't') AS all_cst_rows_as_jsonb)) AS ( nodeid INT, colocation_id INT, tenant_attribute TEXT, read_count_in_this_period INT, read_count_in_last_period INT, query_count_in_this_period INT, query_count_in_last_period INT, cpu_usage_in_this_period DOUBLE PRECISION, cpu_usage_in_last_period DOUBLE PRECISION, score BIGINT ) ORDER BY score DESC LIMIT CASE WHEN NOT return_all_tenants THEN current_setting('citus.stat_tenants_limit')::BIGINT END; END; $function$; CREATE OR REPLACE VIEW citus.citus_stat_tenants AS SELECT nodeid, colocation_id, tenant_attribute, read_count_in_this_period, read_count_in_last_period, query_count_in_this_period, query_count_in_last_period, cpu_usage_in_this_period, cpu_usage_in_last_period FROM pg_catalog.citus_stat_tenants(FALSE); ALTER VIEW citus.citus_stat_tenants SET SCHEMA pg_catalog; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants(BOOLEAN) TO pg_monitor; REVOKE ALL ON pg_catalog.citus_stat_tenants FROM PUBLIC; GRANT SELECT ON pg_catalog.citus_stat_tenants TO pg_monitor; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants/latest.sql ================================================ -- cts in the query is an abbreviation for citus_stat_tenants CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants ( return_all_tenants BOOLEAN DEFAULT FALSE, OUT nodeid INT, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT ) RETURNS SETOF record LANGUAGE plpgsql AS $function$ BEGIN IF array_position(enumvals, 'log') >= array_position(enumvals, setting) AND setting != 'off' FROM pg_settings WHERE name = 'citus.stat_tenants_log_level' THEN RAISE LOG 'Generating citus_stat_tenants'; END IF; RETURN QUERY SELECT * FROM jsonb_to_recordset(( SELECT jsonb_agg(all_cst_rows_as_jsonb.cst_row_as_jsonb)::jsonb FROM ( SELECT jsonb_array_elements(run_command_on_all_nodes.result::jsonb)::jsonb || ('{"nodeid":' || run_command_on_all_nodes.nodeid || '}')::jsonb AS cst_row_as_jsonb FROM run_command_on_all_nodes ( $$ SELECT coalesce(to_jsonb (array_agg(cstl.*)), '[]'::jsonb) FROM citus_stat_tenants_local($$||return_all_tenants||$$) cstl; $$, parallel:= TRUE, give_warning_for_connection_errors:= TRUE) WHERE success = 't') AS all_cst_rows_as_jsonb)) AS ( nodeid INT, colocation_id INT, tenant_attribute TEXT, read_count_in_this_period INT, read_count_in_last_period INT, query_count_in_this_period INT, query_count_in_last_period INT, cpu_usage_in_this_period DOUBLE PRECISION, cpu_usage_in_last_period DOUBLE PRECISION, score BIGINT ) ORDER BY score DESC LIMIT CASE WHEN NOT return_all_tenants THEN current_setting('citus.stat_tenants_limit')::BIGINT END; END; $function$; CREATE OR REPLACE VIEW citus.citus_stat_tenants AS SELECT nodeid, colocation_id, tenant_attribute, read_count_in_this_period, read_count_in_last_period, query_count_in_this_period, query_count_in_last_period, cpu_usage_in_this_period, cpu_usage_in_last_period FROM pg_catalog.citus_stat_tenants(FALSE); ALTER VIEW citus.citus_stat_tenants SET SCHEMA pg_catalog; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants(BOOLEAN) TO pg_monitor; REVOKE ALL ON pg_catalog.citus_stat_tenants FROM PUBLIC; GRANT SELECT ON pg_catalog.citus_stat_tenants TO pg_monitor; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_local/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local( return_all_tenants BOOLEAN DEFAULT FALSE, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT) RETURNS SETOF RECORD LANGUAGE C AS 'citus', $$citus_stat_tenants_local$$; CREATE OR REPLACE VIEW citus.citus_stat_tenants_local AS SELECT colocation_id, tenant_attribute, read_count_in_this_period, read_count_in_last_period, query_count_in_this_period, query_count_in_last_period, cpu_usage_in_this_period, cpu_usage_in_last_period FROM pg_catalog.citus_stat_tenants_local() ORDER BY score DESC; ALTER VIEW citus.citus_stat_tenants_local SET SCHEMA pg_catalog; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) TO pg_monitor; REVOKE ALL ON pg_catalog.citus_stat_tenants_local FROM PUBLIC; GRANT SELECT ON pg_catalog.citus_stat_tenants_local TO pg_monitor; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_local/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local_internal( return_all_tenants BOOLEAN DEFAULT FALSE, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT) RETURNS SETOF RECORD LANGUAGE C AS 'citus', $$citus_stat_tenants_local$$; CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local( return_all_tenants BOOLEAN DEFAULT FALSE, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT) RETURNS SETOF RECORD LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT L.colocation_id, CASE WHEN L.tenant_attribute IS NULL THEN N.nspname ELSE L.tenant_attribute END COLLATE "default" as tenant_attribute, L.read_count_in_this_period, L.read_count_in_last_period, L.query_count_in_this_period, L.query_count_in_last_period, L.cpu_usage_in_this_period, L.cpu_usage_in_last_period, L.score FROM pg_catalog.citus_stat_tenants_local_internal(return_all_tenants) L LEFT JOIN pg_dist_schema S ON L.tenant_attribute IS NULL AND L.colocation_id = S.colocationid LEFT JOIN pg_namespace N ON N.oid = S.schemaid ORDER BY L.score DESC; END; $function$; CREATE OR REPLACE VIEW pg_catalog.citus_stat_tenants_local AS SELECT colocation_id, tenant_attribute, read_count_in_this_period, read_count_in_last_period, query_count_in_this_period, query_count_in_last_period, cpu_usage_in_this_period, cpu_usage_in_last_period FROM pg_catalog.citus_stat_tenants_local() ORDER BY score DESC; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants_local_internal(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants_local_internal(BOOLEAN) TO pg_monitor; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) TO pg_monitor; REVOKE ALL ON pg_catalog.citus_stat_tenants_local FROM PUBLIC; GRANT SELECT ON pg_catalog.citus_stat_tenants_local TO pg_monitor; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_local/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local_internal( return_all_tenants BOOLEAN DEFAULT FALSE, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT) RETURNS SETOF RECORD LANGUAGE C AS 'citus', $$citus_stat_tenants_local$$; CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local( return_all_tenants BOOLEAN DEFAULT FALSE, OUT colocation_id INT, OUT tenant_attribute TEXT, OUT read_count_in_this_period INT, OUT read_count_in_last_period INT, OUT query_count_in_this_period INT, OUT query_count_in_last_period INT, OUT cpu_usage_in_this_period DOUBLE PRECISION, OUT cpu_usage_in_last_period DOUBLE PRECISION, OUT score BIGINT) RETURNS SETOF RECORD LANGUAGE plpgsql AS $function$ BEGIN RETURN QUERY SELECT L.colocation_id, CASE WHEN L.tenant_attribute IS NULL THEN N.nspname ELSE L.tenant_attribute END COLLATE "default" as tenant_attribute, L.read_count_in_this_period, L.read_count_in_last_period, L.query_count_in_this_period, L.query_count_in_last_period, L.cpu_usage_in_this_period, L.cpu_usage_in_last_period, L.score FROM pg_catalog.citus_stat_tenants_local_internal(return_all_tenants) L LEFT JOIN pg_dist_schema S ON L.tenant_attribute IS NULL AND L.colocation_id = S.colocationid LEFT JOIN pg_namespace N ON N.oid = S.schemaid ORDER BY L.score DESC; END; $function$; CREATE OR REPLACE VIEW pg_catalog.citus_stat_tenants_local AS SELECT colocation_id, tenant_attribute, read_count_in_this_period, read_count_in_last_period, query_count_in_this_period, query_count_in_last_period, cpu_usage_in_this_period, cpu_usage_in_last_period FROM pg_catalog.citus_stat_tenants_local() ORDER BY score DESC; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants_local_internal(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants_local_internal(BOOLEAN) TO pg_monitor; REVOKE ALL ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.citus_stat_tenants_local(BOOLEAN) TO pg_monitor; REVOKE ALL ON pg_catalog.citus_stat_tenants_local FROM PUBLIC; GRANT SELECT ON pg_catalog.citus_stat_tenants_local TO pg_monitor; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_local_reset/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local_reset() RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_stat_tenants_local_reset$$; COMMENT ON FUNCTION pg_catalog.citus_stat_tenants_local_reset() IS 'resets the local tenant statistics'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_local_reset/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_local_reset() RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_stat_tenants_local_reset$$; COMMENT ON FUNCTION pg_catalog.citus_stat_tenants_local_reset() IS 'resets the local tenant statistics'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_reset/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_reset() RETURNS VOID LANGUAGE plpgsql AS $function$ BEGIN PERFORM run_command_on_all_nodes($$SELECT citus_stat_tenants_local_reset()$$); END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stat_tenants_reset/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_stat_tenants_reset() RETURNS VOID LANGUAGE plpgsql AS $function$ BEGIN PERFORM run_command_on_all_nodes($$SELECT citus_stat_tenants_local_reset()$$); END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stats/13.2-1.sql ================================================ SET search_path = 'pg_catalog'; DROP VIEW IF EXISTS pg_catalog.citus_stats; CREATE OR REPLACE VIEW citus.citus_stats AS WITH most_common_vals_double_json AS ( SELECT ( SELECT json_agg(row_to_json(f)) FROM ( SELECT * FROM run_command_on_shards(logicalrelid, $$ SELECT json_agg(row_to_json(shard_stats)) FROM ( SELECT '$$ || logicalrelid || $$' AS citus_table, attname, s.null_frac, most_common_vals, most_common_freqs, c.reltuples AS reltuples -- join on tablename is enough here, no need to join with pg_namespace -- since shards have unique ids in their names, hence two shard names -- could never be the same FROM pg_stats s RIGHT JOIN pg_class c ON (s.tablename = c.relname) WHERE c.oid = '%s'::regclass) shard_stats $$ ))f) FROM pg_dist_partition), most_common_vals_json AS ( SELECT (json_array_elements(json_agg)->>'result') AS result, (json_array_elements(json_agg)->>'shardid') AS shardid FROM most_common_vals_double_json), table_reltuples_json AS ( SELECT distinct(shardid), CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples, (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table FROM most_common_vals_json), table_reltuples AS ( SELECT citus_table, sum(shard_reltuples) AS table_reltuples FROM table_reltuples_json GROUP BY 1 ORDER BY 1), null_frac_json AS ( SELECT (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table, CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples, CAST((json_array_elements(result::json)->>'null_frac') AS float4) AS null_frac, (json_array_elements(result::json)->>'attname')::text AS attname FROM most_common_vals_json ), null_occurrences AS ( SELECT citus_table, attname, sum(null_frac * shard_reltuples)::bigint AS null_occurrences FROM null_frac_json GROUP BY 1, 2 ORDER BY 1, 2 ), most_common_vals AS ( SELECT (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table, (json_array_elements(result::json)->>'attname')::text AS attname, json_array_elements_text((json_array_elements(result::json)->>'most_common_vals')::json)::text AS common_val, CAST(json_array_elements_text((json_array_elements(result::json)->>'most_common_freqs')::json) AS float4) AS common_freq, CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples FROM most_common_vals_json), common_val_occurrence AS ( SELECT citus_table, m.attname, common_val, sum(common_freq * shard_reltuples)::bigint AS occurrence FROM most_common_vals m GROUP BY citus_table, m.attname, common_val ORDER BY 1, 2, occurrence DESC, 3) SELECT nsp.nspname AS schemaname, p.relname AS tablename, c.attname, CASE WHEN max(t.table_reltuples::bigint) = 0 THEN 0 ELSE max(n.null_occurrences/t.table_reltuples)::float4 END AS null_frac, ARRAY_agg(common_val) AS most_common_vals, CASE WHEN max(t.table_reltuples::bigint) = 0 THEN NULL ELSE ARRAY_agg((occurrence/t.table_reltuples)::float4) END AS most_common_freqs FROM common_val_occurrence c, table_reltuples t, null_occurrences n, pg_class p, pg_namespace nsp WHERE c.citus_table = t.citus_table AND c.citus_table = n.citus_table AND c.attname = n.attname AND c.citus_table::regclass::oid = p.oid AND p.relnamespace = nsp.oid GROUP BY nsp.nspname, c.citus_table, p.relname, c.attname; ALTER VIEW citus.citus_stats SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stats TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_stats/latest.sql ================================================ SET search_path = 'pg_catalog'; DROP VIEW IF EXISTS pg_catalog.citus_stats; CREATE OR REPLACE VIEW citus.citus_stats AS WITH most_common_vals_double_json AS ( SELECT ( SELECT json_agg(row_to_json(f)) FROM ( SELECT * FROM run_command_on_shards(logicalrelid, $$ SELECT json_agg(row_to_json(shard_stats)) FROM ( SELECT '$$ || logicalrelid || $$' AS citus_table, attname, s.null_frac, most_common_vals, most_common_freqs, c.reltuples AS reltuples -- join on tablename is enough here, no need to join with pg_namespace -- since shards have unique ids in their names, hence two shard names -- could never be the same FROM pg_stats s RIGHT JOIN pg_class c ON (s.tablename = c.relname) WHERE c.oid = '%s'::regclass) shard_stats $$ ))f) FROM pg_dist_partition), most_common_vals_json AS ( SELECT (json_array_elements(json_agg)->>'result') AS result, (json_array_elements(json_agg)->>'shardid') AS shardid FROM most_common_vals_double_json), table_reltuples_json AS ( SELECT distinct(shardid), CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples, (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table FROM most_common_vals_json), table_reltuples AS ( SELECT citus_table, sum(shard_reltuples) AS table_reltuples FROM table_reltuples_json GROUP BY 1 ORDER BY 1), null_frac_json AS ( SELECT (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table, CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples, CAST((json_array_elements(result::json)->>'null_frac') AS float4) AS null_frac, (json_array_elements(result::json)->>'attname')::text AS attname FROM most_common_vals_json ), null_occurrences AS ( SELECT citus_table, attname, sum(null_frac * shard_reltuples)::bigint AS null_occurrences FROM null_frac_json GROUP BY 1, 2 ORDER BY 1, 2 ), most_common_vals AS ( SELECT (json_array_elements(result::json)->>'citus_table')::regclass AS citus_table, (json_array_elements(result::json)->>'attname')::text AS attname, json_array_elements_text((json_array_elements(result::json)->>'most_common_vals')::json)::text AS common_val, CAST(json_array_elements_text((json_array_elements(result::json)->>'most_common_freqs')::json) AS float4) AS common_freq, CAST( CAST((json_array_elements(result::json)->>'reltuples') AS DOUBLE PRECISION) AS bigint) AS shard_reltuples FROM most_common_vals_json), common_val_occurrence AS ( SELECT citus_table, m.attname, common_val, sum(common_freq * shard_reltuples)::bigint AS occurrence FROM most_common_vals m GROUP BY citus_table, m.attname, common_val ORDER BY 1, 2, occurrence DESC, 3) SELECT nsp.nspname AS schemaname, p.relname AS tablename, c.attname, CASE WHEN max(t.table_reltuples::bigint) = 0 THEN 0 ELSE max(n.null_occurrences/t.table_reltuples)::float4 END AS null_frac, ARRAY_agg(common_val) AS most_common_vals, CASE WHEN max(t.table_reltuples::bigint) = 0 THEN NULL ELSE ARRAY_agg((occurrence/t.table_reltuples)::float4) END AS most_common_freqs FROM common_val_occurrence c, table_reltuples t, null_occurrences n, pg_class p, pg_namespace nsp WHERE c.citus_table = t.citus_table AND c.citus_table = n.citus_table AND c.attname = n.attname AND c.citus_table::regclass::oid = p.oid AND p.relnamespace = nsp.oid GROUP BY nsp.nspname, c.citus_table, p.relname, c.attname; ALTER VIEW citus.citus_stats SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_stats TO PUBLIC; RESET search_path; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/10.0-1.sql ================================================ CREATE VIEW public.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' ELSE 'reference' END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(citus_total_relation_size(logicalrelid, fail_on_error := false)) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) WHERE partkey IS NOT NULL OR repmodel = 't' ORDER BY logicalrelid::text; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/10.0-4.sql ================================================ DO $$ declare citus_tables_create_query text; BEGIN citus_tables_create_query=$CTCQ$ CREATE OR REPLACE VIEW %I.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' ELSE 'reference' END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(citus_total_relation_size(logicalrelid, fail_on_error := false)) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) WHERE partkey IS NOT NULL OR repmodel = 't' ORDER BY logicalrelid::text; $CTCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_tables_create_query, 'public'); GRANT SELECT ON public.citus_tables TO public; ELSE EXECUTE format(citus_tables_create_query, 'citus'); ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_tables TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/11.1-1.sql ================================================ DO $$ declare citus_tables_create_query text; BEGIN citus_tables_create_query=$CTCQ$ CREATE OR REPLACE VIEW %I.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' ELSE CASE when repmodel = 't' THEN 'reference' ELSE 'local' END END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(citus_total_relation_size(logicalrelid, fail_on_error := false)) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) WHERE -- filter out tables owned by extensions logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY logicalrelid::text; $CTCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_tables_create_query, 'public'); GRANT SELECT ON public.citus_tables TO public; ELSE EXECUTE format(citus_tables_create_query, 'citus'); ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_tables TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/11.3-2.sql ================================================ DO $$ declare citus_tables_create_query text; BEGIN citus_tables_create_query=$CTCQ$ CREATE OR REPLACE VIEW %I.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN partkey IS NOT NULL THEN 'distributed' ELSE CASE when repmodel = 't' THEN 'reference' ELSE 'local' END END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(table_sizes.table_size) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) JOIN ( SELECT ds.logicalrelid AS table_id, SUM(css.size) AS table_size FROM citus_shard_sizes() css, pg_dist_shard ds WHERE css.shard_id = ds.shardid GROUP BY ds.logicalrelid ) table_sizes ON (table_sizes.table_id = p.logicalrelid) WHERE -- filter out tables owned by extensions logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY logicalrelid::text; $CTCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_tables_create_query, 'public'); GRANT SELECT ON public.citus_tables TO public; ELSE EXECUTE format(citus_tables_create_query, 'citus'); ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_tables TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/12.0-1.sql ================================================ DO $$ declare citus_tables_create_query text; BEGIN citus_tables_create_query=$CTCQ$ CREATE OR REPLACE VIEW %I.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN colocationid IN (SELECT colocationid FROM pg_dist_schema) THEN 'schema' WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' WHEN colocationid = 0 THEN 'local' ELSE 'distributed' END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(table_sizes.table_size) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) JOIN ( SELECT ds.logicalrelid AS table_id, SUM(css.size) AS table_size FROM citus_shard_sizes() css, pg_dist_shard ds WHERE css.shard_id = ds.shardid GROUP BY ds.logicalrelid ) table_sizes ON (table_sizes.table_id = p.logicalrelid) WHERE -- filter out tables owned by extensions logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY logicalrelid::text; $CTCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_tables_create_query, 'public'); GRANT SELECT ON public.citus_tables TO public; ELSE EXECUTE format(citus_tables_create_query, 'citus'); ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_tables TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_tables/latest.sql ================================================ DO $$ declare citus_tables_create_query text; BEGIN citus_tables_create_query=$CTCQ$ CREATE OR REPLACE VIEW %I.citus_tables AS SELECT logicalrelid AS table_name, CASE WHEN colocationid IN (SELECT colocationid FROM pg_dist_schema) THEN 'schema' WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' WHEN colocationid = 0 THEN 'local' ELSE 'distributed' END AS citus_table_type, coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, colocationid AS colocation_id, pg_size_pretty(table_sizes.table_size) AS table_size, (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, pg_get_userbyid(relowner) AS table_owner, amname AS access_method FROM pg_dist_partition p JOIN pg_class c ON (p.logicalrelid = c.oid) LEFT JOIN pg_am a ON (a.oid = c.relam) JOIN ( SELECT ds.logicalrelid AS table_id, SUM(css.size) AS table_size FROM citus_shard_sizes() css, pg_dist_shard ds WHERE css.shard_id = ds.shardid GROUP BY ds.logicalrelid ) table_sizes ON (table_sizes.table_id = p.logicalrelid) WHERE -- filter out tables owned by extensions logicalrelid NOT IN ( SELECT objid FROM pg_depend WHERE classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' ) ORDER BY logicalrelid::text; $CTCQ$; IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN EXECUTE format(citus_tables_create_query, 'public'); GRANT SELECT ON public.citus_tables TO public; ELSE EXECUTE format(citus_tables_create_query, 'citus'); ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.citus_tables TO public; END IF; END; $$; ================================================ FILE: src/backend/distributed/sql/udfs/citus_task_wait/11.2-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME',$$citus_task_wait$$; COMMENT ON FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status) IS 'blocks till the task identified by taskid is at the specified status, or reached a terminal status. Only waits for terminal status when no desired_status was specified. The return value indicates if the desired status was reached or not. When no desired status was specified it will assume any terminal status was desired'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_task_wait/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status DEFAULT NULL) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME',$$citus_task_wait$$; COMMENT ON FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status) IS 'blocks till the task identified by taskid is at the specified status, or reached a terminal status. Only waits for terminal status when no desired_status was specified. The return value indicates if the desired status was reached or not. When no desired status was specified it will assume any terminal status was desired'; GRANT EXECUTE ON FUNCTION pg_catalog.citus_task_wait(taskid bigint, desired_status pg_catalog.citus_task_status) TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_total_relation_size/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass, fail_on_error boolean default true) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_total_relation_size$$; COMMENT ON FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass, boolean) IS 'get total disk space used by the specified table'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_total_relation_size/7.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_total_relation_size$$; COMMENT ON FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass) IS 'get total disk space used by the specified table'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_total_relation_size/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass, fail_on_error boolean default true) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_total_relation_size$$; COMMENT ON FUNCTION pg_catalog.citus_total_relation_size(logicalrelid regclass, boolean) IS 'get total disk space used by the specified table'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_unmark_object_distributed/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_unmark_object_distributed$$; COMMENT ON FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int) IS 'remove an object address from citus.pg_dist_object once the object has been deleted'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_unmark_object_distributed/13.1-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int, checkobjectexistence boolean DEFAULT true) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_unmark_object_distributed$$; COMMENT ON FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int, checkobjectexistence boolean) IS 'Removes an object from citus.pg_dist_object after deletion. If checkobjectexistence is true, object existence check performed.' 'Otherwise, object existence check is skipped.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_unmark_object_distributed/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int, checkobjectexistence boolean DEFAULT true) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_unmark_object_distributed$$; COMMENT ON FUNCTION pg_catalog.citus_unmark_object_distributed(classid oid, objid oid, objsubid int, checkobjectexistence boolean) IS 'Removes an object from citus.pg_dist_object after deletion. If checkobjectexistence is true, object existence check performed.' 'Otherwise, object existence check is skipped.'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_node/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_update_node(node_id int, new_node_name text, new_node_port int, force bool DEFAULT false, lock_cooldown int DEFAULT 10000) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_node$$; COMMENT ON FUNCTION pg_catalog.citus_update_node(node_id int, new_node_name text, new_node_port int, force bool, lock_cooldown int) IS 'change the location of a node. when force => true it will wait lock_cooldown ms before killing competing locks'; REVOKE ALL ON FUNCTION pg_catalog.citus_update_node(int,text,int,bool,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_update_node(node_id int, new_node_name text, new_node_port int, force bool DEFAULT false, lock_cooldown int DEFAULT 10000) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_node$$; COMMENT ON FUNCTION pg_catalog.citus_update_node(node_id int, new_node_name text, new_node_port int, force bool, lock_cooldown int) IS 'change the location of a node. when force => true it will wait lock_cooldown ms before killing competing locks'; REVOKE ALL ON FUNCTION pg_catalog.citus_update_node(int,text,int,bool,int) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_shard_statistics/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_update_shard_statistics(shard_id bigint) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_shard_statistics$$; COMMENT ON FUNCTION pg_catalog.citus_update_shard_statistics(bigint) IS 'updates shard statistics and returns the updated shard size'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_shard_statistics/latest.sql ================================================ CREATE FUNCTION pg_catalog.citus_update_shard_statistics(shard_id bigint) RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_shard_statistics$$; COMMENT ON FUNCTION pg_catalog.citus_update_shard_statistics(bigint) IS 'updates shard statistics and returns the updated shard size'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_table_statistics/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.citus_update_table_statistics(relation regclass) RETURNS VOID AS $$ DECLARE colocated_tables regclass[]; BEGIN SELECT get_colocated_table_array(relation) INTO colocated_tables; PERFORM master_update_shard_statistics(shardid) FROM pg_dist_shard WHERE logicalrelid = ANY (colocated_tables); END; $$ LANGUAGE 'plpgsql'; COMMENT ON FUNCTION pg_catalog.citus_update_table_statistics(regclass) IS 'updates shard statistics of the given table and its colocated tables'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_table_statistics/10.0-3.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.citus_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_update_table_statistics/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_update_table_statistics(relation regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_update_table_statistics$$; COMMENT ON FUNCTION pg_catalog.citus_update_table_statistics(regclass) IS 'updates shard statistics of the given table'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_validate_rebalance_strategy_functions/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_validate_rebalance_strategy_functions( shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_validate_rebalance_strategy_functions(regproc,regproc,regproc) IS 'internal function used by citus to validate signatures of functions used in rebalance strategy'; ================================================ FILE: src/backend/distributed/sql/udfs/citus_validate_rebalance_strategy_functions/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_validate_rebalance_strategy_functions( shard_cost_function regproc, node_capacity_function regproc, shard_allowed_on_node_function regproc ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.citus_validate_rebalance_strategy_functions(regproc,regproc,regproc) IS 'internal function used by citus to validate signatures of functions used in rebalance strategy'; ================================================ FILE: src/backend/distributed/sql/udfs/commit_management_command_2pc/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.commit_management_command_2pc() RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$commit_management_command_2pc$$; COMMENT ON FUNCTION citus_internal.commit_management_command_2pc() IS 'commits the coordinated remote transactions, is a wrapper function for CoordinatedRemoteTransactionsCommit'; ================================================ FILE: src/backend/distributed/sql/udfs/commit_management_command_2pc/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.commit_management_command_2pc() RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$commit_management_command_2pc$$; COMMENT ON FUNCTION citus_internal.commit_management_command_2pc() IS 'commits the coordinated remote transactions, is a wrapper function for CoordinatedRemoteTransactionsCommit'; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg/14.0-1.sql ================================================ -- select coord_binary_combine_agg(agg, col) is similar to coord_combine_agg but -- takes binary representation of the state as input CREATE AGGREGATE pg_catalog.coord_binary_combine_agg(oid, bytea, anyelement) ( STYPE = internal, SFUNC = pg_catalog.coord_binary_combine_agg_sfunc, FINALFUNC = pg_catalog.coord_binary_combine_agg_ffunc, FINALFUNC_EXTRA ); COMMENT ON AGGREGATE pg_catalog.coord_binary_combine_agg(oid, bytea, anyelement) IS 'support aggregate for implementing combining partial aggregate results from workers'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg/latest.sql ================================================ -- select coord_binary_combine_agg(agg, col) is similar to coord_combine_agg but -- takes binary representation of the state as input CREATE AGGREGATE pg_catalog.coord_binary_combine_agg(oid, bytea, anyelement) ( STYPE = internal, SFUNC = pg_catalog.coord_binary_combine_agg_sfunc, FINALFUNC = pg_catalog.coord_binary_combine_agg_ffunc, FINALFUNC_EXTRA ); COMMENT ON AGGREGATE pg_catalog.coord_binary_combine_agg(oid, bytea, anyelement) IS 'support aggregate for implementing combining partial aggregate results from workers'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg_ffunc/14.0-1.sql ================================================ CREATE FUNCTION pg_catalog.coord_binary_combine_agg_ffunc(internal, oid, bytea, anyelement) RETURNS anyelement AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc(internal, oid, bytea, anyelement) IS 'finalizer for coord_binary_combine_agg'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg_ffunc/latest.sql ================================================ CREATE FUNCTION pg_catalog.coord_binary_combine_agg_ffunc(internal, oid, bytea, anyelement) RETURNS anyelement AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc(internal, oid, bytea, anyelement) IS 'finalizer for coord_binary_combine_agg'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg_ffunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg_sfunc/14.0-1.sql ================================================ CREATE FUNCTION pg_catalog.coord_binary_combine_agg_sfunc(internal, oid, bytea, anyelement) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc(internal, oid, bytea, anyelement) IS 'transition function for coord_binary_combine_agg'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/coord_binary_combine_agg_sfunc/latest.sql ================================================ CREATE FUNCTION pg_catalog.coord_binary_combine_agg_sfunc(internal, oid, bytea, anyelement) RETURNS internal AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc(internal, oid, bytea, anyelement) IS 'transition function for coord_binary_combine_agg'; REVOKE ALL ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.coord_binary_combine_agg_sfunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/create_citus_local_table/10.0-1.sql ================================================ DROP FUNCTION pg_catalog.create_citus_local_table(regclass); CREATE OR REPLACE FUNCTION pg_catalog.citus_add_local_table_to_metadata(table_name regclass, cascade_via_foreign_keys boolean default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_local_table_to_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_add_local_table_to_metadata(table_name regclass, cascade_via_foreign_keys boolean) IS 'create a citus local table'; ================================================ FILE: src/backend/distributed/sql/udfs/create_citus_local_table/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.create_citus_local_table(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$create_citus_local_table$$; COMMENT ON FUNCTION pg_catalog.create_citus_local_table(table_name regclass) IS 'create a citus local table'; ================================================ FILE: src/backend/distributed/sql/udfs/create_citus_local_table/latest.sql ================================================ DROP FUNCTION pg_catalog.create_citus_local_table(regclass); CREATE OR REPLACE FUNCTION pg_catalog.citus_add_local_table_to_metadata(table_name regclass, cascade_via_foreign_keys boolean default false) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_add_local_table_to_metadata$$; COMMENT ON FUNCTION pg_catalog.citus_add_local_table_to_metadata(table_name regclass, cascade_via_foreign_keys boolean) IS 'create a citus local table'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_function/11.0-1.sql ================================================ DROP FUNCTION pg_catalog.create_distributed_function(regprocedure, text, text); CREATE OR REPLACE FUNCTION pg_catalog.create_distributed_function(function_name regprocedure, distribution_arg_name text DEFAULT NULL, colocate_with text DEFAULT 'default', force_delegation bool DEFAULT NULL) RETURNS void LANGUAGE C CALLED ON NULL INPUT AS 'MODULE_PATHNAME', $$create_distributed_function$$; COMMENT ON FUNCTION pg_catalog.create_distributed_function(function_name regprocedure, distribution_arg_name text, colocate_with text, force_delegation bool) IS 'creates a distributed function'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_function/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION create_distributed_function(function_name regprocedure, distribution_arg_name text DEFAULT NULL, colocate_with text DEFAULT 'default') RETURNS void LANGUAGE C CALLED ON NULL INPUT AS 'MODULE_PATHNAME', $$create_distributed_function$$; COMMENT ON FUNCTION create_distributed_function(function_name regprocedure, distribution_arg_name text, colocate_with text) IS 'creates a distributed function'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_function/latest.sql ================================================ DROP FUNCTION pg_catalog.create_distributed_function(regprocedure, text, text); CREATE OR REPLACE FUNCTION pg_catalog.create_distributed_function(function_name regprocedure, distribution_arg_name text DEFAULT NULL, colocate_with text DEFAULT 'default', force_delegation bool DEFAULT NULL) RETURNS void LANGUAGE C CALLED ON NULL INPUT AS 'MODULE_PATHNAME', $$create_distributed_function$$; COMMENT ON FUNCTION pg_catalog.create_distributed_function(function_name regprocedure, distribution_arg_name text, colocate_with text, force_delegation bool) IS 'creates a distributed function'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_table/10.1-1.sql ================================================ DROP FUNCTION create_distributed_table(regclass, text, citus.distribution_type, text); CREATE OR REPLACE FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default', shard_count int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$create_distributed_table$$; COMMENT ON FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text, shard_count int) IS 'creates a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_table/latest.sql ================================================ DROP FUNCTION create_distributed_table(regclass, text, citus.distribution_type, text); CREATE OR REPLACE FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default', shard_count int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$create_distributed_table$$; COMMENT ON FUNCTION create_distributed_table(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text, shard_count int) IS 'creates a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_table_concurrently/11.1-1.sql ================================================ CREATE FUNCTION pg_catalog.create_distributed_table_concurrently(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default', shard_count int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$create_distributed_table_concurrently$$; COMMENT ON FUNCTION pg_catalog.create_distributed_table_concurrently(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text, shard_count int) IS 'creates a distributed table and avoids blocking writes'; ================================================ FILE: src/backend/distributed/sql/udfs/create_distributed_table_concurrently/latest.sql ================================================ CREATE FUNCTION pg_catalog.create_distributed_table_concurrently(table_name regclass, distribution_column text, distribution_type citus.distribution_type DEFAULT 'hash', colocate_with text DEFAULT 'default', shard_count int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', $$create_distributed_table_concurrently$$; COMMENT ON FUNCTION pg_catalog.create_distributed_table_concurrently(table_name regclass, distribution_column text, distribution_type citus.distribution_type, colocate_with text, shard_count int) IS 'creates a distributed table and avoids blocking writes'; ================================================ FILE: src/backend/distributed/sql/udfs/create_time_partitions/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz DEFAULT now()) returns boolean LANGUAGE plpgsql AS $$ DECLARE -- partitioned table name schema_name_text name; table_name_text name; -- record for to-be-created partition missing_partition_record record; -- result indiciates whether any partitions were created partition_created bool := false; BEGIN IF start_from >= end_at THEN RAISE 'start_from (%) must be older than end_at (%)', start_from, end_at; END IF; SELECT nspname, relname INTO schema_name_text, table_name_text FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid WHERE pg_class.oid = table_name::oid; -- Get missing partition range info using the get_missing_partition_ranges -- and create partitions using that info. FOR missing_partition_record IN SELECT * FROM get_missing_time_partition_ranges(table_name, partition_interval, end_at, start_from) LOOP EXECUTE format('CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%L) TO (%L)', schema_name_text, missing_partition_record.partition_name, schema_name_text, table_name_text, missing_partition_record.range_from_value, missing_partition_record.range_to_value); partition_created := true; END LOOP; RETURN partition_created; END; $$; COMMENT ON FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz) IS 'create time partitions for the given range'; ================================================ FILE: src/backend/distributed/sql/udfs/create_time_partitions/13.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz DEFAULT now()) returns boolean LANGUAGE plpgsql AS $$ DECLARE -- partitioned table name schema_name_text name; table_name_text name; -- record for to-be-created partition missing_partition_record record; -- result indiciates whether any partitions were created partition_created bool := false; BEGIN IF start_from >= end_at THEN RAISE 'start_from (%) must be older than end_at (%)', start_from, end_at; END IF; IF NOT isfinite(partition_interval) THEN RAISE 'Partition interval must be a finite value'; END IF; SELECT nspname, relname INTO schema_name_text, table_name_text FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid WHERE pg_class.oid = table_name::oid; -- Get missing partition range info using the get_missing_partition_ranges -- and create partitions using that info. FOR missing_partition_record IN SELECT * FROM get_missing_time_partition_ranges(table_name, partition_interval, end_at, start_from) LOOP EXECUTE format('CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%L) TO (%L)', schema_name_text, missing_partition_record.partition_name, schema_name_text, table_name_text, missing_partition_record.range_from_value, missing_partition_record.range_to_value); partition_created := true; END LOOP; RETURN partition_created; END; $$; COMMENT ON FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz) IS 'create time partitions for the given range'; ================================================ FILE: src/backend/distributed/sql/udfs/create_time_partitions/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz DEFAULT now()) returns boolean LANGUAGE plpgsql AS $$ DECLARE -- partitioned table name schema_name_text name; table_name_text name; -- record for to-be-created partition missing_partition_record record; -- result indiciates whether any partitions were created partition_created bool := false; BEGIN IF start_from >= end_at THEN RAISE 'start_from (%) must be older than end_at (%)', start_from, end_at; END IF; IF NOT isfinite(partition_interval) THEN RAISE 'Partition interval must be a finite value'; END IF; SELECT nspname, relname INTO schema_name_text, table_name_text FROM pg_class JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid WHERE pg_class.oid = table_name::oid; -- Get missing partition range info using the get_missing_partition_ranges -- and create partitions using that info. FOR missing_partition_record IN SELECT * FROM get_missing_time_partition_ranges(table_name, partition_interval, end_at, start_from) LOOP EXECUTE format('CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%L) TO (%L)', schema_name_text, missing_partition_record.partition_name, schema_name_text, table_name_text, missing_partition_record.range_from_value, missing_partition_record.range_to_value); partition_created := true; END LOOP; RETURN partition_created; END; $$; COMMENT ON FUNCTION pg_catalog.create_time_partitions( table_name regclass, partition_interval INTERVAL, end_at timestamptz, start_from timestamptz) IS 'create time partitions for the given range'; ================================================ FILE: src/backend/distributed/sql/udfs/distributed_tables_colocated/9.0-2.sql ================================================ -- -- distributed_tables_colocated returns true if given tables are co-located, false otherwise. -- The function checks shard definitions, matches shard placements for given tables. -- CREATE OR REPLACE FUNCTION pg_catalog.distributed_tables_colocated(table1 regclass, table2 regclass) RETURNS bool LANGUAGE plpgsql AS $function$ DECLARE table1_colocationid int; table2_colocationid int; BEGIN SELECT colocationid INTO table1_colocationid FROM pg_catalog.pg_dist_partition WHERE logicalrelid = table1; SELECT colocationid INTO table2_colocationid FROM pg_catalog.pg_dist_partition WHERE logicalrelid = table2; RETURN table1_colocationid = table2_colocationid; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/distributed_tables_colocated/latest.sql ================================================ -- -- distributed_tables_colocated returns true if given tables are co-located, false otherwise. -- The function checks shard definitions, matches shard placements for given tables. -- CREATE OR REPLACE FUNCTION pg_catalog.distributed_tables_colocated(table1 regclass, table2 regclass) RETURNS bool LANGUAGE plpgsql AS $function$ DECLARE table1_colocationid int; table2_colocationid int; BEGIN SELECT colocationid INTO table1_colocationid FROM pg_catalog.pg_dist_partition WHERE logicalrelid = table1; SELECT colocationid INTO table2_colocationid FROM pg_catalog.pg_dist_partition WHERE logicalrelid = table2; RETURN table1_colocationid = table2_colocationid; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/drop_old_time_partitions/10.2-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table number_of_partition_columns int; partition_column_index int; partition_column_type regtype; r record; BEGIN -- check whether the table is time partitioned table, if not error out SELECT partnatts, partattrs[0] INTO number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table WHERE partrelid = table_name; IF NOT FOUND THEN RAISE '% is not partitioned', table_name::text; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- get datatype here to check interval-table type SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; FOR r IN SELECT partition, nspname AS schema_name, relname AS table_name, from_value, to_value FROM pg_catalog.time_partitions, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE parent_table = table_name AND partition = c.oid AND c.relnamespace = n.oid AND to_value IS NOT NULL AND to_value::timestamptz <= older_than ORDER BY to_value::timestamptz LOOP RAISE NOTICE 'dropping % with start time % and end time %', r.partition, r.from_value, r.to_value; EXECUTE format('DROP TABLE %I.%I', r.schema_name, r.table_name); END LOOP; END; $$; COMMENT ON PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) IS 'drop old partitions of a time-partitioned table'; ================================================ FILE: src/backend/distributed/sql/udfs/drop_old_time_partitions/12.0-1.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table number_of_partition_columns int; partition_column_index int; partition_column_type regtype; -- used to support dynamic type casting between the partition column type and timestamptz custom_cast text; is_partition_column_castable boolean; older_partitions_query text; r record; BEGIN -- check whether the table is time partitioned table, if not error out SELECT partnatts, partattrs[0] INTO number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table WHERE partrelid = table_name; IF NOT FOUND THEN RAISE '% is not partitioned', table_name::text; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- get datatype here to check interval-table type SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz custom_cast = ''; IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN SELECT EXISTS(SELECT OID FROM pg_cast WHERE castsource = partition_column_type AND casttarget = 'timestamptz'::regtype) AND EXISTS(SELECT OID FROM pg_cast WHERE castsource = 'timestamptz'::regtype AND casttarget = partition_column_type) INTO is_partition_column_castable; IF not is_partition_column_castable THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; custom_cast = format('::%s', partition_column_type); END IF; older_partitions_query = format('SELECT partition, nspname AS schema_name, relname AS table_name, from_value, to_value FROM pg_catalog.time_partitions, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE parent_table = $1 AND partition = c.oid AND c.relnamespace = n.oid AND to_value IS NOT NULL AND to_value%1$s::timestamptz <= $2 ORDER BY to_value%1$s::timestamptz', custom_cast); FOR r IN EXECUTE older_partitions_query USING table_name, older_than LOOP RAISE NOTICE 'dropping % with start time % and end time %', r.partition, r.from_value, r.to_value; EXECUTE format('DROP TABLE %I.%I', r.schema_name, r.table_name); END LOOP; END; $$; COMMENT ON PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) IS 'drop old partitions of a time-partitioned table'; ================================================ FILE: src/backend/distributed/sql/udfs/drop_old_time_partitions/latest.sql ================================================ CREATE OR REPLACE PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table number_of_partition_columns int; partition_column_index int; partition_column_type regtype; -- used to support dynamic type casting between the partition column type and timestamptz custom_cast text; is_partition_column_castable boolean; older_partitions_query text; r record; BEGIN -- check whether the table is time partitioned table, if not error out SELECT partnatts, partattrs[0] INTO number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table WHERE partrelid = table_name; IF NOT FOUND THEN RAISE '% is not partitioned', table_name::text; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- get datatype here to check interval-table type SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz custom_cast = ''; IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN SELECT EXISTS(SELECT OID FROM pg_cast WHERE castsource = partition_column_type AND casttarget = 'timestamptz'::regtype) AND EXISTS(SELECT OID FROM pg_cast WHERE castsource = 'timestamptz'::regtype AND casttarget = partition_column_type) INTO is_partition_column_castable; IF not is_partition_column_castable THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; custom_cast = format('::%s', partition_column_type); END IF; older_partitions_query = format('SELECT partition, nspname AS schema_name, relname AS table_name, from_value, to_value FROM pg_catalog.time_partitions, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE parent_table = $1 AND partition = c.oid AND c.relnamespace = n.oid AND to_value IS NOT NULL AND to_value%1$s::timestamptz <= $2 ORDER BY to_value%1$s::timestamptz', custom_cast); FOR r IN EXECUTE older_partitions_query USING table_name, older_than LOOP RAISE NOTICE 'dropping % with start time % and end time %', r.partition, r.from_value, r.to_value; EXECUTE format('DROP TABLE %I.%I', r.schema_name, r.table_name); END LOOP; END; $$; COMMENT ON PROCEDURE pg_catalog.drop_old_time_partitions( table_name regclass, older_than timestamptz) IS 'drop old partitions of a time-partitioned table'; ================================================ FILE: src/backend/distributed/sql/udfs/fetch_intermediate_results/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fetch_intermediate_results( result_ids text[], node_name text, node_port int) RETURNS bigint LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$fetch_intermediate_results$$; COMMENT ON FUNCTION pg_catalog.fetch_intermediate_results(text[],text,int) IS 'fetch array of intermediate results from a remote node. returns number of bytes read.'; ================================================ FILE: src/backend/distributed/sql/udfs/fetch_intermediate_results/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fetch_intermediate_results( result_ids text[], node_name text, node_port int) RETURNS bigint LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$fetch_intermediate_results$$; COMMENT ON FUNCTION pg_catalog.fetch_intermediate_results(text[],text,int) IS 'fetch array of intermediate results from a remote node. returns number of bytes read.'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_all_partition_shard_index_names/10.2-4.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fix_all_partition_shard_index_names() RETURNS SETOF regclass LANGUAGE plpgsql AS $$ DECLARE dist_partitioned_table_name regclass; BEGIN FOR dist_partitioned_table_name IN SELECT p.logicalrelid FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid WHERE c.relkind = 'p' ORDER BY c.relname, c.oid LOOP EXECUTE 'SELECT fix_partition_shard_index_names( ' || quote_literal(dist_partitioned_table_name) || ' )'; RETURN NEXT dist_partitioned_table_name; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.fix_all_partition_shard_index_names() IS 'fix index names on partition shards of all tables'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_all_partition_shard_index_names/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fix_all_partition_shard_index_names() RETURNS SETOF regclass LANGUAGE plpgsql AS $$ DECLARE dist_partitioned_table_name regclass; BEGIN FOR dist_partitioned_table_name IN SELECT p.logicalrelid FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid WHERE c.relkind = 'p' ORDER BY c.relname, c.oid LOOP EXECUTE 'SELECT fix_partition_shard_index_names( ' || quote_literal(dist_partitioned_table_name) || ' )'; RETURN NEXT dist_partitioned_table_name; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.fix_all_partition_shard_index_names() IS 'fix index names on partition shards of all tables'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_partition_shard_index_names/10.2-4.sql ================================================ CREATE FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$fix_partition_shard_index_names$$; COMMENT ON FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass) IS 'fix index names on partition shards of given table'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_partition_shard_index_names/latest.sql ================================================ CREATE FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$fix_partition_shard_index_names$$; COMMENT ON FUNCTION pg_catalog.fix_partition_shard_index_names(table_name regclass) IS 'fix index names on partition shards of given table'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_pre_citus10_partitioned_table_constraint_names/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$fix_pre_citus10_partitioned_table_constraint_names$$; COMMENT ON FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(table_name regclass) IS 'fix constraint names on partition shards'; CREATE OR REPLACE FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names() RETURNS SETOF regclass LANGUAGE plpgsql AS $$ DECLARE oid regclass; BEGIN FOR oid IN SELECT c.oid FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.relkind = 'p' ORDER BY n.nspname, c.relname LOOP EXECUTE 'SELECT fix_pre_citus10_partitioned_table_constraint_names( ' || quote_literal(oid) || ' )'; RETURN NEXT oid; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names() IS 'fix constraint names on all partition shards'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_pre_citus10_partitioned_table_constraint_names/latest.sql ================================================ CREATE FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$fix_pre_citus10_partitioned_table_constraint_names$$; COMMENT ON FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names(table_name regclass) IS 'fix constraint names on partition shards'; CREATE OR REPLACE FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names() RETURNS SETOF regclass LANGUAGE plpgsql AS $$ DECLARE oid regclass; BEGIN FOR oid IN SELECT c.oid FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.relkind = 'p' ORDER BY n.nspname, c.relname LOOP EXECUTE 'SELECT fix_pre_citus10_partitioned_table_constraint_names( ' || quote_literal(oid) || ' )'; RETURN NEXT oid; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.fix_pre_citus10_partitioned_table_constraint_names() IS 'fix constraint names on all partition shards'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_pre_citus14_colocation_group_collation_mismatches/14.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fix_pre_citus14_colocation_group_collation_mismatches() RETURNS VOID AS $func$ DECLARE v_colocationid oid; v_tables_to_move_out_grouped_by_collation json; v_collationid oid; v_tables_to_move_out oid[]; v_table_to_move_out oid; v_first_table oid; BEGIN SET LOCAL search_path TO pg_catalog; FOR v_colocationid, v_tables_to_move_out_grouped_by_collation IN WITH colocation_groups_and_tables_with_collation_mismatches AS ( SELECT pdc.colocationid, pa.attcollation as distkeycollation, pdp.logicalrelid FROM pg_dist_colocation pdc JOIN pg_dist_partition pdp ON pdc.colocationid = pdp.colocationid JOIN pg_attribute pa ON pa.attrelid = pdp.logicalrelid AND pa.attname = column_to_column_name(pdp.logicalrelid, pdp.partkey) -- column_to_column_name() returns NULL if partkey is NULL, so we're already -- implicitly ignoring the tables that don't have a distribution column, such -- as reference tables, but let's still explicitly discard such tables below. WHERE pdp.partkey IS NOT NULL -- ignore the table if its distribution column collation matches the collation saved for the colocation group AND pdc.distributioncolumncollation != pa.attcollation ) SELECT colocationid, json_object_agg(distkeycollation, rels) AS tables_to_move_out_grouped_by_collation FROM ( SELECT colocationid, distkeycollation, array_agg(logicalrelid::oid) AS rels FROM colocation_groups_and_tables_with_collation_mismatches GROUP BY colocationid, distkeycollation ) q GROUP BY colocationid LOOP RAISE DEBUG 'Processing colocation group with id %', v_colocationid; FOR v_collationid, v_tables_to_move_out IN SELECT key::oid AS collationid, array_agg(elem::oid) AS tables_to_move_out FROM json_each(v_tables_to_move_out_grouped_by_collation) AS e(key, value), LATERAL json_array_elements_text(e.value) AS elem GROUP BY key LOOP RAISE DEBUG 'Moving out tables with collation id % from colocation group %', v_collationid, v_colocationid; v_first_table := NULL; FOR v_table_to_move_out IN SELECT unnest(v_tables_to_move_out) LOOP IF v_first_table IS NULL then -- Move the first table out to start a new colocation group. -- -- Could check if there is an appropriate colocation group to move to instead of 'none', -- but this won't be super easy. Plus, even if we had such a colocation group, the user -- was anyways okay with having this in a different colocation group in the first place. RAISE DEBUG 'Moving out table with oid % to a new colocation group', v_table_to_move_out; PERFORM update_distributed_table_colocation(v_table_to_move_out, colocate_with => 'none'); -- save the first table to colocate the rest of the tables with it v_first_table := v_table_to_move_out; ELSE -- Move the rest of the tables to colocate with the first table. RAISE DEBUG 'Moving out table with oid % to colocate with table with oid %', v_table_to_move_out, v_first_table; PERFORM update_distributed_table_colocation(v_table_to_move_out, colocate_with => v_first_table::regclass::text); END IF; END LOOP; END LOOP; END LOOP; END; $func$ LANGUAGE plpgsql; COMMENT ON FUNCTION pg_catalog.fix_pre_citus14_colocation_group_collation_mismatches() IS 'Fix distributed tables whose colocation group collations do not match their distribution columns by moving them to new colocation groups'; ================================================ FILE: src/backend/distributed/sql/udfs/fix_pre_citus14_colocation_group_collation_mismatches/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.fix_pre_citus14_colocation_group_collation_mismatches() RETURNS VOID AS $func$ DECLARE v_colocationid oid; v_tables_to_move_out_grouped_by_collation json; v_collationid oid; v_tables_to_move_out oid[]; v_table_to_move_out oid; v_first_table oid; BEGIN SET LOCAL search_path TO pg_catalog; FOR v_colocationid, v_tables_to_move_out_grouped_by_collation IN WITH colocation_groups_and_tables_with_collation_mismatches AS ( SELECT pdc.colocationid, pa.attcollation as distkeycollation, pdp.logicalrelid FROM pg_dist_colocation pdc JOIN pg_dist_partition pdp ON pdc.colocationid = pdp.colocationid JOIN pg_attribute pa ON pa.attrelid = pdp.logicalrelid AND pa.attname = column_to_column_name(pdp.logicalrelid, pdp.partkey) -- column_to_column_name() returns NULL if partkey is NULL, so we're already -- implicitly ignoring the tables that don't have a distribution column, such -- as reference tables, but let's still explicitly discard such tables below. WHERE pdp.partkey IS NOT NULL -- ignore the table if its distribution column collation matches the collation saved for the colocation group AND pdc.distributioncolumncollation != pa.attcollation ) SELECT colocationid, json_object_agg(distkeycollation, rels) AS tables_to_move_out_grouped_by_collation FROM ( SELECT colocationid, distkeycollation, array_agg(logicalrelid::oid) AS rels FROM colocation_groups_and_tables_with_collation_mismatches GROUP BY colocationid, distkeycollation ) q GROUP BY colocationid LOOP RAISE DEBUG 'Processing colocation group with id %', v_colocationid; FOR v_collationid, v_tables_to_move_out IN SELECT key::oid AS collationid, array_agg(elem::oid) AS tables_to_move_out FROM json_each(v_tables_to_move_out_grouped_by_collation) AS e(key, value), LATERAL json_array_elements_text(e.value) AS elem GROUP BY key LOOP RAISE DEBUG 'Moving out tables with collation id % from colocation group %', v_collationid, v_colocationid; v_first_table := NULL; FOR v_table_to_move_out IN SELECT unnest(v_tables_to_move_out) LOOP IF v_first_table IS NULL then -- Move the first table out to start a new colocation group. -- -- Could check if there is an appropriate colocation group to move to instead of 'none', -- but this won't be super easy. Plus, even if we had such a colocation group, the user -- was anyways okay with having this in a different colocation group in the first place. RAISE DEBUG 'Moving out table with oid % to a new colocation group', v_table_to_move_out; PERFORM update_distributed_table_colocation(v_table_to_move_out, colocate_with => 'none'); -- save the first table to colocate the rest of the tables with it v_first_table := v_table_to_move_out; ELSE -- Move the rest of the tables to colocate with the first table. RAISE DEBUG 'Moving out table with oid % to colocate with table with oid %', v_table_to_move_out, v_first_table; PERFORM update_distributed_table_colocation(v_table_to_move_out, colocate_with => v_first_table::regclass::text); END IF; END LOOP; END LOOP; END LOOP; END; $func$ LANGUAGE plpgsql; COMMENT ON FUNCTION pg_catalog.fix_pre_citus14_colocation_group_collation_mismatches() IS 'Fix distributed tables whose colocation group collations do not match their distribution columns by moving them to new colocation groups'; ================================================ FILE: src/backend/distributed/sql/udfs/get_all_active_transactions/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.get_all_active_transactions(); CREATE OR REPLACE FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT datname text, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) IS 'returns transaction information for all Citus initiated transactions'; ================================================ FILE: src/backend/distributed/sql/udfs/get_all_active_transactions/11.1-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.get_all_active_transactions(); CREATE OR REPLACE FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) IS 'returns transaction information for all Citus initiated transactions'; ================================================ FILE: src/backend/distributed/sql/udfs/get_all_active_transactions/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.get_all_active_transactions(); CREATE OR REPLACE FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_all_active_transactions$$; COMMENT ON FUNCTION pg_catalog.get_all_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) IS 'returns transaction information for all Citus initiated transactions'; ================================================ FILE: src/backend/distributed/sql/udfs/get_global_active_transactions/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.get_global_active_transactions(); CREATE OR REPLACE FUNCTION pg_catalog.get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_global_active_transactions$$; COMMENT ON FUNCTION pg_catalog.get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) IS 'returns transaction information for all Citus initiated transactions from each node of the cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/get_global_active_transactions/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.get_global_active_transactions(); CREATE OR REPLACE FUNCTION pg_catalog.get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) RETURNS SETOF RECORD LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$get_global_active_transactions$$; COMMENT ON FUNCTION pg_catalog.get_global_active_transactions(OUT datid oid, OUT process_id int, OUT initiator_node_identifier int4, OUT worker_query BOOL, OUT transaction_number int8, OUT transaction_stamp timestamptz, OUT global_pid int8) IS 'returns transaction information for all Citus initiated transactions from each node of the cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/get_missing_time_partition_ranges/10.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz DEFAULT now()) returns table( partition_name text, range_from_value text, range_to_value text) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table table_name_text text; table_schema_text text; number_of_partition_columns int; partition_column_index int; partition_column_type regtype; -- used for generating time ranges current_range_from_value timestamptz := NULL; current_range_to_value timestamptz := NULL; current_range_from_value_text text; current_range_to_value_text text; -- used to check whether there are misaligned (manually created) partitions manual_partition regclass; manual_partition_from_value_text text; manual_partition_to_value_text text; -- used for partition naming partition_name_format text; max_table_name_length int := current_setting('max_identifier_length'); -- used to determine whether the partition_interval is a day multiple is_day_multiple boolean; BEGIN -- check whether the table is time partitioned table, if not error out SELECT relname, nspname, partnatts, partattrs[0] INTO table_name_text, table_schema_text, number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE partrelid = c.oid AND c.oid = table_name AND c.relnamespace = n.oid; IF NOT FOUND THEN RAISE '% is not partitioned', table_name; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- to not to have partitions to be created in parallel EXECUTE format('LOCK TABLE %I.%I IN SHARE UPDATE EXCLUSIVE MODE', table_schema_text, table_name_text); -- get datatype here to check interval-table type alignment and generate range values in the right data format SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; IF partition_column_type = 'date'::regtype AND partition_interval IS NOT NULL THEN SELECT date_trunc('day', partition_interval) = partition_interval INTO is_day_multiple; IF NOT is_day_multiple THEN RAISE 'partition interval of date partitioned table must be day or multiple days'; END IF; END IF; -- If no partition exists, truncate from_value to find intuitive initial value. -- If any partition exist, use the initial partition as the pivot partition. -- tp.to_value and tp.from_value are equal to '', if default partition exists. SELECT tp.from_value::timestamptz, tp.to_value::timestamptz INTO current_range_from_value, current_range_to_value FROM pg_catalog.time_partitions tp WHERE parent_table = table_name AND tp.to_value <> '' AND tp.from_value <> '' ORDER BY tp.from_value::timestamptz ASC LIMIT 1; IF NOT FOUND THEN -- Decide on the current_range_from_value of the initial partition according to interval of the table. -- Since we will create all other partitions by adding intervals, truncating given start time will provide -- more intuitive interval ranges, instead of starting from from_value directly. IF partition_interval < INTERVAL '1 hour' THEN current_range_from_value = date_trunc('minute', from_value); ELSIF partition_interval < INTERVAL '1 day' THEN current_range_from_value = date_trunc('hour', from_value); ELSIF partition_interval < INTERVAL '1 week' THEN current_range_from_value = date_trunc('day', from_value); ELSIF partition_interval < INTERVAL '1 month' THEN current_range_from_value = date_trunc('week', from_value); ELSIF partition_interval = INTERVAL '3 months' THEN current_range_from_value = date_trunc('quarter', from_value); ELSIF partition_interval < INTERVAL '1 year' THEN current_range_from_value = date_trunc('month', from_value); ELSE current_range_from_value = date_trunc('year', from_value); END IF; current_range_to_value := current_range_from_value + partition_interval; ELSE -- if from_value is newer than pivot's from value, go forward, else go backward IF from_value >= current_range_from_value THEN WHILE current_range_from_value < from_value LOOP current_range_from_value := current_range_from_value + partition_interval; END LOOP; ELSE WHILE current_range_from_value > from_value LOOP current_range_from_value := current_range_from_value - partition_interval; END LOOP; END IF; current_range_to_value := current_range_from_value + partition_interval; END IF; -- reuse pg_partman naming scheme for back-and-forth migration IF partition_interval = INTERVAL '3 months' THEN -- include quarter in partition name partition_name_format = 'YYYY"q"Q'; ELSIF partition_interval = INTERVAL '1 week' THEN -- include week number in partition name partition_name_format := 'IYYY"w"IW'; ELSE -- always start with the year partition_name_format := 'YYYY'; IF partition_interval < INTERVAL '1 year' THEN -- include month in partition name partition_name_format := partition_name_format || '_MM'; END IF; IF partition_interval < INTERVAL '1 month' THEN -- include day of month in partition name partition_name_format := partition_name_format || '_DD'; END IF; IF partition_interval < INTERVAL '1 day' THEN -- include time of day in partition name partition_name_format := partition_name_format || '_HH24MI'; END IF; IF partition_interval < INTERVAL '1 minute' THEN -- include seconds in time of day in partition name partition_name_format := partition_name_format || 'SS'; END IF; END IF; WHILE current_range_from_value < to_value LOOP -- Check whether partition with given range has already been created -- Since partition interval can be given with different types, we are converting -- all variables to timestamptz to make sure that we are comparing same type of parameters PERFORM * FROM pg_catalog.time_partitions tp WHERE tp.from_value::timestamptz = current_range_from_value::timestamptz AND tp.to_value::timestamptz = current_range_to_value::timestamptz AND parent_table = table_name; IF found THEN current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; CONTINUE; END IF; -- Check whether any other partition covers from_value or to_value -- That means some partitions doesn't align with the initial partition. -- In other words, gap(s) exist between partitions which is not multiple of intervals. SELECT partition, tp.from_value::text, tp.to_value::text INTO manual_partition, manual_partition_from_value_text, manual_partition_to_value_text FROM pg_catalog.time_partitions tp WHERE ((current_range_from_value::timestamptz >= tp.from_value::timestamptz AND current_range_from_value < tp.to_value::timestamptz) OR (current_range_to_value::timestamptz > tp.from_value::timestamptz AND current_range_to_value::timestamptz < tp.to_value::timestamptz)) AND parent_table = table_name; IF found THEN RAISE 'partition % with the range from % to % does not align with the initial partition given the partition interval', manual_partition::text, manual_partition_from_value_text, manual_partition_to_value_text USING HINT = 'Only use partitions of the same size, without gaps between partitions.'; END IF; IF partition_column_type = 'date'::regtype THEN SELECT current_range_from_value::date::text INTO current_range_from_value_text; SELECT current_range_to_value::date::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp without time zone'::regtype THEN SELECT current_range_from_value::timestamp::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamp::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp with time zone'::regtype THEN SELECT current_range_from_value::timestamptz::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamptz::text INTO current_range_to_value_text; ELSE RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; -- use range values within the name of partition to have unique partition names RETURN QUERY SELECT substring(table_name_text, 0, max_table_name_length - length(to_char(current_range_from_value, partition_name_format)) - 1) || '_p' || to_char(current_range_from_value, partition_name_format), current_range_from_value_text, current_range_to_value_text; current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz) IS 'get missing partitions ranges for table within the range using the given interval'; ================================================ FILE: src/backend/distributed/sql/udfs/get_missing_time_partition_ranges/12.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz DEFAULT now()) returns table( partition_name text, range_from_value text, range_to_value text) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table table_name_text text; table_schema_text text; number_of_partition_columns int; partition_column_index int; partition_column_type regtype; -- used for generating time ranges current_range_from_value timestamptz := NULL; current_range_to_value timestamptz := NULL; current_range_from_value_text text; current_range_to_value_text text; -- used to check whether there are misaligned (manually created) partitions manual_partition regclass; manual_partition_from_value_text text; manual_partition_to_value_text text; -- used for partition naming partition_name_format text; max_table_name_length int := current_setting('max_identifier_length'); -- used to determine whether the partition_interval is a day multiple is_day_multiple boolean; -- used to support dynamic type casting between the partition column type and timestamptz custom_cast text; is_partition_column_castable boolean; partition regclass; partition_covers_query text; partition_exist_query text; BEGIN -- check whether the table is time partitioned table, if not error out SELECT relname, nspname, partnatts, partattrs[0] INTO table_name_text, table_schema_text, number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE partrelid = c.oid AND c.oid = table_name AND c.relnamespace = n.oid; IF NOT FOUND THEN RAISE '% is not partitioned', table_name; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- to not to have partitions to be created in parallel EXECUTE format('LOCK TABLE %I.%I IN SHARE UPDATE EXCLUSIVE MODE', table_schema_text, table_name_text); -- get datatype here to check interval-table type alignment and generate range values in the right data format SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz custom_cast = ''; IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN SELECT EXISTS(SELECT OID FROM pg_cast WHERE castsource = partition_column_type AND casttarget = 'timestamptz'::regtype) AND EXISTS(SELECT OID FROM pg_cast WHERE castsource = 'timestamptz'::regtype AND casttarget = partition_column_type) INTO is_partition_column_castable; IF not is_partition_column_castable THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; custom_cast = format('::%s', partition_column_type); END IF; IF partition_column_type = 'date'::regtype AND partition_interval IS NOT NULL THEN SELECT date_trunc('day', partition_interval) = partition_interval INTO is_day_multiple; IF NOT is_day_multiple THEN RAISE 'partition interval of date partitioned table must be day or multiple days'; END IF; END IF; -- If no partition exists, truncate from_value to find intuitive initial value. -- If any partition exist, use the initial partition as the pivot partition. -- tp.to_value and tp.from_value are equal to '', if default partition exists. EXECUTE format('SELECT tp.from_value%1$s::timestamptz, tp.to_value%1$s::timestamptz FROM pg_catalog.time_partitions tp WHERE parent_table = $1 AND tp.to_value <> '' AND tp.from_value <> '' ORDER BY tp.from_value%1$s::timestamptz ASC LIMIT 1', custom_cast) INTO current_range_from_value, current_range_to_value USING table_name; IF current_range_from_value is NULL THEN -- Decide on the current_range_from_value of the initial partition according to interval of the table. -- Since we will create all other partitions by adding intervals, truncating given start time will provide -- more intuitive interval ranges, instead of starting from from_value directly. IF partition_interval < INTERVAL '1 hour' THEN current_range_from_value = date_trunc('minute', from_value); ELSIF partition_interval < INTERVAL '1 day' THEN current_range_from_value = date_trunc('hour', from_value); ELSIF partition_interval < INTERVAL '1 week' THEN current_range_from_value = date_trunc('day', from_value); ELSIF partition_interval < INTERVAL '1 month' THEN current_range_from_value = date_trunc('week', from_value); ELSIF partition_interval = INTERVAL '3 months' THEN current_range_from_value = date_trunc('quarter', from_value); ELSIF partition_interval < INTERVAL '1 year' THEN current_range_from_value = date_trunc('month', from_value); ELSE current_range_from_value = date_trunc('year', from_value); END IF; current_range_to_value := current_range_from_value + partition_interval; ELSE -- if from_value is newer than pivot's from value, go forward, else go backward IF from_value >= current_range_from_value THEN WHILE current_range_from_value < from_value LOOP current_range_from_value := current_range_from_value + partition_interval; END LOOP; ELSE WHILE current_range_from_value > from_value LOOP current_range_from_value := current_range_from_value - partition_interval; END LOOP; END IF; current_range_to_value := current_range_from_value + partition_interval; END IF; -- reuse pg_partman naming scheme for back-and-forth migration IF partition_interval = INTERVAL '3 months' THEN -- include quarter in partition name partition_name_format = 'YYYY"q"Q'; ELSIF partition_interval = INTERVAL '1 week' THEN -- include week number in partition name partition_name_format := 'IYYY"w"IW'; ELSE -- always start with the year partition_name_format := 'YYYY'; IF partition_interval < INTERVAL '1 year' THEN -- include month in partition name partition_name_format := partition_name_format || '_MM'; END IF; IF partition_interval < INTERVAL '1 month' THEN -- include day of month in partition name partition_name_format := partition_name_format || '_DD'; END IF; IF partition_interval < INTERVAL '1 day' THEN -- include time of day in partition name partition_name_format := partition_name_format || '_HH24MI'; END IF; IF partition_interval < INTERVAL '1 minute' THEN -- include seconds in time of day in partition name partition_name_format := partition_name_format || 'SS'; END IF; END IF; partition_exist_query = format('SELECT partition FROM pg_catalog.time_partitions tp WHERE tp.from_value%1$s::timestamptz = $1 AND tp.to_value%1$s::timestamptz = $2 AND parent_table = $3', custom_cast); partition_covers_query = format('SELECT partition, tp.from_value, tp.to_value FROM pg_catalog.time_partitions tp WHERE (($1 >= tp.from_value%1$s::timestamptz AND $1 < tp.to_value%1$s::timestamptz) OR ($2 > tp.from_value%1$s::timestamptz AND $2 < tp.to_value%1$s::timestamptz)) AND parent_table = $3', custom_cast); WHILE current_range_from_value < to_value LOOP -- Check whether partition with given range has already been created -- Since partition interval can be given with different types, we are converting -- all variables to timestamptz to make sure that we are comparing same type of parameters EXECUTE partition_exist_query into partition using current_range_from_value, current_range_to_value, table_name; IF partition is not NULL THEN current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; CONTINUE; END IF; -- Check whether any other partition covers from_value or to_value -- That means some partitions doesn't align with the initial partition. -- In other words, gap(s) exist between partitions which is not multiple of intervals. EXECUTE partition_covers_query INTO manual_partition, manual_partition_from_value_text, manual_partition_to_value_text using current_range_from_value, current_range_to_value, table_name; IF manual_partition is not NULL THEN RAISE 'partition % with the range from % to % does not align with the initial partition given the partition interval', manual_partition::text, manual_partition_from_value_text, manual_partition_to_value_text USING HINT = 'Only use partitions of the same size, without gaps between partitions.'; END IF; IF partition_column_type = 'date'::regtype THEN SELECT current_range_from_value::date::text INTO current_range_from_value_text; SELECT current_range_to_value::date::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp without time zone'::regtype THEN SELECT current_range_from_value::timestamp::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamp::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp with time zone'::regtype THEN SELECT current_range_from_value::timestamptz::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamptz::text INTO current_range_to_value_text; ELSE EXECUTE format('SELECT $1%s::text', custom_cast) INTO current_range_from_value_text using current_range_from_value; EXECUTE format('SELECT $1%s::text', custom_cast) INTO current_range_to_value_text using current_range_to_value; END IF; -- use range values within the name of partition to have unique partition names RETURN QUERY SELECT substring(table_name_text, 0, max_table_name_length - length(to_char(current_range_from_value, partition_name_format)) - 1) || '_p' || to_char(current_range_from_value, partition_name_format), current_range_from_value_text, current_range_to_value_text; current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz) IS 'get missing partitions ranges for table within the range using the given interval'; ================================================ FILE: src/backend/distributed/sql/udfs/get_missing_time_partition_ranges/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz DEFAULT now()) returns table( partition_name text, range_from_value text, range_to_value text) LANGUAGE plpgsql AS $$ DECLARE -- properties of the partitioned table table_name_text text; table_schema_text text; number_of_partition_columns int; partition_column_index int; partition_column_type regtype; -- used for generating time ranges current_range_from_value timestamptz := NULL; current_range_to_value timestamptz := NULL; current_range_from_value_text text; current_range_to_value_text text; -- used to check whether there are misaligned (manually created) partitions manual_partition regclass; manual_partition_from_value_text text; manual_partition_to_value_text text; -- used for partition naming partition_name_format text; max_table_name_length int := current_setting('max_identifier_length'); -- used to determine whether the partition_interval is a day multiple is_day_multiple boolean; -- used to support dynamic type casting between the partition column type and timestamptz custom_cast text; is_partition_column_castable boolean; partition regclass; partition_covers_query text; partition_exist_query text; BEGIN -- check whether the table is time partitioned table, if not error out SELECT relname, nspname, partnatts, partattrs[0] INTO table_name_text, table_schema_text, number_of_partition_columns, partition_column_index FROM pg_catalog.pg_partitioned_table, pg_catalog.pg_class c, pg_catalog.pg_namespace n WHERE partrelid = c.oid AND c.oid = table_name AND c.relnamespace = n.oid; IF NOT FOUND THEN RAISE '% is not partitioned', table_name; ELSIF number_of_partition_columns <> 1 THEN RAISE 'partitioned tables with multiple partition columns are not supported'; END IF; -- to not to have partitions to be created in parallel EXECUTE format('LOCK TABLE %I.%I IN SHARE UPDATE EXCLUSIVE MODE', table_schema_text, table_name_text); -- get datatype here to check interval-table type alignment and generate range values in the right data format SELECT atttypid INTO partition_column_type FROM pg_attribute WHERE attrelid = table_name::oid AND attnum = partition_column_index; -- we currently only support partitioning by date, timestamp, and timestamptz custom_cast = ''; IF partition_column_type <> 'date'::regtype AND partition_column_type <> 'timestamp'::regtype AND partition_column_type <> 'timestamptz'::regtype THEN SELECT EXISTS(SELECT OID FROM pg_cast WHERE castsource = partition_column_type AND casttarget = 'timestamptz'::regtype) AND EXISTS(SELECT OID FROM pg_cast WHERE castsource = 'timestamptz'::regtype AND casttarget = partition_column_type) INTO is_partition_column_castable; IF not is_partition_column_castable THEN RAISE 'type of the partition column of the table % must be date, timestamp or timestamptz', table_name; END IF; custom_cast = format('::%s', partition_column_type); END IF; IF partition_column_type = 'date'::regtype AND partition_interval IS NOT NULL THEN SELECT date_trunc('day', partition_interval) = partition_interval INTO is_day_multiple; IF NOT is_day_multiple THEN RAISE 'partition interval of date partitioned table must be day or multiple days'; END IF; END IF; -- If no partition exists, truncate from_value to find intuitive initial value. -- If any partition exist, use the initial partition as the pivot partition. -- tp.to_value and tp.from_value are equal to '', if default partition exists. EXECUTE format('SELECT tp.from_value%1$s::timestamptz, tp.to_value%1$s::timestamptz FROM pg_catalog.time_partitions tp WHERE parent_table = $1 AND tp.to_value <> '' AND tp.from_value <> '' ORDER BY tp.from_value%1$s::timestamptz ASC LIMIT 1', custom_cast) INTO current_range_from_value, current_range_to_value USING table_name; IF current_range_from_value is NULL THEN -- Decide on the current_range_from_value of the initial partition according to interval of the table. -- Since we will create all other partitions by adding intervals, truncating given start time will provide -- more intuitive interval ranges, instead of starting from from_value directly. IF partition_interval < INTERVAL '1 hour' THEN current_range_from_value = date_trunc('minute', from_value); ELSIF partition_interval < INTERVAL '1 day' THEN current_range_from_value = date_trunc('hour', from_value); ELSIF partition_interval < INTERVAL '1 week' THEN current_range_from_value = date_trunc('day', from_value); ELSIF partition_interval < INTERVAL '1 month' THEN current_range_from_value = date_trunc('week', from_value); ELSIF partition_interval = INTERVAL '3 months' THEN current_range_from_value = date_trunc('quarter', from_value); ELSIF partition_interval < INTERVAL '1 year' THEN current_range_from_value = date_trunc('month', from_value); ELSE current_range_from_value = date_trunc('year', from_value); END IF; current_range_to_value := current_range_from_value + partition_interval; ELSE -- if from_value is newer than pivot's from value, go forward, else go backward IF from_value >= current_range_from_value THEN WHILE current_range_from_value < from_value LOOP current_range_from_value := current_range_from_value + partition_interval; END LOOP; ELSE WHILE current_range_from_value > from_value LOOP current_range_from_value := current_range_from_value - partition_interval; END LOOP; END IF; current_range_to_value := current_range_from_value + partition_interval; END IF; -- reuse pg_partman naming scheme for back-and-forth migration IF partition_interval = INTERVAL '3 months' THEN -- include quarter in partition name partition_name_format = 'YYYY"q"Q'; ELSIF partition_interval = INTERVAL '1 week' THEN -- include week number in partition name partition_name_format := 'IYYY"w"IW'; ELSE -- always start with the year partition_name_format := 'YYYY'; IF partition_interval < INTERVAL '1 year' THEN -- include month in partition name partition_name_format := partition_name_format || '_MM'; END IF; IF partition_interval < INTERVAL '1 month' THEN -- include day of month in partition name partition_name_format := partition_name_format || '_DD'; END IF; IF partition_interval < INTERVAL '1 day' THEN -- include time of day in partition name partition_name_format := partition_name_format || '_HH24MI'; END IF; IF partition_interval < INTERVAL '1 minute' THEN -- include seconds in time of day in partition name partition_name_format := partition_name_format || 'SS'; END IF; END IF; partition_exist_query = format('SELECT partition FROM pg_catalog.time_partitions tp WHERE tp.from_value%1$s::timestamptz = $1 AND tp.to_value%1$s::timestamptz = $2 AND parent_table = $3', custom_cast); partition_covers_query = format('SELECT partition, tp.from_value, tp.to_value FROM pg_catalog.time_partitions tp WHERE (($1 >= tp.from_value%1$s::timestamptz AND $1 < tp.to_value%1$s::timestamptz) OR ($2 > tp.from_value%1$s::timestamptz AND $2 < tp.to_value%1$s::timestamptz)) AND parent_table = $3', custom_cast); WHILE current_range_from_value < to_value LOOP -- Check whether partition with given range has already been created -- Since partition interval can be given with different types, we are converting -- all variables to timestamptz to make sure that we are comparing same type of parameters EXECUTE partition_exist_query into partition using current_range_from_value, current_range_to_value, table_name; IF partition is not NULL THEN current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; CONTINUE; END IF; -- Check whether any other partition covers from_value or to_value -- That means some partitions doesn't align with the initial partition. -- In other words, gap(s) exist between partitions which is not multiple of intervals. EXECUTE partition_covers_query INTO manual_partition, manual_partition_from_value_text, manual_partition_to_value_text using current_range_from_value, current_range_to_value, table_name; IF manual_partition is not NULL THEN RAISE 'partition % with the range from % to % does not align with the initial partition given the partition interval', manual_partition::text, manual_partition_from_value_text, manual_partition_to_value_text USING HINT = 'Only use partitions of the same size, without gaps between partitions.'; END IF; IF partition_column_type = 'date'::regtype THEN SELECT current_range_from_value::date::text INTO current_range_from_value_text; SELECT current_range_to_value::date::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp without time zone'::regtype THEN SELECT current_range_from_value::timestamp::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamp::text INTO current_range_to_value_text; ELSIF partition_column_type = 'timestamp with time zone'::regtype THEN SELECT current_range_from_value::timestamptz::text INTO current_range_from_value_text; SELECT current_range_to_value::timestamptz::text INTO current_range_to_value_text; ELSE EXECUTE format('SELECT $1%s::text', custom_cast) INTO current_range_from_value_text using current_range_from_value; EXECUTE format('SELECT $1%s::text', custom_cast) INTO current_range_to_value_text using current_range_to_value; END IF; -- use range values within the name of partition to have unique partition names RETURN QUERY SELECT substring(table_name_text, 0, max_table_name_length - length(to_char(current_range_from_value, partition_name_format)) - 1) || '_p' || to_char(current_range_from_value, partition_name_format), current_range_from_value_text, current_range_to_value_text; current_range_from_value := current_range_to_value; current_range_to_value := current_range_to_value + partition_interval; END LOOP; RETURN; END; $$; COMMENT ON FUNCTION pg_catalog.get_missing_time_partition_ranges( table_name regclass, partition_interval INTERVAL, to_value timestamptz, from_value timestamptz) IS 'get missing partitions ranges for table within the range using the given interval'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_progress/10.1-1.sql ================================================ DROP FUNCTION pg_catalog.get_rebalance_progress(); CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_progress() RETURNS TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int, progress bigint, source_shard_size bigint, target_shard_size bigint) AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.get_rebalance_progress() IS 'provides progress information about the ongoing rebalance operations'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_progress/11.1-1.sql ================================================ DROP FUNCTION pg_catalog.get_rebalance_progress(); CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_progress() RETURNS TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text ) AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.get_rebalance_progress() IS 'provides progress information about the ongoing rebalance operations'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_progress/11.2-1.sql ================================================ DROP FUNCTION pg_catalog.get_rebalance_progress(); CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_progress() RETURNS TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text, source_lsn pg_lsn, target_lsn pg_lsn, status text ) AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.get_rebalance_progress() IS 'provides progress information about the ongoing rebalance operations'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_progress/9.0-1.sql ================================================ -- get_rebalance_progress returns the list of shard placement move operations along with -- their progressions for ongoing rebalance operations. -- CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_progress() RETURNS TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int, progress bigint) AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.get_rebalance_progress() IS 'provides progress information about the ongoing rebalance operations'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_progress/latest.sql ================================================ DROP FUNCTION pg_catalog.get_rebalance_progress(); CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_progress() RETURNS TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text, source_lsn pg_lsn, target_lsn pg_lsn, status text ) AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.get_rebalance_progress() IS 'provides progress information about the ongoing rebalance operations'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_table_shards_plan/10.1-1.sql ================================================ -- get_rebalance_table_shards_plan shows the actual events that will be performed -- if a rebalance operation will be performed with the same arguments, which allows users -- to understand the impact of the change overall availability of the application and -- network trafic. -- DROP FUNCTION pg_catalog.get_rebalance_table_shards_plan; CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_table_shards_plan( relation regclass default NULL, threshold float4 default NULL, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', drain_only boolean default false, rebalance_strategy name default NULL, improvement_threshold float4 DEFAULT NULL ) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_rebalance_table_shards_plan(regclass, float4, int, bigint[], boolean, name, float4) IS 'returns the list of shard placement moves to be done on a rebalance operation'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_table_shards_plan/9.0-1.sql ================================================ -- get_rebalance_table_shards_plan shows the actual events that will be performed -- if a rebalance operation will be performed with the same arguments, which allows users -- to understand the impact of the change overall availability of the application and -- network trafic. -- CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_table_shards_plan( relation regclass, threshold float4 default 0, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}') RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int) AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.get_rebalance_table_shards_plan(regclass, float4, int, bigint[]) IS 'returns the list of shard placement moves to be done on a rebalance operation'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_table_shards_plan/9.1-1.sql ================================================ -- get_rebalance_table_shards_plan shows the actual events that will be performed -- if a rebalance operation will be performed with the same arguments, which allows users -- to understand the impact of the change overall availability of the application and -- network trafic. -- DROP FUNCTION pg_catalog.get_rebalance_table_shards_plan; CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_table_shards_plan( relation regclass default NULL, threshold float4 default 0, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', drain_only boolean default false) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_rebalance_table_shards_plan(regclass, float4, int, bigint[], boolean) IS 'returns the list of shard placement moves to be done on a rebalance operation'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_table_shards_plan/9.2-1.sql ================================================ -- get_rebalance_table_shards_plan shows the actual events that will be performed -- if a rebalance operation will be performed with the same arguments, which allows users -- to understand the impact of the change overall availability of the application and -- network trafic. -- DROP FUNCTION pg_catalog.get_rebalance_table_shards_plan; CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_table_shards_plan( relation regclass default NULL, threshold float4 default NULL, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', drain_only boolean default false, rebalance_strategy name default NULL ) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_rebalance_table_shards_plan(regclass, float4, int, bigint[], boolean, name) IS 'returns the list of shard placement moves to be done on a rebalance operation'; ================================================ FILE: src/backend/distributed/sql/udfs/get_rebalance_table_shards_plan/latest.sql ================================================ -- get_rebalance_table_shards_plan shows the actual events that will be performed -- if a rebalance operation will be performed with the same arguments, which allows users -- to understand the impact of the change overall availability of the application and -- network trafic. -- DROP FUNCTION pg_catalog.get_rebalance_table_shards_plan; CREATE OR REPLACE FUNCTION pg_catalog.get_rebalance_table_shards_plan( relation regclass default NULL, threshold float4 default NULL, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', drain_only boolean default false, rebalance_strategy name default NULL, improvement_threshold float4 DEFAULT NULL ) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport int, targetname text, targetport int) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_rebalance_table_shards_plan(regclass, float4, int, bigint[], boolean, name, float4) IS 'returns the list of shard placement moves to be done on a rebalance operation'; ================================================ FILE: src/backend/distributed/sql/udfs/get_snapshot_based_node_split_plan/13.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.get_snapshot_based_node_split_plan( primary_node_name text, primary_node_port integer, replica_node_name text, replica_node_port integer, rebalance_strategy name DEFAULT NULL ) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, placement_node text) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_snapshot_based_node_split_plan(text, int, text, int, name) IS 'shows the shard placements to balance shards between primary and replica worker nodes'; REVOKE ALL ON FUNCTION pg_catalog.get_snapshot_based_node_split_plan(text, int, text, int, name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/get_snapshot_based_node_split_plan/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.get_snapshot_based_node_split_plan( primary_node_name text, primary_node_port integer, replica_node_name text, replica_node_port integer, rebalance_strategy name DEFAULT NULL ) RETURNS TABLE (table_name regclass, shardid bigint, shard_size bigint, placement_node text) AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.get_snapshot_based_node_split_plan(text, int, text, int, name) IS 'shows the shard placements to balance shards between primary and replica worker nodes'; REVOKE ALL ON FUNCTION pg_catalog.get_snapshot_based_node_split_plan(text, int, text, int, name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/isolate_tenant_to_new_shard/11.1-1.sql ================================================ CREATE FUNCTION pg_catalog.isolate_tenant_to_new_shard( table_name regclass, tenant_id "any", cascade_option text DEFAULT '', shard_transfer_mode citus.shard_transfer_mode DEFAULT 'auto') RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$isolate_tenant_to_new_shard$$; COMMENT ON FUNCTION pg_catalog.isolate_tenant_to_new_shard( table_name regclass, tenant_id "any", cascade_option text, shard_transfer_mode citus.shard_transfer_mode) IS 'isolate a tenant to its own shard and return the new shard id'; ================================================ FILE: src/backend/distributed/sql/udfs/isolate_tenant_to_new_shard/8.0-1.sql ================================================ CREATE FUNCTION pg_catalog.isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text DEFAULT '') RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$isolate_tenant_to_new_shard$$; COMMENT ON FUNCTION pg_catalog.isolate_tenant_to_new_shard(table_name regclass, tenant_id "any", cascade_option text) IS 'isolate a tenant to its own shard and return the new shard id'; ================================================ FILE: src/backend/distributed/sql/udfs/isolate_tenant_to_new_shard/latest.sql ================================================ CREATE FUNCTION pg_catalog.isolate_tenant_to_new_shard( table_name regclass, tenant_id "any", cascade_option text DEFAULT '', shard_transfer_mode citus.shard_transfer_mode DEFAULT 'auto') RETURNS bigint LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$isolate_tenant_to_new_shard$$; COMMENT ON FUNCTION pg_catalog.isolate_tenant_to_new_shard( table_name regclass, tenant_id "any", cascade_option text, shard_transfer_mode citus.shard_transfer_mode) IS 'isolate a tenant to its own shard and return the new shard id'; ================================================ FILE: src/backend/distributed/sql/udfs/master_add_inactive_node/9.1-1.sql ================================================ -- Update the default groupId to -1 DROP FUNCTION master_add_inactive_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_inactive_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_add_inactive_node$$; COMMENT ON FUNCTION master_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; REVOKE ALL ON FUNCTION master_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_add_inactive_node/latest.sql ================================================ -- Update the default groupId to -1 DROP FUNCTION master_add_inactive_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_inactive_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME',$$master_add_inactive_node$$; COMMENT ON FUNCTION master_add_inactive_node(nodename text,nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'prepare node by adding it to pg_dist_node'; REVOKE ALL ON FUNCTION master_add_inactive_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_add_node/9.1-1.sql ================================================ -- Update the default groupId to -1 DROP FUNCTION master_add_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_node$$; COMMENT ON FUNCTION master_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; REVOKE ALL ON FUNCTION master_add_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_add_node/latest.sql ================================================ -- Update the default groupId to -1 DROP FUNCTION master_add_node(text, integer, integer, noderole, name); CREATE FUNCTION master_add_node(nodename text, nodeport integer, groupid integer default -1, noderole noderole default 'primary', nodecluster name default 'default') RETURNS INTEGER LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_add_node$$; COMMENT ON FUNCTION master_add_node(nodename text, nodeport integer, groupid integer, noderole noderole, nodecluster name) IS 'add node to the cluster'; REVOKE ALL ON FUNCTION master_add_node(text,int,int,noderole,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_drain_node/9.1-1.sql ================================================ CREATE FUNCTION pg_catalog.master_drain_node( nodename text, nodeport integer, shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_drain_node$$; COMMENT ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode) IS 'mark a node to be drained of data and actually drain it as well'; REVOKE ALL ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_drain_node/9.2-1.sql ================================================ DROP FUNCTION pg_catalog.master_drain_node; CREATE FUNCTION pg_catalog.master_drain_node( nodename text, nodeport integer, shard_transfer_mode citus.shard_transfer_mode default 'auto', rebalance_strategy name default NULL ) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$master_drain_node$$; COMMENT ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode,name) IS 'mark a node to be drained of data and actually drain it as well'; REVOKE ALL ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_drain_node/latest.sql ================================================ DROP FUNCTION pg_catalog.master_drain_node; CREATE FUNCTION pg_catalog.master_drain_node( nodename text, nodeport integer, shard_transfer_mode citus.shard_transfer_mode default 'auto', rebalance_strategy name default NULL ) RETURNS VOID LANGUAGE C AS 'MODULE_PATHNAME', $$master_drain_node$$; COMMENT ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode,name) IS 'mark a node to be drained of data and actually drain it as well'; REVOKE ALL ON FUNCTION pg_catalog.master_drain_node(text,int,citus.shard_transfer_mode,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_set_node_property/9.1-1.sql ================================================ CREATE FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'master_set_node_property'; COMMENT ON FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) IS 'set a property of a node in pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/master_set_node_property/latest.sql ================================================ CREATE FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'master_set_node_property'; COMMENT ON FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) IS 'set a property of a node in pg_dist_node'; REVOKE ALL ON FUNCTION pg_catalog.master_set_node_property( nodename text, nodeport integer, property text, value boolean) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/pg_cancel_backend/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.pg_cancel_backend(global_pid bigint) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.pg_cancel_backend(global_pid bigint) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$citus_cancel_backend$$; COMMENT ON FUNCTION pg_catalog.pg_cancel_backend(global_pid bigint) IS 'cancels a Citus query which might be on any node in the Citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/pg_cancel_backend/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.pg_cancel_backend(global_pid bigint) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.pg_cancel_backend(global_pid bigint) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$citus_cancel_backend$$; COMMENT ON FUNCTION pg_catalog.pg_cancel_backend(global_pid bigint) IS 'cancels a Citus query which might be on any node in the Citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/pg_dist_rebalance_strategy_trigger_func/9.2-1.sql ================================================ -- Ensures that only a single default strategy is possible CREATE OR REPLACE FUNCTION citus_internal.pg_dist_rebalance_strategy_trigger_func() RETURNS TRIGGER AS $$ BEGIN -- citus_add_rebalance_strategy also takes out a ShareRowExclusiveLock LOCK TABLE pg_dist_rebalance_strategy IN SHARE ROW EXCLUSIVE MODE; PERFORM citus_validate_rebalance_strategy_functions( NEW.shard_cost_function, NEW.node_capacity_function, NEW.shard_allowed_on_node_function); IF NEW.default_threshold < NEW.minimum_threshold THEN RAISE EXCEPTION 'default_threshold cannot be smaller than minimum_threshold'; END IF; IF NOT NEW.default_strategy THEN RETURN NEW; END IF; IF TG_OP = 'UPDATE' AND NEW.default_strategy = OLD.default_strategy THEN return NEW; END IF; IF EXISTS (SELECT 1 FROM pg_dist_rebalance_strategy WHERE default_strategy) THEN RAISE EXCEPTION 'there cannot be two default strategies'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; ================================================ FILE: src/backend/distributed/sql/udfs/pg_dist_rebalance_strategy_trigger_func/latest.sql ================================================ -- Ensures that only a single default strategy is possible CREATE OR REPLACE FUNCTION citus_internal.pg_dist_rebalance_strategy_trigger_func() RETURNS TRIGGER AS $$ BEGIN -- citus_add_rebalance_strategy also takes out a ShareRowExclusiveLock LOCK TABLE pg_dist_rebalance_strategy IN SHARE ROW EXCLUSIVE MODE; PERFORM citus_validate_rebalance_strategy_functions( NEW.shard_cost_function, NEW.node_capacity_function, NEW.shard_allowed_on_node_function); IF NEW.default_threshold < NEW.minimum_threshold THEN RAISE EXCEPTION 'default_threshold cannot be smaller than minimum_threshold'; END IF; IF NOT NEW.default_strategy THEN RETURN NEW; END IF; IF TG_OP = 'UPDATE' AND NEW.default_strategy = OLD.default_strategy THEN return NEW; END IF; IF EXISTS (SELECT 1 FROM pg_dist_rebalance_strategy WHERE default_strategy) THEN RAISE EXCEPTION 'there cannot be two default strategies'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; ================================================ FILE: src/backend/distributed/sql/udfs/pg_dist_shard_placement_trigger_func/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.pg_dist_shard_placement_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'DELETE') THEN DELETE FROM pg_dist_placement WHERE placementid = OLD.placementid; RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN UPDATE pg_dist_placement SET shardid = NEW.shardid, shardstate = NEW.shardstate, shardlength = NEW.shardlength, placementid = NEW.placementid, groupid = citus_internal.find_groupid_for_node(NEW.nodename, NEW.nodeport) WHERE placementid = OLD.placementid; RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO pg_dist_placement (placementid, shardid, shardstate, shardlength, groupid) VALUES (NEW.placementid, NEW.shardid, NEW.shardstate, NEW.shardlength, citus_internal.find_groupid_for_node(NEW.nodename, NEW.nodeport)); RETURN NEW; END IF; END; $$ LANGUAGE plpgsql; ================================================ FILE: src/backend/distributed/sql/udfs/pg_dist_shard_placement_trigger_func/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.pg_dist_shard_placement_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'DELETE') THEN DELETE FROM pg_dist_placement WHERE placementid = OLD.placementid; RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN UPDATE pg_dist_placement SET shardid = NEW.shardid, shardstate = NEW.shardstate, shardlength = NEW.shardlength, placementid = NEW.placementid, groupid = citus_internal.find_groupid_for_node(NEW.nodename, NEW.nodeport) WHERE placementid = OLD.placementid; RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO pg_dist_placement (placementid, shardid, shardstate, shardlength, groupid) VALUES (NEW.placementid, NEW.shardid, NEW.shardstate, NEW.shardlength, citus_internal.find_groupid_for_node(NEW.nodename, NEW.nodeport)); RETURN NEW; END IF; END; $$ LANGUAGE plpgsql; ================================================ FILE: src/backend/distributed/sql/udfs/pg_terminate_backend/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint DEFAULT 0) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$citus_terminate_backend$$; COMMENT ON FUNCTION pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint) IS 'terminates a Citus query which might be on any node in the Citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/pg_terminate_backend/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint) CASCADE; CREATE OR REPLACE FUNCTION pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint DEFAULT 0) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$citus_terminate_backend$$; COMMENT ON FUNCTION pg_catalog.pg_terminate_backend(global_pid bigint, timeout bigint) IS 'terminates a Citus query which might be on any node in the Citus cluster'; ================================================ FILE: src/backend/distributed/sql/udfs/read_intermediate_results/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.read_intermediate_results( result_ids text[], format pg_catalog.citus_copy_format default 'csv') RETURNS SETOF record LANGUAGE C STRICT VOLATILE PARALLEL SAFE AS 'MODULE_PATHNAME', $$read_intermediate_result_array$$; COMMENT ON FUNCTION pg_catalog.read_intermediate_results(text[],pg_catalog.citus_copy_format) IS 'read a set files and return them as a set of records'; ================================================ FILE: src/backend/distributed/sql/udfs/read_intermediate_results/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.read_intermediate_results( result_ids text[], format pg_catalog.citus_copy_format default 'csv') RETURNS SETOF record LANGUAGE C STRICT VOLATILE PARALLEL SAFE AS 'MODULE_PATHNAME', $$read_intermediate_result_array$$; COMMENT ON FUNCTION pg_catalog.read_intermediate_results(text[],pg_catalog.citus_copy_format) IS 'read a set files and return them as a set of records'; ================================================ FILE: src/backend/distributed/sql/udfs/rebalance_table_shards/9.0-1.sql ================================================ -- rebalance_table_shards uses the shard rebalancer's C UDF functions to rebalance -- shards of the given relation. -- CREATE OR REPLACE FUNCTION pg_catalog.rebalance_table_shards( relation regclass, threshold float4 default 0, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; COMMENT ON FUNCTION pg_catalog.rebalance_table_shards(regclass, float4, int, bigint[], citus.shard_transfer_mode) IS 'rebalance the shards of the given table across the worker nodes (including colocated shards of other tables)'; ================================================ FILE: src/backend/distributed/sql/udfs/rebalance_table_shards/9.1-1.sql ================================================ -- rebalance_table_shards uses the shard rebalancer's C UDF functions to rebalance -- shards of the given relation. -- DROP FUNCTION pg_catalog.rebalance_table_shards; CREATE OR REPLACE FUNCTION pg_catalog.rebalance_table_shards( relation regclass default NULL, threshold float4 default 0, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto', drain_only boolean default false) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.rebalance_table_shards(regclass, float4, int, bigint[], citus.shard_transfer_mode, boolean) IS 'rebalance the shards of the given table across the worker nodes (including colocated shards of other tables)'; ================================================ FILE: src/backend/distributed/sql/udfs/rebalance_table_shards/9.2-1.sql ================================================ -- rebalance_table_shards uses the shard rebalancer's C UDF functions to rebalance -- shards of the given relation. -- DROP FUNCTION pg_catalog.rebalance_table_shards; CREATE OR REPLACE FUNCTION pg_catalog.rebalance_table_shards( relation regclass default NULL, threshold float4 default NULL, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto', drain_only boolean default false, rebalance_strategy name default NULL ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.rebalance_table_shards(regclass, float4, int, bigint[], citus.shard_transfer_mode, boolean, name) IS 'rebalance the shards of the given table across the worker nodes (including colocated shards of other tables)'; ================================================ FILE: src/backend/distributed/sql/udfs/rebalance_table_shards/latest.sql ================================================ -- rebalance_table_shards uses the shard rebalancer's C UDF functions to rebalance -- shards of the given relation. -- DROP FUNCTION pg_catalog.rebalance_table_shards; CREATE OR REPLACE FUNCTION pg_catalog.rebalance_table_shards( relation regclass default NULL, threshold float4 default NULL, max_shard_moves int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto', drain_only boolean default false, rebalance_strategy name default NULL ) RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; COMMENT ON FUNCTION pg_catalog.rebalance_table_shards(regclass, float4, int, bigint[], citus.shard_transfer_mode, boolean, name) IS 'rebalance the shards of the given table across the worker nodes (including colocated shards of other tables)'; ================================================ FILE: src/backend/distributed/sql/udfs/remove_local_tables_from_metadata/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.remove_local_tables_from_metadata() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$remove_local_tables_from_metadata$$; COMMENT ON FUNCTION pg_catalog.remove_local_tables_from_metadata() IS 'undistribute citus local tables that are not chained with any reference tables via foreign keys'; ================================================ FILE: src/backend/distributed/sql/udfs/remove_local_tables_from_metadata/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.remove_local_tables_from_metadata() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$remove_local_tables_from_metadata$$; COMMENT ON FUNCTION pg_catalog.remove_local_tables_from_metadata() IS 'undistribute citus local tables that are not chained with any reference tables via foreign keys'; ================================================ FILE: src/backend/distributed/sql/udfs/repl_origin_helper/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_start_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() IS 'To start replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_stop_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() IS 'To stop replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_is_replication_origin_tracking_active$$; COMMENT ON FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() IS 'To check if replication origin tracking is active for skipping publishing of duplicated events during internal data movements for CDC'; ================================================ FILE: src/backend/distributed/sql/udfs/repl_origin_helper/13.1-1.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.start_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_start_replication_origin_tracking$$; COMMENT ON FUNCTION citus_internal.start_replication_origin_tracking() IS 'To start replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION citus_internal.stop_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_stop_replication_origin_tracking$$; COMMENT ON FUNCTION citus_internal.stop_replication_origin_tracking() IS 'To stop replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION citus_internal.is_replication_origin_tracking_active() RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_is_replication_origin_tracking_active$$; COMMENT ON FUNCTION citus_internal.is_replication_origin_tracking_active() IS 'To check if replication origin tracking is active for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_start_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() IS 'To start replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_stop_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() IS 'To stop replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_is_replication_origin_tracking_active$$; COMMENT ON FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() IS 'To check if replication origin tracking is active for skipping publishing of duplicated events during internal data movements for CDC'; ================================================ FILE: src/backend/distributed/sql/udfs/repl_origin_helper/latest.sql ================================================ CREATE OR REPLACE FUNCTION citus_internal.start_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_start_replication_origin_tracking$$; COMMENT ON FUNCTION citus_internal.start_replication_origin_tracking() IS 'To start replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION citus_internal.stop_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_stop_replication_origin_tracking$$; COMMENT ON FUNCTION citus_internal.stop_replication_origin_tracking() IS 'To stop replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION citus_internal.is_replication_origin_tracking_active() RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_is_replication_origin_tracking_active$$; COMMENT ON FUNCTION citus_internal.is_replication_origin_tracking_active() IS 'To check if replication origin tracking is active for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_start_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_start_replication_origin_tracking() IS 'To start replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_stop_replication_origin_tracking$$; COMMENT ON FUNCTION pg_catalog.citus_internal_stop_replication_origin_tracking() IS 'To stop replication origin tracking for skipping publishing of duplicated events during internal data movements for CDC'; CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() RETURNS boolean LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$citus_internal_is_replication_origin_tracking_active$$; COMMENT ON FUNCTION pg_catalog.citus_internal_is_replication_origin_tracking_active() IS 'To check if replication origin tracking is active for skipping publishing of duplicated events during internal data movements for CDC'; ================================================ FILE: src/backend/distributed/sql/udfs/replicate_reference_tables/11.1-1.sql ================================================ DROP FUNCTION pg_catalog.replicate_reference_tables; CREATE FUNCTION pg_catalog.replicate_reference_tables(shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$replicate_reference_tables$$; COMMENT ON FUNCTION pg_catalog.replicate_reference_tables(citus.shard_transfer_mode) IS 'replicate reference tables to all nodes'; REVOKE ALL ON FUNCTION pg_catalog.replicate_reference_tables(citus.shard_transfer_mode) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-2.sql ================================================ CREATE FUNCTION pg_catalog.replicate_reference_tables() RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$replicate_reference_tables$$; COMMENT ON FUNCTION pg_catalog.replicate_reference_tables() IS 'replicate reference tables to all nodes'; REVOKE ALL ON FUNCTION pg_catalog.replicate_reference_tables() FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/replicate_reference_tables/latest.sql ================================================ DROP FUNCTION pg_catalog.replicate_reference_tables; CREATE FUNCTION pg_catalog.replicate_reference_tables(shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$replicate_reference_tables$$; COMMENT ON FUNCTION pg_catalog.replicate_reference_tables(citus.shard_transfer_mode) IS 'replicate reference tables to all nodes'; REVOKE ALL ON FUNCTION pg_catalog.replicate_reference_tables(citus.shard_transfer_mode) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/replicate_table_shards/9.0-1.sql ================================================ -- replicate_table_shards uses the shard rebalancer's C UDF functions to replicate -- under-replicated shards of the given table. -- CREATE FUNCTION pg_catalog.replicate_table_shards( relation regclass, shard_replication_factor int default current_setting('citus.shard_replication_factor')::int, max_shard_copies int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.replicate_table_shards(regclass, int, int, bigint[], citus.shard_transfer_mode) IS 'replicates under replicated shards of the the given table'; ================================================ FILE: src/backend/distributed/sql/udfs/replicate_table_shards/latest.sql ================================================ -- replicate_table_shards uses the shard rebalancer's C UDF functions to replicate -- under-replicated shards of the given table. -- CREATE FUNCTION pg_catalog.replicate_table_shards( relation regclass, shard_replication_factor int default current_setting('citus.shard_replication_factor')::int, max_shard_copies int default 1000000, excluded_shard_list bigint[] default '{}', shard_transfer_mode citus.shard_transfer_mode default 'auto') RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C STRICT; COMMENT ON FUNCTION pg_catalog.replicate_table_shards(regclass, int, int, bigint[], citus.shard_transfer_mode) IS 'replicates under replicated shards of the the given table'; ================================================ FILE: src/backend/distributed/sql/udfs/run_command_on_all_nodes/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.run_command_on_all_nodes; CREATE FUNCTION pg_catalog.run_command_on_all_nodes(command text, parallel bool default true, give_warning_for_connection_errors bool default false, OUT nodeid int, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE nodenames text[]; ports int[]; commands text[]; current_node_is_in_metadata boolean; command_result_of_current_node text; BEGIN WITH citus_nodes AS ( SELECT * FROM pg_dist_node WHERE isactive = 't' AND nodecluster = current_setting('citus.cluster_name') AND ( (current_setting('citus.use_secondary_nodes') = 'never' AND noderole = 'primary') OR (current_setting('citus.use_secondary_nodes') = 'always' AND noderole = 'secondary') ) ORDER BY nodename, nodeport ) SELECT array_agg(citus_nodes.nodename), array_agg(citus_nodes.nodeport), array_agg(command) INTO nodenames, ports, commands FROM citus_nodes; SELECT count(*) > 0 FROM pg_dist_node WHERE isactive = 't' AND nodecluster = current_setting('citus.cluster_name') AND groupid IN (SELECT groupid FROM pg_dist_local_group) INTO current_node_is_in_metadata; -- This will happen when we call this function on coordinator and -- the coordinator is not added to the metadata. -- We'll manually add current node to the lists to actually run on all nodes. -- But when the coordinator is not added to metadata and this function -- is called from a worker node, this will not be enough and we'll -- not be able run on all nodes. IF NOT current_node_is_in_metadata THEN SELECT array_append(nodenames, current_setting('citus.local_hostname')), array_append(ports, current_setting('port')::int), array_append(commands, command) INTO nodenames, ports, commands; END IF; FOR nodeid, success, result IN SELECT coalesce(pg_dist_node.nodeid, 0) AS nodeid, mrow.success, mrow.result FROM master_run_on_worker(nodenames, ports, commands, parallel) mrow LEFT JOIN pg_dist_node ON mrow.node_name = pg_dist_node.nodename AND mrow.node_port = pg_dist_node.nodeport LOOP IF give_warning_for_connection_errors AND NOT success THEN RAISE WARNING 'Error on node with node id %: %', nodeid, result; END IF; RETURN NEXT; END LOOP; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/run_command_on_all_nodes/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.run_command_on_all_nodes; CREATE FUNCTION pg_catalog.run_command_on_all_nodes(command text, parallel bool default true, give_warning_for_connection_errors bool default false, OUT nodeid int, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE nodenames text[]; ports int[]; commands text[]; current_node_is_in_metadata boolean; command_result_of_current_node text; BEGIN WITH citus_nodes AS ( SELECT * FROM pg_dist_node WHERE isactive = 't' AND nodecluster = current_setting('citus.cluster_name') AND ( (current_setting('citus.use_secondary_nodes') = 'never' AND noderole = 'primary') OR (current_setting('citus.use_secondary_nodes') = 'always' AND noderole = 'secondary') ) ORDER BY nodename, nodeport ) SELECT array_agg(citus_nodes.nodename), array_agg(citus_nodes.nodeport), array_agg(command) INTO nodenames, ports, commands FROM citus_nodes; SELECT count(*) > 0 FROM pg_dist_node WHERE isactive = 't' AND nodecluster = current_setting('citus.cluster_name') AND groupid IN (SELECT groupid FROM pg_dist_local_group) INTO current_node_is_in_metadata; -- This will happen when we call this function on coordinator and -- the coordinator is not added to the metadata. -- We'll manually add current node to the lists to actually run on all nodes. -- But when the coordinator is not added to metadata and this function -- is called from a worker node, this will not be enough and we'll -- not be able run on all nodes. IF NOT current_node_is_in_metadata THEN SELECT array_append(nodenames, current_setting('citus.local_hostname')), array_append(ports, current_setting('port')::int), array_append(commands, command) INTO nodenames, ports, commands; END IF; FOR nodeid, success, result IN SELECT coalesce(pg_dist_node.nodeid, 0) AS nodeid, mrow.success, mrow.result FROM master_run_on_worker(nodenames, ports, commands, parallel) mrow LEFT JOIN pg_dist_node ON mrow.node_name = pg_dist_node.nodename AND mrow.node_port = pg_dist_node.nodeport LOOP IF give_warning_for_connection_errors AND NOT success THEN RAISE WARNING 'Error on node with node id %: %', nodeid, result; END IF; RETURN NEXT; END LOOP; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/run_command_on_coordinator/11.0-2.sql ================================================ -- run_command_on_coordinator tries to closely follow the semantics of run_command_on_all_nodes, -- but only runs the command on the coordinator CREATE FUNCTION pg_catalog.run_command_on_coordinator(command text, give_warning_for_connection_errors bool default false, OUT nodeid int, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE nodenames text[]; ports int[]; commands text[]; coordinator_is_in_metadata boolean; parallel boolean := false; BEGIN WITH citus_nodes AS ( SELECT * FROM pg_dist_node WHERE isactive AND nodecluster = current_setting('citus.cluster_name') AND groupid = 0 AND ( (current_setting('citus.use_secondary_nodes') = 'never' AND noderole = 'primary') OR (current_setting('citus.use_secondary_nodes') = 'always' AND noderole = 'secondary') ) ORDER BY nodename, nodeport ) SELECT array_agg(citus_nodes.nodename), array_agg(citus_nodes.nodeport), array_agg(command), count(*) > 0 FROM citus_nodes INTO nodenames, ports, commands, coordinator_is_in_metadata; IF NOT coordinator_is_in_metadata THEN -- This will happen when we call this function on coordinator and -- the coordinator is not added to the metadata. -- We'll manually add current node to the lists to actually run on all nodes. -- But when the coordinator is not added to metadata and this function -- is called from a worker node, this will not be enough and we'll -- not be able run on all nodes. IF citus_is_coordinator() THEN SELECT array_append(nodenames, current_setting('citus.local_hostname')), array_append(ports, current_setting('port')::int), array_append(commands, command) INTO nodenames, ports, commands; ELSE RAISE EXCEPTION 'the coordinator is not added to the metadata' USING HINT = 'Add the node as a coordinator by using: SELECT citus_set_coordinator_host('''')'; END IF; END IF; FOR nodeid, success, result IN SELECT coalesce(pg_dist_node.nodeid, 0) AS nodeid, mrow.success, mrow.result FROM master_run_on_worker(nodenames, ports, commands, parallel) mrow LEFT JOIN pg_dist_node ON mrow.node_name = pg_dist_node.nodename AND mrow.node_port = pg_dist_node.nodeport LOOP IF give_warning_for_connection_errors AND NOT success THEN RAISE WARNING 'Error on node with node id %: %', nodeid, result; END IF; RETURN NEXT; END LOOP; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/run_command_on_coordinator/latest.sql ================================================ -- run_command_on_coordinator tries to closely follow the semantics of run_command_on_all_nodes, -- but only runs the command on the coordinator CREATE FUNCTION pg_catalog.run_command_on_coordinator(command text, give_warning_for_connection_errors bool default false, OUT nodeid int, OUT success bool, OUT result text) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE nodenames text[]; ports int[]; commands text[]; coordinator_is_in_metadata boolean; parallel boolean := false; BEGIN WITH citus_nodes AS ( SELECT * FROM pg_dist_node WHERE isactive AND nodecluster = current_setting('citus.cluster_name') AND groupid = 0 AND ( (current_setting('citus.use_secondary_nodes') = 'never' AND noderole = 'primary') OR (current_setting('citus.use_secondary_nodes') = 'always' AND noderole = 'secondary') ) ORDER BY nodename, nodeport ) SELECT array_agg(citus_nodes.nodename), array_agg(citus_nodes.nodeport), array_agg(command), count(*) > 0 FROM citus_nodes INTO nodenames, ports, commands, coordinator_is_in_metadata; IF NOT coordinator_is_in_metadata THEN -- This will happen when we call this function on coordinator and -- the coordinator is not added to the metadata. -- We'll manually add current node to the lists to actually run on all nodes. -- But when the coordinator is not added to metadata and this function -- is called from a worker node, this will not be enough and we'll -- not be able run on all nodes. IF citus_is_coordinator() THEN SELECT array_append(nodenames, current_setting('citus.local_hostname')), array_append(ports, current_setting('port')::int), array_append(commands, command) INTO nodenames, ports, commands; ELSE RAISE EXCEPTION 'the coordinator is not added to the metadata' USING HINT = 'Add the node as a coordinator by using: SELECT citus_set_coordinator_host('''')'; END IF; END IF; FOR nodeid, success, result IN SELECT coalesce(pg_dist_node.nodeid, 0) AS nodeid, mrow.success, mrow.result FROM master_run_on_worker(nodenames, ports, commands, parallel) mrow LEFT JOIN pg_dist_node ON mrow.node_name = pg_dist_node.nodename AND mrow.node_port = pg_dist_node.nodeport LOOP IF give_warning_for_connection_errors AND NOT success THEN RAISE WARNING 'Error on node with node id %: %', nodeid, result; END IF; RETURN NEXT; END LOOP; END; $function$; ================================================ FILE: src/backend/distributed/sql/udfs/shard_name/13.1-1.sql ================================================ -- skip_qualify_public is set to true by default just for backward compatibility DROP FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint); CREATE FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint, skip_qualify_public boolean DEFAULT true) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$shard_name$$; COMMENT ON FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint, skip_qualify_public boolean) IS 'returns schema-qualified, shard-extended identifier of object name'; ================================================ FILE: src/backend/distributed/sql/udfs/shard_name/latest.sql ================================================ -- skip_qualify_public is set to true by default just for backward compatibility DROP FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint); CREATE FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint, skip_qualify_public boolean DEFAULT true) RETURNS text LANGUAGE C STABLE STRICT AS 'MODULE_PATHNAME', $$shard_name$$; COMMENT ON FUNCTION pg_catalog.shard_name(object_name regclass, shard_id bigint, skip_qualify_public boolean) IS 'returns schema-qualified, shard-extended identifier of object name'; ================================================ FILE: src/backend/distributed/sql/udfs/start_metadata_sync_to_all_nodes/11.0-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$start_metadata_sync_to_all_nodes$$; COMMENT ON FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() IS 'sync metadata to all active primary nodes'; REVOKE ALL ON FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/start_metadata_sync_to_all_nodes/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$start_metadata_sync_to_all_nodes$$; COMMENT ON FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() IS 'sync metadata to all active primary nodes'; REVOKE ALL ON FUNCTION pg_catalog.start_metadata_sync_to_all_nodes() FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/stop_metadata_sync_to_node/10.2-1.sql ================================================ CREATE FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer, clear_metadata bool DEFAULT true) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$stop_metadata_sync_to_node$$; COMMENT ON FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer, clear_metadata bool) IS 'stop metadata sync to node'; ================================================ FILE: src/backend/distributed/sql/udfs/stop_metadata_sync_to_node/latest.sql ================================================ CREATE FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer, clear_metadata bool DEFAULT true) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$stop_metadata_sync_to_node$$; COMMENT ON FUNCTION pg_catalog.stop_metadata_sync_to_node(nodename text, nodeport integer, clear_metadata bool) IS 'stop metadata sync to node'; ================================================ FILE: src/backend/distributed/sql/udfs/time_partition_range/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.time_partition_range( table_name regclass, OUT lower_bound text, OUT upper_bound text) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$time_partition_range$$; COMMENT ON FUNCTION pg_catalog.time_partition_range(regclass) IS 'returns the start and end of partition boundaries'; ================================================ FILE: src/backend/distributed/sql/udfs/time_partition_range/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.time_partition_range( table_name regclass, OUT lower_bound text, OUT upper_bound text) RETURNS record LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$time_partition_range$$; COMMENT ON FUNCTION pg_catalog.time_partition_range(regclass) IS 'returns the start and end of partition boundaries'; ================================================ FILE: src/backend/distributed/sql/udfs/time_partitions/10.0-1.sql ================================================ CREATE VIEW citus.time_partitions AS SELECT partrelid AS parent_table, attname AS partition_column, relid AS partition, lower_bound AS from_value, upper_bound AS to_value, amname AS access_method FROM ( SELECT partrelid::regclass AS partrelid, attname, c.oid::regclass AS relid, lower_bound, upper_bound, amname FROM pg_class c JOIN pg_inherits i ON (c.oid = inhrelid) JOIN pg_partitioned_table p ON (inhparent = partrelid) JOIN pg_attribute a ON (partrelid = attrelid) JOIN pg_type t ON (atttypid = t.oid) JOIN pg_namespace tn ON (t.typnamespace = tn.oid) LEFT JOIN pg_am am ON (c.relam = am.oid), pg_catalog.time_partition_range(c.oid) WHERE c.relpartbound IS NOT NULL AND p.partstrat = 'r' AND p.partnatts = 1 AND a.attnum = ANY(partattrs::int2[]) ) partitions ORDER BY partrelid::text, lower_bound; ALTER VIEW citus.time_partitions SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.time_partitions TO public; ================================================ FILE: src/backend/distributed/sql/udfs/time_partitions/latest.sql ================================================ CREATE VIEW citus.time_partitions AS SELECT partrelid AS parent_table, attname AS partition_column, relid AS partition, lower_bound AS from_value, upper_bound AS to_value, amname AS access_method FROM ( SELECT partrelid::regclass AS partrelid, attname, c.oid::regclass AS relid, lower_bound, upper_bound, amname FROM pg_class c JOIN pg_inherits i ON (c.oid = inhrelid) JOIN pg_partitioned_table p ON (inhparent = partrelid) JOIN pg_attribute a ON (partrelid = attrelid) JOIN pg_type t ON (atttypid = t.oid) JOIN pg_namespace tn ON (t.typnamespace = tn.oid) LEFT JOIN pg_am am ON (c.relam = am.oid), pg_catalog.time_partition_range(c.oid) WHERE c.relpartbound IS NOT NULL AND p.partstrat = 'r' AND p.partnatts = 1 AND a.attnum = ANY(partattrs::int2[]) ) partitions ORDER BY partrelid::text, lower_bound; ALTER VIEW citus.time_partitions SET SCHEMA pg_catalog; GRANT SELECT ON pg_catalog.time_partitions TO public; ================================================ FILE: src/backend/distributed/sql/udfs/truncate_local_data_after_distributing_table/9.3-2.sql ================================================ CREATE OR REPLACE FUNCTION truncate_local_data_after_distributing_table(function_name regclass) RETURNS void LANGUAGE C CALLED ON NULL INPUT AS 'MODULE_PATHNAME', $$truncate_local_data_after_distributing_table$$; COMMENT ON FUNCTION truncate_local_data_after_distributing_table(function_name regclass) IS 'truncates local records of a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/truncate_local_data_after_distributing_table/latest.sql ================================================ CREATE OR REPLACE FUNCTION truncate_local_data_after_distributing_table(function_name regclass) RETURNS void LANGUAGE C CALLED ON NULL INPUT AS 'MODULE_PATHNAME', $$truncate_local_data_after_distributing_table$$; COMMENT ON FUNCTION truncate_local_data_after_distributing_table(function_name regclass) IS 'truncates local records of a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/undistribute_table/10.0-1.sql ================================================ DROP FUNCTION pg_catalog.undistribute_table(regclass); CREATE OR REPLACE FUNCTION pg_catalog.undistribute_table( table_name regclass, cascade_via_foreign_keys boolean default false) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$undistribute_table$$; COMMENT ON FUNCTION pg_catalog.undistribute_table( table_name regclass, cascade_via_foreign_keys boolean) IS 'undistributes a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/undistribute_table/9.5-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.undistribute_table( table_name regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$undistribute_table$$; COMMENT ON FUNCTION pg_catalog.undistribute_table( table_name regclass) IS 'undistributes a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/undistribute_table/latest.sql ================================================ DROP FUNCTION pg_catalog.undistribute_table(regclass); CREATE OR REPLACE FUNCTION pg_catalog.undistribute_table( table_name regclass, cascade_via_foreign_keys boolean default false) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$undistribute_table$$; COMMENT ON FUNCTION pg_catalog.undistribute_table( table_name regclass, cascade_via_foreign_keys boolean) IS 'undistributes a distributed table'; ================================================ FILE: src/backend/distributed/sql/udfs/update_distributed_table_colocation/9.3-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.update_distributed_table_colocation(table_name regclass, colocate_with text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$update_distributed_table_colocation$$; COMMENT ON FUNCTION pg_catalog.update_distributed_table_colocation(table_name regclass, colocate_with text) IS 'updates colocation of a table'; ================================================ FILE: src/backend/distributed/sql/udfs/update_distributed_table_colocation/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.update_distributed_table_colocation(table_name regclass, colocate_with text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$update_distributed_table_colocation$$; COMMENT ON FUNCTION pg_catalog.update_distributed_table_colocation(table_name regclass, colocate_with text) IS 'updates colocation of a table'; ================================================ FILE: src/backend/distributed/sql/udfs/upgrade_to_reference_table/8.0-1.sql ================================================ CREATE FUNCTION pg_catalog.upgrade_to_reference_table(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$upgrade_to_reference_table$$; COMMENT ON FUNCTION pg_catalog.upgrade_to_reference_table(table_name regclass) IS 'upgrades an existing broadcast table to a reference table'; ================================================ FILE: src/backend/distributed/sql/udfs/upgrade_to_reference_table/latest.sql ================================================ CREATE FUNCTION pg_catalog.upgrade_to_reference_table(table_name regclass) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$upgrade_to_reference_table$$; COMMENT ON FUNCTION pg_catalog.upgrade_to_reference_table(table_name regclass) IS 'upgrades an existing broadcast table to a reference table'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/11.2-2.sql ================================================ -- Since we backported the UDF below from version 11.3, the definition is the same #include "11.3-1.sql" ================================================ FILE: src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/11.3-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_adjust_identity_column_seq_ranges$$; COMMENT ON FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) IS 'modify identity column seq ranges to produce globally unique values'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_adjust_identity_column_seq_ranges/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_adjust_identity_column_seq_ranges$$; COMMENT ON FUNCTION pg_catalog.worker_adjust_identity_column_seq_ranges(regclass) IS 'modify identity column seq ranges to produce globally unique values'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_binary_partial_agg/14.0-1.sql ================================================ -- similar to worker_partial_agg but returns binary representation of the state CREATE AGGREGATE pg_catalog.worker_binary_partial_agg(oid, anyelement) ( STYPE = internal, SFUNC = pg_catalog.worker_partial_agg_sfunc, FINALFUNC = pg_catalog.worker_binary_partial_agg_ffunc ); COMMENT ON AGGREGATE pg_catalog.worker_binary_partial_agg(oid, anyelement) IS 'support aggregate for implementing partial binary aggregation on workers'; REVOKE ALL ON FUNCTION pg_catalog.worker_binary_partial_agg FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_binary_partial_agg TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_binary_partial_agg/latest.sql ================================================ -- similar to worker_partial_agg but returns binary representation of the state CREATE AGGREGATE pg_catalog.worker_binary_partial_agg(oid, anyelement) ( STYPE = internal, SFUNC = pg_catalog.worker_partial_agg_sfunc, FINALFUNC = pg_catalog.worker_binary_partial_agg_ffunc ); COMMENT ON AGGREGATE pg_catalog.worker_binary_partial_agg(oid, anyelement) IS 'support aggregate for implementing partial binary aggregation on workers'; REVOKE ALL ON FUNCTION pg_catalog.worker_binary_partial_agg FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_binary_partial_agg TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_binary_partial_agg_ffunc/14.0-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_binary_partial_agg_ffunc(internal) RETURNS bytea AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc(internal) IS 'finalizer for worker_binary_partial_agg'; REVOKE ALL ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_binary_partial_agg_ffunc/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_binary_partial_agg_ffunc(internal) RETURNS bytea AS 'MODULE_PATHNAME' LANGUAGE C PARALLEL SAFE; COMMENT ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc(internal) IS 'finalizer for worker_binary_partial_agg'; REVOKE ALL ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc FROM PUBLIC; GRANT EXECUTE ON FUNCTION pg_catalog.worker_binary_partial_agg_ffunc TO PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_change_sequence_dependency/10.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_change_sequence_dependency( sequence regclass, source_table regclass, target_table regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_change_sequence_dependency$$; COMMENT ON FUNCTION pg_catalog.worker_change_sequence_dependency( sequence regclass, source_table regclass, target_table regclass) IS 'changes sequence''s dependency from source table to target table'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_change_sequence_dependency/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_change_sequence_dependency( sequence regclass, source_table regclass, target_table regclass) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_change_sequence_dependency$$; COMMENT ON FUNCTION pg_catalog.worker_change_sequence_dependency( sequence regclass, source_table regclass, target_table regclass) IS 'changes sequence''s dependency from source table to target table'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_copy_table_to_node/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_copy_table_to_node( source_table regclass, target_node_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_copy_table_to_node$$; COMMENT ON FUNCTION pg_catalog.worker_copy_table_to_node(regclass, integer) IS 'Perform copy of a shard'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_copy_table_to_node/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_copy_table_to_node( source_table regclass, target_node_id integer) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_copy_table_to_node$$; COMMENT ON FUNCTION pg_catalog.worker_copy_table_to_node(regclass, integer) IS 'Perform copy of a shard'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_or_alter_role/9.3-2.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_alter_role( role_name text, create_role_utility_query text, alter_role_utility_query text) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$worker_create_or_alter_role$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_alter_role( role_name text, create_role_utility_query text, alter_role_utility_query text) IS 'runs the create role query, if the role doesn''t exists, runs the alter role query if it does'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_or_alter_role/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_alter_role( role_name text, create_role_utility_query text, alter_role_utility_query text) RETURNS BOOL LANGUAGE C AS 'MODULE_PATHNAME', $$worker_create_or_alter_role$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_alter_role( role_name text, create_role_utility_query text, alter_role_utility_query text) IS 'runs the create role query, if the role doesn''t exists, runs the alter role query if it does'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_or_replace_object/11.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_replace_object(statement text) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_or_replace_object$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_replace_object(statement text) IS 'takes a sql CREATE statement, before executing the create it will check if an object with that name already exists and safely replaces that named object with the new object'; CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_replace_object(statements text[]) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_or_replace_object_array$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_replace_object(statements text[]) IS 'takes an array of sql statements, before executing these it will check if the object already exists in that exact state otherwise replaces that named object with the new object'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_or_replace_object/9.0-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_replace_object(statement text) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_or_replace_object$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_replace_object(statement text) IS 'takes a sql CREATE statement, before executing the create it will check if an object with that name already exists and safely replaces that named object with the new object'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_or_replace_object/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_replace_object(statement text) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_or_replace_object$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_replace_object(statement text) IS 'takes a sql CREATE statement, before executing the create it will check if an object with that name already exists and safely replaces that named object with the new object'; CREATE OR REPLACE FUNCTION pg_catalog.worker_create_or_replace_object(statements text[]) RETURNS bool LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_or_replace_object_array$$; COMMENT ON FUNCTION pg_catalog.worker_create_or_replace_object(statements text[]) IS 'takes an array of sql statements, before executing these it will check if the object already exists in that exact state otherwise replaces that named object with the new object'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_schema/9.1-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_create_schema(bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_schema$$; COMMENT ON FUNCTION pg_catalog.worker_create_schema(bigint) IS 'create schema in remote node'; REVOKE ALL ON FUNCTION pg_catalog.worker_create_schema(bigint) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_schema/9.2-2.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.worker_create_schema(jobid bigint); CREATE FUNCTION pg_catalog.worker_create_schema(jobid bigint, username text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_schema$$; COMMENT ON FUNCTION pg_catalog.worker_create_schema(bigint, text) IS 'create schema in remote node'; REVOKE ALL ON FUNCTION pg_catalog.worker_create_schema(bigint, text) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_create_schema/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.worker_create_schema(jobid bigint); CREATE FUNCTION pg_catalog.worker_create_schema(jobid bigint, username text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_create_schema$$; COMMENT ON FUNCTION pg_catalog.worker_create_schema(bigint, text) IS 'create schema in remote node'; REVOKE ALL ON FUNCTION pg_catalog.worker_create_schema(bigint, text) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_all_shell_tables/11.3-1.sql ================================================ -- During metadata sync, when we send many ddls over single transaction, worker node can error due -- to reaching at max allocation block size for invalidation messages. To find a workaround for the problem, -- we added nontransactional metadata sync mode where we create many transaction while dropping shell tables -- via https://github.com/citusdata/citus/pull/6728. CREATE OR REPLACE PROCEDURE pg_catalog.worker_drop_all_shell_tables(singleTransaction bool DEFAULT true) LANGUAGE plpgsql AS $$ DECLARE table_name text; BEGIN -- drop shell tables within single or multiple transactions according to the flag singleTransaction FOR table_name IN SELECT logicalrelid::regclass::text FROM pg_dist_partition LOOP PERFORM pg_catalog.worker_drop_shell_table(table_name); IF not singleTransaction THEN COMMIT; END IF; END LOOP; END; $$; COMMENT ON PROCEDURE worker_drop_all_shell_tables(singleTransaction bool) IS 'drop all distributed tables only without the metadata within single transaction or ' 'multiple transaction specified by the flag singleTransaction'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_all_shell_tables/latest.sql ================================================ -- During metadata sync, when we send many ddls over single transaction, worker node can error due -- to reaching at max allocation block size for invalidation messages. To find a workaround for the problem, -- we added nontransactional metadata sync mode where we create many transaction while dropping shell tables -- via https://github.com/citusdata/citus/pull/6728. CREATE OR REPLACE PROCEDURE pg_catalog.worker_drop_all_shell_tables(singleTransaction bool DEFAULT true) LANGUAGE plpgsql AS $$ DECLARE table_name text; BEGIN -- drop shell tables within single or multiple transactions according to the flag singleTransaction FOR table_name IN SELECT logicalrelid::regclass::text FROM pg_dist_partition LOOP PERFORM pg_catalog.worker_drop_shell_table(table_name); IF not singleTransaction THEN COMMIT; END IF; END LOOP; END; $$; COMMENT ON PROCEDURE worker_drop_all_shell_tables(singleTransaction bool) IS 'drop all distributed tables only without the metadata within single transaction or ' 'multiple transaction specified by the flag singleTransaction'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_sequence_dependency/11.0-1.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.worker_drop_sequence_dependency(table_name text); CREATE OR REPLACE FUNCTION pg_catalog.worker_drop_sequence_dependency(table_name text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_sequence_dependency$$; COMMENT ON FUNCTION pg_catalog.worker_drop_sequence_dependency(table_name text) IS 'drop the Citus tables sequence dependency'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_sequence_dependency/latest.sql ================================================ DROP FUNCTION IF EXISTS pg_catalog.worker_drop_sequence_dependency(table_name text); CREATE OR REPLACE FUNCTION pg_catalog.worker_drop_sequence_dependency(table_name text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_sequence_dependency$$; COMMENT ON FUNCTION pg_catalog.worker_drop_sequence_dependency(table_name text) IS 'drop the Citus tables sequence dependency'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_shell_table/11.0-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_drop_shell_table(table_name text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_shell_table$$; COMMENT ON FUNCTION worker_drop_shell_table(table_name text) IS 'drop the distributed table only without the metadata'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_drop_shell_table/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_drop_shell_table(table_name text) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_drop_shell_table$$; COMMENT ON FUNCTION worker_drop_shell_table(table_name text) IS 'drop the distributed table only without the metadata'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_fix_partition_shard_index_names/10.2-4.sql ================================================ CREATE FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass, partition_shard text, new_partition_shard_index_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fix_partition_shard_index_names$$; COMMENT ON FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass, partition_shard text, new_partition_shard_index_name text) IS 'fix the name of the index on given partition shard that is child of given parent_index'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_fix_partition_shard_index_names/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass, partition_shard text, new_partition_shard_index_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fix_partition_shard_index_names$$; COMMENT ON FUNCTION pg_catalog.worker_fix_partition_shard_index_names(parent_shard_index regclass, partition_shard text, new_partition_shard_index_name text) IS 'fix the name of the index on given partition shard that is child of given parent_index'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_fix_pre_citus10_partitioned_table_constraint_names/10.0-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_fix_pre_citus10_partitioned_table_constraint_names(table_name regclass, shardid bigint, constraint_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fix_pre_citus10_partitioned_table_constraint_names$$; COMMENT ON FUNCTION pg_catalog.worker_fix_pre_citus10_partitioned_table_constraint_names(table_name regclass, shardid bigint, constraint_name text) IS 'fix constraint names on partition shards on worker nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_fix_pre_citus10_partitioned_table_constraint_names/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_fix_pre_citus10_partitioned_table_constraint_names(table_name regclass, shardid bigint, constraint_name text) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_fix_pre_citus10_partitioned_table_constraint_names$$; COMMENT ON FUNCTION pg_catalog.worker_fix_pre_citus10_partitioned_table_constraint_names(table_name regclass, shardid bigint, constraint_name text) IS 'fix constraint names on partition shards on worker nodes'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_last_saved_explain_analyze/13.2-1.sql ================================================ DROP FUNCTION pg_catalog.worker_last_saved_explain_analyze(); CREATE OR REPLACE FUNCTION pg_catalog.worker_last_saved_explain_analyze() RETURNS TABLE(explain_analyze_output TEXT, execution_duration DOUBLE PRECISION, execution_ntuples DOUBLE PRECISION, execution_nloops DOUBLE PRECISION) LANGUAGE C STRICT AS 'citus'; COMMENT ON FUNCTION pg_catalog.worker_last_saved_explain_analyze() IS 'Returns the saved explain analyze output for the last run query'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_last_saved_explain_analyze/9.4-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_last_saved_explain_analyze() RETURNS TABLE(explain_analyze_output TEXT, execution_duration DOUBLE PRECISION) LANGUAGE C STRICT AS 'citus'; COMMENT ON FUNCTION pg_catalog.worker_last_saved_explain_analyze() IS 'Returns the saved explain analyze output for the last run query'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_last_saved_explain_analyze/latest.sql ================================================ DROP FUNCTION pg_catalog.worker_last_saved_explain_analyze(); CREATE OR REPLACE FUNCTION pg_catalog.worker_last_saved_explain_analyze() RETURNS TABLE(explain_analyze_output TEXT, execution_duration DOUBLE PRECISION, execution_ntuples DOUBLE PRECISION, execution_nloops DOUBLE PRECISION) LANGUAGE C STRICT AS 'citus'; COMMENT ON FUNCTION pg_catalog.worker_last_saved_explain_analyze() IS 'Returns the saved explain analyze output for the last run query'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_nextval/10.2-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_nextval(sequence regclass) RETURNS int LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_nextval$$; COMMENT ON FUNCTION pg_catalog.worker_nextval(regclass) IS 'calculates nextval() for column defaults of type int or smallint'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_nextval/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_nextval(sequence regclass) RETURNS int LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_nextval$$; COMMENT ON FUNCTION pg_catalog.worker_nextval(regclass) IS 'calculates nextval() for column defaults of type int or smallint'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partition_query_result/11.0-1.sql ================================================ DROP FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean); CREATE OR REPLACE FUNCTION pg_catalog.worker_partition_query_result( result_prefix text, query text, partition_column_index int, partition_method citus.distribution_type, partition_min_values text[], partition_max_values text[], binary_copy boolean, allow_null_partition_column boolean DEFAULT false, generate_empty_results boolean DEFAULT false, OUT partition_index int, OUT rows_written bigint, OUT bytes_written bigint) RETURNS SETOF record LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$worker_partition_query_result$$; COMMENT ON FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean, boolean, boolean) IS 'execute a query and partitions its results in set of local result files'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partition_query_result/9.2-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_partition_query_result( result_prefix text, query text, partition_column_index int, partition_method citus.distribution_type, partition_min_values text[], partition_max_values text[], binaryCopy boolean, OUT partition_index int, OUT rows_written bigint, OUT bytes_written bigint) RETURNS SETOF record LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$worker_partition_query_result$$; COMMENT ON FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean) IS 'execute a query and partitions its results in set of local result files'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partition_query_result/latest.sql ================================================ DROP FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean); CREATE OR REPLACE FUNCTION pg_catalog.worker_partition_query_result( result_prefix text, query text, partition_column_index int, partition_method citus.distribution_type, partition_min_values text[], partition_max_values text[], binary_copy boolean, allow_null_partition_column boolean DEFAULT false, generate_empty_results boolean DEFAULT false, OUT partition_index int, OUT rows_written bigint, OUT bytes_written bigint) RETURNS SETOF record LANGUAGE C STRICT VOLATILE AS 'MODULE_PATHNAME', $$worker_partition_query_result$$; COMMENT ON FUNCTION pg_catalog.worker_partition_query_result(text, text, int, citus.distribution_type, text[], text[], boolean, boolean, boolean) IS 'execute a query and partitions its results in set of local result files'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_relation_size/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_relation_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_relation_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION pg_catalog.worker_partitioned_relation_size(regclass) IS 'Calculates and returns the size of a partitioned relation'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_relation_size/latest.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_relation_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_relation_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION pg_catalog.worker_partitioned_relation_size(regclass) IS 'Calculates and returns the size of a partitioned relation'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_relation_total_size/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_relation_total_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_total_relation_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION worker_partitioned_relation_total_size(regclass) IS 'Calculates and returns the total size of a partitioned relation'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_relation_total_size/latest.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_relation_total_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_total_relation_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION worker_partitioned_relation_total_size(regclass) IS 'Calculates and returns the total size of a partitioned relation'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_table_size/10.1-1.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_table_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_table_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION pg_catalog.worker_partitioned_table_size(regclass) IS 'Calculates and returns the size of a partitioned table'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_partitioned_table_size/latest.sql ================================================ CREATE OR REPLACE FUNCTION worker_partitioned_table_size(relation regclass) RETURNS bigint AS $$ SELECT sum(pg_table_size(relid))::bigint FROM (SELECT relid from pg_partition_tree(relation)) partition_tree; $$ LANGUAGE SQL; COMMENT ON FUNCTION pg_catalog.worker_partitioned_table_size(regclass) IS 'Calculates and returns the size of a partitioned table'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_record_sequence_dependency/9.5-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_record_sequence_dependency(seq_name regclass, table_name regclass, column_name name) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'worker_record_sequence_dependency'; COMMENT ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) IS 'record the fact that the sequence depends on the table in pg_depend'; REVOKE ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_record_sequence_dependency/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_record_sequence_dependency(seq_name regclass, table_name regclass, column_name name) RETURNS VOID LANGUAGE C STRICT AS 'MODULE_PATHNAME', 'worker_record_sequence_dependency'; COMMENT ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) IS 'record the fact that the sequence depends on the table in pg_depend'; REVOKE ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regclass,name) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_repartition_cleanup/9.1-1.sql ================================================ CREATE FUNCTION pg_catalog.worker_repartition_cleanup(bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_repartition_cleanup$$; COMMENT ON FUNCTION pg_catalog.worker_repartition_cleanup(bigint) IS 'remove job in remote node'; REVOKE ALL ON FUNCTION pg_catalog.worker_repartition_cleanup(bigint) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_repartition_cleanup/latest.sql ================================================ CREATE FUNCTION pg_catalog.worker_repartition_cleanup(bigint) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_repartition_cleanup$$; COMMENT ON FUNCTION pg_catalog.worker_repartition_cleanup(bigint) IS 'remove job in remote node'; REVOKE ALL ON FUNCTION pg_catalog.worker_repartition_cleanup(bigint) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_save_query_explain_analyze/9.4-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_save_query_explain_analyze( query text, options jsonb) RETURNS SETOF record LANGUAGE C STRICT AS 'citus'; COMMENT ON FUNCTION pg_catalog.worker_save_query_explain_analyze(text, jsonb) IS 'Executes and returns results of query while saving its EXPLAIN ANALYZE to be fetched later'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_save_query_explain_analyze/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_save_query_explain_analyze( query text, options jsonb) RETURNS SETOF record LANGUAGE C STRICT AS 'citus'; COMMENT ON FUNCTION pg_catalog.worker_save_query_explain_analyze(text, jsonb) IS 'Executes and returns results of query while saving its EXPLAIN ANALYZE to be fetched later'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_copy/11.1-1.sql ================================================ -- We want to create the type in pg_catalog but doing that leads to an error -- "ERROR: permission denied to create "pg_catalog.split_copy_info" -- "DETAIL: System catalog modifications are currently disallowed. "" -- As a workaround, we create the type in the citus schema and then later modify it to pg_catalog. DROP TYPE IF EXISTS citus.split_copy_info; CREATE TYPE citus.split_copy_info AS ( destination_shard_id bigint, destination_shard_min_value text, destination_shard_max_value text, -- A 'nodeId' is a uint32 in CITUS [1, 4294967296] but postgres does not have unsigned type support. -- Use integer (consistent with other previously defined UDFs that take nodeId as integer) as for all practical purposes it is big enough. destination_shard_node_id integer); ALTER TYPE citus.split_copy_info SET SCHEMA pg_catalog; CREATE OR REPLACE FUNCTION pg_catalog.worker_split_copy( source_shard_id bigint, distribution_column text, splitCopyInfos pg_catalog.split_copy_info[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_copy$$; COMMENT ON FUNCTION pg_catalog.worker_split_copy(source_shard_id bigint, distribution_column text, splitCopyInfos pg_catalog.split_copy_info[]) IS 'Perform split copy for shard'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_copy/latest.sql ================================================ -- We want to create the type in pg_catalog but doing that leads to an error -- "ERROR: permission denied to create "pg_catalog.split_copy_info" -- "DETAIL: System catalog modifications are currently disallowed. "" -- As a workaround, we create the type in the citus schema and then later modify it to pg_catalog. DROP TYPE IF EXISTS citus.split_copy_info; CREATE TYPE citus.split_copy_info AS ( destination_shard_id bigint, destination_shard_min_value text, destination_shard_max_value text, -- A 'nodeId' is a uint32 in CITUS [1, 4294967296] but postgres does not have unsigned type support. -- Use integer (consistent with other previously defined UDFs that take nodeId as integer) as for all practical purposes it is big enough. destination_shard_node_id integer); ALTER TYPE citus.split_copy_info SET SCHEMA pg_catalog; CREATE OR REPLACE FUNCTION pg_catalog.worker_split_copy( source_shard_id bigint, distribution_column text, splitCopyInfos pg_catalog.split_copy_info[]) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_copy$$; COMMENT ON FUNCTION pg_catalog.worker_split_copy(source_shard_id bigint, distribution_column text, splitCopyInfos pg_catalog.split_copy_info[]) IS 'Perform split copy for shard'; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_shard_release_dsm/11.1-1.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_split_shard_release_dsm() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_shard_release_dsm$$; COMMENT ON FUNCTION pg_catalog.worker_split_shard_release_dsm() IS 'Releases shared memory segment allocated by non-blocking split workflow'; REVOKE ALL ON FUNCTION pg_catalog.worker_split_shard_release_dsm() FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_shard_release_dsm/latest.sql ================================================ CREATE OR REPLACE FUNCTION pg_catalog.worker_split_shard_release_dsm() RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_shard_release_dsm$$; COMMENT ON FUNCTION pg_catalog.worker_split_shard_release_dsm() IS 'Releases shared memory segment allocated by non-blocking split workflow'; REVOKE ALL ON FUNCTION pg_catalog.worker_split_shard_release_dsm() FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_shard_replication_setup/11.1-1.sql ================================================ CREATE TYPE citus.split_shard_info AS ( source_shard_id bigint, distribution_column text, child_shard_id bigint, shard_min_value text, shard_max_value text, node_id integer); ALTER TYPE citus.split_shard_info SET SCHEMA pg_catalog; COMMENT ON TYPE pg_catalog.split_shard_info IS 'Stores split child shard information'; CREATE TYPE citus.replication_slot_info AS(node_id integer, slot_owner text, slot_name text); ALTER TYPE citus.replication_slot_info SET SCHEMA pg_catalog; COMMENT ON TYPE pg_catalog.replication_slot_info IS 'Replication slot information to be used for subscriptions during non blocking shard split'; CREATE OR REPLACE FUNCTION pg_catalog.worker_split_shard_replication_setup( splitShardInfo pg_catalog.split_shard_info[]) RETURNS setof pg_catalog.replication_slot_info LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_shard_replication_setup$$; COMMENT ON FUNCTION pg_catalog.worker_split_shard_replication_setup(splitShardInfo pg_catalog.split_shard_info[]) IS 'Replication setup for splitting a shard'; REVOKE ALL ON FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[]) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_shard_replication_setup/11.2-1.sql ================================================ DROP FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[]); CREATE OR REPLACE FUNCTION pg_catalog.worker_split_shard_replication_setup( splitShardInfo pg_catalog.split_shard_info[], operation_id bigint) RETURNS setof pg_catalog.replication_slot_info LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_shard_replication_setup$$; COMMENT ON FUNCTION pg_catalog.worker_split_shard_replication_setup(splitShardInfo pg_catalog.split_shard_info[], operation_id bigint) IS 'Replication setup for splitting a shard'; REVOKE ALL ON FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[], bigint) FROM PUBLIC; ================================================ FILE: src/backend/distributed/sql/udfs/worker_split_shard_replication_setup/latest.sql ================================================ DROP FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[]); CREATE OR REPLACE FUNCTION pg_catalog.worker_split_shard_replication_setup( splitShardInfo pg_catalog.split_shard_info[], operation_id bigint) RETURNS setof pg_catalog.replication_slot_info LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$worker_split_shard_replication_setup$$; COMMENT ON FUNCTION pg_catalog.worker_split_shard_replication_setup(splitShardInfo pg_catalog.split_shard_info[], operation_id bigint) IS 'Replication setup for splitting a shard'; REVOKE ALL ON FUNCTION pg_catalog.worker_split_shard_replication_setup(pg_catalog.split_shard_info[], bigint) FROM PUBLIC; ================================================ FILE: src/backend/distributed/stats/query_stats.c ================================================ /*------------------------------------------------------------------------- * * query_stats.c * Statement-level statistics for distributed queries. * Code is mostly taken from postgres/contrib/pg_stat_statements * and adapted to citus. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "safe_lib.h" #include "access/hash.h" #include "catalog/pg_authid.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/spin.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" #include "distributed/function_utils.h" #include "distributed/hash_helpers.h" #include "distributed/multi_executor.h" #include "distributed/multi_server_executor.h" #include "distributed/stats/query_stats.h" #include "distributed/tuplestore.h" #include "distributed/version_compat.h" #define CITUS_STATS_DUMP_FILE "pg_stat/citus_query_stats.stat" #define CITUS_STAT_STATEMENTS_COLS 6 #define CITUS_STAT_STATAMENTS_QUERY_ID 0 #define CITUS_STAT_STATAMENTS_USER_ID 1 #define CITUS_STAT_STATAMENTS_DB_ID 2 #define CITUS_STAT_STATAMENTS_EXECUTOR_TYPE 3 #define CITUS_STAT_STATAMENTS_PARTITION_KEY 4 #define CITUS_STAT_STATAMENTS_CALLS 5 #define USAGE_DECREASE_FACTOR (0.99) /* decreased every CitusQueryStatsEntryDealloc */ #define STICKY_DECREASE_FACTOR (0.50) /* factor for sticky entries */ #define USAGE_DEALLOC_PERCENT 5 /* free this % of entries at once */ #define USAGE_INIT (1.0) /* including initial planning */ #define MAX_KEY_LENGTH NAMEDATALEN static const uint32 CITUS_QUERY_STATS_FILE_HEADER = 0x0d756e0f; /* time interval in seconds for maintenance daemon to call CitusQueryStatsSynchronizeEntries */ int StatStatementsPurgeInterval = 10; /* maximum number of entries in queryStats hash, controlled by GUC citus.stat_statements_max */ int StatStatementsMax = 50000; /* tracking all or none, for citus_stat_statements, controlled by GUC citus.stat_statements_track */ int StatStatementsTrack = STAT_STATEMENTS_TRACK_NONE; /* * Hashtable key that defines the identity of a hashtable entry. We use the * same hash as pg_stat_statements */ typedef struct QueryStatsHashKey { Oid userid; /* user OID */ Oid dbid; /* database OID */ uint64 queryid; /* query identifier */ MultiExecutorType executorType; /* executor type */ char partitionKey[MAX_KEY_LENGTH]; } QueryStatsHashKey; /* * Statistics per query and executor type */ typedef struct queryStatsEntry { QueryStatsHashKey key; /* hash key of entry - MUST BE FIRST */ int64 calls; /* # of times executed */ double usage; /* hashtable usage factor */ slock_t mutex; /* protects the counters only */ } QueryStatsEntry; /* * Global shared state */ typedef struct QueryStatsSharedState { LWLockId lock; /* protects hashtable search/modification */ double cur_median_usage; /* current median usage in hashtable */ } QueryStatsSharedState; /* lookup table for existing pg_stat_statements entries */ typedef struct ExistingStatsHashKey { Oid userid; /* user OID */ Oid dbid; /* database OID */ uint64 queryid; /* query identifier */ } ExistingStatsHashKey; /* saved hook address in case of unload */ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* Links to shared memory state */ static QueryStatsSharedState *queryStats = NULL; static HTAB *queryStatsHash = NULL; /*--- Functions --- */ Datum citus_query_stats_reset(PG_FUNCTION_ARGS); Datum citus_query_stats(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(citus_stat_statements_reset); PG_FUNCTION_INFO_V1(citus_query_stats); PG_FUNCTION_INFO_V1(citus_executor_name); static char * CitusExecutorName(MultiExecutorType executorType); static void CitusQueryStatsShmemStartup(void); static void CitusQueryStatsShmemShutdown(int code, Datum arg); static QueryStatsEntry * CitusQueryStatsEntryAlloc(QueryStatsHashKey *key, bool sticky); static void CitusQueryStatsEntryDealloc(void); static void CitusQueryStatsEntryReset(void); static uint32 CitusQuerysStatsHashFn(const void *key, Size keysize); static int CitusQuerysStatsMatchFn(const void *key1, const void *key2, Size keysize); static HTAB * BuildExistingQueryIdHash(void); static int GetPGStatStatementsMax(void); static void CitusQueryStatsRemoveExpiredEntries(HTAB *existingQueryIdHash); void InitializeCitusQueryStats(void) { /* Install hook */ prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = CitusQueryStatsShmemStartup; } static void CitusQueryStatsShmemStartup(void) { bool found; HASHCTL info; uint32 header; int32 num; QueryStatsEntry *buffer = NULL; if (prev_shmem_startup_hook) { prev_shmem_startup_hook(); } /* Create or attach to the shared memory state */ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); /* global access lock */ queryStats = ShmemInitStruct(STATS_SHARED_MEM_NAME, sizeof(QueryStatsSharedState), &found); if (!found) { /* First time through ... */ queryStats->lock = &(GetNamedLWLockTranche(STATS_SHARED_MEM_NAME))->lock; } memset(&info, 0, sizeof(info)); info.keysize = sizeof(QueryStatsHashKey); info.entrysize = sizeof(QueryStatsEntry); info.hash = CitusQuerysStatsHashFn; info.match = CitusQuerysStatsMatchFn; /* allocate stats shared memory hash */ queryStatsHash = ShmemInitHash("citus_query_stats hash", StatStatementsMax, StatStatementsMax, &info, HASH_ELEM | HASH_FUNCTION | HASH_COMPARE); LWLockRelease(AddinShmemInitLock); if (!IsUnderPostmaster) { on_shmem_exit(CitusQueryStatsShmemShutdown, (Datum) 0); } /* * Done if some other process already completed our initialization. */ if (found) { return; } /* Load stat file, don't care about locking */ FILE *file = AllocateFile(CITUS_STATS_DUMP_FILE, PG_BINARY_R); if (file == NULL) { if (errno == ENOENT) { return; /* ignore not-found error */ } goto error; } /* check is header is valid */ if (fread(&header, sizeof(uint32), 1, file) != 1 || header != CITUS_QUERY_STATS_FILE_HEADER) { goto error; } /* get number of entries */ if (fread(&num, sizeof(int32), 1, file) != 1) { goto error; } for (int i = 0; i < num; i++) { QueryStatsEntry temp; if (fread(&temp, sizeof(QueryStatsEntry), 1, file) != 1) { goto error; } /* Skip loading "sticky" entries */ if (temp.calls == 0) { continue; } QueryStatsEntry *entry = CitusQueryStatsEntryAlloc(&temp.key, false); /* copy in the actual stats */ entry->calls = temp.calls; entry->usage = temp.usage; /* don't initialize spinlock, already done */ } FreeFile(file); /* * Remove the file so it's not included in backups/replicas, etc. A new file will be * written on next shutdown. */ unlink(CITUS_STATS_DUMP_FILE); return; error: ereport(LOG, (errcode_for_file_access(), errmsg("could not read citus_query_stats file \"%s\": %m", CITUS_STATS_DUMP_FILE))); if (buffer) { pfree(buffer); } if (file) { FreeFile(file); } /* delete bogus file, don't care of errors in this case */ unlink(CITUS_STATS_DUMP_FILE); } /* * CitusQueryStatsShmemShutdown is a shmem_shutdown hook, * it dumps statistics into file. */ static void CitusQueryStatsShmemShutdown(int code, Datum arg) { HASH_SEQ_STATUS hash_seq; QueryStatsEntry *entry; /* Don't try to dump during a crash. */ if (code) { return; } if (!queryStats) { return; } FILE *file = AllocateFile(CITUS_STATS_DUMP_FILE ".tmp", PG_BINARY_W); if (file == NULL) { goto error; } if (fwrite(&CITUS_QUERY_STATS_FILE_HEADER, sizeof(uint32), 1, file) != 1) { goto error; } int32 num_entries = hash_get_num_entries(queryStatsHash); if (fwrite(&num_entries, sizeof(int32), 1, file) != 1) { goto error; } hash_seq_init(&hash_seq, queryStatsHash); while ((entry = hash_seq_search(&hash_seq)) != NULL) { if (fwrite(entry, sizeof(QueryStatsEntry), 1, file) != 1) { /* note: we assume hash_seq_term won't change errno */ hash_seq_term(&hash_seq); goto error; } } if (FreeFile(file)) { file = NULL; goto error; } /* * Rename file inplace */ if (rename(CITUS_STATS_DUMP_FILE ".tmp", CITUS_STATS_DUMP_FILE) != 0) { ereport(LOG, (errcode_for_file_access(), errmsg("could not rename citus_query_stats file \"%s\": %m", CITUS_STATS_DUMP_FILE ".tmp"))); } return; error: ereport(LOG, (errcode_for_file_access(), errmsg("could not read citus_query_stats file \"%s\": %m", CITUS_STATS_DUMP_FILE))); if (file) { FreeFile(file); } unlink(CITUS_STATS_DUMP_FILE); } /* * CitusQueryStatsSharedMemSize calculates and returns shared memory size * required to keep query statistics. */ Size CitusQueryStatsSharedMemSize(void) { Assert(StatStatementsMax >= 0); Size size = MAXALIGN(sizeof(QueryStatsSharedState)); size = add_size(size, hash_estimate_size(StatStatementsMax, sizeof(QueryStatsEntry))); return size; } /* * CitusQueryStatsExecutorsEntry is the function to update statistics * for a given query id. */ void CitusQueryStatsExecutorsEntry(uint64 queryId, MultiExecutorType executorType, char *partitionKey) { QueryStatsHashKey key; /* Safety check... */ if (!queryStats || !queryStatsHash) { return; } /* early return if tracking is disabled */ if (!StatStatementsTrack) { return; } /* Set up key for hashtable search */ key.userid = GetUserId(); key.dbid = MyDatabaseId; key.queryid = queryId; key.executorType = executorType; memset(key.partitionKey, 0, MAX_KEY_LENGTH); if (partitionKey != NULL) { strlcpy(key.partitionKey, partitionKey, MAX_KEY_LENGTH); } /* Lookup the hash table entry with shared lock. */ LWLockAcquire(queryStats->lock, LW_SHARED); QueryStatsEntry *entry = (QueryStatsEntry *) hash_search(queryStatsHash, &key, HASH_FIND, NULL); /* Create new entry, if not present */ if (!entry) { /* Need exclusive lock to make a new hashtable entry - promote */ LWLockRelease(queryStats->lock); LWLockAcquire(queryStats->lock, LW_EXCLUSIVE); /* OK to create a new hashtable entry */ entry = CitusQueryStatsEntryAlloc(&key, false); } /* * Grab the spinlock while updating the counters (see comment about * locking rules at the head of the pg_stat_statements file) */ volatile QueryStatsEntry *e = (volatile QueryStatsEntry *) entry; SpinLockAcquire(&e->mutex); /* "Unstick" entry if it was previously sticky */ if (e->calls == 0) { e->usage = USAGE_INIT; } e->calls += 1; SpinLockRelease(&e->mutex); LWLockRelease(queryStats->lock); } /* * Allocate a new hashtable entry. * caller must hold an exclusive lock on queryStats->lock */ static QueryStatsEntry * CitusQueryStatsEntryAlloc(QueryStatsHashKey *key, bool sticky) { bool found; long StatStatementsMaxLong = StatStatementsMax; /* Make space if needed */ while (hash_get_num_entries(queryStatsHash) >= StatStatementsMaxLong) { CitusQueryStatsEntryDealloc(); } /* Find or create an entry with desired hash code */ QueryStatsEntry *entry = (QueryStatsEntry *) hash_search(queryStatsHash, key, HASH_ENTER, &found); if (!found) { /* New entry, initialize it */ /* set the appropriate initial usage count */ entry->usage = sticky ? queryStats->cur_median_usage : USAGE_INIT; /* re-initialize the mutex each time ... we assume no one using it */ SpinLockInit(&entry->mutex); } entry->calls = 0; entry->usage = (0.0); return entry; } /* * entry_cmp is qsort comparator for sorting into increasing usage order */ static int entry_cmp(const void *lhs, const void *rhs) { double l_usage = (*(QueryStatsEntry *const *) lhs)->usage; double r_usage = (*(QueryStatsEntry *const *) rhs)->usage; if (l_usage < r_usage) { return -1; } else if (l_usage > r_usage) { return +1; } else { return 0; } } /* * CitusQueryStatsEntryDealloc deallocates least used entries. * Caller must hold an exclusive lock on queryStats->lock. */ static void CitusQueryStatsEntryDealloc(void) { HASH_SEQ_STATUS hash_seq; QueryStatsEntry *entry; /* * Sort entries by usage and deallocate USAGE_DEALLOC_PERCENT of them. * While we're scanning the table, apply the decay factor to the usage * values. */ QueryStatsEntry **entries = palloc(hash_get_num_entries(queryStatsHash) * sizeof(QueryStatsEntry *)); int i = 0; hash_seq_init(&hash_seq, queryStatsHash); while ((entry = hash_seq_search(&hash_seq)) != NULL) { entries[i++] = entry; /* "Sticky" entries get a different usage decay rate. */ if (entry->calls == 0) { entry->usage *= STICKY_DECREASE_FACTOR; } else { entry->usage *= USAGE_DECREASE_FACTOR; } } SafeQsort(entries, i, sizeof(QueryStatsEntry *), entry_cmp); if (i > 0) { /* Record the (approximate) median usage */ queryStats->cur_median_usage = entries[i / 2]->usage; } int nvictims = Max(10, i * USAGE_DEALLOC_PERCENT / 100); nvictims = Min(nvictims, i); for (i = 0; i < nvictims; i++) { hash_search(queryStatsHash, &entries[i]->key, HASH_REMOVE, NULL); } pfree(entries); } /* * CitusQueryStatsEntryReset resets statistics. */ static void CitusQueryStatsEntryReset(void) { HASH_SEQ_STATUS hash_seq; QueryStatsEntry *entry; LWLockAcquire(queryStats->lock, LW_EXCLUSIVE); hash_seq_init(&hash_seq, queryStatsHash); while ((entry = hash_seq_search(&hash_seq)) != NULL) { hash_search(queryStatsHash, &entry->key, HASH_REMOVE, NULL); } LWLockRelease(queryStats->lock); } /* * CitusQuerysStatsHashFn calculates and returns hash value for a key */ static uint32 CitusQuerysStatsHashFn(const void *key, Size keysize) { const QueryStatsHashKey *k = (const QueryStatsHashKey *) key; if (k->partitionKey[0] != '\0') { return hash_uint32((uint32) k->userid) ^ hash_uint32((uint32) k->dbid) ^ hash_any((const unsigned char *) &(k->queryid), sizeof(uint64)) ^ hash_uint32((uint32) k->executorType) ^ hash_any((const unsigned char *) (k->partitionKey), strlen( k->partitionKey)); } else { return hash_uint32((uint32) k->userid) ^ hash_uint32((uint32) k->dbid) ^ hash_any((const unsigned char *) &(k->queryid), sizeof(uint64)) ^ hash_uint32((uint32) k->executorType); } } /* * CitusQuerysStatsMatchFn compares two keys - zero means match. * See definition of HashCompareFunc in hsearch.h for more info. */ static int CitusQuerysStatsMatchFn(const void *key1, const void *key2, Size keysize) { const QueryStatsHashKey *k1 = (const QueryStatsHashKey *) key1; const QueryStatsHashKey *k2 = (const QueryStatsHashKey *) key2; if (k1->userid == k2->userid && k1->dbid == k2->dbid && k1->queryid == k2->queryid && k1->executorType == k2->executorType) { return 0; } return 1; } /* * Reset statistics. */ Datum citus_stat_statements_reset(PG_FUNCTION_ARGS) { CitusQueryStatsEntryReset(); PG_RETURN_VOID(); } /* * citus_query_stats returns query stats kept in memory. */ Datum citus_query_stats(PG_FUNCTION_ARGS) { TupleDesc tupdesc; HASH_SEQ_STATUS hash_seq; QueryStatsEntry *entry; Oid currentUserId = GetUserId(); bool canSeeStats = superuser(); if (!queryStats) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("citus_query_stats: shared memory not initialized"))); } if (is_member_of_role(GetUserId(), ROLE_PG_READ_ALL_STATS)) { canSeeStats = true; } Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); /* exclusive lock on queryStats->lock is acquired and released inside the function */ CitusQueryStatsSynchronizeEntries(); LWLockAcquire(queryStats->lock, LW_SHARED); hash_seq_init(&hash_seq, queryStatsHash); while ((entry = hash_seq_search(&hash_seq)) != NULL) { Datum values[CITUS_STAT_STATEMENTS_COLS]; bool nulls[CITUS_STAT_STATEMENTS_COLS]; /* following vars are to keep data for processing after spinlock release */ uint64 queryid = 0; Oid userid = InvalidOid; Oid dbid = InvalidOid; MultiExecutorType executorType = MULTI_EXECUTOR_INVALID_FIRST; char partitionKey[MAX_KEY_LENGTH]; int64 calls = 0; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); memset(partitionKey, 0, MAX_KEY_LENGTH); SpinLockAcquire(&entry->mutex); /* * Skip entry if unexecuted (ie, it's a pending "sticky" entry) or * the user does not have permission to view it. */ if (entry->calls == 0 || !(currentUserId == entry->key.userid || canSeeStats)) { SpinLockRelease(&entry->mutex); continue; } queryid = entry->key.queryid; userid = entry->key.userid; dbid = entry->key.dbid; executorType = entry->key.executorType; if (entry->key.partitionKey[0] != '\0') { memcpy_s(partitionKey, sizeof(partitionKey), entry->key.partitionKey, sizeof(entry->key.partitionKey)); } calls = entry->calls; SpinLockRelease(&entry->mutex); values[CITUS_STAT_STATAMENTS_QUERY_ID] = UInt64GetDatum(queryid); values[CITUS_STAT_STATAMENTS_USER_ID] = ObjectIdGetDatum(userid); values[CITUS_STAT_STATAMENTS_DB_ID] = ObjectIdGetDatum(dbid); values[CITUS_STAT_STATAMENTS_EXECUTOR_TYPE] = UInt32GetDatum( (uint32) executorType); if (partitionKey[0] != '\0') { values[CITUS_STAT_STATAMENTS_PARTITION_KEY] = CStringGetTextDatum( partitionKey); } else { nulls[CITUS_STAT_STATAMENTS_PARTITION_KEY] = true; } values[CITUS_STAT_STATAMENTS_CALLS] = Int64GetDatumFast(calls); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } LWLockRelease(queryStats->lock); return (Datum) 0; } /* * CitusQueryStatsSynchronizeEntries removes all entries in queryStats hash * that does not have matching queryId in pg_stat_statements. * * A function called inside (CitusQueryStatsRemoveExpiredEntries) acquires * an exclusive lock on queryStats->lock. */ void CitusQueryStatsSynchronizeEntries(void) { HTAB *existingQueryIdHash = BuildExistingQueryIdHash(); if (existingQueryIdHash != NULL) { CitusQueryStatsRemoveExpiredEntries(existingQueryIdHash); hash_destroy(existingQueryIdHash); } } /* * BuildExistingQueryIdHash goes over entries in pg_stat_statements and prepare * a hash table of queryId's. The function returns null if * public.pg_stat_statements(bool) function is not available. Returned hash * table is allocated on the CurrentMemoryContext, and caller is responsible * for deallocation. */ static HTAB * BuildExistingQueryIdHash(void) { const int userIdAttributeNumber = 1; const int dbIdAttributeNumber = 2; const int queryIdAttributeNumber = 4; Datum commandTypeDatum = (Datum) 0; bool missingOK = true; Oid pgStatStatementsOid = FunctionOidExtended("public", "pg_stat_statements", 1, missingOK); if (!OidIsValid(pgStatStatementsOid)) { return NULL; } /* fetch pg_stat_statements.max, it is expected to be available, if not bail out */ int pgStatStatementsMax = GetPGStatStatementsMax(); if (pgStatStatementsMax == 0) { ereport(DEBUG1, (errmsg("Cannot access pg_stat_statements.max"))); return NULL; } FmgrInfo *fmgrPGStatStatements = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); commandTypeDatum = BoolGetDatum(false); fmgr_info(pgStatStatementsOid, fmgrPGStatStatements); ReturnSetInfo *statStatementsReturnSet = FunctionCallGetTupleStore1( fmgrPGStatStatements->fn_addr, pgStatStatementsOid, commandTypeDatum); TupleTableSlot *tupleTableSlot = MakeSingleTupleTableSlot( statStatementsReturnSet->setDesc, &TTSOpsMinimalTuple); /* * Allocate more hash slots (twice as much) than necessary to minimize * collisions. */ assert_valid_hash_key3(ExistingStatsHashKey, userid, dbid, queryid); HTAB *queryIdHashTable = CreateSimpleHashSetWithNameAndSize( ExistingStatsHashKey, "pg_stats_statements queryId hash", pgStatStatementsMax * 2); /* iterate over tuples in tuple store, and add queryIds to hash table */ while (true) { bool isNull = false; bool tuplePresent = tuplestore_gettupleslot(statStatementsReturnSet->setResult, true, false, tupleTableSlot); if (!tuplePresent) { break; } Datum userIdDatum = slot_getattr(tupleTableSlot, userIdAttributeNumber, &isNull); Datum dbIdDatum = slot_getattr(tupleTableSlot, dbIdAttributeNumber, &isNull); Datum queryIdDatum = slot_getattr(tupleTableSlot, queryIdAttributeNumber, &isNull); /* * queryId may be returned as NULL when current user is not authorized to see other * users' stats. */ if (!isNull) { ExistingStatsHashKey key; key.userid = DatumGetInt32(userIdDatum); key.dbid = DatumGetInt32(dbIdDatum); key.queryid = DatumGetInt64(queryIdDatum); hash_search(queryIdHashTable, (void *) &key, HASH_ENTER, NULL); } ExecClearTuple(tupleTableSlot); } ExecDropSingleTupleTableSlot(tupleTableSlot); tuplestore_end(statStatementsReturnSet->setResult); pfree(fmgrPGStatStatements); return queryIdHashTable; } /* * GetPGStatStatementsMax returns GUC value pg_stat_statements.max. The * function returns 0 if for some reason it can not access * pg_stat_statements.max value. */ static int GetPGStatStatementsMax(void) { const char *name = "pg_stat_statements.max"; int maxValue = 0; const char *pgssMax = GetConfigOption(name, true, false); /* * Retrieving pg_stat_statements.max can fail if the extension is loaded * after citus in shared_preload_libraries, or not at all. */ if (pgssMax) { maxValue = pg_strtoint32(pgssMax); } return maxValue; } /* * CitusQueryStatsRemoveExpiredEntries iterates over queryStats hash entries * and removes entries with keys that do not exists in the provided hash of * queryIds. * * Acquires and releases exclusive lock on queryStats->lock. */ static void CitusQueryStatsRemoveExpiredEntries(HTAB *existingQueryIdHash) { HASH_SEQ_STATUS hash_seq; QueryStatsEntry *entry; int removedCount = 0; bool canSeeStats = superuser(); Oid currentUserId = GetUserId(); if (is_member_of_role(currentUserId, ROLE_PG_READ_ALL_STATS)) { canSeeStats = true; } LWLockAcquire(queryStats->lock, LW_EXCLUSIVE); hash_seq_init(&hash_seq, queryStatsHash); while ((entry = hash_seq_search(&hash_seq)) != NULL) { bool found = false; ExistingStatsHashKey existingStatsKey = { 0, 0, 0 }; /* * pg_stat_statements returns NULL in the queryId field for queries * belonging to other users. Those queries are therefore not reflected * in the existingQueryIdHash, but that does not mean that we should * remove them as they are relevant to other users. */ if (!(currentUserId == entry->key.userid || canSeeStats)) { continue; } existingStatsKey.userid = entry->key.userid; existingStatsKey.dbid = entry->key.dbid; existingStatsKey.queryid = entry->key.queryid; hash_search(existingQueryIdHash, (void *) &existingStatsKey, HASH_FIND, &found); if (!found) { hash_search(queryStatsHash, &entry->key, HASH_REMOVE, NULL); removedCount++; } } LWLockRelease(queryStats->lock); if (removedCount > 0) { elog(DEBUG2, "citus_stat_statements removed %d expired entries", removedCount); } } /* * citus_executor_name is a UDF that returns the name of the executor * given the internal enum value. */ Datum citus_executor_name(PG_FUNCTION_ARGS) { MultiExecutorType executorType = PG_GETARG_UINT32(0); char *executorName = CitusExecutorName(executorType); PG_RETURN_TEXT_P(cstring_to_text(executorName)); } /* * CitusExecutorName returns the name of the executor given the internal * enum value. */ static char * CitusExecutorName(MultiExecutorType executorType) { switch (executorType) { case MULTI_EXECUTOR_ADAPTIVE: { return "adaptive"; } case MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT: { return "insert-select"; } default: { return "unknown"; } } } ================================================ FILE: src/backend/distributed/stats/stat_counters.c ================================================ /*------------------------------------------------------------------------- * * stat_counters.c * * This file contains functions to track various statistic counters for * Citus. * * We create an array of "BackendStatsSlot"s in shared memory, one for * each backend. Each backend increments its own stat counters in its * own slot via IncrementStatCounterForMyDb(). And when a backend exits, * it saves its stat counters from its slot via * SaveBackendStatsIntoSavedBackendStatsHash() into a hash table in * shared memory, whose entries are "SavedBackendStatsHashEntry"s and * the key is the database id. In other words, each entry of the hash * table is used to aggregate the stat counters for backends that were * connected to that database and exited since the last server restart. * Plus, each entry is responsible for keeping track of the reset * timestamp for both active and exited backends too. * Note that today we don't evict the entries of the said hash table * that point to dropped databases because the wrapper view anyway * filters them out (thanks to LEFT JOIN) and we don't expect a * performance hit due to that unless users have a lot of databases * that are dropped and recreated frequently. * * The reason why we save the stat counters for exited backends in the * shared hash table is that we cannot guarantee that the backend slot * that was used by an exited backend will be reused by another backend * connected to the same database. For this reason, we need to save the * stat counters for exited backends into a shared hash table so that we * can reset the counters within the corresponding backend slots while * the backends exit. * * When citus_stat_counters() is called, we first aggregate the stat * counters from the backend slots of all the active backends and then * we add the aggregated stat counters from the exited backends that * are stored in the shared hash table. Also, we don't persist backend * stats on server shutdown, but we might want to do that in the future. * * Similarly, when citus_stat_counters_reset() is called, we reset the * stat counters for the active backends and the exited backends that are * stored in the shared hash table. Then, it also updates the * resetTimestamp in the shared hash table entry appropriately. So, * similarly, when citus_stat_counters() is called, we just report * resetTimestamp as stats_reset column. * * Caveats: * * There is chance that citus_stat_counters_reset() might race with a * backend that is trying to increment one of the counters in its slot * and as a result it can effectively fail to reset that counter due to * the reasons documented in IncrementStatCounterForMyDb() function. * However, this should be a very rare case and we can live with that * for now. * * Also, citus_stat_counters() might observe the counters for a backend * twice or perhaps unsee it if it's concurrently exiting, depending on * the order we call CollectActiveBackendStatsIntoHTAB() and * CollectSavedBackendStatsIntoHTAB() in citus_stat_counters(). However, * the next call to citus_stat_counters() will see the correct values * for the counters, so we can live with that for now. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "common/hashfn.h" #include "port/atomics.h" #include "storage/ipc.h" #include "storage/proc.h" #include "utils/hsearch.h" #include "pg_version_compat.h" #include "distributed/argutils.h" #include "distributed/metadata_cache.h" #include "distributed/stats/stat_counters.h" #include "distributed/tuplestore.h" /* * saved backend stats - hash table constants * * Configurations used to create the hash table for saved backend stats. * The places where SAVED_BACKEND_STATS_HASH_MAX_DATABASES is used do not * impose a hard limit on the number of databases that can be tracked but * in ShmemInitHash() it's documented that the access efficiency will degrade * if it is exceeded substantially. * * XXX: Consider using dshash_table instead of (shared) HTAB if that becomes * a concern. */ #define SAVED_BACKEND_STATS_HASH_INIT_DATABASES 8 #define SAVED_BACKEND_STATS_HASH_MAX_DATABASES 1024 /* fixed size array types to store the stat counters */ typedef pg_atomic_uint64 AtomicStatCounters[N_CITUS_STAT_COUNTERS]; typedef uint64 StatCounters[N_CITUS_STAT_COUNTERS]; /* * saved backend stats - hash entry definition * * This is used to define & access the shared hash table used to aggregate the stat * counters for the backends exited so far since last server restart. It's also * responsible for keeping track of the reset timestamp. */ typedef struct SavedBackendStatsHashEntry { /* hash entry key, must always be the first */ Oid databaseId; /* * Needs to be locked whenever we read / write counters or resetTimestamp * in this struct since we don't use atomic counters for this struct. Plus, * we want to update the stat counters and resetTimestamp atomically. */ slock_t mutex; /* * While "counters" only represents the stat counters for exited backends, * the "resetTimestamp" doesn't only represent the reset timestamp for exited * backends' stat counters but also for the active backends. */ StatCounters counters; TimestampTz resetTimestamp; } SavedBackendStatsHashEntry; /* * Hash entry definition used for the local hash table created by * citus_stat_counters() at the runtime to aggregate the stat counters * across all backends. */ typedef struct DatabaseStatsHashEntry { /* hash entry key, must always be the first */ Oid databaseId; StatCounters counters; TimestampTz resetTimestamp; } DatabaseStatsHashEntry; /* definition of a one per-backend stat counters slot in shared memory */ typedef struct BackendStatsSlot { AtomicStatCounters counters; } BackendStatsSlot; /* * GUC variable * * This only controls whether we track the stat counters or not, via * IncrementStatCounterForMyDb() and * SaveBackendStatsIntoSavedBackendStatsHash(). In other words, even * when the GUC is disabled, we still allocate the shared memory * structures etc. and citus_stat_counters() / citus_stat_counters_reset() * will still work. */ bool EnableStatCounters = ENABLE_STAT_COUNTERS_DEFAULT; /* saved backend stats - shared memory variables */ static LWLockId *SharedSavedBackendStatsHashLock = NULL; static HTAB *SharedSavedBackendStatsHash = NULL; /* per-backend stat counter slots - shared memory array */ BackendStatsSlot *SharedBackendStatsSlotArray = NULL; /* * We don't expect the callsites that check this (via * EnsureStatCountersShmemInitDone()) to be executed before * StatCountersShmemInit() is done. Plus, once StatCountersShmemInit() * is done, we also don't expect shared memory variables to be * initialized improperly. However, we still set this to true only * once StatCountersShmemInit() is done and if all three of the shared * memory variables above are initialized properly. And in the callsites * where these shared memory variables are accessed, we check this * variable first just to be on the safe side. */ static bool StatCountersShmemInitDone = false; /* saved shmem_startup_hook */ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* shared memory init & management */ static void StatCountersShmemInit(void); static Size SharedBackendStatsSlotArrayShmemSize(void); /* helper functions for citus_stat_counters() */ static void CollectActiveBackendStatsIntoHTAB(Oid databaseId, HTAB *databaseStats); static void CollectSavedBackendStatsIntoHTAB(Oid databaseId, HTAB *databaseStats); static DatabaseStatsHashEntry * DatabaseStatsHashEntryFindOrCreate(Oid databaseId, HTAB *databaseStats); static void StoreDatabaseStatsIntoTupStore(HTAB *databaseStats, Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); /* helper functions for citus_stat_counters_reset() */ static bool ResetActiveBackendStats(Oid databaseId); static void ResetSavedBackendStats(Oid databaseId, bool force); /* saved backend stats */ static SavedBackendStatsHashEntry * SavedBackendStatsHashEntryCreateIfNotExists(Oid databaseId); /* sql exports */ PG_FUNCTION_INFO_V1(citus_stat_counters); PG_FUNCTION_INFO_V1(citus_stat_counters_reset); /* * EnsureStatCountersShmemInitDone returns true if the shared memory * data structures used for keeping track of stat counters have been * properly initialized, otherwise, returns false and emits a warning. */ static inline bool EnsureStatCountersShmemInitDone(void) { if (!StatCountersShmemInitDone) { ereport(WARNING, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("shared memory for stat counters was not properly initialized"))); return false; } return true; } /* * citus_stat_counters returns stats counters for the given database id. * * This only returns rows for the databases which have been connected to * by at least one backend since the last server restart (even if no * observations have been made for none of the counters or if they were * reset) and it considers such a database even if it has been dropped later. * * When InvalidOid is provided, all such databases are considered; otherwise * only the database with the given id is considered. * * So, as an outcome, when a database id that is different than InvalidOid * is provided and no backend has connected to it since the last server * restart, or, if we didn't ever have such a database, then the function * returns an empty set. * * Finally, stats_reset column is set to NULL if the stat counters for the * database were never reset since the last server restart. */ Datum citus_stat_counters(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* * Function's sql definition allows Postgres to silently * ignore NULL, but we still check. */ PG_ENSURE_ARGNOTNULL(0, "database_id"); Oid databaseId = PG_GETARG_OID(0); /* just to be on the safe side */ if (!EnsureStatCountersShmemInitDone()) { PG_RETURN_VOID(); } TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); HASHCTL info; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION); memset(&info, 0, sizeof(info)); info.keysize = sizeof(Oid); info.hash = oid_hash; info.entrysize = sizeof(DatabaseStatsHashEntry); HTAB *databaseStats = hash_create("Citus Database Stats Collect Hash", 8, &info, hashFlags); CollectActiveBackendStatsIntoHTAB(databaseId, databaseStats); CollectSavedBackendStatsIntoHTAB(databaseId, databaseStats); StoreDatabaseStatsIntoTupStore(databaseStats, tupleStore, tupleDescriptor); hash_destroy(databaseStats); PG_RETURN_VOID(); } /* * citus_stat_counters_reset resets Citus stat counters for given database * id or for the current database if InvalidOid is provided. * * If a valid database id is provided, stat counters for that database are * reset, even if it was dropped later. * * Otherwise, if the provided database id is not valid, then the function * effectively does nothing. */ Datum citus_stat_counters_reset(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* * Function's sql definition allows Postgres to silently * ignore NULL, but we still check. */ PG_ENSURE_ARGNOTNULL(0, "database_id"); Oid databaseId = PG_GETARG_OID(0); /* * If the database id is InvalidOid, then we assume that * the caller wants to reset the stat counters for the * current database. */ if (databaseId == InvalidOid) { databaseId = MyDatabaseId; } /* just to be on the safe side */ if (!EnsureStatCountersShmemInitDone()) { PG_RETURN_VOID(); } bool foundAnyBackendsForDb = ResetActiveBackendStats(databaseId); /* * Even when we don't have an entry for the given database id in the * saved backend stats hash table, we still want to create one for * it to save the resetTimestamp if we currently have at least backend * connected to it. By providing foundAnyBackendsForDb, we effectively * let the function do that. Since ResetActiveBackendStats() doesn't * filter the active backends, foundAnyBackendsForDb being true * not always means that at least one backend is connected to it right * now, but it means that we had such a backend at some point in time * since the last server restart. If all backends refered to in the * shared array are already exited, then we should already have an * entry for it in the saved backend stats hash table, so providing * a "true" wouldn't do anything in that case. Otherwise, if at least * one backend is still connected to it, providing a "true" will * effectively create a new entry for it if it doesn't exist yet, * which is what we actually want to do. * * That way, we can save the resetTimestamp for the active backends * into the relevant entry of the saved backend stats hash table. * Note that we don't do that for the databases that don't have * any active backends connected to them because we actually don't * reset anything for such databases. */ ResetSavedBackendStats(databaseId, foundAnyBackendsForDb); PG_RETURN_VOID(); } /* * InitializeStatCountersShmem saves the previous shmem_startup_hook and sets * up a new shmem_startup_hook for initializing the shared memory data structures * used for keeping track of stat counters. */ void InitializeStatCountersShmem(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = StatCountersShmemInit; } /* * StatCountersShmemSize calculates and returns shared memory size * required for the shared memory data structures used for keeping track of * stat counters. */ Size StatCountersShmemSize(void) { Size backendStatsSlotArraySize = SharedBackendStatsSlotArrayShmemSize(); Size savedBackendStatsHashLockSize = MAXALIGN(sizeof(LWLockId)); Size savedBackendStatsHashSize = hash_estimate_size( SAVED_BACKEND_STATS_HASH_MAX_DATABASES, sizeof(SavedBackendStatsHashEntry)); return add_size(add_size(backendStatsSlotArraySize, savedBackendStatsHashLockSize), savedBackendStatsHashSize); } /* * IncrementStatCounterForMyDb increments the stat counter for the given statId * for this backend. */ void IncrementStatCounterForMyDb(int statId) { if (!EnableStatCounters) { return; } /* just to be on the safe side */ if (!EnsureStatCountersShmemInitDone()) { return; } int myBackendSlotIdx = getProcNo_compat(MyProc); BackendStatsSlot *myBackendStatsSlot = &SharedBackendStatsSlotArray[myBackendSlotIdx]; /* * When there cannot be any other writers, incrementing an atomic * counter via pg_atomic_read_u64() and pg_atomic_write_u64() is * same as incrementing it via pg_atomic_fetch_add_u64(). Plus, the * former is cheaper than the latter because the latter has to do * extra work to deal with concurrent writers. * * In our case, the only concurrent writer could be the backend that * is executing citus_stat_counters_reset(). So, there is chance that * we read the counter value, then it gets reset by a concurrent call * made to citus_stat_counters_reset() and then we write the * incremented value back, by effectively overriding the reset value. * But this should be a rare case and we can live with that, for the * sake of lock-free implementation of this function. */ pg_atomic_uint64 *statPtr = &myBackendStatsSlot->counters[statId]; pg_atomic_write_u64(statPtr, pg_atomic_read_u64(statPtr) + 1); } /* * SaveBackendStatsIntoSavedBackendStatsHash saves the stat counters * for this backend into the saved backend stats hash table. * * So, this is only supposed to be called when a backend exits. * * Also, we do our best to avoid throwing errors in this function because * this function is called when a backend is exiting and throwing errors * at that point will cause the backend to crash. */ void SaveBackendStatsIntoSavedBackendStatsHash(void) { if (!EnableStatCounters) { return; } /* just to be on the safe side */ if (!EnsureStatCountersShmemInitDone()) { return; } Oid databaseId = MyDatabaseId; LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_SHARED); SavedBackendStatsHashEntry *dbSavedBackendStatsEntry = (SavedBackendStatsHashEntry *) hash_search( SharedSavedBackendStatsHash, (void *) &databaseId, HASH_FIND, NULL); if (!dbSavedBackendStatsEntry) { /* promote the lock to exclusive to insert the new entry for this database */ LWLockRelease(*SharedSavedBackendStatsHashLock); LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_EXCLUSIVE); dbSavedBackendStatsEntry = SavedBackendStatsHashEntryCreateIfNotExists(databaseId); LWLockRelease(*SharedSavedBackendStatsHashLock); if (!dbSavedBackendStatsEntry) { /* * Couldn't allocate a new hash entry because we're out of * (shared) memory. In that case, we just log a warning and * return, instead of throwing an error due to the reasons * mentioned in function's comment. */ ereport(WARNING, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("failed to allocate saved backend stats hash entry"))); return; } /* re-acquire the shared lock */ LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_SHARED); } int myBackendSlotIdx = getProcNo_compat(MyProc); BackendStatsSlot *myBackendStatsSlot = &SharedBackendStatsSlotArray[myBackendSlotIdx]; SpinLockAcquire(&dbSavedBackendStatsEntry->mutex); for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { dbSavedBackendStatsEntry->counters[statIdx] += pg_atomic_read_u64(&myBackendStatsSlot->counters[statIdx]); /* * Given that this function is only called when a backend exits, later on * another backend might be assigned to the same slot. So, we reset each * stat counter of this slot to 0 after saving it. */ pg_atomic_write_u64(&myBackendStatsSlot->counters[statIdx], 0); } SpinLockRelease(&dbSavedBackendStatsEntry->mutex); LWLockRelease(*SharedSavedBackendStatsHashLock); } /* * StatCountersShmemInit initializes the shared memory data structures used * for keeping track of stat counters. */ static void StatCountersShmemInit(void) { if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); bool sharedBackendStatsSlotArrayAlreadyInit = false; SharedBackendStatsSlotArray = (BackendStatsSlot *) ShmemInitStruct( "Citus Shared Backend Stats Slot Array", SharedBackendStatsSlotArrayShmemSize(), &sharedBackendStatsSlotArrayAlreadyInit); bool sharedSavedBackendStatsHashLockAlreadyInit = false; SharedSavedBackendStatsHashLock = ShmemInitStruct( SAVED_BACKEND_STATS_HASH_LOCK_TRANCHE_NAME, sizeof(LWLockId), & sharedSavedBackendStatsHashLockAlreadyInit); HASHCTL hashInfo = { .keysize = sizeof(Oid), .entrysize = sizeof(SavedBackendStatsHashEntry), .hash = oid_hash, }; SharedSavedBackendStatsHash = ShmemInitHash("Citus Shared Saved Backend Stats Hash", SAVED_BACKEND_STATS_HASH_INIT_DATABASES, SAVED_BACKEND_STATS_HASH_MAX_DATABASES, &hashInfo, HASH_ELEM | HASH_FUNCTION); Assert(sharedBackendStatsSlotArrayAlreadyInit == sharedSavedBackendStatsHashLockAlreadyInit); if (!sharedBackendStatsSlotArrayAlreadyInit) { for (int backendSlotIdx = 0; backendSlotIdx < MaxBackends; ++backendSlotIdx) { BackendStatsSlot *backendStatsSlot = &SharedBackendStatsSlotArray[backendSlotIdx]; for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { pg_atomic_init_u64(&backendStatsSlot->counters[statIdx], 0); } } *SharedSavedBackendStatsHashLock = &( GetNamedLWLockTranche( SAVED_BACKEND_STATS_HASH_LOCK_TRANCHE_NAME) )->lock; } LWLockRelease(AddinShmemInitLock); /* * At this point, they should have been set to non-null values already, * but we still check them just to be sure. */ if (SharedBackendStatsSlotArray && SharedSavedBackendStatsHashLock && SharedSavedBackendStatsHash) { StatCountersShmemInitDone = true; } } /* * SharedBackendStatsSlotArrayShmemSize returns the size of the shared * backend stats slot array. */ static Size SharedBackendStatsSlotArrayShmemSize(void) { return mul_size(sizeof(BackendStatsSlot), MaxBackends); } /* * CollectActiveBackendStatsIntoHTAB aggregates the stat counters for the * given database id from all the active backends into the databaseStats * hash table. The function doesn't actually filter the slots of active * backends but it's just fine to read the stat counters from all because * exited backends anyway zero out their stat counters when they exit. * * If the database id is InvalidOid, then all the active backends will be * considered regardless of the database they are connected to. * * Otherwise, if the database id is different than InvalidOid, then only * the active backends whose PGPROC->databaseId is the same as the given * database id will be considered, if any. */ static void CollectActiveBackendStatsIntoHTAB(Oid databaseId, HTAB *databaseStats) { for (int backendSlotIdx = 0; backendSlotIdx < MaxBackends; ++backendSlotIdx) { PGPROC *backendProc = GetPGProcByNumber(backendSlotIdx); if (backendProc->pid == 0) { /* unused slot */ continue; } Oid procDatabaseId = backendProc->databaseId; if (procDatabaseId == InvalidOid) { /* * Not connected to any database, something like logical replication * launcher, autovacuum launcher or such. */ continue; } if (databaseId != InvalidOid && databaseId != procDatabaseId) { /* not a database we are interested in */ continue; } DatabaseStatsHashEntry *dbStatsEntry = DatabaseStatsHashEntryFindOrCreate(procDatabaseId, databaseStats); BackendStatsSlot *backendStatsSlot = &SharedBackendStatsSlotArray[backendSlotIdx]; for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { dbStatsEntry->counters[statIdx] += pg_atomic_read_u64(&backendStatsSlot->counters[statIdx]); } } } /* * CollectSavedBackendStatsIntoHTAB fetches the saved stat counters and * resetTimestamp for the given database id from the saved backend stats * hash table and saves them into the databaseStats hash table. * * If the database id is InvalidOid, then all the databases that present * in the saved backend stats hash table will be considered. * * Otherwise, if the database id is different than InvalidOid, then only * the entry that belongs to given database will be considered, if there * is such an entry. */ static void CollectSavedBackendStatsIntoHTAB(Oid databaseId, HTAB *databaseStats) { LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_SHARED); if (databaseId != InvalidOid) { SavedBackendStatsHashEntry *dbSavedBackendStatsEntry = (SavedBackendStatsHashEntry *) hash_search( SharedSavedBackendStatsHash, (void *) &databaseId, HASH_FIND, NULL); if (dbSavedBackendStatsEntry) { DatabaseStatsHashEntry *dbStatsEntry = DatabaseStatsHashEntryFindOrCreate(databaseId, databaseStats); SpinLockAcquire(&dbSavedBackendStatsEntry->mutex); for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { dbStatsEntry->counters[statIdx] += dbSavedBackendStatsEntry->counters[statIdx]; } dbStatsEntry->resetTimestamp = dbSavedBackendStatsEntry->resetTimestamp; SpinLockRelease(&dbSavedBackendStatsEntry->mutex); } } else { HASH_SEQ_STATUS hashSeqStatus; hash_seq_init(&hashSeqStatus, SharedSavedBackendStatsHash); SavedBackendStatsHashEntry *dbSavedBackendStatsEntry = NULL; while ((dbSavedBackendStatsEntry = hash_seq_search(&hashSeqStatus)) != NULL) { DatabaseStatsHashEntry *dbStatsEntry = DatabaseStatsHashEntryFindOrCreate(dbSavedBackendStatsEntry->databaseId, databaseStats); SpinLockAcquire(&dbSavedBackendStatsEntry->mutex); for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { dbStatsEntry->counters[statIdx] += dbSavedBackendStatsEntry->counters[statIdx]; } dbStatsEntry->resetTimestamp = dbSavedBackendStatsEntry->resetTimestamp; SpinLockRelease(&dbSavedBackendStatsEntry->mutex); } } LWLockRelease(*SharedSavedBackendStatsHashLock); } /* * DatabaseStatsHashEntryFindOrCreate creates a new entry in databaseStats * hash table for the given database id if it doesn't already exist and * initializes it, or just returns the existing entry if it does. */ static DatabaseStatsHashEntry * DatabaseStatsHashEntryFindOrCreate(Oid databaseId, HTAB *databaseStats) { bool found = false; DatabaseStatsHashEntry *dbStatsEntry = (DatabaseStatsHashEntry *) hash_search(databaseStats, &databaseId, HASH_ENTER, &found); if (!found) { MemSet(dbStatsEntry->counters, 0, sizeof(StatCounters)); dbStatsEntry->resetTimestamp = 0; } return dbStatsEntry; } /* * StoreDatabaseStatsIntoTupStore stores the database stats from the * databaseStats hash table into given tuple store. */ static void StoreDatabaseStatsIntoTupStore(HTAB *databaseStats, Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { HASH_SEQ_STATUS hashSeqStatus; hash_seq_init(&hashSeqStatus, databaseStats); DatabaseStatsHashEntry *dbStatsEntry = NULL; while ((dbStatsEntry = hash_seq_search(&hashSeqStatus)) != NULL) { /* +2 for database_id (first) and the stats_reset (last) column */ Datum values[N_CITUS_STAT_COUNTERS + 2] = { 0 }; bool isNulls[N_CITUS_STAT_COUNTERS + 2] = { 0 }; values[0] = ObjectIdGetDatum(dbStatsEntry->databaseId); for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { uint64 statCounter = dbStatsEntry->counters[statIdx]; values[statIdx + 1] = UInt64GetDatum(statCounter); } /* set stats_reset column to NULL if it was never reset */ if (dbStatsEntry->resetTimestamp == 0) { isNulls[N_CITUS_STAT_COUNTERS + 1] = true; } else { values[N_CITUS_STAT_COUNTERS + 1] = TimestampTzGetDatum(dbStatsEntry->resetTimestamp); } tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } } /* * ResetActiveBackendStats resets the stat counters for the given database * id for all the active backends. The function doesn't actually filter the * slots of active backends but it's just fine to reset the stat counters * for all because doing so just means resetting the stat counters for * exited backends once again, which were already reset when they exited. * * Only active backends whose PGPROC->databaseId is the same as the given * database id will be considered, if any. * * Returns true if any active backend was found. */ static bool ResetActiveBackendStats(Oid databaseId) { bool foundAny = false; for (int backendSlotIdx = 0; backendSlotIdx < MaxBackends; ++backendSlotIdx) { PGPROC *backendProc = GetPGProcByNumber(backendSlotIdx); if (backendProc->pid == 0) { /* unused slot */ continue; } Oid procDatabaseId = backendProc->databaseId; if (procDatabaseId == InvalidOid) { /* * not connected to any database, something like logical replication * launcher, autovacuum launcher, etc. */ continue; } if (databaseId != procDatabaseId) { /* not a database we are interested in */ continue; } foundAny = true; BackendStatsSlot *backendStatsSlot = &SharedBackendStatsSlotArray[backendSlotIdx]; for (int statIdx = 0; statIdx < N_CITUS_STAT_COUNTERS; statIdx++) { pg_atomic_write_u64(&backendStatsSlot->counters[statIdx], 0); } } return foundAny; } /* * ResetSavedBackendStats resets the saved stat counters for the given * database id and sets the resetTimestamp for it to the current timestamp. * * If force is true, then we first make sure that we have an entry for * the given database id in the saved backend stats hash table. */ static void ResetSavedBackendStats(Oid databaseId, bool force) { LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_SHARED); SavedBackendStatsHashEntry *dbSavedBackendStatsEntry = (SavedBackendStatsHashEntry *) hash_search( SharedSavedBackendStatsHash, (void *) &databaseId, HASH_FIND, NULL); if (!dbSavedBackendStatsEntry && force) { /* promote the lock to exclusive to insert the new entry for this database */ LWLockRelease(*SharedSavedBackendStatsHashLock); LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_EXCLUSIVE); dbSavedBackendStatsEntry = SavedBackendStatsHashEntryCreateIfNotExists(databaseId); LWLockRelease(*SharedSavedBackendStatsHashLock); if (!dbSavedBackendStatsEntry) { /* * Couldn't allocate a new hash entry because we're out of * (shared) memory. */ ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("failed to allocate saved backend stats hash entry"))); return; } /* re-acquire the shared lock */ LWLockAcquire(*SharedSavedBackendStatsHashLock, LW_SHARED); } /* * Actually reset the stat counters for the exited backends and set * the resetTimestamp to the current timestamp if we already had * an entry for it or if we just created it. */ if (dbSavedBackendStatsEntry) { SpinLockAcquire(&dbSavedBackendStatsEntry->mutex); memset(dbSavedBackendStatsEntry->counters, 0, sizeof(StatCounters)); dbSavedBackendStatsEntry->resetTimestamp = GetCurrentTimestamp(); SpinLockRelease(&dbSavedBackendStatsEntry->mutex); } LWLockRelease(*SharedSavedBackendStatsHashLock); } /* * SavedBackendStatsHashEntryCreateIfNotExists creates a new entry in the * saved backend stats hash table for the given database id if it doesn't * already exist and initializes it. * * Assumes that the caller has exclusive access to the hash table since it * performs HASH_ENTER_NULL. * * Returns NULL if the entry didn't exist and couldn't be allocated since * we're out of (shared) memory. */ static SavedBackendStatsHashEntry * SavedBackendStatsHashEntryCreateIfNotExists(Oid databaseId) { bool found = false; SavedBackendStatsHashEntry *dbSavedBackendStatsEntry = (SavedBackendStatsHashEntry *) hash_search(SharedSavedBackendStatsHash, (void *) &databaseId, HASH_ENTER_NULL, &found); if (!dbSavedBackendStatsEntry) { /* * As we provided HASH_ENTER_NULL, returning NULL means OOM. * In that case, we return and let the caller decide what to do. */ return NULL; } if (!found) { memset(dbSavedBackendStatsEntry->counters, 0, sizeof(StatCounters)); dbSavedBackendStatsEntry->resetTimestamp = 0; SpinLockInit(&dbSavedBackendStatsEntry->mutex); } return dbSavedBackendStatsEntry; } ================================================ FILE: src/backend/distributed/stats/stat_tenants.c ================================================ /*------------------------------------------------------------------------- * * stat_tenants.c * Routines related to the multi tenant monitor. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "unistd.h" #include "access/hash.h" #include "common/pg_prng.h" #include "executor/execdesc.h" #include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/shmem.h" #include "sys/time.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/citus_safe_lib.h" #include "distributed/colocation_utils.h" #include "distributed/distributed_planner.h" #include "distributed/jsonbutils.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/stats/stat_tenants.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/tuplestore.h" static void AttributeMetricsIfApplicable(void); ExecutorEnd_hook_type prev_ExecutorEnd = NULL; #define ATTRIBUTE_PREFIX "/*{\"cId\":" #define ATTRIBUTE_STRING_FORMAT "/*{\"cId\":%d,\"tId\":%s}*/" #define ATTRIBUTE_STRING_FORMAT_WITHOUT_TID "/*{\"cId\":%d}*/" #define STAT_TENANTS_COLUMNS 9 #define ONE_QUERY_SCORE 1000000000 /* this doesn't attempt dereferencing given input and is computed in compile-time, so it's safe */ #define TENANT_STATS_SCORE_FIELD_BIT_LENGTH (sizeof(((TenantStats *) NULL)->score) * 8) static char AttributeToTenant[MAX_TENANT_ATTRIBUTE_LENGTH] = ""; static CmdType AttributeToCommandType = CMD_UNKNOWN; static int AttributeToColocationGroupId = INVALID_COLOCATION_ID; static clock_t QueryStartClock = { 0 }; static clock_t QueryEndClock = { 0 }; static const char *SharedMemoryNameForMultiTenantMonitor = "Shared memory for multi tenant monitor"; static char *MonitorTrancheName = "Multi Tenant Monitor Tranche"; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static int CompareTenantScore(const void *leftElement, const void *rightElement); static void UpdatePeriodsIfNecessary(TenantStats *tenantStats, TimestampTz queryTime); static void ReduceScoreIfNecessary(TenantStats *tenantStats, TimestampTz queryTime); static void EvictTenantsIfNecessary(TimestampTz queryTime); static void RecordTenantStats(TenantStats *tenantStats, TimestampTz queryTime); static MultiTenantMonitor * CreateSharedMemoryForMultiTenantMonitor(void); static MultiTenantMonitor * GetMultiTenantMonitor(void); static void MultiTenantMonitorSMInit(void); static TenantStats * CreateTenantStats(MultiTenantMonitor *monitor, TimestampTz queryTime); static void FillTenantStatsHashKey(TenantStatsHashKey *key, char *tenantAttribute, uint32 colocationGroupId); static TenantStats * FindTenantStats(MultiTenantMonitor *monitor); static size_t MultiTenantMonitorshmemSize(void); static char * ExtractTopComment(const char *inputString); static char * EscapeCommentChars(const char *str); static char * UnescapeCommentChars(const char *str); int StatTenantsLogLevel = CITUS_LOG_LEVEL_OFF; int StatTenantsPeriod = (time_t) 60; int StatTenantsLimit = 100; int StatTenantsTrack = STAT_TENANTS_TRACK_NONE; double StatTenantsSampleRateForNewTenants = 1; PG_FUNCTION_INFO_V1(citus_stat_tenants_local); PG_FUNCTION_INFO_V1(citus_stat_tenants_local_reset); /* * citus_stat_tenants_local finds, updates and returns the statistics for tenants. */ Datum citus_stat_tenants_local(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* * We keep more than StatTenantsLimit tenants in our monitor. * We do this to not lose data if a tenant falls out of top StatTenantsLimit in case they need to return soon. * Normally we return StatTenantsLimit tenants but if returnAllTenants is true we return all of them. */ bool returnAllTenants = PG_GETARG_BOOL(0); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); TimestampTz monitoringTime = GetCurrentTimestamp(); Datum values[STAT_TENANTS_COLUMNS]; bool isNulls[STAT_TENANTS_COLUMNS]; MultiTenantMonitor *monitor = GetMultiTenantMonitor(); if (monitor == NULL) { PG_RETURN_VOID(); } LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); int numberOfRowsToReturn = 0; int tenantStatsCount = hash_get_num_entries(monitor->tenants); if (returnAllTenants) { numberOfRowsToReturn = tenantStatsCount; } else { numberOfRowsToReturn = Min(tenantStatsCount, StatTenantsLimit); } /* Allocate an array to hold the tenants. */ TenantStats **stats = palloc(tenantStatsCount * sizeof(TenantStats *)); HASH_SEQ_STATUS hash_seq; TenantStats *stat; /* Get all the tenants from the hash table. */ int j = 0; hash_seq_init(&hash_seq, monitor->tenants); while ((stat = hash_seq_search(&hash_seq)) != NULL) { stats[j++] = stat; UpdatePeriodsIfNecessary(stat, monitoringTime); ReduceScoreIfNecessary(stat, monitoringTime); } /* Sort the tenants by their score. */ SafeQsort(stats, j, sizeof(TenantStats *), CompareTenantScore); for (int i = 0; i < numberOfRowsToReturn; i++) { memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); TenantStats *tenantStats = stats[i]; values[0] = Int32GetDatum(tenantStats->key.colocationGroupId); if (tenantStats->key.tenantAttribute[0] == '\0') { isNulls[1] = true; } else { values[1] = PointerGetDatum(cstring_to_text( tenantStats->key.tenantAttribute)); } values[2] = Int32GetDatum(tenantStats->readsInThisPeriod); values[3] = Int32GetDatum(tenantStats->readsInLastPeriod); values[4] = Int32GetDatum(tenantStats->readsInThisPeriod + tenantStats->writesInThisPeriod); values[5] = Int32GetDatum(tenantStats->readsInLastPeriod + tenantStats->writesInLastPeriod); values[6] = Float8GetDatum(tenantStats->cpuUsageInThisPeriod); values[7] = Float8GetDatum(tenantStats->cpuUsageInLastPeriod); values[8] = Int64GetDatum(tenantStats->score); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } pfree(stats); LWLockRelease(&monitor->lock); PG_RETURN_VOID(); } /* * citus_stat_tenants_local_reset resets monitor for tenant statistics * on the local node. */ Datum citus_stat_tenants_local_reset(PG_FUNCTION_ARGS) { MultiTenantMonitor *monitor = GetMultiTenantMonitor(); /* if monitor is not created yet, there is nothing to reset */ if (monitor == NULL) { PG_RETURN_VOID(); } HASH_SEQ_STATUS hash_seq; TenantStats *stats; LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); hash_seq_init(&hash_seq, monitor->tenants); while ((stats = hash_seq_search(&hash_seq)) != NULL) { hash_search(monitor->tenants, &stats->key, HASH_REMOVE, NULL); } LWLockRelease(&monitor->lock); PG_RETURN_VOID(); } /* * AttributeQueryIfAnnotated checks the query annotation and if the query is annotated * for the tenant statistics monitoring this function records the tenant attributes. */ void AttributeQueryIfAnnotated(const char *query_string, CmdType commandType) { if (StatTenantsTrack == STAT_TENANTS_TRACK_NONE) { return; } AttributeToColocationGroupId = INVALID_COLOCATION_ID; if (query_string == NULL) { return; } if (strncmp(ATTRIBUTE_PREFIX, query_string, strlen(ATTRIBUTE_PREFIX)) == 0) { char *annotation = ExtractTopComment(query_string); if (annotation != NULL) { Datum jsonbDatum = DirectFunctionCall1(jsonb_in, PointerGetDatum(annotation)); text *tenantIdTextP = ExtractFieldTextP(jsonbDatum, "tId"); char *tenantId = NULL; if (tenantIdTextP != NULL) { tenantId = UnescapeCommentChars(text_to_cstring(tenantIdTextP)); } int colocationId = ExtractFieldInt32(jsonbDatum, "cId", INVALID_COLOCATION_ID); AttributeTask(tenantId, colocationId, commandType); } } } /* * AttributeTask assigns the given attributes of a tenant and starts a timer */ void AttributeTask(char *tenantId, int colocationId, CmdType commandType) { if (StatTenantsTrack == STAT_TENANTS_TRACK_NONE || colocationId == INVALID_COLOCATION_ID) { return; } TenantStatsHashKey key = { 0 }; FillTenantStatsHashKey(&key, tenantId, colocationId); MultiTenantMonitor *monitor = GetMultiTenantMonitor(); bool found = false; /* Acquire the lock in shared mode to check if the tenant is already in the hash table. */ LWLockAcquire(&monitor->lock, LW_SHARED); hash_search(monitor->tenants, &key, HASH_FIND, &found); LWLockRelease(&monitor->lock); /* If the tenant is not found in the hash table, we will track the query with a probability of StatTenantsSampleRateForNewTenants. */ if (!found) { double randomValue = pg_prng_double(&pg_global_prng_state); bool shouldTrackQuery = randomValue <= StatTenantsSampleRateForNewTenants; if (!shouldTrackQuery) { return; } } /* * if tenantId is NULL, it must be a schema-based tenant and * we try to get the tenantId from the colocationId to lookup schema name and use it as a tenantId */ if (tenantId == NULL) { if (!IsTenantSchemaColocationGroup(colocationId)) { return; } } AttributeToColocationGroupId = colocationId; if (tenantId != NULL) { strncpy_s(AttributeToTenant, MAX_TENANT_ATTRIBUTE_LENGTH, tenantId, MAX_TENANT_ATTRIBUTE_LENGTH - 1); } else { strcpy_s(AttributeToTenant, sizeof(AttributeToTenant), ""); } AttributeToCommandType = commandType; QueryStartClock = clock(); } /* * AnnotateQuery annotates the query with tenant attributes. * if the query has a partition key, we annotate it with the partition key value and colocationId * if the query doesn't have a partition key and if it's a schema-based tenant, we annotate it with the colocationId only. */ char * AnnotateQuery(char *queryString, Const *partitionKeyValue, int colocationId) { if (StatTenantsTrack == STAT_TENANTS_TRACK_NONE || colocationId == INVALID_COLOCATION_ID) { return queryString; } StringInfo newQuery = makeStringInfo(); /* if the query doesn't have a parititon key value, check if it is a tenant schema */ if (partitionKeyValue == NULL) { if (IsTenantSchemaColocationGroup(colocationId)) { /* If it is a schema-based tenant, we only annotate the query with colocationId */ appendStringInfo(newQuery, ATTRIBUTE_STRING_FORMAT_WITHOUT_TID, colocationId); } else { /* If it is not a schema-based tenant query and doesn't have a parititon key, * we don't annotate it */ return queryString; } } else { /* if the query has a partition key value, we annotate it with both tenantId and colocationId */ char *partitionKeyValueString = DatumToString(partitionKeyValue->constvalue, partitionKeyValue->consttype); char *commentCharsEscaped = EscapeCommentChars(partitionKeyValueString); StringInfo escapedSourceName = makeStringInfo(); escape_json(escapedSourceName, commentCharsEscaped); appendStringInfo(newQuery, ATTRIBUTE_STRING_FORMAT, colocationId, escapedSourceName->data ); } appendStringInfoString(newQuery, queryString); return newQuery->data; } /* * CitusAttributeToEnd keeps the statistics for the tenant and calls the previously installed end hook * or the standard executor end function. */ void CitusAttributeToEnd(QueryDesc *queryDesc) { /* * At the end of the Executor is the last moment we have to attribute the previous * attribution to a tenant, if applicable */ AttributeMetricsIfApplicable(); /* now call in to the previously installed hook, or the standard implementation */ if (prev_ExecutorEnd) { prev_ExecutorEnd(queryDesc); } else { standard_ExecutorEnd(queryDesc); } } /* * CompareTenantScore is used to sort the tenant statistics by score * in descending order. */ static int CompareTenantScore(const void *leftElement, const void *rightElement) { double l_usage = (*(TenantStats *const *) leftElement)->score; double r_usage = (*(TenantStats *const *) rightElement)->score; if (l_usage > r_usage) { return -1; } else if (l_usage < r_usage) { return 1; } return 0; } /* * AttributeMetricsIfApplicable updates the metrics for current tenant's statistics */ static void AttributeMetricsIfApplicable() { if (StatTenantsTrack == STAT_TENANTS_TRACK_NONE || AttributeToColocationGroupId == INVALID_COLOCATION_ID) { return; } /* * return if we are not in the top level to make sure we are not * stopping counting time for a sub-level execution */ if (ExecutorLevel != 0 || PlannerLevel != 0) { return; } QueryEndClock = clock(); TimestampTz queryTime = GetCurrentTimestamp(); MultiTenantMonitor *monitor = GetMultiTenantMonitor(); /* * We need to acquire the monitor lock in shared mode to check if the tenant is * already in the monitor. If it is not, we need to acquire the lock in * exclusive mode to add the tenant to the monitor. * * We need to check again if the tenant is in the monitor after acquiring the * exclusive lock to avoid adding the tenant twice. Some other backend might * have added the tenant while we were waiting for the lock. * * After releasing the exclusive lock, we need to acquire the lock in shared * mode to update the tenant's statistics. We need to check again if the tenant * is in the monitor after acquiring the shared lock because some other backend * might have removed the tenant while we were waiting for the lock. */ LWLockAcquire(&monitor->lock, LW_SHARED); TenantStats *tenantStats = FindTenantStats(monitor); if (tenantStats != NULL) { SpinLockAcquire(&tenantStats->lock); UpdatePeriodsIfNecessary(tenantStats, queryTime); ReduceScoreIfNecessary(tenantStats, queryTime); RecordTenantStats(tenantStats, queryTime); SpinLockRelease(&tenantStats->lock); } else { LWLockRelease(&monitor->lock); LWLockAcquire(&monitor->lock, LW_EXCLUSIVE); tenantStats = FindTenantStats(monitor); if (tenantStats == NULL) { tenantStats = CreateTenantStats(monitor, queryTime); } LWLockRelease(&monitor->lock); LWLockAcquire(&monitor->lock, LW_SHARED); tenantStats = FindTenantStats(monitor); if (tenantStats != NULL) { SpinLockAcquire(&tenantStats->lock); UpdatePeriodsIfNecessary(tenantStats, queryTime); ReduceScoreIfNecessary(tenantStats, queryTime); RecordTenantStats(tenantStats, queryTime); SpinLockRelease(&tenantStats->lock); } } LWLockRelease(&monitor->lock); AttributeToColocationGroupId = INVALID_COLOCATION_ID; } /* * UpdatePeriodsIfNecessary moves the query counts to previous periods if a enough time has passed. * * If 1 period has passed after the latest query, this function moves this period's counts to the last period * and cleans this period's statistics. * * If 2 or more periods has passed after the last query, this function cleans all both this and last period's * statistics. */ static void UpdatePeriodsIfNecessary(TenantStats *tenantStats, TimestampTz queryTime) { long long int periodInMicroSeconds = StatTenantsPeriod * USECS_PER_SEC; long long int periodInMilliSeconds = StatTenantsPeriod * 1000; TimestampTz periodStart = queryTime - (queryTime % periodInMicroSeconds); /* * If the last query in this tenant was before the start of current period * but there are some query count for this period we move them to the last period. */ if (tenantStats->lastQueryTime < periodStart && (tenantStats->writesInThisPeriod || tenantStats->readsInThisPeriod)) { tenantStats->writesInLastPeriod = tenantStats->writesInThisPeriod; tenantStats->writesInThisPeriod = 0; tenantStats->readsInLastPeriod = tenantStats->readsInThisPeriod; tenantStats->readsInThisPeriod = 0; tenantStats->cpuUsageInLastPeriod = tenantStats->cpuUsageInThisPeriod; tenantStats->cpuUsageInThisPeriod = 0; } /* * If the last query is more than two periods ago, we clean the last period counts too. */ if (TimestampDifferenceExceeds(tenantStats->lastQueryTime, periodStart, periodInMilliSeconds)) { tenantStats->writesInLastPeriod = 0; tenantStats->readsInLastPeriod = 0; tenantStats->cpuUsageInLastPeriod = 0; } } /* * ReduceScoreIfNecessary reduces the tenant score only if it is necessary. * * We halve the tenants' scores after each period. This function checks the number of * periods that passed after the lsat score reduction and reduces the score accordingly. */ static void ReduceScoreIfNecessary(TenantStats *tenantStats, TimestampTz queryTime) { long long int periodInMicroSeconds = StatTenantsPeriod * USECS_PER_SEC; TimestampTz periodStart = queryTime - (queryTime % periodInMicroSeconds); /* * With each query we increase the score of tenant by ONE_QUERY_SCORE. * After one period we halve the scores. * * Here we calculate how many periods passed after the last time we did score reduction * If the latest score reduction was in this period this number should be 0, * if it was in the last period this number should be 1 and so on. */ int periodCountAfterLastScoreReduction = (periodStart - tenantStats->lastScoreReduction + periodInMicroSeconds - 1) / periodInMicroSeconds; /* * This should not happen but let's make sure */ if (periodCountAfterLastScoreReduction < 0) { periodCountAfterLastScoreReduction = 0; } /* * If the last score reduction was not in this period we do score reduction now. */ if (periodCountAfterLastScoreReduction > 0) { tenantStats->lastScoreReduction = queryTime; /* addtional check to avoid undefined behavior */ tenantStats->score = (periodCountAfterLastScoreReduction < TENANT_STATS_SCORE_FIELD_BIT_LENGTH) ? tenantStats->score >> periodCountAfterLastScoreReduction : 0; } } /* * EvictTenantsIfNecessary sorts and evicts the tenants if the tenant count is more than or * equal to 3 * StatTenantsLimit. */ static void EvictTenantsIfNecessary(TimestampTz queryTime) { MultiTenantMonitor *monitor = GetMultiTenantMonitor(); /* * We keep up to StatTenantsLimit * 3 tenants instead of StatTenantsLimit, * so we don't lose data immediately after a tenant is out of top StatTenantsLimit * * Every time tenant count hits StatTenantsLimit * 3, we reduce it back to StatTenantsLimit * 2. */ long tenantStatsCount = hash_get_num_entries(monitor->tenants); if (tenantStatsCount >= StatTenantsLimit * 3) { HASH_SEQ_STATUS hash_seq; TenantStats *stat; TenantStats **stats = palloc(tenantStatsCount * sizeof(TenantStats *)); int i = 0; hash_seq_init(&hash_seq, monitor->tenants); while ((stat = hash_seq_search(&hash_seq)) != NULL) { stats[i++] = stat; } SafeQsort(stats, i, sizeof(TenantStats *), CompareTenantScore); for (i = StatTenantsLimit * 2; i < tenantStatsCount; i++) { hash_search(monitor->tenants, &stats[i]->key, HASH_REMOVE, NULL); } pfree(stats); } } /* * RecordTenantStats records the query statistics for the tenant. */ static void RecordTenantStats(TenantStats *tenantStats, TimestampTz queryTime) { if (tenantStats->score < LLONG_MAX - ONE_QUERY_SCORE) { tenantStats->score += ONE_QUERY_SCORE; } else { tenantStats->score = LLONG_MAX; } if (AttributeToCommandType == CMD_SELECT) { tenantStats->readsInThisPeriod++; } else if (AttributeToCommandType == CMD_UPDATE || AttributeToCommandType == CMD_INSERT || AttributeToCommandType == CMD_DELETE) { tenantStats->writesInThisPeriod++; } double queryCpuTime = ((double) (QueryEndClock - QueryStartClock)) / CLOCKS_PER_SEC; tenantStats->cpuUsageInThisPeriod += queryCpuTime; tenantStats->lastQueryTime = queryTime; } /* * CreateSharedMemoryForMultiTenantMonitor creates a dynamic shared memory segment for multi tenant monitor. */ static MultiTenantMonitor * CreateSharedMemoryForMultiTenantMonitor() { bool found = false; MultiTenantMonitor *monitor = ShmemInitStruct(SharedMemoryNameForMultiTenantMonitor, MultiTenantMonitorshmemSize(), &found); if (found) { return monitor; } monitor->namedLockTranche.trancheId = LWLockNewTrancheId(); monitor->namedLockTranche.trancheName = MonitorTrancheName; LWLockRegisterTranche(monitor->namedLockTranche.trancheId, monitor->namedLockTranche.trancheName); LWLockInitialize(&monitor->lock, monitor->namedLockTranche.trancheId); HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(TenantStatsHashKey); info.entrysize = sizeof(TenantStats); monitor->tenants = ShmemInitHash("citus_stats_tenants hash", StatTenantsLimit * 3, StatTenantsLimit * 3, &info, HASH_ELEM | HASH_SHARED_MEM | HASH_BLOBS); return monitor; } /* * GetMultiTenantMonitor returns the data structure for multi tenant monitor. */ static MultiTenantMonitor * GetMultiTenantMonitor() { bool found = false; MultiTenantMonitor *monitor = ShmemInitStruct(SharedMemoryNameForMultiTenantMonitor, MultiTenantMonitorshmemSize(), &found); if (!found) { elog(WARNING, "monitor not found"); return NULL; } return monitor; } /* * InitializeMultiTenantMonitorSMHandleManagement sets up the shared memory startup hook * so that the multi tenant monitor can be initialized and stored in shared memory. */ void InitializeMultiTenantMonitorSMHandleManagement() { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = MultiTenantMonitorSMInit; } /* * MultiTenantMonitorSMInit initializes the shared memory for MultiTenantMonitorSMData. */ static void MultiTenantMonitorSMInit() { CreateSharedMemoryForMultiTenantMonitor(); if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * CreateTenantStats creates the data structure for a tenant's statistics. * * Calling this function should be protected by the monitor->lock in LW_EXCLUSIVE mode. */ static TenantStats * CreateTenantStats(MultiTenantMonitor *monitor, TimestampTz queryTime) { /* * If the tenant count reached 3 * StatTenantsLimit, we evict the tenants * with the lowest score. */ EvictTenantsIfNecessary(queryTime); TenantStatsHashKey key = { 0 }; FillTenantStatsHashKey(&key, AttributeToTenant, AttributeToColocationGroupId); TenantStats *stats = (TenantStats *) hash_search(monitor->tenants, &key, HASH_ENTER, NULL); stats->writesInLastPeriod = 0; stats->writesInThisPeriod = 0; stats->readsInLastPeriod = 0; stats->readsInThisPeriod = 0; stats->cpuUsageInLastPeriod = 0; stats->cpuUsageInThisPeriod = 0; stats->score = 0; stats->lastScoreReduction = 0; SpinLockInit(&stats->lock); return stats; } /* * FindTenantStats finds the current tenant's statistics. */ static TenantStats * FindTenantStats(MultiTenantMonitor *monitor) { TenantStatsHashKey key = { 0 }; FillTenantStatsHashKey(&key, AttributeToTenant, AttributeToColocationGroupId); TenantStats *stats = (TenantStats *) hash_search(monitor->tenants, &key, HASH_FIND, NULL); return stats; } static void FillTenantStatsHashKey(TenantStatsHashKey *key, char *tenantAttribute, uint32 colocationGroupId) { memset(key->tenantAttribute, 0, MAX_TENANT_ATTRIBUTE_LENGTH); if (tenantAttribute != NULL) { strlcpy(key->tenantAttribute, tenantAttribute, MAX_TENANT_ATTRIBUTE_LENGTH); } key->colocationGroupId = colocationGroupId; } /* * MultiTenantMonitorshmemSize calculates the size of the multi tenant monitor using * StatTenantsLimit parameter. */ static size_t MultiTenantMonitorshmemSize(void) { Size size = sizeof(MultiTenantMonitor); size = add_size(size, mul_size(sizeof(TenantStats), StatTenantsLimit * 3)); return size; } /* * ExtractTopComment extracts the top-level multi-line comment from a given input string. */ static char * ExtractTopComment(const char *inputString) { int commentCharsLength = 2; int inputStringLen = strlen(inputString); if (inputStringLen < commentCharsLength) { return NULL; } const char *commentStartChars = "/*"; const char *commentEndChars = "*/"; /* If query doesn't start with a comment, return NULL */ if (strstr(inputString, commentStartChars) != inputString) { return NULL; } StringInfo commentData = makeStringInfo(); /* Skip the comment start characters */ const char *commentStart = inputString + commentCharsLength; /* Find the first comment end character */ const char *commentEnd = strstr(commentStart, commentEndChars); if (commentEnd == NULL) { return NULL; } /* Append the comment to the StringInfo buffer */ int commentLength = commentEnd - commentStart; appendStringInfo(commentData, "%.*s", commentLength, commentStart); /* Return the extracted comment */ return commentData->data; } /* EscapeCommentChars adds a backslash before each occurrence of '*' or '/' in the input string */ static char * EscapeCommentChars(const char *str) { int originalStringLength = strlen(str); StringInfo escapedString = makeStringInfo(); for (int originalStringIndex = 0; originalStringIndex < originalStringLength; originalStringIndex++) { if (str[originalStringIndex] == '*' || str[originalStringIndex] == '/') { appendStringInfoChar(escapedString, '\\'); } appendStringInfoChar(escapedString, str[originalStringIndex]); } return escapedString->data; } /* UnescapeCommentChars removes the backslash that precedes '*' or '/' in the input string. */ static char * UnescapeCommentChars(const char *str) { int originalStringLength = strlen(str); StringInfo unescapedString = makeStringInfo(); for (int originalStringindex = 0; originalStringindex < originalStringLength; originalStringindex++) { if (str[originalStringindex] == '\\' && originalStringindex < originalStringLength - 1 && (str[originalStringindex + 1] == '*' || str[originalStringindex + 1] == '/')) { originalStringindex++; } appendStringInfoChar(unescapedString, str[originalStringindex]); } return unescapedString->data; } ================================================ FILE: src/backend/distributed/test/backend_counter.c ================================================ /*------------------------------------------------------------------------- * * test/src/backend_counter.c * * This file contains functions to test the active backend counter * within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "distributed/backend_data.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(get_all_active_client_backend_count); /* * get_all_active_client_backend_count returns the active * client backend count. */ Datum get_all_active_client_backend_count(PG_FUNCTION_ARGS) { PG_RETURN_UINT32(GetExternalClientBackendCount()); } ================================================ FILE: src/backend/distributed/test/citus_depended_object.c ================================================ /* * citus_depended_object.c * * Implements udf function related to hiding citus depended objects while executing * postgres vanilla tests. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_attribute.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_sequence.h" #include "catalog/pg_statistic.h" #include "catalog/pg_trigger.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" #include "distributed/citus_depended_object.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" static bool IsCitusDependentObject(ObjectAddress objectAddress); PG_FUNCTION_INFO_V1(is_citus_depended_object); /* * is_citus_depended_object a wrapper around IsCitusDependentObject, so * see the details there. * * The first parameter expects an oid for * a pg meta class, and the second parameter expects an oid for * the object which is found in the pg meta class. */ Datum is_citus_depended_object(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) { /* Because we want to return false for null arguments, we donot use strict keyword while creating that function. */ PG_RETURN_BOOL(false); } Oid metaTableId = PG_GETARG_OID(0); Oid objectId = PG_GETARG_OID(1); if (!OidIsValid(metaTableId) || !OidIsValid(objectId)) { /* we cannot continue without valid meta table or object oid */ PG_RETURN_BOOL(false); } bool dependsOnCitus = false; ObjectAddress objectAdress = { metaTableId, objectId, 0 }; switch (metaTableId) { case ProcedureRelationId: case AccessMethodRelationId: case EventTriggerRelationId: case TriggerRelationId: case OperatorRelationId: case OperatorClassRelationId: case OperatorFamilyRelationId: case AccessMethodOperatorRelationId: case AccessMethodProcedureRelationId: case TSConfigRelationId: case TSTemplateRelationId: case TSDictionaryRelationId: case LanguageRelationId: case RewriteRelationId: case AttrDefaultRelationId: case NamespaceRelationId: case ConstraintRelationId: case TypeRelationId: case RelationRelationId: { /* meta classes that access their own oid */ dependsOnCitus = IsCitusDependentObject(objectAdress); break; } case EnumRelationId: { /* * we do not directly access the oid in pg_enum, * because it does not exist in pg_depend, but its type does */ objectAdress.classId = TypeRelationId; dependsOnCitus = IsCitusDependentObject(objectAdress); break; } case IndexRelationId: case AttributeRelationId: case SequenceRelationId: case StatisticRelationId: { /* meta classes that access their relation's oid */ objectAdress.classId = RelationRelationId; dependsOnCitus = IsCitusDependentObject(objectAdress); break; } case AggregateRelationId: { /* We access procedure oid for aggregates. */ objectAdress.classId = ProcedureRelationId; dependsOnCitus = IsCitusDependentObject(objectAdress); break; } default: { break; } } PG_RETURN_BOOL(dependsOnCitus); } /* * IsCitusDependentObject returns true if the given object depends on the citus extension. */ static bool IsCitusDependentObject(ObjectAddress objectAddress) { if (IsObjectAddressOwnedByCitus(&objectAddress)) { /* object itself is owned by citus */ return true; } /* check if object's any dependency is owned by citus. */ List *citusDependencies = GetAllCitusDependedDependenciesForObject(&objectAddress); return list_length(citusDependencies) > 0; } ================================================ FILE: src/backend/distributed/test/citus_stat_tenants.c ================================================ /*------------------------------------------------------------------------- * * citus_stat_tenants.c * * This file contains functions to test citus_stat_tenants. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "sys/time.h" #include "distributed/stats/stat_tenants.h" PG_FUNCTION_INFO_V1(sleep_until_next_period); /* * sleep_until_next_period sleeps until the next monitoring period starts. */ Datum sleep_until_next_period(PG_FUNCTION_ARGS) { struct timeval currentTime; gettimeofday(¤tTime, NULL); long int nextPeriodStart = currentTime.tv_sec - (currentTime.tv_sec % StatTenantsPeriod) + StatTenantsPeriod; long int sleepTime = (nextPeriodStart - currentTime.tv_sec) * 1000000 - currentTime.tv_usec + 100000; pg_usleep(sleepTime); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/colocation_utils.c ================================================ /*------------------------------------------------------------------------- * * test/src/colocations_utils.c * * This file contains functions to test co-location functionality * within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "catalog/pg_type.h" #include "distributed/colocation_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/utils/array_type.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(get_table_colocation_id); PG_FUNCTION_INFO_V1(tables_colocated); PG_FUNCTION_INFO_V1(shards_colocated); PG_FUNCTION_INFO_V1(get_colocated_table_array); PG_FUNCTION_INFO_V1(find_shard_interval_index); /* * get_table_colocation_id returns colocation id of given distributed table. */ Datum get_table_colocation_id(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); uint32 colocationId = TableColocationId(distributedTableId); PG_RETURN_INT32(colocationId); } /* * tables_colocated checks if given two tables are co-located or not. If they are * co-located, this function returns true. */ Datum tables_colocated(PG_FUNCTION_ARGS) { Oid leftDistributedTableId = PG_GETARG_OID(0); Oid rightDistributedTableId = PG_GETARG_OID(1); bool tablesColocated = TablesColocated(leftDistributedTableId, rightDistributedTableId); PG_RETURN_BOOL(tablesColocated); } /* * shards_colocated checks if given two shards are co-located or not. If they are * co-located, this function returns true. */ Datum shards_colocated(PG_FUNCTION_ARGS) { uint32 leftShardId = PG_GETARG_UINT32(0); uint32 rightShardId = PG_GETARG_UINT32(1); ShardInterval *leftShard = LoadShardInterval(leftShardId); ShardInterval *rightShard = LoadShardInterval(rightShardId); bool shardsColocated = ShardsColocated(leftShard, rightShard); PG_RETURN_BOOL(shardsColocated); } /* * get_colocated_tables_array returns array of table oids which are co-located with given * distributed table. */ Datum get_colocated_table_array(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); List *colocatedTableList = ColocatedTableList(distributedTableId); int colocatedTableCount = list_length(colocatedTableList); Datum *colocatedTablesDatumArray = palloc0(colocatedTableCount * sizeof(Datum)); Oid arrayTypeId = OIDOID; int colocatedTableIndex = 0; Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { Datum colocatedTableDatum = ObjectIdGetDatum(colocatedTableId); colocatedTablesDatumArray[colocatedTableIndex] = colocatedTableDatum; colocatedTableIndex++; } ArrayType *colocatedTablesArrayType = DatumArrayToArrayType(colocatedTablesDatumArray, colocatedTableCount, arrayTypeId); PG_RETURN_ARRAYTYPE_P(colocatedTablesArrayType); } /* * find_shard_interval_index finds index of given shard in sorted shard interval list. */ Datum find_shard_interval_index(PG_FUNCTION_ARGS) { uint32 shardId = PG_GETARG_UINT32(0); ShardInterval *shardInterval = LoadShardInterval(shardId); uint32 shardIndex = ShardIndex(shardInterval); PG_RETURN_INT32(shardIndex); } ================================================ FILE: src/backend/distributed/test/create_shards.c ================================================ /*------------------------------------------------------------------------- * * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality * within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "distributed/listutils.h" /* local function forward declarations */ static int CompareStrings(const void *leftElement, const void *rightElement); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(sort_names); /* * sort_names accepts three strings, places them in a list, then calls PGSSortList * to test its sort functionality. Returns a string containing sorted lines. */ Datum sort_names(PG_FUNCTION_ARGS) { char *first = PG_GETARG_CSTRING(0); char *second = PG_GETARG_CSTRING(1); char *third = PG_GETARG_CSTRING(2); List *nameList = SortList(list_make3(first, second, third), (int (*)(const void *, const void *))(&CompareStrings)); StringInfo sortedNames = makeStringInfo(); const char *name = NULL; foreach_declared_ptr(name, nameList) { appendStringInfo(sortedNames, "%s\n", name); } PG_RETURN_CSTRING(sortedNames->data); } /* * A simple wrapper around strcmp suitable for use with PGSSortList or qsort. */ static int CompareStrings(const void *leftElement, const void *rightElement) { const char *leftString = *((const char **) leftElement); const char *rightString = *((const char **) rightElement); return strcmp(leftString, rightString); } ================================================ FILE: src/backend/distributed/test/deparse_function_query.c ================================================ /*------------------------------------------------------------------------- * * test/src/deparse_function_query.c * * This file contains functions to exercise deparsing of * CREATE|ALTER|DROP [...] {FUNCTION|PROCEDURE} ... * queries * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/builtins.h" #include "distributed/deparser.h" #include "distributed/multi_executor.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(deparse_test); /* * deparse_test UDF is a UDF to test deparsing in Citus. * * This function accepts a query string; parses, qualifies and then deparses it to create * a qualified query string. */ Datum deparse_test(PG_FUNCTION_ARGS) { text *queryStringText = PG_GETARG_TEXT_P(0); char *queryStringChar = text_to_cstring(queryStringText); Query *query = ParseQueryString(queryStringChar, NULL, 0); QualifyTreeNode(query->utilityStmt); const char *deparsedQuery = DeparseTreeNode(query->utilityStmt); PG_RETURN_TEXT_P(cstring_to_text(deparsedQuery)); } ================================================ FILE: src/backend/distributed/test/deparse_shard_query.c ================================================ /*------------------------------------------------------------------------- * * test/src/depase_shard_query.c * * This file contains functions to exercise deparsing of INSERT .. SELECT queries * for distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/value.h" #include "tcop/tcopprot.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/palloc.h" #include "distributed/citus_ruleutils.h" #include "distributed/coordinator_protocol.h" #include "distributed/insert_select_planner.h" #include "distributed/listutils.h" #include "distributed/multi_router_planner.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(deparse_shard_query_test); Datum deparse_shard_query_test(PG_FUNCTION_ARGS) { text *queryString = PG_GETARG_TEXT_P(0); char *queryStringChar = text_to_cstring(queryString); List *parseTreeList = pg_parse_query(queryStringChar); Node *parsetree = NULL; foreach_declared_ptr(parsetree, parseTreeList) { List *queryTreeList = pg_analyze_and_rewrite_fixedparams((RawStmt *) parsetree, queryStringChar, NULL, 0, NULL); Query *query = NULL; foreach_declared_ptr(query, queryTreeList) { StringInfo buffer = makeStringInfo(); /* reoreder the target list only for INSERT .. SELECT queries */ if (InsertSelectIntoCitusTable(query)) { RangeTblEntry *insertRte = linitial(query->rtable); RangeTblEntry *subqueryRte = lsecond(query->rtable); ReorderInsertSelectTargetLists(query, insertRte, subqueryRte); } deparse_shard_query(query, InvalidOid, 0, buffer); elog(INFO, "query: %s", buffer->data); } } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/dependency.c ================================================ /*------------------------------------------------------------------------- * * test/src/dependency.c * * This file contains functions to exercise dependency resolution for objects. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "fmgr.h" #include "distributed/listutils.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata_cache.h" #include "distributed/tuplestore.h" PG_FUNCTION_INFO_V1(citus_get_all_dependencies_for_object); PG_FUNCTION_INFO_V1(citus_get_dependencies_for_object); /* * citus_get_all_dependencies_for_object(classid oid, objid oid, objsubid int) * * citus_get_all_dependencies_for_object gets an object and returns all of its * dependencies irrespective of whether the dependencies are already distributed * or not. * * This is to emulate what Citus would qualify as dependency when adding a new * node. */ Datum citus_get_all_dependencies_for_object(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid classid = PG_GETARG_OID(0); Oid objid = PG_GETARG_OID(1); int32 objsubid = PG_GETARG_INT32(2); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); ObjectAddress address = { 0 }; ObjectAddressSubSet(address, classid, objid, objsubid); List *dependencies = GetAllSupportedDependenciesForObject(&address); ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { Datum values[3]; bool isNulls[3]; memset(values, 0, sizeof(values)); memset(isNulls, 0, sizeof(isNulls)); values[0] = ObjectIdGetDatum(dependency->classId); values[1] = ObjectIdGetDatum(dependency->objectId); values[2] = Int32GetDatum(dependency->objectSubId); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } PG_RETURN_VOID(); } /* * citus_get_dependencies_for_object(classid oid, objid oid, objsubid int) * * citus_get_dependencies_for_object gets an object and returns all of its * dependencies that are not already distributed. * * This is to emulate what Citus would qualify as dependency when creating * a new object. */ Datum citus_get_dependencies_for_object(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid classid = PG_GETARG_OID(0); Oid objid = PG_GETARG_OID(1); int32 objsubid = PG_GETARG_INT32(2); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); ObjectAddress address = { 0 }; ObjectAddressSubSet(address, classid, objid, objsubid); List *dependencies = GetDependenciesForObject(&address); ObjectAddress *dependency = NULL; foreach_declared_ptr(dependency, dependencies) { Datum values[3]; bool isNulls[3]; memset(values, 0, sizeof(values)); memset(isNulls, 0, sizeof(isNulls)); values[0] = ObjectIdGetDatum(dependency->classId); values[1] = ObjectIdGetDatum(dependency->objectId); values[2] = Int32GetDatum(dependency->objectSubId); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/distributed_deadlock_detection.c ================================================ /*------------------------------------------------------------------------- * * test/src/distributed_deadlock_detection.c * * This file contains functions to exercise distributed deadlock detection * related lower level functionality. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "access/hash.h" #include "nodes/pg_list.h" #include "utils/hsearch.h" #include "utils/timestamp.h" #include "distributed/backend_data.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/transaction_identifier.h" #include "distributed/tuplestore.h" PG_FUNCTION_INFO_V1(get_adjacency_list_wait_graph); /* * get_adjacency_list_wait_graph returns the wait graph in adjacency list format. For the * details see BuildAdjacencyListForWaitGraph(). * * This function is mostly useful for testing and debugging purposes. */ Datum get_adjacency_list_wait_graph(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; HASH_SEQ_STATUS status; TransactionNode *transactionNode = NULL; Datum values[2]; bool isNulls[2]; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); /* distributed deadlock detection only considers distributed txs */ bool onlyDistributedTx = true; WaitGraph *waitGraph = BuildGlobalWaitGraph(onlyDistributedTx); HTAB *adjacencyList = BuildAdjacencyListsForWaitGraph(waitGraph); /* iterate on all nodes */ hash_seq_init(&status, adjacencyList); while ((transactionNode = (TransactionNode *) hash_seq_search(&status)) != 0) { memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[0] = UInt64GetDatum(transactionNode->transactionId.transactionNumber); values[1] = CStringGetDatum(WaitsForToString(transactionNode->waitsFor)); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/distributed_intermediate_results.c ================================================ /*------------------------------------------------------------------------- * * test/src/distributed_intermediate_results.c * * This file contains functions to test functions related to * src/backend/distributed/executor/distributed_intermediate_results.c. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "catalog/pg_type.h" #include "tcop/tcopprot.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/multi_executor.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" PG_FUNCTION_INFO_V1(partition_task_list_results); PG_FUNCTION_INFO_V1(redistribute_task_list_results); /* * partition_task_list_results partitions results of each of distributed * tasks for the given query with the ranges of the given relation. * Partitioned results for a task are stored on the node that the task * was targeted for. */ Datum partition_task_list_results(PG_FUNCTION_ARGS) { text *resultIdPrefixText = PG_GETARG_TEXT_P(0); char *resultIdPrefix = text_to_cstring(resultIdPrefixText); text *queryText = PG_GETARG_TEXT_P(1); char *queryString = text_to_cstring(queryText); Oid relationId = PG_GETARG_OID(2); bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); PlannedStmt *queryPlan = pg_plan_query(parsedQuery, queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) { ereport(ERROR, (errmsg("query must be distributed and shouldn't require " "any merging on the coordinator."))); } CustomScan *customScan = (CustomScan *) queryPlan->planTree; DistributedPlan *distributedPlan = GetDistributedPlan(customScan); Job *job = distributedPlan->workerJob; List *taskList = job->taskList; CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(relationId); /* * Here SELECT query's target list should match column list of target relation, * so their partition column indexes are equal. */ int partitionColumnIndex = 0; if (IsCitusTableTypeCacheEntry(targetRelation, DISTRIBUTED_TABLE) && IsA( targetRelation->partitionColumn, Var)) { partitionColumnIndex = targetRelation->partitionColumn->varattno - 1; } List *fragmentList = PartitionTasklistResults(resultIdPrefix, taskList, partitionColumnIndex, targetRelation, binaryFormat); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); DistributedResultFragment *fragment = NULL; foreach_declared_ptr(fragment, fragmentList) { bool columnNulls[5] = { 0 }; Datum columnValues[5] = { CStringGetTextDatum(fragment->resultId), UInt32GetDatum(fragment->nodeId), Int64GetDatum(fragment->rowCount), UInt64GetDatum(fragment->targetShardId), Int32GetDatum(fragment->targetShardIndex) }; tuplestore_putvalues(tupleStore, tupleDescriptor, columnValues, columnNulls); } PG_RETURN_DATUM(0); } /* * redistribute_task_list_results exposes RedistributeTaskListResult for testing. * It executes a query and repartitions and colocates its results according to * a relation. */ Datum redistribute_task_list_results(PG_FUNCTION_ARGS) { text *resultIdPrefixText = PG_GETARG_TEXT_P(0); char *resultIdPrefix = text_to_cstring(resultIdPrefixText); text *queryText = PG_GETARG_TEXT_P(1); char *queryString = text_to_cstring(queryText); Oid relationId = PG_GETARG_OID(2); bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); PlannedStmt *queryPlan = pg_plan_query(parsedQuery, queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) { ereport(ERROR, (errmsg("query must be distributed and shouldn't require " "any merging on the coordinator."))); } CustomScan *customScan = (CustomScan *) queryPlan->planTree; DistributedPlan *distributedPlan = GetDistributedPlan(customScan); Job *job = distributedPlan->workerJob; List *taskList = job->taskList; CitusTableCacheEntry *targetRelation = GetCitusTableCacheEntry(relationId); /* * Here SELECT query's target list should match column list of target relation, * so their partition column indexes are equal. */ int partitionColumnIndex = IsCitusTableTypeCacheEntry(targetRelation, DISTRIBUTED_TABLE) ? targetRelation->partitionColumn->varattno - 1 : 0; List **shardResultIds = RedistributeTaskListResults(resultIdPrefix, taskList, partitionColumnIndex, targetRelation, binaryFormat); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); int shardCount = targetRelation->shardIntervalArrayLength; for (int shardIndex = 0; shardIndex < shardCount; shardIndex++) { ShardInterval *shardInterval = targetRelation->sortedShardIntervalArray[shardIndex]; uint64 shardId = shardInterval->shardId; int fragmentCount = list_length(shardResultIds[shardIndex]); Datum *resultIdValues = palloc0(fragmentCount * sizeof(Datum)); List *sortedResultIds = SortList(shardResultIds[shardIndex], pg_qsort_strcmp); const char *resultId = NULL; int resultIdIndex = 0; foreach_declared_ptr(resultId, sortedResultIds) { resultIdValues[resultIdIndex++] = CStringGetTextDatum(resultId); } ArrayType *resultIdArray = DatumArrayToArrayType(resultIdValues, fragmentCount, TEXTOID); bool columnNulls[2] = { 0 }; Datum columnValues[2] = { Int64GetDatum(shardId), PointerGetDatum(resultIdArray) }; tuplestore_putvalues(tupleStore, tupleDescriptor, columnValues, columnNulls); } PG_RETURN_DATUM(0); } ================================================ FILE: src/backend/distributed/test/distribution_metadata.c ================================================ /*------------------------------------------------------------------------- * * test/src/distribution_metadata.c * * This file contains functions to exercise distributed table metadata * functionality within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "access/heapam.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "storage/lock.h" #include "tcop/tcopprot.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/palloc.h" #include "distributed/coordinator_protocol.h" #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_join_order.h" #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_shard.h" #include "distributed/query_utils.h" #include "distributed/resource_lock.h" #include "distributed/utils/array_type.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(load_shard_id_array); PG_FUNCTION_INFO_V1(load_shard_interval_array); PG_FUNCTION_INFO_V1(load_shard_placement_array); PG_FUNCTION_INFO_V1(partition_column_id); PG_FUNCTION_INFO_V1(partition_type); PG_FUNCTION_INFO_V1(is_distributed_table); PG_FUNCTION_INFO_V1(create_monolithic_shard_row); PG_FUNCTION_INFO_V1(acquire_shared_shard_lock); PG_FUNCTION_INFO_V1(relation_count_in_query); /* * load_shard_id_array returns the shard identifiers for a particular * distributed table as a bigint array. If the table is not distributed * yet, the function errors-out. */ Datum load_shard_id_array(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); int shardIdIndex = 0; Oid shardIdTypeId = INT8OID; List *shardList = LoadShardIntervalList(distributedTableId); int shardIdCount = list_length(shardList); Datum *shardIdDatumArray = palloc0(shardIdCount * sizeof(Datum)); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardList) { Datum shardIdDatum = Int64GetDatum(shardInterval->shardId); shardIdDatumArray[shardIdIndex] = shardIdDatum; shardIdIndex++; } ArrayType *shardIdArrayType = DatumArrayToArrayType(shardIdDatumArray, shardIdCount, shardIdTypeId); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * load_shard_interval_array loads a shard interval using a provided identifier * and returns a two-element array consisting of min/max values contained in * that shard interval. If no such interval can be found, this function raises * an error instead. */ Datum load_shard_interval_array(PG_FUNCTION_ARGS) { int64 shardId = PG_GETARG_INT64(0); Oid expectedType PG_USED_FOR_ASSERTS_ONLY = get_fn_expr_argtype(fcinfo->flinfo, 1); ShardInterval *shardInterval = LoadShardInterval(shardId); Datum shardIntervalArray[] = { shardInterval->minValue, shardInterval->maxValue }; Assert(expectedType == shardInterval->valueTypeId); ArrayType *shardIntervalArrayType = DatumArrayToArrayType(shardIntervalArray, 2, shardInterval->valueTypeId); PG_RETURN_ARRAYTYPE_P(shardIntervalArrayType); } /* * load_shard_placement_array loads a shard interval using the provided ID * and returns an array of strings containing the node name and port for each * placement of the specified shard interval. If the second argument is true, * only active placements are returned; otherwise, all are. If no such shard * interval can be found, this function raises an error instead. */ Datum load_shard_placement_array(PG_FUNCTION_ARGS) { int64 shardId = PG_GETARG_INT64(0); bool onlyActive = PG_GETARG_BOOL(1); List *placementList = NIL; int placementIndex = 0; Oid placementTypeId = TEXTOID; StringInfo placementInfo = makeStringInfo(); if (onlyActive) { placementList = ActiveShardPlacementList(shardId); } else { placementList = ShardPlacementList(shardId); } placementList = SortList(placementList, CompareShardPlacementsByWorker); int placementCount = list_length(placementList); Datum *placementDatumArray = palloc0(placementCount * sizeof(Datum)); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, placementList) { appendStringInfo(placementInfo, "%s:%d", placement->nodeName, placement->nodePort); placementDatumArray[placementIndex] = CStringGetTextDatum(placementInfo->data); placementIndex++; resetStringInfo(placementInfo); } ArrayType *placementArrayType = DatumArrayToArrayType(placementDatumArray, placementCount, placementTypeId); PG_RETURN_ARRAYTYPE_P(placementArrayType); } /* * partition_column_id simply finds a distributed table using the provided Oid * and returns the column_id of its partition column. If the specified table is * not distributed, this function raises an error instead. */ Datum partition_column_id(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); uint32 rangeTableId = 1; if (!IsCitusTableType(distributedTableId, HASH_DISTRIBUTED)) { ereport(ERROR, (errmsg("table needs to be hash distributed"))); } Var *partitionColumn = PartitionColumn(distributedTableId, rangeTableId); PG_RETURN_INT16((int16) partitionColumn->varattno); } /* * partition_type simply finds a distributed table using the provided Oid and * returns the type of partitioning in use by that table. If the specified * table is not distributed, this function raises an error instead. */ Datum partition_type(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); char partitionType = PartitionMethod(distributedTableId); PG_RETURN_CHAR(partitionType); } /* * is_distributed_table simply returns whether a given table is distributed. No * errors, just a boolean. */ Datum is_distributed_table(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); bool isCitusTable = IsCitusTable(distributedTableId); PG_RETURN_BOOL(isCitusTable); } /* * create_monolithic_shard_row creates a single shard covering all possible * hash values for a given table and inserts a row representing that shard * into the backing store. It returns the primary key of the new row. */ Datum create_monolithic_shard_row(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); StringInfo minInfo = makeStringInfo(); StringInfo maxInfo = makeStringInfo(); uint64 newShardId = GetNextShardId(); appendStringInfo(minInfo, "%d", PG_INT32_MIN); appendStringInfo(maxInfo, "%d", PG_INT32_MAX); text *minInfoText = cstring_to_text(minInfo->data); text *maxInfoText = cstring_to_text(maxInfo->data); InsertShardRow(distributedTableId, newShardId, SHARD_STORAGE_TABLE, minInfoText, maxInfoText); PG_RETURN_INT64(newShardId); } /* * acquire_shared_shard_lock grabs a shared lock for the specified shard. */ Datum acquire_shared_shard_lock(PG_FUNCTION_ARGS) { int64 shardId = PG_GETARG_INT64(0); LockShardResource(shardId, ShareLock); PG_RETURN_VOID(); } /* * relation_count_in_query return the first query's relation count. */ Datum relation_count_in_query(PG_FUNCTION_ARGS) { text *queryString = PG_GETARG_TEXT_P(0); char *queryStringChar = text_to_cstring(queryString); List *parseTreeList = pg_parse_query(queryStringChar); Node *parsetree = NULL; foreach_declared_ptr(parsetree, parseTreeList) { List *queryTreeList = pg_analyze_and_rewrite_fixedparams((RawStmt *) parsetree, queryStringChar, NULL, 0, NULL); Query *query = NULL; foreach_declared_ptr(query, queryTreeList) { List *rangeTableList = NIL; ExtractRangeTableRelationWalker((Node *) query, &rangeTableList); PG_RETURN_INT32(list_length(rangeTableList)); } } PG_RETURN_INT32(0); } ================================================ FILE: src/backend/distributed/test/fake_am.c ================================================ /*------------------------------------------------------------------------- * * fake_am.c * fake table access method code * * Copyright (c) Citus Data, Inc. * * IDENTIFICATION * Based on https://github.com/michaelpq/pg_plugins/blob/master/blackhole_am/blackhole_am.c * * * NOTES * This file introduces the table access method "fake", which delegates * bare minimum functionality for testing to heapam to provide an append * only access method, and doesn't implement rest of the functionality. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/amapi.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/tableam.h" #include "access/xact.h" #include "catalog/index.h" #include "catalog/storage.h" #include "catalog/storage_xlog.h" #include "commands/vacuum.h" #include "executor/tuptable.h" #include "storage/smgr.h" #include "utils/snapmgr.h" #include "pg_version_compat.h" #include "pg_version_constants.h" PG_FUNCTION_INFO_V1(fake_am_handler); static const TableAmRoutine fake_methods; /* ------------------------------------------------------------------------ * Slot related callbacks for fake AM * ------------------------------------------------------------------------ */ static const TupleTableSlotOps * fake_slot_callbacks(Relation relation) { return &TTSOpsBufferHeapTuple; } /* ------------------------------------------------------------------------ * Table Scan Callbacks for fake AM * ------------------------------------------------------------------------ */ static TableScanDesc fake_scan_begin(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, uint32 flags) { return heap_beginscan(relation, snapshot, nkeys, key, parallel_scan, flags); } static void fake_scan_end(TableScanDesc sscan) { heap_endscan(sscan); } static void fake_scan_rescan(TableScanDesc sscan, ScanKey key, bool set_params, bool allow_strat, bool allow_sync, bool allow_pagemode) { heap_rescan(sscan, key, set_params, allow_strat, allow_sync, allow_pagemode); } static bool fake_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot) { ereport(WARNING, (errmsg("fake_scan_getnextslot"))); return heap_getnextslot(sscan, direction, slot); } /* ------------------------------------------------------------------------ * Index Scan Callbacks for fake AM * ------------------------------------------------------------------------ */ static IndexFetchTableData * fake_index_fetch_begin(Relation rel) { elog(ERROR, "fake_index_fetch_begin not implemented"); return NULL; } static void fake_index_fetch_reset(IndexFetchTableData *scan) { elog(ERROR, "fake_index_fetch_reset not implemented"); } static void fake_index_fetch_end(IndexFetchTableData *scan) { elog(ERROR, "fake_index_fetch_end not implemented"); } static bool fake_index_fetch_tuple(struct IndexFetchTableData *scan, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot, bool *call_again, bool *all_dead) { elog(ERROR, "fake_index_fetch_tuple not implemented"); return false; } /* ------------------------------------------------------------------------ * Callbacks for non-modifying operations on individual tuples for * fake AM. * ------------------------------------------------------------------------ */ static bool fake_fetch_row_version(Relation relation, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot) { elog(ERROR, "fake_fetch_row_version not implemented"); return false; } static void fake_get_latest_tid(TableScanDesc sscan, ItemPointer tid) { elog(ERROR, "fake_get_latest_tid not implemented"); } static bool fake_tuple_tid_valid(TableScanDesc scan, ItemPointer tid) { elog(ERROR, "fake_tuple_tid_valid not implemented"); return false; } static bool fake_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, Snapshot snapshot) { elog(ERROR, "fake_tuple_satisfies_snapshot not implemented"); return false; } static TransactionId fake_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate) { elog(ERROR, "fake_index_delete_tuples not implemented"); return InvalidTransactionId; } /* ---------------------------------------------------------------------------- * Functions for manipulations of physical tuples for fake AM. * ---------------------------------------------------------------------------- */ static void fake_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate) { ereport(WARNING, (errmsg("fake_tuple_insert"))); /* * Code below this point is from heapam_tuple_insert from * heapam_handler.c */ bool shouldFree = true; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); /* Update the tuple with table oid */ slot->tts_tableOid = RelationGetRelid(relation); tuple->t_tableOid = slot->tts_tableOid; /* Perform the insertion, and copy the resulting ItemPointer */ heap_insert(relation, tuple, cid, options, bistate); ItemPointerCopy(&tuple->t_self, &slot->tts_tid); if (shouldFree) { pfree(tuple); } } static void fake_tuple_insert_speculative(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate, uint32 specToken) { elog(ERROR, "fake_tuple_insert_speculative not implemented"); } static void fake_tuple_complete_speculative(Relation relation, TupleTableSlot *slot, uint32 spekToken, bool succeeded) { elog(ERROR, "fake_tuple_complete_speculative not implemented"); } static void fake_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, CommandId cid, int options, BulkInsertState bistate) { ereport(WARNING, (errmsg("fake_multi_insert"))); heap_multi_insert(relation, slots, ntuples, cid, options, bistate); } static TM_Result fake_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, Snapshot snapshot, Snapshot crosscheck, bool wait, TM_FailureData *tmfd, bool changingPart) { elog(ERROR, "fake_tuple_delete not implemented"); } static TM_Result fake_tuple_update(Relation relation, ItemPointer otid, TupleTableSlot *slot, CommandId cid, Snapshot snapshot, Snapshot crosscheck, bool wait, TM_FailureData *tmfd, LockTupleMode *lockmode, TU_UpdateIndexes *update_indexes) { elog(ERROR, "fake_tuple_update not implemented"); } static TM_Result fake_tuple_lock(Relation relation, ItemPointer tid, Snapshot snapshot, TupleTableSlot *slot, CommandId cid, LockTupleMode mode, LockWaitPolicy wait_policy, uint8 flags, TM_FailureData *tmfd) { elog(ERROR, "fake_tuple_lock not implemented"); } static void fake_finish_bulk_insert(Relation relation, int options) { /* nothing to do here */ } /* ------------------------------------------------------------------------ * DDL related callbacks for fake AM. * ------------------------------------------------------------------------ */ static void fake_relation_set_new_filenode(Relation rel, const RelFileLocator *newrnode, char persistence, TransactionId *freezeXid, MultiXactId *minmulti) { /* * Code below is copied from heapam_relation_set_new_filenode in * heapam_handler.c. */ /* * Initialize to the minimum XID that could put tuples in the table. We * know that no xacts older than RecentXmin are still running, so that * will do. */ *freezeXid = RecentXmin; /* * Similarly, initialize the minimum Multixact to the first value that * could possibly be stored in tuples in the table. Running transactions * could reuse values from their local cache, so we are careful to * consider all currently running multis. * * XXX this could be refined further, but is it worth the hassle? */ *minmulti = GetOldestMultiXactId(); SMgrRelation srel = RelationCreateStorage(*newrnode, persistence, true); /* * If required, set up an init fork for an unlogged table so that it can * be correctly reinitialized on restart. An immediate sync is required * even if the page has been logged, because the write did not go through * shared_buffers and therefore a concurrent checkpoint may have moved the * redo pointer past our xlog record. Recovery may as well remove it * while replaying, for example, XLOG_DBASE_CREATE or XLOG_TBLSPC_CREATE * record. Therefore, logging is necessary even if wal_level=minimal. */ if (persistence == RELPERSISTENCE_UNLOGGED) { Assert(rel->rd_rel->relkind == RELKIND_RELATION || rel->rd_rel->relkind == RELKIND_MATVIEW || rel->rd_rel->relkind == RELKIND_TOASTVALUE); smgrcreate(srel, INIT_FORKNUM, false); log_smgrcreate(newrnode, INIT_FORKNUM); smgrimmedsync(srel, INIT_FORKNUM); } smgrclose(srel); } static void fake_relation_nontransactional_truncate(Relation rel) { elog(ERROR, "fake_relation_nontransactional_truncate not implemented"); } static void fake_copy_data(Relation rel, const RelFileLocator *newrnode) { elog(ERROR, "fake_copy_data not implemented"); } static void fake_copy_for_cluster(Relation OldTable, Relation NewTable, Relation OldIndex, bool use_sort, TransactionId OldestXmin, TransactionId *xid_cutoff, MultiXactId *multi_cutoff, double *num_tuples, double *tups_vacuumed, double *tups_recently_dead) { elog(ERROR, "fake_copy_for_cluster not implemented"); } static void fake_vacuum(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy) { elog(WARNING, "fake_copy_for_cluster not implemented"); } static bool fake_scan_analyze_next_block(TableScanDesc scan, #if PG_VERSION_NUM >= PG_VERSION_17 ReadStream *stream) #else BlockNumber blockno, BufferAccessStrategy bstrategy) #endif { /* we don't support analyze, so return false */ return false; } static bool fake_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, double *liverows, double *deadrows, TupleTableSlot *slot) { elog(ERROR, "fake_scan_analyze_next_tuple not implemented"); } static double fake_index_build_range_scan(Relation tableRelation, Relation indexRelation, IndexInfo *indexInfo, bool allow_sync, bool anyvisible, bool progress, BlockNumber start_blockno, BlockNumber numblocks, IndexBuildCallback callback, void *callback_state, TableScanDesc scan) { elog(ERROR, "fake_index_build_range_scan not implemented"); } static void fake_index_validate_scan(Relation tableRelation, Relation indexRelation, IndexInfo *indexInfo, Snapshot snapshot, ValidateIndexState *state) { elog(ERROR, "fake_index_build_range_scan not implemented"); } /* ------------------------------------------------------------------------ * Miscellaneous callbacks for the fake AM * ------------------------------------------------------------------------ */ static uint64 fake_relation_size(Relation rel, ForkNumber forkNumber) { /* * Code below is copied from heapam_relation_size from * heapam_handler.c. */ uint64 nblocks = 0; /* InvalidForkNumber indicates returning the size for all forks */ if (forkNumber == InvalidForkNumber) { for (int i = 0; i < MAX_FORKNUM; i++) { nblocks += smgrnblocks(RelationGetSmgr(rel), i); } } else { nblocks = smgrnblocks(RelationGetSmgr(rel), forkNumber); } return nblocks * BLCKSZ; } /* * Check to see whether the table needs a TOAST table. */ static bool fake_relation_needs_toast_table(Relation rel) { /* we don't test toastable data with this, so no toast table needed */ return false; } /* ------------------------------------------------------------------------ * Planner related callbacks for the fake AM * ------------------------------------------------------------------------ */ static void fake_estimate_rel_size(Relation rel, int32 *attr_widths, BlockNumber *pages, double *tuples, double *allvisfrac) { /* no data available */ *attr_widths = 0; *tuples = 0; *allvisfrac = 0; *pages = 0; } /* ------------------------------------------------------------------------ * Executor related callbacks for the fake AM * ------------------------------------------------------------------------ */ #if PG_VERSION_NUM < PG_VERSION_18 static bool fake_scan_bitmap_next_block(TableScanDesc scan, TBMIterateResult *tbmres) { elog(ERROR, "fake_scan_bitmap_next_block not implemented"); } static bool fake_scan_bitmap_next_tuple(TableScanDesc scan, TBMIterateResult *tbmres, TupleTableSlot *slot) { elog(ERROR, "fake_scan_bitmap_next_tuple not implemented"); } #endif static bool fake_scan_sample_next_block(TableScanDesc scan, SampleScanState *scanstate) { elog(ERROR, "fake_scan_sample_next_block not implemented"); } static bool fake_scan_sample_next_tuple(TableScanDesc scan, SampleScanState *scanstate, TupleTableSlot *slot) { elog(ERROR, "fake_scan_sample_next_tuple not implemented"); } /* ------------------------------------------------------------------------ * Definition of the fake table access method. * ------------------------------------------------------------------------ */ static const TableAmRoutine fake_methods = { .type = T_TableAmRoutine, .slot_callbacks = fake_slot_callbacks, .scan_begin = fake_scan_begin, .scan_end = fake_scan_end, .scan_rescan = fake_scan_rescan, .scan_getnextslot = fake_scan_getnextslot, /* these are common helper functions */ .parallelscan_estimate = table_block_parallelscan_estimate, .parallelscan_initialize = table_block_parallelscan_initialize, .parallelscan_reinitialize = table_block_parallelscan_reinitialize, .index_fetch_begin = fake_index_fetch_begin, .index_fetch_reset = fake_index_fetch_reset, .index_fetch_end = fake_index_fetch_end, .index_fetch_tuple = fake_index_fetch_tuple, .tuple_insert = fake_tuple_insert, .tuple_insert_speculative = fake_tuple_insert_speculative, .tuple_complete_speculative = fake_tuple_complete_speculative, .multi_insert = fake_multi_insert, .tuple_delete = fake_tuple_delete, .tuple_update = fake_tuple_update, .tuple_lock = fake_tuple_lock, .finish_bulk_insert = fake_finish_bulk_insert, .tuple_fetch_row_version = fake_fetch_row_version, .tuple_get_latest_tid = fake_get_latest_tid, .tuple_tid_valid = fake_tuple_tid_valid, .tuple_satisfies_snapshot = fake_tuple_satisfies_snapshot, .index_delete_tuples = fake_index_delete_tuples, .relation_set_new_filelocator = fake_relation_set_new_filenode, .relation_nontransactional_truncate = fake_relation_nontransactional_truncate, .relation_copy_data = fake_copy_data, .relation_copy_for_cluster = fake_copy_for_cluster, .relation_vacuum = fake_vacuum, .scan_analyze_next_block = fake_scan_analyze_next_block, .scan_analyze_next_tuple = fake_scan_analyze_next_tuple, .index_build_range_scan = fake_index_build_range_scan, .index_validate_scan = fake_index_validate_scan, .relation_size = fake_relation_size, .relation_needs_toast_table = fake_relation_needs_toast_table, .relation_estimate_size = fake_estimate_rel_size, #if PG_VERSION_NUM < PG_VERSION_18 /* these two fields were removed in PG 18 */ .scan_bitmap_next_block = fake_scan_bitmap_next_block, .scan_bitmap_next_tuple = fake_scan_bitmap_next_tuple, #endif .scan_sample_next_block = fake_scan_sample_next_block, .scan_sample_next_tuple = fake_scan_sample_next_tuple }; Datum fake_am_handler(PG_FUNCTION_ARGS) { PG_RETURN_POINTER(&fake_methods); } ================================================ FILE: src/backend/distributed/test/fake_fdw.c ================================================ /*------------------------------------------------------------------------- * * test/src/fake_fdw.c * * This file contains a barebones FDW implementation, suitable for use in * test code. Inspired by Andrew Dunstan's blackhole_fdw. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "executor/tuptable.h" #include "foreign/fdwapi.h" #include "nodes/execnodes.h" #include "nodes/nodes.h" #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/plannodes.h" #include "optimizer/pathnode.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" #include "utils/palloc.h" #include "pg_version_compat.h" /* local function forward declarations */ static void FakeGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); static void FakeGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); static ForeignScan * FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses, Plan *outer_plan); static void FakeBeginForeignScan(ForeignScanState *node, int eflags); static TupleTableSlot * FakeIterateForeignScan(ForeignScanState *node); static void FakeReScanForeignScan(ForeignScanState *node); static void FakeEndForeignScan(ForeignScanState *node); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(fake_fdw_handler); /* * fake_fdw_handler populates an FdwRoutine with pointers to the functions * implemented within this file. */ Datum fake_fdw_handler(PG_FUNCTION_ARGS) { FdwRoutine *fdwroutine = makeNode(FdwRoutine); fdwroutine->GetForeignRelSize = FakeGetForeignRelSize; fdwroutine->GetForeignPaths = FakeGetForeignPaths; fdwroutine->GetForeignPlan = FakeGetForeignPlan; fdwroutine->BeginForeignScan = FakeBeginForeignScan; fdwroutine->IterateForeignScan = FakeIterateForeignScan; fdwroutine->ReScanForeignScan = FakeReScanForeignScan; fdwroutine->EndForeignScan = FakeEndForeignScan; PG_RETURN_POINTER(fdwroutine); } /* * FakeGetForeignRelSize populates baserel with a fake relation size. */ static void FakeGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { baserel->rows = 0; baserel->fdw_private = (void *) palloc0(1); } /* * FakeGetForeignPaths adds a single fake foreign path to baserel. */ static void FakeGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { Cost startup_cost = 0; Cost total_cost = startup_cost + baserel->rows; add_path(baserel, (Path *) create_foreignscan_path_compat(root, baserel, NULL, baserel->rows, startup_cost, total_cost, NIL, NULL, NULL, NIL, NIL)); } /* * FakeGetForeignPlan builds a fake foreign plan. */ static ForeignScan * FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses, Plan *outer_plan) { Index scan_relid = baserel->relid; scan_clauses = extract_actual_clauses(scan_clauses, false); return make_foreignscan(tlist, scan_clauses, scan_relid, NIL, NIL, NIL, NIL, outer_plan); } /* * FakeBeginForeignScan begins the fake plan (i.e. does nothing). */ static void FakeBeginForeignScan(ForeignScanState *node, int eflags) { /* this comment is for indentation consistency */ } /* * FakeIterateForeignScan continues the fake plan (i.e. does nothing). */ static TupleTableSlot * FakeIterateForeignScan(ForeignScanState *node) { TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; ExecClearTuple(slot); return slot; } /* * FakeReScanForeignScan restarts the fake plan (i.e. does nothing). */ static void FakeReScanForeignScan(ForeignScanState *node) { /* this comment is for indentation consistency */ } /* * FakeEndForeignScan ends the fake plan (i.e. does nothing). */ static void FakeEndForeignScan(ForeignScanState *node) { /* this comment is for indentation consistency */ } ================================================ FILE: src/backend/distributed/test/foreign_key_relationship_query.c ================================================ /*------------------------------------------------------------------------- * * foreign_key_relationship_query.c * * This file contains UDFs for getting foreign constraint relationship between * distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "funcapi.h" #include "catalog/dependency.h" #include "catalog/pg_constraint.h" #include "utils/builtins.h" #include "distributed/coordinator_protocol.h" #include "distributed/foreign_key_relationship.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/tuplestore.h" #include "distributed/version_compat.h" #define GET_FKEY_CONNECTED_RELATIONS_COLUMNS 1 /* these functions are only exported in the regression tests */ PG_FUNCTION_INFO_V1(get_referencing_relation_id_list); PG_FUNCTION_INFO_V1(get_referenced_relation_id_list); PG_FUNCTION_INFO_V1(get_foreign_key_connected_relations); PG_FUNCTION_INFO_V1(drop_constraint_cascade_via_perform_deletion); /* * drop_constraint_cascade_via_perform_deletion simply drops constraint on * relation via performDeletion. */ Datum drop_constraint_cascade_via_perform_deletion(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); if (PG_ARGISNULL(1)) { /* avoid unexpected crashes in regression tests */ ereport(ERROR, (errmsg("cannot perform operation without constraint " "name argument"))); } text *constraintNameText = PG_GETARG_TEXT_P(1); char *constraintName = text_to_cstring(constraintNameText); /* error if constraint does not exist */ bool missingOk = false; Oid constraintId = get_relation_constraint_oid(relationId, constraintName, missingOk); ObjectAddress constraintObjectAddress; constraintObjectAddress.classId = ConstraintRelationId; constraintObjectAddress.objectId = constraintId; constraintObjectAddress.objectSubId = 0; performDeletion(&constraintObjectAddress, DROP_CASCADE, 0); PG_RETURN_VOID(); } /* * get_referencing_relation_id_list returns the list of table oids that is referencing * by given oid recursively. It uses the list cached in the distributed table cache * entry. */ Datum get_referencing_relation_id_list(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); FuncCallContext *functionContext = NULL; ListCell *foreignRelationCell = NULL; /* for the first we call this UDF, we need to populate the result to return set */ if (SRF_IS_FIRSTCALL()) { Oid relationId = PG_GETARG_OID(0); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); /* create a function context for cross-call persistence */ functionContext = SRF_FIRSTCALL_INIT(); MemoryContext oldContext = MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy( cacheEntry->referencingRelationsViaForeignKey); ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); foreignRelationCell = list_head(refList); wrapper->list = refList; wrapper->listCell = foreignRelationCell; functionContext->user_fctx = wrapper; MemoryContextSwitchTo(oldContext); } /* * On every call to this function, we get the current position in the * statement list. We then iterate to the next position in the list and * return the current statement, if we have not yet reached the end of * list. */ functionContext = SRF_PERCALL_SETUP(); ListCellAndListWrapper *wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { Oid refId = lfirst_oid(wrapper->listCell); wrapper->listCell = lnext(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, ObjectIdGetDatum(refId)); } else { SRF_RETURN_DONE(functionContext); } } /* * get_referenced_relation_id_list returns the list of table oids that is referenced * by given oid recursively. It uses the list cached in the distributed table cache * entry. */ Datum get_referenced_relation_id_list(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); FuncCallContext *functionContext = NULL; ListCell *foreignRelationCell = NULL; /* for the first we call this UDF, we need to populate the result to return set */ if (SRF_IS_FIRSTCALL()) { Oid relationId = PG_GETARG_OID(0); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); /* create a function context for cross-call persistence */ functionContext = SRF_FIRSTCALL_INIT(); MemoryContext oldContext = MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy(cacheEntry->referencedRelationsViaForeignKey); foreignRelationCell = list_head(refList); ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); wrapper->list = refList; wrapper->listCell = foreignRelationCell; functionContext->user_fctx = wrapper; MemoryContextSwitchTo(oldContext); } /* * On every call to this function, we get the current position in the * statement list. We then iterate to the next position in the list and * return the current statement, if we have not yet reached the end of * list. */ functionContext = SRF_PERCALL_SETUP(); ListCellAndListWrapper *wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { Oid refId = lfirst_oid(wrapper->listCell); wrapper->listCell = lnext(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, ObjectIdGetDatum(refId)); } else { SRF_RETURN_DONE(functionContext); } } /* * get_foreign_key_connected_relations takes a relation, and returns relations * that are connected to input relation via a foreign key graph. */ Datum get_foreign_key_connected_relations(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); Oid connectedRelationId; List *fkeyConnectedRelationIdList = GetForeignKeyConnectedRelationIdList(relationId); foreach_declared_oid(connectedRelationId, fkeyConnectedRelationIdList) { Datum values[GET_FKEY_CONNECTED_RELATIONS_COLUMNS]; bool nulls[GET_FKEY_CONNECTED_RELATIONS_COLUMNS]; memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); values[0] = ObjectIdGetDatum(connectedRelationId); tuplestore_putvalues(tupleStore, tupleDescriptor, values, nulls); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/global_pid.c ================================================ /*------------------------------------------------------------------------- * * test/src/global_pid.c * * This file contains functions to test the global pid. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "distributed/backend_data.h" #include "distributed/metadata_cache.h" PG_FUNCTION_INFO_V1(test_assign_global_pid); /* * test_assign_global_pid is the wrapper UDF for AssignGlobalPID and is only meant for use * in tests. */ Datum test_assign_global_pid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); AssignGlobalPID(application_name); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/hide_shards.c ================================================ /*------------------------------------------------------------------------- * * hide_shards.c * * This file contains functions to provide helper UDFs for hiding * shards from the applications. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" #include "distributed/metadata_utility.h" #include "distributed/worker_shard_visibility.h" PG_FUNCTION_INFO_V1(set_backend_type); /* * set_backend_type is an external API to set the MyBackendType and * re-checks the shard visibility. */ Datum set_backend_type(PG_FUNCTION_ARGS) { EnsureSuperUser(); MyBackendType = PG_GETARG_INT32(0); elog(NOTICE, "backend type switched to: %s", GetBackendTypeDesc(MyBackendType)); ResetHideShardsDecision(); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/intermediate_results.c ================================================ /*------------------------------------------------------------------------- * * test/src/intermediate_results.c * * This file contains functions to test functions related to * src/backend/distributed/executor/intermediate_results.c. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "distributed/commands/multi_copy.h" #include "distributed/connection_management.h" #include "distributed/intermediate_results.h" #include "distributed/multi_executor.h" #include "distributed/remote_commands.h" PG_FUNCTION_INFO_V1(store_intermediate_result_on_node); /* * store_intermediate_result_on_node executes a query and streams the results * into a file on the given node. */ Datum store_intermediate_result_on_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeNameText = PG_GETARG_TEXT_P(0); char *nodeNameString = text_to_cstring(nodeNameText); int nodePort = PG_GETARG_INT32(1); text *resultIdText = PG_GETARG_TEXT_P(2); char *resultIdString = text_to_cstring(resultIdText); text *queryText = PG_GETARG_TEXT_P(3); char *queryString = text_to_cstring(queryText); bool writeLocalFile = false; ParamListInfo paramListInfo = NULL; WorkerNode *workerNode = FindWorkerNodeOrError(nodeNameString, nodePort); /* * Make sure that this transaction has a distributed transaction ID. * * Intermediate results will be stored in a directory that is derived * from the distributed transaction ID. */ UseCoordinatedTransaction(); EState *estate = CreateExecutorState(); DestReceiver *resultDest = CreateRemoteFileDestReceiver(resultIdString, estate, list_make1(workerNode), writeLocalFile); ExecuteQueryStringIntoDestReceiver(queryString, paramListInfo, (DestReceiver *) resultDest); FreeExecutorState(estate); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/make_external_connection.c ================================================ /*------------------------------------------------------------------------- * * test/src/make_external_connection.c * * This file contains UDF to connect to a node without using the Citus * internal application_name. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/xact.h" #include "executor/spi.h" #include "lib/stringinfo.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_utils.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" #include "distributed/version_compat.h" PG_FUNCTION_INFO_V1(make_external_connection_to_node); /* * make_external_connection_to_node opens a conneciton to a node * and keeps it until the end of the session. */ Datum make_external_connection_to_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); char *nodeName = text_to_cstring(PG_GETARG_TEXT_P(0)); uint32 nodePort = PG_GETARG_UINT32(1); char *userName = text_to_cstring(PG_GETARG_TEXT_P(2)); char *databaseName = text_to_cstring(PG_GETARG_TEXT_P(3)); StringInfo connectionString = makeStringInfo(); appendStringInfo(connectionString, "host=%s port=%d user=%s dbname=%s", nodeName, nodePort, userName, databaseName); PGconn *pgConn = PQconnectdb(connectionString->data); if (PQstatus(pgConn) != CONNECTION_OK) { PQfinish(pgConn); ereport(ERROR, (errmsg("connection failed"))); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/metadata_sync.c ================================================ /*------------------------------------------------------------------------- * * test/src/metadata_sync.c * * This file contains functions to exercise the metadata snapshoy * generation functionality within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "fmgr.h" #include "miscadmin.h" #include "catalog/pg_type.h" #include "postmaster/postmaster.h" #include "storage/latch.h" #include "utils/array.h" #include "utils/builtins.h" #include "distributed/connection_management.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/listutils.h" #include "distributed/maintenanced.h" #include "distributed/metadata_sync.h" #include "distributed/remote_commands.h" #include "distributed/utils/array_type.h" #include "distributed/worker_manager.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(activate_node_snapshot); PG_FUNCTION_INFO_V1(wait_until_metadata_sync); PG_FUNCTION_INFO_V1(trigger_metadata_sync); PG_FUNCTION_INFO_V1(raise_error_in_metadata_sync); /* * activate_node_snapshot prints all the queries that are required * to activate a node. */ Datum activate_node_snapshot(PG_FUNCTION_ARGS) { /* * Activate node commands are created using the given worker node, * so we are using first primary worker node just for test purposes. */ WorkerNode *dummyWorkerNode = GetFirstPrimaryWorkerNode(); if (dummyWorkerNode == NULL) { ereport(ERROR, (errmsg("no worker nodes found"), errdetail("Function activate_node_snapshot is meant to be " "used when running tests on a multi-node cluster " "with workers."))); } /* * Create MetadataSyncContext which is used throughout nodes' activation. * As we set collectCommands to true, it would not create connections to workers. * Instead it would collect and return sync commands to be sent to workers. */ bool collectCommands = true; bool nodesAddedInSameTransaction = false; MetadataSyncContext *context = CreateMetadataSyncContext(list_make1(dummyWorkerNode), collectCommands, nodesAddedInSameTransaction); ActivateNodeList(context); List *activateNodeCommandList = context->collectedCommands; int activateNodeCommandIndex = 0; Oid ddlCommandTypeId = TEXTOID; int activateNodeCommandCount = list_length(activateNodeCommandList); Datum *activateNodeCommandDatumArray = palloc0(activateNodeCommandCount * sizeof(Datum)); const char *activateNodeSnapshotCommand = NULL; foreach_declared_ptr(activateNodeSnapshotCommand, activateNodeCommandList) { Datum activateNodeSnapshotCommandDatum = CStringGetTextDatum( activateNodeSnapshotCommand); activateNodeCommandDatumArray[activateNodeCommandIndex] = activateNodeSnapshotCommandDatum; activateNodeCommandIndex++; } ArrayType *activateNodeCommandArrayType = DatumArrayToArrayType( activateNodeCommandDatumArray, activateNodeCommandCount, ddlCommandTypeId); PG_RETURN_ARRAYTYPE_P(activateNodeCommandArrayType); } /* * IsMetadataSynced checks the workers to see if all workers with metadata are * synced. */ static bool IsMetadataSynced(void) { List *workerList = ActivePrimaryNonCoordinatorNodeList(NoLock); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerList) { if (workerNode->hasMetadata && !workerNode->metadataSynced) { return false; } } return true; } /* * wait_until_metadata_sync waits until the maintenance daemon does a metadata * sync, or times out. */ Datum wait_until_metadata_sync(PG_FUNCTION_ARGS) { uint32 timeout = PG_GETARG_UINT32(0); /* First we start listening. */ MultiConnection *connection = GetNodeConnection(FORCE_NEW_CONNECTION, LocalHostName, PostPortNumber); ExecuteCriticalRemoteCommand(connection, "LISTEN " METADATA_SYNC_CHANNEL); /* * If all the metadata nodes have already been synced, we should not wait. * That's primarily because the maintenance deamon might have already sent * the notification and we'd wait unnecessarily here. Worse, the test outputs * might be inconsistent across executions due to the warning. */ if (IsMetadataSynced()) { CloseConnection(connection); PG_RETURN_VOID(); } int waitFlags = WL_SOCKET_READABLE | WL_TIMEOUT | WL_POSTMASTER_DEATH; int waitResult = WaitLatchOrSocket(NULL, waitFlags, PQsocket(connection->pgConn), timeout, 0); if (waitResult & WL_POSTMASTER_DEATH) { ereport(ERROR, (errmsg("postmaster was shut down, exiting"))); } else if (waitResult & WL_SOCKET_MASK) { ClearResults(connection, true); } else if (waitResult & WL_TIMEOUT && !IsMetadataSynced()) { elog(WARNING, "waiting for metadata sync timed out"); } CloseConnection(connection); PG_RETURN_VOID(); } /* * trigger_metadata_sync triggers metadata sync for testing. */ Datum trigger_metadata_sync(PG_FUNCTION_ARGS) { TriggerNodeMetadataSyncOnCommit(); PG_RETURN_VOID(); } /* * raise_error_in_metadata_sync causes metadata sync to raise an error. */ Datum raise_error_in_metadata_sync(PG_FUNCTION_ARGS) { /* metadata sync uses SIGALRM to test errors */ SignalMetadataSyncDaemon(MyDatabaseId, SIGALRM); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/partitioning_utils.c ================================================ /*------------------------------------------------------------------------- * * test/src/partitioning_utils.c * * This file contains functions to test partitioning utility functions * implemented in Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "catalog/pg_type.h" #include "lib/stringinfo.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "distributed/listutils.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/reference_table_utils.h" PG_FUNCTION_INFO_V1(generate_alter_table_detach_partition_command); PG_FUNCTION_INFO_V1(generate_alter_table_attach_partition_command); PG_FUNCTION_INFO_V1(generate_partition_information); PG_FUNCTION_INFO_V1(print_partitions); PG_FUNCTION_INFO_V1(table_inherits); PG_FUNCTION_INFO_V1(table_inherited); /* * Just a wrapper around GenereateDetachPartitionCommand(). */ Datum generate_alter_table_detach_partition_command(PG_FUNCTION_ARGS) { char *command = ""; command = GenerateDetachPartitionCommand(PG_GETARG_OID(0)); PG_RETURN_TEXT_P(cstring_to_text(command)); } /* * Just a wrapper around GenerateAlterTableAttachPartitionCommand(). */ Datum generate_alter_table_attach_partition_command(PG_FUNCTION_ARGS) { char *command = ""; command = GenerateAlterTableAttachPartitionCommand(PG_GETARG_OID(0)); PG_RETURN_TEXT_P(cstring_to_text(command)); } /* * Just a wrapper around GenereatePartitioningInformation(). */ Datum generate_partition_information(PG_FUNCTION_ARGS) { char *command = ""; command = GeneratePartitioningInformation(PG_GETARG_OID(0)); PG_RETURN_TEXT_P(cstring_to_text(command)); } /* * Just a wrapper around PartitionList() with human readable table name outpus. */ Datum print_partitions(PG_FUNCTION_ARGS) { StringInfo resultRelationNames = makeStringInfo(); List *partitionList = PartitionList(PG_GETARG_OID(0)); partitionList = SortList(partitionList, CompareOids); Oid partitionOid = InvalidOid; foreach_declared_oid(partitionOid, partitionList) { /* at least one table is already added, add comma */ if (resultRelationNames->len > 0) { appendStringInfoString(resultRelationNames, ","); } appendStringInfoString(resultRelationNames, get_rel_name(partitionOid)); } PG_RETURN_TEXT_P(cstring_to_text(resultRelationNames->data)); } /* * Just a wrapper around IsChildTable() */ Datum table_inherits(PG_FUNCTION_ARGS) { return IsChildTable(PG_GETARG_OID(0)); } /* * Just a wrapper around IsParentTable() */ Datum table_inherited(PG_FUNCTION_ARGS) { return IsParentTable(PG_GETARG_OID(0)); } ================================================ FILE: src/backend/distributed/test/progress_utils.c ================================================ /*------------------------------------------------------------------------- * * progress_utils.c * * This file contains functions to exercise progress monitoring functionality * within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" #include "nodes/execnodes.h" #include "utils/tuplestore.h" #include "distributed/listutils.h" #include "distributed/multi_progress.h" #include "distributed/tuplestore.h" PG_FUNCTION_INFO_V1(create_progress); PG_FUNCTION_INFO_V1(update_progress); PG_FUNCTION_INFO_V1(finish_progress); PG_FUNCTION_INFO_V1(show_progress); Datum create_progress(PG_FUNCTION_ARGS) { uint64 magicNumber = PG_GETARG_INT64(0); int stepCount = PG_GETARG_INT32(1); dsm_handle dsmHandle; ProgressMonitorData *monitor = CreateProgressMonitor(stepCount, sizeof(uint64), &dsmHandle); if (monitor != NULL) { uint64 *steps = (uint64 *) ProgressMonitorSteps(monitor); int i = 0; for (; i < stepCount; i++) { steps[i] = 0; } } RegisterProgressMonitor(magicNumber, 0, dsmHandle); PG_RETURN_VOID(); } Datum update_progress(PG_FUNCTION_ARGS) { uint64 step = PG_GETARG_INT64(0); uint64 newValue = PG_GETARG_INT64(1); ProgressMonitorData *monitor = GetCurrentProgressMonitor(); if (monitor != NULL && step < monitor->stepCount) { uint64 *steps = (uint64 *) ProgressMonitorSteps(monitor); steps[step] = newValue; } PG_RETURN_VOID(); } Datum finish_progress(PG_FUNCTION_ARGS) { FinalizeCurrentProgressMonitor(); PG_RETURN_VOID(); } Datum show_progress(PG_FUNCTION_ARGS) { uint64 magicNumber = PG_GETARG_INT64(0); List *attachedDSMSegments = NIL; List *monitorList = ProgressMonitorList(magicNumber, &attachedDSMSegments); TupleDesc tupdesc; Tuplestorestate *tupstore = SetupTuplestore(fcinfo, &tupdesc); ProgressMonitorData *monitor = NULL; foreach_declared_ptr(monitor, monitorList) { uint64 *steps = ProgressMonitorSteps(monitor); for (int stepIndex = 0; stepIndex < monitor->stepCount; stepIndex++) { uint64 step = steps[stepIndex]; Datum values[2]; bool nulls[2]; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = Int32GetDatum(stepIndex); values[1] = UInt64GetDatum(step); tuplestore_putvalues(tupstore, tupdesc, values, nulls); } } DetachFromDSMSegments(attachedDSMSegments); return (Datum) 0; } ================================================ FILE: src/backend/distributed/test/prune_shard_list.c ================================================ /*------------------------------------------------------------------------- * * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality * within Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "c.h" #include "fmgr.h" #include "access/stratnum.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "utils/array.h" #include "utils/palloc.h" #include "pg_version_constants.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_join_order.h" #include "distributed/multi_physical_planner.h" #include "distributed/resource_lock.h" #include "distributed/shard_pruning.h" #include "distributed/utils/array_type.h" /* local function forward declarations */ static Expr * MakeTextPartitionExpression(Oid distributedTableId, text *value); static ArrayType * PrunedShardIdsForTable(Oid distributedTableId, List *whereClauseList); static ArrayType * SortedShardIntervalArray(Oid distributedTableId); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(prune_using_no_values); PG_FUNCTION_INFO_V1(prune_using_single_value); PG_FUNCTION_INFO_V1(prune_using_either_value); PG_FUNCTION_INFO_V1(prune_using_both_values); PG_FUNCTION_INFO_V1(debug_equality_expression); PG_FUNCTION_INFO_V1(print_sorted_shard_intervals); /* * prune_using_no_values returns the shards for the specified distributed table * after pruning using an empty clause list. */ Datum prune_using_no_values(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); List *whereClauseList = NIL; ArrayType *shardIdArrayType = PrunedShardIdsForTable(distributedTableId, whereClauseList); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * prune_using_single_value returns the shards for the specified distributed * table after pruning using a single value provided by the caller. */ Datum prune_using_single_value(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); text *value = (PG_ARGISNULL(1)) ? NULL : PG_GETARG_TEXT_P(1); Expr *equalityExpr = MakeTextPartitionExpression(distributedTableId, value); List *whereClauseList = list_make1(equalityExpr); ArrayType *shardIdArrayType = PrunedShardIdsForTable(distributedTableId, whereClauseList); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * prune_using_either_value returns the shards for the specified distributed * table after pruning using either of two values provided by the caller (OR). */ Datum prune_using_either_value(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); text *firstValue = PG_GETARG_TEXT_P(1); text *secondValue = PG_GETARG_TEXT_P(2); Expr *firstQual = MakeTextPartitionExpression(distributedTableId, firstValue); Expr *secondQual = MakeTextPartitionExpression(distributedTableId, secondValue); Expr *orClause = make_orclause(list_make2(firstQual, secondQual)); List *whereClauseList = list_make1(orClause); ArrayType *shardIdArrayType = PrunedShardIdsForTable(distributedTableId, whereClauseList); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * prune_using_both_values returns the shards for the specified distributed * table after pruning using both of the values provided by the caller (AND). */ Datum prune_using_both_values(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); text *firstValue = PG_GETARG_TEXT_P(1); text *secondValue = PG_GETARG_TEXT_P(2); Expr *firstQual = MakeTextPartitionExpression(distributedTableId, firstValue); Expr *secondQual = MakeTextPartitionExpression(distributedTableId, secondValue); List *whereClauseList = list_make2(firstQual, secondQual); ArrayType *shardIdArrayType = PrunedShardIdsForTable(distributedTableId, whereClauseList); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * debug_equality_expression returns the textual representation of an equality * expression generated by a call to MakeOpExpression. */ Datum debug_equality_expression(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); uint32 rangeTableId = 1; if (!IsCitusTableType(distributedTableId, HASH_DISTRIBUTED)) { ereport(ERROR, (errmsg("table needs to be hash distributed"))); } Var *partitionColumn = PartitionColumn(distributedTableId, rangeTableId); OpExpr *equalityExpression = MakeOpExpression(partitionColumn, BTEqualStrategyNumber); PG_RETURN_CSTRING(nodeToString(equalityExpression)); } /* * print_sorted_shard_intervals prints the sorted shard interval array that is in the * metadata cache. This function aims to test sorting functionality. */ Datum print_sorted_shard_intervals(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); ArrayType *shardIdArrayType = SortedShardIntervalArray(distributedTableId); PG_RETURN_ARRAYTYPE_P(shardIdArrayType); } /* * MakeTextPartitionExpression returns an equality expression between the * specified table's partition column and the provided values. */ static Expr * MakeTextPartitionExpression(Oid distributedTableId, text *value) { uint32 rangeTableId = 1; Var *partitionColumn = PartitionColumn(distributedTableId, rangeTableId); Expr *partitionExpression = NULL; if (value != NULL) { OpExpr *equalityExpr = MakeOpExpression(partitionColumn, BTEqualStrategyNumber); Node *rightOp = get_rightop((Expr *) equalityExpr); Assert(rightOp != NULL); Assert(IsA(rightOp, Const)); Const *rightConst = (Const *) rightOp; rightConst->constvalue = (Datum) value; rightConst->constisnull = false; rightConst->constbyval = false; partitionExpression = (Expr *) equalityExpr; } else { NullTest *nullTest = makeNode(NullTest); nullTest->arg = (Expr *) partitionColumn; nullTest->nulltesttype = IS_NULL; partitionExpression = (Expr *) nullTest; } return partitionExpression; } /* * PrunedShardIdsForTable loads the shard intervals for the specified table, * prunes them using the provided clauses. It returns an ArrayType containing * the shard identifiers, suitable for return from an SQL-facing function. */ static ArrayType * PrunedShardIdsForTable(Oid distributedTableId, List *whereClauseList) { int shardIdIndex = 0; Oid shardIdTypeId = INT8OID; Index tableId = 1; List *shardList = PruneShards(distributedTableId, tableId, whereClauseList, NULL); int shardIdCount = list_length(shardList); Datum *shardIdDatumArray = palloc0(shardIdCount * sizeof(Datum)); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardList) { Datum shardIdDatum = Int64GetDatum(shardInterval->shardId); shardIdDatumArray[shardIdIndex] = shardIdDatum; shardIdIndex++; } ArrayType *shardIdArrayType = DatumArrayToArrayType(shardIdDatumArray, shardIdCount, shardIdTypeId); return shardIdArrayType; } /* * SortedShardIntervalArray simply returns the shard interval ids in the sorted shard * interval cache as a datum array. */ static ArrayType * SortedShardIntervalArray(Oid distributedTableId) { Oid shardIdTypeId = INT8OID; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); ShardInterval **shardIntervalArray = cacheEntry->sortedShardIntervalArray; int shardIdCount = cacheEntry->shardIntervalArrayLength; Datum *shardIdDatumArray = palloc0(shardIdCount * sizeof(Datum)); for (int shardIndex = 0; shardIndex < shardIdCount; ++shardIndex) { ShardInterval *shardId = shardIntervalArray[shardIndex]; Datum shardIdDatum = Int64GetDatum(shardId->shardId); shardIdDatumArray[shardIndex] = shardIdDatum; } ArrayType *shardIdArrayType = DatumArrayToArrayType(shardIdDatumArray, shardIdCount, shardIdTypeId); return shardIdArrayType; } ================================================ FILE: src/backend/distributed/test/relation_access_tracking.c ================================================ /*------------------------------------------------------------------------- * * test/src/relation_acess_tracking.c * * Some test UDF for tracking relation accesses within transaction blocks. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "fmgr.h" #include "distributed/relation_access_tracking.h" /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(relation_select_access_mode); PG_FUNCTION_INFO_V1(relation_dml_access_mode); PG_FUNCTION_INFO_V1(relation_ddl_access_mode); /* * relation_select_access_mode returns the SELECT access * type (e.g., single shard - multi shard) for the given relation. */ Datum relation_select_access_mode(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); PG_RETURN_INT64(GetRelationSelectAccessMode(relationId)); } /* * relation_dml_access_mode returns the DML access type (e.g., * single shard - multi shard) for the given relation. */ Datum relation_dml_access_mode(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); PG_RETURN_INT64(GetRelationDMLAccessMode(relationId)); } /* * relation_ddl_access_mode returns the DDL access type (e.g., * single shard - multi shard) for the given relation. */ Datum relation_ddl_access_mode(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); PG_RETURN_INT64(GetRelationDDLAccessMode(relationId)); } ================================================ FILE: src/backend/distributed/test/run_from_same_connection.c ================================================ /*------------------------------------------------------------------------- * * test/src/run_from_same_connection.c * * This file contains UDF to run consecutive commands on worker node from the * same connection. UDFs will be used to test MX functionalities in isolation * tests. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/xact.h" #include "executor/spi.h" #include "lib/stringinfo.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_utils.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/run_from_same_connection.h" #include "distributed/version_compat.h" #define ALTER_CURRENT_PROCESS_ID \ "ALTER SYSTEM SET citus.isolation_test_session_process_id TO %d" #define ALTER_CURRENT_WORKER_PROCESS_ID \ "ALTER SYSTEM SET citus.isolation_test_session_remote_process_id TO %ld" #define GET_PROCESS_ID "SELECT process_id FROM get_current_transaction_id()" static bool allowNonIdleRemoteTransactionOnXactHandling = false; static MultiConnection *singleConnection = NULL; /* * Config variables which will be used by isolation framework to check transactions * initiated from worker nodes. */ int IsolationTestSessionRemoteProcessID = -1; int IsolationTestSessionProcessID = -1; static int64 GetRemoteProcessId(void); /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(start_session_level_connection_to_node); PG_FUNCTION_INFO_V1(run_commands_on_session_level_connection_to_node); PG_FUNCTION_INFO_V1(stop_session_level_connection_to_node); PG_FUNCTION_INFO_V1(override_backend_data_gpid); /* * AllowNonIdleTransactionOnXactHandling allows connection opened with * SESSION_LIFESPAN remain opened even if it is not idle. */ bool AllowNonIdleTransactionOnXactHandling(void) { return allowNonIdleRemoteTransactionOnXactHandling; } /* * start_session_level_connection_to_node helps us to open and keep connections * open while sending consecutive commands, even if they are outside the transaction. * To use the connection opened with an open transaction, we have implemented a hacky * solution by setting a static flag, allowNonIdleRemoteTransactionOnXactHandling, on * this file to true. That gives us to chance to keep that connection open. * * Note that, this UDF shouldn't be used outside the isolation tests. */ Datum start_session_level_connection_to_node(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *nodeName = PG_GETARG_TEXT_P(0); uint32 nodePort = PG_GETARG_UINT32(1); char *nodeNameString = text_to_cstring(nodeName); int connectionFlags = 0; if (singleConnection != NULL && (strcmp(singleConnection->hostname, nodeNameString) != 0 || singleConnection->port != nodePort)) { elog(ERROR, "can not connect different worker nodes from the same session using start_session_level_connection_to_node"); } /* * In order to keep connection open even with an open transaction, * allowSessionLifeSpanWithOpenTransaction is set to true. */ if (singleConnection == NULL) { singleConnection = GetNodeConnection(connectionFlags, nodeNameString, nodePort); allowNonIdleRemoteTransactionOnXactHandling = true; } if (PQstatus(singleConnection->pgConn) != CONNECTION_OK) { elog(ERROR, "failed to connect to %s:%d", nodeNameString, (int) nodePort); } /* pretend we are a regular client to avoid citus-initiated backend checks */ const char *setAppName = "SET application_name TO run_commands_on_session_level_connection_to_node"; ExecuteCriticalRemoteCommand(singleConnection, setAppName); /* * We are hackily overriding the remote processes' gpid value such that * blocked process detection continues to work. */ StringInfo overrideBackendDataCommandOriginator = makeStringInfo(); appendStringInfo(overrideBackendDataCommandOriginator, "SELECT override_backend_data_gpid(%lu);", GetGlobalPID()); ExecuteCriticalRemoteCommand(singleConnection, overrideBackendDataCommandOriginator->data); PG_RETURN_VOID(); } /* * run_commands_on_session_level_connection_to_node runs to consecutive commands * from the same connection opened by start_session_level_connection_to_node. * * Since transactions can be initiated from worker nodes with MX, we need to * keep them open on the worker node to check whether there exist a waiting * transaction in test steps. In order to release the locks taken in the * transaction we need to send related unlock commands from the same connection * as well. */ Datum run_commands_on_session_level_connection_to_node(PG_FUNCTION_ARGS) { text *queryText = PG_GETARG_TEXT_P(0); char *queryString = text_to_cstring(queryText); StringInfo processStringInfo = makeStringInfo(); StringInfo workerProcessStringInfo = makeStringInfo(); MultiConnection *localConnection = GetNodeConnection(0, LocalHostName, PostPortNumber); if (!singleConnection) { elog(ERROR, "start_session_level_connection_to_node must be called first to open a session level connection"); } appendStringInfo(processStringInfo, ALTER_CURRENT_PROCESS_ID, MyProcPid); appendStringInfo(workerProcessStringInfo, ALTER_CURRENT_WORKER_PROCESS_ID, GetRemoteProcessId()); ExecuteCriticalRemoteCommand(singleConnection, queryString); /* * Since we cannot run `ALTER SYSTEM` command within a transaction, we are * calling it from a self-connected session. */ ExecuteCriticalRemoteCommand(localConnection, processStringInfo->data); ExecuteCriticalRemoteCommand(localConnection, workerProcessStringInfo->data); CloseConnection(localConnection); /* Call pg_reload_conf UDF to update changed GUCs above on each backend */ Oid pgReloadConfOid = FunctionOid("pg_catalog", "pg_reload_conf", 0); OidFunctionCall0(pgReloadConfOid); PG_RETURN_VOID(); } /* * override_backend_data_gpid is a wrapper around SetBackendDataGpid(). * Also sets distributedCommandOriginator to true since the only caller of * this method calls this function actually wants this backend to * be treated as a distributed command originator with the given global pid. */ Datum override_backend_data_gpid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 gpid = PG_GETARG_INT64(0); SetBackendDataGlobalPID(gpid); SetBackendDataDistributedCommandOriginator(true); PG_RETURN_VOID(); } /* * stop_session_level_connection_to_node closes the connection opened by the * start_session_level_connection_to_node and set the flag to false which * allows connection API to keep connections with open transaction. */ Datum stop_session_level_connection_to_node(PG_FUNCTION_ARGS) { allowNonIdleRemoteTransactionOnXactHandling = false; if (singleConnection != NULL) { CloseConnection(singleConnection); singleConnection = NULL; } PG_RETURN_VOID(); } /* * GetRemoteProcessId() get the process id of remote transaction opened * by the connection. */ static int64 GetRemoteProcessId() { StringInfo queryStringInfo = makeStringInfo(); PGresult *result = NULL; appendStringInfo(queryStringInfo, GET_PROCESS_ID); int queryResult = ExecuteOptionalRemoteCommand(singleConnection, queryStringInfo->data, &result); if (queryResult != RESPONSE_OKAY) { PG_RETURN_VOID(); } int64 rowCount = PQntuples(result); if (rowCount != 1) { PG_RETURN_VOID(); } int64 resultValue = ParseIntField(result, 0, 0); PQclear(result); ClearResults(singleConnection, false); return resultValue; } ================================================ FILE: src/backend/distributed/test/sequential_execution.c ================================================ /*------------------------------------------------------------------------- * * test/src/sequential_execution.c * * This file contains functions to test setting citus.multi_shard_modify_mode * GUC. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "distributed/multi_executor.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(set_local_multi_shard_modify_mode_to_sequential); /* * set_local_multi_shard_modify_mode_to_sequential is a SQL * interface for testing SetLocalMultiShardModifyModeToSequential(). */ Datum set_local_multi_shard_modify_mode_to_sequential(PG_FUNCTION_ARGS) { SetLocalMultiShardModifyModeToSequential(); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/shard_rebalancer.c ================================================ /*------------------------------------------------------------------------- * * test/shard_rebalancer.c * * This file contains functions used for unit testing the planning part of the * shard rebalancer. * * Copyright (c) 2014-2019, Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "safe_lib.h" #include "catalog/pg_type.h" #include "utils/builtins.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/metadata_utility.h" #include "distributed/multi_physical_planner.h" #include "distributed/relay_utility.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" /* static declarations for json conversion */ static List * JsonArrayToShardPlacementTestInfoList(ArrayType * shardPlacementJsonArrayObject); static List * JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject); static bool JsonFieldValueBoolDefault(Datum jsonDocument, const char *key, bool defaultValue); static uint32 JsonFieldValueUInt32Default(Datum jsonDocument, const char *key, uint32 defaultValue); static uint64 JsonFieldValueUInt64Default(Datum jsonDocument, const char *key, uint64 defaultValue); static char * JsonFieldValueString(Datum jsonDocument, const char *key); static ArrayType * PlacementUpdateListToJsonArray(List *placementUpdateList); static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *context); static float NodeCapacity(WorkerNode *workerNode, void *context); static ShardCost GetShardCost(uint64 shardId, void *context); PG_FUNCTION_INFO_V1(shard_placement_rebalance_array); PG_FUNCTION_INFO_V1(shard_placement_replication_array); PG_FUNCTION_INFO_V1(worker_node_responsive); PG_FUNCTION_INFO_V1(run_try_drop_marked_resources); typedef struct ShardPlacementTestInfo { ShardPlacement *placement; uint64 cost; bool nextColocationGroup; } ShardPlacementTestInfo; typedef struct WorkerTestInfo { WorkerNode *node; List *disallowedShardIds; float capacity; } WorkerTestInfo; typedef struct RebalancePlanContext { List *workerTestInfoList; List *shardPlacementTestInfoList; } RebalancePlacementContext; /* * run_try_drop_marked_resources is a wrapper to run TryDropOrphanedResources. */ Datum run_try_drop_marked_resources(PG_FUNCTION_ARGS) { TryDropOrphanedResources(); PG_RETURN_VOID(); } /* * shard_placement_rebalance_array returns a list of operations which can make a * cluster consisting of given shard placements and worker nodes balanced with * respect to the given threshold. Threshold is a value between 0 and 1 which * determines the evenness in shard distribution. When threshold is 0, then all * nodes should have equal number of shards. As threshold increases, cluster's * evenness requirements decrease, and we can rebalance the cluster using less * operations. */ Datum shard_placement_rebalance_array(PG_FUNCTION_ARGS) { ArrayType *workerNodeJsonArray = PG_GETARG_ARRAYTYPE_P(0); ArrayType *shardPlacementJsonArray = PG_GETARG_ARRAYTYPE_P(1); float threshold = PG_GETARG_FLOAT4(2); int32 maxShardMoves = PG_GETARG_INT32(3); bool drainOnly = PG_GETARG_BOOL(4); float utilizationImproventThreshold = PG_GETARG_FLOAT4(5); List *workerNodeList = NIL; List *shardPlacementListList = NIL; List *shardPlacementList = NIL; WorkerTestInfo *workerTestInfo = NULL; ShardPlacementTestInfo *shardPlacementTestInfo = NULL; RebalancePlanFunctions rebalancePlanFunctions = { .shardAllowedOnNode = ShardAllowedOnNode, .nodeCapacity = NodeCapacity, .shardCost = GetShardCost, }; RebalancePlacementContext context = { .workerTestInfoList = NULL, }; context.workerTestInfoList = JsonArrayToWorkerTestInfoList(workerNodeJsonArray); context.shardPlacementTestInfoList = JsonArrayToShardPlacementTestInfoList( shardPlacementJsonArray); /* we don't need original arrays any more, so we free them to save memory */ pfree(workerNodeJsonArray); pfree(shardPlacementJsonArray); /* map workerTestInfoList to a list of its WorkerNodes */ foreach_declared_ptr(workerTestInfo, context.workerTestInfoList) { workerNodeList = lappend(workerNodeList, workerTestInfo->node); } /* map shardPlacementTestInfoList to a list of list of its ShardPlacements */ foreach_declared_ptr(shardPlacementTestInfo, context.shardPlacementTestInfoList) { if (shardPlacementTestInfo->nextColocationGroup) { shardPlacementList = SortList(shardPlacementList, CompareShardPlacements); shardPlacementListList = lappend(shardPlacementListList, shardPlacementList); shardPlacementList = NIL; } shardPlacementList = lappend(shardPlacementList, shardPlacementTestInfo->placement); } shardPlacementList = SortList(shardPlacementList, CompareShardPlacements); shardPlacementListList = lappend(shardPlacementListList, shardPlacementList); List *unbalancedShards = NIL; ListCell *shardPlacementListCell = NULL; foreach(shardPlacementListCell, shardPlacementListList) { List *placementList = (List *) lfirst(shardPlacementListCell); if (list_length(placementList) < list_length(workerNodeList)) { unbalancedShards = list_concat(unbalancedShards, placementList); shardPlacementListList = foreach_delete_current(shardPlacementListList, shardPlacementListCell); } } if (list_length(unbalancedShards) > 0) { shardPlacementListList = lappend(shardPlacementListList, unbalancedShards); } rebalancePlanFunctions.context = &context; /* sort the lists to make the function more deterministic */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); List *placementUpdateList = RebalancePlacementUpdates(workerNodeList, shardPlacementListList, threshold, maxShardMoves, drainOnly, utilizationImproventThreshold, &rebalancePlanFunctions); ArrayType *placementUpdateJsonArray = PlacementUpdateListToJsonArray( placementUpdateList); PG_RETURN_ARRAYTYPE_P(placementUpdateJsonArray); } /* * ShardAllowedOnNode is the function that checks if shard is allowed to be on * a worker when running the shard rebalancer unit tests. */ static bool ShardAllowedOnNode(uint64 shardId, WorkerNode *workerNode, void *voidContext) { RebalancePlacementContext *context = voidContext; WorkerTestInfo *workerTestInfo = NULL; uint64 *disallowedShardIdPtr = NULL; foreach_declared_ptr(workerTestInfo, context->workerTestInfoList) { if (workerTestInfo->node == workerNode) { break; } } Assert(workerTestInfo != NULL); foreach_declared_ptr(disallowedShardIdPtr, workerTestInfo->disallowedShardIds) { if (shardId == *disallowedShardIdPtr) { return false; } } return true; } /* * NodeCapacity is the function that gets the capacity of a worker when running * the shard rebalancer unit tests. */ static float NodeCapacity(WorkerNode *workerNode, void *voidContext) { RebalancePlacementContext *context = voidContext; WorkerTestInfo *workerTestInfo = NULL; foreach_declared_ptr(workerTestInfo, context->workerTestInfoList) { if (workerTestInfo->node == workerNode) { break; } } Assert(workerTestInfo != NULL); return workerTestInfo->capacity; } /* * GetShardCost is the function that gets the ShardCost of a shard when running * the shard rebalancer unit tests. */ static ShardCost GetShardCost(uint64 shardId, void *voidContext) { RebalancePlacementContext *context = voidContext; ShardCost shardCost; memset_struct_0(shardCost); shardCost.shardId = shardId; ShardPlacementTestInfo *shardPlacementTestInfo = NULL; foreach_declared_ptr(shardPlacementTestInfo, context->shardPlacementTestInfoList) { if (shardPlacementTestInfo->placement->shardId == shardId) { break; } } Assert(shardPlacementTestInfo != NULL); shardCost.cost = shardPlacementTestInfo->cost; return shardCost; } /* * shard_placement_replication_array returns a list of operations which will * replicate under-replicated shards in a cluster consisting of given shard * placements and worker nodes. A shard is under-replicated if it has less * active placements than the given shard replication factor. */ Datum shard_placement_replication_array(PG_FUNCTION_ARGS) { ArrayType *workerNodeJsonArray = PG_GETARG_ARRAYTYPE_P(0); ArrayType *shardPlacementJsonArray = PG_GETARG_ARRAYTYPE_P(1); uint32 shardReplicationFactor = PG_GETARG_INT32(2); List *workerNodeList = NIL; List *shardPlacementList = NIL; WorkerTestInfo *workerTestInfo = NULL; ShardPlacementTestInfo *shardPlacementTestInfo = NULL; /* validate shard replication factor */ if (shardReplicationFactor < SHARD_REPLICATION_FACTOR_MINIMUM || shardReplicationFactor > SHARD_REPLICATION_FACTOR_MAXIMUM) { ereport(ERROR, (errmsg("invalid shard replication factor"), errhint("Shard replication factor must be an integer " "between %d and %d", SHARD_REPLICATION_FACTOR_MINIMUM, SHARD_REPLICATION_FACTOR_MAXIMUM))); } List *workerTestInfoList = JsonArrayToWorkerTestInfoList(workerNodeJsonArray); List *shardPlacementTestInfoList = JsonArrayToShardPlacementTestInfoList( shardPlacementJsonArray); /* we don't need original arrays any more, so we free them to save memory */ pfree(workerNodeJsonArray); pfree(shardPlacementJsonArray); foreach_declared_ptr(workerTestInfo, workerTestInfoList) { workerNodeList = lappend(workerNodeList, workerTestInfo->node); } foreach_declared_ptr(shardPlacementTestInfo, shardPlacementTestInfoList) { shardPlacementList = lappend(shardPlacementList, shardPlacementTestInfo->placement); } List *activeShardPlacementList = shardPlacementList; /* sort the lists to make the function more deterministic */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); activeShardPlacementList = SortList(activeShardPlacementList, CompareShardPlacements); List *placementUpdateList = ReplicationPlacementUpdates(workerNodeList, activeShardPlacementList, shardReplicationFactor); ArrayType *placementUpdateJsonArray = PlacementUpdateListToJsonArray( placementUpdateList); PG_RETURN_ARRAYTYPE_P(placementUpdateJsonArray); } /* * JsonArrayToShardPlacementTestInfoList converts the given shard placement json array * to a list of ShardPlacement structs. */ static List * JsonArrayToShardPlacementTestInfoList(ArrayType *shardPlacementJsonArrayObject) { List *shardPlacementTestInfoList = NIL; Datum *shardPlacementJsonArray = NULL; int placementCount = 0; /* * Memory is not automatically freed when we call UDFs using DirectFunctionCall. * We call these functions in functionCallContext, so we can free the memory * once they return. */ MemoryContext functionCallContext = AllocSetContextCreate(CurrentMemoryContext, "Function Call Context", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); deconstruct_array(shardPlacementJsonArrayObject, JSONOID, -1, false, 'i', &shardPlacementJsonArray, NULL, &placementCount); for (int placementIndex = 0; placementIndex < placementCount; placementIndex++) { Datum placementJson = shardPlacementJsonArray[placementIndex]; ShardPlacementTestInfo *placementTestInfo = palloc0( sizeof(ShardPlacementTestInfo)); MemoryContext oldContext = MemoryContextSwitchTo(functionCallContext); uint64 shardId = JsonFieldValueUInt64Default( placementJson, FIELD_NAME_SHARD_ID, placementIndex + 1); uint64 shardLength = JsonFieldValueUInt64Default( placementJson, FIELD_NAME_SHARD_LENGTH, 1); char *nodeName = JsonFieldValueString(placementJson, FIELD_NAME_NODE_NAME); if (nodeName == NULL) { ereport(ERROR, (errmsg(FIELD_NAME_NODE_NAME " needs be set"))); } int nodePort = JsonFieldValueUInt32Default( placementJson, FIELD_NAME_NODE_PORT, 5432); uint64 placementId = JsonFieldValueUInt64Default( placementJson, FIELD_NAME_PLACEMENT_ID, placementIndex + 1); uint64 cost = JsonFieldValueUInt64Default(placementJson, "cost", 1); bool nextColocationGroup = JsonFieldValueBoolDefault(placementJson, "next_colocation", false); MemoryContextSwitchTo(oldContext); placementTestInfo->placement = palloc0(sizeof(ShardPlacement)); placementTestInfo->placement->shardId = shardId; placementTestInfo->placement->shardLength = shardLength; placementTestInfo->placement->nodeName = pstrdup(nodeName); placementTestInfo->placement->nodePort = nodePort; placementTestInfo->placement->placementId = placementId; placementTestInfo->cost = cost; placementTestInfo->nextColocationGroup = nextColocationGroup; /* * We have copied whatever we needed from the UDF calls, so we can free * the memory allocated by them. */ MemoryContextReset(functionCallContext); shardPlacementTestInfoList = lappend(shardPlacementTestInfoList, placementTestInfo); } pfree(shardPlacementJsonArray); return shardPlacementTestInfoList; } /* * JsonArrayToWorkerNodeList converts the given worker node json array to a list * of WorkerNode structs. */ static List * JsonArrayToWorkerTestInfoList(ArrayType *workerNodeJsonArrayObject) { List *workerTestInfoList = NIL; Datum *workerNodeJsonArray = NULL; int workerNodeCount = 0; deconstruct_array(workerNodeJsonArrayObject, JSONOID, -1, false, 'i', &workerNodeJsonArray, NULL, &workerNodeCount); for (int workerNodeIndex = 0; workerNodeIndex < workerNodeCount; workerNodeIndex++) { Datum workerNodeJson = workerNodeJsonArray[workerNodeIndex]; char *workerName = JsonFieldValueString(workerNodeJson, FIELD_NAME_WORKER_NAME); if (workerName == NULL) { ereport(ERROR, (errmsg(FIELD_NAME_WORKER_NAME " needs be set"))); } uint32 workerPort = JsonFieldValueUInt32Default(workerNodeJson, FIELD_NAME_WORKER_PORT, 5432); List *disallowedShardIdList = NIL; WorkerTestInfo *workerTestInfo = palloc0(sizeof(WorkerTestInfo)); WorkerNode *workerNode = palloc0(sizeof(WorkerNode)); strncpy_s(workerNode->workerName, sizeof(workerNode->workerName), workerName, WORKER_LENGTH); workerNode->nodeId = workerNodeIndex; workerNode->workerPort = workerPort; workerNode->shouldHaveShards = true; workerNode->nodeRole = PrimaryNodeRoleId(); workerTestInfo->node = workerNode; workerTestInfo->capacity = JsonFieldValueUInt64Default(workerNodeJson, "capacity", 1); workerNode->isActive = JsonFieldValueBoolDefault(workerNodeJson, "isActive", true); workerTestInfoList = lappend(workerTestInfoList, workerTestInfo); char *disallowedShardsString = JsonFieldValueString( workerNodeJson, "disallowed_shards"); if (disallowedShardsString == NULL) { continue; } char *strtokPosition = NULL; char *shardString = strtok_r(disallowedShardsString, ",", &strtokPosition); while (shardString != NULL) { uint64 *shardInt = palloc0(sizeof(uint64)); *shardInt = SafeStringToUint64(shardString); disallowedShardIdList = lappend(disallowedShardIdList, shardInt); shardString = strtok_r(NULL, ",", &strtokPosition); } workerTestInfo->disallowedShardIds = disallowedShardIdList; } return workerTestInfoList; } /* * JsonFieldValueBoolDefault gets the value of the given key in the given json * document and returns it as a boolean. If the field does not exist in the * JSON it returns defaultValue. */ static bool JsonFieldValueBoolDefault(Datum jsonDocument, const char *key, bool defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); if (valueString == NULL) { return defaultValue; } Datum valueBoolDatum = DirectFunctionCall1(boolin, CStringGetDatum(valueString)); return DatumGetBool(valueBoolDatum); } /* * JsonFieldValueUInt32Default gets the value of the given key in the given json * document and returns it as an unsigned 32-bit integer. If the field does not * exist in the JSON it returns defaultValue. */ static uint32 JsonFieldValueUInt32Default(Datum jsonDocument, const char *key, uint32 defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); if (valueString == NULL) { return defaultValue; } Datum valueInt4Datum = DirectFunctionCall1(int4in, CStringGetDatum(valueString)); uint32 valueUInt32 = DatumGetInt32(valueInt4Datum); return valueUInt32; } /* * JsonFieldValueUInt64 gets the value of the given key in the given json * document and returns it as an unsigned 64-bit integer. If the field does not * exist in the JSON it returns defaultValue. */ static uint64 JsonFieldValueUInt64Default(Datum jsonDocument, const char *key, uint64 defaultValue) { char *valueString = JsonFieldValueString(jsonDocument, key); if (valueString == NULL) { return defaultValue; } Datum valueInt8Datum = DirectFunctionCall1(int8in, CStringGetDatum(valueString)); uint64 valueUInt64 = DatumGetInt64(valueInt8Datum); return valueUInt64; } /* * DirectFunctionalCall2Null is a version of DirectFunctionCall2 that can * return NULL. It still does not support NULL arguments though. */ static Datum DirectFunctionCall2Null(PGFunction func, bool *isnull, Datum arg1, Datum arg2) { LOCAL_FCINFO(fcinfo, 2); InitFunctionCallInfoData(*fcinfo, NULL, 2, InvalidOid, NULL, NULL); fcinfo->args[0].value = arg1; fcinfo->args[0].isnull = false; fcinfo->args[1].value = arg2; fcinfo->args[1].isnull = false; Datum result = (*func)(fcinfo); if (fcinfo->isnull) { *isnull = true; return 0; } *isnull = false; return result; } /* * JsonFieldValueString gets the value of the given key in the given json * document and returns it as a string. If the field does not exist in the JSON * it returns NULL. */ static char * JsonFieldValueString(Datum jsonDocument, const char *key) { bool isnull = false; Datum keyDatum = PointerGetDatum(cstring_to_text(key)); Datum valueTextDatum = DirectFunctionCall2Null( json_object_field_text, &isnull, jsonDocument, keyDatum); if (isnull) { return NULL; } char *valueString = TextDatumGetCString(valueTextDatum); return valueString; } /* * PlacementUpdateListToJsonArray converts the given list of placement update * data to a json array. */ static ArrayType * PlacementUpdateListToJsonArray(List *placementUpdateList) { ListCell *placementUpdateCell = NULL; int placementUpdateIndex = 0; int placementUpdateCount = list_length(placementUpdateList); Datum *placementUpdateJsonArray = palloc0(placementUpdateCount * sizeof(Datum)); foreach(placementUpdateCell, placementUpdateList) { PlacementUpdateEvent *placementUpdateEvent = lfirst(placementUpdateCell); WorkerNode *sourceNode = placementUpdateEvent->sourceNode; WorkerNode *targetNode = placementUpdateEvent->targetNode; StringInfo escapedSourceName = makeStringInfo(); escape_json(escapedSourceName, sourceNode->workerName); StringInfo escapedTargetName = makeStringInfo(); escape_json(escapedTargetName, targetNode->workerName); StringInfo placementUpdateJsonString = makeStringInfo(); appendStringInfo(placementUpdateJsonString, PLACEMENT_UPDATE_JSON_FORMAT, placementUpdateEvent->updateType, placementUpdateEvent->shardId, escapedSourceName->data, sourceNode->workerPort, escapedTargetName->data, targetNode->workerPort); Datum placementUpdateStringDatum = CStringGetDatum( placementUpdateJsonString->data); Datum placementUpdateJsonDatum = DirectFunctionCall1(json_in, placementUpdateStringDatum); placementUpdateJsonArray[placementUpdateIndex] = placementUpdateJsonDatum; placementUpdateIndex++; } ArrayType *placementUpdateObject = construct_array(placementUpdateJsonArray, placementUpdateCount, JSONOID, -1, false, 'i'); return placementUpdateObject; } /* * worker_node_responsive returns true if the given worker node is responsive. * Otherwise, it returns false. */ Datum worker_node_responsive(PG_FUNCTION_ARGS) { text *workerNameText = PG_GETARG_TEXT_PP(0); uint32 workerPort = PG_GETARG_INT32(1); int connectionFlag = FORCE_NEW_CONNECTION; bool workerNodeResponsive = false; const char *workerName = text_to_cstring(workerNameText); MultiConnection *connection = GetNodeConnection(connectionFlag, workerName, workerPort); if (connection != NULL && connection->pgConn != NULL) { if (PQstatus(connection->pgConn) == CONNECTION_OK) { workerNodeResponsive = true; } CloseConnection(connection); } PG_RETURN_BOOL(workerNodeResponsive); } ================================================ FILE: src/backend/distributed/test/shared_connection_counters.c ================================================ /*------------------------------------------------------------------------- * * test/src/sequential_execution.c * * This file contains functions to test setting citus.multi_shard_modify_mode * GUC. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "miscadmin.h" #include "nodes/parsenodes.h" #include "utils/guc.h" #include "distributed/listutils.h" #include "distributed/shared_connection_stats.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(wake_up_connection_pool_waiters); PG_FUNCTION_INFO_V1(set_max_shared_pool_size); /* * wake_up_waiters_backends is a SQL * interface for testing WakeupWaiterBackendsForSharedConnection(). */ Datum wake_up_connection_pool_waiters(PG_FUNCTION_ARGS) { WakeupWaiterBackendsForSharedConnection(); PG_RETURN_VOID(); } /* * makeIntConst creates a Const Node that stores a given integer * * copied from backend/parser/gram.c */ static Node * makeIntConst(int val, int location) { A_Const *n = makeNode(A_Const); n->val.ival.type = T_Integer; n->val.ival.ival = val; n->location = location; return (Node *) n; } /* * set_max_shared_pool_size is a SQL * interface for setting MaxSharedPoolSize. We use this function in isolation * tester where ALTER SYSTEM is not allowed. */ Datum set_max_shared_pool_size(PG_FUNCTION_ARGS) { int value = PG_GETARG_INT32(0); AlterSystemStmt *alterSystemStmt = palloc0(sizeof(AlterSystemStmt)); A_Const *aConstValue = castNode(A_Const, makeIntConst(value, 0)); alterSystemStmt->setstmt = makeNode(VariableSetStmt); alterSystemStmt->setstmt->name = "citus.max_shared_pool_size"; alterSystemStmt->setstmt->is_local = false; alterSystemStmt->setstmt->kind = VAR_SET_VALUE; alterSystemStmt->setstmt->args = list_make1(aConstValue); AlterSystemSetConfigFile(alterSystemStmt); kill(PostmasterPid, SIGHUP); PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/test/xact_stats.c ================================================ /*------------------------------------------------------------------------- * * xact_stats.c * * This file contains functions to provide helper UDFs for testing transaction * statistics. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "pgstat.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" static Size MemoryContextTotalSpace(MemoryContext context); PG_FUNCTION_INFO_V1(top_transaction_context_size); PG_FUNCTION_INFO_V1(coordinated_transaction_should_use_2PC); /* * top_transaction_context_size returns current size of TopTransactionContext. */ Datum top_transaction_context_size(PG_FUNCTION_ARGS) { Size totalSpace = MemoryContextTotalSpace(TopTransactionContext); PG_RETURN_INT64(totalSpace); } /* * MemoryContextTotalSpace returns total space allocated in context and its children. */ static Size MemoryContextTotalSpace(MemoryContext context) { Size totalSpace = 0; MemoryContextCounters totals = { 0 }; TopTransactionContext->methods->stats(TopTransactionContext, NULL, NULL, &totals, true); totalSpace += totals.totalspace; for (MemoryContext child = context->firstchild; child != NULL; child = child->nextchild) { totalSpace += MemoryContextTotalSpace(child); } return totalSpace; } /* * coordinated_transaction_should_use_2PC returns true if the transaction is in a * coordinated transaction and uses 2PC. If the transaction is nott in a * coordinated transaction, the function throws an error. */ Datum coordinated_transaction_should_use_2PC(PG_FUNCTION_ARGS) { if (!InCoordinatedTransaction()) { ereport(ERROR, (errmsg("The transaction is not a coordinated transaction"))); } PG_RETURN_BOOL(GetCoordinatedTransactionShouldUse2PC()); } ================================================ FILE: src/backend/distributed/transaction/backend_data.c ================================================ /*------------------------------------------------------------------------- * * backend_data.c * * Infrastructure for managing per backend data that can efficiently * accessed by all sessions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "safe_lib.h" #include "unistd.h" #include "access/htup_details.h" #include "catalog/pg_authid.h" #include "catalog/pg_type.h" #include "datatype/timestamp.h" #include "nodes/execnodes.h" #include "postmaster/autovacuum.h" /* to access autovacuum_max_workers */ #include "replication/walsender.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "storage/procarray.h" #include "storage/s_lock.h" #include "storage/spin.h" #include "utils/timestamp.h" #include "pg_version_compat.h" #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/shared_connection_stats.h" #include "distributed/transaction_identifier.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" #define GET_ACTIVE_TRANSACTION_QUERY "SELECT * FROM get_all_active_transactions();" #define ACTIVE_TRANSACTION_COLUMN_COUNT 7 #define GLOBAL_PID_NODE_ID_MULTIPLIER 10000000000 /* * Each backend's data reside in the shared memory * on the BackendManagementShmemData. */ typedef struct BackendManagementShmemData { int trancheId; NamedLWLockTranche namedLockTranche; LWLock lock; /* * We prefer to use an atomic integer over sequences for two * reasons (i) orders of magnitude performance difference * (ii) allowing read-only replicas to be able to generate ids */ pg_atomic_uint64 nextTransactionNumber; /* * Total number of external client backends that are authenticated. * * Note that the counter does not consider any background workers * or such, and also exludes internal connections between nodes. */ pg_atomic_uint32 externalClientBackendCounter; BackendData backends[FLEXIBLE_ARRAY_MEMBER]; } BackendManagementShmemData; /* * CitusBackendType reflects what type of backend we are in. This * can change depending on the application_name. */ typedef enum CitusBackendType { CITUS_BACKEND_NOT_ASSIGNED, CITUS_INTERNAL_BACKEND, CITUS_REBALANCER_BACKEND, CITUS_RUN_COMMAND_BACKEND, EXTERNAL_CLIENT_BACKEND } CitusBackendType; static const char *CitusBackendPrefixes[] = { CITUS_APPLICATION_NAME_PREFIX, CITUS_REBALANCER_APPLICATION_NAME_PREFIX, CITUS_RUN_COMMAND_APPLICATION_NAME_PREFIX, }; static const CitusBackendType CitusBackendTypes[] = { CITUS_INTERNAL_BACKEND, CITUS_REBALANCER_BACKEND, CITUS_RUN_COMMAND_BACKEND, }; static void StoreAllActiveTransactions(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); static bool UserHasPermissionToViewStatsOf(Oid currentUserId, Oid backendOwnedId); static uint64 CalculateGlobalPID(int32 nodeId, pid_t pid); static uint64 GenerateGlobalPID(void); static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static BackendManagementShmemData *backendManagementShmemData = NULL; static BackendData *MyBackendData = NULL; static CitusBackendType CurrentBackendType = CITUS_BACKEND_NOT_ASSIGNED; PG_FUNCTION_INFO_V1(assign_distributed_transaction_id); PG_FUNCTION_INFO_V1(get_current_transaction_id); PG_FUNCTION_INFO_V1(get_global_active_transactions); PG_FUNCTION_INFO_V1(get_all_active_transactions); PG_FUNCTION_INFO_V1(citus_calculate_gpid); PG_FUNCTION_INFO_V1(citus_backend_gpid); PG_FUNCTION_INFO_V1(citus_nodeid_for_gpid); PG_FUNCTION_INFO_V1(citus_pid_for_gpid); /* * assign_distributed_transaction_id updates the shared memory allocated for this backend * and sets initiatorNodeIdentifier, transactionNumber, timestamp fields with the given * inputs. Also, the function sets the database id and process id via the information that * Postgres provides. * * This function is only intended for internal use for managing distributed transactions. * Users should not use this function for any purpose. */ Datum assign_distributed_transaction_id(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); /* prepare data before acquiring spinlock to protect against errors */ int32 initiatorNodeIdentifier = PG_GETARG_INT32(0); uint64 transactionNumber = PG_GETARG_INT64(1); TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(2); /* MyBackendData should always be avaliable, just out of paranoia */ if (!MyBackendData) { ereport(ERROR, (errmsg("backend is not ready for distributed transactions"))); } /* * Note that we don't need to lock shared memory (i.e., LockBackendSharedMemory()) here * since this function is executed after AssignDistributedTransactionId() issued on the * initiator node, which already takes the required lock to enforce the consistency. */ SpinLockAcquire(&MyBackendData->mutex); /* if an id is already assigned, release the lock and error */ if (MyBackendData->transactionId.transactionNumber != 0) { SpinLockRelease(&MyBackendData->mutex); ereport(ERROR, (errmsg("the backend has already been assigned a " "transaction id"))); } MyBackendData->transactionId.initiatorNodeIdentifier = initiatorNodeIdentifier; MyBackendData->transactionId.transactionNumber = transactionNumber; MyBackendData->transactionId.timestamp = timestamp; MyBackendData->transactionId.transactionOriginator = false; SpinLockRelease(&MyBackendData->mutex); PG_RETURN_VOID(); } /* * get_current_transaction_id returns a tuple with (databaseId, processId, * initiatorNodeIdentifier, transactionNumber, timestamp) that exists in the * shared memory associated with this backend. Note that if the backend * is not in a transaction, the function returns uninitialized data where * transactionNumber equals to 0. */ Datum get_current_transaction_id(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Datum values[5]; bool isNulls[5]; /* build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupleDescriptor) != TYPEFUNC_COMPOSITE) { elog(ERROR, "return type must be a row type"); } /* MyBackendData should always be avaliable, just out of paranoia */ if (!MyBackendData) { ereport(ERROR, (errmsg("backend is not ready for distributed transactions"))); } DistributedTransactionId *distributedTransctionId = GetCurrentDistributedTransactionId(); memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); /* first two fields do not change for this backend, so get directly */ values[0] = ObjectIdGetDatum(MyDatabaseId); values[1] = Int32GetDatum(MyProcPid); values[2] = Int32GetDatum(distributedTransctionId->initiatorNodeIdentifier); values[3] = UInt64GetDatum(distributedTransctionId->transactionNumber); /* provide a better output */ if (distributedTransctionId->transactionNumber != 0) { values[4] = TimestampTzGetDatum(distributedTransctionId->timestamp); } else { isNulls[4] = true; } HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); PG_RETURN_DATUM(HeapTupleGetDatum(heapTuple)); } /* * get_global_active_transactions returns all the available information about all * the active backends from each node of the cluster. If you call that function from * the coordinator, it will returns back active transaction from the coordinator as * well. Yet, if you call it from the worker, result won't include the transactions * on the coordinator node, since worker nodes are not aware of the coordinator. */ Datum get_global_active_transactions(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(NoLock); List *connectionList = NIL; StringInfo queryToSend = makeStringInfo(); Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); appendStringInfo(queryToSend, GET_ACTIVE_TRANSACTION_QUERY); /* add active transactions for local node */ StoreAllActiveTransactions(tupleStore, tupleDescriptor); int32 localGroupId = GetLocalGroupId(); /* open connections in parallel */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int connectionFlags = 0; if (workerNode->groupId == localGroupId) { /* we already get these transactions via GetAllActiveTransactions() */ continue; } MultiConnection *connection = StartNodeConnection(connectionFlags, nodeName, nodePort); connectionList = lappend(connectionList, connection); } FinishConnectionListEstablishment(connectionList); /* send commands in parallel */ MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommand(connection, queryToSend->data); if (querySent == 0) { ReportConnectionError(connection, WARNING); } } /* receive query results */ foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; Datum values[ACTIVE_TRANSACTION_COLUMN_COUNT]; bool isNulls[ACTIVE_TRANSACTION_COLUMN_COUNT]; if (PQstatus(connection->pgConn) != CONNECTION_OK) { continue; } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, WARNING); continue; } int64 rowCount = PQntuples(result); int64 colCount = PQnfields(result); /* Although it is not expected */ if (colCount != ACTIVE_TRANSACTION_COLUMN_COUNT) { ereport(WARNING, (errmsg("unexpected number of columns from " "get_all_active_transactions"))); continue; } for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[0] = ParseIntField(result, rowIndex, 0); values[1] = ParseIntField(result, rowIndex, 1); values[2] = ParseIntField(result, rowIndex, 2); values[3] = ParseBoolField(result, rowIndex, 3); values[4] = ParseIntField(result, rowIndex, 4); values[5] = ParseTimestampTzField(result, rowIndex, 5); values[6] = ParseIntField(result, rowIndex, 6); tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); } PQclear(result); ForgetResults(connection); } PG_RETURN_VOID(); } /* * get_all_active_transactions returns all the avaliable information about all * the active backends. */ Datum get_all_active_transactions(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); StoreAllActiveTransactions(tupleStore, tupleDescriptor); PG_RETURN_VOID(); } /* * StoreAllActiveTransactions gets active transaction from the local node and inserts * them into the given tuplestore. */ static void StoreAllActiveTransactions(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor) { Datum values[ACTIVE_TRANSACTION_COLUMN_COUNT]; bool isNulls[ACTIVE_TRANSACTION_COLUMN_COUNT]; bool showAllBackends = superuser(); const Oid userId = GetUserId(); if (!showAllBackends && is_member_of_role(userId, ROLE_PG_MONITOR)) { showAllBackends = true; } /* we're reading all distributed transactions, prevent new backends */ LockBackendSharedMemory(LW_SHARED); for (int backendIndex = 0; backendIndex < TotalProcCount(); ++backendIndex) { bool showCurrentBackendDetails = showAllBackends; BackendData *currentBackend = &backendManagementShmemData->backends[backendIndex]; PGPROC *currentProc = GetPGProcByNumber(backendIndex); /* to work on data after releasing g spinlock to protect against errors */ uint64 transactionNumber = 0; SpinLockAcquire(¤tBackend->mutex); if (currentProc->pid == 0 || !currentBackend->activeBackend) { /* unused PGPROC slot or the backend already exited */ SpinLockRelease(¤tBackend->mutex); continue; } /* * Unless the user has a role that allows seeing all transactions (superuser, * pg_monitor), we only follow pg_stat_statements owner checks. */ if (!showCurrentBackendDetails && UserHasPermissionToViewStatsOf(userId, currentProc->roleId)) { showCurrentBackendDetails = true; } Oid databaseId = currentBackend->databaseId; int backendPid = GetPGProcByNumber(backendIndex)->pid; /* * We prefer to use worker_query instead of distributedCommandOriginator in * the user facing functions since it's more intuitive. Thus, * we negate the result before returning. */ bool distributedCommandOriginator = currentBackend->distributedCommandOriginator; transactionNumber = currentBackend->transactionId.transactionNumber; TimestampTz transactionIdTimestamp = currentBackend->transactionId.timestamp; SpinLockRelease(¤tBackend->mutex); memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); /* * We imitate pg_stat_activity such that if a user doesn't have enough * privileges, we only show the minimal information including the pid, * global pid and distributedCommandOriginator. * * pid is already can be found in pg_stat_activity for any process, and * the rest doesn't reveal anything critial for under priviledge users * but still could be useful for monitoring purposes of Citus. */ if (showCurrentBackendDetails) { bool missingOk = true; int initiatorNodeId = ExtractNodeIdFromGlobalPID(currentBackend->globalPID, missingOk); values[0] = ObjectIdGetDatum(databaseId); values[1] = Int32GetDatum(backendPid); values[2] = Int32GetDatum(initiatorNodeId); values[3] = !distributedCommandOriginator; values[4] = UInt64GetDatum(transactionNumber); values[5] = TimestampTzGetDatum(transactionIdTimestamp); values[6] = UInt64GetDatum(currentBackend->globalPID); } else { isNulls[0] = true; values[1] = Int32GetDatum(backendPid); isNulls[2] = true; values[3] = !distributedCommandOriginator; isNulls[4] = true; isNulls[5] = true; values[6] = UInt64GetDatum(currentBackend->globalPID); } tuplestore_putvalues(tupleStore, tupleDescriptor, values, isNulls); /* * We don't want to initialize memory while spinlock is held so we * prefer to do it here. This initialization is done for the rows * starting from the second one. */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); } UnlockBackendSharedMemory(); } /* * UserHasPermissionToViewStatsOf returns true if currentUserId can * see backends of backendOwnedId. * * We follow the same approach with pg_stat_activity. */ static bool UserHasPermissionToViewStatsOf(Oid currentUserId, Oid backendOwnedId) { if (has_privs_of_role(currentUserId, backendOwnedId)) { return true; } if (is_member_of_role(currentUserId, ROLE_PG_READ_ALL_STATS)) { return true; } return false; } /* * InitializeBackendManagement requests the necessary shared memory * from Postgres and sets up the shared memory startup hook. */ void InitializeBackendManagement(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = BackendManagementShmemInit; } /* * BackendManagementShmemInit is the callback that is to be called on shared * memory startup hook. The function sets up the necessary shared memory * segment for the backend manager. */ void BackendManagementShmemInit(void) { bool alreadyInitialized = false; /* we may update the shmem, acquire lock exclusively */ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); backendManagementShmemData = (BackendManagementShmemData *) ShmemInitStruct( "Backend Management Shmem", BackendManagementShmemSize(), &alreadyInitialized); if (!alreadyInitialized) { char *trancheName = "Backend Management Tranche"; NamedLWLockTranche *namedLockTranche = &backendManagementShmemData->namedLockTranche; /* start by zeroing out all the memory */ memset(backendManagementShmemData, 0, BackendManagementShmemSize()); namedLockTranche->trancheId = LWLockNewTrancheId(); LWLockRegisterTranche(namedLockTranche->trancheId, trancheName); LWLockInitialize(&backendManagementShmemData->lock, namedLockTranche->trancheId); /* start the distributed transaction ids from 1 */ pg_atomic_init_u64(&backendManagementShmemData->nextTransactionNumber, 1); /* there are no active backends yet, so start with zero */ pg_atomic_init_u32(&backendManagementShmemData->externalClientBackendCounter, 0); /* * We need to init per backend's spinlock before any backend * starts its execution. Note that we initialize TotalProcs (e.g., not * MaxBackends) since some of the blocking processes could be prepared * transactions, which aren't covered by MaxBackends. * * We also initiate initiatorNodeIdentifier to -1, which can never be * used as a node id. */ int totalProcs = TotalProcCount(); for (int backendIndex = 0; backendIndex < totalProcs; ++backendIndex) { BackendData *backendData = &backendManagementShmemData->backends[backendIndex]; SpinLockInit(&backendData->mutex); } } LWLockRelease(AddinShmemInitLock); if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * BackendManagementShmemSize returns the size that should be allocated * on the shared memory for backend management. */ size_t BackendManagementShmemSize(void) { Size size = 0; int totalProcs = TotalProcCount(); size = add_size(size, sizeof(BackendManagementShmemData)); size = add_size(size, mul_size(sizeof(BackendData), totalProcs)); return size; } /* * TotalProcCount returns the total processes that could run via the current * postgres server. See the details in the function comments. * * There is one thing we should warn the readers. Citus enforces to be loaded * as the first extension in shared_preload_libraries. However, if any other * extension overrides MaxConnections, autovacuum_max_workers or * max_worker_processes, our reasoning in this function may not work as expected. * Given that it is not a usual pattern for extension, we consider Citus' behaviour * good enough for now. */ int TotalProcCount(void) { int maxBackends = 0; int totalProcs = 0; #ifdef WIN32 /* autovacuum_max_workers is not PGDLLIMPORT, so use a high estimate for windows */ int estimatedMaxAutovacuumWorkers = 30; maxBackends = MaxConnections + estimatedMaxAutovacuumWorkers + 1 + max_worker_processes; #else /* * We're simply imitating Postgrsql's InitializeMaxBackends(). Given that all * the items used here PGC_POSTMASTER, should be safe to access them * anytime during the execution even before InitializeMaxBackends() is called. */ maxBackends = MaxConnections + autovacuum_max_workers + 1 + max_worker_processes; #endif /* * We prefer to maintain space for auxiliary procs or preperad transactions in * the backend space because they could be blocking processes and our current * implementation of distributed deadlock detection could process them * as a regular backend. In the future, we could consider changing deadlock * detection algorithm to ignore auxiliary procs or prepared transactions and * save some space. */ totalProcs = maxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts; totalProcs += max_wal_senders; return totalProcs; } /* * InitializeBackendData initialises MyBackendData to the shared memory segment * belonging to the current backend. * * The function is called through CitusHasBeenLoaded when we first detect that * the Citus extension is present, and after any subsequent invalidation of * pg_dist_partition (see InvalidateMetadataSystemCache()). * * We only need to initialise MyBackendData once. The main goal here is to make * sure that we don't use the backend data from a previous backend with the same * pgprocno. Resetting the backend data after a distributed transaction happens * on COMMIT/ABORT through transaction callbacks. * * We do also initialize the distributedCommandOriginator and globalPID values * based on these values. This is to make sure that once the backend date is * initialized this backend can be correctly shown in citus_lock_waits. */ void InitializeBackendData(const char *applicationName) { if (MyBackendData != NULL) { /* * We already initialized MyBackendData before. We definitely should * not initialise it again, because we might be in the middle of a * distributed transaction. */ return; } uint64 gpid = ExtractGlobalPID(applicationName); MyBackendData = &backendManagementShmemData->backends[getProcNo_compat(MyProc)]; Assert(MyBackendData); LockBackendSharedMemory(LW_EXCLUSIVE); /* zero out the backend its transaction id */ UnSetDistributedTransactionId(); UnSetGlobalPID(); SpinLockAcquire(&MyBackendData->mutex); MyBackendData->distributedCommandOriginator = IsExternalClientBackend(); MyBackendData->globalPID = gpid; SpinLockRelease(&MyBackendData->mutex); /* * Signal that this backend is active and should show up * on activity monitors. */ SetActiveMyBackend(true); UnlockBackendSharedMemory(); } /* * UnSetDistributedTransactionId simply acquires the mutex and resets the backend's * distributed transaction data in shared memory to the initial values. */ void UnSetDistributedTransactionId(void) { /* backend does not exist if the extension is not created */ if (MyBackendData) { SpinLockAcquire(&MyBackendData->mutex); MyBackendData->cancelledDueToDeadlock = false; MyBackendData->transactionId.initiatorNodeIdentifier = 0; MyBackendData->transactionId.transactionOriginator = false; MyBackendData->transactionId.transactionNumber = 0; MyBackendData->transactionId.timestamp = 0; SpinLockRelease(&MyBackendData->mutex); } } /* * UnSetGlobalPID resets the global pid for the current backend. */ void UnSetGlobalPID(void) { /* backend does not exist if the extension is not created */ if (MyBackendData) { SpinLockAcquire(&MyBackendData->mutex); MyBackendData->globalPID = 0; MyBackendData->databaseId = 0; MyBackendData->distributedCommandOriginator = false; SpinLockRelease(&MyBackendData->mutex); } } /* * SetActiveMyBackend is a wrapper around MyBackendData->activeBackend. */ void SetActiveMyBackend(bool value) { /* backend does not exist if the extension is not created */ if (MyBackendData) { SpinLockAcquire(&MyBackendData->mutex); MyBackendData->activeBackend = value; SpinLockRelease(&MyBackendData->mutex); } } /* * LockBackendSharedMemory is a simple wrapper around LWLockAcquire on the * shared memory lock. * * We use the backend shared memory lock for preventing new backends to be part * of a new distributed transaction or an existing backend to leave a distributed * transaction while we're reading the all backends' data. * * The primary goal is to provide consistent view of the current distributed * transactions while doing the deadlock detection. */ void LockBackendSharedMemory(LWLockMode lockMode) { LWLockAcquire(&backendManagementShmemData->lock, lockMode); } /* * UnlockBackendSharedMemory is a simple wrapper around LWLockRelease on the * shared memory lock. */ void UnlockBackendSharedMemory(void) { LWLockRelease(&backendManagementShmemData->lock); } /* * GetCurrentDistributedTransactionId reads the backend's distributed transaction id and * returns a copy of it. * * When called from a parallel worker, it uses the parent's transaction ID per the logic * in GetBackendDataForProc. */ DistributedTransactionId * GetCurrentDistributedTransactionId(void) { DistributedTransactionId *currentDistributedTransactionId = (DistributedTransactionId *) palloc(sizeof(DistributedTransactionId)); BackendData backendData; GetBackendDataForProc(MyProc, &backendData); currentDistributedTransactionId->initiatorNodeIdentifier = backendData.transactionId.initiatorNodeIdentifier; currentDistributedTransactionId->transactionOriginator = backendData.transactionId.transactionOriginator; currentDistributedTransactionId->transactionNumber = backendData.transactionId.transactionNumber; currentDistributedTransactionId->timestamp = backendData.transactionId.timestamp; return currentDistributedTransactionId; } /* * AssignDistributedTransactionId generates a new distributed transaction id and * sets it for the current backend. It also sets the databaseId and * processId fields. * * This function should only be called on UseCoordinatedTransaction(). Any other * callers is very likely to break the distributed transaction management. */ void AssignDistributedTransactionId(void) { /* * MyBackendData should always be available. However, we observed some * crashes where certain hooks were not executed. * Bug 3697586: Server crashes when assigning distributed transaction */ if (!MyBackendData) { ereport(ERROR, (errmsg("backend is not ready for distributed transactions"))); } pg_atomic_uint64 *transactionNumberSequence = &backendManagementShmemData->nextTransactionNumber; uint64 nextTransactionNumber = pg_atomic_fetch_add_u64(transactionNumberSequence, 1); int32 localGroupId = GetLocalGroupId(); TimestampTz currentTimestamp = GetCurrentTimestamp(); SpinLockAcquire(&MyBackendData->mutex); MyBackendData->transactionId.initiatorNodeIdentifier = localGroupId; MyBackendData->transactionId.transactionOriginator = true; MyBackendData->transactionId.transactionNumber = nextTransactionNumber; MyBackendData->transactionId.timestamp = currentTimestamp; SpinLockRelease(&MyBackendData->mutex); } /* * AssignGlobalPID assigns a global process id for the current backend based on * the given applicationName. If this is a Citus initiated backend, which means * it is distributed part of a distributed query, then this function assigns * the global pid extracted from the application name. If not, this function * assigns a new generated global pid. * * There's one special case where we don't want to assign a new pid and keep * the old pid on purpose: The current backend is an external backend and the * node id of the current node changed since the previous call to * AssingGlobalPID. Updating the gpid to match the nodeid might seem like a * desirable property, but that's not the case. Mainly, because existing cached * connections will still report as the old gpid on the worker. So updating the * gpid with the new nodeid would mess up distributed deadlock and originator * detection of queries done using those old connections. So if this is an * external backend for which a gpid was already generated, then we don't * change the gpid. * * NOTE: This function can be called arbitrary amount of times for the same * backend, due to being called by StartupCitusBackend. */ void AssignGlobalPID(const char *applicationName) { uint64 globalPID = INVALID_CITUS_INTERNAL_BACKEND_GPID; bool distributedCommandOriginator = IsExternalClientBackend(); if (distributedCommandOriginator) { globalPID = GenerateGlobalPID(); } else { globalPID = ExtractGlobalPID(applicationName); } SpinLockAcquire(&MyBackendData->mutex); /* * Skip updating globalpid when we were a command originator and still are * and we already have a valid global pid assigned. * See function comment for detailed explanation. */ if (!MyBackendData->distributedCommandOriginator || !distributedCommandOriginator || MyBackendData->globalPID == INVALID_CITUS_INTERNAL_BACKEND_GPID) { MyBackendData->globalPID = globalPID; MyBackendData->distributedCommandOriginator = distributedCommandOriginator; } SpinLockRelease(&MyBackendData->mutex); } /* * SetBackendDataDatabaseId sets the databaseId in the backend data. * * NOTE: this needs to be run after the auth hook, because in the auth hook the * database is not known yet. */ void SetBackendDataDatabaseId(void) { Assert(MyDatabaseId != InvalidOid); SpinLockAcquire(&MyBackendData->mutex); MyBackendData->databaseId = MyDatabaseId; SpinLockRelease(&MyBackendData->mutex); } /* * SetBackendDataGlobalPID is used to set the gpid field on MyBackendData. * * IMPORTANT: This should not be used for normal operations. It's a very hacky * way of setting the gpid, which is only used in our MX isolation tests. The * main problem is that it does not set distributedCommandOriginator to the * correct value. */ void SetBackendDataGlobalPID(uint64 gpid) { if (!MyBackendData) { return; } SpinLockAcquire(&MyBackendData->mutex); MyBackendData->globalPID = gpid; SpinLockRelease(&MyBackendData->mutex); } /* * SetBackendDataDistributedCommandOriginator sets the distributedCommandOriginator * field on MyBackendData. */ void SetBackendDataDistributedCommandOriginator(bool distributedCommandOriginator) { if (!MyBackendData) { return; } SpinLockAcquire(&MyBackendData->mutex); MyBackendData->distributedCommandOriginator = distributedCommandOriginator; SpinLockRelease(&MyBackendData->mutex); } /* * GetGlobalPID returns the global process id of the current backend. */ uint64 GetGlobalPID(void) { uint64 globalPID = INVALID_CITUS_INTERNAL_BACKEND_GPID; if (MyBackendData) { SpinLockAcquire(&MyBackendData->mutex); globalPID = MyBackendData->globalPID; SpinLockRelease(&MyBackendData->mutex); } return globalPID; } /* * citus_calculate_gpid calculates the gpid for any given process on any * given node. */ Datum citus_calculate_gpid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int32 nodeId = PG_GETARG_INT32(0); int32 pid = PG_GETARG_INT32(1); PG_RETURN_UINT64(CalculateGlobalPID(nodeId, pid)); } /* * CalculateGlobalPID gets a nodeId and pid, and returns the global pid * that can be assigned for a process with the given input. */ static uint64 CalculateGlobalPID(int32 nodeId, pid_t pid) { /* * We try to create a human readable global pid that consists of node id and process id. * By multiplying node id with 10^10 and adding pid we generate a number where the smallest * 10 digit represent the pid and the remaining digits are the node id. * * Both node id and pid are 32 bit. We use 10^10 to fit all possible pids. Some very large * node ids might cause overflow. But even for the applications that scale around 50 nodes every * day it'd take about 100K years. So we are not worried. */ return (((uint64) nodeId) * GLOBAL_PID_NODE_ID_MULTIPLIER) + pid; } /* * GenerateGlobalPID generates the global process id for the current backend. * See CalculateGlobalPID for the details. */ static uint64 GenerateGlobalPID(void) { return CalculateGlobalPID(GetLocalNodeId(), getpid()); } /* * citus_backend_gpid similar to pg_backend_pid, but returns Citus * assigned gpid. */ Datum citus_backend_gpid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); PG_RETURN_UINT64(GetGlobalPID()); } /* * citus_nodeid_for_gpid returns node id for the global process with given global pid */ Datum citus_nodeid_for_gpid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 globalPID = PG_GETARG_INT64(0); bool missingOk = false; PG_RETURN_INT32(ExtractNodeIdFromGlobalPID(globalPID, missingOk)); } /* * citus_pid_for_gpid returns process id for the global process with given global pid */ Datum citus_pid_for_gpid(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 globalPID = PG_GETARG_INT64(0); PG_RETURN_INT32(ExtractProcessIdFromGlobalPID(globalPID)); } /* * ExtractGlobalPID extracts the global process id from the application name and returns it * if the application name is not compatible with Citus' application names returns 0. */ uint64 ExtractGlobalPID(const char *applicationName) { /* does application name exist */ if (!applicationName) { return INVALID_CITUS_INTERNAL_BACKEND_GPID; } /* we create our own copy of application name incase the original changes */ char *applicationNameCopy = pstrdup(applicationName); for (int i = 0; i < lengthof(CitusBackendPrefixes); i++) { uint64 prefixLength = strlen(CitusBackendPrefixes[i]); /* does application name start with this prefix prefix */ if (strncmp(applicationNameCopy, CitusBackendPrefixes[i], prefixLength) != 0) { continue; } char *globalPIDString = &applicationNameCopy[prefixLength]; uint64 globalPID = strtoul(globalPIDString, NULL, 10); if (globalPID == 0) { /* * INVALID_CITUS_INTERNAL_BACKEND_GPID is 0, but just to be explicit * about how we handle strtoul errors. */ return INVALID_CITUS_INTERNAL_BACKEND_GPID; } return globalPID; } return INVALID_CITUS_INTERNAL_BACKEND_GPID; } /* * ExtractNodeIdFromGlobalPID extracts the node id from the global pid. * Global pid is constructed by multiplying node id with GLOBAL_PID_NODE_ID_MULTIPLIER * and adding process id. So integer division of global pid by GLOBAL_PID_NODE_ID_MULTIPLIER * gives us the node id. */ int ExtractNodeIdFromGlobalPID(uint64 globalPID, bool missingOk) { int nodeId = (int) (globalPID / GLOBAL_PID_NODE_ID_MULTIPLIER); if (!missingOk && nodeId == GLOBAL_PID_NODE_ID_FOR_NODES_NOT_IN_METADATA) { ereport(ERROR, (errmsg("originator node of the query with the global pid " "%lu is not in Citus' metadata", globalPID), errhint("connect to the node directly run pg_cancel_backend(pid) " "or pg_terminate_backend(pid)"))); } return nodeId; } /* * ExtractProcessIdFromGlobalPID extracts the process id from the global pid. * Global pid is constructed by multiplying node id with GLOBAL_PID_NODE_ID_MULTIPLIER * and adding process id. So global pid mod GLOBAL_PID_NODE_ID_MULTIPLIER gives us the * process id. */ int ExtractProcessIdFromGlobalPID(uint64 globalPID) { return (int) (globalPID % GLOBAL_PID_NODE_ID_MULTIPLIER); } /* * CurrentDistributedTransactionNumber returns the transaction number of the * current distributed transaction. The caller must make sure a distributed * transaction is in progress. */ uint64 CurrentDistributedTransactionNumber(void) { Assert(MyBackendData != NULL); return MyBackendData->transactionId.transactionNumber; } /* * GetBackendDataForProc writes the backend data for the given process to * result. If the process is part of a lock group (parallel query) it * returns the leader data instead. */ void GetBackendDataForProc(PGPROC *proc, BackendData *result) { int pgprocno = getProcNo_compat(proc); if (proc->lockGroupLeader != NULL) { pgprocno = getProcNo_compat(proc->lockGroupLeader); } BackendData *backendData = &backendManagementShmemData->backends[pgprocno]; SpinLockAcquire(&backendData->mutex); *result = *backendData; SpinLockRelease(&backendData->mutex); } /* * CancelTransactionDueToDeadlock cancels the input proc and also marks the backend * data with this information. */ void CancelTransactionDueToDeadlock(PGPROC *proc) { BackendData *backendData = &backendManagementShmemData->backends[getProcNo_compat( proc)]; /* backend might not have used citus yet and thus not initialized backend data */ if (!backendData) { return; } SpinLockAcquire(&backendData->mutex); /* send a SIGINT only if the process is still in a distributed transaction */ if (backendData->transactionId.transactionNumber != 0) { backendData->cancelledDueToDeadlock = true; SpinLockRelease(&backendData->mutex); if (kill(proc->pid, SIGINT) != 0) { ereport(WARNING, (errmsg("attempted to cancel this backend (pid: %d) to resolve a " "distributed deadlock but the backend could not " "be cancelled", proc->pid))); } } else { SpinLockRelease(&backendData->mutex); } } /* * MyBackendGotCancelledDueToDeadlock returns whether the current distributed * transaction was cancelled due to a deadlock. If the backend is not in a * distributed transaction, the function returns false. * We keep some session level state to keep track of if we were cancelled * because of a distributed deadlock. When clearState is true, this function * also resets that state. So after calling this function with clearState true, * a second would always return false. */ bool MyBackendGotCancelledDueToDeadlock(bool clearState) { bool cancelledDueToDeadlock = false; /* backend might not have used citus yet and thus not initialized backend data */ if (!MyBackendData) { return false; } SpinLockAcquire(&MyBackendData->mutex); if (IsInDistributedTransaction(MyBackendData)) { cancelledDueToDeadlock = MyBackendData->cancelledDueToDeadlock; } if (clearState) { MyBackendData->cancelledDueToDeadlock = false; } SpinLockRelease(&MyBackendData->mutex); return cancelledDueToDeadlock; } /* * ActiveDistributedTransactionNumbers returns a list of pointers to * transaction numbers of distributed transactions that are in progress * and were started by the node on which it is called. */ List * ActiveDistributedTransactionNumbers(void) { List *activeTransactionNumberList = NIL; /* build list of starting procs */ for (int curBackend = 0; curBackend < MaxBackends; curBackend++) { PGPROC *currentProc = GetPGProcByNumber(curBackend); BackendData currentBackendData; if (currentProc->pid == 0) { /* unused PGPROC slot */ continue; } GetBackendDataForProc(currentProc, ¤tBackendData); if (!currentBackendData.activeBackend) { /* * Skip if the PGPROC slot is unused. We should normally use * IsBackendPid() to be able to skip reliably all the exited * processes. However, that is a costly operation. Instead, we * keep track of activeBackend in Citus code. */ continue; } if (!IsInDistributedTransaction(¤tBackendData)) { /* not a distributed transaction */ continue; } if (!currentBackendData.transactionId.transactionOriginator) { /* not a coordinator process */ continue; } uint64 *transactionNumber = (uint64 *) palloc0(sizeof(uint64)); *transactionNumber = currentBackendData.transactionId.transactionNumber; activeTransactionNumberList = lappend(activeTransactionNumberList, transactionNumber); } return activeTransactionNumberList; } /* * GetMyProcLocalTransactionId() is a wrapper for * getting lxid of MyProc. */ LocalTransactionId GetMyProcLocalTransactionId(void) { return getLxid_compat(MyProc); } /* * GetExternalClientBackendCount returns externalClientBackendCounter in * the shared memory. */ int GetExternalClientBackendCount(void) { uint32 activeBackendCount = pg_atomic_read_u32(&backendManagementShmemData->externalClientBackendCounter); return activeBackendCount; } /* * IncrementExternalClientBackendCounter increments externalClientBackendCounter in * the shared memory by one. */ uint32 IncrementExternalClientBackendCounter(void) { return pg_atomic_add_fetch_u32( &backendManagementShmemData->externalClientBackendCounter, 1); } /* * DecrementExternalClientBackendCounter decrements externalClientBackendCounter in * the shared memory by one. */ void DecrementExternalClientBackendCounter(void) { pg_atomic_sub_fetch_u32(&backendManagementShmemData->externalClientBackendCounter, 1); } /* * IsRebalancerInitiatedBackend returns true if we are in a backend that citus * rebalancer initiated. */ bool IsRebalancerInternalBackend(void) { if (CurrentBackendType == CITUS_BACKEND_NOT_ASSIGNED) { DetermineCitusBackendType(application_name); } return CurrentBackendType == CITUS_REBALANCER_BACKEND; } /* * IsCitusInitiatedRemoteBackend returns true if we are in a backend that citus * initiated via remote connection. */ bool IsCitusInternalBackend(void) { if (CurrentBackendType == CITUS_BACKEND_NOT_ASSIGNED) { DetermineCitusBackendType(application_name); } return CurrentBackendType == CITUS_INTERNAL_BACKEND; } /* * IsCitusRunCommandBackend returns true if we are in a backend that one of * the run_command_on_* functions initiated. */ bool IsCitusRunCommandBackend(void) { if (CurrentBackendType == CITUS_BACKEND_NOT_ASSIGNED) { DetermineCitusBackendType(application_name); } return CurrentBackendType == CITUS_RUN_COMMAND_BACKEND; } bool IsExternalClientBackend(void) { if (CurrentBackendType == CITUS_BACKEND_NOT_ASSIGNED) { DetermineCitusBackendType(application_name); } return CurrentBackendType == EXTERNAL_CLIENT_BACKEND; } /* * IsRebalancerInitiatedBackend returns true if we are in a backend that citus * rebalancer initiated. */ bool IsCitusShardTransferBackend(void) { int prefixLength = strlen(CITUS_SHARD_TRANSFER_APPLICATION_NAME_PREFIX); return strncmp(application_name, CITUS_SHARD_TRANSFER_APPLICATION_NAME_PREFIX, prefixLength) == 0; } /* * DetermineCitusBackendType determines the type of backend based on the application_name. */ void DetermineCitusBackendType(const char *applicationName) { if (applicationName && ExtractGlobalPID(applicationName) != INVALID_CITUS_INTERNAL_BACKEND_GPID) { for (int i = 0; i < lengthof(CitusBackendPrefixes); i++) { uint64 prefixLength = strlen(CitusBackendPrefixes[i]); /* does application name start with this prefix prefix */ if (strncmp(applicationName, CitusBackendPrefixes[i], prefixLength) == 0) { CurrentBackendType = CitusBackendTypes[i]; return; } } } CurrentBackendType = EXTERNAL_CLIENT_BACKEND; } ================================================ FILE: src/backend/distributed/transaction/citus_dist_stat_activity.c ================================================ /*------------------------------------------------------------------------- * * citus_dist_stat_activity.c * * The methods in the file are deprecated. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" PG_FUNCTION_INFO_V1(citus_dist_stat_activity); PG_FUNCTION_INFO_V1(citus_worker_stat_activity); /* This UDF is deprecated. */ Datum citus_dist_stat_activity(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_NULL(); } /* This UDF is deprecated. */ Datum citus_worker_stat_activity(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_NULL(); } ================================================ FILE: src/backend/distributed/transaction/distributed_deadlock_detection.c ================================================ /*------------------------------------------------------------------------- * * distributed_deadlock_detection.c * * Functions for performing distributed deadlock detection. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "access/hash.h" #include "nodes/pg_list.h" #include "utils/hsearch.h" #include "utils/timestamp.h" #include "distributed/backend_data.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/errormessage.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/transaction_identifier.h" /* used only for finding the deadlock cycle path */ typedef struct QueuedTransactionNode { TransactionNode *transactionNode; int currentStackDepth; } QueuedTransactionNode; /* GUC, determining whether debug messages for deadlock detection sent to LOG */ bool LogDistributedDeadlockDetection = false; static bool CheckDeadlockForTransactionNode(TransactionNode *startingTransactionNode, int maxStackDepth, List **deadlockPath); static void PrependOutgoingNodesToQueue(TransactionNode *queuedTransactionNode, int currentStackDepth, List **toBeVisitedNodes); static void BuildDeadlockPathList(QueuedTransactionNode *cycledTransactionNode, TransactionNode **transactionNodeStack, List **deadlockPath); static void ResetVisitedFields(HTAB *adjacencyList); static bool AssociateDistributedTransactionWithBackendProc(TransactionNode * transactionNode); static TransactionNode * GetOrCreateTransactionNode(HTAB *adjacencyList, DistributedTransactionId * transactionId); static uint32 DistributedTransactionIdHash(const void *key, Size keysize); static int DistributedTransactionIdCompare(const void *a, const void *b, Size keysize); static void LogCancellingBackend(TransactionNode *transactionNode); static void LogTransactionNode(TransactionNode *transactionNode); static void LogDistributedDeadlockDebugMessage(const char *errorMessage); PG_FUNCTION_INFO_V1(check_distributed_deadlocks); /* * check_distributed_deadlocks is the external API for manually * checking for distributed deadlocks. For the details, see * CheckForDistributedDeadlocks(). */ Datum check_distributed_deadlocks(PG_FUNCTION_ARGS) { bool deadlockFound = CheckForDistributedDeadlocks(); return BoolGetDatum(deadlockFound); } /* * CheckForDistributedDeadlocks is the entry point for detecting * distributed deadlocks. * * In plain words, the function first builds a wait graph by * adding the wait edges from the local node and then adding the * remote wait edges to form a global wait graph. Later, the wait * graph is converted into another graph representation (adjacency * lists) for more efficient searches. Finally, a DFS is done on * the adjacency lists. Finding a cycle in the graph unveils a * distributed deadlock. Upon finding a deadlock, the youngest * participant backend is cancelled. * * The complexity of the algorithm is O(N) for each distributed * transaction that's checked for deadlocks. Note that there exists * 0 to MaxBackends number of transactions. * * The function returns true if a deadlock is found. Otherwise, returns * false. */ bool CheckForDistributedDeadlocks(void) { HASH_SEQ_STATUS status; TransactionNode *transactionNode = NULL; int32 localGroupId = GetLocalGroupId(); List *workerNodeList = ActiveReadableNodeList(); /* * We don't need to do any distributed deadlock checking if there * are no worker nodes. This might even be problematic for a non-mx * worker node which has the same group id with its master (i.e., 0), * which may erroneously decide to kill the deadlocks happening on it. */ if (list_length(workerNodeList) == 0) { return false; } /* distributed deadlock detection only considers distributed txs */ bool onlyDistributedTx = true; WaitGraph *waitGraph = BuildGlobalWaitGraph(onlyDistributedTx); HTAB *adjacencyLists = BuildAdjacencyListsForWaitGraph(waitGraph); int edgeCount = waitGraph->edgeCount; /* * We iterate on transaction nodes and search for deadlocks where the * starting node is the given transaction node. */ hash_seq_init(&status, adjacencyLists); while ((transactionNode = (TransactionNode *) hash_seq_search(&status)) != 0) { List *deadlockPath = NIL; /* * Since we only see nodes which are waiting or being waited upon it's not * possible to have more than edgeCount + 1 nodes. */ int maxStackDepth = edgeCount + 1; /* we're only interested in finding deadlocks originating from this node */ if (transactionNode->transactionId.initiatorNodeIdentifier != localGroupId) { continue; } ResetVisitedFields(adjacencyLists); bool deadlockFound = CheckDeadlockForTransactionNode(transactionNode, maxStackDepth, &deadlockPath); if (deadlockFound) { TransactionNode *youngestAliveTransaction = NULL; /* * There should generally be at least two transactions to get into a * deadlock. However, in case Citus gets into a self-deadlock, we may * find a deadlock with a single transaction. */ Assert(list_length(deadlockPath) >= 1); LogDistributedDeadlockDebugMessage("Distributed deadlock found among the " "following distributed transactions:"); /* * We search for the youngest participant for two reasons * (i) predictable results (ii) cancel the youngest transaction * (i.e., if a DDL continues for 1 hour and deadlocks with a * SELECT continues for 10 msec, we prefer to cancel the SELECT). * * We're also searching for the youngest transactions initiated by * this node. */ TransactionNode *currentNode = NULL; foreach_declared_ptr(currentNode, deadlockPath) { bool transactionAssociatedWithProc = AssociateDistributedTransactionWithBackendProc(currentNode); LogTransactionNode(currentNode); /* we couldn't find the backend process originated the transaction */ if (!transactionAssociatedWithProc) { continue; } if (youngestAliveTransaction == NULL) { youngestAliveTransaction = currentNode; continue; } TimestampTz youngestTimestamp = youngestAliveTransaction->transactionId.timestamp; TimestampTz currentTimestamp = currentNode->transactionId.timestamp; if (timestamptz_cmp_internal(currentTimestamp, youngestTimestamp) == 1) { youngestAliveTransaction = currentNode; } } /* we found the deadlock and its associated proc exists */ if (youngestAliveTransaction) { CancelTransactionDueToDeadlock(youngestAliveTransaction->initiatorProc); LogCancellingBackend(youngestAliveTransaction); hash_seq_term(&status); return true; } } } return false; } /* * CheckDeadlockForTransactionNode does a DFS starting with the given * transaction node and checks for a cycle (i.e., the node can be reached again * while traversing the graph). * * Finding a cycle indicates a distributed deadlock and the function returns * true on that case. Also, the deadlockPath is filled with the transaction * nodes that form the cycle. */ static bool CheckDeadlockForTransactionNode(TransactionNode *startingTransactionNode, int maxStackDepth, List **deadlockPath) { List *toBeVisitedNodes = NIL; const int rootStackDepth = 0; TransactionNode **transactionNodeStack = palloc0(maxStackDepth * sizeof(TransactionNode *)); /* * We keep transactionNodeStack to keep track of the deadlock paths. At this point, * adjust the depth of the starting node and set the stack's first element with * the starting node. */ transactionNodeStack[rootStackDepth] = startingTransactionNode; PrependOutgoingNodesToQueue(startingTransactionNode, rootStackDepth, &toBeVisitedNodes); /* traverse the graph and search for the deadlocks */ while (toBeVisitedNodes != NIL) { QueuedTransactionNode *queuedTransactionNode = (QueuedTransactionNode *) linitial(toBeVisitedNodes); TransactionNode *currentTransactionNode = queuedTransactionNode->transactionNode; toBeVisitedNodes = list_delete_first(toBeVisitedNodes); /* cycle found, let the caller know about the cycle */ if (currentTransactionNode == startingTransactionNode) { BuildDeadlockPathList(queuedTransactionNode, transactionNodeStack, deadlockPath); pfree(transactionNodeStack); return true; } /* don't need to revisit the node again */ if (currentTransactionNode->transactionVisited) { continue; } currentTransactionNode->transactionVisited = true; /* set the stack's corresponding element with the current node */ int currentStackDepth = queuedTransactionNode->currentStackDepth; Assert(currentStackDepth < maxStackDepth); transactionNodeStack[currentStackDepth] = currentTransactionNode; PrependOutgoingNodesToQueue(currentTransactionNode, currentStackDepth, &toBeVisitedNodes); } pfree(transactionNodeStack); return false; } /* * PrependOutgoingNodesToQueue prepends the waiters of the input transaction nodes to the * toBeVisitedNodes. */ static void PrependOutgoingNodesToQueue(TransactionNode *transactionNode, int currentStackDepth, List **toBeVisitedNodes) { /* as we traverse outgoing edges, increment the depth */ currentStackDepth++; /* prepend to the list to continue depth-first search */ TransactionNode *waitForTransaction = NULL; foreach_declared_ptr(waitForTransaction, transactionNode->waitsFor) { QueuedTransactionNode *queuedNode = palloc0(sizeof(QueuedTransactionNode)); queuedNode->transactionNode = waitForTransaction; queuedNode->currentStackDepth = currentStackDepth; *toBeVisitedNodes = lcons(queuedNode, *toBeVisitedNodes); } } /* * BuildDeadlockPathList fills deadlockPath with a list of transactions involved * in a distributed deadlock (i.e. a cycle in the graph). */ static void BuildDeadlockPathList(QueuedTransactionNode *cycledTransactionNode, TransactionNode **transactionNodeStack, List **deadlockPath) { int deadlockStackDepth = cycledTransactionNode->currentStackDepth; *deadlockPath = NIL; for (int stackIndex = 0; stackIndex < deadlockStackDepth; stackIndex++) { *deadlockPath = lappend(*deadlockPath, transactionNodeStack[stackIndex]); } } /* * ResetVisitedFields goes over all the elements of the input adjacency list * and sets transactionVisited to false. */ static void ResetVisitedFields(HTAB *adjacencyList) { HASH_SEQ_STATUS status; TransactionNode *resetNode = NULL; /* reset all visited fields */ hash_seq_init(&status, adjacencyList); while ((resetNode = (TransactionNode *) hash_seq_search(&status)) != 0) { resetNode->transactionVisited = false; } } /* * AssociateDistributedTransactionWithBackendProc gets a transaction node * and searches the corresponding backend. Once found, transactionNodes' * initiatorProc is set to it. * * The function goes over all the backends, checks for the backend with * the same transaction number as the given transaction node. * * If the transaction cannot be associated with a backend process, the function * returns false. Otherwise, the function returns true. */ static bool AssociateDistributedTransactionWithBackendProc(TransactionNode *transactionNode) { int32 localGroupId PG_USED_FOR_ASSERTS_ONLY = GetLocalGroupId(); for (int backendIndex = 0; backendIndex < MaxBackends; ++backendIndex) { PGPROC *currentProc = GetPGProcByNumber(backendIndex); BackendData currentBackendData; /* we're not interested in processes that are not active or waiting on a lock */ if (currentProc->pid <= 0) { continue; } GetBackendDataForProc(currentProc, ¤tBackendData); /* we're only interested in distribtued transactions */ if (!IsInDistributedTransaction(¤tBackendData)) { continue; } DistributedTransactionId *currentTransactionId = ¤tBackendData.transactionId; if (currentTransactionId->transactionNumber != transactionNode->transactionId. transactionNumber) { continue; } /* we're only interested in transactions started on this node */ if (!currentTransactionId->transactionOriginator) { continue; } /* at the point we should only have transactions initiated by this node */ Assert(currentTransactionId->initiatorNodeIdentifier == localGroupId); transactionNode->initiatorProc = currentProc; return true; } return false; } /* * BuildAdjacencyListsForWaitGraph converts the input wait graph to * an adjacency list for further processing. * * The input wait graph consists of set of wait edges between all * backends in the Citus cluster. * * We represent the adjacency list with an HTAB structure. Each node is * represented with a DistributedTransactionId and each edge is represented with * a TransactionNode structure. * * While iterating over the input wait edges, we follow the algorithm * below: * for each edge in waitGraph: * - find the corresponding nodes for waiting and * blocking transactions in the adjacency list * - if not found, add new node(s) to the list * - Add blocking transaction to the waiting transaction's waitFor * list * * The format of the adjacency list becomes the following: * [transactionId] = [transactionNode->waitsFor {list of waiting transaction nodes}] */ extern HTAB * BuildAdjacencyListsForWaitGraph(WaitGraph *waitGraph) { HASHCTL info; int edgeCount = waitGraph->edgeCount; memset(&info, 0, sizeof(info)); info.keysize = sizeof(DistributedTransactionId); info.entrysize = sizeof(TransactionNode); info.hash = DistributedTransactionIdHash; info.match = DistributedTransactionIdCompare; info.hcxt = CurrentMemoryContext; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE); HTAB *adjacencyList = hash_create("distributed deadlock detection", 64, &info, hashFlags); for (int edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) { WaitEdge *edge = &waitGraph->edges[edgeIndex]; bool transactionOriginator = false; DistributedTransactionId waitingId = { edge->waitingNodeId, transactionOriginator, edge->waitingTransactionNum, edge->waitingTransactionStamp }; DistributedTransactionId blockingId = { edge->blockingNodeId, transactionOriginator, edge->blockingTransactionNum, edge->blockingTransactionStamp }; TransactionNode *waitingTransaction = GetOrCreateTransactionNode(adjacencyList, &waitingId); TransactionNode *blockingTransaction = GetOrCreateTransactionNode(adjacencyList, &blockingId); waitingTransaction->waitsFor = lappend(waitingTransaction->waitsFor, blockingTransaction); } return adjacencyList; } /* * GetOrCreateTransactionNode searches distributedTransactionHash for the given * given transactionId. If the transaction is not found, a new transaction node * with the given transaction identifier is added. */ static TransactionNode * GetOrCreateTransactionNode(HTAB *adjacencyList, DistributedTransactionId *transactionId) { bool found = false; TransactionNode *transactionNode = (TransactionNode *) hash_search(adjacencyList, transactionId, HASH_ENTER, &found); if (!found) { transactionNode->waitsFor = NIL; transactionNode->initiatorProc = NULL; } return transactionNode; } /* * DistributedTransactionIdHash returns hashed value for a given distributed * transaction id. */ static uint32 DistributedTransactionIdHash(const void *key, Size keysize) { DistributedTransactionId *entry = (DistributedTransactionId *) key; uint32 hash = hash_uint32(entry->initiatorNodeIdentifier); hash = hash_combine(hash, hash_any((unsigned char *) &entry->transactionNumber, sizeof(int64))); hash = hash_combine(hash, hash_any((unsigned char *) &entry->timestamp, sizeof(TimestampTz))); return hash; } /* * DistributedTransactionIdCompare compares DistributedTransactionId's a and b * and returns -1 if a < b, 1 if a > b, 0 if they are equal. * * DistributedTransactionId are first compared by their timestamp, then transaction * number, then node identifier. */ static int DistributedTransactionIdCompare(const void *a, const void *b, Size keysize) { DistributedTransactionId *xactIdA = (DistributedTransactionId *) a; DistributedTransactionId *xactIdB = (DistributedTransactionId *) b; if (!TimestampDifferenceExceeds(xactIdB->timestamp, xactIdA->timestamp, 0)) { /* ! (B <= A) = A < B */ return -1; } else if (!TimestampDifferenceExceeds(xactIdA->timestamp, xactIdB->timestamp, 0)) { /* ! (A <= B) = A > B */ return 1; } else if (xactIdA->transactionNumber < xactIdB->transactionNumber) { return -1; } else if (xactIdA->transactionNumber > xactIdB->transactionNumber) { return 1; } else if (xactIdA->initiatorNodeIdentifier < xactIdB->initiatorNodeIdentifier) { return -1; } else if (xactIdA->initiatorNodeIdentifier > xactIdB->initiatorNodeIdentifier) { return 1; } else { return 0; } } /* * LogCancellingBackend should only be called when a distributed transaction's * backend is cancelled due to distributed deadlocks. It sends which transaction * is cancelled and its corresponding pid to the log. */ static void LogCancellingBackend(TransactionNode *transactionNode) { if (!LogDistributedDeadlockDetection) { return; } StringInfo logMessage = makeStringInfo(); appendStringInfo(logMessage, "Cancelling the following backend " "to resolve distributed deadlock " "(transaction number = " UINT64_FORMAT ", pid = %d)", transactionNode->transactionId.transactionNumber, transactionNode->initiatorProc->pid); LogDistributedDeadlockDebugMessage(logMessage->data); } /* * LogTransactionNode converts the transaction node to a human readable form * and sends to the logs via LogDistributedDeadlockDebugMessage(). */ static void LogTransactionNode(TransactionNode *transactionNode) { if (!LogDistributedDeadlockDetection) { return; } StringInfo logMessage = makeStringInfo(); DistributedTransactionId *transactionId = &(transactionNode->transactionId); appendStringInfo(logMessage, "[DistributedTransactionId: (%d, " UINT64_FORMAT ", %s)] = ", transactionId->initiatorNodeIdentifier, transactionId->transactionNumber, timestamptz_to_str(transactionId->timestamp)); appendStringInfo(logMessage, "[WaitsFor transaction numbers: %s]", WaitsForToString(transactionNode->waitsFor)); /* log the backend query if the proc is associated with the transaction */ if (transactionNode->initiatorProc != NULL) { const char *backendQuery = pgstat_get_backend_current_activity(transactionNode->initiatorProc->pid, false); appendStringInfo(logMessage, "[Backend Query: %s]", backendQuery); } LogDistributedDeadlockDebugMessage(logMessage->data); } /* * LogDistributedDeadlockDebugMessage checks EnableDistributedDeadlockDebugging flag. If * it is true, the input message is sent to the logs with LOG level. Also, current timestamp * is prepanded to the message. */ static void LogDistributedDeadlockDebugMessage(const char *errorMessage) { if (!LogDistributedDeadlockDetection) { return; } ereport(LOG, (errmsg("[%s] %s", timestamptz_to_str(GetCurrentTimestamp()), errorMessage))); } /* * WaitsForToString is only intended for testing and debugging. It gets a * waitsForList and returns the list of transaction nodes' transactionNumber * in a string. */ char * WaitsForToString(List *waitsFor) { StringInfo transactionIdStr = makeStringInfo(); TransactionNode *waitingNode = NULL; foreach_declared_ptr(waitingNode, waitsFor) { if (transactionIdStr->len != 0) { appendStringInfoString(transactionIdStr, ","); } appendStringInfo(transactionIdStr, UINT64_FORMAT, waitingNode->transactionId.transactionNumber); } return transactionIdStr->data; } ================================================ FILE: src/backend/distributed/transaction/lock_graph.c ================================================ /*------------------------------------------------------------------------- * * lock_graph.c * * Functions for obtaining local and global lock graphs in which each * node is a distributed transaction, and an edge represent a waiting-for * relationship. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/hash.h" #include "storage/proc.h" #include "utils/builtins.h" #include "utils/hsearch.h" #include "utils/timestamp.h" #include "pg_version_compat.h" #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" /* * PROCStack is a stack of PGPROC pointers used to perform a depth-first search * through the lock graph. It also keeps track of which processes have been * added to the stack to avoid visiting the same process multiple times. */ typedef struct PROCStack { int procCount; PGPROC **procs; bool *procAdded; } PROCStack; static void AddWaitEdgeFromResult(WaitGraph *waitGraph, PGresult *result, int rowIndex); static void ReturnWaitGraph(WaitGraph *waitGraph, FunctionCallInfo fcinfo); static void AddWaitEdgeFromBlockedProcessResult(WaitGraph *waitGraph, PGresult *result, int rowIndex); static void ReturnBlockedProcessGraph(WaitGraph *waitGraph, FunctionCallInfo fcinfo); static WaitGraph * BuildLocalWaitGraph(bool onlyDistributedTx); static bool IsProcessWaitingForSafeOperations(PGPROC *proc); static void LockLockData(void); static void UnlockLockData(void); static void AddEdgesForLockWaits(WaitGraph *waitGraph, PGPROC *waitingProc, PROCStack *remaining); static void AddEdgesForWaitQueue(WaitGraph *waitGraph, PGPROC *waitingProc, PROCStack *remaining); static void AddWaitEdge(WaitGraph *waitGraph, PGPROC *waitingProc, PGPROC *blockingProc, PROCStack *remaining); static WaitEdge * AllocWaitEdge(WaitGraph *waitGraph); static void AddProcToVisit(PROCStack *remaining, PGPROC *proc); static bool IsSameLockGroup(PGPROC *leftProc, PGPROC *rightProc); static bool IsConflictingLockMask(int holdMask, int conflictMask); /* * We almost have 2 sets of identical functions. The first set (e.g., dump_wait_edges) * functions are intended for distributed deadlock detection purposes. * * The second set of functions (e.g., citus_internal_local_blocked_processes) are * intended for citus_lock_waits view. * * The main difference is that the former functions only show processes that are blocked * inside a distributed transaction (e.g., see AssignDistributedTransactionId()). * The latter functions return a superset, where any blocked process is returned. * * We kept two different set of functions for two purposes. First, the deadlock detection * is a performance critical code-path happening very frequently and we don't add any * performance overhead. Secondly, to be able to do rolling upgrades, we cannot change * the API of dump_global_wait_edges/dump_local_wait_edges such that they take a boolean * parameter. If we do that, until all nodes are upgraded, the deadlock detection would fail, * which is not acceptable. */ PG_FUNCTION_INFO_V1(dump_local_wait_edges); PG_FUNCTION_INFO_V1(dump_global_wait_edges); PG_FUNCTION_INFO_V1(citus_internal_local_blocked_processes); PG_FUNCTION_INFO_V1(citus_internal_global_blocked_processes); /* * dump_global_wait_edges returns global wait edges for distributed transactions * originating from the node on which it is started. */ Datum dump_global_wait_edges(PG_FUNCTION_ARGS) { bool onlyDistributedTx = true; WaitGraph *waitGraph = BuildGlobalWaitGraph(onlyDistributedTx); ReturnWaitGraph(waitGraph, fcinfo); return (Datum) 0; } /* * citus_internal_global_blocked_processes returns global wait edges * including all processes running on the cluster. */ Datum citus_internal_global_blocked_processes(PG_FUNCTION_ARGS) { bool onlyDistributedTx = false; WaitGraph *waitGraph = BuildGlobalWaitGraph(onlyDistributedTx); ReturnBlockedProcessGraph(waitGraph, fcinfo); return (Datum) 0; } /* * BuildGlobalWaitGraph builds a wait graph for distributed transactions * that originate from this node, including edges from all (other) worker * nodes. * * * If onlyDistributedTx is true, we only return distributed transactions * (e.g., AssignDistributedTransaction() or assign_distributed_transactions()) * has been called for the process. Distributed deadlock detection only * interested in these processes. */ WaitGraph * BuildGlobalWaitGraph(bool onlyDistributedTx) { List *workerNodeList = ActiveReadableNodeList(); char *nodeUser = CitusExtensionOwnerName(); List *connectionList = NIL; int32 localGroupId = GetLocalGroupId(); /* deadlock detection is only interested in distributed transactions */ WaitGraph *waitGraph = BuildLocalWaitGraph(onlyDistributedTx); /* open connections in parallel */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int connectionFlags = 0; if (workerNode->groupId == localGroupId) { /* we already have local wait edges */ continue; } MultiConnection *connection = StartNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, nodeUser, NULL); connectionList = lappend(connectionList, connection); } FinishConnectionListEstablishment(connectionList); /* send commands in parallel */ MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { StringInfo queryString = makeStringInfo(); if (onlyDistributedTx) { appendStringInfo(queryString, "SELECT waiting_pid, waiting_node_id, " "waiting_transaction_num, waiting_transaction_stamp, " "blocking_pid, blocking_node_id, blocking_transaction_num, " "blocking_transaction_stamp, blocking_transaction_waiting " "FROM dump_local_wait_edges()"); } else { appendStringInfo(queryString, "SELECT waiting_global_pid, waiting_pid, " "waiting_node_id, waiting_transaction_num, waiting_transaction_stamp, " "blocking_global_pid,blocking_pid, blocking_node_id, " "blocking_transaction_num, blocking_transaction_stamp, blocking_transaction_waiting " "FROM citus_internal.local_blocked_processes()"); } int querySent = SendRemoteCommand(connection, queryString->data); if (querySent == 0) { ReportConnectionError(connection, WARNING); } } /* receive dump_local_wait_edges results */ foreach_declared_ptr(connection, connectionList) { bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, WARNING); continue; } int64 rowCount = PQntuples(result); int64 colCount = PQnfields(result); if (onlyDistributedTx && colCount != 9) { ereport(WARNING, (errmsg("unexpected number of columns from " "dump_local_wait_edges"))); continue; } else if (!onlyDistributedTx && colCount != 11) { ereport(WARNING, (errmsg("unexpected number of columns from " "citus_internal.local_blocked_processes"))); continue; } for (int64 rowIndex = 0; rowIndex < rowCount; rowIndex++) { if (onlyDistributedTx) { AddWaitEdgeFromResult(waitGraph, result, rowIndex); } else { AddWaitEdgeFromBlockedProcessResult(waitGraph, result, rowIndex); } } PQclear(result); ForgetResults(connection); } return waitGraph; } /* * AddWaitEdgeFromResult adds an edge to the wait graph that is read from * a PGresult. */ static void AddWaitEdgeFromResult(WaitGraph *waitGraph, PGresult *result, int rowIndex) { WaitEdge *waitEdge = AllocWaitEdge(waitGraph); waitEdge->waitingGPid = 0; /* not requested for deadlock detection */ waitEdge->waitingPid = ParseIntField(result, rowIndex, 0); waitEdge->waitingNodeId = ParseIntField(result, rowIndex, 1); waitEdge->waitingTransactionNum = ParseIntField(result, rowIndex, 2); waitEdge->waitingTransactionStamp = ParseTimestampTzField(result, rowIndex, 3); waitEdge->blockingGPid = 0; /* not requested for deadlock detection */ waitEdge->blockingPid = ParseIntField(result, rowIndex, 4); waitEdge->blockingNodeId = ParseIntField(result, rowIndex, 5); waitEdge->blockingTransactionNum = ParseIntField(result, rowIndex, 6); waitEdge->blockingTransactionStamp = ParseTimestampTzField(result, rowIndex, 7); waitEdge->isBlockingXactWaiting = ParseBoolField(result, rowIndex, 8); } /* * AddWaitEdgeFromBlockedProcessResult adds an edge to the wait graph that * is read from a PGresult. */ static void AddWaitEdgeFromBlockedProcessResult(WaitGraph *waitGraph, PGresult *result, int rowIndex) { WaitEdge *waitEdge = AllocWaitEdge(waitGraph); waitEdge->waitingGPid = ParseIntField(result, rowIndex, 0); waitEdge->waitingPid = ParseIntField(result, rowIndex, 1); waitEdge->waitingNodeId = ParseIntField(result, rowIndex, 2); waitEdge->waitingTransactionNum = ParseIntField(result, rowIndex, 3); waitEdge->waitingTransactionStamp = ParseTimestampTzField(result, rowIndex, 4); waitEdge->blockingGPid = ParseIntField(result, rowIndex, 5); waitEdge->blockingPid = ParseIntField(result, rowIndex, 6); waitEdge->blockingNodeId = ParseIntField(result, rowIndex, 7); waitEdge->blockingTransactionNum = ParseIntField(result, rowIndex, 8); waitEdge->blockingTransactionStamp = ParseTimestampTzField(result, rowIndex, 9); waitEdge->isBlockingXactWaiting = ParseBoolField(result, rowIndex, 10); } /* * ParseIntField parses a int64 from a remote result or returns 0 if the * result is NULL. */ int64 ParseIntField(PGresult *result, int rowIndex, int colIndex) { if (PQgetisnull(result, rowIndex, colIndex)) { return 0; } char *resultString = PQgetvalue(result, rowIndex, colIndex); return strtou64(resultString, NULL, 10); } /* * ParseBoolField parses a bool from a remote result or returns false if the * result is NULL. */ bool ParseBoolField(PGresult *result, int rowIndex, int colIndex) { if (PQgetisnull(result, rowIndex, colIndex)) { return false; } char *resultString = PQgetvalue(result, rowIndex, colIndex); if (strlen(resultString) != 1) { return false; } return resultString[0] == 't'; } /* * ParseTimestampTzField parses a timestamptz from a remote result or returns * 0 if the result is NULL. */ TimestampTz ParseTimestampTzField(PGresult *result, int rowIndex, int colIndex) { if (PQgetisnull(result, rowIndex, colIndex)) { return DT_NOBEGIN; } char *resultString = PQgetvalue(result, rowIndex, colIndex); Datum resultStringDatum = CStringGetDatum(resultString); Datum timestampDatum = DirectFunctionCall3(timestamptz_in, resultStringDatum, 0, -1); return DatumGetTimestampTz(timestampDatum); } /* * dump_local_wait_edges returns wait edges for distributed transactions * running on the node on which it is called, which originate from the source node. */ Datum dump_local_wait_edges(PG_FUNCTION_ARGS) { bool onlyDistributedTx = true; WaitGraph *waitGraph = BuildLocalWaitGraph(onlyDistributedTx); ReturnWaitGraph(waitGraph, fcinfo); return (Datum) 0; } /* * citus_internal_local_blocked_processes returns global wait edges * including all processes running on the node. */ Datum citus_internal_local_blocked_processes(PG_FUNCTION_ARGS) { bool onlyDistributedTx = false; WaitGraph *waitGraph = BuildLocalWaitGraph(onlyDistributedTx); ReturnBlockedProcessGraph(waitGraph, fcinfo); return (Datum) 0; } /* * ReturnWaitGraph returns a wait graph for a set returning function. */ static void ReturnWaitGraph(WaitGraph *waitGraph, FunctionCallInfo fcinfo) { TupleDesc tupleDesc; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDesc); /* * Columns: * 00: waiting_pid * 01: waiting_node_id * 02: waiting_transaction_num * 03: waiting_transaction_stamp * 04: blocking_pid * 05: blocking__node_id * 06: blocking_transaction_num * 07: blocking_transaction_stamp * 08: blocking_transaction_waiting */ for (size_t curEdgeNum = 0; curEdgeNum < waitGraph->edgeCount; curEdgeNum++) { Datum values[9]; bool nulls[9]; WaitEdge *curEdge = &waitGraph->edges[curEdgeNum]; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = Int32GetDatum(curEdge->waitingPid); values[1] = Int32GetDatum(curEdge->waitingNodeId); if (curEdge->waitingTransactionNum != 0) { values[2] = Int64GetDatum(curEdge->waitingTransactionNum); values[3] = TimestampTzGetDatum(curEdge->waitingTransactionStamp); } else { nulls[2] = true; nulls[3] = true; } values[4] = Int32GetDatum(curEdge->blockingPid); values[5] = Int32GetDatum(curEdge->blockingNodeId); if (curEdge->blockingTransactionNum != 0) { values[6] = Int64GetDatum(curEdge->blockingTransactionNum); values[7] = TimestampTzGetDatum(curEdge->blockingTransactionStamp); } else { nulls[6] = true; nulls[7] = true; } values[8] = BoolGetDatum(curEdge->isBlockingXactWaiting); tuplestore_putvalues(tupleStore, tupleDesc, values, nulls); } } /* * ReturnBlockedProcessGraph returns a wait graph for a set returning function. */ static void ReturnBlockedProcessGraph(WaitGraph *waitGraph, FunctionCallInfo fcinfo) { TupleDesc tupleDesc; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDesc); /* * Columns: * 00: waiting_global_pid * 01: waiting_pid * 02: waiting_node_id * 03: waiting_transaction_num * 04: waiting_transaction_stamp * 05: blocking_global_pid * 06: blocking_pid * 07: blocking__node_id * 08: blocking_transaction_num * 09: blocking_transaction_stamp * 10: blocking_transaction_waiting */ for (size_t curEdgeNum = 0; curEdgeNum < waitGraph->edgeCount; curEdgeNum++) { Datum values[11]; bool nulls[11]; WaitEdge *curEdge = &waitGraph->edges[curEdgeNum]; memset(values, 0, sizeof(values)); memset(nulls, 0, sizeof(nulls)); values[0] = UInt64GetDatum(curEdge->waitingGPid); values[1] = Int32GetDatum(curEdge->waitingPid); values[2] = Int32GetDatum(curEdge->waitingNodeId); if (curEdge->waitingTransactionNum != 0) { values[3] = Int64GetDatum(curEdge->waitingTransactionNum); values[4] = TimestampTzGetDatum(curEdge->waitingTransactionStamp); } else { nulls[3] = true; nulls[4] = true; } values[5] = UInt64GetDatum(curEdge->blockingGPid); values[6] = Int32GetDatum(curEdge->blockingPid); values[7] = Int32GetDatum(curEdge->blockingNodeId); if (curEdge->blockingTransactionNum != 0) { values[8] = Int64GetDatum(curEdge->blockingTransactionNum); values[9] = TimestampTzGetDatum(curEdge->blockingTransactionStamp); } else { nulls[8] = true; nulls[9] = true; } values[10] = BoolGetDatum(curEdge->isBlockingXactWaiting); tuplestore_putvalues(tupleStore, tupleDesc, values, nulls); } } /* * BuildLocalWaitGraph builds a wait graph for distributed transactions * that originate from the local node. * * If onlyDistributedTx is true, we only return distributed transactions * (e.g., AssignDistributedTransaction() or assign_distributed_transactions()) * has been called for the process. Distributed deadlock detection only * interested in these processes. */ static WaitGraph * BuildLocalWaitGraph(bool onlyDistributedTx) { PROCStack remaining; int totalProcs = TotalProcCount(); /* * Try hard to avoid allocations while holding lock. Thus we pre-allocate * space for locks in large batches - for common scenarios this should be * more than enough space to build the list of wait edges without a single * allocation. */ WaitGraph *waitGraph = (WaitGraph *) palloc0(sizeof(WaitGraph)); waitGraph->localNodeId = GetLocalGroupId(); waitGraph->allocatedSize = totalProcs * 3; waitGraph->edgeCount = 0; waitGraph->edges = (WaitEdge *) palloc(waitGraph->allocatedSize * sizeof(WaitEdge)); remaining.procs = (PGPROC **) palloc(sizeof(PGPROC *) * totalProcs); remaining.procAdded = (bool *) palloc0(sizeof(bool *) * totalProcs); remaining.procCount = 0; LockLockData(); /* * Build lock-graph. We do so by first finding all procs which we are * interested in (in a distributed transaction, and blocked). Once * those are collected, do depth first search over all procs blocking * those. */ /* build list of starting procs */ for (int curBackend = 0; curBackend < totalProcs; curBackend++) { PGPROC *currentProc = GetPGProcByNumber(curBackend); BackendData currentBackendData; if (currentProc->pid == 0) { /* skip if the PGPROC slot is unused */ continue; } GetBackendDataForProc(currentProc, ¤tBackendData); if (!currentBackendData.activeBackend) { /* * Skip if the PGPROC slot is unused. We should normally use * IsBackendPid() to be able to skip reliably all the exited * processes. However, that is a costly operation. Instead, we * keep track of activeBackend in Citus code. */ continue; } /* * Only start searching from distributed transactions, since we only * care about distributed transactions for the purpose of distributed * deadlock detection. */ if (onlyDistributedTx && !IsInDistributedTransaction(¤tBackendData)) { continue; } /* skip if the process is not blocked */ if (!IsProcessWaitingForLock(currentProc)) { continue; } /* skip if the process is blocked for relation extension */ if (IsProcessWaitingForSafeOperations(currentProc)) { continue; } AddProcToVisit(&remaining, currentProc); } while (remaining.procCount > 0) { PGPROC *waitingProc = remaining.procs[--remaining.procCount]; /* only blocked processes result in wait edges */ if (!IsProcessWaitingForLock(waitingProc)) { continue; } /* skip if the process is blocked for relation extension */ if (IsProcessWaitingForSafeOperations(waitingProc)) { continue; } /* * Record an edge for everyone already holding the lock in a * conflicting manner ("hard edges" in postgres parlance). */ AddEdgesForLockWaits(waitGraph, waitingProc, &remaining); /* * Record an edge for everyone in front of us in the wait-queue * for the lock ("soft edges" in postgres parlance). */ AddEdgesForWaitQueue(waitGraph, waitingProc, &remaining); } UnlockLockData(); return waitGraph; } /* * IsProcessWaitingForSafeOperations returns true if the given PROC * waiting on relation extension locks, page locks or speculative locks. * * The function also returns true if the waiting process is an autovacuum * process given that autovacuum cannot contribute to any distributed * deadlocks. * * In general for the purpose of distributed deadlock detection, we should * skip if the process blocked on the locks that may not be part of deadlocks. * Those locks are held for a short duration while the relation or the index * is actually extended on the disk and released as soon as the extension is * done, even before the execution of the command that triggered the extension * finishes. Thus, recording such waits on our lock graphs could yield detecting * wrong distributed deadlocks. */ static bool IsProcessWaitingForSafeOperations(PGPROC *proc) { if (proc->waitStatus != PROC_WAIT_STATUS_WAITING) { return false; } if (proc->statusFlags & PROC_IS_AUTOVACUUM) { return true; } PROCLOCK *waitProcLock = proc->waitProcLock; LOCK *waitLock = waitProcLock->tag.myLock; return waitLock->tag.locktag_type == LOCKTAG_RELATION_EXTEND || waitLock->tag.locktag_type == LOCKTAG_PAGE || waitLock->tag.locktag_type == LOCKTAG_SPECULATIVE_TOKEN; } /* * LockLockData takes locks the shared lock data structure, which prevents * concurrent lock acquisitions/releases. * * The function also acquires lock on the backend shared memory to prevent * new backends to start. */ static void LockLockData(void) { LockBackendSharedMemory(LW_SHARED); for (int partitionNum = 0; partitionNum < NUM_LOCK_PARTITIONS; partitionNum++) { LWLockAcquire(LockHashPartitionLockByIndex(partitionNum), LW_SHARED); } } /* * UnlockLockData unlocks the locks on the shared lock data structure in reverse * order since LWLockRelease searches the given lock from the end of the * held_lwlocks array. * * The function also releases the shared memory lock to allow new backends to * start. */ static void UnlockLockData(void) { for (int partitionNum = NUM_LOCK_PARTITIONS - 1; partitionNum >= 0; partitionNum--) { LWLockRelease(LockHashPartitionLockByIndex(partitionNum)); } UnlockBackendSharedMemory(); } /* * AddEdgesForLockWaits adds an edge to the wait graph for every granted lock * that waitingProc is waiting for. * * This function iterates over the procLocks data structure in shared memory, * which also contains entries for locks which have not been granted yet, but * it does not reflect the order of the wait queue. We therefore handle the * wait queue separately. * * We have separate blocks for PG16 and waitLock; /* determine the conflict mask for the lock level used by the process */ LockMethod lockMethodTable = GetLocksMethodTable(waitLock); int conflictMask = lockMethodTable->conflictTab[waitingProc->waitLockMode]; /* iterate through the queue of processes holding the lock */ dlist_head *procLocks = &waitLock->procLocks; dlist_iter iter; dlist_foreach(iter, procLocks) { PROCLOCK *procLock = dlist_container(PROCLOCK, lockLink, iter.cur); PGPROC *currentProc = procLock->tag.myProc; /* * Skip processes from the same lock group, processes that don't conflict, * and processes that are waiting on safe operations. */ if (!IsSameLockGroup(waitingProc, currentProc) && IsConflictingLockMask(procLock->holdMask, conflictMask) && !IsProcessWaitingForSafeOperations(currentProc)) { AddWaitEdge(waitGraph, waitingProc, currentProc, remaining); } } } static void AddEdgesForWaitQueue(WaitGraph *waitGraph, PGPROC *waitingProc, PROCStack *remaining) { /* the lock for which this process is waiting */ LOCK *waitLock = waitingProc->waitLock; /* determine the conflict mask for the lock level used by the process */ LockMethod lockMethodTable = GetLocksMethodTable(waitLock); int conflictMask = lockMethodTable->conflictTab[waitingProc->waitLockMode]; /* iterate through the wait queue */ dclist_head *waitQueue = &waitLock->waitProcs; dlist_iter iter; dclist_foreach(iter, waitQueue) { PGPROC *currentProc = dlist_container(PGPROC, links, iter.cur); if (currentProc == waitingProc) { /* * Iterate through the queue from the start until we encounter waitingProc, * since we only care about processes in front of waitingProc in the queue. */ break; } int awaitMask = LOCKBIT_ON(currentProc->waitLockMode); /* * Skip processes from the same lock group, processes that don't conflict, * and processes that are waiting on safe operations. */ if (!IsSameLockGroup(waitingProc, currentProc) && IsConflictingLockMask(awaitMask, conflictMask) && !IsProcessWaitingForSafeOperations(currentProc)) { AddWaitEdge(waitGraph, waitingProc, currentProc, remaining); } currentProc = (PGPROC *) currentProc->links.next; } } /* * AddWaitEdge adds a new wait edge to a wait graph. The nodes in the graph are * transactions and an edge indicates the "waiting" process is blocked on a lock * held by the "blocking" process. * * If the blocking process is itself waiting then it is added to the remaining * stack. */ static void AddWaitEdge(WaitGraph *waitGraph, PGPROC *waitingProc, PGPROC *blockingProc, PROCStack *remaining) { WaitEdge *curEdge = AllocWaitEdge(waitGraph); BackendData waitingBackendData; BackendData blockingBackendData; GetBackendDataForProc(waitingProc, &waitingBackendData); GetBackendDataForProc(blockingProc, &blockingBackendData); curEdge->isBlockingXactWaiting = IsProcessWaitingForLock(blockingProc) && !IsProcessWaitingForSafeOperations(blockingProc); if (curEdge->isBlockingXactWaiting) { AddProcToVisit(remaining, blockingProc); } curEdge->waitingPid = waitingProc->pid; curEdge->waitingGPid = waitingBackendData.globalPID; if (IsInDistributedTransaction(&waitingBackendData)) { DistributedTransactionId *waitingTransactionId = &waitingBackendData.transactionId; curEdge->waitingNodeId = waitingTransactionId->initiatorNodeIdentifier; curEdge->waitingTransactionNum = waitingTransactionId->transactionNumber; curEdge->waitingTransactionStamp = waitingTransactionId->timestamp; } else { curEdge->waitingNodeId = waitGraph->localNodeId; curEdge->waitingTransactionNum = 0; curEdge->waitingTransactionStamp = 0; } curEdge->blockingPid = blockingProc->pid; curEdge->blockingGPid = blockingBackendData.globalPID; if (IsInDistributedTransaction(&blockingBackendData)) { DistributedTransactionId *blockingTransactionId = &blockingBackendData.transactionId; curEdge->blockingNodeId = blockingTransactionId->initiatorNodeIdentifier; curEdge->blockingTransactionNum = blockingTransactionId->transactionNumber; curEdge->blockingTransactionStamp = blockingTransactionId->timestamp; } else { curEdge->blockingNodeId = waitGraph->localNodeId; curEdge->blockingTransactionNum = 0; curEdge->blockingTransactionStamp = 0; } } /* * AllocWaitEdge allocates a wait edge as part of the given wait graph. * If the wait graph has insufficient space its size is doubled using * repalloc. */ static WaitEdge * AllocWaitEdge(WaitGraph *waitGraph) { /* ensure space for new edge */ if (waitGraph->allocatedSize == waitGraph->edgeCount) { waitGraph->allocatedSize *= 2; waitGraph->edges = (WaitEdge *) repalloc(waitGraph->edges, sizeof(WaitEdge) * waitGraph->allocatedSize); } return &waitGraph->edges[waitGraph->edgeCount++]; } /* * AddProcToVisit adds a process to the stack of processes to visit * in the depth-first search, unless it was already added. */ static void AddProcToVisit(PROCStack *remaining, PGPROC *proc) { if (remaining->procAdded[getProcNo_compat(proc)]) { return; } Assert(remaining->procCount < TotalProcCount()); remaining->procs[remaining->procCount++] = proc; remaining->procAdded[getProcNo_compat(proc)] = true; } /* * IsProcessWaitingForLock returns whether a given process is waiting for a lock. */ bool IsProcessWaitingForLock(PGPROC *proc) { return proc->waitStatus == PROC_WAIT_STATUS_WAITING; } /* * IsSameLockGroup returns whether two processes are part of the same lock group, * meaning they are either the same process, or have the same lock group leader. */ static bool IsSameLockGroup(PGPROC *leftProc, PGPROC *rightProc) { return leftProc == rightProc || (leftProc->lockGroupLeader != NULL && leftProc->lockGroupLeader == rightProc->lockGroupLeader); } /* * IsConflictingLockMask returns whether the given conflict mask conflicts with the * holdMask. * * holdMask is a bitmask with the i-th bit turned on if a lock mode i is held. * * conflictMask is a bitmask with the j-th bit turned on if it conflicts with * lock mode i. */ static bool IsConflictingLockMask(int holdMask, int conflictMask) { return (holdMask & conflictMask) != 0; } /* * IsInDistributedTransaction returns whether the given backend is in a * distributed transaction. */ bool IsInDistributedTransaction(BackendData *backendData) { return backendData->transactionId.transactionNumber != 0; } ================================================ FILE: src/backend/distributed/transaction/relation_access_tracking.c ================================================ /*------------------------------------------------------------------------- * * relation_access_tracking.c * * Transaction access tracking for Citus. The functions in this file * are intended to track the relation accesses within a transaction. The * logic here is mostly useful when a reference table is referred by * a distributed table via a foreign key. Whenever such a pair of tables * are accessed inside a transaction, Citus should detect and act * accordingly. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/xact.h" #include "common/hashfn.h" #include "utils/hsearch.h" #include "utils/lsyscache.h" #include "utils/memutils.h" /* for ALLOCSET_DEFAULT_MINSIZE, _INITSIZE, _MAXSIZE */ #include "pg_version_constants.h" #include "distributed/colocation_utils.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/relation_access_tracking.h" /* Config variables managed via guc.c */ bool EnforceForeignKeyRestrictions = true; #define PARALLEL_MODE_FLAG_OFFSET 3 /* simply set parallel bits as defined below for select, dml and ddl */ #define PARALLEL_ACCESS_MASK (int) (0 | \ (1 << (PLACEMENT_ACCESS_SELECT + \ PARALLEL_MODE_FLAG_OFFSET)) | \ (1 << (PLACEMENT_ACCESS_DML + \ PARALLEL_MODE_FLAG_OFFSET)) | \ (1 << (PLACEMENT_ACCESS_DDL + \ PARALLEL_MODE_FLAG_OFFSET))) MemoryContext RelationAcessContext = NULL; /* * Hash table mapping relations to the * (relationId) = (relationAccessType and relationAccessMode) * * RelationAccessHash is used to keep track of relation accesses types (e.g., select, * dml or ddl) along with access modes (e.g., no access, sequential access or * parallel access). * * We keep an integer per relation and use some of the bits to identify the access types * and access modes. * * We store the access types in the first 3 bits: * - 0th bit is set for SELECT accesses to a relation * - 1st bit is set for DML accesses to a relation * - 2nd bit is set for DDL accesses to a relation * * and, access modes in the next 3 bits: * - 3rd bit is set for PARALLEL SELECT accesses to a relation * - 4th bit is set for PARALLEL DML accesses to a relation * - 5th bit is set for PARALLEL DDL accesses to a relation * */ typedef struct RelationAccessHashKey { Oid relationId; } RelationAccessHashKey; typedef struct RelationAccessHashEntry { RelationAccessHashKey key; int relationAccessMode; } RelationAccessHashEntry; static HTAB *RelationAccessHash; /* functions related to access recording */ static void AllocateRelationAccessHash(void); static void RecordRelationAccessBase(Oid relationId, ShardPlacementAccessType accessType); static void RecordPlacementAccessToCache(Oid relationId, ShardPlacementAccessType accessType); static void RecordRelationParallelSelectAccessForTask(Task *task); static void RecordRelationParallelModifyAccessForTask(Task *task); static void RecordRelationParallelDDLAccessForTask(Task *task); static RelationAccessMode GetRelationAccessMode(Oid relationId, ShardPlacementAccessType accessType); static void RecordParallelRelationAccess(Oid relationId, ShardPlacementAccessType placementAccess); static void RecordParallelRelationAccessToCache(Oid relationId, ShardPlacementAccessType placementAccess); /* functions related to access conflict checks */ static char * PlacementAccessTypeToText(ShardPlacementAccessType accessType); static void CheckConflictingRelationAccesses(Oid relationId, ShardPlacementAccessType accessType); static bool HoldsConflictingLockWithReferencingRelations(Oid relationId, ShardPlacementAccessType placementAccess, Oid *conflictingRelationId, ShardPlacementAccessType * conflictingAccessMode); static void CheckConflictingParallelRelationAccesses(Oid relationId, ShardPlacementAccessType accessType); static bool HoldsConflictingLockWithReferencedRelations(Oid relationId, ShardPlacementAccessType placementAccess, Oid *conflictingRelationId, ShardPlacementAccessType * conflictingAccessMode); /* * InitRelationAccessHash performs initialization of the * infrastructure in this file at backend start. */ void InitRelationAccessHash(void) { /* allocate (relationId) = [relationAccessMode] hash */ AllocateRelationAccessHash(); } /* * Empty RelationAccessHash, without destroying the hash table itself. */ void ResetRelationAccessHash() { hash_delete_all(RelationAccessHash); } /* * Allocate RelationAccessHash. */ static void AllocateRelationAccessHash(void) { /* * Create a single context for relation access related memory * management. Doing so, instead of allocating in TopMemoryContext, makes * it easier to associate used memory. */ RelationAcessContext = AllocSetContextCreateInternal(TopMemoryContext, "Relation Access Context", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(RelationAccessHashKey); info.entrysize = sizeof(RelationAccessHashEntry); info.hash = tag_hash; info.hcxt = RelationAcessContext; uint32 hashFlags = (HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); RelationAccessHash = hash_create("citus relation access cache (relationid)", 8, &info, hashFlags); } /* * RecordRelationAccessIfNonDistTable marks the relation accessed if it is a * reference relation. * * The function is a wrapper around RecordRelationAccessBase(). */ void RecordRelationAccessIfNonDistTable(Oid relationId, ShardPlacementAccessType accessType) { if (!ShouldRecordRelationAccess()) { return; } /* * We keep track of relation accesses for the purposes of foreign keys to * reference tables. So, other distributed tables are not relevant for now. * Additionally, partitioned tables with lots of partitions might require * recursively calling RecordRelationAccessBase(), so becareful about * removing this check. */ if (IsCitusTableType(relationId, DISTRIBUTED_TABLE)) { return; } RecordRelationAccessBase(relationId, accessType); } /* * PlacementAccessTypeToText converts ShardPlacementAccessType to * text representation. */ static char * PlacementAccessTypeToText(ShardPlacementAccessType accessType) { switch (accessType) { case PLACEMENT_ACCESS_SELECT: { return "SELECT"; break; } case PLACEMENT_ACCESS_DML: { return "DML"; } case PLACEMENT_ACCESS_DDL: { return "DDL"; } default: { return "None"; break; } } } /* * RecordRelationAccessBase associates the access to the distributed relation. The * function takes partitioned relations into account as well. * * We implemented this function to prevent accessing placement metadata during * recursive calls of the function itself (e.g., avoid * RecordRelationAccessBase()). */ static void RecordRelationAccessBase(Oid relationId, ShardPlacementAccessType accessType) { if (IsCitusTableType(relationId, REFERENCE_TABLE)) { /* * We don't support partitioned reference tables. */ Assert(!PartitionedTable(relationId) && !PartitionTable(relationId)); } /* make sure that this is not a conflicting access */ CheckConflictingRelationAccesses(relationId, accessType); /* always record the relation that is being considered */ RecordPlacementAccessToCache(relationId, accessType); } /* * RecordPlacementAccessToCache is a utility function which saves the given * relation id's access to the RelationAccessHash. */ static void RecordPlacementAccessToCache(Oid relationId, ShardPlacementAccessType accessType) { RelationAccessHashKey hashKey; bool found = false; hashKey.relationId = relationId; RelationAccessHashEntry *hashEntry = hash_search(RelationAccessHash, &hashKey, HASH_ENTER, &found); if (!found) { hashEntry->relationAccessMode = 0; } /* set the bit representing the access type */ hashEntry->relationAccessMode |= (1 << (accessType)); } /* * RecordParallelRelationAccessForTaskList gets a task list and records * the necessary parallel relation accesses for the task list. * * This function is used to enforce foreign keys from distributed * tables to reference tables. */ void RecordParallelRelationAccessForTaskList(List *taskList) { if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { /* sequential mode prevents parallel access */ return; } if (list_length(taskList) < 2) { /* single shard task doesn't mean parallel access in our definition */ return; } /* * Since all the tasks in a task list is expected to operate on the same * distributed table(s), we only need to process the first task. */ Task *firstTask = linitial(taskList); if (firstTask->taskType == READ_TASK) { RecordRelationParallelSelectAccessForTask(firstTask); } else if (firstTask->taskType == MODIFY_TASK) { if (firstTask->rowValuesLists != NIL) { /* * We always run multi-row INSERTs in a sequential * mode (hard-coded). Thus, we do not mark as parallel * access even if the prerequisites hold. */ } else { /* * We prefer to mark with all remaining multi-shard modifications * with both modify and select accesses. */ RecordRelationParallelModifyAccessForTask(firstTask); RecordRelationParallelSelectAccessForTask(firstTask); } } else { RecordRelationParallelDDLAccessForTask(firstTask); } } /* * RecordRelationParallelSelectAccessForTask goes over all the relations * in the relationShardList and records the select access per each table. */ static void RecordRelationParallelSelectAccessForTask(Task *task) { Oid lastRelationId = InvalidOid; /* no point in recoding accesses in non-transaction blocks, skip the loop */ if (!ShouldRecordRelationAccess()) { return; } List *relationShardList = task->relationShardList; RelationShard *relationShard = NULL; foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; /* * An optimization, skip going to hash table if we've already * recorded the relation. */ if (currentRelationId == lastRelationId) { continue; } RecordParallelSelectAccess(currentRelationId); lastRelationId = currentRelationId; } } /* * RecordRelationParallelModifyAccessForTask gets a task and records * the accesses. Note that the target relation is recorded with modify access * where as the subqueries inside the modify query is recorded with select * access. */ static void RecordRelationParallelModifyAccessForTask(Task *task) { List *relationShardList = NULL; Oid lastRelationId = InvalidOid; /* no point in recoding accesses in non-transaction blocks, skip the loop */ if (!ShouldRecordRelationAccess()) { return; } /* anchor shard is always associated with modify access */ RecordParallelModifyAccess(RelationIdForShard(task->anchorShardId)); if (task->modifyWithSubquery) { relationShardList = task->relationShardList; RelationShard *relationShard = NULL; foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; /* * An optimization, skip going to hash table if we've already * recorded the relation. */ if (currentRelationId == lastRelationId) { continue; } RecordParallelSelectAccess(currentRelationId); lastRelationId = currentRelationId; } } } /* * RecordRelationParallelDDLAccessForTask marks all the relationShards * with parallel DDL access if exists. That case is valid for inter-shard * DDL commands such as foreign key creation. The function also records * the relation that anchorShardId belongs to. */ static void RecordRelationParallelDDLAccessForTask(Task *task) { List *relationShardList = task->relationShardList; Oid lastRelationId = InvalidOid; RelationShard *relationShard = NULL; foreach_declared_ptr(relationShard, relationShardList) { Oid currentRelationId = relationShard->relationId; /* * An optimization, skip going to hash table if we've already * recorded the relation. */ if (currentRelationId == lastRelationId) { continue; } RecordParallelDDLAccess(currentRelationId); lastRelationId = currentRelationId; } if (task->anchorShardId != INVALID_SHARD_ID) { RecordParallelDDLAccess(RelationIdForShard(task->anchorShardId)); } } /* * RecordParallelSelectAccess is a wrapper around RecordParallelRelationAccess() */ void RecordParallelSelectAccess(Oid relationId) { RecordParallelRelationAccess(relationId, PLACEMENT_ACCESS_SELECT); } /* * RecordParallelModifyAccess is a wrapper around RecordParallelRelationAccess() */ void RecordParallelModifyAccess(Oid relationId) { RecordParallelRelationAccess(relationId, PLACEMENT_ACCESS_DML); } /* * RecordParallelDDLAccess is a wrapper around RecordParallelRelationAccess() */ void RecordParallelDDLAccess(Oid relationId) { RecordParallelRelationAccess(relationId, PLACEMENT_ACCESS_DDL); } /* * RecordParallelRelationAccess records the relation access mode as parallel * for the given access type (e.g., select, dml or ddl) in the RelationAccessHash. * * The function also takes partitions and partitioned tables into account. */ static void RecordParallelRelationAccess(Oid relationId, ShardPlacementAccessType placementAccess) { if (!ShouldRecordRelationAccess()) { return; } /* act accordingly if it's a conflicting access */ CheckConflictingParallelRelationAccesses(relationId, placementAccess); /* * CheckConflictingParallelRelationAccesses might switch to sequential * execution. If that's the case, no need to continue because the executor * would take the necessary actions to switch to sequential execution * immediately. */ if (MultiShardConnectionType == SEQUENTIAL_CONNECTION) { return; } /* If a relation is partitioned, record accesses to all of its partitions as well. */ if (PartitionedTable(relationId)) { List *partitionList = PartitionList(relationId); Oid partitionOid = InvalidOid; foreach_declared_oid(partitionOid, partitionList) { /* recursively record all relation accesses of its partitions */ RecordParallelRelationAccess(partitionOid, placementAccess); } } else if (PartitionTable(relationId)) { Oid parentOid = PartitionParentOid(relationId); /* only record the parent */ RecordParallelRelationAccessToCache(parentOid, placementAccess); } RecordParallelRelationAccessToCache(relationId, placementAccess); } /* * RecordParallelRelationAccessToCache is a utility function which saves the given * relation id's access to the RelationAccessHash. */ static void RecordParallelRelationAccessToCache(Oid relationId, ShardPlacementAccessType placementAccess) { RelationAccessHashKey hashKey; bool found = false; hashKey.relationId = relationId; RelationAccessHashEntry *hashEntry = hash_search(RelationAccessHash, &hashKey, HASH_ENTER, &found); if (!found) { hashEntry->relationAccessMode = 0; } /* set the bit representing the access type */ hashEntry->relationAccessMode |= (1 << (placementAccess)); /* set the bit representing access mode */ int parallelRelationAccessBit = placementAccess + PARALLEL_MODE_FLAG_OFFSET; hashEntry->relationAccessMode |= (1 << parallelRelationAccessBit); } /* * ParallelQueryExecutedInTransaction returns true if any parallel query * is executed in the current transaction. */ bool ParallelQueryExecutedInTransaction(void) { HASH_SEQ_STATUS status; if (!ShouldRecordRelationAccess() || RelationAccessHash == NULL) { return false; } hash_seq_init(&status, RelationAccessHash); RelationAccessHashEntry *hashEntry = (RelationAccessHashEntry *) hash_seq_search( &status); while (hashEntry != NULL) { int relationAccessMode = hashEntry->relationAccessMode; if ((relationAccessMode & PARALLEL_ACCESS_MASK)) { hash_seq_term(&status); return true; } hashEntry = (RelationAccessHashEntry *) hash_seq_search(&status); } return false; } /* * GetRelationSelectAccessMode is a wrapper around GetRelationAccessMode. */ RelationAccessMode GetRelationSelectAccessMode(Oid relationId) { return GetRelationAccessMode(relationId, PLACEMENT_ACCESS_SELECT); } /* * GetRelationDMLAccessMode is a wrapper around GetRelationAccessMode. */ RelationAccessMode GetRelationDMLAccessMode(Oid relationId) { return GetRelationAccessMode(relationId, PLACEMENT_ACCESS_DML); } /* * GetRelationDDLAccessMode is a wrapper around GetRelationAccessMode. */ RelationAccessMode GetRelationDDLAccessMode(Oid relationId) { return GetRelationAccessMode(relationId, PLACEMENT_ACCESS_DDL); } /* * GetRelationAccessMode returns the relation access mode (e.g., none, sequential * or parallel) for the given access type (e.g., select, dml or ddl). */ static RelationAccessMode GetRelationAccessMode(Oid relationId, ShardPlacementAccessType accessType) { RelationAccessHashKey hashKey; bool found = false; int parallelRelationAccessBit = accessType + PARALLEL_MODE_FLAG_OFFSET; /* no point in getting the mode when not inside a transaction block */ if (!ShouldRecordRelationAccess()) { return RELATION_NOT_ACCESSED; } hashKey.relationId = relationId; RelationAccessHashEntry *hashEntry = hash_search(RelationAccessHash, &hashKey, HASH_FIND, &found); if (!found) { /* relation not accessed at all */ return RELATION_NOT_ACCESSED; } int relationAcessMode = hashEntry->relationAccessMode; if (!(relationAcessMode & (1 << accessType))) { /* relation not accessed with the given access type */ return RELATION_NOT_ACCESSED; } if (relationAcessMode & (1 << parallelRelationAccessBit)) { return RELATION_PARALLEL_ACCESSED; } else { return RELATION_REFERENCE_ACCESSED; } } /* * ShouldRecordRelationAccess returns true when we should keep track * of the relation accesses. * * In many cases, we'd only need IsMultiStatementTransaction(), however, for some * cases such as CTEs, where Citus uses the same connections across multiple queries, * we should still record the relation accesses even not inside an explicit transaction * block. Thus, keeping track of the relation accesses inside coordinated transactions * is also required. */ bool ShouldRecordRelationAccess() { if (EnforceForeignKeyRestrictions && (IsMultiStatementTransaction() || InCoordinatedTransaction())) { return true; } return false; } /* * CheckConflictingRelationAccesses is mostly a wrapper around * HoldsConflictingLockWithReferencingRelations(). We're only interested in * accesses to reference tables and citus local tables that are referenced via * a foreign constraint by a hash distributed table. */ static void CheckConflictingRelationAccesses(Oid relationId, ShardPlacementAccessType accessType) { Oid conflictingReferencingRelationId = InvalidOid; ShardPlacementAccessType conflictingAccessType = PLACEMENT_ACCESS_SELECT; if (!EnforceForeignKeyRestrictions || !IsCitusTable(relationId)) { return; } CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE) || cacheEntry->referencingRelationsViaForeignKey == NIL) { return; } if (HoldsConflictingLockWithReferencingRelations(relationId, accessType, &conflictingReferencingRelationId, &conflictingAccessType)) { char *relationName = get_rel_name(relationId); char *conflictingRelationName = get_rel_name(conflictingReferencingRelationId); char *accessTypeText = PlacementAccessTypeToText(accessType); char *conflictingAccessTypeText = PlacementAccessTypeToText(conflictingAccessType); /* * Relation could already be dropped if the accessType is DDL and the * command that we were executing were a DROP command. In that case, * as this function is executed via DROP trigger, standard_ProcessUtility * had already dropped the table from PostgreSQL's perspective. Hence, it * returns NULL pointer for the name of the relation. */ if (relationName == NULL) { ereport(ERROR, (errmsg("cannot execute %s on table because there was " "a parallel %s access to distributed table " "\"%s\" in the same transaction", accessTypeText, conflictingAccessTypeText, conflictingRelationName), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else { ereport(ERROR, (errmsg("cannot execute %s on table \"%s\" because " "there was a parallel %s access to distributed " "table \"%s\" in the same transaction", accessTypeText, relationName, conflictingAccessTypeText, conflictingRelationName), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } } else if (cacheEntry->referencingRelationsViaForeignKey != NIL && accessType > PLACEMENT_ACCESS_SELECT) { char *relationName = get_rel_name(relationId); if (ParallelQueryExecutedInTransaction()) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg("cannot modify table \"%s\" because there was " "a parallel operation on a distributed table", relationName), errdetail("When there is a foreign key to a reference " "table or to a local table, Citus needs " "to perform all operations over a single " "connection per node to ensure consistency."), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else if (MultiShardConnectionType == PARALLEL_CONNECTION) { /* * We can still continue with multi-shard queries in sequential mode, so * set it. */ ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail( "Table \"%s\" is modified, which might lead to data " "inconsistencies or distributed deadlocks via " "parallel accesses to hash distributed tables due to " "foreign keys. Any parallel modification to " "those hash distributed tables in the same " "transaction can only be executed in sequential query " "execution mode", relationName != NULL ? relationName : ""))); /* * Switching to sequential mode is admittedly confusing and, could be useless * and less performant in some cases. However, if we do not switch to * sequential mode at this point, we'd lose the opportunity to do so * later when a parallel query is executed on the hash distributed relations * that are referencing this reference table. */ SetLocalMultiShardModifyModeToSequential(); } } } /* * CheckConflictingParallelRelationAccesses is mostly a wrapper around * HoldsConflictingLockWithReferencedRelations(). We're only interested in parallel * accesses to distributed tables that refers reference tables via foreign constraint. * */ static void CheckConflictingParallelRelationAccesses(Oid relationId, ShardPlacementAccessType accessType) { Oid conflictingReferencingRelationId = InvalidOid; ShardPlacementAccessType conflictingAccessType = PLACEMENT_ACCESS_SELECT; if (!EnforceForeignKeyRestrictions || !IsCitusTable(relationId)) { return; } CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (!(IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) && cacheEntry->referencedRelationsViaForeignKey != NIL)) { return; } if (MultiShardConnectionType == PARALLEL_CONNECTION && HoldsConflictingLockWithReferencedRelations(relationId, accessType, &conflictingReferencingRelationId, &conflictingAccessType)) { char *relationName = get_rel_name(relationId); char *conflictingRelationName = get_rel_name(conflictingReferencingRelationId); char *accessTypeText = PlacementAccessTypeToText(accessType); char *conflictingAccessTypeText = PlacementAccessTypeToText(conflictingAccessType); if (ParallelQueryExecutedInTransaction()) { /* * If there has already been a parallel query executed, the sequential mode * would still use the already opened parallel connections to the workers, * thus contradicting our purpose of using sequential mode. */ ereport(ERROR, (errmsg("cannot execute parallel %s on table \"%s\" " "after %s command on reference table " "\"%s\" because there is a foreign key between " "them and \"%s\" has been accessed in this transaction", accessTypeText, relationName, conflictingAccessTypeText, conflictingRelationName, conflictingRelationName), errdetail("When there is a foreign key to a reference " "table, Citus needs to perform all operations " "over a single connection per node to ensure " "consistency."), errhint("Try re-running the transaction with " "\"SET LOCAL citus.multi_shard_modify_mode TO " "\'sequential\';\""))); } else { ereport(DEBUG1, (errmsg("switching to sequential query execution mode"), errdetail("cannot execute parallel %s on table \"%s\" " "after %s command on reference table " "\"%s\" because there is a foreign key between " "them and \"%s\" has been accessed in this transaction", accessTypeText, relationName, conflictingAccessTypeText, conflictingRelationName, conflictingRelationName))); SetLocalMultiShardModifyModeToSequential(); } } } /* * HoldsConflictingLockWithReferencedRelations returns true if the input relationId is a * hash distributed table and it holds any conflicting locks with the reference tables that * the distributed table has a foreign key to the reference table. */ static bool HoldsConflictingLockWithReferencedRelations(Oid relationId, ShardPlacementAccessType placementAccess, Oid *conflictingRelationId, ShardPlacementAccessType * conflictingAccessMode) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); Oid referencedRelation = InvalidOid; foreach_declared_oid(referencedRelation, cacheEntry->referencedRelationsViaForeignKey) { /* * We're only interested in foreign keys to reference tables and citus * local tables. */ if (IsCitusTableType(referencedRelation, DISTRIBUTED_TABLE)) { continue; } /* * A select on a reference table could conflict with a DDL * on a distributed table. */ RelationAccessMode selectMode = GetRelationSelectAccessMode(referencedRelation); if (placementAccess == PLACEMENT_ACCESS_DDL && selectMode != RELATION_NOT_ACCESSED) { *conflictingRelationId = referencedRelation; *conflictingAccessMode = PLACEMENT_ACCESS_SELECT; return true; } /* * Both DML and DDL operations on a reference table conflicts with * any parallel operation on distributed tables. */ RelationAccessMode dmlMode = GetRelationDMLAccessMode(referencedRelation); if (dmlMode != RELATION_NOT_ACCESSED) { *conflictingRelationId = referencedRelation; *conflictingAccessMode = PLACEMENT_ACCESS_DML; return true; } RelationAccessMode ddlMode = GetRelationDDLAccessMode(referencedRelation); if (ddlMode != RELATION_NOT_ACCESSED) { *conflictingRelationId = referencedRelation; *conflictingAccessMode = PLACEMENT_ACCESS_DDL; return true; } } return false; } /* * HoldsConflictingLockWithReferencingRelations returns true when the input relationId is a * reference table and it holds any conflicting locks with the distributed tables where * the distributed table has a foreign key to the reference table. * * If returns true, the referencing relation and conflictingAccessMode are also set. */ static bool HoldsConflictingLockWithReferencingRelations(Oid relationId, ShardPlacementAccessType placementAccess, Oid *conflictingRelationId, ShardPlacementAccessType * conflictingAccessMode) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); bool holdsConflictingLocks = false; Assert(!IsCitusTableTypeCacheEntry(cacheEntry, DISTRIBUTED_TABLE)); Oid referencingRelation = InvalidOid; foreach_declared_oid(referencingRelation, cacheEntry->referencingRelationsViaForeignKey) { /* * We're only interested in foreign keys to reference tables from * hash distributed tables. */ if (!IsCitusTableType(referencingRelation, HASH_DISTRIBUTED)) { continue; } /* * Rules that we apply: * - SELECT on a reference might table conflict with * a previous parallel DDL on a distributed table * - DML on a reference table might conflict with * a previous parallel DML or DDL on a distributed * table * - DDL on a reference table might conflict with * a parellel SELECT, DML or DDL on a distributed * table */ if (placementAccess == PLACEMENT_ACCESS_SELECT) { RelationAccessMode ddlMode = GetRelationDDLAccessMode(referencingRelation); if (ddlMode == RELATION_PARALLEL_ACCESSED) { /* SELECT on a distributed table conflicts with DDL / TRUNCATE */ holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_DDL; } } else if (placementAccess == PLACEMENT_ACCESS_DML) { RelationAccessMode dmlMode = GetRelationDMLAccessMode(referencingRelation); if (dmlMode == RELATION_PARALLEL_ACCESSED) { holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_DML; } RelationAccessMode ddlMode = GetRelationDDLAccessMode(referencingRelation); if (ddlMode == RELATION_PARALLEL_ACCESSED) { /* SELECT on a distributed table conflicts with DDL / TRUNCATE */ holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_DDL; } } else if (placementAccess == PLACEMENT_ACCESS_DDL) { RelationAccessMode selectMode = GetRelationSelectAccessMode( referencingRelation); if (selectMode == RELATION_PARALLEL_ACCESSED) { holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_SELECT; } RelationAccessMode dmlMode = GetRelationDMLAccessMode(referencingRelation); if (dmlMode == RELATION_PARALLEL_ACCESSED) { holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_DML; } RelationAccessMode ddlMode = GetRelationDDLAccessMode(referencingRelation); if (ddlMode == RELATION_PARALLEL_ACCESSED) { holdsConflictingLocks = true; *conflictingAccessMode = PLACEMENT_ACCESS_DDL; } } if (holdsConflictingLocks) { *conflictingRelationId = referencingRelation; return true; } } return false; } ================================================ FILE: src/backend/distributed/transaction/remote_transaction.c ================================================ /*------------------------------------------------------------------------- * * remote_transaction.c * Management of transaction spanning more than one node. * * Since the functions defined in this file mostly allocate in * CitusXactCallbackContext, we mostly try doing allocations on stack. * And when it's hard to do so, we at least try freeing the heap memory * immediately after an object becomes useless. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/xact.h" #include "postmaster/postmaster.h" #include "utils/builtins.h" #include "utils/hsearch.h" #include "utils/xid8.h" #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/placement_connection.h" #include "distributed/remote_commands.h" #include "distributed/remote_transaction.h" #include "distributed/transaction_identifier.h" #include "distributed/transaction_management.h" #include "distributed/transaction_recovery.h" #include "distributed/worker_manager.h" #define PREPARED_TRANSACTION_NAME_FORMAT "citus_%u_%u_"UINT64_FORMAT "_%u" static char * BeginTransactionCommand(void); static char * AssignDistributedTransactionIdCommand(void); static void StartRemoteTransactionSavepointBegin(MultiConnection *connection, SubTransactionId subId); static void FinishRemoteTransactionSavepointBegin(MultiConnection *connection, SubTransactionId subId); static void StartRemoteTransactionSavepointRelease(MultiConnection *connection, SubTransactionId subId); static void FinishRemoteTransactionSavepointRelease(MultiConnection *connection, SubTransactionId subId); static void StartRemoteTransactionSavepointRollback(MultiConnection *connection, SubTransactionId subId); static void FinishRemoteTransactionSavepointRollback(MultiConnection *connection, SubTransactionId subId); static void Assign2PCIdentifier(MultiConnection *connection); PG_FUNCTION_INFO_V1(start_management_transaction); PG_FUNCTION_INFO_V1(execute_command_on_remote_nodes_as_user); PG_FUNCTION_INFO_V1(commit_management_command_2pc); static char *IsolationLevelName[] = { "READ UNCOMMITTED", "READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE" }; /* * These variables are necessary for running queries from a database that is not * the Citus main database. Some of these queries need to be propagated to the * workers and Citus main database will be used for these queries, such as * CREATE ROLE. For that we create a connection to the Citus main database and * run queries from there. */ /* The MultiConnection used for connecting Citus main database. */ MultiConnection *MainDBConnection = NULL; /* * IsMainDBCommand is true if this is a query in the Citus main database that is started * by a query from a different database. */ bool IsMainDBCommand = false; /* * The transaction id of the query from the other database that started the * main database query. */ FullTransactionId OuterXid; /* * Shows if this is the Citus main database or not. We needed a variable instead of * checking if this database's name is the same as MainDb because we sometimes need * this value outside a transaction where we cannot reach the current database name. */ bool IsMainDB = true; /* * Name of a superuser role to be used during main database connections. */ char *SuperuserRole = NULL; /* * IsMainDBCommandInXact shows if the query sent to the main database requires * a transaction */ bool IsMainDBCommandInXact = true; /* * start_management_transaction starts a management transaction * in the main database by recording the outer transaction's transaction id and setting * IsMainDBCommand to true. */ Datum start_management_transaction(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); OuterXid = PG_GETARG_FULLTRANSACTIONID(0); IsMainDBCommand = true; Use2PCForCoordinatedTransaction(); PG_RETURN_VOID(); } /* * execute_command_on_remote_nodes_as_user executes the query on the nodes * other than the current node, using the user passed. */ Datum execute_command_on_remote_nodes_as_user(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); text *queryText = PG_GETARG_TEXT_P(0); char *query = text_to_cstring(queryText); text *usernameText = PG_GETARG_TEXT_P(1); char *username = text_to_cstring(usernameText); StringInfo queryToSend = makeStringInfo(); appendStringInfo(queryToSend, "%s;%s;%s", DISABLE_METADATA_SYNC, query, ENABLE_METADATA_SYNC); SendCommandToWorkersAsUser(REMOTE_NODES, username, queryToSend->data); PG_RETURN_VOID(); } /* * commit_management_command_2pc is a wrapper UDF for * CoordinatedRemoteTransactionsCommit */ Datum commit_management_command_2pc(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureSuperUser(); RecoverTwoPhaseCommits(); PG_RETURN_VOID(); } /* * RunCitusMainDBQuery creates a connection to Citus main database if necessary * and runs the query over the connection in the main database. */ void RunCitusMainDBQuery(char *query) { if (MainDBConnection == NULL) { if (strlen(SuperuserRole) == 0) { ereport(ERROR, (errmsg("No superuser role is given for Citus main " "database connection"), errhint("Set citus.superuser to a superuser role name"))); } int flags = 0; MainDBConnection = GetNodeUserDatabaseConnection(flags, LocalHostName, PostPortNumber, SuperuserRole, MainDb); if (IsMainDBCommandInXact) { RemoteTransactionBegin(MainDBConnection); } } SendRemoteCommand(MainDBConnection, query); PGresult *result = GetRemoteCommandResult(MainDBConnection, true); if (!IsResponseOK(result)) { ReportResultError(MainDBConnection, result, ERROR); } ForgetResults(MainDBConnection); } /* * CleanCitusMainDBConnection closes and removes the connection to Citus main database. */ void CleanCitusMainDBConnection(void) { if (MainDBConnection == NULL) { return; } CloseConnection(MainDBConnection); MainDBConnection = NULL; } /* * StartRemoteTransactionBegin initiates beginning the remote transaction in * a non-blocking manner. The function sends "BEGIN" followed by * assign_distributed_transaction_id() to assign the distributed transaction * id on the remote node. */ void StartRemoteTransactionBegin(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; Assert(transaction->transactionState == REMOTE_TRANS_NOT_STARTED); /* remember transaction as being in-progress */ dlist_push_tail(&InProgressTransactions, &connection->transactionNode); connection->transactionInProgress = true; transaction->transactionState = REMOTE_TRANS_STARTING; StringInfo beginAndSetDistributedTransactionId = makeStringInfo(); /* * Explicitly specify READ COMMITTED, the default on the remote * side might have been changed, and that would cause problematic * behaviour. */ appendStringInfoString(beginAndSetDistributedTransactionId, BeginTransactionCommand()); /* append context for in-progress SAVEPOINTs for this transaction */ List *activeSubXacts = ActiveSubXactContexts(); transaction->lastSuccessfulSubXact = TopSubTransactionId; transaction->lastQueuedSubXact = TopSubTransactionId; SubXactContext *subXactState = NULL; foreach_declared_ptr(subXactState, activeSubXacts) { /* append SET LOCAL state from when SAVEPOINT was encountered... */ if (subXactState->setLocalCmds != NULL) { appendStringInfoString(beginAndSetDistributedTransactionId, subXactState->setLocalCmds->data); } /* ... then append SAVEPOINT to enter this subxact */ appendStringInfo(beginAndSetDistributedTransactionId, "SAVEPOINT savepoint_%u;", subXactState->subId); transaction->lastQueuedSubXact = subXactState->subId; } /* we've pushed into deepest subxact: apply in-progress SET context */ if (activeSetStmts != NULL) { appendStringInfoString(beginAndSetDistributedTransactionId, activeSetStmts->data); } char *assignDistributedTransactionIdCommand = AssignDistributedTransactionIdCommand(); /* add SELECT assign_distributed_transaction_id ... */ appendStringInfoString(beginAndSetDistributedTransactionId, assignDistributedTransactionIdCommand); pfree(assignDistributedTransactionIdCommand); bool success = SendRemoteCommand(connection, beginAndSetDistributedTransactionId->data); pfree(beginAndSetDistributedTransactionId->data); pfree(beginAndSetDistributedTransactionId); if (!success) { const bool raiseErrors = true; HandleRemoteTransactionConnectionError(connection, raiseErrors); } transaction->beginSent = true; } /* * BeginAndSetDistributedTransactionIdCommand returns a command which starts * a transaction and assigns the current distributed transaction id. */ StringInfo BeginAndSetDistributedTransactionIdCommand(void) { StringInfo beginAndSetDistributedTransactionId = makeStringInfo(); /* * Explicitly specify READ COMMITTED, the default on the remote * side might have been changed, and that would cause problematic * behaviour. */ appendStringInfoString(beginAndSetDistributedTransactionId, BeginTransactionCommand()); appendStringInfoString(beginAndSetDistributedTransactionId, AssignDistributedTransactionIdCommand()); return beginAndSetDistributedTransactionId; } /* * BeginTransactionCommand returns the BEGIN command to use for the current isolation * level. * * Transactions have 3 properties that we care about here: * - XactIsoLevel (isolation level) * - XactDeferrable (deferrable) * - XactReadOnly (read only) * * These properties can be set in several ways: * - via BEGIN TRANSACTION ISOLATION LEVEL ... * - via default_transaction_isolation, ... * - via SET TRANSACTION .. (or plain SET transaction_isolation ...) * * We want to make sure that the properties that are passed to the worker nodes * match the coordinator as much as possible. However, we do not want to waste * bytes repeating the current values ad infinitum. * * The trade-off we make is that we send the isolation level in all cases, * but only set deferrable and read-only if they were explicitly specified * in the BEGIN by the user. The implication is that we may not follow the * default_transaction_* settings on the coordinator if they differ on the * worker. */ static char * BeginTransactionCommand(void) { StringInfo beginCommand = makeStringInfo(); /* * XactIsoLevel can only be set at the start of the transaction, before the * first query. Since Citus does not send BEGIN until the first query, we * can simply use the current values, and they will match the values for the * outer transaction after any BEGIN and SET TRANSACTION that may have occurred. */ appendStringInfo(beginCommand, "BEGIN TRANSACTION ISOLATION LEVEL %s", IsolationLevelName[XactIsoLevel]); if (BeginXactDeferrable == BeginXactDeferrable_Enabled) { appendStringInfoString(beginCommand, " DEFERRABLE"); } else if (BeginXactDeferrable == BeginXactDeferrable_Disabled) { appendStringInfoString(beginCommand, " NOT DEFERRABLE"); } if (BeginXactReadOnly == BeginXactReadOnly_Enabled) { appendStringInfoString(beginCommand, " READ ONLY"); } else if (BeginXactReadOnly == BeginXactReadOnly_Disabled) { appendStringInfoString(beginCommand, " READ WRITE"); } appendStringInfoChar(beginCommand, ';'); return beginCommand->data; } /* * AssignDistributedTransactionIdCommand returns a command to set the local * distributed transaction ID on a remote transaction. */ static char * AssignDistributedTransactionIdCommand(void) { StringInfo assignDistributedTransactionId = makeStringInfo(); /* * Append BEGIN and assign_distributed_transaction_id() statements into a single command * and send both in one step. The reason is purely performance, we don't want * seperate roundtrips for these two statements. */ DistributedTransactionId *distributedTransactionId = GetCurrentDistributedTransactionId(); const char *timestamp = timestamptz_to_str(distributedTransactionId->timestamp); appendStringInfo(assignDistributedTransactionId, "SELECT assign_distributed_transaction_id(%d, " UINT64_FORMAT ", '%s');", distributedTransactionId->initiatorNodeIdentifier, distributedTransactionId->transactionNumber, timestamp); /* free the StringInfo but not the buffer itself */ char *command = assignDistributedTransactionId->data; pfree(assignDistributedTransactionId); return command; } /* * FinishRemoteTransactionBegin finishes the work StartRemoteTransactionBegin * initiated. It blocks if necessary (i.e. if PQisBusy() would return true). */ void FinishRemoteTransactionBegin(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; bool raiseErrors = true; Assert(transaction->transactionState == REMOTE_TRANS_STARTING); bool clearSuccessful = ClearResults(connection, raiseErrors); if (clearSuccessful) { transaction->transactionState = REMOTE_TRANS_STARTED; transaction->lastSuccessfulSubXact = transaction->lastQueuedSubXact; } if (!transaction->transactionFailed) { Assert(PQtransactionStatus(connection->pgConn) == PQTRANS_INTRANS); } } /* * RemoteTransactionBegin begins a remote transaction in a blocking manner. */ void RemoteTransactionBegin(struct MultiConnection *connection) { StartRemoteTransactionBegin(connection); FinishRemoteTransactionBegin(connection); } /* * RemoteTransactionListBegin sends BEGIN over all connections in the * given connection list and waits for all of them to finish. */ void RemoteTransactionListBegin(List *connectionList) { MultiConnection *connection = NULL; /* send BEGIN to all nodes */ foreach_declared_ptr(connection, connectionList) { StartRemoteTransactionBegin(connection); } /* wait for BEGIN to finish on all nodes */ foreach_declared_ptr(connection, connectionList) { FinishRemoteTransactionBegin(connection); } } /* * StartRemoteTransactionCommit initiates transaction commit in a non-blocking * manner. If the transaction is in a failed state, it'll instead get rolled * back. */ void StartRemoteTransactionCommit(MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = false; /* can only commit if transaction is in progress */ Assert(transaction->transactionState != REMOTE_TRANS_NOT_STARTED); /* can't commit if we already started to commit or abort */ Assert(transaction->transactionState < REMOTE_TRANS_1PC_ABORTING); if (transaction->transactionFailed) { /* abort the transaction if it failed */ transaction->transactionState = REMOTE_TRANS_1PC_ABORTING; /* * Try sending an ROLLBACK; Depending on the state that won't * succeed, but let's try. Have to clear previous results * first. */ ForgetResults(connection); /* try to clear pending stuff */ if (!SendRemoteCommand(connection, "ROLLBACK")) { /* no point in reporting a likely redundant message */ } } else if (transaction->transactionState == REMOTE_TRANS_PREPARED) { /* * Commit the prepared transaction. * * We need to allocate 420 bytes for command buffer (including '\0'): * - len("COMMIT PREPARED ") = 16 * - maximum quoted length of transaction->preparedName = 2 * 200 + 3 = 403 */ char command[420]; char *quotedPrepName = quote_literal_cstr(transaction->preparedName); SafeSnprintf(command, sizeof(command), "COMMIT PREPARED %s", quotedPrepName); pfree(quotedPrepName); transaction->transactionState = REMOTE_TRANS_2PC_COMMITTING; if (!SendRemoteCommand(connection, command)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } } else { /* initiate remote transaction commit */ transaction->transactionState = REMOTE_TRANS_1PC_COMMITTING; if (!SendRemoteCommand(connection, "COMMIT")) { /* * For a moment there I thought we were in trouble. * * Failing in this state means that we don't know whether the * commit has succeeded. */ HandleRemoteTransactionConnectionError(connection, raiseErrors); } } } /* * FinishRemoteTransactionCommit finishes the work * StartRemoteTransactionCommit initiated. It blocks if necessary (i.e. if * PQisBusy() would return true). */ void FinishRemoteTransactionCommit(MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = false; Assert(transaction->transactionState == REMOTE_TRANS_1PC_ABORTING || transaction->transactionState == REMOTE_TRANS_1PC_COMMITTING || transaction->transactionState == REMOTE_TRANS_2PC_COMMITTING); PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { HandleRemoteTransactionResultError(connection, result, raiseErrors); /* * Failing in this state means that we will often not know whether * the commit has succeeded (particularly in case of network * troubles). * * XXX: It might be worthwhile to discern cases where we got a * proper error back from postgres (i.e. COMMIT was received but * produced an error) from cases where the connection failed * before getting a reply. */ if (transaction->transactionState == REMOTE_TRANS_1PC_COMMITTING) { ereport(WARNING, (errmsg("failed to commit transaction on %s:%d", connection->hostname, connection->port))); } else if (transaction->transactionState == REMOTE_TRANS_2PC_COMMITTING) { ereport(WARNING, (errmsg("failed to commit transaction on %s:%d", connection->hostname, connection->port))); } } else if (transaction->transactionState == REMOTE_TRANS_1PC_ABORTING || transaction->transactionState == REMOTE_TRANS_2PC_ABORTING) { transaction->transactionState = REMOTE_TRANS_ABORTED; } else { transaction->transactionState = REMOTE_TRANS_COMMITTED; } PQclear(result); ForgetResults(connection); } /* * RemoteTransactionCommit commits (or aborts, if the transaction failed) a * remote transaction in a blocking manner. */ void RemoteTransactionCommit(MultiConnection *connection) { StartRemoteTransactionCommit(connection); FinishRemoteTransactionCommit(connection); } /* * StartRemoteTransactionAbort initiates abortin the transaction in a * non-blocking manner. */ void StartRemoteTransactionAbort(MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = false; Assert(transaction->transactionState != REMOTE_TRANS_NOT_STARTED); /* * Clear previous results, so we have a better chance to send ROLLBACK * [PREPARED]. If we've previously sent a PREPARE TRANSACTION, we always * want to wait for that result, as that shouldn't take long and will * reserve resources. But if there's another query running, we don't want * to wait, because a long running statement may be running, so force it to * be killed in that case. */ if (transaction->transactionState == REMOTE_TRANS_PREPARING || transaction->transactionState == REMOTE_TRANS_PREPARED) { ForgetResults(connection); /* * Await PREPARE TRANSACTION results, closing the connection would leave it dangling. * * We need to allocate 422 bytes for command buffer (including '\0'): * - len("ROLLBACK PREPARED ") = 18 * - maximum quoted length of transaction->preparedName = 2 * 200 + 3 = 403 */ char command[422]; char *quotedPrepName = quote_literal_cstr(transaction->preparedName); SafeSnprintf(command, sizeof(command), "ROLLBACK PREPARED %s", quotedPrepName); pfree(quotedPrepName); if (!SendRemoteCommand(connection, command)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } else { transaction->transactionState = REMOTE_TRANS_2PC_ABORTING; } } else { /* * In case of a cancellation, the connection might still be working * on some commands. Try to consume the results such that the * connection can be reused, but do not want to wait for commands * to finish. Instead we just close the connection if the command * is still busy. */ if (!ClearResultsIfReady(connection)) { ShutdownConnection(connection); /* FinishRemoteTransactionAbort will emit warning */ return; } if (!SendRemoteCommand(connection, "ROLLBACK")) { /* no point in reporting a likely redundant message */ MarkRemoteTransactionFailed(connection, raiseErrors); } else { transaction->transactionState = REMOTE_TRANS_1PC_ABORTING; } } } /* * FinishRemoteTransactionAbort finishes the work StartRemoteTransactionAbort * initiated. It blocks if necessary (i.e. if PQisBusy() would return true). */ void FinishRemoteTransactionAbort(MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = false; if (transaction->transactionState == REMOTE_TRANS_2PC_ABORTING) { PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { HandleRemoteTransactionResultError(connection, result, raiseErrors); } PQclear(result); } /* * Try to consume results of any in-progress commands. In the 1PC case * this is also where we consume the result of the ROLLBACK. * * If we don't succeed the connection will be in a bad state, so we close it. */ if (!ClearResults(connection, raiseErrors)) { ShutdownConnection(connection); } transaction->transactionState = REMOTE_TRANS_ABORTED; } /* * RemoteTransactionAbort aborts a remote transaction in a blocking manner. */ void RemoteTransactionAbort(MultiConnection *connection) { StartRemoteTransactionAbort(connection); FinishRemoteTransactionAbort(connection); } /* * StartRemoteTransactionPrepare initiates preparing the transaction in a * non-blocking manner. */ void StartRemoteTransactionPrepare(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = true; /* can't prepare a nonexistant transaction */ Assert(transaction->transactionState != REMOTE_TRANS_NOT_STARTED); /* can't prepare in a failed transaction */ Assert(!transaction->transactionFailed); /* can't prepare if already started to prepare/abort/commit */ Assert(transaction->transactionState < REMOTE_TRANS_PREPARING); Assign2PCIdentifier(connection); /* log transactions to workers in pg_dist_transaction */ WorkerNode *workerNode = FindWorkerNode(connection->hostname, connection->port); if (workerNode != NULL) { LogTransactionRecord(workerNode->groupId, transaction->preparedName, OuterXid); } /* * We need to allocate 424 bytes for command buffer (including '\0'): * - len("PREPARE TRANSACTION ") = 20 * - maximum quoted length of transaction->preparedName = 2 * 200 + 3 = 403 */ char command[424]; char *quotedPrepName = quote_literal_cstr(transaction->preparedName); SafeSnprintf(command, sizeof(command), "PREPARE TRANSACTION %s", quotedPrepName); pfree(quotedPrepName); if (!SendRemoteCommand(connection, command)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } else { transaction->transactionState = REMOTE_TRANS_PREPARING; } } /* * FinishRemoteTransactionPrepare finishes the work * StartRemoteTransactionPrepare initiated. It blocks if necessary (i.e. if * PQisBusy() would return true). */ void FinishRemoteTransactionPrepare(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; const bool raiseErrors = true; Assert(transaction->transactionState == REMOTE_TRANS_PREPARING); PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { transaction->transactionState = REMOTE_TRANS_ABORTED; HandleRemoteTransactionResultError(connection, result, raiseErrors); } else { transaction->transactionState = REMOTE_TRANS_PREPARED; } PQclear(result); /* * Try to consume results of PREPARE TRANSACTION command. If we don't * succeed, rollback the transaction. Note that we've not committed on * any node yet, and we're not sure about the state of the worker node. * So rollbacking seems to be the safest action if the worker is * in a state where it can actually rollback. */ if (!ClearResults(connection, raiseErrors)) { ereport(ERROR, (errmsg("failed to prepare transaction '%s' on host %s:%d", transaction->preparedName, connection->hostname, connection->port), errhint("Try re-running the command."))); } } /* * RemoteTransactionBeginIfNecessary is a convenience wrapper around * RemoteTransactionsBeginIfNecessary(), for a single connection. */ void RemoteTransactionBeginIfNecessary(MultiConnection *connection) { /* just delegate */ if (InCoordinatedTransaction()) { List *connectionList = list_make1(connection); RemoteTransactionsBeginIfNecessary(connectionList); list_free(connectionList); } } /* * RemoteTransactionsBeginIfNecessary begins, if necessary according to this * session's coordinated transaction state, and the remote transaction's * state, an explicit transaction on all the connections. This is done in * parallel, to lessen latency penalties. */ void RemoteTransactionsBeginIfNecessary(List *connectionList) { MultiConnection *connection = NULL; /* * Don't do anything if not in a coordinated transaction. That allows the * same code to work both in situations that uses transactions, and when * not. */ if (!InCoordinatedTransaction()) { return; } /* issue BEGIN to all connections needing it */ foreach_declared_ptr(connection, connectionList) { RemoteTransaction *transaction = &connection->remoteTransaction; /* can't send BEGIN if a command already is in progress */ Assert(PQtransactionStatus(connection->pgConn) != PQTRANS_ACTIVE); /* * If a transaction already is in progress (including having failed), * don't start it again. That's quite normal if a piece of code allows * cached connections. */ if (transaction->transactionState != REMOTE_TRANS_NOT_STARTED) { continue; } StartRemoteTransactionBegin(connection); } bool raiseInterrupts = true; WaitForAllConnections(connectionList, raiseInterrupts); /* get result of all the BEGINs */ foreach_declared_ptr(connection, connectionList) { RemoteTransaction *transaction = &connection->remoteTransaction; /* * Only handle BEGIN results on connections that are in process of * starting a transaction, and haven't already failed (e.g. by not * being able to send BEGIN due to a network failure). */ if (transaction->transactionFailed || transaction->transactionState != REMOTE_TRANS_STARTING) { continue; } FinishRemoteTransactionBegin(connection); } } /* * HandleRemoteTransactionConnectionError records a transaction as having failed * and throws a connection error if the transaction was critical and raiseErrors * is true, or a warning otherwise. */ void HandleRemoteTransactionConnectionError(MultiConnection *connection, bool raiseErrors) { RemoteTransaction *transaction = &connection->remoteTransaction; transaction->transactionFailed = true; if (transaction->transactionCritical && raiseErrors) { ReportConnectionError(connection, ERROR); } else { ReportConnectionError(connection, WARNING); } } /* * HandleRemoteTransactionResultError records a transaction as having failed * and throws a result error if the transaction was critical and raiseErrors * is true, or a warning otherwise. */ void HandleRemoteTransactionResultError(MultiConnection *connection, PGresult *result, bool raiseErrors) { RemoteTransaction *transaction = &connection->remoteTransaction; transaction->transactionFailed = true; if (transaction->transactionCritical && raiseErrors) { ReportResultError(connection, result, ERROR); } else { ReportResultError(connection, result, WARNING); } } /* * MarkRemoteTransactionFailed records a transaction as having failed. * * If the connection is marked as critical, and allowErrorPromotion is true, * this routine will ERROR out. The allowErrorPromotion case is primarily * required for the transaction management code itself. Usually it is helpful * to fail as soon as possible. If !allowErrorPromotion transaction commit * will instead issue an error before committing on any node. */ void MarkRemoteTransactionFailed(MultiConnection *connection, bool allowErrorPromotion) { RemoteTransaction *transaction = &connection->remoteTransaction; transaction->transactionFailed = true; /* * If the connection is marked as critical, fail the entire coordinated * transaction. If allowed. */ if (transaction->transactionCritical && allowErrorPromotion) { ereport(ERROR, (errmsg("failure on connection marked as essential: %s:%d", connection->hostname, connection->port))); } } /* * MarkRemoteTransactionCritical signals that failures on this remote * transaction should fail the entire coordinated transaction. */ void MarkRemoteTransactionCritical(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; transaction->transactionCritical = true; } /* * ResetRemoteTransaction resets the state of the transaction after the end of * the main transaction, if the connection is being reused. */ void ResetRemoteTransaction(struct MultiConnection *connection) { RemoteTransaction *transaction = &connection->remoteTransaction; /* unlink from list of open transactions, if necessary */ if (connection->transactionInProgress) { /* XXX: Should we error out for a critical transaction? */ dlist_delete(&connection->transactionNode); connection->transactionInProgress = false; memset(&connection->transactionNode, 0, sizeof(connection->transactionNode)); } /* just reset the entire state, relying on 0 being invalid/false */ memset(transaction, 0, sizeof(*transaction)); ResetShardPlacementAssociation(connection); /* reset copy state */ connection->copyBytesWrittenSinceLastFlush = 0; } /* * CoordinatedRemoteTransactionsPrepare PREPAREs a 2PC transaction on all * non-failed transactions participating in the coordinated transaction. */ void CoordinatedRemoteTransactionsPrepare(void) { dlist_iter iter; List *connectionList = NIL; /* issue PREPARE TRANSACTION; to all relevant remote nodes */ /* asynchronously send PREPARE */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; Assert(transaction->transactionState != REMOTE_TRANS_NOT_STARTED); /* can't PREPARE a transaction that failed */ if (transaction->transactionFailed) { continue; } /* * Check if any DML or DDL is executed over the connection on any * placement/table. If yes, we start preparing the transaction, otherwise * we skip prepare since the connection didn't perform any write (read-only) */ if (ConnectionModifiedPlacement(connection)) { StartRemoteTransactionPrepare(connection); connectionList = lappend(connectionList, connection); } } bool raiseInterrupts = true; WaitForAllConnections(connectionList, raiseInterrupts); /* Wait for result */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionState != REMOTE_TRANS_PREPARING) { /* * Verify that either the transaction failed, hence we couldn't prepare * or the connection didn't modify any placement */ Assert(transaction->transactionFailed || !ConnectionModifiedPlacement(connection)); continue; } FinishRemoteTransactionPrepare(connection); } CurrentCoordinatedTransactionState = COORD_TRANS_PREPARED; list_free(connectionList); } /* * CoordinatedRemoteTransactionsCommit performs distributed transactions * handling at commit time. This will be called at XACT_EVENT_PRE_COMMIT if * 1PC commits are used - so shards can still be invalidated - and at * XACT_EVENT_COMMIT if 2PC is being used. * * Note that this routine has to issue rollbacks for failed transactions. */ void CoordinatedRemoteTransactionsCommit(void) { dlist_iter iter; List *connectionList = NIL; /* * Issue appropriate transaction commands to remote nodes. If everything * went well that's going to be COMMIT or COMMIT PREPARED, if individual * connections had errors, some or all of them might require a ROLLBACK. * * First send the command asynchronously over all connections. */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionState == REMOTE_TRANS_NOT_STARTED || transaction->transactionState == REMOTE_TRANS_1PC_COMMITTING || transaction->transactionState == REMOTE_TRANS_2PC_COMMITTING || transaction->transactionState == REMOTE_TRANS_COMMITTED || transaction->transactionState == REMOTE_TRANS_ABORTED) { continue; } StartRemoteTransactionCommit(connection); connectionList = lappend(connectionList, connection); } bool raiseInterrupts = false; WaitForAllConnections(connectionList, raiseInterrupts); /* wait for the replies to the commands to come in */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; /* nothing to do if not committing / aborting */ if (transaction->transactionState != REMOTE_TRANS_1PC_COMMITTING && transaction->transactionState != REMOTE_TRANS_2PC_COMMITTING && transaction->transactionState != REMOTE_TRANS_1PC_ABORTING && transaction->transactionState != REMOTE_TRANS_2PC_ABORTING) { continue; } FinishRemoteTransactionCommit(connection); } list_free(connectionList); } /* * CoordinatedRemoteTransactionsAbort performs distributed transactions * handling at abort time. * * This issues ROLLBACKS and ROLLBACK PREPARED depending on whether the remote * transaction has been prepared or not. */ void CoordinatedRemoteTransactionsAbort(void) { dlist_iter iter; List *connectionList = NIL; /* asynchronously send ROLLBACK [PREPARED] */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionState == REMOTE_TRANS_NOT_STARTED || transaction->transactionState == REMOTE_TRANS_1PC_ABORTING || transaction->transactionState == REMOTE_TRANS_2PC_ABORTING || transaction->transactionState == REMOTE_TRANS_ABORTED) { continue; } StartRemoteTransactionAbort(connection); connectionList = lappend(connectionList, connection); } bool raiseInterrupts = false; WaitForAllConnections(connectionList, raiseInterrupts); /* and wait for the results */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionState != REMOTE_TRANS_1PC_ABORTING && transaction->transactionState != REMOTE_TRANS_2PC_ABORTING) { continue; } FinishRemoteTransactionAbort(connection); } list_free(connectionList); } /* * CoordinatedRemoteTransactionsSavepointBegin sends the SAVEPOINT command for * the given sub-transaction id to all connections participating in the current * transaction. */ void CoordinatedRemoteTransactionsSavepointBegin(SubTransactionId subId) { dlist_iter iter; const bool raiseInterrupts = true; List *connectionList = NIL; /* asynchronously send SAVEPOINT */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } StartRemoteTransactionSavepointBegin(connection, subId); connectionList = lappend(connectionList, connection); } WaitForAllConnections(connectionList, raiseInterrupts); /* and wait for the results */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } FinishRemoteTransactionSavepointBegin(connection, subId); if (!transaction->transactionFailed) { transaction->lastSuccessfulSubXact = subId; } } list_free(connectionList); } /* * CoordinatedRemoteTransactionsSavepointRelease sends the RELEASE SAVEPOINT * command for the given sub-transaction id to all connections participating in * the current transaction. */ void CoordinatedRemoteTransactionsSavepointRelease(SubTransactionId subId) { dlist_iter iter; const bool raiseInterrupts = true; List *connectionList = NIL; /* asynchronously send RELEASE SAVEPOINT */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } StartRemoteTransactionSavepointRelease(connection, subId); connectionList = lappend(connectionList, connection); } WaitForAllConnections(connectionList, raiseInterrupts); /* and wait for the results */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed) { continue; } FinishRemoteTransactionSavepointRelease(connection, subId); } list_free(connectionList); } /* * CoordinatedRemoteTransactionsSavepointRollback sends the ROLLBACK TO SAVEPOINT * command for the given sub-transaction id to all connections participating in * the current transaction. */ void CoordinatedRemoteTransactionsSavepointRollback(SubTransactionId subId) { dlist_iter iter; const bool raiseInterrupts = false; List *connectionList = NIL; /* asynchronously send ROLLBACK TO SAVEPOINT */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; /* cancel any ongoing queries before issuing rollback */ SendCancelationRequest(connection); /* clear results, but don't show cancelation warning messages from workers. */ ClearResultsDiscardWarnings(connection, raiseInterrupts); if (transaction->transactionFailed) { if (transaction->lastSuccessfulSubXact <= subId) { transaction->transactionRecovering = true; /* * Clear the results of the failed query so we can send the ROLLBACK * TO SAVEPOINT command for a savepoint that can recover the transaction * from failure. */ ForgetResults(connection); } else { continue; } } StartRemoteTransactionSavepointRollback(connection, subId); connectionList = lappend(connectionList, connection); } WaitForAllConnections(connectionList, raiseInterrupts); /* and wait for the results */ dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; if (transaction->transactionFailed && !transaction->transactionRecovering) { continue; } FinishRemoteTransactionSavepointRollback(connection, subId); /* * We unclaim the connection now so it can be used again when * continuing after the ROLLBACK TO SAVEPOINT. * XXX: We do not undo our hadDML/hadDDL flags. This could result in * some queries not being allowed on Citus that would actually be fine * to execute. Changing this would require us to keep track for each * savepoint which placement connections had DDL/DML executed at that * point and if they were already. We also do not call * ResetShardPlacementAssociation. This might result in suboptimal * parallelism, because of placement associations that are not really * necessary anymore because of ROLLBACK TO SAVEPOINT. To change this * we would need to keep track of when a connection becomes associated * to a placement. */ UnclaimConnection(connection); } list_free(connectionList); } /* * StartRemoteTransactionSavepointBegin initiates SAVEPOINT command for the given * subtransaction id in a non-blocking manner. */ static void StartRemoteTransactionSavepointBegin(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = true; /* * We need to allocate 31 bytes for command buffer (including '\0'): * - len("SAVEPOINT savepoint_") = 20 * - maximum length of str(subId) = 10 */ char savepointCommand[31]; SafeSnprintf(savepointCommand, sizeof(savepointCommand), "SAVEPOINT savepoint_%u", subId); if (!SendRemoteCommand(connection, savepointCommand)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } } /* * FinishRemoteTransactionSavepointBegin finishes the work * StartRemoteTransactionSavepointBegin initiated. It blocks if necessary (i.e. * if PQisBusy() would return true). */ static void FinishRemoteTransactionSavepointBegin(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = true; PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { HandleRemoteTransactionResultError(connection, result, raiseErrors); } PQclear(result); ForgetResults(connection); } /* * StartRemoteTransactionSavepointRelease initiates RELEASE SAVEPOINT command for * the given subtransaction id in a non-blocking manner. */ static void StartRemoteTransactionSavepointRelease(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = true; /* * We need to allocate 39 bytes for command buffer (including '\0'): * - len("RELEASE SAVEPOINT savepoint_") = 28 * - maximum length of str(subId) = 10 */ char savepointCommand[39]; SafeSnprintf(savepointCommand, sizeof(savepointCommand), "RELEASE SAVEPOINT savepoint_%u", subId); if (!SendRemoteCommand(connection, savepointCommand)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } } /* * FinishRemoteTransactionSavepointRelease finishes the work * StartRemoteTransactionSavepointRelease initiated. It blocks if necessary (i.e. * if PQisBusy() would return true). */ static void FinishRemoteTransactionSavepointRelease(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = true; PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { HandleRemoteTransactionResultError(connection, result, raiseErrors); } PQclear(result); ForgetResults(connection); } /* * StartRemoteTransactionSavepointRollback initiates ROLLBACK TO SAVEPOINT command * for the given subtransaction id in a non-blocking manner. */ static void StartRemoteTransactionSavepointRollback(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = false; /* * We need to allocate 43 bytes for command buffer (including '\0'): * - len("ROLLBACK TO SAVEPOINT savepoint_") = 32 * - maximum length of str(subId) = 10 */ char savepointCommand[43]; SafeSnprintf(savepointCommand, sizeof(savepointCommand), "ROLLBACK TO SAVEPOINT savepoint_%u", subId); if (!SendRemoteCommand(connection, savepointCommand)) { HandleRemoteTransactionConnectionError(connection, raiseErrors); } } /* * FinishRemoteTransactionSavepointRollback finishes the work * StartRemoteTransactionSavepointRollback initiated. It blocks if necessary (i.e. * if PQisBusy() would return true). It also recovers the transaction from failure * if transaction is recovering and the rollback command succeeds. */ static void FinishRemoteTransactionSavepointRollback(MultiConnection *connection, SubTransactionId subId) { const bool raiseErrors = false; RemoteTransaction *transaction = &connection->remoteTransaction; PGresult *result = GetRemoteCommandResult(connection, raiseErrors); if (!IsResponseOK(result)) { HandleRemoteTransactionResultError(connection, result, raiseErrors); } /* ROLLBACK TO SAVEPOINT succeeded, check if it recovers the transaction */ else if (transaction->transactionRecovering) { transaction->transactionFailed = false; transaction->transactionRecovering = false; } PQclear(result); ForgetResults(connection); /* reset transaction state so the executor can accept next commands in transaction */ transaction->transactionState = REMOTE_TRANS_STARTED; } /* * CheckRemoteTransactionsHealth checks if any of the participating transactions in a * coordinated transaction failed, and what consequence that should have. * This needs to be called before the coordinated transaction commits (but * after they've been PREPAREd if 2PC is in use). */ void CheckRemoteTransactionsHealth(void) { dlist_iter iter; dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); RemoteTransaction *transaction = &connection->remoteTransaction; PGTransactionStatusType status = PQtransactionStatus(connection->pgConn); /* if the connection is in a bad state, so is the transaction's state */ if (status == PQTRANS_INERROR || status == PQTRANS_UNKNOWN) { transaction->transactionFailed = true; } /* * If a critical connection is marked as failed (and no error has been * raised yet) do so now. */ if (transaction->transactionFailed && transaction->transactionCritical) { ereport(ERROR, (errmsg("failure on connection marked as essential: %s:%d", connection->hostname, connection->port))); } } } /* * Assign2PCIdentifier computes the 2PC transaction name to use for a * transaction. Every prepared transaction should get a new name, i.e. this * function will need to be called again. * * The format of the name is: * * citus____ * * (at most 5+1+10+1+10+1+20+1+10 = 59 characters, while limit is 64) * * The source group is used to distinguish 2PCs started by different * coordinators. A coordinator will only attempt to recover its own 2PCs. * * The pid is used to distinguish different processes on the coordinator, mainly * to provide some entropy across restarts. * * The distributed transaction number is used to distinguish different * transactions originating from the same node (since restart). * * The connection number is used to distinguish connections made to a node * within the same transaction. * */ static void Assign2PCIdentifier(MultiConnection *connection) { /* local sequence number used to distinguish different connections */ static uint32 connectionNumber = 0; /* transaction identifier that is unique across processes */ uint64 transactionNumber = CurrentDistributedTransactionNumber(); /* print all numbers as unsigned to guarantee no minus symbols appear in the name */ SafeSnprintf(connection->remoteTransaction.preparedName, NAMEDATALEN, PREPARED_TRANSACTION_NAME_FORMAT, GetLocalGroupId(), MyProcPid, transactionNumber, connectionNumber++); } /* * ParsePreparedTransactionName parses a prepared transaction name to extract * the initiator group ID, initiator process ID, distributed transaction number, * and the connection number. If the transaction name does not match the expected * format ParsePreparedTransactionName returns false, and true otherwise. */ bool ParsePreparedTransactionName(char *preparedTransactionName, int32 *groupId, int *procId, uint64 *transactionNumber, uint32 *connectionNumber) { char *currentCharPointer = preparedTransactionName; currentCharPointer = strchr(currentCharPointer, '_'); if (currentCharPointer == NULL) { return false; } /* step ahead of the current '_' character */ ++currentCharPointer; *groupId = strtol(currentCharPointer, NULL, 10); if ((*groupId == COORDINATOR_GROUP_ID && errno == EINVAL) || (*groupId == INT_MAX && errno == ERANGE)) { return false; } currentCharPointer = strchr(currentCharPointer, '_'); if (currentCharPointer == NULL) { return false; } /* step ahead of the current '_' character */ ++currentCharPointer; *procId = strtol(currentCharPointer, NULL, 10); if ((*procId == 0 && errno == EINVAL) || (*procId == INT_MAX && errno == ERANGE)) { return false; } currentCharPointer = strchr(currentCharPointer, '_'); if (currentCharPointer == NULL) { return false; } /* step ahead of the current '_' character */ ++currentCharPointer; *transactionNumber = strtou64(currentCharPointer, NULL, 10); if ((*transactionNumber == 0 && errno != 0) || (*transactionNumber == ULLONG_MAX && errno == ERANGE)) { return false; } currentCharPointer = strchr(currentCharPointer, '_'); if (currentCharPointer == NULL) { return false; } /* step ahead of the current '_' character */ ++currentCharPointer; *connectionNumber = strtoul(currentCharPointer, NULL, 10); if ((*connectionNumber == 0 && errno == EINVAL) || (*connectionNumber == UINT_MAX && errno == ERANGE)) { return false; } return true; } ================================================ FILE: src/backend/distributed/transaction/transaction_management.c ================================================ /*------------------------------------------------------------------------- * * transaction_management.c * * Transaction management for Citus. Most of the work is delegated to other * subsystems, this files, and especially CoordinatedTransactionCallback, * coordinates the work between them. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/twophase.h" #include "access/xact.h" #include "catalog/dependency.h" #include "common/hashfn.h" #include "nodes/print.h" #include "postmaster/postmaster.h" #include "storage/fd.h" #include "utils/datum.h" #include "utils/guc.h" #include "utils/guc_tables.h" #include "utils/hsearch.h" #include "utils/memutils.h" #include "distributed/backend_data.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands.h" #include "distributed/connection_management.h" #include "distributed/distributed_planner.h" #include "distributed/function_call_delegation.h" #include "distributed/hash_helpers.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/locally_reserved_shared_connections.h" #include "distributed/maintenanced.h" #include "distributed/metadata/dependency.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_logical_replication.h" #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/repartition_join_execution.h" #include "distributed/replication_origin_session_utils.h" #include "distributed/shard_cleaner.h" #include "distributed/shared_connection_stats.h" #include "distributed/subplan_execution.h" #include "distributed/transaction_management.h" #include "distributed/version_compat.h" #include "distributed/worker_log_messages.h" #define COMMIT_MANAGEMENT_COMMAND_2PC \ "SELECT citus_internal.commit_management_command_2pc()" CoordinatedTransactionState CurrentCoordinatedTransactionState = COORD_TRANS_NONE; /* * GUC that determines whether a SELECT in a transaction block should also run in * a transaction block on the worker even if no writes have occurred yet. */ bool SelectOpensTransactionBlock = true; /* controls use of locks to enforce safe commutativity */ bool AllModificationsCommutative = false; /* we've deprecated this flag, keeping here for some time not to break existing users */ bool EnableDeadlockPrevention = true; /* number of nested stored procedure call levels we are currently in */ int StoredProcedureLevel = 0; /* number of nested DO block levels we are currently in */ int DoBlockLevel = 0; /* state needed to keep track of operations used during a transaction */ XactModificationType XactModificationLevel = XACT_MODIFICATION_NONE; /* list of connections that are part of the current coordinated transaction */ dlist_head InProgressTransactions = DLIST_STATIC_INIT(InProgressTransactions); /* * activeSetStmts keeps track of SET LOCAL statements executed within the current * subxact and will be set to NULL when pushing into new subxact or ending top xact. */ StringInfo activeSetStmts; /* * Though a list, we treat this as a stack, pushing on subxact contexts whenever * e.g. a SAVEPOINT is executed (though this is actually performed by providing * PostgreSQL with a sub-xact callback). At present, the context of a subxact * includes * - a subxact identifier, * - any SET LOCAL statements propagated to workers during the sub-transaction, * - all objects propagated to workers during the sub-transaction. * * To be clear, last item of activeSubXactContexts list corresponds to top of * stack. */ static List *activeSubXactContexts = NIL; /* * PropagatedObjectsInTx is a set of objects propagated in the root transaction. * We also keep track of objects propagated in sub-transactions in activeSubXactContexts. * Any committed sub-transaction would cause the objects, which are propagated during * the sub-transaction, to be moved to upper transaction's set. Objects are discarded * when the sub-transaction is aborted. */ static HTAB *PropagatedObjectsInTx = NULL; /* some pre-allocated memory so we don't need to call malloc() during callbacks */ MemoryContext CitusXactCallbackContext = NULL; /* * Should this coordinated transaction use 2PC? Set by * CoordinatedTransactionUse2PC(), e.g. if any modification * is issued and us 2PC. But, even if this flag is set, * the transaction manager is smart enough to only * do 2PC on the remote connections that did a modification. * * As a variable name ShouldCoordinatedTransactionUse2PC could * be improved. We use Use2PCForCoordinatedTransaction() as the * public API function, hence couldn't come up with a better name * for the underlying variable at the moment. */ bool ShouldCoordinatedTransactionUse2PC = false; /* * Distribution function argument (along with colocationId) when delegated * using forceDelegation flag. */ AllowedDistributionColumn AllowedDistributionColumnValue; /* if disabled, distributed statements in a function may run as separate transactions */ bool FunctionOpensTransactionBlock = true; /* if true, we should trigger node metadata sync on commit */ bool NodeMetadataSyncOnCommit = false; /* * In an explicit BEGIN ...; we keep track of top-level transaction characteristics * specified by the user. */ BeginXactReadOnlyState BeginXactReadOnly = BeginXactReadOnly_NotSet; BeginXactDeferrableState BeginXactDeferrable = BeginXactDeferrable_NotSet; /* transaction management functions */ static void CoordinatedTransactionCallback(XactEvent event, void *arg); static void CoordinatedSubTransactionCallback(SubXactEvent event, SubTransactionId subId, SubTransactionId parentSubid, void *arg); /* remaining functions */ static void AdjustMaxPreparedTransactions(void); static void PushSubXact(SubTransactionId subId); static void PopSubXact(SubTransactionId subId, bool commit); static void ResetGlobalVariables(void); static bool SwallowErrors(void (*func)(void)); static void ForceAllInProgressConnectionsToClose(void); static void EnsurePrepareTransactionIsAllowed(void); static HTAB * CurrentTransactionPropagatedObjects(bool readonly); static HTAB * ParentTransactionPropagatedObjects(bool readonly); static void MovePropagatedObjectsToParentTransaction(void); static bool DependencyInPropagatedObjectsHash(HTAB *propagatedObjects, const ObjectAddress *dependency); static HTAB * CreateTxPropagatedObjectsHash(void); /* * UseCoordinatedTransaction sets up the necessary variables to use * a coordinated transaction, unless one is already in progress. */ void UseCoordinatedTransaction(void) { if (CurrentCoordinatedTransactionState == COORD_TRANS_STARTED) { return; } if (CurrentCoordinatedTransactionState != COORD_TRANS_NONE && CurrentCoordinatedTransactionState != COORD_TRANS_IDLE) { ereport(ERROR, (errmsg("starting transaction in wrong state"))); } CurrentCoordinatedTransactionState = COORD_TRANS_STARTED; /* * If assign_distributed_transaction_id() has been called, we should reuse * that identifier so distributed deadlock detection works properly. */ DistributedTransactionId *transactionId = GetCurrentDistributedTransactionId(); if (transactionId->transactionNumber == 0) { AssignDistributedTransactionId(); } } /* * EnsureDistributedTransactionId makes sure that the current transaction * has a distributed transaction id. It is either assigned by a previous * call of assign_distributed_transaction_id(), or by starting a coordinated * transaction. */ void EnsureDistributedTransactionId(void) { DistributedTransactionId *transactionId = GetCurrentDistributedTransactionId(); if (transactionId->transactionNumber == 0) { UseCoordinatedTransaction(); } } /* * InCoordinatedTransaction returns whether a coordinated transaction has been * started. */ bool InCoordinatedTransaction(void) { return CurrentCoordinatedTransactionState != COORD_TRANS_NONE && CurrentCoordinatedTransactionState != COORD_TRANS_IDLE; } /* * Use2PCForCoordinatedTransaction() signals that the current coordinated * transaction should use 2PC to commit. * * Note that even if 2PC is enabled, it is only used for connections that make * modification (DML or DDL). */ void Use2PCForCoordinatedTransaction(void) { /* * If this transaction is also a coordinated * transaction, use 2PC. Otherwise, this * state change does nothing. * * In other words, when this flag is set, * we "should" use 2PC when needed (e.g., * we are in a coordinated transaction and * the coordinated transaction does a remote * modification). */ ShouldCoordinatedTransactionUse2PC = true; } /* * GetCoordinatedTransactionShouldUse2PC is a wrapper function to read the value * of CoordinatedTransactionShouldUse2PCFlag. */ bool GetCoordinatedTransactionShouldUse2PC(void) { return ShouldCoordinatedTransactionUse2PC; } void InitializeTransactionManagement(void) { /* hook into transaction machinery */ RegisterXactCallback(CoordinatedTransactionCallback, NULL); RegisterSubXactCallback(CoordinatedSubTransactionCallback, NULL); AdjustMaxPreparedTransactions(); /* set aside 8kb of memory for use in CoordinatedTransactionCallback */ CitusXactCallbackContext = AllocSetContextCreateInternal(TopMemoryContext, "CitusXactCallbackContext", 8 * 1024, 8 * 1024, 8 * 1024); } /* * Transaction management callback, handling coordinated transaction, and * transaction independent connection management. * * NB: There should only ever be a single transaction callback in citus, the * ordering between the callbacks and the actions within those callbacks * otherwise becomes too undeterministic / hard to reason about. */ static void CoordinatedTransactionCallback(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_COMMIT: { /* * ERRORs thrown during XACT_EVENT_COMMIT will cause postgres to abort, at * this point enough work has been done that it's not possible to rollback. * * One possible source of errors is memory allocation failures. To minimize * the chance of those happening we've pre-allocated some memory in the * CitusXactCallbackContext, it has 8kb of memory that we're allowed to use. * * We only do this in the COMMIT callback because: * - Errors thrown in other callbacks (such as PRE_COMMIT) won't cause * crashes, they will simply cause the ABORT handler to be called. * - The exception is ABORT, errors thrown there could also cause crashes, but * postgres already creates a TransactionAbortContext which performs this * trick, so there's no need for us to do it again. */ MemoryContext previousContext = MemoryContextSwitchTo(CitusXactCallbackContext); if (CurrentCoordinatedTransactionState == COORD_TRANS_PREPARED && !IsMainDBCommand) { /* handles both already prepared and open transactions */ CoordinatedRemoteTransactionsCommit(); } /* * If this is a non-Citus main database we should try to commit the prepared * transactions created by the Citus main database on the worker nodes. */ if (!IsMainDB && MainDBConnection != NULL && IsMainDBCommandInXact) { RunCitusMainDBQuery(COMMIT_MANAGEMENT_COMMAND_2PC); CleanCitusMainDBConnection(); } /* close connections etc. */ if (CurrentCoordinatedTransactionState != COORD_TRANS_NONE) { ResetPlacementConnectionManagement(); AfterXactConnectionHandling(true); } /* * Changes to catalog tables are now visible to the metadata sync * daemon, so we can trigger node metadata sync if necessary. */ if (NodeMetadataSyncOnCommit) { TriggerNodeMetadataSync(MyDatabaseId); } ResetGlobalVariables(); ResetRelationAccessHash(); ResetPropagatedObjects(); /* * Make sure that we give the shared connections back to the shared * pool if any. This operation is a no-op if the reserved connections * are already given away. */ DeallocateReservedConnections(); UnSetDistributedTransactionId(); PlacementMovedUsingLogicalReplicationInTX = false; /* empty the CitusXactCallbackContext to ensure we're not leaking memory */ MemoryContextSwitchTo(previousContext); MemoryContextReset(CitusXactCallbackContext); /* Set CreateCitusTransactionLevel to 0 since original transaction is about to be * committed. */ if (GetCitusCreationLevel() > 0) { /* Check CitusCreationLevel was correctly decremented to 1 */ Assert(GetCitusCreationLevel() == 1); SetCreateCitusTransactionLevel(0); } break; } case XACT_EVENT_ABORT: { /* stop propagating notices from workers, we know the query is failed */ DisableWorkerMessagePropagation(); RemoveIntermediateResultsDirectories(); CleanCitusMainDBConnection(); /* handles both already prepared and open transactions */ if (CurrentCoordinatedTransactionState > COORD_TRANS_IDLE) { /* * Since CoordinateRemoteTransactionsAbort may cause an error and it is * not allowed to error out at that point, swallow the error if any. * * Particular error we've observed was CreateWaitEventSet throwing an error * when out of file descriptor. * * If an error is swallowed, connections of all active transactions must * be forced to close at the end of the transaction explicitly. */ bool errorSwallowed = SwallowErrors(CoordinatedRemoteTransactionsAbort); if (errorSwallowed == true) { ForceAllInProgressConnectionsToClose(); } } /* * Close connections etc. Contrary to a successful transaction we reset the * placement connection management irregardless of state of the statemachine * as recorded in CurrentCoordinatedTransactionState. * The hashmaps recording the connection management live a memory context * higher compared to most of the data referenced in the hashmap. This causes * use after free errors when the contents are retained due to an error caused * before the CurrentCoordinatedTransactionState changed. */ ResetPlacementConnectionManagement(); AfterXactConnectionHandling(false); ResetGlobalVariables(); ResetRelationAccessHash(); ResetPropagatedObjects(); /* Reset any local replication origin session since transaction has been aborted.*/ ResetReplicationOriginLocalSession(); /* empty the CitusXactCallbackContext to ensure we're not leaking memory */ MemoryContextReset(CitusXactCallbackContext); /* * Clear MetadataCache table if we're aborting from a CREATE EXTENSION Citus * so that any created OIDs from the table are cleared and invalidated. We * also set CreateCitusTransactionLevel to 0 since that process has been aborted */ if (GetCitusCreationLevel() > 0) { /* Checks CitusCreationLevel correctly decremented to 1 */ Assert(GetCitusCreationLevel() == 1); InvalidateMetadataSystemCache(); SetCreateCitusTransactionLevel(0); } /* * Make sure that we give the shared connections back to the shared * pool if any. This operation is a no-op if the reserved connections * are already given away. */ DeallocateReservedConnections(); /* * We reset these mainly for posterity. The only way we would normally * get here with ExecutorLevel or PlannerLevel > 0 is during a fatal * error when the process is about to end. */ ExecutorLevel = 0; PlannerLevel = 0; /* * We should reset SubPlanLevel in case a transaction is aborted, * otherwise this variable would stay +ve if the transaction is * aborted in the middle of a CTE/complex subquery execution * which would cause the subsequent queries to error out in * case the copy size is greater than * citus.max_intermediate_result_size */ SubPlanLevel = 0; UnSetDistributedTransactionId(); PlacementMovedUsingLogicalReplicationInTX = false; break; } case XACT_EVENT_PARALLEL_COMMIT: case XACT_EVENT_PARALLEL_ABORT: { break; } case XACT_EVENT_PREPARE: { /* we need to reset SavedExplainPlan before TopTransactionContext is deleted */ FreeSavedExplainPlan(); /* * This callback is only relevant for worker queries since * distributed queries cannot be executed with 2PC, see * XACT_EVENT_PRE_PREPARE. * * We should remove the intermediate results before unsetting the * distributed transaction id. That is necessary, otherwise Citus * would try to remove a non-existing folder and leak some of the * existing folders that are associated with distributed transaction * ids on the worker nodes. */ RemoveIntermediateResultsDirectories(); UnSetDistributedTransactionId(); break; } case XACT_EVENT_PRE_COMMIT: { /* * If the distributed query involves 2PC, we already removed * the intermediate result directory on XACT_EVENT_PREPARE. However, * if not, we should remove it here on the COMMIT. Since * RemoveIntermediateResultsDirectories() is idempotent, we're safe * to call it here again even if the transaction involves 2PC. */ RemoveIntermediateResultsDirectories(); /* nothing further to do if there's no managed remote xacts */ if (CurrentCoordinatedTransactionState == COORD_TRANS_NONE) { break; } /* * If this is a non-Citus main database we should commit the Citus * main database query. So if some error happens on the distributed main * database query we wouldn't have committed the current query. */ if (!IsMainDB && MainDBConnection != NULL && IsMainDBCommandInXact) { RunCitusMainDBQuery("COMMIT"); } /* * TODO: It'd probably be a good idea to force constraints and * such to 'immediate' here. Deferred triggers might try to send * stuff to the remote side, which'd not be good. Doing so * remotely would also catch a class of errors where committing * fails, which can lead to divergence when not using 2PC. */ if (ShouldCoordinatedTransactionUse2PC) { CoordinatedRemoteTransactionsPrepare(); CurrentCoordinatedTransactionState = COORD_TRANS_PREPARED; /* * Make sure we did not have any failures on connections marked as * critical before committing. */ CheckRemoteTransactionsHealth(); } else { CheckRemoteTransactionsHealth(); /* * Have to commit remote transactions in PRE_COMMIT, to allow * us to mark failed placements as invalid. Better don't use * this for anything important (i.e. DDL/metadata). */ if (IsMainDB) { CoordinatedRemoteTransactionsCommit(); } CurrentCoordinatedTransactionState = COORD_TRANS_COMMITTED; } /* * Check again whether shards/placement successfully * committed. This handles failure at COMMIT time. */ ErrorIfPostCommitFailedShardPlacements(); break; } case XACT_EVENT_PARALLEL_PRE_COMMIT: case XACT_EVENT_PRE_PREPARE: { EnsurePrepareTransactionIsAllowed(); break; } } } /* * ForceAllInProgressConnectionsToClose forces all connections of in progress transactions * to close at the end of the transaction. */ static void ForceAllInProgressConnectionsToClose(void) { dlist_iter iter; dlist_foreach(iter, &InProgressTransactions) { MultiConnection *connection = dlist_container(MultiConnection, transactionNode, iter.cur); connection->forceCloseAtTransactionEnd = true; } } /* * If an ERROR is thrown while processing a transaction the ABORT handler is called. * ERRORS thrown during ABORT are not treated any differently, the ABORT handler is also * called during processing of those. If an ERROR was raised the first time through it's * unlikely that the second try will succeed; more likely that an ERROR will be thrown * again. This loop continues until Postgres notices and PANICs, complaining about a stack * overflow. * * Instead of looping and crashing, SwallowErrors lets us attempt to continue running the * ABORT logic. This wouldn't be safe in most other parts of the codebase, in * approximately none of the places where we emit ERROR do we first clean up after * ourselves! It's fine inside the ABORT handler though; Postgres is going to clean * everything up before control passes back to us. * * If it swallows any error, returns true. Otherwise, returns false. */ static bool SwallowErrors(void (*func)()) { MemoryContext savedContext = CurrentMemoryContext; volatile bool anyErrorSwallowed = false; PG_TRY(); { func(); } PG_CATCH(); { MemoryContextSwitchTo(savedContext); ErrorData *edata = CopyErrorData(); FlushErrorState(); /* rethrow as WARNING */ edata->elevel = WARNING; ThrowErrorData(edata); anyErrorSwallowed = true; } PG_END_TRY(); return anyErrorSwallowed; } /* * ResetGlobalVariables resets global variables that * might be changed during the execution of queries. */ static void ResetGlobalVariables() { CurrentCoordinatedTransactionState = COORD_TRANS_NONE; XactModificationLevel = XACT_MODIFICATION_NONE; SetLocalExecutionStatus(LOCAL_EXECUTION_OPTIONAL); FreeSavedExplainPlan(); dlist_init(&InProgressTransactions); activeSetStmts = NULL; ShouldCoordinatedTransactionUse2PC = false; TransactionModifiedNodeMetadata = false; NodeMetadataSyncOnCommit = false; InTopLevelDelegatedFunctionCall = false; InTableTypeConversionFunctionCall = false; CurrentOperationId = INVALID_OPERATION_ID; BeginXactReadOnly = BeginXactReadOnly_NotSet; BeginXactDeferrable = BeginXactDeferrable_NotSet; ResetWorkerErrorIndication(); memset(&AllowedDistributionColumnValue, 0, sizeof(AllowedDistributionColumn)); } /* * CoordinatedSubTransactionCallback is the callback used to implement * distributed ROLLBACK TO SAVEPOINT. */ static void CoordinatedSubTransactionCallback(SubXactEvent event, SubTransactionId subId, SubTransactionId parentSubid, void *arg) { switch (event) { /* * Our sub-transaction stack should be consistent with postgres' internal * transaction stack. In case of subxact begin, postgres calls our * callback after it has pushed the transaction into stack, so we have to * do the same even if worker commands fail, so we PushSubXact() first. * In case of subxact commit, callback is called before pushing subxact to * the postgres transaction stack, so we call PopSubXact() after making sure * worker commands didn't fail. Otherwise, Postgres would roll back that * would cause us to call PopSubXact again. */ case SUBXACT_EVENT_START_SUB: { MemoryContext previousContext = MemoryContextSwitchTo(CitusXactCallbackContext); PushSubXact(subId); if (InCoordinatedTransaction()) { CoordinatedRemoteTransactionsSavepointBegin(subId); } MemoryContextSwitchTo(previousContext); break; } case SUBXACT_EVENT_COMMIT_SUB: { MemoryContext previousContext = MemoryContextSwitchTo(CitusXactCallbackContext); if (InCoordinatedTransaction()) { CoordinatedRemoteTransactionsSavepointRelease(subId); } PopSubXact(subId, true); /* Set CachedDuringCitusCreation to one level lower to represent citus creation is done */ if (GetCitusCreationLevel() == GetCurrentTransactionNestLevel()) { SetCreateCitusTransactionLevel(GetCitusCreationLevel() - 1); } MemoryContextSwitchTo(previousContext); break; } case SUBXACT_EVENT_ABORT_SUB: { MemoryContext previousContext = MemoryContextSwitchTo(CitusXactCallbackContext); /* * Stop showing message for now, will re-enable when executing * the next statement. */ DisableWorkerMessagePropagation(); /* * Given that we aborted, worker error indications can be ignored. */ ResetWorkerErrorIndication(); if (InCoordinatedTransaction()) { CoordinatedRemoteTransactionsSavepointRollback(subId); } PopSubXact(subId, false); /* * Clear MetadataCache table if we're aborting from a CREATE EXTENSION Citus * so that any created OIDs from the table are cleared and invalidated. We * also set CreateCitusTransactionLevel to 0 since subtransaction has been aborted */ if (GetCitusCreationLevel() == GetCurrentTransactionNestLevel()) { InvalidateMetadataSystemCache(); SetCreateCitusTransactionLevel(0); } /* Reset any local replication origin session since subtransaction has been aborted.*/ ResetReplicationOriginLocalSession(); MemoryContextSwitchTo(previousContext); break; } case SUBXACT_EVENT_PRE_COMMIT_SUB: { /* nothing to do */ break; } } } /* * AdjustMaxPreparedTransactions configures the number of available prepared * transaction slots at startup. */ static void AdjustMaxPreparedTransactions(void) { /* * As Citus uses 2PC internally, there always should be some available. As * the default is 0, we increase it to something appropriate * (connections * 2 currently). If the user explicitly configured 2PC, we * leave the configuration alone - there might have been intent behind the * decision. * * find_option is declared static in guc.c for older versions, so we can't * really check if max_prepared_xacts is configured by the user explicitly, * so check if it's value is default. */ struct config_generic *gconf = find_option("max_prepared_transactions", false, false, ERROR); if (gconf->source == PGC_S_DEFAULT) { char newvalue[12]; SafeSnprintf(newvalue, sizeof(newvalue), "%d", MaxConnections * 2); SetConfigOption("max_prepared_transactions", newvalue, PGC_POSTMASTER, PGC_S_OVERRIDE); ereport(LOG, (errmsg("number of prepared transactions has not been " "configured, overriding"), errdetail("max_prepared_transactions is now set to %s", newvalue))); } } /* PushSubXact pushes subId to the stack of active sub-transactions. */ static void PushSubXact(SubTransactionId subId) { /* save provided subId as well as propagated SET LOCAL stmts */ SubXactContext *state = palloc(sizeof(SubXactContext)); state->subId = subId; state->setLocalCmds = activeSetStmts; /* we lazily create hashset when any object is propagated during sub-transaction */ state->propagatedObjects = NULL; /* append to list and reset active set stmts for upcoming sub-xact */ activeSubXactContexts = lappend(activeSubXactContexts, state); activeSetStmts = makeStringInfo(); } /* PopSubXact pops subId from the stack of active sub-transactions. */ static void PopSubXact(SubTransactionId subId, bool commit) { SubXactContext *state = llast(activeSubXactContexts); Assert(state->subId == subId); /* * Free activeSetStmts to avoid memory leaks when we create subxacts * for each row, e.g. in exception handling of UDFs. */ if (activeSetStmts != NULL) { pfree(activeSetStmts->data); pfree(activeSetStmts); } /* * SET LOCAL commands are local to subxact blocks. When a subxact commits * or rolls back, we should roll back our set of SET LOCAL commands to the * ones we had in the upper commit. */ activeSetStmts = state->setLocalCmds; /* * Keep subtransaction's propagated objects at parent transaction * if subtransaction committed. Otherwise, discard them. */ if (commit) { MovePropagatedObjectsToParentTransaction(); } hash_destroy(state->propagatedObjects); /* * Free state to avoid memory leaks when we create subxacts for each row, * e.g. in exception handling of UDFs. */ pfree(state); activeSubXactContexts = list_delete_last(activeSubXactContexts); } /* ActiveSubXactContexts returns the list of active sub-xact context in temporal order. */ List * ActiveSubXactContexts(void) { return activeSubXactContexts; } /* * IsMultiStatementTransaction determines whether the current statement is * part of a bigger multi-statement transaction. This is the case when the * statement is wrapped in a transaction block (comes after BEGIN), or it * is called from a stored procedure or function. */ bool IsMultiStatementTransaction(void) { if (IsTransactionBlock()) { /* in a BEGIN...END block */ return true; } else if (DoBlockLevel > 0) { /* in (a transaction within) a do block */ return true; } else if (StoredProcedureLevel > 0) { /* in (a transaction within) a stored procedure */ return true; } else if (MaybeExecutingUDF() && FunctionOpensTransactionBlock) { /* in a language-handler function call, open a transaction if configured to do so */ return true; } else { return false; } } /* * MaybeExecutingUDF returns true if we are possibly executing a function call. * We use nested level of executor to check this, so this can return true for * CTEs, etc. which also start nested executors. * * If the planner is being called from the executor, then we may also be in * a UDF. */ bool MaybeExecutingUDF(void) { return ExecutorLevel > 1 || (ExecutorLevel == 1 && PlannerLevel > 0); } /* * TriggerNodeMetadataSyncOnCommit sets a flag to do node metadata sync * on commit. This is because new metadata only becomes visible to the * metadata sync daemon after commit happens. */ void TriggerNodeMetadataSyncOnCommit(void) { NodeMetadataSyncOnCommit = true; } /* * Function raises an exception, if the current backend started a coordinated * transaction and got a PREPARE event to become a participant in a 2PC * transaction coordinated by another node. */ static void EnsurePrepareTransactionIsAllowed(void) { if (!InCoordinatedTransaction()) { /* If the backend has not started a coordinated transaction. */ return; } if (IsCitusInternalBackend()) { /* * If this is a Citus-initiated backend. */ return; } ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot use 2PC in transactions involving " "multiple servers"))); } /* * CurrentTransactionPropagatedObjects returns the objects propagated in current * sub-transaction or the root transaction if no sub-transaction exists. * * If the propagated objects are readonly it will not create the hashmap if it does not * already exist in the current sub-transaction. */ static HTAB * CurrentTransactionPropagatedObjects(bool readonly) { if (activeSubXactContexts == NIL) { /* hashset in the root transaction if there is no sub-transaction */ if (PropagatedObjectsInTx == NULL && !readonly) { /* lazily create hashset for root transaction, for mutating uses */ PropagatedObjectsInTx = CreateTxPropagatedObjectsHash(); } return PropagatedObjectsInTx; } /* hashset in top level sub-transaction */ SubXactContext *state = llast(activeSubXactContexts); if (state->propagatedObjects == NULL && !readonly) { /* lazily create hashset for sub-transaction, for mutating uses */ state->propagatedObjects = CreateTxPropagatedObjectsHash(); } return state->propagatedObjects; } /* * ParentTransactionPropagatedObjects returns the objects propagated in parent * transaction of active sub-transaction. It returns the root transaction if * no sub-transaction exists. * * If the propagated objects are readonly it will not create the hashmap if it does not * already exist in the target sub-transaction. */ static HTAB * ParentTransactionPropagatedObjects(bool readonly) { int nestingLevel = list_length(activeSubXactContexts); if (nestingLevel <= 1) { /* * The parent is the root transaction, when there is single level sub-transaction * or no sub-transaction. */ if (PropagatedObjectsInTx == NULL && !readonly) { /* lazily create hashset for root transaction, for mutating uses */ PropagatedObjectsInTx = CreateTxPropagatedObjectsHash(); } return PropagatedObjectsInTx; } /* parent is upper sub-transaction */ Assert(nestingLevel >= 2); SubXactContext *state = list_nth(activeSubXactContexts, nestingLevel - 2); if (state->propagatedObjects == NULL && !readonly) { /* lazily create hashset for parent sub-transaction */ state->propagatedObjects = CreateTxPropagatedObjectsHash(); } return state->propagatedObjects; } /* * MovePropagatedObjectsToParentTransaction moves all objects propagated in the current * sub-transaction to the parent transaction. This should only be called when there is * active sub-transaction. */ static void MovePropagatedObjectsToParentTransaction(void) { Assert(llast(activeSubXactContexts) != NULL); HTAB *currentPropagatedObjects = CurrentTransactionPropagatedObjects(true); if (currentPropagatedObjects == NULL) { /* nothing to move */ return; } /* * Only after we know we have objects to move into the parent do we get a handle on * a guaranteed existing parent hash table. This makes sure that the parents only * get populated once there are objects to be tracked. */ HTAB *parentPropagatedObjects = ParentTransactionPropagatedObjects(false); HASH_SEQ_STATUS propagatedObjectsSeq; hash_seq_init(&propagatedObjectsSeq, currentPropagatedObjects); ObjectAddress *objectAddress = NULL; while ((objectAddress = hash_seq_search(&propagatedObjectsSeq)) != NULL) { hash_search(parentPropagatedObjects, objectAddress, HASH_ENTER, NULL); } } /* * DependencyInPropagatedObjectsHash checks if dependency is in given hashset * of propagated objects. */ static bool DependencyInPropagatedObjectsHash(HTAB *propagatedObjects, const ObjectAddress *dependency) { if (propagatedObjects == NULL) { return false; } bool found = false; hash_search(propagatedObjects, dependency, HASH_FIND, &found); return found; } /* * CreateTxPropagatedObjectsHash creates a hashset to keep track of the objects * propagated in the current root transaction or sub-transaction. */ static HTAB * CreateTxPropagatedObjectsHash(void) { HASHCTL info; memset(&info, 0, sizeof(info)); info.keysize = sizeof(ObjectAddress); info.entrysize = sizeof(ObjectAddress); info.hash = tag_hash; info.hcxt = CitusXactCallbackContext; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION); return hash_create("Tx Propagated Objects", 16, &info, hashFlags); } /* * TrackPropagatedObject adds given object into the objects propagated in the current * sub-transaction. */ void TrackPropagatedObject(const ObjectAddress *objectAddress) { HTAB *currentPropagatedObjects = CurrentTransactionPropagatedObjects(false); hash_search(currentPropagatedObjects, objectAddress, HASH_ENTER, NULL); } /* * TrackPropagatedTableAndSequences adds given table and its sequences to the objects * propagated in the current sub-transaction. */ void TrackPropagatedTableAndSequences(Oid relationId) { /* track table */ ObjectAddress *tableAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*tableAddress, RelationRelationId, relationId); TrackPropagatedObject(tableAddress); /* track its sequences */ List *ownedSeqIdList = getOwnedSequences(relationId); Oid ownedSeqId = InvalidOid; foreach_declared_oid(ownedSeqId, ownedSeqIdList) { ObjectAddress *seqAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*seqAddress, RelationRelationId, ownedSeqId); TrackPropagatedObject(seqAddress); } } /* * ResetPropagatedObjects destroys hashset of propagated objects in the root transaction. */ void ResetPropagatedObjects(void) { hash_destroy(PropagatedObjectsInTx); PropagatedObjectsInTx = NULL; } /* * HasAnyObjectInPropagatedObjects decides if any of the objects in given list are * propagated in the current transaction. */ bool HasAnyObjectInPropagatedObjects(List *objectList) { ObjectAddress *object = NULL; foreach_declared_ptr(object, objectList) { /* first search in root transaction */ if (DependencyInPropagatedObjectsHash(PropagatedObjectsInTx, object)) { return true; } /* search in all nested sub-transactions */ if (activeSubXactContexts == NIL) { continue; } SubXactContext *state = NULL; foreach_declared_ptr(state, activeSubXactContexts) { if (DependencyInPropagatedObjectsHash(state->propagatedObjects, object)) { return true; } } } return false; } ================================================ FILE: src/backend/distributed/transaction/transaction_recovery.c ================================================ /*------------------------------------------------------------------------- * * transaction_recovery.c * * Routines for recovering two-phase commits started by this node if a * failure occurs between prepare and commit/abort. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/relscan.h" #include "access/xact.h" #include "catalog/indexing.h" #include "lib/stringinfo.h" #include "storage/lmgr.h" #include "storage/lock.h" #include "storage/procarray.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/xid8.h" #include "pg_version_constants.h" #include "distributed/backend_data.h" #include "distributed/connection_management.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/pg_dist_transaction.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_recovery.h" #include "distributed/version_compat.h" #include "distributed/worker_manager.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(recover_prepared_transactions); /* Local functions forward declarations */ static int RecoverWorkerTransactions(WorkerNode *workerNode, MultiConnection *connection); static List * PendingWorkerTransactionList(MultiConnection *connection); static bool IsTransactionInProgress(HTAB *activeTransactionNumberSet, char *preparedTransactionName); static bool RecoverPreparedTransactionOnWorker(MultiConnection *connection, char *transactionName, bool shouldCommit); /* * recover_prepared_transactions recovers any pending prepared * transactions started by this node on other nodes. */ Datum recover_prepared_transactions(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); int recoveredTransactionCount = RecoverTwoPhaseCommits(); PG_RETURN_INT32(recoveredTransactionCount); } /* * LogTransactionRecord registers the fact that a transaction has been * prepared on a worker. The presence of this record indicates that the * prepared transaction should be committed. */ void LogTransactionRecord(int32 groupId, char *transactionName, FullTransactionId outerXid) { /* open transaction relation */ Relation pgDistTransaction = table_open(DistTransactionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); /* form new transaction tuple */ Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isNulls = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); values[Anum_pg_dist_transaction_groupid - 1] = Int32GetDatum(groupId); values[Anum_pg_dist_transaction_gid - 1] = CStringGetTextDatum(transactionName); values[GetOuterXidAttrIndexInPgDistTransaction(tupleDescriptor)] = FullTransactionIdGetDatum(outerXid); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); /* insert new tuple */ PushActiveSnapshot(GetTransactionSnapshot()); CatalogTupleInsert(pgDistTransaction, heapTuple); PopActiveSnapshot(); CommandCounterIncrement(); /* close relation and invalidate previous cache entry */ table_close(pgDistTransaction, NoLock); pfree(values); pfree(isNulls); } /* * RecoverTwoPhaseCommits recovers any pending prepared * transactions started by this node on other nodes. */ int RecoverTwoPhaseCommits(void) { int recoveredTransactionCount = 0; /* take advisory lock first to avoid running concurrently */ LockTransactionRecovery(ShareUpdateExclusiveLock); List *workerList = ActivePrimaryNodeList(NoLock); List *workerConnections = NIL; WorkerNode *workerNode = NULL; MultiConnection *connection = NULL; /* * Pre-establish all connections to worker nodes. * * We do this to enforce a consistent lock acquisition order and prevent deadlocks. * Currently, during extension updates, we take strong locks on the Citus * catalog tables in a specific order: first on pg_dist_authinfo, then on * pg_dist_transaction. It's critical that any operation locking these two * tables adheres to this order, or a deadlock could occur. * * Note that RecoverWorkerTransactions() retains its lock until the end * of the transaction, while GetNodeConnection() releases its lock after * the catalog lookup. So when there are multiple workers in the active primary * node list, the lock acquisition order may reverse in subsequent iterations * of the loop calling RecoverWorkerTransactions(), increasing the risk * of deadlock. * * By establishing all worker connections upfront, we ensure that * RecoverWorkerTransactions() deals with a single distributed catalog table, * thereby preventing deadlocks regardless of the lock acquisition sequence * used in the upgrade extension script. */ foreach_declared_ptr(workerNode, workerList) { int connectionFlags = 0; char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; connection = GetNodeConnection(connectionFlags, nodeName, nodePort); Assert(connection != NULL); /* * We don't verify connection validity here. * Instead, RecoverWorkerTransactions() performs the necessary * sanity checks on the connection state. */ workerConnections = lappend(workerConnections, connection); } forboth_ptr(workerNode, workerList, connection, workerConnections) { recoveredTransactionCount += RecoverWorkerTransactions(workerNode, connection); } return recoveredTransactionCount; } /* * RecoverWorkerTransactions recovers any pending prepared transactions * started by this node on the specified worker. */ static int RecoverWorkerTransactions(WorkerNode *workerNode, MultiConnection *connection) { int recoveredTransactionCount = 0; int32 groupId = workerNode->groupId; char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; ScanKeyData scanKey[1]; int scanKeyCount = 1; bool indexOK = true; HeapTuple heapTuple = NULL; HASH_SEQ_STATUS status; bool recoveryFailed = false; Assert(connection != NULL); if (connection->pgConn == NULL || PQstatus(connection->pgConn) != CONNECTION_OK) { ereport(WARNING, (errmsg("transaction recovery cannot connect to %s:%d", nodeName, nodePort))); return 0; } MemoryContext localContext = AllocSetContextCreateInternal(CurrentMemoryContext, "RecoverWorkerTransactions", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); MemoryContext oldContext = MemoryContextSwitchTo(localContext); Relation pgDistTransaction = table_open(DistTransactionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); /* * We're going to check the list of prepared transactions on the worker, * but some of those prepared transactions might belong to ongoing * distributed transactions. * * We could avoid this by temporarily blocking new prepared transactions * from being created by taking an ExclusiveLock on pg_dist_transaction. * However, this hurts write performance, so instead we avoid blocking * by consulting the list of active distributed transactions, and follow * a carefully chosen order to avoid race conditions: * * 1) P = prepared transactions on worker * 2) A = active distributed transactions * 3) T = pg_dist_transaction snapshot * 4) Q = prepared transactions on worker * * By observing A after P, we get a conclusive answer to which distributed * transactions we observed in P are still in progress. It is safe to recover * the transactions in P - A based on the presence or absence of a record * in T. * * We also remove records in T if there is no prepared transaction, which * we assume means the transaction committed. However, a transaction could * have left prepared transactions and committed between steps 1 and 2. * In that case, we would incorrectly remove the records, while the * prepared transaction is still in place. * * We therefore observe the set of prepared transactions one more time in * step 4. The aforementioned transactions would show up in Q, but not in * P. We can skip those transactions and recover them later. */ /* find stale prepared transactions on the remote node */ List *pendingTransactionList = PendingWorkerTransactionList(connection); HTAB *pendingTransactionSet = ListToHashSet(pendingTransactionList, NAMEDATALEN, true); /* find in-progress distributed transactions */ List *activeTransactionNumberList = ActiveDistributedTransactionNumbers(); HTAB *activeTransactionNumberSet = ListToHashSet(activeTransactionNumberList, sizeof(uint64), false); /* scan through all recovery records of the current worker */ ScanKeyInit(&scanKey[0], Anum_pg_dist_transaction_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); /* get a snapshot of pg_dist_transaction */ SysScanDesc scanDescriptor = systable_beginscan(pgDistTransaction, DistTransactionGroupIndexId(), indexOK, NULL, scanKeyCount, scanKey); /* find stale prepared transactions on the remote node */ List *recheckTransactionList = PendingWorkerTransactionList(connection); HTAB *recheckTransactionSet = ListToHashSet(recheckTransactionList, NAMEDATALEN, true); while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { bool isNull = false; bool foundPreparedTransactionBeforeCommit = false; bool foundPreparedTransactionAfterCommit = false; Datum transactionNameDatum = heap_getattr(heapTuple, Anum_pg_dist_transaction_gid, tupleDescriptor, &isNull); char *transactionName = TextDatumGetCString(transactionNameDatum); bool isTransactionInProgress = IsTransactionInProgress(activeTransactionNumberSet, transactionName); if (isTransactionInProgress) { /* * Do not touch in progress transactions as we might mistakenly * commit a transaction that is actually in the process of * aborting or vice-versa. */ continue; } bool outerXidIsNull = false; Datum outerXidDatum = 0; if (EnableVersionChecks || SearchSysCacheExistsAttName(DistTransactionRelationId(), "outer_xid")) { /* Check if the transaction is created by an outer transaction from a non-main database */ outerXidDatum = heap_getattr(heapTuple, GetOuterXidAttrIndexInPgDistTransaction(tupleDescriptor) + 1, tupleDescriptor, &outerXidIsNull); } else { /* * Normally we don't try to recover prepared transactions when the * binary version doesn't match the sql version. However, we skip * those checks in regression tests by disabling * citus.enable_version_checks. And when this is the case, while * the C code looks for "outer_xid" attribute, pg_dist_transaction * doesn't yet have it. */ Assert(!EnableVersionChecks); } TransactionId outerXid = 0; if (!outerXidIsNull) { FullTransactionId outerFullXid = DatumGetFullTransactionId(outerXidDatum); outerXid = XidFromFullTransactionId(outerFullXid); } if (outerXid != 0) { bool outerXactIsInProgress = TransactionIdIsInProgress(outerXid); bool outerXactDidCommit = TransactionIdDidCommit(outerXid); if (outerXactIsInProgress && !outerXactDidCommit) { /* * The transaction is initiated from an outer transaction and the outer * transaction is not yet committed, so we should not commit either. * We remove this transaction from the pendingTransactionSet so it'll * not be aborted by the loop below. */ hash_search(pendingTransactionSet, transactionName, HASH_REMOVE, &foundPreparedTransactionBeforeCommit); continue; } else if (!outerXactIsInProgress && !outerXactDidCommit) { /* * Since outer transaction isn't in progress and did not commit we need to * abort the prepared transaction too. We do this by simply doing the same * thing we would do for transactions that are initiated from the main * database. */ continue; } else { /* * Outer transaction did commit, so we can try to commit the prepared * transaction too. */ } } /* * Remove the transaction from the pending list such that only transactions * that need to be aborted remain at the end. */ hash_search(pendingTransactionSet, transactionName, HASH_REMOVE, &foundPreparedTransactionBeforeCommit); hash_search(recheckTransactionSet, transactionName, HASH_FIND, &foundPreparedTransactionAfterCommit); if (foundPreparedTransactionBeforeCommit && foundPreparedTransactionAfterCommit) { /* * The transaction was committed, but the prepared transaction still exists * on the worker. Try committing it. * * We double check that the recovery record exists both before and after * checking ActiveDistributedTransactionNumbers(), since we may have * observed a prepared transaction that was committed immediately after. */ bool shouldCommit = true; bool commitSucceeded = RecoverPreparedTransactionOnWorker(connection, transactionName, shouldCommit); if (!commitSucceeded) { /* * Failed to commit on the current worker. Stop without throwing * an error to allow recover_prepared_transactions to continue with * other workers. */ recoveryFailed = true; break; } recoveredTransactionCount++; /* * We successfully committed the prepared transaction, safe to delete * the recovery record. */ } else if (foundPreparedTransactionAfterCommit) { /* * We found a committed pg_dist_transaction record that initially did * not have a prepared transaction, but did when we checked again. * * If a transaction started and committed just after we observed the * set of prepared transactions, and just before we called * ActiveDistributedTransactionNumbers, then we would see a recovery * record without a prepared transaction in pendingTransactionSet, * but there may be prepared transactions that failed to commit. * We should not delete the records for those prepared transactions, * since we would otherwise roll back them on the next call to * recover_prepared_transactions. * * In addition, if the transaction started after the call to * ActiveDistributedTransactionNumbers and finished just before our * pg_dist_transaction snapshot, then it may still be in the process * of committing the prepared transactions in the post-commit callback * and we should not touch the prepared transactions. * * To handle these cases, we just leave the records and prepared * transactions for the next call to recover_prepared_transactions * and skip them here. */ continue; } else { /* * We found a recovery record without any prepared transaction. It * must have already been committed, so it's safe to delete the * recovery record. * * Transactions that started after we observed pendingTransactionSet, * but successfully committed their prepared transactions before * ActiveDistributedTransactionNumbers are indistinguishable from * transactions that committed at an earlier time, in which case it's * safe delete the recovery record as well. */ } simple_heap_delete(pgDistTransaction, &heapTuple->t_self); } systable_endscan(scanDescriptor); table_close(pgDistTransaction, NoLock); if (!recoveryFailed) { char *pendingTransactionName = NULL; bool abortSucceeded = true; /* * All remaining prepared transactions that are not part of an in-progress * distributed transaction should be aborted since we did not find a recovery * record, which implies the disributed transaction aborted. */ hash_seq_init(&status, pendingTransactionSet); while ((pendingTransactionName = hash_seq_search(&status)) != NULL) { bool isTransactionInProgress = IsTransactionInProgress( activeTransactionNumberSet, pendingTransactionName); if (isTransactionInProgress) { continue; } bool shouldCommit = false; abortSucceeded = RecoverPreparedTransactionOnWorker(connection, pendingTransactionName, shouldCommit); if (!abortSucceeded) { hash_seq_term(&status); break; } recoveredTransactionCount++; } } MemoryContextSwitchTo(oldContext); MemoryContextDelete(localContext); return recoveredTransactionCount; } /* * PendingWorkerTransactionList returns a list of pending prepared * transactions on a remote node that were started by this node. */ static List * PendingWorkerTransactionList(MultiConnection *connection) { StringInfo command = makeStringInfo(); bool raiseInterrupts = true; List *transactionNames = NIL; int32 coordinatorId = GetLocalGroupId(); appendStringInfo(command, "SELECT gid FROM pg_prepared_xacts " "WHERE gid COLLATE pg_catalog.default LIKE 'citus\\_%d\\_%%' COLLATE pg_catalog.default AND database = current_database()", coordinatorId); int querySent = SendRemoteCommand(connection, command->data); if (querySent == 0) { ReportConnectionError(connection, ERROR); } PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } int rowCount = PQntuples(result); for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { const int columnIndex = 0; char *transactionName = PQgetvalue(result, rowIndex, columnIndex); transactionNames = lappend(transactionNames, pstrdup(transactionName)); } PQclear(result); ForgetResults(connection); return transactionNames; } /* * IsTransactionInProgress returns whether the distributed transaction to which * preparedTransactionName belongs is still in progress, or false if the * transaction name cannot be parsed. This can happen when the user manually * inserts into pg_dist_transaction. */ static bool IsTransactionInProgress(HTAB *activeTransactionNumberSet, char *preparedTransactionName) { int32 groupId = 0; int procId = 0; uint32 connectionNumber = 0; uint64 transactionNumber = 0; bool isTransactionInProgress = false; bool isValidName = ParsePreparedTransactionName(preparedTransactionName, &groupId, &procId, &transactionNumber, &connectionNumber); if (isValidName) { hash_search(activeTransactionNumberSet, &transactionNumber, HASH_FIND, &isTransactionInProgress); } return isTransactionInProgress; } /* * RecoverPreparedTransactionOnWorker recovers a single prepared transaction over * the given connection. If shouldCommit is true we send */ static bool RecoverPreparedTransactionOnWorker(MultiConnection *connection, char *transactionName, bool shouldCommit) { StringInfo command = makeStringInfo(); PGresult *result = NULL; bool raiseInterrupts = false; if (shouldCommit) { /* should have committed this prepared transaction */ appendStringInfo(command, "COMMIT PREPARED %s", quote_literal_cstr(transactionName)); } else { /* should have aborted this prepared transaction */ appendStringInfo(command, "ROLLBACK PREPARED %s", quote_literal_cstr(transactionName)); } int executeCommand = ExecuteOptionalRemoteCommand(connection, command->data, &result); if (executeCommand == QUERY_SEND_FAILED) { return false; } if (executeCommand == RESPONSE_NOT_OKAY) { return false; } PQclear(result); ClearResults(connection, raiseInterrupts); ereport(LOG, (errmsg("recovered a prepared transaction on %s:%d", connection->hostname, connection->port), errcontext("%s", command->data))); return true; } /* * DeleteWorkerTransactions deletes the entries on pg_dist_transaction for a given * worker node. It's implemented to be called at master_remove_node. */ void DeleteWorkerTransactions(WorkerNode *workerNode) { if (workerNode == NULL) { /* * We don't expect this, but let's be defensive since crashing is much worse * than leaving pg_dist_transction entries. */ return; } bool indexOK = true; int scanKeyCount = 1; ScanKeyData scanKey[1]; int32 groupId = workerNode->groupId; HeapTuple heapTuple = NULL; Relation pgDistTransaction = table_open(DistTransactionRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_transaction_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistTransaction, DistTransactionGroupIndexId(), indexOK, NULL, scanKeyCount, scanKey); while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) { simple_heap_delete(pgDistTransaction, &heapTuple->t_self); } CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistTransaction, NoLock); } /* * GetOuterXidAttrIndexInPgDistTransaction returns attrnum for outer_xid attr. * * outer_xid attr was added to table pg_dist_transaction using alter operation after * the version where Citus started supporting downgrades, and it's only column that we've * introduced to pg_dist_transaction since then. * * And in case of a downgrade + upgrade, tupleDesc->natts becomes greater than * Natts_pg_dist_transaction and when this happens, then we know that attrnum outer_xid is * not Anum_pg_dist_transaction_outerxid anymore but tupleDesc->natts - 1. */ int GetOuterXidAttrIndexInPgDistTransaction(TupleDesc tupleDesc) { return tupleDesc->natts == Natts_pg_dist_transaction ? (Anum_pg_dist_transaction_outerxid - 1) : tupleDesc->natts - 1; } ================================================ FILE: src/backend/distributed/transaction/worker_transaction.c ================================================ /*------------------------------------------------------------------------- * * worker_transaction.c * * Routines for performing transactions across all workers. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "fmgr.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/xact.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/uuid.h" #include "distributed/connection_management.h" #include "distributed/jsonbutils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/pg_dist_node.h" #include "distributed/pg_dist_transaction.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/transaction_recovery.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" PG_FUNCTION_INFO_V1(citus_server_id); static void SendBareCommandListToMetadataNodesInternal(List *commandList, TargetWorkerSet targetWorkerSet); static void SendCommandToMetadataWorkersParams(const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues); static void SendCommandToWorkersParamsInternal(TargetWorkerSet targetWorkerSet, const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues); static void ErrorIfAnyMetadataNodeOutOfSync(List *metadataNodeList); /* * SendCommandToWorker sends a command to a particular worker as part of the * 2PC. */ void SendCommandToWorker(const char *nodeName, int32 nodePort, const char *command) { const char *nodeUser = CurrentUserName(); SendCommandToWorkerAsUser(nodeName, nodePort, nodeUser, command); } /* * SendCommandToWorkersAsUser sends a command to targetWorkerSet as a particular user * as part of the 2PC. */ void SendCommandToWorkersAsUser(TargetWorkerSet targetWorkerSet, const char *nodeUser, const char *command) { List *workerNodeList = TargetWorkerSetNodeList(targetWorkerSet, RowShareLock); /* run commands serially */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; SendCommandToWorkerAsUser(nodeName, nodePort, nodeUser, command); } } /* * SendCommandToWorkerAsUser sends a command to a particular worker as a particular user * as part of the 2PC. */ void SendCommandToWorkerAsUser(const char *nodeName, int32 nodePort, const char *nodeUser, const char *command) { uint32 connectionFlags = 0; UseCoordinatedTransaction(); Use2PCForCoordinatedTransaction(); MultiConnection *transactionConnection = GetNodeUserDatabaseConnection( connectionFlags, nodeName, nodePort, nodeUser, NULL); MarkRemoteTransactionCritical(transactionConnection); RemoteTransactionBeginIfNecessary(transactionConnection); ExecuteCriticalRemoteCommand(transactionConnection, command); } /* * SendCommandToWorkers sends a command to all workers in * parallel. Commands are committed on the workers when the local * transaction commits. */ void SendCommandToWorkersWithMetadata(const char *command) { SendCommandToMetadataWorkersParams(command, CurrentUserName(), 0, NULL, NULL); } /* * SendCommandToWorkersWithMetadataViaSuperUser sends a command to all workers in * parallel by opening a super user connection. Commands are committed on the workers * when the local transaction commits. The connection are made as the extension * owner to ensure write access to the Citus metadata tables. * * Since we prevent to open superuser connections for metadata tables, it is * discouraged to use it. Consider using it only for propagating pg_dist_object * tuples for dependent objects. */ void SendCommandToWorkersWithMetadataViaSuperUser(const char *command) { SendCommandToMetadataWorkersParams(command, CitusExtensionOwnerName(), 0, NULL, NULL); } /* * SendCommandListToWorkersWithMetadata sends all commands to all metadata workers * with the current user. See `SendCommandToWorkersWithMetadata`for details. */ void SendCommandListToWorkersWithMetadata(List *commands) { char *command = NULL; foreach_declared_ptr(command, commands) { SendCommandToWorkersWithMetadata(command); } } /* * SendCommandToRemoteNodesWithMetadata sends a command to remote nodes in * parallel. Commands are committed on the nodes when the local transaction * commits. */ void SendCommandToRemoteNodesWithMetadata(const char *command) { SendCommandToRemoteMetadataNodesParams(command, CurrentUserName(), 0, NULL, NULL); } /* * SendCommandToRemoteNodesWithMetadataViaSuperUser sends a command to remote * nodes in parallel by opening a super user connection. Commands are committed * on the nodes when the local transaction commits. The connection are made as * the extension owner to ensure write access to the Citus metadata tables. * * Since we prevent to open superuser connections for metadata tables, it is * discouraged to use it. Consider using it only for propagating pg_dist_object * tuples for dependent objects. */ void SendCommandToRemoteNodesWithMetadataViaSuperUser(const char *command) { SendCommandToRemoteMetadataNodesParams(command, CitusExtensionOwnerName(), 0, NULL, NULL); } /* * SendCommandListToRemoteNodesWithMetadata sends all commands to remote nodes * with the current user. See `SendCommandToRemoteNodesWithMetadata`for details. */ void SendCommandListToRemoteNodesWithMetadata(List *commands) { char *command = NULL; foreach_declared_ptr(command, commands) { SendCommandToRemoteNodesWithMetadata(command); } } /* * SendCommandToRemoteMetadataNodesParams is a wrapper around * SendCommandToWorkersParamsInternal() that can be used to send commands * to remote metadata nodes. */ void SendCommandToRemoteMetadataNodesParams(const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues) { /* use METADATA_NODES so that ErrorIfAnyMetadataNodeOutOfSync checks local node as well */ List *workerNodeList = TargetWorkerSetNodeList(METADATA_NODES, RowShareLock); ErrorIfAnyMetadataNodeOutOfSync(workerNodeList); SendCommandToWorkersParamsInternal(REMOTE_METADATA_NODES, command, user, parameterCount, parameterTypes, parameterValues); } /* * TargetWorkerSetNodeList returns a list of WorkerNode's that satisfies the * TargetWorkerSet. */ List * TargetWorkerSetNodeList(TargetWorkerSet targetWorkerSet, LOCKMODE lockMode) { List *workerNodeList = NIL; if (targetWorkerSet == ALL_SHARD_NODES || targetWorkerSet == METADATA_NODES) { workerNodeList = ActivePrimaryNodeList(lockMode); } else if (targetWorkerSet == REMOTE_NODES || targetWorkerSet == REMOTE_METADATA_NODES) { workerNodeList = ActivePrimaryRemoteNodeList(lockMode); } else if (targetWorkerSet == NON_COORDINATOR_METADATA_NODES || targetWorkerSet == NON_COORDINATOR_NODES) { workerNodeList = ActivePrimaryNonCoordinatorNodeList(lockMode); } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid target worker set: %d", targetWorkerSet))); } List *result = NIL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if ((targetWorkerSet == NON_COORDINATOR_METADATA_NODES || targetWorkerSet == REMOTE_METADATA_NODES || targetWorkerSet == METADATA_NODES) && !workerNode->hasMetadata) { continue; } result = lappend(result, workerNode); } return result; } /* * SendBareCommandListToRemoteMetadataNodes is a wrapper around * SendBareCommandListToMetadataNodesInternal() that can be used to send * bare commands to remote metadata nodes. */ void SendBareCommandListToRemoteMetadataNodes(List *commandList) { SendBareCommandListToMetadataNodesInternal(commandList, REMOTE_METADATA_NODES); } /* * SendBareCommandListToMetadataWorkers is a wrapper around * SendBareCommandListToMetadataNodesInternal() that can be used to send * bare commands to metadata workers. */ void SendBareCommandListToMetadataWorkers(List *commandList) { SendBareCommandListToMetadataNodesInternal(commandList, NON_COORDINATOR_METADATA_NODES); } /* * SendBareCommandListToMetadataNodesInternal sends a list of commands to given * target worker set in serial. Commands are committed immediately: new connections * are always used and no transaction block is used (hence "bare"). The connections * are made as the extension owner to ensure write access to the Citus metadata * tables. Primarly useful for INDEX commands using CONCURRENTLY. */ static void SendBareCommandListToMetadataNodesInternal(List *commandList, TargetWorkerSet targetWorkerSet) { List *workerNodeList = TargetWorkerSetNodeList(targetWorkerSet, RowShareLock); char *nodeUser = CurrentUserName(); ErrorIfAnyMetadataNodeOutOfSync(workerNodeList); /* run commands serially */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int connectionFlags = FORCE_NEW_CONNECTION; MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, nodeUser, NULL); /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; foreach_declared_ptr(commandString, commandList) { ExecuteCriticalRemoteCommand(workerConnection, commandString); } CloseConnection(workerConnection); } } /* * SendCommandToMetadataWorkersParams is a wrapper around * SendCommandToWorkersParamsInternal() enforcing some extra checks. */ static void SendCommandToMetadataWorkersParams(const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues) { List *workerNodeList = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); ErrorIfAnyMetadataNodeOutOfSync(workerNodeList); SendCommandToWorkersParamsInternal(NON_COORDINATOR_METADATA_NODES, command, user, parameterCount, parameterTypes, parameterValues); } /* * SendCommandToWorkersParamsInternal sends a command to all workers in parallel. * Commands are committed on the workers when the local transaction commits. The * connection are made as the extension owner to ensure write access to the Citus * metadata tables. Parameters can be specified as for PQexecParams, except that * paramLengths, paramFormats and resultFormat are hard-coded to NULL, NULL and 0 * respectively. */ static void SendCommandToWorkersParamsInternal(TargetWorkerSet targetWorkerSet, const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues) { List *connectionList = NIL; List *workerNodeList = TargetWorkerSetNodeList(targetWorkerSet, RowShareLock); UseCoordinatedTransaction(); Use2PCForCoordinatedTransaction(); /* open connections in parallel */ WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int32 connectionFlags = REQUIRE_METADATA_CONNECTION; MultiConnection *connection = StartNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, user, NULL); /* * connection can only be NULL for optional connections, which we don't * support in this codepath. */ Assert((connectionFlags & OPTIONAL_CONNECTION) == 0); Assert(connection != NULL); MarkRemoteTransactionCritical(connection); connectionList = lappend(connectionList, connection); } /* finish opening connections */ FinishConnectionListEstablishment(connectionList); RemoteTransactionsBeginIfNecessary(connectionList); /* send commands in parallel */ MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommandParams(connection, command, parameterCount, parameterTypes, parameterValues, false); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } /* get results */ foreach_declared_ptr(connection, connectionList) { PGresult *result = GetRemoteCommandResult(connection, true); if (!IsResponseOK(result)) { ReportResultError(connection, result, ERROR); } PQclear(result); ForgetResults(connection); } } /* * EnsureNoModificationsHaveBeenDone reports an error if we have performed any * modification in the current transaction to prevent opening a connection is such cases. */ void EnsureNoModificationsHaveBeenDone() { if (XactModificationLevel > XACT_MODIFICATION_NONE) { ereport(ERROR, (errcode(ERRCODE_ACTIVE_SQL_TRANSACTION), errmsg("cannot open new connections after the first modification " "command within a transaction"))); } } /* * SendCommandListToWorkerOutsideTransaction forces to open a new connection * to the node with the given nodeName and nodePort. Then, the connection starts * a transaction on the remote node and executes the commands in the transaction. * The function raises error if any of the queries fails. */ void SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort, const char *nodeUser, List *commandList) { int connectionFlags = FORCE_NEW_CONNECTION; MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, nodeUser, NULL); SendCommandListToWorkerOutsideTransactionWithConnection(workerConnection, commandList); CloseConnection(workerConnection); } /* * SendCommandListToWorkerOutsideTransactionWithConnection sends the command list * over the specified connection. This opens a new transaction on the * connection, thus it's important that no transaction is currently open. * This function is mainly useful to avoid opening an closing * connections excessively by allowing reusing a single connection to send * multiple separately committing transactions. The function raises an error if * any of the queries fail. */ void SendCommandListToWorkerOutsideTransactionWithConnection(MultiConnection *workerConnection, List *commandList) { MarkRemoteTransactionCritical(workerConnection); RemoteTransactionBegin(workerConnection); /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; foreach_declared_ptr(commandString, commandList) { ExecuteCriticalRemoteCommand(workerConnection, commandString); } RemoteTransactionCommit(workerConnection); ResetRemoteTransaction(workerConnection); } /* * SendCommandListToWorkerListWithBareConnections sends the command list * over the specified bare connections. This function is mainly useful to * avoid opening an closing connections excessively by allowing reusing * connections to send multiple separate bare commands. The function * raises an error if any of the queries fail. */ void SendCommandListToWorkerListWithBareConnections(List *workerConnectionList, List *commandList) { Assert(!InCoordinatedTransaction()); Assert(!GetCoordinatedTransactionShouldUse2PC()); if (list_length(commandList) == 0 || list_length(workerConnectionList) == 0) { /* nothing to do */ return; } /* * In order to avoid round-trips per query in queryStringList, * we join the string and send as a single command. Also, * if there is only a single command, avoid additional call to * StringJoin given that some strings can be quite large. */ char *stringToSend = (list_length(commandList) == 1) ? linitial(commandList) : StringJoin(commandList, ';'); /* send commands in parallel */ MultiConnection *connection = NULL; foreach_declared_ptr(connection, workerConnectionList) { int querySent = SendRemoteCommand(connection, stringToSend); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } bool failOnError = true; foreach_declared_ptr(connection, workerConnectionList) { ClearResults(connection, failOnError); } } /* * SendCommandListToWorkerInCoordinatedTransaction opens connection to the node * with the given nodeName and nodePort. The commands are sent as part of the * coordinated transaction. Any failures aborts the coordinated transaction. */ void SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList, const char *nodeUser, List *commandList) { if (list_length(commandList) == 0 || list_length(workerNodeList) == 0) { /* nothing to do */ return; } ErrorIfAnyMetadataNodeOutOfSync(workerNodeList); UseCoordinatedTransaction(); List *connectionList = NIL; WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { const char *nodeName = workerNode->workerName; int nodePort = workerNode->workerPort; int connectionFlags = REQUIRE_METADATA_CONNECTION; MultiConnection *connection = StartNodeConnection(connectionFlags, nodeName, nodePort); MarkRemoteTransactionCritical(connection); /* * connection can only be NULL for optional connections, which we don't * support in this codepath. */ Assert((connectionFlags & OPTIONAL_CONNECTION) == 0); Assert(connection != NULL); connectionList = lappend(connectionList, connection); } FinishConnectionListEstablishment(connectionList); /* must open transaction blocks to use intermediate results */ RemoteTransactionsBeginIfNecessary(connectionList); /* * In order to avoid round-trips per query in queryStringList, * we join the string and send as a single command. Also, * if there is only a single command, avoid additional call to * StringJoin given that some strings can be quite large. */ char *stringToSend = (list_length(commandList) == 1) ? linitial(commandList) : StringJoin(commandList, ';'); /* send commands in parallel */ bool failOnError = true; MultiConnection *connection = NULL; foreach_declared_ptr(connection, connectionList) { int querySent = SendRemoteCommand(connection, stringToSend); if (querySent == 0) { ReportConnectionError(connection, ERROR); } } foreach_declared_ptr(connection, connectionList) { ClearResults(connection, failOnError); } } /* * SendOptionalCommandListToWorkerOutsideTransactionWithConnection sends the * given command list over a specified connection in a single transaction that * is outside of the coordinated tranaction. * * If any of the commands fail, it rollbacks the transaction, and otherwise commits. * A successful commit is indicated by returning true, and a failed commit by returning * false. */ bool SendOptionalCommandListToWorkerOutsideTransactionWithConnection(MultiConnection * workerConnection, List * commandList) { if (PQstatus(workerConnection->pgConn) != CONNECTION_OK) { return false; } RemoteTransactionBegin(workerConnection); /* iterate over the commands and execute them in the same connection */ bool failed = false; const char *commandString = NULL; foreach_declared_ptr(commandString, commandList) { if (ExecuteOptionalRemoteCommand(workerConnection, commandString, NULL) != 0) { failed = true; break; } } if (failed) { RemoteTransactionAbort(workerConnection); } else { RemoteTransactionCommit(workerConnection); } ResetRemoteTransaction(workerConnection); return !failed; } /* * SendOptionalCommandListToWorkerOutsideTransaction sends the given command * list to the given worker in a single transaction that is outside of the * coordinated tranaction. If any of the commands fail, it rollbacks the * transaction, and otherwise commits. */ bool SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort, const char *nodeUser, List *commandList) { int connectionFlags = FORCE_NEW_CONNECTION; MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, nodeUser, NULL); bool failed = SendOptionalCommandListToWorkerOutsideTransactionWithConnection( workerConnection, commandList); CloseConnection(workerConnection); return !failed; } /* * SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction sends the given * command list to the given worker as part of the coordinated transaction. * If any of the commands fail, the function returns false. */ bool SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(const char *nodeName, int32 nodePort, const char *nodeUser, List *commandList) { int connectionFlags = REQUIRE_METADATA_CONNECTION; bool failed = false; UseCoordinatedTransaction(); MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, nodeName, nodePort, nodeUser, NULL); if (PQstatus(workerConnection->pgConn) != CONNECTION_OK) { return false; } RemoteTransactionsBeginIfNecessary(list_make1(workerConnection)); /* iterate over the commands and execute them in the same connection */ const char *commandString = NULL; foreach_declared_ptr(commandString, commandList) { if (ExecuteOptionalRemoteCommand(workerConnection, commandString, NULL) != RESPONSE_OKAY) { failed = true; bool raiseErrors = false; MarkRemoteTransactionFailed(workerConnection, raiseErrors); break; } } return !failed; } /* * ErrorIfAnyMetadataNodeOutOfSync raises an error if any of the given * metadata nodes are out of sync. It is safer to avoid metadata changing * commands (e.g. DDL or node addition) until all metadata nodes have * been synced. * * An example of we could get in a bad situation without doing so is: * 1. Create a reference table * 2. After the node becomes out of sync, add a new active node * 3. Insert into the reference table from the out of sync node * * Since the out-of-sync might not know about the new node, it won't propagate * the changes to the new node and replicas will be in an inconsistent state. */ static void ErrorIfAnyMetadataNodeOutOfSync(List *metadataNodeList) { WorkerNode *metadataNode = NULL; foreach_declared_ptr(metadataNode, metadataNodeList) { Assert(metadataNode->hasMetadata); if (!metadataNode->metadataSynced) { const char *workerName = metadataNode->workerName; int workerPort = metadataNode->workerPort; ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("%s:%d is a metadata node, but is out of sync", workerName, workerPort), errhint("If the node is up, wait until metadata" " gets synced to it and try again."))); } } } /* * IsWorkerTheCurrentNode checks if the given worker refers to the * the current node by comparing the server id of the worker and of the * current nodefrom pg_dist_node_metadata */ bool IsWorkerTheCurrentNode(WorkerNode *workerNode) { int connectionFlags = REQUIRE_METADATA_CONNECTION; MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags, workerNode->workerName, workerNode->workerPort, CurrentUserName(), NULL); const char *command = "SELECT metadata ->> 'server_id' AS server_id FROM pg_dist_node_metadata"; int resultCode = SendRemoteCommand(workerConnection, command); if (resultCode == 0) { CloseConnection(workerConnection); return false; } PGresult *result = GetRemoteCommandResult(workerConnection, true); if (result == NULL) { return false; } List *commandResult = ReadFirstColumnAsText(result); PQclear(result); ForgetResults(workerConnection); if ((list_length(commandResult) != 1)) { return false; } StringInfo resultInfo = (StringInfo) linitial(commandResult); char *workerServerId = resultInfo->data; Datum metadata = DistNodeMetadata(); text *currentServerIdTextP = ExtractFieldTextP(metadata, "server_id"); if (currentServerIdTextP == NULL) { return false; } char *currentServerId = text_to_cstring(currentServerIdTextP); return strcmp(workerServerId, currentServerId) == 0; } /* * citus_server_id returns a random UUID value as server identifier. This is * modeled after PostgreSQL's pg_random_uuid(). */ Datum citus_server_id(PG_FUNCTION_ARGS) { uint8 *buf = (uint8 *) palloc(UUID_LEN); /* * If pg_strong_random() fails, fall-back to using random(). In previous * versions of postgres we don't have pg_strong_random(), so use it by * default in that case. */ if (!pg_strong_random((char *) buf, UUID_LEN)) { for (int bufIdx = 0; bufIdx < UUID_LEN; bufIdx++) { buf[bufIdx] = (uint8) (random() & 0xFF); } } /* * Set magic numbers for a "version 4" (pseudorandom) UUID, see * http://tools.ietf.org/html/rfc4122#section-4.4 */ buf[6] = (buf[6] & 0x0f) | 0x40; /* "version" field */ buf[8] = (buf[8] & 0x3f) | 0x80; /* "variant" field */ PG_RETURN_UUID_P((pg_uuid_t *) buf); } ================================================ FILE: src/backend/distributed/utils/acquire_lock.c ================================================ /*------------------------------------------------------------------------- * * acquire_lock.c * A dynamic background worker that can help your backend to acquire its locks. This is * an intrusive way of getting your way. The primary use of this will be to allow * master_update_node to make progress during failure. When the system cannot possibly * finish a transaction due to the host required to finish the transaction has failed * it might be better to actively cancel the backend instead of waiting for it to fail. * * This file provides infrastructure for launching exactly one a background * worker for every database in which citus is used. That background worker * can then perform work like deadlock detection, prepared transaction * recovery, and cleanup. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "access/xact.h" #include "catalog/pg_type.h" #include "executor/spi.h" #include "portability/instr_time.h" #include "storage/ipc.h" #include "storage/latch.h" #include "utils/snapmgr.h" #include "distributed/background_worker_utils.h" #include "distributed/citus_acquire_lock.h" #include "distributed/citus_safe_lib.h" #include "distributed/connection_management.h" #include "distributed/version_compat.h" /* forward declaration of background worker entrypoint */ extern PGDLLEXPORT void LockAcquireHelperMain(Datum main_arg); /* forward declaration of helper functions */ static void lock_acquire_helper_sigterm(SIGNAL_ARGS); static void EnsureStopLockAcquireHelper(void *arg); /* LockAcquireHelperArgs contains extra arguments to be used to start the worker */ typedef struct LockAcquireHelperArgs { Oid DatabaseId; int32 lock_cooldown; } LockAcquireHelperArgs; static bool got_sigterm = false; /* * StartLockAcquireHelperBackgroundWorker creates a background worker that will help the * backend passed in as an argument to complete. The worker that is started will be * terminated once the current memory context gets reset, to make sure it is cleaned up in * all situations. It is however advised to call TerminateBackgroundWorker on the handle * returned on the first possible moment the help is no longer required. */ BackgroundWorkerHandle * StartLockAcquireHelperBackgroundWorker(int backendToHelp, int32 lock_cooldown) { LockAcquireHelperArgs args; memset(&args, 0, sizeof(args)); /* collect the extra arguments required for the background worker */ args.DatabaseId = MyDatabaseId; args.lock_cooldown = lock_cooldown; char workerName[BGW_MAXLEN]; SafeSnprintf(workerName, BGW_MAXLEN, "Citus Lock Acquire Helper: %d/%u", backendToHelp, MyDatabaseId); CitusBackgroundWorkerConfig config = { .workerName = workerName, .functionName = "LockAcquireHelperMain", .mainArg = Int32GetDatum(backendToHelp), .extensionOwner = InvalidOid, .needsNotification = false, .waitForStartup = false, .restartTime = CITUS_BGW_NEVER_RESTART, .startTime = BgWorkerStart_RecoveryFinished, .workerType = "citus_lock_aqcuire", .extraData = &args, .extraDataSize = sizeof(args) }; BackgroundWorkerHandle *handle = RegisterCitusBackgroundWorker(&config); if (!handle) { return NULL; } MemoryContextCallback *workerCleanup = palloc0(sizeof(MemoryContextCallback)); workerCleanup->func = EnsureStopLockAcquireHelper; workerCleanup->arg = handle; MemoryContextRegisterResetCallback(CurrentMemoryContext, workerCleanup); return handle; } /* * EnsureStopLockAcquireHelper is designed to be called as a MemoryContextCallback. It * takes a handle to the background worker and Terminates it. It is safe to be called on a * handle that has already been terminated due to the guard around the generation number * implemented in the handle by postgres. */ static void EnsureStopLockAcquireHelper(void *arg) { BackgroundWorkerHandle *handle = (BackgroundWorkerHandle *) arg; TerminateBackgroundWorker(handle); } /* * Signal handler for SIGTERM * Set a flag to let the main loop to terminate, and set our latch to wake * it up. */ static void lock_acquire_helper_sigterm(SIGNAL_ARGS) { int save_errno = errno; got_sigterm = true; SetLatch(MyLatch); errno = save_errno; } /* * ShouldAcquireLock tests if our backend should still proceed with acquiring the lock, * and thus keep terminating conflicting backends. This function returns true until a * SIGTERM, background worker termination signal, has been received. * * The function blocks for at most sleepms when called. During operation without being * terminated this is the time between invocations to the backend termination logic. */ static bool ShouldAcquireLock(long sleepms) { /* early escape in case we already got the signal to stop acquiring the lock */ if (got_sigterm) { return false; } int rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, sleepms * 1L, PG_WAIT_EXTENSION); ResetLatch(MyLatch); /* emergency bailout if postmaster has died */ if (rc & WL_POSTMASTER_DEATH) { proc_exit(1); } CHECK_FOR_INTERRUPTS(); return !got_sigterm; } /* * LockAcquireHelperMain runs in a dynamic background worker to help master_update_node to * acquire its locks. */ void LockAcquireHelperMain(Datum main_arg) { int backendPid = DatumGetInt32(main_arg); StringInfoData sql; LockAcquireHelperArgs *args = (LockAcquireHelperArgs *) MyBgworkerEntry->bgw_extra; long timeout = 0; instr_time connectionStart; INSTR_TIME_SET_CURRENT(connectionStart); /* parameters for sql query to be executed */ const int paramCount = 1; Oid paramTypes[1] = { INT4OID }; Datum paramValues[1]; pqsignal(SIGTERM, lock_acquire_helper_sigterm); BackgroundWorkerUnblockSignals(); elog(LOG, "lock acquiring backend started for backend %d (cooldown %dms)", backendPid, args->lock_cooldown); /* * this loop waits till the deadline is reached (eg. lock_cooldown has passed) OR we * no longer need to acquire the lock due to the termination of this backend. * Only after the timeout the code will continue with the section that will acquire * the lock. */ do { timeout = MillisecondsToTimeout(connectionStart, args->lock_cooldown); } while (timeout > 0 && ShouldAcquireLock(timeout)); /* connecting to the database */ BackgroundWorkerInitializeConnectionByOid(args->DatabaseId, InvalidOid, 0); /* * The query below sends a SIGTERM signal to conflicting backends using * pg_terminate_backend() function. * * The result is are rows of pid,bool indicating a conflicting backend and * whether the SIGTERM was successfully delivered. These will be logged * accordingly below for an administrator to correlate in the logs with the * termination message. */ initStringInfo(&sql); appendStringInfo(&sql, "WITH pids AS (\n" " SELECT DISTINCT pid\n" " FROM pg_catalog.unnest(pg_catalog.pg_blocking_pids($1)) AS pid\n" ") SELECT pid, pg_catalog.pg_terminate_backend(pid) FROM pids"); paramValues[0] = Int32GetDatum(backendPid); while (ShouldAcquireLock(100)) { elog(LOG, "canceling competing backends for backend %d", backendPid); /* * Begin our transaction */ SetCurrentStatementStartTimestamp(); StartTransactionCommand(); SPI_connect(); PushActiveSnapshot(GetTransactionSnapshot()); pgstat_report_activity(STATE_RUNNING, sql.data); int spiStatus = SPI_execute_with_args(sql.data, paramCount, paramTypes, paramValues, NULL, false, 0); if (spiStatus == SPI_OK_SELECT) { for (uint64 row = 0; row < SPI_processed; row++) { bool isnull = false; int signaledPid = DatumGetInt32(SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull)); bool isSignaled = DatumGetBool(SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2, &isnull)); if (isSignaled) { elog(WARNING, "terminating conflicting backend %d", signaledPid); } else { elog(INFO, "attempt to terminate conflicting backend %d was unsuccessful", signaledPid); } } } else { elog(FATAL, "cannot cancel competing backends for backend %d", backendPid); } /* * And finish our transaction. */ SPI_finish(); PopActiveSnapshot(); CommitTransactionCommand(); pgstat_report_stat(false); pgstat_report_activity(STATE_IDLE, NULL); } elog(LOG, "lock acquiring backend finished for backend %d", backendPid); /* safely got to the end, exit without problem */ proc_exit(0); } ================================================ FILE: src/backend/distributed/utils/aggregate_utils.c ================================================ /*------------------------------------------------------------------------- * * aggregate_utils.c * * Implementation of UDFs distributing execution of aggregates across workers. * * When an aggregate has a combinefunc, we use worker_partial_agg to skip * calling finalfunc on workers, instead passing state to coordinator where * it uses combinefunc in coord_combine_agg & applying finalfunc only at end. * * Copyright Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "miscadmin.h" #include "pg_config_manual.h" #include "access/htup_details.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "nodes/nodeFuncs.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/typcache.h" #include "distributed/version_compat.h" PG_FUNCTION_INFO_V1(worker_partial_agg_sfunc); PG_FUNCTION_INFO_V1(worker_partial_agg_ffunc); PG_FUNCTION_INFO_V1(coord_combine_agg_sfunc); PG_FUNCTION_INFO_V1(coord_combine_agg_ffunc); PG_FUNCTION_INFO_V1(worker_binary_partial_agg_ffunc); PG_FUNCTION_INFO_V1(coord_binary_combine_agg_sfunc); PG_FUNCTION_INFO_V1(coord_binary_combine_agg_ffunc); /* * Holds information describing the structure of aggregation arguments * and helps to efficiently handle both a single argument and multiple * arguments wrapped in a tuple/record. It exploits the fact that * aggregation argument types do not change between subsequent * calls to SFUNC. */ typedef struct AggregationArgumentContext { /* immutable fields */ int argumentCount; bool isTuple; TupleDesc tupleDesc; /* mutable fields */ HeapTuple tuple; Datum *values; bool *nulls; } AggregationArgumentContext; /* * internal type for support aggregates to pass transition state alongside * aggregation bookkeeping */ typedef struct StypeBox { Datum value; Oid agg; Oid transtype; int16_t transtypeLen; bool transtypeByVal; bool valueNull; bool valueInit; AggregationArgumentContext *aggregationArgumentContext; } StypeBox; static HeapTuple GetAggregateForm(Oid oid, Form_pg_aggregate *form); static HeapTuple GetProcForm(Oid oid, Form_pg_proc *form); static HeapTuple GetTypeForm(Oid oid, Form_pg_type *form); static void * pallocInAggContext(FunctionCallInfo fcinfo, size_t size); static void aclcheckAggregate(ObjectType objectType, Oid userOid, Oid funcOid); static Datum GetAggInitVal(Datum textInitVal, Oid transtype); static void InitializeStypeBox(FunctionCallInfo fcinfo, StypeBox *box, HeapTuple aggTuple, Oid transtype, AggregationArgumentContext *aggregationArgumentContext); static StypeBox * TryCreateStypeBoxFromFcinfoAggref(FunctionCallInfo fcinfo); static AggregationArgumentContext * CreateAggregationArgumentContext(FunctionCallInfo fcinfo, int argumentIndex); static void ExtractAggregationValues(FunctionCallInfo fcinfo, int argumentIndex, AggregationArgumentContext *aggregationArgumentContext); static void HandleTransition(StypeBox *box, FunctionCallInfo fcinfo, FunctionCallInfo innerFcinfo); static void HandleStrictUninit(StypeBox *box, FunctionCallInfo fcinfo, Datum value); static bool TypecheckWorkerPartialAggArgType(FunctionCallInfo fcinfo, StypeBox *box); static bool TypecheckCoordCombineAggReturnType(FunctionCallInfo fcinfo, Oid ffunc, StypeBox *box); /* * GetAggregateForm loads corresponding tuple & Form_pg_aggregate for oid */ static HeapTuple GetAggregateForm(Oid oid, Form_pg_aggregate *form) { HeapTuple tuple = SearchSysCache1(AGGFNOID, ObjectIdGetDatum(oid)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "citus cache lookup failed for aggregate %u", oid); } *form = (Form_pg_aggregate) GETSTRUCT(tuple); return tuple; } /* * GetProcForm loads corresponding tuple & Form_pg_proc for oid */ static HeapTuple GetProcForm(Oid oid, Form_pg_proc *form) { HeapTuple tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(oid)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "citus cache lookup failed for function %u", oid); } *form = (Form_pg_proc) GETSTRUCT(tuple); return tuple; } /* * GetTypeForm loads corresponding tuple & Form_pg_type for oid */ static HeapTuple GetTypeForm(Oid oid, Form_pg_type *form) { HeapTuple tuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(oid)); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "citus cache lookup failed for type %u", oid); } *form = (Form_pg_type) GETSTRUCT(tuple); return tuple; } /* * pallocInAggContext calls palloc in fcinfo's aggregate context */ static void * pallocInAggContext(FunctionCallInfo fcinfo, size_t size) { MemoryContext aggregateContext; if (!AggCheckCallContext(fcinfo, &aggregateContext)) { elog(ERROR, "Aggregate function called without an aggregate context"); } return MemoryContextAlloc(aggregateContext, size); } /* * aclcheckAggregate verifies that the given user has ACL_EXECUTE to the given proc */ static void aclcheckAggregate(ObjectType objectType, Oid userOid, Oid funcOid) { AclResult aclresult; if (funcOid != InvalidOid) { aclresult = object_aclcheck(ProcedureRelationId, funcOid, userOid, ACL_EXECUTE); if (aclresult != ACLCHECK_OK) { aclcheck_error(aclresult, objectType, get_func_name(funcOid)); } } } /* Copied from nodeAgg.c */ static Datum GetAggInitVal(Datum textInitVal, Oid transtype) { /* *INDENT-OFF* */ Oid typinput, typioparam; char *strInitVal; Datum initVal; getTypeInputInfo(transtype, &typinput, &typioparam); strInitVal = TextDatumGetCString(textInitVal); initVal = OidInputFunctionCall(typinput, strInitVal, typioparam, -1); pfree(strInitVal); return initVal; /* *INDENT-ON* */ } /* * InitializeStypeBox fills in the rest of an StypeBox's fields besides agg, * handling both permission checking & setting up the initial transition state. */ static void InitializeStypeBox(FunctionCallInfo fcinfo, StypeBox *box, HeapTuple aggTuple, Oid transtype, AggregationArgumentContext *aggregationArgumentContext) { Form_pg_aggregate aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); Oid userId = GetUserId(); /* First we make ACL_EXECUTE checks as would be done in nodeAgg.c */ aclcheckAggregate(OBJECT_AGGREGATE, userId, aggform->aggfnoid); aclcheckAggregate(OBJECT_FUNCTION, userId, aggform->aggfinalfn); aclcheckAggregate(OBJECT_FUNCTION, userId, aggform->aggtransfn); aclcheckAggregate(OBJECT_FUNCTION, userId, aggform->aggdeserialfn); aclcheckAggregate(OBJECT_FUNCTION, userId, aggform->aggserialfn); aclcheckAggregate(OBJECT_FUNCTION, userId, aggform->aggcombinefn); Datum textInitVal = SysCacheGetAttr(AGGFNOID, aggTuple, Anum_pg_aggregate_agginitval, &box->valueNull); box->transtype = transtype; box->valueInit = !box->valueNull; box->aggregationArgumentContext = aggregationArgumentContext; if (box->valueNull) { box->value = (Datum) 0; } else { MemoryContext aggregateContext; if (!AggCheckCallContext(fcinfo, &aggregateContext)) { elog(ERROR, "InitializeStypeBox called from non aggregate context"); } MemoryContext oldContext = MemoryContextSwitchTo(aggregateContext); box->value = GetAggInitVal(textInitVal, transtype); MemoryContextSwitchTo(oldContext); } } /* * TryCreateStypeBoxFromFcinfoAggref attempts to initialize an StypeBox through * introspection of the fcinfo's Aggref from AggGetAggref. This is required * when we receive no intermediate rows. * * Returns NULL if the Aggref isn't our expected shape. */ static StypeBox * TryCreateStypeBoxFromFcinfoAggref(FunctionCallInfo fcinfo) { Aggref *aggref = AggGetAggref(fcinfo); if (aggref == NULL || aggref->args == NIL) { return NULL; } TargetEntry *aggArg = linitial(aggref->args); if (!IsA(aggArg->expr, Const)) { return NULL; } Const *aggConst = (Const *) aggArg->expr; if (aggConst->consttype != OIDOID && aggConst->consttype != REGPROCEDUREOID) { return NULL; } Form_pg_aggregate aggform; StypeBox *box = pallocInAggContext(fcinfo, sizeof(StypeBox)); box->agg = DatumGetObjectId(aggConst->constvalue); HeapTuple aggTuple = GetAggregateForm(box->agg, &aggform); InitializeStypeBox(fcinfo, box, aggTuple, aggform->aggtranstype, NULL); ReleaseSysCache(aggTuple); return box; } /* * CreateAggregationArgumentContext creates an AggregationArgumentContext tailored * to handling the aggregation of input arguments identical to type at * 'argumentIndex' in 'fcinfo'. */ static AggregationArgumentContext * CreateAggregationArgumentContext(FunctionCallInfo fcinfo, int argumentIndex) { AggregationArgumentContext *aggregationArgumentContext = pallocInAggContext(fcinfo, sizeof(AggregationArgumentContext)); /* check if input comes combined into tuple/record */ if (RECORDOID == get_fn_expr_argtype(fcinfo->flinfo, argumentIndex)) { /* initialize context to handle aggregation argument combined into tuple */ if (fcGetArgNull(fcinfo, argumentIndex)) { ereport(ERROR, (errmsg("worker_partial_agg_sfunc: null record input"), errhint("Elements of record may be null"))); } /* retrieve tuple header */ HeapTupleHeader tupleHeader = PG_GETARG_HEAPTUPLEHEADER(argumentIndex); /* extract type info from the tuple */ TupleDesc tupleDesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(tupleHeader), HeapTupleHeaderGetTypMod(tupleHeader)); /* create a copy we can keep */ TupleDesc tupleDescCopy = pallocInAggContext(fcinfo, TupleDescSize(tupleDesc)); TupleDescCopy(tupleDescCopy, tupleDesc); ReleaseTupleDesc(tupleDesc); /* build a HeapTuple control structure */ HeapTuple tuple = pallocInAggContext(fcinfo, sizeof(HeapTupleData)); ItemPointerSetInvalid(&(tuple->t_self)); tuple->t_tableOid = InvalidOid; /* initialize context to handle multiple aggregation arguments */ aggregationArgumentContext->argumentCount = tupleDescCopy->natts; aggregationArgumentContext->values = pallocInAggContext(fcinfo, tupleDescCopy->natts * sizeof(Datum)); aggregationArgumentContext->nulls = pallocInAggContext(fcinfo, tupleDescCopy->natts * sizeof(bool)); aggregationArgumentContext->isTuple = true; aggregationArgumentContext->tupleDesc = tupleDescCopy; aggregationArgumentContext->tuple = tuple; } else { /* initialize context to handle single aggregation argument */ aggregationArgumentContext->argumentCount = 1; aggregationArgumentContext->values = pallocInAggContext(fcinfo, sizeof(Datum)); aggregationArgumentContext->nulls = pallocInAggContext(fcinfo, sizeof(bool)); aggregationArgumentContext->isTuple = false; aggregationArgumentContext->tupleDesc = NULL; aggregationArgumentContext->tuple = NULL; } return aggregationArgumentContext; } /* * ExtractAggregationValues extracts aggregation argument values and stores them in * the mutable fields of AggregationArgumentContext. */ static void ExtractAggregationValues(FunctionCallInfo fcinfo, int argumentIndex, AggregationArgumentContext *aggregationArgumentContext) { if (aggregationArgumentContext->isTuple) { if (fcGetArgNull(fcinfo, argumentIndex)) { /* handle null record input */ for (int i = 0; i < aggregationArgumentContext->argumentCount; i++) { aggregationArgumentContext->values[i] = 0; aggregationArgumentContext->nulls[i] = true; } } else { /* handle tuple/record input */ HeapTupleHeader tupleHeader = DatumGetHeapTupleHeader(fcGetArgValue(fcinfo, argumentIndex)); if (HeapTupleHeaderGetNatts( tupleHeader) != aggregationArgumentContext->argumentCount || HeapTupleHeaderGetTypeId( tupleHeader) != aggregationArgumentContext->tupleDesc->tdtypeid || HeapTupleHeaderGetTypMod( tupleHeader) != aggregationArgumentContext->tupleDesc->tdtypmod) { ereport(ERROR, (errmsg("worker_partial_agg_sfunc received " "incompatible record"))); } aggregationArgumentContext->tuple->t_len = HeapTupleHeaderGetDatumLength(tupleHeader); aggregationArgumentContext->tuple->t_data = tupleHeader; /* break down the tuple into fields */ heap_deform_tuple( aggregationArgumentContext->tuple, aggregationArgumentContext->tupleDesc, aggregationArgumentContext->values, aggregationArgumentContext->nulls); } } else { /* extract single argument value */ aggregationArgumentContext->values[0] = fcGetArgValue(fcinfo, argumentIndex); aggregationArgumentContext->nulls[0] = fcGetArgNull(fcinfo, argumentIndex); } } /* * HandleTransition copies logic used in nodeAgg's advance_transition_function * for handling result of transition function. */ static void HandleTransition(StypeBox *box, FunctionCallInfo fcinfo, FunctionCallInfo innerFcinfo) { Datum newVal = FunctionCallInvoke(innerFcinfo); bool newValIsNull = innerFcinfo->isnull; if (!box->transtypeByVal && DatumGetPointer(newVal) != DatumGetPointer(box->value)) { if (!newValIsNull) { MemoryContext aggregateContext; if (!AggCheckCallContext(fcinfo, &aggregateContext)) { elog(ERROR, "HandleTransition called from non aggregate context"); } MemoryContext oldContext = MemoryContextSwitchTo(aggregateContext); if (!(DatumIsReadWriteExpandedObject(newVal, false, box->transtypeLen) && MemoryContextGetParent(DatumGetEOHP(newVal)->eoh_context) == CurrentMemoryContext)) { newVal = datumCopy(newVal, box->transtypeByVal, box->transtypeLen); } MemoryContextSwitchTo(oldContext); } if (!box->valueNull) { if (DatumIsReadWriteExpandedObject(box->value, false, box->transtypeLen)) { DeleteExpandedObject(box->value); } else { pfree(DatumGetPointer(box->value)); } } } box->value = newVal; box->valueNull = newValIsNull; } /* * HandleStrictUninit handles initialization of state for when * transition function is strict & state has not yet been initialized. */ static void HandleStrictUninit(StypeBox *box, FunctionCallInfo fcinfo, Datum value) { MemoryContext aggregateContext; if (!AggCheckCallContext(fcinfo, &aggregateContext)) { elog(ERROR, "HandleStrictUninit called from non aggregate context"); } MemoryContext oldContext = MemoryContextSwitchTo(aggregateContext); box->value = datumCopy(value, box->transtypeByVal, box->transtypeLen); MemoryContextSwitchTo(oldContext); box->valueNull = false; box->valueInit = true; } /* * worker_partial_agg_sfunc advances transition state, * essentially implementing the following pseudocode: * * (box, agg, ...) -> box * box.agg = agg; * box.value = agg.sfunc(box.value, ...); * return box */ Datum worker_partial_agg_sfunc(PG_FUNCTION_ARGS) { StypeBox *box = NULL; Form_pg_aggregate aggform; LOCAL_FCINFO(innerFcinfo, FUNC_MAX_ARGS); FmgrInfo info; int argumentIndex = 0; bool initialCall = PG_ARGISNULL(0); if (initialCall) { if (PG_ARGISNULL(1)) { ereport(ERROR, (errmsg("worker_partial_agg_sfunc received invalid null " "input for second argument"))); } box = pallocInAggContext(fcinfo, sizeof(StypeBox)); box->agg = PG_GETARG_OID(1); box->aggregationArgumentContext = CreateAggregationArgumentContext(fcinfo, 2); if (!TypecheckWorkerPartialAggArgType(fcinfo, box)) { ereport(ERROR, (errmsg("worker_partial_agg_sfunc could not confirm type " "correctness"))); } } else { box = (StypeBox *) PG_GETARG_POINTER(0); Assert(box->agg == PG_GETARG_OID(1)); } HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); Oid aggsfunc = aggform->aggtransfn; if (initialCall) { InitializeStypeBox(fcinfo, box, aggtuple, aggform->aggtranstype, box->aggregationArgumentContext); } ReleaseSysCache(aggtuple); if (initialCall) { get_typlenbyval(box->transtype, &box->transtypeLen, &box->transtypeByVal); } /* * Get aggregation values, which may be either wrapped in a * tuple (multi-argument case) or a singular, unwrapped value. */ ExtractAggregationValues(fcinfo, 2, box->aggregationArgumentContext); fmgr_info(aggsfunc, &info); if (info.fn_strict) { for (argumentIndex = 0; argumentIndex < box->aggregationArgumentContext->argumentCount; argumentIndex++) { if (box->aggregationArgumentContext->nulls[argumentIndex]) { PG_RETURN_POINTER(box); } } if (!box->valueInit) { /* For 'strict' transition functions, if the initial state value is null * then the first argument value of the first row with all-nonnull input * values replaces the state value. */ Datum stateValue = box->aggregationArgumentContext->values[0]; HandleStrictUninit(box, fcinfo, stateValue); PG_RETURN_POINTER(box); } if (box->valueNull) { PG_RETURN_POINTER(box); } } /* if aggregate function has N parameters, corresponding SFUNC has N+1 */ InitFunctionCallInfoData(*innerFcinfo, &info, box->aggregationArgumentContext->argumentCount + 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); for (argumentIndex = 0; argumentIndex < box->aggregationArgumentContext->argumentCount; argumentIndex++) { fcSetArgExt(innerFcinfo, argumentIndex + 1, box->aggregationArgumentContext->values[argumentIndex], box->aggregationArgumentContext->nulls[argumentIndex]); } HandleTransition(box, fcinfo, innerFcinfo); PG_RETURN_POINTER(box); } /* * Given an STypeBox, returns its transition type Oid * for the worker aggregate being computed. */ static Oid GetAggregateTransitionType(StypeBox *box) { Form_pg_aggregate aggform; HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); if (aggform->aggcombinefn == InvalidOid) { ereport(ERROR, (errmsg( "worker_partial_agg_ffunc expects an aggregate with COMBINEFUNC"))); } if (aggform->aggtranstype == INTERNALOID && aggform->aggserialfn == InvalidOid) { ereport(ERROR, (errmsg( "worker_partial_agg_ffunc does not support aggregates with INTERNAL transition state"))); } Oid transType = aggform->aggtranstype; ReleaseSysCache(aggtuple); return transType; } /* * serializes transition state, * returning an StypeBox and setting transtype to the transition type Oid. */ static StypeBox * WorkerPartialAggregateApplyFFunc(PG_FUNCTION_ARGS) { StypeBox *box = (StypeBox *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (box == NULL) { box = TryCreateStypeBoxFromFcinfoAggref(fcinfo); } if (box == NULL || box->valueNull) { return NULL; } return box; } /* If the transtype is internal, we need to use the SERIALFUNC of the aggregate * to serialize the value in the worker. The COMBINEFUNC on the coordinator will * then use the DESERIALFUNC to deserialize the value. */ static Datum CheckAndCallSerialFunc(PG_FUNCTION_ARGS, StypeBox *box, bool *outputIsNull) { LOCAL_FCINFO(serialFcInfo, 1); FmgrInfo serialInfo; Form_pg_aggregate aggform; HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); if (aggform->aggserialfn == InvalidOid) { ereport(ERROR, (errmsg( "worker_partial_agg_ffunc expects aggregates with internal transition state to have a SERIALFUNC"))); } /* Otherwise, first invoke the serialfunc to ensure that we produce a serialiable value from the boxValue */ Assert(get_func_rettype(aggform->aggserialfn) == BYTEAOID); fmgr_info(aggform->aggserialfn, &serialInfo); InitFunctionCallInfoData(*serialFcInfo, &serialInfo, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(serialFcInfo, 0, box->value, box->valueNull); Datum result = FunctionCallInvoke(serialFcInfo); *outputIsNull = serialFcInfo->isnull; ReleaseSysCache(aggtuple); return result; } /* * worker_partial_agg_ffunc serializes transition state, * essentially implementing the following pseudocode: * * (box) -> text * return box.agg.stype.output(box.value) */ Datum worker_partial_agg_ffunc(PG_FUNCTION_ARGS) { LOCAL_FCINFO(innerFcinfo, 1); FmgrInfo info; Oid typoutput = InvalidOid; bool typIsVarlena = false; StypeBox *box = WorkerPartialAggregateApplyFFunc(fcinfo); if (box == NULL) { PG_RETURN_NULL(); } Oid transtype = GetAggregateTransitionType(box); Datum boxValue = box->value; bool boxValueNull = box->valueNull; if (transtype == INTERNALOID) { ereport(ERROR, (errmsg("worker_partial_agg_ffunc does not support output" " of aggregates with INTERNAL transition state"))); } getTypeOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); if (innerFcinfo->isnull) { PG_RETURN_NULL(); } PG_RETURN_DATUM(result); } /* * worker_partial_binary_agg_ffunc serializes transition state, * essentially implementing the following pseudocode: * * (box) -> bytea * return box.agg.stype.output(box.value) */ Datum worker_binary_partial_agg_ffunc(PG_FUNCTION_ARGS) { LOCAL_FCINFO(innerFcinfo, 1); FmgrInfo info; Oid typoutput = InvalidOid; bool typIsVarlena = false; StypeBox *box = WorkerPartialAggregateApplyFFunc(fcinfo); if (box == NULL) { PG_RETURN_NULL(); } Oid transtype = GetAggregateTransitionType(box); Datum boxValue = box->value; bool boxValueNull = box->valueNull; if (transtype == INTERNALOID) { /* Call and store the output of the SERIALFUNC - the output type * then is always BYTEAOID. */ boxValue = CheckAndCallSerialFunc(fcinfo, box, &boxValueNull); transtype = BYTEAOID; } getTypeBinaryOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); if (innerFcinfo->isnull) { PG_RETURN_NULL(); } PG_RETURN_DATUM(result); } static Datum DeserializeBoxValue(Oid deserialFunc, Datum value, bool valueNull, FunctionCallInfo fcinfo, bool *outputIsNull) { LOCAL_FCINFO(deserialFcInfo, 3); FmgrInfo deserialInfo; fmgr_info(deserialFunc, &deserialInfo); InitFunctionCallInfoData(*deserialFcInfo, &deserialInfo, 2, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(deserialFcInfo, 0, value, valueNull); /* Arg1 is not used and is internal */ fcSetArgExt(deserialFcInfo, 1, (Datum) 0, false); Datum result = FunctionCallInvoke(deserialFcInfo); *outputIsNull = deserialFcInfo->isnull; return result; } static Datum CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) { LOCAL_FCINFO(innerFcinfo, 3); FmgrInfo info; Form_pg_aggregate aggform; Form_pg_type transtypeform; Oid deserialFunc = InvalidOid; Datum value; StypeBox *box = NULL; if (PG_ARGISNULL(0)) { box = pallocInAggContext(fcinfo, sizeof(StypeBox)); box->agg = PG_GETARG_OID(1); } else { box = (StypeBox *) PG_GETARG_POINTER(0); Assert(box->agg == PG_GETARG_OID(1)); } HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); if (aggform->aggcombinefn == InvalidOid) { ereport(ERROR, (errmsg( "coord_combine_agg_sfunc expects an aggregate with COMBINEFUNC"))); } if (aggform->aggtranstype == INTERNALOID) { if (aggform->aggdeserialfn == InvalidOid) { ereport(ERROR, (errmsg( "coord_combine_agg_sfunc does not support aggregates with INTERNAL transition state"))); } else { deserialFunc = aggform->aggdeserialfn; } } Oid combine = aggform->aggcombinefn; if (PG_ARGISNULL(0)) { InitializeStypeBox(fcinfo, box, aggtuple, aggform->aggtranstype, NULL); } ReleaseSysCache(aggtuple); if (PG_ARGISNULL(0)) { get_typlenbyval(box->transtype, &box->transtypeLen, &box->transtypeByVal); } bool valueNull = PG_ARGISNULL(2); /* If the stype is internal, this needs to through first * deserializing the wire output to the intermediate state. */ Oid deserializationType = box->transtype; if (box->transtype == INTERNALOID) { /* For a deserialfunc, the input is a BYTEAOID */ deserializationType = BYTEAOID; } HeapTuple deserializationTypeTuple = GetTypeForm(deserializationType, &transtypeform); Oid ioparam = getTypeIOParam(deserializationTypeTuple); Oid deserial = isBinaryInput ? transtypeform->typreceive : transtypeform->typinput; ReleaseSysCache(deserializationTypeTuple); fmgr_info(deserial, &info); if (valueNull && info.fn_strict) { value = (Datum) 0; } else if (isBinaryInput) { StringInfoData buf; InitFunctionCallInfoData(*innerFcinfo, &info, 3, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); if (valueNull) { fcSetArgExt(innerFcinfo, 0, (Datum) 0, valueNull); fcSetArg(innerFcinfo, 1, ObjectIdGetDatum(ioparam)); fcSetArg(innerFcinfo, 2, Int32GetDatum(-1)); /* typmod */ value = FunctionCallInvoke(innerFcinfo); valueNull = innerFcinfo->isnull; } else { bytea *byteaInput = PG_GETARG_BYTEA_PP(2); #if PG_VERSION_NUM >= 170000 initReadOnlyStringInfo(&buf, (char *) VARDATA_ANY(byteaInput), VARSIZE_ANY_EXHDR(byteaInput)); fcSetArg(innerFcinfo, 0, PointerGetDatum(&buf)); fcSetArg(innerFcinfo, 1, ObjectIdGetDatum(ioparam)); fcSetArg(innerFcinfo, 2, Int32GetDatum(-1)); /* typmod */ value = FunctionCallInvoke(innerFcinfo); valueNull = innerFcinfo->isnull; #else /* * Read Only StringInfo is not a characteristic in pg16 * or below. We can't follow what's there in arrayfuncs since * the Send function won't guarantee to append an extra null byte at the end. * So we manually set up a StringInfo with a trailing null byte. */ initStringInfo(&buf); appendBinaryStringInfo(&buf, (char *) VARDATA_ANY(byteaInput), VARSIZE_ANY_EXHDR(byteaInput)); fcSetArg(innerFcinfo, 0, PointerGetDatum(&buf)); fcSetArg(innerFcinfo, 1, ObjectIdGetDatum(ioparam)); fcSetArg(innerFcinfo, 2, Int32GetDatum(-1)); /* typmod */ value = FunctionCallInvoke(innerFcinfo); valueNull = innerFcinfo->isnull; pfree(buf.data); #endif } } else { InitFunctionCallInfoData(*innerFcinfo, &info, 3, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, PG_GETARG_DATUM(2), valueNull); fcSetArg(innerFcinfo, 1, ObjectIdGetDatum(ioparam)); fcSetArg(innerFcinfo, 2, Int32GetDatum(-1)); /* typmod */ value = FunctionCallInvoke(innerFcinfo); valueNull = innerFcinfo->isnull; } /* If the stype is internal, we need to go through one additional step * of now calling the deserialfunc to go from the serialized type to * internal before we call the combine function. */ if (box->transtype == INTERNALOID && !valueNull) { value = DeserializeBoxValue(deserialFunc, value, valueNull, fcinfo, &valueNull); } fmgr_info(combine, &info); if (info.fn_strict) { if (valueNull) { PG_RETURN_POINTER(box); } if (!box->valueInit) { HandleStrictUninit(box, fcinfo, value); PG_RETURN_POINTER(box); } if (box->valueNull) { PG_RETURN_POINTER(box); } } InitFunctionCallInfoData(*innerFcinfo, &info, 2, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); fcSetArgExt(innerFcinfo, 1, value, valueNull); HandleTransition(box, fcinfo, innerFcinfo); PG_RETURN_POINTER(box); } /* * coord_combine_agg_sfunc deserializes transition state from worker * & advances transition state using combinefunc, * essentially implementing the following pseudocode: * * (box, agg, text) -> box * box.agg = agg * box.value = agg.combine(box.value, agg.stype.input(text)) * return box */ Datum coord_combine_agg_sfunc(PG_FUNCTION_ARGS) { bool isBinaryInput = false; return CoordinatorCombineAggSfuncCore(fcinfo, isBinaryInput); } /* * coord_binary_combine_agg_sfunc deserializes transition state from worker * & advances transition state using combinefunc, * essentially implementing the following pseudocode: * * (box, agg, bytea) -> box * box.agg = agg * box.value = agg.combine(box.value, agg.stype.receive(bytea)) * return box */ Datum coord_binary_combine_agg_sfunc(PG_FUNCTION_ARGS) { bool isBinaryInput = true; return CoordinatorCombineAggSfuncCore(fcinfo, isBinaryInput); } /* * Applies finalfunc of aggregate to state, * essentially implementing the following pseudocode: * * (box, ...) -> fval * return box.agg.ffunc(box.value) * Used by both the binary and text versions of the finalfunc. */ static Datum CoordCombineAggFuncCore(PG_FUNCTION_ARGS) { StypeBox *box = (StypeBox *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); LOCAL_FCINFO(innerFcinfo, FUNC_MAX_ARGS); FmgrInfo info; int innerNargs = 0; Form_pg_aggregate aggform; Form_pg_proc ffuncform; if (box == NULL) { box = TryCreateStypeBoxFromFcinfoAggref(fcinfo); if (box == NULL) { PG_RETURN_NULL(); } } HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); Oid ffunc = aggform->aggfinalfn; bool fextra = aggform->aggfinalextra; ReleaseSysCache(aggtuple); if (!TypecheckCoordCombineAggReturnType(fcinfo, ffunc, box)) { ereport(ERROR, (errmsg("coord_combine_agg_ffunc could not " "confirm type correctness"))); } if (ffunc == InvalidOid) { if (box->valueNull) { PG_RETURN_NULL(); } PG_RETURN_DATUM(box->value); } HeapTuple ffunctuple = GetProcForm(ffunc, &ffuncform); bool finalStrict = ffuncform->proisstrict; ReleaseSysCache(ffunctuple); if (finalStrict && box->valueNull) { PG_RETURN_NULL(); } if (fextra) { innerNargs = fcinfo->nargs; } else { innerNargs = 1; } fmgr_info(ffunc, &info); InitFunctionCallInfoData(*innerFcinfo, &info, innerNargs, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); for (int argumentIndex = 1; argumentIndex < innerNargs; argumentIndex++) { fcSetArgNull(innerFcinfo, argumentIndex); } Datum result = FunctionCallInvoke(innerFcinfo); fcinfo->isnull = innerFcinfo->isnull; return result; } /* * coord_combine_agg_ffunc applies finalfunc of aggregate to state, * essentially implementing the following pseudocode: * * (box, ...) -> fval * return box.agg.ffunc(box.value) */ Datum coord_combine_agg_ffunc(PG_FUNCTION_ARGS) { return CoordCombineAggFuncCore(fcinfo); } /* * coord_binary_combine_agg_ffunc applies finalfunc of aggregate to state, * essentially implementing the following pseudocode: * * (box, ...) -> fval * return box.agg.ffunc(box.value) */ Datum coord_binary_combine_agg_ffunc(PG_FUNCTION_ARGS) { return CoordCombineAggFuncCore(fcinfo); } /* * TypecheckWorkerPartialAggArgType returns whether the arguments being passed to * worker_partial_agg match the arguments expected by the aggregate being distributed. */ static bool TypecheckWorkerPartialAggArgType(FunctionCallInfo fcinfo, StypeBox *box) { Aggref *aggref = AggGetAggref(fcinfo); if (aggref == NULL) { return false; } Assert(list_length(aggref->args) == 2); TargetEntry *aggarg = list_nth(aggref->args, 1); bool argtypesNull; HeapTuple proctuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(box->agg)); if (!HeapTupleIsValid(proctuple)) { return false; } Datum argtypes = SysCacheGetAttr(PROCOID, proctuple, Anum_pg_proc_proargtypes, &argtypesNull); Assert(!argtypesNull); ReleaseSysCache(proctuple); if (ARR_NDIM(DatumGetArrayTypeP(argtypes)) != 1) { elog(ERROR, "worker_partial_agg_sfunc cannot type check aggregates " "taking multi-dimensional arguments"); } int aggregateArgCount = ARR_DIMS(DatumGetArrayTypeP(argtypes))[0]; /* we expect aggregate function to have at least a single parameter */ if (box->aggregationArgumentContext->argumentCount != aggregateArgCount) { return false; } int aggregateArgIndex = 0; Datum argType; if (box->aggregationArgumentContext->isTuple) { /* check if record element types match aggregate input parameters */ for (aggregateArgIndex = 0; aggregateArgIndex < aggregateArgCount; aggregateArgIndex++) { argType = array_get_element(argtypes, 1, &aggregateArgIndex, -1, sizeof(Oid), true, 'i', &argtypesNull); Assert(!argtypesNull); TupleDesc tupleDesc = box->aggregationArgumentContext->tupleDesc; if (argType != TupleDescAttr(tupleDesc, aggregateArgIndex)->atttypid) { return false; } } return true; } else { argType = array_get_element(argtypes, 1, &aggregateArgIndex, -1, sizeof(Oid), true, 'i', &argtypesNull); Assert(!argtypesNull); return exprType((Node *) aggarg->expr) == DatumGetObjectId(argType); } } /* * TypecheckCoordCombineAggReturnType returns whether the return type of the aggregate * being distributed by coord_combine_agg matches the null constant used to inform postgres * what the aggregate's expected return type is. */ static bool TypecheckCoordCombineAggReturnType(FunctionCallInfo fcinfo, Oid ffunc, StypeBox *box) { Aggref *aggref = AggGetAggref(fcinfo); if (aggref == NULL) { return false; } Oid finalType = ffunc == InvalidOid ? box->transtype : get_func_rettype(ffunc); Assert(list_length(aggref->args) == 3); TargetEntry *nulltag = list_nth(aggref->args, 2); return nulltag != NULL && IsA(nulltag->expr, Const) && ((Const *) nulltag->expr)->consttype == finalType; } ================================================ FILE: src/backend/distributed/utils/array_type.c ================================================ /*------------------------------------------------------------------------- * * array_type.c * * Utility functions for dealing with array types. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "catalog/pg_type.h" #include "nodes/pg_list.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "pg_version_compat.h" #include "distributed/utils/array_type.h" /* * DeconstructArrayObject takes in a single dimensional array, and deserializes * this array's members into an array of datum objects. The function then * returns this datum array. */ Datum * DeconstructArrayObject(ArrayType *arrayObject) { Datum *datumArray = NULL; bool *datumArrayNulls = NULL; int datumArrayLength = 0; bool typeByVal = false; char typeAlign = 0; int16 typeLength = 0; bool arrayHasNull = ARR_HASNULL(arrayObject); if (arrayHasNull) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("worker array object cannot contain null values"))); } Oid typeId = ARR_ELEMTYPE(arrayObject); get_typlenbyvalalign(typeId, &typeLength, &typeByVal, &typeAlign); deconstruct_array(arrayObject, typeId, typeLength, typeByVal, typeAlign, &datumArray, &datumArrayNulls, &datumArrayLength); return datumArray; } /* * ArrayObjectCount takes in a single dimensional array, and returns the number * of elements in this array. */ int32 ArrayObjectCount(ArrayType *arrayObject) { int32 dimensionCount = ARR_NDIM(arrayObject); int32 *dimensionLengthArray = ARR_DIMS(arrayObject); if (dimensionCount == 0) { return 0; } /* we currently allow split point arrays to have only one subarray */ Assert(dimensionCount == 1); int32 arrayLength = ArrayGetNItems(dimensionCount, dimensionLengthArray); if (arrayLength <= 0) { ereport(ERROR, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("worker array object cannot be empty"))); } return arrayLength; } /* * DatumArrayToArrayType converts the provided Datum array (of the specified * length and type) into an ArrayType suitable for returning from a UDF. */ ArrayType * DatumArrayToArrayType(Datum *datumArray, int datumCount, Oid datumTypeId) { int16 typeLength = 0; bool typeByValue = false; char typeAlignment = 0; get_typlenbyvalalign(datumTypeId, &typeLength, &typeByValue, &typeAlignment); ArrayType *arrayObject = construct_array(datumArray, datumCount, datumTypeId, typeLength, typeByValue, typeAlignment); return arrayObject; } /* * Converts ArrayType to List. */ List * IntegerArrayTypeToList(ArrayType *arrayObject) { List *list = NULL; Datum *datumObjectArray = DeconstructArrayObject(arrayObject); int arrayObjectCount = ArrayObjectCount(arrayObject); for (int index = 0; index < arrayObjectCount; index++) { int32 intObject = DatumGetInt32(datumObjectArray[index]); list = lappend_int(list, intObject); } return list; } /* * Converts Text ArrayType to Integer List. */ extern List * TextArrayTypeToIntegerList(ArrayType *arrayObject) { List *list = NULL; Datum *datumObjectArray = DeconstructArrayObject(arrayObject); int arrayObjectCount = ArrayObjectCount(arrayObject); for (int index = 0; index < arrayObjectCount; index++) { char *intAsStr = text_to_cstring(DatumGetTextP(datumObjectArray[index])); list = lappend_int(list, pg_strtoint32(intAsStr)); } return list; } /* * IntArrayToDatum * * Convert an integer array to the datum int array format * (currently used for nodes_involved in pg_dist_background_task) * * Returns the array in the form of a Datum, or PointerGetDatum(NULL) * if the int_array is empty. */ Datum IntArrayToDatum(uint32 int_array_size, int int_array[]) { if (int_array_size == 0) { return PointerGetDatum(NULL); } ArrayBuildState *astate = NULL; for (int i = 0; i < int_array_size; i++) { Datum dvalue = Int32GetDatum(int_array[i]); bool disnull = false; Oid element_type = INT4OID; astate = accumArrayResult(astate, dvalue, disnull, element_type, CurrentMemoryContext); } return makeArrayResult(astate, CurrentMemoryContext); } ================================================ FILE: src/backend/distributed/utils/background_jobs.c ================================================ /*------------------------------------------------------------------------- * * background_jobs.c * Background jobs run as a background worker, spawned from the * maintenance daemon. Jobs have tasks, tasks can depend on other * tasks before execution. * * This file contains the code for two separate background workers to * achieve the goal of running background tasks asynchronously from the * main database workload. This first background worker is the * Background Tasks Queue Monitor. This background worker keeps track of * tasks recorded in pg_dist_background_task and ensures execution based * on a statemachine. When a task needs to be executed it starts a * Background Task Executor that executes the sql statement defined in the * task. The output of the Executor is shared with the Monitor via a * shared memory queue. * * To make sure there is only ever exactly one monitor running per database * it takes an exclusive lock on the CITUS_BACKGROUND_TASK_MONITOR * operation. This lock is consulted from the maintenance daemon to only * spawn a new monitor when the lock is not held. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "libpq-fe.h" #include "pgstat.h" #include "safe_mem_lib.h" #include "access/xact.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "libpq/pqformat.h" #include "libpq/pqmq.h" #include "libpq/pqsignal.h" #include "parser/analyze.h" #include "storage/dsm.h" #include "storage/ipc.h" #include "storage/procarray.h" #include "storage/shm_mq.h" #include "storage/shm_toc.h" #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "utils/fmgrprotos.h" #include "utils/hsearch.h" #include "utils/memutils.h" #include "utils/portal.h" #include "utils/ps_status.h" #include "utils/resowner.h" #include "utils/snapmgr.h" #include "utils/timeout.h" #include "distributed/background_jobs.h" #include "distributed/background_worker_utils.h" #include "distributed/citus_safe_lib.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/maintenanced.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/shard_rebalancer.h" /* Table-of-contents constants for our dynamic shared memory segment. */ #define CITUS_BACKGROUND_TASK_MAGIC 0x51028081 #define CITUS_BACKGROUND_TASK_KEY_DATABASE 0 #define CITUS_BACKGROUND_TASK_KEY_USERNAME 1 #define CITUS_BACKGROUND_TASK_KEY_COMMAND 2 #define CITUS_BACKGROUND_TASK_KEY_QUEUE 3 #define CITUS_BACKGROUND_TASK_KEY_TASK_ID 4 #define CITUS_BACKGROUND_TASK_KEY_JOB_ID 5 #define CITUS_BACKGROUND_TASK_NKEYS 6 static BackgroundWorkerHandle * StartCitusBackgroundTaskExecutor(char *database, char *user, char *command, int64 taskId, int64 jobId, dsm_segment **pSegment); static void ExecuteSqlString(const char *sql); static shm_mq_result ConsumeTaskWorkerOutput(shm_mq_handle *responseq, StringInfo message, bool *hadError); static void UpdateDependingTasks(BackgroundTask *task); static int64 CalculateBackoffDelay(int retryCount); static bool NewExecutorExceedsCitusLimit(QueueMonitorExecutionContext * queueMonitorExecutionContext); static bool NewExecutorExceedsPgMaxWorkers(BackgroundWorkerHandle *handle, QueueMonitorExecutionContext * queueMonitorExecutionContext); static bool AssignRunnableTaskToNewExecutor(BackgroundTask *runnableTask, QueueMonitorExecutionContext * queueMonitorExecutionContext); static void AssignRunnableTasks(QueueMonitorExecutionContext * queueMonitorExecutionContext); static List * GetRunningTaskEntries(HTAB *currentExecutors); static shm_mq_result ReadFromExecutorQueue(BackgroundExecutorHashEntry * backgroundExecutorHashEntry, bool *hadError); static void CheckAndResetLastWorkerAllocationFailure(QueueMonitorExecutionContext * queueMonitorExecutionContext); static TaskExecutionStatus TaskConcurrentCancelCheck(TaskExecutionContext * taskExecutionContext); static TaskExecutionStatus ConsumeExecutorQueue(TaskExecutionContext * taskExecutionContext); static void TaskHadError(TaskExecutionContext *taskExecutionContext); static void TaskEnded(TaskExecutionContext *taskExecutionContext); static void TerminateAllTaskExecutors(HTAB *currentExecutors); static HTAB * GetRunningUniqueJobIds(HTAB *currentExecutors); static void CancelAllTaskExecutors(HTAB *currentExecutors); static bool MonitorGotTerminationOrCancellationRequest(); static void QueueMonitorSigTermHandler(SIGNAL_ARGS); static void QueueMonitorSigIntHandler(SIGNAL_ARGS); static void QueueMonitorSigHupHandler(SIGNAL_ARGS); static void DecrementParallelTaskCountForNodesInvolved(BackgroundTask *task); /* flags set by signal handlers */ static volatile sig_atomic_t GotSigterm = false; static volatile sig_atomic_t GotSigint = false; static volatile sig_atomic_t GotSighup = false; /* keeping track of parallel background tasks per node */ HTAB *ParallelTasksPerNode = NULL; int MaxBackgroundTaskExecutorsPerNode = 1; PG_FUNCTION_INFO_V1(citus_job_cancel); PG_FUNCTION_INFO_V1(citus_job_wait); PG_FUNCTION_INFO_V1(citus_task_wait); /* * pg_catalog.citus_job_cancel(jobid bigint) void * cancels a scheduled/running job * * When cancelling a job there are two phases. * 1. scan all associated tasks and transition all tasks that are not already in their * terminal state to cancelled. Except if the task is currently running. * 2. for all running tasks we send a cancelation signal to the backend running the * query. The background executor/monitor will transition this task to cancelled. * * We apply the same policy checks as pg_cancel_backend to check if a user can cancel a * job. */ Datum citus_job_cancel(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 jobid = PG_GETARG_INT64(0); /* Cancel all tasks that were scheduled before */ List *pids = CancelTasksForJob(jobid); /* send cancellation to any running backends */ int pid = 0; foreach_declared_int(pid, pids) { Datum pidDatum = Int32GetDatum(pid); Datum signalSuccessDatum = DirectFunctionCall1(pg_cancel_backend, pidDatum); bool signalSuccess = DatumGetBool(signalSuccessDatum); if (!signalSuccess) { ereport(WARNING, (errmsg("could not send signal to process %d: %m", pid))); } } UpdateBackgroundJob(jobid); PG_RETURN_VOID(); } /* * pg_catalog.citus_job_wait(jobid bigint, * desired_status citus_job_status DEFAULT NULL) boolean * waits till a job reaches a desired status, or can't reach the status anymore because * it reached a (different) terminal state. When no desired_status is given it will * assume any terminal state as its desired status. The function returns if the * desired_state was reached. * * The current implementation is a polling implementation with an interval of 1 second. * Ideally we would have some synchronization between the background tasks queue monitor * and any backend calling this function to receive a signal when the job changes state. */ Datum citus_job_wait(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 jobid = PG_GETARG_INT64(0); /* parse the optional desired_status argument */ bool hasDesiredStatus = !PG_ARGISNULL(1); BackgroundJobStatus desiredStatus = { 0 }; if (hasDesiredStatus) { desiredStatus = BackgroundJobStatusByOid(PG_GETARG_OID(1)); } citus_job_wait_internal(jobid, hasDesiredStatus ? &desiredStatus : NULL); PG_RETURN_VOID(); } /* * pg_catalog.citus_task_wait(taskid bigint, * desired_status citus_task_status DEFAULT NULL) boolean * waits till a task reaches a desired status, or can't reach the status anymore because * it reached a (different) terminal state. When no desired_status is given it will * assume any terminal state as its desired status. The function returns if the * desired_state was reached. * * The current implementation is a polling implementation with an interval of 0.1 seconds. * Ideally we would have some synchronization between the background tasks queue monitor * and any backend calling this function to receive a signal when the task changes state. */ Datum citus_task_wait(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); int64 taskid = PG_GETARG_INT64(0); /* parse the optional desired_status argument */ bool hasDesiredStatus = !PG_ARGISNULL(1); BackgroundTaskStatus desiredStatus = { 0 }; if (hasDesiredStatus) { desiredStatus = BackgroundTaskStatusByOid(PG_GETARG_OID(1)); } citus_task_wait_internal(taskid, hasDesiredStatus ? &desiredStatus : NULL); PG_RETURN_VOID(); } /* * citus_job_wait_internal implements the waiting on a job for reuse in other areas where * we want to wait on jobs. eg the background rebalancer. * * When a desiredStatus is provided it will provide an error when a different state is * reached and the state cannot ever reach the desired state anymore. */ void citus_job_wait_internal(int64 jobid, BackgroundJobStatus *desiredStatus) { /* * Since we are wait polling we will actually allocate memory on every poll. To make * sure we don't put unneeded pressure on the memory we create a context that we clear * every iteration. */ MemoryContext waitContext = AllocSetContextCreate(CurrentMemoryContext, "JobsWaitContext", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); MemoryContext oldContext = MemoryContextSwitchTo(waitContext); while (true) { MemoryContextReset(waitContext); BackgroundJob *job = GetBackgroundJobByJobId(jobid); if (!job) { ereport(ERROR, (errmsg("no job found for job with jobid: %ld", jobid))); } if (desiredStatus && job->state == *desiredStatus) { /* job has reached its desired status, done waiting */ break; } if (IsBackgroundJobStatusTerminal(job->state)) { if (desiredStatus) { /* * We have reached a terminal state, which is not the desired state we * were waiting for, otherwise we would have escaped earlier. Since it is * a terminal state we know that we can never reach the desired state. */ Oid reachedStatusOid = BackgroundJobStatusOid(job->state); Datum reachedStatusNameDatum = DirectFunctionCall1(enum_out, reachedStatusOid); char *reachedStatusName = DatumGetCString(reachedStatusNameDatum); Oid desiredStatusOid = BackgroundJobStatusOid(*desiredStatus); Datum desiredStatusNameDatum = DirectFunctionCall1(enum_out, desiredStatusOid); char *desiredStatusName = DatumGetCString(desiredStatusNameDatum); ereport(ERROR, (errmsg("Job reached terminal state \"%s\" instead of desired " "state \"%s\"", reachedStatusName, desiredStatusName))); } /* job has reached its terminal state, done waiting */ break; } /* sleep for a while, before rechecking the job status */ CHECK_FOR_INTERRUPTS(); const long delay_ms = 1000; (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); } MemoryContextSwitchTo(oldContext); MemoryContextDelete(waitContext); } /* * citus_task_wait_internal implements the waiting on a task for reuse in other areas where * we want to wait on tasks. * * When a desiredStatus is provided it will provide an error when a different state is * reached and the state cannot ever reach the desired state anymore. */ void citus_task_wait_internal(int64 taskid, BackgroundTaskStatus *desiredStatus) { /* * Since we are wait polling we will actually allocate memory on every poll. To make * sure we don't put unneeded pressure on the memory we create a context that we clear * every iteration. */ MemoryContext waitContext = AllocSetContextCreate(CurrentMemoryContext, "TasksWaitContext", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); MemoryContext oldContext = MemoryContextSwitchTo(waitContext); while (true) { MemoryContextReset(waitContext); BackgroundTask *task = GetBackgroundTaskByTaskId(taskid); if (!task) { ereport(ERROR, (errmsg("no task found with taskid: %ld", taskid))); } if (desiredStatus && task->status == *desiredStatus) { /* task has reached its desired status, done waiting */ break; } if (IsBackgroundTaskStatusTerminal(task->status)) { if (desiredStatus) { /* * We have reached a terminal state, which is not the desired state we * were waiting for, otherwise we would have escaped earlier. Since it is * a terminal state we know that we can never reach the desired state. */ Oid reachedStatusOid = BackgroundTaskStatusOid(task->status); Datum reachedStatusNameDatum = DirectFunctionCall1(enum_out, reachedStatusOid); char *reachedStatusName = DatumGetCString(reachedStatusNameDatum); Oid desiredStatusOid = BackgroundTaskStatusOid(*desiredStatus); Datum desiredStatusNameDatum = DirectFunctionCall1(enum_out, desiredStatusOid); char *desiredStatusName = DatumGetCString(desiredStatusNameDatum); ereport(ERROR, (errmsg("Task reached terminal state \"%s\" instead of desired " "state \"%s\"", reachedStatusName, desiredStatusName))); } /* task has reached its terminal state, done waiting */ break; } /* sleep for a while, before rechecking the task status */ CHECK_FOR_INTERRUPTS(); const long delay_ms = 100; (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); } MemoryContextSwitchTo(oldContext); MemoryContextDelete(waitContext); } /* * StartCitusBackgroundTaskQueueMonitor spawns a new background worker connected to the * current database and owner. This background worker consumes the tasks that are ready * for execution. */ BackgroundWorkerHandle * StartCitusBackgroundTaskQueueMonitor(Oid database, Oid extensionOwner) { char workerName[BGW_MAXLEN]; SafeSnprintf(workerName, BGW_MAXLEN, "Citus Background Task Queue Monitor: %u/%u", database, extensionOwner); CitusBackgroundWorkerConfig config = { .workerName = workerName, .functionName = "CitusBackgroundTaskQueueMonitorMain", .mainArg = ObjectIdGetDatum(MyDatabaseId), .extensionOwner = extensionOwner, .needsNotification = true, .waitForStartup = true, .restartTime = CITUS_BGW_NEVER_RESTART, .startTime = CITUS_BGW_DEFAULT_START_TIME, .workerType = NULL, /* use default */ .extraData = NULL, .extraDataSize = 0 }; return RegisterCitusBackgroundWorker(&config); } /* * context for any log/error messages emitted from the background task queue monitor. */ typedef struct CitusBackgroundTaskQueueMonitorErrorCallbackContext { const char *database; } CitusBackgroundTaskQueueMonitorCallbackContext; /* * CitusBackgroundTaskQueueMonitorErrorCallback is a callback handler that gets called for * any ereport to add extra context to the message. */ static void CitusBackgroundTaskQueueMonitorErrorCallback(void *arg) { CitusBackgroundTaskQueueMonitorCallbackContext *context = (CitusBackgroundTaskQueueMonitorCallbackContext *) arg; errcontext("Citus Background Task Queue Monitor: %s", context->database); } /* * NewExecutorExceedsCitusLimit returns true if currently we reached Citus' max worker count. */ static bool NewExecutorExceedsCitusLimit(QueueMonitorExecutionContext *queueMonitorExecutionContext) { if (queueMonitorExecutionContext->currentExecutorCount >= MaxBackgroundTaskExecutors) { /* * we hit to Citus' maximum task executor count. Warn for the first failure * after a successful worker allocation happened, that is, we do not warn if * we repeatedly come here without a successful worker allocation. */ if (queueMonitorExecutionContext->backgroundWorkerFailedStartTime == 0) { ereport(WARNING, (errmsg("unable to start background worker for " "background task execution"), errdetail( "Already reached the maximum number of task " "executors: %ld/%d", queueMonitorExecutionContext->currentExecutorCount, MaxBackgroundTaskExecutors))); queueMonitorExecutionContext->backgroundWorkerFailedStartTime = GetCurrentTimestamp(); } return true; } return false; } /* * NewExecutorExceedsPgMaxWorkers returns true if currently we reached Postgres' max worker count. */ static bool NewExecutorExceedsPgMaxWorkers(BackgroundWorkerHandle *handle, QueueMonitorExecutionContext *queueMonitorExecutionContext) { if (handle == NULL) { /* * we are unable to start a background worker for the task execution. * Probably we are out of background workers. Warn for the first failure * after a successful worker allocation happened, that is, we do not warn if * we repeatedly come here without a successful worker allocation. */ if (queueMonitorExecutionContext->backgroundWorkerFailedStartTime == 0) { ereport(WARNING, (errmsg("unable to start background worker for " "background task execution"), errdetail( "Current number of task " "executors: %ld/%d", queueMonitorExecutionContext->currentExecutorCount, MaxBackgroundTaskExecutors))); queueMonitorExecutionContext->backgroundWorkerFailedStartTime = GetCurrentTimestamp(); } return true; } return false; } /* * AssignRunnableTaskToNewExecutor tries to assign given runnable task to a new task executor. * It reports the assignment status as return value. */ static bool AssignRunnableTaskToNewExecutor(BackgroundTask *runnableTask, QueueMonitorExecutionContext * queueMonitorExecutionContext) { Assert(runnableTask && runnableTask->status == BACKGROUND_TASK_STATUS_RUNNABLE); if (NewExecutorExceedsCitusLimit(queueMonitorExecutionContext)) { /* escape if we hit citus executor limit */ return false; } char *databaseName = get_database_name(MyDatabaseId); char *userName = GetUserNameFromId(runnableTask->owner, false); /* try to create new executor and make it alive during queue monitor lifetime */ MemoryContext oldContext = MemoryContextSwitchTo(queueMonitorExecutionContext->ctx); dsm_segment *seg = NULL; BackgroundWorkerHandle *handle = StartCitusBackgroundTaskExecutor(databaseName, userName, runnableTask->command, runnableTask->taskid, runnableTask->jobid, &seg); MemoryContextSwitchTo(oldContext); if (NewExecutorExceedsPgMaxWorkers(handle, queueMonitorExecutionContext)) { /* escape if we hit pg worker limit */ return false; } /* assign the allocated executor to the runnable task and increment total executor count */ bool handleEntryFound = false; BackgroundExecutorHashEntry *handleEntry = hash_search( queueMonitorExecutionContext->currentExecutors, &runnableTask->taskid, HASH_ENTER, &handleEntryFound); Assert(!handleEntryFound); handleEntry->handle = handle; handleEntry->seg = seg; handleEntry->jobid = runnableTask->jobid; /* reset worker allocation timestamp and log time elapsed since the last failure */ CheckAndResetLastWorkerAllocationFailure(queueMonitorExecutionContext); /* make message alive during queue monitor lifetime */ oldContext = MemoryContextSwitchTo(queueMonitorExecutionContext->ctx); handleEntry->message = makeStringInfo(); MemoryContextSwitchTo(oldContext); /* set runnable task's status as running */ runnableTask->status = BACKGROUND_TASK_STATUS_RUNNING; UpdateBackgroundTask(runnableTask); UpdateBackgroundJob(runnableTask->jobid); queueMonitorExecutionContext->currentExecutorCount++; ereport(LOG, (errmsg("task jobid/taskid started: %ld/%ld", runnableTask->jobid, runnableTask->taskid))); return true; } /* * AssignRunnableTasks tries to assign all runnable tasks to a new task executor. * If an assignment fails, it stops in case we hit some limitation. We do not load * all the runnable tasks in memory at once as it can load memory much + we have * limited worker to which we can assign task. */ static void AssignRunnableTasks(QueueMonitorExecutionContext *queueMonitorExecutionContext) { BackgroundTask *runnableTask = NULL; bool taskAssigned = false; do { /* fetch a runnable task from catalog */ runnableTask = GetRunnableBackgroundTask(); if (runnableTask) { taskAssigned = AssignRunnableTaskToNewExecutor(runnableTask, queueMonitorExecutionContext); } else { taskAssigned = false; } } while (taskAssigned); } /* * GetRunningTaskEntries returns list of BackgroundExecutorHashEntry from given hash table */ static List * GetRunningTaskEntries(HTAB *currentExecutors) { List *runningTaskEntries = NIL; HASH_SEQ_STATUS status; BackgroundExecutorHashEntry *backgroundExecutorHashEntry; foreach_htab(backgroundExecutorHashEntry, &status, currentExecutors) { runningTaskEntries = lappend(runningTaskEntries, backgroundExecutorHashEntry); } return runningTaskEntries; } /* * CheckAndResetLastWorkerAllocationFailure checks the last time background worker allocation * is failed. If it is set, we print how long we have waited to successfully allocate the worker. * It also resets the failure timestamp. */ static void CheckAndResetLastWorkerAllocationFailure(QueueMonitorExecutionContext * queueMonitorExecutionContext) { if (queueMonitorExecutionContext->backgroundWorkerFailedStartTime > 0) { /* * we had a delay in starting the background worker for task execution. Report * the actual delay and reset the time. This allows a subsequent task to * report again if it can't start a background worker directly. */ long secs = 0; int microsecs = 0; TimestampDifference( queueMonitorExecutionContext-> backgroundWorkerFailedStartTime, GetCurrentTimestamp(), &secs, µsecs); ereport(LOG, (errmsg( "able to start a background worker with %ld seconds " "delay", secs))); queueMonitorExecutionContext->backgroundWorkerFailedStartTime = 0; } } /* * TaskConcurrentCancelCheck checks if concurrent task cancellation or removal happened by * taking Exclusive lock. It mutates task's pid and status. Returns execution status for the * task. */ static TaskExecutionStatus TaskConcurrentCancelCheck(TaskExecutionContext *taskExecutionContext) { /* * here we take exclusive lock on pg_dist_background_task table to prevent a * concurrent modification. A separate process could have cancelled or removed * the task by now, they would not see the pid and status update, so it is our * responsibility to stop the backend and update the pid and status. * * The lock will release on transaction commit. */ LockRelationOid(DistBackgroundTaskRelationId(), ExclusiveLock); BackgroundExecutorHashEntry *handleEntry = taskExecutionContext->handleEntry; BackgroundTask *task = GetBackgroundTaskByTaskId(handleEntry->taskid); taskExecutionContext->task = task; if (!task) { ereport(ERROR, (errmsg("unexpected missing task id: %ld", handleEntry->taskid))); } if (task->status == BACKGROUND_TASK_STATUS_CANCELLING) { /* * being in that step means that a concurrent cancel or removal happened. we should * mark task status as cancelled. We also want to reflect cancel message by consuming * task executor queue. */ bool hadError = false; ReadFromExecutorQueue(handleEntry, &hadError); ereport(LOG, (errmsg( "task jobid/taskid is cancelled: %ld/%ld", task->jobid, task->taskid))); task->status = BACKGROUND_TASK_STATUS_CANCELLED; return TASK_EXECUTION_STATUS_CANCELLED; } else { /* * now that we have verified the task has not been cancelled and still exist we * update it to reflect the new state. If task is already in running status, * the operation is idempotent. But for runnable tasks, we make their status * as running. */ pid_t pid = 0; GetBackgroundWorkerPid(handleEntry->handle, &pid); task->status = BACKGROUND_TASK_STATUS_RUNNING; SET_NULLABLE_FIELD(task, pid, pid); /* Update task status to indicate it is running */ UpdateBackgroundTask(task); UpdateBackgroundJob(task->jobid); return TASK_EXECUTION_STATUS_RUNNING; } } /* * ConsumeExecutorQueue consumes executor's shared memory queue and returns execution status * for the task. */ static TaskExecutionStatus ConsumeExecutorQueue(TaskExecutionContext *taskExecutionContext) { BackgroundExecutorHashEntry *handleEntry = taskExecutionContext->handleEntry; BackgroundTask *task = taskExecutionContext->task; /* * we consume task executor response queue. * possible response codes can lead us different steps below. */ bool hadError = false; shm_mq_result mq_res = ReadFromExecutorQueue(handleEntry, &hadError); if (hadError) { ereport(LOG, (errmsg("task jobid/taskid failed: %ld/%ld", task->jobid, task->taskid))); return TASK_EXECUTION_STATUS_ERROR; } else if (mq_res == SHM_MQ_DETACHED) { ereport(LOG, (errmsg("task jobid/taskid succeeded: %ld/%ld", task->jobid, task->taskid))); /* update task status as done. */ task->status = BACKGROUND_TASK_STATUS_DONE; return TASK_EXECUTION_STATUS_SUCCESS; } else { /* still running the task */ Assert(mq_res == SHM_MQ_WOULD_BLOCK); return TASK_EXECUTION_STATUS_WOULDBLOCK; } } /* * TaskHadError updates retry count of a failed task inside taskExecutionContext. * If maximum retry count is reached, task status is marked as failed. Otherwise, backoff * delay is calculated, notBefore time is updated and the task is marked as runnable. */ static void TaskHadError(TaskExecutionContext *taskExecutionContext) { BackgroundTask *task = taskExecutionContext->task; /* * when we had an error in response queue, we need to decide if we want to retry (keep the * runnable state), or move to error state */ if (!task->retry_count) { SET_NULLABLE_FIELD(task, retry_count, 1); } else { (*task->retry_count)++; } /* * based on the retry count we either transition the task to its error * state, or we calculate a new backoff time for future execution. */ int64 delayMs = CalculateBackoffDelay(*(task->retry_count)); if (delayMs < 0) { task->status = BACKGROUND_TASK_STATUS_ERROR; UNSET_NULLABLE_FIELD(task, not_before); } else { TimestampTz notBefore = TimestampTzPlusMilliseconds( GetCurrentTimestamp(), delayMs); SET_NULLABLE_FIELD(task, not_before, notBefore); task->status = BACKGROUND_TASK_STATUS_RUNNABLE; } TaskEnded(taskExecutionContext); } /* * TaskEnded updates task inside taskExecutionContext. It also updates depending * tasks and the job to which task belongs. At the end, it also updates executor map and * count inside queueMonitorExecutionContext after terminating the executor. */ static void TaskEnded(TaskExecutionContext *taskExecutionContext) { QueueMonitorExecutionContext *queueMonitorExecutionContext = taskExecutionContext->queueMonitorExecutionContext; HTAB *currentExecutors = queueMonitorExecutionContext->currentExecutors; BackgroundExecutorHashEntry *handleEntry = taskExecutionContext->handleEntry; BackgroundTask *task = taskExecutionContext->task; /* * we update task and job fields. We also update depending jobs. * At the end, do cleanup. */ UNSET_NULLABLE_FIELD(task, pid); task->message = handleEntry->message->data; UpdateBackgroundTask(task); UpdateDependingTasks(task); UpdateBackgroundJob(task->jobid); DecrementParallelTaskCountForNodesInvolved(task); /* we are sure that at least one task did not block on current iteration */ queueMonitorExecutionContext->allTasksWouldBlock = false; hash_search(currentExecutors, &task->taskid, HASH_REMOVE, NULL); WaitForBackgroundWorkerShutdown(handleEntry->handle); queueMonitorExecutionContext->currentExecutorCount--; } /* * IncrementParallelTaskCountForNodesInvolved * Checks whether we have reached the limit of parallel tasks per node * per each of the nodes involved with the task * If at least one limit is reached, it returns false. * If limits aren't reached, it increments the parallel task count * for each of the nodes involved with the task, and returns true. */ bool IncrementParallelTaskCountForNodesInvolved(BackgroundTask *task) { if (task->nodesInvolved) { int node; /* first check whether we have reached the limit for any of the nodes */ foreach_declared_int(node, task->nodesInvolved) { bool found; ParallelTasksPerNodeEntry *hashEntry = hash_search( ParallelTasksPerNode, &(node), HASH_ENTER, &found); if (!found) { hashEntry->counter = 0; } else if (hashEntry->counter >= MaxBackgroundTaskExecutorsPerNode) { /* at least one node's limit is reached */ return false; } } /* then, increment the parallel task count per each node */ foreach_declared_int(node, task->nodesInvolved) { ParallelTasksPerNodeEntry *hashEntry = hash_search( ParallelTasksPerNode, &(node), HASH_FIND, NULL); Assert(hashEntry); hashEntry->counter += 1; } } return true; } /* * DecrementParallelTaskCountForNodesInvolved * Decrements the parallel task count for each of the nodes involved * with the task. * We call this function after the task has gone through Running state * and then has ended. */ static void DecrementParallelTaskCountForNodesInvolved(BackgroundTask *task) { if (task->nodesInvolved) { int node; foreach_declared_int(node, task->nodesInvolved) { ParallelTasksPerNodeEntry *hashEntry = hash_search(ParallelTasksPerNode, &(node), HASH_FIND, NULL); hashEntry->counter -= 1; } } } /* * QueueMonitorSigHupHandler handles SIGHUP to update monitor related config params. */ static void QueueMonitorSigHupHandler(SIGNAL_ARGS) { int saved_errno = errno; GotSighup = true; if (MyProc) { SetLatch(&MyProc->procLatch); } errno = saved_errno; } /* * MonitorGotTerminationOrCancellationRequest returns true if monitor had SIGTERM or SIGINT signals */ static bool MonitorGotTerminationOrCancellationRequest() { return GotSigterm || GotSigint; } /* * QueueMonitorSigTermHandler handles SIGTERM by setting a flag to inform the monitor process * so that it can terminate active task executors properly. It also sets the latch to awake the * monitor if it waits on it. */ static void QueueMonitorSigTermHandler(SIGNAL_ARGS) { int saved_errno = errno; GotSigterm = true; if (MyProc) { SetLatch(&MyProc->procLatch); } errno = saved_errno; } /* * QueueMonitorSigIntHandler handles SIGINT by setting a flag to inform the monitor process * so that it can terminate active task executors properly. It also sets the latch to awake the * monitor if it waits on it. */ static void QueueMonitorSigIntHandler(SIGNAL_ARGS) { int saved_errno = errno; GotSigint = true; if (MyProc) { SetLatch(&MyProc->procLatch); } errno = saved_errno; } /* * TerminateAllTaskExecutors terminates task executors given in the hash map. */ static void TerminateAllTaskExecutors(HTAB *currentExecutors) { HASH_SEQ_STATUS status; BackgroundExecutorHashEntry *backgroundExecutorHashEntry; foreach_htab(backgroundExecutorHashEntry, &status, currentExecutors) { TerminateBackgroundWorker(backgroundExecutorHashEntry->handle); } } /* * GetRunningUniqueJobIds returns unique job ids from currentExecutors */ static HTAB * GetRunningUniqueJobIds(HTAB *currentExecutors) { /* create a set to store unique job ids for currently executing tasks */ HTAB *uniqueJobIds = CreateSimpleHashSetWithSize(int64, MAX_BG_TASK_EXECUTORS); HASH_SEQ_STATUS status; BackgroundExecutorHashEntry *backgroundExecutorHashEntry; foreach_htab(backgroundExecutorHashEntry, &status, currentExecutors) { hash_search(uniqueJobIds, &backgroundExecutorHashEntry->jobid, HASH_ENTER, NULL); } return uniqueJobIds; } /* * CancelAllTaskExecutors cancels task executors given in the hash map. */ static void CancelAllTaskExecutors(HTAB *currentExecutors) { StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); /* get unique job id set for running tasks in currentExecutors */ HTAB *uniqueJobIds = GetRunningUniqueJobIds(currentExecutors); HASH_SEQ_STATUS status; int64 *uniqueJobId; foreach_htab(uniqueJobId, &status, uniqueJobIds) { ereport(DEBUG1, (errmsg("cancelling job: %ld", *uniqueJobId))); Datum jobidDatum = Int64GetDatum(*uniqueJobId); DirectFunctionCall1(citus_job_cancel, jobidDatum); } PopActiveSnapshot(); CommitTransactionCommand(); } /* * CitusBackgroundTaskQueueMonitorMain is the main entry point for the background worker * running the background tasks queue monitor. * * It's mainloop reads a runnable task from pg_dist_background_task and progressing the * tasks and jobs state machines associated with the task. When no new task can be found * it will exit(0) and lets the maintenance daemon poll for new tasks. * * The main loop is implemented as asynchronous loop stepping through the task * and update its state before going to the next. Loop assigns runnable tasks to new task * executors as much as possible. If the max task executor limit is hit, the tasks will be * waiting in runnable status until currently running tasks finish. Each parallel worker * executes one task at a time without blocking each other by using nonblocking api. */ void CitusBackgroundTaskQueueMonitorMain(Datum arg) { /* handle SIGTERM to properly terminate active task executors */ pqsignal(SIGTERM, QueueMonitorSigTermHandler); /* handle SIGINT to properly cancel active task executors */ pqsignal(SIGINT, QueueMonitorSigIntHandler); /* handle SIGHUP to update MaxBackgroundTaskExecutors and MaxBackgroundTaskExecutorsPerNode */ pqsignal(SIGHUP, QueueMonitorSigHupHandler); /* ready to handle signals */ BackgroundWorkerUnblockSignals(); Oid databaseOid = DatumGetObjectId(arg); /* extension owner is passed via bgw_extra */ Oid extensionOwner = InvalidOid; memcpy_s(&extensionOwner, sizeof(extensionOwner), MyBgworkerEntry->bgw_extra, sizeof(Oid)); /* connect to database, after that we can actually access catalogs */ BackgroundWorkerInitializeConnectionByOid(databaseOid, extensionOwner, 0); /* * save old context until monitor loop exits, we use backgroundTaskContext for * all allocations. */ MemoryContext firstContext = CurrentMemoryContext; MemoryContext backgroundTaskContext = AllocSetContextCreate(TopMemoryContext, "BackgroundTaskContext", ALLOCSET_DEFAULT_SIZES); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); const char *databasename = get_database_name(MyDatabaseId); /* make databasename alive during queue monitor lifetime */ MemoryContext oldContext = MemoryContextSwitchTo(backgroundTaskContext); databasename = pstrdup(databasename); MemoryContextSwitchTo(oldContext); /* setup error context to indicate the errors came from a running background task */ ErrorContextCallback errorCallback = { 0 }; struct CitusBackgroundTaskQueueMonitorErrorCallbackContext context = { .database = databasename, }; errorCallback.callback = CitusBackgroundTaskQueueMonitorErrorCallback; errorCallback.arg = (void *) &context; errorCallback.previous = error_context_stack; error_context_stack = &errorCallback; PopActiveSnapshot(); CommitTransactionCommand(); /* * There should be exactly one background task monitor running, running multiple would * cause conflicts on processing the tasks in the catalog table as well as violate * parallelism guarantees. To make sure there is at most, exactly one backend running * we take a session lock on the CITUS_BACKGROUND_TASK_MONITOR operation. */ LOCKTAG tag = { 0 }; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_BACKGROUND_TASK_MONITOR); const bool sessionLock = true; const bool dontWait = true; LockAcquireResult locked = LockAcquire(&tag, AccessExclusiveLock, sessionLock, dontWait); if (locked == LOCKACQUIRE_NOT_AVAIL) { ereport(ERROR, (errmsg("background task queue monitor already running for " "database"))); } /* make worker recognizable in pg_stat_activity */ pgstat_report_appname("citus background task queue monitor"); ereport(DEBUG1, (errmsg("started citus background task queue monitor"))); /* * First we find all jobs that are running, we need to check if they are still running * if not reset their state back to scheduled. */ StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); ResetRunningBackgroundTasks(); PopActiveSnapshot(); CommitTransactionCommand(); /* create a map to store parallel task executors. Persist it in monitor memory context */ oldContext = MemoryContextSwitchTo(backgroundTaskContext); HTAB *currentExecutors = CreateSimpleHashWithNameAndSize(int64, BackgroundExecutorHashEntry, "Background Executor Hash", MAX_BG_TASK_EXECUTORS); MemoryContextSwitchTo(oldContext); /* * monitor execution context that is useful during the monitor loop. * we store current executor count, last background failure timestamp, * currently executed task context and also a memory context to persist * some allocations throughout the loop. */ QueueMonitorExecutionContext queueMonitorExecutionContext = { .currentExecutorCount = 0, .backgroundWorkerFailedStartTime = 0, .allTasksWouldBlock = true, .currentExecutors = currentExecutors, .ctx = backgroundTaskContext }; /* flag to prevent duplicate termination and cancellation of task executors */ bool terminateExecutorsStarted = false; bool cancelExecutorsStarted = false; /* loop exits if there is no running or runnable tasks left */ bool hasAnyTask = true; while (hasAnyTask) { /* handle signals */ CHECK_FOR_INTERRUPTS(); /* invalidate cache for new data in catalog */ InvalidateMetadataSystemCache(); /* * if the flag is set, we should terminate all task executor workers to prevent duplicate * runs of the same task on the next start of the monitor, which is dangerous for non-idempotent * tasks. We do not break the loop here as we want to reflect tasks' messages. Hence, we wait until * all tasks finish and also do not allow new runnable tasks to start running. After all current tasks * finish, we can exit the loop safely. */ if (GotSigterm && !terminateExecutorsStarted) { ereport(LOG, (errmsg("handling termination signal"))); terminateExecutorsStarted = true; TerminateAllTaskExecutors(queueMonitorExecutionContext.currentExecutors); } if (GotSigint && !cancelExecutorsStarted) { ereport(LOG, (errmsg("handling cancellation signal"))); cancelExecutorsStarted = true; CancelAllTaskExecutors(queueMonitorExecutionContext.currentExecutors); } if (GotSighup) { GotSighup = false; /* update max_background_task_executors and max_background_task_executors_per_node if changed */ ProcessConfigFile(PGC_SIGHUP); } if (ParallelTasksPerNode == NULL) { ParallelTasksPerNode = CreateSimpleHash(int32, ParallelTasksPerNodeEntry); } /* assign runnable tasks, if any, to new task executors in a transaction if we do not have SIGTERM or SIGINT */ if (!MonitorGotTerminationOrCancellationRequest()) { StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); AssignRunnableTasks(&queueMonitorExecutionContext); PopActiveSnapshot(); CommitTransactionCommand(); } /* get running task entries from hash table */ List *runningTaskEntries = GetRunningTaskEntries( queueMonitorExecutionContext.currentExecutors); hasAnyTask = list_length(runningTaskEntries) > 0; /* useful to sleep if all tasks ewouldblock on current iteration */ queueMonitorExecutionContext.allTasksWouldBlock = true; /* monitor executors inside transaction */ StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); /* iterate over all handle entries and monitor each task's output */ BackgroundExecutorHashEntry *handleEntry = NULL; foreach_declared_ptr(handleEntry, runningTaskEntries) { /* create task execution context and assign it to queueMonitorExecutionContext */ TaskExecutionContext taskExecutionContext = { .queueMonitorExecutionContext = &queueMonitorExecutionContext, .handleEntry = handleEntry, .task = NULL }; /* check if concurrent cancellation occurred */ TaskExecutionStatus taskExecutionStatus = TaskConcurrentCancelCheck( &taskExecutionContext); /* * check task status. If it is cancelled, we do not need to consume queue * as we already consumed the queue. */ if (taskExecutionStatus == TASK_EXECUTION_STATUS_CANCELLED) { TaskEnded(&taskExecutionContext); continue; } taskExecutionStatus = ConsumeExecutorQueue(&taskExecutionContext); if (taskExecutionStatus == TASK_EXECUTION_STATUS_ERROR) { TaskHadError(&taskExecutionContext); } else if (taskExecutionStatus == TASK_EXECUTION_STATUS_SUCCESS) { TaskEnded(&taskExecutionContext); } } PopActiveSnapshot(); CommitTransactionCommand(); if (queueMonitorExecutionContext.allTasksWouldBlock) { /* * sleep to lower cpu consumption if all tasks responded with EWOULD_BLOCK on the last iteration. * That will also let those tasks to progress to generate some output probably. */ const long delay_ms = 1000; (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, WAIT_EVENT_PG_SLEEP); ResetLatch(MyLatch); } } MemoryContextSwitchTo(firstContext); MemoryContextDelete(backgroundTaskContext); proc_exit(0); } /* * ReadFromExecutorQueue reads from task executor's response queue into the message. * It also sets hadError flag if an error response is encountered in the queue. */ static shm_mq_result ReadFromExecutorQueue(BackgroundExecutorHashEntry *backgroundExecutorHashEntry, bool *hadError) { dsm_segment *seg = backgroundExecutorHashEntry->seg; shm_toc *toc = shm_toc_attach(CITUS_BACKGROUND_TASK_MAGIC, dsm_segment_address(seg)); shm_mq *mq = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_QUEUE, false); shm_mq_handle *responseq = shm_mq_attach(mq, seg, NULL); /* * Consume background executor's queue and get a response code. */ StringInfo message = backgroundExecutorHashEntry->message; shm_mq_result mq_res = ConsumeTaskWorkerOutput(responseq, message, hadError); return mq_res; } /* * CalculateBackoffDelay calculates the time to backoff between retries. * * Per try we increase the delay as follows: * retry 1: 5 sec * retry 2: 20 sec * retry 3-32 (30 tries in total): 1 min * * returns -1 when retrying should stop. * * In the future we would like a callback on the job_type that could * distinguish the retry count and delay + potential jitter on a * job_type basis. For now we only assume this to be used by the * rebalancer and settled on the retry scheme above. */ static int64 CalculateBackoffDelay(int retryCount) { if (retryCount == 1) { return 5 * 1000; } else if (retryCount == 2) { return 20 * 1000; } else if (retryCount <= 32) { return 60 * 1000; } return -1; } /* * bgw_generate_returned_message - * generates the message to be inserted into the job_run_details table * first part is comming from error_severity (elog.c) */ static void bgw_generate_returned_message(StringInfoData *display_msg, ErrorData edata) { const char *prefix = error_severity(edata.elevel); appendStringInfo(display_msg, "%s: %s", prefix, edata.message); if (edata.detail != NULL) { appendStringInfo(display_msg, "\nDETAIL: %s", edata.detail); } if (edata.hint != NULL) { appendStringInfo(display_msg, "\nHINT: %s", edata.hint); } if (edata.context != NULL) { appendStringInfo(display_msg, "\nCONTEXT: %s", edata.context); } } /* * UpdateDependingTasks updates all depending tasks, based on the type of terminal state * the current task reached. */ static void UpdateDependingTasks(BackgroundTask *task) { switch (task->status) { case BACKGROUND_TASK_STATUS_DONE: { UnblockDependingBackgroundTasks(task); break; } case BACKGROUND_TASK_STATUS_ERROR: { /* when we error this task, we need to unschedule all dependant tasks */ UnscheduleDependentTasks(task); break; } default: { /* nothing to do for other states */ break; } } } /* * ConsumeTaskWorkerOutput consumes the output of an executor and sets the message as * the last message read from the queue. It also sets hadError as true if executor had * error. */ static shm_mq_result ConsumeTaskWorkerOutput(shm_mq_handle *responseq, StringInfo message, bool *hadError) { shm_mq_result res; /* * Message-parsing routines operate on a null-terminated StringInfo, * so we must construct one. */ StringInfoData msg = { 0 }; initStringInfo(&msg); for (;;) { resetStringInfo(&msg); /* * non-blocking receive to not block other bg workers */ Size nbytes = 0; void *data = NULL; const bool noWait = true; res = shm_mq_receive(responseq, &nbytes, &data, noWait); if (res != SHM_MQ_SUCCESS) { break; } appendBinaryStringInfo(&msg, data, nbytes); /* * msgtype seems to be documented on * https://www.postgresql.org/docs/current/protocol-message-formats.html * * Here we mostly handle the same message types as supported in pg_cron as the * executor is highly influenced by the implementation there. */ char msgtype = pq_getmsgbyte(&msg); switch (msgtype) { case 'E': /* ErrorResponse */ { if (hadError) { *hadError = true; } } /* FALLTHROUGH */ case 'N': /* NoticeResponse */ { ErrorData edata = { 0 }; StringInfoData display_msg = { 0 }; pq_parse_errornotice(&msg, &edata); initStringInfo(&display_msg); bgw_generate_returned_message(&display_msg, edata); /* we keep only the last message */ resetStringInfo(message); appendStringInfoString(message, display_msg.data); appendStringInfoChar(message, '\n'); pfree(display_msg.data); break; } case 'C': /* CommandComplete */ { const char *tag = pq_getmsgstring(&msg); char *nonconst_tag = pstrdup(tag); /* append the nonconst_tag to the task's message */ appendStringInfoString(message, nonconst_tag); appendStringInfoChar(message, '\n'); pfree(nonconst_tag); break; } case 'A': case 'D': case 'G': case 'H': case 'T': case 'W': case 'Z': { break; } default: { elog(WARNING, "unknown message type: %c (%zu bytes)", msg.data[0], nbytes); break; } } } pfree(msg.data); return res; } /* * StoreArgumentsInDSM creates a dynamic shared memory segment to pass the query and its * environment to the executor. */ static dsm_segment * StoreArgumentsInDSM(char *database, char *username, char *command, int64 taskId, int64 jobId) { /* * Create the shared memory that we will pass to the background * worker process. We use DSM_CREATE_NULL_IF_MAXSEGMENTS so that we * do not ERROR here. This way, we can mark the job as failed and * keep the launcher process running normally. */ shm_toc_estimator e = { 0 }; shm_toc_initialize_estimator(&e); shm_toc_estimate_chunk(&e, strlen(database) + 1); shm_toc_estimate_chunk(&e, strlen(username) + 1); shm_toc_estimate_chunk(&e, strlen(command) + 1); #define QUEUE_SIZE ((Size) 65536) shm_toc_estimate_chunk(&e, QUEUE_SIZE); shm_toc_estimate_chunk(&e, sizeof(int64)); shm_toc_estimate_chunk(&e, sizeof(int64)); shm_toc_estimate_keys(&e, CITUS_BACKGROUND_TASK_NKEYS); Size segsize = shm_toc_estimate(&e); dsm_segment *seg = dsm_create(segsize, DSM_CREATE_NULL_IF_MAXSEGMENTS); if (seg == NULL) { ereport(ERROR, (errmsg("max number of DSM segments may has been reached"))); return NULL; } shm_toc *toc = shm_toc_create(CITUS_BACKGROUND_TASK_MAGIC, dsm_segment_address(seg), segsize); Size size = strlen(database) + 1; char *databaseTarget = shm_toc_allocate(toc, size); strcpy_s(databaseTarget, size, database); shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_DATABASE, databaseTarget); size = strlen(username) + 1; char *usernameTarget = shm_toc_allocate(toc, size); strcpy_s(usernameTarget, size, username); shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_USERNAME, usernameTarget); size = strlen(command) + 1; char *commandTarget = shm_toc_allocate(toc, size); strcpy_s(commandTarget, size, command); shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_COMMAND, commandTarget); shm_mq *mq = shm_mq_create(shm_toc_allocate(toc, QUEUE_SIZE), QUEUE_SIZE); shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_QUEUE, mq); shm_mq_set_receiver(mq, MyProc); int64 *taskIdTarget = shm_toc_allocate(toc, sizeof(int64)); *taskIdTarget = taskId; shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_TASK_ID, taskIdTarget); int64 *jobIdTarget = shm_toc_allocate(toc, sizeof(int64)); *jobIdTarget = jobId; shm_toc_insert(toc, CITUS_BACKGROUND_TASK_KEY_JOB_ID, jobIdTarget); shm_mq_attach(mq, seg, NULL); /* * when we have CurrentResourceOwner != NULL, segment will be released upon CurrentResourceOwner release, * but we may consume the queue in segment even after CurrentResourceOwner released. 'dsm_pin_mapping' helps * persisting the segment until the session ends or the segment is detached explicitly by 'dsm_detach'. */ dsm_pin_mapping(seg); return seg; } /* * StartCitusBackgroundTaskExecutor start a new background worker for the execution of a * background task. Callers interested in the shared memory segment that is created * between the background worker and the current backend can pass in a segOut to get a * pointer to the dynamic shared memory. */ static BackgroundWorkerHandle * StartCitusBackgroundTaskExecutor(char *database, char *user, char *command, int64 taskId, int64 jobId, dsm_segment **pSegment) { dsm_segment *seg = StoreArgumentsInDSM(database, user, command, taskId, jobId); char workerName[BGW_MAXLEN]; SafeSnprintf(workerName, BGW_MAXLEN, "Citus Background Task Queue Executor: %s/%s for (%ld/%ld)", database, user, jobId, taskId); CitusBackgroundWorkerConfig config = { .workerName = workerName, .functionName = "CitusBackgroundTaskExecutor", .mainArg = UInt32GetDatum(dsm_segment_handle(seg)), .extensionOwner = InvalidOid, .needsNotification = true, .waitForStartup = true, .restartTime = CITUS_BGW_NEVER_RESTART, .startTime = CITUS_BGW_DEFAULT_START_TIME, .workerType = NULL, /* use default */ .extraData = NULL, .extraDataSize = 0 }; BackgroundWorkerHandle *handle = RegisterCitusBackgroundWorker(&config); if (!handle) { dsm_detach(seg); return NULL; } if (pSegment) { *pSegment = seg; } return handle; } /* * context for any log/error messages emitted from the background task executor. */ typedef struct CitusBackgroundJobExecutorErrorCallbackContext { const char *database; const char *username; int64 taskId; int64 jobId; } CitusBackgroundJobExecutorErrorCallbackContext; /* * CitusBackgroundJobExecutorErrorCallback is a callback handler that gets called for any * ereport to add extra context to the message. */ static void CitusBackgroundJobExecutorErrorCallback(void *arg) { CitusBackgroundJobExecutorErrorCallbackContext *context = (CitusBackgroundJobExecutorErrorCallbackContext *) arg; errcontext("Citus Background Task Queue Executor: %s/%s for (%ld/%ld)", context->database, context->username, context->jobId, context->taskId); } /* * CitusBackgroundTaskExecutor is the main function of the background tasks queue * executor. This backend attaches to a shared memory segment as identified by the * main_arg of the background worker. * * This is mostly based on the background worker logic in pg_cron */ void CitusBackgroundTaskExecutor(Datum main_arg) { /* handles SIGTERM similar to backends */ pqsignal(SIGTERM, die); BackgroundWorkerUnblockSignals(); /* Set up a dynamic shared memory segment. */ dsm_segment *seg = dsm_attach(DatumGetInt32(main_arg)); if (seg == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("unable to map dynamic shared memory segment"))); } shm_toc *toc = shm_toc_attach(CITUS_BACKGROUND_TASK_MAGIC, dsm_segment_address(seg)); if (toc == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("bad magic number in dynamic shared memory segment"))); } char *database = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_DATABASE, false); char *username = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_USERNAME, false); char *command = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_COMMAND, false); int64 *taskId = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_TASK_ID, false); int64 *jobId = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_JOB_ID, false); shm_mq *mq = shm_toc_lookup(toc, CITUS_BACKGROUND_TASK_KEY_QUEUE, false); shm_mq_set_sender(mq, MyProc); shm_mq_handle *responseq = shm_mq_attach(mq, seg, NULL); pq_redirect_to_shm_mq(seg, responseq); /* setup error context to indicate the errors came from a running background task */ ErrorContextCallback errorCallback = { 0 }; CitusBackgroundJobExecutorErrorCallbackContext context = { .database = database, .username = username, .taskId = *taskId, .jobId = *jobId, }; errorCallback.callback = CitusBackgroundJobExecutorErrorCallback; errorCallback.arg = (void *) &context; errorCallback.previous = error_context_stack; error_context_stack = &errorCallback; BackgroundWorkerInitializeConnection(database, username, 0); /* make sure we are the only backend running for this task */ LOCKTAG locktag = { 0 }; SET_LOCKTAG_BACKGROUND_TASK(locktag, *taskId); const bool sessionLock = true; const bool dontWait = true; LockAcquireResult locked = LockAcquire(&locktag, AccessExclusiveLock, sessionLock, dontWait); if (locked == LOCKACQUIRE_NOT_AVAIL) { ereport(ERROR, (errmsg("unable to acquire background task lock for taskId: %ld", *taskId), errdetail("this indicates that an other backend is already " "executing this task"))); } /* Execute the query. */ StartTransactionCommand(); ExecuteSqlString(command); CommitTransactionCommand(); /* Signal that we are done. */ ReadyForQuery(DestRemote); dsm_detach(seg); proc_exit(0); } /* * Execute given SQL string without SPI or a libpq session. */ static void ExecuteSqlString(const char *sql) { /* * Parse the SQL string into a list of raw parse trees. * * Because we allow statements that perform internal transaction control, * we can't do this in TopTransactionContext; the parse trees might get * blown away before we're done executing them. */ MemoryContext parsecontext = AllocSetContextCreate(CurrentMemoryContext, "query parse/plan", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); MemoryContext oldcontext = MemoryContextSwitchTo(parsecontext); List *raw_parsetree_list = pg_parse_query(sql); int commands_remaining = list_length(raw_parsetree_list); bool isTopLevel = commands_remaining == 1; MemoryContextSwitchTo(oldcontext); /* * Do parse analysis, rule rewrite, planning, and execution for each raw * parsetree. We must fully execute each query before beginning parse * analysis on the next one, since there may be interdependencies. */ RawStmt *parsetree = NULL; foreach_declared_ptr(parsetree, raw_parsetree_list) { /* * We don't allow transaction-control commands like COMMIT and ABORT * here. The entire SQL statement is executed as a single transaction * which commits if no errors are encountered. */ if (IsA(parsetree, TransactionStmt)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "transaction control statements are not allowed in background job"))); } /* * Get the command name for use in status display (it also becomes the * default completion tag, down inside PortalRun). Set ps_status and * do any special start-of-SQL-command processing needed by the * destination. */ CommandTag commandTag = CreateCommandTag(parsetree->stmt); set_ps_display(GetCommandTagName(commandTag)); BeginCommand(commandTag, DestNone); /* Set up a snapshot if parse analysis/planning will need one. */ bool snapshot_set = false; if (analyze_requires_snapshot(parsetree)) { PushActiveSnapshot(GetTransactionSnapshot()); snapshot_set = true; } /* * OK to analyze, rewrite, and plan this query. * * As with parsing, we need to make sure this data outlives the * transaction, because of the possibility that the statement might * perform internal transaction control. */ oldcontext = MemoryContextSwitchTo(parsecontext); #if PG_VERSION_NUM >= 150000 List *querytree_list = pg_analyze_and_rewrite_fixedparams(parsetree, sql, NULL, 0, NULL); #else List *querytree_list = pg_analyze_and_rewrite(parsetree, sql, NULL, 0, NULL); #endif List *plantree_list = pg_plan_queries(querytree_list, sql, 0, NULL); /* Done with the snapshot used for parsing/planning */ if (snapshot_set) { PopActiveSnapshot(); } /* If we got a cancel signal in analysis or planning, quit */ CHECK_FOR_INTERRUPTS(); /* * Execute the query using the unnamed portal. */ Portal portal = CreatePortal("", true, true); /* Don't display the portal in pg_cursors */ portal->visible = false; /* PG17-: six‐arg signature */ PortalDefineQuery( portal, NULL, /* no prepared‐stmt name */ sql, /* the query text */ commandTag, /* the CommandTag */ plantree_list, /* List of PlannedStmt* */ NULL /* no CachedPlan */ ); PortalStart(portal, NULL, 0, InvalidSnapshot); int16 format[] = { 1 }; PortalSetResultFormat(portal, lengthof(format), format); /* binary format */ commands_remaining--; DestReceiver *receiver = CreateDestReceiver(DestNone); /* * Only once the portal and destreceiver have been established can * we return to the transaction context. All that stuff needs to * survive an internal commit inside PortalRun! */ MemoryContextSwitchTo(oldcontext); /* Here's where we actually execute the command. */ QueryCompletion qc = { 0 }; /* Execute the portal, dropping the `run_once` arg on PG18+ */ #if PG_VERSION_NUM >= PG_VERSION_18 (void) PortalRun( portal, FETCH_ALL, /* count */ isTopLevel, /* isTopLevel */ receiver, /* DestReceiver *dest */ receiver, /* DestReceiver *altdest */ &qc /* QueryCompletion *qc */ ); #else (void) PortalRun( portal, FETCH_ALL, /* count */ isTopLevel, /* isTopLevel */ true, /* run_once */ receiver, /* DestReceiver *dest */ receiver, /* DestReceiver *altdest */ &qc /* QueryCompletion *qc */ ); #endif /* Clean up the receiver. */ (*receiver->rDestroy)(receiver); /* * Send a CommandComplete message even if we suppressed the query * results. The user backend will report these in the absence of * any true query results. */ EndCommand(&qc, DestRemote, false); /* Clean up the portal. */ PortalDrop(portal, false); } /* Be sure to advance the command counter after the last script command */ CommandCounterIncrement(); } ================================================ FILE: src/backend/distributed/utils/background_worker_utils.c ================================================ /*------------------------------------------------------------------------- * * background_worker_utils.c * Common utilities for initializing PostgreSQL background workers * used by Citus distributed infrastructure. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/proc.h" #include "distributed/background_worker_utils.h" #include "distributed/citus_safe_lib.h" /* * InitializeCitusBackgroundWorker initializes a BackgroundWorker struct * with common Citus background worker settings. */ void InitializeCitusBackgroundWorker(BackgroundWorker *worker, const CitusBackgroundWorkerConfig *config) { Assert(worker != NULL); Assert(config != NULL); Assert(config->workerName != NULL); Assert(config->functionName != NULL); /* Initialize the worker structure */ memset(worker, 0, sizeof(BackgroundWorker)); /* Set worker name */ strcpy_s(worker->bgw_name, sizeof(worker->bgw_name), config->workerName); /* Set worker type if provided */ if (config->workerType != NULL) { strcpy_s(worker->bgw_type, sizeof(worker->bgw_type), config->workerType); } /* Set standard flags for Citus workers */ worker->bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; /* Set start time - use custom start time if provided, otherwise use default */ worker->bgw_start_time = (config->startTime != 0) ? config->startTime : CITUS_BGW_DEFAULT_START_TIME; /* Set restart behavior */ worker->bgw_restart_time = config->restartTime; /* Set library and function names */ strcpy_s(worker->bgw_library_name, sizeof(worker->bgw_library_name), "citus"); strcpy_s(worker->bgw_function_name, sizeof(worker->bgw_function_name), config->functionName); /* Set main argument */ worker->bgw_main_arg = config->mainArg; /* Set extension owner if provided */ if (OidIsValid(config->extensionOwner)) { memcpy_s(worker->bgw_extra, sizeof(worker->bgw_extra), &config->extensionOwner, sizeof(Oid)); } /* Set additional extra data if provided */ if (config->extraData != NULL && config->extraDataSize > 0) { size_t remainingSpace = sizeof(worker->bgw_extra); size_t usedSpace = OidIsValid(config->extensionOwner) ? sizeof(Oid) : 0; if (usedSpace + config->extraDataSize <= remainingSpace) { memcpy_s(((char *) worker->bgw_extra) + usedSpace, remainingSpace - usedSpace, config->extraData, config->extraDataSize); } } /* Set notification PID if needed */ if (config->needsNotification) { worker->bgw_notify_pid = MyProcPid; } } /* * RegisterCitusBackgroundWorker creates and registers a Citus background worker * with the specified configuration. Returns the worker handle on success, * NULL on failure. */ BackgroundWorkerHandle * RegisterCitusBackgroundWorker(const CitusBackgroundWorkerConfig *config) { BackgroundWorker worker; BackgroundWorkerHandle *handle = NULL; /* Initialize the worker structure */ InitializeCitusBackgroundWorker(&worker, config); /* Register the background worker */ if (!RegisterDynamicBackgroundWorker(&worker, &handle)) { return NULL; } /* Wait for startup if requested */ if (config->waitForStartup && handle != NULL) { pid_t pid = 0; WaitForBackgroundWorkerStartup(handle, &pid); } return handle; } ================================================ FILE: src/backend/distributed/utils/cancel_utils.c ================================================ /* * cancel_utils.c * * Utilities related to query cancellation * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "miscadmin.h" #include "distributed/cancel_utils.h" /* * IsHoldOffCancellationReceived returns true if a cancel signal * was sent and HOLD_INTERRUPTS was called prior to this. The motivation * here is that since our queries can take a long time, in some places * we do not want to wait even if HOLD_INTERRUPTS was called. */ bool IsHoldOffCancellationReceived() { return InterruptHoldoffCount > 0 && (QueryCancelPending || ProcDiePending); } ================================================ FILE: src/backend/distributed/utils/citus_clauses.c ================================================ /* * citus_clauses.c * * Routines roughly equivalent to postgres' util/clauses. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "executor/executor.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/planmain.h" #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "distributed/citus_clauses.h" #include "distributed/insert_select_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_router_planner.h" #include "distributed/version_compat.h" /* private function declarations */ static bool IsVariableExpression(Node *node); static Expr * citus_evaluate_expr(Expr *expr, Oid result_type, int32 result_typmod, Oid result_collation, CoordinatorEvaluationContext * coordinatorEvaluationContext); static bool CitusIsVolatileFunctionIdChecker(Oid func_id, void *context); static bool CitusIsMutableFunctionIdChecker(Oid func_id, void *context); static bool ShouldEvaluateExpression(Expr *expression); static bool ShouldEvaluateFunctions(CoordinatorEvaluationContext *evaluationContext); static void FixFunctionArguments(Node *expr); static bool FixFunctionArgumentsWalker(Node *expr, void *context); static bool CheckExprExecutorSafe(Node *expr); /* * RequiresCoordinatorEvaluation returns the executor needs to reparse and * try to execute this query, which is the case if the query contains * any stable or volatile function. */ bool RequiresCoordinatorEvaluation(Query *query) { if (query->commandType == CMD_SELECT && !query->hasModifyingCTE) { return false; } return FindNodeMatchingCheckFunction((Node *) query, CitusIsMutableFunction); } /* * ExecuteCoordinatorEvaluableExpressions evaluates expressions and parameters * that can be resolved to a constant. */ void ExecuteCoordinatorEvaluableExpressions(Query *query, PlanState *planState) { CoordinatorEvaluationContext coordinatorEvaluationContext; coordinatorEvaluationContext.planState = planState; if (query->commandType == CMD_SELECT) { coordinatorEvaluationContext.evaluationMode = EVALUATE_PARAMS; } else { coordinatorEvaluationContext.evaluationMode = EVALUATE_FUNCTIONS_PARAMS; } PartiallyEvaluateExpression((Node *) query, &coordinatorEvaluationContext); } /* * PartiallyEvaluateExpression descends into an expression tree to evaluate * expressions that can be resolved to a constant on the master. Expressions * containing a Var are skipped, since the value of the Var is not known * on the master. */ Node * PartiallyEvaluateExpression(Node *expression, CoordinatorEvaluationContext *coordinatorEvaluationContext) { if (expression == NULL || IsA(expression, Const)) { return expression; } NodeTag nodeTag = nodeTag(expression); /* ExecInitExpr cannot handle some expressions (PARAM_MULTIEXPR and PARAM_SUBLINK) */ if (!CheckExprExecutorSafe(expression)) { return expression; } /* ExecInitExpr cannot handle PARAM_MULTIEXPR and PARAM_SUBLINK but we have guards */ else if (nodeTag == T_Param) { Assert(((Param *) expression)->paramkind != PARAM_MULTIEXPR && ((Param *) expression)->paramkind != PARAM_SUBLINK); return (Node *) citus_evaluate_expr((Expr *) expression, exprType(expression), exprTypmod(expression), exprCollation(expression), coordinatorEvaluationContext); } else if (ShouldEvaluateExpression((Expr *) expression) && ShouldEvaluateFunctions(coordinatorEvaluationContext)) { /* * The planner normally evaluates constant expressions, but we may be * working on the original query tree. We could rely on * citus_evaluate_expr to evaluate constant expressions, but there are * certain node types that citus_evaluate_expr does not expect because * the planner normally replaces them (in particular, CollateExpr). * Hence, we first evaluate constant expressions using * eval_const_expressions before continuing. * * NOTE: We do not use expression_planner here, since all it does * apart from calling eval_const_expressions is call fix_opfuncids. * We do not need this, since that is already called in * citus_evaluate_expr. So we won't needlessly traverse the expression * tree by calling it another time. */ expression = eval_const_expressions(NULL, expression); /* * It's possible that after evaluating const expressions we * actually don't need to evaluate this expression anymore e.g: * * 1 = 0 AND now() > timestamp '10-10-2000 00:00' * * This statement would simply resolve to false, because 1 = 0 is * false. That's why we now check again if we should evaluate the * expression and only continue if we still do. */ if (!ShouldEvaluateExpression((Expr *) expression)) { return (Node *) expression_tree_mutator(expression, PartiallyEvaluateExpression, coordinatorEvaluationContext); } if (FindNodeMatchingCheckFunction(expression, IsVariableExpression)) { /* * The expression contains a variable expression (e.g. a stable function, * which has a column reference as its input). That means that we cannot * evaluate the expression on the coordinator, since the result depends * on the input. * * Skipping function evaluation for these expressions is safe in most * cases, since the function will always be re-evaluated for every input * value. An exception is function calls that call another stable function * that should not be re-evaluated, such as now(). */ return (Node *) expression_tree_mutator(expression, PartiallyEvaluateExpression, coordinatorEvaluationContext); } return (Node *) citus_evaluate_expr((Expr *) expression, exprType(expression), exprTypmod(expression), exprCollation(expression), coordinatorEvaluationContext); } else if (nodeTag == T_Query) { Query *query = (Query *) expression; CoordinatorEvaluationContext subContext = *coordinatorEvaluationContext; if (query->commandType != CMD_SELECT) { /* * Currently INSERT SELECT evaluates stable functions on master, * while a plain SELECT does not. For evaluating SELECT evaluationMode is * EVALUATE_PARAMS, but if recursing into a modifying CTE switch into * EVALUATE_FUNCTIONS_PARAMS. */ subContext.evaluationMode = EVALUATE_FUNCTIONS_PARAMS; } return (Node *) query_tree_mutator(query, PartiallyEvaluateExpression, &subContext, QTW_DONT_COPY_QUERY); } else { return (Node *) expression_tree_mutator(expression, PartiallyEvaluateExpression, coordinatorEvaluationContext); } return expression; } /* * ShouldEvaluateFunctions is a helper function which is used to * decide whether the function/expression should be evaluated with the input * coordinatorEvaluationContext. */ static bool ShouldEvaluateFunctions(CoordinatorEvaluationContext *evaluationContext) { if (evaluationContext == NULL) { /* if no context provided, evaluate, which is the default behaviour */ return true; } return evaluationContext->evaluationMode == EVALUATE_FUNCTIONS_PARAMS; } /* * ShouldEvaluateExpression returns true if Citus should evaluate the * input node on the coordinator. */ static bool ShouldEvaluateExpression(Expr *expression) { NodeTag nodeTag = nodeTag(expression); switch (nodeTag) { case T_FuncExpr: { FuncExpr *funcExpr = (FuncExpr *) expression; /* we cannot evaluate set returning functions */ bool isSetReturningFunction = funcExpr->funcretset; return !isSetReturningFunction; } case T_OpExpr: case T_DistinctExpr: case T_NullIfExpr: case T_CoerceViaIO: case T_ArrayCoerceExpr: case T_ScalarArrayOpExpr: case T_RowExpr: case T_RowCompareExpr: case T_RelabelType: case T_CoerceToDomain: case T_NextValueExpr: { return true; } default: { return false; } } } /* * IsVariableExpression returns whether the given node is a variable expression, * meaning its result depends on the input data and is not constant for the whole * query. */ static bool IsVariableExpression(Node *node) { if (IsA(node, Aggref)) { return true; } if (IsA(node, WindowFunc)) { return true; } if (IsA(node, Param)) { /* ExecInitExpr cannot handle PARAM_SUBLINK */ return ((Param *) node)->paramkind == PARAM_SUBLINK; } return IsA(node, Var); } /* * a copy of pg's evaluate_expr, pre-evaluate a constant expression * * We use the executor's routine ExecEvalExpr() to avoid duplication of * code and ensure we get the same result as the executor would get. * * *INDENT-OFF* */ static Expr * citus_evaluate_expr(Expr *expr, Oid result_type, int32 result_typmod, Oid result_collation, CoordinatorEvaluationContext *coordinatorEvaluationContext) { PlanState *planState = NULL; EState *estate; ExprState *exprstate; Datum const_val; bool const_is_null; int16 resultTypLen; bool resultTypByVal; if (coordinatorEvaluationContext) { planState = coordinatorEvaluationContext->planState; if (IsA(expr, Param)) { if (coordinatorEvaluationContext->evaluationMode == EVALUATE_NONE) { /* bail out, the caller doesn't want params to be evaluated */ return expr; } } else if (coordinatorEvaluationContext->evaluationMode != EVALUATE_FUNCTIONS_PARAMS) { /* should only get here for node types we should evaluate */ Assert(ShouldEvaluateExpression(expr)); /* bail out, the caller doesn't want functions/expressions to be evaluated */ return expr; } } /* * To use the executor, we need an EState. */ estate = CreateExecutorState(); /* We can use the estate's working context to avoid memory leaks. */ MemoryContext oldcontext = MemoryContextSwitchTo(estate->es_query_cxt); /* handles default values */ FixFunctionArguments((Node *) expr); /* Make sure any opfuncids are filled in. */ fix_opfuncids((Node *) expr); /* * Prepare expr for execution. (Note: we can't use ExecPrepareExpr * because it'd result in recursively invoking eval_const_expressions.) */ exprstate = ExecInitExpr(expr, planState); /* * Get short lived per tuple context as evaluate_expr does. Here we don't * use planState->ExprContext as it might cause double-free'ing executor * state. */ ExprContext *econtext = GetPerTupleExprContext(estate); if (planState) { /* * If planState exists, then we add es_param_list_info to per tuple * ExprContext as we need them when evaluating prepared statements. */ econtext->ecxt_param_list_info = planState->state->es_param_list_info; } /* * And evaluate it. */ const_val = ExecEvalExprSwitchContext(exprstate, econtext, &const_is_null); /* Get info needed about result datatype */ get_typlenbyval(result_type, &resultTypLen, &resultTypByVal); /* Get back to outer memory context */ MemoryContextSwitchTo(oldcontext); /* * Must copy result out of sub-context used by expression eval. * * Also, if it's varlena, forcibly detoast it. This protects us against * storing TOAST pointers into plans that might outlive the referenced * data. (makeConst would handle detoasting anyway, but it's worth a few * extra lines here so that we can do the copy and detoast in one step.) */ if (!const_is_null) { if (resultTypLen == -1) const_val = PointerGetDatum(PG_DETOAST_DATUM_COPY(const_val)); else const_val = datumCopy(const_val, resultTypByVal, resultTypLen); } /* Release all the junk we just created */ FreeExecutorState(estate); /* * Make the constant result node. */ return (Expr *) makeConst(result_type, result_typmod, result_collation, resultTypLen, const_val, const_is_null, resultTypByVal); } /* *INDENT-ON* */ /* * CitusIsVolatileFunctionIdChecker checks if the given function id is * a volatile function other than read_intermediate_result(). */ static bool CitusIsVolatileFunctionIdChecker(Oid func_id, void *context) { if (func_id == CitusReadIntermediateResultFuncId() || func_id == CitusReadIntermediateResultArrayFuncId()) { return false; } return (func_volatile(func_id) == PROVOLATILE_VOLATILE); } /* * CitusIsVolatileFunction checks if the given node is a volatile function * other than Citus's internal functions. */ bool CitusIsVolatileFunction(Node *node) { /* Check for volatile functions in node itself */ if (check_functions_in_node(node, CitusIsVolatileFunctionIdChecker, NULL)) { return true; } if (IsA(node, NextValueExpr)) { /* NextValueExpr is volatile */ return true; } return false; } /* * CitusIsMutableFunctionIdChecker checks if the given function id is * a mutable function other than read_intermediate_result(). */ static bool CitusIsMutableFunctionIdChecker(Oid func_id, void *context) { if (func_id == CitusReadIntermediateResultFuncId() || func_id == CitusReadIntermediateResultArrayFuncId()) { return false; } else { return (func_volatile(func_id) != PROVOLATILE_IMMUTABLE); } } /* * CitusIsMutableFunction checks if the given node is a mutable function * other than Citus's internal functions. */ bool CitusIsMutableFunction(Node *node) { /* Check for mutable functions in node itself */ if (check_functions_in_node(node, CitusIsMutableFunctionIdChecker, NULL)) { return true; } if (IsA(node, SQLValueFunction)) { /* all variants of SQLValueFunction are stable */ return true; } if (IsA(node, NextValueExpr)) { /* NextValueExpr is volatile */ return true; } return false; } /* FixFunctionArguments applies expand_function_arguments to all function calls. */ static void FixFunctionArguments(Node *expr) { FixFunctionArgumentsWalker(expr, NULL); } /* FixFunctionArgumentsWalker is the helper function for fix_funcargs. */ static bool FixFunctionArgumentsWalker(Node *expr, void *context) { if (expr == NULL) { return false; } if (IsA(expr, FuncExpr)) { FuncExpr *funcExpr = castNode(FuncExpr, expr); HeapTuple func_tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcExpr->funcid)); if (!HeapTupleIsValid(func_tuple)) { elog(ERROR, "cache lookup failed for function %u", funcExpr->funcid); } funcExpr->args = expand_function_arguments(funcExpr->args, false, funcExpr->funcresulttype, func_tuple); ReleaseSysCache(func_tuple); } return expression_tree_walker(expr, FixFunctionArgumentsWalker, NULL); } /* * Recursively explore an expression to ensure it can be used in the PostgreSQL * ExecInitExpr. * Currently only search for PARAM_MULTIEXPR or PARAM_SUBLINK. */ static bool CheckExprExecutorSafe(Node *expr) { if (expr == NULL) { return true; } /* * If it's a Param, we're done traversing the tree. * Just check if it contins a sublink or multiexpr. */ else if (IsA(expr, Param)) { Param *param = (Param *) expr; if (param->paramkind == PARAM_MULTIEXPR || param->paramkind == PARAM_SUBLINK) { return false; } } /* If it's a FuncExpr, search in arguments */ else if (IsA(expr, FuncExpr)) { FuncExpr *func = (FuncExpr *) expr; ListCell *lc; foreach(lc, func->args) { if (!CheckExprExecutorSafe((Node *) lfirst(lc))) { return false; } } } return true; } ================================================ FILE: src/backend/distributed/utils/citus_copyfuncs.c ================================================ /*------------------------------------------------------------------------- * * citus_copyfuncs.c * Citus specific node copy functions * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/datum.h" #include "distributed/citus_nodefuncs.h" #include "distributed/listutils.h" #include "distributed/multi_server_executor.h" /* * Macros to simplify copying of different kinds of fields. Use these * wherever possible to reduce the chance for silly typos. Note that these * hard-wire the convention that the local variables in a Copy routine are * named 'newnode' and 'from'. */ static inline Node * CitusSetTag(Node *node, int tag) { CitusNode *citus_node = (CitusNode *) node; citus_node->citus_tag = tag; return node; } #define DECLARE_FROM_AND_NEW_NODE(nodeTypeName) \ nodeTypeName *newnode = \ (nodeTypeName *) CitusSetTag((Node *) target_node, T_ ## nodeTypeName); \ nodeTypeName *from = (nodeTypeName *) source_node /* Copy a simple scalar field (int, float, bool, enum, etc) */ #define COPY_SCALAR_FIELD(fldname) \ (newnode->fldname = from->fldname) /* Copy a field that is a pointer to some kind of Node or Node tree */ #define COPY_NODE_FIELD(fldname) \ (newnode->fldname = copyObject(from->fldname)) /* Copy a field that is a pointer to a C string, or perhaps NULL */ #define COPY_STRING_FIELD(fldname) \ (newnode->fldname = from->fldname ? pstrdup(from->fldname) : (char *) NULL) /* Copy a node array. Target array is also allocated. */ #define COPY_NODE_ARRAY(fldname, type, count) \ do { \ int i = 0; \ newnode->fldname = (type **) palloc(count * sizeof(type *)); \ for (i = 0; i < count; ++i) \ { \ newnode->fldname[i] = copyObject(from->fldname[i]); \ } \ } \ while (0) /* Copy a scalar array. Target array is also allocated. */ #define COPY_SCALAR_ARRAY(fldname, type, count) \ do { \ int i = 0; \ newnode->fldname = (type *) palloc(count * sizeof(type)); \ for (i = 0; i < count; ++i) \ { \ newnode->fldname[i] = from->fldname[i]; \ } \ } \ while (0) #define COPY_STRING_LIST(fldname) \ do { \ char *curString = NULL; \ List *newList = NIL; \ foreach_declared_ptr(curString, from->fldname) { \ char *newString = curString ? pstrdup(curString) : (char *) NULL; \ newList = lappend(newList, newString); \ } \ newnode->fldname = newList; \ } \ while (0) static void CopyTaskQuery(Task *newnode, Task *from); static void copyJobInfo(Job *newnode, Job *from) { COPY_SCALAR_FIELD(jobId); COPY_NODE_FIELD(jobQuery); COPY_NODE_FIELD(taskList); COPY_NODE_FIELD(dependentJobList); COPY_SCALAR_FIELD(subqueryPushdown); COPY_SCALAR_FIELD(requiresCoordinatorEvaluation); COPY_SCALAR_FIELD(deferredPruning); COPY_NODE_FIELD(partitionKeyValue); COPY_NODE_FIELD(localPlannedStatements); COPY_SCALAR_FIELD(parametersInJobQueryResolved); } void CopyNodeJob(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(Job); copyJobInfo(newnode, from); } void CopyNodeDistributedPlan(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(DistributedPlan); COPY_SCALAR_FIELD(planId); COPY_SCALAR_FIELD(modLevel); COPY_SCALAR_FIELD(expectResults); COPY_NODE_FIELD(workerJob); COPY_NODE_FIELD(combineQuery); COPY_SCALAR_FIELD(queryId); COPY_NODE_FIELD(relationIdList); COPY_SCALAR_FIELD(targetRelationId); COPY_NODE_FIELD(modifyQueryViaCoordinatorOrRepartition); COPY_NODE_FIELD(selectPlanForModifyViaCoordinatorOrRepartition); COPY_SCALAR_FIELD(modifyWithSelectMethod); COPY_STRING_FIELD(intermediateResultIdPrefix); COPY_NODE_FIELD(subPlanList); COPY_NODE_FIELD(usedSubPlanNodeList); COPY_SCALAR_FIELD(fastPathRouterPlan); COPY_SCALAR_FIELD(numberOfTimesExecuted); COPY_NODE_FIELD(planningError); COPY_SCALAR_FIELD(sourceResultRepartitionColumnIndex); } void CopyNodeDistributedSubPlan(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(DistributedSubPlan); COPY_SCALAR_FIELD(subPlanId); COPY_NODE_FIELD(plan); COPY_SCALAR_FIELD(bytesSentPerWorker); COPY_SCALAR_FIELD(remoteWorkerCount); COPY_SCALAR_FIELD(durationMillisecs); COPY_SCALAR_FIELD(writeLocalFile); if (newnode->totalExplainOutput) { MemSet(newnode->totalExplainOutput, 0, sizeof(newnode->totalExplainOutput)); } /* copy each SubPlanExplainOutput element */ for (int i = 0; i < from->numTasksOutput; i++) { /* copy the explainOutput string pointer */ COPY_STRING_FIELD(totalExplainOutput[i].explainOutput); /* copy the executionDuration (double) */ COPY_SCALAR_FIELD(totalExplainOutput[i].executionDuration); /* copy the totalReceivedTupleData (uint64) */ COPY_SCALAR_FIELD(totalExplainOutput[i].totalReceivedTupleData); } COPY_SCALAR_FIELD(numTasksOutput); COPY_SCALAR_FIELD(ntuples); } void CopyNodeUsedDistributedSubPlan(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(UsedDistributedSubPlan); COPY_STRING_FIELD(subPlanId); COPY_SCALAR_FIELD(accessType); } void CopyNodeShardInterval(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(ShardInterval); COPY_SCALAR_FIELD(relationId); COPY_SCALAR_FIELD(storageType); COPY_SCALAR_FIELD(valueTypeId); COPY_SCALAR_FIELD(valueTypeLen); COPY_SCALAR_FIELD(valueByVal); COPY_SCALAR_FIELD(minValueExists); COPY_SCALAR_FIELD(maxValueExists); if (from->minValueExists) { newnode->minValue = datumCopy(from->minValue, from->valueByVal, from->valueTypeLen); } if (from->maxValueExists) { newnode->maxValue = datumCopy(from->maxValue, from->valueByVal, from->valueTypeLen); } COPY_SCALAR_FIELD(shardId); COPY_SCALAR_FIELD(shardIndex); } void CopyNodeMapMergeJob(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(MapMergeJob); copyJobInfo(&newnode->job, &from->job); COPY_SCALAR_FIELD(partitionType); COPY_NODE_FIELD(partitionColumn); COPY_SCALAR_FIELD(partitionCount); COPY_SCALAR_FIELD(sortedShardIntervalArrayLength); int arrayLength = from->sortedShardIntervalArrayLength; /* now build & read sortedShardIntervalArray */ COPY_NODE_ARRAY(sortedShardIntervalArray, ShardInterval, arrayLength); COPY_NODE_FIELD(mapTaskList); COPY_NODE_FIELD(mergeTaskList); } void CopyNodeShardPlacement(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(ShardPlacement); COPY_SCALAR_FIELD(placementId); COPY_SCALAR_FIELD(shardId); COPY_SCALAR_FIELD(shardLength); COPY_SCALAR_FIELD(groupId); COPY_STRING_FIELD(nodeName); COPY_SCALAR_FIELD(nodePort); COPY_SCALAR_FIELD(nodeId); COPY_SCALAR_FIELD(partitionMethod); COPY_SCALAR_FIELD(colocationGroupId); COPY_SCALAR_FIELD(representativeValue); } void CopyNodeGroupShardPlacement(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(GroupShardPlacement); COPY_SCALAR_FIELD(placementId); COPY_SCALAR_FIELD(shardId); COPY_SCALAR_FIELD(shardLength); COPY_SCALAR_FIELD(groupId); } void CopyNodeRelationShard(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(RelationShard); COPY_SCALAR_FIELD(relationId); COPY_SCALAR_FIELD(shardId); } void CopyNodeRelationRowLock(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(RelationRowLock); COPY_SCALAR_FIELD(relationId); COPY_SCALAR_FIELD(rowLockStrength); } static void CopyTaskQuery(Task *newnode, Task *from) { COPY_SCALAR_FIELD(taskQuery.queryType); switch (from->taskQuery.queryType) { case TASK_QUERY_TEXT: { COPY_STRING_FIELD(taskQuery.data.queryStringLazy); break; } case TASK_QUERY_OBJECT: { COPY_NODE_FIELD(taskQuery.data.jobQueryReferenceForLazyDeparsing); break; } case TASK_QUERY_TEXT_LIST: { COPY_STRING_LIST(taskQuery.data.queryStringList); break; } case TASK_QUERY_LOCAL_PLAN: { newnode->taskQuery.data.localCompiled = (LocalCompilation *) palloc0(sizeof(LocalCompilation)); COPY_NODE_FIELD(taskQuery.data.localCompiled->plan); COPY_NODE_FIELD(taskQuery.data.localCompiled->query); break; } default: { break; } } } void CopyNodeTask(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(Task); COPY_SCALAR_FIELD(taskType); COPY_SCALAR_FIELD(jobId); COPY_SCALAR_FIELD(taskId); CopyTaskQuery(newnode, from); COPY_SCALAR_FIELD(anchorDistributedTableId); COPY_SCALAR_FIELD(anchorShardId); COPY_NODE_FIELD(taskPlacementList); COPY_NODE_FIELD(dependentTaskList); COPY_SCALAR_FIELD(partitionId); COPY_SCALAR_FIELD(upstreamTaskId); COPY_NODE_FIELD(shardInterval); COPY_SCALAR_FIELD(assignmentConstrained); COPY_SCALAR_FIELD(replicationModel); COPY_SCALAR_FIELD(modifyWithSubquery); COPY_NODE_FIELD(relationShardList); COPY_NODE_FIELD(relationRowLockList); COPY_NODE_FIELD(rowValuesLists); COPY_SCALAR_FIELD(partiallyLocalOrRemote); COPY_SCALAR_FIELD(parametersInQueryStringResolved); COPY_SCALAR_FIELD(tupleDest); COPY_SCALAR_FIELD(queryCount); COPY_SCALAR_FIELD(totalReceivedTupleData); COPY_SCALAR_FIELD(fetchedExplainAnalyzePlacementIndex); COPY_STRING_FIELD(fetchedExplainAnalyzePlan); COPY_SCALAR_FIELD(fetchedExplainAnalyzeExecutionDuration); COPY_SCALAR_FIELD(isLocalTableModification); COPY_SCALAR_FIELD(cannotBeExecutedInTransaction); } void CopyNodeLocalPlannedStatement(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(LocalPlannedStatement); COPY_SCALAR_FIELD(shardId); COPY_SCALAR_FIELD(localGroupId); COPY_NODE_FIELD(localPlan); } void CopyNodeDeferredErrorMessage(COPYFUNC_ARGS) { DECLARE_FROM_AND_NEW_NODE(DeferredErrorMessage); COPY_SCALAR_FIELD(code); COPY_STRING_FIELD(message); COPY_STRING_FIELD(detail); COPY_STRING_FIELD(hint); COPY_STRING_FIELD(filename); COPY_SCALAR_FIELD(linenumber); COPY_STRING_FIELD(functionname); } ================================================ FILE: src/backend/distributed/utils/citus_depended_object.c ================================================ /* * citus_depended_object.c * * Implements exposed functions related to hiding citus depended objects. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "miscadmin.h" #include "catalog/namespace.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_attribute.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_proc.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_sequence.h" #include "catalog/pg_statistic.h" #include "catalog/pg_trigger.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "parser/parse_type.h" #include "storage/large_object.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" #include "distributed/citus_depended_object.h" #include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/shared_library_init.h" /* * GUC hides any objects, which depends on citus extension, from pg meta class queries, * it is intended to be used in vanilla tests to not break postgres test logs */ bool HideCitusDependentObjects = false; static Node * CreateCitusDependentObjectExpr(int pgMetaTableVarno, int pgMetaTableOid); static List * GetCitusDependedObjectArgs(int pgMetaTableVarno, int pgMetaTableOid); static bool AlterRoleSetStatementContainsAll(Node *node); static bool HasDropCommandViolatesOwnership(Node *node); static bool AnyObjectViolatesOwnership(DropStmt *dropStmt); /* * IsPgLocksTable returns true if RTE is pg_locks table. */ bool IsPgLocksTable(RangeTblEntry *rte) { Oid pgLocksId = get_relname_relid("pg_locks", get_namespace_oid("pg_catalog", false)); return rte->relid == pgLocksId; } /* * SetLocalHideCitusDependentObjectsDisabledWhenAlreadyEnabled disables the GUC HideCitusDependentObjects * if only it is enabled for local transaction. */ void SetLocalHideCitusDependentObjectsDisabledWhenAlreadyEnabled(void) { if (!HideCitusDependentObjects) { return; } set_config_option("citus.hide_citus_dependent_objects", "false", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } /* * SetLocalClientMinMessagesIfRunningPGTests sets client_min_message locally to the given value * if EnableUnsupportedFeatureMessages is set to false. */ void SetLocalClientMinMessagesIfRunningPGTests(int clientMinMessageLevel) { if (EnableUnsupportedFeatureMessages) { return; } const char *clientMinMessageLevelName = GetClientMinMessageLevelNameForValue( clientMinMessageLevel); set_config_option("client_min_messages", clientMinMessageLevelName, (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); } /* * HideCitusDependentObjectsOnQueriesOfPgMetaTables adds a NOT is_citus_depended_object(oid, oid) expr * to the quals of meta class RTEs that we are interested in. */ bool HideCitusDependentObjectsOnQueriesOfPgMetaTables(Node *node, void *context) { if (!CitusHasBeenLoaded() || !HideCitusDependentObjects || node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; MemoryContext queryContext = GetMemoryChunkContext(query); /* * We process the whole rtable rather than visiting individual RangeTblEntry's * in the walker, since we need to know the varno to generate the right * filter. */ int varno = 0; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, query->rtable) { varno++; if (rangeTableEntry->rtekind == RTE_RELATION) { /* make sure the expression is in the right memory context */ MemoryContext originalContext = MemoryContextSwitchTo(queryContext); Oid metaTableOid = InvalidOid; /* * add NOT is_citus_depended_object(oid, oid) to the quals * of the RTE if it is a pg meta table that we are interested in. */ switch (rangeTableEntry->relid) { /* pg_class */ case RelationRelationId: /* pg_proc */ case ProcedureRelationId: /* pg_am */ case AccessMethodRelationId: /* pg_type */ case TypeRelationId: /* pg_enum */ case EnumRelationId: /* pg_event_trigger */ case EventTriggerRelationId: /* pg_trigger */ case TriggerRelationId: /* pg_rewrite */ case RewriteRelationId: /* pg_attrdef */ case AttrDefaultRelationId: /* pg_constraint */ case ConstraintRelationId: /* pg_ts_config */ case TSConfigRelationId: /* pg_ts_template */ case TSTemplateRelationId: /* pg_ts_dict */ case TSDictionaryRelationId: /* pg_language */ case LanguageRelationId: /* pg_namespace */ case NamespaceRelationId: /* pg_sequence */ case SequenceRelationId: /* pg_statistic */ case StatisticRelationId: /* pg_attribute */ case AttributeRelationId: /* pg_index */ case IndexRelationId: /* pg_operator */ case OperatorRelationId: /* pg_opclass */ case OperatorClassRelationId: /* pg_opfamily */ case OperatorFamilyRelationId: /* pg_amop */ case AccessMethodOperatorRelationId: /* pg_amproc */ case AccessMethodProcedureRelationId: /* pg_aggregate */ case AggregateRelationId: { metaTableOid = rangeTableEntry->relid; break; } default: { metaTableOid = InvalidOid; break; } } if (OidIsValid(metaTableOid)) { bool mergeJoinCondition = false; #if PG_VERSION_NUM >= PG_VERSION_17 /* * In Postgres 17, the query tree has a specific field for the merge condition. * So we shouldn't modify the jointree, but rather the mergeJoinCondition here * Relevant PG17 commit: 0294df2f1 */ mergeJoinCondition = query->mergeJoinCondition; #endif /* * We found a valid pg meta class in query, * so we assert below conditions. */ Assert(mergeJoinCondition || (query->jointree != NULL && query->jointree->fromlist != NULL)); Node *citusDependentObjExpr = CreateCitusDependentObjectExpr(varno, metaTableOid); /* * We do not use security quals because a postgres vanilla test fails * with a change of order for its result. */ if (!mergeJoinCondition) { query->jointree->quals = make_and_qual( query->jointree->quals, citusDependentObjExpr); } else { #if PG_VERSION_NUM >= PG_VERSION_17 query->mergeJoinCondition = make_and_qual( query->mergeJoinCondition, citusDependentObjExpr); #endif } } MemoryContextSwitchTo(originalContext); } } return query_tree_walker((Query *) node, HideCitusDependentObjectsOnQueriesOfPgMetaTables, context, 0); } return expression_tree_walker(node, HideCitusDependentObjectsOnQueriesOfPgMetaTables, context); } /* * CreateCitusDependentObjectExpr constructs an expression of the form: * NOT pg_catalog.is_citus_depended_object(oid, oid) */ static Node * CreateCitusDependentObjectExpr(int pgMetaTableVarno, int pgMetaTableOid) { /* build the call to read_intermediate_result */ FuncExpr *funcExpr = makeNode(FuncExpr); funcExpr->funcid = CitusDependentObjectFuncId(); funcExpr->funcretset = false; funcExpr->funcvariadic = false; funcExpr->funcformat = 0; funcExpr->funccollid = 0; funcExpr->inputcollid = 0; funcExpr->location = -1; funcExpr->args = GetCitusDependedObjectArgs(pgMetaTableVarno, pgMetaTableOid); BoolExpr *notExpr = makeNode(BoolExpr); notExpr->boolop = NOT_EXPR; notExpr->args = list_make1(funcExpr); notExpr->location = -1; return (Node *) notExpr; } /* * GetCitusDependedObjectArgs returns func arguments for pg_catalog.is_citus_depended_object */ static List * GetCitusDependedObjectArgs(int pgMetaTableVarno, int pgMetaTableOid) { /* * set attribute number for the oid, which we are insterest in, inside pg meta tables. * We are accessing the 1. col(their own oid or their relation's oid) to get the related * object's oid for all of the pg meta tables except pg_enum and pg_index. For pg_enum, * class, we access its 2. col(its type's oid) to see if its type depends on citus, * so it does. For pg_index, we access its 2. col (its relation's oid) to see if its relation * depends on citus, so it does. */ AttrNumber oidAttNum = 1; if (pgMetaTableOid == EnumRelationId || pgMetaTableOid == IndexRelationId) { oidAttNum = 2; } /* create const for meta table oid */ Const *metaTableOidConst = makeConst(OIDOID, -1, InvalidOid, sizeof(Oid), ObjectIdGetDatum(pgMetaTableOid), false, true); /* * create a var for the oid that we are interested in, * col type should be regproc for pg_aggregate table; else oid */ Oid varType = (pgMetaTableOid == AggregateRelationId) ? REGPROCOID : OIDOID; Var *oidVar = makeVar(pgMetaTableVarno, oidAttNum, varType, -1, InvalidOid, 0); return list_make2((Node *) metaTableOidConst, (Node *) oidVar); } /* * DistOpsValidityState returns validation state for given dist ops. */ DistOpsValidationState DistOpsValidityState(Node *node, const DistributeObjectOps *ops) { if (ops && ops->operationType == DIST_OPS_CREATE) { /* * We should beware of qualifying the CREATE statement too early. */ if (nodeTag(node) == T_CreateDomainStmt) { /* * Create Domain statements should be qualified after local creation * because in case of an error in creation, we don't want to print * the error with the qualified name, as that would differ with * vanilla Postgres error output. */ return ShouldQualifyAfterLocalCreation; } /* * We should not validate CREATE statements because no address exists * here yet. */ return NoAddressResolutionRequired; } else if (AlterRoleSetStatementContainsAll(node)) { /* * We should not validate 'ALTER ROLE ALL [SET|UNSET] because for the role ALL * AlterRoleSetStmtObjectAddress returns an invalid address even though it should not. */ return NoAddressResolutionRequired; } else if (HasDropCommandViolatesOwnership(node)) { /* * found object with an invalid ownership, PG will complain if there is any object * with an invalid ownership. */ return HasObjectWithInvalidOwnership; } if (ops && ops->address) { bool missingOk = true; bool isPostprocess = false; List *objectAddresses = ops->address(node, missingOk, isPostprocess); ObjectAddress *objectAddress = NULL; foreach_declared_ptr(objectAddress, objectAddresses) { if (OidIsValid(objectAddress->objectId)) { /* found one valid object */ return HasAtLeastOneValidObject; } } /* no valid objects */ return HasNoneValidObject; } else { /* if the object doesn't have address defined, we donot validate */ return NoAddressResolutionRequired; } } /* * DistOpsInValidState returns true if given state is valid to execute * preprocess and qualify steps. */ bool DistOpsInValidState(DistOpsValidationState distOpsValidationState) { return distOpsValidationState == HasAtLeastOneValidObject || distOpsValidationState == NoAddressResolutionRequired; } /* * AlterRoleSetStatementContainsAll returns true if the statement is a * ALTER ROLE ALL (SET / RESET). */ static bool AlterRoleSetStatementContainsAll(Node *node) { if (node == NULL) { return false; } if (nodeTag(node) == T_AlterRoleSetStmt) { /* rolespec is null for the role 'ALL' */ AlterRoleSetStmt *alterRoleSetStmt = castNode(AlterRoleSetStmt, node); return alterRoleSetStmt->role == NULL; } return false; } /* * HasDropCommandViolatesOwnership returns true if any object in the given * statement violates object ownership. * * Currently there is only one test which fails due to object ownership. * The command that is failing is DROP. If in the future we hit other * commands like this, we should expand this function. */ static bool HasDropCommandViolatesOwnership(Node *node) { if (!IsA(node, DropStmt)) { return false; } DropStmt *dropStmt = castNode(DropStmt, node); if (AnyObjectViolatesOwnership(dropStmt)) { return true; } return false; } /* * AnyObjectViolatesOwnership return true if given object in stmt violates ownership. */ static bool AnyObjectViolatesOwnership(DropStmt *dropStmt) { bool hasOwnershipViolation = false; ObjectAddress objectAddress = { 0 }; volatile Relation relation = NULL; ObjectType objectType = dropStmt->removeType; bool missingOk = dropStmt->missing_ok; MemoryContext savedContext = CurrentMemoryContext; ResourceOwner savedOwner = CurrentResourceOwner; BeginInternalSubTransaction(NULL); MemoryContextSwitchTo(savedContext); PG_TRY(); { Node *object = NULL; foreach_declared_ptr(object, dropStmt->objects) { Relation rel = NULL; objectAddress = get_object_address(objectType, object, &rel, AccessShareLock, missingOk); /* * The object relation is qualified with volatile and its value is obtained from * get_object_address(). Unless we can qualify the corresponding parameter of * get_object_address() with volatile (this is a function defined in PostgreSQL), * we cannot get rid of this assignment. */ relation = rel; if (OidIsValid(objectAddress.objectId)) { /* * if object violates ownership, check_object_ownership will throw error. */ check_object_ownership(GetUserId(), objectType, objectAddress, object, relation); } if (relation != NULL) { relation_close(relation, NoLock); relation = NULL; } } ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(savedContext); CurrentResourceOwner = savedOwner; } PG_CATCH(); { MemoryContextSwitchTo(savedContext); ErrorData *edata = CopyErrorData(); FlushErrorState(); hasOwnershipViolation = true; if (relation != NULL) { relation_close(relation, NoLock); relation = NULL; } RollbackAndReleaseCurrentSubTransaction(); MemoryContextSwitchTo(savedContext); CurrentResourceOwner = savedOwner; /* Rethrow error with LOG_SERVER_ONLY to prevent log to be sent to client */ edata->elevel = LOG_SERVER_ONLY; ThrowErrorData(edata); } PG_END_TRY(); return hasOwnershipViolation; } ================================================ FILE: src/backend/distributed/utils/citus_nodefuncs.c ================================================ /*------------------------------------------------------------------------- * * citus_nodefuncs.c * Helper functions for dealing with nodes * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_type.h" #include "pg_version_constants.h" #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_router_planner.h" #include "distributed/multi_server_executor.h" static const char *CitusNodeTagNamesD[] = { "MultiNode", "MultiTreeRoot", "MultiProject", "MultiCollect", "MultiSelect", "MultiTable", "MultiJoin", "MultiPartition", "MultiCartesianProduct", "MultiExtendedOp", "Job", "MapMergeJob", "DistributedPlan", "DistributedSubPlan", "UsedDistributedSubPlan", "Task", "LocalPlannedStatement", "ShardInterval", "ShardPlacement", "RelationShard", "RelationRowLock", "DeferredErrorMessage", "GroupShardPlacement", "TableDDLCommand" }; const char **CitusNodeTagNames = CitusNodeTagNamesD; /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_extradata_container); /* * SetRangeTblExtraData adds additional data to a RTE, overwriting previous * values, if present. * * The data is stored as RTE_FUNCTION type RTE of a special * citus_extradata_container function, with the extra data serialized into the * function arguments. That works, because these RTEs aren't used by Postgres * to any significant degree, and Citus' variant of ruleutils.c knows how to * deal with these extended RTEs. Note that rte->eref needs to be set prior * to calling SetRangeTblExtraData to ensure the funccolcount can be set * correctly. * * NB: If used for postgres defined RTEKinds, fields specific to that RTEKind * will not be handled by out/readfuncs.c. For the current uses that's ok. */ void SetRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSchemaName, char *fragmentTableName, List *tableIdList, List *funcColumnNames, List *funcColumnTypes, List *funcColumnTypeMods, List *funcCollations) { Assert(rte->eref); /* store RTE kind as a plain int4 */ Const *rteKindData = makeNode(Const); rteKindData->consttype = INT4OID; rteKindData->constlen = 4; rteKindData->constvalue = Int32GetDatum(rteKind); rteKindData->constbyval = true; rteKindData->constisnull = false; rteKindData->location = -1; /* store the fragment schema as a cstring */ Const *fragmentSchemaData = makeNode(Const); fragmentSchemaData->consttype = CSTRINGOID; fragmentSchemaData->constlen = -2; fragmentSchemaData->constvalue = CStringGetDatum(fragmentSchemaName); fragmentSchemaData->constbyval = false; fragmentSchemaData->constisnull = fragmentSchemaName == NULL; fragmentSchemaData->location = -1; /* store the fragment name as a cstring */ Const *fragmentTableData = makeNode(Const); fragmentTableData->consttype = CSTRINGOID; fragmentTableData->constlen = -2; fragmentTableData->constvalue = CStringGetDatum(fragmentTableName); fragmentTableData->constbyval = false; fragmentTableData->constisnull = fragmentTableName == NULL; fragmentTableData->location = -1; /* store the table id list as an array of integers: FIXME */ Const *tableIdListData = makeNode(Const); tableIdListData->consttype = CSTRINGOID; tableIdListData->constbyval = false; tableIdListData->constlen = -2; tableIdListData->location = -1; /* serialize tableIdList to a string, seems simplest that way */ if (tableIdList != NIL) { char *serializedList = nodeToString(tableIdList); tableIdListData->constisnull = false; tableIdListData->constvalue = CStringGetDatum(serializedList); } else { tableIdListData->constisnull = true; } /* create function expression to store our faux arguments in */ FuncExpr *fauxFuncExpr = makeNode(FuncExpr); fauxFuncExpr->funcid = CitusExtraDataContainerFuncId(); fauxFuncExpr->funcresulttype = RECORDOID; fauxFuncExpr->funcretset = true; fauxFuncExpr->location = -1; fauxFuncExpr->args = list_make4(rteKindData, fragmentSchemaData, fragmentTableData, tableIdListData); RangeTblFunction *fauxFunction = makeNode(RangeTblFunction); fauxFunction->funcexpr = (Node *) fauxFuncExpr; /* set the column count to pass ruleutils checks, not used elsewhere */ if (rte->relid != 0) { Relation rel = RelationIdGetRelation(rte->relid); fauxFunction->funccolcount = RelationGetNumberOfAttributes(rel); RelationClose(rel); } else { fauxFunction->funccolcount = list_length(rte->eref->colnames); } fauxFunction->funccolnames = funcColumnNames; fauxFunction->funccoltypes = funcColumnTypes; fauxFunction->funccoltypmods = funcColumnTypeMods; fauxFunction->funccolcollations = funcCollations; rte->rtekind = RTE_FUNCTION; rte->functions = list_make1(fauxFunction); } /* * ExtractRangeTblExtraData extracts extra data stored for a range table entry * that previously has been stored with * Set/ModifyRangeTblExtraData. Parameters can be NULL if unintersting. It is * valid to use the function on a RTE without extra data. */ void ExtractRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind *rteKind, char **fragmentSchemaName, char **fragmentTableName, List **tableIdList) { /* set base rte kind first, so this can be used for 'non-extended' RTEs as well */ if (rteKind != NULL) { *rteKind = (CitusRTEKind) rte->rtekind; } /* reset values of optionally-present fields, will later be overwritten, if present */ if (fragmentSchemaName != NULL) { *fragmentSchemaName = NULL; } if (fragmentTableName != NULL) { *fragmentTableName = NULL; } if (tableIdList != NULL) { *tableIdList = NIL; } /* only function RTEs have our special extra data */ if (rte->rtekind != RTE_FUNCTION) { return; } /* we only ever generate one argument */ if (list_length(rte->functions) != 1) { return; } /* should pretty much always be a FuncExpr, but be liberal in what we expect... */ RangeTblFunction *fauxFunction = linitial(rte->functions); if (!IsA(fauxFunction->funcexpr, FuncExpr)) { return; } FuncExpr *fauxFuncExpr = (FuncExpr *) fauxFunction->funcexpr; /* * There will never be a range table entry with this function id, but for * the purpose of this file. */ if (fauxFuncExpr->funcid != CitusExtraDataContainerFuncId()) { return; } /* * Extra data for rtes is stored in the function arguments. The first * argument stores the rtekind, second fragmentSchemaName, third * fragmentTableName, fourth tableIdList. */ if (list_length(fauxFuncExpr->args) != 4) { ereport(ERROR, (errmsg("unexpected number of function arguments to " "citus_extradata_container"))); return; } /* extract rteKind */ Const *tmpConst = (Const *) linitial(fauxFuncExpr->args); Assert(IsA(tmpConst, Const)); Assert(tmpConst->consttype == INT4OID); if (rteKind != NULL) { *rteKind = DatumGetInt32(tmpConst->constvalue); } /* extract fragmentSchemaName */ tmpConst = (Const *) lsecond(fauxFuncExpr->args); Assert(IsA(tmpConst, Const)); Assert(tmpConst->consttype == CSTRINGOID); if (fragmentSchemaName != NULL && !tmpConst->constisnull) { *fragmentSchemaName = DatumGetCString(tmpConst->constvalue); } /* extract fragmentTableName */ tmpConst = (Const *) lthird(fauxFuncExpr->args); Assert(IsA(tmpConst, Const)); Assert(tmpConst->consttype == CSTRINGOID); if (fragmentTableName != NULL && !tmpConst->constisnull) { *fragmentTableName = DatumGetCString(tmpConst->constvalue); } /* extract tableIdList, stored as a serialized integer list */ tmpConst = (Const *) lfourth(fauxFuncExpr->args); Assert(IsA(tmpConst, Const)); Assert(tmpConst->consttype == CSTRINGOID); if (tableIdList != NULL && !tmpConst->constisnull) { Node *deserializedList = stringToNode(DatumGetCString(tmpConst->constvalue)); Assert(IsA(deserializedList, IntList)); *tableIdList = (List *) deserializedList; } } /* * ModifyRangeTblExtraData sets the RTE extra data fields for the passed * fields, leaving the current values in place for the ones not specified. * * rteKind has to be specified, fragmentSchemaName, fragmentTableName, * tableIdList can be set to NULL/NIL respectively to leave the current values * in-place. */ void ModifyRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSchemaName, char *fragmentTableName, List *tableIdList) { /* load existing values for the arguments not specifying a new value */ ExtractRangeTblExtraData(rte, NULL, fragmentSchemaName == NULL ? &fragmentSchemaName : NULL, fragmentTableName == NULL ? &fragmentTableName : NULL, tableIdList == NIL ? &tableIdList : NULL); SetRangeTblExtraData(rte, rteKind, fragmentSchemaName, fragmentTableName, tableIdList, NIL, NIL, NIL, NIL); } /* GetRangeTblKind returns rtekind of a RTE, be it an extended one or not. */ CitusRTEKind GetRangeTblKind(RangeTblEntry *rte) { CitusRTEKind rteKind = CITUS_RTE_RELATION /* invalid */; switch (rte->rtekind) { /* directly rtekind if it's not possibly an extended RTE */ case RTE_TABLEFUNC: case RTE_NAMEDTUPLESTORE: case RTE_RELATION: case RTE_SUBQUERY: case RTE_JOIN: case RTE_VALUES: case RTE_CTE: case RTE_RESULT: { rteKind = (CitusRTEKind) rte->rtekind; break; } #if PG_VERSION_NUM >= PG_VERSION_18 /* new in PG18: GROUP RTE, just map it straight through */ case RTE_GROUP: { rteKind = (CitusRTEKind) rte->rtekind; break; } #endif case RTE_FUNCTION: { /* * Extract extra data - correct even if a plain RTE_FUNCTION, not * an extended one, ExtractRangeTblExtraData handles that case * transparently. */ ExtractRangeTblExtraData(rte, &rteKind, NULL, NULL, NULL); break; } } return rteKind; } /* * citus_extradata_container is a placeholder function to store information * needed by Citus in plain postgres node trees. Executor and other hooks * should always intercept statements containing calls to this function. It's * not actually SQL callable by the user because of an INTERNAL argument. */ Datum citus_extradata_container(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("not supposed to get here, did you cheat?"))); PG_RETURN_NULL(); } static void CopyUnsupportedCitusNode(struct ExtensibleNode *newnode, const struct ExtensibleNode *oldnode) { ereport(ERROR, (errmsg("not implemented"))); } static bool EqualUnsupportedCitusNode(const struct ExtensibleNode *a, const struct ExtensibleNode *b) { ereport(ERROR, (errmsg("not implemented"))); } /* *INDENT-OFF* */ #define DEFINE_NODE_METHODS(type) \ { \ #type, \ sizeof(type), \ CopyNode##type, \ EqualUnsupportedCitusNode, \ Out##type, \ ReadUnsupportedCitusNode \ } #define DEFINE_NODE_METHODS_NO_READ(type) \ { \ #type, \ sizeof(type), \ CopyUnsupportedCitusNode, \ EqualUnsupportedCitusNode, \ Out##type, \ ReadUnsupportedCitusNode \ } /* *INDENT-ON* */ const ExtensibleNodeMethods nodeMethods[] = { DEFINE_NODE_METHODS(DistributedPlan), DEFINE_NODE_METHODS(DistributedSubPlan), DEFINE_NODE_METHODS(UsedDistributedSubPlan), DEFINE_NODE_METHODS(Job), DEFINE_NODE_METHODS(ShardInterval), DEFINE_NODE_METHODS(MapMergeJob), DEFINE_NODE_METHODS(ShardPlacement), DEFINE_NODE_METHODS(RelationShard), DEFINE_NODE_METHODS(RelationRowLock), DEFINE_NODE_METHODS(Task), DEFINE_NODE_METHODS(LocalPlannedStatement), DEFINE_NODE_METHODS(DeferredErrorMessage), DEFINE_NODE_METHODS(GroupShardPlacement), /* nodes with only output support */ DEFINE_NODE_METHODS_NO_READ(MultiNode), DEFINE_NODE_METHODS_NO_READ(MultiTreeRoot), DEFINE_NODE_METHODS_NO_READ(MultiProject), DEFINE_NODE_METHODS_NO_READ(MultiCollect), DEFINE_NODE_METHODS_NO_READ(MultiSelect), DEFINE_NODE_METHODS_NO_READ(MultiTable), DEFINE_NODE_METHODS_NO_READ(MultiJoin), DEFINE_NODE_METHODS_NO_READ(MultiPartition), DEFINE_NODE_METHODS_NO_READ(MultiCartesianProduct), DEFINE_NODE_METHODS_NO_READ(MultiExtendedOp), DEFINE_NODE_METHODS_NO_READ(TableDDLCommand) }; void RegisterNodes(void) { StaticAssertExpr(lengthof(nodeMethods) == lengthof(CitusNodeTagNamesD), "number of node methods and names do not match"); for (int off = 0; off < lengthof(nodeMethods); off++) { RegisterExtensibleNodeMethods(&nodeMethods[off]); } } ================================================ FILE: src/backend/distributed/utils/citus_outfuncs.c ================================================ /*------------------------------------------------------------------------- * * citus_outfuncs.c * Output functions for Citus tree nodes. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) Citus Data, Inc. * * NOTES * This is a wrapper around postgres' nodeToString() that additionally * supports Citus node types. * * Keep as closely aligned with the upstream version as possible. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "pg_version_constants.h" #include #include "distributed/citus_nodefuncs.h" #include "distributed/citus_nodes.h" #include "distributed/coordinator_protocol.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" #include "distributed/distributed_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/metadata_utility.h" #include "lib/stringinfo.h" #include "nodes/plannodes.h" #include "nodes/pathnodes.h" #include "utils/datum.h" /* * Macros to simplify output of different kinds of fields. Use these * wherever possible to reduce the chance for silly typos. Note that these * hard-wire conventions about the names of the local variables in an Out * routine. */ /* Store const reference to raw input node in local named 'node' */ #define WRITE_LOCALS(nodeTypeName) \ const nodeTypeName *node = (const nodeTypeName *) raw_node /* Write the label for the node type */ #define WRITE_NODE_TYPE(nodelabel) \ (void) 0 /* Write an integer field (anything written as ":fldname %d") */ #define WRITE_INT_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname) /* Write an 64-bit integer field (anything written as ":fldname %d") */ #define WRITE_INT64_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " " INT64_FORMAT, node->fldname) /* Write an unsigned integer field (anything written as ":fldname %u") */ #define WRITE_UINT_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname) /* XXX: Citus: Write an unsigned 64-bit integer field */ #define WRITE_UINT64_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " " UINT64_FORMAT, node->fldname) /* Write an OID field (don't hard-wire assumption that OID is same as uint) */ #define WRITE_OID_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %u", node->fldname) /* Write a char field (ie, one ascii character) */ #define WRITE_CHAR_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %c", node->fldname) /* Write an enumerated-type field as an integer code */ #define WRITE_ENUM_FIELD(fldname, enumtype) \ appendStringInfo(str, " :" CppAsString(fldname) " %d", \ (int) node->fldname) /* Write a float field --- caller must give format to define precision */ #define WRITE_FLOAT_FIELD(fldname,format) \ appendStringInfo(str, " :" CppAsString(fldname) " " format, node->fldname) /* Write a boolean field */ #define WRITE_BOOL_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %s", \ booltostr(node->fldname)) /* Write a character-string (possibly NULL) field */ #define WRITE_STRING_FIELD(fldname) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ outToken(str, node->fldname)) /* Write a parse location field (actually same as INT case) */ #define WRITE_LOCATION_FIELD(fldname) \ appendStringInfo(str, " :" CppAsString(fldname) " %d", node->fldname) /* Write a Node field */ #define WRITE_NODE_FIELD(fldname) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ outNode(str, node->fldname)) /* Write a bitmapset field */ #define WRITE_BITMAPSET_FIELD(fldname) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ _outBitmapset(str, node->fldname)) #define WRITE_CUSTOM_FIELD(fldname, fldvalue) \ (appendStringInfo(str, " :" CppAsString(fldname) " "), \ appendStringInfoString(str, (fldvalue))) /* Write an integer array (anything written as ":fldname (%d, %d") */ #define WRITE_INT_ARRAY(fldname, count) \ appendStringInfo(str, " :" CppAsString(fldname) " ("); \ { \ int i;\ for (i = 0; i < count; i++) \ { \ if (i > 0) \ { \ appendStringInfo(str, ", "); \ } \ appendStringInfo(str, "%d", node->fldname[i]); \ }\ }\ appendStringInfo(str, ")") /* Write an enum array (anything written as ":fldname (%d, %d") */ #define WRITE_ENUM_ARRAY(fldname, count) WRITE_INT_ARRAY(fldname, count) #define booltostr(x) ((x) ? "true" : "false") static void WriteTaskQuery(OUTFUNC_ARGS); /***************************************************************************** * Output routines for Citus node types *****************************************************************************/ static void OutMultiUnaryNodeFields(StringInfo str, const MultiUnaryNode *node) { WRITE_NODE_FIELD(childNode); } static void OutMultiBinaryNodeFields(StringInfo str, const MultiBinaryNode *node) { WRITE_NODE_FIELD(leftChildNode); WRITE_NODE_FIELD(rightChildNode); } void OutMultiNode(OUTFUNC_ARGS) { WRITE_NODE_TYPE("MULTINODE"); } void OutMultiTreeRoot(OUTFUNC_ARGS) { WRITE_LOCALS(MultiTreeRoot); WRITE_NODE_TYPE("MULTITREEROOT"); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutDistributedPlan(OUTFUNC_ARGS) { WRITE_LOCALS(DistributedPlan); WRITE_NODE_TYPE("DISTRIBUTEDPLAN"); WRITE_UINT64_FIELD(planId); WRITE_ENUM_FIELD(modLevel, RowModifyLevel); WRITE_BOOL_FIELD(expectResults); WRITE_NODE_FIELD(workerJob); WRITE_NODE_FIELD(combineQuery); WRITE_UINT64_FIELD(queryId); WRITE_NODE_FIELD(relationIdList); WRITE_OID_FIELD(targetRelationId); WRITE_NODE_FIELD(modifyQueryViaCoordinatorOrRepartition); WRITE_NODE_FIELD(selectPlanForModifyViaCoordinatorOrRepartition); WRITE_ENUM_FIELD(modifyWithSelectMethod, ModifyWithSelectMethod); WRITE_STRING_FIELD(intermediateResultIdPrefix); WRITE_NODE_FIELD(subPlanList); WRITE_NODE_FIELD(usedSubPlanNodeList); WRITE_BOOL_FIELD(fastPathRouterPlan); WRITE_UINT_FIELD(numberOfTimesExecuted); WRITE_NODE_FIELD(planningError); WRITE_INT_FIELD(sourceResultRepartitionColumnIndex); } void OutDistributedSubPlan(OUTFUNC_ARGS) { WRITE_LOCALS(DistributedSubPlan); WRITE_NODE_TYPE("DISTRIBUTEDSUBPLAN"); WRITE_UINT_FIELD(subPlanId); WRITE_NODE_FIELD(plan); WRITE_UINT64_FIELD(bytesSentPerWorker); WRITE_INT_FIELD(remoteWorkerCount); WRITE_FLOAT_FIELD(durationMillisecs, "%.2f"); WRITE_BOOL_FIELD(writeLocalFile); appendStringInfoString(str, " totalExplainOutput ["); for (int i = 0; i < node->numTasksOutput; i++) { const SubPlanExplainOutputData *e = &node->totalExplainOutput[i]; /* skip empty slots */ if (e->explainOutput == NULL && e->executionDuration == 0 && e->totalReceivedTupleData == 0) { continue; } if (i > 0) { appendStringInfoChar(str, ' '); } appendStringInfoChar(str, '('); /* string pointer – prints quoted or NULL */ WRITE_STRING_FIELD(totalExplainOutput[i].explainOutput); /* double field */ WRITE_FLOAT_FIELD(totalExplainOutput[i].executionDuration, "%.2f"); /* 64-bit unsigned – use the uint64 macro */ WRITE_UINT64_FIELD(totalExplainOutput[i].totalReceivedTupleData); appendStringInfoChar(str, ')'); } appendStringInfoChar(str, ']'); WRITE_INT_FIELD(numTasksOutput); WRITE_FLOAT_FIELD(ntuples, "%.2f"); } void OutUsedDistributedSubPlan(OUTFUNC_ARGS) { WRITE_LOCALS(UsedDistributedSubPlan); WRITE_NODE_TYPE("USEDDISTRIBUTEDSUBPLAN"); WRITE_STRING_FIELD(subPlanId); WRITE_ENUM_FIELD(accessType, SubPlanAccessType); } void OutMultiProject(OUTFUNC_ARGS) { WRITE_LOCALS(MultiProject); WRITE_NODE_TYPE("MULTIPROJECT"); WRITE_NODE_FIELD(columnList); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutMultiCollect(OUTFUNC_ARGS) { WRITE_LOCALS(MultiCollect); WRITE_NODE_TYPE("MULTICOLLECT"); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutMultiSelect(OUTFUNC_ARGS) { WRITE_LOCALS(MultiSelect); WRITE_NODE_TYPE("MULTISELECT"); WRITE_NODE_FIELD(selectClauseList); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutMultiTable(OUTFUNC_ARGS) { WRITE_LOCALS(MultiTable); WRITE_NODE_TYPE("MULTITABLE"); WRITE_OID_FIELD(relationId); WRITE_INT_FIELD(rangeTableId); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutMultiJoin(OUTFUNC_ARGS) { WRITE_LOCALS(MultiJoin); WRITE_NODE_TYPE("MULTIJOIN"); WRITE_NODE_FIELD(joinClauseList); WRITE_ENUM_FIELD(joinRuleType, JoinRuleType); WRITE_ENUM_FIELD(joinType, JoinType); OutMultiBinaryNodeFields(str, (const MultiBinaryNode *) node); } void OutMultiPartition(OUTFUNC_ARGS) { WRITE_LOCALS(MultiPartition); WRITE_NODE_TYPE("MULTIPARTITION"); WRITE_NODE_FIELD(partitionColumn); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } void OutMultiCartesianProduct(OUTFUNC_ARGS) { WRITE_LOCALS(MultiCartesianProduct); WRITE_NODE_TYPE("MULTICARTESIANPRODUCT"); OutMultiBinaryNodeFields(str, (const MultiBinaryNode *) node); } void OutMultiExtendedOp(OUTFUNC_ARGS) { WRITE_LOCALS(MultiExtendedOp); WRITE_NODE_TYPE("MULTIEXTENDEDOP"); WRITE_NODE_FIELD(targetList); WRITE_NODE_FIELD(groupClauseList); WRITE_NODE_FIELD(sortClauseList); WRITE_NODE_FIELD(limitCount); WRITE_NODE_FIELD(limitOffset); WRITE_ENUM_FIELD(limitOption, LimitOption); WRITE_NODE_FIELD(havingQual); WRITE_BOOL_FIELD(hasDistinctOn); WRITE_NODE_FIELD(distinctClause); WRITE_BOOL_FIELD(hasWindowFuncs); WRITE_BOOL_FIELD(onlyPushableWindowFunctions); WRITE_NODE_FIELD(windowClause); OutMultiUnaryNodeFields(str, (const MultiUnaryNode *) node); } static void OutJobFields(StringInfo str, const Job *node) { WRITE_UINT64_FIELD(jobId); WRITE_NODE_FIELD(jobQuery); WRITE_NODE_FIELD(taskList); WRITE_NODE_FIELD(dependentJobList); WRITE_BOOL_FIELD(subqueryPushdown); WRITE_BOOL_FIELD(requiresCoordinatorEvaluation); WRITE_BOOL_FIELD(deferredPruning); WRITE_NODE_FIELD(partitionKeyValue); WRITE_NODE_FIELD(localPlannedStatements); WRITE_BOOL_FIELD(parametersInJobQueryResolved); } void OutJob(OUTFUNC_ARGS) { WRITE_LOCALS(Job); WRITE_NODE_TYPE("JOB"); OutJobFields(str, node); } void OutShardInterval(OUTFUNC_ARGS) { WRITE_LOCALS(ShardInterval); WRITE_NODE_TYPE("SHARDINTERVAL"); WRITE_OID_FIELD(relationId); WRITE_CHAR_FIELD(storageType); WRITE_OID_FIELD(valueTypeId); WRITE_INT_FIELD(valueTypeLen); WRITE_BOOL_FIELD(valueByVal); WRITE_BOOL_FIELD(minValueExists); WRITE_BOOL_FIELD(maxValueExists); appendStringInfoString(str, " :minValue "); if (!node->minValueExists) appendStringInfoString(str, "<>"); else outDatum(str, node->minValue, node->valueTypeLen, node->valueByVal); appendStringInfoString(str, " :maxValue "); if (!node->maxValueExists) appendStringInfoString(str, "<>"); else outDatum(str, node->maxValue, node->valueTypeLen, node->valueByVal); WRITE_UINT64_FIELD(shardId); WRITE_INT_FIELD(shardIndex); } void OutMapMergeJob(OUTFUNC_ARGS) { WRITE_LOCALS(MapMergeJob); int arrayLength = node->sortedShardIntervalArrayLength; int i; WRITE_NODE_TYPE("MAPMERGEJOB"); OutJobFields(str, (Job *) node); WRITE_ENUM_FIELD(partitionType, PartitionType); WRITE_NODE_FIELD(partitionColumn); WRITE_UINT_FIELD(partitionCount); WRITE_INT_FIELD(sortedShardIntervalArrayLength); for (i = 0; i < arrayLength; ++i) { outNode(str, node->sortedShardIntervalArray[i]); } WRITE_NODE_FIELD(mapTaskList); WRITE_NODE_FIELD(mergeTaskList); } void OutShardPlacement(OUTFUNC_ARGS) { WRITE_LOCALS(ShardPlacement); WRITE_NODE_TYPE("SHARDPLACEMENT"); WRITE_UINT64_FIELD(placementId); WRITE_UINT64_FIELD(shardId); WRITE_UINT64_FIELD(shardLength); WRITE_INT_FIELD(groupId); WRITE_STRING_FIELD(nodeName); WRITE_UINT_FIELD(nodePort); WRITE_UINT_FIELD(nodeId); /* so we can deal with 0 */ WRITE_INT_FIELD(partitionMethod); WRITE_UINT_FIELD(colocationGroupId); WRITE_UINT_FIELD(representativeValue); } void OutGroupShardPlacement(OUTFUNC_ARGS) { WRITE_LOCALS(GroupShardPlacement); WRITE_NODE_TYPE("GROUPSHARDPLACEMENT"); WRITE_UINT64_FIELD(placementId); WRITE_UINT64_FIELD(shardId); WRITE_UINT64_FIELD(shardLength); WRITE_INT_FIELD(groupId); } void OutRelationShard(OUTFUNC_ARGS) { WRITE_LOCALS(RelationShard); WRITE_NODE_TYPE("RELATIONSHARD"); WRITE_OID_FIELD(relationId); WRITE_UINT64_FIELD(shardId); } void OutRelationRowLock(OUTFUNC_ARGS) { WRITE_LOCALS(RelationRowLock); WRITE_NODE_TYPE("RELATIONROWLOCK"); WRITE_OID_FIELD(relationId); WRITE_ENUM_FIELD(rowLockStrength, LockClauseStrength); } static void WriteTaskQuery(OUTFUNC_ARGS) { WRITE_LOCALS(Task); WRITE_ENUM_FIELD(taskQuery.queryType, TaskQueryType); switch (node->taskQuery.queryType) { case TASK_QUERY_TEXT: { WRITE_STRING_FIELD(taskQuery.data.queryStringLazy); break; } case TASK_QUERY_OBJECT: { WRITE_NODE_FIELD(taskQuery.data.jobQueryReferenceForLazyDeparsing); break; } case TASK_QUERY_TEXT_LIST: { WRITE_NODE_FIELD(taskQuery.data.queryStringList); break; } default: { break; } } } void OutTask(OUTFUNC_ARGS) { WRITE_LOCALS(Task); WRITE_NODE_TYPE("TASK"); WRITE_ENUM_FIELD(taskType, TaskType); WRITE_UINT64_FIELD(jobId); WRITE_UINT_FIELD(taskId); WriteTaskQuery(str, raw_node); WRITE_OID_FIELD(anchorDistributedTableId); WRITE_UINT64_FIELD(anchorShardId); WRITE_NODE_FIELD(taskPlacementList); WRITE_NODE_FIELD(dependentTaskList); WRITE_UINT_FIELD(partitionId); WRITE_UINT_FIELD(upstreamTaskId); WRITE_NODE_FIELD(shardInterval); WRITE_BOOL_FIELD(assignmentConstrained); WRITE_CHAR_FIELD(replicationModel); WRITE_BOOL_FIELD(modifyWithSubquery); WRITE_NODE_FIELD(relationShardList); WRITE_NODE_FIELD(relationRowLockList); WRITE_NODE_FIELD(rowValuesLists); WRITE_BOOL_FIELD(partiallyLocalOrRemote); WRITE_BOOL_FIELD(parametersInQueryStringResolved); WRITE_INT_FIELD(queryCount); WRITE_UINT64_FIELD(totalReceivedTupleData); WRITE_INT_FIELD(fetchedExplainAnalyzePlacementIndex); WRITE_STRING_FIELD(fetchedExplainAnalyzePlan); WRITE_FLOAT_FIELD(fetchedExplainAnalyzeExecutionDuration, "%.2f"); WRITE_BOOL_FIELD(isLocalTableModification); WRITE_BOOL_FIELD(cannotBeExecutedInTransaction); } void OutLocalPlannedStatement(OUTFUNC_ARGS) { WRITE_LOCALS(LocalPlannedStatement); WRITE_NODE_TYPE("LocalPlannedStatement"); WRITE_UINT64_FIELD(shardId); WRITE_UINT_FIELD(localGroupId); WRITE_NODE_FIELD(localPlan); } void OutDeferredErrorMessage(OUTFUNC_ARGS) { WRITE_LOCALS(DeferredErrorMessage); WRITE_NODE_TYPE("DEFERREDERRORMESSAGE"); WRITE_INT_FIELD(code); WRITE_STRING_FIELD(message); WRITE_STRING_FIELD(detail); WRITE_STRING_FIELD(hint); WRITE_STRING_FIELD(filename); WRITE_INT_FIELD(linenumber); WRITE_STRING_FIELD(functionname); } void OutTableDDLCommand(OUTFUNC_ARGS) { WRITE_LOCALS(TableDDLCommand); WRITE_NODE_TYPE("TableDDLCommand"); switch (node->type) { case TABLE_DDL_COMMAND_STRING: { WRITE_STRING_FIELD(commandStr); break; } case TABLE_DDL_COMMAND_FUNCTION: { char *example = node->function.function(node->function.context); WRITE_CUSTOM_FIELD(function, example); break; } } } ================================================ FILE: src/backend/distributed/utils/citus_readfuncs.c ================================================ /*------------------------------------------------------------------------- * * citus_readfuncs.c * Citus specific node functions * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/citus_nodefuncs.h" void ReadUnsupportedCitusNode(READFUNC_ARGS) { ereport(ERROR, (errmsg("not implemented"))); } ================================================ FILE: src/backend/distributed/utils/citus_safe_lib.c ================================================ /*------------------------------------------------------------------------- * * safe_lib.c * * This file contains all SafeXXXX helper functions that we implement to * replace missing xxxx_s functions implemented by safestringlib. It also * contains a constraint handler for use in both our SafeXXX and safestringlib * its xxxx_s functions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "safe_lib.h" #include "lib/stringinfo.h" #include "pg_version_constants.h" #include "distributed/citus_safe_lib.h" /* * ereport_constraint_handler is a constraint handler that calls ereport. A * constraint handler is called whenever an error occurs in any of the * safestringlib xxxx_s functions or our SafeXXXX functions. * * More info on constraint handlers can be found here: * https://en.cppreference.com/w/c/error/set_constraint_handler_s */ void ereport_constraint_handler(const char *message, void *pointer, errno_t error) { if (message && error) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "Memory constraint error: %s (errno %d)", message, error))); } else if (message) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "Memory constraint error: %s", message))); } else if (error) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "Unknown function failed with memory constraint error (errno %d)", error))); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "Unknown function failed with memory constraint error"))); } } /* * SafeStringToInt64 converts a string containing a number to a int64. When it * fails it calls ereport. * * The different error cases are inspired by * https://stackoverflow.com/a/26083517/2570866 */ int64 SafeStringToInt64(const char *str) { char *endptr; errno = 0; long long number = strtoll(str, &endptr, 10); if (str == endptr) { ereport(ERROR, (errmsg("Error parsing %s as int64, no digits found\n", str))); } else if ((errno == ERANGE && number == LLONG_MIN) || number < INT64_MIN) { ereport(ERROR, (errmsg("Error parsing %s as int64, underflow occurred\n", str))); } else if ((errno == ERANGE && number == LLONG_MAX) || number > INT64_MAX) { ereport(ERROR, (errmsg("Error parsing %s as int64, overflow occurred\n", str))); } else if (errno == EINVAL) { ereport(ERROR, (errmsg( "Error parsing %s as int64, base contains unsupported value\n", str))); } else if (errno != 0 && number == 0) { int err = errno; ereport(ERROR, (errmsg("Error parsing %s as int64, errno %d\n", str, err))); } else if (errno == 0 && str && *endptr != '\0') { ereport(ERROR, (errmsg( "Error parsing %s as int64, aditional characters remain after int64\n", str))); } return number; } /* * SafeStringToInt32 converts a string containing a number to a int32. When it * fails it calls ereport. * * The different error cases are inspired by * https://stackoverflow.com/a/26083517/2570866 */ int32 SafeStringToInt32(const char *str) { char *endptr; errno = 0; long number = strtol(str, &endptr, 10); if (str == endptr) { ereport(ERROR, (errmsg("Error parsing %s as int32, no digits found\n", str))); } else if ((errno == ERANGE && number == LONG_MIN) || number < INT32_MIN) { ereport(ERROR, (errmsg("Error parsing %s as int32, underflow occurred\n", str))); } else if ((errno == ERANGE && number == LONG_MAX) || number > INT32_MAX) { ereport(ERROR, (errmsg("Error parsing %s as int32, overflow occurred\n", str))); } else if (errno == EINVAL) { ereport(ERROR, (errmsg( "Error parsing %s as int32, base contains unsupported value\n", str))); } else if (errno != 0 && number == 0) { int err = errno; ereport(ERROR, (errmsg("Error parsing %s as int32, errno %d\n", str, err))); } else if (errno == 0 && str && *endptr != '\0') { ereport(ERROR, (errmsg( "Error parsing %s as int32, aditional characters remain after int32\n", str))); } return number; } /* * SafeStringToUint64 converts a string containing a number to a uint64. When it * fails it calls ereport. * * The different error cases are inspired by * https://stackoverflow.com/a/26083517/2570866 */ uint64 SafeStringToUint64(const char *str) { char *endptr; errno = 0; unsigned long long number = strtoull(str, &endptr, 10); if (str == endptr) { ereport(ERROR, (errmsg("Error parsing %s as uint64, no digits found\n", str))); } else if ((errno == ERANGE && number == ULLONG_MAX) || number > UINT64_MAX) { ereport(ERROR, (errmsg("Error parsing %s as uint64, overflow occurred\n", str))); } else if (errno == EINVAL) { ereport(ERROR, (errmsg( "Error parsing %s as uint64, base contains unsupported value\n", str))); } else if (errno != 0 && number == 0) { int err = errno; ereport(ERROR, (errmsg("Error parsing %s as uint64, errno %d\n", str, err))); } else if (errno == 0 && str && *endptr != '\0') { ereport(ERROR, (errmsg( "Error parsing %s as uint64, aditional characters remain after uint64\n", str))); } return number; } /* * SafeQsort is the non reentrant version of qsort (qsort vs qsort_r), but it * does the input checks required for qsort_s: * 1. count or size is greater than RSIZE_MAX * 2. ptr or comp is a null pointer (unless count is zero) * source: https://en.cppreference.com/w/c/algorithm/qsort * * When it hits these errors it calls the ereport_constraint_handler. * * NOTE: this functions calls pg_qsort instead of stdlib qsort. */ void SafeQsort(void *ptr, rsize_t count, rsize_t size, int (*comp)(const void *, const void *)) { if (count > RSIZE_MAX_MEM) { ereport_constraint_handler("SafeQsort: count exceeds max", NULL, ESLEMAX); } if (size > RSIZE_MAX_MEM) { ereport_constraint_handler("SafeQsort: size exceeds max", NULL, ESLEMAX); } if (size != 0) { if (ptr == NULL) { ereport_constraint_handler("SafeQsort: ptr is NULL", NULL, ESNULLP); } if (comp == NULL) { ereport_constraint_handler("SafeQsort: comp is NULL", NULL, ESNULLP); } } pg_qsort(ptr, count, size, comp); } /* * SafeBsearch is a non reentrant version of bsearch, but it does the * input checks required for bsearch_s: * 1. count or size is greater than RSIZE_MAX * 2. key, ptr or comp is a null pointer (unless count is zero) * source: https://en.cppreference.com/w/c/algorithm/bsearch * * When it hits these errors it calls the ereport_constraint_handler. * * NOTE: this functions calls pg_qsort instead of stdlib qsort. */ void * SafeBsearch(const void *key, const void *ptr, rsize_t count, rsize_t size, int (*comp)(const void *, const void *)) { if (count > RSIZE_MAX_MEM) { ereport_constraint_handler("SafeBsearch: count exceeds max", NULL, ESLEMAX); } if (size > RSIZE_MAX_MEM) { ereport_constraint_handler("SafeBsearch: size exceeds max", NULL, ESLEMAX); } if (size != 0) { if (key == NULL) { ereport_constraint_handler("SafeBsearch: key is NULL", NULL, ESNULLP); } if (ptr == NULL) { ereport_constraint_handler("SafeBsearch: ptr is NULL", NULL, ESNULLP); } if (comp == NULL) { ereport_constraint_handler("SafeBsearch: comp is NULL", NULL, ESNULLP); } } /* * Explanation of IGNORE-BANNED: * bsearch is safe to use here since we check the same thing bsearch_s * does. We cannot use bsearch_s as a replacement, since it's not available * in safestringlib. */ return bsearch(key, ptr, count, size, comp); /* IGNORE-BANNED */ } /* * SafeSnprintf is a safer replacement for snprintf, which is needed since * safestringlib doesn't implement snprintf_s. * * The required failure modes of snprint_s are as follows (in parentheses if * this implements it and how): * 1. the conversion specifier %n is present in format (yes, %n is not * supported by pg_vsnprintf) * 2. any of the arguments corresponding to %s is a null pointer (half, checked * in postgres when asserts are enabled) * 3. format or buffer is a null pointer (yes, checked by this function) * 4. bufsz is zero or greater than RSIZE_MAX (yes, checked by this function) * 5. encoding errors occur in any of string and character conversion * specifiers (no clue what postgres does in this case) * source: https://en.cppreference.com/w/c/io/fprintf */ int SafeSnprintf(char *restrict buffer, rsize_t bufsz, const char *restrict format, ...) { /* failure mode 3 */ if (buffer == NULL) { ereport_constraint_handler("SafeSnprintf: buffer is NULL", NULL, ESNULLP); } if (format == NULL) { ereport_constraint_handler("SafeSnprintf: format is NULL", NULL, ESNULLP); } /* failure mode 4 */ if (bufsz == 0) { ereport_constraint_handler("SafeSnprintf: bufsz is 0", NULL, ESZEROL); } if (bufsz > RSIZE_MAX_STR) { ereport_constraint_handler("SafeSnprintf: bufsz exceeds max", NULL, ESLEMAX); } va_list args; va_start(args, format); int result = pg_vsnprintf(buffer, bufsz, format, args); va_end(args); return result; } ================================================ FILE: src/backend/distributed/utils/citus_version.c ================================================ /*------------------------------------------------------------------------- * * citus_version.c * * This file contains functions for displaying the Citus version string * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/builtins.h" #include "citus_version.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(citus_version); /* GIT_VERSION is passed in as a compiler flag during builds that have git installed */ #ifdef GIT_VERSION #define GIT_REF " gitref: " GIT_VERSION #else #define GIT_REF #endif Datum citus_version(PG_FUNCTION_ARGS) { PG_RETURN_TEXT_P(cstring_to_text(CITUS_VERSION_STR GIT_REF)); } ================================================ FILE: src/backend/distributed/utils/clonenode_utils.c ================================================ #include #include #include #include #include "postgres.h" #include "utils/fmgrprotos.h" #include "utils/pg_lsn.h" #include "distributed/argutils.h" #include "distributed/clonenode_utils.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/remote_commands.h" #include "distributed/shard_rebalancer.h" /* * GetReplicationLag calculates the replication lag between the primary and replica nodes. * It returns the lag in bytes. */ int64 GetReplicationLag(WorkerNode *primaryWorkerNode, WorkerNode *replicaWorkerNode) { /* Input validation */ if (primaryWorkerNode == NULL || replicaWorkerNode == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("primary or replica worker node is NULL"))); } #if PG_VERSION_NUM >= 100000 const char *primary_lsn_query = "SELECT pg_current_wal_lsn()"; const char *replica_lsn_query = "SELECT pg_last_wal_replay_lsn()"; #else const char *primary_lsn_query = "SELECT pg_current_xlog_location()"; const char *replica_lsn_query = "SELECT pg_last_xlog_replay_location()"; #endif int connectionFlag = 0; MultiConnection *primaryConnection = GetNodeConnection(connectionFlag, primaryWorkerNode->workerName, primaryWorkerNode->workerPort); if (PQstatus(primaryConnection->pgConn) != CONNECTION_OK) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "cannot connect to primary node %s:%d to fetch replication status", primaryWorkerNode->workerName, primaryWorkerNode-> workerPort))); } MultiConnection *replicaConnection = GetNodeConnection(connectionFlag, replicaWorkerNode->workerName, replicaWorkerNode->workerPort); if (PQstatus(replicaConnection->pgConn) != CONNECTION_OK) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "cannot connect to clone node %s:%d to fetch replication status", replicaWorkerNode->workerName, replicaWorkerNode-> workerPort))); } int primaryResultCode = SendRemoteCommand(primaryConnection, primary_lsn_query); if (primaryResultCode == 0) { ReportConnectionError(primaryConnection, ERROR); } PGresult *primaryResult = GetRemoteCommandResult(primaryConnection, true); if (!IsResponseOK(primaryResult)) { ReportResultError(primaryConnection, primaryResult, ERROR); } int replicaResultCode = SendRemoteCommand(replicaConnection, replica_lsn_query); if (replicaResultCode == 0) { ReportConnectionError(replicaConnection, ERROR); } PGresult *replicaResult = GetRemoteCommandResult(replicaConnection, true); if (!IsResponseOK(replicaResult)) { ReportResultError(replicaConnection, replicaResult, ERROR); } List *primaryLsnList = ReadFirstColumnAsText(primaryResult); if (list_length(primaryLsnList) != 1) { PQclear(primaryResult); ClearResults(primaryConnection, true); CloseConnection(primaryConnection); PQclear(replicaResult); ClearResults(replicaConnection, true); CloseConnection(replicaConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse primary LSN result from %s:%d", primaryWorkerNode->workerName, primaryWorkerNode->workerPort), errdetail("Expected exactly one row with LSN value"))); } StringInfo primaryLsnQueryResInfo = (StringInfo) linitial(primaryLsnList); char *primary_lsn_str = primaryLsnQueryResInfo->data; List *replicaLsnList = ReadFirstColumnAsText(replicaResult); if (list_length(replicaLsnList) != 1) { PQclear(primaryResult); ClearResults(primaryConnection, true); CloseConnection(primaryConnection); PQclear(replicaResult); ClearResults(replicaConnection, true); CloseConnection(replicaConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse clone LSN result from %s:%d", replicaWorkerNode->workerName, replicaWorkerNode->workerPort), errdetail("Expected exactly one row with LSN value"))); } StringInfo replicaLsnQueryResInfo = (StringInfo) linitial(replicaLsnList); char *replica_lsn_str = replicaLsnQueryResInfo->data; int64 primary_lsn = DatumGetLSN(DirectFunctionCall1(pg_lsn_in, CStringGetDatum( primary_lsn_str))); int64 replica_lsn = DatumGetLSN(DirectFunctionCall1(pg_lsn_in, CStringGetDatum( replica_lsn_str))); int64 lag_bytes = primary_lsn - replica_lsn; PQclear(primaryResult); ForgetResults(primaryConnection); CloseConnection(primaryConnection); PQclear(replicaResult); ForgetResults(replicaConnection); CloseConnection(replicaConnection); ereport(DEBUG2, (errmsg( "successfully measured replication lag: primary LSN %s, clone LSN %s", primary_lsn_str, replica_lsn_str))); ereport(DEBUG1, (errmsg("replication lag between %s:%d and %s:%d is %ld bytes", primaryWorkerNode->workerName, primaryWorkerNode->workerPort, replicaWorkerNode->workerName, replicaWorkerNode->workerPort, lag_bytes))); return lag_bytes; } /* * EnsureValidCloneMode verifies that a clone node has a valid replication * relationship with the specified primary node. * * This function performs several critical checks: * 1. Validates that the clone is actually connected to and replicating from * the specified primary node * 2. Ensures the clone is not configured as a synchronous replica, which * would block 2PC commits on the primary when the clone gets promoted * 3. Verifies the replication connection is active and healthy * * The function connects to the primary node and queries pg_stat_replication * to find the clone's replication slot. It resolves hostnames to IP addresses * for robust matching since PostgreSQL may report different address formats. * * Parameters: * primaryWorkerNode - The primary node that should be sending replication data * cloneHostname - Hostname/IP of the clone node to verify * clonePort - Port of the clone node to verify * operation - Description of the operation being performed (for error messages) * * Throws ERROR if: * - Primary or clone parameters are invalid * - Cannot connect to the primary node * - Clone is not found in the primary's replication slots * - Clone is configured as a synchronous replica * - Replication connection is not active */ void EnsureValidCloneMode(WorkerNode *primaryWorkerNode, char *cloneHostname, int clonePort, char *operation) { Assert(operation != NULL); if (primaryWorkerNode == NULL || cloneHostname == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("primary or clone worker node is NULL"))); } ereport(NOTICE, (errmsg( "checking replication relationship between primary %s:%d and clone %s:%d", primaryWorkerNode->workerName, primaryWorkerNode->workerPort, cloneHostname, clonePort))); /* Connect to primary node to check replication status */ int connectionFlag = 0; MultiConnection *primaryConnection = GetNodeConnection(connectionFlag, primaryWorkerNode->workerName, primaryWorkerNode->workerPort); if (PQstatus(primaryConnection->pgConn) != CONNECTION_OK) { ReportConnectionError(primaryConnection, ERROR); } /* Build query to check if clone is connected and get its sync state */ StringInfo replicationCheckQuery = makeStringInfo(); /* First, try to resolve the hostname to IP address for more robust matching */ char *resolvedIP = NULL; struct addrinfo hints, *result, *rp; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_STREAM; /* TCP socket */ hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ int getaddrinfo_result = getaddrinfo(cloneHostname, NULL, &hints, &result); if (getaddrinfo_result == 0) { /* Get the first resolved IP address */ for (rp = result; rp != NULL; rp = rp->ai_next) { if (rp->ai_family == AF_INET) { /* IPv4 */ struct sockaddr_in *addr_in = (struct sockaddr_in *) rp->ai_addr; resolvedIP = palloc(INET_ADDRSTRLEN); inet_ntop(AF_INET, &(addr_in->sin_addr), resolvedIP, INET_ADDRSTRLEN); break; } else if (rp->ai_family == AF_INET6) { /* IPv6 */ struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *) rp->ai_addr; resolvedIP = palloc(INET6_ADDRSTRLEN); inet_ntop(AF_INET6, &(addr_in6->sin6_addr), resolvedIP, INET6_ADDRSTRLEN); break; } } freeaddrinfo(result); } ereport(NOTICE, (errmsg("checking replication status of clone node %s:%d", cloneHostname, clonePort))); /* Build query to check if clone is connected and get its sync state */ /* We check multiple fields to handle different scenarios: * 1. application_name - if it's set to the node name * 2. client_hostname - if it's the hostname * 3. client_addr - if it's the IP address (most reliable) */ if (resolvedIP != NULL) { appendStringInfo(replicationCheckQuery, "SELECT sync_state, state FROM pg_stat_replication WHERE " "application_name = '%s' OR " "client_hostname = '%s' OR " "client_addr = '%s'", cloneHostname, cloneHostname, resolvedIP); pfree(resolvedIP); } else { /* Fallback to hostname-only check if IP resolution fails */ appendStringInfo(replicationCheckQuery, "SELECT sync_state, state FROM pg_stat_replication WHERE " "application_name = '%s' OR " "client_hostname = '%s'", cloneHostname, cloneHostname); } ereport(DEBUG2, (errmsg("sending replication status check query: %s to primary %s:%d", replicationCheckQuery->data, primaryWorkerNode->workerName, primaryWorkerNode->workerPort))); int replicationCheckResultCode = SendRemoteCommand(primaryConnection, replicationCheckQuery->data); if (replicationCheckResultCode == 0) { pfree(replicationCheckQuery->data); pfree(replicationCheckQuery); CloseConnection(primaryConnection); ReportConnectionError(primaryConnection, ERROR); } PGresult *replicationCheckResult = GetRemoteCommandResult(primaryConnection, true); if (!IsResponseOK(replicationCheckResult)) { ReportResultError(primaryConnection, replicationCheckResult, ERROR); } List *replicationStateList = ReadFirstColumnAsText(replicationCheckResult); /* Check if clone is connected to this primary */ if (list_length(replicationStateList) == 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("clone %s:%d is not connected to primary %s:%d", cloneHostname, clonePort, primaryWorkerNode->workerName, primaryWorkerNode-> workerPort), errdetail( "The clone must be actively replicating from the specified primary node"), errhint( "Verify the clone is running and properly configured for replication"))); } /* Check if clone is synchronous */ if (list_length(replicationStateList) > 0) { StringInfo syncStateInfo = (StringInfo) linitial(replicationStateList); if (syncStateInfo && syncStateInfo->data && (strcmp(syncStateInfo->data, "sync") == 0 || strcmp(syncStateInfo->data, "quorum") == 0)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg( "cannot %s clone %s:%d as it is configured as a synchronous replica", operation, cloneHostname, clonePort), errdetail( "Promoting a synchronous clone can cause data consistency issues"), errhint( "Configure clone as an asynchronous replica"))); } } /* Cleanup resources */ bool raiseErrors = false; PQclear(replicationCheckResult); ClearResults(primaryConnection, raiseErrors); pfree(replicationCheckQuery->data); pfree(replicationCheckQuery); CloseConnection(primaryConnection); ereport(NOTICE, (errmsg( "clone %s:%d is properly connected to primary %s:%d and is not synchronous", cloneHostname, clonePort, primaryWorkerNode->workerName, primaryWorkerNode->workerPort)) ); } /* * EnsureValidStreamingReplica verifies that a node is a valid streaming replica * of the specified primary node. * * This function performs comprehensive validation to ensure the replica is: * 1. Currently in recovery mode (acting as a replica, not a primary) * 2. Has the same system identifier as the primary (ensuring they're part of * the same PostgreSQL cluster/timeline) * * The function connects to both the replica and primary nodes to perform these * checks. This validation is critical before performing operations like promotion * or failover to ensure data consistency and prevent split-brain scenarios. * * Parameters: * primaryWorkerNode - The primary node that should be the source of replication * replicaHostname - Hostname/IP of the replica node to validate * replicaPort - Port of the replica node to validate * * Throws ERROR if: * - Cannot connect to the replica or primary node * - Replica is not in recovery mode (indicating it's not acting as a replica) * - System identifiers don't match between primary and replica * - Any database queries fail during validation * */ void EnsureValidStreamingReplica(WorkerNode *primaryWorkerNode, char *replicaHostname, int replicaPort) { int connectionFlag = FORCE_NEW_CONNECTION; MultiConnection *replicaConnection = GetNodeConnection(connectionFlag, replicaHostname , replicaPort); if (PQstatus(replicaConnection->pgConn) != CONNECTION_OK) { ReportConnectionError(replicaConnection, ERROR); } const char *replica_recovery_query = "SELECT pg_is_in_recovery()"; int resultCode = SendRemoteCommand(replicaConnection, replica_recovery_query); if (resultCode == 0) { ereport(DEBUG2, (errmsg( "cannot connect to %s:%d to check if it is in recovery mode", replicaHostname, replicaPort))); ReportConnectionError(replicaConnection, ERROR); } bool raiseInterrupts = true; PGresult *result = GetRemoteCommandResult(replicaConnection, raiseInterrupts); if (!IsResponseOK(result)) { ereport(DEBUG2, (errmsg("failed to execute pg_is_in_recovery"))); ReportResultError(replicaConnection, result, ERROR); } List *sizeList = ReadFirstColumnAsText(result); if (list_length(sizeList) != 1) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse pg_is_in_recovery() result from %s:%d", replicaHostname, replicaPort))); } StringInfo isInRecoveryQueryResInfo = (StringInfo) linitial(sizeList); char *isInRecoveryQueryResStr = isInRecoveryQueryResInfo->data; if (strcmp(isInRecoveryQueryResStr, "t") != 0 && strcmp(isInRecoveryQueryResStr, "true") != 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("node %s:%d is not in recovery mode", replicaHostname, replicaPort))); } PQclear(result); ForgetResults(replicaConnection); /* Step2: Get the system identifier from replica */ const char *sysidQuery = "SELECT system_identifier FROM pg_control_system()"; resultCode = SendRemoteCommand(replicaConnection, sysidQuery); if (resultCode == 0) { ereport(DEBUG2, (errmsg("cannot connect to %s:%d to get system identifier", replicaHostname, replicaPort))); ReportConnectionError(replicaConnection, ERROR); } result = GetRemoteCommandResult(replicaConnection, raiseInterrupts); if (!IsResponseOK(result)) { ereport(DEBUG2, (errmsg("failed to execute get system identifier"))); ReportResultError(replicaConnection, result, ERROR); } List *sysidList = ReadFirstColumnAsText(result); if (list_length(sysidList) != 1) { CloseConnection(replicaConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse get system identifier result from %s:%d", replicaHostname, replicaPort))); } StringInfo sysidQueryResInfo = (StringInfo) linitial(sysidList); char *sysidQueryResStr = sysidQueryResInfo->data; ereport(DEBUG2, (errmsg("system identifier of %s:%d is %s", replicaHostname, replicaPort, sysidQueryResStr))); /* We do not need the connection anymore */ PQclear(result); ForgetResults(replicaConnection); CloseConnection(replicaConnection); /* Step3: Get system identifier from primary */ ereport(DEBUG2, (errmsg("getting system identifier from primary %s:%d", primaryWorkerNode->workerName, primaryWorkerNode->workerPort))); int primaryConnectionFlag = 0; MultiConnection *primaryConnection = GetNodeConnection(primaryConnectionFlag, primaryWorkerNode->workerName, primaryWorkerNode->workerPort); if (PQstatus(primaryConnection->pgConn) != CONNECTION_OK) { ReportConnectionError(primaryConnection, ERROR); } int primaryResultCode = SendRemoteCommand(primaryConnection, sysidQuery); if (primaryResultCode == 0) { ReportConnectionError(primaryConnection, ERROR); } PGresult *primaryResult = GetRemoteCommandResult(primaryConnection, raiseInterrupts); if (!IsResponseOK(primaryResult)) { ereport(DEBUG2, (errmsg("failed to execute get system identifier"))); ReportResultError(primaryConnection, primaryResult, ERROR); } List *primarySizeList = ReadFirstColumnAsText(primaryResult); if (list_length(primarySizeList) != 1) { CloseConnection(primaryConnection); ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg("cannot parse get system identifier result from %s:%d", primaryWorkerNode->workerName, primaryWorkerNode->workerPort))); } StringInfo primarySysidQueryResInfo = (StringInfo) linitial(primarySizeList); char *primarySysidQueryResStr = primarySysidQueryResInfo->data; ereport(DEBUG2, (errmsg("system identifier of %s:%d is %s", primaryWorkerNode->workerName, primaryWorkerNode->workerPort, primarySysidQueryResStr))); /* verify both identifiers */ if (strcmp(sysidQueryResStr, primarySysidQueryResStr) != 0) { ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), errmsg( "system identifiers do not match: %s (clone) vs %s (primary)", sysidQueryResStr, primarySysidQueryResStr))); } PQclear(primaryResult); ClearResults(primaryConnection, true); CloseConnection(primaryConnection); } ================================================ FILE: src/backend/distributed/utils/colocation_utils.c ================================================ /*------------------------------------------------------------------------- * * colocation_utils.c * * This file contains functions to perform useful operations on co-located tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/indexing.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/pg_dist_colocation.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/tenant_schema_metadata.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" /* local function forward declarations */ static void MarkTablesColocated(Oid sourceRelationId, Oid targetRelationId); static bool ShardsIntervalsEqual(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval); static bool HashPartitionedShardIntervalsEqual(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval); static int CompareShardPlacementsByNode(const void *leftElement, const void *rightElement); static uint32 CreateColocationGroupForRelation(Oid sourceRelationId); static void BreakColocation(Oid sourceRelationId); static uint32 SingleShardTableGetNodeId(Oid relationId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(mark_tables_colocated); PG_FUNCTION_INFO_V1(get_colocated_shard_array); PG_FUNCTION_INFO_V1(update_distributed_table_colocation); /* * mark_tables_colocated puts target tables to same colocation group with the * source table. If the source table is in INVALID_COLOCATION_ID group, then it * creates a new colocation group and assigns all tables to this new colocation * group. */ Datum mark_tables_colocated(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("this function is deprecated and no longer is used"))); } /* * update_distributed_table_colocation updates the colocation of a table. * if colocate_with -> 'none' then the table is assigned a new * colocation group. */ Datum update_distributed_table_colocation(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid targetRelationId = PG_GETARG_OID(0); text *colocateWithTableNameText = PG_GETARG_TEXT_P(1); EnsureTableOwner(targetRelationId); ErrorIfTenantTable(targetRelationId, TenantOperationNames[TENANT_UPDATE_COLOCATION]); char *colocateWithTableName = text_to_cstring(colocateWithTableNameText); if (IsColocateWithNone(colocateWithTableName)) { EnsureHashOrSingleShardDistributedTable(targetRelationId); BreakColocation(targetRelationId); } else { Oid colocateWithTableId = ResolveRelationId(colocateWithTableNameText, false); ErrorIfTenantTable(colocateWithTableId, TenantOperationNames[TENANT_COLOCATE_WITH]); EnsureTableOwner(colocateWithTableId); MarkTablesColocated(colocateWithTableId, targetRelationId); } PG_RETURN_VOID(); } /* * IsColocateWithNone returns true if the given table is * the special keyword "none". */ bool IsColocateWithNone(char *colocateWithTableName) { return pg_strncasecmp(colocateWithTableName, "none", NAMEDATALEN) == 0; } /* * IsColocateWithDefault returns true if the given table is * the special keyword "default". */ bool IsColocateWithDefault(char *colocateWithTableName) { return pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) == 0; } /* * BreakColocation breaks the colocations of the given relation id. * If t1, t2 and t3 are colocated and we call this function with t2, * t1 and t3 will stay colocated but t2 will have a new colocation id. * Note that this function does not move any data around for the new colocation. */ static void BreakColocation(Oid sourceRelationId) { /* * Get an exclusive lock on the colocation system catalog. Therefore, we * can be sure that there will no modifications on the colocation table * until this transaction is committed. */ Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock); uint32 oldColocationId = TableColocationId(sourceRelationId); CreateColocationGroupForRelation(sourceRelationId); /* if there is not any remaining table in the old colocation group, delete it */ DeleteColocationGroupIfNoTablesBelong(oldColocationId); table_close(pgDistColocation, NoLock); } /* * get_colocated_shards_array returns array of shards ids which are co-located with given * shard. */ Datum get_colocated_shard_array(PG_FUNCTION_ARGS) { uint32 shardId = PG_GETARG_UINT32(0); ShardInterval *shardInterval = LoadShardInterval(shardId); List *colocatedShardList = ColocatedShardIntervalList(shardInterval); int colocatedShardCount = list_length(colocatedShardList); Datum *colocatedShardsDatumArray = palloc0(colocatedShardCount * sizeof(Datum)); Oid arrayTypeId = OIDOID; int colocatedShardIndex = 0; ShardInterval *colocatedShardInterval = NULL; foreach_declared_ptr(colocatedShardInterval, colocatedShardList) { uint64 colocatedShardId = colocatedShardInterval->shardId; Datum colocatedShardDatum = Int64GetDatum(colocatedShardId); colocatedShardsDatumArray[colocatedShardIndex] = colocatedShardDatum; colocatedShardIndex++; } ArrayType *colocatedShardsArrayType = DatumArrayToArrayType(colocatedShardsDatumArray, colocatedShardCount, arrayTypeId); PG_RETURN_ARRAYTYPE_P(colocatedShardsArrayType); } /* * CreateColocationGroupForRelation creates colocation entry in * pg_dist_colocation and updated the colocation id in pg_dist_partition * for the given relation. */ static uint32 CreateColocationGroupForRelation(Oid sourceRelationId) { uint32 shardCount = ShardIntervalCount(sourceRelationId); uint32 shardReplicationFactor = TableShardReplicationFactor(sourceRelationId); Var *sourceDistributionColumn = DistPartitionKey(sourceRelationId); Oid sourceDistributionColumnType = InvalidOid; Oid sourceDistributionColumnCollation = InvalidOid; /* reference tables has NULL distribution column */ if (sourceDistributionColumn != NULL) { sourceDistributionColumnType = sourceDistributionColumn->vartype; sourceDistributionColumnCollation = sourceDistributionColumn->varcollid; } uint32 sourceColocationId = CreateColocationGroup(shardCount, shardReplicationFactor, sourceDistributionColumnType, sourceDistributionColumnCollation); bool localOnly = false; UpdateRelationColocationGroup(sourceRelationId, sourceColocationId, localOnly); return sourceColocationId; } /* * MarkTablesColocated puts both tables to same colocation group. If the * source table is in INVALID_COLOCATION_ID group, then it creates a new * colocation group and assigns both tables to same colocation group. Otherwise, * it adds the target table to colocation group of the source table. */ static void MarkTablesColocated(Oid sourceRelationId, Oid targetRelationId) { if (IsCitusTableType(sourceRelationId, CITUS_LOCAL_TABLE) || IsCitusTableType(targetRelationId, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errmsg("local tables cannot be colocated with " "other tables"))); } EnsureHashOrSingleShardDistributedTable(sourceRelationId); EnsureHashOrSingleShardDistributedTable(targetRelationId); CheckReplicationModel(sourceRelationId, targetRelationId); CheckDistributionColumnType(sourceRelationId, targetRelationId); /* * Get an exclusive lock on the colocation system catalog. Therefore, we * can be sure that there will no modifications on the colocation table * until this transaction is committed. */ Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock); /* check if shard placements are colocated */ ErrorIfShardPlacementsNotColocated(sourceRelationId, targetRelationId); /* * Get colocation group of the source table, if the source table does not * have a colocation group, create a new one, and set it for the source table. */ uint32 sourceColocationId = TableColocationId(sourceRelationId); if (sourceColocationId == INVALID_COLOCATION_ID) { sourceColocationId = CreateColocationGroupForRelation(sourceRelationId); } uint32 targetColocationId = TableColocationId(targetRelationId); /* finally set colocation group for the target relation */ bool localOnly = false; UpdateRelationColocationGroup(targetRelationId, sourceColocationId, localOnly); /* if there is not any remaining table in the colocation group, delete it */ DeleteColocationGroupIfNoTablesBelong(targetColocationId); table_close(pgDistColocation, NoLock); } /* * ErrorIfShardPlacementsNotColocated checks if the shard placements of the * given two relations are physically colocated. It errors out in any of * following cases: * 1.Shard counts are different, * 2.Shard intervals don't match * 3.Matching shard intervals have different number of shard placements * 4.Shard placements are not colocated (not on the same node) * 5.Shard placements have different health states * * Note that, this functions assumes that both tables are hash distributed. */ void ErrorIfShardPlacementsNotColocated(Oid leftRelationId, Oid rightRelationId) { /* get sorted shard interval lists for both tables */ List *leftShardIntervalList = LoadShardIntervalList(leftRelationId); List *rightShardIntervalList = LoadShardIntervalList(rightRelationId); /* prevent concurrent placement changes */ LockShardListMetadata(leftShardIntervalList, ShareLock); LockShardListMetadata(rightShardIntervalList, ShareLock); char *leftRelationName = get_rel_name(leftRelationId); char *rightRelationName = get_rel_name(rightRelationId); uint32 leftShardCount = list_length(leftShardIntervalList); uint32 rightShardCount = list_length(rightShardIntervalList); if (leftShardCount != rightShardCount) { ereport(ERROR, (errmsg("cannot colocate tables %s and %s", leftRelationName, rightRelationName), errdetail("Shard counts don't match for %s and %s.", leftRelationName, rightRelationName))); } /* compare shard intervals one by one */ ShardInterval *leftInterval = NULL; ShardInterval *rightInterval = NULL; forboth_ptr(leftInterval, leftShardIntervalList, rightInterval, rightShardIntervalList) { uint64 leftShardId = leftInterval->shardId; uint64 rightShardId = rightInterval->shardId; bool shardsIntervalsEqual = ShardsIntervalsEqual(leftInterval, rightInterval); if (!shardsIntervalsEqual) { ereport(ERROR, (errmsg("cannot colocate tables %s and %s", leftRelationName, rightRelationName), errdetail("Shard intervals don't match for %s and %s.", leftRelationName, rightRelationName))); } List *leftPlacementList = ShardPlacementList(leftShardId); List *rightPlacementList = ShardPlacementList(rightShardId); if (list_length(leftPlacementList) != list_length(rightPlacementList)) { ereport(ERROR, (errmsg("cannot colocate tables %s and %s", leftRelationName, rightRelationName), errdetail("Shard " UINT64_FORMAT " of %s and shard " UINT64_FORMAT " of %s have different number of shard placements.", leftShardId, leftRelationName, rightShardId, rightRelationName))); } /* sort shard placements according to the node */ List *sortedLeftPlacementList = SortList(leftPlacementList, CompareShardPlacementsByNode); List *sortedRightPlacementList = SortList(rightPlacementList, CompareShardPlacementsByNode); /* compare shard placements one by one */ ShardPlacement *leftPlacement = NULL; ShardPlacement *rightPlacement = NULL; forboth_ptr(leftPlacement, sortedLeftPlacementList, rightPlacement, sortedRightPlacementList) { /* * If shard placements are on different nodes, these shard * placements are not colocated. */ int nodeCompare = CompareShardPlacementsByNode((void *) &leftPlacement, (void *) &rightPlacement); if (nodeCompare != 0) { ereport(ERROR, (errmsg("cannot colocate tables %s and %s", leftRelationName, rightRelationName), errdetail("Shard " UINT64_FORMAT " of %s and shard " UINT64_FORMAT " of %s are not colocated.", leftShardId, leftRelationName, rightShardId, rightRelationName))); } } } } /* * ShardsIntervalsEqual checks if two shard intervals of distributed * tables are equal. * * Notes on the function: * (i) The function returns true if both shard intervals are the same. * (ii) The function returns false even if the shard intervals equal, but, * their distribution method are different. * (iii) The function returns false for append and range partitioned tables * excluding (i) case. * (iv) For reference tables, all shards are equal (i.e., same replication factor * and shard min/max values). Thus, always return true for shards of reference * tables. */ static bool ShardsIntervalsEqual(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval) { char leftIntervalPartitionMethod = PartitionMethod(leftShardInterval->relationId); char rightIntervalPartitionMethod = PartitionMethod(rightShardInterval->relationId); /* if both shards are the same, return true */ if (leftShardInterval->shardId == rightShardInterval->shardId) { return true; } /* if partition methods are not the same, shards cannot be considered as co-located */ leftIntervalPartitionMethod = PartitionMethod(leftShardInterval->relationId); rightIntervalPartitionMethod = PartitionMethod(rightShardInterval->relationId); if (leftIntervalPartitionMethod != rightIntervalPartitionMethod) { return false; } if (IsCitusTableType(leftShardInterval->relationId, HASH_DISTRIBUTED)) { return HashPartitionedShardIntervalsEqual(leftShardInterval, rightShardInterval); } else if (!HasDistributionKey(leftShardInterval->relationId)) { /* * Reference tables has only a single shard and all reference tables * are always co-located with each other. */ return true; } /* append and range partitioned shard never co-located */ return false; } /* * HashPartitionedShardIntervalsEqual checks if two shard intervals of hash distributed * tables are equal. Note that, this function doesn't work with non-hash * partitioned table's shards. * * We do min/max value check here to decide whether two shards are colocated, * instead we can simply use ShardIndex function on both shards then * but do index check, but we avoid it because this way it is more cheaper. */ static bool HashPartitionedShardIntervalsEqual(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval) { int32 leftShardMinValue = DatumGetInt32(leftShardInterval->minValue); int32 leftShardMaxValue = DatumGetInt32(leftShardInterval->maxValue); int32 rightShardMinValue = DatumGetInt32(rightShardInterval->minValue); int32 rightShardMaxValue = DatumGetInt32(rightShardInterval->maxValue); bool minValuesEqual = leftShardMinValue == rightShardMinValue; bool maxValuesEqual = leftShardMaxValue == rightShardMaxValue; return minValuesEqual && maxValuesEqual; } /* * CompareShardPlacementsByNode compares two shard placements by their nodename * and nodeport. */ static int CompareShardPlacementsByNode(const void *leftElement, const void *rightElement) { const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); /* if node names are same, check node ports */ if (leftPlacement->nodeId < rightPlacement->nodeId) { return -1; } else if (leftPlacement->nodeId > rightPlacement->nodeId) { return 1; } else { return 0; } } /* * ColocationId searches pg_dist_colocation for shard count, replication factor, * distribution column type, and distribution column collation. If a matching entry * is found, it returns the colocation id, otherwise returns INVALID_COLOCATION_ID. */ uint32 ColocationId(int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation) { uint32 colocationId = INVALID_COLOCATION_ID; const int scanKeyCount = 4; ScanKeyData scanKey[4]; bool indexOK = true; Relation pgDistColocation = table_open(DistColocationRelationId(), AccessShareLock); /* set scan arguments */ ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_distributioncolumntype, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distributionColumnType)); ScanKeyInit(&scanKey[1], Anum_pg_dist_colocation_shardcount, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(shardCount)); ScanKeyInit(&scanKey[2], Anum_pg_dist_colocation_replicationfactor, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(replicationFactor)); ScanKeyInit(&scanKey[3], Anum_pg_dist_colocation_distributioncolumncollation, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum( distributionColumnCollation)); SysScanDesc scanDescriptor = systable_beginscan(pgDistColocation, DistColocationConfigurationIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple colocationTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(colocationTuple)) { Form_pg_dist_colocation colocationForm = (Form_pg_dist_colocation) GETSTRUCT(colocationTuple); /* avoid chosing a colocation group that belongs to a tenant schema */ if (IsTenantSchemaColocationGroup(colocationForm->colocationid)) { colocationTuple = systable_getnext(scanDescriptor); continue; } if (colocationId == INVALID_COLOCATION_ID || colocationId > colocationForm->colocationid) { /* * We assign the smallest colocation id among all the matches so that we * assign the same colocation group for similar distributed tables */ colocationId = colocationForm->colocationid; } colocationTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgDistColocation, AccessShareLock); return colocationId; } /* * AcquireColocationDefaultLock serializes concurrent creation of a colocation entry * for default group. */ void AcquireColocationDefaultLock(void) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_CREATE_COLOCATION_DEFAULT); (void) LockAcquire(&tag, ExclusiveLock, sessionLock, dontWait); } /* * ReleaseColocationDefaultLock releases the lock for concurrent creation of a colocation entry * for default group. */ void ReleaseColocationDefaultLock(void) { LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_CREATE_COLOCATION_DEFAULT); LockRelease(&tag, ExclusiveLock, sessionLock); } /* * CreateColocationGroup creates a new colocation id and writes it into * pg_dist_colocation with the given configuration. It also returns the created * colocation id. */ uint32 CreateColocationGroup(int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation) { uint32 colocationId = GetNextColocationId(); InsertColocationGroupLocally(colocationId, shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); SyncNewColocationGroupToNodes(colocationId, shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); return colocationId; } /* * InsertColocationGroupLocally inserts a record into pg_dist_colocation. */ void InsertColocationGroupLocally(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation) { Datum values[Natts_pg_dist_colocation]; bool isNulls[Natts_pg_dist_colocation]; /* form new colocation tuple */ memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); values[Anum_pg_dist_colocation_colocationid - 1] = UInt32GetDatum(colocationId); values[Anum_pg_dist_colocation_shardcount - 1] = UInt32GetDatum(shardCount); values[Anum_pg_dist_colocation_replicationfactor - 1] = UInt32GetDatum(replicationFactor); values[Anum_pg_dist_colocation_distributioncolumntype - 1] = ObjectIdGetDatum(distributionColumnType); values[Anum_pg_dist_colocation_distributioncolumncollation - 1] = ObjectIdGetDatum(distributionColumnCollation); /* open colocation relation and insert the new tuple */ Relation pgDistColocation = table_open(DistColocationRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistColocation); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); CatalogTupleInsert(pgDistColocation, heapTuple); /* increment the counter so that next command can see the row */ CommandCounterIncrement(); table_close(pgDistColocation, NoLock); } /* * GetNextColocationId allocates and returns a unique colocationId for the * colocation group to be created. This allocation occurs both in shared memory * and in write ahead logs; writing to logs avoids the risk of having * colocationId collisions. * * Please note that the caller is still responsible for finalizing colocationId * with the master node. Further note that this function relies on an internal * sequence created in initdb to generate unique identifiers. */ uint32 GetNextColocationId() { text *sequenceName = cstring_to_text(COLOCATIONID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); Oid savedUserId = InvalidOid; int savedSecurityContext = 0; GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); /* generate new and unique colocation id from sequence */ Datum colocationIdDatum = DirectFunctionCall1(nextval_oid, sequenceIdDatum); SetUserIdAndSecContext(savedUserId, savedSecurityContext); uint32 colocationId = DatumGetUInt32(colocationIdDatum); return colocationId; } /* * CheckReplicationModel checks if given relations are from the same * replication model. Otherwise, it errors out. */ void CheckReplicationModel(Oid sourceRelationId, Oid targetRelationId) { CitusTableCacheEntry *sourceTableEntry = GetCitusTableCacheEntry(sourceRelationId); char sourceReplicationModel = sourceTableEntry->replicationModel; CitusTableCacheEntry *targetTableEntry = GetCitusTableCacheEntry(targetRelationId); char targetReplicationModel = targetTableEntry->replicationModel; if (sourceReplicationModel != targetReplicationModel) { char *sourceRelationName = get_rel_name(sourceRelationId); char *targetRelationName = get_rel_name(targetRelationId); ereport(ERROR, (errmsg("cannot colocate tables %s and %s", sourceRelationName, targetRelationName), errdetail("Replication models don't match for %s and %s.", sourceRelationName, targetRelationName))); } } /* * CheckDistributionColumnType checks if distribution column types of relations * are same. Otherwise, it errors out. */ void CheckDistributionColumnType(Oid sourceRelationId, Oid targetRelationId) { /* reference tables have NULL distribution column */ Var *sourceDistributionColumn = DistPartitionKey(sourceRelationId); /* reference tables have NULL distribution column */ Var *targetDistributionColumn = DistPartitionKey(targetRelationId); EnsureColumnTypeEquality(sourceRelationId, targetRelationId, sourceDistributionColumn, targetDistributionColumn); } /* * EnsureColumnTypeEquality checks if distribution column types and collations * of the given columns are same. The function sets the boolean pointers. */ void EnsureColumnTypeEquality(Oid sourceRelationId, Oid targetRelationId, Var *sourceDistributionColumn, Var *targetDistributionColumn) { Oid sourceDistributionColumnType = InvalidOid; Oid targetDistributionColumnType = InvalidOid; Oid sourceDistributionColumnCollation = InvalidOid; Oid targetDistributionColumnCollation = InvalidOid; if (sourceDistributionColumn != NULL) { sourceDistributionColumnType = sourceDistributionColumn->vartype; sourceDistributionColumnCollation = sourceDistributionColumn->varcollid; } if (targetDistributionColumn != NULL) { targetDistributionColumnType = targetDistributionColumn->vartype; targetDistributionColumnCollation = targetDistributionColumn->varcollid; } bool columnTypesSame = sourceDistributionColumnType == targetDistributionColumnType; bool columnCollationsSame = sourceDistributionColumnCollation == targetDistributionColumnCollation; if (!columnTypesSame) { char *sourceRelationName = get_rel_name(sourceRelationId); char *targetRelationName = get_rel_name(targetRelationId); ereport(ERROR, (errmsg("cannot colocate tables %s and %s", sourceRelationName, targetRelationName), errdetail("Distribution column types don't match for " "%s and %s.", sourceRelationName, targetRelationName))); } if (!columnCollationsSame) { char *sourceRelationName = get_rel_name(sourceRelationId); char *targetRelationName = get_rel_name(targetRelationId); ereport(ERROR, (errmsg("cannot colocate tables %s and %s", sourceRelationName, targetRelationName), errdetail( "Distribution column collations don't match for " "%s and %s.", sourceRelationName, targetRelationName))); } } /* * UpdateRelationColocationGroup updates colocation group in pg_dist_partition * for the given relation. * * When localOnly is true, the function does not propagate changes to the * metadata workers. */ void UpdateRelationColocationGroup(Oid distributedRelationId, uint32 colocationId, bool localOnly) { bool indexOK = true; int scanKeyCount = 1; ScanKeyData scanKey[1]; Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); Datum *values = (Datum *) palloc0(tupleDescriptor->natts * sizeof(Datum)); bool *isNull = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); bool *replace = (bool *) palloc0(tupleDescriptor->natts * sizeof(bool)); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distributedRelationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionLogicalRelidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (!HeapTupleIsValid(heapTuple)) { char *distributedRelationName = get_rel_name(distributedRelationId); ereport(ERROR, (errmsg("could not find valid entry for relation %s", distributedRelationName))); } values[Anum_pg_dist_partition_colocationid - 1] = UInt32GetDatum(colocationId); isNull[Anum_pg_dist_partition_colocationid - 1] = false; replace[Anum_pg_dist_partition_colocationid - 1] = true; heapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isNull, replace); CatalogTupleUpdate(pgDistPartition, &heapTuple->t_self, heapTuple); CitusInvalidateRelcacheByRelid(distributedRelationId); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistPartition, NoLock); pfree(values); pfree(isNull); pfree(replace); bool shouldSyncMetadata = ShouldSyncTableMetadata(distributedRelationId); if (shouldSyncMetadata && !localOnly) { char *updateColocationIdCommand = ColocationIdUpdateCommand(distributedRelationId, colocationId); SendCommandToWorkersWithMetadata(updateColocationIdCommand); } } /* * TableColocationId function returns co-location id of given table. This function * errors out if given table is not distributed. */ uint32 TableColocationId(Oid distributedTableId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); return cacheEntry->colocationId; } /* * TablesColocated function checks whether given two tables are co-located and * returns true if they are co-located. A table is always co-located with itself. * If given two tables are different and they are not distributed, this function * errors out. */ bool TablesColocated(Oid leftDistributedTableId, Oid rightDistributedTableId) { if (leftDistributedTableId == rightDistributedTableId) { return true; } uint32 leftColocationId = TableColocationId(leftDistributedTableId); uint32 rightColocationId = TableColocationId(rightDistributedTableId); if (leftColocationId == INVALID_COLOCATION_ID || rightColocationId == INVALID_COLOCATION_ID) { return false; } return leftColocationId == rightColocationId; } /* * ShardsColocated function checks whether given two shards are co-located and * returns true if they are co-located. Two shards are co-located either; * - They are same (A shard is always co-located with itself). * OR * - Tables are hash partitioned. * - Tables containing the shards are co-located. * - Min/Max values of the shards are same. */ bool ShardsColocated(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval) { bool tablesColocated = TablesColocated(leftShardInterval->relationId, rightShardInterval->relationId); if (tablesColocated) { bool shardIntervalEqual = ShardsIntervalsEqual(leftShardInterval, rightShardInterval); return shardIntervalEqual; } return false; } /* * ColocatedTableList function returns list of relation ids which are co-located * with given table. If given table is not hash distributed, co-location is not * valid for that table and it is only co-located with itself. */ List * ColocatedTableList(Oid distributedTableId) { uint32 tableColocationId = TableColocationId(distributedTableId); List *colocatedTableList = NIL; /* * If distribution type of the table is not hash, the table is only co-located * with itself. */ if (tableColocationId == INVALID_COLOCATION_ID) { colocatedTableList = lappend_oid(colocatedTableList, distributedTableId); return colocatedTableList; } int count = 0; colocatedTableList = ColocationGroupTableList(tableColocationId, count); return colocatedTableList; } /* * ColocationGroupTableList returns the list of tables in the given colocation * group. If the colocation group is INVALID_COLOCATION_ID, it returns NIL. * * If count is zero then the command is executed for all rows that it applies to. * If count is greater than zero, then no more than count rows will be retrieved; * execution stops when the count is reached, much like adding a LIMIT clause * to the query. */ List * ColocationGroupTableList(uint32 colocationId, uint32 count) { List *colocatedTableList = NIL; bool indexOK = true; int scanKeyCount = 1; ScanKeyData scanKey[1]; /* * If distribution type of the table is not hash, the table is only co-located * with itself. */ if (colocationId == INVALID_COLOCATION_ID) { return NIL; } ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_colocationid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(colocationId)); Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionColocationidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); while (HeapTupleIsValid(heapTuple)) { memset(datumArray, 0, tupleDescriptor->natts * sizeof(Datum)); memset(isNullArray, 0, tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); Oid colocatedTableId = DatumGetObjectId( datumArray[Anum_pg_dist_partition_logicalrelid - 1]); colocatedTableList = lappend_oid(colocatedTableList, colocatedTableId); heapTuple = systable_getnext(scanDescriptor); if (count == 0) { /* fetch all rows */ continue; } else if (list_length(colocatedTableList) >= count) { /* we are done */ break; } } pfree(datumArray); pfree(isNullArray); systable_endscan(scanDescriptor); table_close(pgDistPartition, AccessShareLock); return colocatedTableList; } /* * ColocatedShardIntervalList function returns list of shard intervals which are * co-located with given shard. If given shard is belong to append or range distributed * table, co-location is not valid for that shard. Therefore such shard is only co-located * with itself. */ List * ColocatedShardIntervalList(ShardInterval *shardInterval) { Oid distributedTableId = shardInterval->relationId; List *colocatedShardList = NIL; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); /* * If distribution type of the table is append or range, each shard of * the shard is only co-located with itself. */ if (IsCitusTableTypeCacheEntry(cacheEntry, APPEND_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED)) { ShardInterval *copyShardInterval = CopyShardInterval(shardInterval); colocatedShardList = lappend(colocatedShardList, copyShardInterval); return colocatedShardList; } int shardIntervalIndex = ShardIndex(shardInterval); List *colocatedTableList = ColocatedTableList(distributedTableId); /* ShardIndex have to find index of given shard */ Assert(shardIntervalIndex >= 0); Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { CitusTableCacheEntry *colocatedTableCacheEntry = GetCitusTableCacheEntry(colocatedTableId); /* * Since we iterate over co-located tables, shard count of each table should be * same and greater than shardIntervalIndex. */ Assert(cacheEntry->shardIntervalArrayLength == colocatedTableCacheEntry-> shardIntervalArrayLength); ShardInterval *colocatedShardInterval = colocatedTableCacheEntry->sortedShardIntervalArray[shardIntervalIndex]; ShardInterval *copyShardInterval = CopyShardInterval(colocatedShardInterval); colocatedShardList = lappend(colocatedShardList, copyShardInterval); } Assert(list_length(colocatedTableList) == list_length(colocatedShardList)); return SortList(colocatedShardList, CompareShardIntervalsById); } /* * ColocatedNonPartitionShardIntervalList function returns list of shard intervals * which are co-located with given shard, except partitions. If given shard is belong * to append or range distributed table, co-location is not valid for that shard. * Therefore such shard is only co-located with itself. */ List * ColocatedNonPartitionShardIntervalList(ShardInterval *shardInterval) { Oid distributedTableId = shardInterval->relationId; List *colocatedShardList = NIL; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); /* * If distribution type of the table is append or range, each shard of the shard * is only co-located with itself. We don't expect this case to happen, since * distributing partitioned tables in only supported for hash-distributed tables. * Therefore, currently we can't cover here with a test. */ if (IsCitusTableTypeCacheEntry(cacheEntry, APPEND_DISTRIBUTED) || IsCitusTableTypeCacheEntry(cacheEntry, RANGE_DISTRIBUTED)) { ShardInterval *copyShardInterval = CopyShardInterval(shardInterval); colocatedShardList = lappend(colocatedShardList, copyShardInterval); return colocatedShardList; } ereport(DEBUG1, (errmsg("skipping child tables for relation named: %s", get_rel_name(distributedTableId)))); int shardIntervalIndex = ShardIndex(shardInterval); List *colocatedTableList = ColocatedTableList(distributedTableId); /* ShardIndex have to find index of given shard */ Assert(shardIntervalIndex >= 0); Oid colocatedTableId = InvalidOid; foreach_declared_oid(colocatedTableId, colocatedTableList) { if (PartitionTable(colocatedTableId)) { continue; } CitusTableCacheEntry *colocatedTableCacheEntry = GetCitusTableCacheEntry(colocatedTableId); /* * Since we iterate over co-located tables, shard count of each table should be * same and greater than shardIntervalIndex. */ Assert(cacheEntry->shardIntervalArrayLength == colocatedTableCacheEntry-> shardIntervalArrayLength); ShardInterval *colocatedShardInterval = colocatedTableCacheEntry->sortedShardIntervalArray[shardIntervalIndex]; ShardInterval *copyShardInterval = CopyShardInterval(colocatedShardInterval); colocatedShardList = lappend(colocatedShardList, copyShardInterval); } return SortList(colocatedShardList, CompareShardIntervalsById); } /* * ColocatedTableId returns an arbitrary table which belongs to given colocation * group. If there is not such a colocation group, it returns invalid oid. * * This function also takes an AccessShareLock on the co-colocated table to * guarantee that the table isn't dropped for the remainder of the transaction. */ Oid ColocatedTableId(int32 colocationId) { Oid colocatedTableId = InvalidOid; bool indexOK = true; ScanKeyData scanKey[1]; int scanKeyCount = 1; /* * We may have a distributed table whose colocation id is INVALID_COLOCATION_ID. * In this case, we do not want to send that table's id as colocated table id. */ if (colocationId == INVALID_COLOCATION_ID) { return colocatedTableId; } ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_colocationid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(colocationId)); Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionColocationidIndexId(), indexOK, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); Datum *datumArray = (Datum *) palloc(tupleDescriptor->natts * sizeof(Datum)); bool *isNullArray = (bool *) palloc(tupleDescriptor->natts * sizeof(bool)); while (HeapTupleIsValid(heapTuple)) { memset(datumArray, 0, tupleDescriptor->natts * sizeof(Datum)); memset(isNullArray, 0, tupleDescriptor->natts * sizeof(bool)); heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); colocatedTableId = DatumGetObjectId( datumArray[Anum_pg_dist_partition_logicalrelid - 1]); /* * Make sure the relation isn't dropped for the remainder of * the transaction. */ LockRelationOid(colocatedTableId, AccessShareLock); /* * The relation might have been dropped just before we locked it. * Let's look it up. */ Relation colocatedRelation = RelationIdGetRelation(colocatedTableId); if (RelationIsValid(colocatedRelation)) { /* relation still exists, we can use it */ RelationClose(colocatedRelation); break; } /* relation was dropped, try the next one */ colocatedTableId = InvalidOid; heapTuple = systable_getnext(scanDescriptor); } pfree(datumArray); pfree(isNullArray); systable_endscan(scanDescriptor); table_close(pgDistPartition, AccessShareLock); return colocatedTableId; } /* * SingleShardTableColocationNodeId takes a colocation id that presumably * belongs to colocation group used to colocate a set of single-shard * tables and returns id of the node that stores / is expected to store * the shards within the colocation group. */ uint32 SingleShardTableColocationNodeId(uint32 colocationId) { List *tablesInColocationGroup = ColocationGroupTableList(colocationId, 0); if (list_length(tablesInColocationGroup) == 0) { int workerNodeIndex = EmptySingleShardTableColocationDecideNodeId(colocationId); List *workerNodeList = DistributedTablePlacementNodeList(RowShareLock); WorkerNode *workerNode = (WorkerNode *) list_nth(workerNodeList, workerNodeIndex); return workerNode->nodeId; } else { Oid colocatedTableId = ColocatedTableId(colocationId); return SingleShardTableGetNodeId(colocatedTableId); } } /* * SingleShardTableGetNodeId returns id of the node that stores shard of * given single-shard table. */ static uint32 SingleShardTableGetNodeId(Oid relationId) { if (!IsCitusTableType(relationId, SINGLE_SHARD_DISTRIBUTED)) { ereport(ERROR, (errmsg("table is not a single-shard distributed table"))); } int64 shardId = GetFirstShardId(relationId); List *shardPlacementList = ShardPlacementList(shardId); if (list_length(shardPlacementList) != 1) { ereport(ERROR, (errmsg("table shard does not have a single shard placement"))); } return ((ShardPlacement *) linitial(shardPlacementList))->nodeId; } /* * ColocatedShardIdInRelation returns shardId of the shard from given relation, so that * returned shard is co-located with given shard. */ uint64 ColocatedShardIdInRelation(Oid relationId, int shardIndex) { CitusTableCacheEntry *tableCacheEntry = GetCitusTableCacheEntry(relationId); return tableCacheEntry->sortedShardIntervalArray[shardIndex]->shardId; } /* * DeleteColocationGroupIfNoTablesBelong function deletes given co-location group if there * is no relation in that co-location group. A co-location group may become empty after * update_distributed_table_colocation UDF calls. In that case we need to * remove empty co-location group to prevent orphaned co-location groups. */ void DeleteColocationGroupIfNoTablesBelong(uint32 colocationId) { if (colocationId != INVALID_COLOCATION_ID) { int count = 1; List *colocatedTableList = ColocationGroupTableList(colocationId, count); int colocatedTableCount = list_length(colocatedTableList); if (colocatedTableCount == 0) { DeleteColocationGroup(colocationId); } } } /* * DeleteColocationGroup deletes the colocation group from pg_dist_colocation * throughout the cluster and dissociates the tenant schema if any. */ void DeleteColocationGroup(uint32 colocationId) { DeleteColocationGroupLocally(colocationId); SyncDeleteColocationGroupToNodes(colocationId); } /* * DeleteColocationGroupLocally deletes the colocation group from pg_dist_colocation. */ void DeleteColocationGroupLocally(uint32 colocationId) { int scanKeyCount = 1; ScanKeyData scanKey[1]; bool indexOK = false; Relation pgDistColocation = table_open(DistColocationRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_colocationid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(colocationId)); SysScanDesc scanDescriptor = systable_beginscan(pgDistColocation, InvalidOid, indexOK, NULL, scanKeyCount, scanKey); /* if a record is found, delete it */ HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { /* * simple_heap_delete() expects that the caller has at least an * AccessShareLock on primary key index. * * XXX: This does not seem required, do we really need to acquire this lock? * Postgres doesn't acquire such locks on indexes before deleting catalog tuples. * Linking here the reasons we added this lock acquirement: * https://github.com/citusdata/citus/pull/2851#discussion_r306569462 * https://github.com/citusdata/citus/pull/2855#discussion_r313628554 * https://github.com/citusdata/citus/issues/1890 */ #if PG_VERSION_NUM >= PG_VERSION_18 /* PG 18+ expects a second “deferrable_ok” flag */ Relation replicaIndex = index_open( RelationGetPrimaryKeyIndex(pgDistColocation, false), AccessShareLock ); #else /* PG 17- had a single-arg signature */ Relation replicaIndex = index_open( RelationGetPrimaryKeyIndex(pgDistColocation), AccessShareLock ); #endif simple_heap_delete(pgDistColocation, &(heapTuple->t_self)); CitusInvalidateRelcacheByRelid(DistColocationRelationId()); CommandCounterIncrement(); table_close(replicaIndex, AccessShareLock); } systable_endscan(scanDescriptor); table_close(pgDistColocation, NoLock); } /* * FindColocateWithColocationId tries to find a colocation ID for a given * colocate_with clause passed to create_distributed_table. */ uint32 FindColocateWithColocationId(Oid relationId, char replicationModel, Var *distributionColumn, int shardCount, bool shardCountIsStrict, char *colocateWithTableName) { uint32 colocationId = INVALID_COLOCATION_ID; if (IsColocateWithDefault(colocateWithTableName)) { /* distributionColumn can only be null for single-shard tables */ Oid distributionColumnType = distributionColumn ? distributionColumn->vartype : InvalidOid; Oid distributionColumnCollation = distributionColumn ? distributionColumn->varcollid : InvalidOid; /* check for default colocation group */ colocationId = ColocationId(shardCount, ShardReplicationFactor, distributionColumnType, distributionColumnCollation); /* * if the shardCount is strict then we check if the shard count * of the colocated table is actually shardCount */ if (shardCountIsStrict && colocationId != INVALID_COLOCATION_ID) { Oid colocatedTableId = ColocatedTableId(colocationId); if (colocatedTableId != InvalidOid) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(colocatedTableId); int colocatedTableShardCount = cacheEntry->shardIntervalArrayLength; if (colocatedTableShardCount != shardCount) { colocationId = INVALID_COLOCATION_ID; } } } } else if (!IsColocateWithNone(colocateWithTableName)) { text *colocateWithTableNameText = cstring_to_text(colocateWithTableName); Oid sourceRelationId = ResolveRelationId(colocateWithTableNameText, false); EnsureTableCanBeColocatedWith(relationId, replicationModel, distributionColumn, sourceRelationId); colocationId = TableColocationId(sourceRelationId); } return colocationId; } /* * EnsureTableCanBeColocatedWith checks whether a given replication model and * distribution column type is suitable to distribute a table to be colocated * with given source table. * * We only pass relationId to provide meaningful error messages. */ void EnsureTableCanBeColocatedWith(Oid relationId, char replicationModel, Var *distributionColumn, Oid sourceRelationId) { CitusTableCacheEntry *sourceTableEntry = GetCitusTableCacheEntry(sourceRelationId); if (IsCitusTableTypeCacheEntry(sourceTableEntry, APPEND_DISTRIBUTED) || IsCitusTableTypeCacheEntry(sourceTableEntry, RANGE_DISTRIBUTED) || IsCitusTableTypeCacheEntry(sourceTableEntry, CITUS_LOCAL_TABLE)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot distribute relation"), errdetail("Currently, colocate_with option is not supported " "with append / range distributed tables and local " "tables added to metadata."))); } char sourceReplicationModel = sourceTableEntry->replicationModel; if (sourceReplicationModel != replicationModel) { char *relationName = get_rel_name(relationId); char *sourceRelationName = get_rel_name(sourceRelationId); ereport(ERROR, (errmsg("cannot colocate tables %s and %s", sourceRelationName, relationName), errdetail("Replication models don't match for %s and %s.", sourceRelationName, relationName))); } Var *sourceDistributionColumn = DistPartitionKey(sourceRelationId); EnsureColumnTypeEquality(sourceRelationId, relationId, sourceDistributionColumn, distributionColumn); /* prevent colocating regular tables with tenant tables */ Oid sourceRelationSchemaId = get_rel_namespace(sourceRelationId); Oid targetRelationSchemaId = get_rel_namespace(relationId); if (IsTenantSchema(sourceRelationSchemaId) && sourceRelationSchemaId != targetRelationSchemaId) { char *relationName = get_rel_name(relationId); char *sourceRelationName = get_rel_name(sourceRelationId); char *sourceRelationSchemaName = get_namespace_name(sourceRelationSchemaId); ereport(ERROR, (errmsg("cannot colocate tables %s and %s", sourceRelationName, relationName), errdetail("Cannot colocate tables with distributed schema tables" " by using colocate_with option."), errhint("Consider using \"CREATE TABLE\" statement " "to create this table as a single-shard distributed " "table in the same schema to automatically colocate " "it with %s.%s", sourceRelationSchemaName, sourceRelationName))); } } ================================================ FILE: src/backend/distributed/utils/directory.c ================================================ /*------------------------------------------------------------------------- * * directory.c * * Utility functions for dealing with directories. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "distributed/utils/directory.h" /* Local functions forward declarations */ static bool FileIsLink(const char *filename, struct stat filestat); /* * CitusCreateDirectory creates a new directory with the given directory name. */ void CitusCreateDirectory(StringInfo directoryName) { int makeOK = MakePGDirectory(directoryName->data); if (makeOK != 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not create directory \"%s\": %m", directoryName->data))); } } /* * FileIsLink checks whether a file is a symbolic link. */ static bool FileIsLink(const char *filename, struct stat filestat) { return S_ISLNK(filestat.st_mode); } /* * CitusRemoveDirectory first checks if the given directory exists. If it does, the * function recursively deletes the contents of the given directory, and then * deletes the directory itself. This function is modeled on the Boost file * system library's remove_all() method. */ void CitusRemoveDirectory(const char *filename) { /* files may be added during execution, loop when that occurs */ while (true) { struct stat fileStat; int removed = 0; int statOK = stat(filename, &fileStat); if (statOK < 0) { if (errno == ENOENT) { return; /* if file does not exist, return */ } else { ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", filename))); } } /* * If this is a directory, iterate over all its contents and for each * content, recurse into this function. Also, make sure that we do not * recurse into symbolic links. */ if (S_ISDIR(fileStat.st_mode) && !FileIsLink(filename, fileStat)) { const char *directoryName = filename; DIR *directory = AllocateDir(directoryName); if (directory == NULL) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not open directory \"%s\": %m", directoryName))); } StringInfo fullFilename = makeStringInfo(); struct dirent *directoryEntry = ReadDir(directory, directoryName); for (; directoryEntry != NULL; directoryEntry = ReadDir(directory, directoryName)) { const char *baseFilename = directoryEntry->d_name; /* if system file, skip it */ if (strncmp(baseFilename, ".", MAXPGPATH) == 0 || strncmp(baseFilename, "..", MAXPGPATH) == 0) { continue; } resetStringInfo(fullFilename); appendStringInfo(fullFilename, "%s/%s", directoryName, baseFilename); CitusRemoveDirectory(fullFilename->data); } pfree(fullFilename->data); pfree(fullFilename); FreeDir(directory); } /* we now have an empty directory or a regular file, remove it */ if (S_ISDIR(fileStat.st_mode)) { /* * We ignore the TOCTUO race condition static analysis warning * here, since we don't actually read the files or directories. We * simply want to remove them. */ removed = rmdir(filename); /* lgtm[cpp/toctou-race-condition] */ if (errno == ENOTEMPTY || errno == EEXIST) { continue; } } else { /* * We ignore the TOCTUO race condition static analysis warning * here, since we don't actually read the files or directories. We * simply want to remove them. */ removed = unlink(filename); /* lgtm[cpp/toctou-race-condition] */ } if (removed != 0 && errno != ENOENT) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not remove file \"%s\": %m", filename))); } return; } } /* * CleanupJobCacheDirectory cleans up all files in the job cache directory * as part of this process's start-up logic. */ void CleanupJobCacheDirectory(void) { /* use the default tablespace in {datadir}/base */ StringInfo jobCacheDirectory = makeStringInfo(); appendStringInfo(jobCacheDirectory, "base/%s", PG_JOB_CACHE_DIR); CitusRemoveDirectory(jobCacheDirectory->data); CitusCreateDirectory(jobCacheDirectory); pfree(jobCacheDirectory->data); pfree(jobCacheDirectory); } ================================================ FILE: src/backend/distributed/utils/distribution_column.c ================================================ /*------------------------------------------------------------------------- * * distribution_column.c * * This file contains functions for translating distribution columns in * metadata tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/attnum.h" #include "access/heapam.h" #include "access/htup_details.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "nodes/primnodes.h" #include "parser/parse_relation.h" #include "parser/scansup.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/errcodes.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "distributed/distribution_column.h" #include "distributed/metadata_cache.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/version_compat.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(column_name_to_column); PG_FUNCTION_INFO_V1(column_name_to_column_id); PG_FUNCTION_INFO_V1(column_to_column_name); /* * column_name_to_column is an internal UDF to obtain a textual representation * of a particular column node (Var), given a relation identifier and column * name. There is no requirement that the table be distributed; this function * simply returns the textual representation of a Var representing a column. * This function will raise an ERROR if no such column can be found or if the * provided name refers to a system column. */ Datum column_name_to_column(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *columnText = PG_GETARG_TEXT_P(1); char *columnName = text_to_cstring(columnText); Var *column = BuildDistributionKeyFromColumnName(relationId, columnName, AccessShareLock); Assert(column != NULL); char *columnNodeString = nodeToString(column); text *columnNodeText = cstring_to_text(columnNodeString); PG_RETURN_TEXT_P(columnNodeText); } /* * column_name_to_column_id takes a relation identifier and a name of a column * in that relation and returns the index of that column in the relation. If * the provided name is a system column or no column at all, this function will * throw an error instead. */ Datum column_name_to_column_id(PG_FUNCTION_ARGS) { Oid distributedTableId = PG_GETARG_OID(0); char *columnName = PG_GETARG_CSTRING(1); Var *column = BuildDistributionKeyFromColumnName(distributedTableId, columnName, AccessExclusiveLock); Assert(column != NULL); PG_RETURN_INT16((int16) column->varattno); } /* * column_to_column_name is an internal UDF to obtain the human-readable name * of a column given a relation identifier and the column's internal textual * (Var) representation. This function will raise an ERROR if no such column * can be found or if the provided Var refers to a system column. */ Datum column_to_column_name(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); text *columnNodeText = PG_GETARG_TEXT_P(1); char *columnNodeString = text_to_cstring(columnNodeText); Node *columnNode = stringToNode(columnNodeString); char *columnName = ColumnToColumnName(relationId, columnNode); text *columnText = cstring_to_text(columnName); PG_RETURN_TEXT_P(columnText); } /* * BuildDistributionKeyFromColumnName builds a simple distribution key consisting * only out of a reference to the column of name columnName. Errors out if the * specified column does not exist or is not suitable to be used as a * distribution column. * * The function returns NULL if the passed column name is NULL. That case only * corresponds to reference tables. */ Var * BuildDistributionKeyFromColumnName(Oid relationId, char *columnName, LOCKMODE lockMode) { Relation relation = try_relation_open(relationId, lockMode); if (relation == NULL) { ereport(ERROR, (errmsg("relation does not exist"))); } relation_close(relation, NoLock); char *tableName = get_rel_name(relationId); /* short circuit for reference tables and single-shard tables */ if (columnName == NULL) { return NULL; } /* it'd probably better to downcase identifiers consistent with SQL case folding */ truncate_identifier(columnName, strlen(columnName), true); /* lookup column definition */ HeapTuple columnTuple = SearchSysCacheAttName(relationId, columnName); if (!HeapTupleIsValid(columnTuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of relation \"%s\" does not exist", columnName, tableName))); } Form_pg_attribute columnForm = (Form_pg_attribute) GETSTRUCT(columnTuple); /* check if the column may be referenced in the distribution key */ if (columnForm->attnum <= 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reference system column \"%s\" in relation \"%s\"", columnName, tableName))); } /* build Var referencing only the chosen distribution column */ Var *distributionColumn = makeVar(1, columnForm->attnum, columnForm->atttypid, columnForm->atttypmod, columnForm->attcollation, 0); ReleaseSysCache(columnTuple); return distributionColumn; } /* * EnsureValidDistributionColumn Errors out if the * specified column does not exist or is not suitable to be used as a * distribution column. It does not hold locks. */ void EnsureValidDistributionColumn(Oid relationId, char *columnName) { Relation relation = try_relation_open(relationId, AccessShareLock); if (relation == NULL) { ereport(ERROR, (errmsg("relation does not exist"))); } char *tableName = get_rel_name(relationId); /* it'd probably better to downcase identifiers consistent with SQL case folding */ truncate_identifier(columnName, strlen(columnName), true); /* lookup column definition */ HeapTuple columnTuple = SearchSysCacheAttName(relationId, columnName); if (!HeapTupleIsValid(columnTuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of relation \"%s\" does not exist", columnName, tableName))); } Form_pg_attribute columnForm = (Form_pg_attribute) GETSTRUCT(columnTuple); /* check if the column may be referenced in the distribution key */ if (columnForm->attnum <= 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reference system column \"%s\" in relation \"%s\"", columnName, tableName))); } ReleaseSysCache(columnTuple); relation_close(relation, AccessShareLock); } /* * ColumnTypeIdForRelationColumnName returns type id for the given relation's column name. */ Oid ColumnTypeIdForRelationColumnName(Oid relationId, char *columnName) { Assert(columnName != NULL); AttrNumber attNum = get_attnum(relationId, columnName); if (attNum == InvalidAttrNumber) { ereport(ERROR, (errmsg("invalid attr %s", columnName))); } Relation relation = relation_open(relationId, AccessShareLock); Oid typeId = attnumTypeId(relation, attNum); relation_close(relation, AccessShareLock); return typeId; } /* * ColumnToColumnName returns the human-readable name of a column given a * relation identifier and the column's internal (Var) representation. * This function will raise an ERROR if no such column can be found or if the * provided Var refers to a system column. */ char * ColumnToColumnName(Oid relationId, Node *columnNode) { if (columnNode == NULL || !IsA(columnNode, Var)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("not a valid column"))); } Var *column = (Var *) columnNode; AttrNumber columnNumber = column->varattno; if (!AttrNumberIsForUserDefinedAttr(columnNumber)) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), errmsg("attribute %d of relation \"%s\" is a system column", columnNumber, relationName))); } char *columnName = get_attname(relationId, column->varattno, false); if (columnName == NULL) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("attribute %d of relation \"%s\" does not exist", columnNumber, relationName))); } return columnName; } ================================================ FILE: src/backend/distributed/utils/distribution_column_map.c ================================================ /*------------------------------------------------------------------------- * * distribution_column_map.c * Implementation of a relation OID to distribution column map. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/hashfn.h" #include "nodes/primnodes.h" #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/utils/distribution_column_map.h" /* * RelationIdDistributionColumnMapEntry is used to map relation IDs to * distribution column Vars. */ typedef struct RelationIdDistributionColumnMapEntry { /* OID of the relation */ Oid relationId; /* a Var describing the distribution column */ Var *distributionColumn; } RelationIdDistributionColumnMapEntry; /* * CreateDistributionColumnMap creates an empty (OID -> distribution column Var) map. */ DistributionColumnMap * CreateDistributionColumnMap(void) { HASHCTL info = { 0 }; info.keysize = sizeof(Oid); info.entrysize = sizeof(RelationIdDistributionColumnMapEntry); info.hash = oid_hash; info.hcxt = CurrentMemoryContext; uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); HTAB *distributionColumnMap = hash_create("Distribution Column Map", 32, &info, hashFlags); return distributionColumnMap; } /* * AddDistributionColumnForRelation adds the given OID and its distribution column * to the hash, as well as any child partitions. */ void AddDistributionColumnForRelation(DistributionColumnMap *distributionColumnMap, Oid relationId, char *distributionColumnName) { bool entryFound = false; RelationIdDistributionColumnMapEntry *entry = hash_search(distributionColumnMap, &relationId, HASH_ENTER, &entryFound); Assert(!entryFound); entry->distributionColumn = BuildDistributionKeyFromColumnName(relationId, distributionColumnName, NoLock); if (PartitionedTable(relationId)) { /* * Recursively add partitions as well. */ List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { AddDistributionColumnForRelation(distributionColumnMap, partitionRelationId, distributionColumnName); } } } /* * GetDistributionColumnFromMap returns the distribution column for a given * relation ID from the distribution column map. */ Var * GetDistributionColumnFromMap(DistributionColumnMap *distributionColumnMap, Oid relationId) { bool entryFound = false; RelationIdDistributionColumnMapEntry *entry = hash_search(distributionColumnMap, &relationId, HASH_FIND, &entryFound); if (entryFound) { return entry->distributionColumn; } else { return NULL; } } /* * GetDistributionColumnWithOverrides returns the distribution column for a given * relation from the distribution column overrides map, or the metadata if no * override is specified. */ Var * GetDistributionColumnWithOverrides(Oid relationId, DistributionColumnMap *distributionColumnOverrides) { Var *distributionColumn = NULL; if (distributionColumnOverrides != NULL) { distributionColumn = GetDistributionColumnFromMap(distributionColumnOverrides, relationId); if (distributionColumn != NULL) { return distributionColumn; } } /* no override defined, use distribution column from metadata */ return DistPartitionKey(relationId); } ================================================ FILE: src/backend/distributed/utils/enable_ssl.c ================================================ /*------------------------------------------------------------------------- * * enable_ssl.c * UDF and Utilities for enabling ssl during citus setup * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" /* * Make sure that functions marked as deprecated in OpenSSL 3.0 don't trigger * deprecation warnings by indicating that we're using the OpenSSL 1.0.1 * compatibile API. Postgres does this by already in PG14, so we should not do * it otherwise we get warnings about redefining this value. This needs to be * done before including libpq.h. */ #include "miscadmin.h" #include "libpq/libpq.h" #include "nodes/parsenodes.h" #include "postmaster/postmaster.h" #include "utils/guc.h" #include "pg_version_constants.h" #include "distributed/connection_management.h" #include "distributed/memutils.h" #include "distributed/worker_protocol.h" #ifdef USE_OPENSSL #include "openssl/dsa.h" #include "openssl/err.h" #include "openssl/pem.h" #include "openssl/rsa.h" #include "openssl/ssl.h" #include "openssl/x509.h" #endif #define ENABLE_SSL_QUERY "ALTER SYSTEM SET ssl TO on;" #define RESET_CITUS_NODE_CONNINFO \ "ALTER SYSTEM SET citus.node_conninfo TO 'sslmode=prefer';" #define CITUS_AUTO_SSL_COMMON_NAME "citus-auto-ssl" #define X509_SUBJECT_COMMON_NAME "CN" #define POSTGRES_DEFAULT_SSL_CIPHERS "HIGH:MEDIUM:+3DES:!aNULL" /* * Microsoft approved cipher string. * This cipher string implicitely enables only TLSv1.2+, because these ciphers * were all added in TLSv1.2. This can be confirmed by running: * openssl -v */ #define CITUS_DEFAULT_SSL_CIPHERS "ECDHE-ECDSA-AES128-GCM-SHA256:" \ "ECDHE-ECDSA-AES256-GCM-SHA384:" \ "ECDHE-RSA-AES128-GCM-SHA256:" \ "ECDHE-RSA-AES256-GCM-SHA384:" \ "ECDHE-ECDSA-AES128-SHA256:" \ "ECDHE-ECDSA-AES256-SHA384:" \ "ECDHE-RSA-AES128-SHA256:" \ "ECDHE-RSA-AES256-SHA384" #define SET_CITUS_SSL_CIPHERS_QUERY \ "ALTER SYSTEM SET ssl_ciphers TO '" CITUS_DEFAULT_SSL_CIPHERS "';" /* forward declaration of helper functions */ static void GloballyReloadConfig(void); #ifdef USE_SSL /* forward declaration of functions used when compiled with ssl */ static bool ShouldUseAutoSSL(void); static bool CreateCertificatesWhenNeeded(void); static EVP_PKEY * GeneratePrivateKey(void); static X509 * CreateCertificate(EVP_PKEY *privateKey); static bool StoreCertificate(EVP_PKEY *privateKey, X509 *certificate); #endif /* USE_SSL */ PG_FUNCTION_INFO_V1(citus_setup_ssl); PG_FUNCTION_INFO_V1(citus_check_defaults_for_sslmode); /* * citus_setup_ssl is called during the first creation of a citus extension. It configures * postgres to use ssl if not already on. During this process it will create certificates * if they are not already installed in the configured location. */ Datum citus_setup_ssl(PG_FUNCTION_ARGS) { #ifndef USE_SSL ereport(WARNING, (errmsg("can not setup ssl on postgres that is not compiled with " "ssl support"))); #else /* USE_SSL */ if (!EnableSSL && ShouldUseAutoSSL()) { ereport(LOG, (errmsg("citus extension created on postgres without ssl enabled, " "turning it on during creation of the extension"))); /* execute the alter system statement to enable ssl on within postgres */ Node *enableSSLParseTree = ParseTreeNode(ENABLE_SSL_QUERY); AlterSystemSetConfigFile((AlterSystemStmt *) enableSSLParseTree); if (strcmp(SSLCipherSuites, POSTGRES_DEFAULT_SSL_CIPHERS) == 0) { /* * postgres default cipher suite is configured, these allow TSL 1 and TLS 1.1, * citus will upgrade to TLS1.2+HIGH and above. */ Node *citusSSLCiphersParseTree = ParseTreeNode(SET_CITUS_SSL_CIPHERS_QUERY); AlterSystemSetConfigFile((AlterSystemStmt *) citusSSLCiphersParseTree); } /* * ssl=on requires that a key and certificate are present, since we have * enabled ssl mode here chances are the user didn't install credentials already. * * This function will check if they are available and if not it will generate a * self singed certificate. */ CreateCertificatesWhenNeeded(); GloballyReloadConfig(); } #endif /* USE_SSL */ PG_RETURN_NULL(); } /* * citus_check_defaults_for_sslmode is called in the extension upgrade path when * users upgrade from a previous version to a version that has ssl enabled by default, and * only when the changed default value conflicts with the setup of the user. * * Once it is determined that the default value for citus.node_conninfo is used verbatim * with ssl not enabled on the cluster it will reinstate the old default value for * citus.node_conninfo. * * In effect this is to not impose the overhead of ssl on an already existing cluster that * didn't have it enabled already. */ Datum citus_check_defaults_for_sslmode(PG_FUNCTION_ARGS) { bool configChanged = false; if (EnableSSL) { /* since ssl is on we do not have to change any sslmode back to prefer */ PG_RETURN_NULL(); } /* * test if the node_conninfo setting is exactly set to the default value used when * Citus started to enable SSL. This is to make sure upgrades restores the previous * value so users will not have unexpected changes during upgrades. */ if (strcmp(NodeConninfo, "sslmode=require") == 0) { /* execute the alter system statement to reset node_conninfo to the old default */ ereport(LOG, (errmsg("reset citus.node_conninfo to old default value as the new " "value is incompatible with the current ssl setting"))); Node *resetCitusNodeConnInfoParseTree = ParseTreeNode(RESET_CITUS_NODE_CONNINFO); AlterSystemSetConfigFile((AlterSystemStmt *) resetCitusNodeConnInfoParseTree); configChanged = true; } if (configChanged) { GloballyReloadConfig(); } PG_RETURN_NULL(); } /* * GloballyReloadConfig signals postmaster to reload the configuration as well as * reloading the configuration in the current backend. By reloading the configuration in * the current backend the changes will also be reflected in the current transaction. */ static void GloballyReloadConfig() { if (kill(PostmasterPid, SIGHUP)) { ereport(WARNING, (errmsg("failed to send signal to postmaster: %m"))); } ProcessConfigFile(PGC_SIGHUP); } #ifdef USE_SSL /* * ShouldUseAutoSSL checks if citus should enable ssl based on the connection settings it * uses for outward connections. When the outward connection is configured to require ssl * it assumes the other nodes in the network have the same setting and therefore it will * automatically enable ssl during installation. */ static bool ShouldUseAutoSSL(void) { const char *sslmode = NULL; sslmode = GetConnParam("sslmode"); if (sslmode != NULL && strcmp(sslmode, "require") == 0) { return true; } return false; } /* * CreateCertificatesWhenNeeded checks if the certificates exists. When they don't exist * they will be created. The return value tells whether or not new certificates have been * created. After this function it is guaranteed that certificates are in place. It is not * guaranteed they have the right permissions as we will not touch the keys if they exist. */ static bool CreateCertificatesWhenNeeded() { EVP_PKEY *privateKey = NULL; X509 *certificate = NULL; bool certificateWritten = false; SSL_CTX *sslContext = NULL; /* * Ensure the OpenSSL library is initialized so we can create our SSL context. * On OpenSSL ≥ 1.1.0 we call OPENSSL_init_ssl() (which also loads the default * config), and on older versions we fall back to SSL_library_init(). * PostgreSQL itself will perform its full SSL setup when it reloads * its configuration with ssl enabled. */ #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x10100000L /* OpenSSL 1.1.0+ */ OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); #else /* OpenSSL < 1.1.0 */ SSL_library_init(); #endif #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x10100000L sslContext = SSL_CTX_new(TLS_method()); #else sslContext = SSL_CTX_new(SSLv23_method()); #endif if (!sslContext) { ereport(WARNING, (errmsg("unable to create ssl context, please verify ssl " "settings for postgres"), errdetail("Citus could not create the ssl context to verify " "the ssl settings for postgres and possibly setup " "certificates. Since Citus requires connections " "between nodes to use ssl communication between " "nodes might return an error until ssl is setup " "correctly."))); return false; } EnsureReleaseResource((MemoryContextCallbackFunction) (&SSL_CTX_free), sslContext); /* * check if we can load the certificate, when we can we assume the certificates are in * place. No need to create the certificates and we can exit the function. * * This also makes the whole ssl enabling idempotent as writing the certificate is the * last step. */ if (SSL_CTX_use_certificate_chain_file(sslContext, ssl_cert_file) == 1) { return false; } ereport(LOG, (errmsg("no certificate present, generating self signed certificate"))); privateKey = GeneratePrivateKey(); if (!privateKey) { ereport(ERROR, (errmsg("error while generating private key"))); } certificate = CreateCertificate(privateKey); if (!certificate) { ereport(ERROR, (errmsg("error while generating certificate"))); } certificateWritten = StoreCertificate(privateKey, certificate); if (!certificateWritten) { ereport(ERROR, (errmsg("error while storing key and certificate"))); } return true; } /* * GeneratePrivateKey uses open ssl functions to generate an RSA private key of 2048 bits. * All OpenSSL resources created during the process are added to the memory context active * when the function is called and therefore should not be freed by the caller. */ static EVP_PKEY * GeneratePrivateKey() { /* Allocate memory for the EVP_PKEY structure. */ EVP_PKEY *privateKey = EVP_PKEY_new(); if (!privateKey) { ereport(ERROR, (errmsg("unable to allocate space for private key"))); } EnsureReleaseResource((MemoryContextCallbackFunction) (&EVP_PKEY_free), privateKey); BIGNUM *exponent = BN_new(); EnsureReleaseResource((MemoryContextCallbackFunction) (&BN_free), exponent); /* load the exponent to use for the generation of the key */ int success = BN_set_word(exponent, RSA_F4); if (success != 1) { ereport(ERROR, (errmsg("unable to prepare exponent for RSA algorithm"))); } RSA *rsa = RSA_new(); success = RSA_generate_key_ex(rsa, 2048, exponent, NULL); if (success != 1) { ereport(ERROR, (errmsg("unable to generate RSA key"))); } if (!EVP_PKEY_assign_RSA(privateKey, rsa)) { ereport(ERROR, (errmsg("unable to assign RSA key to use as private key"))); } /* The key has been generated, return it. */ return privateKey; } /* * CreateCertificate creates a self signed certificate for citus to use. The certificate * will contain the public parts of the private key and will be signed in the end by the * private part to make it self signed. */ static X509 * CreateCertificate(EVP_PKEY *privateKey) { X509 *certificate = X509_new(); if (!certificate) { ereport(ERROR, (errmsg("unable to allocate space for the x509 certificate"))); } EnsureReleaseResource((MemoryContextCallbackFunction) (&X509_free), certificate); /* Set the serial number. */ ASN1_INTEGER_set(X509_get_serialNumber(certificate), 1); /* * Set the expiry of the certificate. * * the functions X509_get_notBefore and X509_get_notAfter are deprecated, these are * replaced with mutable and non-mutable variants in openssl 1.1, however they are * better supported than the newer versions. In 1.1 they are aliasses to the mutable * variant (X509_getm_notBefore, ...) that we actually need, so they will actually use * the correct function in newer versions. * * Postgres does not check the validity on the certificates, but we can't omit the * dates either to create a certificate that can be parsed. We settled on a validity * of 0 seconds. When postgres would fix the validity check in a future version it * would fail right after an upgrade. Instead of working until the certificate * expiration date and then suddenly erroring out. */ #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x10100000L /* New mutable accessors (present in 1.1, 3.x). */ X509_gmtime_adj(X509_getm_notBefore(certificate), 0); X509_gmtime_adj(X509_getm_notAfter(certificate), 0); #else /* Legacy functions kept for 1.0.x compatibility. */ X509_gmtime_adj(X509_get_notBefore(certificate), 0); X509_gmtime_adj(X509_get_notAfter(certificate), 0); #endif /* Set the public key for our certificate */ X509_set_pubkey(certificate, privateKey); /* Set the common name for the certificate */ X509_NAME *subjectName = X509_get_subject_name(certificate); X509_NAME_add_entry_by_txt(subjectName, X509_SUBJECT_COMMON_NAME, MBSTRING_ASC, (unsigned char *) CITUS_AUTO_SSL_COMMON_NAME, -1, -1, 0); /* For a self signed certificate we set the isser name to our own name */ X509_set_issuer_name(certificate, subjectName); /* With all information filled out we sign the certificate with our own key */ if (!X509_sign(certificate, privateKey, EVP_sha256())) { ereport(ERROR, (errmsg("unable to create signature for the x509 certificate"))); } return certificate; } /* * StoreCertificate stores both the private key and its certificate to the files * configured in postgres. */ static bool StoreCertificate(EVP_PKEY *privateKey, X509 *certificate) { const char *privateKeyFilename = ssl_key_file; const char *certificateFilename = ssl_cert_file; /* Open the private key file and write the private key in PEM format to it */ int privateKeyFileDescriptor = open(privateKeyFilename, O_WRONLY | O_CREAT, 0600); if (privateKeyFileDescriptor == -1) { ereport(ERROR, (errmsg("unable to open private key file '%s' for writing", privateKeyFilename))); } FILE *privateKeyFile = fdopen(privateKeyFileDescriptor, "wb"); if (!privateKeyFile) { ereport(ERROR, (errmsg("unable to open private key file '%s' for writing", privateKeyFilename))); } int success = PEM_write_PrivateKey(privateKeyFile, privateKey, NULL, NULL, 0, NULL, NULL); fclose(privateKeyFile); if (!success) { ereport(ERROR, (errmsg("unable to store private key"))); } int certificateFileDescriptor = open(certificateFilename, O_WRONLY | O_CREAT, 0600); if (certificateFileDescriptor == -1) { ereport(ERROR, (errmsg("unable to open private key file '%s' for writing", privateKeyFilename))); } /* Open the certificate file and write the certificate in the PEM format to it */ FILE *certificateFile = fdopen(certificateFileDescriptor, "wb"); if (!certificateFile) { ereport(ERROR, (errmsg("unable to open certificate file '%s' for writing", certificateFilename))); } success = PEM_write_X509(certificateFile, certificate); fclose(certificateFile); if (!success) { ereport(ERROR, (errmsg("unable to store certificate"))); } return true; } #endif /* USE_SSL */ ================================================ FILE: src/backend/distributed/utils/errormessage.c ================================================ /* * errormessage.c * Error handling related support functionality. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "common/sha2.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" /* * DeferredErrorInternal is a helper function for DeferredError(). */ DeferredErrorMessage * DeferredErrorInternal(int code, const char *message, const char *detail, const char *hint, const char *filename, int linenumber, const char *functionname) { DeferredErrorMessage *error = CitusMakeNode(DeferredErrorMessage); Assert(message != NULL); error->code = code; error->message = message; error->detail = detail; error->hint = hint; error->filename = filename; error->linenumber = linenumber; error->functionname = functionname; return error; } /* * RaiseDeferredErrorInternal is a helper function for RaiseDeferredError(). */ void RaiseDeferredErrorInternal(DeferredErrorMessage *error, int elevel) { ErrorData *errorData = palloc0(sizeof(ErrorData)); errorData->sqlerrcode = error->code; errorData->elevel = elevel; errorData->message = pstrdup(error->message); if (error->detail) { errorData->detail = pstrdup(error->detail); } if (error->hint) { errorData->hint = pstrdup(error->hint); } errorData->filename = pstrdup(error->filename); errorData->lineno = error->linenumber; errorData->funcname = error->functionname; errorData->assoc_context = ErrorContext; ThrowErrorData(errorData); } ================================================ FILE: src/backend/distributed/utils/foreign_key_relationship.c ================================================ /*------------------------------------------------------------------------- * * foreign_key_relationship.c * This file contains functions for creating foreign key relationship graph * between distributed tables. Created relationship graph will be hold by * a static variable defined in this file until an invalidation comes in. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/htup_details.h" #include "access/stratnum.h" #include "access/table.h" #include "catalog/pg_constraint.h" #include "common/hashfn.h" #include "nodes/pg_list.h" #include "storage/lockdefs.h" #include "utils/catcache.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/inval.h" #include "utils/memutils.h" #include "pg_version_constants.h" #include "distributed/commands.h" #include "distributed/foreign_key_relationship.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/version_compat.h" /* * ForeignConstraintRelationshipGraph holds the graph data structure for foreign constraint relationship * between relations. We will only have single static instance of that struct and it * will be invalidated after change on any foreign constraint. */ typedef struct ForeignConstraintRelationshipGraph { HTAB *nodeMap; bool isValid; }ForeignConstraintRelationshipGraph; /* * ForeignConstraintRelationshipNode holds the data for each node of the ForeignConstraintRelationshipGraph * For each node we have relation id, which is the Oid of that relation, visiting * information for that node in the latest DFS and the list of adjacency nodes. * Note that we also hold back adjacency nodes for getting referenced node over * that one. */ typedef struct ForeignConstraintRelationshipNode { Oid relationId; List *adjacencyList; List *backAdjacencyList; }ForeignConstraintRelationshipNode; /* * ForeignConstraintRelationshipEdge will only be used while creating the ForeignConstraintRelationshipGraph. * It won't show edge information on the graph, yet will be used in the pre-processing * phase. */ typedef struct ForeignConstraintRelationshipEdge { Oid referencingRelationOID; Oid referencedRelationOID; }ForeignConstraintRelationshipEdge; static ForeignConstraintRelationshipGraph *fConstraintRelationshipGraph = NULL; static List * GetRelationshipNodesForFKeyConnectedRelations( ForeignConstraintRelationshipNode *relationshipNode); static List * GetAllNeighboursList(ForeignConstraintRelationshipNode *relationshipNode); static ForeignConstraintRelationshipNode * GetRelationshipNodeForRelationId(Oid relationId, bool * isFound); static void CreateForeignConstraintRelationshipGraph(void); static bool IsForeignConstraintRelationshipGraphValid(void); static List * GetNeighbourList(ForeignConstraintRelationshipNode *relationshipNode, bool isReferencing); static List * GetRelationIdsFromRelationshipNodeList(List *fKeyRelationshipNodeList); static void PopulateAdjacencyLists(void); static int CompareForeignConstraintRelationshipEdges(const void *leftElement, const void *rightElement); static void AddForeignConstraintRelationshipEdge(Oid referencingOid, Oid referencedOid); static ForeignConstraintRelationshipNode * CreateOrFindNode(HTAB *adjacencyLists, Oid relid); static List * GetConnectedListHelper(ForeignConstraintRelationshipNode *node, bool isReferencing); static List * GetForeignConstraintRelationshipHelper(Oid relationId, bool isReferencing); MemoryContext ForeignConstraintRelationshipMemoryContext = NULL; /* * GetForeignKeyConnectedRelationIdList returns a list of relation id's for * relations that are connected to relation with relationId via a foreign * key graph. */ List * GetForeignKeyConnectedRelationIdList(Oid relationId) { /* use ShareRowExclusiveLock to prevent concurent foreign key creation */ LOCKMODE lockMode = ShareRowExclusiveLock; Relation relation = try_relation_open(relationId, lockMode); if (!RelationIsValid(relation)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relation with OID %d does not exist", relationId))); } relation_close(relation, NoLock); bool foundInFKeyGraph = false; ForeignConstraintRelationshipNode *relationshipNode = GetRelationshipNodeForRelationId(relationId, &foundInFKeyGraph); if (!foundInFKeyGraph) { /* * Relation could not be found in foreign key graph, then it has no * foreign key relationships. */ return NIL; } List *fKeyConnectedRelationshipNodeList = GetRelationshipNodesForFKeyConnectedRelations(relationshipNode); List *fKeyConnectedRelationIdList = GetRelationIdsFromRelationshipNodeList(fKeyConnectedRelationshipNodeList); return fKeyConnectedRelationIdList; } /* * ShouldUndistributeCitusLocalTable returns true if given relationId needs * to be undistributed. Here we do not undistribute table if it's converted by the user, * or connected to a table converted by the user, or a reference table, via foreign keys. */ bool ShouldUndistributeCitusLocalTable(Oid relationId) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); if (!cacheEntry->autoConverted) { /* * The relation is not added to metadata automatically, * we shouldn't undistribute it. */ return false; } /* * As we will operate on foreign key connected relations, here we * invalidate foreign key graph so that we act on fresh graph. */ InvalidateForeignKeyGraph(); List *fkeyConnectedRelations = GetForeignKeyConnectedRelationIdList(relationId); return !RelationIdListHasReferenceTable(fkeyConnectedRelations); } /* * GetRelationshipNodesForFKeyConnectedRelations performs breadth-first search * starting from input ForeignConstraintRelationshipNode and returns a list * of ForeignConstraintRelationshipNode objects for relations that are connected * to given relation node via a foreign key relationhip graph. */ static List * GetRelationshipNodesForFKeyConnectedRelations(ForeignConstraintRelationshipNode * relationshipNode) { HTAB *oidVisitedMap = CreateSimpleHashSetWithName(Oid, "oid visited hash set"); VisitOid(oidVisitedMap, relationshipNode->relationId); List *relationshipNodeList = list_make1(relationshipNode); ForeignConstraintRelationshipNode *currentNode = NULL; foreach_ptr_append(currentNode, relationshipNodeList) { List *allNeighboursList = GetAllNeighboursList(currentNode); ForeignConstraintRelationshipNode *neighbourNode = NULL; foreach_declared_ptr(neighbourNode, allNeighboursList) { Oid neighbourRelationId = neighbourNode->relationId; if (OidVisited(oidVisitedMap, neighbourRelationId)) { continue; } VisitOid(oidVisitedMap, neighbourRelationId); relationshipNodeList = lappend(relationshipNodeList, neighbourNode); } } return relationshipNodeList; } /* * GetAllNeighboursList returns a list of ForeignConstraintRelationshipNode * objects by concatenating both (referencing & referenced) adjacency lists * of given relationship node. */ static List * GetAllNeighboursList(ForeignConstraintRelationshipNode *relationshipNode) { bool isReferencing = false; List *referencedNeighboursList = GetNeighbourList(relationshipNode, isReferencing); isReferencing = true; List *referencingNeighboursList = GetNeighbourList(relationshipNode, isReferencing); /* * GetNeighbourList returns list from graph as is, so first copy it as * list_concat might invalidate it. */ List *allNeighboursList = list_copy(referencedNeighboursList); allNeighboursList = list_concat_unique_ptr(allNeighboursList, referencingNeighboursList); return allNeighboursList; } /* * ReferencedRelationIdList is a wrapper function around GetForeignConstraintRelationshipHelper * to get list of relation IDs which are referenced by the given relation id. * * Note that, if relation A is referenced by relation B and relation B is referenced * by relation C, then the result list for relation A consists of the relation * IDs of relation B and relation C. */ List * ReferencedRelationIdList(Oid relationId) { return GetForeignConstraintRelationshipHelper(relationId, false); } /* * ReferencingRelationIdList is a wrapper function around GetForeignConstraintRelationshipHelper * to get list of relation IDs which are referencing to given relation id. * * Note that, if relation A is referenced by relation B and relation B is referenced * by relation C, then the result list for relation C consists of the relation * IDs of relation A and relation B. */ List * ReferencingRelationIdList(Oid relationId) { return GetForeignConstraintRelationshipHelper(relationId, true); } /* * GetForeignConstraintRelationshipHelper returns the list of oids referenced or * referencing given relation id. It is a helper function for providing results * to public functions ReferencedRelationIdList and ReferencingRelationIdList. */ static List * GetForeignConstraintRelationshipHelper(Oid relationId, bool isReferencing) { bool isFound = false; ForeignConstraintRelationshipNode *relationshipNode = GetRelationshipNodeForRelationId(relationId, &isFound); if (!isFound) { /* * If there is no node with the given relation id, that means given table * is not referencing and is not referenced by any table */ return NIL; } List *connectedNodeList = GetConnectedListHelper(relationshipNode, isReferencing); List *relationIdList = GetRelationIdsFromRelationshipNodeList(connectedNodeList); return relationIdList; } /* * GetRelationshipNodeForRelationId searches foreign key graph for relation * with relationId and returns ForeignConstraintRelationshipNode object for * relation if it exists in graph. Otherwise, sets isFound to false. * * Also before searching foreign key graph, this function implicitly builds * foreign key graph if it's invalid or not built yet. */ static ForeignConstraintRelationshipNode * GetRelationshipNodeForRelationId(Oid relationId, bool *isFound) { CreateForeignConstraintRelationshipGraph(); ForeignConstraintRelationshipNode *relationshipNode = (ForeignConstraintRelationshipNode *) hash_search( fConstraintRelationshipGraph->nodeMap, &relationId, HASH_FIND, isFound); return relationshipNode; } /* * CreateForeignConstraintRelationshipGraph creates the foreign constraint relation graph using * foreign constraint provided by pg_constraint metadata table. */ static void CreateForeignConstraintRelationshipGraph() { /* if we have already created the graph, use it */ if (IsForeignConstraintRelationshipGraphValid()) { return; } /* * Lazily create our memory context once and reset on every reuse. * Since we have cleared and invalidated the fConstraintRelationshipGraph, right * before we can simply reset the context if it was already existing. */ if (ForeignConstraintRelationshipMemoryContext == NULL) { /* make sure we've initialized CacheMemoryContext */ if (CacheMemoryContext == NULL) { CreateCacheMemoryContext(); } ForeignConstraintRelationshipMemoryContext = AllocSetContextCreate( CacheMemoryContext, "Foreign Constraint Relationship Graph Context", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); } else { fConstraintRelationshipGraph = NULL; MemoryContextReset(ForeignConstraintRelationshipMemoryContext); } Assert(fConstraintRelationshipGraph == NULL); MemoryContext oldContext = MemoryContextSwitchTo( ForeignConstraintRelationshipMemoryContext); fConstraintRelationshipGraph = (ForeignConstraintRelationshipGraph *) palloc( sizeof(ForeignConstraintRelationshipGraph)); fConstraintRelationshipGraph->isValid = false; fConstraintRelationshipGraph->nodeMap = CreateSimpleHash(Oid, ForeignConstraintRelationshipNode); PopulateAdjacencyLists(); fConstraintRelationshipGraph->isValid = true; MemoryContextSwitchTo(oldContext); } /* * IsForeignConstraintGraphValid check whether there is a valid graph. */ static bool IsForeignConstraintRelationshipGraphValid() { /* * We might have some concurrent metadata changes. In order to get the changes, * we first need to accept the cache invalidation messages. */ AcceptInvalidationMessages(); if (fConstraintRelationshipGraph != NULL && fConstraintRelationshipGraph->isValid) { return true; } return false; } /* * SetForeignConstraintGraphInvalid sets the validity of the graph to false. */ void SetForeignConstraintRelationshipGraphInvalid() { if (fConstraintRelationshipGraph != NULL) { fConstraintRelationshipGraph->isValid = false; } } /* * GetConnectedListHelper returns list of ForeignConstraintRelationshipNode * objects for relations referenced by or referencing to given relation * according to isReferencing flag. * */ static List * GetConnectedListHelper(ForeignConstraintRelationshipNode *node, bool isReferencing) { HTAB *oidVisitedMap = CreateSimpleHashSetWithName(Oid, "oid visited hash set"); List *connectedNodeList = NIL; List *relationshipNodeStack = list_make1(node); while (list_length(relationshipNodeStack) != 0) { /* * Note that this loop considers leftmost element of * relationshipNodeStack as top of the stack. */ /* pop top element from stack */ ForeignConstraintRelationshipNode *currentNode = linitial(relationshipNodeStack); relationshipNodeStack = list_delete_first(relationshipNodeStack); Oid currentRelationId = currentNode->relationId; if (!OidVisited(oidVisitedMap, currentRelationId)) { connectedNodeList = lappend(connectedNodeList, currentNode); VisitOid(oidVisitedMap, currentRelationId); } List *neighbourList = GetNeighbourList(currentNode, isReferencing); ForeignConstraintRelationshipNode *neighbourNode = NULL; foreach_declared_ptr(neighbourNode, neighbourList) { Oid neighbourRelationId = neighbourNode->relationId; if (!OidVisited(oidVisitedMap, neighbourRelationId)) { /* push to stack */ relationshipNodeStack = lcons(neighbourNode, relationshipNodeStack); } } } hash_destroy(oidVisitedMap); /* finally remove yourself from list */ connectedNodeList = list_delete_first(connectedNodeList); return connectedNodeList; } /* * OidVisited returns true if given oid is visited according to given oid hash-set. */ bool OidVisited(HTAB *oidVisitedMap, Oid oid) { bool found = false; hash_search(oidVisitedMap, &oid, HASH_FIND, &found); return found; } /* * VisitOid sets given oid as visited in given hash-set. */ void VisitOid(HTAB *oidVisitedMap, Oid oid) { bool found = false; hash_search(oidVisitedMap, &oid, HASH_ENTER, &found); } /* * GetNeighbourList returns copy of relevant adjacency list of given * ForeignConstraintRelationshipNode object depending on the isReferencing * flag. */ static List * GetNeighbourList(ForeignConstraintRelationshipNode *relationshipNode, bool isReferencing) { if (isReferencing) { return relationshipNode->backAdjacencyList; } else { return relationshipNode->adjacencyList; } } /* * GetRelationIdsFromRelationshipNodeList returns list of relationId's for * given ForeignConstraintRelationshipNode object list. */ static List * GetRelationIdsFromRelationshipNodeList(List *fKeyRelationshipNodeList) { List *relationIdList = NIL; ForeignConstraintRelationshipNode *fKeyRelationshipNode = NULL; foreach_declared_ptr(fKeyRelationshipNode, fKeyRelationshipNodeList) { Oid relationId = fKeyRelationshipNode->relationId; relationIdList = lappend_oid(relationIdList, relationId); } return relationIdList; } /* * PopulateAdjacencyLists gets foreign constraint relationship information from pg_constraint * metadata table and populates them to the foreign constraint relation graph. */ static void PopulateAdjacencyLists(void) { HeapTuple tuple; ScanKeyData scanKey[1]; int scanKeyCount = 1; Oid prevReferencingOid = InvalidOid; Oid prevReferencedOid = InvalidOid; List *frelEdgeList = NIL; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, InvalidOid, false, NULL, scanKeyCount, scanKey); while (HeapTupleIsValid(tuple = systable_getnext(scanDescriptor))) { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(tuple); ForeignConstraintRelationshipEdge *currentFConstraintRelationshipEdge = palloc( sizeof(ForeignConstraintRelationshipEdge)); currentFConstraintRelationshipEdge->referencingRelationOID = constraintForm->conrelid; currentFConstraintRelationshipEdge->referencedRelationOID = constraintForm->confrelid; frelEdgeList = lappend(frelEdgeList, currentFConstraintRelationshipEdge); } /* * Since there is no index on columns we are planning to sort tuples * sorting tuples manually instead of using scan keys */ frelEdgeList = SortList(frelEdgeList, CompareForeignConstraintRelationshipEdges); ForeignConstraintRelationshipEdge *currentFConstraintRelationshipEdge = NULL; foreach_declared_ptr(currentFConstraintRelationshipEdge, frelEdgeList) { /* we just saw this edge, no need to add it twice */ if (currentFConstraintRelationshipEdge->referencingRelationOID == prevReferencingOid && currentFConstraintRelationshipEdge->referencedRelationOID == prevReferencedOid ) { continue; } AddForeignConstraintRelationshipEdge( currentFConstraintRelationshipEdge->referencingRelationOID, currentFConstraintRelationshipEdge-> referencedRelationOID); prevReferencingOid = currentFConstraintRelationshipEdge->referencingRelationOID; prevReferencedOid = currentFConstraintRelationshipEdge->referencedRelationOID; } systable_endscan(scanDescriptor); table_close(pgConstraint, AccessShareLock); } /* * CompareForeignConstraintRelationshipEdges is a helper function to compare two * ForeignConstraintRelationshipEdge using referencing and referenced ids respectively. */ static int CompareForeignConstraintRelationshipEdges(const void *leftElement, const void *rightElement) { const ForeignConstraintRelationshipEdge *leftEdge = *((const ForeignConstraintRelationshipEdge **) leftElement); const ForeignConstraintRelationshipEdge *rightEdge = *((const ForeignConstraintRelationshipEdge **) rightElement); int referencingDiff = leftEdge->referencingRelationOID - rightEdge->referencingRelationOID; int referencedDiff = leftEdge->referencedRelationOID - rightEdge->referencedRelationOID; if (referencingDiff != 0) { return referencingDiff; } return referencedDiff; } /* * AddForeignConstraintRelationshipEdge adds edge between the nodes having given OIDs * by adding referenced node to the adjacency list referencing node and adding * referencing node to the back adjacency list of referenced node. */ static void AddForeignConstraintRelationshipEdge(Oid referencingOid, Oid referencedOid) { ForeignConstraintRelationshipNode *referencingNode = CreateOrFindNode( fConstraintRelationshipGraph->nodeMap, referencingOid); ForeignConstraintRelationshipNode *referencedNode = CreateOrFindNode( fConstraintRelationshipGraph->nodeMap, referencedOid); referencingNode->adjacencyList = lappend(referencingNode->adjacencyList, referencedNode); referencedNode->backAdjacencyList = lappend(referencedNode->backAdjacencyList, referencingNode); } /* * CreateOrFindNode either gets or adds new node to the foreign constraint relation graph */ static ForeignConstraintRelationshipNode * CreateOrFindNode(HTAB *adjacencyLists, Oid relid) { bool found = false; ForeignConstraintRelationshipNode *node = (ForeignConstraintRelationshipNode *) hash_search(adjacencyLists, &relid, HASH_ENTER, &found); if (!found) { node->adjacencyList = NIL; node->backAdjacencyList = NIL; } return node; } ================================================ FILE: src/backend/distributed/utils/function.c ================================================ /*------------------------------------------------------------------------- * * function.c * * Utility functions for dealing with functions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "miscadmin.h" #include "commands/defrem.h" #include "utils/lsyscache.h" #include "distributed/utils/function.h" /* * GetFunctionInfo first resolves the operator for the given data type, access * method, and support procedure. The function then uses the resolved operator's * identifier to fill in a function manager object, and returns this object. */ FmgrInfo * GetFunctionInfo(Oid typeId, Oid accessMethodId, int16 procedureId) { FmgrInfo *functionInfo = (FmgrInfo *) palloc0(sizeof(FmgrInfo)); /* get default operator class from pg_opclass for datum type */ Oid operatorClassId = GetDefaultOpClass(typeId, accessMethodId); Oid operatorFamilyId = get_opclass_family(operatorClassId); Oid operatorClassInputType = get_opclass_input_type(operatorClassId); Oid operatorId = get_opfamily_proc(operatorFamilyId, operatorClassInputType, operatorClassInputType, procedureId); if (operatorId == InvalidOid) { ereport(ERROR, (errmsg("could not find function for data typeId %u", typeId))); } /* fill in the FmgrInfo struct using the operatorId */ fmgr_info(operatorId, functionInfo); return functionInfo; } ================================================ FILE: src/backend/distributed/utils/function_utils.c ================================================ /*------------------------------------------------------------------------- * * function_utils.c * Utilities regarding calls to PG functions * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/namespace.h" #include "executor/executor.h" #include "utils/builtins.h" #include "utils/regproc.h" #include "distributed/function_utils.h" #include "distributed/version_compat.h" /* * FunctionOid searches for a function that has the given name and the given * number of arguments, and returns the corresponding function's oid. The * function reports error if the target function is not found, or it found more * matching instances. */ Oid FunctionOid(const char *schemaName, const char *functionName, int argumentCount) { return FunctionOidExtended(schemaName, functionName, argumentCount, false); } /* * FunctionOidExtended searches for a given function identified by schema, * functionName, and argumentCount. It reports error if the function is not * found or there are more than one match. If the missingOK parameter is set * and there are no matches, then the function returns InvalidOid. */ Oid FunctionOidExtended(const char *schemaName, const char *functionName, int argumentCount, bool missingOK) { char *qualifiedFunctionName = quote_qualified_identifier(schemaName, functionName); List *qualifiedFunctionNameList = stringToQualifiedNameList(qualifiedFunctionName, NULL); List *argumentList = NIL; const bool findVariadics = false; const bool findDefaults = false; FuncCandidateList functionList = FuncnameGetCandidates( qualifiedFunctionNameList, argumentCount, argumentList, findVariadics, findDefaults, false, true); if (functionList == NULL) { if (missingOK) { return InvalidOid; } ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("function \"%s\" does not exist", functionName))); } else if (functionList->next != NULL) { ereport(ERROR, (errcode(ERRCODE_AMBIGUOUS_FUNCTION), errmsg("more than one function named \"%s\"", functionName))); } /* get function oid from function list's head */ Oid functionOid = functionList->oid; return functionOid; } /* * FunctionCallGetTupleStore1 calls the given set-returning PGFunction with the given * argument and returns the ResultSetInfo filled by the called function. */ ReturnSetInfo * FunctionCallGetTupleStore1(PGFunction function, Oid functionId, Datum argument) { LOCAL_FCINFO(fcinfo, 1); FmgrInfo flinfo; ReturnSetInfo *rsinfo = makeNode(ReturnSetInfo); EState *estate = CreateExecutorState(); rsinfo->econtext = GetPerTupleExprContext(estate); rsinfo->allowedModes = SFRM_Materialize; fmgr_info(functionId, &flinfo); InitFunctionCallInfoData(*fcinfo, &flinfo, 1, InvalidOid, NULL, (Node *) rsinfo); fcSetArg(fcinfo, 0, argument); (*function)(fcinfo); return rsinfo; } ================================================ FILE: src/backend/distributed/utils/hash_helpers.c ================================================ /*------------------------------------------------------------------------- * * hash_helpers.c * Helpers for dynahash.c style hash tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/hashfn.h" #include "utils/hsearch.h" #include "distributed/citus_safe_lib.h" #include "distributed/hash_helpers.h" /* * Empty a hash, without destroying the hash table itself. */ void hash_delete_all(HTAB *htab) { HASH_SEQ_STATUS status; void *entry = NULL; hash_seq_init(&status, htab); while ((entry = hash_seq_search(&status)) != 0) { bool found = false; hash_search(htab, entry, HASH_REMOVE, &found); Assert(found); } } /* * CreateSimpleHashWithNameAndSize creates a hashmap that hashes its key using * tag_hash function and stores the entries in the current memory context. */ HTAB * CreateSimpleHashWithNameAndSizeInternal(Size keySize, Size entrySize, char *name, long nelem) { HASHCTL info; memset_struct_0(info); info.keysize = keySize; info.entrysize = entrySize; info.hcxt = CurrentMemoryContext; int hashFlags = (HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); HTAB *publicationInfoHash = hash_create(name, nelem, &info, hashFlags); return publicationInfoHash; } /* * foreach_htab_cleanup cleans up the hash iteration state after the iteration * is done. This is only needed when break statements are present in the * foreach block. */ void foreach_htab_cleanup(void *var, HASH_SEQ_STATUS *status) { if ((var) != NULL) { hash_seq_term(status); } } ================================================ FILE: src/backend/distributed/utils/jsonbutils.c ================================================ #include "postgres.h" #include "fmgr.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/json.h" #include "utils/lsyscache.h" #include "pg_version_compat.h" #include "distributed/jsonbutils.h" #include "distributed/metadata_cache.h" /* * ExtractFieldJsonb gets value of fieldName from jsonbDoc and puts it * into result. If not found, returns false. Otherwise, returns true. * The field is returned as a Text* Datum if as_text is true, or a Jsonb* * Datum if as_text is false. */ static bool ExtractFieldJsonb(Datum jsonbDoc, const char *fieldName, Datum *result, bool as_text) { Datum pathArray[1] = { CStringGetTextDatum(fieldName) }; bool pathNulls[1] = { false }; bool typeByValue = false; char typeAlignment = 0; int16 typeLength = 0; int dimensions[1] = { 1 }; int lowerbounds[1] = { 1 }; get_typlenbyvalalign(TEXTOID, &typeLength, &typeByValue, &typeAlignment); ArrayType *pathArrayObject = construct_md_array(pathArray, pathNulls, 1, dimensions, lowerbounds, TEXTOID, typeLength, typeByValue, typeAlignment); Datum pathDatum = PointerGetDatum(pathArrayObject); FmgrInfo fmgrInfo; if (as_text) { fmgr_info(JsonbExtractPathTextFuncId(), &fmgrInfo); } else { fmgr_info(JsonbExtractPathFuncId(), &fmgrInfo); } LOCAL_FCINFO(functionCallInfo, 2); InitFunctionCallInfoData(*functionCallInfo, &fmgrInfo, 2, DEFAULT_COLLATION_OID, NULL, NULL); fcSetArg(functionCallInfo, 0, jsonbDoc); fcSetArg(functionCallInfo, 1, pathDatum); *result = FunctionCallInvoke(functionCallInfo); return !functionCallInfo->isnull; } /* * ExtractFieldBoolean gets value of fieldName from jsonbDoc, or returns * defaultValue if it doesn't exist. */ bool ExtractFieldBoolean(Datum jsonbDoc, const char *fieldName, bool defaultValue) { Datum jsonbDatum = 0; bool found = ExtractFieldJsonb(jsonbDoc, fieldName, &jsonbDatum, false); if (!found) { return defaultValue; } Datum boolDatum = DirectFunctionCall1(jsonb_bool, jsonbDatum); return DatumGetBool(boolDatum); } /* * ExtractFieldInt32 gets value of fieldName from jsonbDoc, or returns * defaultValue if it doesn't exist. */ int32 ExtractFieldInt32(Datum jsonbDoc, const char *fieldName, int32 defaultValue) { Datum jsonbDatum = 0; bool found = ExtractFieldJsonb(jsonbDoc, fieldName, &jsonbDatum, false); if (!found) { return defaultValue; } Datum int32Datum = DirectFunctionCall1(jsonb_int4, jsonbDatum); return DatumGetInt32(int32Datum); } /* * ExtractFieldTextP gets value of fieldName as text* from jsonbDoc, or * returns NULL if it doesn't exist. */ text * ExtractFieldTextP(Datum jsonbDoc, const char *fieldName) { Datum jsonbDatum = 0; bool found = ExtractFieldJsonb(jsonbDoc, fieldName, &jsonbDatum, true); if (!found) { return NULL; } return DatumGetTextP(jsonbDatum); } /* * ExtractFieldJsonbDatum gets value of fieldName from jsonbDoc and puts it * into result. If not found, returns false. Otherwise, returns true. */ bool ExtractFieldJsonbDatum(Datum jsonbDoc, const char *fieldName, Datum *result) { return ExtractFieldJsonb(jsonbDoc, fieldName, result, false); } ================================================ FILE: src/backend/distributed/utils/listutils.c ================================================ /*------------------------------------------------------------------------- * * listutils.c * * This file contains functions to perform useful operations on lists. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "port.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "distributed/citus_safe_lib.h" #include "distributed/listutils.h" /* * SortList takes in a list of void pointers, and sorts these pointers (and the * values they point to) by applying the given comparison function. The function * then returns the sorted list of pointers. * * Because the input list is a list of pointers, and because qsort expects to * compare pointers to the list elements, the provided comparison function must * compare pointers to pointers to elements. In addition, this sort function * naturally exhibits the same lack of stability exhibited by qsort. See that * function's man page for more details. */ List * SortList(List *pointerList, int (*comparisonFunction)(const void *, const void *)) { List *sortedList = NIL; uint32 arrayIndex = 0; uint32 arraySize = (uint32) list_length(pointerList); void **array = (void **) palloc0(arraySize * sizeof(void *)); void *pointer = NULL; foreach_declared_ptr(pointer, pointerList) { array[arrayIndex] = pointer; arrayIndex++; } /* sort the array of pointers using the comparison function */ SafeQsort(array, arraySize, sizeof(void *), comparisonFunction); /* convert the sorted array of pointers back to a sorted list */ for (arrayIndex = 0; arrayIndex < arraySize; arrayIndex++) { void *sortedPointer = array[arrayIndex]; sortedList = lappend(sortedList, sortedPointer); } pfree(array); if (sortedList != NIL) { sortedList->type = pointerList->type; } return sortedList; } /* * PointerArrayFromList converts a list of pointers to an array of pointers. */ void ** PointerArrayFromList(List *pointerList) { int pointerCount = list_length(pointerList); void **pointerArray = (void **) palloc0(pointerCount * sizeof(void *)); int pointerIndex = 0; void *pointer = NULL; foreach_declared_ptr(pointer, pointerList) { pointerArray[pointerIndex] = pointer; pointerIndex += 1; } return pointerArray; } /* * ListToHashSet creates a hash table in which the keys are copied from * from itemList and the values are the same as the keys. This can * be used for fast lookups of the presence of a byte array in a set. * itemList should be a list of pointers. * * If isStringList is true, then look-ups are performed through string * comparison of strings up to keySize in length. If isStringList is * false, then look-ups are performed by comparing exactly keySize * bytes pointed to by the pointers in itemList. */ HTAB * ListToHashSet(List *itemList, Size keySize, bool isStringList) { HASHCTL info; int flags = HASH_ELEM | HASH_CONTEXT; /* allocate sufficient capacity for O(1) expected look-up time */ int capacity = (int) (list_length(itemList) / 0.75) + 1; /* initialise the hash table for looking up keySize bytes */ memset(&info, 0, sizeof(info)); info.keysize = keySize; info.entrysize = keySize; info.hcxt = CurrentMemoryContext; if (isStringList) { flags |= HASH_STRINGS; } else { flags |= HASH_BLOBS; } HTAB *itemSet = hash_create("ListToHashSet", capacity, &info, flags); void *item = NULL; foreach_declared_ptr(item, itemList) { bool foundInSet = false; hash_search(itemSet, item, HASH_ENTER, &foundInSet); } return itemSet; } /* * GeneratePositiveIntSequenceList generates a positive int * sequence list up to the given number. The list will have: * [1:upto] */ List * GeneratePositiveIntSequenceList(int upTo) { List *intList = NIL; for (int i = 1; i <= upTo; i++) { intList = lappend_int(intList, i); } return intList; } /* * StringJoin gets a list of char * and then simply * returns a newly allocated char * joined with the * given delimiter. It uses ';' as the delimiter by * default. */ char * StringJoin(List *stringList, char delimiter) { return StringJoinParams(stringList, delimiter, NULL, NULL); } /* * StringJoin gets a list of char * and then simply * returns a newly allocated char * joined with the * given delimiter, prefix and postfix. */ char * StringJoinParams(List *stringList, char delimiter, char *prefix, char *postfix) { StringInfo joinedString = makeStringInfo(); if (prefix != NULL) { appendStringInfoString(joinedString, prefix); } const char *command = NULL; int curIndex = 0; foreach_declared_ptr(command, stringList) { if (curIndex > 0) { appendStringInfoChar(joinedString, delimiter); } appendStringInfoString(joinedString, command); curIndex++; } if (postfix != NULL) { appendStringInfoString(joinedString, postfix); } return joinedString->data; } /* * ListTake returns the first size elements of given list. If size is greater * than list's length, it returns all elements of list. This is modeled after * the "take" function used in some Scheme implementations. */ List * ListTake(List *pointerList, int size) { List *result = NIL; int listIndex = 0; void *pointer = NULL; foreach_declared_ptr(pointer, pointerList) { result = lappend(result, pointer); listIndex++; if (listIndex >= size) { break; } } return result; } /* * safe_list_nth first checks if given index is valid and errors out if it is * not. Otherwise, it directly calls list_nth. */ void * safe_list_nth(const List *list, int index) { int listLength = list_length(list); if (index < 0 || index >= listLength) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid list access: list length was %d but " "element at index %d was requested ", listLength, index))); } return list_nth(list, index); } /* * GenerateListFromElement returns a new list with length of listLength * such that all the elements are identical with input listElement pointer. */ List * GenerateListFromElement(void *listElement, int listLength) { List *list = NIL; for (int i = 0; i < listLength; i++) { list = lappend(list, listElement); } return list; } /* * list_filter_oid filters a list of oid-s based on a keepElement * function */ List * list_filter_oid(List *list, bool (*keepElement)(Oid element)) { List *result = NIL; Oid element = InvalidOid; foreach_declared_oid(element, list) { if (keepElement(element)) { result = lappend_oid(result, element); } } return result; } ================================================ FILE: src/backend/distributed/utils/log_utils.c ================================================ /*------------------------------------------------------------------------- * * log_utils.c * Utilities regarding logs * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "common/cryptohash.h" #include "common/sha2.h" #include "utils/builtins.h" #include "utils/guc.h" #include "pg_version_constants.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" /* * GUC controls showing of some of the unwanted citus messages, it is intended to be set false * before vanilla tests to not break postgres test logs. */ bool EnableUnsupportedFeatureMessages = true; /* * IsLoggableLevel returns true if either of client or server log guc is configured to * log the given log level. * In postgres, log can be configured differently for clients and servers. */ bool IsLoggableLevel(int logLevel) { return log_min_messages <= logLevel || client_min_messages <= logLevel; } ================================================ FILE: src/backend/distributed/utils/maintenanced.c ================================================ /*------------------------------------------------------------------------- * * maintenanced.c * Background worker run for each citus using database in a postgres * cluster. * * This file provides infrastructure for launching exactly one a background * worker for every database in which citus is used. That background worker * can then perform work like deadlock detection, prepared transaction * recovery, and cleanup. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include #include "postgres.h" #include "miscadmin.h" #include "pgstat.h" #include "access/xact.h" #include "access/xlog.h" #include "catalog/namespace.h" #include "catalog/pg_authid.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "commands/async.h" #include "commands/extension.h" #include "common/hashfn.h" #include "libpq/pqsignal.h" #include "nodes/makefuncs.h" #include "postmaster/bgworker.h" #include "postmaster/postmaster.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/lmgr.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "citus_version.h" #include "pg_version_constants.h" #include "distributed/background_jobs.h" #include "distributed/background_worker_utils.h" #include "distributed/citus_safe_lib.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/maintenanced.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/resource_lock.h" #include "distributed/shard_cleaner.h" #include "distributed/stats/query_stats.h" #include "distributed/transaction_recovery.h" #include "distributed/version_compat.h" /* * Shared memory data for all maintenance workers. */ typedef struct MaintenanceDaemonControlData { /* * Lock protecting the shared memory state. This is to be taken when * looking up (shared mode) or inserting (exclusive mode) per-database * data in MaintenanceDaemonDBHash. */ int trancheId; char *lockTrancheName; LWLock lock; } MaintenanceDaemonControlData; /* * Per database worker state. */ typedef struct MaintenanceDaemonDBData { /* hash key: database to run on */ Oid databaseOid; /* information: which user to use */ Oid userOid; pid_t workerPid; bool daemonStarted; bool triggerNodeMetadataSync; Latch *latch; /* pointer to the background worker's latch */ } MaintenanceDaemonDBData; /* config variable for distributed deadlock detection timeout */ double DistributedDeadlockDetectionTimeoutFactor = 2.0; int Recover2PCInterval = 60000; int DeferShardDeleteInterval = 15000; int BackgroundTaskQueueCheckInterval = 5000; int MaxBackgroundTaskExecutors = 1; char *MainDb = ""; /* config variables for metadata sync timeout */ int MetadataSyncInterval = 60000; int MetadataSyncRetryInterval = 5000; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static MaintenanceDaemonControlData *MaintenanceDaemonControl = NULL; /* * Hash-table of workers, one entry for each database with citus * activated. */ static HTAB *MaintenanceDaemonDBHash; static ErrorContextCallback errorCallback = { 0 }; static volatile sig_atomic_t got_SIGHUP = false; static volatile sig_atomic_t got_SIGTERM = false; /* set to true when becoming a maintenance daemon */ static bool IsMaintenanceDaemon = false; static void MaintenanceDaemonSigTermHandler(SIGNAL_ARGS); static void MaintenanceDaemonSigHupHandler(SIGNAL_ARGS); static void MaintenanceDaemonShmemExit(int code, Datum arg); static void MaintenanceDaemonErrorContext(void *arg); static bool MetadataSyncTriggeredCheckAndReset(MaintenanceDaemonDBData *dbData); static void WarnMaintenanceDaemonNotStarted(void); static MaintenanceDaemonDBData * GetMaintenanceDaemonDBHashEntry(Oid databaseId, bool *found); /* * InitializeMaintenanceDaemon, called at server start, is responsible for * requesting shared memory and related infrastructure required by maintenance * daemons. */ void InitializeMaintenanceDaemon(void) { prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = MaintenanceDaemonShmemInit; } /* * GetMaintenanceDaemonDBHashEntry searches the MaintenanceDaemonDBHash for the * databaseId. It returns the entry if found or creates a new entry and initializes * the value with zeroes. */ MaintenanceDaemonDBData * GetMaintenanceDaemonDBHashEntry(Oid databaseId, bool *found) { MaintenanceDaemonDBData *dbData = (MaintenanceDaemonDBData *) hash_search( MaintenanceDaemonDBHash, &MyDatabaseId, HASH_ENTER_NULL, found); if (!dbData) { elog(LOG, "cannot create or find the maintenance deamon hash entry for database %u", databaseId); return NULL; } if (!*found) { /* ensure the values in MaintenanceDaemonDBData are zero */ memset(((char *) dbData) + sizeof(Oid), 0, sizeof(MaintenanceDaemonDBData) - sizeof(Oid)); } return dbData; } /* * InitializeMaintenanceDaemonForMainDb is called in _PG_Init * at which stage we are not in a transaction or have databaseOid */ void InitializeMaintenanceDaemonForMainDb(void) { if (strcmp(MainDb, "") == 0) { elog(LOG, "There is no designated Main database."); return; } CitusBackgroundWorkerConfig config = { .workerName = "Citus Maintenance Daemon for Main DB", .functionName = "CitusMaintenanceDaemonMain", .mainArg = (Datum) 0, .extensionOwner = InvalidOid, .needsNotification = false, .waitForStartup = false, .restartTime = CITUS_BGW_DEFAULT_RESTART_TIME, .startTime = CITUS_BGW_DEFAULT_START_TIME, .workerType = NULL, /* use default */ .extraData = NULL, .extraDataSize = 0 }; BackgroundWorker worker; InitializeCitusBackgroundWorker(&worker, &config); RegisterBackgroundWorker(&worker); } /* * InitializeMaintenanceDaemonBackend, called at backend start and * configuration changes, is responsible for starting a per-database * maintenance worker if necessary. */ void InitializeMaintenanceDaemonBackend(void) { Oid extensionOwner = CitusExtensionOwner(); bool found = false; LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); MaintenanceDaemonDBData *dbData = GetMaintenanceDaemonDBHashEntry(MyDatabaseId, &found); if (dbData == NULL) { WarnMaintenanceDaemonNotStarted(); LWLockRelease(&MaintenanceDaemonControl->lock); return; } if (IsMaintenanceDaemon) { /* * InitializeMaintenanceDaemonBackend is called by the maintenance daemon * itself. In that case, we clearly don't need to start another maintenance * daemon. */ LWLockRelease(&MaintenanceDaemonControl->lock); return; } if (!found || !dbData->daemonStarted) { Assert(dbData->workerPid == 0); char workerName[BGW_MAXLEN]; SafeSnprintf(workerName, sizeof(workerName), "Citus Maintenance Daemon: %u/%u", MyDatabaseId, extensionOwner); CitusBackgroundWorkerConfig config = { .workerName = workerName, .functionName = "CitusMaintenanceDaemonMain", .mainArg = ObjectIdGetDatum(MyDatabaseId), .extensionOwner = extensionOwner, .needsNotification = true, .waitForStartup = true, .restartTime = CITUS_BGW_DEFAULT_RESTART_TIME, .startTime = CITUS_BGW_DEFAULT_START_TIME, .workerType = NULL, /* use default */ .extraData = NULL, .extraDataSize = 0 }; BackgroundWorkerHandle *handle = RegisterCitusBackgroundWorker(&config); if (!handle) { WarnMaintenanceDaemonNotStarted(); dbData->daemonStarted = false; LWLockRelease(&MaintenanceDaemonControl->lock); return; } dbData->daemonStarted = true; dbData->userOid = extensionOwner; dbData->workerPid = 0; dbData->triggerNodeMetadataSync = false; LWLockRelease(&MaintenanceDaemonControl->lock); pfree(handle); } else { Assert(dbData->daemonStarted); /* * If owner of extension changed, wake up daemon. It'll notice and * restart. */ if (dbData->userOid != extensionOwner) { dbData->userOid = extensionOwner; if (dbData->latch) { SetLatch(dbData->latch); } } LWLockRelease(&MaintenanceDaemonControl->lock); } } /* * WarnMaintenanceDaemonNotStarted warns that maintenanced couldn't be started. */ static void WarnMaintenanceDaemonNotStarted(void) { ereport(WARNING, (errmsg("could not start maintenance background worker"), errhint("Increasing max_worker_processes might help."))); } /* * ConnectToDatabase connects to the database for the given databaseOid. * if databaseOid is 0, connects to MainDb and then creates a hash entry. * If a hash entry cannot be created for MainDb it exits the process requesting a restart. * However for regular databases, it exits without requesting a restart since another * subsequent backend is expected to start the Maintenance Daemon. * If the found hash entry has a valid workerPid, it exits * without requesting a restart since there is already a daemon running. */ static MaintenanceDaemonDBData * ConnectToDatabase(Oid databaseOid) { MaintenanceDaemonDBData *myDbData = NULL; bool isMainDb = false; LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); if (databaseOid == 0) { char *databaseName = MainDb; /* * Since we cannot query databaseOid without initializing Postgres * first, connect to the database by name. */ BackgroundWorkerInitializeConnection(databaseName, NULL, 0); /* * Now we have a valid MyDatabaseId. * Insert the hash entry for the database to the Maintenance Deamon Hash. */ bool found = false; myDbData = GetMaintenanceDaemonDBHashEntry(MyDatabaseId, &found); if (!myDbData) { /* * If an entry cannot be created, * return code of 1 requests worker restart * Since BackgroundWorker for the MainDb is only registered * once during server startup, we need to retry. */ proc_exit(1); } if (found && myDbData->workerPid != 0) { /* Another maintenance daemon is running.*/ proc_exit(0); } databaseOid = MyDatabaseId; myDbData->userOid = GetSessionUserId(); isMainDb = true; } else { myDbData = (MaintenanceDaemonDBData *) hash_search(MaintenanceDaemonDBHash, &databaseOid, HASH_FIND, NULL); if (!myDbData) { /* * When the database crashes, background workers are restarted, but * the state in shared memory is lost. In that case, we exit and * wait for a session to call InitializeMaintenanceDaemonBackend * to properly add it to the hash. */ proc_exit(0); } if (myDbData->workerPid != 0) { /* * Another maintenance daemon is running. This usually happens because * postgres restarts the daemon after an non-zero exit, and * InitializeMaintenanceDaemonBackend started one before postgres did. * In that case, the first one stays and the last one exits. */ proc_exit(0); } } before_shmem_exit(MaintenanceDaemonShmemExit, ObjectIdGetDatum(databaseOid)); /* * Signal that I am the maintenance daemon now. * * From this point, DROP DATABASE/EXTENSION will send a SIGTERM to me. */ myDbData->workerPid = MyProcPid; /* * Signal that we are running. This in mainly needed in case of restart after * an error, otherwise the daemonStarted flag is already true. */ myDbData->daemonStarted = true; /* wire up signals */ pqsignal(SIGTERM, MaintenanceDaemonSigTermHandler); pqsignal(SIGHUP, MaintenanceDaemonSigHupHandler); BackgroundWorkerUnblockSignals(); myDbData->latch = MyLatch; IsMaintenanceDaemon = true; LWLockRelease(&MaintenanceDaemonControl->lock); memset(&errorCallback, 0, sizeof(errorCallback)); errorCallback.callback = MaintenanceDaemonErrorContext; errorCallback.arg = (void *) myDbData; errorCallback.previous = error_context_stack; error_context_stack = &errorCallback; elog(LOG, "starting maintenance daemon on database %u user %u", databaseOid, myDbData->userOid); if (!isMainDb) { /* connect to database, after that we can actually access catalogs */ BackgroundWorkerInitializeConnectionByOid(databaseOid, myDbData->userOid, 0); } return myDbData; } /* * CitusMaintenanceDaemonMain is the maintenance daemon's main routine, it'll * be started by the background worker infrastructure. If it errors out, * it'll be restarted after a few seconds. */ void CitusMaintenanceDaemonMain(Datum main_arg) { Oid databaseOid = DatumGetObjectId(main_arg); TimestampTz lastRecoveryTime = 0; TimestampTz lastShardCleanTime = 0; TimestampTz lastStatStatementsPurgeTime = 0; TimestampTz nextMetadataSyncTime = 0; /* state kept for the background tasks queue monitor */ TimestampTz lastBackgroundTaskQueueCheck = GetCurrentTimestamp(); BackgroundWorkerHandle *backgroundTasksQueueBgwHandle = NULL; bool backgroundTasksQueueWarnedForLock = false; /* * We do metadata sync in a separate background worker. We need its * handle to be able to check its status. */ BackgroundWorkerHandle *metadataSyncBgwHandle = NULL; MaintenanceDaemonDBData *myDbData = ConnectToDatabase(databaseOid); /* make worker recognizable in pg_stat_activity */ pgstat_report_appname("Citus Maintenance Daemon"); /* * Terminate orphaned metadata sync daemons spawned from previously terminated * or crashed maintenanced instances. */ SignalMetadataSyncDaemon(MyDatabaseId, SIGTERM); /* enter main loop */ while (!got_SIGTERM) { int latchFlags = WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH; double timeout = 10000.0; /* use this if the deadlock detection is disabled */ bool foundDeadlock = false; CHECK_FOR_INTERRUPTS(); CitusTableCacheFlushInvalidatedEntries(); /* * XXX: Each task should clear the metadata cache before every iteration * by calling InvalidateMetadataSystemCache(), because otherwise it * might contain stale OIDs. It appears that in some cases invalidation * messages for a DROP EXTENSION may arrive during these tasks and * this causes us to cache a stale pg_dist_node OID. We'd actually expect * all invalidations to arrive after obtaining a lock in LockCitusExtension. */ /* * Perform Work. If a specific task needs to be called sooner than * timeout indicates, it's ok to lower it to that value. Expensive * tasks should do their own time math about whether to re-run checks. */ pid_t metadataSyncBgwPid = 0; BgwHandleStatus metadataSyncStatus = metadataSyncBgwHandle != NULL ? GetBackgroundWorkerPid(metadataSyncBgwHandle, &metadataSyncBgwPid) : BGWH_STOPPED; if (metadataSyncStatus != BGWH_STOPPED && GetCurrentTimestamp() >= nextMetadataSyncTime) { /* * Metadata sync is still running, recheck in a short while. */ int nextTimeout = MetadataSyncRetryInterval; nextMetadataSyncTime = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), nextTimeout); timeout = Min(timeout, nextTimeout); } else if (!RecoveryInProgress() && metadataSyncStatus == BGWH_STOPPED && (MetadataSyncTriggeredCheckAndReset(myDbData) || GetCurrentTimestamp() >= nextMetadataSyncTime)) { if (metadataSyncBgwHandle) { pfree(metadataSyncBgwHandle); metadataSyncBgwHandle = NULL; } InvalidateMetadataSystemCache(); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); int nextTimeout = MetadataSyncRetryInterval; bool syncMetadata = false; if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping metadata sync"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { bool lockFailure = false; syncMetadata = ShouldInitiateMetadataSync(&lockFailure); /* * If lock fails, we need to recheck in a short while. If we are * going to sync metadata, we should recheck in a short while to * see if it failed. Otherwise, we can wait longer. */ nextTimeout = (lockFailure || syncMetadata) ? MetadataSyncRetryInterval : MetadataSyncInterval; } PopActiveSnapshot(); CommitTransactionCommand(); if (syncMetadata) { metadataSyncBgwHandle = SpawnSyncNodeMetadataToNodes(MyDatabaseId, myDbData->userOid); } nextMetadataSyncTime = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), nextTimeout); timeout = Min(timeout, nextTimeout); } /* * If enabled, run 2PC recovery on primary nodes (where !RecoveryInProgress()), * since we'll write to the pg_dist_transaction log. */ if (Recover2PCInterval > 0 && !RecoveryInProgress() && TimestampDifferenceExceeds(lastRecoveryTime, GetCurrentTimestamp(), Recover2PCInterval)) { int recoveredTransactionCount = 0; InvalidateMetadataSystemCache(); StartTransactionCommand(); if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping 2PC recovery"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { /* * Record last recovery time at start to ensure we run once per * Recover2PCInterval even if RecoverTwoPhaseCommits takes some time. */ lastRecoveryTime = GetCurrentTimestamp(); recoveredTransactionCount = RecoverTwoPhaseCommits(); } CommitTransactionCommand(); if (recoveredTransactionCount > 0) { ereport(LOG, (errmsg("maintenance daemon recovered %d distributed " "transactions", recoveredTransactionCount))); } /* make sure we don't wait too long */ timeout = Min(timeout, Recover2PCInterval); } /* the config value -1 disables the distributed deadlock detection */ if (DistributedDeadlockDetectionTimeoutFactor != -1.0) { double deadlockTimeout = DistributedDeadlockDetectionTimeoutFactor * (double) DeadlockTimeout; InvalidateMetadataSystemCache(); StartTransactionCommand(); /* * We skip the deadlock detection if citus extension * is not accessible. * * Similarly, we skip to run the deadlock checks if * there exists any version mismatch or the extension * is not fully created yet. */ if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping deadlock detection"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { foundDeadlock = CheckForDistributedDeadlocks(); } CommitTransactionCommand(); /* * If we find any deadlocks, run the distributed deadlock detection * more often since it is quite possible that there are other * deadlocks need to be resolved. * * Thus, we use 1/20 of the calculated value. With the default * values (i.e., deadlock_timeout 1 seconds, * citus.distributed_deadlock_detection_factor 2), we'd be able to cancel * ~10 distributed deadlocks per second. */ if (foundDeadlock) { deadlockTimeout = deadlockTimeout / 20.0; } /* make sure we don't wait too long */ timeout = Min(timeout, deadlockTimeout); } if (!RecoveryInProgress() && DeferShardDeleteInterval > 0 && TimestampDifferenceExceeds(lastShardCleanTime, GetCurrentTimestamp(), DeferShardDeleteInterval)) { int numberOfDroppedResources = 0; InvalidateMetadataSystemCache(); StartTransactionCommand(); if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg( "could not lock the citus extension, skipping shard cleaning"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { /* * Record last shard clean time at start to ensure we run once per * DeferShardDeleteInterval. */ lastShardCleanTime = GetCurrentTimestamp(); numberOfDroppedResources = TryDropOrphanedResources(); } CommitTransactionCommand(); if (numberOfDroppedResources > 0) { ereport(LOG, (errmsg("maintenance daemon dropped %d " "resources previously marked to be removed", numberOfDroppedResources))); } /* make sure we don't wait too long */ timeout = Min(timeout, DeferShardDeleteInterval); } if (StatStatementsPurgeInterval > 0 && StatStatementsTrack != STAT_STATEMENTS_TRACK_NONE && TimestampDifferenceExceeds(lastStatStatementsPurgeTime, GetCurrentTimestamp(), (StatStatementsPurgeInterval * 1000))) { StartTransactionCommand(); if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping stat statements purging"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { /* * Record last time we perform the purge to ensure we run once per * StatStatementsPurgeInterval. */ lastStatStatementsPurgeTime = GetCurrentTimestamp(); CitusQueryStatsSynchronizeEntries(); } CommitTransactionCommand(); /* make sure we don't wait too long, need to convert seconds to milliseconds */ timeout = Min(timeout, (StatStatementsPurgeInterval * 1000)); } pid_t backgroundTaskQueueWorkerPid = 0; BgwHandleStatus backgroundTaskQueueWorkerStatus = backgroundTasksQueueBgwHandle != NULL ? GetBackgroundWorkerPid( backgroundTasksQueueBgwHandle, &backgroundTaskQueueWorkerPid) : BGWH_STOPPED; if (!RecoveryInProgress() && BackgroundTaskQueueCheckInterval > 0 && TimestampDifferenceExceeds(lastBackgroundTaskQueueCheck, GetCurrentTimestamp(), BackgroundTaskQueueCheckInterval) && backgroundTaskQueueWorkerStatus == BGWH_STOPPED) { /* clear old background worker for task queue before checking for new tasks */ if (backgroundTasksQueueBgwHandle) { pfree(backgroundTasksQueueBgwHandle); backgroundTasksQueueBgwHandle = NULL; } StartTransactionCommand(); bool shouldStartBackgroundTaskQueueBackgroundWorker = false; if (!LockCitusExtension()) { ereport(DEBUG1, (errmsg("could not lock the citus extension, " "skipping stat statements purging"))); } else if (CheckCitusVersion(DEBUG1) && CitusHasBeenLoaded()) { /* perform catalog precheck */ shouldStartBackgroundTaskQueueBackgroundWorker = HasRunnableBackgroundTask(); } CommitTransactionCommand(); if (shouldStartBackgroundTaskQueueBackgroundWorker) { /* * Before we start the background worker we want to check if an orphaned * one is still running. This could happen when the maintenance daemon * restarted in a way where the background task queue monitor wasn't * restarted. * * To check if an orphaned background task queue monitor is still running * we quickly acquire the lock without waiting. If we can't acquire the * lock this means that some other backed still has the lock. We prevent a * new backend from starting and log a warning that we found that another * process still holds the lock. */ LOCKTAG tag = { 0 }; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_BACKGROUND_TASK_MONITOR); const bool sessionLock = false; const bool dontWait = true; LockAcquireResult locked = LockAcquire(&tag, AccessExclusiveLock, sessionLock, dontWait); if (locked == LOCKACQUIRE_NOT_AVAIL) { if (!backgroundTasksQueueWarnedForLock) { ereport(WARNING, (errmsg("background task queue monitor already " "held"), errdetail("the background task queue monitor " "lock is held by another backend, " "indicating the maintenance daemon " "has lost track of an already " "running background task queue " "monitor, not starting a new one"))); backgroundTasksQueueWarnedForLock = true; } } else { LockRelease(&tag, AccessExclusiveLock, sessionLock); /* we were able to acquire the lock, reset the warning tracker */ backgroundTasksQueueWarnedForLock = false; /* spawn background worker */ ereport(LOG, (errmsg("found scheduled background tasks, starting new " "background task queue monitor"))); backgroundTasksQueueBgwHandle = StartCitusBackgroundTaskQueueMonitor(MyDatabaseId, myDbData->userOid); if (!backgroundTasksQueueBgwHandle || GetBackgroundWorkerPid(backgroundTasksQueueBgwHandle, &backgroundTaskQueueWorkerPid) == BGWH_STOPPED) { ereport(WARNING, (errmsg("unable to start background worker for " "background task execution"))); } } } /* interval management */ lastBackgroundTaskQueueCheck = GetCurrentTimestamp(); timeout = Min(timeout, BackgroundTaskQueueCheckInterval); } /* * Wait until timeout, or until somebody wakes us up. Also cast the timeout to * integer where we've calculated it using double for not losing the precision. */ int rc = WaitLatch(MyLatch, latchFlags, (long) timeout, PG_WAIT_EXTENSION); /* emergency bailout if postmaster has died */ if (rc & WL_POSTMASTER_DEATH) { proc_exit(1); } if (rc & WL_LATCH_SET) { ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); /* check for changed configuration */ if (myDbData->userOid != GetSessionUserId()) { /* return code of 1 requests worker restart */ proc_exit(1); } /* * Could also add code checking whether extension still exists, * but that'd complicate things a bit, because we'd have to delete * the shared memory entry. There'd potentially be a race * condition where the extension gets re-created, checking that * this entry still exists, and it getting deleted just after. * Doesn't seem worth catering for that. */ } if (got_SIGHUP) { got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); } } if (metadataSyncBgwHandle) { TerminateBackgroundWorker(metadataSyncBgwHandle); } } /* * MaintenanceDaemonShmemSize computes how much shared memory is required. */ size_t MaintenanceDaemonShmemSize(void) { Size size = 0; size = add_size(size, sizeof(MaintenanceDaemonControlData)); /* * We request enough shared memory to have one hash-table entry for each * worker process. We couldn't start more anyway, so there's little point * in allocating more. */ Size hashSize = hash_estimate_size(max_worker_processes, sizeof(MaintenanceDaemonDBData)); size = add_size(size, hashSize); return size; } /* * MaintenanceDaemonShmemInit initializes the requested shared memory for the * maintenance daemon. */ void MaintenanceDaemonShmemInit(void) { bool alreadyInitialized = false; HASHCTL hashInfo; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); MaintenanceDaemonControl = (MaintenanceDaemonControlData *) ShmemInitStruct("Citus Maintenance Daemon", MaintenanceDaemonShmemSize(), &alreadyInitialized); /* * Might already be initialized on EXEC_BACKEND type platforms that call * shared library initialization functions in every backend. */ if (!alreadyInitialized) { MaintenanceDaemonControl->trancheId = LWLockNewTrancheId(); MaintenanceDaemonControl->lockTrancheName = "Citus Maintenance Daemon"; LWLockRegisterTranche(MaintenanceDaemonControl->trancheId, MaintenanceDaemonControl->lockTrancheName); LWLockInitialize(&MaintenanceDaemonControl->lock, MaintenanceDaemonControl->trancheId); } memset(&hashInfo, 0, sizeof(hashInfo)); hashInfo.keysize = sizeof(Oid); hashInfo.entrysize = sizeof(MaintenanceDaemonDBData); hashInfo.hash = tag_hash; int hashFlags = (HASH_ELEM | HASH_FUNCTION); MaintenanceDaemonDBHash = ShmemInitHash("Maintenance Database Hash", max_worker_processes, max_worker_processes, &hashInfo, hashFlags); LWLockRelease(AddinShmemInitLock); if (prev_shmem_startup_hook != NULL) { prev_shmem_startup_hook(); } } /* * MaintenaceDaemonShmemExit is the before_shmem_exit handler for cleaning up MaintenanceDaemonDBHash */ static void MaintenanceDaemonShmemExit(int code, Datum arg) { Oid databaseOid = DatumGetObjectId(arg); LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); MaintenanceDaemonDBData *myDbData = (MaintenanceDaemonDBData *) hash_search(MaintenanceDaemonDBHash, &databaseOid, HASH_FIND, NULL); /* myDbData is NULL after StopMaintenanceDaemon */ if (myDbData != NULL) { /* * Once the maintenance daemon fails (e.g., due to an error in the main loop), * both Postgres tries to restart the failed daemon and Citus attempt to start * a new one. In that case, the one started by Citus ends up here. * * As the maintenance daemon that Citus tried to start, we might see the entry * for the daemon restarted by Postgres if the system was so slow that it * took a long time for us to be re-scheduled to call MaintenanceDaemonShmemExit(), * e.g., under valgrind testing. * * In that case, we should unregister ourself only if we are still the registered * maintenance daemon. */ if (myDbData->workerPid == MyProcPid) { myDbData->daemonStarted = false; myDbData->workerPid = 0; } else { ereport(LOG, (errmsg( "maintenance daemon for database %u has already been replaced by " "Postgres, skipping to unregister this maintenance daemon", databaseOid))); } } LWLockRelease(&MaintenanceDaemonControl->lock); } /* MaintenanceDaemonSigTermHandler sets the got_SIGTERM flag.*/ static void MaintenanceDaemonSigTermHandler(SIGNAL_ARGS) { int save_errno = errno; got_SIGTERM = true; if (MyProc != NULL) { SetLatch(&MyProc->procLatch); } errno = save_errno; } /* * MaintenanceDaemonSigHupHandler set a flag to re-read config file at next * convenient time. */ static void MaintenanceDaemonSigHupHandler(SIGNAL_ARGS) { int save_errno = errno; got_SIGHUP = true; if (MyProc != NULL) { SetLatch(&MyProc->procLatch); } errno = save_errno; } /* * MaintenanceDaemonErrorContext adds some context to log messages to make it * easier to associate them with the maintenance daemon. */ static void MaintenanceDaemonErrorContext(void *arg) { MaintenanceDaemonDBData *myDbData = (MaintenanceDaemonDBData *) arg; errcontext("Citus maintenance daemon for database %u user %u", myDbData->databaseOid, myDbData->userOid); } /* * LockCitusExtension acquires a lock on the Citus extension or returns * false if the extension does not exist or is being dropped. */ bool LockCitusExtension(void) { Oid extensionOid = get_extension_oid("citus", true); if (extensionOid == InvalidOid) { /* citus extension does not exist */ return false; } LockDatabaseObject(ExtensionRelationId, extensionOid, 0, AccessShareLock); /* * The extension may have been dropped and possibly recreated prior to * obtaining a lock. Check whether we still get the expected OID. */ Oid recheckExtensionOid = get_extension_oid("citus", true); if (recheckExtensionOid != extensionOid) { return false; } return true; } /* * StopMaintenanceDaemon stops the maintenance daemon for the * given database and removes it from the maintenance daemon * control hash. */ void StopMaintenanceDaemon(Oid databaseId) { bool found = false; pid_t workerPid = 0; LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); MaintenanceDaemonDBData *dbData = (MaintenanceDaemonDBData *) hash_search( MaintenanceDaemonDBHash, &databaseId, HASH_REMOVE, &found); if (found) { workerPid = dbData->workerPid; } LWLockRelease(&MaintenanceDaemonControl->lock); if (workerPid > 0) { kill(workerPid, SIGTERM); } } /* * TriggerMetadataSync triggers the maintenance daemon to do * a node metadata sync for the given database. */ void TriggerNodeMetadataSync(Oid databaseId) { bool found = false; LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); MaintenanceDaemonDBData *dbData = (MaintenanceDaemonDBData *) hash_search( MaintenanceDaemonDBHash, &databaseId, HASH_FIND, &found); if (found) { dbData->triggerNodeMetadataSync = true; /* set latch to wake-up the maintenance loop */ SetLatch(dbData->latch); } LWLockRelease(&MaintenanceDaemonControl->lock); } /* * MetadataSyncTriggeredCheckAndReset checks if metadata sync has been * triggered for the given database, and resets the flag. */ static bool MetadataSyncTriggeredCheckAndReset(MaintenanceDaemonDBData *dbData) { LWLockAcquire(&MaintenanceDaemonControl->lock, LW_EXCLUSIVE); bool metadataSyncTriggered = dbData->triggerNodeMetadataSync; dbData->triggerNodeMetadataSync = false; LWLockRelease(&MaintenanceDaemonControl->lock); return metadataSyncTriggered; } ================================================ FILE: src/backend/distributed/utils/multi_partitioning_utils.c ================================================ /* * multi_partitioning_utils.c * Utility functions for declarative partitioning * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "pgstat.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "catalog/index.h" #include "catalog/indexing.h" #include "catalog/partition.h" #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_inherits.h" #include "commands/tablecmds.h" #include "common/string.h" #include "lib/stringinfo.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" #include "partitioning/partdesc.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "pg_version_constants.h" #include "distributed/adaptive_executor.h" #include "distributed/citus_nodes.h" #include "distributed/citus_ruleutils.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_physical_planner.h" #include "distributed/relay_utility.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" static char * PartitionBound(Oid partitionId); static Relation try_relation_open_nolock(Oid relationId); static List * CreateFixPartitionConstraintsTaskList(Oid relationId); static List * WorkerFixPartitionConstraintCommandList(Oid relationId, uint64 shardId, List *checkConstraintList); static void CreateFixPartitionShardIndexNames(Oid parentRelationId, Oid partitionRelationId, Oid parentIndexOid); static List * WorkerFixPartitionShardIndexNamesCommandList(uint64 parentShardId, List *indexIdList, Oid partitionRelationId); static List * WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex(char * qualifiedParentShardIndexName, Oid parentIndexId, Oid partitionRelationId); static List * WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(Oid partitionIndexId, char * qualifiedParentShardIndexName, Oid partitionId); static List * CheckConstraintNameListForRelation(Oid relationId); static bool RelationHasConstraint(Oid relationId, char *constraintName); static char * RenameConstraintCommand(Oid relationId, char *constraintName, char *newConstraintName); PG_FUNCTION_INFO_V1(fix_pre_citus10_partitioned_table_constraint_names); PG_FUNCTION_INFO_V1(worker_fix_pre_citus10_partitioned_table_constraint_names); PG_FUNCTION_INFO_V1(fix_partition_shard_index_names); PG_FUNCTION_INFO_V1(worker_fix_partition_shard_index_names); /* * fix_pre_citus10_partitioned_table_constraint_names fixes the constraint names of * partitioned table shards on workers. * * Constraint names for partitioned table shards should have shardId suffixes if and only * if they are unique or foreign key constraints. We mistakenly appended shardIds to * constraint names on ALTER TABLE dist_part_table ADD CONSTRAINT .. queries prior to * Citus 10. fix_pre_citus10_partitioned_table_constraint_names determines if this is the * case, and renames constraints back to their original names on shards. */ Datum fix_pre_citus10_partitioned_table_constraint_names(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); EnsureCoordinator(); if (!PartitionedTable(relationId)) { ereport(ERROR, (errmsg("could not fix partition constraints: " "relation does not exist or is not partitioned"))); } if (!IsCitusTable(relationId)) { ereport(ERROR, (errmsg("fix_pre_citus10_partitioned_table_constraint_names can " "only be called for distributed partitioned tables"))); } List *taskList = CreateFixPartitionConstraintsTaskList(relationId); /* do not do anything if there are no constraints that should be fixed */ if (taskList != NIL) { bool localExecutionSupported = true; ExecuteUtilityTaskList(taskList, localExecutionSupported); } PG_RETURN_VOID(); } /* * worker_fix_pre_citus10_partitioned_table_constraint_names fixes the constraint names on a worker given a shell * table name and shard id. */ Datum worker_fix_pre_citus10_partitioned_table_constraint_names(PG_FUNCTION_ARGS) { Oid relationId = PG_GETARG_OID(0); int64 shardId = PG_GETARG_INT32(1); text *constraintNameText = PG_GETARG_TEXT_P(2); if (!PartitionedTable(relationId)) { ereport(ERROR, (errmsg("could not fix partition constraints: " "relation does not exist or is not partitioned"))); } char *constraintName = text_to_cstring(constraintNameText); char *shardIdAppendedConstraintName = pstrdup(constraintName); AppendShardIdToName(&shardIdAppendedConstraintName, shardId); /* if shardId was appended to the constraint name, rename back to original */ if (RelationHasConstraint(relationId, shardIdAppendedConstraintName)) { char *renameConstraintDDLCommand = RenameConstraintCommand(relationId, shardIdAppendedConstraintName, constraintName); ExecuteAndLogUtilityCommand(renameConstraintDDLCommand); } PG_RETURN_VOID(); } /* * fix_partition_shard_index_names fixes the index names of shards of partitions of * partitioned tables on workers. If the input is a partition rather than a partitioned * table, we only fix the index names of shards of that particular partition. * * When running CREATE INDEX on parent_table, we didn't explicitly create the index on * each partition as well. Postgres created indexes for partitions in the coordinator, * and also in the workers. Actually, Postgres auto-generates index names when auto-creating * indexes on each partition shard of the parent shards. If index name is too long, it * truncates the name and adds _idx postfix to it. However, when truncating the name, the * shardId of the partition shard can be lost. This may result in the same index name used for * the partition shell table and one of the partition shards. * For more details, check issue #4962 https://github.com/citusdata/citus/issues/4962 * * fix_partition_shard_index_names renames indexes of shards of partition tables to include * the shardId at the end of the name, regardless of whether index name was long or short * As a result there will be no index name ending in _idx, rather all will end in _{shardid} * * Algorithm is: * foreach parentShard in shardListOfParentTableId: * foreach parentIndex on parent: * generate qualifiedParentShardIndexName -> parentShardIndex * foreach inheritedPartitionIndex on parentIndex: * get table relation of inheritedPartitionIndex -> partitionId * foreach partitionShard in shardListOfPartitionid: * generate qualifiedPartitionShardName -> partitionShard * generate newPartitionShardIndexName * (the following happens in the worker node) * foreach inheritedPartitionShardIndex on parentShardIndex: * if table relation of inheritedPartitionShardIndex is partitionShard: * if inheritedPartitionShardIndex does not have proper name: * Rename(inheritedPartitionShardIndex, newPartitionShardIndexName) * break */ Datum fix_partition_shard_index_names(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); EnsureCoordinator(); Oid relationId = PG_GETARG_OID(0); Oid parentIndexOid = InvalidOid; /* fix all the indexes */ if (!IsCitusTable(relationId)) { ereport(ERROR, (errmsg("fix_partition_shard_index_names can only be called " "for Citus tables"))); } EnsureTableOwner(relationId); FixPartitionShardIndexNames(relationId, parentIndexOid); /* * This UDF is called from fix_all_partition_shard_index_names() which iterates * over all the partitioned tables. There is no need to hold all the distributed * table metadata until the end of the transaction for the input table. */ CitusTableCacheFlushInvalidatedEntries(); PG_RETURN_VOID(); } /* * worker_fix_partition_shard_index_names fixes the index name of the index on given * partition shard that has parent the given parent index. * The parent index should be the index of a shard of a distributed partitioned table. */ Datum worker_fix_partition_shard_index_names(PG_FUNCTION_ARGS) { Oid parentShardIndexId = PG_GETARG_OID(0); text *partitionShardName = PG_GETARG_TEXT_P(1); /* resolve partitionShardId from passed in schema and partition shard name */ List *partitionShardNameList = textToQualifiedNameList(partitionShardName); RangeVar *partitionShard = makeRangeVarFromNameList(partitionShardNameList); /* lock the relation with the lock mode */ bool missing_ok = true; Oid partitionShardId = RangeVarGetRelid(partitionShard, NoLock, missing_ok); if (!OidIsValid(partitionShardId)) { PG_RETURN_VOID(); } CheckCitusVersion(ERROR); EnsureTableOwner(partitionShardId); text *newPartitionShardIndexNameText = PG_GETARG_TEXT_P(2); char *newPartitionShardIndexName = text_to_cstring( newPartitionShardIndexNameText); if (!has_subclass(parentShardIndexId)) { ereport(ERROR, (errmsg("could not fix child index names: " "index is not partitioned"))); } List *partitionShardIndexIds = find_inheritance_children(parentShardIndexId, ShareRowExclusiveLock); Oid partitionShardIndexId = InvalidOid; foreach_declared_oid(partitionShardIndexId, partitionShardIndexIds) { if (IndexGetRelation(partitionShardIndexId, false) == partitionShardId) { char *partitionShardIndexName = get_rel_name(partitionShardIndexId); if (ExtractShardIdFromTableName(partitionShardIndexName, missing_ok) == INVALID_SHARD_ID) { /* * ExtractShardIdFromTableName will return INVALID_SHARD_ID if * partitionShardIndexName doesn't end in _shardid. In that case, * we want to rename this partition shard index to newPartitionShardIndexName, * which ends in _shardid, hence we maintain naming consistency: * we can reach this partition shard index by conventional Citus naming */ RenameStmt *stmt = makeNode(RenameStmt); stmt->renameType = OBJECT_INDEX; stmt->missing_ok = false; char *idxNamespace = get_namespace_name(get_rel_namespace( partitionShardIndexId)); stmt->relation = makeRangeVar(idxNamespace, partitionShardIndexName, -1); stmt->newname = newPartitionShardIndexName; RenameRelation(stmt); } break; } } PG_RETURN_VOID(); } /* * FixPartitionShardIndexNames gets a relationId. The input relationId should be * either a parent or partition table. If it is a parent table, then all the * index names on all the partitions are fixed. If it is a partition, only the * specific partition is fixed. * * The second parentIndexOid parameter is optional. If provided a valid Oid, only * that specific index name is fixed. */ void FixPartitionShardIndexNames(Oid relationId, Oid parentIndexOid) { Relation relation = try_relation_open(relationId, AccessShareLock); if (relation == NULL) { ereport(NOTICE, (errmsg("relation with OID %u does not exist, skipping", relationId))); return; } /* at this point, we should only be dealing with Citus tables */ Assert(IsCitusTable(relationId)); Oid parentRelationId = InvalidOid; Oid partitionRelationId = InvalidOid; if (relation->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) { parentRelationId = relationId; } else if (PartitionTable(relationId)) { parentRelationId = PartitionParentOid(relationId); partitionRelationId = relationId; } else { char *relname = pstrdup(RelationGetRelationName(relation)); relation_close(relation, NoLock); ereport(ERROR, (errmsg("Fixing shard index names is only applicable to " "partitioned tables or partitions, " "and \"%s\" is neither", relname))); } CreateFixPartitionShardIndexNames(parentRelationId, partitionRelationId, parentIndexOid); relation_close(relation, NoLock); } /* * CreateFixPartitionConstraintsTaskList goes over all the partitions of a distributed * partitioned table, and creates the list of tasks to execute * worker_fix_pre_citus10_partitioned_table_constraint_names UDF on worker nodes. */ static List * CreateFixPartitionConstraintsTaskList(Oid relationId) { List *taskList = NIL; /* enumerate the tasks when putting them to the taskList */ int taskId = 1; List *checkConstraintList = CheckConstraintNameListForRelation(relationId); /* early exit if the relation does not have any check constraints */ if (checkConstraintList == NIL) { return NIL; } List *shardIntervalList = LoadShardIntervalList(relationId); /* lock metadata before getting placement lists */ LockShardListMetadata(shardIntervalList, ShareLock); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { uint64 shardId = shardInterval->shardId; List *queryStringList = WorkerFixPartitionConstraintCommandList(relationId, shardId, checkConstraintList); Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = DDL_TASK; SetTaskQueryStringList(task, queryStringList); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); taskList = lappend(taskList, task); } return taskList; } /* * CheckConstraintNameListForRelation returns a list of names of CHECK constraints * for a relation. */ static List * CheckConstraintNameListForRelation(Oid relationId) { List *constraintNameList = NIL; int scanKeyCount = 2; ScanKeyData scanKey[2]; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); ScanKeyInit(&scanKey[1], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_CHECK)); bool useIndex = false; SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, InvalidOid, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(heapTuple)) { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); char *constraintName = NameStr(constraintForm->conname); constraintNameList = lappend(constraintNameList, pstrdup(constraintName)); heapTuple = systable_getnext(scanDescriptor); } systable_endscan(scanDescriptor); table_close(pgConstraint, NoLock); return constraintNameList; } /* * WorkerFixPartitionConstraintCommandList creates a list of queries that will fix * all check constraint names of a shard. */ static List * WorkerFixPartitionConstraintCommandList(Oid relationId, uint64 shardId, List *checkConstraintList) { List *commandList = NIL; Oid schemaId = get_rel_namespace(relationId); char *schemaName = get_namespace_name(schemaId); char *relationName = get_rel_name(relationId); char *shardRelationName = pstrdup(relationName); /* build shard relation name */ AppendShardIdToName(&shardRelationName, shardId); char *quotedShardName = quote_qualified_identifier(schemaName, shardRelationName); char *constraintName = NULL; foreach_declared_ptr(constraintName, checkConstraintList) { StringInfo shardQueryString = makeStringInfo(); appendStringInfo(shardQueryString, "SELECT worker_fix_pre_citus10_partitioned_table_constraint_names(%s::regclass, " UINT64_FORMAT ", %s::text)", quote_literal_cstr(quotedShardName), shardId, quote_literal_cstr(constraintName)); commandList = lappend(commandList, shardQueryString->data); } return commandList; } /* * CreateFixPartitionShardIndexNamesTaskList goes over all the * indexes of a distributed partitioned table unless parentIndexOid * is valid. If it is valid, only the given index is processed. * * The function creates the list of tasks to execute * worker_fix_partition_shard_index_names() on worker nodes. * * When the partitionRelationId is a valid Oid, the function only operates on the * given partition. Otherwise, the function create tasks for all the partitions. * * So, for example, if a new partition is created, we only need to fix only for the * new partition, hence partitionRelationId should be a valid Oid. However, if a new * index/constraint is created on the parent, we should fix all the partitions, hence * partitionRelationId should be InvalidOid. * * As a reflection of the above, we always create parent_table_shard_count tasks. * When we need to fix all the partitions, each task with parent_indexes_count * times partition_count query strings. When we need to fix a single * partition each task will have parent_indexes_count query strings. When we need * to fix a single index, parent_indexes_count becomes 1. */ static void CreateFixPartitionShardIndexNames(Oid parentRelationId, Oid partitionRelationId, Oid parentIndexOid) { List *partitionList = PartitionList(parentRelationId); if (partitionList == NIL) { /* early exit if the parent relation does not have any partitions */ return; } Relation parentRelation = RelationIdGetRelation(parentRelationId); if (!RelationIsValid(parentRelation)) { ereport(ERROR, (errmsg("could not open relation with OID %u", parentRelationId))); } List *parentIndexIdList = NIL; if (parentIndexOid != InvalidOid) { parentIndexIdList = list_make1_oid(parentIndexOid); } else { parentIndexIdList = RelationGetIndexList(parentRelation); } if (parentIndexIdList == NIL) { /* early exit if the parent relation does not have any indexes */ RelationClose(parentRelation); return; } /* * Lock shard metadata, if a specific partition is provided, lock that. Otherwise, * lock all partitions. */ if (OidIsValid(partitionRelationId)) { /* if a partition was provided we only need to lock that partition's metadata */ List *partitionShardIntervalList = LoadShardIntervalList(partitionRelationId); LockShardListMetadata(partitionShardIntervalList, ShareLock); } else { Oid partitionId = InvalidOid; foreach_declared_oid(partitionId, partitionList) { List *partitionShardIntervalList = LoadShardIntervalList(partitionId); LockShardListMetadata(partitionShardIntervalList, ShareLock); } } List *parentShardIntervalList = LoadShardIntervalList(parentRelationId); /* lock metadata before getting placement lists */ LockShardListMetadata(parentShardIntervalList, ShareLock); MemoryContext localContext = AllocSetContextCreate(CurrentMemoryContext, "CreateFixPartitionShardIndexNames", ALLOCSET_DEFAULT_SIZES); MemoryContext oldContext = MemoryContextSwitchTo(localContext); int taskId = 1; ShardInterval *parentShardInterval = NULL; foreach_declared_ptr(parentShardInterval, parentShardIntervalList) { uint64 parentShardId = parentShardInterval->shardId; List *queryStringList = WorkerFixPartitionShardIndexNamesCommandList(parentShardId, parentIndexIdList, partitionRelationId); if (queryStringList != NIL) { Task *task = CitusMakeNode(Task); task->jobId = INVALID_JOB_ID; task->taskId = taskId++; task->taskType = DDL_TASK; char *prefix = "SELECT pg_catalog.citus_run_local_command($$"; char *postfix = "$$)"; char *string = StringJoinParams(queryStringList, ';', prefix, postfix); SetTaskQueryString(task, string); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = parentShardId; task->taskPlacementList = ActiveShardPlacementList(parentShardId); bool localExecutionSupported = true; ExecuteUtilityTaskList(list_make1(task), localExecutionSupported); } /* after every iteration, clean-up all the memory associated with it */ MemoryContextReset(localContext); } MemoryContextSwitchTo(oldContext); RelationClose(parentRelation); } /* * WorkerFixPartitionShardIndexNamesCommandList creates a list of queries that will fix * all child index names of parent indexes on given shard of parent partitioned table. */ static List * WorkerFixPartitionShardIndexNamesCommandList(uint64 parentShardId, List *parentIndexIdList, Oid partitionRelationId) { List *commandList = NIL; Oid parentIndexId = InvalidOid; foreach_declared_oid(parentIndexId, parentIndexIdList) { if (!has_subclass(parentIndexId)) { continue; } /* * Get the qualified name of the corresponding index of given parent index * in the parent shard with given parentShardId */ char *parentIndexName = get_rel_name(parentIndexId); char *parentShardIndexName = pstrdup(parentIndexName); AppendShardIdToName(&parentShardIndexName, parentShardId); Oid schemaId = get_rel_namespace(parentIndexId); char *schemaName = get_namespace_name(schemaId); char *qualifiedParentShardIndexName = quote_qualified_identifier(schemaName, parentShardIndexName); List *commands = WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex( qualifiedParentShardIndexName, parentIndexId, partitionRelationId); commandList = list_concat(commandList, commands); } return commandList; } /* * WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex creates a list * of queries that will fix the child index names of given index on shard * of parent partitioned table. * * In case a partition was provided as argument (partitionRelationId isn't InvalidOid) * the list of queries will include only the child indexes whose relation is the * given partition. Otherwise, all the partitions are included. */ static List * WorkerFixPartitionShardIndexNamesCommandListForParentShardIndex(char * qualifiedParentShardIndexName, Oid parentIndexId, Oid partitionRelationId) { List *commandList = NIL; /* * Get the list of all partition indexes that are children of current * index on parent */ List *partitionIndexIds = find_inheritance_children(parentIndexId, ShareRowExclusiveLock); bool addAllPartitions = (partitionRelationId == InvalidOid); Oid partitionIndexId = InvalidOid; foreach_declared_oid(partitionIndexId, partitionIndexIds) { Oid partitionId = IndexGetRelation(partitionIndexId, false); if (addAllPartitions || partitionId == partitionRelationId) { List *commands = WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex( partitionIndexId, qualifiedParentShardIndexName, partitionId); commandList = list_concat(commandList, commands); } } return commandList; } /* * WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex creates a list of queries that will fix * all child index names of given index on shard of parent partitioned table, whose table relation is a shard * of the partition that is the table relation of given partitionIndexId, which is partitionId */ static List * WorkerFixPartitionShardIndexNamesCommandListForPartitionIndex(Oid partitionIndexId, char * qualifiedParentShardIndexName, Oid partitionId) { List *commandList = NIL; /* get info for this partition relation of this index*/ char *partitionIndexName = get_rel_name(partitionIndexId); char *partitionName = get_rel_name(partitionId); char *partitionSchemaName = get_namespace_name(get_rel_namespace(partitionId)); List *partitionShardIntervalList = LoadShardIntervalList(partitionId); ShardInterval *partitionShardInterval = NULL; foreach_declared_ptr(partitionShardInterval, partitionShardIntervalList) { /* * Prepare commands for each shard of current partition * to fix the index name that corresponds to the * current parent index name */ uint64 partitionShardId = partitionShardInterval->shardId; /* get qualified partition shard name */ char *partitionShardName = pstrdup(partitionName); AppendShardIdToName(&partitionShardName, partitionShardId); char *qualifiedPartitionShardName = quote_qualified_identifier( partitionSchemaName, partitionShardName); /* generate the new correct index name */ char *newPartitionShardIndexName = pstrdup(partitionIndexName); AppendShardIdToName(&newPartitionShardIndexName, partitionShardId); /* create worker_fix_partition_shard_index_names command */ StringInfo shardQueryString = makeStringInfo(); appendStringInfo(shardQueryString, "SELECT worker_fix_partition_shard_index_names(%s::regclass, %s, %s)", quote_literal_cstr(qualifiedParentShardIndexName), quote_literal_cstr(qualifiedPartitionShardName), quote_literal_cstr(newPartitionShardIndexName)); commandList = lappend(commandList, shardQueryString->data); } return commandList; } /* * RelationHasConstraint checks if a relation has a constraint with a given name. */ static bool RelationHasConstraint(Oid relationId, char *constraintName) { bool found = false; int scanKeyCount = 2; ScanKeyData scanKey[2]; Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); ScanKeyInit(&scanKey[1], Anum_pg_constraint_conname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(constraintName)); bool useIndex = false; SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, InvalidOid, useIndex, NULL, scanKeyCount, scanKey); HeapTuple heapTuple = systable_getnext(scanDescriptor); if (HeapTupleIsValid(heapTuple)) { found = true; } systable_endscan(scanDescriptor); table_close(pgConstraint, NoLock); return found; } /* * RenameConstraintCommand creates the query string that will rename a constraint */ static char * RenameConstraintCommand(Oid relationId, char *constraintName, char *newConstraintName) { char *qualifiedRelationName = generate_qualified_relation_name(relationId); const char *quotedConstraintName = quote_identifier(constraintName); const char *quotedNewConstraintName = quote_identifier(newConstraintName); StringInfo renameCommand = makeStringInfo(); appendStringInfo(renameCommand, "ALTER TABLE %s RENAME CONSTRAINT %s TO %s", qualifiedRelationName, quotedConstraintName, quotedNewConstraintName); return renameCommand->data; } /* * Returns true if the given relation is a partitioned table. */ bool PartitionedTable(Oid relationId) { Relation rel = try_relation_open(relationId, AccessShareLock); /* don't error out for tables that are dropped */ if (rel == NULL) { return false; } bool partitionedTable = false; if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) { partitionedTable = true; } /* keep the lock */ table_close(rel, NoLock); return partitionedTable; } /* * Returns true if the given relation is a partitioned table. The function * doesn't acquire any locks on the input relation, thus the caller is * reponsible for holding the appropriate locks. */ bool PartitionedTableNoLock(Oid relationId) { Relation rel = try_relation_open_nolock(relationId); bool partitionedTable = false; /* don't error out for tables that are dropped */ if (rel == NULL) { return false; } if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) { partitionedTable = true; } /* keep the lock */ table_close(rel, NoLock); return partitionedTable; } /* * Returns true if the given relation is a partition. */ bool PartitionTable(Oid relationId) { Relation rel = try_relation_open(relationId, AccessShareLock); /* don't error out for tables that are dropped */ if (rel == NULL) { return false; } bool partitionTable = rel->rd_rel->relispartition; /* keep the lock */ table_close(rel, NoLock); return partitionTable; } /* * Returns true if the given relation is a partition. The function * doesn't acquire any locks on the input relation, thus the caller is * reponsible for holding the appropriate locks. */ bool PartitionTableNoLock(Oid relationId) { Relation rel = try_relation_open_nolock(relationId); /* don't error out for tables that are dropped */ if (rel == NULL) { return false; } bool partitionTable = rel->rd_rel->relispartition; /* keep the lock */ table_close(rel, NoLock); return partitionTable; } /* * try_relation_open_nolock opens a relation with given relationId without * acquiring locks. PostgreSQL's try_relation_open() asserts that caller * has already acquired a lock on the relation, which we don't always do. * * ATTENTION: * 1. Sync this with try_relation_open(). It hasn't changed for 10 to 12 * releases though. * 2. We should remove this after we fix the locking/distributed deadlock * issues with MX Truncate. See https://github.com/citusdata/citus/pull/2894 * for more discussion. */ static Relation try_relation_open_nolock(Oid relationId) { if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relationId))) { return NULL; } Relation relation = RelationIdGetRelation(relationId); if (!RelationIsValid(relation)) { return NULL; } pgstat_init_relation(relation); return relation; } /* * IsChildTable returns true if the table is inherited. Note that * partition tables inherites by default. However, this function * returns false if the given table is a partition. */ bool IsChildTable(Oid relationId) { ScanKeyData key[1]; HeapTuple inheritsTuple = NULL; bool tableInherits = false; Relation pgInherits = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scan = systable_beginscan(pgInherits, InvalidOid, false, NULL, 1, key); while ((inheritsTuple = systable_getnext(scan)) != NULL) { Oid inheritedRelationId = ((Form_pg_inherits) GETSTRUCT(inheritsTuple))->inhrelid; if (relationId == inheritedRelationId) { tableInherits = true; break; } } systable_endscan(scan); table_close(pgInherits, AccessShareLock); if (tableInherits && PartitionTable(relationId)) { tableInherits = false; } return tableInherits; } /* * IsParentTable returns true if the table is inherited. Note that * partitioned tables inherited by default. However, this function * returns false if the given table is a partitioned table. */ bool IsParentTable(Oid relationId) { ScanKeyData key[1]; bool tableInherited = false; Relation pgInherits = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_inherits_inhparent, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); SysScanDesc scan = systable_beginscan(pgInherits, InheritsParentIndexId, true, NULL, 1, key); if (systable_getnext(scan) != NULL) { tableInherited = true; } systable_endscan(scan); table_close(pgInherits, AccessShareLock); Relation relation = try_relation_open(relationId, AccessShareLock); if (relation == NULL) { ereport(ERROR, (errmsg("relation with OID %u does not exist", relationId))); } if (tableInherited && PartitionedTableNoLock(relationId)) { tableInherited = false; } relation_close(relation, AccessShareLock); return tableInherited; } /* * Wrapper around get_partition_parent * * Note: Because this function assumes that the relation whose OID is passed * as an argument will have precisely one parent, it should only be called * when it is known that the relation is a partition. */ Oid PartitionParentOid(Oid partitionOid) { Oid partitionParentOid = get_partition_parent(partitionOid, false); return partitionParentOid; } /* * PartitionWithLongestNameRelationId is a utility function that returns the * oid of the partition table that has the longest name in terms of number of * characters. */ Oid PartitionWithLongestNameRelationId(Oid parentRelationId) { Oid longestNamePartitionId = InvalidOid; int longestNameLength = 0; List *partitionList = PartitionList(parentRelationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { char *partitionName = get_rel_name(partitionRelationId); int partitionNameLength = strnlen(partitionName, NAMEDATALEN); if (partitionNameLength > longestNameLength) { longestNamePartitionId = partitionRelationId; longestNameLength = partitionNameLength; } } return longestNamePartitionId; } /* * Takes a parent relation and returns Oid list of its partitions. The * function errors out if the given relation is not a parent. */ List * PartitionList(Oid parentRelationId) { Relation rel = table_open(parentRelationId, AccessShareLock); List *partitionList = NIL; if (!PartitionedTable(parentRelationId)) { char *relationName = get_rel_name(parentRelationId); ereport(ERROR, (errmsg("\"%s\" is not a parent table", relationName))); } PartitionDesc partDesc = RelationGetPartitionDesc(rel, true); Assert(partDesc != NULL); int partitionCount = partDesc->nparts; for (int partitionIndex = 0; partitionIndex < partitionCount; ++partitionIndex) { partitionList = lappend_oid(partitionList, partDesc->oids[partitionIndex]); } /* keep the lock */ table_close(rel, NoLock); return partitionList; } /* * GenerateDetachPartitionCommand gets a partition table and returns * "ALTER TABLE parent_table DETACH PARTITION partitionName" command. */ char * GenerateDetachPartitionCommand(Oid partitionTableId) { StringInfo detachPartitionCommand = makeStringInfo(); if (!PartitionTable(partitionTableId)) { char *relationName = get_rel_name(partitionTableId); ereport(ERROR, (errmsg("\"%s\" is not a partition", relationName))); } Oid parentId = get_partition_parent(partitionTableId, false); char *tableQualifiedName = generate_qualified_relation_name(partitionTableId); char *parentTableQualifiedName = generate_qualified_relation_name(parentId); appendStringInfo(detachPartitionCommand, "ALTER TABLE IF EXISTS %s DETACH PARTITION %s;", parentTableQualifiedName, tableQualifiedName); return detachPartitionCommand->data; } /* * GenerateDetachPartitionCommandRelationIdList returns the necessary command list to * detach the given partitions from their parents. */ List * GenerateDetachPartitionCommandRelationIdList(List *relationIds) { List *detachPartitionCommands = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIds) { Assert(PartitionTable(relationId)); char *detachCommand = GenerateDetachPartitionCommand(relationId); detachPartitionCommands = lappend(detachPartitionCommands, detachCommand); } return detachPartitionCommands; } /* * GenereatePartitioningInformation returns the partitioning type and partition column * for the given parent table in the form of "PARTITION TYPE (partitioning column(s)/expression(s))". */ char * GeneratePartitioningInformation(Oid parentTableId) { char *partitionBoundCString = ""; if (!PartitionedTable(parentTableId)) { char *relationName = get_rel_name(parentTableId); ereport(ERROR, (errmsg("\"%s\" is not a parent table", relationName))); } Datum partitionBoundDatum = DirectFunctionCall1(pg_get_partkeydef, ObjectIdGetDatum(parentTableId)); partitionBoundCString = TextDatumGetCString(partitionBoundDatum); return partitionBoundCString; } /* * GenerateAttachShardPartitionCommand generates command to attach a child table * table to its parent in a partitioning hierarchy. */ char * GenerateAttachShardPartitionCommand(ShardInterval *shardInterval) { Oid schemaId = get_rel_namespace(shardInterval->relationId); char *schemaName = get_namespace_name(schemaId); char *escapedSchemaName = quote_literal_cstr(schemaName); char *command = GenerateAlterTableAttachPartitionCommand(shardInterval->relationId); char *escapedCommand = quote_literal_cstr(command); int shardIndex = ShardIndex(shardInterval); StringInfo attachPartitionCommand = makeStringInfo(); Oid parentRelationId = PartitionParentOid(shardInterval->relationId); if (parentRelationId == InvalidOid) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot attach partition"), errdetail("Referenced relation cannot be found."))); } Oid parentSchemaId = get_rel_namespace(parentRelationId); char *parentSchemaName = get_namespace_name(parentSchemaId); char *escapedParentSchemaName = quote_literal_cstr(parentSchemaName); uint64 parentShardId = ColocatedShardIdInRelation(parentRelationId, shardIndex); appendStringInfo(attachPartitionCommand, WORKER_APPLY_INTER_SHARD_DDL_COMMAND, parentShardId, escapedParentSchemaName, shardInterval->shardId, escapedSchemaName, escapedCommand); return attachPartitionCommand->data; } /* * GenerateAlterTableAttachPartitionCommand returns the necessary command to * attach the given partition to its parent. */ char * GenerateAlterTableAttachPartitionCommand(Oid partitionTableId) { StringInfo createPartitionCommand = makeStringInfo(); if (!PartitionTable(partitionTableId)) { char *relationName = get_rel_name(partitionTableId); ereport(ERROR, (errmsg("\"%s\" is not a partition", relationName))); } Oid parentId = get_partition_parent(partitionTableId, false); char *tableQualifiedName = generate_qualified_relation_name(partitionTableId); char *parentTableQualifiedName = generate_qualified_relation_name(parentId); char *partitionBoundCString = PartitionBound(partitionTableId); appendStringInfo(createPartitionCommand, "ALTER TABLE %s ATTACH PARTITION %s %s;", parentTableQualifiedName, tableQualifiedName, partitionBoundCString); return createPartitionCommand->data; } /* * GenerateAttachPartitionCommandRelationIdList returns the necessary command list to * attach the given partitions to their parents. */ List * GenerateAttachPartitionCommandRelationIdList(List *relationIds) { List *attachPartitionCommands = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIds) { char *attachCommand = GenerateAlterTableAttachPartitionCommand(relationId); attachPartitionCommands = lappend(attachPartitionCommands, attachCommand); } return attachPartitionCommands; } /* * This function heaviliy inspired from RelationBuildPartitionDesc() * which is avaliable in src/backend/catalog/partition.c. * * The function simply reads the pg_class and gets the partition bound. * Later, converts it to text format and returns. */ static char * PartitionBound(Oid partitionId) { bool isnull = false; HeapTuple tuple = SearchSysCache1(RELOID, partitionId); if (!HeapTupleIsValid(tuple)) { elog(ERROR, "cache lookup failed for relation %u", partitionId); } /* * It is possible that the pg_class tuple of a partition has not been * updated yet to set its relpartbound field. The only case where * this happens is when we open the parent relation to check using its * partition descriptor that a new partition's bound does not overlap * some existing partition. */ if (!((Form_pg_class) GETSTRUCT(tuple))->relispartition) { ReleaseSysCache(tuple); return ""; } Datum datum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relpartbound, &isnull); Assert(!isnull); Datum partitionBoundDatum = DirectFunctionCall2(pg_get_expr, datum, ObjectIdGetDatum(partitionId)); char *partitionBoundString = TextDatumGetCString(partitionBoundDatum); ReleaseSysCache(tuple); return partitionBoundString; } /* * ListShardsUnderParentRelation returns a list of ShardInterval for every * shard under a given relation, meaning it includes the shards of child * tables in a partitioning hierarchy. */ List * ListShardsUnderParentRelation(Oid relationId) { List *shardList = LoadShardIntervalList(relationId); if (PartitionedTable(relationId)) { List *partitionList = PartitionList(relationId); Oid partitionRelationId = InvalidOid; foreach_declared_oid(partitionRelationId, partitionList) { List *childShardList = ListShardsUnderParentRelation(partitionRelationId); shardList = list_concat(shardList, childShardList); } } return shardList; } ================================================ FILE: src/backend/distributed/utils/namespace_utils.c ================================================ /*------------------------------------------------------------------------- * * namespace_utils.c * * Utility functions related to namespace changes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "utils/guc.h" #include "utils/regproc.h" #include "distributed/namespace_utils.h" /* * We use the equivalent of a function SET option to allow the setting to * persist for the exact duration of the transaction, guc.c takes care of * undoing the setting on error. * * We set search_path to "pg_catalog" instead of "" to expose useful utilities. */ int PushEmptySearchPath() { int saveNestLevel = NewGUCNestLevel(); (void) set_config_option("search_path", "pg_catalog", PGC_USERSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, 0, false); return saveNestLevel; } /* * Restore the GUC variable search_path we set in PushEmptySearchPath */ void PopEmptySearchPath(int saveNestLevel) { AtEOXact_GUC(true, saveNestLevel); } ================================================ FILE: src/backend/distributed/utils/param_utils.c ================================================ /*------------------------------------------------------------------------- * * param_utils.c * Utilities to process paramaters. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/bitmapset.h" #include "nodes/nodeFuncs.h" #include "nodes/nodes.h" #include "nodes/params.h" #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "distributed/param_utils.h" /* * IsExternParamUsedInQuery returns true if the passed in paramId * is used in the query, false otherwise. */ bool GetParamsUsedInQuery(Node *expression, Bitmapset **paramBitmap) { if (expression == NULL) { return false; } if (IsA(expression, Param)) { Param *param = (Param *) expression; int paramId = param->paramid; /* only care about user supplied parameters */ if (param->paramkind != PARAM_EXTERN) { return false; } /* Found a parameter, mark it in the bitmap and continue */ *paramBitmap = bms_add_member(*paramBitmap, paramId); /* Continue searching */ return false; } /* keep traversing */ if (IsA(expression, Query)) { return query_tree_walker((Query *) expression, GetParamsUsedInQuery, paramBitmap, 0); } else { return expression_tree_walker(expression, GetParamsUsedInQuery, paramBitmap); } } /* * MarkUnreferencedExternParams marks parameter's type to zero if the * parameter is not used in the query. */ void MarkUnreferencedExternParams(Node *expression, ParamListInfo boundParams) { int parameterCount = boundParams->numParams; Bitmapset *paramBitmap = NULL; /* Fetch all parameters used in the query */ GetParamsUsedInQuery(expression, ¶mBitmap); /* Check for any missing parameters */ for (int parameterNum = 1; parameterNum <= parameterCount; parameterNum++) { if (!bms_is_member(parameterNum, paramBitmap)) { boundParams->params[parameterNum - 1].ptype = 0; } } } ================================================ FILE: src/backend/distributed/utils/priority.c ================================================ /*------------------------------------------------------------------------- * * priority.c * Utilities for managing CPU priority. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include #include #include #include #include #include "postgres.h" #include "distributed/priority.h" int CpuPriority = 0; int CpuPriorityLogicalRepSender = CPU_PRIORITY_INHERIT; int MaxHighPriorityBackgroundProcesess = 2; /* * SetOwnPriority changes the CPU priority of the current backend to the given * priority. If the OS disallows us to set the priority to the given value, we * only warn about it. */ void SetOwnPriority(int priority) { if (priority == CPU_PRIORITY_INHERIT) { return; } if (setpriority(PRIO_PROCESS, getpid(), priority) == -1) { ereport(WARNING, ( errmsg("could not set cpu priority to %d: %m", priority), errhint("Try changing the 'nice' resource limit by changing " "/etc/security/limits.conf for the postgres user " "and/or by setting LimitNICE in your the systemd " "service file (depending on how you start " "postgres)." ))); } } /* * GetOwnPriority returns the current CPU priority value of the backend. */ int GetOwnPriority(void) { errno = 0; int result = getpriority(PRIO_PROCESS, getpid()); /* * We explicitly check errno too because getpriority can return -1 on * success too, if the actual priority value is -1 */ if (result == -1 && errno != 0) { ereport(WARNING, (errmsg("could not get current cpu priority value, " "assuming 0: %m"))); return 0; } return result; } ================================================ FILE: src/backend/distributed/utils/query_utils.c ================================================ /*------------------------------------------------------------------------- * * query_utils.c * * Query-walker utility functions to be used to construct a logical plan * tree from the given query tree structure. * * Copyright (c), Citus Data, Inc. * **---------------------------------------------------------------------------- */ #include "postgres.h" #include "catalog/pg_class.h" #include "nodes/nodeFuncs.h" #include "nodes/primnodes.h" #include "distributed/listutils.h" #include "distributed/query_utils.h" #include "distributed/version_compat.h" static bool CitusQueryableRangeTableRelation(RangeTblEntry *rangeTableEntry); /* * ExtractRangeTableList walks over a tree to gather entries. * Execution is parameterized by passing walkerMode flag via ExtractRangeTableWalkerContext * as we cannot pass more than one parameter to query_tree_walker */ bool ExtractRangeTableList(Node *node, ExtractRangeTableWalkerContext *context) { /* get parameters from context */ List **rangeTableRelationList = context->rangeTableList; ExtractRangeTableMode walkerMode = context->walkerMode; bool walkIsComplete = false; if (node == NULL) { return false; } if (IsA(node, RangeTblEntry)) { RangeTblEntry *rangeTable = (RangeTblEntry *) node; if (walkerMode == EXTRACT_ALL_ENTRIES || (walkerMode == EXTRACT_RELATION_ENTRIES && CitusQueryableRangeTableRelation(rangeTable))) { (*rangeTableRelationList) = lappend(*rangeTableRelationList, rangeTable); } } else if (IsA(node, Query)) { Query *query = (Query *) node; if (query->hasSubLinks || query->cteList || query->setOperations) { /* descend into all parts of the query */ walkIsComplete = query_tree_walker(query, ExtractRangeTableList, context, QTW_EXAMINE_RTES_BEFORE); } else { /* descend only into RTEs */ walkIsComplete = range_table_walker(query->rtable, ExtractRangeTableList, context, QTW_EXAMINE_RTES_BEFORE); } } else { walkIsComplete = expression_tree_walker(node, ExtractRangeTableList, context); } return walkIsComplete; } /* * CitusQueryableRangeTableRelation returns true if the input range table * entry is a relation and it can be used in a distributed query, including * local tables and materialized views as well. */ static bool CitusQueryableRangeTableRelation(RangeTblEntry *rangeTableEntry) { char relationKind = '\0'; if (rangeTableEntry->rtekind != RTE_RELATION) { /* we're only interested in relations */ return false; } relationKind = rangeTableEntry->relkind; if (relationKind == RELKIND_RELATION || relationKind == RELKIND_PARTITIONED_TABLE || relationKind == RELKIND_FOREIGN_TABLE || relationKind == RELKIND_MATVIEW) { /* * RELKIND_VIEW are automatically replaced with a subquery in * the query tree, so we ignore them here. * * RELKIND_MATVIEW is equivalent of a local table in postgres. */ return true; } return false; } /* * ExtractRangeTableRelationWalker gathers all range table relation entries * in a query. The caller is responsible for checking whether the returned * entries are distributed or not. */ bool ExtractRangeTableRelationWalker(Node *node, List **rangeTableRelationList) { ExtractRangeTableWalkerContext context; context.rangeTableList = rangeTableRelationList; context.walkerMode = EXTRACT_RELATION_ENTRIES; return ExtractRangeTableList(node, &context); } /* * ExtractRangeTableEntryWalker walks over a query tree, and finds all range * table entries. For recursing into the query tree, this function uses the * query tree walker since the expression tree walker doesn't recurse into * sub-queries. */ bool ExtractRangeTableEntryWalker(Node *node, List **rangeTableList) { ExtractRangeTableWalkerContext context; context.rangeTableList = rangeTableList; context.walkerMode = EXTRACT_ALL_ENTRIES; return ExtractRangeTableList(node, &context); } /* * ExtractRangeTableIndexWalker walks over a join tree, and finds all range * table indexes in that tree. */ bool ExtractRangeTableIndexWalker(Node *node, List **rangeTableIndexList) { bool walkerResult = false; if (node == NULL) { return false; } if (IsA(node, RangeTblRef)) { int rangeTableIndex = ((RangeTblRef *) node)->rtindex; (*rangeTableIndexList) = lappend_int(*rangeTableIndexList, rangeTableIndex); } else { walkerResult = expression_tree_walker(node, ExtractRangeTableIndexWalker, rangeTableIndexList); } return walkerResult; } ================================================ FILE: src/backend/distributed/utils/reference_table_utils.c ================================================ /*------------------------------------------------------------------------- * * reference_table_utils.c * * Declarations for public utility functions related to reference tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" #include "postmaster/postmaster.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "distributed/backend_data.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_logical_planner.h" #include "distributed/reference_table_utils.h" #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shard_transfer.h" #include "distributed/shardinterval_utils.h" #include "distributed/transaction_management.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" /* local function forward declarations */ static List * WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode); static StringInfo CopyShardPlacementToWorkerNodeQuery(ShardPlacement *sourceShardPlacement , WorkerNode *workerNode, char transferMode); static bool AnyRelationsModifiedInTransaction(List *relationIdList); static List * ReplicatedMetadataSyncedDistributedTableList(void); static bool NodeHasAllReferenceTableReplicas(WorkerNode *workerNode); typedef struct ShardTaskEntry { uint64 shardId; int64 taskId; } ShardTaskEntry; /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(upgrade_to_reference_table); PG_FUNCTION_INFO_V1(replicate_reference_tables); /* * replicate_reference_tables is a UDF to ensure that allreference tables are * replicated to all nodes. */ Datum replicate_reference_tables(PG_FUNCTION_ARGS) { Oid shardReplicationModeOid = PG_GETARG_OID(0); char shardReplicationMode = LookupShardTransferMode(shardReplicationModeOid); /* to prevent concurrent node additions while copying reference tables */ LockRelationOid(DistNodeRelationId(), ShareLock); EnsureReferenceTablesExistOnAllNodesExtended(shardReplicationMode); /* * Given the copying of reference tables and updating metadata have been done via a * loopback connection we do not have to retain the lock on pg_dist_node anymore. */ UnlockRelationOid(DistNodeRelationId(), ShareLock); PG_RETURN_VOID(); } /* * EnsureReferenceTablesExistOnAllNodes ensures that a shard placement for every * reference table exists on all nodes. If a node does not have a set of shard * placements, then citus_copy_shard_placement is called in a subtransaction * to pull the data to the new node. */ void EnsureReferenceTablesExistOnAllNodes(void) { EnsureReferenceTablesExistOnAllNodesExtended(TRANSFER_MODE_BLOCK_WRITES); } /* * EnsureReferenceTablesExistOnAllNodesExtended ensures that a shard placement for every * reference table exists on all nodes. If a node does not have a set of shard placements, * then citus_copy_shard_placement is called in a subtransaction to pull the data to the * new node. * * The transferMode is passed on to the implementation of the copy to control the locks * and transferMode. */ void EnsureReferenceTablesExistOnAllNodesExtended(char transferMode) { List *referenceTableIdList = NIL; uint64 shardId = INVALID_SHARD_ID; List *newWorkersList = NIL; const char *referenceTableName = NULL; int colocationId = GetReferenceTableColocationId(); if (colocationId == INVALID_COLOCATION_ID) { /* we have no reference table yet. */ return; } /* * Most of the time this function should result in a conclusion where we do not need * to copy any reference tables. To prevent excessive locking the majority of the time * we run our precondition checks first with a lower lock. If, after checking with the * lower lock, that we might need to copy reference tables we check with a more * aggressive and self conflicting lock. It is important to be self conflicting in the * second run to make sure that two concurrent calls to this routine will actually not * run concurrently after the initial check. * * If after two iterations of precondition checks we still find the need for copying * reference tables we exit the loop with all locks held. This will prevent concurrent * DROP TABLE and create_reference_table calls so that the list of reference tables we * operate on are stable. * * Since the changes to the reference table placements are made via loopback * connections we release the locks held at the end of this function. Due to Citus * only running transactions in READ COMMITTED mode we can be sure that other * transactions correctly find the metadata entries. */ LOCKMODE lockmodes[] = { AccessShareLock, ExclusiveLock }; for (int lockmodeIndex = 0; lockmodeIndex < lengthof(lockmodes); lockmodeIndex++) { LockColocationId(colocationId, lockmodes[lockmodeIndex]); referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE); if (referenceTableIdList == NIL) { /* * No reference tables exist, make sure that any locks obtained earlier are * released. It will probably not matter, but we release the locks in the * reverse order we obtained them in. */ for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } return; } Oid referenceTableId = linitial_oid(referenceTableIdList); referenceTableName = get_rel_name(referenceTableId); List *shardIntervalList = LoadShardIntervalList(referenceTableId); if (list_length(shardIntervalList) == 0) { /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard", referenceTableName))); } ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); shardId = shardInterval->shardId; /* * We only take an access share lock, otherwise we'll hold up citus_add_node. * In case of create_reference_table() where we don't want concurrent writes * to pg_dist_node, we have already acquired ShareLock on pg_dist_node. */ newWorkersList = WorkersWithoutReferenceTablePlacement(shardId, AccessShareLock); if (list_length(newWorkersList) == 0) { /* * All workers already have a copy of the reference tables, make sure that * any locks obtained earlier are released. It will probably not matter, but * we release the locks in the reverse order we obtained them in. */ for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } return; } } /* * citus_copy_shard_placement triggers metadata sync-up, which tries to * acquire a ShareLock on pg_dist_node. We do master_copy_shad_placement * in a separate connection. If we have modified pg_dist_node in the * current backend, this will cause a deadlock. */ if (TransactionModifiedNodeMetadata) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot replicate reference tables in a transaction " "that modified node metadata"))); } /* * Modifications to reference tables in current transaction are not visible * to citus_copy_shard_placement, since it is done in a separate backend. */ if (AnyRelationsModifiedInTransaction(referenceTableIdList)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot replicate reference tables in a transaction " "that modified a reference table"))); } bool missingOk = false; ShardPlacement *sourceShardPlacement = ActiveShardPlacement(shardId, missingOk); if (sourceShardPlacement == NULL) { /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table shard " UINT64_FORMAT " does not have an active shard placement", shardId))); } WorkerNode *newWorkerNode = NULL; foreach_declared_ptr(newWorkerNode, newWorkersList) { ereport(DEBUG2, (errmsg("replicating reference table '%s' to %s:%d ...", referenceTableName, newWorkerNode->workerName, newWorkerNode->workerPort))); /* * Call citus_copy_shard_placement using citus extension owner. Current * user might not have permissions to do the copy. */ const char *userName = CitusExtensionOwnerName(); int connectionFlags = OUTSIDE_TRANSACTION; MultiConnection *connection = GetNodeUserDatabaseConnection( connectionFlags, LocalHostName, PostPortNumber, userName, NULL); if (PQstatus(connection->pgConn) == CONNECTION_OK) { UseCoordinatedTransaction(); RemoteTransactionBegin(connection); StringInfo placementCopyCommand = CopyShardPlacementToWorkerNodeQuery(sourceShardPlacement, newWorkerNode, transferMode); /* * The placement copy command uses distributed execution to copy * the shard. This is allowed when indicating that the backend is a * rebalancer backend. */ ExecuteCriticalRemoteCommand(connection, psprintf( "SET LOCAL application_name TO '%s%ld'", CITUS_REBALANCER_APPLICATION_NAME_PREFIX, GetGlobalPID())); ExecuteCriticalRemoteCommand(connection, placementCopyCommand->data); RemoteTransactionCommit(connection); } else { ereport(ERROR, (errmsg("could not open a connection to localhost " "when replicating reference tables"), errdetail( "citus.replicate_reference_tables_on_activate = false " "requires localhost connectivity."))); } CloseConnection(connection); } /* * Since reference tables have been copied via a loopback connection we do not have * to retain our locks. Since Citus only runs well in READ COMMITTED mode we can be * sure that other transactions will find the reference tables copied. * We have obtained and held multiple locks, here we unlock them all in the reverse * order we have obtained them in. */ for (int releaseLockmodeIndex = lengthof(lockmodes) - 1; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } } /* * ScheduleTasksToParallelCopyReferenceTablesOnAllMissingNodes is essentially a * twin of EnsureReferenceTablesExistOnAllNodesExtended. The difference is instead of * copying the missing tables on to the worker nodes this function creates the background * tasks for each required copy operation and schedule it in the background job. * Another difference is that instead of moving all the colocated shards sequencially * this function creates a seperate background task for each shard, even when the shards * are part of same colocated shard group. * * For transfering the shards in parallel the function creates a task for each shard * move and than schedules another task that creates the shard relationships (if any) * between shards and that task wait for the completion of all shard transfer tasks. * * The function returns an array of task ids that are created for creating the shard * relationships, effectively completion of these tasks signals the completion of * of reference table setup on the worker nodes. Any process that needs to wait for * the completion of the reference table setup can wait for these tasks to complete. * * The transferMode is passed to this function gets ignored for now and it only uses * block write mode. */ int64 * ScheduleTasksToParallelCopyReferenceTablesOnAllMissingNodes(int64 jobId, char transferMode , int *nDependTasks) { List *referenceTableIdList = NIL; uint64 shardId = INVALID_SHARD_ID; List *newWorkersList = NIL; int64 *dependsTaskArray = NULL; const char *referenceTableName = NULL; int colocationId = GetReferenceTableColocationId(); *nDependTasks = 0; if (colocationId == INVALID_COLOCATION_ID) { /* we have no reference table yet. */ return 0; } /* * Most of the time this function should result in a conclusion where we do not need * to copy any reference tables. To prevent excessive locking the majority of the time * we run our precondition checks first with a lower lock. If, after checking with the * lower lock, that we might need to copy reference tables we check with a more * aggressive and self conflicting lock. It is important to be self conflicting in the * second run to make sure that two concurrent calls to this routine will actually not * run concurrently after the initial check. * * If after two iterations of precondition checks we still find the need for copying * reference tables we exit the loop with all locks held. This will prevent concurrent * DROP TABLE and create_reference_table calls so that the list of reference tables we * operate on are stable. * * * Since the changes to the reference table placements are made via loopback * connections we release the locks held at the end of this function. Due to Citus * only running transactions in READ COMMITTED mode we can be sure that other * transactions correctly find the metadata entries. */ LOCKMODE lockmodes[] = { AccessShareLock, ExclusiveLock }; for (int lockmodeIndex = 0; lockmodeIndex < lengthof(lockmodes); lockmodeIndex++) { LockColocationId(colocationId, lockmodes[lockmodeIndex]); referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE); if (referenceTableIdList == NIL) { /* * No reference tables exist, make sure that any locks obtained earlier are * released. It will probably not matter, but we release the locks in the * reverse order we obtained them in. */ for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } return 0; } Oid referenceTableId = linitial_oid(referenceTableIdList); referenceTableName = get_rel_name(referenceTableId); List *shardIntervalList = LoadShardIntervalList(referenceTableId); if (list_length(shardIntervalList) == 0) { /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard", referenceTableName))); } ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); shardId = shardInterval->shardId; /* * We only take an access share lock, otherwise we'll hold up citus_add_node. * In case of create_reference_table() where we don't want concurrent writes * to pg_dist_node, we have already acquired ShareLock on pg_dist_node. */ newWorkersList = WorkersWithoutReferenceTablePlacement(shardId, AccessShareLock); if (list_length(newWorkersList) == 0) { /* * All workers already have a copy of the reference tables, make sure that * any locks obtained earlier are released. It will probably not matter, but * we release the locks in the reverse order we obtained them in. */ for (int releaseLockmodeIndex = lockmodeIndex; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } return 0; } } /* * citus_copy_shard_placement triggers metadata sync-up, which tries to * acquire a ShareLock on pg_dist_node. We do master_copy_shad_placement * in a separate connection. If we have modified pg_dist_node in the * current backend, this will cause a deadlock. */ if (TransactionModifiedNodeMetadata) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot replicate reference tables in a transaction " "that modified node metadata"))); } /* * Modifications to reference tables in current transaction are not visible * to citus_copy_shard_placement, since it is done in a separate backend. */ if (AnyRelationsModifiedInTransaction(referenceTableIdList)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot replicate reference tables in a transaction " "that modified a reference table"))); } bool missingOk = false; ShardPlacement *sourceShardPlacement = ActiveShardPlacement(shardId, missingOk); if (sourceShardPlacement == NULL) { /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table shard " UINT64_FORMAT " does not have an active shard placement", shardId))); } WorkerNode *newWorkerNode = NULL; BackgroundTask *task = NULL; StringInfoData buf = { 0 }; initStringInfo(&buf); List *depTasksList = NIL; const char *transferModeString = transferMode == TRANSFER_MODE_BLOCK_WRITES ? "block_writes" : transferMode == TRANSFER_MODE_FORCE_LOGICAL ? "force_logical" : "auto"; foreach_declared_ptr(newWorkerNode, newWorkersList) { ereport(DEBUG2, (errmsg("replicating reference table '%s' to %s:%d ...", referenceTableName, newWorkerNode->workerName, newWorkerNode->workerPort))); Oid relationId = InvalidOid; List *nodeTasksList = NIL; HTAB *shardTaskMap = CreateSimpleHashWithNameAndSize(uint64, ShardTaskEntry, "Shard_Task_Map", list_length( referenceTableIdList)); foreach_declared_oid(relationId, referenceTableIdList) { referenceTableName = get_rel_name(relationId); List *shardIntervalList = LoadShardIntervalList(relationId); if (list_length(shardIntervalList) != 1) { ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard", referenceTableName))); } ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); shardId = shardInterval->shardId; resetStringInfo(&buf); uint32 shardTransferFlags = SHARD_TRANSFER_SINGLE_SHARD_ONLY | SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS; /* Temporary hack until we get background task config support PR */ appendStringInfo(&buf, "SET LOCAL application_name TO '%s%ld';", CITUS_REBALANCER_APPLICATION_NAME_PREFIX, GetGlobalPID()); /* * In first step just create and load data in the shards but defer the * creation of the shard relationships to the next step. * The reason we want to defer the creation of the shard relationships is that * we want to make sure that all the parallel shard copy task are finished * before we create the relationships. Otherwise we might end up with * a situation where the dependent-shard task is still running and trying to * create the shard relationships will result in ERROR. */ appendStringInfo(&buf, "SELECT " "citus_internal.citus_internal_copy_single_shard_placement" "(%ld,%u,%u,%u,%s)", shardId, sourceShardPlacement->nodeId, newWorkerNode->nodeId, shardTransferFlags, quote_literal_cstr(transferModeString)); ereport(DEBUG2, (errmsg("replicating reference table '%s' to %s:%d ... QUERY= %s", referenceTableName, newWorkerNode->workerName, newWorkerNode->workerPort, buf.data))); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *relatedRelations = list_concat(cacheEntry-> referencedRelationsViaForeignKey, cacheEntry-> referencingRelationsViaForeignKey); List *dependencyTaskList = NIL; Oid relatedRelationId = InvalidOid; foreach_declared_oid(relatedRelationId, relatedRelations) { if (!list_member_oid(referenceTableIdList, relatedRelationId)) { continue; } List *relatedShardIntervalList = LoadShardIntervalList( relatedRelationId); ShardInterval *relatedShardInterval = (ShardInterval *) linitial( relatedShardIntervalList); uint64 relatedShardId = relatedShardInterval->shardId; bool taskFound = false; ShardTaskEntry *taskEntry = hash_search(shardTaskMap, &relatedShardId, HASH_FIND, &taskFound); if (taskFound) { dependencyTaskList = lappend(dependencyTaskList, taskEntry); } } int nDepends = list_length(dependencyTaskList); int64 *dependsArray = NULL; if (nDepends > 0) { dependsArray = (int64 *) palloc(sizeof(int64) * nDepends); int i = 0; ListCell *lc; foreach(lc, dependencyTaskList) { dependsArray[i++] = ((ShardTaskEntry *) lfirst(lc))->taskId; } } int32 nodesInvolved[2] = { 0 }; nodesInvolved[0] = sourceShardPlacement->nodeId; nodesInvolved[1] = newWorkerNode->nodeId; task = ScheduleBackgroundTask(jobId, CitusExtensionOwner(), buf.data, nDepends, dependsArray, 2, nodesInvolved); bool found = false; ShardTaskEntry *taskEntry = hash_search(shardTaskMap, &shardId, HASH_ENTER, &found); if (!found) { taskEntry->taskId = task->taskid; ereport(DEBUG2, (errmsg( "Added hash entry in scheduled task hash " "with task %ld for shard %ld", task->taskid, shardId))); } else { ereport(ERROR, (errmsg("failed to record task dependency for shard %ld", shardId))); } nodeTasksList = lappend(nodeTasksList, task); if (dependsArray) { pfree(dependsArray); } list_free(dependencyTaskList); } if (list_length(nodeTasksList) > 0) { int nDepends = list_length(nodeTasksList); int32 nodesInvolved[2] = { 0 }; nodesInvolved[0] = sourceShardPlacement->nodeId; nodesInvolved[1] = newWorkerNode->nodeId; int64 *dependsArray = palloc(sizeof(int64) * list_length(nodeTasksList)); int idx = 0; foreach_declared_ptr(task, nodeTasksList) { dependsArray[idx++] = task->taskid; } resetStringInfo(&buf); uint32 shardTransferFlags = SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY; appendStringInfo(&buf, "SET LOCAL application_name TO '%s%ld';\n", CITUS_REBALANCER_APPLICATION_NAME_PREFIX, GetGlobalPID()); appendStringInfo(&buf, "SELECT " "citus_internal.citus_internal_copy_single_shard_placement" "(%ld,%u,%u,%u,%s)", shardId, sourceShardPlacement->nodeId, newWorkerNode->nodeId, shardTransferFlags, quote_literal_cstr(transferModeString)); ereport(DEBUG2, (errmsg( "creating relations for reference table '%s' on %s:%d ... " "QUERY= %s", referenceTableName, newWorkerNode->workerName, newWorkerNode->workerPort, buf.data))); task = ScheduleBackgroundTask(jobId, CitusExtensionOwner(), buf.data, nDepends, dependsArray, 2, nodesInvolved); depTasksList = lappend(depTasksList, task); pfree(dependsArray); list_free(nodeTasksList); nodeTasksList = NIL; } hash_destroy(shardTaskMap); } /* * compute a dependent task list array to be used to indicate the completion of all * reference table shards copy, so that we can start with distributed shard copy */ if (list_length(depTasksList) > 0) { *nDependTasks = list_length(depTasksList); dependsTaskArray = palloc(sizeof(int64) * *nDependTasks); int idx = 0; foreach_declared_ptr(task, depTasksList) { dependsTaskArray[idx++] = task->taskid; } list_free(depTasksList); } /* * Since reference tables have been copied via a loopback connection we do not have to * retain our locks. Since Citus only runs well in READ COMMITTED mode we can be sure * that other transactions will find the reference tables copied. * We have obtained and held multiple locks, here we unlock them all in the reverse * order we have obtained them in. */ for (int releaseLockmodeIndex = lengthof(lockmodes) - 1; releaseLockmodeIndex >= 0; releaseLockmodeIndex--) { UnlockColocationId(colocationId, lockmodes[releaseLockmodeIndex]); } return dependsTaskArray; } /* * HasNodesWithMissingReferenceTables checks if all reference tables are already copied to * all nodes. When a node doesn't have a copy of the reference tables we call them missing * and this function will return true. * * The caller might be interested in the list of all reference tables after this check and * this the list of tables is written to *referenceTableList if a non-null pointer is * passed. */ bool HasNodesWithMissingReferenceTables(List **referenceTableList) { int colocationId = GetReferenceTableColocationId(); if (colocationId == INVALID_COLOCATION_ID) { /* we have no reference table yet. */ return false; } LockColocationId(colocationId, AccessShareLock); List *referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE); if (referenceTableList) { *referenceTableList = referenceTableIdList; } if (list_length(referenceTableIdList) <= 0) { return false; } Oid referenceTableId = linitial_oid(referenceTableIdList); List *shardIntervalList = LoadShardIntervalList(referenceTableId); if (list_length(shardIntervalList) == 0) { const char *referenceTableName = get_rel_name(referenceTableId); /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard", referenceTableName))); } ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); uint64 shardId = shardInterval->shardId; List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId, AccessShareLock); if (list_length(newWorkersList) <= 0) { return false; } return true; } /* * AnyRelationsModifiedInTransaction returns true if any of the given relations * were modified in the current transaction. */ static bool AnyRelationsModifiedInTransaction(List *relationIdList) { Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationIdList) { if (GetRelationDDLAccessMode(relationId) != RELATION_NOT_ACCESSED || GetRelationDMLAccessMode(relationId) != RELATION_NOT_ACCESSED) { return true; } } return false; } /* * WorkersWithoutReferenceTablePlacement returns a list of workers (WorkerNode) that * do not yet have a placement for the given reference table shard ID, but are * supposed to. */ static List * WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode) { List *workersWithoutPlacements = NIL; List *shardPlacementList = ActiveShardPlacementList(shardId); List *workerNodeList = ReferenceTablePlacementNodeList(lockMode); workerNodeList = SortList(workerNodeList, CompareWorkerNodes); WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { char *nodeName = workerNode->workerName; uint32 nodePort = workerNode->workerPort; ShardPlacement *targetPlacement = SearchShardPlacementInList(shardPlacementList, nodeName, nodePort); if (targetPlacement == NULL) { workersWithoutPlacements = lappend(workersWithoutPlacements, workerNode); } } return workersWithoutPlacements; } /* * CopyShardPlacementToWorkerNodeQuery returns the citus_copy_shard_placement * command to copy the given shard placement to given node. */ static StringInfo CopyShardPlacementToWorkerNodeQuery(ShardPlacement *sourceShardPlacement, WorkerNode *workerNode, char transferMode) { StringInfo queryString = makeStringInfo(); const char *transferModeString = transferMode == TRANSFER_MODE_BLOCK_WRITES ? "block_writes" : transferMode == TRANSFER_MODE_FORCE_LOGICAL ? "force_logical" : "auto"; appendStringInfo(queryString, "SELECT pg_catalog.citus_copy_shard_placement(" UINT64_FORMAT ", %d, %d, " "transfer_mode := %s)", sourceShardPlacement->shardId, sourceShardPlacement->nodeId, workerNode->nodeId, quote_literal_cstr(transferModeString)); return queryString; } /* * upgrade_to_reference_table was removed, but we maintain a dummy implementation * to support downgrades. */ Datum upgrade_to_reference_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("this function is deprecated and no longer used"))); } /* * CreateReferenceTableColocationId creates a new co-location id for reference tables and * writes it into pg_dist_colocation, then returns the created co-location id. Since there * can be only one colocation group for all kinds of reference tables, if a co-location id * is already created for reference tables, this function returns it without creating * anything. */ uint32 CreateReferenceTableColocationId() { int shardCount = 1; Oid distributionColumnType = InvalidOid; Oid distributionColumnCollation = InvalidOid; /* * We don't maintain replication factor of reference tables anymore and * just use -1 instead. We don't use this value in any places. */ int replicationFactor = -1; /* check for existing colocations */ uint32 colocationId = ColocationId(shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); if (colocationId == INVALID_COLOCATION_ID) { colocationId = CreateColocationGroup(shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); } return colocationId; } uint32 GetReferenceTableColocationId() { int shardCount = 1; Oid distributionColumnType = InvalidOid; Oid distributionColumnCollation = InvalidOid; /* * We don't maintain replication factor of reference tables anymore and * just use -1 instead. We don't use this value in any places. */ int replicationFactor = -1; /* check for existing colocations */ uint32 colocationId = ColocationId(shardCount, replicationFactor, distributionColumnType, distributionColumnCollation); return colocationId; } /* * GetAllReplicatedTableList returns all tables which has replicated placements. * i.e. (all reference tables) + (distributed tables with more than 1 placements) */ List * GetAllReplicatedTableList(void) { List *referenceTableList = CitusTableTypeIdList(REFERENCE_TABLE); List *replicatedMetadataSyncedDistributedTableList = ReplicatedMetadataSyncedDistributedTableList(); List *replicatedTableList = list_concat(referenceTableList, replicatedMetadataSyncedDistributedTableList); return replicatedTableList; } /* * ReplicatedPlacementsForNodeGroup filters all replicated placements for given * node group id. */ List * ReplicatedPlacementsForNodeGroup(int32 groupId) { List *replicatedTableList = GetAllReplicatedTableList(); if (list_length(replicatedTableList) == 0) { return NIL; } List *replicatedPlacementsForNodeGroup = NIL; Oid replicatedTableId = InvalidOid; foreach_declared_oid(replicatedTableId, replicatedTableList) { List *placements = GroupShardPlacementsForTableOnGroup(replicatedTableId, groupId); if (list_length(placements) == 0) { /* * This happens either the node was previously disabled or the table * doesn't have placement on this node. */ continue; } replicatedPlacementsForNodeGroup = list_concat(replicatedPlacementsForNodeGroup, placements); } return replicatedPlacementsForNodeGroup; } /* * DeleteShardPlacementCommand returns a command for deleting given placement from * metadata. */ char * DeleteShardPlacementCommand(uint64 placementId) { StringInfo deletePlacementCommand = makeStringInfo(); appendStringInfo(deletePlacementCommand, "DELETE FROM pg_catalog.pg_dist_placement " "WHERE placementid = " UINT64_FORMAT, placementId); return deletePlacementCommand->data; } /* * DeleteAllReplicatedTablePlacementsFromNodeGroup function iterates over * list of reference and replicated hash distributed tables and deletes * all placements from pg_dist_placement table for given group. */ void DeleteAllReplicatedTablePlacementsFromNodeGroup(int32 groupId, bool localOnly) { List *replicatedPlacementListForGroup = ReplicatedPlacementsForNodeGroup(groupId); /* if there are no replicated tables for the group, we do not need to do anything */ if (list_length(replicatedPlacementListForGroup) == 0) { return; } GroupShardPlacement *placement = NULL; foreach_declared_ptr(placement, replicatedPlacementListForGroup) { LockShardDistributionMetadata(placement->shardId, ExclusiveLock); if (!localOnly) { char *deletePlacementCommand = DeleteShardPlacementCommand(placement->placementId); SendCommandToWorkersWithMetadata(deletePlacementCommand); } DeleteShardPlacementRow(placement->placementId); } } /* * DeleteAllReplicatedTablePlacementsFromNodeGroupViaMetadataContext does the same as * DeleteAllReplicatedTablePlacementsFromNodeGroup except it uses metadataSyncContext for * connections. */ void DeleteAllReplicatedTablePlacementsFromNodeGroupViaMetadataContext(MetadataSyncContext * context, int32 groupId, bool localOnly) { List *replicatedPlacementListForGroup = ReplicatedPlacementsForNodeGroup(groupId); /* if there are no replicated tables for the group, we do not need to do anything */ if (list_length(replicatedPlacementListForGroup) == 0) { return; } MemoryContext oldContext = MemoryContextSwitchTo(context->context); GroupShardPlacement *placement = NULL; foreach_declared_ptr(placement, replicatedPlacementListForGroup) { LockShardDistributionMetadata(placement->shardId, ExclusiveLock); if (!localOnly) { char *deletePlacementCommand = DeleteShardPlacementCommand(placement->placementId); SendOrCollectCommandListToMetadataNodes(context, list_make1(deletePlacementCommand)); } /* do not execute local transaction if we collect commands */ if (!MetadataSyncCollectsCommands(context)) { DeleteShardPlacementRow(placement->placementId); } ResetMetadataSyncMemoryContext(context); } MemoryContextSwitchTo(oldContext); } /* * ReplicatedMetadataSyncedDistributedTableList is a helper function which returns the * list of replicated hash distributed tables. */ static List * ReplicatedMetadataSyncedDistributedTableList(void) { List *distributedRelationList = CitusTableTypeIdList(DISTRIBUTED_TABLE); List *replicatedHashDistributedTableList = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, distributedRelationList) { if (ShouldSyncTableMetadata(relationId) && !SingleReplicatedTable(relationId)) { replicatedHashDistributedTableList = lappend_oid(replicatedHashDistributedTableList, relationId); } } return replicatedHashDistributedTableList; } /* CompareOids is a comparison function for sort shard oids */ int CompareOids(const void *leftElement, const void *rightElement) { Oid *leftId = (Oid *) leftElement; Oid *rightId = (Oid *) rightElement; if (*leftId > *rightId) { return 1; } else if (*leftId < *rightId) { return -1; } else { return 0; } } /* * ErrorIfNotAllNodesHaveReferenceTableReplicas throws an error when one of the * nodes in the list does not have reference table replicas. */ void ErrorIfNotAllNodesHaveReferenceTableReplicas(List *workerNodeList) { WorkerNode *workerNode = NULL; foreach_declared_ptr(workerNode, workerNodeList) { if (!NodeHasAllReferenceTableReplicas(workerNode)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("reference tables have not been replicated to " "node %s:%d yet", workerNode->workerName, workerNode->workerPort), errdetail("Reference tables are lazily replicated after " "adding a node, but must exist before shards can " "be created on that node."), errhint("Run SELECT replicate_reference_tables(); to " "ensure reference tables exist on all nodes."))); } } } /* * NodeHasAllReferenceTablesReplicas returns whether the given worker node has reference * table replicas. If there are no reference tables the function returns true. * * This function does not do any locking, so the situation could change immediately after, * though we can only ever transition from false to true, so only "false" could be the * incorrect answer. * * In the case where the function returns true because no reference tables exist * on the node, a reference table could be created immediately after. However, the * creation logic guarantees that this reference table will be created on all the * nodes, so our answer was correct. */ static bool NodeHasAllReferenceTableReplicas(WorkerNode *workerNode) { List *referenceTableIdList = CitusTableTypeIdList(REFERENCE_TABLE); if (list_length(referenceTableIdList) == 0) { /* no reference tables exist */ return true; } Oid referenceTableId = linitial_oid(referenceTableIdList); List *shardIntervalList = LoadShardIntervalList(referenceTableId); if (list_length(shardIntervalList) != 1) { /* check for corrupt metadata */ ereport(ERROR, (errmsg("reference table \"%s\" can only have 1 shard", get_rel_name(referenceTableId)))); } ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList); List *shardPlacementList = ActiveShardPlacementList(shardInterval->shardId); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacementList) { if (placement->groupId == workerNode->groupId) { /* our worker has a reference table placement */ return true; } } return false; } ================================================ FILE: src/backend/distributed/utils/relation_utils.c ================================================ /*------------------------------------------------------------------------- * * relation_utils.c * * This file contains functions similar to rel.h to perform useful * operations on Relation objects. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "utils/lsyscache.h" #include "utils/rel.h" #include "distributed/relation_utils.h" /* * RelationGetNamespaceName returns the relation's namespace name. */ char * RelationGetNamespaceName(Relation relation) { Oid namespaceId = RelationGetNamespace(relation); char *namespaceName = get_namespace_name(namespaceId); return namespaceName; } /* * GetFilledPermissionInfo creates RTEPermissionInfo for a given RTE * and fills it with given data and returns this RTEPermissionInfo object. * Added this function since Postgres's addRTEPermissionInfo doesn't fill the data. * * Given data consists of relid, inh and requiredPerms * Took a quick look around Postgres, unless specified otherwise, * we are dealing with GetUserId(). * Currently the following entries are filled like this: * perminfo->checkAsUser = GetUserId(); */ RTEPermissionInfo * GetFilledPermissionInfo(Oid relid, bool inh, AclMode requiredPerms) { RTEPermissionInfo *perminfo = makeNode(RTEPermissionInfo); perminfo->relid = relid; perminfo->inh = inh; perminfo->requiredPerms = requiredPerms; perminfo->checkAsUser = GetUserId(); return perminfo; } ================================================ FILE: src/backend/distributed/utils/replication_origin_session_utils.c ================================================ /*------------------------------------------------------------------------- * * replication_origin_session_utils.c * Functions for managing replication origin session. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "utils/builtins.h" #include "distributed/metadata_cache.h" #include "distributed/remote_commands.h" #include "distributed/replication_origin_session_utils.h" static bool IsRemoteReplicationOriginSessionSetup(MultiConnection *connection); static void SetupMemoryContextResetReplicationOriginHandler(void); static void SetupReplicationOriginSessionHelper(bool isContexResetSetupNeeded); PG_FUNCTION_INFO_V1(citus_internal_start_replication_origin_tracking); PG_FUNCTION_INFO_V1(citus_internal_stop_replication_origin_tracking); PG_FUNCTION_INFO_V1(citus_internal_is_replication_origin_tracking_active); /* * This variable is used to remember the replication origin id of the current session * before resetting it to DoNotReplicateId in SetupReplicationOriginLocalSession. */ static RepOriginId OriginalOriginId = InvalidRepOriginId; /* * Setting that controls whether replication origin tracking is enabled */ bool EnableChangeDataCapture = false; /* citus_internal_start_replication_origin_tracking starts a new replication origin session * in the local node. This function is used to avoid publishing the WAL records to the * replication slot by setting replication origin to DoNotReplicateId in WAL records. * It remembers the previous replication origin for the current session which will be * used to reset the replication origin to the previous value when the session ends. */ Datum citus_internal_start_replication_origin_tracking(PG_FUNCTION_ARGS) { if (!EnableChangeDataCapture) { PG_RETURN_VOID(); } SetupReplicationOriginSessionHelper(false); PG_RETURN_VOID(); } /* citus_internal_stop_replication_origin_tracking ends the current replication origin session * in the local node. This function is used to reset the replication origin to the * earlier value of replication origin. */ Datum citus_internal_stop_replication_origin_tracking(PG_FUNCTION_ARGS) { ResetReplicationOriginLocalSession(); PG_RETURN_VOID(); } /* IsLocalReplicationOriginSessionActive checks if the current replication origin * session is active in the local node. */ static inline bool IsLocalReplicationOriginSessionActive(void) { return (replorigin_session_origin == DoNotReplicateId); } /* citus_internal_is_replication_origin_tracking_active checks if the current replication origin * session is active in the local node. */ Datum citus_internal_is_replication_origin_tracking_active(PG_FUNCTION_ARGS) { bool result = IsLocalReplicationOriginSessionActive(); PG_RETURN_BOOL(result); } /* * SetupMemoryContextResetReplicationOriginHandler registers a callback function * that resets the replication origin session in case of any error for the current * memory context. */ static void SetupMemoryContextResetReplicationOriginHandler() { MemoryContextCallback *replicationOriginResetCallback = palloc0( sizeof(MemoryContextCallback)); replicationOriginResetCallback->func = ResetReplicationOriginLocalSessionCallbackHandler; replicationOriginResetCallback->arg = NULL; MemoryContextRegisterResetCallback(CurrentMemoryContext, replicationOriginResetCallback); } /* * SetupReplicationOriginSessionHelper sets up a new replication origin session in a * local session. It takes an argument isContexResetSetupNeeded to decide whether * to register a callback function that resets the replication origin session in case * of any error for the current memory context. */ static void SetupReplicationOriginSessionHelper(bool isContexResetSetupNeeded) { if (!EnableChangeDataCapture) { return; } OriginalOriginId = replorigin_session_origin; replorigin_session_origin = DoNotReplicateId; if (isContexResetSetupNeeded) { SetupMemoryContextResetReplicationOriginHandler(); } } /* * SetupReplicationOriginLocalSession sets up a new replication origin session in a * local session. */ void SetupReplicationOriginLocalSession() { SetupReplicationOriginSessionHelper(true); } /* * ResetReplicationOriginLocalSession resets the replication origin session in a * local node. */ void ResetReplicationOriginLocalSession(void) { if (replorigin_session_origin != DoNotReplicateId) { return; } replorigin_session_origin = OriginalOriginId; } /* * ResetReplicationOriginLocalSessionCallbackHandler is a callback function that * resets the replication origin session in a local node. This is used to register * with MemoryContextRegisterResetCallback to reset the replication origin session * in case of any error for the given memory context. */ void ResetReplicationOriginLocalSessionCallbackHandler(void *arg) { ResetReplicationOriginLocalSession(); } /* * SetupReplicationOriginRemoteSession sets up a new replication origin session in a * remote session. The identifier is used to create a unique replication origin name * for the session in the remote node. */ void SetupReplicationOriginRemoteSession(MultiConnection *connection) { if (!EnableChangeDataCapture) { return; } if (connection != NULL && !IsRemoteReplicationOriginSessionSetup(connection)) { StringInfo replicationOriginSessionSetupQuery = makeStringInfo(); appendStringInfo(replicationOriginSessionSetupQuery, "select citus_internal.start_replication_origin_tracking();"); ExecuteCriticalRemoteCommand(connection, replicationOriginSessionSetupQuery->data); connection->isReplicationOriginSessionSetup = true; } } /* * ResetReplicationOriginRemoteSession resets the replication origin session in a * remote node. */ void ResetReplicationOriginRemoteSession(MultiConnection *connection) { if (connection != NULL && connection->isReplicationOriginSessionSetup) { StringInfo replicationOriginSessionResetQuery = makeStringInfo(); appendStringInfo(replicationOriginSessionResetQuery, "select citus_internal.stop_replication_origin_tracking();"); ExecuteCriticalRemoteCommand(connection, replicationOriginSessionResetQuery->data); connection->isReplicationOriginSessionSetup = false; } } /* * IsRemoteReplicationOriginSessionSetup checks if the replication origin is setup * already in the remote session by calliing the UDF * citus_internal_is_replication_origin_tracking_active(). This is also remembered * in the connection object to avoid calling the UDF again next time. */ static bool IsRemoteReplicationOriginSessionSetup(MultiConnection *connection) { if (connection->isReplicationOriginSessionSetup) { return true; } StringInfo isReplicationOriginSessionSetupQuery = makeStringInfo(); appendStringInfo(isReplicationOriginSessionSetupQuery, "SELECT citus_internal.is_replication_origin_tracking_active()"); bool result = ExecuteRemoteCommandAndCheckResult(connection, isReplicationOriginSessionSetupQuery->data, "t"); connection->isReplicationOriginSessionSetup = result; return result; } ================================================ FILE: src/backend/distributed/utils/resource_lock.c ================================================ /*------------------------------------------------------------------------- * * resource_lock.c * Locking Infrastructure for Citus. * * To avoid introducing a new type of locktag - that then could not be * displayed by core functionality - we reuse advisory locks. If we'd just * reused them directly we'd run into danger conflicting with user-defined * advisory locks, but luckily advisory locks only two values for 'field4' in * the locktag. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "c.h" #include "miscadmin.h" #include "access/xact.h" #include "catalog/namespace.h" #include "commands/tablecmds.h" #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/varlena.h" #include "distributed/colocation_utils.h" #include "distributed/commands.h" #include "distributed/coordinator_protocol.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/multi_join_order.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/reference_table_utils.h" #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/utils/array_type.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_transaction.h" #define LOCK_RELATION_IF_EXISTS \ "SELECT pg_catalog.lock_relation_if_exists(%s, %s);" /* static definition and declarations */ struct LockModeToStringType { LOCKMODE lockMode; const char *name; }; /* * list of lock mode mappings, number of items need to be kept in sync * with lock_mode_to_string_map_count. */ static const struct LockModeToStringType lockmode_to_string_map[] = { { NoLock, "NoLock" }, { AccessShareLock, "ACCESS SHARE" }, { RowShareLock, "ROW SHARE" }, { RowExclusiveLock, "ROW EXCLUSIVE" }, { ShareUpdateExclusiveLock, "SHARE UPDATE EXCLUSIVE" }, { ShareLock, "SHARE" }, { ShareRowExclusiveLock, "SHARE ROW EXCLUSIVE" }, { ExclusiveLock, "EXCLUSIVE" }, { AccessExclusiveLock, "ACCESS EXCLUSIVE" } }; static const int lock_mode_to_string_map_count = sizeof(lockmode_to_string_map) / sizeof(lockmode_to_string_map[0]); /* * LockRelationRecord holds the oid of a relation to be locked * and a boolean inh to determine whether its decendants * should be locked as well */ typedef struct LockRelationRecord { Oid relationId; bool inh; } LockRelationRecord; /* local function forward declarations */ static LOCKMODE IntToLockMode(int mode); static void LockReferencedReferenceShardResources(uint64 shardId, LOCKMODE lockMode); static bool AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList); static void LockShardListResources(List *shardIntervalList, LOCKMODE lockMode); static void LockShardListResourcesOnFirstWorker(LOCKMODE lockmode, List *shardIntervalList); static bool IsFirstWorkerNode(); static void CitusRangeVarCallbackForLockTable(const RangeVar *rangeVar, Oid relationId, Oid oldRelationId, void *arg); static AclResult CitusLockTableAclCheck(Oid relationId, LOCKMODE lockmode, Oid userId); static void SetLocktagForShardDistributionMetadata(int64 shardId, LOCKTAG *tag); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(lock_shard_metadata); PG_FUNCTION_INFO_V1(lock_shard_resources); PG_FUNCTION_INFO_V1(lock_relation_if_exists); /* Config variable managed via guc.c */ bool EnableAcquiringUnsafeLockFromWorkers = false; bool SkipAdvisoryLockPermissionChecks = false; /* * lock_shard_metadata allows the shard distribution metadata to be locked * remotely to block concurrent writes from workers in MX tables. * * This function does not sort the array to avoid deadlock, callers * must ensure a consistent order. */ Datum lock_shard_metadata(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); LOCKMODE lockMode = IntToLockMode(PG_GETARG_INT32(0)); ArrayType *shardIdArrayObject = PG_GETARG_ARRAYTYPE_P(1); if (ARR_NDIM(shardIdArrayObject) == 0) { ereport(ERROR, (errmsg("no locks specified"))); } int shardIdCount = ArrayObjectCount(shardIdArrayObject); Datum *shardIdArrayDatum = DeconstructArrayObject(shardIdArrayObject); for (int shardIdIndex = 0; shardIdIndex < shardIdCount; shardIdIndex++) { int64 shardId = DatumGetInt64(shardIdArrayDatum[shardIdIndex]); /* * We don't want random users to block writes. The callers of this * function either operates on all the colocated placements, such * as shard moves, or requires superuser such as adding node. * In other words, the coordinator initiated operations has already * ensured table owner, we are preventing any malicious attempt to * use this function. */ bool missingOk = true; EnsureShardOwner(shardId, missingOk); LockShardDistributionMetadata(shardId, lockMode); } PG_RETURN_VOID(); } /* * EnsureShardOwner gets the shardId and reads pg_dist_partition to find * the corresponding relationId. If the relation does not exist, the function * returns. If the relation exists, the function ensures if the current * user is the owner of the table. * */ void EnsureShardOwner(uint64 shardId, bool missingOk) { Oid relationId = LookupShardRelationFromCatalog(shardId, missingOk); if (!OidIsValid(relationId) && missingOk) { /* * This could happen in two ways. First, a malicious user is trying * to acquire locks on non-existing shards. Second, the metadata has * not been synced (or not yet visible) to this node. In the second * case, there is no point in locking the shards because no other * transaction can be accessing the table. */ return; } EnsureTableOwner(relationId); } /* * lock_shard_resources allows shard resources to be locked * remotely to serialise non-commutative writes on shards. * * This function does not sort the array to avoid deadlock, callers * must ensure a consistent order. */ Datum lock_shard_resources(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); LOCKMODE lockMode = IntToLockMode(PG_GETARG_INT32(0)); ArrayType *shardIdArrayObject = PG_GETARG_ARRAYTYPE_P(1); if (ARR_NDIM(shardIdArrayObject) == 0) { ereport(ERROR, (errmsg("no locks specified"))); } int shardIdCount = ArrayObjectCount(shardIdArrayObject); Datum *shardIdArrayDatum = DeconstructArrayObject(shardIdArrayObject); /* * The executor calls this UDF for modification queries. So, any user * who has the the rights to modify this table are actually able * to call the UDF. * * So, at this point, we make sure that any malicious user who doesn't * have modification privileges to call this UDF. * * Update/Delete/Truncate commands already acquires ExclusiveLock * on the executor. However, for INSERTs, the user might have only * INSERTs granted, so add a special case for it. */ AclMode aclMode = ACL_UPDATE | ACL_DELETE | ACL_TRUNCATE; AclMaskHow aclMaskHow = ACLMASK_ANY; if (lockMode == RowExclusiveLock) { aclMode |= ACL_INSERT; } for (int shardIdIndex = 0; shardIdIndex < shardIdCount; shardIdIndex++) { int64 shardId = DatumGetInt64(shardIdArrayDatum[shardIdIndex]); /* * We don't want random users to block writes. If the current user * has privileges to modify the shard, then the user can already * acquire the lock. So, we allow. */ bool missingOk = true; Oid relationId = LookupShardRelationFromCatalog(shardId, missingOk); if (!OidIsValid(relationId) && missingOk) { /* * This could happen in two ways. First, a malicious user is trying * to acquire locks on non-existing shards. Second, the metadata has * not been synced (or not yet visible) to this node. In the second * case, there is no point in locking the shards because no other * transaction can be accessing the table. */ continue; } if (!SkipAdvisoryLockPermissionChecks) { EnsureTablePermissions(relationId, aclMode, aclMaskHow); } LockShardResource(shardId, lockMode); } PG_RETURN_VOID(); } /* * LockShardListResourcesOnFirstWorker acquires the resource locks for the specified * shards on the first worker. Acquiring a lock with or without metadata does not * matter for us. So, worker does not have to be an MX node, acquiring the lock * on any worker node is enough. Note that the function does not sort the shard list, * therefore the caller should sort the shard list in order to avoid deadlocks. */ static void LockShardListResourcesOnFirstWorker(LOCKMODE lockmode, List *shardIntervalList) { if (!AllowModificationsFromWorkersToReplicatedTables) { /* * Allowing modifications from worker nodes for replicated tables requires * to serialize modifications, see AcquireExecutorShardLocksForExecution() * for the details. * * If the user opted for disabling modifications from the workers, we do not * need to acquire these remote locks. Returning early saves us from an additional * network round-trip. */ Assert(AnyTableReplicated(shardIntervalList, NULL)); return; } StringInfo lockCommand = makeStringInfo(); int processedShardIntervalCount = 0; int totalShardIntervalCount = list_length(shardIntervalList); WorkerNode *firstWorkerNode = GetFirstPrimaryWorkerNode(); int connectionFlags = 0; const char *currentUser = CurrentUserName(); appendStringInfo(lockCommand, "SELECT lock_shard_resources(%d, ARRAY[", lockmode); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; appendStringInfo(lockCommand, "%lu", shardId); processedShardIntervalCount++; if (processedShardIntervalCount != totalShardIntervalCount) { appendStringInfo(lockCommand, ", "); } } appendStringInfo(lockCommand, "])"); /* need to hold the lock until commit */ UseCoordinatedTransaction(); /* * Use the superuser connection to make sure we are allowed to lock. * This also helps ensure we only use one connection. */ MultiConnection *firstWorkerConnection = GetNodeUserDatabaseConnection( connectionFlags, firstWorkerNode ->workerName, firstWorkerNode ->workerPort, currentUser, NULL); /* the SELECT .. FOR UPDATE breaks if we lose the connection */ MarkRemoteTransactionCritical(firstWorkerConnection); /* make sure we are in a tranasaction block to hold the lock until commit */ RemoteTransactionBeginIfNecessary(firstWorkerConnection); /* grab the lock on the first worker node */ ExecuteCriticalRemoteCommand(firstWorkerConnection, lockCommand->data); } /* * IsFirstWorkerNode checks whether the node is the first worker node sorted * according to the host name and port number. */ static bool IsFirstWorkerNode() { List *workerNodeList = ActivePrimaryNonCoordinatorNodeList(NoLock); workerNodeList = SortList(workerNodeList, CompareWorkerNodes); if (list_length(workerNodeList) == 0) { return false; } WorkerNode *firstWorkerNode = (WorkerNode *) linitial(workerNodeList); if (firstWorkerNode->groupId == GetLocalGroupId()) { return true; } return false; } /* * LockShardListMetadataOnWorkers acquires the matadata locks for the specified shards on * metadata workers. Note that the function does not sort the shard list, therefore the * caller should sort the shard list in order to avoid deadlocks. */ void LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList) { StringInfo lockCommand = makeStringInfo(); int processedShardIntervalCount = 0; int totalShardIntervalCount = list_length(shardIntervalList); if (list_length(shardIntervalList) == 0) { return; } appendStringInfo(lockCommand, "SELECT lock_shard_metadata(%d, ARRAY[", lockmode); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; appendStringInfo(lockCommand, "%lu", shardId); processedShardIntervalCount++; if (processedShardIntervalCount != totalShardIntervalCount) { appendStringInfo(lockCommand, ", "); } } appendStringInfo(lockCommand, "])"); /* * Disable idle_in_transaction_session_timeout on metadata workers before * acquiring locks. In block_writes mode, these connections stay open for * the entire shard copy which can take hours for large shards. Without * this, the timeout would kill the connection and fail the move. * SET LOCAL scopes the change to this transaction only. */ List *commandList = list_make2( "SET LOCAL idle_in_transaction_session_timeout = 0", lockCommand->data); SendCommandListToWorkersWithMetadata(commandList); } /* * IntToLockMode verifies whether the specified integer is an accepted lock mode * and returns it as a LOCKMODE enum. */ static LOCKMODE IntToLockMode(int mode) { if (mode == ExclusiveLock) { return ExclusiveLock; } else if (mode == ShareLock) { return ShareLock; } else if (mode == AccessShareLock) { return AccessShareLock; } else if (mode == RowExclusiveLock) { return RowExclusiveLock; } else { elog(ERROR, "unsupported lockmode %d", mode); } } /* * LockColocationId returns after acquiring a co-location ID lock, typically used * for rebalancing and replication. */ void LockColocationId(int colocationId, LOCKMODE lockMode) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SET_LOCKTAG_REBALANCE_COLOCATION(tag, (int64) colocationId); (void) LockAcquire(&tag, lockMode, sessionLock, dontWait); } /* * UnlockColocationId releases a co-location ID lock. */ void UnlockColocationId(int colocationId, LOCKMODE lockMode) { LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_REBALANCE_COLOCATION(tag, (int64) colocationId); LockRelease(&tag, lockMode, sessionLock); } /* * LockShardDistributionMetadata returns after grabbing a lock for distribution * metadata related to the specified shard, blocking if required. Any locks * acquired using this method are released at transaction end. */ void LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SetLocktagForShardDistributionMetadata(shardId, &tag); (void) LockAcquire(&tag, lockMode, sessionLock, dontWait); } static void SetLocktagForShardDistributionMetadata(int64 shardId, LOCKTAG *tag) { ShardInterval *shardInterval = LoadShardInterval(shardId); Oid citusTableId = shardInterval->relationId; CitusTableCacheEntry *citusTable = GetCitusTableCacheEntry(citusTableId); uint32 colocationId = citusTable->colocationId; if (colocationId == INVALID_COLOCATION_ID || (!IsCitusTableTypeCacheEntry(citusTable, HASH_DISTRIBUTED) && !IsCitusTableTypeCacheEntry(citusTable, SINGLE_SHARD_DISTRIBUTED))) { SET_LOCKTAG_SHARD_METADATA_RESOURCE(*tag, MyDatabaseId, shardId); } else { SET_LOCKTAG_COLOCATED_SHARDS_METADATA_RESOURCE(*tag, MyDatabaseId, colocationId, shardInterval->shardIndex); } } /* * LockReferencedReferenceShardDistributionMetadata acquires shard distribution * metadata locks with the given lock mode on the reference tables which has a * foreign key from the given relation. * * It also gets metadata locks on worker nodes to prevent concurrent write * operations on reference tables from metadata nodes. */ void LockReferencedReferenceShardDistributionMetadata(uint64 shardId, LOCKMODE lockMode) { Oid relationId = RelationIdForShard(shardId); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); List *referencedRelationList = cacheEntry->referencedRelationsViaForeignKey; List *shardIntervalList = GetSortedReferenceShardIntervals(referencedRelationList); if (list_length(shardIntervalList) > 0 && ClusterHasKnownMetadataWorkers()) { LockShardListMetadataOnWorkers(lockMode, shardIntervalList); } ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { LockShardDistributionMetadata(shardInterval->shardId, lockMode); } } /* * LockReferencedReferenceShardResources acquires resource locks with the * given lock mode on the reference tables which has a foreign key from * the given relation. * * It also gets resource locks on worker nodes to prevent concurrent write * operations on reference tables from metadata nodes. */ static void LockReferencedReferenceShardResources(uint64 shardId, LOCKMODE lockMode) { Oid relationId = RelationIdForShard(shardId); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); /* * Note that referencedRelationsViaForeignKey contains transitively referenced * relations too. */ List *referencedRelationList = cacheEntry->referencedRelationsViaForeignKey; List *referencedShardIntervalList = GetSortedReferenceShardIntervals(referencedRelationList); if (list_length(referencedShardIntervalList) > 0 && ClusterHasKnownMetadataWorkers() && !IsFirstWorkerNode()) { /* * When there is metadata, all nodes can write to the reference table, * but the writes need to be serialised. To achieve that, all nodes will * take the shard resource lock on the first worker node via RPC, except * for the first worker node which will just take it the regular way. */ LockShardListResourcesOnFirstWorker(lockMode, referencedShardIntervalList); } ShardInterval *referencedShardInterval = NULL; foreach_declared_ptr(referencedShardInterval, referencedShardIntervalList) { LockShardResource(referencedShardInterval->shardId, lockMode); } } /* * GetSortedReferenceShardIntervals iterates through the given relation list, * lists the shards of reference tables, and returns the list after sorting. */ List * GetSortedReferenceShardIntervals(List *relationList) { List *shardIntervalList = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationList) { if (!IsCitusTableType(relationId, REFERENCE_TABLE)) { continue; } List *currentShardIntervalList = LoadShardIntervalList(relationId); shardIntervalList = lappend(shardIntervalList, linitial( currentShardIntervalList)); } shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); return shardIntervalList; } /* * LockShardResource acquires a lock needed to modify data on a remote shard. * This task may be assigned to multiple backends at the same time, so the lock * manages any concurrency issues associated with shard file fetching and DML * command execution. */ void LockShardResource(uint64 shardId, LOCKMODE lockmode) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SET_LOCKTAG_SHARD_RESOURCE(tag, MyDatabaseId, shardId); (void) LockAcquire(&tag, lockmode, sessionLock, dontWait); } /* LockTransactionRecovery acquires a lock for transaction recovery */ void LockTransactionRecovery(LOCKMODE lockmode) { LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; SET_LOCKTAG_CITUS_OPERATION(tag, CITUS_TRANSACTION_RECOVERY); (void) LockAcquire(&tag, lockmode, sessionLock, dontWait); } /* * LockShardListMetadata takes shared locks on the metadata of all shards in * shardIntervalList to prevents concurrent placement changes. */ void LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode) { /* lock shards in order of shard id to prevent deadlock */ shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; LockShardDistributionMetadata(shardId, lockMode); } } /* * LockShardsInPlacementListMetadata takes locks on the metadata of all shards in * shardPlacementList to prevent concurrent placement changes. */ void LockShardsInPlacementListMetadata(List *shardPlacementList, LOCKMODE lockMode) { /* lock shards in order of shard id to prevent deadlock */ shardPlacementList = SortList(shardPlacementList, CompareShardPlacementsByShardId); GroupShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacementList) { int64 shardId = placement->shardId; LockShardDistributionMetadata(shardId, lockMode); } } /* * SerializeNonCommutativeWrites acquires the required locks to prevent concurrent * writes on the given shards. * * If the modified shard is a reference table's shard and the cluster is an MX * cluster we need to get shard resource lock on the first worker node to * prevent divergence possibility between placements of the reference table. * * In other workers, by acquiring a lock on the first worker, we're serializing * non-commutative modifications to a reference table. If the node executing the * command is the first worker, defined via IsFirstWorker(), we skip acquiring * the lock remotely to avoid an extra round-trip and/or self-deadlocks. * * Finally, if we're not dealing with reference tables on MX cluster, we'll * always acquire the lock with LockShardListResources() call. */ void SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode) { if (shardIntervalList == NIL) { return; } List *replicatedShardList = NIL; bool anyTableReplicated = AnyTableReplicated(shardIntervalList, &replicatedShardList); /* * Acquire locks on the modified table. * If the table is replicated, the locks are first acquired on the first worker node then locally. * But if we're already on the first worker, acquiring on the first worker node and locally are the same operation. * So we only acquire locally in that case. */ if (anyTableReplicated && ClusterHasKnownMetadataWorkers() && !IsFirstWorkerNode()) { LockShardListResourcesOnFirstWorker(lockMode, replicatedShardList); } LockShardListResources(shardIntervalList, lockMode); /* * Next, acquire locks on the reference tables that are referenced by a foreign key if there are any. * Note that LockReferencedReferenceShardResources() first acquires locks on the first worker, * then locally. */ if (anyTableReplicated) { ShardInterval *firstShardInterval = (ShardInterval *) linitial(replicatedShardList); if (ReferenceTableShardId(firstShardInterval->shardId)) { /* * Referenced tables can cascade their changes to this table, and we * want to serialize changes to keep different replicas consistent. * * We currently only support foreign keys to reference tables, which are * single shard. So, getting the first shard should be sufficient here. */ LockReferencedReferenceShardResources(firstShardInterval->shardId, lockMode); } } } /* * AnyTableReplicated iterates on the shard list and returns true * if any of the shard is a replicated table. We qualify replicated * tables as any reference table or any distributed table with * replication factor > 1. * * If the optional replicatedShardIntervalList is passed, the function * fills it with the replicated shard intervals. */ static bool AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList) { List *localList = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; Oid relationId = RelationIdForShard(shardId); if (ReferenceTableShardId(shardId)) { localList = lappend(localList, LoadShardInterval(shardId)); } else if (!SingleReplicatedTable(relationId)) { localList = lappend(localList, LoadShardInterval(shardId)); } } if (replicatedShardIntervalList != NULL) { *replicatedShardIntervalList = localList; } return list_length(localList) > 0; } /* * LockShardListResources takes locks on all shards in shardIntervalList to * prevent concurrent DML statements on those shards. */ static void LockShardListResources(List *shardIntervalList, LOCKMODE lockMode) { /* lock shards in order of shard id to prevent deadlock */ shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { int64 shardId = shardInterval->shardId; LockShardResource(shardId, lockMode); } } /* * LockRelationShardResources takes locks on all shards in a list of RelationShards * to prevent concurrent DML statements on those shards. */ void LockRelationShardResources(List *relationShardList, LOCKMODE lockMode) { if (relationShardList == NIL) { return; } List *shardIntervalList = NIL; RelationShard *relationShard = NULL; foreach_declared_ptr(relationShard, relationShardList) { uint64 shardId = relationShard->shardId; ShardInterval *shardInterval = LoadShardInterval(shardId); shardIntervalList = lappend(shardIntervalList, shardInterval); } /* lock shards in a consistent order to prevent deadlock */ shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById); SerializeNonCommutativeWrites(shardIntervalList, lockMode); } /* * LockParentShardResourceIfPartition checks whether the given shard belongs * to a partition. If it does, LockParentShardResourceIfPartition acquires a * shard resource lock on the colocated shard of the parent table. */ void LockParentShardResourceIfPartition(List *shardIntervalList, LOCKMODE lockMode) { List *parentShardIntervalList = NIL; ShardInterval *shardInterval = NULL; foreach_declared_ptr(shardInterval, shardIntervalList) { Oid relationId = shardInterval->relationId; if (PartitionTable(relationId)) { int shardIndex = ShardIndex(shardInterval); Oid parentRelationId = PartitionParentOid(relationId); uint64 parentShardId = ColocatedShardIdInRelation(parentRelationId, shardIndex); ShardInterval *parentShardInterval = LoadShardInterval(parentShardId); parentShardIntervalList = lappend(parentShardIntervalList, parentShardInterval); } } LockShardListResources(parentShardIntervalList, lockMode); } /* * LockModeTextToLockMode gets a lockMode name and returns its corresponding LOCKMODE. * The function errors out if the input lock mode isn't defined in the PostgreSQL's * explicit locking table. */ LOCKMODE LockModeTextToLockMode(const char *lockModeName) { LOCKMODE lockMode = -1; for (int lockIndex = 0; lockIndex < lock_mode_to_string_map_count; lockIndex++) { const struct LockModeToStringType *lockMap = lockmode_to_string_map + lockIndex; if (pg_strncasecmp(lockMap->name, lockModeName, NAMEDATALEN) == 0) { lockMode = lockMap->lockMode; break; } } /* we could not find the lock mode we are looking for */ if (lockMode == -1) { ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("unknown lock mode: %s", lockModeName))); } return lockMode; } /* * LockModeToLockModeText gets a lockMode enum and returns its corresponding text * representation. * The function errors out if the input lock mode isn't defined in the PostgreSQL's * explicit locking table. */ const char * LockModeToLockModeText(LOCKMODE lockMode) { const char *lockModeText = NULL; for (int lockIndex = 0; lockIndex < lock_mode_to_string_map_count; lockIndex++) { const struct LockModeToStringType *lockMap = lockmode_to_string_map + lockIndex; if (lockMode == lockMap->lockMode) { lockModeText = lockMap->name; break; } } /* we could not find the lock mode we are looking for */ if (lockModeText == NULL) { ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("unknown lock mode enum: %d", (int) lockMode))); } return lockModeText; } /* * lock_relation_if_exists gets a relation name and lock mode * and returns true if the relation exists and can be locked with * the given lock mode. If the relation doesn't exists, the function * return false. * * The relation name should be qualified with the schema name. * * The function errors out if the lockmode isn't defined in the PostgreSQL's * explicit locking table. */ Datum lock_relation_if_exists(PG_FUNCTION_ARGS) { text *relationName = PG_GETARG_TEXT_P(0); text *lockModeText = PG_GETARG_TEXT_P(1); char *lockModeCString = text_to_cstring(lockModeText); /* get the lock mode */ LOCKMODE lockMode = LockModeTextToLockMode(lockModeCString); /* resolve relationId from passed in schema and relation name */ List *relationNameList = textToQualifiedNameList(relationName); RangeVar *relation = makeRangeVarFromNameList(relationNameList); /* lock the relation with the lock mode */ Oid relationId = RangeVarGetRelidExtended(relation, lockMode, RVR_MISSING_OK, CitusRangeVarCallbackForLockTable, (void *) &lockMode); bool relationExists = OidIsValid(relationId); PG_RETURN_BOOL(relationExists); } /* * CitusRangeVarCallbackForLockTable is a callback for RangeVarGetRelidExtended used * to check whether the user has permission to lock a table in a particular mode. * * This function is a copy of RangeVarCallbackForLockTable in lockcmds.c adapted to * Citus code style. */ static void CitusRangeVarCallbackForLockTable(const RangeVar *rangeVar, Oid relationId, Oid oldRelationId, void *arg) { LOCKMODE lockmode = *(LOCKMODE *) arg; if (!OidIsValid(relationId)) { /* table doesn't exist, so no permissions check */ return; } /* we only allow tables, views and foreign tables to be locked */ if (!RegularTable(relationId) && !IsForeignTable(relationId)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", rangeVar->relname))); } /* check permissions */ AclResult aclResult = CitusLockTableAclCheck(relationId, lockmode, GetUserId()); if (aclResult != ACLCHECK_OK) { aclcheck_error(aclResult, get_relkind_objtype(get_rel_relkind(relationId)), rangeVar->relname); } } /* * CitusLockTableAclCheck checks whether a user has permission to lock a relation * in the given lock mode. * * This function is a copy of LockTableAclCheck in lockcmds.c adapted to Citus * code style. */ static AclResult CitusLockTableAclCheck(Oid relationId, LOCKMODE lockmode, Oid userId) { AclMode aclMask; /* verify adequate privilege */ if (lockmode == AccessShareLock) { aclMask = ACL_SELECT; } else if (lockmode == RowExclusiveLock) { aclMask = ACL_INSERT | ACL_UPDATE | ACL_DELETE | ACL_TRUNCATE; } else { aclMask = ACL_UPDATE | ACL_DELETE | ACL_TRUNCATE; } AclResult aclResult = pg_class_aclcheck(relationId, userId, aclMask); return aclResult; } /* * EnsureCanAcquireLock checks if currect user has the permissions * to acquire a lock on the table and throws an error if the user does * not have the permissions */ static void EnsureCanAcquireLock(Oid relationId, LOCKMODE lockMode) { AclResult aclResult = CitusLockTableAclCheck(relationId, lockMode, GetUserId()); if (aclResult != ACLCHECK_OK) { aclcheck_error(aclResult, get_relkind_objtype(get_rel_relkind(relationId)), get_rel_name(relationId)); } } /* * CreateLockTerminationString creates a string that can be appended to the * end of a partial lock command to properly terminate the command */ static const char * CreateLockTerminationString(const char *lockModeText, bool nowait) { StringInfo lockTerminationStringInfo = makeStringInfo(); appendStringInfo(lockTerminationStringInfo, nowait ? " IN %s MODE NOWAIT;\n" : " IN %s MODE;\n", lockModeText); return lockTerminationStringInfo->data; } /* * FinishLockCommandIfNecessary appends the lock termination string if the lock command is partial. * Sets the partialLockCommand flag to false */ static void FinishLockCommandIfNecessary(StringInfo lockCommand, const char *lockTerminationString, bool *partialLockCommand) { if (*partialLockCommand) { appendStringInfo(lockCommand, "%s", lockTerminationString); } *partialLockCommand = false; } /* * LockRelationRecordListMember checks if a relation id is present in the * LockRelationRecord list */ static bool LockRelationRecordListMember(List *lockRelationRecordList, Oid relationId) { LockRelationRecord *record = NULL; foreach_declared_ptr(record, lockRelationRecordList) { if (record->relationId == relationId) { return true; } } return false; } /* * MakeLockRelationRecord makes a LockRelationRecord using the relation oid * and the inh boolean while properly allocating the structure */ static LockRelationRecord * MakeLockRelationRecord(Oid relationId, bool inh) { LockRelationRecord *lockRelationRecord = palloc(sizeof(LockRelationRecord)); lockRelationRecord->relationId = relationId; lockRelationRecord->inh = inh; return lockRelationRecord; } /* * ConcatLockRelationRecordList concats a list of LockRelationRecord with * another list of LockRelationRecord created from a list of relation oid-s * which are not present in the first list and an inh bool which will be * applied across all LockRelationRecords */ static List * ConcatLockRelationRecordList(List *lockRelationRecordList, List *relationOidList, bool inh) { List *constructedList = NIL; Oid relationId = InvalidOid; foreach_declared_oid(relationId, relationOidList) { if (!LockRelationRecordListMember(lockRelationRecordList, relationId)) { LockRelationRecord *record = MakeLockRelationRecord(relationId, inh); constructedList = lappend(constructedList, (void *) record); } } return list_concat(lockRelationRecordList, constructedList); } /* * AcquireDistributedLockOnRelations_Internal acquire a distributed lock on worker nodes * for given list of relations ids. Worker node list is sorted so that the lock * is acquired in the same order regardless of which node it was run on. * * A nowait flag is used to require the locks to be available immediately * and if that is not the case, an error will be thrown * * Notice that no validation or filtering is done on the relationIds, that is the responsibility * of this function's caller. */ static void AcquireDistributedLockOnRelations_Internal(List *lockRelationRecordList, LOCKMODE lockMode, bool nowait) { const char *lockModeText = LockModeToLockModeText(lockMode); UseCoordinatedTransaction(); StringInfo lockRelationsCommand = makeStringInfo(); appendStringInfo(lockRelationsCommand, "%s;\n", DISABLE_DDL_PROPAGATION); /* * In the following loop, when there are foreign tables, we need to switch from * LOCK * statement to lock_relation_if_exists() and vice versa since pg * does not support using LOCK on foreign tables. */ bool startedLockCommand = false; /* create a lock termination string used to terminate a partial lock command */ const char *lockTerminationString = CreateLockTerminationString(lockModeText, nowait); int lockedRelations = 0; LockRelationRecord *lockRelationRecord; foreach_declared_ptr(lockRelationRecord, lockRelationRecordList) { Oid relationId = lockRelationRecord->relationId; bool lockDescendants = lockRelationRecord->inh; char *qualifiedRelationName = generate_qualified_relation_name(relationId); /* * As of pg14 we cannot use LOCK to lock a FOREIGN TABLE * so we have to use the lock_relation_if_exist udf */ if (get_rel_relkind(relationId) == RELKIND_FOREIGN_TABLE) { FinishLockCommandIfNecessary(lockRelationsCommand, lockTerminationString, &startedLockCommand); /* * The user should not be able to trigger this codepath * with nowait = true or lockDescendants = false since the * only way to do that is calling LOCK * IN * MODE NOWAIT; * and LOCK ONLY * IN * MODE; respectively but foreign tables * cannot be locked with LOCK command as of pg14 */ Assert(nowait == false); Assert(lockDescendants == true); /* use lock_relation_if_exits to lock foreign table */ appendStringInfo(lockRelationsCommand, LOCK_RELATION_IF_EXISTS, quote_literal_cstr(qualifiedRelationName), quote_literal_cstr(lockModeText)); appendStringInfoChar(lockRelationsCommand, '\n'); } else if (startedLockCommand) { /* append relation to partial lock statement */ appendStringInfo(lockRelationsCommand, ",%s%s", lockDescendants ? " " : " ONLY ", qualifiedRelationName); } else { /* start a new partial lock statement */ appendStringInfo(lockRelationsCommand, "LOCK%s%s", lockDescendants ? " " : " ONLY ", qualifiedRelationName); startedLockCommand = true; } lockedRelations++; } if (lockedRelations == 0) { return; } FinishLockCommandIfNecessary(lockRelationsCommand, lockTerminationString, &startedLockCommand); appendStringInfo(lockRelationsCommand, ENABLE_DDL_PROPAGATION); char *lockCommand = lockRelationsCommand->data; List *workerNodeList = TargetWorkerSetNodeList(METADATA_NODES, NoLock); /* * We want to acquire locks in the same order across the nodes. * Although relation ids may change, their ordering will not. */ workerNodeList = SortList(workerNodeList, CompareWorkerNodes); int32 localGroupId = GetLocalGroupId(); WorkerNode *workerNode = NULL; const char *currentUser = CurrentUserName(); foreach_declared_ptr(workerNode, workerNodeList) { /* if local node is one of the targets, acquire the lock locally */ if (workerNode->groupId == localGroupId) { ExecuteUtilityCommand(lockCommand); continue; } SendMetadataCommandListToWorkerListInCoordinatedTransaction(list_make1( workerNode), currentUser, list_make1( lockCommand)); } } /* * AcquireDistributedLockOnRelations filters relations before passing them to * AcquireDistributedLockOnRelations_Internal to acquire the locks. * * Only tables, views, and foreign tables can be locked with this function. Other relations * will cause an error. * * Skips non distributed relations. * configs parameter is used to configure what relation will be locked and if the lock * should throw an error if it cannot be acquired immediately */ void AcquireDistributedLockOnRelations(List *relationList, LOCKMODE lockMode, uint32 configs) { if (!ClusterHasKnownMetadataWorkers() || !EnableMetadataSync || !EnableDDLPropagation) { return; } /* used to store the relations that will be locked */ List *distributedRelationRecordsList = NIL; bool nowait = (configs & DIST_LOCK_NOWAIT) > 0; RangeVar *rangeVar = NULL; foreach_declared_ptr(rangeVar, relationList) { Oid relationId = RangeVarGetRelid(rangeVar, NoLock, false); LockRelationRecord *lockRelationRecord = MakeLockRelationRecord(relationId, rangeVar->inh); /* * Note that allowing the user to lock shards could lead to * distributed deadlocks due to shards not being locked when * a distributed table is locked. * However, because citus.enable_manual_changes_to_shards * is a guc which is not visible by default, whoever is using this * guc will hopefully know what they're doing and avoid such scenarios. */ ErrorIfIllegallyChangingKnownShard(relationId); /* * we want to prevent under privileged users to trigger establishing connections, * that's why we have this additional check. Otherwise, we'd get the errors as * soon as we execute the command on any node */ EnsureCanAcquireLock(relationId, lockMode); bool isView = get_rel_relkind(relationId) == RELKIND_VIEW; if ((isView && !IsViewDistributed(relationId)) || (!isView && !ShouldSyncTableMetadata(relationId))) { continue; } if (!LockRelationRecordListMember(distributedRelationRecordsList, relationId)) { distributedRelationRecordsList = lappend(distributedRelationRecordsList, (void *) lockRelationRecord); } char relkind = get_rel_relkind(relationId); bool relationCanBeReferenced = (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE); if (relationCanBeReferenced && (configs & DIST_LOCK_REFERENCING_TABLES) > 0) { CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId); Assert(cacheEntry != NULL); List *referencingTableList = cacheEntry->referencingRelationsViaForeignKey; /* remove the relations which should not be synced */ referencingTableList = list_filter_oid(referencingTableList, &ShouldSyncTableMetadata); distributedRelationRecordsList = ConcatLockRelationRecordList( distributedRelationRecordsList, referencingTableList, true); } } if (distributedRelationRecordsList != NIL) { if (!IsCoordinator() && !CoordinatorAddedAsWorkerNode() && !EnableAcquiringUnsafeLockFromWorkers) { ereport(ERROR, (errmsg( "Cannot acquire a distributed lock from a worker node since the " "coordinator is not in the metadata."), errhint( "Either run this command on the coordinator or add the coordinator " "in the metadata by using: SELECT citus_set_coordinator_host('', );\n" "Alternatively, though it is not recommended, you can allow this command by running: " "SET citus.allow_unsafe_locks_from_workers TO 'on';"))); } AcquireDistributedLockOnRelations_Internal(distributedRelationRecordsList, lockMode, nowait); } } /** * PreprocessLockStatement makes sure that the lock is allowed * before calling AcquireDistributedLockOnRelations on the locked tables/views * with flags DIST_LOCK_VIEWS_RECUR and DIST_LOCK_NOWAIT if nowait is true for * the lock statement */ void PreprocessLockStatement(LockStmt *stmt, ProcessUtilityContext context) { bool isTopLevel = context == PROCESS_UTILITY_TOPLEVEL; RequireTransactionBlock(isTopLevel, "LOCK TABLE"); uint32 nowaitFlag = stmt->nowait ? DIST_LOCK_NOWAIT : 0; AcquireDistributedLockOnRelations(stmt->relations, stmt->mode, nowaitFlag); } ================================================ FILE: src/backend/distributed/utils/role.c ================================================ /*------------------------------------------------------------------------- * * role.c * * Utilities for ALTER ROLE statments. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "distributed/commands/utility_hook.h" #include "distributed/worker_protocol.h" PG_FUNCTION_INFO_V1(alter_role_if_exists); PG_FUNCTION_INFO_V1(worker_create_or_alter_role); /* * alter_role_if_exists checks if the role, whose name is given * in the first parameter exists and then runs the query, which is the second * parameter. This UDF is particularly used for ALTER ROLE queries, how ever it * can run any other query too. */ Datum alter_role_if_exists(PG_FUNCTION_ARGS) { text *rolenameText = PG_GETARG_TEXT_P(0); const char *rolename = text_to_cstring(rolenameText); text *utilityQueryText = PG_GETARG_TEXT_P(1); const char *utilityQuery = text_to_cstring(utilityQueryText); if (get_role_oid(rolename, true) == InvalidOid) { PG_RETURN_BOOL(false); } Node *parseTree = ParseTreeNode(utilityQuery); ProcessUtilityParseTree(parseTree, utilityQuery, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); PG_RETURN_BOOL(true); } /* * worker_create_or_alter_role( * role_name text, * create_role_utility_query text, * alter_role_utility_query text) * * This UDF checks if the role, whose name is given in role_name exists. * * If the role does not exist it will run the query provided in create_role_utility_query * which is expected to be a CreateRoleStmt. If a different statement is provided the call * will raise an error, * * If the role does exist it will run the query provided in alter_role_utility_query to * change the existing user in such a way that it is compatible with the user on the * coordinator. This query is expected to be a AlterRoleStmt, if a different statement is * provided the function will raise an error. * * For both queries a NULL value can be passed to omit the execution of that condition. * * The function returns true if a command has been successfully executed, false if no * command was executed, and raises an error if something went wrong. */ Datum worker_create_or_alter_role(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { ereport(ERROR, (errmsg("role name cannot be NULL"))); } text *rolenameText = PG_GETARG_TEXT_P(0); const char *rolename = text_to_cstring(rolenameText); if (get_role_oid(rolename, true) == InvalidOid) { if (PG_ARGISNULL(1)) { PG_RETURN_BOOL(false); } text *createRoleUtilityQueryText = PG_GETARG_TEXT_P(1); const char *createRoleUtilityQuery = text_to_cstring(createRoleUtilityQueryText); Node *parseTree = ParseTreeNode(createRoleUtilityQuery); if (nodeTag(parseTree) != T_CreateRoleStmt) { ereport(ERROR, (errmsg("cannot create role"), errdetail("the role %s does not exist " "but %s is not a correct CREATE ROLE query", quote_literal_cstr(rolename), quote_literal_cstr(createRoleUtilityQuery)))); } ProcessUtilityParseTree(parseTree, createRoleUtilityQuery, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); PG_RETURN_BOOL(true); } else { if (PG_ARGISNULL(2)) { PG_RETURN_BOOL(false); } text *alterRoleUtilityQueryText = PG_GETARG_TEXT_P(2); const char *alterRoleUtilityQuery = text_to_cstring(alterRoleUtilityQueryText); Node *parseTree = ParseTreeNode(alterRoleUtilityQuery); if (nodeTag(parseTree) != T_AlterRoleStmt) { ereport(ERROR, (errmsg("cannot alter role"), errdetail("the role %s exists " "but %s is not a correct alter ROLE query", quote_literal_cstr(rolename), quote_literal_cstr(alterRoleUtilityQuery)))); } ProcessUtilityParseTree(parseTree, alterRoleUtilityQuery, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); PG_RETURN_BOOL(true); } } ================================================ FILE: src/backend/distributed/utils/shard_utils.c ================================================ /*------------------------------------------------------------------------- * * shard_utils.c * * This file contains functions to perform useful operations on shards. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "utils/builtins.h" #include "utils/fmgrprotos.h" #include "utils/lsyscache.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" #include "distributed/metadata_utility.h" #include "distributed/multi_physical_planner.h" #include "distributed/relay_utility.h" #include "distributed/shard_utils.h" static int GetLargestShardId(void); /* * GetTableLocalShardOid returns the oid of the shard from the given distributed * relation with the shardId. */ Oid GetTableLocalShardOid(Oid citusTableOid, uint64 shardId) { const char *citusTableName = get_rel_name(citusTableOid); Assert(citusTableName != NULL); /* construct shard relation name */ char *shardRelationName = pstrdup(citusTableName); AppendShardIdToName(&shardRelationName, shardId); Oid citusTableSchemaOid = get_rel_namespace(citusTableOid); Oid shardRelationOid = get_relname_relid(shardRelationName, citusTableSchemaOid); return shardRelationOid; } /* * GetLongestShardName is a utility function that returns the name of the shard of a * table that has the longest name in terms of number of characters. * * Both the Oid and name of the table are required so we can create longest shard names * after a RENAME. */ char * GetLongestShardName(Oid citusTableOid, char *finalRelationName) { char *longestShardName = pstrdup(finalRelationName); ShardInterval *shardInterval = LoadShardIntervalWithLongestShardName(citusTableOid); AppendShardIdToName(&longestShardName, shardInterval->shardId); return longestShardName; } /* * GetLongestShardNameForLocalPartition is a utility function that creates a hypothetical shard * name for a partition table that is not distributed yet. */ char * GetLongestShardNameForLocalPartition(Oid parentTableOid, char *partitionRelationName) { char *longestShardName = pstrdup(partitionRelationName); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(parentTableOid); int shardIntervalCount = cacheEntry->shardIntervalArrayLength; int newShardId = GetLargestShardId() + shardIntervalCount; AppendShardIdToName(&longestShardName, newShardId); return longestShardName; } /* * GetLargestShardId returns the biggest shard id, and returns a 10^6 in case of failure * to get the last value from the sequence. */ int GetLargestShardId() { Oid savedUserId = InvalidOid; int savedSecurityContext = 0; GetUserIdAndSecContext(&savedUserId, &savedSecurityContext); SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE); text *sequenceName = cstring_to_text(SHARDID_SEQUENCE_NAME); Oid sequenceId = ResolveRelationId(sequenceName, false); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); volatile int64 largestShardId = 0; /* * pg_sequence_last_value() returns NULL if the sequence value is not yet used. * DirectFunctionCall1() gives an ERROR message on NULL return values, and that's why we * need a PG_TRY block. */ PG_TRY(); { Datum lastShardIdDatum = DirectFunctionCall1(pg_sequence_last_value, sequenceIdDatum); largestShardId = DatumGetInt64(lastShardIdDatum); } PG_CATCH(); { /* assume that we have a shardId with 7 digits */ largestShardId = 1000000; } PG_END_TRY(); SetUserIdAndSecContext(savedUserId, savedSecurityContext); return largestShardId; } ================================================ FILE: src/backend/distributed/utils/shardinterval_utils.c ================================================ /*------------------------------------------------------------------------- * * shardinterval_utils.c * * This file contains functions to perform useful operations on shard intervals. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "stdint.h" #include "access/nbtree.h" #include "catalog/pg_am.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "utils/catcache.h" #include "utils/memutils.h" #include "distributed/distributed_planner.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/pg_dist_partition.h" #include "distributed/shard_pruning.h" #include "distributed/shardinterval_utils.h" #include "distributed/worker_protocol.h" /* * SortedShardIntervalArray sorts the input shardIntervalArray. Shard intervals with * no min/max values are placed at the end of the array. */ ShardInterval ** SortShardIntervalArray(ShardInterval **shardIntervalArray, int shardCount, Oid collation, FmgrInfo *shardIntervalSortCompareFunction) { SortShardIntervalContext sortContext = { .comparisonFunction = shardIntervalSortCompareFunction, .collation = collation }; /* short cut if there are no shard intervals in the array */ if (shardCount == 0) { return shardIntervalArray; } /* if a shard doesn't have min/max values, it's placed in the end of the array */ qsort_arg(shardIntervalArray, shardCount, sizeof(ShardInterval *), (qsort_arg_comparator) CompareShardIntervals, (void *) &sortContext); return shardIntervalArray; } /* * CompareShardIntervals acts as a helper function to compare two shard intervals * by their minimum values, using the value's type comparison function. * * If a shard interval does not have min/max value, it's treated as being greater * than the other. */ int CompareShardIntervals(const void *leftElement, const void *rightElement, SortShardIntervalContext *sortContext) { ShardInterval *leftShardInterval = *((ShardInterval **) leftElement); ShardInterval *rightShardInterval = *((ShardInterval **) rightElement); int comparisonResult = 0; bool leftHasNull = (!leftShardInterval->minValueExists || !leftShardInterval->maxValueExists); bool rightHasNull = (!rightShardInterval->minValueExists || !rightShardInterval->maxValueExists); Assert(sortContext->comparisonFunction != NULL); if (leftHasNull && rightHasNull) { comparisonResult = 0; } else if (leftHasNull) { comparisonResult = 1; } else if (rightHasNull) { comparisonResult = -1; } else { /* if both shard interval have min/max values, calculate comparison result */ Datum leftDatum = leftShardInterval->minValue; Datum rightDatum = rightShardInterval->minValue; Datum comparisonDatum = FunctionCall2Coll(sortContext->comparisonFunction, sortContext->collation, leftDatum, rightDatum); comparisonResult = DatumGetInt32(comparisonDatum); } /* Two different shards should never be equal */ if (comparisonResult == 0) { return CompareShardIntervalsById(leftElement, rightElement); } return comparisonResult; } /* * CompareShardIntervalsById is a comparison function for sort shard * intervals by their shard ID. */ int CompareShardIntervalsById(const void *leftElement, const void *rightElement) { ShardInterval *leftInterval = *((ShardInterval **) leftElement); ShardInterval *rightInterval = *((ShardInterval **) rightElement); int64 leftShardId = leftInterval->shardId; int64 rightShardId = rightInterval->shardId; /* we compare 64-bit integers, instead of casting their difference to int */ if (leftShardId > rightShardId) { return 1; } else if (leftShardId < rightShardId) { return -1; } else { return 0; } } /* * CompareShardPlacementsByShardId is a comparison function for sorting shard * placement by their shard ID. */ int CompareShardPlacementsByShardId(const void *leftElement, const void *rightElement) { GroupShardPlacement *left = *((GroupShardPlacement **) leftElement); GroupShardPlacement *right = *((GroupShardPlacement **) rightElement); int64 leftShardId = left->shardId; int64 rightShardId = right->shardId; /* we compare 64-bit integers, instead of casting their difference to int */ if (leftShardId > rightShardId) { return 1; } else if (leftShardId < rightShardId) { return -1; } else { return 0; } } /* * CompareRelationShards is a comparison function for sorting relation * to shard mappings by their relation ID and then shard ID. */ int CompareRelationShards(const void *leftElement, const void *rightElement) { RelationShard *leftRelationShard = *((RelationShard **) leftElement); RelationShard *rightRelationShard = *((RelationShard **) rightElement); Oid leftRelationId = leftRelationShard->relationId; Oid rightRelationId = rightRelationShard->relationId; int64 leftShardId = leftRelationShard->shardId; int64 rightShardId = rightRelationShard->shardId; if (leftRelationId > rightRelationId) { return 1; } else if (leftRelationId < rightRelationId) { return -1; } else if (leftShardId > rightShardId) { return 1; } else if (leftShardId < rightShardId) { return -1; } else { return 0; } } /* * ShardIndex finds the index of given shard in sorted shard interval array. * * For hash partitioned tables, it calculates hash value of a number in its * range (e.g. min value) and finds which shard should contain the hashed * value. For the tables that don't have a shard key, it simply returns 0. * For the other table types, the function errors out. */ int ShardIndex(ShardInterval *shardInterval) { int shardIndex = INVALID_SHARD_INDEX; Oid distributedTableId = shardInterval->relationId; Datum shardMinValue = shardInterval->minValue; CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(distributedTableId); /* * Note that, we can also support append and range distributed tables, but * currently it is not required. */ if (!IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) && HasDistributionKeyCacheEntry(cacheEntry)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("finding index of a given shard is only supported for " "hash distributed tables, reference tables and local " "tables that are added to citus metadata"))); } /* short-circuit for the tables that don't have a distribution key */ if (!HasDistributionKeyCacheEntry(cacheEntry)) { /* * Such tables have only a single shard, so the index is fixed to 0. */ shardIndex = 0; return shardIndex; } shardIndex = FindShardIntervalIndex(shardMinValue, cacheEntry); return shardIndex; } /* * FindShardInterval finds a single shard interval in the cache for the * given partition column value. Note that reference tables do not have * partition columns, thus, pass partitionColumnValue and compareFunction * as NULL for them. */ ShardInterval * FindShardInterval(Datum partitionColumnValue, CitusTableCacheEntry *cacheEntry) { Datum searchedValue = partitionColumnValue; if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED)) { searchedValue = FunctionCall1Coll(cacheEntry->hashFunction, cacheEntry->partitionColumn->varcollid, partitionColumnValue); } int shardIndex = FindShardIntervalIndex(searchedValue, cacheEntry); if (shardIndex == INVALID_SHARD_INDEX) { return NULL; } return cacheEntry->sortedShardIntervalArray[shardIndex]; } /* * FindShardIntervalIndex finds the index of the shard interval which covers * the searched value. Note that the searched value must be the hashed value * of the original value if the distribution method is hash. * * Note that, if the searched value can not be found for hash partitioned * tables, we error out (unless there are no shards, in which case * INVALID_SHARD_INDEX is returned). This should only happen if something is * terribly wrong, either metadata tables are corrupted or we have a bug * somewhere. Such as a hash function which returns a value not in the range * of [PG_INT32_MIN, PG_INT32_MAX] can fire this. */ int FindShardIntervalIndex(Datum searchedValue, CitusTableCacheEntry *cacheEntry) { ShardInterval **shardIntervalCache = cacheEntry->sortedShardIntervalArray; int shardCount = cacheEntry->shardIntervalArrayLength; FmgrInfo *compareFunction = cacheEntry->shardIntervalCompareFunction; bool useBinarySearch = (!IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED) || !cacheEntry->hasUniformHashDistribution); int shardIndex = INVALID_SHARD_INDEX; if (shardCount == 0) { return INVALID_SHARD_INDEX; } if (IsCitusTableTypeCacheEntry(cacheEntry, HASH_DISTRIBUTED)) { if (useBinarySearch) { Assert(compareFunction != NULL); Oid shardIntervalCollation = cacheEntry->partitionColumn->varcollid; shardIndex = SearchCachedShardInterval(searchedValue, shardIntervalCache, shardCount, shardIntervalCollation, compareFunction); /* we should always return a valid shard index for hash partitioned tables */ if (shardIndex == INVALID_SHARD_INDEX) { ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("cannot find shard interval"), errdetail("Hash of the partition column value " "does not fall into any shards."))); } } else { int hashedValue = DatumGetInt32(searchedValue); shardIndex = CalculateUniformHashRangeIndex(hashedValue, shardCount); } } else if (!HasDistributionKeyCacheEntry(cacheEntry)) { /* non-distributed tables have a single shard, all values mapped to that shard */ Assert(shardCount == 1); shardIndex = 0; } else { Assert(compareFunction != NULL); Oid shardIntervalCollation = cacheEntry->partitionColumn->varcollid; shardIndex = SearchCachedShardInterval(searchedValue, shardIntervalCache, shardCount, shardIntervalCollation, compareFunction); } return shardIndex; } /* * SearchCachedShardInterval performs a binary search for a shard interval * matching a given partition column value and returns its index in the cached * array. If it can not find any shard interval with the given value, it returns * INVALID_SHARD_INDEX. * * TODO: Data re-partitioning logic (worker_partition_query_resul)) * on the worker nodes relies on this function in order to be consistent * with shard pruning. Since the worker nodes don't have the metadata, a * synthetically generated ShardInterval ** is passed to this * function. The synthetic shard intervals contain only shardmin and shardmax * values. A proper implementation of this approach should be introducing an * intermediate data structure (e.g., ShardRange) on which this function * operates instead of operating shard intervals. */ int SearchCachedShardInterval(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, Oid shardIntervalCollation, FmgrInfo *compareFunction) { int lowerBoundIndex = 0; int upperBoundIndex = shardCount; while (lowerBoundIndex < upperBoundIndex) { int middleIndex = (lowerBoundIndex + upperBoundIndex) / 2; int minValueComparison = FunctionCall2Coll(compareFunction, shardIntervalCollation, partitionColumnValue, shardIntervalCache[middleIndex]-> minValue); if (DatumGetInt32(minValueComparison) < 0) { upperBoundIndex = middleIndex; continue; } int maxValueComparison = FunctionCall2Coll(compareFunction, shardIntervalCollation, partitionColumnValue, shardIntervalCache[middleIndex]-> maxValue); if (DatumGetInt32(maxValueComparison) <= 0) { return middleIndex; } lowerBoundIndex = middleIndex + 1; } return INVALID_SHARD_INDEX; } /* * CalculateUniformHashRangeIndex returns the index of the hash range in * which hashedValue falls, assuming shardCount uniform hash ranges. * * We use 64-bit integers to avoid overflow issues during arithmetic. * * NOTE: This function is ONLY for hash-distributed tables with uniform * hash ranges. */ int CalculateUniformHashRangeIndex(int hashedValue, int shardCount) { int64 hashedValue64 = (int64) hashedValue; /* normalize to the 0-UINT32_MAX range */ int64 normalizedHashValue = hashedValue64 - PG_INT32_MIN; /* size of each hash range */ int64 hashRangeSize = HASH_TOKEN_COUNT / shardCount; /* index of hash range into which the hash value falls */ int shardIndex = (int) (normalizedHashValue / hashRangeSize); if (shardIndex < 0 || shardIndex > shardCount) { ereport(ERROR, (errmsg("bug: shard index %d out of bounds", shardIndex))); } /* * If the shard count is not power of 2, the range of the last * shard becomes larger than others. For that extra piece of range, * we still need to use the last shard. */ if (shardIndex == shardCount) { shardIndex = shardCount - 1; } return shardIndex; } /* * SingleReplicatedTable checks whether all shards of a distributed table, do not have * more than one replica. If even one shard has more than one replica, this function * returns false, otherwise it returns true. */ bool SingleReplicatedTable(Oid relationId) { List *shardList = LoadShardList(relationId); List *shardPlacementList = NIL; /* we could have append/range distributed tables without shards */ if (list_length(shardList) == 0) { return false; } uint64 *shardIdPointer = NULL; foreach_declared_ptr(shardIdPointer, shardList) { uint64 shardId = *shardIdPointer; shardPlacementList = ShardPlacementList(shardId); if (list_length(shardPlacementList) != 1) { return false; } } return true; } ================================================ FILE: src/backend/distributed/utils/string_utils.c ================================================ /*------------------------------------------------------------------------- * * string_utils.c * * This file contains functions to perform useful operations on strings. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "distributed/relay_utility.h" #include "distributed/string_utils.h" /* * ConvertIntToString returns the string version of given integer. */ char * ConvertIntToString(int val) { StringInfo str = makeStringInfo(); appendStringInfo(str, "%d", val); return str->data; } ================================================ FILE: src/backend/distributed/utils/task_execution_utils.c ================================================ #include #include #include #include "postgres.h" #include "miscadmin.h" #include "commands/dbcommands.h" #include "common/hashfn.h" #include "storage/fd.h" #include "utils/builtins.h" #include "utils/hsearch.h" #include "utils/timestamp.h" #include "pg_version_constants.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_nodes.h" #include "distributed/connection_management.h" #include "distributed/deparse_shard_query.h" #include "distributed/distributed_execution_locks.h" #include "distributed/hash_helpers.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/multi_executor.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/pg_dist_partition.h" #include "distributed/resource_lock.h" #include "distributed/subplan_execution.h" #include "distributed/task_execution_utils.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" /* TaskMapKey is used as a key in task hash */ typedef struct TaskMapKey { TaskType taskType; uint32 taskId; uint64 jobId; } TaskMapKey; /* * TaskMapEntry is used as entry in task hash. We need to keep a pointer * of the task in the entry. */ typedef struct TaskMapEntry { TaskMapKey key; Task *task; } TaskMapEntry; static Task * TaskHashEnter(HTAB *taskHash, Task *task); static Task * TaskHashLookup(HTAB *trackerHash, TaskType taskType, uint64 jobId, uint32 taskId); /* * CreateTaskListForJobTree visits all tasks in the job tree (by following dependentTaskList), * starting with the given job's task list. The function then returns the list. */ List * CreateTaskListForJobTree(List *jobTaskList) { List *taskList = NIL; const int topLevelTaskHashSize = 32; int taskHashSize = list_length(jobTaskList) * topLevelTaskHashSize; assert_valid_hash_key3(TaskMapKey, taskType, taskId, jobId); HTAB *taskHash = CreateSimpleHashWithSize(TaskMapKey, TaskMapEntry, taskHashSize); /* * We walk over the task tree using breadth-first search. For the search, we * first queue top level tasks in the task tree. */ List *taskQueue = list_copy(jobTaskList); while (taskQueue != NIL) { /* pop first element from the task queue */ Task *task = (Task *) linitial(taskQueue); taskQueue = list_delete_first(taskQueue); taskList = lappend(taskList, task); List *dependendTaskList = task->dependentTaskList; /* * Push task node's children into the task queue, if and only if * they're not already there. As task dependencies have to form a * directed-acyclic-graph and are processed in a breadth-first search * we can never re-encounter nodes we've already processed. * * While we're checking this, we can also fix the problem that * copyObject() might have duplicated nodes in the graph - if a node * isn't pushed to the graph because it is already planned to be * visited, we can simply replace it with the copy. Note that, here * we only consider dependend tasks. Since currently top level tasks * cannot be on any dependend task list, we do not check them for duplicates. * * taskHash is used to reduce the complexity of keeping track of * the tasks that are already encountered. */ ListCell *dependentTaskCell = NULL; foreach(dependentTaskCell, dependendTaskList) { Task *dependendTask = lfirst(dependentTaskCell); Task *dependendTaskInHash = TaskHashLookup(taskHash, dependendTask->taskType, dependendTask->jobId, dependendTask->taskId); /* * If the dependend task encountered for the first time, add it to the hash. * Also, add this task to the task queue. Note that, we do not need to * add the tasks to the queue which are already encountered, because * they are already added to the queue. */ if (!dependendTaskInHash) { dependendTaskInHash = TaskHashEnter(taskHash, dependendTask); taskQueue = lappend(taskQueue, dependendTaskInHash); } /* update dependentTaskList element to the one which is in the hash */ lfirst(dependentTaskCell) = dependendTaskInHash; } } return taskList; } /* * TaskHashEnter creates a reference to the task entry in the given task * hash. The function errors-out if the same key exists multiple times. */ static Task * TaskHashEnter(HTAB *taskHash, Task *task) { bool handleFound = false; TaskMapKey taskKey; memset(&taskKey, 0, sizeof(TaskMapKey)); taskKey.taskType = task->taskType; taskKey.jobId = task->jobId; taskKey.taskId = task->taskId; void *hashKey = (void *) &taskKey; TaskMapEntry *taskInTheHash = (TaskMapEntry *) hash_search(taskHash, hashKey, HASH_ENTER, &handleFound); /* if same node appears twice, we error-out */ if (handleFound) { ereport(ERROR, (errmsg("multiple entries for task: \"%d:" UINT64_FORMAT ":%u\"", task->taskType, task->jobId, task->taskId))); } /* save the pointer to the original task in the hash */ taskInTheHash->task = task; return task; } /* * TaskHashLookup looks for the tasks that corresponds to the given * taskType, jobId and taskId, and returns the found task, NULL otherwise. */ static Task * TaskHashLookup(HTAB *taskHash, TaskType taskType, uint64 jobId, uint32 taskId) { Task *task = NULL; bool handleFound = false; TaskMapKey taskKey; memset(&taskKey, 0, sizeof(TaskMapKey)); taskKey.taskType = taskType; taskKey.jobId = jobId; taskKey.taskId = taskId; void *hashKey = (void *) &taskKey; TaskMapEntry *taskEntry = (TaskMapEntry *) hash_search(taskHash, hashKey, HASH_FIND, &handleFound); if (taskEntry != NULL) { task = taskEntry->task; } return task; } ================================================ FILE: src/backend/distributed/utils/tenant_schema_metadata.c ================================================ /*------------------------------------------------------------------------- * * tenant_schema_metadata.c * * This file contains functions to query and modify tenant schema metadata, * which is used to track the schemas used for schema-based sharding in * Citus. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/htup.h" #include "access/table.h" #include "catalog/pg_namespace_d.h" #include "storage/lockdefs.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/relcache.h" #include "distributed/colocation_utils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/pg_dist_schema.h" #include "distributed/tenant_schema_metadata.h" /* * IsTenantSchema returns true if there is a tenant schema with given schemaId. */ bool IsTenantSchema(Oid schemaId) { /* * Bail out early if there is a version mismatch between the Citus * shared library and the installed SQL extension. This prevents * querying pg_dist_schema when the catalog table may not exist * (e.g., during multi_extension tests that install versions older * than 12.0). * * When version checks are disabled (citus.enable_version_checks=off), * CheckCitusVersion() returns true unconditionally, even if there is * a real version mismatch. In that case, we need to verify that * pg_dist_schema actually exists before querying it, since the * multi_extension test installs old versions (e.g. 8.0-1) with * version checks disabled and creates tables in that state. * * On the normal path (version checks enabled, versions compatible), * this adds no overhead -- the !EnableVersionChecks branch is skipped * entirely. */ if (!CheckCitusVersion(DEBUG4)) { return false; } if (!EnableVersionChecks) { Oid distSchemaOid = get_relname_relid("pg_dist_schema", PG_CATALOG_NAMESPACE); if (!OidIsValid(distSchemaOid)) { return false; } } return SchemaIdGetTenantColocationId(schemaId) != INVALID_COLOCATION_ID; } /* * IsTenantSchemaColocationGroup returns true if there is a tenant schema * that is associated with given colocation id. */ bool IsTenantSchemaColocationGroup(uint32 colocationId) { return OidIsValid(ColocationIdGetTenantSchemaId(colocationId)); } /* * SchemaIdGetTenantColocationId returns the colocation id associated with * the tenant schema with given id. * * Returns INVALID_COLOCATION_ID if there is no tenant schema with given id. */ uint32 SchemaIdGetTenantColocationId(Oid schemaId) { uint32 colocationId = INVALID_COLOCATION_ID; if (!OidIsValid(schemaId)) { ereport(ERROR, (errmsg("schema id is invalid"))); } Relation pgDistTenantSchema = table_open(DistTenantSchemaRelationId(), AccessShareLock); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_dist_schema_schemaid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(schemaId)); bool indexOk = true; SysScanDesc scanDescriptor = systable_beginscan(pgDistTenantSchema, DistTenantSchemaPrimaryKeyIndexId(), indexOk, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); if (HeapTupleIsValid(heapTuple)) { bool isNull = false; colocationId = DatumGetUInt32( heap_getattr(heapTuple, Anum_pg_dist_schema_colocationid, RelationGetDescr(pgDistTenantSchema), &isNull)); Assert(!isNull); } systable_endscan(scanDescriptor); table_close(pgDistTenantSchema, AccessShareLock); return colocationId; } /* * ColocationIdGetTenantSchemaId returns the oid of the tenant schema that * is associated with given colocation id. * * Returns InvalidOid if there is no such tenant schema. */ Oid ColocationIdGetTenantSchemaId(uint32 colocationId) { if (colocationId == INVALID_COLOCATION_ID) { ereport(ERROR, (errmsg("colocation id is invalid"))); } Relation pgDistTenantSchema = table_open(DistTenantSchemaRelationId(), AccessShareLock); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_dist_schema_colocationid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(colocationId)); bool indexOk = true; SysScanDesc scanDescriptor = systable_beginscan(pgDistTenantSchema, DistTenantSchemaUniqueColocationIdIndexId(), indexOk, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); Oid schemaId = InvalidOid; if (HeapTupleIsValid(heapTuple)) { bool isNull = false; schemaId = heap_getattr(heapTuple, Anum_pg_dist_schema_schemaid, RelationGetDescr(pgDistTenantSchema), &isNull); Assert(!isNull); } systable_endscan(scanDescriptor); table_close(pgDistTenantSchema, AccessShareLock); return schemaId; } /* * InsertTenantSchemaLocally inserts an entry into pg_dist_schema * with given schemaId and colocationId. * * Throws a constraint violation error if there is already an entry with * given schemaId, or if given colocation id is already associated with * another tenant schema. */ void InsertTenantSchemaLocally(Oid schemaId, uint32 colocationId) { if (!OidIsValid(schemaId)) { ereport(ERROR, (errmsg("schema id is invalid"))); } if (colocationId == INVALID_COLOCATION_ID) { ereport(ERROR, (errmsg("colocation id is invalid"))); } Datum values[Natts_pg_dist_schema] = { 0 }; bool isNulls[Natts_pg_dist_schema] = { 0 }; values[Anum_pg_dist_schema_schemaid - 1] = ObjectIdGetDatum(schemaId); values[Anum_pg_dist_schema_colocationid - 1] = UInt32GetDatum(colocationId); Relation pgDistTenantSchema = table_open(DistTenantSchemaRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTenantSchema); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); CatalogTupleInsert(pgDistTenantSchema, heapTuple); CommandCounterIncrement(); table_close(pgDistTenantSchema, NoLock); } /* * DeleteTenantSchemaLocally deletes the entry for given schemaId from * pg_dist_schema. * * Throws an error if there is no such tenant schema. */ void DeleteTenantSchemaLocally(Oid schemaId) { if (!OidIsValid(schemaId)) { ereport(ERROR, (errmsg("schema id is invalid"))); } Relation pgDistTenantSchema = table_open(DistTenantSchemaRelationId(), RowExclusiveLock); ScanKeyData scanKey[1]; ScanKeyInit(&scanKey[0], Anum_pg_dist_schema_schemaid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(schemaId)); bool indexOk = true; SysScanDesc scanDescriptor = systable_beginscan(pgDistTenantSchema, DistTenantSchemaPrimaryKeyIndexId(), indexOk, NULL, 1, scanKey); HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection); if (!HeapTupleIsValid(heapTuple)) { ereport(ERROR, (errmsg("could not find tuple for tenant schema %u", schemaId))); } CatalogTupleDelete(pgDistTenantSchema, &heapTuple->t_self); CommandCounterIncrement(); systable_endscan(scanDescriptor); table_close(pgDistTenantSchema, NoLock); } ================================================ FILE: src/backend/distributed/utils/tuplestore.c ================================================ /*------------------------------------------------------------------------- * * tuplestore.c * Utilities regarding calls to PG functions * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "miscadmin.h" #include "distributed/tuplestore.h" /* * CheckTuplestoreReturn checks if a tuplestore can be returned in the callsite * of the UDF. */ ReturnSetInfo * CheckTuplestoreReturn(FunctionCallInfo fcinfo, TupleDesc *tupdesc) { ReturnSetInfo *returnSetInfo = (ReturnSetInfo *) fcinfo->resultinfo; /* check to see if caller supports us returning a tuplestore */ if (returnSetInfo == NULL || !IsA(returnSetInfo, ReturnSetInfo)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("set-valued function called in context that cannot " \ "accept a set"))); } if (!(returnSetInfo->allowedModes & SFRM_Materialize)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("materialize mode required, but it is not " \ "allowed in this context"))); } switch (get_call_result_type(fcinfo, NULL, tupdesc)) { case TYPEFUNC_COMPOSITE: { /* success */ break; } case TYPEFUNC_RECORD: { /* failed to determine actual type of RECORD */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); break; } default: { /* result type isn't composite */ elog(ERROR, "return type must be a row type"); break; } } return returnSetInfo; } /* * SetupTuplestore sets up a tuplestore for returning data. */ Tuplestorestate * SetupTuplestore(FunctionCallInfo fcinfo, TupleDesc *tupleDescriptor) { ReturnSetInfo *resultSet = CheckTuplestoreReturn(fcinfo, tupleDescriptor); MemoryContext perQueryContext = resultSet->econtext->ecxt_per_query_memory; MemoryContext oldContext = MemoryContextSwitchTo(perQueryContext); Tuplestorestate *tupstore = tuplestore_begin_heap(true, false, work_mem); resultSet->returnMode = SFRM_Materialize; resultSet->setResult = tupstore; resultSet->setDesc = *tupleDescriptor; MemoryContextSwitchTo(oldContext); return tupstore; } ================================================ FILE: src/backend/distributed/utils/type_utils.c ================================================ /*------------------------------------------------------------------------- * * type_utils.c * * Utility functions related to types. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "libpq-fe.h" #include "catalog/pg_type.h" #include "libpq/pqformat.h" #include "nodes/pg_list.h" #include "utils/syscache.h" #include "distributed/causal_clock.h" #define NUM_CLUSTER_CLOCK_ARGS 2 #define LDELIM '(' #define RDELIM ')' #define DELIM ',' static ClusterClock * cluster_clock_in_internal(char *clockString); PG_FUNCTION_INFO_V1(cluster_clock_in); PG_FUNCTION_INFO_V1(cluster_clock_out); PG_FUNCTION_INFO_V1(cluster_clock_recv); PG_FUNCTION_INFO_V1(cluster_clock_send); /* * cluster_clock_in_internal generic routine to parse the cluster_clock format of (logical, counter), * (%lu, %u), in string format to ClusterClock struct internal format. */ static ClusterClock * cluster_clock_in_internal(char *clockString) { char *clockFields[NUM_CLUSTER_CLOCK_ARGS]; int numClockField = 0; for (char *currentChar = clockString; (*currentChar) && (numClockField < NUM_CLUSTER_CLOCK_ARGS) && (*currentChar != RDELIM); currentChar++) { if (*currentChar == DELIM || (*currentChar == LDELIM && !numClockField)) { clockFields[numClockField++] = currentChar + 1; } } if (numClockField < NUM_CLUSTER_CLOCK_ARGS) { ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid input syntax for type %s: \"%s\"", "cluster_clock", clockString))); } char *endingChar = NULL; errno = 0; int64 logical = strtoul(clockFields[0], &endingChar, 10); if (errno || (*endingChar != DELIM) || (logical > MAX_LOGICAL) || logical < 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid input syntax for type %s: \"%s\"", "cluster_clock", clockString))); } int64 counter = strtol(clockFields[1], &endingChar, 10); if (errno || (*endingChar != RDELIM) || (counter > MAX_COUNTER) || counter < 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid input syntax for type %s: \"%s\"", "cluster_clock", clockString))); } ClusterClock *clusterClock = (ClusterClock *) palloc(sizeof(ClusterClock)); clusterClock->logical = logical; clusterClock->counter = counter; return clusterClock; } /* * cluster_clock_in converts the cstring input format to the ClusterClock type. */ Datum cluster_clock_in(PG_FUNCTION_ARGS) { char *clockString = PG_GETARG_CSTRING(0); PG_RETURN_POINTER(cluster_clock_in_internal(clockString)); } /* * cluster_clock_out converts the internal ClusterClock format to cstring output. */ Datum cluster_clock_out(PG_FUNCTION_ARGS) { ClusterClock *clusterClock = (ClusterClock *) PG_GETARG_POINTER(0); if (clusterClock == NULL) { PG_RETURN_CSTRING(""); } char *clockString = psprintf("(%lu,%u)", clusterClock->logical, clusterClock->counter); PG_RETURN_CSTRING(clockString); } /* * cluster_clock_recv converts external binary format to ClusterClock. */ Datum cluster_clock_recv(PG_FUNCTION_ARGS) { StringInfo clockBuffer = (StringInfo) PG_GETARG_POINTER(0); ClusterClock *clusterClock = (ClusterClock *) palloc(sizeof(ClusterClock)); clusterClock->logical = pq_getmsgint64(clockBuffer); clusterClock->counter = pq_getmsgint(clockBuffer, sizeof(int)); PG_RETURN_POINTER(clusterClock); } /* * cluster_clock_send converts ClusterClock to binary format. */ Datum cluster_clock_send(PG_FUNCTION_ARGS) { ClusterClock *clusterClock = (ClusterClock *) PG_GETARG_POINTER(0); StringInfoData clockBuffer; pq_begintypsend(&clockBuffer); pq_sendint64(&clockBuffer, clusterClock->logical); pq_sendint32(&clockBuffer, clusterClock->counter); PG_RETURN_BYTEA_P(pq_endtypsend(&clockBuffer)); } /***************************************************************************** * PUBLIC ROUTINES * *****************************************************************************/ PG_FUNCTION_INFO_V1(cluster_clock_lt); PG_FUNCTION_INFO_V1(cluster_clock_le); PG_FUNCTION_INFO_V1(cluster_clock_eq); PG_FUNCTION_INFO_V1(cluster_clock_ne); PG_FUNCTION_INFO_V1(cluster_clock_gt); PG_FUNCTION_INFO_V1(cluster_clock_ge); PG_FUNCTION_INFO_V1(cluster_clock_cmp); PG_FUNCTION_INFO_V1(cluster_clock_logical); /* * cluster_clock_cmp_internal generic compare routine, and must be used for all * operators, including Btree Indexes when comparing cluster_clock data type. * Return values are * 1 -- clock1 is > clock2 * 0 -- clock1 is = clock2 * -1 -- clock1 is < clock2 */ int cluster_clock_cmp_internal(ClusterClock *clusterClock1, ClusterClock *clusterClock2) { Assert(clusterClock1 && clusterClock2); int retcode = 0; /* Logical value takes precedence when comparing two clocks */ if (clusterClock1->logical != clusterClock2->logical) { retcode = (clusterClock1->logical > clusterClock2->logical) ? 1 : -1; return retcode; } /* Logical values are equal, let's compare ticks */ if (clusterClock1->counter != clusterClock2->counter) { retcode = (clusterClock1->counter > clusterClock2->counter) ? 1 : -1; return retcode; } /* Ticks are equal too, return zero */ return retcode; } /* * cluster_clock_lt returns true if clock1 is less than clock2. */ Datum cluster_clock_lt(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) < 0); } /* * cluster_clock_le returns true if clock1 is less than or equal to clock2. */ Datum cluster_clock_le(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) <= 0); } /* * cluster_clock_eq returns true if clock1 is equal to clock2. */ Datum cluster_clock_eq(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) == 0); } /* * cluster_clock_ne returns true if clock1 is not equal to clock2. */ Datum cluster_clock_ne(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) != 0); } /* * cluster_clock_gt returns true if clock1 is greater than clock2. */ Datum cluster_clock_gt(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) > 0); } /* * cluster_clock_ge returns true if clock1 is greater than or equal to clock2 */ Datum cluster_clock_ge(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_BOOL(cluster_clock_cmp_internal(clock1, clock2) >= 0); } /* * cluster_clock_cmp returns 1 if clock1 is greater than clock2, returns -1 if * clock1 is less than clock2, and zero if they are equal. */ Datum cluster_clock_cmp(PG_FUNCTION_ARGS) { ClusterClock *clock1 = (ClusterClock *) PG_GETARG_POINTER(0); ClusterClock *clock2 = (ClusterClock *) PG_GETARG_POINTER(1); PG_RETURN_INT32(cluster_clock_cmp_internal(clock1, clock2)); } /* * cluster_clock_logical return the logical part from type * clock, which is basically the epoch value in milliseconds. */ Datum cluster_clock_logical(PG_FUNCTION_ARGS) { ClusterClock *clusterClock = (ClusterClock *) PG_GETARG_POINTER(0); PG_RETURN_INT64(clusterClock->logical); } /* * ParseClusterClockPGresult parses a ClusterClock remote result and returns the value or * returns 0 if the result is NULL. */ ClusterClock * ParseClusterClockPGresult(PGresult *result, int rowIndex, int colIndex) { if (PQgetisnull(result, rowIndex, colIndex)) { return 0; } char *resultString = PQgetvalue(result, rowIndex, colIndex); return cluster_clock_in_internal(resultString); } ================================================ FILE: src/backend/distributed/worker/task_tracker_protocol.c ================================================ /*------------------------------------------------------------------------- * * task_tracker_protocol.c * * The methods in the file are deprecated. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(task_tracker_assign_task); PG_FUNCTION_INFO_V1(task_tracker_task_status); PG_FUNCTION_INFO_V1(task_tracker_cleanup_job); PG_FUNCTION_INFO_V1(task_tracker_conninfo_cache_invalidate); /* This UDF is deprecated.*/ Datum task_tracker_assign_task(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_NULL(); } /* This UDF is deprecated.*/ Datum task_tracker_task_status(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_UINT32(0); } /* This UDF is deprecated.*/ Datum task_tracker_cleanup_job(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_NULL(); } /* This UDF is deprecated.*/ Datum task_tracker_conninfo_cache_invalidate(PG_FUNCTION_ARGS) { PG_RETURN_DATUM(PointerGetDatum(NULL)); } ================================================ FILE: src/backend/distributed/worker/worker_create_or_replace.c ================================================ /*------------------------------------------------------------------------- * * worker_create_or_replace.c * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "funcapi.h" #include "access/htup_details.h" #include "catalog/dependency.h" #include "catalog/pg_collation.h" #include "catalog/pg_proc.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "parser/parse_type.h" #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/syscache.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_protocol.h" /* * OnCollisionAction describes what to do when the created object * and existing object do not match. */ typedef enum OnCollisionAction { ON_COLLISION_RENAME, ON_COLLISION_DROP } OnCollisionAction; static List * CreateStmtListByObjectAddress(const ObjectAddress *address); static bool CompareStringList(List *list1, List *list2); static OnCollisionAction GetOnCollisionAction(const ObjectAddress *address); PG_FUNCTION_INFO_V1(worker_create_or_replace_object); PG_FUNCTION_INFO_V1(worker_create_or_replace_object_array); static bool WorkerCreateOrReplaceObject(List *sqlStatements); /* * WrapCreateOrReplace takes a sql CREATE command and wraps it in a call to citus' udf to * create or replace the existing object based on its create command. */ char * WrapCreateOrReplace(const char *sql) { StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, CREATE_OR_REPLACE_COMMAND, quote_literal_cstr(sql)); return buf.data; } /* * WrapCreateOrReplaceList takes a list of sql commands and wraps it in a call to citus' * udf to create or replace the existing object based on its create commands. */ char * WrapCreateOrReplaceList(List *sqls) { StringInfoData textArrayLitteral = { 0 }; initStringInfo(&textArrayLitteral); appendStringInfoString(&textArrayLitteral, "ARRAY["); const char *sql = NULL; bool first = true; foreach_declared_ptr(sql, sqls) { if (!first) { appendStringInfoString(&textArrayLitteral, ", "); } appendStringInfoString(&textArrayLitteral, quote_literal_cstr(sql)); first = false; } appendStringInfoString(&textArrayLitteral, "]::text[]"); StringInfoData buf = { 0 }; initStringInfo(&buf); appendStringInfo(&buf, CREATE_OR_REPLACE_COMMAND, textArrayLitteral.data); return buf.data; } /* * worker_create_or_replace_object(statement text) * * function is called, by the coordinator, with a CREATE statement for an object. This * function implements the CREATE ... IF NOT EXISTS functionality for objects that do not * have this functionality or where their implementation is not sufficient. * * Besides checking if an object of said name exists it tries to compare the object to be * created with the one in the local catalog. If there is a difference the one in the local * catalog will be renamed after which the statement can be executed on this worker to * create the object. * * Renaming has two purposes * - free the identifier for creation * - non destructive if there is data store that would be destroyed if the object was * used in a table on this node, eg. types. If the type would be dropped with a cascade * it would drop any column holding user data for this type. */ Datum worker_create_or_replace_object(PG_FUNCTION_ARGS) { text *sqlStatementText = PG_GETARG_TEXT_P(0); char *sqlStatement = text_to_cstring(sqlStatementText); List *sqlStatements = list_make1(sqlStatement); PG_RETURN_BOOL(WorkerCreateOrReplaceObject(sqlStatements)); } /* * worker_create_or_replace_object(statements text[]) * * function is called, by the coordinator, with a CREATE statement for an object. This * function implements the CREATE ... IF NOT EXISTS functionality for objects that do not * have this functionality or where their implementation is not sufficient. * * Besides checking if an object of said name exists it tries to compare the object to be * created with the one in the local catalog. If there is a difference the one in the local * catalog will be renamed after which the statement can be executed on this worker to * create the object. If more statements are provided, all are compared in order with the * statements generated on the worker. This works assuming a) both citus versions are the * same, b) the objects are exactly the same. * * Renaming has two purposes * - free the identifier for creation * - non destructive if there is data store that would be destroyed if the object was * used in a table on this node, eg. types. If the type would be dropped with a cascade * it would drop any column holding user data for this type. */ Datum worker_create_or_replace_object_array(PG_FUNCTION_ARGS) { List *sqlStatements = NIL; Datum *textArray = NULL; int length = 0; deconstruct_array(PG_GETARG_ARRAYTYPE_P(0), TEXTOID, -1, false, 'i', &textArray, NULL, &length); for (int i = 0; i < length; i++) { sqlStatements = lappend(sqlStatements, TextDatumGetCString(textArray[i])); } if (list_length(sqlStatements) < 1) { ereport(ERROR, (errmsg("expected atleast 1 statement to be provided"))); } PG_RETURN_BOOL(WorkerCreateOrReplaceObject(sqlStatements)); } /* * WorkerCreateOrReplaceObject implements the logic used by both variants of * worker_create_or_replace_object to either create the object or coming to the conclusion * the object already exists in the correct state. * * Returns true if the object has been created, false if it was already in the exact state * it was asked for. */ static bool WorkerCreateOrReplaceObject(List *sqlStatements) { /* * To check which object we are changing we find the object address from the first * statement passed into the UDF. Later we will check if all object addresses are the * same. * * Although many of the objects will only have one statement in this call, more * complex objects might come with a list of statements. We assume they all are on the * same subject. */ Node *parseTree = ParseTreeNode(linitial(sqlStatements)); List *addresses = GetObjectAddressListFromParseTree(parseTree, true, false); Assert(list_length(addresses) == 1); /* We have already asserted that we have exactly 1 address in the addresses. */ ObjectAddress *address = linitial(addresses); if (ObjectExists(address)) { /* * Object with name from statement is already found locally, check if states are * identical. If objects differ we will rename the old object (non- destructively) * or drop it (if safe) as to make room to create the new object according to the * spec sent. */ /* * Based on the local catalog we generate the list of commands we would send to * recreate our version of the object. This we can compare to what the coordinator * sent us. If they match we don't do anything. */ List *localSqlStatements = CreateStmtListByObjectAddress(address); if (CompareStringList(sqlStatements, localSqlStatements)) { /* * statements sent by the coordinator are the same as we would create for our * object, therefore we can omit the statements locally and not create the * object as it already exists in the correct shape. * * We let the coordinator know we didn't create the object. */ return false; } Node *utilityStmt = NULL; if (GetOnCollisionAction(address) == ON_COLLISION_DROP) { /* drop the existing object */ utilityStmt = (Node *) CreateDropStmt(address); } else { /* rename the existing object */ char *newName = GenerateBackupNameForCollision(address); utilityStmt = (Node *) CreateRenameStatement(address, newName); } const char *commandString = DeparseTreeNode(utilityStmt); ProcessUtilityParseTree(utilityStmt, commandString, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } /* apply all statement locally */ char *sqlStatement = NULL; foreach_declared_ptr(sqlStatement, sqlStatements) { parseTree = ParseTreeNode(sqlStatement); ProcessUtilityParseTree(parseTree, sqlStatement, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); /* TODO verify all statements are about exactly 1 subject, mostly a sanity check * to prevent unintentional use of this UDF, needs to come after the local * execution to be able to actually resolve the ObjectAddress of the newly created * object */ } /* type has been created */ return true; } static bool CompareStringList(List *list1, List *list2) { if (list_length(list1) != list_length(list2)) { return false; } const char *str1 = NULL; const char *str2 = NULL; forboth_ptr(str1, list1, str2, list2) { if (strcmp(str1, str2) != 0) { return false; } } return true; } /* * CreateStmtByObjectAddress returns a parsetree that will recreate the object addressed * by the ObjectAddress provided. * * Note: this tree does not contain position information that is normally in a parsetree, * therefore you cannot equal this tree against parsed statement. Instead it can be * deparsed to do a string comparison. */ static List * CreateStmtListByObjectAddress(const ObjectAddress *address) { switch (getObjectClass(address)) { case OCLASS_COLLATION: { return list_make1(CreateCollationDDL(address->objectId)); } case OCLASS_PROC: { return list_make1(GetFunctionDDLCommand(address->objectId, false)); } case OCLASS_PUBLICATION: { return list_make1(CreatePublicationDDLCommand(address->objectId)); } case OCLASS_TSCONFIG: { List *stmts = GetCreateTextSearchConfigStatements(address); return DeparseTreeNodes(stmts); } case OCLASS_TSDICT: { List *stmts = GetCreateTextSearchDictionaryStatements(address); return DeparseTreeNodes(stmts); } case OCLASS_TYPE: { return list_make1(DeparseTreeNode(CreateTypeStmtByObjectAddress(address))); } default: { ereport(ERROR, (errmsg( "unsupported object to construct a create statement"))); } } } /* * GetOnCollisionAction decides what to do when the object already exists. */ static OnCollisionAction GetOnCollisionAction(const ObjectAddress *address) { switch (getObjectClass(address)) { case OCLASS_PUBLICATION: { /* * We prefer to drop publications because they can be * harmful (cause update/delete failures) and are relatively * safe to drop. */ return ON_COLLISION_DROP; } case OCLASS_COLLATION: case OCLASS_PROC: case OCLASS_TSCONFIG: case OCLASS_TSDICT: case OCLASS_TYPE: default: { return ON_COLLISION_RENAME; } } } /* * GenerateBackupNameForCollision calculate a backup name for a given object by its * address. This name should be used when renaming an existing object before creating the * new object locally on the worker. */ char * GenerateBackupNameForCollision(const ObjectAddress *address) { switch (getObjectClass(address)) { case OCLASS_COLLATION: { return GenerateBackupNameForCollationCollision(address); } case OCLASS_PROC: { return GenerateBackupNameForProcCollision(address); } case OCLASS_TSCONFIG: { return GenerateBackupNameForTextSearchConfiguration(address); } case OCLASS_TYPE: { return GenerateBackupNameForTypeCollision(address); } case OCLASS_CLASS: { char relKind = get_rel_relkind(address->objectId); if (relKind == RELKIND_SEQUENCE) { return GenerateBackupNameForSequenceCollision(address); } } default: { break; } } ereport(ERROR, (errmsg("unsupported object to construct a rename statement"), errdetail("unable to generate a backup name for the old type"))); } /* * CreateDropPublicationStmt creates a DROP PUBLICATION statement for the * publication at the given address. */ static DropStmt * CreateDropPublicationStmt(const ObjectAddress *address) { Assert(address->classId == PublicationRelationId); DropStmt *dropStmt = makeNode(DropStmt); dropStmt->removeType = OBJECT_PUBLICATION; dropStmt->behavior = DROP_RESTRICT; HeapTuple publicationTuple = SearchSysCache1(PUBLICATIONOID, ObjectIdGetDatum(address->objectId)); if (!HeapTupleIsValid(publicationTuple)) { ereport(ERROR, (errmsg("cannot find publication with oid: %d", address->objectId))); } Form_pg_publication publicationForm = (Form_pg_publication) GETSTRUCT(publicationTuple); char *publicationName = NameStr(publicationForm->pubname); dropStmt->objects = list_make1(makeString(publicationName)); ReleaseSysCache(publicationTuple); return dropStmt; } /* * CreateDropStmt returns a DROP statement for the given object. */ DropStmt * CreateDropStmt(const ObjectAddress *address) { switch (getObjectClass(address)) { case OCLASS_PUBLICATION: { return CreateDropPublicationStmt(address); } default: { break; } } ereport(ERROR, (errmsg("unsupported object to construct a drop statement"), errdetail("unable to generate a parsetree for the drop"))); } /* * CreateRenameTypeStmt creates a rename statement for a type based on its ObjectAddress. * The rename statement will rename the existing object on its address to the value * provided in newName. */ static RenameStmt * CreateRenameCollationStmt(const ObjectAddress *address, char *newName) { RenameStmt *stmt = makeNode(RenameStmt); Oid collid = address->objectId; HeapTuple colltup = SearchSysCache1(COLLOID, collid); if (!HeapTupleIsValid(colltup)) { ereport(ERROR, (errmsg("citus cache lookup error"))); } Form_pg_collation collationForm = (Form_pg_collation) GETSTRUCT(colltup); char *schemaName = get_namespace_name(collationForm->collnamespace); char *collationName = NameStr(collationForm->collname); List *name = list_make2(makeString(schemaName), makeString(collationName)); ReleaseSysCache(colltup); stmt->renameType = OBJECT_COLLATION; stmt->object = (Node *) name; stmt->newname = newName; return stmt; } /* * CreateRenameTypeStmt creates a rename statement for a type based on its ObjectAddress. * The rename statement will rename the existing object on its address to the value * provided in newName. */ static RenameStmt * CreateRenameTypeStmt(const ObjectAddress *address, char *newName) { RenameStmt *stmt = makeNode(RenameStmt); stmt->renameType = OBJECT_TYPE; stmt->object = (Node *) stringToQualifiedNameList(format_type_be_qualified(address-> objectId), NULL); stmt->newname = newName; return stmt; } /* * CreateRenameTextSearchStmt creates a rename statement for a text search configuration * based on its ObjectAddress. The rename statement will rename the existing object on its * address to the value provided in newName. */ static RenameStmt * CreateRenameTextSearchStmt(const ObjectAddress *address, char *newName) { Assert(address->classId == TSConfigRelationId); RenameStmt *stmt = makeNode(RenameStmt); stmt->renameType = OBJECT_TSCONFIGURATION; stmt->object = (Node *) get_ts_config_namelist(address->objectId); stmt->newname = newName; return stmt; } /* * CreateRenameTypeStmt creates a rename statement for a type based on its ObjectAddress. * The rename statement will rename the existing object on its address to the value * provided in newName. */ static RenameStmt * CreateRenameProcStmt(const ObjectAddress *address, char *newName) { RenameStmt *stmt = makeNode(RenameStmt); stmt->renameType = OBJECT_ROUTINE; stmt->object = (Node *) ObjectWithArgsFromOid(address->objectId); stmt->newname = newName; return stmt; } /* * CreateRenameSequenceStmt creates a rename statement for a sequence based on its * ObjectAddress. The rename statement will rename the existing object on its address * to the value provided in newName. */ static RenameStmt * CreateRenameSequenceStmt(const ObjectAddress *address, char *newName) { RenameStmt *stmt = makeNode(RenameStmt); Oid seqOid = address->objectId; HeapTuple seqClassTuple = SearchSysCache1(RELOID, seqOid); if (!HeapTupleIsValid(seqClassTuple)) { ereport(ERROR, (errmsg("citus cache lookup error"))); } Form_pg_class seqClassForm = (Form_pg_class) GETSTRUCT(seqClassTuple); char *schemaName = get_namespace_name(seqClassForm->relnamespace); char *seqName = NameStr(seqClassForm->relname); List *name = list_make2(makeString(schemaName), makeString(seqName)); ReleaseSysCache(seqClassTuple); stmt->renameType = OBJECT_SEQUENCE; stmt->object = (Node *) name; stmt->relation = makeRangeVar(schemaName, seqName, -1); stmt->newname = newName; return stmt; } /* * CreateRenameStatement creates a rename statement for an existing object to rename the * object to newName. */ RenameStmt * CreateRenameStatement(const ObjectAddress *address, char *newName) { switch (getObjectClass(address)) { case OCLASS_COLLATION: { return CreateRenameCollationStmt(address, newName); } case OCLASS_PROC: { return CreateRenameProcStmt(address, newName); } case OCLASS_TSCONFIG: { return CreateRenameTextSearchStmt(address, newName); } case OCLASS_TYPE: { return CreateRenameTypeStmt(address, newName); } case OCLASS_CLASS: { char relKind = get_rel_relkind(address->objectId); if (relKind == RELKIND_SEQUENCE) { return CreateRenameSequenceStmt(address, newName); } } default: { break; } } ereport(ERROR, (errmsg("unsupported object to construct a rename statement"), errdetail("unable to generate a parsetree for the rename"))); } ================================================ FILE: src/backend/distributed/worker/worker_data_fetch_protocol.c ================================================ /*------------------------------------------------------------------------- * * worker_data_fetch_protocol.c * * Routines for fetching remote resources from other nodes to this worker node, * and materializing these resources on this node if necessary. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include #include #include "postgres.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/pg_type.h" #include "commands/copy.h" #include "commands/dbcommands.h" #include "commands/extension.h" #include "commands/sequence.h" #include "executor/spi.h" #include "nodes/makefuncs.h" #include "parser/parse_relation.h" #include "storage/lmgr.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/varlena.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" #include "distributed/coordinator_protocol.h" #include "distributed/deparser.h" #include "distributed/intermediate_results.h" #include "distributed/listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/multi_server_executor.h" #include "distributed/relay_utility.h" #include "distributed/remote_commands.h" #include "distributed/resource_lock.h" #include "distributed/version_compat.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_protocol.h" /* Local functions forward declarations */ static bool check_log_statement(List *stmt_list); static void AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName, Oid sequenceTypeId); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(worker_apply_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_inter_shard_ddl_command); PG_FUNCTION_INFO_V1(worker_apply_sequence_command); PG_FUNCTION_INFO_V1(worker_adjust_identity_column_seq_ranges); PG_FUNCTION_INFO_V1(worker_append_table_to_shard); PG_FUNCTION_INFO_V1(worker_nextval); /* * worker_apply_shard_ddl_command extends table, index, or constraint names in * the given DDL command. The function then applies this extended DDL command * against the database. */ Datum worker_apply_shard_ddl_command(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 shardId = PG_GETARG_INT64(0); text *schemaNameText = PG_GETARG_TEXT_P(1); text *ddlCommandText = PG_GETARG_TEXT_P(2); char *schemaName = text_to_cstring(schemaNameText); const char *ddlCommand = text_to_cstring(ddlCommandText); Node *ddlCommandNode = ParseTreeNode(ddlCommand); /* extend names in ddl command and apply extended command */ RelayEventExtendNames(ddlCommandNode, schemaName, shardId); ProcessUtilityParseTree(ddlCommandNode, ddlCommand, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); PG_RETURN_VOID(); } /* * worker_apply_inter_shard_ddl_command extends table, index, or constraint names in * the given DDL command. The function then applies this extended DDL command * against the database. */ Datum worker_apply_inter_shard_ddl_command(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); uint64 leftShardId = PG_GETARG_INT64(0); text *leftShardSchemaNameText = PG_GETARG_TEXT_P(1); uint64 rightShardId = PG_GETARG_INT64(2); text *rightShardSchemaNameText = PG_GETARG_TEXT_P(3); text *ddlCommandText = PG_GETARG_TEXT_P(4); char *leftShardSchemaName = text_to_cstring(leftShardSchemaNameText); char *rightShardSchemaName = text_to_cstring(rightShardSchemaNameText); const char *ddlCommand = text_to_cstring(ddlCommandText); Node *ddlCommandNode = ParseTreeNode(ddlCommand); /* extend names in ddl command and apply extended command */ RelayEventExtendNamesForInterShardCommands(ddlCommandNode, leftShardId, leftShardSchemaName, rightShardId, rightShardSchemaName); ProcessUtilityParseTree(ddlCommandNode, ddlCommand, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); PG_RETURN_VOID(); } /* * worker_adjust_identity_column_seq_ranges takes a table oid, runs an ALTER SEQUENCE statement * for each identity column to adjust the minvalue and maxvalue of the sequence owned by * identity column such that the sequence creates globally unique values. * We use table oid instead of sequence name to avoid any potential conflicts between sequences of different tables. This way, we can safely iterate through identity columns on a specific table without any issues. While this may introduce a small amount of business logic to workers, it's a much safer approach overall. */ Datum worker_adjust_identity_column_seq_ranges(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid tableRelationId = PG_GETARG_OID(0); EnsureTableOwner(tableRelationId); Relation tableRelation = relation_open(tableRelationId, AccessShareLock); TupleDesc tableTupleDesc = RelationGetDescr(tableRelation); bool missingSequenceOk = false; for (int attributeIndex = 0; attributeIndex < tableTupleDesc->natts; attributeIndex++) { Form_pg_attribute attributeForm = TupleDescAttr(tableTupleDesc, attributeIndex); /* skip dropped columns */ if (attributeForm->attisdropped) { continue; } if (attributeForm->attidentity) { Oid sequenceOid = getIdentitySequence(identitySequenceRelation_compat( tableRelation), attributeForm->attnum, missingSequenceOk); Oid sequenceSchemaOid = get_rel_namespace(sequenceOid); char *sequenceSchemaName = get_namespace_name(sequenceSchemaOid); char *sequenceName = get_rel_name(sequenceOid); Oid sequenceTypeId = pg_get_sequencedef(sequenceOid)->seqtypid; AlterSequenceMinMax(sequenceOid, sequenceSchemaName, sequenceName, sequenceTypeId); } } relation_close(tableRelation, NoLock); PG_RETURN_VOID(); } /* * worker_apply_sequence_command takes a CREATE SEQUENCE command string, runs the * CREATE SEQUENCE command then creates and runs an ALTER SEQUENCE statement * which adjusts the minvalue and maxvalue of the sequence such that the sequence * creates globally unique values. */ Datum worker_apply_sequence_command(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *commandText = PG_GETARG_TEXT_P(0); Oid sequenceTypeId = PG_GETARG_OID(1); const char *commandString = text_to_cstring(commandText); Node *commandNode = ParseTreeNode(commandString); NodeTag nodeType = nodeTag(commandNode); if (nodeType != T_CreateSeqStmt) { ereport(ERROR, (errmsg("must call worker_apply_sequence_command with a CREATE" " SEQUENCE command string"))); } /* * If sequence with the same name exist for different type, it must have been * stayed on that node after a rollbacked create_distributed_table operation. * We must change its name first to create the sequence with the correct type. */ CreateSeqStmt *createSequenceStatement = (CreateSeqStmt *) commandNode; RenameExistingSequenceWithDifferentTypeIfExists(createSequenceStatement->sequence, sequenceTypeId); /* run the CREATE SEQUENCE command */ ProcessUtilityParseTree(commandNode, commandString, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); CommandCounterIncrement(); Oid sequenceRelationId = RangeVarGetRelid(createSequenceStatement->sequence, AccessShareLock, false); char *sequenceName = createSequenceStatement->sequence->relname; char *sequenceSchema = createSequenceStatement->sequence->schemaname; Assert(sequenceRelationId != InvalidOid); AlterSequenceMinMax(sequenceRelationId, sequenceSchema, sequenceName, sequenceTypeId); PG_RETURN_VOID(); } /* * ExtractShardIdFromTableName tries to extract shard id from the given table name, * and returns the shard id if table name is formatted as shard name. * Else, the function returns INVALID_SHARD_ID. */ uint64 ExtractShardIdFromTableName(const char *tableName, bool missingOk) { char *shardIdStringEnd = NULL; /* find the last underscore and increment for shardId string */ char *shardIdString = strrchr(tableName, SHARD_NAME_SEPARATOR); if (shardIdString == NULL && !missingOk) { ereport(ERROR, (errmsg("could not extract shardId from table name \"%s\"", tableName))); } else if (shardIdString == NULL && missingOk) { return INVALID_SHARD_ID; } shardIdString++; errno = 0; uint64 shardId = strtou64(shardIdString, &shardIdStringEnd, 0); if (errno != 0 || (*shardIdStringEnd != '\0')) { if (!missingOk) { ereport(ERROR, (errmsg("could not extract shardId from table name \"%s\"", tableName))); } else { return INVALID_SHARD_ID; } } return shardId; } /* * Parses the given DDL command, and returns the tree node for parsed command. */ Node * ParseTreeNode(const char *ddlCommand) { Node *parseTreeNode = ParseTreeRawStmt(ddlCommand); parseTreeNode = ((RawStmt *) parseTreeNode)->stmt; return parseTreeNode; } /* * Parses the given DDL command, and returns the tree node for parsed command. */ Node * ParseTreeRawStmt(const char *ddlCommand) { List *parseTreeList = pg_parse_query(ddlCommand); /* log immediately if dictated by log statement */ if (check_log_statement(parseTreeList)) { ereport(LOG, (errmsg("statement: %s", ddlCommand), errhidestmt(true))); } uint32 parseTreeCount = list_length(parseTreeList); if (parseTreeCount != 1) { ereport(ERROR, (errmsg("cannot execute multiple utility events"))); } /* * xact.c rejects certain commands that are unsafe to run inside transaction * blocks. Since we only apply commands that relate to creating tables and * those commands are safe, we can safely set the ProcessUtilityContext to * PROCESS_UTILITY_TOPLEVEL. */ Node *parseTreeNode = (Node *) linitial(parseTreeList); return parseTreeNode; } /* * worker_append_table_to_shard is deprecated. */ Datum worker_append_table_to_shard(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("worker_append_table_to_shard has been deprecated"))); } /* * worker_nextval calculates nextval() in worker nodes * for int and smallint column default types * TODO: not error out but get the proper nextval() */ Datum worker_nextval(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg( "nextval(sequence) calls in worker nodes are not supported" " for column defaults of type int or smallint"))); PG_RETURN_INT32(0); } /* * check_log_statement is a copy of postgres' check_log_statement function and * returns whether a statement ought to be logged or not. */ static bool check_log_statement(List *statementList) { if (log_statement == LOGSTMT_NONE) { return false; } if (log_statement == LOGSTMT_ALL) { return true; } /* else we have to inspect the statement(s) to see whether to log */ Node *statement = NULL; foreach_declared_ptr(statement, statementList) { if (GetCommandLogLevel(statement) <= log_statement) { return true; } } return false; } /* * AlterSequenceMinMax arranges the min and max value of the given sequence. The function * creates ALTER SEQUENCE statemenet which sets the start, minvalue and maxvalue of * the given sequence. * * The function provides the uniqueness by shifting the start of the sequence by * GetLocalGroupId() << 48 + 1 and sets a maxvalue which stops it from passing out any * values greater than: (GetLocalGroupID() + 1) << 48. * * For serial we only have 32 bits and therefore shift by 28, and for smallserial * we only have 16 bits and therefore shift by 12. * * This is to ensure every group of workers passes out values from a unique range, * and therefore that all values generated for the sequence are globally unique. */ static void AlterSequenceMinMax(Oid sequenceId, char *schemaName, char *sequenceName, Oid sequenceTypeId) { Form_pg_sequence sequenceData = pg_get_sequencedef(sequenceId); int64 sequenceMaxValue = sequenceData->seqmax; int64 sequenceMinValue = sequenceData->seqmin; int valueBitLength = 48; /* * For int and smallint, we don't currently support insertion from workers * Check issue #5126 and PR #5254 for details. * https://github.com/citusdata/citus/issues/5126 * So, no need to alter sequence min/max for now * We call setval(sequence, maxvalue) such that manually using * nextval(sequence) in the workers will error out as well. */ if (sequenceTypeId != INT8OID) { DirectFunctionCall2(setval_oid, ObjectIdGetDatum(sequenceId), Int64GetDatum(sequenceMaxValue)); return; } /* calculate min/max values that the sequence can generate in this worker */ int64 startValue = (((int64) GetLocalGroupId()) << valueBitLength) + 1; int64 maxValue = startValue + ((int64) 1 << valueBitLength); /* * We alter the sequence if the previously set min and max values are not equal to * their correct values. */ if (sequenceMinValue != startValue || sequenceMaxValue != maxValue) { StringInfo startNumericString = makeStringInfo(); StringInfo maxNumericString = makeStringInfo(); AlterSeqStmt *alterSequenceStatement = makeNode(AlterSeqStmt); const char *dummyString = "-"; alterSequenceStatement->sequence = makeRangeVar(schemaName, sequenceName, -1); /* * DefElem->arg can only hold literal ints up to int4, in order to represent * larger numbers we need to construct a float represented as a string. */ appendStringInfo(startNumericString, INT64_FORMAT, startValue); Node *startFloatArg = (Node *) makeFloat(startNumericString->data); appendStringInfo(maxNumericString, INT64_FORMAT, maxValue); Node *maxFloatArg = (Node *) makeFloat(maxNumericString->data); SetDefElemArg(alterSequenceStatement, "start", startFloatArg); SetDefElemArg(alterSequenceStatement, "minvalue", startFloatArg); SetDefElemArg(alterSequenceStatement, "maxvalue", maxFloatArg); SetDefElemArg(alterSequenceStatement, "restart", startFloatArg); /* since the command is an AlterSeqStmt, a dummy command string works fine */ ProcessUtilityParseTree((Node *) alterSequenceStatement, dummyString, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } } /* * SetDefElemArg scans through all the DefElem's of an AlterSeqStmt and * and sets the arg of the one with a defname of name to arg. * * If a DefElem with the given defname does not exist it is created and * added to the AlterSeqStmt. */ void SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg) { DefElem *defElem = NULL; foreach_declared_ptr(defElem, statement->options) { if (strcmp(defElem->defname, name) == 0) { pfree(defElem->arg); defElem->arg = arg; return; } } defElem = makeDefElem((char *) name, arg, -1); statement->options = lappend(statement->options, defElem); } ================================================ FILE: src/backend/distributed/worker/worker_drop_protocol.c ================================================ /*------------------------------------------------------------------------- * * worker_drop_protocol.c * * Routines for dropping distributed tables and their metadata on worker nodes. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "access/genam.h" #include "access/heapam.h" #include "access/xact.h" #include "catalog/dependency.h" #include "catalog/pg_depend.h" #include "catalog/pg_foreign_server.h" #include "foreign/foreign.h" #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/distribution_column.h" #include "distributed/listutils.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/worker_protocol.h" PG_FUNCTION_INFO_V1(worker_drop_distributed_table); PG_FUNCTION_INFO_V1(worker_drop_shell_table); PG_FUNCTION_INFO_V1(worker_drop_sequence_dependency); static void WorkerDropDistributedTable(Oid relationId); /* * worker_drop_distributed_table drops the distributed table with the given oid, * then, removes the associated rows from pg_dist_partition, pg_dist_shard and * pg_dist_placement. * * Note that drop fails if any dependent objects are present for any of the * distributed tables. Also, shard placements of the distributed tables are * not dropped as in the case of "DROP TABLE distributed_table;" command. * * The function errors out if the input relation Oid is not a regular or foreign table. * The function is meant to be called only by the coordinator, therefore requires * superuser privileges. */ Datum worker_drop_distributed_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *relationName = PG_GETARG_TEXT_P(0); Oid relationId = ResolveRelationId(relationName, true); if (!OidIsValid(relationId)) { ereport(NOTICE, (errmsg("relation %s does not exist, skipping", text_to_cstring(relationName)))); PG_RETURN_VOID(); } EnsureTableOwner(relationId); if (PartitionedTable(relationId)) { /* * When "DROP SCHEMA .. CASCADE" happens, we rely on Postgres' drop trigger * to send the individual DROP TABLE commands for tables. * * In case of partitioned tables, we have no control on the order of DROP * commands that is sent to the extension. We can try to sort while processing * on the coordinator, but we prefer to handle it in a more flexible manner. * * That's why, whenever we see a partitioned table, we drop all the corresponding * partitions first. Otherwise, WorkerDropDistributedTable() would already drop * the shell tables of the partitions (e.g., due to performDeletion(..CASCADE), * and further WorkerDropDistributedTable() on the partitions would become no-op. * * If, say one partition has already been dropped earlier, that should also be fine * because we read the existing partitions. */ List *partitionList = PartitionList(relationId); Oid partitionOid = InvalidOid; foreach_declared_oid(partitionOid, partitionList) { WorkerDropDistributedTable(partitionOid); } } WorkerDropDistributedTable(relationId); PG_RETURN_VOID(); } /* * WorkerDropDistributedTable is a helper function for worker_drop_distributed_table, see * tha function for the details. */ static void WorkerDropDistributedTable(Oid relationId) { /* first check the relation type */ Relation distributedRelation = relation_open(relationId, AccessShareLock); EnsureRelationKindSupported(relationId); /* close the relation since we do not need anymore */ relation_close(distributedRelation, AccessShareLock); /* prepare distributedTableObject for dropping the table */ ObjectAddress *distributedTableObject = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*distributedTableObject, RelationRelationId, relationId); /* Drop dependent sequences from pg_dist_object */ List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; foreach_declared_oid(ownedSequenceOid, ownedSequences) { ObjectAddress ownedSequenceAddress = { 0 }; ObjectAddressSet(ownedSequenceAddress, RelationRelationId, ownedSequenceOid); UnmarkObjectDistributed(&ownedSequenceAddress); } UnmarkObjectDistributed(distributedTableObject); /* * Remove metadata before object's itself to make functions no-op within * drop event trigger for undistributed objects on worker nodes except * removing pg_dist_object entries. */ List *shardList = LoadShardList(relationId); uint64 *shardIdPointer = NULL; foreach_declared_ptr(shardIdPointer, shardList) { uint64 shardId = *shardIdPointer; List *shardPlacementList = ShardPlacementList(shardId); ShardPlacement *placement = NULL; foreach_declared_ptr(placement, shardPlacementList) { /* delete the row from pg_dist_placement */ DeleteShardPlacementRow(placement->placementId); } /* delete the row from pg_dist_shard */ DeleteShardRow(shardId); } /* delete the row from pg_dist_partition */ DeletePartitionRow(relationId); /* * If the table is owned by an extension, we cannot drop it, nor should we * until the user runs DROP EXTENSION. Therefore, we skip dropping the * table. */ if (!IsAnyObjectAddressOwnedByExtension(list_make1(distributedTableObject), NULL)) { StringInfo dropCommand = makeStringInfo(); appendStringInfo(dropCommand, "DROP%sTABLE %s CASCADE", IsForeignTable(relationId) ? " FOREIGN " : " ", generate_qualified_relation_name(relationId)); Node *dropCommandNode = ParseTreeNode(dropCommand->data); /* * We use ProcessUtilityParseTree (instead of performDeletion) to make sure that * we also drop objects that depend on the table and call the drop event trigger * which removes them from pg_dist_object. */ ProcessUtilityParseTree(dropCommandNode, dropCommand->data, PROCESS_UTILITY_QUERY, NULL, None_Receiver, NULL); } } /* * worker_drop_shell_table drops the shell table of with the given distributed * table without deleting related entries on pg_dist_placement, pg_dist_shard * and pg_dist_placement. We've separated that logic since we handle object * dependencies and table metadata separately while activating nodes. */ Datum worker_drop_shell_table(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); text *relationName = PG_GETARG_TEXT_P(0); Oid relationId = ResolveRelationId(relationName, true); if (!OidIsValid(relationId)) { ereport(NOTICE, (errmsg("relation %s does not exist, skipping", text_to_cstring(relationName)))); PG_RETURN_VOID(); } EnsureTableOwner(relationId); if (GetLocalGroupId() == COORDINATOR_GROUP_ID) { ereport(ERROR, (errmsg("worker_drop_shell_table is only allowed to run" " on worker nodes"))); } /* first check the relation type */ Relation distributedRelation = relation_open(relationId, AccessShareLock); EnsureRelationKindSupported(relationId); /* close the relation since we do not need anymore */ relation_close(distributedRelation, AccessShareLock); /* prepare distributedTableObject for dropping the table */ ObjectAddress *distributedTableObject = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*distributedTableObject, RelationRelationId, relationId); if (IsAnyObjectAddressOwnedByExtension(list_make1(distributedTableObject), NULL)) { PG_RETURN_VOID(); } /* Drop dependent sequences from pg_dist_object */ List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; foreach_declared_oid(ownedSequenceOid, ownedSequences) { ObjectAddress ownedSequenceAddress = { 0 }; ObjectAddressSet(ownedSequenceAddress, RelationRelationId, ownedSequenceOid); UnmarkObjectDistributed(&ownedSequenceAddress); } /* * If the table is owned by an extension, we cannot drop it, nor should we * until the user runs DROP EXTENSION. Therefore, we skip dropping the * table and only delete the metadata. * * We drop the table with cascade since other tables may be referring to it. */ performDeletion(distributedTableObject, DROP_CASCADE, PERFORM_DELETION_INTERNAL); PG_RETURN_VOID(); } /* * worker_drop_sequence_dependency is a UDF that removes the dependency * of all the sequences for the given table. * * The main purpose of this UDF is to prevent dropping the sequences while * re-creating the same table such as changing the shard count, converting * a citus local table to a distributed table or re-syncing the metadata. */ Datum worker_drop_sequence_dependency(PG_FUNCTION_ARGS) { text *relationName = PG_GETARG_TEXT_P(0); Oid relationId = ResolveRelationId(relationName, true); if (!OidIsValid(relationId)) { ereport(NOTICE, (errmsg("relation %s does not exist, skipping", text_to_cstring(relationName)))); PG_RETURN_VOID(); } EnsureTableOwner(relationId); /* break the dependent sequences from the table */ List *ownedSequences = getOwnedSequences(relationId); Oid ownedSequenceOid = InvalidOid; foreach_declared_oid(ownedSequenceOid, ownedSequences) { /* the caller doesn't want to drop the sequence, so break the dependency */ deleteDependencyRecordsForSpecific(RelationRelationId, ownedSequenceOid, DEPENDENCY_AUTO, RelationRelationId, relationId); } if (list_length(ownedSequences) > 0) { /* if we delete at least one dependency, let next commands know */ CommandCounterIncrement(); } PG_RETURN_VOID(); } ================================================ FILE: src/backend/distributed/worker/worker_partition_protocol.c ================================================ /*------------------------------------------------------------------------- * * worker_partition_protocol.c * * Deprecated functions related to generating re-partitioning files. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(worker_cleanup_job_schema_cache); PG_FUNCTION_INFO_V1(worker_create_schema); PG_FUNCTION_INFO_V1(worker_fetch_foreign_file); PG_FUNCTION_INFO_V1(worker_fetch_partition_file); PG_FUNCTION_INFO_V1(worker_hash_partition_table); PG_FUNCTION_INFO_V1(worker_merge_files_into_table); PG_FUNCTION_INFO_V1(worker_merge_files_and_run_query); PG_FUNCTION_INFO_V1(worker_range_partition_table); PG_FUNCTION_INFO_V1(worker_repartition_cleanup); /* * worker_range_partition_table is a deprecated function that we keep around for * testing downgrade scripts. */ Datum worker_range_partition_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_hash_partition_table is a deprecated function that we keep around for * testing downgrade scripts. */ Datum worker_hash_partition_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_create_schema is deprecated and only kept for testing downgrade scripts. */ Datum worker_create_schema(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_repartition_cleanup removes the job directory and schema with the given job id . */ Datum worker_repartition_cleanup(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_merge_files_into_table is deprecated and only kept for testing downgrade * scripts. */ Datum worker_merge_files_into_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_merge_files_and_run_query is deprecated and only kept for testing downgrade * scripts. */ Datum worker_merge_files_and_run_query(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_cleanup_job_schema_cache is deprecated and only kept for testing downgrade * scripts. */ Datum worker_cleanup_job_schema_cache(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } /* * worker_fetch_foreign_file UDF is a stub UDF to install Citus flawlessly. * Otherwise we need to delete them from our sql files, which is confusing */ Datum worker_fetch_foreign_file(PG_FUNCTION_ARGS) { ereport(DEBUG2, (errmsg("this function is deprecated and no longer is used"))); PG_RETURN_VOID(); } /* * worker_fetch_partition_file is a deprecated function that we keep around for * testing downgrade scripts. */ Datum worker_fetch_partition_file(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("this function is deprecated and only kept for testing " "downgrade scripts"))); } ================================================ FILE: src/backend/distributed/worker/worker_shard_visibility.c ================================================ /* * worker_shard_visibility.c * * Implements the functions for hiding shards on the Citus MX * worker (data) nodes. * * Copyright (c) Citus Data, Inc. */ #include "postgres.h" #include "miscadmin.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/varlena.h" #include "distributed/backend_data.h" #include "distributed/coordinator_protocol.h" #include "distributed/listutils.h" #include "distributed/local_executor.h" #include "distributed/metadata_cache.h" #include "distributed/query_colocation_checker.h" #include "distributed/worker_protocol.h" #include "distributed/worker_shard_visibility.h" /* HideShardsMode is used to determine whether to hide shards */ typedef enum HideShardsMode { CHECK_APPLICATION_NAME, HIDE_SHARDS_FROM_APPLICATION, DO_NOT_HIDE_SHARDS } HideShardsMode; /* Config variable managed via guc.c */ bool OverrideTableVisibility = true; bool EnableManualChangesToShards = false; /* show shards when the application_name starts with one of: */ char *ShowShardsForAppNamePrefixes = ""; /* cache of whether or not to hide shards */ static HideShardsMode HideShards = CHECK_APPLICATION_NAME; static bool ShouldHideShards(void); static bool ShouldHideShardsInternal(void); static bool IsPgBgWorker(void); static bool FilterShardsFromPgclass(Node *node, void *context); static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno); static bool HasRangeTableRef(Node *node, int *varno); PG_FUNCTION_INFO_V1(citus_table_is_visible); PG_FUNCTION_INFO_V1(relation_is_a_known_shard); /* * relation_is_a_known_shard a wrapper around RelationIsAKnownShard(), so * see the details there. The function also treats the indexes on shards * as if they were shards. */ Datum relation_is_a_known_shard(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); PG_RETURN_BOOL(RelationIsAKnownShard(relationId)); } /* * citus_table_is_visible aims to behave exactly the same with * pg_table_is_visible with only one exception. The former one * returns false for the relations that are known to be shards. */ Datum citus_table_is_visible(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); char relKind = '\0'; /* * We don't want to deal with not valid/existing relations * as pg_table_is_visible does. */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relationId))) { PG_RETURN_NULL(); } if (!RelationIsVisible(relationId)) { /* relation is not on the search path */ PG_RETURN_BOOL(false); } if (RelationIsAKnownShard(relationId)) { /* * If the input relation is an index we simply replace the * relationId with the corresponding relation to hide indexes * as well. See RelationIsAKnownShard() for the details and give * more meaningful debug message here. */ relKind = get_rel_relkind(relationId); if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { ereport(DEBUG2, (errmsg("skipping index \"%s\" since it belongs to a shard", get_rel_name(relationId)))); } else { ereport(DEBUG2, (errmsg("skipping relation \"%s\" since it is a shard", get_rel_name(relationId)))); } PG_RETURN_BOOL(false); } PG_RETURN_BOOL(RelationIsVisible(relationId)); } /* * ErrorIfRelationIsAKnownShard errors out if the relation with relationId is * a shard relation. */ void ErrorIfRelationIsAKnownShard(Oid relationId) { if (!RelationIsAKnownShard(relationId)) { return; } const char *relationName = get_rel_name(relationId); ereport(ERROR, (errmsg("relation \"%s\" is a shard relation ", relationName))); } /* * ErrorIfIllegallyChangingKnownShard errors out if the relation with relationId is * a known shard and manual changes on known shards are disabled. This is * valid for only non-citus (external) connections. */ void ErrorIfIllegallyChangingKnownShard(Oid relationId) { /* allow Citus to make changes, and allow the user if explicitly enabled */ if (LocalExecutorShardId != INVALID_SHARD_ID || IsCitusInternalBackend() || IsRebalancerInternalBackend() || EnableManualChangesToShards) { return; } if (RelationIsAKnownShard(relationId)) { const char *relationName = get_rel_name(relationId); ereport(ERROR, (errmsg("cannot modify \"%s\" because it is a shard of " "a distributed table", relationName), errhint("Use the distributed table or set " "citus.enable_manual_changes_to_shards to on " "to modify shards directly"))); } } /* * RelationIsAKnownShard gets a relationId, check whether it's a shard of * any distributed table. */ bool RelationIsAKnownShard(Oid shardRelationId) { bool missingOk = true; char relKind = '\0'; if (!OidIsValid(shardRelationId)) { /* we cannot continue without a valid Oid */ return false; } if (IsCoordinator()) { bool coordinatorIsKnown = false; PrimaryNodeForGroup(0, &coordinatorIsKnown); if (!coordinatorIsKnown) { /* * We're not interested in shards in the coordinator * or non-mx worker nodes, unless the coordinator is * in pg_dist_node. */ return false; } } /* * We do not take locks here, because that might block a query on pg_class. */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(shardRelationId))) { /* relation does not exist */ return false; } /* * If the input relation is an index we simply replace the * relationId with the corresponding relation to hide indexes * as well. */ relKind = get_rel_relkind(shardRelationId); if (relKind == RELKIND_INDEX || relKind == RELKIND_PARTITIONED_INDEX) { shardRelationId = IndexGetRelation(shardRelationId, false); } /* get the shard's relation name */ char *shardRelationName = get_rel_name(shardRelationId); uint64 shardId = ExtractShardIdFromTableName(shardRelationName, missingOk); if (shardId == INVALID_SHARD_ID) { /* * The format of the table name does not align with * our shard name definition. */ return false; } /* try to get the relation id */ Oid relationId = LookupShardRelationFromCatalog(shardId, true); if (!OidIsValid(relationId)) { /* there is no such relation */ return false; } /* verify that their namespaces are the same */ if (get_rel_namespace(shardRelationId) != get_rel_namespace(relationId)) { return false; } /* * Now get the relation name and append the shardId to it. We need * to do that because otherwise a local table with a valid shardId * appended to its name could be misleading. */ char *generatedRelationName = get_rel_name(relationId); AppendShardIdToName(&generatedRelationName, shardId); if (strncmp(shardRelationName, generatedRelationName, NAMEDATALEN) == 0) { /* we found the distributed table that the input shard belongs to */ return true; } return false; } /* * HideShardsFromSomeApplications transforms queries to pg_class to * filter out known shards if the application_name does not match any of * the prefixes in citus.show_shards_for_app_name_prefix. */ void HideShardsFromSomeApplications(Query *query) { if (!OverrideTableVisibility || HideShards == DO_NOT_HIDE_SHARDS || !CitusHasBeenLoaded() || !CheckCitusVersion(DEBUG2)) { return; } if (ShouldHideShards()) { FilterShardsFromPgclass((Node *) query, NULL); } } /* * ShouldHideShards returns whether we should hide shards in the current * session. It only checks the application_name once and then uses a * cached response unless either the application_name or * citus.show_shards_for_app_name_prefix changes. */ static bool ShouldHideShards(void) { if (HideShards == CHECK_APPLICATION_NAME) { if (ShouldHideShardsInternal()) { HideShards = HIDE_SHARDS_FROM_APPLICATION; return true; } else { HideShards = DO_NOT_HIDE_SHARDS; return false; } } else { return HideShards == HIDE_SHARDS_FROM_APPLICATION; } } /* * ResetHideShardsDecision resets the decision whether to hide shards. */ void ResetHideShardsDecision(void) { HideShards = CHECK_APPLICATION_NAME; } /* * ShouldHideShardsInternal determines whether we should hide shards based on * the current application name. */ static bool ShouldHideShardsInternal(void) { if (MyBackendType == B_BG_WORKER) { if (IsPgBgWorker()) { /* * If a background worker belongs to Postgres, we should * never hide shards. For other background workers, enforce * the application_name check below. */ return false; } } else if (MyBackendType != B_BACKEND && MyBackendType != B_WAL_SENDER) { /* * We are aiming only to hide shards from client * backends or certain background workers(see above), */ return false; } if (IsCitusInternalBackend() || IsRebalancerInternalBackend() || IsCitusRunCommandBackend() || IsCitusShardTransferBackend()) { /* we never hide shards from Citus */ return false; } List *prefixList = NIL; /* SplitGUCList scribbles on the input */ char *splitCopy = pstrdup(ShowShardsForAppNamePrefixes); if (!SplitGUCList(splitCopy, ',', &prefixList)) { /* invalid GUC value, ignore */ return true; } char *appNamePrefix = NULL; foreach_declared_ptr(appNamePrefix, prefixList) { /* never hide shards when one of the prefixes is * */ if (strcmp(appNamePrefix, "*") == 0) { return false; } /* compare only the first first characters */ int prefixLength = strlen(appNamePrefix); if (strncmp(application_name, appNamePrefix, prefixLength) == 0) { return false; } } /* default behaviour: hide shards */ return true; } /* * IsPgBgWorker returns true if the current background worker * belongs to Postgres. */ static bool IsPgBgWorker(void) { Assert(MyBackendType == B_BG_WORKER); if (MyBgworkerEntry) { return strcmp(MyBgworkerEntry->bgw_library_name, "postgres") == 0; } return false; } /* * FilterShardsFromPgclass adds a "relation_is_a_known_shard(oid) IS NOT TRUE" * filter to the quals of queries that query pg_class. */ static bool FilterShardsFromPgclass(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Query)) { Query *query = (Query *) node; MemoryContext queryContext = GetMemoryChunkContext(query); /* * We process the whole rtable rather than visiting individual RangeTblEntry's * in the walker, since we need to know the varno to generate the right * filter. */ int varno = 0; RangeTblEntry *rangeTableEntry = NULL; foreach_declared_ptr(rangeTableEntry, query->rtable) { varno++; if (rangeTableEntry->rtekind != RTE_RELATION || rangeTableEntry->relid != RelationRelationId) { /* not pg_class */ continue; } /* * Skip if pg_class is not actually queried. This is possible on * INSERT statements that insert into pg_class. */ if (!expression_tree_walker((Node *) query->jointree->fromlist, HasRangeTableRef, &varno)) { /* the query references pg_class */ continue; } /* make sure the expression is in the right memory context */ MemoryContext originalContext = MemoryContextSwitchTo(queryContext); /* add relation_is_a_known_shard(oid) IS NOT TRUE to the quals of the query */ Node *newQual = CreateRelationIsAKnownShardFilter(varno); #if PG_VERSION_NUM >= PG_VERSION_17 /* * In PG17, MERGE queries introduce a new struct `mergeJoinCondition`. * We need to handle this condition safely. */ if (query->mergeJoinCondition != NULL) { /* Add the filter to mergeJoinCondition */ query->mergeJoinCondition = (Node *) makeBoolExpr( AND_EXPR, list_make2(query->mergeJoinCondition, newQual), -1); } else #endif { /* Handle older versions or queries without mergeJoinCondition */ Node *oldQuals = query->jointree->quals; if (oldQuals) { query->jointree->quals = (Node *) makeBoolExpr( AND_EXPR, list_make2(oldQuals, newQual), -1); } else { query->jointree->quals = newQual; } } MemoryContextSwitchTo(originalContext); } return query_tree_walker((Query *) node, FilterShardsFromPgclass, context, 0); } return expression_tree_walker(node, FilterShardsFromPgclass, context); } /* * HasRangeTableRef passed to expression_tree_walker to check if a node is a * RangeTblRef of the given varno is present in a fromlist. */ static bool HasRangeTableRef(Node *node, int *varno) { if (node == NULL) { return false; } if (IsA(node, RangeTblRef)) { RangeTblRef *rangeTblRef = (RangeTblRef *) node; return rangeTblRef->rtindex == *varno; } return expression_tree_walker(node, HasRangeTableRef, varno); } /* * CreateRelationIsAKnownShardFilter constructs an expression of the form: * pg_catalog.relation_is_a_known_shard(oid) IS NOT TRUE * * The difference between "NOT pg_catalog.relation_is_a_known_shard(oid)" and * "pg_catalog.relation_is_a_known_shard(oid) IS NOT TRUE" is that the former * will return FALSE if the function returns NULL, while the second will return * TRUE. This difference is important in the case of outer joins, because this * filter might be applied on an oid that is then NULL. */ static Node * CreateRelationIsAKnownShardFilter(int pgClassVarno) { /* oid is always the first column */ AttrNumber oidAttNum = 1; Var *oidVar = makeVar(pgClassVarno, oidAttNum, OIDOID, -1, InvalidOid, 0); /* build the call to read_intermediate_result */ FuncExpr *funcExpr = makeNode(FuncExpr); funcExpr->funcid = RelationIsAKnownShardFuncId(); funcExpr->funcretset = false; funcExpr->funcvariadic = false; funcExpr->funcformat = 0; funcExpr->funccollid = 0; funcExpr->inputcollid = 0; funcExpr->location = -1; funcExpr->args = list_make1(oidVar); BooleanTest *notExpr = makeNode(BooleanTest); notExpr->booltesttype = IS_NOT_TRUE; notExpr->arg = (Expr *) funcExpr; notExpr->location = -1; return (Node *) notExpr; } ================================================ FILE: src/backend/distributed/worker/worker_sql_task_protocol.c ================================================ /*------------------------------------------------------------------------- * * worker_sql_task_protocol.c * * Routines for executing SQL tasks. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ /* necessary to get S_IRUSR, S_IWUSR definitions on illumos */ #include #include "postgres.h" #include "funcapi.h" #include "pgstat.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "distributed/commands/multi_copy.h" #include "distributed/multi_executor.h" #include "distributed/transmit.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #define COPY_BUFFER_SIZE (4 * 1024 * 1024) /* TaskFileDestReceiver can be used to stream results into a file */ typedef struct TaskFileDestReceiver { /* public DestReceiver interface */ DestReceiver pub; /* descriptor of the tuples that are sent to the worker */ TupleDesc tupleDescriptor; /* context for per-tuple memory allocation */ MemoryContext tupleContext; /* MemoryContext for DestReceiver session */ MemoryContext memoryContext; /* output file */ char *filePath; FileCompat fileCompat; bool binaryCopyFormat; /* state on how to copy out data types */ CopyOutState copyOutState; FmgrInfo *columnOutputFunctions; /* statistics */ uint64 tuplesSent; uint64 bytesSent; } TaskFileDestReceiver; static void TaskFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static bool TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); static void WriteToLocalFile(StringInfo copyData, TaskFileDestReceiver *taskFileDest); static void TaskFileDestReceiverShutdown(DestReceiver *destReceiver); static void TaskFileDestReceiverDestroy(DestReceiver *destReceiver); /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(worker_execute_sql_task); /* * worker_execute_sql_task executes a query and writes the results to * a file according to the usual task naming scheme. */ Datum worker_execute_sql_task(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("This UDF is deprecated."))); PG_RETURN_INT64(0); } /* * CreateFileDestReceiver creates a DestReceiver for writing query results * to a file. */ DestReceiver * CreateFileDestReceiver(char *filePath, MemoryContext tupleContext, bool binaryCopyFormat) { TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) palloc0( sizeof(TaskFileDestReceiver)); /* set up the DestReceiver function pointers */ taskFileDest->pub.receiveSlot = TaskFileDestReceiverReceive; taskFileDest->pub.rStartup = TaskFileDestReceiverStartup; taskFileDest->pub.rShutdown = TaskFileDestReceiverShutdown; taskFileDest->pub.rDestroy = TaskFileDestReceiverDestroy; taskFileDest->pub.mydest = DestCopyOut; /* set up output parameters */ taskFileDest->tupleContext = tupleContext; taskFileDest->memoryContext = CurrentMemoryContext; taskFileDest->filePath = pstrdup(filePath); taskFileDest->binaryCopyFormat = binaryCopyFormat; return (DestReceiver *) taskFileDest; } /* * TaskFileDestReceiverStartup implements the rStartup interface of * TaskFileDestReceiver. It opens the destination file and sets up * the CopyOutState. */ static void TaskFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor) { TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) dest; const char *delimiterCharacter = "\t"; const char *nullPrintCharacter = "\\N"; const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); /* use the memory context that was in place when the DestReceiver was created */ MemoryContext oldContext = MemoryContextSwitchTo(taskFileDest->memoryContext); taskFileDest->tupleDescriptor = inputTupleDescriptor; /* define how tuples will be serialised */ CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; copyOutState->null_print = (char *) nullPrintCharacter; copyOutState->null_print_client = (char *) nullPrintCharacter; copyOutState->binary = taskFileDest->binaryCopyFormat; copyOutState->fe_msgbuf = makeStringInfo(); copyOutState->rowcontext = taskFileDest->tupleContext; taskFileDest->copyOutState = copyOutState; taskFileDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor, copyOutState->binary); taskFileDest->fileCompat = FileCompatFromFileStart(FileOpenForTransmit( taskFileDest->filePath, fileFlags)); if (copyOutState->binary) { /* write headers when using binary encoding */ AppendCopyBinaryHeaders(copyOutState); } MemoryContextSwitchTo(oldContext); } /* * TaskFileDestReceiverReceive implements the receiveSlot function of * TaskFileDestReceiver. It takes a TupleTableSlot and writes the contents * to a local file. */ static bool TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) { TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) dest; TupleDesc tupleDescriptor = taskFileDest->tupleDescriptor; CopyOutState copyOutState = taskFileDest->copyOutState; FmgrInfo *columnOutputFunctions = taskFileDest->columnOutputFunctions; StringInfo copyData = copyOutState->fe_msgbuf; MemoryContext executorTupleContext = taskFileDest->tupleContext; MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext); slot_getallattrs(slot); Datum *columnValues = slot->tts_values; bool *columnNulls = slot->tts_isnull; /* construct row in COPY format */ AppendCopyRowData(columnValues, columnNulls, tupleDescriptor, copyOutState, columnOutputFunctions, NULL); if (copyData->len > COPY_BUFFER_SIZE) { WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); resetStringInfo(copyData); } MemoryContextSwitchTo(oldContext); taskFileDest->tuplesSent++; MemoryContextReset(executorTupleContext); return true; } /* * WriteToLocalResultsFile writes the bytes in a StringInfo to a local file. */ static void WriteToLocalFile(StringInfo copyData, TaskFileDestReceiver *taskFileDest) { int bytesWritten = FileWriteCompat(&taskFileDest->fileCompat, copyData->data, copyData->len, PG_WAIT_IO); if (bytesWritten < 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not append to file: %m"))); } taskFileDest->bytesSent += bytesWritten; } /* * TaskFileDestReceiverShutdown implements the rShutdown interface of * TaskFileDestReceiver. It writes the footer and closes the file. * the relation. */ static void TaskFileDestReceiverShutdown(DestReceiver *destReceiver) { TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) destReceiver; CopyOutState copyOutState = taskFileDest->copyOutState; if (copyOutState->fe_msgbuf->len > 0) { WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); resetStringInfo(copyOutState->fe_msgbuf); } if (copyOutState->binary) { /* write footers when using binary encoding */ AppendCopyBinaryFooters(copyOutState); WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); resetStringInfo(copyOutState->fe_msgbuf); } FileClose(taskFileDest->fileCompat.fd); } /* * TaskFileDestReceiverDestroy frees memory allocated as part of the * TaskFileDestReceiver and closes file descriptors. */ static void TaskFileDestReceiverDestroy(DestReceiver *destReceiver) { TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) destReceiver; if (taskFileDest->copyOutState) { pfree(taskFileDest->copyOutState); taskFileDest->copyOutState = NULL; } if (taskFileDest->columnOutputFunctions) { pfree(taskFileDest->columnOutputFunctions); taskFileDest->columnOutputFunctions = NULL; } if (taskFileDest->filePath) { pfree(taskFileDest->filePath); taskFileDest->filePath = NULL; } } /* * FileDestReceiverStats returns statistics for the given file dest receiver. */ void FileDestReceiverStats(DestReceiver *dest, uint64 *rowsSent, uint64 *bytesSent) { TaskFileDestReceiver *fileDestReceiver = (TaskFileDestReceiver *) dest; *rowsSent = fileDestReceiver->tuplesSent; *bytesSent = fileDestReceiver->bytesSent; } ================================================ FILE: src/backend/distributed/worker/worker_truncate_trigger_protocol.c ================================================ /*------------------------------------------------------------------------- * * worker_create_truncate_trigger_protocol.c * * Routines for creating truncate triggers on distributed tables on worker nodes. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #include "postgres.h" #include "fmgr.h" #include "utils/elog.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "distributed/citus_ruleutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" PG_FUNCTION_INFO_V1(worker_create_truncate_trigger); /* * worker_create_truncate_trigger creates a truncate trigger for the given distributed * table on current metadata worker. The function is intended to be called by the * coordinator node during metadata propagation of mx tables or during the upgrades from * citus version <=5.2 to >=6.1. The function requires superuser permissions. */ Datum worker_create_truncate_trigger(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); Oid relationId = PG_GETARG_OID(0); EnsureTableOwner(relationId); /* Create the truncate trigger */ CreateTruncateTrigger(relationId); PG_RETURN_VOID(); } ================================================ FILE: src/include/.gitignore ================================================ /stamp-h /stamp-ext-h /citus_config.h /citus_config.h.in~ /citus_version.h /citus_version.h.in~ ================================================ FILE: src/include/citus_config.h.in ================================================ /* src/include/citus_config.h.in. Generated from configure.ac by autoheader. */ /* * citus_config.h.in is generated by autoconf/autoheader and * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! */ /* Citus edition as a string */ #undef CITUS_EDITION /* Extension version expected by this Citus build */ #undef CITUS_EXTENSIONVERSION /* Citus major version as a string */ #undef CITUS_MAJORVERSION /* Citus full name as a string */ #undef CITUS_NAME /* Citus version as a string */ #undef CITUS_VERSION /* Citus version as a number */ #undef CITUS_VERSION_NUM /* A string containing the version number, platform, and C compiler */ #undef CITUS_VERSION_STR /* Define to 1 to build with lz4 support. (--with-lz4) */ #undef HAVE_CITUS_LIBLZ4 /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H /* Define to 1 if you have the `lz4' library (-llz4). */ #undef HAVE_LIBLZ4 /* Define to 1 if you have the `zstd' library (-lzstd). */ #undef HAVE_LIBZSTD /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H /* Define to 1 if you have the header file. */ #undef HAVE_STRINGS_H /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_STAT_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TYPES_H /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT /* Define to the full name of this package. */ #undef PACKAGE_NAME /* Define to the full name and version of this package. */ #undef PACKAGE_STRING /* Define to the one symbol short name of this package. */ #undef PACKAGE_TARNAME /* Define to the home page for this package. */ #undef PACKAGE_URL /* Define to the version of this package. */ #undef PACKAGE_VERSION /* The size of `void *', as computed by sizeof. */ #undef SIZEOF_VOID_P /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS ================================================ FILE: src/include/citus_version.h.in ================================================ /* This file is created manually */ /* Citus full name as a string */ #undef CITUS_NAME /* Citus edition as a string */ #undef CITUS_EDITION /* Extension version expected by this Citus build */ #undef CITUS_EXTENSIONVERSION /* Citus major version as a string */ #undef CITUS_MAJORVERSION /* Citus version as a string */ #undef CITUS_VERSION /* Citus version as a number */ #undef CITUS_VERSION_NUM /* A string containing the version number, platform, and C compiler */ #undef CITUS_VERSION_STR /* Define to 1 if you have the `liblz4' library (-llz4). */ #undef HAVE_CITUS_LIBLZ4 /* Define to 1 if you have the `libzstd' library (-lzstd). */ #undef HAVE_LIBZSTD /* Base URL for statistics collection and update checks */ #undef REPORTS_BASE_URL ================================================ FILE: src/include/columnar/columnar.h ================================================ /*------------------------------------------------------------------------- * * columnar.h * * Type and function declarations for Columnar * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_H #define COLUMNAR_H #include "postgres.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "storage/bufpage.h" #include "storage/lockdefs.h" #include "storage/relfilelocator.h" #include "utils/relcache.h" #include "utils/snapmgr.h" #include "pg_version_compat.h" #include "columnar/columnar_compression.h" #include "columnar/columnar_metadata.h" #define COLUMNAR_AM_NAME "columnar" #define COLUMNAR_MODULE_NAME "citus_columnar" #define COLUMNAR_SETOPTIONS_HOOK_SYM "ColumnarTableSetOptions_hook" /* Defines for valid option names */ #define OPTION_NAME_COMPRESSION_TYPE "compression" #define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_limit" #define OPTION_NAME_CHUNK_ROW_COUNT "chunk_group_row_limit" /* Limits for option parameters */ #define STRIPE_ROW_COUNT_MINIMUM 1000 #define STRIPE_ROW_COUNT_MAXIMUM 10000000 #define CHUNK_ROW_COUNT_MINIMUM 1000 #define CHUNK_ROW_COUNT_MAXIMUM 100000 #define COMPRESSION_LEVEL_MIN 1 #define COMPRESSION_LEVEL_MAX 19 /* Columnar file signature */ #define COLUMNAR_VERSION_MAJOR 2 #define COLUMNAR_VERSION_MINOR 0 /* miscellaneous defines */ #define COLUMNAR_TUPLE_COST_MULTIPLIER 10 #define COLUMNAR_POSTSCRIPT_SIZE_LENGTH 1 #define COLUMNAR_POSTSCRIPT_SIZE_MAX 256 #define COLUMNAR_BYTES_PER_PAGE (BLCKSZ - SizeOfPageHeaderData) /*global variables for citus_columnar fake version Y */ #define CITUS_COLUMNAR_INTERNAL_VERSION "11.1-0" /* * We can't rely on RelidByRelfilenumber for temp tables since PG18(it was backpatched * through PG13), so we can use this macro to define relid within relation in case of * temp relations. Otherwise RelidByRelfilenumber should be used. */ #define RelationPrecomputeOid(a) (RelationUsesLocalBuffers(a) ? RelationGetRelid(a) : \ InvalidOid) /* * ColumnarOptions holds the option values to be used when reading or writing * a columnar table. To resolve these values, we first check foreign table's options, * and if not present, we then fall back to the default values specified above. */ typedef struct ColumnarOptions { uint64 stripeRowCount; uint32 chunkRowCount; CompressionType compressionType; int compressionLevel; } ColumnarOptions; /* ColumnChunkSkipNode contains statistics for a ColumnChunkData. */ typedef struct ColumnChunkSkipNode { /* statistics about values of a column chunk */ bool hasMinMax; Datum minimumValue; Datum maximumValue; uint64 rowCount; /* * Offsets and sizes of value and exists streams in the column data. * These enable us to skip reading suppressed row chunks, and start reading * a chunk without reading previous chunks. */ uint64 valueChunkOffset; uint64 valueLength; uint64 existsChunkOffset; uint64 existsLength; /* * This is used for (1) determining destination size when decompressing, * (2) calculating compression rates when logging stats. */ uint64 decompressedValueSize; CompressionType valueCompressionType; int valueCompressionLevel; } ColumnChunkSkipNode; /* * StripeSkipList can be used for skipping row chunks. It contains a column chunk * skip node for each chunk of each column. chunkSkipNodeArray[column][chunk] * is the entry for the specified column chunk. */ typedef struct StripeSkipList { ColumnChunkSkipNode **chunkSkipNodeArray; uint32 *chunkGroupRowCounts; uint32 columnCount; uint32 chunkCount; } StripeSkipList; /* * ChunkData represents a chunk of data for multiple columns. valueArray stores * the values of data, and existsArray stores whether a value is present. * valueBuffer is used to store (uncompressed) serialized values * referenced by Datum's in valueArray. It is only used for by-reference Datum's. * There is a one-to-one correspondence between valueArray and existsArray. */ typedef struct ChunkData { uint32 rowCount; uint32 columnCount; /* * Following are indexed by [column][row]. If a column is not projected, * then existsArray[column] and valueArray[column] are NULL. */ bool **existsArray; Datum **valueArray; /* valueBuffer keeps actual data for type-by-reference datums from valueArray. */ StringInfo *valueBufferArray; } ChunkData; /* * ColumnChunkBuffers represents a chunk of serialized data in a column. * valueBuffer stores the serialized values of data, and existsBuffer stores * serialized value of presence information. valueCompressionType contains * compression type if valueBuffer is compressed. Finally rowCount has * the number of rows in this chunk. */ typedef struct ColumnChunkBuffers { StringInfo existsBuffer; StringInfo valueBuffer; CompressionType valueCompressionType; uint64 decompressedValueSize; } ColumnChunkBuffers; /* * ColumnBuffers represents data buffers for a column in a row stripe. Each * column is made of multiple column chunks. */ typedef struct ColumnBuffers { ColumnChunkBuffers **chunkBuffersArray; } ColumnBuffers; /* StripeBuffers represents data for a row stripe. */ typedef struct StripeBuffers { uint32 columnCount; uint32 rowCount; ColumnBuffers **columnBuffersArray; uint32 *selectedChunkGroupRowCounts; } StripeBuffers; /* return value of StripeWriteState to decide stripe write state */ typedef enum StripeWriteStateEnum { /* stripe write is flushed to disk, so it's readable */ STRIPE_WRITE_FLUSHED, /* * Writer transaction did abort either before inserting into * columnar.stripe or after. */ STRIPE_WRITE_ABORTED, /* * Writer transaction is still in-progress. Note that it is not certain * if it is being written by current backend's current transaction or * another backend. */ STRIPE_WRITE_IN_PROGRESS } StripeWriteStateEnum; typedef bool (*ColumnarSupportsIndexAM_type)(char *); typedef const char *(*CompressionTypeStr_type)(CompressionType); typedef bool (*IsColumnarTableAmTable_type)(Oid); typedef bool (*ReadColumnarOptions_type)(Oid, ColumnarOptions *); /* ColumnarReadState represents state of a columnar scan. */ struct ColumnarReadState; typedef struct ColumnarReadState ColumnarReadState; /* ColumnarWriteState represents state of a columnar write operation. */ struct ColumnarWriteState; typedef struct ColumnarWriteState ColumnarWriteState; /* GUCs */ extern int columnar_compression; extern int columnar_stripe_row_limit; extern int columnar_chunk_group_row_limit; extern int columnar_compression_level; /* called when the user changes options on the given relation */ typedef void (*ColumnarTableSetOptions_hook_type)(Oid relid, ColumnarOptions options); extern void columnar_init(void); extern void columnar_init_gucs(void); extern CompressionType ParseCompressionType(const char *compressionTypeString); /* Function declarations for writing to a columnar table */ extern ColumnarWriteState * ColumnarBeginWrite(Relation rel, ColumnarOptions options, TupleDesc tupleDescriptor); extern uint64 ColumnarWriteRow(ColumnarWriteState *state, Datum *columnValues, bool *columnNulls); extern void ColumnarFlushPendingWrites(ColumnarWriteState *state); extern void ColumnarEndWrite(ColumnarWriteState *state); extern bool ContainsPendingWrites(ColumnarWriteState *state); extern MemoryContext ColumnarWritePerTupleContext(ColumnarWriteState *state); /* Function declarations for reading from columnar table */ /* functions applicable for both sequential and random access */ extern ColumnarReadState * ColumnarBeginRead(Relation relation, TupleDesc tupleDescriptor, List *projectedColumnList, List *qualConditions, MemoryContext scanContext, Snapshot snaphot, bool randomAccess); extern void ColumnarReadFlushPendingWrites(ColumnarReadState *readState); extern void ColumnarEndRead(ColumnarReadState *state); extern void ColumnarResetRead(ColumnarReadState *readState); /* functions only applicable for sequential access */ extern bool ColumnarReadNextRow(ColumnarReadState *state, Datum *columnValues, bool *columnNulls, uint64 *rowNumber); extern int64 ColumnarReadChunkGroupsFiltered(ColumnarReadState *state); extern void ColumnarRescan(ColumnarReadState *readState, List *scanQual); /* functions only applicable for random access */ extern void ColumnarReadRowByRowNumberOrError(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls); extern bool ColumnarReadRowByRowNumber(ColumnarReadState *readState, uint64 rowNumber, Datum *columnValues, bool *columnNulls); /* Function declarations for common functions */ extern FmgrInfo * GetFunctionInfoOrNull(Oid typeId, Oid accessMethodId, int16 procedureId); extern ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask, uint32 chunkGroupRowCount); extern void FreeChunkData(ChunkData *chunkData); extern uint64 ColumnarTableRowCount(Relation relation); extern PGDLLEXPORT const char * CompressionTypeStr(CompressionType type); /* columnar_metadata_tables.c */ extern PGDLLEXPORT void InitColumnarOptions(Oid regclass); extern PGDLLEXPORT void SetColumnarOptions(Oid regclass, ColumnarOptions *options); extern PGDLLEXPORT bool DeleteColumnarTableOptions(Oid regclass, bool missingOk); extern PGDLLEXPORT bool ReadColumnarOptions(Oid regclass, ColumnarOptions *options); extern PGDLLEXPORT bool IsColumnarTableAmTable(Oid relationId); /* columnar_metadata_tables.c */ extern void DeleteMetadataRows(Relation rel); extern uint64 ColumnarMetadataNewStorageId(void); extern uint64 GetHighestUsedAddress(Relation rel); extern EmptyStripeReservation * ReserveEmptyStripe(Relation rel, uint64 columnCount, uint64 chunkGroupRowCount, uint64 stripeRowCount); extern StripeMetadata * CompleteStripeReservation(Relation rel, uint64 stripeId, uint64 sizeBytes, uint64 rowCount, uint64 chunkCount); extern void SaveStripeSkipList(Oid relid, RelFileLocator relfilelocator, uint64 stripe, StripeSkipList *stripeSkipList, TupleDesc tupleDescriptor); extern void SaveChunkGroups(Oid relid, RelFileLocator relfilelocator, uint64 stripe, List *chunkGroupRowCounts); extern StripeSkipList * ReadStripeSkipList(Relation rel, uint64 stripe, TupleDesc tupleDescriptor, uint32 chunkCount, Snapshot snapshot); extern StripeMetadata * FindNextStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot); extern StripeMetadata * FindStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot); extern StripeMetadata * FindStripeWithMatchingFirstRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot); extern StripeWriteStateEnum StripeWriteState(StripeMetadata *stripeMetadata); extern uint64 StripeGetHighestRowNumber(StripeMetadata *stripeMetadata); extern StripeMetadata * FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot); extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS); extern Oid ColumnarRelationId(Oid relid, RelFileLocator relfilelocator); /* write_state_management.c */ extern ColumnarWriteState * columnar_init_write_state(Relation relation, TupleDesc tupdesc, Oid tupSlotRelationId, SubTransactionId currentSubXid); extern void FlushWriteStateForRelfilenumber(RelFileNumber relfilenumber, SubTransactionId currentSubXid); extern void FlushWriteStateForAllRels(SubTransactionId currentSubXid, SubTransactionId parentSubXid); extern void DiscardWriteStateForAllRels(SubTransactionId currentSubXid, SubTransactionId parentSubXid); extern void MarkRelfilenumberDropped(RelFileNumber relfilenumber, SubTransactionId currentSubXid); extern void NonTransactionDropWriteState(RelFileNumber relfilenumber); extern bool PendingWritesInUpperTransactions(RelFileNumber relfilenumber, SubTransactionId currentSubXid); extern MemoryContext GetWriteContextForDebug(void); #endif /* COLUMNAR_H */ ================================================ FILE: src/include/columnar/columnar_compression.h ================================================ /*------------------------------------------------------------------------- * * columnar_compression.h * * Type and function declarations for compression methods. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_COMPRESSION_H #define COLUMNAR_COMPRESSION_H /* Enumaration for columnar table's compression method */ typedef enum { COMPRESSION_TYPE_INVALID = -1, COMPRESSION_NONE = 0, COMPRESSION_PG_LZ = 1, COMPRESSION_LZ4 = 2, COMPRESSION_ZSTD = 3, COMPRESSION_COUNT } CompressionType; extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, CompressionType compressionType, int compressionLevel); extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType, uint64 decompressedSize); #endif /* COLUMNAR_COMPRESSION_H */ ================================================ FILE: src/include/columnar/columnar_customscan.h ================================================ /*------------------------------------------------------------------------- * * columnar_customscan.h * * Forward declarations of functions to hookup the custom scan feature of * columnar. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_CUSTOMSCAN_H #define COLUMNAR_CUSTOMSCAN_H void columnar_customscan_init(void); #endif /* COLUMNAR_CUSTOMSCAN_H */ ================================================ FILE: src/include/columnar/columnar_metadata.h ================================================ /*------------------------------------------------------------------------- * * columnar_metadata.h * * Type and function declarations for Columnar metadata. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_METADATA_H #define COLUMNAR_METADATA_H #include "postgres.h" #include "storage/relfilelocator.h" #include "pg_version_compat.h" #include "pg_version_constants.h" /* * StripeMetadata represents information about a stripe. This information is * stored in the metadata table "columnar.stripe". */ typedef struct StripeMetadata { uint64 fileOffset; uint64 dataLength; uint32 columnCount; uint32 chunkCount; uint32 chunkGroupRowCount; uint64 rowCount; uint64 id; uint64 firstRowNumber; /* see StripeWriteState */ bool aborted; /* * If write operation is in-progress (i.e. StripeWriteState returned * STRIPE_WRITE_IN_PROGRESS), then insertedByCurrentXact is used to * distinguish whether it's being written by current transaction or * not. */ bool insertedByCurrentXact; } StripeMetadata; /* * EmptyStripeReservation represents information for an empty stripe * reservation. */ typedef struct EmptyStripeReservation { uint64 stripeId; uint64 stripeFirstRowNumber; } EmptyStripeReservation; extern List * StripesForRelfilelocator(Relation rel); extern void ColumnarStorageUpdateIfNeeded(Relation rel, bool isUpgrade); extern List * ExtractColumnarRelOptions(List *inOptions, List **outColumnarOptions); extern void SetColumnarRelOptions(RangeVar *rv, List *reloptions); #endif /* COLUMNAR_METADATA_H */ ================================================ FILE: src/include/columnar/columnar_storage.h ================================================ /*------------------------------------------------------------------------- * * columnar_storage.h * * Type and function declarations for storage of columnar data in blocks. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_STORAGE_H #define COLUMNAR_STORAGE_H #include "postgres.h" #include "storage/smgr.h" #include "utils/rel.h" #include "columnar/columnar_tableam.h" #define COLUMNAR_INVALID_ROW_NUMBER ((uint64) 0) #define COLUMNAR_FIRST_ROW_NUMBER ((uint64) 1) #define COLUMNAR_MAX_ROW_NUMBER ((uint64) \ (COLUMNAR_FIRST_ROW_NUMBER + \ VALID_ITEMPOINTER_OFFSETS * \ VALID_BLOCKNUMBERS)) /* * Logical offsets never fall on the first two physical pages. See * comments in columnar_storage.c. */ #define ColumnarInvalidLogicalOffset 0 #define ColumnarFirstLogicalOffset ((BLCKSZ - SizeOfPageHeaderData) * 2) #define ColumnarLogicalOffsetIsValid(X) ((X) >= ColumnarFirstLogicalOffset) extern void ColumnarStorageInit(SMgrRelation srel, uint64 storageId); extern bool ColumnarStorageIsCurrent(Relation rel); extern void ColumnarStorageUpdateCurrent(Relation rel, bool upgrade, uint64 reservedStripeId, uint64 reservedRowNumber, uint64 reservedOffset); extern uint64 ColumnarStorageGetVersionMajor(Relation rel, bool force); extern uint64 ColumnarStorageGetVersionMinor(Relation rel, bool force); extern uint64 ColumnarStorageGetStorageId(Relation rel, bool force); extern uint64 ColumnarStorageGetReservedStripeId(Relation rel, bool force); extern uint64 ColumnarStorageGetReservedRowNumber(Relation rel, bool force); extern uint64 ColumnarStorageGetReservedOffset(Relation rel, bool force); extern uint64 ColumnarStorageReserveData(Relation rel, uint64 amount); extern uint64 ColumnarStorageReserveRowNumber(Relation rel, uint64 nrows); extern uint64 ColumnarStorageReserveStripeId(Relation rel); extern void ColumnarStorageRead(Relation rel, uint64 logicalOffset, char *data, uint32 amount); extern void ColumnarStorageWrite(Relation rel, uint64 logicalOffset, char *data, uint32 amount); extern bool ColumnarStorageTruncate(Relation rel, uint64 newDataReservation); #endif /* COLUMNAR_STORAGE_H */ ================================================ FILE: src/include/columnar/columnar_tableam.h ================================================ #ifndef COLUMNAR_TABLEAM_H #define COLUMNAR_TABLEAM_H #include "postgres.h" #include "fmgr.h" #include "access/heapam.h" #include "access/skey.h" #include "access/tableam.h" #include "catalog/indexing.h" #include "nodes/bitmapset.h" #include "utils/acl.h" #include "citus_version.h" /* * Number of valid ItemPointer Offset's for "row number" <> "ItemPointer" * mapping. * * Postgres has some asserts calling either ItemPointerIsValid or * OffsetNumberIsValid. That constraints itemPointer.offsetNumber * for columnar tables to the following interval: * [FirstOffsetNumber, MaxOffsetNumber]. * * However, bitmap scan logic assumes that itemPointer.offsetNumber cannot * be larger than MaxHeapTuplesPerPage (see tbm_add_tuples). * * For this reason, we restrict itemPointer.offsetNumber * to the following interval: [FirstOffsetNumber, MaxHeapTuplesPerPage]. */ #define VALID_ITEMPOINTER_OFFSETS \ ((uint64) (MaxHeapTuplesPerPage - FirstOffsetNumber + 1)) /* * Number of valid ItemPointer BlockNumber's for "row number" <> "ItemPointer" * mapping. * * Similar to VALID_ITEMPOINTER_OFFSETS, due to asserts around * itemPointer.blockNumber, we can only use values upto and including * MaxBlockNumber. * Note that postgres doesn't restrict blockNumber to a lower boundary. * * For this reason, we restrict itemPointer.blockNumber * to the following interval: [0, MaxBlockNumber]. */ #define VALID_BLOCKNUMBERS ((uint64) (MaxBlockNumber + 1)) struct ColumnarScanDescData; typedef struct ColumnarScanDescData *ColumnarScanDesc; const TableAmRoutine * GetColumnarTableAmRoutine(void); extern void columnar_tableam_init(void); extern TableScanDesc columnar_beginscan_extended(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, uint32 flags, Bitmapset *attr_needed, List *scanQual); extern int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc); extern PGDLLEXPORT bool ColumnarSupportsIndexAM(char *indexAMName); extern bool IsColumnarTableAmTable(Oid relationId); extern void CheckCitusColumnarCreateExtensionStmt(Node *parseTree); extern void CheckCitusColumnarAlterExtensionStmt(Node *parseTree); extern DefElem * GetExtensionOption(List *extensionOptions, const char *defname); #endif /* COLUMNAR_TABLEAM_H */ ================================================ FILE: src/include/columnar/columnar_version_compat.h ================================================ /*------------------------------------------------------------------------- * * columnar_version_compat.h * * Compatibility macros for writing code agnostic to PostgreSQL versions * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLUMNAR_VERSION_COMPAT_H #define COLUMNAR_VERSION_COMPAT_H #include "pg_version_constants.h" /* for PG_VERSION_NUM and TupleDescAttr() */ #include "postgres.h" #include "access/htup_details.h" #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE #define ExplainPropertyLong(qlabel, value, es) \ ExplainPropertyInteger(qlabel, NULL, value, es) #endif /* COLUMNAR_COMPAT_H */ ================================================ FILE: src/include/distributed/adaptive_executor.h ================================================ #ifndef ADAPTIVE_EXECUTOR_H #define ADAPTIVE_EXECUTOR_H #include "distributed/multi_physical_planner.h" /* GUC, determining whether Citus opens 1 connection per task */ extern bool ForceMaxQueryParallelization; extern int MaxAdaptiveExecutorPoolSize; extern bool EnableBinaryProtocol; /* GUC, number of ms to wait between opening connections to the same worker */ extern int ExecutorSlowStartInterval; extern bool EnableCostBasedConnectionEstablishment; extern bool PreventIncompleteConnectionEstablishment; extern uint64 ExecuteTaskList(RowModifyLevel modLevel, List *taskList); extern uint64 ExecuteUtilityTaskList(List *utilityTaskList, bool localExecutionSupported); extern uint64 ExecuteUtilityTaskListExtended(List *utilityTaskList, int poolSize, bool localExecutionSupported); extern uint64 ExecuteTaskListOutsideTransaction(RowModifyLevel modLevel, List *taskList, int targetPoolSize, List *jobIdList); #endif /* ADAPTIVE_EXECUTOR_H */ ================================================ FILE: src/include/distributed/argutils.h ================================================ /*------------------------------------------------------------------------- * * argutils.h * * Macros to help with argument parsing in UDFs. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ /* * PG_ENSURE_ARGNOTNULL ensures that a UDF argument is not NULL and throws an * error otherwise. This is useful for non STRICT UDFs where only some * arguments are allowed to be NULL. */ #define PG_ENSURE_ARGNOTNULL(argIndex, argName) \ if (PG_ARGISNULL(argIndex)) \ { \ ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), \ errmsg("%s cannot be NULL", argName))); \ } /* * PG_GETARG_TEXT_TO_CSTRING is the same as PG_GETARG_TEXT_P, but instead of * text* it returns char*. Just like most other PG_GETARG_* macros this assumes * the argument is not NULL. */ #define PG_GETARG_TEXT_TO_CSTRING(argIndex) \ text_to_cstring(PG_GETARG_TEXT_P(argIndex)) /* * PG_GETARG_TEXT_TO_CSTRING_OR_NULL is the same as PG_GETARG_TEXT_TO_CSTRING, * but it supports the case where the argument is NULL. In this case it will * return a NULL pointer. */ #define PG_GETARG_TEXT_TO_CSTRING_OR_NULL(argIndex) \ PG_ARGISNULL(argIndex) ? NULL : PG_GETARG_TEXT_TO_CSTRING(argIndex) /* * PG_GETARG_NAME_OR_NULL is the same as PG_GETARG_NAME, but it supports the * case where the argument is NULL. In this case it will return a NULL pointer. */ #define PG_GETARG_NAME_OR_NULL(argIndex) \ PG_ARGISNULL(argIndex) ? NULL : PG_GETARG_NAME(argIndex) /* * PG_GETARG_FLOAT4_OR is the same as PG_GETARG_FLOAT4, but it supports the * case where the argument is NULL. In that case it will return the provided * fallback. */ #define PG_GETARG_FLOAT4_OR_DEFAULT(argIndex, fallback) \ PG_ARGISNULL(argIndex) ? (fallback) : PG_GETARG_FLOAT4(argIndex) ================================================ FILE: src/include/distributed/backend_data.h ================================================ /* * backend_data.h * * Data structure definition for managing backend data and related function * declarations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef BACKEND_DATA_H #define BACKEND_DATA_H #include "access/twophase.h" #include "datatype/timestamp.h" #include "nodes/pg_list.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "storage/s_lock.h" #include "distributed/transaction_identifier.h" /* * Each backend's active distributed transaction information is tracked via * BackendData in shared memory. * * DistributedTransactionId already has the same fields that CitusInitiatedBackend * has. However, we prefer to keep them seperate since CitusInitiatedBackend is a * broader concept which covers the backends that are not initiated via a distributed * transaction as well. In other words, we could have backends that * CitusInitiatedBackend is set but DistributedTransactionId is not set such as an * "INSERT" query which is not inside a transaction block. */ typedef struct BackendData { Oid databaseId; slock_t mutex; bool cancelledDueToDeadlock; uint64 globalPID; bool distributedCommandOriginator; DistributedTransactionId transactionId; bool activeBackend; /* set to false when backend exists */ } BackendData; extern void BackendManagementShmemInit(void); extern size_t BackendManagementShmemSize(void); extern void InitializeBackendManagement(void); extern int TotalProcCount(void); extern void InitializeBackendData(const char *applicationName); extern void LockBackendSharedMemory(LWLockMode lockMode); extern void UnlockBackendSharedMemory(void); extern void UnSetDistributedTransactionId(void); extern void UnSetGlobalPID(void); extern void SetActiveMyBackend(bool value); extern void AssignDistributedTransactionId(void); extern void AssignGlobalPID(const char *applicationName); extern uint64 GetGlobalPID(void); extern void SetBackendDataDatabaseId(void); extern void SetBackendDataGlobalPID(uint64 gpid); extern void SetBackendDataDistributedCommandOriginator(bool distributedCommandOriginator); extern uint64 ExtractGlobalPID(const char *applicationName); extern int ExtractNodeIdFromGlobalPID(uint64 globalPID, bool missingOk); extern int ExtractProcessIdFromGlobalPID(uint64 globalPID); extern void GetBackendDataForProc(PGPROC *proc, BackendData *result); extern void CancelTransactionDueToDeadlock(PGPROC *proc); extern bool MyBackendGotCancelledDueToDeadlock(bool clearState); extern List * ActiveDistributedTransactionNumbers(void); extern LocalTransactionId GetMyProcLocalTransactionId(void); extern int GetExternalClientBackendCount(void); extern uint32 IncrementExternalClientBackendCounter(void); extern void DecrementExternalClientBackendCounter(void); extern void DetermineCitusBackendType(const char *applicationName); extern bool IsCitusInternalBackend(void); extern bool IsRebalancerInternalBackend(void); extern bool IsCitusRunCommandBackend(void); extern bool IsExternalClientBackend(void); extern bool IsCitusShardTransferBackend(void); #define INVALID_CITUS_INTERNAL_BACKEND_GPID 0 #define GLOBAL_PID_NODE_ID_FOR_NODES_NOT_IN_METADATA 99999999 #endif /* BACKEND_DATA_H */ ================================================ FILE: src/include/distributed/background_jobs.h ================================================ /*------------------------------------------------------------------------- * * background_jobs.h * Functions related to running the background tasks queue monitor. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_BACKGROUND_JOBS_H #define CITUS_BACKGROUND_JOBS_H #include "postgres.h" #include "postmaster/bgworker.h" #include "distributed/metadata_utility.h" /* * BackgroundExecutorHashEntry hash table entry to refer existing task executors */ typedef struct BackgroundExecutorHashEntry { /* hash key must be the first to hash correctly */ int64 taskid; BackgroundWorkerHandle *handle; dsm_segment *seg; int64 jobid; StringInfo message; } BackgroundExecutorHashEntry; /* * TaskExecutionStatus status for task execution in queue monitor */ typedef enum TaskExecutionStatus { TASK_EXECUTION_STATUS_SUCCESS = 0, TASK_EXECUTION_STATUS_ERROR, TASK_EXECUTION_STATUS_CANCELLED, TASK_EXECUTION_STATUS_RUNNING, TASK_EXECUTION_STATUS_WOULDBLOCK } TaskExecutionStatus; /* * QueueMonitorExecutionContext encapsulates info related to executors and tasks * in queue monitor */ typedef struct QueueMonitorExecutionContext { /* current total # of parallel task executors */ int64 currentExecutorCount; /* map of current executors */ HTAB *currentExecutors; /* last background allocation failure timestamp */ TimestampTz backgroundWorkerFailedStartTime; /* useful to track if all tasks EWOULDBLOCK'd at current iteration */ bool allTasksWouldBlock; /* context for monitor related allocations */ MemoryContext ctx; } QueueMonitorExecutionContext; /* * TaskExecutionContext encapsulates info for currently executed task in queue monitor */ typedef struct TaskExecutionContext { /* active background executor entry */ BackgroundExecutorHashEntry *handleEntry; /* active background task */ BackgroundTask *task; /* context for queue monitor */ QueueMonitorExecutionContext *queueMonitorExecutionContext; } TaskExecutionContext; /* * ParallelTasksPerNodeEntry is the struct used * to track the number of concurrent background tasks that * involve a particular node (the key to the entry) */ typedef struct ParallelTasksPerNodeEntry { /* Used as hash key. */ int32 node_id; /* number of concurrent background tasks that involve node node_id */ uint32 counter; } ParallelTasksPerNodeEntry; extern BackgroundWorkerHandle * StartCitusBackgroundTaskQueueMonitor(Oid database, Oid extensionOwner); extern PGDLLEXPORT void CitusBackgroundTaskQueueMonitorMain(Datum arg); extern PGDLLEXPORT void CitusBackgroundTaskExecutor(Datum main_arg); extern Datum citus_job_cancel(PG_FUNCTION_ARGS); extern Datum citus_job_wait(PG_FUNCTION_ARGS); extern Datum citus_task_wait(PG_FUNCTION_ARGS); extern void citus_job_wait_internal(int64 jobid, BackgroundJobStatus *desiredStatus); extern void citus_task_wait_internal(int64 taskid, BackgroundTaskStatus *desiredStatus); extern bool IncrementParallelTaskCountForNodesInvolved(BackgroundTask *task); #endif /*CITUS_BACKGROUND_JOBS_H */ ================================================ FILE: src/include/distributed/background_worker_utils.h ================================================ /*------------------------------------------------------------------------- * * background_worker_utils.h * Common utilities for initializing PostgreSQL background workers * used by Citus distributed infrastructure. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef BACKGROUND_WORKER_UTILS_H #define BACKGROUND_WORKER_UTILS_H #include "postgres.h" #include "postmaster/bgworker.h" /* * Background worker configuration parameters */ typedef struct CitusBackgroundWorkerConfig { /* Worker identification */ const char *workerName; const char *functionName; const char *workerType; /* Worker parameters */ Datum mainArg; Oid extensionOwner; /* Worker behavior flags */ bool needsNotification; bool waitForStartup; int restartTime; /* Worker timing */ BgWorkerStartTime startTime; /* Optional extra data */ const void *extraData; size_t extraDataSize; } CitusBackgroundWorkerConfig; /* Default configuration values */ #define CITUS_BGW_DEFAULT_RESTART_TIME 5 #define CITUS_BGW_NEVER_RESTART BGW_NEVER_RESTART #define CITUS_BGW_DEFAULT_START_TIME BgWorkerStart_ConsistentState /* Function declarations */ extern BackgroundWorkerHandle * RegisterCitusBackgroundWorker(const CitusBackgroundWorkerConfig *config); extern void InitializeCitusBackgroundWorker(BackgroundWorker *worker, const CitusBackgroundWorkerConfig *config); #endif /* BACKGROUND_WORKER_UTILS_H */ ================================================ FILE: src/include/distributed/cancel_utils.h ================================================ /*------------------------------------------------------------------------- * * cencel_utils.h * Utilities related to query cancellation * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CANCEL_UTILS_H #define CANCEL_UTILS_H extern bool IsHoldOffCancellationReceived(void); #endif /* CANCEL_UTILS_H */ ================================================ FILE: src/include/distributed/causal_clock.h ================================================ /* * causal_clock.h * * Data structure definitions for managing hybrid logical clock and * related function declarations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CAUSAL_CLOCK_H #define CAUSAL_CLOCK_H #include "distributed/type_utils.h" /* * Clock components - Unsigned 64 bit * Logical clock (LC): 42 bits * Counter (C): 22 bits * * 2^42 milliseconds - 4398046511104 milliseconds, which is ~139 years. * 2^22 ticks - maximum of four million operations per millisecond. * */ #define LOGICAL_BITS 42 #define COUNTER_BITS 22 #define LOGICAL_MASK ((1U << COUNTER_BITS) - 1) #define MAX_LOGICAL ((1LU << LOGICAL_BITS) - 1) #define MAX_COUNTER LOGICAL_MASK extern bool EnableClusterClock; extern void LogicalClockShmemInit(void); extern size_t LogicalClockShmemSize(void); extern void InitializeClusterClockMem(void); extern ClusterClock * GetEpochTimeAsClock(void); #endif /* CAUSAL_CLOCK_H */ ================================================ FILE: src/include/distributed/citus_acquire_lock.h ================================================ /*------------------------------------------------------------------------- * * citus_acquire_lock.h * Background worker to help with acquiering locks by canceling competing backends. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_ACQUIRE_LOCK_H #define CITUS_ACQUIRE_LOCK_H #include "postmaster/bgworker.h" BackgroundWorkerHandle * StartLockAcquireHelperBackgroundWorker(int backendToHelp, int32 lock_cooldown); #endif /* CITUS_ACQUIRE_LOCK_H */ ================================================ FILE: src/include/distributed/citus_clauses.h ================================================ /*------------------------------------------------------------------------- * * citus_clauses.h * Routines roughly equivalent to postgres' util/clauses. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_CLAUSES_H #define CITUS_CLAUSES_H #include "nodes/execnodes.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" /* * CoordinatorEvaluationMode is used to signal what expressions in the query * should be evaluated on the coordinator. */ typedef enum CoordinatorEvaluationMode { /* evaluate nothing */ EVALUATE_NONE = 0, /* evaluate only external parameters */ EVALUATE_PARAMS, /* evaluate both the functions/expressions and the external paramaters */ EVALUATE_FUNCTIONS_PARAMS } CoordinatorEvaluationMode; /* * This struct is used to pass information to master * evaluation logic. */ typedef struct CoordinatorEvaluationContext { PlanState *planState; CoordinatorEvaluationMode evaluationMode; } CoordinatorEvaluationContext; extern bool RequiresCoordinatorEvaluation(Query *query); extern void ExecuteCoordinatorEvaluableExpressions(Query *query, PlanState *planState); extern Node * PartiallyEvaluateExpression(Node *expression, CoordinatorEvaluationContext * coordinatorEvaluationContext); extern bool CitusIsVolatileFunction(Node *node); extern bool CitusIsMutableFunction(Node *node); #endif /* CITUS_CLAUSES_H */ ================================================ FILE: src/include/distributed/citus_custom_scan.h ================================================ /*------------------------------------------------------------------------- * * citus_custom_scan.h * Export all custom scan and custom exec methods. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_CUSTOM_SCAN_H #define CITUS_CUSTOM_SCAN_H #include "executor/execdesc.h" #include "nodes/plannodes.h" #include "distributed/distributed_planner.h" #include "distributed/multi_server_executor.h" typedef struct CitusScanState { CustomScanState customScanState; /* underlying custom scan node */ /* function that gets called before postgres starts its execution */ bool finishedPreScan; /* flag to check if the pre scan is finished */ void (*PreExecScan)(struct CitusScanState *scanState); DistributedPlan *distributedPlan; /* distributed execution plan */ MultiExecutorType executorType; /* distributed executor type */ bool finishedRemoteScan; /* flag to check if remote scan is finished */ Tuplestorestate *tuplestorestate; /* tuple store to store distributed results */ } CitusScanState; /* custom scan methods for all executors */ extern CustomScanMethods AdaptiveExecutorCustomScanMethods; extern CustomScanMethods NonPushableInsertSelectCustomScanMethods; extern CustomScanMethods DelayedErrorCustomScanMethods; extern CustomScanMethods NonPushableMergeCommandCustomScanMethods; extern void RegisterCitusCustomScanMethods(void); extern void CitusExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es); extern TupleDesc ScanStateGetTupleDescriptor(CitusScanState *scanState); extern EState * ScanStateGetExecutorState(CitusScanState *scanState); extern CustomScan * FetchCitusCustomScanIfExists(Plan *plan); extern bool IsCitusPlan(Plan *plan); extern bool IsCitusCustomScan(Plan *plan); extern void SetJobColocationId(Job *job); #endif /* CITUS_CUSTOM_SCAN_H */ ================================================ FILE: src/include/distributed/citus_depended_object.h ================================================ /*------------------------------------------------------------------------- * * citus_depended_object.h * Exposes functions related to hiding citus depended objects while executing * postgres vanilla tests. * * Copyright (c) CitusDependent Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_DEPENDED_OBJECT_H #define CITUS_DEPENDED_OBJECT_H #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "distributed/commands.h" extern bool HideCitusDependentObjects; /* DistOpsValidationState to be used to determine validity of dist ops */ typedef enum DistOpsValidationState { HasAtLeastOneValidObject, HasNoneValidObject, HasObjectWithInvalidOwnership, NoAddressResolutionRequired, ShouldQualifyAfterLocalCreation } DistOpsValidationState; extern void SetLocalClientMinMessagesIfRunningPGTests(int clientMinMessageLevel); extern void SetLocalHideCitusDependentObjectsDisabledWhenAlreadyEnabled(void); extern bool HideCitusDependentObjectsOnQueriesOfPgMetaTables(Node *node, void *context); extern bool IsPgLocksTable(RangeTblEntry *rte); extern DistOpsValidationState DistOpsValidityState(Node *node, const DistributeObjectOps *ops); extern bool DistOpsInValidState(DistOpsValidationState distOpsValidationState); #endif /* CITUS_DEPENDED_OBJECT_H */ ================================================ FILE: src/include/distributed/citus_nodefuncs.h ================================================ /*------------------------------------------------------------------------- * * citus_nodefuncs.h * Node (de-)serialization support for Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_NODEFUNCS_H #define CITUS_NODEFUNCS_H #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "distributed/multi_physical_planner.h" /* citus_nodefuncs.c */ extern void SetRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSchemaName, char *fragmentTableName, List *tableIdList, List *funcColumnNames, List *funcColumnTypes, List *funcColumnTypeMods, List *funcCollations); extern void ModifyRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind rteKind, char *fragmentSchemaName, char *fragmentTableName, List *tableIdList); extern void ExtractRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind *rteKind, char **fragmentSchemaName, char **fragmentTableName, List **tableIdList); extern CitusRTEKind GetRangeTblKind(RangeTblEntry *rte); extern void RegisterNodes(void); #define READFUNC_ARGS struct ExtensibleNode *node #define OUTFUNC_ARGS StringInfo str, const struct ExtensibleNode *raw_node #define COPYFUNC_ARGS struct ExtensibleNode *target_node, const struct \ ExtensibleNode *source_node extern void ReadUnsupportedCitusNode(READFUNC_ARGS); extern void OutJob(OUTFUNC_ARGS); extern void OutDistributedPlan(OUTFUNC_ARGS); extern void OutDistributedSubPlan(OUTFUNC_ARGS); extern void OutUsedDistributedSubPlan(OUTFUNC_ARGS); extern void OutShardInterval(OUTFUNC_ARGS); extern void OutMapMergeJob(OUTFUNC_ARGS); extern void OutShardPlacement(OUTFUNC_ARGS); extern void OutRelationShard(OUTFUNC_ARGS); extern void OutRelationRowLock(OUTFUNC_ARGS); extern void OutTask(OUTFUNC_ARGS); extern void OutLocalPlannedStatement(OUTFUNC_ARGS); extern void OutDeferredErrorMessage(OUTFUNC_ARGS); extern void OutGroupShardPlacement(OUTFUNC_ARGS); extern void OutMultiNode(OUTFUNC_ARGS); extern void OutMultiTreeRoot(OUTFUNC_ARGS); extern void OutMultiProject(OUTFUNC_ARGS); extern void OutMultiCollect(OUTFUNC_ARGS); extern void OutMultiSelect(OUTFUNC_ARGS); extern void OutMultiTable(OUTFUNC_ARGS); extern void OutMultiJoin(OUTFUNC_ARGS); extern void OutMultiPartition(OUTFUNC_ARGS); extern void OutMultiCartesianProduct(OUTFUNC_ARGS); extern void OutMultiExtendedOp(OUTFUNC_ARGS); extern void OutTableDDLCommand(OUTFUNC_ARGS); extern void CopyNodeJob(COPYFUNC_ARGS); extern void CopyNodeDistributedPlan(COPYFUNC_ARGS); extern void CopyNodeDistributedSubPlan(COPYFUNC_ARGS); extern void CopyNodeUsedDistributedSubPlan(COPYFUNC_ARGS); extern void CopyNodeShardInterval(COPYFUNC_ARGS); extern void CopyNodeMapMergeJob(COPYFUNC_ARGS); extern void CopyNodeShardPlacement(COPYFUNC_ARGS); extern void CopyNodeGroupShardPlacement(COPYFUNC_ARGS); extern void CopyNodeRelationShard(COPYFUNC_ARGS); extern void CopyNodeRelationRowLock(COPYFUNC_ARGS); extern void CopyNodeTask(COPYFUNC_ARGS); extern void CopyNodeLocalPlannedStatement(COPYFUNC_ARGS); extern void CopyNodeTaskQuery(COPYFUNC_ARGS); extern void CopyNodeDeferredErrorMessage(COPYFUNC_ARGS); #endif /* CITUS_NODEFUNCS_H */ ================================================ FILE: src/include/distributed/citus_nodes.h ================================================ /*------------------------------------------------------------------------- * * citus_nodes.h * Additional node types, and related infrastructure, for Citus. * * To add a new node type to Citus, perform the following: * * * Add a new CitusNodeTag value to use as a tag for the node. Add * the node's name at a corresponding offset within the array named * CitusNodeTagNamesD at the top of citus_nodefuncs.c * * * Describe the node in a struct, which must have a CitusNode as * its first element * * * Implement an 'outfunc' for the node in citus_outfuncs.c, using * the macros defined within that file. This function will handle * converting the node to a string * * * Implement a 'readfunc' for the node in citus_readfuncs.c, using * the macros defined within that file. This function will handle * converting strings into instances of the node * * * Use DEFINE_NODE_METHODS within the nodeMethods array (near the * bottom of citus_nodefuncs.c) to register the node in PostgreSQL * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_NODES_H #define CITUS_NODES_H #include "nodes/extensible.h" /* * Citus Node Tags * * These have to be distinct from the ideas used in postgres' nodes.h * * NOTE: This list must match CitusNodeTagNamesD from citus_nodefuncs.c */ #define CITUS_NODE_TAG_START 1200 typedef enum CitusNodeTag { T_MultiNode = CITUS_NODE_TAG_START, /* FIXME: perhaps use something less predicable? */ T_MultiTreeRoot, T_MultiProject, T_MultiCollect, T_MultiSelect, T_MultiTable, T_MultiJoin, T_MultiPartition, T_MultiCartesianProduct, T_MultiExtendedOp, T_Job, T_MapMergeJob, T_DistributedPlan, T_DistributedSubPlan, T_UsedDistributedSubPlan, T_Task, T_LocalPlannedStatement, T_ShardInterval, T_ShardPlacement, T_RelationShard, T_RelationRowLock, T_DeferredErrorMessage, T_GroupShardPlacement, T_TableDDLCommand } CitusNodeTag; extern const char** CitusNodeTagNames; typedef struct CitusNode { ExtensibleNode extensible; CitusNodeTag citus_tag; /* for quick type determination */ } CitusNode; #define CitusNodeTag(nodeptr) CitusNodeTagI((Node*) nodeptr) static inline int CitusNodeTagI(Node *node) { if (!IsA(node, ExtensibleNode)) { return nodeTag(node); } return ((CitusNode*)(node))->citus_tag; } /* Citus variant of newNode(), don't use directly. */ static inline CitusNode * CitusNewNode(size_t size, CitusNodeTag tag) { CitusNode *result; Assert(size >= sizeof(CitusNode)); /* need the ExtensibleNode and the tag, at least */ result = (CitusNode *) palloc0(size); result->extensible.type = T_ExtensibleNode; result->extensible.extnodename = CitusNodeTagNames[tag - CITUS_NODE_TAG_START]; result->citus_tag = (int) (tag); return result; } /* * IsA equivalent that compares node tags, including Citus-specific nodes. */ #define CitusIsA(nodeptr,_type_) (CitusNodeTag(nodeptr) == T_##_type_) /* * CitusMakeNode is Citus variant of makeNode(). Use it to create nodes of * the types listed in the CitusNodeTag enum and plain NodeTag. Initializes * memory, besides the node tag, to 0. */ #define CitusMakeNode(_type_) ((_type_ *) CitusNewNode(sizeof(_type_),T_##_type_)) #endif /* CITUS_NODES_H */ ================================================ FILE: src/include/distributed/citus_ruleutils.h ================================================ /*------------------------------------------------------------------------- * * citus_ruleutils.h * Citus ruleutils wrapper functions and exported PostgreSQL ruleutils * functions. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_RULEUTILS_H #define CITUS_RULEUTILS_H #include "postgres.h" /* IWYU pragma: keep */ #include "catalog/pg_sequence.h" #include "commands/sequence.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "distributed/coordinator_protocol.h" /* Function declarations for version independent Citus ruleutils wrapper functions */ extern char * pg_get_extensiondef_string(Oid tableRelationId); extern Oid get_extension_schema(Oid ext_oid); extern char * get_extension_version(Oid extensionId); extern char * pg_get_serverdef_string(Oid tableRelationId); extern char * pg_get_sequencedef_string(Oid sequenceRelid); extern Form_pg_sequence pg_get_sequencedef(Oid sequenceRelationId); extern char * pg_get_tableschemadef_string(Oid tableRelationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, char *accessMethod); extern void EnsureRelationKindSupported(Oid relationId); extern char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); extern void deparse_shard_index_statement(IndexStmt *origStmt, Oid distrelid, int64 shardid, StringInfo buffer); extern void deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shardid, StringInfo buffer); extern char * pg_get_indexclusterdef_string(Oid indexRelationId); extern List * pg_get_table_grants(Oid relationId); extern bool contain_nextval_expression_walker(Node *node, void *context); extern char * pg_get_replica_identity_command(Oid tableRelationId); extern List * pg_get_row_level_security_commands(Oid relationId); extern const char * RoleSpecString(RoleSpec *spec, bool withQuoteIdentifier); extern List * ExpandMergedSubscriptingRefEntries(List *targetEntryList); extern char * flatten_reloptions(Oid relid); /* Function declarations for version dependent PostgreSQL ruleutils functions */ extern void pg_get_query_def(Query *query, StringInfo buffer); bool get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList, Oid **mergedNamedArgTypes, List **mergedArgumentList, int *totalArguments); char * pg_get_rule_expr(Node *expression); extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer); extern char * generate_relation_name(Oid relid, List *namespaces); extern char * generate_qualified_relation_name(Oid relid); extern char * generate_operator_name(Oid operid, Oid arg1, Oid arg2); extern List * getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype); extern void AppendOptionListToString(StringInfo stringData, List *options); extern void ensure_update_targetlist_in_param_order(List *targetList); #endif /* CITUS_RULEUTILS_H */ ================================================ FILE: src/include/distributed/citus_safe_lib.h ================================================ /*------------------------------------------------------------------------- * * safe_lib.h * * This file contains helper functions to expand on the _s functions from * safestringlib. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_safe_lib_H #define CITUS_safe_lib_H #include "postgres.h" #include "safe_lib.h" extern void ereport_constraint_handler(const char *message, void *pointer, errno_t error); extern int64 SafeStringToInt64(const char *str); extern int32 SafeStringToInt32(const char *str); extern uint64 SafeStringToUint64(const char *str); extern void SafeQsort(void *ptr, rsize_t count, rsize_t size, int (*comp)(const void *, const void *)); void * SafeBsearch(const void *key, const void *ptr, rsize_t count, rsize_t size, int (*comp)(const void *, const void *)); int SafeSnprintf(char *str, rsize_t count, const char *fmt, ...) pg_attribute_printf(3, 4); #define memset_struct_0(variable) memset(&variable, 0, sizeof(variable)) #endif ================================================ FILE: src/include/distributed/clonenode_utils.h ================================================ #ifndef CLONENODE_UTILS_H #define CLONENODE_UTILS_H #include "distributed/metadata_cache.h" extern int64 GetReplicationLag(WorkerNode *primaryWorkerNode, WorkerNode * replicaWorkerNode); extern void EnsureValidStreamingReplica(WorkerNode *primaryWorkerNode, char * replicaHostname, int replicaPort); extern void EnsureValidCloneMode(WorkerNode *primaryWorkerNode, char *cloneHostname, int clonePort, char *operation); #endif /* CLONE_UTILS_H */ ================================================ FILE: src/include/distributed/colocation_utils.h ================================================ /*------------------------------------------------------------------------- * * colocation_utils.h * * Declarations for public utility functions related to co-located tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COLOCATION_UTILS_H_ #define COLOCATION_UTILS_H_ #include "nodes/pg_list.h" #include "distributed/shardinterval_utils.h" #define INVALID_COLOCATION_ID 0 extern uint32 TableColocationId(Oid distributedTableId); extern bool TablesColocated(Oid leftDistributedTableId, Oid rightDistributedTableId); extern bool ShardsColocated(ShardInterval *leftShardInterval, ShardInterval *rightShardInterval); extern List * ColocatedTableList(Oid distributedTableId); extern List * ColocatedShardIntervalList(ShardInterval *shardInterval); extern List * ColocatedNonPartitionShardIntervalList(ShardInterval *shardInterval); extern Oid ColocatedTableId(int32 colocationId); extern uint32 SingleShardTableColocationNodeId(uint32 colocationId); extern uint64 ColocatedShardIdInRelation(Oid relationId, int shardIndex); uint32 ColocationId(int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation); extern uint32 CreateColocationGroup(int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation); extern void InsertColocationGroupLocally(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumnType, Oid distributionColumnCollation); extern bool IsColocateWithNone(char *colocateWithTableName); extern bool IsColocateWithDefault(char *colocateWithTableName); extern uint32 GetNextColocationId(void); extern void ErrorIfShardPlacementsNotColocated(Oid leftRelationId, Oid rightRelationId); extern void CheckReplicationModel(Oid sourceRelationId, Oid targetRelationId); extern void CheckDistributionColumnType(Oid sourceRelationId, Oid targetRelationId); extern void EnsureColumnTypeEquality(Oid sourceRelationId, Oid targetRelationId, Var *sourceDistributionColumn, Var *targetDistributionColumn); extern void UpdateRelationColocationGroup(Oid distributedRelationId, uint32 colocationId, bool localOnly); extern void DeleteColocationGroupIfNoTablesBelong(uint32 colocationId); extern List * ColocationGroupTableList(uint32 colocationId, uint32 count); extern void DeleteColocationGroup(uint32 colocationId); extern void DeleteColocationGroupLocally(uint32 colocationId); extern uint32 FindColocateWithColocationId(Oid relationId, char replicationModel, Var *distributionColumn, int shardCount, bool shardCountIsStrict, char *colocateWithTableName); extern void EnsureTableCanBeColocatedWith(Oid relationId, char replicationModel, Var *distributionColumn, Oid sourceRelationId); extern void AcquireColocationDefaultLock(void); extern void ReleaseColocationDefaultLock(void); #endif /* COLOCATION_UTILS_H_ */ ================================================ FILE: src/include/distributed/combine_query_planner.h ================================================ /*------------------------------------------------------------------------- * * combine_query_planner.h * Function declarations for building planned statements; these statements * are then executed on the coordinator node. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COMBINE_QUERY_PLANNER_H #define COMBINE_QUERY_PLANNER_H #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "nodes/pathnodes.h" #include "nodes/plannodes.h" /* Function declarations for building local plans on the coordinator node */ struct DistributedPlan; struct CustomScan; extern Path * CreateCitusCustomScanPath(PlannerInfo *root, RelOptInfo *relOptInfo, Index restrictionIndex, RangeTblEntry *rte, CustomScan *remoteScan); extern PlannedStmt * PlanCombineQuery(struct DistributedPlan *distributedPlan, struct CustomScan *dataScan); extern bool FindCitusExtradataContainerRTE(Node *node, RangeTblEntry **result); extern bool ReplaceCitusExtraDataContainer; extern CustomScan *ReplaceCitusExtraDataContainerWithCustomScan; #endif /* COMBINE_QUERY_PLANNER_H */ ================================================ FILE: src/include/distributed/commands/multi_copy.h ================================================ /*------------------------------------------------------------------------- * * multi_copy.h * Declarations for public functions and variables used in COPY for * distributed tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MULTI_COPY_H #define MULTI_COPY_H #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_coerce.h" #include "tcop/dest.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #include "distributed/version_compat.h" #define INVALID_PARTITION_COLUMN_INDEX -1 /* * CitusCopyDest indicates the source or destination of a COPY command. */ typedef enum CitusCopyDest { COPY_FILE, /* to/from file (or a piped program) */ COPY_FRONTEND, /* to frontend */ COPY_CALLBACK /* to/from callback function */ } CitusCopyDest; /* * A smaller version of copy.c's CopyStateData, trimmed to the elements * necessary to copy out results. While it'd be a bit nicer to share code, * it'd require changing core postgres code. */ typedef struct CopyOutStateData { CitusCopyDest copy_dest; /* type of copy source/destination */ StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for * dest == COPY_NEW_FE in COPY FROM */ List *attnumlist; /* integer list of attnums to copy */ int file_encoding; /* file or remote side's character encoding */ bool need_transcoding; /* file encoding diff from server? */ bool binary; /* binary format? */ char *null_print; /* NULL marker string (server encoding!) */ char *null_print_client; /* same converted to file encoding */ char *delim; /* column delimiter (must be 1 byte) */ MemoryContext rowcontext; /* per-row evaluation context */ } CopyOutStateData; typedef struct CopyOutStateData *CopyOutState; /* struct type to keep both hostname and port */ typedef struct NodeAddress { char *nodeName; int32 nodePort; } NodeAddress; /* struct to allow rReceive to coerce tuples before sending them to workers */ typedef struct CopyCoercionData { CoercionPathType coercionType; FmgrInfo coerceFunction; FmgrInfo inputFunction; FmgrInfo outputFunction; Oid typioparam; /* inputFunction has an extra param */ } CopyCoercionData; /* CopyDestReceiver can be used to stream results into a distributed table */ typedef struct CitusCopyDestReceiver { /* public DestReceiver interface */ DestReceiver pub; /* relation and columns to which to copy */ Oid distributedRelationId; List *columnNameList; int partitionColumnIndex; /* open relation handle */ Relation distributedRelation; /* descriptor of the tuples that are sent to the worker */ TupleDesc tupleDescriptor; /* EState for per-tuple memory allocation */ EState *executorState; /* MemoryContext for DestReceiver session */ MemoryContext memoryContext; /* template for COPY statement to send to workers */ CopyStmt *copyStatement; /* * shardId to CopyShardState map. Also used in insert_select_executor.c for * task pruning. */ HTAB *shardStateHash; /* socket to CopyConnectionState map */ HTAB *connectionStateHash; /* state on how to copy out data types */ CopyOutState copyOutState; FmgrInfo *columnOutputFunctions; /* instructions for coercing incoming tuples */ CopyCoercionData *columnCoercionPaths; /* number of tuples sent */ int64 tuplesSent; /* useful for tracking multi shard accesses */ bool multiShardCopy; /* if true, should copy to local placements in the current session */ bool shouldUseLocalCopy; /* * if true, the data from this dest receiver should be published for CDC clients. * This is set tot false for internal transfers like shard split/move/rebalance etc. */ bool isPublishable; /* * Copy into colocated intermediate result. When this is set, the * COPY assumes there are hypothetical colocated shards to the * relation that are files. And, COPY writes the data to the * files as if they are shards. */ char *colocatedIntermediateResultIdPrefix; /* * When copying into append-partitioned tables, the destination shard is chosen * upfront. */ uint64 appendShardId; /* * When copying to intermediate files, we can skip coercions and run them * when merging into the target tables. */ bool skipCoercions; /* * Determines whether the COPY command should track query stat counters. */ bool trackQueryCounters; } CitusCopyDestReceiver; /* GUCs */ extern bool SkipJsonbValidationInCopy; /* managed via GUC, the default is 4MB */ extern int CopySwitchOverThresholdBytes; /* function declarations for copying into a distributed table */ extern CitusCopyDestReceiver * CreateCitusCopyDestReceiver(Oid relationId, List *columnNameList, int partitionColumnIndex, EState *executorState, char *intermediateResultPrefix, bool isPublishable, bool trackQueryCounters); extern FmgrInfo * ColumnOutputFunctions(TupleDesc rowDescriptor, bool binaryFormat); extern bool CanUseBinaryCopyFormat(TupleDesc tupleDescription); extern bool CanUseBinaryCopyFormatForTargetList(List *targetEntryList); extern bool CanUseBinaryCopyFormatForType(Oid typeId); extern void AppendCopyRowData(Datum *valueArray, bool *isNullArray, TupleDesc rowDescriptor, CopyOutState rowOutputState, FmgrInfo *columnOutputFunctions, CopyCoercionData *columnCoercionPaths); extern void AppendCopyBinaryHeaders(CopyOutState headerOutputState); extern void AppendCopyBinaryFooters(CopyOutState footerOutputState); extern void EndRemoteCopy(int64 shardId, List *connectionList); extern List * CreateRangeTable(Relation rel); extern Node * ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletion *completionTag, const char *queryString); extern void CheckCopyPermissions(CopyStmt *copyStatement); extern bool IsCopyResultStmt(CopyStmt *copyStatement); extern void ConversionPathForTypes(Oid inputType, Oid destType, CopyCoercionData *result); extern Datum CoerceColumnValue(Datum inputValue, CopyCoercionData *coercionPath); extern void ReportCopyError(MultiConnection *connection, PGresult *result); #endif /* MULTI_COPY_H */ ================================================ FILE: src/include/distributed/commands/sequence.h ================================================ /*------------------------------------------------------------------------- * * sequence.h * Functions for dealing with sequences * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_SEQUENCE_H #define CITUS_SEQUENCE_H #include "access/attnum.h" #include "nodes/pg_list.h" extern bool ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber); extern void ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList, List **ownedSequenceIdList); #endif /* CITUS_SEQUENCE_H */ ================================================ FILE: src/include/distributed/commands/serialize_distributed_ddls.h ================================================ /*------------------------------------------------------------------------- * * serialize_distributed_ddls.h * * Declarations for public functions related to serializing distributed * DDLs. * *------------------------------------------------------------------------- */ #ifndef SERIALIZE_DDLS_OVER_CATALOG_H #define SERIALIZE_DDLS_OVER_CATALOG_H #include "postgres.h" #include "catalog/dependency.h" /* * Note that those two lock types don't conflict with each other and are * acquired for different purposes. The lock on the object class * --SerializeDistributedDDLsOnObjectClass()-- is used to serialize DDLs * that target the object class itself, e.g., when creating a new object * of that class, and the latter one --SerializeDistributedDDLsOnObjectClassObject()-- * is used to serialize DDLs that target a specific object of that class, * e.g., when altering an object. * * In some cases, we may want to acquire both locks at the same time. For * example, when renaming a database, we want to acquire both lock types * because while the object class lock is used to ensure that another session * doesn't create a new database with the same name, the object lock is used * to ensure that another session doesn't alter the same database. */ extern void SerializeDistributedDDLsOnObjectClass(ObjectClass objectClass); extern void SerializeDistributedDDLsOnObjectClassObject(ObjectClass objectClass, char *qualifiedObjectName); #endif /* SERIALIZE_DDLS_OVER_CATALOG_H */ ================================================ FILE: src/include/distributed/commands/utility_hook.h ================================================ /*------------------------------------------------------------------------- * * utility_hook.h * Citus utility hook and related functionality. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef MULTI_UTILITY_H #define MULTI_UTILITY_H #include "postgres.h" #include "tcop/utility.h" #include "utils/relcache.h" #include "pg_version_constants.h" #include "distributed/coordinator_protocol.h" #include "distributed/function_call_delegation.h" #include "distributed/version_compat.h" #include "distributed/worker_transaction.h" typedef enum { CREATE_OBJECT_PROPAGATION_DEFERRED = 0, CREATE_OBJECT_PROPAGATION_AUTOMATIC = 1, CREATE_OBJECT_PROPAGATION_IMMEDIATE = 2 } CreateObjectPropagationOptions; typedef enum { PROPSETCMD_INVALID = -1, PROPSETCMD_NONE, /* do not propagate SET commands */ PROPSETCMD_LOCAL, /* propagate SET LOCAL commands */ PROPSETCMD_SESSION, /* propagate SET commands, but not SET LOCAL ones */ PROPSETCMD_ALL /* propagate all SET commands */ } PropSetCmdBehavior; extern PropSetCmdBehavior PropagateSetCommands; extern bool EnableDDLPropagation; extern int CreateObjectPropagationMode; extern bool EnableCreateDatabasePropagation; extern bool EnableCreateTypePropagation; extern bool EnableCreateRolePropagation; extern bool EnableAlterRolePropagation; extern bool EnableAlterRoleSetPropagation; extern bool EnableAlterDatabaseOwner; extern int UtilityHookLevel; extern bool InDelegatedProcedureCall; /* * A DDLJob encapsulates the remote tasks and commands needed to process all or * part of a distributed DDL command. It hold the target object's address, * the original DDL command string (for MX DDL propagation), and a task list of * DDL_TASK-type Tasks to be executed. */ typedef struct DDLJob { ObjectAddress targetObjectAddress; /* target distributed object address */ /* * Whether to commit and start a new transaction before sending commands * (only applies to CONCURRENTLY commands). This is needed for REINDEX CONCURRENTLY * and CREATE INDEX CONCURRENTLY on local shards, which would otherwise * get blocked waiting for the current transaction to finish. */ bool startNewTransaction; /* * Command to run in MX nodes when metadata is synced * This is usually the initial (coordinator) DDL command string */ const char *metadataSyncCommand; List *taskList; /* worker DDL tasks to execute */ /* * Only applicable when any of the tasks cannot be executed in a * transaction block. * * Controls whether to emit a warning within the utility hook in case of a * failure. */ bool warnForPartialFailure; } DDLJob; extern ProcessUtility_hook_type PrevProcessUtility; extern void citus_ProcessUtility(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completionTag ); extern void ProcessUtilityParseTree(Node *node, const char *queryString, ProcessUtilityContext context, ParamListInfo params, DestReceiver *dest, QueryCompletion *completionTag ); extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); extern List * DDLTaskList(Oid relationId, const char *commandString); extern List * NontransactionalNodeDDLTaskList(TargetWorkerSet targets, List *commands, bool warnForPartialFailure); extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands); extern bool AlterTableInProgress(void); extern bool DropSchemaOrDBInProgress(void); extern void UndistributeDisconnectedCitusLocalTables(void); extern void NotifyUtilityHookConstraintDropped(void); extern void ResetConstraintDropped(void); extern void ExecuteDistributedDDLJob(DDLJob *ddlJob); extern bool IsDroppedOrGenerated(Form_pg_attribute attr); #endif /* MULTI_UTILITY_H */ ================================================ FILE: src/include/distributed/commands.h ================================================ /*------------------------------------------------------------------------- * * commands.h * Declarations for public functions and variables used for all commands * and DDL operations for citus. All declarations are grouped by the * file that implements them. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_COMMANDS_H #define CITUS_COMMANDS_H #include "postgres.h" #include "nodes/parsenodes.h" #include "tcop/dest.h" #include "tcop/utility.h" #include "utils/acl.h" #include "utils/rel.h" #include "distributed/metadata_utility.h" extern bool AddAllLocalTablesToMetadata; extern bool EnableSchemaBasedSharding; /* controlled via GUC, should be accessed via EnableLocalReferenceForeignKeys() */ extern bool EnableLocalReferenceForeignKeys; /* * GUC that controls whether to allow unique/exclude constraints without * distribution column. */ extern bool AllowUnsafeConstraints; extern bool EnableUnsafeTriggers; extern int MaxMatViewSizeToAutoRecreate; extern bool EnforceLocalObjectRestrictions; extern void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId, char * finalRelationName); extern void SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid parentRelationId, Oid partitionRelationId) ; /* DistOpsOperationType to be used in DistributeObjectOps */ typedef enum DistOpsOperationType { DIST_OPS_NONE, DIST_OPS_CREATE, DIST_OPS_ALTER, DIST_OPS_DROP, } DistOpsOperationType; /* * DistributeObjectOps specifies handlers for node/object type pairs. * Instances of this type should all be declared in deparse.c. * Handlers expect to receive the same Node passed to GetDistributeObjectOps. * Handlers may be NULL. * * Handlers: * deparse: return a string representation of the node. * qualify: replace names in tree with qualified names. * preprocess: executed before standard_ProcessUtility. * postprocess: executed after standard_ProcessUtility. * address: return an ObjectAddress for the subject of the statement. * 2nd parameter is missing_ok, and * 3rd parameter is isPostProcess. * markDistribued: true if the object will be distributed. * * preprocess/postprocess return a List of DDLJobs. */ typedef struct DistributeObjectOps { char *(*deparse)(Node *); void (*qualify)(Node *); List *(*preprocess)(Node *, const char *, ProcessUtilityContext); List *(*postprocess)(Node *, const char *); List *(*address)(Node *, bool, bool); bool markDistributed; /* fields used by common implementations, omitted for specialized implementations */ ObjectType objectType; /* * Points to the varriable that contains the GUC'd feature flag, when turned off the * common propagation functions will not propagate the creation of the object. */ bool *featureFlag; /* specifies the type of the operation */ DistOpsOperationType operationType; } DistributeObjectOps; #define CITUS_TRUNCATE_TRIGGER_NAME "citus_truncate_trigger" const DistributeObjectOps * GetDistributeObjectOps(Node *node); /* functions to support node-wide object management commands from non-main dbs */ extern bool RunPreprocessNonMainDBCommand(Node *parsetree); extern void RunPostprocessNonMainDBCommand(Node *parsetree); /* * Flags that can be passed to GetForeignKeyOids to indicate * which foreign key constraint OIDs are to be extracted */ typedef enum ExtractForeignKeyConstraintsMode { /* extract the foreign key OIDs where the table is the referencing one */ INCLUDE_REFERENCING_CONSTRAINTS = 1 << 0, /* extract the foreign key OIDs the table is the referenced one */ INCLUDE_REFERENCED_CONSTRAINTS = 1 << 1, /* exclude the self-referencing foreign keys */ EXCLUDE_SELF_REFERENCES = 1 << 2, /* any combination of the 5 flags below is supported */ /* include foreign keys when the other table is a distributed table*/ INCLUDE_DISTRIBUTED_TABLES = 1 << 3, /* include foreign keys when the other table is a reference table*/ INCLUDE_REFERENCE_TABLES = 1 << 4, /* include foreign keys when the other table is a citus local table*/ INCLUDE_CITUS_LOCAL_TABLES = 1 << 5, /* include foreign keys when the other table is a Postgres local table*/ INCLUDE_LOCAL_TABLES = 1 << 6, /* include foreign keys when the other table is a single shard table*/ INCLUDE_SINGLE_SHARD_TABLES = 1 << 7, /* include foreign keys regardless of the other table's type */ INCLUDE_ALL_TABLE_TYPES = INCLUDE_DISTRIBUTED_TABLES | INCLUDE_REFERENCE_TABLES | INCLUDE_CITUS_LOCAL_TABLES | INCLUDE_LOCAL_TABLES | INCLUDE_SINGLE_SHARD_TABLES } ExtractForeignKeyConstraintMode; /* * Flags that can be passed to GetForeignKeyIdsForColumn to * indicate whether relationId argument should match: * - referencing relation or, * - referenced relation, * or we are searching for both sides. */ typedef enum SearchForeignKeyColumnFlags { /* relationId argument should match referencing relation */ SEARCH_REFERENCING_RELATION = 1 << 0, /* relationId argument should match referenced relation */ SEARCH_REFERENCED_RELATION = 1 << 1, /* callers can also pass union of above flags */ } SearchForeignKeyColumnFlags; typedef enum TenantOperation { TENANT_UNDISTRIBUTE_TABLE = 0, TENANT_ALTER_TABLE, TENANT_COLOCATE_WITH, TENANT_UPDATE_COLOCATION, TENANT_SET_SCHEMA, } TenantOperation; #define TOTAL_TENANT_OPERATION 5 extern const char *TenantOperationNames[TOTAL_TENANT_OPERATION]; /* begin.c - forward declarations */ extern void SaveBeginCommandProperties(TransactionStmt *transactionStmt); /* cluster.c - forward declarations */ extern List * PreprocessClusterStmt(Node *node, const char *clusterCommand, ProcessUtilityContext processUtilityContext); /* common.c - forward declarations*/ extern List * PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, const char *queryString); extern List * PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString); extern List * PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); /* call.c */ extern bool CallDistributedProcedureRemotely(CallStmt *callStmt, DestReceiver *dest); /* collation.c - forward declarations */ extern char * CreateCollationDDL(Oid collationId); extern List * CreateCollationDDLsIdempotent(Oid collationId); extern List * AlterCollationOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterCollationSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern char * GenerateBackupNameForCollationCollision(const ObjectAddress *address); extern List * DefineCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); /* database.c - forward declarations */ extern List * AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * DatabaseOwnerDDLCommands(const ObjectAddress *address); extern List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * GetDatabaseMetadataSyncCommands(Oid dbOid); extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * GenerateGrantDatabaseCommandList(void); extern List * PreprocessAlterDatabaseRenameStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterDatabaseRenameStmt(Node *node, const char *queryString); extern void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); extern char * CreateDatabaseDDLCommand(Oid dbId); /* domain.c - forward declarations */ extern List * CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * DomainRenameConstraintStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern CreateDomainStmt * RecreateDomainStmt(Oid domainOid); extern Oid get_constraint_typid(Oid conoid); /* extension.c - forward declarations */ extern bool IsDropCitusExtensionStmt(Node *parsetree); extern List * GetDependentFDWsToExtension(Oid extensionId); extern bool IsCreateAlterExtensionUpdateCitusStmt(Node *parsetree); extern void PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree); extern void PreprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern void PostprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern bool ShouldMarkRelationDistributed(Oid relationId); extern void ErrorIfUnstableCreateOrAlterExtensionStmt(Node *parsetree); extern List * PostprocessCreateExtensionStmt(Node *stmt, const char *queryString); extern List * PreprocessDropExtensionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterExtensionSchemaStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterExtensionSchemaStmt(Node *stmt, const char *queryString); extern List * PreprocessAlterExtensionUpdateStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern void PostprocessAlterExtensionCitusUpdateStmt(Node *node); extern List * PreprocessAlterExtensionContentsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * CreateExtensionDDLCommand(const ObjectAddress *extensionAddress); extern List * AlterExtensionSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterExtensionUpdateStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern void CreateExtensionWithVersion(char *extname, char *extVersion); extern void AlterExtensionUpdateStmt(char *extname, char *extVersion); extern int GetExtensionVersionNumber(char *extVersion); /* foreign_constraint.c - forward declarations */ extern bool ConstraintIsAForeignKeyToReferenceTable(char *constraintName, Oid leftRelationId); extern void ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingReplicationModel, char distributionMethod, Var *distributionColumn, uint32 colocationId); extern void EnsureNoFKeyFromTableType(Oid relationId, int tableTypeFlag); extern void EnsureNoFKeyToTableType(Oid relationId, int tableTypeFlag); extern void ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(Oid localTableId); extern bool ColumnReferencedByAnyForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId); extern List * GetReferencingForeignConstaintCommands(Oid relationOid); extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId); extern List * GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId); extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId); extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId); extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags); extern Oid DropFKeysAndUndistributeTable(Oid relationId); extern void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag); extern List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag); extern bool AnyForeignKeyDependsOnIndex(Oid indexId); extern bool HasForeignKeyWithLocalTable(Oid relationId); extern bool HasForeignKeyToReferenceTable(Oid relationOid); extern List * GetForeignKeysFromLocalTables(Oid relationId); extern bool TableReferenced(Oid relationOid); extern bool TableReferencing(Oid relationOid); extern bool ConstraintIsAUniquenessConstraint(char *inputConstaintName, Oid relationId); extern bool ConstraintIsAForeignKey(char *inputConstaintName, Oid relationOid); extern bool ConstraintWithNameIsOfType(char *inputConstaintName, Oid relationId, char targetConstraintType); extern bool ConstraintWithIdIsOfType(Oid constraintId, char targetConstraintType); extern bool TableHasExternalForeignKeys(Oid relationId); extern List * GetForeignKeyOids(Oid relationId, int flags); extern Oid GetReferencedTableId(Oid foreignKeyId); extern Oid GetReferencingTableId(Oid foreignKeyId); extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId); /* foreign_data_wrapper.c - forward declarations */ extern List * PreprocessGrantOnFDWStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern Acl * GetPrivilegesForFDW(Oid FDWOid); /* foreign_server.c - forward declarations */ extern List * PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * GetForeignServerCreateDDLCommand(Oid serverId); /* foreign_table.c - forward declarations */ extern List * PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); /* function.c - forward declarations */ extern List * PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessCreateFunctionStmt(Node *stmt, const char *queryString); extern List * CreateFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * DefineAggregateStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * PreprocessAlterFunctionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * AlterFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterFunctionOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterFunctionSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * PreprocessAlterFunctionDependsStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * AlterFunctionDependsStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); /* grant.c - forward declarations */ extern List * PreprocessGrantStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern void deparsePrivileges(StringInfo privsString, GrantStmt *grantStmt); extern void deparseGrantees(StringInfo granteesString, GrantStmt *grantStmt); /* index.c - forward declarations */ extern bool IsIndexRenameStmt(RenameStmt *renameStmt); extern List * PreprocessIndexStmt(Node *createIndexStatement, const char *createIndexCommand, ProcessUtilityContext processUtilityContext); extern char * ChooseIndexName(const char *tabname, Oid namespaceId, List *colnames, List *exclusionOpNames, bool primary, bool isconstraint); extern char * ChooseIndexNameAddition(List *colnames); extern List * ChooseIndexColumnNames(List *indexElems); extern LOCKMODE GetCreateIndexRelationLockMode(IndexStmt *createIndexStatement); extern List * PreprocessReindexStmt(Node *ReindexStatement, const char *ReindexCommand, ProcessUtilityContext processUtilityContext); extern List * ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * PreprocessDropIndexStmt(Node *dropIndexStatement, const char *dropIndexCommand, ProcessUtilityContext processUtilityContext); extern List * PostprocessIndexStmt(Node *node, const char *queryString); extern void ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement); extern void MarkIndexValid(IndexStmt *indexStmt); extern List * ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcessor, int flags); extern bool IsReindexWithParam_compat(ReindexStmt *stmt, char *paramName); /* objectaddress.c - forward declarations */ extern List * CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); /* owned.c - forward declarations */ extern List * PreprocessDropOwnedStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessReassignOwnedStmt(Node *node, const char *queryString); /* policy.c - forward declarations */ extern List * CreatePolicyCommands(Oid relationId); extern void ErrorIfUnsupportedPolicy(Relation relation); extern void ErrorIfUnsupportedPolicyExpr(Node *expr); extern List * PostprocessCreatePolicyStmt(Node *node, const char *queryString); extern List * PreprocessAlterPolicyStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessDropPolicyStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern bool IsPolicyRenameStmt(RenameStmt *stmt); extern void CreatePolicyEventExtendNames(CreatePolicyStmt *stmt, const char *schemaName, uint64 shardId); extern void AlterPolicyEventExtendNames(AlterPolicyStmt *stmt, const char *schemaName, uint64 shardId); extern void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 shardId); extern void DropPolicyEventExtendNames(DropStmt *stmt, const char *schemaName, uint64 shardId); extern void AddRangeTableEntryToQueryCompat(ParseState *parseState, Relation relation); /* publication.c - forward declarations */ extern List * PostProcessCreatePublicationStmt(Node *node, const char *queryString); extern List * CreatePublicationDDLCommandsIdempotent(const ObjectAddress *address); extern char * CreatePublicationDDLCommand(Oid publicationId); extern List * PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityCtx); extern List * GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd); extern char * GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, bool isAdd); extern List * AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); extern List * AlterPublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); extern List * CreatePublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); extern List * RenamePublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); /* rename.c - forward declarations*/ extern List * PreprocessRenameStmt(Node *renameStmt, const char *renameCommand, ProcessUtilityContext processUtilityContext); extern void ErrorIfUnsupportedRenameStmt(RenameStmt *renameStmt); extern List * PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); /* role.c - forward declarations*/ extern List * PostprocessAlterRoleStmt(Node *stmt, const char *queryString); extern List * PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterRoleRenameStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * GenerateAlterRoleSetCommandForRole(Oid roleid); extern List * AlterRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterRoleSetStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * PreprocessCreateRoleStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessDropRoleStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessGrantRoleStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessGrantRoleStmt(Node *stmt, const char *queryString); extern List * GenerateCreateOrAlterRoleCommand(Oid roleOid); extern List * CreateRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern void UnmarkRolesDistributed(List *roles); extern List * FilterDistributedRoles(List *roles); /* schema.c - forward declarations */ extern List * PostprocessCreateSchemaStmt(Node *node, const char *queryString); extern List * PreprocessDropSchemaStmt(Node *dropSchemaStatement, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterObjectSchemaStmt(Node *alterObjectSchemaStmt, const char *alterObjectSchemaCommand); extern List * PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); /* seclabel.c - forward declarations*/ extern List * PostprocessAnySecLabelStmt(Node *node, const char *queryString); extern List * PostprocessRoleSecLabelStmt(Node *node, const char *queryString); extern List * PostprocessTableOrColumnSecLabelStmt(Node *node, const char *queryString); extern List * SecLabelStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern void citus_test_object_relabel(const ObjectAddress *object, const char *seclabel); /* sequence.c - forward declarations */ extern List * PreprocessAlterSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString); extern List * PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); extern List * PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessDropSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * PreprocessRenameSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessGrantOnSequenceStmt(Node *node, const char *queryString); extern List * AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt); extern void ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt); extern char * GenerateBackupNameForSequenceCollision(const ObjectAddress *address); extern void RenameExistingSequenceWithDifferentTypeIfExists(RangeVar *sequence, Oid desiredSeqTypeId); /* statistics.c - forward declarations */ extern List * PreprocessCreateStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessCreateStatisticsStmt(Node *node, const char *queryString); extern List * CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * PreprocessDropStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * DropStatisticsObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString); extern List * AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * PreprocessAlterStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString); extern List * GetExplicitStatisticsCommandList(Oid relationId); extern List * GetExplicitStatisticsSchemaIdList(Oid relationId); extern List * GetAlterIndexStatisticsCommands(Oid indexOid); /* subscription.c - forward declarations */ extern Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); /* table.c - forward declarations */ extern List * PreprocessDropTableStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern void PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString); extern bool ShouldEnableLocalReferenceForeignKeys(void); extern List * PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, const char *queryString); extern List * PostprocessAlterTableSchemaStmt(Node *node, const char *queryString); extern void PrepareAlterTableStmtForConstraint(AlterTableStmt *alterTableStatement, Oid relationId, Constraint *constraint); extern List * PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt, bool processLocalRelation); extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt); extern void ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement); extern void PostprocessAlterTableStmt(AlterTableStmt *pStmt); extern void FixAlterTableStmtIndexNames(AlterTableStmt *pStmt); extern void ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *command, Constraint *constraint); extern void ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, char referencingReplicationModel, Var *distributionColumn, uint32 colocationId); extern List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, const char *commandString); extern List * AlterTableSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * MakeNameListFromRangeVar(const RangeVar *rel); extern Oid GetSequenceOid(Oid relationId, AttrNumber attnum); extern bool ConstrTypeUsesIndex(ConstrType constrType); extern bool ConstrTypeCitusCanDefaultName(ConstrType constrType); extern char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, char *colname, bool missingTableOk); extern void ErrorIfTableHasIdentityColumn(Oid relationId); extern void ConvertNewTableIfNecessary(Node *createStmt); extern void ConvertToTenantTableIfNecessary(AlterObjectSchemaStmt *alterObjectSchemaStmt); /* text_search.c - forward declarations */ extern List * GetCreateTextSearchConfigStatements(const ObjectAddress *address); extern List * GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); extern List * CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address); extern List * CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address); extern List * CreateTextSearchConfigurationObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * CreateTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * RenameTextSearchConfigurationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * RenameTextSearchDictionaryStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchConfigurationStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchDictionaryStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchConfigurationOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterTextSearchDictOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern char * GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address); extern char * GenerateBackupNameForTextSearchDict(const ObjectAddress *address); extern List * get_ts_config_namelist(Oid tsconfigOid); /* truncate.c - forward declarations */ extern void PreprocessTruncateStatement(TruncateStmt *truncateStatement); /* type.c - forward declarations */ extern List * PreprocessRenameTypeAttributeStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); extern Node * CreateTypeStmtByObjectAddress(const ObjectAddress *address); extern List * CompositeTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * CreateEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * AlterTypeSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameTypeAttributeStmtObjectAddress(Node *stmt, bool missing_ok); extern List * AlterTypeOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress); extern char * GenerateBackupNameForTypeCollision(const ObjectAddress *address); /* function.c - forward declarations */ extern List * CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress); extern char * GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); extern char * GenerateBackupNameForProcCollision(const ObjectAddress *address); extern ObjectWithArgs * ObjectWithArgsFromOid(Oid funcOid); extern void UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, int *distribution_argument_index, int *colocationId, bool *forceDelegation); /* vacuum.c - forward declarations */ extern List * PostprocessVacuumStmt(Node *node, const char *vacuumCommand); /* view.c - forward declarations */ extern List * PreprocessViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessViewStmt(Node *node, const char *queryString); extern List * ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * PreprocessDropViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * DropViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern char * CreateViewDDLCommand(Oid viewOid); extern List * GetViewCreationCommandsOfTable(Oid relationId); extern char * AlterViewOwnerCommand(Oid viewOid); extern char * DeparseViewStmt(Node *node); extern char * DeparseDropViewStmt(Node *node); extern bool IsViewDistributed(Oid viewOid); extern List * CreateViewDDLCommandsIdempotent(Oid viewOid); extern List * PreprocessAlterViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterViewStmt(Node *node, const char *queryString); extern List * PreprocessRenameViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessAlterViewSchemaStmt(Node *node, const char *queryString); extern List * AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern bool IsViewRenameStmt(RenameStmt *renameStmt); /* trigger.c - forward declarations */ extern List * GetExplicitTriggerCommandList(Oid relationId); extern HeapTuple GetTriggerTupleById(Oid triggerId, bool missingOk); extern List * GetExplicitTriggerIdList(Oid relationId); extern List * PostprocessCreateTriggerStmt(Node *node, const char *queryString); extern List * CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern void CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaName, uint64 shardId); extern List * PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); extern void AlterTriggerRenameEventExtendNames(RenameStmt *renameTriggerStmt, char *schemaName, uint64 shardId); extern List * PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString); extern List * PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern void AlterTriggerDependsEventExtendNames(AlterObjectDependsStmt * alterTriggerDependsStmt, char *schemaName, uint64 shardId); extern void ErrorOutForTriggerIfNotSupported(Oid relationId); extern void ErrorIfRelationHasUnsupportedTrigger(Oid relationId); extern List * PreprocessDropTriggerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern void DropTriggerEventExtendNames(DropStmt *dropTriggerStmt, char *schemaName, uint64 shardId); extern List * CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, const char *queryString); extern Oid GetTriggerFunctionId(Oid triggerId); /* cascade_table_operation_for_connected_relations.c */ /* * Flags that can be passed to CascadeOperationForFkeyConnectedRelations, and * CascadeOperationForRelationIdList to specify * citus table function to be executed in cascading mode. */ typedef enum CascadeOperationType { INVALID_OPERATION = 1 << 0, /* execute UndistributeTable on each relation */ CASCADE_FKEY_UNDISTRIBUTE_TABLE = 1 << 1, /* execute CreateCitusLocalTable on each relation, with autoConverted = false */ CASCADE_USER_ADD_LOCAL_TABLE_TO_METADATA = 1 << 2, /* execute CreateCitusLocalTable on each relation, with autoConverted = true */ CASCADE_AUTO_ADD_LOCAL_TABLE_TO_METADATA = 1 << 3, } CascadeOperationType; extern void CascadeOperationForFkeyConnectedRelations(Oid relationId, LOCKMODE relLockMode, CascadeOperationType cascadeOperationType); extern void CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, CascadeOperationType cascadeOperationType); extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList); extern bool RelationIdListHasReferenceTable(List *relationIdList); extern List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList); extern void DropRelationForeignKeys(Oid relationId, int flags); extern void SetLocalEnableLocalReferenceForeignKeys(bool state); extern void ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI(List * utilityCmdList); extern void ExecuteAndLogUtilityCommandList(List *ddlCommandList); extern void ExecuteAndLogUtilityCommand(const char *commandString); extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, bool skip_validation); /* create_citus_local_table.c */ extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConverted); extern bool ShouldAddNewTableToMetadata(Oid relationId); extern List * GetExplicitIndexOidList(Oid relationId); extern bool ShouldPropagateSetCommand(VariableSetStmt *setStmt); extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *setCommand); extern void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, Oid parentRelationId); extern void UpdateAutoConvertedForConnectedRelations(List *relationId, bool autoConverted); /* schema_based_sharding.c */ extern bool ShouldUseSchemaBasedSharding(char *schemaName); extern bool ShouldCreateTenantSchemaTable(Oid relationId); extern void EnsureTenantTable(Oid relationId, char *operationName); extern void ErrorIfIllegalPartitioningInTenantSchema(Oid parentRelationId, Oid partitionRelationId); extern void CreateTenantSchemaTable(Oid relationId); extern void ErrorIfTenantTable(Oid relationId, const char *operationName); extern uint32 CreateTenantSchemaColocationId(void); #endif /*CITUS_COMMANDS_H */ ================================================ FILE: src/include/distributed/comment.h ================================================ /*------------------------------------------------------------------------- * * comment.h * Declarations for comment related operations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COMMENT_H #define COMMENT_H #include "postgres.h" #include "nodes/parsenodes.h" extern const char *ObjectTypeNames[]; extern List * GetCommentPropagationCommands(Oid classOid, Oid oid, char *objectName, ObjectType objectType); extern List * CommentObjectAddress(Node *node, bool missing_ok, bool isPostprocess); # endif /* COMMENT_H */ ================================================ FILE: src/include/distributed/connection_management.h ================================================ /*------------------------------------------------------------------------- * * connection_management.h * Central management of connections and their life-cycle * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CONNECTION_MANAGMENT_H #define CONNECTION_MANAGMENT_H #include "postgres.h" #include "pg_config.h" #include "lib/ilist.h" #include "portability/instr_time.h" #include "storage/latch.h" #include "utils/guc.h" #include "utils/hsearch.h" #include "utils/timestamp.h" #include "distributed/remote_transaction.h" #include "distributed/transaction_management.h" /* maximum (textual) lengths of hostname and port */ #define MAX_NODE_LENGTH 255 /* includes 0 byte */ /* used for libpq commands that get an error buffer. Postgres docs recommend 256. */ #define ERROR_BUFFER_SIZE 256 /* values with special behavior for authinfo lookup */ #define WILDCARD_NODE_ID 0 #define LOCALHOST_NODE_ID -1 /* application name used for internal connections in Citus */ #define CITUS_APPLICATION_NAME_PREFIX "citus_internal gpid=" /* application name used for internal connections in rebalancer */ #define CITUS_REBALANCER_APPLICATION_NAME_PREFIX "citus_rebalancer gpid=" /* application name used for connections made by run_command_on_* */ #define CITUS_RUN_COMMAND_APPLICATION_NAME_PREFIX "citus_run_command gpid=" /* * application name prefix for move/split replication connections. * * This application_name is set to the subscription name by logical replication * workers, so there is no GPID. */ #define CITUS_SHARD_TRANSFER_APPLICATION_NAME_PREFIX "citus_shard_" /* deal with waiteventset errors */ #define WAIT_EVENT_SET_INDEX_NOT_INITIALIZED -1 #define WAIT_EVENT_SET_INDEX_FAILED -2 /* * UINT32_MAX is reserved in pg_dist_node, so we can use it safely. */ #define LOCAL_NODE_ID UINT32_MAX /* forward declare, to avoid forcing large headers on everyone */ struct pg_conn; /* target of the PGconn typedef */ struct MemoryContextData; /* * Flags determining connection establishment behaviour. */ enum MultiConnectionMode { /* force establishment of a new connection */ FORCE_NEW_CONNECTION = 1 << 0, FOR_DDL = 1 << 1, FOR_DML = 1 << 2, /* * During COPY we do not want to use a connection that accessed non-co-located * placements. If there is a connection that did not access another placement, * then use it. Otherwise open a new clean connection. */ REQUIRE_CLEAN_CONNECTION = 1 << 3, OUTSIDE_TRANSACTION = 1 << 4, /* * All metadata changes should go through the same connection, otherwise * self-deadlocks are possible. That is because the same metadata (e.g., * metadata includes the distributed table on the workers) can be modified * accross multiple connections. * * With this flag, we guarantee that there is a single metadata connection. * But note that this connection can be used for any other operation. * In other words, this connection is not exclusively reserved for metadata * operations. */ REQUIRE_METADATA_CONNECTION = 1 << 5, /* * Some connections are optional such as when adaptive executor is executing * a multi-shard command and requires the second (or further) connections * per node. In that case, the connection manager may decide not to allow the * connection. */ OPTIONAL_CONNECTION = 1 << 6, /* * When this flag is passed, via connection throttling, the connection * establishments may be suspended until a connection slot is available to * the remote host. */ WAIT_FOR_CONNECTION = 1 << 7, /* * Use the flag to start a connection for streaming replication. * This flag constructs additional libpq connection parameters needed for streaming * replication protocol. It adds 'replication=database' param which instructs * the backend to go into logical replication walsender mode. * https://www.postgresql.org/docs/current/protocol-replication.html * * This is need to run 'CREATE_REPLICATION_SLOT' command. */ REQUIRE_REPLICATION_CONNECTION_PARAM = 1 << 8 }; /* * This state is used for keeping track of the initialization * of the underlying pg_conn struct. */ typedef enum MultiConnectionState { MULTI_CONNECTION_INITIAL, MULTI_CONNECTION_CONNECTING, MULTI_CONNECTION_CONNECTED, MULTI_CONNECTION_FAILED, MULTI_CONNECTION_LOST, MULTI_CONNECTION_TIMED_OUT } MultiConnectionState; /* * This state is used for keeping track of the initialization * of MultiConnection struct, not specifically the underlying * pg_conn. The state is useful to determine the action during * clean-up of connections. */ typedef enum MultiConnectionStructInitializationState { POOL_STATE_NOT_INITIALIZED, POOL_STATE_COUNTER_INCREMENTED, POOL_STATE_INITIALIZED } MultiConnectionStructInitializationState; /* declaring this directly above causes uncrustify to format it badly */ typedef enum MultiConnectionMode MultiConnectionMode; typedef struct MultiConnection { /* connection details, useful for error messages and such. */ char hostname[MAX_NODE_LENGTH]; int32 port; char user[NAMEDATALEN]; char database[NAMEDATALEN]; /* underlying libpq connection */ struct pg_conn *pgConn; /* connection id */ uint64 connectionId; /* state of the connection */ MultiConnectionState connectionState; /* signal that the connection is ready for read/write */ bool ioReady; /* whether to wait for read/write */ int waitFlags; /* force the connection to be closed at the end of the transaction */ bool forceCloseAtTransactionEnd; /* is the connection currently in use, and shouldn't be used by anything else */ bool claimedExclusively; /* is the replication origin session has already been setup for this connection. */ bool isReplicationOriginSessionSetup; /* * Should be used to access/modify metadata. See REQUIRE_METADATA_CONNECTION for * the details. */ bool useForMetadataOperations; /* time connection establishment was started, for timeout and executor stats */ instr_time connectionEstablishmentStart; instr_time connectionEstablishmentEnd; /* membership in list of connections in ConnectionHashEntry */ dlist_node connectionNode; /* information about the associated remote transaction */ RemoteTransaction remoteTransaction; /* * membership in list of in-progress transactions and a flag to indicate * that the connection was added to this list */ dlist_node transactionNode; bool transactionInProgress; /* list of all placements referenced by this connection */ dlist_head referencedPlacements; /* number of bytes sent to PQputCopyData() since last flush */ uint64 copyBytesWrittenSinceLastFlush; /* replication option */ bool requiresReplication; MultiConnectionStructInitializationState initializationState; } MultiConnection; /* * Central connection management hash, mapping (host, port, user, database) to * a list of connections. * * This hash is used to keep track of which connections are open to which * node. Besides allowing connection reuse, that information is e.g. used to * handle closing connections after the end of a transaction. */ /* hash key */ typedef struct ConnectionHashKey { char hostname[MAX_NODE_LENGTH]; int32 port; char user[NAMEDATALEN]; char database[NAMEDATALEN]; bool replicationConnParam; } ConnectionHashKey; /* hash entry */ typedef struct ConnectionHashEntry { ConnectionHashKey key; dlist_head *connections; /* connections list is valid or not */ bool isValid; } ConnectionHashEntry; /* hash entry for cached connection parameters */ typedef struct ConnParamsHashEntry { ConnectionHashKey key; bool isValid; Index runtimeParamStart; char **keywords; char **values; } ConnParamsHashEntry; /* maximum duration to wait for connection */ extern int NodeConnectionTimeout; /* maximum number of connections to cache per worker per session */ extern int MaxCachedConnectionsPerWorker; /* maximum lifetime of connections in miliseconds */ extern int MaxCachedConnectionLifetime; /* parameters used for outbound connections */ extern char *NodeConninfo; extern char *LocalHostName; extern bool checkAtBootPassed; /* the hash tables are externally accessiable */ extern HTAB *ConnectionHash; extern HTAB *ConnParamsHash; /* context for all connection and transaction related memory */ extern struct MemoryContextData *ConnectionContext; extern void AfterXactConnectionHandling(bool isCommit); extern void InitializeConnectionManagement(void); extern char * GetAuthinfo(char *hostname, int32 port, char *user); extern void InitConnParams(void); extern void ResetConnParams(void); extern void InvalidateConnParamsHashEntries(void); extern void AddConnParam(const char *keyword, const char *value); extern void GetConnParams(ConnectionHashKey *key, char ***keywords, char ***values, Index *runtimeParamStart, MemoryContext context); extern const char * GetConnParam(const char *keyword); extern bool CheckConninfo(const char *conninfo, const char **allowedConninfoKeywords, Size allowedConninfoKeywordsLength, char **errmsg); /* Low-level connection establishment APIs */ extern MultiConnection * GetNodeConnection(uint32 flags, const char *hostname, int32 port); extern MultiConnection * StartNodeConnection(uint32 flags, const char *hostname, int32 port); extern MultiConnection * GetNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port, const char *user, const char *database); extern MultiConnection * GetConnectionForLocalQueriesOutsideTransaction(char *userName); extern MultiConnection * StartNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port, const char *user, const char *database); extern void RestartConnection(MultiConnection *connection); extern void CloseAllConnectionsAfterTransaction(void); extern void CloseNodeConnectionsAfterTransaction(char *nodeName, int nodePort); extern MultiConnection * ConnectionAvailableToNode(char *hostName, int nodePort, const char *userName, const char *database); extern void CloseConnection(MultiConnection *connection); extern void ShutdownAllConnections(void); extern void ShutdownConnection(MultiConnection *connection); /* dealing with a connection */ extern void FinishConnectionListEstablishment(List *multiConnectionList); extern void FinishConnectionEstablishment(MultiConnection *connection); extern void ForceConnectionCloseAtTransactionEnd(MultiConnection *connection); extern void ClaimConnectionExclusively(MultiConnection *connection); extern void UnclaimConnection(MultiConnection *connection); extern void MarkConnectionConnected(MultiConnection *connection, bool newConnection); /* waiteventset utilities */ extern int CitusAddWaitEventSetToSet(WaitEventSet *set, uint32 events, pgsocket fd, Latch *latch, void *user_data); extern bool CitusModifyWaitEvent(WaitEventSet *set, int pos, uint32 events, Latch *latch); /* time utilities */ extern double MillisecondsPassedSince(instr_time moment); extern long MillisecondsToTimeout(instr_time start, long msAfterStart); #endif /* CONNECTION_MANAGMENT_H */ ================================================ FILE: src/include/distributed/coordinator_protocol.h ================================================ /*------------------------------------------------------------------------- * * coordinator_protocol.h * Header for shared declarations for access to coordinator node data. * These data are used to create new shards or update existing ones. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef COORDINATOR_PROTOCOL_H #define COORDINATOR_PROTOCOL_H #include "postgres.h" #include "c.h" #include "fmgr.h" #include "nodes/pg_list.h" #include "columnar/columnar.h" #include "distributed/connection_management.h" #include "distributed/metadata_utility.h" #include "distributed/shardinterval_utils.h" /* * In our distributed database, we need a mechanism to make remote procedure * calls between clients, the coordinator node, and worker nodes. These remote calls * require serializing and deserializing values and function signatures between * nodes; and for these, we currently use PostgreSQL's built-in type and * function definition system. This approach is by no means ideal however; and * our implementation: (i) cannot perform compile-time type checks, (ii) * requires additional effort when upgrading to new function signatures, and * (iii) hides argument and return value names and types behind complicated * pg_proc.h definitions. * * An ideal implementation should overcome these problems, and make it much * easier to pass values back and forth between nodes. One such implementation * that comes close to ideal is Google's Protocol Buffers. Nonetheless, we do * not use it in here as its inclusion requires changes to PostgreSQL's make * system, and a native C version is currently unavailable. */ /* Number of tuple fields that coordinator node functions return */ #define TABLE_METADATA_FIELDS 7 #define CANDIDATE_NODE_FIELDS 2 #define WORKER_NODE_FIELDS 2 /* transfer mode for citus_copy_shard_placement */ #define TRANSFER_MODE_AUTOMATIC 'a' #define TRANSFER_MODE_FORCE_LOGICAL 'l' #define TRANSFER_MODE_BLOCK_WRITES 'b' #define SHARDID_SEQUENCE_NAME "pg_dist_shardid_seq" #define PLACEMENTID_SEQUENCE_NAME "pg_dist_placement_placementid_seq" /* Remote call definitions to help with data staging and deletion */ #define WORKER_APPLY_SHARD_DDL_COMMAND \ "SELECT worker_apply_shard_ddl_command (" UINT64_FORMAT ", %s, %s)" #define WORKER_APPLY_SHARD_DDL_COMMAND_WITHOUT_SCHEMA \ "SELECT worker_apply_shard_ddl_command (" UINT64_FORMAT ", %s)" #define WORKER_APPLY_INTER_SHARD_DDL_COMMAND \ "SELECT worker_apply_inter_shard_ddl_command (" UINT64_FORMAT \ ", %s, " UINT64_FORMAT \ ", %s, %s)" #define SHARD_RANGE_QUERY "SELECT min(%s), max(%s) FROM %s" #define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size(%s)" #define SHARD_CSTORE_TABLE_SIZE_QUERY "SELECT cstore_table_size(%s)" #define DROP_REGULAR_TABLE_COMMAND "DROP TABLE IF EXISTS %s CASCADE" #define DROP_FOREIGN_TABLE_COMMAND "DROP FOREIGN TABLE IF EXISTS %s CASCADE" #define CREATE_SCHEMA_COMMAND "CREATE SCHEMA IF NOT EXISTS %s AUTHORIZATION %s" /* * TableDDLCommandType encodes the implementation used by TableDDLCommand. See comments in * TableDDLCpmmand for details. */ typedef enum TableDDLCommandType { TABLE_DDL_COMMAND_STRING, TABLE_DDL_COMMAND_FUNCTION, } TableDDLCommandType; /* * IndexDefinitionDeparseFlags helps to control which parts of the * index creation commands are deparsed. */ typedef enum IndexDefinitionDeparseFlags { INCLUDE_CREATE_INDEX_STATEMENTS = 1 << 0, INCLUDE_CREATE_CONSTRAINT_STATEMENTS = 1 << 1, INCLUDE_INDEX_CLUSTERED_STATEMENTS = 1 << 2, INCLUDE_INDEX_STATISTICS_STATEMENTTS = 1 << 3, INCLUDE_INDEX_ALL_STATEMENTS = INCLUDE_CREATE_INDEX_STATEMENTS | INCLUDE_CREATE_CONSTRAINT_STATEMENTS | INCLUDE_INDEX_CLUSTERED_STATEMENTS | INCLUDE_INDEX_STATISTICS_STATEMENTTS } IndexDefinitionDeparseFlags; /* * IncludeSequenceDefaults decides on inclusion of DEFAULT clauses for columns * getting their default values from a sequence when creating the definition * of a table. */ typedef enum IncludeSequenceDefaults { NO_SEQUENCE_DEFAULTS = 0, /* don't include sequence defaults */ NEXTVAL_SEQUENCE_DEFAULTS = 1, /* include sequence defaults */ /* * Include sequence defaults, but use worker_nextval instead of nextval * when the default will be called in worker node, and the column type is * int or smallint. */ WORKER_NEXTVAL_SEQUENCE_DEFAULTS = 2, } IncludeSequenceDefaults; /* * IncludeIdentities decides on how we include identity information * when creating the definition of a table. */ typedef enum IncludeIdentities { NO_IDENTITY = 0, /* don't include identities */ INCLUDE_IDENTITY = 1 /* include identities as-is*/ } IncludeIdentities; struct TableDDLCommand; typedef struct TableDDLCommand TableDDLCommand; typedef char *(*TableDDLFunction)(void *context); typedef char *(*TableDDLShardedFunction)(uint64 shardId, void *context); /* * TableDDLCommand holds the definition of a command to be executed to bring the table and * or shard into a certain state. The command needs to be able to serialized into two * versions: * - one version should have the vanilla commands operating on the base table. These are * used for example to create the MX table shards * - the second versions should replace all identifiers with an identifier containing the * shard id. * * Current implementations are * - command string, created via makeTableDDLCommandString. This variant contains a ddl * command that will be wrapped in `worker_apply_shard_ddl_command` when applied * against a shard. */ struct TableDDLCommand { CitusNode node; /* encoding the type this TableDDLCommand contains */ TableDDLCommandType type; /* * This union contains one (1) typed field for every implementation for * TableDDLCommand. A union enforces no overloading of fields but instead requires at * most one of the fields to be used at any time. */ union { /* * CommandStr is used when type is set to TABLE_DDL_COMMAND_STRING. It holds the * sql ddl command string representing the ddl command. */ char *commandStr; /* * function is used when type is set to TABLE_DDL_COMMAND_FUNCTION. It contains * function pointers and a context to be passed to the functions to be able to * construct the sql commands for sharded and non-sharded tables. */ struct { TableDDLFunction function; TableDDLShardedFunction shardedFunction; void *context; } function; }; }; /* * ColumnarTableDDLContext holds the instance variable for the TableDDLCommandFunction * instance described below. */ typedef struct ColumnarTableDDLContext { char *schemaName; char *relationName; ColumnarOptions options; } ColumnarTableDDLContext; /* make functions for TableDDLCommand */ extern TableDDLCommand * makeTableDDLCommandString(char *commandStr); extern TableDDLCommand * makeTableDDLCommandFunction(TableDDLFunction function, TableDDLShardedFunction shardedFunction, void *context); extern char * GetShardedTableDDLCommand(TableDDLCommand *command, uint64 shardId, char *schemaName); extern char * GetShardedTableDDLCommandColumnar(uint64 shardId, void *context); extern char * GetTableDDLCommand(TableDDLCommand *command); extern TableDDLCommand * ColumnarGetCustomTableOptionsDDL(char *schemaName, char *relationName, ColumnarOptions options); /* Config variables managed via guc.c */ extern int ShardCount; extern int ShardReplicationFactor; extern int NextShardId; extern int NextPlacementId; extern bool IsCoordinator(void); /* Function declarations local to the distributed module */ extern uint64 GetNextShardId(void); extern uint64 GetNextPlacementId(void); extern Oid ResolveRelationId(text *relationName, bool missingOk); extern List * GetFullTableCreationCommands(Oid relationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, bool creatingShellTableOnRemoteNode); extern List * GetPostLoadTableCreationCommands(Oid relationId, bool includeIndexes, bool includeReplicaIdentity); extern List * GetPreLoadTableCreationCommands(Oid relationId, IncludeSequenceDefaults includeSequenceDefaults, IncludeIdentities includeIdentityDefaults, char *accessMethod); extern List * GetTableRowLevelSecurityCommands(Oid relationId); extern List * GetTableIndexAndConstraintCommands(Oid relationId, int indexFlags); extern List * GetTableIndexAndConstraintCommandsExcludingReplicaIdentity(Oid relationId, int indexFlags); extern Oid GetRelationIdentityOrPK(Relation rel); extern void GatherIndexAndConstraintDefinitionList(Form_pg_index indexForm, List **indexDDLEventList, int indexFlags); extern bool IndexImpliedByAConstraint(Form_pg_index indexForm); extern List * GetTableReplicaIdentityCommand(Oid relationId); extern char ShardStorageType(Oid relationId); extern bool DistributedTableReplicationIsEnabled(void); extern void CheckDistributedTable(Oid relationId); extern void CreateAppendDistributedShardPlacements(Oid relationId, int64 shardId, List *workerNodeList, int replicationFactor); extern void CreateShardsOnWorkers(Oid distributedRelationId, List *shardPlacements, bool useExclusiveConnection); extern void InsertShardPlacementRows(Oid relationId, int64 shardId, List *workerNodeList, int workerStartIndex, int replicationFactor); extern uint64 UpdateShardStatistics(int64 shardId); extern void CreateShardsWithRoundRobinPolicy(Oid distributedTableId, int32 shardCount, int32 replicationFactor, bool useExclusiveConnections); extern void CreateColocatedShards(Oid targetRelationId, Oid sourceRelationId, bool useExclusiveConnections); extern void CreateReferenceTableShard(Oid distributedTableId); extern void CreateSingleShardTableShardWithRoundRobinPolicy(Oid relationId, uint32 colocationId); extern int EmptySingleShardTableColocationDecideNodeId(uint32 colocationId); extern List * WorkerCreateShardCommandList(Oid relationId, uint64 shardId, List *ddlCommandList); extern Oid ForeignConstraintGetReferencedTableId(const char *queryString); extern void CheckHashPartitionedTable(Oid distributedTableId); extern void CheckTableSchemaNameForDrop(Oid relationId, char **schemaName, char **tableName); extern text * IntegerToText(int32 value); /* Function declarations for generating metadata for shard and placement creation */ extern Datum master_get_table_metadata(PG_FUNCTION_ARGS); extern Datum master_get_table_ddl_events(PG_FUNCTION_ARGS); extern Datum master_get_new_shardid(PG_FUNCTION_ARGS); extern Datum master_get_new_placementid(PG_FUNCTION_ARGS); extern Datum master_get_active_worker_nodes(PG_FUNCTION_ARGS); extern Datum master_get_round_robin_candidate_nodes(PG_FUNCTION_ARGS); extern Datum master_stage_shard_row(PG_FUNCTION_ARGS); extern Datum master_stage_shard_placement_row(PG_FUNCTION_ARGS); /* Function declarations to help with data staging and deletion */ extern Datum master_create_empty_shard(PG_FUNCTION_ARGS); extern Datum master_update_shard_statistics(PG_FUNCTION_ARGS); extern Datum master_drop_sequences(PG_FUNCTION_ARGS); extern Datum master_modify_multiple_shards(PG_FUNCTION_ARGS); extern Datum lock_relation_if_exists(PG_FUNCTION_ARGS); extern Datum citus_drop_all_shards(PG_FUNCTION_ARGS); extern Datum master_drop_all_shards(PG_FUNCTION_ARGS); extern int MasterDropAllShards(Oid relationId, char *schemaName, char *relationName); /* function declarations for shard creation functionality */ extern Datum master_create_worker_shards(PG_FUNCTION_ARGS); extern Datum isolate_tenant_to_new_shard(PG_FUNCTION_ARGS); /* function declarations for shard split functionality */ extern Datum citus_split_shard_by_split_points(PG_FUNCTION_ARGS); /* function declarations for shard copy functinality */ extern List * CopyShardForeignConstraintCommandList(ShardInterval *shardInterval); extern void CopyShardForeignConstraintCommandListGrouped(ShardInterval *shardInterval, List ** colocatedShardForeignConstraintCommandList, List ** referenceTableForeignConstraintList); extern ShardPlacement * SearchShardPlacementInList(List *shardPlacementList, const char *nodeName, uint32 nodePort); extern ShardPlacement * SearchShardPlacementInListOrError(List *shardPlacementList, const char *nodeName, uint32 nodePort); extern char LookupShardTransferMode(Oid shardReplicationModeOid); extern void BlockWritesToShardList(List *shardList); extern List * WorkerApplyShardDDLCommandList(List *ddlCommandList, int64 shardId); #endif /* COORDINATOR_PROTOCOL_H */ ================================================ FILE: src/include/distributed/cte_inline.h ================================================ /*------------------------------------------------------------------------- * * cte_inline.h * Functions and global variables to control cte inlining. * * Copyright (c) 2019, Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CTE_INLINE_H #define CTE_INLINE_H #include "nodes/parsenodes.h" extern void RecursivelyInlineCtesInQueryTree(Query *query); extern bool QueryTreeContainsInlinableCTE(Query *queryTree); #endif /* CTE_INLINE_H */ ================================================ FILE: src/include/distributed/deparse_shard_query.h ================================================ /*------------------------------------------------------------------------- * * deparse_shard_query.h * * Declarations for public functions and types related to deparsing shard * queries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef DEPARSE_SHARD_QUERY_H #define DEPARSE_SHARD_QUERY_H #include "c.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "distributed/citus_custom_scan.h" #include "distributed/query_utils.h" extern void RebuildQueryStrings(Job *workerJob); extern bool UpdateRelationToShardNames(Node *node, List *relationShardList); extern void UpdateWhereClauseToPushdownRecurringOuterJoin(Query *query, List *relationShardList); extern bool UpdateWhereClauseToPushdownRecurringOuterJoinWalker(Node *node, List *relationShardList); Node * CreateQualsForShardInterval(RelationShard *relationShard, int attnum, int outerRtIndex); extern void SetTaskQueryIfShouldLazyDeparse(Task *task, Query *query); extern void SetTaskQueryString(Task *task, char *queryString); extern void SetTaskQueryStringList(Task *task, List *queryStringList); extern void SetTaskQueryPlan(Task *task, Query *query, PlannedStmt *localPlan); extern char * TaskQueryString(Task *task); extern PlannedStmt * TaskQueryLocalPlan(Task *task); extern char * TaskQueryStringAtIndex(Task *task, int index); extern int GetTaskQueryType(Task *task); extern void AddInsertAliasIfNeeded(Query *query); #endif /* DEPARSE_SHARD_QUERY_H */ ================================================ FILE: src/include/distributed/deparser.h ================================================ /*------------------------------------------------------------------------- * * deparser.h * Used when deparsing any ddl parsetree into its sql from. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef CITUS_DEPARSER_H #define CITUS_DEPARSER_H #include "postgres.h" #include "catalog/objectaddress.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" /* forward declarations for format_collate.c */ /* Control flags for FormatCollateExtended, compatible with format_type_extended */ #define FORMAT_COLLATE_ALLOW_INVALID 0x02 /* allow invalid types */ #define FORMAT_COLLATE_FORCE_QUALIFY 0x04 /* force qualification of collate */ extern char * FormatCollateBEQualified(Oid collate_oid); extern char * FormatCollateExtended(Oid collid, bits16 flags); extern void AssertObjectTypeIsFunctional(ObjectType type); extern void QualifyTreeNode(Node *stmt); extern char * DeparseTreeNode(Node *stmt); extern List * DeparseTreeNodes(List *stmts); /* forward declarations for qualify_aggregate_stmts.c */ extern void QualifyDefineAggregateStmt(Node *node); /* forward declarations for deparse_attribute_stmts.c */ extern char * DeparseRenameAttributeStmt(Node *); /* forward declarations for deparse_collation_stmts.c */ extern char * DeparseDropCollationStmt(Node *stmt); extern char * DeparseRenameCollationStmt(Node *stmt); extern char * DeparseAlterCollationSchemaStmt(Node *stmt); extern char * DeparseAlterCollationOwnerStmt(Node *stmt); extern void QualifyDropCollationStmt(Node *stmt); extern void QualifyRenameCollationStmt(Node *stmt); extern void QualifyAlterCollationSchemaStmt(Node *stmt); extern void QualifyAlterCollationOwnerStmt(Node *stmt); /* forward declarations for deparse_domain_stmts.c */ extern char * DeparseCreateDomainStmt(Node *node); extern char * DeparseDropDomainStmt(Node *node); extern char * DeparseAlterDomainStmt(Node *node); extern char * DeparseDomainRenameConstraintStmt(Node *node); extern char * DeparseAlterDomainOwnerStmt(Node *node); extern char * DeparseRenameDomainStmt(Node *node); extern char * DeparseAlterDomainSchemaStmt(Node *node); extern void QualifyCreateDomainStmt(Node *node); extern void QualifyDropDomainStmt(Node *node); extern void QualifyAlterDomainStmt(Node *node); extern void QualifyDomainRenameConstraintStmt(Node *node); extern void QualifyAlterDomainOwnerStmt(Node *node); extern void QualifyRenameDomainStmt(Node *node); extern void QualifyAlterDomainSchemaStmt(Node *node); /* forward declarations for deparse_foreign_data_wrapper_stmts.c */ extern char * DeparseGrantOnFDWStmt(Node *node); /* forward declarations for deparse_foreign_server_stmts.c */ extern char * DeparseCreateForeignServerStmt(Node *node); extern char * DeparseAlterForeignServerStmt(Node *node); extern char * DeparseAlterForeignServerRenameStmt(Node *node); extern char * DeparseAlterForeignServerOwnerStmt(Node *node); extern char * DeparseDropForeignServerStmt(Node *node); extern char * DeparseGrantOnForeignServerStmt(Node *node); /* forward declarations for deparse_table_stmts.c */ extern char * DeparseAlterTableSchemaStmt(Node *stmt); extern char * DeparseAlterTableStmt(Node *node); extern void QualifyAlterTableSchemaStmt(Node *stmt); /* forward declarations for deparse_text_search.c */ extern char * DeparseAlterTextSearchConfigurationOwnerStmt(Node *node); extern char * DeparseAlterTextSearchConfigurationSchemaStmt(Node *node); extern char * DeparseAlterTextSearchConfigurationStmt(Node *node); extern char * DeparseAlterTextSearchDictionaryOwnerStmt(Node *node); extern char * DeparseAlterTextSearchDictionarySchemaStmt(Node *node); extern char * DeparseAlterTextSearchDictionaryStmt(Node *node); extern char * DeparseCreateTextSearchConfigurationStmt(Node *node); extern char * DeparseCreateTextSearchDictionaryStmt(Node *node); extern char * DeparseDropTextSearchConfigurationStmt(Node *node); extern char * DeparseDropTextSearchDictionaryStmt(Node *node); extern char * DeparseRenameTextSearchConfigurationStmt(Node *node); extern char * DeparseRenameTextSearchDictionaryStmt(Node *node); extern char * DeparseTextSearchConfigurationCommentStmt(Node *node); extern char * DeparseTextSearchDictionaryCommentStmt(Node *node); /* forward declarations for deparse_schema_stmts.c */ extern char * DeparseCreateSchemaStmt(Node *node); extern char * DeparseDropSchemaStmt(Node *node); extern char * DeparseGrantOnSchemaStmt(Node *stmt); extern char * DeparseAlterSchemaRenameStmt(Node *stmt); extern char * DeparseAlterSchemaOwnerStmt(Node *node); extern void AppendGrantPrivileges(StringInfo buf, GrantStmt *stmt); extern void AppendGrantGrantees(StringInfo buf, GrantStmt *stmt); extern void AppendWithGrantOption(StringInfo buf, GrantStmt *stmt); extern void AppendGrantOptionFor(StringInfo buf, GrantStmt *stmt); extern void AppendGrantRestrictAndCascadeForRoleSpec(StringInfo buf, DropBehavior behavior, bool isGrant); extern void AppendGrantRestrictAndCascade(StringInfo buf, GrantStmt *stmt); extern void AppendGrantedByInGrantForRoleSpec(StringInfo buf, RoleSpec *grantor, bool isGrant); extern void AppendGrantedByInGrant(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedPrefix(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedSuffix(StringInfo buf, GrantStmt *stmt); extern void AppendColumnNameList(StringInfo buf, List *columns); /* Common deparser utils */ typedef struct DefElemOptionFormat { char *name; char *format; int type; } DefElemOptionFormat; typedef enum OptionFormatType { OPTION_FORMAT_STRING, OPTION_FORMAT_LITERAL_CSTR, OPTION_FORMAT_BOOLEAN, OPTION_FORMAT_INTEGER } OptionFormatType; extern void DefElemOptionToStatement(StringInfo buf, DefElem *option, const DefElemOptionFormat *opt_formats, int opt_formats_len); /* forward declarations for deparse_comment_stmts.c */ extern char * DeparseCommentStmt(Node *node); /* forward declarations for deparse_statistics_stmts.c */ extern char * DeparseCreateStatisticsStmt(Node *node); extern char * DeparseDropStatisticsStmt(List *nameList, bool ifExists); extern char * DeparseAlterStatisticsRenameStmt(Node *node); extern char * DeparseAlterStatisticsSchemaStmt(Node *node); extern char * DeparseAlterStatisticsStmt(Node *node); extern char * DeparseAlterStatisticsOwnerStmt(Node *node); extern void QualifyCreateStatisticsStmt(Node *node); extern void QualifyDropStatisticsStmt(Node *node); extern void QualifyAlterStatisticsRenameStmt(Node *node); extern void QualifyAlterStatisticsSchemaStmt(Node *node); extern void QualifyAlterStatisticsStmt(Node *node); extern void QualifyAlterStatisticsOwnerStmt(Node *node); /* forward declarations for deparse_type_stmts.c */ extern char * DeparseCompositeTypeStmt(Node *stmt); extern char * DeparseCreateEnumStmt(Node *stmt); extern char * DeparseDropTypeStmt(Node *stmt); extern char * DeparseAlterEnumStmt(Node *stmt); extern char * DeparseAlterTypeStmt(Node *stmt); extern char * DeparseRenameTypeStmt(Node *stmt); extern char * DeparseRenameTypeAttributeStmt(Node *stmt); extern char * DeparseAlterTypeSchemaStmt(Node *stmt); extern char * DeparseAlterTypeOwnerStmt(Node *stmt); extern void QualifyRenameAttributeStmt(Node *stmt); extern void QualifyRenameTypeStmt(Node *stmt); extern void QualifyRenameTypeAttributeStmt(Node *stmt); extern void QualifyAlterEnumStmt(Node *stmt); extern void QualifyAlterTypeStmt(Node *stmt); extern void QualifyCompositeTypeStmt(Node *stmt); extern void QualifyCreateEnumStmt(Node *stmt); extern void QualifyAlterTypeSchemaStmt(Node *stmt); extern void QualifyAlterTypeOwnerStmt(Node *stmt); extern char * GetTypeNamespaceNameByNameList(List *names); extern Oid TypeOidGetNamespaceOid(Oid typeOid); extern List * GetObjectAddressListFromParseTree(Node *parseTree, bool missing_ok, bool isPostprocess); extern List * RenameAttributeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); /* forward declarations for deparse_view_stmts.c */ extern void QualifyDropViewStmt(Node *node); extern void QualifyAlterViewStmt(Node *node); extern void QualifyRenameViewStmt(Node *node); extern void QualifyAlterViewSchemaStmt(Node *node); extern char * DeparseRenameViewStmt(Node *stmt); extern char * DeparseAlterViewStmt(Node *node); extern char * DeparseDropViewStmt(Node *node); extern char * DeparseAlterViewSchemaStmt(Node *node); /* forward declarations for deparse_function_stmts.c */ extern bool isFunction(ObjectType objectType); extern char * DeparseDropFunctionStmt(Node *stmt); extern char * DeparseAlterFunctionStmt(Node *stmt); extern char * DeparseRenameFunctionStmt(Node *stmt); extern char * DeparseAlterFunctionSchemaStmt(Node *stmt); extern char * DeparseAlterFunctionOwnerStmt(Node *stmt); extern char * DeparseAlterFunctionDependsStmt(Node *stmt); extern char * DeparseGrantOnFunctionStmt(Node *node); extern void AppendVariableSet(StringInfo buf, VariableSetStmt *setStmt); extern void QualifyAlterFunctionStmt(Node *stmt); extern void QualifyRenameFunctionStmt(Node *stmt); extern void QualifyAlterFunctionSchemaStmt(Node *stmt); extern void QualifyAlterFunctionOwnerStmt(Node *stmt); extern void QualifyAlterFunctionDependsStmt(Node *stmt); /* forward declarations for deparse_role_stmts.c */ extern char * DeparseAlterRoleStmt(Node *stmt); extern char * DeparseAlterRoleSetStmt(Node *stmt); extern char * DeparseRenameRoleStmt(Node *stmt); extern List * MakeSetStatementArguments(char *configurationName, char *configurationValue); extern void QualifyAlterRoleSetStmt(Node *stmt); extern char * DeparseCreateRoleStmt(Node *stmt); extern char * DeparseDropRoleStmt(Node *stmt); extern char * DeparseGrantRoleStmt(Node *stmt); extern char * DeparseReassignOwnedStmt(Node *node); /* forward declarations for deparse_owned_stmts.c */ extern char * DeparseDropOwnedStmt(Node *node); /* forward declarations for deparse_extension_stmts.c */ extern DefElem * GetExtensionOption(List *extensionOptions, const char *defname); extern char * DeparseCreateExtensionStmt(Node *stmt); extern char * DeparseDropExtensionStmt(Node *stmt); extern char * DeparseAlterExtensionSchemaStmt(Node *stmt); extern char * DeparseAlterExtensionStmt(Node *stmt); /* forward declarations for deparse_database_stmts.c */ extern char * DeparseAlterDatabaseOwnerStmt(Node *node); extern char * DeparseGrantOnDatabaseStmt(Node *node); extern char * DeparseAlterDatabaseStmt(Node *node); extern char * DeparseAlterDatabaseRefreshCollStmt(Node *node); extern char * DeparseAlterDatabaseSetStmt(Node *node); extern char * DeparseCreateDatabaseStmt(Node *node); extern char * DeparseDropDatabaseStmt(Node *node); extern char * DeparseAlterDatabaseRenameStmt(Node *node); /* forward declaration for deparse_publication_stmts.c */ extern char * DeparseCreatePublicationStmt(Node *stmt); extern char * DeparseCreatePublicationStmtExtended(Node *node, bool whereClauseNeedsTransform, bool includeLocalTables); extern char * DeparseAlterPublicationStmt(Node *stmt); extern char * DeparseAlterPublicationStmtExtended(Node *stmt, bool whereClauseNeedsTransform, bool includeLocalTables); extern char * DeparseAlterPublicationOwnerStmt(Node *stmt); extern char * DeparseAlterPublicationSchemaStmt(Node *node); extern char * DeparseDropPublicationStmt(Node *stmt); extern char * DeparseRenamePublicationStmt(Node *node); extern void QualifyCreatePublicationStmt(Node *node); extern void QualifyAlterPublicationStmt(Node *node); /* forward declatations for deparse_text_search_stmts.c */ extern void QualifyAlterTextSearchConfigurationOwnerStmt(Node *node); extern void QualifyAlterTextSearchConfigurationSchemaStmt(Node *node); extern void QualifyAlterTextSearchConfigurationStmt(Node *node); extern void QualifyAlterTextSearchDictionaryOwnerStmt(Node *node); extern void QualifyAlterTextSearchDictionarySchemaStmt(Node *node); extern void QualifyAlterTextSearchDictionaryStmt(Node *node); extern void QualifyDropTextSearchConfigurationStmt(Node *node); extern void QualifyDropTextSearchDictionaryStmt(Node *node); extern void QualifyRenameTextSearchConfigurationStmt(Node *node); extern void QualifyRenameTextSearchDictionaryStmt(Node *node); extern void QualifyTextSearchConfigurationCommentStmt(Node *node); extern void QualifyTextSearchDictionaryCommentStmt(Node *node); /* forward declarations for deparse_seclabel_stmts.c */ extern char * DeparseRoleSecLabelStmt(Node *node); extern char * DeparseTableSecLabelStmt(Node *node); extern char * DeparseColumnSecLabelStmt(Node *node); /* forward declarations for deparse_sequence_stmts.c */ extern char * DeparseDropSequenceStmt(Node *node); extern char * DeparseRenameSequenceStmt(Node *node); extern char * DeparseAlterSequenceSchemaStmt(Node *node); extern char * DeparseAlterSequenceOwnerStmt(Node *node); extern char * DeparseAlterSequencePersistenceStmt(Node *node); extern char * DeparseGrantOnSequenceStmt(Node *node); /* forward declarations for qualify_sequence_stmt.c */ extern void QualifyRenameSequenceStmt(Node *node); extern void QualifyDropSequenceStmt(Node *node); extern void QualifyAlterSequenceSchemaStmt(Node *node); extern void QualifyAlterSequenceOwnerStmt(Node *node); extern void QualifyAlterSequencePersistenceStmt(Node *node); extern void QualifyGrantOnSequenceStmt(Node *node); #endif /* CITUS_DEPARSER_H */ ================================================ FILE: src/include/distributed/directed_acyclic_graph_execution.h ================================================ /*------------------------------------------------------------------------- * * directed_acyclic_graph_execution.h * Execution logic for directed acyclic graph tasks. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef DIRECTED_ACYCLIC_GRAPH_EXECUTION_H #define DIRECTED_ACYCLIC_GRAPH_EXECUTION_H #include "postgres.h" #include "nodes/pg_list.h" extern void ExecuteTasksInDependencyOrder(List *allTasks, List *excludedTasks, List *jobIds); #endif /* DIRECTED_ACYCLIC_GRAPH_EXECUTION_H */ ================================================ FILE: src/include/distributed/distributed_deadlock_detection.h ================================================ /*------------------------------------------------------------------------- * * distributed_deadlock_detection.h * Type and function declarations used for performing distributed deadlock * detection. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef DISTRIBUTED_DEADLOCK_DETECTION_H #define DISTRIBUTED_DEADLOCK_DETECTION_H #include "postgres.h" #include "access/hash.h" #include "nodes/pg_list.h" #include "distributed/backend_data.h" #include "distributed/listutils.h" #include "distributed/lock_graph.h" #include "distributed/transaction_identifier.h" typedef struct TransactionNode { DistributedTransactionId transactionId; /* list of TransactionNode that this distributed transaction is waiting for */ List *waitsFor; /* backend that is on the initiator node */ PGPROC *initiatorProc; bool transactionVisited; } TransactionNode; /* GUC, determining whether debug messages for deadlock detection sent to LOG */ extern bool LogDistributedDeadlockDetection; extern bool CheckForDistributedDeadlocks(void); extern HTAB * BuildAdjacencyListsForWaitGraph(WaitGraph *waitGraph); extern char * WaitsForToString(List *waitsFor); #endif /* DISTRIBUTED_DEADLOCK_DETECTION_H */ ================================================ FILE: src/include/distributed/distributed_execution_locks.h ================================================ /*------------------------------------------------------------------------- * * distributed_execution_locks.h * Locking Infrastructure for distributed execution. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef DISTRIBUTED_EXECUTION_LOCKS_H #define DISTRIBUTED_EXECUTION_LOCKS_H #include "postgres.h" #include "nodes/pg_list.h" #include "storage/lockdefs.h" #include "distributed/multi_physical_planner.h" extern void AcquireExecutorShardLocksForExecution(RowModifyLevel modLevel, List *taskList); extern void AcquireExecutorShardLocksForRelationRowLockList(List *relationRowLockList); extern bool RequiresConsistentSnapshot(Task *task); extern void AcquireMetadataLocks(List *taskList); extern void LockPartitionsInRelationList(List *relationIdList, LOCKMODE lockmode); extern void LockPartitionRelations(Oid relationId, LOCKMODE lockMode); extern void LockPartitionsForDistributedPlan(DistributedPlan *distributedPlan); #endif /* DISTRIBUTED_EXECUTION_LOCKS_H */ ================================================ FILE: src/include/distributed/distributed_planner.h ================================================ /*------------------------------------------------------------------------- * * distributed_planner.h * General Citus planner code. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef DISTRIBUTED_PLANNER_H #define DISTRIBUTED_PLANNER_H #include "postgres.h" #include "nodes/pathnodes.h" #include "nodes/plannodes.h" #include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" /* values used by jobs and tasks which do not require identifiers */ #define INVALID_JOB_ID 0 #define INVALID_TASK_ID 0 #define CURSOR_OPT_FORCE_DISTRIBUTED 0x080000 /* level of planner calls */ extern int PlannerLevel; /* RouterPlanType is used to determine the router plan to invoke */ typedef enum RouterPlanType { INSERT_SELECT_INTO_CITUS_TABLE, INSERT_SELECT_INTO_LOCAL_TABLE, DML_QUERY, SELECT_QUERY, MERGE_QUERY, REPLAN_WITH_BOUND_PARAMETERS } RouterPlanType; typedef struct RelationRestrictionContext { bool allReferenceTables; List *relationRestrictionList; } RelationRestrictionContext; typedef struct RootPlanParams { PlannerInfo *root; /* * Copy of root->plan_params. root->plan_params is not preserved in * relation_restriction_equivalence, so we need to create a copy. */ List *plan_params; } RootPlanParams; typedef struct RelationRestriction { Index index; Oid relationId; bool citusTable; RangeTblEntry *rte; RelOptInfo *relOptInfo; PlannerInfo *plannerInfo; /* list of RootPlanParams for all outer nodes */ List *outerPlanParamsList; /* list of translated vars, this is copied from postgres since it gets deleted on postgres*/ List *translatedVars; } RelationRestriction; typedef struct JoinRestrictionContext { List *joinRestrictionList; bool hasSemiJoin; bool hasOuterJoin; } JoinRestrictionContext; typedef struct JoinRestriction { JoinType joinType; List *joinRestrictInfoList; PlannerInfo *plannerInfo; Relids innerrelRelids; Relids outerrelRelids; } JoinRestriction; typedef struct FastPathRestrictionContext { bool fastPathRouterQuery; /* * While calculating fastPathRouterQuery, we could sometimes be * able to extract the distribution key value as well (such as when * there are no prepared statements). Could be NULL when the distribution * key contains parameter, so check for it before using. */ Const *distributionKeyValue; /* * Set to true when distKey = Param; in the queryTree */ bool distributionKeyHasParam; /* * Indicates to hold off calling the fast path planner until its * known if the shard is local or not. */ bool delayFastPathPlanning; } FastPathRestrictionContext; struct DistributedPlanningContext; typedef struct PlannerRestrictionContext { RelationRestrictionContext *relationRestrictionContext; JoinRestrictionContext *joinRestrictionContext; /* * When the query is qualified for fast path, we don't have * the RelationRestrictionContext and JoinRestrictionContext * since those are dependent to calling standard_planner. * Instead, we keep this struct to pass some extra information. */ FastPathRestrictionContext *fastPathRestrictionContext; MemoryContext memoryContext; #if PG_VERSION_NUM >= PG_VERSION_18 /* * Enable access to the distributed planning context from * planner hooks called by Postgres. Enables Citus to track * changes made by Postgres to the query tree (such as * expansion of virtual columns) and ensure they are reflected * back to subsequent distributed planning. */ struct DistributedPlanningContext *planContext; #endif } PlannerRestrictionContext; typedef struct RelationShard { CitusNode type; Oid relationId; uint64 shardId; } RelationShard; typedef struct RelationRowLock { CitusNode type; Oid relationId; LockClauseStrength rowLockStrength; } RelationRowLock; /* * Parameters to be set according to range table entries of a query. */ typedef struct RTEListProperties { bool hasPostgresLocalTable; bool hasReferenceTable; bool hasCitusLocalTable; /* includes hash, single-shard, append and range partitioned tables */ bool hasDistributedTable; /* * Effectively, hasDistributedTable is equal to * "hasDistTableWithShardKey || hasSingleShardDistTable". * * We provide below two for the callers that want to know what kind of * distributed tables that given query has references to. */ bool hasDistTableWithShardKey; bool hasSingleShardDistTable; /* union of hasReferenceTable, hasCitusLocalTable and hasDistributedTable */ bool hasCitusTable; bool hasMaterializedView; } RTEListProperties; typedef struct DistributedPlanningContext { /* The parsed query that is given to the planner. It is a slightly modified * to work with the standard_planner */ Query *query; /* A copy of the original parsed query that is given to the planner. This * doesn't contain most of the changes that are made to parse. There's one * that change that is made for non fast path router queries though, which * is the assigning of RTE identities using AssignRTEIdentities. This is * NULL for non distributed plans, since those don't need it. */ Query *originalQuery; /* the cursor options given to the planner */ int cursorOptions; /* the ParamListInfo that is given to the planner */ ParamListInfo boundParams; /* Plan created either by standard_planner or by FastPathPlanner */ PlannedStmt *plan; /* Our custom restriction context */ PlannerRestrictionContext *plannerRestrictionContext; } DistributedPlanningContext; /* * CitusCustomScanPath is injected into the planner during the combine query planning * phase of the logical planner. * * We call out to the standard planner to plan the combine query part for the output of * the logical planner. This makes it easier to implement new sql features into the * logical planner by not having to manually implement the plan creation for the combine * query on the coordinator.. */ typedef struct CitusCustomScanPath { CustomPath custom_path; /* * Custom scan node computed by the citus planner that will produce the tuples for the * path we are injecting during the planning of the combine query */ CustomScan *remoteScan; } CitusCustomScanPath; extern PlannedStmt * distributed_planner(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); /* * Common hint message to workaround using postgres local and citus local tables * in distributed queries */ #define LOCAL_TABLE_SUBQUERY_CTE_HINT \ "Use CTE's or subqueries to select from local tables and use them in joins" extern List * ExtractRangeTableEntryList(Query *query); extern bool NeedsDistributedPlanning(Query *query); extern List * TranslatedVarsForRteIdentity(int rteIdentity); extern struct DistributedPlan * GetDistributedPlan(CustomScan *node); extern void multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo, Index restrictionIndex, RangeTblEntry *rte); extern void multi_get_relation_info_hook(PlannerInfo *root, Oid relationObjectId, bool inhparent, RelOptInfo *rel); extern void multi_join_restriction_hook(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, RelOptInfo *innerrel, JoinType jointype, JoinPathExtraData *extra); extern bool HasUnresolvedExternParamsWalker(Node *expression, ParamListInfo boundParams); extern bool IsModifyCommand(Query *query); extern void EnsurePartitionTableNotReplicated(Oid relationId); extern Node * ResolveExternalParams(Node *inputNode, ParamListInfo boundParams); extern bool IsMultiTaskPlan(struct DistributedPlan *distributedPlan); extern RangeTblEntry * RemoteScanRangeTableEntry(List *columnNameList); extern int GetRTEIdentity(RangeTblEntry *rte); extern bool GetOriginalInh(RangeTblEntry *rte); extern LOCKMODE GetQueryLockMode(Query *query); extern int32 BlessRecordExpression(Expr *expr); extern void DissuadePlannerFromUsingPlan(PlannedStmt *plan); extern PlannedStmt * FinalizePlan(PlannedStmt *localPlan, struct DistributedPlan *distributedPlan); extern bool ContainsSingleShardTable(Query *query); extern RTEListProperties * GetRTEListPropertiesForQuery(Query *query); extern struct DistributedPlan * CreateDistributedPlan(uint64 planId, bool allowRecursivePlanning, Query *originalQuery, Query *query, ParamListInfo boundParams, bool hasUnresolvedParams, PlannerRestrictionContext * plannerRestrictionContext); #endif /* DISTRIBUTED_PLANNER_H */ ================================================ FILE: src/include/distributed/distribution_column.h ================================================ /*------------------------------------------------------------------------- * * distribution_column.h * Type and function declarations used for handling the distribution * column of distributed tables. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef DISTRIBUTION_COLUMN_H #define DISTRIBUTION_COLUMN_H #include "utils/rel.h" /* Remaining metadata utility functions */ extern Var * BuildDistributionKeyFromColumnName(Oid relationId, char *columnName, LOCKMODE lockMode); extern char * ColumnToColumnName(Oid relationId, Node *columnNode); extern Oid ColumnTypeIdForRelationColumnName(Oid relationId, char *columnName); extern void EnsureValidDistributionColumn(Oid relationId, char *columnName); #endif /* DISTRIBUTION_COLUMN_H */ ================================================ FILE: src/include/distributed/enterprise.h ================================================ /*------------------------------------------------------------------------- * * enterprise.h * * Utilities related to enterprise code in the community version. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_ENTERPRISE_H #define CITUS_ENTERPRISE_H #include "postgres.h" #include "fmgr.h" #define NOT_SUPPORTED_IN_COMMUNITY(name) \ PG_FUNCTION_INFO_V1(name); \ Datum name(PG_FUNCTION_ARGS) { \ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \ errmsg(# name "() is only supported on Citus Enterprise"))); \ } #endif ================================================ FILE: src/include/distributed/error_codes.h ================================================ /*------------------------------------------------------------------------- * * error_codes.h * Error codes that are specific to Citus * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_ERROR_CODES_H #define CITUS_ERROR_CODES_H #include "utils/elog.h" #define ERRCODE_CITUS_INTERMEDIATE_RESULT_NOT_FOUND MAKE_SQLSTATE('C', 'I', 'I', 'N', 'F') #endif /* CITUS_ERROR_CODES_H */ ================================================ FILE: src/include/distributed/errormessage.h ================================================ /*------------------------------------------------------------------------- * * errormessage.h * Error handling related support functionality. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef ERRORMESSAGE_H #define ERRORMESSAGE_H #include "c.h" #include "pg_version_compat.h" #include "distributed/citus_nodes.h" typedef struct DeferredErrorMessage { CitusNode tag; int code; const char *message; const char *detail; const char *hint; const char *filename; int linenumber; const char *functionname; } DeferredErrorMessage; /* * DeferredError allocates a deferred error message, that can later be emitted * using RaiseDeferredError(). These error messages can be * serialized/copied/deserialized, i.e. can be embedded in plans and such. */ #define DeferredError(code, message, detail, hint) \ DeferredErrorInternal(code, message, detail, hint, __FILE__, __LINE__, \ __func__) DeferredErrorMessage * DeferredErrorInternal(int code, const char *message, const char *detail, const char *hint, const char *filename, int linenumber, const char *functionname); /* * RaiseDeferredError emits a previously allocated error using the specified * severity. * * The trickery with __builtin_constant_p/pg_unreachable aims to have the * compiler understand that the function will not return if elevel >= ERROR. */ #ifdef HAVE__BUILTIN_CONSTANT_P #define RaiseDeferredError(error, elevel) \ do { \ RaiseDeferredErrorInternal(error, elevel); \ if (__builtin_constant_p(elevel) && (elevel) >= ERROR) { \ pg_unreachable(); } \ } \ while (0) #else /* !HAVE_BUILTIN_CONSTANT_P */ #define RaiseDeferredError(error, elevel) \ do { \ const int elevel_ = (elevel); \ RaiseDeferredErrorInternal(error, elevel_); \ if (elevel_ >= ERROR) { \ pg_unreachable(); } \ } \ while (0) #endif /* HAVE_BUILTIN_CONSTANT_P */ void RaiseDeferredErrorInternal(DeferredErrorMessage *error, int elevel); #endif ================================================ FILE: src/include/distributed/executor_util.h ================================================ /*------------------------------------------------------------------------- * * executor_util.h * Utility functions for executing task lists. * *------------------------------------------------------------------------- */ #ifndef EXECUTOR_UTIL_H #define EXECUTOR_UTIL_H #include "fmgr.h" #include "funcapi.h" #include "access/tupdesc.h" #include "nodes/params.h" #include "nodes/pg_list.h" #include "distributed/multi_physical_planner.h" /* utility functions for dealing with tasks in the executor */ extern bool TaskListModifiesDatabase(RowModifyLevel modLevel, List *taskList); extern bool TaskListRequiresRollback(List *taskList); extern bool TaskListRequires2PC(List *taskList); extern bool TaskListCannotBeExecutedInTransaction(List *taskList); extern bool SelectForUpdateOnReferenceTable(List *taskList); extern bool ReadOnlyTask(TaskType taskType); extern bool ModifiedTableReplicated(List *taskList); extern bool ShouldRunTasksSequentially(List *taskList); /* utility functions for handling parameters in the executor */ extern void ExtractParametersForRemoteExecution(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues); extern void ExtractParametersFromParamList(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues, bool useOriginalCustomTypeOids); /* utility functions for processing tuples in the executor */ extern AttInMetadata * TupleDescGetAttBinaryInMetadata(TupleDesc tupdesc); extern HeapTuple BuildTupleFromBytes(AttInMetadata *attinmeta, fmStringInfo *values); #endif /* EXECUTOR_UTIL_H */ ================================================ FILE: src/include/distributed/extended_op_node_utils.h ================================================ /*------------------------------------------------------------------------- * * extended_op_node_utils.h * General Citus planner code. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef EXTENDED_OP_NODE_UTILS_H_ #define EXTENDED_OP_NODE_UTILS_H_ #include "distributed/multi_logical_planner.h" /* * ExtendedOpNodeProperties is a helper structure that is used to * share the common information among the worker and coordinator extended * op nodes. * * It is designed to be a read-only singleton object per extended op node * generation and processing. */ typedef struct ExtendedOpNodeProperties { bool groupedByDisjointPartitionColumn; bool repartitionSubquery; bool hasNonPartitionColumnDistinctAgg; bool pullDistinctColumns; bool hasWindowFuncs; bool onlyPushableWindowFunctions; bool pullUpIntermediateRows; bool pushDownGroupingAndHaving; /* indicates whether the MultiExtendedOp has a GROUP BY */ bool hasGroupBy; /* indicates whether the MultiExtendedOp has an aggregate on the target list */ bool hasAggregate; } ExtendedOpNodeProperties; extern ExtendedOpNodeProperties BuildExtendedOpNodeProperties(MultiExtendedOp * extendedOpNode, bool hasNonDistributableAggregates); #endif /* EXTENDED_OP_NODE_UTILS_H_ */ ================================================ FILE: src/include/distributed/foreign_key_relationship.h ================================================ /*------------------------------------------------------------------------- * foreign_key_relationship.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef FOREIGN_KEY_RELATIONSHIP_H #define FOREIGN_KEY_RELATIONSHIP_H #include "postgres.h" #include "postgres_ext.h" #include "nodes/primnodes.h" #include "utils/hsearch.h" #include "utils/relcache.h" extern List * GetForeignKeyConnectedRelationIdList(Oid relationId); extern bool ShouldUndistributeCitusLocalTable(Oid relationId); extern List * ReferencedRelationIdList(Oid relationId); extern List * ReferencingRelationIdList(Oid relationId); extern void SetForeignConstraintRelationshipGraphInvalid(void); extern bool OidVisited(HTAB *oidVisitedMap, Oid oid); extern void VisitOid(HTAB *oidVisitedMap, Oid oid); #endif ================================================ FILE: src/include/distributed/function_call_delegation.h ================================================ /* * function_call_delegation.h * Declarations for public functions and variables used to delegate * function calls to worker nodes. * * Copyright (c), Citus Data, Inc. */ #ifndef FUNCTION_CALL_DELEGATION_H #define FUNCTION_CALL_DELEGATION_H #include "postgres.h" #include "distributed/distributed_planner.h" #include "distributed/multi_physical_planner.h" /* * These flags keep track of whether the process is currently in a delegated * function or procedure call. */ extern bool InTopLevelDelegatedFunctionCall; extern bool InDelegatedProcedureCall; PlannedStmt * TryToDelegateFunctionCall(DistributedPlanningContext *planContext); extern void CheckAndResetAllowedShardKeyValueIfNeeded(void); extern bool IsShardKeyValueAllowed(Const *shardKey, uint32 colocationId); #endif /* FUNCTION_CALL_DELEGATION_H */ ================================================ FILE: src/include/distributed/function_utils.h ================================================ /*------------------------------------------------------------------------- * * function_utils.h * Utilities regarding calls to PG functions * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_FUNCTION_UTILS_H #define CITUS_FUNCTION_UTILS_H #include "postgres.h" #include "nodes/execnodes.h" /* Function declaration for getting oid for the given function name */ extern Oid FunctionOid(const char *schemaName, const char *functionName, int argumentCount); extern Oid FunctionOidExtended(const char *schemaName, const char *functionName, int argumentCount, bool missingOK); extern ReturnSetInfo * FunctionCallGetTupleStore1(PGFunction function, Oid functionId, Datum argument); #endif /* CITUS_FUNCTION_UTILS_H */ ================================================ FILE: src/include/distributed/hash_helpers.h ================================================ /*------------------------------------------------------------------------- * hash_helpers.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef HASH_HELPERS_H #define HASH_HELPERS_H #include "postgres.h" #include "utils/hsearch.h" #include "pg_version_constants.h" /* * assert_valid_hash_key2 checks if a type that contains 2 fields contains no * padding bytes. This is necessary to use a type as a hash key with tag_hash. */ #define assert_valid_hash_key2(type, field1, field2) \ StaticAssertDecl( \ sizeof(type) == sizeof(((type) { 0 }).field1) \ + sizeof(((type) { 0 }).field2), \ # type " has padding bytes, but is used as a hash key in a simple hash"); /* * assert_valid_hash_key3 checks if a type that contains 3 fields contains no * padding bytes. This is necessary to use a type as a hash key with tag_hash. */ #define assert_valid_hash_key3(type, field1, field2, field3) \ StaticAssertDecl( \ sizeof(type) == sizeof(((type) { 0 }).field1) \ + sizeof(((type) { 0 }).field2) \ + sizeof(((type) { 0 }).field3), \ # type " has padding bytes, but is used as a hash key in a simple hash"); extern void hash_delete_all(HTAB *htab); /* * foreach_htab - * a convenience macro which loops through a HTAB */ #define foreach_htab(var, status, htab) \ hash_seq_init((status), (htab)); \ for ((var) = hash_seq_search(status); \ (var) != NULL; \ (var) = hash_seq_search(status)) extern void foreach_htab_cleanup(void *var, HASH_SEQ_STATUS *status); extern HTAB * CreateSimpleHashWithNameAndSizeInternal(Size keysize, Size entrysize, char *name, long nelem); /* * CreatesSimpleHash creates a hash table that hash its key using the tag_hash * and stores then entries in the CurrentMemoryContext. * * We use 32 as the initial number of elements that fit into this hash * table. This value seems a reasonable tradeof between two issues: * 1. An empty hashmap shouldn't take up a lot of space * 2. Doing a few inserts shouldn't require growing the hashmap * * NOTE: No performance testing has been performed when choosing this * value. If this ever turns out to be a problem, feel free to do some * performance tests. * * IMPORTANT: Because this uses tag_hash it's required that the keyType * contains no automatic padding bytes, because that will result in tag_hash * returning undefined values. You can check this using assert_valid_hash_keyX. */ #define CreateSimpleHash(keyType, entryType) \ CreateSimpleHashWithNameAndSize(keyType, entryType, \ # entryType "Hash", 32) /* * Same as CreateSimpleHash but allows specifying the name */ #define CreateSimpleHashWithName(keyType, entryType, name) \ CreateSimpleHashWithNameAndSize(keyType, entryType, \ name, 32) /* * CreateSimpleHashWithSize is the same as CreateSimpleHash, but allows * configuring of the amount of elements that initially fit in the hash table. */ #define CreateSimpleHashWithSize(keyType, entryType, size) \ CreateSimpleHashWithNameAndSize(keyType, entryType, \ # entryType "Hash", size) #define CreateSimpleHashWithNameAndSize(keyType, entryType, name, size) \ CreateSimpleHashWithNameAndSizeInternal(sizeof(keyType), \ sizeof(entryType), \ name, size) /* * CreatesSimpleHashSet creates a hash set that hashes its values using the * tag_hash and stores the values in the CurrentMemoryContext. */ #define CreateSimpleHashSet(keyType) \ CreateSimpleHashWithName(keyType, keyType, \ # keyType "HashSet") /* * CreatesSimpleHashSetWithSize creates a hash set that hashes its values using * the tag_hash and stores the values in the CurrentMemoryContext. It allows * specifying its number of elements. */ #define CreateSimpleHashSetWithSize(keyType, size) \ CreateSimpleHashWithNameAndSize(keyType, keyType, # keyType "HashSet", size) /* * CreatesSimpleHashSetWithName creates a hash set that hashes its values using the * tag_hash and stores the values in the CurrentMemoryContext. It allows * specifying its name. */ #define CreateSimpleHashSetWithName(keyType, name) \ CreateSimpleHashWithName(keyType, keyType, name) /* * CreatesSimpleHashSetWithName creates a hash set that hashes its values using the * tag_hash and stores the values in the CurrentMemoryContext. It allows * specifying its name and number of elements. */ #define CreateSimpleHashSetWithNameAndSize(keyType, name, size) \ CreateSimpleHashWithNameAndSize(keyType, keyType, name, size) #endif ================================================ FILE: src/include/distributed/insert_select_executor.h ================================================ /*------------------------------------------------------------------------- * * insert_select_executor.h * * Declarations for public functions and types related to executing * INSERT..SELECT commands. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef INSERT_SELECT_EXECUTOR_H #define INSERT_SELECT_EXECUTOR_H #include "executor/execdesc.h" extern TupleTableSlot * NonPushableInsertSelectExecScan(CustomScanState *node); extern List * BuildColumnNameListFromTargetList(Oid targetRelationId, List *insertTargetList); #endif /* INSERT_SELECT_EXECUTOR_H */ ================================================ FILE: src/include/distributed/insert_select_planner.h ================================================ /*------------------------------------------------------------------------- * * insert_select_planner.h * * Declarations for public functions and types related to planning * INSERT..SELECT commands. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef INSERT_SELECT_PLANNER_H #define INSERT_SELECT_PLANNER_H #include "postgres.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h" #include "distributed/distributed_planner.h" #include "distributed/multi_physical_planner.h" extern bool InsertSelectIntoCitusTable(Query *query); extern bool CheckInsertSelectQuery(Query *query); extern bool InsertSelectIntoLocalTable(Query *query); extern Query * ReorderInsertSelectTargetLists(Query *originalQuery, RangeTblEntry *insertRte, RangeTblEntry *subqueryRte); extern void NonPushableInsertSelectExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es); extern DistributedPlan * CreateInsertSelectPlan(uint64 planId, Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext, ParamListInfo boundParams); extern DistributedPlan * CreateInsertSelectIntoLocalTablePlan(uint64 planId, Query *originalQuery, ParamListInfo boundParams, bool hasUnresolvedParams, PlannerRestrictionContext * plannerRestrictionContext); extern char * InsertSelectResultIdPrefix(uint64 planId); extern bool PlanningInsertSelect(void); extern Query * WrapSubquery(Query *subquery); #endif /* INSERT_SELECT_PLANNER_H */ ================================================ FILE: src/include/distributed/intermediate_result_pruning.h ================================================ /*------------------------------------------------------------------------- * * intermediate_result_pruning.h * Functions for pruning intermediate result broadcasting. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef INTERMEDIATE_RESULT_PRUNING_H #define INTERMEDIATE_RESULT_PRUNING_H #include "distributed/subplan_execution.h" extern bool LogIntermediateResults; extern List * FindSubPlanUsages(DistributedPlan *plan); extern List * FindAllWorkerNodesUsingSubplan(HTAB *intermediateResultsHash, char *resultId); extern HTAB * MakeIntermediateResultHTAB(void); extern void RecordSubplanExecutionsOnNodes(HTAB *intermediateResultsHash, DistributedPlan *distributedPlan); extern IntermediateResultsHashEntry * SearchIntermediateResult(HTAB *resultsHash, char *resultId); #endif /* INTERMEDIATE_RESULT_PRUNING_H */ ================================================ FILE: src/include/distributed/intermediate_results.h ================================================ /*------------------------------------------------------------------------- * * intermediate_results.h * Functions for writing and reading intermediate results. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef INTERMEDIATE_RESULTS_H #define INTERMEDIATE_RESULTS_H #include "fmgr.h" #include "nodes/execnodes.h" #include "nodes/pg_list.h" #include "tcop/dest.h" #include "utils/builtins.h" #include "utils/palloc.h" #include "distributed/commands/multi_copy.h" /* * DistributedResultFragment represents a fragment of a distributed result. */ typedef struct DistributedResultFragment { /* result's id, which can be used by read_intermediate_results(), ... */ char *resultId; /* location of the result */ uint32 nodeId; /* number of rows in the result file */ int rowCount; /* * The fragment contains the rows which match the partitioning method * and partitioning ranges of targetShardId. The shape of each row matches * the schema of the relation to which targetShardId belongs to. */ uint64 targetShardId; /* what is the index of targetShardId in its relation's sorted shard list? */ int targetShardIndex; } DistributedResultFragment; /* * NodePair contains the source and destination node in a NodeToNodeFragmentsTransfer. * It is a separate struct to use it as a key in a hash table. */ typedef struct NodePair { uint32 sourceNodeId; uint32 targetNodeId; } NodePair; /* * NodeToNodeFragmentsTransfer contains all fragments that need to be fetched from * the source node to the destination node in the NodePair. */ typedef struct NodeToNodeFragmentsTransfer { NodePair nodes; List *fragmentList; } NodeToNodeFragmentsTransfer; /* Forward Declarations */ struct CitusTableCacheEntry; /* intermediate_results.c */ extern DestReceiver * CreateRemoteFileDestReceiver(const char *resultId, EState *executorState, List *initialNodeList, bool writeLocalFile); extern DestReceiver * CreatePartitionedResultDestReceiver(int partitionColumnIndex, int partitionCount, CitusTableCacheEntry * shardSearchInfo, DestReceiver ** partitionedDestReceivers, bool lazyStartup, bool allowNullPartitionValues); extern CitusTableCacheEntry * QueryTupleShardSearchInfo(ArrayType *minValuesArray, ArrayType *maxValuesArray, char partitionMethod, Var *partitionColumn); extern void WriteToLocalFile(StringInfo copyData, FileCompat *fileCompat); extern uint64 RemoteFileDestReceiverBytesSent(DestReceiver *destReceiver); extern void SendQueryResultViaCopy(const char *resultId); extern void ReceiveQueryResultViaCopy(const char *resultId); extern void RemoveIntermediateResultsDirectories(void); extern int64 IntermediateResultSize(const char *resultId); extern char * QueryResultFileName(const char *resultId); extern char * CreateIntermediateResultsDirectory(void); extern ArrayType * CreateArrayFromDatums(Datum *datumArray, bool *nullsArray, int datumCount, Oid typeId); /* distributed_intermediate_results.c */ extern List ** RedistributeTaskListResults(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *targetRelation, bool binaryFormat); extern List * PartitionTasklistResults(const char *resultIdPrefix, List *selectTaskList, int partitionColumnIndex, CitusTableCacheEntry *distributionScheme, bool binaryFormat); extern char * QueryStringForFragmentsTransfer(NodeToNodeFragmentsTransfer * fragmentsTransfer); extern void ShardMinMaxValueArrays(ShardInterval **shardIntervalArray, int shardCount, Oid intervalTypeId, ArrayType **minValueArray, ArrayType **maxValueArray); #endif /* INTERMEDIATE_RESULTS_H */ ================================================ FILE: src/include/distributed/jsonbutils.h ================================================ /*------------------------------------------------------------------------- * * jsonbutils.h * * Declarations for public utility functions related to jsonb. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_JSONBUTILS_H #define CITUS_JSONBUTILS_H #include "postgres.h" bool ExtractFieldJsonbDatum(Datum jsonbDoc, const char *fieldName, Datum *result); text * ExtractFieldTextP(Datum jsonbDoc, const char *fieldName); bool ExtractFieldBoolean(Datum jsonbDoc, const char *fieldName, bool defaultValue); int32 ExtractFieldInt32(Datum jsonbDoc, const char *fieldName, int32 defaultValue); #endif /* CITUS_JSONBUTILS_H */ ================================================ FILE: src/include/distributed/listutils.h ================================================ /*------------------------------------------------------------------------- * * listutils.h * * Declarations for public utility functions related to lists. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_LISTUTILS_H #define CITUS_LISTUTILS_H #include "postgres.h" #include "c.h" #include "nodes/pg_list.h" #include "utils/array.h" #include "utils/hsearch.h" #include "pg_version_compat.h" /* * ListCellAndListWrapper stores a list and list cell. * This struct is used for functionContext. When iterating a list * in separate function calls, we need both the list and the current cell. * Therefore this wrapper stores both of them. */ typedef struct ListCellAndListWrapper { List *list; ListCell *listCell; } ListCellAndListWrapper; /* * foreach_declared_ptr - * a convenience macro which loops through a pointer list without needing a * ListCell, just a declared pointer variable to store the pointer of the * cell in. * * How it works: * - A ListCell is declared with the name {var}CellDoNotUse and used * throughout the for loop using ## to concat. * - To assign to var it needs to be done in the condition of the for loop, * because we cannot use the initializer since a ListCell* variable is * declared there. * - || true is used to always enter the loop when cell is not null even if * var is NULL. */ #define foreach_declared_ptr(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst(var ## CellDoNotUse)) || true); \ var ## CellDoNotUse = lnext(l, var ## CellDoNotUse)) /* * foreach_declared_int - * a convenience macro which loops through an int list without needing a * ListCell, just a declared int variable to store the int of the cell in. * For explanation of how it works see foreach_declared_ptr. */ #define foreach_declared_int(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_int(var ## CellDoNotUse)) || true); \ var ## CellDoNotUse = lnext(l, var ## CellDoNotUse)) /* * foreach_declared_oid - * a convenience macro which loops through an oid list without needing a * ListCell, just a declared Oid variable to store the oid of the cell in. * For explanation of how it works see foreach_declared_ptr. */ #define foreach_declared_oid(var, l) \ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_oid(var ## CellDoNotUse)) || true); \ var ## CellDoNotUse = lnext(l, var ## CellDoNotUse)) /* * forboth_ptr - * a convenience macro which loops through two lists of pointers at the same * time, without needing a ListCell. It only needs two declared pointer * variables to store the pointer of each of the two cells in. */ #define forboth_ptr(var1, l1, var2, l2) \ for (ListCell *(var1 ## CellDoNotUse) = list_head(l1), \ *(var2 ## CellDoNotUse) = list_head(l2); \ (var1 ## CellDoNotUse) != NULL && \ (var2 ## CellDoNotUse) != NULL && \ (((var1) = lfirst(var1 ## CellDoNotUse)) || true) && \ (((var2) = lfirst(var2 ## CellDoNotUse)) || true); \ var1 ## CellDoNotUse = lnext(l1, var1 ## CellDoNotUse), \ var2 ## CellDoNotUse = lnext(l2, var2 ## CellDoNotUse) \ ) /* * forboth_ptr_oid - * a convenience macro which loops through two lists at the same time. The * first list should contain pointers and the second list should contain * Oids. It does not need a ListCell to do this. It only needs two declared * variables to store the pointer and the Oid of each of the two cells in. */ #define forboth_ptr_oid(var1, l1, var2, l2) \ for (ListCell *(var1 ## CellDoNotUse) = list_head(l1), \ *(var2 ## CellDoNotUse) = list_head(l2); \ (var1 ## CellDoNotUse) != NULL && \ (var2 ## CellDoNotUse) != NULL && \ (((var1) = lfirst(var1 ## CellDoNotUse)) || true) && \ (((var2) = lfirst_oid(var2 ## CellDoNotUse)) || true); \ var1 ## CellDoNotUse = lnext(l1, var1 ## CellDoNotUse), \ var2 ## CellDoNotUse = lnext(l2, var2 ## CellDoNotUse) \ ) /* * forboth_int_oid - * a convenience macro which loops through two lists at the same time. The * first list should contain integers and the second list should contain * Oids. It does not need a ListCell to do this. It only needs two declared * variables to store the int and the Oid of each of the two cells in. */ #define forboth_int_oid(var1, l1, var2, l2) \ for (ListCell *(var1 ## CellDoNotUse) = list_head(l1), \ *(var2 ## CellDoNotUse) = list_head(l2); \ (var1 ## CellDoNotUse) != NULL && \ (var2 ## CellDoNotUse) != NULL && \ (((var1) = lfirst_int(var1 ## CellDoNotUse)) || true) && \ (((var2) = lfirst_oid(var2 ## CellDoNotUse)) || true); \ var1 ## CellDoNotUse = lnext(l1, var1 ## CellDoNotUse), \ var2 ## CellDoNotUse = lnext(l2, var2 ## CellDoNotUse) \ ) /* * foreach_ptr_append - * a convenience macro which loops through a pointer List and can append list * elements without needing a ListCell or and index variable, just a declared * pointer variable to store the iterated values. * * PostgreSQL 13 changed the representation of Lists to expansible arrays, * not chains of cons-cells. This changes the costs for accessing and * mutating List contents. Therefore different implementations are provided. * * For more information, see postgres commit with sha * 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164 * * How it works: * - An index is declared with the name {var}PositionDoNotUse and used * throughout the for loop using ## to concat. * - To assign to var it needs to be done in the condition of the for loop, * because we cannot use the initializer since the index variable is * declared there. * - || true is used to always enter the loop even if var is NULL. */ #define foreach_ptr_append(var, l) \ for (int var ## PositionDoNotUse = 0; \ (var ## PositionDoNotUse) < list_length(l) && \ (((var) = list_nth(l, var ## PositionDoNotUse)) || true); \ var ## PositionDoNotUse++) /* utility functions declaration shared within this module */ extern List * SortList(List *pointerList, int (*ComparisonFunction)(const void *, const void *)); extern void ** PointerArrayFromList(List *pointerList); extern HTAB * ListToHashSet(List *pointerList, Size keySize, bool isStringList); extern char * StringJoin(List *stringList, char delimiter); extern char * StringJoinParams(List *stringList, char delimiter, char *prefix, char *postfix); extern List * ListTake(List *pointerList, int size); extern void * safe_list_nth(const List *list, int index); extern List * GeneratePositiveIntSequenceList(int upTo); extern List * GenerateListFromElement(void *listElement, int listLength); extern List * list_filter_oid(List *list, bool (*keepElement)(Oid element)); #endif /* CITUS_LISTUTILS_H */ ================================================ FILE: src/include/distributed/local_distributed_join_planner.h ================================================ /*------------------------------------------------------------------------- * * local_distributed_join_planner.h * * Declarations for functions to handle local-distributed table joins. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef LOCAL_DISTRIBUTED_JOIN_PLANNER_H #define LOCAL_DISTRIBUTED_JOIN_PLANNER_H #include "postgres.h" #include "distributed/recursive_planning.h" /* managed via guc.c */ typedef enum { LOCAL_JOIN_POLICY_NEVER = 0, LOCAL_JOIN_POLICY_PREFER_LOCAL = 1, LOCAL_JOIN_POLICY_PREFER_DISTRIBUTED = 2, LOCAL_JOIN_POLICY_AUTO = 3, } LocalJoinPolicy; extern int LocalTableJoinPolicy; extern bool ShouldConvertLocalTableJoinsToSubqueries(List *rangeTableList); extern void RecursivelyPlanLocalTableJoins(Query *query, RecursivePlanningContext *context); extern List * RequiredAttrNumbersForRelation(RangeTblEntry *relationRte, PlannerRestrictionContext * plannerRestrictionContext); extern List * RequiredAttrNumbersForRelationInternal(Query *queryToProcess, int rteIndex); #endif /* LOCAL_DISTRIBUTED_JOIN_PLANNER_H */ ================================================ FILE: src/include/distributed/local_executor.h ================================================ /*------------------------------------------------------------------------- * * local_executor.h * Functions and global variables to control local query execution. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef LOCAL_EXECUTION_H #define LOCAL_EXECUTION_H #include "distributed/citus_custom_scan.h" #include "distributed/tuple_destination.h" /* enabled with GUCs*/ extern bool EnableLocalExecution; extern bool LogLocalCommands; /* global variable that tracks whether the local execution is on a shard */ extern uint64 LocalExecutorShardId; typedef enum LocalExecutionStatus { LOCAL_EXECUTION_REQUIRED, LOCAL_EXECUTION_OPTIONAL, LOCAL_EXECUTION_DISABLED } LocalExecutionStatus; /* extern function declarations */ extern LocalExecutionStatus GetCurrentLocalExecutionStatus(void); extern uint64 ExecuteLocalTaskList(List *taskList, TupleDestination *defaultTupleDest); extern uint64 ExecuteLocalUtilityTaskList(List *utilityTaskList); extern uint64 ExecuteLocalTaskListExtended(List *taskList, ParamListInfo orig_paramListInfo, DistributedPlan *distributedPlan, TupleDestination *defaultTupleDest, bool isUtilityCommand); extern void ExtractLocalAndRemoteTasks(bool readOnlyPlan, List *taskList, List **localTaskList, List **remoteTaskList); extern void ExecuteUtilityCommand(const char *utilityCommand); extern bool ShouldExecuteTasksLocally(List *taskList); extern bool AnyTaskAccessesLocalNode(List *taskList); extern bool TaskAccessesLocalNode(Task *task); extern void EnsureCompatibleLocalExecutionState(List *taskList); extern void ErrorIfTransactionAccessedPlacementsLocally(void); extern void DisableLocalExecution(void); extern void SetLocalExecutionStatus(LocalExecutionStatus newStatus); extern void ExtractParametersForLocalExecution(ParamListInfo paramListInfo, Oid **parameterTypes, const char ***parameterValues); #endif /* LOCAL_EXECUTION_H */ ================================================ FILE: src/include/distributed/local_multi_copy.h ================================================ #ifndef LOCAL_MULTI_COPY #define LOCAL_MULTI_COPY /* * LocalCopyFlushThresholdByte is the threshold for local copy to be flushed. * There will be one buffer for each local placement, when the buffer size * exceeds this threshold, it will be flushed. * * Managed via GUC, the default is 512 kB. */ extern int LocalCopyFlushThresholdByte; extern void WriteTupleToLocalShard(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localCopyOutState); extern void WriteTupleToLocalFile(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localFileCopyOutState, FileCompat *fileCompat); extern void FinishLocalCopyToShard(CitusCopyDestReceiver *copyDest, int64 shardId, CopyOutState localCopyOutState); extern void FinishLocalCopyToFile(CopyOutState localFileCopyOutState, FileCompat *fileCompat); #endif /* LOCAL_MULTI_COPY */ ================================================ FILE: src/include/distributed/local_plan_cache.h ================================================ #ifndef LOCAL_PLAN_CACHE #define LOCAL_PLAN_CACHE extern bool IsLocalPlanCachingSupported(Job *currentJob, DistributedPlan *originalDistributedPlan); extern PlannedStmt * GetCachedLocalPlan(Task *task, DistributedPlan *distributedPlan); extern void CacheLocalPlanForShardQuery(Task *task, DistributedPlan *originalDistributedPlan, ParamListInfo paramListInfo); #endif /* LOCAL_PLAN_CACHE */ ================================================ FILE: src/include/distributed/locally_reserved_shared_connections.h ================================================ /*------------------------------------------------------------------------- * * locally_reserved_shared_connection_stats.h * Management of connection reservations in shard memory pool * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef LOCALLY_RESERVED_SHARED_CONNECTIONS_H_ #define LOCALLY_RESERVED_SHARED_CONNECTIONS_H_ #include "distributed/connection_management.h" extern void InitializeLocallyReservedSharedConnections(void); extern bool CanUseReservedConnection(const char *hostName, int nodePort, Oid userId, Oid databaseOid); extern void MarkReservedConnectionUsed(const char *hostName, int nodePort, Oid userId, Oid databaseOid); extern void DeallocateReservedConnections(void); extern void EnsureConnectionPossibilityForRemotePrimaryNodes(void); extern bool TryConnectionPossibilityForLocalPrimaryNode(void); extern bool IsReservationPossible(void); #endif /* LOCALLY_RESERVED_SHARED_CONNECTIONS_H_ */ ================================================ FILE: src/include/distributed/lock_graph.h ================================================ /* * lock_graph.h * * Data structures and functions for gathering lock graphs between * distributed transactions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef LOCK_GRAPH_H #define LOCK_GRAPH_H #include "postgres.h" #include "libpq-fe.h" #include "datatype/timestamp.h" #include "storage/lock.h" #include "distributed/backend_data.h" /* * Describes an edge in a waiting-for graph of locks. This isn't used for * deadlock-checking directly, but to gather the information necessary to * do so. * * The datatypes here are a bit looser than strictly necessary, because * they're transported as the return type from an SQL function. */ typedef struct WaitEdge { uint64 waitingGPid; int waitingPid; int waitingNodeId; int64 waitingTransactionNum; TimestampTz waitingTransactionStamp; uint64 blockingGPid; int blockingPid; int blockingNodeId; int64 blockingTransactionNum; TimestampTz blockingTransactionStamp; /* blocking transaction is also waiting on a lock */ bool isBlockingXactWaiting; } WaitEdge; /* * WaitGraph represent a graph of wait edges as an adjacency list. */ typedef struct WaitGraph { int localNodeId; int allocatedSize; int edgeCount; WaitEdge *edges; } WaitGraph; extern WaitGraph * BuildGlobalWaitGraph(bool onlyDistributedTx); extern bool IsProcessWaitingForLock(PGPROC *proc); extern bool IsInDistributedTransaction(BackendData *backendData); extern TimestampTz ParseTimestampTzField(PGresult *result, int rowIndex, int colIndex); extern int64 ParseIntField(PGresult *result, int rowIndex, int colIndex); /* some utility function to parse results */ extern int64 ParseIntField(PGresult *result, int rowIndex, int colIndex); extern bool ParseBoolField(PGresult *result, int rowIndex, int colIndex); extern TimestampTz ParseTimestampTzField(PGresult *result, int rowIndex, int colIndex); #endif /* LOCK_GRAPH_H */ ================================================ FILE: src/include/distributed/log_utils.h ================================================ /*------------------------------------------------------------------------- * log_utils.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef LOG_UTILS_H #define LOG_UTILS_H #include "utils/guc.h" /* do not log */ #define CITUS_LOG_LEVEL_OFF 0 extern bool EnableUnsupportedFeatureMessages; extern bool IsLoggableLevel(int logLevel); #undef ereport #define ereport(elevel, rest) \ do { \ int ereport_loglevel = elevel; \ (void) (ereport_loglevel); \ ereport_domain(elevel, TEXTDOMAIN, rest); \ } while (0) #endif /* LOG_UTILS_H */ ================================================ FILE: src/include/distributed/maintenanced.h ================================================ /*------------------------------------------------------------------------- * * maintenanced.h * Background worker run for each citus using database in a postgres * cluster. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MAINTENANCED_H #define MAINTENANCED_H /* collect statistics every 24 hours */ #define STATS_COLLECTION_TIMEOUT_MILLIS (24 * 60 * 60 * 1000) /* if statistics collection fails, retry in 1 minute */ #define STATS_COLLECTION_RETRY_TIMEOUT_MILLIS (60 * 1000) /* config variable for */ extern double DistributedDeadlockDetectionTimeoutFactor; extern char *MainDb; extern void StopMaintenanceDaemon(Oid databaseId); extern void TriggerNodeMetadataSync(Oid databaseId); extern void InitializeMaintenanceDaemon(void); extern size_t MaintenanceDaemonShmemSize(void); extern void MaintenanceDaemonShmemInit(void); extern void InitializeMaintenanceDaemonBackend(void); extern void InitializeMaintenanceDaemonForMainDb(void); extern bool LockCitusExtension(void); extern PGDLLEXPORT void CitusMaintenanceDaemonMain(Datum main_arg); #endif /* MAINTENANCED_H */ ================================================ FILE: src/include/distributed/memutils.h ================================================ /* * memutils.h * utility functions to help with postgres' memory management primitives */ #ifndef CITUS_MEMUTILS_H #define CITUS_MEMUTILS_H #include "utils/palloc.h" /* * EnsureReleaseResource is an abstraction on MemoryContextRegisterResetCallback that * allocates the space for the MemoryContextCallback and registers it to the current * MemoryContext, ensuring the call of callback with arg as its argument during either the * Reset of Delete of a MemoryContext. */ static inline void EnsureReleaseResource(MemoryContextCallbackFunction callback, void *arg) { MemoryContextCallback *cb = MemoryContextAllocZero(CurrentMemoryContext, sizeof(MemoryContextCallback)); cb->func = callback; cb->arg = arg; MemoryContextRegisterResetCallback(CurrentMemoryContext, cb); } #endif /*CITUS_MEMUTILS_H */ ================================================ FILE: src/include/distributed/merge_executor.h ================================================ /*------------------------------------------------------------------------- * * merge_executor.h * * Declarations for public functions and types related to executing * MERGE INTO ... SQL commands. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MERGE_EXECUTOR_H #define MERGE_EXECUTOR_H extern TupleTableSlot * NonPushableMergeCommandExecScan(CustomScanState *node); #endif /* MERGE_EXECUTOR_H */ ================================================ FILE: src/include/distributed/merge_planner.h ================================================ /*------------------------------------------------------------------------- * * merge_planner.h * * Declarations for public functions and types related to router planning. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MERGE_PLANNER_H #define MERGE_PLANNER_H #include "c.h" #include "nodes/parsenodes.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/multi_physical_planner.h" extern DistributedPlan * CreateMergePlan(uint64 planId, Query *originalQuery, Query *query, PlannerRestrictionContext * plannerRestrictionContext, ParamListInfo boundParams); extern bool IsLocalTableModification(Oid targetRelationId, Query *query, uint64 shardId, RTEListProperties *rteProperties); extern void NonPushableMergeCommandExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es); extern Var * FetchAndValidateInsertVarIfExists(Oid targetRelationId, Query *query); extern RangeTblEntry * ExtractMergeSourceRangeTableEntry(Query *query, bool joinSourceOk); extern FromExpr * GetMergeJoinTree(Query *mergeQuery); #endif /* MERGE_PLANNER_H */ ================================================ FILE: src/include/distributed/metadata/dependency.h ================================================ /*------------------------------------------------------------------------- * * dependency.c * Functions to follow and record dependencies for objects to be * created in the right order. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_DEPENDENCY_H #define CITUS_DEPENDENCY_H #include "postgres.h" #include "catalog/objectaddress.h" #include "catalog/pg_depend.h" #include "nodes/pg_list.h" #include "distributed/errormessage.h" typedef bool (*AddressPredicate)(const ObjectAddress *); extern List * GetUniqueDependenciesList(List *objectAddressesList); extern List * GetDependenciesForObject(const ObjectAddress *target); extern List * GetAllSupportedDependenciesForObject(const ObjectAddress *target); extern List * GetAllDependenciesForObject(const ObjectAddress *target); extern bool ErrorOrWarnIfAnyObjectHasUnsupportedDependency(List *objectAddresses); extern DeferredErrorMessage * DeferErrorIfAnyObjectHasUnsupportedDependency(const List * objectAddresses); extern List * GetAllCitusDependedDependenciesForObject(const ObjectAddress *target); extern List * OrderObjectAddressListInDependencyOrder(List *objectAddressList); extern bool SupportedDependencyByCitus(const ObjectAddress *address); extern List * GetPgDependTuplesForDependingObjects(Oid targetObjectClassId, Oid targetObjectId); extern List * GetDependingViews(Oid relationId); extern Oid GetDependingView(Form_pg_depend pg_depend); extern List * FilterObjectAddressListByPredicate(List *objectAddressList, AddressPredicate predicate); #endif /* CITUS_DEPENDENCY_H */ ================================================ FILE: src/include/distributed/metadata/distobject.h ================================================ /*------------------------------------------------------------------------- * * distobject.h * Declarations for functions to work with pg_dist_object * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_METADATA_DISTOBJECT_H #define CITUS_METADATA_DISTOBJECT_H #include "postgres.h" #include "catalog/objectaddress.h" #define INVALID_DISTRIBUTION_ARGUMENT_INDEX -1 #define NO_FORCE_PUSHDOWN 0 extern bool ObjectExists(const ObjectAddress *address); extern bool CitusExtensionObject(const ObjectAddress *objectAddress); extern bool IsAnyObjectDistributed(const List *addresses); extern bool IsAnyParentObjectDistributed(const List *addresses); extern bool ClusterHasDistributedFunctionWithDistArgument(void); extern void MarkObjectDistributed(const ObjectAddress *distAddress); extern void MarkObjectDistributedWithName(const ObjectAddress *distAddress, char *name, bool useConnectionForLocalQuery, char *connectionUser); extern void MarkObjectDistributedViaSuperUser(const ObjectAddress *distAddress); extern void MarkObjectDistributedLocally(const ObjectAddress *distAddress); extern void UnmarkObjectDistributed(const ObjectAddress *address); extern void UnmarkNodeWideObjectsDistributed(Node *node); extern bool IsTableOwnedByExtension(Oid relationId); extern bool ObjectAddressDependsOnExtension(const ObjectAddress *target); extern bool IsAnyObjectAddressOwnedByExtension(const List *targets, ObjectAddress *extensionAddress); extern ObjectAddress * FirstExtensionWithSchema(Oid schemaId); extern bool IsObjectAddressOwnedByCitus(const ObjectAddress *objectAddress); extern ObjectAddress PgGetObjectAddress(char *ttype, ArrayType *namearr, ArrayType *argsarr); extern List * GetDistributedObjectAddressList(void); extern RoleSpec * GetRoleSpecObjectForUser(Oid roleOid); extern void UpdateDistributedObjectColocationId(uint32 oldColocationId, uint32 newColocationId); extern List * DistributedFunctionList(void); extern List * DistributedSequenceList(void); #endif /* CITUS_METADATA_DISTOBJECT_H */ ================================================ FILE: src/include/distributed/metadata/pg_dist_object.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_object.h * definition of the system distributed objects relation (pg_dist_object). * * This table keeps metadata on all postgres objects that are distributed * to all the nodes in the network. Objects in this table should all be * present on all workers and kept in sync throughout their existance. * This also means that all nodes joining the network are assumed to * recreate all these objects. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_OBJECT_H #define PG_DIST_OBJECT_H /* ---------------- * pg_dist_object definition. * ---------------- */ typedef struct FormData_pg_dist_object { Oid classid; /* class of the distributed object */ Oid objid; /* object id of the distributed object */ int32 objsubid; /* object sub id of the distributed object, eg. attnum */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ text type; text[] object_names; text[] object_arguments; uint32 distribution_argument_index; /* only valid for distributed functions/procedures */ uint32 colocationid; /* only valid for distributed functions/procedures */ boolean forced_pushdown; /* only valid for distributed functions */ #endif } FormData_pg_dist_object; /* ---------------- * Form_pg_dist_partitions corresponds to a pointer to a tuple with * the format of pg_dist_partitions relation. * ---------------- */ typedef FormData_pg_dist_object *Form_pg_dist_object; /* ---------------- * compiler constants for pg_dist_object * ---------------- */ #define Natts_pg_dist_object 9 #define Anum_pg_dist_object_classid 1 #define Anum_pg_dist_object_objid 2 #define Anum_pg_dist_object_objsubid 3 #define Anum_pg_dist_object_type 4 #define Anum_pg_dist_object_object_names 5 #define Anum_pg_dist_object_object_args 6 #define Anum_pg_dist_object_distribution_argument_index 7 #define Anum_pg_dist_object_colocationid 8 #define Anum_pg_dist_object_force_delegation 9 extern int GetForceDelegationAttrIndexInPgDistObject(TupleDesc tupleDesc); #endif /* PG_DIST_OBJECT_H */ ================================================ FILE: src/include/distributed/metadata_cache.h ================================================ /*------------------------------------------------------------------------- * * metadata_cache.h * Executor support for Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef METADATA_CACHE_H #define METADATA_CACHE_H #include "postgres.h" #include "fmgr.h" #include "utils/hsearch.h" #include "distributed/metadata_utility.h" #include "distributed/pg_dist_partition.h" #include "distributed/worker_manager.h" extern bool EnableVersionChecks; /* managed via guc.c */ typedef enum { USE_SECONDARY_NODES_NEVER = 0, USE_SECONDARY_NODES_ALWAYS = 1 } ReadFromSecondariesType; extern int ReadFromSecondaries; /* * While upgrading pg_dist_local_group can be empty temporarily, in that * case we use GROUP_ID_UPGRADING as the local group id to communicate * this to other functions. */ #define GROUP_ID_UPGRADING -2 /* * Representation of a table's metadata that is frequently used for * distributed execution. Cached. */ typedef struct { /* lookup key - must be first. A pg_class.oid oid. */ Oid relationId; /* * Has an invalidation been received for this entry, requiring a rebuild * of the cache entry? */ bool isValid; bool hasUninitializedShardInterval; bool hasUniformHashDistribution; /* valid for hash partitioned tables */ bool hasOverlappingShardInterval; /* pg_dist_partition metadata for this table */ char *partitionKeyString; Var *partitionColumn; char partitionMethod; uint32 colocationId; char replicationModel; bool autoConverted; /* table auto-added to metadata, valid for citus local tables */ /* pg_dist_shard metadata (variable-length ShardInterval array) for this table */ int shardIntervalArrayLength; ShardInterval **sortedShardIntervalArray; /* comparator for partition column's type, NULL if DISTRIBUTE_BY_NONE */ FmgrInfo *shardColumnCompareFunction; /* * Comparator for partition interval type (different from * shardColumnCompareFunction if hash-partitioned), NULL if * DISTRIBUTE_BY_NONE. */ FmgrInfo *shardIntervalCompareFunction; FmgrInfo *hashFunction; /* NULL if table is not distributed by hash */ /* * The following two lists consists of relationIds that this distributed * relation has a foreign key to (e.g., referencedRelationsViaForeignKey) or * other relations has a foreign key to this relation (e.g., * referencingRelationsViaForeignKey). * * Note that we're keeping all transitive foreign key references as well * such that if relation A refers to B, and B refers to C, we keep A and B * in C's referencingRelationsViaForeignKey. */ List *referencedRelationsViaForeignKey; List *referencingRelationsViaForeignKey; /* pg_dist_placement metadata */ GroupShardPlacement **arrayOfPlacementArrays; int *arrayOfPlacementArrayLengths; } CitusTableCacheEntry; typedef struct DistObjectCacheEntryKey { Oid classid; Oid objid; int32 objsubid; } DistObjectCacheEntryKey; typedef struct DistObjectCacheEntry { /* lookup key - must be first. */ DistObjectCacheEntryKey key; bool isValid; bool isDistributed; int distributionArgIndex; int colocationId; bool forceDelegation; } DistObjectCacheEntry; typedef enum { HASH_DISTRIBUTED, APPEND_DISTRIBUTED, RANGE_DISTRIBUTED, SINGLE_SHARD_DISTRIBUTED, /* hash, range or append distributed table */ DISTRIBUTED_TABLE, /* hash- or range-distributed table */ STRICTLY_PARTITIONED_DISTRIBUTED_TABLE, REFERENCE_TABLE, CITUS_LOCAL_TABLE, ANY_CITUS_TABLE_TYPE } CitusTableType; void InvalidateDistRelationCacheCallback(Datum argument, Oid relationId); extern List * AllCitusTableIds(void); extern bool IsCitusTableType(Oid relationId, CitusTableType tableType); extern CitusTableType GetCitusTableType(CitusTableCacheEntry *tableEntry); extern bool IsCitusTableTypeCacheEntry(CitusTableCacheEntry *tableEtnry, CitusTableType tableType); extern bool IsFirstShard(CitusTableCacheEntry *tableEntry, uint64 shardId); bool HasDistributionKey(Oid relationId); bool HasDistributionKeyCacheEntry(CitusTableCacheEntry *tableEntry); extern char * GetTableTypeName(Oid tableId); extern void SetCreateCitusTransactionLevel(int val); extern int GetCitusCreationLevel(void); extern bool IsCitusTable(Oid relationId); extern bool IsCitusTableRangeVar(RangeVar *rangeVar, LOCKMODE lockMode, bool missingOk); extern bool IsCitusTableViaCatalog(Oid relationId); extern char PgDistPartitionViaCatalog(Oid relationId); extern List * LookupDistShardTuples(Oid relationId); extern char PartitionMethodViaCatalog(Oid relationId); extern Var * PartitionColumnViaCatalog(Oid relationId); extern uint32 ColocationIdViaCatalog(Oid relationId); bool IsReferenceTableByDistParams(char partitionMethod, char replicationModel); extern bool IsCitusLocalTableByDistParams(char partitionMethod, char replicationModel, uint32 colocationId); extern bool IsSingleShardTableByDistParams(char partitionMethod, char replicationModel, uint32 colocationId); extern List * CitusTableList(void); extern ShardInterval * LoadShardInterval(uint64 shardId); extern bool ShardExists(uint64 shardId); extern Oid RelationIdForShard(uint64 shardId); extern bool ReferenceTableShardId(uint64 shardId); extern bool DistributedTableShardId(uint64 shardId); extern ShardPlacement * ShardPlacementOnGroupIncludingOrphanedPlacements(int32 groupId, uint64 shardId); extern ShardPlacement * ActiveShardPlacementOnGroup(int32 groupId, uint64 shardId); extern GroupShardPlacement * LoadGroupShardPlacement(uint64 shardId, uint64 placementId); extern ShardPlacement * LoadShardPlacement(uint64 shardId, uint64 placementId); extern CitusTableCacheEntry * GetCitusTableCacheEntry(Oid distributedRelationId); extern CitusTableCacheEntry * LookupCitusTableCacheEntry(Oid relationId); extern DistObjectCacheEntry * LookupDistObjectCacheEntry(Oid classid, Oid objid, int32 objsubid); extern int32 GetLocalGroupId(void); extern int32 GetLocalNodeId(void); extern void CitusTableCacheFlushInvalidatedEntries(void); extern Oid LookupShardRelationFromCatalog(int64 shardId, bool missing_ok); extern List * ShardPlacementList(uint64 shardId); extern void CitusInvalidateRelcacheByRelid(Oid relationId); extern void CitusInvalidateRelcacheByShardId(int64 shardId); extern void InvalidateForeignKeyGraph(void); extern void FlushDistTableCache(void); extern void InvalidateMetadataSystemCache(void); extern List * CitusTableTypeIdList(CitusTableType citusTableType); extern Datum DistNodeMetadata(void); extern bool HasUniformHashDistribution(ShardInterval **shardIntervalArray, int shardIntervalArrayLength); extern bool HasUninitializedShardInterval(ShardInterval **sortedShardIntervalArray, int shardCount); extern bool HasOverlappingShardInterval(ShardInterval **shardIntervalArray, int shardIntervalArrayLength, Oid shardIntervalCollation, FmgrInfo *shardIntervalSortCompareFunction); extern ShardPlacement * ShardPlacementForFunctionColocatedWithSingleShardTable( CitusTableCacheEntry *cacheEntry); extern ShardPlacement * ShardPlacementForFunctionColocatedWithReferenceTable( CitusTableCacheEntry *cacheEntry); extern ShardPlacement * ShardPlacementForFunctionColocatedWithDistTable( DistObjectCacheEntry *procedure, List *argumentList, Var *partitionColumn, CitusTableCacheEntry *cacheEntry, PlannedStmt * plan); extern bool CitusHasBeenLoaded(void); extern bool CheckCitusVersion(int elevel); extern bool CheckAvailableVersion(int elevel); extern bool InstalledAndAvailableVersionsSame(void); extern bool MajorVersionsCompatible(char *leftVersion, char *rightVersion); extern bool MinorVersionsCompatibleRelaxed(char *leftVersion, char *rightVersion); extern void ErrorIfInconsistentShardIntervals(CitusTableCacheEntry *cacheEntry); extern void EnsureModificationsCanRun(void); extern void EnsureModificationsCanRunOnRelation(Oid relationId); extern char LookupDistributionMethod(Oid distributionMethodOid); extern bool RelationExists(Oid relationId); extern ShardInterval * TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid intervalTypeId, int32 intervalTypeMod); /* access WorkerNodeHash */ extern bool HasAnyNodes(void); extern HTAB * GetWorkerNodeHash(void); extern WorkerNode * LookupNodeByNodeId(uint32 nodeId); extern WorkerNode * LookupNodeByNodeIdOrError(uint32 nodeId); extern WorkerNode * LookupNodeForGroup(int32 groupId); /* namespace oids */ extern Oid CitusCatalogNamespaceId(void); /* relation oids */ extern Oid DistCleanupRelationId(void); extern Oid DistColocationRelationId(void); extern Oid DistColocationConfigurationIndexId(void); extern Oid DistPartitionRelationId(void); extern Oid DistShardRelationId(void); extern Oid DistPlacementRelationId(void); extern Oid DistNodeRelationId(void); extern Oid DistBackgroundJobRelationId(void); extern Oid DistBackgroundTaskRelationId(void); extern Oid DistRebalanceStrategyRelationId(void); extern Oid DistLocalGroupIdRelationId(void); extern Oid DistObjectRelationId(void); extern Oid DistEnabledCustomAggregatesId(void); extern Oid DistTenantSchemaRelationId(void); /* index oids */ extern Oid DistNodeNodeIdIndexId(void); extern Oid DistPartitionLogicalRelidIndexId(void); extern Oid DistPartitionColocationidIndexId(void); extern Oid DistBackgroundJobPKeyIndexId(void); extern Oid DistBackgroundTaskPKeyIndexId(void); extern Oid DistBackgroundTaskJobIdTaskIdIndexId(void); extern Oid DistBackgroundTaskStatusTaskIdIndexId(void); extern Oid DistBackgroundTaskDependRelationId(void); extern Oid DistBackgroundTaskDependTaskIdIndexId(void); extern Oid DistBackgroundTaskDependDependsOnIndexId(void); extern Oid DistShardLogicalRelidIndexId(void); extern Oid DistShardShardidIndexId(void); extern Oid DistPlacementShardidIndexId(void); extern Oid DistPlacementPlacementidIndexId(void); extern Oid DistColocationIndexId(void); extern Oid DistTransactionRelationId(void); extern Oid DistTransactionGroupIndexId(void); extern Oid DistPlacementGroupidIndexId(void); extern Oid DistObjectPrimaryKeyIndexId(void); extern Oid DistCleanupPrimaryKeyIndexId(void); extern Oid DistTenantSchemaPrimaryKeyIndexId(void); extern Oid DistTenantSchemaUniqueColocationIdIndexId(void); /* sequence oids */ extern Oid DistBackgroundJobJobIdSequenceId(void); extern Oid DistBackgroundTaskTaskIdSequenceId(void); extern Oid DistClockLogicalSequenceId(void); /* type oids */ extern Oid LookupTypeOid(char *schemaNameSting, char *typeNameString); extern Oid CitusCopyFormatTypeId(void); /* function oids */ extern Oid CitusReadIntermediateResultFuncId(void); Oid CitusReadIntermediateResultArrayFuncId(void); extern Oid CitusExtraDataContainerFuncId(void); extern Oid CitusAnyValueFunctionId(void); extern Oid CitusTextSendAsJsonbFunctionId(void); extern Oid TextOutFunctionId(void); extern Oid RelationIsAKnownShardFuncId(void); extern Oid JsonbExtractPathFuncId(void); extern Oid JsonbExtractPathTextFuncId(void); extern Oid CitusDependentObjectFuncId(void); /* enum oids */ extern Oid PrimaryNodeRoleId(void); extern Oid SecondaryNodeRoleId(void); extern Oid UnavailableNodeRoleId(void); extern Oid CitusCopyFormatTypeId(void); extern Oid TextCopyFormatId(void); extern Oid BinaryCopyFormatId(void); extern Oid CitusJobStatusScheduledId(void); extern Oid CitusJobStatusRunningId(void); extern Oid CitusJobStatusCancellingId(void); extern Oid CitusJobStatusFinishedId(void); extern Oid CitusJobStatusCancelledId(void); extern Oid CitusJobStatusFailedId(void); extern Oid CitusJobStatusFailingId(void); extern Oid CitusTaskStatusBlockedId(void); extern Oid CitusTaskStatusRunnableId(void); extern Oid CitusTaskStatusRunningId(void); extern Oid CitusTaskStatusDoneId(void); extern Oid CitusTaskStatusErrorId(void); extern Oid CitusTaskStatusUnscheduledId(void); extern Oid CitusTaskStatusCancelledId(void); extern Oid CitusTaskStatusCancellingId(void); /* user related functions */ extern Oid CitusExtensionOwner(void); extern char * CitusExtensionOwnerName(void); extern char * CurrentUserName(void); extern const char * CurrentDatabaseName(void); /* connection-related functions */ extern char * GetAuthinfoViaCatalog(const char *roleName, int64 nodeId); extern char * GetPoolinfoViaCatalog(int32 nodeId); #endif /* METADATA_CACHE_H */ ================================================ FILE: src/include/distributed/metadata_sync.h ================================================ /*------------------------------------------------------------------------- * * metadata_sync.h * Type and function declarations used to sync metadata across all * workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef METADATA_SYNC_H #define METADATA_SYNC_H #include "nodes/pg_list.h" #include "distributed/commands/utility_hook.h" #include "distributed/coordinator_protocol.h" #include "distributed/metadata_cache.h" /* managed via guc.c */ typedef enum { METADATA_SYNC_TRANSACTIONAL = 0, METADATA_SYNC_NON_TRANSACTIONAL = 1 } MetadataSyncTransactionMode; /* config variables */ extern int MetadataSyncInterval; extern int MetadataSyncRetryInterval; extern int MetadataSyncTransMode; /* * MetadataSyncContext is used throughout metadata sync. */ typedef struct MetadataSyncContext { List *activatedWorkerNodeList; /* activated worker nodes */ List *activatedWorkerBareConnections; /* bare connections to activated nodes */ MemoryContext context; /* memory context for all allocations */ MetadataSyncTransactionMode transactionMode; /* transaction mode for the sync */ bool collectCommands; /* if we collect commands instead of sending and resetting */ List *collectedCommands; /* collected commands. (NIL if collectCommands == false) */ bool nodesAddedInSameTransaction; /* if the nodes are added just before activation */ } MetadataSyncContext; typedef enum { NODE_METADATA_SYNC_SUCCESS = 0, NODE_METADATA_SYNC_FAILED_LOCK = 1, NODE_METADATA_SYNC_FAILED_SYNC = 2 } NodeMetadataSyncResult; /* * Information about dependent sequences. We do not have the * dependent relationId as no caller needs. But, could be added * here if needed. */ typedef struct SequenceInfo { Oid sequenceOid; int attributeNumber; /* * true for nexval(seq) -- which also includes serials * false when only OWNED BY col */ bool isNextValDefault; } SequenceInfo; /* Functions declarations for metadata syncing */ extern void citus_internal_add_placement_metadata_internal(int64 shardId, int64 shardLength, int32 groupId, int64 placementId); extern void SyncCitusTableMetadata(Oid relationId); extern void EnsureSequentialModeMetadataOperations(void); extern bool ClusterHasKnownMetadataWorkers(void); extern char * LocalGroupIdUpdateCommand(int32 groupId); extern bool ShouldSyncUserCommandForObject(ObjectAddress objectAddress); extern bool ShouldSyncTableMetadata(Oid relationId); extern bool ShouldSyncTableMetadataViaCatalog(Oid relationId); extern Oid FetchRelationIdFromPgPartitionHeapTuple(HeapTuple heapTuple, TupleDesc tupleDesc); extern bool ShouldSyncSequenceMetadata(Oid relationId); extern List * NodeMetadataCreateCommands(void); extern List * CitusTableMetadataCreateCommandList(Oid relationId); extern List * NodeMetadataDropCommands(void); extern char * MarkObjectsDistributedCreateCommand(List *addresses, List *names, List *distributionArgumentIndexes, List *colocationIds, List *forceDelegations); extern char * DistributionCreateCommand(CitusTableCacheEntry *cacheEntry); extern char * DistributionDeleteCommand(const char *schemaName, const char *tableName); extern char * DistributionDeleteMetadataCommand(Oid relationId); extern char * TableOwnerResetCommand(Oid distributedRelationId); extern char * NodeListInsertCommand(List *workerNodeList); char * NodeListIdempotentInsertCommand(List *workerNodeList); extern List * ShardListInsertCommand(List *shardIntervalList); extern List * ShardDeleteCommandList(ShardInterval *shardInterval); extern char * NodeDeleteCommand(uint32 nodeId); extern char * NodeStateUpdateCommand(uint32 nodeId, bool isActive); extern char * ShouldHaveShardsUpdateCommand(uint32 nodeId, bool shouldHaveShards); extern char * ColocationIdUpdateCommand(Oid relationId, uint32 colocationId); extern char * CreateSchemaDDLCommand(Oid schemaId); extern List * GrantOnSchemaDDLCommands(Oid schemaId); extern List * GrantOnFunctionDDLCommands(Oid functionOid); extern List * GrantOnDatabaseDDLCommands(Oid databaseOid); extern List * GrantOnForeignServerDDLCommands(Oid serverId); extern List * GenerateGrantOnForeignServerQueriesFromAclItem(Oid serverId, AclItem *aclItem); extern List * GenerateGrantOnFDWQueriesFromAclItem(Oid serverId, AclItem *aclItem); extern char * PlacementUpsertCommand(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId); extern TableDDLCommand * TruncateTriggerCreateCommand(Oid relationId); extern void CreateInterTableRelationshipOfRelationOnWorkers(Oid relationId); extern List * InterTableRelationshipOfRelationCommandList(Oid relationId); extern List * DetachPartitionCommandList(void); extern void SyncNodeMetadataToNodes(void); extern BackgroundWorkerHandle * SpawnSyncNodeMetadataToNodes(Oid database, Oid owner); extern PGDLLEXPORT void SyncNodeMetadataToNodesMain(Datum main_arg); extern void SignalMetadataSyncDaemon(Oid database, int sig); extern bool ShouldInitiateMetadataSync(bool *lockFailure); extern List * SequenceDependencyCommandList(Oid relationId); extern List * IdentitySequenceDependencyCommandList(Oid targetRelationId); extern List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName); extern List * GetSequencesFromAttrDef(Oid attrdefOid); extern List * GetAttrDefsFromSequence(Oid seqOid); extern void GetDependentSequencesWithRelation(Oid relationId, List **seqInfoList, AttrNumber attnum, char depType); extern List * GetDependentRelationsWithSequence(Oid seqId, char depType); extern List * GetDependentFunctionsWithRelation(Oid relationId); extern Oid GetAttributeTypeOid(Oid relationId, AttrNumber attnum); extern void SetLocalEnableMetadataSync(bool state); extern void SyncNewColocationGroupToNodes(uint32 colocationId, int shardCount, int replicationFactor, Oid distributionColumType, Oid distributionColumnCollation); extern void SyncDeleteColocationGroupToNodes(uint32 colocationId); extern char * TenantSchemaInsertCommand(Oid schemaId, uint32 colocationId); extern char * TenantSchemaDeleteCommand(char *schemaName); extern char * UpdateNoneDistTableMetadataCommand(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted); extern char * AddPlacementMetadataCommand(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId); extern char * DeletePlacementMetadataCommand(uint64 placementId); extern MetadataSyncContext * CreateMetadataSyncContext(List *nodeList, bool collectCommands, bool nodesAddedInSameTransaction); extern void EstablishAndSetMetadataSyncBareConnections(MetadataSyncContext *context); extern void SetMetadataSyncNodesFromNodeList(MetadataSyncContext *context, List *nodeList); extern void ResetMetadataSyncMemoryContext(MetadataSyncContext *context); extern bool MetadataSyncCollectsCommands(MetadataSyncContext *context); extern void SendOrCollectCommandListToActivatedNodes(MetadataSyncContext *context, List *commands); extern void SendOrCollectCommandListToMetadataNodes(MetadataSyncContext *context, List *commands); extern void SendOrCollectCommandListToSingleNode(MetadataSyncContext *context, List *commands, int nodeIdx); extern void ActivateNodeList(MetadataSyncContext *context); extern char * WorkerDropAllShellTablesCommand(bool singleTransaction); extern char * WorkerDropSequenceDependencyCommand(Oid relationId); extern void SyncDistributedObjects(MetadataSyncContext *context); extern void SendNodeWideObjectsSyncCommands(MetadataSyncContext *context); extern void SendShellTableDeletionCommands(MetadataSyncContext *context); extern void SendMetadataDeletionCommands(MetadataSyncContext *context); extern void SendColocationMetadataCommands(MetadataSyncContext *context); extern void SendTenantSchemaMetadataCommands(MetadataSyncContext *context); extern void SendDependencyCreationCommands(MetadataSyncContext *context); extern void SendDistTableMetadataCommands(MetadataSyncContext *context); extern void SendDistObjectCommands(MetadataSyncContext *context); extern void SendInterTableRelationshipCommands(MetadataSyncContext *context); #define DELETE_ALL_NODES "DELETE FROM pg_dist_node" #define DELETE_ALL_PLACEMENTS "DELETE FROM pg_dist_placement" #define DELETE_ALL_SHARDS "DELETE FROM pg_dist_shard" #define DELETE_ALL_DISTRIBUTED_OBJECTS "DELETE FROM pg_catalog.pg_dist_object" #define DELETE_ALL_PARTITIONS "DELETE FROM pg_dist_partition" #define DELETE_ALL_COLOCATION "DELETE FROM pg_catalog.pg_dist_colocation" #define DELETE_ALL_TENANT_SCHEMAS "DELETE FROM pg_catalog.pg_dist_schema" #define WORKER_DROP_ALL_SHELL_TABLES \ "CALL pg_catalog.worker_drop_all_shell_tables(%s)" #define CITUS_INTERNAL_MARK_NODE_NOT_SYNCED \ "SELECT citus_internal.mark_node_not_synced(%d, %d)" #define REMOVE_ALL_CITUS_TABLES_COMMAND \ "SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition" #define BREAK_ALL_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND \ "SELECT pg_catalog.worker_drop_sequence_dependency(logicalrelid::regclass::text) FROM pg_dist_partition" #define BREAK_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND \ "SELECT pg_catalog.worker_drop_sequence_dependency(%s);" #define DISABLE_DDL_PROPAGATION "SET citus.enable_ddl_propagation TO 'off'" #define ENABLE_DDL_PROPAGATION "SET citus.enable_ddl_propagation TO 'on'" #define DISABLE_METADATA_SYNC "SET citus.enable_metadata_sync TO 'off'" #define ENABLE_METADATA_SYNC "SET citus.enable_metadata_sync TO 'on'" #define WORKER_APPLY_SEQUENCE_COMMAND "SELECT worker_apply_sequence_command (%s,%s)" #define UPSERT_PLACEMENT \ "INSERT INTO pg_dist_placement " \ "(shardid, shardstate, shardlength, " \ "groupid, placementid) " \ "VALUES (" UINT64_FORMAT ", 1, " UINT64_FORMAT \ ", %d, " UINT64_FORMAT \ ") " \ "ON CONFLICT (shardid, groupid) DO UPDATE SET " \ "shardstate = EXCLUDED.shardstate, " \ "shardlength = EXCLUDED.shardlength, " \ "placementid = EXCLUDED.placementid" #define METADATA_SYNC_CHANNEL "metadata_sync" #define WORKER_ADJUST_IDENTITY_COLUMN_SEQ_RANGES \ "SELECT pg_catalog.worker_adjust_identity_column_seq_ranges(%s)" /* controlled via GUC */ extern char *EnableManualMetadataChangesForUser; extern bool EnableMetadataSync; #endif /* METADATA_SYNC_H */ ================================================ FILE: src/include/distributed/metadata_utility.h ================================================ /*------------------------------------------------------------------------- * * metadata_utility.h * Type and function declarations used for reading and modifying * coordinator node's metadata. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef METADATA_UTILITY_H #define METADATA_UTILITY_H #include "postgres.h" #include "access/heapam.h" #include "access/htup.h" #include "access/tupdesc.h" #include "catalog/indexing.h" #include "catalog/objectaddress.h" #include "utils/acl.h" #include "utils/relcache.h" #include "distributed/citus_nodes.h" #include "distributed/connection_management.h" #include "distributed/errormessage.h" #include "distributed/relay_utility.h" #include "distributed/worker_manager.h" /* total number of hash tokens (2^32) */ #define HASH_TOKEN_COUNT INT64CONST(4294967296) #define SELECT_TRUE_QUERY "SELECT TRUE FROM %s LIMIT 1" #define PG_TABLE_SIZE_FUNCTION "pg_table_size(%s)" #define PG_RELATION_SIZE_FUNCTION "pg_relation_size(%s)" #define PG_TOTAL_RELATION_SIZE_FUNCTION "pg_total_relation_size(%s)" #define WORKER_PARTITIONED_TABLE_SIZE_FUNCTION "worker_partitioned_table_size(%s)" #define WORKER_PARTITIONED_RELATION_SIZE_FUNCTION "worker_partitioned_relation_size(%s)" #define WORKER_PARTITIONED_RELATION_TOTAL_SIZE_FUNCTION \ "worker_partitioned_relation_total_size(%s)" #define SHARD_SIZES_COLUMN_COUNT (2) /* * Flag to keep track of whether the process is currently in a function converting the * type of the table. Since it only affects the level of the log shown while dropping/ * recreating table within the table type conversion, rollbacking to the savepoint hasn't * been implemented for the sake of simplicity. If you are planning to use that flag for * any other purpose, please consider implementing that. */ extern bool InTableTypeConversionFunctionCall; /* In-memory representation of a typed tuple in pg_dist_shard. */ typedef struct ShardInterval { CitusNode type; Oid relationId; char storageType; Oid valueTypeId; /* min/max value datum's typeId */ int valueTypeLen; /* min/max value datum's typelen */ bool valueByVal; /* min/max value datum's byval */ bool minValueExists; bool maxValueExists; Datum minValue; /* a shard's typed min value datum */ Datum maxValue; /* a shard's typed max value datum */ uint64 shardId; int shardIndex; } ShardInterval; /* In-memory representation of a tuple in pg_dist_placement. */ typedef struct GroupShardPlacement { CitusNode type; uint64 placementId; /* sequence that implies this placement creation order */ uint64 shardId; uint64 shardLength; int32 groupId; } GroupShardPlacement; /* A GroupShardPlacement which has had some extra data resolved */ typedef struct ShardPlacement { /* * careful, the rest of the code assumes this exactly matches GroupShardPlacement */ CitusNode type; uint64 placementId; uint64 shardId; uint64 shardLength; int32 groupId; /* the rest of the fields aren't from pg_dist_placement */ char *nodeName; uint32 nodePort; uint32 nodeId; char partitionMethod; uint32 colocationGroupId; uint32 representativeValue; } ShardPlacement; typedef enum CascadeToColocatedOption { CASCADE_TO_COLOCATED_UNSPECIFIED, CASCADE_TO_COLOCATED_YES, CASCADE_TO_COLOCATED_NO, CASCADE_TO_COLOCATED_NO_ALREADY_CASCADED }CascadeToColocatedOption; /* * TableConversionParameters are the parameters that are given to * table conversion UDFs: undistribute_table, alter_distributed_table, * alter_table_set_access_method. * * When passing a TableConversionParameters object to one of the table * conversion functions some of the parameters needs to be set: * UndistributeTable: relationId * AlterDistributedTable: relationId, distributionColumn, shardCountIsNull, * shardCount, colocateWith, cascadeToColocated * AlterTableSetAccessMethod: relationId, accessMethod * * conversionType parameter will be automatically set by the function. * * TableConversionState objects can be created using TableConversionParameters * objects with CreateTableConversion function. */ typedef struct TableConversionParameters { /* * Determines type of conversion: UNDISTRIBUTE_TABLE, * ALTER_DISTRIBUTED_TABLE, ALTER_TABLE_SET_ACCESS_METHOD. */ char conversionType; /* Oid of the table to do conversion on */ Oid relationId; /* * Options to do conversions on the table * distributionColumn is the name of the new distribution column, * shardCountIsNull is if the shardCount variable is not given * shardCount is the new shard count, * colocateWith is the name of the table to colocate with, 'none', or * 'default' * accessMethod is the name of the new accessMethod for the table */ char *distributionColumn; bool shardCountIsNull; int shardCount; char *colocateWith; char *accessMethod; /* * cascadeToColocated determines whether the shardCount and * colocateWith will be cascaded to the currently colocated tables */ CascadeToColocatedOption cascadeToColocated; /* * cascadeViaForeignKeys determines if the conversion operation * will be cascaded to the graph connected with foreign keys * to the table */ bool cascadeViaForeignKeys; /* * suppressNoticeMessages determines if we want to suppress NOTICE * messages that we explicitly issue */ bool suppressNoticeMessages; /* * bypassTenantCheck skips tenant table checks to allow some internal * operations which are normally disallowed */ bool bypassTenantCheck; } TableConversionParameters; typedef struct TableConversionReturn { /* * commands to create foreign keys for the table * * When the table conversion is cascaded we can recreate * some of the foreign keys of the cascaded tables. So with this * list we can return it to the initial conversion operation so * foreign keys can be created after every colocated table is * converted. */ List *foreignKeyCommands; }TableConversionReturn; /* * Size query types for PG and Citus * For difference details, please see: * https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE */ typedef enum SizeQueryType { RELATION_SIZE, /* pg_relation_size() */ TOTAL_RELATION_SIZE, /* pg_total_relation_size() */ TABLE_SIZE /* pg_table_size() */ } SizeQueryType; typedef enum { COLOCATE_WITH_TABLE_LIKE_OPT, COLOCATE_WITH_COLOCATION_ID } ColocationParamType; /* * Param used to specify the colocation target of a distributed table. It can * be either a table name or a colocation id. * * When colocationParamType is COLOCATE_WITH_COLOCATION_ID, colocationId is * expected to be a valid colocation id. When colocationParamType is set to * COLOCATE_WITH_TABLE_LIKE_OPT, colocateWithTableName is expected to * be a valid table name, "default" or "none". * * Among the functions used to create a Citus table, right now only * CreateSingleShardTable() accepts a ColocationParam. */ typedef struct { union { char *colocateWithTableName; uint32 colocationId; }; ColocationParamType colocationParamType; } ColocationParam; typedef enum BackgroundJobStatus { BACKGROUND_JOB_STATUS_SCHEDULED, BACKGROUND_JOB_STATUS_RUNNING, BACKGROUND_JOB_STATUS_FINISHED, BACKGROUND_JOB_STATUS_CANCELLING, BACKGROUND_JOB_STATUS_CANCELLED, BACKGROUND_JOB_STATUS_FAILING, BACKGROUND_JOB_STATUS_FAILED } BackgroundJobStatus; typedef struct BackgroundJob { int64 jobid; BackgroundJobStatus state; char *jobType; char *description; TimestampTz *started_at; TimestampTz *finished_at; /* extra space to store values for nullable value types above */ struct { TimestampTz started_at; TimestampTz finished_at; } __nullable_storage; } BackgroundJob; typedef enum BackgroundTaskStatus { BACKGROUND_TASK_STATUS_BLOCKED, BACKGROUND_TASK_STATUS_RUNNABLE, BACKGROUND_TASK_STATUS_RUNNING, BACKGROUND_TASK_STATUS_CANCELLING, BACKGROUND_TASK_STATUS_DONE, BACKGROUND_TASK_STATUS_ERROR, BACKGROUND_TASK_STATUS_UNSCHEDULED, BACKGROUND_TASK_STATUS_CANCELLED } BackgroundTaskStatus; typedef struct BackgroundTask { int64 jobid; int64 taskid; Oid owner; int32 *pid; BackgroundTaskStatus status; char *command; int32 *retry_count; TimestampTz *not_before; char *message; List *nodesInvolved; /* extra space to store values for nullable value types above */ struct { int32 pid; int32 retry_count; TimestampTz not_before; } __nullable_storage; } BackgroundTask; #define SET_NULLABLE_FIELD(ptr, field, value) \ (ptr)->__nullable_storage.field = (value); \ (ptr)->field = &((ptr)->__nullable_storage.field) #define UNSET_NULLABLE_FIELD(ptr, field) \ (ptr)->field = NULL; \ memset_struct_0((ptr)->__nullable_storage.field) /* Size functions */ extern Datum citus_table_size(PG_FUNCTION_ARGS); extern Datum citus_total_relation_size(PG_FUNCTION_ARGS); extern Datum citus_relation_size(PG_FUNCTION_ARGS); /* Function declarations to read shard and shard placement data */ extern uint32 TableShardReplicationFactor(Oid relationId); extern List * LoadShardIntervalList(Oid relationId); extern List * LoadUnsortedShardIntervalListViaCatalog(Oid relationId); extern ShardInterval * LoadShardIntervalWithLongestShardName(Oid relationId); extern int ShardIntervalCount(Oid relationId); extern List * LoadShardList(Oid relationId); extern ShardInterval * CopyShardInterval(ShardInterval *srcInterval); extern uint64 ShardLength(uint64 shardId); extern bool NodeGroupHasShardPlacements(int32 groupId); extern bool IsActiveShardPlacement(ShardPlacement *ShardPlacement); extern bool IsRemoteShardPlacement(ShardPlacement *shardPlacement); extern bool IsPlacementOnWorkerNode(ShardPlacement *placement, WorkerNode *workerNode); extern List * FilterShardPlacementList(List *shardPlacementList, bool (*filter)( ShardPlacement *)); extern List * FilterActiveShardPlacementListByNode(List *shardPlacementList, WorkerNode *workerNode); extern List * ActiveShardPlacementListOnGroup(uint64 shardId, int32 groupId); extern List * ActiveShardPlacementList(uint64 shardId); extern List * ShardPlacementListSortedByWorker(uint64 shardId); extern ShardPlacement * ActiveShardPlacement(uint64 shardId, bool missingOk); extern WorkerNode * ActiveShardPlacementWorkerNode(uint64 shardId); extern List * BuildShardPlacementList(int64 shardId); extern List * AllShardPlacementsOnNodeGroup(int32 groupId); extern List * GroupShardPlacementsForTableOnGroup(Oid relationId, int32 groupId); extern void LookupTaskPlacementHostAndPort(ShardPlacement *taskPlacement, char **nodeName, int *nodePort); extern bool IsDummyPlacement(ShardPlacement *taskPlacement); extern StringInfo GenerateSizeQueryOnMultiplePlacements(List *shardIntervalList, Oid indexId, SizeQueryType sizeQueryType, bool optimizePartitionCalculations ); extern List * RemoveCoordinatorPlacementIfNotSingleNode(List *placementList); /* Function declarations to modify shard and shard placement data */ extern void InsertShardRow(Oid relationId, uint64 shardId, char storageType, text *shardMinValue, text *shardMaxValue); extern void DeleteShardRow(uint64 shardId); extern ShardPlacement * InsertShardPlacementRowGlobally(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId); extern uint64 InsertShardPlacementRow(uint64 shardId, uint64 placementId, uint64 shardLength, int32 groupId); extern void InsertIntoPgDistPartition(Oid relationId, char distributionMethod, Var *distributionColumn, uint32 colocationId, char replicationModel, bool autoConverted); extern void UpdatePgDistPartitionAutoConverted(Oid citusTableId, bool autoConverted); extern void UpdateDistributionColumnGlobally(Oid relationId, char distributionMethod, Var *distributionColumn, int colocationId); extern void UpdateDistributionColumn(Oid relationId, char distributionMethod, Var *distributionColumn, int colocationId); extern void DeletePartitionRow(Oid distributedRelationId); extern void UpdateNoneDistTableMetadataGlobally(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted); extern void UpdateNoneDistTableMetadata(Oid relationId, char replicationModel, uint32 colocationId, bool autoConverted); extern void DeleteShardRow(uint64 shardId); extern void UpdatePlacementGroupId(uint64 placementId, int groupId); extern void DeleteShardPlacementRowGlobally(uint64 placementId); extern void DeleteShardPlacementRow(uint64 placementId); extern void CreateSingleShardTable(Oid relationId, ColocationParam colocationParam); extern void CreateDistributedTable(Oid relationId, char *distributionColumnName, char distributionMethod, int shardCount, bool shardCountIsStrict, char *colocateWithTableName); extern void CreateReferenceTable(Oid relationId); extern void CreateTruncateTrigger(Oid relationId); extern uint64 CopyFromLocalTableIntoDistTable(Oid localTableId, Oid distributedTableId); extern void EnsureUndistributeTenantTableSafe(Oid relationId, const char *operationName); extern TableConversionReturn * UndistributeTable(TableConversionParameters *params); extern void UndistributeTables(List *relationIdList); extern void EnsureObjectAndDependenciesExistOnAllNodes(const ObjectAddress *target); extern void EnsureAllObjectDependenciesExistOnAllNodes(const List *targets); extern DeferredErrorMessage * DeferErrorIfCircularDependencyExists(const ObjectAddress * objectAddress); extern List * GetDistributableDependenciesForObject(const ObjectAddress *target); extern List * GetAllDependencyCreateDDLCommands(const List *dependencies); extern bool ShouldPropagate(void); extern bool ShouldPropagateCreateInCoordinatedTransction(void); extern bool ShouldPropagateAnyObject(List *addresses); /* Remaining metadata utility functions */ extern Oid TableOwnerOid(Oid relationId); extern char * TableOwner(Oid relationId); extern void EnsureTablePermissions(Oid relationId, AclMode mode, AclMaskHow mask); extern void EnsureTableOwner(Oid relationId); extern void EnsureHashDistributedTable(Oid relationId); extern void EnsureHashOrSingleShardDistributedTable(Oid relationId); extern void EnsureFunctionOwner(Oid functionId); extern void EnsureSchemaOwner(Oid schemaId); extern void EnsureSuperUser(void); extern void ErrorIfTableIsACatalogTable(Relation relation); extern void EnsureTableNotDistributed(Oid relationId); extern void EnsureRelationExists(Oid relationId); extern bool RegularTable(Oid relationId); extern bool TableEmpty(Oid tableId); extern bool IsForeignTable(Oid relationId); extern bool ForeignTableDropsTableNameOption(List *optionList); extern bool ServerUsesPostgresFdw(Oid serverId); extern char * ConstructQualifiedShardName(ShardInterval *shardInterval); extern uint64 GetFirstShardId(Oid relationId); extern Datum StringToDatum(char *inputString, Oid dataType); extern char * DatumToString(Datum datum, Oid dataType); extern int CompareShardPlacementsByWorker(const void *leftElement, const void *rightElement); extern int CompareShardPlacementsByGroupId(const void *leftElement, const void *rightElement); extern ShardInterval * DeformedDistShardTupleToShardInterval(Datum *datumArray, bool *isNullArray, Oid intervalTypeId, int32 intervalTypeMod); extern void GetIntervalTypeInfo(char partitionMethod, Var *partitionColumn, Oid *intervalTypeId, int32 *intervalTypeMod); extern List * SendShardStatisticsQueriesInParallel(List *citusTableIds, bool useDistributedTransaction); extern bool GetNodeDiskSpaceStatsForConnection(MultiConnection *connection, uint64 *availableBytes, uint64 *totalBytes); extern void ExecuteQueryViaSPI(char *query, int SPIOK); extern void ExecuteAndLogQueryViaSPI(char *query, int SPIOK, int logLevel); extern void EnsureSequenceTypeSupported(Oid seqOid, Oid attributeTypeId, Oid ownerRelationId); extern void AlterSequenceType(Oid seqOid, Oid typeOid); extern void EnsureRelationHasCompatibleSequenceTypes(Oid relationId); extern bool HasRunnableBackgroundTask(void); extern bool HasNonTerminalJobOfType(const char *jobType, int64 *jobIdOut); extern int64 CreateBackgroundJob(const char *jobType, const char *description); extern BackgroundTask * ScheduleBackgroundTask(int64 jobId, Oid owner, char *command, int dependingTaskCount, int64 dependingTaskIds[], int nodesInvolvedCount, int32 nodesInvolved[]); extern BackgroundTask * GetRunnableBackgroundTask(void); extern void ResetRunningBackgroundTasks(void); extern BackgroundJob * GetBackgroundJobByJobId(int64 jobId); extern BackgroundTask * GetBackgroundTaskByTaskId(int64 taskId); extern void UpdateBackgroundJob(int64 jobId); extern void UpdateBackgroundTask(BackgroundTask *task); extern void UpdateJobStatus(int64 taskId, const pid_t *pid, BackgroundTaskStatus status, const int32 *retry_count, char *message); extern bool UpdateJobError(BackgroundTask *job, ErrorData *edata); extern List * CancelTasksForJob(int64 jobid); extern void UnscheduleDependentTasks(BackgroundTask *task); extern void UnblockDependingBackgroundTasks(BackgroundTask *task); extern BackgroundJobStatus BackgroundJobStatusByOid(Oid enumOid); extern BackgroundTaskStatus BackgroundTaskStatusByOid(Oid enumOid); extern bool IsBackgroundJobStatusTerminal(BackgroundJobStatus status); extern bool IsBackgroundTaskStatusTerminal(BackgroundTaskStatus status); extern Oid BackgroundJobStatusOid(BackgroundJobStatus status); extern Oid BackgroundTaskStatusOid(BackgroundTaskStatus status); extern int GetAutoConvertedAttrIndexInPgDistPartition(TupleDesc tupleDEsc); /* from node_metadata.c */ extern void LockShardsInWorkerPlacementList(WorkerNode *workerNode, LOCKMODE lockMode); extern void ActivateCloneNodeAsPrimary(WorkerNode *workerNode); #endif /* METADATA_UTILITY_H */ ================================================ FILE: src/include/distributed/multi_executor.h ================================================ /*------------------------------------------------------------------------- * * multi_executor.h * Executor support for Citus. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef MULTI_EXECUTOR_H #define MULTI_EXECUTOR_H #include "executor/execdesc.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "distributed/citus_custom_scan.h" #include "distributed/multi_physical_planner.h" #include "distributed/multi_server_executor.h" #include "distributed/tuple_destination.h" /* managed via guc.c */ typedef enum { PARALLEL_CONNECTION = 0, SEQUENTIAL_CONNECTION = 1 } MultiShardConnectionTypes; /* * TransactionBlocksUsage indicates whether to use remote transaction * blocks according to one of the following policies: * - opening a remote transaction is required * - opening a remote transaction does not matter, so it is allowed but not required. * - opening a remote transaction is disallowed */ typedef enum TransactionBlocksUsage { TRANSACTION_BLOCKS_REQUIRED, TRANSACTION_BLOCKS_ALLOWED, TRANSACTION_BLOCKS_DISALLOWED, } TransactionBlocksUsage; /* * TransactionProperties reflects how we should execute a task list * given previous commands in the transaction and the type of task list. */ typedef struct TransactionProperties { /* if true, any failure on the worker causes the execution to end immediately */ bool errorOnAnyFailure; /* * Determines whether transaction blocks on workers are required, disallowed, or * allowed (will use them if already in a coordinated transaction). */ TransactionBlocksUsage useRemoteTransactionBlocks; /* if true, the current execution requires 2PC to be globally enabled */ bool requires2PC; } TransactionProperties; extern bool AllowNestedDistributedExecution; extern bool SkipConstraintValidation; extern int MultiShardConnectionType; extern bool WritableStandbyCoordinator; extern bool AllowModificationsFromWorkersToReplicatedTables; extern bool ForceMaxQueryParallelization; extern int MaxAdaptiveExecutorPoolSize; extern int ExecutorSlowStartInterval; extern bool SortReturning; extern int ExecutorLevel; extern void CitusExecutorStart(QueryDesc *queryDesc, int eflags); extern void CitusExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once); extern void AdaptiveExecutorPreExecutorRun(CitusScanState *scanState); extern TupleTableSlot * AdaptiveExecutor(CitusScanState *scanState); /* * ExecutionParams contains parameters that are used during the execution. * Some of these can be the zero value if it is not needed during the execution. */ typedef struct ExecutionParams { /* modLevel is the access level for rows.*/ RowModifyLevel modLevel; /* taskList contains the tasks for the execution.*/ List *taskList; /* where to forward each tuple received */ TupleDestination *tupleDestination; /* expectResults is true if this execution will return some result. */ bool expectResults; /* targetPoolSize is the maximum amount of connections per worker */ int targetPoolSize; /* xactProperties contains properties for transactions, such as if we should use 2pc. */ TransactionProperties xactProperties; /* jobIdList contains all job ids for the execution */ List *jobIdList; /* localExecutionSupported is true if we can use local execution, if it is false * local execution will not be used. */ bool localExecutionSupported; /* isUtilityCommand is true if the current execution is for a utility * command such as a DDL command.*/ bool isUtilityCommand; /* pass bind parameters to the distributed executor for parameterized plans */ ParamListInfo paramListInfo; } ExecutionParams; ExecutionParams * CreateBasicExecutionParams(RowModifyLevel modLevel, List *taskList, int targetPoolSize, bool localExecutionSupported); extern uint64 ExecuteTaskListExtended(ExecutionParams *executionParams); extern uint64 ExecuteTaskListIntoTupleDestWithParam(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults, ParamListInfo paramListInfo); extern uint64 ExecuteTaskListIntoTupleDest(RowModifyLevel modLevel, List *taskList, TupleDestination *tupleDest, bool expectResults); extern bool IsCitusCustomState(PlanState *planState); extern TupleTableSlot * CitusExecScan(CustomScanState *node); extern TupleTableSlot * ReturnTupleFromTuplestore(CitusScanState *scanState); extern void ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc tupleDescriptor, Tuplestorestate *tupstore); extern Query * ParseQueryString(const char *queryString, Oid *paramOids, int numParams); extern Query * RewriteRawQueryStmt(RawStmt *rawStmt, const char *queryString, Oid *paramOids, int numParams); extern void ExecuteQueryStringIntoDestReceiver(const char *queryString, ParamListInfo params, DestReceiver *dest); extern void ExecuteQueryIntoDestReceiver(Query *query, ParamListInfo params, DestReceiver *dest); extern uint64 ExecutePlanIntoDestReceiver(PlannedStmt *queryPlan, ParamListInfo params, DestReceiver *dest); extern void SetLocalMultiShardModifyModeToSequential(void); extern void EnsureSequentialMode(ObjectType objType); extern void SetLocalForceMaxQueryParallelization(void); extern void SortTupleStore(CitusScanState *scanState); extern ParamListInfo ExecutorBoundParams(void); extern void EnsureTaskExecutionAllowed(bool isRemote); #endif /* MULTI_EXECUTOR_H */ ================================================ FILE: src/include/distributed/multi_explain.h ================================================ /*------------------------------------------------------------------------- * * multi_explain.h * Explain support for Citus. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef MULTI_EXPLAIN_H #define MULTI_EXPLAIN_H #include "tuple_destination.h" #include "executor/executor.h" typedef enum { EXPLAIN_ANALYZE_SORT_BY_TIME = 0, EXPLAIN_ANALYZE_SORT_BY_TASK_ID = 1 } ExplainAnalyzeSortMethods; /* Config variables managed via guc.c to explain distributed query plans */ extern bool ExplainDistributedQueries; extern bool ExplainAllTasks; extern int ExplainAnalyzeSortMethod; extern void FreeSavedExplainPlan(void); extern void CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv); extern List * ExplainAnalyzeTaskList(List *originalTaskList, TupleDestination *defaultTupleDest, TupleDesc tupleDesc, ParamListInfo params); extern bool RequestedForExplainAnalyze(CitusScanState *node); extern void ResetExplainAnalyzeData(List *taskList); #endif /* MULTI_EXPLAIN_H */ ================================================ FILE: src/include/distributed/multi_join_order.h ================================================ /*------------------------------------------------------------------------- * * multi_join_order.h * * Type and function declarations for determining a left-only join order for a * distributed query plan. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef MULTI_JOIN_ORDER_H #define MULTI_JOIN_ORDER_H #include "postgres.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" /* * JoinRuleType determines the type of the join rule that applies between two * tables or row sources. The rule types are ordered below according to their * costs, with the cheapes rule appearing at the top. Note that changing the * order of these enums *will* change the order in which the rules are applied. */ typedef enum JoinRuleType { JOIN_RULE_INVALID_FIRST = 0, REFERENCE_JOIN = 1, LOCAL_PARTITION_JOIN = 2, SINGLE_HASH_PARTITION_JOIN = 3, SINGLE_RANGE_PARTITION_JOIN = 4, DUAL_PARTITION_JOIN = 5, CARTESIAN_PRODUCT_REFERENCE_JOIN = 6, CARTESIAN_PRODUCT = 7, /* * Add new join rule types above this comment. After adding, you must also * update these arrays: RuleEvalFunctionArray, RuleApplyFunctionArray, and * RuleNameArray. */ JOIN_RULE_LAST } JoinRuleType; /* * TableEntry represents a table used when determining the join order. A table * entry corresponds to an ordinary relation reference (RTE_RELATION) in the * query range table list. */ typedef struct TableEntry { Oid relationId; uint32 rangeTableId; } TableEntry; /* * JoinOrderNode represents an element in the join order list; and this list * keeps the total join order for a distributed query. The first node in this * list later becomes the leftmost table in the join tree, and the successive * elements in the list are the joining tables in the left-deep tree. */ typedef struct JoinOrderNode { TableEntry *tableEntry; /* this node's relation and range table id */ JoinRuleType joinRuleType; /* not relevant for the first table */ JoinType joinType; /* not relevant for the first table */ /* * We keep track of all unique partition columns in the relation to correctly find * join clauses that can be applied locally. */ List *partitionColumnList; char partitionMethod; List *joinClauseList; /* not relevant for the first table */ TableEntry *anchorTable; } JoinOrderNode; /* Config variables managed via guc.c */ extern bool LogMultiJoinOrder; extern bool EnableSingleHashRepartitioning; /* Function declaration for determining table join orders */ extern List * JoinExprList(FromExpr *fromExpr); extern List * JoinOrderList(List *rangeTableEntryList, List *joinClauseList); extern bool IsApplicableJoinClause(List *leftTableIdList, uint32 rightTableId, Node *joinClause); extern List * ApplicableJoinClauses(List *leftTableIdList, uint32 rightTableId, List *joinClauseList); extern bool NodeIsEqualsOpExpr(Node *node); extern bool IsSupportedReferenceJoin(JoinType joinType, bool leftIsReferenceTable, bool rightIsReferenceTable); extern OpExpr * SinglePartitionJoinClause(List *partitionColumnList, List *applicableJoinClauses, bool *foundTypeMismatch); extern OpExpr * DualPartitionJoinClause(List *applicableJoinClauses); extern Var * LeftColumnOrNULL(OpExpr *joinClause); extern Var * RightColumnOrNULL(OpExpr *joinClause); extern Var * PartitionColumn(Oid relationId, uint32 rangeTableId); extern Var * DistPartitionKey(Oid relationId); extern Var * DistPartitionKeyOrError(Oid relationId); extern char PartitionMethod(Oid relationId); extern char TableReplicationModel(Oid relationId); extern bool JoinOnColumns(List *currentPartitionColumnList, Var *candidatePartitionColumn, List *joinClauseList); #endif /* MULTI_JOIN_ORDER_H */ ================================================ FILE: src/include/distributed/multi_logical_optimizer.h ================================================ /*------------------------------------------------------------------------- * * multi_logical_optimizer.h * Type and function declarations for optimizing multi-relation based logical * plans. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef MULTI_LOGICAL_OPTIMIZER_H #define MULTI_LOGICAL_OPTIMIZER_H #include "distributed/metadata_utility.h" #include "distributed/multi_logical_planner.h" #include "distributed/relation_restriction_equivalence.h" /* Definitions local to logical plan optimizer */ #define DIVISION_OPER_NAME "/" #define DISABLE_LIMIT_APPROXIMATION -1 #define DISABLE_DISTINCT_APPROXIMATION 0.0 #define ARRAY_CAT_AGGREGATE_NAME "array_cat_agg" #define JSONB_CAT_AGGREGATE_NAME "jsonb_cat_agg" #define JSON_CAT_AGGREGATE_NAME "json_cat_agg" #define WORKER_PARTIAL_AGGREGATE_NAME "worker_partial_agg" #define COORD_COMBINE_AGGREGATE_NAME "coord_combine_agg" #define WORKER_BINARY_PARTIAL_AGGREGATE_NAME "worker_binary_partial_agg" #define COORD_BINARY_COMBINE_AGGREGATE_NAME "coord_binary_combine_agg" #define WORKER_COLUMN_FORMAT "worker_column_%d" /* Definitions related to count(distinct) approximations */ #define HLL_EXTENSION_NAME "hll" #define HLL_TYPE_NAME "hll" #define HLL_HASH_INTEGER_FUNC_NAME "hll_hash_integer" #define HLL_HASH_BIGINT_FUNC_NAME "hll_hash_bigint" #define HLL_HASH_TEXT_FUNC_NAME "hll_hash_text" #define HLL_HASH_ANY_FUNC_NAME "hll_hash_any" #define HLL_ADD_AGGREGATE_NAME "hll_add_agg" #define HLL_UNION_AGGREGATE_NAME "hll_union_agg" #define HLL_CARDINALITY_FUNC_NAME "hll_cardinality" #define HLL_FORCE_GROUPAGG_GUC_NAME "hll.force_groupagg" /* Definitions related to Top-N approximations */ #define TOPN_ADD_AGGREGATE_NAME "topn_add_agg" #define TOPN_UNION_AGGREGATE_NAME "topn_union_agg" /* * AggregateType represents an aggregate function's type, where the function is * used in the context of a query. We use this function type to determine how to * modify the plan when creating the logical distributed plan. * * Please note that the order of values in this enumeration is tied to the order * of elements in the following AggregateNames array. This order needs to be * preserved. */ typedef enum { AGGREGATE_INVALID_FIRST = 0, AGGREGATE_AVERAGE = 1, AGGREGATE_MIN = 2, AGGREGATE_MAX = 3, AGGREGATE_SUM = 4, AGGREGATE_COUNT = 5, AGGREGATE_ARRAY_AGG = 6, AGGREGATE_JSONB_AGG = 7, AGGREGATE_JSONB_OBJECT_AGG = 8, AGGREGATE_JSON_AGG = 9, AGGREGATE_JSON_OBJECT_AGG = 10, AGGREGATE_BIT_AND = 11, AGGREGATE_BIT_OR = 12, AGGREGATE_BOOL_AND = 13, AGGREGATE_BOOL_OR = 14, AGGREGATE_EVERY = 15, AGGREGATE_HLL_ADD = 16, AGGREGATE_HLL_UNION = 17, AGGREGATE_TOPN_ADD_AGG = 18, AGGREGATE_TOPN_UNION_AGG = 19, AGGREGATE_ANY_VALUE = 20, /* support for github.com/tvondra/tdigest */ AGGREGATE_TDIGEST_COMBINE = 21, AGGREGATE_TDIGEST_ADD_DOUBLE = 22, AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLE = 23, AGGREGATE_TDIGEST_PERCENTILE_ADD_DOUBLEARRAY = 24, AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLE = 25, AGGREGATE_TDIGEST_PERCENTILE_TDIGEST_DOUBLEARRAY = 26, AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLE = 27, AGGREGATE_TDIGEST_PERCENTILE_OF_ADD_DOUBLEARRAY = 28, AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLE = 29, AGGREGATE_TDIGEST_PERCENTILE_OF_TDIGEST_DOUBLEARRAY = 30, /* AGGREGATE_CUSTOM must come last */ AGGREGATE_CUSTOM_COMBINE = 31, AGGREGATE_CUSTOM_ROW_GATHER = 32, } AggregateType; /* Enumeration for citus.coordinator_aggregation GUC */ typedef enum { COORDINATOR_AGGREGATION_DISABLED, COORDINATOR_AGGREGATION_ROW_GATHER, } CoordinatorAggregationStrategyType; /* * PushDownStatus indicates whether a node can be pushed down below its child * using the commutative and distributive relational algebraic properties. */ typedef enum { PUSH_DOWN_INVALID_FIRST = 0, PUSH_DOWN_VALID = 1, PUSH_DOWN_NOT_VALID = 2, PUSH_DOWN_SPECIAL_CONDITIONS = 3 } PushDownStatus; /* * PullUpStatus indicates whether a node can be pulled up above its parent using * the commutative and factorizable relational algebraic properties. */ typedef enum { PULL_UP_INVALID_FIRST = 0, PULL_UP_VALID = 1, PULL_UP_NOT_VALID = 2 } PullUpStatus; /* * AggregateNames is an array that stores cstring names for aggregate functions; * these cstring names act as an intermediary when mapping aggregate function * oids to AggregateType enumerations. For this mapping to occur, we use the * aggregate function oid to find the corresponding cstring name in pg_proc. We * then compare that name against entries in this array, and return the * appropriate AggregateType value. * * Please note that the order of elements in this array is tied to the order of * values in the preceding AggregateType enum. This order needs to be preserved. */ static const char *const AggregateNames[] = { "invalid", "avg", "min", "max", "sum", "count", "array_agg", "jsonb_agg", "jsonb_object_agg", "json_agg", "json_object_agg", "bit_and", "bit_or", "bool_and", "bool_or", "every", "hll_add_agg", "hll_union_agg", "topn_add_agg", "topn_union_agg", "any_value" }; /* Config variable managed via guc.c */ extern int LimitClauseRowFetchCount; extern double CountDistinctErrorRate; extern int CoordinatorAggregationStrategy; /* Function declaration for optimizing logical plans */ extern void MultiLogicalPlanOptimize(MultiTreeRoot *multiTree); /* Function declaration for getting partition method for the given relation */ extern char PartitionMethod(Oid relationId); /* Function declaration for helper functions in subquery pushdown */ extern bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column); extern List * SubqueryMultiTableList(MultiNode *multiNode); extern List * GroupTargetEntryList(List *groupClauseList, List *targetEntryList); extern bool ExtractQueryWalker(Node *node, List **queryList); extern bool IsPartitionColumn(Expr *columnExpression, Query *query, bool skipOuterVars); extern void FindReferencedTableColumn(Expr *columnExpression, List *parentQueryList, Query *query, Var **column, RangeTblEntry **rteContainingReferencedColumn, bool skipOuterVars); extern char * WorkerColumnName(AttrNumber resno); extern bool IsGroupBySubsetOfDistinct(List *groupClauses, List *distinctClauses); extern bool TargetListHasAggregates(List *targetEntryList); #endif /* MULTI_LOGICAL_OPTIMIZER_H */ ================================================ FILE: src/include/distributed/multi_logical_planner.h ================================================ /*------------------------------------------------------------------------- * * multi_logical_planner.h * Type declarations for multi-relational algebra operators, and function * declarations for building logical plans over distributed relations. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef MULTI_LOGICAL_PLANNER_H #define MULTI_LOGICAL_PLANNER_H #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_join_order.h" #include "distributed/relation_restriction_equivalence.h" #define SUBQUERY_RANGE_TABLE_ID -1 #define SUBQUERY_RELATION_ID 10000 #define SUBQUERY_PUSHDOWN_RELATION_ID 10001 /* * MultiNode represents the base node type for all multi-relational algebra * nodes. By creating this base node, we can simulate inheritance and represent * derived operator nodes as a MultiNode type. A similar structure to simulate * inheritance is also used in PostgreSQL's plan nodes. */ typedef struct MultiNode { CitusNode type; struct MultiNode *parentNode; /* child node(s) are defined in unary and binary nodes */ } MultiNode; /* Represents unary nodes that have only one child */ typedef struct MultiUnaryNode { MultiNode node; struct MultiNode *childNode; } MultiUnaryNode; /* Represents binary nodes that have two children */ typedef struct MultiBinaryNode { MultiNode node; struct MultiNode *leftChildNode; struct MultiNode *rightChildNode; } MultiBinaryNode; /* * MultiTreeRoot keeps a pointer to the root node in the multi-relational * operator tree. This node is always on the top of every logical plan. */ typedef struct MultiTreeRoot { MultiUnaryNode unaryNode; } MultiTreeRoot; /* * MultiTable represents a distributed table in a logical query plan. Note that * this node does not represent a query operator, and differs from the nodes * that follow in that sense. */ typedef struct MultiTable { MultiUnaryNode unaryNode; Oid relationId; int rangeTableId; Var *partitionColumn; Alias *alias; Alias *referenceNames; Query *subquery; /* this field is only valid for non-relation subquery types */ bool includePartitions; /* FROM .. TABLESAMPLE clause */ TableSampleClause *tablesample; } MultiTable; /* Defines the columns to project in multi-relational algebra */ typedef struct MultiProject { MultiUnaryNode unaryNode; List *columnList; } MultiProject; /* * MultiCollect defines the Collect operator in multi-relational algebra. This * operator collects data from remote nodes; the collected data are output from * the query operator that is beneath this Collect in the logical query tree. */ typedef struct MultiCollect { MultiUnaryNode unaryNode; } MultiCollect; /* * MultiSelect defines the MultiSelect operator in multi-relational algebra. * This operator contains select predicates which apply to a selected logical * relation. */ typedef struct MultiSelect { MultiUnaryNode unaryNode; List *selectClauseList; } MultiSelect; /* * MultiJoin joins the output of two query operators that are beneath it in the * query tree. The operator also keeps the join rule that applies between the * two operators, and the partition key to use if the join is distributed. */ typedef struct MultiJoin { MultiBinaryNode binaryNode; List *joinClauseList; JoinRuleType joinRuleType; JoinType joinType; } MultiJoin; /* Defines the (re-)Partition operator in multi-relational algebra */ typedef struct MultiPartition { MultiUnaryNode unaryNode; Var *partitionColumn; uint32 splitPointTableId; } MultiPartition; /* Defines the CartesianProduct operator in multi-relational algebra */ typedef struct MultiCartesianProduct { MultiBinaryNode binaryNode; } MultiCartesianProduct; /* * MultiExtendedOp defines a set of extended operators that operate on columns * in relational algebra. This node allows us to distinguish between operations * in the coordinator and worker nodes, and also captures the following: * * (1) Aggregate functions such as sums or averages; * (2) Grouping of attributes; these groupings may also be tied to aggregates; * (3) Extended projection expressions including columns, arithmetic and string * functions; * (4) User's intented display information, such as column display order; * (5) Sort clauses on columns, expressions, or aggregates; and * (6) Limit count and offset clause. */ typedef struct MultiExtendedOp { MultiUnaryNode unaryNode; List *targetList; List *groupClauseList; List *sortClauseList; Node *limitCount; Node *limitOffset; LimitOption limitOption; Node *havingQual; List *distinctClause; List *windowClause; bool hasDistinctOn; bool hasWindowFuncs; bool onlyPushableWindowFunctions; } MultiExtendedOp; /* Function declarations for building logical plans */ extern MultiTreeRoot * MultiLogicalPlanCreate(Query *originalQuery, Query *queryTree, PlannerRestrictionContext * plannerRestrictionContext); extern bool FindNodeMatchingCheckFunction(Node *node, bool (*check)(Node *)); extern bool TargetListOnPartitionColumn(Query *query, List *targetEntryList); extern bool FindNodeMatchingCheckFunctionInRangeTableList(List *rtable, bool (*check)( Node *)); extern bool IsCitusTableRTE(Node *node); extern bool IsDistributedOrReferenceTableRTE(Node *node); extern bool IsDistributedTableRTE(Node *node); extern bool IsReferenceTableRTE(Node *node); extern bool IsTableWithDistKeyRTE(Node *node); extern bool IsCitusExtraDataContainerRelation(RangeTblEntry *rte); extern bool ContainsReadIntermediateResultFunction(Node *node); extern bool ContainsReadIntermediateResultArrayFunction(Node *node); extern bool IsReadIntermediateResultFunction(Node *node); extern char * FindIntermediateResultIdIfExists(RangeTblEntry *rte); extern MultiNode * ParentNode(MultiNode *multiNode); extern MultiNode * ChildNode(MultiUnaryNode *multiNode); extern MultiNode * GrandChildNode(MultiUnaryNode *multiNode); extern void SetChild(MultiUnaryNode *parent, MultiNode *child); extern void SetLeftChild(MultiBinaryNode *parent, MultiNode *leftChild); extern void SetRightChild(MultiBinaryNode *parent, MultiNode *rightChild); extern bool UnaryOperator(MultiNode *node); extern bool BinaryOperator(MultiNode *node); extern List * OutputTableIdList(MultiNode *multiNode); extern List * FindNodesOfType(MultiNode *node, int type); extern List * JoinClauseList(List *whereClauseList); extern bool IsJoinClause(Node *clause); extern List * SubqueryEntryList(Query *queryTree); extern DeferredErrorMessage * DeferErrorIfQueryNotSupported(Query *queryTree); extern List * WhereClauseList(FromExpr *fromExpr); extern List * QualifierList(FromExpr *fromExpr); extern List * TableEntryList(List *rangeTableList); extern List * UsedTableEntryList(Query *query); extern List * pull_var_clause_default(Node *node); extern bool OperatorImplementsEquality(Oid opno); extern DeferredErrorMessage * DeferErrorIfUnsupportedClause(List *clauseList); extern MultiProject * MultiProjectNode(List *targetEntryList); extern MultiExtendedOp * MultiExtendedOpNode(Query *queryTree, Query *originalQuery); extern DeferredErrorMessage * DeferErrorIfUnsupportedSubqueryRepartition(Query * subqueryTree); extern MultiNode * MultiNodeTree(Query *queryTree); #endif /* MULTI_LOGICAL_PLANNER_H */ ================================================ FILE: src/include/distributed/multi_logical_replication.h ================================================ /*------------------------------------------------------------------------- * * multi_logical_replication.h * * Declarations for public functions and variables used in logical replication * on the distributed tables while moving shards. * * Copyright (c) 2017, Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MULTI_LOGICAL_REPLICATION_H_ #define MULTI_LOGICAL_REPLICATION_H_ #include "c.h" #include "nodes/pg_list.h" #include "distributed/connection_management.h" #include "distributed/hash_helpers.h" #include "distributed/shard_cleaner.h" /* Config variables managed via guc.c */ extern int LogicalReplicationTimeout; extern bool PlacementMovedUsingLogicalReplicationInTX; /* * NodeAndOwner should be used as a key for structs that should be hashed by a * combination of node and owner. */ typedef struct NodeAndOwner { uint32_t nodeId; Oid tableOwnerId; } NodeAndOwner; assert_valid_hash_key2(NodeAndOwner, nodeId, tableOwnerId); /* * ReplicationSlotInfo stores the info that defines a replication slot. For * shard splits this information is built by parsing the result of the * 'worker_split_shard_replication_setup' UDF. */ typedef struct ReplicationSlotInfo { uint32 targetNodeId; Oid tableOwnerId; char *name; } ReplicationSlotInfo; /* * PublicationInfo stores the information that defines a publication. */ typedef struct PublicationInfo { NodeAndOwner key; char *name; List *shardIntervals; struct LogicalRepTarget *target; } PublicationInfo; /* * Stores information necesary to create all the th */ typedef struct LogicalRepTarget { /* * The Oid of the user that owns the shards in newShards. This Oid is the * Oid of the user on the coordinator, this Oid is likely different than * the Oid of the user on the logical replication source or target. */ Oid tableOwnerId; char *subscriptionName; /* * The name of the user that's used as the owner of the subscription. This * is not the same as the name of the user that matches tableOwnerId. * Instead we create a temporary user with the same permissions as that * user, with its only purpose being owning the subscription. */ char *subscriptionOwnerName; ReplicationSlotInfo *replicationSlot; PublicationInfo *publication; /* * The shardIntervals that we want to create on this logical replication * target. This can be different from the shard intervals that are part of * the publication for two reasons: * 1. The publication does not contain partitioned tables, only their * children. The partition parent tables ARE part of newShards. * 2. For shard splits the publication also contains dummy shards, these * ARE NOT part of newShards. */ List *newShards; /* * The superuserConnection is shared between all LogicalRepTargets that have * the same node. This can be initialized easily by using * CreateGroupedLogicalRepTargetsConnections. */ MultiConnection *superuserConnection; } LogicalRepTarget; /* * GroupedLogicalRepTargets groups LogicalRepTargets by node. This allows to * create a hashmap where we can filter by search by nodeId. Which is useful * because these targets can all use the same superuserConection for * management, which allows us to batch certain operations such as getting * state of the subscriptions. */ typedef struct GroupedLogicalRepTargets { uint32 nodeId; List *logicalRepTargetList; MultiConnection *superuserConnection; } GroupedLogicalRepTargets; /* * LogicalRepType is used for various functions to do something different for * shard moves than for shard splits. Such as using a different prefix for a * subscription name. */ typedef enum LogicalRepType { SHARD_MOVE, SHARD_SPLIT, } LogicalRepType; extern void LogicallyReplicateShards(List *shardList, char *sourceNodeName, int sourceNodePort, char *targetNodeName, int targetNodePort, bool skipInterShardRelationshipCreation); extern void ConflictWithIsolationTestingBeforeCopy(void); extern void ConflictWithIsolationTestingAfterCopy(void); extern void CreateReplicaIdentities(List *subscriptionInfoList); extern void CreateReplicaIdentitiesOnNode(List *shardList, char *nodeName, int32 nodePort); extern XLogRecPtr GetRemoteLogPosition(MultiConnection *connection); extern List * GetQueryResultStringList(MultiConnection *connection, char *query); extern MultiConnection * GetReplicationConnection(char *nodeName, int nodePort); extern void CreatePublications(MultiConnection *sourceConnection, HTAB *publicationInfoHash); extern void CreateSubscriptions(MultiConnection *sourceConnection, char *databaseName, List *subscriptionInfoList); extern char * CreateReplicationSlots(MultiConnection *sourceConnection, MultiConnection *sourceReplicationConnection, List *subscriptionInfoList, char *outputPlugin); extern void EnableSubscriptions(List *subscriptionInfoList); extern char * PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId); extern char * ReplicationSlotNameForNodeAndOwnerForOperation(LogicalRepType type, uint32_t nodeId, Oid ownerId, OperationId operationId); extern char * SubscriptionName(LogicalRepType type, Oid ownerId); extern char * SubscriptionRoleName(LogicalRepType type, Oid ownerId); extern void WaitForAllSubscriptionsToCatchUp(MultiConnection *sourceConnection, HTAB *groupedLogicalRepTargetsHash); extern void WaitForShardSubscriptionToCatchUp(MultiConnection *targetConnection, XLogRecPtr sourcePosition, Bitmapset *tableOwnerIds, char *operationPrefix); extern HTAB * CreateGroupedLogicalRepTargetsHash(List *subscriptionInfoList); extern void CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash, char *user, char *databaseName); extern void CloseGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash); extern void CompleteNonBlockingShardTransfer(List *shardList, MultiConnection *sourceConnection, HTAB *publicationInfoHash, List *logicalRepTargetList, HTAB *groupedLogicalRepTargetsHash, LogicalRepType type, bool skipInterShardRelationshipCreation); extern void CreateUncheckedForeignKeyConstraints(List *logicalRepTargetList); extern void CreatePartitioningHierarchy(List *logicalRepTargetList); #endif /* MULTI_LOGICAL_REPLICATION_H_ */ ================================================ FILE: src/include/distributed/multi_partitioning_utils.h ================================================ /* * multi_partitioning_utils.h * Utility functions declarations for declarative partitioning * * Copyright (c) Citus Data, Inc. */ #ifndef MULTI_PARTITIONING_UTILS_H_ #define MULTI_PARTITIONING_UTILS_H_ #include "nodes/pg_list.h" #include "distributed/metadata_utility.h" extern bool PartitionedTable(Oid relationId); extern bool PartitionedTableNoLock(Oid relationId); extern bool PartitionTable(Oid relationId); extern bool PartitionTableNoLock(Oid relationId); extern bool IsChildTable(Oid relationId); extern bool IsParentTable(Oid relationId); extern Oid PartitionParentOid(Oid partitionOid); extern Oid PartitionWithLongestNameRelationId(Oid parentRelationId); extern List * PartitionList(Oid parentRelationId); extern char * GenerateDetachPartitionCommand(Oid partitionTableId); extern List * GenerateDetachPartitionCommandRelationIdList(List *relationIds); extern char * GenerateAttachShardPartitionCommand(ShardInterval *shardInterval); extern char * GenerateAlterTableAttachPartitionCommand(Oid partitionTableId); extern List * GenerateAttachPartitionCommandRelationIdList(List *relationIds); extern char * GeneratePartitioningInformation(Oid tableId); extern void FixPartitionConstraintsOnWorkers(Oid relationId); extern void FixLocalPartitionConstraints(Oid relationId, int64 shardId); extern void FixPartitionShardIndexNames(Oid relationId, Oid parentIndexOid); extern List * ListShardsUnderParentRelation(Oid relationId); #endif /* MULTI_PARTITIONING_UTILS_H_ */ ================================================ FILE: src/include/distributed/multi_physical_planner.h ================================================ /*------------------------------------------------------------------------- * * multi_physical_planner.h * Type and function declarations used in creating the distributed execution * plan. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef MULTI_PHYSICAL_PLANNER_H #define MULTI_PHYSICAL_PLANNER_H #include "postgres.h" #include "c.h" #include "datatype/timestamp.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/array.h" #include "pg_version_constants.h" #include "distributed/citus_nodes.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/metadata_utility.h" #include "distributed/multi_logical_planner.h" #include "distributed/worker_manager.h" /* Definitions local to the physical planner */ #define NON_PRUNABLE_JOIN -1 #define RESERVED_HASHED_COLUMN_ID MaxAttrNumber extern int RepartitionJoinBucketCountPerNode; typedef enum CitusRTEKind { CITUS_RTE_RELATION = RTE_RELATION, /* ordinary relation reference */ CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ CITUS_RTE_JOIN = RTE_JOIN, /* join */ CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ CITUS_RTE_TABLEFUNC = RTE_TABLEFUNC, /* TableFunc(.., column list) */ CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ CITUS_RTE_NAMEDTUPLESTORE = RTE_NAMEDTUPLESTORE, /* tuplestore, e.g. for triggers */ CITUS_RTE_RESULT = RTE_RESULT, /* RTE represents an empty FROM clause */ CITUS_RTE_SHARD, CITUS_RTE_REMOTE_QUERY } CitusRTEKind; /* Enumeration that defines the partition type for a remote job */ typedef enum { PARTITION_INVALID_FIRST = 0, RANGE_PARTITION_TYPE = 1, SINGLE_HASH_PARTITION_TYPE = 2, DUAL_HASH_PARTITION_TYPE = 3 } PartitionType; /* Enumeration that defines different task types */ typedef enum { TASK_TYPE_INVALID_FIRST, READ_TASK, MAP_TASK, MERGE_TASK, MAP_OUTPUT_FETCH_TASK, MERGE_FETCH_TASK, MODIFY_TASK, DDL_TASK, VACUUM_ANALYZE_TASK } TaskType; /* Enumeration that defines the task assignment policy to use */ typedef enum { TASK_ASSIGNMENT_INVALID_FIRST = 0, TASK_ASSIGNMENT_GREEDY = 1, TASK_ASSIGNMENT_ROUND_ROBIN = 2, TASK_ASSIGNMENT_FIRST_REPLICA = 3 } TaskAssignmentPolicyType; /* Enumeration that defines different job types */ typedef enum { JOB_INVALID_FIRST = 0, JOIN_MAP_MERGE_JOB = 1, TOP_LEVEL_WORKER_JOB = 2 } BoundaryNodeJobType; /* Enumeration that specifies extent of DML modifications */ typedef enum RowModifyLevel { ROW_MODIFY_NONE = 0, ROW_MODIFY_READONLY = 1, ROW_MODIFY_COMMUTATIVE = 2, ROW_MODIFY_NONCOMMUTATIVE = 3 } RowModifyLevel; /* * LocalPlannedStatement represents a local plan of a shard. The scope * for the LocalPlannedStatement is Task. */ typedef struct LocalPlannedStatement { CitusNode type; uint64 shardId; uint32 localGroupId; PlannedStmt *localPlan; } LocalPlannedStatement; /* * Job represents a logical unit of work that contains one set of data transfers * in our physical plan. The physical planner maps each SQL query into one or * more jobs depending on the query's complexity, and sets dependencies between * these jobs. Each job consists of multiple executable tasks; and these tasks * either operate on base shards, or repartitioned tables. */ typedef struct Job { CitusNode type; uint64 jobId; Query *jobQuery; List *taskList; List *dependentJobList; bool subqueryPushdown; bool requiresCoordinatorEvaluation; /* only applies to modify jobs */ bool deferredPruning; Const *partitionKeyValue; /* for local shard queries, we may save the local plan here */ List *localPlannedStatements; /* * When we evaluate functions and parameters in jobQuery then we * should no longer send the list of parameters along with the * query. */ bool parametersInJobQueryResolved; uint32 colocationId; /* common colocation group ID of the relations */ } Job; /* Defines a repartitioning job and holds additional related data. */ typedef struct MapMergeJob { Job job; PartitionType partitionType; Var *partitionColumn; uint32 partitionCount; int sortedShardIntervalArrayLength; ShardInterval **sortedShardIntervalArray; /* only applies to range partitioning */ List *mapTaskList; List *mergeTaskList; } MapMergeJob; typedef enum TaskQueryType { TASK_QUERY_NULL, TASK_QUERY_TEXT, TASK_QUERY_OBJECT, TASK_QUERY_TEXT_LIST, TASK_QUERY_LOCAL_PLAN, } TaskQueryType; typedef struct LocalCompilation { PlannedStmt *plan; /* the local plan for this task */ Query *query; /* query to deparse for EXPLAIN ANALYZE or local command logging */ } LocalCompilation; typedef struct TaskQuery { TaskQueryType queryType; union { /* * For most queries jobQueryReferenceForLazyDeparsing and/or queryStringLazy is not * NULL. This means we have a single query for all placements. * * If this is not the case, the length of perPlacementQueryStrings is * non-zero and equal to length of taskPlacementList. Like this it can * assign a different query for each placement. We need this flexibility * when a query should return node specific values. For example, on which * node did we succeed storing some result files? * * jobQueryReferenceForLazyDeparsing is only not null when the planner thinks the * query could possibly be locally executed. In that case deparsing+parsing * the query might not be necessary, so we do that lazily. * * jobQueryReferenceForLazyDeparsing should only be set by using SetTaskQueryIfShouldLazyDeparse() */ Query *jobQueryReferenceForLazyDeparsing; /* * In almost all cases queryStringLazy should be read only indirectly by * using TaskQueryString(). This will populate the field if only the * jobQueryReferenceForLazyDeparsing field is not NULL. * * This field should only be set by using SetTaskQueryString() (or as a * side effect from TaskQueryString()). Otherwise it might not be in sync * with jobQueryReferenceForLazyDeparsing. */ char *queryStringLazy; /* * queryStringList contains query strings. They should be * run sequentially. The concatenated version of this list * will already be set for queryStringLazy, this can be useful * when we want to access each query string. */ List *queryStringList; /* * For tasks that can be executed locally, this field contains the * local plan for the task. This is only set when the shard that the * task is assigned to is local to the node that executes the task. * The query field is used to deparse the query for EXPLAIN ANALYZE * or local command logging. */ LocalCompilation *localCompiled; /* only applies to local tasks */ }data; }TaskQuery; struct TupleDestination; typedef struct Task { CitusNode type; TaskType taskType; uint64 jobId; uint32 taskId; /* * taskQuery contains query string information. The way we get queryString can be different * so this is abstracted with taskQuery. */ TaskQuery taskQuery; /* * A task can have multiple queries, in which case queryCount will be > 1, and * taskQuery->queryType == TASK_QUERY_TEXT_LIST. */ int queryCount; Oid anchorDistributedTableId; /* only applies to insert tasks */ uint64 anchorShardId; /* only applies to compute tasks */ List *taskPlacementList; /* only applies to compute tasks */ List *dependentTaskList; /* only applies to compute tasks */ uint32 partitionId; uint32 upstreamTaskId; /* only applies to data fetch tasks */ ShardInterval *shardInterval; /* only applies to merge tasks */ bool assignmentConstrained; /* only applies to merge tasks */ /* for merge tasks, this is set to the target list of the map task */ List *mapJobTargetList; char replicationModel; /* only applies to modify tasks */ /* * List of struct RelationRowLock. This contains an entry for each * query identified as a FOR [KEY] UPDATE/SHARE target. Citus * converts PostgreSQL's RowMarkClause to RelationRowLock in * RowLocksOnRelations(). */ List *relationRowLockList; bool modifyWithSubquery; /* * List of struct RelationShard. This represents the mapping of relations * in the RTE list to shard IDs for a task for the purposes of: * - Locking: See AcquireExecutorShardLocks() * - Deparsing: See UpdateRelationToShardNames() * - Relation Access Tracking */ List *relationShardList; List *rowValuesLists; /* rows to use when building multi-row INSERT */ /* * Used only when local execution happens. Indicates that this task is part of * both local and remote executions. We use "or" in the field name because this * is set to true for both the remote and local tasks generated for such * executions. The most common example is modifications to reference tables where * the task splitted into local and remote tasks. */ bool partiallyLocalOrRemote; /* * When we evaluate functions and parameters in the query string then * we should no longer send the list of parameters along with the * query. */ bool parametersInQueryStringResolved; /* * Destination of tuples generated as a result of executing this task. Can be * NULL, in which case executor might use a default destination. */ struct TupleDestination *tupleDest; /* * totalReceivedTupleData only counts the data for a single placement. So * for RETURNING DML this is not really correct. This is used by * EXPLAIN ANALYZE, to display the amount of received bytes. The local execution * does not increment this value, so only used for remote execution. */ uint64 totalReceivedTupleData; /* * EXPLAIN ANALYZE output fetched from worker. This is saved to be used later * by RemoteExplain(). */ char *fetchedExplainAnalyzePlan; int fetchedExplainAnalyzePlacementIndex; /* * Execution Duration fetched from worker. This is saved to be used later by * ExplainTaskList(). */ double fetchedExplainAnalyzeExecutionDuration; /* * isLocalTableModification is true if the task is on modifying a local table. */ bool isLocalTableModification; /* * Vacuum, create/drop/reindex concurrently cannot be executed in a transaction. */ bool cannotBeExecutedInTransaction; Const *partitionKeyValue; int colocationId; } Task; /* * RangeTableFragment represents a fragment of a range table. This fragment * could be a regular shard or a merged table formed in a MapMerge job. */ typedef struct RangeTableFragment { CitusRTEKind fragmentType; void *fragmentReference; uint32 rangeTableId; } RangeTableFragment; /* * JoinSequenceNode represents a range table in an ordered sequence of tables * joined together. This representation helps build combinations of all range * table fragments during task generation. */ typedef struct JoinSequenceNode { uint32 rangeTableId; int32 joiningRangeTableId; } JoinSequenceNode; /* * ModifyWithSelectMethod represents the method to use for INSERT INTO ... SELECT * or MERGE type of queries. * * Note that there is a third method which is not represented here, which is * pushing down the MERGE/INSERT INTO ... SELECT to workers. This method is * executed similar to other distributed queries and doesn't need a special * execution code, so we don't need to represent it here. */ typedef enum ModifyWithSelectMethod { MODIFY_WITH_SELECT_VIA_COORDINATOR, MODIFY_WITH_SELECT_REPARTITION } ModifyWithSelectMethod; /* * DistributedPlan contains all information necessary to execute a * distribute query. */ typedef struct DistributedPlan { CitusNode type; /* unique identifier of the plan within the session */ uint64 planId; /* specifies nature of modifications in query */ RowModifyLevel modLevel; /* * specifies whether plan returns results, * either as a SELECT or a DML which has RETURNING. */ bool expectResults; /* job tree containing the tasks to be executed on workers */ Job *workerJob; /* local query that merges results from the workers */ Query *combineQuery; /* query identifier (copied from the top-level PlannedStmt) */ uint64 queryId; /* which relations are accessed by this distributed plan */ List *relationIdList; /* target relation of a modification */ Oid targetRelationId; /* * Modifications performed using the output of a source query via * the coordinator or repartition. */ Query *modifyQueryViaCoordinatorOrRepartition; PlannedStmt *selectPlanForModifyViaCoordinatorOrRepartition; ModifyWithSelectMethod modifyWithSelectMethod; /* * If intermediateResultIdPrefix is non-null, the source query * results are written to a set of intermediate results named * according to _. * That way we can run a distributed modification query which * requires evaluating source query results at the coordinator. * Once results are captured in intermediate files, modification * is done from the intermediate results into the target relation. * */ char *intermediateResultIdPrefix; /* list of subplans to execute before the distributed query */ List *subPlanList; /* * List of subPlans that are used in the DistributedPlan * Note that this is different that "subPlanList" field which * contains the subplans generated as part of the DistributedPlan. * * On the other hand, usedSubPlanNodeList keeps track of which subPlans * are used within this distributed plan as a list of * UsedDistributedSubPlan pointers. * * The list may contain duplicates if the subplan is referenced multiple * times (e.g. a CTE appears in the query tree multiple times). */ List *usedSubPlanNodeList; /* * When the query is very simple such that we don't need to call * standard_planner(). See FastPathRouterQuery() for the definition. */ bool fastPathRouterPlan; /* number of times this plan has been used (as a prepared statement) */ uint32 numberOfTimesExecuted; /* * NULL if this a valid plan, an error description otherwise. This will * e.g. be set if SQL features are present that a planner doesn't support, * or if prepared statement parameters prevented successful planning. */ DeferredErrorMessage *planningError; /* * When performing query execution scenarios that require repartitioning * the source rows, this field stores the index of the column in the list * of source rows to be repartitioned for colocation with the target. */ int sourceResultRepartitionColumnIndex; } DistributedPlan; /* * SubPlanExplainOutputData Holds the EXPLAIN ANALYZE output and collected * statistics for a single task executed by a worker during distributed * query execution. * explainOutput — raw EXPLAIN ANALYZE output for the task * executionDuration — wall‑clock time taken to run the task * totalReceivedTupleData — total bytes of tuple data received from the worker */ typedef struct SubPlanExplainOutputData { char *explainOutput; double executionDuration; double executionNtuples; double executionNloops; uint64 totalReceivedTupleData; } SubPlanExplainOutputData; /* * DistributedSubPlan contains a subplan of a distributed plan. Subplans are * executed before the distributed query and their results are written to * temporary files. This is used to execute CTEs and subquery joins that * cannot be distributed. */ typedef struct DistributedSubPlan { CitusNode type; uint32 subPlanId; PlannedStmt *plan; /* EXPLAIN ANALYZE instrumentations */ uint64 bytesSentPerWorker; uint32 remoteWorkerCount; double durationMillisecs; bool writeLocalFile; SubPlanExplainOutputData *totalExplainOutput; uint32 numTasksOutput; /* actual size of the above array */ double ntuples; /* total tuples produced */ } DistributedSubPlan; /* defines how a subplan is used by a distributed query */ typedef enum SubPlanAccessType { SUBPLAN_ACCESS_NONE, SUBPLAN_ACCESS_LOCAL, SUBPLAN_ACCESS_REMOTE, SUBPLAN_ACCESS_ANYWHERE } SubPlanAccessType; /* * UsedDistributedSubPlan contains information about a subPlan that is used in a * distributed plan. */ typedef struct UsedDistributedSubPlan { CitusNode type; /* subplan used by the distributed query */ char *subPlanId; /* how the subplan is used by a distributed query */ SubPlanAccessType accessType; } UsedDistributedSubPlan; /* OperatorCacheEntry contains information for each element in OperatorCache */ typedef struct OperatorCacheEntry { /* cache key consists of typeId, accessMethodId and strategyNumber */ Oid typeId; Oid accessMethodId; int16 strategyNumber; Oid operatorId; Oid operatorClassInputType; char typeType; } OperatorCacheEntry; /* Named function pointer type for reordering Task lists */ typedef List *(*ReorderFunction)(List *); /* Config variable managed via guc.c */ extern int TaskAssignmentPolicy; extern bool EnableUniqueJobIds; /* Function declarations for building physical plans and constructing queries */ extern DistributedPlan * CreatePhysicalDistributedPlan(MultiTreeRoot *multiTree, PlannerRestrictionContext * plannerRestrictionContext); extern Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType, char *queryString); extern OpExpr * MakeOpExpressionExtended(Var *leftVar, Expr *rightArg, int16 strategyNumber); extern OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber); extern Node * WrapUngroupedVarsInAnyValueAggregate(Node *expression, List *groupClauseList, List *targetList, bool checkExpressionEquality); /* * Function declarations for building, updating constraints and simple operator * expression check. */ extern Node * BuildBaseConstraint(Var *column); extern void UpdateConstraint(Node *baseConstraint, ShardInterval *shardInterval); extern bool BinaryOpExpression(Expr *clause, Node **leftOperand, Node **rightOperand); /* helper functions */ extern Var * MakeInt4Column(void); extern int CompareShardPlacements(const void *leftElement, const void *rightElement); extern int CompareGroupShardPlacements(const void *leftElement, const void *rightElement); extern bool ShardIntervalsOverlap(ShardInterval *firstInterval, ShardInterval *secondInterval); extern bool ShardIntervalsOverlapWithParams(Datum firstMin, Datum firstMax, Datum secondMin, Datum secondMax, FmgrInfo *comparisonFunction, Oid collation); extern bool CoPartitionedTables(Oid firstRelationId, Oid secondRelationId); extern ShardInterval ** GenerateSyntheticShardIntervalArray(int partitionCount); extern RowModifyLevel RowModifyLevelForQuery(Query *query); extern StringInfo ArrayObjectToString(ArrayType *arrayObject, Oid columnType, int32 columnTypeMod); /* function declarations for Task and Task list operations */ extern bool TasksEqual(const Task *a, const Task *b); extern bool TaskListMember(const List *taskList, const Task *task); extern List * TaskListDifference(const List *list1, const List *list2); extern List * AssignAnchorShardTaskList(List *taskList); extern List * FirstReplicaAssignTaskList(List *taskList); extern List * RoundRobinAssignTaskList(List *taskList); extern List * RoundRobinReorder(List *placementList); extern void SetPlacementNodeMetadata(ShardPlacement *placement, WorkerNode *workerNode); extern int CompareTasksByTaskId(const void *leftElement, const void *rightElement); extern int CompareTasksByExecutionDuration(const void *leftElement, const void *rightElement); /* function declaration for creating Task */ extern List * QueryPushdownSqlTaskList(Query *query, uint64 jobId, RelationRestrictionContext * relationRestrictionContext, List *prunedRelationShardList, TaskType taskType, bool modifyRequiresCoordinatorEvaluation, DeferredErrorMessage **planningError); extern bool ModifyLocalTableJob(Job *job); /* function declarations for managing jobs */ extern uint64 UniqueJobId(void); extern List * DerivedColumnNameList(uint32 columnCount, uint64 generatingJobId); extern RangeTblEntry * DerivedRangeTableEntry(MultiNode *multiNode, List *columnList, List *tableIdList, List *funcColumnNames, List *funcColumnTypes, List *funcColumnTypeMods, List *funcCollations); extern List * FetchEqualityAttrNumsForRTE(Node *quals); #endif /* MULTI_PHYSICAL_PLANNER_H */ ================================================ FILE: src/include/distributed/multi_progress.h ================================================ /*------------------------------------------------------------------------- * * multi_progress.h * Declarations for public functions and variables used in progress * tracking functions in Citus. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MULTI_PROGRESS_H #define MULTI_PROGRESS_H #include "postgres.h" #include "fmgr.h" #include "nodes/pg_list.h" #include "storage/dsm.h" typedef struct ProgressMonitorData { uint64 processId; int stepCount; } ProgressMonitorData; extern ProgressMonitorData * CreateProgressMonitor(int stepCount, Size stepSize, dsm_handle *dsmHandle); extern void RegisterProgressMonitor(uint64 progressTypeMagicNumber, Oid relationId, dsm_handle dsmHandle); extern ProgressMonitorData * GetCurrentProgressMonitor(void); extern void FinalizeCurrentProgressMonitor(void); extern bool HasProgressMonitor(void); extern List * ProgressMonitorList(uint64 commandTypeMagicNumber, List **attachedDSMSegmentList); extern void DetachFromDSMSegments(List *dsmSegmentList); extern void * ProgressMonitorSteps(ProgressMonitorData *monitor); #endif /* MULTI_PROGRESS_H */ ================================================ FILE: src/include/distributed/multi_router_planner.h ================================================ /*------------------------------------------------------------------------- * * multi_router_planner.h * * Declarations for public functions and types related to router planning. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef MULTI_ROUTER_PLANNER_H #define MULTI_ROUTER_PLANNER_H #include "c.h" #include "nodes/parsenodes.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" /* reserved alias name for UPSERTs */ #define CITUS_TABLE_ALIAS "citus_table_alias" extern bool EnableRouterExecution; extern bool EnableFastPathRouterPlanner; extern bool EnableLocalFastPathQueryOptimization; extern bool EnableNonColocatedRouterQueryPushdown; extern DistributedPlan * CreateRouterPlan(Query *originalQuery, Query *query, PlannerRestrictionContext * plannerRestrictionContext); extern DistributedPlan * CreateModifyPlan(Query *originalQuery, Query *query, PlannerRestrictionContext * plannerRestrictionContext); extern DeferredErrorMessage * PlanRouterQuery(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext, List **placementList, uint64 *anchorShardId, List **relationShardList, List **prunedShardIntervalListList, bool replacePrunedQueryWithDummy, bool *multiShardModifyQuery, Const **partitionValueConst, bool *containOnlyLocalTable); extern List * RelationShardListForShardIntervalList(List *shardIntervalList, bool *shardsPresent); extern List * CreateTaskPlacementListForShardIntervals(List *shardIntervalList, bool shardsPresent, bool generateDummyPlacement, bool hasLocalRelation); extern List * RouterInsertTaskList(Query *query, bool parametersInQueryResolved, DeferredErrorMessage **planningError); extern Const * ExtractInsertPartitionKeyValue(Query *query); extern List * TargetShardIntervalsForRestrictInfo(RelationRestrictionContext * restrictionContext, bool *multiShardQuery, Const **partitionValueConst); extern List * PlacementsForWorkersContainingAllShards(List *shardIntervalListList); extern List * IntersectPlacementList(List *lhsPlacementList, List *rhsPlacementList); extern DeferredErrorMessage * ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuery, PlannerRestrictionContext * plannerRestrictionContext); extern DeferredErrorMessage * ErrorIfOnConflictNotSupported(Query *queryTree); extern List * ShardIntervalOpExpressions(ShardInterval *shardInterval, Index rteIndex); extern RelationRestrictionContext * CopyRelationRestrictionContext( RelationRestrictionContext *oldContext); extern Oid ExtractFirstCitusTableId(Query *query); extern RangeTblEntry * ExtractSelectRangeTableEntry(Query *query); extern Oid ModifyQueryResultRelationId(Query *query); extern RangeTblEntry * ExtractResultRelationRTE(Query *query); extern RangeTblEntry * ExtractResultRelationRTEOrError(Query *query); extern RangeTblEntry * ExtractDistributedInsertValuesRTE(Query *query); extern bool IsMultiRowInsert(Query *query); extern void AddPartitionKeyNotNullFilterToSelect(Query *subqery); extern bool UpdateOrDeleteOrMergeQuery(Query *query); extern bool IsMergeQuery(Query *query); extern uint64 GetAnchorShardId(List *relationShardList); extern List * TargetShardIntervalForFastPathQuery(Query *query, bool *isMultiShardQuery, Const *inputDistributionKeyValue, Const **outGoingPartitionValueConst); extern void GenerateSingleShardRouterTaskList(Job *job, List *relationShardList, List *placementList, uint64 shardId, bool isLocalTableModification, bool delayedFastPath); /* * FastPathPlanner is a subset of router planner, that's why we prefer to * keep the external function here. */extern PlannedStmt * GeneratePlaceHolderPlannedStmt(Query *parse); extern void FastPathPreprocessParseTree(Query *parse); extern PlannedStmt * FastPathPlanner(Query *originalQuery, Query *parse, ParamListInfo boundParams); extern bool FastPathRouterQuery(Query *query, FastPathRestrictionContext *fastPathContext); extern bool JoinConditionIsOnFalse(List *relOptInfo); extern Oid ResultRelationOidForQuery(Query *query); extern DeferredErrorMessage * TargetlistAndFunctionsSupported(Oid resultRelationId, FromExpr *joinTree, Node *quals, List *targetList, CmdType commandType, List *returningList); extern bool NodeIsFieldStore(Node *node); extern bool TargetEntryChangesValue(TargetEntry *targetEntry, Var *column, FromExpr *joinTree); extern bool MasterIrreducibleExpression(Node *expression, bool *varArgument, bool *badCoalesce); extern bool HasDangerousJoinUsing(List *rtableList, Node *jtnode); extern Job * RouterJob(Query *originalQuery, PlannerRestrictionContext *plannerRestrictionContext, DeferredErrorMessage **planningError); extern bool ContainsOnlyLocalOrReferenceTables(RTEListProperties *rteProperties); extern RangeTblEntry * ExtractSourceResultRangeTableEntry(Query *query); extern void CheckAndBuildDelayedFastPathPlan(DistributedPlanningContext *planContext, DistributedPlan *plan); #endif /* MULTI_ROUTER_PLANNER_H */ ================================================ FILE: src/include/distributed/multi_server_executor.h ================================================ /*------------------------------------------------------------------------- * * multi_server_executor.h * Type and function declarations for executing remote jobs from a backend; * the ensemble of these jobs form the distributed execution plan. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef MULTI_SERVER_EXECUTOR_H #define MULTI_SERVER_EXECUTOR_H #include "distributed/multi_physical_planner.h" #include "distributed/worker_manager.h" /* Adaptive executor repartitioning related defines */ #define WORKER_CREATE_SCHEMA_QUERY "SELECT worker_create_schema (" UINT64_FORMAT ", %s);" #define WORKER_REPARTITION_CLEANUP_QUERY "SELECT worker_repartition_cleanup (" \ UINT64_FORMAT \ ");" /* Enumeration that represents distributed executor types */ typedef enum { MULTI_EXECUTOR_INVALID_FIRST = 0, MULTI_EXECUTOR_ADAPTIVE = 1, MULTI_EXECUTOR_NON_PUSHABLE_INSERT_SELECT = 2, MULTI_EXECUTOR_NON_PUSHABLE_MERGE_QUERY = 3 } MultiExecutorType; /* Config variable managed via guc.c */ extern int RemoteTaskCheckInterval; extern int TaskExecutorType; extern bool EnableRepartitionJoins; extern int MultiTaskQueryLogLevel; /* Function declarations common to more than one executor */ extern MultiExecutorType JobExecutorType(DistributedPlan *distributedPlan); #endif /* MULTI_SERVER_EXECUTOR_H */ ================================================ FILE: src/include/distributed/namespace_utils.h ================================================ /*------------------------------------------------------------------------- * * namespace_utils.h * Utility function declarations related to namespace changes. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef NAMESPACE_UTILS_H #define NAMESPACE_UTILS_H extern int PushEmptySearchPath(void); extern void PopEmptySearchPath(int saveNestLevel); #endif /* NAMESPACE_UTILS_H */ ================================================ FILE: src/include/distributed/param_utils.h ================================================ /*------------------------------------------------------------------------- * param_utils.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PARAM_UTILS_H #define PARAM_UTILS_H extern bool GetParamsUsedInQuery(Node *expression, Bitmapset **paramBitmap); extern void MarkUnreferencedExternParams(Node *expression, ParamListInfo boundParams); #endif /* PARAM_UTILS_H */ ================================================ FILE: src/include/distributed/pg_dist_background_job.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_background_job.h * definition of the relation that holds the jobs metadata * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_PG_DIST_BACKGROUND_JOB_H #define CITUS_PG_DIST_BACKGROUND_JOB_H /* ---------------- * compiler constants for pg_dist_background_job * ---------------- */ #define Natts_pg_dist_background_job 6 #define Anum_pg_dist_background_job_job_id 1 #define Anum_pg_dist_background_job_state 2 #define Anum_pg_dist_background_job_job_type 3 #define Anum_pg_dist_background_job_description 4 #define Anum_pg_dist_background_job_started_at 5 #define Anum_pg_dist_background_job_finished_at 6 #endif /* CITUS_PG_DIST_BACKGROUND_JOB_H */ ================================================ FILE: src/include/distributed/pg_dist_background_task.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_background_task.h * definition of the relation that holds the tasks metadata * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_PG_DIST_BACKGROUND_TASK_H #define CITUS_PG_DIST_BACKGROUND_TASK_H /* ---------------- * compiler constants for pg_dist_background_task * ---------------- */ #define Natts_pg_dist_background_task 10 #define Anum_pg_dist_background_task_job_id 1 #define Anum_pg_dist_background_task_task_id 2 #define Anum_pg_dist_background_task_owner 3 #define Anum_pg_dist_background_task_pid 4 #define Anum_pg_dist_background_task_status 5 #define Anum_pg_dist_background_task_command 6 #define Anum_pg_dist_background_task_retry_count 7 #define Anum_pg_dist_background_task_not_before 8 #define Anum_pg_dist_background_task_message 9 #define Anum_pg_dist_background_task_nodes_involved 10 extern int GetNodesInvolvedAttrIndexInPgDistBackgroundTask(TupleDesc tupleDesc); #endif /* CITUS_PG_DIST_BACKGROUND_TASK_H */ ================================================ FILE: src/include/distributed/pg_dist_backrgound_task_depend.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_background_task_depend.h * definition of the relation that holds which tasks depend on each * other. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_PG_DIST_BACKGROUND_TASK_DEPEND_H #define CITUS_PG_DIST_BACKGROUND_TASK_DEPEND_H typedef struct FormData_pg_dist_background_task_depend { int64 job_id; int64 task_id; int64 depends_on; #ifdef CATALOG_VARLEN /* variable-length fields start here */ #endif } FormData_pg_dist_background_task_depend; typedef FormData_pg_dist_background_task_depend *Form_pg_dist_background_task_depend; /* ---------------- * compiler constants for pg_dist_background_task_depend * ---------------- */ #define Natts_pg_dist_background_task_depend 3 #define Anum_pg_dist_background_task_depend_job_id 1 #define Anum_pg_dist_background_task_depend_task_id 2 #define Anum_pg_dist_background_task_depend_depends_on 3 #endif /* CITUS_PG_DIST_BACKGROUND_TASK_DEPEND_H */ ================================================ FILE: src/include/distributed/pg_dist_cleanup.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_cleanup.h * definition of the relation that holds the resources to be cleaned up * in cluster (pg_dist_cleanup). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_CLEANUP_H #define PG_DIST_CLEANUP_H /* ---------------- * compiler constants for pg_dist_cleanup * ---------------- */ #define Natts_pg_dist_cleanup 6 #define Anum_pg_dist_cleanup_record_id 1 #define Anum_pg_dist_cleanup_operation_id 2 #define Anum_pg_dist_cleanup_object_type 3 #define Anum_pg_dist_cleanup_object_name 4 #define Anum_pg_dist_cleanup_node_group_id 5 #define Anum_pg_dist_cleanup_policy_type 6 #define PG_CATALOG "pg_catalog" #define PG_DIST_CLEANUP "pg_dist_cleanup" #define OPERATIONID_SEQUENCE_NAME "pg_dist_operationid_seq" #define CLEANUPRECORDID_SEQUENCE_NAME "pg_dist_cleanup_recordid_seq" #endif /* PG_DIST_CLEANUP_H */ ================================================ FILE: src/include/distributed/pg_dist_colocation.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_colocation.h * definition of the relation that holds the colocation information on the * cluster (pg_dist_colocation). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_COLOCATION_H #define PG_DIST_COLOCATION_H /* ---------------- * pg_dist_colocation definition. * ---------------- */ typedef struct FormData_pg_dist_colocation { uint32 colocationid; uint32 shardcount; uint32 replicationfactor; Oid distributioncolumntype; Oid distributioncolumncollation; } FormData_pg_dist_colocation; /* ---------------- * Form_pg_dist_colocation corresponds to a pointer to a tuple with * the format of pg_dist_colocation relation. * ---------------- */ typedef FormData_pg_dist_colocation *Form_pg_dist_colocation; /* ---------------- * compiler constants for pg_dist_colocation * ---------------- */ #define Natts_pg_dist_colocation 5 #define Anum_pg_dist_colocation_colocationid 1 #define Anum_pg_dist_colocation_shardcount 2 #define Anum_pg_dist_colocation_replicationfactor 3 #define Anum_pg_dist_colocation_distributioncolumntype 4 #define Anum_pg_dist_colocation_distributioncolumncollation 5 #define COLOCATIONID_SEQUENCE_NAME "pg_dist_colocationid_seq" #endif /* PG_DIST_COLOCATION_H */ ================================================ FILE: src/include/distributed/pg_dist_local_group.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_local_group.h * definition of the relation that holds the local group id (pg_dist_local_group). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_LOCAL_GROUP_H #define PG_DIST_LOCAL_GROUP_H /* ---------------- * pg_dist_local_group definition. * ---------------- */ typedef struct FormData_pg_dist_local_group { int groupid; } FormData_pg_dist_local_group; /* ---------------- * FormData_pg_dist_local_group corresponds to a pointer to a tuple with * the format of pg_dist_local_group relation. * ---------------- */ typedef FormData_pg_dist_local_group *Form_pg_dist_local_group; /* ---------------- * compiler constants for pg_dist_local_group * ---------------- */ #define Natts_pg_dist_local_group 1 #define Anum_pg_dist_local_groupid 1 #endif /* PG_DIST_LOCAL_GROUP_H */ ================================================ FILE: src/include/distributed/pg_dist_node.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_node.h * definition of the relation that holds the nodes on the cluster (pg_dist_node). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_NODE_H #define PG_DIST_NODE_H /* ---------------- * compiler constants for pg_dist_node * ---------------- * * n.b. citus_add_node, citus_add_inactive_node, and citus_activate_node all * directly return pg_dist_node tuples. This means their definitions (and * in particular their OUT parameters) must be changed whenever the definition of * pg_dist_node changes. */ #define Natts_pg_dist_node 13 #define Anum_pg_dist_node_nodeid 1 #define Anum_pg_dist_node_groupid 2 #define Anum_pg_dist_node_nodename 3 #define Anum_pg_dist_node_nodeport 4 #define Anum_pg_dist_node_noderack 5 #define Anum_pg_dist_node_hasmetadata 6 #define Anum_pg_dist_node_isactive 7 #define Anum_pg_dist_node_noderole 8 #define Anum_pg_dist_node_nodecluster 9 #define Anum_pg_dist_node_metadatasynced 10 #define Anum_pg_dist_node_shouldhaveshards 11 #define Anum_pg_dist_node_nodeisclone 12 #define Anum_pg_dist_node_nodeprimarynodeid 13 #define GROUPID_SEQUENCE_NAME "pg_dist_groupid_seq" #define NODEID_SEQUENCE_NAME "pg_dist_node_nodeid_seq" #endif /* PG_DIST_NODE_H */ ================================================ FILE: src/include/distributed/pg_dist_node_metadata.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_node_metadata.h * definition of the relation that holds the metadata table (pg_dist_node_metadata). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_NODE_METADATA_H #define PG_DIST_NODE_METADATA_H /* ---------------- * compiler constants for pg_dist_node_metadata * ---------------- */ #define Natts_pg_dist_node_metadata 1 #define Anum_pg_dist_node_metadata_metadata 1 extern int FindCoordinatorNodeId(void); #endif /* PG_DIST_NODE_METADATA_H */ ================================================ FILE: src/include/distributed/pg_dist_partition.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_partition.h * definition of the system "remote partition" relation (pg_dist_partition). * * This table keeps metadata on logical tables that the user requested remote * partitioning for (smaller physical tables that we partition data to are * handled in another system catalog). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_PARTITION_H #define PG_DIST_PARTITION_H /* ---------------- * pg_dist_partition definition. * ---------------- */ typedef struct FormData_pg_dist_partition { Oid logicalrelid; /* logical relation id; references pg_class oid */ char partmethod; /* partition method; see codes below */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ text partkey; /* partition key expression */ uint32 colocationid; /* id of the co-location group of particular table belongs to */ char repmodel; /* replication model; see codes below */ #endif bool autoconverted; } FormData_pg_dist_partition; /* ---------------- * Form_pg_dist_partitions corresponds to a pointer to a tuple with * the format of pg_dist_partitions relation. * ---------------- */ typedef FormData_pg_dist_partition *Form_pg_dist_partition; /* ---------------- * compiler constants for pg_dist_partitions * ---------------- */ #define Natts_pg_dist_partition 6 #define Anum_pg_dist_partition_logicalrelid 1 #define Anum_pg_dist_partition_partmethod 2 #define Anum_pg_dist_partition_partkey 3 #define Anum_pg_dist_partition_colocationid 4 #define Anum_pg_dist_partition_repmodel 5 #define Anum_pg_dist_partition_autoconverted 6 /* valid values for partmethod include append, hash, and range */ #define DISTRIBUTE_BY_APPEND 'a' #define DISTRIBUTE_BY_HASH 'h' #define DISTRIBUTE_BY_RANGE 'r' #define DISTRIBUTE_BY_NONE 'n' #define REDISTRIBUTE_BY_HASH 'x' #define DISTRIBUTE_BY_INVALID '\0' /* * Valid values for repmodel are 'c' for coordinator, 's' for streaming * and 't' for two-phase-commit. We also use an invalid replication model * ('i') for distinguishing uninitialized variables where necessary. */ #define REPLICATION_MODEL_COORDINATOR 'c' #define REPLICATION_MODEL_STREAMING 's' #define REPLICATION_MODEL_2PC 't' #define REPLICATION_MODEL_INVALID 'i' #endif /* PG_DIST_PARTITION_H */ ================================================ FILE: src/include/distributed/pg_dist_placement.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_placement.h * definition of the "server" relation (pg_dist_placement). * * This table keeps information on remote shards and their whereabouts on the * coordinator node. The table's contents are updated and used as follows: (i) the * worker nodes send periodic reports about the shards they contain, and (ii) * the coordinator reconciles these shard reports, and determines outdated, under- * and over-replicated shards. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_PLACEMENT_H #define PG_DIST_PLACEMENT_H /* ---------------- * pg_dist_placement definition. * ---------------- */ typedef struct FormData_pg_dist_placement { int64 placementid; /* global placementId on remote node */ int64 shardid; /* global shardId on remote node */ int32 shardstate; /* shard state on remote node; see ShardState */ int64 shardlength; /* shard length on remote node; stored as bigint */ int32 groupid; /* the group the shard is placed on */ } FormData_pg_dist_placement; /* ---------------- * Form_pg_dist_placement corresponds to a pointer to a tuple with * the format of pg_dist_placement relation. * ---------------- */ typedef FormData_pg_dist_placement *Form_pg_dist_placement; /* ---------------- * compiler constants for pg_dist_placement * ---------------- */ #define Natts_pg_dist_placement 5 #define Anum_pg_dist_placement_placementid 1 #define Anum_pg_dist_placement_shardid 2 #define Anum_pg_dist_placement_shardstate 3 #define Anum_pg_dist_placement_shardlength 4 #define Anum_pg_dist_placement_groupid 5 #endif /* PG_DIST_PLACEMENT_H */ ================================================ FILE: src/include/distributed/pg_dist_rebalance_strategy.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_rebalance_strategy.h * definition of the "rebalance strategy" relation (pg_dist_rebalance_strategy). * * This table contains all the available strategies for rebalancing. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_REBALANCE_STRATEGY_H #define PG_DIST_REBALANCE_STRATEGY_H #include "postgres.h" /* ---------------- * pg_dist_shard definition. * ---------------- */ typedef struct FormData_pg_dist_rebalance_strategy { NameData name; /* user readable name of the strategy */ bool default_strategy; /* if this strategy is the default strategy */ Oid shardCostFunction; /* function to calculate the shard cost */ Oid nodeCapacityFunction; /* function to get the capacity of a node */ Oid shardAllowedOnNodeFunction; /* function to check if shard is allowed on node */ float4 defaultThreshold; /* default threshold that is used */ float4 minimumThreshold; /* minimum threshold that is allowed */ float4 improvementThreshold; /* the shard size threshold that is used */ } FormData_pg_dist_rebalance_strategy; /* ---------------- * Form_pg_dist_shards corresponds to a pointer to a tuple with * the format of pg_dist_shards relation. * ---------------- */ typedef FormData_pg_dist_rebalance_strategy *Form_pg_dist_rebalance_strategy; /* ---------------- * compiler constants for pg_dist_rebalance_strategy * ---------------- */ #define Natts_pg_dist_rebalance_strategy 7 #define Anum_pg_dist_rebalance_strategy_name 1 #define Anum_pg_dist_rebalance_strategy_default_strategy 2 #define Anum_pg_dist_rebalance_strategy_shard_cost_function 3 #define Anum_pg_dist_rebalance_strategy_node_capacity_function 4 #define Anum_pg_dist_rebalance_strategy_shard_allowed_on_node_function 5 #define Anum_pg_dist_rebalance_strategy_default_threshold 6 #define Anum_pg_dist_rebalance_strategy_minimum_threshold 7 #endif /* PG_DIST_REBALANCE_STRATEGY_H */ ================================================ FILE: src/include/distributed/pg_dist_schema.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_schema.h * definition of the system catalog for the schemas used for schema-based * sharding in Citus. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_SCHEMA_H #define PG_DIST_SCHEMA_H #include "postgres.h" /* ---------------- * pg_dist_schema definition. * ---------------- */ typedef struct FormData_pg_dist_schema { Oid schemaid; uint32 colocationid; } FormData_pg_dist_schema; /* ---------------- * Form_pg_dist_schema corresponds to a pointer to a tuple with * the format of pg_dist_schema relation. * ---------------- */ typedef FormData_pg_dist_schema *Form_pg_dist_schema; /* ---------------- * compiler constants for pg_dist_schema * ---------------- */ #define Natts_pg_dist_schema 2 #define Anum_pg_dist_schema_schemaid 1 #define Anum_pg_dist_schema_colocationid 2 #endif /* PG_DIST_SCHEMA_H */ ================================================ FILE: src/include/distributed/pg_dist_shard.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_shard.h * definition of the "shard" relation (pg_dist_shard). * * This table maps logical tables to their remote partitions (from this point * on, we use the terms remote partition and shard interchangeably). All changes * concerning the creation, deletion, merging, and split of remote partitions * reference this table. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_SHARD_H #define PG_DIST_SHARD_H /* ---------------- * pg_dist_shard definition. * ---------------- */ typedef struct FormData_pg_dist_shard { Oid logicalrelid; /* logical relation id; references pg_class oid */ int64 shardid; /* global shardId representing remote partition */ char shardstorage; /* shard storage type; see codes below */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ text shardalias_DROPPED; /* dropped column, not in use */ text shardminvalue; /* partition key's minimum value in shard */ text shardmaxvalue; /* partition key's maximum value in shard */ #endif } FormData_pg_dist_shard; /* ---------------- * Form_pg_dist_shards corresponds to a pointer to a tuple with * the format of pg_dist_shards relation. * ---------------- */ typedef FormData_pg_dist_shard *Form_pg_dist_shard; /* ---------------- * compiler constants for pg_dist_shards * ---------------- */ #define Natts_pg_dist_shard 6 #define Anum_pg_dist_shard_logicalrelid 1 #define Anum_pg_dist_shard_shardid 2 #define Anum_pg_dist_shard_shardstorage 3 #define Anum_pg_dist_shard_shardalias_DROPPED 4 #define Anum_pg_dist_shard_shardminvalue 5 #define Anum_pg_dist_shard_shardmaxvalue 6 /* * Valid values for shard storage types include foreign table, (standard) table * and columnar table. */ #define SHARD_STORAGE_FOREIGN 'f' #define SHARD_STORAGE_TABLE 't' #define SHARD_STORAGE_VIRTUAL 'v' #endif /* PG_DIST_SHARD_H */ ================================================ FILE: src/include/distributed/pg_dist_transaction.h ================================================ /*------------------------------------------------------------------------- * * pg_dist_transaction.h * definition of the "transaction" relation (pg_dist_transaction). * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_DIST_TRANSACTION_H #define PG_DIST_TRANSACTION_H /* ---------------- * pg_dist_transaction definition. * ---------------- */ typedef struct FormData_pg_dist_transaction { int32 groupid; /* id of the replication group */ text gid; /* global transaction identifier */ } FormData_pg_dist_transaction; /* ---------------- * Form_pg_dist_transactions corresponds to a pointer to a tuple with * the format of pg_dist_transactions relation. * ---------------- */ typedef FormData_pg_dist_transaction *Form_pg_dist_transaction; /* ---------------- * compiler constants for pg_dist_transaction * ---------------- */ #define Natts_pg_dist_transaction 3 #define Anum_pg_dist_transaction_groupid 1 #define Anum_pg_dist_transaction_gid 2 #define Anum_pg_dist_transaction_outerxid 3 extern int GetOuterXidAttrIndexInPgDistTransaction(TupleDesc tupleDesc); #endif /* PG_DIST_TRANSACTION_H */ ================================================ FILE: src/include/distributed/placement_access.h ================================================ /*------------------------------------------------------------------------- * * placement_access.h * Declarations of the structs and functions used in generating the * placement accesses for distributed query execution. * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef PLACEMENT_ACCESS_H #define PLACEMENT_ACCESS_H #include "postgres.h" #include "nodes/pg_list.h" #include "distributed/multi_physical_planner.h" /* forward declare, to avoid dependency on ShardPlacement definition */ struct ShardPlacement; /* represents the way in which a placement is accessed */ typedef enum ShardPlacementAccessType { /* read from placement */ PLACEMENT_ACCESS_SELECT, /* modify rows in placement */ PLACEMENT_ACCESS_DML, /* modify placement schema */ PLACEMENT_ACCESS_DDL } ShardPlacementAccessType; /* represents access to a placement */ typedef struct ShardPlacementAccess { /* placement that is accessed */ struct ShardPlacement *placement; /* the way in which the placement is accessed */ ShardPlacementAccessType accessType; } ShardPlacementAccess; extern List * PlacementAccessListForTask(Task *task, ShardPlacement *taskPlacement); extern ShardPlacementAccess * CreatePlacementAccess(ShardPlacement *placement, ShardPlacementAccessType accessType); #endif /* PLACEMENT_ACCESS_H */ ================================================ FILE: src/include/distributed/placement_connection.h ================================================ /*------------------------------------------------------------------------- * placement_connection.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PLACEMENT_CONNECTION_H #define PLACEMENT_CONNECTION_H #include "distributed/connection_management.h" #include "distributed/placement_access.h" extern MultiConnection * GetPlacementConnection(uint32 flags, struct ShardPlacement *placement, const char *userName); extern MultiConnection * StartPlacementConnection(uint32 flags, struct ShardPlacement *placement, const char *userName); extern MultiConnection * GetConnectionIfPlacementAccessedInXact(int flags, List * placementAccessList, const char *userName); extern MultiConnection * StartPlacementListConnection(uint32 flags, List *placementAccessList, const char *userName); extern ShardPlacementAccess * CreatePlacementAccess(ShardPlacement *placement, ShardPlacementAccessType accessType); extern void AssignPlacementListToConnection(List *placementAccessList, MultiConnection *connection); extern void ResetPlacementConnectionManagement(void); extern void ErrorIfPostCommitFailedShardPlacements(void); extern void CloseShardPlacementAssociation(struct MultiConnection *connection); extern void ResetShardPlacementAssociation(struct MultiConnection *connection); extern void InitPlacementConnectionManagement(void); extern bool ConnectionModifiedPlacement(MultiConnection *connection); extern bool UseConnectionPerPlacement(void); #endif /* PLACEMENT_CONNECTION_H */ ================================================ FILE: src/include/distributed/priority.h ================================================ /*------------------------------------------------------------------------- * * priority.h * Shared declarations for managing CPU priority. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_PRIORITY_H #define CITUS_PRIORITY_H #include "c.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "storage/fd.h" extern int CpuPriority; extern int CpuPriorityLogicalRepSender; extern int MaxHighPriorityBackgroundProcesess; #define CPU_PRIORITY_INHERIT 1234 /* Function declarations for transmitting files between two nodes */ extern void SetOwnPriority(int priority); extern int GetOwnPriority(void); #endif /* CITUS_PRIORITY_H */ ================================================ FILE: src/include/distributed/query_colocation_checker.h ================================================ /*------------------------------------------------------------------------- * * query_colocation_checker.h * General Citus planner code. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef QUERY_COLOCATION_CHECKER_H #define QUERY_COLOCATION_CHECKER_H #include "nodes/parsenodes.h" #include "nodes/primnodes.h" #include "distributed/distributed_planner.h" /* * ColocatedJoinChecker is a helper structure that is used to decide * whether any subqueries should be recursively planned due joins non * colocated joins. */ typedef struct ColocatedJoinChecker { Query *subquery; List *anchorAttributeEquivalences; List *anchorRelationRestrictionList; PlannerRestrictionContext *subqueryPlannerRestriction; } ColocatedJoinChecker; extern ColocatedJoinChecker CreateColocatedJoinChecker(Query *subquery, PlannerRestrictionContext * restrictionContext); extern bool SubqueryColocated(Query *subquery, ColocatedJoinChecker *context); extern Query * WrapRteRelationIntoSubquery(RangeTblEntry *rteRelation, List *requiredAttributes, RTEPermissionInfo *perminfo); extern List * CreateAllTargetListForRelation(Oid relationId, List *requiredAttributes); extern List * CreateFilteredTargetListForRelation(Oid relationId, List *requiredAttributes); #endif /* QUERY_COLOCATION_CHECKER_H */ ================================================ FILE: src/include/distributed/query_pushdown_planning.h ================================================ /*------------------------------------------------------------------------- * query_pushdown_planning.h * * Copyright (c) Citus Data, Inc. * Function declarations used in query pushdown logic. * *------------------------------------------------------------------------- */ #ifndef QUERY_PUSHDOWN_PLANNING #define QUERY_PUSHDOWN_PLANNING #include "postgres.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/multi_logical_planner.h" #include "distributed/multi_physical_planner.h" /* Config variables managed via guc.c */ extern bool SubqueryPushdown; extern int ValuesMaterializationThreshold; extern bool AllowAggregateWorkerCombineOnInternalTypes; extern bool CanPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit); extern bool ShouldUseSubqueryPushDown(Query *originalQuery, Query *rewrittenQuery, PlannerRestrictionContext * plannerRestrictionContext); extern bool JoinTreeContainsSubquery(Query *query); extern bool IsNodeSubquery(Node *node); extern bool HasEmptyJoinTree(Query *query); extern bool WhereOrHavingClauseContainsSubquery(Query *query); extern bool TargetListContainsSubquery(List *targetList); extern bool SafeToPushdownWindowFunction(Query *query, StringInfo *errorDetail); extern MultiNode * SubqueryMultiNodeTree(Query *originalQuery, Query *queryTree, PlannerRestrictionContext * plannerRestrictionContext); extern DeferredErrorMessage * DeferErrorIfUnsupportedSubqueryPushdown(Query * originalQuery, PlannerRestrictionContext * plannerRestrictionContext, bool plannerPhase); extern DeferredErrorMessage * DeferErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit); extern DeferredErrorMessage * DeferErrorIfUnsupportedUnionQuery(Query *queryTree); extern bool IsJsonTableRTE(RangeTblEntry *rte); extern bool IsOuterJoinExpr(Node *node); extern void FlattenGroupExprs(Query *query); #endif /* QUERY_PUSHDOWN_PLANNING_H */ ================================================ FILE: src/include/distributed/query_utils.h ================================================ /*------------------------------------------------------------------------- * * query_utils.h * * Declarations for query-walker utility functions and related types. * * Copyright (c), Citus Data, Inc. * **--------------------------------------------------------------------------- */ #ifndef QUERY_UTILS_H #define QUERY_UTILS_H #include "postgres.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" /* Enum to define execution flow of ExtractRangeTableList */ typedef enum ExtractRangeTableMode { EXTRACT_RELATION_ENTRIES, /* inclduding local, foreign and partitioned tables */ EXTRACT_ALL_ENTRIES } ExtractRangeTableMode; /* Struct to pass rtable list and execution flow flag to query_walker_tree */ typedef struct ExtractRangeTableWalkerContext { List **rangeTableList; ExtractRangeTableMode walkerMode; } ExtractRangeTableWalkerContext; /* Function declarations for query-walker utility functions */ extern bool ExtractRangeTableList(Node *node, ExtractRangeTableWalkerContext *context); /* Below two functions wrap ExtractRangeTableList function to determine the execution flow */ extern bool ExtractRangeTableRelationWalker(Node *node, List **rangeTableList); extern bool ExtractRangeTableEntryWalker(Node *node, List **rangeTableList); extern bool ExtractRangeTableIndexWalker(Node *node, List **rangeTableIndexList); #endif /* QUERY_UTILS_H */ ================================================ FILE: src/include/distributed/recursive_planning.h ================================================ /*------------------------------------------------------------------------- * * recursive_planning.h * General Citus planner code. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef RECURSIVE_PLANNING_H #define RECURSIVE_PLANNING_H #include "nodes/pathnodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "pg_version_constants.h" #include "distributed/distributed_planner.h" #include "distributed/errormessage.h" #include "distributed/log_utils.h" #include "distributed/relation_restriction_equivalence.h" extern bool EnableRecurringOuterJoinPushdown; extern bool EnableOuterJoinsWithPseudoconstantQualsPrePG17; typedef struct RecursivePlanningContextInternal RecursivePlanningContext; typedef struct RangeTblEntryIndex { RangeTblEntry *rangeTableEntry; Index rteIndex; }RangeTblEntryIndex; extern PlannerRestrictionContext * GetPlannerRestrictionContext(RecursivePlanningContext * recursivePlanningContext); extern List * GenerateSubplansForSubqueriesAndCTEs(uint64 planId, Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext, RouterPlanType routerPlan); extern char * GenerateResultId(uint64 planId, uint32 subPlanId); extern Query * BuildSubPlanResultQuery(List *targetEntryList, List *columnAliasList, char *resultId); extern Query * BuildReadIntermediateResultsArrayQuery(List *targetEntryList, List *columnAliasList, List *resultIdList, bool useBinaryCopyFormat); extern Query * BuildEmptyResultQuery(List *targetEntryList, char *resultId); extern bool GeneratingSubplans(void); extern bool ContainsLocalTableDistributedTableJoin(List *rangeTableList); extern void ReplaceRTERelationWithRteSubquery(RangeTblEntry *rangeTableEntry, List *requiredAttrNumbers, RecursivePlanningContext *context, RTEPermissionInfo *perminfo); extern bool IsRecursivelyPlannableRelation(RangeTblEntry *rangeTableEntry); extern bool IsRelationLocalTableOrMatView(Oid relationId); extern bool ContainsReferencesToOuterQuery(Query *query); extern void UpdateVarNosInNode(Node *node, Index newVarNo); extern bool CanPushdownRecurringOuterJoinExtended(JoinExpr *joinExpr, Query *query, int *outerRtIndex, RangeTblEntry **outerRte, RangeTblEntry **distRte, int *attnum); bool ResolveBaseVarFromSubquery(Var *var, Query *query, Var **baseVar, RangeTblEntry **baseRte); #endif /* RECURSIVE_PLANNING_H */ ================================================ FILE: src/include/distributed/reference_table_utils.h ================================================ /*------------------------------------------------------------------------- * * reference_table_utils.h * * Declarations for public utility functions related to reference tables. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REFERENCE_TABLE_UTILS_H_ #define REFERENCE_TABLE_UTILS_H_ #include "postgres.h" #include "listutils.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_sync.h" extern void EnsureReferenceTablesExistOnAllNodes(void); extern void EnsureReferenceTablesExistOnAllNodesExtended(char transferMode); extern int64 * ScheduleTasksToParallelCopyReferenceTablesOnAllMissingNodes(int64 jobId, char transferMode, int * nDependTasks); extern bool HasNodesWithMissingReferenceTables(List **referenceTableList); extern uint32 CreateReferenceTableColocationId(void); extern uint32 GetReferenceTableColocationId(void); extern List * GetAllReplicatedTableList(void); extern List * ReplicatedPlacementsForNodeGroup(int32 groupId); extern char * DeleteShardPlacementCommand(uint64 placementId); extern void DeleteAllReplicatedTablePlacementsFromNodeGroup(int32 groupId, bool localOnly); extern void DeleteAllReplicatedTablePlacementsFromNodeGroupViaMetadataContext( MetadataSyncContext *context, int32 groupId, bool localOnly); extern int CompareOids(const void *leftElement, const void *rightElement); extern void ReplicateAllReferenceTablesToNode(WorkerNode *workerNode); extern void ErrorIfNotAllNodesHaveReferenceTableReplicas(List *workerNodeList); #endif /* REFERENCE_TABLE_UTILS_H_ */ ================================================ FILE: src/include/distributed/relation_access_tracking.h ================================================ /* * relation_access_tracking.h * * Function declartions for transaction access tracking. * * Copyright (c) Citus Data, Inc. */ #ifndef RELATION_ACCESS_TRACKING_H_ #define RELATION_ACCESS_TRACKING_H_ #include "distributed/metadata_utility.h" #include "distributed/multi_physical_planner.h" /* access Task struct */ #include "distributed/placement_connection.h" /* Config variables managed via guc.c */ extern bool EnforceForeignKeyRestrictions; /* forward declare, to avoid dependency on ShardPlacement definition */ struct ShardPlacement; typedef enum RelationAccessMode { RELATION_NOT_ACCESSED, /* only valid for reference tables */ RELATION_REFERENCE_ACCESSED, /* * Only valid for distributed tables and set * if table is accessed in parallel mode */ RELATION_PARALLEL_ACCESSED } RelationAccessMode; extern void InitRelationAccessHash(void); extern void ResetRelationAccessHash(void); extern void RecordRelationAccessIfNonDistTable(Oid relationId, ShardPlacementAccessType accessType); extern void RecordParallelRelationAccessForTaskList(List *taskList); extern void RecordParallelSelectAccess(Oid relationId); extern void RecordParallelModifyAccess(Oid relationId); extern void RecordParallelDDLAccess(Oid relationId); extern RelationAccessMode GetRelationDDLAccessMode(Oid relationId); extern RelationAccessMode GetRelationDMLAccessMode(Oid relationId); extern RelationAccessMode GetRelationSelectAccessMode(Oid relationId); extern bool ShouldRecordRelationAccess(void); extern bool ParallelQueryExecutedInTransaction(void); #endif /* RELATION_ACCESS_TRACKING_H_ */ ================================================ FILE: src/include/distributed/relation_restriction_equivalence.h ================================================ /* * relation_restriction_equivalence.h * * This file contains functions helper functions for planning * queries with colocated tables and subqueries. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef RELATION_RESTRICTION_EQUIVALENCE_H #define RELATION_RESTRICTION_EQUIVALENCE_H #include "distributed/distributed_planner.h" #include "distributed/metadata_cache.h" #define SINGLE_RTE_INDEX 1 extern bool AllDistributionKeysInQueryAreEqual(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); extern bool IsRelOptOuterJoin(PlannerInfo *root, int varNo); extern bool SafeToPushdownUnionSubquery(Query *originalQuery, PlannerRestrictionContext * plannerRestrictionContext); extern bool ContainsUnionSubquery(Query *queryTree); extern bool RestrictionEquivalenceForPartitionKeys(PlannerRestrictionContext * plannerRestrictionContext); bool RestrictionEquivalenceForPartitionKeysViaEquivalences(PlannerRestrictionContext * plannerRestrictionContext, List * allAttributeEquivalenceList); extern List * GenerateAllAttributeEquivalences(PlannerRestrictionContext * plannerRestrictionContext); extern uint32 UniqueRelationCount(RelationRestrictionContext *restrictionContext, CitusTableType tableType); extern List * DistributedRelationIdList(Query *query); extern PlannerRestrictionContext * FilterPlannerRestrictionForQuery( PlannerRestrictionContext *plannerRestrictionContext, Query *query); extern List * GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, PlannerRestrictionContext * plannerRestrictionContext); extern RelationRestriction * RelationRestrictionForRelation(RangeTblEntry * rangeTableEntry, PlannerRestrictionContext * plannerRestrictionContext); extern JoinRestrictionContext * RemoveDuplicateJoinRestrictions(JoinRestrictionContext * joinRestrictionContext); extern bool EquivalenceListContainsRelationsEquality(List *attributeEquivalenceList, RelationRestrictionContext * restrictionContext); extern RelationRestrictionContext * FilterRelationRestrictionContext( RelationRestrictionContext *relationRestrictionContext, Relids queryRteIdentities); extern bool AllDistributedRelationsInRTEListColocated(List *rangeTableEntryList); extern bool AllDistributedRelationsInListColocated(List *relationList); #endif /* RELATION_RESTRICTION_EQUIVALENCE_H */ ================================================ FILE: src/include/distributed/relation_utils.h ================================================ /*------------------------------------------------------------------------- * * relation_utils.h * Utilities related to Relation objects. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef RELATION_UTILS_H #define RELATION_UTILS_H #include "postgres.h" #include "parser/parse_relation.h" #include "utils/relcache.h" #include "pg_version_constants.h" extern char * RelationGetNamespaceName(Relation relation); extern RTEPermissionInfo * GetFilledPermissionInfo(Oid relid, bool inh, AclMode requiredPerms); #endif /* RELATION_UTILS_H */ ================================================ FILE: src/include/distributed/relay_utility.h ================================================ /*------------------------------------------------------------------------- * * relay_utility.h * * Header and type declarations that extend relation, index and constraint names * with the appropriate shard identifiers. * * Copyright (c) Citus Data, Inc. * * $Id:$ * *------------------------------------------------------------------------- */ #ifndef RELAY_UTILITY_H #define RELAY_UTILITY_H #include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" /* Shard name and identifier related defines */ #define SHARD_NAME_SEPARATOR '_' #define INVALID_SHARD_ID 0 #define INVALID_PLACEMENT_ID 0 /* Function declarations to extend names in DDL commands */ extern void RelayEventExtendNames(Node *parseTree, char *schemaName, uint64 shardId); extern void RelayEventExtendNamesForInterShardCommands(Node *parseTree, uint64 leftShardId, char *leftShardSchemaName, uint64 rightShardId, char *rightShardSchemaName); extern void AppendShardIdToName(char **name, uint64 shardId); extern void SetSchemaNameIfNotExist(char **schemaName, const char *newSchemaName); #endif /* RELAY_UTILITY_H */ ================================================ FILE: src/include/distributed/remote_commands.h ================================================ /*------------------------------------------------------------------------- * * remote_commands.h * Helpers to execute commands on remote nodes, over libpq. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REMOTE_COMMAND_H #define REMOTE_COMMAND_H #include "distributed/connection_management.h" /* errors which ExecuteRemoteCommand might return */ #define RESPONSE_OKAY 0 #define QUERY_SEND_FAILED 1 #define RESPONSE_NOT_OKAY 2 /* GUC, determining whether statements sent to remote nodes are logged */ extern bool LogRemoteCommands; extern char *GrepRemoteCommands; /* GUC that determines the number of bytes after which remote COPY is flushed */ extern int RemoteCopyFlushThreshold; /* simple helpers */ extern bool IsResponseOK(PGresult *result); extern void ForgetResults(MultiConnection *connection); extern bool ClearResults(MultiConnection *connection, bool raiseErrors); extern bool ClearResultsDiscardWarnings(MultiConnection *connection, bool raiseErrors); extern bool ClearResultsIfReady(MultiConnection *connection); /* report errors & warnings */ extern void ReportConnectionError(MultiConnection *connection, int elevel); extern void ReportResultError(MultiConnection *connection, PGresult *result, int elevel); extern char * pchomp(const char *in); extern void LogRemoteCommand(MultiConnection *connection, const char *command); extern bool CommandMatchesLogGrepPattern(const char *command); /* wrappers around libpq functions, with command logging support */ extern void ExecuteCriticalRemoteCommandList(MultiConnection *connection, List *commandList); extern void ExecuteCriticalRemoteCommand(MultiConnection *connection, const char *command); extern void ExecuteRemoteCommandInConnectionList(List *nodeConnectionList, const char *command); extern bool ExecuteRemoteCommandAndCheckResult(MultiConnection *connection, char *command, char *expected); extern int ExecuteOptionalRemoteCommand(MultiConnection *connection, const char *command, PGresult **result); extern int SendRemoteCommand(MultiConnection *connection, const char *command); extern int SendRemoteCommandParams(MultiConnection *connection, const char *command, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues, bool binaryResults); extern List * ReadFirstColumnAsText(PGresult *queryResult); extern PGresult * GetRemoteCommandResult(MultiConnection *connection, bool raiseInterrupts); extern bool PutRemoteCopyData(MultiConnection *connection, const char *buffer, int nbytes); extern bool PutRemoteCopyEnd(MultiConnection *connection, const char *errormsg); /* waiting for multiple command results */ extern void WaitForAllConnections(List *connectionList, bool raiseInterrupts); extern bool SendCancelationRequest(MultiConnection *connection); extern bool EvaluateSingleQueryResult(MultiConnection *connection, PGresult *queryResult, StringInfo queryResultString); extern void StoreErrorMessage(MultiConnection *connection, StringInfo queryResultString); extern bool IsSettingSafeToPropagate(const char *name); #endif /* REMOTE_COMMAND_H */ ================================================ FILE: src/include/distributed/remote_transaction.h ================================================ /*------------------------------------------------------------------------- * remote_transaction.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REMOTE_TRANSACTION_H #define REMOTE_TRANSACTION_H #include "libpq-fe.h" #include "lib/ilist.h" #include "nodes/pg_list.h" /* forward declare, to avoid recursive includes */ struct MultiConnection; /* * Enum that defines different remote transaction states, of a single remote * transaction. */ typedef enum { /* no transaction active */ REMOTE_TRANS_NOT_STARTED = 0, /* transaction start */ REMOTE_TRANS_STARTING, REMOTE_TRANS_STARTED, /* command execution */ REMOTE_TRANS_SENT_BEGIN, REMOTE_TRANS_SENT_COMMAND, REMOTE_TRANS_FETCHING_RESULTS, REMOTE_TRANS_CLEARING_RESULTS, /* 2pc prepare */ REMOTE_TRANS_PREPARING, REMOTE_TRANS_PREPARED, /* transaction abort */ REMOTE_TRANS_1PC_ABORTING, REMOTE_TRANS_2PC_ABORTING, REMOTE_TRANS_ABORTED, /* transaction commit */ REMOTE_TRANS_1PC_COMMITTING, REMOTE_TRANS_2PC_COMMITTING, REMOTE_TRANS_COMMITTED } RemoteTransactionState; /* * Transaction state associated associated with a single MultiConnection. */ typedef struct RemoteTransaction { /* what state is the remote side transaction in */ RemoteTransactionState transactionState; /* failures on this connection should abort entire coordinated transaction */ bool transactionCritical; /* failed in current transaction */ bool transactionFailed; /* * Id of last savepoint that successfully began before transaction failure. * Since savepoint ids are assigned incrementally, rolling back to any savepoint * with id equal to or less than this id recovers the transaction from failures. */ SubTransactionId lastSuccessfulSubXact; /* Id of last savepoint queued before first query of transaction */ SubTransactionId lastQueuedSubXact; /* waiting for the result of a recovering ROLLBACK TO SAVEPOINT command */ bool transactionRecovering; /* 2PC transaction name currently associated with connection */ char preparedName[NAMEDATALEN]; /* set when BEGIN is sent over the connection */ bool beginSent; } RemoteTransaction; /* utility functions for dealing with remote transactions */ extern bool ParsePreparedTransactionName(char *preparedTransactionName, int32 *groupId, int *procId, uint64 *transactionNumber, uint32 *connectionNumber); /* change an individual remote transaction's state */ extern void StartRemoteTransactionBegin(struct MultiConnection *connection); extern void FinishRemoteTransactionBegin(struct MultiConnection *connection); extern void RemoteTransactionBegin(struct MultiConnection *connection); extern void RemoteTransactionListBegin(List *connectionList); extern void StartRemoteTransactionPrepare(struct MultiConnection *connection); extern void FinishRemoteTransactionPrepare(struct MultiConnection *connection); extern void StartRemoteTransactionCommit(struct MultiConnection *connection); extern void FinishRemoteTransactionCommit(struct MultiConnection *connection); extern void RemoteTransactionCommit(struct MultiConnection *connection); extern void StartRemoteTransactionAbort(struct MultiConnection *connection); extern void FinishRemoteTransactionAbort(struct MultiConnection *connection); extern void RemoteTransactionAbort(struct MultiConnection *connection); /* start transaction if necessary */ extern void RemoteTransactionBeginIfNecessary(struct MultiConnection *connection); extern void RemoteTransactionsBeginIfNecessary(List *connectionList); /* other public functionality */ extern void HandleRemoteTransactionConnectionError(struct MultiConnection *connection, bool raiseError); extern void HandleRemoteTransactionResultError(struct MultiConnection *connection, PGresult *result, bool raiseErrors); extern void MarkRemoteTransactionFailed(struct MultiConnection *connection, bool allowErrorPromotion); extern void MarkRemoteTransactionCritical(struct MultiConnection *connection); /* * The following functions should all only be called by connection / * transaction managment code. */ extern void ResetRemoteTransaction(struct MultiConnection *connection); /* perform handling for all in-progress transactions */ extern void CoordinatedRemoteTransactionsPrepare(void); extern void CoordinatedRemoteTransactionsCommit(void); extern void CoordinatedRemoteTransactionsAbort(void); extern void CheckRemoteTransactionsHealth(void); /* remote savepoint commands */ extern void CoordinatedRemoteTransactionsSavepointBegin(SubTransactionId subId); extern void CoordinatedRemoteTransactionsSavepointRelease(SubTransactionId subId); extern void CoordinatedRemoteTransactionsSavepointRollback(SubTransactionId subId); extern void RunCitusMainDBQuery(char *query); extern void CleanCitusMainDBConnection(void); extern bool IsMainDBCommand; extern bool IsMainDB; extern char *SuperuserRole; extern char *MainDb; extern struct MultiConnection *MainDBConnection; extern bool IsMainDBCommandInXact; #endif /* REMOTE_TRANSACTION_H */ ================================================ FILE: src/include/distributed/repartition_executor.h ================================================ /*------------------------------------------------------------------------- * * repartition_executor.h * * Declarations for public functions and types related to repartition of * select query results. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REPARTITION_EXECUTOR_H #define REPARTITION_EXECUTOR_H extern bool EnableRepartitionedInsertSelect; extern int DistributionColumnIndex(List *insertTargetList, Var *distributionColumn); extern List * GenerateTaskListWithColocatedIntermediateResults(Oid targetRelationId, Query * modifyQueryViaCoordinatorOrRepartition, char *resultIdPrefix); extern List * GenerateTaskListWithRedistributedResults(Query * modifyQueryViaCoordinatorOrRepartition, CitusTableCacheEntry * targetRelation, List **redistributedResults, bool useBinaryFormat); extern bool IsSupportedRedistributionTarget(Oid targetRelationId); extern bool IsRedistributablePlan(Plan *selectPlan); extern bool HasMergeNotMatchedBySource(Query *query); extern void AdjustTaskQueryForEmptySource(Oid targetRelationId, Query *mergeQuery, List *emptySourceTaskList, char *resultIdPrefix); #endif /* REPARTITION_EXECUTOR_H */ ================================================ FILE: src/include/distributed/repartition_join_execution.h ================================================ /*------------------------------------------------------------------------- * * repartition_join_execution.h * Execution logic for repartition queries. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef REPARTITION_JOIN_EXECUTION_H #define REPARTITION_JOIN_EXECUTION_H #include "nodes/pg_list.h" extern List * ExecuteDependentTasks(List *taskList, Job *topLevelJob); #endif /* REPARTITION_JOIN_EXECUTION_H */ ================================================ FILE: src/include/distributed/replicate_none_dist_table_shard.h ================================================ /*------------------------------------------------------------------------- * * replicate_none_dist_table_shard.h * Routines to replicate shard of none-distributed table to * a remote node. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REPLICA_LOCAL_TABLE_SHARD_H #define REPLICA_LOCAL_TABLE_SHARD_H extern void NoneDistTableReplicateCoordinatorPlacement(Oid noneDistTableId, List *targetNodeList); extern void NoneDistTableDeleteCoordinatorPlacement(Oid noneDistTableId); extern void NoneDistTableDropCoordinatorPlacementTable(Oid noneDistTableId); #endif /* REPLICA_LOCAL_TABLE_SHARD_H */ ================================================ FILE: src/include/distributed/replication_origin_session_utils.h ================================================ /*------------------------------------------------------------------------- * * replication_origin_utils.h * Utilities related to replication origin. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef REPLICATION_ORIGIN_SESSION_UTILS_H #define REPLICATION_ORIGIN_SESSION_UTILS_H #include "postgres.h" #include "replication/origin.h" #include "distributed/connection_management.h" extern void InitializeReplicationOriginSessionUtils(void); extern void SetupReplicationOriginRemoteSession(MultiConnection *connection); extern void ResetReplicationOriginRemoteSession(MultiConnection *connection); extern void SetupReplicationOriginLocalSession(void); extern void ResetReplicationOriginLocalSession(void); extern void ResetReplicationOriginLocalSessionCallbackHandler(void *arg); extern bool EnableChangeDataCapture; #endif /* REPLICATION_ORIGIN_SESSION_UTILS_H */ ================================================ FILE: src/include/distributed/resource_lock.h ================================================ /*------------------------------------------------------------------------- * * resource_lock.h * Locking Infrastructure for Citus. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef RESOURCE_LOCK_H #define RESOURCE_LOCK_H #include "postgres.h" /* IWYU pragma: keep */ #include "c.h" #include "catalog/dependency.h" #include "nodes/pg_list.h" #include "storage/lock.h" #include "tcop/utility.h" #include "distributed/worker_transaction.h" /* * Postgres' advisory locks use 'field4' to discern between different kind of * advisory locks. Only 1 and 2 are used allowing us to define non-conflicting * lock methods. * * In case postgres starts to use additional values, Citus's values * will have to be changed. That just requires re-compiling and a restart. */ typedef enum AdvisoryLocktagClass { /* values defined in postgres' lockfuncs.c */ ADV_LOCKTAG_CLASS_INT64 = 1, ADV_LOCKTAG_CLASS_INT32 = 2, /* Citus lock types */ ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4, ADV_LOCKTAG_CLASS_CITUS_SHARD = 5, ADV_LOCKTAG_CLASS_CITUS_JOB = 6, ADV_LOCKTAG_CLASS_CITUS_REBALANCE_COLOCATION = 7, ADV_LOCKTAG_CLASS_CITUS_COLOCATED_SHARDS_METADATA = 8, ADV_LOCKTAG_CLASS_CITUS_OPERATIONS = 9, ADV_LOCKTAG_CLASS_CITUS_CLEANUP_OPERATION_ID = 10, ADV_LOCKTAG_CLASS_CITUS_LOGICAL_REPLICATION = 12, /* Not used anymore */ ADV_LOCKTAG_CLASS_CITUS_REBALANCE_PLACEMENT_COLOCATION = 13, ADV_LOCKTAG_CLASS_CITUS_BACKGROUND_TASK = 14, ADV_LOCKTAG_CLASS_CITUS_GLOBAL_DDL_SERIALIZATION = 15, ADV_LOCKTAG_CLASS_CITUS_SHARD_MOVE = 16 } AdvisoryLocktagClass; /* CitusOperations has constants for citus operations */ typedef enum CitusOperations { CITUS_TRANSACTION_RECOVERY = 0, CITUS_NONBLOCKING_SPLIT = 1, CITUS_CREATE_DISTRIBUTED_TABLE_CONCURRENTLY = 2, CITUS_CREATE_COLOCATION_DEFAULT = 3, CITUS_BACKGROUND_TASK_MONITOR = 4 } CitusOperations; /* reuse advisory lock, but with different, unused field 4 (4)*/ #define SET_LOCKTAG_SHARD_METADATA_RESOURCE(tag, db, shardid) \ SET_LOCKTAG_ADVISORY(tag, \ db, \ (uint32) ((shardid) >> 32), \ (uint32) (shardid), \ ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA) #define SET_LOCKTAG_COLOCATED_SHARDS_METADATA_RESOURCE(tag, db, colocationId, \ shardIntervalIndex) \ SET_LOCKTAG_ADVISORY(tag, \ db, \ (uint32) shardIntervalIndex, \ (uint32) colocationId, \ ADV_LOCKTAG_CLASS_CITUS_COLOCATED_SHARDS_METADATA) /* reuse advisory lock, but with different, unused field 4 (5)*/ #define SET_LOCKTAG_SHARD_RESOURCE(tag, db, shardid) \ SET_LOCKTAG_ADVISORY(tag, \ db, \ (uint32) ((shardid) >> 32), \ (uint32) (shardid), \ ADV_LOCKTAG_CLASS_CITUS_SHARD) /* advisory lock for citus shard move/copy operations, * also it has the database hardcoded to MyDatabaseId, * to ensure the locks are local to each database */ #define SET_LOCKTAG_SHARD_MOVE(tag, shardid) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) ((shardid) >> 32), \ (uint32) (shardid), \ ADV_LOCKTAG_CLASS_CITUS_SHARD_MOVE) /* reuse advisory lock, but with different, unused field 4 (7) * Also it has the database hardcoded to MyDatabaseId, to ensure the locks * are local to each database */ #define SET_LOCKTAG_REBALANCE_COLOCATION(tag, colocationOrTableId) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) ((colocationOrTableId) >> 32), \ (uint32) (colocationOrTableId), \ ADV_LOCKTAG_CLASS_CITUS_REBALANCE_COLOCATION) /* reuse advisory lock, but with different, unused field 4 (13) * Also it has the database hardcoded to MyDatabaseId, to ensure the locks * are local to each database */ #define SET_LOCKTAG_REBALANCE_PLACEMENT_COLOCATION(tag, colocationOrTableId) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) ((colocationOrTableId) >> 32), \ (uint32) (colocationOrTableId), \ ADV_LOCKTAG_CLASS_CITUS_REBALANCE_PLACEMENT_COLOCATION) /* advisory lock for citus operations, also it has the database hardcoded to MyDatabaseId, * to ensure the locks are local to each database */ #define SET_LOCKTAG_CITUS_OPERATION(tag, operationId) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) 0, \ (uint32) operationId, \ ADV_LOCKTAG_CLASS_CITUS_OPERATIONS) /* reuse advisory lock, but with different, unused field 4 (10) * Also it has the database hardcoded to MyDatabaseId, to ensure the locks * are local to each database */ #define SET_LOCKTAG_CLEANUP_OPERATION_ID(tag, operationId) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) ((operationId) >> 32), \ (uint32) operationId, \ ADV_LOCKTAG_CLASS_CITUS_CLEANUP_OPERATION_ID) /* reuse advisory lock, but with different, unused field 4 (14) * Also it has the database hardcoded to MyDatabaseId, to ensure the locks * are local to each database */ #define SET_LOCKTAG_BACKGROUND_TASK(tag, taskId) \ SET_LOCKTAG_ADVISORY(tag, \ MyDatabaseId, \ (uint32) ((taskId) >> 32), \ (uint32) (taskId), \ ADV_LOCKTAG_CLASS_CITUS_BACKGROUND_TASK) /* * IsNodeWideObjectClass returns true if the given object class is node-wide, * i.e., that is not bound to a particular database but to whole server. * * Defined here as an inlined function so that SET_LOCKTAG_GLOBAL_DDL_SERIALIZATION * macro can use it. */ static inline bool IsNodeWideObjectClass(ObjectClass objectClass) { if ((int) objectClass < 0 || objectClass > LAST_OCLASS) { elog(ERROR, "invalid object class: %d", objectClass); } /* * We don't expect Postgres to change an object class to a node-wide one in the * future, but a newly added object class may be node-wide. * * So we put a static assert here to make sure that the developer who adds support * for a new Postgres version is aware of this. * * If new object classes are added and none of them are node-wide, then update * this assertion check based on latest supported major Postgres version. */ StaticAssertStmt(PG_MAJORVERSION_NUM <= 18, "better to check if any of newly added ObjectClass'es are node-wide") ; switch (objectClass) { case OCLASS_ROLE: case OCLASS_DATABASE: case OCLASS_TBLSPACE: case OCLASS_PARAMETER_ACL: case OCLASS_ROLE_MEMBERSHIP: { return true; } default: { return false; } } } /* * Automatically sets databaseId to InvalidOid if the object class is * node-wide, i.e., that is not bound to a particular database but to * whole server. If the object class is not node-wide, sets databaseId * to MyDatabaseId. * * That way, the lock is local to each database if the object class is * not node-wide, and global if it is. */ #define SET_LOCKTAG_GLOBAL_DDL_SERIALIZATION(tag, objectClass, oid) \ SET_LOCKTAG_ADVISORY(tag, \ (uint32) (IsNodeWideObjectClass(objectClass) ? InvalidOid : \ MyDatabaseId), \ (uint32) objectClass, \ (uint32) oid, \ ADV_LOCKTAG_CLASS_CITUS_GLOBAL_DDL_SERIALIZATION) /* * DistLockConfigs are used to configure the locking behaviour of AcquireDistributedLockOnRelations */ enum DistLockConfigs { /* * lock citus tables */ DIST_LOCK_DEFAULT = 0, /* * lock tables that refer to locked citus tables with a foreign key */ DIST_LOCK_REFERENCING_TABLES = 1, /* * throw an error if the lock is not immediately available */ DIST_LOCK_NOWAIT = 2 }; /* Lock shard/relation metadata for safe modifications */ extern void LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode); extern void EnsureShardOwner(uint64 shardId, bool missingOk); extern void LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList); extern void BlockWritesToShardList(List *shardList); /* Lock shard/relation metadata of the referenced reference table if exists */ extern void LockReferencedReferenceShardDistributionMetadata(uint64 shardId, LOCKMODE lock); /* Lock shard data, for DML commands or remote fetches */ extern void LockShardResource(uint64 shardId, LOCKMODE lockmode); /* Lock a co-location group */ extern void LockColocationId(int colocationId, LOCKMODE lockMode); extern void UnlockColocationId(int colocationId, LOCKMODE lockMode); /* Lock multiple shards for safe modification */ extern void LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode); extern void LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList); extern void LockShardsInPlacementListMetadata(List *shardPlacementList, LOCKMODE lockMode); extern void LockTransactionRecovery(LOCKMODE lockMode); extern void SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode); extern void LockRelationShardResources(List *relationShardList, LOCKMODE lockMode); extern List * GetSortedReferenceShardIntervals(List *relationList); void AcquireCreateDistributedTableConcurrentlyLock(Oid relationId); /* Lock parent table's colocated shard resource */ extern void LockParentShardResourceIfPartition(List *shardIntervalList, LOCKMODE lockMode); /* Lock mode translation between text and enum */ extern LOCKMODE LockModeTextToLockMode(const char *lockModeName); extern const char * LockModeToLockModeText(LOCKMODE lockMode); extern void AcquireDistributedLockOnRelations(List *relationList, LOCKMODE lockMode, uint32 configs); extern void PreprocessLockStatement(LockStmt *stmt, ProcessUtilityContext context); extern bool EnableAcquiringUnsafeLockFromWorkers; extern bool SkipAdvisoryLockPermissionChecks; #endif /* RESOURCE_LOCK_H */ ================================================ FILE: src/include/distributed/run_from_same_connection.h ================================================ /* * run_from_same_connection.h * * Sending commands from same connection to test transactions initiated from * worker nodes in the isolation framework. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef RUN_FROM_SAME_CONNECTION_H #define RUN_FROM_SAME_CONNECTION_H /* * Config variables which will be used by isolation framework to check transactions * initiated from worker nodes. */ extern int IsolationTestSessionRemoteProcessID; extern int IsolationTestSessionProcessID; bool AllowNonIdleTransactionOnXactHandling(void); #endif /* RUN_FROM_SAME_CONNECTION_H */ ================================================ FILE: src/include/distributed/shard_cleaner.h ================================================ /*------------------------------------------------------------------------- * * shard_cleaner.h * Type and function declarations used in background shard cleaning * * Copyright (c) 2018, Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_SHARD_CLEANER_H #define CITUS_SHARD_CLEANER_H #define MAX_BG_TASK_EXECUTORS 1000 /* GUC to configure deferred shard deletion */ extern int DeferShardDeleteInterval; extern int BackgroundTaskQueueCheckInterval; extern int MaxBackgroundTaskExecutors; extern double DesiredPercentFreeAfterMove; extern bool CheckAvailableSpaceBeforeMove; extern int NextOperationId; extern int NextCleanupRecordId; extern int TryDropOrphanedResources(void); extern void DropOrphanedResourcesInSeparateTransaction(void); extern void ErrorIfCleanupRecordForShardExists(char *shardName); /* Members for cleanup infrastructure */ typedef uint64 OperationId; extern OperationId CurrentOperationId; /* * CleanupResource represents the Resource type in cleanup records. */ typedef enum CleanupObject { CLEANUP_OBJECT_INVALID = 0, CLEANUP_OBJECT_SHARD_PLACEMENT = 1, CLEANUP_OBJECT_SUBSCRIPTION = 2, CLEANUP_OBJECT_REPLICATION_SLOT = 3, CLEANUP_OBJECT_PUBLICATION = 4, CLEANUP_OBJECT_USER = 5, CLEANUP_OBJECT_DATABASE = 6 } CleanupObject; /* * CleanupPolicy represents the policy type for cleanup records. */ typedef enum CleanupPolicy { /* * Resources that are transient and always need clean up after the operation is completed. * (Example: Dummy Shards for Non-Blocking splits) */ CLEANUP_ALWAYS = 0, /* * Resources that are cleanup only on failure. * (Example: Split Children for Blocking/Non-Blocking splits) */ CLEANUP_ON_FAILURE = 1, /* * Resources that need 'deferred' clean up only on success . * (Example: Parent child being split for Blocking/Non-Blocking splits) */ CLEANUP_DEFERRED_ON_SUCCESS = 2, } CleanupPolicy; /* Global Constants */ #define INVALID_OPERATION_ID 0 #define INVALID_CLEANUP_RECORD_ID 0 /* APIs for cleanup infrastructure */ /* * RegisterOperationNeedingCleanup is be called by an operation to register * for cleanup. */ extern OperationId RegisterOperationNeedingCleanup(void); /* * InsertCleanupOnSuccessRecordInCurrentTransaction inserts a new pg_dist_cleanup entry * as part of the current transaction. * * This is primarily useful for deferred cleanup (CLEANUP_DEFERRED_ON_SUCCESS) * scenarios, since the records would roll back in case of failure. And for the * same reason, always sets the policy type to CLEANUP_DEFERRED_ON_SUCCESS. */ extern void InsertCleanupOnSuccessRecordInCurrentTransaction(CleanupObject objectType, char *objectName, int nodeGroupId); /* * InsertCleanupRecordInSeparateTransaction inserts a new pg_dist_cleanup entry * in a separate transaction to ensure the record persists after rollback. * * This is used in scenarios where we need to cleanup resources on operation * completion (CLEANUP_ALWAYS) or on failure (CLEANUP_ON_FAILURE). */ extern void InsertCleanupRecordOutsideTransaction(CleanupObject objectType, char *objectName, int nodeGroupId, CleanupPolicy policy); /* * FinalizeOperationNeedingCleanupOnSuccess is be called by an operation to signal * completion on success. This will trigger cleanup of appropriate resources * and cleanup records. */ extern void FinalizeOperationNeedingCleanupOnSuccess(const char *operationName); #endif /*CITUS_SHARD_CLEANER_H */ ================================================ FILE: src/include/distributed/shard_pruning.h ================================================ /*------------------------------------------------------------------------- * * shard_pruning.h * Shard pruning infrastructure. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARD_PRUNING_H_ #define SHARD_PRUNING_H_ #include "nodes/primnodes.h" #include "distributed/metadata_cache.h" #define INVALID_SHARD_INDEX -1 /* Function declarations for shard pruning */ extern List * PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, Const **partitionValueConst); extern bool ContainsFalseClause(List *whereClauseList); extern List * get_all_actual_clauses(List *restrictinfo_list); extern Const * TransformPartitionRestrictionValue(Var *partitionColumn, Const *restrictionValue, bool missingOk); bool VarConstOpExprClause(OpExpr *opClause, Var **varClause, Const **constantClause); #endif /* SHARD_PRUNING_H_ */ ================================================ FILE: src/include/distributed/shard_rebalancer.h ================================================ /*------------------------------------------------------------------------- * * shard_rebalancer.h * * Type and function declarations for the shard rebalancer tool. * * Copyright (c) 2016, Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef SHARD_REBALANCER_H #define SHARD_REBALANCER_H #include "postgres.h" #include "fmgr.h" #include "nodes/pg_list.h" #include "distributed/coordinator_protocol.h" #include "distributed/worker_manager.h" /* Limits for function parameters */ #define SHARD_REPLICATION_FACTOR_MINIMUM 1 #define SHARD_REPLICATION_FACTOR_MAXIMUM 100 /* Definitions for metadata update commands */ #define INSERT_SHARD_PLACEMENT_COMMAND "INSERT INTO pg_dist_shard_placement VALUES(" \ UINT64_FORMAT ", %d, " UINT64_FORMAT ", '%s', %d)" #define DELETE_SHARD_PLACEMENT_COMMAND "DELETE FROM pg_dist_shard_placement WHERE " \ "shardid=" UINT64_FORMAT \ " AND nodename='%s' AND nodeport=%d" /* * Definitions for shard placement json field names. These names should match * the column names in pg_dist_shard_placement. */ #define FIELD_NAME_SHARD_ID "shardid" #define FIELD_NAME_SHARD_LENGTH "shardlength" #define FIELD_NAME_SHARD_STATE "shardstate" #define FIELD_NAME_NODE_NAME "nodename" #define FIELD_NAME_NODE_PORT "nodeport" #define FIELD_NAME_PLACEMENT_ID "placementid" /* * Definitions for worker node json field names. These names should match the * column names in master_get_active_worker_nodes(). */ #define FIELD_NAME_WORKER_NAME "node_name" #define FIELD_NAME_WORKER_PORT "node_port" /* Definitions for placement update json field names */ #define FIELD_NAME_UPDATE_TYPE "updatetype" #define FIELD_NAME_SOURCE_NAME "sourcename" #define FIELD_NAME_SOURCE_PORT "sourceport" #define FIELD_NAME_TARGET_NAME "targetname" #define FIELD_NAME_TARGET_PORT "targetport" /* *INDENT-OFF* */ /* Definition for format of placement update json document */ #define PLACEMENT_UPDATE_JSON_FORMAT \ "{"\ "\"" FIELD_NAME_UPDATE_TYPE "\":%d,"\ "\"" FIELD_NAME_SHARD_ID "\":" UINT64_FORMAT ","\ "\"" FIELD_NAME_SOURCE_NAME "\":%s,"\ "\"" FIELD_NAME_SOURCE_PORT "\":%d,"\ "\"" FIELD_NAME_TARGET_NAME "\":%s,"\ "\"" FIELD_NAME_TARGET_PORT "\":%d"\ "}" /* *INDENT-ON* */ #define REBALANCE_ACTIVITY_MAGIC_NUMBER 1337 #define REBALANCE_PROGRESS_WAITING 0 #define REBALANCE_PROGRESS_MOVING 1 #define REBALANCE_PROGRESS_MOVED 2 /* Enumeration that defines different placement update types */ typedef enum { PLACEMENT_UPDATE_INVALID_FIRST = 0, PLACEMENT_UPDATE_MOVE = 1, PLACEMENT_UPDATE_COPY = 2 } PlacementUpdateType; typedef enum { PLACEMENT_UPDATE_STATUS_NOT_STARTED_YET = 0, PLACEMENT_UPDATE_STATUS_SETTING_UP = 1, PLACEMENT_UPDATE_STATUS_COPYING_DATA = 2, PLACEMENT_UPDATE_STATUS_CATCHING_UP = 3, PLACEMENT_UPDATE_STATUS_CREATING_CONSTRAINTS = 4, PLACEMENT_UPDATE_STATUS_FINAL_CATCH_UP = 5, PLACEMENT_UPDATE_STATUS_CREATING_FOREIGN_KEYS = 6, PLACEMENT_UPDATE_STATUS_COMPLETING = 7, PLACEMENT_UPDATE_STATUS_COMPLETED = 8, } PlacementUpdateStatus; /* * PlacementUpdateEvent represents a logical unit of work that copies or * moves a shard placement. */ typedef struct PlacementUpdateEvent { PlacementUpdateType updateType; uint64 shardId; WorkerNode *sourceNode; WorkerNode *targetNode; } PlacementUpdateEvent; typedef struct PlacementUpdateEventProgress { uint64 shardId; char sourceName[255]; int sourcePort; char targetName[255]; int targetPort; PlacementUpdateType updateType; pg_atomic_uint64 progress; pg_atomic_uint64 updateStatus; } PlacementUpdateEventProgress; typedef struct NodeFillState { WorkerNode *node; /* * capacity is how big this node is, relative to the other nodes in the * cluster. This has no unit, it can represent whatever the user wants. * Some examples: * 1. GBs of RAM * 2. number of CPUs * 3. GBs of disk * 4. relative improvement of new CPU generation in newly added nodes */ float4 capacity; /* * totalCost is the costs of ShardCosts on the node added together. This * doesn't have a unit. See the ShardCost->cost comment for some examples. */ float4 totalCost; /* * utilization is how "full" the node is. This is always totalCost divided * by capacity. Since neither of those have a unit, this also doesn't have * one. */ float4 utilization; /* * shardCostListDesc contains all ShardCosts that are on the current node, * ordered from high cost to low cost. */ List *shardCostListDesc; } NodeFillState; typedef struct ShardCost { uint64 shardId; /* * cost is the cost of the shard. This doesn't have a unit. * Some examples of what this could represent: * 1. GBs of data * 2. number of queries per day */ float4 cost; } ShardCost; typedef struct DisallowedPlacement { ShardCost *shardCost; NodeFillState *fillState; } DisallowedPlacement; typedef struct RebalancePlanFunctions { bool (*shardAllowedOnNode)(uint64 shardId, WorkerNode *workerNode, void *context); float4 (*nodeCapacity)(WorkerNode *workerNode, void *context); ShardCost (*shardCost)(uint64 shardId, void *context); void *context; } RebalancePlanFunctions; extern char *VariablesToBePassedToNewConnections; extern int MaxRebalancerLoggedIgnoredMoves; extern int RebalancerByDiskSizeBaseCost; extern bool RunningUnderCitusTestSuite; extern bool PropagateSessionSettingsForLoopbackConnection; extern int MaxBackgroundTaskExecutorsPerNode; /* External function declarations */ extern Datum shard_placement_rebalance_array(PG_FUNCTION_ARGS); extern Datum shard_placement_replication_array(PG_FUNCTION_ARGS); extern Datum worker_node_responsive(PG_FUNCTION_ARGS); extern Datum update_shard_placement(PG_FUNCTION_ARGS); extern Datum init_rebalance_monitor(PG_FUNCTION_ARGS); extern Datum finalize_rebalance_monitor(PG_FUNCTION_ARGS); extern Datum get_rebalance_progress(PG_FUNCTION_ARGS); extern List * RebalancePlacementUpdates(List *workerNodeList, List *shardPlacementListList, double threshold, int32 maxShardMoves, bool drainOnly, float4 utilizationImproventThreshold, RebalancePlanFunctions *rebalancePlanFunctions); extern List * ReplicationPlacementUpdates(List *workerNodeList, List *shardPlacementList, int shardReplicationFactor); extern void ExecuteRebalancerCommandInSeparateTransaction(char *command); extern void AcquirePlacementColocationLock(Oid relationId, int lockMode, const char *operationName); extern void SetupRebalanceMonitor(List *placementUpdateList, Oid relationId, uint64 initialProgressState, PlacementUpdateStatus initialStatus); extern void SplitShardsBetweenPrimaryAndClone(WorkerNode *primaryNode, WorkerNode *cloneNode, Name strategyName); #endif /* SHARD_REBALANCER_H */ ================================================ FILE: src/include/distributed/shard_split.h ================================================ /*------------------------------------------------------------------------- * * shard_split.h * * API for shard splits. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARDSPLIT_H_ #define SHARDSPLIT_H_ #include "distributed/utils/distribution_column_map.h" /* Split Modes supported by Shard Split API */ typedef enum SplitMode { BLOCKING_SPLIT = 0, NON_BLOCKING_SPLIT = 1, AUTO_SPLIT = 2 } SplitMode; /* * User Scenario calling Split Shard API. * The 'SplitOperation' type is used to customize info/error messages based on user scenario. */ typedef enum SplitOperation { SHARD_SPLIT_API = 0, ISOLATE_TENANT_TO_NEW_SHARD, CREATE_DISTRIBUTED_TABLE } SplitOperation; /* * SplitShard API to split a given shard (or shard group) using split mode and * specified split points to a set of destination nodes. */ extern void SplitShard(SplitMode splitMode, SplitOperation splitOperation, uint64 shardIdToSplit, List *shardSplitPointsList, List *nodeIdsForPlacementList, DistributionColumnMap *distributionColumnOverrides, List *colocatedShardIntervalList, uint32 targetColocationId); extern SplitMode LookupSplitMode(Oid shardTransferModeOid); extern void ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction(void); #endif /* SHARDSPLIT_H_ */ ================================================ FILE: src/include/distributed/shard_transfer.h ================================================ /*------------------------------------------------------------------------- * * shard_transfer.h * Code used to move shards around. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #include "postgres.h" #include "nodes/pg_list.h" #include "distributed/shard_rebalancer.h" extern Datum citus_move_shard_placement(PG_FUNCTION_ARGS); extern Datum citus_move_shard_placement_with_nodeid(PG_FUNCTION_ARGS); typedef enum { SHARD_TRANSFER_INVALID_FIRST = 0, SHARD_TRANSFER_MOVE = 1, SHARD_TRANSFER_COPY = 2 } ShardTransferType; /* * ShardTransferOperationMode is used to pass flags to the shard transfer * function. The flags are used to control the behavior of the transfer * function. * Currently, optionFlags are only used to customize reference table transfers. * For distributed tables, optionFlags should always be set to 0. */ typedef enum { /* * This flag instructs the transfer function to only transfer single shard * rather than transfer all the colocated shards for the shard interval. * Using this flag mean we might break the colocated shard * relationship on the source node. So this is only usefull when setting up * the new node and we are sure that the node would not be used until we have * transfered all the shards. * The reason we need this flag is that we want to be able to transfer * colocated shards in parallel and for now it is only used for the reference * table shards. * Finally if you are using this flag, you should also use consider defering * the creation of the relationships on the source node until all colocated * shards are transfered (see: SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS). */ SHARD_TRANSFER_SINGLE_SHARD_ONLY = 1 << 0, /* With this flag the shard transfer function does not create any constrainsts * or foreign relations defined on the shard, This can be used to defer the * creation of the relationships until all the shards are transfered. * This is usefull when we are transfering colocated shards in parallel and * we want to avoid the creation of the relationships on the source node * until all the shards are transfered. */ SHARD_TRANSFER_SKIP_CREATE_RELATIONSHIPS = 1 << 1, /* This flag is used to indicate that the shard transfer function should * only create the relationships on the target node and not transfer any data. * This is can be used to create the relationships that were defered * during the transfering of shards. */ SHARD_TRANSFER_CREATE_RELATIONSHIPS_ONLY = 1 << 2 } ShardTransferOperationMode; extern void TransferShards(int64 shardId, char *sourceNodeName, int32 sourceNodePort, char *targetNodeName, int32 targetNodePort, char shardReplicationMode, ShardTransferType transferType, uint32 optionFlags); extern uint64 ShardListSizeInBytes(List *colocatedShardList, char *workerNodeName, uint32 workerNodePort); extern void ErrorIfMoveUnsupportedTableType(Oid relationId); extern void CopyShardsToNode(WorkerNode *sourceNode, WorkerNode *targetNode, List *shardIntervalList, char *snapshotName); extern void VerifyTablesHaveReplicaIdentity(List *colocatedTableList); extern bool RelationCanPublishAllModifications(Oid relationId); extern void UpdatePlacementUpdateStatusForShardIntervalList(List *shardIntervalList, char *sourceName, int sourcePort, PlacementUpdateStatus status); extern void InsertDeferredDropCleanupRecordsForShards(List *shardIntervalList); extern void InsertCleanupRecordsForShardPlacementsOnNode(List *shardIntervalList, int32 groupId); extern void AdjustShardsForPrimaryCloneNodeSplit(WorkerNode *primaryNode, WorkerNode *cloneNode, List *primaryShardList, List *cloneShardList); ================================================ FILE: src/include/distributed/shard_utils.h ================================================ /*------------------------------------------------------------------------- * * shard_utils.h * Utilities related to shards. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARD_UTILS_H #define SHARD_UTILS_H #include "postgres.h" extern Oid GetTableLocalShardOid(Oid citusTableOid, uint64 shardId); extern char * GetLongestShardName(Oid citusTableOid, char *finalRelationName); extern char * GetLongestShardNameForLocalPartition(Oid parentTableOid, char *partitionRelationName); #endif /* SHARD_UTILS_H */ ================================================ FILE: src/include/distributed/shardinterval_utils.h ================================================ /*------------------------------------------------------------------------- * * shardinterval_utils.h * * Declarations for public utility functions related to shard intervals. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARDINTERVAL_UTILS_H_ #define SHARDINTERVAL_UTILS_H_ #include "nodes/primnodes.h" #include "distributed/metadata_cache.h" #include "distributed/metadata_utility.h" #define INVALID_SHARD_INDEX -1 /* OperatorCacheEntry contains information for each element in OperatorCache */ typedef struct ShardIntervalCompareFunctionCacheEntry { Var *partitionColumn; char partitionMethod; FmgrInfo *functionInfo; } ShardIntervalCompareFunctionCacheEntry; /* * SortShardIntervalContext is the context parameter in SortShardIntervalArray */ typedef struct SortShardIntervalContext { FmgrInfo *comparisonFunction; Oid collation; } SortShardIntervalContext; extern ShardInterval ** SortShardIntervalArray(ShardInterval **shardIntervalArray, int shardCount, Oid collation, FmgrInfo * shardIntervalSortCompareFunction); extern int CompareShardIntervals(const void *leftElement, const void *rightElement, SortShardIntervalContext *sortContext); extern int CompareShardIntervalsById(const void *leftElement, const void *rightElement); extern int CompareShardPlacementsByShardId(const void *leftElement, const void *rightElement); extern int CompareRelationShards(const void *leftElement, const void *rightElement); extern int ShardIndex(ShardInterval *shardInterval); extern int CalculateUniformHashRangeIndex(int hashedValue, int shardCount); extern ShardInterval * FindShardInterval(Datum partitionColumnValue, CitusTableCacheEntry *cacheEntry); extern int FindShardIntervalIndex(Datum searchedValue, CitusTableCacheEntry *cacheEntry); extern int SearchCachedShardInterval(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, Oid shardIntervalCollation, FmgrInfo *compareFunction); extern bool SingleReplicatedTable(Oid relationId); #endif /* SHARDINTERVAL_UTILS_H_ */ ================================================ FILE: src/include/distributed/shardsplit_logical_replication.h ================================================ /*------------------------------------------------------------------------- * * shardsplit_logical_replication.h * * Function declarations for logically replicating shard to split children. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARDSPLIT_LOGICAL_REPLICATION_H #define SHARDSPLIT_LOGICAL_REPLICATION_H #include "distributed/metadata_utility.h" #include "distributed/multi_logical_replication.h" #include "distributed/worker_manager.h" /* * GroupedShardSplitInfos groups all ShardSplitInfos belonging to the same node * and table owner together. This data structure its only purpose is creating a * hashmap that allows us to search ShardSplitInfos by node and owner. */ typedef struct GroupedShardSplitInfos { NodeAndOwner key; List *shardSplitInfoList; } GroupedShardSplitInfos; /* Functions for subscriber metadata management */ extern List * PopulateShardSplitSubscriptionsMetadataList(HTAB *shardSplitInfoHashMap, List *replicationSlotInfoList, List * shardGroupSplitIntervalListList, List *workersForPlacementList); extern HTAB * CreateShardSplitInfoMapForPublication(List * sourceColocatedShardIntervalList, List * shardGroupSplitIntervalListList, List *destinationWorkerNodesList); /* Functions to drop publisher-subscriber resources */ extern void DropAllShardSplitLeftOvers(WorkerNode *sourceNode, HTAB *shardSplitMapOfPublications); extern void DropShardSplitPublications(MultiConnection *sourceConnection, HTAB *shardInfoHashMapForPublication); extern void DropShardSplitSubsriptions(List *shardSplitSubscribersMetadataList); extern void DropShardSplitReplicationSlots(MultiConnection *sourceConnection, List *replicationSlotInfoList); #endif /* SHARDSPLIT_LOGICAL_REPLICATION_H */ ================================================ FILE: src/include/distributed/shardsplit_shared_memory.h ================================================ /*------------------------------------------------------------------------- * * shardsplit_shared_memory.h * API's for creating and accessing shared memory segments to store * shard split information. 'worker_split_shard_replication_setup' UDF creates the * shared memory and populates the contents. WAL sender processes are consumer * of split information for appropriate tuple routing. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARDSPLIT_SHARED_MEMORY_H #define SHARDSPLIT_SHARED_MEMORY_H #include "postgres.h" /* * In-memory mapping of a split child shard. */ typedef struct ShardSplitInfo { Oid distributedTableOid; /* citus distributed table Oid */ int partitionColumnIndex; /* partition column index */ Oid sourceShardOid; /* parent shard Oid */ Oid splitChildShardOid; /* child shard Oid */ int32 shardMinValue; /* min hash value */ int32 shardMaxValue; /* max hash value */ uint32_t nodeId; /* node where child shard is to be placed */ uint64 sourceShardId; /* parent shardId */ uint64 splitChildShardId; /* child shardId*/ char slotName[NAMEDATALEN]; /* replication slot name belonging to this node */ } ShardSplitInfo; /* * Header of the shared memory segment where shard split information is stored. */ typedef struct ShardSplitInfoSMHeader { int count; /* number of elements in the shared memory */ ShardSplitInfo splitInfoArray[FLEXIBLE_ARRAY_MEMBER]; } ShardSplitInfoSMHeader; /* * Shard split information is populated and stored in shared memory in the form of one dimensional * array by 'worker_split_shard_replication_setup'. Information belonging to same replication * slot is grouped together and stored contiguously within this array. * 'SourceToDestinationShardMap' maps list of child(destination) shards that should be processed by a replication * slot corresponding to a parent(source) shard. When a parent shard receives a change, the decoder can use this map * to traverse only the list of child shards corresponding the given parent. */ typedef struct SourceToDestinationShardMapEntry { Oid sourceShardKey; List *shardSplitInfoList; } SourceToDestinationShardMapEntry; typedef struct ShardSplitShmemData { int trancheId; NamedLWLockTranche namedLockTranche; LWLock lock; dsm_handle dsmHandle; } ShardSplitShmemData; /* Functions for creating and accessing shared memory used for dsm handle managment */ void InitializeShardSplitSMHandleManagement(void); void StoreShardSplitSharedMemoryHandle(dsm_handle dsmHandle); /* Functions for creating and accessing shared memory segments consisting shard split information */ extern ShardSplitInfoSMHeader * CreateSharedMemoryForShardSplitInfo(int shardSplitInfoCount, dsm_handle * dsmHandle); extern void ReleaseSharedMemoryOfShardSplitInfo(void); extern ShardSplitInfoSMHeader * GetShardSplitInfoSMHeader(void); extern HTAB * PopulateSourceToDestinationShardMapForSlot(char *slotName, MemoryContext cxt); #endif /* SHARDSPLIT_SHARED_MEMORY_H */ ================================================ FILE: src/include/distributed/shared_connection_stats.h ================================================ /*------------------------------------------------------------------------- * * shared_connection_stats.h * Central management of connections and their life-cycle * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARED_CONNECTION_STATS_H #define SHARED_CONNECTION_STATS_H #define ADJUST_POOLSIZE_AUTOMATICALLY 0 #define DISABLE_CONNECTION_THROTTLING -1 #define DISABLE_REMOTE_CONNECTIONS_FOR_LOCAL_QUERIES -1 #define ALLOW_ALL_EXTERNAL_CONNECTIONS -1 extern int MaxSharedPoolSize; extern int LocalSharedPoolSize; extern int MaxClientConnections; extern void InitializeSharedConnectionStats(void); extern void WaitForSharedConnection(void); extern void WakeupWaiterBackendsForSharedConnection(void); extern size_t SharedConnectionStatsShmemSize(void); extern void SharedConnectionStatsShmemInit(void); extern int GetMaxClientConnections(void); extern int GetMaxSharedPoolSize(void); extern int GetLocalSharedPoolSize(void); extern bool TryToIncrementSharedConnectionCounter(const char *hostname, int port); extern void WaitLoopForSharedConnection(const char *hostname, int port); extern void DecrementSharedConnectionCounter(const char *hostname, int port); extern void IncrementSharedConnectionCounter(const char *hostname, int port); extern int AdaptiveConnectionManagementFlag(bool connectToLocalNode, int activeConnectionCount); #endif /* SHARED_CONNECTION_STATS_H */ ================================================ FILE: src/include/distributed/shared_library_init.h ================================================ /*------------------------------------------------------------------------- * * shared_library_init.h * Functionality related to the initialization of the Citus extension. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef SHARED_LIBRARY_INIT_H #define SHARED_LIBRARY_INIT_H #include "columnar/columnar.h" #define GUC_STANDARD 0 #define MAX_SHARD_COUNT 64000 #define MAX_SHARD_REPLICATION_FACTOR 100 extern PGDLLEXPORT ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM; extern PGDLLEXPORT CompressionTypeStr_type extern_CompressionTypeStr; extern PGDLLEXPORT IsColumnarTableAmTable_type extern_IsColumnarTableAmTable; extern PGDLLEXPORT ReadColumnarOptions_type extern_ReadColumnarOptions; extern void StartupCitusBackend(void); extern const char * GetClientMinMessageLevelNameForValue(int minMessageLevel); #endif /* SHARED_LIBRARY_INIT_H */ ================================================ FILE: src/include/distributed/stats/query_stats.h ================================================ /*------------------------------------------------------------------------- * * stats_statements.h * Statement-level statistics for distributed queries. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef QUERY_STATS_H #define QUERY_STATS_H #include "distributed/multi_server_executor.h" #define STATS_SHARED_MEM_NAME "citus_query_stats" extern Size CitusQueryStatsSharedMemSize(void); extern void InitializeCitusQueryStats(void); extern void CitusQueryStatsExecutorsEntry(uint64 queryId, MultiExecutorType executorType, char *partitionKey); extern void CitusQueryStatsSynchronizeEntries(void); extern int StatStatementsPurgeInterval; extern int StatStatementsMax; extern int StatStatementsTrack; typedef enum { STAT_STATEMENTS_TRACK_NONE = 0, STAT_STATEMENTS_TRACK_ALL = 1 } StatStatementsTrackType; #endif /* QUERY_STATS_H */ ================================================ FILE: src/include/distributed/stats/stat_counters.h ================================================ /*------------------------------------------------------------------------- * * stat_counters.h * * This file contains the exported functions to track various statistic * counters for Citus. * * ------------------------------------------------------------------------- */ #ifndef STAT_COUNTERS_H #define STAT_COUNTERS_H /* saved backend stats - constants */ #define SAVED_BACKEND_STATS_HASH_LOCK_TRANCHE_NAME \ "citus_stat_counters saved backend stats hash" /* default value for the GUC variable */ #define ENABLE_STAT_COUNTERS_DEFAULT false /* * Must be in the same order as the output columns defined in citus_stat_counters() UDF, * see src/backend/distributed/sql/udfs/citus_stat_counters/latest.sql */ typedef enum { /* * These are mainly tracked by connection_management.c and * adaptive_executor.c. */ STAT_CONNECTION_ESTABLISHMENT_SUCCEEDED, STAT_CONNECTION_ESTABLISHMENT_FAILED, STAT_CONNECTION_REUSED, /* * These are maintained by ExecCustomScan methods implemented * for CustomScan nodes provided by Citus to account for actual * execution of the queries and subplans. By maintaining these * counters in ExecCustomScan callbacks, we ensure avoid * incrementing them for plain EXPLAIN (i.e., without ANALYZE). * queries. And, prefering the executor methods rather than the * planner methods helps us capture the execution of prepared * statements too. */ STAT_QUERY_EXECUTION_SINGLE_SHARD, STAT_QUERY_EXECUTION_MULTI_SHARD, /* do not use this and ensure it is the last entry */ N_CITUS_STAT_COUNTERS } StatType; /* GUC variable */ extern bool EnableStatCounters; /* shared memory init */ extern void InitializeStatCountersShmem(void); extern Size StatCountersShmemSize(void); /* main entry point for the callers who want to increment the stat counters */ extern void IncrementStatCounterForMyDb(int statId); /* * Exported to define a before_shmem_exit() callback that saves * the stat counters for exited backends into the shared memory. */ extern void SaveBackendStatsIntoSavedBackendStatsHash(void); #endif /* STAT_COUNTERS_H */ ================================================ FILE: src/include/distributed/stats/stat_tenants.h ================================================ /*------------------------------------------------------------------------- * * stat_tenants.h * Routines related to the multi tenant monitor. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_ATTRIBUTE_H #define CITUS_ATTRIBUTE_H #include "executor/execdesc.h" #include "executor/executor.h" #include "storage/lwlock.h" #include "utils/datetime.h" #include "utils/hsearch.h" #include "distributed/hash_helpers.h" #define MAX_TENANT_ATTRIBUTE_LENGTH 100 /* * Hashtable key that defines the identity of a hashtable entry. * The key is the attribute value, e.g distribution column and the colocation group id of the tenant. */ typedef struct TenantStatsHashKey { char tenantAttribute[MAX_TENANT_ATTRIBUTE_LENGTH]; int colocationGroupId; } TenantStatsHashKey; assert_valid_hash_key2(TenantStatsHashKey, tenantAttribute, colocationGroupId); /* * TenantStats is the struct that keeps statistics about one tenant. */ typedef struct TenantStats { TenantStatsHashKey key; /* hash key of entry - MUST BE FIRST */ /* * Number of SELECT queries this tenant ran in this and last periods. */ int readsInLastPeriod; int readsInThisPeriod; /* * Number of INSERT, UPDATE, and DELETE queries this tenant ran in this and last periods. */ int writesInLastPeriod; int writesInThisPeriod; /* * CPU time usage of this tenant in this and last periods. */ double cpuUsageInLastPeriod; double cpuUsageInThisPeriod; /* * The latest time this tenant ran a query. This value is used to update the score later. */ TimestampTz lastQueryTime; /* * The tenant monitoring score of this tenant. This value is increased by ONE_QUERY_SCORE at every query * and halved after every period. This custom scoring mechanism is used to rank the tenants based on * the recency and frequency of their activity. The score is used to rank the tenants and decide which * tenants should be removed from the monitor. */ long long score; /* * The latest time the score of this tenant is halved. This value is used to correctly calculate the reduction later. */ TimestampTz lastScoreReduction; /* * Locks needed to update this tenant's statistics. */ slock_t lock; } TenantStats; /* * MultiTenantMonitor is the struct for keeping the statistics * of the tenants */ typedef struct MultiTenantMonitor { /* * Lock mechanism for the monitor. * Each tenant update acquires the lock in shared mode and * the tenant number reduction and monitor view acquires in exclusive mode. */ NamedLWLockTranche namedLockTranche; LWLock lock; /* * The max length of tenants hashtable is 3 * citus.stat_tenants_limit */ HTAB *tenants; } MultiTenantMonitor; typedef enum { STAT_TENANTS_TRACK_NONE = 0, STAT_TENANTS_TRACK_ALL = 1 } StatTenantsTrackType; extern void CitusAttributeToEnd(QueryDesc *queryDesc); extern void AttributeQueryIfAnnotated(const char *queryString, CmdType commandType); extern char * AnnotateQuery(char *queryString, Const *partitionKeyValue, int colocationId); extern void InitializeMultiTenantMonitorSMHandleManagement(void); extern void AttributeTask(char *tenantId, int colocationGroupId, CmdType commandType); extern ExecutorEnd_hook_type prev_ExecutorEnd; extern int StatTenantsLogLevel; extern int StatTenantsPeriod; extern int StatTenantsLimit; extern int StatTenantsTrack; extern double StatTenantsSampleRateForNewTenants; #endif /*CITUS_ATTRIBUTE_H */ ================================================ FILE: src/include/distributed/string_utils.h ================================================ /*------------------------------------------------------------------------- * * string_utils.h * Utilities related to strings. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_STRING_UTILS_H #define CITUS_STRING_UTILS_H #include "postgres.h" extern char * ConvertIntToString(int val); #define StringStartsWith(str, prefix) \ (strncmp(str, prefix, strlen(prefix)) == 0) #endif /* CITUS_STRING_UTILS_H */ ================================================ FILE: src/include/distributed/subplan_execution.h ================================================ /*------------------------------------------------------------------------- * * subplan_execution.h * * Functions for execution subplans. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef SUBPLAN_EXECUTION_H #define SUBPLAN_EXECUTION_H #include "distributed/multi_physical_planner.h" extern int MaxIntermediateResult; extern int SubPlanLevel; extern void ExecuteSubPlans(DistributedPlan *distributedPlan, bool explainAnalyzeEnabled); /** * IntermediateResultsHashEntry is used to store which nodes need to receive * intermediate results. Given an intermediate result name, you can lookup * the list of nodes that can possibly run a query that will use the * intermediate results. * * The nodeIdList contains a set of unique WorkerNode ids that have placements * that can be used in non-colocated subquery joins with the intermediate result * given in the key. * * writeLocalFile indicates if the intermediate result is accessed during local * execution. Note that there can possibly be an item for the local node in the * NodeIdList. */ typedef struct IntermediateResultsHashEntry { char key[NAMEDATALEN]; List *nodeIdList; bool writeLocalFile; } IntermediateResultsHashEntry; #endif /* SUBPLAN_EXECUTION_H */ ================================================ FILE: src/include/distributed/task_execution_utils.h ================================================ #ifndef TASK_EXECUTION_UTILS_H #define TASK_EXECUTION_UTILS_H extern List * CreateTaskListForJobTree(List *jobTaskList); #endif /* TASK_EXECUTION_UTILS_H */ ================================================ FILE: src/include/distributed/tdigest_extension.h ================================================ /*------------------------------------------------------------------------- * * tdigest_extension.c * Helper functions to get access to tdigest specific data. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_TDIGEST_EXTENSION_H #define CITUS_TDIGEST_EXTENSION_H /* tdigest related functions */ extern Oid TDigestExtensionSchema(void); extern Oid TDigestExtensionTypeOid(void); extern Oid TDigestExtensionAggTDigest1(void); extern Oid TDigestExtensionAggTDigest2(void); extern Oid TDigestExtensionAggTDigestPercentile2(void); extern Oid TDigestExtensionAggTDigestPercentile2a(void); extern Oid TDigestExtensionAggTDigestPercentile3(void); extern Oid TDigestExtensionAggTDigestPercentile3a(void); extern Oid TDigestExtensionAggTDigestPercentileOf2(void); extern Oid TDigestExtensionAggTDigestPercentileOf2a(void); extern Oid TDigestExtensionAggTDigestPercentileOf3(void); extern Oid TDigestExtensionAggTDigestPercentileOf3a(void); #endif /* CITUS_TDIGEST_EXTENSION_H */ ================================================ FILE: src/include/distributed/tenant_schema_metadata.h ================================================ /*------------------------------------------------------------------------- * * tenant_schema_metadata.h * * This file contains functions to query and modify tenant schema metadata, * which is used to track the schemas used for schema-based sharding in * Citus. * * ------------------------------------------------------------------------- */ #ifndef TENANT_SCHEMA_METADATA_H #define TENANT_SCHEMA_METADATA_H #include "postgres.h" /* accessors */ extern Oid ColocationIdGetTenantSchemaId(uint32 colocationId); extern uint32 SchemaIdGetTenantColocationId(Oid schemaId); extern bool IsTenantSchema(Oid schemaId); extern bool IsTenantSchemaColocationGroup(uint32 colocationId); /* * Local only modifiers. * * These functions may not make much sense by themselves. They are mainly * exported for tenant-schema management (schema_based_sharding.c) and * metadata-sync layer (metadata_sync.c). */ extern void InsertTenantSchemaLocally(Oid schemaId, uint32 colocationId); extern void DeleteTenantSchemaLocally(Oid schemaId); #endif /* TENANT_SCHEMA_METADATA_H */ ================================================ FILE: src/include/distributed/time_constants.h ================================================ /*------------------------------------------------------------------------- * time_constants.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef TIME_CONSTANTS_H #define TIME_CONSTANTS_H #define MS 1 #define MS_PER_SECOND MS * 1000 #define MS_PER_MINUTE MS_PER_SECOND * 60 #define MS_PER_HOUR MS_PER_MINUTE * 60 #define MS_PER_DAY MS_PER_HOUR * 24 #endif /* TIME_CONSTANTS_H */ ================================================ FILE: src/include/distributed/transaction_identifier.h ================================================ /* * transaction_identifier.h * * Data structure for distributed transaction id and related function * declarations. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef TRANSACTION_IDENTIFIER_H #define TRANSACTION_IDENTIFIER_H #include "datatype/timestamp.h" /* * Citus identifies a distributed transaction with a triplet consisting of * * - initiatorNodeIdentifier: A unique identifier of the node that initiated * the distributed transaction * - transactionOriginator: Set to true only for the transactions initialized on * the coordinator. This is only useful for MX in order to distinguish the transaction * that started the distributed transaction on the coordinator where we could * have the same transactions' worker queries on the same node * - transactionNumber: A locally unique identifier assigned for the distributed * transaction on the node that initiated the distributed transaction * - timestamp: The current timestamp of distributed transaction initiation * */ typedef struct DistributedTransactionId { int initiatorNodeIdentifier; bool transactionOriginator; uint64 transactionNumber; TimestampTz timestamp; } DistributedTransactionId; extern DistributedTransactionId * GetCurrentDistributedTransactionId(void); extern uint64 CurrentDistributedTransactionNumber(void); #endif /* TRANSACTION_IDENTIFIER_H */ ================================================ FILE: src/include/distributed/transaction_management.h ================================================ /*------------------------------------------------------------------------- * transaction_management.h * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef TRANSACTION_MANAGMENT_H #define TRANSACTION_MANAGMENT_H #include "access/xact.h" #include "catalog/objectaddress.h" #include "lib/ilist.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "utils/hsearch.h" /* forward declare, to avoid recursive includes */ struct DistObjectCacheEntry; /* describes what kind of modifications have occurred in the current transaction */ typedef enum { XACT_MODIFICATION_INVALID = 0, /* placeholder initial value */ XACT_MODIFICATION_NONE, /* no modifications have taken place */ XACT_MODIFICATION_DATA, /* data modifications (DML) have occurred */ XACT_MODIFICATION_MULTI_SHARD /* multi-shard modifications have occurred */ } XactModificationType; /* * Enum defining the state of a coordinated (i.e. a transaction potentially * spanning several nodes). */ typedef enum CoordinatedTransactionState { /* no coordinated transaction in progress, no connections established */ COORD_TRANS_NONE, /* no coordinated transaction in progress, but connections established */ COORD_TRANS_IDLE, /* coordinated transaction in progress */ COORD_TRANS_STARTED, /* coordinated transaction prepared on all workers */ COORD_TRANS_PREPARED, /* coordinated transaction committed */ COORD_TRANS_COMMITTED } CoordinatedTransactionState; /* Enumeration to keep track of context within nested sub-transactions */ typedef struct SubXactContext { SubTransactionId subId; StringInfo setLocalCmds; HTAB *propagatedObjects; } SubXactContext; /* * Function delegated with force_delegation call enforces the distribution argument * along with the colocationId. The latter one is equally important to not allow * the same partition key value into another distributed table which is not co-located * and therefore might be on a different node. */ typedef struct AllowedDistributionColumn { Const *distributionColumnValue; uint32 colocationId; bool isActive; /* In nested executor, track the level at which value is set */ int executorLevel; } AllowedDistributionColumn; /* * BeginXactDeferrableState reflects the value of the DEFERRABLE property * in the BEGIN of a transaction block. */ typedef enum BeginXactDeferrableState { BeginXactDeferrable_NotSet, BeginXactDeferrable_Disabled, BeginXactDeferrable_Enabled, } BeginXactDeferrableState; /* * BeginXactReadOnlyState reflects the value of the READ ONLY property * in the BEGIN of a transaction block. */ typedef enum BeginXactReadOnlyState { BeginXactReadOnly_NotSet, BeginXactReadOnly_Disabled, BeginXactReadOnly_Enabled, } BeginXactReadOnlyState; /* * The current distribution column value passed as an argument to a forced * function call delegation. */ extern AllowedDistributionColumn AllowedDistributionColumnValue; /* * GUC that determines whether a SELECT in a transaction block should also run in * a transaction block on the worker. */ extern bool SelectOpensTransactionBlock; /* * GUC that determines whether a function should be considered a transaction * block. */ extern bool FunctionOpensTransactionBlock; /* state needed to prevent new connections during modifying transactions */ extern XactModificationType XactModificationLevel; extern CoordinatedTransactionState CurrentCoordinatedTransactionState; /* list of connections that are part of the current coordinated transaction */ extern dlist_head InProgressTransactions; /* controls use of locks to enforce safe commutativity */ extern bool AllModificationsCommutative; /* we've deprecated this flag, keeping here for some time not to break existing users */ extern bool EnableDeadlockPrevention; /* number of nested stored procedure call levels we are currently in */ extern int StoredProcedureLevel; /* number of nested DO block levels we are currently in */ extern int DoBlockLevel; /* SET LOCAL statements active in the current (sub-)transaction. */ extern StringInfo activeSetStmts; /* did current transaction modify pg_dist_node? */ extern bool TransactionModifiedNodeMetadata; /* after an explicit BEGIN, keep track of top-level transaction characteristics */ extern BeginXactReadOnlyState BeginXactReadOnly; extern BeginXactDeferrableState BeginXactDeferrable; /* * Coordinated transaction management. */ extern void UseCoordinatedTransaction(void); extern bool InCoordinatedTransaction(void); extern void Use2PCForCoordinatedTransaction(void); extern bool GetCoordinatedTransactionShouldUse2PC(void); extern bool IsMultiStatementTransaction(void); extern void EnsureDistributedTransactionId(void); extern bool MaybeExecutingUDF(void); /* functions for tracking the objects propagated in current transaction */ extern void TrackPropagatedObject(const ObjectAddress *objectAddress); extern void TrackPropagatedTableAndSequences(Oid relationId); extern void ResetPropagatedObjects(void); extern bool HasAnyObjectInPropagatedObjects(List *objectList); /* initialization function(s) */ extern void InitializeTransactionManagement(void); /* other functions */ extern List * ActiveSubXactContexts(void); extern StringInfo BeginAndSetDistributedTransactionIdCommand(void); extern void TriggerNodeMetadataSyncOnCommit(void); #endif /* TRANSACTION_MANAGMENT_H */ ================================================ FILE: src/include/distributed/transaction_recovery.h ================================================ /*------------------------------------------------------------------------- * * transaction_recovery.h * Type and function declarations used in recovering 2PC transactions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef TRANSACTION_RECOVERY_H #define TRANSACTION_RECOVERY_H /* GUC to configure interval for 2PC auto-recovery */ extern int Recover2PCInterval; /* Functions declarations for worker transactions */ extern void LogTransactionRecord(int32 groupId, char *transactionName, FullTransactionId outerXid); extern int RecoverTwoPhaseCommits(void); extern void DeleteWorkerTransactions(WorkerNode *workerNode); #endif /* TRANSACTION_RECOVERY_H */ ================================================ FILE: src/include/distributed/transmit.h ================================================ /*------------------------------------------------------------------------- * * transmit.h * Shared declarations for transmitting files between remote nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef TRANSMIT_H #define TRANSMIT_H #include "c.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "storage/fd.h" /* Function declarations for transmitting files between two nodes */ extern void RedirectCopyDataToRegularFile(const char *filename); extern void SendRegularFile(const char *filename); extern File FileOpenForTransmit(const char *filename, int fileFlags); extern File FileOpenForTransmitPerm(const char *filename, int fileFlags, int fileMode); #endif /* TRANSMIT_H */ ================================================ FILE: src/include/distributed/tuple_destination.h ================================================ /*------------------------------------------------------------------------- * * tuple_destination.h * Tuple destination generic struct. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef TUPLE_DESTINATION_H #define TUPLE_DESTINATION_H #include "access/tupdesc.h" #include "tcop/dest.h" #include "utils/tuplestore.h" #include "distributed/multi_physical_planner.h" typedef struct TupleDestination TupleDestination; /* * TupleDestinationStats holds the size related stats. * * totalIntermediateResultSize is a counter to keep the size * of the intermediate results of complex subqueries and CTEs * so that we can put a limit on the size. */ typedef struct TupleDestinationStats { uint64 totalIntermediateResultSize; } TupleDestinationStats; /* * TupleDestination provides a generic interface for where to send tuples. * * Users of the executor can set task->tupleDest for custom processing of * the result tuples. * * Since a task can have multiple queries, methods of TupleDestination also * accept a queryNumber parameter which denotes the index of the query that * tuple belongs to. */ struct TupleDestination { /* putTuple implements custom processing of a tuple */ void (*putTuple)(TupleDestination *self, Task *task, int placementIndex, int queryNumber, HeapTuple tuple, uint64 tupleLibpqSize); /* tupleDescForQuery returns tuple descriptor for a query number. Can return NULL. */ TupleDesc (*tupleDescForQuery)(TupleDestination *self, int queryNumber); /* * Used to enforce citus.max_intermediate_result_size, could be NULL * if the caller is not interested in the size. */ TupleDestinationStats *tupleDestinationStats; }; extern TupleDestination * CreateTupleStoreTupleDest(Tuplestorestate *tupleStore, TupleDesc tupleDescriptor); extern TupleDestination * CreateTupleDestNone(void); extern DestReceiver * CreateTupleDestDestReceiver(TupleDestination *tupleDest, Task *task, int placementIndex); #endif ================================================ FILE: src/include/distributed/tuplestore.h ================================================ /*------------------------------------------------------------------------- * * tuplestore.h * Utilities regarding calls to PG functions * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef CITUS_TUPLESTORE_H #define CITUS_TUPLESTORE_H #include "funcapi.h" /* Function declaration for getting oid for the given function name */ extern ReturnSetInfo * CheckTuplestoreReturn(FunctionCallInfo fcinfo, TupleDesc *tupdesc); extern Tuplestorestate * SetupTuplestore(FunctionCallInfo fcinfo, TupleDesc *tupdesc); #endif ================================================ FILE: src/include/distributed/type_utils.h ================================================ /*------------------------------------------------------------------------- * * type_utils.h * Utility functions related to types. * * Copyright (c) Citus Data, Inc. *------------------------------------------------------------------------- */ #ifndef TYPE_UTILS_H #define TYPE_UTILS_H typedef struct ClusterClock { uint64 logical; /* cluster clock logical timestamp at the commit */ uint32 counter; /* cluster clock counter value at the commit */ } ClusterClock; extern ClusterClock * ParseClusterClockPGresult(PGresult *result, int rowIndex, int colIndex); extern int cluster_clock_cmp_internal(ClusterClock *clusterClock1, ClusterClock *clusterClock2); #endif /* TYPE_UTILS_H */ ================================================ FILE: src/include/distributed/utils/array_type.h ================================================ /*------------------------------------------------------------------------- * * array_type.h * Utility functions for dealing with array types. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_ARRAY_TYPE_H #define CITUS_ARRAY_TYPE_H #include "postgres.h" #include "utils/array.h" extern Datum * DeconstructArrayObject(ArrayType *arrayObject); extern int32 ArrayObjectCount(ArrayType *arrayObject); extern ArrayType * DatumArrayToArrayType(Datum *datumArray, int datumCount, Oid datumTypeId); extern List * IntegerArrayTypeToList(ArrayType *arrayObject); extern List * TextArrayTypeToIntegerList(ArrayType *arrayObject); extern Datum IntArrayToDatum(uint32 int_array_size, int int_array[]); #endif /* CITUS_ARRAY_TYPE_H */ ================================================ FILE: src/include/distributed/utils/directory.h ================================================ /*------------------------------------------------------------------------- * * directory.h * Utility functions for dealing with directories. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_DIRECTORY_H #define CITUS_DIRECTORY_H #include "postgres.h" #include "lib/stringinfo.h" #define PG_JOB_CACHE_DIR "pgsql_job_cache" extern void CleanupJobCacheDirectory(void); extern void CitusCreateDirectory(StringInfo directoryName); extern void CitusRemoveDirectory(const char *filename); #endif /* CITUS_DIRECTORY_H */ ================================================ FILE: src/include/distributed/utils/distribution_column_map.h ================================================ /*------------------------------------------------------------------------- * * distribution_column_map.h * Declarations for a relation OID to distribution column hash. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef DISTRIBUTION_COLUMN_HASH_H #define DISTRIBUTION_COLUMN_HASH_H #include "postgres.h" #include "nodes/primnodes.h" #include "utils/hsearch.h" typedef HTAB DistributionColumnMap; extern DistributionColumnMap * CreateDistributionColumnMap(void); extern void AddDistributionColumnForRelation(DistributionColumnMap *distributionColumns, Oid relationId, char *distributionColumnName); extern Var * GetDistributionColumnFromMap(DistributionColumnMap *distributionColumnMap, Oid relationId); extern Var * GetDistributionColumnWithOverrides(Oid relationId, DistributionColumnMap *overrides); #endif /* DISTRIBUTION_COLUMN_HASH_H */ ================================================ FILE: src/include/distributed/utils/function.h ================================================ /*------------------------------------------------------------------------- * * function.h * Utility functions for dealing with functions. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef CITUS_FUNCTION_H #define CITUS_FUNCTION_H #include "postgres.h" #include "fmgr.h" extern FmgrInfo * GetFunctionInfo(Oid typeId, Oid accessMethodId, int16 procedureId); #endif /* CITUS_FUNCTION_H */ ================================================ FILE: src/include/distributed/version_compat.h ================================================ /*------------------------------------------------------------------------- * * version_compat.h * Compatibility macros for writing code agnostic to PostgreSQL versions * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef VERSION_COMPAT_H #define VERSION_COMPAT_H #include "postgres.h" #include "access/heapam.h" #include "access/sdir.h" #include "catalog/namespace.h" #include "commands/explain.h" #include "executor/tuptable.h" #include "nodes/parsenodes.h" #include "optimizer/optimizer.h" #include "parser/parse_func.h" #include "tcop/tcopprot.h" #include "pg_version_compat.h" #include "distributed/citus_ruleutils.h" #include "distributed/citus_safe_lib.h" typedef struct { File fd; off_t offset; } FileCompat; static inline int FileWriteCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) { int count = FileWrite(file->fd, buffer, amount, file->offset, wait_event_info); if (count > 0) { file->offset += count; } return count; } static inline int FileReadCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) { int count = FileRead(file->fd, buffer, amount, file->offset, wait_event_info); if (count > 0) { file->offset += count; } return count; } static inline FileCompat FileCompatFromFileStart(File fileDesc) { FileCompat fc; /* ensure uninitialized padding doesn't escape the function */ memset_struct_0(fc); fc.fd = fileDesc; fc.offset = 0; return fc; } #endif /* VERSION_COMPAT_H */ ================================================ FILE: src/include/distributed/worker_create_or_replace.h ================================================ /*------------------------------------------------------------------------- * * worker_create_or_replace.h * Header for handling CREATE OR REPLACE of objects, * even if postgres lacks CREATE OR REPLACE for those objects * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef WORKER_CREATE_OR_REPLACE_H #define WORKER_CREATE_OR_REPLACE_H #include "catalog/objectaddress.h" #define CREATE_OR_REPLACE_COMMAND "SELECT worker_create_or_replace_object(%s);" extern char * WrapCreateOrReplace(const char *sql); extern char * WrapCreateOrReplaceList(List *sqls); extern char * GenerateBackupNameForCollision(const ObjectAddress *address); extern DropStmt * CreateDropStmt(const ObjectAddress *address); extern RenameStmt * CreateRenameStatement(const ObjectAddress *address, char *newName); #endif /* WORKER_CREATE_OR_REPLACE_H */ ================================================ FILE: src/include/distributed/worker_log_messages.h ================================================ /*------------------------------------------------------------------------- * * worker_log_messages.h * Functions for handling log messages from the workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef WORKER_LOG_MESSAGES_H #define WORKER_LOG_MESSAGES_H #include "distributed/connection_management.h" /* minimum log level for worker messages to be propagated */ extern int WorkerMinMessages; void SetCitusNoticeReceiver(MultiConnection *connection); void EnableWorkerMessagePropagation(void); void DisableWorkerMessagePropagation(void); void ErrorIfWorkerErrorIndicationReceived(void); void ResetWorkerErrorIndication(void); #endif /* WORKER_LOG_MESSAGES_H */ ================================================ FILE: src/include/distributed/worker_manager.h ================================================ /*------------------------------------------------------------------------- * * worker_manager.h * Header and type declarations for managing worker nodes and for placing * shards on worker nodes in an intelligent manner. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef WORKER_MANAGER_H #define WORKER_MANAGER_H #include "postgres.h" #include "nodes/pg_list.h" #include "storage/lmgr.h" #include "storage/lockdefs.h" /* Worker nodeName's, nodePort's, and nodeCluster's maximum length */ #define WORKER_LENGTH 256 /* Maximum length of worker port number (represented as string) */ #define MAX_PORT_LENGTH 10 /* Implementation specific definitions used in finding worker nodes */ #define WORKER_RACK_TRIES 5 #define WORKER_DEFAULT_RACK "default" #define WORKER_DEFAULT_CLUSTER "default" #define COORDINATOR_GROUP_ID 0 /* * In memory representation of pg_dist_node table elements. The elements are hold in * WorkerNodeHash table. * IMPORTANT: The order of the fields in this definition should match the * column order of pg_dist_node */ typedef struct WorkerNode { uint32 nodeId; /* node's unique id, key of the hash table */ uint32 workerPort; /* node's port */ char workerName[WORKER_LENGTH]; /* node's name */ int32 groupId; /* node's groupId; same for the nodes that are in the same group */ char workerRack[WORKER_LENGTH]; /* node's network location */ bool hasMetadata; /* node gets metadata changes */ bool isActive; /* node's state */ Oid nodeRole; /* the node's role in its group */ char nodeCluster[NAMEDATALEN]; /* the cluster the node is a part of */ bool metadataSynced; /* node has the most recent metadata */ bool shouldHaveShards; /* if the node should have distributed table shards on it or not */ bool nodeisclone; /* whether this node is a replica */ int32 nodeprimarynodeid; /* nodeid of the primary for this replica */ } WorkerNode; /* Config variables managed via guc.c */ extern int MaxWorkerNodesTracked; extern char *WorkerListFileName; extern char *CurrentCluster; /* Function declarations for finding worker nodes to place shards on */ extern WorkerNode * WorkerGetRandomCandidateNode(List *currentNodeList); extern WorkerNode * WorkerGetRoundRobinCandidateNode(List *workerNodeList, uint64 shardId, uint32 placementIndex); extern uint32 ActivePrimaryNonCoordinatorNodeCount(void); extern uint32 ActiveReadableNodeCount(void); extern List * ActivePrimaryNonCoordinatorNodeList(LOCKMODE lockMode); extern List * ActivePrimaryNodeList(LOCKMODE lockMode); extern List * ActivePrimaryRemoteNodeList(LOCKMODE lockMode); extern bool CoordinatorAddedAsWorkerNode(void); extern List * ReferenceTablePlacementNodeList(LOCKMODE lockMode); extern WorkerNode * CoordinatorNodeIfAddedAsWorkerOrError(void); extern void ErrorIfCoordinatorNotAddedAsWorkerNode(void); extern List * DistributedTablePlacementNodeList(LOCKMODE lockMode); extern bool NodeCanHaveDistTablePlacements(WorkerNode *node); extern List * ActiveReadableNonCoordinatorNodeList(void); extern List * ActiveReadableNodeList(void); extern WorkerNode * FindWorkerNode(const char *nodeName, int32 nodePort); extern WorkerNode * FindWorkerNodeOrError(const char *nodeName, int32 nodePort); extern WorkerNode * FindWorkerNodeAnyCluster(const char *nodeName, int32 nodePort); extern WorkerNode * FindNodeWithNodeId(int nodeId, bool missingOk); extern WorkerNode * FindNodeAnyClusterByNodeId(uint32 nodeId); extern WorkerNode * ModifiableWorkerNode(const char *nodeName, int32 nodePort); extern List * ReadDistNode(bool includeNodesFromOtherClusters); extern void EnsureCoordinator(void); extern void EnsurePropagationToCoordinator(void); extern void EnsureCoordinatorIsInMetadata(void); extern void InsertCoordinatorIfClusterEmpty(void); extern uint32 GroupForNode(char *nodeName, int32 nodePort); extern WorkerNode * PrimaryNodeForGroup(int32 groupId, bool *groupContainsNodes); extern bool NodeIsPrimaryAndRemote(WorkerNode *worker); extern bool NodeIsPrimary(WorkerNode *worker); extern bool NodeIsSecondary(WorkerNode *worker); extern bool NodeIsReadable(WorkerNode *worker); extern bool NodeIsCoordinator(WorkerNode *node); extern WorkerNode * SetWorkerColumn(WorkerNode *workerNode, int columnIndex, Datum value); extern WorkerNode * SetWorkerColumnOptional(WorkerNode *workerNode, int columnIndex, Datum value); extern WorkerNode * SetWorkerColumnLocalOnly(WorkerNode *workerNode, int columnIndex, Datum value); extern uint32 CountPrimariesWithMetadata(void); extern WorkerNode * GetFirstPrimaryWorkerNode(void); /* Function declarations for worker node utilities */ extern int CompareWorkerNodes(const void *leftElement, const void *rightElement); extern uint32 WorkerNodeHashCode(const void *key, Size keySize); extern int WorkerNodeCompare(const void *lhsKey, const void *rhsKey, Size keySize); extern int NodeNamePortCompare(const char *workerLhsName, const char *workerRhsName, int workerLhsPort, int workerRhsPort); #endif /* WORKER_MANAGER_H */ ================================================ FILE: src/include/distributed/worker_protocol.h ================================================ /*------------------------------------------------------------------------- * * worker_protocol.h * Header for shared declarations that are used for caching remote resources * on worker nodes, and also for applying distributed execution primitives. * * Copyright (c) Citus Data, Inc. * * $Id$ * *------------------------------------------------------------------------- */ #ifndef WORKER_PROTOCOL_H #define WORKER_PROTOCOL_H #include "postgres.h" #include "fmgr.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "storage/fd.h" #include "utils/array.h" #include "distributed/shardinterval_utils.h" #include "distributed/version_compat.h" /* Number of rows to prefetch when reading data with a cursor */ #define ROW_PREFETCH_COUNT 50 /* Defines that relate to creating tables */ #define GET_TABLE_DDL_EVENTS "SELECT master_get_table_ddl_events('%s')" #define SET_SEARCH_PATH_COMMAND "SET search_path TO %s" #define CREATE_TABLE_COMMAND "CREATE TABLE %s (%s)" #define CREATE_TABLE_AS_COMMAND "CREATE TABLE %s (%s) AS (%s)" /* Function declarations local to the worker module */ extern uint64 ExtractShardIdFromTableName(const char *tableName, bool missingOk); extern void SetDefElemArg(AlterSeqStmt *statement, const char *name, Node *arg); /* Function declarations shared with the master planner */ extern DestReceiver * CreateFileDestReceiver(char *filePath, MemoryContext tupleContext, bool binaryCopyFormat); extern void FileDestReceiverStats(DestReceiver *dest, uint64 *rowsSent, uint64 *bytesSent); /* Function declaration for parsing tree node */ extern Node * ParseTreeNode(const char *ddlCommand); extern Node * ParseTreeRawStmt(const char *ddlCommand); /* Function declarations for applying distributed execution primitives */ extern Datum worker_apply_shard_ddl_command(PG_FUNCTION_ARGS); /* Function declarations for fetching regular and foreign tables */ extern Datum worker_append_table_to_shard(PG_FUNCTION_ARGS); /* Function declaration for calculating hashed value */ extern Datum worker_hash(PG_FUNCTION_ARGS); /* Function declaration for calculating nextval() in worker */ extern Datum worker_nextval(PG_FUNCTION_ARGS); #endif /* WORKER_PROTOCOL_H */ ================================================ FILE: src/include/distributed/worker_shard_copy.h ================================================ /*------------------------------------------------------------------------- * * worker_shard_copy.c * Copy data to destination shard in a push approach. * * Copyright (c) Citus Data, Inc. * * *------------------------------------------------------------------------- */ #ifndef WORKER_SHARD_COPY_H_ #define WORKER_SHARD_COPY_H_ /* GUC, determining whether Binary Copy is enabled */ extern bool EnableBinaryProtocol; extern DestReceiver * CreateShardCopyDestReceiver(EState *executorState, List * destinationShardFullyQualifiedName, uint32_t destinationNodeId); extern const char * CopyableColumnNamesFromRelationName(const char *schemaName, const char *relationName); extern const char * CopyableColumnNamesFromTupleDesc(TupleDesc tupdesc); #endif /* WORKER_SHARD_COPY_H_ */ ================================================ FILE: src/include/distributed/worker_shard_visibility.h ================================================ /*------------------------------------------------------------------------- * * worker_shard_visibility.h * Hide shard names on MX worker nodes. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef WORKER_SHARD_VISIBILITY_H #define WORKER_SHARD_VISIBILITY_H #include "nodes/nodes.h" extern bool OverrideTableVisibility; extern bool EnableManualChangesToShards; extern char *ShowShardsForAppNamePrefixes; extern void HideShardsFromSomeApplications(Query *query); extern void ResetHideShardsDecision(void); extern void ErrorIfRelationIsAKnownShard(Oid relationId); extern void ErrorIfIllegallyChangingKnownShard(Oid relationId); extern bool RelationIsAKnownShard(Oid shardRelationId); #endif /* WORKER_SHARD_VISIBILITY_H */ ================================================ FILE: src/include/distributed/worker_transaction.h ================================================ /*------------------------------------------------------------------------- * * worker_transaction.h * Type and function declarations used in performing transactions across * workers. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef WORKER_TRANSACTION_H #define WORKER_TRANSACTION_H #include "storage/lockdefs.h" #include "distributed/connection_management.h" #include "distributed/worker_manager.h" /* * TargetWorkerSet is used for determining the type of workers that a command * is targeted to. */ typedef enum TargetWorkerSet { /* * All the active primary nodes in the metadata which have metadata * except the coordinator */ NON_COORDINATOR_METADATA_NODES, /* * All the active primary nodes in the metadata which have metadata * except the local node */ REMOTE_METADATA_NODES, /* * All the active primary nodes in the metadata except the coordinator */ NON_COORDINATOR_NODES, /* * All the active primary nodes in the metadata except the local node */ REMOTE_NODES, /* * All active primary nodes in the metadata */ ALL_SHARD_NODES, /* * All the active primary nodes in the metadata which have metadata * (includes the coodinator if it is added) */ METADATA_NODES } TargetWorkerSet; /* Functions declarations for worker transactions */ extern List * GetWorkerTransactions(void); extern List * TargetWorkerSetNodeList(TargetWorkerSet targetWorkerSet, LOCKMODE lockMode); extern void SendCommandToWorker(const char *nodeName, int32 nodePort, const char *command); extern void SendCommandToWorkersAsUser(TargetWorkerSet targetWorkerSet, const char *nodeUser, const char *command); extern void SendCommandToWorkerAsUser(const char *nodeName, int32 nodePort, const char *nodeUser, const char *command); extern void SendCommandToRemoteMetadataNodesParams(const char *command, const char *user, int parameterCount, const Oid *parameterTypes, const char *const *parameterValues); extern bool SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort, const char *nodeUser, List *commandList); extern bool SendOptionalCommandListToWorkerOutsideTransactionWithConnection( MultiConnection *workerConnection, List * commandList); extern bool SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(const char * nodeName, int32 nodePort, const char * nodeUser, List * commandList); extern void SendCommandToWorkersWithMetadata(const char *command); extern void SendCommandToWorkersWithMetadataViaSuperUser(const char *command); extern void SendCommandListToWorkersWithMetadata(List *commands); extern void SendCommandToRemoteNodesWithMetadata(const char *command); extern void SendCommandToRemoteNodesWithMetadataViaSuperUser(const char *command); extern void SendCommandListToRemoteNodesWithMetadata(List *commands); extern void SendBareCommandListToRemoteMetadataNodes(List *commandList); extern void SendBareCommandListToMetadataWorkers(List *commandList); extern void EnsureNoModificationsHaveBeenDone(void); extern void SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort, const char *nodeUser, List *commandList); extern void SendCommandListToWorkerOutsideTransactionWithConnection(MultiConnection * workerConnection, List *commandList); extern void SendCommandListToWorkerListWithBareConnections(List *workerConnections, List *commandList); extern void SendMetadataCommandListToWorkerListInCoordinatedTransaction(List * workerNodeList, const char * nodeUser, List * commandList); extern void RemoveWorkerTransaction(const char *nodeName, int32 nodePort); /* helper functions for worker transactions */ extern bool IsWorkerTransactionActive(void); extern bool IsWorkerTheCurrentNode(WorkerNode *workerNode); #endif /* WORKER_TRANSACTION_H */ ================================================ FILE: src/include/pg_version_compat.h ================================================ /*------------------------------------------------------------------------- * * pg_version_compat.h * Compatibility macros for writing code agnostic to PostgreSQL versions * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_VERSION_COMPAT_H #define PG_VERSION_COMPAT_H #include "pg_version_constants.h" #if PG_VERSION_NUM >= PG_VERSION_18 #define create_foreignscan_path_compat(a, b, c, d, e, f, g, h, i, j, k) \ create_foreignscan_path( \ (a), /* root */ \ (b), /* rel */ \ (c), /* target */ \ (d), /* rows */ \ 0, /* disabled_nodes */ \ (e), /* startup_cost */ \ (f), /* total_cost */ \ (g), /* pathkeys */ \ (h), /* required_outer */ \ (i), /* fdw_outerpath */ \ (j), /* fdw_restrictinfo*/ \ (k) /* fdw_private */ \ ) /* PG-18 introduced get_op_index_interpretation, old name was get_op_btree_interpretation */ #define get_op_btree_interpretation(opno) get_op_index_interpretation(opno) /* PG-18 unified row-compare operator codes under COMPARE_* */ #define ROWCOMPARE_NE COMPARE_NE #elif PG_VERSION_NUM >= PG_VERSION_17 #define create_foreignscan_path_compat(a, b, c, d, e, f, g, h, i, j, k) \ create_foreignscan_path( \ (a), (b), (c), (d), \ (e), (f), \ (g), (h), (i), (j), (k) \ ) #endif #if PG_VERSION_NUM >= PG_VERSION_17 #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_conversion.h" #include "catalog/pg_database.h" #include "catalog/pg_default_acl.h" #include "catalog/pg_depend.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_init_privs.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" #include "catalog/pg_operator.h" #include "catalog/pg_opfamily.h" #include "catalog/pg_parameter_acl.h" #include "catalog/pg_policy.h" #include "catalog/pg_proc.h" #include "catalog/pg_publication.h" #include "catalog/pg_publication_namespace.h" #include "catalog/pg_publication_rel.h" #include "catalog/pg_rewrite.h" #include "catalog/pg_statistic_ext.h" #include "catalog/pg_subscription.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_transform.h" #include "catalog/pg_trigger.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" #include "catalog/pg_ts_parser.h" #include "catalog/pg_ts_template.h" #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" /* * This enum covers all system catalogs whose OIDs can appear in * pg_depend.classId or pg_shdepend.classId. */ typedef enum ObjectClass { OCLASS_CLASS, /* pg_class */ OCLASS_PROC, /* pg_proc */ OCLASS_TYPE, /* pg_type */ OCLASS_CAST, /* pg_cast */ OCLASS_COLLATION, /* pg_collation */ OCLASS_CONSTRAINT, /* pg_constraint */ OCLASS_CONVERSION, /* pg_conversion */ OCLASS_DEFAULT, /* pg_attrdef */ OCLASS_LANGUAGE, /* pg_language */ OCLASS_LARGEOBJECT, /* pg_largeobject */ OCLASS_OPERATOR, /* pg_operator */ OCLASS_OPCLASS, /* pg_opclass */ OCLASS_OPFAMILY, /* pg_opfamily */ OCLASS_AM, /* pg_am */ OCLASS_AMOP, /* pg_amop */ OCLASS_AMPROC, /* pg_amproc */ OCLASS_REWRITE, /* pg_rewrite */ OCLASS_TRIGGER, /* pg_trigger */ OCLASS_SCHEMA, /* pg_namespace */ OCLASS_STATISTIC_EXT, /* pg_statistic_ext */ OCLASS_TSPARSER, /* pg_ts_parser */ OCLASS_TSDICT, /* pg_ts_dict */ OCLASS_TSTEMPLATE, /* pg_ts_template */ OCLASS_TSCONFIG, /* pg_ts_config */ OCLASS_ROLE, /* pg_authid */ OCLASS_ROLE_MEMBERSHIP, /* pg_auth_members */ OCLASS_DATABASE, /* pg_database */ OCLASS_TBLSPACE, /* pg_tablespace */ OCLASS_FDW, /* pg_foreign_data_wrapper */ OCLASS_FOREIGN_SERVER, /* pg_foreign_server */ OCLASS_USER_MAPPING, /* pg_user_mapping */ OCLASS_DEFACL, /* pg_default_acl */ OCLASS_EXTENSION, /* pg_extension */ OCLASS_EVENT_TRIGGER, /* pg_event_trigger */ OCLASS_PARAMETER_ACL, /* pg_parameter_acl */ OCLASS_POLICY, /* pg_policy */ OCLASS_PUBLICATION, /* pg_publication */ OCLASS_PUBLICATION_NAMESPACE, /* pg_publication_namespace */ OCLASS_PUBLICATION_REL, /* pg_publication_rel */ OCLASS_SUBSCRIPTION, /* pg_subscription */ OCLASS_TRANSFORM, /* pg_transform */ } ObjectClass; #define LAST_OCLASS OCLASS_TRANSFORM /* * Determine the class of a given object identified by objectAddress. * * We implement it as a function instead of an array because the OIDs aren't * consecutive. */ static inline ObjectClass getObjectClass(const ObjectAddress *object) { /* only pg_class entries can have nonzero objectSubId */ if (object->classId != RelationRelationId && object->objectSubId != 0) { elog(ERROR, "invalid non-zero objectSubId for object class %u", object->classId); } switch (object->classId) { case RelationRelationId: { /* caller must check objectSubId */ return OCLASS_CLASS; } case ProcedureRelationId: { return OCLASS_PROC; } case TypeRelationId: { return OCLASS_TYPE; } case CastRelationId: { return OCLASS_CAST; } case CollationRelationId: { return OCLASS_COLLATION; } case ConstraintRelationId: { return OCLASS_CONSTRAINT; } case ConversionRelationId: { return OCLASS_CONVERSION; } case AttrDefaultRelationId: { return OCLASS_DEFAULT; } case LanguageRelationId: { return OCLASS_LANGUAGE; } case LargeObjectRelationId: { return OCLASS_LARGEOBJECT; } case OperatorRelationId: { return OCLASS_OPERATOR; } case OperatorClassRelationId: { return OCLASS_OPCLASS; } case OperatorFamilyRelationId: { return OCLASS_OPFAMILY; } case AccessMethodRelationId: { return OCLASS_AM; } case AccessMethodOperatorRelationId: { return OCLASS_AMOP; } case AccessMethodProcedureRelationId: { return OCLASS_AMPROC; } case RewriteRelationId: { return OCLASS_REWRITE; } case TriggerRelationId: { return OCLASS_TRIGGER; } case NamespaceRelationId: { return OCLASS_SCHEMA; } case StatisticExtRelationId: { return OCLASS_STATISTIC_EXT; } case TSParserRelationId: { return OCLASS_TSPARSER; } case TSDictionaryRelationId: { return OCLASS_TSDICT; } case TSTemplateRelationId: { return OCLASS_TSTEMPLATE; } case TSConfigRelationId: { return OCLASS_TSCONFIG; } case AuthIdRelationId: { return OCLASS_ROLE; } case AuthMemRelationId: { return OCLASS_ROLE_MEMBERSHIP; } case DatabaseRelationId: { return OCLASS_DATABASE; } case TableSpaceRelationId: { return OCLASS_TBLSPACE; } case ForeignDataWrapperRelationId: { return OCLASS_FDW; } case ForeignServerRelationId: { return OCLASS_FOREIGN_SERVER; } case UserMappingRelationId: { return OCLASS_USER_MAPPING; } case DefaultAclRelationId: { return OCLASS_DEFACL; } case ExtensionRelationId: { return OCLASS_EXTENSION; } case EventTriggerRelationId: { return OCLASS_EVENT_TRIGGER; } case ParameterAclRelationId: { return OCLASS_PARAMETER_ACL; } case PolicyRelationId: { return OCLASS_POLICY; } case PublicationNamespaceRelationId: { return OCLASS_PUBLICATION_NAMESPACE; } case PublicationRelationId: { return OCLASS_PUBLICATION; } case PublicationRelRelationId: { return OCLASS_PUBLICATION_REL; } case SubscriptionRelationId: { return OCLASS_SUBSCRIPTION; } case TransformRelationId: { return OCLASS_TRANSFORM; } } /* shouldn't get here */ elog(ERROR, "unrecognized object class: %u", object->classId); return OCLASS_CLASS; /* keep compiler quiet */ } #include "commands/tablecmds.h" static inline void RangeVarCallbackOwnsTable(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg) { return RangeVarCallbackMaintainsTable(relation, relId, oldRelId, arg); } #include "catalog/pg_attribute.h" #include "utils/syscache.h" static inline int getAttstattarget_compat(HeapTuple attTuple) { bool isnull; Datum dat = SysCacheGetAttr(ATTNUM, attTuple, Anum_pg_attribute_attstattarget, &isnull); return (isnull ? -1 : DatumGetInt16(dat)); } #include "catalog/pg_statistic_ext.h" static inline int getStxstattarget_compat(HeapTuple tup) { bool isnull; Datum dat = SysCacheGetAttr(STATEXTOID, tup, Anum_pg_statistic_ext_stxstattarget, &isnull); return (isnull ? -1 : DatumGetInt16(dat)); } #define getAlterStatsStxstattarget_compat(a) ((Node *) makeInteger(a)) #define getIntStxstattarget_compat(a) (intVal(a)) #define WaitEventSetTracker_compat CurrentResourceOwner #define identitySequenceRelation_compat(a) (a) #define matched_compat(a) (a->matchKind == MERGE_WHEN_MATCHED) #define getProcNo_compat(a) (a->vxid.procNumber) #define getLxid_compat(a) (a->vxid.lxid) #else #define Anum_pg_collation_colllocale Anum_pg_collation_colliculocale #define Anum_pg_database_datlocale Anum_pg_database_daticulocale #include "access/htup_details.h" static inline int getAttstattarget_compat(HeapTuple attTuple) { return ((Form_pg_attribute) GETSTRUCT(attTuple))->attstattarget; } #include "catalog/pg_statistic_ext.h" static inline int getStxstattarget_compat(HeapTuple tup) { return ((Form_pg_statistic_ext) GETSTRUCT(tup))->stxstattarget; } #define getAlterStatsStxstattarget_compat(a) (a) #define getIntStxstattarget_compat(a) (a) #define WaitEventSetTracker_compat CurrentMemoryContext #define identitySequenceRelation_compat(a) (RelationGetRelid(a)) #define matched_compat(a) (a->matched) #define create_foreignscan_path_compat(a, b, c, d, e, f, g, h, i, j, \ k) create_foreignscan_path(a, b, c, d, e, f, g, h, \ i, k) #define getProcNo_compat(a) (a->pgprocno) #define getLxid_compat(a) (a->lxid) #define COLLPROVIDER_BUILTIN 'b' #endif #define SetListCellPtr(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define fcGetArgValue(fc, n) ((fc)->args[n].value) #define fcGetArgNull(fc, n) ((fc)->args[n].isnull) #define fcSetArgExt(fc, n, val, is_null) \ (((fc)->args[n].isnull = (is_null)), ((fc)->args[n].value = (val))) #define fcSetArg(fc, n, value) fcSetArgExt(fc, n, value, false) #define fcSetArgNull(fc, n) fcSetArgExt(fc, n, (Datum) 0, true) #define CREATE_SEQUENCE_COMMAND \ "CREATE %sSEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \ " MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \ " START WITH " INT64_FORMAT " CACHE " INT64_FORMAT " %sCYCLE" #endif /* PG_VERSION_COMPAT_H */ ================================================ FILE: src/include/pg_version_constants.h ================================================ /*------------------------------------------------------------------------- * * pg_version_constants.h * pg version related constants. * * Copyright (c) Citus Data, Inc. * *------------------------------------------------------------------------- */ #ifndef PG_VERSION_CONSTANTS #define PG_VERSION_CONSTANTS #define PG_VERSION_16 160000 #define PG_VERSION_17 170000 #define PG_VERSION_18 180000 #define PG_VERSION_19 190000 #endif /* PG_VERSION_CONSTANTS */ ================================================ FILE: src/test/cdc/Makefile ================================================ #------------------------------------------------------------------------- # # Makefile for src/test/cdc # # Test that CDC publication works correctly. # #------------------------------------------------------------------------- subdir = src/test/cdc top_builddir = ../../.. include $(top_builddir)/Makefile.global pg_version = $(shell $(PG_CONFIG) --version 2>/dev/null) pg_whole_version = $(shell echo "$(pg_version)"| sed -e 's/^PostgreSQL \([0-9]*\)\(\.[0-9]*\)\{0,1\}\(.*\)/\1\2/') pg_major_version = $(shell echo "$(pg_whole_version)"| sed -e 's/^\([0-9]\{2\}\)\(.*\)/\1/') export pg_major_version test_path = t/*.pl # copied from pgxs/Makefile.global to use postgres' abs build dir for pg_regress ifeq ($(enable_tap_tests),yes) define citus_prove_installcheck rm -rf '$(CURDIR)'/tmp_check $(MKDIR_P) '$(CURDIR)'/tmp_check cd $(srcdir) && \ TESTDIR='$(CURDIR)' \ PATH="$(bindir):$$PATH" \ PGPORT='6$(DEF_PGPORT)' \ top_builddir='$(CURDIR)/$(top_builddir)' \ PG_REGRESS='$(pgxsdir)/src/test/regress/pg_regress' \ TEMP_CONFIG='$(CURDIR)'/postgresql.conf \ $(PROVE) $(PG_PROVE_FLAGS) $(PROVE_FLAGS) $(if $(PROVE_TESTS),$(PROVE_TESTS),$(test_path)) endef else citus_prove_installcheck = @echo "TAP tests not enabled when postgres was compiled" endif installcheck: $(citus_prove_installcheck) clean distclean maintainer-clean: rm -rf tmp_check ================================================ FILE: src/test/cdc/postgresql.conf ================================================ shared_preload_libraries='citus' ================================================ FILE: src/test/cdc/t/001_cdc_create_distributed_table_test.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE sensors REPLICA IDENTITY FULL;'); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table insert data'); # Update some data in the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," UPDATE sensors SET eventdatetime=NOW(), measure_data = jsonb_set(measure_data, '{val}', measureid::text::jsonb , TRUE), measure_status = CASE WHEN measureid % 2 = 0 THEN 'y' ELSE 'n' END, measure_comment= 'Comment:' || measureid::text;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table update data'); # Delete some data from the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," DELETE FROM sensors WHERE (measureid % 2) = 0;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table delete data'); $node_coordinator->safe_psql('postgres',"TRUNCATE sensors;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table delete data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/002_cdc_create_distributed_table_concurrently.pl ================================================ # CDC test for create_distributed_table_concurrently use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table_concurrently('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC test - create_distributed_table_concurrently insert data'); # Update some data in the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," UPDATE sensors SET eventdatetime=NOW(), measure_data = jsonb_set(measure_data, '{val}', measureid::text::jsonb , TRUE), measure_status = CASE WHEN measureid % 2 = 0 THEN 'y' ELSE 'n' END, measure_comment= 'Comment:' || measureid::text;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC test - create_distributed_table_concurrently update data'); # Delete some data from the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," DELETE FROM sensors WHERE (measureid % 2) = 0;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC test - create_distributed_table_concurrently delete data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/003_cdc_parallel_insert.pl ================================================ # CDC test for inserts during create distributed table concurrently use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; use threads; # Initialize co-ordinator node our $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $add_local_meta_data_stmt = qq(SELECT citus_add_local_table_to_metadata('sensors');); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639, ""); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres',$add_local_meta_data_stmt); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_replication_slots_for_citus_cluster($node_coordinator,\@workers,'sensors'); connect_cdc_client_to_citus_cluster_publications($node_coordinator,\@workers,$node_cdc_client); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - insert data'); sub create_distributed_table_thread() { $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table_concurrently('sensors', 'measureid');"); } sub insert_data_into_distributed_table_thread() { # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(-10,-1)i;"); } # Create the distributed table concurrently in a separate thread. my $thr_create = threads->create(\&create_distributed_table_thread); my $thr_insert = threads->create(\&insert_data_into_distributed_table_thread); $thr_create->join(); $thr_insert->join(); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - insert data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/004_cdc_move_shard.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); print("worker1 port:" . $workers[1]->port() . "\n"); print("cdc_client port:" .$node_cdc_client->port() . "\n"); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,100)i;"); $node_coordinator->safe_psql('postgres',"SET citus.shard_count = 2; SELECT create_distributed_table_concurrently('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - schema change before move'); my $shard_to_move = $node_coordinator->safe_psql('postgres', "SELECT shardid FROM citus_shards ORDER BY shardid LIMIT 1;"); my $host1 = $node_coordinator->safe_psql('postgres', "SELECT nodename FROM citus_shards ORDER BY shardid LIMIT 1;"); my $port1 = $node_coordinator->safe_psql('postgres', "SELECT nodeport FROM citus_shards ORDER BY shardid LIMIT 1;"); my $shard_last = $node_coordinator->safe_psql('postgres', "SELECT shardid FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $host2 = $node_coordinator->safe_psql('postgres', "SELECT nodename FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $port2 = $node_coordinator->safe_psql('postgres', "SELECT nodeport FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $move_params = "select citus_move_shard_placement($shard_to_move,'$host1',$port1,'$host2',$port2,'force_logical');"; print("move_params: $move_params\n"); $node_coordinator->safe_psql('postgres',$move_params); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); #wait_for_cdc_client_to_catch_up_with_workers(\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - schema change and move shard'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/005_cdc_reference_table_test.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; my $ref_select_stmt = qq(SELECT * FROM reference_table ORDER BY measureid;); ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); $node_coordinator->safe_psql('postgres',"CREATE TABLE reference_table(measureid integer PRIMARY KEY);"); $node_cdc_client->safe_psql('postgres',"CREATE TABLE reference_table(measureid integer PRIMARY KEY);"); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'reference_table'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); # Create the reference table in the coordinator and cdc client nodes. $node_coordinator->safe_psql('postgres',"SELECT create_reference_table('reference_table');"); create_cdc_slots_for_workers(\@workers); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$ref_select_stmt); is($result, 1, 'CDC reference taable test 1'); # Insert data to the reference table in the coordinator node. $node_coordinator->safe_psql('postgres',"INSERT INTO reference_table SELECT i FROM generate_series(0,100)i;"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$ref_select_stmt); is($result, 1, 'CDC reference taable test 2'); $node_coordinator->safe_psql('postgres',"INSERT INTO reference_table SELECT i FROM generate_series(101,200)i;"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$ref_select_stmt); is($result, 1, 'CDC reference taable test 3'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/006_cdc_schema_change_and_move.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $select_stmt_after_drop = qq(SELECT measureid, eventdatetime, measure_data, measure_status, measure_comment FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); print("worker1 port:" . $workers[1]->port() . "\n"); print("cdc_client port:" .$node_cdc_client->port() . "\n"); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE sensors REPLICA IDENTITY FULL;'); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,100)i;"); $node_coordinator->safe_psql('postgres',"SET citus.shard_count = 2; SELECT create_distributed_table_concurrently('sensors', 'measureid');"); #connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); create_cdc_slots_for_workers(\@workers); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - schema change before move'); $node_coordinator->safe_psql('postgres',"ALTER TABLE sensors DROP COLUMN meaure_quantity;"); my $shard_to_move = $node_coordinator->safe_psql('postgres', "SELECT shardid FROM citus_shards ORDER BY shardid LIMIT 1;"); my $host1 = $node_coordinator->safe_psql('postgres', "SELECT nodename FROM citus_shards ORDER BY shardid LIMIT 1;"); my $port1 = $node_coordinator->safe_psql('postgres', "SELECT nodeport FROM citus_shards ORDER BY shardid LIMIT 1;"); my $shard_last = $node_coordinator->safe_psql('postgres', "SELECT shardid FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $host2 = $node_coordinator->safe_psql('postgres', "SELECT nodename FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $port2 = $node_coordinator->safe_psql('postgres', "SELECT nodeport FROM citus_shards ORDER BY shardid DESC LIMIT 1;"); my $move_params = "select citus_move_shard_placement($shard_to_move,'$host1',$port1,'$host2',$port2,'force_logical');"; print("move_params: $move_params\n"); $node_coordinator->safe_psql('postgres',$move_params); $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 'A', 'I <3 Citus' FROM generate_series(-10,-1)i;"); $node_cdc_client->safe_psql('postgres',"ALTER TABLE sensors DROP COLUMN meaure_quantity;"); wait_for_cdc_client_to_catch_up_with_workers(\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - schema change and move shard'); # Update some data in the sensors table to check the schema change handling logic in CDC decoder. $node_coordinator->safe_psql('postgres'," UPDATE sensors SET measure_status = CASE WHEN measureid % 2 = 0 THEN 'y' ELSE 'n' END;"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - update data after schema change'); # Update some data in the sensors table to check the schema change handling logic in CDC decoder. $node_coordinator->safe_psql('postgres'," DELETE FROM sensors WHERE measure_status = 'n';"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - delete data after schem change'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/007_cdc_undistributed_table_test.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); my $command = "UPDATE pg_dist_node SET shouldhaveshards = true;"; $node_coordinator->safe_psql('postgres',$command); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table insert data'); # Update some data in the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," UPDATE sensors SET eventdatetime=NOW(), measure_data = jsonb_set(measure_data, '{val}', measureid::text::jsonb , TRUE), measure_status = CASE WHEN measureid % 2 = 0 THEN 'y' ELSE 'n' END, measure_comment= 'Comment:' || measureid::text;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table update data'); # Delete some data from the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," DELETE FROM sensors WHERE (measureid % 2) = 0;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table delete data'); $node_coordinator->safe_psql('postgres'," SELECT undistribute_table('sensors',cascade_via_foreign_keys=>true);"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table delete data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/008_cdc_shard_split_test.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; my $citus_config = " citus.shard_count = 2 citus.shard_replication_factor = 1 "; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636, $citus_config); my $command = "UPDATE pg_dist_node SET shouldhaveshards = true;"; $node_coordinator->safe_psql('postgres',$command); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table create data'); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(-100,100)i;"); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table insert data'); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $node_coordinator->safe_psql('postgres'," SELECT citus_split_shard_by_split_points(102008,ARRAY['-50'],ARRAY[1,2], 'block_writes');"); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table split data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/009_cdc_shard_split_test_non_blocking.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; my $citus_config = " citus.shard_count = 2 citus.shard_replication_factor = 1 "; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636, $citus_config); my $command = "UPDATE pg_dist_node SET shouldhaveshards = true;"; $node_coordinator->safe_psql('postgres',$command); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table create data'); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(-100,100)i;"); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table insert data'); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $node_coordinator->safe_psql('postgres'," SELECT citus_split_shard_by_split_points(102008,ARRAY['-50'],ARRAY[1,2], 'force_logical');"); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table split data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/010_cdc_shard_split_parallel_insert.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; use threads; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; my $citus_config = " citus.shard_count = 2 citus.shard_replication_factor = 1 "; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636, $citus_config); my $command = "UPDATE pg_dist_node SET shouldhaveshards = true;"; $node_coordinator->safe_psql('postgres',$command); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table create data'); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(-100,100)i;"); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table insert data'); sub insert_data_into_distributed_table_thread() { # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(101,200)i;"); } sub split_distributed_table_thread() { $node_coordinator->safe_psql('postgres'," SELECT citus_split_shard_by_split_points(102008,ARRAY['-50'],ARRAY[1,2], 'force_logical');"); } # Create the distributed table concurrently in a separate thread. my $thr_create = threads->create(\&split_distributed_table_thread); # Insert some data to the sensors table in the coordinator node while the table is being distributed. my $thr_insert = threads->create(\&insert_data_into_distributed_table_thread); # Wait for the threads to finish. $thr_create->join(); $thr_insert->join(); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table split data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/011_cdc_alter_distributed_table.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $add_local_meta_data_stmt = qq(SELECT citus_add_local_table_to_metadata('sensors');); my $result = 0; my $citus_config = " citus.shard_count = 2 citus.shard_replication_factor = 1 "; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636, $citus_config); my $command = "UPDATE pg_dist_node SET shouldhaveshards = true;"; $node_coordinator->safe_psql('postgres',$command); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres',$add_local_meta_data_stmt); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_replication_slots_for_citus_cluster($node_coordinator,\@workers,'sensors'); connect_cdc_client_to_citus_cluster_publications($node_coordinator,\@workers,$node_cdc_client); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table_concurrently('sensors', 'measureid');"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator,\@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - distributed table create data'); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," SELECT alter_distributed_table('sensors', shard_count:=6, cascade_to_colocated:=true);"); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Compare the data in the coordinator and cdc client nodes. $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC split test - alter distributed table '); #$node_cdc_client->safe_psql("postgres","alter subscription cdc_subscription refresh publication;"); $node_cdc_client->safe_psql("postgres","alter subscription cdc_subscription_1 refresh publication;"); #Drop the CDC client subscription and recreate them , since the #alter_distributed_table has changed the Oid of the distributed table. #So the CDC client has to create Oid to table mappings again for #CDC to work again. drop_cdc_client_subscriptions($node_cdc_client,\@workers); create_cdc_publication_and_replication_slots_for_citus_cluster($node_coordinator,\@workers,'sensors'); connect_cdc_client_to_citus_cluster_publications($node_coordinator,\@workers,$node_cdc_client); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC basic test - distributed table insert data'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/012_cdc_restart_test.pl ================================================ # Basic CDC test for create_distributed_table use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); # Create the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); create_cdc_slots_for_workers(\@workers); # Distribut the sensors table to worker nodes. $node_coordinator->safe_psql('postgres',"SELECT create_distributed_table('sensors', 'measureid');"); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC restart test - distributed table creation'); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,10)i;"); # Wait for the data changes to be replicated to the cdc client node. wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC restart test - distributed table insert data'); print("stopping worker 0"); $workers[0]->stop(); print("starting worker 0 againg.."); $workers[0]->start(); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); # Insert some data to the sensors table in the coordinator node. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(11,20)i;"); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC restart test - distributed table after restart'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/013_cdc_drop_last_column_for_one_shard.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $select_stmt_after_drop = qq(SELECT measureid, eventdatetime, measure_data, meaure_quantity, measure_status FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); print("worker1 port:" . $workers[1]->port() . "\n"); print("cdc_client port:" .$node_cdc_client->port() . "\n"); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE sensors REPLICA IDENTITY FULL;'); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,100)i;"); $node_coordinator->safe_psql('postgres',"SET citus.shard_count = 2; SELECT create_distributed_table_concurrently('sensors', 'measureid');"); #connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); create_cdc_slots_for_workers(\@workers); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - schema change before move'); my $shard_id = $workers[1]->safe_psql('postgres', "SELECT shardid FROM citus_shards ORDER BY shardid LIMIT 1;"); my $shard_to_drop_column = "sensors_" . $shard_id; $workers[1]->safe_psql('postgres',"ALTER TABLE $shard_to_drop_column DROP COLUMN measure_comment;"); $workers[1]->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A' FROM generate_series(-10,-1)i;"); wait_for_cdc_client_to_catch_up_with_workers(\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt_after_drop); is($result, 1, 'CDC create_distributed_table - schema change and move shard'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/014_cdc_with_table_like_shard_name.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $select_stmt_after_drop = qq(SELECT measureid, eventdatetime, measure_data, meaure_quantity, measure_status FROM sensors ORDER BY measureid, eventdatetime, measure_data;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); print("worker1 port:" . $workers[1]->port() . "\n"); print("cdc_client port:" .$node_cdc_client->port() . "\n"); # Creeate the sensors table and ndexes. my $initial_schema = " CREATE TABLE sensors( measureid integer, eventdatetime timestamptz, measure_data jsonb, meaure_quantity decimal(15, 2), measure_status char(1), measure_comment varchar(44), PRIMARY KEY (measureid, eventdatetime, measure_data)); CREATE INDEX index_on_sensors ON sensors(lower(measureid::text)); ALTER INDEX index_on_sensors ALTER COLUMN 1 SET STATISTICS 1000; CREATE INDEX hash_index_on_sensors ON sensors USING HASH((measure_data->'IsFailed')); CREATE INDEX index_with_include_on_sensors ON sensors ((measure_data->'IsFailed')) INCLUDE (measure_data, eventdatetime, measure_status); CREATE STATISTICS stats_on_sensors (dependencies) ON measureid, eventdatetime FROM sensors;"; my $shard_like_table_schema = " CREATE TABLE data_100008( id integer, data integer, PRIMARY KEY (data));"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres',$shard_like_table_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE sensors REPLICA IDENTITY FULL;'); $node_cdc_client->safe_psql('postgres',$initial_schema); $node_cdc_client->safe_psql('postgres',$shard_like_table_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'sensors,data_100008'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,100)i;"); $node_coordinator->safe_psql('postgres',"SET citus.shard_count = 2; SELECT create_distributed_table_concurrently('sensors', 'measureid');"); #connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); create_cdc_slots_for_workers(\@workers); connect_cdc_client_to_workers_publication(\@workers, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_citus_cluster($node_coordinator, \@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - basic test'); $workers[1]->safe_psql('postgres',$shard_like_table_schema); $workers[1]->safe_psql('postgres','\d'); $workers[1]->safe_psql('postgres'," INSERT INTO data_100008 SELECT i, i*10 FROM generate_series(-10,10)i;"); wait_for_cdc_client_to_catch_up_with_workers(\@workers); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt_after_drop); is($result, 1, 'CDC create_distributed_table - normal table with name like shard'); drop_cdc_client_subscriptions($node_cdc_client,\@workers); done_testing(); ================================================ FILE: src/test/cdc/t/015_cdc_without_citus.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(2,"localhost",57636); our $node_cdc_client = create_node('cdc_client', 0, "localhost", 57639); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); print("worker1 port:" . $workers[1]->port() . "\n"); print("cdc_client port:" .$node_cdc_client->port() . "\n"); my $initial_schema = " CREATE TABLE data_100008( id integer, data integer, PRIMARY KEY (data));"; $node_coordinator->safe_psql('postgres','DROP extension citus'); $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE data_100008 REPLICA IDENTITY FULL;'); $node_cdc_client->safe_psql('postgres',$initial_schema); create_cdc_publication_and_slots_for_coordinator($node_coordinator,'data_100008'); connect_cdc_client_to_coordinator_publication($node_coordinator, $node_cdc_client); wait_for_cdc_client_to_catch_up_with_coordinator($node_coordinator); #insert data into the sensors table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO data_100008 SELECT i, i*10 FROM generate_series(-10,10)i;"); $result = compare_tables_in_different_nodes($node_coordinator,$node_cdc_client,'postgres',$select_stmt); is($result, 1, 'CDC create_distributed_table - basic test'); $node_cdc_client->safe_psql('postgres',"drop subscription cdc_subscription"); done_testing(); ================================================ FILE: src/test/cdc/t/016_cdc_wal2json.pl ================================================ # Schema change CDC test for Citus use strict; use warnings; use Test::More; use lib './t'; use cdctestlib; use threads; # Initialize co-ordinator node my $select_stmt = qq(SELECT * FROM data_100008 ORDER BY id;); my $result = 0; ### Create the citus cluster with coordinator and two worker nodes our ($node_coordinator, @workers) = create_citus_cluster(1,"localhost",57636); print("coordinator port: " . $node_coordinator->port() . "\n"); print("worker0 port:" . $workers[0]->port() . "\n"); my $initial_schema = " CREATE TABLE data_100008( id integer, data text, PRIMARY KEY (data));"; $node_coordinator->safe_psql('postgres',$initial_schema); $node_coordinator->safe_psql('postgres','ALTER TABLE data_100008 REPLICA IDENTITY FULL;'); $node_coordinator->safe_psql('postgres',"SELECT pg_catalog.pg_create_logical_replication_slot('cdc_replication_slot','wal2json');"); #insert data into the data_100008 table in the coordinator node before distributing the table. $node_coordinator->safe_psql('postgres'," INSERT INTO data_100008 SELECT i, 'my test data ' || i FROM generate_series(-1,1)i;"); my $output = $node_coordinator->safe_psql('postgres',"SELECT * FROM pg_logical_slot_get_changes('cdc_replication_slot', NULL, NULL);"); print($output); my $change_string_expected = '[0,"my test data 0"]'; if ($output =~ /$change_string_expected/) { $result = 1; } else { $result = 0; } is($result, 1, 'CDC create_distributed_table - wal2json test'); $node_coordinator->safe_psql('postgres',"SELECT pg_drop_replication_slot('cdc_replication_slot');"); done_testing(); ================================================ FILE: src/test/cdc/t/cdctestlib.pm ================================================ use strict; use warnings; my $pg_major_version = int($ENV{'pg_major_version'}); print("working with PG major version : $pg_major_version\n"); if ($pg_major_version >= 15) { eval "use PostgreSQL::Test::Cluster"; eval "use PostgreSQL::Test::Utils"; } else { eval "use PostgresNode"; } #use PostgresNode; use DBI; our $NODE_TYPE_COORDINATOR = 1; our $NODE_TYPE_WORKER = 2; our $NODE_TYPE_CDC_CLIENT = 3; sub compare_tables_in_different_nodes { my $result = 1; my ($node1, $node2, $dbname, $stmt) = @_; # Connect to the first database node my $dbh1 = DBI->connect("dbi:Pg:" . $node1->connstr($dbname)); # Connect to the second database node my $dbh2 = DBI->connect("dbi:Pg:" . $node2->connstr($dbname)); # Define the SQL query for the first database node my $sth1 = $dbh1->prepare($stmt); $sth1->execute(); # Define the SQL query for the second database node my $sth2 = $dbh2->prepare($stmt); $sth2->execute(); # Get the field names for the table my @field_names = @{$sth2->{NAME}}; #$sth1->dump_results(); #$sth2->dump_results(); our @row1, our @row2; # Use a cursor to iterate over the first database node's data while (1) { @row1 = $sth1->fetchrow_array(); @row2 = $sth2->fetchrow_array(); #print("row1: @row1\n"); #print("row2: @row2\n"); # Use a cursor to iterate over the second database node's data if (@row1 and @row2) { #print("row1: @row1\n"); #print("row2: @row2\n"); my $field_count_row1 = scalar @row1; my $field_count_row2 = scalar @row2; if ($field_count_row1 != $field_count_row2) { print "Field count mismatch: $field_count_row1 != $field_count_row2 \n"; print "First row: @row1\n"; #print "Second row: @row2\n"; for (my $i = 0; $i < scalar @row2; $i++) { print("Field $i, field name: $field_names[$i], value: $row2[$i] \n"); } $result = 0; last; } # Compare the data in each field in each row of the two nodes for (my $i = 0; $i < scalar @row1; $i++) { if ($row1[$i] ne $row2[$i]) { print "Data mismatch in field '$field_names[$i]'\n"; print "$row1[$i] != $row2[$i]\n"; print "First row: @row1\n"; print "Second row: @row2\n"; $result = 0; last; } } } elsif (@row1 and !@row2) { print "First node has more rows than the second node\n"; $result = 0; last; } elsif (!@row1 and @row2) { print "Second node has more rows than the first node\n"; $result = 0; last; } else { last; } } $sth1->finish(); $sth2->finish(); $dbh1->disconnect(); $dbh2->disconnect(); return $result; } sub create_node { my ($name,$node_type,$host, $port, $config) = @_; if (!defined($config)) { $config = "" } our $node; if ($pg_major_version >= 15) { $PostgreSQL::Test::Cluster::use_unix_sockets = 0; $PostgreSQL::Test::Cluster::use_tcp = 1; $PostgreSQL::Test::Cluster::test_pghost = 'localhost'; my %params = ( "port" => $port, "host" => "localhost"); $node = PostgreSQL::Test::Cluster->new($name, %params); } else { $PostgresNode::use_tcp = 1; $PostgresNode::test_pghost = '127.0.0.1'; my %params = ( "port" => $port, "host" => "localhost"); $node = get_new_node($name, %params); } print("node's port:" . $node->port . "\n"); $port += 1; my $citus_config_options = " max_connections = 100 max_wal_senders = 100 max_replication_slots = 100 citus.enable_change_data_capture = on log_statement = 'all' citus.override_table_visibility = off"; if ($config ne "") { $citus_config_options = $citus_config_options . $config } my $client_config_options = " max_connections = 100 max_wal_senders = 100 max_replication_slots = 100 "; $node->init(allows_streaming => 'logical'); if ($node_type == $NODE_TYPE_COORDINATOR || $node_type == $NODE_TYPE_WORKER) { $node->append_conf("postgresql.conf",$citus_config_options); } else { $node->append_conf("postgresql.conf",$citus_config_options); } $node->start(); if ($node_type == $NODE_TYPE_COORDINATOR || $node_type == $NODE_TYPE_WORKER) { $node->safe_psql('postgres', "CREATE EXTENSION citus;"); my $value = $node->safe_psql('postgres', "SHOW citus.enable_change_data_capture;"); print("citus.enable_change_data_capture value is $value\n") } return $node; } # Create a Citus cluster with the given number of workers sub create_citus_cluster { my ($no_workers,$host,$port,$citus_config) = @_; my @workers = (); my $node_coordinator; print("citus_config :", $citus_config); if ($citus_config ne "") { $node_coordinator = create_node('coordinator', $NODE_TYPE_COORDINATOR,$host, $port, $citus_config); } else { $node_coordinator = create_node('coordinator', $NODE_TYPE_COORDINATOR,$host, $port); } my $coord_host = $node_coordinator->host(); my $coord_port = $node_coordinator->port(); $node_coordinator->safe_psql('postgres',"SELECT pg_catalog.citus_set_coordinator_host('$coord_host', $coord_port);"); for (my $i = 0; $i < $no_workers; $i++) { $port = $port + 1; my $node_worker; if ($citus_config ne "") { $node_worker = create_node("worker$i", $NODE_TYPE_WORKER,"localhost", $port, $citus_config); } else { $node_worker = create_node("worker$i", $NODE_TYPE_WORKER,"localhost", $port); } my $node_worker_host = $node_worker->host(); my $node_worker_port = $node_worker->port(); $node_coordinator->safe_psql('postgres',"SELECT pg_catalog.citus_add_node('$node_worker_host', $node_worker_port);"); push @workers, $node_worker; } return $node_coordinator, @workers; } sub create_cdc_publication_and_replication_slots_for_citus_cluster { my $node_coordinator = $_[0]; my $workersref = $_[1]; my $table_names = $_[2]; create_cdc_publication_and_slots_for_coordinator($node_coordinator, $table_names); create_cdc_slots_for_workers($workersref); } sub create_cdc_publication_and_slots_for_coordinator { my $node_coordinator = $_[0]; my $table_names = $_[1]; print("node node_coordinator connstr: \n" . $node_coordinator->connstr()); my $pub = $node_coordinator->safe_psql('postgres',"SELECT * FROM pg_publication WHERE pubname = 'cdc_publication';"); if ($pub ne "") { $node_coordinator->safe_psql('postgres',"DROP PUBLICATION IF EXISTS cdc_publication;"); } $node_coordinator->safe_psql('postgres',"CREATE PUBLICATION cdc_publication FOR TABLE $table_names;"); $node_coordinator->safe_psql('postgres',"SELECT pg_catalog.pg_create_logical_replication_slot('cdc_replication_slot','pgoutput',false)"); } sub create_cdc_slots_for_workers { my $workersref = $_[0]; for (@$workersref) { my $slot = $_->safe_psql('postgres',"select * from pg_replication_slots where slot_name = 'cdc_replication_slot';"); if ($slot ne "") { $_->safe_psql('postgres',"SELECT pg_catalog.pg_drop_replication_slot('cdc_replication_slot');"); } $_->safe_psql('postgres',"SELECT pg_catalog.pg_create_logical_replication_slot('cdc_replication_slot','pgoutput',false)"); } } sub connect_cdc_client_to_citus_cluster_publications { my $node_coordinator = $_[0]; my $workersref = $_[1]; my $node_cdc_client = $_[2]; my $num_args = scalar(@_); if ($num_args > 3) { my $copy_arg = $_[3]; connect_cdc_client_to_coordinator_publication($node_coordinator,$node_cdc_client, $copy_arg); } else { connect_cdc_client_to_coordinator_publication($node_coordinator,$node_cdc_client); } connect_cdc_client_to_workers_publication($workersref, $node_cdc_client); } sub connect_cdc_client_to_coordinator_publication { my $node_coordinator = $_[0]; my $node_cdc_client = $_[1]; my $num_args = scalar(@_); my $copy_data = ""; if ($num_args > 2) { my $copy_arg = $_[2]; $copy_data = 'copy_data='. $copy_arg; } else { $copy_data = 'copy_data=false'; } my $conn_str = $node_coordinator->connstr() . " dbname=postgres"; my $subscription = 'cdc_subscription'; print "creating subscription $subscription for coordinator: $conn_str\n"; $node_cdc_client->safe_psql('postgres'," CREATE SUBSCRIPTION $subscription CONNECTION '$conn_str' PUBLICATION cdc_publication WITH ( create_slot=false, enabled=true, slot_name=cdc_replication_slot," . $copy_data. ");" ); } sub connect_cdc_client_to_workers_publication { my $workersref = $_[0]; my $node_cdc_client = $_[1]; my $i = 1; for (@$workersref) { my $conn_str = $_->connstr() . " dbname=postgres"; my $subscription = 'cdc_subscription_' . $i; print "creating subscription $subscription for node$i: $conn_str\n"; my $subscription_stmt = "CREATE SUBSCRIPTION $subscription CONNECTION '$conn_str' PUBLICATION cdc_publication WITH ( create_slot=false, enabled=true, slot_name=cdc_replication_slot, copy_data=false); "; $node_cdc_client->safe_psql('postgres',$subscription_stmt); $i++; } } sub wait_for_cdc_client_to_catch_up_with_citus_cluster { my $node_coordinator = $_[0]; my ($workersref) = $_[1]; my $subscription = 'cdc_subscription'; print "coordinator: waiting for cdc client subscription $subscription to catch up\n"; $node_coordinator->wait_for_catchup($subscription); wait_for_cdc_client_to_catch_up_with_workers($workersref); } sub wait_for_cdc_client_to_catch_up_with_coordinator { my $node_coordinator = $_[0]; my $subscription = 'cdc_subscription'; print "coordinator: waiting for cdc client subscription $subscription to catch up\n"; $node_coordinator->wait_for_catchup($subscription); } sub wait_for_cdc_client_to_catch_up_with_workers { my ($workersref) = $_[0]; my $i = 1; for (@$workersref) { my $subscription = 'cdc_subscription_' . $i; print "node$i: waiting for cdc client subscription $subscription to catch up\n"; $_->wait_for_catchup($subscription); $i++; } } sub drop_cdc_client_subscriptions { my $node = $_[0]; my ($workersref) = $_[1]; $node->safe_psql('postgres',"drop subscription cdc_subscription"); my $i = 1; for (@$workersref) { my $subscription = 'cdc_subscription_' . $i; $node->safe_psql('postgres',"drop subscription " . $subscription); $i++; } } ================================================ FILE: src/test/hammerdb/README.md ================================================ # How to trigger hammerdb benchmark jobs You can trigger two types of hammerdb benchmark jobs: -ch_benchmark (analytical and transactional queries) -tpcc_benchmark (only transactional queries) Your branch will be run against `master` branch. In order to trigger the jobs prepend `ch_benchmark/` or `tpcc_benchmark/` to your branch and push it. For example if you were running on a feature/improvement branch with name `improve/adaptive_executor`. In order to trigger a tpcc benchmark, you can do the following: ```bash git checkout improve/adaptive_executor git checkout -b tpcc_benchmark/improve/adaptive_executor git push origin tpcc_benchmark/improve/adaptive_executor # the tpcc benchmark job will be triggered. ``` You will see the results in a branch in [https://github.com/citusdata/release-test-results](https://github.com/citusdata/release-test-results). The branch name will be something like: `citus_github_push/citusbot_tpcc_benchmark_rg//`. On success, which is the vast majority of the cases, the resource groups on Azure which are automatically created for this purpose are deleted automatically. On failure, you need to manually drop the resource groups named: `citusbot_ch_benchmark_rg` and `citusbot_tpcc_benchmark_rg`. To check whether the job failed, go to Azure portal and search for these resource group names. If the resource group doesn't have any virtual machines or the virtual machines don't have any CPU/Memory load at all, it is highly possible that the job failed. Delete the resource groups and re-try with another branch. These jobs use the hammerdb tool under the hood, please see [https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark](https://github.com/citusdata/test-automation#running-automated-hammerdb-benchmark) for more details. ================================================ FILE: src/test/hammerdb/run_hammerdb.sh ================================================ #!/bin/bash # fail if trying to reference a variable that is not set. set -u # exit immediately if a command fails set -e # echo commands set -x rg=$1 export RESOURCE_GROUP_NAME="${rg}" export AZURE_REGION=westus2 # the branch name is stored in CIRCLE_BRANCH env variable in CI jobs. export BRANCH="${CIRCLE_BRANCH}" # add github to known hosts echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts set +x git config --global credential.helper store git clone https://${GIT_USERNAME}:${GIT_TOKEN}@github.com/citusdata/test-automation set -x cd test-automation test_automation_dir=$(pwd) # add the ssh keys eval `ssh-agent -s` ssh-add ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub now=$(date +"%m_%d_%Y_%s") new_branch_name=citus_github_push/"${rg}"/"${now}" git checkout -b "${new_branch_name}" cd ./fabfile/hammerdb_confs branch_config=current_branch.ini # create a config for this branch cp master.ini "${branch_config}" # put the branch name to the config file. sed -i "s@master@${BRANCH}@g" "${branch_config}" cd "${test_automation_dir}" cd ./hammerdb git config --global user.email "citus-bot@microsoft.com" git config --global user.name "citus bot" git add -A git commit -m "test hammerdb: ${rg} vs master" git push origin "${new_branch_name}" cd "${test_automation_dir}" cd ./hammerdb if [ "$rg" = "citusbot_ch_benchmark_rg" ]; then export IS_CH=true export IS_TPCC=true fi if [ "$rg" = "citusbot_tpcc_benchmark_rg" ]; then export IS_CH=false export IS_TPCC=true fi # create cluster and run the hammerd benchmark ./create-run.sh ================================================ FILE: src/test/regress/.gitignore ================================================ # Local binaries /pg_regress # Generated subdirectories /tmp_check/ /tmp_upgrade/ /tmp_citus_upgrade/ /tmp_citus_tarballs/ /tmp_citus_test/ /build/ /results/ /log/ /bin/test/results/ # Regression test output /regression.diffs /regression.out /test_times.log # Regression test timing /test_times.log # Failure test side effets /proxy.output # python *.pyc # core dumps core # pg_vanilla output folders pg_vanilla_outputs ================================================ FILE: src/test/regress/Makefile ================================================ # Makefile for tests of the Citus extension citus_subdir = src/test/regress citus_top_builddir = ../../.. include $(citus_top_builddir)/Makefile.global # ensure MAJORVERSION is defined (missing in older versions) ifndef MAJORVERSION MAJORVERSION := $(basename $(VERSION)) endif ifndef CITUSVERSION CITUSVERSION := endif ifndef CITUSLIBDIR CITUSLIBDIR := endif ifndef N1MODE N1MODE := endif # Add ./bin to $PATH so we use our custom diff instead of the default diff tool. # We do this to be able to mask shard Ids, placement Ids, node ports, etc. MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) export PATH := $(MAKEFILE_DIR)/bin:$(PATH) export PG_REGRESS_DIFF_OPTS = -dU10 -w # Use lower isolation test timeout, the 5 minute default is waaay too long for # us so we use 60 seconds instead. We should detect blockages very quickly and # most queries that we run are also very fast. So fast even that 60 seconds is # usually too long. However, any commands involving logical replication can be # quite slow, especially shard splits and especially on CI. So we still keep # this value at the pretty high 60 seconds because even those slow commands are # definitly stuck when they take longer than that. export PGISOLATIONTIMEOUT = 60 ## ## Citus regression support ## MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" --postgres-builddir="$(postgres_abs_builddir)" --postgres-srcdir="$(postgres_abs_srcdir)" --citus_abs_srcdir="$(citus_abs_srcdir)" $(if $(CITUSVERSION),--citus-version=$(CITUSVERSION)) $(if $(CITUSLIBDIR),--citus-libdir="$(CITUSLIBDIR)") $(if $(N1MODE),--n-1-mode=$(N1MODE)) MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) --launcher="$(citus_abs_srcdir)/log_test_times" pg_upgrade_check = $(citus_abs_srcdir)/citus_tests/upgrade/pg_upgrade_test.py citus_upgrade_check =CITUS_OLD_VERSION=$(citus-old-version) $(citus_abs_srcdir)/citus_tests/upgrade/citus_upgrade_test.py arbitrary_config_check = $(citus_abs_srcdir)/citus_tests/arbitrary_configs/citus_arbitrary_configs.py query_generator_check = $(citus_abs_srcdir)/citus_tests/query_generator/bin/run_query_compare_test.py template_isolation_files = $(shell find $(citus_abs_srcdir)/spec/ -name '*.spec') generated_isolation_files = $(patsubst $(citus_abs_srcdir)/spec/%,$(citus_abs_srcdir)/build/specs/%,$(template_isolation_files)) vanilla_diffs_file = $(citus_abs_srcdir)/pg_vanilla_outputs/$(MAJORVERSION)/regression.diffs # have make check actually run all tests, but keep check-full as an # intermediate, for muscle memory backward compatibility. check: check-full check-enterprise-full # check-full triggers all tests that ought to be run routinely check-full: check-multi check-multi-mx check-multi-1 check-multi-1-create-citus check-operations check-add-backup-node check-follower-cluster check-isolation check-failure check-split check-vanilla check-columnar check-columnar-isolation check-pg-upgrade check-arbitrary-configs check-citus-upgrade check-citus-upgrade-mixed check-citus-upgrade-local check-citus-upgrade-mixed-local check-pytest check-query-generator check-tap # check-enterprise-full triggers all enterprise specific tests check-enterprise-full: check-enterprise check-enterprise-isolation check-enterprise-failure check-enterprise-isolation-logicalrep-1 check-enterprise-isolation-logicalrep-2 check-enterprise-isolation-logicalrep-3 ISOLATION_DEPDIR=.deps/isolation ISOLATION_BUILDDIR=build/specs # this can be used to print a value of variable # ex: make print-generated_isolation_files print-% : ; @echo $* = $($*) .PHONY: create-symbolic-link create-symbolic-link: mkdir -p $(citus_abs_srcdir)/build ln -fsn $(citus_abs_srcdir)/expected $(citus_abs_srcdir)/build/ # How this target works: # cpp is used before running isolation tests to preprocess spec files. # This way we can include any file we want to. Currently this is used to include mx common part. # spec files are put to /build/specs for clear separation between generated files and template files # a symbolic link is created for /expected in build/expected/. # when running isolation tests, as the inputdir, build is passed so # it runs the spec files from build/specs and checks the expected output from build/expected. # /specs is renamed as /spec because postgres first look at the specs file under current directory, # so this is renamed to avoid that since we are running the isolation tests from build/specs now. $(generated_isolation_files): $(citus_abs_srcdir)/build/specs/%: $(citus_abs_srcdir)/spec/% Makefile @mkdir -p $(citus_abs_srcdir)/$(ISOLATION_DEPDIR) $(citus_abs_srcdir)/$(ISOLATION_BUILDDIR) @# -MF is used to store dependency files(.Po) in another directory for separation @# -MT is used to change the target of the rule emitted by dependency generation. @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. @# -undef is used to not predefine any system-specific or GCC-specific macros. @# `man cpp` for further information @# sed is used to strip the final // comments, because OSX cpp is weird cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(ISOLATION_DEPDIR)/$(*F).Po -MT$@ $< | sed '/^\/\//d' | sed '/^$$/d' > $@ Isolation_Po_files := $(wildcard $(ISOLATION_DEPDIR)/*.Po) ifneq (,$(Isolation_Po_files)) include $(Isolation_Po_files) endif isolation_test_files=$(generated_isolation_files) create-symbolic-link ifndef CITUS_VALGRIND_LOG_FILE CITUS_VALGRIND_LOG_FILE := citus_valgrind_test_log.txt endif # check-base only sets up a testing environment so you can specify all your tests using EXTRA_TESTS check-base: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/base_schedule $(EXTRA_TESTS) # check-minimal only sets up the cluster check-minimal: all $(pg_regress_multi_check) --load-extension=citus --load-extension=hll \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/minimal_schedule $(EXTRA_TESTS) check-base-vg: all $(pg_regress_multi_check) --load-extension=citus \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/base_schedule $(EXTRA_TESTS) check-base-mx: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/mx_base_schedule $(EXTRA_TESTS) check-minimal-mx: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/mx_minimal_schedule $(EXTRA_TESTS) check-custom-schedule: all $(pg_regress_multi_check) --load-extension=citus --worker-count=$(WORKERCOUNT) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-failure-custom-schedule: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy --worker-count=$(WORKERCOUNT) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-isolation-custom-schedule: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester --worker-count=$(WORKERCOUNT) \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-custom-schedule-vg: all $(pg_regress_multi_check) --load-extension=citus \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 --worker-count=$(WORKERCOUNT) \ --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-failure-custom-schedule-vg: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 --worker-count=$(WORKERCOUNT) \ --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-isolation-custom-schedule-vg: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester --worker-count=$(WORKERCOUNT) \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-empty: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) $(EXTRA_TESTS) check-multi: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-enterprise: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/enterprise_schedule $(EXTRA_TESTS) check-multi-1: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_1_schedule $(EXTRA_TESTS) check-multi-1-create-citus: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_1_create_citus_schedule $(EXTRA_TESTS) check-multi-hyperscale: all $(pg_regress_multi_check) --conninfo="$(conninfo)" --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule_hyperscale $(EXTRA_TESTS) check-multi-hyperscale-superuser: all $(pg_regress_multi_check) --conninfo="$(conninfo)" --worker-1-public-hostname="$(worker_1_public_hostname)" --worker-2-public-hostname="$(worker_2_public_hostname)" --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule_hyperscale_superuser $(EXTRA_TESTS) check-multi-vg: all $(pg_regress_multi_check) --load-extension=citus --valgrind \ --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-1-vg: all $(pg_regress_multi_check) --load-extension=citus --valgrind \ --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_1_schedule $(EXTRA_TESTS) check-isolation: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/isolation_schedule $(EXTRA_TESTS) check-enterprise-isolation: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/enterprise_isolation_schedule $(EXTRA_TESTS) # we have separate targets for logical replication tests because they take very long to complete # hence this increases parallelism a lot without sacrifing any coverage. check-enterprise-isolation-logicalrep-1: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/enterprise_isolation_logicalrep_1_schedule $(EXTRA_TESTS) check-enterprise-isolation-logicalrep-2: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/enterprise_isolation_logicalrep_2_schedule $(EXTRA_TESTS) check-enterprise-isolation-logicalrep-3: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/enterprise_isolation_logicalrep_3_schedule $(EXTRA_TESTS) check-isolation-base: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/base_isolation_schedule $(EXTRA_TESTS) # ci takes regression.diffs output from another location, so copy diffs file at the end. check-vanilla: all $(pg_regress_multi_check) --vanillatest || (cp $(vanilla_diffs_file) $(citus_abs_srcdir)/regression.diffs && false) check-multi-mx: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_mx_schedule $(EXTRA_TESTS) check-tap: $(MAKE) -C $(citus_top_srcdir)/src/test/tap installcheck check-follower-cluster: all $(pg_regress_multi_check) --load-extension=citus --follower-cluster \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_follower_schedule $(EXTRA_TESTS) check-add-backup-node: all $(pg_regress_multi_check) --load-extension=citus --follower-cluster --backupnodetest --worker-count=6 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_add_backup_node_schedule $(EXTRA_TESTS) check-operations: all $(pg_regress_multi_check) --load-extension=citus --worker-count=6 \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/operations_schedule $(EXTRA_TESTS) check-columnar-minimal: $(pg_regress_multi_check) --load-extension=citus_columnar \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/minimal_columnar_schedule $(EXTRA_TESTS) check-columnar: all $(pg_regress_multi_check) --load-extension=citus_columnar \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/columnar_schedule $(EXTRA_TESTS) check-columnar-vg: all $(pg_regress_multi_check) --load-extension=citus_columnar --valgrind \ --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/columnar_schedule $(EXTRA_TESTS) check-columnar-isolation: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus_columnar --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/columnar_isolation_schedule $(EXTRA_TESTS) check-columnar-custom-schedule: all $(pg_regress_multi_check) --load-extension=citus_columnar \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-columnar-custom-schedule-vg: all $(pg_regress_multi_check) --load-extension=citus_columnar \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 \ --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-columnar-isolation-custom-schedule: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus_columnar --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-columnar-isolation-custom-schedule-vg: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus_columnar --isolationtester \ --valgrind --pg_ctl-timeout=360 --connection-timeout=500000 --valgrind-path=valgrind --valgrind-log-file=$(CITUS_VALGRIND_LOG_FILE) \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/$(SCHEDULE) $(EXTRA_TESTS) check-split: all $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/split_schedule $(EXTRA_TESTS) check-failure: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/failure_schedule $(EXTRA_TESTS) check-failure-base: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/failure_base_schedule $(EXTRA_TESTS) check-enterprise-failure: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/enterprise_failure_schedule $(EXTRA_TESTS) check-pg-upgrade: all $(pg_upgrade_check) --old-bindir=$(old-bindir) --new-bindir=$(new-bindir) --pgxsdir=$(pgxsdir) $(if $(filter true,$(test-with-columnar)),--test-with-columnar) check-arbitrary-configs: all ${arbitrary_config_check} --bindir=$(bindir) --pgxsdir=$(pgxsdir) --parallel=$(parallel) --configs=$(CONFIGS) --seed=$(seed) check-arbitrary-base: all ${arbitrary_config_check} --bindir=$(bindir) --pgxsdir=$(pgxsdir) --parallel=$(parallel) --configs=$(CONFIGS) --seed=$(seed) --base check-pytest: pytest -n auto check-query-generator: all ${query_generator_check} --bindir=$(bindir) --pgxsdir=$(pgxsdir) --seed=$(seed) check-citus-upgrade: all $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-pre-tar=$(citus-pre-tar) \ --citus-post-tar=$(citus-post-tar) check-citus-upgrade-mixed: all $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-pre-tar=$(citus-pre-tar) \ --citus-post-tar=$(citus-post-tar) \ --mixed check-citus-upgrade-local: all clean-upgrade-artifacts $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-old-version=$(citus-old-version) check-citus-upgrade-mixed-local: all clean-upgrade-artifacts $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-old-version=$(citus-old-version) \ --mixed check-citus-minor-upgrade: all $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-pre-tar=$(citus-pre-tar) \ --citus-post-tar=$(citus-post-tar) \ --minor-upgrade check-citus-minor-upgrade-local: all clean-upgrade-artifacts $(citus_upgrade_check) \ --bindir=$(bindir) \ --pgxsdir=$(pgxsdir) \ --citus-old-version=$(citus-old-version) \ --minor-upgrade clean-upgrade-artifacts: rm -rf $(citus_abs_srcdir)/tmp_citus_upgrade/ /tmp/citus_copy/ clean distclean maintainer-clean: rm -rf input/ output/ rm -rf tmp_check/ rm -rf tmp_citus_test/ ================================================ FILE: src/test/regress/Pipfile ================================================ [[source]] name = "pypi" url = "https://pypi.python.org/simple" verify_ssl = true [packages] mitmproxy = {git = "https://github.com/citusdata/mitmproxy.git", ref = "main"} "aioquic" = ">=1.2.0,<1.3.0" "mitmproxy-rs" = ">=0.12.6,<0.13.0" argon2-cffi = ">=23.1.0" bcrypt = ">=4.1.2" brotli = "<=1.2.0" h11 = "==0.16.0" h2 = "==4.3.0" tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" pytest-asyncio = "*" pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" [dev-packages] black = "==24.10.0" isort = "*" flake8 = "*" flake8-bugbear = "*" [requires] python_version = "3.12" ================================================ FILE: src/test/regress/README.md ================================================ # How our testing works We use the test tooling of postgres to run our tests. This tooling is very simple but effective. The basics it runs a series of `.sql` scripts, gets their output and stores that in `results/$sqlfilename.out`. It then compares the actual output to the expected output with a simple `diff` command: ```bash diff results/$sqlfilename.out expected/$sqlfilename.out ``` ## Schedules Which sql scripts to run is defined in a schedule file, e.g. `multi_schedule`, `multi_mx_schedule`. ## Makefile In our `Makefile` we have rules to run the different types of test schedules. You can run them from the root of the repository like so: ```bash # e.g. the multi_schedule make install -j9 && make -C src/test/regress/ check-multi ``` Take a look at the makefile for a list of all the testing targets. ### Running a specific test Often you want to run a specific test and don't want to run everything. You can simply use `run_test.py [test_name]` script like below in that case. It detects the test schedule and make target to run the given test. ```bash src/test/regress/citus_tests/run_test.py multi_utility_warnings ``` You can pass `--repeat` or `r` parameter to run the given test for multiple times. ```bash src/test/regress/citus_tests/run_test.py multi_utility_warnings -r 1000 ``` To force the script to use base schedules rather than minimal ones, you can pass `-b` or `--use-base-schedule`. ```bash src/test/regress/citus_tests/run_test.py coordinator_shouldhaveshards -r 1000 --use-base-schedule ``` If you would like to run a specific test on a certain target you can use one of the following commands to do so: ```bash # If your tests needs almost no setup you can use check-minimal make install -j9 && make -C src/test/regress/ check-minimal EXTRA_TESTS='multi_utility_warnings' # For columnar specific tests, use check-columnar-minimal instead of check-minimal make install -j9 && make -C src/test/regress/ check-columnar-minimal # Often tests need some testing data, if you get missing table errors using # check-minimal you should try check-base make install -j9 && make -C src/test/regress/ check-base EXTRA_TESTS='with_prepare' # Sometimes this is still not enough and some other test needs to be run before # the test you want to run. You can do so by adding it to EXTRA_TESTS too. make install -j9 && make -C src/test/regress/ check-base EXTRA_TESTS='add_coordinator coordinator_shouldhaveshards' ``` ## Normalization The output of tests is sadly not completely predictable. Still we want to compare the output of different runs and error when the important things are different. We do this by not using the regular system `diff` to compare files. Instead we use `src/test/regress/bin/diff` which does the following things: 1. Change the `$sqlfilename.out` file by running it through `sed` using the `src/test/regress/bin/normalize.sed` file. This does stuff like replacing numbers that keep changing across runs with an `XXX` string, e.g. portnumbers or transaction numbers. 2. Backup the original output to `$sqlfilename.out.unmodified` in case it's needed for debugging 3. Compare the changed `results` and `expected` files with the system `diff` command. ## Updating the expected test output Sometimes you add a test to an existing file, or test output changes in a way that's not bad (possibly even good if support for queries is added). In those cases you want to update the expected test output. The way to do this is very simple, you run the test and copy the new .out file in the `results` directory to the `expected` directory, e.g.: ```bash make install -j9 && make -C src/test/regress/ check-minimal EXTRA_TESTS='multi_utility_warnings' cp src/test/regress/{results,expected}/multi_utility_warnings.out ``` Or if it's a columnar test, you can use: ```bash make install -j9 && make -C src/test/regress/ check-columnar-minimal EXTRA_TESTS='multi_utility_warnings' cp src/test/regress/{results,expected}/multi_utility_warnings.out ``` ## Adding a new test file Adding a new test file is quite simple: 1. Write the SQL file in the `sql` directory 2. Add it to a schedule file, to make sure it's run in CI 3. Run the test 4. Check that the output is as expected 5. Copy the `.out` file from `results` to `expected` ## Isolation testing See [`src/test/regress/spec/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/spec/README.md) ## Pytest testing See [`src/test/regress/citus_tests/test/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/citus_tests/test/README.md) ## Upgrade testing See [`src/test/regress/citus_tests/upgrade/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/citus_tests/upgrade/README.md) ## Arbitrary configs testing See [`src/test/regress/citus_tests/arbitrary_configs/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/citus_tests/arbitrary_configsupgrade/README.md) ## Failure testing See [`src/test/regress/mitmscripts/README.md`](https://github.com/citusdata/citus/blob/master/src/test/regress/mitmscripts/README.md) ## Perl test setup script To automatically setup a citus cluster in tests we use our `src/test/regress/pg_regress_multi.pl` script. This sets up a citus cluster and then starts the standard postgres test tooling. You almost never have to change this file. ## Handling different test outputs Sometimes the test output changes because we run tests in different configurations. The most common example is an output that changes in different Postgres versions. We highly encourage to find a way to avoid these test outputs. You can try the following, if applicable to the changing output: - Change the test such that you still test what you want, but you avoid the different test outputs. - Reduce the test verbosity via: `\set VERBOSITY terse`, `SET client_min_messages TO error`, etc - Drop the specific test lines altogether, if the test is not critical. - Use utility functions that modify the output to your preference, like [coordinator_plan](https://github.com/citusdata/citus/blob/main/src/test/regress/sql/multi_test_helpers.sql#L23), which modifies EXPLAIN output - Add [a normalization rule](https://github.com/citusdata/citus/blob/main/ci/README.md#normalize_expectedsh) Alternative test output files are highly discouraged, so only add one when strictly necessary. In order to maintain a clean test suite, make sure to explain why it has an alternative output in the test header, and when we can drop the alternative output file in the future. For example: ```sql -- -- MULTI_INSERT_SELECT -- -- This test file has an alternative output because of the change in the -- display of SQL-standard function's arguments in INSERT/SELECT in PG15. -- The alternative output can be deleted when we drop support for PG14 -- ``` Including important keywords, like "PG14", "PG15", "alternative output" will help cleaning up in the future. ## Randomly failing tests In CI sometimes a test fails randomly, we call these tests "flaky". To fix these flaky tests see [`src/test/regress/flaky_tests.md`](https://github.com/citusdata/citus/blob/main/src/test/regress/flaky_tests.md) ## Running regression tests in mixed-version scenarios The test runner supports running regression tests in mixed-version (N/N-1) scenarios, where some or all nodes use an older Citus version. This is useful for verifying backward compatibility. ### Environment variables Three environment variables control mixed-version behavior: | Variable | Description | |---|---| | `CITUSVERSION` | The Citus extension version to create (e.g. `13.2-1`). When set, `CREATE EXTENSION citus VERSION ''` is used instead of the default. | | `CITUSLIBDIR` | Path to a directory containing the older `citus.so` library (e.g. `~/citus-libs/17/v13.2.0`). When set, the specified library is loaded instead of the currently installed one. | | `N1MODE` | Controls which nodes use the older version. Valid values: `all` (all nodes), `workeronly` (only worker 1), `coordinatoronly` (only the coordinator). | When `CITUSVERSION` or `CITUSLIBDIR` is set, the test runner automatically disables version checks (`citus.enable_version_checks=off`, `columnar.enable_version_checks=off`). ### Supported scenarios | Scenario | Description | Variables | |---|---|---| | SQL version N-1 | All nodes create the Citus extension at an older version | `CITUSVERSION=13.2-1 N1MODE=all` | | Library version N-1 | All nodes load `citus.so` from an older release | `CITUSLIBDIR=~/citus-libs/17/v13.2.0 N1MODE=all` | | Worker N-1 | Only worker 1 uses the older library and extension version | `CITUSLIBDIR=~/citus-libs/18/v14.0 CITUSVERSION=14.0-1` | | Coordinator N-1 | Only the coordinator uses the older library and extension version | `CITUSLIBDIR=~/citus-libs/17/v13.2.0 CITUSVERSION=13.2-1 N1MODE=coordinatoronly` | ### Schedule considerations Not all schedules are compatible with N-1 testing: - **`check-multi-1-create-citus`** and **`check-multi-mx`**: These schedules drop and recreate the Citus extension using the default version, which is incompatible with N-1 setups. - **`check-vanilla`**: The test preparation steps have not yet been adapted for N-1 workflows. The `multi_1_create_citus_schedule` was split out from `multi_1_schedule` to isolate tests that drop/create the Citus extension, so the remaining `multi_1_schedule` tests can be reused safely in N-1 scenarios. ### Local testing #### Prerequisites 1. Install the older Citus version you want to test against (e.g. 13.2). 2. Copy the `citus.so` from that installation to a known directory: ```bash mkdir -p ~/citus-libs/17/v13.2.0 cp /path/to/old/citus.so ~/citus-libs/17/v13.2.0/ ``` 3. Install the current (HEAD) version of Citus: ```bash make install -j9 ``` #### Example commands ```bash # Test with only the SQL extension version pinned to N-1 on all nodes CITUSVERSION=13.2-1 N1MODE=all make -C src/test/regress check-minimal # Test with both the library and extension version pinned to N-1 on worker only CITUSLIBDIR=~/citus-libs/18/v14.0 CITUSVERSION=14.0-1 N1MODE=workeronly make -C src/test/regress check-minimal # Run a full schedule with coordinator at N-1 CITUSLIBDIR=~/citus-libs/18/v14.0 CITUSVERSION=14.0-1 N1MODE=coordinatoronly make -C src/test/regress check-multi-1 # Run a single test using run_test.py CITUSLIBDIR=~/citus-libs/18/v14.0 CITUSVERSION=14.0-1 N1MODE=workeronly src/test/regress/citus_tests/run_test.py check_mx ``` # Regression test best practices * Instead of connecting to different nodes to check catalog tables, should use `run_command_on_all_nodes()` because it's faster than keep disconnecting / connecting to different nodes. * Tests should **define functions** for repetitive actions, e.g., by wrapping usual queries used to check catalog tables. If the function is presumed to be used by other tests in future, then the function needs to defined in `multi_test_helpers.sql`. * If you're adding a new file, consider using `src/test/regress/bin/create_test.py your_new_test_name` to create the file. Or if you want to manually create it, make sure that your test file creates a schema and that it drops the schema at the end of the test to make sure that it doesn't leak any objects behind. See which lines `src/test/regress/bin/create_test.py` adds to the test file to understand what you need to do. For the object that are not bound to a schema, make sure to drop them at the end of the test too, such as databases and roles. ================================================ FILE: src/test/regress/after_citus_upgrade_coord_schedule ================================================ # this schedule is to be run only on coordinators test: upgrade_citus_finish_citus_upgrade test: upgrade_pg_dist_cleanup_after test: upgrade_basic_after test: upgrade_basic_after_non_mixed test: upgrade_post_11_after test: upgrade_post_14_after ================================================ FILE: src/test/regress/after_pg_upgrade_with_columnar_schedule ================================================ test: upgrade_basic_after upgrade_ref2ref_after upgrade_type_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects upgrade_autoconverted_after upgrade_citus_stat_activity upgrade_citus_locks upgrade_single_shard_table_after upgrade_schema_based_sharding_after upgrade_basic_after_non_mixed # This test cannot be run with run_test.py currently due to its dependence on # the specific PG versions that we use to run upgrade tests. For now we leave # it out of the parallel line, so that flaky test detection can at least work # for the other tests. test: upgrade_distributed_triggers_after # This attempts dropping citus extension (and rollbacks), so please do # not run in parallel with any other tests. test: upgrade_columnar_after ================================================ FILE: src/test/regress/after_pg_upgrade_without_columnar_schedule ================================================ test: upgrade_basic_after upgrade_ref2ref_after upgrade_type_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects upgrade_autoconverted_after upgrade_citus_stat_activity upgrade_citus_locks upgrade_single_shard_table_after upgrade_schema_based_sharding_after upgrade_basic_after_non_mixed # This test cannot be run with run_test.py currently due to its dependence on # the specific PG versions that we use to run upgrade tests. For now we leave # it out of the parallel line, so that flaky test detection can at least work # for the other tests. test: upgrade_distributed_triggers_after # The last test to ensure citus columnar not automatically created and upgrade # went fine without automatically creating it. test: ensure_citus_columnar_not_exists ================================================ FILE: src/test/regress/base_isolation_schedule ================================================ # ---------- # isolation setup steps # ---------- test: isolation_setup test: isolation_cluster_management ================================================ FILE: src/test/regress/base_schedule ================================================ # ---------- # Only run few basic tests to set up a testing environment # ---------- test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw failure_test_helpers test: multi_cluster_management test: multi_test_catalog_views test: multi_create_table test: multi_behavioral_analytics_create_table test: multi_create_table_superuser multi_behavioral_analytics_create_table_superuser test: multi_load_data multi_load_data_superuser tablespace test: check_mx ================================================ FILE: src/test/regress/before_citus_upgrade_coord_schedule ================================================ # this schedule is to be run on only coordinators test: upgrade_basic_before upgrade_basic_before_non_mixed test: upgrade_pg_dist_cleanup_before test: upgrade_post_11_before test: upgrade_post_14_before ================================================ FILE: src/test/regress/before_pg_upgrade_with_columnar_schedule ================================================ # The basic tests runs analyze which depends on shard numbers test: multi_test_helpers multi_test_helpers_superuser upgrade_basic_before_non_mixed test: multi_test_catalog_views test: upgrade_basic_before test: upgrade_ref2ref_before test: upgrade_type_before test: upgrade_distributed_function_before upgrade_rebalance_strategy_before test: upgrade_autoconverted_before upgrade_single_shard_table_before upgrade_schema_based_sharding_before test: upgrade_citus_stat_activity test: upgrade_citus_locks test: upgrade_distributed_triggers_before # upgrade_columnar_before renames public schema to citus_schema, so let's # run this test as the last one. test: upgrade_columnar_before ================================================ FILE: src/test/regress/before_pg_upgrade_without_columnar_schedule ================================================ # The basic tests runs analyze which depends on shard numbers test: multi_test_helpers multi_test_helpers_superuser upgrade_basic_before_non_mixed test: multi_test_catalog_views test: upgrade_basic_before test: upgrade_ref2ref_before test: upgrade_type_before test: upgrade_distributed_function_before upgrade_rebalance_strategy_before test: upgrade_autoconverted_before upgrade_single_shard_table_before upgrade_schema_based_sharding_before test: upgrade_citus_stat_activity test: upgrade_citus_locks test: upgrade_distributed_triggers_before # The last test, i.e., upgrade_columnar_before, in before_pg_upgrade_with_columnar_schedule # renames public schema to citus_schema and re-creates public schema, so we also do the same # here to have compatible output in after schedule tests for both schedules. test: rename_public_to_citus_schema_and_recreate ================================================ FILE: src/test/regress/bin/copy_modified ================================================ #!/bin/bash set -euo pipefail # This is to make the following command work to update the test output: # cp src/test/regress/{results,expected}/multi_cluster_management.out # # This can not be done in the custom "diff" command, because that is executed # multiple times for modified_file in {results,expected}/*.modified; do original=${modified_file%.modified} mv "$original" "$original.unmodified" mv "$modified_file" "$original" done ================================================ FILE: src/test/regress/bin/copy_modified_wrapper ================================================ #!/bin/bash set -euo pipefail ## Set mydir to the directory containing the script ## The ${var%pattern} format will remove the shortest match of ## pattern from the end of the string. Here, it will remove the ## script's name,. leaving only the directory. datadir="${0%/*}" cd "${datadir}" # shellcheck source=copy_modified source copy_modified ================================================ FILE: src/test/regress/bin/create_test.py ================================================ #!/usr/bin/env python3 import os import random import sys if __name__ == "__main__": if len(sys.argv) != 2: print( "ERROR: Expected the name of the new test as an argument, such as:\n" "src/test/regress/bin/create_test.py my_awesome_test" ) sys.exit(1) test_name = sys.argv[1] regress_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) filename = os.path.join(regress_dir, "sql", f"{test_name}.sql") if os.path.isfile(filename): print(f"ERROR: test file '{filename}' already exists") sys.exit(1) shard_id = random.randint(1, 999999) * 100 contents = f"""CREATE SCHEMA {test_name}; SET search_path TO {test_name}; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; SET citus.next_shard_id TO {shard_id}; -- add tests here SET client_min_messages TO WARNING; DROP SCHEMA {test_name} CASCADE; """ with open(filename, "w") as f: f.write(contents) print(f"Created {filename}") print(f"Don't forget to add '{test_name}' in multi_schedule somewhere") ================================================ FILE: src/test/regress/bin/diff ================================================ #!/usr/bin/env bash # Our custom diff tool which normalizes result and expected files before # doing the actual diff. Rules for normalization are in normalize.sed. See # the comments there for more information. # # Note that src/test/regress/Makefile adds this directory to $PATH so # pg_regress uses this diff tool instead of the system diff tool. set -eu -o pipefail file1="${@:(-2):1}" file2="${@:(-1):1}" # pg_regress passes the expected file as the first argument ($file1 above), # and results file as the second argument ($file2 above). We take the base # filename of the expected file as the test name. We can have multiple # expected files for a single test with _0, _1, ... suffixes. # So for the test name, we also strip the additional suffix. test=$(basename "$file1" .out | sed -E "s/_[0-9]+$//") args=${@:1:$#-2} BASEDIR=$(dirname "$0") DIFF=/usr/bin/diff if [ ! -f "$DIFF" ] then # whereis searches for standard unix places before $PATH. So select the # first entry as the original diff tool. # With the default WSL2 configuration whereis is very slow though ~400ms, # so we only use it if /usr/bin/diff does not exist. DIFF=$(whereis diff | sed "s/diff://g" | awk '{print $1}') if [ -z "$DIFF" ] then echo "ERROR: could not find diff command" 1>&2 exit 1 fi fi if test -z "${VANILLATEST:-}" then touch "$file1" # when adding a new test the expected file does not exist normalize_file="$BASEDIR/normalize.sed" # when running tests on an existing cluster some changes need to be done on # normalize.sed file. So a new file is used. if [[ -f "$BASEDIR/normalize_modified.sed" ]] then normalize_file="$BASEDIR/normalize_modified.sed" fi sed -Ef "$normalize_file" < "$file1" > "$file1.modified" sed -Ef "$normalize_file" < "$file2" > "$file2.modified" "$DIFF" -w $args "$file1.modified" "$file2.modified" | LC_CTYPE=C.UTF-8 diff-filter "$BASEDIR/normalize.sed" exit ${PIPESTATUS[0]} else exec "$DIFF" -w $args "$file1" "$file2" fi ================================================ FILE: src/test/regress/bin/diff-filter ================================================ #!/usr/bin/env python3 """ diff-filter denormalizes diff output by having lines beginning with ' ' or '+' come from file2's unmodified version. """ import re from sys import argv, stdin, stdout class FileScanner: """ FileScanner is an iterator over the lines of a file. It can apply a rewrite rule which can be used to skip lines. """ def __init__(self, file, rewrite=lambda x: x): self.file = file self.line = 1 self.rewrite = rewrite def __next__(self): while True: nextline = self.rewrite(next(self.file)) if nextline is not None: self.line += 1 return nextline def main(): # we only test //d rules, as we need to ignore those lines regexregex = re.compile(r"^/(?P.*)/d$") regexpipeline = [] for line in open(argv[1]): line = line.strip() if not line or line.startswith("#") or not line.endswith("d"): continue rule = regexregex.match(line) if not rule: raise "Failed to parse regex rule: %s" % line regexpipeline.append(re.compile(rule.group("rule"))) def sed(line): if any(regex.search(line) for regex in regexpipeline): return None return line for line in stdin: if line.startswith("+++ "): tab = line.rindex("\t") fname = line[4:tab] file2 = FileScanner( open(fname.replace(".modified", ""), encoding="utf8"), sed ) stdout.write(line) elif line.startswith("@@ "): idx_start = line.index("+") + 1 idx_end = idx_start + 1 while line[idx_end].isdigit(): idx_end += 1 linenum = int(line[idx_start:idx_end]) while file2.line < linenum: next(file2) stdout.write(line) elif line.startswith(" "): stdout.write(" ") stdout.write(next(file2)) elif line.startswith("+"): stdout.write("+") stdout.write(next(file2)) else: stdout.write(line) main() ================================================ FILE: src/test/regress/bin/normalize.sed ================================================ # Rules to normalize test outputs. Our custom diff tool passes test output # of tests through the substitution rules in this file before doing the # actual comparison. # # An example of when this is useful is when an error happens on a different # port number, or a different worker shard, or a different placement, etc. # because we are running the tests in a different configuration. # In all tests, normalize worker ports, placement ids, and shard ids s/localhost:[0-9]+/localhost:xxxxx/g s/ port=[0-9]+ / port=xxxxx /g s/placement [0-9]+/placement xxxxx/g s/shard [0-9]+/shard xxxxx/g s/Shard [0-9]+/Shard xxxxx/g s/assigned task [0-9]+ to node/assigned task to node/ s/node group [12] (but|does)/node group \1/ # Differing names can have differing table column widths s/^-[+-]{2,}$/---------------------------------------------------------------------/g # In foreign_key_to_reference_table, normalize shard table names, etc in # the generated plan s/"(foreign_key_2_|fkey_ref_to_dist_|fkey_ref_|fkey_to_ref_)[0-9]+"/"\1xxxxxxx"/g s/"(referenced_table_|referencing_table_|referencing_table2_)[0-9]+"/"\1xxxxxxx"/g s/"(referencing_table_0_|referencing_table_4_|referenced_table2_)[0-9]+"/"\1xxxxxxx"/g s/\(id\)=\([0-9]+\)/(id)=(X)/g s/\(ref_id\)=\([0-9]+\)/(ref_id)=(X)/g # shard table names for multi_subtransactions s/"t2_[0-9]+"/"t2_xxxxxxx"/g # shard table names for MERGE tests s/merge_schema\.([_a-z0-9]+)_40[0-9]+ /merge_schema.\1_xxxxxxx /g s/pgmerge_schema\.([_a-z0-9]+)_40[0-9]+ /pgmerge_schema.\1_xxxxxxx /g s/merge_vcore_schema\.([_a-z0-9]+)_40[0-9]+ /pgmerge_schema.\1_xxxxxxx /g # shard table names for multi_subquery s/ keyval(1|2|ref)_[0-9]+ / keyval\1_xxxxxxx /g # shard table names for custom_aggregate_support s/ daily_uniques_[0-9]+ / daily_uniques_xxxxxxx /g # shard table names for isolation_create_citus_local_table s/"citus_local_table_([0-9]+)_[0-9]+"/"citus_local_table_\1_xxxxxxx"/g # normalize relation oid suffix for the truncate triggers created by citus s/truncate_trigger_[0-9]+/truncate_trigger_xxxxxxx/g # shard move subscription and publication names contain the oid of the # table owner, which can change across runs s/(citus_shard_(move|split)_subscription_role_)[0-9]+_[0-9]+/\1xxxxxxx_xxxxxxx/g s/(citus_shard_(move|split)_subscription_)[0-9]+_[0-9]+/\1xxxxxxx_xxxxxxx/g s/(citus_shard_(move|split)_(slot|publication)_)[0-9]+_[0-9]+_[0-9]+/\1xxxxxxx_xxxxxxx_xxxxxxx/g # In foreign_key_restriction_enforcement, normalize shard names s/"(on_update_fkey_table_|fkey_)[0-9]+"/"\1xxxxxxx"/g # In multi_insert_select_conflict, normalize shard name and constraints s/"(target_table_|target_table_|test_ref_table_)[0-9]+"/"\1xxxxxxx"/g s/\(col_1\)=\([0-9]+\)/(col_1)=(X)/g # In multi_name_lengths, normalize shard names s/name_len_12345678901234567890123456789012345678_fcd8ab6f_[0-9]+/name_len_12345678901234567890123456789012345678_fcd8ab6f_xxxxx/g # ignore page split with pg13, and WAL warnings # they can randomly appear when DEBUG logs are on /DEBUG: concurrent ROOT page split/d /DEBUG: .+creating and filling new WAL file/d # normalize debug connection failure s/DEBUG: connection to the remote node/WARNING: connection to the remote node/g # normalize file names for partitioned files s/(task_[0-9]+\.)[0-9]+/\1xxxx/g s/(job_[0-9]+\/task_[0-9]+\/p_[0-9]+\.)[0-9]+/\1xxxx/g # isolation_ref2ref_foreign_keys s/"(ref_table_[0-9]_|ref_table_[0-9]_value_fkey_)[0-9]+"/"\1xxxxxxx"/g # commands cascading to shard relations s/(NOTICE: .*_)[0-9]{5,}( CASCADE)/\1xxxxx\2/g s/(NOTICE: [a-z]+ cascades to table ".*)_[0-9]{5,}"/\1_xxxxx"/g # Line info varies between versions /^LINE [0-9]+:.*$/d /^ *\^$/d # connection id s/connectionId: [0-9]+/connectionId: xxxxxxx/g # Remove trailing whitespace s/ *$//g # pg13 changes s/of relation ".*" violates not-null constraint/violates not-null constraint/g /DEBUG: index ".*" can safely use deduplication.*$/d /DEBUG: index ".*" cannot use deduplication.*$/d /DEBUG: building index ".*" on table ".*" serially.*$/d s/partition ".*" would be violated by some row/partition would be violated by some row/g s/of relation ".*" contains null values/contains null values/g s/(Citus Background Task Queue Executor: regression\/postgres for \()[0-9]+\/[0-9]+\)/\1xxxxx\/xxxxx\)/g # Changed outputs after minor bump to PG14.5 and PG13.8 s/(ERROR: |WARNING: |error:) invalid socket/\1 connection not open/g # Extra outputs after minor bump to PG14.5 and PG13.8 /^\s*invalid socket$/d # pg15 changes s/ AS "\?column\?"//g # We ignore multiline error messages, and substitute first line with a single line # alternative that is used in some older libpq versions. s/(ERROR: |WARNING: |error:) server closed the connection unexpectedly/\1 connection not open/g /^\s*This probably means the server terminated abnormally$/d /^\s*before or while processing the request.$/d /^\s*connection not open$/d # intermediate_results s/(ERROR.*)pgsql_job_cache\/([0-9]+_[0-9]+_[0-9]+)\/(.*).data/\1pgsql_job_cache\/xx_x_xxx\/\3.data/g # assign_distributed_transaction id params s/(NOTICE.*)assign_distributed_transaction_id\([0-9]+, [0-9]+, '.*'\)/\1assign_distributed_transaction_id\(xx, xx, 'xxxxxxx'\)/g s/(NOTICE.*)PREPARE TRANSACTION 'citus_[0-9]+_[0-9]+_[0-9]+_[0-9]+'/\1PREPARE TRANSACTION 'citus_xx_xx_xx_xx'/g s/(NOTICE.*)COMMIT PREPARED 'citus_[0-9]+_[0-9]+_[0-9]+_[0-9]+'/\1COMMIT PREPARED 'citus_xx_xx_xx_xx'/g # toast tables s/pg_toast_[0-9]+/pg_toast_xxxxx/g # Plan numbers are not very stable, so we normalize those # subplan numbers are quite stable so we keep those s/DEBUG: Plan [0-9]+/DEBUG: Plan XXX/g s/generating subplan [0-9]+\_/generating subplan XXX\_/g s/read_intermediate_result\('[0-9]+_/read_intermediate_result('XXX_/g s/Subplan [0-9]+\_/Subplan XXX\_/g # Plan numbers in insert select s/read_intermediate_result\('insert_select_[0-9]+_/read_intermediate_result('insert_select_XXX_/g # Plan numbers in merge into s/read_intermediate_result\('merge_into_[0-9]+_/read_intermediate_result('merge_into_XXX_/g # ignore job id in repartitioned insert/select s/repartitioned_results_[0-9]+/repartitioned_results_xxxxx/g # ignore job id in worker_hash_partition_table s/worker_hash_partition_table \([0-9]+/worker_hash_partition_table \(xxxxxxx/g # ignore referene table replication messages /replicating reference table.*$/d # ignore memory usage output /.*Memory Usage:.*/d # Warnings in multi_explain s/prepared transaction with identifier .* does not exist/prepared transaction with identifier "citus_x_yyyyyy_zzz_w" does not exist/g s/failed to roll back prepared transaction '.*'/failed to roll back prepared transaction 'citus_x_yyyyyy_zzz_w'/g # Errors with binary decoding where OIDs should be normalized s/wrong data type: [0-9]+, expected [0-9]+/wrong data type: XXXX, expected XXXX/g # Errors with relation OID does not exist s/relation with OID [0-9]+ does not exist/relation with OID XXXX does not exist/g # ignore event triggers, mainly due to the event trigger for columnar /^DEBUG: EventTriggerInvoke [0-9]+$/d # ignore DEBUG1 messages that Postgres generates /^DEBUG: rehashing catalog cache id .*$/d # ignore JIT related messages /^DEBUG: probing availability of JIT.*/d /^DEBUG: provider not available, disabling JIT for current session.*/d /^DEBUG: time to inline:.*/d /^DEBUG: successfully loaded JIT.*/d # ignore timing statistics for VACUUM VERBOSE /CPU: user: .*s, system: .*s, elapsed: .*s/d # normalize storage id of columnar tables s/^storage id: [0-9]+$/storage id: xxxxx/g # normalize notice messages in citus_local_tables s/(NOTICE: executing.*)citus_local_tables_test_schema.citus_local_table_4_[0-9]+(.*)/\1citus_local_tables_test_schema.citus_local_table_4_xxxx\2/g s/(NOTICE: executing.*)\([0-9]+, 'citus_local_tables_test_schema', [0-9]+(.*)/\1\(xxxxx, 'citus_local_tables_test_schema', xxxxx\2/g s/citus_local_table_4_idx_[0-9]+/citus_local_table_4_idx_xxxxxx/g s/citus_local_table_4_[0-9]+/citus_local_table_4_xxxxxx/g s/ERROR: cannot append to shardId [0-9]+/ERROR: cannot append to shardId xxxxxx/g # hide notice/hint message that we get when converting local tables automatically /local tables that are added to metadata automatically by citus, but not chained with reference tables via foreign keys might be automatically converted back to postgres tables$/d /Executing citus_add_local_table_to_metadata(.*) prevents this for the given relation, and all of the connected relations$/d # normalize for distributed deadlock delay in isolation_metadata_sync_deadlock # isolation tester first detects a lock, but then deadlock detector cancels the # session. Sometimes happens that deadlock detector cancels the session before # lock detection, so we normalize it by removing these two lines. /^ $/ { N; /\nstep s1-update-2: <... completed>$/ { s/.*//g } } # normalize long table shard name errors for alter_table_set_access_method and alter_distributed_table s/^(ERROR: child table is missing constraint "\w+)_([0-9])+"/\1_xxxxxx"/g s/^(DEBUG: the name of the shard \(abcde_01234567890123456789012345678901234567890_f7ff6612)_([0-9])+/\1_xxxxxx/g s/^(ERROR: cannot distribute relation: numeric_negative_scale)_([0-9]+)/\1_xxxxxx"/g # normalize long index name errors for multi_index_statements s/^(ERROR: The index name \(test_index_creation1_p2020_09_26)_([0-9])+_(tenant_id_timeperiod_idx)/\1_xxxxxx_\3/g s/^(DEBUG: the index name on the shards of the partition is too long, switching to sequential and local execution mode to prevent self deadlocks: test_index_creation1_p2020_09_26)_([0-9])+_(tenant_id_timeperiod_idx)/\1_xxxxxx_\3/g # normalize errors for not being able to connect to a non-existing host s/could not translate host name "([A-Za-z0-9\.\-]+)" to address: .*$/could not translate host name "\1" to address: /g # ignore PL/pgSQL line numbers that differ on Mac builds s/(CONTEXT: PL\/pgSQL function .* line )([0-9]+)/\1XX/g s/^(PL\/pgSQL function .* line) [0-9]+ (.*)/\1 XX \2/g # normalize a test difference in multi_move_mx s/ connection to server at "\w+" (\(127\.0\.0\.1\)|\(::1\)), port [0-9]+ failed://g # normalize differences in tablespace of new index s/pg14\.idx.*/pg14\.xxxxx/g s/CREATE TABLESPACE test_tablespace LOCATION.*/CREATE TABLESPACE test_tablespace LOCATION XXXX/g # columnar log for var correlation s/(.*absolute correlation \()([0,1]\.[0-9]+)(\) of var attribute [0-9]+ is smaller than.*)/\1X\.YZ\3/g # normalize differences in multi_fix_partition_shard_index_names test s/NOTICE: issuing WITH placement_data\(shardid, shardlength, groupid, placementid\) AS \(VALUES \([0-9]+, [0-9]+, [0-9]+, [0-9]+\)\)/NOTICE: issuing WITH placement_data\(shardid, shardlength, groupid, placementid\) AS \(VALUES \(xxxxxx, xxxxxx, xxxxxx, xxxxxx\)\)/g # global_pid when pg_cancel_backend is sent to workers s/pg_cancel_backend\('[0-9]+'::bigint\)/pg_cancel_backend('xxxxx'::bigint)/g s/issuing SELECT pg_cancel_backend\([0-9]+::integer\)/issuing SELECT pg_cancel_backend(xxxxx::integer)/g # shard_rebalancer output for flaky nodeIds s/issuing SELECT pg_catalog.citus_copy_shard_placement\(43[0-9]+,[0-9]+,[0-9]+,'block_writes'\)/issuing SELECT pg_catalog.citus_copy_shard_placement(43xxxx,xx,xx,'block_writes')/g # node id in run_command_on_all_nodes warning s/Error on node with node id [0-9]+/Error on node with node id xxxxx/g # Temp schema names in error messages regarding dependencies that we cannot distribute # # 1) Schema of the depending object in the error message: # # e.g.: # WARNING: "function pg_temp_3.f(bigint)" has dependency on unsupported object "" # will be replaced with # WARNING: "function pg_temp_xxx.f(bigint)" has dependency on unsupported object "" s/^(WARNING|ERROR)(: "[a-z\ ]+ )pg_temp_[0-9]+(\..*" has dependency on unsupported object ".*")$/\1\2pg_temp_xxx\3/g # 2) Schema of the depending object in the error detail: s/^(DETAIL: "[a-z\ ]+ )pg_temp_[0-9]+(\..*" will be created only locally)$/\1pg_temp_xxx\2/g # 3) Schema that the object depends in the error message: # e.g.: # WARNING: "function func(bigint)" has dependency on unsupported object "schema pg_temp_3" # will be replaced with # WARNING: "function func(bigint)" has dependency on unsupported object "schema pg_temp_xxx" s/^(WARNING|ERROR)(: "[a-z\ ]+ .*" has dependency on unsupported object) "schema pg_temp_[0-9]+"$/\1\2 "schema pg_temp_xxx"/g # remove jobId's from the messages of the background rebalancer s/^ERROR: A rebalance is already running as job [0-9]+$/ERROR: A rebalance is already running as job xxx/g s/^NOTICE: Scheduled ([0-9]+) moves as job [0-9]+$/NOTICE: Scheduled \1 moves as job xxx/g s/^HINT: (.*) job_id = [0-9]+ (.*)$/HINT: \1 job_id = xxx \2/g # In clock tests, normalize epoch value(s) and the DEBUG messages printed s/^(DEBUG: |LOG: )(coordinator|final global|Set) transaction clock [0-9]+.*$/\1\2 transaction clock xxxxxx/g # Look for >= 13 digit logical value s/^ (\(localhost,)([0-9]+)(,t,"\([1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]+,[0-9]+\)")/\1 xxx,t,"(xxxxxxxxxxxxx,x)"/g s/^ \([1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]+,[0-9]+\)/ (xxxxxxxxxxxxx,x)/g s/^(DEBUG: |LOG: )(node\([0-9]+\)) transaction clock [0-9]+.*$/\1node xxxx transaction clock xxxxxx/g s/^(NOTICE: )(clock).*LC:[0-9]+,.*C:[0-9]+,.*$/\1\2 xxxxxx/g /^(DEBUG: )(adjusted to remote clock: $/d /^DEBUG: persisting transaction.*counter.*$/d /^DEBUG: both logical clock values are equal\([0-9]+\), pick remote.*$/d # The following 2 lines are to normalize duration and cost in the EXPLAIN output s/LOG: duration: [0-9].[0-9]+ ms/LOG: duration: xxxx ms/g s/"Total Cost": [0-9].[0-9]+/"Total Cost": xxxx/g # normalize gpids s/(NOTICE: issuing SET LOCAL application_name TO 'citus_rebalancer gpid=)[0-9]+/\1xxxxx/g # shard_rebalancer output, flaky improvement number s/improvement of 0.1[0-9]* is lower/improvement of 0.1xxxxx is lower/g # normalize tenants statistics annotations s/\/\*\{"cId":.*\*\///g # Notice message that contains current columnar version that makes it harder to bump versions s/(NOTICE: issuing CREATE EXTENSION IF NOT EXISTS citus_columnar WITH SCHEMA pg_catalog VERSION )"[0-9]+\.[0-9]+-[0-9]+"/\1 "x.y-z"/ # pg17 changes # can be removed when dropping PG16 support #if PG_VERSION_NUM < PG_VERSION_17 # (This is not preprocessor directive, but a reminder for the developer that will drop PG16 support ) s/COPY DEFAULT only available using COPY FROM/COPY DEFAULT cannot be used with COPY TO/ s/COPY delimiter must not appear in the DEFAULT specification/COPY delimiter character must not appear in the DEFAULT specification/ #endif /* PG_VERSION_NUM < PG_VERSION_17 */ # PG 17 Removes outer parentheses from CHECK constraints # we add them back for pg16 compatibility # e.g. change CHECK other_col >= 100 to CHECK (other_col >= 100) s/\| CHECK ([a-zA-Z])(.*)/| CHECK \(\1\2\)/g # pg17 change: this is a rule that ignores additional DEBUG logging # for CREATE MATERIALIZED VIEW (commit b4da732fd64). This could be # changed to a normalization rule when 17 becomes the minimum # supported Postgres version. /DEBUG: drop auto-cascades to type [a-zA-Z_]*.pg_temp_[0-9]*/d # --- PG18 Actual Rows normalization --- # New in PG18: Actual Rows in EXPLAIN output are now rounded to # 1) 0.50 (and 0.5, 0.5000...) -> 0 s/(actual[[:space:]]*rows[[:space:]]*[=:][[:space:]]*)0\.50*/\10/gI s/(actual[^)]*rows[[:space:]]*=[[:space:]]*)0\.50*/\10/gI # 2) 0.51+ -> 1 s/(actual[[:space:]]*rows[[:space:]]*[=:][[:space:]]*)0\.(5[1-9][0-9]*|[6-9][0-9]*)/\11/gI s/(actual[^)]*rows[[:space:]]*=[[:space:]]*)0\.(5[1-9][0-9]*|[6-9][0-9]*)/\11/gI # 3) Strip trivial trailing ".0..." (6.00 -> 6) [keep your existing cross-format rules] s/(actual[[:space:]]*rows[[:space:]]*[=:][[:space:]]*)([0-9]+)\.0+/\1\2/gI s/(actual[^)]*rows[[:space:]]*=[[:space:]]*)([0-9]+)\.0+/\1\2/gI # 4) YAML/XML/JSON: strip trailing ".0..." s/(Actual[[:space:]]+Rows:[[:space:]]*[0-9]+)\.0+/\1/gI s/([0-9]+)\.0+(<\/Actual-Rows>)/\1\2/g s/("Actual[[:space:]]+Rows":[[:space:]]*[0-9]+)\.0+/\1/gI # 5) Placeholder cleanups (kept from existing rules; harmless if unused) # JSON placeholder cleanup: '"Actual Rows": N.N' -> N s/("Actual[[:space:]]+Rows":[[:space:]]*)N\.N/\1N/gI # Text EXPLAIN collapse: "rows=N.N" -> "rows=N" s/(rows[[:space:]]*=[[:space:]]*)N\.N/\1N/gI # YAML placeholder: "Actual Rows: N.N" -> "Actual Rows: N" s/(Actual[[:space:]]+Rows:[[:space:]]*)N\.N/\1N/gI # --- PG18 Actual Rows normalization --- # pg18 “Disabled” change start # ignore any “Disabled:” lines in test output /^\s*Disabled:/d # ignore XML true or false /^\s*.*<\/Disabled>/d # pg18 “Disabled” change end # PG18 psql: headings changed from "List of relations" to per-type titles s/^([ \t]*)List of tables$/\1List of relations/g s/^([ \t]*)List of indexes$/\1List of relations/g s/^([ \t]*)List of sequences$/\1List of relations/g # --- PG18 FK wording -> legacy generic form --- # e.g., "violates RESTRICT setting of foreign key constraint" -> "violates foreign key constraint" s/violates RESTRICT setting of foreign key constraint/violates foreign key constraint/g # DETAIL line changed "is referenced" -> old "is still referenced" s/\/is still referenced from table/g # pg18 extension_control_path GUC debugs # ignore any "find_in_path:" lines in test output /DEBUG: find_in_path: trying .*/d # EXPLAIN (PG18+): hide Materialize storage instrumentation # this rule can be removed when PG18 is the minimum supported version /^[ \t]*Storage:[ \t].*$/d # PG18: drop 'subscription ""' prefix # this rule can be removed when PG18 is the minimum supported version s/^[[:space:]]*ERROR:[[:space:]]+subscription "[^"]+" could not connect to the publisher:[[:space:]]*/ERROR: could not connect to the publisher: /I # PG18: drop verbose 'connection to server … failed:' preamble s/^[[:space:]]*ERROR:[[:space:]]+could not connect to the publisher:[[:space:]]*connection to server .* failed:[[:space:]]*/ERROR: could not connect to the publisher: /I # PG18: replace named window refs like "OVER w1" with neutral "OVER (?)" # this rule can be removed when PG18 is the minimum supported version # only on Sort Key / Group Key / Output lines # Sort Key /^[[:space:]]*Sort Key:/ s/(OVER[[:space:]]+)w[0-9]+/\1(?)/g # Group Key /^[[:space:]]*Group Key:/ s/(OVER[[:space:]]+)w[0-9]+/\1(?)/g # Output /^[[:space:]]*Output:/ s/(OVER[[:space:]]+)w[0-9]+/\1(?)/g # end PG18 window ref normalization # pg18 varreturningtype - change needed for PG16, PG17 tests # can be removed when dropping pg17 support s/(:varnullingrels \(b\) :varlevelsup 0) (:varnosyn 1)/\1 :varreturningtype 0 \2/g # end pg18 varreturningtype ================================================ FILE: src/test/regress/bin/test/expected/different.out ================================================ --- file.out.modified +++ file_different.out.modified @@ -1,3 +1,2 @@ -This line is missing in file_different Ports are replaced with xxxxx: localhost:2187 This line is the same @@ -7,6 +6,8 @@ Filler 2, localhost:1111 Filler 3, localhost:111 -This line is missing in file_different +This line has been inserted +This line has also been inserted, localhost:10812 Line below will be removed, localhost:2781 -This line will be changed ✓ +This line has been changed ✇ End of file +New line at end ================================================ FILE: src/test/regress/bin/test/expected/same.out ================================================ ================================================ FILE: src/test/regress/bin/test/file.out ================================================ This line is missing in file_different Ports are replaced with xxxxx: localhost:1728 This line is the same This line is also the same DEBUG: we remove this line creating and filling new WAL file Line above was deleted é Filler 1, localhost:21870 Filler 2, localhost:2187 Filler 3, localhost:218 This line is missing in file_different Line below will be removed, localhost:2187 DEBUG: we remove this line creating and filling new WAL file This line will be changed ✓ End of file ================================================ FILE: src/test/regress/bin/test/file_different.out ================================================ Ports are replaced with xxxxx: localhost:2187 This line is the same This line is also the same DEBUG: we remove this line creating and filling new WAL file Line above was deleted é Filler 1, localhost:11111 Filler 2, localhost:1111 Filler 3, localhost:111 This line has been inserted This line has also been inserted, localhost:10812 DEBUG: we remove this line creating and filling new WAL file Line below will be removed, localhost:2781 This line has been changed ✇ End of file New line at end ================================================ FILE: src/test/regress/bin/test/file_same.out ================================================ This line is missing in file_different Ports are replaced with xxxxx: localhost:1728 This line is the same This line is also the same DEBUG: we remove this line creating and filling new WAL file Line above was deleted é Filler 1, localhost:21870 Filler 2, localhost:2187 Filler 3, localhost:218 This line is missing in file_different Line below will be removed, localhost:2187 DEBUG: we remove this line creating and filling new WAL file This line will be changed ✓ End of file ================================================ FILE: src/test/regress/bin/test_diff ================================================ #!/bin/bash SCRIPT=`realpath -s "$0"` SCRIPTPATH=`dirname "$SCRIPT"` PATH="$SCRIPTPATH:$PATH" cp "$SCRIPTPATH/test/file.out" "$SCRIPTPATH/test/file_same.out" mkdir -p "$SCRIPTPATH/test/results" # diff file.out against file_$1.out, also strip out timestamps & file paths function create_result() { diff -dU2 -w "$SCRIPTPATH/test/file.out" "$SCRIPTPATH/test/file_$1.out" \ | sed -E 's/^(\+\+\+|---).+\/([^/]+)\t.+$/\1 \2/' \ > "$SCRIPTPATH/test/results/$1.out" } # compare whether result is same as expected function check_result() { cmp -s "$SCRIPTPATH/test/expected/$1.out" "$SCRIPTPATH/test/results/$1.out" if test $? -ne 0 then diff -u "$SCRIPTPATH/test/expected/$1.out" "$SCRIPTPATH/test/results/$1.out" exit 1 fi } function test_case() { # call twice as tests invoke diff multiple times create_result "$1" create_result "$1" check_result "$1" } test_case "same" test_case "different" ================================================ FILE: src/test/regress/citus_tests/__init__.py ================================================ ================================================ FILE: src/test/regress/citus_tests/arbitrary_configs/README.md ================================================ # Arbitrary Configs ## Usage To run tests in parallel use: ```bash # will run 4 configs in parallel make check-arbitrary-configs parallel=4 ``` To run tests sequentially use: ```bash make check-arbitrary-configs parallel=1 ``` To run only some configs: ```bash # Config names should be comma separated make check-arbitrary-configs CONFIGS=CitusSingleNodeClusterConfig,CitusSmallSharedPoolSizeConfig ``` To run only some test files with some config: ```bash make check-arbitrary-base CONFIGS=CitusSingleNodeClusterConfig EXTRA_TESTS=dropped_columns_1 ``` To get a deterministic run, you can give the random's seed: ```bash make check-arbitrary-configs parallel=4 seed=12312 ``` The `seed` will be in the output of the run. ## General Info In our regular regression tests, we can see all the details about either planning or execution but this means we need to run the same query under different configs/cluster setups again and again, which is not really maintanable. When we don't care about the internals of how planning/execution is done but the correctness, especially with different configs this infrastructure can be used. With `check-arbitrary-configs` target, the following happens: - a bunch of configs are loaded, which are defined in `config.py`. These configs have different settings such as different shard count, different citus settings, postgres settings, worker amount, or different metadata. - For each config, a separate data directory is created for tests in `tmp_citus_test` with the config's name. - For each config, `create_schedule` is run on the coordinator to setup the necessary tables. - For each config, `sql_schedule` is run. `sql_schedule` is run on the coordinator if it is a non-mx cluster. And if it is mx, it is either run on the coordinator or a random worker. - Tests results are checked if they match with the expected. When tests results don't match, you can see the regression diffs in a config's datadir, such as `tmp_citus_tests/dataCitusSingleNodeClusterConfig`. We also have a PostgresConfig which runs all the test suite with Postgres. By default configs use regular user, but we have a config to run as a superuser as well. So the infrastructure tests: - Postgres vs Citus - Mx vs Non-Mx - Superuser vs regular user - Arbitrary Citus configs ## Adding a new test When you want to add a new test, you can add the create statements to `create_schedule` and add the sql queries to `sql_schedule`. If you are adding Citus UDFs that should be a NO-OP for Postgres, make sure to override the UDFs in `postgres.sql`. If the test needs to be skipped in some configs, you can do that by adding the test names in the `skip_tests` array for each config. The test files associated with the skipped test will be set to empty so the test will pass without the actual test being run. ## Adding a new config You can add your new config to `config.py`. Make sure to extend either `CitusDefaultClusterConfig` or `CitusMXBaseClusterConfig`. ## Debugging failures On the CI, upon a failure, all logfiles will be uploaded as artifacts, so you can check the artifacts tab. All the regressions will be shown as part of the job on CI. In your local, you can check the regression diffs in config's datadirs as in `tmp_citus_tests/dataCitusSingleNodeClusterConfig`. ================================================ FILE: src/test/regress/citus_tests/arbitrary_configs/__init__.py ================================================ ================================================ FILE: src/test/regress/citus_tests/arbitrary_configs/citus_arbitrary_configs.py ================================================ #!/usr/bin/env python3 """citus_arbitrary_configs Usage: citus_arbitrary_configs --bindir= --pgxsdir= --parallel= --configs= --seed= [--base] Options: --bindir= The PostgreSQL executable directory(ex: '~/.pgenv/pgsql-11.3/bin') --pgxsdir= Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3) --parallel= how many configs to run in parallel --configs= the config names to run --seed= random number seed --base whether to use the base sql schedule or not """ import concurrent.futures import multiprocessing import os import random import shutil import sys import time from docopt import docopt # https://stackoverflow.com/questions/14132789/relative-imports-for-the-billionth-time/14132912#14132912 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ignore E402 because these imports require addition to path import common # noqa: E402 import config as cfg # noqa: E402 testResults = {} parallel_thread_amount = 1 def _run_pg_regress_on_port(config, port, schedule_name, extra_tests=""): return common.run_pg_regress_without_exit( config.bindir, config.pg_srcdir, port, schedule_name, config.output_dir, config.input_dir, config.user, extra_tests, ) def run_for_config(config, lock, sql_schedule_name): name = config.name print("Running test for: {}".format(name)) start_time = time.time() common.initialize_citus_cluster( config.bindir, config.datadir, config.settings, config ) if config.user == cfg.REGULAR_USER_NAME: common.create_role( config.bindir, config.node_name_to_ports.values(), config.user, ) copy_copy_modified_binary(config.datadir) copy_test_files(config) exitCode = 0 if not config.is_citus: exitCode |= common.run_pg_regress_without_exit( config.bindir, config.pg_srcdir, config.coordinator_port(), cfg.POSTGRES_SCHEDULE, config.output_dir, config.input_dir, cfg.SUPER_USER_NAME, ) common.save_regression_diff("postgres", config.output_dir) elif config.all_null_dist_key: exitCode |= common.run_pg_regress_without_exit( config.bindir, config.pg_srcdir, config.coordinator_port(), cfg.SINGLE_SHARD_PREP_SCHEDULE, config.output_dir, config.input_dir, cfg.SUPER_USER_NAME, ) common.save_regression_diff( "single_shard_table_prep_regression", config.output_dir ) exitCode |= _run_pg_regress_on_port( config, config.coordinator_port(), cfg.CREATE_SCHEDULE ) common.save_regression_diff("create", config.output_dir) extra_tests = os.getenv("EXTRA_TESTS", "") if config.is_mx and config.worker_amount > 0: exitCode |= _run_pg_regress_on_port( config, config.random_port(), sql_schedule_name, extra_tests=extra_tests ) else: exitCode |= _run_pg_regress_on_port( config, config.coordinator_port(), sql_schedule_name, extra_tests=extra_tests, ) run_time = time.time() - start_time with lock: testResults[name] = ( "SUCCESS" if exitCode == 0 else "FAIL: see {}".format(config.output_dir + "/run.out") ) testResults[name] += " runtime: {} seconds".format(run_time) common.stop_databases( config.bindir, config.datadir, config.node_name_to_ports, config.name ) common.save_regression_diff("sql", config.output_dir) return exitCode def copy_copy_modified_binary(datadir): shutil.copy("bin/copy_modified", datadir) shutil.copy("bin/copy_modified_wrapper", datadir) def copy_test_files(config): sql_dir_path = os.path.join(config.datadir, "sql") expected_dir_path = os.path.join(config.datadir, "expected") common.initialize_temp_dir(sql_dir_path) common.initialize_temp_dir(expected_dir_path) for scheduleName in cfg.ARBITRARY_SCHEDULE_NAMES: with open(scheduleName) as file: lines = file.readlines() for line in lines: colon_index = line.find(":") # skip empty lines if colon_index == -1: continue line = line[colon_index + 1 :].strip() test_names = line.split(" ") copy_test_files_with_names( test_names, sql_dir_path, expected_dir_path, config ) def copy_test_files_with_names(test_names, sql_dir_path, expected_dir_path, config): for test_name in test_names: # make empty files for the skipped tests if test_name in config.skip_tests: expected_sql_file = os.path.join(sql_dir_path, test_name + ".sql") open(expected_sql_file, "x").close() expected_out_file = os.path.join(expected_dir_path, test_name + ".out") open(expected_out_file, "x").close() continue sql_name = os.path.join("./sql", test_name + ".sql") shutil.copy(sql_name, sql_dir_path) # for a test named , all files: # .out, _0.out, _1.out ... # are considered as valid outputs for the test # by the testing tool (pg_regress) # so copy such files to the testing directory output_name = os.path.join("./expected", test_name + ".out") alt_output_version_no = 0 while os.path.isfile(output_name): # it might be the first time we run this test and the expected file # might not be there yet, in that case, we don't want to error out # while copying the file. shutil.copy(output_name, expected_dir_path) output_name = os.path.join( "./expected", f"{test_name}_{alt_output_version_no}.out" ) alt_output_version_no += 1 def run_tests(configs, sql_schedule_name): failCount = 0 common.initialize_temp_dir(cfg.CITUS_ARBITRARY_TEST_DIR) with concurrent.futures.ThreadPoolExecutor( max_workers=parallel_thread_amount ) as executor: manager = multiprocessing.Manager() lock = manager.Lock() futures = [ executor.submit(run_for_config, config, lock, sql_schedule_name) for config in configs ] try: for future in futures: exitCode = future.result() if exitCode != 0: failCount += 1 except KeyboardInterrupt: exit(1) return failCount def read_configs(docoptRes): configs = [] # We fill the configs from all of the possible classes in config.py so that if we add a new config, # we don't need to add it here. And this avoids the problem where we forget to add it here for x in cfg.__dict__.values(): if cfg.should_include_config(x): configs.append(x(docoptRes)) return configs def read_arguments(docoptRes): global parallel_thread_amount if "--parallel" in docoptRes and docoptRes["--parallel"] != "": parallel_thread_amount = int(docoptRes["--parallel"]) seed = random.randint(1, 1000000) if "--seed" in docoptRes and docoptRes["--seed"] != "": seed = int(docoptRes["--seed"]) random.seed(seed) configs = read_configs(docoptRes) if "--configs" in docoptRes and docoptRes["--configs"] != "": given_configs = docoptRes["--configs"].split(",") new_configs = [] for config in configs: if config.name in given_configs: new_configs.append(config) if len(new_configs) > 0: configs = new_configs sql_schedule_name = cfg.SQL_SCHEDULE if "--base" in docoptRes and docoptRes["--base"]: sql_schedule_name = cfg.SQL_BASE_SCHEDULE return configs, sql_schedule_name, seed def show_results(configs, testResults, runtime, seed): for testName, testResult in testResults.items(): print("{}: {}".format(testName, testResult)) print("--- {} seconds to run all tests! ---".format(end_time - start_time)) print("---SEED: {} ---".format(seed)) configCount = len(configs) if len(testResults) != configCount or failCount > 0: print( "actual {} expected {}, failCount: {}".format( len(testResults), configCount, failCount ) ) sys.exit(1) if __name__ == "__main__": docoptRes = docopt(__doc__) start_time = time.time() configs, sql_schedule_name, seed = read_arguments(docoptRes) failCount = run_tests(configs, sql_schedule_name) end_time = time.time() show_results(configs, testResults, end_time - start_time, seed) ================================================ FILE: src/test/regress/citus_tests/common.py ================================================ import asyncio import atexit import concurrent.futures import os import pathlib import platform import random import re import shutil import socket import subprocess import sys import time import typing from abc import ABC, abstractmethod from contextlib import asynccontextmanager, closing, contextmanager from datetime import datetime, timedelta from pathlib import Path from tempfile import gettempdir import filelock import psycopg import psycopg.sql import utils from psycopg import sql from utils import USER # This SQL returns true ( 't' ) if the Citus version >= 11.0. IS_CITUS_VERSION_11_SQL = "SELECT (split_part(extversion, '.', 1)::int >= 11) as is_11 FROM pg_extension WHERE extname = 'citus';" LINUX = False MACOS = False FREEBSD = False OPENBSD = False if platform.system() == "Linux": LINUX = True elif platform.system() == "Darwin": MACOS = True elif platform.system() == "FreeBSD": FREEBSD = True elif platform.system() == "OpenBSD": OPENBSD = True BSD = MACOS or FREEBSD or OPENBSD TIMEOUT_DEFAULT = timedelta(seconds=int(os.getenv("PG_TEST_TIMEOUT_DEFAULT", "10"))) FORCE_PORTS = os.getenv("PG_FORCE_PORTS", "NO").lower() not in ("no", "0", "n", "") REGRESS_DIR = pathlib.Path(os.path.realpath(__file__)).parent.parent REPO_ROOT = REGRESS_DIR.parent.parent.parent CI = os.environ.get("CI") == "true" def eprint(*args, **kwargs): """eprint prints to stderr""" print(*args, file=sys.stderr, **kwargs) def run(command, *args, check=True, shell=True, silent=False, **kwargs): """run runs the given command and prints it to stderr""" if not silent: eprint(f"+ {command} ") if silent: kwargs.setdefault("stdout", subprocess.DEVNULL) return subprocess.run(command, *args, check=check, shell=shell, **kwargs) def capture(command, *args, **kwargs): """runs the given command and returns its output as a string""" return run(command, *args, stdout=subprocess.PIPE, text=True, **kwargs).stdout PG_CONFIG = os.environ.get("PG_CONFIG", "pg_config") PG_CONFIG_ARGS = capture([PG_CONFIG, "--configure"], shell=False).rstrip() PG_SUPPORTS_SSL = "--with-ssl" in PG_CONFIG_ARGS or "--with-openssl" in PG_CONFIG_ARGS PG_BINDIR = capture([PG_CONFIG, "--bindir"], shell=False).rstrip() os.environ["PATH"] = PG_BINDIR + os.pathsep + os.environ["PATH"] def get_pg_major_version(): full_version_string = run( "initdb --version", stdout=subprocess.PIPE, encoding="utf-8", silent=True ).stdout major_version_string = re.search("[0-9]+", full_version_string) assert major_version_string is not None return int(major_version_string.group(0)) PG_MAJOR_VERSION = get_pg_major_version() OLDEST_SUPPORTED_CITUS_VERSION_MATRIX = { 14: "10.2.0", 15: "11.1.5", 16: "12.1.5", 17: "13.0.1", 18: "15.0devel", } OLDEST_SUPPORTED_CITUS_VERSION = OLDEST_SUPPORTED_CITUS_VERSION_MATRIX[PG_MAJOR_VERSION] def initialize_temp_dir(temp_dir): if os.path.exists(temp_dir): shutil.rmtree(temp_dir) os.mkdir(temp_dir) # Give full access to TEMP_DIR so that postgres user can use it. os.chmod(temp_dir, 0o777) def initialize_temp_dir_if_not_exists(temp_dir): if os.path.exists(temp_dir): return os.mkdir(temp_dir) # Give full access to TEMP_DIR so that postgres user can use it. os.chmod(temp_dir, 0o777) def parallel_run(function, items, *args, **kwargs): with concurrent.futures.ThreadPoolExecutor() as executor: futures = [executor.submit(function, item, *args, **kwargs) for item in items] for future in futures: future.result() def initialize_db_for_cluster(pg_path, rel_data_path, settings, node_names): subprocess.run(["mkdir", rel_data_path], check=True) def initialize(node_name): abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) command = [ os.path.join(pg_path, "initdb"), "--pgdata", abs_data_path, "--username", USER, "--no-sync", # --allow-group-access is used to ensure we set permissions on # private keys correctly "--allow-group-access", "--data-checksums", "--encoding", "UTF8", "--locale", "POSIX", ] subprocess.run(command, check=True) add_settings(abs_data_path, settings) parallel_run(initialize, node_names) def add_settings(abs_data_path, settings): conf_path = os.path.join(abs_data_path, "postgresql.conf") with open(conf_path, "a") as conf_file: for setting_key, setting_val in settings.items(): setting = "{setting_key} = '{setting_val}'\n".format( setting_key=setting_key, setting_val=setting_val ) conf_file.write(setting) def create_role(pg_path, node_ports, user_name): def create(port): command = ( "SET citus.enable_ddl_propagation TO OFF;" + "SELECT worker_create_or_alter_role('{}', 'CREATE ROLE {} WITH LOGIN CREATEROLE CREATEDB;', NULL)".format( user_name, user_name ) ) utils.psql(pg_path, port, command) command = "SET citus.enable_ddl_propagation TO OFF; GRANT CREATE ON DATABASE postgres to {}".format( user_name ) utils.psql(pg_path, port, command) parallel_run(create, node_ports) def coordinator_should_haveshards(pg_path, port): command = "SELECT citus_set_node_property('localhost', {}, 'shouldhaveshards', true)".format( port ) utils.psql(pg_path, port, command) def start_databases( pg_path, rel_data_path, node_name_to_ports, logfile_prefix, env_variables ): def start(node_name): abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) node_port = node_name_to_ports[node_name] command = [ os.path.join(pg_path, "pg_ctl"), "start", "--pgdata", abs_data_path, "-U", USER, "-o", "-p {}".format(node_port), "--log", os.path.join(abs_data_path, logfile_name(logfile_prefix, node_name)), ] # set the application name if requires if env_variables != {}: os.environ.update(env_variables) subprocess.run(command, check=True) parallel_run(start, node_name_to_ports.keys()) # We don't want parallel shutdown here because that will fail when it's # tried in this atexit call with an error like: # cannot schedule new futures after interpreter shutdown atexit.register( stop_databases, pg_path, rel_data_path, node_name_to_ports, logfile_prefix, no_output=True, parallel=False, ) def create_citus_extension(pg_path, node_ports): def create(port): utils.psql(pg_path, port, "CREATE EXTENSION citus;") parallel_run(create, node_ports) def run_pg_regress(pg_path, pg_srcdir, port, schedule): should_exit = True try: _run_pg_regress(pg_path, pg_srcdir, port, schedule, should_exit) finally: subprocess.run("bin/copy_modified", check=True) def run_pg_regress_without_exit( pg_path, pg_srcdir, port, schedule, output_dir=".", input_dir=".", user="postgres", extra_tests="", ): should_exit = False exit_code = _run_pg_regress( pg_path, pg_srcdir, port, schedule, should_exit, output_dir, input_dir, user, extra_tests, ) copy_binary_path = os.path.join(input_dir, "copy_modified_wrapper") exit_code |= subprocess.call(copy_binary_path) return exit_code def _run_pg_regress( pg_path, pg_srcdir, port, schedule, should_exit, output_dir=".", input_dir=".", user="postgres", extra_tests="", ): command = [ os.path.join(pg_srcdir, "src/test/regress/pg_regress"), "--port", str(port), "--schedule", schedule, "--bindir", pg_path, "--user", user, "--dbname", "postgres", "--inputdir", input_dir, "--outputdir", output_dir, "--use-existing", ] if PG_MAJOR_VERSION >= 16: command.append("--expecteddir") command.append(output_dir) if extra_tests != "": command.append(extra_tests) exit_code = subprocess.call(command) if should_exit and exit_code != 0: sys.exit(exit_code) return exit_code def save_regression_diff(name, output_dir): path = os.path.join(output_dir, "regression.diffs") if not os.path.exists(path): return new_file_path = os.path.join(output_dir, "./regression_{}.diffs".format(name)) print("new file path:", new_file_path) shutil.move(path, new_file_path) def stop_metadata_to_workers(pg_path, worker_ports, coordinator_port): for port in worker_ports: command = ( "SELECT * from stop_metadata_sync_to_node('localhost', {port});".format( port=port ) ) utils.psql(pg_path, coordinator_port, command) def add_coordinator_to_metadata(pg_path, coordinator_port): command = "SELECT citus_set_coordinator_host('localhost');" utils.psql(pg_path, coordinator_port, command) def add_workers(pg_path, worker_ports, coordinator_port): for port in worker_ports: command = "SELECT * from master_add_node('localhost', {port});".format( port=port ) utils.psql(pg_path, coordinator_port, command) def logfile_name(logfile_prefix, node_name): return "logfile_" + logfile_prefix + "_" + node_name def stop_databases( pg_path, rel_data_path, node_name_to_ports, logfile_prefix, no_output=False, parallel=True, ): def stop(node_name): abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) node_port = node_name_to_ports[node_name] command = [ os.path.join(pg_path, "pg_ctl"), "stop", "--pgdata", abs_data_path, "-U", USER, "-o", "-p {}".format(node_port), "--log", os.path.join(abs_data_path, logfile_name(logfile_prefix, node_name)), ] if no_output: subprocess.call( command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) else: subprocess.call(command) if parallel: parallel_run(stop, node_name_to_ports.keys()) else: for node_name in node_name_to_ports.keys(): stop(node_name) def is_citus_set_coordinator_host_udf_exist(pg_path, port): return utils.psql_capture(pg_path, port, IS_CITUS_VERSION_11_SQL) == b" t\n\n" def initialize_citus_cluster(bindir, datadir, settings, config): # In case there was a leftover from previous runs, stop the databases stop_databases( bindir, datadir, config.node_name_to_ports, config.name, no_output=True ) initialize_db_for_cluster( bindir, datadir, settings, config.node_name_to_ports.keys() ) start_databases( bindir, datadir, config.node_name_to_ports, config.name, config.env_variables ) create_citus_extension(bindir, config.node_name_to_ports.values()) # In upgrade tests, it is possible that Citus version < 11.0 # where the citus_set_coordinator_host UDF does not exist. if is_citus_set_coordinator_host_udf_exist(bindir, config.coordinator_port()): add_coordinator_to_metadata(bindir, config.coordinator_port()) add_workers(bindir, config.worker_ports, config.coordinator_port()) if not config.is_mx: stop_metadata_to_workers(bindir, config.worker_ports, config.coordinator_port()) config.setup_steps() def sudo(command, *args, shell=True, **kwargs): """ A version of run that prefixes the command with sudo when the process is not already run as root """ effective_user_id = os.geteuid() if effective_user_id == 0: return run(command, *args, shell=shell, **kwargs) if shell: return run(f"sudo {command}", *args, shell=shell, **kwargs) else: return run(["sudo", *command]) # this is out of ephemeral port range for many systems hence # it is a lower chance that it will conflict with "in-use" ports PORT_LOWER_BOUND = 10200 # ephemeral port start on many Linux systems PORT_UPPER_BOUND = 32768 next_port = PORT_LOWER_BOUND def notice_handler(diag: psycopg.errors.Diagnostic): print(f"{diag.severity}: {diag.message_primary}") if diag.message_detail: print(f"DETAIL: {diag.message_detail}") if diag.message_hint: print(f"HINT: {diag.message_hint}") if diag.context: print(f"CONTEXT: {diag.context}") def cleanup_test_leftovers(nodes): """ Cleaning up test leftovers needs to be done in a specific order, because some of these leftovers depend on others having been removed. They might even depend on leftovers on other nodes being removed. So this takes a list of nodes, so that we can clean up all test leftovers globally in the correct order. """ for node in nodes: node.cleanup_subscriptions() for node in nodes: node.cleanup_publications() for node in nodes: node.cleanup_replication_slots() for node in nodes: node.cleanup_schemas() for node in nodes: node.cleanup_databases() for node in nodes: node.cleanup_users() class PortLock: """PortLock allows you to take a lock an a specific port. While a port is locked by one process, other processes using PortLock won't get the same port. """ def __init__(self): global next_port first_port = next_port while True: next_port += 1 if next_port >= PORT_UPPER_BOUND: next_port = PORT_LOWER_BOUND # avoid infinite loop if first_port == next_port: raise Exception("Could not find port") self.lock = filelock.FileLock(Path(gettempdir()) / f"port-{next_port}.lock") try: self.lock.acquire(timeout=0) except filelock.Timeout: continue if FORCE_PORTS: self.port = next_port break with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: try: s.bind(("127.0.0.1", next_port)) self.port = next_port break except Exception: self.lock.release() continue def release(self): """Call release when you are done with the port. This way other processes can use it again. """ self.lock.release() class QueryRunner(ABC): """A subclassable interface class that can be used to run queries. This is mostly useful to be generic across differnt types of things that implement the Postgres interface, such as Postgres, PgBouncer, or a Citus cluster. This implements some helpers send queries in a simpler manner than psycopg allows by default. """ @abstractmethod def set_default_connection_options(self, options: dict[str, typing.Any]): """Sets the default connection options on the given options dictionary This is the only method that the class that subclasses QueryRunner needs to implement. """ ... def make_conninfo(self, **kwargs) -> str: self.set_default_connection_options(kwargs) return psycopg.conninfo.make_conninfo(**kwargs) def conn(self, *, autocommit=True, **kwargs): """Open a psycopg connection to this server""" self.set_default_connection_options(kwargs) conn = psycopg.connect( autocommit=autocommit, **kwargs, ) conn.add_notice_handler(notice_handler) return conn def aconn(self, *, autocommit=True, **kwargs): """Open an asynchronous psycopg connection to this server""" self.set_default_connection_options(kwargs) return psycopg.AsyncConnection.connect( autocommit=autocommit, **kwargs, ) @contextmanager def cur(self, autocommit=True, **kwargs): """Open an psycopg cursor to this server The connection and the cursors automatically close once you leave the "with" block """ with self.conn( autocommit=autocommit, **kwargs, ) as conn: with conn.cursor() as cur: yield cur @asynccontextmanager async def acur(self, **kwargs): """Open an asynchronous psycopg cursor to this server The connection and the cursors automatically close once you leave the "async with" block """ async with await self.aconn(**kwargs) as conn: async with conn.cursor() as cur: yield cur def sql(self, query, params=None, **kwargs): """Run an SQL query This opens a new connection and closes it once the query is done """ with self.cur(**kwargs) as cur: cur.execute(query, params=params) def sql_prepared(self, query, params=None, **kwargs): """Run an SQL query, with prepare=True This opens a new connection and closes it once the query is done """ with self.cur(**kwargs) as cur: cur.execute(query, params=params, prepare=True) def sql_row(self, query, params=None, allow_empty_result=False, **kwargs): """Run an SQL query that returns a single row and returns this row This opens a new connection and closes it once the query is done """ with self.cur(**kwargs) as cur: cur.execute(query, params=params) result = cur.fetchall() if allow_empty_result and len(result) == 0: return None assert len(result) == 1, "sql_row returns more than one row" return result[0] def sql_value(self, query, params=None, allow_empty_result=False, **kwargs): """Run an SQL query that returns a single cell and return this value This opens a new connection and closes it once the query is done """ with self.cur(**kwargs) as cur: cur.execute(query, params=params) result = cur.fetchall() if allow_empty_result and len(result) == 0: return None assert len(result) == 1 assert len(result[0]) == 1 value = result[0][0] return value def asql(self, query, **kwargs): """Run an SQL query in asynchronous task This opens a new connection and closes it once the query is done """ return asyncio.ensure_future(self.asql_coroutine(query, **kwargs)) async def asql_coroutine( self, query, params=None, **kwargs ) -> typing.Optional[typing.List[typing.Any]]: async with self.acur(**kwargs) as cur: await cur.execute(query, params=params) try: return await cur.fetchall() except psycopg.ProgrammingError as e: if "the last operation didn't produce a result" == str(e): return None raise def psql(self, query, **kwargs): """Run an SQL query using psql instead of psycopg This opens a new connection and closes it once the query is done """ conninfo = self.make_conninfo(**kwargs) run( ["psql", "-X", f"{conninfo}", "-c", query], shell=False, silent=True, ) def poll_query_until(self, query, params=None, expected=True, **kwargs): """Run query repeatedly until it returns the expected result""" start = datetime.now() result = None while start + TIMEOUT_DEFAULT > datetime.now(): result = self.sql_value( query, params=params, allow_empty_result=True, **kwargs ) if result == expected: return time.sleep(0.1) raise Exception( f"Timeout reached while polling query, last result was: {result}" ) @contextmanager def transaction(self, **kwargs): with self.cur(**kwargs) as cur: with cur.connection.transaction(): yield cur def sleep(self, duration=3, **kwargs): """Run pg_sleep""" return self.sql(f"select pg_sleep({duration})", **kwargs) def asleep(self, duration=3, times=1, sequentially=False, **kwargs): """Run pg_sleep asynchronously in a task. times: You can create a single task that opens multiple connections, which run pg_sleep concurrently. The asynchronous task will only complete once all these pg_sleep calls are finished. sequentially: Instead of running all pg_sleep calls spawned by providing times > 1 concurrently, this will run them sequentially. """ return asyncio.ensure_future( self.asleep_coroutine( duration=duration, times=times, sequentially=sequentially, **kwargs ) ) async def asleep_coroutine(self, duration=3, times=1, sequentially=False, **kwargs): """This is the coroutine that the asleep task runs internally""" if not sequentially: await asyncio.gather( *[ self.asql(f"select pg_sleep({duration})", **kwargs) for _ in range(times) ] ) else: for _ in range(times): await self.asql(f"select pg_sleep({duration})", **kwargs) def test(self, **kwargs): """Test if you can connect""" return self.sql("select 1", **kwargs) def atest(self, **kwargs): """Test if you can connect asynchronously""" return self.asql("select 1", **kwargs) def psql_test(self, **kwargs): """Test if you can connect with psql instead of psycopg""" return self.psql("select 1", **kwargs) def debug(self): print("Connect manually to:\n ", repr(self.make_conninfo())) print("Press Enter to continue running the test...") input() def psql_debug(self, **kwargs): conninfo = self.make_conninfo(**kwargs) run( ["psql", f"{conninfo}"], shell=False, silent=True, ) class Postgres(QueryRunner): """A class that represents a Postgres instance on this machine You can query it by using the interface provided by QueryRunner or use many of the helper methods. """ def __init__(self, pgdata): self.port_lock = PortLock() # These values should almost never be changed after initialization self.host = "127.0.0.1" self.port = self.port_lock.port # These values can be changed when needed self.dbname = "postgres" self.user = "postgres" self.schema = None self.pgdata = pgdata self.log_path = self.pgdata / "pg.log" # Used to track objects that we want to clean up at the end of a test self.subscriptions = set() self.publications = set() self.replication_slots = set() self.databases = set() self.schemas = set() self.users = set() def set_default_connection_options(self, options): options.setdefault("host", self.host) options.setdefault("port", self.port) options.setdefault("dbname", self.dbname) options.setdefault("user", self.user) if self.schema is not None: options.setdefault("options", f"-c search_path={self.schema}") options.setdefault("connect_timeout", 3) # needed for Ubuntu 18.04 options.setdefault("client_encoding", "UTF8") def initdb(self): run( f"initdb -A trust --nosync --username postgres --pgdata {self.pgdata} --allow-group-access --encoding UTF8 --locale POSIX", stdout=subprocess.DEVNULL, ) with self.conf_path.open(mode="a") as pgconf: # Allow connecting over unix sockets pgconf.write("unix_socket_directories = '/tmp'\n") # Useful logs for debugging issues pgconf.write("log_replication_commands = on\n") # The following to are also useful for debugging, but quite noisy. # So better to enable them manually by uncommenting. # pgconf.write("log_connections = on\n") # pgconf.write("log_disconnections = on\n") # Enable citus pgconf.write("shared_preload_libraries = 'citus'\n") # Allow CREATE SUBSCRIPTION to work pgconf.write("wal_level = 'logical'\n") # Faster logical replication status update so tests with logical replication # run faster pgconf.write("wal_receiver_status_interval = 1\n") # Faster logical replication apply worker launch so tests with logical # replication run faster. This is used in ApplyLauncherMain in # src/backend/replication/logical/launcher.c. pgconf.write("wal_retrieve_retry_interval = '250ms'\n") # Make sure there's enough logical replication resources for most # of our tests pgconf.write("max_logical_replication_workers = 50\n") pgconf.write("max_wal_senders = 50\n") pgconf.write("max_worker_processes = 50\n") pgconf.write("max_replication_slots = 50\n") # We need to make the log go to stderr so that the tests can # check what is being logged. This should be the default, but # some packagings change the default configuration. pgconf.write("log_destination = stderr\n") # We don't need the logs anywhere else than stderr pgconf.write("logging_collector = off\n") # This makes tests run faster and we don't care about crash safety # of our test data. pgconf.write("fsync = false\n") # conservative settings to ensure we can run multiple postmasters: pgconf.write("shared_buffers = 1MB\n") # limit disk space consumption, too: pgconf.write("max_wal_size = 128MB\n") # don't restart after crashes to make it obvious that a crash # happened pgconf.write("restart_after_crash = off\n") os.truncate(self.hba_path, 0) if PG_SUPPORTS_SSL: self.ssl_access("all", "trust") self.nossl_access("all", "trust") self.commit_hba() def init_with_citus(self): self.initdb() self.start() self.sql("CREATE EXTENSION citus") if PG_SUPPORTS_SSL: # Manually turn on ssl, so that we can safely truncate # postgresql.auto.conf later. We can only do this after creating the # citus extension because that creates the self signed certificates. with self.conf_path.open(mode="a") as pgconf: pgconf.write("ssl = on\n") def pgctl(self, command, **kwargs): run(f"pg_ctl -w --pgdata {self.pgdata} {command}", **kwargs) def apgctl(self, command, **kwargs): return asyncio.create_subprocess_shell( f"pg_ctl -w --pgdata {self.pgdata} {command}", **kwargs ) def start(self): try: self.pgctl(f'-o "-p {self.port}" -l {self.log_path} start') except Exception: print(f"\n\nPG_LOG: {self.pgdata}\n") with self.log_path.open() as f: print(f.read()) raise def stop(self, mode="fast"): self.pgctl(f"-m {mode} stop", check=False) def cleanup(self): self.stop() self.port_lock.release() def restart(self): self.stop() self.start() def reload(self): self.pgctl("reload") # Sadly UNIX signals are asynchronous, so we sleep a bit and hope that # Postgres actually processed the SIGHUP signal after the sleep. time.sleep(0.1) async def arestart(self): process = await self.apgctl("-m fast restart") await process.communicate() def nossl_access(self, dbname, auth_type): """Prepends a local non-SSL access to the HBA file""" with self.hba_path.open() as pghba: old_contents = pghba.read() with self.hba_path.open(mode="w") as pghba: pghba.write(f"local {dbname} all {auth_type}\n") pghba.write(f"hostnossl {dbname} all 127.0.0.1/32 {auth_type}\n") pghba.write(f"hostnossl {dbname} all ::1/128 {auth_type}\n") pghba.write(old_contents) def ssl_access(self, dbname, auth_type): """Prepends a local SSL access rule to the HBA file""" with self.hba_path.open() as pghba: old_contents = pghba.read() with self.hba_path.open(mode="w") as pghba: pghba.write(f"hostssl {dbname} all 127.0.0.1/32 {auth_type}\n") pghba.write(f"hostssl {dbname} all ::1/128 {auth_type}\n") pghba.write(old_contents) @property def hba_path(self): return self.pgdata / "pg_hba.conf" @property def conf_path(self): return self.pgdata / "postgresql.conf" def commit_hba(self): """Mark the current HBA contents as non-resetable by reset_hba""" with self.hba_path.open() as pghba: old_contents = pghba.read() with self.hba_path.open(mode="w") as pghba: pghba.write("# committed-rules\n") pghba.write(old_contents) def reset_hba(self): """Remove any HBA rules that were added after the last call to commit_hba""" with self.hba_path.open() as f: hba_contents = f.read() committed = hba_contents[hba_contents.find("# committed-rules\n") :] with self.hba_path.open("w") as f: f.write(committed) def prepare_reset(self): """Prepares all changes to reset Postgres settings and objects To actually apply the prepared changes a restart might still be needed. """ self.reset_hba() os.truncate(self.pgdata / "postgresql.auto.conf", 0) def reset(self): """Resets any changes to Postgres settings from previous tests""" self.prepare_reset() self.restart() async def delayed_start(self, delay=1): """Start Postgres after a delay NOTE: The sleep is asynchronous, but while waiting for Postgres to start the pg_ctl start command will block the event loop. This is currently acceptable for our usage of this method in the existing tests and this way it was easiest to implement. However, it seems totally reasonable to change this behaviour in the future if necessary. """ await asyncio.sleep(delay) self.start() def configure(self, *configs): """Configure specific Postgres settings using ALTER SYSTEM SET NOTE: after configuring a call to reload or restart is needed for the settings to become effective. """ for config in configs: self.sql(f"alter system set {config}") def log_handle(self): """Returns the opened logfile at the current end of the log By later calling read on this file you can read the contents that were written from this moment on. IMPORTANT: This handle should be closed once it's not needed anymore """ f = self.log_path.open() f.seek(0, os.SEEK_END) return f @contextmanager def log_contains(self, re_string, times=None): """Checks if during this with block the log matches re_string re_string: The regex to search for. times: If None, any number of matches is accepted. If a number, only that specific number of matches is accepted. """ with self.log_handle() as f: yield content = f.read() if times is None: assert re.search(re_string, content) else: match_count = len(re.findall(re_string, content)) assert match_count == times def create_user(self, name, args: typing.Optional[psycopg.sql.Composable] = None): self.users.add(name) if args is None: args = sql.SQL("") self.sql(sql.SQL("CREATE USER {} {}").format(sql.Identifier(name), args)) def create_database(self, name): self.databases.add(name) self.sql(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) def create_schema(self, name): self.schemas.add(name) self.sql(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(name))) def create_publication(self, name: str, args: psycopg.sql.Composable): self.publications.add(name) self.sql(sql.SQL("CREATE PUBLICATION {} {}").format(sql.Identifier(name), args)) def create_logical_replication_slot( self, name, plugin, temporary=False, twophase=False ): self.replication_slots.add(name) self.sql( "SELECT pg_catalog.pg_create_logical_replication_slot(%s,%s,%s,%s)", (name, plugin, temporary, twophase), ) def create_subscription(self, name: str, args: psycopg.sql.Composable): self.subscriptions.add(name) self.sql( sql.SQL("CREATE SUBSCRIPTION {} {}").format(sql.Identifier(name), args) ) def cleanup_users(self): for user in self.users: self.sql(sql.SQL("DROP USER IF EXISTS {}").format(sql.Identifier(user))) def cleanup_databases(self): for database in self.databases: self.sql( sql.SQL("DROP DATABASE IF EXISTS {}").format(sql.Identifier(database)) ) def cleanup_schemas(self): for schema in self.schemas: self.sql( sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format( sql.Identifier(schema) ) ) def cleanup_publications(self): for publication in self.publications: self.sql( sql.SQL("DROP PUBLICATION IF EXISTS {}").format( sql.Identifier(publication) ) ) def cleanup_replication_slots(self): for slot in self.replication_slots: start = time.time() while True: try: self.sql( "SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = %s", (slot,), ) except psycopg.errors.ObjectInUse: if time.time() < start + 10: time.sleep(0.5) continue raise break def cleanup_subscriptions(self): for subscription in self.subscriptions: try: self.sql( sql.SQL("ALTER SUBSCRIPTION {} DISABLE").format( sql.Identifier(subscription) ) ) except psycopg.errors.UndefinedObject: # Subscription didn't exist already continue self.sql( sql.SQL("ALTER SUBSCRIPTION {} SET (slot_name = NONE)").format( sql.Identifier(subscription) ) ) self.sql( sql.SQL("DROP SUBSCRIPTION {}").format(sql.Identifier(subscription)) ) def lsn(self, mode): """Returns the lsn for the given mode""" queries = { "insert": "SELECT pg_current_wal_insert_lsn()", "flush": "SELECT pg_current_wal_flush_lsn()", "write": "SELECT pg_current_wal_lsn()", "receive": "SELECT pg_last_wal_receive_lsn()", "replay": "SELECT pg_last_wal_replay_lsn()", } return self.sql_value(queries[mode]) def wait_for_catchup(self, subscription_name, mode="replay", target_lsn=None): """Waits until the subscription has caught up""" if target_lsn is None: target_lsn = self.lsn("write") # Before release 12 walreceiver just set the application name to # "walreceiver" self.poll_query_until( sql.SQL( """ SELECT {} <= {} AND state = 'streaming' FROM pg_catalog.pg_stat_replication WHERE application_name IN ({}, 'walreceiver') """ ).format(target_lsn, sql.Identifier(f"{mode}_lsn"), subscription_name) ) @contextmanager def _enable_firewall(self): """Enables the firewall for the platform that you are running Normally this should not be called directly, and instead drop_traffic or reject_traffic should be used. """ fw_token = None if BSD: if MACOS: command_stderr = sudo( "pfctl -E", stderr=subprocess.PIPE, text=True ).stderr match = re.search(r"^Token : (\d+)", command_stderr, flags=re.MULTILINE) assert match is not None fw_token = match.group(1) sudo( 'bash -c "' f"echo 'anchor \\\"port_{self.port}\\\"'" f' | pfctl -a citus_test -f -"' ) try: yield finally: if MACOS: sudo(f"pfctl -X {fw_token}") @contextmanager def drop_traffic(self): """Drops all TCP packets to this query runner""" with self._enable_firewall(): if LINUX: sudo( "iptables --append OUTPUT " "--protocol tcp " f"--destination {self.host} " f"--destination-port {self.port} " "--jump DROP " ) elif BSD: sudo( "bash -c '" f'echo "block drop out proto tcp from any to {self.host} port {self.port}"' f"| pfctl -a citus_test/port_{self.port} -f -'" ) else: raise Exception("This OS cannot run this test") try: yield finally: if LINUX: sudo( "iptables --delete OUTPUT " "--protocol tcp " f"--destination {self.host} " f"--destination-port {self.port} " "--jump DROP " ) elif BSD: sudo(f"pfctl -a citus_test/port_{self.port} -F all") @contextmanager def reject_traffic(self): """Rejects all traffic to this query runner with a TCP RST message""" with self._enable_firewall(): if LINUX: sudo( "iptables --append OUTPUT " "--protocol tcp " f"--destination {self.host} " f"--destination-port {self.port} " "--jump REJECT " "--reject-with tcp-reset" ) elif BSD: sudo( "bash -c '" f'echo "block return-rst out out proto tcp from any to {self.host} port {self.port}"' f"| pfctl -a citus_test/port_{self.port} -f -'" ) else: raise Exception("This OS cannot run this test") try: yield finally: if LINUX: sudo( "iptables --delete OUTPUT " "--protocol tcp " f"--destination {self.host} " f"--destination-port {self.port} " "--jump REJECT " "--reject-with tcp-reset" ) elif BSD: sudo(f"pfctl -a citus_test/port_{self.port} -F all") class CitusCluster(QueryRunner): """A class that represents a Citus cluster on this machine The nodes in the cluster can be accessed directly using the coordinator, workers, and nodes properties. If it doesn't matter which of the nodes in the cluster is used to run a query, then you can use the methods provided by QueryRunner directly on the cluster. In that case a random node will be chosen to run your query. """ def __init__(self, basedir: Path, worker_count: int): self.coordinator = Postgres(basedir / "coordinator") self.workers = [Postgres(basedir / f"worker{i}") for i in range(worker_count)] self.nodes = [self.coordinator] + self.workers self._schema = None self.failed_reset = False parallel_run(Postgres.init_with_citus, self.nodes) with self.coordinator.cur() as cur: cur.execute( "SELECT pg_catalog.citus_set_coordinator_host(%s, %s)", (self.coordinator.host, self.coordinator.port), ) for worker in self.workers: cur.execute( "SELECT pg_catalog.citus_add_node(%s, %s)", (worker.host, worker.port), ) def set_default_connection_options(self, options): random.choice(self.nodes).set_default_connection_options(options) @property def schema(self): return self._schema @schema.setter def schema(self, value): self._schema = value for node in self.nodes: node.schema = value def reset(self): """Resets any changes to Postgres settings from previous tests""" parallel_run(Postgres.prepare_reset, self.nodes) parallel_run(Postgres.restart, self.nodes) def cleanup(self): parallel_run(Postgres.cleanup, self.nodes) def debug(self): """Print information to stdout to help with debugging your cluster""" print("The nodes in this cluster and their connection strings are:") for node in self.nodes: print(f"{node.pgdata}:\n ", repr(node.make_conninfo())) print("Press Enter to continue running the test...") input() ================================================ FILE: src/test/regress/citus_tests/config.py ================================================ import inspect import os import random import socket import threading from contextlib import closing from os.path import expanduser import common COORDINATOR_NAME = "coordinator" WORKER1 = "worker1" WORKER2 = "worker2" REGULAR_USER_NAME = "regularuser" SUPER_USER_NAME = "postgres" DATABASE_NAME = "postgres" ARBITRARY_SCHEDULE_NAMES = [ "create_schedule", "sql_schedule", "sql_base_schedule", "postgres_schedule", "single_shard_table_prep_schedule", ] BEFORE_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE = "./before_pg_upgrade_with_columnar_schedule" BEFORE_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE = ( "./before_pg_upgrade_without_columnar_schedule" ) AFTER_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE = "./after_pg_upgrade_with_columnar_schedule" AFTER_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE = ( "./after_pg_upgrade_without_columnar_schedule" ) CREATE_SCHEDULE = "./create_schedule" POSTGRES_SCHEDULE = "./postgres_schedule" SINGLE_SHARD_PREP_SCHEDULE = "./single_shard_table_prep_schedule" SQL_SCHEDULE = "./sql_schedule" SQL_BASE_SCHEDULE = "./sql_base_schedule" AFTER_CITUS_UPGRADE_COORD_SCHEDULE = "./after_citus_upgrade_coord_schedule" BEFORE_CITUS_UPGRADE_COORD_SCHEDULE = "./before_citus_upgrade_coord_schedule" MIXED_BEFORE_CITUS_UPGRADE_SCHEDULE = "./mixed_before_citus_upgrade_schedule" MIXED_AFTER_CITUS_UPGRADE_SCHEDULE = "./mixed_after_citus_upgrade_schedule" CITUS_ARBITRARY_TEST_DIR = "./tmp_citus_test" MASTER = "master" # This should be updated when citus version changes MASTER_VERSION = "15.0" HOME = expanduser("~") CITUS_VERSION_SQL = "SELECT extversion FROM pg_extension WHERE extname = 'citus';" # this is out of ephemeral port range for many systems hence # it is a lower change that it will conflict with "in-use" ports next_port = 10200 # ephemeral port start on many linux systems PORT_UPPER = 32768 port_lock = threading.Lock() def should_include_config(class_name): if inspect.isclass(class_name) and issubclass( class_name, CitusDefaultClusterConfig ): return True return False def find_free_port(): global next_port with port_lock: while next_port < PORT_UPPER: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: try: s.bind(("localhost", next_port)) port = next_port next_port += 1 return port except Exception: next_port += 1 # we couldn't find a port raise Exception("Couldn't find a port to use") class NewInitCaller(type): def __call__(cls, *args, **kwargs): obj = type.__call__(cls, *args, **kwargs) obj.post_init() return obj class CitusBaseClusterConfig(object, metaclass=NewInitCaller): def __init__(self, arguments): if "--bindir" in arguments: self.bindir = arguments["--bindir"] self.pg_srcdir = arguments["--pgxsdir"] self.temp_dir = CITUS_ARBITRARY_TEST_DIR self.worker_amount = 2 self.user = REGULAR_USER_NAME self.dbname = DATABASE_NAME self.is_mx = True self.is_citus = True self.all_null_dist_key = False self.test_with_columnar = False self.name = type(self).__name__ self.settings = { "shared_preload_libraries": "citus", "log_error_verbosity": "terse", "fsync": False, "citus.node_conninfo": "sslmode=prefer", "citus.enable_repartition_joins": True, "citus.repartition_join_bucket_count_per_node": 2, "citus.log_distributed_deadlock_detection": True, "max_connections": 1200, } self.new_settings = {} self.env_variables = {} self.skip_tests = [] def post_init(self): self._init_node_name_ports() self.datadir = self.temp_dir + "/data" self.datadir += self.name self.input_dir = self.datadir self.output_dir = self.datadir self.output_file = os.path.join(self.datadir, "run.out") if self.worker_amount > 0: self.chosen_random_worker_port = self.random_worker_port() self.settings.update(self.new_settings) def coordinator_port(self): return self.node_name_to_ports[COORDINATOR_NAME] def setup_steps(self): pass def random_worker_port(self): return random.choice(self.worker_ports) def random_port(self): return random.choice(list(self.node_name_to_ports.values())) def _init_node_name_ports(self): self.node_name_to_ports = {} self.worker_ports = [] cur_port = self._get_and_update_next_port() self.node_name_to_ports[COORDINATOR_NAME] = cur_port for i in range(self.worker_amount): cur_port = self._get_and_update_next_port() cur_worker_name = "worker{}".format(i) self.node_name_to_ports[cur_worker_name] = cur_port self.worker_ports.append(cur_port) def _get_and_update_next_port(self): if hasattr(self, "fixed_port"): next_port = self.fixed_port self.fixed_port += 1 return next_port return find_free_port() class CitusDefaultClusterConfig(CitusBaseClusterConfig): def __init__(self, arguments): super().__init__(arguments) new_settings = { "client_min_messages": "WARNING", "citus.sort_returning": True, "citus.use_citus_managed_tables": True, } self.settings.update(new_settings) self.skip_tests = [ # Alter Table statement cannot be run from an arbitrary node so this test will fail "arbitrary_configs_alter_table_add_constraint_without_name_create", "arbitrary_configs_alter_table_add_constraint_without_name", ] class CitusUpgradeConfig(CitusBaseClusterConfig): def __init__(self, arguments, pre_tar, post_tar): super().__init__(arguments) self.pre_tar_path = pre_tar self.post_tar_path = post_tar self.temp_dir = "./tmp_citus_upgrade" self.new_settings = {"citus.enable_version_checks": "false"} self.user = SUPER_USER_NAME self.mixed_mode = arguments["--mixed"] self.minor_upgrade = arguments.get("--minor-upgrade", False) self.fixed_port = 57635 class PostgresConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.worker_amount = 0 self.is_citus = False self.new_settings = { "citus.use_citus_managed_tables": False, } self.skip_tests = [ "nested_execution", # Alter Table statement cannot be run from an arbitrary node so this test will fail "arbitrary_configs_alter_table_add_constraint_without_name_create", "arbitrary_configs_alter_table_add_constraint_without_name", ] class AllSingleShardTableDefaultConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.all_null_dist_key = True self.skip_tests += [ # One of the distributed functions created in "function_create" # requires setting a distribution column, which cannot be the # case with single shard tables. "function_create", "functions", # In "nested_execution", one of the tests that query # "dist_query_single_shard" table acts differently when the table # has a single shard. This is explained with a comment in the test. "nested_execution", "merge_arbitrary", ] class CitusSingleNodeClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.worker_amount = 0 def setup_steps(self): common.coordinator_should_haveshards(self.bindir, self.coordinator_port()) class CitusSingleWorkerClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.worker_amount = 1 class CitusSuperUserDefaultClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.user = SUPER_USER_NAME class CitusThreeWorkersManyShardsClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = {"citus.shard_count": 191} self.worker_amount = 3 def setup_steps(self): common.coordinator_should_haveshards(self.bindir, self.coordinator_port()) class CitusSmallSharedPoolSizeConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.local_shared_pool_size": 5, "citus.max_shared_pool_size": 5, } class CitusSmallExecutorPoolSizeConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.max_adaptive_executor_pool_size": 2, } class CitusSequentialExecutionConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.multi_shard_modify_mode": "sequential", } class CitusCacheManyConnectionsConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.max_cached_conns_per_worker": 4, } class CitusUnusualExecutorConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.max_adaptive_executor_pool_size": 7, "citus.executor_slow_start_interval": 1, "citus.prevent_incomplete_connection_establishment": False, "citus.enable_cost_based_connection_establishment": False, "citus.max_cached_connection_lifetime": "10ms", # https://github.com/citusdata/citus/issues/5345 # "citus.force_max_query_parallelization": "on", "citus.enable_binary_protocol": False, "citus.local_table_join_policy": "prefer-distributed", } # this setting does not necessarily need to be here # could go any other test self.env_variables = {"PGAPPNAME": "test_app"} class CitusSmallCopyBuffersConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.copy_switchover_threshold": "1B", "citus.local_copy_flush_threshold": "1B", "citus.remote_copy_flush_threshold": "1B", } class CitusUnusualQuerySettingsConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = { "citus.use_citus_managed_tables": False, "citus.task_assignment_policy": "first-replica", "citus.enable_fast_path_router_planner": False, "citus.enable_local_execution": False, "citus.enable_single_hash_repartition_joins": True, "citus.recover_2pc_interval": "1s", "citus.remote_task_check_interval": "1ms", "citus.values_materialization_threshold": "0", } self.skip_tests = [ # Creating a reference table from a table referred to by a fk # requires the table with the fk to be converted to a citus_local_table. # As of c11, there is no way to do that through remote execution so this test # will fail "arbitrary_configs_truncate_cascade_create", "arbitrary_configs_truncate_cascade", # Alter Table statement cannot be run from an arbitrary node so this test will fail "arbitrary_configs_alter_table_add_constraint_without_name_create", "arbitrary_configs_alter_table_add_constraint_without_name", ] class CitusSingleNodeSingleShardClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.worker_amount = 0 self.new_settings = {"citus.shard_count": 1} def setup_steps(self): common.coordinator_should_haveshards(self.bindir, self.coordinator_port()) class CitusShardReplicationFactorClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = {"citus.shard_replication_factor": 2} self.skip_tests = [ # citus does not support foreign keys in distributed tables # when citus.shard_replication_factor >= 2 "arbitrary_configs_truncate_partition_create", "arbitrary_configs_truncate_partition", # citus does not support modifying a partition when # citus.shard_replication_factor >= 2 "arbitrary_configs_truncate_cascade_create", "arbitrary_configs_truncate_cascade", # citus does not support colocating functions with distributed tables when # citus.shard_replication_factor >= 2 "function_create", "functions", # Alter Table statement cannot be run from an arbitrary node so this test will fail "arbitrary_configs_alter_table_add_constraint_without_name_create", "arbitrary_configs_alter_table_add_constraint_without_name", ] class CitusSingleShardClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.new_settings = {"citus.shard_count": 1} class CitusNonMxClusterConfig(CitusDefaultClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.is_mx = False # citus does not support distributed functions # when metadata is not synced self.skip_tests = ["function_create", "functions", "nested_execution"] class PGUpgradeConfig(CitusBaseClusterConfig): def __init__(self, arguments): super().__init__(arguments) self.old_bindir = arguments["--old-bindir"] self.new_bindir = arguments["--new-bindir"] self.temp_dir = "./tmp_upgrade" self.old_datadir = self.temp_dir + "/oldData" self.new_datadir = self.temp_dir + "/newData" self.user = SUPER_USER_NAME self.test_with_columnar = arguments["--test-with-columnar"] ================================================ FILE: src/test/regress/citus_tests/print_test_names.py ================================================ #!/usr/bin/env python3 import config as cfg def read_config_names(): config_names = [] # We fill the configs from all of the possible classes in config.py so that if we add a new config, # we don't need to add it here. And this avoids the problem where we forget to add it here for x in cfg.__dict__.values(): if cfg.should_include_config(x): config_names.append(x.__name__) return config_names def print_config_names(): config_names = read_config_names() for config_name in config_names: print(config_name) if __name__ == "__main__": print_config_names() ================================================ FILE: src/test/regress/citus_tests/query_generator/.gitignore ================================================ __pycache__ *-env ================================================ FILE: src/test/regress/citus_tests/query_generator/README.md ================================================ ## Goal of the Tool Tool supports a simple syntax to be useful to generate queries with different join orders. Main motivation for me to create the tool was to compare results of the generated queries for different [Citus](https://github.com/citusdata/citus) tables and Postgres tables. That is why we support a basic syntax for now. It can be extended to support different queries. ## Query Generator for Postgres Tool generates SELECT queries, whose depth can be configured, with different join orders. It also generates DDLs required for query execution. You can also tweak configuration parameters for data inserting command generation. ## How to Run Citus Join Verification? You can verify if Citus breaks any default PG join behaviour via `citus_compare_dist_local_joins.sh`. It creates tables specified in config. Then, it runs generated queries on those tables and saves the results into `out/dist_queries.out`. After running those queries for Citus tables, it creates PG tables with the same names as previous run, executes the same queries, and saves the results into `out/local_queries.out`. In final step, it generates diff between local and distributed results. You can see the contents of `out/local_dist_diffs` to see if there is any Citus unsupported query. 1. Create a Citus local cluster with 2 workers by using [citus_dev](https://github.com/citusdata/tools/tree/develop/citus_dev) tool (Note: make sure you do not configure psql via .psqlrc file as it would fail the test.) ```bash citus_dev make testCluster --destroy ``` 2. Run the test, ```bash cd src/test/regress/citus_tests/query_generator/bin bash citus_compare_dist_local_joins.sh Optional: ``` 3. See the diff content in `src/test/regress/citus_tests/query_generator/out/local_dist_diffs` Note: `seed` can be used to reproduce a run of Citus test by generating the same queries and results via the given seed. ### Configuration You can configure 3 different parts: - [DDL Configuration](#ddl-configuration) - [Data Insertion Configuration](#data-insertion-configuration) - [Query Configuration](#query-configuration) ## DDL Configuration Tool generates related ddl commands before generating queries. Schema for DDL configuration: ```yaml ddlOutFile: commonColName: targetTables: - Table: name: citusType: maxAllowedUseOnQuery: rowCount: nullRate: duplicateRate: columns: - Column: name: type: distinctCopyCount: ``` Explanation: ```yaml ddlOutFile: "file to write generated DDL commands" commonColName: "name of the column that will be used as distribution column, filter column in restrictions and target column in selections" targetTables: "array of tables that will be used in generated queries" - Table: name: "name prefix of table" citusType: "citus type of table" maxAllowedUseOnQuery: "limits how many times table can appear in query" rowCount: "total # of rows that will be inserted into table" nullRate: "percentage of null rows in rowCount that will be inserted into table" duplicateRate: "percentage of duplicates in rowCount that will be inserted into table" columns: "array of columns in table" - Column: name: "name of column" type: "name of data type of column(only support 'int' now)" distinctCopyCount: "how many tables with the same configuration we should create(only by changing full name, still using the same name prefix)" ``` ## Data Insertion Configuration Tool generates data insertion commands if you want tables with filled data. You can configure total number of rows, what percentage of them should be null and what percentage of them should be duplicated. For related configuration see Table schema at [DDL Configuration](#ddl-configuration). You can also configure range of the randomly generated data. See `dataRange` at [Query Configuration](#query-configuration). ## Query Configuration After generation of ddls and data insertion commands, the tool generates queries. Schema for Query configuration: ```yaml queryCount: queryOutFile: semiAntiJoin: cartesianProduct: limit: orderby: forceOrderbyWithLimit: useAvgAtTopLevelTarget: dataRange: from: to: filterRange: from: to: limitRange: from: to: targetRteCount: targetCteCount: targetCteRteCount: targetJoinTypes: targetRteTypes: targetRestrictOps: ``` Explanation: ```yaml queryCount: "number of queries to generate" queryOutFile: "file to write generated queries" semiAntiJoin: "should we support semi joins (WHERE col IN (Subquery))" cartesianProduct: "should we support cartesian joins" limit: "should we support limit clause" orderby: "should we support order by clause" forceOrderbyWithLimit: "should we force order by when we use limit" useAvgAtTopLevelTarget: "should we make top level query as select avg() from (subquery)" dataRange: from: "starting boundary for data generation" to: "end boundary for data generation" filterRange: from: "starting boundary for restriction clause" to: "end boundary for restriction clause" limitRange: from: "starting boundary for limit clause" to: "end boundary for data limit clause" targetRteCount: "limits how many rtes should exist in non-cte part of the query" targetCteCount: "limits how many ctes should exist in query" targetCteRteCount: "limits how many rtes should exist in cte part of the query" targetJoinTypes: "supported join types" targetRteTypes: "supported rte types" targetRestrictOps: "supported restrict ops" ``` ## Misc Configuration Tool has some configuration options which does not suit above 3 parts. Schema for misc configuration: ```yaml interactiveMode: ``` Explanation: ```yaml interactiveMode: "when true, interactively prints generated ddls and queries. Otherwise, it writes them to configured files." ``` ## Supported Operations It uses `commonColName` for any kind of target selection required for any supported query clause. ### Column Type Support Tool currently supports only int data type, but plans to support other basic types. ### Join Support Tool supports following joins: ```yaml targetJoinTypes: - INNER - LEFT - RIGHT - FULL ``` ### Citus Table Support Tool supports following citus table types: ```yaml targetTables: - Table: ... citusType: ... ``` ### Restrict Operation Support Tool supports following restrict operations: ```yaml targetRestrictOps: - LT - GT - EQ ``` ### Rte Support Tool supports following rtes: ```yaml targetRteTypes: - RELATION - SUBQUERY - CTE - VALUES ``` ## How to Generate Queries? You have 2 different options. - [Interactive Mode](#interactive-mode) - [File Mode](#file-mode) ### Interactive Mode In this mode, you will be prompted to continue generating a query. When you hit to `Enter`, it will continue generating them. You will need to hit to `x` to exit. 1. Configure `interactiveMode: true` in config.yml, 2. Run the command shown below ```bash cd src/test/regress/citus_tests/query_generator python generate_queries.py ``` ### File Mode In this mode, generated ddls and queries will be written into the files configured in [config.yml](./config/config.yaml). 1. Configure `interactiveMode: false`, 2. Configure `queryCount: `, 3. Configure `queryOutFile: ` and `ddlOutFile: ` 4. Run the command shown below ```bash cd src/test/regress/citus_tests/query_generator python generate_queries.py ``` ================================================ FILE: src/test/regress/citus_tests/query_generator/bin/citus_compare_dist_local_joins.sh ================================================ #!/bin/bash # make bash behave set -euo pipefail psql_user=$1 psql_db=$2 psql_port=$3 seed=${4:-""} runDDLs() { # run ddls psql -U "${psql_user}" -d "${psql_db}" -p "${psql_port}" -f "${out_folder}"/ddls.sql > /dev/null } runUndistributeTables() { undistribute_all_tables_command='SELECT undistribute_table(logicalrelid) FROM pg_dist_partition;' # run undistribute all tables psql -U "${psql_user}" -d "${psql_db}" -p "${psql_port}" -c "${undistribute_all_tables_command}" > /dev/null } runQueries() { out_filename=$1 # run dmls # echo queries and comments for query tracing psql -U "${psql_user}" -d "${psql_db}" -p "${psql_port}" \ --echo-all \ -f "${out_folder}"/queries.sql > "${out_filename}" 2>&1 } showDiffs() { pushd . && cd "${script_folder}" && python3 diff-checker.py && popd } # run query generator and let it create output ddls and queries script_folder=$(dirname "$0") out_folder="${script_folder}"/../out pushd . && cd "${script_folder}"/.. && python3 generate_queries.py --seed="${seed}" && popd # remove result files if exists rm -rf "${out_folder}"/dist_queries.out "${out_folder}"/local_queries.out # run ddls runDDLs # runs dmls for distributed tables runQueries "${out_folder}"/dist_queries.out # undistribute all dist tables runUndistributeTables # runs the same dmls for pg local tables runQueries "${out_folder}"/local_queries.out # see diffs in results showDiffs ================================================ FILE: src/test/regress/citus_tests/query_generator/bin/diff-checker.py ================================================ import os import re import subprocess import sys def createPatternForFailedQueryBlock(acceptableErrors): totalAcceptableOrders = len(acceptableErrors) failedQueryBlockPattern = "-- queryId: [0-9]+\n.*\npsql:.*(?:" for acceptableErrorIdx in range(totalAcceptableOrders): failedQueryBlockPattern += acceptableErrors[acceptableErrorIdx] if acceptableErrorIdx < totalAcceptableOrders - 1: failedQueryBlockPattern += "|" failedQueryBlockPattern += ")" return failedQueryBlockPattern def findFailedQueriesFromFile(queryOutFile, acceptableErrors): failedQueryIds = [] outFileContent = "" failedQueryBlockPattern = createPatternForFailedQueryBlock(acceptableErrors) queryIdPattern = "queryId: ([0-9]+)" with open(queryOutFile, "r") as f: outFileContent = f.read() failedQueryContents = re.findall(failedQueryBlockPattern, outFileContent) failedQueryIds = [ re.search(queryIdPattern, failedQueryContent)[1] for failedQueryContent in failedQueryContents ] return failedQueryIds def removeFailedQueryOutputFromFile(outFile, failedQueryIds): if len(failedQueryIds) == 0: return distOutFileContentAsLines = [] with open(outFile, "r") as f: distOutFileContentAsLines = f.readlines() with open(outFile, "w") as f: clear = False nextIdx = 0 nextQueryIdToDelete = failedQueryIds[nextIdx] queryIdPattern = "queryId: ([0-9]+)" for line in distOutFileContentAsLines: matched = re.search(queryIdPattern, line) # founded line which contains query id if matched: # query id matches with the next failed query's id if nextQueryIdToDelete == matched[1]: # clear lines until we find succesfull query clear = True nextIdx += 1 if nextIdx < len(failedQueryIds): nextQueryIdToDelete = failedQueryIds[nextIdx] else: # we found successfull query clear = False if not clear: f.write(line) return def removeFailedQueryOutputFromFiles(distQueryOutFile, localQueryOutFile): # remove the failed distributed query from both local and distributed query files to prevent diff for acceptable errors # some of generated queries fails with below errors due to https://github.com/citusdata/citus/issues/6653, we skip those until we support them acceptableErrors = [ "ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns", "ERROR: recursive complex joins are only supported when all distributed tables are co-located and joined on their distribution columns", ] failedDistQueryIds = findFailedQueriesFromFile(distQueryOutFile, acceptableErrors) removeFailedQueryOutputFromFile(distQueryOutFile, failedDistQueryIds) removeFailedQueryOutputFromFile(localQueryOutFile, failedDistQueryIds) return def showDiffs(distQueryOutFile, localQueryOutFile, diffFile): exitCode = 1 with open(diffFile, "w") as f: diffCommand = "diff -u {} {}".format(localQueryOutFile, distQueryOutFile) process = subprocess.Popen(diffCommand.split(), stdout=f, stderr=f, shell=False) process.wait() exitCode = process.returncode print("diff exit {}".format(exitCode)) return exitCode def exitIfAnyLocalQueryFailed(localQueryOutFile): allErrors = ["ERROR:"] failedLocalQueryIds = findFailedQueriesFromFile(localQueryOutFile, allErrors) assert ( len(failedLocalQueryIds) == 0 ), """There might be an internal error related to query generator or we might find a Postgres bug. Check local_queries.out to see the error.""" return if __name__ == "__main__": scriptDirPath = os.path.dirname(os.path.abspath(__file__)) outFolderPath = scriptDirPath + "/../out" localQueryOutFile = "{}/local_queries.out".format(outFolderPath) distQueryOutFile = "{}/dist_queries.out".format(outFolderPath) diffFile = "{}/local_dist.diffs".format(outFolderPath) # exit if we have any error from local queries exitIfAnyLocalQueryFailed(localQueryOutFile) # find failed queries from distQueryOutFile and then remove failed queries and their results from # both distQueryOutFile and localQueryOutFile removeFailedQueryOutputFromFiles(distQueryOutFile, localQueryOutFile) # show diffs in unified format exitCode = showDiffs(distQueryOutFile, localQueryOutFile, diffFile) sys.exit(exitCode) ================================================ FILE: src/test/regress/citus_tests/query_generator/bin/run_query_compare_test.py ================================================ #!/usr/bin/env python3 """query_gen_test Usage: run_query_compare_test --bindir= --pgxsdir= --seed= Options: --bindir= PostgreSQL executable directory(ex: '~/.pgenv/pgsql-10.4/bin') --pgxsdir= Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3) --seed= Seed number used by the query generator.(ex: 123) """ import os import subprocess import sys from docopt import docopt # https://stackoverflow.com/questions/14132789/relative-imports-for-the-billionth-time/14132912#14132912 sys.path.append( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) # ignore E402 because these imports require addition to path import common # noqa: E402 import config as cfg # noqa: E402 def run_test(config, seed): # start cluster common.initialize_temp_dir(cfg.CITUS_ARBITRARY_TEST_DIR) common.initialize_citus_cluster( config.bindir, config.datadir, config.settings, config ) # run test scriptDirPath = os.path.dirname(os.path.abspath(__file__)) testRunCommand = "bash {}/citus_compare_dist_local_joins.sh {} {} {} {}".format( scriptDirPath, config.user, config.dbname, config.coordinator_port(), seed ) process = subprocess.Popen( testRunCommand.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = process.communicate() # stop cluster common.stop_databases( config.bindir, config.datadir, config.node_name_to_ports, config.name ) print(stdout) print(stderr) print(process.returncode) sys.exit(process.returncode) if __name__ == "__main__": arguments = docopt(__doc__, version="run_query_compare_test") citusClusterConfig = cfg.CitusSuperUserDefaultClusterConfig(arguments) seed = "" if "--seed" in arguments and arguments["--seed"] != "": seed = arguments["--seed"] run_test(citusClusterConfig, seed) ================================================ FILE: src/test/regress/citus_tests/query_generator/config/config.py ================================================ import copy import os import yaml from config.config_parser import ( parseJoinTypeArray, parseRange, parseRestrictOpArray, parseRteTypeArray, parseTableArray, ) from node_defs import CitusType class Config: def __init__(self): configObj = Config.parseConfigFile( f"{os.path.dirname(os.path.abspath(__file__))}/config.yaml" ) self.targetTables = _distinctCopyTables( parseTableArray(configObj["targetTables"]) ) self.targetJoinTypes = parseJoinTypeArray(configObj["targetJoinTypes"]) self.targetRteTypes = parseRteTypeArray(configObj["targetRteTypes"]) self.targetRestrictOps = parseRestrictOpArray(configObj["targetRestrictOps"]) self.commonColName = configObj["commonColName"] self.targetRteCount = configObj["targetRteCount"] self.targetCteCount = configObj["targetCteCount"] self.targetCteRteCount = configObj["targetCteRteCount"] self.semiAntiJoin = configObj["semiAntiJoin"] self.cartesianProduct = configObj["cartesianProduct"] self.limit = configObj["limit"] self.orderby = configObj["orderby"] self.forceOrderbyWithLimit = configObj["forceOrderbyWithLimit"] self.useAvgAtTopLevelTarget = configObj["useAvgAtTopLevelTarget"] self.interactiveMode = configObj["interactiveMode"] self.queryOutFile = configObj["queryOutFile"] self.ddlOutFile = configObj["ddlOutFile"] self.queryCount = configObj["queryCount"] self.dataRange = parseRange(configObj["dataRange"]) self.filterRange = parseRange(configObj["filterRange"]) self.limitRange = parseRange(configObj["limitRange"]) self._ensureRteLimitsAreSane() # print(self) def __repr__(self): rep = "targetRteCount: {}\n".format(self.targetRteCount) rep += "targetCteCount: {}\n".format(self.targetCteCount) rep += "targetCteRteCount: {}\n".format(self.targetCteRteCount) rep += "targetRteTypes:\n" for rteType in self.targetRteTypes: rep += "\t{}\n".format(rteType) rep += "targetJoinTypes:\n" for joinType in self.targetJoinTypes: rep += "\t{}\n".format(joinType) rep += "restrictOps:\n" for restrictOp in self.targetRestrictOps: rep += "\t{}\n".format(restrictOp) return rep def _ensureRteLimitsAreSane(self): totalAllowedUseForAllTables = 0 for table in self.targetTables: totalAllowedUseForAllTables += table.maxAllowedUseOnQuery assert ( totalAllowedUseForAllTables >= self.targetRteCount ), """targetRteCount cannot be greater than sum of maxAllowedUseOnQuery for all tables. Set targetRteCount to a lower value or increase maxAllowedUseOnQuery for tables at config.yml.""" @staticmethod def parseConfigFile(path): try: with open(path, "r") as configFile: return yaml.load(configFile, yaml.Loader) except IOError as e: print(f"I/O error({0}): {1}".format(e.errno, e.strerror)) raise BaseException("cannot parse config.yaml") except Exception: print("Unexpected error while parsing config.yml.") _config = None def resetConfig(): global _config _config = Config() def getConfig(): return _config def getAllTableNames(): """returns table names from target tables given at config""" tables = getConfig().targetTables tableNames = [table.name for table in tables] return tableNames def getMaxAllowedCountForTable(tableName): tables = getConfig().targetTables filtered = filter(lambda el: el.name == tableName, tables) filtered = list(filtered) assert len(filtered) == 1 return filtered[0].maxAllowedUseOnQuery def isTableDistributed(table): return table.citusType == CitusType.DISTRIBUTED def isTableReference(table): return table.citusType == CitusType.REFERENCE def _distinctCopyTables(tables): distinctCopyTables = [] for table in tables: distinctCopyCount = table.distinctCopyCount for tblIdx in range(1, distinctCopyCount): distinctCopyTable = copy.deepcopy(table) distinctCopyTable.name += str(tblIdx) distinctCopyTables.append(distinctCopyTable) table.name += "0" tables.extend(distinctCopyTables) return tables ================================================ FILE: src/test/regress/citus_tests/query_generator/config/config.yaml ================================================ interactiveMode: false queryCount: 250 queryOutFile: queries.sql ddlOutFile: ddls.sql semiAntiJoin: true cartesianProduct: false limit: true orderby: true forceOrderbyWithLimit: true useAvgAtTopLevelTarget: true dataRange: from: 0 to: 10 filterRange: from: 0 to: 10 limitRange: from: 0 to: 10 targetRteCount: 40 targetCteCount: 3 targetCteRteCount: 2 commonColName: id targetTables: - Table: name: dist citusType: DISTRIBUTED maxAllowedUseOnQuery: 10 rowCount: 10 nullRate: 0.1 duplicateRate: 0.1 columns: - Column: name: id type: int distinctCopyCount: 2 - Table: name: ref citusType: REFERENCE maxAllowedUseOnQuery: 10 rowCount: 10 nullRate: 0.1 duplicateRate: 0.1 columns: - Column: name: id type: int distinctCopyCount: 2 targetJoinTypes: - INNER - LEFT - RIGHT - FULL targetRteTypes: - RELATION - SUBQUERY - CTE targetRestrictOps: - LT - GT ================================================ FILE: src/test/regress/citus_tests/query_generator/config/config_parser.py ================================================ from node_defs import CitusType, Column, JoinType, RestrictOp, RTEType, Table def parseJoinType(joinTypeText): return JoinType[joinTypeText] def parseJoinTypeArray(joinTypeTexts): joinTypes = [] for joinTypeText in joinTypeTexts: joinType = parseJoinType(joinTypeText) joinTypes.append(joinType) return joinTypes def parseRteType(rteTypeText): return RTEType[rteTypeText] def parseRteTypeArray(rteTypeTexts): rteTypes = [] for rteTypeText in rteTypeTexts: rteType = parseRteType(rteTypeText) rteTypes.append(rteType) return rteTypes def parseRestrictOp(restrictOpText): return RestrictOp[restrictOpText] def parseRestrictOpArray(restrictOpTexts): restrictOps = [] for restrictOpText in restrictOpTexts: restrictOp = parseRestrictOp(restrictOpText) restrictOps.append(restrictOp) return restrictOps def parseTable(targetTableDict): name = targetTableDict["name"] citusType = CitusType[targetTableDict["citusType"]] maxAllowedUseOnQuery = targetTableDict["maxAllowedUseOnQuery"] rowCount = targetTableDict["rowCount"] nullRate = targetTableDict["nullRate"] duplicateRate = targetTableDict["duplicateRate"] columns = [] for columnDict in targetTableDict["columns"]: col = parseColumn(columnDict) columns.append(col) distinctCopyCount = targetTableDict["distinctCopyCount"] return Table( name, citusType, maxAllowedUseOnQuery, rowCount, nullRate, duplicateRate, columns, distinctCopyCount, ) def parseTableArray(targetTableDicts): tables = [] for targetTableDict in targetTableDicts: table = parseTable(targetTableDict["Table"]) tables.append(table) return tables def parseColumn(targetColumnDict): name = targetColumnDict["name"] type = targetColumnDict["type"] return Column(name, type) def parseRange(rangeDict): fromVal = rangeDict["from"] toVal = rangeDict["to"] return (fromVal, toVal) ================================================ FILE: src/test/regress/citus_tests/query_generator/data_gen.py ================================================ from node_defs import CitusType from config.config import getConfig def getTableData(): dataGenerationSql = "" tableIdx = 1 (fromVal, toVal) = getConfig().dataRange tables = getConfig().targetTables for table in tables: # generate base rows dataGenerationSql += _genOverlappingData(table.name, fromVal, table.rowCount) dataGenerationSql += "\n" dataGenerationSql += _genNonOverlappingData(table.name, toVal, tableIdx) dataGenerationSql += "\n" # generate null rows if not table.citusType == CitusType.DISTRIBUTED: targetNullRows = int(table.rowCount * table.nullRate) dataGenerationSql += _genNullData(table.name, targetNullRows) dataGenerationSql += "\n" # generate duplicate rows targetDuplicateRows = int(table.rowCount * table.duplicateRate) dataGenerationSql += _genDupData(table.name, targetDuplicateRows) dataGenerationSql += "\n\n" tableIdx += 1 return dataGenerationSql def _genOverlappingData(tableName, startVal, rowCount): """returns string to fill table with [startVal,startVal+rowCount] range of integers""" dataGenerationSql = "" dataGenerationSql += "INSERT INTO " + tableName dataGenerationSql += ( " SELECT i FROM generate_series(" + str(startVal) + "," + str(startVal + rowCount) + ") i;" ) return dataGenerationSql def _genNullData(tableName, nullCount): """returns string to fill table with NULLs""" dataGenerationSql = "" dataGenerationSql += "INSERT INTO " + tableName dataGenerationSql += ( " SELECT NULL FROM generate_series(0," + str(nullCount) + ") i;" ) return dataGenerationSql def _genDupData(tableName, dupRowCount): """returns string to fill table with duplicate integers which are fetched from given table""" dataGenerationSql = "" dataGenerationSql += "INSERT INTO " + tableName dataGenerationSql += ( " SELECT * FROM " + tableName + " ORDER BY " + getConfig().commonColName + " LIMIT " + str(dupRowCount) + ";" ) return dataGenerationSql def _genNonOverlappingData(tableName, startVal, tableIdx): """returns string to fill table with different integers for given table""" startVal = startVal + tableIdx * 20 endVal = startVal + 20 dataGenerationSql = "" dataGenerationSql += "INSERT INTO " + tableName dataGenerationSql += ( " SELECT i FROM generate_series(" + str(startVal) + "," + str(endVal) + ") i;" ) return dataGenerationSql ================================================ FILE: src/test/regress/citus_tests/query_generator/ddl_gen.py ================================================ from config.config import getConfig, isTableDistributed, isTableReference def getTableDDLs(): ddls = "" tables = getConfig().targetTables for table in tables: ddls += _genTableDDL(table) ddls += "\n" return ddls def _genTableDDL(table): ddl = "" ddl += "DROP TABLE IF EXISTS " + table.name + ";" ddl += "\n" ddl += "CREATE TABLE " + table.name + "(" for column in table.columns[:-1]: ddl += _genColumnDDL(column) ddl += ",\n" if len(table.columns) > 0: ddl += _genColumnDDL(table.columns[-1]) ddl += ");\n" if isTableDistributed(table): ddl += ( "SELECT create_distributed_table(" + "'" + table.name + "','" + getConfig().commonColName + "'" + ");" ) ddl += "\n" elif isTableReference(table): ddl += "SELECT create_reference_table(" + "'" + table.name + "'" + ");" ddl += "\n" return ddl def _genColumnDDL(column): ddl = "" ddl += column.name ddl += " " ddl += column.type return ddl ================================================ FILE: src/test/regress/citus_tests/query_generator/generate_queries.py ================================================ #!/usr/bin/env python3 """generate_queries Usage: generate_queries --seed= Options: --seed= Seed number used by the query generator.(ex: 123) """ import os import random import signal import sys from data_gen import getTableData from ddl_gen import getTableDDLs from docopt import docopt from query_gen import newQuery from random_selections import currentMilliSecs from config.config import getConfig, resetConfig def _signal_handler(sig, frame): sys.exit(0) def _interactiveMode(ddls, data): print(ddls) print(data) while True: res = input("Press x to exit or Enter to generate more") if res.lower() == "x": print("Exit from query generation mode!") sys.exit(0) query = newQuery() print(query) resetConfig() def _fileMode(ddls, data): ddlFileName = ( f"{os.path.dirname(os.path.abspath(__file__))}/out/{getConfig().ddlOutFile}" ) with open(ddlFileName, "w") as ddlFile: ddlFile.writelines([ddls, data]) queryCount = getConfig().queryCount fileName = ( f"{os.path.dirname(os.path.abspath(__file__))}/out/{getConfig().queryOutFile}" ) with open(fileName, "w") as f: # enable repartition joins due to https://github.com/citusdata/citus/issues/6865 enableRepartitionJoinCommand = "SET citus.enable_repartition_joins TO on;\n" queryLines = [enableRepartitionJoinCommand] queryId = 1 for _ in range(queryCount): query = newQuery() queryLine = "-- queryId: " + str(queryId) + "\n" queryLine += query + "\n\n" queryLines.append(queryLine) queryId += 1 f.writelines(queryLines) if __name__ == "__main__": signal.signal(signal.SIGINT, _signal_handler) arguments = docopt(__doc__, version="generate_queries") seed = -1 if "--seed" in arguments and arguments["--seed"] != "": seed = int(arguments["--seed"]) else: seed = currentMilliSecs() assert seed > 0 random.seed(seed) print(f"---SEED: {seed} ---") resetConfig() ddls = getTableDDLs() data = getTableData() if getConfig().interactiveMode: _interactiveMode(ddls, data) else: _fileMode(ddls, data) ================================================ FILE: src/test/regress/citus_tests/query_generator/node_defs.py ================================================ from enum import Enum class JoinType(Enum): INNER = 1 LEFT = 2 RIGHT = 3 FULL = 4 class RTEType(Enum): RELATION = 1 SUBQUERY = 2 CTE = 3 VALUES = 4 class RestrictOp(Enum): LT = 1 GT = 2 EQ = 3 class CitusType(Enum): DISTRIBUTED = 1 REFERENCE = 2 POSTGRES = 3 class Table: def __init__( self, name, citusType, maxAllowedUseOnQuery, rowCount, nullRate, duplicateRate, columns, distinctCopyCount, ): self.name = name self.citusType = citusType self.maxAllowedUseOnQuery = maxAllowedUseOnQuery self.rowCount = rowCount self.nullRate = nullRate self.duplicateRate = duplicateRate self.columns = columns self.distinctCopyCount = distinctCopyCount class Column: def __init__(self, name, type): self.name = name self.type = type ================================================ FILE: src/test/regress/citus_tests/query_generator/out/.gitignore ================================================ * !.gitignore ================================================ FILE: src/test/regress/citus_tests/query_generator/query_gen.py ================================================ import random from node_defs import RTEType from random_selections import ( randomJoinOp, randomRestrictOp, randomRteType, shouldSelectThatBranch, ) from config.config import getAllTableNames, getConfig, getMaxAllowedCountForTable # query_gen.py generates a new query from grammar rules below. It randomly chooses allowed rules # to generate a query. Here are some important notes about the query generation: # # - We enforce the max allowed # of usage for each table. It also enforces global total # of tables. # - Table names, restriction types and any other selections are chosen between the values provided by # configuration file (config.yml). # - Entry point for the generator is newQuery and all other methods are internal methods. We pass a context # object named GeneratorContext to all internal methods as parameter to perform checks and generations # on the query via the context. # - shouldSelectThatBranch() is useful utility method to randomly chose a grammar rule. Some of the rules # are only selected if we allowed them in configuration file. # - We enforce table limits separately if we are inside cte part of the query(see targetCteRteCount). # We also enforce max # of ctes for a query. # - commonColName from the config is used at select and where clauses. # - useAvgAtTopLevelTarget is useful to return single row as query result. It is also useful to track Citus # query bugs via run_query_compare_test.py. # - '=' restriction is removed from the config by default to return values different than null most of the time. # - 'RTE.VALUES' is also removed from the config for the same reason as above. # - Filter range is selected as same with data range for the same reason as above. # - aliasStack at GeneratorContext is useful to put correct table names into where clause. # grammar syntax # # ======Assumptions====== # 1. Tables has common dist col # 2. All operations execute on common column for all tables. # # TODO: RTE_FUNCTION, RTE_TABLEFUNC # # ====SYNTAX==== # ===Nonterminals=== # Query # SelectExpr # FromExpr # RestrictExpr # RteList # Rte # SubqueryRte # RelationRte # JoinList # JoinOp # Using # RestrictList # Restrict # CteRte # CteList # Cte # ValuesRte # Limit # OrderBy # # ===Terminals=== # e 'SELECT' 'FROM' 'INNER JOIN' 'LEFT JOIN' 'RIGHT JOIN' 'FULL JOIN' 'WHERE' 'LIMIT' 'USING' 'WITH' # 'ORDER BY' 'VALUES' 'IN' 'NOT' 'AS' '<' '>' '=' '*' ',' ';' '(' ')' # # ===Rules=== # Start -> Query ';' || 'WITH' CteList Query ';' # Query -> SelectExpr FromExpr [OrderBy] [Limit] || 'SELECT' 'avg(avgsub.DistColName)' 'FROM' SubqueryRte 'AS avgsub' # SelectExpr -> 'SELECT' 'curAlias()' '.' DistColName # FromExpr -> 'FROM' (Rte JoinList JoinOp Rte Using || RteList) ['WHERE' 'nextRandomAlias()' '.' DistColName RestrictExpr] # RestrictExpr -> ('<' || '>' || '=') Int || ['NOT'] 'IN' SubqueryRte # JoinList -> JoinOp Rte Using JoinList || e # Using -> 'USING' '(' DistColName ')' # RteList -> Rte [, RteList] || Rte # Rte -> SubqueryRte 'AS' 'nextRandomAlias()' || RelationRte 'AS' 'nextRandomAlias()' || # CteRte 'AS' 'nextRandomAlias()' || ValuesRte 'AS' 'nextRandomAlias()' # SubqueryRte -> '(' Query ')' # RelationRte -> 'nextRandomTableName()' # CteRte -> 'randomCteName()' # CteList -> Cte [',' CteList] || Cte # Cte -> 'nextRandomAlias()' 'AS' '(' Query ')' # ValuesRte -> '(' 'VALUES' '(' 'random()' ')' ')' # JoinOp -> 'INNER JOIN' || 'LEFT JOIN' || 'RIGHT JOIN' || 'FULL JOIN' # Limit -> 'LIMIT' 'random()' # OrderBy -> 'ORDER BY' DistColName # DistColName -> 'hardwired(get from config)' class GeneratorContext: """context to store some variables which should be available during generation""" def __init__(self): # each level's last table is used in WHERE clause for the level self.aliasStack = [] # tracks if we are inside cte as we should not refer cte inside cte self.insideCte = False # total rtes in cte + non-cte parts self.totalRteCount = 0 # rte count in non-cte part to enforce non-cte rte limit self.currentRteCount = 0 # cte count to enforce cte limit self.currentCteCount = 0 # rte count in cte part to enforce rte limit in cte self.currentCteRteCount = 0 # rte counts per table to enforce rte limit per table self.perTableRtes = {} # tables which hit count limit self.disallowedTables = set() # useful to track usage avg only at first select self.usedAvg = False def randomCteName(self): """returns a randomly selected cte name""" randCteRef = random.randint(0, self.currentCteCount - 1) return " cte_" + str(randCteRef) def curAlias(self): """returns current alias name to be used for the current table""" return " table_" + str(self.totalRteCount) def curCteAlias(self): """returns current alias name to be used for the current cte""" return " cte_" + str(self.currentCteCount) def hasAnyCte(self): """returns if context has any cte""" return self.currentCteCount > 0 def canGenerateNewRte(self): """checks if context exceeds allowed rte count""" return self.currentRteCount < getConfig().targetRteCount def canGenerateNewCte(self): """checks if context exceeds allowed cte count""" return self.currentCteCount < getConfig().targetCteCount def canGenerateNewRteInsideCte(self): """checks if context exceeds allowed rte count inside a cte""" return self.currentCteRteCount < getConfig().targetCteRteCount def addAlias(self, alias): """adds given alias to the stack""" self.aliasStack.append(alias) def removeLastAlias(self): """removes the latest added alias from the stack""" return self.aliasStack.pop() def getRteNameEnforcingRteLimits(self): """returns rteName by enforcing rte count limits for tables""" # do not enforce per table rte limit if we are inside cte if self.insideCte: rteName = random.choice(getAllTableNames()) return " " + rteName + " " while True: # keep trying to find random table by eliminating the ones which hit table limit allowedNames = set(getAllTableNames()) - self.disallowedTables assert len(allowedNames) > 0 rteName = random.choice(list(allowedNames)) # not yet added to rte count map, so we can allow the name if rteName not in self.perTableRtes: self.perTableRtes[rteName] = 0 break # limit is not exceeded, so we can allow the name if self.perTableRtes[rteName] < getMaxAllowedCountForTable(rteName): break else: self.disallowedTables.add(rteName) # increment rte count for the table name self.perTableRtes[rteName] += 1 return " " + rteName + " " def newQuery(): """returns generated query""" genCtx = GeneratorContext() return _start(genCtx) def _start(genCtx): # Query ';' || 'WITH' CteList Query ';' query = "" if not genCtx.canGenerateNewCte() or shouldSelectThatBranch(): query += _genQuery(genCtx) else: genCtx.insideCte = True query += " WITH " query += _genCteList(genCtx) genCtx.insideCte = False query += _genQuery(genCtx) query += ";" return query def _genQuery(genCtx): # SelectExpr FromExpr [OrderBy] [Limit] || 'SELECT' 'avg(avgsub.DistColName)' 'FROM' SubqueryRte 'AS avgsub' query = "" if ( getConfig().useAvgAtTopLevelTarget and not genCtx.insideCte and not genCtx.usedAvg ): genCtx.usedAvg = True query += "SELECT " query += "count(*), avg(avgsub." + getConfig().commonColName + ") FROM " query += _genSubqueryRte(genCtx) query += " AS avgsub" else: query += _genSelectExpr(genCtx) query += _genFromExpr(genCtx) choseOrderby = False if getConfig().orderby and shouldSelectThatBranch(): query += _genOrderBy(genCtx) choseOrderby = True if getConfig().limit and shouldSelectThatBranch(): if not choseOrderby and getConfig().forceOrderbyWithLimit: query += _genOrderBy(genCtx) query += _genLimit(genCtx) return query def _genOrderBy(genCtx): # 'ORDER BY' DistColName query = "" query += " ORDER BY " query += getConfig().commonColName + " " return query def _genLimit(genCtx): # 'LIMIT' 'random()' query = "" query += " LIMIT " (fromVal, toVal) = getConfig().limitRange query += str(random.randint(fromVal, toVal)) return query def _genSelectExpr(genCtx): # 'SELECT' 'curAlias()' '.' DistColName query = "" query += " SELECT " commonColName = getConfig().commonColName query += genCtx.curAlias() + "." + commonColName + " " return query def _genFromExpr(genCtx): # 'FROM' (Rte JoinList JoinOp Rte Using || RteList) ['WHERE' 'nextRandomAlias()' '.' DistColName RestrictExpr] query = "" query += " FROM " if shouldSelectThatBranch(): query += _genRte(genCtx) query += _genJoinList(genCtx) query += randomJoinOp() query += _genRte(genCtx) query += _genUsing(genCtx) else: query += _genRteList(genCtx) alias = genCtx.removeLastAlias() if shouldSelectThatBranch(): query += " WHERE " query += alias + "." + getConfig().commonColName query += _genRestrictExpr(genCtx) return query def _genRestrictExpr(genCtx): # ('<' || '>' || '=') Int || ['NOT'] 'IN' '(' SubqueryRte ')' query = "" if ( not getConfig().semiAntiJoin or not genCtx.canGenerateNewRte() or shouldSelectThatBranch() ): query += randomRestrictOp() (fromVal, toVal) = getConfig().filterRange query += str(random.randint(fromVal, toVal)) else: if shouldSelectThatBranch(): query += " NOT" query += " IN " query += _genSubqueryRte(genCtx) return query def _genCteList(genCtx): # Cte [',' CteList] || Cte query = "" if shouldSelectThatBranch(): query += _genCte(genCtx) if not genCtx.canGenerateNewCte(): return query query += "," query += _genCteList(genCtx) else: query += _genCte(genCtx) return query def _genCte(genCtx): # 'nextRandomAlias()' 'AS' '(' Query ')' query = "" query += genCtx.curCteAlias() genCtx.currentCteCount += 1 query += " AS " query += " ( " query += _genQuery(genCtx) query += " ) " return query def _genRteList(genCtx): # RteList -> Rte [, RteList] || Rte query = "" if getConfig().cartesianProduct and shouldSelectThatBranch(): query += _genRte(genCtx) if not genCtx.canGenerateNewRte(): return query if genCtx.insideCte and not genCtx.canGenerateNewRteInsideCte(): return query query += "," query += _genRteList(genCtx) else: query += _genRte(genCtx) return query def _genJoinList(genCtx): # JoinOp Rte Using JoinList || e query = "" if shouldSelectThatBranch(): if not genCtx.canGenerateNewRte(): return query if genCtx.insideCte and not genCtx.canGenerateNewRteInsideCte(): return query query += randomJoinOp() query += _genRte(genCtx) query += _genUsing(genCtx) query += _genJoinList(genCtx) return query def _genUsing(genCtx): # 'USING' '(' DistColName ')' query = "" query += " USING (" + getConfig().commonColName + " ) " return query def _genRte(genCtx): # SubqueryRte as 'nextRandomAlias()' || RelationRte as 'curAlias()' || # CteRte as 'curAlias()' || ValuesRte 'AS' 'nextRandomAlias()' alias = genCtx.curAlias() modifiedAlias = None if genCtx.insideCte: genCtx.currentCteRteCount += 1 else: genCtx.currentRteCount += 1 genCtx.totalRteCount += 1 # donot dive into recursive subquery further if we hit into rte limit, replace it with relation rte rteType = randomRteType() if not genCtx.canGenerateNewRte(): rteType = RTEType.RELATION # donot dive into recursive subquery further if we hit into rte in cte limit, replace it with relation rte if genCtx.insideCte and not genCtx.canGenerateNewRteInsideCte(): rteType = RTEType.RELATION # we cannot refer to cte if we are inside it or we donot have any cte if (genCtx.insideCte or not genCtx.hasAnyCte()) and rteType == RTEType.CTE: rteType = RTEType.RELATION query = "" if rteType == RTEType.SUBQUERY: query += _genSubqueryRte(genCtx) elif rteType == RTEType.RELATION: query += _genRelationRte(genCtx) elif rteType == RTEType.CTE: query += _genCteRte(genCtx) elif rteType == RTEType.VALUES: query += _genValuesRte(genCtx) modifiedAlias = alias + "(" + getConfig().commonColName + ") " else: raise BaseException("unknown RTE type") query += " AS " query += alias if not modifiedAlias else modifiedAlias genCtx.addAlias(alias) return query def _genSubqueryRte(genCtx): # '(' Query ')' query = "" query += " ( " query += _genQuery(genCtx) query += " ) " return query def _genRelationRte(genCtx): # 'randomAllowedTableName()' query = "" query += genCtx.getRteNameEnforcingRteLimits() return query def _genCteRte(genCtx): # 'randomCteName()' query = "" query += genCtx.randomCteName() return query def _genValuesRte(genCtx): # '( VALUES(random()) )' query = "" (fromVal, toVal) = getConfig().dataRange query += " ( VALUES(" + str(random.randint(fromVal, toVal)) + " ) ) " return query ================================================ FILE: src/test/regress/citus_tests/query_generator/random_selections.py ================================================ import random import time from node_defs import RestrictOp from config.config import getConfig def shouldSelectThatBranch(): """returns 0 or 1 randomly""" return random.randint(0, 1) def currentMilliSecs(): """returns total milliseconds since epoch""" return round(time.time() * 1000) def randomRteType(): """returns a randomly selected RteType given at config""" rtes = getConfig().targetRteTypes return random.choice(rtes) def randomJoinOp(): """returns a randomly selected JoinOp given at config""" joinTypes = getConfig().targetJoinTypes return " " + random.choice(joinTypes).name + " JOIN" def randomRestrictOp(): """returns a randomly selected RestrictOp given at config""" restrictOps = getConfig().targetRestrictOps restrictOp = random.choice(restrictOps) opText = "" if restrictOp == RestrictOp.LT: opText = "<" elif restrictOp == RestrictOp.GT: opText = ">" elif restrictOp == RestrictOp.EQ: opText = "=" else: raise BaseException("Unknown restrict op") return " " + opText + " " ================================================ FILE: src/test/regress/citus_tests/run_test.py ================================================ #!/usr/bin/env python3 from __future__ import annotations import argparse import os import pathlib import random import re import shutil import sys from collections import OrderedDict from contextlib import contextmanager from typing import Optional import common from common import OLDEST_SUPPORTED_CITUS_VERSION, PG_BINDIR, REGRESS_DIR, capture, run from upgrade import generate_citus_tarball, run_citus_upgrade_tests from config import ARBITRARY_SCHEDULE_NAMES, CitusBaseClusterConfig, CitusUpgradeConfig def main(): args = parse_arguments() test_name = get_test_name(args) # All python tests start with test_ and all other tests don't. This is by # convention. if test_name.startswith("test_"): run_python_test(test_name, args) # above function never returns else: run_regress_test(test_name, args) def parse_arguments(): args = argparse.ArgumentParser() args.add_argument( "test_name", help="Test name (must be included in a schedule.)", nargs="?" ) args.add_argument( "-p", "--path", required=False, help="Relative path for test file (must have a .sql or .spec extension)", type=pathlib.Path, ) args.add_argument( "-r", "--repeat", help="Number of test to run", type=int, default=1 ) args.add_argument( "-b", "--use-base-schedule", required=False, help="Choose base-schedules rather than minimal-schedules", action="store_true", ) args.add_argument( "-w", "--use-whole-schedule-line", required=False, help="Use the whole line found in related schedule", action="store_true", ) args.add_argument( "--valgrind", required=False, help="Run the test with valgrind enabled", action="store_true", ) return vars(args.parse_args()) class TestDeps: schedule: Optional[str] direct_extra_tests: list[str] def __init__( self, schedule, extra_tests=None, repeatable=True, worker_count=2, citus_upgrade_infra=False, ): self.schedule = schedule self.direct_extra_tests = extra_tests or [] self.repeatable = repeatable self.worker_count = worker_count self.citus_upgrade_infra = citus_upgrade_infra def extra_tests(self): all_deps = OrderedDict() for direct_dep in self.direct_extra_tests: if direct_dep in DEPS: for indirect_dep in DEPS[direct_dep].extra_tests(): all_deps[indirect_dep] = True all_deps[direct_dep] = True return list(all_deps.keys()) DEPS = { "multi_cluster_management": TestDeps( None, ["multi_test_helpers_superuser"], repeatable=False ), "minimal_cluster_management": TestDeps( None, ["multi_test_helpers_superuser"], repeatable=False ), "multi_behavioral_analytics_create_table": TestDeps( "minimal_schedule", ["multi_test_helpers_superuser"], repeatable=False ), "create_role_propagation": TestDeps(None, ["multi_cluster_management"]), "single_node_enterprise": TestDeps(None), "multi_add_node_from_backup": TestDeps(None, repeatable=False, worker_count=5), "multi_add_node_from_backup_negative": TestDeps( None, ["multi_add_node_from_backup"], worker_count=5, repeatable=False ), "multi_add_node_from_backup_sync_replica": TestDeps( None, repeatable=False, worker_count=5 ), "single_node": TestDeps(None, ["multi_test_helpers"]), "single_node_truncate": TestDeps(None), "multi_explain": TestDeps( "base_schedule", ["multi_insert_select_non_pushable_queries"] ), "multi_extension": TestDeps(None, repeatable=False), "multi_test_helpers": TestDeps(None), "multi_insert_select": TestDeps("base_schedule"), "multi_partitioning": TestDeps("base_schedule"), "multi_mx_create_table": TestDeps( None, [ "multi_mx_node_metadata", "multi_cluster_management", "multi_mx_function_table_reference", ], ), "alter_distributed_table": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "alter_role_propagation": TestDeps("minimal_schedule"), "background_rebalance": TestDeps( None, ["multi_test_helpers", "multi_cluster_management"], worker_count=3 ), "background_rebalance_parallel": TestDeps( None, ["multi_test_helpers", "multi_cluster_management"], worker_count=6 ), "background_rebalance_parallel_reference_tables": TestDeps( None, ["multi_test_helpers", "multi_cluster_management"], repeatable=False, worker_count=6, ), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), "multi_modifying_xacts": TestDeps("minimal_schedule"), "multi_mx_modifying_xacts": TestDeps(None, ["multi_mx_create_table"]), "multi_mx_router_planner": TestDeps(None, ["multi_mx_create_table"]), "multi_mx_copy_data": TestDeps(None, ["multi_mx_create_table"]), "multi_mx_modifications": TestDeps(None, ["multi_mx_create_table"]), "multi_mx_schema_support": TestDeps(None, ["multi_mx_copy_data"]), "multi_simple_queries": TestDeps("base_schedule"), "create_single_shard_table": TestDeps("minimal_schedule"), "isolation_extension_commands": TestDeps( None, ["isolation_setup", "isolation_add_remove_node"] ), "isolation_update_node": TestDeps( None, ["isolation_setup", "isolation_add_remove_node"] ), "schema_based_sharding": TestDeps("minimal_schedule"), "multi_sequence_default": TestDeps( None, ["multi_test_helpers", "multi_cluster_management", "multi_table_ddl"] ), "grant_on_schema_propagation": TestDeps("minimal_schedule"), "propagate_extension_commands": TestDeps("minimal_schedule"), "multi_size_queries": TestDeps("base_schedule", ["multi_copy"]), "multi_mx_node_metadata": TestDeps( None, ["multi_extension", "multi_test_helpers", "multi_test_helpers_superuser"] ), "multi_mx_function_table_reference": TestDeps( None, ["multi_cluster_management", "remove_coordinator_from_metadata"], # because it queries node group id and it changes as we add / remove nodes repeatable=False, ), "multi_mx_add_coordinator": TestDeps( None, [ "multi_cluster_management", "remove_coordinator_from_metadata", "multi_mx_function_table_reference", ], ), "metadata_sync_helpers": TestDeps( None, ["multi_mx_node_metadata", "multi_cluster_management"] ), "multi_utilities": TestDeps("minimal_schedule", ["multi_data_types"]), "multi_tenant_isolation_nonblocking": TestDeps( "minimal_schedule", ["multi_data_types", "remove_coordinator_from_metadata"] ), "remove_non_default_nodes": TestDeps( None, ["multi_mx_node_metadata", "multi_cluster_management"], repeatable=False ), "citus_split_shard_columnar_partitioned": TestDeps( "minimal_schedule", ["remove_coordinator_from_metadata"] ), "add_coordinator": TestDeps( "minimal_schedule", ["remove_coordinator_from_metadata"], repeatable=False ), "multi_multiuser_auth": TestDeps( "minimal_schedule", ["multi_create_table", "multi_create_users", "multi_multiuser_load_data"], repeatable=False, ), "multi_prepare_plsql": TestDeps("base_schedule"), "pg15": TestDeps("base_schedule"), "foreign_key_to_reference_shard_rebalance": TestDeps( "minimal_schedule", ["remove_coordinator_from_metadata"] ), "limit_intermediate_size": TestDeps("base_schedule"), "columnar_drop": TestDeps( "minimal_columnar_schedule", ["columnar_create", "columnar_load"], repeatable=False, ), "multi_metadata_sync": TestDeps( None, [ "multi_sequence_default", "alter_database_propagation", "alter_role_propagation", "multi_test_catalog_views", "multi_drop_extension", ], repeatable=False, ), "pg17": TestDeps("minimal_schedule", ["multi_behavioral_analytics_create_table"]), "multi_subquery_misc": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "multi_subquery_union": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "multi_subquery_in_where_clause": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "multi_limit_clause_approximate": TestDeps( "minimal_schedule", ["multi_create_table", "multi_create_users", "multi_load_data"], ), "multi_single_relation_subquery": TestDeps( "minimal_schedule", ["multi_create_table", "multi_create_users", "multi_load_data"], ), "multi_subquery_complex_reference_clause": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "multi_subquery_in_where_reference_clause": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "subquery_in_where": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "subquery_in_targetlist": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "window_functions": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "multi_subquery_window_functions": TestDeps( "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), } def run_python_test(test_name, args): """Runs the test using pytest This function never returns as it usese os.execlp to replace the current process with a new pytest process. """ test_path = REGRESS_DIR / "citus_tests" / "test" / f"{test_name}.py" if not test_path.exists(): raise Exception("Test could not be found in any schedule") os.execlp( "pytest", "pytest", "--numprocesses", "auto", "--count", str(args["repeat"]), str(test_path), ) def run_regress_test(test_name, args): original_schedule, schedule_line = find_test_schedule_and_line(test_name, args) print(f"SCHEDULE: {original_schedule}") print(f"SCHEDULE_LINE: {schedule_line}") dependencies = test_dependencies(test_name, original_schedule, schedule_line, args) with tmp_schedule(test_name, dependencies, schedule_line, args) as schedule: if "upgrade" in original_schedule: run_schedule_with_python(test_name, schedule, dependencies) else: run_schedule_with_multiregress(test_name, schedule, dependencies, args) def run_schedule_with_python(test_name, schedule, dependencies): pgxs_path = pathlib.Path(capture("pg_config --pgxs").rstrip()) os.chdir(REGRESS_DIR) os.environ["PATH"] = str(REGRESS_DIR / "bin") + os.pathsep + os.environ["PATH"] os.environ["PG_REGRESS_DIFF_OPTS"] = "-dU10 -w" fake_config_args = { "--pgxsdir": str(pgxs_path.parent.parent.parent), "--bindir": PG_BINDIR, "--mixed": False, } if dependencies.citus_upgrade_infra: run_single_citus_upgrade_test(test_name, schedule, fake_config_args) return config = CitusBaseClusterConfig(fake_config_args) common.initialize_temp_dir(config.temp_dir) common.initialize_citus_cluster( config.bindir, config.datadir, config.settings, config ) common.run_pg_regress( config.bindir, config.pg_srcdir, config.coordinator_port(), schedule ) def run_single_citus_upgrade_test(test_name, schedule, fake_config_args): os.environ["CITUS_OLD_VERSION"] = f"v{OLDEST_SUPPORTED_CITUS_VERSION}" citus_tarball_path = generate_citus_tarball(f"v{OLDEST_SUPPORTED_CITUS_VERSION}") config = CitusUpgradeConfig(fake_config_args, citus_tarball_path, None) # Before tests are a simple case, because no actual upgrade is needed if "_before" in test_name: run_citus_upgrade_tests(config, schedule, None) return before_schedule_name = f"{schedule}_before" before_schedule_path = REGRESS_DIR / before_schedule_name before_test_name = test_name.replace("_after", "_before") with open(before_schedule_path, "w") as before_schedule_file: before_schedule_file.write(f"test: {before_test_name}\n") try: run_citus_upgrade_tests(config, before_schedule_name, schedule) finally: os.remove(before_schedule_path) def run_schedule_with_multiregress(test_name, schedule, dependencies, args): worker_count = needed_worker_count(test_name, dependencies) # find suitable make recipe if dependencies.schedule == "base_isolation_schedule" or test_name.startswith( "isolation" ): make_recipe = "check-isolation-custom-schedule" elif dependencies.schedule == "failure_base_schedule" or test_name.startswith( "failure" ): make_recipe = "check-failure-custom-schedule" elif test_name.startswith("columnar"): if dependencies.schedule is None: # Columanar isolation tests don't depend on a base schedule, # so this must be a columnar isolation test. make_recipe = "check-columnar-isolation-custom-schedule" elif dependencies.schedule == "minimal_columnar_schedule": make_recipe = "check-columnar-custom-schedule" else: raise Exception("Columnar test could not be found in any schedule") else: make_recipe = "check-custom-schedule" if args["valgrind"]: make_recipe += "-vg" # prepare command to run tests test_command = ( f"make -C {REGRESS_DIR} {make_recipe} " f"WORKERCOUNT={worker_count} " f"SCHEDULE='{schedule}'" ) run(test_command) def default_base_schedule(test_schedule, args): if "isolation" in test_schedule: if "columnar" in test_schedule: # we don't have pre-requisites for columnar isolation tests return None return "base_isolation_schedule" if "failure" in test_schedule: return "failure_base_schedule" if "enterprise" in test_schedule: return "enterprise_minimal_schedule" if "split" in test_schedule: return "minimal_schedule" if "mx" in test_schedule: if args["use_base_schedule"]: return "mx_base_schedule" return "mx_minimal_schedule" if "operations" in test_schedule: return "minimal_schedule" if "pg_upgrade" in test_schedule: return "minimal_pg_upgrade_schedule" if "columnar" in test_schedule: return "minimal_columnar_schedule" if test_schedule in ARBITRARY_SCHEDULE_NAMES: print(f"WARNING: Arbitrary config schedule ({test_schedule}) is not supported.") sys.exit(0) if args["use_base_schedule"]: return "base_schedule" return "minimal_schedule" # we run the tests with 2 workers by default. # If we find any dependency which requires more workers, we update the worker count. def worker_count_for(test_name): if test_name in DEPS: return DEPS[test_name].worker_count return 2 def get_test_name(args): if args["test_name"]: return args["test_name"] if not args["path"]: print("FATAL: No test given.") sys.exit(2) absolute_test_path = os.path.join(os.getcwd(), args["path"]) if not os.path.isfile(absolute_test_path): print(f"ERROR: test file '{absolute_test_path}' does not exist") sys.exit(2) if pathlib.Path(absolute_test_path).suffix not in (".spec", ".sql", ".py"): print( "ERROR: Unrecognized test extension. Valid extensions are: .sql, .spec, and .py" ) sys.exit(1) return pathlib.Path(absolute_test_path).stem def find_test_schedule_and_line(test_name, args): for schedule_file_path in sorted(REGRESS_DIR.glob("*_schedule")): for schedule_line in open(schedule_file_path, "r"): if re.search(r"^test:.*\b" + test_name + r"\b", schedule_line): test_schedule = pathlib.Path(schedule_file_path).stem if args["use_whole_schedule_line"]: return test_schedule, schedule_line return test_schedule, f"test: {test_name}\n" raise Exception("Test could not be found in any schedule") def test_dependencies(test_name, test_schedule, schedule_line, args): if test_name in DEPS: return DEPS[test_name] if "citus_upgrade" in test_schedule: return TestDeps(None, citus_upgrade_infra=True) if schedule_line_is_upgrade_after(schedule_line): # upgrade_xxx_after tests always depend on upgrade_xxx_before test_names = schedule_line.split()[1:] before_tests = [] # _after tests have implicit dependencies on _before tests for test_name in test_names: if "_after" in test_name: before_tests.append(test_name.replace("_after", "_before")) # the upgrade_columnar_before renames the schema, on which other # "after" tests depend. So we make sure to execute it always. if "upgrade_columnar_before" not in before_tests: before_tests.append("upgrade_columnar_before") return TestDeps(default_base_schedule(test_schedule, args), before_tests) # before_ tests leave stuff around on purpose for the after tests. So they # are not repeatable by definition. if "before_" in test_schedule: repeatable = False else: repeatable = True return TestDeps(default_base_schedule(test_schedule, args), repeatable=repeatable) # Returns true if given test_schedule_line is of the form: # "test: upgrade_ ... _after .." def schedule_line_is_upgrade_after(test_schedule_line: str) -> bool: return ( test_schedule_line.startswith("test: upgrade_") and "_after" in test_schedule_line ) @contextmanager def tmp_schedule(test_name, dependencies, schedule_line, args): tmp_schedule_path = REGRESS_DIR / f"tmp_schedule_{random.randint(1, 10000)}" # Prefill the temporary schedule with the base schedule that this test # depends on. Some tests don't need a base schedule to run though, # e.g tests that are in the first place in their own schedule if dependencies.schedule: shutil.copy2(REGRESS_DIR / dependencies.schedule, tmp_schedule_path) with open(tmp_schedule_path, "a") as myfile: # Add any specific dependencies for dependency in dependencies.extra_tests(): myfile.write(f"test: {dependency}\n") repetition_cnt = args["repeat"] if repetition_cnt > 1 and not dependencies.repeatable: repetition_cnt = 1 print(f"WARNING: Cannot repeatably run this test: '{test_name}'") for _ in range(repetition_cnt): myfile.write(schedule_line) try: yield tmp_schedule_path.stem finally: os.remove(tmp_schedule_path) def needed_worker_count(test_name, dependencies): worker_count = worker_count_for(test_name) for dependency in dependencies.extra_tests(): worker_count = max(worker_count_for(dependency), worker_count) return worker_count if __name__ == "__main__": main() ================================================ FILE: src/test/regress/citus_tests/test/README.md ================================================ # Pytest based tests ## Usage Run all tests in parallel: ```bash pytest -n auto ``` Run all tests sequentially: ```bash pytest ``` Run a specific test: ```bash pytest test/test_columnar.py::test_recovery ``` Run a specific test file in parallel: ```bash pytest -n auto test/test_columnar.py ``` Run any test that contains a certain string in the name: ```bash pytest -k recovery ``` Run tests without it capturing stdout/stderr. This can be useful to see the logs of a passing test: ```bash pytest -s test/test_columnar.py::test_recovery ``` ## General info Our other tests work by comparing output of a sequence of SQL commands that's executed by `psql` to an expected output. If there's a difference between the expected and actual output, then the tests fails. This works fine for many cases, but certain types of tests are hard to write and a lot of care usually has to be taken to make sure output is completely identical in every run. The tests in this directory use a different approach and use [`pytest`][pytest-docs] to run tests that are written in the Python programming language. This idea is similar to TAP tests that are part of Postgres, with the important difference that those are written in Perl. In the sections below you can find most stuff you'll need to know about `pytest` to run and write such tests, but if you want more detailed info some useful references are: - [A blog with pytest tips and tricks][pytest-tips] - [The official pytest docs][pytest-docs] [pytest-docs]: https://docs.pytest.org/en/stable/ [pytest-tips]: https://pythontest.com/pytest-tips-tricks/ ## Adding a new test Tests are automatically discovered by `pytest` using a simple but effective heuristic. In this directory (`src/test/regress/citus_tests/test`) it finds all of the files that are named `test_{some name}.py`. Those files are then searched for function names starting with the `test_` prefix. All those functions are considered tests by `pytest`. ### Fixtures aka Dependency Injection aka Teardown/Cleanup An important part of tests is that they have some dependencies. The most important dependency for us is usually a running Citus cluster. These dependencies are provided by what `pytest` calls [fixtures]. Fixtures are functions that `yield` a value. Anything before the `yield` is done during setup and anything after the yield is done during teardown of the test (or whole session). All our fixtures are defined in `conftest.py`. Using a fixture in a test is very easy, but looks like a lot of magic. All you have to do is make sure your test function has an argument with the same name as the name of the fixture. For example: ```python def test_some_query(cluster): cluster.coordinator.sql("SELECT 1") assert cluster.workers[0].sql_value('SELECT 2') == 2 ``` If you need a cluster of a specific size you can use the `cluster_factory` fixture: ```python def test_with_100_workers(cluster_factory): cluster = cluster_factory(100) ``` If you want more details on how fixtures work a few useful pages of the pytest docs are: - [About fixtures][fixtures] - [How to use fixtures][fixtures-how-to] - [Fixtures reference][fixtures-reference] [fixtures]: https://docs.pytest.org/en/stable/explanation/fixtures.html [fixtures-how-to]: https://docs.pytest.org/en/stable/how-to/fixtures.html [fixtures-reference]: https://docs.pytest.org/en/stable/reference/fixtures.html ## Connecting to a test postgres Sometimes your test is failing in an unexpected way and the easiest way to find out why is to connect to Postgres at a certain point interactively. ### Using `psql_debug` The easiest way is to use the `psql_debug()` method of your `Cluster` or `Postgres` instance. ```python def test_something(cluster): # working stuff cluster.coordinator.psql_debug() # unexpectedly failing test ``` Then run this test with stdout/stderr capturing disabled (`-s`) and it will show you an interactive `psql` prompt right at that point in the test: ```bash $ pytest -s test/test_your_thing.py::test_something ... psql (15.2) SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off) Type "help" for help. 127.0.0.1 postgres@postgres:10201-20016= > select 1; ``` ### Debug manually Sometimes you need to connect to more than one node though. For that you can use a `Cluster` its `debug` method instead. ```python def test_something(cluster): # working stuff cluster.debug() # unexpectedly failing test ``` Then run this test with stdout/stderr capturing disabled (`-s`) and it will show you the connection string for each of the nodes in the cluster: ```bash $ PG_FORCE_PORTS=true pytest -s test/test_your_thing.py::test_something ... The nodes in this cluster and their connection strings are: /tmp/pytest-of-jelte/pytest-752/cluster2-0/coordinator: "host=127.0.0.1 port=10202 dbname=postgres user=postgres options='-c search_path=test_recovery' connect_timeout=3 client_encoding=UTF8" /tmp/pytest-of-jelte/pytest-752/cluster2-0/worker0: "host=127.0.0.1 port=10203 dbname=postgres user=postgres options='-c search_path=test_recovery' connect_timeout=3 client_encoding=UTF8" /tmp/pytest-of-jelte/pytest-752/cluster2-0/worker1: "host=127.0.0.1 port=10204 dbname=postgres user=postgres options='-c search_path=test_recovery' connect_timeout=3 client_encoding=UTF8" Press Enter to continue running the test... ``` Then in another terminal you can manually connect to as many of them as you want. Using `PG_FORCE_PORTS` is recommended here, to make sure that the ports will stay the same across runs of the tests. That way you can reuse the connection strings that you got from a previous run, if you need to debug again. ================================================ FILE: src/test/regress/citus_tests/test/__init__.py ================================================ ================================================ FILE: src/test/regress/citus_tests/test/conftest.py ================================================ import pytest from common import CitusCluster, Postgres, cleanup_test_leftovers, parallel_run @pytest.fixture(scope="session") def cluster_factory_session(tmp_path_factory): """The session level pytest fixture that creates and caches citus clusters IMPORTANT: This should not be used directly, but only indirectly through the cluster_factory fixture. """ clusters = {} def _make_or_get_cluster(worker_count: int): if worker_count not in clusters: clusters[worker_count] = CitusCluster( tmp_path_factory.mktemp(f"cluster{worker_count}-"), worker_count ) return clusters[worker_count] yield _make_or_get_cluster parallel_run(CitusCluster.cleanup, clusters.values()) @pytest.fixture def cluster_factory(cluster_factory_session, request): """The pytest fixture that creates and caches citus clusters When the function provided by the factory is called, it returns a cluster with the given worker count. This cluster is cached across tests, so that future invocations with the same worker count don't need to create a cluster again, but can reuse the previously created one. To try and make sure that tests don't depend on eachother this tries very hard to clean up anything that is created during the test. It also prints the Postgres logs that were produced during the test to stdout. Normally these will be hidden, but when a test fails pytest will show all stdout output produced during the test. Thus showing the Postgres logs in that case makes it easier to debug. """ log_handles = [] clusters = [] nodes = [] def _make_or_get_cluster(worker_count: int): nonlocal log_handles nonlocal nodes cluster = cluster_factory_session(worker_count) if cluster.failed_reset: cluster.reset() cluster.failed_reset = False clusters.append(cluster) log_handles += [(node, node.log_handle()) for node in cluster.nodes] nodes += cluster.nodes # Create a dedicated schema for the test and use it by default cluster.coordinator.create_schema(request.node.originalname) cluster.schema = request.node.originalname return cluster yield _make_or_get_cluster try: # We clean up test leftovers on all nodes together, instead of per # cluster. The reason for this is that some subscriptions/publication # pairs might be between different clusters. And by cleaning them up # all together, the ordering of the DROPs is easy to make correct. cleanup_test_leftovers(nodes) parallel_run(Postgres.prepare_reset, nodes) parallel_run(Postgres.restart, nodes) except Exception: for cluster in clusters: cluster.failed_reset = True raise finally: for node, log in log_handles: print(f"\n\nPG_LOG: {node.pgdata}\n") print(log.read()) log.close() @pytest.fixture(name="coord") def coordinator(cluster_factory): """Sets up a clean single-node Citus cluster for this test""" yield cluster_factory(0).coordinator @pytest.fixture def cluster(cluster_factory): """Sets up a clean 2-worker Citus cluste for this test""" yield cluster_factory(2) ================================================ FILE: src/test/regress/citus_tests/test/test_columnar.py ================================================ import psycopg import pytest def test_freezing(coord): coord.sql("CREATE EXTENSION IF NOT EXISTS citus_columnar") coord.configure("vacuum_freeze_min_age = 50000", "vacuum_freeze_table_age = 50000") coord.restart() # create columnar table and insert simple data to verify the data survives # a crash coord.sql("CREATE TABLE test_row(i int)") coord.sql("INSERT INTO test_row VALUES (1) ") coord.sql( "CREATE TABLE test_columnar_freeze(i int) USING columnar WITH(autovacuum_enabled=false)" ) coord.sql("INSERT INTO test_columnar_freeze VALUES (1)") for _ in range(0, 7): with coord.cur() as cur: for _ in range(0, 10_000): cur.execute("UPDATE test_row SET i = i + 1") frozen_age = coord.sql_value( """ select age(relfrozenxid) from pg_class where relname='test_columnar_freeze'; """ ) assert frozen_age > 70_000, "columnar table was frozen" coord.sql("VACUUM FREEZE test_columnar_freeze") frozen_age = coord.sql_value( """ select age(relfrozenxid) from pg_class where relname='test_columnar_freeze'; """ ) assert frozen_age < 70_000, "columnar table was not frozen" coord.sql("DROP EXTENSION citus_columnar CASCADE") def test_recovery(coord): coord.sql("CREATE EXTENSION IF NOT EXISTS citus_columnar") # create columnar table and insert simple data to verify the data survives a crash coord.sql("CREATE TABLE t1 (a int, b text) USING columnar") coord.sql( "INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1002) AS a" ) # simulate crash coord.stop("immediate") coord.start() row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 1002, "columnar didn't recover data before crash correctly" # truncate the table to verify the truncation survives a crash coord.sql("TRUNCATE t1") # simulate crash coord.stop("immediate") coord.start() row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 0, "columnar didn't recover the truncate correctly" # test crashing while having an open transaction with pytest.raises( psycopg.OperationalError, match=( "server closed the connection unexpectedly|" "consuming input failed: EOF detected|" "SSL SYSCALL error: EOF detected|" "SSL error: unexpected eof while reading" ), ): with coord.transaction() as cur: cur.execute( "INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1003) AS a" ) # simulate crash coord.stop("immediate") coord.start() row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 0, "columnar didn't recover uncommited transaction" # test crashing while having a prepared transaction with pytest.raises( psycopg.OperationalError, match=( "server closed the connection unexpectedly|" "consuming input failed: EOF detected|" "consuming input failed: SSL SYSCALL error: EOF detected|" "SSL error: unexpected eof while reading" ), ): with coord.transaction() as cur: cur.execute( "INSERT INTO t1 SELECT a, 'hello world ' || a FROM generate_series(1,1004) AS a" ) cur.execute("PREPARE TRANSACTION 'prepared_xact_crash'") # simulate crash coord.stop("immediate") coord.start() row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 0, "columnar didn't recover uncommitted prepared transaction" coord.sql("COMMIT PREPARED 'prepared_xact_crash'") row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 1004, "columnar didn't recover committed transaction" # test crash recovery with copied data with coord.cur() as cur: with cur.copy("COPY t1 FROM STDIN") as copy: copy.write_row((1, "a")) copy.write_row((2, "b")) copy.write_row((3, "c")) # simulate crash coord.stop("immediate") coord.start() row_count = coord.sql_value("SELECT count(*) FROM t1") assert row_count == 1007, "columnar didn't recover after copy" coord.sql("DROP EXTENSION citus_columnar CASCADE") ================================================ FILE: src/test/regress/citus_tests/test/test_extension.py ================================================ import psycopg import pytest def test_create_drop_citus(coord): with coord.cur() as cur1: with coord.cur() as cur2: # Conn1 drops the extension # and Conn2 cannot use it. cur1.execute("DROP EXTENSION citus") with pytest.raises(psycopg.errors.UndefinedFunction): # Conn1 dropped the extension. citus_version udf # cannot be found.sycopg.errors.UndefinedFunction # is expected here. cur2.execute("SELECT citus_version();") # Conn2 creates the extension, # Conn1 is able to use it immediadtely. cur2.execute("CREATE EXTENSION citus") cur1.execute("SELECT citus_version();") cur1.execute("DROP EXTENSION citus;") with coord.cur() as cur1: with coord.cur() as cur2: # A connection is able to create and use the extension # within a transaction block. cur1.execute("BEGIN;") cur1.execute("CREATE TABLE t1(id int);") cur1.execute("CREATE EXTENSION citus;") cur1.execute("SELECT create_reference_table('t1')") cur1.execute("ABORT;") # Conn1 aborted so Conn2 is be able to create and # use the extension within a transaction block. cur2.execute("BEGIN;") cur2.execute("CREATE TABLE t1(id int);") cur2.execute("CREATE EXTENSION citus;") cur2.execute("SELECT create_reference_table('t1')") cur2.execute("COMMIT;") # Conn2 commited so Conn1 is be able to use the # extension immediately. cur1.execute("SELECT citus_version();") ================================================ FILE: src/test/regress/citus_tests/test/test_prepared_statements.py ================================================ def test_call_param(cluster): # create a distributed table and an associated distributed procedure # to ensure parameterized CALL succeed, even when the param is the # distribution key. coord = cluster.coordinator coord.sql("CREATE TABLE test(i int)") coord.sql( """ CREATE PROCEDURE p(_i INT) LANGUAGE plpgsql AS $$ BEGIN INSERT INTO test(i) VALUES (_i); END; $$ """ ) sql = "CALL p(%s)" # prepare/exec before distributing coord.sql_prepared(sql, (1,)) coord.sql("SELECT create_distributed_table('test', 'i')") coord.sql( "SELECT create_distributed_function('p(int)', distribution_arg_name := '_i', colocate_with := 'test')" ) # prepare/exec after distribution coord.sql_prepared(sql, (2,)) sum_i = coord.sql_value("select sum(i) from test;") assert sum_i == 3 ================================================ FILE: src/test/regress/citus_tests/upgrade/README.md ================================================ # Upgrade Tests ## Postgres Upgrade Test Postgres upgrade test is used for testing postgres version upgrade with citus installed. Before running the script, make sure that: - You have downloaded citus. - You have two different postgres versions. - Citus is installed to both of the postgres versions. For each postgres version: - In citus source directory run: ```bash make clean ./configure PG_CONFIG= PG_CONFIG= make sudo PG_CONFIG= make install ``` Make sure you do this for both postgres versions, pg_config should be different for each postgres version. - Install `pipenv` and run in `citus/src/test/regress`: ```bash pipenv install pipenv shell ``` - Finally run upgrade test in `citus/src/test/regress`: ```bash pipenv run make check-pg-upgrade old-bindir= new-bindir= test-with-columnar= ``` When test-with-columnar is provided as true, before_pg_upgrade_with_columnar_schedule / after_pg_upgrade_with_columnar_schedule is used before / after upgrading Postgres during the tests and before_pg_upgrade_without_columnar_schedule / after_pg_upgrade_without_columnar_schedule is used otherwise. To see full command list: ```bash pipenv run citus_tests/upgrade/pg_upgrade_test.py -help ``` How the postgres upgrade test works: - Temporary folder `tmp_upgrade` is created in `src/test/regress/`, if one exists it is removed first. - Database is initialized and citus cluster is created(1 coordinator + 2 workers) with old postgres. - `before_pg_upgrade_with_columnar_schedule` / `before_pg_upgrade_without_columnar_schedule` is run with `pg_regress`. This schedule sets up any objects and data that will be tested for preservation after the upgrade. It - `after_pg_upgrade_with_columnar_schedule` / `after_pg_upgrade_without_columnar_schedule` is run with `pg_regress` to verify that the output of those tests is the same before the upgrade as after. - `citus_prepare_pg_upgrade` is run in coordinators and workers. - Old database is stopped. - A new database is initialized with new postgres under `tmp_upgrade`. - Postgres upgrade is performed. - New database is started in both coordinators and workers. - `citus_finish_pg_upgrade` is run in coordinators and workers to finalize the upgrade step. - `after_pg_upgrade_with_columnar_schedule` / `after_pg_upgrade_without_columnar_schedule` is run with `pg_regress` to verify that the previously created tables, and data still exist. Router and realtime queries are used to verify this. ### Writing new PG upgrade tests The main important thing is that we have `upgrade_{name}_before` and `upgrade_{name}_after` tests. The `before` files are used to setup any objects and data before the upgrade. The `after` tests shouldn't have any side effects since they are run twice (once before and once after the upgrade). Furthermore, anything that is basic Citus functionality should go in the `upgrade_basic_before`/`upgrade_basic_after` tests. This test file is used during PG upgrades and Citus upgrades. Any features that don't work in old Citus versions should thus be added to their own file, because that file will then only be run during PG versions. ## Citus Upgrade Test Citus upgrade test is used for testing citus version upgrades from specific version to master. The purpose of this test is to ensure that a newly made change does not result in unexpected upgrade errors. Currently the citus upgrade test assumes that: - You have citus artifact tarballs, both for old version and master. How the citus upgrade test work: - The script takes `citus-pre-tar` and `citus-post-tar` which should contain citus artifacts. - It installs the given citus version from `citus-pre-tar`. - It creates a citus cluster(1 coordinator 2 workers). - It reports the initial versions. - It installs the checked out citus version from `citus-post-tar`. - It restarts the database and runs `ALTER EXTENSION citus UPGRADE`. - It runs `after_citus_upgrade` schedule to verify that the upgrade is successful. - It stops the cluster. Note that when the version of citus changes, we should update `MASTER_VERSION` with the new version of citus otherwise that will be outdated and it will fail. There is a target for citus upgrade test. We run citus upgrade tests both in normal mode and in mixed mode. In mixed mode, we don't upgrade one of the workers. `'citus.enable_version_checks' : 'false'` is used to prevent citus from giving an error for mixed mode. To see full command list: ```bash pipenv run upgrade/citus_upgrade_test.py -help ``` In order to run citus upgrade tests locally you can use: ```bash pipenv run make check-citus-upgrade-local citus-old-version=v8.0.0 ``` For mixed mode: ```bash pipenv run make check-citus-upgrade-mixed-local citus-old-version=v8.0.0 ``` ================================================ FILE: src/test/regress/citus_tests/upgrade/__init__.py ================================================ from .citus_upgrade_test import generate_citus_tarball, run_citus_upgrade_tests # noqa ================================================ FILE: src/test/regress/citus_tests/upgrade/citus_upgrade_test.py ================================================ #!/usr/bin/env python3 """citus_upgrade_test Usage: citus_upgrade_test [options] --bindir= --pgxsdir= [--citus-old-version=] Options: --bindir= The PostgreSQL executable directory(ex: '~/.pgenv/pgsql-11.3/bin') --citus-pre-tar= Tarball with the citus artifacts to use as the base version to upgrade from --citus-post-tar= Tarball with the citus artifacts to use as the new version to upgrade to --pgxsdir= Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3) --citus-old-version= Citus old version for local run(ex v8.0.0) --mixed Run the verification phase with one node not upgraded. --minor-upgrade Use minor version upgrade test schedules instead of major version schedules. """ import multiprocessing import os import re import subprocess import sys from docopt import docopt # https://stackoverflow.com/questions/14132789/relative-imports-for-the-billionth-time/14132912#14132912 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ignore E402 because these imports require addition to path import common # noqa: E402 import utils # noqa: E402 from common import CI, PG_MAJOR_VERSION, REPO_ROOT, run # noqa: E402 from utils import USER # noqa: E402 from config import ( # noqa: E402 AFTER_CITUS_UPGRADE_COORD_SCHEDULE, BEFORE_CITUS_UPGRADE_COORD_SCHEDULE, CITUS_VERSION_SQL, MASTER_VERSION, MIXED_AFTER_CITUS_UPGRADE_SCHEDULE, MIXED_BEFORE_CITUS_UPGRADE_SCHEDULE, CitusUpgradeConfig, ) def main(config): before_upgrade_schedule = get_before_upgrade_schedule(config.mixed_mode) after_upgrade_schedule = get_after_upgrade_schedule(config.mixed_mode) run_citus_upgrade_tests(config, before_upgrade_schedule, after_upgrade_schedule) def run_citus_upgrade_tests(config, before_upgrade_schedule, after_upgrade_schedule): install_citus(config.pre_tar_path) common.initialize_temp_dir(config.temp_dir) common.initialize_citus_cluster( config.bindir, config.datadir, config.settings, config ) report_initial_version(config) # Store the pre-upgrade GUCs and UDFs for minor version upgrades pre_upgrade = None if config.minor_upgrade: pre_upgrade = get_citus_catalog_info(config) run_test_on_coordinator(config, before_upgrade_schedule) remove_citus(config.pre_tar_path) if after_upgrade_schedule is None: return install_citus(config.post_tar_path) restart_databases(config.bindir, config.datadir, config.mixed_mode, config) run_alter_citus(config.bindir, config.mixed_mode, config) verify_upgrade(config, config.mixed_mode, config.node_name_to_ports.values()) # For minor version upgrades, verify GUCs and UDFs does not have breaking changes breaking_changes = [] if config.minor_upgrade: breaking_changes = compare_citus_catalog_info(config, pre_upgrade) run_test_on_coordinator(config, after_upgrade_schedule) remove_citus(config.post_tar_path) # Fail the test if there are any breaking changes if breaking_changes: common.eprint("\n=== BREAKING CHANGES DETECTED ===") for change in breaking_changes: common.eprint(f" - {change}") common.eprint("==================================\n") sys.exit(1) def get_citus_catalog_info(config): results = {} # Store GUCs guc_results = utils.psql_capture( config.bindir, config.coordinator_port(), "SELECT name, boot_val FROM pg_settings WHERE name LIKE 'citus.%' ORDER BY name;", ) guc_lines = guc_results.decode("utf-8").strip().split("\n") results["gucs"] = {} for line in guc_lines[2:]: # Skip header lines name, boot_val = line.split("|") results["gucs"][name.strip()] = boot_val.strip() # Store UDFs udf_results = utils.psql_capture( config.bindir, config.coordinator_port(), """ SELECT n.nspname AS schema_name, p.proname AS function_name, pg_get_function_arguments(p.oid) AS full_args, pg_get_function_result(p.oid) AS return_type FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_depend d ON d.objid = p.oid JOIN pg_extension e ON e.oid = d.refobjid WHERE e.extname = 'citus' AND d.deptype = 'e' ORDER BY schema_name, function_name, full_args; """, ) udf_lines = udf_results.decode("utf-8").strip().split("\n") results["udfs"] = {} for line in udf_lines[2:]: # Skip header lines schema_name, function_name, full_args, return_type = line.split("|") key = (schema_name.strip(), function_name.strip()) signature = (full_args.strip(), return_type.strip()) if key not in results["udfs"]: results["udfs"][key] = set() results["udfs"][key].add(signature) # Store types, exclude composite types (t.typrelid = 0) and # exclude auto-created array types # (t.typname LIKE '\_%' AND t.typelem <> 0) type_results = utils.psql_capture( config.bindir, config.coordinator_port(), """ SELECT n.nspname, t.typname, t.typtype FROM pg_type t JOIN pg_depend d ON d.objid = t.oid JOIN pg_extension e ON e.oid = d.refobjid JOIN pg_namespace n ON n.oid = t.typnamespace WHERE e.extname = 'citus' AND t.typrelid = 0 AND NOT (t.typname LIKE '\\_%%' AND t.typelem <> 0) ORDER BY n.nspname, t.typname; """, ) type_lines = type_results.decode("utf-8").strip().split("\n") results["types"] = {} for line in type_lines[2:]: # Skip header lines nspname, typname, typtype = line.split("|") key = (nspname.strip(), typname.strip()) results["types"][key] = typtype.strip() # Store tables and views table_results = utils.psql_capture( config.bindir, config.coordinator_port(), """ SELECT n.nspname, c.relname, a.attname, t.typname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_attribute a ON a.attrelid = c.oid JOIN pg_type t ON t.oid = a.atttypid JOIN pg_depend d ON d.objid = c.oid JOIN pg_extension e ON e.oid = d.refobjid WHERE e.extname = 'citus' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY n.nspname, c.relname, a.attname; """, ) table_lines = table_results.decode("utf-8").strip().split("\n") results["tables"] = {} for line in table_lines[2:]: # Skip header lines nspname, relname, attname, typname = line.split("|") key = (nspname.strip(), relname.strip()) if key not in results["tables"]: results["tables"][key] = {} results["tables"][key][attname.strip()] = typname.strip() return results def compare_citus_catalog_info(config, pre_upgrade): post_upgrade = get_citus_catalog_info(config) breaking_changes = [] # Compare GUCs for name, boot_val in pre_upgrade["gucs"].items(): if name not in post_upgrade["gucs"]: breaking_changes.append(f"GUC {name} was removed") elif post_upgrade["gucs"][name] != boot_val and name != "citus.version": breaking_changes.append( f"The default value of GUC {name} was changed from {boot_val} to {post_upgrade['gucs'][name]}" ) # Compare UDFs - check if any pre-upgrade signatures were removed for (schema_name, function_name), pre_signatures in pre_upgrade["udfs"].items(): if (schema_name, function_name) not in post_upgrade["udfs"]: breaking_changes.append( f"UDF {schema_name}.{function_name} was completely removed" ) else: post_signatures = post_upgrade["udfs"][(schema_name, function_name)] removed_signatures = pre_signatures - post_signatures if removed_signatures: for full_args, return_type in removed_signatures: if not find_compatible_udf_signature( full_args, return_type, post_signatures ): breaking_changes.append( f"UDF signature removed: {schema_name}.{function_name}({full_args}) RETURNS {return_type}" ) # Compare Types - check if any pre-upgrade types were removed or changed for (nspname, typname), typtype in pre_upgrade["types"].items(): if (nspname, typname) not in post_upgrade["types"]: breaking_changes.append(f"Type {nspname}.{typname} was removed") elif post_upgrade["types"][(nspname, typname)] != typtype: breaking_changes.append( f"Type {nspname}.{typname} changed type from {typtype} to {post_upgrade['types'][(nspname, typname)]}" ) # Compare tables / views - check if any pre-upgrade tables or columns were removed or changed for (nspname, relname), columns in pre_upgrade["tables"].items(): if (nspname, relname) not in post_upgrade["tables"]: breaking_changes.append(f"Table/view {nspname}.{relname} was removed") else: post_columns = post_upgrade["tables"][(nspname, relname)] for col_name, col_type in columns.items(): if col_name not in post_columns: breaking_changes.append( f"Column {col_name} in table/view {nspname}.{relname} was removed" ) elif post_columns[col_name] != col_type: breaking_changes.append( f"Column {col_name} in table/view {nspname}.{relname} changed type from {col_type} to {post_columns[col_name]}" ) return breaking_changes def find_compatible_udf_signature(full_args, return_type, post_signatures): pre_args_list = [arg.strip() for arg in full_args.split(",") if arg.strip()] for post_full_args, post_return_type in post_signatures: if post_return_type == return_type: post_args_list = [ arg.strip() for arg in post_full_args.split(",") if arg.strip() ] """ Here check if the function signatures are compatible, they are compatible if: post_args_list has all the arguments of pre_args_list in the same order, but can have additional arguments with default values """ pre_index = 0 post_index = 0 compatible = True while pre_index < len(pre_args_list) and post_index < len(post_args_list): if pre_args_list[pre_index] == post_args_list[post_index]: pre_index += 1 else: # Check if the argument in post_args_list has a default value if "default" not in post_args_list[post_index].lower(): compatible = False break post_index += 1 if pre_index < len(pre_args_list): compatible = False continue while post_index < len(post_args_list): if "default" not in post_args_list[post_index].lower(): compatible = False break post_index += 1 if compatible: return True return False def install_citus(tar_path): if tar_path: with utils.cd("/"): run(["tar", "xvf", tar_path], shell=False) else: with utils.cd(REPO_ROOT): run(f"make -j{multiprocessing.cpu_count()} -s install") def report_initial_version(config): for port in config.node_name_to_ports.values(): actual_citus_version = get_actual_citus_version(config.bindir, port) print("port:{} citus version {}".format(port, actual_citus_version)) def get_version_number(version): return re.findall(r"\d+.\d+", version)[0] def get_actual_citus_version(pg_path, port): citus_version = utils.psql_capture(pg_path, port, CITUS_VERSION_SQL) citus_version = citus_version.decode("utf-8") return get_version_number(citus_version) def run_test_on_coordinator(config, schedule): common.run_pg_regress( config.bindir, config.pg_srcdir, config.coordinator_port(), schedule ) def remove_citus(tar_path): if tar_path: with utils.cd("/"): remove_tar_files(tar_path) def remove_tar_files(tar_path): ps = subprocess.Popen(("tar", "tf", tar_path), stdout=subprocess.PIPE) subprocess.check_output(("xargs", "rm", "-v"), stdin=ps.stdout) ps.wait() def restart_databases(pg_path, rel_data_path, mixed_mode, config): for node_name in config.node_name_to_ports.keys(): if mixed_mode and config.node_name_to_ports[node_name] in ( config.chosen_random_worker_port, config.coordinator_port(), ): continue abs_data_path = os.path.abspath(os.path.join(rel_data_path, node_name)) restart_database( pg_path=pg_path, abs_data_path=abs_data_path, node_name=node_name, node_ports=config.node_name_to_ports, logfile_prefix=config.name, ) def restart_database(pg_path, abs_data_path, node_name, node_ports, logfile_prefix): command = [ os.path.join(pg_path, "pg_ctl"), "restart", "--pgdata", abs_data_path, "-U", USER, "-o", "-p {}".format(node_ports[node_name]), "--log", os.path.join(abs_data_path, common.logfile_name(logfile_prefix, node_name)), ] subprocess.run(command, check=True) def run_alter_citus(pg_path, mixed_mode, config): for port in config.node_name_to_ports.values(): if mixed_mode and port in ( config.chosen_random_worker_port, config.coordinator_port(), ): continue utils.psql(pg_path, port, "ALTER EXTENSION citus UPDATE;") def verify_upgrade(config, mixed_mode, node_ports): for port in node_ports: actual_citus_version = get_actual_citus_version(config.bindir, port) expected_citus_version = MASTER_VERSION if expected_citus_version != actual_citus_version and not ( mixed_mode and port in (config.chosen_random_worker_port, config.coordinator_port()) ): print( "port: {} citus version {} expected {}".format( port, actual_citus_version, expected_citus_version ) ) sys.exit(1) else: print("port:{} citus version {}".format(port, actual_citus_version)) def get_before_upgrade_schedule(mixed_mode): if mixed_mode: return MIXED_BEFORE_CITUS_UPGRADE_SCHEDULE else: return BEFORE_CITUS_UPGRADE_COORD_SCHEDULE def get_after_upgrade_schedule(mixed_mode): if mixed_mode: return MIXED_AFTER_CITUS_UPGRADE_SCHEDULE else: return AFTER_CITUS_UPGRADE_COORD_SCHEDULE def generate_citus_tarball(citus_version): tmp_dir = "tmp_citus_tarballs" citus_old_tarpath = os.path.abspath( os.path.join(tmp_dir, f"install-pg{PG_MAJOR_VERSION}-citus{citus_version}.tar") ) common.initialize_temp_dir_if_not_exists(tmp_dir) dirpath = os.path.dirname(os.path.realpath(__file__)) local_script_path = os.path.join(dirpath, "generate_citus_tarballs.sh") with utils.cd(tmp_dir): subprocess.check_call([local_script_path, str(PG_MAJOR_VERSION), citus_version]) return citus_old_tarpath if __name__ == "__main__": args = docopt(__doc__, version="citus_upgrade_test") if not CI: citus_tarball_path = generate_citus_tarball(args["--citus-old-version"]) config = CitusUpgradeConfig(args, citus_tarball_path, None) else: config = CitusUpgradeConfig( args, args["--citus-pre-tar"], args["--citus-post-tar"] ) main(config) ================================================ FILE: src/test/regress/citus_tests/upgrade/generate_citus_tarballs.sh ================================================ #!/bin/bash set -euxo pipefail pg_version=$1 citus_old_version=$2 base="$(pwd)" install_citus_and_tar() { # do everything in a subdirectory to avoid clutter in current directory mkdir -p "${builddir}" && cd "${builddir}" "${citus_dir}/configure" installdir="${builddir}/install" make "-j$(nproc)" && mkdir -p "${installdir}" && make DESTDIR="${installdir}" install cd "${installdir}" && find . -type f -print >"${builddir}/files.lst" tar cvf "${basedir}/install-pg${pg_version}-citus${citus_version}.tar" $(cat "${builddir}"/files.lst) mv "${basedir}/install-pg${pg_version}-citus${citus_version}.tar" "${base}/install-pg${pg_version}-citus${citus_version}.tar" cd "${builddir}" && rm -rf install files.lst && make clean } build_ext() { citus_version="$1" # If tarball already exsists we're good if [ -f "${base}/install-pg${pg_version}-citus${citus_version}.tar" ]; then return fi basedir="${base}/${citus_version}" rm -rf "${basedir}" mkdir -p "${basedir}" cd "${basedir}" citus_dir=${basedir}/citus_$citus_version git clone --branch "$citus_version" https://github.com/citusdata/citus.git --depth 1 citus_"$citus_version" builddir="${basedir}/build" install_citus_and_tar } build_ext "${citus_old_version}" ================================================ FILE: src/test/regress/citus_tests/upgrade/pg_upgrade_test.py ================================================ #!/usr/bin/env python3 """upgrade_test Usage: upgrade_test --old-bindir= --new-bindir= --pgxsdir= [--test-with-columnar] Options: --old-bindir= The old PostgreSQL executable directory(ex: '~/.pgenv/pgsql-10.4/bin') --new-bindir= The new PostgreSQL executable directory(ex: '~/.pgenv/pgsql-11.3/bin') --pgxsdir= Path to the PGXS directory(ex: ~/.pgenv/src/postgresql-11.3) --test-with-columnar Enable automatically creating citus_columnar extension """ import atexit import os import subprocess import sys from docopt import docopt # https://stackoverflow.com/questions/14132789/relative-imports-for-the-billionth-time/14132912#14132912 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ignore E402 because these imports require addition to path import common # noqa: E402 import utils # noqa: E402 from utils import USER # noqa: E402 from config import ( # noqa: E402 AFTER_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE, AFTER_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE, BEFORE_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE, BEFORE_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE, PGUpgradeConfig, ) def citus_prepare_pg_upgrade(pg_path, node_ports): for port in node_ports: utils.psql(pg_path, port, "SELECT citus_prepare_pg_upgrade();") def perform_postgres_upgrade( old_bindir, new_bindir, old_datadir, new_datadir, node_names ): for node_name in node_names: base_new_data_path = os.path.abspath(new_datadir) base_old_data_path = os.path.abspath(old_datadir) with utils.cd(base_new_data_path): abs_new_data_path = os.path.join(base_new_data_path, node_name) abs_old_data_path = os.path.join(base_old_data_path, node_name) command = [ os.path.join(new_bindir, "pg_upgrade"), "--username", USER, "--old-bindir", old_bindir, "--new-bindir", new_bindir, "--old-datadir", abs_old_data_path, "--new-datadir", abs_new_data_path, ] subprocess.run(command, check=True) def citus_finish_pg_upgrade(pg_path, node_ports): for port in node_ports: utils.psql(pg_path, port, "SELECT citus_finish_pg_upgrade();") def stop_all_databases(old_bindir, new_bindir, old_datadir, new_datadir, config): common.stop_databases( old_bindir, old_datadir, config.node_name_to_ports, config.name ) common.stop_databases( new_bindir, new_datadir, config.node_name_to_ports, config.name ) def main(config): common.initialize_temp_dir(config.temp_dir) common.initialize_citus_cluster( config.old_bindir, config.old_datadir, config.settings, config ) common.run_pg_regress( config.old_bindir, config.pg_srcdir, config.coordinator_port(), ( BEFORE_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE if config.test_with_columnar else BEFORE_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE ), ) common.run_pg_regress( config.old_bindir, config.pg_srcdir, config.coordinator_port(), ( AFTER_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE if config.test_with_columnar else AFTER_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE ), ) citus_prepare_pg_upgrade(config.old_bindir, config.node_name_to_ports.values()) # prepare should be idempotent, calling it a second time should never fail. citus_prepare_pg_upgrade(config.old_bindir, config.node_name_to_ports.values()) common.stop_databases( config.old_bindir, config.old_datadir, config.node_name_to_ports, config.name ) common.initialize_db_for_cluster( config.new_bindir, config.new_datadir, config.settings, config.node_name_to_ports.keys(), ) perform_postgres_upgrade( config.old_bindir, config.new_bindir, config.old_datadir, config.new_datadir, config.node_name_to_ports.keys(), ) common.start_databases( config.new_bindir, config.new_datadir, config.node_name_to_ports, config.name, {}, ) citus_finish_pg_upgrade(config.new_bindir, config.node_name_to_ports.values()) common.run_pg_regress( config.new_bindir, config.pg_srcdir, config.coordinator_port(), ( AFTER_PG_UPGRADE_WITH_COLUMNAR_SCHEDULE if config.test_with_columnar else AFTER_PG_UPGRADE_WITHOUT_COLUMNAR_SCHEDULE ), ) if __name__ == "__main__": config = PGUpgradeConfig(docopt(__doc__, version="upgrade_test")) atexit.register( stop_all_databases, config.old_bindir, config.new_bindir, config.old_datadir, config.new_datadir, config, ) main(config) ================================================ FILE: src/test/regress/citus_tests/utils.py ================================================ import os import subprocess USER = "postgres" def psql(pg_path, port, command): return subprocess.run( [ os.path.join(pg_path, "psql"), "-U", USER, "-p", str(port), "-c", command, "-P", "pager=off", "--no-psqlrc", ], check=True, ) def psql_capture(pg_path, port, command): return subprocess.check_output( [ os.path.join(pg_path, "psql"), "-U", USER, "-p", str(port), "-c", command, "-P", "pager=off", "--no-psqlrc", "--tuples-only", ], ) # Taken from https://stackoverflow.com/questions/431684/how-do-i-change-directory-cd-in-python/13197763#13197763 class cd(object): """Context manager for changing the current working directory""" def __init__(self, newPath): self.newPath = os.path.expanduser(newPath) def __enter__(self): self.savedPath = os.getcwd() print(self.savedPath) os.chdir(self.newPath) def __exit__(self, etype, value, traceback): os.chdir(self.savedPath) ================================================ FILE: src/test/regress/columnar_isolation_schedule ================================================ test: columnar_write_concurrency columnar_write_concurrency_index test: columnar_vacuum_vs_insert test: columnar_temp_tables test: columnar_index_concurrency ================================================ FILE: src/test/regress/columnar_schedule ================================================ test: columnar_test_helpers test: columnar_create test: columnar_load test: columnar_query test: columnar_first_row_number test: columnar_analyze test: columnar_data_types test: columnar_drop test: columnar_indexes test: columnar_fallback_scan columnar_paths test: columnar_partitioning test: columnar_permissions test: columnar_empty test: columnar_insert test: columnar_update_delete test: columnar_cursor test: columnar_copyto test: columnar_alter test: columnar_alter_set_type test: columnar_lz4 columnar_zstd test: columnar_rollback test: columnar_truncate test: columnar_vacuum test: columnar_clean test: columnar_types_without_comparison test: columnar_chunk_filtering test: columnar_join test: columnar_pg15 test: columnar_trigger test: columnar_tableoptions test: columnar_recursive test: columnar_transactions test: columnar_matview test: columnar_memory ================================================ FILE: src/test/regress/create_schedule ================================================ test: intermediate_result_pruning_create test: prepared_statements_create_load ch_benchmarks_create_load test: dropped_columns_create_load distributed_planning_create_load test: local_dist_join_load nested_execution_create test: partitioned_indexes_create test: connectivity_checks test: schemas_create test: views_create test: sequences_create test: index_create test: function_create test: arbitrary_configs_truncate_create test: arbitrary_configs_truncate_cascade_create test: arbitrary_configs_truncate_partition_create test: arbitrary_configs_alter_table_add_constraint_without_name_create test: merge_arbitrary_create test: arbitrary_configs_router_create ================================================ FILE: src/test/regress/data/ads.csv ================================================ 474,8,59,Accuracy,https://placehold.it/600x100.png,http://wunsch.com/wilson.corwin,0,0,2017-06-13 16:42:34.608325,2017-06-13 16:42:34.67835 475,8,59,Mind Blast,https://placehold.it/600x100.png,http://wunsch.com/valentina,0,0,2017-06-13 16:42:34.691426,2017-06-13 16:42:34.769046 476,8,59,Telekinesis,https://placehold.it/600x100.png,http://wunsch.com/emmet.bogisich,0,0,2017-06-13 16:42:34.78347,2017-06-13 16:42:34.842037 477,8,59,Cloaking,https://placehold.it/600x100.png,http://wunsch.com/lottie.fahey,0,0,2017-06-13 16:42:34.856078,2017-06-13 16:42:34.945625 478,8,59,Sub-Mariner,https://placehold.it/600x100.png,http://wunsch.com/percival,0,0,2017-06-13 16:42:34.961286,2017-06-13 16:42:35.01756 479,8,59,Human physical perfection,https://placehold.it/600x100.png,http://wunsch.com/kelly,0,0,2017-06-13 16:42:35.03094,2017-06-13 16:42:35.111088 480,8,59,Radiation Absorption,https://placehold.it/600x100.png,http://wunsch.com/dudley.towne,0,0,2017-06-13 16:42:35.125719,2017-06-13 16:42:35.194242 481,8,60,Time Travel,https://placehold.it/600x100.png,http://tromp.co/cecilia.bernier,0,0,2017-06-13 16:42:35.234614,2017-06-13 16:42:35.315544 482,8,60,Power Augmentation,https://placehold.it/600x100.png,http://tromp.co/josiah,0,0,2017-06-13 16:42:35.330366,2017-06-13 16:42:35.381389 483,8,60,Size Changing,https://placehold.it/600x100.png,http://tromp.co/nichole.lindgren,0,0,2017-06-13 16:42:35.395987,2017-06-13 16:42:35.471563 484,8,60,Hydrokinesis,https://placehold.it/600x100.png,http://tromp.co/anastacio,0,0,2017-06-13 16:42:35.490591,2017-06-13 16:42:35.576482 485,8,60,Animal Oriented Powers,https://placehold.it/600x100.png,http://tromp.co/adeline_prosacco,0,0,2017-06-13 16:42:35.59005,2017-06-13 16:42:35.65499 486,8,60,Vision - Thermal,https://placehold.it/600x100.png,http://tromp.co/jamaal.langosh,0,0,2017-06-13 16:42:35.670428,2017-06-13 16:42:35.754608 487,8,60,Entropy Projection,https://placehold.it/600x100.png,http://tromp.co/je_ullrich,0,0,2017-06-13 16:42:35.767417,2017-06-13 16:42:35.832125 488,8,60,Cold Resistance,https://placehold.it/600x100.png,http://tromp.co/ashly,0,0,2017-06-13 16:42:35.846377,2017-06-13 16:42:35.922403 489,8,60,Enhanced Smell,https://placehold.it/600x100.png,http://tromp.co/georgianna.champlin,0,0,2017-06-13 16:42:35.935626,2017-06-13 16:42:35.997004 490,8,60,Latent Abilities,https://placehold.it/600x100.png,http://tromp.co/elyse,0,0,2017-06-13 16:42:36.011106,2017-06-13 16:42:36.079066 491,8,61,Matter Absorption,https://placehold.it/600x100.png,http://pourosmclaughlin.info/isaac,0,0,2017-06-13 16:42:36.112581,2017-06-13 16:42:36.17948 492,8,61,Separation,https://placehold.it/600x100.png,http://pourosmclaughlin.info/dallin.kutch,0,0,2017-06-13 16:42:36.195558,2017-06-13 16:42:36.263427 493,8,61,Omnilingualism,https://placehold.it/600x100.png,http://pourosmclaughlin.info/sarai.blick,0,0,2017-06-13 16:42:36.277127,2017-06-13 16:42:36.344861 494,8,61,Enhanced Senses,https://placehold.it/600x100.png,http://pourosmclaughlin.info/monique.paucek,0,0,2017-06-13 16:42:36.360244,2017-06-13 16:42:36.415685 495,8,61,Web Creation,https://placehold.it/600x100.png,http://pourosmclaughlin.info/erick,0,0,2017-06-13 16:42:36.431581,2017-06-13 16:42:36.498975 496,8,61,Invulnerability,https://placehold.it/600x100.png,http://pourosmclaughlin.info/lucinda.keeling,0,0,2017-06-13 16:42:36.513616,2017-06-13 16:42:36.591947 497,8,61,Phasing,https://placehold.it/600x100.png,http://pourosmclaughlin.info/palma.waters,0,0,2017-06-13 16:42:36.606594,2017-06-13 16:42:36.680658 498,8,62,Longevity,https://placehold.it/600x100.png,http://casper.co/helene.tillman,0,0,2017-06-13 16:42:36.706329,2017-06-13 16:42:36.773721 499,8,62,Ability Shift,https://placehold.it/600x100.png,http://casper.co/myrtice_pouros,0,0,2017-06-13 16:42:36.787875,2017-06-13 16:42:36.828517 500,8,62,Symbiote Costume,https://placehold.it/600x100.png,http://casper.co/meaghan,0,0,2017-06-13 16:42:36.843316,2017-06-13 16:42:36.901191 501,8,62,Enhanced Smell,https://placehold.it/600x100.png,http://casper.co/carmine_hickle,0,0,2017-06-13 16:42:36.914554,2017-06-13 16:42:36.983131 502,8,62,Vaporising Beams,https://placehold.it/600x100.png,http://casper.co/joshuah_barton,0,0,2017-06-13 16:42:36.99614,2017-06-13 16:42:37.039987 503,8,62,Changing Armor,https://placehold.it/600x100.png,http://casper.co/manuela,0,0,2017-06-13 16:42:37.053766,2017-06-13 16:42:37.122629 504,8,63,Telepathy,https://placehold.it/600x100.png,http://kerlukegibson.info/karine,0,0,2017-06-13 16:42:37.14871,2017-06-13 16:42:37.229672 505,8,63,Energy Manipulation,https://placehold.it/600x100.png,http://kerlukegibson.info/freeman.koepp,0,0,2017-06-13 16:42:37.245137,2017-06-13 16:42:37.288871 506,8,63,Orbing,https://placehold.it/600x100.png,http://kerlukegibson.info/nathanael,0,0,2017-06-13 16:42:37.304128,2017-06-13 16:42:37.370034 507,8,63,Mind Control,https://placehold.it/600x100.png,http://kerlukegibson.info/harvey_batz,0,0,2017-06-13 16:42:37.385162,2017-06-13 16:42:37.462222 508,8,63,Physical Anomaly,https://placehold.it/600x100.png,http://kerlukegibson.info/brannon.dickens,0,0,2017-06-13 16:42:37.476992,2017-06-13 16:42:37.544666 509,8,63,Physical Anomaly,https://placehold.it/600x100.png,http://kerlukegibson.info/wilbert,0,0,2017-06-13 16:42:37.55834,2017-06-13 16:42:37.631954 510,8,63,Vision - Night,https://placehold.it/600x100.png,http://kerlukegibson.info/jamarcus.graham,0,0,2017-06-13 16:42:37.646964,2017-06-13 16:42:37.72198 511,8,63,Cryokinesis,https://placehold.it/600x100.png,http://kerlukegibson.info/tyson,0,0,2017-06-13 16:42:37.737892,2017-06-13 16:42:37.793224 512,8,63,Weapon-based Powers,https://placehold.it/600x100.png,http://kerlukegibson.info/kasandra,0,0,2017-06-13 16:42:37.80835,2017-06-13 16:42:37.883791 513,8,64,Vision - Infrared,https://placehold.it/600x100.png,http://sawaynkris.name/brent.mccullough,0,0,2017-06-13 16:42:37.913044,2017-06-13 16:42:37.972777 514,8,64,Wishing,https://placehold.it/600x100.png,http://sawaynkris.name/scotty.torphy,0,0,2017-06-13 16:42:37.987515,2017-06-13 16:42:38.046315 515,8,64,Clairvoyance,https://placehold.it/600x100.png,http://sawaynkris.name/jeremy.ward,0,0,2017-06-13 16:42:38.063407,2017-06-13 16:42:38.142325 1,1,1,Astral Projection,https://placehold.it/600x100.png,http://ullrichreynolds.info/miouri,0,0,2017-06-13 16:41:52.743418,2017-06-13 16:41:52.821056 2,1,1,Endurance,https://placehold.it/600x100.png,http://ullrichreynolds.info/ally,0,0,2017-06-13 16:41:52.842525,2017-06-13 16:41:52.896993 3,1,1,Self-Sustenance,https://placehold.it/600x100.png,http://ullrichreynolds.info/kyler_kaulke,0,0,2017-06-13 16:41:52.91236,2017-06-13 16:41:52.976566 4,1,1,Flight,https://placehold.it/600x100.png,http://ullrichreynolds.info/earl,0,0,2017-06-13 16:41:52.992233,2017-06-13 16:41:53.040091 5,1,1,Web Creation,https://placehold.it/600x100.png,http://ullrichreynolds.info/lonie,0,0,2017-06-13 16:41:53.062708,2017-06-13 16:41:53.145829 6,1,1,Enhanced Senses,https://placehold.it/600x100.png,http://ullrichreynolds.info/fred,0,0,2017-06-13 16:41:53.161932,2017-06-13 16:41:53.239383 7,1,1,Psionic Powers,https://placehold.it/600x100.png,http://ullrichreynolds.info/emil_cole,0,0,2017-06-13 16:41:53.255107,2017-06-13 16:41:53.326385 8,1,2,Qwardian Power Ring,https://placehold.it/600x100.png,http://gottlieb.org/janelle,0,0,2017-06-13 16:41:53.356046,2017-06-13 16:41:53.417944 9,1,2,Photokinesis,https://placehold.it/600x100.png,http://gottlieb.org/kamron,0,0,2017-06-13 16:41:53.431432,2017-06-13 16:41:53.502793 10,1,2,Orbing,https://placehold.it/600x100.png,http://gottlieb.org/stefan_cruickshank,0,0,2017-06-13 16:41:53.51603,2017-06-13 16:41:53.564467 11,1,2,Vision - Infrared,https://placehold.it/600x100.png,http://gottlieb.org/nyah.yost,0,0,2017-06-13 16:41:53.579014,2017-06-13 16:41:53.641534 12,1,2,Magic,https://placehold.it/600x100.png,http://gottlieb.org/abraham_vonrueden,0,0,2017-06-13 16:41:53.657284,2017-06-13 16:41:53.719554 13,1,2,Projection,https://placehold.it/600x100.png,http://gottlieb.org/johnnie,0,0,2017-06-13 16:41:53.73225,2017-06-13 16:41:53.807951 14,1,3,Empathy,https://placehold.it/600x100.png,http://bartell.org/abigail,0,0,2017-06-13 16:41:53.831665,2017-06-13 16:41:53.899213 15,1,3,Aerokinesis,https://placehold.it/600x100.png,http://bartell.org/clementine.harvey,0,0,2017-06-13 16:41:53.912298,2017-06-13 16:41:53.977334 16,1,3,Qwardian Power Ring,https://placehold.it/600x100.png,http://bartell.org/lilyan,0,0,2017-06-13 16:41:53.991443,2017-06-13 16:41:54.057883 17,1,3,Animal Oriented Powers,https://placehold.it/600x100.png,http://bartell.org/ayana_considine,0,0,2017-06-13 16:41:54.073168,2017-06-13 16:41:54.150101 18,1,3,Lantern Power Ring,https://placehold.it/600x100.png,http://bartell.org/violet_white,0,0,2017-06-13 16:41:54.16378,2017-06-13 16:41:54.241198 19,1,3,Force Fields,https://placehold.it/600x100.png,http://bartell.org/lewis,0,0,2017-06-13 16:41:54.257117,2017-06-13 16:41:54.303101 20,1,3,Animal Control,https://placehold.it/600x100.png,http://bartell.org/louie,0,0,2017-06-13 16:41:54.320371,2017-06-13 16:41:54.393469 21,1,4,Symbiote Costume,https://placehold.it/600x100.png,http://weberwilkinson.biz/rose,0,0,2017-06-13 16:41:54.423479,2017-06-13 16:41:54.479025 22,1,4,Flight,https://placehold.it/600x100.png,http://weberwilkinson.biz/orland,0,0,2017-06-13 16:41:54.493675,2017-06-13 16:41:54.553835 23,1,4,Astral Projection,https://placehold.it/600x100.png,http://weberwilkinson.biz/albin,0,0,2017-06-13 16:41:54.568075,2017-06-13 16:41:54.630676 24,1,4,Apotheosis,https://placehold.it/600x100.png,http://weberwilkinson.biz/ralph.lesch,0,0,2017-06-13 16:41:54.646647,2017-06-13 16:41:54.730156 25,1,4,Elemental Transmogrification,https://placehold.it/600x100.png,http://weberwilkinson.biz/haley_bradtke,0,0,2017-06-13 16:41:54.746373,2017-06-13 16:41:54.818255 26,1,4,Physical Anomaly,https://placehold.it/600x100.png,http://weberwilkinson.biz/grayson_block,0,0,2017-06-13 16:41:54.833244,2017-06-13 16:41:54.886698 27,1,5,Intangibility,https://placehold.it/600x100.png,http://tromphauck.com/aliyah_cremin,0,0,2017-06-13 16:41:54.916506,2017-06-13 16:41:54.984771 28,1,5,Immortality,https://placehold.it/600x100.png,http://tromphauck.com/kariane_haley,0,0,2017-06-13 16:41:54.99844,2017-06-13 16:41:55.076524 29,1,5,Grim Reaping,https://placehold.it/600x100.png,http://tromphauck.com/leila_mckenzie,0,0,2017-06-13 16:41:55.089888,2017-06-13 16:41:55.158945 30,1,5,Vision - Cryo,https://placehold.it/600x100.png,http://tromphauck.com/kylee,0,0,2017-06-13 16:41:55.173877,2017-06-13 16:41:55.235255 31,1,5,Omnitrix,https://placehold.it/600x100.png,http://tromphauck.com/athena_fritsch,0,0,2017-06-13 16:41:55.250178,2017-06-13 16:41:55.31541 32,1,5,Density Control,https://placehold.it/600x100.png,http://tromphauck.com/jedediah_miller,0,0,2017-06-13 16:41:55.3313,2017-06-13 16:41:55.392267 33,1,5,Regeneration,https://placehold.it/600x100.png,http://tromphauck.com/rodrick.windler,0,0,2017-06-13 16:41:55.407676,2017-06-13 16:41:55.486182 34,1,6,Entropy Projection,https://placehold.it/600x100.png,http://pfeffer.info/gerson_mayer,0,0,2017-06-13 16:41:55.514712,2017-06-13 16:41:55.588303 35,1,6,Chlorokinesis,https://placehold.it/600x100.png,http://pfeffer.info/larry,0,0,2017-06-13 16:41:55.603978,2017-06-13 16:41:55.671785 36,1,6,Jump,https://placehold.it/600x100.png,http://pfeffer.info/declan_waters,0,0,2017-06-13 16:41:55.686352,2017-06-13 16:41:55.76281 37,1,6,Matter Absorption,https://placehold.it/600x100.png,http://pfeffer.info/leda_klein,0,0,2017-06-13 16:41:55.777003,2017-06-13 16:41:55.844437 38,1,6,Force Fields,https://placehold.it/600x100.png,http://pfeffer.info/jay,0,0,2017-06-13 16:41:55.857952,2017-06-13 16:41:55.915902 39,1,6,Self-Sustenance,https://placehold.it/600x100.png,http://pfeffer.info/lennie,0,0,2017-06-13 16:41:55.928361,2017-06-13 16:41:56.005911 40,1,6,Phasing,https://placehold.it/600x100.png,http://pfeffer.info/rachel,0,0,2017-06-13 16:41:56.01984,2017-06-13 16:41:56.092961 41,1,6,Self-Sustenance,https://placehold.it/600x100.png,http://pfeffer.info/idella.pacocha,0,0,2017-06-13 16:41:56.109324,2017-06-13 16:41:56.172858 42,1,6,Umbrakinesis,https://placehold.it/600x100.png,http://pfeffer.info/guillermo,0,0,2017-06-13 16:41:56.187473,2017-06-13 16:41:56.271745 43,1,6,Melting,https://placehold.it/600x100.png,http://pfeffer.info/tre,0,0,2017-06-13 16:41:56.286616,2017-06-13 16:41:56.348768 516,8,64,Vision - Night,https://placehold.it/600x100.png,http://sawaynkris.name/karley.schamberger,0,0,2017-06-13 16:42:38.155149,2017-06-13 16:42:38.213451 517,8,64,Magic Resistance,https://placehold.it/600x100.png,http://sawaynkris.name/karolann,0,0,2017-06-13 16:42:38.227688,2017-06-13 16:42:38.309546 518,8,64,Time Travel,https://placehold.it/600x100.png,http://sawaynkris.name/kareem_rodriguez,0,0,2017-06-13 16:42:38.322352,2017-06-13 16:42:38.40506 519,8,64,Vitakinesis,https://placehold.it/600x100.png,http://sawaynkris.name/reinhold,0,0,2017-06-13 16:42:38.420398,2017-06-13 16:42:38.474493 520,8,65,Orbing,https://placehold.it/600x100.png,http://altenwerth.biz/carson,0,0,2017-06-13 16:42:38.50261,2017-06-13 16:42:38.552952 521,8,65,Enhanced Memory,https://placehold.it/600x100.png,http://altenwerth.biz/halle,0,0,2017-06-13 16:42:38.567391,2017-06-13 16:42:38.6301 522,8,65,Gravitokinesis,https://placehold.it/600x100.png,http://altenwerth.biz/isaias.bauch,0,0,2017-06-13 16:42:38.645625,2017-06-13 16:42:38.7071 523,8,65,Sonic Scream,https://placehold.it/600x100.png,http://altenwerth.biz/emelie,0,0,2017-06-13 16:42:38.721179,2017-06-13 16:42:38.800573 524,8,65,Power Cosmic,https://placehold.it/600x100.png,http://altenwerth.biz/imani,0,0,2017-06-13 16:42:38.815318,2017-06-13 16:42:38.889674 525,8,65,Intuitive aptitude,https://placehold.it/600x100.png,http://altenwerth.biz/macie,0,0,2017-06-13 16:42:38.905747,2017-06-13 16:42:38.969082 44,1,7,Duplication,https://placehold.it/600x100.png,http://oconner.io/kamren,0,0,2017-06-13 16:41:56.380371,2017-06-13 16:41:56.440708 45,1,7,Astral Trap,https://placehold.it/600x100.png,http://oconner.io/austyn,0,0,2017-06-13 16:41:56.456295,2017-06-13 16:41:56.520117 46,1,7,Magic,https://placehold.it/600x100.png,http://oconner.io/leie,0,0,2017-06-13 16:41:56.537561,2017-06-13 16:41:56.623912 47,1,7,Death Touch,https://placehold.it/600x100.png,http://oconner.io/francisco,0,0,2017-06-13 16:41:56.640652,2017-06-13 16:41:56.698514 48,1,7,Astral Projection,https://placehold.it/600x100.png,http://oconner.io/shyann,0,0,2017-06-13 16:41:56.713923,2017-06-13 16:41:56.782984 49,1,7,Duplication,https://placehold.it/600x100.png,http://oconner.io/kaela.white,0,0,2017-06-13 16:41:56.799672,2017-06-13 16:41:56.874138 50,1,7,Cross-Dimensional Awareness,https://placehold.it/600x100.png,http://oconner.io/frederique,0,0,2017-06-13 16:41:56.889036,2017-06-13 16:41:56.955786 51,1,7,Enhanced Memory,https://placehold.it/600x100.png,http://oconner.io/isaiah.stoltenberg,0,0,2017-06-13 16:41:56.972734,2017-06-13 16:41:57.043972 263,5,34,Vision - Cryo,https://placehold.it/600x100.png,http://smith.name/colby,0,0,2017-06-13 16:42:16.162406,2017-06-13 16:42:16.207805 264,5,34,Enhanced Hearing,https://placehold.it/600x100.png,http://smith.name/liliana.sawayn,0,0,2017-06-13 16:42:16.221792,2017-06-13 16:42:16.306247 265,5,34,Underwater breathing,https://placehold.it/600x100.png,http://smith.name/kennith_jacobs,0,0,2017-06-13 16:42:16.319694,2017-06-13 16:42:16.385433 266,5,34,Empathy,https://placehold.it/600x100.png,http://smith.name/johnnie_marks,0,0,2017-06-13 16:42:16.398534,2017-06-13 16:42:16.47872 267,5,34,Photographic Reflexes,https://placehold.it/600x100.png,http://smith.name/arnulfo,0,0,2017-06-13 16:42:16.492138,2017-06-13 16:42:16.556796 268,5,34,Hypnokinesis,https://placehold.it/600x100.png,http://smith.name/damian_abshire,0,0,2017-06-13 16:42:16.571833,2017-06-13 16:42:16.634833 269,5,35,Vitakinesis,https://placehold.it/600x100.png,http://herman.name/maximilian,0,0,2017-06-13 16:42:16.660121,2017-06-13 16:42:16.746777 270,5,35,Illumination,https://placehold.it/600x100.png,http://herman.name/ebony.stanton,0,0,2017-06-13 16:42:16.76317,2017-06-13 16:42:16.842035 271,5,35,Flight,https://placehold.it/600x100.png,http://herman.name/jonas,0,0,2017-06-13 16:42:16.855602,2017-06-13 16:42:16.920217 272,5,35,Latent Abilities,https://placehold.it/600x100.png,http://herman.name/jaydon,0,0,2017-06-13 16:42:16.935107,2017-06-13 16:42:16.999501 273,5,35,Power Sense,https://placehold.it/600x100.png,http://herman.name/germaine_gerlach,0,0,2017-06-13 16:42:17.012844,2017-06-13 16:42:17.074256 274,5,35,Animation,https://placehold.it/600x100.png,http://herman.name/cynthia_hackett,0,0,2017-06-13 16:42:17.086759,2017-06-13 16:42:17.145545 275,5,35,Web Creation,https://placehold.it/600x100.png,http://herman.name/jerel,0,0,2017-06-13 16:42:17.159392,2017-06-13 16:42:17.224966 276,5,35,Adaptation,https://placehold.it/600x100.png,http://herman.name/hilda,0,0,2017-06-13 16:42:17.239934,2017-06-13 16:42:17.311339 277,5,35,Elasticity,https://placehold.it/600x100.png,http://herman.name/shanny.hills,0,0,2017-06-13 16:42:17.325049,2017-06-13 16:42:17.3875 278,5,35,Electrokinesis,https://placehold.it/600x100.png,http://herman.name/eduardo,0,0,2017-06-13 16:42:17.404187,2017-06-13 16:42:17.46846 279,5,36,Reality Warping,https://placehold.it/600x100.png,http://feil.biz/nya_little,0,0,2017-06-13 16:42:17.496607,2017-06-13 16:42:17.571994 280,5,36,Animal Control,https://placehold.it/600x100.png,http://feil.biz/joshua_connelly,0,0,2017-06-13 16:42:17.586525,2017-06-13 16:42:17.649273 281,5,36,Darkforce Manipulation,https://placehold.it/600x100.png,http://feil.biz/amparo_hilll,0,0,2017-06-13 16:42:17.674162,2017-06-13 16:42:17.752422 282,5,36,Astral Projection,https://placehold.it/600x100.png,http://feil.biz/gardner_feil,0,0,2017-06-13 16:42:17.769774,2017-06-13 16:42:17.8617 283,5,36,Thirstokinesis,https://placehold.it/600x100.png,http://feil.biz/dariana_rodriguez,0,0,2017-06-13 16:42:17.877147,2017-06-13 16:42:17.949145 284,5,36,Banish,https://placehold.it/600x100.png,http://feil.biz/howell.collins,0,0,2017-06-13 16:42:17.966361,2017-06-13 16:42:18.027674 285,5,36,Audiokinesis,https://placehold.it/600x100.png,http://feil.biz/kiara,0,0,2017-06-13 16:42:18.042808,2017-06-13 16:42:18.118722 286,5,36,Hypnokinesis,https://placehold.it/600x100.png,http://feil.biz/benny_gibson,0,0,2017-06-13 16:42:18.131505,2017-06-13 16:42:18.186998 287,5,36,Nova Force,https://placehold.it/600x100.png,http://feil.biz/lacy,0,0,2017-06-13 16:42:18.202092,2017-06-13 16:42:18.256864 288,5,37,Super Breath,https://placehold.it/600x100.png,http://harberbarton.org/augusta_spinka,0,0,2017-06-13 16:42:18.281835,2017-06-13 16:42:18.335603 289,5,37,Vision - Heat,https://placehold.it/600x100.png,http://harberbarton.org/mabel.king,0,0,2017-06-13 16:42:18.347493,2017-06-13 16:42:18.416212 290,5,37,Vitakinesis,https://placehold.it/600x100.png,http://harberbarton.org/patience,0,0,2017-06-13 16:42:18.431689,2017-06-13 16:42:18.50215 291,5,37,Stamina,https://placehold.it/600x100.png,http://harberbarton.org/zella_reynolds,0,0,2017-06-13 16:42:18.515605,2017-06-13 16:42:18.564853 292,5,37,Vision - Telescopic,https://placehold.it/600x100.png,http://harberbarton.org/treie,0,0,2017-06-13 16:42:18.580649,2017-06-13 16:42:18.660398 293,5,37,Longevity,https://placehold.it/600x100.png,http://harberbarton.org/emmie.miller,0,0,2017-06-13 16:42:18.674778,2017-06-13 16:42:18.735704 294,5,37,Darkforce Manipulation,https://placehold.it/600x100.png,http://harberbarton.org/carter_runolfsdottir,0,0,2017-06-13 16:42:18.748656,2017-06-13 16:42:18.808957 295,5,38,Thirstokinesis,https://placehold.it/600x100.png,http://wehner.info/briana,0,0,2017-06-13 16:42:18.836199,2017-06-13 16:42:18.900668 296,5,38,Regeneration,https://placehold.it/600x100.png,http://wehner.info/jammie,0,0,2017-06-13 16:42:18.913762,2017-06-13 16:42:18.972692 297,5,38,Force Fields,https://placehold.it/600x100.png,http://wehner.info/vella,0,0,2017-06-13 16:42:18.985338,2017-06-13 16:42:19.042965 298,5,38,Reflexes,https://placehold.it/600x100.png,http://wehner.info/morgan,0,0,2017-06-13 16:42:19.05659,2017-06-13 16:42:19.126796 299,5,38,Sonic Scream,https://placehold.it/600x100.png,http://wehner.info/courtney,0,0,2017-06-13 16:42:19.142045,2017-06-13 16:42:19.198281 300,5,38,Illusions,https://placehold.it/600x100.png,http://wehner.info/rose,0,0,2017-06-13 16:42:19.214866,2017-06-13 16:42:19.2976 301,5,39,Changing Armor,https://placehold.it/600x100.png,http://altenwerthfriesen.com/brielle.ferry,0,0,2017-06-13 16:42:19.323564,2017-06-13 16:42:19.391349 302,5,39,Probability Manipulation,https://placehold.it/600x100.png,http://altenwerthfriesen.com/letha.kiehn,0,0,2017-06-13 16:42:19.405213,2017-06-13 16:42:19.498867 303,5,39,Toxin and Disease Resistance,https://placehold.it/600x100.png,http://altenwerthfriesen.com/merl_rempel,0,0,2017-06-13 16:42:19.517393,2017-06-13 16:42:19.590909 304,5,39,Thermokinesis,https://placehold.it/600x100.png,http://altenwerthfriesen.com/nia.kuhn,0,0,2017-06-13 16:42:19.606901,2017-06-13 16:42:19.687948 305,5,39,Physical Anomaly,https://placehold.it/600x100.png,http://altenwerthfriesen.com/susan_lind,0,0,2017-06-13 16:42:19.704308,2017-06-13 16:42:19.763318 306,5,39,Underwater breathing,https://placehold.it/600x100.png,http://altenwerthfriesen.com/liza_okeefe,0,0,2017-06-13 16:42:19.799245,2017-06-13 16:42:19.874997 307,5,39,Radiation Immunity,https://placehold.it/600x100.png,http://altenwerthfriesen.com/ciara,0,0,2017-06-13 16:42:19.897113,2017-06-13 16:42:19.994026 308,5,40,Absorption,https://placehold.it/600x100.png,http://simonis.co/adelia,0,0,2017-06-13 16:42:20.020978,2017-06-13 16:42:20.085306 309,5,40,Lantern Power Ring,https://placehold.it/600x100.png,http://simonis.co/alva_balistreri,0,0,2017-06-13 16:42:20.099438,2017-06-13 16:42:20.162317 310,5,40,Omnitrix,https://placehold.it/600x100.png,http://simonis.co/riley,0,0,2017-06-13 16:42:20.178052,2017-06-13 16:42:20.253952 311,5,40,Power Nullifier,https://placehold.it/600x100.png,http://simonis.co/camren_wisoky,0,0,2017-06-13 16:42:20.269262,2017-06-13 16:42:20.341557 312,5,40,Force Fields,https://placehold.it/600x100.png,http://simonis.co/karlie_zemlak,0,0,2017-06-13 16:42:20.356619,2017-06-13 16:42:20.425887 313,5,40,Apotheosis,https://placehold.it/600x100.png,http://simonis.co/malcolm_streich,0,0,2017-06-13 16:42:20.441071,2017-06-13 16:42:20.505276 314,5,40,Photokinesis,https://placehold.it/600x100.png,http://simonis.co/addison,0,0,2017-06-13 16:42:20.518554,2017-06-13 16:42:20.596769 315,5,40,Molecular Dissipation,https://placehold.it/600x100.png,http://simonis.co/ernestina.larson,0,0,2017-06-13 16:42:20.609757,2017-06-13 16:42:20.661025 316,5,40,Intangibility,https://placehold.it/600x100.png,http://simonis.co/heber_barton,0,0,2017-06-13 16:42:20.674742,2017-06-13 16:42:20.754247 317,5,41,Energy Armor,https://placehold.it/600x100.png,http://mraz.info/meghan.nienow,0,0,2017-06-13 16:42:20.782879,2017-06-13 16:42:20.840814 318,5,41,Toxikinesis,https://placehold.it/600x100.png,http://mraz.info/enoch_leuschke,0,0,2017-06-13 16:42:20.854318,2017-06-13 16:42:20.921452 319,5,41,Death Touch,https://placehold.it/600x100.png,http://mraz.info/elisa.powlowski,0,0,2017-06-13 16:42:20.936985,2017-06-13 16:42:20.987241 320,5,41,Hyperkinesis,https://placehold.it/600x100.png,http://mraz.info/myles.ohara,0,0,2017-06-13 16:42:21.003609,2017-06-13 16:42:21.079362 321,5,41,Magic Resistance,https://placehold.it/600x100.png,http://mraz.info/talia,0,0,2017-06-13 16:42:21.09468,2017-06-13 16:42:21.177664 322,5,41,Omnilingualism,https://placehold.it/600x100.png,http://mraz.info/ewald_bruen,0,0,2017-06-13 16:42:21.194168,2017-06-13 16:42:21.253242 323,5,41,Mind Blast,https://placehold.it/600x100.png,http://mraz.info/salvatore.marvin,0,0,2017-06-13 16:42:21.265753,2017-06-13 16:42:21.327758 324,5,41,Duplication,https://placehold.it/600x100.png,http://mraz.info/khalid.gleason,0,0,2017-06-13 16:42:21.341988,2017-06-13 16:42:21.408118 325,5,41,Animal Control,https://placehold.it/600x100.png,http://mraz.info/simeon,0,0,2017-06-13 16:42:21.421917,2017-06-13 16:42:21.48722 326,5,41,Telepathy Resistance,https://placehold.it/600x100.png,http://mraz.info/karli,0,0,2017-06-13 16:42:21.502431,2017-06-13 16:42:21.563172 327,5,42,Insanity,https://placehold.it/600x100.png,http://hermann.net/kiana_watsica,0,0,2017-06-13 16:42:21.588969,2017-06-13 16:42:21.648471 328,5,42,Omnipotence,https://placehold.it/600x100.png,http://hermann.net/elody_pfeffer,0,0,2017-06-13 16:42:21.662236,2017-06-13 16:42:21.747112 329,5,42,Vision - Night,https://placehold.it/600x100.png,http://hermann.net/michael.keebler,0,0,2017-06-13 16:42:21.762926,2017-06-13 16:42:21.844281 330,5,42,Weapons Master,https://placehold.it/600x100.png,http://hermann.net/vilma,0,0,2017-06-13 16:42:21.859198,2017-06-13 16:42:21.913085 331,5,42,Radiation Control,https://placehold.it/600x100.png,http://hermann.net/hillard.olson,0,0,2017-06-13 16:42:21.937447,2017-06-13 16:42:22.016871 332,5,42,Enhanced Sight,https://placehold.it/600x100.png,http://hermann.net/ettie,0,0,2017-06-13 16:42:22.035241,2017-06-13 16:42:22.081754 333,5,42,Energy Manipulation,https://placehold.it/600x100.png,http://hermann.net/brandyn_flatley,0,0,2017-06-13 16:42:22.095659,2017-06-13 16:42:22.17818 334,5,42,Enhanced Smell,https://placehold.it/600x100.png,http://hermann.net/christy.farrell,0,0,2017-06-13 16:42:22.192815,2017-06-13 16:42:22.263038 335,5,42,Human physical perfection,https://placehold.it/600x100.png,http://hermann.net/chris,0,0,2017-06-13 16:42:22.277711,2017-06-13 16:42:22.34046 195,4,26,Melting,https://placehold.it/600x100.png,http://reichertemard.info/nels.bailey,0,0,2017-06-13 16:42:09.926522,2017-06-13 16:42:09.98811 196,4,26,Vision - Thermal,https://placehold.it/600x100.png,http://reichertemard.info/destiney.parker,0,0,2017-06-13 16:42:10.01493,2017-06-13 16:42:10.098757 197,4,26,Cryokinesis,https://placehold.it/600x100.png,http://reichertemard.info/carmel.green,0,0,2017-06-13 16:42:10.115465,2017-06-13 16:42:10.171335 198,4,26,Enhanced Senses,https://placehold.it/600x100.png,http://reichertemard.info/bertha,0,0,2017-06-13 16:42:10.184966,2017-06-13 16:42:10.272063 199,4,26,Qwardian Power Ring,https://placehold.it/600x100.png,http://reichertemard.info/mack_schoen,0,0,2017-06-13 16:42:10.288014,2017-06-13 16:42:10.400737 200,4,26,Molecular Combustion,https://placehold.it/600x100.png,http://reichertemard.info/tracey,0,0,2017-06-13 16:42:10.415436,2017-06-13 16:42:10.462692 201,4,26,Hypnokinesis,https://placehold.it/600x100.png,http://reichertemard.info/jermey_homenick,0,0,2017-06-13 16:42:10.476775,2017-06-13 16:42:10.547303 202,4,26,Size Changing,https://placehold.it/600x100.png,http://reichertemard.info/karson,0,0,2017-06-13 16:42:10.562942,2017-06-13 16:42:10.648992 203,4,26,Spatial Awareness,https://placehold.it/600x100.png,http://reichertemard.info/aglae,0,0,2017-06-13 16:42:10.662146,2017-06-13 16:42:10.749766 204,4,26,Reality Warping,https://placehold.it/600x100.png,http://reichertemard.info/trinity,0,0,2017-06-13 16:42:10.765629,2017-06-13 16:42:10.85367 205,4,27,Vaporising Beams,https://placehold.it/600x100.png,http://brekke.info/gaetano_kreiger,0,0,2017-06-13 16:42:10.881735,2017-06-13 16:42:10.949966 206,4,27,Death Touch,https://placehold.it/600x100.png,http://brekke.info/americo,0,0,2017-06-13 16:42:10.965366,2017-06-13 16:42:11.033852 207,4,27,Entropy Projection,https://placehold.it/600x100.png,http://brekke.info/ezequiel,0,0,2017-06-13 16:42:11.046569,2017-06-13 16:42:11.104646 208,4,27,Vision - X-Ray,https://placehold.it/600x100.png,http://brekke.info/eduardo,0,0,2017-06-13 16:42:11.120397,2017-06-13 16:42:11.180013 209,4,27,Energy Armor,https://placehold.it/600x100.png,http://brekke.info/yadira,0,0,2017-06-13 16:42:11.19538,2017-06-13 16:42:11.250759 210,4,27,Mind Control,https://placehold.it/600x100.png,http://brekke.info/isabell.legros,0,0,2017-06-13 16:42:11.264326,2017-06-13 16:42:11.324623 211,4,27,Illumination,https://placehold.it/600x100.png,http://brekke.info/jacky.osinski,0,0,2017-06-13 16:42:11.341134,2017-06-13 16:42:11.428056 212,4,28,Cold Resistance,https://placehold.it/600x100.png,http://nader.net/quincy.adams,0,0,2017-06-13 16:42:11.455535,2017-06-13 16:42:11.537865 213,4,28,Animal Attributes,https://placehold.it/600x100.png,http://nader.net/craig,0,0,2017-06-13 16:42:11.552403,2017-06-13 16:42:11.631115 214,4,28,Aerokinesis,https://placehold.it/600x100.png,http://nader.net/wade,0,0,2017-06-13 16:42:11.644744,2017-06-13 16:42:11.697 215,4,28,Super Speed,https://placehold.it/600x100.png,http://nader.net/era.bahringer,0,0,2017-06-13 16:42:11.712336,2017-06-13 16:42:11.787007 216,4,28,Immortality,https://placehold.it/600x100.png,http://nader.net/jaquan_auer,0,0,2017-06-13 16:42:11.801254,2017-06-13 16:42:11.864441 217,4,28,Power Augmentation,https://placehold.it/600x100.png,http://nader.net/van,0,0,2017-06-13 16:42:11.876843,2017-06-13 16:42:11.944514 218,4,28,Gravitokinesis,https://placehold.it/600x100.png,http://nader.net/esta,0,0,2017-06-13 16:42:11.958474,2017-06-13 16:42:12.035329 219,4,28,Magnetokinesis,https://placehold.it/600x100.png,http://nader.net/percival,0,0,2017-06-13 16:42:12.050091,2017-06-13 16:42:12.113839 220,4,28,Geokinesis,https://placehold.it/600x100.png,http://nader.net/paxton.fritsch,0,0,2017-06-13 16:42:12.128613,2017-06-13 16:42:12.186174 221,4,29,Illumination,https://placehold.it/600x100.png,http://powlowski.info/melvin,0,0,2017-06-13 16:42:12.214896,2017-06-13 16:42:12.28574 222,4,29,Telekinesis,https://placehold.it/600x100.png,http://powlowski.info/odie,0,0,2017-06-13 16:42:12.298268,2017-06-13 16:42:12.354219 223,4,29,Molecular Immobilization,https://placehold.it/600x100.png,http://powlowski.info/leland,0,0,2017-06-13 16:42:12.365801,2017-06-13 16:42:12.480926 224,4,29,Telepathy Resistance,https://placehold.it/600x100.png,http://powlowski.info/linnie_gerhold,0,0,2017-06-13 16:42:12.495242,2017-06-13 16:42:12.548984 225,4,29,Thirstokinesis,https://placehold.it/600x100.png,http://powlowski.info/leopold.hayes,0,0,2017-06-13 16:42:12.563367,2017-06-13 16:42:12.626435 226,4,29,Elemental Transmogrification,https://placehold.it/600x100.png,http://powlowski.info/turner.prohaska,0,0,2017-06-13 16:42:12.641204,2017-06-13 16:42:12.713979 227,4,29,Sonic Scream,https://placehold.it/600x100.png,http://powlowski.info/audrey.frami,0,0,2017-06-13 16:42:12.731361,2017-06-13 16:42:12.800301 228,4,29,Invisibility,https://placehold.it/600x100.png,http://powlowski.info/imogene_steuber,0,0,2017-06-13 16:42:12.819249,2017-06-13 16:42:12.902741 229,4,29,Thermokinesis,https://placehold.it/600x100.png,http://powlowski.info/kelvin.schmitt,0,0,2017-06-13 16:42:12.921349,2017-06-13 16:42:13.001917 230,4,29,The Force,https://placehold.it/600x100.png,http://powlowski.info/hollis,0,0,2017-06-13 16:42:13.019277,2017-06-13 16:42:13.096331 231,4,30,Spatial Awareness,https://placehold.it/600x100.png,http://kovacek.biz/kiley_weimann,0,0,2017-06-13 16:42:13.128857,2017-06-13 16:42:13.19951 232,4,30,Hypnokinesis,https://placehold.it/600x100.png,http://kovacek.biz/werner,0,0,2017-06-13 16:42:13.218297,2017-06-13 16:42:13.299542 233,4,30,Omnitrix,https://placehold.it/600x100.png,http://kovacek.biz/bartholome,0,0,2017-06-13 16:42:13.314483,2017-06-13 16:42:13.396859 234,4,30,Empathy,https://placehold.it/600x100.png,http://kovacek.biz/peter,0,0,2017-06-13 16:42:13.413567,2017-06-13 16:42:13.483935 235,4,30,Agility,https://placehold.it/600x100.png,http://kovacek.biz/tatum_ebert,0,0,2017-06-13 16:42:13.500539,2017-06-13 16:42:13.582682 236,4,30,Probability Manipulation,https://placehold.it/600x100.png,http://kovacek.biz/hillary,0,0,2017-06-13 16:42:13.599466,2017-06-13 16:42:13.695078 237,4,30,Illumination,https://placehold.it/600x100.png,http://kovacek.biz/geovanny_lockman,0,0,2017-06-13 16:42:13.729134,2017-06-13 16:42:13.810826 238,4,31,Probability Manipulation,https://placehold.it/600x100.png,http://hyattbosco.name/darrin,0,0,2017-06-13 16:42:13.852567,2017-06-13 16:42:13.966448 239,4,31,Vision - Infrared,https://placehold.it/600x100.png,http://hyattbosco.name/edmond_feeney,0,0,2017-06-13 16:42:13.980191,2017-06-13 16:42:14.051847 240,4,31,Shapeshifting,https://placehold.it/600x100.png,http://hyattbosco.name/mia,0,0,2017-06-13 16:42:14.06857,2017-06-13 16:42:14.12891 241,4,31,Matter Absorption,https://placehold.it/600x100.png,http://hyattbosco.name/emmie,0,0,2017-06-13 16:42:14.143312,2017-06-13 16:42:14.229655 242,4,31,Cross-Dimensional Travel,https://placehold.it/600x100.png,http://hyattbosco.name/ellie.moore,0,0,2017-06-13 16:42:14.245168,2017-06-13 16:42:14.319963 243,4,31,Vitakinesis,https://placehold.it/600x100.png,http://hyattbosco.name/howard,0,0,2017-06-13 16:42:14.334896,2017-06-13 16:42:14.397326 244,4,31,Astral Trap,https://placehold.it/600x100.png,http://hyattbosco.name/vita,0,0,2017-06-13 16:42:14.411435,2017-06-13 16:42:14.467397 245,4,31,Geokinesis,https://placehold.it/600x100.png,http://hyattbosco.name/antonetta_schamberger,0,0,2017-06-13 16:42:14.480336,2017-06-13 16:42:14.551423 246,4,31,Entropy Projection,https://placehold.it/600x100.png,http://hyattbosco.name/frida,0,0,2017-06-13 16:42:14.564282,2017-06-13 16:42:14.637422 247,4,31,Agility,https://placehold.it/600x100.png,http://hyattbosco.name/cyril_douglas,0,0,2017-06-13 16:42:14.651284,2017-06-13 16:42:14.716513 248,4,32,Elasticity,https://placehold.it/600x100.png,http://macgyverdach.name/lesley.leuschke,0,0,2017-06-13 16:42:14.744501,2017-06-13 16:42:14.801894 249,4,32,Elasticity,https://placehold.it/600x100.png,http://macgyverdach.name/simeon.renner,0,0,2017-06-13 16:42:14.815489,2017-06-13 16:42:14.881319 250,4,32,Energy Blasts,https://placehold.it/600x100.png,http://macgyverdach.name/eusebio,0,0,2017-06-13 16:42:14.894441,2017-06-13 16:42:14.951624 251,4,32,Jump,https://placehold.it/600x100.png,http://macgyverdach.name/mohamed_mcdermott,0,0,2017-06-13 16:42:14.964376,2017-06-13 16:42:15.052968 252,4,32,Energy Constructs,https://placehold.it/600x100.png,http://macgyverdach.name/jewell,0,0,2017-06-13 16:42:15.066124,2017-06-13 16:42:15.123939 253,4,32,Magic,https://placehold.it/600x100.png,http://macgyverdach.name/emie,0,0,2017-06-13 16:42:15.13865,2017-06-13 16:42:15.194685 254,4,32,Resurrection,https://placehold.it/600x100.png,http://macgyverdach.name/halie_heaney,0,0,2017-06-13 16:42:15.209184,2017-06-13 16:42:15.292166 255,4,32,Telepathy,https://placehold.it/600x100.png,http://macgyverdach.name/wilhelmine,0,0,2017-06-13 16:42:15.308347,2017-06-13 16:42:15.384001 256,4,32,Echokinesis,https://placehold.it/600x100.png,http://macgyverdach.name/heaven_flatley,0,0,2017-06-13 16:42:15.397995,2017-06-13 16:42:15.463031 257,4,33,Cryokinesis,https://placehold.it/600x100.png,http://christiansen.biz/humberto_bechtelar,0,0,2017-06-13 16:42:15.489538,2017-06-13 16:42:15.56185 258,4,33,Molecular Combustion,https://placehold.it/600x100.png,http://christiansen.biz/aron_macgyver,0,0,2017-06-13 16:42:15.577872,2017-06-13 16:42:15.65217 259,4,33,Hypnokinesis,https://placehold.it/600x100.png,http://christiansen.biz/tatum_ankunding,0,0,2017-06-13 16:42:15.665302,2017-06-13 16:42:15.70958 260,4,33,Orbing,https://placehold.it/600x100.png,http://christiansen.biz/emery,0,0,2017-06-13 16:42:15.728428,2017-06-13 16:42:15.798829 261,4,33,Vision - X-Ray,https://placehold.it/600x100.png,http://christiansen.biz/michelle_cruickshank,0,0,2017-06-13 16:42:15.812196,2017-06-13 16:42:15.852943 262,4,33,Molecular Combustion,https://placehold.it/600x100.png,http://christiansen.biz/danika_ratke,0,0,2017-06-13 16:42:15.865508,2017-06-13 16:42:15.929214 403,7,51,Durability,https://placehold.it/600x100.png,http://klinghansen.com/ron,0,0,2017-06-13 16:42:28.604309,2017-06-13 16:42:28.678298 404,7,51,Electrokinesis,https://placehold.it/600x100.png,http://klinghansen.com/jamison.murray,0,0,2017-06-13 16:42:28.693114,2017-06-13 16:42:28.73854 405,7,51,Hydrokinesis,https://placehold.it/600x100.png,http://klinghansen.com/carmen_murray,0,0,2017-06-13 16:42:28.752645,2017-06-13 16:42:28.815855 406,7,51,Ability Shift,https://placehold.it/600x100.png,http://klinghansen.com/alvera_murray,0,0,2017-06-13 16:42:28.829952,2017-06-13 16:42:28.882325 407,7,51,Toxin and Disease Resistance,https://placehold.it/600x100.png,http://klinghansen.com/reyna_greenfelder,0,0,2017-06-13 16:42:28.897239,2017-06-13 16:42:28.965269 408,7,51,Atmokinesis,https://placehold.it/600x100.png,http://klinghansen.com/caroline,0,0,2017-06-13 16:42:28.979254,2017-06-13 16:42:29.033103 409,7,51,Radiation Immunity,https://placehold.it/600x100.png,http://klinghansen.com/deron,0,0,2017-06-13 16:42:29.047041,2017-06-13 16:42:29.102933 410,7,51,Power Absorption,https://placehold.it/600x100.png,http://klinghansen.com/litzy,0,0,2017-06-13 16:42:29.115026,2017-06-13 16:42:29.19022 411,7,51,Hydrokinesis,https://placehold.it/600x100.png,http://klinghansen.com/roscoe.thompson,0,0,2017-06-13 16:42:29.203687,2017-06-13 16:42:29.262901 412,7,52,Molecular Dissipation,https://placehold.it/600x100.png,http://grady.co/genoveva.hartmann,0,0,2017-06-13 16:42:29.296123,2017-06-13 16:42:29.377637 413,7,52,Portal Creation,https://placehold.it/600x100.png,http://grady.co/danika,0,0,2017-06-13 16:42:29.391003,2017-06-13 16:42:29.458785 414,7,52,Stamina,https://placehold.it/600x100.png,http://grady.co/salma,0,0,2017-06-13 16:42:29.472321,2017-06-13 16:42:29.545044 415,7,52,Geokinesis,https://placehold.it/600x100.png,http://grady.co/matt_klocko,0,0,2017-06-13 16:42:29.558831,2017-06-13 16:42:29.627347 416,7,52,Cloaking,https://placehold.it/600x100.png,http://grady.co/otilia,0,0,2017-06-13 16:42:29.642138,2017-06-13 16:42:29.716773 417,7,52,Animation,https://placehold.it/600x100.png,http://grady.co/audreanne_connelly,0,0,2017-06-13 16:42:29.730007,2017-06-13 16:42:29.807748 418,7,52,Molecular Combustion,https://placehold.it/600x100.png,http://grady.co/maybell_huels,0,0,2017-06-13 16:42:29.820471,2017-06-13 16:42:29.889149 419,7,52,Photographic Reflexes,https://placehold.it/600x100.png,http://grady.co/ethelyn,0,0,2017-06-13 16:42:29.90302,2017-06-13 16:42:29.981066 420,7,52,Radiation Immunity,https://placehold.it/600x100.png,http://grady.co/leilani,0,0,2017-06-13 16:42:29.996324,2017-06-13 16:42:30.053946 421,7,53,Enhanced Senses,https://placehold.it/600x100.png,http://davistowne.biz/keaton,0,0,2017-06-13 16:42:30.080147,2017-06-13 16:42:30.155064 422,7,53,Elasticity,https://placehold.it/600x100.png,http://davistowne.biz/isobel,0,0,2017-06-13 16:42:30.168813,2017-06-13 16:42:30.239331 423,7,53,Adaptation,https://placehold.it/600x100.png,http://davistowne.biz/wendell.emmerich,0,0,2017-06-13 16:42:30.254269,2017-06-13 16:42:30.313076 424,7,53,Vision - Night,https://placehold.it/600x100.png,http://davistowne.biz/jerome_cummings,0,0,2017-06-13 16:42:30.326348,2017-06-13 16:42:30.406846 425,7,53,Animal Attributes,https://placehold.it/600x100.png,http://davistowne.biz/kaylie,0,0,2017-06-13 16:42:30.41931,2017-06-13 16:42:30.484838 426,7,53,Size Changing,https://placehold.it/600x100.png,http://davistowne.biz/laria,0,0,2017-06-13 16:42:30.49803,2017-06-13 16:42:30.556733 427,7,53,Portal Creation,https://placehold.it/600x100.png,http://davistowne.biz/eveline,0,0,2017-06-13 16:42:30.570974,2017-06-13 16:42:30.641847 428,7,53,Enhanced Smell,https://placehold.it/600x100.png,http://davistowne.biz/patience_dach,0,0,2017-06-13 16:42:30.657014,2017-06-13 16:42:30.699382 429,7,53,Enhanced Smell,https://placehold.it/600x100.png,http://davistowne.biz/geraldine.wolff,0,0,2017-06-13 16:42:30.714557,2017-06-13 16:42:30.77683 430,7,53,Durability,https://placehold.it/600x100.png,http://davistowne.biz/arthur.reichert,0,0,2017-06-13 16:42:30.796571,2017-06-13 16:42:30.872621 431,7,54,Self-Sustenance,https://placehold.it/600x100.png,http://skiles.co/darien,0,0,2017-06-13 16:42:30.900325,2017-06-13 16:42:30.981206 432,7,54,Timeframe Control,https://placehold.it/600x100.png,http://skiles.co/mara.olson,0,0,2017-06-13 16:42:30.995774,2017-06-13 16:42:31.053294 433,7,54,Cryokinesis,https://placehold.it/600x100.png,http://skiles.co/myrtle,0,0,2017-06-13 16:42:31.067833,2017-06-13 16:42:31.135892 434,7,54,Gravitokinesis,https://placehold.it/600x100.png,http://skiles.co/eileen.carter,0,0,2017-06-13 16:42:31.150801,2017-06-13 16:42:31.210469 435,7,54,Physical Anomaly,https://placehold.it/600x100.png,http://skiles.co/domenic,0,0,2017-06-13 16:42:31.224606,2017-06-13 16:42:31.282036 436,7,54,Orbing,https://placehold.it/600x100.png,http://skiles.co/amelia,0,0,2017-06-13 16:42:31.296307,2017-06-13 16:42:31.364396 437,7,54,Hydrokinesis,https://placehold.it/600x100.png,http://skiles.co/kayleigh_stark,0,0,2017-06-13 16:42:31.37953,2017-06-13 16:42:31.463535 438,7,54,Banish,https://placehold.it/600x100.png,http://skiles.co/annalise,0,0,2017-06-13 16:42:31.477699,2017-06-13 16:42:31.540072 439,7,54,Astral Trap,https://placehold.it/600x100.png,http://skiles.co/kenna,0,0,2017-06-13 16:42:31.556365,2017-06-13 16:42:31.629206 440,7,54,Vision - Cryo,https://placehold.it/600x100.png,http://skiles.co/norris,0,0,2017-06-13 16:42:31.644734,2017-06-13 16:42:31.713326 441,7,55,Molecular Combustion,https://placehold.it/600x100.png,http://swift.co/lambert.witting,0,0,2017-06-13 16:42:31.744059,2017-06-13 16:42:31.805383 442,7,55,Energy Blasts,https://placehold.it/600x100.png,http://swift.co/marlene,0,0,2017-06-13 16:42:31.819,2017-06-13 16:42:31.886085 443,7,55,Chronokinesis,https://placehold.it/600x100.png,http://swift.co/marilou,0,0,2017-06-13 16:42:31.900555,2017-06-13 16:42:31.979509 444,7,55,Longevity,https://placehold.it/600x100.png,http://swift.co/maureen.dickens,0,0,2017-06-13 16:42:31.992875,2017-06-13 16:42:32.067624 445,7,55,Adaptation,https://placehold.it/600x100.png,http://swift.co/judson.keler,0,0,2017-06-13 16:42:32.080885,2017-06-13 16:42:32.154812 446,7,55,Cloaking,https://placehold.it/600x100.png,http://swift.co/magnus,0,0,2017-06-13 16:42:32.170243,2017-06-13 16:42:32.232727 447,7,55,Projection,https://placehold.it/600x100.png,http://swift.co/makayla.padberg,0,0,2017-06-13 16:42:32.246729,2017-06-13 16:42:32.292045 448,7,55,Magnetokinesis,https://placehold.it/600x100.png,http://swift.co/nikolas.muller,0,0,2017-06-13 16:42:32.306589,2017-06-13 16:42:32.361817 449,7,55,Radiation Control,https://placehold.it/600x100.png,http://swift.co/gwen,0,0,2017-06-13 16:42:32.376305,2017-06-13 16:42:32.435641 450,7,55,Separation,https://placehold.it/600x100.png,http://swift.co/carlotta,0,0,2017-06-13 16:42:32.447937,2017-06-13 16:42:32.506509 451,7,56,Precognition,https://placehold.it/600x100.png,http://muller.com/emie,0,0,2017-06-13 16:42:32.532825,2017-06-13 16:42:32.586822 452,7,56,Vision - Telescopic,https://placehold.it/600x100.png,http://muller.com/savanah.weinat,0,0,2017-06-13 16:42:32.598892,2017-06-13 16:42:32.661789 453,7,56,Energy Armor,https://placehold.it/600x100.png,http://muller.com/roselyn,0,0,2017-06-13 16:42:32.67583,2017-06-13 16:42:32.747526 454,7,56,Symbiote Costume,https://placehold.it/600x100.png,http://muller.com/devan_schuppe,0,0,2017-06-13 16:42:32.761974,2017-06-13 16:42:32.829835 455,7,56,Enhanced Smell,https://placehold.it/600x100.png,http://muller.com/kasey_farrell,0,0,2017-06-13 16:42:32.844693,2017-06-13 16:42:32.904123 456,7,56,Apotheosis,https://placehold.it/600x100.png,http://muller.com/zoila_gibson,0,0,2017-06-13 16:42:32.9171,2017-06-13 16:42:32.984825 457,7,57,Absorption,https://placehold.it/600x100.png,http://price.info/heidi,0,0,2017-06-13 16:42:33.015761,2017-06-13 16:42:33.087538 458,7,57,Omnipotence,https://placehold.it/600x100.png,http://price.info/hal,0,0,2017-06-13 16:42:33.103637,2017-06-13 16:42:33.176102 459,7,57,Invulnerability,https://placehold.it/600x100.png,http://price.info/yasmeen_runolfsdottir,0,0,2017-06-13 16:42:33.1894,2017-06-13 16:42:33.261465 460,7,57,Wallcrawling,https://placehold.it/600x100.png,http://price.info/vena_legros,0,0,2017-06-13 16:42:33.275881,2017-06-13 16:42:33.326074 461,7,57,Shapeshifting,https://placehold.it/600x100.png,http://price.info/walter.balistreri,0,0,2017-06-13 16:42:33.340048,2017-06-13 16:42:33.421558 462,7,57,Nova Force,https://placehold.it/600x100.png,http://price.info/imani_kihn,0,0,2017-06-13 16:42:33.436373,2017-06-13 16:42:33.507189 463,7,57,Duplication,https://placehold.it/600x100.png,http://price.info/merle,0,0,2017-06-13 16:42:33.520825,2017-06-13 16:42:33.610244 464,7,57,Intuitive aptitude,https://placehold.it/600x100.png,http://price.info/mattie,0,0,2017-06-13 16:42:33.623721,2017-06-13 16:42:33.703095 465,7,57,Latent Abilities,https://placehold.it/600x100.png,http://price.info/leonie_leuschke,0,0,2017-06-13 16:42:33.717448,2017-06-13 16:42:33.778371 466,7,58,Psionic Powers,https://placehold.it/600x100.png,http://heaneyhoppe.biz/hugh.barton,0,0,2017-06-13 16:42:33.805653,2017-06-13 16:42:33.857479 467,7,58,Qwardian Power Ring,https://placehold.it/600x100.png,http://heaneyhoppe.biz/kattie.howe,0,0,2017-06-13 16:42:33.871106,2017-06-13 16:42:33.914541 468,7,58,Illumination,https://placehold.it/600x100.png,http://heaneyhoppe.biz/colin_pagac,0,0,2017-06-13 16:42:33.932708,2017-06-13 16:42:33.99832 469,7,58,Clairvoyance,https://placehold.it/600x100.png,http://heaneyhoppe.biz/jalon,0,0,2017-06-13 16:42:34.011514,2017-06-13 16:42:34.061216 470,7,58,Stamina,https://placehold.it/600x100.png,http://heaneyhoppe.biz/erin_corkery,0,0,2017-06-13 16:42:34.075924,2017-06-13 16:42:34.138604 471,7,58,Omnitrix,https://placehold.it/600x100.png,http://heaneyhoppe.biz/erica,0,0,2017-06-13 16:42:34.151339,2017-06-13 16:42:34.218896 472,7,58,Size Changing,https://placehold.it/600x100.png,http://heaneyhoppe.biz/rebeka,0,0,2017-06-13 16:42:34.232041,2017-06-13 16:42:34.294881 473,7,58,Mind Control Resistance,https://placehold.it/600x100.png,http://heaneyhoppe.biz/chaim_cummerata,0,0,2017-06-13 16:42:34.310382,2017-06-13 16:42:34.3921 126,3,18,Telekinesis,https://placehold.it/600x100.png,http://feeneyking.org/eve_jones,0,0,2017-06-13 16:42:03.809046,2017-06-13 16:42:03.893836 127,3,18,Cryokinesis,https://placehold.it/600x100.png,http://feeneyking.org/anderson,0,0,2017-06-13 16:42:03.907537,2017-06-13 16:42:03.949249 128,3,18,Energy Armor,https://placehold.it/600x100.png,http://feeneyking.org/teagan,0,0,2017-06-13 16:42:03.963271,2017-06-13 16:42:04.042923 129,3,18,Natural Armor,https://placehold.it/600x100.png,http://feeneyking.org/sidney.parker,0,0,2017-06-13 16:42:04.057484,2017-06-13 16:42:04.140856 130,3,18,Natural Weapons,https://placehold.it/600x100.png,http://feeneyking.org/ettie,0,0,2017-06-13 16:42:04.156869,2017-06-13 16:42:04.233039 131,3,18,Banish,https://placehold.it/600x100.png,http://feeneyking.org/susanna.anderson,0,0,2017-06-13 16:42:04.246535,2017-06-13 16:42:04.313148 132,3,18,Camouflage,https://placehold.it/600x100.png,http://feeneyking.org/mitchel_doyle,0,0,2017-06-13 16:42:04.330694,2017-06-13 16:42:04.415642 133,3,18,Stamina,https://placehold.it/600x100.png,http://feeneyking.org/nina,0,0,2017-06-13 16:42:04.429892,2017-06-13 16:42:04.487595 134,3,18,Portal Creation,https://placehold.it/600x100.png,http://feeneyking.org/sterling,0,0,2017-06-13 16:42:04.502836,2017-06-13 16:42:04.565242 135,3,18,Illusions,https://placehold.it/600x100.png,http://feeneyking.org/tyrese_cole,0,0,2017-06-13 16:42:04.581041,2017-06-13 16:42:04.655729 136,3,19,Natural Weapons,https://placehold.it/600x100.png,http://paucek.biz/lamont,0,0,2017-06-13 16:42:04.686014,2017-06-13 16:42:04.743433 137,3,19,Probability Manipulation,https://placehold.it/600x100.png,http://paucek.biz/chesley.heathcote,0,0,2017-06-13 16:42:04.757826,2017-06-13 16:42:04.844329 138,3,19,Telekinesis,https://placehold.it/600x100.png,http://paucek.biz/houston,0,0,2017-06-13 16:42:04.860048,2017-06-13 16:42:04.945833 139,3,19,Cross-Dimensional Travel,https://placehold.it/600x100.png,http://paucek.biz/giovanny_marquardt,0,0,2017-06-13 16:42:04.960175,2017-06-13 16:42:05.021506 140,3,19,Underwater breathing,https://placehold.it/600x100.png,http://paucek.biz/lynn,0,0,2017-06-13 16:42:05.03948,2017-06-13 16:42:05.111025 141,3,19,Physical Anomaly,https://placehold.it/600x100.png,http://paucek.biz/ned,0,0,2017-06-13 16:42:05.127417,2017-06-13 16:42:05.204073 142,3,19,Illusions,https://placehold.it/600x100.png,http://paucek.biz/federico,0,0,2017-06-13 16:42:05.216885,2017-06-13 16:42:05.303466 143,3,19,Cloaking,https://placehold.it/600x100.png,http://paucek.biz/arden,0,0,2017-06-13 16:42:05.317382,2017-06-13 16:42:05.3914 144,3,19,Teleportation,https://placehold.it/600x100.png,http://paucek.biz/jayne,0,0,2017-06-13 16:42:05.406914,2017-06-13 16:42:05.4594 145,3,19,Longevity,https://placehold.it/600x100.png,http://paucek.biz/jeffery_hilll,0,0,2017-06-13 16:42:05.476436,2017-06-13 16:42:05.547683 146,3,20,Echokinesis,https://placehold.it/600x100.png,http://gutmann.com/gladys.mckenzie,0,0,2017-06-13 16:42:05.579384,2017-06-13 16:42:05.626708 147,3,20,Elasticity,https://placehold.it/600x100.png,http://gutmann.com/mohammed.bashirian,0,0,2017-06-13 16:42:05.642674,2017-06-13 16:42:05.705984 148,3,20,Substance Secretion,https://placehold.it/600x100.png,http://gutmann.com/penelope.corkery,0,0,2017-06-13 16:42:05.720665,2017-06-13 16:42:05.780659 149,3,20,Animal Attributes,https://placehold.it/600x100.png,http://gutmann.com/efren.torp,0,0,2017-06-13 16:42:05.797722,2017-06-13 16:42:05.865206 150,3,20,Self-Sustenance,https://placehold.it/600x100.png,http://gutmann.com/lavon.schuster,0,0,2017-06-13 16:42:05.877946,2017-06-13 16:42:05.941094 151,3,20,Vision - Infrared,https://placehold.it/600x100.png,http://gutmann.com/naomie,0,0,2017-06-13 16:42:05.955333,2017-06-13 16:42:06.02343 152,3,20,Orbing,https://placehold.it/600x100.png,http://gutmann.com/tad,0,0,2017-06-13 16:42:06.038315,2017-06-13 16:42:06.10998 153,3,20,Clairvoyance,https://placehold.it/600x100.png,http://gutmann.com/tyrique,0,0,2017-06-13 16:42:06.122497,2017-06-13 16:42:06.179545 154,3,20,Geokinesis,https://placehold.it/600x100.png,http://gutmann.com/winnifred,0,0,2017-06-13 16:42:06.191793,2017-06-13 16:42:06.246337 155,3,20,Vision - Heat,https://placehold.it/600x100.png,http://gutmann.com/carlos,0,0,2017-06-13 16:42:06.261953,2017-06-13 16:42:06.31341 156,3,21,Omnitrix,https://placehold.it/600x100.png,http://dare.io/carroll,0,0,2017-06-13 16:42:06.341123,2017-06-13 16:42:06.387925 157,3,21,Super Strength,https://placehold.it/600x100.png,http://dare.io/lisandro,0,0,2017-06-13 16:42:06.40589,2017-06-13 16:42:06.465698 158,3,21,Astral Travel,https://placehold.it/600x100.png,http://dare.io/laverna.feil,0,0,2017-06-13 16:42:06.481883,2017-06-13 16:42:06.558315 159,3,21,Vision - Microscopic,https://placehold.it/600x100.png,http://dare.io/macey,0,0,2017-06-13 16:42:06.574333,2017-06-13 16:42:06.634626 160,3,21,Timeframe Control,https://placehold.it/600x100.png,http://dare.io/hudson,0,0,2017-06-13 16:42:06.648858,2017-06-13 16:42:06.724307 161,3,21,The Force,https://placehold.it/600x100.png,http://dare.io/zora_mosciski,0,0,2017-06-13 16:42:06.739704,2017-06-13 16:42:06.793667 162,3,22,Energy Absorption,https://placehold.it/600x100.png,http://jerde.net/charlotte,0,0,2017-06-13 16:42:06.820607,2017-06-13 16:42:06.869208 163,3,22,Omnitrix,https://placehold.it/600x100.png,http://jerde.net/abelardo,0,0,2017-06-13 16:42:06.882798,2017-06-13 16:42:06.955947 164,3,22,Enhanced Touch,https://placehold.it/600x100.png,http://jerde.net/coralie_steuber,0,0,2017-06-13 16:42:06.974081,2017-06-13 16:42:07.040691 165,3,22,Physical Anomaly,https://placehold.it/600x100.png,http://jerde.net/leopoldo.hamill,0,0,2017-06-13 16:42:07.056526,2017-06-13 16:42:07.139174 166,3,22,Photographic Reflexes,https://placehold.it/600x100.png,http://jerde.net/jody_becker,0,0,2017-06-13 16:42:07.155253,2017-06-13 16:42:07.219213 167,3,22,Molecular Immobilization,https://placehold.it/600x100.png,http://jerde.net/marguerite,0,0,2017-06-13 16:42:07.233126,2017-06-13 16:42:07.320242 168,3,22,The Force,https://placehold.it/600x100.png,http://jerde.net/americo,0,0,2017-06-13 16:42:07.334481,2017-06-13 16:42:07.407912 169,3,22,Gliding,https://placehold.it/600x100.png,http://jerde.net/alejandrin,0,0,2017-06-13 16:42:07.421185,2017-06-13 16:42:07.486228 170,3,22,Vision - Infrared,https://placehold.it/600x100.png,http://jerde.net/judson.beier,0,0,2017-06-13 16:42:07.500938,2017-06-13 16:42:07.559577 171,3,22,Power Cosmic,https://placehold.it/600x100.png,http://jerde.net/uriah.stehr,0,0,2017-06-13 16:42:07.574059,2017-06-13 16:42:07.622781 172,3,23,Stealth,https://placehold.it/600x100.png,http://pfefferjenkins.co/janick_crist,0,0,2017-06-13 16:42:07.651837,2017-06-13 16:42:07.740867 173,3,23,Insanity,https://placehold.it/600x100.png,http://pfefferjenkins.co/francesca.schimmel,0,0,2017-06-13 16:42:07.75642,2017-06-13 16:42:07.834273 174,3,23,Cross-Dimensional Travel,https://placehold.it/600x100.png,http://pfefferjenkins.co/gene,0,0,2017-06-13 16:42:07.84881,2017-06-13 16:42:07.932184 175,3,23,Intangibility,https://placehold.it/600x100.png,http://pfefferjenkins.co/jade.waelchi,0,0,2017-06-13 16:42:07.948427,2017-06-13 16:42:08.021368 176,3,23,Terrakinesis,https://placehold.it/600x100.png,http://pfefferjenkins.co/helen.weinat,0,0,2017-06-13 16:42:08.037737,2017-06-13 16:42:08.110735 177,3,23,Hydrokinesis,https://placehold.it/600x100.png,http://pfefferjenkins.co/berniece,0,0,2017-06-13 16:42:08.128169,2017-06-13 16:42:08.199452 178,3,24,Illumination,https://placehold.it/600x100.png,http://yostrunolfon.info/jakayla_towne,0,0,2017-06-13 16:42:08.231144,2017-06-13 16:42:08.31204 179,3,24,Reality Warping,https://placehold.it/600x100.png,http://yostrunolfon.info/maud.haley,0,0,2017-06-13 16:42:08.325801,2017-06-13 16:42:08.387129 180,3,24,Elemental Transmogrification,https://placehold.it/600x100.png,http://yostrunolfon.info/nat,0,0,2017-06-13 16:42:08.401184,2017-06-13 16:42:08.451499 181,3,24,Elasticity,https://placehold.it/600x100.png,http://yostrunolfon.info/effie.hudson,0,0,2017-06-13 16:42:08.464181,2017-06-13 16:42:08.51401 182,3,24,Technopath/Cyberpath,https://placehold.it/600x100.png,http://yostrunolfon.info/jackson,0,0,2017-06-13 16:42:08.528378,2017-06-13 16:42:08.585054 183,3,24,Astral Travel,https://placehold.it/600x100.png,http://yostrunolfon.info/abraham,0,0,2017-06-13 16:42:08.59736,2017-06-13 16:42:08.671576 184,3,24,Changing Armor,https://placehold.it/600x100.png,http://yostrunolfon.info/pasquale.streich,0,0,2017-06-13 16:42:08.684427,2017-06-13 16:42:08.734522 185,3,25,Molecular Combustion,https://placehold.it/600x100.png,http://mueller.biz/enrique,0,0,2017-06-13 16:42:08.777743,2017-06-13 16:42:08.868558 186,3,25,The Force,https://placehold.it/600x100.png,http://mueller.biz/marcelina.stracke,0,0,2017-06-13 16:42:08.885621,2017-06-13 16:42:08.955883 187,3,25,Duplication,https://placehold.it/600x100.png,http://mueller.biz/margaret,0,0,2017-06-13 16:42:08.973615,2017-06-13 16:42:09.045737 188,3,25,Gravitokinesis,https://placehold.it/600x100.png,http://mueller.biz/schuyler,0,0,2017-06-13 16:42:09.064227,2017-06-13 16:42:09.125999 189,3,25,Heat Generation,https://placehold.it/600x100.png,http://mueller.biz/norene.treutel,0,0,2017-06-13 16:42:09.142832,2017-06-13 16:42:09.219551 190,3,25,Illumination,https://placehold.it/600x100.png,http://mueller.biz/roy.hamill,0,0,2017-06-13 16:42:09.238599,2017-06-13 16:42:09.284909 191,3,25,Mind Control Resistance,https://placehold.it/600x100.png,http://mueller.biz/lavinia.willms,0,0,2017-06-13 16:42:09.300653,2017-06-13 16:42:09.384197 192,3,25,Time Travel,https://placehold.it/600x100.png,http://mueller.biz/arianna_nolan,0,0,2017-06-13 16:42:09.401135,2017-06-13 16:42:09.494179 193,3,25,Levitation,https://placehold.it/600x100.png,http://mueller.biz/aliya.ebert,0,0,2017-06-13 16:42:09.511977,2017-06-13 16:42:09.589935 194,3,25,Cold Resistance,https://placehold.it/600x100.png,http://mueller.biz/beau,0,0,2017-06-13 16:42:09.606813,2017-06-13 16:42:09.696173 336,6,43,Cold Resistance,https://placehold.it/600x100.png,http://trantow.info/allene,0,0,2017-06-13 16:42:22.54747,2017-06-13 16:42:22.619579 337,6,43,Energy Resistance,https://placehold.it/600x100.png,http://trantow.info/zackery,0,0,2017-06-13 16:42:22.633866,2017-06-13 16:42:22.708133 338,6,43,Portal Creation,https://placehold.it/600x100.png,http://trantow.info/valentine.schultz,0,0,2017-06-13 16:42:22.723297,2017-06-13 16:42:22.802213 339,6,43,Weapon-based Powers,https://placehold.it/600x100.png,http://trantow.info/lea_bartell,0,0,2017-06-13 16:42:22.816411,2017-06-13 16:42:22.883117 340,6,43,Umbrakinesis,https://placehold.it/600x100.png,http://trantow.info/conner.buckridge,0,0,2017-06-13 16:42:22.89868,2017-06-13 16:42:22.961783 341,6,43,Sonar,https://placehold.it/600x100.png,http://trantow.info/sedrick.kuphal,0,0,2017-06-13 16:42:22.97485,2017-06-13 16:42:23.040428 342,6,43,Elasticity,https://placehold.it/600x100.png,http://trantow.info/patricia,0,0,2017-06-13 16:42:23.051936,2017-06-13 16:42:23.121264 343,6,43,Danger Sense,https://placehold.it/600x100.png,http://trantow.info/patience_baumbach,0,0,2017-06-13 16:42:23.134886,2017-06-13 16:42:23.187287 344,6,43,Physical Anomaly,https://placehold.it/600x100.png,http://trantow.info/laurine,0,0,2017-06-13 16:42:23.200617,2017-06-13 16:42:23.279953 345,6,44,Technopath/Cyberpath,https://placehold.it/600x100.png,http://schmittkeler.org/joey,0,0,2017-06-13 16:42:23.307659,2017-06-13 16:42:23.361456 346,6,44,Hydrokinesis,https://placehold.it/600x100.png,http://schmittkeler.org/terrell,0,0,2017-06-13 16:42:23.37575,2017-06-13 16:42:23.426955 347,6,44,Separation,https://placehold.it/600x100.png,http://schmittkeler.org/constantin.torphy,0,0,2017-06-13 16:42:23.443194,2017-06-13 16:42:23.504532 348,6,44,Radiation Immunity,https://placehold.it/600x100.png,http://schmittkeler.org/kameron_bruen,0,0,2017-06-13 16:42:23.518493,2017-06-13 16:42:23.602878 349,6,44,Dexterity,https://placehold.it/600x100.png,http://schmittkeler.org/marques_vonrueden,0,0,2017-06-13 16:42:23.617001,2017-06-13 16:42:23.687554 350,6,44,Vision - Microscopic,https://placehold.it/600x100.png,http://schmittkeler.org/kaleb,0,0,2017-06-13 16:42:23.70204,2017-06-13 16:42:23.762041 351,6,45,Radiation Control,https://placehold.it/600x100.png,http://kingwalsh.org/kavon.murray,0,0,2017-06-13 16:42:23.788708,2017-06-13 16:42:23.838007 352,6,45,Accuracy,https://placehold.it/600x100.png,http://kingwalsh.org/jennie.little,0,0,2017-06-13 16:42:23.852718,2017-06-13 16:42:23.932507 353,6,45,Photokinesis,https://placehold.it/600x100.png,http://kingwalsh.org/dereck,0,0,2017-06-13 16:42:23.945676,2017-06-13 16:42:24.047288 354,6,45,Animal Control,https://placehold.it/600x100.png,http://kingwalsh.org/yasmine_purdy,0,0,2017-06-13 16:42:24.060147,2017-06-13 16:42:24.130783 355,6,45,Animal Control,https://placehold.it/600x100.png,http://kingwalsh.org/royal.wuckert,0,0,2017-06-13 16:42:24.145601,2017-06-13 16:42:24.210153 356,6,45,Geokinesis,https://placehold.it/600x100.png,http://kingwalsh.org/angelita,0,0,2017-06-13 16:42:24.222364,2017-06-13 16:42:24.315466 357,6,46,Substance Secretion,https://placehold.it/600x100.png,http://maggio.name/tina_vandervort,0,0,2017-06-13 16:42:24.3455,2017-06-13 16:42:24.398655 358,6,46,Molecular Manipulation,https://placehold.it/600x100.png,http://maggio.name/brennon,0,0,2017-06-13 16:42:24.411667,2017-06-13 16:42:24.467224 359,6,46,Enhanced Touch,https://placehold.it/600x100.png,http://maggio.name/genesis_sawayn,0,0,2017-06-13 16:42:24.480213,2017-06-13 16:42:24.544346 360,6,46,Energy Constructs,https://placehold.it/600x100.png,http://maggio.name/cydney,0,0,2017-06-13 16:42:24.558557,2017-06-13 16:42:24.611128 361,6,46,Substance Secretion,https://placehold.it/600x100.png,http://maggio.name/bridgette,0,0,2017-06-13 16:42:24.625051,2017-06-13 16:42:24.675576 362,6,46,Atmokinesis,https://placehold.it/600x100.png,http://maggio.name/ayana,0,0,2017-06-13 16:42:24.690354,2017-06-13 16:42:24.737523 363,6,46,Photokinesis,https://placehold.it/600x100.png,http://maggio.name/mavis,0,0,2017-06-13 16:42:24.754694,2017-06-13 16:42:24.824135 364,6,46,Regeneration,https://placehold.it/600x100.png,http://maggio.name/margaretta,0,0,2017-06-13 16:42:24.836943,2017-06-13 16:42:25.205806 365,6,46,Size Changing,https://placehold.it/600x100.png,http://maggio.name/edwina_brakus,0,0,2017-06-13 16:42:25.220389,2017-06-13 16:42:25.286963 366,6,46,Fire Resistance,https://placehold.it/600x100.png,http://maggio.name/pauline_stanton,0,0,2017-06-13 16:42:25.301557,2017-06-13 16:42:25.359201 367,6,47,Possession,https://placehold.it/600x100.png,http://keebler.name/serenity,0,0,2017-06-13 16:42:25.389001,2017-06-13 16:42:25.443057 368,6,47,Anti-Gravity,https://placehold.it/600x100.png,http://keebler.name/zula,0,0,2017-06-13 16:42:25.462868,2017-06-13 16:42:25.507943 369,6,47,Cross-Dimensional Awareness,https://placehold.it/600x100.png,http://keebler.name/cleo,0,0,2017-06-13 16:42:25.522579,2017-06-13 16:42:25.576707 370,6,47,Longevity,https://placehold.it/600x100.png,http://keebler.name/brannon_donnelly,0,0,2017-06-13 16:42:25.590399,2017-06-13 16:42:25.661514 371,6,47,Vision - X-Ray,https://placehold.it/600x100.png,http://keebler.name/angelo_rippin,0,0,2017-06-13 16:42:25.674631,2017-06-13 16:42:25.742287 372,6,47,Energy Manipulation,https://placehold.it/600x100.png,http://keebler.name/dario,0,0,2017-06-13 16:42:25.756989,2017-06-13 16:42:25.836984 373,6,47,Self-Sustenance,https://placehold.it/600x100.png,http://keebler.name/felipe.oreilly,0,0,2017-06-13 16:42:25.853061,2017-06-13 16:42:25.916555 374,6,47,Teleportation,https://placehold.it/600x100.png,http://keebler.name/enos,0,0,2017-06-13 16:42:25.931299,2017-06-13 16:42:26.003001 375,6,47,Astral Travel,https://placehold.it/600x100.png,http://keebler.name/paula,0,0,2017-06-13 16:42:26.01667,2017-06-13 16:42:26.082117 376,6,47,Vision - Cryo,https://placehold.it/600x100.png,http://keebler.name/mandy,0,0,2017-06-13 16:42:26.097139,2017-06-13 16:42:26.158086 377,6,48,Power Suit,https://placehold.it/600x100.png,http://haag.io/melvin,0,0,2017-06-13 16:42:26.184963,2017-06-13 16:42:26.244193 378,6,48,Telekinesis,https://placehold.it/600x100.png,http://haag.io/yolanda,0,0,2017-06-13 16:42:26.259224,2017-06-13 16:42:26.3272 379,6,48,Photographic Reflexes,https://placehold.it/600x100.png,http://haag.io/bella,0,0,2017-06-13 16:42:26.341133,2017-06-13 16:42:26.413343 380,6,48,Power Cosmic,https://placehold.it/600x100.png,http://haag.io/jordan_dach,0,0,2017-06-13 16:42:26.429433,2017-06-13 16:42:26.499291 381,6,48,Astral Projection,https://placehold.it/600x100.png,http://haag.io/johanna,0,0,2017-06-13 16:42:26.514178,2017-06-13 16:42:26.582176 382,6,48,Biokinesis,https://placehold.it/600x100.png,http://haag.io/dorothea,0,0,2017-06-13 16:42:26.596519,2017-06-13 16:42:26.665202 383,6,48,Lantern Power Ring,https://placehold.it/600x100.png,http://haag.io/lonie.kuphal,0,0,2017-06-13 16:42:26.680984,2017-06-13 16:42:26.750244 384,6,48,Invisibility,https://placehold.it/600x100.png,http://haag.io/gino,0,0,2017-06-13 16:42:26.76451,2017-06-13 16:42:26.821239 385,6,48,Super Strength,https://placehold.it/600x100.png,http://haag.io/mikel.jakubowski,0,0,2017-06-13 16:42:26.83487,2017-06-13 16:42:26.922366 386,6,48,Web Creation,https://placehold.it/600x100.png,http://haag.io/urban_block,0,0,2017-06-13 16:42:26.939612,2017-06-13 16:42:27.033537 387,6,49,Astral Projection,https://placehold.it/600x100.png,http://pollich.net/heaven,0,0,2017-06-13 16:42:27.062225,2017-06-13 16:42:27.132599 388,6,49,Gravitokinesis,https://placehold.it/600x100.png,http://pollich.net/mittie,0,0,2017-06-13 16:42:27.151083,2017-06-13 16:42:27.225265 389,6,49,Enhanced Senses,https://placehold.it/600x100.png,http://pollich.net/keyshawn.hauck,0,0,2017-06-13 16:42:27.244387,2017-06-13 16:42:27.308748 390,6,49,Vision - Heat,https://placehold.it/600x100.png,http://pollich.net/clementine_littel,0,0,2017-06-13 16:42:27.323688,2017-06-13 16:42:27.390503 391,6,49,Super Strength,https://placehold.it/600x100.png,http://pollich.net/jaren,0,0,2017-06-13 16:42:27.405382,2017-06-13 16:42:27.473043 392,6,49,Banish,https://placehold.it/600x100.png,http://pollich.net/adrienne.eichmann,0,0,2017-06-13 16:42:27.486391,2017-06-13 16:42:27.560828 393,6,49,Toxin and Disease Resistance,https://placehold.it/600x100.png,http://pollich.net/dell_kuvalis,0,0,2017-06-13 16:42:27.575557,2017-06-13 16:42:27.633337 394,6,49,Force Fields,https://placehold.it/600x100.png,http://pollich.net/carroll_schamberger,0,0,2017-06-13 16:42:27.647365,2017-06-13 16:42:27.701748 395,6,49,Animation,https://placehold.it/600x100.png,http://pollich.net/alice,0,0,2017-06-13 16:42:27.717997,2017-06-13 16:42:27.778461 396,6,50,Vision - Infrared,https://placehold.it/600x100.png,http://ratke.name/john,0,0,2017-06-13 16:42:27.807222,2017-06-13 16:42:27.895033 397,6,50,Longevity,https://placehold.it/600x100.png,http://ratke.name/natalia,0,0,2017-06-13 16:42:27.915154,2017-06-13 16:42:28.011595 398,6,50,Vision - Infrared,https://placehold.it/600x100.png,http://ratke.name/torrey.balistreri,0,0,2017-06-13 16:42:28.02799,2017-06-13 16:42:28.098461 399,6,50,Cold Resistance,https://placehold.it/600x100.png,http://ratke.name/christine,0,0,2017-06-13 16:42:28.114082,2017-06-13 16:42:28.180828 400,6,50,Heat Resistance,https://placehold.it/600x100.png,http://ratke.name/robb.kub,0,0,2017-06-13 16:42:28.195933,2017-06-13 16:42:28.253087 401,6,50,Resurrection,https://placehold.it/600x100.png,http://ratke.name/angeline,0,0,2017-06-13 16:42:28.266944,2017-06-13 16:42:28.331602 402,6,50,Healing Factor,https://placehold.it/600x100.png,http://ratke.name/hildegard_rowe,0,0,2017-06-13 16:42:28.344034,2017-06-13 16:42:28.401812 52,2,8,Possession,https://placehold.it/600x100.png,http://renner.org/dwight,0,0,2017-06-13 16:41:57.254284,2017-06-13 16:41:57.320506 53,2,8,Power Nullifier,https://placehold.it/600x100.png,http://renner.org/fae,0,0,2017-06-13 16:41:57.336567,2017-06-13 16:41:57.423065 54,2,8,Healing Factor,https://placehold.it/600x100.png,http://renner.org/derrick.shields,0,0,2017-06-13 16:41:57.438646,2017-06-13 16:41:57.515199 55,2,8,Probability Manipulation,https://placehold.it/600x100.png,http://renner.org/ole,0,0,2017-06-13 16:41:57.528455,2017-06-13 16:41:57.599034 56,2,8,Thermokinesis,https://placehold.it/600x100.png,http://renner.org/leopold.dibbert,0,0,2017-06-13 16:41:57.611384,2017-06-13 16:41:57.670595 57,2,8,Power Augmentation,https://placehold.it/600x100.png,http://renner.org/mohammed_schneider,0,0,2017-06-13 16:41:57.684605,2017-06-13 16:41:57.755518 58,2,9,Animation,https://placehold.it/600x100.png,http://rath.net/alda,0,0,2017-06-13 16:41:57.783929,2017-06-13 16:41:57.864984 59,2,9,Seismic Power,https://placehold.it/600x100.png,http://rath.net/jamarcus,0,0,2017-06-13 16:41:57.881804,2017-06-13 16:41:57.945827 60,2,9,Apotheosis,https://placehold.it/600x100.png,http://rath.net/felicity,0,0,2017-06-13 16:41:57.959157,2017-06-13 16:41:58.035709 61,2,9,Cloaking,https://placehold.it/600x100.png,http://rath.net/holden,0,0,2017-06-13 16:41:58.050128,2017-06-13 16:41:58.122997 62,2,9,Cryokinesis,https://placehold.it/600x100.png,http://rath.net/lonnie,0,0,2017-06-13 16:41:58.136565,2017-06-13 16:41:58.190822 63,2,9,Omnilingualism,https://placehold.it/600x100.png,http://rath.net/bernard_rempel,0,0,2017-06-13 16:41:58.211175,2017-06-13 16:41:58.302561 64,2,10,Duplication,https://placehold.it/600x100.png,http://davis.io/mason,0,0,2017-06-13 16:41:58.334676,2017-06-13 16:41:58.411971 65,2,10,Power Absorption,https://placehold.it/600x100.png,http://davis.io/norval_gleichner,0,0,2017-06-13 16:41:58.429915,2017-06-13 16:41:58.514911 66,2,10,Chlorokinesis,https://placehold.it/600x100.png,http://davis.io/onie,0,0,2017-06-13 16:41:58.533822,2017-06-13 16:41:58.606623 67,2,10,Aerokinesis,https://placehold.it/600x100.png,http://davis.io/paris,0,0,2017-06-13 16:41:58.621321,2017-06-13 16:41:58.702946 68,2,10,Flight,https://placehold.it/600x100.png,http://davis.io/abdullah,0,0,2017-06-13 16:41:58.721529,2017-06-13 16:41:58.773786 69,2,10,Radiation Immunity,https://placehold.it/600x100.png,http://davis.io/alfonso,0,0,2017-06-13 16:41:58.788581,2017-06-13 16:41:58.858677 70,2,10,Cross-Dimensional Travel,https://placehold.it/600x100.png,http://davis.io/chris,0,0,2017-06-13 16:41:58.87368,2017-06-13 16:41:58.958767 71,2,10,Electrical Transport,https://placehold.it/600x100.png,http://davis.io/crystal_macgyver,0,0,2017-06-13 16:41:58.974751,2017-06-13 16:41:59.041036 72,2,11,Magic Resistance,https://placehold.it/600x100.png,http://murphy.com/toney,0,0,2017-06-13 16:41:59.067952,2017-06-13 16:41:59.11737 73,2,11,Sonar,https://placehold.it/600x100.png,http://murphy.com/lorenza_cummings,0,0,2017-06-13 16:41:59.131646,2017-06-13 16:41:59.205101 74,2,11,Radiation Absorption,https://placehold.it/600x100.png,http://murphy.com/vida,0,0,2017-06-13 16:41:59.21827,2017-06-13 16:41:59.271412 75,2,11,Vision - Infrared,https://placehold.it/600x100.png,http://murphy.com/jean.wiegand,0,0,2017-06-13 16:41:59.284775,2017-06-13 16:41:59.374152 76,2,11,Omnipotence,https://placehold.it/600x100.png,http://murphy.com/alia_gleason,0,0,2017-06-13 16:41:59.386785,2017-06-13 16:41:59.457859 77,2,11,Resurrection,https://placehold.it/600x100.png,http://murphy.com/nina,0,0,2017-06-13 16:41:59.473568,2017-06-13 16:41:59.5416 78,2,12,Vision - Telescopic,https://placehold.it/600x100.png,http://heller.com/van,0,0,2017-06-13 16:41:59.565909,2017-06-13 16:41:59.647627 79,2,12,Time Travel,https://placehold.it/600x100.png,http://heller.com/ayla,0,0,2017-06-13 16:41:59.665771,2017-06-13 16:41:59.714259 80,2,12,Symbiote Costume,https://placehold.it/600x100.png,http://heller.com/kirsten,0,0,2017-06-13 16:41:59.729906,2017-06-13 16:41:59.785213 81,2,12,Umbrakinesis,https://placehold.it/600x100.png,http://heller.com/simeon,0,0,2017-06-13 16:41:59.799354,2017-06-13 16:41:59.867031 82,2,12,Jump,https://placehold.it/600x100.png,http://heller.com/polly_kerluke,0,0,2017-06-13 16:41:59.883848,2017-06-13 16:41:59.961005 83,2,12,Enhanced Memory,https://placehold.it/600x100.png,http://heller.com/cordelia_braun,0,0,2017-06-13 16:41:59.975421,2017-06-13 16:42:00.041536 84,2,12,Telekinesis,https://placehold.it/600x100.png,http://heller.com/antonio,0,0,2017-06-13 16:42:00.055007,2017-06-13 16:42:00.120545 85,2,12,Vaporising Beams,https://placehold.it/600x100.png,http://heller.com/lindsay,0,0,2017-06-13 16:42:00.134856,2017-06-13 16:42:00.178369 86,2,13,Stamina,https://placehold.it/600x100.png,http://kunze.biz/victoria.rowe,0,0,2017-06-13 16:42:00.207665,2017-06-13 16:42:00.2827 87,2,13,Fire Resistance,https://placehold.it/600x100.png,http://kunze.biz/marcos.jacobi,0,0,2017-06-13 16:42:00.297293,2017-06-13 16:42:00.376913 88,2,13,Probability Manipulation,https://placehold.it/600x100.png,http://kunze.biz/roy.oconnell,0,0,2017-06-13 16:42:00.390982,2017-06-13 16:42:00.451078 89,2,13,Intelligence,https://placehold.it/600x100.png,http://kunze.biz/carroll,0,0,2017-06-13 16:42:00.46511,2017-06-13 16:42:00.547278 90,2,13,Banish,https://placehold.it/600x100.png,http://kunze.biz/manuel.bosco,0,0,2017-06-13 16:42:00.563048,2017-06-13 16:42:00.645009 91,2,13,The Force,https://placehold.it/600x100.png,http://kunze.biz/ebba_mcdermott,0,0,2017-06-13 16:42:00.663591,2017-06-13 16:42:00.716856 92,2,13,Possession,https://placehold.it/600x100.png,http://kunze.biz/cristian,0,0,2017-06-13 16:42:00.729407,2017-06-13 16:42:00.797286 93,2,13,Clairvoyance,https://placehold.it/600x100.png,http://kunze.biz/norbert.jakubowski,0,0,2017-06-13 16:42:00.812074,2017-06-13 16:42:00.889136 94,2,13,Bullet Time,https://placehold.it/600x100.png,http://kunze.biz/efrain_mcclure,0,0,2017-06-13 16:42:00.905391,2017-06-13 16:42:00.971897 95,2,13,Pyrokinesis,https://placehold.it/600x100.png,http://kunze.biz/jee,0,0,2017-06-13 16:42:00.987795,2017-06-13 16:42:01.05311 96,2,14,Vitakinesis,https://placehold.it/600x100.png,http://conn.biz/minerva_skiles,0,0,2017-06-13 16:42:01.07882,2017-06-13 16:42:01.146378 97,2,14,Self-Sustenance,https://placehold.it/600x100.png,http://conn.biz/gillian_erdman,0,0,2017-06-13 16:42:01.163896,2017-06-13 16:42:01.239235 98,2,14,Telepathy Resistance,https://placehold.it/600x100.png,http://conn.biz/kamren,0,0,2017-06-13 16:42:01.254879,2017-06-13 16:42:01.310392 99,2,14,Radar Sense,https://placehold.it/600x100.png,http://conn.biz/anabelle.bahringer,0,0,2017-06-13 16:42:01.324379,2017-06-13 16:42:01.381244 100,2,14,Energy Resistance,https://placehold.it/600x100.png,http://conn.biz/moie,0,0,2017-06-13 16:42:01.397053,2017-06-13 16:42:01.46703 101,2,14,Intangibility,https://placehold.it/600x100.png,http://conn.biz/vena_rosenbaum,0,0,2017-06-13 16:42:01.482865,2017-06-13 16:42:01.550407 102,2,15,Electrical Transport,https://placehold.it/600x100.png,http://hansen.net/georgiana_stokes,0,0,2017-06-13 16:42:01.580298,2017-06-13 16:42:01.661925 103,2,15,Phasing,https://placehold.it/600x100.png,http://hansen.net/freeda,0,0,2017-06-13 16:42:01.678295,2017-06-13 16:42:01.743686 104,2,15,Pyrokinesis,https://placehold.it/600x100.png,http://hansen.net/martina_casper,0,0,2017-06-13 16:42:01.756784,2017-06-13 16:42:01.830191 105,2,15,Elasticity,https://placehold.it/600x100.png,http://hansen.net/kane,0,0,2017-06-13 16:42:01.845698,2017-06-13 16:42:01.915996 106,2,15,Animal Control,https://placehold.it/600x100.png,http://hansen.net/murphy,0,0,2017-06-13 16:42:01.931099,2017-06-13 16:42:01.988951 107,2,15,Enhanced Senses,https://placehold.it/600x100.png,http://hansen.net/griffin_gleason,0,0,2017-06-13 16:42:02.004217,2017-06-13 16:42:02.082313 108,2,15,Animation,https://placehold.it/600x100.png,http://hansen.net/shemar,0,0,2017-06-13 16:42:02.096371,2017-06-13 16:42:02.154741 109,2,15,Omnipotence,https://placehold.it/600x100.png,http://hansen.net/clark.davis,0,0,2017-06-13 16:42:02.16873,2017-06-13 16:42:02.235534 110,2,15,Symbiote Costume,https://placehold.it/600x100.png,http://hansen.net/camron_wintheiser,0,0,2017-06-13 16:42:02.249841,2017-06-13 16:42:02.304958 111,2,16,Regeneration,https://placehold.it/600x100.png,http://vandervortbayer.co/linnie,0,0,2017-06-13 16:42:02.336109,2017-06-13 16:42:02.412784 112,2,16,Power Sense,https://placehold.it/600x100.png,http://vandervortbayer.co/katrina,0,0,2017-06-13 16:42:02.4283,2017-06-13 16:42:02.508817 113,2,16,Mind Blast,https://placehold.it/600x100.png,http://vandervortbayer.co/adalberto,0,0,2017-06-13 16:42:02.522811,2017-06-13 16:42:02.572642 114,2,16,Invisibility,https://placehold.it/600x100.png,http://vandervortbayer.co/annabel,0,0,2017-06-13 16:42:02.588723,2017-06-13 16:42:02.656822 115,2,16,Summoning,https://placehold.it/600x100.png,http://vandervortbayer.co/leonardo.gutkowski,0,0,2017-06-13 16:42:02.671477,2017-06-13 16:42:02.728037 116,2,16,Duplication,https://placehold.it/600x100.png,http://vandervortbayer.co/scot,0,0,2017-06-13 16:42:02.741896,2017-06-13 16:42:02.81086 117,2,16,Astral Trap,https://placehold.it/600x100.png,http://vandervortbayer.co/dora.hackett,0,0,2017-06-13 16:42:02.823574,2017-06-13 16:42:02.893394 118,2,16,Vision - Cryo,https://placehold.it/600x100.png,http://vandervortbayer.co/kitty_beahan,0,0,2017-06-13 16:42:02.907004,2017-06-13 16:42:02.987976 119,2,17,Hyperkinesis,https://placehold.it/600x100.png,http://donnellypredovic.info/sean,0,0,2017-06-13 16:42:03.019672,2017-06-13 16:42:03.094867 120,2,17,Absorption,https://placehold.it/600x100.png,http://donnellypredovic.info/sonny,0,0,2017-06-13 16:42:03.108791,2017-06-13 16:42:03.186472 121,2,17,Power Nullifier,https://placehold.it/600x100.png,http://donnellypredovic.info/lilliana_watsica,0,0,2017-06-13 16:42:03.204521,2017-06-13 16:42:03.261758 122,2,17,Geokinesis,https://placehold.it/600x100.png,http://donnellypredovic.info/king.heaney,0,0,2017-06-13 16:42:03.276369,2017-06-13 16:42:03.330756 123,2,17,Cold Resistance,https://placehold.it/600x100.png,http://donnellypredovic.info/ansel_schuster,0,0,2017-06-13 16:42:03.345959,2017-06-13 16:42:03.427401 124,2,17,Cross-Dimensional Awareness,https://placehold.it/600x100.png,http://donnellypredovic.info/kayden_ziemann,0,0,2017-06-13 16:42:03.442446,2017-06-13 16:42:03.506236 125,2,17,Teleportation,https://placehold.it/600x100.png,http://donnellypredovic.info/daisy.wuckert,0,0,2017-06-13 16:42:03.51888,2017-06-13 16:42:03.606551 ================================================ FILE: src/test/regress/data/agg.data ================================================ 56 7.8 100 99.097 0 0.09561 42 324.78 ================================================ FILE: src/test/regress/data/agg_type.data ================================================ 1.0 2.343 23.44 2.0 3.544 324.4 3.0 3.5666 2332.9 4.5 6.34343 2332.9 ================================================ FILE: src/test/regress/data/array_types.csv ================================================ "{1,2,3}","{1,2,3}","{a,b,c}" {},{},{} "{-2147483648,2147483647}","{-9223372036854775808,9223372036854775807}","{""""}" ================================================ FILE: src/test/regress/data/campaigns.csv ================================================ 59,8,Gorilla Grodd I,cost_per_click,running,935,,2017-06-13 16:42:34.594681,2017-06-13 16:42:34.594681 60,8,The Doppelganger the Hunter,cost_per_impression,paused,8975,,2017-06-13 16:42:35.207761,2017-06-13 16:42:35.207761 61,8,Falcon Brain,cost_per_click,running,8428,,2017-06-13 16:42:36.097274,2017-06-13 16:42:36.097274 62,8,Red Goku of Hearts,cost_per_click,archived,6531,,2017-06-13 16:42:36.693675,2017-06-13 16:42:36.693675 63,8,Green Rambo,cost_per_click,archived,2451,,2017-06-13 16:42:37.136045,2017-06-13 16:42:37.136045 64,8,Winter Soldier Dragon,cost_per_click,running,1752,,2017-06-13 16:42:37.897502,2017-06-13 16:42:37.897502 65,8,Overtkill,cost_per_click,running,8232,,2017-06-13 16:42:38.489345,2017-06-13 16:42:38.489345 1,1,Mandarin,cost_per_impression,paused,7895,,2017-06-13 16:41:52.707903,2017-06-13 16:41:52.707903 2,1,Red Leader,cost_per_click,paused,4891,,2017-06-13 16:41:53.341818,2017-06-13 16:41:53.341818 3,1,Morlun,cost_per_impression,paused,1182,,2017-06-13 16:41:53.819101,2017-06-13 16:41:53.819101 4,1,Illustrious Absorbing IX,cost_per_click,archived,4763,,2017-06-13 16:41:54.407686,2017-06-13 16:41:54.407686 5,1,Captain Azrael Woman,cost_per_click,paused,9005,,2017-06-13 16:41:54.902299,2017-06-13 16:41:54.902299 6,1,Cyborg Hancock II,cost_per_click,archived,8604,,2017-06-13 16:41:55.498523,2017-06-13 16:41:55.498523 7,1,Ultra Justice Eyes,cost_per_click,paused,6982,,2017-06-13 16:41:56.367163,2017-06-13 16:41:56.367163 34,5,Supah Bat Lord,cost_per_impression,archived,4307,,2017-06-13 16:42:16.142651,2017-06-13 16:42:16.142651 35,5,Shocker,cost_per_impression,paused,3914,,2017-06-13 16:42:16.647064,2017-06-13 16:42:16.647064 36,5,Giant Toxin Girl,cost_per_click,running,4057,,2017-06-13 16:42:17.481264,2017-06-13 16:42:17.481264 37,5,Supah Scorpion Wolf,cost_per_impression,running,7420,,2017-06-13 16:42:18.269976,2017-06-13 16:42:18.269976 38,5,Lyja,cost_per_click,archived,2369,,2017-06-13 16:42:18.821974,2017-06-13 16:42:18.821974 39,5,Captain Annihilus,cost_per_click,archived,9352,,2017-06-13 16:42:19.310638,2017-06-13 16:42:19.310638 40,5,Agent Rachel Pirzad of Hearts,cost_per_click,archived,3538,,2017-06-13 16:42:20.009205,2017-06-13 16:42:20.009205 41,5,Giant Maya Herrera I,cost_per_click,archived,3712,,2017-06-13 16:42:20.769031,2017-06-13 16:42:20.769031 42,5,Dark Groot Boy,cost_per_impression,running,6496,,2017-06-13 16:42:21.576833,2017-06-13 16:42:21.576833 26,4,Magnificent Cy-Gor Thirteen,cost_per_impression,paused,5900,,2017-06-13 16:42:09.902924,2017-06-13 16:42:09.902924 27,4,Wolverine Dragon,cost_per_impression,running,3713,,2017-06-13 16:42:10.866183,2017-06-13 16:42:10.866183 28,4,Goku,cost_per_click,paused,1960,,2017-06-13 16:42:11.441643,2017-06-13 16:42:11.441643 29,4,Ripcord Claw,cost_per_impression,archived,6994,,2017-06-13 16:42:12.198094,2017-06-13 16:42:12.198094 30,4,Dark Aqua,cost_per_impression,paused,2663,,2017-06-13 16:42:13.113204,2017-06-13 16:42:13.113204 31,4,Warbird Girl,cost_per_click,paused,4425,,2017-06-13 16:42:13.834842,2017-06-13 16:42:13.834842 32,4,Cyborg Rogue,cost_per_click,archived,6281,,2017-06-13 16:42:14.730579,2017-06-13 16:42:14.730579 33,4,Mr Mera III,cost_per_impression,paused,2192,,2017-06-13 16:42:15.47731,2017-06-13 16:42:15.47731 51,7,Giant Chameleon Man,cost_per_click,running,3176,,2017-06-13 16:42:28.591585,2017-06-13 16:42:28.591585 52,7,Tinkerer Strike,cost_per_impression,running,5198,,2017-06-13 16:42:29.277142,2017-06-13 16:42:29.277142 53,7,Zatanna,cost_per_impression,running,150,,2017-06-13 16:42:30.065958,2017-06-13 16:42:30.065958 54,7,Giant Question Claw,cost_per_impression,paused,2819,,2017-06-13 16:42:30.886949,2017-06-13 16:42:30.886949 55,7,Question,cost_per_impression,paused,1642,,2017-06-13 16:42:31.728957,2017-06-13 16:42:31.728957 56,7,Annihilus,cost_per_impression,running,137,,2017-06-13 16:42:32.519651,2017-06-13 16:42:32.519651 57,7,Venom,cost_per_impression,archived,8340,,2017-06-13 16:42:32.999851,2017-06-13 16:42:32.999851 58,7,The Longshot Strike,cost_per_click,paused,8977,,2017-06-13 16:42:33.791989,2017-06-13 16:42:33.791989 18,3,Red Siren,cost_per_click,paused,2065,,2017-06-13 16:42:03.79514,2017-06-13 16:42:03.79514 19,3,Titan the Fated,cost_per_impression,archived,7617,,2017-06-13 16:42:04.670896,2017-06-13 16:42:04.670896 20,3,Metamorpho,cost_per_click,paused,6950,,2017-06-13 16:42:05.56454,2017-06-13 16:42:05.56454 21,3,Mr Forge the Hunter,cost_per_impression,running,3085,,2017-06-13 16:42:06.326961,2017-06-13 16:42:06.326961 22,3,Supah Scorpia,cost_per_click,running,480,,2017-06-13 16:42:06.808002,2017-06-13 16:42:06.808002 23,3,Giant Maya Herrera,cost_per_impression,archived,8309,,2017-06-13 16:42:07.636385,2017-06-13 16:42:07.636385 24,3,Mandarin,cost_per_impression,archived,3492,,2017-06-13 16:42:08.215445,2017-06-13 16:42:08.215445 25,3,The Huntress,cost_per_click,paused,5367,,2017-06-13 16:42:08.748961,2017-06-13 16:42:08.748961 43,6,Mystique Brain,cost_per_click,paused,1342,,2017-06-13 16:42:22.534609,2017-06-13 16:42:22.534609 44,6,Mr Doomsday Brain,cost_per_impression,paused,1958,,2017-06-13 16:42:23.293625,2017-06-13 16:42:23.293625 45,6,Ultra Monarch,cost_per_click,archived,5374,,2017-06-13 16:42:23.775571,2017-06-13 16:42:23.775571 46,6,Supah Curse,cost_per_click,running,7865,,2017-06-13 16:42:24.331765,2017-06-13 16:42:24.331765 47,6,Thing,cost_per_impression,running,2707,,2017-06-13 16:42:25.374042,2017-06-13 16:42:25.374042 48,6,Red Colin Wagner,cost_per_click,paused,9873,,2017-06-13 16:42:26.172275,2017-06-13 16:42:26.172275 49,6,Green Speedball,cost_per_click,archived,1032,,2017-06-13 16:42:27.047698,2017-06-13 16:42:27.047698 50,6,Boom Boom I,cost_per_impression,archived,9896,,2017-06-13 16:42:27.792139,2017-06-13 16:42:27.792139 8,2,Meltdown of Hearts,cost_per_impression,paused,4293,,2017-06-13 16:41:57.24128,2017-06-13 16:41:57.24128 9,2,Bloodaxe I,cost_per_click,running,5130,,2017-06-13 16:41:57.770862,2017-06-13 16:41:57.770862 10,2,Blink Claw,cost_per_impression,archived,1374,,2017-06-13 16:41:58.319465,2017-06-13 16:41:58.319465 11,2,Electro Wolf,cost_per_impression,running,7194,,2017-06-13 16:41:59.053908,2017-06-13 16:41:59.053908 12,2,Polaris,cost_per_impression,paused,4748,,2017-06-13 16:41:59.553231,2017-06-13 16:41:59.553231 13,2,Omega Red,cost_per_click,paused,748,,2017-06-13 16:42:00.19296,2017-06-13 16:42:00.19296 14,2,Spawn,cost_per_click,archived,5445,,2017-06-13 16:42:01.064903,2017-06-13 16:42:01.064903 15,2,Blink,cost_per_click,archived,3643,,2017-06-13 16:42:01.564672,2017-06-13 16:42:01.564672 16,2,Spectre Fist,cost_per_click,running,3841,,2017-06-13 16:42:02.320521,2017-06-13 16:42:02.320521 17,2,Carnage Knight,cost_per_impression,paused,7221,,2017-06-13 16:42:03.006055,2017-06-13 16:42:03.006055 ================================================ FILE: src/test/regress/data/clicks.csv ================================================ 1,1,1,2017-06-12 08:44:51,http://pollich.info/dameon.bayer,,130.70.206.43,"{""location"": ""MR"", ""is_mobile"": true}" 2,1,1,2017-01-21 06:53:55,http://lednerfranecki.io/ward,,176.239.100.21,"{""location"": ""BQ"", ""is_mobile"": true}" 3,1,1,2017-05-02 11:06:11,http://krajcik.com/eldred,,102.154.56.47,"{""location"": ""VC"", ""is_mobile"": false}" 4,1,1,2017-01-27 21:44:27,http://schulist.info/fae_stoltenberg,,161.226.163.83,"{""location"": ""MF"", ""is_mobile"": true}" 5,1,1,2017-05-07 07:43:20,http://ullrich.io/jeie,,176.239.100.21,"{""location"": ""RW"", ""is_mobile"": false}" 6,1,2,2017-05-23 03:07:37,http://friesenpredovic.io/wilfredo,,136.239.79.72,"{""location"": ""MW"", ""is_mobile"": true}" 7,1,2,2017-04-18 05:41:12,http://homenick.name/broderick.dibbert,,148.127.195.70,"{""location"": ""PA"", ""is_mobile"": false}" 8,1,3,2017-02-01 21:22:59,http://boehm.io/dedrick_konopelski,,79.251.206.155,"{""location"": ""PN"", ""is_mobile"": true}" 9,1,3,2017-05-16 15:06:39,http://haagkertzmann.com/gregory,,197.29.133.181,"{""location"": ""IQ"", ""is_mobile"": false}" 10,1,3,2017-03-04 07:14:14,http://leuschkeko.co/daija,,197.29.133.181,"{""location"": ""TC"", ""is_mobile"": false}" 11,1,3,2016-12-30 19:38:21,http://windler.org/jerod,,218.4.220.11,"{""location"": ""ML"", ""is_mobile"": true}" 12,1,3,2016-12-19 01:38:41,http://white.org/genevieve.schuster,,3.232.171.243,"{""location"": ""FR"", ""is_mobile"": false}" 13,1,3,2017-03-28 15:17:07,http://dickens.com/kristy,,188.50.189.129,"{""location"": ""AR"", ""is_mobile"": false}" 14,1,3,2016-12-29 15:52:33,http://reynolds.co/virgie,,224.86.119.31,"{""location"": ""IS"", ""is_mobile"": false}" 15,1,3,2017-03-23 00:41:26,http://boyer.info/mercedes,,2.153.179.122,"{""location"": ""AR"", ""is_mobile"": false}" 16,1,3,2017-05-30 12:47:03,http://ondrickanader.biz/arianna,,184.181.183.61,"{""location"": ""AG"", ""is_mobile"": false}" 17,1,4,2017-02-26 16:39:46,http://batz.biz/jaida,,4.45.90.244,"{""location"": ""MP"", ""is_mobile"": true}" 18,1,4,2017-04-04 16:32:24,http://kovacek.info/kyle,,171.63.44.91,"{""location"": ""KE"", ""is_mobile"": false}" 19,1,4,2017-03-18 05:24:03,http://blicklang.info/krystel.hoppe,,90.200.195.135,"{""location"": ""MW"", ""is_mobile"": true}" 20,1,4,2017-04-28 01:30:31,http://oconnell.com/alvis,,123.108.32.214,"{""location"": ""CI"", ""is_mobile"": true}" 21,1,4,2017-02-09 19:17:21,http://renner.co/nettie.olson,,56.166.226.70,"{""location"": ""DK"", ""is_mobile"": true}" 22,1,4,2017-04-29 03:14:02,http://cormiergibson.io/fausto,,84.123.247.153,"{""location"": ""IR"", ""is_mobile"": false}" 23,1,5,2017-05-04 11:11:30,http://champlin.org/thora_harber,,46.164.41.83,"{""location"": ""MC"", ""is_mobile"": true}" 24,1,5,2017-06-08 11:27:19,http://ornzboncak.net/orlo_hoeger,,214.18.177.74,"{""location"": ""NF"", ""is_mobile"": true}" 25,1,5,2017-03-13 05:20:57,http://bosco.org/sienna,,9.35.183.96,"{""location"": ""IT"", ""is_mobile"": true}" 26,1,5,2017-02-12 02:34:13,http://ziemegutkowski.com/ro_boehm,,127.116.151.208,"{""location"": ""ET"", ""is_mobile"": true}" 27,1,5,2017-04-21 05:03:22,http://marquardt.org/okey,,87.92.164.74,"{""location"": ""AQ"", ""is_mobile"": true}" 28,1,5,2017-04-08 02:33:38,http://friesen.info/timothy,,247.232.194.24,"{""location"": ""AE"", ""is_mobile"": true}" 29,1,5,2017-03-05 22:37:27,http://doyle.biz/rosamond_rutherford,,58.113.118.10,"{""location"": ""BJ"", ""is_mobile"": true}" 30,1,5,2017-05-31 18:47:21,http://harveydouglas.com/caesar_kozey,,182.225.238.154,"{""location"": ""HU"", ""is_mobile"": false}" 31,1,5,2017-04-27 05:41:05,http://mayert.info/thora.boyer,,207.201.119.79,"{""location"": ""TT"", ""is_mobile"": false}" 32,1,5,2017-04-15 16:39:33,http://jacobson.info/aliza.brekke,,248.160.88.37,"{""location"": ""WS"", ""is_mobile"": true}" 33,1,5,2017-04-25 23:44:51,http://binscollins.info/rhoda.cormier,,229.202.70.147,"{""location"": ""HN"", ""is_mobile"": true}" 34,1,5,2017-02-11 02:50:31,http://wyman.name/tina_toy,,58.113.118.10,"{""location"": ""NG"", ""is_mobile"": true}" 35,1,5,2017-05-24 09:28:53,http://green.com/saige,,230.175.93.46,"{""location"": ""DO"", ""is_mobile"": false}" 36,1,5,2017-04-29 17:31:56,http://kuhncartwright.com/fanny_durgan,,56.50.239.44,"{""location"": ""IT"", ""is_mobile"": false}" 37,1,5,2017-05-25 23:28:02,http://ortiz.info/rico_waelchi,,21.199.217.190,"{""location"": ""NC"", ""is_mobile"": false}" 38,1,5,2017-05-04 14:00:30,http://millerdaugherty.biz/leora.reynolds,,222.252.41.76,"{""location"": ""KP"", ""is_mobile"": false}" 39,1,6,2017-04-11 09:49:39,http://maggiofunk.com/edwina.gorczany,,220.221.95.245,"{""location"": ""UM"", ""is_mobile"": false}" 40,1,6,2017-05-24 22:13:06,http://kohler.name/freddy,,63.200.15.158,"{""location"": ""MD"", ""is_mobile"": true}" 41,1,6,2017-03-28 18:45:25,http://steuber.co/jeie.olson,,156.44.156.135,"{""location"": ""ER"", ""is_mobile"": true}" 42,1,6,2017-05-06 23:39:13,http://larsonkihn.net/kavon.morar,,32.185.126.197,"{""location"": ""MU"", ""is_mobile"": false}" 43,1,6,2016-12-23 09:10:18,http://wilkinson.com/wellington.murphy,,32.88.25.183,"{""location"": ""TL"", ""is_mobile"": false}" 44,1,6,2017-05-26 11:07:53,http://yost.net/kennedy,,160.154.45.137,"{""location"": ""LS"", ""is_mobile"": true}" 45,1,6,2017-03-15 15:03:19,http://dach.net/delbert,,97.77.185.183,"{""location"": ""NR"", ""is_mobile"": true}" 46,1,6,2017-04-11 19:16:34,http://fayweber.com/jamal,,121.26.155.40,"{""location"": ""FM"", ""is_mobile"": false}" 47,1,6,2017-05-01 06:41:41,http://weimannhodkiewicz.name/gus,,29.84.70.245,"{""location"": ""TG"", ""is_mobile"": true}" 48,1,6,2017-05-07 11:02:47,http://deckow.org/ayden,,12.89.127.107,"{""location"": ""SE"", ""is_mobile"": false}" 49,1,6,2017-01-15 13:55:22,http://cartwright.org/rosanna,,29.84.70.245,"{""location"": ""AE"", ""is_mobile"": false}" 50,1,6,2016-12-15 12:18:53,http://spencerratke.org/harmon_paucek,,153.142.61.93,"{""location"": ""OM"", ""is_mobile"": true}" 51,1,6,2017-01-21 02:38:38,http://mclaughlin.org/gladys_okon,,106.187.40.128,"{""location"": ""MR"", ""is_mobile"": false}" 52,1,6,2017-02-23 16:45:25,http://lebsack.name/earnest_crist,,68.29.181.197,"{""location"": ""GF"", ""is_mobile"": true}" 53,1,6,2017-05-29 09:52:34,http://jenkins.net/gianni,,134.203.215.223,"{""location"": ""GD"", ""is_mobile"": false}" 54,1,7,2016-12-30 18:45:55,http://parkerlueilwitz.com/franz,,172.246.70.245,"{""location"": ""VE"", ""is_mobile"": true}" 55,1,7,2017-05-21 01:28:20,http://hartmannhauck.net/mia,,194.216.49.11,"{""location"": ""GB"", ""is_mobile"": false}" 4995,8,474,2016-12-18 02:40:30,http://bogisichtorphy.co/susie,0.0000000000,89.148.71.17,"{""location"": ""GM"", ""is_mobile"": false}" 4996,8,474,2017-05-24 02:14:09,http://gutkowski.org/leonardo,0.0000000000,103.6.226.229,"{""location"": ""SE"", ""is_mobile"": true}" 4997,8,474,2017-03-29 20:07:44,http://barrowpencer.name/darby_nolan,1.0000000000,243.212.193.204,"{""location"": ""GA"", ""is_mobile"": true}" 4998,8,474,2017-04-23 03:12:09,http://bins.org/isadore,1.0000000000,241.111.144.230,"{""location"": ""SL"", ""is_mobile"": true}" 4999,8,474,2017-04-17 12:14:23,http://feestsipes.org/jackson.kunde,0.0000000000,22.178.2.131,"{""location"": ""CU"", ""is_mobile"": true}" 5000,8,474,2017-03-01 10:21:41,http://weberstrosin.io/aaron,0.0000000000,73.125.121.41,"{""location"": ""AM"", ""is_mobile"": true}" 5001,8,474,2017-03-23 00:53:10,http://schmitt.com/rodrick,0.0000000000,211.17.120.135,"{""location"": ""NC"", ""is_mobile"": true}" 5002,8,474,2016-12-24 01:53:19,http://abernathybradtke.com/ocie,1.0000000000,89.148.71.17,"{""location"": ""SN"", ""is_mobile"": true}" 5003,8,474,2017-02-25 04:50:28,http://osinskibeier.biz/celia.weinat,1.0000000000,146.91.118.53,"{""location"": ""LC"", ""is_mobile"": true}" 5004,8,474,2017-02-11 19:04:55,http://kohlerhegmann.io/rogers,0.0000000000,89.148.71.17,"{""location"": ""KY"", ""is_mobile"": true}" 5005,8,475,2016-12-24 17:11:28,http://graham.io/josiane,2.0000000000,173.158.19.64,"{""location"": ""KN"", ""is_mobile"": true}" 5006,8,475,2017-04-05 07:30:47,http://roob.net/major.lang,0.0000000000,161.145.20.150,"{""location"": ""BQ"", ""is_mobile"": true}" 5007,8,475,2017-03-16 18:18:32,http://baumbach.co/annalise.fritsch,0.0000000000,159.227.94.130,"{""location"": ""IS"", ""is_mobile"": true}" 5008,8,475,2017-01-05 03:32:15,http://lebsack.io/carol,2.0000000000,18.66.152.170,"{""location"": ""WF"", ""is_mobile"": true}" 5009,8,475,2017-04-06 23:16:22,http://thielroberts.biz/ari_blick,0.0000000000,200.53.155.78,"{""location"": ""SX"", ""is_mobile"": false}" 5010,8,475,2017-01-30 15:32:32,http://feest.co/keeley,0.0000000000,140.171.144.91,"{""location"": ""CK"", ""is_mobile"": false}" 5011,8,475,2016-12-31 18:31:09,http://donnelly.io/elvera,0.0000000000,232.67.218.242,"{""location"": ""LT"", ""is_mobile"": false}" 5012,8,476,2017-05-25 22:20:45,http://huelcorkery.co/lisandro.tillman,2.0000000000,204.246.119.117,"{""location"": ""AG"", ""is_mobile"": true}" 5013,8,476,2017-05-18 20:39:35,http://hayeshartmann.name/aliya,0.0000000000,21.173.187.17,"{""location"": ""AE"", ""is_mobile"": false}" 5014,8,476,2017-02-09 08:41:54,http://schaden.biz/enrico,2.0000000000,197.239.245.27,"{""location"": ""RW"", ""is_mobile"": false}" 5015,8,477,2017-04-11 15:53:48,http://hayesboyle.name/lou,2.0000000000,196.30.65.147,"{""location"": ""TK"", ""is_mobile"": true}" 5016,8,477,2017-01-07 02:59:49,http://funk.info/nedra.hoppe,1.0000000000,98.135.245.39,"{""location"": ""CV"", ""is_mobile"": true}" 5017,8,477,2017-02-13 23:12:29,http://reynolds.org/elmore_baumbach,0.0000000000,196.14.194.73,"{""location"": ""AI"", ""is_mobile"": false}" 5018,8,477,2017-02-24 02:06:34,http://moriette.net/rachel,1.0000000000,98.135.245.39,"{""location"": ""PY"", ""is_mobile"": true}" 5019,8,477,2017-05-05 17:48:53,http://kunzestracke.io/jaydon,2.0000000000,103.125.27.164,"{""location"": ""BV"", ""is_mobile"": true}" 5020,8,477,2017-05-15 04:45:50,http://witting.net/annamarie,0.0000000000,16.218.59.75,"{""location"": ""BR"", ""is_mobile"": false}" 5021,8,477,2017-05-03 01:33:18,http://gulgowski.info/augustus,2.0000000000,142.71.188.161,"{""location"": ""NR"", ""is_mobile"": true}" 5022,8,477,2017-03-11 16:36:36,http://ruel.co/manuela,1.0000000000,121.63.170.93,"{""location"": ""RS"", ""is_mobile"": true}" 5023,8,477,2017-03-20 02:43:12,http://rempel.co/kathleen,0.0000000000,172.40.227.12,"{""location"": ""NR"", ""is_mobile"": false}" 5024,8,477,2017-06-09 13:50:27,http://harber.com/elya_hammes,1.0000000000,93.234.91.159,"{""location"": ""MF"", ""is_mobile"": true}" 5025,8,477,2016-12-28 16:06:13,http://ruelrath.info/alysa,1.0000000000,172.40.227.12,"{""location"": ""BT"", ""is_mobile"": true}" 5026,8,477,2017-01-26 16:49:03,http://mcglynn.io/emilia,1.0000000000,238.97.190.84,"{""location"": ""BD"", ""is_mobile"": false}" 5027,8,478,2017-02-23 11:55:02,http://considinequitzon.info/liam_kaulke,1.0000000000,22.204.95.37,"{""location"": ""BJ"", ""is_mobile"": false}" 5028,8,478,2017-05-09 07:13:43,http://lubowitz.org/lori,0.0000000000,163.14.145.192,"{""location"": ""GQ"", ""is_mobile"": true}" 5029,8,478,2017-03-06 23:24:01,http://stoltenberg.biz/keith_conn,1.0000000000,249.206.51.98,"{""location"": ""AQ"", ""is_mobile"": true}" 5030,8,479,2017-02-01 22:02:28,http://dickenswehner.com/jamir_stamm,1.0000000000,217.133.182.65,"{""location"": ""NI"", ""is_mobile"": false}" 5031,8,479,2017-04-13 23:35:21,http://schroederthiel.info/abagail.johnston,0.0000000000,3.220.12.202,"{""location"": ""BD"", ""is_mobile"": false}" 5032,8,479,2017-03-12 18:12:24,http://wuckert.biz/napoleon,2.0000000000,3.12.27.210,"{""location"": ""QA"", ""is_mobile"": true}" 5033,8,479,2017-01-27 05:09:22,http://bartonbuckridge.info/lonnie,2.0000000000,210.157.215.166,"{""location"": ""VA"", ""is_mobile"": true}" 5034,8,479,2016-12-15 18:00:26,http://bogan.org/roie.boyer,1.0000000000,126.218.143.193,"{""location"": ""SG"", ""is_mobile"": false}" 5035,8,479,2017-06-07 05:39:10,http://pacocha.co/arlene_schimmel,0.0000000000,165.131.167.110,"{""location"": ""SX"", ""is_mobile"": true}" 5036,8,479,2017-02-17 03:29:05,http://vonkuphal.io/jamel.west,1.0000000000,28.162.133.101,"{""location"": ""BM"", ""is_mobile"": true}" 5037,8,479,2017-04-18 20:23:33,http://marvin.net/mallory_prosacco,1.0000000000,240.124.109.138,"{""location"": ""CK"", ""is_mobile"": true}" 5038,8,479,2016-12-16 17:12:34,http://walker.co/estevan,1.0000000000,242.218.26.164,"{""location"": ""HU"", ""is_mobile"": false}" 5039,8,479,2017-04-16 08:20:32,http://littel.name/dorthy,0.0000000000,229.216.76.98,"{""location"": ""MP"", ""is_mobile"": false}" 5040,8,479,2017-04-21 18:18:12,http://ortizstark.info/eliseo_rice,1.0000000000,231.178.99.86,"{""location"": ""PN"", ""is_mobile"": true}" 5041,8,479,2017-01-09 21:22:14,http://bogisich.info/arturo,1.0000000000,13.164.126.97,"{""location"": ""ER"", ""is_mobile"": false}" 5042,8,479,2017-04-27 23:58:06,http://casper.org/guadalupe,1.0000000000,75.35.50.27,"{""location"": ""KI"", ""is_mobile"": false}" 5043,8,479,2017-02-12 03:47:02,http://armstrong.info/cloyd,2.0000000000,109.235.104.62,"{""location"": ""MF"", ""is_mobile"": true}" 5044,8,479,2017-02-11 14:15:19,http://dietrichborer.org/hardy,2.0000000000,46.83.48.83,"{""location"": ""AE"", ""is_mobile"": true}" 56,1,7,2017-04-25 19:58:57,http://dicki.net/norberto.considine,,113.50.173.188,"{""location"": ""UZ"", ""is_mobile"": true}" 57,1,7,2017-03-29 10:38:36,http://torphyruecker.name/monty_hilpert,,86.198.75.31,"{""location"": ""GQ"", ""is_mobile"": true}" 58,1,7,2017-02-11 23:04:06,http://crooks.com/bruce_swift,,86.198.75.31,"{""location"": ""SJ"", ""is_mobile"": false}" 59,1,7,2016-12-15 07:09:56,http://mclaughlin.com/lucious_rolfson,,153.190.154.47,"{""location"": ""CU"", ""is_mobile"": false}" 60,1,7,2016-12-23 09:24:23,http://klinghettinger.org/josefa.ruecker,,190.210.172.134,"{""location"": ""TV"", ""is_mobile"": true}" 61,1,7,2017-01-02 09:08:45,http://jacobs.com/iac,,172.246.70.245,"{""location"": ""IT"", ""is_mobile"": true}" 62,1,7,2017-06-04 21:43:08,http://bins.biz/rickie.hauck,,137.27.224.89,"{""location"": ""BD"", ""is_mobile"": false}" 63,1,7,2017-01-06 15:20:56,http://walter.name/alice,,134.247.219.94,"{""location"": ""BO"", ""is_mobile"": false}" 64,1,7,2017-06-03 17:59:51,http://sauerleannon.net/raina_christiansen,,229.37.123.85,"{""location"": ""VN"", ""is_mobile"": true}" 65,1,7,2017-05-07 15:56:22,http://johnson.name/destiney.crist,,137.27.224.89,"{""location"": ""BT"", ""is_mobile"": true}" 66,1,7,2017-01-15 19:49:48,http://thiel.org/mae.towne,,177.26.106.161,"{""location"": ""MD"", ""is_mobile"": false}" 67,1,7,2017-06-10 02:14:40,http://markswaters.io/anabel_toy,,177.184.46.13,"{""location"": ""PH"", ""is_mobile"": true}" 68,1,7,2017-05-11 16:36:45,http://balistreri.info/chanelle,,153.190.154.47,"{""location"": ""BM"", ""is_mobile"": true}" 69,1,7,2017-03-23 08:49:50,http://treutel.io/forrest.abbott,,190.210.172.134,"{""location"": ""BN"", ""is_mobile"": false}" 70,1,7,2017-05-15 21:55:59,http://schuppemarks.info/joannie,,194.216.49.11,"{""location"": ""SN"", ""is_mobile"": true}" 71,1,7,2016-12-15 19:02:29,http://ziemann.info/eda_berge,,29.244.43.243,"{""location"": ""ES"", ""is_mobile"": false}" 72,1,7,2017-06-08 23:34:22,http://bernhard.com/kiara,,242.200.40.75,"{""location"": ""MM"", ""is_mobile"": false}" 73,1,8,2016-12-25 22:53:05,http://konopelski.net/molly,1.0000000000,153.220.141.226,"{""location"": ""NA"", ""is_mobile"": false}" 74,1,8,2016-12-31 21:30:52,http://zulauf.co/sally_wyman,1.0000000000,150.150.105.101,"{""location"": ""BL"", ""is_mobile"": true}" 75,1,8,2017-04-02 08:59:22,http://herzog.com/dorothea,0.0000000000,153.120.103.128,"{""location"": ""JO"", ""is_mobile"": false}" 76,1,8,2017-05-20 19:57:01,http://durgan.net/joan,0.0000000000,9.234.14.36,"{""location"": ""LK"", ""is_mobile"": true}" 77,1,8,2017-01-31 04:00:54,http://grady.info/stephanie_labadie,1.0000000000,156.96.140.159,"{""location"": ""JO"", ""is_mobile"": true}" 78,1,8,2017-05-22 04:47:38,http://rippin.name/jeica.roberts,1.0000000000,153.120.103.128,"{""location"": ""MG"", ""is_mobile"": true}" 79,1,8,2017-05-15 00:47:49,http://kihn.org/zackary.marvin,0.0000000000,92.49.186.30,"{""location"": ""CX"", ""is_mobile"": false}" 80,1,8,2016-12-19 07:26:28,http://murphy.io/betsy_schuppe,0.0000000000,184.47.164.254,"{""location"": ""UA"", ""is_mobile"": false}" 81,1,8,2017-05-21 08:23:29,http://stanton.co/milford,1.0000000000,110.208.27.149,"{""location"": ""JP"", ""is_mobile"": false}" 82,1,8,2017-03-25 04:22:35,http://schuppe.info/clinton,1.0000000000,184.47.164.254,"{""location"": ""VC"", ""is_mobile"": true}" 83,1,9,2017-04-06 17:45:08,http://boganschmeler.biz/amalia,1.0000000000,197.91.193.64,"{""location"": ""BE"", ""is_mobile"": false}" 84,1,9,2017-03-13 23:54:40,http://pfannerstillbarton.co/lois,0.0000000000,165.79.104.156,"{""location"": ""WS"", ""is_mobile"": true}" 85,1,9,2017-02-15 12:21:12,http://jenkins.name/mayra,0.0000000000,217.60.75.15,"{""location"": ""ID"", ""is_mobile"": true}" 86,1,9,2017-01-29 13:30:54,http://wiza.name/bernard_upton,1.0000000000,10.218.34.162,"{""location"": ""LK"", ""is_mobile"": true}" 87,1,10,2017-03-30 22:17:08,http://crist.net/scotty.rice,2.0000000000,57.155.245.154,"{""location"": ""ZW"", ""is_mobile"": false}" 88,1,10,2017-02-22 08:38:10,http://marks.com/sandra,3.0000000000,124.152.162.119,"{""location"": ""LY"", ""is_mobile"": false}" 89,1,10,2017-05-21 08:54:16,http://lynchjakubowski.name/elmo,0.0000000000,249.133.137.221,"{""location"": ""EE"", ""is_mobile"": false}" 90,1,10,2017-01-09 17:21:09,http://wuckert.net/terrence,0.0000000000,125.144.122.228,"{""location"": ""ML"", ""is_mobile"": true}" 91,1,10,2017-05-04 05:15:38,http://schultzspencer.co/chandler,1.0000000000,125.144.122.228,"{""location"": ""GD"", ""is_mobile"": false}" 92,1,11,2017-03-28 17:47:18,http://pollich.name/joel,0.0000000000,235.157.40.79,"{""location"": ""PR"", ""is_mobile"": true}" 93,1,11,2017-02-22 06:52:18,http://ferrywalsh.org/cecile.green,0.0000000000,235.157.40.79,"{""location"": ""TK"", ""is_mobile"": false}" 94,1,12,2017-01-08 20:58:48,http://becker.net/luisa,1.0000000000,212.122.53.48,"{""location"": ""BF"", ""is_mobile"": true}" 95,1,12,2017-02-11 06:45:01,http://greenfelder.net/aron_bartell,0.0000000000,34.131.97.72,"{""location"": ""GW"", ""is_mobile"": true}" 96,1,12,2017-04-14 12:57:57,http://konopelski.com/ruben_jenkins,0.0000000000,233.148.42.6,"{""location"": ""TZ"", ""is_mobile"": true}" 97,1,12,2017-05-17 00:45:30,http://langworth.io/joannie,1.0000000000,103.73.204.105,"{""location"": ""AX"", ""is_mobile"": false}" 98,1,12,2017-01-02 06:40:06,http://waelchi.io/tony_kulas,0.0000000000,132.252.152.28,"{""location"": ""CO"", ""is_mobile"": true}" 99,1,12,2017-03-08 11:42:42,http://sanford.info/sanford,0.0000000000,132.252.152.28,"{""location"": ""MQ"", ""is_mobile"": false}" 100,1,13,2017-03-12 13:57:04,http://lind.io/ryder_jacobs,1.0000000000,100.2.206.179,"{""location"": ""MG"", ""is_mobile"": false}" 101,1,13,2017-01-01 10:35:05,http://luettgen.com/jalyn,1.0000000000,64.108.234.232,"{""location"": ""VC"", ""is_mobile"": true}" 102,1,13,2017-03-16 14:09:52,http://gleichner.net/devante,0.0000000000,209.136.120.80,"{""location"": ""TM"", ""is_mobile"": false}" 103,1,13,2017-05-15 18:05:38,http://swift.com/katlynn.romaguera,0.0000000000,203.32.168.121,"{""location"": ""TF"", ""is_mobile"": false}" 104,1,13,2017-04-12 19:17:11,http://von.io/bailee,1.0000000000,226.6.126.75,"{""location"": ""GH"", ""is_mobile"": false}" 105,1,13,2017-05-25 15:48:32,http://yost.org/felix,1.0000000000,65.211.177.182,"{""location"": ""TL"", ""is_mobile"": false}" 106,1,13,2017-03-16 01:18:22,http://collier.net/carter,0.0000000000,48.167.63.144,"{""location"": ""NE"", ""is_mobile"": false}" 107,1,13,2017-03-20 08:44:24,http://mraz.net/verlie,1.0000000000,97.15.210.10,"{""location"": ""PE"", ""is_mobile"": true}" 108,1,13,2017-04-01 12:34:57,http://runolfsdottir.co/claudine,1.0000000000,203.158.111.152,"{""location"": ""AI"", ""is_mobile"": true}" 109,1,13,2017-03-27 07:12:39,http://bruenrenner.io/percy.denesik,0.0000000000,175.113.113.181,"{""location"": ""BY"", ""is_mobile"": true}" 5045,8,480,2017-05-13 17:32:59,http://spinka.name/muriel,0.0000000000,58.185.207.95,"{""location"": ""MY"", ""is_mobile"": false}" 5046,8,480,2017-05-14 10:52:08,http://ferrykirlin.co/cielo.olson,1.0000000000,142.208.114.223,"{""location"": ""DK"", ""is_mobile"": true}" 5047,8,480,2017-05-28 10:32:26,http://hermannhomenick.info/jadon.jacobson,0.0000000000,208.168.29.4,"{""location"": ""FM"", ""is_mobile"": false}" 5048,8,481,2017-06-09 21:50:23,http://hermann.org/helene.raynor,,15.254.59.75,"{""location"": ""CX"", ""is_mobile"": false}" 5049,8,481,2017-01-01 21:58:09,http://mertzrath.io/ursula,,17.178.107.53,"{""location"": ""VN"", ""is_mobile"": true}" 5050,8,481,2016-12-18 04:20:14,http://cummerataherzog.biz/meda,,146.85.159.244,"{""location"": ""HT"", ""is_mobile"": false}" 5051,8,481,2017-04-22 06:48:09,http://schultzfeeney.biz/reanna_gleason,,252.213.232.3,"{""location"": ""MS"", ""is_mobile"": true}" 5052,8,481,2017-02-26 07:03:48,http://gleichner.io/aurelia,,191.209.228.129,"{""location"": ""CL"", ""is_mobile"": false}" 5053,8,481,2017-02-17 00:00:15,http://gibson.info/rosalind,,178.98.49.155,"{""location"": ""NA"", ""is_mobile"": true}" 5054,8,481,2017-04-27 01:37:27,http://stracke.org/jan,,59.140.183.157,"{""location"": ""LI"", ""is_mobile"": false}" 5055,8,481,2017-06-05 06:40:05,http://satterfield.com/helene.johns,,67.142.20.145,"{""location"": ""GS"", ""is_mobile"": true}" 5056,8,481,2017-03-12 06:10:30,http://schiller.org/zack,,252.213.232.3,"{""location"": ""SN"", ""is_mobile"": false}" 5057,8,481,2017-03-15 04:21:23,http://funk.name/sammy_balistreri,,129.167.7.147,"{""location"": ""LA"", ""is_mobile"": true}" 5058,8,481,2017-05-01 05:36:12,http://wildermanemard.name/murl,,228.35.169.167,"{""location"": ""AQ"", ""is_mobile"": true}" 5059,8,481,2017-03-31 23:46:50,http://skilestillman.co/mathias,,36.113.42.69,"{""location"": ""GF"", ""is_mobile"": false}" 5060,8,482,2017-04-23 23:43:54,http://dickinsonokuneva.org/liam.emmerich,,144.233.252.168,"{""location"": ""LY"", ""is_mobile"": false}" 5061,8,483,2017-02-04 08:11:43,http://harberkautzer.io/brendan,,244.181.17.231,"{""location"": ""KW"", ""is_mobile"": false}" 5062,8,483,2016-12-25 02:56:39,http://stoltenbergmcclure.com/mariam,,2.116.204.19,"{""location"": ""TT"", ""is_mobile"": false}" 5063,8,483,2017-01-09 23:16:10,http://ruel.co/maxime,,194.190.28.160,"{""location"": ""KG"", ""is_mobile"": false}" 5064,8,483,2017-05-16 05:46:41,http://cormierziemann.info/aleen,,11.139.79.174,"{""location"": ""LU"", ""is_mobile"": true}" 5065,8,483,2017-05-09 04:58:01,http://schowalteryost.org/jedidiah_wolff,,83.169.207.2,"{""location"": ""LV"", ""is_mobile"": false}" 5066,8,483,2017-06-12 06:35:20,http://sanford.org/esperanza_mitchell,,198.137.173.251,"{""location"": ""HR"", ""is_mobile"": false}" 5067,8,483,2017-03-07 17:42:50,http://conroylarson.org/kolby.collier,,97.192.165.188,"{""location"": ""MU"", ""is_mobile"": true}" 5068,8,483,2016-12-31 18:55:17,http://okuneva.org/dustin_witting,,81.13.168.109,"{""location"": ""NI"", ""is_mobile"": false}" 5069,8,483,2017-06-06 09:38:26,http://frami.name/roberta,,97.63.168.166,"{""location"": ""TD"", ""is_mobile"": true}" 5070,8,483,2017-06-11 06:29:44,http://legros.org/monique,,83.169.207.2,"{""location"": ""PL"", ""is_mobile"": true}" 5071,8,483,2017-01-01 11:06:05,http://bins.info/camilla_kemmer,,58.38.35.70,"{""location"": ""IN"", ""is_mobile"": false}" 5072,8,483,2017-03-01 12:27:33,http://hicklekrajcik.io/jerald.herzog,,194.190.28.160,"{""location"": ""PE"", ""is_mobile"": false}" 5073,8,483,2017-05-08 17:27:02,http://hermann.com/olin.langosh,,131.123.28.145,"{""location"": ""CH"", ""is_mobile"": false}" 5074,8,483,2017-04-09 09:32:24,http://will.biz/selina.sawayn,,11.139.79.174,"{""location"": ""ES"", ""is_mobile"": false}" 5075,8,483,2017-05-11 12:13:07,http://tillman.com/erika,,97.63.168.166,"{""location"": ""SL"", ""is_mobile"": true}" 5076,8,483,2017-02-02 20:59:15,http://robel.co/esther_leuschke,,74.125.33.224,"{""location"": ""TH"", ""is_mobile"": false}" 5077,8,484,2017-01-24 09:27:22,http://jast.biz/richard,,194.223.148.51,"{""location"": ""IN"", ""is_mobile"": false}" 5078,8,484,2017-04-15 03:50:03,http://daugherty.org/augusta.koepp,,239.178.146.73,"{""location"": ""EE"", ""is_mobile"": false}" 5079,8,484,2017-04-18 14:07:01,http://eichmann.io/katelynn_rutherford,,68.134.239.18,"{""location"": ""LS"", ""is_mobile"": true}" 5080,8,484,2016-12-14 22:20:40,http://corkery.org/jena,,234.17.57.162,"{""location"": ""AE"", ""is_mobile"": false}" 5081,8,484,2017-06-02 05:49:03,http://bailey.biz/adeline,,194.223.148.51,"{""location"": ""ZM"", ""is_mobile"": true}" 5082,8,484,2017-01-31 05:49:07,http://abbott.org/emery.adams,,205.177.182.6,"{""location"": ""FJ"", ""is_mobile"": true}" 5083,8,484,2017-04-10 11:38:18,http://stehr.io/hilda_roob,,144.164.59.136,"{""location"": ""BQ"", ""is_mobile"": false}" 5084,8,484,2017-05-19 17:17:08,http://howell.net/marlene,,116.83.106.83,"{""location"": ""HR"", ""is_mobile"": true}" 5085,8,484,2017-04-21 04:53:41,http://murazik.name/kris,,64.204.178.136,"{""location"": ""GB"", ""is_mobile"": false}" 5086,8,484,2017-05-25 09:13:48,http://dachlang.info/obie,,81.78.145.202,"{""location"": ""PR"", ""is_mobile"": true}" 5087,8,484,2017-03-31 13:02:25,http://hettinger.co/ova_senger,,4.74.124.44,"{""location"": ""JP"", ""is_mobile"": false}" 5088,8,484,2017-03-29 11:43:38,http://gutkowskilind.com/jarrett,,168.214.174.152,"{""location"": ""VI"", ""is_mobile"": false}" 5089,8,485,2017-05-25 23:53:18,http://beer.com/george.frami,,235.137.76.4,"{""location"": ""RE"", ""is_mobile"": false}" 5090,8,485,2017-04-02 11:45:54,http://lubowitzbosco.info/orrin_nader,,28.170.58.79,"{""location"": ""GQ"", ""is_mobile"": false}" 5091,8,485,2016-12-22 18:38:36,http://watsicadicki.biz/devan,,14.201.188.4,"{""location"": ""IE"", ""is_mobile"": true}" 5092,8,485,2017-01-29 05:21:42,http://wehnertromp.biz/brice_witting,,6.103.81.39,"{""location"": ""MU"", ""is_mobile"": false}" 5093,8,485,2016-12-19 00:05:48,http://lesch.info/zoey,,197.19.140.105,"{""location"": ""BZ"", ""is_mobile"": true}" 5094,8,485,2017-04-18 08:42:03,http://hills.biz/coy,,227.44.4.9,"{""location"": ""LV"", ""is_mobile"": true}" 5095,8,485,2017-03-17 23:16:05,http://kemmerrosenbaum.com/adell,,180.22.172.27,"{""location"": ""MQ"", ""is_mobile"": true}" 5096,8,485,2016-12-15 07:01:22,http://yost.info/golden.gutkowski,,15.37.168.67,"{""location"": ""ID"", ""is_mobile"": false}" 5097,8,486,2017-01-02 17:17:52,http://hoeger.name/keagan,,114.169.202.150,"{""location"": ""KH"", ""is_mobile"": true}" 5098,8,486,2017-01-27 22:25:42,http://kozey.name/kaleigh_schimmel,,206.175.146.183,"{""location"": ""GA"", ""is_mobile"": true}" 5099,8,486,2017-06-12 23:30:37,http://andersonkub.co/thora,,158.5.248.197,"{""location"": ""GH"", ""is_mobile"": true}" 110,1,13,2017-04-06 01:18:58,http://johnstonbarton.net/roderick_altenwerth,1.0000000000,125.203.157.34,"{""location"": ""UM"", ""is_mobile"": true}" 111,1,13,2016-12-26 03:52:53,http://brekke.org/ora,1.0000000000,122.111.204.246,"{""location"": ""KM"", ""is_mobile"": false}" 112,1,13,2017-02-28 17:51:27,http://dibbert.info/frances,1.0000000000,165.219.39.78,"{""location"": ""HT"", ""is_mobile"": true}" 113,1,13,2017-03-22 22:02:07,http://collins.com/shawn,1.0000000000,53.194.85.149,"{""location"": ""CI"", ""is_mobile"": false}" 114,1,13,2017-04-17 22:36:27,http://casper.co/dagmar.macejkovic,1.0000000000,235.231.120.52,"{""location"": ""GT"", ""is_mobile"": false}" 115,1,13,2017-02-27 03:36:52,http://swift.org/katherine,0.0000000000,49.145.31.127,"{""location"": ""KE"", ""is_mobile"": false}" 116,1,13,2017-04-08 09:18:30,http://nitzsche.biz/kristin,1.0000000000,92.29.37.200,"{""location"": ""RU"", ""is_mobile"": false}" 117,1,13,2017-01-26 09:17:40,http://hegmann.biz/audreanne.lesch,1.0000000000,122.111.204.246,"{""location"": ""MT"", ""is_mobile"": false}" 118,1,14,2017-05-04 22:21:33,http://damore.biz/reggie.ledner,,4.16.78.229,"{""location"": ""CK"", ""is_mobile"": false}" 119,1,14,2016-12-16 20:26:47,http://schultz.com/addison,,91.144.186.14,"{""location"": ""ST"", ""is_mobile"": false}" 120,1,14,2017-06-08 02:58:56,http://rogahnlittle.com/breanne.ratke,,166.244.207.39,"{""location"": ""TO"", ""is_mobile"": false}" 121,1,14,2016-12-13 19:06:07,http://jaskolski.org/johnathon,,180.69.135.140,"{""location"": ""DE"", ""is_mobile"": true}" 122,1,14,2017-01-05 02:26:17,http://johnson.io/rosie,,77.215.102.213,"{""location"": ""HU"", ""is_mobile"": true}" 123,1,14,2017-01-05 15:07:15,http://bogisich.co/talia,,112.219.73.105,"{""location"": ""PT"", ""is_mobile"": false}" 124,1,14,2017-03-07 01:58:46,http://gusikowskihane.info/katheryn,,163.173.151.22,"{""location"": ""ME"", ""is_mobile"": true}" 125,1,15,2017-05-28 09:10:04,http://ernsernader.co/milford.ebert,,92.34.252.21,"{""location"": ""MC"", ""is_mobile"": true}" 126,1,15,2017-06-11 09:59:01,http://stammmertz.com/vernon.eichmann,,101.66.100.220,"{""location"": ""AO"", ""is_mobile"": true}" 127,1,15,2017-05-18 09:39:54,http://lehner.info/kay_pouros,,87.160.127.187,"{""location"": ""TW"", ""is_mobile"": false}" 128,1,15,2017-06-10 04:17:35,http://reinger.name/americo,,211.178.80.176,"{""location"": ""GF"", ""is_mobile"": false}" 129,1,15,2017-01-16 01:12:28,http://moen.net/dayne_herman,,92.34.252.21,"{""location"": ""AE"", ""is_mobile"": true}" 130,1,15,2017-05-12 20:22:43,http://gusikowski.co/davin,,211.178.80.176,"{""location"": ""OM"", ""is_mobile"": false}" 131,1,15,2017-05-29 04:05:26,http://gusikowskipurdy.org/arvel_zieme,,251.217.158.103,"{""location"": ""IE"", ""is_mobile"": false}" 132,1,15,2016-12-24 01:37:52,http://deckow.org/nikolas,,223.222.220.239,"{""location"": ""KI"", ""is_mobile"": true}" 133,1,15,2017-01-20 05:39:14,http://lynchhayes.name/laila,,161.84.223.200,"{""location"": ""BG"", ""is_mobile"": true}" 134,1,15,2017-03-06 21:33:28,http://rohanbashirian.net/braxton,,221.233.25.115,"{""location"": ""IS"", ""is_mobile"": true}" 135,1,15,2017-02-16 07:05:02,http://bartoletti.info/enid,,210.30.115.194,"{""location"": ""YT"", ""is_mobile"": true}" 136,1,15,2017-03-01 11:01:53,http://koelpin.org/eloise.hayes,,141.99.108.10,"{""location"": ""EH"", ""is_mobile"": false}" 137,1,15,2017-01-28 04:51:51,http://walsh.net/joey,,68.3.160.12,"{""location"": ""PH"", ""is_mobile"": true}" 138,1,15,2017-01-16 11:22:16,http://champlin.biz/mavis.stamm,,144.141.166.77,"{""location"": ""MS"", ""is_mobile"": false}" 139,1,15,2017-01-19 02:15:44,http://rath.org/brown,,211.178.80.176,"{""location"": ""HR"", ""is_mobile"": true}" 140,1,15,2017-02-13 21:27:11,http://ferryschmeler.io/breanna_graham,,178.109.238.51,"{""location"": ""NA"", ""is_mobile"": false}" 141,1,16,2017-01-19 22:05:02,http://greenfelderoberbrunner.com/antone,,208.239.95.189,"{""location"": ""CN"", ""is_mobile"": true}" 142,1,16,2016-12-18 15:47:21,http://padberggerhold.co/hilma,,111.159.232.223,"{""location"": ""DE"", ""is_mobile"": false}" 143,1,16,2017-06-13 02:24:11,http://schambergerankunding.org/kylie,,137.254.40.244,"{""location"": ""BE"", ""is_mobile"": true}" 144,1,16,2017-06-11 09:40:11,http://hegmann.net/jody,,160.86.178.11,"{""location"": ""MY"", ""is_mobile"": true}" 145,1,16,2017-05-27 18:49:50,http://bogan.name/rosario.fay,,164.78.228.42,"{""location"": ""WS"", ""is_mobile"": false}" 146,1,16,2016-12-31 13:21:15,http://schulist.com/malvina.lakin,,241.121.25.102,"{""location"": ""PT"", ""is_mobile"": true}" 147,1,16,2017-05-01 14:07:19,http://littelpadberg.net/max,,180.211.239.105,"{""location"": ""PA"", ""is_mobile"": true}" 148,1,16,2017-01-08 20:42:11,http://gutmann.net/jaleel.schmitt,,196.226.154.33,"{""location"": ""AX"", ""is_mobile"": false}" 149,1,16,2016-12-31 20:10:13,http://robel.co/noah,,200.251.12.43,"{""location"": ""MC"", ""is_mobile"": false}" 150,1,16,2016-12-21 02:10:46,http://murphy.name/tristin,,28.227.57.197,"{""location"": ""BJ"", ""is_mobile"": true}" 151,1,16,2017-01-06 08:49:00,http://little.name/edyth.wyman,,18.4.135.176,"{""location"": ""GI"", ""is_mobile"": true}" 152,1,16,2017-05-13 19:24:24,http://pollich.com/austin,,217.94.118.186,"{""location"": ""AS"", ""is_mobile"": true}" 153,1,16,2017-04-24 13:42:56,http://nikolauspouros.biz/eric_balistreri,,18.4.135.176,"{""location"": ""SD"", ""is_mobile"": false}" 154,1,16,2017-05-15 12:44:22,http://larsonkeebler.co/misael,,65.64.24.12,"{""location"": ""NG"", ""is_mobile"": false}" 155,1,16,2017-04-01 15:35:34,http://brekke.org/helene,,99.222.163.122,"{""location"": ""AU"", ""is_mobile"": false}" 156,1,16,2017-01-09 19:50:05,http://dooleyweber.info/devon,,160.86.178.11,"{""location"": ""EC"", ""is_mobile"": true}" 157,1,16,2017-05-10 20:32:31,http://torpheel.name/alfreda,,172.245.58.254,"{""location"": ""SS"", ""is_mobile"": false}" 158,1,16,2016-12-29 03:06:25,http://effertzfahey.net/wendell,,65.64.24.12,"{""location"": ""SN"", ""is_mobile"": false}" 159,1,16,2017-04-23 09:07:58,http://satterfield.co/roxanne,,46.245.224.45,"{""location"": ""MA"", ""is_mobile"": true}" 160,1,16,2017-03-30 15:19:04,http://heaney.com/hubert,,99.222.163.122,"{""location"": ""MY"", ""is_mobile"": false}" 161,1,17,2016-12-24 23:59:46,http://becker.info/howell,,11.127.105.64,"{""location"": ""BE"", ""is_mobile"": true}" 162,1,17,2017-01-04 12:05:08,http://white.co/lily_kohler,,64.142.118.61,"{""location"": ""JO"", ""is_mobile"": true}" 163,1,17,2017-01-10 01:14:26,http://fay.org/dangelo_goodwin,,3.68.20.223,"{""location"": ""DO"", ""is_mobile"": true}" 164,1,17,2017-01-04 13:35:02,http://runte.biz/moshe,,211.132.206.71,"{""location"": ""BE"", ""is_mobile"": false}" 165,1,17,2017-04-19 06:40:49,http://kuhn.name/emely,,217.120.35.226,"{""location"": ""MG"", ""is_mobile"": false}" 166,1,17,2017-01-19 19:10:37,http://buckridge.biz/juanita,,217.120.35.226,"{""location"": ""BV"", ""is_mobile"": false}" 5100,8,486,2017-01-28 19:00:18,http://bednar.name/kaycee_schowalter,,157.108.156.25,"{""location"": ""CZ"", ""is_mobile"": false}" 5101,8,486,2017-04-27 08:52:16,http://nienow.biz/anne.hodkiewicz,,234.122.140.185,"{""location"": ""ER"", ""is_mobile"": false}" 5102,8,486,2017-04-05 18:32:14,http://wiegandrohan.net/dawson_monahan,,78.150.100.36,"{""location"": ""BL"", ""is_mobile"": true}" 5103,8,486,2017-01-07 03:43:28,http://collierprohaska.info/davion.reynolds,,50.49.223.196,"{""location"": ""MQ"", ""is_mobile"": true}" 5104,8,486,2017-03-09 06:35:36,http://blicklindgren.io/humberto,,156.23.75.123,"{""location"": ""YT"", ""is_mobile"": false}" 5105,8,486,2017-01-13 09:27:07,http://corkeryfay.com/noble.jast,,191.204.173.17,"{""location"": ""BE"", ""is_mobile"": false}" 5106,8,486,2017-04-28 15:06:56,http://schiller.io/eduardo.little,,134.27.223.82,"{""location"": ""IE"", ""is_mobile"": false}" 5107,8,486,2017-01-14 07:25:13,http://lueilwitzbreitenberg.info/aaliyah_kling,,209.236.156.35,"{""location"": ""MF"", ""is_mobile"": true}" 5108,8,486,2017-05-05 03:50:21,http://nicolas.org/jaquelin,,114.169.202.150,"{""location"": ""AG"", ""is_mobile"": true}" 5109,8,486,2016-12-14 01:51:12,http://ankunding.info/arnoldo_corwin,,220.74.83.110,"{""location"": ""ZA"", ""is_mobile"": false}" 5110,8,486,2017-03-01 22:00:07,http://vandervort.com/gabrielle.hauck,,156.23.75.123,"{""location"": ""MR"", ""is_mobile"": false}" 5111,8,486,2016-12-13 08:36:28,http://rohan.net/leopoldo.wintheiser,,157.7.220.14,"{""location"": ""ID"", ""is_mobile"": false}" 5112,8,486,2017-03-20 09:23:51,http://ullrich.info/elvis_hagenes,,209.236.156.35,"{""location"": ""PH"", ""is_mobile"": false}" 5113,8,486,2017-04-19 05:51:14,http://paucek.info/lucius_swaniawski,,166.97.171.170,"{""location"": ""KG"", ""is_mobile"": false}" 5114,8,486,2017-02-18 19:22:59,http://hills.com/kylie,,220.74.83.110,"{""location"": ""RE"", ""is_mobile"": true}" 5115,8,486,2016-12-21 23:48:58,http://framibeatty.co/elmer,,50.49.223.196,"{""location"": ""KW"", ""is_mobile"": true}" 5116,8,486,2017-03-10 07:15:16,http://bartell.net/josie.grant,,14.250.45.44,"{""location"": ""MX"", ""is_mobile"": false}" 5117,8,487,2017-01-12 20:44:54,http://stehr.io/kelley,,188.224.198.161,"{""location"": ""GQ"", ""is_mobile"": false}" 5118,8,487,2016-12-16 05:24:13,http://mrazfarrell.info/uriel,,158.223.154.120,"{""location"": ""KE"", ""is_mobile"": true}" 5119,8,487,2017-04-05 10:06:01,http://huelheel.co/kameron_schuppe,,229.8.167.173,"{""location"": ""AD"", ""is_mobile"": true}" 5120,8,487,2017-03-04 08:39:10,http://macejkovic.biz/ryan,,108.108.213.63,"{""location"": ""AX"", ""is_mobile"": true}" 5121,8,487,2017-05-12 10:39:41,http://flatley.name/kitty,,148.131.181.80,"{""location"": ""AG"", ""is_mobile"": true}" 5122,8,487,2017-03-15 11:02:52,http://veum.info/forrest_keeling,,197.178.21.250,"{""location"": ""TW"", ""is_mobile"": true}" 5123,8,487,2017-02-24 22:55:35,http://feest.org/dayne,,134.115.247.156,"{""location"": ""PW"", ""is_mobile"": false}" 5124,8,487,2017-03-06 11:28:18,http://waters.io/audreanne,,188.224.198.161,"{""location"": ""FO"", ""is_mobile"": false}" 5125,8,487,2017-04-08 09:28:35,http://dicki.net/tiana_zboncak,,50.101.87.119,"{""location"": ""CY"", ""is_mobile"": true}" 5126,8,487,2017-03-06 05:42:13,http://kshlerin.org/yasmin_bauch,,104.157.94.128,"{""location"": ""TO"", ""is_mobile"": false}" 5127,8,487,2017-03-06 00:46:26,http://mohr.name/annetta.veum,,69.108.223.44,"{""location"": ""KG"", ""is_mobile"": true}" 5128,8,487,2017-04-20 21:28:33,http://kling.io/johnpaul,,49.189.202.195,"{""location"": ""IQ"", ""is_mobile"": false}" 5129,8,487,2017-06-10 05:47:39,http://miller.biz/cicero_leannon,,4.51.36.51,"{""location"": ""AF"", ""is_mobile"": true}" 5130,8,487,2017-01-03 07:33:23,http://abbott.org/erick,,51.116.171.120,"{""location"": ""GL"", ""is_mobile"": true}" 5131,8,487,2017-03-02 20:23:18,http://harveygrimes.name/theodora_rau,,215.132.27.109,"{""location"": ""FJ"", ""is_mobile"": true}" 5132,8,488,2016-12-18 17:13:04,http://gerlachschultz.com/enola.kris,,97.50.68.95,"{""location"": ""MO"", ""is_mobile"": false}" 5133,8,488,2017-02-28 07:27:14,http://volkman.biz/tyson,,38.153.211.113,"{""location"": ""DO"", ""is_mobile"": true}" 5134,8,488,2017-06-08 03:48:53,http://batz.com/gianni,,82.134.17.167,"{""location"": ""CO"", ""is_mobile"": true}" 5135,8,488,2017-03-14 04:28:34,http://hilll.com/mckayla,,25.223.182.214,"{""location"": ""MT"", ""is_mobile"": true}" 5136,8,488,2017-05-16 17:06:25,http://kihn.net/benny,,181.197.185.219,"{""location"": ""GG"", ""is_mobile"": false}" 5137,8,488,2017-04-21 09:36:22,http://willms.io/janie,,38.153.211.113,"{""location"": ""WF"", ""is_mobile"": true}" 5138,8,488,2016-12-19 23:09:36,http://langworthpfeffer.name/brent.jakubowski,,108.89.12.177,"{""location"": ""RU"", ""is_mobile"": false}" 5139,8,488,2017-06-09 06:05:49,http://fritsch.co/randall.veum,,101.209.132.21,"{""location"": ""SK"", ""is_mobile"": false}" 5140,8,488,2017-03-06 00:55:26,http://streichspencer.io/murphy.stokes,,115.248.193.24,"{""location"": ""IM"", ""is_mobile"": true}" 5141,8,488,2016-12-25 22:01:49,http://wilderman.io/kyla,,73.156.26.52,"{""location"": ""TC"", ""is_mobile"": false}" 5142,8,488,2017-06-03 17:42:26,http://ondrickalangworth.name/elouise.wilderman,,207.238.22.60,"{""location"": ""BR"", ""is_mobile"": false}" 5143,8,488,2017-02-03 10:37:45,http://andersonpollich.com/pablo_nolan,,87.160.54.29,"{""location"": ""DK"", ""is_mobile"": true}" 5144,8,488,2017-02-11 03:33:29,http://sawayn.co/makayla,,30.251.83.117,"{""location"": ""BZ"", ""is_mobile"": true}" 5145,8,489,2017-04-20 20:23:49,http://christiansenhodkiewicz.co/ernestina,,239.23.190.63,"{""location"": ""VI"", ""is_mobile"": false}" 5146,8,490,2017-05-21 09:07:42,http://goyette.biz/rafael,,208.176.155.69,"{""location"": ""SV"", ""is_mobile"": true}" 5147,8,490,2017-03-29 12:25:12,http://king.co/rollin_dach,,182.202.228.145,"{""location"": ""TN"", ""is_mobile"": false}" 5148,8,490,2017-04-21 15:41:16,http://leannon.net/shad,,102.175.191.121,"{""location"": ""GP"", ""is_mobile"": true}" 5149,8,490,2017-02-14 16:35:25,http://mills.biz/arvid,,132.98.39.221,"{""location"": ""MC"", ""is_mobile"": true}" 5150,8,490,2016-12-28 15:09:11,http://schmidtbergnaum.info/diana,,182.202.228.145,"{""location"": ""BZ"", ""is_mobile"": true}" 5151,8,490,2017-03-12 15:33:14,http://connelly.co/jamel.will,,254.208.112.78,"{""location"": ""NU"", ""is_mobile"": false}" 5152,8,490,2017-04-04 14:09:56,http://wisoky.biz/myles,,81.61.139.203,"{""location"": ""PN"", ""is_mobile"": true}" 5153,8,490,2016-12-31 01:01:04,http://botsfordkihn.com/linda,,102.175.191.121,"{""location"": ""GE"", ""is_mobile"": true}" 5154,8,490,2017-02-22 16:45:10,http://pouros.net/joseph,,11.19.6.197,"{""location"": ""KW"", ""is_mobile"": true}" 5155,8,490,2017-02-28 22:48:17,http://hahnskiles.info/yasmine,,254.208.112.78,"{""location"": ""BS"", ""is_mobile"": false}" 5156,8,490,2016-12-27 18:29:02,http://oberbrunnervonrueden.com/kailyn_boyle,,254.208.112.78,"{""location"": ""BV"", ""is_mobile"": false}" 5157,8,490,2017-01-14 09:22:08,http://renner.biz/christy.stroman,,132.98.39.221,"{""location"": ""BL"", ""is_mobile"": false}" 5158,8,490,2017-05-06 19:22:28,http://pagac.net/burley,,142.83.111.54,"{""location"": ""GL"", ""is_mobile"": true}" 5159,8,491,2017-06-02 14:23:00,http://kuphalconnelly.name/savanah,1.0000000000,68.125.71.247,"{""location"": ""MV"", ""is_mobile"": true}" 5160,8,491,2017-01-29 13:25:09,http://ratke.info/aliya,1.0000000000,54.112.138.141,"{""location"": ""DO"", ""is_mobile"": true}" 5161,8,491,2017-06-13 19:01:56,http://gutmann.org/guy,0.0000000000,38.176.207.118,"{""location"": ""ST"", ""is_mobile"": false}" 5162,8,491,2017-01-08 02:19:46,http://hammes.name/kamryn_sporer,2.0000000000,222.124.33.186,"{""location"": ""LB"", ""is_mobile"": true}" 5163,8,491,2017-02-20 18:39:35,http://schimmel.net/gladyce_oconnell,2.0000000000,49.106.162.10,"{""location"": ""NO"", ""is_mobile"": false}" 5164,8,491,2017-03-18 14:47:18,http://smithbartell.org/brice,0.0000000000,119.154.206.69,"{""location"": ""DE"", ""is_mobile"": true}" 5165,8,491,2017-05-06 22:21:40,http://fay.name/margaretta.ankunding,1.0000000000,171.136.111.246,"{""location"": ""AF"", ""is_mobile"": true}" 5166,8,491,2017-01-05 17:31:48,http://hilll.co/asia.greenholt,1.0000000000,101.60.159.103,"{""location"": ""TG"", ""is_mobile"": true}" 5167,8,491,2017-05-30 04:04:53,http://klockowisozk.com/clair.hodkiewicz,2.0000000000,120.232.60.19,"{""location"": ""LV"", ""is_mobile"": true}" 5168,8,491,2017-05-29 19:51:49,http://dickinsonkeebler.net/franz,0.0000000000,181.6.59.27,"{""location"": ""MQ"", ""is_mobile"": true}" 5169,8,491,2017-01-13 00:24:54,http://hackett.org/vicente,0.0000000000,250.227.210.90,"{""location"": ""UA"", ""is_mobile"": false}" 5170,8,491,2017-01-22 22:22:22,http://mccullough.net/marguerite,0.0000000000,89.124.68.64,"{""location"": ""KH"", ""is_mobile"": false}" 5171,8,491,2016-12-25 19:39:03,http://blick.net/modesto.frami,2.0000000000,27.103.205.92,"{""location"": ""ME"", ""is_mobile"": false}" 5172,8,491,2017-04-17 12:05:23,http://hicklegutmann.info/ariane,0.0000000000,254.77.2.32,"{""location"": ""NE"", ""is_mobile"": true}" 5173,8,491,2017-04-18 23:48:14,http://krajcik.io/randal,2.0000000000,107.232.112.244,"{""location"": ""KI"", ""is_mobile"": false}" 5174,8,491,2017-06-14 01:06:04,http://kub.co/chyna.miller,1.0000000000,146.20.27.20,"{""location"": ""CV"", ""is_mobile"": true}" 5175,8,491,2016-12-23 08:08:33,http://mccullough.biz/krystina,1.0000000000,222.124.33.186,"{""location"": ""BW"", ""is_mobile"": true}" 5176,8,491,2017-01-17 01:25:57,http://hoppe.com/vivienne.reilly,1.0000000000,181.6.59.27,"{""location"": ""UG"", ""is_mobile"": true}" 5177,8,492,2017-02-19 15:50:18,http://corkery.com/samir,3.0000000000,84.226.91.111,"{""location"": ""SA"", ""is_mobile"": true}" 5178,8,492,2017-01-18 08:04:20,http://waelchi.co/owen,0.0000000000,240.206.93.248,"{""location"": ""SA"", ""is_mobile"": true}" 5179,8,492,2017-05-29 00:57:36,http://klocko.net/twila_torphy,1.0000000000,172.159.253.45,"{""location"": ""ML"", ""is_mobile"": false}" 5180,8,492,2017-02-25 12:46:19,http://luettgen.io/zelma,0.0000000000,177.54.237.98,"{""location"": ""NA"", ""is_mobile"": true}" 5181,8,492,2017-02-15 00:30:29,http://emardjenkins.info/breana,1.0000000000,112.111.95.177,"{""location"": ""BB"", ""is_mobile"": true}" 5182,8,492,2016-12-18 00:52:05,http://weinatwalter.org/toby_beer,0.0000000000,38.18.62.64,"{""location"": ""PF"", ""is_mobile"": true}" 5183,8,492,2017-06-13 09:30:34,http://steuber.info/lillie,1.0000000000,205.188.73.202,"{""location"": ""UG"", ""is_mobile"": false}" 5184,8,492,2017-02-07 22:17:30,http://streich.net/jettie,3.0000000000,226.186.156.92,"{""location"": ""NR"", ""is_mobile"": false}" 5185,8,492,2017-01-15 12:43:55,http://turcotte.biz/rollin,2.0000000000,32.167.82.157,"{""location"": ""LA"", ""is_mobile"": false}" 5186,8,492,2017-03-07 20:42:27,http://botsford.com/vivienne,1.0000000000,240.254.239.236,"{""location"": ""GT"", ""is_mobile"": true}" 5187,8,492,2017-04-08 19:59:16,http://romaguerastamm.biz/hector_ernser,1.0000000000,240.254.239.236,"{""location"": ""TC"", ""is_mobile"": true}" 5188,8,492,2017-04-02 13:03:49,http://vandervortwhite.biz/bart,0.0000000000,35.136.67.33,"{""location"": ""YT"", ""is_mobile"": true}" 5189,8,492,2017-06-02 07:22:26,http://raynor.co/romaine.vonrueden,1.0000000000,157.53.197.53,"{""location"": ""BM"", ""is_mobile"": false}" 5190,8,492,2017-04-18 03:51:41,http://frami.io/clint,3.0000000000,186.22.41.30,"{""location"": ""MO"", ""is_mobile"": false}" 5191,8,492,2017-02-02 15:03:00,http://lakindare.info/sandy.padberg,2.0000000000,191.86.44.17,"{""location"": ""SX"", ""is_mobile"": true}" 5192,8,492,2017-05-20 22:34:58,http://abbottsanford.net/trycia,3.0000000000,2.16.168.30,"{""location"": ""NZ"", ""is_mobile"": false}" 5193,8,493,2017-02-13 12:15:09,http://fritsch.org/jaylon,0.0000000000,180.220.98.82,"{""location"": ""AT"", ""is_mobile"": true}" 5194,8,493,2017-01-23 05:57:04,http://howell.io/icie.macejkovic,2.0000000000,116.237.233.225,"{""location"": ""CX"", ""is_mobile"": false}" 5195,8,493,2017-06-08 12:35:36,http://rueckerkrajcik.com/breanna,2.0000000000,144.153.229.244,"{""location"": ""MM"", ""is_mobile"": false}" 5196,8,493,2017-01-17 08:41:39,http://harris.co/rasheed_goodwin,0.0000000000,180.220.98.82,"{""location"": ""PG"", ""is_mobile"": false}" 5197,8,493,2017-02-19 10:37:46,http://ondricka.name/therese,2.0000000000,222.12.97.124,"{""location"": ""BZ"", ""is_mobile"": false}" 5198,8,493,2016-12-14 23:56:58,http://sawayn.biz/xavier.williamson,1.0000000000,48.26.139.25,"{""location"": ""OM"", ""is_mobile"": false}" 5199,8,493,2017-06-10 02:57:29,http://murphy.biz/eldridge.berge,0.0000000000,206.50.193.114,"{""location"": ""MU"", ""is_mobile"": true}" 5200,8,493,2017-03-14 19:07:46,http://klocko.io/dorian,2.0000000000,140.28.118.101,"{""location"": ""TM"", ""is_mobile"": false}" 5201,8,493,2017-05-09 06:13:37,http://jones.name/donny_miller,2.0000000000,140.28.118.101,"{""location"": ""CX"", ""is_mobile"": true}" 5202,8,493,2017-05-20 08:11:01,http://leuschke.name/verda_williamson,0.0000000000,116.237.233.225,"{""location"": ""DZ"", ""is_mobile"": false}" 5203,8,493,2017-03-08 15:44:29,http://pacocha.info/nikko.ko,1.0000000000,123.67.83.25,"{""location"": ""AQ"", ""is_mobile"": false}" 5204,8,493,2017-05-24 11:49:44,http://turneradams.io/floy.johnston,1.0000000000,158.221.125.212,"{""location"": ""AQ"", ""is_mobile"": false}" 5205,8,493,2017-01-05 00:57:22,http://mosciski.com/maye.daugherty,1.0000000000,41.162.226.178,"{""location"": ""NA"", ""is_mobile"": false}" 5206,8,493,2017-05-16 19:49:21,http://kuphalebert.org/archibald,1.0000000000,111.158.102.106,"{""location"": ""KZ"", ""is_mobile"": false}" 5207,8,493,2017-05-22 08:03:11,http://gutkowskiupton.net/annamarie,0.0000000000,151.45.8.74,"{""location"": ""NC"", ""is_mobile"": false}" 5208,8,493,2017-03-13 11:51:57,http://koepp.io/ellsworth_gibson,0.0000000000,215.253.125.219,"{""location"": ""MS"", ""is_mobile"": false}" 5209,8,493,2017-01-10 18:50:08,http://wiegand.org/ernestina.toy,2.0000000000,186.155.110.97,"{""location"": ""SA"", ""is_mobile"": false}" 5210,8,493,2017-01-19 09:57:07,http://botsford.name/marlon,0.0000000000,186.155.110.97,"{""location"": ""GP"", ""is_mobile"": true}" 5211,8,493,2017-04-07 13:26:39,http://conroy.name/rickey,2.0000000000,79.182.78.127,"{""location"": ""AL"", ""is_mobile"": false}" 5212,8,493,2017-06-01 04:31:14,http://block.name/coty,2.0000000000,158.221.125.212,"{""location"": ""DO"", ""is_mobile"": false}" 5213,8,494,2017-02-22 17:49:06,http://kerluke.biz/tyson_lind,0.0000000000,224.214.234.239,"{""location"": ""HU"", ""is_mobile"": true}" 5214,8,494,2017-01-30 17:23:38,http://hyattsanford.com/chelsey,0.0000000000,180.211.252.58,"{""location"": ""TF"", ""is_mobile"": true}" 5215,8,494,2017-01-05 02:40:14,http://hackett.biz/santino,1.0000000000,153.98.128.98,"{""location"": ""SZ"", ""is_mobile"": true}" 5216,8,494,2017-03-04 13:59:10,http://okunevagoodwin.name/elvie_champlin,1.0000000000,156.176.3.63,"{""location"": ""SD"", ""is_mobile"": false}" 5217,8,494,2017-04-07 03:49:08,http://herzog.org/alberta,0.0000000000,136.72.126.15,"{""location"": ""EG"", ""is_mobile"": false}" 5218,8,494,2017-03-08 01:38:13,http://braunhahn.io/florian.klein,1.0000000000,224.214.234.239,"{""location"": ""MA"", ""is_mobile"": true}" 5219,8,495,2017-04-21 22:22:37,http://wunschkemmer.co/else,1.0000000000,202.147.206.115,"{""location"": ""ER"", ""is_mobile"": false}" 5220,8,495,2017-03-27 18:34:05,http://oconner.biz/ferne_thiel,1.0000000000,91.63.25.60,"{""location"": ""SA"", ""is_mobile"": true}" 5221,8,495,2017-02-19 07:05:29,http://nikolaus.name/lorenz,0.0000000000,178.161.204.244,"{""location"": ""VC"", ""is_mobile"": true}" 5222,8,495,2017-04-30 09:43:25,http://sporerboehm.org/waldo,1.0000000000,119.32.137.126,"{""location"": ""UG"", ""is_mobile"": false}" 5223,8,495,2017-01-25 07:59:03,http://franecki.co/rory,1.0000000000,2.159.121.233,"{""location"": ""PL"", ""is_mobile"": true}" 5224,8,495,2017-01-16 10:32:40,http://stantongreenfelder.org/ronny.wyman,2.0000000000,12.183.199.191,"{""location"": ""KG"", ""is_mobile"": false}" 5225,8,495,2017-03-21 06:48:04,http://wintheiser.biz/katharina,1.0000000000,37.72.219.225,"{""location"": ""OM"", ""is_mobile"": true}" 5226,8,495,2017-03-16 03:16:42,http://conroy.name/efrain_erdman,1.0000000000,36.45.243.203,"{""location"": ""AX"", ""is_mobile"": true}" 5227,8,495,2017-03-26 14:19:48,http://bradtke.co/jodie,0.0000000000,218.120.174.50,"{""location"": ""TK"", ""is_mobile"": false}" 5228,8,495,2017-03-26 18:30:26,http://lind.biz/rene,0.0000000000,225.214.27.204,"{""location"": ""GQ"", ""is_mobile"": true}" 5229,8,495,2017-04-19 01:12:09,http://moriette.net/hudson_littel,1.0000000000,158.23.123.7,"{""location"": ""BV"", ""is_mobile"": false}" 5230,8,495,2017-04-29 14:23:19,http://predovic.net/lorenzo.ryan,2.0000000000,113.196.197.220,"{""location"": ""AE"", ""is_mobile"": false}" 5231,8,495,2017-01-31 17:21:53,http://will.biz/hannah,0.0000000000,81.89.235.61,"{""location"": ""DE"", ""is_mobile"": false}" 5232,8,495,2016-12-27 18:01:05,http://kuhic.net/clemens,1.0000000000,184.66.214.13,"{""location"": ""GS"", ""is_mobile"": false}" 5233,8,495,2017-04-16 20:03:15,http://doyle.co/lorena,2.0000000000,118.95.243.63,"{""location"": ""CF"", ""is_mobile"": false}" 5234,8,495,2016-12-29 07:06:54,http://pfefferlangworth.biz/george.rath,2.0000000000,225.214.27.204,"{""location"": ""AR"", ""is_mobile"": false}" 5235,8,496,2017-04-01 18:16:08,http://mayer.io/waylon,0.0000000000,103.27.160.227,"{""location"": ""CK"", ""is_mobile"": true}" 5236,8,496,2017-03-28 17:35:43,http://wolfzboncak.info/armando,0.0000000000,30.247.46.66,"{""location"": ""NG"", ""is_mobile"": true}" 5237,8,496,2017-03-19 22:01:37,http://stark.biz/laurence.douglas,0.0000000000,92.85.87.154,"{""location"": ""PG"", ""is_mobile"": false}" 5238,8,496,2017-02-11 02:32:01,http://krisolson.io/richard.wisozk,0.0000000000,203.32.75.137,"{""location"": ""MU"", ""is_mobile"": false}" 5239,8,496,2016-12-26 01:55:10,http://marvin.info/ania,0.0000000000,89.174.203.75,"{""location"": ""EE"", ""is_mobile"": true}" 5240,8,496,2017-04-20 16:43:18,http://hartmann.net/lacey.mueller,0.0000000000,92.213.107.2,"{""location"": ""TO"", ""is_mobile"": false}" 5241,8,496,2017-06-13 23:22:46,http://franecki.info/lesley.dickinson,0.0000000000,14.27.132.213,"{""location"": ""CM"", ""is_mobile"": false}" 5242,8,496,2017-01-12 21:07:17,http://will.com/miles.kilback,0.0000000000,185.196.173.237,"{""location"": ""KW"", ""is_mobile"": true}" 5243,8,496,2017-04-24 10:43:49,http://wymanreilly.info/kacey_hane,0.0000000000,78.81.125.120,"{""location"": ""PW"", ""is_mobile"": false}" 5244,8,497,2017-01-13 06:34:57,http://shieldsemard.org/thalia.bruen,0.0000000000,146.156.212.234,"{""location"": ""ZA"", ""is_mobile"": true}" 5245,8,497,2017-04-26 20:05:35,http://kling.info/korbin_carter,0.0000000000,239.125.204.25,"{""location"": ""PA"", ""is_mobile"": false}" 5246,8,497,2017-02-02 20:06:16,http://batz.name/kylie_kuhic,0.0000000000,180.96.105.99,"{""location"": ""BV"", ""is_mobile"": true}" 5247,8,497,2017-05-29 12:11:53,http://heaney.org/rae_gleason,0.0000000000,29.120.136.66,"{""location"": ""BJ"", ""is_mobile"": true}" 5248,8,497,2017-05-26 18:57:57,http://hyatt.net/johnson_kuhic,1.0000000000,105.238.242.61,"{""location"": ""EC"", ""is_mobile"": false}" 5249,8,497,2017-05-09 21:11:11,http://lebsack.io/virgie,1.0000000000,202.107.42.65,"{""location"": ""LV"", ""is_mobile"": true}" 5250,8,497,2017-06-07 00:59:00,http://graham.info/elliot,1.0000000000,67.143.35.3,"{""location"": ""KZ"", ""is_mobile"": false}" 5251,8,497,2017-04-07 19:44:34,http://dibbert.name/yoshiko,1.0000000000,16.39.113.108,"{""location"": ""AF"", ""is_mobile"": false}" 5252,8,497,2017-02-20 19:32:53,http://satterfieldvon.com/caterina,1.0000000000,4.11.63.54,"{""location"": ""TM"", ""is_mobile"": true}" 5253,8,497,2017-04-14 17:27:25,http://abshire.net/demetris.hartmann,0.0000000000,102.252.130.43,"{""location"": ""MF"", ""is_mobile"": false}" 5254,8,497,2017-03-09 14:10:05,http://lindgren.io/milo_marks,0.0000000000,16.39.113.108,"{""location"": ""VA"", ""is_mobile"": true}" 5255,8,497,2017-02-10 07:25:22,http://schimmelreichert.co/stephania.runolfon,1.0000000000,35.176.199.233,"{""location"": ""KY"", ""is_mobile"": false}" 5256,8,497,2017-02-16 18:48:33,http://langhills.io/lexi_ebert,1.0000000000,44.108.145.51,"{""location"": ""LS"", ""is_mobile"": true}" 5257,8,497,2017-04-18 23:02:22,http://abshireheller.info/cornelius,1.0000000000,56.174.69.148,"{""location"": ""EE"", ""is_mobile"": false}" 5258,8,497,2017-02-22 21:02:47,http://breitenberg.io/stone.farrell,0.0000000000,16.39.113.108,"{""location"": ""KI"", ""is_mobile"": false}" 5259,8,497,2017-01-06 05:50:31,http://gusikowskithompson.co/hunter_wuckert,1.0000000000,233.172.28.108,"{""location"": ""TV"", ""is_mobile"": false}" 5260,8,497,2017-03-03 17:47:38,http://price.co/haleigh_kub,0.0000000000,209.184.143.56,"{""location"": ""BV"", ""is_mobile"": false}" 5261,8,497,2016-12-19 08:01:02,http://frami.biz/lou.gutkowski,0.0000000000,77.115.39.58,"{""location"": ""PG"", ""is_mobile"": false}" 5262,8,497,2017-01-06 05:38:41,http://mayerkoch.info/grant_lubowitz,1.0000000000,180.96.105.99,"{""location"": ""PM"", ""is_mobile"": true}" 5263,8,498,2017-06-03 11:52:40,http://mcdermott.com/suzanne,1.0000000000,106.52.179.97,"{""location"": ""GL"", ""is_mobile"": true}" 5264,8,498,2017-01-12 19:20:52,http://prosacco.net/bridie,0.0000000000,69.43.203.250,"{""location"": ""LK"", ""is_mobile"": true}" 5265,8,498,2017-04-28 19:16:23,http://kuhn.name/roger,0.0000000000,215.172.85.51,"{""location"": ""NZ"", ""is_mobile"": true}" 5266,8,499,2017-04-05 17:54:14,http://williamson.biz/kayleigh,1.0000000000,44.21.20.24,"{""location"": ""TO"", ""is_mobile"": true}" 5267,8,499,2017-04-05 01:24:03,http://dooley.co/hillard_hettinger,0.0000000000,202.93.42.79,"{""location"": ""LI"", ""is_mobile"": false}" 5268,8,499,2017-01-24 23:52:16,http://ruel.io/jovany,1.0000000000,44.21.20.24,"{""location"": ""LI"", ""is_mobile"": true}" 5269,8,499,2017-01-09 01:17:21,http://grantmclaughlin.info/madison.kunze,0.0000000000,61.12.179.59,"{""location"": ""PA"", ""is_mobile"": false}" 5270,8,500,2017-06-01 05:01:00,http://goodwin.com/elwyn.nicolas,0.0000000000,49.68.161.238,"{""location"": ""AZ"", ""is_mobile"": true}" 5271,8,500,2017-05-05 12:57:26,http://turcotte.co/jeanette,0.0000000000,145.125.14.43,"{""location"": ""VA"", ""is_mobile"": false}" 5272,8,500,2017-04-11 09:30:58,http://fritschfisher.name/josiane,0.0000000000,2.47.51.155,"{""location"": ""FI"", ""is_mobile"": false}" 5273,8,500,2017-05-22 14:26:08,http://reichelhettinger.name/jeremie.crona,0.0000000000,13.150.27.243,"{""location"": ""IR"", ""is_mobile"": true}" 5274,8,500,2017-01-13 00:58:45,http://trantownicolas.net/kaitlyn_quigley,0.0000000000,193.4.4.92,"{""location"": ""GR"", ""is_mobile"": false}" 5275,8,500,2017-04-07 21:48:24,http://witting.io/milan,0.0000000000,46.81.58.21,"{""location"": ""GU"", ""is_mobile"": false}" 5276,8,500,2017-02-19 22:02:12,http://marvin.info/reginald.reichert,0.0000000000,116.69.131.253,"{""location"": ""LK"", ""is_mobile"": false}" 5277,8,501,2017-06-08 01:32:28,http://lockmanheathcote.org/darion,0.0000000000,66.100.91.108,"{""location"": ""ZA"", ""is_mobile"": false}" 5278,8,501,2016-12-20 22:33:13,http://funk.biz/dion,2.0000000000,30.35.124.38,"{""location"": ""PF"", ""is_mobile"": true}" 5279,8,501,2017-05-09 04:24:18,http://yundt.name/karl_mcclure,2.0000000000,109.62.56.22,"{""location"": ""GY"", ""is_mobile"": false}" 5280,8,501,2017-04-10 04:43:47,http://marvin.io/dylan,1.0000000000,228.46.142.103,"{""location"": ""PE"", ""is_mobile"": true}" 5281,8,501,2017-05-04 05:55:24,http://andersonkuvalis.biz/shyanne.will,2.0000000000,16.49.212.67,"{""location"": ""IS"", ""is_mobile"": false}" 5282,8,501,2017-04-19 02:03:30,http://collinscrist.io/kris.reichel,0.0000000000,205.127.51.193,"{""location"": ""BL"", ""is_mobile"": false}" 5283,8,501,2017-03-18 00:10:29,http://olson.co/darrin_cartwright,1.0000000000,85.86.158.15,"{""location"": ""NR"", ""is_mobile"": false}" 5284,8,501,2017-03-16 02:03:21,http://windlerhowe.io/leola_brakus,2.0000000000,26.54.145.98,"{""location"": ""KN"", ""is_mobile"": false}" 5285,8,502,2017-04-15 22:00:19,http://hoppewehner.biz/herta_marks,0.0000000000,183.64.51.38,"{""location"": ""KM"", ""is_mobile"": true}" 5286,8,502,2017-05-17 15:32:35,http://welch.name/elise,1.0000000000,115.11.28.82,"{""location"": ""JP"", ""is_mobile"": false}" 5287,8,503,2017-04-23 17:14:43,http://torp.name/adam,3.0000000000,212.55.202.24,"{""location"": ""GE"", ""is_mobile"": true}" 5288,8,503,2017-01-14 15:28:38,http://schroedersteuber.co/theodore_doyle,1.0000000000,246.160.80.126,"{""location"": ""BV"", ""is_mobile"": false}" 5289,8,503,2017-03-28 21:16:08,http://haneblock.com/marianna.steuber,1.0000000000,239.158.39.115,"{""location"": ""WS"", ""is_mobile"": false}" 5290,8,503,2017-03-17 12:30:48,http://krisvon.co/camylle,3.0000000000,88.70.251.62,"{""location"": ""GH"", ""is_mobile"": true}" 5291,8,503,2017-02-20 07:39:11,http://wintheiser.org/frieda.douglas,0.0000000000,85.56.238.200,"{""location"": ""KW"", ""is_mobile"": false}" 5292,8,503,2017-03-13 16:54:17,http://hoegergleichner.biz/nicolas,0.0000000000,132.153.143.205,"{""location"": ""SN"", ""is_mobile"": false}" 5293,8,503,2017-01-16 18:00:01,http://powlowskidaniel.co/francesca,1.0000000000,26.127.228.35,"{""location"": ""ST"", ""is_mobile"": true}" 5294,8,503,2017-02-14 11:28:28,http://sipes.biz/arlie,3.0000000000,14.253.125.60,"{""location"": ""MA"", ""is_mobile"": true}" 5295,8,503,2017-04-07 06:06:44,http://lubowitz.net/connor,1.0000000000,218.3.166.102,"{""location"": ""NU"", ""is_mobile"": false}" 5296,8,503,2017-04-12 00:14:27,http://goldner.org/damon,3.0000000000,40.2.240.157,"{""location"": ""BG"", ""is_mobile"": false}" 5297,8,503,2017-06-02 18:35:34,http://becker.co/dax,3.0000000000,246.160.80.126,"{""location"": ""LT"", ""is_mobile"": false}" 5298,8,503,2017-03-11 01:33:41,http://baileyyost.co/molly_macejkovic,1.0000000000,212.55.202.24,"{""location"": ""IN"", ""is_mobile"": true}" 5299,8,503,2017-01-01 03:36:41,http://lynch.name/herminio.buckridge,3.0000000000,218.3.166.102,"{""location"": ""BW"", ""is_mobile"": false}" 5300,8,503,2017-06-05 03:07:37,http://oconner.name/rigoberto,3.0000000000,201.175.247.60,"{""location"": ""DJ"", ""is_mobile"": false}" 5301,8,503,2017-02-04 22:55:17,http://davis.co/deie_kuhn,2.0000000000,124.178.3.191,"{""location"": ""CI"", ""is_mobile"": false}" 5302,8,503,2017-05-22 11:12:00,http://willjacobs.net/candace.hermann,0.0000000000,252.136.82.196,"{""location"": ""GF"", ""is_mobile"": true}" 5303,8,503,2017-02-28 21:40:43,http://fritsch.org/antonetta,1.0000000000,246.160.80.126,"{""location"": ""NC"", ""is_mobile"": false}" 5304,8,503,2017-04-06 23:37:30,http://metz.org/delmer,0.0000000000,251.248.245.191,"{""location"": ""SH"", ""is_mobile"": false}" 5305,8,504,2017-05-24 14:32:16,http://fahey.net/hattie,0.0000000000,72.93.124.208,"{""location"": ""AR"", ""is_mobile"": false}" 5306,8,504,2017-06-10 16:13:52,http://satterfield.io/jadon_hauck,0.0000000000,187.194.91.107,"{""location"": ""MY"", ""is_mobile"": true}" 5307,8,504,2017-05-09 23:15:18,http://collier.io/kelsie.champlin,0.0000000000,86.63.23.45,"{""location"": ""EG"", ""is_mobile"": true}" 5308,8,504,2017-03-24 06:20:12,http://reynolds.name/glen,0.0000000000,232.214.87.247,"{""location"": ""AX"", ""is_mobile"": false}" 5309,8,504,2017-03-17 12:00:48,http://cremin.io/victor,0.0000000000,56.221.94.164,"{""location"": ""TW"", ""is_mobile"": false}" 5310,8,504,2017-04-10 19:08:57,http://gradyolson.info/isidro,0.0000000000,40.196.114.41,"{""location"": ""LI"", ""is_mobile"": false}" 5311,8,504,2017-01-10 02:05:24,http://handgutmann.co/laria,0.0000000000,72.57.194.73,"{""location"": ""BR"", ""is_mobile"": false}" 5312,8,504,2017-01-06 07:37:13,http://leannon.biz/toby.cormier,0.0000000000,117.68.23.220,"{""location"": ""FI"", ""is_mobile"": false}" 5313,8,504,2017-02-08 05:09:10,http://senger.info/gino_morar,0.0000000000,117.68.23.220,"{""location"": ""LT"", ""is_mobile"": true}" 5314,8,504,2017-03-23 11:17:23,http://lueilwitz.net/janice,0.0000000000,155.86.59.112,"{""location"": ""HN"", ""is_mobile"": false}" 5315,8,504,2017-01-08 13:35:04,http://legros.io/solon.zulauf,0.0000000000,130.253.54.245,"{""location"": ""KN"", ""is_mobile"": false}" 5316,8,504,2017-02-05 20:49:07,http://klingnolan.co/joe.hilll,0.0000000000,6.179.177.119,"{""location"": ""LU"", ""is_mobile"": false}" 5317,8,504,2017-01-19 13:07:03,http://wizabradtke.co/winona,0.0000000000,48.113.74.188,"{""location"": ""RO"", ""is_mobile"": false}" 5318,8,505,2017-02-28 14:44:43,http://von.biz/hazel.adams,3.0000000000,132.209.245.7,"{""location"": ""GP"", ""is_mobile"": true}" 5319,8,505,2017-03-10 04:58:08,http://heidenreich.biz/deven.schoen,0.0000000000,45.209.243.200,"{""location"": ""VC"", ""is_mobile"": true}" 5320,8,505,2017-03-21 11:41:47,http://jenkins.name/clifford_deckow,2.0000000000,181.210.55.214,"{""location"": ""NA"", ""is_mobile"": false}" 5321,8,505,2017-05-29 17:41:38,http://tromproberts.info/michel,1.0000000000,162.78.174.133,"{""location"": ""GQ"", ""is_mobile"": false}" 5322,8,505,2017-01-13 09:04:46,http://lynch.co/alfredo.keebler,0.0000000000,181.210.55.214,"{""location"": ""RS"", ""is_mobile"": false}" 5323,8,505,2017-06-09 02:51:06,http://jacobsonhagenes.biz/dallas_crona,3.0000000000,228.117.188.3,"{""location"": ""PE"", ""is_mobile"": false}" 5324,8,505,2017-03-28 21:18:49,http://hegmann.net/albert,0.0000000000,60.224.137.248,"{""location"": ""KY"", ""is_mobile"": true}" 5325,8,505,2017-06-08 17:28:33,http://dach.io/dashawn,1.0000000000,228.117.188.3,"{""location"": ""GI"", ""is_mobile"": true}" 5326,8,505,2017-01-30 08:27:19,http://aufderhar.io/sammy.robel,1.0000000000,7.95.10.121,"{""location"": ""HK"", ""is_mobile"": false}" 5327,8,506,2017-04-06 04:27:04,http://schmeler.org/mackenzie,0.0000000000,100.153.104.82,"{""location"": ""SG"", ""is_mobile"": true}" 5328,8,506,2017-04-08 19:38:56,http://volkmandare.net/rubye.hackett,0.0000000000,174.21.5.161,"{""location"": ""NG"", ""is_mobile"": true}" 5329,8,506,2017-02-01 11:33:35,http://erdman.io/derrick.carroll,0.0000000000,236.143.131.215,"{""location"": ""NZ"", ""is_mobile"": true}" 5330,8,506,2017-01-05 23:22:34,http://homenickblock.info/fidel,0.0000000000,236.143.131.215,"{""location"": ""MA"", ""is_mobile"": false}" 5331,8,506,2017-04-12 19:27:37,http://rohan.info/colton,0.0000000000,231.66.221.4,"{""location"": ""MX"", ""is_mobile"": false}" 5332,8,506,2017-01-16 06:13:24,http://walterdach.co/bridgette,0.0000000000,85.140.253.70,"{""location"": ""BB"", ""is_mobile"": false}" 5333,8,506,2017-06-08 18:33:41,http://koeppboyle.org/llewellyn_spinka,0.0000000000,54.167.226.30,"{""location"": ""LB"", ""is_mobile"": false}" 5334,8,506,2017-04-11 04:32:23,http://eichmann.net/lyda_runolfon,0.0000000000,42.100.126.145,"{""location"": ""GL"", ""is_mobile"": true}" 5335,8,506,2017-04-06 06:10:17,http://miller.info/olin,0.0000000000,168.150.197.249,"{""location"": ""AD"", ""is_mobile"": true}" 5336,8,506,2017-01-17 18:44:26,http://veum.biz/hoyt_howell,0.0000000000,158.90.189.144,"{""location"": ""GH"", ""is_mobile"": true}" 5337,8,506,2017-05-19 03:12:12,http://thiel.name/mallory,0.0000000000,168.150.197.249,"{""location"": ""BE"", ""is_mobile"": false}" 5338,8,506,2017-01-29 09:54:12,http://terry.info/chadrick,0.0000000000,102.222.189.236,"{""location"": ""CG"", ""is_mobile"": false}" 5339,8,506,2017-02-02 07:29:16,http://weber.io/darien,0.0000000000,16.42.129.190,"{""location"": ""AW"", ""is_mobile"": true}" 5340,8,506,2017-01-25 19:13:23,http://maggio.net/neal_weinat,0.0000000000,42.100.126.145,"{""location"": ""IS"", ""is_mobile"": true}" 5341,8,506,2016-12-29 01:01:01,http://jakubowski.co/ward.friesen,0.0000000000,231.66.221.4,"{""location"": ""DZ"", ""is_mobile"": true}" 5342,8,506,2016-12-23 13:36:54,http://daniel.biz/raquel_mitchell,0.0000000000,155.111.169.43,"{""location"": ""SM"", ""is_mobile"": true}" 5343,8,507,2017-01-24 09:24:16,http://streichpagac.co/corene,0.0000000000,9.104.152.83,"{""location"": ""NA"", ""is_mobile"": false}" 5344,8,507,2017-04-06 01:11:54,http://haley.co/carlee.oberbrunner,0.0000000000,165.67.161.193,"{""location"": ""GH"", ""is_mobile"": false}" 5345,8,507,2017-06-02 23:38:54,http://ortiz.org/buster_pagac,0.0000000000,141.203.12.242,"{""location"": ""PF"", ""is_mobile"": true}" 5346,8,507,2017-03-28 02:10:41,http://terryterry.io/maxwell.fritsch,0.0000000000,222.173.40.128,"{""location"": ""TG"", ""is_mobile"": false}" 5347,8,507,2017-02-04 12:27:38,http://schuster.com/carlee.cole,0.0000000000,239.209.219.246,"{""location"": ""KZ"", ""is_mobile"": true}" 5348,8,507,2017-02-25 05:12:56,http://roberts.info/maya,0.0000000000,42.84.231.207,"{""location"": ""AM"", ""is_mobile"": false}" 5349,8,507,2017-03-17 04:19:17,http://ledner.info/astrid,0.0000000000,44.17.83.177,"{""location"": ""KM"", ""is_mobile"": true}" 5350,8,507,2017-03-09 11:14:24,http://mohr.info/arvid_shanahan,0.0000000000,211.240.105.187,"{""location"": ""SZ"", ""is_mobile"": false}" 5351,8,507,2017-04-01 01:33:43,http://hermiston.co/nora.breitenberg,0.0000000000,132.241.217.118,"{""location"": ""OM"", ""is_mobile"": true}" 5352,8,507,2017-03-13 03:34:31,http://marquardt.org/jaeden.balistreri,0.0000000000,89.218.139.92,"{""location"": ""PT"", ""is_mobile"": true}" 5353,8,507,2017-04-06 22:52:40,http://sipesheller.com/aurelio,0.0000000000,231.228.122.123,"{""location"": ""MN"", ""is_mobile"": false}" 5354,8,507,2017-05-12 15:53:17,http://wehnerweinat.com/maxwell.jacobi,0.0000000000,161.102.121.155,"{""location"": ""CH"", ""is_mobile"": true}" 5355,8,507,2017-01-13 18:06:06,http://schroedermuller.org/pascale,0.0000000000,92.143.45.72,"{""location"": ""NA"", ""is_mobile"": false}" 5356,8,507,2017-01-11 00:14:15,http://homenick.io/ricardo,0.0000000000,103.133.128.65,"{""location"": ""KR"", ""is_mobile"": false}" 5357,8,508,2017-01-22 07:08:52,http://pfeffercasper.biz/alberto,0.0000000000,39.86.47.114,"{""location"": ""BW"", ""is_mobile"": false}" 5358,8,508,2017-03-26 04:04:03,http://schaefer.co/cortney,0.0000000000,68.210.118.62,"{""location"": ""IM"", ""is_mobile"": false}" 5359,8,508,2017-05-05 15:55:19,http://gutkowskikunde.io/bradford,0.0000000000,48.33.116.180,"{""location"": ""BQ"", ""is_mobile"": true}" 5360,8,509,2017-01-12 12:55:44,http://schusterpaucek.name/eliezer_schowalter,1.0000000000,236.239.104.216,"{""location"": ""PK"", ""is_mobile"": false}" 5361,8,509,2017-03-09 06:48:17,http://padberg.co/kaylah,2.0000000000,160.130.107.6,"{""location"": ""TZ"", ""is_mobile"": false}" 5362,8,509,2017-01-23 11:33:32,http://turcotte.co/fae.ruel,2.0000000000,33.50.22.113,"{""location"": ""BT"", ""is_mobile"": true}" 5363,8,509,2017-03-08 18:04:58,http://wolf.org/raul,0.0000000000,200.42.157.40,"{""location"": ""SN"", ""is_mobile"": true}" 5364,8,509,2016-12-23 11:07:56,http://gutkowski.net/amber.ziemann,2.0000000000,163.65.70.68,"{""location"": ""MU"", ""is_mobile"": false}" 5365,8,509,2017-02-08 05:50:11,http://rowe.net/jee_bradtke,2.0000000000,201.40.208.243,"{""location"": ""RS"", ""is_mobile"": false}" 5366,8,509,2017-05-14 12:54:04,http://bartonlueilwitz.biz/kayleigh_robel,2.0000000000,166.160.24.180,"{""location"": ""ZA"", ""is_mobile"": false}" 5367,8,510,2017-02-27 08:21:24,http://kuhn.info/horace.jerde,0.0000000000,47.239.211.15,"{""location"": ""FK"", ""is_mobile"": true}" 5368,8,510,2017-01-02 09:44:06,http://harris.co/kiley,2.0000000000,143.228.38.33,"{""location"": ""LS"", ""is_mobile"": true}" 5369,8,510,2017-02-15 07:08:45,http://kerluke.io/floy_jaskolski,1.0000000000,19.138.242.34,"{""location"": ""CO"", ""is_mobile"": true}" 5370,8,510,2017-01-22 03:12:32,http://tillmanmckenzie.net/antonetta.fadel,1.0000000000,235.178.36.105,"{""location"": ""KY"", ""is_mobile"": true}" 5371,8,510,2017-01-12 14:52:27,http://vandervort.org/peggie_towne,0.0000000000,128.80.57.154,"{""location"": ""FI"", ""is_mobile"": false}" 5372,8,510,2017-05-26 07:15:02,http://mayer.biz/rae,1.0000000000,169.126.195.34,"{""location"": ""PG"", ""is_mobile"": true}" 5373,8,511,2016-12-31 13:57:33,http://hermann.net/erika,1.0000000000,26.80.109.112,"{""location"": ""LA"", ""is_mobile"": false}" 5374,8,512,2017-05-25 22:45:31,http://dach.net/pete_gislason,0.0000000000,179.156.226.18,"{""location"": ""IL"", ""is_mobile"": false}" 5375,8,512,2017-02-09 08:43:56,http://kovacekreinger.co/cordelia,0.0000000000,161.87.104.73,"{""location"": ""TW"", ""is_mobile"": true}" 5376,8,512,2017-04-18 13:15:20,http://jerde.co/ari.robel,1.0000000000,202.150.249.114,"{""location"": ""LB"", ""is_mobile"": false}" 5377,8,512,2016-12-28 01:02:26,http://beattyquitzon.io/andre.stoltenberg,0.0000000000,202.150.249.114,"{""location"": ""VU"", ""is_mobile"": true}" 5378,8,512,2016-12-13 06:40:00,http://greenjaskolski.co/sylvan,0.0000000000,62.70.214.228,"{""location"": ""FK"", ""is_mobile"": false}" 5379,8,512,2017-04-16 08:56:30,http://lind.org/mariano.larkin,1.0000000000,13.162.125.248,"{""location"": ""CA"", ""is_mobile"": true}" 5380,8,512,2017-01-28 16:04:07,http://hermistonswaniawski.biz/nichole.franecki,0.0000000000,137.213.121.48,"{""location"": ""TK"", ""is_mobile"": false}" 5381,8,512,2016-12-13 08:33:07,http://hoegerdeckow.org/ludie.price,1.0000000000,163.115.226.245,"{""location"": ""AF"", ""is_mobile"": false}" 5382,8,512,2017-04-18 19:21:38,http://kuhic.net/deondre,1.0000000000,5.126.85.253,"{""location"": ""UY"", ""is_mobile"": false}" 5383,8,512,2017-03-09 22:44:00,http://fadel.name/evert,0.0000000000,161.87.104.73,"{""location"": ""LA"", ""is_mobile"": false}" 5384,8,512,2017-02-15 08:27:08,http://ko.name/armand.blanda,1.0000000000,223.231.138.91,"{""location"": ""CR"", ""is_mobile"": true}" 5385,8,512,2017-05-02 07:24:15,http://hamilldickinson.net/reece,1.0000000000,172.50.119.105,"{""location"": ""GH"", ""is_mobile"": true}" 5386,8,512,2017-04-22 06:35:54,http://mullerwolff.com/bruce_ziemann,1.0000000000,132.52.96.115,"{""location"": ""PM"", ""is_mobile"": false}" 5387,8,512,2017-06-13 19:05:40,http://hoppe.com/major.kertzmann,1.0000000000,179.156.226.18,"{""location"": ""DJ"", ""is_mobile"": true}" 5388,8,512,2017-04-30 11:58:49,http://wolff.co/theresia,1.0000000000,167.81.87.6,"{""location"": ""PN"", ""is_mobile"": false}" 5389,8,512,2017-03-25 05:26:28,http://wisozktillman.info/nannie,0.0000000000,8.125.64.24,"{""location"": ""HR"", ""is_mobile"": true}" 5390,8,513,2017-03-04 00:03:51,http://conroy.com/delaney_mccullough,2.0000000000,143.224.189.42,"{""location"": ""BO"", ""is_mobile"": false}" 5391,8,513,2017-05-02 11:11:08,http://murraywuckert.info/cali_mayert,0.0000000000,96.33.181.204,"{""location"": ""KZ"", ""is_mobile"": false}" 5392,8,513,2016-12-17 18:42:38,http://schuppe.org/moshe.mayer,2.0000000000,218.95.13.43,"{""location"": ""AS"", ""is_mobile"": false}" 5393,8,513,2017-01-03 08:36:22,http://labadie.info/gustave.boyer,1.0000000000,111.247.110.127,"{""location"": ""KI"", ""is_mobile"": true}" 5394,8,513,2017-05-08 20:26:16,http://fisher.info/marilou_dicki,3.0000000000,212.56.197.211,"{""location"": ""SN"", ""is_mobile"": true}" 5395,8,513,2017-03-16 13:13:12,http://gerhold.info/efrain,3.0000000000,212.56.197.211,"{""location"": ""BT"", ""is_mobile"": false}" 5396,8,514,2017-04-16 08:17:40,http://goyette.co/mackenzie,2.0000000000,42.103.103.187,"{""location"": ""CR"", ""is_mobile"": true}" 5397,8,514,2016-12-31 14:54:07,http://schneidercarroll.info/idell,0.0000000000,51.218.89.51,"{""location"": ""MW"", ""is_mobile"": true}" 5398,8,514,2016-12-16 15:22:42,http://weber.co/rosalee.wiza,1.0000000000,195.246.116.251,"{""location"": ""IQ"", ""is_mobile"": true}" 5399,8,514,2017-06-01 10:24:46,http://lockmanoreilly.info/leta,2.0000000000,38.215.49.114,"{""location"": ""PG"", ""is_mobile"": false}" 5400,8,514,2017-04-20 08:19:36,http://schuster.co/abner,3.0000000000,195.246.116.251,"{""location"": ""GL"", ""is_mobile"": false}" 5401,8,514,2016-12-26 14:36:03,http://hermistonkris.name/adan,3.0000000000,41.9.37.197,"{""location"": ""NU"", ""is_mobile"": true}" 5402,8,514,2017-03-14 01:40:31,http://bosco.name/andreane,0.0000000000,252.240.229.100,"{""location"": ""GU"", ""is_mobile"": false}" 5403,8,514,2017-06-13 01:45:46,http://gorczany.io/lupe,1.0000000000,82.70.24.237,"{""location"": ""FM"", ""is_mobile"": false}" 5404,8,514,2017-02-07 06:09:41,http://cristpadberg.org/enola,0.0000000000,37.64.79.10,"{""location"": ""LS"", ""is_mobile"": false}" 5405,8,514,2017-05-01 06:13:56,http://hammes.info/leone,3.0000000000,19.158.134.237,"{""location"": ""AI"", ""is_mobile"": true}" 5406,8,514,2017-03-10 05:45:18,http://beer.org/garrick,0.0000000000,239.170.174.78,"{""location"": ""FJ"", ""is_mobile"": true}" 5407,8,514,2017-04-29 03:07:30,http://hahn.co/maudie,3.0000000000,42.103.103.187,"{""location"": ""GM"", ""is_mobile"": true}" 5408,8,514,2017-03-27 21:16:01,http://borer.com/rita,3.0000000000,252.240.229.100,"{""location"": ""MN"", ""is_mobile"": true}" 5409,8,514,2017-02-17 19:05:58,http://ondricka.org/stanford,1.0000000000,139.75.82.166,"{""location"": ""VC"", ""is_mobile"": false}" 5410,8,515,2017-05-13 20:53:45,http://leffler.com/lexi_mccullough,0.0000000000,25.200.119.190,"{""location"": ""JE"", ""is_mobile"": true}" 5411,8,515,2017-03-11 04:02:52,http://pfefferbogan.com/syble,0.0000000000,237.39.47.158,"{""location"": ""EE"", ""is_mobile"": false}" 5412,8,515,2017-02-01 06:26:18,http://maggio.com/christophe.ohara,0.0000000000,56.209.24.155,"{""location"": ""MQ"", ""is_mobile"": true}" 5413,8,515,2017-03-28 02:57:17,http://halvorson.net/calista_friesen,2.0000000000,153.230.167.162,"{""location"": ""OM"", ""is_mobile"": false}" 167,1,17,2017-05-22 00:50:04,http://johnson.info/kareem.bernier,,129.192.50.219,"{""location"": ""VE"", ""is_mobile"": false}" 168,1,17,2017-06-10 00:34:32,http://effertz.info/edythe,,16.187.217.144,"{""location"": ""BT"", ""is_mobile"": false}" 169,1,17,2017-02-14 06:25:38,http://ryan.net/vida_brown,,129.192.50.219,"{""location"": ""LK"", ""is_mobile"": true}" 170,1,17,2016-12-18 09:38:18,http://torphy.org/rahsaan,,192.92.225.49,"{""location"": ""AU"", ""is_mobile"": false}" 171,1,17,2017-01-23 16:35:35,http://pagac.com/crystel.feeney,,177.101.164.145,"{""location"": ""BE"", ""is_mobile"": true}" 172,1,17,2016-12-17 11:01:18,http://prosacco.biz/kayli,,238.245.225.92,"{""location"": ""VN"", ""is_mobile"": false}" 173,1,17,2017-02-02 20:18:48,http://kilback.biz/tommie.ledner,,238.245.225.92,"{""location"": ""SV"", ""is_mobile"": true}" 174,1,17,2017-04-15 19:07:54,http://rath.co/omari,,224.105.29.6,"{""location"": ""CZ"", ""is_mobile"": false}" 175,1,17,2017-05-07 13:39:46,http://ritchie.info/ronaldo,,177.101.164.145,"{""location"": ""EC"", ""is_mobile"": true}" 176,1,18,2017-02-18 23:57:16,http://franecki.biz/trea_stark,,159.180.41.98,"{""location"": ""NC"", ""is_mobile"": true}" 177,1,18,2017-04-18 06:35:03,http://weber.org/kathryne,,97.148.9.70,"{""location"": ""KM"", ""is_mobile"": false}" 178,1,18,2017-05-09 08:14:20,http://kertzmann.biz/zaria,,208.81.138.12,"{""location"": ""CH"", ""is_mobile"": true}" 179,1,18,2017-03-12 05:21:14,http://ortiz.net/aric.ruel,,24.185.14.98,"{""location"": ""RW"", ""is_mobile"": false}" 180,1,18,2017-01-20 22:58:43,http://nicolas.info/darien.zulauf,,97.148.9.70,"{""location"": ""PR"", ""is_mobile"": true}" 181,1,18,2017-04-03 09:01:20,http://cronin.biz/antonia,,188.216.5.102,"{""location"": ""CF"", ""is_mobile"": false}" 182,1,18,2017-06-05 03:42:13,http://batz.info/harmony_lockman,,43.207.221.234,"{""location"": ""LY"", ""is_mobile"": false}" 183,1,18,2017-02-08 13:50:31,http://pfannerstill.name/cindy_pouros,,246.205.23.181,"{""location"": ""AX"", ""is_mobile"": true}" 184,1,18,2017-03-26 02:27:35,http://conn.biz/luna,,28.205.158.179,"{""location"": ""ZM"", ""is_mobile"": true}" 185,1,18,2016-12-23 06:35:31,http://lynchrolfson.info/sasha_bahringer,,228.45.251.38,"{""location"": ""VC"", ""is_mobile"": true}" 186,1,18,2017-02-09 02:51:07,http://fritschreichel.co/irma.veum,,126.215.74.135,"{""location"": ""BS"", ""is_mobile"": true}" 187,1,18,2017-04-18 20:31:19,http://donnelly.org/keith.muller,,159.186.19.58,"{""location"": ""FI"", ""is_mobile"": true}" 188,1,18,2017-03-29 22:51:25,http://muller.io/leila,,136.68.23.53,"{""location"": ""GT"", ""is_mobile"": true}" 189,1,18,2017-01-16 14:01:45,http://greenkeeling.com/merlin,,213.130.58.10,"{""location"": ""SI"", ""is_mobile"": true}" 190,1,18,2017-05-07 06:15:31,http://dare.org/hildegard,,228.45.251.38,"{""location"": ""BN"", ""is_mobile"": false}" 191,1,18,2017-06-11 03:41:18,http://nader.name/mollie,,210.159.76.243,"{""location"": ""BZ"", ""is_mobile"": true}" 192,1,18,2017-01-12 12:27:14,http://strosin.org/citlalli_tromp,,129.231.100.200,"{""location"": ""MN"", ""is_mobile"": true}" 193,1,19,2016-12-25 19:08:18,http://nikolaus.co/horace,,226.193.71.137,"{""location"": ""NU"", ""is_mobile"": true}" 194,1,19,2017-05-28 11:36:14,http://ullrich.net/lura,,103.78.135.172,"{""location"": ""MW"", ""is_mobile"": false}" 195,1,20,2017-01-12 01:51:29,http://runtemuller.co/wellington_robel,,205.55.12.116,"{""location"": ""UY"", ""is_mobile"": false}" 196,1,21,2017-02-01 20:10:13,http://corkerymurazik.biz/dena.corwin,1.0000000000,225.168.159.171,"{""location"": ""KP"", ""is_mobile"": true}" 197,1,21,2017-03-06 09:22:07,http://lemke.com/angus.pouros,1.0000000000,237.34.84.211,"{""location"": ""MY"", ""is_mobile"": true}" 198,1,22,2017-04-30 05:01:22,http://nikolaus.info/jayson,0.0000000000,95.170.190.50,"{""location"": ""IR"", ""is_mobile"": true}" 199,1,22,2017-01-03 12:21:45,http://cartwright.net/malvina_walker,0.0000000000,213.61.68.163,"{""location"": ""KI"", ""is_mobile"": false}" 200,1,22,2017-03-13 02:11:35,http://dubuque.com/kameron_glover,0.0000000000,59.193.153.48,"{""location"": ""PY"", ""is_mobile"": false}" 201,1,22,2017-01-08 02:59:50,http://quitzon.net/lisandro,0.0000000000,16.193.193.97,"{""location"": ""AE"", ""is_mobile"": false}" 202,1,22,2017-01-22 18:35:25,http://hodkiewicz.info/robyn,0.0000000000,95.75.144.22,"{""location"": ""FM"", ""is_mobile"": true}" 203,1,22,2016-12-28 22:19:58,http://greenholt.name/terrance,0.0000000000,233.248.190.222,"{""location"": ""LK"", ""is_mobile"": false}" 204,1,23,2017-01-08 06:52:18,http://klocko.info/aubrey,0.0000000000,144.78.75.75,"{""location"": ""EE"", ""is_mobile"": true}" 205,1,23,2017-06-01 01:42:38,http://towne.info/alia.kling,1.0000000000,209.9.110.159,"{""location"": ""SY"", ""is_mobile"": true}" 206,1,23,2017-01-19 08:37:42,http://carrolldooley.io/jeanne,0.0000000000,144.78.75.75,"{""location"": ""VC"", ""is_mobile"": true}" 207,1,23,2017-04-27 12:45:42,http://doyleziemann.io/jasper,0.0000000000,84.132.209.189,"{""location"": ""ET"", ""is_mobile"": true}" 208,1,23,2017-01-05 09:43:23,http://botsford.org/cathryn,1.0000000000,218.62.30.246,"{""location"": ""VI"", ""is_mobile"": true}" 209,1,23,2016-12-27 22:46:36,http://hilpert.info/rudy.herzog,0.0000000000,39.44.143.75,"{""location"": ""GT"", ""is_mobile"": true}" 210,1,23,2017-05-13 17:15:54,http://nikolausmurphy.net/letha,0.0000000000,177.225.19.77,"{""location"": ""PG"", ""is_mobile"": false}" 211,1,23,2017-01-20 14:23:08,http://schumm.biz/myrl,1.0000000000,160.122.117.59,"{""location"": ""AU"", ""is_mobile"": true}" 212,1,23,2017-01-04 01:56:05,http://fritsch.io/aniyah,0.0000000000,39.44.143.75,"{""location"": ""NF"", ""is_mobile"": true}" 213,1,23,2017-01-08 20:30:34,http://leschgibson.org/samara,1.0000000000,253.121.59.225,"{""location"": ""GD"", ""is_mobile"": false}" 214,1,24,2017-02-07 03:12:46,http://wyman.io/velma,0.0000000000,106.177.249.40,"{""location"": ""RO"", ""is_mobile"": true}" 215,1,24,2017-05-20 02:35:42,http://bartell.co/tania.pacocha,1.0000000000,202.157.4.188,"{""location"": ""FM"", ""is_mobile"": false}" 216,1,24,2017-02-07 04:42:36,http://vandervortdaniel.biz/jordy_bechtelar,1.0000000000,170.8.171.20,"{""location"": ""SJ"", ""is_mobile"": false}" 217,1,24,2017-05-14 19:55:26,http://schuppe.org/reymundo_bechtelar,1.0000000000,23.40.252.204,"{""location"": ""BR"", ""is_mobile"": true}" 218,1,24,2017-06-08 04:49:59,http://goldner.name/yvette_lynch,1.0000000000,152.36.134.85,"{""location"": ""CR"", ""is_mobile"": true}" 219,1,24,2017-04-12 05:06:35,http://nader.co/herminia_runolfsdottir,0.0000000000,240.101.40.157,"{""location"": ""CL"", ""is_mobile"": true}" 220,1,24,2017-04-28 13:55:33,http://cronalehner.net/jerrell.cummerata,1.0000000000,170.8.171.20,"{""location"": ""BT"", ""is_mobile"": false}" 5414,8,515,2017-02-22 06:20:31,http://abbott.co/olga,1.0000000000,134.149.41.34,"{""location"": ""NA"", ""is_mobile"": false}" 5415,8,515,2017-01-23 03:16:47,http://crooks.net/gay.bode,2.0000000000,243.155.231.110,"{""location"": ""TZ"", ""is_mobile"": true}" 5416,8,515,2017-02-08 13:55:10,http://becker.co/darby,2.0000000000,168.109.190.124,"{""location"": ""TV"", ""is_mobile"": false}" 5417,8,515,2017-06-07 21:40:51,http://bartolettimonahan.net/lydia,2.0000000000,114.195.49.188,"{""location"": ""CZ"", ""is_mobile"": true}" 5418,8,515,2017-04-04 04:53:26,http://crona.biz/keanu.stiedemann,1.0000000000,181.125.243.61,"{""location"": ""BR"", ""is_mobile"": true}" 5419,8,515,2017-06-02 19:33:55,http://goodwin.com/marina,2.0000000000,119.232.138.87,"{""location"": ""NC"", ""is_mobile"": true}" 5420,8,515,2017-01-22 00:03:09,http://armstrong.name/kaley.torphy,2.0000000000,105.42.172.116,"{""location"": ""CK"", ""is_mobile"": true}" 5421,8,515,2017-05-14 21:07:24,http://jenkins.io/maxwell,1.0000000000,216.158.121.162,"{""location"": ""PH"", ""is_mobile"": false}" 5422,8,515,2017-01-10 20:02:30,http://reichel.name/beulah_stokes,0.0000000000,219.249.114.77,"{""location"": ""BQ"", ""is_mobile"": true}" 5423,8,515,2016-12-31 06:32:22,http://mullerzieme.org/warren.schinner,2.0000000000,25.200.119.190,"{""location"": ""ES"", ""is_mobile"": false}" 5424,8,515,2017-03-27 13:15:57,http://goldnercummerata.io/osbaldo.schaden,1.0000000000,233.64.81.102,"{""location"": ""CR"", ""is_mobile"": false}" 5425,8,515,2017-01-27 19:45:26,http://wiegandmarvin.co/jackson,2.0000000000,83.76.99.203,"{""location"": ""KM"", ""is_mobile"": false}" 5426,8,515,2017-04-13 17:39:03,http://kub.io/ansley.mclaughlin,2.0000000000,124.204.209.57,"{""location"": ""JO"", ""is_mobile"": false}" 5427,8,516,2017-06-12 06:27:13,http://walter.io/chyna.stanton,1.0000000000,33.246.121.161,"{""location"": ""CH"", ""is_mobile"": false}" 5428,8,516,2017-03-04 17:26:51,http://lehnermertz.name/caterina_schmidt,0.0000000000,200.96.180.237,"{""location"": ""VU"", ""is_mobile"": true}" 5429,8,516,2017-02-13 02:12:07,http://carterfadel.name/reie_murray,1.0000000000,201.174.228.176,"{""location"": ""JM"", ""is_mobile"": true}" 5430,8,516,2017-05-27 20:17:20,http://dare.org/benny.dicki,1.0000000000,203.28.140.229,"{""location"": ""GI"", ""is_mobile"": false}" 5431,8,516,2016-12-20 17:31:40,http://handchristiansen.info/hildegard.rogahn,0.0000000000,86.180.4.123,"{""location"": ""UY"", ""is_mobile"": true}" 5432,8,517,2017-02-14 15:31:11,http://sawayn.org/aidan,0.0000000000,53.240.156.251,"{""location"": ""PF"", ""is_mobile"": false}" 5433,8,517,2016-12-18 10:58:32,http://jaskolski.name/dameon,0.0000000000,76.129.41.252,"{""location"": ""JP"", ""is_mobile"": false}" 5434,8,517,2017-04-27 19:52:42,http://nader.net/noah,0.0000000000,57.152.64.49,"{""location"": ""PA"", ""is_mobile"": false}" 5435,8,517,2017-05-25 19:30:48,http://baumbachbednar.biz/lesly,0.0000000000,218.177.246.177,"{""location"": ""VU"", ""is_mobile"": true}" 5436,8,517,2017-06-06 06:04:54,http://davis.info/llewellyn,0.0000000000,39.114.39.139,"{""location"": ""DE"", ""is_mobile"": true}" 5437,8,517,2017-01-30 06:08:48,http://ziemannschoen.io/bud_hammes,0.0000000000,156.216.219.110,"{""location"": ""VC"", ""is_mobile"": false}" 5438,8,517,2017-04-30 10:01:43,http://danielbatz.com/wellington,0.0000000000,202.128.107.213,"{""location"": ""IN"", ""is_mobile"": true}" 5439,8,517,2017-03-04 16:04:43,http://conroy.io/wyman,0.0000000000,127.232.122.71,"{""location"": ""DM"", ""is_mobile"": false}" 5440,8,517,2017-02-25 05:03:16,http://barton.info/clay.kreiger,0.0000000000,163.72.158.20,"{""location"": ""GE"", ""is_mobile"": true}" 5441,8,517,2017-03-15 09:04:24,http://hills.io/darrin,0.0000000000,73.163.60.198,"{""location"": ""SL"", ""is_mobile"": false}" 5442,8,517,2016-12-26 11:10:59,http://ohara.name/fae,0.0000000000,52.239.105.222,"{""location"": ""GH"", ""is_mobile"": true}" 5443,8,517,2017-05-29 02:07:53,http://hilpert.io/skylar,0.0000000000,205.8.163.70,"{""location"": ""GB"", ""is_mobile"": true}" 5444,8,517,2017-02-02 08:32:44,http://bergstrom.name/jazmyn.mueller,0.0000000000,27.190.37.96,"{""location"": ""BH"", ""is_mobile"": true}" 5445,8,517,2017-01-19 09:18:29,http://bartoletti.io/ladarius_lemke,0.0000000000,205.8.163.70,"{""location"": ""KW"", ""is_mobile"": true}" 5446,8,518,2017-01-18 13:29:10,http://bartoletti.co/marjolaine,0.0000000000,180.14.175.133,"{""location"": ""NZ"", ""is_mobile"": false}" 5447,8,518,2017-03-01 01:17:39,http://bernhard.org/kelly,0.0000000000,16.213.51.116,"{""location"": ""DK"", ""is_mobile"": true}" 5448,8,518,2017-05-06 18:17:42,http://wildermanhomenick.info/william.kautzer,0.0000000000,180.14.175.133,"{""location"": ""ES"", ""is_mobile"": true}" 5449,8,518,2017-03-05 23:50:09,http://howesawayn.info/jacky,0.0000000000,183.124.96.242,"{""location"": ""TR"", ""is_mobile"": true}" 5450,8,518,2017-01-23 10:16:54,http://raynorsimonis.co/daniella_botsford,0.0000000000,84.39.80.227,"{""location"": ""ER"", ""is_mobile"": true}" 5451,8,518,2016-12-26 20:59:37,http://pfeffer.biz/joel_nitzsche,0.0000000000,189.45.213.142,"{""location"": ""EE"", ""is_mobile"": false}" 5452,8,518,2017-04-10 01:30:15,http://ratke.info/maud_macejkovic,0.0000000000,56.251.219.82,"{""location"": ""CL"", ""is_mobile"": true}" 5453,8,518,2017-05-11 12:22:34,http://schmittpollich.com/dortha,0.0000000000,207.16.222.156,"{""location"": ""EE"", ""is_mobile"": true}" 5454,8,518,2017-03-05 00:34:00,http://hayes.biz/darrell,0.0000000000,189.66.110.233,"{""location"": ""NP"", ""is_mobile"": true}" 5455,8,518,2017-04-18 11:50:23,http://bradtke.org/millie,0.0000000000,74.127.145.24,"{""location"": ""PR"", ""is_mobile"": false}" 5456,8,518,2017-03-26 20:38:35,http://cartwright.net/willis,0.0000000000,189.45.213.142,"{""location"": ""AS"", ""is_mobile"": true}" 5457,8,518,2016-12-20 17:55:47,http://kuvalis.info/lela_ernser,0.0000000000,61.28.148.247,"{""location"": ""NU"", ""is_mobile"": false}" 5458,8,519,2017-02-17 23:40:02,http://ziemann.info/howard.lubowitz,0.0000000000,167.222.246.35,"{""location"": ""GL"", ""is_mobile"": true}" 5459,8,519,2017-04-03 11:48:50,http://bergnaum.io/roie_collins,1.0000000000,139.6.4.59,"{""location"": ""SA"", ""is_mobile"": true}" 5460,8,519,2017-02-21 16:01:37,http://dibbertnader.net/daisy.zieme,1.0000000000,127.197.105.202,"{""location"": ""ZA"", ""is_mobile"": true}" 5461,8,519,2017-03-03 21:21:18,http://hammes.name/nichole_mante,2.0000000000,127.197.105.202,"{""location"": ""BS"", ""is_mobile"": true}" 5462,8,519,2017-03-06 10:22:32,http://schaeferwelch.io/stanley.konopelski,3.0000000000,122.77.78.119,"{""location"": ""AM"", ""is_mobile"": false}" 5463,8,519,2016-12-19 19:17:14,http://fisher.io/cora,1.0000000000,229.146.160.40,"{""location"": ""CD"", ""is_mobile"": true}" 5464,8,519,2017-04-13 10:42:22,http://schaden.org/godfrey,0.0000000000,238.123.47.53,"{""location"": ""BG"", ""is_mobile"": true}" 221,1,24,2017-02-01 15:36:56,http://murrayfay.name/beryl,1.0000000000,121.83.83.231,"{""location"": ""KH"", ""is_mobile"": false}" 222,1,24,2017-01-26 14:07:58,http://cruickshank.org/june,1.0000000000,57.118.31.98,"{""location"": ""AQ"", ""is_mobile"": false}" 223,1,24,2017-04-07 14:59:55,http://hand.info/bella_padberg,1.0000000000,24.214.117.177,"{""location"": ""AT"", ""is_mobile"": false}" 224,1,24,2017-01-28 22:59:55,http://trantowlubowitz.co/fermin_howe,0.0000000000,183.96.146.199,"{""location"": ""CX"", ""is_mobile"": true}" 225,1,24,2017-05-10 12:09:19,http://zemlakbeahan.biz/moshe,0.0000000000,102.240.56.15,"{""location"": ""AE"", ""is_mobile"": false}" 226,1,24,2017-03-01 14:19:28,http://oconner.biz/carmelo,1.0000000000,241.40.6.49,"{""location"": ""GF"", ""is_mobile"": false}" 227,1,24,2016-12-16 14:26:51,http://adams.info/waino.rosenbaum,1.0000000000,24.214.117.177,"{""location"": ""PY"", ""is_mobile"": false}" 228,1,24,2017-04-26 22:06:24,http://bartell.biz/teagan,0.0000000000,37.110.174.148,"{""location"": ""MQ"", ""is_mobile"": false}" 229,1,24,2017-01-21 10:51:16,http://cain.org/jennyfer,0.0000000000,226.80.88.161,"{""location"": ""CW"", ""is_mobile"": true}" 230,1,24,2017-02-08 05:20:02,http://hermiston.name/elinore.kaulke,0.0000000000,49.72.126.82,"{""location"": ""BA"", ""is_mobile"": false}" 231,1,24,2016-12-29 07:22:42,http://beattymedhurst.biz/lucienne.gusikowski,0.0000000000,131.48.157.169,"{""location"": ""IM"", ""is_mobile"": false}" 232,1,25,2017-01-24 06:54:47,http://hoppe.io/thea_lang,0.0000000000,189.173.223.128,"{""location"": ""BY"", ""is_mobile"": false}" 233,1,25,2017-05-16 07:29:49,http://blockaltenwerth.io/mozell.leannon,0.0000000000,61.35.36.172,"{""location"": ""NF"", ""is_mobile"": false}" 234,1,25,2017-05-17 01:36:45,http://schimmelschaefer.co/georgianna,0.0000000000,215.70.175.52,"{""location"": ""MA"", ""is_mobile"": true}" 235,1,25,2017-02-01 06:23:33,http://sipes.name/terrance_mckenzie,0.0000000000,215.82.122.123,"{""location"": ""MR"", ""is_mobile"": true}" 236,1,25,2017-05-19 21:09:06,http://goyetteturner.org/david.robel,0.0000000000,69.32.156.26,"{""location"": ""KN"", ""is_mobile"": true}" 237,1,25,2017-01-17 09:46:54,http://anderson.net/johnathan.oconnell,0.0000000000,224.27.231.43,"{""location"": ""PE"", ""is_mobile"": false}" 238,1,25,2016-12-20 06:58:05,http://shanahan.io/jeica_wisoky,0.0000000000,17.30.86.76,"{""location"": ""CR"", ""is_mobile"": false}" 239,1,25,2017-02-28 15:04:31,http://mrazleannon.name/alison.goodwin,0.0000000000,166.11.125.91,"{""location"": ""IM"", ""is_mobile"": false}" 240,1,25,2017-04-18 00:07:58,http://marquardtweimann.org/jovany.mayer,0.0000000000,197.12.249.27,"{""location"": ""BH"", ""is_mobile"": true}" 241,1,25,2017-02-25 17:49:03,http://gislasoncollier.name/abbie.turner,0.0000000000,16.16.253.108,"{""location"": ""LK"", ""is_mobile"": false}" 242,1,25,2017-01-16 21:37:18,http://heathcote.net/werner_deckow,0.0000000000,106.43.236.8,"{""location"": ""KW"", ""is_mobile"": false}" 243,1,25,2017-02-22 12:29:15,http://mayer.com/carole.hickle,0.0000000000,68.69.166.152,"{""location"": ""AM"", ""is_mobile"": false}" 244,1,25,2016-12-19 03:16:36,http://morietteharris.name/mafalda,0.0000000000,17.30.86.76,"{""location"": ""VN"", ""is_mobile"": true}" 245,1,25,2017-03-27 23:15:28,http://jacobihermann.biz/dariana,0.0000000000,139.46.201.117,"{""location"": ""FI"", ""is_mobile"": false}" 246,1,25,2017-01-20 19:50:27,http://boehm.biz/kade,0.0000000000,61.68.220.177,"{""location"": ""BR"", ""is_mobile"": true}" 247,1,25,2017-02-20 03:49:46,http://kris.com/amos_wisozk,0.0000000000,195.232.106.119,"{""location"": ""UM"", ""is_mobile"": false}" 248,1,25,2017-01-01 19:46:14,http://johnstonbauch.biz/adriana,0.0000000000,8.91.29.216,"{""location"": ""NE"", ""is_mobile"": false}" 249,1,25,2016-12-18 13:43:32,http://howell.info/gudrun_mertz,0.0000000000,52.221.47.70,"{""location"": ""AR"", ""is_mobile"": true}" 250,1,25,2017-06-10 03:24:24,http://jacobscrist.com/constantin,0.0000000000,17.30.86.76,"{""location"": ""TT"", ""is_mobile"": false}" 251,1,26,2017-01-06 17:43:06,http://ankundingmoen.io/shaylee_beahan,0.0000000000,94.211.243.83,"{""location"": ""VN"", ""is_mobile"": true}" 252,1,26,2017-02-24 07:36:31,http://konopelski.net/valerie_shanahan,0.0000000000,90.251.103.22,"{""location"": ""KP"", ""is_mobile"": true}" 253,1,26,2017-02-13 21:55:08,http://abbott.info/mariah_lubowitz,1.0000000000,193.128.177.24,"{""location"": ""GQ"", ""is_mobile"": false}" 254,1,26,2017-04-02 03:21:34,http://swaniawskihoeger.name/savanna.rosenbaum,0.0000000000,187.233.127.122,"{""location"": ""TJ"", ""is_mobile"": false}" 255,1,26,2017-01-07 23:58:24,http://osinskiferry.biz/linnea_mckenzie,2.0000000000,178.194.40.165,"{""location"": ""KE"", ""is_mobile"": false}" 256,1,26,2017-02-15 04:40:38,http://feesthills.info/jamal.oreilly,2.0000000000,59.112.58.252,"{""location"": ""KG"", ""is_mobile"": true}" 257,1,26,2017-03-17 00:04:03,http://ratke.net/jordane,1.0000000000,235.180.98.160,"{""location"": ""FJ"", ""is_mobile"": true}" 258,1,26,2017-01-10 23:09:35,http://schroeder.info/haylie.jacobi,2.0000000000,179.142.252.238,"{""location"": ""LS"", ""is_mobile"": true}" 259,1,26,2017-05-11 03:54:16,http://gottlieb.com/leie_jacobson,2.0000000000,18.218.175.123,"{""location"": ""MG"", ""is_mobile"": true}" 260,1,27,2016-12-15 05:37:31,http://streich.co/colby.oconner,0.0000000000,189.29.200.198,"{""location"": ""DM"", ""is_mobile"": true}" 261,1,27,2017-05-27 19:42:14,http://zulauf.net/keith,0.0000000000,219.23.174.109,"{""location"": ""SN"", ""is_mobile"": true}" 262,1,27,2017-04-14 00:14:12,http://jacobi.org/vincenza,0.0000000000,51.146.109.132,"{""location"": ""IS"", ""is_mobile"": true}" 263,1,27,2017-06-08 15:03:58,http://hegmann.co/marie.ohara,0.0000000000,247.45.150.148,"{""location"": ""KP"", ""is_mobile"": false}" 264,1,27,2016-12-22 01:00:23,http://nicolas.net/eva,0.0000000000,122.94.29.235,"{""location"": ""GF"", ""is_mobile"": true}" 265,1,28,2017-02-24 08:43:07,http://hettingerlakin.co/frederic,0.0000000000,254.64.57.168,"{""location"": ""KI"", ""is_mobile"": true}" 266,1,28,2017-01-17 10:39:31,http://howe.org/zander_weinat,0.0000000000,49.164.92.24,"{""location"": ""MW"", ""is_mobile"": false}" 267,1,29,2017-01-18 22:19:22,http://wardharvey.org/lela,2.0000000000,186.28.111.145,"{""location"": ""SE"", ""is_mobile"": false}" 268,1,29,2017-02-17 15:33:46,http://mayertgreen.info/jammie,2.0000000000,72.234.226.191,"{""location"": ""BD"", ""is_mobile"": false}" 269,1,29,2017-05-21 19:28:24,http://kertzmannbartell.co/fleta_ratke,1.0000000000,27.99.147.44,"{""location"": ""CC"", ""is_mobile"": false}" 270,1,29,2017-03-04 12:57:43,http://wiegand.io/theodora,0.0000000000,55.66.181.77,"{""location"": ""BM"", ""is_mobile"": false}" 271,1,29,2017-02-13 18:19:13,http://crona.net/shyann,1.0000000000,151.122.12.247,"{""location"": ""KP"", ""is_mobile"": false}" 272,1,29,2017-01-15 01:54:51,http://trantowprohaska.name/dannie.marvin,1.0000000000,221.115.95.86,"{""location"": ""KE"", ""is_mobile"": false}" 5465,8,519,2017-06-05 21:12:27,http://swaniawski.biz/keegan.douglas,2.0000000000,247.151.26.47,"{""location"": ""BJ"", ""is_mobile"": false}" 5466,8,519,2017-04-04 18:51:25,http://stamm.name/gwendolyn.grady,3.0000000000,104.85.190.132,"{""location"": ""CD"", ""is_mobile"": true}" 5467,8,519,2017-01-13 09:41:31,http://tillman.org/eliseo_ritchie,2.0000000000,199.92.225.99,"{""location"": ""TN"", ""is_mobile"": true}" 5468,8,519,2017-03-11 05:22:12,http://swiftjohnson.org/elbert_emard,3.0000000000,247.151.26.47,"{""location"": ""CH"", ""is_mobile"": true}" 5469,8,519,2017-02-14 00:44:13,http://tromp.co/melisa,0.0000000000,247.151.26.47,"{""location"": ""TN"", ""is_mobile"": true}" 5470,8,519,2017-04-22 21:08:37,http://trantow.com/osbaldo,2.0000000000,212.119.22.175,"{""location"": ""NC"", ""is_mobile"": false}" 5471,8,519,2017-06-03 02:58:41,http://leuschke.name/shayna_keler,0.0000000000,212.119.22.175,"{""location"": ""AU"", ""is_mobile"": true}" 5472,8,520,2016-12-14 17:14:52,http://durgan.org/rex_swaniawski,0.0000000000,160.55.113.28,"{""location"": ""QA"", ""is_mobile"": true}" 5473,8,520,2017-04-24 06:33:27,http://emmerichmacejkovic.com/edward.abernathy,1.0000000000,178.99.85.175,"{""location"": ""FK"", ""is_mobile"": false}" 5474,8,520,2017-04-04 21:59:43,http://johns.info/blanca,1.0000000000,21.64.110.127,"{""location"": ""LC"", ""is_mobile"": false}" 5475,8,520,2017-02-04 17:28:34,http://lakinkulas.net/brigitte.gleichner,1.0000000000,252.170.102.79,"{""location"": ""SZ"", ""is_mobile"": false}" 5476,8,520,2017-04-14 19:24:22,http://parisian.com/nash_hagenes,1.0000000000,4.170.250.119,"{""location"": ""PH"", ""is_mobile"": false}" 5477,8,520,2017-01-10 21:09:41,http://stark.biz/roslyn.bergnaum,0.0000000000,140.19.214.145,"{""location"": ""GS"", ""is_mobile"": false}" 5478,8,520,2017-05-14 12:14:33,http://larkin.co/mittie,0.0000000000,252.170.102.79,"{""location"": ""BQ"", ""is_mobile"": false}" 5479,8,520,2017-03-22 17:05:18,http://bartell.biz/nat,1.0000000000,172.237.162.43,"{""location"": ""MY"", ""is_mobile"": false}" 5480,8,521,2017-06-07 18:16:53,http://mclaughlinfay.info/maybelle.hoppe,0.0000000000,177.114.95.198,"{""location"": ""NE"", ""is_mobile"": true}" 5481,8,521,2017-02-10 18:47:53,http://lubowitz.net/retta,0.0000000000,218.240.22.159,"{""location"": ""NO"", ""is_mobile"": false}" 5482,8,521,2017-03-24 05:28:06,http://goyette.net/juwan.mertz,0.0000000000,26.46.200.18,"{""location"": ""MF"", ""is_mobile"": false}" 5483,8,521,2017-06-08 14:22:47,http://mertz.org/wilmer_connelly,0.0000000000,20.73.172.106,"{""location"": ""AM"", ""is_mobile"": true}" 5484,8,521,2017-03-06 01:58:20,http://hayes.org/eva,0.0000000000,173.190.12.187,"{""location"": ""NC"", ""is_mobile"": true}" 5485,8,521,2016-12-22 04:33:40,http://durgan.com/florencio,0.0000000000,113.252.139.69,"{""location"": ""SA"", ""is_mobile"": false}" 5486,8,521,2017-04-25 20:28:49,http://heller.biz/jaime,0.0000000000,41.49.176.156,"{""location"": ""BW"", ""is_mobile"": true}" 5487,8,521,2016-12-26 02:45:25,http://robel.biz/maddison,0.0000000000,150.112.216.138,"{""location"": ""BH"", ""is_mobile"": true}" 5488,8,521,2017-03-10 11:40:06,http://emmerich.org/blanche_smitham,0.0000000000,245.132.147.222,"{""location"": ""ET"", ""is_mobile"": false}" 5489,8,521,2017-01-16 01:56:35,http://streich.org/rosina.gibson,0.0000000000,67.36.34.241,"{""location"": ""BH"", ""is_mobile"": false}" 5490,8,521,2017-06-02 09:53:27,http://waelchi.net/torrey,0.0000000000,66.240.3.109,"{""location"": ""KY"", ""is_mobile"": false}" 5491,8,521,2017-03-22 05:49:47,http://lesch.com/jolie,0.0000000000,66.240.3.109,"{""location"": ""BW"", ""is_mobile"": true}" 5492,8,521,2017-05-16 08:04:58,http://hudson.biz/joanne.olson,0.0000000000,94.35.220.128,"{""location"": ""QA"", ""is_mobile"": true}" 5493,8,521,2017-03-10 11:38:49,http://hartmann.biz/chesley.abernathy,0.0000000000,173.42.196.106,"{""location"": ""EH"", ""is_mobile"": false}" 5494,8,522,2017-01-22 18:41:57,http://olson.info/terence,0.0000000000,60.71.131.224,"{""location"": ""CU"", ""is_mobile"": true}" 5495,8,522,2017-01-23 04:51:50,http://haley.biz/herbert.bartoletti,0.0000000000,208.31.160.41,"{""location"": ""RU"", ""is_mobile"": false}" 5496,8,522,2017-05-11 08:32:26,http://schowalterschmidt.biz/dino,0.0000000000,60.71.131.224,"{""location"": ""PR"", ""is_mobile"": false}" 5497,8,522,2017-04-06 16:10:19,http://handherzog.org/elouise,0.0000000000,43.169.198.35,"{""location"": ""GH"", ""is_mobile"": true}" 5498,8,522,2017-05-25 16:24:12,http://vandervort.co/elisha,0.0000000000,11.29.123.149,"{""location"": ""CV"", ""is_mobile"": true}" 5499,8,522,2017-03-19 00:00:13,http://oconnerlehner.info/pasquale,0.0000000000,134.18.32.116,"{""location"": ""SJ"", ""is_mobile"": false}" 5500,8,522,2017-06-06 22:07:41,http://ankunding.info/leopold,0.0000000000,123.64.75.133,"{""location"": ""SO"", ""is_mobile"": true}" 5501,8,522,2017-05-14 09:44:50,http://rempel.org/gustave_langworth,0.0000000000,168.172.24.124,"{""location"": ""LK"", ""is_mobile"": true}" 5502,8,522,2017-01-24 21:38:43,http://tillman.io/angel,0.0000000000,27.113.167.114,"{""location"": ""GB"", ""is_mobile"": true}" 5503,8,522,2017-02-04 01:08:58,http://konopelskikeler.com/marlin,0.0000000000,168.172.24.124,"{""location"": ""NI"", ""is_mobile"": false}" 5504,8,522,2017-02-01 03:29:42,http://fay.name/enrico.corwin,0.0000000000,165.43.140.213,"{""location"": ""KH"", ""is_mobile"": false}" 5505,8,522,2017-03-03 12:56:15,http://prohaskareilly.biz/winifred,0.0000000000,71.172.75.240,"{""location"": ""BQ"", ""is_mobile"": false}" 5506,8,523,2017-06-11 02:57:29,http://lowe.io/raven_king,1.0000000000,216.46.254.175,"{""location"": ""MH"", ""is_mobile"": true}" 5507,8,523,2016-12-28 07:50:37,http://johnson.net/deontae_breitenberg,1.0000000000,243.136.88.60,"{""location"": ""MN"", ""is_mobile"": true}" 5508,8,523,2017-04-10 23:58:09,http://crist.name/rory.lemke,2.0000000000,160.128.53.211,"{""location"": ""SB"", ""is_mobile"": false}" 5509,8,523,2016-12-23 08:15:23,http://goldnerfeil.io/marcia.welch,0.0000000000,252.22.80.89,"{""location"": ""CU"", ""is_mobile"": true}" 5510,8,523,2017-02-15 08:00:56,http://hammes.io/jay,2.0000000000,28.98.68.157,"{""location"": ""CK"", ""is_mobile"": true}" 5511,8,523,2017-05-30 15:07:51,http://baumbachhammes.co/kaya,1.0000000000,198.190.238.131,"{""location"": ""SE"", ""is_mobile"": false}" 5512,8,523,2017-02-23 15:22:07,http://gulgowski.io/brando_dach,2.0000000000,200.192.220.153,"{""location"": ""HR"", ""is_mobile"": false}" 5513,8,523,2017-02-17 04:58:02,http://wiegand.co/kevon.ankunding,0.0000000000,16.148.120.113,"{""location"": ""HT"", ""is_mobile"": false}" 5514,8,523,2016-12-18 05:54:51,http://mohrhintz.net/lon,2.0000000000,70.182.196.176,"{""location"": ""TJ"", ""is_mobile"": false}" 5515,8,523,2017-01-18 08:23:03,http://reingerhuel.name/wilson_crist,0.0000000000,56.44.228.44,"{""location"": ""NR"", ""is_mobile"": false}" 5516,8,523,2017-02-13 08:07:55,http://starkkautzer.name/jett_runolfon,2.0000000000,16.134.14.192,"{""location"": ""TN"", ""is_mobile"": true}" 273,1,29,2017-02-02 05:54:08,http://gerhold.biz/norris_stamm,0.0000000000,148.102.86.198,"{""location"": ""BG"", ""is_mobile"": true}" 274,1,29,2017-04-16 18:39:20,http://ernser.biz/fanny,2.0000000000,18.188.161.28,"{""location"": ""AW"", ""is_mobile"": true}" 275,1,29,2017-03-24 02:30:32,http://rogahn.io/bethel,3.0000000000,245.38.130.246,"{""location"": ""EC"", ""is_mobile"": true}" 276,1,29,2017-03-25 03:00:58,http://gaylordsteuber.biz/maia,3.0000000000,60.224.111.123,"{""location"": ""PG"", ""is_mobile"": false}" 277,1,29,2017-05-17 05:17:49,http://lehner.com/alberta,0.0000000000,18.188.161.28,"{""location"": ""IS"", ""is_mobile"": false}" 278,1,29,2017-04-01 03:36:50,http://nader.co/rubye,0.0000000000,151.13.53.59,"{""location"": ""NG"", ""is_mobile"": false}" 279,1,29,2017-03-02 05:26:26,http://bartell.org/katarina,3.0000000000,224.245.95.149,"{""location"": ""BY"", ""is_mobile"": false}" 280,1,29,2017-03-28 19:40:48,http://mitchell.info/dorthy,1.0000000000,55.66.181.77,"{""location"": ""TR"", ""is_mobile"": true}" 281,1,30,2017-03-02 11:58:28,http://cartwrighthand.co/novella.rohan,0.0000000000,111.36.12.238,"{""location"": ""PA"", ""is_mobile"": false}" 282,1,30,2017-05-13 17:44:46,http://waelchimacgyver.name/jerrold.boehm,0.0000000000,125.246.138.12,"{""location"": ""LA"", ""is_mobile"": false}" 283,1,30,2017-01-15 07:05:18,http://turnerhintz.co/aurore_reinger,0.0000000000,168.194.154.31,"{""location"": ""JM"", ""is_mobile"": false}" 284,1,30,2017-06-13 15:01:21,http://beckerdavis.io/israel,0.0000000000,25.147.67.91,"{""location"": ""NF"", ""is_mobile"": false}" 285,1,30,2017-02-11 19:28:47,http://hauck.biz/maureen_durgan,0.0000000000,50.43.62.62,"{""location"": ""NU"", ""is_mobile"": false}" 286,1,30,2016-12-29 01:32:25,http://gorczany.net/xavier_schiller,0.0000000000,215.91.116.164,"{""location"": ""MX"", ""is_mobile"": false}" 287,1,30,2017-05-08 02:40:50,http://pollich.net/roosevelt.keeling,0.0000000000,230.236.77.86,"{""location"": ""RW"", ""is_mobile"": false}" 288,1,30,2017-05-01 08:08:54,http://leffler.co/tatum.nikolaus,0.0000000000,68.17.173.171,"{""location"": ""TL"", ""is_mobile"": true}" 289,1,30,2017-05-16 13:14:09,http://rolfsonbruen.io/obie_simonis,0.0000000000,155.236.15.168,"{""location"": ""AX"", ""is_mobile"": true}" 290,1,31,2017-04-18 04:01:14,http://bins.org/mae,1.0000000000,136.125.159.147,"{""location"": ""VN"", ""is_mobile"": false}" 291,1,31,2017-02-02 00:36:04,http://heidenreichborer.biz/brady.kiehn,0.0000000000,33.26.244.205,"{""location"": ""NR"", ""is_mobile"": false}" 292,1,31,2017-03-24 16:22:52,http://schulist.net/jedediah_wehner,1.0000000000,211.96.214.86,"{""location"": ""CF"", ""is_mobile"": true}" 293,1,31,2017-05-18 11:54:36,http://denesik.co/micaela,1.0000000000,169.121.11.85,"{""location"": ""SC"", ""is_mobile"": true}" 294,1,31,2017-05-16 21:40:39,http://mrazmills.org/davonte.von,1.0000000000,173.183.191.184,"{""location"": ""BV"", ""is_mobile"": false}" 295,1,31,2017-04-21 21:37:55,http://rogahn.io/amara_sauer,1.0000000000,227.86.15.230,"{""location"": ""MT"", ""is_mobile"": false}" 296,1,32,2016-12-27 12:54:03,http://boehmmedhurst.org/dangelo,0.0000000000,118.174.96.212,"{""location"": ""SB"", ""is_mobile"": true}" 297,1,32,2017-05-15 02:22:56,http://harber.net/raleigh,0.0000000000,141.227.254.7,"{""location"": ""NO"", ""is_mobile"": true}" 298,1,32,2017-04-16 15:03:50,http://crooks.io/francisco_hayes,1.0000000000,113.58.165.151,"{""location"": ""KP"", ""is_mobile"": true}" 299,1,32,2017-06-03 12:14:42,http://hintzcorwin.info/lela_grimes,0.0000000000,229.250.195.180,"{""location"": ""LI"", ""is_mobile"": true}" 300,1,32,2017-04-27 02:34:29,http://leuschkevon.name/ivory,1.0000000000,147.86.154.248,"{""location"": ""EG"", ""is_mobile"": false}" 301,1,32,2017-01-11 11:12:44,http://wintheiser.name/norma.ernser,1.0000000000,229.250.195.180,"{""location"": ""SY"", ""is_mobile"": true}" 302,1,32,2017-01-25 15:33:33,http://kling.info/deshaun.mills,0.0000000000,134.236.50.140,"{""location"": ""BB"", ""is_mobile"": false}" 303,1,32,2017-06-01 12:50:06,http://ritchieskiles.info/marjorie_kautzer,0.0000000000,144.152.69.220,"{""location"": ""AW"", ""is_mobile"": true}" 304,1,32,2017-04-04 23:08:48,http://bogisich.biz/kamille,1.0000000000,248.14.251.188,"{""location"": ""TF"", ""is_mobile"": false}" 305,1,32,2017-05-23 22:37:13,http://rohan.name/oral.abernathy,1.0000000000,134.236.50.140,"{""location"": ""AR"", ""is_mobile"": false}" 306,1,32,2017-01-14 09:45:38,http://larson.info/hildegard,0.0000000000,11.200.134.64,"{""location"": ""NA"", ""is_mobile"": true}" 307,1,32,2017-03-10 06:15:54,http://kunze.org/shanna,0.0000000000,199.80.210.50,"{""location"": ""KG"", ""is_mobile"": true}" 308,1,32,2016-12-30 01:03:10,http://oberbrunnerkeeling.org/deion,1.0000000000,181.23.74.127,"{""location"": ""RS"", ""is_mobile"": false}" 309,1,32,2017-02-21 22:44:51,http://collierpadberg.biz/ed_kuvalis,1.0000000000,147.86.154.248,"{""location"": ""NA"", ""is_mobile"": false}" 310,1,32,2017-05-05 22:18:50,http://reilly.org/timothy.wolff,1.0000000000,228.195.75.204,"{""location"": ""RU"", ""is_mobile"": true}" 311,1,32,2017-05-06 08:54:58,http://herzog.co/donavon,1.0000000000,114.49.140.83,"{""location"": ""HM"", ""is_mobile"": true}" 312,1,33,2017-03-06 09:38:27,http://lakinboyer.info/eloise,0.0000000000,216.230.179.184,"{""location"": ""SB"", ""is_mobile"": true}" 313,1,33,2017-02-10 07:55:48,http://gutkowski.info/marc_langworth,0.0000000000,163.60.88.62,"{""location"": ""MV"", ""is_mobile"": true}" 314,1,33,2017-03-20 13:52:27,http://runolfon.org/beatrice.streich,0.0000000000,95.17.231.156,"{""location"": ""LU"", ""is_mobile"": false}" 315,1,33,2017-02-10 02:37:03,http://weimann.org/eudora,0.0000000000,93.41.207.99,"{""location"": ""BE"", ""is_mobile"": false}" 316,1,33,2017-03-31 06:38:41,http://hodkiewiczrunolfon.info/blanche,0.0000000000,183.92.82.238,"{""location"": ""LU"", ""is_mobile"": false}" 317,1,33,2017-01-08 09:07:55,http://roberts.net/paul,0.0000000000,95.17.231.156,"{""location"": ""LV"", ""is_mobile"": true}" 318,1,33,2017-03-04 21:49:34,http://dietrichhaley.co/lolita,0.0000000000,123.157.129.112,"{""location"": ""FR"", ""is_mobile"": true}" 319,1,33,2017-06-03 00:58:46,http://durgan.co/stefan_reilly,0.0000000000,84.110.191.115,"{""location"": ""GG"", ""is_mobile"": true}" 320,1,33,2017-04-12 04:43:15,http://auerziemann.co/mackenzie.hilpert,0.0000000000,38.230.23.42,"{""location"": ""VA"", ""is_mobile"": true}" 321,1,33,2017-03-04 22:51:28,http://heaney.io/yolanda_nikolaus,0.0000000000,102.11.214.137,"{""location"": ""NE"", ""is_mobile"": false}" 322,1,33,2017-05-03 19:13:43,http://murphystehr.org/mabel,0.0000000000,100.199.206.163,"{""location"": ""KZ"", ""is_mobile"": false}" 323,1,33,2017-04-24 05:07:03,http://colekuhn.biz/mona.dickinson,0.0000000000,214.230.233.180,"{""location"": ""BL"", ""is_mobile"": false}" 324,1,33,2017-06-11 04:04:51,http://volkmanpadberg.org/mortimer,0.0000000000,114.40.180.171,"{""location"": ""SA"", ""is_mobile"": true}" 5517,8,523,2017-02-23 09:40:19,http://wisozk.net/ima_turcotte,0.0000000000,116.140.95.123,"{""location"": ""UZ"", ""is_mobile"": false}" 5518,8,523,2017-05-17 20:05:05,http://okon.net/zita.willms,2.0000000000,241.159.195.79,"{""location"": ""US"", ""is_mobile"": false}" 5519,8,523,2017-03-15 14:42:53,http://mante.co/alba,0.0000000000,16.134.14.192,"{""location"": ""AU"", ""is_mobile"": true}" 5520,8,523,2016-12-25 15:58:52,http://fahey.io/turner,2.0000000000,31.61.75.201,"{""location"": ""MC"", ""is_mobile"": true}" 5521,8,523,2017-01-11 06:43:04,http://volkman.biz/aidan.howe,2.0000000000,164.12.207.122,"{""location"": ""DE"", ""is_mobile"": false}" 5522,8,523,2017-02-13 11:19:47,http://medhurststamm.name/demetris,1.0000000000,66.56.237.93,"{""location"": ""ST"", ""is_mobile"": true}" 5523,8,523,2017-04-01 12:13:33,http://metz.biz/emmy,1.0000000000,129.3.207.134,"{""location"": ""LC"", ""is_mobile"": true}" 5524,8,523,2017-03-29 02:05:24,http://rowegoyette.co/kiley,1.0000000000,30.168.245.207,"{""location"": ""WF"", ""is_mobile"": false}" 5525,8,524,2017-01-23 19:41:21,http://prosacco.co/gerhard.jacobs,1.0000000000,204.54.186.123,"{""location"": ""GG"", ""is_mobile"": false}" 5526,8,524,2016-12-19 13:50:22,http://moen.info/retta,2.0000000000,190.216.84.246,"{""location"": ""SK"", ""is_mobile"": false}" 5527,8,524,2017-06-10 00:10:52,http://kshlerin.biz/diamond,1.0000000000,87.90.185.8,"{""location"": ""NF"", ""is_mobile"": false}" 5528,8,524,2017-02-18 14:56:00,http://bergstromabshire.com/jaleel_padberg,1.0000000000,187.26.16.84,"{""location"": ""PY"", ""is_mobile"": false}" 5529,8,524,2016-12-30 07:14:14,http://auer.biz/eugene_turcotte,1.0000000000,87.250.146.56,"{""location"": ""BE"", ""is_mobile"": true}" 5530,8,524,2017-02-19 11:50:50,http://johns.io/vergie_aufderhar,0.0000000000,81.74.143.164,"{""location"": ""AQ"", ""is_mobile"": false}" 5531,8,524,2017-01-14 04:50:26,http://leannon.io/tyrique_blanda,0.0000000000,81.74.143.164,"{""location"": ""GG"", ""is_mobile"": false}" 5532,8,524,2017-05-09 16:12:27,http://okuneva.org/eleanora_bradtke,0.0000000000,11.132.48.249,"{""location"": ""HR"", ""is_mobile"": true}" 5533,8,524,2017-03-19 08:21:35,http://raynor.name/syble,1.0000000000,159.157.253.62,"{""location"": ""AG"", ""is_mobile"": true}" 5534,8,524,2017-02-15 19:23:22,http://kautzer.org/allison,0.0000000000,26.199.118.88,"{""location"": ""MV"", ""is_mobile"": false}" 5535,8,524,2017-05-31 00:03:58,http://tillmanjacobi.io/dorian_stehr,1.0000000000,187.26.16.84,"{""location"": ""JE"", ""is_mobile"": true}" 5536,8,524,2017-05-29 17:28:12,http://stamm.net/delores,1.0000000000,153.8.120.28,"{""location"": ""KM"", ""is_mobile"": false}" 5537,8,524,2017-01-19 00:59:48,http://bernier.net/eladio,2.0000000000,122.149.77.244,"{""location"": ""VA"", ""is_mobile"": false}" 5538,8,524,2017-03-17 15:40:12,http://nienowrunolfon.co/esta_stehr,2.0000000000,70.131.79.131,"{""location"": ""MT"", ""is_mobile"": true}" 5539,8,524,2017-04-07 12:30:49,http://heathcote.net/major,0.0000000000,190.217.76.170,"{""location"": ""MR"", ""is_mobile"": false}" 5540,8,525,2017-01-16 03:23:32,http://altenwerthmarks.name/rosalinda,2.0000000000,173.151.26.146,"{""location"": ""PA"", ""is_mobile"": true}" 5541,8,525,2017-03-10 09:27:14,http://klein.name/joanie_keeling,1.0000000000,178.83.232.95,"{""location"": ""KH"", ""is_mobile"": true}" 325,1,33,2017-04-26 23:29:33,http://boscorau.co/alta.ebert,0.0000000000,123.157.129.112,"{""location"": ""SA"", ""is_mobile"": false}" 326,1,33,2017-03-12 15:04:38,http://wisozkrolfson.io/wilfred,0.0000000000,201.19.214.183,"{""location"": ""SG"", ""is_mobile"": true}" 327,1,33,2017-02-24 11:25:12,http://blandawuckert.co/lilla,0.0000000000,149.51.104.181,"{""location"": ""UY"", ""is_mobile"": false}" 328,1,33,2017-03-18 07:48:17,http://predovic.info/shanie,0.0000000000,57.4.56.71,"{""location"": ""KN"", ""is_mobile"": true}" 329,1,33,2017-04-08 17:57:18,http://herzog.io/troy_okuneva,0.0000000000,7.68.103.8,"{""location"": ""US"", ""is_mobile"": true}" 330,1,33,2017-04-15 19:50:06,http://mertzhaag.org/germaine,0.0000000000,63.56.20.77,"{""location"": ""GP"", ""is_mobile"": true}" 331,1,34,2017-05-27 02:20:40,http://litteltoy.co/eloise,2.0000000000,238.232.229.17,"{""location"": ""IN"", ""is_mobile"": true}" 332,1,34,2017-02-20 17:06:39,http://hegmann.io/eldora_stroman,3.0000000000,51.246.130.100,"{""location"": ""AQ"", ""is_mobile"": false}" 333,1,34,2017-06-09 19:45:44,http://pollich.co/mellie_kutch,1.0000000000,210.125.100.109,"{""location"": ""SJ"", ""is_mobile"": false}" 334,1,34,2017-04-12 14:12:07,http://runolfon.info/richie,1.0000000000,50.222.59.171,"{""location"": ""MD"", ""is_mobile"": false}" 335,1,34,2017-04-07 08:48:06,http://schulist.co/julio.kohler,0.0000000000,10.248.130.230,"{""location"": ""TW"", ""is_mobile"": false}" 336,1,34,2016-12-17 05:37:24,http://ko.com/sabryna,2.0000000000,91.10.113.168,"{""location"": ""CY"", ""is_mobile"": true}" 337,1,34,2016-12-25 18:54:57,http://sipes.biz/destiny_feeney,0.0000000000,95.152.139.55,"{""location"": ""ET"", ""is_mobile"": false}" 338,1,34,2017-06-09 08:30:41,http://bednargraham.biz/jakob,1.0000000000,56.194.191.194,"{""location"": ""KN"", ""is_mobile"": false}" 339,1,34,2016-12-22 07:43:51,http://krishuels.name/oral,0.0000000000,103.169.127.143,"{""location"": ""ML"", ""is_mobile"": true}" 340,1,34,2017-02-28 13:33:44,http://heaney.biz/rhiannon,3.0000000000,137.107.37.200,"{""location"": ""SZ"", ""is_mobile"": false}" 341,1,34,2017-03-27 18:04:24,http://rohanschmeler.io/kasey,2.0000000000,148.70.250.49,"{""location"": ""AG"", ""is_mobile"": true}" 342,1,34,2017-04-07 15:44:18,http://schinnerframi.biz/laron,2.0000000000,203.184.153.242,"{""location"": ""YT"", ""is_mobile"": false}" 343,1,34,2017-03-11 15:22:57,http://bashirianrenner.biz/vivienne.dooley,2.0000000000,137.107.37.200,"{""location"": ""VI"", ""is_mobile"": false}" 344,1,34,2017-02-25 19:47:40,http://lubowitz.org/florian_gibson,0.0000000000,210.125.100.109,"{""location"": ""BF"", ""is_mobile"": false}" 345,1,34,2017-05-08 16:47:49,http://corkery.name/hilbert,3.0000000000,50.222.59.171,"{""location"": ""TO"", ""is_mobile"": false}" 346,1,34,2017-05-28 23:00:45,http://vandervort.net/alfred,1.0000000000,34.32.100.22,"{""location"": ""TM"", ""is_mobile"": true}" 347,1,35,2016-12-16 11:46:49,http://shieldswilliamson.biz/delbert_wilderman,0.0000000000,143.210.138.177,"{""location"": ""GL"", ""is_mobile"": true}" 348,1,35,2017-01-27 14:30:28,http://lynch.com/asha.bahringer,0.0000000000,84.78.118.133,"{""location"": ""BQ"", ""is_mobile"": true}" 349,1,35,2017-03-14 18:20:31,http://wiegand.io/amber,0.0000000000,246.77.30.185,"{""location"": ""ES"", ""is_mobile"": true}" 350,1,35,2017-03-25 08:02:47,http://quitzon.info/elmo,0.0000000000,173.140.171.156,"{""location"": ""TW"", ""is_mobile"": true}" 351,1,35,2017-03-16 21:29:17,http://macejkovic.net/giovani.pagac,0.0000000000,85.248.193.179,"{""location"": ""RO"", ""is_mobile"": false}" 352,1,36,2017-04-04 04:07:35,http://okon.net/hannah,2.0000000000,85.121.215.213,"{""location"": ""UZ"", ""is_mobile"": true}" 353,1,36,2016-12-26 01:21:14,http://johnsonschowalter.info/kristofer,1.0000000000,237.223.135.138,"{""location"": ""CL"", ""is_mobile"": false}" 354,1,36,2017-03-27 04:07:06,http://stokeskutch.org/kolby,2.0000000000,240.182.153.72,"{""location"": ""GD"", ""is_mobile"": true}" 355,1,36,2016-12-24 07:50:31,http://klockoferry.co/theresia_hauck,2.0000000000,64.152.24.50,"{""location"": ""ST"", ""is_mobile"": true}" 356,1,36,2017-02-10 08:22:23,http://hermanrath.co/wilton,0.0000000000,194.55.8.74,"{""location"": ""IT"", ""is_mobile"": true}" 357,1,36,2017-06-10 12:04:13,http://murphy.com/ricardo,0.0000000000,85.121.215.213,"{""location"": ""KR"", ""is_mobile"": true}" 358,1,36,2017-02-23 20:03:19,http://johns.info/denis_koch,1.0000000000,115.37.239.81,"{""location"": ""MZ"", ""is_mobile"": true}" 359,1,36,2016-12-24 18:24:43,http://kirlingreenfelder.biz/madaline.ritchie,1.0000000000,180.191.169.118,"{""location"": ""NR"", ""is_mobile"": true}" 360,1,36,2017-04-23 06:40:34,http://oberbrunner.info/daniela_beer,0.0000000000,248.203.213.119,"{""location"": ""ID"", ""is_mobile"": true}" 361,1,36,2017-06-10 20:30:29,http://gottlieb.com/lon,1.0000000000,25.251.205.155,"{""location"": ""LI"", ""is_mobile"": false}" 362,1,36,2017-04-26 16:08:00,http://lemkecasper.name/aisha_ferry,0.0000000000,194.55.8.74,"{""location"": ""SS"", ""is_mobile"": false}" 363,1,36,2017-03-26 21:59:37,http://oberbrunner.com/martina,2.0000000000,26.57.15.169,"{""location"": ""SC"", ""is_mobile"": true}" 364,1,36,2017-01-29 15:24:26,http://yundt.co/lynn_koelpin,2.0000000000,158.72.146.86,"{""location"": ""BR"", ""is_mobile"": false}" 365,1,36,2017-02-09 14:25:54,http://bayervandervort.biz/floy.wolff,2.0000000000,237.251.225.144,"{""location"": ""CN"", ""is_mobile"": false}" 366,1,36,2016-12-27 19:58:51,http://maggio.info/myriam_doyle,2.0000000000,248.203.213.119,"{""location"": ""ML"", ""is_mobile"": false}" 367,1,36,2017-03-29 15:48:28,http://gorczany.org/sylvan,0.0000000000,60.50.110.155,"{""location"": ""AS"", ""is_mobile"": false}" 368,1,36,2017-02-16 02:01:05,http://pfeffer.biz/mayra,0.0000000000,234.199.239.10,"{""location"": ""VI"", ""is_mobile"": true}" 369,1,36,2017-05-18 08:30:53,http://schummullrich.name/claude,0.0000000000,5.219.171.61,"{""location"": ""BZ"", ""is_mobile"": true}" 370,1,37,2016-12-19 12:51:40,http://fadeldamore.net/piper,0.0000000000,186.109.222.26,"{""location"": ""BJ"", ""is_mobile"": false}" 371,1,37,2017-01-06 21:56:47,http://bradtke.net/haylee,2.0000000000,233.2.156.103,"{""location"": ""VN"", ""is_mobile"": true}" 372,1,37,2017-02-01 18:09:21,http://moriette.com/alford_watsica,3.0000000000,206.68.69.216,"{""location"": ""TZ"", ""is_mobile"": true}" 373,1,37,2017-06-12 08:45:05,http://swaniawski.io/dangelo.kiehn,3.0000000000,234.52.151.227,"{""location"": ""LS"", ""is_mobile"": true}" 374,1,37,2017-02-07 04:12:13,http://hudson.net/jennifer,0.0000000000,167.40.30.133,"{""location"": ""BJ"", ""is_mobile"": true}" 375,1,37,2016-12-18 11:24:08,http://oreilly.org/elmira_kertzmann,2.0000000000,128.124.105.187,"{""location"": ""PM"", ""is_mobile"": true}" 376,1,37,2016-12-25 05:48:09,http://blandarenner.biz/jolie.reichert,1.0000000000,3.159.228.202,"{""location"": ""RE"", ""is_mobile"": true}" 377,1,38,2017-05-12 05:27:38,http://dickenshoppe.info/marisa,0.0000000000,113.214.152.158,"{""location"": ""LR"", ""is_mobile"": false}" 378,1,39,2017-03-06 20:54:44,http://moriettegleichner.net/noelia.heaney,1.0000000000,60.150.72.124,"{""location"": ""CH"", ""is_mobile"": false}" 379,1,39,2016-12-26 09:15:02,http://beierchristiansen.biz/trevion,1.0000000000,96.85.73.213,"{""location"": ""PS"", ""is_mobile"": false}" 380,1,39,2017-02-24 16:39:22,http://wisozk.com/rylan,1.0000000000,228.194.188.82,"{""location"": ""ZM"", ""is_mobile"": false}" 381,1,39,2017-02-23 01:04:17,http://connelly.info/drew,1.0000000000,174.66.71.52,"{""location"": ""SL"", ""is_mobile"": true}" 382,1,39,2017-06-06 09:47:16,http://feeneyaufderhar.name/blaise,1.0000000000,81.52.192.165,"{""location"": ""DJ"", ""is_mobile"": true}" 383,1,39,2017-04-18 23:55:23,http://wilkinson.co/hershel,1.0000000000,197.160.115.195,"{""location"": ""ME"", ""is_mobile"": false}" 384,1,39,2017-05-19 03:57:43,http://leuschke.co/vada,0.0000000000,49.222.204.13,"{""location"": ""PS"", ""is_mobile"": true}" 385,1,39,2017-03-26 09:10:56,http://darehilpert.info/trycia,0.0000000000,252.13.179.103,"{""location"": ""BQ"", ""is_mobile"": false}" 386,1,39,2017-02-14 07:33:20,http://jakubowski.biz/cordelia.hoppe,1.0000000000,34.127.16.132,"{""location"": ""FR"", ""is_mobile"": true}" 387,1,39,2017-03-07 19:29:37,http://brakus.info/loyal,0.0000000000,91.50.70.140,"{""location"": ""VE"", ""is_mobile"": true}" 388,1,39,2017-05-14 01:24:23,http://schulistboehm.name/shawn,1.0000000000,174.66.71.52,"{""location"": ""MO"", ""is_mobile"": true}" 389,1,39,2017-02-18 16:59:30,http://rempel.info/nikita,0.0000000000,15.57.28.86,"{""location"": ""KG"", ""is_mobile"": true}" 390,1,39,2017-01-18 09:46:09,http://batz.info/kay,0.0000000000,96.85.73.213,"{""location"": ""LS"", ""is_mobile"": true}" 391,1,39,2017-04-25 00:45:36,http://bruenebert.biz/jacinthe_mertz,1.0000000000,252.13.179.103,"{""location"": ""TH"", ""is_mobile"": false}" 392,1,39,2017-04-02 12:11:42,http://kelergutmann.org/cary_dooley,0.0000000000,127.110.237.58,"{""location"": ""SC"", ""is_mobile"": false}" 393,1,39,2017-05-16 06:55:14,http://oreilly.com/beverly,0.0000000000,119.109.217.133,"{""location"": ""AR"", ""is_mobile"": false}" 394,1,39,2017-03-11 10:20:00,http://will.info/antonetta,0.0000000000,181.52.126.101,"{""location"": ""PA"", ""is_mobile"": false}" 395,1,39,2017-03-07 12:50:56,http://luettgen.net/rosemary,0.0000000000,253.8.79.81,"{""location"": ""GF"", ""is_mobile"": true}" 396,1,40,2016-12-15 09:33:34,http://williamsonoconnell.io/felicity_gerlach,2.0000000000,44.76.228.249,"{""location"": ""CR"", ""is_mobile"": false}" 397,1,40,2017-03-02 06:11:42,http://heller.io/floyd.anderson,0.0000000000,200.109.87.172,"{""location"": ""MR"", ""is_mobile"": true}" 398,1,41,2017-04-28 04:57:31,http://cronin.info/deion_bartoletti,1.0000000000,148.178.76.76,"{""location"": ""SK"", ""is_mobile"": false}" 399,1,41,2017-02-25 01:30:35,http://kris.net/eveline_rippin,2.0000000000,211.218.92.248,"{""location"": ""DE"", ""is_mobile"": false}" 400,1,41,2017-04-18 23:35:28,http://gislason.name/jerel,2.0000000000,242.13.82.58,"{""location"": ""ZM"", ""is_mobile"": true}" 401,1,42,2017-02-19 09:47:02,http://herzog.biz/chance.kuhn,0.0000000000,174.199.4.142,"{""location"": ""BD"", ""is_mobile"": true}" 402,1,42,2016-12-22 11:40:10,http://haley.info/ona_zulauf,1.0000000000,140.249.60.242,"{""location"": ""NG"", ""is_mobile"": false}" 403,1,42,2017-05-12 05:36:59,http://tremblay.info/jolie.sporer,0.0000000000,93.63.207.120,"{""location"": ""TJ"", ""is_mobile"": true}" 404,1,42,2017-04-23 12:19:00,http://gaylord.biz/jamison.turner,1.0000000000,162.206.47.14,"{""location"": ""BA"", ""is_mobile"": false}" 405,1,42,2017-05-29 08:09:35,http://howell.biz/aniyah,0.0000000000,139.122.54.11,"{""location"": ""CI"", ""is_mobile"": false}" 406,1,42,2017-03-29 20:00:50,http://labadie.biz/alex.weinat,1.0000000000,237.50.15.127,"{""location"": ""NO"", ""is_mobile"": true}" 407,1,42,2017-06-04 14:18:32,http://gibsondach.net/bell.pouros,1.0000000000,243.54.10.53,"{""location"": ""HN"", ""is_mobile"": true}" 408,1,42,2017-01-14 20:42:45,http://pfefferpowlowski.org/allie,1.0000000000,35.137.90.72,"{""location"": ""GW"", ""is_mobile"": true}" 409,1,42,2017-03-06 08:19:45,http://graham.com/garry_parisian,1.0000000000,19.121.56.122,"{""location"": ""NP"", ""is_mobile"": true}" 410,1,42,2017-02-07 07:08:46,http://bartoletti.net/joel_sporer,1.0000000000,75.197.230.219,"{""location"": ""ZA"", ""is_mobile"": false}" 411,1,42,2017-04-06 07:35:35,http://eichmann.io/demond_powlowski,0.0000000000,9.133.138.50,"{""location"": ""BI"", ""is_mobile"": false}" 412,1,42,2017-04-09 17:31:43,http://sawayn.org/mattie,0.0000000000,164.153.178.188,"{""location"": ""VU"", ""is_mobile"": false}" 413,1,42,2017-03-18 16:25:08,http://marvinbeahan.io/torey,0.0000000000,40.245.155.197,"{""location"": ""IT"", ""is_mobile"": false}" 414,1,42,2017-05-27 04:04:50,http://andersonyost.biz/felicia_botsford,1.0000000000,146.13.186.192,"{""location"": ""LS"", ""is_mobile"": false}" 415,1,42,2017-01-07 10:32:27,http://kuphalswaniawski.org/jamar_nicolas,1.0000000000,202.204.68.250,"{""location"": ""PY"", ""is_mobile"": true}" 416,1,42,2017-05-20 01:35:14,http://lakinwuckert.co/oliver,1.0000000000,126.140.153.86,"{""location"": ""AW"", ""is_mobile"": true}" 417,1,42,2017-03-19 09:28:44,http://champlinhand.org/jermey_mraz,0.0000000000,143.240.218.162,"{""location"": ""JO"", ""is_mobile"": false}" 418,1,42,2017-02-27 18:58:01,http://brekke.co/keyon,0.0000000000,126.140.153.86,"{""location"": ""CO"", ""is_mobile"": true}" 419,1,42,2017-03-25 08:15:07,http://hansen.biz/antonio,1.0000000000,124.44.208.166,"{""location"": ""LB"", ""is_mobile"": true}" 420,1,43,2017-05-12 23:57:04,http://beahanwunsch.info/carmelo_auer,0.0000000000,150.189.10.12,"{""location"": ""CA"", ""is_mobile"": false}" 421,1,43,2017-01-07 18:20:46,http://emmerichhaley.org/velda_jerde,0.0000000000,151.189.159.101,"{""location"": ""TH"", ""is_mobile"": true}" 422,1,43,2017-04-23 05:27:57,http://yost.biz/germaine.hagenes,0.0000000000,217.13.145.150,"{""location"": ""UM"", ""is_mobile"": true}" 423,1,43,2017-03-03 06:27:12,http://bartoletti.com/colton.cain,0.0000000000,20.47.164.202,"{""location"": ""KG"", ""is_mobile"": false}" 424,1,43,2017-01-22 02:22:52,http://conroy.co/caleigh_swift,0.0000000000,151.189.159.101,"{""location"": ""KI"", ""is_mobile"": true}" 425,1,43,2017-01-25 05:36:11,http://gerlach.biz/ryder,0.0000000000,171.239.212.6,"{""location"": ""RS"", ""is_mobile"": false}" 426,1,43,2017-03-10 20:19:27,http://bahringer.name/alize_abshire,0.0000000000,151.189.159.101,"{""location"": ""AI"", ""is_mobile"": false}" 427,1,43,2017-04-24 23:10:11,http://mrazledner.info/curt,0.0000000000,5.121.151.172,"{""location"": ""SB"", ""is_mobile"": true}" 428,1,43,2016-12-22 16:49:12,http://reichel.org/stone,0.0000000000,143.3.169.189,"{""location"": ""TO"", ""is_mobile"": false}" 429,1,43,2017-02-06 14:23:49,http://considinejohnston.name/chanel_kihn,0.0000000000,231.210.250.32,"{""location"": ""AT"", ""is_mobile"": true}" 430,1,43,2017-03-18 02:56:32,http://okon.net/shany,0.0000000000,22.12.155.110,"{""location"": ""CI"", ""is_mobile"": true}" 431,1,43,2017-06-02 17:18:40,http://bergepredovic.name/marion,0.0000000000,5.121.151.172,"{""location"": ""NE"", ""is_mobile"": false}" 432,1,43,2017-03-28 01:28:42,http://lowe.io/alexie.kozey,0.0000000000,252.104.87.25,"{""location"": ""VC"", ""is_mobile"": false}" 433,1,43,2017-03-23 07:35:31,http://schusterglover.info/linwood_breitenberg,0.0000000000,217.13.145.150,"{""location"": ""TJ"", ""is_mobile"": false}" 434,1,44,2017-05-07 02:00:15,http://collins.org/darwin.dicki,0.0000000000,210.235.156.78,"{""location"": ""NZ"", ""is_mobile"": false}" 435,1,44,2017-01-25 14:35:24,http://doylestehr.io/sydni,1.0000000000,183.245.173.220,"{""location"": ""SS"", ""is_mobile"": false}" 436,1,44,2017-01-18 20:29:19,http://heathcote.biz/miracle_nitzsche,1.0000000000,72.166.120.241,"{""location"": ""PF"", ""is_mobile"": true}" 437,1,44,2017-03-13 21:16:50,http://gaylordwhite.co/mona,2.0000000000,7.59.32.185,"{""location"": ""FK"", ""is_mobile"": true}" 438,1,44,2017-03-01 19:01:08,http://runolfonweinat.com/dena,2.0000000000,122.119.37.198,"{""location"": ""GS"", ""is_mobile"": false}" 439,1,44,2017-03-17 16:52:15,http://gerhold.org/felipa_pagac,0.0000000000,129.117.3.120,"{""location"": ""SN"", ""is_mobile"": true}" 440,1,44,2017-01-18 06:31:21,http://walker.net/braeden,1.0000000000,234.40.19.160,"{""location"": ""MN"", ""is_mobile"": true}" 441,1,44,2017-04-22 22:22:42,http://ondricka.io/kyra,2.0000000000,21.26.53.45,"{""location"": ""PN"", ""is_mobile"": false}" 442,1,45,2017-03-02 14:23:37,http://hayesdoyle.info/estefania.mcglynn,1.0000000000,15.89.103.157,"{""location"": ""TZ"", ""is_mobile"": false}" 443,1,45,2017-03-05 04:37:42,http://dubuque.io/crystal,0.0000000000,152.107.172.120,"{""location"": ""BM"", ""is_mobile"": true}" 444,1,45,2017-05-13 23:17:15,http://herman.name/leopold.weimann,0.0000000000,80.96.163.145,"{""location"": ""KR"", ""is_mobile"": false}" 445,1,45,2017-04-09 19:24:31,http://greentreutel.info/lavada,2.0000000000,72.155.240.217,"{""location"": ""GI"", ""is_mobile"": false}" 446,1,45,2017-01-14 21:09:45,http://schaden.biz/laura_keebler,0.0000000000,39.23.44.142,"{""location"": ""TH"", ""is_mobile"": true}" 447,1,45,2017-05-28 09:14:34,http://hoppe.org/blanche_wuckert,0.0000000000,220.205.192.6,"{""location"": ""CX"", ""is_mobile"": true}" 448,1,45,2017-04-24 19:58:01,http://hills.io/vince.labadie,0.0000000000,111.192.110.72,"{""location"": ""FO"", ""is_mobile"": true}" 449,1,45,2017-02-02 06:04:29,http://wiegand.name/romaine,0.0000000000,39.23.44.142,"{""location"": ""DK"", ""is_mobile"": true}" 450,1,45,2017-05-23 00:47:02,http://schimmel.org/sabina_donnelly,1.0000000000,28.251.174.219,"{""location"": ""DZ"", ""is_mobile"": true}" 451,1,45,2017-06-05 21:23:38,http://cremin.io/jaylon.cartwright,0.0000000000,50.242.196.128,"{""location"": ""UM"", ""is_mobile"": false}" 452,1,45,2017-06-12 13:00:32,http://beatty.co/murray_feeney,3.0000000000,73.12.79.101,"{""location"": ""GQ"", ""is_mobile"": false}" 453,1,45,2017-05-15 11:05:44,http://baileynader.co/earl,3.0000000000,134.65.51.101,"{""location"": ""UY"", ""is_mobile"": false}" 454,1,45,2016-12-14 07:54:32,http://shanahan.info/mabelle,1.0000000000,164.241.75.15,"{""location"": ""CV"", ""is_mobile"": false}" 455,1,45,2017-05-22 22:50:46,http://brekkedietrich.net/isac.macejkovic,1.0000000000,98.156.168.195,"{""location"": ""GF"", ""is_mobile"": false}" 456,1,45,2017-04-06 05:28:45,http://bodeupton.net/hillard.konopelski,0.0000000000,72.155.240.217,"{""location"": ""IE"", ""is_mobile"": false}" 457,1,46,2017-03-28 09:42:54,http://stammgrady.net/xavier.ratke,0.0000000000,22.247.15.145,"{""location"": ""BQ"", ""is_mobile"": false}" 458,1,46,2017-05-10 00:46:12,http://hagenesroob.com/arno,0.0000000000,242.36.59.99,"{""location"": ""MK"", ""is_mobile"": true}" 459,1,46,2017-05-30 17:24:58,http://keebler.io/efren.beatty,0.0000000000,101.45.9.175,"{""location"": ""VC"", ""is_mobile"": false}" 460,1,46,2017-06-11 12:23:46,http://kuphal.com/annamae,0.0000000000,47.79.126.156,"{""location"": ""KR"", ""is_mobile"": true}" 461,1,46,2017-06-06 09:13:39,http://vonsauer.net/sim_crist,0.0000000000,244.105.239.47,"{""location"": ""EE"", ""is_mobile"": false}" 462,1,46,2017-04-24 16:50:14,http://ryan.info/aleia,0.0000000000,254.209.239.47,"{""location"": ""AT"", ""is_mobile"": false}" 463,1,46,2017-01-11 00:13:08,http://graham.name/camille.langworth,0.0000000000,245.143.218.131,"{""location"": ""TN"", ""is_mobile"": false}" 464,1,46,2017-05-29 10:45:53,http://lockman.co/arne,0.0000000000,235.49.207.188,"{""location"": ""GP"", ""is_mobile"": true}" 465,1,46,2017-02-21 01:32:50,http://baumbach.biz/wendell_jakubowski,0.0000000000,110.56.99.166,"{""location"": ""SH"", ""is_mobile"": true}" 466,1,46,2017-02-02 02:17:18,http://runolfon.info/melya,0.0000000000,45.89.180.220,"{""location"": ""MG"", ""is_mobile"": false}" 467,1,46,2017-06-11 11:06:22,http://sawayn.biz/javier.moriette,0.0000000000,245.143.218.131,"{""location"": ""NZ"", ""is_mobile"": false}" 468,1,46,2017-01-27 12:01:49,http://kohlerstiedemann.co/francis.kuphal,0.0000000000,22.131.41.39,"{""location"": ""GE"", ""is_mobile"": true}" 469,1,46,2017-06-05 22:56:18,http://larkindamore.biz/keegan.medhurst,0.0000000000,245.143.218.131,"{""location"": ""BB"", ""is_mobile"": false}" 470,1,46,2017-02-18 04:45:00,http://hauck.net/milan,0.0000000000,175.235.76.151,"{""location"": ""ER"", ""is_mobile"": true}" 471,1,46,2017-04-29 17:36:11,http://goyetteokeefe.io/lyric,0.0000000000,68.187.177.175,"{""location"": ""SE"", ""is_mobile"": false}" 472,1,46,2017-04-20 12:24:29,http://morar.name/lorna,0.0000000000,216.192.109.140,"{""location"": ""PM"", ""is_mobile"": true}" 473,1,46,2017-01-23 06:08:36,http://cummerataratke.org/alva.schulist,0.0000000000,28.225.210.94,"{""location"": ""VU"", ""is_mobile"": true}" 474,1,46,2017-05-25 13:03:07,http://witting.net/emile,0.0000000000,72.233.9.17,"{""location"": ""AO"", ""is_mobile"": false}" 475,1,46,2017-05-28 15:12:46,http://jacobsondietrich.org/marianna_ohara,0.0000000000,41.140.108.64,"{""location"": ""WS"", ""is_mobile"": false}" 476,1,46,2017-02-08 09:11:02,http://cain.io/bette,0.0000000000,242.36.59.99,"{""location"": ""JP"", ""is_mobile"": false}" 477,1,47,2017-03-04 14:23:15,http://marvinmueller.name/daphne.hills,0.0000000000,71.45.221.159,"{""location"": ""LV"", ""is_mobile"": true}" 478,1,48,2017-05-02 23:51:05,http://stroman.info/ahmad,1.0000000000,108.20.202.152,"{""location"": ""EH"", ""is_mobile"": false}" 479,1,48,2017-03-11 12:10:37,http://auer.biz/estrella_olson,1.0000000000,44.148.211.113,"{""location"": ""JP"", ""is_mobile"": false}" 480,1,48,2017-04-29 13:01:31,http://turner.co/kenna,1.0000000000,228.141.57.79,"{""location"": ""BY"", ""is_mobile"": true}" 481,1,48,2017-05-26 21:13:29,http://macgyver.io/terrill_dickinson,0.0000000000,178.146.96.253,"{""location"": ""PT"", ""is_mobile"": false}" 482,1,48,2017-02-05 00:41:53,http://krajcikschinner.biz/ima,0.0000000000,172.74.244.32,"{""location"": ""JO"", ""is_mobile"": true}" 483,1,48,2017-03-07 08:34:51,http://deckow.info/yolanda_jacobson,0.0000000000,172.74.244.32,"{""location"": ""CD"", ""is_mobile"": false}" 484,1,48,2017-04-07 09:49:16,http://schiller.co/horace.wolf,0.0000000000,189.26.134.53,"{""location"": ""TF"", ""is_mobile"": true}" 485,1,48,2017-06-08 01:46:56,http://hermiston.net/toy,1.0000000000,148.13.88.206,"{""location"": ""SG"", ""is_mobile"": true}" 486,1,48,2016-12-17 05:56:27,http://little.biz/stone,1.0000000000,228.141.57.79,"{""location"": ""MG"", ""is_mobile"": true}" 487,1,48,2017-04-26 13:07:43,http://heller.io/amiya.wunsch,0.0000000000,195.142.199.210,"{""location"": ""AW"", ""is_mobile"": false}" 488,1,48,2017-01-08 10:56:01,http://leuschke.com/tamara,0.0000000000,133.39.231.42,"{""location"": ""ET"", ""is_mobile"": false}" 489,1,48,2017-03-21 18:54:08,http://hammes.info/rosalia,1.0000000000,2.188.128.203,"{""location"": ""CO"", ""is_mobile"": false}" 490,1,48,2017-05-14 15:31:10,http://danieltrantow.name/raphael,1.0000000000,44.148.211.113,"{""location"": ""IM"", ""is_mobile"": false}" 491,1,48,2017-04-01 17:33:39,http://parkerbuckridge.biz/perry.volkman,0.0000000000,133.39.231.42,"{""location"": ""SH"", ""is_mobile"": false}" 492,1,49,2017-02-06 19:36:33,http://schneiderrunte.info/rahul,1.0000000000,196.102.251.120,"{""location"": ""CF"", ""is_mobile"": false}" 493,1,49,2016-12-17 03:04:30,http://harris.name/billy,1.0000000000,84.27.149.171,"{""location"": ""TT"", ""is_mobile"": false}" 494,1,49,2017-03-08 18:33:11,http://powlowskigislason.net/sandy.christiansen,1.0000000000,79.51.94.198,"{""location"": ""GW"", ""is_mobile"": true}" 495,1,49,2017-02-04 09:21:10,http://adamsrunolfon.info/jamaal,1.0000000000,105.120.52.68,"{""location"": ""MS"", ""is_mobile"": true}" 496,1,49,2016-12-14 08:12:14,http://lowe.co/josh.schoen,1.0000000000,184.24.238.15,"{""location"": ""GH"", ""is_mobile"": false}" 497,1,49,2017-02-18 06:03:43,http://damore.co/ubaldo_smitham,2.0000000000,131.110.43.20,"{""location"": ""YT"", ""is_mobile"": true}" 498,1,49,2017-04-26 20:52:19,http://dickinsonstiedemann.info/carolyn.ullrich,0.0000000000,217.173.252.118,"{""location"": ""CC"", ""is_mobile"": false}" 499,1,49,2017-03-11 12:51:50,http://fadel.co/bernadette,0.0000000000,217.173.252.118,"{""location"": ""PM"", ""is_mobile"": true}" 500,1,49,2017-05-06 16:03:47,http://stehrwill.net/mervin.kuhn,1.0000000000,2.82.162.233,"{""location"": ""CC"", ""is_mobile"": true}" 501,1,49,2017-03-21 20:56:10,http://gusikowskischmitt.info/candace_schmitt,0.0000000000,103.105.163.101,"{""location"": ""FR"", ""is_mobile"": false}" 502,1,50,2017-01-10 16:09:15,http://gaylordwilliamson.info/emily,3.0000000000,133.216.251.27,"{""location"": ""YT"", ""is_mobile"": true}" 503,1,50,2017-02-21 01:47:11,http://feestfeil.biz/easter,3.0000000000,121.237.90.228,"{""location"": ""BN"", ""is_mobile"": false}" 504,1,50,2017-05-26 11:28:50,http://baumbachstiedemann.info/michele,2.0000000000,159.23.141.243,"{""location"": ""AS"", ""is_mobile"": true}" 505,1,50,2017-01-22 16:54:51,http://rauhyatt.net/ransom.wilderman,2.0000000000,203.251.254.161,"{""location"": ""SC"", ""is_mobile"": false}" 506,1,50,2016-12-22 13:10:55,http://morarjenkins.io/dereck.brekke,1.0000000000,172.69.195.245,"{""location"": ""LR"", ""is_mobile"": true}" 507,1,51,2017-02-07 07:39:18,http://kovacek.io/meda_gleason,3.0000000000,159.78.11.64,"{""location"": ""TD"", ""is_mobile"": false}" 508,1,51,2017-04-22 22:40:24,http://borerschroeder.net/rosetta_vandervort,2.0000000000,244.231.121.250,"{""location"": ""PM"", ""is_mobile"": true}" 2794,5,263,2017-01-11 11:09:43,http://marks.biz/loma,,6.155.218.162,"{""location"": ""GH"", ""is_mobile"": false}" 2795,5,263,2017-01-03 15:42:55,http://stoltenbergschowalter.name/clovis,,206.129.103.236,"{""location"": ""BQ"", ""is_mobile"": true}" 2796,5,263,2017-03-04 22:05:17,http://bruenrobel.co/america,,191.162.136.31,"{""location"": ""SS"", ""is_mobile"": false}" 2797,5,263,2017-02-20 03:31:04,http://schaefer.net/ardith_erdman,,6.155.218.162,"{""location"": ""KN"", ""is_mobile"": false}" 2798,5,263,2017-03-17 07:51:13,http://hermann.info/dorcas,,72.126.67.66,"{""location"": ""PH"", ""is_mobile"": true}" 2799,5,263,2017-01-30 17:45:00,http://swaniawskitremblay.biz/derrick.schuster,,167.149.114.171,"{""location"": ""CL"", ""is_mobile"": true}" 2800,5,263,2017-03-31 04:13:26,http://hilpert.biz/gus_beer,,87.227.33.129,"{""location"": ""AX"", ""is_mobile"": false}" 2801,5,264,2017-05-28 00:53:26,http://brown.net/marge.roob,,14.178.243.6,"{""location"": ""LC"", ""is_mobile"": false}" 2802,5,264,2017-04-05 15:12:21,http://mclaughlinko.biz/marc_oberbrunner,,23.114.236.213,"{""location"": ""RO"", ""is_mobile"": true}" 2803,5,264,2017-05-30 04:36:29,http://franecki.com/alejandrin_carroll,,40.185.6.234,"{""location"": ""RS"", ""is_mobile"": false}" 2804,5,264,2017-06-13 05:41:43,http://spinka.co/linnie.okuneva,,70.28.161.218,"{""location"": ""MF"", ""is_mobile"": false}" 2805,5,264,2017-02-11 04:03:09,http://mclaughlin.com/wilber,,149.76.60.11,"{""location"": ""BE"", ""is_mobile"": true}" 2806,5,264,2017-01-12 20:53:38,http://wisokyparker.org/adriel_king,,224.142.76.83,"{""location"": ""MH"", ""is_mobile"": true}" 2807,5,264,2017-03-07 17:22:53,http://kshlerinkutch.org/myrtle,,165.251.22.166,"{""location"": ""UG"", ""is_mobile"": true}" 2808,5,264,2017-01-11 08:51:03,http://medhurstkuphal.biz/june,,8.212.9.27,"{""location"": ""QA"", ""is_mobile"": true}" 2809,5,264,2017-01-05 12:28:04,http://handskiles.co/ruel.bayer,,70.86.169.17,"{""location"": ""IM"", ""is_mobile"": true}" 2810,5,264,2017-02-22 22:38:30,http://stromanhermann.co/derrick,,77.63.173.180,"{""location"": ""AZ"", ""is_mobile"": true}" 2811,5,264,2017-04-16 10:02:41,http://klockogreenholt.name/kavon,,242.35.119.54,"{""location"": ""IN"", ""is_mobile"": true}" 2812,5,264,2017-06-13 14:27:21,http://gutmannflatley.org/sam_fay,,60.173.24.178,"{""location"": ""MR"", ""is_mobile"": true}" 2813,5,264,2017-05-31 17:20:49,http://lowedaniel.org/damon.schuster,,165.251.22.166,"{""location"": ""MN"", ""is_mobile"": true}" 2814,5,264,2017-06-03 19:23:56,http://goldner.info/madalyn_collins,,165.251.22.166,"{""location"": ""ST"", ""is_mobile"": true}" 2815,5,264,2017-05-31 14:02:37,http://gutkowskirippin.com/daniella,,60.173.24.178,"{""location"": ""BI"", ""is_mobile"": true}" 2816,5,265,2017-04-07 18:18:25,http://mccullough.biz/santa.wunsch,,156.178.99.112,"{""location"": ""CR"", ""is_mobile"": false}" 2817,5,265,2017-06-03 23:13:58,http://oconnerpouros.org/destiney.ward,,88.53.3.32,"{""location"": ""PG"", ""is_mobile"": true}" 2818,5,265,2017-06-05 16:54:16,http://kshlerindaugherty.info/elinore,,33.71.216.109,"{""location"": ""SD"", ""is_mobile"": false}" 2819,5,265,2017-05-26 11:16:21,http://emard.name/weston,,156.178.99.112,"{""location"": ""CZ"", ""is_mobile"": false}" 2820,5,265,2017-01-06 15:42:41,http://okeefe.io/magdalena,,241.195.93.72,"{""location"": ""MM"", ""is_mobile"": false}" 2821,5,265,2017-03-29 09:55:16,http://kochdach.io/rocio,,120.109.185.19,"{""location"": ""TL"", ""is_mobile"": true}" 2822,5,265,2017-04-01 13:27:46,http://rowe.biz/mayra,,71.218.16.77,"{""location"": ""VU"", ""is_mobile"": false}" 2823,5,265,2017-06-03 13:14:56,http://maggiodaugherty.net/darren_leffler,,33.71.216.109,"{""location"": ""SE"", ""is_mobile"": false}" 2824,5,265,2017-04-22 23:49:18,http://pouroschristiansen.com/aiyana,,71.218.16.77,"{""location"": ""HR"", ""is_mobile"": true}" 2825,5,265,2017-05-17 08:47:01,http://schuppe.com/torrance.cartwright,,89.146.186.107,"{""location"": ""LA"", ""is_mobile"": true}" 2826,5,265,2017-01-22 04:09:19,http://gradywisoky.com/alfreda,,152.48.109.120,"{""location"": ""NE"", ""is_mobile"": true}" 2827,5,265,2017-02-25 02:57:57,http://mannwintheiser.org/erna.bernhard,,189.87.216.55,"{""location"": ""AT"", ""is_mobile"": false}" 2828,5,265,2017-01-11 03:04:17,http://fritsch.org/milan.wilkinson,,88.53.3.32,"{""location"": ""CW"", ""is_mobile"": true}" 2829,5,265,2017-04-23 08:03:15,http://braunkrajcik.name/gail.schmidt,,126.246.183.11,"{""location"": ""MW"", ""is_mobile"": false}" 2830,5,265,2017-01-07 18:07:16,http://hamill.com/fanny,,168.59.186.35,"{""location"": ""GM"", ""is_mobile"": true}" 2831,5,265,2017-01-12 17:49:21,http://rathweimann.io/dora.wiegand,,78.33.99.52,"{""location"": ""SX"", ""is_mobile"": false}" 2832,5,265,2017-05-10 17:51:42,http://green.net/brown_bernhard,,93.83.249.37,"{""location"": ""CL"", ""is_mobile"": false}" 2833,5,265,2017-03-07 10:32:20,http://champlin.name/hunter,,106.77.219.204,"{""location"": ""LV"", ""is_mobile"": true}" 2834,5,266,2017-01-31 14:31:16,http://becker.co/reyes.pouros,,71.41.213.81,"{""location"": ""TW"", ""is_mobile"": true}" 2835,5,266,2017-05-05 04:51:34,http://luettgensawayn.info/rollin,,145.27.239.118,"{""location"": ""NR"", ""is_mobile"": false}" 2836,5,266,2017-03-19 08:55:08,http://metzruecker.co/marjory.borer,,42.173.5.212,"{""location"": ""SK"", ""is_mobile"": true}" 2837,5,266,2017-05-22 13:47:47,http://cartwrightkiehn.co/cydney,,205.218.20.118,"{""location"": ""BO"", ""is_mobile"": false}" 2838,5,266,2016-12-22 19:55:57,http://littlewiegand.org/carolina,,245.249.188.101,"{""location"": ""GD"", ""is_mobile"": false}" 2839,5,266,2017-04-21 13:10:53,http://bartellmcglynn.co/virginie,,23.194.56.72,"{""location"": ""BO"", ""is_mobile"": false}" 2840,5,266,2017-05-17 10:32:31,http://erdmanvon.io/jeffrey.franecki,,248.38.79.91,"{""location"": ""ES"", ""is_mobile"": false}" 2841,5,266,2017-01-26 02:36:55,http://ruecker.info/deanna,,196.135.94.71,"{""location"": ""AS"", ""is_mobile"": true}" 2842,5,266,2017-03-06 09:55:19,http://dubuquewunsch.co/saige,,82.94.196.146,"{""location"": ""LS"", ""is_mobile"": false}" 2843,5,266,2016-12-27 22:49:31,http://botsford.info/brody,,102.168.210.52,"{""location"": ""MR"", ""is_mobile"": true}" 2844,5,266,2017-01-03 02:13:43,http://kshlerinberge.co/arlene.gulgowski,,145.130.57.251,"{""location"": ""UM"", ""is_mobile"": true}" 2845,5,266,2017-01-29 10:26:26,http://kuhnwalker.org/aubree.little,,82.91.128.159,"{""location"": ""TZ"", ""is_mobile"": true}" 2846,5,266,2017-03-24 19:22:09,http://macejkovic.name/bailee_runolfon,,46.147.21.116,"{""location"": ""GQ"", ""is_mobile"": true}" 2847,5,266,2016-12-19 06:01:22,http://stehr.co/myrtice_cole,,185.33.211.161,"{""location"": ""US"", ""is_mobile"": false}" 2848,5,266,2017-01-19 15:01:50,http://danielhegmann.com/kristina,,53.134.183.87,"{""location"": ""NL"", ""is_mobile"": true}" 2849,5,266,2017-01-31 11:21:58,http://cormier.info/lisa.walter,,191.153.239.203,"{""location"": ""SA"", ""is_mobile"": false}" 2850,5,266,2017-06-05 08:59:14,http://kutchswift.com/nash.batz,,71.41.213.81,"{""location"": ""TM"", ""is_mobile"": true}" 2851,5,266,2017-04-21 08:20:17,http://nitzsche.name/adan.funk,,248.38.79.91,"{""location"": ""OM"", ""is_mobile"": false}" 2852,5,266,2017-01-17 11:19:02,http://cormier.name/lisa.schimmel,,196.120.226.78,"{""location"": ""PT"", ""is_mobile"": true}" 2853,5,266,2017-05-11 13:15:43,http://fisher.io/cristobal,,46.147.21.116,"{""location"": ""UM"", ""is_mobile"": false}" 2854,5,267,2017-03-11 12:57:03,http://murazikstrosin.biz/diego,,154.32.27.198,"{""location"": ""GB"", ""is_mobile"": true}" 2855,5,267,2017-04-24 02:55:26,http://legros.info/ephraim,,74.171.125.104,"{""location"": ""ST"", ""is_mobile"": true}" 2856,5,267,2017-04-12 04:03:33,http://runolfsdottir.net/alexandria,,124.10.213.9,"{""location"": ""GB"", ""is_mobile"": true}" 2857,5,267,2017-02-15 23:19:14,http://watsica.info/godfrey,,76.239.27.202,"{""location"": ""AM"", ""is_mobile"": false}" 2858,5,267,2017-02-13 07:04:47,http://abshire.com/verna,,63.221.218.5,"{""location"": ""CZ"", ""is_mobile"": true}" 2859,5,267,2017-01-11 04:03:36,http://millchmitt.com/garrett_ward,,124.10.213.9,"{""location"": ""KY"", ""is_mobile"": false}" 2860,5,267,2017-02-16 07:28:34,http://lueilwitzreichert.com/christelle_fahey,,102.186.233.219,"{""location"": ""BR"", ""is_mobile"": false}" 2861,5,267,2017-05-07 01:07:48,http://lehner.net/deonte_oreilly,,21.188.188.183,"{""location"": ""IT"", ""is_mobile"": true}" 2862,5,267,2017-04-16 13:28:08,http://koelpin.info/bridget,,3.13.50.113,"{""location"": ""GN"", ""is_mobile"": false}" 2863,5,267,2017-03-05 08:00:04,http://grahamleuschke.biz/river_ruel,,159.69.221.135,"{""location"": ""MY"", ""is_mobile"": false}" 2864,5,267,2017-03-15 17:38:01,http://hettinger.io/henriette.stamm,,146.109.146.104,"{""location"": ""ZW"", ""is_mobile"": false}" 2865,5,267,2017-06-03 15:31:35,http://quitzon.biz/giovanny_rosenbaum,,63.221.218.5,"{""location"": ""NO"", ""is_mobile"": false}" 2866,5,267,2016-12-17 16:08:29,http://turner.io/meggie.keeling,,101.186.77.9,"{""location"": ""MY"", ""is_mobile"": true}" 2867,5,267,2016-12-24 10:32:38,http://bailey.info/merl,,241.196.40.160,"{""location"": ""WF"", ""is_mobile"": true}" 2868,5,267,2017-04-10 19:56:12,http://kaulke.info/orie_cain,,63.221.218.5,"{""location"": ""AI"", ""is_mobile"": false}" 2869,5,267,2017-01-21 04:55:22,http://gradyabshire.org/ford,,173.229.124.123,"{""location"": ""PS"", ""is_mobile"": false}" 2870,5,267,2017-02-06 21:20:24,http://stoltenberg.name/elmer.yundt,,131.57.97.68,"{""location"": ""WS"", ""is_mobile"": false}" 2871,5,267,2017-04-11 19:34:39,http://jacobi.io/mona,,43.213.198.8,"{""location"": ""DK"", ""is_mobile"": true}" 2872,5,268,2017-04-29 05:12:18,http://walsh.com/rebeka_kunde,,163.107.166.14,"{""location"": ""NZ"", ""is_mobile"": true}" 2873,5,268,2017-03-26 17:08:04,http://eichmann.net/frieda,,87.70.168.129,"{""location"": ""KZ"", ""is_mobile"": true}" 2874,5,268,2017-02-12 06:47:12,http://adamsheaney.com/jeffrey_romaguera,,127.142.118.201,"{""location"": ""RW"", ""is_mobile"": false}" 2875,5,268,2017-01-24 16:02:19,http://cruickshankschroeder.com/flo_raynor,,15.112.231.73,"{""location"": ""AI"", ""is_mobile"": false}" 2876,5,268,2017-02-03 10:51:46,http://fisherbotsford.biz/woodrow.mcglynn,,87.70.168.129,"{""location"": ""PR"", ""is_mobile"": false}" 2877,5,268,2017-04-04 21:19:53,http://kuvalis.name/heber,,69.42.150.203,"{""location"": ""SI"", ""is_mobile"": false}" 2878,5,268,2017-05-15 04:44:59,http://wittingschmidt.name/sam,,224.74.10.112,"{""location"": ""SD"", ""is_mobile"": false}" 2879,5,268,2017-03-26 18:01:47,http://stehr.biz/shanie_harris,,116.154.218.71,"{""location"": ""BV"", ""is_mobile"": true}" 2880,5,268,2017-04-12 20:07:43,http://gutmann.name/stefan.goodwin,,166.172.58.116,"{""location"": ""ER"", ""is_mobile"": true}" 2881,5,268,2017-03-28 16:38:25,http://romaguera.net/leila,,15.209.191.43,"{""location"": ""KP"", ""is_mobile"": true}" 2882,5,268,2017-01-20 09:20:09,http://ohara.co/dion_lebsack,,238.78.93.113,"{""location"": ""ID"", ""is_mobile"": true}" 2883,5,268,2017-01-22 03:20:47,http://feest.net/martin,,238.78.93.113,"{""location"": ""FR"", ""is_mobile"": false}" 2884,5,268,2017-04-11 06:18:38,http://grantnolan.co/priscilla.quitzon,,229.212.213.55,"{""location"": ""FK"", ""is_mobile"": false}" 2885,5,268,2017-03-10 14:36:13,http://bahringerhaley.io/alvis_gerhold,,127.142.118.201,"{""location"": ""IO"", ""is_mobile"": false}" 2886,5,268,2017-03-21 13:56:30,http://reichelmitchell.biz/gage.abshire,,24.140.127.63,"{""location"": ""SS"", ""is_mobile"": true}" 2887,5,268,2017-03-29 17:01:27,http://schoenbarton.name/lizeth,,83.165.9.127,"{""location"": ""LK"", ""is_mobile"": true}" 2888,5,268,2017-03-17 18:16:08,http://shields.info/kaci_cormier,,137.12.21.86,"{""location"": ""VI"", ""is_mobile"": true}" 2889,5,268,2017-03-06 20:28:32,http://fadeldaniel.io/teie,,24.140.127.63,"{""location"": ""FM"", ""is_mobile"": false}" 2890,5,269,2017-04-22 16:18:07,http://creminwilkinson.info/branson,,235.226.98.225,"{""location"": ""SR"", ""is_mobile"": false}" 2891,5,269,2017-04-20 07:41:26,http://hyatt.info/ivy.mohr,,99.35.226.59,"{""location"": ""GB"", ""is_mobile"": false}" 2892,5,269,2017-01-12 03:19:16,http://anderson.info/berenice_oconnell,,215.198.35.11,"{""location"": ""GW"", ""is_mobile"": false}" 2893,5,269,2017-02-27 06:22:48,http://williamson.com/beryl.schulist,,192.238.199.150,"{""location"": ""AM"", ""is_mobile"": true}" 2894,5,269,2017-01-01 10:41:33,http://beer.name/savion,,101.243.115.11,"{""location"": ""TT"", ""is_mobile"": false}" 2895,5,269,2017-02-23 21:15:34,http://deckow.org/thaddeus,,29.253.101.128,"{""location"": ""EG"", ""is_mobile"": true}" 2896,5,269,2017-04-03 22:13:55,http://beier.io/agnes,,25.143.50.128,"{""location"": ""SK"", ""is_mobile"": false}" 2897,5,269,2017-05-13 20:02:14,http://walter.biz/gust.kris,,29.253.101.128,"{""location"": ""LB"", ""is_mobile"": true}" 2898,5,269,2017-01-27 03:54:10,http://nienowstark.org/malcolm,,220.181.231.72,"{""location"": ""TZ"", ""is_mobile"": true}" 2899,5,269,2017-05-15 13:39:54,http://windler.io/junior,,193.198.155.100,"{""location"": ""PE"", ""is_mobile"": true}" 2900,5,269,2017-01-13 18:24:59,http://crona.biz/carson,,50.67.30.126,"{""location"": ""GG"", ""is_mobile"": true}" 2901,5,269,2017-02-27 06:33:09,http://gradyhirthe.co/enid.witting,,71.81.205.96,"{""location"": ""NG"", ""is_mobile"": false}" 2902,5,269,2017-05-05 14:05:18,http://rath.net/terrence,,235.226.98.225,"{""location"": ""UZ"", ""is_mobile"": false}" 2903,5,269,2017-04-25 19:51:03,http://mcglynn.com/brenna,,238.57.203.153,"{""location"": ""AF"", ""is_mobile"": false}" 2904,5,269,2017-02-23 22:15:58,http://huel.com/roderick_collier,,38.152.50.205,"{""location"": ""UM"", ""is_mobile"": true}" 2905,5,269,2017-03-28 01:05:54,http://osinskinienow.org/dante_nitzsche,,72.196.133.76,"{""location"": ""MN"", ""is_mobile"": false}" 2906,5,269,2017-01-04 17:43:54,http://paucekwehner.org/khalil,,161.9.133.164,"{""location"": ""GL"", ""is_mobile"": false}" 2907,5,269,2017-01-29 04:48:28,http://corkeryzemlak.net/rhianna,,38.152.50.205,"{""location"": ""YT"", ""is_mobile"": false}" 2908,5,270,2017-05-07 00:13:10,http://beattylynch.co/jamison,,66.102.174.43,"{""location"": ""CG"", ""is_mobile"": false}" 2909,5,270,2017-01-28 10:52:50,http://sawaynokon.io/hal,,95.155.168.112,"{""location"": ""FJ"", ""is_mobile"": false}" 2910,5,270,2017-03-14 19:46:27,http://feestkuhlman.com/ruel.nader,,124.138.142.92,"{""location"": ""FK"", ""is_mobile"": false}" 2911,5,270,2017-04-05 20:23:36,http://carterschmeler.co/jennings.champlin,,164.141.144.152,"{""location"": ""CC"", ""is_mobile"": false}" 2912,5,270,2017-01-28 08:57:49,http://jast.name/anabelle_bartell,,71.167.240.38,"{""location"": ""WF"", ""is_mobile"": false}" 2913,5,270,2017-06-07 23:12:04,http://mayert.io/tomasa,,78.34.23.74,"{""location"": ""AS"", ""is_mobile"": false}" 2914,5,270,2017-03-19 00:29:50,http://littel.net/jamison.klocko,,95.155.168.112,"{""location"": ""YT"", ""is_mobile"": true}" 2915,5,270,2017-01-02 13:34:38,http://keelingwiza.co/mellie.prosacco,,99.86.245.182,"{""location"": ""MN"", ""is_mobile"": false}" 2916,5,270,2017-02-09 18:25:13,http://zboncak.info/deron,,19.37.177.140,"{""location"": ""DZ"", ""is_mobile"": false}" 2917,5,270,2017-06-05 19:06:21,http://ryanhayes.name/dangelo,,130.185.101.87,"{""location"": ""BZ"", ""is_mobile"": false}" 2918,5,270,2017-05-19 11:53:16,http://corwin.com/anya.hodkiewicz,,149.37.223.19,"{""location"": ""MV"", ""is_mobile"": false}" 2919,5,270,2017-04-19 02:49:56,http://kuhn.com/demario_parker,,124.138.142.92,"{""location"": ""MC"", ""is_mobile"": false}" 2920,5,270,2017-04-18 18:03:02,http://wehner.name/irving,,157.10.80.246,"{""location"": ""AU"", ""is_mobile"": false}" 2921,5,270,2017-04-12 09:06:32,http://gottliebeichmann.io/mackenzie.hudson,,68.139.213.70,"{""location"": ""KI"", ""is_mobile"": true}" 2922,5,271,2016-12-31 08:16:02,http://pfefferkutch.net/marisa,,68.247.177.215,"{""location"": ""GM"", ""is_mobile"": true}" 2923,5,271,2017-02-05 11:03:34,http://franecki.name/susana.walter,,32.38.198.224,"{""location"": ""BV"", ""is_mobile"": false}" 2924,5,271,2017-01-22 22:44:19,http://corkery.biz/gabriel,,139.93.208.204,"{""location"": ""NE"", ""is_mobile"": true}" 2925,5,271,2017-05-28 03:54:52,http://kihn.com/kaylie,,77.30.59.138,"{""location"": ""GT"", ""is_mobile"": true}" 2926,5,271,2017-02-06 09:02:16,http://corkery.net/elia,,21.247.62.123,"{""location"": ""TO"", ""is_mobile"": false}" 2927,5,271,2017-06-11 19:38:43,http://gleasonheidenreich.info/paula_ryan,,102.177.96.148,"{""location"": ""AX"", ""is_mobile"": true}" 2928,5,271,2017-01-06 04:55:50,http://gorczany.biz/maye.medhurst,,206.17.195.99,"{""location"": ""SD"", ""is_mobile"": false}" 2929,5,271,2017-05-17 15:01:39,http://cristwest.com/kirsten.pollich,,204.145.129.117,"{""location"": ""BN"", ""is_mobile"": true}" 2930,5,271,2017-01-03 09:20:21,http://watsica.co/andre.hodkiewicz,,93.41.73.124,"{""location"": ""KW"", ""is_mobile"": true}" 2931,5,271,2017-04-10 12:39:24,http://schoenstehr.co/gust.langworth,,93.41.73.124,"{""location"": ""SS"", ""is_mobile"": false}" 2932,5,271,2017-01-15 13:20:39,http://daniel.info/brent_schiller,,126.115.246.73,"{""location"": ""LI"", ""is_mobile"": false}" 2933,5,271,2017-02-09 17:31:11,http://ohara.co/augusta,,206.17.195.99,"{""location"": ""PK"", ""is_mobile"": true}" 2934,5,272,2017-05-30 23:55:41,http://schaeferwitting.com/jarrett_pouros,,95.52.75.143,"{""location"": ""CY"", ""is_mobile"": true}" 2935,5,272,2016-12-31 14:02:05,http://hudson.com/rosemarie,,118.168.130.80,"{""location"": ""EH"", ""is_mobile"": true}" 2936,5,272,2017-04-30 07:04:09,http://tromp.biz/theo.moore,,196.144.198.15,"{""location"": ""GN"", ""is_mobile"": false}" 2937,5,272,2017-02-24 21:33:59,http://marquardt.io/annette.mann,,188.155.44.250,"{""location"": ""WS"", ""is_mobile"": true}" 2938,5,272,2017-01-30 04:11:35,http://medhurstfeeney.org/shea_wolf,,14.112.244.125,"{""location"": ""SJ"", ""is_mobile"": true}" 2939,5,272,2017-02-12 09:04:19,http://hills.co/reinhold_ryan,,247.73.150.51,"{""location"": ""ST"", ""is_mobile"": false}" 2940,5,272,2017-06-11 22:56:44,http://joneshuels.biz/nigel,,215.51.15.37,"{""location"": ""GM"", ""is_mobile"": false}" 2941,5,272,2017-04-03 08:14:15,http://little.co/chad_ko,,247.73.150.51,"{""location"": ""BF"", ""is_mobile"": false}" 2942,5,272,2017-03-28 08:06:11,http://streichblock.biz/garry,,167.37.95.54,"{""location"": ""MM"", ""is_mobile"": false}" 2943,5,273,2017-03-26 00:34:54,http://grimes.io/roscoe.price,,202.137.232.251,"{""location"": ""NR"", ""is_mobile"": true}" 2944,5,273,2017-01-26 08:27:13,http://kertzmannmurphy.com/rocio.bauch,,38.19.69.145,"{""location"": ""NU"", ""is_mobile"": true}" 2945,5,273,2017-05-11 07:26:38,http://lockman.net/kennedy_bode,,234.211.126.100,"{""location"": ""SI"", ""is_mobile"": true}" 2946,5,273,2017-01-24 03:59:04,http://lowe.net/bennett,,130.145.175.184,"{""location"": ""IQ"", ""is_mobile"": false}" 2947,5,273,2016-12-17 16:54:43,http://schiller.net/alene.kulas,,125.129.41.4,"{""location"": ""MC"", ""is_mobile"": true}" 2948,5,273,2017-03-02 10:15:51,http://kohlerprice.info/cory,,101.95.137.101,"{""location"": ""BW"", ""is_mobile"": false}" 2949,5,273,2017-04-15 15:54:20,http://krisblock.biz/lily.mcclure,,128.49.219.148,"{""location"": ""GI"", ""is_mobile"": false}" 2950,5,273,2017-02-14 08:56:11,http://schmidt.io/giovanni.williamson,,202.137.232.251,"{""location"": ""SL"", ""is_mobile"": true}" 2951,5,273,2017-01-13 07:19:24,http://schowalterherman.name/katlyn.keeling,,128.49.219.148,"{""location"": ""AI"", ""is_mobile"": false}" 2952,5,273,2017-01-23 05:55:10,http://vonrueden.com/earlene,,130.145.175.184,"{""location"": ""AI"", ""is_mobile"": true}" 2953,5,273,2017-01-24 23:39:41,http://romagueraklein.org/emile.kuhic,,102.144.41.244,"{""location"": ""NG"", ""is_mobile"": true}" 2954,5,273,2016-12-21 14:24:57,http://larsonwaelchi.io/orville,,32.146.231.144,"{""location"": ""GI"", ""is_mobile"": false}" 2955,5,273,2017-03-10 17:48:22,http://moriette.com/mozelle_dooley,,101.95.137.101,"{""location"": ""AM"", ""is_mobile"": true}" 2956,5,273,2017-05-04 03:27:22,http://ohara.com/ollie,,130.145.175.184,"{""location"": ""NE"", ""is_mobile"": false}" 2957,5,273,2017-04-06 02:36:35,http://rohan.net/angie,,94.241.248.233,"{""location"": ""AF"", ""is_mobile"": true}" 2958,5,273,2017-05-05 09:26:30,http://renner.co/ronny.kuvalis,,13.23.119.23,"{""location"": ""CH"", ""is_mobile"": true}" 2959,5,273,2017-05-31 22:28:12,http://marvin.org/ashlynn,,222.243.219.127,"{""location"": ""TC"", ""is_mobile"": false}" 2960,5,274,2017-04-06 15:16:50,http://reillyjacobson.com/jayce,,235.139.211.201,"{""location"": ""BB"", ""is_mobile"": false}" 2961,5,274,2017-03-19 05:30:47,http://fritschcrona.name/ally,,249.161.9.23,"{""location"": ""IN"", ""is_mobile"": false}" 2962,5,274,2017-04-24 15:03:42,http://kirlin.info/payton,,171.163.199.18,"{""location"": ""MF"", ""is_mobile"": true}" 2963,5,274,2017-03-19 04:28:04,http://erdman.com/loren,,72.135.55.12,"{""location"": ""EG"", ""is_mobile"": false}" 2964,5,274,2017-01-08 19:53:57,http://quitzon.net/trycia,,137.232.143.73,"{""location"": ""BJ"", ""is_mobile"": true}" 2965,5,274,2017-01-09 22:51:17,http://schmeler.io/vicenta,,219.66.231.107,"{""location"": ""EE"", ""is_mobile"": true}" 2966,5,274,2017-03-05 05:25:27,http://dubuque.org/abbigail.mills,,72.135.55.12,"{""location"": ""NU"", ""is_mobile"": false}" 2967,5,274,2017-03-11 23:26:18,http://huel.com/axel,,95.56.222.222,"{""location"": ""BL"", ""is_mobile"": true}" 2968,5,274,2017-05-13 14:11:33,http://rosenbaummarquardt.net/yasmine,,105.56.178.152,"{""location"": ""GF"", ""is_mobile"": false}" 2969,5,274,2017-03-20 21:01:54,http://kunde.com/patricia_bins,,50.65.24.236,"{""location"": ""SI"", ""is_mobile"": false}" 2970,5,274,2016-12-30 05:57:26,http://herzoglindgren.biz/lawson,,137.232.143.73,"{""location"": ""WS"", ""is_mobile"": true}" 2971,5,274,2017-03-31 06:03:36,http://rempel.org/marietta_feest,,212.247.162.115,"{""location"": ""CA"", ""is_mobile"": false}" 2972,5,274,2017-05-31 07:47:26,http://strosin.org/gudrun_zulauf,,72.135.55.12,"{""location"": ""CG"", ""is_mobile"": false}" 2973,5,274,2017-04-06 17:20:48,http://lubowitz.biz/sallie.bartoletti,,219.66.231.107,"{""location"": ""CU"", ""is_mobile"": false}" 2974,5,274,2017-01-21 16:15:30,http://marquardt.info/joanie,,22.101.187.92,"{""location"": ""PA"", ""is_mobile"": true}" 2975,5,274,2017-01-18 17:25:01,http://kemmer.name/adolphus,,105.56.178.152,"{""location"": ""OM"", ""is_mobile"": false}" 2976,5,274,2017-01-01 22:09:06,http://turnerprice.org/jimmy_kautzer,,235.139.211.201,"{""location"": ""SS"", ""is_mobile"": false}" 2977,5,274,2017-01-18 11:23:44,http://kuvalisbraun.org/francisco,,26.110.97.213,"{""location"": ""YT"", ""is_mobile"": true}" 2978,5,274,2017-01-29 19:45:58,http://dietrich.org/meghan.welch,,159.251.72.204,"{""location"": ""GD"", ""is_mobile"": true}" 2979,5,274,2017-04-01 16:49:01,http://zieme.co/rupert,,50.65.24.236,"{""location"": ""RS"", ""is_mobile"": true}" 2980,5,275,2017-02-21 16:34:20,http://toy.io/ima,,211.42.187.89,"{""location"": ""AX"", ""is_mobile"": true}" 2981,5,275,2017-05-14 00:35:42,http://beatty.com/ardith_keler,,38.62.136.78,"{""location"": ""CN"", ""is_mobile"": true}" 2982,5,275,2017-05-11 12:44:57,http://weber.info/caandra.monahan,,38.62.136.78,"{""location"": ""CL"", ""is_mobile"": true}" 2983,5,275,2017-02-27 22:37:14,http://olson.io/terrell.dickens,,195.64.10.59,"{""location"": ""CK"", ""is_mobile"": true}" 2984,5,275,2017-01-07 01:25:11,http://parisianromaguera.info/erich,,131.77.27.240,"{""location"": ""HM"", ""is_mobile"": false}" 2985,5,275,2017-04-01 15:19:35,http://lubowitz.info/rae,,150.136.21.39,"{""location"": ""MQ"", ""is_mobile"": false}" 2986,5,275,2017-04-11 05:04:33,http://wisoky.org/salvador.leuschke,,134.207.8.76,"{""location"": ""AU"", ""is_mobile"": false}" 2987,5,275,2017-01-25 00:57:17,http://corkery.org/max,,195.64.10.59,"{""location"": ""LS"", ""is_mobile"": false}" 2988,5,275,2017-03-24 16:10:50,http://ward.org/will_mosciski,,180.251.2.246,"{""location"": ""PM"", ""is_mobile"": true}" 2989,5,276,2016-12-17 09:04:24,http://adams.io/nichole_kirlin,,239.39.141.104,"{""location"": ""BI"", ""is_mobile"": false}" 2990,5,276,2017-02-08 01:18:02,http://mosciski.name/layla_steuber,,141.7.176.167,"{""location"": ""SY"", ""is_mobile"": true}" 2991,5,276,2017-05-24 16:00:22,http://crona.org/jennings,,65.250.233.20,"{""location"": ""ZW"", ""is_mobile"": true}" 2992,5,276,2017-01-26 02:53:37,http://johnsoconner.name/sarai,,72.47.128.72,"{""location"": ""MS"", ""is_mobile"": false}" 2993,5,276,2017-01-04 08:35:51,http://schummbashirian.info/kitty,,34.5.253.114,"{""location"": ""DO"", ""is_mobile"": false}" 2994,5,276,2017-04-24 08:49:25,http://howell.net/dillon_greenfelder,,83.8.126.207,"{""location"": ""NA"", ""is_mobile"": false}" 2995,5,276,2016-12-29 17:51:53,http://walker.net/kallie_cain,,174.95.6.228,"{""location"": ""HM"", ""is_mobile"": false}" 2996,5,276,2017-03-20 22:16:19,http://koelpin.biz/garnet,,44.21.122.109,"{""location"": ""BB"", ""is_mobile"": true}" 2997,5,276,2017-01-25 05:39:49,http://dareschamberger.org/lazaro.kuphal,,48.152.146.10,"{""location"": ""IM"", ""is_mobile"": true}" 2998,5,277,2017-03-10 04:19:12,http://kirlinwiegand.net/taryn.daugherty,,53.193.228.191,"{""location"": ""DJ"", ""is_mobile"": false}" 2999,5,277,2017-01-03 16:11:35,http://miller.co/joannie.ritchie,,211.131.196.132,"{""location"": ""CH"", ""is_mobile"": false}" 3000,5,277,2017-02-24 10:27:07,http://crooks.biz/kale.lockman,,55.160.84.241,"{""location"": ""MH"", ""is_mobile"": false}" 3001,5,277,2017-01-20 10:10:08,http://sawayn.io/kristy_king,,95.224.22.18,"{""location"": ""IQ"", ""is_mobile"": true}" 3002,5,277,2017-04-13 19:47:44,http://joneskreiger.org/clemens,,44.66.119.63,"{""location"": ""GE"", ""is_mobile"": false}" 3003,5,277,2017-03-20 02:41:35,http://pfeffer.co/mallory_heathcote,,229.51.113.84,"{""location"": ""PM"", ""is_mobile"": true}" 3004,5,277,2017-04-05 15:39:37,http://leuschkehammes.co/telly,,211.131.196.132,"{""location"": ""PH"", ""is_mobile"": false}" 3005,5,277,2017-04-15 17:55:28,http://damore.org/katarina,,130.90.194.202,"{""location"": ""GL"", ""is_mobile"": false}" 3006,5,277,2017-04-10 17:10:28,http://buckridge.net/mandy_prosacco,,206.219.70.243,"{""location"": ""NF"", ""is_mobile"": false}" 3007,5,277,2017-05-31 21:02:29,http://wisozkwaters.name/alicia,,242.19.25.219,"{""location"": ""CN"", ""is_mobile"": false}" 3008,5,277,2017-02-08 16:36:15,http://jacobs.biz/connor_kozey,,82.94.109.75,"{""location"": ""RS"", ""is_mobile"": true}" 3009,5,277,2017-04-03 15:10:30,http://miller.name/danika,,95.224.22.18,"{""location"": ""CR"", ""is_mobile"": true}" 3010,5,277,2016-12-24 21:38:18,http://wiza.name/alize,,206.29.80.132,"{""location"": ""RU"", ""is_mobile"": true}" 3011,5,277,2017-02-25 03:29:21,http://hackett.biz/johann.toy,,55.160.84.241,"{""location"": ""TV"", ""is_mobile"": false}" 3012,5,277,2017-05-02 11:23:12,http://beatty.co/ebba,,193.120.147.15,"{""location"": ""MS"", ""is_mobile"": true}" 3013,5,277,2016-12-19 03:08:09,http://price.info/annamae,,226.54.21.49,"{""location"": ""NG"", ""is_mobile"": true}" 3014,5,277,2017-06-02 01:11:51,http://spencer.net/candida,,229.51.113.84,"{""location"": ""UZ"", ""is_mobile"": true}" 3015,5,278,2017-05-04 00:10:19,http://rohanerdman.name/clementine,,28.34.213.153,"{""location"": ""CV"", ""is_mobile"": true}" 3016,5,278,2017-02-06 17:48:19,http://huels.info/dominic,,81.251.23.109,"{""location"": ""MK"", ""is_mobile"": false}" 3017,5,278,2017-02-12 07:17:59,http://bins.com/alexie.tillman,,109.109.135.182,"{""location"": ""KZ"", ""is_mobile"": true}" 3018,5,278,2017-01-08 05:43:47,http://bode.com/dax_hintz,,117.169.141.12,"{""location"": ""DK"", ""is_mobile"": false}" 3019,5,278,2017-02-22 21:18:43,http://bernier.name/uriel.collins,,182.205.196.250,"{""location"": ""IT"", ""is_mobile"": true}" 3020,5,278,2017-01-11 07:00:46,http://cartwright.com/ray,,12.37.184.178,"{""location"": ""GE"", ""is_mobile"": false}" 3021,5,278,2017-01-01 04:51:49,http://muller.io/enrico.nienow,,246.251.143.42,"{""location"": ""ES"", ""is_mobile"": false}" 3022,5,278,2016-12-27 01:53:34,http://hagenes.io/heloise_buckridge,,122.195.77.72,"{""location"": ""EC"", ""is_mobile"": false}" 3023,5,278,2016-12-19 12:04:07,http://bailey.io/garland_schuster,,131.65.86.101,"{""location"": ""CU"", ""is_mobile"": false}" 3024,5,278,2017-04-26 23:50:09,http://bergstrom.com/darrell.mccullough,,175.159.169.159,"{""location"": ""SC"", ""is_mobile"": false}" 3025,5,278,2017-01-14 19:13:26,http://waters.io/garett,,12.37.184.178,"{""location"": ""PS"", ""is_mobile"": true}" 3026,5,278,2017-02-03 21:12:48,http://torp.net/ethelyn,,121.230.19.226,"{""location"": ""NF"", ""is_mobile"": false}" 3027,5,278,2017-03-01 15:54:36,http://hamill.com/wilma,,177.56.220.222,"{""location"": ""SX"", ""is_mobile"": false}" 3028,5,278,2017-04-17 01:12:05,http://fadel.io/donato_rodriguez,,95.26.152.2,"{""location"": ""DM"", ""is_mobile"": false}" 3029,5,278,2017-04-05 21:52:45,http://pfannerstill.biz/mckenzie.sauer,,146.10.183.220,"{""location"": ""MG"", ""is_mobile"": true}" 3030,5,278,2017-03-26 21:55:19,http://beierschimmel.co/mozell,,12.37.184.178,"{""location"": ""CF"", ""is_mobile"": true}" 3031,5,278,2016-12-23 20:43:27,http://crona.biz/brandi.waters,,28.34.213.153,"{""location"": ""ME"", ""is_mobile"": true}" 3032,5,278,2017-05-17 11:58:59,http://schoenortiz.co/bailee,,63.104.137.168,"{""location"": ""ML"", ""is_mobile"": false}" 3033,5,279,2017-03-28 12:55:53,http://wiegand.io/kirk_hamill,0.0000000000,60.98.123.200,"{""location"": ""GN"", ""is_mobile"": false}" 3034,5,279,2017-03-22 03:44:15,http://hodkiewiczemard.net/benny,0.0000000000,82.91.227.158,"{""location"": ""NZ"", ""is_mobile"": false}" 3035,5,279,2017-02-08 13:44:18,http://kerluke.org/marcelle.jenkins,0.0000000000,96.135.96.100,"{""location"": ""MN"", ""is_mobile"": false}" 3036,5,279,2017-06-05 11:04:07,http://nienow.io/eli.friesen,0.0000000000,176.68.147.103,"{""location"": ""LT"", ""is_mobile"": false}" 3037,5,279,2017-04-20 19:46:37,http://lehner.info/marvin,0.0000000000,176.8.46.228,"{""location"": ""SY"", ""is_mobile"": true}" 3038,5,279,2017-06-10 15:33:05,http://gulgowski.io/emiliano,0.0000000000,243.163.30.42,"{""location"": ""PS"", ""is_mobile"": true}" 3039,5,279,2016-12-17 10:34:10,http://cummings.co/maryse,0.0000000000,228.118.191.225,"{""location"": ""PA"", ""is_mobile"": false}" 3040,5,279,2017-01-05 02:49:17,http://cainschumm.name/citlalli.schneider,0.0000000000,80.23.113.32,"{""location"": ""SH"", ""is_mobile"": false}" 3041,5,279,2016-12-21 18:29:08,http://kuhlmanabbott.info/nicholas,0.0000000000,99.187.63.223,"{""location"": ""LB"", ""is_mobile"": false}" 3042,5,280,2017-05-14 08:58:48,http://oreilly.net/deshawn,2.0000000000,5.79.102.34,"{""location"": ""IT"", ""is_mobile"": false}" 3043,5,280,2017-04-04 21:14:58,http://pacocha.io/berta_parisian,2.0000000000,227.249.155.170,"{""location"": ""MG"", ""is_mobile"": true}" 3044,5,280,2017-03-19 02:14:54,http://vonrueden.co/elian,1.0000000000,64.184.136.34,"{""location"": ""AO"", ""is_mobile"": true}" 3045,5,280,2017-03-24 08:48:00,http://dach.co/rashawn,1.0000000000,171.217.125.91,"{""location"": ""MU"", ""is_mobile"": false}" 3046,5,280,2017-03-30 05:24:28,http://eichmann.net/lucio,1.0000000000,227.249.155.170,"{""location"": ""BI"", ""is_mobile"": true}" 3047,5,280,2017-01-08 13:54:21,http://carrollwyman.name/abdiel,0.0000000000,227.249.155.170,"{""location"": ""OM"", ""is_mobile"": true}" 3048,5,280,2017-03-29 05:53:05,http://denesik.net/maynard.morar,2.0000000000,111.235.217.176,"{""location"": ""GI"", ""is_mobile"": true}" 3049,5,280,2017-01-29 10:17:36,http://hermann.com/imelda,2.0000000000,60.164.242.47,"{""location"": ""NE"", ""is_mobile"": true}" 3050,5,280,2017-02-10 06:10:25,http://stanton.biz/boris_harris,0.0000000000,181.231.111.41,"{""location"": ""LA"", ""is_mobile"": true}" 3051,5,280,2016-12-24 21:11:52,http://runolfonjohnson.org/emerson_kunde,3.0000000000,218.177.174.36,"{""location"": ""CR"", ""is_mobile"": false}" 3052,5,280,2017-04-21 23:38:09,http://okon.net/jamel,0.0000000000,163.226.246.54,"{""location"": ""AT"", ""is_mobile"": false}" 3053,5,280,2016-12-21 02:05:55,http://raynor.name/nova,3.0000000000,218.53.150.5,"{""location"": ""MA"", ""is_mobile"": true}" 3054,5,281,2017-03-26 06:22:03,http://toy.info/easter.lynch,1.0000000000,111.41.230.229,"{""location"": ""PS"", ""is_mobile"": false}" 3055,5,281,2017-05-30 11:40:58,http://casperhilpert.org/maximillia.greenfelder,2.0000000000,134.5.192.109,"{""location"": ""FK"", ""is_mobile"": false}" 3056,5,281,2017-04-22 18:25:03,http://marks.co/kathlyn.ziemann,2.0000000000,111.41.230.229,"{""location"": ""CU"", ""is_mobile"": true}" 3057,5,281,2017-03-04 09:08:58,http://raulangosh.co/devon.wilkinson,0.0000000000,134.5.192.109,"{""location"": ""JP"", ""is_mobile"": true}" 3058,5,281,2017-03-13 08:39:00,http://leschdubuque.net/darrin,0.0000000000,121.228.123.252,"{""location"": ""GH"", ""is_mobile"": false}" 3059,5,281,2017-06-09 22:21:18,http://purdycollins.net/elijah_runolfon,2.0000000000,231.77.84.67,"{""location"": ""OM"", ""is_mobile"": false}" 3060,5,281,2017-03-03 02:17:11,http://kiehnbaumbach.name/brennon.dach,2.0000000000,134.5.192.109,"{""location"": ""GA"", ""is_mobile"": true}" 3061,5,281,2017-02-13 06:04:05,http://grady.name/frieda,3.0000000000,164.123.171.152,"{""location"": ""SL"", ""is_mobile"": false}" 3062,5,281,2017-03-02 23:04:54,http://boehm.net/perry_breitenberg,3.0000000000,254.203.236.245,"{""location"": ""TM"", ""is_mobile"": false}" 3063,5,281,2017-05-16 13:05:05,http://lemke.io/elroy.reichel,2.0000000000,216.120.118.50,"{""location"": ""UG"", ""is_mobile"": true}" 3064,5,282,2017-01-19 09:30:42,http://gutmann.biz/sandrine_ledner,1.0000000000,203.207.214.143,"{""location"": ""TW"", ""is_mobile"": false}" 3065,5,282,2016-12-17 10:28:23,http://kohler.io/aniyah,1.0000000000,161.224.203.16,"{""location"": ""GD"", ""is_mobile"": false}" 3066,5,282,2017-02-05 18:29:36,http://robelbalistreri.org/neil.bauch,2.0000000000,137.176.82.202,"{""location"": ""SX"", ""is_mobile"": false}" 3067,5,282,2017-01-03 21:06:47,http://thompson.net/lucy_spinka,0.0000000000,169.148.187.213,"{""location"": ""EC"", ""is_mobile"": false}" 3068,5,282,2017-01-09 18:36:37,http://cormier.name/erna_leannon,1.0000000000,57.28.194.234,"{""location"": ""SS"", ""is_mobile"": true}" 3069,5,282,2017-05-14 10:41:40,http://hudson.co/joan,0.0000000000,203.207.214.143,"{""location"": ""GB"", ""is_mobile"": false}" 3070,5,282,2017-05-01 14:23:35,http://stiedemann.com/izaiah.boyle,0.0000000000,241.199.39.5,"{""location"": ""CK"", ""is_mobile"": true}" 3071,5,282,2017-04-12 23:52:01,http://wilderman.io/gracie_tillman,0.0000000000,166.100.133.65,"{""location"": ""LK"", ""is_mobile"": true}" 3072,5,282,2017-02-28 13:04:15,http://wilkinsonabernathy.name/hallie,1.0000000000,166.100.133.65,"{""location"": ""MU"", ""is_mobile"": false}" 3073,5,282,2017-03-14 02:55:26,http://hahnmetz.io/isabell.weinat,0.0000000000,220.193.145.128,"{""location"": ""NF"", ""is_mobile"": false}" 3074,5,282,2017-05-25 14:20:45,http://jaskolskiklocko.org/cristal,0.0000000000,254.78.210.48,"{""location"": ""PN"", ""is_mobile"": true}" 3075,5,283,2017-02-28 04:00:16,http://roberts.co/brown,1.0000000000,9.59.160.216,"{""location"": ""AS"", ""is_mobile"": true}" 3076,5,283,2017-05-10 13:36:19,http://jakubowski.org/hadley_torp,1.0000000000,56.132.157.83,"{""location"": ""TO"", ""is_mobile"": false}" 3077,5,283,2017-01-08 17:42:52,http://gaylord.com/danyka,1.0000000000,156.128.206.209,"{""location"": ""CR"", ""is_mobile"": true}" 3078,5,283,2017-06-12 06:40:39,http://murazik.co/sincere_kulas,0.0000000000,199.90.150.138,"{""location"": ""PS"", ""is_mobile"": true}" 3079,5,283,2016-12-17 07:37:24,http://gorczany.info/ansel,0.0000000000,78.87.44.139,"{""location"": ""LI"", ""is_mobile"": false}" 3080,5,283,2017-06-13 02:32:51,http://flatley.io/irving,0.0000000000,44.16.234.13,"{""location"": ""SK"", ""is_mobile"": true}" 3081,5,283,2017-05-04 22:37:22,http://powlowskihowell.biz/randall,0.0000000000,104.20.43.197,"{""location"": ""MY"", ""is_mobile"": false}" 3082,5,283,2017-01-25 21:09:27,http://okunevagorczany.name/elvie.mitchell,0.0000000000,3.197.215.104,"{""location"": ""KR"", ""is_mobile"": false}" 3083,5,283,2017-03-25 02:25:53,http://fay.name/buford,0.0000000000,231.26.193.67,"{""location"": ""GR"", ""is_mobile"": false}" 3084,5,283,2016-12-28 06:50:49,http://quitzon.co/alison,1.0000000000,132.68.78.153,"{""location"": ""OM"", ""is_mobile"": true}" 3085,5,283,2016-12-27 21:48:15,http://batzweinat.org/eunice.mccullough,1.0000000000,12.151.195.196,"{""location"": ""IL"", ""is_mobile"": true}" 3086,5,283,2017-02-02 18:53:38,http://stiedemann.info/dorris.corwin,1.0000000000,89.32.150.92,"{""location"": ""BY"", ""is_mobile"": false}" 3087,5,283,2017-02-12 06:39:46,http://wilkinson.info/percy.howell,0.0000000000,12.151.195.196,"{""location"": ""UG"", ""is_mobile"": false}" 3088,5,283,2017-01-21 11:54:14,http://farrell.name/damian.lowe,1.0000000000,44.16.234.13,"{""location"": ""FI"", ""is_mobile"": true}" 3089,5,284,2017-01-22 03:50:46,http://mertz.io/gust_harris,1.0000000000,209.81.247.43,"{""location"": ""MC"", ""is_mobile"": false}" 3090,5,284,2017-04-19 23:54:10,http://creminbotsford.org/ona,1.0000000000,254.179.149.164,"{""location"": ""GS"", ""is_mobile"": true}" 3091,5,284,2017-02-11 02:23:51,http://breitenberg.com/colby_treutel,1.0000000000,180.239.187.156,"{""location"": ""MM"", ""is_mobile"": true}" 3092,5,284,2017-01-30 16:48:32,http://zboncak.org/sheila,0.0000000000,168.164.190.77,"{""location"": ""UY"", ""is_mobile"": false}" 3093,5,285,2017-04-09 03:13:07,http://goodwin.com/rylan.ferry,3.0000000000,95.232.121.161,"{""location"": ""BQ"", ""is_mobile"": true}" 3094,5,285,2017-05-18 00:12:25,http://heaney.info/cristina,2.0000000000,129.200.46.203,"{""location"": ""GI"", ""is_mobile"": true}" 3095,5,285,2016-12-19 16:18:54,http://larson.biz/marcelina.braun,3.0000000000,156.243.215.26,"{""location"": ""GG"", ""is_mobile"": true}" 3096,5,285,2017-01-23 09:14:25,http://abernathy.name/zachariah_bernhard,0.0000000000,100.90.200.253,"{""location"": ""UA"", ""is_mobile"": true}" 3097,5,285,2017-06-08 17:32:50,http://whiterunte.net/alysa,1.0000000000,238.221.135.205,"{""location"": ""SV"", ""is_mobile"": false}" 3098,5,285,2017-04-16 15:31:37,http://gusikowski.com/lambert.bins,3.0000000000,129.31.92.208,"{""location"": ""AS"", ""is_mobile"": false}" 3099,5,285,2017-02-13 18:50:46,http://hettingerhermann.com/christian.gusikowski,2.0000000000,19.93.150.105,"{""location"": ""CN"", ""is_mobile"": true}" 3100,5,285,2017-02-28 10:43:18,http://boscobeer.net/christa.herman,0.0000000000,80.186.156.140,"{""location"": ""HU"", ""is_mobile"": false}" 3101,5,285,2017-04-07 21:05:42,http://ondricka.name/stephanie_upton,1.0000000000,129.31.92.208,"{""location"": ""MK"", ""is_mobile"": false}" 3102,5,285,2017-06-12 19:14:58,http://flatley.org/josie.rodriguez,2.0000000000,197.138.45.83,"{""location"": ""UZ"", ""is_mobile"": true}" 3103,5,285,2017-05-22 06:20:05,http://kuhlman.biz/kelli.bartoletti,1.0000000000,148.156.111.158,"{""location"": ""JO"", ""is_mobile"": true}" 3104,5,285,2017-02-21 11:29:15,http://mcdermottromaguera.info/marquis.gislason,0.0000000000,246.247.188.185,"{""location"": ""VN"", ""is_mobile"": false}" 3105,5,285,2017-02-28 16:50:28,http://sawaynrowe.net/nelda_veum,2.0000000000,192.167.72.153,"{""location"": ""SV"", ""is_mobile"": true}" 3106,5,285,2017-04-23 02:19:41,http://fritsch.name/eldora_kertzmann,0.0000000000,100.90.200.253,"{""location"": ""TM"", ""is_mobile"": false}" 3107,5,285,2017-05-31 05:37:29,http://gorczanykunde.name/darryl,2.0000000000,238.221.135.205,"{""location"": ""BD"", ""is_mobile"": false}" 3108,5,285,2017-06-09 09:35:05,http://moriette.io/ladarius,3.0000000000,24.234.80.157,"{""location"": ""SZ"", ""is_mobile"": false}" 3109,5,285,2017-02-22 16:32:59,http://corwin.net/wendell.cummerata,3.0000000000,192.167.72.153,"{""location"": ""JP"", ""is_mobile"": true}" 3110,5,285,2017-03-12 09:43:00,http://rice.net/houston,1.0000000000,197.138.45.83,"{""location"": ""NO"", ""is_mobile"": true}" 3111,5,285,2017-01-02 21:24:59,http://shanahanglover.net/bryon,1.0000000000,12.178.198.24,"{""location"": ""AZ"", ""is_mobile"": true}" 3112,5,285,2017-01-02 15:24:33,http://wunsch.net/clark,3.0000000000,21.72.181.40,"{""location"": ""MV"", ""is_mobile"": true}" 3113,5,286,2017-02-08 17:56:26,http://barton.co/alverta,1.0000000000,190.76.86.8,"{""location"": ""BO"", ""is_mobile"": false}" 3114,5,286,2017-01-18 11:21:19,http://halvorson.org/orpha,1.0000000000,126.220.183.132,"{""location"": ""KP"", ""is_mobile"": true}" 3115,5,286,2017-02-11 20:00:51,http://doyle.net/norberto,0.0000000000,61.7.17.163,"{""location"": ""CF"", ""is_mobile"": false}" 3116,5,286,2017-05-13 07:42:02,http://hudson.net/kasey.lang,0.0000000000,200.191.34.127,"{""location"": ""FK"", ""is_mobile"": true}" 3117,5,286,2017-03-07 01:31:27,http://kshlerin.com/amparo_keeling,0.0000000000,58.72.47.44,"{""location"": ""MV"", ""is_mobile"": true}" 3118,5,286,2017-03-10 23:57:08,http://legros.biz/clinton.kulas,0.0000000000,82.66.163.94,"{""location"": ""FR"", ""is_mobile"": true}" 3119,5,287,2017-01-07 14:01:20,http://hartmannkerluke.org/elna,1.0000000000,167.169.45.70,"{""location"": ""PA"", ""is_mobile"": false}" 3120,5,287,2017-02-18 23:47:41,http://bins.info/pete.abernathy,1.0000000000,167.169.45.70,"{""location"": ""TW"", ""is_mobile"": true}" 3121,5,287,2017-01-23 06:57:25,http://kaulke.biz/austyn.ryan,2.0000000000,50.130.174.129,"{""location"": ""ET"", ""is_mobile"": false}" 3122,5,288,2017-06-11 09:21:39,http://haley.com/ophelia,,174.63.17.95,"{""location"": ""ME"", ""is_mobile"": true}" 3123,5,288,2017-03-11 04:32:30,http://franecki.io/marjory.moore,,44.250.113.125,"{""location"": ""BN"", ""is_mobile"": true}" 3124,5,288,2017-04-21 16:06:55,http://walterbrakus.biz/taryn_goldner,,170.72.207.199,"{""location"": ""TJ"", ""is_mobile"": false}" 3125,5,288,2017-04-01 04:11:33,http://purdy.com/percy_pfeffer,,170.72.207.199,"{""location"": ""BM"", ""is_mobile"": true}" 3126,5,288,2017-06-04 00:28:32,http://jones.name/brannon_welch,,174.63.17.95,"{""location"": ""TH"", ""is_mobile"": false}" 3127,5,288,2017-02-05 04:45:04,http://herzog.net/gavin.gibson,,100.142.27.116,"{""location"": ""TT"", ""is_mobile"": true}" 3128,5,288,2016-12-30 01:11:34,http://shieldshickle.co/reid,,27.154.71.32,"{""location"": ""DE"", ""is_mobile"": false}" 3129,5,288,2017-01-01 17:01:11,http://schoen.co/adrianna,,109.63.98.158,"{""location"": ""MD"", ""is_mobile"": false}" 3130,5,288,2016-12-17 03:56:50,http://jacobs.com/keely_murphy,,119.52.214.167,"{""location"": ""WS"", ""is_mobile"": false}" 3131,5,288,2017-02-05 17:55:35,http://lesch.co/berneice,,105.181.169.129,"{""location"": ""MU"", ""is_mobile"": true}" 3132,5,288,2017-02-28 14:06:03,http://ziemann.org/chris,,83.79.21.19,"{""location"": ""GA"", ""is_mobile"": false}" 3133,5,288,2016-12-28 22:26:42,http://fisher.name/kiara.ko,,119.52.214.167,"{""location"": ""LB"", ""is_mobile"": true}" 3134,5,288,2017-01-29 10:46:20,http://gleason.com/ines_hane,,146.10.241.14,"{""location"": ""CL"", ""is_mobile"": true}" 3135,5,288,2017-03-30 21:09:55,http://bartonwaters.name/mary,,105.17.77.86,"{""location"": ""SX"", ""is_mobile"": false}" 3136,5,288,2017-02-06 14:12:37,http://stehrvonrueden.com/adele,,17.129.97.248,"{""location"": ""SV"", ""is_mobile"": true}" 3137,5,289,2016-12-31 11:06:49,http://collins.info/sister.jerde,,28.145.201.223,"{""location"": ""BI"", ""is_mobile"": true}" 3138,5,289,2017-01-17 14:25:40,http://roberts.net/eladio,,58.73.244.7,"{""location"": ""GS"", ""is_mobile"": false}" 3139,5,289,2017-04-06 22:05:06,http://harris.com/gillian.reichert,,28.70.136.96,"{""location"": ""RO"", ""is_mobile"": true}" 3140,5,289,2017-03-13 15:39:41,http://schmelerpfeffer.com/clemmie,,160.127.174.7,"{""location"": ""ZM"", ""is_mobile"": true}" 3141,5,289,2017-01-24 16:51:50,http://bayer.co/megane.fadel,,12.148.102.227,"{""location"": ""GA"", ""is_mobile"": true}" 3142,5,289,2017-02-03 07:54:55,http://weber.name/hosea_schuster,,44.206.232.19,"{""location"": ""JM"", ""is_mobile"": false}" 3143,5,289,2017-03-03 00:52:39,http://grimes.org/dakota.batz,,173.91.151.16,"{""location"": ""TG"", ""is_mobile"": false}" 3144,5,289,2017-04-02 23:08:06,http://hilpert.net/jamil.damore,,34.145.244.196,"{""location"": ""KY"", ""is_mobile"": false}" 3145,5,289,2017-02-14 19:20:31,http://mckenzie.org/moie,,138.229.120.99,"{""location"": ""IT"", ""is_mobile"": false}" 3146,5,289,2017-03-30 20:18:42,http://quigley.name/gerda,,173.231.24.48,"{""location"": ""SR"", ""is_mobile"": true}" 3147,5,289,2017-05-12 21:25:46,http://borer.co/mikayla,,107.140.161.117,"{""location"": ""CH"", ""is_mobile"": true}" 3148,5,289,2017-04-18 13:46:01,http://raynorrippin.co/geovanny,,96.123.158.170,"{""location"": ""MT"", ""is_mobile"": true}" 3149,5,289,2017-03-01 16:56:11,http://legroskuphal.name/ernestine.weinat,,50.155.99.238,"{""location"": ""BF"", ""is_mobile"": true}" 3150,5,289,2017-06-02 07:24:13,http://hermann.co/deron,,107.236.24.58,"{""location"": ""MU"", ""is_mobile"": true}" 3151,5,289,2017-05-23 00:48:31,http://krajcik.co/adele_hegmann,,12.148.102.227,"{""location"": ""DK"", ""is_mobile"": true}" 3152,5,289,2017-01-28 06:29:26,http://dickinson.name/spencer.prohaska,,252.28.136.77,"{""location"": ""CK"", ""is_mobile"": false}" 3153,5,290,2017-05-16 14:52:02,http://schadenspencer.name/ashton_wintheiser,,242.149.248.251,"{""location"": ""YE"", ""is_mobile"": false}" 3154,5,290,2017-02-12 11:24:32,http://labadiewintheiser.co/annabelle_rutherford,,244.96.214.5,"{""location"": ""NU"", ""is_mobile"": false}" 3155,5,290,2017-03-16 03:56:00,http://donnellyschiller.co/cole,,68.56.225.48,"{""location"": ""CA"", ""is_mobile"": false}" 3156,5,290,2017-06-10 09:44:11,http://torp.info/ania,,173.197.73.230,"{""location"": ""RU"", ""is_mobile"": true}" 3157,5,290,2017-04-08 04:23:27,http://lakinhahn.org/linwood,,244.244.32.253,"{""location"": ""BZ"", ""is_mobile"": true}" 3158,5,290,2017-02-11 18:40:11,http://trantow.name/brenda.wunsch,,190.243.88.181,"{""location"": ""NR"", ""is_mobile"": false}" 3159,5,290,2017-05-27 22:38:18,http://rosenbaum.co/tremaine,,143.242.33.158,"{""location"": ""TR"", ""is_mobile"": false}" 3160,5,290,2017-02-27 07:48:24,http://doyle.info/abdiel.hodkiewicz,,156.217.240.44,"{""location"": ""MD"", ""is_mobile"": false}" 3161,5,290,2017-02-09 05:01:44,http://hamill.biz/benedict_zieme,,244.96.214.5,"{""location"": ""AM"", ""is_mobile"": true}" 3162,5,290,2017-05-30 14:01:24,http://jakubowski.net/everett_mckenzie,,217.72.83.73,"{""location"": ""AZ"", ""is_mobile"": true}" 3163,5,290,2017-02-02 11:20:42,http://bartell.co/bernadine_mayert,,217.72.83.73,"{""location"": ""JP"", ""is_mobile"": false}" 3164,5,290,2017-01-22 08:51:16,http://hoppe.io/eloise,,156.217.240.44,"{""location"": ""SX"", ""is_mobile"": true}" 3165,5,290,2017-01-26 11:36:46,http://wilderman.org/ora,,244.244.32.253,"{""location"": ""DM"", ""is_mobile"": false}" 3166,5,290,2017-05-01 05:04:49,http://ankunding.io/francisca,,224.90.243.143,"{""location"": ""ID"", ""is_mobile"": false}" 3167,5,290,2017-02-26 05:01:33,http://considine.io/kole.parisian,,224.90.243.143,"{""location"": ""VN"", ""is_mobile"": false}" 3168,5,290,2017-01-12 05:40:53,http://harberchamplin.name/hipolito,,217.72.83.73,"{""location"": ""NI"", ""is_mobile"": false}" 3169,5,290,2017-04-20 21:06:53,http://paucekjaskolski.com/clifton.turcotte,,167.209.138.234,"{""location"": ""BM"", ""is_mobile"": false}" 3170,5,290,2017-06-09 14:50:21,http://emard.io/lolita,,244.244.32.253,"{""location"": ""UA"", ""is_mobile"": false}" 3171,5,290,2017-06-12 10:37:48,http://blick.io/juston_hartmann,,143.242.33.158,"{""location"": ""LS"", ""is_mobile"": false}" 3172,5,290,2017-01-02 03:45:57,http://dibbert.co/fabian,,244.244.32.253,"{""location"": ""CF"", ""is_mobile"": false}" 3173,5,291,2017-05-18 00:19:21,http://marvin.name/odie,,195.122.235.106,"{""location"": ""AX"", ""is_mobile"": false}" 3174,5,291,2016-12-20 01:13:40,http://reinger.co/marty,,134.45.57.187,"{""location"": ""SI"", ""is_mobile"": true}" 3175,5,291,2017-05-09 22:39:51,http://hartmanngulgowski.com/mireya,,141.68.20.248,"{""location"": ""CM"", ""is_mobile"": false}" 3176,5,291,2017-02-20 23:53:52,http://oharaschneider.info/laron,,248.115.111.243,"{""location"": ""RW"", ""is_mobile"": true}" 3177,5,291,2017-01-25 03:51:47,http://jasttreutel.com/caleb,,170.59.45.150,"{""location"": ""HT"", ""is_mobile"": true}" 3178,5,291,2017-06-11 10:18:54,http://bosco.info/buster,,115.206.160.244,"{""location"": ""GU"", ""is_mobile"": true}" 3179,5,291,2017-03-15 20:33:28,http://predovic.io/allie,,73.241.207.190,"{""location"": ""MC"", ""is_mobile"": false}" 3180,5,291,2017-02-12 10:16:16,http://baumbach.biz/ro_gleichner,,221.111.173.139,"{""location"": ""PR"", ""is_mobile"": true}" 3181,5,291,2017-02-14 22:26:46,http://fahey.name/terence.mcdermott,,115.206.160.244,"{""location"": ""FR"", ""is_mobile"": true}" 3182,5,291,2017-01-04 23:35:35,http://gorczany.info/baron,,149.53.20.197,"{""location"": ""LC"", ""is_mobile"": false}" 3183,5,292,2016-12-18 09:08:35,http://feeney.io/marty,,75.188.235.6,"{""location"": ""BF"", ""is_mobile"": false}" 3184,5,292,2017-03-15 17:31:41,http://barton.org/ashly.bauch,,97.42.245.27,"{""location"": ""BI"", ""is_mobile"": false}" 3185,5,292,2017-04-25 06:05:54,http://bauch.co/aida,,250.60.82.214,"{""location"": ""AX"", ""is_mobile"": false}" 3186,5,292,2017-05-23 13:55:25,http://ziemannfranecki.info/idell.swift,,105.166.194.111,"{""location"": ""IE"", ""is_mobile"": false}" 3187,5,292,2017-05-22 23:17:10,http://emmerichgreen.net/kenya.williamson,,71.80.232.252,"{""location"": ""GY"", ""is_mobile"": false}" 3188,5,292,2017-02-09 07:49:41,http://thompson.com/glennie,,25.235.71.189,"{""location"": ""UZ"", ""is_mobile"": true}" 3189,5,292,2017-03-05 00:38:20,http://kunde.com/fay,,197.23.182.254,"{""location"": ""GD"", ""is_mobile"": false}" 3301,5,304,2017-02-08 08:03:05,http://mann.name/luz,0.0000000000,161.66.146.85,"{""location"": ""IL"", ""is_mobile"": true}" 3190,5,292,2017-04-14 07:11:27,http://bashirian.org/maddison,,7.78.111.93,"{""location"": ""PR"", ""is_mobile"": true}" 3191,5,292,2017-03-05 02:27:20,http://mitchell.org/luciano_abshire,,21.72.97.213,"{""location"": ""LR"", ""is_mobile"": false}" 3192,5,292,2017-05-22 04:12:48,http://kiehn.com/gregg.padberg,,100.56.166.220,"{""location"": ""LS"", ""is_mobile"": false}" 3193,5,292,2017-04-16 01:40:25,http://rolfsonfahey.name/trenton_klein,,68.103.238.184,"{""location"": ""HU"", ""is_mobile"": true}" 3194,5,292,2016-12-23 16:07:56,http://raynor.name/caterina.bergstrom,,245.117.206.3,"{""location"": ""AS"", ""is_mobile"": false}" 3195,5,292,2016-12-19 22:00:22,http://oconnell.biz/leone.mosciski,,188.169.92.181,"{""location"": ""BM"", ""is_mobile"": true}" 3196,5,292,2017-02-08 13:39:15,http://mohrrosenbaum.org/roman,,106.145.147.229,"{""location"": ""VG"", ""is_mobile"": false}" 3197,5,293,2017-06-11 20:57:01,http://lubowitzwatsica.biz/katelyn.bins,,100.143.224.236,"{""location"": ""CL"", ""is_mobile"": false}" 3198,5,293,2017-05-15 07:30:23,http://lesch.name/aaron,,108.248.85.147,"{""location"": ""BQ"", ""is_mobile"": true}" 3199,5,293,2017-01-23 16:11:15,http://spencergoldner.biz/darlene,,252.165.244.200,"{""location"": ""AT"", ""is_mobile"": true}" 3200,5,293,2017-05-21 18:27:17,http://towne.info/gianni,,78.24.244.164,"{""location"": ""IN"", ""is_mobile"": false}" 3201,5,293,2017-02-13 15:04:14,http://swaniawski.co/delia,,199.250.222.102,"{""location"": ""TL"", ""is_mobile"": false}" 3202,5,293,2017-01-29 14:45:19,http://bartoletti.com/esta_crist,,100.143.224.236,"{""location"": ""GD"", ""is_mobile"": true}" 3203,5,294,2017-02-20 09:07:14,http://donnelly.net/alayna,,230.212.214.129,"{""location"": ""JM"", ""is_mobile"": false}" 3204,5,294,2017-01-12 02:26:45,http://kleinvandervort.name/mafalda_gottlieb,,250.172.221.126,"{""location"": ""AU"", ""is_mobile"": true}" 3205,5,294,2016-12-15 19:10:46,http://weber.io/doyle,,250.155.133.59,"{""location"": ""AT"", ""is_mobile"": false}" 3206,5,295,2017-03-31 07:39:04,http://pouros.name/alexis,2.0000000000,192.35.15.96,"{""location"": ""TD"", ""is_mobile"": true}" 3207,5,295,2017-04-24 13:12:04,http://roobquitzon.name/garland,2.0000000000,116.119.172.221,"{""location"": ""PH"", ""is_mobile"": false}" 3208,5,295,2017-01-12 02:39:20,http://runolfon.co/xzavier_harber,1.0000000000,113.202.114.115,"{""location"": ""MR"", ""is_mobile"": true}" 3209,5,295,2017-05-15 14:49:09,http://torphy.info/mark.wiegand,1.0000000000,94.84.142.215,"{""location"": ""ZW"", ""is_mobile"": true}" 3210,5,295,2017-02-13 17:11:52,http://senger.io/nicolas,1.0000000000,9.26.183.188,"{""location"": ""JM"", ""is_mobile"": false}" 3211,5,295,2017-01-21 23:26:10,http://ziemannrunolfsdottir.io/cameron_nitzsche,0.0000000000,9.219.126.149,"{""location"": ""MK"", ""is_mobile"": true}" 3212,5,295,2017-02-22 19:30:58,http://conn.net/mitchell,0.0000000000,95.106.37.172,"{""location"": ""TH"", ""is_mobile"": true}" 3213,5,295,2017-01-05 09:58:23,http://rice.name/tyree_pacocha,2.0000000000,116.84.12.251,"{""location"": ""JE"", ""is_mobile"": false}" 3214,5,295,2017-01-20 17:37:38,http://schulist.co/colten,1.0000000000,183.239.185.104,"{""location"": ""SL"", ""is_mobile"": true}" 3215,5,295,2017-03-07 05:41:37,http://bailey.io/mathias,0.0000000000,222.173.244.21,"{""location"": ""IO"", ""is_mobile"": true}" 3216,5,295,2017-06-02 22:44:27,http://baileygreen.biz/charles,1.0000000000,241.221.93.23,"{""location"": ""AL"", ""is_mobile"": true}" 3217,5,295,2017-02-03 21:42:43,http://funkkirlin.name/garfield,2.0000000000,94.84.142.215,"{""location"": ""MZ"", ""is_mobile"": false}" 3218,5,295,2017-03-31 04:56:28,http://sawayn.info/jerome_hauck,1.0000000000,24.148.116.113,"{""location"": ""DM"", ""is_mobile"": true}" 3219,5,295,2017-04-21 15:37:32,http://donnellylakin.net/vaughn.daniel,0.0000000000,183.239.185.104,"{""location"": ""BR"", ""is_mobile"": true}" 3220,5,295,2017-05-04 04:37:27,http://mayerhirthe.biz/amparo_prohaska,0.0000000000,178.211.35.164,"{""location"": ""MK"", ""is_mobile"": true}" 3221,5,295,2017-05-29 04:51:40,http://rathkuhn.com/maud,2.0000000000,222.173.244.21,"{""location"": ""LB"", ""is_mobile"": true}" 3222,5,295,2017-02-16 22:20:05,http://parker.org/astrid.thiel,2.0000000000,192.35.15.96,"{""location"": ""FJ"", ""is_mobile"": true}" 3223,5,295,2017-01-17 14:05:08,http://lednerreichel.co/elmo_walker,1.0000000000,72.172.8.6,"{""location"": ""ZM"", ""is_mobile"": false}" 3224,5,295,2017-02-28 15:30:33,http://bergstrom.info/jana_wolff,1.0000000000,178.211.35.164,"{""location"": ""MZ"", ""is_mobile"": true}" 3225,5,295,2017-01-29 16:29:45,http://kaulke.com/chasity.greenholt,2.0000000000,38.44.81.86,"{""location"": ""SN"", ""is_mobile"": false}" 3226,5,296,2016-12-16 01:59:48,http://moriette.biz/layla,2.0000000000,21.142.120.228,"{""location"": ""CM"", ""is_mobile"": true}" 3227,5,296,2017-01-15 01:00:54,http://bednar.com/wilford_mills,2.0000000000,106.53.18.68,"{""location"": ""DO"", ""is_mobile"": true}" 3228,5,296,2017-01-03 20:32:59,http://kautzer.name/haven,2.0000000000,120.244.152.76,"{""location"": ""JE"", ""is_mobile"": true}" 3229,5,296,2017-04-07 14:13:37,http://cainzemlak.biz/herminia,1.0000000000,126.157.201.64,"{""location"": ""TD"", ""is_mobile"": true}" 3230,5,296,2017-05-11 00:39:01,http://wolff.org/stan,1.0000000000,72.218.111.159,"{""location"": ""IL"", ""is_mobile"": true}" 3231,5,296,2017-04-27 11:32:57,http://kulas.io/concepcion.cremin,1.0000000000,105.190.248.175,"{""location"": ""FK"", ""is_mobile"": true}" 3232,5,296,2017-02-07 23:26:50,http://nitzsche.io/rubye,0.0000000000,232.250.41.85,"{""location"": ""AO"", ""is_mobile"": true}" 3233,5,297,2017-02-24 23:41:58,http://beahanbernier.com/leonie,2.0000000000,210.185.143.108,"{""location"": ""SG"", ""is_mobile"": true}" 3234,5,297,2017-04-02 15:07:32,http://hagenes.org/nicolas,1.0000000000,116.191.2.177,"{""location"": ""SI"", ""is_mobile"": true}" 3235,5,297,2017-01-31 20:02:23,http://zemlak.name/kobe.mclaughlin,2.0000000000,117.47.209.90,"{""location"": ""EC"", ""is_mobile"": false}" 3236,5,297,2017-06-14 02:21:21,http://rohanpouros.org/lawrence,0.0000000000,62.132.245.171,"{""location"": ""IT"", ""is_mobile"": true}" 3237,5,297,2017-05-31 07:44:56,http://emardwiegand.name/trevor,0.0000000000,252.198.83.7,"{""location"": ""LU"", ""is_mobile"": true}" 3238,5,297,2017-01-06 08:55:01,http://frami.io/major,1.0000000000,47.150.217.203,"{""location"": ""RO"", ""is_mobile"": true}" 3239,5,297,2017-01-31 14:13:51,http://kubschimmel.io/vincent,2.0000000000,242.136.58.116,"{""location"": ""CI"", ""is_mobile"": true}" 3240,5,297,2017-04-08 10:08:01,http://oconner.info/may.bartoletti,2.0000000000,47.150.217.203,"{""location"": ""JP"", ""is_mobile"": false}" 3241,5,297,2017-01-19 01:20:01,http://gibsonheaney.info/kaia_schumm,3.0000000000,210.185.143.108,"{""location"": ""PH"", ""is_mobile"": false}" 3242,5,297,2017-04-27 10:52:35,http://schuppekuhlman.org/gavin,0.0000000000,219.204.37.55,"{""location"": ""ME"", ""is_mobile"": false}" 3243,5,297,2017-02-09 04:03:29,http://sauerdoyle.io/lew_johns,2.0000000000,210.185.143.108,"{""location"": ""ZW"", ""is_mobile"": true}" 3244,5,297,2017-03-15 09:25:07,http://gibson.net/shane,0.0000000000,116.191.2.177,"{""location"": ""MW"", ""is_mobile"": false}" 3245,5,298,2017-06-01 18:14:16,http://gibson.co/gracie.abshire,0.0000000000,54.39.37.101,"{""location"": ""BG"", ""is_mobile"": true}" 3246,5,299,2017-03-07 20:58:35,http://wunschjerde.biz/kelli,2.0000000000,148.216.81.254,"{""location"": ""KH"", ""is_mobile"": true}" 3247,5,299,2017-04-17 22:58:37,http://turcotte.co/eileen.robel,1.0000000000,248.9.149.56,"{""location"": ""PE"", ""is_mobile"": false}" 3248,5,299,2017-03-10 06:21:05,http://bergehartmann.info/macie,0.0000000000,123.215.90.26,"{""location"": ""BD"", ""is_mobile"": true}" 3249,5,299,2017-05-12 07:47:12,http://johns.com/jalen.terry,0.0000000000,212.243.170.229,"{""location"": ""BY"", ""is_mobile"": true}" 3250,5,299,2017-05-30 15:07:29,http://osinski.net/raphaelle,0.0000000000,93.211.177.187,"{""location"": ""AM"", ""is_mobile"": true}" 3251,5,299,2016-12-30 02:21:02,http://bergnaum.co/hipolito_collier,2.0000000000,93.211.177.187,"{""location"": ""ST"", ""is_mobile"": true}" 3252,5,299,2017-04-11 22:33:23,http://leannon.info/oda,0.0000000000,148.216.81.254,"{""location"": ""MD"", ""is_mobile"": false}" 3253,5,299,2016-12-24 15:31:25,http://casper.biz/timmy.cole,1.0000000000,58.168.204.203,"{""location"": ""PS"", ""is_mobile"": true}" 3254,5,299,2017-03-08 13:43:31,http://zboncak.com/elenor,0.0000000000,218.78.254.136,"{""location"": ""EH"", ""is_mobile"": false}" 3255,5,299,2017-02-08 08:40:58,http://brekkebernhard.info/kamryn.dibbert,1.0000000000,212.243.170.229,"{""location"": ""YE"", ""is_mobile"": true}" 3256,5,299,2017-02-22 15:44:29,http://kohler.io/friedrich,1.0000000000,84.13.127.115,"{""location"": ""EE"", ""is_mobile"": false}" 3257,5,299,2017-03-08 21:11:03,http://fisherblock.net/forrest,2.0000000000,77.80.22.226,"{""location"": ""GN"", ""is_mobile"": false}" 3258,5,299,2017-02-23 08:26:03,http://okonoberbrunner.info/leone,0.0000000000,8.177.90.237,"{""location"": ""GL"", ""is_mobile"": true}" 3259,5,299,2017-03-18 18:51:21,http://grady.info/velda_wyman,0.0000000000,128.239.117.117,"{""location"": ""NF"", ""is_mobile"": true}" 3260,5,299,2017-03-09 12:04:52,http://borerking.biz/vena,0.0000000000,13.66.203.3,"{""location"": ""BZ"", ""is_mobile"": false}" 3261,5,299,2017-01-15 01:43:47,http://jakubowskischaefer.org/karina,1.0000000000,123.215.90.26,"{""location"": ""PG"", ""is_mobile"": false}" 3262,5,299,2017-04-19 22:14:55,http://spinkarolfson.io/cara,2.0000000000,8.177.90.237,"{""location"": ""GD"", ""is_mobile"": false}" 3263,5,299,2017-03-16 02:44:40,http://abshireosinski.biz/fausto,1.0000000000,164.36.238.52,"{""location"": ""CH"", ""is_mobile"": false}" 3264,5,299,2017-05-04 12:59:42,http://swaniawski.co/bill,0.0000000000,123.215.90.26,"{""location"": ""BF"", ""is_mobile"": true}" 3265,5,300,2017-06-08 17:34:43,http://batz.co/taylor,0.0000000000,183.174.159.65,"{""location"": ""CI"", ""is_mobile"": false}" 3266,5,300,2016-12-31 12:32:09,http://gradyheidenreich.info/kiana_sporer,2.0000000000,171.128.25.193,"{""location"": ""KG"", ""is_mobile"": true}" 3267,5,300,2017-06-09 10:54:12,http://raynorgraham.biz/madalyn.robel,1.0000000000,237.68.253.55,"{""location"": ""MF"", ""is_mobile"": true}" 3268,5,300,2017-02-01 21:16:08,http://will.net/jerry,1.0000000000,22.122.49.159,"{""location"": ""PM"", ""is_mobile"": true}" 3269,5,300,2017-05-14 11:51:48,http://jacobi.net/stephon.schuppe,2.0000000000,46.96.133.141,"{""location"": ""ID"", ""is_mobile"": true}" 3270,5,300,2017-04-08 15:59:28,http://stehr.com/elian,1.0000000000,68.8.220.4,"{""location"": ""KM"", ""is_mobile"": false}" 3271,5,300,2016-12-14 10:36:58,http://prohaska.org/dusty,1.0000000000,82.91.188.47,"{""location"": ""PH"", ""is_mobile"": true}" 3272,5,300,2017-06-13 02:12:47,http://considine.net/marshall_kozey,0.0000000000,189.114.137.118,"{""location"": ""AR"", ""is_mobile"": true}" 3273,5,300,2017-01-08 20:26:27,http://mcdermott.com/lela_luettgen,0.0000000000,118.174.113.248,"{""location"": ""CH"", ""is_mobile"": true}" 3274,5,300,2017-02-09 21:47:16,http://thompson.com/paxton.rowe,0.0000000000,4.114.127.112,"{""location"": ""CY"", ""is_mobile"": false}" 3275,5,300,2017-03-08 10:45:30,http://bernier.net/blair.doyle,2.0000000000,111.114.141.80,"{""location"": ""GG"", ""is_mobile"": true}" 3276,5,300,2017-05-15 06:30:29,http://ullrich.info/rebekah_oreilly,0.0000000000,216.17.47.157,"{""location"": ""DO"", ""is_mobile"": true}" 3277,5,300,2017-04-20 02:00:25,http://robel.io/marianna_donnelly,2.0000000000,101.55.142.175,"{""location"": ""GU"", ""is_mobile"": true}" 3278,5,300,2017-04-20 09:53:32,http://stoltenbergheidenreich.net/velma,2.0000000000,245.179.16.9,"{""location"": ""CZ"", ""is_mobile"": true}" 3279,5,300,2017-06-12 06:58:30,http://stamm.biz/melya,1.0000000000,4.114.127.112,"{""location"": ""FK"", ""is_mobile"": true}" 3280,5,300,2017-06-03 19:14:29,http://koelpinbergnaum.co/hubert,1.0000000000,146.216.178.144,"{""location"": ""AI"", ""is_mobile"": true}" 3281,5,300,2017-01-17 23:25:05,http://hammes.org/beverly_beier,1.0000000000,183.174.159.65,"{""location"": ""UG"", ""is_mobile"": false}" 3282,5,300,2017-03-28 05:28:25,http://parisianmayert.org/valentina,2.0000000000,93.123.216.46,"{""location"": ""GI"", ""is_mobile"": true}" 3283,5,300,2017-01-24 23:16:05,http://konopelskistanton.info/cary.cummerata,1.0000000000,118.174.113.248,"{""location"": ""GS"", ""is_mobile"": true}" 3284,5,300,2017-02-08 04:55:07,http://becker.org/bette_dickens,1.0000000000,245.179.16.9,"{""location"": ""AO"", ""is_mobile"": true}" 3285,5,301,2017-05-10 21:53:57,http://goldner.org/gianni,0.0000000000,73.88.183.19,"{""location"": ""VU"", ""is_mobile"": false}" 3286,5,301,2017-05-06 12:48:47,http://koelpingrant.net/keagan.schuppe,0.0000000000,31.94.30.11,"{""location"": ""LT"", ""is_mobile"": false}" 3287,5,301,2017-01-23 10:47:39,http://larkinschuster.biz/alize,0.0000000000,73.88.183.19,"{""location"": ""JM"", ""is_mobile"": true}" 3288,5,301,2017-01-09 16:23:44,http://monahan.org/keeley.cole,0.0000000000,67.161.31.249,"{""location"": ""GM"", ""is_mobile"": true}" 3289,5,301,2017-01-19 09:26:51,http://veum.net/dolly.moen,0.0000000000,24.83.53.102,"{""location"": ""AU"", ""is_mobile"": false}" 3290,5,302,2017-03-10 14:57:21,http://jakubowski.com/sadye_homenick,0.0000000000,68.156.189.107,"{""location"": ""BB"", ""is_mobile"": false}" 3291,5,302,2017-03-07 00:19:31,http://streichwilkinson.com/solon_dickens,3.0000000000,106.189.146.206,"{""location"": ""GH"", ""is_mobile"": true}" 3292,5,302,2017-02-23 10:48:52,http://armstrong.io/magnus,1.0000000000,161.120.20.37,"{""location"": ""CO"", ""is_mobile"": false}" 3293,5,302,2017-02-17 01:57:32,http://schuppe.com/isidro.sporer,1.0000000000,225.32.204.191,"{""location"": ""TF"", ""is_mobile"": false}" 3294,5,303,2017-04-06 07:40:25,http://nitzsche.name/forest_dietrich,1.0000000000,142.39.246.161,"{""location"": ""VU"", ""is_mobile"": true}" 3295,5,303,2017-04-10 10:21:04,http://kodooley.org/vinnie,1.0000000000,145.2.63.170,"{""location"": ""AS"", ""is_mobile"": true}" 3296,5,304,2017-05-28 14:29:50,http://ankundingcorkery.net/dwight_schaden,0.0000000000,101.153.214.157,"{""location"": ""TN"", ""is_mobile"": false}" 3297,5,304,2017-06-12 16:51:29,http://rohanwalsh.net/mina,0.0000000000,188.65.162.58,"{""location"": ""SS"", ""is_mobile"": true}" 3298,5,304,2017-02-17 16:07:42,http://schummschultz.io/susana,0.0000000000,64.61.241.81,"{""location"": ""PT"", ""is_mobile"": false}" 3299,5,304,2017-06-12 00:30:44,http://nader.co/jany,0.0000000000,11.9.116.213,"{""location"": ""MN"", ""is_mobile"": false}" 3300,5,304,2017-01-05 12:21:04,http://halvorsonkohler.com/pedro,0.0000000000,64.61.241.81,"{""location"": ""NZ"", ""is_mobile"": false}" 3302,5,304,2017-06-10 11:32:41,http://metz.info/daphne,0.0000000000,142.156.185.218,"{""location"": ""BI"", ""is_mobile"": false}" 3303,5,304,2016-12-16 02:20:26,http://bashirian.net/scot.hoppe,0.0000000000,11.9.116.213,"{""location"": ""TN"", ""is_mobile"": true}" 3304,5,304,2017-03-07 08:14:42,http://smitham.name/dallas,0.0000000000,184.202.245.135,"{""location"": ""TW"", ""is_mobile"": true}" 3305,5,304,2017-05-28 21:44:19,http://kutch.net/lindsey,0.0000000000,188.65.162.58,"{""location"": ""GF"", ""is_mobile"": true}" 3306,5,304,2017-04-17 13:00:34,http://daugherty.biz/cristian.stroman,0.0000000000,34.230.78.35,"{""location"": ""AU"", ""is_mobile"": false}" 3307,5,304,2016-12-16 08:37:14,http://littelvandervort.io/glen.nicolas,0.0000000000,218.32.46.141,"{""location"": ""CK"", ""is_mobile"": true}" 3308,5,304,2017-03-19 00:03:04,http://effertz.io/sean,0.0000000000,64.61.241.81,"{""location"": ""AZ"", ""is_mobile"": true}" 3309,5,304,2017-04-01 00:59:36,http://brakus.org/odell_reichert,0.0000000000,64.61.241.81,"{""location"": ""LV"", ""is_mobile"": true}" 3310,5,304,2017-04-08 22:05:19,http://jakubowski.com/noemi,0.0000000000,101.153.214.157,"{""location"": ""TW"", ""is_mobile"": true}" 3311,5,304,2017-01-03 18:44:10,http://hagenesbailey.info/aimee,0.0000000000,246.191.60.228,"{""location"": ""CF"", ""is_mobile"": false}" 3312,5,304,2017-05-11 19:10:45,http://pollich.io/corrine,0.0000000000,237.137.80.136,"{""location"": ""CI"", ""is_mobile"": false}" 3313,5,304,2017-05-25 13:19:58,http://jacobchmitt.co/tillman,0.0000000000,213.80.60.137,"{""location"": ""NR"", ""is_mobile"": false}" 3314,5,304,2017-01-20 15:40:53,http://heaneycollier.info/willa_larkin,0.0000000000,184.202.245.135,"{""location"": ""SX"", ""is_mobile"": true}" 3315,5,304,2017-01-29 16:09:35,http://towne.name/shad_yundt,0.0000000000,101.162.148.197,"{""location"": ""AU"", ""is_mobile"": true}" 3316,5,305,2017-05-12 17:26:54,http://feest.co/caitlyn_aufderhar,0.0000000000,47.233.43.59,"{""location"": ""NP"", ""is_mobile"": false}" 3317,5,305,2017-06-04 07:45:05,http://oberbrunnerschaden.io/allison,0.0000000000,104.191.162.248,"{""location"": ""MF"", ""is_mobile"": false}" 3318,5,305,2017-03-12 04:34:13,http://kuhn.biz/karolann,0.0000000000,253.136.40.228,"{""location"": ""PF"", ""is_mobile"": false}" 3319,5,305,2017-02-22 08:43:58,http://gibsonthiel.net/malinda_reilly,0.0000000000,149.62.50.236,"{""location"": ""UY"", ""is_mobile"": false}" 3320,5,305,2017-01-10 00:33:21,http://nicolaswuckert.com/heloise,0.0000000000,211.220.134.254,"{""location"": ""MH"", ""is_mobile"": true}" 3321,5,305,2017-04-04 18:31:41,http://schulist.io/tanya_macejkovic,0.0000000000,30.155.10.163,"{""location"": ""CD"", ""is_mobile"": true}" 3322,5,305,2017-05-19 02:54:38,http://kutch.org/kaycee_hettinger,0.0000000000,104.191.162.248,"{""location"": ""DO"", ""is_mobile"": false}" 3323,5,305,2017-03-25 10:10:58,http://mcdermott.biz/barrett_bradtke,0.0000000000,168.162.91.28,"{""location"": ""PT"", ""is_mobile"": false}" 3324,5,305,2017-05-13 12:02:55,http://beahan.com/delores,0.0000000000,93.204.193.142,"{""location"": ""AG"", ""is_mobile"": false}" 3325,5,305,2017-04-13 04:04:03,http://boscomoriette.name/judah,0.0000000000,247.215.201.114,"{""location"": ""TO"", ""is_mobile"": true}" 3326,5,305,2017-03-20 08:43:25,http://altenwerth.info/alejandrin.white,0.0000000000,219.12.25.143,"{""location"": ""ID"", ""is_mobile"": true}" 3327,5,305,2017-05-21 18:03:04,http://schowalter.co/annie.quigley,0.0000000000,30.155.10.163,"{""location"": ""BF"", ""is_mobile"": true}" 3328,5,305,2017-04-03 23:54:28,http://sporer.io/jovany,0.0000000000,43.97.73.171,"{""location"": ""KH"", ""is_mobile"": true}" 3329,5,305,2017-01-17 23:03:46,http://stehrhaley.info/reinhold,0.0000000000,160.136.222.206,"{""location"": ""BL"", ""is_mobile"": true}" 3330,5,305,2017-05-22 17:41:18,http://fritsch.net/dalton_wiegand,0.0000000000,149.62.50.236,"{""location"": ""NR"", ""is_mobile"": true}" 3331,5,305,2017-03-10 04:52:21,http://mayert.com/destini,0.0000000000,211.220.134.254,"{""location"": ""TR"", ""is_mobile"": true}" 3332,5,305,2017-01-19 01:23:13,http://schiller.info/maymie,0.0000000000,158.15.137.153,"{""location"": ""FI"", ""is_mobile"": true}" 3333,5,305,2017-05-31 22:08:06,http://kuhlmanlittle.org/jedidiah.jast,0.0000000000,247.215.201.114,"{""location"": ""TZ"", ""is_mobile"": true}" 3334,5,305,2017-02-02 17:04:44,http://torpkiehn.biz/catherine.miller,0.0000000000,208.40.239.144,"{""location"": ""CW"", ""is_mobile"": true}" 3335,5,306,2017-05-13 21:29:16,http://osinski.name/ruthe.cronin,2.0000000000,174.164.91.183,"{""location"": ""KI"", ""is_mobile"": true}" 3336,5,306,2017-02-05 17:27:09,http://smithamgoyette.org/van,1.0000000000,228.240.120.86,"{""location"": ""NC"", ""is_mobile"": true}" 3337,5,306,2017-01-16 09:56:14,http://torphybartoletti.org/daisy,0.0000000000,25.146.250.101,"{""location"": ""IT"", ""is_mobile"": false}" 3338,5,306,2017-03-08 10:29:31,http://swiftwaelchi.name/elsa,0.0000000000,174.164.91.183,"{""location"": ""IS"", ""is_mobile"": true}" 3339,5,306,2017-02-27 17:46:01,http://wuckert.io/kayla,0.0000000000,181.46.209.17,"{""location"": ""TH"", ""is_mobile"": true}" 3340,5,306,2017-03-07 04:44:32,http://williamson.name/xander,1.0000000000,40.24.36.133,"{""location"": ""NF"", ""is_mobile"": true}" 3341,5,306,2017-05-24 09:43:19,http://kunze.com/stanley_upton,0.0000000000,109.187.30.240,"{""location"": ""BZ"", ""is_mobile"": true}" 3342,5,306,2017-03-06 04:05:35,http://barrowsbednar.com/jennifer,1.0000000000,25.146.250.101,"{""location"": ""UG"", ""is_mobile"": true}" 3343,5,306,2017-03-04 21:53:56,http://kuhn.org/merle_stanton,2.0000000000,127.95.203.24,"{""location"": ""UZ"", ""is_mobile"": true}" 3344,5,306,2017-06-12 07:09:14,http://walsh.net/margaretta,0.0000000000,2.211.150.59,"{""location"": ""GF"", ""is_mobile"": true}" 3345,5,306,2017-02-08 19:37:40,http://erdman.info/maude,2.0000000000,104.242.137.169,"{""location"": ""MX"", ""is_mobile"": true}" 3346,5,306,2017-04-15 11:59:21,http://kulasborer.name/keon,2.0000000000,130.213.115.198,"{""location"": ""TM"", ""is_mobile"": false}" 3347,5,306,2017-02-07 05:41:59,http://pollichledner.name/milo,0.0000000000,181.46.209.17,"{""location"": ""OM"", ""is_mobile"": true}" 3348,5,306,2017-05-23 17:12:18,http://gulgowski.name/marguerite,2.0000000000,164.139.252.151,"{""location"": ""PH"", ""is_mobile"": false}" 3349,5,306,2016-12-17 22:08:27,http://funkmcglynn.com/zachariah,1.0000000000,2.211.150.59,"{""location"": ""CW"", ""is_mobile"": true}" 3350,5,306,2017-04-17 06:33:24,http://zemlak.info/therese,1.0000000000,2.211.150.59,"{""location"": ""WF"", ""is_mobile"": false}" 3351,5,306,2016-12-24 12:46:31,http://huelmoen.io/harvey.farrell,1.0000000000,181.46.209.17,"{""location"": ""GM"", ""is_mobile"": true}" 3352,5,307,2017-06-13 06:48:03,http://moriettestroman.biz/nichole,0.0000000000,113.17.225.153,"{""location"": ""TL"", ""is_mobile"": false}" 3353,5,307,2017-01-04 08:49:50,http://donnelly.info/kristina.bayer,0.0000000000,93.89.193.243,"{""location"": ""AU"", ""is_mobile"": false}" 3354,5,307,2017-02-18 16:56:37,http://stoltenberg.com/bernard.gottlieb,0.0000000000,72.46.29.169,"{""location"": ""VG"", ""is_mobile"": false}" 3355,5,307,2017-02-26 17:36:34,http://thiel.net/rickey_goyette,0.0000000000,134.49.93.203,"{""location"": ""CC"", ""is_mobile"": false}" 3356,5,307,2017-01-15 03:57:15,http://kochullrich.com/jayson.brakus,0.0000000000,155.66.94.57,"{""location"": ""CM"", ""is_mobile"": true}" 3357,5,307,2017-02-05 18:53:31,http://schuster.com/joshua,0.0000000000,107.93.173.70,"{""location"": ""AS"", ""is_mobile"": true}" 3358,5,307,2017-01-20 12:27:10,http://ritchie.co/delaney.gorczany,0.0000000000,247.19.75.205,"{""location"": ""SS"", ""is_mobile"": false}" 3359,5,307,2017-01-16 09:15:33,http://mayert.com/simone,0.0000000000,22.102.127.253,"{""location"": ""CU"", ""is_mobile"": true}" 3360,5,307,2017-05-28 09:44:14,http://muller.biz/jacey.johns,0.0000000000,133.77.147.34,"{""location"": ""GH"", ""is_mobile"": false}" 3361,5,307,2017-03-21 18:40:48,http://jakubowski.name/sister,0.0000000000,110.5.192.241,"{""location"": ""AR"", ""is_mobile"": true}" 3362,5,307,2017-01-09 05:31:10,http://schuppe.biz/gertrude_bruen,0.0000000000,45.25.183.83,"{""location"": ""MT"", ""is_mobile"": true}" 3363,5,307,2017-06-03 23:54:37,http://howell.info/rusty,0.0000000000,134.49.93.203,"{""location"": ""PK"", ""is_mobile"": false}" 3364,5,307,2017-02-06 08:16:16,http://kohler.com/lupe.roberts,0.0000000000,159.240.113.78,"{""location"": ""CR"", ""is_mobile"": false}" 3365,5,307,2016-12-22 13:32:48,http://schinner.io/ryann,0.0000000000,232.185.47.39,"{""location"": ""KR"", ""is_mobile"": false}" 3366,5,307,2017-05-30 21:20:01,http://schumm.biz/lilian_nienow,0.0000000000,134.49.93.203,"{""location"": ""QA"", ""is_mobile"": false}" 3367,5,307,2017-03-25 07:40:31,http://schaefer.name/harley,0.0000000000,114.141.228.158,"{""location"": ""LB"", ""is_mobile"": true}" 3368,5,307,2017-01-04 12:30:56,http://stoltenberghickle.name/daphnee_mayer,0.0000000000,36.56.188.201,"{""location"": ""GM"", ""is_mobile"": true}" 3369,5,308,2017-05-14 22:34:34,http://wolf.co/chaim,0.0000000000,148.24.51.124,"{""location"": ""LC"", ""is_mobile"": false}" 3370,5,308,2017-05-08 12:52:28,http://lang.org/doris_farrell,0.0000000000,254.105.44.38,"{""location"": ""NE"", ""is_mobile"": true}" 3371,5,308,2017-03-01 21:35:43,http://bashirian.org/deron,0.0000000000,223.79.155.232,"{""location"": ""BG"", ""is_mobile"": true}" 3372,5,308,2017-06-08 02:48:22,http://reichertcain.info/cordelia,0.0000000000,152.202.49.34,"{""location"": ""JP"", ""is_mobile"": false}" 3373,5,308,2017-03-20 08:52:09,http://renner.co/eleazar.okeefe,0.0000000000,158.176.35.222,"{""location"": ""GI"", ""is_mobile"": true}" 3374,5,308,2017-06-05 19:38:24,http://medhurstgreen.io/montana,0.0000000000,227.248.110.53,"{""location"": ""TL"", ""is_mobile"": true}" 3375,5,308,2017-04-30 17:49:24,http://okunevagutmann.org/colt,0.0000000000,206.141.43.196,"{""location"": ""GS"", ""is_mobile"": false}" 3376,5,308,2016-12-24 16:24:18,http://beierlesch.co/elvie,0.0000000000,219.90.113.55,"{""location"": ""SL"", ""is_mobile"": true}" 3377,5,308,2017-05-21 05:26:20,http://whiteschimmel.info/dewitt.greenholt,0.0000000000,204.204.35.248,"{""location"": ""HT"", ""is_mobile"": false}" 3378,5,308,2017-02-09 02:13:24,http://kirlin.io/dasia,0.0000000000,225.100.252.128,"{""location"": ""NC"", ""is_mobile"": false}" 3379,5,308,2017-02-03 08:25:35,http://bergstrom.name/maudie.kirlin,0.0000000000,29.227.13.109,"{""location"": ""CC"", ""is_mobile"": true}" 3380,5,308,2017-03-23 14:14:29,http://tromp.name/jadon_schultz,0.0000000000,220.114.164.237,"{""location"": ""HR"", ""is_mobile"": true}" 3381,5,308,2017-04-16 05:59:36,http://schultz.co/sophie,0.0000000000,180.83.128.230,"{""location"": ""PL"", ""is_mobile"": true}" 3382,5,309,2017-03-30 08:14:46,http://olsonjast.org/nannie.oreilly,0.0000000000,7.2.97.100,"{""location"": ""KI"", ""is_mobile"": true}" 3383,5,310,2016-12-20 07:47:57,http://mullergrant.com/jayda,0.0000000000,227.213.134.43,"{""location"": ""MG"", ""is_mobile"": false}" 3384,5,310,2017-05-24 21:33:43,http://gloverskiles.org/ciara,0.0000000000,11.161.42.201,"{""location"": ""RW"", ""is_mobile"": false}" 3385,5,310,2016-12-20 03:11:11,http://treutel.name/nolan_gerlach,0.0000000000,136.22.110.68,"{""location"": ""CG"", ""is_mobile"": false}" 3386,5,310,2017-04-13 13:00:34,http://muellerbailey.io/jimmie_kuphal,0.0000000000,227.101.43.131,"{""location"": ""GN"", ""is_mobile"": false}" 3387,5,310,2017-04-02 08:10:54,http://bartellkonopelski.name/rogers,0.0000000000,45.31.8.15,"{""location"": ""BO"", ""is_mobile"": false}" 3388,5,310,2017-03-28 17:19:50,http://krajcikko.io/nick.lemke,0.0000000000,227.101.43.131,"{""location"": ""IT"", ""is_mobile"": false}" 3389,5,310,2017-03-03 14:48:32,http://dubuque.com/alphonso,0.0000000000,42.155.137.230,"{""location"": ""CW"", ""is_mobile"": true}" 3390,5,310,2017-05-01 09:51:37,http://dietrichswift.net/margie,0.0000000000,141.142.252.93,"{""location"": ""OM"", ""is_mobile"": false}" 3391,5,310,2016-12-28 00:04:07,http://schamberger.info/penelope,0.0000000000,52.169.55.212,"{""location"": ""BY"", ""is_mobile"": true}" 3392,5,310,2017-03-22 17:57:35,http://fadel.org/shemar_feil,0.0000000000,243.236.167.116,"{""location"": ""HN"", ""is_mobile"": true}" 3393,5,310,2017-04-17 05:40:16,http://weinatbode.com/parker_walsh,0.0000000000,185.228.193.157,"{""location"": ""BI"", ""is_mobile"": false}" 3394,5,310,2017-03-28 15:15:11,http://beckerokeefe.co/mariana,0.0000000000,99.106.25.162,"{""location"": ""IO"", ""is_mobile"": true}" 3395,5,310,2017-01-11 10:35:02,http://stoltenbergboyer.com/kaylah.mosciski,0.0000000000,45.56.150.52,"{""location"": ""MD"", ""is_mobile"": true}" 3396,5,310,2017-01-09 20:57:53,http://hettinger.net/tommie_zulauf,0.0000000000,91.5.242.158,"{""location"": ""UA"", ""is_mobile"": true}" 3397,5,310,2017-05-26 20:04:28,http://pagac.info/rachelle.schimmel,0.0000000000,227.101.43.131,"{""location"": ""ML"", ""is_mobile"": false}" 3398,5,310,2017-06-06 18:13:12,http://treutellesch.com/gaylord.green,0.0000000000,99.106.25.162,"{""location"": ""PM"", ""is_mobile"": false}" 3399,5,311,2017-01-20 22:01:51,http://barton.org/nettie_collier,0.0000000000,210.198.231.42,"{""location"": ""IO"", ""is_mobile"": false}" 3400,5,311,2017-05-21 00:41:27,http://abbott.name/georgianna,0.0000000000,74.214.64.66,"{""location"": ""GQ"", ""is_mobile"": false}" 3401,5,311,2017-03-19 22:25:41,http://dietrich.io/laurie,0.0000000000,150.65.136.97,"{""location"": ""UZ"", ""is_mobile"": true}" 3402,5,312,2017-05-22 09:30:27,http://schamberger.info/terrell,0.0000000000,215.53.222.204,"{""location"": ""KZ"", ""is_mobile"": false}" 3403,5,312,2017-01-11 00:41:28,http://rippin.co/clovis.legros,0.0000000000,159.189.66.122,"{""location"": ""NO"", ""is_mobile"": false}" 3404,5,312,2017-03-13 20:51:57,http://jones.net/shad_reichel,0.0000000000,148.117.17.109,"{""location"": ""PT"", ""is_mobile"": true}" 3405,5,312,2017-01-13 05:08:15,http://dach.org/haie.nolan,0.0000000000,201.116.228.245,"{""location"": ""GT"", ""is_mobile"": false}" 3406,5,312,2017-02-08 03:56:32,http://cummerata.co/penelope_stark,0.0000000000,5.84.89.45,"{""location"": ""NF"", ""is_mobile"": true}" 3407,5,312,2017-03-12 06:04:00,http://hamill.io/edythe_jacobs,0.0000000000,232.87.160.149,"{""location"": ""HU"", ""is_mobile"": true}" 3408,5,312,2017-01-19 01:40:09,http://heidenreichbahringer.info/gilbert,0.0000000000,167.164.54.165,"{""location"": ""KI"", ""is_mobile"": true}" 3409,5,312,2017-06-13 09:48:25,http://bashirianritchie.org/allene,0.0000000000,180.141.207.73,"{""location"": ""CK"", ""is_mobile"": false}" 3410,5,312,2017-03-28 07:05:22,http://feest.info/kenyon,0.0000000000,24.228.214.17,"{""location"": ""VE"", ""is_mobile"": false}" 3411,5,313,2017-06-04 23:11:02,http://morar.co/dejah.swift,1.0000000000,251.148.250.17,"{""location"": ""KM"", ""is_mobile"": false}" 3412,5,313,2017-02-02 20:53:54,http://oconnell.biz/marques,1.0000000000,114.126.99.3,"{""location"": ""GL"", ""is_mobile"": true}" 3413,5,313,2017-04-09 05:32:43,http://lockman.org/concepcion_leannon,1.0000000000,89.21.221.233,"{""location"": ""NE"", ""is_mobile"": true}" 3414,5,313,2017-04-11 07:39:47,http://thompson.net/broderick.hoppe,0.0000000000,153.207.2.207,"{""location"": ""BG"", ""is_mobile"": false}" 3415,5,313,2017-05-06 06:04:50,http://beier.org/nadia,0.0000000000,120.99.143.40,"{""location"": ""NP"", ""is_mobile"": true}" 3416,5,313,2017-03-26 18:08:36,http://strosincrona.com/raven,1.0000000000,11.215.86.248,"{""location"": ""JO"", ""is_mobile"": false}" 3417,5,313,2017-02-23 08:46:16,http://yundt.com/lenora_ferry,1.0000000000,7.46.251.158,"{""location"": ""BA"", ""is_mobile"": true}" 3418,5,313,2017-01-14 03:03:24,http://jaskolski.biz/graham.walker,0.0000000000,151.236.105.104,"{""location"": ""BF"", ""is_mobile"": false}" 3419,5,313,2017-03-14 17:55:58,http://christiansensteuber.info/florida,0.0000000000,168.248.197.4,"{""location"": ""AI"", ""is_mobile"": false}" 3420,5,313,2017-01-20 22:53:54,http://keeling.name/matilde.rau,1.0000000000,83.74.88.254,"{""location"": ""KP"", ""is_mobile"": true}" 3421,5,313,2017-02-20 19:15:23,http://schroeder.io/terrence,0.0000000000,165.112.226.194,"{""location"": ""LB"", ""is_mobile"": false}" 3422,5,313,2017-01-09 06:13:19,http://medhurstdonnelly.name/merl_waelchi,1.0000000000,198.151.165.211,"{""location"": ""AL"", ""is_mobile"": false}" 3423,5,313,2017-02-07 18:10:36,http://rosenbaum.co/davin,1.0000000000,124.211.252.187,"{""location"": ""HK"", ""is_mobile"": true}" 3424,5,313,2017-03-01 18:36:20,http://powlowskigoldner.com/cindy.jacobi,0.0000000000,89.21.221.233,"{""location"": ""BQ"", ""is_mobile"": false}" 3425,5,313,2017-03-26 01:23:08,http://pagac.org/angelina.roberts,0.0000000000,91.101.65.54,"{""location"": ""NI"", ""is_mobile"": true}" 3426,5,313,2017-04-12 19:40:41,http://kerlukerippin.io/deja,1.0000000000,9.188.200.241,"{""location"": ""NE"", ""is_mobile"": false}" 3427,5,313,2017-04-06 08:36:48,http://lowe.co/coby.balistreri,1.0000000000,174.62.79.230,"{""location"": ""BQ"", ""is_mobile"": true}" 3428,5,313,2017-01-27 03:31:37,http://thiel.net/quinten_willms,0.0000000000,214.108.223.144,"{""location"": ""DE"", ""is_mobile"": false}" 3429,5,313,2017-04-27 13:35:56,http://stoltenberg.io/michele.romaguera,0.0000000000,153.207.2.207,"{""location"": ""AI"", ""is_mobile"": true}" 3430,5,314,2016-12-21 22:34:18,http://botsford.com/duane.turcotte,1.0000000000,190.47.176.97,"{""location"": ""BH"", ""is_mobile"": true}" 3431,5,314,2016-12-18 01:32:05,http://kuphal.info/edwin,1.0000000000,8.20.6.148,"{""location"": ""IS"", ""is_mobile"": true}" 3432,5,314,2017-03-04 12:51:45,http://hegmannhahn.com/halle,1.0000000000,72.150.130.150,"{""location"": ""US"", ""is_mobile"": true}" 3433,5,314,2017-03-27 18:00:39,http://schmeler.co/walker.becker,2.0000000000,13.108.251.64,"{""location"": ""AR"", ""is_mobile"": false}" 3434,5,314,2017-02-11 11:28:30,http://hilpert.biz/narciso,0.0000000000,13.108.251.64,"{""location"": ""AE"", ""is_mobile"": false}" 3435,5,314,2017-02-27 23:30:09,http://hilll.io/naomie,1.0000000000,13.108.251.64,"{""location"": ""AW"", ""is_mobile"": true}" 3436,5,314,2017-01-09 15:45:49,http://millsblanda.com/kade,3.0000000000,147.218.59.65,"{""location"": ""NL"", ""is_mobile"": false}" 3437,5,314,2017-03-21 08:45:03,http://weimannbruen.info/rashad,3.0000000000,139.60.201.217,"{""location"": ""SZ"", ""is_mobile"": true}" 3438,5,314,2017-03-15 20:44:55,http://kovacekkuhlman.org/romaine,2.0000000000,136.213.27.58,"{""location"": ""TO"", ""is_mobile"": false}" 3439,5,314,2017-06-03 22:47:25,http://ornupton.com/zora_marvin,3.0000000000,59.107.153.121,"{""location"": ""SK"", ""is_mobile"": true}" 3440,5,314,2017-05-29 07:00:22,http://schultzrobel.biz/lia_hagenes,2.0000000000,249.167.83.42,"{""location"": ""YE"", ""is_mobile"": true}" 3441,5,314,2017-02-24 04:54:00,http://beckerlakin.info/bret,1.0000000000,239.19.132.245,"{""location"": ""LK"", ""is_mobile"": true}" 3442,5,314,2017-05-16 04:43:06,http://tromp.net/malinda.kuhlman,0.0000000000,166.14.40.136,"{""location"": ""KN"", ""is_mobile"": true}" 3443,5,314,2017-05-29 08:34:22,http://cummerataweber.info/ola,1.0000000000,101.217.133.204,"{""location"": ""PL"", ""is_mobile"": false}" 3444,5,314,2017-05-21 13:53:31,http://murray.name/danial,3.0000000000,41.176.249.59,"{""location"": ""MG"", ""is_mobile"": false}" 3445,5,314,2017-02-27 06:17:24,http://schaefer.info/clint,2.0000000000,8.20.6.148,"{""location"": ""KI"", ""is_mobile"": true}" 3446,5,314,2017-05-12 05:24:08,http://christiansenfeil.org/pascale,3.0000000000,121.115.171.224,"{""location"": ""PE"", ""is_mobile"": false}" 3447,5,315,2017-03-30 21:07:23,http://sipeszieme.io/pedro_koepp,0.0000000000,230.254.10.203,"{""location"": ""US"", ""is_mobile"": true}" 3448,5,315,2017-02-12 19:45:32,http://hudsondamore.info/lilyan,3.0000000000,96.43.98.162,"{""location"": ""WF"", ""is_mobile"": true}" 3449,5,315,2017-06-01 09:11:26,http://marquardtlowe.io/elsie,3.0000000000,197.45.140.235,"{""location"": ""PR"", ""is_mobile"": false}" 3450,5,315,2017-03-18 20:22:11,http://homenick.biz/nikolas_champlin,3.0000000000,185.135.147.33,"{""location"": ""LB"", ""is_mobile"": true}" 3451,5,315,2017-02-25 21:42:22,http://adams.co/lennie_moen,1.0000000000,226.163.156.168,"{""location"": ""UG"", ""is_mobile"": false}" 3452,5,315,2017-04-18 14:08:26,http://torphy.org/shany,3.0000000000,242.20.250.229,"{""location"": ""TN"", ""is_mobile"": true}" 3453,5,315,2017-01-10 15:09:34,http://turcotteherman.biz/cedrick,3.0000000000,63.9.169.102,"{""location"": ""MO"", ""is_mobile"": true}" 3454,5,315,2017-06-05 03:25:12,http://cronin.name/malika,3.0000000000,172.119.40.158,"{""location"": ""MC"", ""is_mobile"": true}" 3455,5,315,2017-04-09 02:08:36,http://bergstrom.info/burley,3.0000000000,226.163.156.168,"{""location"": ""ME"", ""is_mobile"": false}" 3456,5,315,2017-05-27 13:11:03,http://goyettemoen.io/nicholas_connelly,3.0000000000,110.68.18.230,"{""location"": ""AS"", ""is_mobile"": true}" 3457,5,315,2017-01-08 11:01:51,http://jaskolski.info/randi_king,1.0000000000,235.28.86.7,"{""location"": ""LI"", ""is_mobile"": true}" 3458,5,315,2017-01-14 23:13:18,http://lowe.info/oceane.adams,0.0000000000,235.28.86.7,"{""location"": ""SS"", ""is_mobile"": true}" 3459,5,315,2017-03-28 12:01:23,http://hamill.net/stefan_miller,0.0000000000,54.142.12.188,"{""location"": ""GS"", ""is_mobile"": true}" 3460,5,316,2017-05-25 09:09:02,http://rohan.net/elia,0.0000000000,36.122.153.102,"{""location"": ""AE"", ""is_mobile"": false}" 3461,5,316,2017-05-23 13:58:28,http://bosco.net/guy_klein,0.0000000000,218.145.100.126,"{""location"": ""BH"", ""is_mobile"": false}" 3462,5,316,2017-05-14 09:11:10,http://hageneswindler.com/jevon.heel,0.0000000000,197.45.159.235,"{""location"": ""ES"", ""is_mobile"": false}" 3463,5,316,2017-02-20 09:09:24,http://mcglynn.net/destin_nikolaus,0.0000000000,213.91.174.54,"{""location"": ""VU"", ""is_mobile"": false}" 3464,5,316,2017-04-17 06:49:07,http://howe.co/oral,0.0000000000,70.156.24.65,"{""location"": ""VI"", ""is_mobile"": false}" 3465,5,316,2017-01-08 06:36:35,http://daniel.co/zoe.denesik,1.0000000000,117.105.28.138,"{""location"": ""GS"", ""is_mobile"": true}" 3466,5,316,2017-01-20 11:50:15,http://feil.name/maynard,0.0000000000,238.13.57.49,"{""location"": ""US"", ""is_mobile"": true}" 3467,5,316,2017-04-04 23:12:46,http://gottliebferry.biz/letha_homenick,1.0000000000,212.166.19.135,"{""location"": ""AM"", ""is_mobile"": true}" 3468,5,316,2017-05-03 02:44:26,http://walterrippin.com/sydnee,1.0000000000,32.102.174.241,"{""location"": ""PN"", ""is_mobile"": true}" 3469,5,316,2017-05-11 13:18:12,http://adams.com/ro,0.0000000000,225.76.245.47,"{""location"": ""EC"", ""is_mobile"": true}" 3470,5,316,2017-01-17 01:22:40,http://renner.co/ardith_gerlach,0.0000000000,52.227.91.7,"{""location"": ""JP"", ""is_mobile"": false}" 3471,5,316,2017-04-19 23:51:46,http://considine.com/hope.eichmann,1.0000000000,52.227.91.7,"{""location"": ""PH"", ""is_mobile"": false}" 3472,5,316,2017-01-13 15:19:14,http://morar.io/israel_homenick,1.0000000000,142.30.243.30,"{""location"": ""GQ"", ""is_mobile"": true}" 3473,5,316,2016-12-23 09:11:39,http://conroy.co/retta.gleason,1.0000000000,62.94.180.175,"{""location"": ""RO"", ""is_mobile"": false}" 3474,5,317,2017-03-03 15:44:53,http://funkrau.biz/edna_lakin,2.0000000000,158.211.84.60,"{""location"": ""NC"", ""is_mobile"": false}" 3475,5,317,2017-02-10 08:14:22,http://smithambrown.com/tania_vandervort,0.0000000000,163.152.26.220,"{""location"": ""TN"", ""is_mobile"": false}" 3476,5,317,2016-12-24 03:29:10,http://swaniawski.biz/aidan,3.0000000000,157.197.251.207,"{""location"": ""NF"", ""is_mobile"": true}" 3477,5,317,2017-04-25 13:59:21,http://padberggoodwin.info/emmet,1.0000000000,116.173.103.71,"{""location"": ""ES"", ""is_mobile"": false}" 3478,5,317,2017-01-04 09:31:53,http://ohara.org/manuela_lowe,1.0000000000,167.135.111.144,"{""location"": ""NF"", ""is_mobile"": true}" 3479,5,317,2016-12-31 06:37:43,http://rogahn.com/jacey,1.0000000000,83.97.127.45,"{""location"": ""PY"", ""is_mobile"": false}" 3480,5,317,2017-05-08 23:11:27,http://wilderman.biz/hermina.towne,3.0000000000,97.226.188.198,"{""location"": ""DK"", ""is_mobile"": false}" 3481,5,317,2017-03-07 14:05:15,http://smitham.info/melyna,3.0000000000,167.135.111.144,"{""location"": ""GN"", ""is_mobile"": true}" 3482,5,317,2017-01-30 17:18:30,http://collintamm.com/estella_cummerata,1.0000000000,73.40.107.124,"{""location"": ""SX"", ""is_mobile"": true}" 3483,5,317,2017-01-08 16:46:36,http://macejkovicschimmel.org/angelo_hammes,2.0000000000,163.152.26.220,"{""location"": ""GW"", ""is_mobile"": true}" 3484,5,317,2017-03-16 11:14:30,http://goyetteoreilly.com/dedrick,1.0000000000,97.226.188.198,"{""location"": ""LR"", ""is_mobile"": true}" 3485,5,317,2016-12-25 16:08:11,http://kihn.co/jody.terry,1.0000000000,204.199.164.38,"{""location"": ""FK"", ""is_mobile"": false}" 3486,5,317,2017-04-23 04:21:05,http://shanahan.co/zoila,1.0000000000,194.243.123.33,"{""location"": ""LY"", ""is_mobile"": true}" 3487,5,317,2017-04-21 05:11:56,http://schowalterkshlerin.info/lee.orn,1.0000000000,5.29.64.238,"{""location"": ""SV"", ""is_mobile"": false}" 3488,5,317,2017-01-21 22:21:03,http://aufderhar.org/marquise_johns,0.0000000000,182.39.69.14,"{""location"": ""MC"", ""is_mobile"": false}" 3489,5,318,2017-06-10 05:39:13,http://breitenbergmayer.biz/ricardo_gleichner,1.0000000000,71.96.82.115,"{""location"": ""MR"", ""is_mobile"": false}" 3490,5,318,2017-03-25 05:17:18,http://bruenward.io/rowena_greenfelder,0.0000000000,136.151.76.102,"{""location"": ""AO"", ""is_mobile"": true}" 3491,5,318,2017-02-12 02:09:52,http://jaskolski.io/rebeca,1.0000000000,52.38.161.120,"{""location"": ""US"", ""is_mobile"": true}" 3492,5,318,2017-06-03 09:51:41,http://schumm.co/everette,2.0000000000,51.237.144.179,"{""location"": ""TR"", ""is_mobile"": true}" 3493,5,318,2017-01-19 14:18:56,http://nienow.biz/lucinda,3.0000000000,30.133.76.110,"{""location"": ""BL"", ""is_mobile"": false}" 3494,5,318,2017-01-22 23:27:50,http://quitzon.biz/louvenia.williamson,2.0000000000,10.66.124.168,"{""location"": ""KW"", ""is_mobile"": true}" 3495,5,318,2017-02-28 14:20:12,http://schaden.name/lenore.pfeffer,3.0000000000,230.17.191.75,"{""location"": ""IS"", ""is_mobile"": false}" 3496,5,318,2017-01-03 03:52:17,http://moensatterfield.net/eli_hansen,2.0000000000,103.79.6.118,"{""location"": ""GG"", ""is_mobile"": false}" 3497,5,318,2017-03-13 04:10:08,http://cartwright.io/jakob,1.0000000000,196.222.242.5,"{""location"": ""MS"", ""is_mobile"": false}" 3498,5,318,2017-06-12 01:08:09,http://abshire.org/rudy.predovic,1.0000000000,10.66.124.168,"{""location"": ""GM"", ""is_mobile"": true}" 3499,5,318,2017-02-01 06:27:14,http://oconnerkihn.name/lelah,3.0000000000,204.4.158.143,"{""location"": ""MP"", ""is_mobile"": true}" 3500,5,318,2017-03-29 10:11:59,http://gutmann.co/donato_crist,1.0000000000,10.66.124.168,"{""location"": ""JO"", ""is_mobile"": false}" 3501,5,318,2017-03-12 03:25:51,http://ruel.org/vaughn.corwin,0.0000000000,154.81.253.105,"{""location"": ""AF"", ""is_mobile"": false}" 3502,5,318,2016-12-24 02:10:32,http://christiansen.org/annabel_morar,3.0000000000,72.90.192.14,"{""location"": ""UY"", ""is_mobile"": true}" 3503,5,318,2017-02-01 05:33:57,http://schimmelskiles.io/ola,2.0000000000,32.78.26.117,"{""location"": ""TH"", ""is_mobile"": false}" 3504,5,318,2017-05-14 00:49:45,http://fisher.name/orrin,3.0000000000,103.79.6.118,"{""location"": ""KE"", ""is_mobile"": false}" 3505,5,319,2017-01-30 11:51:43,http://erdman.io/johnathon,2.0000000000,171.108.16.138,"{""location"": ""NU"", ""is_mobile"": true}" 3506,5,319,2017-03-25 07:07:47,http://zieme.io/warren,2.0000000000,95.148.123.98,"{""location"": ""LR"", ""is_mobile"": true}" 3507,5,319,2017-01-06 04:20:03,http://friesenlockman.io/sedrick,0.0000000000,222.46.137.219,"{""location"": ""DO"", ""is_mobile"": false}" 3508,5,319,2017-04-16 14:26:25,http://oconnell.biz/sofia,0.0000000000,192.159.12.61,"{""location"": ""HT"", ""is_mobile"": false}" 3509,5,319,2016-12-30 18:44:44,http://okeefe.biz/idella_leannon,1.0000000000,205.231.253.17,"{""location"": ""KP"", ""is_mobile"": true}" 3510,5,319,2017-02-08 12:14:07,http://willmccullough.net/emelia.leffler,0.0000000000,97.247.97.40,"{""location"": ""CC"", ""is_mobile"": false}" 3511,5,319,2017-04-20 17:19:24,http://ankundingkuhlman.biz/hilbert,1.0000000000,171.108.16.138,"{""location"": ""SI"", ""is_mobile"": false}" 3512,5,319,2017-03-19 05:01:43,http://lubowitz.info/jacynthe_schulist,2.0000000000,205.231.253.17,"{""location"": ""MH"", ""is_mobile"": false}" 3513,5,320,2017-05-03 20:45:16,http://doyle.com/omer,2.0000000000,25.52.234.72,"{""location"": ""NL"", ""is_mobile"": true}" 3514,5,320,2017-06-02 04:27:21,http://okunevaziemann.info/ned,0.0000000000,47.241.221.45,"{""location"": ""IE"", ""is_mobile"": true}" 3515,5,320,2017-05-03 15:41:51,http://krajcik.biz/violet_tremblay,2.0000000000,45.126.231.7,"{""location"": ""TJ"", ""is_mobile"": false}" 3516,5,320,2016-12-18 08:24:14,http://parisianryan.io/domingo.mckenzie,1.0000000000,151.159.73.50,"{""location"": ""HU"", ""is_mobile"": true}" 3517,5,320,2017-04-13 04:13:06,http://jacobi.co/kristian,2.0000000000,17.144.98.113,"{""location"": ""HK"", ""is_mobile"": true}" 3518,5,320,2017-05-23 15:07:52,http://lueilwitzwunsch.io/jefferey.mueller,2.0000000000,47.241.221.45,"{""location"": ""VG"", ""is_mobile"": true}" 3519,5,320,2017-05-17 19:18:06,http://herzog.io/domenick_wunsch,2.0000000000,116.59.242.226,"{""location"": ""AE"", ""is_mobile"": true}" 3520,5,320,2017-06-06 16:54:55,http://schuppe.org/kadin,2.0000000000,200.156.216.27,"{""location"": ""MY"", ""is_mobile"": false}" 3521,5,320,2017-01-26 06:38:07,http://funk.io/peyton,0.0000000000,8.91.110.94,"{""location"": ""SJ"", ""is_mobile"": false}" 3522,5,320,2017-03-09 20:17:20,http://herzog.biz/javier,0.0000000000,240.6.157.11,"{""location"": ""MW"", ""is_mobile"": false}" 3523,5,320,2017-04-02 06:07:38,http://harris.name/reymundo,0.0000000000,89.11.248.232,"{""location"": ""FK"", ""is_mobile"": true}" 3524,5,320,2017-01-03 23:18:42,http://coleschulist.com/eugene,2.0000000000,28.16.17.127,"{""location"": ""IS"", ""is_mobile"": false}" 3525,5,320,2017-01-09 07:07:27,http://feil.com/pink,0.0000000000,28.16.17.127,"{""location"": ""SZ"", ""is_mobile"": false}" 3526,5,320,2017-03-30 05:47:51,http://ohara.info/moriah.mayer,2.0000000000,35.56.243.154,"{""location"": ""UG"", ""is_mobile"": false}" 3527,5,320,2017-04-08 15:25:21,http://kihn.io/jonatan,2.0000000000,121.153.172.240,"{""location"": ""CF"", ""is_mobile"": false}" 3528,5,320,2017-01-23 18:49:08,http://haley.name/demetrius_sporer,1.0000000000,75.194.196.139,"{""location"": ""PM"", ""is_mobile"": false}" 3529,5,320,2016-12-17 23:52:47,http://schumm.net/bridgette,2.0000000000,125.37.235.183,"{""location"": ""IT"", ""is_mobile"": false}" 3530,5,320,2017-02-26 00:26:46,http://kemmer.biz/gustave.turner,1.0000000000,246.243.181.204,"{""location"": ""KE"", ""is_mobile"": true}" 3531,5,321,2017-05-01 03:56:41,http://thiel.info/emanuel_price,1.0000000000,191.24.184.80,"{""location"": ""MZ"", ""is_mobile"": true}" 3532,5,321,2017-01-28 14:59:17,http://hamill.org/fay_stanton,0.0000000000,105.156.91.206,"{""location"": ""SC"", ""is_mobile"": false}" 3533,5,321,2017-05-11 23:51:56,http://shieldskrajcik.info/kristian.harris,2.0000000000,148.90.189.30,"{""location"": ""CI"", ""is_mobile"": false}" 3534,5,321,2016-12-20 00:23:36,http://hoppe.io/earl_langworth,0.0000000000,169.150.235.18,"{""location"": ""SA"", ""is_mobile"": false}" 3535,5,322,2017-02-16 01:56:12,http://greenholt.org/candida_harber,2.0000000000,197.171.26.224,"{""location"": ""IT"", ""is_mobile"": true}" 3536,5,323,2017-03-04 19:00:37,http://hermiston.biz/marlon.bruen,0.0000000000,82.17.217.232,"{""location"": ""HU"", ""is_mobile"": true}" 3537,5,323,2017-05-03 22:29:13,http://terry.info/adonis,1.0000000000,7.196.153.22,"{""location"": ""EC"", ""is_mobile"": true}" 3538,5,323,2017-01-22 03:15:27,http://johns.co/alize_blanda,1.0000000000,143.185.13.169,"{""location"": ""BN"", ""is_mobile"": false}" 3539,5,323,2017-02-03 09:51:06,http://herman.name/murphy.thompson,1.0000000000,173.177.63.224,"{""location"": ""BG"", ""is_mobile"": false}" 3540,5,323,2017-02-17 06:51:33,http://gusikowskieichmann.co/angie_weimann,2.0000000000,216.174.122.210,"{""location"": ""AQ"", ""is_mobile"": false}" 3541,5,324,2017-04-05 15:59:57,http://dietrichschamberger.info/earnestine.mayert,1.0000000000,94.23.114.132,"{""location"": ""NG"", ""is_mobile"": false}" 3542,5,324,2017-01-31 08:03:26,http://maggio.info/alysa,0.0000000000,104.36.108.185,"{""location"": ""FR"", ""is_mobile"": false}" 3543,5,324,2016-12-28 00:15:53,http://bergstromkeeling.net/meredith,0.0000000000,77.145.35.85,"{""location"": ""TO"", ""is_mobile"": false}" 3544,5,324,2017-04-16 04:44:07,http://pouros.biz/mckayla,0.0000000000,243.6.72.201,"{""location"": ""CD"", ""is_mobile"": false}" 3545,5,324,2017-01-10 19:32:56,http://von.com/gaetano_fahey,0.0000000000,216.126.118.30,"{""location"": ""CL"", ""is_mobile"": false}" 3546,5,324,2017-02-05 21:42:33,http://sawayn.co/erich_zemlak,1.0000000000,40.197.166.140,"{""location"": ""VU"", ""is_mobile"": true}" 3547,5,324,2017-05-13 04:56:43,http://farrell.name/easton_johns,0.0000000000,110.51.42.217,"{""location"": ""MC"", ""is_mobile"": true}" 3548,5,324,2017-01-31 05:54:57,http://pagachammes.io/mya_damore,1.0000000000,94.23.114.132,"{""location"": ""AG"", ""is_mobile"": true}" 3549,5,324,2017-06-12 12:49:49,http://abbott.io/vida.koch,1.0000000000,251.93.157.251,"{""location"": ""RS"", ""is_mobile"": true}" 3550,5,324,2017-01-22 15:09:44,http://sporerquitzon.com/kaley.rolfson,0.0000000000,46.177.14.15,"{""location"": ""BJ"", ""is_mobile"": true}" 3551,5,325,2017-06-12 14:32:04,http://mrazprosacco.co/crystel,2.0000000000,230.120.91.128,"{""location"": ""GR"", ""is_mobile"": true}" 3552,5,325,2016-12-14 00:03:59,http://lebsackpouros.io/isadore.schultz,1.0000000000,31.11.57.59,"{""location"": ""SA"", ""is_mobile"": false}" 3553,5,325,2016-12-16 06:39:14,http://gorczany.info/esther.rowe,2.0000000000,187.169.33.62,"{""location"": ""SN"", ""is_mobile"": false}" 3554,5,325,2017-04-11 07:34:50,http://lowe.name/robin,1.0000000000,194.137.88.108,"{""location"": ""FM"", ""is_mobile"": false}" 3555,5,325,2017-06-11 14:56:01,http://pollich.name/arnoldo_raynor,2.0000000000,102.127.58.156,"{""location"": ""CD"", ""is_mobile"": false}" 3556,5,325,2017-03-21 08:45:45,http://johnston.co/izabella_grady,0.0000000000,45.147.168.211,"{""location"": ""CO"", ""is_mobile"": true}" 3557,5,325,2017-04-29 20:14:33,http://ratkemckenzie.io/dee,0.0000000000,87.30.57.103,"{""location"": ""PR"", ""is_mobile"": true}" 3558,5,326,2016-12-19 13:25:07,http://mante.com/lyla,1.0000000000,181.181.104.72,"{""location"": ""HM"", ""is_mobile"": false}" 3559,5,326,2017-04-12 04:15:00,http://schneider.info/arden,0.0000000000,58.159.119.232,"{""location"": ""GM"", ""is_mobile"": true}" 3560,5,326,2017-01-13 20:55:40,http://mckenziereichel.info/jeromy_rutherford,2.0000000000,217.237.169.29,"{""location"": ""SL"", ""is_mobile"": false}" 3561,5,326,2017-05-12 01:22:51,http://conn.co/grayson.stiedemann,2.0000000000,27.77.134.103,"{""location"": ""EE"", ""is_mobile"": false}" 3562,5,326,2017-04-23 19:45:34,http://kirlinstrosin.biz/thaddeus.wiza,2.0000000000,30.121.94.125,"{""location"": ""GW"", ""is_mobile"": false}" 3563,5,326,2017-03-31 09:16:43,http://feeney.net/hellen,2.0000000000,239.178.218.171,"{""location"": ""BH"", ""is_mobile"": true}" 3564,5,326,2017-01-21 03:59:44,http://weimannrolfson.org/brandyn,2.0000000000,27.77.134.103,"{""location"": ""TL"", ""is_mobile"": false}" 3565,5,326,2017-05-29 17:50:45,http://gulgowski.com/enrique,1.0000000000,181.181.104.72,"{""location"": ""GI"", ""is_mobile"": true}" 3566,5,326,2017-06-07 11:47:30,http://weinatschimmel.org/dovie,1.0000000000,63.206.18.140,"{""location"": ""TZ"", ""is_mobile"": false}" 3567,5,326,2017-01-09 18:08:56,http://kirlin.net/wilfrid.lindgren,3.0000000000,58.159.119.232,"{""location"": ""NZ"", ""is_mobile"": true}" 3568,5,326,2017-02-13 12:04:48,http://willms.org/mattie_jacobs,2.0000000000,95.203.126.55,"{""location"": ""YT"", ""is_mobile"": true}" 3569,5,326,2017-04-30 06:46:13,http://balistreri.io/tevin,3.0000000000,19.112.7.211,"{""location"": ""TC"", ""is_mobile"": true}" 3570,5,326,2017-01-15 09:30:58,http://schmitt.biz/lavada_morar,1.0000000000,228.147.115.203,"{""location"": ""SV"", ""is_mobile"": false}" 3571,5,327,2017-01-28 17:40:54,http://friesenrunte.net/leann,,133.180.9.101,"{""location"": ""CH"", ""is_mobile"": false}" 3572,5,327,2017-01-11 06:24:47,http://weber.org/charlotte_parisian,,19.199.36.244,"{""location"": ""DO"", ""is_mobile"": false}" 3573,5,327,2017-02-08 18:08:45,http://medhurstcrooks.biz/marjory,,69.123.134.76,"{""location"": ""CR"", ""is_mobile"": true}" 3574,5,328,2017-04-07 05:37:35,http://kuvalichaefer.biz/willow.barrows,,151.104.240.45,"{""location"": ""PY"", ""is_mobile"": true}" 3575,5,328,2017-02-03 19:58:16,http://murray.io/deie_carter,,244.41.31.51,"{""location"": ""DJ"", ""is_mobile"": false}" 3576,5,328,2017-05-26 00:18:40,http://hills.net/vivien,,213.139.140.231,"{""location"": ""GB"", ""is_mobile"": true}" 3577,5,328,2017-04-04 01:59:22,http://parker.org/riley.hartmann,,166.125.82.133,"{""location"": ""SV"", ""is_mobile"": false}" 3578,5,328,2017-03-18 17:06:20,http://wymanjacobson.co/luigi_crist,,169.71.105.103,"{""location"": ""HN"", ""is_mobile"": true}" 3579,5,328,2016-12-26 21:17:15,http://lakin.info/wilton,,165.143.28.172,"{""location"": ""TT"", ""is_mobile"": true}" 3580,5,328,2017-05-01 00:14:10,http://hansen.org/meghan,,220.32.50.133,"{""location"": ""KH"", ""is_mobile"": true}" 3581,5,328,2017-01-05 13:12:18,http://schuster.biz/enoch_franecki,,88.135.48.85,"{""location"": ""HR"", ""is_mobile"": true}" 3582,5,328,2017-04-03 10:28:39,http://schinner.name/nicolette,,169.71.105.103,"{""location"": ""BY"", ""is_mobile"": false}" 3583,5,328,2017-05-07 03:54:50,http://rosenbaum.info/anabel_koelpin,,188.248.163.143,"{""location"": ""LB"", ""is_mobile"": true}" 3584,5,328,2017-05-17 22:53:03,http://turnerpowlowski.co/alycia,,2.172.63.126,"{""location"": ""BE"", ""is_mobile"": true}" 3585,5,328,2016-12-18 20:45:07,http://tremblay.io/kariane,,27.197.2.81,"{""location"": ""CH"", ""is_mobile"": false}" 3586,5,328,2017-02-22 00:11:11,http://heller.co/chad,,5.244.61.52,"{""location"": ""DK"", ""is_mobile"": false}" 3587,5,328,2017-02-03 09:23:37,http://purdy.org/eldora,,240.163.25.105,"{""location"": ""DK"", ""is_mobile"": true}" 3588,5,328,2017-03-12 01:49:55,http://ruecker.info/jamarcus_kohler,,103.80.196.225,"{""location"": ""KG"", ""is_mobile"": false}" 3589,5,328,2017-01-04 08:32:40,http://kuhic.net/cleo.cormier,,17.27.86.250,"{""location"": ""CM"", ""is_mobile"": true}" 3590,5,328,2017-04-10 08:40:32,http://schulist.net/patrick.sawayn,,145.195.143.146,"{""location"": ""BM"", ""is_mobile"": false}" 3591,5,328,2017-03-21 23:20:05,http://ryan.info/oswaldo.heller,,27.197.2.81,"{""location"": ""GG"", ""is_mobile"": false}" 3592,5,328,2017-05-01 04:49:08,http://hilpert.io/gwen.mayer,,215.103.163.53,"{""location"": ""MT"", ""is_mobile"": true}" 3593,5,328,2016-12-17 10:09:32,http://heaneydickens.info/cathy,,17.27.86.250,"{""location"": ""TC"", ""is_mobile"": true}" 3594,5,329,2017-06-08 02:48:22,http://bodebrakus.com/casimer,,121.33.164.155,"{""location"": ""EE"", ""is_mobile"": false}" 3595,5,329,2017-05-05 21:59:42,http://mills.net/joan_langworth,,210.214.206.202,"{""location"": ""UA"", ""is_mobile"": true}" 3596,5,329,2016-12-23 15:28:02,http://gerlach.biz/ada,,36.226.23.80,"{""location"": ""FJ"", ""is_mobile"": false}" 3597,5,329,2017-03-19 10:46:45,http://boscostiedemann.name/rafael,,121.199.233.125,"{""location"": ""KZ"", ""is_mobile"": true}" 3598,5,329,2017-03-06 23:40:22,http://wisokybarrows.info/janie.bogisich,,113.120.216.11,"{""location"": ""MP"", ""is_mobile"": false}" 3599,5,329,2017-02-04 11:15:22,http://feestmcclure.io/myrtis.medhurst,,89.91.186.114,"{""location"": ""SE"", ""is_mobile"": false}" 3600,5,329,2017-04-08 06:42:08,http://wehnerokuneva.net/maybelle,,230.57.19.208,"{""location"": ""KM"", ""is_mobile"": false}" 3601,5,329,2017-03-12 05:17:44,http://padberg.name/brian,,199.118.30.123,"{""location"": ""CM"", ""is_mobile"": false}" 3602,5,329,2017-04-04 00:38:30,http://sipesgutkowski.biz/karolann,,202.189.70.149,"{""location"": ""SL"", ""is_mobile"": false}" 3603,5,329,2016-12-31 00:58:22,http://ledner.org/rolando,,148.169.154.186,"{""location"": ""LA"", ""is_mobile"": true}" 3604,5,330,2017-05-30 17:09:18,http://keebler.org/gaston,,213.199.174.78,"{""location"": ""NR"", ""is_mobile"": false}" 3605,5,330,2017-02-07 12:27:33,http://smith.biz/libby,,230.215.104.2,"{""location"": ""AW"", ""is_mobile"": true}" 3606,5,330,2017-06-01 15:58:22,http://boehm.biz/tyler,,213.199.174.78,"{""location"": ""MC"", ""is_mobile"": false}" 3607,5,330,2017-03-02 12:51:19,http://gaylordgibson.info/jonathan,,213.199.174.78,"{""location"": ""GQ"", ""is_mobile"": false}" 3608,5,330,2017-02-19 01:44:25,http://reichel.name/ulises,,165.253.55.228,"{""location"": ""SN"", ""is_mobile"": true}" 3609,5,330,2017-05-18 12:17:02,http://blanda.biz/geo.stark,,230.215.104.2,"{""location"": ""MN"", ""is_mobile"": true}" 3610,5,330,2017-04-22 22:03:54,http://gradyklein.name/iac,,97.190.105.241,"{""location"": ""RO"", ""is_mobile"": false}" 3611,5,330,2017-03-09 05:25:52,http://block.io/lila,,165.253.55.228,"{""location"": ""VI"", ""is_mobile"": true}" 3612,5,330,2017-03-05 11:00:18,http://brown.name/rudy_hettinger,,18.149.156.121,"{""location"": ""LV"", ""is_mobile"": true}" 3613,5,331,2017-04-16 00:28:16,http://senger.com/candido_walker,,217.229.57.148,"{""location"": ""BW"", ""is_mobile"": false}" 3614,5,331,2017-04-01 08:44:09,http://ebert.io/chance_kohler,,56.13.46.20,"{""location"": ""LC"", ""is_mobile"": false}" 3615,5,331,2017-06-04 03:30:30,http://willms.info/ayden,,205.49.160.216,"{""location"": ""LK"", ""is_mobile"": true}" 3616,5,331,2017-04-15 06:54:54,http://kub.info/gilda_beier,,9.2.253.172,"{""location"": ""YE"", ""is_mobile"": false}" 3617,5,331,2017-03-11 09:51:00,http://bailey.info/gustave,,157.253.189.113,"{""location"": ""GR"", ""is_mobile"": false}" 3618,5,331,2017-03-11 11:48:32,http://schmidtlind.com/rosalinda_ortiz,,219.99.161.207,"{""location"": ""LK"", ""is_mobile"": true}" 3619,5,331,2017-05-19 11:48:37,http://simonisokeefe.info/larue_schumm,,189.166.182.178,"{""location"": ""AR"", ""is_mobile"": false}" 3620,5,331,2017-03-25 04:40:40,http://lemke.co/tiara_swaniawski,,62.120.90.73,"{""location"": ""SR"", ""is_mobile"": true}" 3621,5,331,2017-01-26 07:30:41,http://heel.biz/brook,,47.3.74.83,"{""location"": ""PL"", ""is_mobile"": true}" 3622,5,331,2017-05-10 04:01:00,http://sporer.co/vallie.keler,,56.69.238.83,"{""location"": ""SX"", ""is_mobile"": true}" 3623,5,331,2017-04-26 17:58:45,http://brekke.biz/katelin,,184.8.211.169,"{""location"": ""KM"", ""is_mobile"": false}" 3624,5,332,2017-06-04 03:37:55,http://padberg.org/isai_rosenbaum,,233.184.101.15,"{""location"": ""HK"", ""is_mobile"": true}" 3625,5,332,2017-02-05 09:26:40,http://gottlieb.co/lyric.metz,,35.144.79.195,"{""location"": ""LR"", ""is_mobile"": false}" 3626,5,332,2016-12-15 00:02:30,http://witting.net/clifton,,212.125.77.251,"{""location"": ""HT"", ""is_mobile"": false}" 3627,5,332,2017-01-19 21:11:31,http://okongrady.net/citlalli,,51.23.169.170,"{""location"": ""NP"", ""is_mobile"": true}" 3628,5,332,2017-06-10 19:49:34,http://veum.info/ara_bode,,2.127.65.248,"{""location"": ""SN"", ""is_mobile"": true}" 3629,5,332,2017-03-16 17:33:30,http://koch.net/adella,,2.127.65.248,"{""location"": ""CF"", ""is_mobile"": false}" 3630,5,332,2017-03-14 15:06:14,http://kutch.biz/justus_lueilwitz,,159.43.123.182,"{""location"": ""FK"", ""is_mobile"": true}" 3631,5,333,2017-06-05 20:41:48,http://marquardt.com/alford.nolan,,16.171.192.72,"{""location"": ""PS"", ""is_mobile"": true}" 3632,5,333,2017-06-12 12:52:24,http://schaeferbednar.org/elenor_moore,,209.133.109.169,"{""location"": ""EG"", ""is_mobile"": true}" 3633,5,333,2016-12-30 06:37:38,http://rutherford.org/jeffery,,163.84.95.34,"{""location"": ""BZ"", ""is_mobile"": true}" 3634,5,333,2017-01-12 23:56:01,http://stiedemannstrosin.name/fred.sanford,,45.193.206.106,"{""location"": ""GD"", ""is_mobile"": false}" 3635,5,333,2017-02-26 01:31:54,http://emmerichspinka.com/kyler,,191.35.104.19,"{""location"": ""AU"", ""is_mobile"": true}" 3636,5,333,2017-05-29 20:00:01,http://satterfield.name/adolf,,139.44.2.22,"{""location"": ""AF"", ""is_mobile"": false}" 3637,5,333,2017-04-01 16:23:10,http://cormierhickle.io/earl.lubowitz,,96.173.131.44,"{""location"": ""LA"", ""is_mobile"": false}" 3638,5,333,2016-12-23 21:58:05,http://gerlach.biz/noe,,50.229.229.20,"{""location"": ""UM"", ""is_mobile"": true}" 3639,5,334,2017-02-19 05:51:03,http://ledner.co/josh.ohara,,215.191.90.86,"{""location"": ""DK"", ""is_mobile"": false}" 3640,5,334,2016-12-28 16:58:47,http://mckenzie.io/haven.mueller,,168.203.249.241,"{""location"": ""FR"", ""is_mobile"": true}" 3641,5,334,2016-12-22 23:35:39,http://friesencrooks.org/destiny.rice,,195.30.231.237,"{""location"": ""JE"", ""is_mobile"": true}" 3642,5,334,2017-01-03 23:30:06,http://wyman.co/estel,,43.189.95.77,"{""location"": ""HN"", ""is_mobile"": false}" 3643,5,334,2017-05-10 02:12:25,http://gibsonhammes.info/emmett,,167.137.220.218,"{""location"": ""AL"", ""is_mobile"": true}" 3644,5,334,2017-02-05 16:47:03,http://robertsolson.com/amely,,99.207.66.134,"{""location"": ""LK"", ""is_mobile"": true}" 3645,5,334,2017-03-12 04:14:27,http://rolfsonemard.co/elias_watsica,,195.30.231.237,"{""location"": ""CI"", ""is_mobile"": true}" 3646,5,334,2017-02-08 07:09:57,http://croninolson.co/caterina,,248.216.44.124,"{""location"": ""SI"", ""is_mobile"": true}" 3647,5,334,2017-01-24 06:35:04,http://keeling.org/marcella,,99.207.66.134,"{""location"": ""NL"", ""is_mobile"": true}" 3648,5,334,2017-01-20 16:23:54,http://conn.io/autumn_wintheiser,,163.184.161.23,"{""location"": ""MU"", ""is_mobile"": false}" 3649,5,334,2017-01-13 00:56:50,http://bashirian.io/shanelle.carroll,,195.30.231.237,"{""location"": ""GY"", ""is_mobile"": true}" 3650,5,334,2017-03-18 12:21:14,http://weber.biz/victoria,,19.184.230.27,"{""location"": ""PM"", ""is_mobile"": false}" 3651,5,334,2017-05-18 15:33:11,http://schinnerlynch.co/fatima_armstrong,,180.75.132.57,"{""location"": ""AR"", ""is_mobile"": false}" 3652,5,334,2017-03-31 11:58:31,http://casper.name/ilene_hintz,,61.123.204.241,"{""location"": ""BB"", ""is_mobile"": false}" 3653,5,334,2016-12-14 10:27:50,http://greenkiehn.name/nils_maggio,,167.137.220.218,"{""location"": ""ES"", ""is_mobile"": true}" 3654,5,334,2017-04-19 22:51:24,http://stoltenbergcasper.org/roel_johns,,167.137.220.218,"{""location"": ""AM"", ""is_mobile"": false}" 3655,5,334,2016-12-13 17:02:52,http://quitzonherman.name/carmella.hudson,,163.184.161.23,"{""location"": ""LS"", ""is_mobile"": true}" 3656,5,334,2017-06-08 19:12:50,http://heel.biz/emely,,11.85.204.213,"{""location"": ""RO"", ""is_mobile"": true}" 3657,5,334,2017-04-05 23:27:56,http://manncarroll.info/danielle_borer,,149.102.182.30,"{""location"": ""ST"", ""is_mobile"": false}" 3658,5,335,2017-03-21 14:40:43,http://mooreko.name/sibyl,,28.237.40.186,"{""location"": ""NU"", ""is_mobile"": true}" 3659,5,335,2017-05-13 22:07:21,http://hirthe.io/krystal_wuckert,,218.37.157.171,"{""location"": ""BN"", ""is_mobile"": false}" 3660,5,335,2017-04-13 05:00:54,http://abbottfeil.info/brett,,140.99.244.85,"{""location"": ""JO"", ""is_mobile"": true}" 3661,5,335,2017-04-06 06:35:31,http://anderson.io/retha,,226.39.16.98,"{""location"": ""PS"", ""is_mobile"": true}" 3662,5,335,2017-03-09 19:15:20,http://yostspencer.net/curtis_schultz,,140.99.244.85,"{""location"": ""TG"", ""is_mobile"": false}" 3663,5,335,2017-02-06 12:00:04,http://klockowolf.net/mattie,,226.39.16.98,"{""location"": ""AE"", ""is_mobile"": true}" 3664,5,335,2016-12-26 13:07:08,http://jakubowskiherman.net/aliyah_stark,,109.247.32.66,"{""location"": ""CO"", ""is_mobile"": true}" 3665,5,335,2017-05-12 10:19:52,http://adams.info/justen,,134.147.49.114,"{""location"": ""DZ"", ""is_mobile"": false}" 3666,5,335,2017-05-26 13:02:25,http://borerfunk.org/macy_rempel,,70.71.2.114,"{""location"": ""TN"", ""is_mobile"": false}" 3667,5,335,2017-06-03 15:05:42,http://dareadams.biz/camren,,134.147.49.114,"{""location"": ""GQ"", ""is_mobile"": true}" 3668,5,335,2017-06-11 21:35:53,http://keebler.biz/leda.metz,,49.171.90.166,"{""location"": ""PT"", ""is_mobile"": true}" 3669,5,335,2017-03-29 00:32:49,http://labadie.com/eldon,,134.147.49.114,"{""location"": ""TO"", ""is_mobile"": true}" 3670,5,335,2017-03-01 11:14:23,http://tillman.com/roberto_schroeder,,73.161.192.153,"{""location"": ""PE"", ""is_mobile"": true}" 3671,5,335,2017-04-28 23:41:00,http://shields.co/mariah,,213.124.78.111,"{""location"": ""PF"", ""is_mobile"": false}" 2084,4,195,2017-04-28 20:26:38,http://bins.org/velma.mcdermott,,238.14.207.110,"{""location"": ""KY"", ""is_mobile"": true}" 2085,4,195,2017-02-09 08:28:27,http://heelwillms.biz/alvera,,88.233.168.141,"{""location"": ""KI"", ""is_mobile"": true}" 2086,4,195,2017-03-13 05:28:12,http://colebeer.name/lulu_roob,,188.195.187.121,"{""location"": ""KY"", ""is_mobile"": false}" 2087,4,195,2017-03-12 04:29:01,http://swaniawski.org/liliana,,92.62.116.82,"{""location"": ""CG"", ""is_mobile"": true}" 2088,4,195,2017-05-16 12:36:26,http://cummings.info/sonia,,57.116.161.131,"{""location"": ""SG"", ""is_mobile"": true}" 2089,4,195,2017-03-26 02:54:36,http://vandervort.biz/cameron,,37.112.254.131,"{""location"": ""PR"", ""is_mobile"": true}" 2090,4,195,2017-01-07 14:54:59,http://fisher.org/walker_kilback,,250.247.83.99,"{""location"": ""NR"", ""is_mobile"": true}" 2091,4,195,2017-05-17 10:29:36,http://moore.org/carmel.hand,,165.159.3.148,"{""location"": ""LT"", ""is_mobile"": true}" 2092,4,195,2016-12-31 10:49:37,http://cormier.co/casimir_cremin,,243.246.226.152,"{""location"": ""BF"", ""is_mobile"": true}" 2093,4,196,2017-02-16 00:41:56,http://kaulke.name/zoe_huels,,213.234.252.61,"{""location"": ""SJ"", ""is_mobile"": true}" 2094,4,196,2017-04-29 07:05:33,http://padbergheel.com/newton_bartoletti,,31.17.96.120,"{""location"": ""BB"", ""is_mobile"": false}" 2095,4,196,2017-06-07 07:38:48,http://rempellowe.co/devan,,145.31.59.226,"{""location"": ""ZA"", ""is_mobile"": false}" 2096,4,196,2017-02-06 08:27:37,http://murphypurdy.name/margaretta_borer,,31.17.96.120,"{""location"": ""AE"", ""is_mobile"": false}" 2097,4,197,2017-06-05 20:38:51,http://durgan.biz/harry_sawayn,,206.221.57.82,"{""location"": ""PG"", ""is_mobile"": true}" 2098,4,197,2017-02-15 01:09:14,http://connelly.co/noemie.cummerata,,241.61.109.196,"{""location"": ""IN"", ""is_mobile"": true}" 2099,4,197,2017-04-01 19:30:04,http://streich.io/nasir,,103.222.165.178,"{""location"": ""SK"", ""is_mobile"": false}" 2100,4,197,2017-01-30 21:14:11,http://hickle.io/brooke,,230.254.236.236,"{""location"": ""HK"", ""is_mobile"": true}" 2101,4,197,2017-04-21 01:54:53,http://balistreri.org/kendall,,228.245.225.43,"{""location"": ""KR"", ""is_mobile"": true}" 2102,4,197,2017-01-01 12:31:44,http://murray.io/caroline,,33.149.53.106,"{""location"": ""TM"", ""is_mobile"": true}" 2103,4,198,2017-04-12 05:51:24,http://klein.net/jordan_bechtelar,,91.200.210.215,"{""location"": ""MK"", ""is_mobile"": true}" 2104,4,198,2017-05-31 13:09:38,http://shields.biz/wilson,,217.56.198.79,"{""location"": ""AI"", ""is_mobile"": true}" 2105,4,198,2016-12-21 03:29:20,http://mills.org/winnifred,,146.189.109.114,"{""location"": ""AQ"", ""is_mobile"": false}" 2106,4,198,2017-04-08 07:39:56,http://macejkovic.biz/forest,,241.13.64.198,"{""location"": ""IE"", ""is_mobile"": false}" 2107,4,198,2017-06-13 04:09:32,http://botsford.com/dameon.schuster,,64.112.110.232,"{""location"": ""PK"", ""is_mobile"": true}" 2108,4,198,2017-05-05 19:38:27,http://roob.biz/bernice_beer,,9.246.5.210,"{""location"": ""SS"", ""is_mobile"": true}" 2109,4,198,2016-12-30 05:23:24,http://vonrueden.info/libbie.sawayn,,46.147.247.254,"{""location"": ""AI"", ""is_mobile"": true}" 2110,4,198,2017-06-04 23:56:06,http://murphy.co/agustina,,143.246.188.216,"{""location"": ""AS"", ""is_mobile"": false}" 2111,4,198,2017-02-03 01:58:14,http://schiller.biz/curtis.lakin,,222.156.240.16,"{""location"": ""NA"", ""is_mobile"": true}" 2112,4,198,2016-12-25 16:01:00,http://murray.info/laney_ortiz,,202.132.44.211,"{""location"": ""BE"", ""is_mobile"": true}" 2113,4,198,2017-03-22 05:31:27,http://bruen.io/meggie,,132.70.114.73,"{""location"": ""CF"", ""is_mobile"": true}" 2114,4,198,2017-05-07 11:56:48,http://kundefeil.net/otis_cronin,,218.96.245.187,"{""location"": ""PA"", ""is_mobile"": false}" 2115,4,198,2017-01-02 03:07:36,http://johnston.name/jaden.dach,,217.56.198.79,"{""location"": ""IL"", ""is_mobile"": true}" 2116,4,198,2017-01-25 10:58:27,http://hoppe.co/gordon,,51.164.133.145,"{""location"": ""NC"", ""is_mobile"": false}" 2117,4,199,2017-02-04 11:21:00,http://stehr.com/verdie_quitzon,,68.38.78.199,"{""location"": ""DK"", ""is_mobile"": false}" 2118,4,199,2017-03-29 08:46:25,http://parker.com/chester,,120.133.67.103,"{""location"": ""FR"", ""is_mobile"": false}" 2119,4,199,2017-04-16 02:01:22,http://durganwaelchi.biz/emma_feil,,46.33.72.25,"{""location"": ""PL"", ""is_mobile"": false}" 2120,4,199,2017-03-05 06:23:37,http://koeppbogisich.name/ferne,,160.182.241.62,"{""location"": ""IS"", ""is_mobile"": true}" 2121,4,199,2017-01-16 21:18:40,http://effertz.net/gail,,105.12.97.69,"{""location"": ""LS"", ""is_mobile"": true}" 2122,4,199,2017-04-05 18:12:36,http://torphy.info/coty,,68.145.103.219,"{""location"": ""JM"", ""is_mobile"": true}" 2123,4,199,2017-04-11 17:51:12,http://lubowitzjones.co/zena.grady,,77.181.56.188,"{""location"": ""KY"", ""is_mobile"": true}" 2124,4,199,2017-04-20 15:14:09,http://maggioratke.info/isadore,,174.55.87.141,"{""location"": ""FI"", ""is_mobile"": true}" 2125,4,199,2017-06-10 01:05:04,http://witting.com/luna,,86.135.45.14,"{""location"": ""WS"", ""is_mobile"": true}" 2126,4,199,2017-01-21 00:35:23,http://waelchigoodwin.biz/kennedy,,247.41.63.144,"{""location"": ""DM"", ""is_mobile"": true}" 2127,4,199,2016-12-22 06:10:23,http://schaefer.name/javier,,174.55.87.141,"{""location"": ""ZM"", ""is_mobile"": false}" 2128,4,199,2017-05-09 19:54:21,http://nitzschecasper.info/dallas_mcglynn,,174.55.87.141,"{""location"": ""PY"", ""is_mobile"": false}" 2129,4,199,2016-12-21 05:26:00,http://rolfson.io/rey,,218.14.232.187,"{""location"": ""PM"", ""is_mobile"": false}" 2130,4,199,2017-02-11 03:40:10,http://murphylebsack.io/edythe_lueilwitz,,246.28.87.203,"{""location"": ""SE"", ""is_mobile"": false}" 2131,4,199,2017-05-17 03:19:31,http://lueilwitz.io/bette,,66.143.187.56,"{""location"": ""CM"", ""is_mobile"": false}" 2132,4,200,2017-02-02 21:46:58,http://padberg.biz/orlando.borer,,9.42.162.59,"{""location"": ""GA"", ""is_mobile"": true}" 2133,4,200,2017-04-01 21:06:23,http://torp.name/addison_kerluke,,30.193.74.133,"{""location"": ""ME"", ""is_mobile"": true}" 2134,4,200,2017-03-27 20:04:12,http://oberbrunner.org/hector.cormier,,106.13.251.95,"{""location"": ""LR"", ""is_mobile"": false}" 2135,4,200,2017-03-17 23:20:29,http://borergleichner.co/ari.denesik,,206.60.110.60,"{""location"": ""KG"", ""is_mobile"": true}" 2136,4,201,2017-01-28 06:50:30,http://huel.info/meda_predovic,,228.215.116.150,"{""location"": ""AF"", ""is_mobile"": true}" 2137,4,201,2017-01-19 02:15:19,http://corwinparisian.org/leanna_hermiston,,207.164.114.203,"{""location"": ""AZ"", ""is_mobile"": false}" 2138,4,201,2017-01-06 06:52:51,http://ebert.org/vanea.reichel,,179.211.41.200,"{""location"": ""GW"", ""is_mobile"": false}" 2139,4,201,2017-05-18 19:44:57,http://deckowabshire.co/michaela,,124.133.40.247,"{""location"": ""SJ"", ""is_mobile"": true}" 2140,4,201,2017-04-04 09:10:23,http://steuber.co/antone.bosco,,47.123.238.76,"{""location"": ""AF"", ""is_mobile"": false}" 2141,4,201,2017-04-12 11:48:25,http://hettinger.biz/earlene_upton,,245.168.56.77,"{""location"": ""CF"", ""is_mobile"": true}" 2142,4,201,2017-01-26 22:57:36,http://wymanebert.com/kenna,,51.19.65.215,"{""location"": ""GQ"", ""is_mobile"": true}" 2143,4,201,2017-05-16 07:24:47,http://morietterosenbaum.biz/nedra.dibbert,,252.154.122.236,"{""location"": ""FJ"", ""is_mobile"": true}" 2144,4,201,2016-12-31 19:34:25,http://lockman.org/dwight,,215.112.21.228,"{""location"": ""NE"", ""is_mobile"": false}" 2145,4,201,2017-03-04 12:06:31,http://baileyshanahan.info/rosemary_murray,,252.154.122.236,"{""location"": ""TC"", ""is_mobile"": false}" 2146,4,201,2016-12-24 23:39:34,http://schaden.io/elmore_runolfsdottir,,99.73.153.187,"{""location"": ""HM"", ""is_mobile"": false}" 2147,4,202,2017-04-28 02:59:32,http://yundt.org/anabelle,,24.71.70.149,"{""location"": ""SL"", ""is_mobile"": false}" 2148,4,202,2017-02-26 04:01:10,http://weber.co/josiane_sporer,,151.84.82.96,"{""location"": ""VG"", ""is_mobile"": true}" 2149,4,202,2016-12-25 17:34:22,http://marquardt.biz/cyrus.kiehn,,209.34.115.234,"{""location"": ""GM"", ""is_mobile"": false}" 2150,4,202,2016-12-20 10:03:02,http://mannrunolfsdottir.info/columbus,,68.134.104.32,"{""location"": ""JM"", ""is_mobile"": false}" 2151,4,202,2017-01-21 13:41:04,http://hauck.info/garett,,22.192.134.146,"{""location"": ""SC"", ""is_mobile"": false}" 2152,4,202,2017-05-18 19:27:13,http://kuphal.org/darrion_goodwin,,109.238.197.52,"{""location"": ""CW"", ""is_mobile"": true}" 2153,4,202,2016-12-20 10:34:18,http://bahringer.info/suzanne.mertz,,201.11.237.191,"{""location"": ""MX"", ""is_mobile"": true}" 2154,4,202,2017-04-18 16:01:52,http://zemlak.net/seamus.hamill,,37.164.144.118,"{""location"": ""BQ"", ""is_mobile"": true}" 2155,4,202,2017-05-20 19:24:49,http://nader.com/bailee_boyer,,112.5.72.68,"{""location"": ""KI"", ""is_mobile"": true}" 2156,4,202,2016-12-20 05:08:39,http://bahringerkoch.co/treie,,77.232.63.43,"{""location"": ""ML"", ""is_mobile"": false}" 2157,4,202,2017-02-08 21:31:38,http://jacobi.com/jaiden,,116.219.76.223,"{""location"": ""LV"", ""is_mobile"": false}" 2158,4,202,2017-03-03 23:43:27,http://pacochaokeefe.io/marjory,,167.166.83.232,"{""location"": ""PE"", ""is_mobile"": false}" 2159,4,202,2017-06-01 11:04:03,http://littel.name/savanna,,136.105.209.153,"{""location"": ""PA"", ""is_mobile"": true}" 2160,4,202,2017-01-11 14:30:52,http://weinat.biz/madie,,152.38.142.162,"{""location"": ""AD"", ""is_mobile"": false}" 2161,4,202,2017-05-16 17:48:32,http://davis.co/kaitlin,,215.196.126.164,"{""location"": ""AX"", ""is_mobile"": false}" 2162,4,202,2017-03-03 18:02:29,http://bins.co/mabel,,22.192.134.146,"{""location"": ""UZ"", ""is_mobile"": true}" 2163,4,202,2017-01-12 19:19:33,http://howewisozk.biz/martina.mitchell,,70.133.69.213,"{""location"": ""MH"", ""is_mobile"": true}" 2164,4,203,2017-06-05 19:32:17,http://heelskiles.com/piper,,159.40.126.234,"{""location"": ""WS"", ""is_mobile"": false}" 2165,4,203,2016-12-16 08:16:40,http://aufderhar.info/marcelo,,211.234.58.206,"{""location"": ""NP"", ""is_mobile"": false}" 2166,4,203,2017-05-27 23:32:15,http://upton.net/hubert,,198.67.60.180,"{""location"": ""KH"", ""is_mobile"": true}" 2167,4,203,2017-04-30 01:06:21,http://rice.com/clifton,,61.155.4.103,"{""location"": ""SO"", ""is_mobile"": true}" 2168,4,203,2017-02-04 08:12:40,http://zboncak.biz/elbert.feil,,80.124.123.50,"{""location"": ""TT"", ""is_mobile"": true}" 2169,4,203,2017-01-18 17:24:31,http://vonrueden.biz/boris_ryan,,196.116.253.85,"{""location"": ""PM"", ""is_mobile"": true}" 2170,4,203,2017-06-12 13:51:33,http://bradtke.name/nina.mosciski,,55.114.87.35,"{""location"": ""IS"", ""is_mobile"": true}" 2171,4,203,2017-05-27 03:37:16,http://nolan.info/julius,,197.167.52.253,"{""location"": ""GN"", ""is_mobile"": true}" 2172,4,203,2017-04-01 16:02:42,http://kutch.biz/granville_cruickshank,,96.61.169.31,"{""location"": ""ER"", ""is_mobile"": true}" 2173,4,203,2016-12-14 04:47:01,http://zboncak.co/stephania,,213.104.46.181,"{""location"": ""WS"", ""is_mobile"": true}" 2174,4,203,2017-03-08 08:30:52,http://satterfield.org/ila_satterfield,,249.150.237.139,"{""location"": ""RU"", ""is_mobile"": false}" 2175,4,203,2016-12-20 20:56:50,http://kiehn.info/harmon.ruecker,,189.78.239.163,"{""location"": ""NU"", ""is_mobile"": true}" 2176,4,203,2017-02-11 23:45:46,http://swaniawskikaulke.info/eugene,,142.117.231.16,"{""location"": ""US"", ""is_mobile"": true}" 2177,4,203,2017-04-15 23:44:23,http://king.co/cecile,,40.50.204.51,"{""location"": ""PH"", ""is_mobile"": false}" 2178,4,203,2017-04-19 06:28:09,http://cummerata.org/dante,,126.151.12.17,"{""location"": ""GR"", ""is_mobile"": false}" 2179,4,203,2017-05-08 21:18:14,http://gulgowskifeil.io/bethel,,59.52.156.28,"{""location"": ""LC"", ""is_mobile"": true}" 2180,4,203,2017-06-06 21:46:14,http://millerwillms.co/benny,,254.74.27.27,"{""location"": ""CD"", ""is_mobile"": false}" 2181,4,204,2017-04-23 06:40:41,http://murphy.co/dorris.dietrich,,247.132.216.127,"{""location"": ""SR"", ""is_mobile"": false}" 2182,4,204,2017-04-21 08:54:44,http://crooksdurgan.net/jovanny,,60.104.164.253,"{""location"": ""JM"", ""is_mobile"": true}" 2183,4,204,2017-01-18 09:09:42,http://rippin.org/eudora_marvin,,160.180.191.214,"{""location"": ""KR"", ""is_mobile"": false}" 2184,4,204,2017-05-24 21:56:20,http://marksosinski.net/david_bahringer,,40.214.137.148,"{""location"": ""LI"", ""is_mobile"": true}" 2185,4,204,2017-04-19 07:13:52,http://casper.name/ara.franecki,,30.122.208.222,"{""location"": ""WF"", ""is_mobile"": true}" 2186,4,204,2017-06-09 16:13:27,http://cremin.io/janie,,109.73.20.220,"{""location"": ""UY"", ""is_mobile"": true}" 2187,4,204,2017-03-18 16:11:41,http://brekke.io/jovany,,180.253.223.29,"{""location"": ""BW"", ""is_mobile"": true}" 2188,4,204,2017-01-24 20:10:25,http://yostkub.org/garret.toy,,140.113.215.34,"{""location"": ""LY"", ""is_mobile"": true}" 2189,4,204,2017-04-06 03:38:27,http://hermistongerlach.co/jorge,,154.145.129.85,"{""location"": ""BZ"", ""is_mobile"": false}" 2190,4,204,2017-03-03 12:48:21,http://fritschsmith.name/marion.price,,140.113.215.34,"{""location"": ""AF"", ""is_mobile"": false}" 2191,4,204,2017-01-02 22:18:34,http://vandervortkertzmann.io/letitia,,33.136.169.253,"{""location"": ""SM"", ""is_mobile"": true}" 2192,4,204,2017-04-13 13:44:10,http://dietrichlang.io/shakira,,134.9.55.74,"{""location"": ""PK"", ""is_mobile"": true}" 2193,4,205,2017-05-23 10:07:07,http://wardbeahan.info/rhea.heel,,105.231.142.219,"{""location"": ""DK"", ""is_mobile"": false}" 2194,4,205,2017-06-06 04:02:28,http://tillman.co/danny,,226.46.222.104,"{""location"": ""ZM"", ""is_mobile"": false}" 2195,4,205,2016-12-17 23:34:14,http://schulistlegros.co/domenico.wyman,,237.95.120.142,"{""location"": ""HN"", ""is_mobile"": false}" 2196,4,205,2017-03-29 02:04:26,http://oconnell.net/jefferey_miller,,82.130.187.195,"{""location"": ""IS"", ""is_mobile"": false}" 2197,4,205,2017-02-24 14:40:16,http://prohaskagleason.info/frederique,,40.43.220.201,"{""location"": ""SB"", ""is_mobile"": false}" 2198,4,205,2017-03-23 18:15:48,http://nicolas.biz/garrick_wehner,,98.204.29.122,"{""location"": ""JP"", ""is_mobile"": false}" 2199,4,205,2017-02-28 20:02:23,http://rohan.com/kenyon,,243.117.179.33,"{""location"": ""YE"", ""is_mobile"": true}" 2200,4,206,2017-02-17 13:55:07,http://schulist.co/monte,,18.58.60.23,"{""location"": ""UA"", ""is_mobile"": false}" 2201,4,206,2017-04-15 12:43:50,http://hoeger.net/chaim,,167.144.49.13,"{""location"": ""WF"", ""is_mobile"": false}" 2202,4,206,2017-05-20 06:33:22,http://rogahndickens.name/clarabelle_howe,,113.128.85.13,"{""location"": ""TG"", ""is_mobile"": true}" 2203,4,206,2017-03-21 20:34:23,http://hilll.io/elwin,,18.58.60.23,"{""location"": ""GD"", ""is_mobile"": false}" 2204,4,206,2017-06-08 12:05:24,http://shanahan.net/iva.volkman,,252.87.153.156,"{""location"": ""EH"", ""is_mobile"": true}" 2205,4,206,2017-01-07 21:18:18,http://adamskutch.co/rachael,,114.222.238.8,"{""location"": ""AQ"", ""is_mobile"": false}" 2206,4,206,2017-06-13 10:41:44,http://brownlarkin.io/colleen.gutmann,,219.68.72.45,"{""location"": ""NR"", ""is_mobile"": true}" 2207,4,206,2017-01-18 16:53:08,http://osinski.org/nona,,18.58.60.23,"{""location"": ""BS"", ""is_mobile"": true}" 2208,4,206,2017-03-16 08:42:58,http://carroll.net/jeica,,225.195.167.216,"{""location"": ""TK"", ""is_mobile"": true}" 2209,4,206,2017-06-03 10:27:03,http://hyatt.net/jan_bahringer,,74.201.243.117,"{""location"": ""LU"", ""is_mobile"": false}" 2210,4,206,2017-02-26 23:16:57,http://hodkiewicz.com/virgil.renner,,25.52.186.215,"{""location"": ""ID"", ""is_mobile"": true}" 2211,4,206,2017-01-02 04:13:19,http://hackett.io/mariam,,58.51.227.200,"{""location"": ""HK"", ""is_mobile"": true}" 2212,4,206,2017-04-16 09:31:31,http://damore.com/junior_farrell,,137.198.146.116,"{""location"": ""HK"", ""is_mobile"": false}" 2213,4,206,2016-12-15 22:04:59,http://damorerenner.org/alfred,,137.198.146.116,"{""location"": ""VI"", ""is_mobile"": true}" 2214,4,207,2017-03-27 04:35:33,http://zemlakkoelpin.info/jazmin.mclaughlin,,131.182.203.249,"{""location"": ""YT"", ""is_mobile"": false}" 2215,4,207,2017-02-01 21:10:54,http://mcglynn.co/zack.graham,,137.201.114.46,"{""location"": ""TK"", ""is_mobile"": true}" 2216,4,207,2017-01-13 20:29:17,http://runolfon.net/virgie,,246.164.117.245,"{""location"": ""MZ"", ""is_mobile"": true}" 2217,4,207,2017-04-21 15:22:30,http://hills.org/destiny.oconner,,212.50.230.230,"{""location"": ""ML"", ""is_mobile"": false}" 2218,4,207,2017-05-20 02:41:23,http://fritsch.biz/richard.larkin,,137.201.114.46,"{""location"": ""VA"", ""is_mobile"": false}" 2219,4,207,2017-02-05 13:03:28,http://uptonemard.info/mara,,212.50.230.230,"{""location"": ""CU"", ""is_mobile"": true}" 2220,4,207,2017-05-24 06:38:17,http://goyette.co/amari.roob,,45.179.239.10,"{""location"": ""KE"", ""is_mobile"": false}" 2221,4,207,2017-02-04 02:09:54,http://cremingreenholt.name/trace.marvin,,63.134.92.186,"{""location"": ""AF"", ""is_mobile"": true}" 2222,4,207,2016-12-14 20:35:32,http://hegmanndare.name/evangeline,,45.179.239.10,"{""location"": ""CV"", ""is_mobile"": false}" 2223,4,208,2017-04-27 09:54:39,http://gislasonkozey.org/imelda.bednar,,183.13.183.196,"{""location"": ""LV"", ""is_mobile"": true}" 2224,4,208,2017-06-05 08:08:58,http://mayertgrant.org/benny,,105.50.207.112,"{""location"": ""IN"", ""is_mobile"": false}" 2225,4,208,2017-06-13 03:04:40,http://willms.co/beryl.abshire,,224.31.216.135,"{""location"": ""LR"", ""is_mobile"": true}" 2226,4,208,2016-12-13 23:09:46,http://shieldsdooley.net/lew,,204.94.15.23,"{""location"": ""IS"", ""is_mobile"": false}" 2227,4,208,2017-01-19 19:37:44,http://bechtelar.io/maximillian,,25.140.118.103,"{""location"": ""PA"", ""is_mobile"": false}" 2228,4,208,2016-12-20 17:07:06,http://lynch.org/garfield.schowalter,,108.57.239.78,"{""location"": ""MX"", ""is_mobile"": false}" 2229,4,208,2017-02-24 22:23:13,http://williamson.co/devante_stehr,,61.36.82.158,"{""location"": ""BL"", ""is_mobile"": true}" 2230,4,208,2017-01-07 20:16:31,http://bednarrempel.com/evert,,125.220.90.233,"{""location"": ""BF"", ""is_mobile"": true}" 2231,4,208,2017-03-31 16:52:01,http://stokeshodkiewicz.biz/aurore.schimmel,,252.47.127.198,"{""location"": ""DZ"", ""is_mobile"": true}" 2232,4,208,2017-01-08 15:02:47,http://hanebeer.com/kenyon,,204.94.15.23,"{""location"": ""CY"", ""is_mobile"": true}" 2233,4,209,2017-04-19 09:06:22,http://connelly.name/isabelle,,223.56.89.18,"{""location"": ""AI"", ""is_mobile"": false}" 2234,4,209,2017-05-24 07:47:49,http://mueller.biz/toby,,34.4.226.25,"{""location"": ""BY"", ""is_mobile"": false}" 2235,4,209,2017-02-25 20:15:03,http://mayer.co/patricia.feil,,7.9.99.173,"{""location"": ""KM"", ""is_mobile"": false}" 2236,4,209,2017-05-25 20:37:43,http://bartonwiegand.biz/lizeth,,34.4.226.25,"{""location"": ""EC"", ""is_mobile"": true}" 2237,4,209,2017-04-26 14:14:31,http://turnernikolaus.biz/arnoldo,,83.232.84.97,"{""location"": ""ET"", ""is_mobile"": false}" 2238,4,210,2017-05-27 23:05:38,http://hayes.net/roma.heaney,,145.122.75.58,"{""location"": ""BS"", ""is_mobile"": true}" 2239,4,210,2017-05-29 05:47:27,http://crooks.biz/jaquelin,,246.88.79.129,"{""location"": ""FR"", ""is_mobile"": false}" 2240,4,210,2017-05-09 13:34:25,http://sawayn.info/jerome,,202.127.99.170,"{""location"": ""FI"", ""is_mobile"": true}" 2241,4,210,2017-05-10 02:08:00,http://gibson.net/sylvia_turner,,205.247.13.144,"{""location"": ""ET"", ""is_mobile"": false}" 2242,4,210,2017-05-14 22:43:06,http://stantonhahn.com/spencer,,222.192.133.201,"{""location"": ""ZW"", ""is_mobile"": true}" 2243,4,210,2017-01-31 17:39:26,http://pourosweber.org/josue_kaulke,,222.192.133.201,"{""location"": ""KM"", ""is_mobile"": false}" 2244,4,210,2017-05-23 14:35:39,http://swiftgrady.biz/gaston,,83.142.12.254,"{""location"": ""MZ"", ""is_mobile"": false}" 2245,4,210,2017-05-24 17:22:33,http://hoppe.co/yolanda,,52.252.7.158,"{""location"": ""DO"", ""is_mobile"": true}" 2246,4,210,2017-05-05 23:56:40,http://parker.co/reanna.nienow,,52.169.124.88,"{""location"": ""MC"", ""is_mobile"": true}" 2247,4,210,2017-03-27 21:57:33,http://walsh.com/estella_heidenreich,,147.35.43.46,"{""location"": ""BO"", ""is_mobile"": true}" 2248,4,211,2017-06-09 15:12:26,http://trompsimonis.biz/cecil.wiegand,,63.181.235.66,"{""location"": ""BN"", ""is_mobile"": false}" 2249,4,211,2017-01-24 17:27:24,http://collier.org/merlin.willms,,218.49.172.193,"{""location"": ""MK"", ""is_mobile"": false}" 2250,4,211,2017-05-14 04:46:16,http://schaeferbashirian.biz/anabelle,,194.142.86.134,"{""location"": ""CM"", ""is_mobile"": false}" 2251,4,211,2017-03-26 07:42:30,http://dach.io/esperanza,,173.60.76.34,"{""location"": ""IT"", ""is_mobile"": true}" 2252,4,211,2017-05-10 16:27:12,http://howellboyle.net/rebekah.metz,,86.198.10.91,"{""location"": ""AE"", ""is_mobile"": true}" 2253,4,211,2017-01-06 20:41:10,http://greenschuster.co/emmett_hyatt,,74.165.206.30,"{""location"": ""SB"", ""is_mobile"": false}" 2254,4,211,2017-02-13 14:07:05,http://reillykihn.com/mercedes,,12.51.42.24,"{""location"": ""NG"", ""is_mobile"": true}" 2255,4,211,2017-06-06 10:33:03,http://wildermanwitting.info/ines.kihn,,247.228.45.45,"{""location"": ""CL"", ""is_mobile"": false}" 2256,4,211,2017-03-05 21:57:03,http://oberbrunnerreynolds.biz/franz,,214.58.229.244,"{""location"": ""BM"", ""is_mobile"": false}" 2257,4,211,2017-03-18 18:12:46,http://kovacekdurgan.biz/rosendo,,92.112.89.227,"{""location"": ""LI"", ""is_mobile"": false}" 2258,4,211,2017-02-10 21:24:18,http://will.com/ruell,,138.143.114.43,"{""location"": ""FK"", ""is_mobile"": true}" 2259,4,211,2016-12-24 06:33:24,http://terry.io/brandyn_schumm,,9.65.116.125,"{""location"": ""GH"", ""is_mobile"": false}" 2260,4,211,2017-01-18 01:15:00,http://gusikowskiskiles.com/flavio,,194.142.86.134,"{""location"": ""KW"", ""is_mobile"": false}" 2261,4,211,2017-02-18 15:45:12,http://bergnaum.net/makenzie.schaefer,,205.93.226.17,"{""location"": ""AQ"", ""is_mobile"": true}" 2262,4,211,2017-04-17 01:50:25,http://trompsmith.biz/haskell_fahey,,151.45.200.112,"{""location"": ""HK"", ""is_mobile"": false}" 2263,4,211,2017-04-30 16:00:17,http://purdy.org/bernita,,63.84.148.7,"{""location"": ""YT"", ""is_mobile"": true}" 2264,4,211,2017-02-20 22:27:45,http://connfunk.io/andre,,87.114.64.240,"{""location"": ""UA"", ""is_mobile"": true}" 2265,4,211,2016-12-26 00:31:02,http://haley.org/camila_stracke,,182.252.170.233,"{""location"": ""ZM"", ""is_mobile"": false}" 2266,4,211,2016-12-26 19:33:48,http://abshire.info/rafaela_dach,,167.111.252.179,"{""location"": ""OM"", ""is_mobile"": false}" 2267,4,211,2017-04-20 13:10:13,http://bergstromglover.com/kayden,,63.84.148.7,"{""location"": ""YE"", ""is_mobile"": true}" 2268,4,212,2017-03-11 17:52:52,http://kovacek.net/karley,0.0000000000,89.111.166.114,"{""location"": ""ID"", ""is_mobile"": true}" 2269,4,212,2017-06-12 13:15:12,http://cummeratarohan.info/cleora.pagac,0.0000000000,219.216.26.193,"{""location"": ""US"", ""is_mobile"": false}" 2270,4,212,2017-05-10 04:25:18,http://gutkowski.net/donnie.davis,0.0000000000,41.123.124.18,"{""location"": ""CH"", ""is_mobile"": true}" 2271,4,212,2017-01-18 14:01:01,http://weinat.net/baron,0.0000000000,216.201.24.159,"{""location"": ""GE"", ""is_mobile"": true}" 2272,4,212,2017-01-04 17:44:23,http://gerholdkaulke.com/scarlett,0.0000000000,168.110.80.196,"{""location"": ""ES"", ""is_mobile"": true}" 2273,4,212,2017-05-19 12:47:12,http://nicolas.org/icie,0.0000000000,66.73.29.100,"{""location"": ""KP"", ""is_mobile"": false}" 2274,4,212,2017-05-09 01:39:21,http://renner.org/elnora.ruecker,0.0000000000,243.209.69.128,"{""location"": ""UZ"", ""is_mobile"": true}" 2275,4,213,2017-04-15 18:25:18,http://rosenbaum.net/lorenzo.moore,1.0000000000,202.11.115.198,"{""location"": ""MV"", ""is_mobile"": true}" 2276,4,213,2017-04-28 10:58:36,http://larkin.com/edwin_mccullough,1.0000000000,23.127.42.184,"{""location"": ""CV"", ""is_mobile"": false}" 2277,4,213,2017-03-02 00:08:30,http://schaefer.info/kianna.damore,1.0000000000,207.5.84.31,"{""location"": ""MG"", ""is_mobile"": false}" 2278,4,213,2017-04-09 18:40:57,http://orn.name/theron_sawayn,1.0000000000,35.195.203.140,"{""location"": ""TJ"", ""is_mobile"": false}" 2279,4,213,2016-12-15 18:32:57,http://roberts.net/flavie.langosh,0.0000000000,58.178.48.181,"{""location"": ""LU"", ""is_mobile"": true}" 2280,4,213,2016-12-31 11:21:05,http://green.org/nick,1.0000000000,244.17.163.200,"{""location"": ""DO"", ""is_mobile"": false}" 2281,4,213,2017-04-05 16:30:18,http://williamson.biz/deshawn,0.0000000000,161.51.16.46,"{""location"": ""GL"", ""is_mobile"": true}" 2282,4,214,2017-05-05 11:17:27,http://barrowsfeil.org/taryn,1.0000000000,189.181.113.169,"{""location"": ""AD"", ""is_mobile"": true}" 2283,4,214,2017-01-21 11:39:56,http://reichel.com/audrey,0.0000000000,49.81.143.132,"{""location"": ""PF"", ""is_mobile"": true}" 2284,4,214,2017-04-22 13:27:08,http://blanda.biz/alexandra,0.0000000000,38.203.43.22,"{""location"": ""AL"", ""is_mobile"": true}" 2285,4,214,2017-01-03 12:50:52,http://darehauck.co/alec,0.0000000000,29.207.133.27,"{""location"": ""AL"", ""is_mobile"": false}" 2286,4,214,2017-01-30 17:14:16,http://schumm.info/suzanne_hermiston,0.0000000000,38.203.43.22,"{""location"": ""IT"", ""is_mobile"": false}" 2287,4,214,2017-04-09 15:59:02,http://baumbach.name/connor,1.0000000000,49.81.143.132,"{""location"": ""GY"", ""is_mobile"": true}" 2288,4,214,2016-12-13 11:38:31,http://whitespencer.biz/billie,0.0000000000,138.10.143.15,"{""location"": ""NO"", ""is_mobile"": false}" 2289,4,214,2017-03-22 19:27:28,http://deckow.net/bria.kunze,0.0000000000,238.175.158.187,"{""location"": ""BD"", ""is_mobile"": true}" 2290,4,214,2017-03-26 05:48:49,http://reilly.net/ezekiel.wolf,1.0000000000,49.81.143.132,"{""location"": ""DE"", ""is_mobile"": true}" 2291,4,214,2017-02-16 04:52:14,http://heathcote.org/prince,1.0000000000,68.56.104.2,"{""location"": ""ZA"", ""is_mobile"": true}" 2292,4,214,2017-02-14 07:43:12,http://schuppegerhold.info/nadia_hauck,1.0000000000,49.81.143.132,"{""location"": ""LB"", ""is_mobile"": false}" 2293,4,214,2017-03-21 07:20:12,http://bartoletti.net/kara,0.0000000000,138.10.143.15,"{""location"": ""VN"", ""is_mobile"": false}" 2294,4,214,2017-03-02 11:08:16,http://daniel.info/gregg.sauer,1.0000000000,91.177.191.144,"{""location"": ""TZ"", ""is_mobile"": true}" 2295,4,214,2016-12-28 12:16:20,http://friesenvolkman.name/juston,0.0000000000,49.81.143.132,"{""location"": ""BQ"", ""is_mobile"": false}" 2296,4,214,2017-03-03 11:28:20,http://schneider.io/lucas_toy,1.0000000000,51.29.209.47,"{""location"": ""PW"", ""is_mobile"": true}" 2297,4,214,2017-04-30 13:12:19,http://sawayn.info/jayce_monahan,1.0000000000,252.47.76.166,"{""location"": ""LB"", ""is_mobile"": false}" 2298,4,214,2017-05-21 06:19:44,http://schroeder.com/bernadine_sauer,0.0000000000,87.137.71.80,"{""location"": ""NG"", ""is_mobile"": false}" 2299,4,215,2017-03-11 21:34:50,http://ferry.co/ralph.cole,1.0000000000,116.9.119.127,"{""location"": ""CR"", ""is_mobile"": true}" 2300,4,215,2017-04-28 02:05:43,http://gerlachpfannerstill.info/valentine,2.0000000000,162.115.248.24,"{""location"": ""FR"", ""is_mobile"": true}" 2301,4,215,2017-01-06 19:49:11,http://lueilwitzprosacco.com/freeman,1.0000000000,184.190.125.125,"{""location"": ""AI"", ""is_mobile"": false}" 2302,4,215,2017-02-04 08:30:28,http://lindgren.org/ayana.weimann,2.0000000000,251.204.39.99,"{""location"": ""ID"", ""is_mobile"": false}" 2303,4,215,2017-04-18 03:26:46,http://luettgengusikowski.com/ulises.jenkins,0.0000000000,225.15.44.249,"{""location"": ""SA"", ""is_mobile"": false}" 2304,4,215,2017-05-18 08:18:10,http://kuhnflatley.io/alberto.kling,0.0000000000,208.81.79.152,"{""location"": ""AO"", ""is_mobile"": true}" 2305,4,215,2017-05-09 15:40:45,http://corwingleason.org/alf,2.0000000000,149.60.22.154,"{""location"": ""HR"", ""is_mobile"": true}" 2306,4,215,2017-03-14 15:05:26,http://christiansen.co/torrance,1.0000000000,41.154.55.36,"{""location"": ""JM"", ""is_mobile"": false}" 2307,4,215,2017-06-11 06:18:58,http://hackett.com/lauren,1.0000000000,176.222.113.206,"{""location"": ""KI"", ""is_mobile"": false}" 2308,4,215,2017-04-20 04:45:16,http://balistreri.biz/elton,1.0000000000,53.31.12.88,"{""location"": ""BF"", ""is_mobile"": true}" 2309,4,215,2017-04-10 03:24:00,http://ritchie.io/christy,0.0000000000,116.9.119.127,"{""location"": ""KY"", ""is_mobile"": false}" 2310,4,215,2017-02-19 22:51:07,http://flatley.com/hertha,1.0000000000,53.31.12.88,"{""location"": ""LC"", ""is_mobile"": false}" 2311,4,215,2017-01-14 08:21:56,http://legros.com/gabe,0.0000000000,87.22.109.19,"{""location"": ""IQ"", ""is_mobile"": true}" 2312,4,215,2017-02-04 03:35:20,http://blick.com/frederick.schimmel,0.0000000000,199.223.103.65,"{""location"": ""AL"", ""is_mobile"": false}" 2313,4,215,2016-12-18 11:37:13,http://bechtelar.biz/maximillia,2.0000000000,225.3.50.122,"{""location"": ""ST"", ""is_mobile"": false}" 2314,4,215,2017-04-19 03:46:15,http://leffler.info/lisandro.kautzer,2.0000000000,84.204.125.112,"{""location"": ""BG"", ""is_mobile"": false}" 2315,4,216,2016-12-25 12:24:28,http://greenholt.org/dangelo_borer,1.0000000000,13.251.77.22,"{""location"": ""IL"", ""is_mobile"": false}" 2316,4,216,2017-03-16 15:48:36,http://schroeder.info/edyth_schmidt,3.0000000000,78.101.26.168,"{""location"": ""LY"", ""is_mobile"": false}" 2317,4,216,2016-12-30 01:10:40,http://blick.org/francis,2.0000000000,13.88.46.99,"{""location"": ""BY"", ""is_mobile"": true}" 2318,4,216,2017-02-19 16:58:58,http://schimmel.name/lesley.lockman,2.0000000000,215.47.100.221,"{""location"": ""CO"", ""is_mobile"": true}" 2319,4,216,2017-03-22 12:29:31,http://veum.io/frieda_reichel,2.0000000000,111.224.62.253,"{""location"": ""PW"", ""is_mobile"": false}" 2320,4,216,2017-02-18 08:15:01,http://barrows.org/liliana_greenfelder,2.0000000000,55.167.246.241,"{""location"": ""AI"", ""is_mobile"": false}" 2321,4,216,2017-01-13 13:42:57,http://carter.name/lelia,3.0000000000,81.155.212.204,"{""location"": ""SO"", ""is_mobile"": false}" 2322,4,217,2017-01-23 20:08:45,http://harrisgreen.co/lorenzo_kling,2.0000000000,160.118.41.61,"{""location"": ""CA"", ""is_mobile"": false}" 2323,4,217,2017-01-25 08:32:31,http://predovic.info/gerda.borer,0.0000000000,129.122.187.140,"{""location"": ""PL"", ""is_mobile"": true}" 2324,4,217,2017-01-17 17:16:54,http://gulgowski.org/una,1.0000000000,240.88.223.177,"{""location"": ""HK"", ""is_mobile"": false}" 2325,4,217,2017-03-16 12:43:38,http://buckridge.co/moriah.kovacek,1.0000000000,57.210.119.150,"{""location"": ""AX"", ""is_mobile"": true}" 2326,4,217,2017-01-07 03:48:03,http://grady.info/clotilde.kovacek,0.0000000000,218.233.77.163,"{""location"": ""CK"", ""is_mobile"": false}" 2327,4,217,2017-01-28 13:34:20,http://christiansen.name/darrion_senger,1.0000000000,5.24.128.253,"{""location"": ""SG"", ""is_mobile"": true}" 2328,4,217,2017-04-02 11:45:34,http://wehner.com/mohammed_schultz,2.0000000000,194.179.68.197,"{""location"": ""SE"", ""is_mobile"": false}" 2329,4,217,2016-12-25 13:11:37,http://simonis.com/damien_boyle,0.0000000000,145.227.194.224,"{""location"": ""BF"", ""is_mobile"": true}" 2330,4,217,2017-01-17 01:14:25,http://shieldsbednar.co/raoul_stroman,2.0000000000,41.222.57.51,"{""location"": ""MN"", ""is_mobile"": false}" 2331,4,218,2017-02-22 14:49:21,http://naderdenesik.net/jannie_auer,1.0000000000,196.51.164.53,"{""location"": ""LY"", ""is_mobile"": true}" 2332,4,218,2017-02-22 02:38:23,http://pouros.org/allen,0.0000000000,74.54.65.176,"{""location"": ""MK"", ""is_mobile"": false}" 2333,4,218,2017-06-09 20:59:43,http://windler.biz/romaine.howell,0.0000000000,66.136.74.113,"{""location"": ""MC"", ""is_mobile"": true}" 2334,4,218,2017-04-04 11:10:50,http://stamm.co/camylle.rodriguez,2.0000000000,30.112.236.197,"{""location"": ""GI"", ""is_mobile"": true}" 2335,4,218,2017-03-31 00:32:03,http://lynch.co/ida,1.0000000000,232.87.36.208,"{""location"": ""MH"", ""is_mobile"": true}" 2336,4,218,2017-05-30 05:56:50,http://jerde.com/maybell_dubuque,2.0000000000,138.237.13.241,"{""location"": ""KI"", ""is_mobile"": true}" 2337,4,218,2017-03-19 03:24:56,http://stromangusikowski.io/danial,2.0000000000,202.205.78.105,"{""location"": ""MV"", ""is_mobile"": true}" 2338,4,218,2017-04-23 16:36:35,http://leschborer.org/amani,0.0000000000,74.54.65.176,"{""location"": ""FR"", ""is_mobile"": false}" 2339,4,218,2017-02-20 03:22:01,http://oconner.org/rhett,0.0000000000,72.17.87.177,"{""location"": ""PT"", ""is_mobile"": true}" 2340,4,218,2017-01-16 20:24:49,http://leffler.net/mohamed.predovic,2.0000000000,120.66.103.103,"{""location"": ""MZ"", ""is_mobile"": false}" 2341,4,218,2017-02-04 07:31:10,http://trantowkerluke.org/loma,1.0000000000,82.123.194.73,"{""location"": ""GS"", ""is_mobile"": false}" 2342,4,218,2017-05-25 03:38:25,http://sanford.co/jolie,2.0000000000,196.51.164.53,"{""location"": ""AS"", ""is_mobile"": false}" 2343,4,218,2017-04-30 06:13:38,http://crist.net/manuela_cruickshank,0.0000000000,128.107.31.184,"{""location"": ""ER"", ""is_mobile"": false}" 2344,4,218,2017-03-13 14:51:57,http://weber.info/anya,1.0000000000,63.159.211.208,"{""location"": ""MR"", ""is_mobile"": true}" 2345,4,218,2017-05-26 17:11:25,http://mcclure.net/lamont_armstrong,1.0000000000,72.17.87.177,"{""location"": ""WS"", ""is_mobile"": true}" 2346,4,218,2017-06-13 18:41:11,http://berge.name/asa,1.0000000000,30.112.236.197,"{""location"": ""CC"", ""is_mobile"": true}" 2347,4,218,2017-04-15 06:55:28,http://stokes.net/dayton.homenick,2.0000000000,203.48.193.226,"{""location"": ""TM"", ""is_mobile"": true}" 2348,4,219,2017-04-21 02:25:34,http://cartwrighthilll.co/keyon,0.0000000000,177.74.17.14,"{""location"": ""MS"", ""is_mobile"": true}" 2349,4,219,2017-03-21 18:00:34,http://walkerleannon.co/alycia,0.0000000000,208.137.247.4,"{""location"": ""PW"", ""is_mobile"": true}" 2350,4,219,2017-04-15 22:35:07,http://rennerbatz.biz/vernon,0.0000000000,162.236.80.172,"{""location"": ""SG"", ""is_mobile"": false}" 2351,4,219,2017-04-03 15:34:09,http://goldner.name/stephanie,0.0000000000,76.196.161.249,"{""location"": ""MV"", ""is_mobile"": true}" 2352,4,219,2017-03-23 22:28:33,http://ruel.com/candida_rosenbaum,0.0000000000,249.165.104.26,"{""location"": ""LB"", ""is_mobile"": true}" 2353,4,219,2017-02-15 14:46:36,http://carter.name/tristian,0.0000000000,162.236.80.172,"{""location"": ""TK"", ""is_mobile"": false}" 2354,4,219,2017-05-23 00:27:58,http://connelly.com/phoebe_blanda,0.0000000000,254.134.20.12,"{""location"": ""KR"", ""is_mobile"": true}" 2355,4,219,2017-05-23 23:53:35,http://legroscormier.info/mathew.pfannerstill,0.0000000000,86.76.161.175,"{""location"": ""BV"", ""is_mobile"": false}" 2356,4,219,2017-06-08 02:07:29,http://lynchdickinson.net/reese,0.0000000000,193.171.16.81,"{""location"": ""GS"", ""is_mobile"": false}" 2357,4,219,2017-01-04 01:41:08,http://bartellmcglynn.net/ramiro_considine,0.0000000000,233.247.128.87,"{""location"": ""IO"", ""is_mobile"": false}" 2358,4,219,2017-02-21 15:18:32,http://welch.org/lauren,0.0000000000,13.71.148.214,"{""location"": ""LR"", ""is_mobile"": false}" 2359,4,219,2017-03-02 12:07:30,http://shanahanhansen.info/muriel.turner,0.0000000000,86.76.161.175,"{""location"": ""UM"", ""is_mobile"": false}" 2360,4,219,2017-03-30 06:28:28,http://kautzer.co/candelario,0.0000000000,90.245.133.181,"{""location"": ""LY"", ""is_mobile"": false}" 2361,4,219,2017-05-15 00:07:26,http://turner.io/shanna,0.0000000000,19.193.4.175,"{""location"": ""TG"", ""is_mobile"": false}" 2362,4,219,2017-01-04 20:33:32,http://satterfieldkutch.io/emmitt_cummings,0.0000000000,203.84.214.156,"{""location"": ""BO"", ""is_mobile"": true}" 2363,4,219,2016-12-25 10:33:25,http://windler.org/clark,0.0000000000,207.118.20.154,"{""location"": ""NC"", ""is_mobile"": true}" 2364,4,220,2017-01-23 07:48:30,http://adams.biz/athena.olson,0.0000000000,87.182.47.244,"{""location"": ""EG"", ""is_mobile"": true}" 2365,4,221,2017-05-30 12:11:18,http://leffler.com/amelia.denesik,,64.189.128.171,"{""location"": ""CF"", ""is_mobile"": true}" 2366,4,221,2016-12-22 16:47:39,http://klocko.co/rubye,,247.44.138.18,"{""location"": ""MD"", ""is_mobile"": true}" 2367,4,221,2017-06-06 20:04:32,http://kuvalis.co/omari,,194.122.31.144,"{""location"": ""CC"", ""is_mobile"": true}" 2368,4,221,2017-02-16 07:30:37,http://stamm.info/mae_schneider,,46.207.39.66,"{""location"": ""SA"", ""is_mobile"": true}" 2369,4,221,2017-04-12 01:09:51,http://dietrich.com/crystel_glover,,244.25.34.97,"{""location"": ""AZ"", ""is_mobile"": true}" 2370,4,221,2016-12-19 19:16:59,http://ward.biz/gregg,,240.169.18.162,"{""location"": ""NC"", ""is_mobile"": false}" 2371,4,222,2017-03-13 04:05:07,http://runolfoncollier.info/octavia,,170.78.156.208,"{""location"": ""SG"", ""is_mobile"": false}" 2372,4,222,2017-05-06 00:18:12,http://leffler.io/trenton,,170.78.156.208,"{""location"": ""GG"", ""is_mobile"": true}" 2373,4,222,2017-06-04 22:42:07,http://ruel.com/cleta,,250.119.252.2,"{""location"": ""GH"", ""is_mobile"": true}" 2374,4,223,2017-05-03 05:03:44,http://windlergreenfelder.name/donny.dare,,41.171.106.92,"{""location"": ""MY"", ""is_mobile"": true}" 2375,4,223,2017-01-01 05:06:15,http://rogahn.info/ludwig,,147.212.182.23,"{""location"": ""SD"", ""is_mobile"": false}" 2376,4,223,2017-03-06 23:11:40,http://skileslittle.name/trevor.kuhic,,118.168.135.138,"{""location"": ""SR"", ""is_mobile"": true}" 2377,4,223,2017-04-06 17:56:40,http://gaylord.co/trever,,113.16.230.252,"{""location"": ""SY"", ""is_mobile"": false}" 2378,4,223,2017-05-24 11:33:01,http://wymanboyer.com/jasen_wolff,,110.103.42.180,"{""location"": ""BM"", ""is_mobile"": true}" 2379,4,223,2017-02-09 22:27:23,http://kilback.name/jacinthe_legros,,244.70.156.109,"{""location"": ""GE"", ""is_mobile"": false}" 2380,4,223,2017-04-12 01:27:40,http://treutel.com/stanford_jacobi,,102.157.85.223,"{""location"": ""KP"", ""is_mobile"": false}" 2381,4,223,2017-03-14 10:21:41,http://watsicabednar.org/johnpaul.leffler,,16.244.77.96,"{""location"": ""GH"", ""is_mobile"": true}" 2382,4,223,2016-12-13 09:08:15,http://conroy.com/madelynn,,196.60.176.22,"{""location"": ""AM"", ""is_mobile"": false}" 2383,4,223,2017-01-15 13:44:18,http://mitchellmarvin.name/chelsie_rice,,147.114.192.59,"{""location"": ""GT"", ""is_mobile"": false}" 2384,4,223,2016-12-27 17:56:48,http://keeblerhermiston.name/andrew_jast,,226.219.139.117,"{""location"": ""IO"", ""is_mobile"": false}" 2385,4,223,2017-03-29 08:46:34,http://schoen.net/mariah,,170.167.181.24,"{""location"": ""KH"", ""is_mobile"": true}" 2386,4,224,2017-02-21 03:16:54,http://goldnerwest.io/marisa_nader,,148.109.110.112,"{""location"": ""LY"", ""is_mobile"": false}" 2387,4,224,2017-04-25 07:20:26,http://botsford.org/telly,,77.22.107.118,"{""location"": ""BY"", ""is_mobile"": true}" 2388,4,224,2017-01-07 00:54:03,http://yundt.io/dulce,,168.224.90.232,"{""location"": ""AX"", ""is_mobile"": true}" 2389,4,224,2017-02-26 17:55:35,http://wardwilkinson.name/keara.doyle,,77.22.107.118,"{""location"": ""GR"", ""is_mobile"": true}" 2390,4,225,2017-06-06 12:28:08,http://purdy.biz/marvin,,147.163.45.139,"{""location"": ""YE"", ""is_mobile"": true}" 2391,4,225,2016-12-19 07:26:26,http://glover.biz/nickolas_haley,,47.225.162.87,"{""location"": ""CL"", ""is_mobile"": true}" 2392,4,225,2017-04-14 00:28:02,http://jacobi.biz/rick_johnson,,128.214.219.78,"{""location"": ""VN"", ""is_mobile"": true}" 2393,4,225,2017-05-28 00:01:34,http://lemke.name/eladio,,47.225.162.87,"{""location"": ""GP"", ""is_mobile"": true}" 2394,4,225,2017-02-03 12:31:12,http://christiansenschulist.net/jaiden,,25.226.250.94,"{""location"": ""BQ"", ""is_mobile"": false}" 2395,4,225,2017-04-21 13:23:08,http://fay.net/mac,,109.252.215.28,"{""location"": ""PM"", ""is_mobile"": true}" 2396,4,225,2017-04-27 04:15:36,http://morar.com/madyson.kertzmann,,3.153.18.135,"{""location"": ""NZ"", ""is_mobile"": true}" 2397,4,225,2017-05-06 00:43:51,http://turner.net/rhianna_rohan,,243.133.33.144,"{""location"": ""PL"", ""is_mobile"": true}" 2398,4,225,2017-05-06 22:25:50,http://thiel.io/effie,,6.100.99.33,"{""location"": ""CG"", ""is_mobile"": true}" 2399,4,225,2017-05-12 14:59:43,http://gutkowskiwisoky.io/houston.bauch,,196.143.119.199,"{""location"": ""CA"", ""is_mobile"": false}" 2400,4,225,2017-05-23 06:11:30,http://cummerata.net/haley,,194.213.156.176,"{""location"": ""NR"", ""is_mobile"": false}" 2401,4,225,2017-03-28 02:03:04,http://robelbins.name/loraine,,95.242.60.145,"{""location"": ""MQ"", ""is_mobile"": false}" 2402,4,225,2017-01-28 02:02:07,http://ruecker.org/maureen.hyatt,,66.85.250.254,"{""location"": ""GT"", ""is_mobile"": true}" 2403,4,225,2017-05-17 10:07:18,http://cruickshank.net/anya,,109.252.215.28,"{""location"": ""KN"", ""is_mobile"": false}" 2404,4,225,2017-01-12 03:13:20,http://orn.org/jamir_renner,,225.199.50.127,"{""location"": ""JE"", ""is_mobile"": true}" 2405,4,226,2017-04-04 01:08:24,http://erdmaneffertz.biz/rashawn.davis,,126.173.180.19,"{""location"": ""CF"", ""is_mobile"": false}" 2406,4,226,2017-06-10 14:52:26,http://kovacek.biz/brandon,,48.107.226.5,"{""location"": ""GN"", ""is_mobile"": true}" 2407,4,226,2017-05-24 15:34:34,http://jast.biz/ashleigh.heaney,,184.124.59.236,"{""location"": ""MZ"", ""is_mobile"": true}" 2408,4,226,2017-03-07 18:19:43,http://wardjacobs.net/modesta_hauck,,206.208.143.37,"{""location"": ""PS"", ""is_mobile"": true}" 2409,4,226,2017-01-15 07:39:04,http://reynolds.com/trevion,,49.8.148.227,"{""location"": ""ML"", ""is_mobile"": false}" 2410,4,226,2017-05-11 00:12:15,http://goodwinmacejkovic.info/crawford_luettgen,,242.129.254.204,"{""location"": ""NO"", ""is_mobile"": false}" 2411,4,226,2017-02-04 14:55:28,http://larsonwaters.biz/keagan,,48.107.226.5,"{""location"": ""CF"", ""is_mobile"": false}" 2412,4,226,2017-04-28 21:46:58,http://kris.io/federico,,50.252.49.178,"{""location"": ""JO"", ""is_mobile"": false}" 2413,4,226,2017-05-11 08:06:26,http://dickinsonkautzer.name/noe.koch,,126.173.180.19,"{""location"": ""VE"", ""is_mobile"": true}" 2414,4,226,2016-12-23 02:00:48,http://gutkowski.name/hazel.weimann,,242.129.254.204,"{""location"": ""MG"", ""is_mobile"": true}" 2415,4,226,2017-06-10 02:31:10,http://prohaskakulas.co/jailyn.bogan,,50.252.49.178,"{""location"": ""AX"", ""is_mobile"": true}" 2416,4,226,2017-05-21 00:37:00,http://connrohan.name/libbie,,151.78.107.195,"{""location"": ""FR"", ""is_mobile"": false}" 2417,4,226,2017-01-05 23:41:06,http://hoppe.com/tabitha,,59.132.144.74,"{""location"": ""PL"", ""is_mobile"": true}" 2418,4,226,2017-01-31 02:59:03,http://murphy.co/tomas,,183.94.30.97,"{""location"": ""BJ"", ""is_mobile"": true}" 2419,4,226,2017-05-21 03:15:53,http://thompson.org/alana_stiedemann,,48.107.226.5,"{""location"": ""AS"", ""is_mobile"": false}" 2420,4,226,2017-05-12 05:03:54,http://mckenziepfannerstill.com/kattie,,126.173.180.19,"{""location"": ""GI"", ""is_mobile"": false}" 2421,4,226,2017-02-20 00:46:32,http://schneider.io/kaylah.hackett,,165.39.79.87,"{""location"": ""MA"", ""is_mobile"": true}" 2422,4,227,2017-04-03 05:51:39,http://tremblay.biz/letitia,,229.172.214.86,"{""location"": ""CM"", ""is_mobile"": true}" 2423,4,227,2017-05-03 22:39:02,http://medhurst.co/antonio.crist,,246.63.8.85,"{""location"": ""FK"", ""is_mobile"": false}" 2424,4,227,2017-01-24 19:12:43,http://kutch.info/lester,,183.250.107.64,"{""location"": ""NF"", ""is_mobile"": false}" 2425,4,227,2017-01-21 05:12:30,http://olsongrimes.info/vilma_daniel,,71.242.165.69,"{""location"": ""MC"", ""is_mobile"": false}" 2426,4,227,2017-01-26 02:58:43,http://raustanton.info/walker,,77.176.235.3,"{""location"": ""MY"", ""is_mobile"": false}" 2427,4,227,2017-04-08 12:54:34,http://aufderharbecker.com/adela_romaguera,,46.184.215.189,"{""location"": ""AZ"", ""is_mobile"": false}" 2428,4,227,2017-02-12 07:35:05,http://connelly.io/mattie.herman,,22.139.247.42,"{""location"": ""DE"", ""is_mobile"": true}" 2429,4,227,2016-12-19 14:43:31,http://hyatt.info/malvina.keeling,,245.114.243.230,"{""location"": ""AO"", ""is_mobile"": true}" 2430,4,227,2017-06-09 12:45:02,http://schimmeldicki.com/vince,,77.176.235.3,"{""location"": ""MX"", ""is_mobile"": true}" 2431,4,227,2017-03-01 16:22:01,http://streichwiegand.name/reilly,,245.246.103.142,"{""location"": ""WS"", ""is_mobile"": true}" 2432,4,228,2017-05-30 23:42:13,http://hayes.net/mac,,102.163.59.43,"{""location"": ""US"", ""is_mobile"": true}" 2433,4,228,2017-05-25 17:16:14,http://maggio.io/annalise,,150.187.42.142,"{""location"": ""BV"", ""is_mobile"": true}" 2434,4,228,2017-03-16 01:24:36,http://haag.net/olen_labadie,,111.211.84.190,"{""location"": ""SA"", ""is_mobile"": false}" 2435,4,228,2017-05-02 23:18:36,http://monahan.com/jadyn,,152.201.210.190,"{""location"": ""YT"", ""is_mobile"": true}" 2436,4,228,2017-01-27 02:01:10,http://nienowschinner.io/lorine,,150.187.42.142,"{""location"": ""ER"", ""is_mobile"": false}" 2437,4,228,2017-05-01 12:20:02,http://trantow.com/shanelle,,10.134.51.58,"{""location"": ""IM"", ""is_mobile"": true}" 2438,4,228,2017-04-07 00:24:14,http://roobbogisich.net/reinhold.hirthe,,95.92.39.178,"{""location"": ""VN"", ""is_mobile"": false}" 2439,4,228,2017-01-02 09:34:53,http://krajcik.biz/joshua_heathcote,,249.161.218.225,"{""location"": ""SJ"", ""is_mobile"": false}" 2440,4,228,2017-04-23 02:11:35,http://willms.name/delpha.goldner,,246.182.162.33,"{""location"": ""DZ"", ""is_mobile"": true}" 2441,4,228,2017-05-05 20:11:28,http://howe.name/idella.rodriguez,,251.44.46.110,"{""location"": ""WS"", ""is_mobile"": true}" 2442,4,228,2017-04-11 03:39:56,http://lockmancarroll.org/orie,,53.145.162.254,"{""location"": ""RE"", ""is_mobile"": false}" 2443,4,228,2017-05-03 07:59:26,http://murphy.com/berniece.hudson,,233.185.78.206,"{""location"": ""BS"", ""is_mobile"": true}" 2444,4,228,2017-05-07 18:34:27,http://satterfield.com/glennie.windler,,131.101.120.43,"{""location"": ""NE"", ""is_mobile"": false}" 2445,4,228,2017-01-07 23:37:02,http://marks.co/hannah,,150.187.42.142,"{""location"": ""GW"", ""is_mobile"": false}" 2446,4,228,2017-04-06 10:22:23,http://eichmann.org/jovani,,48.111.121.64,"{""location"": ""GE"", ""is_mobile"": false}" 2447,4,228,2017-03-15 08:43:21,http://hermiston.biz/ryan_kuhic,,246.182.162.33,"{""location"": ""VG"", ""is_mobile"": false}" 2448,4,228,2017-04-18 01:03:42,http://gleason.io/marisa_turcotte,,251.44.46.110,"{""location"": ""PR"", ""is_mobile"": true}" 2449,4,228,2017-04-18 15:02:44,http://braun.net/shyann,,24.9.56.162,"{""location"": ""JP"", ""is_mobile"": true}" 2450,4,228,2016-12-20 15:15:11,http://deckow.name/thad_okon,,185.56.162.184,"{""location"": ""NA"", ""is_mobile"": false}" 2451,4,229,2017-06-13 17:14:22,http://kuphalhagenes.net/whitney,,134.98.190.242,"{""location"": ""FI"", ""is_mobile"": false}" 2452,4,229,2017-05-08 10:08:45,http://torphy.biz/germaine_lang,,31.119.228.223,"{""location"": ""HK"", ""is_mobile"": true}" 2453,4,229,2017-04-08 20:05:53,http://erdman.co/ewell,,147.2.241.125,"{""location"": ""TJ"", ""is_mobile"": true}" 2454,4,229,2017-03-17 08:55:12,http://spencer.org/elisha,,204.124.177.41,"{""location"": ""GM"", ""is_mobile"": true}" 2455,4,229,2017-06-02 16:46:47,http://hahn.org/charlene,,156.202.35.163,"{""location"": ""IN"", ""is_mobile"": true}" 2456,4,229,2017-04-29 09:02:29,http://wolfkunde.io/marcos.padberg,,46.19.254.9,"{""location"": ""BY"", ""is_mobile"": false}" 2457,4,229,2017-01-17 18:02:45,http://altenwerth.name/leie,,173.46.94.108,"{""location"": ""TK"", ""is_mobile"": false}" 2458,4,229,2017-01-22 21:09:15,http://wyman.net/miguel,,5.40.155.163,"{""location"": ""SO"", ""is_mobile"": true}" 2459,4,229,2017-03-10 23:10:49,http://ryan.org/liza,,136.222.20.139,"{""location"": ""GR"", ""is_mobile"": false}" 2460,4,229,2016-12-25 21:34:45,http://maggio.name/moriah_romaguera,,150.178.251.183,"{""location"": ""BI"", ""is_mobile"": false}" 2461,4,229,2017-04-17 22:55:53,http://schultz.info/eduardo_stamm,,103.63.22.112,"{""location"": ""SG"", ""is_mobile"": true}" 2462,4,229,2017-06-04 23:55:29,http://treutelemard.com/vivian.kunde,,120.2.15.157,"{""location"": ""QA"", ""is_mobile"": true}" 2463,4,229,2017-05-11 08:27:16,http://mullerkoch.org/dimitri.denesik,,222.45.21.236,"{""location"": ""CV"", ""is_mobile"": true}" 2464,4,229,2017-02-27 08:56:07,http://mantejaskolski.org/eula_bogisich,,66.68.174.125,"{""location"": ""AO"", ""is_mobile"": false}" 2465,4,229,2017-01-19 15:04:08,http://balistreri.net/stone,,204.124.177.41,"{""location"": ""BH"", ""is_mobile"": false}" 2466,4,229,2017-02-13 21:55:15,http://blanda.co/felix.graham,,235.215.198.129,"{""location"": ""DO"", ""is_mobile"": true}" 2467,4,230,2017-04-08 21:19:04,http://reichel.org/fausto.keler,,254.67.12.95,"{""location"": ""KN"", ""is_mobile"": false}" 2468,4,230,2017-03-26 08:26:36,http://moore.com/waldo,,4.94.207.219,"{""location"": ""SV"", ""is_mobile"": false}" 2469,4,230,2017-06-05 15:46:35,http://vandervort.net/florencio.sipes,,12.213.154.195,"{""location"": ""KH"", ""is_mobile"": false}" 2470,4,230,2017-04-18 21:47:20,http://mann.com/robb_kerluke,,74.20.103.28,"{""location"": ""AL"", ""is_mobile"": false}" 2471,4,230,2017-04-24 13:52:02,http://lowe.io/myles_beatty,,250.116.210.220,"{""location"": ""KP"", ""is_mobile"": false}" 2472,4,230,2017-01-10 02:35:42,http://rath.io/jaida,,12.213.154.195,"{""location"": ""UZ"", ""is_mobile"": false}" 2473,4,230,2017-05-07 05:59:24,http://mcglynn.info/kallie_feeney,,162.115.223.126,"{""location"": ""CX"", ""is_mobile"": true}" 2474,4,230,2016-12-18 00:43:03,http://schamberger.org/andy_mraz,,127.100.141.72,"{""location"": ""CI"", ""is_mobile"": false}" 2475,4,230,2017-05-29 13:11:39,http://quigley.info/colby.nienow,,242.166.239.65,"{""location"": ""BR"", ""is_mobile"": false}" 2476,4,230,2017-03-09 13:15:27,http://lubowitzbahringer.co/desiree,,117.186.222.52,"{""location"": ""MA"", ""is_mobile"": false}" 2477,4,230,2017-05-03 01:22:44,http://lehner.info/lois,,162.115.223.126,"{""location"": ""NO"", ""is_mobile"": false}" 2478,4,230,2017-04-02 16:52:32,http://feestfriesen.name/vincenzo,,85.235.200.5,"{""location"": ""DJ"", ""is_mobile"": true}" 2479,4,230,2017-01-18 13:06:20,http://orn.com/lukas.rogahn,,113.2.154.190,"{""location"": ""GB"", ""is_mobile"": true}" 2480,4,230,2017-03-22 23:12:02,http://gibson.io/jerad,,97.20.112.91,"{""location"": ""SV"", ""is_mobile"": false}" 2481,4,230,2017-05-04 23:32:22,http://greenholt.co/dasia_wisoky,,42.144.172.62,"{""location"": ""IE"", ""is_mobile"": true}" 2482,4,230,2017-03-21 23:46:37,http://quigleystoltenberg.io/anderson_padberg,,4.94.207.219,"{""location"": ""FI"", ""is_mobile"": false}" 2483,4,230,2017-02-15 02:15:40,http://hoegereichmann.co/katelynn,,160.143.224.50,"{""location"": ""NC"", ""is_mobile"": true}" 2484,4,230,2017-06-11 10:17:02,http://emard.com/efrain.breitenberg,,197.197.70.46,"{""location"": ""SX"", ""is_mobile"": false}" 2485,4,230,2017-05-23 06:01:47,http://buckridge.com/shanny,,127.100.141.72,"{""location"": ""BG"", ""is_mobile"": true}" 2486,4,231,2016-12-27 06:00:54,http://schmitt.co/lurline.prosacco,,67.234.183.196,"{""location"": ""SM"", ""is_mobile"": false}" 2487,4,231,2017-03-09 16:51:12,http://gloverhuel.name/teie.reichert,,63.245.221.231,"{""location"": ""SC"", ""is_mobile"": false}" 2488,4,231,2017-05-13 20:36:30,http://roobroob.info/hilton,,35.150.108.209,"{""location"": ""UM"", ""is_mobile"": true}" 2489,4,231,2017-02-05 03:20:17,http://abernathychamplin.org/lamar.feest,,35.150.108.209,"{""location"": ""LR"", ""is_mobile"": false}" 2490,4,231,2017-02-19 01:02:40,http://bergstromokon.info/tyrell_bernier,,36.81.11.69,"{""location"": ""KZ"", ""is_mobile"": true}" 2491,4,231,2017-04-03 04:27:25,http://macgyver.io/kara,,84.73.126.185,"{""location"": ""PA"", ""is_mobile"": false}" 2492,4,231,2017-05-30 06:37:29,http://rogahnfranecki.com/jazmin,,234.115.69.241,"{""location"": ""JO"", ""is_mobile"": true}" 2493,4,231,2017-03-02 19:17:18,http://upton.net/flo.ohara,,84.73.126.185,"{""location"": ""SL"", ""is_mobile"": true}" 2494,4,231,2017-05-07 07:25:29,http://hudsonmcglynn.info/dale.bernhard,,112.127.81.170,"{""location"": ""LS"", ""is_mobile"": false}" 2495,4,231,2016-12-20 17:40:30,http://kub.biz/oswaldo,,144.43.100.129,"{""location"": ""BN"", ""is_mobile"": false}" 2496,4,231,2017-01-05 00:32:46,http://webergulgowski.biz/filiberto.runolfon,,64.156.153.54,"{""location"": ""VN"", ""is_mobile"": true}" 2497,4,231,2017-01-05 08:18:17,http://lakin.io/edmund_klein,,179.48.56.20,"{""location"": ""BL"", ""is_mobile"": false}" 2498,4,232,2017-03-20 03:40:46,http://steuberhills.io/lilian_sporer,,190.109.227.162,"{""location"": ""LR"", ""is_mobile"": false}" 2499,4,232,2017-04-30 02:04:43,http://kub.biz/quentin.baumbach,,72.156.122.52,"{""location"": ""US"", ""is_mobile"": true}" 2500,4,232,2017-06-02 11:14:55,http://kreiger.com/ignacio.strosin,,127.195.248.179,"{""location"": ""BG"", ""is_mobile"": false}" 2501,4,232,2017-02-02 05:59:25,http://dietrich.net/keara.schroeder,,14.68.124.42,"{""location"": ""NG"", ""is_mobile"": true}" 2502,4,232,2017-04-01 20:58:10,http://klein.biz/everette,,190.109.227.162,"{""location"": ""CW"", ""is_mobile"": true}" 2503,4,232,2017-02-05 19:19:55,http://white.org/ceasar,,190.109.227.162,"{""location"": ""MP"", ""is_mobile"": false}" 2504,4,232,2017-03-20 23:02:38,http://bradtke.info/jacey,,38.92.131.122,"{""location"": ""GF"", ""is_mobile"": true}" 2505,4,232,2017-05-06 12:27:28,http://ziemann.name/delmer,,230.237.191.118,"{""location"": ""TJ"", ""is_mobile"": false}" 2506,4,232,2017-01-27 20:16:28,http://krisbins.info/marjorie,,210.34.74.19,"{""location"": ""ID"", ""is_mobile"": true}" 2507,4,232,2017-03-31 21:12:07,http://rogahn.com/pasquale_bernier,,38.92.131.122,"{""location"": ""VI"", ""is_mobile"": true}" 2508,4,232,2017-04-09 20:18:53,http://haagpfannerstill.co/imani,,156.14.220.246,"{""location"": ""TL"", ""is_mobile"": true}" 2509,4,232,2017-01-24 10:15:51,http://schiller.biz/jaycee_kuhn,,116.218.67.69,"{""location"": ""NL"", ""is_mobile"": false}" 2510,4,233,2017-04-21 19:11:39,http://halvorson.name/paxton.beahan,,33.226.56.126,"{""location"": ""TK"", ""is_mobile"": true}" 2511,4,233,2017-04-28 12:39:18,http://klockohickle.co/fermin_champlin,,244.135.113.8,"{""location"": ""HK"", ""is_mobile"": false}" 2512,4,233,2017-01-23 09:47:10,http://bartellmccullough.io/lucinda,,18.5.127.184,"{""location"": ""DZ"", ""is_mobile"": true}" 2513,4,233,2017-05-28 19:10:47,http://schimmel.co/mateo,,27.55.144.248,"{""location"": ""IQ"", ""is_mobile"": true}" 2514,4,233,2017-04-09 10:31:55,http://wisozk.com/vivianne,,48.60.144.104,"{""location"": ""LU"", ""is_mobile"": false}" 2515,4,233,2017-05-03 21:25:45,http://okeefecummings.com/dee.rempel,,62.148.139.173,"{""location"": ""PY"", ""is_mobile"": false}" 2516,4,233,2017-02-21 18:50:50,http://okuneva.io/viva.feil,,28.108.73.63,"{""location"": ""TD"", ""is_mobile"": false}" 2517,4,233,2017-03-30 08:55:58,http://ankundingparisian.biz/sister_wisozk,,121.223.105.159,"{""location"": ""MK"", ""is_mobile"": true}" 2518,4,233,2017-06-08 12:08:51,http://bartonhowe.com/graciela.kshlerin,,145.142.190.60,"{""location"": ""EH"", ""is_mobile"": true}" 1337,3,126,2017-06-11 16:46:29,http://gleichneraltenwerth.net/karolann_lindgren,0.0000000000,250.206.28.76,"{""location"": ""VU"", ""is_mobile"": false}" 1338,3,126,2017-04-18 19:59:06,http://kohlertoy.name/aliza.hilll,0.0000000000,18.97.64.164,"{""location"": ""LA"", ""is_mobile"": true}" 1339,3,126,2016-12-20 22:56:28,http://batzlowe.com/cesar,1.0000000000,25.14.201.92,"{""location"": ""RW"", ""is_mobile"": true}" 1340,3,126,2017-03-17 08:16:53,http://abernathy.com/edwin,1.0000000000,194.238.112.173,"{""location"": ""BJ"", ""is_mobile"": true}" 1341,3,126,2017-02-18 17:33:26,http://osinski.com/kitty,0.0000000000,68.18.167.224,"{""location"": ""CH"", ""is_mobile"": false}" 1342,3,126,2016-12-24 08:46:50,http://osinski.biz/christop_torphy,1.0000000000,7.77.133.61,"{""location"": ""GG"", ""is_mobile"": false}" 1343,3,126,2017-04-24 13:23:50,http://greenholt.net/tristian_cronin,0.0000000000,98.192.86.187,"{""location"": ""US"", ""is_mobile"": false}" 1344,3,126,2017-04-29 00:37:57,http://cummings.com/devan,0.0000000000,98.192.86.187,"{""location"": ""CF"", ""is_mobile"": true}" 1345,3,126,2017-01-22 07:59:18,http://jast.co/letha_tremblay,1.0000000000,188.232.224.103,"{""location"": ""TR"", ""is_mobile"": true}" 1346,3,126,2017-03-27 00:28:21,http://gottlieb.biz/brooks_gislason,0.0000000000,217.67.28.93,"{""location"": ""KW"", ""is_mobile"": false}" 1347,3,126,2017-02-21 18:50:36,http://bailey.com/marquise,1.0000000000,142.43.172.147,"{""location"": ""ZA"", ""is_mobile"": false}" 1348,3,126,2017-06-06 12:12:49,http://jerde.net/samir,0.0000000000,7.77.133.61,"{""location"": ""TV"", ""is_mobile"": false}" 1349,3,126,2017-04-28 10:05:30,http://quitzon.biz/green,1.0000000000,52.2.218.176,"{""location"": ""LK"", ""is_mobile"": false}" 1350,3,126,2017-04-19 15:25:44,http://gloverhalvorson.co/trea_leannon,1.0000000000,20.140.219.112,"{""location"": ""SO"", ""is_mobile"": true}" 1351,3,126,2017-02-04 13:33:41,http://trantow.com/jimmy_barrows,0.0000000000,52.153.156.78,"{""location"": ""LI"", ""is_mobile"": true}" 1352,3,126,2017-02-21 04:24:02,http://bernier.info/yesenia.olson,1.0000000000,52.153.156.78,"{""location"": ""MW"", ""is_mobile"": false}" 1353,3,126,2017-01-20 07:30:33,http://altenwerth.name/hoyt,1.0000000000,111.219.40.19,"{""location"": ""LV"", ""is_mobile"": false}" 1354,3,126,2017-03-15 02:15:42,http://bartoletti.biz/karlie.schamberger,1.0000000000,247.254.202.254,"{""location"": ""TT"", ""is_mobile"": true}" 1355,3,126,2017-04-24 15:32:47,http://mckenziefahey.info/erica.cruickshank,0.0000000000,250.206.28.76,"{""location"": ""RS"", ""is_mobile"": false}" 1356,3,127,2017-01-01 10:51:17,http://maggiocarroll.name/ludie.conroy,3.0000000000,174.254.240.70,"{""location"": ""NL"", ""is_mobile"": false}" 1357,3,127,2017-03-13 17:23:54,http://schoenwest.info/christina.thompson,3.0000000000,62.175.124.96,"{""location"": ""ZM"", ""is_mobile"": true}" 1358,3,127,2017-05-29 10:47:35,http://towne.org/kaden,0.0000000000,130.251.188.145,"{""location"": ""GY"", ""is_mobile"": false}" 1359,3,127,2017-05-09 19:38:17,http://kshlerin.io/ola_berge,3.0000000000,246.250.199.6,"{""location"": ""KH"", ""is_mobile"": true}" 1360,3,127,2017-04-09 09:25:20,http://keeling.org/glenna,1.0000000000,111.209.128.75,"{""location"": ""CV"", ""is_mobile"": false}" 1361,3,127,2017-04-18 02:09:17,http://price.com/chaya.macgyver,3.0000000000,202.174.202.86,"{""location"": ""CN"", ""is_mobile"": true}" 1362,3,128,2016-12-16 00:32:19,http://thiel.org/dulce,0.0000000000,187.8.177.13,"{""location"": ""KE"", ""is_mobile"": true}" 1363,3,128,2016-12-28 10:51:18,http://krajcikjaskolski.co/marielle.stanton,3.0000000000,145.169.253.252,"{""location"": ""ME"", ""is_mobile"": true}" 1364,3,128,2017-04-23 03:58:21,http://dach.name/sven.glover,3.0000000000,115.193.43.30,"{""location"": ""DK"", ""is_mobile"": true}" 1365,3,128,2016-12-23 14:39:42,http://schmidtebert.biz/sonia.kris,1.0000000000,108.241.47.103,"{""location"": ""PE"", ""is_mobile"": true}" 1366,3,128,2017-05-11 14:40:15,http://fahey.biz/korey,3.0000000000,176.100.50.231,"{""location"": ""MQ"", ""is_mobile"": true}" 1367,3,128,2017-01-15 10:27:20,http://braun.com/halle_ernser,0.0000000000,163.57.126.241,"{""location"": ""MK"", ""is_mobile"": false}" 1368,3,128,2017-05-19 13:53:08,http://lang.net/laney,3.0000000000,98.13.92.24,"{""location"": ""KE"", ""is_mobile"": false}" 1369,3,128,2017-02-17 21:35:39,http://wymanlebsack.name/jaclyn,3.0000000000,19.45.61.239,"{""location"": ""MK"", ""is_mobile"": true}" 1370,3,128,2017-01-10 15:08:47,http://nitzsche.info/rashawn,0.0000000000,203.237.152.66,"{""location"": ""UY"", ""is_mobile"": false}" 1371,3,128,2017-02-10 08:17:40,http://spinkabaumbach.com/pinkie.little,0.0000000000,169.6.103.14,"{""location"": ""VI"", ""is_mobile"": false}" 1372,3,128,2017-01-18 23:22:17,http://considinedooley.net/rosalind,2.0000000000,85.181.39.92,"{""location"": ""GL"", ""is_mobile"": true}" 1373,3,129,2017-03-30 09:43:04,http://mcdermott.com/shaina,0.0000000000,113.250.78.227,"{""location"": ""HT"", ""is_mobile"": false}" 1374,3,129,2017-05-27 21:40:42,http://balistreri.net/enrico.glover,0.0000000000,24.138.154.4,"{""location"": ""BL"", ""is_mobile"": true}" 1375,3,129,2017-02-26 22:47:34,http://jacobsjast.io/bertha,0.0000000000,14.126.56.193,"{""location"": ""ZM"", ""is_mobile"": true}" 1376,3,129,2017-05-05 21:09:03,http://bailey.org/tod.pagac,0.0000000000,63.10.23.107,"{""location"": ""SO"", ""is_mobile"": false}" 1377,3,129,2016-12-28 10:03:27,http://balistreri.co/caesar.leannon,0.0000000000,215.203.184.70,"{""location"": ""LA"", ""is_mobile"": false}" 1378,3,129,2017-04-05 06:45:41,http://lang.net/jayde.harris,0.0000000000,78.94.13.117,"{""location"": ""HN"", ""is_mobile"": true}" 1379,3,129,2017-03-31 06:41:28,http://eichmann.org/simeon,0.0000000000,197.53.209.107,"{""location"": ""MX"", ""is_mobile"": false}" 1380,3,129,2017-05-08 14:18:28,http://klocko.co/lori.carroll,0.0000000000,78.190.208.75,"{""location"": ""HN"", ""is_mobile"": false}" 1381,3,129,2016-12-21 22:03:46,http://conroyboyer.org/maureen_abernathy,0.0000000000,140.31.60.102,"{""location"": ""SI"", ""is_mobile"": false}" 1382,3,129,2017-01-15 15:22:19,http://eichmann.org/anne_gutmann,0.0000000000,165.198.5.172,"{""location"": ""KY"", ""is_mobile"": false}" 1383,3,129,2017-03-16 02:35:55,http://beattygraham.com/rasheed,0.0000000000,143.3.135.235,"{""location"": ""AE"", ""is_mobile"": true}" 1384,3,129,2017-04-20 00:32:53,http://franecki.net/cornelius_sanford,0.0000000000,131.54.76.24,"{""location"": ""CO"", ""is_mobile"": true}" 1385,3,129,2016-12-31 12:43:40,http://zieme.net/zackery.doyle,0.0000000000,27.235.235.131,"{""location"": ""NL"", ""is_mobile"": false}" 2519,4,233,2017-03-04 15:24:54,http://cronin.name/kay.paucek,,60.33.84.162,"{""location"": ""LA"", ""is_mobile"": false}" 2520,4,233,2016-12-14 23:41:31,http://gaylord.com/betty,,27.55.144.248,"{""location"": ""NG"", ""is_mobile"": false}" 2521,4,233,2017-02-23 16:38:07,http://roberts.info/sabryna,,85.55.40.230,"{""location"": ""JO"", ""is_mobile"": false}" 2522,4,233,2017-01-22 15:58:37,http://wyman.org/lavern_nicolas,,145.82.54.97,"{""location"": ""VG"", ""is_mobile"": false}" 2523,4,233,2016-12-29 21:29:24,http://kutch.net/lydia,,33.226.56.126,"{""location"": ""BT"", ""is_mobile"": false}" 2524,4,233,2017-03-06 11:14:45,http://conn.info/luciano_stanton,,48.141.147.8,"{""location"": ""BV"", ""is_mobile"": false}" 2525,4,234,2017-02-03 10:30:49,http://skiles.info/alivia_anderson,,171.206.135.227,"{""location"": ""GP"", ""is_mobile"": true}" 2526,4,234,2017-04-08 12:45:38,http://kertzmann.biz/cameron.simonis,,233.94.20.173,"{""location"": ""MR"", ""is_mobile"": false}" 2527,4,234,2017-01-31 09:36:46,http://kleinmedhurst.io/caria_corkery,,240.40.43.191,"{""location"": ""LA"", ""is_mobile"": true}" 2528,4,234,2017-06-02 06:26:44,http://dickens.org/alisha.roberts,,31.49.30.143,"{""location"": ""PT"", ""is_mobile"": true}" 2529,4,234,2017-01-27 19:11:13,http://pfefferjacobs.com/torrance,,11.217.42.229,"{""location"": ""AE"", ""is_mobile"": true}" 2530,4,234,2017-01-07 07:14:28,http://larson.net/ozzie_gleason,,149.169.115.190,"{""location"": ""SS"", ""is_mobile"": false}" 2531,4,235,2017-03-26 00:36:58,http://corwin.io/carol,,174.72.201.138,"{""location"": ""IT"", ""is_mobile"": false}" 2532,4,235,2017-05-10 08:43:30,http://boyle.name/bobbie_okuneva,,64.125.162.102,"{""location"": ""IM"", ""is_mobile"": false}" 2533,4,235,2016-12-14 17:01:56,http://hellerconroy.com/major,,64.125.162.102,"{""location"": ""BG"", ""is_mobile"": true}" 2534,4,235,2017-02-19 15:00:32,http://gutkowski.io/wallace.rath,,20.241.156.163,"{""location"": ""BT"", ""is_mobile"": false}" 2535,4,235,2016-12-31 20:43:53,http://paucekheathcote.name/christy,,107.144.39.193,"{""location"": ""WS"", ""is_mobile"": true}" 2536,4,235,2016-12-26 23:18:49,http://flatleykoepp.net/edwin_ondricka,,74.32.185.166,"{""location"": ""TR"", ""is_mobile"": true}" 2537,4,235,2017-01-06 12:48:57,http://rice.biz/adelle,,63.71.98.67,"{""location"": ""NZ"", ""is_mobile"": true}" 2538,4,235,2016-12-19 20:02:31,http://balistreriterry.io/rocio.wuckert,,40.7.158.85,"{""location"": ""GM"", ""is_mobile"": false}" 2539,4,236,2017-04-11 22:48:04,http://predovicrice.name/harmon,,59.170.6.150,"{""location"": ""JO"", ""is_mobile"": false}" 2540,4,236,2017-03-16 13:44:54,http://gutkowski.name/christop_orn,,234.20.224.247,"{""location"": ""CW"", ""is_mobile"": false}" 2541,4,236,2017-05-16 17:09:09,http://ward.info/isom,,107.249.200.220,"{""location"": ""CH"", ""is_mobile"": true}" 2542,4,236,2017-02-09 02:29:17,http://heller.name/alvah_rolfson,,222.171.193.205,"{""location"": ""GE"", ""is_mobile"": false}" 2543,4,236,2016-12-25 16:20:09,http://cummings.com/hunter_hackett,,200.203.135.164,"{""location"": ""DZ"", ""is_mobile"": false}" 2544,4,236,2017-02-18 07:04:31,http://blandamedhurst.biz/arlie,,27.51.186.72,"{""location"": ""SR"", ""is_mobile"": false}" 2545,4,236,2017-03-07 11:12:53,http://tremblayschowalter.info/hortense,,118.180.197.210,"{""location"": ""VE"", ""is_mobile"": true}" 2546,4,236,2017-02-03 07:28:01,http://thompsonsanford.io/jamaal,,234.20.224.247,"{""location"": ""BR"", ""is_mobile"": true}" 2547,4,236,2017-03-31 19:31:21,http://ratke.info/lucienne,,216.61.151.27,"{""location"": ""SZ"", ""is_mobile"": true}" 2548,4,236,2017-03-19 02:58:43,http://shanahan.net/maria_wisoky,,147.162.155.188,"{""location"": ""CF"", ""is_mobile"": false}" 2549,4,236,2017-01-14 14:08:57,http://hettinger.net/ivah.murazik,,245.242.225.5,"{""location"": ""AO"", ""is_mobile"": true}" 2550,4,236,2017-03-30 07:49:37,http://yundt.info/lucile_grady,,64.120.197.228,"{""location"": ""SV"", ""is_mobile"": false}" 2551,4,236,2016-12-21 22:25:48,http://howe.com/jolie.bruen,,107.249.200.220,"{""location"": ""TR"", ""is_mobile"": false}" 2552,4,236,2016-12-22 05:03:32,http://murphy.info/tevin,,175.51.241.162,"{""location"": ""CN"", ""is_mobile"": true}" 2553,4,236,2017-04-27 20:25:44,http://predovicwhite.com/ansel_huels,,109.109.27.169,"{""location"": ""JP"", ""is_mobile"": false}" 2554,4,236,2017-04-15 04:40:40,http://wilkinson.io/danny.wilkinson,,92.77.115.111,"{""location"": ""SR"", ""is_mobile"": false}" 2555,4,236,2017-05-19 17:08:54,http://yost.name/denis,,144.6.189.169,"{""location"": ""IT"", ""is_mobile"": true}" 2556,4,237,2016-12-26 19:59:24,http://kuhn.info/aleia_frami,,76.203.228.34,"{""location"": ""NG"", ""is_mobile"": false}" 2557,4,237,2017-01-04 22:17:28,http://littlecruickshank.net/layla,,24.87.25.103,"{""location"": ""SN"", ""is_mobile"": false}" 2558,4,237,2017-03-22 09:50:08,http://rau.org/leonard.kuhic,,228.157.247.103,"{""location"": ""PF"", ""is_mobile"": true}" 2559,4,237,2017-01-08 15:43:32,http://wyman.biz/marion_west,,194.83.95.133,"{""location"": ""GR"", ""is_mobile"": true}" 2560,4,237,2017-04-25 11:47:58,http://hane.biz/dale,,113.173.8.6,"{""location"": ""GU"", ""is_mobile"": true}" 2561,4,238,2017-04-09 22:07:41,http://luettgen.io/kacey.reynolds,2.0000000000,180.241.27.97,"{""location"": ""JE"", ""is_mobile"": true}" 2562,4,238,2017-03-25 15:52:14,http://heel.io/nicholaus_kuphal,2.0000000000,144.236.7.79,"{""location"": ""EG"", ""is_mobile"": true}" 2563,4,238,2017-02-25 09:36:09,http://walsh.name/hugh.pfannerstill,1.0000000000,183.128.67.113,"{""location"": ""GL"", ""is_mobile"": true}" 2564,4,238,2017-02-27 14:13:39,http://bednar.name/rick.bashirian,0.0000000000,171.70.63.217,"{""location"": ""FM"", ""is_mobile"": false}" 2565,4,238,2016-12-24 02:11:15,http://sawayn.info/francesca.halvorson,2.0000000000,213.173.204.154,"{""location"": ""NA"", ""is_mobile"": false}" 2566,4,238,2017-02-28 12:35:20,http://schuster.biz/monique,1.0000000000,100.58.122.219,"{""location"": ""SX"", ""is_mobile"": false}" 2567,4,238,2017-05-29 08:45:33,http://rohan.com/aaron,2.0000000000,228.62.99.73,"{""location"": ""FJ"", ""is_mobile"": true}" 2568,4,238,2017-01-20 16:25:38,http://ankunding.com/selena,2.0000000000,41.102.81.148,"{""location"": ""GT"", ""is_mobile"": true}" 2569,4,238,2017-01-21 08:30:17,http://casper.info/edmund.berge,0.0000000000,135.175.106.238,"{""location"": ""ZM"", ""is_mobile"": true}" 2570,4,238,2017-05-17 07:01:31,http://mills.io/hilma.rice,0.0000000000,206.161.86.238,"{""location"": ""ER"", ""is_mobile"": false}" 2571,4,238,2017-03-18 05:19:38,http://marks.co/kacie,1.0000000000,206.161.86.238,"{""location"": ""MN"", ""is_mobile"": false}" 2572,4,238,2017-02-01 00:55:52,http://koepp.info/jakob,2.0000000000,147.217.116.92,"{""location"": ""AI"", ""is_mobile"": false}" 1386,3,129,2017-03-24 02:53:34,http://runte.com/alberta,0.0000000000,190.229.212.181,"{""location"": ""UA"", ""is_mobile"": true}" 1387,3,130,2017-05-08 03:33:49,http://gusikowski.io/vella_greenfelder,0.0000000000,48.30.229.215,"{""location"": ""PS"", ""is_mobile"": true}" 1388,3,130,2017-05-21 04:17:14,http://eichmannhammes.io/glenna.gleichner,0.0000000000,178.222.46.111,"{""location"": ""CR"", ""is_mobile"": false}" 1389,3,130,2017-04-05 09:33:11,http://little.com/jordi_kunze,0.0000000000,48.30.229.215,"{""location"": ""YT"", ""is_mobile"": true}" 1390,3,130,2017-01-07 06:20:05,http://vonmueller.co/forrest,1.0000000000,207.183.44.217,"{""location"": ""MS"", ""is_mobile"": true}" 1391,3,130,2017-05-19 06:04:44,http://ruel.co/malcolm,0.0000000000,219.243.74.128,"{""location"": ""PA"", ""is_mobile"": false}" 1392,3,130,2017-03-04 21:06:19,http://keeling.io/hellen.beahan,1.0000000000,142.115.219.164,"{""location"": ""PM"", ""is_mobile"": true}" 1393,3,130,2017-02-01 01:35:56,http://towne.net/charlene,0.0000000000,174.241.253.18,"{""location"": ""MF"", ""is_mobile"": true}" 1394,3,130,2017-04-12 02:52:10,http://kling.com/dangelo.corwin,1.0000000000,48.30.229.215,"{""location"": ""KE"", ""is_mobile"": true}" 1395,3,130,2017-04-11 20:01:23,http://mohr.net/rebeka_trantow,1.0000000000,159.126.57.66,"{""location"": ""WF"", ""is_mobile"": true}" 1396,3,130,2017-02-05 18:47:22,http://kunde.net/shawna,1.0000000000,218.205.249.198,"{""location"": ""CW"", ""is_mobile"": false}" 1397,3,130,2017-06-08 23:54:21,http://roobosinski.name/kamren.gottlieb,0.0000000000,217.36.85.81,"{""location"": ""GS"", ""is_mobile"": true}" 1398,3,130,2017-01-24 23:59:51,http://mohr.co/maia,1.0000000000,221.29.144.166,"{""location"": ""UM"", ""is_mobile"": true}" 1399,3,130,2017-06-09 10:58:48,http://turner.net/kenya.swaniawski,1.0000000000,164.69.201.39,"{""location"": ""RE"", ""is_mobile"": false}" 1400,3,130,2016-12-24 01:55:28,http://schmidt.info/cameron.klein,0.0000000000,115.204.176.89,"{""location"": ""NF"", ""is_mobile"": false}" 1401,3,131,2017-06-09 00:13:56,http://bergnaum.net/wiley,3.0000000000,216.238.223.10,"{""location"": ""RO"", ""is_mobile"": false}" 1402,3,131,2017-02-26 03:44:44,http://damore.org/kristina,1.0000000000,73.50.23.80,"{""location"": ""BO"", ""is_mobile"": true}" 1403,3,131,2017-02-14 11:26:07,http://moen.net/carroll_swaniawski,0.0000000000,87.68.195.35,"{""location"": ""SN"", ""is_mobile"": false}" 1404,3,131,2016-12-23 06:10:45,http://gibson.net/milford.schiller,2.0000000000,43.87.66.46,"{""location"": ""SS"", ""is_mobile"": true}" 1405,3,131,2017-05-13 00:02:23,http://hettingerwisozk.co/aliza_boyle,2.0000000000,216.238.223.10,"{""location"": ""BD"", ""is_mobile"": true}" 1406,3,131,2017-04-22 11:08:54,http://funk.biz/adelle.rath,1.0000000000,109.12.47.27,"{""location"": ""BZ"", ""is_mobile"": false}" 1407,3,131,2017-06-01 17:28:34,http://ziemann.com/kayli.oreilly,1.0000000000,23.86.247.148,"{""location"": ""GG"", ""is_mobile"": true}" 1408,3,131,2017-06-03 09:45:03,http://andersonromaguera.net/vernie_schmitt,2.0000000000,149.133.43.209,"{""location"": ""MO"", ""is_mobile"": true}" 1409,3,131,2017-03-11 22:41:06,http://krajcikwalter.com/reinhold.kling,1.0000000000,197.94.159.119,"{""location"": ""SV"", ""is_mobile"": true}" 1410,3,131,2017-04-10 05:04:49,http://harveywaters.net/jefferey_kunde,1.0000000000,43.87.66.46,"{""location"": ""US"", ""is_mobile"": false}" 1411,3,131,2017-01-22 13:39:31,http://gleason.org/deshawn,0.0000000000,109.12.47.27,"{""location"": ""RU"", ""is_mobile"": false}" 1412,3,131,2017-03-18 04:57:00,http://stoltenberg.io/gillian,2.0000000000,150.200.120.68,"{""location"": ""CC"", ""is_mobile"": false}" 1413,3,131,2017-06-08 07:02:59,http://lubowitz.org/virgil.white,2.0000000000,150.231.205.225,"{""location"": ""GN"", ""is_mobile"": false}" 1414,3,131,2017-04-16 00:24:57,http://cummings.co/quentin,1.0000000000,73.50.23.80,"{""location"": ""PR"", ""is_mobile"": true}" 1415,3,131,2017-06-11 16:57:12,http://beahan.biz/toni,0.0000000000,149.44.193.213,"{""location"": ""BI"", ""is_mobile"": false}" 1416,3,131,2017-05-24 05:49:29,http://ko.com/hattie_keler,3.0000000000,190.111.202.213,"{""location"": ""RW"", ""is_mobile"": true}" 1417,3,131,2017-05-14 05:19:05,http://yundt.biz/jose_ward,2.0000000000,149.133.43.209,"{""location"": ""SR"", ""is_mobile"": false}" 1418,3,131,2017-05-03 01:04:36,http://dubuque.name/clair.nolan,2.0000000000,150.200.120.68,"{""location"": ""MF"", ""is_mobile"": true}" 1419,3,132,2016-12-13 10:56:16,http://fritschrodriguez.org/anahi,0.0000000000,152.118.62.27,"{""location"": ""MU"", ""is_mobile"": true}" 1420,3,132,2017-05-23 08:56:53,http://collier.org/eldridge,0.0000000000,45.252.254.246,"{""location"": ""JO"", ""is_mobile"": false}" 1421,3,132,2017-06-05 10:00:58,http://altenwerthwunsch.net/bryana.zulauf,0.0000000000,170.28.127.189,"{""location"": ""KH"", ""is_mobile"": true}" 1422,3,132,2017-05-25 20:55:58,http://padberglebsack.org/myrtice_konopelski,0.0000000000,208.19.231.234,"{""location"": ""GN"", ""is_mobile"": false}" 1423,3,132,2017-01-07 01:01:18,http://durgancrona.co/ottis,0.0000000000,208.19.231.234,"{""location"": ""KG"", ""is_mobile"": false}" 1424,3,132,2017-03-06 21:58:27,http://wisozkkling.com/xander_pollich,0.0000000000,169.248.240.41,"{""location"": ""MZ"", ""is_mobile"": true}" 1425,3,132,2017-02-07 14:27:47,http://pollichledner.net/josianne.sporer,0.0000000000,10.143.72.213,"{""location"": ""FR"", ""is_mobile"": true}" 1426,3,132,2017-06-02 13:18:45,http://macejkovic.com/treie.terry,0.0000000000,234.61.199.61,"{""location"": ""TK"", ""is_mobile"": true}" 1427,3,132,2017-06-04 06:14:19,http://haagwolff.com/arnoldo_torphy,0.0000000000,87.68.10.254,"{""location"": ""DO"", ""is_mobile"": false}" 1428,3,132,2017-04-12 22:54:30,http://anderson.net/alycia,0.0000000000,166.53.86.25,"{""location"": ""CV"", ""is_mobile"": false}" 1429,3,133,2017-05-29 14:10:28,http://armstrongherman.co/jazlyn,0.0000000000,12.249.173.224,"{""location"": ""IQ"", ""is_mobile"": false}" 1430,3,133,2017-05-14 07:57:28,http://barton.name/terence,2.0000000000,199.36.136.61,"{""location"": ""KE"", ""is_mobile"": true}" 1431,3,134,2017-04-15 18:00:59,http://schmitt.io/judy_bosco,3.0000000000,144.25.165.86,"{""location"": ""TW"", ""is_mobile"": false}" 1432,3,134,2017-02-07 23:40:53,http://bogisichpowlowski.name/antwan.effertz,0.0000000000,46.43.193.176,"{""location"": ""HM"", ""is_mobile"": true}" 1433,3,134,2017-03-18 18:51:57,http://watsica.com/jewel,1.0000000000,142.45.15.54,"{""location"": ""DJ"", ""is_mobile"": false}" 1434,3,134,2017-03-07 22:22:10,http://fay.org/francesca,0.0000000000,76.214.60.18,"{""location"": ""VG"", ""is_mobile"": false}" 1435,3,134,2017-05-24 15:56:32,http://beahan.com/ryann.johns,3.0000000000,26.72.87.214,"{""location"": ""CR"", ""is_mobile"": true}" 1436,3,134,2017-06-01 06:39:12,http://reichel.com/reyna,3.0000000000,147.31.204.47,"{""location"": ""ZW"", ""is_mobile"": false}" 1437,3,134,2016-12-17 00:29:34,http://hilll.co/alanna.hamill,0.0000000000,22.122.7.161,"{""location"": ""MZ"", ""is_mobile"": false}" 2573,4,238,2017-03-26 02:00:30,http://beer.name/marco,1.0000000000,213.173.204.154,"{""location"": ""JM"", ""is_mobile"": false}" 2574,4,238,2017-05-23 16:39:42,http://kleinzboncak.com/roel,2.0000000000,192.210.208.147,"{""location"": ""AF"", ""is_mobile"": true}" 2575,4,239,2017-04-29 22:28:29,http://kuvalis.info/tavares,2.0000000000,75.205.40.131,"{""location"": ""WS"", ""is_mobile"": false}" 2576,4,239,2017-03-22 20:09:07,http://heathcote.io/hyman.tremblay,2.0000000000,142.211.2.213,"{""location"": ""WS"", ""is_mobile"": false}" 2577,4,239,2017-05-21 16:41:35,http://upton.co/reymundo.ebert,1.0000000000,17.173.152.38,"{""location"": ""MV"", ""is_mobile"": false}" 2578,4,239,2017-06-08 07:02:53,http://kuhnhegmann.com/candido.rippin,2.0000000000,22.212.2.38,"{""location"": ""GY"", ""is_mobile"": true}" 2579,4,239,2017-03-28 06:53:07,http://runtepfannerstill.biz/taryn.padberg,0.0000000000,203.243.114.60,"{""location"": ""LT"", ""is_mobile"": false}" 2580,4,239,2017-06-08 14:34:08,http://stoltenbergheathcote.com/eldon,0.0000000000,83.190.3.127,"{""location"": ""CH"", ""is_mobile"": false}" 2581,4,239,2017-06-06 22:31:22,http://mills.com/lera,3.0000000000,83.190.3.127,"{""location"": ""IE"", ""is_mobile"": false}" 2582,4,239,2017-05-03 04:25:02,http://larson.name/alphonso,0.0000000000,107.110.55.195,"{""location"": ""SM"", ""is_mobile"": false}" 2583,4,239,2017-02-28 08:51:55,http://homenick.co/cullen.greenfelder,0.0000000000,137.196.16.251,"{""location"": ""TT"", ""is_mobile"": false}" 2584,4,239,2017-05-28 21:47:30,http://ullrich.com/aurelia.gleason,0.0000000000,22.18.56.141,"{""location"": ""NL"", ""is_mobile"": true}" 2585,4,239,2017-02-23 22:15:05,http://halvorsonko.info/cade,1.0000000000,225.218.9.173,"{""location"": ""MZ"", ""is_mobile"": false}" 2586,4,239,2017-05-14 06:53:25,http://gorczany.name/alberta,1.0000000000,214.103.183.130,"{""location"": ""TK"", ""is_mobile"": true}" 2587,4,240,2017-03-01 07:47:31,http://roobstrosin.biz/reyna.thompson,0.0000000000,19.201.89.107,"{""location"": ""LU"", ""is_mobile"": false}" 2588,4,240,2017-05-22 06:34:40,http://mohr.org/jarrett.stokes,1.0000000000,160.57.196.163,"{""location"": ""PK"", ""is_mobile"": false}" 2589,4,240,2017-06-04 15:20:22,http://daniel.org/emmanuel.stracke,0.0000000000,93.166.110.162,"{""location"": ""KM"", ""is_mobile"": false}" 2590,4,241,2017-05-21 15:48:50,http://murphy.net/aunta_gottlieb,0.0000000000,126.245.140.197,"{""location"": ""BH"", ""is_mobile"": true}" 2591,4,241,2017-01-25 08:55:50,http://abernathy.io/sandrine_mante,0.0000000000,18.171.7.177,"{""location"": ""MK"", ""is_mobile"": false}" 2592,4,241,2017-06-01 04:11:55,http://ferry.com/devonte_mclaughlin,0.0000000000,7.132.42.172,"{""location"": ""NR"", ""is_mobile"": true}" 2593,4,241,2016-12-23 04:59:53,http://bednaremmerich.biz/brent,1.0000000000,183.191.114.202,"{""location"": ""AM"", ""is_mobile"": true}" 2594,4,241,2017-06-02 19:52:14,http://kihnfisher.name/floy,2.0000000000,208.217.126.246,"{""location"": ""MP"", ""is_mobile"": true}" 2595,4,241,2017-06-02 10:37:23,http://goodwin.biz/ethelyn,3.0000000000,194.243.73.245,"{""location"": ""SH"", ""is_mobile"": true}" 2596,4,241,2017-01-30 04:38:00,http://ferry.io/joesph.kuhlman,3.0000000000,253.98.183.20,"{""location"": ""PK"", ""is_mobile"": true}" 2597,4,241,2017-01-06 08:41:25,http://stoltenberg.org/taryn_fahey,0.0000000000,18.171.7.177,"{""location"": ""FM"", ""is_mobile"": true}" 2598,4,241,2016-12-28 07:52:59,http://walter.com/rocky,0.0000000000,132.165.74.139,"{""location"": ""CX"", ""is_mobile"": true}" 2599,4,241,2017-02-27 01:25:55,http://will.biz/alana_gutkowski,0.0000000000,223.34.84.244,"{""location"": ""SC"", ""is_mobile"": true}" 2600,4,241,2017-02-03 16:40:17,http://carterwisozk.co/javonte.mclaughlin,2.0000000000,119.82.45.120,"{""location"": ""HM"", ""is_mobile"": true}" 2601,4,241,2017-03-25 13:58:30,http://gleichner.info/eriberto,0.0000000000,66.10.94.107,"{""location"": ""KR"", ""is_mobile"": true}" 2602,4,241,2017-02-06 00:47:08,http://tromp.co/francis_stamm,3.0000000000,116.67.33.224,"{""location"": ""JO"", ""is_mobile"": true}" 2603,4,242,2017-05-04 15:31:41,http://runtebuckridge.name/beie,0.0000000000,124.231.249.197,"{""location"": ""LK"", ""is_mobile"": true}" 2604,4,242,2016-12-25 10:05:02,http://aufderhar.com/isabelle,1.0000000000,204.69.46.138,"{""location"": ""SX"", ""is_mobile"": true}" 2605,4,242,2017-02-02 06:05:28,http://rosenbaumframi.com/alvera_bailey,0.0000000000,223.22.182.16,"{""location"": ""RU"", ""is_mobile"": false}" 2606,4,242,2017-03-04 09:51:06,http://metz.name/jackie,0.0000000000,138.175.254.207,"{""location"": ""ZA"", ""is_mobile"": true}" 2607,4,242,2017-04-25 08:21:18,http://deckowkuphal.name/edmund_reilly,0.0000000000,144.141.213.141,"{""location"": ""MZ"", ""is_mobile"": true}" 2608,4,242,2017-03-22 07:36:41,http://hintzwintheiser.info/dangelo_mcdermott,0.0000000000,220.31.18.130,"{""location"": ""TM"", ""is_mobile"": true}" 2609,4,242,2017-03-30 03:09:55,http://greenfelder.net/vincenza,0.0000000000,165.170.225.4,"{""location"": ""MD"", ""is_mobile"": false}" 2610,4,242,2017-05-19 07:39:06,http://trompsipes.biz/devante_predovic,1.0000000000,242.2.216.29,"{""location"": ""AO"", ""is_mobile"": true}" 2611,4,242,2017-02-23 00:04:35,http://hayes.com/emelie,1.0000000000,68.187.181.13,"{""location"": ""KG"", ""is_mobile"": true}" 2612,4,242,2017-03-05 05:36:50,http://bechtelar.org/allene.mann,0.0000000000,204.69.46.138,"{""location"": ""PL"", ""is_mobile"": false}" 2613,4,243,2017-04-06 22:06:08,http://mcculloughpouros.net/elise,3.0000000000,230.105.148.155,"{""location"": ""JM"", ""is_mobile"": true}" 2614,4,243,2017-05-01 21:53:20,http://greenholt.net/morris.bashirian,3.0000000000,193.231.143.170,"{""location"": ""GM"", ""is_mobile"": true}" 2615,4,243,2017-04-27 00:30:28,http://grimesroob.com/antwan_batz,0.0000000000,28.56.119.28,"{""location"": ""IL"", ""is_mobile"": false}" 2616,4,243,2017-03-25 08:23:45,http://bradtke.info/hipolito.miller,0.0000000000,28.56.119.28,"{""location"": ""BR"", ""is_mobile"": true}" 2617,4,243,2017-03-16 01:34:14,http://dietrich.io/grace,2.0000000000,124.199.26.247,"{""location"": ""DJ"", ""is_mobile"": true}" 2618,4,244,2017-04-28 13:23:24,http://hansen.org/gail_reilly,0.0000000000,182.34.249.82,"{""location"": ""HU"", ""is_mobile"": true}" 2619,4,244,2017-04-05 09:19:38,http://mayert.io/marshall.zboncak,1.0000000000,79.222.214.242,"{""location"": ""GE"", ""is_mobile"": true}" 2620,4,244,2017-03-13 18:26:03,http://mosciskigleason.org/rashad,0.0000000000,98.147.61.116,"{""location"": ""ME"", ""is_mobile"": false}" 2621,4,245,2017-04-12 08:03:04,http://pfannerstill.net/amaya_kiehn,1.0000000000,175.59.52.63,"{""location"": ""FO"", ""is_mobile"": false}" 2622,4,245,2017-01-12 17:34:27,http://treutel.org/lera,1.0000000000,175.59.52.63,"{""location"": ""LC"", ""is_mobile"": false}" 2623,4,245,2017-03-11 22:30:38,http://pagac.org/maurine_rohan,1.0000000000,224.163.203.58,"{""location"": ""ER"", ""is_mobile"": true}" 2624,4,245,2017-01-25 13:44:50,http://johnston.info/joan_fadel,1.0000000000,173.31.9.140,"{""location"": ""MN"", ""is_mobile"": true}" 1438,3,134,2017-02-17 09:15:48,http://cartwrightrau.co/tatum.kemmer,1.0000000000,82.218.105.28,"{""location"": ""SX"", ""is_mobile"": false}" 1439,3,134,2017-03-26 21:22:09,http://mraz.org/richmond.schinner,2.0000000000,72.124.59.45,"{""location"": ""BA"", ""is_mobile"": true}" 1440,3,134,2017-01-05 09:05:01,http://bayerbartell.com/reyna,2.0000000000,26.72.87.214,"{""location"": ""ZM"", ""is_mobile"": false}" 1441,3,134,2017-06-12 06:46:07,http://langworth.biz/faye_anderson,2.0000000000,183.46.42.136,"{""location"": ""BV"", ""is_mobile"": true}" 1442,3,134,2017-04-15 16:07:40,http://auer.org/earnest,3.0000000000,138.238.61.210,"{""location"": ""SL"", ""is_mobile"": true}" 1443,3,134,2017-05-13 06:07:38,http://harvey.org/maxine.botsford,0.0000000000,72.124.59.45,"{""location"": ""TH"", ""is_mobile"": true}" 1444,3,134,2017-06-07 08:48:49,http://blick.info/stanford,0.0000000000,110.119.144.137,"{""location"": ""SV"", ""is_mobile"": true}" 1445,3,134,2017-02-18 21:59:29,http://white.info/johnson_thiel,3.0000000000,118.192.159.250,"{""location"": ""SX"", ""is_mobile"": false}" 1446,3,134,2017-04-21 06:15:54,http://rice.biz/pietro.leuschke,1.0000000000,124.189.71.202,"{""location"": ""SJ"", ""is_mobile"": true}" 1447,3,134,2017-03-17 10:21:11,http://lindgrenhickle.io/eleanora,2.0000000000,161.107.204.3,"{""location"": ""QA"", ""is_mobile"": false}" 1448,3,134,2017-05-29 17:11:37,http://kshlerin.biz/wilhelmine,0.0000000000,147.31.204.47,"{""location"": ""RO"", ""is_mobile"": false}" 1449,3,135,2017-04-20 18:17:55,http://zemlak.info/madilyn_batz,3.0000000000,241.86.229.87,"{""location"": ""MA"", ""is_mobile"": true}" 1450,3,135,2017-01-31 17:02:07,http://dickens.info/andres.schuster,2.0000000000,28.61.26.192,"{""location"": ""MF"", ""is_mobile"": false}" 1451,3,135,2017-03-07 01:51:40,http://boehm.biz/carley.corkery,0.0000000000,32.163.161.14,"{""location"": ""TR"", ""is_mobile"": true}" 1452,3,135,2017-03-19 12:59:30,http://littelupton.io/destinee,2.0000000000,28.61.26.192,"{""location"": ""MK"", ""is_mobile"": true}" 1453,3,135,2017-02-10 10:35:12,http://nicolas.io/celestino,2.0000000000,8.226.40.193,"{""location"": ""ET"", ""is_mobile"": true}" 1454,3,135,2017-01-16 11:45:03,http://goldner.io/emanuel_langosh,1.0000000000,133.129.245.237,"{""location"": ""BT"", ""is_mobile"": false}" 1455,3,135,2017-02-24 17:40:45,http://corkery.co/sylvia,1.0000000000,181.222.93.101,"{""location"": ""TV"", ""is_mobile"": false}" 1456,3,135,2017-04-15 23:54:51,http://nienow.com/nasir,2.0000000000,15.206.99.104,"{""location"": ""NZ"", ""is_mobile"": true}" 1457,3,135,2017-01-10 00:13:11,http://nolansanford.co/green.carter,0.0000000000,75.194.71.113,"{""location"": ""CR"", ""is_mobile"": false}" 1458,3,135,2017-03-16 12:18:02,http://altenwerthbeer.io/devon_kovacek,0.0000000000,97.172.15.81,"{""location"": ""BL"", ""is_mobile"": true}" 1459,3,135,2017-05-25 00:18:01,http://zemlak.biz/doyle,3.0000000000,58.208.184.97,"{""location"": ""SC"", ""is_mobile"": false}" 1460,3,135,2017-04-13 16:46:25,http://kovacek.info/brenden_kuhic,3.0000000000,5.213.129.40,"{""location"": ""VN"", ""is_mobile"": false}" 1461,3,135,2016-12-27 06:32:04,http://graham.io/rupert,0.0000000000,230.104.104.239,"{""location"": ""QA"", ""is_mobile"": false}" 1462,3,135,2017-03-16 02:03:43,http://padbergmcdermott.info/hellen.hintz,1.0000000000,53.101.140.128,"{""location"": ""LV"", ""is_mobile"": false}" 1463,3,136,2017-03-27 07:13:53,http://gleichner.info/ashly_bode,,170.118.170.86,"{""location"": ""SC"", ""is_mobile"": false}" 1464,3,136,2017-02-26 21:34:41,http://kerluke.co/idell_leuschke,,217.158.167.186,"{""location"": ""GU"", ""is_mobile"": false}" 1465,3,136,2017-03-03 04:52:07,http://lesch.co/rusty_goodwin,,176.92.17.250,"{""location"": ""FR"", ""is_mobile"": false}" 1466,3,136,2017-05-31 20:39:03,http://lebsackjaskolski.com/jefferey,,22.37.134.195,"{""location"": ""PH"", ""is_mobile"": false}" 1467,3,136,2017-03-03 01:54:42,http://gorczany.co/dana.franecki,,27.98.183.95,"{""location"": ""PK"", ""is_mobile"": false}" 1468,3,136,2016-12-19 02:14:57,http://kemmertowne.com/evalyn,,176.92.17.250,"{""location"": ""SO"", ""is_mobile"": true}" 1469,3,136,2017-01-12 02:58:05,http://towne.info/amya,,165.143.73.226,"{""location"": ""GW"", ""is_mobile"": true}" 1470,3,136,2017-01-01 21:44:30,http://mclaughlindonnelly.info/gladyce.ward,,40.129.46.136,"{""location"": ""TW"", ""is_mobile"": true}" 1471,3,136,2017-01-21 01:05:18,http://barton.com/manuel,,198.25.4.91,"{""location"": ""AU"", ""is_mobile"": false}" 1472,3,136,2017-03-08 19:54:50,http://kleinshanahan.co/rupert_wehner,,222.39.85.46,"{""location"": ""GS"", ""is_mobile"": false}" 1473,3,136,2017-04-29 23:31:47,http://considineferry.com/woodrow_moore,,170.118.170.86,"{""location"": ""AT"", ""is_mobile"": false}" 1474,3,136,2017-02-16 04:26:47,http://lockman.net/otho.grimes,,198.25.4.91,"{""location"": ""MC"", ""is_mobile"": true}" 1475,3,136,2017-05-28 05:30:18,http://funk.co/geovanny_pagac,,43.198.114.153,"{""location"": ""AU"", ""is_mobile"": true}" 1476,3,136,2017-01-04 18:49:54,http://powlowski.net/annamae_mueller,,89.20.13.104,"{""location"": ""GF"", ""is_mobile"": false}" 1477,3,136,2017-03-29 17:51:57,http://collierpurdy.com/edgar,,170.118.170.86,"{""location"": ""BI"", ""is_mobile"": false}" 1478,3,136,2017-05-20 15:40:04,http://thiel.name/yasmin_braun,,11.138.106.168,"{""location"": ""SX"", ""is_mobile"": true}" 1479,3,136,2017-01-05 23:49:46,http://willms.info/alia,,232.96.201.173,"{""location"": ""DE"", ""is_mobile"": true}" 1480,3,136,2016-12-22 10:49:55,http://okeefetromp.org/camila_kunze,,97.83.164.244,"{""location"": ""SV"", ""is_mobile"": true}" 1481,3,136,2017-03-27 15:10:14,http://okeefe.biz/calista,,170.118.170.86,"{""location"": ""BF"", ""is_mobile"": true}" 1482,3,137,2017-05-26 07:50:40,http://treutel.com/heidi_abernathy,,170.112.63.150,"{""location"": ""EH"", ""is_mobile"": true}" 1483,3,137,2017-04-29 07:23:37,http://senger.org/yesenia_ebert,,191.188.108.136,"{""location"": ""QA"", ""is_mobile"": true}" 1484,3,137,2017-03-11 04:11:08,http://jacobi.net/bridget_farrell,,83.176.18.153,"{""location"": ""DJ"", ""is_mobile"": true}" 1485,3,137,2017-03-26 09:59:03,http://schuppewilliamson.co/jamil_donnelly,,66.248.240.114,"{""location"": ""US"", ""is_mobile"": false}" 1486,3,137,2017-02-11 20:59:13,http://kautzer.co/allan.huels,,252.147.120.169,"{""location"": ""IQ"", ""is_mobile"": false}" 1487,3,137,2017-04-07 12:35:30,http://renner.io/jodie,,56.79.186.246,"{""location"": ""BA"", ""is_mobile"": true}" 1488,3,137,2017-02-04 21:50:53,http://satterfield.info/jasper,,70.36.170.76,"{""location"": ""EH"", ""is_mobile"": false}" 1489,3,137,2017-04-20 08:34:27,http://rolfson.co/bertrand_rempel,,69.213.114.123,"{""location"": ""TZ"", ""is_mobile"": true}" 1490,3,137,2017-04-17 23:05:09,http://terry.org/cleta.schiller,,124.137.231.49,"{""location"": ""KN"", ""is_mobile"": true}" 2625,4,245,2017-05-06 06:52:31,http://spinkashields.info/cheyanne.kunze,2.0000000000,56.53.163.114,"{""location"": ""RS"", ""is_mobile"": true}" 2626,4,245,2017-01-02 09:28:53,http://cruickshank.name/wyman_nicolas,0.0000000000,5.95.163.74,"{""location"": ""ZW"", ""is_mobile"": true}" 2627,4,245,2017-04-06 23:26:11,http://treutelrice.net/wanda,1.0000000000,100.151.222.153,"{""location"": ""FR"", ""is_mobile"": true}" 2628,4,245,2017-01-12 07:29:23,http://larkinkovacek.co/leila.sipes,2.0000000000,150.116.181.126,"{""location"": ""YE"", ""is_mobile"": true}" 2629,4,245,2017-05-23 05:28:56,http://lind.biz/freida,0.0000000000,108.216.253.160,"{""location"": ""DK"", ""is_mobile"": true}" 2630,4,245,2017-03-10 13:41:07,http://nienow.biz/erwin,2.0000000000,8.152.186.247,"{""location"": ""CK"", ""is_mobile"": false}" 2631,4,245,2017-03-17 01:11:29,http://sengerjaskolski.org/elroy.abbott,0.0000000000,104.56.40.224,"{""location"": ""FR"", ""is_mobile"": true}" 2632,4,245,2016-12-29 05:36:04,http://bechtelar.name/noemie_pacocha,1.0000000000,171.138.152.67,"{""location"": ""SX"", ""is_mobile"": false}" 2633,4,245,2017-03-03 01:56:20,http://flatley.io/santino_bradtke,0.0000000000,142.113.2.82,"{""location"": ""NO"", ""is_mobile"": true}" 2634,4,245,2017-04-16 20:45:36,http://heaneystiedemann.biz/simone.baumbach,0.0000000000,108.216.253.160,"{""location"": ""VC"", ""is_mobile"": false}" 2635,4,246,2017-01-07 05:07:22,http://robel.biz/merl,2.0000000000,61.226.131.19,"{""location"": ""SC"", ""is_mobile"": true}" 2636,4,246,2017-01-20 03:49:52,http://monahanhuels.co/sim,0.0000000000,14.5.125.162,"{""location"": ""SS"", ""is_mobile"": true}" 2637,4,246,2017-04-08 22:54:02,http://ritchie.io/bryana,0.0000000000,101.220.226.220,"{""location"": ""AW"", ""is_mobile"": false}" 2638,4,246,2017-01-26 16:38:21,http://stroman.co/zander_gutmann,0.0000000000,252.60.12.251,"{""location"": ""AR"", ""is_mobile"": true}" 2639,4,246,2017-04-18 21:35:18,http://swift.name/rashawn,2.0000000000,14.119.186.243,"{""location"": ""AS"", ""is_mobile"": false}" 2640,4,246,2017-03-16 10:52:05,http://mclaughlin.org/clemmie.torphy,0.0000000000,93.36.29.82,"{""location"": ""GF"", ""is_mobile"": true}" 2641,4,246,2017-05-09 15:38:06,http://schmeler.io/talon,2.0000000000,57.235.205.189,"{""location"": ""LI"", ""is_mobile"": true}" 2642,4,246,2017-02-10 01:13:53,http://stanton.biz/esmeralda.rau,0.0000000000,46.246.183.58,"{""location"": ""BS"", ""is_mobile"": false}" 2643,4,246,2017-04-08 16:22:32,http://lowe.org/angelina.dare,2.0000000000,174.38.20.197,"{""location"": ""SC"", ""is_mobile"": false}" 2644,4,246,2017-06-13 06:47:35,http://ruel.name/garett_halvorson,2.0000000000,136.189.75.12,"{""location"": ""NL"", ""is_mobile"": false}" 2645,4,246,2017-02-08 19:22:48,http://kemmer.io/gayle.walter,2.0000000000,140.181.79.20,"{""location"": ""BV"", ""is_mobile"": true}" 2646,4,247,2017-02-03 02:42:53,http://schoen.io/jerome.senger,1.0000000000,14.85.43.242,"{""location"": ""BQ"", ""is_mobile"": false}" 2647,4,247,2017-05-05 07:40:19,http://murraysmith.biz/alia,2.0000000000,92.134.110.179,"{""location"": ""GE"", ""is_mobile"": false}" 2648,4,247,2017-02-18 20:24:32,http://murazik.net/retha,0.0000000000,113.216.32.162,"{""location"": ""KR"", ""is_mobile"": true}" 2649,4,247,2017-06-09 07:09:40,http://jastfahey.biz/melia_west,1.0000000000,18.140.125.194,"{""location"": ""AW"", ""is_mobile"": false}" 2650,4,247,2017-04-11 23:57:11,http://faymacejkovic.co/hellen,1.0000000000,213.115.219.208,"{""location"": ""SX"", ""is_mobile"": true}" 2651,4,247,2017-05-22 00:32:09,http://harber.name/sherwood,2.0000000000,32.84.172.220,"{""location"": ""AM"", ""is_mobile"": false}" 2652,4,247,2017-04-14 20:36:08,http://schneider.io/vincenza,3.0000000000,236.15.49.212,"{""location"": ""BV"", ""is_mobile"": true}" 2653,4,247,2016-12-19 13:49:52,http://crookanford.name/raven_jerde,2.0000000000,36.78.32.240,"{""location"": ""CV"", ""is_mobile"": true}" 2654,4,247,2017-03-06 22:05:33,http://beer.co/eriberto,1.0000000000,204.111.60.3,"{""location"": ""SZ"", ""is_mobile"": true}" 2655,4,248,2017-05-09 03:32:01,http://abshirebednar.io/bradly,2.0000000000,152.202.200.40,"{""location"": ""CG"", ""is_mobile"": true}" 2656,4,248,2016-12-31 11:37:01,http://bauch.info/davonte_flatley,2.0000000000,114.203.226.44,"{""location"": ""GS"", ""is_mobile"": false}" 2657,4,248,2017-02-10 11:12:10,http://schimmelabernathy.io/casey.hegmann,3.0000000000,152.127.133.21,"{""location"": ""SV"", ""is_mobile"": false}" 2658,4,248,2017-03-02 00:49:01,http://greenfeldercrooks.biz/lila,2.0000000000,66.203.111.86,"{""location"": ""KH"", ""is_mobile"": false}" 2659,4,248,2017-05-20 05:29:19,http://ferryheel.io/jaydon,3.0000000000,66.203.111.86,"{""location"": ""HM"", ""is_mobile"": true}" 2660,4,248,2017-03-08 17:47:02,http://mcdermott.com/krystal.pouros,1.0000000000,97.216.131.116,"{""location"": ""EE"", ""is_mobile"": false}" 2661,4,248,2017-01-22 09:06:51,http://senger.net/gretchen,1.0000000000,22.149.177.7,"{""location"": ""IL"", ""is_mobile"": false}" 2662,4,248,2017-02-20 13:59:26,http://douglas.net/modesto,1.0000000000,243.154.126.57,"{""location"": ""KZ"", ""is_mobile"": true}" 2663,4,248,2017-01-25 08:22:59,http://rohan.org/trenton,3.0000000000,233.46.160.139,"{""location"": ""LI"", ""is_mobile"": false}" 2664,4,248,2017-03-11 05:59:27,http://kohler.info/rylan.johnson,1.0000000000,40.224.254.212,"{""location"": ""RU"", ""is_mobile"": true}" 2665,4,248,2017-01-20 06:30:11,http://stoltenberg.org/claria.kshlerin,0.0000000000,96.227.172.253,"{""location"": ""IM"", ""is_mobile"": false}" 2666,4,248,2017-05-21 06:08:15,http://kutch.biz/bette_haley,3.0000000000,73.36.202.234,"{""location"": ""ES"", ""is_mobile"": true}" 2667,4,249,2017-03-25 15:52:30,http://jastnitzsche.net/santiago,0.0000000000,49.23.163.128,"{""location"": ""HM"", ""is_mobile"": true}" 2668,4,249,2017-03-16 09:55:04,http://gaylordparker.biz/eliezer,0.0000000000,248.182.87.119,"{""location"": ""CU"", ""is_mobile"": true}" 2669,4,250,2016-12-25 07:31:49,http://kertzmannzieme.info/mckayla.watsica,2.0000000000,50.170.121.182,"{""location"": ""EE"", ""is_mobile"": true}" 2670,4,250,2017-02-16 22:08:04,http://bosco.info/alvis_nader,1.0000000000,14.90.12.160,"{""location"": ""CM"", ""is_mobile"": false}" 2671,4,250,2017-05-16 13:04:30,http://kochklocko.name/teresa_franecki,0.0000000000,67.185.163.155,"{""location"": ""AO"", ""is_mobile"": true}" 2672,4,250,2017-05-12 14:33:30,http://denesikdoyle.com/hudson,1.0000000000,81.45.185.144,"{""location"": ""GF"", ""is_mobile"": true}" 2673,4,251,2017-05-10 15:49:12,http://waltergleason.net/samanta,3.0000000000,213.33.106.9,"{""location"": ""MW"", ""is_mobile"": false}" 2674,4,251,2017-05-29 19:39:01,http://armstrongbednar.org/jackeline,3.0000000000,68.238.79.20,"{""location"": ""SH"", ""is_mobile"": false}" 2675,4,251,2017-05-22 01:36:17,http://schaefer.biz/kade_casper,1.0000000000,197.160.168.47,"{""location"": ""KR"", ""is_mobile"": false}" 1491,3,137,2017-03-11 16:11:26,http://franecki.name/agustin_boehm,,133.55.116.145,"{""location"": ""MH"", ""is_mobile"": true}" 1492,3,137,2016-12-15 00:13:32,http://hermiston.net/elijah,,202.86.130.173,"{""location"": ""GI"", ""is_mobile"": true}" 1493,3,137,2017-03-02 17:31:48,http://hansen.biz/alphonso,,29.2.189.70,"{""location"": ""EE"", ""is_mobile"": false}" 1494,3,137,2017-05-30 09:37:24,http://oreillymetz.io/kade_berge,,191.188.108.136,"{""location"": ""VN"", ""is_mobile"": true}" 1495,3,137,2017-05-12 19:14:10,http://kuhn.co/vita,,104.196.82.62,"{""location"": ""EH"", ""is_mobile"": false}" 1496,3,137,2017-03-27 10:23:53,http://stanton.net/jayda,,188.68.183.101,"{""location"": ""TG"", ""is_mobile"": true}" 1497,3,137,2016-12-31 14:30:16,http://willms.info/dameon,,137.234.71.24,"{""location"": ""RS"", ""is_mobile"": true}" 1498,3,137,2017-04-30 09:15:57,http://ko.biz/gabrielle.schneider,,133.55.116.145,"{""location"": ""SB"", ""is_mobile"": true}" 1499,3,137,2017-06-07 11:56:33,http://klockobosco.name/cecile,,161.45.112.132,"{""location"": ""NG"", ""is_mobile"": true}" 1500,3,138,2016-12-29 12:29:28,http://haleymacgyver.io/greta,,26.81.188.149,"{""location"": ""AL"", ""is_mobile"": false}" 1501,3,138,2016-12-20 10:17:01,http://wolf.net/savanna,,92.221.21.97,"{""location"": ""PL"", ""is_mobile"": false}" 1502,3,138,2017-01-11 18:06:43,http://huel.biz/eloisa.erdman,,49.243.85.121,"{""location"": ""TL"", ""is_mobile"": false}" 1503,3,138,2017-03-07 01:15:48,http://heaney.io/creola.smith,,26.81.188.149,"{""location"": ""AG"", ""is_mobile"": false}" 1504,3,138,2017-05-08 04:40:15,http://kuvalis.com/kaylie,,3.252.78.199,"{""location"": ""ME"", ""is_mobile"": true}" 1505,3,138,2016-12-20 05:34:40,http://rippin.com/flo,,250.147.200.102,"{""location"": ""SD"", ""is_mobile"": false}" 1506,3,138,2016-12-14 08:15:45,http://green.com/fritz,,26.147.241.94,"{""location"": ""EE"", ""is_mobile"": true}" 1507,3,138,2017-05-23 23:00:43,http://roberts.info/zoila,,6.246.90.42,"{""location"": ""SN"", ""is_mobile"": true}" 1508,3,138,2017-03-16 14:42:53,http://pricerice.net/mandy,,66.24.223.143,"{""location"": ""GM"", ""is_mobile"": true}" 1509,3,138,2017-02-16 11:40:05,http://gorczanyweinat.io/katherine,,6.246.90.42,"{""location"": ""PH"", ""is_mobile"": true}" 1510,3,138,2017-02-03 19:43:35,http://jaskolski.info/lily.medhurst,,218.93.158.104,"{""location"": ""ST"", ""is_mobile"": false}" 1511,3,138,2016-12-16 18:45:05,http://predoviccronin.net/leopoldo_lebsack,,237.171.32.116,"{""location"": ""AG"", ""is_mobile"": true}" 1512,3,138,2017-01-20 18:58:36,http://gibson.biz/elmore,,173.233.106.253,"{""location"": ""TK"", ""is_mobile"": false}" 1513,3,139,2017-06-10 10:50:17,http://runolfsdottirmurray.org/nakia.raynor,,227.153.58.120,"{""location"": ""CW"", ""is_mobile"": false}" 1514,3,139,2017-04-11 14:55:22,http://hand.net/anabel_smitham,,181.104.56.250,"{""location"": ""CL"", ""is_mobile"": true}" 1515,3,139,2017-04-20 11:56:37,http://kuhncormier.net/darwin_hermann,,214.164.229.134,"{""location"": ""TM"", ""is_mobile"": false}" 1516,3,139,2017-03-09 21:45:40,http://jerde.net/fermin,,162.93.236.100,"{""location"": ""TM"", ""is_mobile"": false}" 1517,3,139,2017-01-17 23:18:00,http://veum.net/norval.gulgowski,,186.124.152.28,"{""location"": ""MP"", ""is_mobile"": false}" 1518,3,139,2017-06-10 21:11:53,http://pacocha.org/guido_donnelly,,101.85.228.20,"{""location"": ""AW"", ""is_mobile"": true}" 1519,3,140,2017-01-21 01:31:48,http://becker.info/javier,,128.241.168.21,"{""location"": ""DK"", ""is_mobile"": true}" 1520,3,140,2017-02-12 09:09:51,http://borerkozey.org/laurie.prohaska,,145.220.60.248,"{""location"": ""LT"", ""is_mobile"": true}" 1521,3,140,2017-05-01 10:32:47,http://daniel.org/rae_white,,65.130.85.36,"{""location"": ""TV"", ""is_mobile"": true}" 1522,3,140,2017-03-13 10:20:44,http://koepphyatt.name/domenic,,128.241.168.21,"{""location"": ""RW"", ""is_mobile"": false}" 1523,3,140,2017-05-27 20:15:31,http://mertzkerluke.org/clementina.dach,,56.59.220.7,"{""location"": ""SA"", ""is_mobile"": true}" 1524,3,140,2017-04-15 07:02:07,http://goodwinmurray.io/turner,,237.140.171.225,"{""location"": ""MG"", ""is_mobile"": true}" 1525,3,140,2017-05-16 02:41:06,http://hartmann.co/maymie_douglas,,168.36.254.21,"{""location"": ""GL"", ""is_mobile"": true}" 1526,3,140,2016-12-28 05:33:41,http://vonrueden.biz/myrtle,,47.3.209.151,"{""location"": ""FI"", ""is_mobile"": false}" 1527,3,140,2017-06-03 21:26:56,http://botsfordheller.info/jacquelyn,,237.189.193.242,"{""location"": ""IS"", ""is_mobile"": false}" 1528,3,140,2017-01-28 20:02:47,http://ferry.co/velva,,66.45.117.224,"{""location"": ""PK"", ""is_mobile"": false}" 1529,3,140,2017-02-11 08:57:23,http://helleraufderhar.com/porter,,240.235.134.138,"{""location"": ""SB"", ""is_mobile"": false}" 1530,3,140,2017-05-10 20:57:09,http://hauck.net/idell,,22.24.177.25,"{""location"": ""UM"", ""is_mobile"": true}" 1531,3,141,2016-12-17 03:46:00,http://kemmer.net/pascale,,148.242.141.89,"{""location"": ""MO"", ""is_mobile"": true}" 1532,3,141,2017-04-12 12:34:46,http://grady.biz/halle,,60.193.83.165,"{""location"": ""TV"", ""is_mobile"": true}" 1533,3,141,2017-01-24 05:26:35,http://effertzko.co/lucious.dare,,45.164.167.192,"{""location"": ""ST"", ""is_mobile"": true}" 1534,3,141,2017-05-12 22:51:54,http://swift.info/earnest,,204.171.34.189,"{""location"": ""PS"", ""is_mobile"": false}" 1535,3,141,2017-05-06 17:19:47,http://schumm.io/lenna,,235.174.13.211,"{""location"": ""SX"", ""is_mobile"": true}" 1536,3,141,2017-02-22 19:58:41,http://ratke.name/estelle_bergnaum,,61.19.8.222,"{""location"": ""MQ"", ""is_mobile"": false}" 1537,3,141,2017-01-21 14:29:02,http://volkman.org/annetta.rolfson,,100.184.180.4,"{""location"": ""SK"", ""is_mobile"": true}" 1538,3,141,2016-12-20 19:06:26,http://reichelraynor.io/aleandra.borer,,248.180.108.194,"{""location"": ""AR"", ""is_mobile"": true}" 1539,3,141,2017-01-02 18:28:57,http://rauhilll.org/simeon.monahan,,221.191.43.72,"{""location"": ""NU"", ""is_mobile"": true}" 1540,3,141,2017-03-09 00:21:07,http://breitenberg.io/esperanza,,21.52.81.158,"{""location"": ""BB"", ""is_mobile"": true}" 1541,3,141,2017-05-28 05:22:08,http://wisozk.com/doug_kunde,,56.247.139.240,"{""location"": ""ER"", ""is_mobile"": false}" 1542,3,141,2017-05-25 06:54:39,http://turcotte.com/moshe,,241.59.183.181,"{""location"": ""CG"", ""is_mobile"": true}" 1543,3,141,2017-02-07 02:41:09,http://schaeferrosenbaum.co/maximus,,47.16.199.73,"{""location"": ""IS"", ""is_mobile"": false}" 1544,3,141,2017-03-03 19:12:38,http://dubuqueblock.co/henderson.steuber,,230.162.153.96,"{""location"": ""MX"", ""is_mobile"": false}" 1545,3,141,2017-03-14 17:58:34,http://kunde.biz/joan,,235.174.13.211,"{""location"": ""CF"", ""is_mobile"": true}" 1546,3,141,2017-01-30 20:28:43,http://okon.io/nathanial.wyman,,186.244.68.68,"{""location"": ""HT"", ""is_mobile"": true}" 2676,4,251,2017-01-19 03:12:08,http://marquardtjacobs.net/kieran,1.0000000000,31.107.120.98,"{""location"": ""MO"", ""is_mobile"": false}" 2677,4,251,2017-04-06 13:02:20,http://osinskidouglas.io/freda,1.0000000000,92.72.100.136,"{""location"": ""TL"", ""is_mobile"": true}" 2678,4,251,2017-05-10 19:42:11,http://feest.io/johnny,1.0000000000,31.107.120.98,"{""location"": ""AI"", ""is_mobile"": false}" 2679,4,251,2017-02-27 04:54:43,http://jacobs.co/orval_cronin,0.0000000000,49.211.128.33,"{""location"": ""MN"", ""is_mobile"": true}" 2680,4,251,2017-04-30 13:36:55,http://heidenreichjast.info/bernhard,0.0000000000,16.115.121.242,"{""location"": ""GG"", ""is_mobile"": true}" 2681,4,251,2016-12-15 01:14:19,http://dietrich.name/deangelo_gleason,0.0000000000,96.124.98.230,"{""location"": ""MD"", ""is_mobile"": true}" 2682,4,251,2017-06-01 08:59:50,http://wiegand.io/travon_bins,2.0000000000,61.130.239.89,"{""location"": ""BH"", ""is_mobile"": false}" 2683,4,251,2017-06-12 06:37:26,http://donnelly.org/jennie,0.0000000000,203.252.127.25,"{""location"": ""NF"", ""is_mobile"": false}" 2684,4,251,2017-05-10 08:22:53,http://luettgen.com/walton.cruickshank,3.0000000000,63.156.167.124,"{""location"": ""GN"", ""is_mobile"": true}" 2685,4,251,2017-01-01 08:34:22,http://zulauf.name/sadye_monahan,2.0000000000,168.133.63.148,"{""location"": ""JP"", ""is_mobile"": false}" 2686,4,251,2017-05-22 10:47:14,http://kuhn.biz/yadira,0.0000000000,30.151.93.247,"{""location"": ""ZW"", ""is_mobile"": true}" 2687,4,251,2017-01-22 15:09:30,http://prohaska.info/ezekiel.steuber,3.0000000000,232.199.62.15,"{""location"": ""GH"", ""is_mobile"": false}" 2688,4,251,2017-06-05 20:41:49,http://schroeder.org/alena.keeling,1.0000000000,116.9.37.217,"{""location"": ""UG"", ""is_mobile"": false}" 2689,4,251,2017-03-24 09:59:26,http://collins.name/antonio,3.0000000000,206.173.231.215,"{""location"": ""PT"", ""is_mobile"": false}" 2690,4,252,2016-12-31 19:54:43,http://homenickhettinger.info/leanne,0.0000000000,28.174.140.215,"{""location"": ""KZ"", ""is_mobile"": true}" 2691,4,252,2017-05-12 14:15:48,http://gerhold.biz/eudora.rolfson,0.0000000000,50.100.13.131,"{""location"": ""LA"", ""is_mobile"": false}" 2692,4,252,2017-03-20 19:13:40,http://kaulke.org/francisca,0.0000000000,200.156.187.68,"{""location"": ""VE"", ""is_mobile"": false}" 2693,4,252,2016-12-29 08:11:05,http://powlowskikerluke.io/brittany,0.0000000000,11.124.188.78,"{""location"": ""CU"", ""is_mobile"": true}" 2694,4,252,2017-03-05 10:27:46,http://kulasgleichner.biz/aimee,0.0000000000,70.199.102.13,"{""location"": ""CG"", ""is_mobile"": false}" 2695,4,252,2017-02-08 13:42:25,http://pacocha.org/bud,0.0000000000,133.207.81.168,"{""location"": ""NI"", ""is_mobile"": true}" 2696,4,252,2017-01-10 23:41:58,http://lakin.net/cale_kirlin,0.0000000000,104.75.41.44,"{""location"": ""KE"", ""is_mobile"": false}" 2697,4,252,2017-02-23 11:49:42,http://moore.org/reagan,0.0000000000,30.34.68.137,"{""location"": ""AR"", ""is_mobile"": false}" 2698,4,252,2017-04-11 15:09:16,http://gislason.io/dustin.mohr,0.0000000000,51.119.125.130,"{""location"": ""EC"", ""is_mobile"": true}" 2699,4,252,2017-04-14 14:38:55,http://stracke.biz/tiffany.keeling,0.0000000000,157.203.194.77,"{""location"": ""BQ"", ""is_mobile"": true}" 2700,4,253,2017-05-08 14:06:33,http://schultz.name/marlin,2.0000000000,137.146.143.247,"{""location"": ""RO"", ""is_mobile"": false}" 2701,4,253,2017-06-02 00:25:03,http://krajcikhane.net/cathrine_ko,1.0000000000,47.191.112.217,"{""location"": ""MO"", ""is_mobile"": true}" 2702,4,253,2017-01-19 06:06:26,http://fisher.co/zoila_oreilly,0.0000000000,26.220.132.238,"{""location"": ""CZ"", ""is_mobile"": false}" 2703,4,253,2017-06-04 06:32:43,http://casper.co/oral_zboncak,0.0000000000,234.26.209.20,"{""location"": ""CU"", ""is_mobile"": true}" 2704,4,253,2017-01-16 08:52:14,http://schuppe.com/danika,0.0000000000,52.28.170.44,"{""location"": ""SS"", ""is_mobile"": false}" 2705,4,254,2017-03-18 12:59:48,http://kihnfisher.info/wilber_zboncak,2.0000000000,67.178.60.163,"{""location"": ""BB"", ""is_mobile"": true}" 2706,4,254,2017-03-29 17:55:36,http://wilderman.io/ruell,2.0000000000,198.75.177.79,"{""location"": ""AM"", ""is_mobile"": true}" 2707,4,254,2017-04-04 23:46:48,http://borer.org/sabryna_goodwin,0.0000000000,59.56.146.178,"{""location"": ""ES"", ""is_mobile"": true}" 2708,4,254,2017-02-02 23:55:30,http://paucek.com/beryl,0.0000000000,130.226.55.130,"{""location"": ""RO"", ""is_mobile"": true}" 2709,4,254,2017-02-14 09:05:08,http://cole.com/bria_ruecker,1.0000000000,57.94.145.253,"{""location"": ""CU"", ""is_mobile"": false}" 2710,4,254,2017-01-26 15:07:57,http://jones.name/ernestina.cremin,1.0000000000,101.161.146.14,"{""location"": ""AL"", ""is_mobile"": true}" 2711,4,254,2016-12-22 16:14:55,http://goyettecormier.biz/ashleigh,2.0000000000,239.100.57.26,"{""location"": ""GI"", ""is_mobile"": false}" 2712,4,254,2017-05-18 03:03:41,http://wolf.co/sage,2.0000000000,59.56.146.178,"{""location"": ""GI"", ""is_mobile"": false}" 2713,4,254,2017-01-18 07:08:54,http://reynolds.com/brandt.bailey,2.0000000000,104.181.127.13,"{""location"": ""BI"", ""is_mobile"": false}" 2714,4,254,2017-03-03 09:24:28,http://windlerschaden.biz/genesis,0.0000000000,101.161.146.14,"{""location"": ""LR"", ""is_mobile"": false}" 2715,4,255,2017-05-08 09:47:55,http://schmidt.io/keeley_fay,0.0000000000,202.116.24.188,"{""location"": ""BI"", ""is_mobile"": true}" 2716,4,255,2017-01-14 12:12:17,http://jacobson.org/hillard,0.0000000000,108.15.92.133,"{""location"": ""SE"", ""is_mobile"": false}" 2717,4,255,2017-02-12 09:29:42,http://vandervort.org/nakia_weimann,1.0000000000,100.221.70.117,"{""location"": ""TH"", ""is_mobile"": false}" 2718,4,255,2016-12-25 00:24:04,http://wisoky.org/darron,1.0000000000,108.15.92.133,"{""location"": ""AD"", ""is_mobile"": true}" 2719,4,255,2017-01-15 02:05:29,http://cristmante.net/julian,0.0000000000,62.81.209.163,"{""location"": ""MH"", ""is_mobile"": false}" 2720,4,255,2017-01-01 16:44:01,http://becker.org/manuel.bahringer,1.0000000000,247.185.34.132,"{""location"": ""IS"", ""is_mobile"": true}" 2721,4,255,2017-05-18 21:04:07,http://denesikpouros.io/lura_heel,0.0000000000,42.251.178.114,"{""location"": ""SS"", ""is_mobile"": true}" 2722,4,255,2017-05-15 23:10:41,http://oreillyzulauf.co/jevon.weber,0.0000000000,209.191.154.209,"{""location"": ""WS"", ""is_mobile"": true}" 2723,4,255,2016-12-16 15:55:30,http://senger.biz/candice,1.0000000000,238.3.138.73,"{""location"": ""TV"", ""is_mobile"": false}" 2724,4,255,2016-12-18 02:38:42,http://homenick.com/florida.buckridge,0.0000000000,92.140.247.192,"{""location"": ""JE"", ""is_mobile"": false}" 2725,4,255,2017-01-23 05:47:13,http://dickinsoncremin.net/malika_harber,0.0000000000,34.180.4.211,"{""location"": ""MA"", ""is_mobile"": true}" 2726,4,255,2017-06-08 03:56:11,http://dare.com/trudie,2.0000000000,179.148.102.246,"{""location"": ""UA"", ""is_mobile"": true}" 1547,3,141,2017-04-13 15:39:29,http://boehm.biz/elenor.abbott,,34.161.17.72,"{""location"": ""BA"", ""is_mobile"": false}" 1548,3,141,2017-01-03 07:21:40,http://raynor.biz/kory.nienow,,172.241.137.191,"{""location"": ""BW"", ""is_mobile"": false}" 1549,3,141,2017-02-04 17:45:31,http://wisoky.com/jeromy_mills,,198.219.229.235,"{""location"": ""HM"", ""is_mobile"": true}" 1550,3,142,2017-05-12 01:15:40,http://cormier.co/charlotte_farrell,,203.10.249.16,"{""location"": ""AU"", ""is_mobile"": true}" 1551,3,142,2017-01-03 07:20:40,http://kertzmann.com/kiera,,116.78.115.248,"{""location"": ""LA"", ""is_mobile"": true}" 1552,3,142,2017-02-21 13:18:18,http://beer.net/brionna,,67.202.84.251,"{""location"": ""CV"", ""is_mobile"": false}" 1553,3,142,2017-03-12 17:10:50,http://trantow.io/emie,,223.116.24.71,"{""location"": ""MS"", ""is_mobile"": true}" 1554,3,142,2017-06-06 13:25:04,http://windler.org/nelson,,47.66.119.45,"{""location"": ""BY"", ""is_mobile"": false}" 1555,3,143,2017-02-26 09:29:39,http://anderson.name/camylle,,170.46.183.52,"{""location"": ""EG"", ""is_mobile"": false}" 1556,3,143,2017-01-12 14:41:45,http://sauerframi.io/taylor_spinka,,200.19.25.102,"{""location"": ""SD"", ""is_mobile"": true}" 1557,3,143,2017-06-09 23:13:39,http://vonrueden.com/rodrigo_hamill,,200.19.25.102,"{""location"": ""SV"", ""is_mobile"": false}" 1558,3,143,2017-01-11 20:55:06,http://herman.name/madaline,,185.26.172.39,"{""location"": ""CV"", ""is_mobile"": true}" 1559,3,143,2017-03-27 16:34:27,http://kshlerin.name/diamond,,241.73.9.188,"{""location"": ""BS"", ""is_mobile"": true}" 1560,3,143,2016-12-28 06:56:16,http://bechtelarwill.info/lolita.johnson,,224.77.100.220,"{""location"": ""BZ"", ""is_mobile"": false}" 1561,3,143,2017-04-30 13:40:42,http://ortizjerde.org/dion,,126.155.135.150,"{""location"": ""IL"", ""is_mobile"": false}" 1562,3,143,2017-01-29 04:26:31,http://pollichhaag.org/grant,,151.117.106.249,"{""location"": ""MC"", ""is_mobile"": true}" 1563,3,143,2017-04-24 23:03:42,http://labadie.net/jane.aufderhar,,241.73.9.188,"{""location"": ""HN"", ""is_mobile"": true}" 1564,3,143,2017-03-15 23:29:55,http://kutch.info/elsa,,218.18.52.179,"{""location"": ""KW"", ""is_mobile"": true}" 1565,3,143,2017-02-16 00:58:01,http://purdy.biz/amya,,177.91.63.75,"{""location"": ""TT"", ""is_mobile"": true}" 1566,3,144,2017-01-16 19:58:41,http://bode.co/orin,,230.141.47.235,"{""location"": ""MN"", ""is_mobile"": true}" 1567,3,144,2017-02-15 06:49:11,http://mueller.biz/ralph,,2.153.161.54,"{""location"": ""MZ"", ""is_mobile"": false}" 1568,3,144,2017-02-05 06:45:16,http://bernhard.info/amara_sauer,,48.223.77.21,"{""location"": ""PF"", ""is_mobile"": true}" 1569,3,145,2017-04-28 20:17:50,http://boehmwhite.io/danika,,154.116.95.164,"{""location"": ""KG"", ""is_mobile"": false}" 1570,3,145,2017-04-18 14:11:30,http://koelpin.co/elinor,,17.56.199.174,"{""location"": ""KE"", ""is_mobile"": true}" 1571,3,145,2016-12-26 20:21:08,http://hoppe.net/demetris,,234.49.209.78,"{""location"": ""KE"", ""is_mobile"": false}" 1572,3,145,2017-01-19 10:15:16,http://larkin.info/sigurd,,251.110.79.249,"{""location"": ""PG"", ""is_mobile"": false}" 1573,3,145,2017-03-15 23:18:18,http://treutelkeeling.biz/justen_conn,,106.49.243.37,"{""location"": ""UZ"", ""is_mobile"": false}" 1574,3,145,2017-02-07 20:47:57,http://hansen.co/cristian_prohaska,,251.110.79.249,"{""location"": ""MZ"", ""is_mobile"": true}" 1575,3,145,2017-03-14 12:18:00,http://pouros.biz/jodie.feest,,106.49.243.37,"{""location"": ""MW"", ""is_mobile"": true}" 1576,3,145,2017-04-09 16:12:05,http://jakubowskiwolf.org/blanche_huel,,133.37.233.236,"{""location"": ""TL"", ""is_mobile"": false}" 1577,3,145,2017-01-09 10:22:47,http://murazikmetz.net/nick.mosciski,,137.138.74.186,"{""location"": ""KW"", ""is_mobile"": false}" 1578,3,145,2017-03-08 02:27:52,http://armstrong.biz/ilene,,234.49.209.78,"{""location"": ""NE"", ""is_mobile"": true}" 1579,3,145,2017-04-27 04:09:36,http://thompson.biz/dewayne.pfeffer,,127.177.20.5,"{""location"": ""MH"", ""is_mobile"": true}" 1580,3,145,2017-04-18 14:19:45,http://ornklocko.name/eladio.kautzer,,122.113.96.42,"{""location"": ""MS"", ""is_mobile"": false}" 1581,3,145,2017-03-06 19:15:18,http://lind.com/isabella,,190.96.163.188,"{""location"": ""AS"", ""is_mobile"": true}" 1582,3,145,2017-01-03 17:01:19,http://brown.biz/toney_mante,,81.22.125.253,"{""location"": ""AZ"", ""is_mobile"": false}" 1583,3,145,2017-02-18 16:16:44,http://lemke.name/haskell.cruickshank,,131.60.227.228,"{""location"": ""DE"", ""is_mobile"": true}" 1584,3,145,2017-05-11 07:19:40,http://terry.name/jacques_emard,,127.177.20.5,"{""location"": ""NA"", ""is_mobile"": true}" 1585,3,145,2017-05-22 00:00:46,http://grimes.name/marlee.abshire,,133.37.233.236,"{""location"": ""JP"", ""is_mobile"": true}" 1586,3,145,2017-02-22 14:36:17,http://rueckerbogisich.io/cyril,,214.167.237.39,"{""location"": ""SH"", ""is_mobile"": true}" 1587,3,145,2017-02-19 02:35:06,http://farrell.io/melvin,,17.56.199.174,"{""location"": ""TW"", ""is_mobile"": true}" 1588,3,146,2017-02-15 19:21:07,http://altenwerth.name/laurence,0.0000000000,185.64.126.37,"{""location"": ""LU"", ""is_mobile"": false}" 1589,3,146,2017-03-22 05:42:26,http://torpromaguera.com/lloyd,0.0000000000,45.37.55.45,"{""location"": ""NZ"", ""is_mobile"": false}" 1590,3,146,2016-12-13 13:06:03,http://osinski.io/winnifred,0.0000000000,225.207.5.162,"{""location"": ""AS"", ""is_mobile"": false}" 1591,3,146,2017-01-06 16:04:26,http://hayes.com/anderson,0.0000000000,33.50.161.158,"{""location"": ""KG"", ""is_mobile"": false}" 1592,3,146,2017-05-04 13:01:46,http://ratkeweimann.io/tyson.turner,0.0000000000,185.64.126.37,"{""location"": ""IO"", ""is_mobile"": false}" 1593,3,147,2017-02-09 06:41:43,http://ullrichgottlieb.name/kristian,1.0000000000,127.73.173.173,"{""location"": ""CW"", ""is_mobile"": true}" 1594,3,147,2017-04-21 09:46:33,http://ferrylakin.io/aiden,0.0000000000,110.244.225.61,"{""location"": ""WF"", ""is_mobile"": true}" 1595,3,147,2017-01-01 19:43:24,http://littelsimonis.com/margaret.oberbrunner,0.0000000000,174.2.172.149,"{""location"": ""AU"", ""is_mobile"": true}" 1596,3,147,2016-12-24 01:39:34,http://quitzonyost.com/clovis_bergstrom,0.0000000000,230.167.68.47,"{""location"": ""SC"", ""is_mobile"": true}" 1597,3,148,2017-03-26 20:36:27,http://gislason.info/kenny_willms,0.0000000000,22.108.121.156,"{""location"": ""CO"", ""is_mobile"": true}" 1598,3,148,2017-03-18 02:19:28,http://gleasonbeier.com/zack.wunsch,0.0000000000,233.162.94.143,"{""location"": ""KY"", ""is_mobile"": true}" 1599,3,148,2017-05-06 10:03:42,http://wolf.biz/una.morar,0.0000000000,193.51.20.129,"{""location"": ""CW"", ""is_mobile"": false}" 1600,3,148,2017-06-07 22:30:19,http://johnstonjenkins.io/kieran_nitzsche,0.0000000000,47.127.220.253,"{""location"": ""NE"", ""is_mobile"": true}" 1601,3,148,2017-02-04 05:19:09,http://cummeratawilliamson.biz/oral_paucek,0.0000000000,193.51.20.129,"{""location"": ""UM"", ""is_mobile"": false}" 2727,4,256,2017-05-25 17:23:54,http://cronin.info/elza_hackett,0.0000000000,54.64.128.8,"{""location"": ""MV"", ""is_mobile"": true}" 2728,4,256,2017-04-13 06:00:05,http://little.info/austen,0.0000000000,99.150.241.122,"{""location"": ""CI"", ""is_mobile"": false}" 2729,4,256,2017-02-15 15:44:27,http://fay.net/ara_borer,0.0000000000,125.13.42.136,"{""location"": ""LT"", ""is_mobile"": true}" 2730,4,256,2017-04-06 22:54:53,http://moore.net/laura,0.0000000000,56.8.81.97,"{""location"": ""SG"", ""is_mobile"": false}" 2731,4,256,2017-05-22 14:33:07,http://herman.org/hermann.waelchi,0.0000000000,218.120.173.33,"{""location"": ""DM"", ""is_mobile"": true}" 2732,4,256,2016-12-25 10:13:17,http://rippin.org/stephania,0.0000000000,78.217.2.117,"{""location"": ""KE"", ""is_mobile"": true}" 2733,4,256,2017-02-24 04:05:14,http://nienow.org/claudia,0.0000000000,143.229.95.192,"{""location"": ""CC"", ""is_mobile"": true}" 2734,4,256,2017-03-13 23:46:04,http://parisian.io/kirstin,0.0000000000,151.130.161.150,"{""location"": ""UA"", ""is_mobile"": true}" 2735,4,256,2017-03-06 12:39:39,http://mayer.org/birdie,0.0000000000,143.229.95.192,"{""location"": ""NG"", ""is_mobile"": true}" 2736,4,256,2016-12-17 11:36:27,http://kohler.name/casandra_gorczany,0.0000000000,125.13.42.136,"{""location"": ""SD"", ""is_mobile"": true}" 2737,4,256,2017-03-02 13:51:14,http://ruecker.co/durward,0.0000000000,172.168.189.222,"{""location"": ""SG"", ""is_mobile"": false}" 2738,4,256,2017-03-31 11:45:22,http://osinski.co/cooper.cormier,0.0000000000,238.230.52.80,"{""location"": ""FJ"", ""is_mobile"": true}" 2739,4,256,2017-05-12 12:57:44,http://hettinger.name/will,0.0000000000,106.122.143.63,"{""location"": ""JP"", ""is_mobile"": false}" 2740,4,256,2017-01-10 08:04:15,http://wunsch.org/enrique,0.0000000000,69.188.176.69,"{""location"": ""IL"", ""is_mobile"": true}" 2741,4,256,2017-03-04 12:01:59,http://yundt.co/rhianna,0.0000000000,177.200.201.39,"{""location"": ""LC"", ""is_mobile"": true}" 2742,4,256,2017-05-17 00:10:39,http://macgyver.org/winfield,0.0000000000,56.8.81.97,"{""location"": ""JP"", ""is_mobile"": false}" 2743,4,256,2017-04-18 10:59:29,http://keler.name/jaden,0.0000000000,109.8.149.246,"{""location"": ""LT"", ""is_mobile"": true}" 2744,4,256,2017-02-15 09:08:23,http://quitzonlarkin.co/domenic_sporer,0.0000000000,218.120.173.33,"{""location"": ""DK"", ""is_mobile"": true}" 2745,4,257,2017-01-31 08:33:24,http://krajcikjakubowski.com/colby,,187.119.115.130,"{""location"": ""MU"", ""is_mobile"": true}" 2746,4,257,2017-02-16 15:27:59,http://moriettereynolds.info/novella_johnston,,74.144.32.213,"{""location"": ""ER"", ""is_mobile"": false}" 2747,4,257,2017-01-10 06:01:44,http://armstrong.co/jamel,,233.145.81.211,"{""location"": ""KY"", ""is_mobile"": true}" 2748,4,257,2017-06-13 12:03:00,http://watsicaking.org/kathleen.streich,,80.95.53.184,"{""location"": ""TW"", ""is_mobile"": true}" 2749,4,257,2017-02-01 09:45:26,http://metz.biz/chelsea.bayer,,252.198.110.23,"{""location"": ""CC"", ""is_mobile"": false}" 2750,4,257,2017-03-04 00:48:43,http://kutchsawayn.com/estrella,,216.43.233.205,"{""location"": ""TO"", ""is_mobile"": false}" 2751,4,257,2017-02-06 22:55:00,http://ritchie.co/idell,,116.245.7.183,"{""location"": ""TG"", ""is_mobile"": false}" 2752,4,257,2017-05-21 12:37:05,http://gleichner.info/wiley.heidenreich,,166.216.54.242,"{""location"": ""AM"", ""is_mobile"": false}" 2753,4,257,2017-02-22 18:54:31,http://jacobiwisoky.info/sunny,,18.124.18.195,"{""location"": ""CL"", ""is_mobile"": true}" 2754,4,257,2017-02-23 15:50:09,http://hettingerbrown.info/edward,,51.136.240.25,"{""location"": ""GT"", ""is_mobile"": true}" 2755,4,257,2017-05-22 11:39:19,http://bogisich.net/jamey_oconner,,252.198.110.23,"{""location"": ""RS"", ""is_mobile"": true}" 2756,4,257,2017-05-24 22:54:28,http://rogahn.net/morton.kuhn,,68.79.137.162,"{""location"": ""SB"", ""is_mobile"": true}" 2757,4,258,2016-12-19 20:38:49,http://ruelkuvalis.org/erik,,179.39.206.171,"{""location"": ""BW"", ""is_mobile"": true}" 2758,4,258,2017-05-07 08:54:16,http://kuhlman.biz/kenyon.nader,,229.160.20.161,"{""location"": ""CO"", ""is_mobile"": true}" 2759,4,258,2017-06-02 22:36:12,http://abbott.co/karianne,,33.8.179.51,"{""location"": ""MU"", ""is_mobile"": true}" 2760,4,258,2017-03-09 07:20:47,http://pollichdare.net/nathanial_schuster,,102.220.121.247,"{""location"": ""BT"", ""is_mobile"": true}" 2761,4,259,2017-02-07 00:37:59,http://rohan.co/beaulah_bashirian,,223.5.23.118,"{""location"": ""MN"", ""is_mobile"": false}" 2762,4,259,2017-06-12 19:52:20,http://durganfay.name/nickolas.hand,,215.216.233.169,"{""location"": ""MG"", ""is_mobile"": false}" 2763,4,259,2017-03-12 11:48:31,http://carroll.biz/savanah,,98.179.241.82,"{""location"": ""CU"", ""is_mobile"": true}" 2764,4,259,2017-04-08 19:36:07,http://cremin.net/jamison.funk,,218.95.204.105,"{""location"": ""ET"", ""is_mobile"": true}" 2765,4,259,2017-01-09 19:49:59,http://grahamgusikowski.net/esperanza_zieme,,218.95.204.105,"{""location"": ""MW"", ""is_mobile"": true}" 2766,4,259,2017-04-29 18:32:55,http://kulas.net/lera,,5.221.111.165,"{""location"": ""LA"", ""is_mobile"": true}" 2767,4,259,2017-01-24 00:50:23,http://kihn.io/erwin_mante,,5.221.111.165,"{""location"": ""KY"", ""is_mobile"": false}" 2768,4,259,2016-12-29 20:19:31,http://prohaska.org/nella,,196.144.96.234,"{""location"": ""FO"", ""is_mobile"": false}" 2769,4,259,2017-02-18 03:14:14,http://beahanmiller.org/blanca,,185.219.144.53,"{""location"": ""SB"", ""is_mobile"": true}" 2770,4,259,2017-01-22 19:12:25,http://pollich.info/savanah_romaguera,,98.175.232.164,"{""location"": ""BJ"", ""is_mobile"": false}" 2771,4,259,2017-02-03 15:22:51,http://quitzonhintz.io/ivah.walker,,98.179.241.82,"{""location"": ""LA"", ""is_mobile"": true}" 2772,4,260,2017-04-04 18:22:37,http://cormier.info/alexandre_kohler,,12.147.228.167,"{""location"": ""SI"", ""is_mobile"": true}" 2773,4,260,2016-12-23 10:20:16,http://schadenjones.biz/maureen_price,,216.93.33.134,"{""location"": ""SN"", ""is_mobile"": false}" 2774,4,260,2017-03-09 15:09:05,http://wardfeil.co/erica_hintz,,73.243.131.254,"{""location"": ""BH"", ""is_mobile"": true}" 2775,4,260,2017-05-26 20:16:41,http://hickle.biz/micaela,,240.188.88.249,"{""location"": ""TC"", ""is_mobile"": false}" 2776,4,260,2017-01-08 19:48:50,http://botsfordbotsford.name/krystina_oreilly,,172.215.15.107,"{""location"": ""JO"", ""is_mobile"": true}" 2777,4,260,2016-12-18 01:26:40,http://romaguera.info/albertha,,240.188.88.249,"{""location"": ""HU"", ""is_mobile"": true}" 2778,4,260,2017-04-30 07:55:41,http://kreiger.biz/simone,,66.90.203.59,"{""location"": ""BN"", ""is_mobile"": false}" 2779,4,260,2017-02-20 13:24:22,http://schneider.com/marcia,,137.190.251.195,"{""location"": ""PK"", ""is_mobile"": false}" 2780,4,260,2017-03-21 05:04:56,http://baumbach.biz/jennifer,,156.215.229.211,"{""location"": ""GQ"", ""is_mobile"": false}" 2781,4,260,2017-02-11 20:17:12,http://emmerich.org/mariela.raynor,,40.38.189.208,"{""location"": ""BZ"", ""is_mobile"": true}" 1602,3,148,2017-05-28 09:28:04,http://kunde.net/floyd.fisher,0.0000000000,22.108.121.156,"{""location"": ""HM"", ""is_mobile"": true}" 1603,3,148,2017-04-25 09:47:44,http://brown.name/jayson,0.0000000000,131.232.24.32,"{""location"": ""KP"", ""is_mobile"": true}" 1604,3,148,2017-02-02 19:36:34,http://bashirian.info/emmett,0.0000000000,76.102.30.212,"{""location"": ""IR"", ""is_mobile"": false}" 1605,3,148,2017-05-15 15:38:43,http://boylejerde.co/morton,0.0000000000,135.95.233.79,"{""location"": ""SX"", ""is_mobile"": false}" 1606,3,148,2017-03-04 10:30:10,http://ward.info/dariana,0.0000000000,131.232.24.32,"{""location"": ""KN"", ""is_mobile"": true}" 1607,3,148,2017-01-16 03:57:44,http://hagenes.com/natasha,0.0000000000,238.204.144.63,"{""location"": ""CR"", ""is_mobile"": true}" 1608,3,148,2017-04-25 20:35:59,http://willms.net/electa,0.0000000000,47.127.220.253,"{""location"": ""AQ"", ""is_mobile"": true}" 1609,3,148,2017-05-17 06:14:51,http://friesen.net/te,0.0000000000,171.225.232.132,"{""location"": ""ER"", ""is_mobile"": true}" 1610,3,148,2017-02-14 06:02:41,http://beierbruen.name/mohammad,0.0000000000,204.12.250.211,"{""location"": ""EH"", ""is_mobile"": true}" 1611,3,149,2017-03-19 16:16:20,http://mertzmaggio.net/kayden,2.0000000000,147.21.105.83,"{""location"": ""NL"", ""is_mobile"": true}" 1612,3,149,2017-05-04 17:45:00,http://gaylordconsidine.net/allison_rutherford,1.0000000000,10.192.17.149,"{""location"": ""RO"", ""is_mobile"": false}" 1613,3,149,2017-03-31 08:16:39,http://sporergreenfelder.biz/amber,0.0000000000,113.6.74.3,"{""location"": ""TL"", ""is_mobile"": true}" 1614,3,149,2017-01-10 00:48:15,http://lynchdamore.io/alba,2.0000000000,200.213.239.170,"{""location"": ""PL"", ""is_mobile"": false}" 1615,3,149,2017-02-28 20:18:47,http://bahringer.net/marquis.bergnaum,1.0000000000,221.242.98.102,"{""location"": ""RU"", ""is_mobile"": false}" 1616,3,149,2017-01-24 12:33:20,http://stiedemann.io/humberto,3.0000000000,190.41.77.148,"{""location"": ""ST"", ""is_mobile"": true}" 1617,3,149,2017-05-27 09:56:55,http://hammes.com/jeanie_romaguera,3.0000000000,39.19.70.113,"{""location"": ""BB"", ""is_mobile"": true}" 1618,3,149,2017-05-06 23:29:01,http://hane.com/valentina,3.0000000000,39.19.70.113,"{""location"": ""CA"", ""is_mobile"": true}" 1619,3,149,2017-02-13 15:13:38,http://cronaraynor.name/miouri,1.0000000000,189.66.65.55,"{""location"": ""KP"", ""is_mobile"": false}" 1620,3,149,2017-05-27 12:04:01,http://mohr.co/pearline,1.0000000000,91.246.6.246,"{""location"": ""BB"", ""is_mobile"": true}" 1621,3,149,2016-12-29 06:33:08,http://pfeffer.com/johan_beatty,3.0000000000,189.66.65.55,"{""location"": ""PL"", ""is_mobile"": false}" 1622,3,150,2017-02-06 11:20:26,http://vonrueden.io/kieran,2.0000000000,182.89.210.50,"{""location"": ""CV"", ""is_mobile"": true}" 1623,3,150,2017-03-23 23:44:05,http://bartoletti.io/aurelie.dubuque,2.0000000000,231.221.136.217,"{""location"": ""AM"", ""is_mobile"": true}" 1624,3,150,2017-03-15 09:27:24,http://predovicbednar.name/dustin,0.0000000000,148.242.132.119,"{""location"": ""RS"", ""is_mobile"": false}" 1625,3,150,2017-04-28 16:14:39,http://schamberger.co/rhoda.terry,1.0000000000,179.223.68.220,"{""location"": ""GL"", ""is_mobile"": false}" 1626,3,150,2017-01-13 22:49:01,http://cummerata.net/herta.langosh,0.0000000000,230.188.98.105,"{""location"": ""PE"", ""is_mobile"": true}" 1627,3,150,2017-04-09 05:34:23,http://emard.biz/eloy,0.0000000000,248.75.80.68,"{""location"": ""TW"", ""is_mobile"": true}" 1628,3,150,2017-06-02 00:04:28,http://kutch.io/myah,1.0000000000,230.188.98.105,"{""location"": ""BL"", ""is_mobile"": true}" 1629,3,150,2017-05-14 12:36:58,http://tillman.info/hope,2.0000000000,114.231.176.161,"{""location"": ""NO"", ""is_mobile"": true}" 1630,3,150,2017-05-20 19:58:13,http://keeblershields.name/josiah.hickle,0.0000000000,52.48.194.115,"{""location"": ""UZ"", ""is_mobile"": true}" 1631,3,150,2017-05-31 13:46:41,http://balistreriabshire.co/jonatan,1.0000000000,163.204.165.183,"{""location"": ""AI"", ""is_mobile"": false}" 1632,3,150,2017-02-10 11:11:43,http://zulauf.org/raegan.wolff,1.0000000000,148.242.132.119,"{""location"": ""MA"", ""is_mobile"": true}" 1633,3,150,2017-02-25 06:31:17,http://huelsokuneva.io/mason_hermiston,1.0000000000,202.196.134.178,"{""location"": ""VA"", ""is_mobile"": false}" 1634,3,150,2017-04-29 18:31:26,http://doyle.name/dovie,2.0000000000,222.32.67.65,"{""location"": ""GH"", ""is_mobile"": true}" 1635,3,150,2017-04-09 04:51:22,http://volkmanjakubowski.org/gavin.green,1.0000000000,250.155.8.46,"{""location"": ""ZA"", ""is_mobile"": false}" 1636,3,150,2017-03-08 17:48:01,http://mrazschmidt.biz/flavio,2.0000000000,200.77.18.194,"{""location"": ""ER"", ""is_mobile"": true}" 1637,3,150,2017-01-22 12:40:25,http://wuckert.io/aisha,2.0000000000,52.48.194.115,"{""location"": ""RE"", ""is_mobile"": true}" 1638,3,150,2017-01-04 19:58:51,http://schamberger.net/fredy,0.0000000000,130.243.234.14,"{""location"": ""CV"", ""is_mobile"": true}" 1639,3,150,2017-05-11 04:22:40,http://casper.net/marcus,0.0000000000,174.150.106.7,"{""location"": ""ZA"", ""is_mobile"": false}" 1640,3,150,2017-02-21 00:38:00,http://batzwilderman.biz/dawn.bernhard,0.0000000000,202.196.134.178,"{""location"": ""VG"", ""is_mobile"": false}" 1641,3,150,2017-02-16 16:21:38,http://bradtke.info/heaven,0.0000000000,227.228.122.187,"{""location"": ""RO"", ""is_mobile"": true}" 1642,3,151,2017-04-27 03:26:50,http://veumreichel.com/jacquelyn.mueller,0.0000000000,111.101.40.92,"{""location"": ""BM"", ""is_mobile"": false}" 1643,3,151,2017-03-11 21:13:58,http://roob.net/britney.conroy,1.0000000000,148.254.14.220,"{""location"": ""MM"", ""is_mobile"": true}" 1644,3,151,2017-03-20 04:49:12,http://west.co/brayan.emmerich,1.0000000000,22.115.73.77,"{""location"": ""KM"", ""is_mobile"": true}" 1645,3,151,2017-04-27 07:07:16,http://brekke.co/meghan.mcglynn,1.0000000000,189.120.93.243,"{""location"": ""SV"", ""is_mobile"": false}" 1646,3,151,2017-03-01 09:06:42,http://schmidt.info/dorcas_grady,0.0000000000,156.37.254.110,"{""location"": ""IM"", ""is_mobile"": false}" 1647,3,151,2017-05-24 03:47:30,http://swift.biz/abagail,1.0000000000,183.111.130.148,"{""location"": ""BV"", ""is_mobile"": false}" 1648,3,151,2017-05-31 21:04:53,http://hicklearmstrong.info/stevie,1.0000000000,86.250.147.55,"{""location"": ""EH"", ""is_mobile"": false}" 1649,3,151,2017-03-08 00:13:31,http://kuhickozey.io/malvina_glover,0.0000000000,212.157.53.80,"{""location"": ""NG"", ""is_mobile"": true}" 1650,3,151,2017-05-14 20:45:00,http://blanda.org/allen,0.0000000000,25.144.202.211,"{""location"": ""SR"", ""is_mobile"": false}" 1651,3,151,2017-04-21 01:43:52,http://sawaynbeier.biz/ruby,0.0000000000,139.215.121.63,"{""location"": ""AW"", ""is_mobile"": true}" 1652,3,151,2017-01-22 23:21:26,http://lueilwitz.co/stanford,1.0000000000,167.195.191.167,"{""location"": ""CY"", ""is_mobile"": false}" 2782,4,261,2017-02-22 12:05:03,http://herzog.name/gerardo_schmitt,,97.243.27.252,"{""location"": ""HM"", ""is_mobile"": true}" 2783,4,261,2017-01-16 07:47:42,http://williamson.org/carroll.hilpert,,140.7.233.63,"{""location"": ""AD"", ""is_mobile"": true}" 2784,4,261,2016-12-20 04:47:49,http://pfannerstill.org/jennings,,162.83.151.157,"{""location"": ""EH"", ""is_mobile"": true}" 2785,4,261,2017-01-24 10:52:46,http://schumm.biz/lillie_keler,,97.243.27.252,"{""location"": ""SB"", ""is_mobile"": false}" 2786,4,261,2017-01-28 07:53:35,http://kunzekutch.biz/destinee,,248.253.16.35,"{""location"": ""KR"", ""is_mobile"": false}" 2787,4,262,2017-06-12 21:27:33,http://kaulkewyman.biz/grayson,,240.82.203.202,"{""location"": ""ER"", ""is_mobile"": true}" 2788,4,262,2017-03-04 06:28:20,http://kuhlman.net/matilde.hermann,,37.20.75.113,"{""location"": ""SY"", ""is_mobile"": true}" 2789,4,262,2017-03-17 08:29:42,http://dibbert.org/joe,,209.156.199.230,"{""location"": ""DO"", ""is_mobile"": true}" 2790,4,262,2016-12-27 18:56:08,http://beatty.org/gust,,16.253.82.176,"{""location"": ""BQ"", ""is_mobile"": false}" 2791,4,262,2017-02-08 03:44:49,http://lehnerschulist.co/macy,,99.67.35.187,"{""location"": ""GH"", ""is_mobile"": true}" 2792,4,262,2017-03-24 19:04:00,http://lubowitz.biz/kaitlyn,,41.108.62.230,"{""location"": ""MP"", ""is_mobile"": false}" 2793,4,262,2017-01-13 19:24:16,http://harriskertzmann.com/olen_gaylord,,149.211.69.118,"{""location"": ""MP"", ""is_mobile"": true}" 4321,7,403,2017-01-07 15:04:40,http://schuppe.net/tony,3.0000000000,238.79.126.118,"{""location"": ""BY"", ""is_mobile"": true}" 4322,7,403,2017-05-25 03:54:11,http://miller.com/dawson,3.0000000000,159.132.175.52,"{""location"": ""NC"", ""is_mobile"": false}" 4323,7,403,2017-03-11 13:52:15,http://pfeffer.com/jaleel,0.0000000000,115.25.242.19,"{""location"": ""HM"", ""is_mobile"": false}" 4324,7,403,2017-02-09 11:58:50,http://skiles.co/gail_jast,0.0000000000,168.116.52.24,"{""location"": ""UG"", ""is_mobile"": false}" 4325,7,403,2016-12-21 02:02:03,http://turner.name/robbie,3.0000000000,181.24.220.109,"{""location"": ""LA"", ""is_mobile"": false}" 4326,7,403,2017-04-25 07:15:35,http://huels.org/brady.reichert,2.0000000000,46.176.32.50,"{""location"": ""HR"", ""is_mobile"": false}" 4327,7,403,2017-02-19 18:02:11,http://thompsonschimmel.io/earline,3.0000000000,192.253.184.168,"{""location"": ""IM"", ""is_mobile"": true}" 4328,7,403,2017-03-14 07:39:59,http://rohanhirthe.name/eudora_mosciski,1.0000000000,40.148.209.206,"{""location"": ""CV"", ""is_mobile"": false}" 4329,7,403,2017-03-13 16:26:41,http://hagenes.org/rowan_zieme,3.0000000000,131.41.214.75,"{""location"": ""IE"", ""is_mobile"": false}" 4330,7,403,2017-06-12 15:26:22,http://steuber.name/cale.welch,2.0000000000,78.45.133.126,"{""location"": ""BR"", ""is_mobile"": true}" 4331,7,403,2017-01-20 15:23:34,http://champlin.info/carolyne,1.0000000000,115.25.242.19,"{""location"": ""SM"", ""is_mobile"": false}" 4332,7,403,2017-05-23 01:19:18,http://hegmann.info/turner.schuppe,3.0000000000,54.128.34.174,"{""location"": ""PM"", ""is_mobile"": false}" 4333,7,403,2016-12-13 10:48:53,http://marquardtlarson.info/christelle,3.0000000000,155.230.246.40,"{""location"": ""PE"", ""is_mobile"": false}" 4334,7,403,2017-04-23 02:50:30,http://oreillycartwright.co/velda.windler,1.0000000000,70.216.196.161,"{""location"": ""GH"", ""is_mobile"": false}" 4335,7,403,2017-05-18 21:42:01,http://macgyvergottlieb.net/dameon,1.0000000000,76.40.132.13,"{""location"": ""SV"", ""is_mobile"": true}" 4336,7,403,2017-02-05 01:27:47,http://strackeroob.name/randy,3.0000000000,54.128.34.174,"{""location"": ""LA"", ""is_mobile"": false}" 4337,7,403,2017-05-25 21:32:56,http://zboncaklubowitz.info/bianka,1.0000000000,23.74.38.30,"{""location"": ""CG"", ""is_mobile"": true}" 4338,7,403,2016-12-26 03:07:40,http://wilkinson.org/austen.rath,3.0000000000,176.219.208.171,"{""location"": ""LI"", ""is_mobile"": true}" 4339,7,403,2017-01-22 02:36:43,http://lang.name/stanley_bernier,1.0000000000,183.3.121.210,"{""location"": ""SH"", ""is_mobile"": false}" 4340,7,403,2017-02-19 15:20:31,http://hermann.io/ora_kautzer,3.0000000000,186.5.89.84,"{""location"": ""VN"", ""is_mobile"": false}" 4341,7,404,2017-04-15 15:47:29,http://mann.net/monica_herman,0.0000000000,13.79.40.84,"{""location"": ""BI"", ""is_mobile"": false}" 4342,7,404,2017-02-01 20:23:38,http://connankunding.net/stella_yundt,2.0000000000,182.82.177.224,"{""location"": ""LB"", ""is_mobile"": false}" 4343,7,404,2017-03-08 12:08:59,http://gleichner.info/cedrick,0.0000000000,10.93.242.239,"{""location"": ""US"", ""is_mobile"": true}" 4344,7,404,2017-04-14 04:38:16,http://stoltenberg.org/donnell_leffler,0.0000000000,10.93.242.239,"{""location"": ""RE"", ""is_mobile"": false}" 4345,7,404,2016-12-23 03:24:13,http://stammskiles.biz/marcelino.hand,2.0000000000,52.186.141.52,"{""location"": ""IL"", ""is_mobile"": false}" 4346,7,405,2016-12-23 02:54:11,http://satterfield.co/camylle,2.0000000000,250.21.142.60,"{""location"": ""PK"", ""is_mobile"": false}" 4347,7,405,2017-04-11 14:51:38,http://powlowski.org/geovanny,2.0000000000,250.21.142.60,"{""location"": ""NC"", ""is_mobile"": false}" 4348,7,405,2017-03-04 13:17:40,http://funkmiller.biz/tiara,1.0000000000,9.73.244.104,"{""location"": ""KN"", ""is_mobile"": true}" 4349,7,406,2017-04-15 04:26:18,http://mcclure.info/santiago,1.0000000000,103.120.221.150,"{""location"": ""VE"", ""is_mobile"": true}" 4350,7,406,2016-12-18 15:10:03,http://tillman.org/ettie,0.0000000000,97.43.143.189,"{""location"": ""TH"", ""is_mobile"": false}" 4351,7,406,2017-05-24 18:16:28,http://cummings.org/armand,0.0000000000,216.34.141.27,"{""location"": ""CO"", ""is_mobile"": true}" 4352,7,407,2017-04-07 00:32:00,http://towne.name/carmela,1.0000000000,242.3.58.204,"{""location"": ""VG"", ""is_mobile"": false}" 4353,7,407,2017-05-27 17:48:58,http://borerkutch.info/marjolaine,0.0000000000,123.179.228.183,"{""location"": ""KN"", ""is_mobile"": true}" 4354,7,407,2017-01-08 02:37:49,http://ferry.biz/deshawn,1.0000000000,82.10.134.192,"{""location"": ""BG"", ""is_mobile"": false}" 4355,7,407,2017-02-25 00:07:21,http://ullrich.net/bianka,1.0000000000,138.49.145.235,"{""location"": ""SS"", ""is_mobile"": true}" 4356,7,407,2017-01-15 13:37:45,http://block.net/tracey,0.0000000000,183.243.67.234,"{""location"": ""IS"", ""is_mobile"": true}" 4357,7,407,2017-04-12 15:02:41,http://hammesjerde.com/jada.hills,0.0000000000,248.67.23.174,"{""location"": ""CH"", ""is_mobile"": true}" 4358,7,407,2017-01-29 18:07:38,http://bauch.org/raina_bogan,1.0000000000,187.138.48.43,"{""location"": ""GE"", ""is_mobile"": false}" 4359,7,407,2017-06-07 04:48:39,http://schamberger.info/kameron.padberg,0.0000000000,9.58.152.212,"{""location"": ""OM"", ""is_mobile"": false}" 4360,7,407,2017-03-24 04:45:57,http://harber.io/adelle,0.0000000000,187.138.48.43,"{""location"": ""CX"", ""is_mobile"": false}" 1653,3,151,2017-06-01 06:30:59,http://bradtkerohan.name/jaime,0.0000000000,221.120.162.48,"{""location"": ""RE"", ""is_mobile"": true}" 1654,3,151,2017-04-16 22:48:05,http://lubowitz.name/noemie,0.0000000000,148.254.14.220,"{""location"": ""PA"", ""is_mobile"": true}" 1655,3,152,2017-04-23 08:49:20,http://barton.name/ilene,1.0000000000,15.238.96.175,"{""location"": ""SY"", ""is_mobile"": true}" 1656,3,152,2016-12-22 02:03:06,http://dubuque.org/carlo_reinger,1.0000000000,179.195.168.149,"{""location"": ""ML"", ""is_mobile"": false}" 1657,3,152,2017-02-18 19:01:37,http://quigley.info/irma,0.0000000000,201.101.168.4,"{""location"": ""KG"", ""is_mobile"": true}" 1658,3,152,2017-03-10 14:34:01,http://fay.net/joe,1.0000000000,170.251.126.98,"{""location"": ""PY"", ""is_mobile"": false}" 1659,3,152,2017-03-21 22:17:18,http://hilllglover.com/braeden_botsford,0.0000000000,163.60.121.195,"{""location"": ""OM"", ""is_mobile"": false}" 1660,3,152,2017-06-09 03:15:57,http://langosh.com/joana.goyette,0.0000000000,82.154.179.169,"{""location"": ""CW"", ""is_mobile"": true}" 1661,3,152,2017-02-20 03:49:15,http://stark.io/john_schmidt,0.0000000000,24.113.203.65,"{""location"": ""KH"", ""is_mobile"": false}" 1662,3,152,2017-01-01 04:23:23,http://turcottecasper.com/myrl,1.0000000000,84.105.36.87,"{""location"": ""GF"", ""is_mobile"": true}" 1663,3,152,2017-01-24 13:54:51,http://schowalter.org/jaime_bradtke,1.0000000000,29.232.98.162,"{""location"": ""MY"", ""is_mobile"": true}" 1664,3,152,2017-05-12 12:23:21,http://kozey.name/carolyne_funk,1.0000000000,24.113.203.65,"{""location"": ""MV"", ""is_mobile"": true}" 1665,3,152,2017-05-23 03:31:33,http://hansen.name/emanuel.heidenreich,0.0000000000,246.66.6.8,"{""location"": ""NL"", ""is_mobile"": true}" 1666,3,152,2017-04-10 13:06:21,http://howell.net/geovany_fahey,1.0000000000,104.115.217.83,"{""location"": ""BN"", ""is_mobile"": true}" 1667,3,152,2017-01-12 06:51:35,http://larkin.net/blake,0.0000000000,15.70.183.109,"{""location"": ""KY"", ""is_mobile"": true}" 1668,3,152,2017-06-12 00:06:16,http://hilpert.io/meredith.hoeger,1.0000000000,92.83.129.48,"{""location"": ""BI"", ""is_mobile"": false}" 1669,3,152,2017-01-01 00:43:51,http://reinger.name/aglae_pouros,1.0000000000,163.60.121.195,"{""location"": ""GA"", ""is_mobile"": true}" 1670,3,153,2017-04-27 18:54:01,http://upton.org/lauryn,0.0000000000,6.202.48.23,"{""location"": ""FK"", ""is_mobile"": true}" 1671,3,153,2017-05-20 18:38:09,http://ward.org/erick,3.0000000000,137.246.95.23,"{""location"": ""PK"", ""is_mobile"": false}" 1672,3,153,2017-02-08 10:47:37,http://macejkovic.info/kobe,2.0000000000,231.75.84.206,"{""location"": ""BY"", ""is_mobile"": false}" 1673,3,154,2017-04-16 21:23:55,http://lind.org/everardo.ferry,0.0000000000,45.31.134.130,"{""location"": ""CF"", ""is_mobile"": true}" 1674,3,154,2017-05-03 11:11:15,http://bruen.co/vanea_sawayn,0.0000000000,206.243.44.99,"{""location"": ""BG"", ""is_mobile"": true}" 1675,3,154,2017-06-05 04:19:25,http://mann.io/bernie.kunde,0.0000000000,36.43.123.187,"{""location"": ""TR"", ""is_mobile"": true}" 1676,3,154,2017-01-08 17:21:13,http://watsica.co/timmothy,0.0000000000,110.253.18.210,"{""location"": ""AW"", ""is_mobile"": true}" 1677,3,154,2016-12-20 18:55:35,http://prohaskakovacek.net/rafaela.wilkinson,0.0000000000,6.159.169.196,"{""location"": ""DJ"", ""is_mobile"": true}" 1678,3,154,2017-03-16 11:44:39,http://stokes.name/tyshawn,0.0000000000,69.66.93.12,"{""location"": ""ST"", ""is_mobile"": true}" 1679,3,155,2017-02-17 22:03:35,http://cain.org/ezra,0.0000000000,2.57.161.73,"{""location"": ""HU"", ""is_mobile"": false}" 1680,3,155,2016-12-20 14:17:53,http://willms.name/sedrick,1.0000000000,11.205.230.155,"{""location"": ""RU"", ""is_mobile"": false}" 1681,3,155,2017-06-03 03:03:53,http://wisoky.co/aileen,1.0000000000,86.110.236.217,"{""location"": ""NO"", ""is_mobile"": false}" 1682,3,155,2017-04-20 00:44:00,http://towne.biz/noemie,0.0000000000,143.220.194.91,"{""location"": ""SG"", ""is_mobile"": false}" 1683,3,155,2016-12-22 20:03:41,http://effertzfranecki.info/gennaro_volkman,1.0000000000,196.74.108.219,"{""location"": ""LS"", ""is_mobile"": true}" 1684,3,156,2017-02-15 13:37:19,http://zieme.org/holden,,178.197.31.87,"{""location"": ""MP"", ""is_mobile"": true}" 1685,3,156,2017-01-10 11:33:49,http://lowebeahan.net/norwood.walker,,241.253.46.200,"{""location"": ""AI"", ""is_mobile"": false}" 1686,3,156,2016-12-27 10:15:25,http://hicklemacejkovic.org/hillary,,155.53.24.179,"{""location"": ""SR"", ""is_mobile"": false}" 1687,3,156,2017-06-01 03:06:05,http://heller.co/diego,,84.200.94.200,"{""location"": ""SB"", ""is_mobile"": true}" 1688,3,156,2017-01-05 07:57:17,http://bahringer.co/imani,,149.93.36.108,"{""location"": ""SA"", ""is_mobile"": false}" 1689,3,156,2017-05-31 00:57:29,http://price.io/eloise_ohara,,152.146.215.47,"{""location"": ""NZ"", ""is_mobile"": false}" 1690,3,156,2017-05-15 17:25:42,http://macejkovicfranecki.io/stephan_roberts,,55.72.83.242,"{""location"": ""MM"", ""is_mobile"": false}" 1691,3,156,2017-04-27 06:41:12,http://langosh.net/rose,,152.71.16.54,"{""location"": ""UM"", ""is_mobile"": false}" 1692,3,156,2017-03-21 18:28:56,http://fritschmorar.com/elton,,246.51.236.43,"{""location"": ""IN"", ""is_mobile"": false}" 1693,3,156,2017-05-01 14:11:40,http://walkerwilderman.net/manley_abbott,,201.230.70.102,"{""location"": ""FI"", ""is_mobile"": true}" 1694,3,156,2017-01-25 09:47:58,http://hermanhodkiewicz.net/cristal_effertz,,249.62.162.122,"{""location"": ""TZ"", ""is_mobile"": true}" 1695,3,157,2017-06-13 16:55:30,http://lowe.name/oscar,,194.95.100.100,"{""location"": ""ZW"", ""is_mobile"": true}" 1696,3,157,2017-04-25 06:39:46,http://williamson.biz/frederic,,65.130.186.183,"{""location"": ""VU"", ""is_mobile"": false}" 1697,3,157,2017-06-06 01:19:37,http://rogahngusikowski.net/joelle,,88.244.35.187,"{""location"": ""GT"", ""is_mobile"": true}" 1698,3,157,2017-03-14 07:43:29,http://hoppe.co/jaden_ebert,,165.12.116.6,"{""location"": ""HM"", ""is_mobile"": false}" 1699,3,157,2017-06-09 15:36:08,http://zulaufwilderman.org/marcelo_parisian,,4.49.48.55,"{""location"": ""SB"", ""is_mobile"": true}" 1700,3,157,2017-04-08 00:24:24,http://lemkegleason.net/virgie,,98.154.194.120,"{""location"": ""CD"", ""is_mobile"": false}" 1701,3,157,2017-03-20 11:39:04,http://berge.co/addison.bailey,,4.49.48.55,"{""location"": ""MX"", ""is_mobile"": false}" 1702,3,157,2017-01-28 13:49:40,http://lowe.io/kaley.macgyver,,23.139.119.35,"{""location"": ""AQ"", ""is_mobile"": true}" 1703,3,157,2017-05-11 20:37:24,http://aufderhar.org/heloise_graham,,206.39.33.102,"{""location"": ""UY"", ""is_mobile"": true}" 1704,3,157,2017-04-19 18:17:11,http://turcottehudson.com/malachi,,88.244.35.187,"{""location"": ""MY"", ""is_mobile"": false}" 1705,3,157,2017-05-17 07:20:41,http://bechtelar.com/shea.okuneva,,88.244.35.187,"{""location"": ""SD"", ""is_mobile"": true}" 1706,3,157,2017-01-31 00:25:24,http://koelpinklein.biz/misty,,90.62.72.176,"{""location"": ""SN"", ""is_mobile"": false}" 4361,7,407,2017-03-13 17:29:01,http://ko.name/aleen,0.0000000000,139.101.229.179,"{""location"": ""SN"", ""is_mobile"": true}" 4362,7,407,2016-12-23 03:01:47,http://homenick.com/jewell_emard,0.0000000000,187.138.48.43,"{""location"": ""BB"", ""is_mobile"": false}" 4363,7,407,2017-05-23 03:53:00,http://okeefebotsford.name/clementine.rohan,0.0000000000,199.94.165.130,"{""location"": ""CD"", ""is_mobile"": false}" 4364,7,407,2017-02-07 18:12:54,http://bogan.io/dolly_howell,1.0000000000,185.106.84.72,"{""location"": ""NE"", ""is_mobile"": false}" 4365,7,407,2016-12-16 23:42:30,http://keelinglebsack.info/angie.bogan,0.0000000000,139.101.229.179,"{""location"": ""MM"", ""is_mobile"": false}" 4366,7,407,2017-02-04 17:31:19,http://wintheiser.com/macey,1.0000000000,187.138.48.43,"{""location"": ""CX"", ""is_mobile"": true}" 4367,7,407,2017-04-20 18:36:26,http://treutelcruickshank.io/zetta,1.0000000000,250.137.218.215,"{""location"": ""MU"", ""is_mobile"": false}" 4368,7,407,2017-03-20 20:01:43,http://quitzon.io/trea,1.0000000000,132.111.203.177,"{""location"": ""BF"", ""is_mobile"": true}" 4369,7,407,2017-01-30 01:45:26,http://osinski.biz/arvid_braun,0.0000000000,185.106.84.72,"{""location"": ""IO"", ""is_mobile"": true}" 4370,7,407,2017-03-23 10:25:51,http://becker.co/laila,0.0000000000,2.136.191.137,"{""location"": ""CA"", ""is_mobile"": false}" 4371,7,408,2017-01-05 10:42:12,http://hilll.org/blaze.treutel,3.0000000000,252.49.103.206,"{""location"": ""AG"", ""is_mobile"": false}" 4372,7,408,2017-03-10 18:20:11,http://romaguera.biz/antwan,3.0000000000,198.10.93.42,"{""location"": ""BZ"", ""is_mobile"": true}" 4373,7,408,2017-01-17 00:04:50,http://konopelski.co/kamren.kris,0.0000000000,70.249.236.73,"{""location"": ""SC"", ""is_mobile"": true}" 4374,7,408,2017-04-04 09:03:55,http://zulauf.biz/sam_kunze,2.0000000000,228.234.152.126,"{""location"": ""ZM"", ""is_mobile"": true}" 4375,7,408,2017-01-31 02:49:37,http://schmitt.co/dannie,1.0000000000,199.12.17.119,"{""location"": ""SS"", ""is_mobile"": true}" 4376,7,408,2017-05-31 06:37:54,http://reinger.co/donato_upton,2.0000000000,41.241.20.106,"{""location"": ""PH"", ""is_mobile"": true}" 4377,7,408,2017-03-12 23:47:38,http://little.io/aurore_willms,3.0000000000,199.12.17.119,"{""location"": ""BT"", ""is_mobile"": true}" 4378,7,408,2017-03-28 07:44:45,http://bartell.net/chester,1.0000000000,73.230.4.135,"{""location"": ""NG"", ""is_mobile"": false}" 4379,7,408,2017-01-17 06:46:50,http://mayert.co/jarrod,3.0000000000,245.128.243.140,"{""location"": ""NL"", ""is_mobile"": false}" 4380,7,408,2017-03-15 17:51:32,http://auerkulas.org/lucius_wolff,3.0000000000,73.230.4.135,"{""location"": ""TL"", ""is_mobile"": false}" 4381,7,408,2017-01-14 21:21:41,http://hickle.org/alden_farrell,2.0000000000,87.100.69.116,"{""location"": ""PT"", ""is_mobile"": true}" 4382,7,408,2017-05-05 05:48:13,http://keebler.io/tyrique,3.0000000000,88.11.41.6,"{""location"": ""SN"", ""is_mobile"": true}" 4383,7,408,2017-04-29 12:37:28,http://hermanruecker.biz/mckayla,0.0000000000,252.49.103.206,"{""location"": ""PT"", ""is_mobile"": false}" 4384,7,408,2017-04-15 20:30:24,http://kuhlman.com/jaqueline,0.0000000000,199.12.17.119,"{""location"": ""BA"", ""is_mobile"": true}" 4385,7,408,2016-12-20 11:47:11,http://quigley.io/salvatore.leffler,0.0000000000,87.100.69.116,"{""location"": ""ES"", ""is_mobile"": true}" 4386,7,409,2017-04-01 06:07:01,http://dickinson.io/franz.cummerata,0.0000000000,90.4.13.225,"{""location"": ""SO"", ""is_mobile"": false}" 4387,7,409,2017-03-24 19:12:20,http://stehr.biz/chris,0.0000000000,94.43.243.113,"{""location"": ""SD"", ""is_mobile"": false}" 4388,7,409,2017-01-07 06:17:41,http://hauck.net/jackson_mckenzie,0.0000000000,168.237.163.15,"{""location"": ""AZ"", ""is_mobile"": true}" 4389,7,409,2017-01-16 20:19:17,http://willms.io/johan.torphy,1.0000000000,176.68.91.130,"{""location"": ""GE"", ""is_mobile"": false}" 4390,7,409,2017-05-29 11:50:55,http://balistreri.net/gerda,1.0000000000,247.162.57.135,"{""location"": ""CM"", ""is_mobile"": false}" 4391,7,409,2017-01-21 13:31:14,http://wuckert.io/zena_jast,1.0000000000,206.67.67.190,"{""location"": ""WF"", ""is_mobile"": false}" 4392,7,409,2017-01-08 23:36:15,http://stehr.co/oda,0.0000000000,168.237.163.15,"{""location"": ""ME"", ""is_mobile"": false}" 4393,7,409,2017-03-27 01:04:20,http://jacobs.name/tomasa,1.0000000000,217.129.230.215,"{""location"": ""AG"", ""is_mobile"": false}" 4394,7,410,2016-12-19 22:40:32,http://murphywisoky.net/alva.davis,0.0000000000,147.177.161.209,"{""location"": ""DJ"", ""is_mobile"": false}" 4395,7,410,2017-03-22 08:03:06,http://schmidtbeier.com/rosemary.mayert,1.0000000000,147.177.161.209,"{""location"": ""CD"", ""is_mobile"": true}" 4396,7,410,2017-02-10 12:55:44,http://feil.io/obie.barton,1.0000000000,211.112.82.98,"{""location"": ""HT"", ""is_mobile"": false}" 4397,7,410,2017-04-26 00:10:27,http://satterfieldschowalter.name/muhammad_marquardt,0.0000000000,16.172.245.102,"{""location"": ""TV"", ""is_mobile"": false}" 4398,7,410,2017-05-29 15:56:43,http://schusterframi.info/jeie,1.0000000000,63.125.133.192,"{""location"": ""JE"", ""is_mobile"": true}" 4399,7,410,2017-05-21 19:47:59,http://hermannschulist.co/leola_heller,1.0000000000,221.199.147.216,"{""location"": ""TC"", ""is_mobile"": true}" 4400,7,410,2017-03-16 19:01:13,http://smith.co/elisabeth.lynch,1.0000000000,172.23.177.218,"{""location"": ""CH"", ""is_mobile"": false}" 4401,7,410,2017-01-28 19:54:41,http://oconner.io/brett_streich,1.0000000000,122.92.144.34,"{""location"": ""SK"", ""is_mobile"": true}" 4402,7,410,2017-01-27 20:32:36,http://runolfon.name/madie.sanford,1.0000000000,16.172.245.102,"{""location"": ""NE"", ""is_mobile"": false}" 4403,7,410,2017-02-17 20:21:05,http://thiel.name/tina,0.0000000000,90.250.225.145,"{""location"": ""VA"", ""is_mobile"": true}" 4404,7,410,2017-03-23 11:29:19,http://leffler.co/macie,0.0000000000,128.22.149.238,"{""location"": ""UZ"", ""is_mobile"": true}" 4405,7,410,2017-01-31 14:16:48,http://brekkerodriguez.net/erin,1.0000000000,206.27.219.30,"{""location"": ""CC"", ""is_mobile"": true}" 4406,7,410,2017-05-28 12:18:52,http://wildermanhermann.com/gerda,0.0000000000,175.23.112.67,"{""location"": ""SE"", ""is_mobile"": true}" 4407,7,410,2016-12-17 23:20:11,http://wildermanmayer.org/charley,0.0000000000,68.153.104.170,"{""location"": ""IS"", ""is_mobile"": true}" 4408,7,410,2016-12-14 01:16:21,http://rempelsatterfield.biz/gudrun,1.0000000000,206.27.219.30,"{""location"": ""AZ"", ""is_mobile"": true}" 4409,7,410,2017-04-18 14:24:32,http://barrows.info/chesley,0.0000000000,211.160.210.187,"{""location"": ""MU"", ""is_mobile"": true}" 4410,7,410,2017-02-26 04:07:04,http://glover.co/hildegard,0.0000000000,161.15.20.8,"{""location"": ""CW"", ""is_mobile"": false}" 4411,7,410,2017-01-12 19:33:19,http://conroy.org/pete_ortiz,0.0000000000,122.92.144.34,"{""location"": ""TL"", ""is_mobile"": false}" 4412,7,410,2017-02-11 16:03:36,http://roob.net/abbie_moriette,1.0000000000,211.160.210.187,"{""location"": ""HN"", ""is_mobile"": false}" 1707,3,157,2017-01-07 08:44:31,http://adams.net/jaden,,251.102.64.231,"{""location"": ""CW"", ""is_mobile"": true}" 1708,3,157,2017-02-12 14:31:51,http://veumkemmer.info/tanya.ritchie,,23.139.119.35,"{""location"": ""BQ"", ""is_mobile"": true}" 1709,3,157,2017-01-16 17:56:32,http://hudson.name/adrian_mitchell,,194.95.100.100,"{""location"": ""PF"", ""is_mobile"": false}" 1710,3,157,2017-01-27 07:19:29,http://kulas.co/kristopher_cummings,,23.139.119.35,"{""location"": ""IM"", ""is_mobile"": false}" 1711,3,157,2017-04-22 21:48:37,http://okuneva.org/alysha,,43.103.222.200,"{""location"": ""PT"", ""is_mobile"": true}" 1712,3,157,2016-12-14 07:32:28,http://oberbrunnerlangosh.biz/ava_williamson,,42.213.211.116,"{""location"": ""KZ"", ""is_mobile"": true}" 1713,3,157,2017-03-24 00:26:31,http://ferryauer.name/brent,,4.49.48.55,"{""location"": ""NO"", ""is_mobile"": false}" 1714,3,157,2017-04-30 00:43:12,http://larkinvon.net/gardner,,176.177.160.109,"{""location"": ""KI"", ""is_mobile"": false}" 1715,3,158,2017-01-07 19:44:00,http://stammmarquardt.org/dwight,,158.81.223.15,"{""location"": ""TT"", ""is_mobile"": true}" 1716,3,158,2017-03-08 13:56:30,http://sawaynlabadie.com/athena,,228.220.91.80,"{""location"": ""SR"", ""is_mobile"": true}" 1717,3,158,2017-04-23 20:45:13,http://oberbrunner.io/marcus,,55.4.145.65,"{""location"": ""TC"", ""is_mobile"": false}" 1718,3,158,2016-12-14 04:12:53,http://bartolettiweimann.biz/jovan_kertzmann,,117.138.164.147,"{""location"": ""IT"", ""is_mobile"": false}" 1719,3,158,2017-03-28 19:50:23,http://flatley.io/julio.bruen,,45.151.93.62,"{""location"": ""ML"", ""is_mobile"": false}" 1720,3,158,2017-05-05 12:21:28,http://feeneywalsh.biz/waino,,195.149.150.85,"{""location"": ""KW"", ""is_mobile"": false}" 1721,3,158,2017-05-29 02:53:00,http://christiansen.biz/joannie_hettinger,,216.103.188.13,"{""location"": ""YT"", ""is_mobile"": true}" 1722,3,158,2017-02-05 01:09:37,http://simonis.io/margaretta,,80.54.198.157,"{""location"": ""MH"", ""is_mobile"": true}" 1723,3,158,2017-03-16 10:56:38,http://roberts.com/emmy_cartwright,,158.81.223.15,"{""location"": ""BE"", ""is_mobile"": false}" 1724,3,158,2017-03-09 22:58:13,http://runolfon.com/katelynn.ondricka,,88.201.217.167,"{""location"": ""VG"", ""is_mobile"": true}" 1725,3,158,2017-01-29 22:12:54,http://mcglynn.io/earl_jacobs,,84.101.88.27,"{""location"": ""HU"", ""is_mobile"": true}" 1726,3,158,2017-04-11 08:53:45,http://lowefeeney.io/elsa_oconner,,72.46.79.24,"{""location"": ""TC"", ""is_mobile"": false}" 1727,3,158,2017-02-09 04:28:26,http://gutmann.co/sylvan,,117.138.164.147,"{""location"": ""RE"", ""is_mobile"": true}" 1728,3,158,2017-01-07 15:08:59,http://pacocha.info/juliet,,119.100.42.23,"{""location"": ""PT"", ""is_mobile"": true}" 1729,3,158,2017-03-17 17:47:57,http://cartwright.biz/ruel.jenkins,,121.31.16.249,"{""location"": ""LA"", ""is_mobile"": true}" 1730,3,158,2017-02-24 15:06:24,http://adams.net/lucile,,121.102.204.182,"{""location"": ""AI"", ""is_mobile"": true}" 1731,3,158,2017-05-29 05:14:01,http://hirthe.name/alden,,179.60.152.250,"{""location"": ""GE"", ""is_mobile"": false}" 1732,3,158,2017-05-08 01:21:48,http://turner.info/anais,,80.54.198.157,"{""location"": ""HK"", ""is_mobile"": true}" 1733,3,158,2016-12-21 14:25:02,http://swaniawskipfeffer.org/brock_fahey,,113.58.93.42,"{""location"": ""MH"", ""is_mobile"": false}" 1734,3,158,2017-01-08 00:01:18,http://cremincremin.io/erin,,117.138.164.147,"{""location"": ""NC"", ""is_mobile"": true}" 1735,3,159,2017-03-06 04:35:20,http://berniermueller.io/houston_spinka,,230.45.87.235,"{""location"": ""AO"", ""is_mobile"": false}" 1736,3,159,2017-03-22 06:12:30,http://mayert.io/paul,,165.215.98.227,"{""location"": ""WF"", ""is_mobile"": true}" 1737,3,159,2017-01-03 05:11:57,http://boyle.name/ludwig.mclaughlin,,219.2.229.222,"{""location"": ""BV"", ""is_mobile"": false}" 1738,3,160,2017-04-29 07:26:34,http://abshireschoen.co/orville,,86.28.213.59,"{""location"": ""GR"", ""is_mobile"": false}" 1739,3,160,2017-04-30 10:49:54,http://hills.info/maximo_senger,,133.244.13.12,"{""location"": ""TM"", ""is_mobile"": false}" 1740,3,160,2017-01-06 00:24:22,http://hansenkovacek.org/kenyatta_littel,,66.51.28.173,"{""location"": ""FK"", ""is_mobile"": false}" 1741,3,160,2017-05-27 12:10:54,http://reinger.io/ena,,86.28.213.59,"{""location"": ""VN"", ""is_mobile"": false}" 1742,3,161,2017-01-03 12:36:56,http://deckow.co/ulises,,127.51.194.62,"{""location"": ""BI"", ""is_mobile"": true}" 1743,3,161,2017-06-03 23:35:08,http://quigley.org/ally,,81.199.12.157,"{""location"": ""AL"", ""is_mobile"": true}" 1744,3,161,2017-03-13 03:54:05,http://hilperttoy.org/elinore_lesch,,3.205.225.243,"{""location"": ""KI"", ""is_mobile"": false}" 1745,3,162,2017-03-27 06:06:07,http://vandervort.biz/sally_hahn,0.0000000000,91.70.192.95,"{""location"": ""SN"", ""is_mobile"": true}" 1746,3,163,2016-12-24 06:35:08,http://stoltenberg.org/felicia,1.0000000000,75.195.242.86,"{""location"": ""FK"", ""is_mobile"": false}" 1747,3,163,2017-01-08 07:35:17,http://haley.name/aurelio,1.0000000000,72.58.35.112,"{""location"": ""RO"", ""is_mobile"": true}" 1748,3,163,2016-12-17 08:51:32,http://koepp.name/jazmyne.kunde,1.0000000000,62.110.170.69,"{""location"": ""MT"", ""is_mobile"": true}" 1749,3,163,2017-05-09 14:53:27,http://stark.io/annette.monahan,0.0000000000,87.193.175.243,"{""location"": ""FK"", ""is_mobile"": true}" 1750,3,163,2017-04-12 01:17:53,http://altenwerth.io/caitlyn.doyle,0.0000000000,72.58.35.112,"{""location"": ""NE"", ""is_mobile"": false}" 1751,3,164,2017-06-09 15:11:21,http://collinsprohaska.co/darian,0.0000000000,75.127.129.85,"{""location"": ""BL"", ""is_mobile"": false}" 1752,3,164,2017-05-02 19:17:20,http://wisozkfranecki.org/mikayla_predovic,0.0000000000,99.172.128.31,"{""location"": ""BR"", ""is_mobile"": false}" 1753,3,164,2017-05-29 18:19:22,http://braunwunsch.io/neoma,0.0000000000,12.119.54.120,"{""location"": ""GM"", ""is_mobile"": false}" 1754,3,164,2017-01-19 15:28:53,http://powlowski.com/javier.wunsch,0.0000000000,238.179.37.42,"{""location"": ""PN"", ""is_mobile"": false}" 1755,3,164,2017-03-14 00:10:46,http://jakubowskimiller.io/morton,0.0000000000,146.36.41.85,"{""location"": ""MZ"", ""is_mobile"": true}" 1756,3,164,2017-02-14 15:08:13,http://stanton.org/madelyn,0.0000000000,253.251.177.33,"{""location"": ""VN"", ""is_mobile"": true}" 1757,3,164,2017-04-11 15:11:14,http://labadie.com/abraham,0.0000000000,179.213.223.57,"{""location"": ""AT"", ""is_mobile"": false}" 1758,3,164,2016-12-15 09:49:51,http://medhurstruel.io/icie_rolfson,0.0000000000,16.89.34.116,"{""location"": ""MW"", ""is_mobile"": true}" 1759,3,164,2017-03-11 20:21:48,http://hahn.com/lawrence_osinski,0.0000000000,69.35.157.93,"{""location"": ""ST"", ""is_mobile"": false}" 1760,3,164,2016-12-15 07:35:41,http://considine.biz/clint.douglas,0.0000000000,238.230.164.233,"{""location"": ""NU"", ""is_mobile"": true}" 4413,7,411,2017-05-10 21:17:54,http://koepp.info/salvador,1.0000000000,162.53.77.92,"{""location"": ""PE"", ""is_mobile"": true}" 4414,7,411,2017-04-27 12:16:25,http://shanahan.co/madaline,0.0000000000,10.30.138.71,"{""location"": ""LK"", ""is_mobile"": false}" 4415,7,411,2017-04-03 17:59:58,http://parkerkoch.biz/eric.hintz,1.0000000000,129.119.140.146,"{""location"": ""LK"", ""is_mobile"": false}" 4416,7,411,2017-03-03 03:49:39,http://runolfsdottir.biz/sherman,1.0000000000,224.100.213.172,"{""location"": ""GY"", ""is_mobile"": false}" 4417,7,411,2017-03-18 23:47:58,http://wildermankirlin.co/elliot.crona,2.0000000000,140.152.217.97,"{""location"": ""UY"", ""is_mobile"": true}" 4418,7,411,2017-06-12 15:09:48,http://feestquitzon.info/kristopher,2.0000000000,235.130.63.53,"{""location"": ""VU"", ""is_mobile"": true}" 4419,7,411,2017-06-13 00:25:23,http://prosacco.info/edmund,1.0000000000,237.117.128.155,"{""location"": ""VN"", ""is_mobile"": true}" 4420,7,412,2017-04-09 19:56:42,http://johnsgrady.info/destinee_stanton,,184.220.93.59,"{""location"": ""GD"", ""is_mobile"": false}" 4421,7,412,2017-04-06 15:59:56,http://monahan.net/roel_ward,,55.126.218.64,"{""location"": ""MM"", ""is_mobile"": true}" 4422,7,412,2017-02-22 01:01:41,http://kozey.biz/ubaldo_mohr,,30.71.237.191,"{""location"": ""AQ"", ""is_mobile"": false}" 4423,7,412,2017-03-18 13:50:12,http://dibbert.name/eulalia,,184.220.93.59,"{""location"": ""VG"", ""is_mobile"": true}" 4424,7,412,2017-02-16 18:33:25,http://casper.io/dante,,37.108.200.64,"{""location"": ""LT"", ""is_mobile"": false}" 4425,7,412,2017-04-15 09:46:09,http://koepp.io/christian,,13.58.229.173,"{""location"": ""GN"", ""is_mobile"": true}" 4426,7,412,2017-02-12 18:09:49,http://mcglynnnader.net/judd.gleason,,190.34.65.112,"{""location"": ""TW"", ""is_mobile"": true}" 4427,7,413,2017-06-10 11:46:09,http://windler.biz/aric,,49.220.24.253,"{""location"": ""TT"", ""is_mobile"": false}" 4428,7,413,2017-02-10 07:35:44,http://marquardtrowe.org/kattie_pfannerstill,,44.210.148.194,"{""location"": ""AM"", ""is_mobile"": false}" 4429,7,413,2017-01-31 19:35:35,http://flatley.info/kareem_borer,,33.233.50.200,"{""location"": ""MK"", ""is_mobile"": true}" 4430,7,413,2017-02-15 03:28:03,http://hills.name/jaqueline.kovacek,,50.72.186.216,"{""location"": ""BQ"", ""is_mobile"": false}" 4431,7,413,2016-12-24 20:25:26,http://carter.org/darwin,,52.79.228.206,"{""location"": ""CF"", ""is_mobile"": true}" 4432,7,414,2017-02-28 15:30:57,http://goodwin.name/darryl,,17.169.156.199,"{""location"": ""SV"", ""is_mobile"": true}" 4433,7,414,2017-04-05 20:14:34,http://hodkiewiczdickens.com/milford,,145.54.129.28,"{""location"": ""PA"", ""is_mobile"": true}" 4434,7,414,2017-06-10 14:24:55,http://bartell.name/mathias_barton,,53.50.107.85,"{""location"": ""CU"", ""is_mobile"": false}" 4435,7,414,2017-03-05 17:40:12,http://mraz.co/charity,,241.249.222.134,"{""location"": ""TR"", ""is_mobile"": true}" 4436,7,414,2017-01-15 13:10:47,http://monahangerhold.biz/vernice,,44.214.86.249,"{""location"": ""BN"", ""is_mobile"": true}" 4437,7,414,2016-12-17 16:09:22,http://quitzon.net/dana,,88.8.20.116,"{""location"": ""CV"", ""is_mobile"": false}" 4438,7,414,2017-01-08 13:24:36,http://bartonnicolas.biz/gerda,,83.175.210.203,"{""location"": ""LY"", ""is_mobile"": false}" 4439,7,414,2017-05-23 05:42:06,http://bahringer.name/angelina.kling,,145.6.174.237,"{""location"": ""AT"", ""is_mobile"": true}" 4440,7,414,2017-05-30 04:05:58,http://lang.name/jazmyne,,225.94.136.242,"{""location"": ""VE"", ""is_mobile"": false}" 4441,7,414,2017-05-27 03:37:20,http://steuber.io/marcelina,,54.200.21.151,"{""location"": ""MQ"", ""is_mobile"": false}" 4442,7,414,2017-02-05 21:22:04,http://sanford.co/nelle,,38.251.220.20,"{""location"": ""MW"", ""is_mobile"": false}" 4443,7,414,2017-05-19 14:11:07,http://brekke.info/myles,,167.91.46.135,"{""location"": ""TG"", ""is_mobile"": true}" 4444,7,414,2017-06-01 10:57:15,http://cummingswisozk.org/halie,,70.175.48.223,"{""location"": ""SE"", ""is_mobile"": true}" 4445,7,414,2017-02-11 00:30:15,http://torpschoen.org/lucienne_kling,,135.174.3.128,"{""location"": ""NC"", ""is_mobile"": true}" 4446,7,414,2017-01-31 12:19:13,http://lynch.org/wendell,,235.248.252.213,"{""location"": ""BV"", ""is_mobile"": false}" 4447,7,414,2017-03-28 12:47:06,http://ruel.biz/everette,,21.139.197.106,"{""location"": ""GU"", ""is_mobile"": false}" 4448,7,414,2017-05-22 17:36:29,http://larsonschmidt.net/jermey.bartell,,188.25.233.169,"{""location"": ""BD"", ""is_mobile"": true}" 4449,7,414,2017-01-12 09:27:52,http://bruenpollich.biz/madonna_von,,12.35.248.179,"{""location"": ""AO"", ""is_mobile"": true}" 4450,7,415,2017-02-07 01:03:56,http://hahnschmeler.name/corrine_ondricka,,149.121.27.77,"{""location"": ""IE"", ""is_mobile"": false}" 4451,7,415,2017-02-17 07:33:21,http://bayer.com/christine.thompson,,15.49.38.106,"{""location"": ""ID"", ""is_mobile"": true}" 4452,7,415,2017-01-31 20:44:43,http://bogisichglover.org/dena.lehner,,149.121.27.77,"{""location"": ""SJ"", ""is_mobile"": false}" 4453,7,416,2017-04-03 20:09:59,http://padberg.info/alanna.gerlach,,83.226.129.208,"{""location"": ""AT"", ""is_mobile"": false}" 4454,7,416,2017-01-21 03:11:14,http://davis.net/jerome_jacobson,,100.220.71.66,"{""location"": ""SX"", ""is_mobile"": false}" 4455,7,416,2016-12-28 06:26:55,http://ullrich.biz/melvin,,123.73.175.42,"{""location"": ""AT"", ""is_mobile"": false}" 4456,7,416,2017-03-20 10:45:49,http://klockomayer.net/lonie,,152.73.236.145,"{""location"": ""MZ"", ""is_mobile"": true}" 4457,7,416,2017-01-15 05:53:46,http://oconnelldoyle.info/alan,,6.141.207.185,"{""location"": ""LC"", ""is_mobile"": true}" 4458,7,416,2017-01-04 12:19:58,http://crona.name/ayden,,225.145.247.109,"{""location"": ""MK"", ""is_mobile"": true}" 4459,7,416,2017-03-05 15:49:14,http://rempel.com/tracy,,243.135.52.203,"{""location"": ""AX"", ""is_mobile"": true}" 4460,7,416,2017-03-06 23:50:50,http://pacochakunde.co/jailyn.cronin,,212.194.182.135,"{""location"": ""MZ"", ""is_mobile"": true}" 4461,7,416,2017-05-04 15:50:17,http://bradtkegerhold.com/adolph_west,,95.216.150.22,"{""location"": ""SD"", ""is_mobile"": true}" 4462,7,416,2017-05-11 11:12:36,http://johnston.io/ruel,,79.139.203.30,"{""location"": ""GS"", ""is_mobile"": true}" 4463,7,416,2017-01-12 20:51:52,http://rogahnfay.info/lavonne.rau,,22.142.58.62,"{""location"": ""SV"", ""is_mobile"": true}" 4464,7,416,2016-12-25 17:30:31,http://marvin.io/sidney.ryan,,97.177.237.69,"{""location"": ""BH"", ""is_mobile"": false}" 4465,7,416,2017-01-19 15:47:43,http://littleschaefer.net/isabelle.schamberger,,39.93.126.242,"{""location"": ""PE"", ""is_mobile"": false}" 4466,7,416,2017-06-07 03:17:51,http://jenkins.org/flavio_koelpin,,8.131.71.87,"{""location"": ""GE"", ""is_mobile"": false}" 4467,7,416,2017-04-05 12:35:47,http://lindgrenrutherford.co/nina.ko,,140.69.68.180,"{""location"": ""KM"", ""is_mobile"": false}" 1761,3,164,2017-04-12 02:33:42,http://kingnader.biz/jeyca,0.0000000000,37.80.142.147,"{""location"": ""BE"", ""is_mobile"": false}" 1762,3,164,2017-03-31 03:58:58,http://kuhic.com/dell.macgyver,0.0000000000,109.64.172.103,"{""location"": ""FI"", ""is_mobile"": true}" 1763,3,164,2017-04-04 08:21:33,http://becker.io/jordi,0.0000000000,199.121.60.179,"{""location"": ""GN"", ""is_mobile"": true}" 1764,3,164,2017-05-03 19:56:16,http://cummerata.org/nathaniel,0.0000000000,195.160.220.132,"{""location"": ""CA"", ""is_mobile"": false}" 1765,3,165,2017-05-21 15:39:28,http://kshlerinmarks.info/caden,1.0000000000,188.233.90.228,"{""location"": ""GD"", ""is_mobile"": false}" 1766,3,165,2017-03-23 06:00:29,http://bartonhintz.com/elyse.volkman,2.0000000000,56.12.154.166,"{""location"": ""ST"", ""is_mobile"": false}" 1767,3,165,2017-06-05 15:11:53,http://okon.name/stevie,0.0000000000,244.102.176.246,"{""location"": ""IR"", ""is_mobile"": true}" 1768,3,165,2017-01-27 14:07:34,http://oconnerpacocha.info/sallie,0.0000000000,164.185.21.232,"{""location"": ""LR"", ""is_mobile"": true}" 1769,3,165,2017-04-19 18:50:58,http://rohan.io/rhianna,1.0000000000,66.57.115.208,"{""location"": ""NG"", ""is_mobile"": true}" 1770,3,165,2017-01-24 12:16:21,http://gusikowskihoeger.io/chet,0.0000000000,134.83.129.211,"{""location"": ""MM"", ""is_mobile"": true}" 1771,3,165,2017-03-12 23:20:53,http://wyman.com/ryder,1.0000000000,178.112.224.240,"{""location"": ""ZA"", ""is_mobile"": false}" 1772,3,165,2017-01-20 00:53:28,http://jacobson.co/liliana,0.0000000000,250.143.250.186,"{""location"": ""GW"", ""is_mobile"": true}" 1773,3,165,2017-05-14 16:45:43,http://grimes.biz/gino,0.0000000000,114.42.156.7,"{""location"": ""NC"", ""is_mobile"": false}" 1774,3,165,2017-01-04 18:40:20,http://marks.net/sterling,0.0000000000,185.21.168.171,"{""location"": ""PL"", ""is_mobile"": false}" 1775,3,165,2017-04-05 21:39:07,http://rodriguez.com/makenna_rice,0.0000000000,229.150.50.36,"{""location"": ""IM"", ""is_mobile"": true}" 1776,3,166,2017-05-20 18:01:53,http://mclaughlin.org/gilbert.farrell,0.0000000000,175.13.235.72,"{""location"": ""EH"", ""is_mobile"": false}" 1777,3,166,2017-02-28 06:58:14,http://vandervort.co/tyrese,2.0000000000,68.216.126.88,"{""location"": ""KI"", ""is_mobile"": false}" 1778,3,166,2017-01-13 12:46:22,http://moendach.name/jimmy,2.0000000000,215.45.128.193,"{""location"": ""MP"", ""is_mobile"": false}" 1779,3,166,2017-03-25 23:06:58,http://cremin.name/gwen_mitchell,2.0000000000,30.229.213.70,"{""location"": ""VG"", ""is_mobile"": true}" 1780,3,166,2017-05-20 00:02:43,http://heel.biz/irwin,0.0000000000,114.101.173.37,"{""location"": ""TF"", ""is_mobile"": false}" 1781,3,166,2016-12-25 04:40:28,http://bernier.com/meggie,0.0000000000,185.13.133.198,"{""location"": ""FO"", ""is_mobile"": true}" 1782,3,167,2017-05-16 17:14:28,http://smitham.net/camren_wiegand,1.0000000000,20.138.13.140,"{""location"": ""PW"", ""is_mobile"": false}" 1783,3,167,2017-06-11 06:13:21,http://rowecormier.biz/sienna,1.0000000000,240.68.185.58,"{""location"": ""TO"", ""is_mobile"": true}" 1784,3,167,2017-01-04 14:34:03,http://wintheiser.com/berta_johnson,1.0000000000,93.174.219.193,"{""location"": ""PG"", ""is_mobile"": true}" 1785,3,167,2017-01-15 14:04:33,http://lefflerwalker.com/mozell_konopelski,0.0000000000,240.68.185.58,"{""location"": ""NE"", ""is_mobile"": false}" 1786,3,167,2017-06-03 09:06:09,http://jones.name/elenor_leuschke,0.0000000000,159.8.116.11,"{""location"": ""SI"", ""is_mobile"": true}" 1787,3,167,2016-12-28 23:50:59,http://buckridge.org/josephine.jones,0.0000000000,85.165.172.9,"{""location"": ""AU"", ""is_mobile"": false}" 1788,3,167,2017-01-23 01:10:47,http://fritsch.io/ila,1.0000000000,183.20.61.70,"{""location"": ""LU"", ""is_mobile"": true}" 1789,3,167,2017-04-10 19:48:19,http://kutch.biz/serena,1.0000000000,136.93.29.97,"{""location"": ""HM"", ""is_mobile"": true}" 1790,3,167,2017-01-05 08:46:54,http://oharamayert.net/dora_altenwerth,0.0000000000,47.107.105.164,"{""location"": ""CO"", ""is_mobile"": false}" 1791,3,167,2017-01-19 18:26:30,http://townehaley.biz/isobel.kuhn,1.0000000000,93.160.89.195,"{""location"": ""MY"", ""is_mobile"": false}" 1792,3,167,2017-05-03 12:01:53,http://fritsch.com/rahsaan,0.0000000000,157.141.240.185,"{""location"": ""NE"", ""is_mobile"": true}" 1793,3,167,2017-06-07 12:06:50,http://shieldporer.net/kylee.ohara,0.0000000000,254.235.69.78,"{""location"": ""NC"", ""is_mobile"": true}" 1794,3,167,2017-04-24 08:55:54,http://marvin.co/brittany_funk,1.0000000000,252.42.29.165,"{""location"": ""FJ"", ""is_mobile"": false}" 1795,3,167,2017-01-16 11:55:36,http://kuhlmanbauch.co/hope,0.0000000000,117.230.202.130,"{""location"": ""KE"", ""is_mobile"": true}" 1796,3,168,2017-01-10 07:47:32,http://mills.org/javonte_erdman,2.0000000000,215.151.203.178,"{""location"": ""MW"", ""is_mobile"": true}" 1797,3,168,2017-05-09 21:08:00,http://aufderhar.com/kiel_simonis,2.0000000000,207.61.100.178,"{""location"": ""RU"", ""is_mobile"": false}" 1798,3,168,2017-01-25 08:59:45,http://adamsdoyle.com/guie,1.0000000000,96.57.32.124,"{""location"": ""NC"", ""is_mobile"": true}" 1799,3,168,2017-02-24 14:01:20,http://oreillyprice.biz/brett,1.0000000000,84.171.241.138,"{""location"": ""VG"", ""is_mobile"": false}" 1800,3,168,2017-03-08 05:19:43,http://veum.net/sven,2.0000000000,52.237.89.142,"{""location"": ""GY"", ""is_mobile"": false}" 1801,3,168,2017-03-02 01:02:32,http://rodriguezemmerich.co/gabrielle.jenkins,2.0000000000,205.33.231.167,"{""location"": ""JM"", ""is_mobile"": false}" 1802,3,168,2017-01-02 07:28:29,http://dachthiel.name/bettie,0.0000000000,17.163.115.204,"{""location"": ""SL"", ""is_mobile"": true}" 1803,3,168,2017-05-29 00:26:14,http://hermistonbrakus.io/aniya.rolfson,0.0000000000,103.116.139.95,"{""location"": ""NP"", ""is_mobile"": false}" 1804,3,168,2017-01-12 22:44:46,http://parisian.biz/brad,1.0000000000,205.33.231.167,"{""location"": ""CL"", ""is_mobile"": false}" 1805,3,168,2017-04-03 11:55:09,http://stracke.io/oswaldo,0.0000000000,131.211.124.244,"{""location"": ""RW"", ""is_mobile"": true}" 1806,3,168,2017-03-21 17:55:28,http://daughertyziemann.net/maudie.bernier,0.0000000000,215.151.203.178,"{""location"": ""SA"", ""is_mobile"": true}" 1807,3,168,2017-02-03 08:49:39,http://price.co/ima,0.0000000000,205.33.231.167,"{""location"": ""AU"", ""is_mobile"": false}" 1808,3,168,2017-05-09 20:42:09,http://sporer.info/sidney.powlowski,1.0000000000,178.113.109.208,"{""location"": ""LU"", ""is_mobile"": false}" 1809,3,168,2017-05-13 13:20:28,http://dickens.org/bennett,0.0000000000,249.13.182.34,"{""location"": ""PY"", ""is_mobile"": true}" 1810,3,168,2017-06-02 11:11:43,http://heidenreichbatz.co/elton_olson,0.0000000000,52.237.89.142,"{""location"": ""AZ"", ""is_mobile"": true}" 1811,3,168,2017-03-08 09:13:24,http://kreiger.biz/breanne,0.0000000000,2.49.22.175,"{""location"": ""PS"", ""is_mobile"": true}" 1812,3,168,2017-02-04 05:51:57,http://emmerich.name/marlene,1.0000000000,169.135.85.242,"{""location"": ""UG"", ""is_mobile"": false}" 4468,7,416,2017-05-02 03:59:28,http://mohr.co/jefferey_grady,,79.139.203.30,"{""location"": ""ZA"", ""is_mobile"": false}" 4469,7,416,2017-06-13 10:15:51,http://simonis.info/roel_schmidt,,201.123.101.43,"{""location"": ""ID"", ""is_mobile"": true}" 4470,7,416,2017-03-17 05:47:51,http://hilll.name/annabel,,22.186.158.99,"{""location"": ""KN"", ""is_mobile"": false}" 4471,7,416,2016-12-16 14:11:03,http://hills.net/elna,,79.139.203.30,"{""location"": ""DK"", ""is_mobile"": true}" 4472,7,417,2017-03-23 10:50:35,http://robel.com/gerda,,2.32.148.250,"{""location"": ""IS"", ""is_mobile"": true}" 4473,7,417,2017-04-26 06:34:30,http://muellerborer.io/camylle,,210.241.232.73,"{""location"": ""ME"", ""is_mobile"": true}" 4474,7,417,2017-06-09 04:17:47,http://bradtke.io/alec_donnelly,,6.34.18.2,"{""location"": ""TC"", ""is_mobile"": true}" 4475,7,417,2017-01-15 20:06:12,http://hudson.org/theresia_kerluke,,127.101.115.75,"{""location"": ""AE"", ""is_mobile"": false}" 4476,7,417,2016-12-27 00:12:40,http://reingerorn.name/selmer.donnelly,,2.32.148.250,"{""location"": ""CV"", ""is_mobile"": false}" 4477,7,417,2017-05-15 06:47:06,http://jacobskrajcik.com/doug,,60.64.167.17,"{""location"": ""FM"", ""is_mobile"": false}" 4478,7,417,2017-03-17 22:33:13,http://ferry.biz/denis.hoppe,,77.242.33.236,"{""location"": ""LB"", ""is_mobile"": false}" 4479,7,417,2017-01-12 08:58:39,http://turcotte.co/eda,,177.25.10.229,"{""location"": ""CM"", ""is_mobile"": false}" 4480,7,417,2017-02-23 20:10:24,http://gorczany.net/milo,,248.72.218.169,"{""location"": ""LR"", ""is_mobile"": false}" 4481,7,417,2016-12-16 08:15:26,http://predovic.io/milo_zemlak,,74.95.112.22,"{""location"": ""TV"", ""is_mobile"": false}" 4482,7,417,2017-02-21 10:21:33,http://trompmayert.info/amos.macejkovic,,147.188.96.50,"{""location"": ""MO"", ""is_mobile"": false}" 4483,7,417,2017-01-27 00:21:43,http://hermiston.biz/davion.rodriguez,,81.122.171.186,"{""location"": ""LA"", ""is_mobile"": false}" 4484,7,417,2016-12-27 16:44:16,http://halvorson.info/nathaniel,,127.101.115.75,"{""location"": ""NF"", ""is_mobile"": false}" 4485,7,417,2017-03-05 09:44:08,http://jones.info/nora_hirthe,,178.39.123.108,"{""location"": ""AZ"", ""is_mobile"": false}" 4486,7,417,2017-05-21 12:54:37,http://runolfsdottir.biz/creola.mcdermott,,161.40.219.230,"{""location"": ""BQ"", ""is_mobile"": false}" 4487,7,417,2017-02-23 05:38:47,http://schaefer.org/mohamed.runolfon,,248.223.246.220,"{""location"": ""LS"", ""is_mobile"": false}" 4488,7,417,2017-02-16 01:03:35,http://little.info/shane,,81.122.171.186,"{""location"": ""AL"", ""is_mobile"": false}" 4489,7,417,2017-05-27 08:27:42,http://jerdejast.net/maximillia.hauck,,189.227.14.156,"{""location"": ""AX"", ""is_mobile"": true}" 4490,7,418,2017-04-24 06:31:37,http://kerluke.co/janelle.runte,,197.221.9.214,"{""location"": ""DJ"", ""is_mobile"": false}" 4491,7,418,2017-03-05 01:18:22,http://keler.net/aurelio.denesik,,243.148.4.25,"{""location"": ""CN"", ""is_mobile"": true}" 4492,7,419,2017-03-22 12:31:52,http://deckow.io/kasandra_beer,,143.110.22.213,"{""location"": ""KY"", ""is_mobile"": true}" 4493,7,419,2017-02-03 03:35:36,http://greenholtbernhard.io/isaiah,,45.47.91.79,"{""location"": ""BV"", ""is_mobile"": true}" 4494,7,419,2017-04-08 00:07:36,http://kihn.name/palma,,139.165.242.169,"{""location"": ""TF"", ""is_mobile"": false}" 4495,7,419,2017-02-25 04:06:56,http://senger.com/dejah_harber,,116.22.118.15,"{""location"": ""KZ"", ""is_mobile"": true}" 4496,7,419,2017-05-10 07:26:00,http://hahnberge.biz/leopoldo,,139.114.44.157,"{""location"": ""YT"", ""is_mobile"": true}" 4497,7,419,2017-01-01 14:10:41,http://morietteziemann.io/davonte,,139.118.236.124,"{""location"": ""LU"", ""is_mobile"": false}" 4498,7,419,2017-04-26 22:18:03,http://bauchconroy.biz/nellie.jakubowski,,218.33.181.135,"{""location"": ""AE"", ""is_mobile"": false}" 4499,7,419,2016-12-27 16:40:27,http://huels.name/guie,,194.40.74.91,"{""location"": ""KR"", ""is_mobile"": false}" 4500,7,419,2017-03-23 06:03:24,http://mraz.biz/ramon.feeney,,184.28.240.198,"{""location"": ""BQ"", ""is_mobile"": false}" 4501,7,419,2017-05-19 21:58:07,http://bergnaum.co/otis,,115.94.164.240,"{""location"": ""MA"", ""is_mobile"": true}" 4502,7,419,2017-01-07 17:54:13,http://little.org/jadon.bruen,,194.40.74.91,"{""location"": ""OM"", ""is_mobile"": true}" 4503,7,419,2016-12-22 15:35:52,http://jenkins.io/lyric.ruecker,,121.24.117.119,"{""location"": ""AF"", ""is_mobile"": true}" 4504,7,419,2017-01-06 06:52:02,http://satterfield.com/rocky.gislason,,164.16.55.55,"{""location"": ""BQ"", ""is_mobile"": false}" 4505,7,419,2017-02-28 05:45:22,http://howemacgyver.com/eduardo,,234.245.190.191,"{""location"": ""YT"", ""is_mobile"": true}" 4506,7,419,2017-05-17 05:05:02,http://legros.info/kelli,,174.244.205.177,"{""location"": ""PA"", ""is_mobile"": true}" 4507,7,419,2017-03-25 23:29:24,http://trantowbosco.biz/dannie,,194.115.219.73,"{""location"": ""GY"", ""is_mobile"": false}" 4508,7,419,2017-01-24 20:07:07,http://pollich.org/samanta.sauer,,45.47.91.79,"{""location"": ""TV"", ""is_mobile"": false}" 4509,7,419,2017-03-01 20:49:13,http://quigleywyman.info/olga,,45.47.91.79,"{""location"": ""KP"", ""is_mobile"": false}" 4510,7,420,2016-12-31 10:00:35,http://harriskovacek.co/darrick_hand,,181.156.34.107,"{""location"": ""LT"", ""is_mobile"": true}" 4511,7,420,2017-05-14 14:20:36,http://mckenzie.name/pietro,,145.106.154.177,"{""location"": ""US"", ""is_mobile"": false}" 4512,7,420,2017-04-11 09:24:00,http://pacocha.co/elfrieda_marvin,,151.56.100.231,"{""location"": ""IS"", ""is_mobile"": true}" 4513,7,420,2017-02-16 14:04:30,http://welchmaggio.info/louie,,26.55.148.60,"{""location"": ""LR"", ""is_mobile"": true}" 4514,7,420,2017-01-16 06:41:28,http://conn.com/billy_mclaughlin,,75.127.167.100,"{""location"": ""MT"", ""is_mobile"": true}" 4515,7,420,2017-01-31 19:08:47,http://jacobsonschroeder.com/roxane,,209.225.228.21,"{""location"": ""VU"", ""is_mobile"": true}" 4516,7,420,2017-02-04 14:57:50,http://gutmannkovacek.org/lee,,62.133.168.180,"{""location"": ""RW"", ""is_mobile"": false}" 4517,7,420,2017-04-09 08:38:34,http://hillsmosciski.org/christopher,,26.55.148.60,"{""location"": ""HU"", ""is_mobile"": true}" 4518,7,421,2017-02-09 12:25:07,http://langworth.info/quinten_schuppe,,88.74.171.226,"{""location"": ""BQ"", ""is_mobile"": true}" 4519,7,421,2017-02-12 13:33:25,http://gibson.net/bernadine,,253.214.66.135,"{""location"": ""CZ"", ""is_mobile"": true}" 4520,7,421,2016-12-30 16:06:40,http://mcclure.name/ariane.huels,,253.214.66.135,"{""location"": ""BJ"", ""is_mobile"": true}" 4521,7,421,2016-12-31 13:48:20,http://sporer.biz/sonya.gottlieb,,39.83.151.245,"{""location"": ""FK"", ""is_mobile"": true}" 4522,7,421,2016-12-28 07:25:45,http://schroeder.net/eduardo,,230.56.197.43,"{""location"": ""PS"", ""is_mobile"": true}" 1813,3,168,2017-01-27 04:36:07,http://hickle.net/jacynthe_murphy,2.0000000000,51.4.89.107,"{""location"": ""CL"", ""is_mobile"": true}" 1814,3,168,2017-03-24 04:19:19,http://kobins.com/aisha.swift,0.0000000000,53.123.124.236,"{""location"": ""MT"", ""is_mobile"": false}" 1815,3,169,2017-03-29 17:06:19,http://kris.io/gordon.beatty,1.0000000000,205.31.25.123,"{""location"": ""GE"", ""is_mobile"": false}" 1816,3,169,2017-06-02 13:37:30,http://considine.info/modesto,1.0000000000,141.248.114.82,"{""location"": ""PF"", ""is_mobile"": true}" 1817,3,169,2017-06-04 22:01:53,http://abernathy.name/gregg,1.0000000000,93.243.184.91,"{""location"": ""BS"", ""is_mobile"": true}" 1818,3,169,2017-05-29 23:01:43,http://cremin.io/elza,1.0000000000,135.232.193.209,"{""location"": ""RU"", ""is_mobile"": true}" 1819,3,169,2017-03-27 03:15:52,http://kuhn.name/gage,0.0000000000,141.152.184.15,"{""location"": ""BS"", ""is_mobile"": true}" 1820,3,169,2017-05-21 07:19:57,http://ortizmante.org/alvis.barrows,0.0000000000,234.167.187.70,"{""location"": ""JO"", ""is_mobile"": true}" 1821,3,169,2017-06-10 02:01:50,http://olson.com/destiny,1.0000000000,201.158.49.46,"{""location"": ""FR"", ""is_mobile"": false}" 1822,3,169,2017-05-30 16:56:43,http://tremblay.biz/mariah,0.0000000000,135.232.193.209,"{""location"": ""CK"", ""is_mobile"": true}" 1823,3,169,2016-12-23 01:32:00,http://parker.io/vivianne,1.0000000000,7.164.243.4,"{""location"": ""SL"", ""is_mobile"": true}" 1824,3,170,2017-03-04 04:16:11,http://pagachamill.name/sasha_braun,2.0000000000,32.191.224.142,"{""location"": ""SS"", ""is_mobile"": true}" 1825,3,170,2017-01-25 20:01:35,http://feil.com/jaylen,0.0000000000,84.9.231.6,"{""location"": ""MQ"", ""is_mobile"": false}" 1826,3,170,2017-05-19 00:34:19,http://mccullough.net/ken,0.0000000000,146.54.49.3,"{""location"": ""MH"", ""is_mobile"": true}" 1827,3,170,2017-04-21 23:53:09,http://herzog.io/sid,2.0000000000,189.143.166.202,"{""location"": ""ME"", ""is_mobile"": true}" 1828,3,170,2017-04-13 11:48:13,http://langoshgutmann.org/brent,0.0000000000,27.211.206.153,"{""location"": ""WF"", ""is_mobile"": false}" 1829,3,170,2017-04-11 19:57:36,http://langosh.co/ruthie.welch,1.0000000000,11.214.149.8,"{""location"": ""IE"", ""is_mobile"": true}" 1830,3,170,2017-01-19 07:35:14,http://quitzon.name/kaci,1.0000000000,32.191.224.142,"{""location"": ""BS"", ""is_mobile"": false}" 1831,3,171,2017-03-04 02:53:50,http://shields.co/nyah.sipes,2.0000000000,181.235.41.245,"{""location"": ""NI"", ""is_mobile"": false}" 1832,3,171,2017-05-25 22:17:09,http://beatty.org/leonard_schmidt,3.0000000000,120.86.147.38,"{""location"": ""BS"", ""is_mobile"": true}" 1833,3,171,2017-04-30 09:45:57,http://jacobson.biz/roberta,2.0000000000,60.177.67.67,"{""location"": ""SM"", ""is_mobile"": true}" 1834,3,171,2017-01-08 20:45:26,http://rodriguez.biz/rex_schuster,2.0000000000,123.37.244.138,"{""location"": ""UZ"", ""is_mobile"": false}" 1835,3,171,2017-01-26 17:09:32,http://greenwalter.org/jillian_tromp,2.0000000000,251.149.155.4,"{""location"": ""GB"", ""is_mobile"": true}" 1836,3,171,2017-01-21 05:03:02,http://borer.co/demarco,3.0000000000,95.49.216.169,"{""location"": ""LR"", ""is_mobile"": true}" 1837,3,171,2017-04-24 17:41:37,http://spinka.info/anjali.nolan,1.0000000000,60.177.67.67,"{""location"": ""SN"", ""is_mobile"": false}" 1838,3,171,2017-03-20 06:33:45,http://doyleleffler.com/laury.graham,2.0000000000,175.101.249.28,"{""location"": ""HN"", ""is_mobile"": false}" 1839,3,171,2016-12-17 22:27:13,http://ebert.biz/euna,0.0000000000,163.44.76.179,"{""location"": ""MQ"", ""is_mobile"": true}" 1840,3,171,2017-01-24 04:41:20,http://homenick.co/finn.emard,0.0000000000,123.37.244.138,"{""location"": ""FO"", ""is_mobile"": true}" 1841,3,171,2017-03-07 21:00:15,http://okeefe.org/doris,2.0000000000,181.97.49.244,"{""location"": ""MP"", ""is_mobile"": true}" 1842,3,171,2017-02-02 20:18:57,http://langosh.co/roosevelt,0.0000000000,215.163.214.208,"{""location"": ""KI"", ""is_mobile"": true}" 1843,3,171,2017-04-28 05:10:26,http://dietrichgutkowski.info/dennis_hauck,3.0000000000,123.37.244.138,"{""location"": ""MN"", ""is_mobile"": false}" 1844,3,171,2016-12-19 07:21:45,http://stiedemann.co/jude_smith,0.0000000000,95.49.216.169,"{""location"": ""BW"", ""is_mobile"": true}" 1845,3,171,2017-04-10 04:11:06,http://schumm.org/markus,2.0000000000,120.86.147.38,"{""location"": ""DE"", ""is_mobile"": true}" 1846,3,172,2017-04-09 06:03:01,http://donnelly.org/leone_simonis,,226.38.207.137,"{""location"": ""CN"", ""is_mobile"": true}" 1847,3,172,2017-01-18 20:17:59,http://eichmannmraz.io/precious.gaylord,,136.3.64.145,"{""location"": ""BZ"", ""is_mobile"": true}" 1848,3,172,2017-02-19 21:04:25,http://kihn.co/webster.okeefe,,143.23.113.166,"{""location"": ""VN"", ""is_mobile"": true}" 1849,3,172,2017-03-02 20:31:15,http://hammes.biz/kelley.spinka,,244.127.63.226,"{""location"": ""GE"", ""is_mobile"": false}" 1850,3,172,2017-04-19 21:53:59,http://upton.info/vella.schultz,,202.18.244.244,"{""location"": ""BB"", ""is_mobile"": false}" 1851,3,172,2016-12-13 10:41:49,http://mraznader.com/reggie_schiller,,150.77.23.190,"{""location"": ""SR"", ""is_mobile"": true}" 1852,3,172,2017-03-18 03:44:57,http://schamberger.co/sammy,,160.145.67.18,"{""location"": ""CC"", ""is_mobile"": true}" 1853,3,172,2017-01-03 14:30:43,http://balistrerimraz.name/rubye_corkery,,199.13.242.55,"{""location"": ""MS"", ""is_mobile"": false}" 1854,3,172,2017-03-20 11:48:34,http://luettgenjakubowski.biz/kendra_maggio,,158.118.245.201,"{""location"": ""HN"", ""is_mobile"": true}" 1855,3,172,2017-05-13 12:13:02,http://ullrich.info/glennie,,118.43.83.138,"{""location"": ""CR"", ""is_mobile"": true}" 1856,3,172,2017-04-23 06:49:31,http://leffler.info/shayne.gerlach,,231.69.251.32,"{""location"": ""SY"", ""is_mobile"": false}" 1857,3,172,2017-04-09 11:07:45,http://volkmanwisozk.info/kurtis_farrell,,231.69.251.32,"{""location"": ""SY"", ""is_mobile"": false}" 1858,3,172,2017-02-04 10:41:09,http://konopelski.net/paul_roob,,161.103.20.231,"{""location"": ""US"", ""is_mobile"": false}" 1859,3,172,2016-12-21 18:32:37,http://greenholtmorar.net/mittie,,113.114.83.247,"{""location"": ""SG"", ""is_mobile"": false}" 1860,3,172,2016-12-24 02:04:23,http://langworth.com/preston,,41.147.38.230,"{""location"": ""FR"", ""is_mobile"": false}" 1861,3,172,2017-02-04 11:57:31,http://raynor.net/newell,,231.69.251.32,"{""location"": ""SL"", ""is_mobile"": true}" 1862,3,173,2017-05-24 02:29:49,http://stokesbartoletti.com/maegan.turcotte,,114.80.90.81,"{""location"": ""PF"", ""is_mobile"": true}" 1863,3,173,2017-06-08 03:43:51,http://kuphalherman.io/unique,,48.227.90.218,"{""location"": ""BI"", ""is_mobile"": true}" 1864,3,173,2017-02-09 23:25:34,http://lind.name/braulio_kunze,,203.20.186.214,"{""location"": ""BB"", ""is_mobile"": true}" 1865,3,173,2016-12-26 10:05:11,http://ko.info/dario_lebsack,,126.136.40.134,"{""location"": ""SK"", ""is_mobile"": false}" 4523,7,421,2017-02-15 09:22:47,http://goyette.com/thurman_larson,,240.71.193.226,"{""location"": ""GI"", ""is_mobile"": false}" 4524,7,421,2017-03-09 23:53:13,http://gorczany.info/luciano_ruecker,,183.153.195.153,"{""location"": ""SI"", ""is_mobile"": true}" 4525,7,421,2017-04-14 23:31:22,http://corkery.org/alejandra_kub,,88.74.171.226,"{""location"": ""CI"", ""is_mobile"": true}" 4526,7,421,2016-12-19 18:50:19,http://fay.biz/kaci,,240.71.193.226,"{""location"": ""DK"", ""is_mobile"": false}" 4527,7,421,2017-04-06 20:05:00,http://ferry.io/levi,,228.122.111.103,"{""location"": ""SN"", ""is_mobile"": false}" 4528,7,421,2017-05-07 23:58:14,http://braun.biz/annabel_hudson,,233.113.146.61,"{""location"": ""NG"", ""is_mobile"": false}" 4529,7,422,2017-03-22 19:35:06,http://mcdermottbogisich.info/kay,,29.108.163.134,"{""location"": ""HN"", ""is_mobile"": false}" 4530,7,422,2017-01-18 22:52:13,http://toy.info/magdalena,,175.131.173.136,"{""location"": ""BJ"", ""is_mobile"": true}" 4531,7,422,2017-05-29 09:39:15,http://cartwrightstoltenberg.com/norma.hickle,,124.228.244.139,"{""location"": ""BZ"", ""is_mobile"": true}" 4532,7,422,2017-04-07 19:52:24,http://monahan.name/jeramy,,224.220.122.194,"{""location"": ""BY"", ""is_mobile"": false}" 4533,7,422,2017-01-09 05:30:43,http://reingeradams.com/emie_cummings,,61.110.200.130,"{""location"": ""WS"", ""is_mobile"": false}" 4534,7,422,2017-01-26 14:11:26,http://gulgowskibernier.io/sabrina.olson,,249.92.31.210,"{""location"": ""NF"", ""is_mobile"": true}" 4535,7,422,2017-01-03 18:55:51,http://corkery.net/flavie.gerlach,,48.12.25.71,"{""location"": ""CG"", ""is_mobile"": true}" 4536,7,422,2016-12-31 01:29:00,http://prohaska.org/arden,,156.31.154.227,"{""location"": ""CI"", ""is_mobile"": true}" 4537,7,423,2017-03-21 00:04:02,http://jenkins.net/ethel_cummings,,253.18.184.100,"{""location"": ""FJ"", ""is_mobile"": false}" 4538,7,423,2017-02-19 12:00:59,http://gerholdtreutel.co/emma,,34.229.144.194,"{""location"": ""ZW"", ""is_mobile"": false}" 4539,7,423,2017-04-04 01:53:21,http://oharalueilwitz.com/friedrich,,214.119.36.130,"{""location"": ""CO"", ""is_mobile"": false}" 4540,7,423,2017-05-07 08:17:43,http://abernathy.info/oma.hilll,,23.65.181.76,"{""location"": ""PK"", ""is_mobile"": true}" 4541,7,423,2016-12-23 07:23:43,http://stantonprosacco.org/tyree,,26.155.133.236,"{""location"": ""SR"", ""is_mobile"": true}" 4542,7,423,2017-02-14 13:24:08,http://emard.name/nakia,,26.155.133.236,"{""location"": ""MW"", ""is_mobile"": true}" 4543,7,423,2016-12-30 20:25:14,http://zemlak.net/shane_heaney,,69.70.244.149,"{""location"": ""SY"", ""is_mobile"": false}" 4544,7,423,2017-04-26 07:35:12,http://cruickshank.com/adam,,94.164.60.44,"{""location"": ""BZ"", ""is_mobile"": false}" 4545,7,423,2017-06-08 01:52:05,http://kihnruel.biz/filomena_bartell,,149.89.214.146,"{""location"": ""NE"", ""is_mobile"": false}" 4546,7,424,2017-04-03 07:38:15,http://cummings.info/chadrick,,160.137.192.108,"{""location"": ""MN"", ""is_mobile"": true}" 4547,7,424,2017-01-03 22:12:00,http://hayes.org/dave,,173.63.211.162,"{""location"": ""ES"", ""is_mobile"": true}" 4548,7,424,2017-04-21 10:55:45,http://gerhold.com/jimmie_bailey,,121.217.38.246,"{""location"": ""BN"", ""is_mobile"": false}" 4549,7,424,2017-05-26 02:01:51,http://johnson.biz/joshua_hoppe,,210.37.191.178,"{""location"": ""AZ"", ""is_mobile"": true}" 4550,7,424,2017-03-16 21:58:26,http://bergstrom.org/jeff,,18.180.96.78,"{""location"": ""UG"", ""is_mobile"": false}" 4551,7,424,2016-12-14 08:36:23,http://moenoberbrunner.com/carolyne.walker,,231.184.210.226,"{""location"": ""LR"", ""is_mobile"": false}" 4552,7,424,2017-03-15 17:21:56,http://vandervort.info/dion.kilback,,173.63.211.162,"{""location"": ""BA"", ""is_mobile"": false}" 4553,7,424,2017-01-02 09:47:47,http://frami.biz/katherine,,64.104.70.131,"{""location"": ""CC"", ""is_mobile"": false}" 4554,7,424,2017-05-01 05:37:15,http://reinger.io/ken,,74.210.94.9,"{""location"": ""PE"", ""is_mobile"": true}" 4555,7,424,2016-12-26 09:04:16,http://walterbins.biz/domenic.breitenberg,,210.37.191.178,"{""location"": ""BB"", ""is_mobile"": false}" 4556,7,424,2017-02-03 00:08:25,http://ritchie.org/teagan.boyer,,48.244.116.212,"{""location"": ""KY"", ""is_mobile"": false}" 4557,7,424,2017-02-25 05:15:40,http://stanton.io/evie_dubuque,,187.205.8.90,"{""location"": ""KN"", ""is_mobile"": true}" 4558,7,424,2017-03-15 15:04:33,http://beer.com/kailey,,121.217.38.246,"{""location"": ""TZ"", ""is_mobile"": false}" 4559,7,424,2017-03-18 22:18:53,http://heelhilll.biz/jackeline,,131.234.29.227,"{""location"": ""NE"", ""is_mobile"": false}" 4560,7,424,2017-01-02 19:31:43,http://skiles.org/akeem.waelchi,,130.127.38.18,"{""location"": ""LT"", ""is_mobile"": false}" 4561,7,425,2017-04-18 15:33:03,http://haag.org/rhianna,,97.163.56.35,"{""location"": ""CY"", ""is_mobile"": false}" 4562,7,425,2017-03-03 10:33:55,http://auerkonopelski.org/hipolito_brown,,161.190.65.146,"{""location"": ""BE"", ""is_mobile"": false}" 4563,7,425,2017-03-24 22:55:25,http://vonrueden.com/dale_morar,,120.85.29.159,"{""location"": ""DK"", ""is_mobile"": true}" 4564,7,425,2017-01-24 14:19:09,http://swaniawski.org/hope_wintheiser,,24.12.74.226,"{""location"": ""MS"", ""is_mobile"": false}" 4565,7,425,2017-06-04 14:03:22,http://abbottkaulke.co/fanny.mills,,185.203.49.132,"{""location"": ""IO"", ""is_mobile"": true}" 4566,7,425,2016-12-18 03:51:52,http://schmitt.com/mike,,226.227.212.57,"{""location"": ""MT"", ""is_mobile"": false}" 4567,7,425,2017-02-25 04:15:14,http://dibbert.name/raymundo,,97.163.56.35,"{""location"": ""LT"", ""is_mobile"": true}" 4568,7,425,2017-01-31 20:43:08,http://streichschultz.biz/modesto,,195.26.6.45,"{""location"": ""MK"", ""is_mobile"": true}" 4569,7,425,2017-05-09 15:20:31,http://moriette.net/zelma.mitchell,,41.70.47.37,"{""location"": ""SI"", ""is_mobile"": true}" 4570,7,425,2017-03-12 11:18:53,http://cartwright.biz/deangelo,,89.106.9.149,"{""location"": ""TV"", ""is_mobile"": false}" 4571,7,425,2017-03-21 09:25:31,http://hayetokes.biz/zaria_yundt,,222.26.244.87,"{""location"": ""VG"", ""is_mobile"": false}" 4572,7,425,2017-01-16 17:19:39,http://metz.io/hal,,166.19.229.38,"{""location"": ""MV"", ""is_mobile"": false}" 4573,7,425,2017-04-21 17:38:56,http://roberts.name/elenora_sporer,,64.118.207.78,"{""location"": ""GL"", ""is_mobile"": false}" 4574,7,425,2017-02-07 03:06:35,http://stracke.info/jeramie,,135.150.24.147,"{""location"": ""GF"", ""is_mobile"": false}" 4575,7,425,2017-01-29 10:43:46,http://hermiston.io/curt.erdman,,38.109.185.45,"{""location"": ""ME"", ""is_mobile"": false}" 4576,7,425,2017-04-30 12:45:44,http://ortiz.info/keanu,,60.204.163.71,"{""location"": ""AD"", ""is_mobile"": true}" 4577,7,426,2017-05-29 08:37:48,http://koch.com/erich,,116.243.191.73,"{""location"": ""IM"", ""is_mobile"": false}" 4578,7,427,2016-12-27 02:30:35,http://funkreichel.io/devonte,,242.176.119.226,"{""location"": ""SK"", ""is_mobile"": false}" 1866,3,173,2017-06-07 14:45:59,http://stark.biz/abel_pacocha,,191.83.226.228,"{""location"": ""GB"", ""is_mobile"": false}" 1867,3,174,2017-03-05 17:09:50,http://heathcote.com/rasheed,,197.141.244.39,"{""location"": ""DE"", ""is_mobile"": true}" 1868,3,174,2017-05-26 14:25:07,http://collins.info/odie_kub,,22.198.131.82,"{""location"": ""PF"", ""is_mobile"": false}" 1869,3,174,2017-04-26 19:02:35,http://dibbertmraz.name/guido,,153.190.139.140,"{""location"": ""JP"", ""is_mobile"": true}" 1870,3,174,2017-05-14 14:07:27,http://rodriguez.org/earl_breitenberg,,33.44.42.254,"{""location"": ""AS"", ""is_mobile"": false}" 1871,3,174,2017-04-27 23:54:50,http://haagjenkins.net/myriam.price,,22.198.131.82,"{""location"": ""MH"", ""is_mobile"": true}" 1872,3,174,2017-03-11 03:46:10,http://danielturcotte.info/anabel.dibbert,,80.52.237.78,"{""location"": ""CF"", ""is_mobile"": true}" 1873,3,174,2017-03-20 02:04:12,http://ohararau.info/levi,,129.123.94.198,"{""location"": ""LC"", ""is_mobile"": true}" 1874,3,174,2017-02-18 07:05:53,http://robertserdman.io/myah,,238.102.115.248,"{""location"": ""ML"", ""is_mobile"": false}" 1875,3,175,2017-04-02 07:19:06,http://hettinger.io/roger.legros,,217.125.64.197,"{""location"": ""AM"", ""is_mobile"": false}" 1876,3,175,2017-03-08 21:54:52,http://altenwerthblock.org/mozelle,,62.29.23.214,"{""location"": ""NE"", ""is_mobile"": false}" 1877,3,175,2017-03-05 02:03:34,http://bins.io/conrad,,138.223.173.84,"{""location"": ""IQ"", ""is_mobile"": true}" 1878,3,176,2017-05-12 01:16:31,http://grimes.org/angus,,29.116.32.144,"{""location"": ""ML"", ""is_mobile"": false}" 1879,3,176,2017-05-03 09:26:03,http://donnelly.info/tremayne,,145.47.126.102,"{""location"": ""DJ"", ""is_mobile"": true}" 1880,3,176,2017-03-25 18:58:20,http://oconnell.org/joesph,,153.101.35.206,"{""location"": ""CZ"", ""is_mobile"": false}" 1881,3,176,2017-04-08 07:03:32,http://mante.info/christian_fahey,,41.148.12.75,"{""location"": ""EH"", ""is_mobile"": false}" 1882,3,176,2016-12-16 09:26:03,http://schiller.org/jamal,,187.132.16.224,"{""location"": ""AT"", ""is_mobile"": false}" 1883,3,176,2016-12-22 16:07:28,http://schumm.net/cali,,52.81.229.164,"{""location"": ""YT"", ""is_mobile"": false}" 1884,3,176,2016-12-23 17:42:59,http://mills.com/amparo_romaguera,,187.132.16.224,"{""location"": ""TO"", ""is_mobile"": true}" 1885,3,176,2017-01-28 14:53:14,http://breitenberg.com/vernie,,153.101.35.206,"{""location"": ""BD"", ""is_mobile"": false}" 1886,3,176,2017-04-17 08:22:08,http://rath.io/christop,,250.223.15.215,"{""location"": ""BF"", ""is_mobile"": true}" 1887,3,176,2016-12-14 11:41:00,http://ernser.io/augustus,,106.195.199.145,"{""location"": ""PM"", ""is_mobile"": true}" 1888,3,176,2017-01-27 12:38:16,http://rogahnankunding.co/lula,,90.243.14.23,"{""location"": ""GH"", ""is_mobile"": true}" 1889,3,176,2017-01-29 10:55:47,http://reynoldsmacgyver.biz/karlee_rodriguez,,70.188.75.108,"{""location"": ""CR"", ""is_mobile"": false}" 1890,3,176,2017-01-25 18:36:38,http://schummtorphy.net/retta,,195.181.193.147,"{""location"": ""PR"", ""is_mobile"": false}" 1891,3,176,2017-05-12 04:26:58,http://konopelski.name/tillman_prohaska,,152.217.34.46,"{""location"": ""FK"", ""is_mobile"": false}" 1892,3,176,2017-05-31 17:31:23,http://tromp.co/madalyn,,209.168.41.244,"{""location"": ""BL"", ""is_mobile"": true}" 1893,3,176,2017-03-15 03:26:42,http://wilkinsonwolf.name/bertha_wyman,,187.132.16.224,"{""location"": ""DE"", ""is_mobile"": true}" 1894,3,176,2017-06-03 22:17:43,http://gibson.io/scot.toy,,243.216.143.95,"{""location"": ""RO"", ""is_mobile"": false}" 1895,3,176,2017-04-03 00:12:02,http://pfannerstill.name/burnice_pfeffer,,249.136.15.48,"{""location"": ""LT"", ""is_mobile"": false}" 1896,3,177,2017-01-23 22:32:18,http://williamsonmosciski.co/mertie,,211.209.232.100,"{""location"": ""AF"", ""is_mobile"": false}" 1897,3,177,2016-12-15 04:06:22,http://thompson.io/quentin,,190.176.70.107,"{""location"": ""BV"", ""is_mobile"": false}" 1898,3,177,2017-02-13 08:43:41,http://reillyauer.info/amir,,137.55.229.238,"{""location"": ""ZM"", ""is_mobile"": true}" 1899,3,178,2017-03-16 00:45:00,http://reichertbecker.biz/franco,,216.208.125.128,"{""location"": ""GG"", ""is_mobile"": false}" 1900,3,178,2017-01-19 03:20:25,http://douglas.net/agustina,,227.37.114.91,"{""location"": ""MF"", ""is_mobile"": false}" 1901,3,178,2017-06-01 08:12:21,http://sipes.io/pearline,,83.117.73.154,"{""location"": ""MG"", ""is_mobile"": true}" 1902,3,178,2017-01-12 18:19:31,http://smithamwaters.org/jaquan,,208.35.237.81,"{""location"": ""LB"", ""is_mobile"": true}" 1903,3,178,2017-02-28 08:01:07,http://hammes.com/grover,,231.30.202.237,"{""location"": ""CM"", ""is_mobile"": false}" 1904,3,178,2017-01-03 15:35:38,http://mosciski.net/sigmund,,75.195.26.87,"{""location"": ""SK"", ""is_mobile"": true}" 1905,3,178,2016-12-17 02:16:21,http://kaulke.name/hank_schimmel,,216.237.27.134,"{""location"": ""TL"", ""is_mobile"": true}" 1906,3,178,2017-04-01 18:47:58,http://murray.name/hal,,244.62.96.220,"{""location"": ""CL"", ""is_mobile"": false}" 1907,3,178,2017-01-18 20:35:14,http://kuhn.net/johnpaul.wintheiser,,68.104.67.230,"{""location"": ""IS"", ""is_mobile"": false}" 1908,3,178,2017-05-31 18:15:25,http://harveyrippin.co/ahmed,,157.81.223.139,"{""location"": ""PS"", ""is_mobile"": false}" 1909,3,178,2017-04-11 15:51:03,http://wolfluettgen.net/riley.torphy,,24.248.47.39,"{""location"": ""SO"", ""is_mobile"": false}" 1910,3,178,2017-05-04 23:32:32,http://wardhirthe.io/alisha.kuphal,,253.89.115.251,"{""location"": ""LU"", ""is_mobile"": false}" 1911,3,178,2017-06-10 04:31:52,http://lang.name/mavis,,75.195.26.87,"{""location"": ""NA"", ""is_mobile"": false}" 1912,3,178,2017-03-24 23:38:00,http://schultzluettgen.co/shakira,,203.12.71.156,"{""location"": ""ES"", ""is_mobile"": false}" 1913,3,179,2017-04-10 02:38:25,http://dietrichreinger.info/myron,,70.252.178.31,"{""location"": ""OM"", ""is_mobile"": false}" 1914,3,179,2017-04-11 00:51:51,http://oconnellbergnaum.net/karianne.terry,,102.69.153.25,"{""location"": ""LS"", ""is_mobile"": true}" 1915,3,179,2017-02-22 05:39:55,http://willbogisich.name/jade,,48.80.3.149,"{""location"": ""KW"", ""is_mobile"": false}" 1916,3,179,2017-06-12 11:18:34,http://lockmanlangworth.info/alvah,,62.122.106.196,"{""location"": ""MF"", ""is_mobile"": false}" 1917,3,179,2017-01-30 18:26:57,http://schamberger.com/mariane_rodriguez,,71.29.82.113,"{""location"": ""LB"", ""is_mobile"": false}" 1918,3,180,2017-05-26 02:54:32,http://altenwerth.biz/andy,,96.92.33.43,"{""location"": ""MW"", ""is_mobile"": true}" 1919,3,180,2016-12-19 05:19:15,http://leannon.com/adeline,,50.243.24.191,"{""location"": ""NA"", ""is_mobile"": true}" 1920,3,180,2017-06-08 12:09:14,http://lebsack.info/wilma.nitzsche,,76.159.60.49,"{""location"": ""GY"", ""is_mobile"": false}" 1921,3,180,2017-05-04 17:50:54,http://senger.io/zena.goyette,,158.250.29.41,"{""location"": ""WF"", ""is_mobile"": false}" 4579,7,428,2017-04-22 19:15:00,http://connellykulas.org/nicolette_hodkiewicz,,206.250.107.94,"{""location"": ""IT"", ""is_mobile"": true}" 4580,7,428,2017-04-10 00:09:33,http://nienow.com/angel_kunde,,90.52.186.168,"{""location"": ""NR"", ""is_mobile"": false}" 4581,7,428,2017-02-19 15:12:36,http://brownfahey.com/leanne_cain,,233.32.190.202,"{""location"": ""GG"", ""is_mobile"": true}" 4582,7,429,2017-02-26 13:49:40,http://sporer.co/muriel,,57.55.57.242,"{""location"": ""AI"", ""is_mobile"": true}" 4583,7,429,2017-02-22 00:05:47,http://romaguerabailey.co/shemar,,170.212.190.166,"{""location"": ""BJ"", ""is_mobile"": true}" 4584,7,429,2017-01-01 08:20:46,http://kautzerhansen.com/johathan_ernser,,89.159.147.227,"{""location"": ""VC"", ""is_mobile"": false}" 4585,7,429,2017-06-05 08:20:54,http://stark.info/amber,,120.219.181.106,"{""location"": ""FI"", ""is_mobile"": false}" 4586,7,430,2017-01-27 06:52:43,http://wuckert.org/craig_fahey,,90.26.234.72,"{""location"": ""TL"", ""is_mobile"": true}" 4587,7,430,2017-05-19 21:12:16,http://senger.io/janae,,201.142.201.96,"{""location"": ""BL"", ""is_mobile"": false}" 4588,7,430,2017-05-20 02:38:29,http://danielko.info/lenna,,135.47.32.71,"{""location"": ""CC"", ""is_mobile"": false}" 4589,7,430,2017-01-21 06:39:37,http://wilkinsoncarter.co/annette_pagac,,156.121.29.145,"{""location"": ""TT"", ""is_mobile"": false}" 4590,7,430,2017-03-02 20:23:47,http://okon.io/raegan_hermann,,93.136.75.60,"{""location"": ""SY"", ""is_mobile"": false}" 4591,7,430,2017-04-26 15:17:05,http://heaney.co/thea.kohler,,93.136.75.60,"{""location"": ""LV"", ""is_mobile"": false}" 4592,7,430,2016-12-28 15:32:44,http://smithrogahn.org/jasmin.swift,,16.130.176.189,"{""location"": ""NZ"", ""is_mobile"": true}" 4593,7,430,2017-04-20 21:29:37,http://predovicerdman.com/ariel,,246.127.18.205,"{""location"": ""MQ"", ""is_mobile"": true}" 4594,7,430,2017-05-25 23:18:36,http://langworth.net/brent,,230.168.142.153,"{""location"": ""KZ"", ""is_mobile"": true}" 4595,7,430,2016-12-14 16:01:34,http://schusterweimann.com/hellen,,201.142.201.96,"{""location"": ""IR"", ""is_mobile"": false}" 4596,7,430,2017-02-10 23:25:04,http://adamsfriesen.io/kayley.schoen,,80.22.76.105,"{""location"": ""LV"", ""is_mobile"": true}" 4597,7,431,2017-01-30 10:17:56,http://romaguerahalvorson.info/dorthy,,24.184.195.38,"{""location"": ""TK"", ""is_mobile"": false}" 4598,7,431,2017-04-20 21:58:03,http://jenkinskirlin.biz/eloy,,191.75.33.200,"{""location"": ""LY"", ""is_mobile"": true}" 4599,7,431,2017-05-25 12:41:19,http://feeney.name/june,,191.75.33.200,"{""location"": ""LI"", ""is_mobile"": false}" 4600,7,431,2017-03-26 07:49:46,http://grantcorwin.info/flo,,90.252.233.46,"{""location"": ""AW"", ""is_mobile"": true}" 4601,7,431,2017-04-26 13:32:12,http://brekke.io/glenda.ziemann,,181.172.13.246,"{""location"": ""SJ"", ""is_mobile"": true}" 4602,7,431,2017-02-02 06:11:35,http://sporer.biz/vladimir.bashirian,,239.82.225.47,"{""location"": ""BS"", ""is_mobile"": true}" 4603,7,431,2017-04-26 23:37:17,http://maggio.name/kay.gusikowski,,28.88.38.188,"{""location"": ""IS"", ""is_mobile"": true}" 4604,7,431,2017-01-09 19:01:04,http://hoppe.biz/tianna_kiehn,,216.110.113.224,"{""location"": ""MD"", ""is_mobile"": false}" 4605,7,431,2016-12-25 02:00:31,http://rodriguez.org/jovanny_kris,,195.47.44.38,"{""location"": ""MZ"", ""is_mobile"": true}" 4606,7,431,2017-01-07 17:25:32,http://bartell.biz/fanny,,18.136.59.74,"{""location"": ""CG"", ""is_mobile"": false}" 4607,7,431,2017-04-29 10:12:23,http://beier.name/tyreek,,138.12.234.163,"{""location"": ""NU"", ""is_mobile"": false}" 4608,7,431,2017-02-15 06:46:50,http://abshire.io/erick,,104.208.102.131,"{""location"": ""AL"", ""is_mobile"": true}" 4609,7,431,2017-01-17 22:13:09,http://cronin.info/caitlyn,,21.170.196.238,"{""location"": ""JO"", ""is_mobile"": true}" 4610,7,431,2017-03-17 03:56:14,http://muellerquigley.io/estel,,111.152.218.206,"{""location"": ""RU"", ""is_mobile"": false}" 4611,7,431,2017-04-11 08:23:51,http://walsh.org/grady.morar,,187.202.62.225,"{""location"": ""BF"", ""is_mobile"": false}" 4612,7,431,2017-01-19 13:51:08,http://zboncakwisozk.co/janick,,191.75.33.200,"{""location"": ""NZ"", ""is_mobile"": true}" 4613,7,431,2017-05-28 22:49:17,http://greenholt.com/leie.parker,,179.161.141.200,"{""location"": ""GA"", ""is_mobile"": false}" 4614,7,431,2017-01-13 09:45:24,http://bogisichbechtelar.info/irma.moore,,183.61.79.211,"{""location"": ""AG"", ""is_mobile"": true}" 4615,7,431,2017-04-11 18:33:34,http://reichert.com/marquis,,148.105.206.150,"{""location"": ""VC"", ""is_mobile"": true}" 4616,7,432,2017-02-12 08:58:54,http://block.io/zakary,,87.56.76.116,"{""location"": ""NR"", ""is_mobile"": false}" 4617,7,433,2016-12-30 05:01:55,http://mohr.com/angeline,,49.27.9.247,"{""location"": ""AW"", ""is_mobile"": false}" 4618,7,433,2017-01-11 09:51:24,http://mueller.name/hank.eichmann,,185.153.2.51,"{""location"": ""KZ"", ""is_mobile"": true}" 4619,7,433,2017-05-16 10:45:43,http://weber.info/kristin,,197.132.77.129,"{""location"": ""AR"", ""is_mobile"": true}" 4620,7,433,2017-01-07 11:02:57,http://corkery.biz/antwon.hane,,10.188.10.36,"{""location"": ""AX"", ""is_mobile"": false}" 4621,7,433,2017-03-13 21:07:45,http://cole.com/lamont,,154.237.210.111,"{""location"": ""PW"", ""is_mobile"": true}" 4622,7,433,2017-01-31 10:22:50,http://rauharvey.co/rebecca,,197.132.77.129,"{""location"": ""BT"", ""is_mobile"": true}" 4623,7,433,2017-01-19 10:54:55,http://greenholt.com/myles,,197.132.77.129,"{""location"": ""BB"", ""is_mobile"": false}" 4624,7,433,2017-02-20 12:00:23,http://prosacco.io/eve.hettinger,,10.188.10.36,"{""location"": ""BB"", ""is_mobile"": false}" 4625,7,433,2017-02-24 21:18:51,http://senger.info/micheal_schiller,,182.230.141.79,"{""location"": ""GN"", ""is_mobile"": false}" 4626,7,433,2016-12-22 18:41:21,http://aufderharmonahan.co/naomi_jerde,,244.168.87.81,"{""location"": ""IR"", ""is_mobile"": true}" 4627,7,433,2017-01-09 22:44:20,http://goyette.net/filiberto.schultz,,249.104.225.210,"{""location"": ""EG"", ""is_mobile"": false}" 4628,7,433,2017-01-30 23:31:11,http://bauch.net/vince.yundt,,67.37.16.120,"{""location"": ""CU"", ""is_mobile"": false}" 4629,7,433,2017-06-02 14:36:55,http://mullerturcotte.org/meta,,205.36.198.39,"{""location"": ""ZW"", ""is_mobile"": true}" 4630,7,434,2017-03-24 19:52:44,http://kulas.org/josephine,,120.171.34.47,"{""location"": ""BM"", ""is_mobile"": false}" 4631,7,434,2017-02-24 17:23:46,http://hansenruecker.name/savannah,,175.35.155.227,"{""location"": ""TF"", ""is_mobile"": false}" 4632,7,434,2017-06-07 11:13:23,http://miller.co/alyon,,195.51.198.33,"{""location"": ""DJ"", ""is_mobile"": false}" 4633,7,434,2017-05-27 21:10:37,http://oberbrunner.co/floy_macgyver,,39.236.5.194,"{""location"": ""SC"", ""is_mobile"": true}" 4634,7,434,2017-02-25 19:23:28,http://kohler.io/wilhelm,,108.123.110.175,"{""location"": ""ET"", ""is_mobile"": true}" 1922,3,180,2017-05-13 17:50:19,http://hackett.co/hannah.crooks,,242.126.154.88,"{""location"": ""UA"", ""is_mobile"": false}" 1923,3,180,2017-05-09 07:07:51,http://dicki.io/olaf.hahn,,193.136.184.11,"{""location"": ""PL"", ""is_mobile"": true}" 1924,3,181,2017-02-26 01:39:23,http://doyle.biz/ramiro,,97.90.32.78,"{""location"": ""VI"", ""is_mobile"": true}" 1925,3,181,2017-06-09 06:38:10,http://metz.info/maximo,,244.116.9.235,"{""location"": ""BD"", ""is_mobile"": false}" 1926,3,181,2017-03-27 21:44:28,http://greenfelder.biz/roger,,111.142.223.84,"{""location"": ""HK"", ""is_mobile"": true}" 1927,3,181,2017-01-15 04:11:50,http://lind.net/rosetta,,120.153.176.227,"{""location"": ""TW"", ""is_mobile"": true}" 1928,3,181,2016-12-15 20:12:17,http://stehrhamill.io/jeromy,,230.245.7.38,"{""location"": ""GA"", ""is_mobile"": true}" 1929,3,181,2017-04-16 23:58:12,http://hayes.info/chasity.keebler,,67.127.128.240,"{""location"": ""AR"", ""is_mobile"": true}" 1930,3,181,2017-04-30 13:34:35,http://tremblay.org/corine_beatty,,120.153.176.227,"{""location"": ""CF"", ""is_mobile"": true}" 1931,3,181,2017-06-13 22:57:07,http://tremblay.com/roy_kovacek,,118.48.81.179,"{""location"": ""AF"", ""is_mobile"": false}" 1932,3,181,2017-03-04 01:08:09,http://berge.io/summer_kuvalis,,22.168.101.31,"{""location"": ""PW"", ""is_mobile"": true}" 1933,3,181,2017-01-14 18:04:43,http://shanahan.net/iliana.gusikowski,,22.168.101.31,"{""location"": ""PE"", ""is_mobile"": true}" 1934,3,182,2017-04-05 20:12:36,http://jerde.biz/julianne,,232.9.210.116,"{""location"": ""BD"", ""is_mobile"": true}" 1935,3,182,2017-04-04 03:21:37,http://prohaska.net/skyla_green,,202.202.9.117,"{""location"": ""GE"", ""is_mobile"": true}" 1936,3,182,2017-02-26 20:50:50,http://wunschnienow.info/luigi.weimann,,41.167.185.5,"{""location"": ""LS"", ""is_mobile"": false}" 1937,3,182,2017-05-27 11:14:01,http://swaniawski.biz/chaz_harber,,202.202.9.117,"{""location"": ""PY"", ""is_mobile"": true}" 1938,3,182,2017-05-26 20:23:05,http://schimmelmarks.co/giovanna,,32.74.127.21,"{""location"": ""MU"", ""is_mobile"": false}" 1939,3,182,2017-06-07 19:10:55,http://terry.com/hildegard.mcglynn,,64.22.192.142,"{""location"": ""SY"", ""is_mobile"": true}" 1940,3,182,2017-06-11 19:21:19,http://west.biz/dashawn.runolfsdottir,,163.206.171.216,"{""location"": ""IL"", ""is_mobile"": true}" 1941,3,182,2017-05-28 03:02:34,http://deckow.com/adah,,21.141.141.76,"{""location"": ""BF"", ""is_mobile"": false}" 1942,3,182,2017-03-16 12:08:39,http://mohr.name/ernest_jones,,161.138.57.117,"{""location"": ""WS"", ""is_mobile"": true}" 1943,3,182,2017-05-29 12:44:13,http://weinat.biz/hellen.schowalter,,91.59.129.8,"{""location"": ""SZ"", ""is_mobile"": false}" 1944,3,182,2017-02-09 08:33:20,http://simonis.com/mattie,,114.46.231.125,"{""location"": ""CC"", ""is_mobile"": false}" 1945,3,182,2017-03-15 00:52:39,http://paucek.info/holly.schuppe,,208.49.8.5,"{""location"": ""TW"", ""is_mobile"": true}" 1946,3,182,2017-01-20 07:43:54,http://rogahn.com/van,,91.59.129.8,"{""location"": ""MN"", ""is_mobile"": false}" 1947,3,183,2017-04-16 18:07:27,http://robertsdibbert.com/joelle,,132.58.27.89,"{""location"": ""UZ"", ""is_mobile"": false}" 1948,3,183,2017-03-29 06:10:37,http://stamm.info/bella_effertz,,162.81.235.124,"{""location"": ""SS"", ""is_mobile"": true}" 1949,3,183,2017-05-14 13:38:42,http://vandervort.name/shayne_gleason,,142.253.16.83,"{""location"": ""MP"", ""is_mobile"": false}" 1950,3,183,2017-05-20 01:41:01,http://kertzmann.io/josefa,,126.214.141.239,"{""location"": ""MC"", ""is_mobile"": false}" 1951,3,183,2017-04-11 00:44:01,http://dietrich.org/amina.lynch,,57.111.46.95,"{""location"": ""ET"", ""is_mobile"": true}" 1952,3,183,2017-02-12 03:02:24,http://kihn.net/paige,,243.232.183.61,"{""location"": ""GH"", ""is_mobile"": false}" 1953,3,183,2017-03-30 17:14:37,http://hartmann.co/ludwig,,2.21.48.235,"{""location"": ""GE"", ""is_mobile"": true}" 1954,3,183,2016-12-28 19:49:05,http://gleichner.co/grover.tremblay,,149.212.50.31,"{""location"": ""GM"", ""is_mobile"": false}" 1955,3,183,2017-03-03 15:25:44,http://mckenziehackett.com/august_keeling,,204.247.135.39,"{""location"": ""DM"", ""is_mobile"": false}" 1956,3,183,2017-06-13 22:35:21,http://rau.net/hadley_murazik,,185.157.96.10,"{""location"": ""BI"", ""is_mobile"": false}" 1957,3,183,2017-05-14 06:13:05,http://haley.co/lilly,,254.188.137.215,"{""location"": ""BN"", ""is_mobile"": false}" 1958,3,183,2017-05-13 22:40:06,http://mohr.biz/camron_rogahn,,21.129.194.194,"{""location"": ""CG"", ""is_mobile"": true}" 1959,3,183,2017-02-07 18:15:10,http://ritchiecrooks.name/kianna_wilkinson,,251.150.89.113,"{""location"": ""IN"", ""is_mobile"": false}" 1960,3,183,2017-06-03 19:04:46,http://dooley.name/eliza,,132.58.27.89,"{""location"": ""ID"", ""is_mobile"": true}" 1961,3,183,2017-02-26 10:39:24,http://fritschrunolfsdottir.com/joany,,4.102.146.57,"{""location"": ""RO"", ""is_mobile"": false}" 1962,3,183,2017-01-07 18:40:35,http://gibson.net/guiseppe.ortiz,,215.143.77.74,"{""location"": ""CO"", ""is_mobile"": true}" 1963,3,184,2017-06-01 02:21:21,http://orn.info/anabel,,210.89.86.100,"{""location"": ""TJ"", ""is_mobile"": false}" 1964,3,184,2016-12-14 00:39:21,http://fisher.info/gardner,,119.12.150.23,"{""location"": ""CZ"", ""is_mobile"": true}" 1965,3,184,2017-03-18 06:29:03,http://spencer.info/ashtyn,,37.92.143.50,"{""location"": ""CO"", ""is_mobile"": false}" 1966,3,184,2017-01-23 12:36:12,http://howellpagac.biz/billie_larkin,,79.224.6.114,"{""location"": ""SG"", ""is_mobile"": true}" 1967,3,185,2017-06-06 07:56:51,http://friesen.name/miller.hettinger,1.0000000000,204.161.83.125,"{""location"": ""HM"", ""is_mobile"": true}" 1968,3,185,2016-12-24 22:40:51,http://wilkinsonolson.com/zachariah,1.0000000000,166.41.212.144,"{""location"": ""BW"", ""is_mobile"": true}" 1969,3,185,2017-03-03 19:16:06,http://hodkiewiczoconnell.info/daisha.crooks,1.0000000000,52.18.167.24,"{""location"": ""IO"", ""is_mobile"": false}" 1970,3,185,2016-12-21 10:40:12,http://kunde.com/dolores_romaguera,0.0000000000,194.199.204.225,"{""location"": ""BM"", ""is_mobile"": true}" 1971,3,185,2017-04-04 12:20:17,http://pfannerstillveum.io/wanda_volkman,1.0000000000,191.165.179.231,"{""location"": ""TG"", ""is_mobile"": true}" 1972,3,185,2017-01-17 04:16:59,http://rowehartmann.org/bettye_thiel,0.0000000000,22.175.25.12,"{""location"": ""PE"", ""is_mobile"": false}" 1973,3,185,2017-05-18 06:41:57,http://waelchi.io/noemi.jacobson,0.0000000000,210.251.44.245,"{""location"": ""GY"", ""is_mobile"": false}" 1974,3,185,2017-03-17 11:58:41,http://stracke.name/gerry_walker,1.0000000000,166.41.212.144,"{""location"": ""SO"", ""is_mobile"": true}" 1975,3,185,2016-12-14 02:08:04,http://rogahn.info/nicklaus,0.0000000000,190.36.116.76,"{""location"": ""CV"", ""is_mobile"": true}" 4635,7,435,2017-03-08 20:06:14,http://mueller.info/alanna.kuhlman,,163.161.44.146,"{""location"": ""KR"", ""is_mobile"": false}" 4636,7,435,2017-03-23 16:35:35,http://ratkevandervort.org/anita_okeefe,,210.45.92.156,"{""location"": ""AR"", ""is_mobile"": true}" 4637,7,435,2017-05-11 23:26:51,http://haleypagac.net/susie_oconner,,235.195.250.23,"{""location"": ""MF"", ""is_mobile"": false}" 4638,7,435,2017-05-28 03:04:27,http://mannschuster.name/austyn,,163.161.44.146,"{""location"": ""PA"", ""is_mobile"": false}" 4639,7,435,2017-03-21 00:38:58,http://nitzsche.name/agustina,,227.230.42.124,"{""location"": ""AQ"", ""is_mobile"": true}" 4640,7,435,2017-01-27 13:54:39,http://gerholdhalvorson.net/scotty.wolf,,7.145.18.113,"{""location"": ""BN"", ""is_mobile"": false}" 4641,7,435,2017-05-24 02:55:37,http://schaefer.co/astrid,,227.230.42.124,"{""location"": ""CI"", ""is_mobile"": false}" 4642,7,435,2017-03-07 14:25:56,http://koch.com/jennings.rutherford,,241.212.190.57,"{""location"": ""EE"", ""is_mobile"": true}" 4643,7,435,2017-05-01 19:25:17,http://mcclurewolf.biz/joelle.beier,,157.141.85.23,"{""location"": ""SJ"", ""is_mobile"": true}" 4644,7,435,2017-02-14 15:22:52,http://tremblay.biz/elroy.barrows,,222.15.9.50,"{""location"": ""PM"", ""is_mobile"": false}" 4645,7,436,2016-12-21 01:32:25,http://swaniawskibradtke.net/teie_lowe,,195.229.246.38,"{""location"": ""CL"", ""is_mobile"": true}" 4646,7,436,2017-05-17 21:24:09,http://rosenbaumreichel.org/lew,,59.19.248.161,"{""location"": ""ID"", ""is_mobile"": false}" 4647,7,436,2017-04-10 07:05:53,http://ullrich.name/bradly,,110.223.82.214,"{""location"": ""EC"", ""is_mobile"": false}" 4648,7,437,2017-06-12 14:33:06,http://kris.biz/dawn,,235.33.238.110,"{""location"": ""VU"", ""is_mobile"": true}" 4649,7,437,2017-03-12 03:31:01,http://emmerich.org/dee_robel,,20.149.196.234,"{""location"": ""ZM"", ""is_mobile"": true}" 4650,7,437,2017-04-27 03:14:31,http://nienow.net/odie_stanton,,29.6.112.247,"{""location"": ""MU"", ""is_mobile"": false}" 4651,7,437,2017-04-17 16:54:41,http://sauer.com/frankie,,174.223.128.77,"{""location"": ""EC"", ""is_mobile"": false}" 4652,7,437,2017-04-07 13:17:47,http://kulasdurgan.com/aurelia,,102.75.217.148,"{""location"": ""PG"", ""is_mobile"": false}" 4653,7,437,2017-05-25 09:44:55,http://lynchjohnston.net/josephine_kuhn,,3.137.121.179,"{""location"": ""BQ"", ""is_mobile"": false}" 4654,7,437,2017-05-15 10:17:22,http://murraydubuque.com/sierra.rosenbaum,,217.248.57.88,"{""location"": ""FJ"", ""is_mobile"": true}" 4655,7,437,2017-05-31 02:52:00,http://bauchlockman.com/tierra,,145.206.124.176,"{""location"": ""LV"", ""is_mobile"": false}" 4656,7,437,2017-01-18 09:10:44,http://emmerich.com/jules_corkery,,235.33.238.110,"{""location"": ""CA"", ""is_mobile"": false}" 4657,7,437,2017-05-06 07:23:34,http://marquardt.org/thora_bechtelar,,70.56.135.157,"{""location"": ""BY"", ""is_mobile"": false}" 4658,7,437,2016-12-16 15:13:16,http://kiehn.name/vergie,,245.77.116.195,"{""location"": ""LU"", ""is_mobile"": false}" 4659,7,437,2017-02-20 11:57:05,http://purdy.biz/river,,154.145.53.191,"{""location"": ""AR"", ""is_mobile"": true}" 4660,7,437,2017-01-18 19:06:35,http://heaneyhammes.name/frederique,,70.56.135.157,"{""location"": ""DZ"", ""is_mobile"": false}" 4661,7,437,2017-04-23 23:56:09,http://ankunding.name/tania,,18.19.143.129,"{""location"": ""TF"", ""is_mobile"": false}" 4662,7,437,2017-02-14 14:04:50,http://wisokyvandervort.name/shana.wyman,,88.173.33.78,"{""location"": ""MF"", ""is_mobile"": false}" 4663,7,437,2016-12-30 17:18:10,http://lednerheathcote.org/ludie.miller,,240.158.206.127,"{""location"": ""TD"", ""is_mobile"": true}" 4664,7,437,2017-01-03 19:45:17,http://ohara.org/mae.ryan,,43.117.12.97,"{""location"": ""CD"", ""is_mobile"": true}" 4665,7,437,2017-05-04 13:48:36,http://grady.net/jacey,,245.77.116.195,"{""location"": ""UA"", ""is_mobile"": false}" 4666,7,438,2017-05-18 02:00:29,http://gutmann.org/bertha,,140.141.117.141,"{""location"": ""MH"", ""is_mobile"": true}" 4667,7,438,2017-02-07 19:36:37,http://hyatt.io/chesley,,61.167.7.92,"{""location"": ""EG"", ""is_mobile"": false}" 4668,7,438,2017-05-14 11:52:10,http://hyatthilpert.name/luther.auer,,210.187.92.59,"{""location"": ""SH"", ""is_mobile"": true}" 4669,7,438,2017-02-11 22:23:48,http://gutmann.biz/alvina,,44.105.205.181,"{""location"": ""KH"", ""is_mobile"": false}" 4670,7,438,2017-01-27 08:44:40,http://wuckerthoeger.org/danny.gulgowski,,113.210.153.54,"{""location"": ""SC"", ""is_mobile"": false}" 4671,7,439,2017-04-25 12:19:38,http://brekkeharber.org/millie,,2.226.169.138,"{""location"": ""MX"", ""is_mobile"": true}" 4672,7,439,2017-03-03 17:54:12,http://borer.name/nola_schuppe,,200.38.169.156,"{""location"": ""GE"", ""is_mobile"": true}" 4673,7,439,2017-01-28 17:28:26,http://simoniskunze.info/larue.harber,,79.38.133.76,"{""location"": ""CK"", ""is_mobile"": false}" 4674,7,439,2017-03-31 18:52:34,http://yundt.io/claudia_weinat,,119.52.63.38,"{""location"": ""NE"", ""is_mobile"": false}" 4675,7,439,2017-06-04 05:43:53,http://conroy.com/elia.schimmel,,94.160.201.19,"{""location"": ""SV"", ""is_mobile"": true}" 4676,7,439,2017-04-18 16:33:17,http://mcdermottrohan.info/jonathon,,162.133.162.110,"{""location"": ""TV"", ""is_mobile"": false}" 4677,7,439,2017-01-23 00:46:00,http://price.info/leonora_schinner,,13.127.120.195,"{""location"": ""BB"", ""is_mobile"": false}" 4678,7,440,2017-02-28 12:46:30,http://kolynch.info/fleta,,90.142.142.54,"{""location"": ""HU"", ""is_mobile"": false}" 4679,7,440,2017-03-10 09:42:05,http://conroycartwright.biz/nick.wiegand,,186.10.242.234,"{""location"": ""KN"", ""is_mobile"": true}" 4680,7,440,2016-12-27 07:59:31,http://quitzon.co/dandre,,206.51.133.225,"{""location"": ""MS"", ""is_mobile"": false}" 4681,7,440,2017-02-10 14:35:52,http://blick.info/donnie.ullrich,,90.142.142.54,"{""location"": ""GD"", ""is_mobile"": true}" 4682,7,440,2017-03-02 19:55:40,http://gulgowski.com/alivia_mills,,19.210.52.177,"{""location"": ""VN"", ""is_mobile"": false}" 4683,7,440,2017-02-21 04:48:34,http://reichel.co/efrain_bechtelar,,46.16.67.82,"{""location"": ""EC"", ""is_mobile"": false}" 4684,7,440,2017-02-27 09:08:44,http://mcdermott.org/fredrick.mitchell,,126.115.176.56,"{""location"": ""JO"", ""is_mobile"": true}" 4685,7,440,2017-05-08 02:12:55,http://bins.info/brendan,,254.70.60.222,"{""location"": ""GI"", ""is_mobile"": false}" 4686,7,440,2017-02-07 21:53:40,http://schuppe.info/ulises,,192.66.126.228,"{""location"": ""MZ"", ""is_mobile"": true}" 4687,7,440,2016-12-24 10:08:47,http://manndeckow.info/oceane_jones,,110.154.146.193,"{""location"": ""TM"", ""is_mobile"": false}" 4688,7,440,2017-03-27 18:07:00,http://cain.name/elia_collins,,28.159.193.222,"{""location"": ""BM"", ""is_mobile"": true}" 4689,7,440,2017-03-31 14:11:39,http://quigley.net/citlalli,,135.165.130.108,"{""location"": ""CR"", ""is_mobile"": true}" 1976,3,185,2016-12-19 21:18:33,http://deckow.net/carol,0.0000000000,215.89.210.165,"{""location"": ""SS"", ""is_mobile"": true}" 1977,3,185,2017-06-12 04:45:58,http://bruen.name/alize,1.0000000000,52.18.167.24,"{""location"": ""AZ"", ""is_mobile"": true}" 1978,3,185,2017-03-13 21:59:23,http://batz.org/maverick_treutel,1.0000000000,26.66.118.123,"{""location"": ""GL"", ""is_mobile"": true}" 1979,3,185,2017-06-03 11:07:21,http://cainstehr.org/frankie.schulist,1.0000000000,176.218.219.222,"{""location"": ""RE"", ""is_mobile"": true}" 1980,3,185,2017-04-01 00:02:15,http://danieldooley.net/adrien,1.0000000000,18.176.153.136,"{""location"": ""LV"", ""is_mobile"": false}" 1981,3,185,2017-01-15 15:35:32,http://friesen.org/hannah,0.0000000000,26.66.118.123,"{""location"": ""EC"", ""is_mobile"": true}" 1982,3,185,2017-01-20 14:14:00,http://connelly.com/vickie_cronin,1.0000000000,204.66.140.187,"{""location"": ""GI"", ""is_mobile"": true}" 1983,3,186,2017-01-16 15:08:01,http://simonis.org/rosella,0.0000000000,194.230.130.110,"{""location"": ""JE"", ""is_mobile"": false}" 1984,3,186,2017-04-07 13:57:29,http://konopelskimarquardt.org/geraldine.becker,1.0000000000,254.99.118.243,"{""location"": ""PR"", ""is_mobile"": true}" 1985,3,186,2017-03-23 20:01:50,http://langworth.name/newton.glover,3.0000000000,41.138.177.6,"{""location"": ""TD"", ""is_mobile"": true}" 1986,3,186,2017-04-30 19:01:27,http://hauck.biz/brooklyn,2.0000000000,55.79.11.203,"{""location"": ""SO"", ""is_mobile"": true}" 1987,3,186,2017-04-15 06:01:09,http://graham.co/sven,2.0000000000,23.28.74.68,"{""location"": ""AG"", ""is_mobile"": false}" 1988,3,186,2017-05-13 14:15:22,http://kreiger.biz/joanie.lockman,0.0000000000,183.104.81.108,"{""location"": ""PK"", ""is_mobile"": false}" 1989,3,186,2017-02-07 20:05:31,http://rippinkohler.net/amira,0.0000000000,25.40.112.109,"{""location"": ""GU"", ""is_mobile"": false}" 1990,3,186,2017-01-13 03:56:49,http://wizakoepp.net/hector_conn,0.0000000000,114.35.37.106,"{""location"": ""CH"", ""is_mobile"": false}" 1991,3,186,2017-01-29 06:17:18,http://hoppe.name/arlo_carter,2.0000000000,23.28.74.68,"{""location"": ""MO"", ""is_mobile"": false}" 1992,3,186,2017-01-30 13:22:46,http://schroeder.com/noemi_hackett,1.0000000000,202.87.236.216,"{""location"": ""IS"", ""is_mobile"": true}" 1993,3,186,2017-03-15 20:49:02,http://millerlebsack.com/noelia,2.0000000000,67.128.157.145,"{""location"": ""FJ"", ""is_mobile"": false}" 1994,3,186,2016-12-18 18:52:15,http://pollichquitzon.net/edgardo,2.0000000000,194.230.130.110,"{""location"": ""NE"", ""is_mobile"": false}" 1995,3,186,2017-05-18 06:18:09,http://howell.info/madonna_stroman,0.0000000000,25.40.112.109,"{""location"": ""VE"", ""is_mobile"": true}" 1996,3,186,2017-02-08 03:49:40,http://gleasonwill.com/eldon,3.0000000000,183.104.81.108,"{""location"": ""QA"", ""is_mobile"": true}" 1997,3,186,2017-04-11 19:49:24,http://heaneydaugherty.io/jamison.davis,0.0000000000,194.230.130.110,"{""location"": ""TT"", ""is_mobile"": true}" 1998,3,186,2017-02-06 00:48:00,http://grant.info/antwan,3.0000000000,59.13.216.153,"{""location"": ""GQ"", ""is_mobile"": false}" 1999,3,186,2017-02-12 01:39:08,http://koch.biz/dangelo_littel,2.0000000000,141.171.126.232,"{""location"": ""BF"", ""is_mobile"": false}" 2000,3,186,2017-03-06 06:07:12,http://vonrueden.info/brice.mraz,1.0000000000,46.7.117.248,"{""location"": ""QA"", ""is_mobile"": false}" 2001,3,186,2017-03-02 07:09:34,http://marvin.io/dina,2.0000000000,183.104.81.108,"{""location"": ""MN"", ""is_mobile"": true}" 2002,3,187,2017-03-04 15:48:46,http://murphyreichel.org/jacey,1.0000000000,60.235.147.157,"{""location"": ""UZ"", ""is_mobile"": true}" 2003,3,187,2017-05-21 19:13:45,http://brekke.io/rickey,0.0000000000,87.48.234.69,"{""location"": ""RE"", ""is_mobile"": true}" 2004,3,187,2017-04-30 20:29:30,http://hoppewiegand.biz/theron,0.0000000000,54.187.136.187,"{""location"": ""BT"", ""is_mobile"": true}" 2005,3,187,2016-12-31 22:54:12,http://emmerich.name/juliana,3.0000000000,198.92.230.78,"{""location"": ""KZ"", ""is_mobile"": true}" 2006,3,187,2016-12-31 10:10:50,http://johnson.org/jovan,3.0000000000,105.5.113.175,"{""location"": ""EH"", ""is_mobile"": true}" 2007,3,187,2017-02-22 01:06:00,http://dare.biz/crystel.brakus,1.0000000000,155.16.87.48,"{""location"": ""NE"", ""is_mobile"": false}" 2008,3,187,2017-03-01 11:33:03,http://hahn.io/paul_halvorson,0.0000000000,4.131.201.25,"{""location"": ""MO"", ""is_mobile"": false}" 2009,3,187,2017-05-29 13:57:05,http://dickinson.com/stefanie,0.0000000000,190.26.163.225,"{""location"": ""SD"", ""is_mobile"": true}" 2010,3,187,2017-04-25 04:39:37,http://kuvalis.io/conner,3.0000000000,21.132.243.146,"{""location"": ""SB"", ""is_mobile"": true}" 2011,3,188,2017-05-19 15:53:03,http://bahringerbayer.co/ceasar_bradtke,3.0000000000,65.27.5.82,"{""location"": ""NG"", ""is_mobile"": false}" 2012,3,189,2017-05-31 21:29:54,http://jaskolski.biz/mohammad.wehner,1.0000000000,106.193.27.120,"{""location"": ""RS"", ""is_mobile"": false}" 2013,3,189,2017-04-30 12:56:11,http://will.name/roman.daugherty,0.0000000000,249.86.66.79,"{""location"": ""AD"", ""is_mobile"": true}" 2014,3,189,2017-03-27 07:15:33,http://padberg.co/stevie,1.0000000000,21.108.116.37,"{""location"": ""PH"", ""is_mobile"": false}" 2015,3,189,2017-06-13 02:39:59,http://skiles.net/lucinda.gislason,1.0000000000,174.128.189.115,"{""location"": ""PR"", ""is_mobile"": false}" 2016,3,189,2017-05-02 20:06:32,http://leannon.co/amara_jacobson,0.0000000000,174.128.189.115,"{""location"": ""GI"", ""is_mobile"": false}" 2017,3,189,2017-01-23 13:13:44,http://little.net/danika,2.0000000000,19.41.229.37,"{""location"": ""TT"", ""is_mobile"": true}" 2018,3,189,2017-05-16 14:36:01,http://jakubowski.name/mckayla_legros,1.0000000000,125.116.176.125,"{""location"": ""YT"", ""is_mobile"": false}" 2019,3,189,2017-05-31 04:48:47,http://hahnnienow.co/jayden,1.0000000000,174.128.189.115,"{""location"": ""BN"", ""is_mobile"": false}" 2020,3,189,2017-04-23 18:09:14,http://langsporer.org/seamus_zemlak,2.0000000000,250.77.42.66,"{""location"": ""IE"", ""is_mobile"": true}" 2021,3,189,2017-02-10 07:46:17,http://murazik.io/berneice.steuber,1.0000000000,188.191.203.189,"{""location"": ""NR"", ""is_mobile"": true}" 2022,3,189,2017-02-07 16:20:10,http://dubuquebeahan.org/moses,0.0000000000,167.79.244.178,"{""location"": ""DJ"", ""is_mobile"": false}" 2023,3,190,2017-03-12 10:19:40,http://bayer.biz/neha_doyle,3.0000000000,129.222.235.173,"{""location"": ""BN"", ""is_mobile"": false}" 2024,3,190,2017-03-07 16:55:57,http://jakubowskihegmann.net/jasmin,0.0000000000,37.101.21.89,"{""location"": ""AT"", ""is_mobile"": true}" 2025,3,190,2017-01-12 22:21:13,http://dooley.info/colten_kilback,3.0000000000,64.189.155.78,"{""location"": ""RU"", ""is_mobile"": false}" 2026,3,190,2017-03-17 01:07:57,http://cristbergnaum.io/guillermo_rowe,3.0000000000,226.89.140.188,"{""location"": ""UZ"", ""is_mobile"": true}" 2027,3,190,2016-12-27 15:42:10,http://bode.org/macey_anderson,0.0000000000,136.107.159.116,"{""location"": ""VI"", ""is_mobile"": false}" 4690,7,440,2017-04-18 13:35:28,http://danielkunde.org/kaleigh,,110.154.146.193,"{""location"": ""GT"", ""is_mobile"": false}" 4691,7,440,2017-01-28 21:30:03,http://gulgowski.org/shanon_west,,122.51.13.2,"{""location"": ""RE"", ""is_mobile"": false}" 4692,7,440,2017-02-27 13:39:01,http://mohr.com/vance_mcdermott,,206.51.133.225,"{""location"": ""PG"", ""is_mobile"": true}" 4693,7,441,2017-03-06 23:31:46,http://cummeratawunsch.name/mark,,113.209.172.182,"{""location"": ""CO"", ""is_mobile"": false}" 4694,7,441,2017-05-06 13:17:39,http://nicolas.net/myrtie.yost,,132.215.234.54,"{""location"": ""TF"", ""is_mobile"": true}" 4695,7,441,2017-01-17 00:35:49,http://witting.name/verona,,213.154.233.165,"{""location"": ""TF"", ""is_mobile"": true}" 4696,7,441,2017-03-30 10:27:51,http://hueldaugherty.net/name,,135.129.122.175,"{""location"": ""QA"", ""is_mobile"": false}" 4697,7,442,2017-02-13 01:32:22,http://roberts.net/lea_moriette,,234.182.178.71,"{""location"": ""EG"", ""is_mobile"": true}" 4698,7,442,2017-02-16 03:17:15,http://runolfon.biz/carmen,,159.108.17.208,"{""location"": ""NR"", ""is_mobile"": true}" 4699,7,442,2017-03-23 13:05:01,http://okuneva.info/tyrell,,95.235.90.20,"{""location"": ""CW"", ""is_mobile"": true}" 4700,7,442,2017-06-13 02:51:48,http://waters.info/joel,,18.120.188.247,"{""location"": ""CL"", ""is_mobile"": false}" 4701,7,442,2016-12-17 01:01:10,http://christiansen.biz/kiana,,207.85.133.18,"{""location"": ""BD"", ""is_mobile"": true}" 4702,7,442,2017-03-28 19:59:03,http://raynor.com/kattie_spinka,,56.54.157.221,"{""location"": ""PM"", ""is_mobile"": false}" 4703,7,443,2017-01-31 19:32:46,http://cartwright.info/jerrold.hudson,,162.242.233.211,"{""location"": ""MX"", ""is_mobile"": false}" 4704,7,443,2017-04-02 22:17:26,http://ruel.biz/jensen,,161.171.191.246,"{""location"": ""PH"", ""is_mobile"": false}" 4705,7,443,2017-06-02 18:37:58,http://reichert.info/jeanette,,224.52.230.42,"{""location"": ""DO"", ""is_mobile"": false}" 4706,7,443,2017-04-28 16:01:17,http://lindgren.io/carey,,117.224.111.20,"{""location"": ""VA"", ""is_mobile"": true}" 4707,7,443,2017-03-04 05:07:41,http://kulas.org/darrell,,168.168.24.164,"{""location"": ""PG"", ""is_mobile"": true}" 4708,7,443,2017-02-17 21:16:38,http://bogan.co/ima.gusikowski,,195.111.155.142,"{""location"": ""ML"", ""is_mobile"": true}" 4709,7,443,2017-02-22 04:06:54,http://lehnergreenholt.io/karelle_schuster,,190.46.193.20,"{""location"": ""IE"", ""is_mobile"": true}" 4710,7,443,2017-04-19 18:00:43,http://wiegandprohaska.net/america,,233.232.180.152,"{""location"": ""PA"", ""is_mobile"": false}" 4711,7,444,2017-05-15 06:08:00,http://heathcote.info/kenneth.zboncak,,106.28.52.207,"{""location"": ""IT"", ""is_mobile"": false}" 4712,7,444,2017-05-03 13:25:00,http://kuhlmanlueilwitz.org/gregg,,118.162.201.5,"{""location"": ""PR"", ""is_mobile"": false}" 4713,7,444,2017-03-08 10:00:26,http://pfannerstill.io/gerry,,204.19.242.138,"{""location"": ""CO"", ""is_mobile"": false}" 4714,7,444,2017-06-13 01:52:11,http://uptonbraun.info/braden.farrell,,246.112.130.187,"{""location"": ""KH"", ""is_mobile"": false}" 4715,7,444,2016-12-22 00:22:58,http://connelly.io/baylee,,233.154.231.76,"{""location"": ""TF"", ""is_mobile"": true}" 4716,7,444,2017-02-19 11:34:51,http://lindgren.co/nikki,,203.165.169.33,"{""location"": ""LV"", ""is_mobile"": false}" 4717,7,444,2017-02-09 01:51:36,http://yundtgrimes.org/nelda.schuster,,188.61.178.42,"{""location"": ""MU"", ""is_mobile"": true}" 4718,7,444,2017-02-12 07:51:56,http://kris.com/ryley,,204.19.242.138,"{""location"": ""IR"", ""is_mobile"": false}" 4719,7,444,2017-03-24 00:03:13,http://kreiger.info/bulah,,106.28.52.207,"{""location"": ""BM"", ""is_mobile"": false}" 4720,7,444,2017-04-15 10:39:37,http://hermann.com/anais,,131.139.37.67,"{""location"": ""BS"", ""is_mobile"": false}" 4721,7,445,2017-03-31 03:53:01,http://schowalter.net/genesis.smitham,,71.229.22.234,"{""location"": ""NC"", ""is_mobile"": true}" 4722,7,445,2017-04-12 21:14:26,http://johnston.org/troy.franecki,,198.133.88.32,"{""location"": ""IE"", ""is_mobile"": true}" 4723,7,445,2017-04-03 07:16:22,http://hoppe.info/josianne_damore,,19.37.183.148,"{""location"": ""LA"", ""is_mobile"": true}" 4724,7,445,2016-12-19 14:20:54,http://parker.com/mabel,,121.71.97.214,"{""location"": ""CF"", ""is_mobile"": true}" 4725,7,445,2017-04-05 19:27:20,http://bernierreynolds.biz/thurman_bartell,,48.90.251.169,"{""location"": ""BB"", ""is_mobile"": false}" 4726,7,445,2017-03-12 16:43:45,http://labadieschroeder.biz/kennedy_haag,,247.107.142.162,"{""location"": ""NZ"", ""is_mobile"": true}" 4727,7,445,2017-05-01 00:36:42,http://windlerschmidt.com/obie,,100.2.186.95,"{""location"": ""SM"", ""is_mobile"": false}" 4728,7,446,2017-02-13 07:17:11,http://hermiston.org/ottilie,,147.228.41.201,"{""location"": ""FM"", ""is_mobile"": true}" 4729,7,446,2017-01-10 21:30:07,http://cole.name/stan_emard,,128.241.231.28,"{""location"": ""WS"", ""is_mobile"": false}" 4730,7,446,2017-05-23 13:34:33,http://collier.name/yazmin.wintheiser,,254.3.234.220,"{""location"": ""GH"", ""is_mobile"": true}" 4731,7,446,2017-02-12 17:38:38,http://boscomcglynn.net/clifford,,228.197.201.81,"{""location"": ""PG"", ""is_mobile"": true}" 4732,7,446,2017-04-14 17:12:57,http://durgan.com/freeman,,71.19.106.211,"{""location"": ""ST"", ""is_mobile"": false}" 4733,7,446,2017-01-15 17:06:01,http://strackewaters.biz/sheldon,,164.97.15.102,"{""location"": ""SZ"", ""is_mobile"": true}" 4734,7,446,2017-03-05 09:22:12,http://spinka.com/arch,,133.218.163.19,"{""location"": ""ML"", ""is_mobile"": true}" 4735,7,446,2017-03-28 10:55:59,http://kirlinmcclure.name/bettie.wisozk,,228.19.222.60,"{""location"": ""BZ"", ""is_mobile"": false}" 4736,7,446,2017-05-03 12:50:25,http://trantowbode.net/lucie.collier,,90.102.241.179,"{""location"": ""LB"", ""is_mobile"": false}" 4737,7,446,2017-01-05 20:25:35,http://mohr.name/marcia_kuhlman,,170.165.211.96,"{""location"": ""NU"", ""is_mobile"": true}" 4738,7,446,2017-03-24 16:02:04,http://bergnaum.info/raheem_becker,,101.191.250.189,"{""location"": ""MX"", ""is_mobile"": true}" 4739,7,447,2017-06-11 02:23:31,http://schmidt.com/modesto_will,,204.66.243.246,"{""location"": ""IR"", ""is_mobile"": false}" 4740,7,447,2017-02-13 11:45:05,http://sanfordgrimes.name/maribel_wiza,,193.169.73.200,"{""location"": ""MG"", ""is_mobile"": true}" 4741,7,447,2016-12-30 05:52:56,http://kunde.com/cordie,,216.185.40.241,"{""location"": ""ST"", ""is_mobile"": true}" 4742,7,447,2017-03-07 02:42:29,http://langosh.biz/kathryne.greenholt,,216.185.40.241,"{""location"": ""KY"", ""is_mobile"": true}" 4743,7,447,2017-02-04 03:10:48,http://crooks.info/jewell_swift,,110.214.183.142,"{""location"": ""MT"", ""is_mobile"": false}" 4744,7,447,2017-05-31 20:51:12,http://morarankunding.name/billy.blanda,,83.81.14.168,"{""location"": ""NU"", ""is_mobile"": false}" 2028,3,190,2017-01-26 13:15:18,http://volkmankozey.org/rosalind_homenick,3.0000000000,93.238.6.177,"{""location"": ""BE"", ""is_mobile"": false}" 2029,3,190,2017-01-09 07:39:03,http://bednar.info/bobby,2.0000000000,129.222.235.173,"{""location"": ""MP"", ""is_mobile"": false}" 2030,3,191,2017-03-22 01:43:42,http://reynolds.com/imani_cummerata,0.0000000000,217.80.168.68,"{""location"": ""UZ"", ""is_mobile"": true}" 2031,3,191,2017-01-26 23:35:15,http://harvey.net/judah_koepp,0.0000000000,169.139.198.68,"{""location"": ""ME"", ""is_mobile"": false}" 2032,3,191,2017-02-02 07:48:39,http://sporer.biz/andy.nikolaus,3.0000000000,101.19.25.199,"{""location"": ""TM"", ""is_mobile"": true}" 2033,3,191,2017-04-30 05:54:40,http://sauermills.name/antonio.toy,3.0000000000,154.242.211.48,"{""location"": ""ZW"", ""is_mobile"": false}" 2034,3,191,2017-05-29 23:46:59,http://herman.biz/emilia,3.0000000000,56.167.37.6,"{""location"": ""AF"", ""is_mobile"": true}" 2035,3,191,2017-06-03 08:19:18,http://kaulke.io/saige,1.0000000000,32.137.9.40,"{""location"": ""FJ"", ""is_mobile"": false}" 2036,3,191,2017-03-19 13:23:07,http://legroshoppe.com/gregorio,2.0000000000,61.140.5.92,"{""location"": ""TD"", ""is_mobile"": false}" 2037,3,191,2017-01-12 00:25:06,http://schimmel.net/samson.wintheiser,3.0000000000,228.9.79.167,"{""location"": ""PT"", ""is_mobile"": true}" 2038,3,191,2017-02-20 14:47:11,http://heathcote.biz/raheem,2.0000000000,4.4.182.34,"{""location"": ""SS"", ""is_mobile"": true}" 2039,3,191,2017-04-11 08:27:08,http://ohara.co/rose_huels,3.0000000000,101.19.25.199,"{""location"": ""BW"", ""is_mobile"": false}" 2040,3,191,2016-12-23 16:35:17,http://tremblay.net/jayden,3.0000000000,217.80.168.68,"{""location"": ""KR"", ""is_mobile"": true}" 2041,3,191,2017-03-07 02:00:04,http://schroeder.com/emely,2.0000000000,158.109.166.17,"{""location"": ""GM"", ""is_mobile"": true}" 2042,3,191,2017-04-26 01:23:39,http://monahan.co/wilburn,2.0000000000,144.107.12.137,"{""location"": ""PH"", ""is_mobile"": false}" 2043,3,191,2017-04-06 20:37:40,http://auerrippin.name/destin,0.0000000000,115.152.118.138,"{""location"": ""BY"", ""is_mobile"": false}" 2044,3,191,2017-05-22 04:50:54,http://king.com/kayley_hahn,1.0000000000,160.226.102.96,"{""location"": ""EE"", ""is_mobile"": false}" 2045,3,192,2017-03-29 13:43:10,http://gutmanntrantow.co/brock,0.0000000000,96.77.89.166,"{""location"": ""CY"", ""is_mobile"": false}" 2046,3,192,2017-02-23 11:19:35,http://gerlach.com/lauriane,0.0000000000,246.231.204.127,"{""location"": ""GF"", ""is_mobile"": false}" 2047,3,192,2017-06-09 20:34:37,http://koch.co/valentine,2.0000000000,173.193.194.188,"{""location"": ""VC"", ""is_mobile"": true}" 2048,3,192,2017-03-18 19:16:52,http://kunze.org/nicola,0.0000000000,237.204.99.2,"{""location"": ""NP"", ""is_mobile"": true}" 2049,3,192,2017-04-19 09:59:56,http://cummings.com/henderson,1.0000000000,252.13.175.209,"{""location"": ""AE"", ""is_mobile"": false}" 2050,3,192,2017-02-12 04:45:09,http://beattykeebler.org/leonie_purdy,1.0000000000,77.157.104.92,"{""location"": ""SD"", ""is_mobile"": true}" 2051,3,192,2017-06-08 08:37:40,http://prohaska.net/brayan_kemmer,0.0000000000,95.10.107.54,"{""location"": ""NZ"", ""is_mobile"": false}" 2052,3,192,2017-02-26 13:01:08,http://ebert.org/mathilde.huel,1.0000000000,96.77.89.166,"{""location"": ""VI"", ""is_mobile"": true}" 2053,3,192,2017-02-03 21:31:52,http://vandervortcasper.info/murray,2.0000000000,252.13.175.209,"{""location"": ""RW"", ""is_mobile"": true}" 2054,3,192,2017-02-05 22:39:46,http://roobdietrich.io/leland,0.0000000000,251.85.5.178,"{""location"": ""CA"", ""is_mobile"": false}" 2055,3,193,2017-01-10 14:50:16,http://ferryerdman.net/alison.hoppe,0.0000000000,172.105.228.61,"{""location"": ""BQ"", ""is_mobile"": true}" 2056,3,193,2017-01-24 05:26:32,http://altenwerth.com/jorge,1.0000000000,18.175.244.198,"{""location"": ""ET"", ""is_mobile"": true}" 2057,3,193,2017-03-02 01:25:44,http://stracke.name/elmira_murazik,0.0000000000,38.129.235.92,"{""location"": ""SI"", ""is_mobile"": false}" 2058,3,193,2016-12-22 14:15:38,http://reynoldshuels.name/brando.lockman,1.0000000000,44.166.240.204,"{""location"": ""SD"", ""is_mobile"": true}" 2059,3,193,2017-04-12 21:14:58,http://miller.com/reinhold,1.0000000000,58.111.129.120,"{""location"": ""BH"", ""is_mobile"": true}" 2060,3,193,2017-04-10 07:23:46,http://townekling.biz/hellen,2.0000000000,105.106.132.40,"{""location"": ""BT"", ""is_mobile"": false}" 2061,3,193,2017-02-19 20:52:14,http://wunschfranecki.info/zelda,3.0000000000,189.244.196.14,"{""location"": ""BQ"", ""is_mobile"": true}" 2062,3,193,2016-12-22 01:31:19,http://pricecasper.name/emilio.terry,2.0000000000,189.244.196.14,"{""location"": ""TR"", ""is_mobile"": false}" 2063,3,193,2017-04-23 10:33:37,http://berge.net/daphnee_bosco,2.0000000000,113.30.199.122,"{""location"": ""ET"", ""is_mobile"": true}" 2064,3,193,2017-02-07 02:34:08,http://schulist.co/jannie,2.0000000000,238.79.70.28,"{""location"": ""AS"", ""is_mobile"": false}" 2065,3,193,2017-04-26 12:32:56,http://altenwerthgreenfelder.co/michele,3.0000000000,165.176.73.62,"{""location"": ""PM"", ""is_mobile"": true}" 2066,3,193,2017-03-13 17:57:46,http://hauckrath.name/barrett.jacobs,3.0000000000,189.244.196.14,"{""location"": ""ME"", ""is_mobile"": false}" 2067,3,193,2017-01-07 18:11:28,http://hills.com/stefan_pagac,0.0000000000,189.244.196.14,"{""location"": ""AL"", ""is_mobile"": false}" 2068,3,193,2016-12-26 20:32:29,http://macejkovic.io/reuben,3.0000000000,58.111.129.120,"{""location"": ""ML"", ""is_mobile"": false}" 2069,3,193,2017-05-29 16:33:15,http://bernhard.org/arnulfo.steuber,3.0000000000,35.170.85.206,"{""location"": ""LC"", ""is_mobile"": true}" 2070,3,193,2016-12-21 09:33:49,http://hermann.io/roderick_ruel,2.0000000000,119.119.219.176,"{""location"": ""AW"", ""is_mobile"": false}" 2071,3,194,2017-02-24 12:55:31,http://pagac.net/mortimer,0.0000000000,49.107.73.14,"{""location"": ""BL"", ""is_mobile"": true}" 2072,3,194,2017-03-28 18:51:21,http://heel.biz/samanta,0.0000000000,203.38.71.77,"{""location"": ""IL"", ""is_mobile"": false}" 2073,3,194,2017-05-28 23:54:56,http://waelchiquitzon.org/jamil,0.0000000000,156.26.214.55,"{""location"": ""GY"", ""is_mobile"": false}" 2074,3,194,2017-03-29 01:53:41,http://beer.net/dane,0.0000000000,124.108.77.57,"{""location"": ""CN"", ""is_mobile"": true}" 2075,3,194,2017-02-14 14:21:11,http://kulas.net/kristopher,0.0000000000,44.178.162.197,"{""location"": ""SN"", ""is_mobile"": false}" 2076,3,194,2017-01-08 00:36:57,http://kuhn.biz/joel,0.0000000000,88.26.98.128,"{""location"": ""FO"", ""is_mobile"": false}" 2077,3,194,2017-02-04 15:21:01,http://schuster.biz/jonathon,0.0000000000,173.219.136.211,"{""location"": ""CM"", ""is_mobile"": false}" 2078,3,194,2016-12-26 05:32:36,http://blandaherman.io/helen,0.0000000000,4.51.96.198,"{""location"": ""MU"", ""is_mobile"": false}" 2079,3,194,2017-04-30 23:49:59,http://klocko.co/kameron,0.0000000000,117.253.95.202,"{""location"": ""MY"", ""is_mobile"": false}" 4745,7,448,2017-01-11 19:54:58,http://dubuque.com/jaylan.feest,,44.223.226.149,"{""location"": ""LB"", ""is_mobile"": false}" 4746,7,448,2016-12-19 01:56:06,http://labadie.name/ramiro_stanton,,43.202.192.104,"{""location"": ""SD"", ""is_mobile"": true}" 4747,7,448,2017-03-14 07:58:54,http://hyatt.org/jordyn,,130.231.197.171,"{""location"": ""RE"", ""is_mobile"": false}" 4748,7,448,2017-04-20 16:38:25,http://pfeffer.info/adelbert.jacobson,,140.29.249.132,"{""location"": ""AI"", ""is_mobile"": true}" 4749,7,448,2017-06-03 00:38:40,http://hahn.co/bertha,,196.213.66.158,"{""location"": ""AL"", ""is_mobile"": false}" 4750,7,448,2017-02-25 17:04:44,http://towne.info/daisha,,17.32.209.216,"{""location"": ""LS"", ""is_mobile"": false}" 4751,7,448,2017-05-08 18:33:31,http://gibson.net/quinn,,10.200.193.16,"{""location"": ""MK"", ""is_mobile"": false}" 4752,7,448,2017-03-20 03:03:02,http://jakubowskiwilliamson.name/zelda.mayert,,4.71.68.76,"{""location"": ""GS"", ""is_mobile"": true}" 4753,7,449,2017-02-11 16:04:15,http://ernser.info/orlo_boyle,,20.238.97.96,"{""location"": ""NG"", ""is_mobile"": false}" 4754,7,449,2017-03-28 06:06:45,http://bauch.org/guadalupe,,27.14.235.66,"{""location"": ""FI"", ""is_mobile"": false}" 4755,7,449,2017-02-14 20:04:07,http://kozey.info/tyler_borer,,247.14.78.42,"{""location"": ""PG"", ""is_mobile"": true}" 4756,7,450,2017-02-21 09:40:07,http://hoeger.co/harley,,206.193.41.44,"{""location"": ""CD"", ""is_mobile"": true}" 4757,7,450,2017-01-08 17:30:31,http://doyleokeefe.co/burdette_funk,,238.110.174.75,"{""location"": ""SA"", ""is_mobile"": true}" 4758,7,451,2017-06-07 05:19:05,http://johnson.name/keaton,,251.200.138.3,"{""location"": ""SV"", ""is_mobile"": false}" 4759,7,451,2016-12-28 19:57:27,http://bruen.org/adrienne,,177.169.138.211,"{""location"": ""ME"", ""is_mobile"": true}" 4760,7,451,2017-03-16 19:33:43,http://runolfsdottir.io/nella.batz,,110.40.212.149,"{""location"": ""BD"", ""is_mobile"": false}" 4761,7,452,2017-01-09 04:52:47,http://kling.org/ardella,,242.38.184.102,"{""location"": ""FJ"", ""is_mobile"": true}" 4762,7,452,2017-06-10 12:03:28,http://kihnbruen.info/cleora.lockman,,17.107.66.137,"{""location"": ""BQ"", ""is_mobile"": true}" 4763,7,452,2017-06-06 20:13:21,http://mayer.io/deborah,,246.130.252.188,"{""location"": ""RO"", ""is_mobile"": true}" 4764,7,452,2017-04-27 11:00:20,http://prohaska.com/sophia,,228.241.195.148,"{""location"": ""GG"", ""is_mobile"": false}" 4765,7,452,2017-02-01 10:56:09,http://armstrongterry.net/louisa,,212.155.250.138,"{""location"": ""MG"", ""is_mobile"": false}" 4766,7,452,2017-04-06 03:13:51,http://dibbertdavis.net/horace_hodkiewicz,,251.144.218.67,"{""location"": ""LR"", ""is_mobile"": true}" 4767,7,452,2017-02-06 10:02:33,http://bergstrom.net/gunner.schumm,,25.70.8.162,"{""location"": ""SV"", ""is_mobile"": true}" 4768,7,452,2017-02-11 05:11:32,http://hansenorn.name/je.stracke,,228.241.195.148,"{""location"": ""GF"", ""is_mobile"": false}" 4769,7,452,2017-04-26 22:38:30,http://bahringer.io/duncan_greenfelder,,242.38.184.102,"{""location"": ""PL"", ""is_mobile"": true}" 4770,7,452,2017-04-09 18:31:44,http://christiansenmonahan.io/newell,,209.84.251.163,"{""location"": ""VE"", ""is_mobile"": true}" 4771,7,452,2017-02-03 14:09:03,http://abshire.net/norwood.altenwerth,,63.215.204.217,"{""location"": ""CF"", ""is_mobile"": true}" 4772,7,452,2017-03-08 20:37:48,http://kub.info/abbey.thompson,,52.136.243.56,"{""location"": ""AZ"", ""is_mobile"": false}" 4773,7,452,2017-04-26 22:29:24,http://kshlerin.org/jadon.homenick,,17.107.66.137,"{""location"": ""MZ"", ""is_mobile"": true}" 4774,7,452,2017-01-04 12:11:53,http://reichertgleason.io/madelyn.welch,,231.134.58.105,"{""location"": ""AO"", ""is_mobile"": false}" 4775,7,452,2017-06-10 01:19:26,http://sporer.co/tyreek.graham,,144.8.152.117,"{""location"": ""ST"", ""is_mobile"": false}" 4776,7,453,2017-03-18 02:24:27,http://quitzonsteuber.com/mckenzie.cruickshank,,87.60.182.218,"{""location"": ""ZW"", ""is_mobile"": false}" 4777,7,453,2017-02-08 11:56:41,http://abernathy.co/roman,,124.212.45.22,"{""location"": ""EG"", ""is_mobile"": true}" 4778,7,453,2017-02-15 10:05:27,http://skiles.net/keon,,222.245.175.181,"{""location"": ""RS"", ""is_mobile"": false}" 4779,7,453,2016-12-31 22:25:48,http://gleichner.com/johnathan,,37.16.219.77,"{""location"": ""PY"", ""is_mobile"": true}" 4780,7,453,2017-03-28 17:01:42,http://okeefe.info/nico,,87.60.182.218,"{""location"": ""NA"", ""is_mobile"": false}" 4781,7,453,2017-01-19 05:03:30,http://ledner.net/carole,,161.113.123.39,"{""location"": ""PN"", ""is_mobile"": true}" 4782,7,453,2017-05-20 15:02:09,http://daniel.co/valentin,,175.172.25.172,"{""location"": ""OM"", ""is_mobile"": false}" 4783,7,453,2017-03-13 13:16:46,http://runolfsdottir.co/roy,,103.237.246.95,"{""location"": ""PF"", ""is_mobile"": false}" 4784,7,453,2017-05-28 07:59:07,http://stiedemann.io/cletus,,175.172.25.172,"{""location"": ""MR"", ""is_mobile"": true}" 4785,7,453,2017-02-04 00:45:02,http://treutel.org/elenora,,161.113.123.39,"{""location"": ""TG"", ""is_mobile"": false}" 4786,7,454,2017-03-06 23:19:12,http://tromp.com/eve,,62.192.181.112,"{""location"": ""KN"", ""is_mobile"": false}" 4787,7,454,2017-03-04 01:39:32,http://kuhicgoyette.name/jerrold,,154.222.232.66,"{""location"": ""LA"", ""is_mobile"": true}" 4788,7,454,2017-05-15 04:22:04,http://stracke.org/teagan,,87.161.181.70,"{""location"": ""HN"", ""is_mobile"": true}" 4789,7,454,2017-04-01 05:47:09,http://millerstiedemann.org/wellington,,175.23.149.200,"{""location"": ""DO"", ""is_mobile"": true}" 4790,7,455,2016-12-20 02:39:51,http://williamson.info/octavia.sanford,,121.6.191.84,"{""location"": ""GU"", ""is_mobile"": true}" 4791,7,455,2016-12-15 03:59:36,http://williamson.co/selena,,71.153.117.200,"{""location"": ""KE"", ""is_mobile"": true}" 4792,7,455,2017-02-13 10:43:54,http://mclaughlin.info/bridget_oreilly,,29.252.55.236,"{""location"": ""PG"", ""is_mobile"": false}" 4793,7,455,2016-12-17 06:49:33,http://crona.name/hildegard,,189.198.97.19,"{""location"": ""SX"", ""is_mobile"": true}" 4794,7,455,2017-02-16 03:05:46,http://block.biz/jany.runolfon,,92.37.137.230,"{""location"": ""HT"", ""is_mobile"": true}" 4795,7,455,2017-01-28 19:21:46,http://fahey.org/brandy,,216.236.51.133,"{""location"": ""LB"", ""is_mobile"": true}" 4796,7,455,2017-01-19 19:22:01,http://abernathy.co/glennie,,126.141.127.72,"{""location"": ""IL"", ""is_mobile"": false}" 4797,7,455,2017-05-16 11:14:35,http://predoviccollier.name/noelia_mcclure,,221.75.173.160,"{""location"": ""UZ"", ""is_mobile"": false}" 4798,7,455,2017-02-04 16:11:24,http://feest.com/alicia_bashirian,,222.76.46.163,"{""location"": ""FM"", ""is_mobile"": false}" 4799,7,455,2017-01-22 02:19:49,http://johnsongislason.net/geovany.beier,,71.153.117.200,"{""location"": ""MW"", ""is_mobile"": false}" 4800,7,455,2017-01-25 21:50:42,http://fadel.org/maverick.schmidt,,70.163.75.129,"{""location"": ""MK"", ""is_mobile"": false}" 2080,3,194,2017-05-22 18:09:27,http://gutkowskitremblay.com/filomena,0.0000000000,41.13.228.96,"{""location"": ""DJ"", ""is_mobile"": false}" 2081,3,194,2017-01-26 05:00:48,http://spencer.co/paige.kirlin,0.0000000000,88.26.98.128,"{""location"": ""US"", ""is_mobile"": true}" 2082,3,194,2017-03-17 18:06:59,http://sauer.name/richie.kemmer,0.0000000000,172.250.191.101,"{""location"": ""VC"", ""is_mobile"": true}" 2083,3,194,2017-01-26 21:25:08,http://kemmer.info/eloisa_feest,0.0000000000,4.51.96.198,"{""location"": ""BG"", ""is_mobile"": true}" 4801,7,455,2017-02-12 20:32:43,http://tremblay.biz/beth.yundt,,92.37.137.230,"{""location"": ""CM"", ""is_mobile"": true}" 4802,7,455,2017-04-07 05:14:14,http://rolfson.co/devonte,,17.104.175.12,"{""location"": ""FM"", ""is_mobile"": true}" 4803,7,455,2017-06-11 19:00:45,http://williamson.biz/ransom_fritsch,,228.14.192.215,"{""location"": ""GL"", ""is_mobile"": true}" 4804,7,455,2017-03-23 15:04:26,http://ratke.org/dewayne,,186.117.211.248,"{""location"": ""LC"", ""is_mobile"": true}" 4805,7,455,2017-05-01 16:39:24,http://luettgen.co/janice_friesen,,178.161.41.37,"{""location"": ""MY"", ""is_mobile"": true}" 4806,7,455,2017-02-23 21:16:32,http://zboncakkoch.com/deja,,221.75.173.160,"{""location"": ""RW"", ""is_mobile"": false}" 4807,7,456,2017-04-18 16:57:57,http://cremin.net/hallie.marvin,,81.170.227.31,"{""location"": ""PL"", ""is_mobile"": true}" 4808,7,456,2017-03-02 08:50:07,http://bradtke.com/hope.homenick,,12.34.42.229,"{""location"": ""MZ"", ""is_mobile"": false}" 4809,7,456,2017-03-28 01:09:46,http://rodriguezemard.net/emmy,,86.21.175.7,"{""location"": ""IT"", ""is_mobile"": false}" 4810,7,456,2016-12-24 10:25:36,http://gaylordroberts.info/tristian,,141.156.97.23,"{""location"": ""NZ"", ""is_mobile"": false}" 4811,7,456,2017-02-01 02:15:13,http://conroy.net/heloise,,66.59.146.239,"{""location"": ""MA"", ""is_mobile"": true}" 4812,7,456,2017-03-10 21:17:22,http://trantow.net/selina_bernier,,11.27.240.148,"{""location"": ""GT"", ""is_mobile"": false}" 4813,7,456,2016-12-30 19:32:40,http://halvorson.net/chaya_armstrong,,141.186.254.129,"{""location"": ""LK"", ""is_mobile"": false}" 4814,7,457,2016-12-27 15:03:29,http://labadie.info/camille,,42.2.222.69,"{""location"": ""PH"", ""is_mobile"": false}" 4815,7,457,2017-03-27 03:15:02,http://pagac.net/dane,,162.65.240.229,"{""location"": ""TG"", ""is_mobile"": false}" 4816,7,457,2017-03-07 08:50:51,http://gorczanyrolfson.net/berniece_casper,,224.7.103.189,"{""location"": ""RU"", ""is_mobile"": true}" 4817,7,457,2017-01-17 11:18:00,http://waters.info/jarret.dicki,,229.54.234.30,"{""location"": ""UZ"", ""is_mobile"": true}" 4818,7,457,2017-03-02 18:27:12,http://schusterdach.io/abbie.pollich,,106.152.135.147,"{""location"": ""AT"", ""is_mobile"": true}" 4819,7,457,2017-05-30 20:07:52,http://gusikowski.biz/shayne.treutel,,119.75.219.19,"{""location"": ""LV"", ""is_mobile"": true}" 4820,7,457,2017-04-18 19:39:35,http://herzog.io/isidro,,208.82.75.160,"{""location"": ""SI"", ""is_mobile"": true}" 4821,7,457,2017-01-06 13:06:41,http://emard.name/bailee.bernier,,57.82.127.253,"{""location"": ""FR"", ""is_mobile"": false}" 4822,7,457,2017-03-30 01:03:55,http://ebertfriesen.info/maybell,,57.82.127.253,"{""location"": ""TJ"", ""is_mobile"": true}" 4823,7,458,2017-02-05 09:19:33,http://zemlak.io/mallory,,242.185.33.67,"{""location"": ""IN"", ""is_mobile"": true}" 4824,7,458,2017-04-08 11:23:18,http://stehr.com/etha,,16.250.65.85,"{""location"": ""MH"", ""is_mobile"": true}" 4825,7,458,2017-03-01 20:05:19,http://weimann.org/katelynn_balistreri,,179.156.11.32,"{""location"": ""AG"", ""is_mobile"": false}" 4826,7,458,2017-01-17 20:40:53,http://sengerboyer.net/jan,,185.103.248.77,"{""location"": ""NA"", ""is_mobile"": true}" 4827,7,458,2017-04-02 16:41:24,http://bruen.org/jeramie,,12.26.105.154,"{""location"": ""SJ"", ""is_mobile"": false}" 4828,7,458,2017-04-28 13:13:04,http://croninraynor.com/quinton_stiedemann,,64.57.56.21,"{""location"": ""AZ"", ""is_mobile"": true}" 4829,7,458,2017-03-23 21:15:16,http://gerlach.name/jermain.fisher,,64.57.56.21,"{""location"": ""BZ"", ""is_mobile"": false}" 4830,7,459,2017-03-29 08:05:36,http://predovic.org/jee_daniel,,65.209.63.246,"{""location"": ""LK"", ""is_mobile"": false}" 4831,7,459,2017-04-29 07:42:17,http://langoshkoelpin.org/aron,,107.125.48.86,"{""location"": ""KZ"", ""is_mobile"": false}" 4832,7,459,2017-01-08 11:31:23,http://lubowitzbeier.name/emelia,,224.39.114.31,"{""location"": ""BI"", ""is_mobile"": false}" 4833,7,459,2017-03-11 05:55:53,http://kirlin.io/delta,,69.20.76.124,"{""location"": ""KZ"", ""is_mobile"": false}" 4834,7,459,2017-06-07 04:15:50,http://ondrickaharris.com/everette,,160.154.13.144,"{""location"": ""GN"", ""is_mobile"": false}" 4835,7,459,2017-04-06 21:28:54,http://feiladams.biz/lila,,6.104.73.53,"{""location"": ""BI"", ""is_mobile"": true}" 4836,7,459,2017-01-13 16:58:31,http://marvin.biz/alf_spencer,,212.226.166.175,"{""location"": ""NI"", ""is_mobile"": true}" 4837,7,459,2017-05-02 13:05:36,http://gislasonleuschke.io/alek.bernhard,,32.3.190.19,"{""location"": ""AR"", ""is_mobile"": false}" 4838,7,459,2017-03-05 13:38:39,http://reichert.info/orland,,78.212.245.56,"{""location"": ""KY"", ""is_mobile"": true}" 4839,7,460,2017-05-24 22:47:31,http://quigley.io/myron,,236.52.166.112,"{""location"": ""IL"", ""is_mobile"": true}" 4840,7,460,2017-01-13 02:29:23,http://hilll.net/xander_wiza,,114.18.166.142,"{""location"": ""AT"", ""is_mobile"": false}" 4841,7,460,2017-05-01 18:14:52,http://bartellpagac.org/jed,,170.22.199.211,"{""location"": ""KY"", ""is_mobile"": true}" 4842,7,461,2017-02-20 17:05:31,http://pacocha.info/chelsea.fritsch,,153.126.193.51,"{""location"": ""AO"", ""is_mobile"": false}" 4843,7,461,2017-04-17 12:22:53,http://koelpin.name/tre_jacobson,,214.143.167.215,"{""location"": ""HM"", ""is_mobile"": false}" 4844,7,461,2017-05-21 21:55:53,http://blandarice.org/elbert,,148.191.58.228,"{""location"": ""GA"", ""is_mobile"": false}" 4845,7,461,2017-03-19 05:29:08,http://daugherty.io/lexie,,136.220.178.41,"{""location"": ""GA"", ""is_mobile"": true}" 4846,7,461,2017-02-10 08:05:28,http://kunze.io/brannon.daniel,,5.98.143.76,"{""location"": ""CV"", ""is_mobile"": false}" 4847,7,461,2017-06-08 00:28:00,http://gleichner.name/brionna.smith,,244.219.231.195,"{""location"": ""KZ"", ""is_mobile"": false}" 4848,7,461,2017-01-29 14:55:54,http://powlowskileffler.co/gerda,,205.73.69.146,"{""location"": ""RU"", ""is_mobile"": true}" 4849,7,461,2017-01-09 09:01:31,http://ebert.net/ed_shields,,206.57.162.9,"{""location"": ""TM"", ""is_mobile"": false}" 4850,7,461,2017-02-25 09:27:48,http://quigley.net/ozzie.dickinson,,214.196.181.238,"{""location"": ""TZ"", ""is_mobile"": false}" 4851,7,461,2017-04-27 15:37:59,http://ruel.co/valentin,,131.155.122.121,"{""location"": ""IE"", ""is_mobile"": true}" 4852,7,461,2016-12-19 12:56:30,http://tillman.io/leie.bergstrom,,113.231.99.143,"{""location"": ""AS"", ""is_mobile"": true}" 4853,7,461,2017-05-20 05:52:04,http://kelercormier.biz/marguerite_schowalter,,148.191.58.228,"{""location"": ""BN"", ""is_mobile"": true}" 4854,7,461,2017-03-22 04:37:44,http://heaney.net/otto,,219.117.166.241,"{""location"": ""PS"", ""is_mobile"": false}" 4855,7,461,2017-01-03 11:02:43,http://romaguera.io/vida_green,,231.204.2.21,"{""location"": ""KR"", ""is_mobile"": false}" 4856,7,461,2017-02-16 13:33:01,http://purdy.info/alverta.schaefer,,223.66.84.155,"{""location"": ""ME"", ""is_mobile"": false}" 4857,7,461,2017-01-15 17:47:53,http://mcdermott.io/margarete,,105.210.148.25,"{""location"": ""GG"", ""is_mobile"": true}" 4858,7,461,2017-01-27 04:10:46,http://rice.io/jameson,,247.157.7.247,"{""location"": ""NG"", ""is_mobile"": false}" 4859,7,461,2017-06-04 08:58:51,http://steuber.io/paul.bode,,78.77.13.13,"{""location"": ""GU"", ""is_mobile"": false}" 4860,7,461,2017-01-25 02:04:56,http://lesch.com/andreanne,,246.228.50.2,"{""location"": ""ID"", ""is_mobile"": false}" 4861,7,462,2017-04-11 10:15:13,http://schimmelbuckridge.org/cecil.hartmann,,123.63.148.123,"{""location"": ""VA"", ""is_mobile"": true}" 4862,7,462,2017-01-09 12:22:36,http://abernathyerdman.name/lennie,,33.154.234.164,"{""location"": ""CU"", ""is_mobile"": false}" 4863,7,462,2017-04-05 12:35:13,http://feest.com/lennie_gerlach,,240.20.123.253,"{""location"": ""MW"", ""is_mobile"": true}" 4864,7,462,2017-05-31 21:27:38,http://keeling.name/emie,,132.219.58.211,"{""location"": ""SJ"", ""is_mobile"": true}" 4865,7,463,2017-04-23 17:49:50,http://weimannvon.info/will,,199.71.170.182,"{""location"": ""GG"", ""is_mobile"": false}" 4866,7,463,2017-01-05 23:17:01,http://medhurst.biz/lloyd,,75.185.73.111,"{""location"": ""CI"", ""is_mobile"": false}" 4867,7,463,2017-05-14 11:42:47,http://rippintreutel.info/lenny,,104.8.119.230,"{""location"": ""FR"", ""is_mobile"": false}" 4868,7,463,2017-01-09 00:43:07,http://marvin.biz/iliana.blanda,,126.188.215.106,"{""location"": ""MT"", ""is_mobile"": false}" 4869,7,463,2017-02-05 15:41:34,http://dicki.net/lucile_reichel,,86.142.225.39,"{""location"": ""TC"", ""is_mobile"": true}" 4870,7,463,2017-01-07 12:21:41,http://carrolldooley.org/elnora_jenkins,,140.163.173.211,"{""location"": ""RW"", ""is_mobile"": true}" 4871,7,463,2017-01-23 04:48:08,http://farrellmcdermott.com/pauline,,41.132.15.179,"{""location"": ""LV"", ""is_mobile"": true}" 4872,7,463,2017-06-13 16:23:19,http://durgan.org/susanna,,188.46.162.14,"{""location"": ""IM"", ""is_mobile"": true}" 4873,7,463,2017-05-16 03:10:08,http://bahringer.co/brittany,,135.164.191.148,"{""location"": ""AW"", ""is_mobile"": false}" 4874,7,463,2017-05-11 05:24:26,http://oberbrunner.co/clovis,,98.189.175.63,"{""location"": ""GH"", ""is_mobile"": true}" 4875,7,463,2017-05-10 18:39:30,http://white.net/nayeli,,199.71.170.182,"{""location"": ""LB"", ""is_mobile"": false}" 4876,7,463,2016-12-25 17:54:45,http://sporer.co/triston_mohr,,4.87.27.183,"{""location"": ""EC"", ""is_mobile"": true}" 4877,7,463,2017-01-17 17:14:50,http://reichert.co/reba.white,,245.54.159.66,"{""location"": ""WF"", ""is_mobile"": false}" 4878,7,463,2017-04-20 21:04:32,http://nicolas.name/karli,,208.66.83.212,"{""location"": ""AS"", ""is_mobile"": true}" 4879,7,463,2017-05-28 10:38:16,http://mitchell.net/pete,,179.19.116.47,"{""location"": ""AF"", ""is_mobile"": true}" 4880,7,463,2017-02-09 12:27:28,http://ebertrice.name/armando.little,,164.185.161.38,"{""location"": ""PG"", ""is_mobile"": false}" 4881,7,463,2017-04-28 09:12:55,http://toykerluke.org/lisa,,104.8.119.230,"{""location"": ""UA"", ""is_mobile"": true}" 4882,7,463,2017-04-27 07:50:03,http://sporer.io/jennyfer,,55.46.46.200,"{""location"": ""LB"", ""is_mobile"": true}" 4883,7,463,2017-01-20 23:46:59,http://bruen.name/ena,,190.111.112.148,"{""location"": ""BV"", ""is_mobile"": true}" 4884,7,463,2017-06-12 22:23:54,http://jonesblanda.org/georgette,,228.194.69.150,"{""location"": ""CF"", ""is_mobile"": true}" 4885,7,464,2017-01-31 22:26:08,http://harvey.co/opal,,99.14.172.56,"{""location"": ""LC"", ""is_mobile"": true}" 4886,7,464,2017-04-21 12:23:45,http://ratkekling.biz/judge,,180.189.118.154,"{""location"": ""SG"", ""is_mobile"": false}" 4887,7,464,2017-05-30 18:51:40,http://becker.org/dora,,182.141.153.61,"{""location"": ""NU"", ""is_mobile"": false}" 4888,7,464,2017-01-19 21:21:57,http://waelchi.info/theresia_kerluke,,197.180.183.41,"{""location"": ""SJ"", ""is_mobile"": false}" 4889,7,464,2017-02-14 18:27:32,http://kuhic.co/ima_haley,,177.38.12.91,"{""location"": ""ME"", ""is_mobile"": true}" 4890,7,464,2017-03-10 19:37:43,http://hermanluettgen.io/sammy.gibson,,180.189.118.154,"{""location"": ""TR"", ""is_mobile"": false}" 4891,7,464,2017-05-27 17:07:52,http://collins.org/clifford.tremblay,,112.163.158.236,"{""location"": ""CX"", ""is_mobile"": true}" 4892,7,464,2017-01-20 18:34:21,http://strosincollier.name/lucio.pacocha,,133.193.173.210,"{""location"": ""SG"", ""is_mobile"": false}" 4893,7,465,2017-03-26 19:38:41,http://hahnpagac.org/destiney.greenholt,,117.251.108.222,"{""location"": ""VN"", ""is_mobile"": true}" 4894,7,465,2017-04-18 22:36:26,http://leannon.name/fae.hirthe,,49.50.184.57,"{""location"": ""AW"", ""is_mobile"": true}" 4895,7,465,2017-02-07 18:22:41,http://gutmannosinski.name/finn,,218.55.94.78,"{""location"": ""IM"", ""is_mobile"": false}" 4896,7,465,2017-06-04 23:04:12,http://lednerchamplin.co/marguerite,,194.51.59.247,"{""location"": ""RO"", ""is_mobile"": true}" 4897,7,465,2017-06-03 22:46:23,http://bartoletti.io/leopoldo_durgan,,49.50.184.57,"{""location"": ""GR"", ""is_mobile"": false}" 4898,7,465,2017-04-13 06:46:38,http://emmerich.org/alize,,208.173.21.118,"{""location"": ""HR"", ""is_mobile"": true}" 4899,7,465,2017-04-14 11:43:16,http://runolfon.biz/andrew.fritsch,,157.89.200.132,"{""location"": ""IS"", ""is_mobile"": true}" 4900,7,465,2017-04-03 00:08:36,http://schimmel.io/oma,,157.37.46.11,"{""location"": ""JO"", ""is_mobile"": false}" 4901,7,465,2017-02-04 20:41:22,http://ruelosinski.net/ona,,157.37.46.11,"{""location"": ""OM"", ""is_mobile"": true}" 4902,7,465,2017-01-22 18:03:14,http://schneidermarvin.io/quentin.kulas,,55.117.225.44,"{""location"": ""FI"", ""is_mobile"": true}" 4903,7,466,2017-01-01 11:50:51,http://ankundinglabadie.info/domenic.jones,1.0000000000,3.201.213.83,"{""location"": ""MC"", ""is_mobile"": false}" 4904,7,466,2017-03-11 12:53:46,http://goyette.com/antonetta,3.0000000000,211.39.235.248,"{""location"": ""DO"", ""is_mobile"": true}" 4905,7,466,2017-01-15 21:33:14,http://connelly.com/ashton,1.0000000000,182.163.33.68,"{""location"": ""DO"", ""is_mobile"": true}" 4906,7,466,2017-05-17 07:49:39,http://keler.net/savanah.kohler,0.0000000000,166.190.83.80,"{""location"": ""HT"", ""is_mobile"": true}" 4907,7,467,2017-02-25 12:18:07,http://goldner.com/oma,1.0000000000,26.165.44.241,"{""location"": ""PM"", ""is_mobile"": false}" 4908,7,467,2017-02-26 08:07:41,http://stracke.com/jaydon.thiel,1.0000000000,105.225.97.91,"{""location"": ""SI"", ""is_mobile"": false}" 4909,7,467,2017-01-05 21:22:36,http://corwinberge.com/jalyn,1.0000000000,117.61.134.112,"{""location"": ""CF"", ""is_mobile"": true}" 4910,7,467,2017-02-28 11:40:18,http://schiller.net/emely_schneider,0.0000000000,124.148.69.25,"{""location"": ""SK"", ""is_mobile"": true}" 4911,7,467,2017-01-07 07:32:57,http://turcotte.name/dylan_glover,1.0000000000,117.253.51.74,"{""location"": ""CU"", ""is_mobile"": false}" 4912,7,468,2017-01-19 00:47:30,http://bauch.net/tanner,0.0000000000,210.55.219.187,"{""location"": ""RS"", ""is_mobile"": false}" 4913,7,468,2017-03-26 07:15:35,http://ebert.co/teagan,0.0000000000,113.191.15.208,"{""location"": ""CH"", ""is_mobile"": false}" 4914,7,468,2017-05-30 00:03:55,http://wehner.name/helga_cole,1.0000000000,26.218.115.48,"{""location"": ""SZ"", ""is_mobile"": false}" 4915,7,468,2017-04-28 02:40:28,http://hilll.info/ciara.glover,1.0000000000,37.207.254.43,"{""location"": ""CW"", ""is_mobile"": true}" 4916,7,468,2017-02-28 09:55:09,http://yostraynor.biz/harley.ryan,2.0000000000,37.207.254.43,"{""location"": ""TO"", ""is_mobile"": true}" 4917,7,468,2017-02-18 13:04:54,http://haag.net/julianne.harvey,0.0000000000,222.152.6.148,"{""location"": ""MT"", ""is_mobile"": false}" 4918,7,468,2017-02-23 06:20:26,http://lang.co/emie.doyle,0.0000000000,175.55.148.101,"{""location"": ""RO"", ""is_mobile"": true}" 4919,7,469,2017-01-27 16:47:57,http://anderson.name/bennett_koch,0.0000000000,39.135.41.76,"{""location"": ""RO"", ""is_mobile"": true}" 4920,7,469,2017-06-12 13:49:30,http://satterfield.com/samanta.stokes,0.0000000000,208.183.68.105,"{""location"": ""CF"", ""is_mobile"": false}" 4921,7,469,2017-03-13 19:13:22,http://mcglynn.io/tremayne,0.0000000000,193.143.183.124,"{""location"": ""MH"", ""is_mobile"": false}" 4922,7,469,2017-03-19 18:30:04,http://hyatt.biz/annamarie_okeefe,0.0000000000,193.143.183.124,"{""location"": ""MR"", ""is_mobile"": true}" 4923,7,469,2017-05-13 23:24:31,http://lind.net/shakira,0.0000000000,161.67.78.175,"{""location"": ""RS"", ""is_mobile"": false}" 4924,7,470,2017-02-13 12:46:25,http://toy.co/sydni_emard,2.0000000000,203.9.207.79,"{""location"": ""PN"", ""is_mobile"": false}" 4925,7,470,2017-01-27 09:56:15,http://boylelang.name/steve.gusikowski,2.0000000000,203.9.207.79,"{""location"": ""VU"", ""is_mobile"": true}" 4926,7,470,2017-01-27 18:40:04,http://powlowski.co/shany,1.0000000000,230.81.165.195,"{""location"": ""TT"", ""is_mobile"": false}" 4927,7,470,2017-03-04 06:45:13,http://lockman.org/hazle,3.0000000000,211.17.130.70,"{""location"": ""UA"", ""is_mobile"": false}" 4928,7,470,2017-05-29 03:25:17,http://croninkoch.io/reymundo_fritsch,1.0000000000,225.130.193.102,"{""location"": ""SV"", ""is_mobile"": true}" 4929,7,470,2017-05-11 14:57:01,http://smithamrempel.co/tobin.witting,3.0000000000,211.17.130.70,"{""location"": ""MC"", ""is_mobile"": true}" 4930,7,470,2017-04-25 18:22:17,http://aufderhar.info/aileen,2.0000000000,190.109.89.48,"{""location"": ""NU"", ""is_mobile"": true}" 4931,7,470,2017-05-01 22:27:16,http://wilkinson.com/chanel_sawayn,0.0000000000,106.32.236.177,"{""location"": ""SS"", ""is_mobile"": true}" 4932,7,470,2017-04-14 05:06:45,http://champlin.net/lea,1.0000000000,241.161.167.245,"{""location"": ""SG"", ""is_mobile"": true}" 4933,7,470,2017-04-19 09:09:03,http://rogahnhickle.info/garnett_walter,2.0000000000,150.56.155.125,"{""location"": ""MH"", ""is_mobile"": true}" 4934,7,470,2017-03-19 05:51:01,http://konopelskiullrich.co/gerry_braun,2.0000000000,150.56.155.125,"{""location"": ""CG"", ""is_mobile"": true}" 4935,7,470,2017-04-29 16:37:38,http://parker.org/jamie,3.0000000000,86.74.112.248,"{""location"": ""ZW"", ""is_mobile"": true}" 4936,7,470,2017-02-25 09:25:15,http://hahnwindler.org/zachary_skiles,0.0000000000,199.106.89.147,"{""location"": ""MV"", ""is_mobile"": false}" 4937,7,470,2017-05-24 13:33:59,http://grantbosco.org/maryse,1.0000000000,94.115.223.78,"{""location"": ""ES"", ""is_mobile"": true}" 4938,7,470,2017-05-28 05:03:34,http://stanton.co/frederic.krajcik,3.0000000000,225.141.8.61,"{""location"": ""DZ"", ""is_mobile"": false}" 4939,7,470,2017-03-31 02:04:53,http://blick.biz/river,3.0000000000,3.78.72.78,"{""location"": ""TK"", ""is_mobile"": false}" 4940,7,470,2017-02-15 11:32:27,http://bartell.info/eudora.jacobi,0.0000000000,203.9.207.79,"{""location"": ""FM"", ""is_mobile"": true}" 4941,7,470,2017-06-05 09:40:08,http://gusikowski.com/cali,3.0000000000,63.123.174.144,"{""location"": ""TR"", ""is_mobile"": true}" 4942,7,471,2017-04-29 23:56:33,http://douglasullrich.net/lila_schowalter,0.0000000000,24.223.19.152,"{""location"": ""ES"", ""is_mobile"": true}" 4943,7,471,2017-03-09 01:14:43,http://shieldsyundt.biz/georgiana,0.0000000000,224.24.234.227,"{""location"": ""ER"", ""is_mobile"": true}" 4944,7,471,2017-03-07 03:34:34,http://schneider.org/eduardo.lehner,0.0000000000,53.49.185.125,"{""location"": ""BZ"", ""is_mobile"": false}" 4945,7,471,2017-05-30 12:41:52,http://treutelanderson.io/royce,2.0000000000,158.11.249.172,"{""location"": ""RO"", ""is_mobile"": true}" 4946,7,471,2017-01-12 01:21:11,http://dickinson.net/christine.ruel,2.0000000000,205.197.206.145,"{""location"": ""FM"", ""is_mobile"": false}" 4947,7,471,2017-02-12 09:41:52,http://gaylordleannon.biz/alanna.koepp,2.0000000000,135.176.50.96,"{""location"": ""MW"", ""is_mobile"": true}" 4948,7,471,2017-05-10 00:17:14,http://schillerquigley.co/hallie.walker,2.0000000000,158.11.249.172,"{""location"": ""BN"", ""is_mobile"": true}" 4949,7,471,2017-02-03 16:29:13,http://hageneskuhic.net/rosina,0.0000000000,236.116.160.221,"{""location"": ""SE"", ""is_mobile"": false}" 4950,7,471,2016-12-29 02:28:42,http://mcdermott.com/helen,0.0000000000,135.176.50.96,"{""location"": ""HM"", ""is_mobile"": false}" 4951,7,471,2017-02-13 12:09:43,http://altenwerth.com/marlee,2.0000000000,162.213.235.212,"{""location"": ""PL"", ""is_mobile"": true}" 4952,7,471,2017-02-04 04:39:53,http://daniel.com/quincy,2.0000000000,94.221.83.143,"{""location"": ""YE"", ""is_mobile"": false}" 4953,7,471,2017-02-17 18:05:21,http://smith.info/krystal_jakubowski,1.0000000000,150.181.135.232,"{""location"": ""PT"", ""is_mobile"": true}" 4954,7,471,2017-03-09 18:08:50,http://denesikcartwright.info/frederik_blick,0.0000000000,162.213.235.212,"{""location"": ""ES"", ""is_mobile"": false}" 4955,7,471,2017-01-21 23:35:47,http://parker.biz/aleandro,0.0000000000,4.34.144.221,"{""location"": ""MM"", ""is_mobile"": false}" 4956,7,471,2017-01-14 00:13:58,http://block.info/mohammed.greenfelder,2.0000000000,71.69.169.54,"{""location"": ""KY"", ""is_mobile"": false}" 4957,7,471,2017-01-09 12:49:08,http://torp.net/skyla_padberg,2.0000000000,170.49.130.156,"{""location"": ""BQ"", ""is_mobile"": true}" 4958,7,471,2017-01-07 17:08:15,http://ziemelubowitz.net/chase.farrell,0.0000000000,162.213.235.212,"{""location"": ""PM"", ""is_mobile"": true}" 4959,7,471,2016-12-29 23:20:49,http://boehm.info/patrick,2.0000000000,24.223.19.152,"{""location"": ""QA"", ""is_mobile"": false}" 4960,7,471,2017-06-11 12:21:40,http://satterfield.name/caleigh.schinner,2.0000000000,128.14.20.206,"{""location"": ""ZW"", ""is_mobile"": false}" 4961,7,471,2017-04-14 20:49:00,http://sporerkozey.info/jacquelyn.hammes,1.0000000000,147.119.70.50,"{""location"": ""YT"", ""is_mobile"": false}" 4962,7,472,2017-02-26 11:16:41,http://okonherzog.org/arianna,1.0000000000,159.99.103.146,"{""location"": ""AR"", ""is_mobile"": false}" 4963,7,472,2017-03-03 10:30:05,http://aufderhar.info/adonis_okuneva,0.0000000000,110.213.146.44,"{""location"": ""WF"", ""is_mobile"": false}" 4964,7,472,2017-01-21 18:19:49,http://smitham.info/jaydon,1.0000000000,181.59.190.173,"{""location"": ""KN"", ""is_mobile"": true}" 4965,7,472,2017-03-07 19:04:42,http://beier.org/angelo,0.0000000000,207.83.215.249,"{""location"": ""CA"", ""is_mobile"": false}" 4966,7,472,2017-02-25 17:19:17,http://heathcote.org/rigoberto_bauch,1.0000000000,181.59.190.173,"{""location"": ""LC"", ""is_mobile"": true}" 4967,7,472,2017-04-20 16:20:42,http://williamson.name/stanley_bode,1.0000000000,235.196.94.164,"{""location"": ""MQ"", ""is_mobile"": false}" 4968,7,472,2017-05-22 03:52:59,http://sawayn.com/cyril_glover,1.0000000000,46.186.187.169,"{""location"": ""TZ"", ""is_mobile"": true}" 4969,7,472,2017-06-03 21:15:36,http://walkerjaskolski.name/vella,1.0000000000,28.118.221.96,"{""location"": ""SN"", ""is_mobile"": true}" 4970,7,472,2017-04-20 06:41:14,http://koelpin.info/mara,0.0000000000,149.55.233.193,"{""location"": ""LI"", ""is_mobile"": true}" 4971,7,472,2017-04-29 20:45:36,http://wymanfritsch.name/victor.baumbach,0.0000000000,46.186.187.169,"{""location"": ""MC"", ""is_mobile"": false}" 4972,7,472,2017-02-28 19:07:14,http://harvey.net/andreanne_macgyver,0.0000000000,152.60.42.238,"{""location"": ""NF"", ""is_mobile"": false}" 4973,7,472,2017-05-09 08:36:58,http://nicolas.name/delbert.bogisich,0.0000000000,81.37.251.142,"{""location"": ""SN"", ""is_mobile"": false}" 4974,7,472,2017-05-06 12:52:25,http://gerhold.biz/meredith,1.0000000000,89.34.204.152,"{""location"": ""KM"", ""is_mobile"": true}" 4975,7,473,2016-12-27 20:49:39,http://hettinger.io/graham.jacobi,0.0000000000,148.164.200.48,"{""location"": ""SN"", ""is_mobile"": true}" 4976,7,473,2017-03-05 03:54:30,http://sanford.org/casimer.robel,1.0000000000,29.126.254.21,"{""location"": ""MG"", ""is_mobile"": false}" 4977,7,473,2017-01-31 22:13:45,http://denesik.biz/mary_casper,1.0000000000,154.153.219.22,"{""location"": ""KG"", ""is_mobile"": false}" 4978,7,473,2017-05-25 18:19:24,http://ledner.name/chaz_zulauf,1.0000000000,173.46.214.118,"{""location"": ""FJ"", ""is_mobile"": true}" 4979,7,473,2016-12-31 04:27:51,http://oconnellblanda.name/jeie_connelly,0.0000000000,137.151.68.82,"{""location"": ""IN"", ""is_mobile"": true}" 4980,7,473,2017-04-20 05:22:09,http://thompsonbahringer.info/reva,0.0000000000,55.34.238.71,"{""location"": ""YE"", ""is_mobile"": false}" 4981,7,473,2017-06-09 08:05:07,http://brown.org/porter_schmitt,0.0000000000,164.22.252.99,"{""location"": ""MG"", ""is_mobile"": false}" 4982,7,473,2017-01-16 22:02:15,http://leffler.biz/gardner_hickle,0.0000000000,46.164.229.23,"{""location"": ""TM"", ""is_mobile"": true}" 4983,7,473,2017-01-28 05:24:31,http://veum.org/wanda,1.0000000000,55.34.238.71,"{""location"": ""TJ"", ""is_mobile"": false}" 4984,7,473,2017-01-25 05:13:45,http://heathcote.net/colin_hayes,0.0000000000,29.126.254.21,"{""location"": ""DM"", ""is_mobile"": true}" 4985,7,473,2017-05-21 11:03:09,http://jerde.co/amara,0.0000000000,43.40.201.248,"{""location"": ""CU"", ""is_mobile"": true}" 4986,7,473,2017-02-07 04:42:40,http://feeney.name/christ_miller,0.0000000000,154.153.219.22,"{""location"": ""FO"", ""is_mobile"": true}" 4987,7,473,2017-03-17 15:59:37,http://turner.info/gunner,1.0000000000,38.136.185.67,"{""location"": ""AX"", ""is_mobile"": true}" 4988,7,473,2017-04-30 19:35:28,http://gislason.biz/earlene_murray,1.0000000000,129.252.140.176,"{""location"": ""GU"", ""is_mobile"": true}" 4989,7,473,2017-01-29 10:59:40,http://volkmandaugherty.org/anika_leuschke,1.0000000000,24.134.234.121,"{""location"": ""IL"", ""is_mobile"": false}" 4990,7,473,2017-06-13 17:57:37,http://powlowskihoeger.com/darrel,1.0000000000,173.46.214.118,"{""location"": ""US"", ""is_mobile"": true}" 4991,7,473,2017-01-25 13:35:52,http://mclaughlinparisian.org/santina.monahan,1.0000000000,99.131.147.82,"{""location"": ""LB"", ""is_mobile"": true}" 4992,7,473,2017-04-15 21:18:00,http://thompson.net/maude_beahan,1.0000000000,34.92.172.92,"{""location"": ""DE"", ""is_mobile"": true}" 4993,7,473,2017-01-13 19:45:20,http://dubuquewisoky.org/caleb,0.0000000000,38.136.185.67,"{""location"": ""CU"", ""is_mobile"": false}" 4994,7,473,2016-12-29 02:33:41,http://fisherjacobi.org/skyla,1.0000000000,88.240.104.13,"{""location"": ""NI"", ""is_mobile"": true}" 509,2,52,2017-03-26 01:47:37,http://macejkovicbernier.info/elena.terry,,136.114.18.117,"{""location"": ""BM"", ""is_mobile"": false}" 510,2,52,2016-12-30 06:10:02,http://gutmann.org/conor,,246.22.72.58,"{""location"": ""LK"", ""is_mobile"": false}" 511,2,52,2017-03-26 15:26:00,http://jacobi.name/shaylee,,40.51.228.228,"{""location"": ""TW"", ""is_mobile"": false}" 512,2,52,2017-01-01 07:06:21,http://funkcole.io/rahul_schiller,,243.24.5.206,"{""location"": ""JP"", ""is_mobile"": false}" 513,2,52,2017-01-31 12:48:12,http://lowebartoletti.io/telly,,18.168.220.134,"{""location"": ""GG"", ""is_mobile"": true}" 514,2,52,2017-05-05 19:47:25,http://gutkowski.biz/milo,,113.204.192.139,"{""location"": ""MW"", ""is_mobile"": true}" 515,2,52,2017-04-20 23:25:49,http://kshlerin.com/carmel_kulas,,40.51.228.228,"{""location"": ""FR"", ""is_mobile"": false}" 516,2,52,2017-02-23 21:47:18,http://keeling.io/lorenzo,,116.234.223.235,"{""location"": ""KR"", ""is_mobile"": true}" 517,2,53,2017-02-03 10:07:47,http://jenkins.net/karen_bradtke,,8.130.223.123,"{""location"": ""TL"", ""is_mobile"": true}" 518,2,53,2017-03-12 20:39:48,http://pfannerstill.net/bernhard.west,,128.163.186.212,"{""location"": ""TR"", ""is_mobile"": false}" 519,2,53,2017-05-22 11:19:37,http://dietrich.co/loyce,,155.100.133.85,"{""location"": ""MC"", ""is_mobile"": false}" 520,2,53,2017-05-02 22:54:30,http://langosh.name/zack,,154.155.250.88,"{""location"": ""ME"", ""is_mobile"": false}" 521,2,53,2017-03-09 02:22:55,http://lehner.name/reid,,167.185.68.138,"{""location"": ""IT"", ""is_mobile"": false}" 522,2,53,2017-04-09 23:36:15,http://cremin.io/elinore,,34.42.155.254,"{""location"": ""PH"", ""is_mobile"": true}" 523,2,53,2017-02-07 12:56:24,http://lind.io/orrin.gleason,,108.243.121.244,"{""location"": ""KG"", ""is_mobile"": true}" 524,2,53,2017-04-25 12:47:32,http://anderson.co/blaise,,124.95.214.23,"{""location"": ""LC"", ""is_mobile"": false}" 525,2,53,2017-06-08 10:39:51,http://stantonmurray.biz/maxwell,,212.119.250.114,"{""location"": ""HN"", ""is_mobile"": false}" 526,2,53,2017-04-14 01:04:21,http://jakubowski.info/sierra,,130.65.140.114,"{""location"": ""MY"", ""is_mobile"": true}" 527,2,53,2017-02-21 19:59:22,http://wolff.org/elmo_carroll,,158.195.56.201,"{""location"": ""IT"", ""is_mobile"": true}" 528,2,53,2016-12-19 04:06:34,http://stoltenbergshanahan.biz/tiffany_wiza,,158.195.56.201,"{""location"": ""GI"", ""is_mobile"": false}" 529,2,53,2017-03-25 11:44:14,http://treutelaltenwerth.info/wade,,60.234.145.107,"{""location"": ""TK"", ""is_mobile"": false}" 530,2,53,2016-12-29 09:30:56,http://kiehn.name/levi.braun,,141.27.31.37,"{""location"": ""BW"", ""is_mobile"": true}" 531,2,53,2017-05-11 06:02:12,http://mcculloughschmeler.com/laisha,,3.220.131.177,"{""location"": ""GA"", ""is_mobile"": false}" 532,2,53,2017-01-30 06:29:12,http://carroll.org/milford,,170.65.49.23,"{""location"": ""PE"", ""is_mobile"": true}" 533,2,53,2017-03-20 01:51:00,http://mohr.name/melisa,,189.37.198.112,"{""location"": ""SB"", ""is_mobile"": true}" 534,2,53,2017-04-03 10:40:31,http://strosinschamberger.io/maxine.zemlak,,51.128.118.193,"{""location"": ""BY"", ""is_mobile"": false}" 535,2,53,2017-05-28 04:09:32,http://willmsweimann.info/laurence_fay,,102.185.160.154,"{""location"": ""LB"", ""is_mobile"": false}" 536,2,54,2017-01-27 00:20:29,http://smitham.io/freeda_gottlieb,,112.40.248.83,"{""location"": ""LB"", ""is_mobile"": true}" 537,2,54,2017-01-18 02:10:07,http://jakubowskihyatt.co/mitchell.grimes,,140.74.58.119,"{""location"": ""CY"", ""is_mobile"": true}" 538,2,54,2016-12-19 08:23:22,http://nienow.name/broderick,,151.243.92.203,"{""location"": ""SB"", ""is_mobile"": false}" 539,2,54,2017-03-06 22:19:29,http://cummeratakeebler.com/rubie.beer,,126.161.4.228,"{""location"": ""FO"", ""is_mobile"": false}" 540,2,54,2016-12-22 18:47:20,http://mraz.name/kadin.weber,,245.194.234.143,"{""location"": ""JO"", ""is_mobile"": true}" 541,2,54,2017-05-28 16:38:25,http://hilpert.info/marlen,,110.54.126.83,"{""location"": ""MA"", ""is_mobile"": true}" 542,2,54,2017-04-15 05:46:15,http://parisiangrant.net/mariane,,176.165.162.38,"{""location"": ""BG"", ""is_mobile"": false}" 543,2,54,2017-03-14 18:27:14,http://mayert.com/polly_heller,,9.35.68.134,"{""location"": ""IN"", ""is_mobile"": true}" 544,2,54,2017-01-01 01:53:02,http://dare.com/theron_hirthe,,121.67.38.77,"{""location"": ""CV"", ""is_mobile"": true}" 545,2,54,2017-05-23 11:16:36,http://muller.biz/amya,,149.34.122.94,"{""location"": ""CW"", ""is_mobile"": true}" 546,2,54,2017-05-21 23:33:24,http://kuhlman.org/loyal_terry,,199.34.17.239,"{""location"": ""KN"", ""is_mobile"": true}" 547,2,54,2017-03-20 12:49:48,http://stroman.name/trever,,110.54.126.83,"{""location"": ""FM"", ""is_mobile"": true}" 548,2,54,2017-05-21 09:58:17,http://carroll.co/ed,,231.154.31.89,"{""location"": ""WF"", ""is_mobile"": true}" 549,2,54,2017-06-09 07:15:53,http://spencer.name/verner_schmidt,,191.201.245.36,"{""location"": ""TZ"", ""is_mobile"": true}" 550,2,54,2017-02-01 10:39:53,http://greenholt.com/deon.reichert,,213.51.147.155,"{""location"": ""NF"", ""is_mobile"": false}" 551,2,54,2017-04-10 23:38:30,http://stark.name/katarina_mertz,,14.179.65.191,"{""location"": ""JE"", ""is_mobile"": true}" 552,2,55,2017-04-15 21:52:20,http://ratke.io/krista,,173.223.150.174,"{""location"": ""IL"", ""is_mobile"": false}" 553,2,55,2017-01-18 10:05:26,http://yundt.name/eleanore,,216.9.4.237,"{""location"": ""VG"", ""is_mobile"": true}" 554,2,55,2017-05-23 12:31:37,http://doylestroman.name/asha_kilback,,46.191.180.85,"{""location"": ""PK"", ""is_mobile"": true}" 555,2,55,2017-04-16 07:28:56,http://toy.co/german,,179.192.21.80,"{""location"": ""YT"", ""is_mobile"": false}" 556,2,55,2016-12-18 21:02:26,http://windler.net/carleton_mohr,,235.205.45.56,"{""location"": ""GS"", ""is_mobile"": false}" 557,2,55,2017-03-06 04:40:09,http://mertz.name/gino_pagac,,79.83.80.125,"{""location"": ""NZ"", ""is_mobile"": true}" 558,2,55,2017-03-04 09:24:12,http://terry.org/nikolas.wintheiser,,21.203.253.252,"{""location"": ""MA"", ""is_mobile"": false}" 559,2,55,2017-04-03 03:48:09,http://emmerichdubuque.io/cierra,,17.106.125.161,"{""location"": ""EE"", ""is_mobile"": false}" 560,2,55,2016-12-19 13:32:10,http://schadensauer.info/hollie,,21.203.253.252,"{""location"": ""LR"", ""is_mobile"": true}" 561,2,55,2017-03-16 15:43:25,http://denesik.com/lenna,,35.107.151.36,"{""location"": ""HU"", ""is_mobile"": false}" 562,2,55,2017-01-15 06:04:31,http://gleichnerfritsch.net/myra,,71.81.245.244,"{""location"": ""TN"", ""is_mobile"": false}" 3672,6,336,2017-02-15 03:54:54,http://goldnerbotsford.com/reilly,1.0000000000,134.119.158.66,"{""location"": ""BA"", ""is_mobile"": true}" 3673,6,336,2017-05-11 01:33:23,http://jenkinscarroll.org/royce_schmitt,2.0000000000,29.56.188.198,"{""location"": ""RU"", ""is_mobile"": true}" 3674,6,336,2017-01-14 13:12:53,http://lindgren.name/robbie.rogahn,1.0000000000,147.124.128.139,"{""location"": ""LB"", ""is_mobile"": false}" 3675,6,336,2017-02-12 01:01:13,http://schmittborer.com/martina_balistreri,2.0000000000,14.129.6.193,"{""location"": ""ST"", ""is_mobile"": true}" 3676,6,336,2017-01-09 23:38:58,http://torphy.io/dejon.bradtke,0.0000000000,105.242.129.53,"{""location"": ""DO"", ""is_mobile"": false}" 3677,6,336,2017-01-28 22:33:12,http://brownreichert.name/jamir,0.0000000000,203.121.66.30,"{""location"": ""BW"", ""is_mobile"": true}" 3678,6,336,2017-05-11 14:19:32,http://dickensullrich.net/jackeline.berge,2.0000000000,232.84.16.228,"{""location"": ""GU"", ""is_mobile"": true}" 3679,6,336,2017-03-02 02:55:16,http://waelchi.name/logan.kovacek,0.0000000000,54.95.178.82,"{""location"": ""MX"", ""is_mobile"": false}" 3680,6,336,2017-01-25 11:17:26,http://becker.biz/stefanie.bauch,0.0000000000,190.7.39.130,"{""location"": ""EE"", ""is_mobile"": false}" 3681,6,336,2017-06-05 10:44:47,http://borer.net/hugh,1.0000000000,55.163.221.208,"{""location"": ""SV"", ""is_mobile"": false}" 3682,6,336,2017-03-09 03:27:50,http://beckerlehner.co/brielle,0.0000000000,222.162.136.158,"{""location"": ""PW"", ""is_mobile"": true}" 3683,6,336,2017-05-31 22:05:23,http://koeppsipes.net/gunner,1.0000000000,179.155.67.59,"{""location"": ""PS"", ""is_mobile"": false}" 3684,6,336,2017-03-03 16:22:30,http://rippin.net/elfrieda_cummings,1.0000000000,14.129.6.193,"{""location"": ""IL"", ""is_mobile"": true}" 3685,6,336,2017-04-25 09:43:17,http://medhurst.info/marcus_langworth,2.0000000000,73.215.40.65,"{""location"": ""NI"", ""is_mobile"": false}" 3686,6,337,2017-02-12 20:33:06,http://willms.name/myrl,0.0000000000,3.177.96.178,"{""location"": ""CX"", ""is_mobile"": true}" 3687,6,337,2017-05-04 10:38:05,http://marquardt.net/jamar.schmidt,0.0000000000,181.149.95.247,"{""location"": ""CI"", ""is_mobile"": true}" 3688,6,337,2017-06-10 08:49:56,http://streich.biz/adrienne.grant,3.0000000000,220.111.167.51,"{""location"": ""TT"", ""is_mobile"": false}" 3689,6,337,2017-05-31 07:20:46,http://hermann.info/susie_haley,2.0000000000,37.98.139.198,"{""location"": ""AR"", ""is_mobile"": true}" 3690,6,337,2017-05-06 22:42:49,http://mayer.io/kyler,3.0000000000,31.130.7.87,"{""location"": ""GP"", ""is_mobile"": true}" 3691,6,337,2017-05-27 22:09:20,http://jacobimurphy.name/reed_hauck,1.0000000000,60.142.121.14,"{""location"": ""SO"", ""is_mobile"": false}" 3692,6,337,2017-02-20 15:34:01,http://toy.com/joannie,2.0000000000,183.173.230.194,"{""location"": ""BG"", ""is_mobile"": false}" 3693,6,337,2017-05-07 11:28:39,http://mannreichert.io/kiel_runte,2.0000000000,48.251.51.225,"{""location"": ""BV"", ""is_mobile"": true}" 3694,6,337,2017-01-07 19:56:00,http://stiedemannauer.info/lafayette_ryan,2.0000000000,54.128.236.96,"{""location"": ""AI"", ""is_mobile"": false}" 3695,6,337,2016-12-27 20:36:47,http://kautzerpagac.org/mathilde,1.0000000000,58.75.96.120,"{""location"": ""SH"", ""is_mobile"": false}" 3696,6,337,2017-02-13 11:06:42,http://rogahn.co/ervin_doyle,3.0000000000,130.218.3.247,"{""location"": ""CH"", ""is_mobile"": false}" 3697,6,337,2017-01-20 05:18:11,http://towne.org/elody,2.0000000000,205.156.188.36,"{""location"": ""LS"", ""is_mobile"": false}" 3698,6,337,2017-03-15 19:04:19,http://fritschsmith.net/amber.murphy,1.0000000000,60.142.121.14,"{""location"": ""VC"", ""is_mobile"": false}" 3699,6,337,2017-01-29 23:07:30,http://cruickshankeffertz.io/mikel,1.0000000000,138.243.222.160,"{""location"": ""SJ"", ""is_mobile"": false}" 3700,6,337,2017-04-22 20:09:00,http://lockman.info/fern,1.0000000000,54.128.236.96,"{""location"": ""TT"", ""is_mobile"": true}" 3701,6,337,2017-04-29 03:12:07,http://konopelski.info/landen,1.0000000000,124.71.99.130,"{""location"": ""NE"", ""is_mobile"": false}" 3702,6,338,2017-02-01 19:43:09,http://koepp.io/omari,1.0000000000,76.51.193.17,"{""location"": ""HT"", ""is_mobile"": true}" 3703,6,338,2016-12-22 11:45:35,http://jerdekunde.io/tristin,1.0000000000,239.114.192.4,"{""location"": ""ZA"", ""is_mobile"": true}" 3704,6,338,2017-01-21 06:05:20,http://farrellhackett.org/rosalyn_gerhold,1.0000000000,76.51.193.17,"{""location"": ""CW"", ""is_mobile"": false}" 3705,6,338,2017-04-26 15:29:03,http://kulaswalsh.co/damaris,0.0000000000,238.78.253.135,"{""location"": ""MF"", ""is_mobile"": true}" 3706,6,338,2017-03-27 15:29:25,http://vandervortpadberg.info/tatyana_gulgowski,0.0000000000,37.79.215.75,"{""location"": ""CI"", ""is_mobile"": true}" 3707,6,338,2017-05-01 03:48:04,http://corwinschultz.biz/ryann,1.0000000000,196.106.178.189,"{""location"": ""MR"", ""is_mobile"": false}" 3708,6,338,2017-02-02 01:33:21,http://nienowvolkman.name/alfonso,0.0000000000,29.55.175.207,"{""location"": ""GB"", ""is_mobile"": true}" 3709,6,338,2017-04-27 07:32:35,http://leffler.org/chauncey,0.0000000000,92.151.149.17,"{""location"": ""PE"", ""is_mobile"": false}" 3710,6,338,2017-01-13 21:29:14,http://yost.name/saul,0.0000000000,229.167.185.30,"{""location"": ""RO"", ""is_mobile"": true}" 3711,6,338,2017-03-01 09:53:53,http://mcglynn.name/trystan,1.0000000000,84.90.175.173,"{""location"": ""KZ"", ""is_mobile"": false}" 3712,6,338,2017-03-10 20:31:23,http://cruickshank.info/nyah,0.0000000000,83.247.236.89,"{""location"": ""BV"", ""is_mobile"": true}" 3713,6,338,2017-04-04 01:30:42,http://kundejohns.io/lauriane_pfannerstill,0.0000000000,84.90.175.173,"{""location"": ""VC"", ""is_mobile"": true}" 3714,6,339,2016-12-26 12:05:43,http://shieldscarroll.name/bertha.welch,0.0000000000,120.98.214.109,"{""location"": ""BZ"", ""is_mobile"": false}" 3715,6,339,2017-02-22 01:46:49,http://herzog.io/susie,0.0000000000,91.134.108.172,"{""location"": ""SE"", ""is_mobile"": false}" 3716,6,339,2017-02-22 15:01:04,http://boscoko.info/wilford_leuschke,0.0000000000,171.141.53.240,"{""location"": ""SD"", ""is_mobile"": true}" 3717,6,339,2017-02-14 21:20:05,http://gibson.io/dorothy,0.0000000000,218.35.62.250,"{""location"": ""JO"", ""is_mobile"": true}" 3718,6,339,2017-01-22 18:21:59,http://barrows.co/carolyn.klein,0.0000000000,240.143.14.156,"{""location"": ""ML"", ""is_mobile"": true}" 3719,6,339,2017-05-27 16:33:55,http://runolfsdottir.net/zachariah.rolfson,0.0000000000,81.36.125.167,"{""location"": ""NL"", ""is_mobile"": false}" 3720,6,339,2017-02-08 20:10:05,http://pollich.info/chauncey_feil,0.0000000000,113.74.91.192,"{""location"": ""IL"", ""is_mobile"": false}" 563,2,55,2017-05-23 12:40:44,http://jaskolskireynolds.info/emerson,,99.229.130.181,"{""location"": ""DO"", ""is_mobile"": true}" 564,2,56,2017-05-30 17:03:02,http://macejkovic.io/raleigh,,209.205.172.191,"{""location"": ""SJ"", ""is_mobile"": false}" 565,2,56,2017-05-23 12:17:47,http://ferrygoldner.com/eugenia,,236.33.10.218,"{""location"": ""EG"", ""is_mobile"": false}" 566,2,56,2017-05-30 12:23:08,http://reinger.info/hilma,,212.224.19.102,"{""location"": ""AW"", ""is_mobile"": false}" 567,2,56,2017-06-09 18:56:42,http://gutkowskicronin.io/baylee,,97.92.146.190,"{""location"": ""IT"", ""is_mobile"": false}" 568,2,56,2017-05-12 20:19:15,http://reilly.org/alice_carroll,,15.110.67.28,"{""location"": ""BT"", ""is_mobile"": true}" 569,2,56,2017-04-08 23:54:43,http://schmidtdicki.com/shannon,,27.151.60.110,"{""location"": ""VN"", ""is_mobile"": false}" 570,2,56,2017-05-29 09:57:06,http://witting.biz/aunta,,145.216.246.154,"{""location"": ""FK"", ""is_mobile"": true}" 571,2,56,2017-02-21 09:04:49,http://gislason.org/ivy_willms,,209.205.172.191,"{""location"": ""CR"", ""is_mobile"": false}" 572,2,56,2017-02-01 13:16:10,http://bernhard.co/allison.macgyver,,145.216.246.154,"{""location"": ""MG"", ""is_mobile"": false}" 573,2,56,2017-05-14 02:03:15,http://casper.name/enid.huel,,108.25.72.163,"{""location"": ""AQ"", ""is_mobile"": true}" 574,2,56,2017-05-03 07:59:27,http://goodwin.com/myrl,,204.81.240.251,"{""location"": ""AX"", ""is_mobile"": false}" 575,2,56,2017-05-02 06:40:27,http://emmerich.name/rocio,,234.114.161.249,"{""location"": ""DJ"", ""is_mobile"": false}" 576,2,56,2017-05-24 19:09:30,http://murphy.io/arianna_jerde,,204.81.240.251,"{""location"": ""GT"", ""is_mobile"": true}" 577,2,56,2017-03-16 20:35:16,http://mertz.net/genevieve,,145.216.246.154,"{""location"": ""BN"", ""is_mobile"": false}" 578,2,56,2017-04-24 06:57:34,http://ornhintz.org/mollie.sawayn,,225.249.29.250,"{""location"": ""CH"", ""is_mobile"": false}" 579,2,56,2017-03-16 08:35:56,http://turnerklein.info/jewel,,38.42.224.180,"{""location"": ""CL"", ""is_mobile"": true}" 580,2,56,2017-03-14 22:23:30,http://ondricka.biz/cara.cruickshank,,107.236.142.203,"{""location"": ""BS"", ""is_mobile"": false}" 581,2,56,2017-05-23 01:01:22,http://bartonferry.co/ozella.conroy,,169.163.4.89,"{""location"": ""GY"", ""is_mobile"": false}" 582,2,56,2017-03-14 08:05:12,http://luettgen.com/desiree,,212.224.19.102,"{""location"": ""FK"", ""is_mobile"": false}" 583,2,57,2017-04-26 09:51:08,http://gottliebbarton.biz/edmond_schiller,,12.6.165.74,"{""location"": ""SX"", ""is_mobile"": true}" 584,2,57,2017-04-17 02:00:01,http://hahnauer.info/justyn.schmidt,,171.78.150.213,"{""location"": ""VA"", ""is_mobile"": false}" 585,2,57,2016-12-18 18:54:35,http://goldner.biz/howard.fahey,,219.206.157.41,"{""location"": ""TM"", ""is_mobile"": true}" 586,2,57,2017-01-10 03:08:52,http://berge.biz/haskell.lind,,251.149.181.216,"{""location"": ""LK"", ""is_mobile"": true}" 587,2,57,2017-02-01 12:56:55,http://gerholdblock.info/esmeralda.moen,,251.149.181.216,"{""location"": ""KG"", ""is_mobile"": false}" 588,2,57,2016-12-31 00:34:19,http://rutherford.co/buford,,206.68.55.89,"{""location"": ""HM"", ""is_mobile"": true}" 589,2,57,2017-01-05 08:28:33,http://champlinmiller.com/rocio,,106.121.171.40,"{""location"": ""GL"", ""is_mobile"": false}" 590,2,57,2017-04-02 19:56:54,http://armstrong.co/josiah,,113.197.235.122,"{""location"": ""RW"", ""is_mobile"": false}" 591,2,57,2017-03-03 19:24:55,http://yundtpagac.biz/gino.jacobs,,190.233.15.235,"{""location"": ""SX"", ""is_mobile"": true}" 592,2,57,2017-02-25 19:35:56,http://powlowski.net/wilton,,113.197.235.122,"{""location"": ""LY"", ""is_mobile"": true}" 593,2,57,2017-06-12 21:09:33,http://kertzmannokuneva.name/savannah,,126.212.145.230,"{""location"": ""BF"", ""is_mobile"": true}" 594,2,57,2017-04-02 15:51:12,http://will.name/anabelle,,117.128.207.122,"{""location"": ""KM"", ""is_mobile"": false}" 595,2,57,2017-03-03 23:11:35,http://bergnaum.net/luciano,,187.85.2.166,"{""location"": ""CN"", ""is_mobile"": true}" 596,2,57,2016-12-30 04:19:10,http://smith.co/annetta,,65.229.169.90,"{""location"": ""MH"", ""is_mobile"": true}" 597,2,57,2017-01-21 01:29:17,http://ritchiehauck.com/loren.ziemann,,126.212.145.230,"{""location"": ""SZ"", ""is_mobile"": true}" 598,2,57,2017-05-03 19:54:58,http://reynolds.com/alfonzo_murray,,187.85.2.166,"{""location"": ""GQ"", ""is_mobile"": false}" 599,2,57,2017-05-10 06:03:29,http://breitenberghyatt.com/garrett,,12.167.241.67,"{""location"": ""OM"", ""is_mobile"": false}" 600,2,57,2017-04-17 07:04:25,http://kuphal.io/adriel,,251.149.181.216,"{""location"": ""BO"", ""is_mobile"": true}" 601,2,57,2017-04-13 17:39:56,http://schaefer.com/grace,,56.35.72.163,"{""location"": ""IL"", ""is_mobile"": true}" 602,2,57,2017-06-13 18:54:23,http://wisozkcrooks.net/vicky_yost,,165.92.188.6,"{""location"": ""MP"", ""is_mobile"": false}" 603,2,58,2017-01-15 10:06:50,http://farrell.name/martina,0.0000000000,242.207.134.237,"{""location"": ""ZW"", ""is_mobile"": false}" 604,2,58,2017-04-23 02:12:27,http://greenmacgyver.info/addison,0.0000000000,16.42.179.107,"{""location"": ""CU"", ""is_mobile"": false}" 605,2,58,2017-04-14 02:42:49,http://hermannrath.io/claude_rutherford,1.0000000000,111.218.156.72,"{""location"": ""VN"", ""is_mobile"": true}" 606,2,58,2016-12-31 04:34:11,http://jerde.name/effie.runte,0.0000000000,64.5.129.44,"{""location"": ""ML"", ""is_mobile"": true}" 607,2,58,2016-12-19 22:42:12,http://reynoldscummings.name/tristian_tillman,0.0000000000,35.92.64.230,"{""location"": ""IL"", ""is_mobile"": true}" 608,2,58,2017-04-20 07:00:19,http://kiehn.biz/justice_rice,1.0000000000,111.218.156.72,"{""location"": ""EE"", ""is_mobile"": true}" 609,2,58,2017-01-07 01:19:30,http://rohan.biz/brandi,0.0000000000,245.101.143.202,"{""location"": ""HU"", ""is_mobile"": false}" 610,2,58,2017-03-14 09:33:05,http://bauch.info/geoffrey,1.0000000000,147.153.234.70,"{""location"": ""US"", ""is_mobile"": false}" 611,2,58,2016-12-19 02:12:36,http://wisoky.io/kailey_ruecker,1.0000000000,144.144.178.215,"{""location"": ""MF"", ""is_mobile"": true}" 612,2,58,2016-12-29 10:46:39,http://oberbrunner.net/katelin,1.0000000000,229.194.39.21,"{""location"": ""LY"", ""is_mobile"": true}" 613,2,58,2017-03-05 03:15:54,http://goldner.com/vena,0.0000000000,118.86.227.27,"{""location"": ""NU"", ""is_mobile"": true}" 614,2,58,2017-06-01 03:36:54,http://hackettdooley.io/layla_murazik,0.0000000000,83.90.119.171,"{""location"": ""GF"", ""is_mobile"": false}" 615,2,58,2017-02-10 00:34:34,http://mertzgaylord.info/herta_simonis,0.0000000000,225.10.238.111,"{""location"": ""FO"", ""is_mobile"": false}" 616,2,58,2017-05-16 22:08:10,http://dare.biz/olin_kuhn,1.0000000000,87.140.122.185,"{""location"": ""FJ"", ""is_mobile"": false}" 617,2,58,2017-06-02 21:28:19,http://mertz.com/jackeline,0.0000000000,196.204.157.4,"{""location"": ""FJ"", ""is_mobile"": false}" 3721,6,339,2017-01-09 10:32:12,http://howellziemann.net/jermaine,0.0000000000,202.69.45.238,"{""location"": ""LR"", ""is_mobile"": true}" 3722,6,339,2016-12-19 17:44:41,http://lindgrady.org/cristal.hahn,0.0000000000,204.201.243.201,"{""location"": ""CU"", ""is_mobile"": false}" 3723,6,339,2017-03-13 16:10:14,http://harbermurazik.com/granville,0.0000000000,39.136.149.120,"{""location"": ""LS"", ""is_mobile"": true}" 3724,6,339,2017-02-03 12:00:05,http://cummerata.net/loma,0.0000000000,59.43.79.20,"{""location"": ""BH"", ""is_mobile"": false}" 3725,6,339,2016-12-24 01:27:47,http://oberbrunner.info/levi,0.0000000000,171.141.53.240,"{""location"": ""BE"", ""is_mobile"": false}" 3726,6,339,2017-01-19 05:21:07,http://price.net/gerardo_grady,0.0000000000,174.12.58.141,"{""location"": ""IM"", ""is_mobile"": true}" 3727,6,339,2016-12-15 11:27:43,http://breitenberg.org/margot,0.0000000000,41.137.170.185,"{""location"": ""CU"", ""is_mobile"": true}" 3728,6,339,2017-02-21 23:30:49,http://schimmel.co/odea.koepp,0.0000000000,40.174.201.133,"{""location"": ""SR"", ""is_mobile"": false}" 3729,6,339,2016-12-29 16:20:45,http://ruecker.name/beie_lemke,0.0000000000,120.98.214.109,"{""location"": ""KP"", ""is_mobile"": true}" 3730,6,340,2017-05-03 21:37:12,http://dare.name/cristal,0.0000000000,97.133.161.136,"{""location"": ""NR"", ""is_mobile"": false}" 3731,6,340,2017-02-20 13:21:55,http://sawayn.org/wade.pfannerstill,0.0000000000,76.19.80.126,"{""location"": ""AZ"", ""is_mobile"": true}" 3732,6,341,2017-03-08 05:47:05,http://bogan.co/corene,0.0000000000,7.194.57.104,"{""location"": ""GP"", ""is_mobile"": false}" 3733,6,341,2017-05-12 22:05:24,http://haley.biz/sydni,2.0000000000,120.185.159.218,"{""location"": ""VC"", ""is_mobile"": true}" 3734,6,341,2017-04-21 06:38:33,http://schuppe.org/kristian_jacobi,0.0000000000,129.183.172.93,"{""location"": ""DO"", ""is_mobile"": true}" 3735,6,341,2017-03-12 20:44:29,http://mitchell.com/alphonso.moore,1.0000000000,194.114.126.236,"{""location"": ""MM"", ""is_mobile"": false}" 3736,6,341,2017-01-12 20:25:32,http://beier.net/dameon_ortiz,0.0000000000,51.90.230.89,"{""location"": ""BA"", ""is_mobile"": true}" 3737,6,341,2017-03-19 10:25:57,http://pfannerstill.net/kiarra,2.0000000000,44.44.249.197,"{""location"": ""IE"", ""is_mobile"": false}" 3738,6,341,2017-04-29 19:52:56,http://heidenreich.net/hilma.tromp,0.0000000000,230.224.152.216,"{""location"": ""GN"", ""is_mobile"": true}" 3739,6,341,2016-12-31 20:21:59,http://connelly.com/ashlee,1.0000000000,93.18.205.221,"{""location"": ""GB"", ""is_mobile"": true}" 3740,6,341,2017-01-18 10:51:23,http://mraz.biz/deshawn,2.0000000000,42.36.164.25,"{""location"": ""TT"", ""is_mobile"": true}" 3741,6,341,2017-01-20 21:04:41,http://ebert.co/wendell,0.0000000000,169.118.182.220,"{""location"": ""MT"", ""is_mobile"": false}" 3742,6,341,2017-02-23 21:40:13,http://sanford.net/dwight,1.0000000000,42.36.164.25,"{""location"": ""HT"", ""is_mobile"": false}" 3743,6,341,2016-12-28 05:36:46,http://simonis.biz/ruthe,2.0000000000,166.99.243.200,"{""location"": ""TK"", ""is_mobile"": false}" 3744,6,341,2016-12-28 00:27:45,http://gutkowski.org/kaylie,2.0000000000,65.188.53.110,"{""location"": ""TM"", ""is_mobile"": false}" 3745,6,341,2017-04-14 13:22:58,http://prohaska.net/reva,0.0000000000,44.44.249.197,"{""location"": ""TC"", ""is_mobile"": false}" 3746,6,341,2017-05-20 14:26:31,http://abernathy.name/velma_bruen,0.0000000000,166.99.243.200,"{""location"": ""BV"", ""is_mobile"": false}" 3747,6,341,2017-02-12 18:07:22,http://purdy.name/anthony.lemke,0.0000000000,120.185.159.218,"{""location"": ""CK"", ""is_mobile"": true}" 3748,6,341,2016-12-13 06:05:35,http://king.info/maryse.boyer,1.0000000000,230.224.152.216,"{""location"": ""ET"", ""is_mobile"": false}" 3749,6,341,2017-05-22 11:16:57,http://witting.io/robin,2.0000000000,42.36.164.25,"{""location"": ""GW"", ""is_mobile"": false}" 3750,6,341,2016-12-19 04:38:36,http://conn.biz/aurelio,2.0000000000,21.77.184.141,"{""location"": ""MA"", ""is_mobile"": false}" 3751,6,342,2017-01-02 12:35:14,http://pacocha.name/amaya.osinski,0.0000000000,220.142.130.138,"{""location"": ""HT"", ""is_mobile"": false}" 3752,6,342,2017-05-31 17:35:01,http://hartmannschumm.net/elva_kerluke,0.0000000000,188.219.2.23,"{""location"": ""GI"", ""is_mobile"": false}" 3753,6,343,2017-05-24 12:15:12,http://donnelly.org/mina,3.0000000000,132.54.169.115,"{""location"": ""SJ"", ""is_mobile"": false}" 3754,6,343,2017-03-19 07:21:51,http://kreiger.info/keith,3.0000000000,132.54.169.115,"{""location"": ""TC"", ""is_mobile"": true}" 3755,6,343,2017-02-12 02:01:27,http://turnerbartoletti.co/raleigh_luettgen,2.0000000000,160.155.140.244,"{""location"": ""BI"", ""is_mobile"": false}" 3756,6,343,2017-02-10 01:07:24,http://barton.name/linnie,0.0000000000,246.132.133.210,"{""location"": ""DO"", ""is_mobile"": false}" 3757,6,343,2017-04-20 17:09:54,http://halvorson.biz/elisabeth,1.0000000000,50.126.41.36,"{""location"": ""JE"", ""is_mobile"": true}" 3758,6,343,2017-01-27 14:23:28,http://lang.name/virgie,1.0000000000,254.246.142.182,"{""location"": ""SG"", ""is_mobile"": true}" 3759,6,344,2017-01-31 06:50:28,http://murazik.co/anastasia.wolff,0.0000000000,240.99.62.164,"{""location"": ""IM"", ""is_mobile"": true}" 3760,6,344,2017-05-19 10:53:27,http://lebsack.name/reid,1.0000000000,141.130.113.240,"{""location"": ""KZ"", ""is_mobile"": true}" 3761,6,344,2016-12-13 19:42:37,http://rodriguezking.co/eve,0.0000000000,243.53.204.55,"{""location"": ""TD"", ""is_mobile"": false}" 3762,6,344,2017-01-14 04:06:15,http://schoen.org/corene,0.0000000000,84.171.219.189,"{""location"": ""UY"", ""is_mobile"": true}" 3763,6,344,2017-04-05 18:57:05,http://kris.io/hallie,0.0000000000,84.160.109.203,"{""location"": ""IM"", ""is_mobile"": true}" 3764,6,344,2017-04-02 06:25:46,http://welch.io/kitty_batz,0.0000000000,58.162.149.22,"{""location"": ""NF"", ""is_mobile"": true}" 3765,6,344,2017-05-20 08:17:27,http://lindgren.biz/isom_shields,1.0000000000,44.168.99.26,"{""location"": ""VN"", ""is_mobile"": false}" 3766,6,344,2017-01-25 23:19:56,http://murphywiegand.org/zaria_bechtelar,0.0000000000,243.53.204.55,"{""location"": ""ST"", ""is_mobile"": false}" 3767,6,344,2017-02-22 06:20:57,http://kuhn.info/muhammad,1.0000000000,99.14.73.155,"{""location"": ""BT"", ""is_mobile"": true}" 3768,6,344,2017-02-07 19:12:46,http://rosenbaumkreiger.net/harold,1.0000000000,8.69.9.172,"{""location"": ""MP"", ""is_mobile"": false}" 3769,6,344,2017-06-11 17:41:40,http://littlebarrows.biz/lenore_mayert,0.0000000000,43.28.224.119,"{""location"": ""CO"", ""is_mobile"": true}" 3770,6,344,2017-03-08 05:35:25,http://cronin.info/larry,0.0000000000,47.185.82.141,"{""location"": ""JP"", ""is_mobile"": true}" 3771,6,344,2017-04-29 12:17:10,http://schinnerrau.io/nakia,0.0000000000,213.225.97.172,"{""location"": ""BB"", ""is_mobile"": true}" 3772,6,344,2017-05-29 20:19:08,http://smitham.info/benedict_boyle,0.0000000000,15.254.24.23,"{""location"": ""PN"", ""is_mobile"": true}" 618,2,58,2017-01-20 05:33:22,http://halvorson.co/harry,0.0000000000,22.140.173.127,"{""location"": ""BY"", ""is_mobile"": false}" 619,2,58,2017-04-02 08:52:33,http://bahringer.com/patrick,1.0000000000,10.31.92.156,"{""location"": ""RE"", ""is_mobile"": true}" 620,2,59,2017-05-21 05:42:40,http://rosenbaum.biz/teie,2.0000000000,59.130.205.221,"{""location"": ""GY"", ""is_mobile"": false}" 621,2,59,2017-02-28 03:01:39,http://vandervort.name/viviane.lindgren,1.0000000000,121.22.189.85,"{""location"": ""AE"", ""is_mobile"": true}" 1292,2,123,2017-05-02 08:23:03,http://kuphal.org/brown,,195.245.131.144,"{""location"": ""WF"", ""is_mobile"": true}" 622,2,59,2017-05-27 00:11:38,http://feeney.info/arjun,0.0000000000,207.137.90.249,"{""location"": ""ER"", ""is_mobile"": true}" 623,2,59,2017-01-24 06:21:29,http://murazik.org/jadon.wehner,0.0000000000,24.86.213.81,"{""location"": ""MC"", ""is_mobile"": true}" 624,2,60,2017-05-19 08:13:57,http://frami.com/jerrell,0.0000000000,149.115.183.169,"{""location"": ""CY"", ""is_mobile"": true}" 625,2,60,2017-06-09 04:20:14,http://emmerich.io/florine.nicolas,0.0000000000,209.66.63.27,"{""location"": ""ZA"", ""is_mobile"": true}" 626,2,60,2017-04-01 13:04:07,http://wyman.com/ida,0.0000000000,3.246.82.71,"{""location"": ""MA"", ""is_mobile"": false}" 627,2,60,2017-02-03 08:17:14,http://renneremmerich.net/noah,0.0000000000,14.48.201.18,"{""location"": ""EE"", ""is_mobile"": false}" 628,2,60,2017-05-04 02:56:44,http://crooks.org/darlene,0.0000000000,92.178.81.132,"{""location"": ""TZ"", ""is_mobile"": true}" 629,2,60,2017-05-25 13:03:13,http://gislason.co/cary_lesch,0.0000000000,246.243.203.99,"{""location"": ""PS"", ""is_mobile"": false}" 630,2,60,2017-04-08 07:36:26,http://schmidt.name/greta.keler,0.0000000000,246.243.203.99,"{""location"": ""BY"", ""is_mobile"": false}" 631,2,60,2017-05-22 13:02:05,http://streich.net/adelia,0.0000000000,158.34.113.242,"{""location"": ""SR"", ""is_mobile"": true}" 632,2,60,2016-12-24 09:03:11,http://rennerschowalter.co/travis,0.0000000000,96.19.40.83,"{""location"": ""CX"", ""is_mobile"": true}" 633,2,60,2017-01-25 03:09:23,http://bodehackett.com/brook,0.0000000000,224.174.203.199,"{""location"": ""SS"", ""is_mobile"": true}" 634,2,60,2017-03-28 23:30:57,http://gleichner.name/mabelle_emmerich,0.0000000000,214.111.179.2,"{""location"": ""MW"", ""is_mobile"": true}" 635,2,60,2017-01-05 20:30:39,http://howehills.io/raheem,0.0000000000,7.98.147.74,"{""location"": ""ME"", ""is_mobile"": false}" 636,2,61,2017-01-24 11:09:52,http://hilpert.com/lacy.little,1.0000000000,228.119.145.31,"{""location"": ""SA"", ""is_mobile"": true}" 637,2,61,2017-01-26 06:57:04,http://miller.co/hortense,1.0000000000,155.157.69.198,"{""location"": ""BG"", ""is_mobile"": false}" 638,2,61,2017-01-03 13:14:03,http://johnson.com/jaquan,0.0000000000,28.224.240.211,"{""location"": ""LS"", ""is_mobile"": false}" 639,2,61,2017-02-01 14:01:38,http://gleasonbechtelar.io/rebeca,1.0000000000,186.254.44.16,"{""location"": ""SJ"", ""is_mobile"": true}" 640,2,62,2017-04-07 05:22:51,http://runolfon.com/cristal,0.0000000000,236.184.6.169,"{""location"": ""FO"", ""is_mobile"": true}" 641,2,62,2016-12-16 20:45:59,http://steuberpfannerstill.biz/laury.crist,0.0000000000,40.203.231.253,"{""location"": ""DK"", ""is_mobile"": true}" 642,2,62,2017-01-21 18:03:32,http://buckridgebruen.info/bette,0.0000000000,131.51.63.53,"{""location"": ""BE"", ""is_mobile"": false}" 643,2,62,2017-01-02 16:15:08,http://leffler.biz/emilia,0.0000000000,217.197.136.180,"{""location"": ""KI"", ""is_mobile"": false}" 644,2,62,2017-01-27 15:23:21,http://kozey.name/demond,0.0000000000,217.197.136.180,"{""location"": ""BL"", ""is_mobile"": false}" 645,2,62,2017-03-07 07:07:48,http://herzogyost.org/scarlett.shanahan,0.0000000000,131.51.63.53,"{""location"": ""CU"", ""is_mobile"": true}" 646,2,62,2017-05-29 07:10:36,http://dooley.biz/rosendo,0.0000000000,122.208.117.82,"{""location"": ""HM"", ""is_mobile"": true}" 647,2,62,2017-05-11 16:01:19,http://pollich.com/darrell.schaefer,0.0000000000,18.237.6.24,"{""location"": ""AX"", ""is_mobile"": false}" 648,2,62,2017-05-25 04:38:11,http://mckenzie.co/cruz,0.0000000000,39.216.55.243,"{""location"": ""AQ"", ""is_mobile"": false}" 649,2,62,2017-01-11 21:21:06,http://leannonfritsch.biz/emmet,0.0000000000,42.118.248.238,"{""location"": ""CD"", ""is_mobile"": false}" 650,2,62,2017-02-01 20:01:16,http://olsonhirthe.org/kole_nicolas,0.0000000000,103.86.224.168,"{""location"": ""RS"", ""is_mobile"": false}" 651,2,63,2017-01-20 18:51:19,http://raynor.org/raleigh.bode,1.0000000000,150.18.21.66,"{""location"": ""NG"", ""is_mobile"": false}" 652,2,63,2016-12-14 21:06:02,http://moenhamill.io/kurt_walsh,1.0000000000,95.242.164.113,"{""location"": ""TK"", ""is_mobile"": true}" 653,2,63,2017-01-01 19:16:00,http://aufderharlueilwitz.org/mozell.abbott,3.0000000000,35.171.116.38,"{""location"": ""FR"", ""is_mobile"": false}" 654,2,63,2017-03-19 20:17:50,http://schulistkulas.biz/nikita,3.0000000000,172.56.141.44,"{""location"": ""CU"", ""is_mobile"": false}" 655,2,63,2017-03-06 18:49:25,http://yostdonnelly.info/jameson,1.0000000000,11.208.13.63,"{""location"": ""RU"", ""is_mobile"": true}" 656,2,63,2017-05-14 15:03:57,http://stehrluettgen.org/jordan,0.0000000000,95.242.164.113,"{""location"": ""GD"", ""is_mobile"": true}" 657,2,63,2017-01-21 20:02:59,http://ortiz.org/francis,0.0000000000,75.76.113.194,"{""location"": ""JO"", ""is_mobile"": false}" 658,2,63,2017-04-03 09:08:07,http://mayert.net/kenton_ritchie,3.0000000000,63.112.155.212,"{""location"": ""MR"", ""is_mobile"": false}" 659,2,63,2017-02-07 00:02:55,http://morarkozey.net/adam_wyman,2.0000000000,133.147.230.215,"{""location"": ""CA"", ""is_mobile"": false}" 660,2,63,2016-12-14 01:00:59,http://mohr.biz/dayana.moore,3.0000000000,168.130.238.203,"{""location"": ""MS"", ""is_mobile"": false}" 661,2,63,2017-03-04 10:18:15,http://treutel.info/maximillian.bergstrom,2.0000000000,150.18.21.66,"{""location"": ""SG"", ""is_mobile"": false}" 662,2,63,2017-05-31 17:29:55,http://trantowsmitham.biz/ova_reichel,3.0000000000,205.228.242.109,"{""location"": ""SG"", ""is_mobile"": true}" 663,2,63,2017-04-09 03:31:33,http://cruickshankmorar.com/stephan,0.0000000000,146.49.105.154,"{""location"": ""YT"", ""is_mobile"": false}" 664,2,63,2017-04-02 16:25:10,http://walker.info/erica_volkman,3.0000000000,41.52.203.28,"{""location"": ""MM"", ""is_mobile"": false}" 665,2,63,2017-04-22 05:18:21,http://lindgrenrutherford.name/name,2.0000000000,84.82.22.124,"{""location"": ""NL"", ""is_mobile"": true}" 666,2,63,2017-02-06 00:41:23,http://hilpert.name/sarina,1.0000000000,172.56.141.44,"{""location"": ""EE"", ""is_mobile"": true}" 667,2,63,2017-05-06 01:14:54,http://ryan.org/jazmyn,0.0000000000,99.50.111.133,"{""location"": ""JE"", ""is_mobile"": true}" 668,2,63,2017-01-07 13:21:53,http://goodwin.io/carley,2.0000000000,168.130.238.203,"{""location"": ""SN"", ""is_mobile"": false}" 669,2,63,2017-03-07 08:19:11,http://cummings.io/tremaine.gaylord,3.0000000000,172.56.141.44,"{""location"": ""TN"", ""is_mobile"": false}" 3773,6,344,2017-06-03 02:06:21,http://kilback.info/jarret,0.0000000000,108.32.5.196,"{""location"": ""BR"", ""is_mobile"": false}" 3774,6,344,2017-04-28 22:44:26,http://williamsonkemmer.co/rickie,0.0000000000,160.131.66.36,"{""location"": ""NL"", ""is_mobile"": true}" 3775,6,344,2017-02-23 08:43:52,http://bruenprohaska.biz/dedric_dubuque,1.0000000000,160.131.66.36,"{""location"": ""KI"", ""is_mobile"": true}" 3776,6,344,2017-03-14 21:41:03,http://metz.biz/jewel,1.0000000000,105.91.51.3,"{""location"": ""JP"", ""is_mobile"": false}" 3777,6,344,2017-03-15 03:39:23,http://jacobsmcglynn.name/nikki,0.0000000000,250.15.107.36,"{""location"": ""PL"", ""is_mobile"": true}" 3778,6,344,2017-04-22 17:34:52,http://heelruel.net/hilbert_strosin,1.0000000000,243.53.204.55,"{""location"": ""GG"", ""is_mobile"": false}" 3779,6,345,2017-01-06 07:18:56,http://bruen.net/eino,,62.54.75.185,"{""location"": ""VE"", ""is_mobile"": false}" 3780,6,345,2017-04-04 03:14:51,http://berge.com/keyshawn,,128.108.28.56,"{""location"": ""VA"", ""is_mobile"": true}" 3781,6,345,2017-06-06 06:07:38,http://durgan.info/orrin_schneider,,62.54.75.185,"{""location"": ""GI"", ""is_mobile"": false}" 3782,6,345,2017-03-06 17:34:14,http://schroederschmitt.biz/andres,,94.241.138.212,"{""location"": ""HM"", ""is_mobile"": false}" 3783,6,345,2017-04-06 08:30:24,http://wolff.info/amber,,114.237.191.214,"{""location"": ""JO"", ""is_mobile"": true}" 3784,6,345,2017-05-26 17:05:55,http://oberbrunner.com/ed.kulas,,151.202.152.39,"{""location"": ""CO"", ""is_mobile"": true}" 3785,6,345,2017-01-06 09:28:16,http://ferry.name/nella_block,,114.237.191.214,"{""location"": ""CA"", ""is_mobile"": false}" 3786,6,345,2017-04-25 04:51:32,http://bosco.info/patrick.leuschke,,151.202.152.39,"{""location"": ""VG"", ""is_mobile"": false}" 3787,6,346,2017-03-05 08:17:44,http://cummings.info/macey_kunze,,107.179.169.168,"{""location"": ""CN"", ""is_mobile"": true}" 3788,6,346,2017-01-12 03:13:16,http://zboncak.com/harmony,,241.194.72.86,"{""location"": ""NA"", ""is_mobile"": false}" 3789,6,346,2017-03-31 23:34:31,http://reingerhilll.name/chance_schuppe,,4.202.27.246,"{""location"": ""KN"", ""is_mobile"": true}" 3790,6,346,2017-02-23 18:31:09,http://hills.org/skyla,,76.127.124.83,"{""location"": ""SD"", ""is_mobile"": true}" 3791,6,347,2017-01-22 20:27:29,http://zemlak.io/ciara,,41.92.198.26,"{""location"": ""MU"", ""is_mobile"": false}" 3792,6,348,2017-01-24 21:49:13,http://hane.io/drew_skiles,,87.23.114.79,"{""location"": ""CM"", ""is_mobile"": false}" 3793,6,348,2017-02-02 02:28:43,http://bernierhartmann.io/ulices.spencer,,165.113.185.41,"{""location"": ""MK"", ""is_mobile"": true}" 3794,6,348,2017-05-16 19:49:58,http://naderstehr.net/maida_sporer,,142.106.95.24,"{""location"": ""IE"", ""is_mobile"": true}" 3795,6,348,2017-04-16 15:13:43,http://eichmanngerhold.info/rosalind,,81.161.194.233,"{""location"": ""CG"", ""is_mobile"": false}" 3796,6,348,2017-02-20 18:23:58,http://johnston.com/mateo,,243.184.204.37,"{""location"": ""GE"", ""is_mobile"": false}" 3797,6,348,2017-06-05 18:47:36,http://toy.biz/filomena.boyle,,69.49.70.28,"{""location"": ""SY"", ""is_mobile"": false}" 3798,6,348,2017-05-20 06:45:57,http://kautzer.co/christiana.harvey,,193.120.155.208,"{""location"": ""CX"", ""is_mobile"": false}" 3799,6,348,2017-04-01 23:36:23,http://langworthklocko.name/tiffany,,211.15.109.232,"{""location"": ""AX"", ""is_mobile"": true}" 3800,6,348,2016-12-28 00:26:08,http://wilderman.io/perry_bogan,,37.224.46.26,"{""location"": ""BY"", ""is_mobile"": false}" 3801,6,349,2017-05-22 13:39:09,http://trantow.com/denis.nikolaus,,181.104.108.139,"{""location"": ""MM"", ""is_mobile"": false}" 3802,6,349,2017-05-02 23:33:47,http://mills.co/janae,,57.142.19.90,"{""location"": ""NI"", ""is_mobile"": false}" 3803,6,349,2017-02-26 23:31:12,http://cormier.biz/ebba_willms,,135.124.28.180,"{""location"": ""PW"", ""is_mobile"": true}" 3804,6,349,2017-02-02 11:45:57,http://goyette.name/lindsey,,248.251.107.180,"{""location"": ""MP"", ""is_mobile"": true}" 3805,6,349,2017-03-24 07:13:42,http://schowaltermaggio.name/ada_nader,,204.112.150.36,"{""location"": ""MN"", ""is_mobile"": false}" 3806,6,349,2017-04-08 10:23:18,http://ziemann.co/holden,,121.69.235.203,"{""location"": ""PG"", ""is_mobile"": false}" 3807,6,349,2017-03-29 05:18:41,http://yost.com/gaston_mosciski,,5.159.121.63,"{""location"": ""BH"", ""is_mobile"": false}" 3808,6,349,2016-12-20 05:12:35,http://runolfsdottirrunte.org/marcelo,,224.8.100.63,"{""location"": ""GE"", ""is_mobile"": true}" 3809,6,349,2017-03-14 00:57:04,http://goldner.biz/reie.feest,,116.54.64.97,"{""location"": ""ZA"", ""is_mobile"": true}" 3810,6,349,2017-05-19 03:11:22,http://tremblay.co/carmen,,250.208.4.101,"{""location"": ""TL"", ""is_mobile"": false}" 3811,6,349,2017-06-03 06:13:12,http://greenholt.info/kaycee,,223.217.55.166,"{""location"": ""KG"", ""is_mobile"": false}" 3812,6,350,2017-01-18 04:41:36,http://kozeyschuppe.io/lydia.mcglynn,,179.78.232.172,"{""location"": ""NE"", ""is_mobile"": true}" 3813,6,350,2016-12-25 09:01:44,http://jaskolski.com/shea,,23.87.149.216,"{""location"": ""VN"", ""is_mobile"": true}" 3814,6,350,2017-03-21 03:54:00,http://parisian.io/daphne,,205.13.95.42,"{""location"": ""GT"", ""is_mobile"": false}" 3815,6,350,2017-05-01 21:05:24,http://runolfon.co/darrin_sipes,,63.184.150.98,"{""location"": ""GQ"", ""is_mobile"": false}" 3816,6,350,2017-02-14 15:16:05,http://brakusbrakus.io/jensen,,129.89.3.209,"{""location"": ""LV"", ""is_mobile"": false}" 3817,6,350,2017-05-05 18:05:27,http://wiegand.biz/mikel.dietrich,,179.78.232.172,"{""location"": ""BL"", ""is_mobile"": true}" 3818,6,350,2017-06-04 23:18:55,http://strosin.org/dean,,68.16.181.120,"{""location"": ""BL"", ""is_mobile"": true}" 3819,6,350,2016-12-29 18:12:44,http://bailey.com/jadyn,,23.87.149.216,"{""location"": ""PS"", ""is_mobile"": false}" 3820,6,350,2017-03-13 14:30:42,http://kerluke.org/colten.gorczany,,160.53.31.165,"{""location"": ""PL"", ""is_mobile"": true}" 3821,6,350,2017-03-11 10:12:53,http://vandervort.io/estel,,63.184.150.98,"{""location"": ""VI"", ""is_mobile"": true}" 3822,6,351,2017-02-02 13:06:48,http://ortiz.io/emie_lakin,0.0000000000,220.250.70.88,"{""location"": ""CL"", ""is_mobile"": true}" 3823,6,351,2017-05-29 05:20:06,http://lindgren.biz/carol,3.0000000000,182.5.132.26,"{""location"": ""HR"", ""is_mobile"": false}" 3824,6,352,2017-04-16 12:00:23,http://bogisichfahey.net/emilie,2.0000000000,82.41.221.49,"{""location"": ""RS"", ""is_mobile"": false}" 3825,6,352,2017-02-27 06:39:26,http://boyle.net/monserrate,0.0000000000,125.2.8.27,"{""location"": ""CO"", ""is_mobile"": true}" 3826,6,352,2017-04-08 06:08:54,http://lockman.co/valerie,2.0000000000,188.61.63.58,"{""location"": ""RE"", ""is_mobile"": true}" 3827,6,352,2017-06-10 22:44:51,http://borermueller.net/zane_grimes,2.0000000000,183.32.196.171,"{""location"": ""MN"", ""is_mobile"": true}" 670,2,63,2017-01-29 16:13:31,http://schmidt.name/gerson.strosin,0.0000000000,90.157.196.215,"{""location"": ""VN"", ""is_mobile"": false}" 671,2,64,2017-04-19 11:03:45,http://eichmann.name/adam,,72.37.223.252,"{""location"": ""SH"", ""is_mobile"": false}" 672,2,64,2017-06-09 11:38:04,http://volkman.info/elta.howell,,189.58.231.216,"{""location"": ""TR"", ""is_mobile"": false}" 673,2,64,2017-02-08 13:18:35,http://mantethiel.info/gayle.ratke,,250.114.7.204,"{""location"": ""BG"", ""is_mobile"": true}" 674,2,64,2017-02-07 12:59:58,http://pouros.co/wilber,,165.162.213.186,"{""location"": ""GE"", ""is_mobile"": false}" 675,2,64,2017-03-28 23:56:12,http://kling.info/mafalda_hudson,,129.108.166.80,"{""location"": ""FJ"", ""is_mobile"": true}" 676,2,64,2016-12-30 11:14:06,http://baileyerdman.info/thea,,16.145.181.55,"{""location"": ""ES"", ""is_mobile"": true}" 677,2,64,2017-06-10 03:17:15,http://schiller.net/amiya,,198.89.119.146,"{""location"": ""NI"", ""is_mobile"": false}" 678,2,64,2017-03-16 06:42:59,http://lebsack.org/arch.fisher,,2.9.181.44,"{""location"": ""GB"", ""is_mobile"": true}" 679,2,64,2017-04-26 16:01:14,http://tillmanabshire.co/skye_hamill,,115.208.79.242,"{""location"": ""BS"", ""is_mobile"": true}" 680,2,64,2016-12-25 00:46:35,http://greenholthodkiewicz.co/millie,,16.204.208.214,"{""location"": ""ES"", ""is_mobile"": false}" 681,2,64,2017-02-10 17:44:52,http://mann.co/alfonzo.rath,,129.62.119.213,"{""location"": ""IL"", ""is_mobile"": false}" 682,2,64,2017-03-27 08:49:45,http://emard.org/wiley.davis,,129.62.119.213,"{""location"": ""PH"", ""is_mobile"": true}" 683,2,64,2017-04-19 09:54:06,http://ornkling.io/kavon.stark,,69.133.147.85,"{""location"": ""ET"", ""is_mobile"": false}" 684,2,64,2017-01-18 00:16:22,http://adamsdouglas.net/leanne_emmerich,,230.98.180.68,"{""location"": ""PG"", ""is_mobile"": false}" 685,2,65,2017-05-11 13:36:33,http://kerlukemayert.biz/casey_howell,,227.184.101.246,"{""location"": ""PM"", ""is_mobile"": false}" 686,2,65,2017-01-30 17:14:03,http://vandervortsmitham.biz/karli_mertz,,184.214.125.122,"{""location"": ""AE"", ""is_mobile"": true}" 687,2,65,2016-12-19 00:40:11,http://daugherty.net/alex,,66.252.143.149,"{""location"": ""SO"", ""is_mobile"": true}" 688,2,65,2017-05-30 06:55:37,http://rice.io/ubaldo,,247.24.25.108,"{""location"": ""MO"", ""is_mobile"": true}" 689,2,65,2017-05-03 06:04:18,http://kertzmann.biz/casey,,113.36.100.73,"{""location"": ""MH"", ""is_mobile"": true}" 690,2,65,2017-04-24 21:24:26,http://lebsack.co/oswaldo.pollich,,54.3.235.205,"{""location"": ""GL"", ""is_mobile"": false}" 691,2,65,2017-05-10 07:42:02,http://kuhic.io/aaliyah,,147.241.227.130,"{""location"": ""BR"", ""is_mobile"": true}" 692,2,65,2017-03-05 06:03:45,http://rosenbaum.biz/rosamond.gerlach,,31.7.89.67,"{""location"": ""KI"", ""is_mobile"": false}" 693,2,65,2017-06-02 07:53:31,http://orn.info/name.predovic,,162.19.98.187,"{""location"": ""RU"", ""is_mobile"": true}" 694,2,65,2017-05-13 13:09:43,http://carter.biz/enrique_nikolaus,,31.7.89.67,"{""location"": ""SK"", ""is_mobile"": true}" 695,2,65,2017-04-04 15:46:24,http://lindgren.info/jerome_bartoletti,,69.116.65.45,"{""location"": ""VI"", ""is_mobile"": false}" 696,2,65,2016-12-27 12:59:19,http://boyle.co/elias_hartmann,,224.92.149.187,"{""location"": ""LU"", ""is_mobile"": false}" 697,2,65,2017-02-22 21:42:58,http://hegmann.name/fern_mcdermott,,31.98.148.11,"{""location"": ""SM"", ""is_mobile"": false}" 698,2,66,2017-01-27 10:54:28,http://cain.io/erica,,118.116.21.133,"{""location"": ""MR"", ""is_mobile"": true}" 699,2,66,2017-04-04 11:33:43,http://nader.net/oswaldo,,213.94.223.147,"{""location"": ""NO"", ""is_mobile"": true}" 700,2,66,2017-01-24 20:34:20,http://vonruedenlehner.com/raleigh,,29.147.115.131,"{""location"": ""CO"", ""is_mobile"": false}" 701,2,66,2017-02-03 05:01:28,http://welch.com/christian_kling,,251.115.229.91,"{""location"": ""BG"", ""is_mobile"": false}" 702,2,67,2017-02-07 16:24:39,http://torphy.io/vergie,,183.77.25.246,"{""location"": ""ZM"", ""is_mobile"": true}" 703,2,67,2017-04-29 03:09:10,http://konopelski.io/bruce,,135.152.82.60,"{""location"": ""CA"", ""is_mobile"": false}" 704,2,67,2017-05-20 22:05:47,http://rodriguez.co/april.corkery,,242.240.82.43,"{""location"": ""TF"", ""is_mobile"": true}" 705,2,67,2017-05-26 01:59:39,http://steuber.biz/stephany,,144.148.223.18,"{""location"": ""CD"", ""is_mobile"": true}" 706,2,67,2017-05-22 14:00:22,http://stanton.name/ian.conn,,86.59.142.135,"{""location"": ""MK"", ""is_mobile"": false}" 707,2,67,2017-02-08 04:44:16,http://wildermanlakin.org/scotty,,177.15.254.124,"{""location"": ""VA"", ""is_mobile"": false}" 708,2,67,2017-05-17 06:50:45,http://ritchie.org/donavon.lowe,,104.223.204.123,"{""location"": ""UG"", ""is_mobile"": true}" 709,2,67,2017-01-16 23:40:54,http://bosco.com/makenna_corwin,,44.16.202.177,"{""location"": ""BB"", ""is_mobile"": false}" 710,2,67,2017-06-07 21:24:35,http://simonis.net/una,,219.61.171.236,"{""location"": ""KG"", ""is_mobile"": false}" 711,2,68,2017-05-15 02:33:48,http://renner.name/milton,,240.32.102.200,"{""location"": ""MW"", ""is_mobile"": false}" 712,2,68,2017-01-20 17:55:02,http://kling.com/lauren,,147.213.122.245,"{""location"": ""LI"", ""is_mobile"": true}" 713,2,68,2017-02-25 15:19:21,http://kuhic.info/rocky.friesen,,221.11.132.24,"{""location"": ""SZ"", ""is_mobile"": false}" 714,2,68,2017-03-14 02:09:06,http://gradyjerde.org/genevieve.okon,,165.206.27.33,"{""location"": ""RW"", ""is_mobile"": true}" 715,2,68,2017-03-17 00:37:40,http://powlowskidickens.net/emmet,,49.241.168.221,"{""location"": ""AG"", ""is_mobile"": true}" 716,2,69,2017-01-01 09:46:48,http://parker.com/yasmine,,186.204.52.137,"{""location"": ""FO"", ""is_mobile"": true}" 717,2,69,2017-04-18 04:49:05,http://dach.biz/marilyne_upton,,143.103.225.98,"{""location"": ""CN"", ""is_mobile"": false}" 718,2,69,2017-05-09 15:42:44,http://ernserschaefer.org/athena_stiedemann,,33.72.161.166,"{""location"": ""GU"", ""is_mobile"": false}" 719,2,69,2017-02-04 01:10:08,http://jakubowski.info/clyde,,109.44.215.139,"{""location"": ""VU"", ""is_mobile"": true}" 720,2,69,2017-05-27 04:59:49,http://roberts.com/janie_frami,,22.163.39.128,"{""location"": ""BR"", ""is_mobile"": false}" 721,2,69,2017-06-07 16:28:22,http://dach.info/jalyn_kulas,,132.187.19.90,"{""location"": ""RE"", ""is_mobile"": true}" 722,2,69,2016-12-26 04:15:51,http://schumm.info/meda_casper,,235.68.212.180,"{""location"": ""BM"", ""is_mobile"": true}" 723,2,69,2017-06-08 22:20:18,http://wintheiserkeeling.name/gabriel,,15.24.196.54,"{""location"": ""EC"", ""is_mobile"": false}" 724,2,69,2017-04-16 18:26:09,http://franecki.co/juwan,,220.197.155.202,"{""location"": ""BD"", ""is_mobile"": true}" 725,2,69,2017-02-24 17:20:42,http://kutchgreenholt.org/angelo.mante,,15.24.196.54,"{""location"": ""AG"", ""is_mobile"": false}" 3828,6,352,2017-02-06 10:49:49,http://altenwerthberge.info/virgil,2.0000000000,241.131.137.60,"{""location"": ""SI"", ""is_mobile"": false}" 3829,6,352,2017-01-24 10:22:31,http://homenick.org/tatyana_kreiger,1.0000000000,253.71.131.50,"{""location"": ""PS"", ""is_mobile"": false}" 3830,6,352,2017-04-05 12:43:16,http://hermistondamore.biz/faye.hudson,2.0000000000,213.67.228.221,"{""location"": ""GI"", ""is_mobile"": true}" 3831,6,352,2017-01-07 21:36:16,http://murray.info/abner,2.0000000000,253.71.131.50,"{""location"": ""CH"", ""is_mobile"": false}" 3832,6,352,2017-06-03 21:51:11,http://murphytowne.io/janiya_mcclure,0.0000000000,241.131.137.60,"{""location"": ""KZ"", ""is_mobile"": true}" 3833,6,352,2017-03-02 19:35:51,http://greenfelder.org/devan,2.0000000000,153.170.167.221,"{""location"": ""BE"", ""is_mobile"": false}" 3834,6,352,2017-03-02 03:00:04,http://schaeferabbott.com/fleta.bernier,0.0000000000,47.137.58.230,"{""location"": ""SN"", ""is_mobile"": true}" 3835,6,352,2017-01-25 13:40:05,http://ruel.biz/elliot.ledner,2.0000000000,238.156.179.135,"{""location"": ""ID"", ""is_mobile"": false}" 3836,6,352,2016-12-15 20:22:36,http://farrell.name/colby,0.0000000000,178.20.86.119,"{""location"": ""CZ"", ""is_mobile"": false}" 3837,6,352,2017-05-21 04:03:17,http://parisian.com/adonis,2.0000000000,8.6.173.37,"{""location"": ""TT"", ""is_mobile"": true}" 3838,6,352,2017-01-03 05:10:11,http://nikolauslynch.org/maria_leuschke,1.0000000000,47.96.115.132,"{""location"": ""JE"", ""is_mobile"": false}" 3839,6,352,2017-02-16 20:59:28,http://gradycasper.info/efren.terry,0.0000000000,86.253.233.107,"{""location"": ""MX"", ""is_mobile"": false}" 3840,6,352,2017-04-08 02:39:32,http://lebsack.co/jeramie.dooley,2.0000000000,31.88.68.86,"{""location"": ""AW"", ""is_mobile"": false}" 3841,6,352,2017-04-08 02:20:01,http://runolfsdottir.org/tanya_wuckert,0.0000000000,84.32.77.20,"{""location"": ""MD"", ""is_mobile"": false}" 3842,6,352,2017-02-08 05:08:23,http://wisozknienow.net/heaven,0.0000000000,47.137.58.230,"{""location"": ""MF"", ""is_mobile"": true}" 3843,6,353,2017-04-19 15:03:52,http://jenkinstorphy.net/salvatore,2.0000000000,186.197.46.47,"{""location"": ""BF"", ""is_mobile"": true}" 3844,6,353,2017-06-03 02:54:39,http://hansensenger.biz/abe.yundt,0.0000000000,106.100.10.163,"{""location"": ""PF"", ""is_mobile"": true}" 3845,6,354,2017-04-26 07:32:52,http://adamchaefer.co/carmine,1.0000000000,90.88.87.48,"{""location"": ""TN"", ""is_mobile"": false}" 3846,6,354,2017-01-21 11:47:03,http://schimmel.com/hayden.reichel,0.0000000000,158.196.251.49,"{""location"": ""HR"", ""is_mobile"": false}" 3847,6,354,2017-04-14 12:23:36,http://lowerolfson.org/westley.hodkiewicz,1.0000000000,135.149.214.156,"{""location"": ""NZ"", ""is_mobile"": true}" 3848,6,354,2017-05-17 18:26:01,http://hirthe.org/keeley,0.0000000000,49.199.186.207,"{""location"": ""AZ"", ""is_mobile"": true}" 3849,6,354,2017-02-14 12:15:40,http://heel.com/buford_dietrich,1.0000000000,150.202.94.20,"{""location"": ""HT"", ""is_mobile"": true}" 3850,6,354,2017-02-18 06:42:34,http://krajcik.biz/tiffany,1.0000000000,141.96.210.243,"{""location"": ""HK"", ""is_mobile"": false}" 3851,6,354,2017-02-05 00:35:16,http://bergstrom.co/mitchell_yundt,1.0000000000,143.48.10.190,"{""location"": ""KZ"", ""is_mobile"": false}" 3852,6,354,2017-05-10 09:47:38,http://morar.info/vidal,0.0000000000,115.237.185.209,"{""location"": ""ME"", ""is_mobile"": true}" 3853,6,354,2016-12-30 19:39:51,http://dicki.name/rudolph.windler,1.0000000000,22.35.189.232,"{""location"": ""LB"", ""is_mobile"": false}" 3854,6,354,2017-01-18 15:45:22,http://schowalter.io/juliana_bernhard,0.0000000000,71.85.15.39,"{""location"": ""LK"", ""is_mobile"": true}" 3855,6,354,2017-03-30 02:09:03,http://king.co/green,0.0000000000,49.199.186.207,"{""location"": ""RU"", ""is_mobile"": true}" 3856,6,355,2017-04-26 03:01:33,http://walkersatterfield.name/victor_buckridge,0.0000000000,166.90.45.203,"{""location"": ""TT"", ""is_mobile"": false}" 3857,6,355,2017-03-20 05:19:46,http://west.io/giovanna_tillman,0.0000000000,187.51.14.191,"{""location"": ""GN"", ""is_mobile"": false}" 3858,6,355,2017-01-26 23:50:01,http://ohara.com/gust_ledner,0.0000000000,107.169.230.172,"{""location"": ""GH"", ""is_mobile"": true}" 3859,6,355,2017-04-18 16:10:22,http://pricetorphy.com/keagan,0.0000000000,251.99.139.54,"{""location"": ""VA"", ""is_mobile"": false}" 3860,6,355,2017-02-24 06:55:51,http://miller.info/mathew,0.0000000000,109.69.242.210,"{""location"": ""CI"", ""is_mobile"": true}" 3861,6,355,2016-12-16 08:01:15,http://kunde.co/fay_corkery,0.0000000000,6.197.99.129,"{""location"": ""GR"", ""is_mobile"": false}" 3862,6,355,2017-04-13 09:03:53,http://legros.net/kendrick.quitzon,0.0000000000,217.205.215.61,"{""location"": ""TR"", ""is_mobile"": false}" 3863,6,355,2017-01-10 04:21:19,http://hegmannbernhard.info/dashawn,0.0000000000,42.150.58.145,"{""location"": ""IM"", ""is_mobile"": true}" 3864,6,355,2017-05-22 03:14:56,http://howellquitzon.io/deondre_dubuque,0.0000000000,201.125.143.5,"{""location"": ""MY"", ""is_mobile"": true}" 3865,6,355,2017-05-20 12:34:46,http://lesch.com/hank.swift,0.0000000000,216.44.115.200,"{""location"": ""EG"", ""is_mobile"": false}" 3866,6,355,2017-02-03 21:03:14,http://bednarkoelpin.com/cletus,0.0000000000,6.197.99.129,"{""location"": ""MK"", ""is_mobile"": true}" 3867,6,355,2017-04-22 16:44:44,http://abshire.net/ricky,0.0000000000,112.79.129.252,"{""location"": ""DE"", ""is_mobile"": false}" 3868,6,355,2017-02-18 12:45:59,http://lowe.org/constantin,0.0000000000,112.79.129.252,"{""location"": ""KM"", ""is_mobile"": false}" 3869,6,355,2017-03-06 17:49:11,http://ritchie.org/braulio_rogahn,0.0000000000,113.24.223.92,"{""location"": ""TT"", ""is_mobile"": false}" 3870,6,355,2016-12-28 18:48:19,http://larkintillman.org/christian,0.0000000000,209.99.242.152,"{""location"": ""VE"", ""is_mobile"": false}" 3871,6,355,2017-04-22 20:36:10,http://johnston.com/anika_murray,0.0000000000,142.220.42.13,"{""location"": ""ZW"", ""is_mobile"": true}" 3872,6,355,2016-12-29 23:43:41,http://mraz.biz/mike_hilll,0.0000000000,107.169.230.172,"{""location"": ""AX"", ""is_mobile"": true}" 3873,6,356,2017-03-12 08:48:53,http://davis.io/kirstin,1.0000000000,71.74.246.192,"{""location"": ""TF"", ""is_mobile"": false}" 3874,6,356,2017-05-27 21:29:55,http://torphy.io/jalen.nicolas,0.0000000000,151.109.253.110,"{""location"": ""SG"", ""is_mobile"": false}" 3875,6,356,2017-04-18 18:44:46,http://stoltenberg.biz/june,0.0000000000,248.157.100.104,"{""location"": ""CU"", ""is_mobile"": true}" 3876,6,356,2017-01-12 13:56:22,http://daughertyfadel.biz/alysha,0.0000000000,138.150.187.129,"{""location"": ""ML"", ""is_mobile"": false}" 3877,6,357,2016-12-26 15:25:59,http://durgan.biz/amalia,1.0000000000,205.20.117.45,"{""location"": ""AM"", ""is_mobile"": true}" 3878,6,357,2017-05-01 05:57:54,http://jacobsmraz.net/rasheed.jenkins,0.0000000000,32.85.12.192,"{""location"": ""MM"", ""is_mobile"": true}" 726,2,69,2017-04-15 06:56:55,http://littlestoltenberg.net/ted,,152.151.43.91,"{""location"": ""SR"", ""is_mobile"": true}" 727,2,70,2017-05-08 19:57:00,http://jonesnienow.net/shannon_franecki,,245.132.194.115,"{""location"": ""EG"", ""is_mobile"": true}" 728,2,70,2017-01-25 19:21:26,http://altenwerth.net/eric.streich,,71.200.107.59,"{""location"": ""SR"", ""is_mobile"": true}" 729,2,70,2017-02-23 16:25:09,http://schoenweinat.co/lula.okuneva,,189.134.57.14,"{""location"": ""VA"", ""is_mobile"": true}" 730,2,70,2017-06-02 16:03:01,http://tromp.org/keagan_wehner,,132.24.183.209,"{""location"": ""RS"", ""is_mobile"": false}" 731,2,70,2017-01-01 09:36:27,http://watsica.org/jordon,,188.86.81.231,"{""location"": ""BO"", ""is_mobile"": true}" 732,2,70,2017-05-18 01:49:24,http://daugherty.info/margarete.friesen,,157.221.235.125,"{""location"": ""BY"", ""is_mobile"": true}" 733,2,70,2017-05-22 08:21:31,http://waelchichristiansen.com/carol,,65.4.54.208,"{""location"": ""CX"", ""is_mobile"": true}" 734,2,70,2017-02-01 16:53:10,http://mosciski.net/miller,,77.145.165.197,"{""location"": ""SV"", ""is_mobile"": false}" 735,2,70,2016-12-28 03:51:35,http://johnson.io/hattie_connelly,,127.238.105.202,"{""location"": ""AU"", ""is_mobile"": false}" 736,2,70,2017-05-11 11:54:14,http://bins.info/bailee,,28.188.144.199,"{""location"": ""SI"", ""is_mobile"": false}" 737,2,70,2017-01-12 21:21:30,http://vonrueden.biz/america_sipes,,128.76.254.135,"{""location"": ""TH"", ""is_mobile"": false}" 738,2,70,2016-12-13 10:44:56,http://nikolaus.io/bert_rosenbaum,,233.154.134.69,"{""location"": ""BZ"", ""is_mobile"": false}" 739,2,70,2017-05-18 11:11:33,http://volkman.net/ansel,,189.134.57.14,"{""location"": ""TH"", ""is_mobile"": true}" 740,2,70,2016-12-13 06:53:02,http://connelly.co/wilhelmine_welch,,160.173.211.112,"{""location"": ""MO"", ""is_mobile"": false}" 741,2,70,2017-06-13 17:50:22,http://emmerichjacobson.info/dejon_yost,,150.100.82.111,"{""location"": ""WF"", ""is_mobile"": true}" 742,2,70,2017-05-30 08:30:25,http://gerhold.co/esther.lind,,65.4.54.208,"{""location"": ""MS"", ""is_mobile"": true}" 743,2,70,2017-04-30 01:36:51,http://colezemlak.io/devante.cartwright,,240.239.78.233,"{""location"": ""IT"", ""is_mobile"": true}" 744,2,71,2016-12-22 20:42:45,http://howell.io/ed,,190.203.67.98,"{""location"": ""EE"", ""is_mobile"": false}" 745,2,71,2017-05-10 08:44:30,http://batzbuckridge.co/brain,,96.16.109.58,"{""location"": ""HR"", ""is_mobile"": false}" 746,2,71,2017-01-19 07:01:56,http://purdybeahan.info/zita,,69.55.203.19,"{""location"": ""VI"", ""is_mobile"": true}" 747,2,72,2017-01-28 01:57:19,http://wunsch.com/jerald.yundt,,14.29.229.209,"{""location"": ""RO"", ""is_mobile"": true}" 748,2,72,2017-04-11 08:09:40,http://herzogstreich.net/thelma_beatty,,83.181.47.51,"{""location"": ""UG"", ""is_mobile"": true}" 749,2,72,2017-04-17 14:36:21,http://cronabarton.biz/cale_shanahan,,169.244.74.103,"{""location"": ""TR"", ""is_mobile"": true}" 750,2,72,2017-04-18 21:28:02,http://nienow.com/lewis.bashirian,,180.223.78.82,"{""location"": ""GW"", ""is_mobile"": false}" 751,2,72,2017-02-04 23:09:21,http://bechtelar.net/edd_berge,,14.29.229.209,"{""location"": ""LC"", ""is_mobile"": false}" 752,2,72,2017-01-18 15:17:23,http://kutch.name/evert_okeefe,,151.53.226.48,"{""location"": ""GH"", ""is_mobile"": true}" 753,2,72,2017-05-02 11:11:45,http://harber.com/dave_green,,151.53.226.48,"{""location"": ""LI"", ""is_mobile"": false}" 754,2,72,2017-02-08 07:55:16,http://murphy.biz/rogers,,171.15.188.220,"{""location"": ""IS"", ""is_mobile"": true}" 755,2,72,2017-04-13 02:43:39,http://kunde.co/marlin.swaniawski,,167.7.40.183,"{""location"": ""PF"", ""is_mobile"": false}" 756,2,72,2016-12-25 10:24:54,http://wunschlittle.org/trevion_kshlerin,,171.15.188.220,"{""location"": ""DK"", ""is_mobile"": false}" 757,2,72,2017-03-11 15:05:38,http://blick.org/nicole_king,,180.223.78.82,"{""location"": ""CW"", ""is_mobile"": false}" 758,2,73,2017-02-04 17:31:19,http://price.io/filiberto_mills,,186.190.179.110,"{""location"": ""EH"", ""is_mobile"": false}" 759,2,73,2017-05-23 08:34:16,http://heller.net/teagan.lehner,,119.77.12.204,"{""location"": ""TC"", ""is_mobile"": false}" 760,2,73,2017-04-21 11:18:46,http://armstrongheaney.info/thea,,32.179.175.95,"{""location"": ""PT"", ""is_mobile"": true}" 761,2,73,2017-01-02 12:42:06,http://satterfieldcarter.io/oscar.collier,,23.228.236.126,"{""location"": ""NZ"", ""is_mobile"": true}" 762,2,73,2017-05-02 07:01:53,http://nikolauskunde.com/judson_bernhard,,201.189.136.201,"{""location"": ""DM"", ""is_mobile"": false}" 763,2,73,2017-05-24 22:21:00,http://jast.name/jamir,,204.22.203.208,"{""location"": ""IQ"", ""is_mobile"": true}" 764,2,73,2017-06-09 08:18:28,http://bosco.net/caandre_bruen,,103.94.247.167,"{""location"": ""UY"", ""is_mobile"": true}" 765,2,73,2017-01-25 11:25:19,http://schambergeradams.com/gina,,201.189.136.201,"{""location"": ""KZ"", ""is_mobile"": true}" 766,2,73,2017-06-06 03:44:35,http://rath.net/ofelia_kiehn,,114.44.50.224,"{""location"": ""DK"", ""is_mobile"": true}" 767,2,73,2017-03-23 03:50:55,http://schuster.org/arlie,,48.131.138.226,"{""location"": ""US"", ""is_mobile"": true}" 768,2,73,2017-05-09 09:02:47,http://hodkiewicz.co/octavia,,239.211.230.200,"{""location"": ""EH"", ""is_mobile"": false}" 769,2,73,2017-06-07 06:09:31,http://hirthe.name/felicia,,107.137.127.179,"{""location"": ""IN"", ""is_mobile"": true}" 770,2,73,2017-03-17 20:25:25,http://leannon.co/dolly.beier,,200.75.60.70,"{""location"": ""PS"", ""is_mobile"": true}" 771,2,73,2017-01-30 09:56:07,http://millsrodriguez.org/tiana,,84.57.92.120,"{""location"": ""PM"", ""is_mobile"": false}" 772,2,73,2017-03-14 22:17:42,http://lowemiller.io/grant,,6.131.100.251,"{""location"": ""ZW"", ""is_mobile"": true}" 773,2,73,2016-12-16 18:44:58,http://sanfordweimann.co/lydia,,239.211.230.200,"{""location"": ""FK"", ""is_mobile"": false}" 774,2,74,2016-12-14 23:20:13,http://schroederwhite.org/halle,,3.150.124.36,"{""location"": ""TJ"", ""is_mobile"": true}" 775,2,74,2017-03-22 01:24:33,http://reingerhowe.org/dejuan_parisian,,124.62.249.97,"{""location"": ""VE"", ""is_mobile"": false}" 776,2,74,2017-05-20 07:22:15,http://cormierhaag.net/jesus_klocko,,221.173.46.105,"{""location"": ""MH"", ""is_mobile"": true}" 777,2,74,2017-05-20 20:53:21,http://hermiston.co/isabell_paucek,,95.214.40.90,"{""location"": ""ML"", ""is_mobile"": true}" 778,2,74,2017-04-10 13:17:27,http://larson.io/elton,,3.150.124.36,"{""location"": ""VA"", ""is_mobile"": false}" 779,2,74,2017-05-28 15:05:18,http://friesenzieme.org/norwood.orn,,119.247.161.123,"{""location"": ""KN"", ""is_mobile"": true}" 780,2,74,2016-12-21 06:09:26,http://reinger.net/mylene_gleichner,,177.61.32.107,"{""location"": ""AM"", ""is_mobile"": true}" 781,2,74,2017-04-30 06:12:30,http://cremin.co/amina,,112.58.59.207,"{""location"": ""TV"", ""is_mobile"": true}" 3879,6,357,2017-05-05 16:40:27,http://hartmanncollier.biz/marge,1.0000000000,201.212.10.254,"{""location"": ""WS"", ""is_mobile"": true}" 3880,6,357,2017-03-08 11:20:32,http://koch.name/shea_wiza,0.0000000000,65.203.234.226,"{""location"": ""DM"", ""is_mobile"": false}" 3881,6,357,2017-01-23 07:44:52,http://harber.net/selena.nitzsche,2.0000000000,149.237.66.14,"{""location"": ""TV"", ""is_mobile"": false}" 3882,6,357,2017-01-29 12:33:59,http://ruelbailey.org/nedra,2.0000000000,153.48.54.234,"{""location"": ""KP"", ""is_mobile"": true}" 3883,6,357,2016-12-15 02:07:17,http://connellywaelchi.com/nathanael,1.0000000000,195.114.174.18,"{""location"": ""MO"", ""is_mobile"": false}" 3884,6,357,2017-03-09 04:12:23,http://schimmel.name/larry.waelchi,2.0000000000,28.113.123.24,"{""location"": ""NZ"", ""is_mobile"": false}" 3885,6,357,2017-01-24 19:50:49,http://schmelerconnelly.biz/stacy.conn,2.0000000000,105.191.204.52,"{""location"": ""PR"", ""is_mobile"": false}" 3886,6,357,2017-01-21 16:11:33,http://bartell.name/jennyfer,1.0000000000,192.120.183.37,"{""location"": ""BG"", ""is_mobile"": false}" 3887,6,357,2017-03-24 14:19:35,http://satterfield.biz/joesph,2.0000000000,195.114.174.18,"{""location"": ""HT"", ""is_mobile"": true}" 3888,6,357,2017-04-10 23:25:29,http://powlowski.org/cleta,0.0000000000,111.34.217.211,"{""location"": ""ET"", ""is_mobile"": true}" 3889,6,357,2017-02-08 10:49:09,http://renner.biz/geraldine_bernhard,0.0000000000,192.120.183.37,"{""location"": ""HK"", ""is_mobile"": true}" 3890,6,357,2017-03-27 13:39:53,http://kiehn.org/gonzalo_considine,2.0000000000,214.34.91.115,"{""location"": ""BA"", ""is_mobile"": true}" 3891,6,357,2017-01-29 18:03:59,http://veumhansen.biz/claude,0.0000000000,61.213.239.33,"{""location"": ""SG"", ""is_mobile"": true}" 3892,6,358,2017-05-30 02:41:37,http://rennerosinski.org/katherine,3.0000000000,235.85.225.137,"{""location"": ""RW"", ""is_mobile"": true}" 3893,6,358,2017-03-16 10:20:27,http://rolfson.co/enola,2.0000000000,176.20.240.216,"{""location"": ""SG"", ""is_mobile"": false}" 3894,6,358,2017-04-30 21:13:03,http://hilpert.org/vallie,0.0000000000,108.207.85.139,"{""location"": ""SR"", ""is_mobile"": true}" 3895,6,358,2017-04-30 00:37:38,http://pacocha.biz/jeanette.gibson,0.0000000000,235.85.225.137,"{""location"": ""CI"", ""is_mobile"": true}" 3896,6,358,2017-04-17 09:38:26,http://weimann.org/michael_batz,1.0000000000,213.232.26.245,"{""location"": ""BM"", ""is_mobile"": false}" 3897,6,358,2017-06-02 13:06:51,http://okeefebednar.org/vance_mann,0.0000000000,252.53.183.52,"{""location"": ""AR"", ""is_mobile"": false}" 3898,6,358,2017-04-30 12:15:22,http://larson.info/karianne_boyer,2.0000000000,176.20.240.216,"{""location"": ""TH"", ""is_mobile"": true}" 3899,6,359,2016-12-26 05:38:27,http://ruecker.net/cordelia,3.0000000000,114.187.179.174,"{""location"": ""GW"", ""is_mobile"": true}" 3900,6,360,2017-01-27 03:46:25,http://wiza.net/sonya.mayer,2.0000000000,85.116.149.16,"{""location"": ""TO"", ""is_mobile"": true}" 3901,6,360,2016-12-31 03:11:14,http://lindgrendavis.com/felton,1.0000000000,85.116.149.16,"{""location"": ""TZ"", ""is_mobile"": true}" 3902,6,360,2017-05-01 16:24:04,http://daughertyeichmann.co/shad,1.0000000000,102.44.196.132,"{""location"": ""GH"", ""is_mobile"": false}" 3903,6,360,2016-12-22 12:52:53,http://streich.info/odie,0.0000000000,111.98.79.129,"{""location"": ""NO"", ""is_mobile"": false}" 3904,6,360,2017-04-30 22:31:23,http://lubowitzrenner.info/rusty.olson,1.0000000000,248.194.192.127,"{""location"": ""NZ"", ""is_mobile"": false}" 3905,6,360,2017-01-10 18:35:10,http://runte.com/lewis,0.0000000000,248.194.192.127,"{""location"": ""BH"", ""is_mobile"": true}" 3906,6,361,2017-06-10 17:20:10,http://wuckert.co/edmund,0.0000000000,54.188.235.36,"{""location"": ""MQ"", ""is_mobile"": true}" 3907,6,361,2016-12-25 05:45:39,http://dicki.net/lizzie_bogisich,1.0000000000,123.49.27.110,"{""location"": ""PH"", ""is_mobile"": false}" 3908,6,361,2017-05-03 18:34:14,http://white.org/payton.lubowitz,2.0000000000,14.179.128.90,"{""location"": ""BE"", ""is_mobile"": false}" 3909,6,361,2017-02-03 21:29:24,http://wisokywintheiser.io/alfonso_dooley,1.0000000000,120.181.159.198,"{""location"": ""TD"", ""is_mobile"": false}" 3910,6,361,2017-02-18 03:22:20,http://fritsch.io/ewald,2.0000000000,233.134.209.182,"{""location"": ""AU"", ""is_mobile"": false}" 3911,6,361,2017-03-22 03:02:58,http://rosenbaumfahey.biz/ricardo_swaniawski,0.0000000000,244.133.70.218,"{""location"": ""VN"", ""is_mobile"": false}" 3912,6,361,2017-01-27 09:47:24,http://howe.net/nathan,0.0000000000,123.49.27.110,"{""location"": ""BQ"", ""is_mobile"": false}" 3913,6,361,2016-12-26 12:26:09,http://okuneva.net/rubye,0.0000000000,36.111.218.159,"{""location"": ""SX"", ""is_mobile"": true}" 3914,6,361,2017-01-20 12:04:54,http://trompkeler.net/raegan.johnson,0.0000000000,175.145.161.162,"{""location"": ""PG"", ""is_mobile"": true}" 3915,6,361,2017-03-18 14:43:05,http://adams.info/estrella,0.0000000000,242.195.35.51,"{""location"": ""PK"", ""is_mobile"": true}" 3916,6,362,2017-03-21 11:20:05,http://bergedenesik.io/jee,1.0000000000,127.248.215.175,"{""location"": ""UY"", ""is_mobile"": false}" 3917,6,362,2016-12-18 11:15:34,http://swaniawski.org/hilma_turner,2.0000000000,62.82.184.3,"{""location"": ""VE"", ""is_mobile"": false}" 3918,6,362,2016-12-15 14:33:10,http://murray.info/clementina_heidenreich,2.0000000000,79.123.241.58,"{""location"": ""SI"", ""is_mobile"": true}" 3919,6,363,2017-05-07 19:44:26,http://stoltenberg.org/quinn_marvin,0.0000000000,145.20.107.44,"{""location"": ""BQ"", ""is_mobile"": true}" 3920,6,363,2017-03-23 21:09:40,http://borertowne.info/julius,0.0000000000,42.96.151.136,"{""location"": ""VA"", ""is_mobile"": false}" 3921,6,363,2016-12-14 21:53:09,http://mcdermott.info/morris,0.0000000000,29.175.119.115,"{""location"": ""MC"", ""is_mobile"": false}" 3922,6,363,2017-04-12 12:40:30,http://sanfordlangworth.co/keely.wehner,0.0000000000,74.38.34.190,"{""location"": ""MA"", ""is_mobile"": false}" 3923,6,363,2017-01-15 09:46:22,http://graham.org/uriah.huels,0.0000000000,134.8.110.230,"{""location"": ""AU"", ""is_mobile"": false}" 3924,6,363,2017-01-21 21:30:30,http://koch.biz/lizzie.heathcote,0.0000000000,109.64.133.53,"{""location"": ""BN"", ""is_mobile"": false}" 3925,6,363,2017-02-19 05:00:44,http://skiles.net/darron,0.0000000000,152.210.235.21,"{""location"": ""ES"", ""is_mobile"": false}" 3926,6,363,2017-04-06 22:54:39,http://senger.io/lisette.gutmann,0.0000000000,18.185.175.133,"{""location"": ""GH"", ""is_mobile"": true}" 3927,6,363,2017-02-23 11:46:17,http://macejkovicgerhold.io/casey_wisoky,0.0000000000,74.38.34.190,"{""location"": ""GW"", ""is_mobile"": true}" 3928,6,363,2017-04-30 23:42:47,http://abernathy.org/belle_stehr,0.0000000000,177.41.161.59,"{""location"": ""BA"", ""is_mobile"": true}" 3929,6,363,2017-01-10 07:39:28,http://hills.co/cameron.boehm,0.0000000000,4.172.224.221,"{""location"": ""KM"", ""is_mobile"": true}" 3930,6,363,2017-04-16 19:33:05,http://kertzmann.net/pasquale,0.0000000000,209.66.113.241,"{""location"": ""NZ"", ""is_mobile"": false}" 782,2,74,2017-01-16 13:10:32,http://littelkeeling.co/jerod_rolfson,,32.219.151.162,"{""location"": ""MR"", ""is_mobile"": true}" 783,2,74,2016-12-15 02:45:23,http://walker.info/maximillian.rosenbaum,,95.214.40.90,"{""location"": ""UZ"", ""is_mobile"": true}" 784,2,74,2017-01-25 02:41:24,http://roberts.io/keith.dickinson,,119.247.161.123,"{""location"": ""SD"", ""is_mobile"": true}" 785,2,74,2016-12-13 08:00:04,http://wisozk.biz/delphia,,61.26.6.126,"{""location"": ""MF"", ""is_mobile"": true}" 786,2,74,2017-01-16 17:51:05,http://yundtwilliamson.io/diego_koch,,124.62.249.97,"{""location"": ""HU"", ""is_mobile"": false}" 787,2,74,2017-03-30 03:46:51,http://mraz.biz/fiona,,221.173.46.105,"{""location"": ""IE"", ""is_mobile"": false}" 788,2,74,2017-03-12 13:41:51,http://littelbayer.co/alexane.blanda,,108.107.121.161,"{""location"": ""HR"", ""is_mobile"": false}" 789,2,74,2017-06-10 13:35:17,http://boehm.co/kelton,,95.214.40.90,"{""location"": ""GR"", ""is_mobile"": true}" 790,2,75,2017-06-08 18:18:27,http://mcglynn.org/francesco.wisozk,,134.83.37.40,"{""location"": ""CR"", ""is_mobile"": false}" 791,2,75,2017-02-22 18:14:11,http://sipetokes.info/aracely_hudson,,48.210.93.176,"{""location"": ""NG"", ""is_mobile"": false}" 792,2,75,2017-02-18 19:30:14,http://littel.io/newell.marquardt,,87.137.232.101,"{""location"": ""BB"", ""is_mobile"": false}" 793,2,75,2017-05-18 13:03:54,http://herzog.com/margarette,,192.29.43.24,"{""location"": ""YE"", ""is_mobile"": false}" 794,2,75,2017-01-06 01:25:17,http://mueller.biz/tianna,,47.101.67.125,"{""location"": ""HK"", ""is_mobile"": true}" 795,2,75,2017-02-21 16:53:25,http://ankunding.biz/bertha,,21.239.210.51,"{""location"": ""GG"", ""is_mobile"": false}" 796,2,75,2016-12-22 04:36:16,http://kshlerinfeest.com/gunner,,6.19.183.245,"{""location"": ""TN"", ""is_mobile"": false}" 797,2,75,2017-03-15 13:04:46,http://hauck.org/velma,,110.134.245.74,"{""location"": ""CN"", ""is_mobile"": true}" 798,2,76,2017-03-01 01:19:31,http://lemke.io/kirstin.morar,,183.3.215.229,"{""location"": ""DJ"", ""is_mobile"": true}" 799,2,76,2017-04-01 18:09:53,http://runtecain.io/sierra_nienow,,179.2.142.51,"{""location"": ""KG"", ""is_mobile"": false}" 800,2,76,2017-03-04 17:17:47,http://leffler.net/ransom,,137.6.143.20,"{""location"": ""HM"", ""is_mobile"": false}" 801,2,76,2017-01-02 11:30:24,http://langworthbreitenberg.info/demarcus.daniel,,243.224.149.185,"{""location"": ""IR"", ""is_mobile"": false}" 802,2,76,2017-01-02 11:33:17,http://dickensmitchell.info/ellis,,208.195.59.250,"{""location"": ""GL"", ""is_mobile"": false}" 803,2,76,2017-05-19 04:42:27,http://smith.io/rhoda.mills,,5.90.248.133,"{""location"": ""MT"", ""is_mobile"": false}" 804,2,76,2017-05-06 23:27:38,http://ko.io/jett_kovacek,,153.111.253.194,"{""location"": ""BW"", ""is_mobile"": false}" 805,2,76,2017-01-27 01:01:30,http://schaefer.co/alene_champlin,,142.113.5.240,"{""location"": ""MY"", ""is_mobile"": true}" 806,2,76,2017-05-05 02:03:56,http://millchroeder.com/colton_grant,,208.195.59.250,"{""location"": ""CX"", ""is_mobile"": false}" 807,2,76,2017-06-02 11:27:50,http://lubowitzsteuber.net/manuela.grant,,137.6.143.20,"{""location"": ""KY"", ""is_mobile"": true}" 808,2,76,2017-05-02 12:29:46,http://donnelly.net/kiarra_marvin,,205.157.141.233,"{""location"": ""MF"", ""is_mobile"": false}" 809,2,76,2017-06-11 05:27:52,http://beer.net/haley_kuvalis,,201.46.216.193,"{""location"": ""CI"", ""is_mobile"": true}" 810,2,76,2017-02-10 15:54:47,http://langosh.biz/oswaldo_cartwright,,107.195.136.143,"{""location"": ""IN"", ""is_mobile"": true}" 811,2,76,2016-12-25 19:15:55,http://wymangoldner.biz/raquel_bins,,183.3.215.229,"{""location"": ""VC"", ""is_mobile"": true}" 812,2,76,2016-12-22 13:33:13,http://spinka.co/pierre.kris,,107.195.136.143,"{""location"": ""BM"", ""is_mobile"": true}" 813,2,77,2017-06-01 14:16:06,http://spinkacruickshank.co/sincere.hoeger,,205.40.30.246,"{""location"": ""IE"", ""is_mobile"": false}" 814,2,77,2017-01-18 15:21:06,http://beatty.com/izaiah.wolff,,157.199.53.135,"{""location"": ""PF"", ""is_mobile"": false}" 815,2,77,2017-05-17 00:52:10,http://kozeypredovic.info/erin.runolfon,,28.227.225.155,"{""location"": ""KG"", ""is_mobile"": false}" 816,2,77,2017-02-20 07:42:54,http://heaney.biz/larue,,198.67.77.138,"{""location"": ""HR"", ""is_mobile"": false}" 817,2,77,2017-05-06 07:06:18,http://gottliebschimmel.com/rebekah,,23.240.244.244,"{""location"": ""KG"", ""is_mobile"": true}" 818,2,77,2017-06-04 19:57:55,http://schulist.info/bertrand.boyer,,227.112.35.97,"{""location"": ""EH"", ""is_mobile"": false}" 819,2,77,2017-04-22 05:56:30,http://hahn.com/verdie,,222.217.196.181,"{""location"": ""TM"", ""is_mobile"": false}" 820,2,78,2017-04-14 16:02:27,http://skiles.com/felipe_johns,,117.10.186.121,"{""location"": ""NA"", ""is_mobile"": false}" 821,2,78,2016-12-23 00:37:51,http://koeppgrady.io/sibyl.hoppe,,218.83.38.201,"{""location"": ""VE"", ""is_mobile"": true}" 822,2,78,2017-04-15 17:06:01,http://emmerich.info/kayla_cruickshank,,234.123.37.216,"{""location"": ""DO"", ""is_mobile"": false}" 823,2,78,2017-06-04 03:22:24,http://glover.com/misty,,39.207.194.120,"{""location"": ""FM"", ""is_mobile"": false}" 824,2,78,2017-02-12 14:03:49,http://rodriguez.com/ethan,,30.183.44.48,"{""location"": ""CX"", ""is_mobile"": false}" 825,2,78,2017-03-08 17:31:22,http://reinger.com/justyn_hayes,,30.183.44.48,"{""location"": ""KP"", ""is_mobile"": true}" 826,2,78,2017-06-01 04:54:09,http://bruen.co/aylin.little,,146.252.155.201,"{""location"": ""KR"", ""is_mobile"": true}" 827,2,78,2017-05-16 12:15:12,http://grady.net/syble,,60.219.185.33,"{""location"": ""BY"", ""is_mobile"": true}" 828,2,78,2017-04-11 12:55:21,http://boganprice.name/mohammed,,218.83.38.201,"{""location"": ""MS"", ""is_mobile"": false}" 829,2,78,2017-01-13 17:33:53,http://olsonokeefe.org/adriana.wolff,,234.123.37.216,"{""location"": ""GI"", ""is_mobile"": false}" 830,2,78,2017-03-09 14:50:44,http://wehner.co/armand_heller,,117.10.186.121,"{""location"": ""CA"", ""is_mobile"": false}" 831,2,78,2017-02-08 12:44:00,http://rowe.biz/mittie.grant,,117.10.186.121,"{""location"": ""AS"", ""is_mobile"": false}" 832,2,79,2017-03-14 16:37:19,http://batz.net/moses,,176.76.241.79,"{""location"": ""TZ"", ""is_mobile"": false}" 833,2,79,2017-05-30 17:48:32,http://hodkiewicz.biz/uriel.schaden,,63.238.24.176,"{""location"": ""VA"", ""is_mobile"": true}" 834,2,79,2017-02-26 00:34:56,http://donnellyharber.biz/wilbert.hodkiewicz,,167.77.167.229,"{""location"": ""UA"", ""is_mobile"": false}" 835,2,80,2017-01-26 00:00:36,http://wuckertbosco.org/rose_konopelski,,31.185.243.211,"{""location"": ""GS"", ""is_mobile"": true}" 836,2,80,2017-03-14 15:07:12,http://auer.org/claud,,14.48.181.30,"{""location"": ""NP"", ""is_mobile"": true}" 837,2,80,2017-05-11 19:50:19,http://stehrkuhic.name/kiarra,,67.245.90.212,"{""location"": ""GE"", ""is_mobile"": false}" 3931,6,363,2017-01-15 00:23:37,http://white.biz/tyra.kuvalis,0.0000000000,4.172.224.221,"{""location"": ""UA"", ""is_mobile"": false}" 3932,6,363,2016-12-23 12:40:28,http://jacobsbosco.name/genoveva,0.0000000000,65.12.166.72,"{""location"": ""YT"", ""is_mobile"": true}" 3933,6,363,2017-01-15 22:13:18,http://harberhomenick.name/stacey,0.0000000000,90.219.235.249,"{""location"": ""MD"", ""is_mobile"": false}" 3934,6,363,2017-02-03 14:33:40,http://rolfson.org/nicklaus,0.0000000000,199.191.169.77,"{""location"": ""CW"", ""is_mobile"": true}" 3935,6,363,2017-04-25 06:50:58,http://lemke.net/ibrahim_miller,0.0000000000,145.20.107.44,"{""location"": ""CN"", ""is_mobile"": true}" 3936,6,363,2017-04-02 21:02:29,http://schneider.io/shad.lueilwitz,0.0000000000,116.193.180.201,"{""location"": ""AM"", ""is_mobile"": false}" 3937,6,363,2017-04-12 12:57:03,http://feil.org/gaylord,0.0000000000,99.85.112.98,"{""location"": ""FK"", ""is_mobile"": false}" 3938,6,364,2017-02-07 14:39:29,http://donnelly.org/aurore,3.0000000000,55.9.135.106,"{""location"": ""JP"", ""is_mobile"": false}" 3939,6,364,2017-05-03 02:54:02,http://leannon.biz/lola,0.0000000000,183.171.103.52,"{""location"": ""MK"", ""is_mobile"": true}" 3940,6,364,2017-05-21 07:43:23,http://zulauf.co/elsie.feeney,3.0000000000,30.251.10.66,"{""location"": ""MC"", ""is_mobile"": false}" 3941,6,364,2017-04-08 17:47:55,http://weinatdubuque.org/kayli_mccullough,2.0000000000,244.65.159.248,"{""location"": ""LR"", ""is_mobile"": true}" 3942,6,364,2017-01-23 12:02:54,http://crona.info/jeanne,2.0000000000,101.14.86.95,"{""location"": ""SG"", ""is_mobile"": false}" 3943,6,364,2017-02-20 18:16:56,http://lakin.info/tod,3.0000000000,190.243.72.46,"{""location"": ""KR"", ""is_mobile"": false}" 3944,6,364,2017-04-30 11:17:02,http://armstrong.com/hallie.bogisich,2.0000000000,220.218.130.3,"{""location"": ""CG"", ""is_mobile"": true}" 3945,6,364,2017-04-03 03:33:41,http://jast.biz/kathleen,2.0000000000,3.91.225.165,"{""location"": ""VU"", ""is_mobile"": false}" 3946,6,364,2017-04-30 19:45:46,http://haley.io/retta.mayer,1.0000000000,216.110.55.154,"{""location"": ""GS"", ""is_mobile"": false}" 3947,6,364,2017-05-08 12:39:52,http://bins.org/bernita,0.0000000000,56.176.124.91,"{""location"": ""BD"", ""is_mobile"": true}" 3948,6,364,2017-03-17 17:53:27,http://rodriguez.io/gretchen.pagac,1.0000000000,49.176.236.126,"{""location"": ""ET"", ""is_mobile"": true}" 3949,6,364,2017-05-26 22:49:26,http://monahanfahey.com/dayna,2.0000000000,209.109.230.161,"{""location"": ""MV"", ""is_mobile"": true}" 3950,6,364,2017-01-18 00:57:49,http://dicki.net/velma,1.0000000000,209.109.230.161,"{""location"": ""PN"", ""is_mobile"": false}" 3951,6,364,2017-01-08 03:02:04,http://wildermanbernhard.name/jaime,0.0000000000,56.176.124.91,"{""location"": ""YT"", ""is_mobile"": false}" 3952,6,364,2017-05-27 08:56:39,http://carroll.name/mark,3.0000000000,88.246.53.133,"{""location"": ""NF"", ""is_mobile"": true}" 3953,6,364,2017-01-10 00:03:40,http://franecki.biz/chandler,1.0000000000,200.74.75.208,"{""location"": ""GD"", ""is_mobile"": false}" 3954,6,365,2017-05-31 18:47:12,http://veum.co/milford_will,2.0000000000,18.164.162.8,"{""location"": ""DE"", ""is_mobile"": false}" 3955,6,365,2017-02-21 03:12:41,http://kiehn.name/jovani.howell,2.0000000000,37.254.51.13,"{""location"": ""MT"", ""is_mobile"": false}" 3956,6,365,2017-02-03 07:25:19,http://nader.info/hailey,0.0000000000,83.86.222.238,"{""location"": ""VA"", ""is_mobile"": true}" 3957,6,365,2017-03-29 11:08:50,http://dietrich.net/ardella,2.0000000000,179.46.188.252,"{""location"": ""FO"", ""is_mobile"": true}" 3958,6,365,2017-03-04 22:07:05,http://ziemann.net/tremaine,0.0000000000,218.125.53.176,"{""location"": ""BW"", ""is_mobile"": true}" 3959,6,366,2016-12-15 19:32:45,http://feeneyleuschke.info/golden.ohara,3.0000000000,195.48.66.144,"{""location"": ""ER"", ""is_mobile"": true}" 3960,6,366,2017-03-19 18:07:42,http://stokes.biz/amiya.hilpert,3.0000000000,31.38.39.35,"{""location"": ""LY"", ""is_mobile"": false}" 3961,6,366,2016-12-31 21:07:50,http://franecki.name/jackeline,1.0000000000,73.22.57.77,"{""location"": ""MY"", ""is_mobile"": false}" 3962,6,366,2017-03-17 10:24:46,http://ruelsmitham.org/jarret,0.0000000000,107.215.232.15,"{""location"": ""AE"", ""is_mobile"": false}" 3963,6,366,2017-03-28 23:27:26,http://kreiger.co/eulalia_kovacek,0.0000000000,31.38.39.35,"{""location"": ""CG"", ""is_mobile"": true}" 3964,6,367,2017-03-04 08:41:10,http://price.biz/marilie,,229.203.73.132,"{""location"": ""IL"", ""is_mobile"": false}" 3965,6,367,2017-02-26 19:22:11,http://stantongreenfelder.com/ryann,,53.96.23.40,"{""location"": ""KP"", ""is_mobile"": false}" 3966,6,367,2017-05-18 18:35:11,http://mosciski.info/delaney.ullrich,,29.158.244.71,"{""location"": ""TT"", ""is_mobile"": false}" 3967,6,367,2017-05-31 05:01:13,http://spinkamcclure.info/kolby,,241.39.246.7,"{""location"": ""KP"", ""is_mobile"": true}" 3968,6,367,2017-01-13 15:59:24,http://wunsch.org/erling.heaney,,225.150.43.192,"{""location"": ""JE"", ""is_mobile"": true}" 3969,6,367,2017-05-24 17:39:40,http://ferry.org/paolo.rogahn,,161.177.132.29,"{""location"": ""ES"", ""is_mobile"": true}" 3970,6,367,2017-02-01 14:40:05,http://dietrich.info/hildegard,,79.176.47.13,"{""location"": ""CN"", ""is_mobile"": true}" 3971,6,367,2017-04-23 18:21:21,http://okeefe.net/ethan,,2.216.162.28,"{""location"": ""KR"", ""is_mobile"": true}" 3972,6,367,2017-04-07 20:07:17,http://bradtkeoconnell.io/kathryn,,145.25.23.221,"{""location"": ""CZ"", ""is_mobile"": false}" 3973,6,368,2017-04-21 00:43:31,http://mckenziedickens.org/anabelle,,95.159.156.158,"{""location"": ""IM"", ""is_mobile"": true}" 3974,6,368,2016-12-18 05:16:45,http://huels.co/kayla.torphy,,63.242.161.113,"{""location"": ""FK"", ""is_mobile"": true}" 3975,6,368,2017-04-05 18:45:30,http://dietrich.co/maggie.klocko,,109.117.98.56,"{""location"": ""TW"", ""is_mobile"": false}" 3976,6,368,2017-05-05 20:20:37,http://daniel.info/rozella,,63.242.161.113,"{""location"": ""NZ"", ""is_mobile"": true}" 3977,6,368,2017-03-14 03:10:01,http://blick.name/retha.smith,,143.181.3.95,"{""location"": ""GM"", ""is_mobile"": true}" 3978,6,369,2017-01-12 17:02:47,http://bins.org/maggie,,32.26.184.232,"{""location"": ""NG"", ""is_mobile"": false}" 3979,6,369,2017-01-09 14:05:59,http://stracke.com/ashtyn,,21.210.239.237,"{""location"": ""AM"", ""is_mobile"": false}" 3980,6,369,2017-02-21 22:15:47,http://mooreblanda.io/lorenz,,150.48.41.74,"{""location"": ""SO"", ""is_mobile"": false}" 3981,6,369,2017-01-05 08:28:30,http://buckridge.com/athena,,254.153.64.167,"{""location"": ""AW"", ""is_mobile"": true}" 3982,6,370,2017-03-07 23:58:50,http://paucekshields.co/jenifer.king,,8.114.52.81,"{""location"": ""PH"", ""is_mobile"": true}" 3983,6,370,2017-06-12 10:22:12,http://tromp.info/carmel,,119.115.170.115,"{""location"": ""CU"", ""is_mobile"": false}" 838,2,80,2017-06-10 19:58:59,http://connelly.name/avery_schimmel,,67.245.90.212,"{""location"": ""MK"", ""is_mobile"": true}" 839,2,80,2017-01-16 11:53:57,http://parker.org/demetrius,,103.126.177.58,"{""location"": ""BY"", ""is_mobile"": true}" 840,2,80,2017-02-28 16:03:28,http://upton.co/leif.bernier,,234.228.81.84,"{""location"": ""WF"", ""is_mobile"": false}" 841,2,80,2017-02-11 13:31:25,http://stark.io/madaline,,136.204.143.202,"{""location"": ""AU"", ""is_mobile"": false}" 842,2,80,2017-06-05 19:37:46,http://reynolds.info/roberto_rau,,136.204.143.202,"{""location"": ""HN"", ""is_mobile"": false}" 843,2,80,2016-12-20 12:54:17,http://stracke.biz/emory.schmidt,,165.117.60.32,"{""location"": ""TC"", ""is_mobile"": false}" 844,2,80,2017-02-03 16:56:53,http://walker.com/jed,,234.228.81.84,"{""location"": ""MG"", ""is_mobile"": false}" 845,2,80,2016-12-15 10:05:55,http://wisozk.org/dario.buckridge,,97.36.171.192,"{""location"": ""LB"", ""is_mobile"": false}" 846,2,81,2017-05-27 20:15:40,http://runolfon.io/shanelle_roberts,,185.233.7.47,"{""location"": ""CZ"", ""is_mobile"": false}" 847,2,81,2017-02-18 01:38:49,http://osinski.com/misty,,34.124.53.111,"{""location"": ""NU"", ""is_mobile"": false}" 848,2,81,2017-03-08 11:12:33,http://tromp.io/dario_schmeler,,81.147.112.87,"{""location"": ""ES"", ""is_mobile"": true}" 849,2,81,2016-12-31 06:12:17,http://feilaltenwerth.info/sim.fisher,,193.90.163.180,"{""location"": ""FK"", ""is_mobile"": true}" 850,2,81,2017-02-12 11:43:16,http://christiansen.co/sherman,,6.14.46.107,"{""location"": ""BW"", ""is_mobile"": true}" 851,2,81,2017-02-02 11:42:35,http://wolf.info/una.rice,,99.100.90.179,"{""location"": ""GS"", ""is_mobile"": false}" 852,2,81,2017-02-02 02:04:52,http://larkin.co/priscilla_goodwin,,94.204.114.194,"{""location"": ""SJ"", ""is_mobile"": false}" 853,2,81,2017-03-22 04:52:06,http://heel.info/sophie_aufderhar,,244.13.11.157,"{""location"": ""GG"", ""is_mobile"": false}" 854,2,81,2017-03-10 13:58:23,http://donnelly.net/giovanny.larkin,,34.124.53.111,"{""location"": ""AZ"", ""is_mobile"": true}" 855,2,81,2016-12-31 10:33:41,http://cruickshank.biz/waldo_bruen,,92.214.73.156,"{""location"": ""KR"", ""is_mobile"": true}" 856,2,81,2017-02-24 16:00:28,http://watsica.biz/ethel,,197.98.30.207,"{""location"": ""SH"", ""is_mobile"": true}" 857,2,81,2017-02-09 00:11:41,http://stehrhoppe.biz/michael_mayert,,244.13.11.157,"{""location"": ""RW"", ""is_mobile"": true}" 858,2,81,2017-03-10 07:04:34,http://kuphalrippin.co/marley,,81.147.112.87,"{""location"": ""US"", ""is_mobile"": false}" 859,2,82,2017-01-18 08:54:41,http://caspercole.net/oma,,41.152.243.37,"{""location"": ""UZ"", ""is_mobile"": true}" 860,2,82,2017-03-05 17:33:00,http://labadie.info/tyreek_turner,,46.162.244.21,"{""location"": ""UM"", ""is_mobile"": true}" 861,2,82,2017-06-12 22:22:42,http://zulauf.biz/madilyn_beer,,129.226.19.84,"{""location"": ""LY"", ""is_mobile"": true}" 862,2,82,2017-01-28 19:50:59,http://gradyhane.org/karli,,81.46.107.133,"{""location"": ""BH"", ""is_mobile"": true}" 863,2,82,2017-01-22 14:13:44,http://schillergulgowski.info/lysanne_pacocha,,57.13.107.77,"{""location"": ""PM"", ""is_mobile"": true}" 864,2,83,2017-05-17 02:32:40,http://kshlerin.org/bonita,,216.148.228.159,"{""location"": ""UM"", ""is_mobile"": true}" 865,2,83,2016-12-22 06:24:04,http://gorczanyprice.biz/bettye,,152.125.108.240,"{""location"": ""CI"", ""is_mobile"": false}" 866,2,84,2017-02-10 03:03:52,http://donnelly.org/bria.kreiger,,58.190.103.240,"{""location"": ""PF"", ""is_mobile"": false}" 867,2,84,2017-01-08 23:03:58,http://kunde.biz/theresa,,96.104.241.149,"{""location"": ""NP"", ""is_mobile"": false}" 868,2,84,2017-03-09 14:46:11,http://farrell.name/delores.langworth,,168.138.93.245,"{""location"": ""BB"", ""is_mobile"": false}" 869,2,84,2017-03-14 16:06:15,http://price.org/samanta.bashirian,,62.221.145.130,"{""location"": ""PN"", ""is_mobile"": false}" 870,2,84,2017-01-25 07:05:57,http://anderson.name/conrad,,150.96.187.2,"{""location"": ""HT"", ""is_mobile"": false}" 871,2,84,2017-05-26 11:18:37,http://swaniawskiwillms.info/lavinia,,30.242.253.84,"{""location"": ""VG"", ""is_mobile"": true}" 872,2,84,2017-02-04 22:59:57,http://gaylord.org/roie_mayer,,96.104.241.149,"{""location"": ""LT"", ""is_mobile"": true}" 873,2,84,2017-02-19 02:30:49,http://wilderman.com/miller.mayert,,3.215.19.95,"{""location"": ""CX"", ""is_mobile"": true}" 874,2,84,2017-04-05 21:27:21,http://cain.com/furman_fay,,96.104.241.149,"{""location"": ""TL"", ""is_mobile"": false}" 875,2,84,2017-04-19 20:38:10,http://bruen.co/devon,,178.167.248.96,"{""location"": ""CR"", ""is_mobile"": false}" 876,2,84,2017-04-18 10:44:19,http://paucek.info/helen_grant,,166.214.34.203,"{""location"": ""AQ"", ""is_mobile"": false}" 877,2,84,2017-03-14 23:18:30,http://walsh.io/jesus.mohr,,164.78.106.242,"{""location"": ""BW"", ""is_mobile"": false}" 878,2,84,2017-01-28 03:32:57,http://zboncak.biz/broderick,,168.72.54.45,"{""location"": ""CM"", ""is_mobile"": true}" 879,2,84,2017-01-22 00:30:58,http://champlin.info/yadira,,166.214.34.203,"{""location"": ""MC"", ""is_mobile"": false}" 880,2,84,2017-05-17 11:10:52,http://schroeder.io/dale,,231.215.221.2,"{""location"": ""PE"", ""is_mobile"": true}" 881,2,85,2017-04-17 08:55:42,http://kihn.info/jana,,61.103.181.167,"{""location"": ""VN"", ""is_mobile"": false}" 882,2,85,2017-03-14 18:50:44,http://haley.biz/ignacio.emard,,19.87.198.44,"{""location"": ""EE"", ""is_mobile"": true}" 883,2,85,2017-04-20 23:10:22,http://ebert.info/elenora.swift,,107.41.113.247,"{""location"": ""BW"", ""is_mobile"": false}" 884,2,86,2017-05-29 21:56:21,http://heelvolkman.info/favian,0.0000000000,84.19.249.202,"{""location"": ""KY"", ""is_mobile"": true}" 885,2,86,2017-05-26 03:18:06,http://hintzhammes.com/meta,0.0000000000,34.122.77.249,"{""location"": ""VG"", ""is_mobile"": true}" 886,2,86,2017-06-09 07:47:00,http://faybailey.com/roslyn_walsh,0.0000000000,253.77.104.48,"{""location"": ""KW"", ""is_mobile"": true}" 887,2,86,2017-04-20 13:22:24,http://johnsonmarvin.org/caie.blick,0.0000000000,173.212.122.73,"{""location"": ""CC"", ""is_mobile"": false}" 888,2,86,2017-02-13 22:01:05,http://ernser.biz/fernando_braun,0.0000000000,196.8.150.24,"{""location"": ""BS"", ""is_mobile"": true}" 889,2,86,2017-02-07 11:43:23,http://schinner.co/weston,0.0000000000,198.121.124.151,"{""location"": ""TT"", ""is_mobile"": false}" 890,2,86,2017-04-14 05:50:39,http://mckenzieparisian.name/bill,0.0000000000,10.159.57.209,"{""location"": ""ET"", ""is_mobile"": false}" 891,2,86,2017-03-03 18:10:14,http://sipes.com/lyric,0.0000000000,184.120.220.219,"{""location"": ""GT"", ""is_mobile"": true}" 892,2,86,2017-02-21 21:16:44,http://donnelly.org/german,0.0000000000,181.136.101.166,"{""location"": ""SX"", ""is_mobile"": true}" 893,2,86,2017-02-14 00:28:35,http://strackelangosh.name/marge.oconner,0.0000000000,80.198.145.118,"{""location"": ""SK"", ""is_mobile"": false}" 3984,6,370,2017-03-21 23:16:09,http://kautzer.info/chelsea,,88.160.122.184,"{""location"": ""MS"", ""is_mobile"": false}" 3985,6,370,2017-03-04 02:30:02,http://prohaska.org/ruthe,,7.130.55.106,"{""location"": ""EG"", ""is_mobile"": false}" 3986,6,370,2017-05-26 19:21:57,http://weinatherman.biz/mikayla,,241.150.61.44,"{""location"": ""RS"", ""is_mobile"": false}" 3987,6,370,2016-12-21 20:12:31,http://schmidt.io/delia,,174.144.133.13,"{""location"": ""GL"", ""is_mobile"": false}" 3988,6,370,2017-02-17 03:12:45,http://lebsack.name/rubye.thompson,,231.7.250.179,"{""location"": ""VG"", ""is_mobile"": true}" 3989,6,370,2017-03-15 16:44:31,http://toy.net/koby,,83.95.104.249,"{""location"": ""SA"", ""is_mobile"": false}" 3990,6,370,2016-12-24 23:54:30,http://harber.info/earnestine_armstrong,,195.103.234.224,"{""location"": ""HU"", ""is_mobile"": true}" 3991,6,371,2017-04-23 15:01:51,http://parisian.com/boyd.donnelly,,137.107.180.153,"{""location"": ""AR"", ""is_mobile"": true}" 3992,6,371,2016-12-24 12:27:23,http://hayesreichel.info/keira.kohler,,137.107.180.153,"{""location"": ""KR"", ""is_mobile"": true}" 3993,6,371,2017-05-17 09:49:54,http://abshire.co/shea_hermiston,,42.252.157.93,"{""location"": ""GW"", ""is_mobile"": false}" 3994,6,371,2017-01-16 00:27:34,http://hudson.co/sallie,,143.236.24.183,"{""location"": ""NP"", ""is_mobile"": true}" 3995,6,371,2017-06-03 07:51:05,http://hahn.name/lonie,,119.7.226.11,"{""location"": ""ER"", ""is_mobile"": true}" 3996,6,371,2017-05-30 21:13:27,http://rutherford.name/rafael_oreilly,,119.7.226.11,"{""location"": ""GM"", ""is_mobile"": true}" 3997,6,371,2017-04-16 12:51:05,http://ratkekozey.info/annabell,,66.134.190.138,"{""location"": ""ZA"", ""is_mobile"": true}" 3998,6,371,2017-05-31 17:03:49,http://dickens.net/manuel,,167.223.38.128,"{""location"": ""CF"", ""is_mobile"": true}" 3999,6,371,2017-04-23 23:21:45,http://littleeichmann.com/eda.quitzon,,4.72.34.163,"{""location"": ""FI"", ""is_mobile"": false}" 4000,6,371,2017-02-12 03:39:13,http://mcglynn.co/reina,,170.201.248.109,"{""location"": ""SH"", ""is_mobile"": false}" 4001,6,371,2017-05-16 16:47:59,http://lubowitz.io/carlotta,,4.72.34.163,"{""location"": ""GP"", ""is_mobile"": true}" 4002,6,371,2017-05-12 06:47:51,http://boganspinka.name/haan_lindgren,,66.134.190.138,"{""location"": ""ZW"", ""is_mobile"": true}" 4003,6,371,2017-03-20 04:37:12,http://turcottebotsford.co/garland,,208.43.216.250,"{""location"": ""EH"", ""is_mobile"": true}" 4004,6,371,2017-04-08 15:23:16,http://heidenreich.biz/cornell,,39.136.242.86,"{""location"": ""LY"", ""is_mobile"": false}" 4005,6,371,2016-12-21 19:54:51,http://keeling.org/durward.bashirian,,77.168.254.23,"{""location"": ""AF"", ""is_mobile"": false}" 4006,6,371,2017-05-29 19:09:19,http://hansen.org/antonette,,241.141.229.81,"{""location"": ""LU"", ""is_mobile"": true}" 4007,6,371,2017-04-14 15:36:19,http://durgan.name/rafaela_tremblay,,26.16.22.152,"{""location"": ""BN"", ""is_mobile"": false}" 4008,6,371,2017-05-26 06:38:07,http://gutmannzemlak.io/joana.cummerata,,141.180.240.169,"{""location"": ""BN"", ""is_mobile"": false}" 4009,6,372,2017-01-15 04:56:43,http://prosacco.com/elvie.schiller,,254.48.59.33,"{""location"": ""RS"", ""is_mobile"": true}" 4010,6,372,2017-03-25 15:13:04,http://okeefe.net/brielle,,116.37.229.156,"{""location"": ""AS"", ""is_mobile"": false}" 4011,6,372,2017-04-08 18:53:50,http://deckow.info/kristina.corwin,,119.25.96.133,"{""location"": ""EE"", ""is_mobile"": false}" 4012,6,372,2016-12-17 07:15:17,http://murrayreinger.com/eileen,,34.223.92.68,"{""location"": ""KN"", ""is_mobile"": false}" 4013,6,372,2017-01-05 04:56:40,http://schamberger.info/nora,,57.85.4.220,"{""location"": ""MQ"", ""is_mobile"": false}" 4014,6,372,2017-01-22 13:15:40,http://moriettethiel.info/erik,,103.28.116.22,"{""location"": ""YE"", ""is_mobile"": false}" 4015,6,372,2017-02-15 05:53:19,http://mcclure.biz/savanah_christiansen,,103.7.149.107,"{""location"": ""NI"", ""is_mobile"": true}" 4016,6,372,2017-05-09 04:33:52,http://bruen.com/emile,,140.112.133.106,"{""location"": ""CM"", ""is_mobile"": true}" 4017,6,372,2017-04-26 18:02:19,http://walkerboehm.net/vernon_wisoky,,48.77.211.133,"{""location"": ""PL"", ""is_mobile"": false}" 4018,6,372,2017-05-03 12:01:52,http://toyrutherford.co/jaydon_huel,,88.60.28.181,"{""location"": ""AX"", ""is_mobile"": true}" 4019,6,372,2016-12-20 03:29:45,http://cole.biz/kaleigh.okuneva,,236.211.33.136,"{""location"": ""VE"", ""is_mobile"": true}" 4020,6,372,2017-02-09 06:46:15,http://vonrueden.biz/elmore,,28.141.136.35,"{""location"": ""AM"", ""is_mobile"": true}" 4021,6,372,2016-12-18 07:01:57,http://torphydouglas.name/bridget,,254.48.59.33,"{""location"": ""BA"", ""is_mobile"": false}" 4022,6,373,2017-03-10 06:32:18,http://rutherfordlubowitz.biz/nash,,61.219.209.200,"{""location"": ""HU"", ""is_mobile"": false}" 4023,6,373,2017-06-09 14:18:51,http://krajcik.net/kyra,,161.10.210.85,"{""location"": ""EE"", ""is_mobile"": true}" 4024,6,373,2017-01-03 02:32:36,http://zemlakflatley.com/levi,,253.134.126.231,"{""location"": ""VA"", ""is_mobile"": true}" 4025,6,373,2017-01-14 08:56:39,http://larkin.name/jaylan,,222.216.220.76,"{""location"": ""BW"", ""is_mobile"": true}" 4026,6,373,2017-04-14 13:42:17,http://sporer.com/jordy,,74.160.71.71,"{""location"": ""KZ"", ""is_mobile"": false}" 4027,6,373,2017-03-01 04:32:10,http://stromanokuneva.io/myrtice,,177.53.93.40,"{""location"": ""GS"", ""is_mobile"": true}" 4028,6,373,2017-05-17 22:57:49,http://schaden.biz/clay_kirlin,,202.225.174.118,"{""location"": ""PE"", ""is_mobile"": false}" 4029,6,373,2017-02-16 08:16:54,http://crist.info/cecilia_flatley,,86.254.205.75,"{""location"": ""RW"", ""is_mobile"": true}" 4030,6,373,2017-01-16 12:03:11,http://kub.info/bernard.ledner,,193.26.213.185,"{""location"": ""VU"", ""is_mobile"": false}" 4031,6,373,2017-04-12 07:17:13,http://feil.info/vincenza,,158.9.61.87,"{""location"": ""MS"", ""is_mobile"": true}" 4032,6,373,2017-01-15 08:36:34,http://herzog.info/hosea,,222.216.220.76,"{""location"": ""VG"", ""is_mobile"": true}" 4033,6,373,2017-03-26 16:44:44,http://fadelchristiansen.com/austen,,225.145.155.239,"{""location"": ""ES"", ""is_mobile"": false}" 4034,6,373,2017-03-11 02:06:29,http://crooks.name/maynard,,78.45.164.164,"{""location"": ""NU"", ""is_mobile"": true}" 4035,6,373,2017-02-28 06:44:52,http://gottlieblittel.net/pamela,,159.75.33.117,"{""location"": ""LR"", ""is_mobile"": false}" 4036,6,373,2016-12-15 20:47:14,http://kertzmannlangworth.org/delfina,,98.179.220.253,"{""location"": ""MP"", ""is_mobile"": false}" 4037,6,374,2017-05-30 10:25:24,http://willmskoelpin.io/stefanie,,50.146.250.117,"{""location"": ""CG"", ""is_mobile"": true}" 4038,6,374,2017-03-12 12:02:48,http://nikolaus.biz/jarvis.miller,,190.249.246.60,"{""location"": ""SH"", ""is_mobile"": true}" 4039,6,374,2017-02-21 15:06:04,http://johnston.com/sincere_denesik,,112.122.48.61,"{""location"": ""GS"", ""is_mobile"": false}" 894,2,86,2017-02-28 06:49:31,http://beerweinat.biz/gerard,0.0000000000,82.245.199.148,"{""location"": ""PW"", ""is_mobile"": false}" 895,2,87,2016-12-19 12:01:49,http://bartell.biz/carey,0.0000000000,168.37.120.85,"{""location"": ""UA"", ""is_mobile"": true}" 896,2,87,2017-04-15 05:35:56,http://boscocorwin.co/janice,0.0000000000,211.241.95.191,"{""location"": ""ZA"", ""is_mobile"": true}" 897,2,87,2017-05-16 10:38:22,http://millerwatsica.net/william_hermann,0.0000000000,130.66.186.222,"{""location"": ""NR"", ""is_mobile"": false}" 898,2,87,2017-06-02 01:00:34,http://terry.name/enrico.mcglynn,0.0000000000,238.183.14.155,"{""location"": ""KW"", ""is_mobile"": true}" 899,2,87,2017-03-10 11:57:45,http://senger.com/jaqueline,0.0000000000,22.238.207.105,"{""location"": ""BE"", ""is_mobile"": true}" 900,2,87,2017-03-08 22:44:53,http://satterfieldondricka.info/joey.murray,0.0000000000,211.241.95.191,"{""location"": ""TC"", ""is_mobile"": false}" 901,2,87,2017-06-10 08:12:15,http://kris.org/rickey,0.0000000000,179.85.80.137,"{""location"": ""MU"", ""is_mobile"": false}" 902,2,87,2016-12-31 12:00:01,http://kihn.com/genesis,0.0000000000,10.139.233.123,"{""location"": ""LV"", ""is_mobile"": true}" 903,2,87,2017-06-06 20:11:12,http://walsh.co/troy,0.0000000000,70.164.94.149,"{""location"": ""SB"", ""is_mobile"": true}" 904,2,87,2017-02-09 15:23:07,http://rogahn.org/deshawn_thiel,0.0000000000,174.144.164.52,"{""location"": ""SZ"", ""is_mobile"": true}" 905,2,87,2017-05-25 11:15:54,http://schuppe.name/chelsey,0.0000000000,174.144.164.52,"{""location"": ""CA"", ""is_mobile"": false}" 906,2,88,2017-03-07 05:56:29,http://ferry.biz/lucinda_waters,2.0000000000,89.159.225.34,"{""location"": ""KM"", ""is_mobile"": true}" 907,2,88,2017-02-12 19:45:28,http://bergnaummoore.com/quinn,3.0000000000,26.167.49.106,"{""location"": ""KM"", ""is_mobile"": true}" 908,2,88,2017-01-14 12:35:21,http://upton.info/gerda.moore,3.0000000000,86.187.55.244,"{""location"": ""DZ"", ""is_mobile"": true}" 909,2,88,2017-02-18 18:09:55,http://collierschinner.biz/opal.douglas,2.0000000000,239.56.63.78,"{""location"": ""SY"", ""is_mobile"": true}" 910,2,88,2017-01-04 12:07:09,http://morarkuphal.org/joy,3.0000000000,138.2.175.22,"{""location"": ""BQ"", ""is_mobile"": false}" 911,2,88,2017-01-15 00:50:19,http://nitzsche.info/nora.bode,3.0000000000,186.223.134.157,"{""location"": ""MC"", ""is_mobile"": false}" 912,2,88,2017-04-04 09:10:27,http://cruickshankspinka.org/johnpaul,3.0000000000,236.23.214.136,"{""location"": ""CZ"", ""is_mobile"": false}" 913,2,88,2017-04-11 12:07:34,http://orn.co/reymundo_reichert,1.0000000000,155.244.57.179,"{""location"": ""NO"", ""is_mobile"": true}" 914,2,88,2017-01-07 11:28:57,http://grahamstark.name/jazmyn,2.0000000000,239.56.63.78,"{""location"": ""EG"", ""is_mobile"": true}" 915,2,88,2017-02-19 20:55:19,http://smithamschultz.org/santino.schimmel,3.0000000000,189.232.98.188,"{""location"": ""TO"", ""is_mobile"": false}" 916,2,88,2017-01-21 19:58:11,http://kunde.info/kadin,3.0000000000,200.74.236.8,"{""location"": ""SS"", ""is_mobile"": false}" 917,2,88,2017-02-02 02:13:41,http://johnson.info/chanelle.von,1.0000000000,254.194.189.12,"{""location"": ""PH"", ""is_mobile"": false}" 918,2,88,2017-01-18 08:08:14,http://weinat.net/luz_kris,1.0000000000,214.250.168.89,"{""location"": ""ML"", ""is_mobile"": false}" 919,2,88,2017-03-23 03:15:44,http://kozeyhills.co/bryce.kihn,3.0000000000,19.204.208.228,"{""location"": ""DZ"", ""is_mobile"": true}" 920,2,88,2017-05-16 19:27:41,http://stamm.net/skye,2.0000000000,144.8.239.4,"{""location"": ""PH"", ""is_mobile"": false}" 921,2,88,2017-05-09 18:41:19,http://grimes.co/alfredo_doyle,1.0000000000,186.223.134.157,"{""location"": ""HM"", ""is_mobile"": false}" 922,2,88,2017-01-27 10:01:53,http://lebsack.io/roscoe_schamberger,2.0000000000,214.250.168.89,"{""location"": ""KP"", ""is_mobile"": false}" 923,2,88,2017-03-27 03:08:29,http://murray.biz/antone,1.0000000000,166.54.154.216,"{""location"": ""DO"", ""is_mobile"": true}" 924,2,89,2017-02-19 01:13:20,http://robelfritsch.io/zachary_cremin,1.0000000000,170.93.42.95,"{""location"": ""VE"", ""is_mobile"": true}" 925,2,89,2017-01-15 21:16:55,http://mcdermott.name/dora_hickle,2.0000000000,26.246.150.114,"{""location"": ""FJ"", ""is_mobile"": false}" 926,2,89,2016-12-26 06:59:26,http://gorczanyhauck.io/yvonne,1.0000000000,53.241.25.155,"{""location"": ""NF"", ""is_mobile"": true}" 927,2,89,2017-05-26 14:32:49,http://reichel.info/jerrod_casper,2.0000000000,118.18.39.87,"{""location"": ""SE"", ""is_mobile"": true}" 928,2,89,2017-02-03 00:54:16,http://bogan.co/eldon,1.0000000000,115.242.87.74,"{""location"": ""ES"", ""is_mobile"": false}" 929,2,89,2017-02-23 17:11:19,http://franecki.biz/jeanie.towne,0.0000000000,115.71.51.194,"{""location"": ""GH"", ""is_mobile"": false}" 930,2,89,2017-03-03 14:55:35,http://watsica.io/bernadine.pfeffer,1.0000000000,8.111.204.89,"{""location"": ""BE"", ""is_mobile"": true}" 931,2,89,2017-01-07 16:32:00,http://monahanberge.net/filomena_ward,1.0000000000,148.94.83.48,"{""location"": ""AL"", ""is_mobile"": true}" 932,2,89,2017-02-18 20:59:54,http://ziemann.co/daren,2.0000000000,184.221.126.98,"{""location"": ""MD"", ""is_mobile"": false}" 933,2,89,2017-02-12 21:15:21,http://durgan.name/kylee,1.0000000000,182.130.134.114,"{""location"": ""CI"", ""is_mobile"": false}" 934,2,89,2017-03-10 14:52:58,http://lind.io/shaun,0.0000000000,8.111.204.89,"{""location"": ""PH"", ""is_mobile"": true}" 935,2,89,2017-04-21 13:06:05,http://little.org/javier,2.0000000000,139.47.157.59,"{""location"": ""AI"", ""is_mobile"": true}" 936,2,89,2017-03-16 13:01:47,http://jaskolski.org/sabryna_gottlieb,2.0000000000,254.186.94.37,"{""location"": ""CH"", ""is_mobile"": true}" 937,2,89,2017-01-20 20:22:09,http://crist.info/jasmin,0.0000000000,47.208.33.91,"{""location"": ""CZ"", ""is_mobile"": true}" 938,2,89,2017-05-23 01:39:38,http://kemmer.org/zachariah,2.0000000000,186.196.104.128,"{""location"": ""UY"", ""is_mobile"": true}" 939,2,89,2017-01-10 21:15:02,http://lebsack.info/minnie,1.0000000000,17.124.90.231,"{""location"": ""PY"", ""is_mobile"": true}" 940,2,89,2017-01-20 02:31:58,http://quigley.com/major.hilll,2.0000000000,231.49.65.15,"{""location"": ""LA"", ""is_mobile"": true}" 941,2,89,2017-06-08 21:24:01,http://armstrong.info/halie,2.0000000000,186.217.227.50,"{""location"": ""ML"", ""is_mobile"": true}" 942,2,89,2017-03-08 11:36:52,http://macejkovicprice.org/dion,2.0000000000,139.47.157.59,"{""location"": ""GR"", ""is_mobile"": false}" 943,2,89,2017-02-08 13:14:59,http://spencer.net/terrance,2.0000000000,148.94.83.48,"{""location"": ""TG"", ""is_mobile"": false}" 944,2,90,2017-04-18 19:46:28,http://johnsonrogahn.co/adolphus_harvey,1.0000000000,167.254.172.198,"{""location"": ""VE"", ""is_mobile"": true}" 945,2,90,2017-02-02 15:18:19,http://collins.co/brenda,1.0000000000,164.152.196.138,"{""location"": ""SV"", ""is_mobile"": false}" 4040,6,374,2017-05-12 17:34:50,http://wiza.net/alfreda_jaskolski,,242.58.193.148,"{""location"": ""NZ"", ""is_mobile"": false}" 4041,6,374,2017-04-03 17:21:53,http://kiehnvonrueden.info/precious_keeling,,74.75.235.156,"{""location"": ""LT"", ""is_mobile"": true}" 4042,6,374,2017-04-16 22:21:48,http://roberts.info/casandra_ryan,,147.89.104.121,"{""location"": ""AT"", ""is_mobile"": false}" 4043,6,374,2017-02-13 05:49:03,http://gerholdhammes.com/berta_predovic,,65.177.90.215,"{""location"": ""KH"", ""is_mobile"": true}" 4044,6,374,2017-04-20 23:12:22,http://goldner.com/mohammed,,176.228.97.200,"{""location"": ""KG"", ""is_mobile"": true}" 4045,6,374,2017-06-03 06:54:08,http://botsford.co/bettie,,251.144.75.232,"{""location"": ""DM"", ""is_mobile"": false}" 4046,6,374,2017-02-02 05:21:03,http://herzog.name/jan,,50.146.250.117,"{""location"": ""HK"", ""is_mobile"": true}" 4047,6,374,2017-01-05 15:23:04,http://dachhalvorson.org/gina,,177.62.14.78,"{""location"": ""SI"", ""is_mobile"": true}" 4048,6,374,2017-04-24 09:01:13,http://litteltoy.io/okey.herman,,112.122.48.61,"{""location"": ""CV"", ""is_mobile"": false}" 4049,6,374,2017-05-06 20:54:02,http://marvin.com/michaela,,6.38.134.150,"{""location"": ""SA"", ""is_mobile"": true}" 4050,6,374,2017-04-05 19:30:54,http://schumm.co/golda_wehner,,178.104.32.117,"{""location"": ""NE"", ""is_mobile"": false}" 4051,6,374,2016-12-18 23:17:27,http://greenweinat.biz/aimee_mitchell,,119.107.173.207,"{""location"": ""MQ"", ""is_mobile"": false}" 4052,6,374,2017-05-06 11:34:05,http://kirlin.co/june,,13.143.254.38,"{""location"": ""IT"", ""is_mobile"": false}" 4053,6,374,2017-05-06 11:05:50,http://pollich.io/otilia,,125.213.150.214,"{""location"": ""BD"", ""is_mobile"": false}" 4054,6,374,2017-03-25 09:20:08,http://hartmann.io/jermain_rosenbaum,,147.89.104.121,"{""location"": ""SZ"", ""is_mobile"": true}" 4055,6,375,2017-02-23 23:35:17,http://rutherfordabshire.co/walter_heathcote,,252.54.210.190,"{""location"": ""GR"", ""is_mobile"": true}" 4056,6,375,2017-01-28 17:57:11,http://predovic.com/odie.strosin,,45.195.177.173,"{""location"": ""UY"", ""is_mobile"": true}" 4057,6,375,2017-04-24 20:55:58,http://kulasgusikowski.io/jennings.lueilwitz,,22.230.157.37,"{""location"": ""DE"", ""is_mobile"": true}" 4058,6,375,2017-04-26 18:47:16,http://sawayn.co/gretchen,,107.235.56.88,"{""location"": ""KR"", ""is_mobile"": true}" 4059,6,375,2017-02-18 11:58:02,http://swift.co/serena,,25.64.179.226,"{""location"": ""GU"", ""is_mobile"": true}" 4060,6,375,2017-01-18 04:07:46,http://heaneywolff.name/aleandra,,153.151.84.207,"{""location"": ""PG"", ""is_mobile"": false}" 4061,6,375,2017-01-20 09:28:54,http://kris.com/makenna,,208.175.62.97,"{""location"": ""BD"", ""is_mobile"": true}" 4062,6,375,2017-03-28 09:53:29,http://wunsch.biz/milton_kub,,67.92.181.179,"{""location"": ""IR"", ""is_mobile"": false}" 4063,6,375,2016-12-23 09:02:03,http://mertz.io/maybelle.botsford,,121.48.239.26,"{""location"": ""MY"", ""is_mobile"": true}" 4064,6,375,2017-02-15 19:30:36,http://kirlin.co/milton,,47.11.11.244,"{""location"": ""AT"", ""is_mobile"": false}" 4065,6,375,2017-02-15 20:53:11,http://fay.org/destin_turcotte,,206.164.246.143,"{""location"": ""GB"", ""is_mobile"": true}" 4066,6,375,2017-05-07 18:58:39,http://streich.name/freeman.adams,,78.48.161.217,"{""location"": ""AX"", ""is_mobile"": false}" 4067,6,375,2016-12-29 10:06:29,http://cormierkshlerin.net/ethel,,208.175.62.97,"{""location"": ""TO"", ""is_mobile"": false}" 4068,6,375,2017-04-10 11:26:11,http://oreillystamm.biz/makayla.volkman,,78.48.161.217,"{""location"": ""CF"", ""is_mobile"": false}" 4069,6,375,2017-03-30 05:28:28,http://boyergreenholt.name/tad,,121.226.95.251,"{""location"": ""DZ"", ""is_mobile"": true}" 4070,6,375,2017-02-16 03:34:27,http://bradtke.io/romaine.considine,,25.64.179.226,"{""location"": ""GI"", ""is_mobile"": true}" 4071,6,375,2017-02-04 18:00:08,http://paucek.info/geovanni,,208.175.62.97,"{""location"": ""PW"", ""is_mobile"": true}" 4072,6,375,2017-04-16 04:29:25,http://blanda.net/laisha,,22.230.157.37,"{""location"": ""ME"", ""is_mobile"": false}" 4073,6,375,2017-01-09 13:02:34,http://wolff.co/urban,,244.188.36.115,"{""location"": ""IR"", ""is_mobile"": true}" 4074,6,376,2017-05-29 02:00:13,http://feilzboncak.biz/antonia.eichmann,,205.37.58.23,"{""location"": ""DJ"", ""is_mobile"": false}" 4075,6,376,2016-12-19 00:31:40,http://roob.com/eliza,,215.131.208.181,"{""location"": ""GH"", ""is_mobile"": true}" 4076,6,376,2017-02-07 07:17:57,http://sporer.info/destinee.dare,,164.52.141.199,"{""location"": ""HN"", ""is_mobile"": true}" 4077,6,376,2017-03-11 05:41:16,http://hoeger.com/elbert.cain,,87.57.214.97,"{""location"": ""AM"", ""is_mobile"": true}" 4078,6,377,2016-12-19 12:31:25,http://bartell.net/abagail,1.0000000000,72.192.24.163,"{""location"": ""TW"", ""is_mobile"": false}" 4079,6,377,2017-01-03 05:13:55,http://mitchellgorczany.com/dallas.hermann,1.0000000000,76.178.6.134,"{""location"": ""LY"", ""is_mobile"": false}" 4080,6,377,2017-05-19 16:12:24,http://wunsch.info/montana,0.0000000000,240.149.116.106,"{""location"": ""VE"", ""is_mobile"": true}" 4081,6,378,2017-03-29 08:53:17,http://jacobson.info/shyann_white,0.0000000000,253.120.147.129,"{""location"": ""AS"", ""is_mobile"": false}" 4082,6,378,2017-03-30 00:03:24,http://murazik.name/kaya.waelchi,0.0000000000,55.252.95.214,"{""location"": ""TM"", ""is_mobile"": true}" 4083,6,378,2017-03-20 07:11:01,http://corkery.net/lisa.lehner,0.0000000000,91.239.5.247,"{""location"": ""ER"", ""is_mobile"": false}" 4084,6,378,2017-01-01 00:01:14,http://lockmanterry.com/jamarcus.ernser,0.0000000000,205.162.28.167,"{""location"": ""TM"", ""is_mobile"": true}" 4085,6,378,2017-01-12 03:58:11,http://vonbeer.info/christelle,0.0000000000,128.218.66.233,"{""location"": ""OM"", ""is_mobile"": false}" 4086,6,378,2017-06-03 13:33:02,http://leannonbernhard.biz/annabelle,1.0000000000,196.180.19.159,"{""location"": ""FR"", ""is_mobile"": true}" 4087,6,378,2017-02-21 22:28:34,http://ziemann.io/adaline,0.0000000000,135.142.50.101,"{""location"": ""JO"", ""is_mobile"": false}" 4088,6,379,2017-05-27 00:11:25,http://hackett.net/isai,1.0000000000,102.214.68.89,"{""location"": ""PR"", ""is_mobile"": true}" 4089,6,379,2016-12-28 16:24:44,http://cronaconn.io/frieda_schulist,3.0000000000,81.159.9.49,"{""location"": ""QA"", ""is_mobile"": true}" 4090,6,379,2017-02-20 01:07:39,http://hirthe.name/jarod,3.0000000000,79.191.16.63,"{""location"": ""FJ"", ""is_mobile"": false}" 4091,6,379,2017-01-25 22:43:53,http://parisianfeil.biz/marilie,1.0000000000,202.141.228.78,"{""location"": ""MF"", ""is_mobile"": false}" 4092,6,379,2017-04-19 16:54:50,http://kihn.net/stanton,2.0000000000,171.47.14.181,"{""location"": ""CL"", ""is_mobile"": true}" 4093,6,379,2016-12-14 17:20:55,http://green.com/rosie,2.0000000000,193.169.41.213,"{""location"": ""SJ"", ""is_mobile"": false}" 946,2,90,2017-04-13 18:05:16,http://hane.co/ernie.little,1.0000000000,4.70.159.55,"{""location"": ""BM"", ""is_mobile"": false}" 947,2,90,2017-05-13 03:51:09,http://armstrong.name/hailey_denesik,0.0000000000,4.70.159.55,"{""location"": ""SR"", ""is_mobile"": true}" 948,2,90,2017-04-27 13:05:34,http://nolan.io/liza,3.0000000000,134.209.179.253,"{""location"": ""AS"", ""is_mobile"": false}" 949,2,90,2017-05-18 04:39:36,http://barton.com/jane.medhurst,1.0000000000,3.179.179.118,"{""location"": ""VC"", ""is_mobile"": false}" 950,2,90,2017-03-29 11:38:44,http://pollich.co/josh,2.0000000000,152.33.71.251,"{""location"": ""GQ"", ""is_mobile"": false}" 951,2,90,2017-05-10 23:50:25,http://kuhn.biz/eleanore,0.0000000000,207.157.213.208,"{""location"": ""ET"", ""is_mobile"": true}" 952,2,90,2017-05-16 04:34:37,http://gutkowski.net/christiana,2.0000000000,109.44.141.196,"{""location"": ""MG"", ""is_mobile"": true}" 953,2,90,2017-01-21 14:28:59,http://darefahey.net/candido_bashirian,1.0000000000,131.253.170.169,"{""location"": ""IR"", ""is_mobile"": true}" 954,2,90,2017-04-17 13:08:21,http://bradtkeprohaska.biz/tracy,2.0000000000,117.151.9.61,"{""location"": ""GS"", ""is_mobile"": true}" 955,2,90,2017-04-04 01:51:22,http://flatley.io/howell_shanahan,0.0000000000,152.33.71.251,"{""location"": ""SN"", ""is_mobile"": false}" 956,2,90,2017-04-21 01:29:04,http://pagac.org/johann_goyette,2.0000000000,101.127.239.213,"{""location"": ""CF"", ""is_mobile"": true}" 957,2,90,2017-03-08 20:01:17,http://damorewolff.io/beau,2.0000000000,129.66.68.4,"{""location"": ""MV"", ""is_mobile"": true}" 958,2,90,2016-12-23 18:33:09,http://heaneyhudson.org/jan.jaskolski,0.0000000000,3.179.179.118,"{""location"": ""SO"", ""is_mobile"": true}" 959,2,91,2017-03-28 22:07:49,http://gibson.co/savannah_batz,3.0000000000,193.19.158.18,"{""location"": ""KR"", ""is_mobile"": true}" 960,2,91,2017-05-20 10:14:36,http://mckenzie.name/kristian,3.0000000000,70.44.115.250,"{""location"": ""GT"", ""is_mobile"": true}" 961,2,91,2017-03-06 07:28:39,http://hammes.net/gregoria,0.0000000000,2.83.218.113,"{""location"": ""GE"", ""is_mobile"": true}" 962,2,91,2016-12-24 17:45:05,http://goldner.name/elya,0.0000000000,19.163.88.229,"{""location"": ""HU"", ""is_mobile"": false}" 963,2,91,2017-01-15 15:20:39,http://mertzschowalter.net/vernie,0.0000000000,44.233.246.114,"{""location"": ""SZ"", ""is_mobile"": false}" 964,2,91,2017-05-25 15:20:46,http://price.name/vincenzo,2.0000000000,165.135.115.15,"{""location"": ""UZ"", ""is_mobile"": true}" 965,2,92,2017-05-13 12:02:14,http://pouros.biz/floy,3.0000000000,248.58.23.41,"{""location"": ""HN"", ""is_mobile"": false}" 966,2,92,2016-12-17 10:38:59,http://nader.name/jalen,2.0000000000,79.137.214.68,"{""location"": ""NO"", ""is_mobile"": true}" 967,2,92,2017-03-06 04:30:40,http://sipeslebsack.org/jaclyn,1.0000000000,12.29.12.37,"{""location"": ""CY"", ""is_mobile"": true}" 968,2,92,2017-04-11 00:28:42,http://littlegraham.info/craig,0.0000000000,46.159.139.120,"{""location"": ""KH"", ""is_mobile"": false}" 969,2,92,2017-01-12 20:41:13,http://gorczanysatterfield.com/marisa,1.0000000000,97.22.10.241,"{""location"": ""US"", ""is_mobile"": true}" 970,2,92,2017-03-11 06:09:09,http://lindgrenmarquardt.info/roscoe_gutkowski,1.0000000000,46.159.139.120,"{""location"": ""GI"", ""is_mobile"": false}" 971,2,92,2017-01-17 21:17:23,http://veum.org/cecilia_mann,0.0000000000,185.186.207.161,"{""location"": ""MS"", ""is_mobile"": false}" 972,2,92,2017-02-05 20:01:04,http://bruenfriesen.info/trenton.sanford,2.0000000000,10.148.166.114,"{""location"": ""BB"", ""is_mobile"": true}" 973,2,92,2017-04-22 13:06:30,http://lueilwitz.org/guillermo.yost,1.0000000000,145.154.91.130,"{""location"": ""AS"", ""is_mobile"": true}" 974,2,92,2016-12-31 20:13:54,http://macejkovic.org/alex,2.0000000000,246.17.41.5,"{""location"": ""VA"", ""is_mobile"": false}" 975,2,92,2017-04-02 01:38:45,http://kowelch.org/keshaun,2.0000000000,10.70.18.244,"{""location"": ""LC"", ""is_mobile"": false}" 976,2,92,2017-03-08 05:31:35,http://batz.org/delphine_predovic,2.0000000000,85.119.156.149,"{""location"": ""GY"", ""is_mobile"": false}" 977,2,92,2017-06-12 05:59:34,http://braun.net/silas,2.0000000000,82.216.199.5,"{""location"": ""GL"", ""is_mobile"": false}" 978,2,93,2016-12-16 21:44:04,http://mann.net/naomi.collins,3.0000000000,200.163.123.109,"{""location"": ""MG"", ""is_mobile"": true}" 979,2,93,2016-12-15 07:33:20,http://hegmannhermiston.net/nyah,2.0000000000,70.79.35.203,"{""location"": ""CI"", ""is_mobile"": true}" 980,2,93,2017-03-24 11:35:31,http://turcotte.io/quincy,0.0000000000,98.128.220.245,"{""location"": ""LI"", ""is_mobile"": true}" 981,2,93,2017-06-09 12:21:27,http://schamberger.name/kenneth_fadel,1.0000000000,170.172.239.109,"{""location"": ""SN"", ""is_mobile"": true}" 982,2,93,2017-05-14 22:32:04,http://jaskolski.biz/myron,1.0000000000,200.163.123.109,"{""location"": ""BJ"", ""is_mobile"": true}" 983,2,93,2017-04-28 19:13:29,http://wilkinson.org/dylan_sawayn,0.0000000000,14.113.90.40,"{""location"": ""UG"", ""is_mobile"": true}" 984,2,93,2017-03-20 01:57:45,http://carroll.info/ocie,1.0000000000,168.50.33.115,"{""location"": ""MC"", ""is_mobile"": true}" 985,2,93,2017-04-30 03:20:08,http://dach.co/raymond,0.0000000000,179.105.210.164,"{""location"": ""PR"", ""is_mobile"": false}" 986,2,93,2017-05-16 20:18:08,http://jasthermiston.io/chyna,3.0000000000,44.79.36.39,"{""location"": ""EE"", ""is_mobile"": true}" 987,2,93,2017-06-14 01:06:36,http://wisoky.com/izaiah.schmeler,1.0000000000,213.186.189.228,"{""location"": ""PE"", ""is_mobile"": false}" 988,2,93,2017-03-27 13:50:33,http://olson.org/litzy,1.0000000000,129.139.57.128,"{""location"": ""KY"", ""is_mobile"": false}" 989,2,93,2016-12-21 00:10:39,http://durgankaulke.com/chance,0.0000000000,20.68.196.75,"{""location"": ""ST"", ""is_mobile"": false}" 990,2,93,2016-12-16 12:26:55,http://schroeder.biz/kacey,3.0000000000,20.68.196.75,"{""location"": ""TC"", ""is_mobile"": true}" 991,2,93,2017-05-17 07:08:44,http://tromp.name/hyman.rolfson,1.0000000000,168.50.33.115,"{""location"": ""GI"", ""is_mobile"": false}" 992,2,93,2017-05-25 07:53:30,http://wymanspinka.info/fred,0.0000000000,70.181.167.176,"{""location"": ""ET"", ""is_mobile"": false}" 993,2,93,2017-04-09 12:09:52,http://yundthauck.com/brad_bergstrom,3.0000000000,170.172.239.109,"{""location"": ""UZ"", ""is_mobile"": false}" 994,2,93,2017-03-21 00:17:15,http://huel.name/terry.mclaughlin,2.0000000000,83.251.124.232,"{""location"": ""MA"", ""is_mobile"": true}" 995,2,93,2017-04-10 20:30:35,http://collins.io/wava.rohan,0.0000000000,142.183.176.156,"{""location"": ""GI"", ""is_mobile"": true}" 996,2,93,2017-01-31 20:07:17,http://ondrickawalker.biz/florine,2.0000000000,177.142.16.21,"{""location"": ""FJ"", ""is_mobile"": true}" 997,2,93,2017-03-16 05:10:02,http://klingstiedemann.org/shirley,1.0000000000,235.65.183.253,"{""location"": ""BA"", ""is_mobile"": false}" 998,2,94,2017-05-16 05:58:40,http://bernhardtreutel.info/haie_lynch,0.0000000000,166.251.207.11,"{""location"": ""AF"", ""is_mobile"": true}" 4094,6,379,2017-01-04 05:59:30,http://moen.net/jarrell,3.0000000000,193.169.41.213,"{""location"": ""SY"", ""is_mobile"": true}" 4095,6,379,2017-06-10 14:27:57,http://toy.net/winnifred,0.0000000000,121.198.196.112,"{""location"": ""TW"", ""is_mobile"": true}" 4096,6,379,2017-03-24 22:13:07,http://rohan.io/lea,0.0000000000,89.192.216.14,"{""location"": ""TC"", ""is_mobile"": false}" 4097,6,379,2017-05-29 21:34:33,http://ward.org/rolando,2.0000000000,171.126.139.159,"{""location"": ""GM"", ""is_mobile"": true}" 4098,6,379,2017-01-31 07:06:52,http://kuhn.net/sigurd,0.0000000000,136.104.82.93,"{""location"": ""KH"", ""is_mobile"": true}" 4099,6,379,2017-05-09 17:34:58,http://shanahantoy.name/delaney.graham,0.0000000000,18.5.228.92,"{""location"": ""JP"", ""is_mobile"": false}" 4100,6,379,2017-05-06 19:04:26,http://kub.com/may.lang,2.0000000000,89.192.216.14,"{""location"": ""KM"", ""is_mobile"": false}" 4101,6,380,2017-05-17 02:06:29,http://howell.net/alena,1.0000000000,12.22.82.105,"{""location"": ""DZ"", ""is_mobile"": false}" 4102,6,380,2017-03-20 14:39:10,http://graham.co/carolina,1.0000000000,59.198.139.140,"{""location"": ""EH"", ""is_mobile"": false}" 4103,6,380,2017-04-11 06:51:28,http://champlin.name/gaylord.goyette,2.0000000000,25.212.195.241,"{""location"": ""MS"", ""is_mobile"": true}" 4104,6,380,2017-04-15 21:25:50,http://schimmel.info/robyn,1.0000000000,170.179.9.21,"{""location"": ""LB"", ""is_mobile"": true}" 4105,6,380,2017-05-21 05:09:38,http://hudson.net/anjali_price,2.0000000000,101.162.98.37,"{""location"": ""MS"", ""is_mobile"": true}" 4106,6,380,2017-02-03 17:23:09,http://paucekdubuque.biz/rhianna_von,0.0000000000,78.176.22.238,"{""location"": ""SH"", ""is_mobile"": false}" 4107,6,380,2017-05-16 15:50:25,http://mosciskiweinat.name/orrin_hagenes,1.0000000000,123.150.112.102,"{""location"": ""BI"", ""is_mobile"": true}" 4108,6,380,2017-05-12 06:21:37,http://batz.info/george,0.0000000000,12.22.82.105,"{""location"": ""BV"", ""is_mobile"": false}" 4109,6,380,2017-06-12 02:49:19,http://mueller.co/krista,1.0000000000,161.36.66.187,"{""location"": ""MM"", ""is_mobile"": false}" 4110,6,380,2017-06-06 07:40:06,http://beer.net/helen.hilll,2.0000000000,78.176.22.238,"{""location"": ""TL"", ""is_mobile"": false}" 4111,6,380,2017-04-28 02:14:49,http://emmerich.info/braden_parisian,0.0000000000,73.137.177.250,"{""location"": ""DO"", ""is_mobile"": true}" 4112,6,381,2017-06-09 14:39:31,http://carter.net/sigmund,0.0000000000,56.141.152.134,"{""location"": ""CH"", ""is_mobile"": true}" 4113,6,381,2017-04-10 13:32:01,http://mertzdeckow.io/alyson,1.0000000000,140.180.254.223,"{""location"": ""BQ"", ""is_mobile"": true}" 4114,6,381,2017-04-05 15:32:02,http://kogleichner.biz/esta,1.0000000000,27.148.146.69,"{""location"": ""KG"", ""is_mobile"": false}" 4115,6,381,2017-04-03 10:47:00,http://botsford.org/armando.hartmann,1.0000000000,240.208.231.59,"{""location"": ""CU"", ""is_mobile"": true}" 4116,6,382,2017-01-31 03:06:19,http://trantow.info/dale.eichmann,0.0000000000,49.230.127.71,"{""location"": ""NR"", ""is_mobile"": true}" 4117,6,382,2017-03-29 01:05:56,http://bauch.org/jeanette_schulist,0.0000000000,245.110.112.195,"{""location"": ""VI"", ""is_mobile"": false}" 4118,6,382,2017-05-09 05:43:36,http://tremblaymclaughlin.biz/cathrine,0.0000000000,248.113.46.35,"{""location"": ""BR"", ""is_mobile"": true}" 4119,6,382,2017-04-17 01:52:35,http://gaylord.org/dorothea.ohara,0.0000000000,64.206.121.83,"{""location"": ""NL"", ""is_mobile"": false}" 4120,6,382,2017-06-06 11:57:52,http://quigley.co/austin.miller,0.0000000000,190.21.60.193,"{""location"": ""LB"", ""is_mobile"": true}" 4121,6,382,2017-01-22 23:15:41,http://naderbruen.org/nigel,0.0000000000,104.167.146.220,"{""location"": ""BN"", ""is_mobile"": false}" 4122,6,382,2017-05-16 22:53:59,http://parisian.net/lia,0.0000000000,49.230.127.71,"{""location"": ""AE"", ""is_mobile"": true}" 4123,6,382,2016-12-31 02:45:32,http://ziemannprice.net/macey,0.0000000000,169.220.219.73,"{""location"": ""ME"", ""is_mobile"": true}" 4124,6,382,2017-02-22 21:03:58,http://west.net/ramona.ortiz,0.0000000000,129.161.192.114,"{""location"": ""JO"", ""is_mobile"": false}" 4125,6,382,2016-12-13 06:13:08,http://mitchell.co/chanelle,0.0000000000,157.247.38.20,"{""location"": ""CV"", ""is_mobile"": true}" 4126,6,382,2017-02-21 06:58:45,http://bosco.name/gaetano_schultz,0.0000000000,33.149.77.90,"{""location"": ""DM"", ""is_mobile"": true}" 4127,6,382,2017-01-18 09:40:07,http://wyman.org/lupe_franecki,0.0000000000,98.207.157.23,"{""location"": ""SA"", ""is_mobile"": true}" 4128,6,382,2017-05-02 15:42:16,http://tremblayblick.com/karolann_crona,0.0000000000,116.34.46.213,"{""location"": ""UM"", ""is_mobile"": true}" 4129,6,382,2017-02-24 09:55:37,http://wilderman.io/lula.heathcote,0.0000000000,169.220.219.73,"{""location"": ""VG"", ""is_mobile"": false}" 4130,6,382,2017-03-02 03:38:00,http://pfefferlarson.name/chris,0.0000000000,57.225.199.201,"{""location"": ""JE"", ""is_mobile"": false}" 4131,6,383,2016-12-21 13:55:42,http://sengerthiel.io/heath,2.0000000000,178.118.219.145,"{""location"": ""BS"", ""is_mobile"": true}" 4132,6,383,2017-02-14 00:37:26,http://cremin.info/brain,0.0000000000,123.119.221.223,"{""location"": ""SR"", ""is_mobile"": true}" 4133,6,383,2017-05-26 20:09:17,http://okon.org/valerie_shanahan,2.0000000000,60.237.195.179,"{""location"": ""PG"", ""is_mobile"": true}" 4134,6,383,2017-01-25 02:09:40,http://mosciski.co/rozella_heidenreich,2.0000000000,211.253.58.232,"{""location"": ""MP"", ""is_mobile"": true}" 4135,6,383,2017-02-23 19:13:23,http://batz.name/aryanna_howe,0.0000000000,72.151.24.161,"{""location"": ""GI"", ""is_mobile"": true}" 4136,6,383,2017-02-14 20:40:34,http://langworthkilback.net/samantha,1.0000000000,185.192.212.152,"{""location"": ""LA"", ""is_mobile"": true}" 4137,6,383,2017-04-05 15:56:06,http://homenickwiegand.info/lauryn_moore,1.0000000000,121.202.167.152,"{""location"": ""AG"", ""is_mobile"": false}" 4138,6,384,2017-04-30 19:14:42,http://gerhold.co/gina,1.0000000000,109.78.66.42,"{""location"": ""NC"", ""is_mobile"": false}" 4139,6,384,2017-02-25 20:44:07,http://kovacek.name/bradly,2.0000000000,226.51.41.7,"{""location"": ""MK"", ""is_mobile"": false}" 4140,6,384,2017-01-15 21:16:13,http://streich.io/victoria.carter,1.0000000000,163.21.123.25,"{""location"": ""BN"", ""is_mobile"": true}" 4141,6,384,2017-04-16 11:21:45,http://ohara.io/alberto_klocko,0.0000000000,33.245.212.28,"{""location"": ""RU"", ""is_mobile"": true}" 4142,6,384,2017-02-03 17:19:32,http://pfannerstill.info/brady.kshlerin,2.0000000000,240.57.217.217,"{""location"": ""PM"", ""is_mobile"": false}" 4143,6,384,2017-05-24 20:30:48,http://kuhlman.co/mertie,1.0000000000,225.176.247.61,"{""location"": ""TO"", ""is_mobile"": true}" 4144,6,384,2017-03-04 09:32:36,http://dach.org/mitchell,1.0000000000,99.128.151.235,"{""location"": ""UM"", ""is_mobile"": false}" 4145,6,384,2017-02-01 19:40:31,http://bechtelar.biz/kiara.rutherford,1.0000000000,99.128.151.235,"{""location"": ""CC"", ""is_mobile"": false}" 999,2,94,2017-01-21 01:15:46,http://nicolas.com/markus.jenkins,0.0000000000,51.4.148.50,"{""location"": ""MW"", ""is_mobile"": true}" 1000,2,94,2017-03-18 13:46:11,http://spinka.com/josue,0.0000000000,40.8.58.181,"{""location"": ""PS"", ""is_mobile"": false}" 1001,2,94,2017-03-09 17:25:41,http://zboncak.co/macey,2.0000000000,57.134.198.183,"{""location"": ""GF"", ""is_mobile"": false}" 1002,2,94,2017-04-20 12:47:53,http://schiller.info/chyna_bosco,0.0000000000,107.78.226.182,"{""location"": ""RS"", ""is_mobile"": true}" 1003,2,94,2017-04-19 21:13:52,http://jakubowski.com/maverick_gleichner,2.0000000000,40.8.58.181,"{""location"": ""HR"", ""is_mobile"": false}" 1004,2,94,2017-05-31 11:31:13,http://hellerhackett.com/roslyn,2.0000000000,254.190.88.174,"{""location"": ""IT"", ""is_mobile"": true}" 1005,2,94,2017-06-03 17:54:00,http://kuphal.com/genesis,2.0000000000,202.247.88.24,"{""location"": ""BQ"", ""is_mobile"": true}" 1006,2,94,2017-01-16 12:22:52,http://littel.org/judd_hyatt,1.0000000000,40.8.58.181,"{""location"": ""GG"", ""is_mobile"": false}" 1007,2,94,2017-04-20 20:54:21,http://windlerbartoletti.co/mose.schulist,1.0000000000,52.155.164.169,"{""location"": ""JP"", ""is_mobile"": true}" 1008,2,94,2017-01-13 18:54:24,http://marks.com/fritz_koepp,2.0000000000,68.134.168.157,"{""location"": ""GS"", ""is_mobile"": false}" 1009,2,94,2017-02-16 21:42:08,http://murray.io/kim.durgan,3.0000000000,202.247.88.24,"{""location"": ""PT"", ""is_mobile"": true}" 1010,2,94,2017-01-21 15:51:03,http://erdman.net/moriah.macejkovic,0.0000000000,254.190.88.174,"{""location"": ""CU"", ""is_mobile"": true}" 1011,2,94,2017-04-07 04:06:42,http://bergstromhowe.com/vivianne.mertz,1.0000000000,145.77.134.115,"{""location"": ""VC"", ""is_mobile"": true}" 1012,2,94,2017-06-01 15:20:25,http://brekkebeahan.org/addie.eichmann,2.0000000000,211.154.216.172,"{""location"": ""KM"", ""is_mobile"": true}" 1013,2,95,2016-12-16 02:53:51,http://trantowbatz.name/enola_reichert,0.0000000000,10.231.45.227,"{""location"": ""SI"", ""is_mobile"": false}" 1014,2,95,2017-01-28 05:36:37,http://paucek.io/irving,1.0000000000,10.231.45.227,"{""location"": ""FI"", ""is_mobile"": true}" 1015,2,95,2017-05-12 02:33:46,http://lesch.name/armand,2.0000000000,248.127.77.252,"{""location"": ""AT"", ""is_mobile"": true}" 1016,2,95,2017-04-30 21:04:35,http://rosenbaum.info/obie.rowe,2.0000000000,239.39.29.102,"{""location"": ""TJ"", ""is_mobile"": false}" 1017,2,96,2017-04-21 10:53:52,http://purdy.com/hilton_upton,0.0000000000,94.182.43.254,"{""location"": ""NZ"", ""is_mobile"": false}" 1018,2,96,2017-04-12 10:27:01,http://naderprosacco.biz/oleta,0.0000000000,96.247.237.123,"{""location"": ""NG"", ""is_mobile"": true}" 1019,2,96,2017-05-14 10:12:26,http://daugherty.name/mortimer,0.0000000000,62.234.207.134,"{""location"": ""IQ"", ""is_mobile"": false}" 1020,2,96,2017-04-04 16:18:33,http://davisdooley.org/javonte_bosco,0.0000000000,197.115.29.10,"{""location"": ""NP"", ""is_mobile"": true}" 1021,2,96,2017-04-13 20:33:51,http://damore.info/noe.conn,0.0000000000,40.223.243.94,"{""location"": ""GU"", ""is_mobile"": false}" 1022,2,96,2016-12-21 22:59:59,http://cummerata.org/berneice,0.0000000000,254.156.115.206,"{""location"": ""PM"", ""is_mobile"": false}" 1023,2,96,2017-04-29 22:35:12,http://jasthane.net/mathias_larson,0.0000000000,154.73.250.103,"{""location"": ""GG"", ""is_mobile"": false}" 1024,2,96,2017-05-25 22:09:37,http://okonbuckridge.co/cleora_hahn,0.0000000000,197.115.29.10,"{""location"": ""YT"", ""is_mobile"": true}" 1025,2,96,2017-05-05 01:47:33,http://wyman.biz/aleen.botsford,0.0000000000,242.72.205.242,"{""location"": ""ET"", ""is_mobile"": false}" 1026,2,96,2017-01-30 21:53:15,http://hermiston.name/peter,0.0000000000,134.249.7.66,"{""location"": ""TV"", ""is_mobile"": false}" 1027,2,96,2017-05-11 13:01:12,http://hegmann.co/terrance,0.0000000000,5.33.127.247,"{""location"": ""KZ"", ""is_mobile"": false}" 1028,2,96,2017-05-10 20:45:59,http://lang.io/muhammad.hudson,0.0000000000,242.72.205.242,"{""location"": ""MH"", ""is_mobile"": false}" 1029,2,96,2017-02-15 18:25:20,http://hermiston.org/parker,0.0000000000,197.115.29.10,"{""location"": ""KY"", ""is_mobile"": true}" 1030,2,96,2017-04-02 15:58:05,http://carter.com/cody_runolfsdottir,0.0000000000,153.117.199.153,"{""location"": ""NF"", ""is_mobile"": false}" 1031,2,96,2017-04-17 03:06:39,http://waelchi.net/carlee.mayert,0.0000000000,79.196.131.229,"{""location"": ""BL"", ""is_mobile"": true}" 1032,2,96,2017-02-19 05:30:26,http://grimesabernathy.io/sim,0.0000000000,154.73.250.103,"{""location"": ""CC"", ""is_mobile"": true}" 1033,2,96,2017-05-24 13:21:22,http://bergnaumkunde.co/sibyl.mclaughlin,0.0000000000,206.184.148.87,"{""location"": ""AZ"", ""is_mobile"": false}" 1034,2,96,2017-03-25 09:06:49,http://lebsackrodriguez.com/dwight.lueilwitz,0.0000000000,90.191.183.94,"{""location"": ""MP"", ""is_mobile"": true}" 1035,2,96,2017-06-06 01:08:44,http://skiles.co/zelda_flatley,0.0000000000,90.191.183.94,"{""location"": ""KM"", ""is_mobile"": true}" 1036,2,96,2017-05-22 05:20:38,http://gottlieb.net/ian,0.0000000000,49.138.113.173,"{""location"": ""BI"", ""is_mobile"": true}" 1037,2,97,2017-02-14 20:30:00,http://corwin.io/erich,1.0000000000,99.19.55.183,"{""location"": ""DK"", ""is_mobile"": true}" 1038,2,97,2017-01-15 03:24:34,http://stoltenbergwhite.io/tito,2.0000000000,20.92.83.154,"{""location"": ""GE"", ""is_mobile"": true}" 1039,2,97,2016-12-27 12:29:06,http://hirthe.org/sister.moen,1.0000000000,27.75.199.186,"{""location"": ""LU"", ""is_mobile"": false}" 1040,2,97,2017-04-13 08:54:51,http://framithompson.info/dallin.breitenberg,3.0000000000,114.110.242.4,"{""location"": ""UG"", ""is_mobile"": true}" 1041,2,97,2017-03-10 09:38:53,http://schmittfahey.com/elliott,3.0000000000,95.157.203.178,"{""location"": ""MP"", ""is_mobile"": false}" 1042,2,97,2017-04-18 13:26:00,http://halvorson.org/georgiana,1.0000000000,236.127.180.143,"{""location"": ""PL"", ""is_mobile"": true}" 1043,2,97,2017-03-18 00:36:03,http://nikolaus.net/vincenzo.lebsack,2.0000000000,185.90.196.249,"{""location"": ""MX"", ""is_mobile"": false}" 1044,2,97,2017-04-17 13:18:43,http://runolfsdottir.io/rhiannon.klein,3.0000000000,188.53.17.73,"{""location"": ""QA"", ""is_mobile"": true}" 1045,2,97,2017-03-03 06:55:01,http://goldner.net/bennett_friesen,3.0000000000,65.188.48.174,"{""location"": ""SY"", ""is_mobile"": true}" 1046,2,97,2017-02-04 13:19:55,http://lang.co/dasia.toy,0.0000000000,84.5.239.162,"{""location"": ""VE"", ""is_mobile"": false}" 1047,2,97,2017-02-11 13:49:02,http://kaulketillman.co/raoul,3.0000000000,185.81.106.202,"{""location"": ""LT"", ""is_mobile"": true}" 1048,2,98,2016-12-23 18:40:29,http://hills.biz/van,1.0000000000,129.85.129.59,"{""location"": ""GE"", ""is_mobile"": false}" 1049,2,98,2017-05-30 10:24:35,http://murray.biz/antonina.koch,1.0000000000,165.103.196.6,"{""location"": ""TG"", ""is_mobile"": false}" 4146,6,384,2017-02-08 13:10:48,http://connelly.com/lilly_bosco,2.0000000000,99.128.151.235,"{""location"": ""CI"", ""is_mobile"": true}" 4147,6,384,2017-04-28 18:00:10,http://spinka.org/margarette_goldner,1.0000000000,95.111.65.21,"{""location"": ""VG"", ""is_mobile"": true}" 4148,6,384,2017-01-17 11:15:09,http://stammwalker.biz/diamond,1.0000000000,197.213.155.164,"{""location"": ""CF"", ""is_mobile"": false}" 4149,6,384,2017-03-05 15:13:44,http://graham.com/ernest,0.0000000000,33.245.212.28,"{""location"": ""LC"", ""is_mobile"": false}" 4150,6,384,2017-04-17 13:26:03,http://prosaccorenner.co/cicero,1.0000000000,39.92.92.86,"{""location"": ""WS"", ""is_mobile"": false}" 4151,6,384,2017-02-18 10:03:08,http://trantowstreich.com/tre,0.0000000000,198.151.108.141,"{""location"": ""HU"", ""is_mobile"": false}" 4152,6,384,2016-12-31 12:20:24,http://medhurst.name/fausto,1.0000000000,113.6.29.235,"{""location"": ""IQ"", ""is_mobile"": false}" 4153,6,384,2017-03-21 15:28:37,http://jakubowski.io/lori,2.0000000000,200.126.218.178,"{""location"": ""WS"", ""is_mobile"": true}" 4154,6,384,2017-05-01 23:49:58,http://bashirian.com/caandre_bailey,0.0000000000,209.94.18.120,"{""location"": ""AE"", ""is_mobile"": true}" 4155,6,384,2017-02-26 17:25:50,http://nienow.com/general_bruen,0.0000000000,225.176.247.61,"{""location"": ""DZ"", ""is_mobile"": true}" 4156,6,385,2017-04-09 00:52:08,http://mantefritsch.co/reyna_gislason,2.0000000000,244.139.245.7,"{""location"": ""HK"", ""is_mobile"": true}" 4157,6,385,2017-03-05 02:30:35,http://koeppcarroll.org/millie_kertzmann,1.0000000000,123.41.129.83,"{""location"": ""ME"", ""is_mobile"": true}" 4158,6,385,2017-02-01 20:24:04,http://swiftbatz.net/jerad,3.0000000000,174.252.25.58,"{""location"": ""GB"", ""is_mobile"": true}" 4159,6,385,2016-12-19 07:44:34,http://gradyoberbrunner.com/marge_deckow,3.0000000000,69.42.192.172,"{""location"": ""SK"", ""is_mobile"": false}" 4160,6,385,2017-03-05 12:25:45,http://abbottreinger.net/beatrice,3.0000000000,48.123.125.55,"{""location"": ""HT"", ""is_mobile"": true}" 4161,6,385,2017-03-24 15:17:36,http://krajcik.biz/orion_mills,3.0000000000,153.45.143.73,"{""location"": ""PM"", ""is_mobile"": false}" 4162,6,385,2017-01-24 08:09:17,http://satterfield.net/terrell.franecki,3.0000000000,169.147.131.98,"{""location"": ""KZ"", ""is_mobile"": false}" 4163,6,385,2017-01-04 23:00:30,http://kshlerin.name/dion,1.0000000000,182.106.131.46,"{""location"": ""BS"", ""is_mobile"": true}" 4164,6,385,2016-12-31 16:26:49,http://welchheller.name/nash,0.0000000000,224.185.219.216,"{""location"": ""RW"", ""is_mobile"": false}" 4165,6,385,2017-01-21 21:13:48,http://sanford.name/wilma_beahan,0.0000000000,251.30.154.237,"{""location"": ""BQ"", ""is_mobile"": true}" 4166,6,385,2017-04-26 16:37:25,http://schulistnitzsche.net/dasia.stiedemann,3.0000000000,45.136.101.22,"{""location"": ""CV"", ""is_mobile"": true}" 4167,6,385,2017-03-26 17:25:36,http://schmeler.name/judson_gleason,1.0000000000,114.187.113.223,"{""location"": ""HN"", ""is_mobile"": true}" 4168,6,385,2017-01-13 02:50:38,http://wilderman.biz/zachariah.lubowitz,2.0000000000,224.185.219.216,"{""location"": ""PA"", ""is_mobile"": true}" 4169,6,386,2017-03-20 05:31:57,http://keelingkohler.io/estelle_bosco,2.0000000000,208.7.175.140,"{""location"": ""NA"", ""is_mobile"": false}" 4170,6,386,2017-03-30 00:34:19,http://jacobson.net/stella.shanahan,0.0000000000,218.26.147.62,"{""location"": ""SC"", ""is_mobile"": true}" 4171,6,386,2017-05-16 00:01:12,http://heidenreich.info/myrtle,2.0000000000,9.248.14.79,"{""location"": ""GF"", ""is_mobile"": false}" 4172,6,386,2017-06-04 01:20:33,http://eichmann.io/coby,1.0000000000,248.212.236.179,"{""location"": ""PL"", ""is_mobile"": true}" 4173,6,386,2017-02-06 19:34:23,http://cruickshanktoy.org/ceasar,0.0000000000,236.248.15.206,"{""location"": ""ME"", ""is_mobile"": true}" 4174,6,386,2016-12-26 17:46:59,http://gleichnerblick.biz/cletus.halvorson,1.0000000000,109.35.170.241,"{""location"": ""MO"", ""is_mobile"": true}" 4175,6,386,2017-01-15 07:27:48,http://harberhowell.com/lindsay,2.0000000000,230.75.42.18,"{""location"": ""VN"", ""is_mobile"": false}" 4176,6,386,2016-12-29 06:45:26,http://corwin.biz/katelynn_barrows,0.0000000000,36.94.140.169,"{""location"": ""GD"", ""is_mobile"": true}" 4177,6,386,2017-06-10 22:37:02,http://schoen.info/ezra.johnson,0.0000000000,160.230.207.86,"{""location"": ""ST"", ""is_mobile"": false}" 4178,6,386,2017-04-03 18:16:02,http://treutelmcclure.biz/berneice,1.0000000000,9.248.14.79,"{""location"": ""BW"", ""is_mobile"": true}" 4179,6,386,2017-02-23 00:47:40,http://gleichnerlang.io/derek,0.0000000000,80.138.102.19,"{""location"": ""MQ"", ""is_mobile"": true}" 4180,6,386,2017-06-11 16:10:31,http://conn.biz/keeley.hills,1.0000000000,93.14.40.175,"{""location"": ""LI"", ""is_mobile"": true}" 4181,6,386,2017-03-03 18:36:11,http://hermiston.net/cristopher,1.0000000000,72.212.227.151,"{""location"": ""IO"", ""is_mobile"": false}" 4182,6,386,2017-01-27 03:46:55,http://ruecker.name/myrna_keeling,0.0000000000,200.106.190.35,"{""location"": ""LS"", ""is_mobile"": true}" 4183,6,386,2017-03-24 15:08:44,http://erdman.net/hellen_kihn,2.0000000000,177.214.16.189,"{""location"": ""SN"", ""is_mobile"": false}" 4184,6,386,2017-05-11 03:47:02,http://feil.net/eulalia,0.0000000000,93.14.40.175,"{""location"": ""GL"", ""is_mobile"": false}" 4185,6,386,2017-03-14 01:06:57,http://miller.org/franco,0.0000000000,36.94.140.169,"{""location"": ""HN"", ""is_mobile"": false}" 4186,6,387,2017-01-26 17:49:03,http://ohara.biz/trycia,1.0000000000,147.172.244.207,"{""location"": ""MM"", ""is_mobile"": false}" 4187,6,387,2017-02-22 21:28:25,http://crona.io/al.streich,0.0000000000,77.160.80.35,"{""location"": ""MK"", ""is_mobile"": true}" 4188,6,387,2016-12-16 03:48:00,http://gleasonklein.com/lilian,1.0000000000,200.113.80.239,"{""location"": ""ST"", ""is_mobile"": true}" 4189,6,387,2016-12-27 03:19:39,http://klein.info/matilda_runolfon,0.0000000000,200.113.80.239,"{""location"": ""AL"", ""is_mobile"": false}" 4190,6,388,2017-05-20 00:57:21,http://considine.org/adella.kiehn,1.0000000000,195.143.4.171,"{""location"": ""FJ"", ""is_mobile"": true}" 4191,6,388,2016-12-18 18:20:52,http://greenfelderwitting.net/ayden,1.0000000000,206.25.216.127,"{""location"": ""PS"", ""is_mobile"": false}" 4192,6,388,2017-01-05 13:11:21,http://kuhlmanjast.org/bryon,1.0000000000,40.213.147.154,"{""location"": ""GF"", ""is_mobile"": false}" 4193,6,388,2017-05-31 09:59:51,http://feil.org/meta_wiegand,0.0000000000,84.13.211.172,"{""location"": ""PH"", ""is_mobile"": false}" 4194,6,388,2017-05-02 20:39:58,http://kutch.org/rhianna,0.0000000000,90.95.82.158,"{""location"": ""LA"", ""is_mobile"": false}" 4195,6,388,2017-05-17 05:18:13,http://ondrickagaylord.info/lenore,1.0000000000,7.57.169.183,"{""location"": ""UG"", ""is_mobile"": true}" 4196,6,388,2017-05-30 05:22:20,http://lueilwitz.name/aurelia,1.0000000000,163.64.5.4,"{""location"": ""RU"", ""is_mobile"": true}" 1050,2,98,2017-06-07 20:51:55,http://kautzer.io/johnnie,1.0000000000,63.155.185.207,"{""location"": ""BE"", ""is_mobile"": false}" 1051,2,98,2017-05-25 21:13:37,http://rogahnbarrows.info/theresa_huel,1.0000000000,28.123.59.27,"{""location"": ""MS"", ""is_mobile"": true}" 1052,2,98,2017-01-15 04:47:39,http://okuneva.info/caesar,0.0000000000,56.71.107.209,"{""location"": ""SN"", ""is_mobile"": false}" 1053,2,98,2017-03-14 05:33:09,http://ankunding.name/henri.daugherty,0.0000000000,30.140.43.59,"{""location"": ""AR"", ""is_mobile"": true}" 1054,2,99,2017-02-10 22:45:10,http://windler.com/ebba_price,0.0000000000,74.43.60.150,"{""location"": ""DK"", ""is_mobile"": true}" 1055,2,99,2017-02-04 01:37:29,http://ortiz.co/dolores,0.0000000000,240.145.6.250,"{""location"": ""LI"", ""is_mobile"": true}" 1056,2,99,2017-03-29 14:39:32,http://lesch.biz/rosalind,1.0000000000,103.160.232.163,"{""location"": ""SC"", ""is_mobile"": false}" 1057,2,99,2017-05-19 22:42:46,http://buckridge.co/victoria,1.0000000000,219.245.130.191,"{""location"": ""CM"", ""is_mobile"": false}" 1058,2,99,2017-04-13 03:05:59,http://wisozk.net/angelica,3.0000000000,56.93.157.16,"{""location"": ""GP"", ""is_mobile"": true}" 1059,2,99,2017-04-24 08:24:21,http://rau.net/jedediah_metz,3.0000000000,175.88.125.50,"{""location"": ""MA"", ""is_mobile"": false}" 1060,2,99,2017-01-08 04:45:58,http://steubertorphy.com/terry,0.0000000000,88.194.65.245,"{""location"": ""ES"", ""is_mobile"": true}" 1061,2,99,2017-03-28 20:46:34,http://beermcdermott.org/anna_olson,1.0000000000,225.98.152.231,"{""location"": ""KP"", ""is_mobile"": true}" 1062,2,99,2017-01-09 15:34:05,http://fisher.info/luz,3.0000000000,127.248.237.241,"{""location"": ""MD"", ""is_mobile"": false}" 1063,2,100,2017-03-29 22:07:21,http://cainrodriguez.com/te_schowalter,1.0000000000,236.222.115.118,"{""location"": ""SI"", ""is_mobile"": false}" 1064,2,100,2017-03-19 05:08:36,http://shields.name/agnes,3.0000000000,125.64.185.157,"{""location"": ""VG"", ""is_mobile"": false}" 1065,2,100,2017-05-03 16:07:11,http://runolfon.name/karine,1.0000000000,81.125.184.120,"{""location"": ""HT"", ""is_mobile"": false}" 1066,2,100,2017-05-04 08:52:32,http://larkin.name/johathan_gleason,1.0000000000,119.80.218.10,"{""location"": ""PA"", ""is_mobile"": true}" 1067,2,100,2017-02-23 01:21:30,http://braunturcotte.net/jude.haag,0.0000000000,74.131.229.207,"{""location"": ""BW"", ""is_mobile"": true}" 1068,2,100,2017-02-18 23:45:39,http://moriettewisozk.com/raoul,3.0000000000,93.70.150.79,"{""location"": ""LI"", ""is_mobile"": false}" 1069,2,100,2017-05-22 22:36:15,http://kubbrekke.info/conrad_balistreri,2.0000000000,172.248.142.213,"{""location"": ""LR"", ""is_mobile"": false}" 1070,2,100,2016-12-20 10:12:41,http://tillman.com/sherwood,2.0000000000,222.187.21.159,"{""location"": ""PE"", ""is_mobile"": false}" 1071,2,100,2017-03-27 16:01:17,http://bernhard.info/presley,3.0000000000,81.125.184.120,"{""location"": ""MC"", ""is_mobile"": false}" 1072,2,100,2017-05-14 16:03:46,http://pouros.co/mikayla.johnston,3.0000000000,125.64.185.157,"{""location"": ""AS"", ""is_mobile"": false}" 1073,2,100,2017-01-23 04:57:15,http://baumbachmoriette.io/mireille,0.0000000000,49.35.135.225,"{""location"": ""FR"", ""is_mobile"": false}" 1074,2,100,2017-06-03 10:55:08,http://dachlittle.io/kaleigh,1.0000000000,70.236.185.228,"{""location"": ""PH"", ""is_mobile"": false}" 1075,2,100,2016-12-31 09:21:38,http://smitham.info/colin,0.0000000000,72.139.147.193,"{""location"": ""TF"", ""is_mobile"": true}" 1076,2,100,2016-12-31 11:43:58,http://rowemann.co/abigale,2.0000000000,93.70.150.79,"{""location"": ""JP"", ""is_mobile"": true}" 1077,2,100,2017-06-13 01:52:39,http://yost.name/wilbert,2.0000000000,84.156.245.33,"{""location"": ""IO"", ""is_mobile"": false}" 1078,2,101,2017-02-27 20:05:02,http://schimmel.net/mike,1.0000000000,100.243.5.194,"{""location"": ""SY"", ""is_mobile"": true}" 1079,2,101,2017-01-14 02:24:26,http://kuhlmanlittle.io/emiliano_schneider,0.0000000000,138.159.38.233,"{""location"": ""DZ"", ""is_mobile"": true}" 1080,2,101,2017-03-03 13:46:27,http://schroeder.com/anjali_beahan,1.0000000000,37.163.208.78,"{""location"": ""DM"", ""is_mobile"": false}" 1081,2,101,2017-02-13 08:40:13,http://bashirianemard.net/alayna_lueilwitz,1.0000000000,219.20.93.50,"{""location"": ""ZM"", ""is_mobile"": false}" 1082,2,101,2016-12-20 22:12:28,http://mayer.name/mattie.bashirian,1.0000000000,100.4.49.192,"{""location"": ""IS"", ""is_mobile"": true}" 1083,2,101,2017-02-14 10:54:09,http://monahanmraz.co/brianne,0.0000000000,219.20.93.50,"{""location"": ""YE"", ""is_mobile"": true}" 1084,2,101,2017-01-10 10:43:12,http://morar.net/ova,1.0000000000,100.4.49.192,"{""location"": ""BJ"", ""is_mobile"": false}" 1085,2,101,2017-06-02 21:11:37,http://hagenes.biz/christopher_stark,0.0000000000,206.66.105.66,"{""location"": ""IE"", ""is_mobile"": false}" 1086,2,101,2016-12-21 09:44:49,http://hagenes.biz/mario.blanda,1.0000000000,205.150.12.159,"{""location"": ""AO"", ""is_mobile"": true}" 1087,2,101,2017-01-03 23:23:00,http://greenfelder.net/mariano_donnelly,0.0000000000,219.20.93.50,"{""location"": ""RS"", ""is_mobile"": true}" 1088,2,101,2017-04-05 09:07:00,http://leuschkecrist.net/damaris,0.0000000000,206.66.105.66,"{""location"": ""IO"", ""is_mobile"": true}" 1089,2,101,2017-04-27 04:52:19,http://rogahn.info/kenyon,0.0000000000,78.76.126.64,"{""location"": ""KR"", ""is_mobile"": true}" 1090,2,101,2017-06-09 11:26:53,http://fay.net/giuseppe.volkman,0.0000000000,48.92.122.126,"{""location"": ""AS"", ""is_mobile"": false}" 1091,2,101,2017-01-09 17:54:49,http://herzogrobel.org/leanna.dare,1.0000000000,96.233.23.219,"{""location"": ""LB"", ""is_mobile"": true}" 1092,2,101,2017-01-07 01:42:13,http://gislasongraham.name/leie,0.0000000000,100.243.5.194,"{""location"": ""MU"", ""is_mobile"": true}" 1093,2,101,2017-01-30 16:18:28,http://franeckitreutel.info/dee_gaylord,1.0000000000,37.23.89.112,"{""location"": ""DO"", ""is_mobile"": true}" 1094,2,101,2017-03-15 14:03:51,http://hilll.net/kyleigh,1.0000000000,103.34.227.221,"{""location"": ""UA"", ""is_mobile"": false}" 1095,2,101,2017-01-16 12:20:52,http://ankunding.biz/rolando,1.0000000000,120.156.225.18,"{""location"": ""KE"", ""is_mobile"": false}" 1096,2,101,2017-05-18 08:15:57,http://mertz.biz/lyda_runolfsdottir,0.0000000000,206.66.105.66,"{""location"": ""KG"", ""is_mobile"": true}" 1097,2,101,2017-05-31 19:36:44,http://kertzmann.co/harmon.roberts,1.0000000000,63.158.79.67,"{""location"": ""KH"", ""is_mobile"": false}" 1098,2,102,2017-06-08 00:28:18,http://jacobs.org/brisa,1.0000000000,100.227.45.71,"{""location"": ""AO"", ""is_mobile"": true}" 1099,2,102,2017-01-05 21:01:04,http://schultz.info/rudolph_gerlach,1.0000000000,24.55.203.124,"{""location"": ""EC"", ""is_mobile"": true}" 1100,2,102,2017-02-09 16:18:02,http://lang.co/ottis_gibson,2.0000000000,7.248.109.244,"{""location"": ""FR"", ""is_mobile"": false}" 1101,2,102,2017-02-05 10:03:17,http://rosenbaum.name/miles,1.0000000000,104.201.28.103,"{""location"": ""AZ"", ""is_mobile"": true}" 4197,6,388,2017-01-14 07:33:49,http://jacobs.co/loraine,0.0000000000,195.143.4.171,"{""location"": ""BR"", ""is_mobile"": false}" 4198,6,388,2017-06-12 10:57:40,http://goldner.co/meda.rogahn,0.0000000000,206.25.216.127,"{""location"": ""BT"", ""is_mobile"": true}" 4199,6,388,2016-12-29 10:39:08,http://kihn.org/jaime,1.0000000000,248.60.119.157,"{""location"": ""NC"", ""is_mobile"": false}" 4200,6,388,2017-01-03 22:57:27,http://nitzsche.io/gaston.wisoky,1.0000000000,230.157.31.242,"{""location"": ""CU"", ""is_mobile"": true}" 4201,6,388,2017-03-21 02:41:38,http://hamillkeeling.org/abbie,0.0000000000,70.246.158.56,"{""location"": ""GQ"", ""is_mobile"": true}" 4202,6,389,2017-02-18 13:48:31,http://sauerwiza.biz/shyann,2.0000000000,136.64.234.72,"{""location"": ""MR"", ""is_mobile"": false}" 4203,6,389,2017-04-20 18:02:34,http://langworthspinka.io/adan,1.0000000000,193.229.240.88,"{""location"": ""BQ"", ""is_mobile"": true}" 4204,6,390,2017-04-15 06:27:24,http://hammes.co/alf_strosin,1.0000000000,5.35.236.33,"{""location"": ""BT"", ""is_mobile"": false}" 4205,6,390,2017-05-08 00:06:35,http://litteldouglas.org/cory,0.0000000000,106.74.105.63,"{""location"": ""CM"", ""is_mobile"": false}" 4206,6,390,2017-04-03 01:50:10,http://klein.net/darius.schmeler,0.0000000000,104.77.146.219,"{""location"": ""WF"", ""is_mobile"": true}" 4207,6,390,2017-04-28 09:51:37,http://schroeder.biz/josephine_lindgren,1.0000000000,51.143.219.104,"{""location"": ""SD"", ""is_mobile"": true}" 4208,6,390,2017-01-07 15:46:34,http://robel.info/austyn,0.0000000000,112.139.212.193,"{""location"": ""RU"", ""is_mobile"": false}" 4209,6,390,2017-05-20 11:02:22,http://cruickshankmayer.name/daniela_glover,1.0000000000,206.215.21.55,"{""location"": ""MF"", ""is_mobile"": false}" 4210,6,390,2017-05-18 11:19:15,http://mclaughlin.info/laury.doyle,1.0000000000,104.77.146.219,"{""location"": ""MA"", ""is_mobile"": true}" 4211,6,390,2017-02-06 16:08:47,http://prosaccoabbott.net/kailee.konopelski,0.0000000000,15.194.237.229,"{""location"": ""BN"", ""is_mobile"": false}" 4212,6,390,2017-01-09 21:58:14,http://mertz.org/peggie,0.0000000000,21.147.74.226,"{""location"": ""AW"", ""is_mobile"": true}" 4213,6,390,2016-12-17 16:51:18,http://raynor.biz/orie,0.0000000000,106.74.105.63,"{""location"": ""ID"", ""is_mobile"": false}" 4214,6,391,2017-05-08 05:42:52,http://rogahn.net/erik,1.0000000000,186.130.175.236,"{""location"": ""GB"", ""is_mobile"": false}" 4215,6,391,2017-05-18 12:45:16,http://ullrichhintz.org/lilla,0.0000000000,87.83.247.248,"{""location"": ""IQ"", ""is_mobile"": false}" 4216,6,391,2017-03-16 10:11:42,http://lakin.com/katrina_beer,0.0000000000,10.225.225.143,"{""location"": ""HK"", ""is_mobile"": true}" 4217,6,391,2017-04-10 17:43:05,http://hagenes.io/jermain.parisian,1.0000000000,239.177.253.149,"{""location"": ""BJ"", ""is_mobile"": true}" 4218,6,391,2017-01-25 10:02:25,http://farrell.info/jeremie,1.0000000000,207.163.49.122,"{""location"": ""GT"", ""is_mobile"": true}" 4219,6,391,2017-03-06 05:41:02,http://bernierlakin.net/nikolas,1.0000000000,16.221.132.104,"{""location"": ""PA"", ""is_mobile"": false}" 4220,6,391,2017-06-10 18:50:55,http://schneider.io/litzy_mosciski,0.0000000000,183.86.222.89,"{""location"": ""GF"", ""is_mobile"": false}" 4221,6,391,2017-04-02 19:06:14,http://keeling.com/cyrus_goyette,1.0000000000,103.250.56.171,"{""location"": ""CA"", ""is_mobile"": false}" 4222,6,391,2017-01-24 00:31:13,http://rowe.biz/anna.smith,0.0000000000,84.122.214.100,"{""location"": ""DZ"", ""is_mobile"": false}" 4223,6,392,2017-04-01 19:01:24,http://ledner.io/katrina.schamberger,0.0000000000,82.211.213.81,"{""location"": ""SV"", ""is_mobile"": false}" 4224,6,392,2017-01-09 01:45:57,http://greenfelder.net/aron,0.0000000000,33.228.50.45,"{""location"": ""UG"", ""is_mobile"": false}" 4225,6,392,2017-01-13 21:33:56,http://dubuquereilly.name/betsy,0.0000000000,245.25.18.252,"{""location"": ""MW"", ""is_mobile"": false}" 4226,6,392,2017-02-05 06:13:09,http://glovercollier.biz/jaylin.keeling,1.0000000000,189.153.130.133,"{""location"": ""TR"", ""is_mobile"": true}" 4227,6,393,2017-05-30 04:20:36,http://brekke.com/jeyca,0.0000000000,147.169.88.2,"{""location"": ""MU"", ""is_mobile"": true}" 4228,6,393,2017-05-13 21:30:42,http://erdman.co/lavinia_hansen,0.0000000000,120.41.13.234,"{""location"": ""VU"", ""is_mobile"": false}" 4229,6,393,2017-01-31 12:57:12,http://bergnaum.name/carli.kautzer,0.0000000000,91.218.14.56,"{""location"": ""IL"", ""is_mobile"": true}" 4230,6,393,2017-03-20 22:32:30,http://nitzschekoch.net/dewayne.klein,0.0000000000,147.7.193.84,"{""location"": ""BV"", ""is_mobile"": true}" 4231,6,394,2017-05-30 19:34:54,http://towne.name/destany_rau,0.0000000000,18.114.128.147,"{""location"": ""SL"", ""is_mobile"": true}" 4232,6,394,2016-12-25 13:32:13,http://bartoletti.org/kattie.stoltenberg,2.0000000000,41.165.243.185,"{""location"": ""PL"", ""is_mobile"": true}" 4233,6,394,2017-05-13 14:12:32,http://gislason.org/bernadine,3.0000000000,138.208.198.208,"{""location"": ""NU"", ""is_mobile"": true}" 4234,6,395,2017-02-19 11:53:41,http://beahanklocko.io/emelie_dach,1.0000000000,96.178.215.81,"{""location"": ""PS"", ""is_mobile"": true}" 4235,6,395,2017-02-10 21:05:21,http://nicolas.info/malachi,0.0000000000,148.64.216.180,"{""location"": ""GE"", ""is_mobile"": true}" 4236,6,395,2016-12-17 13:28:31,http://johnston.co/tianna,2.0000000000,82.31.178.240,"{""location"": ""TL"", ""is_mobile"": true}" 4237,6,395,2017-02-15 05:59:48,http://padberg.name/eliezer_little,2.0000000000,153.74.48.31,"{""location"": ""VG"", ""is_mobile"": false}" 4238,6,395,2017-04-25 00:32:03,http://reinger.com/reese,1.0000000000,153.74.48.31,"{""location"": ""GE"", ""is_mobile"": true}" 4239,6,395,2017-03-28 03:13:55,http://armstrong.name/orlando,1.0000000000,28.2.146.208,"{""location"": ""GQ"", ""is_mobile"": true}" 4240,6,395,2016-12-21 07:11:36,http://zieme.info/breana,2.0000000000,181.184.36.101,"{""location"": ""AS"", ""is_mobile"": true}" 4241,6,395,2017-06-08 07:22:40,http://labadie.co/irma,1.0000000000,148.64.216.180,"{""location"": ""GL"", ""is_mobile"": true}" 4242,6,395,2016-12-17 07:09:58,http://larsonohara.io/natasha,1.0000000000,112.146.203.125,"{""location"": ""RE"", ""is_mobile"": false}" 4243,6,395,2017-03-19 10:02:46,http://mckenzieyundt.biz/irma,2.0000000000,38.52.84.15,"{""location"": ""US"", ""is_mobile"": false}" 4244,6,395,2017-03-19 13:18:15,http://nitzsche.io/gianni,2.0000000000,79.253.79.88,"{""location"": ""CD"", ""is_mobile"": true}" 4245,6,395,2017-05-12 06:48:37,http://jacobsonwiza.info/cheyenne.skiles,2.0000000000,13.169.212.31,"{""location"": ""BJ"", ""is_mobile"": false}" 4246,6,395,2017-04-19 09:57:02,http://pouros.org/leila.oberbrunner,1.0000000000,33.23.248.224,"{""location"": ""BQ"", ""is_mobile"": true}" 4247,6,395,2017-05-11 11:36:58,http://williamson.info/susanna_conn,1.0000000000,224.39.40.247,"{""location"": ""JP"", ""is_mobile"": true}" 4248,6,395,2017-05-04 14:32:05,http://mraz.co/cyrus.schinner,2.0000000000,153.74.48.31,"{""location"": ""TC"", ""is_mobile"": true}" 1102,2,102,2017-03-01 03:04:24,http://zboncak.net/dovie_pfannerstill,3.0000000000,43.177.73.69,"{""location"": ""MK"", ""is_mobile"": true}" 1103,2,102,2016-12-24 22:05:23,http://kuhlman.net/josh,0.0000000000,79.42.153.123,"{""location"": ""LB"", ""is_mobile"": false}" 1104,2,102,2017-02-22 13:51:03,http://cummerata.com/herbert.crona,2.0000000000,246.246.42.192,"{""location"": ""UM"", ""is_mobile"": true}" 1105,2,102,2017-06-10 19:43:27,http://bartell.info/delilah.waters,1.0000000000,143.212.133.154,"{""location"": ""SE"", ""is_mobile"": false}" 1106,2,102,2017-04-18 15:02:36,http://pfannerstill.net/tristin,0.0000000000,55.229.122.65,"{""location"": ""RE"", ""is_mobile"": false}" 1107,2,102,2016-12-17 12:21:28,http://mayertbruen.biz/deion,3.0000000000,104.201.28.103,"{""location"": ""MK"", ""is_mobile"": false}" 1108,2,102,2016-12-30 21:57:55,http://cormier.biz/christelle_purdy,3.0000000000,134.103.51.208,"{""location"": ""SS"", ""is_mobile"": false}" 1109,2,102,2017-02-08 09:23:43,http://schmidt.info/elise,3.0000000000,248.62.120.57,"{""location"": ""KM"", ""is_mobile"": true}" 1110,2,102,2017-01-11 22:28:03,http://nolan.com/joyce,0.0000000000,233.161.35.210,"{""location"": ""VE"", ""is_mobile"": true}" 1111,2,102,2017-05-12 03:44:58,http://kuphalhirthe.net/flavie,0.0000000000,208.138.71.208,"{""location"": ""GG"", ""is_mobile"": true}" 1112,2,102,2017-03-03 09:39:29,http://cartwright.biz/jayme.brakus,2.0000000000,163.19.227.163,"{""location"": ""LU"", ""is_mobile"": false}" 1113,2,103,2017-01-10 10:30:07,http://veumgleichner.io/laurence.lowe,2.0000000000,175.74.236.203,"{""location"": ""GY"", ""is_mobile"": true}" 1114,2,103,2017-05-14 08:38:23,http://rowemarquardt.biz/wilmer.murray,0.0000000000,246.53.192.144,"{""location"": ""ER"", ""is_mobile"": false}" 1115,2,103,2017-04-10 09:13:56,http://harvey.io/marley_gutkowski,0.0000000000,5.119.110.42,"{""location"": ""BM"", ""is_mobile"": false}" 1116,2,103,2017-04-27 06:04:56,http://quigley.com/jett.grimes,0.0000000000,66.151.162.227,"{""location"": ""WF"", ""is_mobile"": true}" 1117,2,103,2017-05-30 04:22:08,http://treutel.name/elna,1.0000000000,36.13.50.152,"{""location"": ""AQ"", ""is_mobile"": true}" 1118,2,103,2017-05-13 06:36:41,http://raynormurphy.co/baylee,1.0000000000,211.161.21.56,"{""location"": ""ML"", ""is_mobile"": false}" 1119,2,103,2017-06-06 06:26:42,http://wisoky.biz/monty,0.0000000000,211.161.21.56,"{""location"": ""RU"", ""is_mobile"": true}" 1120,2,103,2017-06-05 21:34:59,http://conn.info/freida_bartell,2.0000000000,246.148.55.15,"{""location"": ""HR"", ""is_mobile"": false}" 1121,2,103,2017-04-13 01:53:18,http://reichertchristiansen.name/aglae_roberts,2.0000000000,231.199.31.88,"{""location"": ""NC"", ""is_mobile"": true}" 1122,2,103,2017-04-26 16:40:35,http://yundtswift.co/janiya,1.0000000000,35.81.86.95,"{""location"": ""UG"", ""is_mobile"": true}" 1123,2,103,2017-02-05 22:53:50,http://goldner.org/garnett_wehner,1.0000000000,222.206.217.252,"{""location"": ""MR"", ""is_mobile"": false}" 1124,2,103,2017-03-03 17:28:00,http://kuhic.name/arvid,1.0000000000,109.184.67.235,"{""location"": ""ZM"", ""is_mobile"": true}" 1125,2,103,2017-03-15 09:28:02,http://nicolas.info/lue.thompson,1.0000000000,231.199.31.88,"{""location"": ""GH"", ""is_mobile"": false}" 1126,2,103,2017-04-28 03:09:08,http://mcclure.info/walter,2.0000000000,96.228.118.116,"{""location"": ""SA"", ""is_mobile"": false}" 1127,2,104,2017-01-29 11:15:39,http://abshire.com/jerrell_kautzer,1.0000000000,22.15.63.248,"{""location"": ""AQ"", ""is_mobile"": true}" 1128,2,104,2017-03-22 08:32:30,http://deckow.info/angelo_howell,1.0000000000,248.156.251.90,"{""location"": ""SB"", ""is_mobile"": true}" 1129,2,104,2017-05-27 23:34:49,http://schultz.name/alejandra_nolan,1.0000000000,242.61.123.84,"{""location"": ""PM"", ""is_mobile"": true}" 1130,2,104,2017-05-15 11:20:42,http://tillmanbarton.biz/nolan.ratke,2.0000000000,128.50.91.51,"{""location"": ""PY"", ""is_mobile"": false}" 1131,2,104,2016-12-22 11:58:25,http://shanahanshields.biz/isidro,0.0000000000,128.196.54.78,"{""location"": ""PG"", ""is_mobile"": true}" 1132,2,104,2017-03-08 22:33:30,http://rice.biz/royal,0.0000000000,18.226.164.119,"{""location"": ""ET"", ""is_mobile"": false}" 1133,2,104,2017-06-08 11:59:19,http://schulist.biz/catharine_kreiger,1.0000000000,132.73.4.193,"{""location"": ""KI"", ""is_mobile"": true}" 1134,2,104,2017-01-27 09:40:24,http://king.net/amani.altenwerth,0.0000000000,248.156.251.90,"{""location"": ""AS"", ""is_mobile"": false}" 1135,2,104,2017-04-19 05:30:05,http://marquardt.io/dorthy_lang,2.0000000000,180.62.248.147,"{""location"": ""TW"", ""is_mobile"": true}" 1136,2,104,2016-12-24 08:03:40,http://schumm.biz/deondre.brekke,2.0000000000,180.62.248.147,"{""location"": ""MH"", ""is_mobile"": true}" 1137,2,104,2017-05-30 06:45:55,http://waltergreenfelder.io/iliana,1.0000000000,178.210.92.74,"{""location"": ""NF"", ""is_mobile"": false}" 1138,2,105,2017-01-21 16:10:16,http://powlowskimueller.name/ole,3.0000000000,38.22.118.117,"{""location"": ""VU"", ""is_mobile"": true}" 1139,2,105,2017-01-19 09:57:09,http://kuhlman.com/vada_maggio,0.0000000000,9.105.174.39,"{""location"": ""SO"", ""is_mobile"": true}" 1140,2,105,2017-05-31 17:42:13,http://lindgren.org/vivian.davis,1.0000000000,98.89.173.231,"{""location"": ""IL"", ""is_mobile"": false}" 1141,2,105,2017-04-27 22:49:51,http://ornlittle.com/ilene.gutkowski,0.0000000000,178.65.108.16,"{""location"": ""TV"", ""is_mobile"": true}" 1142,2,105,2017-01-23 21:54:23,http://heidenreich.info/rashawn.gaylord,0.0000000000,170.50.24.250,"{""location"": ""EG"", ""is_mobile"": true}" 1143,2,105,2017-05-10 17:21:56,http://macgyver.name/luisa,3.0000000000,16.44.10.195,"{""location"": ""VC"", ""is_mobile"": true}" 1144,2,105,2017-03-11 22:25:05,http://schultz.co/terry.heel,0.0000000000,178.65.108.16,"{""location"": ""CY"", ""is_mobile"": false}" 1145,2,105,2017-06-10 13:28:55,http://walter.com/neoma_fay,1.0000000000,165.111.80.205,"{""location"": ""KZ"", ""is_mobile"": false}" 1146,2,105,2017-01-01 17:32:13,http://gorczanylabadie.biz/winnifred,0.0000000000,98.89.173.231,"{""location"": ""CO"", ""is_mobile"": false}" 1147,2,105,2017-03-10 17:52:25,http://hammesrosenbaum.org/josephine.rohan,2.0000000000,100.56.36.195,"{""location"": ""UY"", ""is_mobile"": true}" 1148,2,105,2017-01-18 17:40:12,http://collier.net/glennie,3.0000000000,73.82.203.73,"{""location"": ""GL"", ""is_mobile"": false}" 1149,2,105,2017-01-24 02:01:34,http://jenkins.io/stewart,0.0000000000,9.242.164.166,"{""location"": ""GQ"", ""is_mobile"": false}" 1150,2,105,2016-12-30 23:00:26,http://beatty.io/laria,0.0000000000,178.65.108.16,"{""location"": ""MY"", ""is_mobile"": true}" 1151,2,105,2017-06-10 10:09:15,http://gleason.co/maddison,0.0000000000,67.94.88.217,"{""location"": ""DJ"", ""is_mobile"": false}" 1152,2,105,2017-01-24 12:16:45,http://emmerich.net/berry,1.0000000000,16.55.132.222,"{""location"": ""NG"", ""is_mobile"": false}" 4249,6,396,2016-12-25 16:44:47,http://denesik.co/marcelina,,231.196.211.121,"{""location"": ""JO"", ""is_mobile"": false}" 4250,6,396,2017-05-30 02:07:03,http://deckowcrist.info/eloy,,96.115.195.74,"{""location"": ""MV"", ""is_mobile"": false}" 4251,6,396,2017-04-30 18:17:35,http://wolf.com/shany_nitzsche,,230.100.237.89,"{""location"": ""ID"", ""is_mobile"": false}" 4252,6,396,2017-01-29 23:16:04,http://beerraynor.info/alexandre.baumbach,,39.72.93.43,"{""location"": ""BA"", ""is_mobile"": true}" 4253,6,396,2017-02-19 13:05:52,http://welch.biz/merl.kohler,,147.114.234.122,"{""location"": ""CH"", ""is_mobile"": false}" 4254,6,396,2017-05-18 18:48:59,http://kemmernicolas.io/dagmar,,204.166.123.67,"{""location"": ""LT"", ""is_mobile"": false}" 4255,6,396,2017-02-14 00:36:56,http://macejkovic.org/abel,,103.109.207.52,"{""location"": ""MS"", ""is_mobile"": false}" 4256,6,396,2017-03-20 20:22:48,http://crona.co/karson,,176.131.191.108,"{""location"": ""BV"", ""is_mobile"": true}" 4257,6,396,2017-02-15 02:48:03,http://beattykautzer.biz/leta,,179.117.95.192,"{""location"": ""AM"", ""is_mobile"": false}" 4258,6,396,2017-01-26 20:18:31,http://purdy.io/kelvin.lindgren,,179.117.95.192,"{""location"": ""DE"", ""is_mobile"": false}" 4259,6,396,2017-02-02 00:34:34,http://ratke.net/cole.konopelski,,59.133.252.234,"{""location"": ""KY"", ""is_mobile"": true}" 4260,6,396,2017-04-30 08:05:04,http://schaden.co/benjamin,,20.95.83.137,"{""location"": ""NO"", ""is_mobile"": true}" 4261,6,396,2017-03-02 19:49:35,http://schoenbernier.org/kali_metz,,229.97.54.190,"{""location"": ""UY"", ""is_mobile"": true}" 4262,6,396,2017-01-24 02:24:12,http://bailey.com/annetta,,40.211.177.154,"{""location"": ""IL"", ""is_mobile"": true}" 4263,6,396,2017-06-09 23:52:57,http://streichhirthe.com/chauncey,,57.186.74.80,"{""location"": ""EE"", ""is_mobile"": false}" 4264,6,397,2017-01-11 01:48:50,http://gusikowski.co/breanne.ondricka,,230.135.47.177,"{""location"": ""LA"", ""is_mobile"": false}" 4265,6,397,2017-05-29 08:46:00,http://quigley.com/rosalee,,176.105.100.179,"{""location"": ""SX"", ""is_mobile"": false}" 4266,6,397,2016-12-20 17:59:34,http://kuhic.info/heaven,,86.58.244.67,"{""location"": ""AR"", ""is_mobile"": true}" 4267,6,397,2017-03-23 03:39:52,http://nolan.io/hester_wuckert,,60.58.254.196,"{""location"": ""SI"", ""is_mobile"": true}" 4268,6,397,2017-02-23 20:26:44,http://hackettconn.io/nicholas,,64.225.23.210,"{""location"": ""GT"", ""is_mobile"": false}" 4269,6,398,2017-01-22 01:40:01,http://bahringer.com/christine,,115.133.247.160,"{""location"": ""BJ"", ""is_mobile"": false}" 4270,6,398,2017-02-17 11:30:41,http://westfay.com/lillian_walsh,,121.133.166.147,"{""location"": ""CL"", ""is_mobile"": true}" 4271,6,398,2016-12-31 08:42:46,http://maggio.org/edmond,,35.208.58.239,"{""location"": ""PA"", ""is_mobile"": false}" 4272,6,398,2017-05-30 03:31:58,http://legros.biz/virginie,,11.231.231.34,"{""location"": ""RW"", ""is_mobile"": false}" 4273,6,398,2017-04-02 00:45:23,http://friesen.net/mafalda,,37.181.122.228,"{""location"": ""PF"", ""is_mobile"": false}" 4274,6,398,2017-06-07 06:38:57,http://hellerfeil.io/wilber_bosco,,152.37.35.78,"{""location"": ""GT"", ""is_mobile"": true}" 4275,6,398,2017-02-04 16:18:12,http://gerhold.info/erika,,83.166.55.203,"{""location"": ""FJ"", ""is_mobile"": true}" 4276,6,398,2017-03-12 22:20:08,http://ankunding.name/ramon,,189.152.140.26,"{""location"": ""TF"", ""is_mobile"": true}" 4277,6,398,2017-01-21 05:17:52,http://schulist.io/florencio_hettinger,,35.208.58.239,"{""location"": ""FJ"", ""is_mobile"": false}" 4278,6,398,2017-04-27 10:18:36,http://lowe.net/genesis,,152.37.35.78,"{""location"": ""IQ"", ""is_mobile"": false}" 4279,6,398,2017-04-04 05:58:38,http://doyle.info/annetta.johnston,,44.41.223.150,"{""location"": ""GN"", ""is_mobile"": true}" 4280,6,398,2017-03-02 04:28:34,http://braun.org/te,,150.149.142.169,"{""location"": ""FI"", ""is_mobile"": true}" 4281,6,398,2017-04-07 20:31:56,http://zemlakrice.name/miller,,61.172.225.104,"{""location"": ""SA"", ""is_mobile"": true}" 4282,6,398,2017-03-18 01:55:42,http://hansenjacobson.co/caie.miller,,145.168.3.191,"{""location"": ""PA"", ""is_mobile"": false}" 4283,6,398,2017-03-18 16:31:28,http://macgyver.info/clare,,83.166.55.203,"{""location"": ""GQ"", ""is_mobile"": false}" 4284,6,398,2017-01-18 23:44:31,http://kling.io/gwen,,37.181.122.228,"{""location"": ""PN"", ""is_mobile"": true}" 4285,6,398,2017-03-28 07:02:21,http://gutmann.io/alverta,,123.8.250.153,"{""location"": ""NE"", ""is_mobile"": true}" 4286,6,398,2017-06-07 21:50:53,http://prosacco.org/esther_lueilwitz,,79.118.183.112,"{""location"": ""LS"", ""is_mobile"": false}" 4287,6,399,2017-03-11 15:42:31,http://littel.org/zora,,145.175.242.151,"{""location"": ""WF"", ""is_mobile"": false}" 4288,6,399,2016-12-26 10:11:57,http://krajcikwunsch.com/zula_torp,,117.164.100.63,"{""location"": ""GH"", ""is_mobile"": false}" 4289,6,399,2017-04-04 06:45:13,http://sanforddaniel.info/aisha,,225.91.79.75,"{""location"": ""NC"", ""is_mobile"": true}" 4290,6,399,2017-05-11 02:46:01,http://jacobson.co/lyla,,136.209.169.75,"{""location"": ""ST"", ""is_mobile"": false}" 4291,6,399,2017-02-13 11:54:57,http://macejkovic.biz/natalia.strosin,,136.209.169.75,"{""location"": ""SE"", ""is_mobile"": true}" 4292,6,399,2017-02-12 17:47:56,http://reichertcrona.name/kendra_dibbert,,127.86.101.186,"{""location"": ""GD"", ""is_mobile"": false}" 4293,6,399,2017-01-30 04:58:35,http://cormier.biz/elisha,,85.18.83.58,"{""location"": ""PA"", ""is_mobile"": true}" 4294,6,399,2017-03-19 12:15:43,http://handhansen.net/leilani,,145.175.242.151,"{""location"": ""FR"", ""is_mobile"": true}" 4295,6,399,2017-01-18 19:01:54,http://buckridge.io/sherman_quigley,,85.18.83.58,"{""location"": ""UA"", ""is_mobile"": false}" 4296,6,399,2017-03-28 23:12:23,http://brekkehowe.info/allan,,238.189.184.40,"{""location"": ""GW"", ""is_mobile"": true}" 4297,6,400,2017-01-10 01:37:19,http://legrosgorczany.name/grayson,,128.176.35.221,"{""location"": ""GQ"", ""is_mobile"": true}" 4298,6,400,2017-04-07 00:51:57,http://little.biz/ryley,,128.176.35.221,"{""location"": ""VC"", ""is_mobile"": false}" 4299,6,400,2017-05-23 03:49:32,http://wittingtreutel.info/leonie.crona,,233.112.8.139,"{""location"": ""AE"", ""is_mobile"": true}" 4300,6,400,2017-01-08 11:35:59,http://kovacek.io/garrick,,129.126.44.67,"{""location"": ""GQ"", ""is_mobile"": false}" 4301,6,400,2017-05-21 14:16:07,http://marquardtconn.info/ola_stehr,,233.112.8.139,"{""location"": ""VA"", ""is_mobile"": true}" 4302,6,400,2017-03-20 16:51:10,http://erdman.co/maud,,244.59.12.15,"{""location"": ""US"", ""is_mobile"": false}" 4303,6,401,2017-04-28 15:19:29,http://marvinauer.org/lily,,184.231.34.196,"{""location"": ""MD"", ""is_mobile"": false}" 4304,6,401,2016-12-19 16:26:02,http://kuphal.biz/isaias.von,,2.131.189.190,"{""location"": ""SK"", ""is_mobile"": true}" 1153,2,105,2017-04-22 19:55:42,http://reynolds.net/ola_little,0.0000000000,109.134.234.146,"{""location"": ""PE"", ""is_mobile"": true}" 1154,2,106,2017-05-11 23:21:11,http://moen.com/elza,1.0000000000,12.46.167.65,"{""location"": ""PR"", ""is_mobile"": true}" 1155,2,106,2017-01-28 09:27:26,http://pricegulgowski.biz/milan.graham,2.0000000000,206.63.199.140,"{""location"": ""GY"", ""is_mobile"": true}" 1156,2,107,2017-04-26 16:56:04,http://wilkinson.name/thora.smith,2.0000000000,38.34.52.38,"{""location"": ""CA"", ""is_mobile"": false}" 1157,2,107,2017-02-25 20:40:52,http://schaefer.biz/thora,2.0000000000,109.13.72.71,"{""location"": ""AQ"", ""is_mobile"": true}" 1158,2,107,2017-03-23 21:36:24,http://gutkowski.biz/demetrius.wiegand,0.0000000000,62.153.191.47,"{""location"": ""KN"", ""is_mobile"": true}" 1159,2,107,2017-02-17 10:31:05,http://bayer.com/hyman,0.0000000000,71.45.115.132,"{""location"": ""KY"", ""is_mobile"": false}" 1160,2,107,2017-03-15 04:50:54,http://erdman.org/mekhi,0.0000000000,86.250.31.238,"{""location"": ""CY"", ""is_mobile"": true}" 1161,2,107,2017-01-29 17:19:59,http://cormier.info/norwood,1.0000000000,184.220.73.3,"{""location"": ""TM"", ""is_mobile"": false}" 1162,2,107,2017-01-23 16:11:32,http://schuster.name/amara,2.0000000000,96.120.136.47,"{""location"": ""CC"", ""is_mobile"": true}" 1163,2,107,2017-02-09 11:37:36,http://mante.net/maybelle.oconner,0.0000000000,86.250.31.238,"{""location"": ""CY"", ""is_mobile"": true}" 1164,2,107,2017-01-07 15:30:52,http://cormierstroman.io/kelly.johnson,1.0000000000,189.205.251.150,"{""location"": ""PY"", ""is_mobile"": false}" 1165,2,107,2017-05-04 02:14:22,http://marksmccullough.com/alexandrine_bogan,1.0000000000,221.198.247.242,"{""location"": ""BA"", ""is_mobile"": false}" 1166,2,107,2017-01-05 14:30:58,http://wyman.name/ashtyn_olson,2.0000000000,9.238.220.73,"{""location"": ""ID"", ""is_mobile"": true}" 1167,2,108,2017-02-18 21:04:31,http://yundthegmann.biz/titus_schumm,2.0000000000,233.13.239.67,"{""location"": ""LA"", ""is_mobile"": false}" 1168,2,109,2017-01-08 16:41:27,http://flatley.net/quinten,0.0000000000,221.33.219.79,"{""location"": ""FK"", ""is_mobile"": true}" 1169,2,109,2017-01-20 19:23:48,http://pagac.info/cheyanne_heidenreich,0.0000000000,172.202.28.41,"{""location"": ""MG"", ""is_mobile"": false}" 1170,2,109,2017-06-06 22:35:47,http://klein.biz/general.stiedemann,0.0000000000,15.179.85.126,"{""location"": ""CI"", ""is_mobile"": true}" 1171,2,109,2017-06-10 09:40:20,http://balistrerireichert.co/theresia,0.0000000000,207.128.15.13,"{""location"": ""BJ"", ""is_mobile"": true}" 1172,2,109,2016-12-22 05:09:45,http://tremblaysauer.co/crystel,0.0000000000,102.171.12.159,"{""location"": ""GF"", ""is_mobile"": true}" 1173,2,109,2017-05-22 05:53:17,http://batzwalker.io/gerard_walter,0.0000000000,155.158.55.212,"{""location"": ""PT"", ""is_mobile"": false}" 1174,2,109,2017-04-11 02:33:02,http://feeney.info/pablo_rowe,0.0000000000,123.254.92.198,"{""location"": ""GB"", ""is_mobile"": true}" 1175,2,109,2016-12-14 07:29:52,http://funk.com/heaven,0.0000000000,221.168.40.20,"{""location"": ""EC"", ""is_mobile"": false}" 1176,2,109,2017-02-11 05:52:04,http://rau.name/kip,0.0000000000,61.18.136.96,"{""location"": ""MQ"", ""is_mobile"": true}" 1177,2,109,2017-05-29 00:41:00,http://bosco.name/cortez_heaney,0.0000000000,233.147.78.99,"{""location"": ""FJ"", ""is_mobile"": false}" 1178,2,109,2017-06-08 21:34:13,http://okuneva.net/eudora_nolan,0.0000000000,167.124.247.178,"{""location"": ""WS"", ""is_mobile"": false}" 1179,2,110,2017-03-31 23:49:59,http://graham.biz/magnus,1.0000000000,202.131.110.33,"{""location"": ""NZ"", ""is_mobile"": false}" 1180,2,110,2017-04-17 17:38:39,http://jerde.com/lorenzo.leuschke,0.0000000000,119.91.95.155,"{""location"": ""BA"", ""is_mobile"": true}" 1181,2,110,2017-05-05 17:37:22,http://zieme.net/erika,0.0000000000,22.110.67.173,"{""location"": ""SS"", ""is_mobile"": true}" 1182,2,110,2017-04-26 18:01:39,http://yostmayert.co/kathryne.prohaska,0.0000000000,166.108.231.88,"{""location"": ""AQ"", ""is_mobile"": true}" 1183,2,110,2017-03-06 21:34:57,http://legros.biz/christa.stehr,1.0000000000,105.37.107.23,"{""location"": ""FR"", ""is_mobile"": true}" 1184,2,110,2017-03-28 12:21:26,http://mitchell.info/cordell,1.0000000000,22.110.67.173,"{""location"": ""RW"", ""is_mobile"": false}" 1185,2,110,2017-04-30 20:17:17,http://oreilly.com/tracey,2.0000000000,166.108.231.88,"{""location"": ""PT"", ""is_mobile"": false}" 1186,2,111,2017-03-01 05:58:23,http://wintheiser.net/beverly,0.0000000000,120.83.216.245,"{""location"": ""UA"", ""is_mobile"": true}" 1187,2,111,2017-02-01 08:39:30,http://wiza.co/bernie,0.0000000000,118.78.144.196,"{""location"": ""TH"", ""is_mobile"": true}" 1188,2,111,2017-01-11 17:34:10,http://legros.net/blair_crist,0.0000000000,152.148.147.191,"{""location"": ""DZ"", ""is_mobile"": true}" 1189,2,111,2017-04-03 17:20:39,http://quitzon.name/pink_herman,0.0000000000,48.214.173.19,"{""location"": ""SE"", ""is_mobile"": true}" 1190,2,111,2017-03-18 05:55:02,http://haucklarson.io/gabriel,0.0000000000,184.222.170.68,"{""location"": ""IE"", ""is_mobile"": false}" 1191,2,111,2017-04-06 20:57:32,http://jacobs.name/wellington,0.0000000000,8.105.129.12,"{""location"": ""GF"", ""is_mobile"": false}" 1192,2,111,2017-02-07 20:13:21,http://hand.org/deion_breitenberg,0.0000000000,219.135.151.147,"{""location"": ""IE"", ""is_mobile"": true}" 1193,2,111,2017-04-07 16:30:20,http://braunrodriguez.net/iva,0.0000000000,182.60.183.104,"{""location"": ""EC"", ""is_mobile"": false}" 1194,2,112,2017-06-09 04:48:59,http://leuschke.biz/marcus_murazik,0.0000000000,222.218.217.92,"{""location"": ""AE"", ""is_mobile"": true}" 1195,2,112,2017-06-03 04:55:58,http://wildermanroob.co/keenan,0.0000000000,191.118.39.136,"{""location"": ""NA"", ""is_mobile"": false}" 1196,2,112,2017-05-05 11:00:53,http://collins.co/nova_deckow,0.0000000000,180.144.94.38,"{""location"": ""US"", ""is_mobile"": true}" 1197,2,112,2017-05-18 16:07:33,http://feest.info/erick,0.0000000000,191.118.39.136,"{""location"": ""MM"", ""is_mobile"": false}" 1198,2,112,2017-02-27 03:49:40,http://kutch.org/hailey_cormier,0.0000000000,170.135.44.123,"{""location"": ""GU"", ""is_mobile"": true}" 1199,2,112,2017-04-21 00:29:09,http://durgancruickshank.info/michael.donnelly,0.0000000000,222.218.217.92,"{""location"": ""AT"", ""is_mobile"": false}" 1200,2,112,2017-05-12 23:41:54,http://kozey.io/adolf.reinger,0.0000000000,230.240.68.145,"{""location"": ""CK"", ""is_mobile"": false}" 1201,2,112,2017-05-17 02:13:17,http://ortiz.name/carlo,0.0000000000,253.44.209.73,"{""location"": ""SG"", ""is_mobile"": true}" 1202,2,112,2017-05-23 05:54:40,http://friesen.co/novella,0.0000000000,5.76.3.209,"{""location"": ""SD"", ""is_mobile"": true}" 1203,2,112,2017-01-10 16:26:10,http://gleichner.com/aniyah_zemlak,0.0000000000,180.144.94.38,"{""location"": ""BI"", ""is_mobile"": false}" 1204,2,112,2017-04-10 11:29:08,http://effertz.info/korey_powlowski,0.0000000000,106.125.190.162,"{""location"": ""KR"", ""is_mobile"": false}" 4305,6,401,2017-04-28 22:20:21,http://schillersimonis.io/carli,,5.185.83.189,"{""location"": ""SS"", ""is_mobile"": true}" 4306,6,401,2017-04-29 13:02:43,http://hudson.io/adah,,113.133.96.131,"{""location"": ""CR"", ""is_mobile"": false}" 4307,6,401,2017-01-21 16:59:27,http://kemmerhansen.net/jillian.collier,,108.219.66.203,"{""location"": ""PK"", ""is_mobile"": false}" 4308,6,401,2017-05-17 04:42:40,http://grant.com/beverly,,121.221.29.48,"{""location"": ""SR"", ""is_mobile"": true}" 4309,6,401,2017-03-30 15:53:06,http://nader.com/stefanie_hegmann,,113.133.96.131,"{""location"": ""SR"", ""is_mobile"": true}" 4310,6,401,2017-02-11 11:50:18,http://bechtelar.io/martine.fadel,,139.233.234.201,"{""location"": ""AO"", ""is_mobile"": false}" 4311,6,401,2017-04-06 01:22:28,http://mcdermottkoepp.com/lysanne.kohler,,78.170.75.215,"{""location"": ""ES"", ""is_mobile"": true}" 4312,6,402,2016-12-23 16:28:28,http://wolfgleason.org/darryl,,9.227.62.196,"{""location"": ""RO"", ""is_mobile"": false}" 4313,6,402,2017-05-20 06:40:53,http://howellhyatt.io/diego,,95.61.151.30,"{""location"": ""BI"", ""is_mobile"": false}" 4314,6,402,2017-01-09 21:02:29,http://hackett.io/taryn.trantow,,27.11.66.24,"{""location"": ""VI"", ""is_mobile"": true}" 4315,6,402,2017-02-21 01:20:49,http://rohan.co/cristal.bradtke,,27.52.87.199,"{""location"": ""SO"", ""is_mobile"": true}" 4316,6,402,2017-06-06 05:24:26,http://herman.net/helga,,185.89.175.138,"{""location"": ""WS"", ""is_mobile"": true}" 4317,6,402,2017-02-07 06:25:33,http://gusikowskiklein.biz/loy,,9.227.62.196,"{""location"": ""LC"", ""is_mobile"": true}" 4318,6,402,2017-02-18 17:07:55,http://starkcollier.com/emmie.lockman,,13.5.15.158,"{""location"": ""BQ"", ""is_mobile"": true}" 4319,6,402,2017-01-22 20:45:10,http://purdy.io/hallie.weimann,,117.47.16.6,"{""location"": ""RO"", ""is_mobile"": false}" 4320,6,402,2017-04-18 18:49:46,http://nikolausgorczany.com/virgil,,136.77.194.227,"{""location"": ""HU"", ""is_mobile"": false}" 1205,2,112,2017-05-03 11:45:24,http://huels.name/hunter,0.0000000000,41.251.202.168,"{""location"": ""TR"", ""is_mobile"": false}" 1206,2,112,2017-02-28 22:25:55,http://rowe.net/nya,0.0000000000,180.144.94.38,"{""location"": ""PM"", ""is_mobile"": false}" 1207,2,112,2017-01-11 18:23:28,http://cruickshank.io/katherine,0.0000000000,59.36.133.199,"{""location"": ""NL"", ""is_mobile"": false}" 1208,2,112,2017-04-14 01:16:27,http://runte.org/sincere.graham,0.0000000000,81.214.228.105,"{""location"": ""VN"", ""is_mobile"": false}" 1209,2,112,2017-01-30 11:20:06,http://greenfelderosinski.net/willy,0.0000000000,137.90.241.214,"{""location"": ""IE"", ""is_mobile"": true}" 1210,2,112,2017-04-28 08:35:15,http://bergstrom.org/terrill_bradtke,0.0000000000,105.246.104.218,"{""location"": ""MC"", ""is_mobile"": false}" 1211,2,113,2017-05-17 09:59:57,http://pfannerstill.info/kian.runolfon,0.0000000000,216.65.188.184,"{""location"": ""TF"", ""is_mobile"": false}" 1212,2,113,2017-01-16 22:54:24,http://hegmannnienow.info/kailyn,0.0000000000,91.93.115.91,"{""location"": ""GR"", ""is_mobile"": true}" 1213,2,114,2017-02-25 17:11:30,http://ricequitzon.co/alicia.considine,1.0000000000,40.122.238.149,"{""location"": ""PT"", ""is_mobile"": false}" 1214,2,114,2017-03-04 21:16:33,http://osinski.co/lazaro_beier,2.0000000000,122.22.250.105,"{""location"": ""CI"", ""is_mobile"": true}" 1215,2,114,2017-03-03 14:25:49,http://harveyflatley.co/gunnar.durgan,1.0000000000,4.71.149.103,"{""location"": ""BI"", ""is_mobile"": true}" 1216,2,114,2017-06-13 22:34:31,http://bergnaumdooley.name/edwina.marvin,0.0000000000,246.64.144.97,"{""location"": ""BJ"", ""is_mobile"": false}" 1217,2,114,2017-03-22 16:04:57,http://wisoky.io/ali_mueller,0.0000000000,145.195.175.116,"{""location"": ""MT"", ""is_mobile"": false}" 1218,2,114,2017-04-10 02:10:00,http://langworth.net/nelda.kling,1.0000000000,148.84.96.56,"{""location"": ""LB"", ""is_mobile"": true}" 1219,2,114,2017-02-27 22:36:40,http://schoencarroll.net/dortha_barton,2.0000000000,18.88.187.77,"{""location"": ""RS"", ""is_mobile"": true}" 1220,2,114,2017-06-02 00:31:45,http://wiegand.net/kennith_schaefer,2.0000000000,128.43.233.144,"{""location"": ""SN"", ""is_mobile"": false}" 1221,2,114,2017-05-08 10:31:58,http://reillygottlieb.net/vito,1.0000000000,42.120.4.95,"{""location"": ""BJ"", ""is_mobile"": true}" 1222,2,114,2016-12-24 07:50:40,http://stamm.name/mellie_stiedemann,0.0000000000,36.68.69.175,"{""location"": ""ST"", ""is_mobile"": true}" 1223,2,114,2017-01-14 15:06:47,http://grady.io/lempi_fadel,2.0000000000,148.84.96.56,"{""location"": ""KW"", ""is_mobile"": true}" 1224,2,115,2017-05-07 12:40:39,http://effertzschultz.io/hobart_morar,0.0000000000,53.95.80.197,"{""location"": ""DK"", ""is_mobile"": true}" 1225,2,115,2017-04-06 05:12:13,http://hintz.net/krystal_jast,1.0000000000,134.208.69.114,"{""location"": ""ZA"", ""is_mobile"": false}" 1226,2,115,2017-03-17 04:44:11,http://sauerokuneva.info/katelin,1.0000000000,40.204.74.114,"{""location"": ""CW"", ""is_mobile"": false}" 1227,2,115,2017-05-29 11:28:00,http://johnston.org/oma.carroll,1.0000000000,235.142.203.141,"{""location"": ""KN"", ""is_mobile"": false}" 1228,2,115,2017-05-13 00:49:51,http://dare.net/clay.kozey,1.0000000000,169.60.138.15,"{""location"": ""SK"", ""is_mobile"": false}" 1229,2,115,2017-04-08 21:51:52,http://halvorsonrowe.info/chanelle,0.0000000000,237.150.99.156,"{""location"": ""IM"", ""is_mobile"": false}" 1230,2,116,2016-12-31 21:14:38,http://renner.net/destiney,0.0000000000,195.135.65.212,"{""location"": ""EH"", ""is_mobile"": false}" 1231,2,116,2017-03-09 22:24:29,http://millshagenes.info/shane,0.0000000000,86.94.88.134,"{""location"": ""CD"", ""is_mobile"": false}" 1232,2,116,2017-01-26 14:37:55,http://welch.io/donato_weimann,0.0000000000,67.42.118.8,"{""location"": ""OM"", ""is_mobile"": true}" 1233,2,116,2017-05-02 22:35:44,http://runte.biz/leilani,0.0000000000,32.166.3.201,"{""location"": ""FJ"", ""is_mobile"": false}" 1234,2,116,2017-03-10 22:28:39,http://eichmann.name/charlie.goyette,0.0000000000,64.103.2.236,"{""location"": ""MT"", ""is_mobile"": false}" 1235,2,117,2017-06-06 07:20:28,http://pouros.info/ambrose_mckenzie,1.0000000000,47.132.145.213,"{""location"": ""VC"", ""is_mobile"": true}" 1236,2,117,2017-04-11 15:20:26,http://schowalter.info/sigurd_armstrong,0.0000000000,146.243.38.198,"{""location"": ""MS"", ""is_mobile"": false}" 1237,2,117,2017-04-15 03:28:45,http://macgyverdaniel.io/kacie,0.0000000000,41.176.120.16,"{""location"": ""GF"", ""is_mobile"": true}" 1238,2,118,2017-02-22 08:43:29,http://thompson.co/adeline_bogisich,0.0000000000,152.254.228.73,"{""location"": ""RW"", ""is_mobile"": true}" 1239,2,118,2017-06-01 02:08:23,http://becker.com/lilyan_jacobs,3.0000000000,169.115.176.219,"{""location"": ""PA"", ""is_mobile"": false}" 1240,2,118,2017-05-17 05:34:34,http://simonis.co/carlos.reynolds,0.0000000000,179.32.231.69,"{""location"": ""MR"", ""is_mobile"": true}" 1241,2,118,2017-04-03 12:24:16,http://gibson.biz/angus.deckow,3.0000000000,78.35.191.111,"{""location"": ""TF"", ""is_mobile"": true}" 1242,2,118,2017-02-24 13:08:24,http://krajcikcronin.info/eleanora,2.0000000000,114.240.223.133,"{""location"": ""MO"", ""is_mobile"": true}" 1243,2,118,2017-03-04 20:51:17,http://graham.net/alexandrea.quigley,3.0000000000,152.254.228.73,"{""location"": ""YE"", ""is_mobile"": true}" 1244,2,118,2017-03-24 07:11:08,http://bogisich.com/nikita.ko,2.0000000000,25.48.154.83,"{""location"": ""ID"", ""is_mobile"": true}" 1245,2,118,2017-02-05 09:44:56,http://gerholdaufderhar.com/mohammed.lueilwitz,2.0000000000,136.207.184.21,"{""location"": ""JO"", ""is_mobile"": false}" 1246,2,118,2017-03-27 18:42:12,http://little.co/ashleigh_kilback,3.0000000000,125.12.218.11,"{""location"": ""KW"", ""is_mobile"": false}" 1247,2,118,2017-05-01 23:27:59,http://marvin.net/della,2.0000000000,228.81.124.158,"{""location"": ""MD"", ""is_mobile"": false}" 1248,2,118,2016-12-26 22:24:03,http://kovacekbrown.com/durward,2.0000000000,136.51.55.250,"{""location"": ""FK"", ""is_mobile"": true}" 1249,2,118,2017-03-12 04:45:30,http://hagenes.co/hermann_wisozk,1.0000000000,75.216.205.204,"{""location"": ""JE"", ""is_mobile"": true}" 1250,2,118,2017-02-08 07:26:02,http://mertz.name/dariana,1.0000000000,152.254.228.73,"{""location"": ""CC"", ""is_mobile"": true}" 1251,2,118,2017-02-11 10:17:35,http://gusikowskigulgowski.name/bo,1.0000000000,246.5.207.242,"{""location"": ""UY"", ""is_mobile"": true}" 1252,2,119,2017-05-19 03:29:40,http://terrybeatty.com/madge.hagenes,,156.6.202.29,"{""location"": ""SA"", ""is_mobile"": true}" 1253,2,119,2017-02-28 06:13:31,http://kunde.com/leie,,5.219.187.130,"{""location"": ""SX"", ""is_mobile"": true}" 1254,2,120,2017-04-02 23:19:31,http://schroeder.org/aidan.bergstrom,,239.85.56.16,"{""location"": ""NO"", ""is_mobile"": false}" 1255,2,120,2017-05-29 18:16:26,http://wunschwilderman.name/jazlyn,,168.245.53.4,"{""location"": ""NG"", ""is_mobile"": false}" 1256,2,120,2017-03-09 16:02:48,http://kaulkecrist.net/maximillia.von,,176.242.68.251,"{""location"": ""MD"", ""is_mobile"": true}" 1257,2,120,2017-04-10 18:44:57,http://carrolllittle.co/yolanda_nicolas,,206.141.126.157,"{""location"": ""IL"", ""is_mobile"": true}" 1258,2,120,2017-01-18 11:01:17,http://homenick.net/hilario,,128.92.114.191,"{""location"": ""LV"", ""is_mobile"": true}" 1259,2,120,2017-02-07 09:02:06,http://ebert.io/cali,,90.140.85.175,"{""location"": ""LS"", ""is_mobile"": false}" 1260,2,120,2017-06-08 15:13:43,http://goyette.name/rhett_heel,,41.184.169.213,"{""location"": ""RO"", ""is_mobile"": false}" 1261,2,120,2017-01-23 09:49:56,http://grimes.name/verna.jerde,,130.17.113.22,"{""location"": ""BF"", ""is_mobile"": false}" 1262,2,121,2017-04-20 00:27:35,http://denesikebert.biz/helmer,,236.22.122.238,"{""location"": ""MV"", ""is_mobile"": true}" 1263,2,122,2017-04-03 08:35:40,http://ritchie.info/laurel.crooks,,41.237.87.138,"{""location"": ""FI"", ""is_mobile"": true}" 1264,2,122,2017-04-20 21:23:51,http://mitchell.com/stephania,,93.223.101.131,"{""location"": ""LR"", ""is_mobile"": true}" 1265,2,122,2016-12-14 07:21:39,http://mante.info/alexandre_stokes,,41.237.87.138,"{""location"": ""GL"", ""is_mobile"": true}" 1266,2,122,2017-01-31 22:15:33,http://fay.name/lazaro,,113.11.182.230,"{""location"": ""OM"", ""is_mobile"": false}" 1267,2,122,2017-05-28 16:10:48,http://heathcote.io/mohammed.predovic,,4.196.238.40,"{""location"": ""FO"", ""is_mobile"": true}" 1268,2,122,2017-03-27 12:32:56,http://bartoletti.org/hoyt.hills,,78.253.211.228,"{""location"": ""CY"", ""is_mobile"": true}" 1269,2,122,2017-04-14 09:50:25,http://batz.io/andre.christiansen,,207.219.102.3,"{""location"": ""BE"", ""is_mobile"": false}" 1270,2,122,2017-05-03 11:57:24,http://dicki.net/carmelo_mann,,97.27.152.3,"{""location"": ""MX"", ""is_mobile"": false}" 1271,2,122,2017-06-08 03:04:00,http://johnsdach.biz/murphy_schowalter,,93.223.101.131,"{""location"": ""LS"", ""is_mobile"": true}" 1272,2,122,2017-04-29 07:02:20,http://goodwin.net/guadalupe,,207.219.102.3,"{""location"": ""PA"", ""is_mobile"": true}" 1273,2,122,2017-03-29 08:50:21,http://moorehoppe.name/raymundo,,78.253.211.228,"{""location"": ""AO"", ""is_mobile"": false}" 1274,2,122,2017-04-03 16:04:57,http://johnsonorn.net/hilbert,,54.102.28.51,"{""location"": ""MN"", ""is_mobile"": false}" 1275,2,122,2017-03-25 08:50:51,http://heathcote.org/emmett_thiel,,180.92.34.211,"{""location"": ""UY"", ""is_mobile"": true}" 1276,2,122,2017-03-13 22:55:07,http://bruen.com/joannie,,4.196.238.40,"{""location"": ""BT"", ""is_mobile"": true}" 1277,2,122,2017-04-02 21:48:47,http://hirthe.biz/joany,,9.153.147.254,"{""location"": ""NF"", ""is_mobile"": true}" 1278,2,122,2017-03-02 11:12:30,http://brakusberge.org/prince_beahan,,41.237.87.138,"{""location"": ""US"", ""is_mobile"": true}" 1279,2,123,2017-04-09 01:08:59,http://mayerdoyle.org/carole,,11.31.30.4,"{""location"": ""SO"", ""is_mobile"": false}" 1280,2,123,2017-02-08 12:16:02,http://jacobi.io/kelvin.botsford,,38.93.174.201,"{""location"": ""RU"", ""is_mobile"": false}" 1281,2,123,2017-05-04 21:39:43,http://schowalter.co/taylor.moriette,,115.200.63.80,"{""location"": ""SD"", ""is_mobile"": true}" 1282,2,123,2017-01-15 03:28:15,http://dibbert.io/walter,,151.71.167.149,"{""location"": ""JM"", ""is_mobile"": false}" 1283,2,123,2017-02-22 06:38:23,http://brown.name/jace,,194.232.136.118,"{""location"": ""TN"", ""is_mobile"": false}" 1284,2,123,2017-01-26 18:42:38,http://schulist.info/armando,,52.226.14.230,"{""location"": ""SY"", ""is_mobile"": false}" 1285,2,123,2017-02-12 06:10:25,http://millshilll.name/katheryn,,116.153.115.35,"{""location"": ""BG"", ""is_mobile"": true}" 1286,2,123,2017-04-19 01:02:19,http://west.info/brice.zulauf,,184.177.238.72,"{""location"": ""UM"", ""is_mobile"": true}" 1287,2,123,2017-02-23 04:56:14,http://bosco.org/destiney,,151.71.167.149,"{""location"": ""GM"", ""is_mobile"": true}" 1288,2,123,2017-05-08 01:28:37,http://boyle.org/garnett_muller,,209.165.149.48,"{""location"": ""AM"", ""is_mobile"": false}" 1289,2,123,2017-05-15 17:59:53,http://cartwrighttorphy.org/davin_reilly,,63.38.22.15,"{""location"": ""NU"", ""is_mobile"": false}" 1290,2,123,2017-03-23 12:00:10,http://weimann.name/robert.kertzmann,,4.46.228.211,"{""location"": ""MZ"", ""is_mobile"": true}" 1291,2,123,2017-01-07 01:45:38,http://kinghudson.org/owen,,105.75.128.238,"{""location"": ""UZ"", ""is_mobile"": false}" 1293,2,123,2017-04-15 07:18:36,http://armstrong.info/susanna.dare,,195.245.131.144,"{""location"": ""AE"", ""is_mobile"": true}" 1294,2,123,2017-02-16 06:37:22,http://bechtelar.name/jabari_emmerich,,209.165.149.48,"{""location"": ""MT"", ""is_mobile"": true}" 1295,2,123,2017-03-14 06:11:46,http://sipes.io/savanah,,193.132.233.14,"{""location"": ""MY"", ""is_mobile"": false}" 1296,2,123,2017-06-06 19:13:08,http://thiel.name/felipe.simonis,,225.182.184.149,"{""location"": ""SA"", ""is_mobile"": false}" 1297,2,123,2017-05-30 10:22:04,http://marquardt.info/mikel,,209.165.149.48,"{""location"": ""CW"", ""is_mobile"": true}" 1298,2,124,2017-03-11 20:33:03,http://champlin.net/elvera_damore,,166.165.59.167,"{""location"": ""TM"", ""is_mobile"": false}" 1299,2,124,2017-04-15 15:13:58,http://uptonhilll.org/bobbie,,184.3.202.217,"{""location"": ""US"", ""is_mobile"": true}" 1300,2,124,2017-01-15 20:11:21,http://jacobi.info/arlene_ernser,,184.3.202.217,"{""location"": ""HR"", ""is_mobile"": true}" 1301,2,124,2017-01-05 00:48:34,http://rogahnnader.biz/dawson.waelchi,,252.98.229.152,"{""location"": ""ID"", ""is_mobile"": true}" 1302,2,124,2017-02-26 04:34:31,http://legros.org/llewellyn,,227.74.211.72,"{""location"": ""UZ"", ""is_mobile"": false}" 1303,2,124,2017-01-09 23:52:52,http://deckow.io/lucio,,203.98.228.115,"{""location"": ""PM"", ""is_mobile"": true}" 1304,2,124,2017-03-16 07:34:20,http://marquardt.net/rozella.sauer,,192.185.69.234,"{""location"": ""TL"", ""is_mobile"": true}" 1305,2,124,2017-02-14 14:57:49,http://altenwerth.com/edmond,,66.49.38.129,"{""location"": ""PL"", ""is_mobile"": true}" 1306,2,124,2017-04-12 23:58:36,http://batz.net/ena_oconnell,,197.40.226.228,"{""location"": ""DJ"", ""is_mobile"": false}" 1307,2,124,2017-03-25 05:44:09,http://bahringer.info/rylee_crona,,88.239.142.245,"{""location"": ""MF"", ""is_mobile"": true}" 1308,2,124,2017-02-26 12:18:31,http://schinnerconsidine.org/quinton.gulgowski,,53.118.162.128,"{""location"": ""AI"", ""is_mobile"": false}" 1309,2,124,2017-05-17 23:35:16,http://leannon.net/reese,,239.33.243.139,"{""location"": ""GP"", ""is_mobile"": false}" 1310,2,124,2017-01-16 06:18:52,http://gutmannstreich.io/oceane,,53.118.162.128,"{""location"": ""AD"", ""is_mobile"": false}" 1311,2,124,2017-01-09 10:10:24,http://rutherfordblock.info/myrtis,,197.40.226.228,"{""location"": ""SH"", ""is_mobile"": false}" 1312,2,124,2017-02-24 18:11:14,http://crist.io/miller,,25.98.49.80,"{""location"": ""CN"", ""is_mobile"": false}" 1313,2,124,2017-02-14 15:11:21,http://westweimann.info/leonardo.leffler,,162.163.247.122,"{""location"": ""FO"", ""is_mobile"": true}" 1314,2,124,2016-12-16 10:56:25,http://upton.name/retta,,162.163.247.122,"{""location"": ""IO"", ""is_mobile"": true}" 1315,2,124,2016-12-28 08:42:29,http://crist.co/zita,,100.210.219.233,"{""location"": ""EG"", ""is_mobile"": false}" 1316,2,124,2017-05-16 14:34:13,http://rogahnlindgren.biz/estevan.koch,,227.74.211.72,"{""location"": ""SI"", ""is_mobile"": true}" 1317,2,125,2017-02-16 05:35:29,http://mraz.com/tyree,,58.162.43.14,"{""location"": ""MS"", ""is_mobile"": true}" 1318,2,125,2017-06-05 00:33:03,http://hoegerschinner.info/ava_gorczany,,10.11.165.225,"{""location"": ""TG"", ""is_mobile"": true}" 1319,2,125,2017-01-15 14:34:26,http://smitham.name/odie,,102.9.170.186,"{""location"": ""AE"", ""is_mobile"": false}" 1320,2,125,2017-05-04 21:29:06,http://cummerata.net/derek,,102.9.170.186,"{""location"": ""UZ"", ""is_mobile"": false}" 1321,2,125,2017-05-15 15:04:43,http://conroy.net/vladimir,,13.204.94.211,"{""location"": ""PT"", ""is_mobile"": false}" 1322,2,125,2017-01-15 18:07:40,http://cummings.net/elmira.schmeler,,139.20.184.101,"{""location"": ""DJ"", ""is_mobile"": false}" 1323,2,125,2017-01-01 08:28:09,http://walsh.io/zaria,,159.67.71.59,"{""location"": ""CY"", ""is_mobile"": false}" 1324,2,125,2017-03-28 20:20:45,http://champlinwalker.net/erling,,29.228.229.96,"{""location"": ""AF"", ""is_mobile"": false}" 1325,2,125,2016-12-14 04:05:39,http://koch.co/kristopher,,160.253.61.116,"{""location"": ""MG"", ""is_mobile"": true}" 1326,2,125,2017-01-06 09:41:01,http://colliergoodwin.net/mya,,251.31.125.33,"{""location"": ""ET"", ""is_mobile"": false}" 1327,2,125,2016-12-25 03:58:36,http://rodriguezskiles.info/valerie,,10.11.165.225,"{""location"": ""PK"", ""is_mobile"": true}" 1328,2,125,2017-03-29 14:46:14,http://rau.io/lowell,,34.101.113.246,"{""location"": ""LI"", ""is_mobile"": false}" 1329,2,125,2017-03-13 15:43:17,http://hand.co/casandra,,18.115.126.112,"{""location"": ""TH"", ""is_mobile"": false}" 1330,2,125,2017-01-08 04:39:11,http://ondrickahuels.name/bert_muller,,160.253.61.116,"{""location"": ""LB"", ""is_mobile"": true}" 1331,2,125,2017-05-03 22:32:58,http://marvinprice.io/jayme.beer,,189.197.201.142,"{""location"": ""SX"", ""is_mobile"": false}" 1332,2,125,2017-03-26 08:42:09,http://hills.info/earl_larson,,31.14.152.202,"{""location"": ""BE"", ""is_mobile"": false}" 1333,2,125,2017-05-22 15:53:12,http://haag.com/amya_conroy,,212.9.109.59,"{""location"": ""QA"", ""is_mobile"": false}" 1334,2,125,2017-05-21 02:13:51,http://steuber.co/jose,,12.52.177.103,"{""location"": ""TF"", ""is_mobile"": true}" 1335,2,125,2017-04-14 13:39:34,http://hamill.biz/cydney.kutch,,12.52.177.103,"{""location"": ""SC"", ""is_mobile"": false}" 1336,2,125,2016-12-30 14:46:51,http://tillman.name/elva,,243.67.151.103,"{""location"": ""PA"", ""is_mobile"": false}" ================================================ FILE: src/test/regress/data/companies.csv ================================================ 8,Luna Rogahn,https://robohash.org/idfugavero.png?size=300x300&set=set1,2017-06-13 16:42:34.405636,2017-06-13 16:42:34.405636 1,Dulce Corkery,https://robohash.org/cumqueautquia.png?size=300x300&set=set1,2017-06-13 16:41:52.460952,2017-06-13 16:41:52.460952 5,Lindsey Durgan,https://robohash.org/sintexfacilis.png?size=300x300&set=set1,2017-06-13 16:42:15.946372,2017-06-13 16:42:15.946372 4,Ms. Morton Stehr,https://robohash.org/optionamquis.png?size=300x300&set=set1,2017-06-13 16:42:09.712621,2017-06-13 16:42:09.712621 7,Sienna Collier,https://robohash.org/autemquiamolestiae.png?size=300x300&set=set1,2017-06-13 16:42:28.41381,2017-06-13 16:42:28.41381 3,Mrs. Nels Jacobi,https://robohash.org/quiapraesentiumeos.png?size=300x300&set=set1,2017-06-13 16:42:03.620678,2017-06-13 16:42:03.620678 6,Maximo Hegmann,https://robohash.org/aliquamreprehenderitdolor.png?size=300x300&set=set1,2017-06-13 16:42:22.35326,2017-06-13 16:42:22.35326 2,Jamir Larkin,https://robohash.org/teneturiureducimus.png?size=300x300&set=set1,2017-06-13 16:41:57.057374,2017-06-13 16:41:57.057374 ================================================ FILE: src/test/regress/data/contestants.1.csv ================================================ a,1990-01-10,2090,97.1,XA ,{a} b,1990-11-01,2203,98.1,XA ,"{a,b}" c,1988-11-01,2907,99.4,XB ,"{w,y}" d,1985-05-05,2314,98.3,XB ,{} e,1995-05-05,2236,98.2,XC ,{a} ================================================ FILE: src/test/regress/data/contestants.2.csv ================================================ f,1983-04-02,3090,99.6,XD ,"{a,b,c,y}" g,1991-12-13,1803,85.1,XD ,"{a,c}" h,1987-10-26,2112,95.4,XD ,"{w,a}" ================================================ FILE: src/test/regress/data/customer-1-10.data ================================================ 1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e 2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref 3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov 4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou 5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor 6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious 7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp 8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide 9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl 10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur ================================================ FILE: src/test/regress/data/customer-1-15.data ================================================ 1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e 2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref 3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov 4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou 5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor 6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious 7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp 8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide 9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl 10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf ================================================ FILE: src/test/regress/data/customer-1-20.data ================================================ 1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e 2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref 3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov 4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou 5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor 6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious 7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp 8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide 9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl 10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf 16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after 17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint 18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t 19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in 20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake ================================================ FILE: src/test/regress/data/customer-1-30.data ================================================ 1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e 2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref 3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov 4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou 5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor 6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious 7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp 8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide 9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl 10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf 16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after 17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint 18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t 19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in 20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake 21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u 22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. 23|Customer#000000023|OdY W13N7Be3OC5MpgfmcYss0Wn6TKT|3|13-312-472-8245|3332.02|HOUSEHOLD|deposits. special deposits cajole slyly. fluffily special deposits about the furiously 24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily 25|Customer#000000025|Hp8GyFQgGHFYSilH5tBfe|12|22-603-468-3533|7133.70|FURNITURE|y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final 26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us 27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun 28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac 29|Customer#000000029|sJ5adtfyAkCK63df2,vF25zyQMVYE34uh|0|10-773-203-7342|7618.27|FURNITURE|its after the carefully final platelets x-ray against 30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account ================================================ FILE: src/test/regress/data/customer-11-20.data ================================================ 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf 16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after 17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint 18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t 19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in 20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake ================================================ FILE: src/test/regress/data/customer-21-30.data ================================================ 21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u 22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. 23|Customer#000000023|OdY W13N7Be3OC5MpgfmcYss0Wn6TKT|3|13-312-472-8245|3332.02|HOUSEHOLD|deposits. special deposits cajole slyly. fluffily special deposits about the furiously 24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily 25|Customer#000000025|Hp8GyFQgGHFYSilH5tBfe|12|22-603-468-3533|7133.70|FURNITURE|y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final 26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us 27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun 28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac 29|Customer#000000029|sJ5adtfyAkCK63df2,vF25zyQMVYE34uh|0|10-773-203-7342|7618.27|FURNITURE|its after the carefully final platelets x-ray against 30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account ================================================ FILE: src/test/regress/data/customer-subset-11-20.data ================================================ 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after 17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint 18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t 20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake ================================================ FILE: src/test/regress/data/customer-subset-21-30.data ================================================ 21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u 22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. 24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily 26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us 27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun 28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac 30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account ================================================ FILE: src/test/regress/data/customer.1.data ================================================ 1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e 2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref 3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov 4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou 5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor 6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious 7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp 8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide 9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl 10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur 11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. 12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along 13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely 14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus 15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf 16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after 17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint 18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t 19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in 20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake 21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u 22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. 23|Customer#000000023|OdY W13N7Be3OC5MpgfmcYss0Wn6TKT|3|13-312-472-8245|3332.02|HOUSEHOLD|deposits. special deposits cajole slyly. fluffily special deposits about the furiously 24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily 25|Customer#000000025|Hp8GyFQgGHFYSilH5tBfe|12|22-603-468-3533|7133.70|FURNITURE|y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final 26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us 27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun 28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac 29|Customer#000000029|sJ5adtfyAkCK63df2,vF25zyQMVYE34uh|0|10-773-203-7342|7618.27|FURNITURE|its after the carefully final platelets x-ray against 30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account 31|Customer#000000031|LUACbO0viaAv6eXOAebryDB xjVst|23|33-197-837-7094|5236.89|HOUSEHOLD|s use among the blithely pending depo 32|Customer#000000032|jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J|15|25-430-914-2194|3471.53|BUILDING|cial ideas. final, furious requests across the e 33|Customer#000000033|qFSlMuLucBmx9xnn5ib2csWUweg D|17|27-375-391-1280|-78.56|AUTOMOBILE|s. slyly regular accounts are furiously. carefully pending requests 34|Customer#000000034|Q6G9wZ6dnczmtOx509xgE,M2KV|15|25-344-968-5422|8589.70|HOUSEHOLD|nder against the even, pending accounts. even 35|Customer#000000035|TEjWGE4nBzJL2|17|27-566-888-7431|1228.24|HOUSEHOLD|requests. special, express requests nag slyly furiousl 36|Customer#000000036|3TvCzjuPzpJ0,DdJ8kW5U|21|31-704-669-5769|4987.27|BUILDING|haggle. enticing, quiet platelets grow quickly bold sheaves. carefully regular acc 37|Customer#000000037|7EV4Pwh,3SboctTWt|8|18-385-235-7162|-917.75|FURNITURE|ilent packages are carefully among the deposits. furiousl 38|Customer#000000038|a5Ee5e9568R8RLP 2ap7|12|22-306-880-7212|6345.11|HOUSEHOLD|lar excuses. closely even asymptotes cajole blithely excuses. carefully silent pinto beans sleep carefully fin 39|Customer#000000039|nnbRg,Pvy33dfkorYE FdeZ60|2|12-387-467-6509|6264.31|AUTOMOBILE|tions. slyly silent excuses slee 40|Customer#000000040|gOnGWAyhSV1ofv|3|13-652-915-8939|1335.30|BUILDING|rges impress after the slyly ironic courts. foxes are. blithely 41|Customer#000000041|IM9mzmyoxeBmvNw8lA7G3Ydska2nkZF|10|20-917-711-4011|270.95|HOUSEHOLD|ly regular accounts hang bold, silent packages. unusual foxes haggle slyly above the special, final depo 42|Customer#000000042|ziSrvyyBke|5|15-416-330-4175|8727.01|BUILDING|ssly according to the pinto beans: carefully special requests across the even, pending accounts wake special 43|Customer#000000043|ouSbjHk8lh5fKX3zGso3ZSIj9Aa3PoaFd|19|29-316-665-2897|9904.28|MACHINERY|ial requests: carefully pending foxes detect quickly. carefully final courts cajole quickly. carefully 44|Customer#000000044|Oi,dOSPwDu4jo4x,,P85E0dmhZGvNtBwi|16|26-190-260-5375|7315.94|AUTOMOBILE|r requests around the unusual, bold a 45|Customer#000000045|4v3OcpFgoOmMG,CbnF,4mdC|9|19-715-298-9917|9983.38|AUTOMOBILE|nto beans haggle slyly alongside of t 46|Customer#000000046|eaTXWWm10L9|6|16-357-681-2007|5744.59|AUTOMOBILE|ctions. accounts sleep furiously even requests. regular, regular accounts cajole blithely around the final pa 47|Customer#000000047|b0UgocSqEW5 gdVbhNT|2|12-427-271-9466|274.58|BUILDING|ions. express, ironic instructions sleep furiously ironic ideas. furi 48|Customer#000000048|0UU iPhBupFvemNB|0|10-508-348-5882|3792.50|BUILDING|re fluffily pending foxes. pending, bold platelets sleep slyly. even platelets cajo 49|Customer#000000049|cNgAeX7Fqrdf7HQN9EwjUa4nxT,68L FKAxzl|10|20-908-631-4424|4573.94|FURNITURE|nusual foxes! fluffily pending packages maintain to the regular 50|Customer#000000050|9SzDYlkzxByyJ1QeTI o|6|16-658-112-3221|4266.13|MACHINERY|ts. furiously ironic accounts cajole furiously slyly ironic dinos. 51|Customer#000000051|uR,wEaiTvo4|12|22-344-885-4251|855.87|FURNITURE|eposits. furiously regular requests integrate carefully packages. furious 52|Customer#000000052|7 QOqGqqSy9jfV51BC71jcHJSD0|11|21-186-284-5998|5630.28|HOUSEHOLD|ic platelets use evenly even accounts. stealthy theodolites cajole furiou 53|Customer#000000053|HnaxHzTfFTZs8MuCpJyTbZ47Cm4wFOOgib|15|25-168-852-5363|4113.64|HOUSEHOLD|ar accounts are. even foxes are blithely. fluffily pending deposits boost 54|Customer#000000054|,k4vf 5vECGWFy,hosTE,|4|14-776-370-4745|868.90|AUTOMOBILE|sual, silent accounts. furiously express accounts cajole special deposits. final, final accounts use furi 55|Customer#000000055|zIRBR4KNEl HzaiV3a i9n6elrxzDEh8r8pDom|10|20-180-440-8525|4572.11|MACHINERY|ully unusual packages wake bravely bold packages. unusual requests boost deposits! blithely ironic packages ab 56|Customer#000000056|BJYZYJQk4yD5B|10|20-895-685-6920|6530.86|FURNITURE|. notornis wake carefully. carefully fluffy requests are furiously even accounts. slyly expre 57|Customer#000000057|97XYbsuOPRXPWU|21|31-835-306-1650|4151.93|AUTOMOBILE|ove the carefully special packages. even, unusual deposits sleep slyly pend 58|Customer#000000058|g9ap7Dk1Sv9fcXEWjpMYpBZIRUohi T|13|23-244-493-2508|6478.46|HOUSEHOLD|ideas. ironic ideas affix furiously express, final instructions. regular excuses use quickly e 59|Customer#000000059|zLOCP0wh92OtBihgspOGl4|1|11-355-584-3112|3458.60|MACHINERY|ously final packages haggle blithely after the express deposits. furiou 60|Customer#000000060|FyodhjwMChsZmUz7Jz0H|12|22-480-575-5866|2741.87|MACHINERY|latelets. blithely unusual courts boost furiously about the packages. blithely final instruct 61|Customer#000000061|9kndve4EAJxhg3veF BfXr7AqOsT39o gtqjaYE|17|27-626-559-8599|1536.24|FURNITURE|egular packages shall have to impress along the 62|Customer#000000062|upJK2Dnw13,|7|17-361-978-7059|595.61|MACHINERY|kly special dolphins. pinto beans are slyly. quickly regular accounts are furiously a 63|Customer#000000063|IXRSpVWWZraKII|21|31-952-552-9584|9331.13|AUTOMOBILE|ithely even accounts detect slyly above the fluffily ir 64|Customer#000000064|MbCeGY20kaKK3oalJD,OT|3|13-558-731-7204|-646.64|BUILDING|structions after the quietly ironic theodolites cajole be 65|Customer#000000065|RGT yzQ0y4l0H90P783LG4U95bXQFDRXbWa1sl,X|23|33-733-623-5267|8795.16|AUTOMOBILE|y final foxes serve carefully. theodolites are carefully. pending i 66|Customer#000000066|XbsEqXH1ETbJYYtA1A|22|32-213-373-5094|242.77|HOUSEHOLD|le slyly accounts. carefully silent packages benea 67|Customer#000000067|rfG0cOgtr5W8 xILkwp9fpCS8|9|19-403-114-4356|8166.59|MACHINERY|indle furiously final, even theodo 68|Customer#000000068|o8AibcCRkXvQFh8hF,7o|12|22-918-832-2411|6853.37|HOUSEHOLD| pending pinto beans impress realms. final dependencies 69|Customer#000000069|Ltx17nO9Wwhtdbe9QZVxNgP98V7xW97uvSH1prEw|9|19-225-978-5670|1709.28|HOUSEHOLD|thely final ideas around the quickly final dependencies affix carefully quickly final theodolites. final accounts c 70|Customer#000000070|mFowIuhnHjp2GjCiYYavkW kUwOjIaTCQ|22|32-828-107-2832|4867.52|FURNITURE|fter the special asymptotes. ideas after the unusual frets cajole quickly regular pinto be 71|Customer#000000071|TlGalgdXWBmMV,6agLyWYDyIz9MKzcY8gl,w6t1B|7|17-710-812-5403|-611.19|HOUSEHOLD|g courts across the regular, final pinto beans are blithely pending ac 72|Customer#000000072|putjlmskxE,zs,HqeIA9Wqu7dhgH5BVCwDwHHcf|2|12-759-144-9689|-362.86|FURNITURE|ithely final foxes sleep always quickly bold accounts. final wat 73|Customer#000000073|8IhIxreu4Ug6tt5mog4|0|10-473-439-3214|4288.50|BUILDING|usual, unusual packages sleep busily along the furiou 74|Customer#000000074|IkJHCA3ZThF7qL7VKcrU nRLl,kylf |4|14-199-862-7209|2764.43|MACHINERY|onic accounts. blithely slow packages would haggle carefully. qui 75|Customer#000000075|Dh 6jZ,cwxWLKQfRKkiGrzv6pm|18|28-247-803-9025|6684.10|AUTOMOBILE| instructions cajole even, even deposits. finally bold deposits use above the even pains. slyl 76|Customer#000000076|m3sbCvjMOHyaOofH,e UkGPtqc4|0|10-349-718-3044|5745.33|FURNITURE|pecial deposits. ironic ideas boost blithely according to the closely ironic theodolites! furiously final deposits n 77|Customer#000000077|4tAE5KdMFGD4byHtXF92vx|17|27-269-357-4674|1738.87|BUILDING|uffily silent requests. carefully ironic asymptotes among the ironic hockey players are carefully bli 78|Customer#000000078|HBOta,ZNqpg3U2cSL0kbrftkPwzX|9|19-960-700-9191|7136.97|FURNITURE|ests. blithely bold pinto beans h 79|Customer#000000079|n5hH2ftkVRwW8idtD,BmM2|15|25-147-850-4166|5121.28|MACHINERY|es. packages haggle furiously. regular, special requests poach after the quickly express ideas. blithely pending re 80|Customer#000000080|K,vtXp8qYB |0|10-267-172-7101|7383.53|FURNITURE|tect among the dependencies. bold accounts engage closely even pinto beans. ca 81|Customer#000000081|SH6lPA7JiiNC6dNTrR|20|30-165-277-3269|2023.71|BUILDING|r packages. fluffily ironic requests cajole fluffily. ironically regular theodolit 82|Customer#000000082|zhG3EZbap4c992Gj3bK,3Ne,Xn|18|28-159-442-5305|9468.34|AUTOMOBILE|s wake. bravely regular accounts are furiously. regula 83|Customer#000000083|HnhTNB5xpnSF20JBH4Ycs6psVnkC3RDf|22|32-817-154-4122|6463.51|BUILDING|ccording to the quickly bold warhorses. final, regular foxes integrate carefully. bold packages nag blithely ev 84|Customer#000000084|lpXz6Fwr9945rnbtMc8PlueilS1WmASr CB|11|21-546-818-3802|5174.71|FURNITURE|ly blithe foxes. special asymptotes haggle blithely against the furiously regular depo 85|Customer#000000085|siRerlDwiolhYR 8FgksoezycLj|5|15-745-585-8219|3386.64|FURNITURE|ronic ideas use above the slowly pendin 86|Customer#000000086|US6EGGHXbTTXPL9SBsxQJsuvy|0|10-677-951-2353|3306.32|HOUSEHOLD|quests. pending dugouts are carefully aroun 87|Customer#000000087|hgGhHVSWQl 6jZ6Ev|23|33-869-884-7053|6327.54|FURNITURE|hely ironic requests integrate according to the ironic accounts. slyly regular pla 88|Customer#000000088|wtkjBN9eyrFuENSMmMFlJ3e7jE5KXcg|16|26-516-273-2566|8031.44|AUTOMOBILE|s are quickly above the quickly ironic instructions; even requests about the carefully final deposi 89|Customer#000000089|dtR, y9JQWUO6FoJExyp8whOU|14|24-394-451-5404|1530.76|FURNITURE|counts are slyly beyond the slyly final accounts. quickly final ideas wake. r 90|Customer#000000090|QxCzH7VxxYUWwfL7|16|26-603-491-1238|7354.23|BUILDING|sly across the furiously even 91|Customer#000000091|S8OMYFrpHwoNHaGBeuS6E 6zhHGZiprw1b7 q|8|18-239-400-3677|4643.14|AUTOMOBILE|onic accounts. fluffily silent pinto beans boost blithely according to the fluffily exp 92|Customer#000000092|obP PULk2LH LqNF,K9hcbNqnLAkJVsl5xqSrY,|2|12-446-416-8471|1182.91|MACHINERY|. pinto beans hang slyly final deposits. ac 93|Customer#000000093|EHXBr2QGdh|7|17-359-388-5266|2182.52|MACHINERY|press deposits. carefully regular platelets r 94|Customer#000000094|IfVNIN9KtkScJ9dUjK3Pg5gY1aFeaXewwf|9|19-953-499-8833|5500.11|HOUSEHOLD|latelets across the bold, final requests sleep according to the fluffily bold accounts. unusual deposits amon 95|Customer#000000095|EU0xvmWvOmUUn5J,2z85DQyG7QCJ9Xq7|15|25-923-255-2929|5327.38|MACHINERY|ithely. ruthlessly final requests wake slyly alongside of the furiously silent pinto beans. even the 96|Customer#000000096|vWLOrmXhRR|8|18-422-845-1202|6323.92|AUTOMOBILE|press requests believe furiously. carefully final instructions snooze carefully. 97|Customer#000000097|OApyejbhJG,0Iw3j rd1M|17|27-588-919-5638|2164.48|AUTOMOBILE|haggle slyly. bold, special ideas are blithely above the thinly bold theo 98|Customer#000000098|7yiheXNSpuEAwbswDW|12|22-885-845-6889|-551.37|BUILDING|ages. furiously pending accounts are quickly carefully final foxes: busily pe 99|Customer#000000099|szsrOiPtCHVS97Lt|15|25-515-237-9232|4088.65|HOUSEHOLD|cajole slyly about the regular theodolites! furiously bold requests nag along the pending, regular packages. somas 100|Customer#000000100|fptUABXcmkC5Wx|20|30-749-445-4907|9889.89|FURNITURE|was furiously fluffily quiet deposits. silent, pending requests boost against 101|Customer#000000101|sMmL2rNeHDltovSm Y|2|12-514-298-3699|7470.96|MACHINERY| sleep. pending packages detect slyly ironic pack 102|Customer#000000102|UAtflJ06 fn9zBfKjInkQZlWtqaA|19|29-324-978-8538|8462.17|BUILDING|ously regular dependencies nag among the furiously express dinos. blithely final 103|Customer#000000103|8KIsQX4LJ7QMsj6DrtFtXu0nUEdV,8a|9|19-216-107-2107|2757.45|BUILDING|furiously pending notornis boost slyly around the blithely ironic ideas? final, even instructions cajole fl 104|Customer#000000104|9mcCK L7rt0SwiYtrbO88DiZS7U d7M|10|20-966-284-8065|-588.38|FURNITURE|rate carefully slyly special pla 105|Customer#000000105|4iSJe4L SPjg7kJj98Yz3z0B|10|20-793-553-6417|9091.82|MACHINERY|l pains cajole even accounts. quietly final instructi 106|Customer#000000106|xGCOEAUjUNG|1|11-751-989-4627|3288.42|MACHINERY|lose slyly. ironic accounts along the evenly regular theodolites wake about the special, final gifts. 107|Customer#000000107|Zwg64UZ,q7GRqo3zm7P1tZIRshBDz|15|25-336-529-9919|2514.15|AUTOMOBILE|counts cajole slyly. regular requests wake. furiously regular deposits about the blithely final fo 108|Customer#000000108|GPoeEvpKo1|5|15-908-619-7526|2259.38|BUILDING|refully ironic deposits sleep. regular, unusual requests wake slyly 109|Customer#000000109|OOOkYBgCMzgMQXUmkocoLb56rfrdWp2NE2c|16|26-992-422-8153|-716.10|BUILDING|es. fluffily final dependencies sleep along the blithely even pinto beans. final deposits haggle furiously furiou 110|Customer#000000110|mymPfgphaYXNYtk|10|20-893-536-2069|7462.99|AUTOMOBILE|nto beans cajole around the even, final deposits. quickly bold packages according to the furiously regular dept 111|Customer#000000111|CBSbPyOWRorloj2TBvrK9qp9tHBs|22|32-582-283-7528|6505.26|MACHINERY|ly unusual instructions detect fluffily special deposits-- theodolites nag carefully during the ironic dependencies 112|Customer#000000112|RcfgG3bO7QeCnfjqJT1|19|29-233-262-8382|2953.35|FURNITURE|rmanently unusual multipliers. blithely ruthless deposits are furiously along the 113|Customer#000000113|eaOl5UBXIvdY57rglaIzqvfPD,MYfK|12|22-302-930-4756|2912.00|BUILDING|usly regular theodolites boost furiously doggedly pending instructio 114|Customer#000000114|xAt 5f5AlFIU|14|24-805-212-7646|1027.46|FURNITURE|der the carefully express theodolites are after the packages. packages are. bli 115|Customer#000000115|0WFt1IXENmUT2BgbsB0ShVKJZt0HCBCbFl0aHc|8|18-971-699-1843|7508.92|HOUSEHOLD|sits haggle above the carefully ironic theodolite 116|Customer#000000116|yCuVxIgsZ3,qyK2rloThy3u|16|26-632-309-5792|8403.99|BUILDING|as. quickly final sauternes haggle slyly carefully even packages. brave, ironic pinto beans are above the furious 117|Customer#000000117|uNhM,PzsRA3S,5Y Ge5Npuhi|24|34-403-631-3505|3950.83|FURNITURE|affix. instructions are furiously sl 118|Customer#000000118|OVnFuHygK9wx3xpg8|18|28-639-943-7051|3582.37|AUTOMOBILE|uick packages alongside of the furiously final deposits haggle above the fluffily even foxes. blithely dogged dep 119|Customer#000000119|M1ETOIecuvH8DtM0Y0nryXfW|7|17-697-919-8406|3930.35|FURNITURE|express ideas. blithely ironic foxes thrash. special acco 120|Customer#000000120|zBNna00AEInqyO1|12|22-291-534-1571|363.75|MACHINERY| quickly. slyly ironic requests cajole blithely furiously final dependen 121|Customer#000000121|tv nCR2YKupGN73mQudO|17|27-411-990-2959|6428.32|BUILDING|uriously stealthy ideas. carefully final courts use carefully 122|Customer#000000122|yp5slqoNd26lAENZW3a67wSfXA6hTF|3|13-702-694-4520|7865.46|HOUSEHOLD| the special packages hinder blithely around the permanent requests. bold depos 123|Customer#000000123|YsOnaaER8MkvK5cpf4VSlq|5|15-817-151-1168|5897.83|BUILDING|ependencies. regular, ironic requests are fluffily regu 124|Customer#000000124|aTbyVAW5tCd,v09O|18|28-183-750-7809|1842.49|AUTOMOBILE|le fluffily even dependencies. quietly s 125|Customer#000000125|,wSZXdVR xxIIfm9s8ITyLl3kgjT6UC07GY0Y|19|29-261-996-3120|-234.12|FURNITURE|x-ray finally after the packages? regular requests c 126|Customer#000000126|ha4EHmbx3kg DYCsP6DFeUOmavtQlHhcfaqr|22|32-755-914-7592|1001.39|HOUSEHOLD|s about the even instructions boost carefully furiously ironic pearls. ruthless, 127|Customer#000000127|Xyge4DX2rXKxXyye1Z47LeLVEYMLf4Bfcj|21|31-101-672-2951|9280.71|MACHINERY|ic, unusual theodolites nod silently after the final, ironic instructions: pending r 128|Customer#000000128|AmKUMlJf2NRHcKGmKjLS|4|14-280-874-8044|-986.96|HOUSEHOLD|ing packages integrate across the slyly unusual dugouts. blithely silent ideas sublate carefully. blithely expr 129|Customer#000000129|q7m7rbMM0BpaCdmxloCgBDRCleXsXkdD8kf|7|17-415-148-7416|9127.27|HOUSEHOLD| unusual deposits boost carefully furiously silent ideas. pending accounts cajole slyly across 130|Customer#000000130|RKPx2OfZy0Vn 8wGWZ7F2EAvmMORl1k8iH|9|19-190-993-9281|5073.58|HOUSEHOLD|ix slowly. express packages along the furiously ironic requests integrate daringly deposits. fur 131|Customer#000000131|jyN6lAjb1FtH10rMC,XzlWyCBrg75|11|21-840-210-3572|8595.53|HOUSEHOLD|jole special packages. furiously final dependencies about the furiously speci 132|Customer#000000132|QM5YabAsTLp9|4|14-692-150-9717|162.57|HOUSEHOLD|uickly carefully special theodolites. carefully regular requests against the blithely unusual instructions 133|Customer#000000133|IMCuXdpIvdkYO92kgDGuyHgojcUs88p|17|27-408-997-8430|2314.67|AUTOMOBILE|t packages. express pinto beans are blithely along the unusual, even theodolites. silent packages use fu 134|Customer#000000134|sUiZ78QCkTQPICKpA9OBzkUp2FM|11|21-200-159-5932|4608.90|BUILDING|yly fluffy foxes boost final ideas. b 135|Customer#000000135|oZK,oC0 fdEpqUML|19|29-399-293-6241|8732.91|FURNITURE| the slyly final accounts. deposits cajole carefully. carefully sly packag 136|Customer#000000136|QoLsJ0v5C1IQbh,DS1|7|17-501-210-4726|-842.39|FURNITURE|ackages sleep ironic, final courts. even requests above the blithely bold requests g 137|Customer#000000137|cdW91p92rlAEHgJafqYyxf1Q|16|26-777-409-5654|7838.30|HOUSEHOLD|carefully regular theodolites use. silent dolphins cajo 138|Customer#000000138|5uyLAeY7HIGZqtu66Yn08f|5|15-394-860-4589|430.59|MACHINERY|ts doze on the busy ideas. regular 139|Customer#000000139|3ElvBwudHKL02732YexGVFVt |9|19-140-352-1403|7897.78|MACHINERY|nstructions. quickly ironic ideas are carefully. bold, 140|Customer#000000140|XRqEPiKgcETII,iOLDZp5jA|4|14-273-885-6505|9963.15|MACHINERY|ies detect slyly ironic accounts. slyly ironic theodolites hag 141|Customer#000000141|5IW,WROVnikc3l7DwiUDGQNGsLBGOL6Dc0|1|11-936-295-6204|6706.14|FURNITURE|packages nag furiously. carefully unusual accounts snooze according to the fluffily regular pinto beans. slyly spec 142|Customer#000000142|AnJ5lxtLjioClr2khl9pb8NLxG2,|9|19-407-425-2584|2209.81|AUTOMOBILE|. even, express theodolites upo 143|Customer#000000143|681r22uL452zqk 8By7I9o9enQfx0|16|26-314-406-7725|2186.50|MACHINERY|across the blithely unusual requests haggle theodo 144|Customer#000000144|VxYZ3ebhgbltnetaGjNC8qCccjYU05 fePLOno8y|1|11-717-379-4478|6417.31|MACHINERY|ges. slyly regular accounts are slyly. bold, idle reque 145|Customer#000000145|kQjHmt2kcec cy3hfMh969u|13|23-562-444-8454|9748.93|HOUSEHOLD|ests? express, express instructions use. blithely fina 146|Customer#000000146|GdxkdXG9u7iyI1,,y5tq4ZyrcEy|3|13-835-723-3223|3328.68|FURNITURE|ffily regular dinos are slyly unusual requests. slyly specia 147|Customer#000000147|6VvIwbVdmcsMzuu,C84GtBWPaipGfi7DV|18|28-803-187-4335|8071.40|AUTOMOBILE|ress packages above the blithely regular packages sleep fluffily blithely ironic accounts. 148|Customer#000000148|BhSPlEWGvIJyT9swk vCWE|11|21-562-498-6636|2135.60|HOUSEHOLD|ing to the carefully ironic requests. carefully regular dependencies about the theodolites wake furious 149|Customer#000000149|3byTHCp2mNLPigUrrq|19|29-797-439-6760|8959.65|AUTOMOBILE|al instructions haggle against the slyly bold w 150|Customer#000000150|zeoGShTjCwGPplOWFkLURrh41O0AZ8dwNEEN4 |18|28-328-564-7630|3849.48|MACHINERY|ole blithely among the furiously pending packages. furiously bold ideas wake fluffily ironic idea 151|Customer#000000151|LlyEtNEXT6kkZ,kGP46H|19|29-433-197-6339|5187.02|HOUSEHOLD|regular dugouts: blithely even dolphins cajole furiously carefull 152|Customer#000000152|PDrllSkScKLh4lr19gmUZnK|8|18-585-850-3926|1215.18|BUILDING|ously ironic accounts. furiously even accounts accord 153|Customer#000000153|kDzx11sIjjWJm1|6|16-342-316-2815|5454.26|HOUSEHOLD|promise carefully. unusual deposits x-ray. carefully regular tithes u 154|Customer#000000154|2LAlU fDHkOqbXjHHDqw1mJQNC|19|29-522-835-6914|4695.12|FURNITURE|nic packages haggle blithely across the 155|Customer#000000155|l,sSphiStMgdrxpxi|0|10-566-282-8705|5902.85|AUTOMOBILE| sleep ironic, bold requests. regular packages on the quiet dependencies 156|Customer#000000156|5OS0edX2Y6B1cf9wJNuOQWgrrZccXk9|9|19-723-913-3943|9302.95|AUTOMOBILE| regular foxes above the theodolites haggle 157|Customer#000000157|HGEouzCcFrNd nBAdsCRjsMxKOvYZdbwA7he5w9v|15|25-207-442-1556|9768.73|BUILDING| pinto beans against the carefully bold requests wake quickly alongside of the final accounts. accounts 158|Customer#000000158|2HaYxi0J1620aoI1CdFyrW,rWOy|10|20-383-680-1329|6160.95|AUTOMOBILE|ecoys. fluffily quick requests use flu 159|Customer#000000159|KotsdDO6EHnysVu922s6pjZpG,vlT|10|20-888-668-2668|2060.06|HOUSEHOLD|cingly express somas haggle above the theodolites. pinto beans use special theodolites. theodolites sleep 160|Customer#000000160|5soVQ3dOCRBWBS|13|23-428-666-4806|4363.17|MACHINERY|olites. silently ironic accounts cajole furious 161|Customer#000000161|2oRkx,NtjFUh|7|17-805-718-2449|3714.06|MACHINERY|ptotes nag carefully instructions. silent accounts are. furiously even accounts alongside 162|Customer#000000162|JE398sXZt2QuKXfJd7poNpyQFLFtth|8|18-131-101-2267|6268.99|MACHINERY|accounts along the doggedly special asymptotes boost blithely during the quickly regular theodolites. slyly 163|Customer#000000163|OgrGcOnm4whd0f|21|31-863-349-4121|2948.61|FURNITURE| nag furiously furiously final requests. slyly s 164|Customer#000000164|YDW51PBWLXLnbQlKC|4|14-565-638-9768|208.45|HOUSEHOLD|ironic, special pinto beans. ironic 165|Customer#000000165|8pc6kwBmwBdEnfVP53aqL9DM4LymC4|0|10-927-209-5601|3349.92|HOUSEHOLD| requests. final ideas cajole quickly at the special, ironic acco 166|Customer#000000166|15HWGtwoP77EJfd95HxtMSTZUelV8NOKne2|10|20-320-530-5920|2042.21|FURNITURE|the packages. blithely final packages are furiously unusual asymptotes. regular frets promise carefully u 167|Customer#000000167|QNc2eOlRIzL6jpthwgDuB866uCIUPiOX|5|15-288-395-5501|1468.09|AUTOMOBILE|espite the ironic excuses. furiously final deposits wake slyly. slyly ex 168|Customer#000000168|GDcL5qU86P8,oaTwVBCLE6asM8rlxpE,211uziU|12|22-354-984-5361|-808.56|FURNITURE|blithely final accounts sleep quickly along the regular ideas. furiously sly foxes nag across the 169|Customer#000000169|NjhmHa7xrcjE|18|28-362-499-3728|4483.83|FURNITURE|fully unusual pinto beans. blithely express asymptotes lose carefully regular instructions? accounts b 170|Customer#000000170|5QmxmYubNhn6HAgLwTvphevM3OmpZTGsM|15|25-879-984-9818|7687.89|BUILDING| regular requests. carefully regu 171|Customer#000000171|RIhjJCrth89EU7xRSvN|7|17-513-603-7451|2379.91|MACHINERY|ly furiously final requests. slyly final requests wake silently pending, silent accounts. exp 172|Customer#000000172|KwgdKUL1G2WacsMNF50yX|22|32-178-964-1847|1134.40|MACHINERY|losely regular, unusual instructions. 173|Customer#000000173|Aue7KVz,FinSHpov Vk5ed,wSQ2BRSioJ0|9|19-443-196-8008|845.84|BUILDING|s pinto beans use thinly slyly regular packages. instructions print along the s 174|Customer#000000174|R5 fCPMSeDXtUpp5Ax|23|33-845-455-8799|1944.73|FURNITURE|oldly even requests haggle quickly blithely ironic accounts. idly final foxes doze slyly pending dep 175|Customer#000000175|8YK1ZyTqoY3wMWnExl4itPMLL793GpEZb6T|10|20-427-617-9922|1975.35|FURNITURE|ly final platelets are final pinto b 176|Customer#000000176|9hBepY2uz88HlCqToOLgeU770u81FeL|13|23-432-942-8830|-375.76|FURNITURE|uriously. final requests sleep ironic packages. quickly 177|Customer#000000177|6wzEKPyZE9dmBCJZ8e7x7fiiK,k|1|11-917-786-9955|7457.50|BUILDING|nal dolphins: blithely bold gifts wake slyly afte 178|Customer#000000178|p HUSDg8Cgan4Fj8Drvcdz4gi4dSqV0a7n 0ag|21|31-436-268-6327|2272.50|FURNITURE|unts. blithely regular dependencies kindle pending deposits. quietly express deposits wake above the Tiresias-- ex 179|Customer#000000179|djez3CWg0nnCiu60jsF|4|14-703-953-2987|-43.08|MACHINERY|st furiously. idly regular instructions wake fluffily slyl 180|Customer#000000180|DSGW3RFoYJE opVw,Y3wGCGcNULZi|13|23-678-802-2105|-92.58|FURNITURE|lar accounts sublate above the slyly final 181|Customer#000000181|YNviWd WrRkZvSw1OxIewBq|9|19-653-305-8440|3929.96|FURNITURE|final requests cajole furiously acro 182|Customer#000000182|tdwvgepG316CCTHtMaF8Q|3|13-199-211-9023|4810.22|AUTOMOBILE|quickly against the blithely even deposits; epitaphs unwind quickly along the carefully regular excuses. furio 183|Customer#000000183|aMAB2QSb8 86MAx|22|32-771-279-8154|4419.89|HOUSEHOLD|sual accounts across the slyl 184|Customer#000000184|uoOpBuRr42f1WIqnVYAhxbAA9bkK6HUGpOt|21|31-739-340-5476|170.46|AUTOMOBILE|hely according to the furiously unusual accounts. furiously bold platele 185|Customer#000000185|iHXzQgienOQ|5|15-760-572-8760|2788.76|BUILDING|t the ironic accounts. fluffily regular requests wake slyly ironic pinto beans. slyly unusu 186|Customer#000000186|BeVr6MzaobBENXRBC8pmOmkByMJI|3|13-518-743-2576|8737.50|HOUSEHOLD|e slyly final dependencies. unusual instructions against the carefully pending instructions boost quickly 187|Customer#000000187|OIlgR6oIRXV5g63q5YGudCjRD8kpod2p|4|14-716-294-6674|-774.22|FURNITURE|r deposits. carefully silent packages after the fluffily even instructio 188|Customer#000000188|58Srs6gEEoD3ZfwgXDM1OayRiaSY6K9YsveWwV|5|15-613-528-7811|9533.37|BUILDING|st slyly special platelets. bold, 189|Customer#000000189|r51HSq Rg8wQgF1CBfG1Vbye3GK|22|32-980-348-1114|-594.05|MACHINERY|sly express patterns. ideas on the regular d 190|Customer#000000190|F2X,GhSqLz8k u0gWsirsraFaEDEo6vIGtOTaO1T|11|21-730-373-8193|1657.46|AUTOMOBILE|uickly-- fluffily pending instructions boo 191|Customer#000000191|P1eCXsPWkv2y6ENQv|16|26-811-707-6869|2945.16|BUILDING|o beans hinder slyly bold accounts. 192|Customer#000000192|rDmB2c9d1BJQ y6R9jTx86YI77D|10|20-750-712-2481|8239.96|MACHINERY|ely unusual packages are fluffily 193|Customer#000000193|dUT4dtsPTZ6ZpkWLc,KGJCHY6JDJgPFH4|23|33-182-978-6287|8024.55|MACHINERY|y even theodolites. final foxes print along the final pinto beans. theodoli 194|Customer#000000194|mksKhdWuQ1pjbc4yffHp8rRmLOMcJ|16|26-597-636-3003|6696.49|HOUSEHOLD|quickly across the fluffily dogged requests. regular platelets around the ironic, even requests cajole quickl 195|Customer#000000195|WiqQD8hscyKekjMcSBA7AX 0AbxvBV|22|32-757-684-6845|4873.91|AUTOMOBILE| should detect blithely. quickly even packages above the deposits wak 196|Customer#000000196|68RstNo6a2B|18|28-135-177-2472|7760.33|FURNITURE|accounts wake. express instructions according to the s 197|Customer#000000197|UeVqssepNuXmtZ38D|1|11-107-312-6585|9860.22|AUTOMOBILE|ickly final accounts cajole. furiously re 198|Customer#000000198|,7fcZHIUn,fUaQtK8U,Q8|1|11-237-758-6141|3824.76|AUTOMOBILE|tions. slyly ironic waters wa 199|Customer#000000199|lBU3xll,a7e9TYm3 UyjDPCVMvnHKpq,9HW1X|4|14-136-924-5232|7654.31|FURNITURE|fully busy pinto beans. packages cajole around the express, bold packages! quickly ironic tithes 200|Customer#000000200|x1 H5c66DUgH2pgNTJhw6eZKgrAz|16|26-472-302-4189|9967.60|BUILDING|e after the ironic, even realms. fluffily regular packages doze-- courts haggle carefully! blithely 201|Customer#000000201|yWLtmd5usyjsCvyL1QJsBorC|2|12-759-183-9859|4614.40|MACHINERY| blithely even packages sleep carefully bold, unus 202|Customer#000000202|Q0uJ1frCbi9yvu|7|17-905-805-4635|2237.64|AUTOMOBILE|fully along the carefully pending Tiresias; special packages along the carefully special deposits try to 203|Customer#000000203|2fRlubh lWRinCs1nimADdn|1|11-886-563-6149|7960.63|MACHINERY| packages are. requests integrate regularly across th 204|Customer#000000204|7U7u2KryFP|6|16-761-837-4820|-627.76|BUILDING|ages. accounts wake slyly. dolphins nag blithely. final, regular requests haggle blithely furiously even 205|Customer#000000205|jOTQBGb nhfBMu3,LIN62WogLDBO0w|12|22-356-437-1311|7161.52|BUILDING| furiously pending accounts. ideas along the slyly final deposits cajole blithel 206|Customer#000000206|xsg,ehRHS5OKqyBR5YtoPm8myz|9|19-976-832-3312|-274.79|AUTOMOBILE| the carefully regular foxes. regular accounts wake furiously braids. bold ideas are carefu 207|Customer#000000207|ewz5JNnxJPmPGY|21|31-562-675-6475|-439.98|AUTOMOBILE|n theodolites against the evenly even requests boost carefully pinto beans! fi 208|Customer#000000208|Abye1MwcNfY0KO6yqv,Wwe|19|29-859-139-6234|6239.89|MACHINERY|le carefully according to the quickly silent packages. quickly ironic packages affix according to the ruthles 209|Customer#000000209|iBvmxOZV3qXMYQW3W4Oo7YFhdV|16|26-207-121-7721|8873.46|FURNITURE|deposits. furiously regular ideas across the quietly regular accounts cajole about the express packages. quickly reg 210|Customer#000000210|13cFL9sG1nrGERURN9WZI0|20|30-876-248-9750|7250.14|HOUSEHOLD|nusual instructions sleep regular acc 211|Customer#000000211|URhlVPzz4FqXem|13|23-965-335-9471|4198.72|BUILDING|furiously regular foxes boost fluffily special ideas. carefully regular dependencies are. slyly ironic 212|Customer#000000212|19U0iZ3GtDdrsn|7|17-382-405-4333|957.58|BUILDING|symptotes are blithely special pinto beans. blithely ironic 213|Customer#000000213|NpqMYBhBcWk8mnEta|24|34-768-700-9764|9987.71|HOUSEHOLD|al deposits. final instructions boost carefully. even deposits sleep quickly. furiously regul 214|Customer#000000214|MpCwhcLrbcIM7AeKS9tRM09by|8|18-180-678-6165|1526.59|MACHINERY|grow. fluffily regular pinto beans according to the regular accounts affix quickly pe 215|Customer#000000215|8H76xbBhde HY70BrYqGEFmVPXqlm8pgjjxh|9|19-564-446-4758|3379.20|FURNITURE|al pinto beans. ironic foxes serve. i 216|Customer#000000216|LXH7wSv4I6GG6TAkLOyLcMh559a8Y|21|31-296-111-5448|-776.08|FURNITURE|hely at the pending warhorses; blithe 217|Customer#000000217|YIy05RMdthrXqdfnNKud|23|33-159-298-3849|378.33|AUTOMOBILE|ven frays wake according to the carefully 218|Customer#000000218| V1FCIeSseuyNGYfHS Rx0,sc4IsBfReV|4|14-480-931-8567|9541.19|MACHINERY|lar courts. furiously pending dependencies cajole blithely? fluffily regular deposits cajol 219|Customer#000000219|eTjiL01eyoKiAe2WQoz3EpPg2lvSLeOu2X2wyxK|11|21-159-138-6090|9858.57|AUTOMOBILE|ckly multipliers. carefully eve 220|Customer#000000220|TbUHVhkttz|16|26-201-301-7371|9131.64|BUILDING| even, even accounts are. ironic 221|Customer#000000221|ripNyyPOewg8AahnZlsM|16|26-351-738-1001|1609.39|BUILDING| instructions above the regular requests cajole packages. pending, even 222|Customer#000000222|gAPkFjwxX1Zq 2Yq6 FIfLdJ4yUOt4Al7DL18Ou|1|11-722-672-5418|8893.76|BUILDING|regular accounts haggle furiously around the c 223|Customer#000000223|ftau6Pk,brboMyEl,,kFm|20|30-193-643-1517|7476.20|BUILDING|al, regular requests run furiously blithely silent packages. blithely ironic accounts across the furious 224|Customer#000000224|4tCJvf30WagGfacqcAqmfCptu2cbMVcj2M7Y0W|15|25-224-867-3668|8465.15|BUILDING|counts. bold packages doubt according to the furiously pending packages. bold, regular pinto beans 225|Customer#000000225|2HFk1E0fmqs|13|23-979-183-7021|8893.20|AUTOMOBILE|ages boost among the special foxes. quiet, final foxes lose carefully about the furiously unusual th 226|Customer#000000226|ToEmqB90fM TkLqyEgX8MJ8T8NkK|3|13-452-318-7709|9008.61|AUTOMOBILE|ic packages. ideas cajole furiously slyly special theodolites: carefully express pinto beans acco 227|Customer#000000227|7wlpEBswtXBPNODASgCUt8OZQ|13|23-951-816-2439|1808.23|MACHINERY|lar, ironic pinto beans use! quickly regular theodolites maintain slyly pending pac 228|Customer#000000228|A1Zvuxjdpt8TZP6i41H3fn9csGqOJUm5x0NIS1LA|20|30-435-915-1603|6868.12|FURNITURE| blithely ironic theodolites 229|Customer#000000229|Sbvjxgmwy4u6Ks1FH7lxo7toMmeU5dG|1|11-243-298-4029|7568.07|BUILDING|bold accounts haggle furiously even deposits. regular instruct 230|Customer#000000230|CnR8xt3MYqID0tiHwYh|21|31-744-950-8047|1682.83|MACHINERY|c decoys impress even deposits. thinly final asymptotes 231|Customer#000000231|WFOhG9Z9ohRdsyuYnPvBSv|10|20-825-880-1065|283.55|BUILDING|ly final deposits. fluffily ironic requests wake carefully carefully regular accounts. quickly sp 232|Customer#000000232|oA9o,3YOXu2rzKONdd,cxpqCFXUv5kuxBYKp|22|32-283-563-2674|554.71|HOUSEHOLD|ges sleep. final, bold theodolites are quickly final packages. furiously ironic packages are slyly fi 233|Customer#000000233|mFm45wZ7rV4VIbEE1F4|3|13-574-104-3221|3998.24|FURNITURE|st the special instructions. theodolites detect blithely according 234|Customer#000000234|ILyuJbixVmrNEVxsfQOMFxByySs|18|28-243-424-1393|8383.51|AUTOMOBILE| fluffily regular ideas play s 235|Customer#000000235|bp0rIBMh4fMdQnHBmMnB|3|13-350-790-6416|754.41|AUTOMOBILE|hely ruthless instructions again 236|Customer#000000236|kcW,mM0rhIstAcVaol1,6DVkS FPKlhY|14|24-808-967-4503|5384.59|AUTOMOBILE|te slyly along the requests. carefully final requests sleep slyly blithe frets. furiously ruthless dep 237|Customer#000000237|R dtznB5ocPPo|19|29-414-970-5238|-160.02|HOUSEHOLD|regular pinto beans sleep furiously ironically silent theodolites. quickly ironic courts after the deposits sleep f 238|Customer#000000238|tE0lVKK3tz5AG2 Hal2XHwE485g5MX7|16|26-307-925-1236|3482.32|HOUSEHOLD|uffily ironic theodolites are. regular, regular ideas cajole according to the blithely pending epitaphs. slyly 239|Customer#000000239|w8eRmMOmUTjVOkucbfcGDh2AqCixTOC|9|19-699-117-6988|5398.77|FURNITURE|uctions. furiously even dolphins haggle fluffily according to the furiously regular dep 240|Customer#000000240|SXfeEOwRZsXArtY3C5UWqXgLcJBAMmaynaTJs8|9|19-756-548-7835|7139.68|MACHINERY|al accounts about the slyly pending p 241|Customer#000000241|FBuwHkPR450PvnZnAezcaeMaS,hX3Ifdk|9|19-344-614-2207|6569.34|AUTOMOBILE| across the enticingly even requests. blithely iro 242|Customer#000000242|apgzK3HWAjKHFteJ16Tg3OERViesqBbx|3|13-324-350-3564|1975.41|MACHINERY|riously ironic pinto beans cajole silently. regular foxes wake slyly. bravely 243|Customer#000000243|te2FOn8xJzJinZc|7|17-297-684-7972|620.73|AUTOMOBILE|nic deposits. evenly pending deposits boost fluffily careful 244|Customer#000000244|FBVbCpEVaFaP8KogqQO2VuXeVx|15|25-621-225-8173|2506.38|HOUSEHOLD|encies. requests nag carefully. regularly final accounts h 245|Customer#000000245|IseFIO7jTGPTzAdZPoO2X4VX48Hy|12|22-952-232-2729|3720.15|MACHINERY|s. regular foxes against the s 246|Customer#000000246|WrRUR0ds6iypmopww9y9t0NJBRbke78qJm|15|25-608-618-2590|9584.96|AUTOMOBILE| requests shall have to integrate furiously pending courts. sil 247|Customer#000000247|N8q4W4QQG2mHY47Dg6|20|30-151-905-3513|8495.92|HOUSEHOLD|es affix furiously regular deposits. blithely ironic asymptotes after the blithely e 248|Customer#000000248|mgT15r8asLyaED|10|20-447-727-8914|8908.35|FURNITURE|s detect blithely. blithely pending dolphins along the fluffily final accounts haggle fu 249|Customer#000000249|0XE2fhX7j2uivaHBrFuRl1NoJnaTSIQT|3|13-226-455-7727|-234.01|MACHINERY|its are after the special deposits. ironic, final deposits against the slyl 250|Customer#000000250|9hif3yif6z8w8pW88F755PU7uz|16|26-464-852-1461|2869.97|FURNITURE|s. slyly unusual instructions cajole quickly carefully bold dep 251|Customer#000000251|Z9fdQmv07C3k hxwt9nchhuQiqC4wox85se8EW7L|13|23-975-623-5949|9585.32|HOUSEHOLD|fully blithely regular requests. fluffily even dugouts detect furiously final ideas. sometimes ironic depos 252|Customer#000000252|db1bPFF xUkJYzvE3cBtqYeDn2 u|16|26-330-347-9201|3561.74|FURNITURE|ngside of the pending foxes. furiously ironic requests wake. blithely ironic acco 253|Customer#000000253|naGyIRPFPH E|15|25-461-140-9884|9139.52|AUTOMOBILE| regular deposits sleep against the accounts. foxes cajole carefully special 254|Customer#000000254|vQ,pEzMQaFgJzK4TJ2eA|1|11-451-622-6325|1915.35|MACHINERY|equests. carefully ironic deposits detect carefully abo 255|Customer#000000255|I8Wz9sJBZTnEFG08lhcbfTZq3S|3|13-924-679-8287|3196.07|BUILDING|ges mold. bold, regular courts boost furiously at the 256|Customer#000000256|eJ6AggYh80JMEzZNwYK4CIC2flT|10|20-229-271-4429|1299.92|HOUSEHOLD|ld boost about the carefully ironic foxes. slyly special packages cajole alongside of the slyly final accounts. q 257|Customer#000000257|LyIa26EXYaSU|7|17-816-687-2155|-339.85|AUTOMOBILE|s cajole quickly along the ironic pinto beans: even, regular foxes are 258|Customer#000000258|7VbADek8qYezQYotxNUmnNI|12|22-278-425-9944|6022.27|MACHINERY|about the regular, bold accounts; pending packages use furiously stealthy warhorses. bold accounts sleep fur 259|Customer#000000259|kwh9i86Wj1Zq14nwTdhxapIkLEI|5|15-907-674-2046|3335.29|HOUSEHOLD|furiously unusual instructions. s 260|Customer#000000260|CrHY2zx4vner4|1|11-708-529-9446|9196.11|MACHINERY|carefully. furiously bold accounts nag furiously carefully regular accounts-- final decoys prin 261|Customer#000000261|dXkVi8qahjP|12|22-494-898-7855|7094.22|AUTOMOBILE|he special instructions integrate carefully final request 262|Customer#000000262|DcUOAFBxMu8oGKvIqbDx7xgeZ|4|14-698-169-5201|1561.80|AUTOMOBILE|ress packages above the ironic accounts are against the ironic pinto beans. carefully final accoun 263|Customer#000000263|Y2pxeGWkTyaq,0RCzIbZ3|1|11-276-906-3193|1162.03|FURNITURE|usly ironic theodolites cajole furiously. final ep 264|Customer#000000264|24Akixb4hqpRD|11|21-881-683-3829|3195.83|MACHINERY|ular packages cajole blithely a 265|Customer#000000265|sthiqpj6CPAKbD7BBSz9ulRuF9d,ebfaiTc|17|27-716-734-2046|8275.80|MACHINERY|lar, ironic platelets. furiously unu 266|Customer#000000266|VSIEruiMdDvjDaTQxkuK60Yw3AGxO|0|10-474-243-3974|5481.00|HOUSEHOLD|ccounts. quickly ironic excuses after the regular foxes wake along the ironic, fina 267|Customer#000000267|el7 bYzj1USp6T5i3KpfZ43jKegbdO,Jd69|15|25-402-954-8909|3166.94|AUTOMOBILE| detect slyly alongside of the foxes. closely regular pinto beans nag quickly of the blithely bold r 268|Customer#000000268|tkSLQoOpfOa601itad05EcN0UmhjZXdyKRc0r|3|13-720-469-5207|6821.01|MACHINERY|press ideas print quickly. fluffily unusual deposits use blithely eve 269|Customer#000000269|J7kLF9iPOQA 7CVwAmQRpwfZPDJ2q5Seu2Vj1gh|14|24-570-874-6232|7667.35|MACHINERY| close packages-- quickly regular instructions sleep. carefully 270|Customer#000000270|,rdHVwNKXKAgREU|7|17-241-806-3530|9192.50|AUTOMOBILE|ldly final instructions mold carefully along the ironic accounts. 271|Customer#000000271|O,HceV3 XE77Yx|6|16-621-282-5689|1490.35|MACHINERY|ly pending deposits cajole slyly sl 272|Customer#000000272| YDjKpjXEe0A6rDE|2|12-324-877-9650|-746.03|MACHINERY|he regular requests. slyly special 273|Customer#000000273|sOA,alhAw1juArjRLOd|2|12-197-772-5736|-675.05|FURNITURE|ng frets sleep. slyly express dolphins doubt ironically ironic accounts. final de 274|Customer#000000274|adesXwNumnPqsKgsE1groEAwdKNgZ|19|29-330-389-1442|4425.42|FURNITURE|gular dependencies. ironic foxes haggle du 275|Customer#000000275|M1UCTKrZLOgSyr|22|32-194-864-6861|5067.31|AUTOMOBILE|y regular deposits. fluffily ironic packages cajole along the 276|Customer#000000276|iSWxETEMKe5cF|16|26-716-357-3851|2292.67|AUTOMOBILE|eans. even, ironic accounts affix sl 277|Customer#000000277|BWGsQevHk0BfRJV3RRB ElFc|23|33-696-831-5394|8876.10|BUILDING|phins; bold, final accounts print. carefully silent 278|Customer#000000278|4jqLjG 2aeYMFEJi|20|30-445-570-5841|7621.56|BUILDING| pending, express requests cajole carefully special packages. blithely pending accounts affix furiously. fluffily 279|Customer#000000279|9t2Wo1jK1TYnDGg6ODSMGf1W9hRT3F3VK5zxJOC|9|19-220-605-9025|9663.23|AUTOMOBILE|l platelets sleep fluffily against the fluffily enticing excuses. blithely special requests wake somet 280|Customer#000000280|3fDiGmN64En0ei|11|21-537-461-3965|3952.84|BUILDING|accounts. quiet deposits sleep. slyly even instructions detect about the blithely bold instru 281|Customer#000000281|x5gJ8IGm2Fo9Jigt|6|16-809-382-6446|4361.70|BUILDING|fully quiet ideas detect quickly even packages. regular instructions accor 282|Customer#000000282|wcCc, y1996DnOwnXu1i|18|28-251-599-2415|1125.45|HOUSEHOLD|ole daringly against the carefully ir 283|Customer#000000283|jysHDuTKUKYEjdBhtI LqTttTp 7i2kvx1 O3A|7|17-111-303-1282|4450.03|AUTOMOBILE|y alongside of the accounts. slyly express dependencies wake quickly above the carefully ironic package 284|Customer#000000284|2ZgAkaBgb6aigORfIfUd3kHbPi42|6|16-161-235-2690|593.52|AUTOMOBILE|lar gifts. carefully even deposits boost! furiously even braids use afte 285|Customer#000000285|ApUL7bgFMUXGXewpoQyQOSnLeL9Vc1rrkW |20|30-235-130-1313|7276.72|FURNITURE|dolphins after the slyly ironic packages boost furiously among the furiously pending theodolites. bl 286|Customer#000000286|7 7uVDrpkWuozyEd|22|32-274-308-4633|-109.73|HOUSEHOLD|ly special accounts haggle slyly slyly fluffy req 287|Customer#000000287|KTsaTAJRC0eMYkyFm7EK3eeamHs7s|4|14-330-840-6321|1734.18|MACHINERY|requests. bold, silent depths lose f 288|Customer#000000288|eEs5rwc9AOJaKhvV|2|12-674-136-5397|5339.43|HOUSEHOLD| furiously about the carefully ironic packages. express reques 289|Customer#000000289|NUilehg0nVOkK3K1SW0,BAHCeST2JqKzuTMoGS|10|20-456-773-7693|-215.75|AUTOMOBILE|ending foxes across the carefully 290|Customer#000000290|8OlPT9G 8UqVXmVZNbmxVTPO8|4|14-458-625-5633|1811.35|MACHINERY|sts. blithely pending requests sleep fluffily on the regular excuses. carefully expre 291|Customer#000000291|ZlLNbGxnQYMubQ9K|8|18-657-656-2318|4261.68|HOUSEHOLD|e slyly silent deposits. bold deposits haggle slyly special packages. furiously bold requests cajole carefully abo 292|Customer#000000292|hCXh3vxC4uje9|11|21-457-910-2923|2975.43|HOUSEHOLD|usly regular, ironic accounts. blithely regular platelets are carefully. blithely unusual ideas affi 293|Customer#000000293|7ynwX7lZ3o2cmAWSkKAc3edKa 8yT|2|12-887-984-5485|-43.79|MACHINERY|ironic foxes are final packages. requests about the furiousl 294|Customer#000000294|hSaNqI1P2IyEFHY0r0PsPkMqt|18|28-187-946-4260|-994.79|BUILDING|bold packages. regular, final asymptotes use quickly fluffily even waters. blithely express requests wake into th 295|Customer#000000295|mk649IH6njR14woTVZ1cxtlNs URxBHD5o5z2|0|10-340-773-4322|9497.89|HOUSEHOLD|play according to the quickly ironic instructions-- unusual, bol 296|Customer#000000296|4eyqk2zpg4m V JGEtgwNmCq3c|15|25-875-178-1959|8081.52|BUILDING|es need to affix furiously. ironic, final foxes are against the regular instructions: pinto beans haggle q 297|Customer#000000297|hzg,409pj0|15|25-915-518-8800|7096.32|HOUSEHOLD|de of the regular asymptotes detect slyly ironic theod 298|Customer#000000298|jFKF3w 8aegECg7mP,qtuR9IsTSYQlEXq|21|31-542-157-4074|3812.84|BUILDING|sleep slyly. stealthy, bold pinto beans sleep blit 299|Customer#000000299|3F3Q0fTkjIv1UfJbcN7|4|14-948-474-7353|5380.50|HOUSEHOLD|tes sleep fluffily. furiously regular requests boost fluffily evenly even asympt 300|Customer#000000300|I0fJfo60DRqQ|7|17-165-193-5964|8084.92|AUTOMOBILE|p fluffily among the slyly express grouches. furiously express instruct 301|Customer#000000301|FtFq9Cgg5UAzUL|7|17-265-345-9265|9305.05|HOUSEHOLD|ular, regular notornis sleep along the furiously pending foxes 302|Customer#000000302|cJ3cHoAjAiaxTU2t87EJM|4|14-152-594-2967|1107.42|MACHINERY|dolphins haggle fluffily across the final requests. regularly unusual sentiments detect fluffily requests. regular 303|Customer#000000303|5pSw0OIoNRcpyTEEI1gZ6zRMyJ0UGhJdD|3|13-184-254-6407|9339.57|AUTOMOBILE|mise ironically against the unusual foxes. deposits cajole asymptotes. ironic ideas shall have to sleep 304|Customer#000000304|Cilvb3k8ghDX4|0|10-321-698-7663|9217.55|MACHINERY|s integrate at the carefully ironic instructions. fin 305|Customer#000000305|x8kcl,R4Wk|11|21-250-654-3339|4356.59|FURNITURE|nts. even, regular courts nag. dugouts use blithely a 306|Customer#000000306|ADoOEIr5aQcLIoGJM6nCvPEP 91|10|20-109-305-9629|3268.01|AUTOMOBILE|ill have to are. final, express deposits hag 307|Customer#000000307|xvkJ13gs7GH|13|23-836-934-5394|346.59|FURNITURE| ironic platelets nag against the bold pinto 308|Customer#000000308|c9WuNBiEYmGxeBmZaELg WWb|9|19-992-128-2013|4150.76|HOUSEHOLD|ilent accounts haggle carefully unusual dolphins. carefully regular requests wake along the 309|Customer#000000309|6Jg4ECVS2u7i,E|21|31-231-377-9535|8824.78|FURNITURE|lyly. furiously enticing instructions haggle. carefull 310|Customer#000000310|QZnc5mkLIPh6JGrzcHmRzCiL0AmdE92vyM|1|11-838-647-9285|3186.57|FURNITURE|mise fluffily blithely ironic courts 311|Customer#000000311|dvpNARle3mR19GD4s2gpEbkL2mZV3uvV6P|23|33-919-292-8822|6589.50|AUTOMOBILE|essly even escapades. blithely regular Tiresias cajole blithely furiously close packages. furiously ironic pi 312|Customer#000000312|cH6XucXV0V|6|16-316-482-2555|-178.84|AUTOMOBILE|e slyly. furiously regular pinto beans wake slyly according to the fluffily even excuses. ca 313|Customer#000000313|Ay52vCrTXsSmp7TmQ1kujvuItfLGx|0|10-401-786-6040|6115.81|HOUSEHOLD|g to the even dependencies. accoun 314|Customer#000000314|8,tdTVYGYoYRaAKwG 6aDJna4Cfjt,F9DDCC2|13|23-366-243-4713|2394.92|MACHINERY|ets alongside of the slyly pending pinto bean 315|Customer#000000315|pXaKKTCTyc UI3tglBaWRimosymG6ZyOCyb6Vb3M|7|17-442-286-3594|348.58|FURNITURE|s. slyly regular sentiments are carefully. slyly ironic asymptot 316|Customer#000000316|zE dN3aqjaG|8|18-171-394-5011|4571.78|MACHINERY|egular ideas cajole around the ironic, pending deposits. furiously pending dolphins serve blithely regular 317|Customer#000000317|uOeuL8DG1j|19|29-615-537-8871|956.88|HOUSEHOLD|ages. hockey players are. dependencie 318|Customer#000000318|PtJQn0IjYtShb1f2uYTYBnnmUeGNiwcALU|0|10-229-548-7118|9149.98|HOUSEHOLD|nding requests. special, bold instruction 319|Customer#000000319| UQ5mF3sdoZT2|6|16-734-928-1642|1834.36|FURNITURE| packages use slyly always ironic deposits. unusual, even notornis above 320|Customer#000000320|pO8rWSwK j|12|22-358-857-3698|6082.74|MACHINERY|ing requests. furiously regular accounts hinder slyly. final, regular theodolites against the slyly quiet requests 321|Customer#000000321|g3,8g XHACSvjZtJuiNk5BYiyPFnIxg|20|30-114-675-9153|7718.77|FURNITURE|special requests! express dugouts can affix furiously blithely regular platelets. fu 322|Customer#000000322|bWRyCyjH5OfGX|20|30-660-202-7517|4489.98|HOUSEHOLD|usual sauternes are among the slyly even instructions! thinly regular 323|Customer#000000323|ZLnVZ CXRi2,QDrlo|18|28-347-223-6024|1137.67|AUTOMOBILE|ely special foxes. express, final excuses across the packages are quickly amon 324|Customer#000000324|fiW1n6dLSVRkXj7kU1768UI2w1vMxEde5a |2|12-722-560-7023|806.59|FURNITURE|, regular requests kindle slyly furio 325|Customer#000000325|Z I43vl3ta3iYmjXNaSM d6Pe24ibjhdvPSi|15|25-823-702-9630|2377.34|HOUSEHOLD|nal foxes alongside of the always bold 326|Customer#000000326|nWDOTh6X019pfBtV3ikACYZiMjGykLrFnuyAo2|2|12-447-614-7494|1906.52|HOUSEHOLD|ckey players. carefully ironic a 327|Customer#000000327|UyKulwfNnX4l4ba1vQtwCWw8WNP50U8DCU|8|18-606-718-3062|8762.16|MACHINERY| unusual braids. daringly final ideas are quickly c 328|Customer#000000328|9pu j2HoEf1uhiY3jxE9l9fCRfjoVU|5|15-817-180-1487|6709.90|BUILDING|y about the daring accounts. furiously thin escapades integrate furiously against the furiously ironi 329|Customer#000000329|67r6XnIxUVgAc3pRX8tmGOw|11|21-106-357-8302|-651.91|BUILDING|ans. fluffily unusual instructions haggle about the slyly ironic platelets. never regular pinto beans sleep fl 330|Customer#000000330|UfNb7T9CTCnsfN3b|20|30-476-852-2371|8244.73|MACHINERY|en pinto beans. quickly final excuses haggle furiously. slyly pendin 331|Customer#000000331|Ug e2IBbl,LJuqjNz5XeQV|5|15-411-430-7917|170.27|AUTOMOBILE|r the silent ideas. carefully ironic deposits was carefully above the furiously even excuses. evenly regu 332|Customer#000000332|Jfosq,G6ziag7M04IvCx7SMRafyYvSI,Do|22|32-767-972-2596|-267.09|HOUSEHOLD| around the pinto beans. final theodolites haggle 333|Customer#000000333|heiloGYs Yey7NKhEFoiNhUBb,QFbjtn5wt|11|21-908-534-7709|8018.89|AUTOMOBILE|uriously close theodolites! slyly express foxes cajole-- final pinto beans boost blithely along the ironic 334|Customer#000000334|OPN1N7t4aQ23TnCpc|4|14-947-291-5002|-405.91|BUILDING|fully busily special ideas. carefully final excuses lose slyly carefully express accounts. even, ironic platelets ar 335|Customer#000000335|d2JCYLr2F9tC1AZMIvbIYPDQA|21|31-772-165-3138|6837.46|HOUSEHOLD| requests haggle carefully about the quickly special escapades. regular a 336|Customer#000000336|yC zy1i6AGrnykrV McJyjg|2|12-345-190-9898|9241.49|AUTOMOBILE|es. dependencies lose carefully blithely regular deposits. t 337|Customer#000000337|EluRTlO4pE7u0XSKKyvKvVyt4sADWFRLZuiyn|0|10-337-165-1106|-270.59|MACHINERY|ld requests sleep quickly. carefully express tithes wake carefully ac 338|Customer#000000338| aiYAeWgI0okGSJv7OgvKqMvPLhxF3blT8josX|23|33-302-620-7535|4092.49|FURNITURE|ckages nag blithely regular requests: carefully final packages between the slyly regular instructions sleep 339|Customer#000000339|jUs1Im28boIduGhp5vbKK50gM5ov7xH9G|24|34-992-529-2023|8438.07|HOUSEHOLD|ix. ironic, special tithes detect dog 340|Customer#000000340|WRnPrKQmAmoMQgHQERoVOhyTklcHMajJlc|2|12-730-681-4571|4667.12|BUILDING|es sleep according to the even, unusual Tiresias. carefully bold packages haggle. furiously pending s 341|Customer#000000341|4,zQfld2YV9TSeNgCSOvqlxhJvVW8WD|9|19-870-813-8585|8247.11|FURNITURE|low, special platelets alongside of the even, bold theodolites are carefully 342|Customer#000000342|SpDDdUfraEAfCULAuGLE|18|28-690-119-9571|7186.74|AUTOMOBILE|luffily final ideas. finally unusual requests boost slyly above the furio 343|Customer#000000343|ejvvSNHIkJVm8I1zpQINNn5yyJbA|3|13-877-910-5134|5521.36|HOUSEHOLD| unusual requests cajole blithely about the carefully express ideas. blithely even excuses above the pint 344|Customer#000000344|Zasc8,E0VVY|2|12-810-788-6699|-544.95|FURNITURE|le according to the regular instruction 345|Customer#000000345|dGFK ICPKxnsAzlX4UYOUf,n200yyEWhIeG|9|19-209-576-4513|1936.77|AUTOMOBILE|en pinto beans nag along the slyly regular deposits. slyly ir 346|Customer#000000346|K61SvIue3Emcwfel,7f9tO5WyJ58MbT7k3iS|2|12-100-890-4659|238.14|FURNITURE|ickly even pinto beans affix across the bravel 347|Customer#000000347|qRT7WRrnykLDfTc5Ei|1|11-519-832-9913|7348.92|BUILDING|ts use blithely blithely regular theodolites. even requests after the 348|Customer#000000348|ciP7BWkhOe1IbbVGlqJePBI6ZwqENkS|13|23-986-141-5327|3310.49|HOUSEHOLD|al foxes are on the carefully final excuses. careful dependen 349|Customer#000000349|vjJBjxjW9uoRZP02nS p6XY5wU6Ic,6xHpxUKA|23|33-818-229-3473|-565.35|BUILDING|y. bold, ironic instructions after the theodolites sleep blithely ironic packages. ideas c 350|Customer#000000350|G vBMGVmIOHl7tc4HeNMiMkKY|15|25-960-809-3690|19.31|BUILDING|tions. quietly unusual accounts sleep blithely afte 351|Customer#000000351|De35Hx1QiyS0uy|7|17-873-420-4342|3419.54|AUTOMOBILE|telets haggle blithely against the ironic 352|Customer#000000352|HqhIE5GRTK0dFtWpJUQENU4aa1bwdsUBEWtzUw|9|19-906-158-8420|6257.88|HOUSEHOLD|ts are. blithely special requests wake. furiously bold packages among the blithely eve 353|Customer#000000353|eftGCmL4b5rAKdvUe9biJXzAH|10|20-733-644-2244|3199.03|BUILDING|nal theodolites nag carefully. requests wake. slyly ironic ideas according to the blithely pe 354|Customer#000000354|sV3WgvJA06WngO4|2|12-545-101-2447|7095.95|BUILDING|. regular, final requests cajole fluffily. express attainments wake slyly until the even acco 355|Customer#000000355|205r3Xg9ZWjPZNX1z|14|24-656-787-6091|8727.90|FURNITURE|ly bold requests detect furiously. unusual instructions sleep aft 356|Customer#000000356|9RfNXUJivKTonL2bp1eG5IT|10|20-415-457-4421|2934.06|FURNITURE|al packages haggle always. daringly bold inst 357|Customer#000000357|l2C0Xkdib4t4 qKFUcRDOhRQMK7U0|18|28-452-965-8560|8747.36|AUTOMOBILE|ress platelets cajole fluffily final accounts: slyly ironic foxes s 358|Customer#000000358|F z jplpUKWz1Hn7p3ez2qTsiIh|5|15-457-255-3822|-44.66|MACHINERY|e furiously pending requests. slyly bold requests wake deposits. furiously express 359|Customer#000000359|z4lUH9ssc3K2w0UjRIuNRrdqw|14|24-608-547-4751|6375.23|FURNITURE|ifts wake fluffily ironic ideas. slyly ironic deposits above the 360|Customer#000000360|S,6ajyDFO3WUQ0Qr|17|27-604-646-1645|6542.83|FURNITURE|engage. quickly final platelets about the fluffily unusual accounts wake 361|Customer#000000361|l0F8jMJVe63cb|20|30-164-267-4590|7451.84|BUILDING|fully busy ideas. regular foxes cajole 362|Customer#000000362|UscV00TNrNTDddxF7BTk|17|27-651-653-4122|6149.01|AUTOMOBILE|ut the fluffily ironic platelets. ironi 363|Customer#000000363|2Koh mYARhsVcFn0U2Abt35qIyedAr1TxP|17|27-460-529-3937|-573.86|HOUSEHOLD|s. carefully unusual deposits are foxes. furiously even foxes nag carefully according to the furiously express 364|Customer#000000364|SQ3b5Q5OtrmmZjJ87tq,o1TiXKVJQ0M7ZOuud|23|33-492-647-4972|32.24|HOUSEHOLD| dependencies? pending requests use carefull 365|Customer#000000365|QiZRz y1xU|24|34-708-696-5226|737.03|HOUSEHOLD|counts. unusual packages are blithely foxes. unusual dinos 366|Customer#000000366|pPQektSfn55AC7s9SRFkj07I2yXqakvCa|3|13-915-531-6826|-729.74|MACHINERY|nos wake quickly. regular, regula 367|Customer#000000367|yZaDoEZCqt2VMTVKoZUkf6gJ4yj|13|23-939-319-4691|9108.65|HOUSEHOLD|eodolites under the ironic, stealthy requests affix furiously among the unusual tit 368|Customer#000000368|9p ReFA4fseKWYUaUHi|22|32-552-596-4994|84.72|MACHINERY|ic asymptotes. quickly special packages along the bravely bold depos 369|Customer#000000369|ge1XhgI3ADIkvLr5GPMqpup,hzlTVv|8|18-333-644-9832|2881.06|FURNITURE| theodolites? quickly quick foxes are fluffily slyly regular instructions. fluffily 370|Customer#000000370|oyAPndV IN|12|22-524-280-8721|8982.79|FURNITURE|ges. final packages haggle quickly. slyly bold 371|Customer#000000371|dnxjCYwhuSHx 9KX38nV0R16fG|22|32-119-346-2028|7789.14|AUTOMOBILE|equests shall boost furiously special pinto beans. express, ironic ideas sleep across the ironi 372|Customer#000000372|aKPMNZfbgV0neVIBo|19|29-226-339-6392|-921.91|MACHINERY|. furiously even foxes sleep at the forges. bold accounts sleep after the ironic theodolites. ironi 373|Customer#000000373|2hrQ wHkbaNlJCzY,mVkugMIE 8ryNlaA3JHDTjJ|20|30-883-170-4010|2354.06|MACHINERY|requests wake blithely even packages. slyly ironic deposits haggle blithely 374|Customer#000000374|fg4eklU1,UaFOan|22|32-282-723-3627|6718.78|AUTOMOBILE|ges are carefully. slyly ironic deposits about the fin 375|Customer#000000375|e53JADEeGvM1ikhN7aa|15|25-575-273-9756|5562.22|HOUSEHOLD|st the pending accounts. final courts above the pending pinto beans use furiously ironic requests. dolphins 376|Customer#000000376|4NwsvFQU T4mSgzvU1Rx2ZtHOGyaNyhe|16|26-437-952-8986|4231.45|AUTOMOBILE|gs cajole quickly. bold asymptotes wake regularly along the quickly 377|Customer#000000377|PA4levhyD,Rvr0JHQ4QNOqJ9gW YXE|23|33-260-610-4079|1043.72|MACHINERY|. slyly regular ideas cajole blithely. slyly ironic foxes are carefully after the thinly special accou 378|Customer#000000378|133stqM9 LT,a2BSlbm49 nXreFggaZgW6P6J|22|32-147-793-4825|5718.05|BUILDING|ackages haggle fluffily ironic packages. 379|Customer#000000379|t3QzCf,q1NbshmjOIUY|7|17-228-550-9246|5348.11|AUTOMOBILE|l deposits cajole blithely blithely final deposits. express, even foxes grow carefully about the sile 380|Customer#000000380|n2w3Jd1bipwICbOVgrELzcNRexmWSklo|21|31-538-493-4229|2755.46|BUILDING|riously special accounts. slyly final accounts sleep; blithely special requests integrate carefully slyly en 381|Customer#000000381|w3zVseYDbjBbzLld|5|15-860-208-7093|9931.71|BUILDING|t regular, bold accounts. carefully quick packages haggle. care 382|Customer#000000382|UdgAMamK5JnSykA,ZPfR5W5zRFatDUye|8|18-942-650-6657|6269.42|AUTOMOBILE|. blithely express notornis according to the blithely even requests are never fina 383|Customer#000000383|iBIHYgXvVDpu6qq7FlqXVcAIDAzv4qs|2|12-868-920-9034|-849.44|MACHINERY|slyly express ideas haggle blithely unusual dugouts. ironic pinto beans are ironic ideas. 384|Customer#000000384|kDDMb3ML nUSu2Sn7CUHyZVedAFUx9|9|19-271-453-8361|-614.30|HOUSEHOLD|olites. express, unusual dolphins cajole carefully about the 385|Customer#000000385|zJvPI24TSPpiFzYfu3RvTKQ9|3|13-741-675-6890|2457.09|AUTOMOBILE|rs. blithely ironic deposits nag furiously across the furiously ironic accounts. bold deposits sleep express 386|Customer#000000386|DeQxsCxixT8RQ7JV6mddRYGDGQ2WM94|24|34-193-143-1425|232.01|BUILDING|counts. blithely permanent deposits wake slyly! unusual, even theodolites u 387|Customer#000000387|Yj 9g1mNu00rKRkc1ovOmptsPI|18|28-694-363-3673|3404.23|HOUSEHOLD|oach. blithely regular instructions sublate across the quickly regular ideas. qui 388|Customer#000000388|dV4lqEufXkF8R|7|17-856-814-6352|1938.05|HOUSEHOLD| carefully bold deposits: final pinto beans sleep slyly idl 389|Customer#000000389|ij8KNM0,HRvIvnvY w8jQK4zvr1EOO9YM|9|19-264-943-1253|-307.61|AUTOMOBILE|o beans affix fluffily. slyly ironic notornis wake 390|Customer#000000390|Nsc3VZZnVsw0mLAnqqzVz,|4|14-812-253-6693|8862.18|HOUSEHOLD| final packages promise quickly. pending theodolites haggle quickly above the doggedly ironic 391|Customer#000000391|q10SV05KB1038lzUR8P|11|21-604-451-4462|4801.30|HOUSEHOLD|le blithely final forges. furiously even deposits cajole fluffily even patterns. furious 392|Customer#000000392|H7M6JObndO|17|27-601-793-2507|8492.33|BUILDING|efully bold ideas. bold requests sleep carefully blithe instructions. carefully final accounts are blithely quickly 393|Customer#000000393|RSELskV44I3LFA9VLGY2Qe|20|30-749-949-5915|3593.57|FURNITURE|ake furiously express notornis. pending accounts hang slyly slyly blithe theod 394|Customer#000000394|nxW1jt,MQvImdr z72gAt1bslnfEipCh,bKZN|23|33-422-600-6936|5200.96|MACHINERY| instructions. carefully special ideas after the fluffily unusual r 395|Customer#000000395|b06rg6Cl5W|15|25-804-388-6600|4582.28|HOUSEHOLD|s mold blithely regular platelets. slyly silent instructions use slowly slyly specia 396|Customer#000000396|miE7JrCdGpQkF4zYJ27tBdSu IYhQ HXx0 |22|32-902-936-4845|1433.50|BUILDING|xcuses. regular pains wake slyly across the ruthlessly ironic dependencies. e 397|Customer#000000397|EzR2BKJ85SmBDS|7|17-103-357-8777|709.46|FURNITURE|al theodolites. regular accounts are regular, silent foxes. unusual asymptotes above t 398|Customer#000000398|cq9NmtIT4b6JB8L79iLzljlHs4 3|15|25-110-215-3747|8865.61|HOUSEHOLD|l deposits breach slyly ironic asymptotes. carefully pend 399|Customer#000000399|ZBvzMa6N1wdCGaPmG13xVusIxdjSiA94jTXN|8|18-882-664-5454|7358.53|BUILDING|yly even excuses. ironic theodolites wake furiously. blithely regular pinto beans cajole. fin 400|Customer#000000400|U23zy17EPxqmJn7neVc|14|24-522-746-1247|-98.46|BUILDING|fully bold accounts cajole bravel 401|Customer#000000401|aKALIG526OK4veQfUh2KmKcE,oRyg|19|29-667-766-5291|4146.43|BUILDING|l instructions wake. slyly express deposits us 402|Customer#000000402|8Cw4p1m1gKYVUgomkAq,es1ZtrnmHaO|6|16-950-729-1638|2106.67|AUTOMOBILE|dolites. furiously regular theodolites integrate furiously. bravely bold requests are. furiously 403|Customer#000000403|9,BVYegfkFLsEMDkeVW|14|24-753-433-1769|6693.36|HOUSEHOLD|al hockey players; ironic dependencies after t 404|Customer#000000404|2orgvLJ05jOvM292mhkS7iJmHG0jk|22|32-840-785-1776|7408.73|BUILDING|uickly brave requests haggle furiously carefully special idea 405|Customer#000000405|mCQNH1rJtqjjQ9Piauc2bZr4pRFydscZtbD9d|10|20-509-301-7901|7519.14|MACHINERY|nts. pending, express foxes sleep? ironic, pending instructions haggle. ironic, pending theodolites detect slyly. bl 406|Customer#000000406|j1fOG9WsIr2JI6Yi9jgJ M|9|19-426-693-4043|4286.94|FURNITURE|nal foxes. unusual pinto beans wake. special excuses cajole ironic 407|Customer#000000407|cfCP9bE3HnI|1|11-975-454-8499|9537.08|MACHINERY|ect among the carefully regular theodolites. regular dep 408|Customer#000000408|TBjb3m,3aea4JtP833HD4VDk7STz2Y9FB|10|20-177-807-5661|6825.37|BUILDING|unts. furiously ironic depths among the instructions wake carefully along the blithely ironi 409|Customer#000000409|mtrMiDvQxNsy1Cj0cU4ITEW5wGKLPQ2IPHNE9r4|11|21-466-412-4731|3969.86|FURNITURE|fily pending courts. express, regular packages are furiously along the quickly regular packages. 410|Customer#000000410|nYak2u Q9,gYUiLfh1N|7|17-576-345-5940|4349.27|BUILDING| sublate across the pending, express asymptotes. quickly 411|Customer#000000411|V3e,FX5x50scsQDzt5,ESxfOQBt4OzjHRoTZxF|18|28-483-924-1955|1209.32|HOUSEHOLD|refully. slyly even packages above the evenly regular asymptotes are blithely ironic dependencies. deposi 412|Customer#000000412|5IN2Y,QrhDJ2YBVGKiDbMpzi2hk1fmozIy2zQ|22|32-940-318-3191|6044.02|BUILDING|ithely silent notornis haggle. regular requests haggle according to the ironic deposits. blithely final dep 413|Customer#000000413|,4Jm5N0ruhJCB7cBR6Kw|6|16-158-285-7336|5817.90|FURNITURE|ular packages integrate furiously fluffily final accounts. carefully regular 414|Customer#000000414|i49DWI61AFb 45vb1RMH|19|29-552-380-2475|527.78|AUTOMOBILE|sily silent, even accounts. careful, final ideas boost fluffily. slyly final pinto be 415|Customer#000000415|334jCRiUb,gx3|23|33-346-876-2972|2317.93|FURNITURE|egular deposits. blithely ironic inst 416|Customer#000000416|fm7H7k6sYhKfXttOT|12|22-651-146-4780|4365.28|MACHINERY|p the pending pinto beans. furiously express reques 417|Customer#000000417|X3LMSpIn4FgjgJxldHVUlUvKzyX|11|21-794-364-5100|6187.73|BUILDING|lent multipliers. quickly express theodolites kindle blithely. ironic re 418|Customer#000000418|,e0q82drO rgVHXHrJRQ0GDrRoUOl|5|15-826-508-1218|1211.39|FURNITURE|d foxes against the furiously special packages snooze blithely quickly 419|Customer#000000419|gvbZNJ4UVBAo5yOZ2UOWcvV9TeTj|16|26-338-447-2399|7786.69|BUILDING|ideas affix alongside of the final accounts. quickly ironic deposits abo 420|Customer#000000420|HV0YB82MWw93 9K|20|30-776-366-5869|1999.35|BUILDING|ideas wake. fluffily ironic packages hang furiously above the regular, even platelets; packages haggle slyly 421|Customer#000000421|it3mUlkZAe9J8gmy|13|23-918-228-2560|7073.17|FURNITURE|lithely final deposits haggle furiously above the 422|Customer#000000422|AyNzZBvmIDo42JtjP9xzaK3pnvkh Qc0o08ssnvq|9|19-299-247-2444|-272.14|HOUSEHOLD|eposits; furiously ironic packages accordi 423|Customer#000000423|Y2B EbOg39GpFLS0n|13|23-201-501-7824|95.79|BUILDING|ts cajole after the silent, pending instructions. ironic, even asymptotes use carefully. furi 424|Customer#000000424|i4cf3kmRE9IJr,cu,1|19|29-891-311-6778|1866.42|HOUSEHOLD|bove the express, final deposits wake furiously furiou 425|Customer#000000425|lp3aCRBK11qFY|16|26-756-407-4828|5824.88|HOUSEHOLD|ajole even, pending accounts. carefully brave accounts 426|Customer#000000426|GjFjM4zjbyhNrV6XlE|19|29-768-330-6311|7818.25|HOUSEHOLD|ar instructions are against the ironic platelets. slyly final acc 427|Customer#000000427|LHzXa71U2AGqfbqj1yYYqw2MEXq99dWmY|2|12-124-309-3821|4376.80|MACHINERY|y even multipliers according to the regu 428|Customer#000000428|TCVjlzbX7x,kWcHN33LRdEjO38mAGmPR|21|31-587-557-8211|1952.36|BUILDING|furiously quick accounts. slyly bold dependencies cajole carefully. quickly even requests int 429|Customer#000000429|kZBtY,LQAFu4iaSagjfIk8Q8dzgmT|15|25-989-936-1954|9247.21|FURNITURE|ly regular requests haggle enticing excuses. carefully ironic requests on th 430|Customer#000000430|s2yfPEGGOqHfgkVSs5Rs6 qh,SuVmR|3|13-406-611-4228|7905.17|BUILDING|ly slyly ironic attainments. slyly special instructions until the deposits nag quickly whithout the bo 431|Customer#000000431|RNfSXbUJkgUlBBPn|6|16-326-904-6643|2273.50|HOUSEHOLD|e quickly. final, even excuses against the even accounts sleep agai 432|Customer#000000432|FDConiq g20GI9dH QTM ZNX4OB9KU|23|33-307-912-9016|5715.64|BUILDING|wake carefully close, special deposits. regu 433|Customer#000000433|7XFuE4 euQR0w|20|30-659-445-3595|8746.23|FURNITURE|sual ideas affix carefully always regular accou 434|Customer#000000434|6LGAf2hv4MB5MJhfvNsg|3|13-325-443-1474|2940.46|MACHINERY|lly final Tiresias. blithely regular ideas nag stealthily about the furiously 435|Customer#000000435|diwjNQSb3wLYLy WfCDATo5rc1I3 s|2|12-741-309-6377|6217.46|MACHINERY|quickly excuses. blithely express theodolites poach slyly along the theodolites. slyly reg 436|Customer#000000436|4DCNzAT842cVYTcaUS94kR0QXHSRM5oco0D6Z|19|29-927-687-6390|5896.87|FURNITURE|olites engage carefully. slyly ironic asymptotes about the ironi 437|Customer#000000437|0PM1xuHd0q2ElcJp 77F2MykOVBSQnZR8u3jkn|4|14-364-492-8498|7760.52|AUTOMOBILE| foxes sleep across the slyly unusual pack 438|Customer#000000438|eqo9A9oaE2CA7 7,L|23|33-394-388-4375|2131.13|MACHINERY|al deposits mold alongside of the fluffily brave requests. 439|Customer#000000439|3deBblz2syRv8yMf0yAVKkE4mDH20uDRj4tJVHUm|14|24-873-368-6801|-61.29|BUILDING|ions may impress thinly for the deposits? even packages towa 440|Customer#000000440|w4fKMgiBuGmV,nLn7NgJl1DoUWwNQMV8z 5,R|3|13-244-480-5751|1809.04|MACHINERY| even theodolites: fluffily final requests cajole about the quickly regular 441|Customer#000000441|gjYpcBx6MP8GvDa6|23|33-438-355-3491|9451.84|HOUSEHOLD|r requests wake theodolites. quickly final ideas haggle fluffily. blithely f 442|Customer#000000442|rvgayfJFLO2cjzMA|1|11-240-523-8711|4157.00|FURNITURE|lets would affix fluffily. regular, regular ideas ought to haggle carefully blit 443|Customer#000000443|UdyNGZ6GSz5aNpMO5N2|3|13-241-131-1632|3726.22|FURNITURE|t the special, final platelets. bold req 444|Customer#000000444|D8l4G8i9aZ7KRbqp6ajvR8h1wjr|1|11-402-300-1949|1505.27|HOUSEHOLD| express accounts along the pending deposits lose carefully above the furiously regular requests. pen 445|Customer#000000445|MX1UA0KUJzIGyWM p2hbLg5dCpVLws8KNcwEsP|20|30-849-846-6070|8018.81|FURNITURE|e ironic, special accounts. quickly regular packages integrate fluffily slyly 446|Customer#000000446|mJOJwYfch izLCuw70,qhlJSmH|24|34-321-168-5681|9225.60|FURNITURE|ending instructions. boldly ironic foxes across the regularly ironic pains sleep along the carefully final deposits. 447|Customer#000000447|hVZBzP8Pii|3|13-438-344-7007|7665.98|HOUSEHOLD|telets around the furiously unusual foxes detect carefully against the 448|Customer#000000448|BH4vtnDpabk0NgoGNJWu4OUXnidfJ|24|34-985-422-6009|8117.27|BUILDING|unts. final pinto beans boost carefully. furiously even foxes according to the express, regular pa 449|Customer#000000449|DiUXazp8EYcJFsX2a7nciEpo9W5BRB4iqdb9HWL|4|14-893-381-6454|3001.94|MACHINERY|posits boost slyly carefully regular requests. final, bold fo 450|Customer#000000450|KVpuYa4dDW8lZZVBttyK614C2qdS|9|19-782-397-9006|5544.42|HOUSEHOLD|gular decoys nod slyly express requests. slyly bold theodolites are along the regular 451|Customer#000000451|ZJKTC1Ck,B01fYZ xdN2|20|30-939-275-3248|2110.59|HOUSEHOLD|quests grow furiously final deposits. ironic, even pi 452|Customer#000000452|,TI7FdTc gCXUMi09qD|6|16-335-974-9174|6633.70|BUILDING|aggle quickly. unusual instructions i 453|Customer#000000453|PZ4mmWL7R,El0MtLWMfLXp120lo0,itmO|8|18-209-381-8571|5678.18|HOUSEHOLD|sts. slyly even dolphins across the bold, regular foxes haggle blithely 454|Customer#000000454|d9oQCm3onNsFlIoteVjFcQDv|7|17-818-915-9400|6134.40|AUTOMOBILE|ions print slyly platelets. carefully regular packages according to the fluffy, even foxes wake carefu 455|Customer#000000455|sssuscPJ,ZYQ8viO|6|16-863-225-9454|6860.34|BUILDING|l wake. blithely final instructions integrate furiously above the final, regular req 456|Customer#000000456|IgUSuulguDJ5|0|10-784-971-7777|8815.78|FURNITURE|ly even warhorses. quickly even requests wake slyly. 457|Customer#000000457|eaAWe Vqr0x17Uwj1uzQRb wQpXxZVDWS3Wg|20|30-543-684-2857|5867.61|FURNITURE|the foxes. carefully pending instructions integrate fluffily blithely pending packages. careful 458|Customer#000000458|iIKwI3HrgNlD9|4|14-651-706-4016|-38.42|BUILDING|ng. final, express requests are furious 459|Customer#000000459|CkGH34iK 9vAHXeY7 wAQIzJa1cmA8DAEA7m|6|16-927-662-8584|1207.97|MACHINERY|ronic, regular dependencies use above the ironic deposits. carefully express packages use car 460|Customer#000000460|Gbx5Hnw,ctlI7|11|21-643-955-6555|5222.83|FURNITURE|old dependencies mold slyly above the foxes. dogged, express ins 461|Customer#000000461|5vxNLzSASzkbrUr8CRf5|21|31-533-226-4307|9177.63|AUTOMOBILE|sits breach blithely. slyly regular ideas haggle fluffily; special ideas cajole q 462|Customer#000000462|MSqsCvNEkowp7FnscRXP6OUWm|21|31-157-561-4106|4522.60|HOUSEHOLD|ly special accounts? ideas engage regular dependencies. fluffily even pinto beans x-ray blith 463|Customer#000000463|LV7MN7Tkm2NSo4Q3lwvjxGQyRJjRZRf,M|8|18-167-214-5805|-654.50|HOUSEHOLD| quickly along the final ideas. slyly regular accounts are iro 464|Customer#000000464|kAALP9gEt3,G9XtxCXjv38HjKBEP|9|19-269-971-9738|8730.85|AUTOMOBILE|efully express accounts play. special requests use carefully. regular courts sle 465|Customer#000000465|gngnTNn7azjgQlQJnakTZto|2|12-137-838-1346|8432.74|FURNITURE|es. quick asymptotes integrate carefully alongside of the ideas. even requests believe slyly even ac 466|Customer#000000466|ZI1c8,ZanegEu5CEQxNf5,bkuYPwn7H7JIK7|12|22-280-738-3240|3168.41|MACHINERY|foxes. express, ironic accounts boost? carefully silent deposits engage. accou 467|Customer#000000467|amwRkh0nDQ6r6MU|11|21-449-581-5158|9398.51|MACHINERY|manently special warthogs. final ideas a 468|Customer#000000468|IcbihAtOVWcnswfyE|10|20-489-960-5023|9834.19|FURNITURE| accounts cajole quickly above the blithely final packages. even, express package 469|Customer#000000469|JWOULMa5Qtt|12|22-406-988-6460|6343.64|BUILDING|cajole carefully slyly regular packages. 470|Customer#000000470|v9 gWSuP4WrOjNJRgyJtjbNCChQME|20|30-507-458-4433|3597.53|HOUSEHOLD|ilent excuses. never ironic requests sleep furiously. daringly f 471|Customer#000000471|tGr0DtrK 91IgzfeZrSPpPIia3|4|14-574-118-1005|5716.90|FURNITURE|es. unusual accounts try to solve ca 472|Customer#000000472|hWgfnsmTAEOx9Mqp87YwztGrgLLqNkjMPh4|12|22-940-478-1933|7929.90|MACHINERY|deas sleep slyly blithely final foxes. slyly final e 473|Customer#000000473|zO3W9pYj PvlsQGe|9|19-209-647-5704|-202.22|HOUSEHOLD|ter the quickly pending requests sleep above the carefully iron 474|Customer#000000474|mvEKw,6zT0V8Yb2yTG hu990UX|21|31-247-536-6143|9165.47|MACHINERY|ns integrate against the quickly special courts. slyly 475|Customer#000000475|JJMbj6myLUzMlbUmg63hNtFv4pWL8nq|14|24-485-422-9361|9043.55|BUILDING|egular requests. ironic requests detect furiously; deposits ha 476|Customer#000000476|68r87HCBbQkVYaVfes8mgKs|2|12-996-628-9902|5973.10|BUILDING|sly. carefully quick instructions sleep carefully deposits. final, pending pinto beans use closely fluffily final in 477|Customer#000000477|5aW5WHphNgFdIS1Qdp2cIJXG8ER8|23|33-845-877-6997|1836.61|AUTOMOBILE|totes are blithely among the furiously final foxes. slyly 478|Customer#000000478|clyq458DIkXXt4qLyHlbe,n JueoniF|1|11-655-291-2694|-210.40|BUILDING|o the foxes. ironic requests sleep. c 479|Customer#000000479|RdIiG8NbwYtamReRwhR|18|28-336-406-1631|3653.64|AUTOMOBILE|ages. bravely even foxes detect careful 480|Customer#000000480|XyQSPswCeO WPD37K3 mYZ4hnCMJO5p|7|17-231-147-5851|2750.71|FURNITURE|posits. slyly ironic theodolites nag carefully about the quickly final accounts. s 481|Customer#000000481|o4xa7J20NqHM8E0ykH,NKe1gPz04OqIn|21|31-363-392-6461|7157.21|FURNITURE|s can nag slyly instructions. regular, regular asymptotes haggle sly 482|Customer#000000482|389RgNCsmVUKiRskmrQQm90xx JiIxOM0|13|23-732-448-1610|4333.37|HOUSEHOLD|carefully bold instructions. carefully final instructions wake carefully accounts. accounts cajole slyly ironic acc 483|Customer#000000483|Yv1QV 1JsV 9sVbNufRvdnprt0grx52|11|21-799-189-1135|8877.20|MACHINERY|pecial ideas. furiously final i 484|Customer#000000484|ismzlUzrqRMRGWmCEUUjkBsi|20|30-777-953-8902|4245.00|BUILDING|y against the express, even packages. blithely pending pearls haggle furiously above the fur 485|Customer#000000485|XeFbvXCQ,J|19|29-434-961-1623|8695.45|MACHINERY|ecial pinto beans. instructions ought to cajole even 486|Customer#000000486|2cXXa6MSx9CGU|21|31-787-534-8723|7487.40|AUTOMOBILE|nstructions. unusual, special pinto beans sleep about the slyly pending requests. fu 487|Customer#000000487|oTc,l9dAf8O0qOOMP4P0WFTuGS|2|12-111-401-4259|9749.37|AUTOMOBILE|as. excuses use carefully carefully pending i 488|Customer#000000488|bBcMjFPTysSTaTdHcoO|3|13-513-778-1881|-275.58|AUTOMOBILE|thely above the carefully ironic accounts. excuse 489|Customer#000000489|GIdW4IVgeqWMBXnNFZGHS8kmhw|4|14-916-241-6195|8255.83|AUTOMOBILE|lar accounts. finally pending dependencies solve fluffily 490|Customer#000000490| 66fG3Fyb946cVQsH9Z3VMNzR,yfHMKIEB|22|32-268-147-7824|-213.85|FURNITURE|ash carefully never bold instructions. regular, bold asymptotes cajole regularly. quickly bold foxes wak 491|Customer#000000491|r3zPOuenxHl0oqInxWlEyLP1ZH|0|10-856-259-7548|785.37|AUTOMOBILE| slyly special requests hang dogged, express epitaphs. 492|Customer#000000492|JexAgMLuUHoElYFaKx,hJcAP1b1GknYoYHQLyx|8|18-686-244-1077|8635.18|AUTOMOBILE|gle furiously furiously final packages. carefully bold pinto beans promise quickly alongside of the close 493|Customer#000000493|G dRBjxmBBug1 xRSa6VwRchFDtU5b|16|26-514-558-7246|6582.04|MACHINERY|er the furiously express excuses use above the regular accounts. regular instructions after the 494|Customer#000000494|GKgTjHFlQrDZWcketSqhZCopBhmChknI|10|20-330-453-6579|6295.47|FURNITURE|al courts. regular, ironic requests serve furiously. pending 495|Customer#000000495|QhFbEv6KbQIwfZs 1krt1eACKI31v3iyM|7|17-400-405-6060|7997.81|BUILDING| dependencies. silent accounts cajole quickly furiously pendin 496|Customer#000000496|Y8oYLlHme6Z4fEzkTu|12|22-173-644-7922|8174.82|MACHINERY| quickly bold packages. decoys among the blithely pending accounts lose according to the deposits. 497|Customer#000000497|0 qRRXAxUbo1J KDwDMjFde5fXDwn |23|33-937-724-3506|2191.59|BUILDING|fluffy ideas detect carefully 498|Customer#000000498|1Wnja9i7KAC3HxS5yATK,In8Q6AHcEUr0f5Tp|19|29-210-810-1479|3945.64|BUILDING|yly pending requests according to the slyly special asymptotes sleep carefully against the slyly even pack 499|Customer#000000499|m1hO3VXQVbwTbJ99Hw|14|24-387-817-9149|4293.76|HOUSEHOLD|old sentiments cajole carefully among the blithely unusual requests. final packages nag careful 500|Customer#000000500|fy7qx5fHLhcbFL93duj9|4|14-194-736-4233|3300.82|AUTOMOBILE|s boost furiously. slyly special deposits sleep quickly above the furiously i 501|Customer#000000501|lzkYA5C6wa,wX|13|23-867-672-1331|1909.35|FURNITURE|ual deposits wake. quickly ironic platelets along the careful deposits haggle 502|Customer#000000502|nouAF6kednGsWEhQYyVpSnnPt|11|21-405-590-9919|1378.67|HOUSEHOLD|even asymptotes haggle. final, unusual theodolites haggle. carefully bo 503|Customer#000000503|7xCLYGLCpFU,toJBIPIrJbLIuLok81h IxK ae5Z|20|30-441-755-3094|3213.66|MACHINERY| even deposits haggle. packages i 504|Customer#000000504|2GuRx4pOLEQWU7fJOa, DYiK8IuMsXRLO5D 0|10|20-916-264-7594|0.51|FURNITURE|slyly final theodolites are across the carefully 505|Customer#000000505|MAUkwAyEvg61RlCMomspMs0WzYa,Ns|2|12-530-647-8313|6557.51|HOUSEHOLD|mptotes haggle around the theodolites. furiously bold accounts detect quickly packages. special pinto beans 506|Customer#000000506|dT kFaJww1B|13|23-895-781-8227|1179.85|HOUSEHOLD| idle instructions impress blithely along the carefully unusual notornis. furiously even packages 507|Customer#000000507|QlA0Fc 6e,r67ugESzq|14|24-158-185-4455|5727.00|MACHINERY|nst the furiously even deposits cajole slyly among the furiously ironic requests. blithely unusual depo 508|Customer#000000508|q9Vq9 nTrUvx|18|28-344-250-3166|1685.90|BUILDING|uses dazzle since the carefully regular accounts. patterns around the furiously even accounts wake blithely abov 509|Customer#000000509|LHLR0IKQJHVF1 0UvBNPLq0|4|14-115-338-1002|7885.50|FURNITURE|ily! requests cajole fluffily. slyly regular waters na 510|Customer#000000510|r6f34uxtNID YBuAXpO94BKyqjkM0qmT5n0Rmd9L|5|15-846-260-5139|1572.48|HOUSEHOLD|symptotes. furiously careful re 511|Customer#000000511|lQC9KfW W77IYtJjAgSZguNzxjY rYk3t6lcxfSh|13|23-247-728-9743|4571.31|FURNITURE|he slyly close deposits. special, ironic ideas detect furiously carefull 512|Customer#000000512|e5 kymvjf6Vja7tNsL 3dfiK|2|12-144-416-6035|3937.58|BUILDING|packages are slyly after the slyly express packages. bold d 513|Customer#000000513|sbWV6FIPas6C0puqgnKUI|1|11-861-303-6887|955.37|HOUSEHOLD|press along the quickly regular instructions. regular requests against the carefully ironic s 514|Customer#000000514|0qD6Nwp3tG3QqCq9qvRAzT6N8L|23|33-194-775-6756|5840.97|BUILDING|carefully final ideas. quickly final packages are. requests haggle slyly. blithely pending sauternes lose bl 515|Customer#000000515|oXxHtgXP5pXYTh|15|25-204-592-4731|3225.07|BUILDING|ackages cajole furiously special, ironic deposits. carefully even Tiresias according to 516|Customer#000000516|EJwOQMTQnFwvd8r Y7f9i5POy6ZlNkIYxCL hg8t|6|16-947-309-2690|4768.96|MACHINERY|final requests after the furiously 517|Customer#000000517|mSo5eI8F4E6Kgl63nWtU84vfyQjOBg4y|10|20-475-741-4234|3959.71|FURNITURE|al, ironic foxes. packages wake according to the pending 518|Customer#000000518|EsCrt4chk,3IRIzwMHTu 6VQWrfh|17|27-651-256-7682|9871.66|BUILDING|as. quickly regular requests are carefully above th 519|Customer#000000519|Z6ke6Y9J2pYuPBp7jE|5|15-452-860-5592|9074.45|BUILDING|es. fluffily regular accounts should have to sleep quickly against the carefully ironic foxes. furiously daring 520|Customer#000000520|yaOGc9Ve92Bi4F6e0GcheU2MmEOXJE0zqyDT sEA|3|13-612-111-7765|8315.09|HOUSEHOLD| haggle across the even, bold instructions. final, even ideas might wake blithely against the 521|Customer#000000521|MUEAEA1ZuvRofNY453Ckr4Apqk1GlOe|2|12-539-480-8897|5830.69|MACHINERY|ackages. stealthily even attainments sleep carefull 522|Customer#000000522|gPz4FuAGpjvaU4YB9J,fGSnLBr9scEovGO1KkTx|12|22-771-454-9561|6358.46|BUILDING|instructions. doggedly express requests doze blithely. regular theodolites hagg 523|Customer#000000523|sHeOSgsSnJi6pwYSr0v5ugiGhgnx7ZB|10|20-638-320-5977|-275.73|BUILDING| fluffily deposits. slyly regular instructions sleep e 524|Customer#000000524|bpsO77xiAmjwOxqIgAszRu4Y|9|19-844-888-9800|5706.19|HOUSEHOLD|ending pinto beans unwind slyly. slyly final theodolites above the quickly ironic pinto beans haggle ev 525|Customer#000000525|w0pOG5FhH45aYg7mKtHQhAWQKe|19|29-365-641-8287|3931.68|AUTOMOBILE| blithely bold accounts about the quietl 526|Customer#000000526|0oAVPhh1I4JdrDafVG2Z8|1|11-170-679-3115|705.93|HOUSEHOLD|ctions cajole after the furiously unusual ideas. ironic packages among the instructions are carefully carefully iro 527|Customer#000000527|giJAUjnTtxX,HXIy0adwwvg,uu5Y3RVP|13|23-139-567-9286|4429.81|HOUSEHOLD|ending, ironic instructions. blithely regular deposits about the deposits wake pinto beans. closely silent 528|Customer#000000528|SRYjG5Wgp8ZG8GyDFhRIR5ep8yNs3nrCmYa|15|25-985-381-5453|1802.50|AUTOMOBILE| the slyly even instructions. carefully idle packages sleep about the platelets. bol 529|Customer#000000529|oGKgweC odpyORKPJ9oxTqzzdlYyFOwXm2F97C|15|25-383-240-7326|9647.58|FURNITURE| deposits after the fluffily special foxes integrate carefully blithely dogged dolphins. enticingly bold d 530|Customer#000000530|wG6AC7G6Y0DRuzJiroWCByzbrkqeySQDvRXzH|13|23-614-884-1055|4990.92|BUILDING|uctions cajole blithely across the ironic packages. slyly regular deposits wa 531|Customer#000000531|ceI1iHfAaZ4DVVcm6GU370dAuIEmUW1wxG|19|29-151-567-1296|5342.82|HOUSEHOLD|e the brave, pending accounts. pending pinto beans above the 532|Customer#000000532|xwWO3lWjgVJTZwhnltyH6zj5ddkzgH8RbF|15|25-875-978-2232|1725.68|MACHINERY|usly regular deposits kindle. quickly even depos 533|Customer#000000533|mSt8Gj4JqXXeDScn2CB PIrlnhvqxY,w6Ohku|15|25-525-957-4486|5432.77|HOUSEHOLD|even dolphins boost furiously among the theodo 534|Customer#000000534|3PI4ZATXq8yaHFt,sZOQccGl Fc1TA3Y 2|1|11-137-389-2888|6520.97|AUTOMOBILE|deas. blithely regular foxes use carefully bold accounts-- ruth 535|Customer#000000535|,2Y kklprPasEp6DcthUibs|2|12-787-866-1808|2912.80|BUILDING|even dinos breach. fluffily ironic 536|Customer#000000536|jf8PSOQDvqQj4uF8|12|22-521-348-9030|3342.75|MACHINERY|tes? blithely enticing theodolites wake. braids sleep. sly 537|Customer#000000537|wyXvxD,4jc|10|20-337-488-6765|2614.79|FURNITURE|e carefully blithely pending platelets. furiously final packages dazzle. ironic foxes wake f 538|Customer#000000538|u9jYEMPoKwrH5wXivkSebbxAx1PU|19|29-632-471-2852|-303.95|MACHINERY|uffily special requests nag around the quickly stealthy 539|Customer#000000539|FoGcDu9llpFiB LELF3rdjaiw RQe1S|6|16-166-785-8571|4390.33|HOUSEHOLD|ent instructions. pending patter 540|Customer#000000540|YkaXu3o1X8|16|26-933-117-7482|9195.77|MACHINERY| enticingly express excuses. quickly regular notornis cajole near th 541|Customer#000000541|,Cris88wkHw4Q0XlCLLYVOAJfkxw|0|10-362-308-9442|1295.54|FURNITURE|according to the final platelets. final, busy requests wake blithely across th 542|Customer#000000542|XU2ffxnW3TQasrfF0u2KwKWmMarPyY4q7Q|16|26-674-545-2517|3109.96|BUILDING|r forges! requests alongside of the bold, final deposits 543|Customer#000000543|JvbSKX7RG3xuqiKQ93C|17|27-972-408-3265|6089.13|AUTOMOBILE|l, even theodolites. carefully bold accounts sleep about the sly 544|Customer#000000544|Jv7vcm,oE,HEyxekXKia1V5H1up23|5|15-572-651-1323|4974.68|AUTOMOBILE|bout the packages integrate above the regular instructions. regular ideas hinder s 545|Customer#000000545|AsYw6k,nDUQcMOpEws|10|20-849-123-8918|7505.33|AUTOMOBILE| carefully final deposits. slyly unusual pinto beans may wake bold requests. unusual courts alongside 546|Customer#000000546|GZtBXX3OaqFLbC9JNi1hmF1JFLbmRs9|19|29-936-444-8248|3116.50|MACHINERY|ly fluffy braids. blithely special theodolites use express deposits-- slyly regular attainments 547|Customer#000000547|4h SK3dVkE1tQ0NCh|22|32-696-724-2981|6058.08|BUILDING|y express deposits. slyly ironic deposits nod slyly slyly ironic instructions. carefully quick idea 548|Customer#000000548|98nP31ToAGK tCCkYm7HqBZt0dLjy0JzlMMRCmkj|4|14-787-370-8722|90.45|BUILDING|c pinto beans. quickly even requests haggle against the blithel 549|Customer#000000549|v5uqfeHLiL1IELejUDnagWqP5pKWa9LtoemziGV|24|34-825-998-8579|91.53|BUILDING|n asymptotes grow blithely. blithely fluffy deposits boost furiously. busily fu 550|Customer#000000550|q5 gKwc7PBQOyd,H|17|27-938-997-6262|7270.82|FURNITURE|ully regular deposits. slyly ironic requests wake along the depos 551|Customer#000000551|holp1DkjYzznatSwjG|15|25-209-544-4006|-334.89|MACHINERY|y special ideas. slyly ironic foxes wake. regular packages alongside of the deposit 552|Customer#000000552|EbjtaRaiok7eqbQ5VJi7q|2|12-669-784-2191|1353.24|FURNITURE|ickly final accounts cajole fluffily according to the bold, regular accounts. 553|Customer#000000553|8tTlavJ sT|4|14-454-146-3094|4804.57|BUILDING|ully regular requests are blithely about the express, bold platelets. slyly permanent deposits across the 554|Customer#000000554|RluaguNRAJhYXmn,CWxcOC,Ly7|2|12-938-503-7317|8395.57|HOUSEHOLD|jole along the blithely bold 555|Customer#000000555|chm8jY6TfQ8CEnsvpuL6azNZzkqGcZcO8|15|25-548-367-9974|5486.52|BUILDING|lites are blithely ironic ideas. blithely special pinto beans dazzl 556|Customer#000000556|UMHllVkuyQUQ3aLXCVRxrXatsyd0AL6Xw|1|11-934-412-5846|7944.22|MACHINERY|nt excuses! carefully final requests solve quick 557|Customer#000000557|Nt6FUuDR7v|15|25-390-153-6699|9559.04|BUILDING|furiously pending dolphins use. carefully unusual ideas must have to are carefully. express instructions a 558|Customer#000000558|PB1ZV4kQnRHiC|17|27-866-273-7672|1912.23|HOUSEHOLD|ly final requests. regular requests hag 559|Customer#000000559|A3ACFoVbP,gPe xknVJMWC,wmRxb Nmg fWFS,UP|7|17-395-429-6655|5872.94|AUTOMOBILE|al accounts cajole carefully across the accounts. furiously pending pinto beans across the 560|Customer#000000560|gU5FQf0WM0sxTYQ|19|29-618-467-8489|1469.59|BUILDING|sly pending packages boost slyly-- fluffily ironic ideas bel 561|Customer#000000561|Z1kPCTbeTqGfdly2Ab9KEdE,jIKW|18|28-286-185-3047|2323.45|FURNITURE|across the furiously ironic theodolites. final requests cajole. slowly unusual foxes haggle carefully 562|Customer#000000562|04xjB,zuffnhVyEY0 PeiJPtdjh 0ji|15|25-271-465-6971|9234.50|BUILDING|accounts. ideas cajole. quick 563|Customer#000000563|2RSC1g7cVd,j23HusdkhdCGmiiE|12|22-544-152-1215|3231.71|FURNITURE| pinto beans believe fluffily. excuses wake blithely silent requests. b 564|Customer#000000564|qPQOo94iVl|4|14-865-332-8571|6307.59|MACHINERY|onic patterns about the furiously pending 565|Customer#000000565|HCBXAou,1eP6Z3IynHFI7XmEBgu27Sx|4|14-798-211-2891|2688.88|FURNITURE|e. carefully bold deposits sleep regu 566|Customer#000000566|5NmdMIwTpF8tj7O92363ycA6EL5Yh,vW|24|34-443-780-3708|1928.10|FURNITURE|ke express, ironic requests. regularly even sauternes detect de 567|Customer#000000567|KNE6mpW69IgTjVN|21|31-389-883-3371|8475.17|BUILDING|blithe, even ideas. fluffily special requests wake. c 568|Customer#000000568|ZddVCnzeABTTBgV3GvkvNtw9,KOHHpME2GELhz|13|23-603-795-8611|1317.56|BUILDING|gular decoys haggle slyly. blithely special packages slee 569|Customer#000000569|Kk20Q5HiysjcPpMlL6pNUZXXuE|2|12-648-567-6776|-795.23|MACHINERY|sh. blithely special excuses sleep. blithely ironic accounts slee 570|Customer#000000570|0Zo0P6m,sie 1,VXacPX2ccDIyWFolj6R|15|25-264-442-3057|8480.87|AUTOMOBILE|gular instructions unwind bold escapades. special asymptotes snooze according to the 571|Customer#000000571|hCrDDrMzGhsa6,5K4rGXQ|2|12-115-414-4819|8993.23|HOUSEHOLD|le fluffily. ironic, pending accounts poach quickly iron 572|Customer#000000572|Nf4Yqb49BqGkzmmtf6|11|21-425-209-5033|7252.65|AUTOMOBILE|leep. pending requests affix blithely. ironic theodol 573|Customer#000000573|BEluH7it7jUcWqb tNLbMIKjU9hrnL7K|4|14-354-826-9743|2333.96|HOUSEHOLD|as. furiously even packages sleep quickly final excu 574|Customer#000000574|ratVLdmp070|8|18-676-218-1058|9787.56|FURNITURE|al pinto beans. carefully ironic foxes cajole idly finally express theodolites. fluff 575|Customer#000000575|4K6h0pYH,bg2FS5cYL,qqejhvp7EfTlBjRjeVPkq|1|11-980-134-7627|3652.29|BUILDING| final requests cajole after the ironic, bold instructio 576|Customer#000000576|JI7ZI3BRrkt40uuUmg oyZC3pQ2lS65SnSGL|1|11-777-499-8213|2091.63|HOUSEHOLD|sual platelets. furiously final theodolit 577|Customer#000000577|a73SSq2cip7C8nSzdmmscpZyLCZ7KL|14|24-662-826-1317|7059.15|FURNITURE|int furiously. slyly express pin 578|Customer#000000578|nxUZ BCBO1 HAymUcopl2NtyWMuWVnE3bqPVDB|14|24-278-860-9263|6181.23|FURNITURE|ly. carefully pending packages cajole among the carefu 579|Customer#000000579|9ST2x,snyY3s|0|10-374-175-6181|1924.96|MACHINERY|ndencies detect slyly fluffil 580|Customer#000000580|wpvPbaPtx5QN|11|21-444-589-3830|-181.63|BUILDING|dependencies. final asymptotes haggle among the bold packages. slyly silent 581|Customer#000000581|s9SoN9XeVuCri|24|34-415-978-2518|3242.10|MACHINERY|ns. quickly regular pinto beans must sleep fluffily 582|Customer#000000582|KqH6uOztVK55zDxLA9kvdtny i5OYXt|3|13-484-591-9280|4879.55|MACHINERY|carefully against the quickly s 583|Customer#000000583|V3i6Gu9,LZtvdnNppXnI2eKQFx0b36WvL,F |13|23-234-625-4041|3686.07|HOUSEHOLD| haggle. regular, regular accounts hinder carefully i 584|Customer#000000584|jebKvptmHtS9,YE1qOjl2AOw38P,8skngJZh|15|25-352-778-1041|8825.71|MACHINERY|ages boost regular deposits. blithely stealthy depo 585|Customer#000000585|OAnZOqr6A,,77WC001ck8BAqvJTW6,dRGoRdX|16|26-397-693-4170|7820.26|MACHINERY|ickly ironic requests sleep regularly pending requ 586|Customer#000000586|vGaA9XBtn,hlswFhSjLIXGlLEDD2flE8UXwj|11|21-239-369-7791|5134.35|AUTOMOBILE|above the blithely express ideas. slyly r 587|Customer#000000587|J2UwoJEQzAOTtuBrxGVag9iWSUPTp|6|16-585-233-5906|7077.79|AUTOMOBILE|ve the final asymptotes. carefully final deposits wake fu 588|Customer#000000588|ex9SkK7K uM,ki1dsO7PgZLlIuQFKJUQZpD2oS|17|27-988-546-2598|483.89|FURNITURE|ic requests haggle quickly across the deposits. regular, express ideas along the 589|Customer#000000589|TvdYNogIzDfr 1UyJE4b9RTENPmffmIoH|19|29-479-316-3576|1647.05|FURNITURE|s; blithely ironic theodolites sleep-- accounts haggle around the furiously silent ideas. silent, final packages in 590|Customer#000000590|4sHhhAZWHYRxJVz0KRgjW9IlKu,55IuT|8|18-734-215-6394|3993.54|MACHINERY|es. regular dependencies cajole furiously blithely regular ideas. regular dependencies cajole carefully a 591|Customer#000000591|wGE7AnEtiX7cmCkYA|20|30-584-309-7885|6344.66|MACHINERY| regular requests after the deposits cajole blithely ironic pinto beans. platelets about the regular, sp 592|Customer#000000592|srNO5Hu10z1Ru4rRPU,QpXzFwY8759wqZ|24|34-832-574-7217|9712.75|BUILDING|lithely final requests use slyly. special theodolites nag carefully-- carefully pending deposits cajol 593|Customer#000000593|SYyEL2nytJXBbFemMseCiivA32USVEDbvGzZS|9|19-621-217-1535|233.51|AUTOMOBILE|ve the regular, ironic deposits. requests along the special, regular theodolites lose furi 594|Customer#000000594|sbcKWltfCAnXrc Z27ZYDzsH1ztd,ZhgaD9xIMsh|9|19-286-925-8440|6518.42|HOUSEHOLD|. fluffily final instructions are slyly toward the slyly 595|Customer#000000595|7Q17BacxM,liY2AwhnHGR0Pjf1180sMz1U|19|29-554-215-7805|4177.17|HOUSEHOLD|gular accounts x-ray carefully against the slyl 596|Customer#000000596|hoByQV2JchlIWfzPFW8I0nCI|5|15-484-811-5482|1722.88|MACHINERY|ecial deposits after the slyly regular packages dazzle furiously across the courts. accounts wake. reg 597|Customer#000000597|Dbv,XVGzl4X|15|25-687-952-9485|2443.52|AUTOMOBILE|es across the slyly brave packages maintain quickly quickly dogged excuses 598|Customer#000000598|9ICLFWFZa6|9|19-113-384-3847|3244.78|FURNITURE|es. furiously pending packages haggle fluffily carefully silent foxes. carefully unusual dependencies boost fu 599|Customer#000000599|fIvpza0tlXAVjOAPkWN5,DiU0DO4e5NkfgOlXpDI|4|14-916-825-6916|6004.52|HOUSEHOLD|thely even requests wake carefully regular theodolites. instructions haggle alongside of the f 600|Customer#000000600|LOtVjPC,Eu,0I2BRCqWf,K|12|22-675-907-7888|2003.44|HOUSEHOLD|nstructions sleep among the final, even pinto beans. fluffily pending theodolites according to the 601|Customer#000000601|P3 Dv,6yllTNmL9yt6NUZZPZjvM2coWJd|1|11-104-635-9839|9768.21|BUILDING|ly according to the unusual foxes. carefully ironic accounts haggle accounts-- regular dolphins will integ 602|Customer#000000602|NCryKIpG3W,FDV2|13|23-434-900-7213|8404.90|BUILDING|nstructions. asymptotes above the forges are against the carefully 603|Customer#000000603|DFAIWiyqZ0GzuF6AWCZX3DcDxyICb3EWxEw|19|29-629-573-6194|8161.13|AUTOMOBILE|n packages wake carefully. special requests haggle slyly carefully bold deposits. furiously regular pinto b 604|Customer#000000604|qCQsFELZ3W hlmi,zOHBcZGo0PZl3jbFu1jsijqE|21|31-757-951-9827|3195.96|HOUSEHOLD|le furiously express instructions. ideas hag 605|Customer#000000605|QAxZ0IXgCzUfNjseQCLfh95HEi|23|33-269-948-8039|-549.73|HOUSEHOLD|ly regular foxes are quickly 606|Customer#000000606|vBIUd7LjRJ5rZXSzITHIvpZwBCClyt4Hjr Tlnf,|1|11-284-540-8460|9676.98|AUTOMOBILE|, bold packages. regular, final theodolites haggle slyly carefully final accounts. silently specia 607|Customer#000000607|m61hvYPASIGmNJx7Tu|24|34-601-151-4029|4038.45|FURNITURE|ymptotes. blithely bold requests shall are about the furiously final platelets. 608|Customer#000000608|luMI1JpfrrILCEeTgz8k98z|16|26-767-193-8671|2256.36|BUILDING|ld packages. special requests along the accounts are after the carefully unusual deposits. blithely quiet excuses c 609|Customer#000000609|dSpUFl8IR8Gh|21|31-869-580-1707|3651.06|FURNITURE|pending, express platelets poach furiously after th 610|Customer#000000610|Fo8RfPq1kgzD 0|11|21-782-663-7023|3374.92|BUILDING|uriously final deposits against the deposits detect alongsi 611|Customer#000000611|E1dtWGHE7NrLfnSKLPFU|19|29-924-242-5243|4272.43|HOUSEHOLD|inst the slyly final accounts. final packages wake after the even pinto beans. carefully 612|Customer#000000612|oNFqorGhq3a3woEp5q8xVDX|14|24-818-339-9984|7669.16|HOUSEHOLD|ns wake quickly quickly ironic accounts. regular accounts toward the 613|Customer#000000613|AJT,26RbanTdEHOBgTWg|4|14-275-416-1669|6679.75|AUTOMOBILE|ironic, pending deposits: quickl 614|Customer#000000614|YKweqHJfVok|18|28-698-510-6194|9630.24|FURNITURE|y even multipliers. pinto beans nag busily after the busily ironic reques 615|Customer#000000615|6aITapYMqM1fJQKuJD05Yb,6FhjrW|2|12-639-391-3956|-482.48|BUILDING|yly. blithely even accounts sleep blithely unusual in 616|Customer#000000616|yvUE7Qy3Ub6uGhPkuEJeOI|1|11-275-121-4443|6898.65|FURNITURE|ts. blithely bold packages sleep 617|Customer#000000617|Ifjxbt3Y4mGu|14|24-527-532-7752|3625.93|HOUSEHOLD|deas sleep slyly? final, even gifts about the furiously regular 618|Customer#000000618|9O4fhgteQdyFvCkrFm|0|10-675-573-1877|-932.38|HOUSEHOLD|uickly even ideas sleep slyly pending foxes. final, pending foxes nag slyly. permanent instructio 619|Customer#000000619|6bxrNxQA oes7cMa23R 5lDmIOIRThvd|24|34-245-618-6317|2336.99|FURNITURE|ts breach slyly after the slyly regu 620|Customer#000000620|3ztw9KQqKGNsiMM,I1 6g2f,u2Pm5LhlSEe8ZK1k|8|18-466-916-8135|5795.15|AUTOMOBILE|ructions boost furiously among the slyly final dolphins. regular, regular ideas w 621|Customer#000000621|IpFo6e22CRink74PUEPthY9DJJnSeORmQJ4|17|27-667-987-3718|3164.28|HOUSEHOLD|nstructions! final dependencies s 622|Customer#000000622|qdRHTTnVf9O2iFMG1sDm2GSnlM24tKWK|13|23-925-151-9771|5974.11|FURNITURE| asymptotes. slyly ironic excus 623|Customer#000000623|HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6|9|19-113-202-7085|7887.60|BUILDING| requests. dolphins above the busily regular dependencies cajole after 624|Customer#000000624|L1hGsNrx4BiN5DIZGk7WMsB90T4ag|4|14-558-935-8773|3907.11|MACHINERY|le carefully. sly asymptotes sleep fluffily unusual packages. final deposi 625|Customer#000000625|uvgDE6eQ2bJp4BkHyVdpYYC8|13|23-789-801-2873|5744.89|FURNITURE|, pending deposits. sly theodolites along the carefully unusual 626|Customer#000000626|PDeE61VY2Q96efuewIZ|5|15-540-121-5663|5447.12|FURNITURE|t brave foxes. slyly pending packages wake furiously along the deposits. carefully pendin 627|Customer#000000627|uOFz ,iMYi02Ksr13Q2nBCETCpSp|15|25-811-790-3533|5826.68|FURNITURE|ages. regular ideas sleep. bold foxes affix. regular instructions haggle. bravely unusual requests haggl 628|Customer#000000628|Vzraru5KbgcC3V|17|27-367-742-4090|4954.25|FURNITURE|l dependencies. pending warhorses haggle. fluffily final accounts slee 629|Customer#000000629|LeXGhXX1mFQ0Cq,7taW ruvRHTpG3q,KkW|12|22-260-205-9116|5100.77|AUTOMOBILE|ic attainments. stealthily pending pinto beans affix carefully ironic theodolites. fluffily final deposits sleep da 630|Customer#000000630|XAw3WrAa mt0DnOuycb16LG9zbUv04DXsS|24|34-396-743-8684|3649.05|HOUSEHOLD|ely across the blithely stealthy ac 631|Customer#000000631|By LIK3TbJ67sJLlpaoOCXZheuX|19|29-864-813-2575|2603.00|MACHINERY|bold platelets haggle. slyly even pearls can solve slyly among the final foxes. slyly unusual r 632|Customer#000000632|sUlni97rSK6DIL|3|13-310-645-6928|-487.92|MACHINERY|ld dependencies sleep slyly along the special requests. furiously regular asymptotes use furiously accounts. 633|Customer#000000633|0pgCxndi1coDLkAV,UJJDMV0wtVVahCQaQap 0M|2|12-450-116-1239|3385.52|HOUSEHOLD|ully. even, final requests use slyly. blithely special packages wake carefully furiously busy 634|Customer#000000634|O09TejHJ6UszNfmqTR cmal8zcs|20|30-997-704-1110|6397.58|FURNITURE|e above the regular deposits. slyly even requests integrate slyly blithely express forges. regular platele 635|Customer#000000635|Ftqi0UYvzz56Ov,J6,ySp5WE4vJ2rtY|2|12-399-186-7550|8216.79|AUTOMOBILE|efully bold deposits doubt above the clos 636|Customer#000000636|7urmO1zY77WBPOWn7pXA OqCm3upL9gOtL4V|7|17-420-712-5063|3660.47|FURNITURE|ions boost furiously at the final dolphins. blithe 637|Customer#000000637|Ey7g4q2oH Q1vs|7|17-983-923-8985|7511.17|HOUSEHOLD|ly furiously even accounts: final, unusual Tiresias do snooze theodolites. slyly regular dependenci 638|Customer#000000638|yyRRorZ HHzU1yJwNJwF72dvUVJ nMlzpKAXEb|24|34-102-347-8343|2258.40|HOUSEHOLD|nts x-ray fluffily across the theodolites. carefully final pearls cajole ruthl 639|Customer#000000639|8OiPHefIPoalRjUoCIwbXz|15|25-221-133-2233|4899.15|MACHINERY|ly bold asymptotes. ironic, even d 640|Customer#000000640|j3vjr0 n,pJFG4gIOtC|2|12-702-315-6637|3025.84|HOUSEHOLD|lly. furiously quick deposits haggle quickly regular packages. pinto 641|Customer#000000641|gbIvFlCygHjj5NG7U|24|34-761-987-7777|339.49|AUTOMOBILE|uriously final requests boost regular, final ideas. fluffily busy packages promise? requests along the bold, s 642|Customer#000000642|6Y1gEH0gMLh0yzlipNR|22|32-925-597-9911|5684.01|AUTOMOBILE| requests cajole blithely quick 643|Customer#000000643|9T 2avhfyF PQ|0|10-978-597-2747|5184.70|FURNITURE|fily along the quickly ironic ideas. final, final 644|Customer#000000644|bfZrdZE0QHtMc,ksudO|18|28-489-845-4801|6183.15|HOUSEHOLD|ke slyly alongside of the silent, even req 645|Customer#000000645|I,Kso,IZ,AL2rK4HhIB1wRWWrx3 yoaZlFs|11|21-242-974-5799|1146.49|BUILDING|. special packages haggle ruthles 646|Customer#000000646|gogI8kweD 2H6U,01lbIp0UWUwljSue,KRcC|2|12-177-329-4316|6074.42|HOUSEHOLD|ptotes. silent requests cajole fluffily along the slyly permanent ideas. pending instr 647|Customer#000000647|2Bx7,7i87h5cagC,ZBz49lyiziLqQoD|1|11-873-931-2886|-132.97|BUILDING|. excuses use alongside of the furiously final deposits. blithely express foxes wake furiously. blithely 648|Customer#000000648|pYCT1OLD5Y7rBjjAKnf0Lqd 41RC,0n,nT3oNWz9|7|17-473-731-8352|7029.24|FURNITURE| wake carefully outside the ironic, bold packages. blithely special dependencies boost slyly even ideas. careful 649|Customer#000000649|EntXL7MF4lU|11|21-656-678-1337|9442.55|BUILDING|unusual deposits engage along the carefully pen 650|Customer#000000650|1hqwYdlDhaqnkRrovbulo0rrDhJZgUz|5|15-842-586-1263|2086.91|AUTOMOBILE|g to the final deposits. quickly regular requests dazzle along the slyly express courts. 651|Customer#000000651|K7rCTwTb3UX9wAO6ihvYxwBOuJNx51|9|19-610-541-4787|4716.31|HOUSEHOLD| fluffily quick dependencies cajole slyly along the blithely final dependencies. carefully pen 652|Customer#000000652|0WlNKsoRqdjU9,5 Qz,bgm5swI,i0Kg|20|30-254-389-5987|-919.65|AUTOMOBILE|p carefully blithely bold platelets. furiously unusual deposits wake. sl 653|Customer#000000653|la,ROBPJ4I2YNzQw,RpbB0sTOjyv3F ZBeWRiQ|11|21-715-573-6928|7307.55|BUILDING|aggle bravely deposits. even du 654|Customer#000000654|Ip,OhnmOeHu9PezZBvw7AHJcBDOWVoVUJKsJyA|16|26-416-434-3449|1067.04|AUTOMOBILE| nag slyly. final deposits sleep along the ironic, bold i 655|Customer#000000655|SceikyyffYL5OUI8rFnAgrT5E|14|24-916-820-5158|1724.68|FURNITURE|e accounts. unusual, pending platelets are quickly. req 656|Customer#000000656|DeZPec2K1U1fCBR2Ul9mnBNjKuXPcRcm|5|15-379-993-4446|-390.09|AUTOMOBILE|uests cajole quickly. final, pending pinto beans wake furiously among the requests. ironic foxes with the forg 657|Customer#000000657|BpXQ3sbx8bCU0|19|29-952-164-8914|8729.97|FURNITURE|ies boost slyly bravely express instructions. fluffily even theodolites integrate s 658|Customer#000000658|9SboxGtXimmJg49IyT3Zt|1|11-675-750-9832|4716.68|HOUSEHOLD|r ideas about the slyly ironic theodolites integrate according to 659|Customer#000000659|ThR9miOedPuwVEZyz 3MMjHPwB|0|10-834-287-1466|5297.68|HOUSEHOLD|s cajole final, regular dependencies. final accounts sleep! furiously regular requests wake slyly silent 660|Customer#000000660|ZD4fphyxb5pyE|0|10-987-901-3986|2110.30|MACHINERY|ding excuses haggle furiously regular foxes. finally 661|Customer#000000661|1MqWuuRVM5y5NlT1kakwke|4|14-983-203-6472|1735.44|HOUSEHOLD| regular theodolites. brave platelets 662|Customer#000000662|ss AgOrB5VFubLk bsmTgbw2ddJD|0|10-728-355-4532|4517.33|BUILDING|ckages haggle: quickly regular ideas are after the furi 663|Customer#000000663| fqLQWxHWWC40GUOgVvb4idYDbE1Gxc6|9|19-613-882-5677|9698.21|AUTOMOBILE|r the slyly final packages. slyly bold attainments detect blithely acco 664|Customer#000000664|viRe1P6HiyL4LXpU7HPwu|15|25-329-655-5068|8878.22|BUILDING|yly ironic requests cajole pending pla 665|Customer#000000665|Fo9QgQsDOP28D3zR|22|32-759-858-2984|-616.79|BUILDING|y along the special requests. blithely regular waters thrash abou 666|Customer#000000666|dD32Q8kL6KW|2|12-594-508-9621|4538.54|AUTOMOBILE| special packages boost across the even accounts. ironic accounts serv 667|Customer#000000667|oQqeEC,OD9XC1JXyOsHqcpv0fPUdP9ek5KKb70tQ|6|16-917-453-2490|3288.76|AUTOMOBILE|lithely about the furiously sil 668|Customer#000000668|PCmw0r6KkLjXZsljablJ|15|25-582-501-2346|8184.21|FURNITURE|uffy excuses are carefully regular, ironic pinto 669|Customer#000000669|axdO3iaVyYXSxnqnwC0p2Dx6Mn3lDDMp|10|20-471-280-4789|6019.20|MACHINERY|ronic decoys. blithely ironic accoun 670|Customer#000000670|RJtcLv6Tjpx|2|12-839-426-4266|6738.93|BUILDING|y unusual deposits are carefully regular foxes. packages dazzle carefully. furiously unusual d 671|Customer#000000671|ic6qGrt0giB,HDEiBK,,FYGHXQpc|21|31-593-213-9388|3227.87|FURNITURE|bold ideas above the ironic packages affix blithely about the furiou 672|Customer#000000672|Rsq8zHIbqRRB8VlgldFlU56j,0SN |18|28-774-971-2700|7778.95|BUILDING|ake. regular requests about the idle asymptotes haggle slyly final, silent packages. carefully special 673|Customer#000000673|Q2S9DoW6mQN5iQ0A8DxD6UxNmPZky|4|14-769-230-8609|3942.58|MACHINERY|ons will cajole blithely. always even packages dazzle blithely across the regular, unusual 674|Customer#000000674|GLZCUQrtiNTrPKdK 0O86ZF|5|15-543-819-4391|7889.64|BUILDING|ites cajole ironic accounts. regular, ironic gifts nag furiously against the blithely express deposits; unusual, 675|Customer#000000675|canew8kOzr8RDTmenqJOn|20|30-299-640-9565|5295.26|HOUSEHOLD|riously even requests boost slowly. ex 676|Customer#000000676|JzJnD kA3KTjTYl|24|34-710-820-8362|5527.61|HOUSEHOLD|es. slyly even theodolites across the slyly ironic accounts nag from the slyly even accounts. accounts are. 677|Customer#000000677|,wTtWOOr wqX2sL,h79myT6nxG1EgY|3|13-398-309-9122|5582.63|AUTOMOBILE|ly brave ideas haggle slyly blithe acc 678|Customer#000000678|BiQyhSp oiDp,,MFVxyrOwMT810qiQlISEPye|9|19-508-735-4910|6302.93|AUTOMOBILE|gainst the carefully pending requests-- furiously special requests haggle? not 679|Customer#000000679|IJf1FlZL9I9m,rvofcoKy5pRUOjUQV|10|20-146-696-9508|1394.44|AUTOMOBILE|ely pending frays boost carefully 680|Customer#000000680|TuONtFVII8TR2QtJyl1lS5f0iagaWWsBm1IQo|12|22-593-663-2200|4366.23|AUTOMOBILE|kages: final deposits believe unusual deposits: b 681|Customer#000000681|x85ncT W41KEcUQCtxE,LD iMNO6j4Hh4x|20|30-809-878-5822|235.53|MACHINERY|ate carefully. accounts hang carefully along the blithely final theodolites. bravely sile 682|Customer#000000682|y5J0HJVI6Mb k4rXTAPbxE9aw|15|25-233-924-7389|7420.60|AUTOMOBILE|he bold, even accounts breach 683|Customer#000000683|G0, q8c6vBykpiLvcuSJLYvqE|6|16-566-251-5446|9120.93|MACHINERY| somas use-- slyly express foxes wake quickly blithely iro 684|Customer#000000684|5kSJ3qMS,7YKdfmwBFzQN4y8EgPiH,8ln19|15|25-456-286-6398|8545.90|HOUSEHOLD| of the bold requests. blithely regular accounts nag alongside of the carefully silent dolphins. carefully 685|Customer#000000685|eRvmYOdl5v|17|27-269-224-9902|2039.84|MACHINERY|nusual ideas. pending packages use carefully. slyly special sentiments cajole after the blithely even accounts. blit 686|Customer#000000686|1j C80VWHe ITCVCV|6|16-682-293-3599|5503.36|HOUSEHOLD| even deposits print quickly. foxes wake. furiously ironic asymptotes across the bold foxes 687|Customer#000000687|UJTC3 WtzvoD39r1GuoCP|21|31-149-119-1456|4999.31|FURNITURE|special packages haggle carefully slyly ironic pinto beans. sl 688|Customer#000000688|2QBxK8WIryWMZTsDM JS7GxWCB6Y71Swa,f1EUxk|8|18-210-546-2836|3354.47|HOUSEHOLD| use alongside of the ironic, pending foxes. final packages use. ironic, unusual dependencies haggle 689|Customer#000000689|Gcie9Q,Wc6J0QvKcefqflajjOtOVchnxqBn9|24|34-325-146-3591|1481.24|MACHINERY|ions. furiously unusual platelets cajole blithely. caref 690|Customer#000000690|xH61m,Si5X4REvi|3|13-489-760-5455|103.45|HOUSEHOLD|nt deposits. final, unusual requests use blithely. slyly unusual packages against the carefully bol 691|Customer#000000691|0RkDX6OLU1hISYCLmdLD C|16|26-741-688-4189|9566.15|MACHINERY|p. ironic, regular foxes against the ironic, special theodolites nod carefully quietly unusu 692|Customer#000000692|d6XE9sg, wINvIz8aZS b8n XuhAQU5|19|29-804-421-1703|276.71|AUTOMOBILE|cuses. slyly even asymptotes doubt quickly. fluffily thin theodolites boost. ca 693|Customer#000000693|r5gn5SUW0tsfkOw42x84|1|11-391-870-8153|6577.21|AUTOMOBILE|onic packages. carefully final sauternes across the even, express deposits h 694|Customer#000000694|3ToF2HqyF9mEcP1QgW NMN,6,KsFq6x4u14MS|10|20-646-375-1939|2222.65|BUILDING|efully final requests detect blithely. furiously ironic p 695|Customer#000000695|fbT9dQCc,su7JboB5FWI6|8|18-131-151-7466|9126.54|FURNITURE|ly express deposits according to the fina 696|Customer#000000696|dYyj4azN0EE GqeZxv|22|32-923-916-5643|3778.97|HOUSEHOLD| ironic ideas among the carefully bold foxes na 697|Customer#000000697|LFwdGCTUYDenZGoA9|4|14-613-651-2263|8121.84|AUTOMOBILE|he regular pinto beans. courts cajole carefully according to the blithely even theodolites. quickly 698|Customer#000000698|FLZ2NG5pKHpPtAh|23|33-282-178-6799|2894.22|BUILDING| sauternes. gifts cajole. slyly pending platelets cajole at the iron 699|Customer#000000699|4R6pspbuk5Sp,036VraUEKPBzs|4|14-865-140-8680|-809.22|HOUSEHOLD|. finally even ideas wake slyly pending dolphins. slyly even instructions wake whithout the p 700|Customer#000000700|zyWvi,SGc,tXTls|0|10-351-119-7514|4367.53|MACHINERY| bold excuses. furiously even asymptotes across the carefully regular dugouts cajole fur 701|Customer#000000701|yQU8LcjocNHkk1hse30M0U1f46MrU6dB|2|12-323-784-5793|148.98|FURNITURE|c packages. blithely final instructions according to the regular pinto beans 702|Customer#000000702|BDKtDAva8rBuCWXT6jXb2JJY7YoRS|3|13-549-296-5659|4782.05|FURNITURE| silent accounts. regular, regul 703|Customer#000000703|ge1GEYt4ewGUiSeqBA4rNB5JhyQ92uKF|6|16-741-513-6919|3998.42|AUTOMOBILE|arefully final accounts cajole quickly blith 704|Customer#000000704|41s9yU0ossOKgBvjPPSTZqhfciy|9|19-998-886-1551|441.44|AUTOMOBILE|ously ironic instructions believe about the pending, regular deposits; final accounts cajole alongside of t 705|Customer#000000705|YKdZRFEGcclF6rmdoiNCY8|18|28-991-405-7914|3199.00|BUILDING|s pinto beans sleep busily. ironic, bold theo 706|Customer#000000706|ycxysGDuWHN98FS4lZ2obT9ZPNRWjJTsHbQXvi|19|29-468-275-6045|6496.08|AUTOMOBILE|ses cajole furiously after the ironic requests. pinto beans sleep busi 707|Customer#000000707|DT0nzkijELHyI|21|31-796-903-4461|2619.52|FURNITURE| the deposits. fluffily ironic pinto beans wake quickly. slyly even foxes sleep slyly quickly final deposits. ev 708|Customer#000000708|9iJJ868sXAdFgZyo0V8cfPRTRaUc3d|17|27-796-490-6026|2297.33|BUILDING|p along the carefully ironic pinto beans. even pinto beans are fluffily against the furiously quick dolphins. s 709|Customer#000000709|drDnC6YXTJgcdfJkpbhrQ7z7XmCsoym2t22jwg|18|28-117-812-5493|-867.94|BUILDING|g the blithely ironic multipliers sleep a 710|Customer#000000710|OCLSZuXw1AEK NLvlofMkuK,YNe,bJD40a|22|32-459-427-9559|7412.12|FURNITURE|ges integrate express, even ideas 711|Customer#000000711|EmACeG3r2Y9bxf7KLLgX,ZdQlATjGaafINO|15|25-306-725-3622|9591.51|MACHINERY|ackages promise. theodolites haggle along the express dolphins. special, special deposits affix slyly final pac 712|Customer#000000712| 8w2pIiA4wWAhtjAdXR|6|16-843-486-5087|8667.09|BUILDING| express dependencies use fluffily final platelets. furiously regular accounts wake furiously. carefully careful cou 713|Customer#000000713|ov fZJgESFHP P|10|20-595-832-3185|3829.03|MACHINERY|lites sleep slyly furiously express requests. even, even notornis affix furious, ironic p 714|Customer#000000714|2,ARep1aMyhgNZqrkc,toQ3XY6FOiCZqNW|9|19-209-782-4244|2438.10|BUILDING|cross the bold pains. bold instructions haggle. even accounts wake slowly. carefully busy ideas need to x-ray sly 715|Customer#000000715|9qLvF42uxUarKl4I 2pEKOMNJmo8Ro5EK|19|29-500-408-6392|85.05|AUTOMOBILE|hins boost quickly. quickly regular epitaphs haggle fluffily quickly bold pinto beans. regular 716|Customer#000000716|jbXpMEcV9j,6ciftBXEMXDjE|12|22-501-849-6084|8364.99|AUTOMOBILE|as. even accounts about the slyly pending excuses lose bl 717|Customer#000000717|TqWi1c aNhdExPcDD7KQ18W|12|22-660-518-8009|8510.94|MACHINERY|ns poach slyly. carefully express pinto beans ab 718|Customer#000000718|w,GXCSSI4NEHAFPKG|20|30-605-635-8197|8438.40|HOUSEHOLD| regular packages-- pinto beans detect ironically. furiously even accounts detect unusual, pendin 719|Customer#000000719|wry Gj3xd8QX ylUCulG|15|25-498-699-7824|-774.47|HOUSEHOLD|gly express pinto beans sleep furiously around the quick foxes. 720|Customer#000000720|8shFEL7J9sq1NJXR8dixBqaTO,kbSx|22|32-575-838-4260|5357.32|HOUSEHOLD|furiously above the furiously ironic c 721|Customer#000000721|N6hr4gV9EkPBuE3Ayu |21|31-174-552-2949|3420.64|AUTOMOBILE|ar instructions. packages haggle stealthily ironic deposits. even platelets detect quickly. even sheaves along 722|Customer#000000722|aymiuXFyrALTRIzfbLDvtaj37ydq|20|30-633-109-9587|2724.80|BUILDING|leep sometimes express accounts. regular theodolites wake quickly ab 723|Customer#000000723|5tKvCvmVB0yxyAF|21|31-542-178-2520|738.57|BUILDING|e quickly against the blithely final requests. regular pinto beans nag fluffily fur 724|Customer#000000724|dF80enl1y5MfccEMqYz0JSY|5|15-316-638-4703|3035.29|AUTOMOBILE|s. bold accounts about the carefully final packages detect slyly pending platele 725|Customer#000000725|fm8t3X05Wm,PQxPSPHZy,rHI,wUBLTFO5S |16|26-923-317-3870|5030.24|FURNITURE|ts. accounts after the even pinto beans boost across the ironic, silent accounts. reques 726|Customer#000000726|4w7DOLtN9Hy,xzZMR|3|13-168-233-7513|6253.81|BUILDING|wake after the regular, final instructions. final epitaphs mold fluffily blithely ironic id 727|Customer#000000727|wLIX8hKnk0qjUAAGqm|18|28-815-316-3525|807.37|FURNITURE| requests-- quickly regular pinto beans need to detect 728|Customer#000000728|uACufK5vIMlsq,v9d4U5ZWi,|24|34-195-384-1105|4349.73|BUILDING|re stealthily furiously bold requests. carefully final pinto beans lose slyly 729|Customer#000000729|YZxBLBAqBmW53g2ia6s4u,MlJ4WhW|20|30-900-572-2883|8947.26|MACHINERY| packages about the furiously even foxes sleep furiously furiously pending ideas. slyly ironic epitaphs 730|Customer#000000730|ORP6ofUwyD1|2|12-513-973-2702|4718.23|AUTOMOBILE|foxes would cajole furiously slyly special forges; slyly regular pinto beans inste 731|Customer#000000731|D0ceqTVRO3EctrC|7|17-731-915-9753|9311.17|MACHINERY|lar dolphins are carefully slyly final theodolites. unusual accounts integrate across the regular, final deposit 732|Customer#000000732|tQ,r4voHok7oeKw9msDh4ORGje|16|26-154-285-2634|4830.51|BUILDING|structions are after the even deposits 733|Customer#000000733|Ks7Ed2g7zDP905tLGdGcFtomDNchNdaPK2cd|13|23-110-948-6754|4351.09|HOUSEHOLD| furiously. pending, quiet packages nag slyly. stealthy pinto beans haggle 734|Customer#000000734|xULRxhbfzPT5nBh|18|28-880-175-5173|9750.71|AUTOMOBILE|tions against the ideas haggle furiously quickly regular forges. carefully final requests solve even, 735|Customer#000000735|8Gn WOTY4cgGaoHwezrZ,JN,Px8e sr|5|15-959-576-4327|293.88|BUILDING|ites. furiously unusual frets boost among the slyly ironic theodolites. packages cajole 736|Customer#000000736|zQNs5e2aFcVY8MigFQEBtoCaQ9e|1|11-617-726-2039|9114.68|MACHINERY| the deposits. blithely regular asymptotes boost q 737|Customer#000000737|NdjG1k243iCLSoy1lYqMIrpvuH1Uf75|18|28-658-938-1102|2501.74|AUTOMOBILE|ding to the final platelets. regular packages against the carefully final ideas hag 738|Customer#000000738|iecb89zRRNeN3KywZSaPXEWMjz|10|20-722-370-5220|2734.95|MACHINERY| carefully slow instructions. furiously express dependencies alongside of the ca 739|Customer#000000739|pAROUfTi3wCEHi1PXu|14|24-309-302-6776|6344.18|HOUSEHOLD|ly regular accounts. blithely final theodolites sleep blithely. furiously regular decoys cajole blithely. bold de 740|Customer#000000740|FCerGpsfsWAsBrQTyqdzMxUQnbQembHKGg|10|20-215-156-3727|1733.76|BUILDING|ets. final, blithe theodolites abou 741|Customer#000000741|jiPAOQuJ5gIauMfvcbE8lKUVPMp|12|22-560-854-2068|9195.41|HOUSEHOLD|ve the fluffily regular accounts wake carefully ironic grouches. slyly bold theodo 742|Customer#000000742|2qRObRkFktME6SsNV0Pa3L8txbA0AFtXuWsKrkW|12|22-610-582-8610|6381.24|HOUSEHOLD|eodolites alongside of the daringly ironic deposi 743|Customer#000000743|WJ lVLsFSgZJCHHLqwRmvCjWvYlCs0c2TvO|20|30-743-559-7934|8450.37|HOUSEHOLD|ptotes. furiously even accounts haggle slyly. ironic foxes sl 744|Customer#000000744| nYSoGuQkf|7|17-121-555-4268|2458.81|HOUSEHOLD|n packages. furiously silent foxes sleep regular, regular accounts. slyly ironic patterns cajole fluffily 745|Customer#000000745|vjuHvDKdaomsivy l|18|28-913-438-9403|7115.14|FURNITURE|o beans. bold, regular theodolites haggle carefully about the quickl 746|Customer#000000746|JOKj8N2QKUm8Gi,F4qX0fLVy|20|30-154-354-9928|1164.46|MACHINERY| final theodolites. final pinto beans haggle-- furiously 747|Customer#000000747|uuichgTY7NjlZaaRZ6S7KzvapZWvAiCUWAQr|8|18-793-504-2931|67.95|FURNITURE|eodolites. blithely regular pinto bea 748|Customer#000000748|1 nkl3GMSnweulTNAMPeB8Sa5aSIk|23|33-403-226-2580|6959.09|AUTOMOBILE|nts. special excuses thrash blithe 749|Customer#000000749|U1Dvu0r793a|24|34-158-697-9591|7491.42|MACHINERY|accounts was. final, final requests wake. theodolites was slyly. blithely even foxes wake carefully ac 750|Customer#000000750|5OyNRajjgjjbaXtI rkxvB2lX4c6u|8|18-235-587-1274|269.90|BUILDING|s. regular, regular deposits sleep carefully blithely bol 751|Customer#000000751|e OSrreG6sx7l1t3wAg8u11DWk D 9|0|10-658-550-2257|2130.98|FURNITURE|ges sleep furiously bold deposits. furiously regular requests cajole slyly. unusual accounts nag unusual ide 752|Customer#000000752|KtdEacPUecPdPLt99kwZrnH9oIxUxpw|8|18-924-993-6038|8363.66|MACHINERY|mong the ironic, final waters. regular deposits above the fluffily ironic instructions 753|Customer#000000753|9k2PLlDRbMq4oSvW5Hh7Ak5iRDH|17|27-817-126-3646|8114.44|HOUSEHOLD|cies. deposits snooze. final, regular excuses wake furiously about the furiously final foxes. dependencies 754|Customer#000000754|8r5wwhhlL9MkAxOhRK|0|10-646-595-5871|-566.86|BUILDING|er regular accounts against the furiously unusual somas sleep carefull 755|Customer#000000755|F2YYbRT2EV|16|26-395-247-2207|7631.94|HOUSEHOLD|xpress instructions breach; pending request 756|Customer#000000756|Lv7cG by4Wyd8Hzmumwp8hSIZg9|14|24-267-298-7503|8116.99|AUTOMOBILE|ly unusual deposits. fluffily express deposits nag blithely above the silent, even instructions. expr 757|Customer#000000757|VFnouow3LhLvEDy|3|13-704-408-2991|9334.82|AUTOMOBILE|riously furiously unusual asymptotes. slyly 758|Customer#000000758|8fJLXfS5Zup0GQ3xBKL3eAC Q|17|27-175-799-9168|6352.14|HOUSEHOLD|eposits. blithely unusual deposits affix care 759|Customer#000000759|IX1uj4NFhOmu0V xDtiYzHVzWfi8bl,5EHtJ|1|11-731-806-1019|3477.59|FURNITURE|above the quickly pending requests nag final, ex 760|Customer#000000760|jp8DYJ7GPQSDQC|2|12-176-116-3113|2883.24|BUILDING|uriously alongside of the ironic deposits. slyly thin pinto beans a 761|Customer#000000761|oObRVLlulGS5xikRk8La|19|29-835-631-4258|1525.96|BUILDING|ress Tiresias haggle across the never ironic ideas. blithely pending theodolites cajole ironically 762|Customer#000000762|n5QsjD,gTSrdNRoRpvWqS|11|21-757-455-7898|3764.90|FURNITURE| sublate carefully carefully regular deposits. bold foxes along the gifts nag 763|Customer#000000763|ACMrkbcf3a2J3aobVEmU5hGnHuc|1|11-232-719-3610|2650.27|BUILDING| ironic asymptotes are. slyly bold realms alongside of the fluffily specia 764|Customer#000000764|F8WBznjtO2bX2knXl4ghnsp ixWylkf|16|26-714-322-4800|2061.45|MACHINERY|tructions. quick theodolites kindle 765|Customer#000000765|b7w2edOUZNe0QX 3Ab3y5RFlMZX|20|30-544-657-7473|5683.33|AUTOMOBILE|ter the blithely regular foxes. slyly bold packages use bl 766|Customer#000000766|zGTH6uWKoQxIE|16|26-283-847-8946|4677.55|BUILDING|ly special deposits boost blithely above the slyly ruthless ideas. carefully bold asymptotes unwind about the e 767|Customer#000000767|9f3 XsYXdvP0E39ZBi7r7oRCns3PTx,H|17|27-526-143-3959|7647.09|BUILDING|ngly express dolphins boost carefu 768|Customer#000000768| ,cIZ,06Kg|18|28-851-493-8588|9618.84|BUILDING|to cajole blithely express asymptotes. foxes use quickly. carefully special ideas could have to are bravely bl 769|Customer#000000769|0eGzsjpDFsN0|5|15-102-635-4835|5950.86|HOUSEHOLD|ggle slyly pinto beans. furiously even excuses above the 770|Customer#000000770|IjwJR6TjBJZbKIeouH2d|8|18-856-112-5677|2066.79|AUTOMOBILE|dolites haggle express, express requests. furiou 771|Customer#000000771|J9UMiYJznHVHZDuCG,ErV0iiMStETS|23|33-761-371-4753|7461.74|AUTOMOBILE|arefully unusual accounts shall boost unusual pinto beans. ironic, final ideas boost. slyly i 772|Customer#000000772|TBxlR0AAHeSYl0vyK 8joF|20|30-405-614-4887|7555.16|MACHINERY|ular requests. final ideas sleep. regular, even platelets could haggle blithely bol 773|Customer#000000773|NyRSeog kIkD7YOb0EuSfSGxfptN5nkX26Mk6|8|18-456-377-3723|4578.24|FURNITURE|hely silent ideas could doubt carefully al 774|Customer#000000774|95 O8gd08tdtmJwM0ebHUnDc|11|21-463-652-6686|-506.37|HOUSEHOLD|nd the bold platelets affix furiously doggedly express accounts. carefully unusual deposits serve furiousl 775|Customer#000000775|Cg3M4gTXeIY7llMN2puop7D2|17|27-903-936-7924|1376.67|BUILDING|ts. blithely regular requests upon the ironic, final courts haggle sly, regular deposits. final requests 776|Customer#000000776|rzhIStRHsiWoc6K,7yv3YMqVdrz|15|25-941-650-8313|2669.01|AUTOMOBILE| express accounts. furiously ironic theodolites serve blithely. blithely thin packages are among the eve 777|Customer#000000777|27adTXaVp7araW|20|30-765-163-9750|9097.52|AUTOMOBILE|pinto beans; furiously special platelets haggle quickly against the slyly unusual foxes. 778|Customer#000000778|tCuRA2W9y5iiGrcT7a4TzK|23|33-702-179-3134|52.43|MACHINERY|hely ironic instructions. regular, ironic requests affix along the carefully 779|Customer#000000779|2cTZiS4ulZ74edT,RmDnh4ZaCrphMMh Ff2|5|15-940-483-5702|-902.48|HOUSEHOLD|old dependencies. pains haggle fluffily carefull 780|Customer#000000780|CMxcdzgEUkCWP1|8|18-844-576-7345|9874.12|FURNITURE|, final packages use slyly regular deposits. slyly ironic instructions nag careful 781|Customer#000000781|FQCAkyfV0 kL3,FNA1OlBjABak|18|28-478-388-5881|6403.62|MACHINERY|ake blithely blithely final foxes. blithely silent pinto beans haggle furiously. fluffily bold acco 782|Customer#000000782|HFuyemzqz0g QhkL|19|29-850-576-7450|-326.32|AUTOMOBILE|usly bold deposits-- furiously ironic accoun 783|Customer#000000783|01bR7OOM6zPqo29DpAq|1|11-920-256-8525|2436.32|FURNITURE| slyly carefully pending packages; doggedly bold theodolites boost slyly slyly dogged excuses: slyly expr 784|Customer#000000784|evcGXqbosO6,qhx|14|24-975-574-2063|3170.47|BUILDING|cajole quickly. ironic dependencies wake quickly. silent ideas use furiously 785|Customer#000000785|gEkI8kSq8RYgO6tiTA0AB7urZX8s2w03JGwtMi9|18|28-808-670-2983|108.14|BUILDING|nal theodolites. dogged, ironic deposits wake carefully. slyly regular requests after the furiously ironic 786|Customer#000000786|viZtkiJ gbMcPrINM3Ez,33vOJW|13|23-413-365-2022|400.03|MACHINERY|ccording to the regular deposits. carefully final packages run. slyly final deposits h 787|Customer#000000787|wwPe2vMZZ1n1Mm2z0qzDfS43FPj2Ndn|10|20-210-617-3870|212.16|AUTOMOBILE|e. ironic theodolites serve regular foxes. pending requests haggle slyly busy instruct 788|Customer#000000788|LbFHoDpNUSu3AyDS7KLgjoQBJV|16|26-388-689-9272|-330.69|FURNITURE|. unusual packages against the 789|Customer#000000789|DW0NMV Ci5V2bnsX0Al98plG1J0QZqNwcjEVW|9|19-176-517-4263|6038.60|BUILDING|as. quickly special courts integrate slowly final accounts. ironic packages are slyly about th 790|Customer#000000790|CR bzmYYVP|0|10-368-832-9671|2724.98|BUILDING|riously final requests haggle to the blithely special requests. blithely pending gifts after the carefully silent id 791|Customer#000000791|3ZWQ5xexnnLDEmxpmbg|13|23-575-775-4059|3694.81|HOUSEHOLD|posits cajole carefully along the slyly final packages 792|Customer#000000792|icVt7HjGs,p3YL3nr1MHgaQIY5Gmzej57nB,b|7|17-392-500-4370|1672.46|AUTOMOBILE|latelets along the carefully even packages cajole blithely packages-- even pinto beans haggle 793|Customer#000000793| SltK1IMp2Xvwb,A0x3Co1uhcwr|0|10-404-953-9048|2072.99|BUILDING|uickly silent foxes use after the 794|Customer#000000794|RMY8 LyGnJ67NGc5cxPYiIDSF|23|33-633-470-5945|1709.50|FURNITURE|gular patterns cajole slyly blithely final ideas? furiously unusual courts wake among 795|Customer#000000795|droXvSIcNEElsEYS|14|24-973-990-6608|8443.18|HOUSEHOLD|packages cajole furiously since the slyly 796|Customer#000000796|79bj5Rk3jJj9ked7M|3|13-554-411-6773|2584.23|HOUSEHOLD|re slyly even deposits. pending pac 797|Customer#000000797|bdGkzA5duas6LZ1ywB96K6Av3x 99q95h|0|10-994-609-7082|3063.11|HOUSEHOLD|iously regular packages mold fluffily. express, idle accounts nag furiously across the carefully ironic req 798|Customer#000000798|wW2OgnHj6dBz tO9OXFqCLm|4|14-670-423-7529|-391.13|HOUSEHOLD|sleep slyly ironic, express ideas. slyly special packag 799|Customer#000000799|LVk8ljWeIYTQQFMKCmxEeRpWQT|12|22-909-693-7833|2263.25|MACHINERY|ets haggle. busily final packages nag carefully after the bo 800|Customer#000000800|mpI6pkdnWLZsBbQi4,uUC5Y3TcM9vmRIgZelrQ|14|24-555-630-2261|9443.39|AUTOMOBILE|ly alongside of the carefully ironic deposits. blithely ironic packages are blithely agains 801|Customer#000000801|UQ67hfDJlxgX68hiFPmDuHav12Vx|16|26-439-495-8236|5207.32|FURNITURE|sits wake blithely according to the slyly un 802|Customer#000000802|ZDk4Suvi8gMp2LLAOW6nFA 3u|0|10-606-236-5778|1377.52|BUILDING|the carefully even pinto beans. carefully unusual pinto beans against the asymptotes cajol 803|Customer#000000803|zm7Xs6RJJJfZ|18|28-855-429-9109|6003.09|BUILDING|. blithely special instructions 804|Customer#000000804|u6xYwCLD,Vd4ODt8|24|34-861-760-4796|3.43|AUTOMOBILE|usly. final accounts integrate slyly above the furiously sly pinto beans. furiously unusual ideas 805|Customer#000000805|wCKx5zcHvwpSffyc9qfi9dvqcm9LT,cLAG|10|20-732-989-5653|511.69|BUILDING|busy sentiments. pending packages haggle among the express requests-- slyly regular excuses above the slyl 806|Customer#000000806|FTM62Dujm2BoblKFY w|12|22-888-883-2475|6859.28|FURNITURE|the carefully regular accounts breach above 807|Customer#000000807|zKjtrn5FGIs|17|27-333-779-5333|1929.89|MACHINERY|y final patterns? carefully pendi 808|Customer#000000808|S2WkSKCGtnbhcFOp6MWcuB3rzFlFemVNrg |19|29-531-319-7726|5561.93|BUILDING| unusual deposits. furiously even packages against the furiously even ac 809|Customer#000000809|eApaa5hW3Mqp11ZuOP|16|26-776-223-5427|-594.23|HOUSEHOLD|ackages. quickly unusual dugouts c 810|Customer#000000810|3J1wwVhwzVT7XM1v3Rzx90a8c4|9|19-997-929-2765|9632.77|FURNITURE|ep pending deposits. quickly bold accounts are carefully instructions. ruth 811|Customer#000000811|Rau3ADOLCNXM42D6fjQTFuueQce2BHw9r|15|25-575-731-2159|9010.02|BUILDING|kly pending foxes dazzle fluffily after the furiously bold packages. fluffy 812|Customer#000000812|u9uI3BE68quHja6k0,UXRYFgvLHl3jKhn|17|27-714-618-2239|3428.67|BUILDING|ar packages. packages thrash amo 813|Customer#000000813|VBNrCYm67O|9|19-585-173-1514|5673.47|HOUSEHOLD|ts cajole silently bold pinto be 814|Customer#000000814|KmMC1nd5iOXTRlhzMu13Wtx|15|25-342-269-3824|4011.18|BUILDING|yly at the blithely regular excuses. carefully thin pinto beans boost slyly furiously regu 815|Customer#000000815|TmpTFwt3b0 WQUmljE1LKbg50dtQszn67qEog|12|22-941-606-7227|5841.33|HOUSEHOLD|unusual accounts. slyly ironic pinto beans nag against the regular, regular requests. slyly p 816|Customer#000000816|k2dnnQZowe|24|34-457-910-7430|9239.32|BUILDING|accounts cajole slyly quickly express platelets. carefully silent theodolites wake carefully. pe 817|Customer#000000817|jDJt0Wfk0,Wcx5HOI5itS XqUJmMpjzJ|0|10-566-341-4382|7297.64|MACHINERY|y against the carefully special patterns. unusual accounts sleep quickly final instructions. 818|Customer#000000818|CvQvKClYoh9lPjMngrjbCoxqqnp2QFiTe8eF|20|30-943-141-5174|-574.39|BUILDING|yly. blithely final platelets haggle regular, regular ideas: furiously r 819|Customer#000000819|cug3zDy qHUaZMQNEtYlWK3R,mGrid4Of|12|22-424-990-1743|2152.05|MACHINERY|y final accounts sleep slyly express ideas. carefully express foxes sleep. carefully stealthy pin 820|Customer#000000820|xsYy3Nu7RhNYyBqL8dD0fONh|4|14-993-368-5047|-614.28|AUTOMOBILE|g, final ideas. slyly ironic packages dazz 821|Customer#000000821|PVdTfF7cfPueJS0MBncE5v4bKfPV64Zg|5|15-901-460-2033|2691.42|FURNITURE|wake carefully. slyly express deposits wake furiously among the fluffily final deposits. pa 822|Customer#000000822|ZoRr18ZiDTtkRduo6PO60gNRg,b46zOe QdS8B|13|23-410-801-1644|736.47|BUILDING|ding dependencies hang alongs 823|Customer#000000823|HuoOuJLxx7S8YDQhexfmIX|0|10-109-430-5638|738.79|AUTOMOBILE|l requests are against the slyly ironic pinto beans. blithely unusual packages wake alongsid 824|Customer#000000824|qHxq5m g6Ug,SDq7R17|18|28-454-124-5859|6840.29|AUTOMOBILE|ffily unusual packages. furiously final ideas 825|Customer#000000825|L8P98o,xq9E78kmuadw,Z7Rwy|10|20-492-863-1129|1809.01|HOUSEHOLD| busy deposits. careful, bold foxes eat furiously according to the final as 826|Customer#000000826|f5Y UZoE nR|13|23-762-328-7631|8938.14|BUILDING|usly final dinos. final depths wake slyly across the final, bold requests. bl 827|Customer#000000827|fmmKD8aBlF9bFndkYf402cHVrUj8FhGZtHeL|14|24-196-343-6537|5411.20|HOUSEHOLD|om the carefully even requests are slowly along the regular 828|Customer#000000828|0PkG ELwBIgU4AsNcjDc5Q,9Gj|23|33-108-680-4317|7059.68|BUILDING|cial instructions. accounts nod special accounts. slowly even deposits belie 829|Customer#000000829|4oht3f64ZWA1,ACbBP9DNxjjG1CbIqOCK|8|18-404-707-6095|4086.56|MACHINERY|deposits cajole slyly ironic, express packages. asymptotes cajole ruthlessly 830|Customer#000000830|4fNmWCmfys1jUI|23|33-408-548-6806|7775.65|AUTOMOBILE|ironic packages integrate alo 831|Customer#000000831|1DyQwHuUncB7BSnQUiW8ksexb|20|30-495-591-5384|3401.49|AUTOMOBILE|c instructions. fluffily bold requests are furiously quickly ironic deposits. carefully 832|Customer#000000832|eNIy9PatAWlh0|19|29-264-864-8387|-201.49|HOUSEHOLD|the special ideas. finally ironic accounts doubt. slyly regular ideas 833|Customer#000000833|t3qDCo,Yh MZcJFV6PibeY,MUunz|6|16-624-307-4875|-526.14|FURNITURE|uickly final orbits across the blithely express accounts integrate furiously among the final sheaves. blithe 834|Customer#000000834|y4mA, TY6DW4gAY|14|24-637-803-7812|-976.25|MACHINERY|regular dolphins. furiously careful ideas eat beyond the bravely regular packages. slyly unusual deposi 835|Customer#000000835|Sjqs42jh111|21|31-700-242-6347|2106.81|BUILDING|gside of the furiously final packages wake quiet, permanent accounts. unusual packages into the unusual, ironic req 836|Customer#000000836|rayz3tXswDbL4hyIr5SMrEIOTqvmok48e|12|22-181-313-6281|9184.72|FURNITURE|press deposits-- fluffily express accounts cajole carefully according to the daringly even deposits-- furi 837|Customer#000000837|mMNC8wjT5aC655e3|9|19-572-730-8324|2701.29|BUILDING|he bold, pending theodolites. special, bold accounts haggle. blithely regular ideas nag fluffily regular packages 838|Customer#000000838|BxDlz44b56kXjsTpl|5|15-919-192-7197|1605.76|AUTOMOBILE|elets. never express accounts haggle slyly. depths along the requests haggle bl 839|Customer#000000839|Ch8 wZk5UKK45BSq1uZLJL0Z A8UUoms|22|32-435-825-7973|2924.78|HOUSEHOLD| close accounts. unusual pinto beans sleep quickly slyly bold instructions. blithely unusu 840|Customer#000000840|9wA4l70okCEW3GnYQvlIAXetDgD50l|22|32-802-156-1748|6799.00|AUTOMOBILE|usly according to the slyly regular pinto beans. blithely regula 841|Customer#000000841|pSISKRxFIgVasW71fLyaEuODHZQLBfd6E c7jo|9|19-556-695-9964|884.20|FURNITURE|ly. slyly special instructions wake. carefully even accounts sleep quickly slyly silent deposits. s 842|Customer#000000842|stRBRwdFCkdN|0|10-272-126-1413|8635.49|FURNITURE| silent pinto beans. bold, pending platelets sleep furiously express theodolites. fluffily express reque 843|Customer#000000843|OEJ7AElvHgEtDpjcrXgLK|14|24-979-584-7928|675.73|HOUSEHOLD|ronic somas. accounts was among the carefully unusual requests. unusual pinto beans are blithely regular depths. s 844|Customer#000000844|1nUzjsH9HS1sPAGLwDIom9IESivLeEh1BvyynjU|11|21-285-410-4046|2954.90|BUILDING|ymptotes. ironic, unusual notornis wake after the ironic, special deposits. blithely fina 845|Customer#000000845|fIq5p0GpDtw6FIsPMdbqNYgfSw3gL9ep|13|23-125-871-9246|6898.89|AUTOMOBILE|deposits above the deposits wake bold instructions. special accounts cajole. ideas along the regu 846|Customer#000000846|EWQcGkx0CU5ZGUNcTV9bBCQ4qnuKsHC|18|28-766-714-9136|7650.73|MACHINERY|s. quickly thin requests against the ironic, bold requests haggle above the final foxe 847|Customer#000000847|oR9VbMI LkR2GUv4MRmylhb|23|33-645-447-3944|4822.08|HOUSEHOLD|haggle carefully. pending, bold foxes play carefully ironic frets: slyly unusual ins 848|Customer#000000848|a nIm5Bk7 RMqMZ6|15|25-655-714-7125|5685.59|BUILDING|st furiously blithely pending packages. deposits across the 849|Customer#000000849|kqRGW2JQtTM,a6 DzJMgcU9U|24|34-718-798-7751|9670.64|MACHINERY|ly silent ideas. fluffily even packages boost carefully fur 850|Customer#000000850|GdhPVh9rkPqSt v17ZuxIlx8c1N8G|22|32-546-203-4000|7576.55|FURNITURE|ly ironic accounts whithout the regular waters are blithely abo 851|Customer#000000851|H9HRTaOz4yI9elrUUiS|9|19-678-843-9850|1144.23|FURNITURE|cross the final packages. blithely ironic excuses serve slyly about the final accounts. fluffily 852|Customer#000000852|fR1Cq8d6m,zJwS3FFrOBV7u|5|15-252-941-6247|1098.98|AUTOMOBILE|unts thrash quickly through the express, regular theodolites. 853|Customer#000000853|U0 9PrwAgWK8AE0GHmnCGtH9BTexWWv87k|2|12-869-161-3468|-444.73|MACHINERY|yly special deposits wake alongside of 854|Customer#000000854|flSR,SlEXwxrDcm3uedfK1Oiq,c9mZI|0|10-544-967-2382|3393.86|AUTOMOBILE|t slyly after the furiously even escapades. carefully regular pa 855|Customer#000000855|i8mS 0Plk2tI8HG1Mnzj8v5RIl5JqmwTSW2Wq|1|11-379-392-2701|3550.49|BUILDING|s. carefully final deposits detect furiously slyly even packages. final, special accounts eat along 856|Customer#000000856|X4U7LH4YtDzephie|15|25-336-316-9641|6988.55|FURNITURE|y bold pinto beans according to the pending foxes 857|Customer#000000857|TFCCMlSEyrItYvEZy068NhrUxJz|5|15-856-649-3113|7537.16|AUTOMOBILE|g to the deposits cajole furiously final deposits. furiously silent dependencies agains 858|Customer#000000858|8qSqHq,Fc378KIPPbTNmaL 0bpAwO|8|18-509-453-8977|-829.37|BUILDING|, special accounts. ironic, pending requests might cajole flu 859|Customer#000000859|yp6dlWnpbzQboP5Xj8W|17|27-379-135-4463|6737.18|BUILDING|uffy foxes boost about the eve 860|Customer#000000860|t0GtEsh39KvInVMH7CDn0xi|12|22-895-319-1388|6836.89|BUILDING| deposits above the blithe requests maintain furiously slyly even packages. slyly regular requests boost. blith 861|Customer#000000861|GvlCZ4fJYbHTOTYczE1iH2k6K9edUyNxWZsl6x|14|24-939-734-5650|-336.80|BUILDING|arefully regular requests unwind slyly 862|Customer#000000862|zkaZ,iOp8t9MBK9T,JIPGL5hmBmK,xjkjHEP|2|12-479-479-6941|2338.92|MACHINERY|yly regular foxes sleep. quickly ironic foxes use furiously express deposits! carefully regular 863|Customer#000000863| oaBBxuQLKPawG1yqOc7cyVhOezCy|3|13-801-392-5922|5274.52|AUTOMOBILE|d accounts detect. bold pinto beans against the slyly ironic 864|Customer#000000864|lKcUKtfu0myNF5msATCFVHHXfMg3,cdb|12|22-897-966-6672|8932.76|AUTOMOBILE| instructions detect slyly. never pending dependencies use slyly enticing accounts. slowl 865|Customer#000000865|UIv4t5cbA7j1ftOSatj2EKah3p|9|19-245-153-1471|3016.38|BUILDING|nic deposits print blithely slyly final accounts. special, express accounts are fluffily a 866|Customer#000000866|u,b0GdP7dZg|20|30-436-285-7224|6265.88|FURNITURE|ts. bravely express pinto beans after the blithely bold requests x-ray busy orbits. furiously pending 867|Customer#000000867|KmET9DxfPVs15pIUUWQ|14|24-522-194-4543|5680.23|FURNITURE|t according to the careful packages. regular 868|Customer#000000868|cFG8Fa5h1e uvHJ9pgZAO|22|32-850-421-3334|7616.48|FURNITURE|the requests. regular dinos at the even requests use furiously around the sheaves. slyly final theodolites bo 869|Customer#000000869|PDGU7BXDNXAo0vmo7QtDm,yCMVcD|2|12-379-344-7132|1228.06|HOUSEHOLD|. final packages wake slyly? blithely even deposits haggle carefully regular packages. unusual requests 870|Customer#000000870|6wGPZX1SbXLGtweqx8jK|2|12-953-532-5903|1970.76|BUILDING|es cajole slyly. furiously even asymptotes are furiously regular packages. special, final ideas s 871|Customer#000000871|KcLmBKitbx7NvU7bpu9clIyccxWG|20|30-933-714-8982|-395.89|HOUSEHOLD|ts. blithely silent courts doze. regular atta 872|Customer#000000872|vLP7iNZBK4B,HANFTKabVI3AO Y9O8H|17|27-357-139-7164|-858.61|BUILDING| detect. packages wake slyly express foxes. even deposits ru 873|Customer#000000873|XFnr9C2bANXL|6|16-375-385-5712|-797.38|AUTOMOBILE|lithely ironic, silent forges. furious 874|Customer#000000874|rdo knkGhtvpH6dbLkQon8QsrK1z4LFUpaVDTOn2|0|10-886-494-4217|5391.74|AUTOMOBILE|st the deposits. furiously even requests cajole slyly. bol 875|Customer#000000875|8pQ4YUYox0d|3|13-146-810-5423|-949.28|FURNITURE|ar theodolites snooze slyly. furiously express packages cajole blithely around the carefully r 876|Customer#000000876|NMzgtdV zCuRIMK0vV,DP9ynDd6Z9X3T|20|30-320-481-3076|4367.63|MACHINERY|ronic requests haggle blithely. slowly ironic ideas against the fluffily 877|Customer#000000877|uYO2BKogrHcOcHEgzjlmZAa1QYyR45i8|16|26-400-912-7812|1755.33|MACHINERY|packages cajole carefully. slyly regular pinto beans bo 878|Customer#000000878|hUCH2juGwk4OtThyY8p35Hi0,IfOGA|15|25-465-180-9022|8781.53|FURNITURE|the quickly pending deposits cajole care 879|Customer#000000879|EJcG18hFrS0SPT0yvl1b|2|12-878-466-6505|2235.20|HOUSEHOLD|ntegrate by the carefully special requests. dolphins sleep e 880|Customer#000000880|ogwHmUUFa1QB69pAoYAAoB0rjbdsVpAQ552e5Q,|8|18-763-990-8618|-77.63|FURNITURE|regular requests. regular deposits ar 881|Customer#000000881|XJ94RTR2oXI omeh|4|14-127-261-7876|2141.71|FURNITURE|pinto beans. asymptotes about the slowly even theodolites are pending requests 882|Customer#000000882|hsKaXwHCLD|2|12-437-842-6799|1650.12|HOUSEHOLD|ts. quickly regular packages alongside of the furiously silent theodolites nag slyly after 883|Customer#000000883|qVQ8rWNU5KZYDcS|3|13-526-239-6950|479.96|FURNITURE|uctions are carefully across the regular, regular asymptote 884|Customer#000000884|5KyisO0Tv9ZtlJhzyI7vAe88|21|31-483-489-6172|1601.60|FURNITURE| among the quickly express theodolites. accounts sleep furiously along the special, pending depende 885|Customer#000000885|nNUbC73nPBCKLg0|5|15-874-471-4903|-959.94|HOUSEHOLD|sits impress regular deposits. slyly silent excuses grow 886|Customer#000000886|QOTGbGPJjNPD7IrfAILA1da|12|22-771-691-7229|1194.33|AUTOMOBILE|slyly even foxes according to the pending, special accounts use carefully against the courts. regular, 887|Customer#000000887|CoInl1fmf5MjYn15AdA|1|11-136-651-8293|7009.42|MACHINERY|ording to the fluffily regular foxes nag fluffily instructions. thinly careful accounts around the furio 888|Customer#000000888|3vlJp0W8cniEXV|12|22-855-455-1154|6358.04|HOUSEHOLD|sleep carefully quick pinto beans. packages hinder beneath the instructions? ironic, unusual theodoli 889|Customer#000000889|pLvfd7drswfAcH8oh2seEct|13|23-625-369-6714|3635.35|FURNITURE|inal ideas. slowly pending frays are. fluff 890|Customer#000000890|rvsLCrRX9z,IPaaF9kqwvLLxueobbPiH4pz|4|14-938-708-4678|3329.21|BUILDING|ironic accounts cajole ruthlessly above the carefully pending accounts. quickly regular theodolites cajole. iro 891|Customer#000000891|AW0m6YSpe,BNPHvBj|11|21-439-958-7518|6032.18|FURNITURE|sits. final foxes run. quickly pendin 892|Customer#000000892|N KwiRAwIk6KL9WJ6vt0G|9|19-589-784-1249|4799.98|FURNITURE|ses are carefully. quickly regular theodolites cajole. carefully express accounts wake sly 893|Customer#000000893|W6m7LofOZoik72ku|13|23-827-724-6816|8250.87|BUILDING|ets. ironic instructions nag even, regular courts. slyly iro 894|Customer#000000894|5y7m8Ts4kDf|16|26-844-679-1540|4483.42|AUTOMOBILE|carefully regular accounts sleep carefully slyly ironic dependencies. blithely ironic accounts wa 895|Customer#000000895|MDaJr8ekGTS79bS7CH8f1WgWPU|0|10-933-819-2037|904.43|AUTOMOBILE|ggle final packages. slyly regular instructions affix fur 896|Customer#000000896|Tu1ZBNgiSEL9Ns|0|10-425-565-3199|7659.72|AUTOMOBILE|affix carefully unusual requests. furiously fin 897|Customer#000000897|nW1X1Hl9uWycuBEu3F3|6|16-988-776-4568|1999.42|MACHINERY|riously regular ideas sleep into the final, unusual 898|Customer#000000898|JrvrIEzAre,VJzJCi3SEmib1T2,YVXVvOGxaVZwR|3|13-265-738-4361|8137.24|AUTOMOBILE|s haggle around the special dependencies. slyly regular requests are according to the idly sly ideas! fluffily final 899|Customer#000000899|Th5XO5ImeCe9nHFQfQMCkNcmf5WHSeYQaR5TJ|2|12-594-534-9654|8605.53|FURNITURE|rding to the furiously unusual accounts. express, express accounts nag furiously. ide 900|Customer#000000900|kEhE1Y,OoZTDv,Auh d5G ALINN0rND|1|11-422-328-1612|3195.39|BUILDING| packages sleep slyly around the quickly special packages. final accounts are furiously. bol 901|Customer#000000901|QUyXt94YM6Ou6rDqK|12|22-202-667-4372|938.35|HOUSEHOLD| foxes maintain. theodolites sleep above the regular deposits. slyly bold excuses boost careful 902|Customer#000000902|A1hnMyYPSkXf7QgOPD2H|4|14-209-883-5797|5858.48|AUTOMOBILE| sometimes regular epitaphs. furiously regular gifts against 903|Customer#000000903|URTiQupkhObWG39,kZ3CfU|7|17-706-779-2078|509.23|FURNITURE|kly doggedly even instructions. regular, regular accounts along the even, bold packa 904|Customer#000000904|YdJEbNygDU6DrgWXQY6orasq|5|15-940-929-4572|9562.82|BUILDING|nic dolphins alongside of the slyly final ideas run 905|Customer#000000905|f iyVEgCU2lZZPCebx5bGp5|3|13-803-156-2231|-600.73|BUILDING|slyly closely ironic dolphins. blithely ironic asymptotes haggle carefully ironic theodolites. furiously 906|Customer#000000906|1Uavkms1A5z|6|16-594-569-6627|-613.45|HOUSEHOLD| accounts. furiously silent ins 907|Customer#000000907|UeVLwnnpccsG1pbQmN7pzD|10|20-501-816-7673|5751.31|HOUSEHOLD|sits haggle quickly above the excuses. slyly ironic packages print furiously. carefully pending reques 908|Customer#000000908|Fa5bchMKUMsaNKOXAiu9pX ME|12|22-814-669-9320|3215.96|AUTOMOBILE|quickly express packages. blithely fina 909|Customer#000000909|b 2X284A5AGpt8skuYwMvTyK68srMKikPst6X|24|34-717-350-9722|5565.58|MACHINERY|se carefully around the special, regular requests. ironic theodolites cajole quickly theodolites. regular, ev 910|Customer#000000910|Qg8TJTCT1mJ9H|9|19-899-463-4292|5794.69|BUILDING|uctions. silent requests after the regular theodolites haggle furiously across the ca 911|Customer#000000911|VS0fia,lJ RvUf68 l4Unv,Vx|13|23-121-746-7339|6364.60|BUILDING|gular foxes! permanently regular packages wake. quickly regular deposits detect blithely. carefully express 912|Customer#000000912|dQA12NEPQK1A5mvD|14|24-348-437-3105|3861.36|FURNITURE|sits use slyly carefully final dependen 913|Customer#000000913|aohNRUjsMbBNE1Ax|14|24-500-946-3315|6935.16|FURNITURE|y blithely final dugouts. accounts wake accounts. asymptotes above the even 914|Customer#000000914| LErnJFeOuDeMgvVzPKHS|5|15-915-758-7313|1230.79|MACHINERY|accounts. slyly final attainments boost slyly express, pen 915|Customer#000000915|mtGezp1BRzcfPVl,1,G8Wl|0|10-452-398-2445|3776.53|AUTOMOBILE|r ideas. final dependencies haggle fluffily. express ideas behin 916|Customer#000000916|9Zo7nkmzJla4Q4PE5mbw|23|33-511-587-8754|130.59|MACHINERY|sual ideas cajole carefully ironic excuses. final platelets use slyly. quickly special instructions nag. bold escapa 917|Customer#000000917|KZ TS0omSFmUQkIOmzlXhwQS,OcF3wzz5|1|11-100-917-5264|3679.65|HOUSEHOLD|arefully fluffy pinto beans. enticingly silent requests affix furiously busily regular ide 918|Customer#000000918|WSptkDdGQQyJ6|19|29-992-318-6425|-155.06|MACHINERY|s boost furiously slyly final dependencies! fluffily regular sheaves nag fluffily. slyly even courts sleep sl 919|Customer#000000919|cHGtsqVvXRiK|9|19-537-180-2200|9774.97|MACHINERY|lar instructions. blithely final deposits haggle furiously bold pinto bean 920|Customer#000000920|oDBFWKIP6M6OlYRPmqCBkVpVSj6uFa|2|12-905-464-3299|990.58|HOUSEHOLD|. ironic, pending frets haggle carefully. ironic, 921|Customer#000000921|XYBVDdDifSYrW gUeDPhITqMjpjtbnc|8|18-765-936-2316|3651.09|HOUSEHOLD|g the furiously unusual theodolites are carefully accounts. slyl 922|Customer#000000922|Az9RFaut7NkPnc5zSD2PwHgVwr4jRzq|7|17-945-916-9648|3869.25|BUILDING|luffily fluffy deposits. packages c 923|Customer#000000923|ckBLWkfYtn2VZXWWqUGbDgbP|11|21-476-142-5086|7462.20|HOUSEHOLD|s. furiously express depths boost. regular requests boost furiously. even, unusual deposits cajole blithely. expre 924|Customer#000000924|yKEtokQYXiuSSh8ZP5|15|25-518-232-9865|4212.53|BUILDING|ake slyly furiously even deposits. express pinto beans are carefully quic 925|Customer#000000925|jn Razhw70hWtHN4iRBWsf1UmrFUPn36Ni562ex|10|20-753-609-6699|1939.39|AUTOMOBILE|ironic accounts cajole carefully. even, expre 926|Customer#000000926| 3b8K2YhfbGDJOpSAUrvq82MnkhHBdwL|16|26-783-803-1329|274.50|AUTOMOBILE|ly quickly unusual foxes. fluffily regular ideas among the regular plat 927|Customer#000000927|Uy7xvOwo4Ndha1tSxDKrQ gXUTdS ,YDqwE2YSO|17|27-332-891-1391|2417.65|MACHINERY|es affix quickly carefully regular platelets. slyly special theodolites would sleep furiously after the special, 928|Customer#000000928|A9 UduEb48ffOe27FxMXF|21|31-508-509-6393|8330.70|HOUSEHOLD| instructions cajole fluffily alongside of the blithely regular re 929|Customer#000000929|c dPfaAmmoLjR3m|23|33-487-459-1026|4079.18|MACHINERY|ously silent foxes cajole slyly pe 930|Customer#000000930|84jHYR8u2XvhyT|19|29-562-904-5451|4787.20|FURNITURE|sual ideas according to the furiously regul 931|Customer#000000931|M,lWVafqdRIO, WnAyLLt|0|10-349-498-1720|2409.69|BUILDING|s the carefully regular packages: pending 932|Customer#000000932|HN9Ap0NsJG7Mb8O|13|23-300-708-7927|6553.37|BUILDING|packages boost slyly along the furiously express foxes. ev 933|Customer#000000933|V0SPv2VbrNo7Pj|14|24-623-803-8018|8541.16|MACHINERY| accounts haggle quickly against the blithely even accounts. deposits sleep blithely quickly p 934|Customer#000000934|UMAFCPYfCxn LhawyoEYoU9GZC7TORCX|12|22-119-576-7222|-592.69|AUTOMOBILE|fluffily requests. carefully even ideas snooze above the accounts. blithely bold platelets cajole 935|Customer#000000935|XkVT4jvetY4JV76IAkd91sSp9CqsICE|0|10-724-445-8323|2531.25|MACHINERY| furiously pending ideas. daringly 936|Customer#000000936|hwJIFpxofea6CLEbWZFsWUxNrGFLANp|11|21-100-538-9635|3650.90|MACHINERY| furiously enticingly final foxes. pending requests wake quickly according to the slyly regu 937|Customer#000000937|usrG6ohdPROyd98c9|23|33-869-990-3946|668.51|AUTOMOBILE|ickly alongside of the express, express ideas. sly 938|Customer#000000938|wrq9S3rEW8zXUVCXpa7uKi|12|22-157-321-7590|2584.52|BUILDING| the quickly special accounts are regular patt 939|Customer#000000939|jYaDdfxAlL1aVKPfN|19|29-627-844-1293|8059.51|FURNITURE|ages integrate carefully. sometimes even sheaves wake sometimes unusual ac 940|Customer#000000940|T7ROXBXdajS,vkwy3VuC8wNvA|19|29-958-573-1004|253.59|AUTOMOBILE|ent accounts. slyly even accounts breach across the dolphins. quickly regular pains dazzle carefully. sl 941|Customer#000000941|297w97UgOfpV3pv2QniJUWBKq0BRpcawOfpj|19|29-745-875-1061|2990.18|FURNITURE|e furiously along the slyly bold pains. ironic, ironic foxes affix quickly even deposits. packages caj 942|Customer#000000942|y0OKxFyfXeYuklJDY9RwujlNIC2ETXo9HxZCVhg1|18|28-560-449-7675|5898.17|HOUSEHOLD|lar depths! carefully regular pinto beans after the deposits wake about the packages. final, final instruc 943|Customer#000000943|74dBRGOKLFEQEqCgH2x8WGL9tubtgJAbHu|5|15-483-251-8603|7541.05|FURNITURE| quickly along the silent, express braids. quickly pending packages boost fluffily furiously regular foxes? a 944|Customer#000000944|8lO9F4WK6PKWXiocyE,ojIfPNfVY8|4|14-485-139-5142|9454.78|BUILDING|ilent pinto beans are according to the regular, final somas. idle, bold foxes was. som 945|Customer#000000945|300zKNJ9lg|15|25-542-662-1673|9615.39|BUILDING|. ironic deposits haggle among the carefully regular excuse 946|Customer#000000946|ufHQOmRhQoLSiyfQ|21|31-152-357-9762|2990.73|FURNITURE|ly after the furiously regular asymptotes. foxes play quickly ironic packages. dinos along the silently expre 947|Customer#000000947|JnzDRxqCwjRPyeq70wrxzKWLXI|22|32-838-393-6825|4092.24|BUILDING| blithely final instructions grow furiously ironic requests. furiously special theodolites sleep after 948|Customer#000000948|yxBr7nLGxxwECEk|17|27-125-968-3750|4346.90|AUTOMOBILE|he final, even deposits. furiously special pl 949|Customer#000000949|tOBbBIfhWM lNI3YxTYH8Or8Ki|2|12-391-316-1861|5340.67|AUTOMOBILE|pecial, pending dolphins. slyly final ideas boost quickly. carefull 950|Customer#000000950|mi3o6cp47mo8Miqh9d R1XWStjxatcQqHQZW|15|25-849-760-2017|9609.77|FURNITURE|s wake quickly after the carefully brave multipliers. regular excuses wake care 951|Customer#000000951|PnC4Xlds,v|0|10-813-916-8297|7499.47|FURNITURE|ts. permanently special accounts mold quickly. requests boost slyly bo 952|Customer#000000952|jg0YsHARdoULvVtP2vGHLVoAfWKFNz6QdTeAi|24|34-527-524-9172|3710.14|AUTOMOBILE|slyly regular ideas. even theodolite 953|Customer#000000953|5HJQ8UzSSl1PJv28MpZqWvNwUm|22|32-829-961-2870|6292.06|BUILDING| according to the carefully bold dependencies use boldly slyly express deposits. car 954|Customer#000000954|rr0Gz2iuYNuLgrIcLCdi5Zr2SnY8,wpzD9A|19|29-639-437-3775|5740.00|FURNITURE| haggle furiously. furiously ironic 955|Customer#000000955|FIis0dJhR5DwVCLy|0|10-918-863-8880|138.31|AUTOMOBILE|ts cajole quickly according to the pending, unusual dolphins. special, ironic c 956|Customer#000000956|aI12bsLSd1Y4dIx2Me5BLbGDCZPn |20|30-627-947-8311|1587.14|HOUSEHOLD|eans. regular, regular foxes wake furiously carefully even pinto beans. furiously r 957|Customer#000000957|9F8p,XsLLxyiZ3b8NN|19|29-941-553-8245|9076.68|FURNITURE|al excuses boost to the ideas. unusual requests are across the slyly 958|Customer#000000958|OrOUBBV7NlzVFXtuSOECmQFOkw8r|5|15-455-954-2914|1791.65|FURNITURE|s deposits. blithely even packages sleep carefully ironic deposits. quickly ironic accou 959|Customer#000000959|O FdrkZxCx PK|4|14-546-329-6898|3266.14|FURNITURE|press accounts wake busily after the furiously final theodolites: fluffily final deposits above the carefully iro 960|Customer#000000960|meekxaMlz5c1uE3wV7a,u h WcU,1OJz|4|14-664-604-8633|1932.59|HOUSEHOLD|s are carefully after the permanent foxes. fluffily 961|Customer#000000961|5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn|12|22-989-463-6089|6963.68|MACHINERY|e final requests: busily final accounts believe a 962|Customer#000000962|lDp572JGdrL34kB YOQuC|4|14-792-232-1645|7557.00|BUILDING|. carefully brave foxes wake furiously final orbits. furiously pending theodolites along the bold the 963|Customer#000000963|40EdWkddaWhQyiQ6FfUo8VOZwgb MetJ2jV,QPL|13|23-921-332-7635|2557.49|HOUSEHOLD|ns snooze slyly daringly pending instructions. regular requests above the regular, even requests sleep blithely fin 964|Customer#000000964|ZnBNwMqvW7y3FSn6025pwkzgTDfsG2A|12|22-974-772-2802|4756.58|FURNITURE|al accounts haggle blithely! regular platelets haggle blit 965|Customer#000000965|UigBc,9d1iLtQAVatnWACSDc9mNx0mYl|2|12-893-735-6415|4768.80|MACHINERY|lly enticing pinto beans haggle requests. evenly express grouch 966|Customer#000000966|V9c8SR8WK7wEd|8|18-539-933-5176|1283.26|MACHINERY|ts. regular accounts cajole about the ideas. slyly unusual idea 967|Customer#000000967|xKdAl6HSWvAmptzHgQHX3cMmxZDhfyrMqx|23|33-687-917-3598|5710.41|BUILDING|iously quickly silent ideas. blithely pending pinto beans except t 968|Customer#000000968|eu 5FA1WHs9jq0pcdlVVA|0|10-470-740-2657|8921.97|BUILDING|ic foxes haggle slyly according to the dependencies. even, regular acc 969|Customer#000000969|N9NSGc0Bj6FlSw3d9k GI7VAd1jW|8|18-148-790-2039|8601.63|BUILDING|lithely according to the fluffily silent patterns. furiously fin 970|Customer#000000970|DXEgz7JHSFW401|14|24-266-486-1615|3623.60|MACHINERY|ronic requests sleep slyly at the slyl 971|Customer#000000971|z29DUY Utsi6mWKI|1|11-256-718-6928|3914.88|AUTOMOBILE|ular theodolites haggle carefully: f 972|Customer#000000972|ImKvHrrNc3rfWejksbCPyIQ|4|14-405-229-6174|4453.46|MACHINERY|deposits: slyly regular deposits among the furiously bold asymptotes are furiously along the slyly even 973|Customer#000000973|FT4jTOdVCpmYW|0|10-749-928-5415|3229.18|FURNITURE|sly special requests integrate carefully along the special foxes. regular, silent 974|Customer#000000974|7RcY6fOjTMbbOnVaFV,,6Dk5FIiHGrCpwXJNI|20|30-473-948-7149|7826.10|HOUSEHOLD|uickly. regular, ironic waters sleep blithely. blithely regular foxes are blithel 975|Customer#000000975|qPFceGMB0xDjY6BhTGdIxe2Z F4MVuKIXHqQ1|16|26-428-220-2070|4364.06|FURNITURE|furiously express packages. even, bold sheaves haggle fluffily. slyly ironic accounts wake slyly across the quickl 976|Customer#000000976|I78UJ2ks3sbcd0c2NQ7aH|19|29-436-660-4732|7772.85|BUILDING|special requests wake carefully regular somas. special theodolites wake regular, unusual pinto beans. furi 977|Customer#000000977|JcKxPwHPM7akg5IiCs,ZVAfO73KE3|2|12-602-807-5055|311.00|FURNITURE|lly against the busily unusual requests! busily even requests haggle blit 978|Customer#000000978|zpvQ6LYE89Inl40Yz,7NJ|24|34-261-243-2624|-50.51|BUILDING|ely unusual packages nag fluffily above the quickly regular requests. regular accounts run. blithely 979|Customer#000000979|DvzUxD35ohKtUnalLGO9kDsCzZxtfcjO|18|28-113-574-4962|7055.13|MACHINERY|. accounts wake carefully special accounts. de 980|Customer#000000980|UsrigSqZBnmbXhXNR6ibloq60qHBUj42kwX|1|11-572-281-8212|4586.33|AUTOMOBILE|st have to integrate above the regularly regular accounts. regular, final waters breach blithely 981|Customer#000000981|pM4DXkl6Y,7S6a6jlJf8dZogp9QOdv|18|28-202-962-8429|3383.26|MACHINERY|ts doze quickly. platelets are quickly agains 982|Customer#000000982|EN9aD5Xgh2q|23|33-460-986-9418|1437.55|FURNITURE|lithely express requests along the carefully pendi 983|Customer#000000983|9jgCxRufEbwbGwW0PmG1RDIOwCUYlHs8z|7|17-283-610-6143|2902.95|BUILDING|mas cajole furiously across the fluffily special deposits. pending foxes sleep regular, silent packages. ironic pla 984|Customer#000000984|fgAELFO9RS 6q9|11|21-247-588-5181|2811.97|AUTOMOBILE|t daringly against the even foxes. furiously silent forges sleep furiously busily ruthless requests. exp 985|Customer#000000985|0uAMe1ICB,wts4STD4eLL|23|33-408-194-5161|2701.21|HOUSEHOLD|gular deposits among the thin instructions haggle since the furiously final packages. ideas use. regular i 986|Customer#000000986|Cei2QidV0GC3OQWfJTNHLYPd|1|11-537-225-3800|178.73|BUILDING|cies impress blithely furiously final package 987|Customer#000000987|SO 0UTuH26eduKI|8|18-887-394-2506|9850.64|AUTOMOBILE|eposits. even packages cajole boldly bold ideas. even, regular accounts haggle. packages among 988|Customer#000000988|3BNYKEUyMbzfb40SEr 8OTb|3|13-862-722-3298|7746.97|FURNITURE|he courts. carefully silent courts impress. carefully close ideas run slyly pinto beans. even, ironic accounts are a 989|Customer#000000989|pKuixWbH6XZFJY0uZjGi0oRaH1Xl|10|20-646-819-6827|5453.74|AUTOMOBILE| accounts. fluffily regular requests use carefully after the furiously special instructions! slyly 990|Customer#000000990|uF idg4bq8Ij7ghxJ5KuTnU8w|0|10-403-137-1064|6988.49|BUILDING|dolites for the fluffily bold 991|Customer#000000991|dK1Gzw1glT|21|31-977-971-6175|6533.53|AUTOMOBILE|n somas. slyly ironic instructions solve quickly at the final dolphins. requests cajole carefully 992|Customer#000000992|Vbi1NGfPeKw,XU|5|15-262-535-3924|5027.75|MACHINERY| across the regular, pending requests. slyly ironic accounts wake furiously about the pending, regular 993|Customer#000000993|56K JjC bMcgbXlJA4KI Icu uggsRoviMQm,F|7|17-494-757-5759|8421.87|FURNITURE| according to the slyly bold accounts. 994|Customer#000000994|sZjdeW4LT9EKopmlv3M Xbnbe3gXQ9JkoxPv |16|26-638-159-5836|7461.27|HOUSEHOLD|ake furiously across the quickly idl 995|Customer#000000995|5tCSAsm4qL5OvHdRZsiwSlVTdqPZws3f|13|23-272-700-1002|-341.79|BUILDING|wake slyly fluffily unusual requests. stealthily regular pinto beans are along the slyly final dugouts. slyly 996|Customer#000000996|yjrSjcG z0Rm5PYrVMFTrU pFRMw|22|32-902-625-1946|6450.78|AUTOMOBILE|hely against the final, brave asymptotes. final ideas haggle slyly bold pinto beans. slyly unusu 997|Customer#000000997|85KMCT2D2RIGayG99ozpk85ppHE6i9gJE|17|27-218-645-5219|367.03|FURNITURE|ven asymptotes. carefully regular packages are blithely. special requests according to the care 998|Customer#000000998|fHRMFCGphazw9KvR1,EmNOUBG|17|27-951-935-6514|6679.20|MACHINERY|ular courts nag quickly unusual, sly pinto beans. special foxes thrash blithely up the foxes. 999|Customer#000000999|r2SFEmfqrRu3M7ouE4zvI2ApOAtD|16|26-876-956-1302|403.89|BUILDING|riously special instructions ac 1000|Customer#000001000|hzM1shTwWlLuk|22|32-730-275-2976|-881.70|BUILDING| closely against the slyly special deposits. regular, ironic p ================================================ FILE: src/test/regress/data/customer.2.data ================================================ 6001|Customer#000006001|e,jhgkgWnN|3|13-247-356-9056|9139.66|FURNITURE|eans haggle blithely. requests hinder furiously alongside of the slyly final foxes. carefully even instructions wake 6002|Customer#000006002|ZnHuM0Y9nONIKR5TFrHGuJnmxq9GLULVLitL8|20|30-330-985-9161|4400.25|HOUSEHOLD|final instructions. quickly regular epitaphs are according to the sl 6003|Customer#000006003|TD5JS9ULaDBUHIy5J7FfT|16|26-402-596-3552|-30.04|MACHINERY|yly furiously regular instructions. requests sleep about the carefully special accounts. 6004|Customer#000006004|Oo0mrAFH,KrRuF0eCxbklLZC|4|14-523-907-2485|-987.86|FURNITURE|the regular theodolites. carefully silent deposits hagg 6005|Customer#000006005|Qhudh0ioaEafuJ Rfr6DAUqtKkZ33nby|0|10-930-477-2232|4911.63|MACHINERY|p fluffily fluffily pending notornis. requests ser 6006|Customer#000006006|5VfR3EWqbrt0zdkyy8 |8|18-760-236-6029|3751.22|FURNITURE| instructions wake about the slyly ironic deposits! regular excuses wake. furiously pendin 6007|Customer#000006007|eFTUQxFkePYCDkt2YiTlP1oUNww1lUPA|16|26-238-936-7519|3752.79|MACHINERY|quickly special pinto beans serve carefully. blithely ironic requests cajole 6008|Customer#000006008|S1yv Nmjxkhb2yQU7sEX ,poB5f0ijkeRWsY|2|12-448-135-5947|8049.94|FURNITURE|uses boost carefully throughout the blithely express theodolites. furiously i 6009|Customer#000006009|tJ3M102q9VHZ0jX|15|25-821-240-1667|4357.29|AUTOMOBILE|endencies. carefully silent attainments cajole according to the ironic multipliers. final dependencies integr 6010|Customer#000006010|Oyw3CVbhnakZby zVlRd9jZPUygik65nK6UCg|23|33-702-784-8705|3887.84|HOUSEHOLD|uickly final pinto beans kindle instructions. slyly regular instructions sleep carefully. slyly ironic 6011|Customer#000006011|wP2fviXCIC9E6kZ2StshJetAa9x0vNMBZ0I|5|15-600-545-9353|895.26|HOUSEHOLD|en requests wake against the fur 6012|Customer#000006012|mSI9TkywG,fJgIV,mr24o,JupCD36mw0|14|24-197-638-5900|7639.44|FURNITURE|ourts against the slyly final ideas sleep quickly pint 6013|Customer#000006013|NOTH5 kyVefl4bJA|20|30-137-646-6576|2051.61|MACHINERY|nal packages are quickly regular, even instructions. even, ironic deposits sleep; slyly 6014|Customer#000006014|ZB9wS3fmWrQi8cZil6mzcKYo0PQLXrKw2ZM Rv5|24|34-570-696-6422|5302.25|BUILDING|y blithely express accounts. furiousl 6015|Customer#000006015|AAc,PFYt20otDa85nT6wDwOVJ|21|31-376-506-1388|1950.33|FURNITURE|riously theodolites; fluffily enticing 6016|Customer#000006016|Of93IVKgMknchAkEf16|22|32-695-578-4478|8103.98|BUILDING|requests cajole packages: quickly express accoun 6017|Customer#000006017|013MNSbWeJML ohf|23|33-579-732-1955|-402.46|FURNITURE|iously. furiously bold theodolites cajole blithely regular 6018|Customer#000006018|XNLYh8cfpNGERz1O|0|10-261-311-5038|9806.55|AUTOMOBILE|ng the blithely pending platelets detect s 6019|Customer#000006019|5,lbJvlCV l8V|18|28-475-771-1730|3695.26|MACHINERY|carefully. furiously pending p 6020|Customer#000006020|SIdmPkG3HQKTwU4p5hGrJttj,km|16|26-990-181-4165|9256.73|BUILDING|cial deposits x-ray after the blithely regular requests. final requests 6021|Customer#000006021|r5k7syDoG41,Uhtpoii9Hp,oxJKKME9|21|31-358-419-1438|3002.73|AUTOMOBILE|ly pending accounts about the blithely ironic package 6022|Customer#000006022|z9Gbi2AkLS0s6HNIDyerHNGE7V98G|10|20-331-281-6929|7892.25|BUILDING|tructions detect among the bold, regular accounts. ironic ideas inte 6023|Customer#000006023|uz9mw3nZBoJ0j2zkMueLynANFVN|21|31-825-465-7964|5150.58|AUTOMOBILE| must have to sleep furiously at the pending, regular accounts. fluffily unusual deposits nag along the carefull 6024|Customer#000006024|LZ80EMXjRz7JULb75V5n3qg|3|13-664-584-4042|4732.68|AUTOMOBILE|cajole blithely about the furiously pending theodolit 6025|Customer#000006025|cawvuhdgRy KaqlhXcWab y31A37F8IPT|14|24-233-488-3262|-815.07|BUILDING| final packages boost slyly above the blithely regular requests. slyly ir 6026|Customer#000006026|ius6eByivZ4BVYaESvB9p0bWsZ|12|22-822-409-2653|4993.97|AUTOMOBILE|xpress requests. carefully unusual accounts cajole blithely even ideas. carefully regular deposits are quickly iro 6027|Customer#000006027|hST46enQLI8TzdOmvA8J|7|17-178-716-9690|1657.83|HOUSEHOLD|deposits sleep slyly regular deposits. furiously ironic accounts boost carefully according to the carefully regular 6028|Customer#000006028|mL,IJFVI1MA9|0|10-424-273-2141|9173.51|MACHINERY|ckly ironic theodolites cajole after the quickly regular ide 6029|Customer#000006029|Bhz6SV,wpp|2|12-230-673-1285|1247.91|FURNITURE|ites wake blithely final, unusual accounts? quietly final foxes hinder. furiously express theodoli 6030|Customer#000006030|thigRH2AJJ5ay1akT2MoIWgGS3iH|0|10-385-880-2306|5585.25|BUILDING|uickly final deposits: even packages after the deposits wake regularly under the blithely ironic theodolites. furio 6031|Customer#000006031|ovto GrnDjTqwxdA0L|24|34-257-441-6562|6268.52|BUILDING|oss the carefully even deposits. furiously ironic deposits haggle furiously furio 6032|Customer#000006032|U9Y1LGOIyhU1r|24|34-124-269-1297|6098.48|AUTOMOBILE|its. pending excuses detect among the carefully ironic packages. carefully final gifts are. ironic in 6033|Customer#000006033|CY9eVCYfqrI1WlamyV,2h2fAuwRof4Vi|20|30-194-447-2847|8203.11|HOUSEHOLD| regular deposits wake carefully final accounts. regular, thin 6034|Customer#000006034|9C1trxxcizUyNm,rADOUt5UB|15|25-369-506-8757|4765.57|HOUSEHOLD|ding, even gifts about the regular, final requests sleep fluffily by the carefully silent dolphins. even, s 6035|Customer#000006035|pzz5CQ3wWzqmawu811Zfm|2|12-929-596-2859|1907.14|FURNITURE|es sleep fluffily besides the furiously qui 6036|Customer#000006036|oArQJruOuhJxRvLWjeDX3 h6CspeJwV7U6by|0|10-960-488-8572|8282.11|HOUSEHOLD| after the carefully even theodolites. furiously unusual pinto beans among the ironic theodolites k 6037|Customer#000006037|SnqQMacIw1UlhWOZib7iEU|6|16-600-315-6012|1463.98|AUTOMOBILE|encies cajole carefully according to the final, pending packages. regular, ironic packages b 6038|Customer#000006038| bIKiOVO1JVZiEtTF4T7HxjimXgL5|9|19-485-680-7672|7087.20|AUTOMOBILE|quests are furiously besides the blithely ironic requests. slow asymptotes cajole about the furiously 6039|Customer#000006039|a5rSHRtBDeEFAd8f,suDA8ve|5|15-453-414-1942|4616.61|HOUSEHOLD| pending instructions must have to sleep carefully 6040|Customer#000006040|al9P9X5hZsq4GVzXaf45nmYGre S|18|28-602-749-7354|6309.48|BUILDING|nstructions along the blithely final foxes solve according to t 6041|Customer#000006041|BFIr9VI8QcoMd4mjXtXH|22|32-422-646-4800|1445.52|BUILDING|ests boost furiously blithely ironic pinto beans. daringly ironic pac 6042|Customer#000006042|5HJbh0QShvTfZretlUSjRk|0|10-526-919-1776|4839.54|FURNITURE| express requests boost carefully. deposits boost blithely quickly ironic deposits. ironic accounts x 6043|Customer#000006043|04Ln0amBet|9|19-335-750-5425|2143.97|AUTOMOBILE| asymptotes haggle above the final gifts. ironic accounts sleep. pending, regu 6044|Customer#000006044|DOpmBtl1j5gVB36A57vtsI6Z|10|20-311-967-3855|6649.41|HOUSEHOLD|thely special packages. express pinto beans sleep about the final, unu 6045|Customer#000006045|FH8AwQ54fTohmWWZEFluKjNP|0|10-579-571-4767|5587.35|HOUSEHOLD| instructions. special accounts hinder quickly unusual depend 6046|Customer#000006046|3pqu0wwA NYnUQAx|1|11-901-156-9794|8566.39|FURNITURE|telets sleep around the accounts. busily special theodolites along the fluffily silent deposits use fu 6047|Customer#000006047|WJIOtP1XJSsFYl7yCfPo2|4|14-601-562-8081|5439.08|FURNITURE|y express, ironic deposits. final deposits nag slyly slyly bold excuses; blithely regular accounts affix slyly. ir 6048|Customer#000006048|DNtLkGdov3MnFfAGa40Pi6c6Y1r5j NF|24|34-996-904-3634|2634.46|AUTOMOBILE| sleep blithely across the regular deposits. accounts wake. slyly ironic gifts above th 6049|Customer#000006049|b1Wlcr2IYgYvlZWW3V,JNkpDg,Msiwmm9|23|33-822-276-9791|357.21|AUTOMOBILE| instructions. instructions boost fu 6050|Customer#000006050|orhXeGjD4RRH5QU|0|10-472-892-7595|6612.70|AUTOMOBILE|ts cajole quickly furiously express packages. evenly final theodolites use quickly 6051|Customer#000006051|k7txuddD z|4|14-674-349-1580|4706.51|MACHINERY|platelets. slyly bold packages sleep permanently along the c 6052|Customer#000006052|5R8 L5TJey,G7Ta7YGFnezUXaI4eIZu|21|31-792-184-4320|-624.49|FURNITURE|gular, final dolphins. sauternes against the slyly final dependencies cajol 6053|Customer#000006053|Yzq6dtXjEfbY PEVyUkJ|15|25-363-543-1931|7102.91|MACHINERY|r, ironic foxes. packages wake 6054|Customer#000006054|bXUQWTwB29ox1ganSx,QcxemZ|6|16-472-362-3917|4601.79|HOUSEHOLD|uriously after the silent deposits. blithely express instructions integrate carefully alongsi 6055|Customer#000006055|FnwMVA8yGuU SgVGARNz22st,8|9|19-663-939-5531|-631.98|FURNITURE|g to the final, regular requests haggle furiously quick foxes. regular requests p 6056|Customer#000006056|nyg076hIRk|2|12-687-122-1788|2218.75|AUTOMOBILE|se accounts. ironic pearls are 6057|Customer#000006057|R4Bt0gWYUrIshQjPi7UaKDrFs|20|30-448-516-3117|9526.94|FURNITURE|ly regular ideas haggle furiously enticingly final packages. regular, bold dolphins cajole slyly 6058|Customer#000006058|noUsJ1sGwBlm9KfvZ|21|31-818-798-9205|1401.74|MACHINERY|y ironic requests. silent deposits grow fluffily. regular, ironic pearls among the dolphins integrate even senti 6059|Customer#000006059|hbYI6RCnsefLu,WWFcArgxqy5xQdygs8tv9|15|25-133-925-3453|6051.11|BUILDING|s detect slyly. even pinto beans across the quickly ironic forges use furiously slyly regular courts. furiou 6060|Customer#000006060|G ziIQMNWzroTjGIHiWM 0pG|15|25-174-729-5653|4030.26|BUILDING|express, regular multipliers haggle after the fluffily final requests. regular excuses ar 6061|Customer#000006061|FRN1 emJhjvn9yjnKN9HjoNG,X aW3kdDmBA|15|25-551-122-4107|7532.06|MACHINERY|during the blithely express requests. quickly ironic frays across the blithe 6062|Customer#000006062|UtdDwozzDvfMtgj3W|3|13-756-700-4918|1370.35|FURNITURE|ourts-- regular packages hang furious 6063|Customer#000006063|P8McFZy0XZk2tO0fd6e|13|23-185-189-2252|6385.75|BUILDING|, special foxes. regular asymptotes sleep slyly 6064|Customer#000006064|,izgs4ldAZyamftqvB|20|30-205-718-3890|7335.40|AUTOMOBILE|theodolites. carefully pending packages sleep fluffily blithe 6065|Customer#000006065|Gb72LtD9HVv9slwDCWiufPxBYW6qVgnfe|16|26-150-752-2624|7191.82|HOUSEHOLD| alongside of the furiously unusual instr 6066|Customer#000006066|s,Txrg7qYwMSykuMvxhA26sOs1KNm8t|6|16-166-390-9922|2967.12|BUILDING|arefully blithely regular courts. furiously express request 6067|Customer#000006067|wktXPx5LSL5ZSP0shdmYDauSiNMRiAHaaKVJ|4|14-386-119-4898|3407.75|AUTOMOBILE|inal dependencies nag blithely car 6068|Customer#000006068|RCTOrn8 qmIVd6qGMMLmjWCvz7|4|14-138-821-9164|6028.37|FURNITURE|arefully even instructions sleep slyly regular, silent pinto b 6069|Customer#000006069|lP56RbvJxUsPWR7AJ3nGdcXjPYhDzw|8|18-520-115-6373|6775.83|FURNITURE|le furiously carefully brave ideas. f 6070|Customer#000006070|0Pycomq4a6KgwcZYyaH5g6t5hdLgsKF|14|24-195-934-6766|7430.40|AUTOMOBILE|even asymptotes wake about the furiously express accounts. special platelets between 6071|Customer#000006071|bSImC9SNAZBpJfS|3|13-976-555-2947|8764.04|AUTOMOBILE|after the dependencies eat carefully against the blithely regular pinto beans. 6072|Customer#000006072|dJTs qvHOtIuFDNIsyGkzkuz|16|26-754-599-4979|5312.01|MACHINERY|theodolites nag evenly among the bold excuses. ironic accounts sleep along 6073|Customer#000006073|pfb,24MPwsE8,0LUyO|9|19-689-677-5011|1045.91|HOUSEHOLD|. furiously regular ideas are regularly ironic theodolites. slyly even packages sleep slyly fluffily express p 6074|Customer#000006074|nrcHwNPzOQ x|8|18-837-412-1792|9486.43|AUTOMOBILE|olve furiously about the furiously bold deposits. slyl 6075|Customer#000006075|DOsAAUjRKdmOtvgEMibSALCKR M5Nj|1|11-593-566-6284|2455.27|FURNITURE|s the furiously ironic requests a 6076|Customer#000006076|At1D9HHVnICLHSzLkebTdUFubbpiizOsfsnT1|7|17-850-828-6128|2626.39|FURNITURE| pending deposits solve. slowly bold ideas haggle fluffily packages. blithely 6077|Customer#000006077|uhQEW4hX BiYzeK vM4p5nmxPwa|9|19-473-215-4783|7755.29|MACHINERY|e the pinto beans. final somas affix slyly: fluffily special instructions are careful 6078|Customer#000006078|SBe5gejYZc lZ|9|19-127-604-4037|2143.99|MACHINERY|o beans. quickly ironic accounts 6079|Customer#000006079|JrsyZ3aOo7pyfy1Nfcu7wv2y9MssX9Fl2j|24|34-956-792-4754|7035.43|AUTOMOBILE|pinto beans cajole carefully special pinto beans. bold packages was across the carefully ironic ideas. even, regular 6080|Customer#000006080|TO8wL2kHE7DAZOlyh7U4aW|2|12-853-465-2990|3557.21|FURNITURE|as sleep carefully along the ironic deposits. furiously dogged deposits among the fluffily regular frets sleep blit 6081|Customer#000006081|g9sWPKJZrAa2yRmoTwz|14|24-898-630-6492|-956.69|MACHINERY| courts affix around the furiously stealthy deposits. packages wake evenly 6082|Customer#000006082|A VQviGoD71daDZOZv|11|21-155-663-3196|898.19|BUILDING|usual pinto beans after the carefully pending fox 6083|Customer#000006083|Zk4jDSMrlCCH,MIpgre hsHn8XZJn62|11|21-852-430-5106|1845.68|AUTOMOBILE|ial accounts wake slyly. furiously unusual depths cajole blithely furiously 6084|Customer#000006084|E0NXucb5MGwfV5BnCW1qPSpeMoqf|15|25-403-768-3899|9300.11|AUTOMOBILE|s are carefully. doggedly pending foxes boost carefully pending, ironic accounts. closely bold 6085|Customer#000006085|N280OwVf0BbPajbB89YmFrHEihif|12|22-247-826-9697|6041.35|MACHINERY|slyly permanent courts was blithely packages. furiously b 6086|Customer#000006086|kBBqMZHke10dGuoDBvzv|20|30-613-894-3303|5029.81|MACHINERY|ays? dugouts sleep carefully around the slyly regular attainments. iron 6087|Customer#000006087|HZvlNvQ41HBORsewqAXWc1i5z,V|17|27-176-985-1293|9991.42|FURNITURE|fluffily. furiously final deposits believe about the furiously final excuses. slyly ironic accoun 6088|Customer#000006088|yxbYU1ogtMNgiSUAFNsA5 aYG|24|34-944-696-5888|8895.30|MACHINERY|thely. packages at the final packages use carefully around the regular, even deposits. 6089|Customer#000006089|1xUlqn0cOHlIKlRHeBz8Mfn|18|28-312-896-5088|458.27|HOUSEHOLD|es. accounts cajole slyly ironic c 6090|Customer#000006090|NqM,WhTKaDb4uzuNsmHPiSvWFsEGHJA|8|18-417-809-3338|61.15|HOUSEHOLD| special sheaves. quickly final requests are. quickly fina 6091|Customer#000006091|xqRatn5FncOIxHo9oZ9lBbPrJQ0e aoISrI|1|11-922-360-2419|2753.48|MACHINERY|quests. furiously ironic requests are above the carefully special packages. 6092|Customer#000006092|Z,XTnww1rI6aWDjmfk7w0p8kxLcqCIfD7h8|5|15-941-440-2662|9761.72|FURNITURE| foxes are above the ironic excuses. special instructions dazzle blithely about the regular asymptotes. fur 6093|Customer#000006093|BOuFVeQZ,E|8|18-806-815-4057|9343.42|AUTOMOBILE|gedly regular pinto beans. express pinto beans nag slyly. regular ideas wake thi 6094|Customer#000006094|X7kFGROKaQpglhpv20A|12|22-421-694-6690|7983.44|FURNITURE|to beans boost. carefully special pinto beans wake alongside of the slyly regular accounts. furiously regular pinto 6095|Customer#000006095|qP9OXihxntW60ybk|17|27-151-511-8515|8802.47|BUILDING|thely. furiously ironic request 6096|Customer#000006096|wy UjJzsFOKGCMfqtaHK5NI|1|11-517-930-3964|2237.40|HOUSEHOLD|silent excuses detect slyly silent accounts. slyly final instructions nag furiously. fur 6097|Customer#000006097|KCW2zgHZRm36j90QVmQ9b5Ml|2|12-742-944-8759|3963.90|MACHINERY|ding to the carefully silent accounts. special excuses wake. furiously final foxes about t 6098|Customer#000006098|c Bp7dnCuklVVNOM7Gc11k2gCRv|3|13-223-820-6932|3312.28|HOUSEHOLD|ts! carefully final foxes haggle slyly: slyly express tithes wake furiously about the final, final deposits. 6099|Customer#000006099|Zw,1lluCeZSlbwLKZo2i37|23|33-797-420-9100|7413.61|HOUSEHOLD| deposits are slyly pinto beans. carefully express packages wake furiously. even deposits 6100|Customer#000006100|BFnRAdDK0EMZAr2zS3js6,Jsh|14|24-425-432-3048|1569.09|AUTOMOBILE|olphins after the requests wake furiously silent requests. quickly ironic dependencies are furiously around the reg 6101|Customer#000006101|KI6M7iuiqLHAb|1|11-152-108-8875|2651.06|AUTOMOBILE|efully even requests. ironic, unusual deposits wake furiously. blithely pending req 6102|Customer#000006102|s5ViRaDEjv2nKox7c6Y|3|13-533-564-6439|4257.41|HOUSEHOLD|cial asymptotes are slyly regular sauternes-- slyly ironic deposits integrate regular, final theodolites-- furi 6103|Customer#000006103|YYS9 3AI8tkKqUqMtWssXA44M|4|14-405-808-5807|8364.07|HOUSEHOLD| pinto beans sleep. carefully regular sentiments sleep blithely. final packages cajole 6104|Customer#000006104|jcKuAbr5WFuVxd2xt,4KqyvQ7kz|15|25-250-924-1873|4010.86|MACHINERY| final packages sleep above the slyly bold 6105|Customer#000006105|EfQPHmIraBqSvNY15deoIPsc|4|14-506-543-9059|506.68|BUILDING|he unusual deposits. furiously even a 6106|Customer#000006106|nfroWgkspby66r6BkrvQCMX,f,i4Myrykv3k pa|7|17-151-647-7940|-337.05|FURNITURE| special excuses cajole. carefully bold req 6107|Customer#000006107|,4I3WYJQN9ZAWX5I5J54dJna|19|29-717-849-8757|9064.29|FURNITURE|long the excuses. slyly special instructions unwind slyly beyond the regular, final pinto beans. requests use blit 6108|Customer#000006108|0SPg7CHlEjunN0dHuKF932w|10|20-564-384-5926|2570.78|BUILDING|asymptotes sleep carefully. fluffily regular pinto 6109|Customer#000006109|gpgBDs5krmN|7|17-896-737-8894|8151.07|HOUSEHOLD|uriously pending asymptotes. fluffily bold pinto beans are above the 6110|Customer#000006110|FAxMqfHBG36oB|8|18-793-720-8574|-888.62|BUILDING|iously final ideas. furiously regular packages nag. quickly regular pinto beans above the regular theo 6111|Customer#000006111|CLSQ2jkX5hmEPyKFJkKxemAH|18|28-252-831-5956|857.85|HOUSEHOLD|ly final deposits. slyly special accounts against the furious 6112|Customer#000006112|wPzFnLHJ Eg3EzH |24|34-654-913-4569|5894.96|HOUSEHOLD| final, dogged packages wake furiously slyly regular platelets. fluffily silent pinto bea 6113|Customer#000006113|YV9LsSwnYEjuQENdPBp8Sfl|18|28-599-409-3070|8646.86|MACHINERY| blithely at the blithely ironic deposits. carefully final ideas cajole fluffily. unusual pa 6114|Customer#000006114|87W0S9L9zjjL,lPnNxpXEJKSjaOY|12|22-957-665-4649|4860.40|HOUSEHOLD|haggle slyly above the slyly express deposits. slyly even ideas sleep fluffily above the slyly ironic accoun 6115|Customer#000006115|r3MSiTaTNNKpIXe1x,RDRqPx1s|21|31-465-527-1774|821.94|MACHINERY|er the slyly express platelets affix carefully above the careful requests. pending 6116|Customer#000006116|Q1HHnkKguo6sSYNWgNEk2tTeEe6db3s6osO|4|14-743-932-4071|7234.12|BUILDING|he blithely express requests haggle furiously regular, final excuses; carefully regular pinto beans unwind ca 6117|Customer#000006117|4CHkWzZ3fT|24|34-380-887-8865|9826.19|FURNITURE|ly final packages. fluffily bold escapades sleep fluffily after the carefully express 6118|Customer#000006118|GZJ6Dctbr0USrtx|11|21-581-376-6339|3810.27|HOUSEHOLD|e fluffily special packages integrate quickly final dependencies. regular 6119|Customer#000006119|p8I8iRWN3HIQZPoMD5y1qogRuycR7VnAiEyOZff9|1|11-137-505-5251|6246.11|FURNITURE|wake blithely fluffily ironic theodolites. bold deposits cajole about the blithely fin 6120|Customer#000006120|ZTjNH4Si5g9pTrul2fRGaHEB|6|16-141-186-1813|3351.67|HOUSEHOLD| excuses nag fluffily ironic requests. slyly 6121|Customer#000006121|uAXqrguis17T1SGF9Od0sJ|13|23-689-704-1607|9531.41|FURNITURE|ss packages are furiously final depths-- final, ev 6122|Customer#000006122|48JzmrY5BZiS2C5Ts,wgJkRR7z5SOkJhg7|18|28-417-497-5317|-485.36|BUILDING|yly express accounts. slyly regular f 6123|Customer#000006123|rriyD1ssl4dg,ur1WRPBG|19|29-912-115-6013|2397.49|BUILDING|ogged deposits breach blithely. blithely express theodolites use. packages about the fina 6124|Customer#000006124|1FAv28GTpQRD2Nw ULiQG3qCi6PTGFXC|7|17-962-756-6916|-383.24|MACHINERY|final courts. furiously ironic deposits run blithely furiously express instructions. slyly special deposits slee 6125|Customer#000006125|q4dT2taZKQmIDI8,V|11|21-143-295-8049|2811.99|MACHINERY| ideas. slyly even packages are fluffily instead of t 6126|Customer#000006126|7NRolRPvqN3QIrcKU|23|33-988-127-6540|3047.89|BUILDING|hely. quickly even braids haggle slyly. quickly even theodolites 6127|Customer#000006127|4xg9rQlpgsZ7NJl4k9IL47ODv3E26jIRCvnlgd|23|33-236-693-8622|7908.12|BUILDING|le carefully. even requests grow carefully carefully bold theodolites. furiously regul 6128|Customer#000006128|c88JZp1so3TfKB6o5|16|26-586-949-7031|7899.60|FURNITURE|efully enticing braids according to the furiously ironic accounts detect furiously around the fluffi 6129|Customer#000006129|MVydnT2OcBdbSLF|22|32-368-121-9588|7571.87|HOUSEHOLD|ously after the regularly pending excuses. accounts boost slyly final ideas. special account 6130|Customer#000006130|uzgA6cyeuysYLVKMv|6|16-405-646-2657|-555.94|BUILDING|efully regular deposits about the regular packages believe blithely alongside of the ideas. bl 6131|Customer#000006131|EwoT1zKxHYj8|13|23-237-602-7871|4223.57|BUILDING|e doggedly permanent platelets slee 6132|Customer#000006132|LMFK0bKFaBIX8tW74Yoxb8,bw9XS|1|11-890-884-8734|468.90|HOUSEHOLD|ly close courts among the express, even packages believe doggedly among the fluffily regular ins 6133|Customer#000006133|FwwrG68tR4k|13|23-849-670-9143|2698.12|HOUSEHOLD|final ideas detect furiously permanent accounts. slyly regular courts boost final instructions. theodolite 6134|Customer#000006134| DMIOsEg5VDDKmWzjjShHILJhBrFVea|8|18-349-804-2162|3214.77|FURNITURE|kages sleep. furiously thin requests affix ac 6135|Customer#000006135|IED4rGsufuEnT4NrO7KvhQfFfcwGzvMHgQiUnoP|1|11-117-608-4110|7552.48|MACHINERY| deposits. deposits along the final packages haggle furiously ironi 6136|Customer#000006136|pDoECZ7k3AwOSp9wDO|19|29-974-254-4381|3649.73|BUILDING|ely ironic tithes sleep finally after the 6137|Customer#000006137|Uv1p49Ppo7lO9zpW05z|5|15-958-508-2113|3037.82|BUILDING|riously express, regular hockey players. quickly final pinto beans use packages. carefully e 6138|Customer#000006138|Dm6,p6hst9ub9FiYivgUm3FapKRiEYPG|15|25-872-607-4602|3638.97|FURNITURE|snooze slyly quickly ironic packages. furiously final deposits sleep quickly specia 6139|Customer#000006139|3r3Uc,7NMqKxby7w641xEV3su|17|27-704-480-4139|9119.98|AUTOMOBILE|s cajole blithely regular packages. furiously ironic deposits are final, regular fo 6140|Customer#000006140|QPlaw,8zEWbcabsVf,wRBMC1hvedP0|11|21-263-388-7344|3593.93|AUTOMOBILE|ke furiously slyly bold warhors 6141|Customer#000006141|4QBCP6mT5dep|10|20-961-227-7319|247.76|BUILDING| alongside of the patterns. pending ideas doze 6142|Customer#000006142|6rJlz 6aiuufZqN8Ldf9|7|17-856-660-5846|4045.08|BUILDING|the carefully bold deposits-- requests haggle slyly 6143|Customer#000006143|zuh77l1EZ6xUn6H,sZIYq|1|11-253-915-7383|3706.32|FURNITURE| nag slyly. final, pending dependencies haggle slyly about the furiously ironic o 6144|Customer#000006144|zPKzjSPczgTN9Yb3ybnDmhplv6ymvG|1|11-306-913-9602|8057.08|FURNITURE|equests cajole carefully pending, even packages. regular platelets 6145|Customer#000006145|CybtvIEF1DXOFCddFGr|21|31-879-282-9414|3423.05|HOUSEHOLD|sits are slyly even requests. regular packages across t 6146|Customer#000006146|RNKo8PgfKqgW6hGWTJMp|19|29-723-257-4271|-832.84|FURNITURE|ld requests alongside of the regular packages wake fluff 6147|Customer#000006147|fr6iYAVpls5Zy8UF1qn|21|31-729-268-4470|3418.83|FURNITURE|accounts. silent, ironic orbits believe furiously escapades. slyly regular packag 6148|Customer#000006148|OiRtO4N,szTYNMjqyl,|14|24-345-723-7631|5246.25|AUTOMOBILE|s poach carefully after the furiously ironic packages. even pinto beans are fluffily. blithely unusual pinto beans 6149|Customer#000006149|Dv0FCsfLC8tg I,NGqcnL3uO|9|19-981-118-5032|6737.15|BUILDING|mong the blithely final requests sleep according to the b 6150|Customer#000006150|aX47rrqgyvzy74tFd0o Wn1wr5XK|2|12-895-769-4268|6249.24|BUILDING|o beans cajole doggedly above the slyly unusual deposits. enticingly regular theodolites snooze c 6151|Customer#000006151|ojCmk5FfGZm4Oset4R|1|11-178-162-9290|1016.84|BUILDING|inos should have to use furiously among the quickly express somas. ironic requests cajole furiously after the slyly 6152|Customer#000006152|Bqxwa70bCy,M8Vs6cWn|19|29-492-966-5351|4957.70|MACHINERY|y final foxes. express packages wake across the ruthlessly regular instructions. express 6153|Customer#000006153|,oEYlRJs0L76hU5MtnT17H5qULk|6|16-303-738-6232|8070.72|MACHINERY|ctions use slyly above the blithely express deposits. 6154|Customer#000006154|FI1U42PTLrAG n3a5JxzHabcH|10|20-372-167-3439|1571.98|AUTOMOBILE|tithes nag regular, final theodolites. deposits sleep along the ironic theodolites. carefully 6155|Customer#000006155|03gJRZx3TGwScCd8d1 MhitTEHY i|20|30-511-190-6866|7246.91|HOUSEHOLD|ess accounts. carefully even theodolites are silent requests. slyly unusual packages 6156|Customer#000006156|cVhwUHk5,b74V|12|22-119-245-3413|4873.83|HOUSEHOLD|ly daring, express deposits. special, regular packages haggle furiously ironic package 6157|Customer#000006157|3hTpojm17PUW3OvoLxeqQtz2|11|21-578-529-7649|6444.10|HOUSEHOLD|are slyly sometimes final deposits. special, final dependencies cajole quickly. slyly e 6158|Customer#000006158|GmqBG39QiPPqq1YA|10|20-393-733-8825|8966.55|FURNITURE| even asymptotes. fluffily pending requests against the quickly silent platelets cajo 6159|Customer#000006159|naaCoV9ztjZ2YVP4hmQAtTnFsb,DAWFoAJp5|6|16-192-111-5212|9877.27|FURNITURE|g the silent, ironic theodolite 6160|Customer#000006160|NrxXWav6LihuuSlWGszrc|13|23-830-408-5650|4333.77|FURNITURE|pending orbits sleep quickly. slyly unusual requests cajole. furiously regular requests 6161|Customer#000006161|pbRxOFTDqY0lPBucGOSzCy|18|28-703-556-9515|4706.39|FURNITURE|ing deposits? furiously regular accounts alongside of the fluffily even requests are blithely above the 6162|Customer#000006162|wFK59S80D3m iXYc96LznPltGTpDcB|3|13-525-195-7035|8277.40|HOUSEHOLD|ackages. requests use blithely according to the final, regular requests. even, special accounts 6163|Customer#000006163|tmxBpVae0jKAe9vXikmLaCRGPAoyiDVGz|21|31-674-263-4416|-678.87|AUTOMOBILE|requests sleep. quickly final depths affix quickly unusual excuses-- final packages af 6164|Customer#000006164|PSgfAO1LnbP8 5aNt2lWBD|14|24-782-729-4216|6171.41|FURNITURE| ironic courts solve closely according to the carefully ironic requests; special, 6165|Customer#000006165|31aZoMFnTFf|12|22-414-194-1058|1982.00|AUTOMOBILE| the even, unusual excuses. slyly special foxes nag quickly special, expres 6166|Customer#000006166|df2AQz9BJgmAj XoIOiyoUd|14|24-891-533-8945|1444.98|HOUSEHOLD|ng packages affix even, final pi 6167|Customer#000006167|oPGRaBt5IuSxtQhwgztJyv1miL|7|17-798-361-3738|6729.32|FURNITURE|ly final requests grow alongside of t 6168|Customer#000006168|TfeEEZ9Ds8x3eQljQUYaNzpu6tE1ap,JGRn2gZm|24|34-877-467-2443|6633.98|MACHINERY|ily. special platelets about the slyly 6169|Customer#000006169|UDeLABnhksFK7tfInLsDPRtynoWxesfiJ|7|17-656-209-2093|-681.05|HOUSEHOLD|y final pinto beans. slyly even dependencies boost 6170|Customer#000006170|PLJi9aPgRlkkIUsc8LGRVLOGrz1IBd|19|29-558-582-4733|2570.49|BUILDING|t requests cajole slyly. slyly ironic courts ca 6171|Customer#000006171|SNyHcZsmA5EydGjLu0MT7Y|19|29-222-112-6966|3125.83|MACHINERY|ts nag fluffily even instructions. furiously even pint 6172|Customer#000006172|uafeB6k3L MWgR6k8Wokrv0gtSwTLmWRW8,UVhv|5|15-750-174-4232|8749.91|BUILDING|ound the slyly regular theodolites wake slyly across the platelets. carefu 6173|Customer#000006173|jd3so7 leJO5Y0SF1YFrMJ|20|30-743-439-3998|9725.31|MACHINERY|es boost above the carefully regular 6174|Customer#000006174|UOkeiQv5WK1OBw5CjtHQDh84JAV|11|21-570-543-7869|2314.46|FURNITURE|ly bold foxes-- final ideas x-ray among the regular theodo 6175|Customer#000006175|4fRoxmxFa4n|23|33-387-822-4617|1065.86|MACHINERY|ress pinto beans. fluffily final ideas dazzle closely slyly ironic packages. furiousl 6176|Customer#000006176|kXM67uOjA5sj|24|34-919-259-2224|3918.54|FURNITURE|nts wake after the even packages. ironic requests alongside of the slyly regular excuses boost slyly regular reque 6177|Customer#000006177|pZZ8D,yKhfsFatXwk|24|34-161-190-1931|-149.78|MACHINERY|are across the slyly ironic requests; carefully dogged accounts run blithely bli 6178|Customer#000006178|ejgXNsz sI0Dl3F,FVziTAF4mPWXczkmXlu|9|19-567-173-3123|-369.66|FURNITURE| furiously alongside of the pending, ironic theodolites. ideas among the silent pinto beans dazzle silently alongsi 6179|Customer#000006179|0MU9 AVKw SeY6kbL5VJm|6|16-699-827-5744|38.38|AUTOMOBILE|l accounts lose furiously slyly final tithes. quickly stealthy attainm 6180|Customer#000006180|4csqeJ8yw1y r 6Bzi49uv|11|21-527-929-7958|4979.33|AUTOMOBILE|ymptotes solve beside the ironic packages. carefully special deposits ar 6181|Customer#000006181|cZ9B3p5D4poouVdTvh0Sol7ODKuWa|2|12-716-858-1804|8630.17|MACHINERY|es. unusual instructions among the slyly regular deposits affix carefully after the furiously ironic platelets. 6182|Customer#000006182|8U367QGaD8IUUdHyHtwSj3pmJoeLcVrMccGMZ5J|6|16-291-418-8009|-745.50|AUTOMOBILE|es serve blithely alongside of the regular instructions: unusual deposits cajole ruthlessly alo 6183|Customer#000006183|iDx,aeynoLw|20|30-808-423-5478|8330.40|MACHINERY|gular, express accounts. requests use. furiously pending instructions detect within the quickly 6184|Customer#000006184|dkynJLYBPBCkx 1paCFtwxmixcoPoqaFVyGQ|19|29-373-987-6278|-960.96|MACHINERY|hely blithely special deposits. furiously pending requests boost carefully instructions. excuses nag 6185|Customer#000006185|4qtODzt Kxhkagjgtc5U 6l|24|34-493-651-6114|6109.32|HOUSEHOLD|ests are final ideas-- ironic, pe 6186|Customer#000006186|0dwkPKvOkPIniv3,Fahd1rq9nwc|16|26-982-329-6333|3360.22|AUTOMOBILE|sly bold grouches. ironic pinto beans sleep furiously furiously regular depths. platelets breach t 6187|Customer#000006187|LJJpyf,OZivsv6IQBJG3gEisgc7d QC7oKRuXOBj|15|25-545-311-4634|-838.33|AUTOMOBILE| requests. slyly regular requests after the accounts hagg 6188|Customer#000006188|Z85HZ6fRUEl3|15|25-648-100-5980|-895.88|AUTOMOBILE|gle carefully. bold theodolites x-ray fluffily. bold gifts alongside of the sent 6189|Customer#000006189|X6edGVb,Osa emoLHSaQKn|14|24-556-862-5258|-213.89|AUTOMOBILE|ly regular deposits across the regular theodolites serve blithely express 6190|Customer#000006190|mV9CzSEQr,nE3CC, xJ1EsQOw|2|12-212-128-8305|33.63|MACHINERY| finally regular theodolites wa 6191|Customer#000006191|7XGJ0ugPk dVdCm1nJQ|22|32-229-609-5050|2167.56|BUILDING|s dugouts. carefully bold pinto beans 6192|Customer#000006192|oI8CXoK1w9PnZDUEbvj|0|10-276-595-2077|7179.63|BUILDING|ly bold decoys are slyly. slyly even epitaphs around the regular requests cajole at the blit 6193|Customer#000006193|RnHj1jACEqFgLpCQfzgsZtgoZu1Jck|16|26-200-646-6714|7033.00|MACHINERY|dependencies. furiously special foxes cajole 6194|Customer#000006194|9GsTKrC4NgB2bqd4ui9kuijhjxDlw IU1|9|19-886-790-6122|6639.12|AUTOMOBILE|ts boost after the carefully ex 6195|Customer#000006195|eVLaPYm6NRhqQzuMx3vk|0|10-891-840-4980|8707.05|FURNITURE|y. quickly pending accounts against the blithely thin ins 6196|Customer#000006196|KNWdwsj7hGyO0lrvqr6G1o|15|25-200-325-2383|4462.11|BUILDING|c courts. final instructions a 6197|Customer#000006197|Ce9LTBhp7GkCqZy|11|21-395-292-5975|5631.85|FURNITURE| unusual instructions wake always above the regular excuses 6198|Customer#000006198|OglD6pbHC9ovv2mfZ rDO iay cVjX5SRng2|23|33-381-544-1422|8515.39|FURNITURE|y unusual platelets are slyly across the carefully final pack 6199|Customer#000006199|3LH72AxUTOqvuazpB6dk5i80YVo3,H2YZiyir|22|32-514-462-3884|6178.78|MACHINERY|ideas wake fluffily. pending, pending p 6200|Customer#000006200|oGCR8cSGI,rHCpRMi2|3|13-609-502-2266|4767.80|FURNITURE|l dolphins. sometimes unusual instructi 6201|Customer#000006201|oK3Q7pkcEZaGXxOeNB4okAaAxNDbZB8K1y|0|10-385-198-4441|6397.29|HOUSEHOLD|ly. regular, slow packages along the carefully express foxes doze to the blithely even foxes-- furiou 6202|Customer#000006202|E73qW mbEQINh gPymXB,ed4O nKl|15|25-477-944-7482|3330.24|FURNITURE|l excuses. fluffily ironic packages use. quickly final Tiresias a 6203|Customer#000006203|xVMuglbV53zSLL4wb7Mxb,pkD8MrP6 R,CO ehMc|19|29-735-432-9939|1280.15|BUILDING| ideas atop the even, unusual excuses sleep against the furiously careful pearls. closely express 6204|Customer#000006204|6MBV7BG qK9BOmGQny|16|26-983-171-4809|8460.54|FURNITURE|eful requests. quickly ironic instructions use b 6205|Customer#000006205|Bd2A0KLWCYN1WZ1XJc,N |18|28-920-423-1966|8416.88|FURNITURE|osits over the pending instructions wa 6206|Customer#000006206|EZ6yws0GmtHJXSHiV|4|14-448-256-3507|486.36|HOUSEHOLD|nding platelets are after the pending foxes. unusual, regula 6207|Customer#000006207|Ba2gZAYGD 74QLT8T7,uRQwIi0rqbJ9|7|17-896-188-3890|-740.28|FURNITURE|foxes. blithely pending platelets haggle quickly bold, silent asymptotes 6208|Customer#000006208|m,PxS2pByk43RfabxsV6in7n|18|28-322-291-1770|8045.41|FURNITURE|deposits wake slyly. slyly even ideas use blithely. furiously iro 6209|Customer#000006209|C1Ls6INP7D9jHTugpjUUWD9kS9cpKclB|2|12-685-144-7597|5241.88|BUILDING|ges. final accounts breach slyly. furiously regular deposits according to the express instructions wake after 6210|Customer#000006210|d5fRx4ruNET9kj6LqGhVxCYwT|17|27-245-729-5781|-791.51|AUTOMOBILE|special pinto beans poach quickly careful theodolites. fluffily pending instructions cajo 6211|Customer#000006211|,bXG5MlIamKtG8mMbce|2|12-200-723-4029|5045.01|BUILDING| furiously special requests. carefully final deposits wake even requests. carefully final accounts are alongside o 6212|Customer#000006212|ArifsT45MN,N2HR,CoiDLwG0|5|15-251-873-7969|1570.87|AUTOMOBILE|n foxes. ironic, final foxes boost blithely bold instructions. furiously dogged deposi 6213|Customer#000006213|jpXKO9LktOMMIPpfE xyGA7uurVPqSzOn|5|15-600-409-7048|-445.21|AUTOMOBILE|nts use carefully above the b 6214|Customer#000006214| ,MAKZxDGF3QKwDERiLKFCaCOcIOlN|22|32-811-917-7230|6202.90|HOUSEHOLD|. carefully express theodolites sleep furiously even requests. quickly express instructions lose slyly silent idea 6215|Customer#000006215|fAfdqCTURbOu,|5|15-792-734-1509|3206.02|BUILDING|deposits. furiously even hockey players sleep carefully. quickly bold requests across the slyly spe 6216|Customer#000006216|bGTXGAHg72BEDM09QZEFI|0|10-585-359-5566|3548.03|MACHINERY|ickly. slyly final theodolites integrate fluffily quickly ironic asymptotes. 6217|Customer#000006217|k9NiqQlFJVv6|9|19-612-407-3150|9647.49|MACHINERY|tions wake quickly slyly regular pinto beans. ironic, final asymptotes affix blithely around the hockey players. 6218|Customer#000006218|4,z5xJL2IWWO5LMrF36cZISGaq77Q6 7DvAh5|16|26-501-652-8685|3504.23|HOUSEHOLD|slyly final instructions. final e 6219|Customer#000006219|lM 6tdVkyERY,wQ6n7ZHD11,G|8|18-384-857-8254|-443.88|FURNITURE|lyly regular theodolites doze furiously bold ac 6220|Customer#000006220|je,Ssek0XNsaWRGsiKF 0,hDjGVOaSNsN7TkZz|7|17-755-898-9664|6694.27|BUILDING|ffily express deposits. carefully ironic packages haggle furiously final excuses. pending dolphins af 6221|Customer#000006221|J2Zx,5YWGMpbyd9yupa1PMuDhhHCFtDqmtzx0CE|6|16-641-636-3853|712.36|HOUSEHOLD| the furiously even braids. instructions use blithely 6222|Customer#000006222|8p2FbclgqcvoFR29P,OwwCkWR|8|18-981-975-1436|2589.32|FURNITURE|usual pinto beans. slyly special deposits breach about the slyly i 6223|Customer#000006223|kOimz7buzOsZP8DPRQfB pa8a7bWyA0Axx|7|17-805-445-2530|644.72|BUILDING|each slyly quickly even requests. quickly regular instructions cajole blithely. deposits wake along 6224|Customer#000006224|LFoSAlF,JOEn4gVU0qGhIpu|5|15-952-723-9945|9794.49|FURNITURE|ly blithe dependencies use across the slow requests. 6225|Customer#000006225|cqOhtNVujJSTrsZlLLvt1k|21|31-221-435-4954|-847.96|BUILDING|; slyly unusual deposits cajole pinto beans. fluffily regular accounts above the accounts lose slyly 6226|Customer#000006226|8gPu8,NPGkfyQQ0hcIYUGPIBWc,ybP5g,|23|33-657-701-3391|2230.09|BUILDING|ending platelets along the express deposits cajole carefully final 6227|Customer#000006227|hQpDSUJLnjcvDZ4WbiVrWDSYBjCou1kJ|11|21-159-594-1232|2062.51|FURNITURE|uffily stealthy deposits haggle quickly carefully final request 6228|Customer#000006228|Zb2Vj1EhkIivE CSYkb936,JYTQaWYT0a|5|15-417-317-1397|1524.63|FURNITURE|ts. furiously regular requests wake carefully regular packages. furiously regular packages about 6229|Customer#000006229|6jBRlUNs3Q,XQZsgUuaWybaSY|17|27-996-380-7890|3480.92|HOUSEHOLD| packages. even asymptotes nag ironic excuses. slyly ironic deposits cajole. express requests 6230|Customer#000006230|MGXvPZQ6UMzc4PbZcSUl8kGfew|18|28-226-342-3356|3763.78|MACHINERY|ously pending platelets; busily special theodolites sleep carefully! express platelets sleep slyly acros 6231|Customer#000006231|DssbpsUtrcZVi81wG|11|21-183-229-4023|5805.58|HOUSEHOLD|ding accounts thrash carefully alongside of the even, regular ex 6232|Customer#000006232|T0JFOylApn8YQr,|18|28-847-729-1271|2455.95|HOUSEHOLD|ly pending packages are. regularly express accounts past the fluffily regular p 6233|Customer#000006233|4ZbcmGRlrqQ5|22|32-482-650-3134|3186.31|FURNITURE| instructions along the regular packages maintain above th 6234|Customer#000006234|lQaP7fCR8lVTQY7pkMi1BLv|23|33-658-619-9537|2250.65|BUILDING|dle packages boost slyly. quickly special theodolites wake furiou 6235|Customer#000006235|YpIuvasSAUFQ027rL|11|21-845-791-5239|6718.32|AUTOMOBILE|e the ironic, final instructions. carefully unusual somas haggle quickly quickly ironic requests. quickly eve 6236|Customer#000006236|zQjCvBPfacwUT0nrJ2uF,4FtSAC0ldB|17|27-557-761-6785|8836.84|MACHINERY|uriously bold accounts boost blithely ironic theodolites. quickly even packages mold along the 6237|Customer#000006237|jgXVEi0rAQCxBmRGM1vE1|0|10-312-679-5009|7806.93|AUTOMOBILE|oze slyly. thin, unusual depths integrate quickly alo 6238|Customer#000006238|AckKNmINwCbVw kC5bZ0u3Hh7C3,7Rdva0|15|25-749-568-5891|1052.60|HOUSEHOLD|hely special deposits. regular requests solve un 6239|Customer#000006239|PklZCqNbiyA9|22|32-305-456-4164|1007.44|AUTOMOBILE|ular packages. final, even ideas print slowly along the pending acco 6240|Customer#000006240|kxWW41iCbmQJFT3GxGtmHakhcHZ07nW0diOTG|8|18-432-692-3478|939.14|BUILDING| even dependencies. pending, regular packages wake carefully above the deposits. 6241|Customer#000006241|ucocla,JAAW amojIyO Ow |7|17-623-128-3126|8018.46|FURNITURE|tructions wake fluffily. enticingly regular requests sleep furiously silent packages. slyly special 6242|Customer#000006242|1J2Yz5f1lOLcsSkMTme|14|24-293-990-7292|3379.23|FURNITURE|uickly ironic packages sleep. fluffily regular accounts wake qui 6243|Customer#000006243|JotibBD2zs87K8|19|29-356-403-8790|-377.14|AUTOMOBILE|ly even accounts. final instructions haggle. silent, silent platelets are around the carefully even accounts. caref 6244|Customer#000006244|5XQ1QzJOWRsj2dkV0WnG2lFantHSML9hToTT2tw|21|31-123-516-6719|6385.54|FURNITURE| furiously even requests sleep quickly under the pinto beans. ironic warthogs thrash sp 6245|Customer#000006245|okJFjGfCET8m41TwZdswM4nqi2,wx8qP6tpM8GE|19|29-589-374-8352|3766.89|BUILDING|ounts cajole. deposits are. quickly final ideas across the carefully ironic notor 6246|Customer#000006246|5l,sIIFQVnUUtzs1|10|20-124-894-8392|3090.40|FURNITURE|ckly regular theodolites use. furiously regular instructions against the regular dolphins are sl 6247|Customer#000006247|22fOQj6uLxyE6L,ev4yMzPv8FuBvstp9JJAmez|3|13-961-966-3385|7408.81|HOUSEHOLD|s should have to haggle furiously even requests. furiously bold d 6248|Customer#000006248|F3NgALjt9qjhFKEDz|10|20-237-638-2221|8834.94|AUTOMOBILE|ckly according to the theodolites. foxes along the ironic, bold accou 6249|Customer#000006249|I3e rogGC4PkngfpXmtiw|15|25-311-257-8481|8946.03|MACHINERY|efully across the fluffily bold tithes. fluffily regular frays nag blithely along the sometimes eve 6250|Customer#000006250|swZXFAFQ2O|17|27-861-967-3557|5926.27|FURNITURE|after the furiously ironic asymptotes. pinto beans nag. fluffily regular ideas pro 6251|Customer#000006251|GWig6svLrx4Lcr0I tMBGMgd7HuyLGjbDJnFkGTl|8|18-245-505-4627|6920.15|AUTOMOBILE| unusual dependencies wake furiously even foxes. regular pinto beans mold furiously spe 6252|Customer#000006252|iGu23V6R50fBd4,WCMeGmcF9QaffCWOZiiY|22|32-311-215-9551|1928.88|AUTOMOBILE|packages cajole. carefully ironic packages nod along the even excuses. foxes wake against the packages. 6253|Customer#000006253|dHtDAyg0dAsmNZUJ6yonI6|9|19-583-938-5958|-356.48|BUILDING|ffix blithely. special, special packages believe. slyly regular platel 6254|Customer#000006254|XpXxuBjXYzFBXCvTp8sO4 0zwCiWD9ggiF|12|22-251-775-8243|1538.38|FURNITURE|. blithely final courts are quickly across the ironic deposit 6255|Customer#000006255|l6XtT3yvhZ,VzKzrXl2vQiIgwcrFB3qL7fuARSYk|5|15-472-424-3063|3495.68|BUILDING|even excuses. fluffily regular pa 6256|Customer#000006256|,fpJxiGODuy,EqJmiD9qM1DN|23|33-628-309-9349|9846.83|FURNITURE|lithely ironic, bold deposits. blithely bold 6257|Customer#000006257|etreRcLjc7uC|17|27-604-361-9761|9836.21|AUTOMOBILE| asymptotes wake slyly. bold excuses are furiously carefully furious accounts. slyly even packa 6258|Customer#000006258|CzJGapE7fxQx3x9eN A7O|18|28-429-156-3365|7221.20|FURNITURE| foxes haggle slyly slyly final asymptotes. slyly even packages wake furiously along the 6259|Customer#000006259|fpEAMihvplhOKyor ZRcf2bEUOwQgGz6SkilLk|16|26-449-972-6429|1723.42|AUTOMOBILE|instructions are evenly unusual ideas. furiously regular pinto beans among the furiously bold deposits wake 6260|Customer#000006260|oJLJxevvYZqj,n6Dq,L5V6C|4|14-214-869-6336|405.59|FURNITURE| haggle above the slyly regular platelets. pending theodolites wake carefully even 6261|Customer#000006261|rbTkXWn,HeQxxR8SLRuBS3m,LK|12|22-636-489-9192|5612.90|AUTOMOBILE|ructions. final, unusual pinto beans are furiously specia 6262|Customer#000006262|HlUg CpG1hqLgHf|14|24-761-458-3272|8970.62|FURNITURE| ironic requests believe quickly after the furious dependencies. express, fin 6263|Customer#000006263|nTwk5ECJ6elmDX8zLW7Fta9u9PlmTaqqRNja7|14|24-936-988-1040|1314.66|FURNITURE|ckages use furiously slyly final accounts. bravely bold dependencies about the carefully ironic asymptotes are 6264|Customer#000006264|pV82CJ4rNyOcMzCXNmdy|17|27-583-887-6592|5392.22|AUTOMOBILE|al, careful instructions affix blithely. ironic 6265|Customer#000006265|,8NdehNjF5ojMMEKikadmc2ng|23|33-192-952-7496|6977.71|BUILDING|y after the quickly pending requests. unusual 6266|Customer#000006266|9OKHx1,rXIAV0pq6Vj,uERU44LaT|22|32-443-596-1740|1468.66|BUILDING|g blithely. ironic foxes cajole blithely around the packages. 6267|Customer#000006267|Hxi,BwRbqRQUkum7Ts3R ugk4w58Ozmpp|20|30-657-865-4960|8650.75|HOUSEHOLD|l platelets sleep blithely. quickly pe 6268|Customer#000006268|cG,c4luyALcY|5|15-764-581-5523|4236.85|BUILDING|special hockey players wake fluffily express, bold pinto bean 6269|Customer#000006269|t3tcDR3QxuXh1Q5eHbzBls8jxxc4eCZSKKu Rh|16|26-790-547-6046|3793.75|FURNITURE|ccording to the furiously ironic deposits. final decoys wake along the patterns. blithely even 6270|Customer#000006270|PGn,pJmM gsA1tDtDbbuiiGra57c4FL|18|28-717-120-2144|3485.91|AUTOMOBILE|deposits. pending braids cajole fluffily pinto beans. carefully ir 6271|Customer#000006271|C61IFNXGXjOgzUAf8drHHKFksk,dASWbIXele|15|25-214-187-2123|-342.40|AUTOMOBILE|ptotes haggle slyly regular accounts-- regular pinto beans use carefully against the blithely reg 6272|Customer#000006272|HnzXtYtwH8Jco2wa,L|2|12-494-911-3342|7209.51|FURNITURE|ajole slyly. furiously final packages affix silent, final theodolites. even asymptotes x-ray bravely. reg 6273|Customer#000006273|ZD7bJedn3FdCd3p1SLGq4rZGqBCMlic|0|10-717-770-2411|3278.98|AUTOMOBILE|uffily among the fluffily regular accounts. even asymptotes haggle finally 6274|Customer#000006274|SgRPgKV3mB1oPuGe1ccFvjhDBRYiopGwIsWTNuOL|24|34-845-579-7944|8778.79|AUTOMOBILE|uriously regular requests. slyly regular instructions haggle. in 6275|Customer#000006275|vFGTLUjxiQu4HEiY16P1jSBrn380WinK,|0|10-194-385-3660|187.26|AUTOMOBILE| poach above the furiously unusual sauternes. accounts poach slyly. blithe 6276|Customer#000006276|qrFgLgA0RCdrQioauSbVb8g|11|21-790-836-6047|-777.89|AUTOMOBILE|uriously regular deposits. even deposits alongside of the slyly 6277|Customer#000006277|0EKQ4D5RMYQ6,NHCtnq14Es7OIWwNDPRtBEswyFP|0|10-237-523-6848|2907.96|BUILDING|s boost furiously final, special r 6278|Customer#000006278|F kMurdAhFU0C2KEiojmsS5gWlgxPZ5Q49iZl|5|15-299-327-8860|9996.76|AUTOMOBILE|detect slyly unusual hockey players. regular requests after the final 6279|Customer#000006279|Ft4nZfY7lsZ ws|5|15-206-922-2248|-319.90|FURNITURE|ial, unusual accounts. final asymptot 6280|Customer#000006280|eA0MCPdIfSK06GY JDS,GKrDGrr3e7ZqkB|19|29-971-689-4133|407.63|AUTOMOBILE|cuses. ironic deposits wake above the permanently ironic deposits. final ideas are slyly ca 6281|Customer#000006281|yORAuTtjrCJF9lOKTJtS9|4|14-318-809-5732|7055.03|BUILDING|refully even frays cajole during the carefully even acc 6282|Customer#000006282|fCqZiNSOZ46KUCaRvVFPG60DMq|16|26-750-142-4294|2969.47|AUTOMOBILE| furiously regular deposits. slyly ironic packages 6283|Customer#000006283|tmVqD0BrhOBHlA|14|24-539-196-5846|9860.06|HOUSEHOLD|regular, silent requests. blithely ironic theodolites sleep. blithely even deposits nag r 6284|Customer#000006284|zEh7eHXMGoSNP9h7Bk7G8axMfEkBVkj,NADgz9|13|23-583-198-6369|1178.44|BUILDING|fily alongside of the fluffily speci 6285|Customer#000006285|nf 4a5KU8QuLSMPZWmEkyVq6UAD065pgsgI|3|13-353-741-3596|3913.54|MACHINERY|rmanent deposits print furiously among the slyly e 6286|Customer#000006286|WbndcI8V39JL1oxVYtRHMvESj1|16|26-227-468-4312|3895.62|AUTOMOBILE|le blithely ironic packages. furiously bold instructions wake blithely ruthless, ironic pinto beans. even 6287|Customer#000006287|b1WalykIgCZUEIk,KPnxg7ytSfEtEHsVRCX98H5|20|30-755-248-6558|-296.05|BUILDING| beans wake slowly according to the regular, ironic deposits. furiously regular requests cajole. express 6288|Customer#000006288|iN4rgnOJ5RH8M5r6fvF75YNLBiT4loi|11|21-622-156-3974|5184.42|HOUSEHOLD|unts sleep carefully quickly even foxes. fluffily bold requests about the silent theodolites breach quickly above 6289|Customer#000006289|MuOF83xgQwBnj42OUkVScHj7RKGR3U7NkdzVdLc|17|27-437-457-4918|1509.92|MACHINERY|efully furiously pending pinto beans. pinto beans after the blithely express pinto beans haggle carefully packa 6290|Customer#000006290|Vi6,QwAcedleabbr0SEv6LeHEU9SluHi57,|3|13-927-921-7780|9223.20|HOUSEHOLD|ual, express deposits wake carefully final instructions. slow, ironic requests across the furiously regular package 6291|Customer#000006291|JwSPtW9LtBALXgfhtQ3H|15|25-657-320-9686|313.68|HOUSEHOLD|ess instructions wake slyly by the 6292|Customer#000006292|xR0ShtQF06IrULp,|20|30-313-386-9424|910.80|FURNITURE|ely regular requests cajole blithely regular, unusual accounts. ironic, ironic packages about the slyl 6293|Customer#000006293|40Q UY9xqRxuXTIF3Kh58iCsPTn6g6FXAU|3|13-828-651-7919|4785.81|BUILDING|uickly final excuses was slyly silent, express requests. carefully regular requests g 6294|Customer#000006294|TEYcUTvYiWvxYjLqLx1a7dI7nqlcLDSG6S732 |10|20-377-548-8347|1065.94|AUTOMOBILE|uriously. furiously bold platelets detect theodolites. quickly final accounts can cajole according to the bli 6295|Customer#000006295|YEDvY2dxIZA5AFzrYqM2R,Qu0BWeRzqZ|11|21-326-272-7171|5134.99|BUILDING|sts boost final, final packages. unusual pinto beans against the carefully final re 6296|Customer#000006296|igCv4BEwY9,779Tix1Jw|21|31-763-742-7377|6768.63|BUILDING|could sleep blithely inside the regular accounts. furiously ironic accounts slee 6297|Customer#000006297|DX,A9MX7Xpum|22|32-968-252-6956|7365.56|BUILDING|ctions haggle furiously carefully final pinto beans. pending deposits dazzle regular, regul 6298|Customer#000006298|v5JDDFsvutMitkNO|10|20-805-968-6774|8028.38|FURNITURE| furiously fluffily pending packages. silent pinto beans across the final foxes nag perm 6299|Customer#000006299|,4uQAq3HIX7qNb2tA yA|4|14-120-953-8397|4991.04|HOUSEHOLD|refully regular ideas wake carefully dependencies. slyly regular excuses serve. fur 6300|Customer#000006300|iFuTBsELWUD|14|24-935-162-6227|7456.50|AUTOMOBILE|bold ideas. depths hang ideas. slyly final requests boost fluffily above the quickly bold escapades 6301|Customer#000006301|JdPIarVVF5vd0laNpjFh|18|28-940-337-3490|7399.19|AUTOMOBILE|nal instructions eat slyly. pendin 6302|Customer#000006302|Pmy5pzuh1YeOHkpciY0bCwMtDvMEPy2816MEBz|11|21-127-281-7462|1273.35|AUTOMOBILE|o beans. bold requests promise carefully slyly regular pains-- boldly ironic accounts are blithely pending 6303|Customer#000006303|YlPFtsXInnJ9pylCP3WVnzHDGX,RbchWeoZ|21|31-241-146-4709|4079.40|HOUSEHOLD|ans run quickly slyly special deposits. carefully 6304|Customer#000006304|Zg6sBmlhMs9XVZeDS3D|13|23-243-641-3155|8426.13|FURNITURE|ccording to the furiously express instructions. asymptotes affix slyly carefully even do 6305|Customer#000006305|26EjrYGIc38wIM,rkf nwGVNIaWPfmsEpjmyyfP|0|10-141-833-6715|2088.12|FURNITURE|dencies. blithely final packages integrate furiously. pending platelets use across the quickly bold de 6306|Customer#000006306|Bp0L3gbYTOCVi9N3Tq6CtAMW6jFuqWTmOEB|7|17-595-455-5504|3048.38|BUILDING|luffily across the special requests. quickly special packages among 6307|Customer#000006307|FJlRZXhd4LA,uWwwa78AoylhQkOOJrNZ|19|29-453-201-8768|5047.86|AUTOMOBILE|eat. quickly pending courts lose furiously against the slyly special platelets. carefully special 6308|Customer#000006308|FmSPcC6tPeT6M6|10|20-499-286-6565|5329.24|BUILDING|s. stealthy, silent packages print. deposits are slyly even, pending accounts. blithely even 6309|Customer#000006309|pHdznY21xwitGpZp|7|17-525-121-8608|4575.56|AUTOMOBILE|deas. quickly quiet instructions can are. express accounts use after the 6310|Customer#000006310|n2yStHsmbyEE6P|6|16-432-150-5510|9468.61|AUTOMOBILE|e final packages haggle blithely above the qui 6311|Customer#000006311|wSwN,5PelFjtbAIv,SzZXy05,GTmiXBsXSTfI|17|27-553-115-9891|-71.60|HOUSEHOLD|inal, ironic accounts above the excuses use evenly among the bold pinto beans. carefully silent foxes above the r 6312|Customer#000006312|QMhVQC0PiraO8oEHoMJ14b8Gxf9gK4h5ATkCn|4|14-870-343-5163|4562.08|AUTOMOBILE|ily silent foxes above the ironically unusual packages cajole quickl 6313|Customer#000006313|g66sNXSi5LC9tvZ|6|16-911-713-4691|2933.72|FURNITURE|beans. boldly final deposits wake blithely. slyly ironic packag 6314|Customer#000006314|lqbO7daGUg0T0QppRJTLXXan2PJ4YiZUtA|24|34-155-178-2373|2996.03|HOUSEHOLD|uickly regular packages alongside of the slow, special requests wake slyly against the furiously regular accounts. 6315|Customer#000006315|H3lTGfGTxl|6|16-965-207-3063|5307.12|BUILDING|packages. packages cajole notornis. closely express accounts according to the id 6316|Customer#000006316|jbMHTjobBPcepRl|14|24-427-805-3364|6714.53|HOUSEHOLD|l, bold requests. fluffily unusual accounts s 6317|Customer#000006317|1i8b72gMGW4MBizzmt2G2j9FiUhxKUV2xcJ|20|30-302-599-4639|154.99|MACHINERY| with the silent accounts? theodolites affix. ironic asymptotes across the idle, express requests h 6318|Customer#000006318|iYPP2u47,ZVs pK,|21|31-463-681-6877|6923.15|BUILDING|quick packages wake slyly unusual 6319|Customer#000006319|j1p9Rogz9sN1u9g0cyHBNsV5Uj0sT,mYh2Z5|14|24-464-821-9726|8487.59|BUILDING|ic deposits integrate slyly across the quickly ironic pinto beans. furiously re 6320|Customer#000006320|utTqGM30xwZPpmVURyoh7jWE4emjju6JHYuUAWFs|3|13-381-184-9600|937.20|MACHINERY|ly ironic requests detect blithely inside the sentiments 6321|Customer#000006321|QQ3MOdoHCo9I6SvghQ10xY|19|29-754-952-3500|8433.20|AUTOMOBILE|odolites. even, final requests through 6322|Customer#000006322|NK2pKqhhwp LJnExfiTmLeCZm6bhLkXWRIfxBPQj|9|19-998-647-9970|-95.54|FURNITURE|counts wake quickly pending packages. silent instructions above the even, r 6323|Customer#000006323|CSFI2KxIkCJ7O,KVsI9rtPZZi1cYypJbtN|23|33-716-180-9533|9240.58|MACHINERY|to cajole about the slyly ironic foxes. i 6324|Customer#000006324|o7FUm0oOy,5Cz|24|34-225-267-1395|6470.97|HOUSEHOLD|furiously idle accounts wake among the closely bold foxes. requests grow fluffily abo 6325|Customer#000006325|eI IZBMs2Neixixf|14|24-113-347-2651|9653.04|MACHINERY|he quickly regular sentiments. ironic, ironic theodolites doze. final deposits run blithely ironi 6326|Customer#000006326|Vvne n,VXT ykC6eBZ202wy4ev360a7jWQugfmC|9|19-653-669-5014|8701.02|MACHINERY|yly. furiously express foxes sublate slyly. pinto beans grow. slyly final deposits cajole quickly f 6327|Customer#000006327|zfiwjLhIm3ykcc0PExS1enEBsWkf|16|26-366-282-8221|2114.22|BUILDING|gle. accounts might nag silent deposits. pending packages use fl 6328|Customer#000006328|wibapP1Bq,wY|23|33-499-740-7766|1651.24|FURNITURE|eposits wake carefully regular, final accounts. pending pinto beans after the unusu 6329|Customer#000006329|GgzjDBrJgBnzfhNzdcbe7XSOs5a9CVHoO|22|32-532-430-2910|8415.80|AUTOMOBILE|ully regular accounts. carefully unusual grouches are. slyly even requests boost b 6330|Customer#000006330|6c1MemjbKFOa41b5CGI4rox|16|26-223-365-1109|4490.78|HOUSEHOLD|counts haggle alongside of the careful requests. fluffily regular deposits are. ideas grow: slow braids sleep 6331|Customer#000006331|7,qAyD7LhheRuOcwIJEzmPI|9|19-824-332-5078|3583.47|MACHINERY| between the pinto beans cajole quickly among the slyly special foxes. furio 6332|Customer#000006332|s7FTXH37X2fnlxS|6|16-913-396-4738|436.62|MACHINERY|yly ironic theodolites cajole fu 6333|Customer#000006333|IExKkHfdJck,eoVnei8NhqREtBsDfpFKG6otE|4|14-571-939-5220|-856.26|HOUSEHOLD|hinder carefully. deposits cajole 6334|Customer#000006334|RwSGFTlf,AMKTl2|24|34-336-748-8274|5795.61|HOUSEHOLD|to haggle. furiously final packages haggle. fluffily final pinto beans a 6335|Customer#000006335|4V2jDP,swx9N|18|28-547-289-9779|-197.98|AUTOMOBILE|xes across the ideas wake above the requests. carefully unusual theodolites according to the slyly fina 6336|Customer#000006336|IZ8GS3783y5 K6zMxFP,wa9cvcmzVIe4noGXa|14|24-979-385-1940|3438.49|MACHINERY| foxes sleep alongside of the ironic, express gifts. de 6337|Customer#000006337|PiZCe6IVnzD4lUtquLcVYM0eH|22|32-187-888-4292|5610.32|BUILDING|s. carefully final requests use bli 6338|Customer#000006338|PUuGLhCv0G0OYIeBs|21|31-679-556-1647|9653.55|FURNITURE|ully regular accounts haggle. furiously ironic sheaves affix. furio 6339|Customer#000006339|tk70NFG92XanAXo,NySdfkSiQcpyXHW|12|22-301-127-4981|6226.35|BUILDING| pending packages. slyly ironic accounts wake across the furiously even instructions. pinto beans 6340|Customer#000006340|oxxavhy2E4A No|2|12-958-657-2432|1279.15|HOUSEHOLD|inal dinos use. ironic, regular theodolites sleep quickly theodolites. even pinto beans wake accord 6341|Customer#000006341|JPBISK7sJEEmLqyjH8gwtLCqIgByZEh|16|26-657-318-9584|2573.55|AUTOMOBILE|ites. special ideas are across the reg 6342|Customer#000006342|70kmTAuDuG9pOnUUXeUp en555nKm3lyh|11|21-100-162-4466|-326.28|BUILDING|n, regular ideas haggle blithely. bold, regular packages will boost blithely. car 6343|Customer#000006343|RxSzYVBobmzOfG7NTC3JnFIvUoIcgSo|20|30-972-622-2287|3104.63|HOUSEHOLD|ngside of the close accounts are final, e 6344|Customer#000006344|ad,0JVklm5JfxVGyc LkIojMtdbWpZnXT2a1aKSX|7|17-693-904-1827|4576.81|AUTOMOBILE|uffily special dependencies boost bl 6345|Customer#000006345|es2go,e,Lr4TRItOoUAQKzu1OJIS,8cq50Yb|19|29-627-959-8977|8513.41|BUILDING|he carefully final excuses. final pinto beans sleep 6346|Customer#000006346|50RfbxMaJICGgfSOXGA|6|16-291-905-7678|5910.74|FURNITURE| requests. ideas nag fluffily alongside of the slyly pending dependencies. quickly 6347|Customer#000006347|70IEncoPD8K5Vin9BRkBxdndJkIraSM|23|33-977-252-8569|9804.85|MACHINERY|egular requests sleep furiously fluffily thin somas. special instructions after th 6348|Customer#000006348|1W8CqVR0os|22|32-400-947-4328|8251.10|AUTOMOBILE|sits after the unusual, even pinto beans are sl 6349|Customer#000006349|mb,ZNqwU0WkZFYDX6hw|8|18-762-580-1517|8319.51|MACHINERY|mong the ironic dependencies. fluffily special excuses solve blithely. regul 6350|Customer#000006350|9PHlaqUJG84BxDkb8fe|9|19-525-320-9404|3019.75|FURNITURE|efully according to the quickly final requests. regular packages wake furiously after the quickly furious f 6351|Customer#000006351|mXVsAYZWVd8rduL2Ndnd20a bFxcZ39umb|22|32-148-461-6773|7852.79|BUILDING|uickly regular accounts cajole slyly around the fluffily ironic courts. unusual theodolites a 6352|Customer#000006352|RoiheUBTOtjUEm L2kUWHswZBOmhW|23|33-416-643-4777|1902.93|MACHINERY|beans detect ironic, express requests. furiously final asymptotes boost furiously. express acc 6353|Customer#000006353|TiJAQNquw5b57kjKHrf6RLX|19|29-104-330-1710|3465.89|AUTOMOBILE|equests. asymptotes are along 6354|Customer#000006354|NUj4uSOE6ZlEKd5dhc|1|11-652-847-7151|4231.75|HOUSEHOLD|ons wake quickly above the busily ironic deposit 6355|Customer#000006355|0gR84cQxuKr0m6V3z6YnGms2Kqj5 FT FHE5YRwg|3|13-206-334-2025|6066.98|HOUSEHOLD|ven, final deposits are carefully carefully regular excuses. quickly unusual excuses 6356|Customer#000006356|6xjYndpkg 0HArUyB56Xqv7EyDD7JrEi|4|14-568-481-2395|6800.46|AUTOMOBILE|ronic ideas integrate furiously careful platelets. blithely regular packages sleep carefully. 6357|Customer#000006357|,HVhNgTVe,T|22|32-202-348-8130|3312.51|HOUSEHOLD|e carefully final deposits. carefully regular deposits run 6358|Customer#000006358|aynGZ5F8MJIzxoEJc kCqh7J9vj|16|26-600-100-6775|297.48|AUTOMOBILE|ages sleep slyly decoys. sometimes bold accounts nag quickly despite the 6359|Customer#000006359|fhA9rzJfqM686ozX RLctv|22|32-890-744-3388|3722.61|MACHINERY|oost carefully. quickly ironic deposits eat carefully. ironic requests boost furiou 6360|Customer#000006360|WYPts48L0tn7iuLS58Cw8JUY2GUY0enx|13|23-508-750-7646|6814.61|AUTOMOBILE|boost slyly blithely bold accounts. careful 6361|Customer#000006361|VzLr6guzIzrZpZfDZjndwix,|23|33-598-626-9499|-780.01|AUTOMOBILE|ggle furiously regular requests. regular, special requests use. fluffily express asymptotes are slyly alon 6362|Customer#000006362|0H7VGv7MTGlMxF8igvaoXhElF8S bqj|15|25-667-257-1612|236.71|HOUSEHOLD|riously regular theodolites-- quickly unusual packages are fluffily after the quickly silent packages: 6363|Customer#000006363|M,pjP 1RfkcWiPc0wMlUmiaV4cmlz57JMV1BI,|15|25-466-653-4756|5382.91|AUTOMOBILE|carefully ironic requests cajole carefully across the carefully regular asymptotes. quickly pending packages slee 6364|Customer#000006364|P7n15pH0vDHA|3|13-693-752-4345|4585.65|AUTOMOBILE|according to the carefully final accounts maintain carefully among the even packages. ironic, i 6365|Customer#000006365|q9FqvI49NhkzJH6lwSPbM,8sD|23|33-576-256-7432|2994.15|HOUSEHOLD| use blithely pending courts. even, ironic deposits about the quickly special dep 6366|Customer#000006366|EAiyyzQp,q7GUp0wQzThB4OasmaTEnE4z|12|22-771-559-9705|6622.87|FURNITURE|s are according to the fluffily ironic instructions. quickly ironic packages haggle slyly among the furiously 6367|Customer#000006367|hGP9UlKdD2BN3LePLnOJ|2|12-744-594-5061|-340.83|MACHINERY|y final foxes believe ironically. carefully pending theodolites haggle. carefully final pinto beans haggle care 6368|Customer#000006368|9wlLx9qDuskAA5Lg3CgbgK2,RYX3|7|17-527-847-1825|591.04|HOUSEHOLD|arefully regular ideas use furiously after the special, special waters: accounts alongside of t 6369|Customer#000006369|2GoiGrH9dEHSKHX3Y8fbA|3|13-817-557-7983|9563.67|HOUSEHOLD|e carefully final accounts. quickly final d 6370|Customer#000006370|mpRkYWqUoJQncfc7Q3VEEgI3eyoGyKFzPJ|3|13-586-788-3167|2734.72|HOUSEHOLD|wake furiously carefully regular theodolites. unusual foxes haggle slyly among the regular depos 6371|Customer#000006371|sqDry7KOh6ztkEJrGoX Y7NPIBW3|16|26-326-418-1698|2443.10|MACHINERY|onic packages dazzle furiously. ironic, final foxes wake slyly. re 6372|Customer#000006372|cN5aX8yXPYruA4rITmdh8e1QaOEyjEeub|7|17-585-758-4417|7197.10|MACHINERY|ithely special requests are alongside of the quickly final requests? furiously regular instructions lose furi 6373|Customer#000006373|Ap2nqlR9SntNoWkk5DFSON84r|22|32-608-822-4087|3700.15|MACHINERY|he packages. unusual asymptotes haggle alongside of the final, 6374|Customer#000006374|6U,tnx3EQ6ymmLLtrRYD8FwwaQEiwvEqA2pL|20|30-879-852-8442|3388.38|AUTOMOBILE|s at the ironic, final pinto beans nag quickly after the packages. slyly even instructions wake furiously 6375|Customer#000006375|,f00Mk7z1TQ4lHuZQA|17|27-709-238-4692|-510.18|HOUSEHOLD|nal excuses. quickly bold packages across the express requests sleep slyly about the slyly final foxes. ironic de 6376|Customer#000006376|9xX7j8zvAb3rY,y2N6rakYVhkcNShJZ|6|16-888-795-8432|3376.54|AUTOMOBILE|lent foxes affix against the even deposits. fin 6377|Customer#000006377|fb5rLUH6Hn|18|28-929-173-8781|9538.01|HOUSEHOLD|lly. instructions are. regular realms after the deposits integra 6378|Customer#000006378|jk4,Yt5J6YsUnLKs3Fj8PWF8y|21|31-227-241-6464|330.88|BUILDING|. express pinto beans doubt quickly. silently bold platelets haggle. furiously regular packages sleep alongside 6379|Customer#000006379|fJOnWmoLfEKa8FDt9T6foVT2njtBKSEnnsQlV0|1|11-989-205-6094|4057.27|MACHINERY|lyly regular deposits. foxes cajole blithely regular, ironic accounts. reg 6380|Customer#000006380|A3CZEoXr2U3M9TJq|14|24-601-213-9207|-453.65|AUTOMOBILE|its nag quickly. furiously unusual platelets x-ray. car 6381|Customer#000006381|qgLG,UnydWWKKYvXEe6g,|7|17-877-502-9214|7346.88|HOUSEHOLD|onic instructions wake furiously according to the carefully express instructions; pending theodolites 6382|Customer#000006382|iT3z1RZJfuFhTVQWn gUQQ30J59FLhUtNa1y|5|15-884-325-7498|2101.57|MACHINERY|ely pending requests haggle s 6383|Customer#000006383|pZMLOQVPjfqg3JvDp|8|18-910-470-6748|9130.34|MACHINERY|its. blithely express asymptote 6384|Customer#000006384|eRUjfmK9XOnuIxULs,H3g2jasWFLc|23|33-767-423-3297|1756.89|FURNITURE|sleep alongside of the slyly idle packages. slyly si 6385|Customer#000006385|feb0lDE33xT7COEBl4CL|15|25-446-991-1683|1948.61|AUTOMOBILE|eposits haggle blithely regular, pending requests. carefully even 6386|Customer#000006386| ,6di,DsCLRoei7glq03Dv03xA|24|34-991-678-6036|1252.97|HOUSEHOLD|sts sleep carefully blithely bold packages-- the 6387|Customer#000006387|YU5Ai0APMax|10|20-581-220-3948|2536.86|MACHINERY|ckages use slyly regular, regular deposits. special, regular deposits 6388|Customer#000006388|p8xZE4LxJw,BhxqdqhnokSPs7TIAZyieKyyzse|14|24-525-264-8052|9925.28|HOUSEHOLD| according to the regular requests cajole carefully above the even depend 6389|Customer#000006389|h1DyF2DxWF0|10|20-749-986-1120|8270.04|BUILDING|slyly express packages affix. carefully ironic requests use among the bold foxes? blithely regular hockey p 6390|Customer#000006390|bn2EoZThG0 s3CAOW0V|17|27-231-144-4566|7571.48|AUTOMOBILE| bold asymptotes use fluffily special packages! slyly final theodolites unwind slyly about th 6391|Customer#000006391|tJFRcLobeDbMdgzr1EfSw7PtH8Uqx97Pd|10|20-447-685-8271|3923.68|HOUSEHOLD|yly thin braids haggle quickly unusual, pending deposits. ca 6392|Customer#000006392| Nn14Ugte47JXBLpdJj4NAXbnsmyk2ykL8v0B|6|16-236-622-1653|9277.66|FURNITURE|s use. bold accounts affix about the ironic theodolites. ironic, ironic packages according to the special pack 6393|Customer#000006393|LrSfeTLkBD8iJmD4,mkCP0awj6UnAxhohA|10|20-600-941-3910|2762.42|BUILDING|ial sentiments cajole quickly. slyly unusual accounts boost blithely regular 6394|Customer#000006394|Qsb,p1hXnullljxxWg7svdXJSXRT58Tjxm2|2|12-653-895-4516|5972.35|HOUSEHOLD|rs doubt even requests. slyly express requests haggle ironic packages. pending accounts nag fluffily after 6395|Customer#000006395|g qNyxByuAIIKgSdK8ye9mw B1ujFXueIu9|17|27-165-881-6729|379.03|MACHINERY|are furiously. carefully silent accounts snooze quickly along the bold, final depos 6396|Customer#000006396|OkUGWXg2r42,|21|31-720-445-5419|-695.45|HOUSEHOLD| even theodolites nag furiously. busy instructions affix slyly after the careful accounts. doggedl 6397|Customer#000006397|OupmJTZAYWzow1H3zbv,v|3|13-204-711-1078|2415.67|AUTOMOBILE|ly against the furiously express notorn 6398|Customer#000006398|nE3eEiDxvFRyoT|5|15-144-311-5947|8155.12|AUTOMOBILE|ly special requests. bold, ironic accounts nag furiously sly 6399|Customer#000006399|dQbyCKR,g2,MXLYPiFE1NlLxzL|20|30-783-132-6452|1103.67|MACHINERY|tes. pending dolphins nag blith 6400|Customer#000006400|B7M0M7FZy06iyPxs VmGJaRJeYgSiL|13|23-177-650-5024|1362.29|AUTOMOBILE|ctions are above the permanently ironic packages. 6401|Customer#000006401|yveLKpTUIx7K6WtnhXAcz5|23|33-945-761-8571|2891.62|HOUSEHOLD|yly final foxes integrate across the carefully expres 6402|Customer#000006402|GLZyBHPsfH,y|9|19-260-224-4337|-594.88|HOUSEHOLD|uctions cajole slyly according to the f 6403|Customer#000006403|9eX1E4GyIXptoHrL5db b7wqVYaUB55u|24|34-376-305-4671|9717.24|FURNITURE|ly unusual accounts cajole slyly even platelets. carefully even package 6404|Customer#000006404|s,kUf5HrSnq,g8un,UAcz7HvRQ MV|19|29-917-546-1913|9678.44|FURNITURE| furiously bold deposits eat ironic ideas. slyly regular theodolites across the even i 6405|Customer#000006405|I0BmHPY,nxSz0vA8a|24|34-382-195-4386|6618.54|AUTOMOBILE|sly pending braids. slyly even platelets sleep fluffily final packages. express patterns integrate finally pen 6406|Customer#000006406|lL3dzJv7lVqtqz,DiuUHcvH1EcpM9X8jba|21|31-342-422-4630|3830.68|AUTOMOBILE|es would haggle after the final pinto beans. slyly ironic deposits sleep quickly. asymptotes h 6407|Customer#000006407|BtK4gy01cd|6|16-862-127-3910|8151.95|MACHINERY| ironic platelets. carefully bold excuses mold slyly above the pending pinto beans. asympt 6408|Customer#000006408|oa9bZUw3jwpkK,Gd|9|19-912-624-3972|2836.65|AUTOMOBILE|boost against the quickly final requests. daring 6409|Customer#000006409|T0HCEe4pnQrvw|17|27-890-763-2774|-451.44|BUILDING|its about the accounts nag even foxes! slyly i 6410|Customer#000006410|GGGN,NDpImMdzCHOjMg6D,35uOrTr3yJ71fW|9|19-197-885-8569|530.65|FURNITURE|side of the special pinto beans. dependencies are. regular, ironic foxes cajole 6411|Customer#000006411|z3ej9PD1gmKm1uaQySQhjFndp0kz062lnUVt|0|10-759-591-4295|6576.83|FURNITURE|y bold foxes. foxes haggle slyly about the silent, special ideas. slyly regular requests haggle across the pendi 6412|Customer#000006412|wJVfQt,87Ckyca4rS4yh8PPGbzQyVxCXqil9ex|24|34-744-473-7572|6753.76|AUTOMOBILE| furious accounts after the furiously regular dependenci 6413|Customer#000006413|okFK1CC8ibk3ml7X4ZhRl|10|20-949-158-4934|-563.78|AUTOMOBILE|ades. frays wake slyly. carefully final realms sleep. qui 6414|Customer#000006414|3mdeAURniRPufSi6dz2|24|34-799-456-8937|4444.30|AUTOMOBILE| final pinto beans. furiously bold pinto beans nag. blithely fina 6415|Customer#000006415|0SOe8iGHkEtlpwV7 e|10|20-971-763-1702|3396.66|HOUSEHOLD|ependencies. quickly ironic pack 6416|Customer#000006416|WGVJj9TyQ5ac3qL7RHW8jSOYQH6XZV|2|12-415-551-2448|6601.37|FURNITURE|lar theodolites. accounts use sometimes fluffily ironic packages 6417|Customer#000006417|06dHuGAxyFTfEygMp6ZU|23|33-360-140-2353|1256.36|MACHINERY|c sauternes. carefully final fox 6418|Customer#000006418| ydC4OuGkly|1|11-186-687-6620|4592.96|AUTOMOBILE|sts. unusual deposits haggle against the regular, 6419|Customer#000006419|JisGDcZRiDHg4yLNKSc2HPy32KfTIL,|12|22-243-778-7315|4838.38|FURNITURE|ound the fluffily furious accounts. courts haggle furiously about the quickly final packages. carefully eve 6420|Customer#000006420|ROdVxbHD0,GVCY,9NwYfDO|22|32-469-891-2051|4658.51|BUILDING|sts cajole according to the dependencies. final requests cajole carefully. final courts are fluffily express, slow 6421|Customer#000006421|fv4AI3OzYxdrM3I4cuDUoxMfOubOlO1m6Oxmv|3|13-450-314-2988|6650.59|FURNITURE| according to the slyly bold accounts are fu 6422|Customer#000006422|ZltvyXcMiO|2|12-501-888-2104|3415.78|HOUSEHOLD|nto beans. fluffily unusual deposits boost slyly. quickly 6423|Customer#000006423|7nudsRA5wmNNjs4,FThBletUyOIq|19|29-421-996-4033|9077.24|FURNITURE|inal, special accounts. furiously final pinto beans a 6424|Customer#000006424|ieZS lxnQBTqJO8BOfz7mcPbUsnlS|6|16-525-665-8732|258.29|MACHINERY|lly even accounts sleep slyly alongside of the blithely final asymptotes. bl 6425|Customer#000006425|y4kyZJUM4AOCGcaIjuj8as6XMgEb,tZyXaWTY2yS|9|19-638-440-4991|5006.99|AUTOMOBILE|fully even requests integrate quickly final pinto beans. carefully silent ideas cajole 6426|Customer#000006426|MBbafnBIIVTjNxVB8,yZcLEMlp5FbyMCFozSR|16|26-200-900-4409|529.42|MACHINERY| bold requests wake slyly. even instructions are. slyly express accounts nag blithely 6427|Customer#000006427|Xa17u3du45DiLnp8SEaIgKaS8l|5|15-852-433-6135|7907.55|HOUSEHOLD|s integrate even, regular foxes. blith 6428|Customer#000006428|jiZdycWOv3UcTXmeQYadg7xOV6LRX|20|30-758-610-5037|9217.56|HOUSEHOLD|kages boost slyly carefully pending packages. b 6429|Customer#000006429|723rhJstUlnqt siNCyOV67ZVvTEXcJ0PqJ1uE|14|24-314-971-8033|4648.01|AUTOMOBILE|thely pending theodolites haggle furiously express requests. fluffily even requests are furiously pending requests. 6430|Customer#000006430|xXFJR6GUfqeCyN5PiAvrw7HV2U|11|21-549-125-1175|6077.54|BUILDING|quickly among the special accounts: even, special packages according to the quickly sp 6431|Customer#000006431|alr2AvVdnwHuR6MYiNcZvdfHfN seFOpZMS|4|14-847-121-5795|8310.76|MACHINERY|inal orbits about the final deposits wake furiously blithe instruc 6432|Customer#000006432|4PGhIJsN8,hNjTsk6|22|32-173-374-5893|9988.42|HOUSEHOLD|encies. special requests after the regular, regular packages use 6433|Customer#000006433|a3pPw8Sauu6hhR4k5uL7wg1H95kiZ64Tk|19|29-909-421-8085|2412.87|FURNITURE|ickly final deposits use carefully. blithely 6434|Customer#000006434|XEo,6jjEzKR0aOUcozaEu8 gisw|2|12-500-275-2120|5131.87|BUILDING|e slyly bold pinto beans. final decoys use against the special ideas. furiously ironic courts against t 6435|Customer#000006435|Bg7iFjWjLVQKT0AEclRbHPBTDpvlrEXKAD9K2|21|31-382-296-1618|9295.79|AUTOMOBILE| bold deposits cajole furiously theodolit 6436|Customer#000006436|yfj8DtQgQrb52|13|23-860-371-1102|-651.11|AUTOMOBILE|ular dependencies cajole evenly final the 6437|Customer#000006437|pZ639dzBsf14zfwauL0jOSYTGQ870YRctiD|23|33-994-170-5321|-282.52|AUTOMOBILE| about the final accounts. slyly bold instructions are iro 6438|Customer#000006438|RyVaFQyhkoASdbPjYYBI|3|13-147-903-2434|1684.90|BUILDING|lithely unusual instructions nag according to the slyly special theodolites. express, r 6439|Customer#000006439|JoIRrAaO8XlKsPF35eCO8Vcy0o5L|13|23-346-902-8254|2048.44|MACHINERY|cuses. final, special ideas boost furiously. furiously close dependencies above the daring deposits wake according 6440|Customer#000006440|EhPeQVf4268eEE81yT7L3kHeBFC1nsvYjF|22|32-963-990-1859|7040.39|MACHINERY|aggle quickly excuses. regular packages nag ruthlessly regular, idle sentiments. bold account 6441|Customer#000006441|XG8rZ2j868y,WH1VUBZ|19|29-490-926-8692|9951.53|BUILDING|es affix carefully. slyly regul 6442|Customer#000006442|KHfAHw,EYDgYxunkuoCi5UZb 0nZL6i868CET|19|29-922-394-2085|9442.52|AUTOMOBILE|osits haggle quickly slowly final requests. even, fina 6443|Customer#000006443|pHv34YqZSMUDlEyxk Mlvf6Ub,|6|16-852-419-2587|1478.34|BUILDING|nic courts haggle slyly blithely regular theodolites-- special excuses use slyly. ironic dolphins are slyly f 6444|Customer#000006444|7wOibQuMiSsum0zAfieMfvToucnh|15|25-268-268-9763|6469.44|HOUSEHOLD|sits use among the fluffily even foxes. furiously regular dependenc 6445|Customer#000006445|J8HlYAyAUaCQpMoj4cTlS,TDmGK8gcKnrDxmUPG,|5|15-192-490-3051|5966.09|BUILDING|, regular theodolites according to the quickly even pa 6446|Customer#000006446|V0Dd,KY57jzAlKGMwRdstqlBUgAj12janY O8|2|12-139-856-4622|5131.95|FURNITURE|bove the regular, even instruc 6447|Customer#000006447|2iQk,R9iPSQKkb3e7oQi |23|33-450-821-7164|-380.57|MACHINERY|after the ironic requests thrash slyly carefully special 6448|Customer#000006448|13Wg2FUg2Ffki TBsY517Ztb4fytGvRe8peQtJ0|0|10-486-606-3968|3677.31|MACHINERY|ing requests. ironic, ironic deposits cajole above the carefully bold deposits. pending i 6449|Customer#000006449|pE4YcxWffV,7kwV8BX4XyOY4S7|1|11-503-687-8564|4467.60|MACHINERY|ptotes sleep against the slyly regular deposits. express requests wake ironically across the blithely r 6450|Customer#000006450|Gc,ojHknX6MUSgmXpv8EmEhKPkU|24|34-272-661-2143|9888.45|FURNITURE|riously slyly silent courts. ironic, final excuses sleep after the regula 6451|Customer#000006451|E,ekihaEI,uN4kGRUwQ csQjx3hVRvUDh,|12|22-457-204-2046|5307.46|AUTOMOBILE|ts integrate carefully carefully pending theodolites. quic 6452|Customer#000006452|Qrin8a98Frzmr0kMzVeX9h6|18|28-426-597-9924|1512.79|HOUSEHOLD|ironic theodolites. requests detect carefully silent courts. slyly 6453|Customer#000006453|pZ0jI IY sh07aKPdmQ7uv3ta8eU,jCXFbL1|22|32-372-439-2517|2372.25|BUILDING|into beans sleep at the blithely brave accounts. unusual deposits wake 6454|Customer#000006454|ypyPdN8tQvcYERecN,3Oufizu|1|11-276-495-7172|4437.76|FURNITURE|ly special packages cajole ruthlessly around t 6455|Customer#000006455|unZZGWq2HBJ4EKR3UlbpcORxMlO|2|12-239-754-9487|3482.98|MACHINERY|c pinto beans sleep blithely carefully express accounts 6456|Customer#000006456|XF8NeBd23gN|23|33-589-473-6210|5405.47|FURNITURE|dependencies. final accounts are furiously. bold instructions use quickly. express, br 6457|Customer#000006457|RWwZBhG0WF8ixJJ9csB|2|12-292-531-2808|9754.94|MACHINERY|olites cajole blithely. blithely bold foxes wake blithely. regular deposits sleep blit 6458|Customer#000006458|9hveScYbQY iKmqJuNs9PetcC|9|19-701-840-4110|5425.87|MACHINERY|pending asymptotes are alongside of the ironic, pending dependencies. blithe 6459|Customer#000006459|zd0TdwWz,IhDrPeawC3G9SHV2Tr0r|16|26-264-700-6190|633.59|AUTOMOBILE|carefully regular requests are furiously about the packages. furiously special accou 6460|Customer#000006460|Pr5bp2A90JjbrjE1XN5HM1|12|22-504-912-3129|9685.49|BUILDING|ending ideas. regular platelets use quickly final theodolites. foxes affix above the regular epi 6461|Customer#000006461|vgpOxXZRI,,Y92 1NO7N98jKwnOY7|5|15-362-557-8104|4166.62|AUTOMOBILE| blithe packages boost slyly. blit 6462|Customer#000006462|3njV5Ft7NhNxW50neyjlryC0Ylq0fI0n,P|18|28-848-982-8505|433.66|AUTOMOBILE|l requests play carefully across the instructions. blithely 6463|Customer#000006463|M9Oy,Xx9CbN GSl3L6khL0LTRgCi8MgX0gTzVbCQ|23|33-939-251-7239|4447.04|HOUSEHOLD|final foxes. accounts should nod against the unusual, even instructions. blithely regular theodolites hag 6464|Customer#000006464|eF9E6ScHCw9,z8nF0py9 ySlB0 iHTIEEZRWl6H|1|11-870-572-9943|5468.53|FURNITURE|telets could are quickly regular packages. fluffily iro 6465|Customer#000006465|hh2iSvQ3ixAtfId4QsSzJOMORcy4t|24|34-206-617-3619|2544.19|FURNITURE|express packages. ironic requests wake carefully 6466|Customer#000006466|nxPwv4px0 1LW05BtXaT7m,b2a63Dq8s25b09|20|30-314-197-3871|1131.19|MACHINERY|believe furiously. unusual, pending deposits wake. quickly pen 6467|Customer#000006467|VKsGXyxDaRzjYny fn 1zOOjPmkA9ZDp1C6lu|11|21-298-482-2423|1798.41|BUILDING| slyly special instructions boost carefully. final packages 6468|Customer#000006468|NTWZxBhUwmLUSwoXnrTwKdFs2|8|18-741-953-3789|6840.45|MACHINERY|y ironic deposits are along the furiously ironic requests. theodolites can are. accounts use after the f 6469|Customer#000006469|kDCpN,X8RAKe6EHXL6zsygq57|7|17-884-283-2518|4429.67|MACHINERY|osits hinder slowly along the regular accounts. even, unusual foxes wi 6470|Customer#000006470|BHgwAOTI0WReuiU9Ub1y|24|34-790-284-2739|-501.95|HOUSEHOLD|c platelets. carefully regular platelets sleep carefully. blithely unusual dependencies haggle. packages use furio 6471|Customer#000006471|z64U8Zsf 1ItZlk|1|11-284-691-1178|9608.94|AUTOMOBILE|bove the slyly even requests haggle even ideas. slyly r 6472|Customer#000006472|sLyBV0ZtMvAH1QSOnxg6koHGsKzzjm7RWuEMRWQV|20|30-220-128-1332|4515.01|HOUSEHOLD|onic pinto beans. furiously final platelets promise never. fluffily ironic instructions p 6473|Customer#000006473|Ee0nC9W1Y9j6e4V|9|19-156-916-4569|3728.77|AUTOMOBILE|al theodolites boost across the even dependencies. express theodolites sleep enticingly fluffily ironic i 6474|Customer#000006474|AqmYPBgr9r vxl8sMzecS5ak3JJ4ByqN5 |9|19-500-356-4558|478.16|BUILDING|e blithely pending dolphins. packages 6475|Customer#000006475|itSIjP3CZMWKp3MjgSuPFim0l6T|8|18-427-310-9935|4338.27|FURNITURE|pades. blithe attainments impress quickly ab 6476|Customer#000006476|NbYct5LvnQfrajINpQm34t3yxD32UQED5bk5dNPG|16|26-531-511-8701|3327.81|AUTOMOBILE|side of the deposits are slyly around the ideas. quickly final accounts wake furiously. bol 6477|Customer#000006477|qylnoy8Yoou7au1IjGNmAR,amKok|1|11-336-397-1897|4554.64|MACHINERY|ly permanently final warhorses. deposits ca 6478|Customer#000006478|E1ZXZROw4YBwc4HhN2M|6|16-483-594-1126|9874.64|MACHINERY|lar accounts are quickly fluffily pending packages. special pinto beans above the slyly bold excuses hag 6479|Customer#000006479|r,YZ8Bqo9gHbHFfnEITjeU4riKFay4mvovDTmc|11|21-143-392-9914|4160.96|FURNITURE| quickly express accounts. slyly furio 6480|Customer#000006480|jZCx1qKm8AJtV|14|24-732-299-9650|4276.33|BUILDING|ending realms cajole at the caref 6481|Customer#000006481|QM0JCcwqe 2OSh5LpTjoQVL0dIbCJ3unJO5V|9|19-708-287-2759|7278.40|BUILDING|pecial packages are busily according to the ironic escapades. slyly 6482|Customer#000006482|dOVykuZo,txNWSMQncRnWc4Goi9sslnW5zv|2|12-270-253-5013|6111.54|MACHINERY|ncies. slyly unusual platelets haggle slyly above the even, express deposi 6483|Customer#000006483|ZT8TwDuABR77|11|21-679-759-6880|7606.73|FURNITURE| are. carefully bold instructions by the foxes are along the slyly express theodolites. eve 6484|Customer#000006484|VbH7IC5F ZR1CDcyyjxX|3|13-467-401-7503|2438.45|MACHINERY|ress whithout the blithely regular deposit 6485|Customer#000006485|sCWkSZkKOpsnFhyNUIccdfvFvipe|1|11-541-386-1396|5926.60|HOUSEHOLD|ly express foxes integrate after 6486|Customer#000006486|huOvXcWg648ez69Yyebn8|11|21-909-289-7137|1380.84|MACHINERY|l theodolites. slyly regular ideas sleep carefully slyly idle asymptotes. fluffily unusual ideas 6487|Customer#000006487|1JfLyCcXKGeHiEV|12|22-967-924-9865|6789.80|HOUSEHOLD|uests across the packages could have to nod furiously silent dependencies. final or 6488|Customer#000006488|fEUg4BCUhNIcaNVKNdkadz2N2lIDWybFm,quvpnO|8|18-915-822-5617|144.97|MACHINERY|final deposits serve carefully about the slyly ironic or 6489|Customer#000006489|z6IrxkffxNyMYDRx4qR,PQnTDYSMXG3bsdBD8mx|8|18-713-226-2794|3494.92|HOUSEHOLD|s haggle around the carefully unusual deposits. carefully even theodolites are enticingly ironic dep 6490|Customer#000006490|VjpZnXeSD2oIK0U,CNLNfRxZF1NDFadS W|1|11-122-897-9952|8180.23|FURNITURE|n instructions lose carefully alongside of the fu 6491|Customer#000006491|J3,jSVfCkxq91fHjycRrqx1mQ,sGEFBT7iCtgu|6|16-977-273-4547|5239.92|AUTOMOBILE|al foxes sleep fluffily along the 6492|Customer#000006492|nPlHrfRAlpx|0|10-440-853-6387|-723.71|FURNITURE|ruthlessly regular packages. pending platelets near the slyly ironic packages haggle quickly against the blith 6493|Customer#000006493|ZT2dynxWMfOTM|19|29-295-246-8539|500.11|MACHINERY|ending decoys. furiously thin ideas cajole according to the f 6494|Customer#000006494|Vu5i1odOtPFZVNY53su76yFnal |12|22-121-359-7339|-795.34|HOUSEHOLD|ts across the final packages are slyly ideas. blithely unusual ac 6495|Customer#000006495|Hs,w9KO2RR At|18|28-465-937-3117|6679.86|AUTOMOBILE|deposits. theodolites hang blithely after the carefully ironic 6496|Customer#000006496|tdAcewkJ,9IL7tYdP6xFSlR4sxhzqYx|14|24-755-676-4374|517.43|BUILDING|cies wake slyly according to the blithely ironic accounts. slyly even packages across the slyly ironic a 6497|Customer#000006497|0m1WM8YSt2E1jhOAohr3Zs9NJuVlgORZFF85yp|21|31-417-915-1172|4355.91|MACHINERY|refully pending packages. even pinto beans haggle across the pinto beans. final instructions after the foxes 6498|Customer#000006498|M,,F70BZbI7wFy Eij7bct0rFLaJkqFN5DS8vS8|3|13-779-226-9489|2765.60|MACHINERY|side of the pinto beans. slyly express theodolites sleep regular, special platelets. slyly bold depos 6499|Customer#000006499|zXN1H4vSEbL VEvBO5lT8IICF|22|32-496-645-5422|4390.89|FURNITURE|tegrate alongside of the final, final theodolite 6500|Customer#000006500|SQLIuXRcQ3Q7G38eUgMXV4ybIKPQxKS0|21|31-954-475-8701|7764.33|HOUSEHOLD|ggle against the silent excuses. frays doubt quickly. carefully pending deposits sleep quickly final 6501|Customer#000006501|9v8gAkVB KcQoBRKbQ2rykps4fKqYX|24|34-433-352-7689|184.55|FURNITURE|pliers. platelets eat slyly against the even accounts. fluffily express requests are furiously unusua 6502|Customer#000006502|mbRY4j5rudW|11|21-562-896-1594|8710.50|BUILDING|eep among the fluffily ironic req 6503|Customer#000006503|4p3W4XHBSKbswkOpK3jehDgW|11|21-476-545-9987|7492.32|FURNITURE|st the quickly careful instructions wake quickly according to 6504|Customer#000006504|,XoJJFtPqo6Hk7GxI4agFf9r5|8|18-223-523-7723|4395.75|HOUSEHOLD|al requests. carefully unusual ideas en 6505|Customer#000006505|,35tS 0nsQ,|10|20-116-997-7935|1320.29|MACHINERY|al theodolites: carefully bold dependencies haggle pending accou 6506|Customer#000006506|Mol5UoqoOCHvOfBHz3s07NcF7HoU2rIWZGlkcI|7|17-831-994-7711|1146.56|HOUSEHOLD|wake blithely after the idle, even packages. final pains c 6507|Customer#000006507|ewNKMjvFSXSHZRYQnPbMCYqB|21|31-762-523-7126|7134.11|FURNITURE|ar foxes around the carefully even deposits haggle quickly quickly final i 6508|Customer#000006508|Swm,c vbMg36c1z5,4fdBedBNG1OrfhsTPco|18|28-986-386-9035|952.00|FURNITURE|ters are above the express, ironic dependencies. express deposits according to the furiously furious packages pri 6509|Customer#000006509|70iy4ZLmA6,OjYW7tTtxlnxkJf N5|24|34-658-604-7430|4234.26|FURNITURE|ly final theodolites affix fl 6510|Customer#000006510|mHOy9Hl0Qj|0|10-992-585-3630|4211.77|MACHINERY|express accounts according to the quickly specia 6511|Customer#000006511|Hqpb ssH,HNi|0|10-987-779-8094|238.09|FURNITURE|ost slyly carefully even packages. slyly final theo 6512|Customer#000006512| 33vAzHrrXIgd68KBr4mD|7|17-511-867-8977|6571.18|BUILDING| after the final requests. carefully ironic packages nag slyly. fluffily eve 6513|Customer#000006513|0yWTa31spHkfkccVy,mRh|4|14-344-180-9079|-125.66|MACHINERY|ites doubt fluffily. furiously regular deposits 6514|Customer#000006514|F8eUA3o2KlPxQoOP42h,cNYShHe|10|20-816-198-3141|8911.35|AUTOMOBILE| sly deposits alongside of the requests are carefully carefully unusual a 6515|Customer#000006515|knMzD7lwT9|16|26-565-114-8675|6121.09|AUTOMOBILE|t ideas. excuses across the furiously fluffy accounts use quickly about the express, 6516|Customer#000006516|zL3M2qRYhxxz|4|14-657-773-6508|3208.09|HOUSEHOLD|ole furiously among the slyly express requests. foxes sleep blithely blithely bold ide 6517|Customer#000006517|tjhz2CB9lXX0TcmPRX|17|27-923-897-4118|5407.70|FURNITURE|the furiously ironic packages haggle blithely pending req 6518|Customer#000006518|1gxNOhqNUnOgcA8pfdSuYB1ROX|13|23-117-625-2924|3187.34|HOUSEHOLD|inal requests above the furiously final asymptotes are bold, 6519|Customer#000006519|NomK60Y3wXy06J7YEMGp 3,XRpg0j9IxcDtVEAM|21|31-885-482-7206|8593.58|MACHINERY|pinto beans. furiously unusual platelets integrate. furiously special packages are. fluffily express fox 6520|Customer#000006520|GgmUKbGO4YqccuU|2|12-135-446-1603|326.88|FURNITURE|p slyly about the silent requests. qui 6521|Customer#000006521|ocqjyIQB3nrDvcPfNK6YdWLJRV,aNNfYjBl|3|13-856-838-6626|2762.22|AUTOMOBILE|ironic braids above the regular r 6522|Customer#000006522|9lIS6iU8xfx2UB77M,cRbhUYe7WaawYE|2|12-106-450-9122|2611.92|MACHINERY|ress notornis integrate blithely inside the slyly thin deposits; ironic, regular platelets hinder 6523|Customer#000006523|ppBY8l3kJDLcOjqD0mF5H|21|31-855-623-3767|6472.78|BUILDING|counts wake blithely. carefully pending requests haggle carefully. slyly ruthless asymptotes wake furiou 6524|Customer#000006524|3NMnX6Cbi83z3Cul C|14|24-198-901-2901|7024.93|BUILDING|rges. even packages sleep across the furiously 6525|Customer#000006525|9TeyC0eRLeHUiJYR9EA8fimqlu6biVgxdxVx|18|28-268-185-9919|2973.93|MACHINERY|s promise furiously across the final grouches. furiously final accounts according to the ironically final re 6526|Customer#000006526|j2e,TUUVZxaFSuiJauFDS2eo|5|15-609-391-3715|7755.32|BUILDING| courts cajole blithely above the fluffily bol 6527|Customer#000006527|R0sIPtavn8fgJTfsLCVTfSuuFHkTVFh0ZscC6Rub|13|23-215-568-8678|2722.00|FURNITURE|furiously: furiously unusual packages cajole. pinto beans are furiously across the unusual accou 6528|Customer#000006528|5F4Zd7t2UCw COOvRxsTqSFYbNuT3LNWqAbE|15|25-123-454-7331|8627.24|FURNITURE|l requests boost carefully across the furiously regular dependencies 6529|Customer#000006529|z MsqjmOtVW8ynDudk6XiUsAurx57YfL9|6|16-605-167-6202|5032.68|BUILDING| orbits are? unusual, ironic accounts are slyly. unusual requests sleep after the ironic, ironic ex 6530|Customer#000006530|MV,sirXvlYHdNS|17|27-993-759-6289|3190.15|FURNITURE|e requests; quickly regular excuses nag 6531|Customer#000006531|mS8cUs YGyXTFQ6,raiT IVNEprioVqOkf02J|2|12-613-384-6941|889.74|MACHINERY|ng warthogs print furiously i 6532|Customer#000006532|TvhgzSH6z6mBFPH4PVuV8WYBdwaL|1|11-390-945-2159|2857.43|AUTOMOBILE|xes. quickly bold accounts detect accordi 6533|Customer#000006533|nlQQZE8X8Gcs0DTc4FbikdL|0|10-292-966-8076|7733.20|AUTOMOBILE|ions. requests wake slyly alongside of the even, even depths. always regular accounts after the blithely exp 6534|Customer#000006534|VxsPQuMbLvQ|14|24-272-292-9916|4916.15|HOUSEHOLD|ly alongside of the furiously unusual packages. special, ironic orbits 6535|Customer#000006535|x nBq56OShrH,akf9CsjKBDGZXYjPPD0JClsM|22|32-183-174-5882|2786.57|HOUSEHOLD|e blithely express requests. packages breach along the packages 6536|Customer#000006536|LCjRJ4bGqSQQ|7|17-209-711-6334|4800.56|HOUSEHOLD|ate slyly slyly regular accounts. furiously speci 6537|Customer#000006537|crEEScryYpiugrKGzY2vjFvO22Bq,wCnTwIzpYN|10|20-987-261-6336|7057.74|BUILDING|ymptotes. fluffily final packages above the packages cajole carefully blithely regular excuses. packag 6538|Customer#000006538|zKfgcSybmZ8|19|29-312-894-5415|4435.01|FURNITURE|nal, regular packages. furiou 6539|Customer#000006539|7Il6KjzRIE xHg8B2wsoIP5y 5t|19|29-262-259-8994|1175.83|AUTOMOBILE| with the carefully even requests. even, bo 6540|Customer#000006540|I8kQ2XzSVZ3sXkl|16|26-663-746-1303|4042.13|HOUSEHOLD|instructions integrate carefully. final requests sleep. quickly special de 6541|Customer#000006541|3ua8FZTnQ5aC|18|28-614-720-7659|9117.32|HOUSEHOLD|al dependencies. carefully final instructions cajole account 6542|Customer#000006542|0r NUZcHVeN5ZImNTuc2Gjf3st|4|14-681-824-2073|2821.51|FURNITURE|r packages. carefully even accounts cajole carefully along the regular, ironi 6543|Customer#000006543|vv4vWlUAksCLLGzokFhUypa0d67QFAE2pxiH,|18|28-439-480-5859|9050.06|FURNITURE|express requests wake slyly even dependencies. close requests around the car 6544|Customer#000006544|blMRmjCIk9qZDmu|8|18-430-406-1141|5344.32|BUILDING|totes sleep fluffily. ruthless, special accounts alongside of the carefully 6545|Customer#000006545|EVlv0HyoPBatXo6VRaH0uSmnfd0YQZggZBaeHrB0|3|13-174-449-5530|5016.87|BUILDING| thin pinto beans boost furiously ironic instructi 6546|Customer#000006546|zYQtqlAP5jGhQOtMELGmOF QYmKwL,|22|32-365-893-3285|1530.52|AUTOMOBILE|e carefully against the pending requ 6547|Customer#000006547|3zLebAX1KgiP7|7|17-710-138-4406|3321.43|HOUSEHOLD|ide of the slyly unusual requests. blithely ironic packages haggle. quickly silent pinto bea 6548|Customer#000006548|32TjRFavvtwh M|1|11-397-769-2069|7325.51|HOUSEHOLD| have to are always pending deposits. slyly silent requests wake carefully regular accoun 6549|Customer#000006549|qAhHr41OY7bbo|7|17-522-198-9893|5850.07|FURNITURE|encies are blithely. special theodolites above the slyly regular realms cajole fluffily foxes. unusual requests sl 6550|Customer#000006550|3bUibIk2 e0g9upYw7f|21|31-679-244-3796|2107.94|BUILDING|ending, bold dependencies boost. regular, silent foxes haggle. deposits h 6551|Customer#000006551|CFd12CDJGcvLDzKaWj SOIjHwSGuIjnqclcJwqJh|11|21-528-245-9647|6066.94|MACHINERY|ans? furiously ironic foxes hinder blithely pending, pending deposits. blith 6552|Customer#000006552|Nf6QCAunWDenuiZJmxzANjkcjzjl|20|30-889-967-1134|6162.01|AUTOMOBILE|. furiously express foxes according to the slyly idle accounts mold ruthlessly slyly slow packages. special, p 6553|Customer#000006553| ocLpu754,ol|16|26-166-724-4677|8985.90|HOUSEHOLD|refully regular courts. requests sleep: special, final accounts sle 6554|Customer#000006554|A9l1P,V5p9431yso381EbxGEBIrjzbQqe4 Hn|8|18-845-570-6654|5037.71|AUTOMOBILE|e along the warhorses. fluffily fina 6555|Customer#000006555|UMegOBlfpGA0IaM|21|31-923-419-2629|-120.55|AUTOMOBILE|alongside of the furiously special 6556|Customer#000006556|p8DBE,GLPulEItM,G,YMkdQ|10|20-224-737-2850|6794.45|AUTOMOBILE|lites. blithely special accounts use furiously deposits. blithely regular accounts use evenly pending accounts. 6557|Customer#000006557|wUknF8m7MjQL,,6nUI1gB LWN|0|10-666-886-5603|1405.40|MACHINERY|even attainments are carefully pending asymptotes. special deposits could have 6558|Customer#000006558|Kzbfegyh P0YRTxW9aCqgoNrx3jYa8j6j|13|23-111-200-2537|1525.10|MACHINERY|ages after the slyly regular r 6559|Customer#000006559|j4vrCbCwWy4ZdMUF, bHX58wklRdI0|0|10-180-831-4219|314.77|FURNITURE|ts. carefully bold packages cajole permanently besides the quickly final foxes. bol 6560|Customer#000006560|nicoCxzD22IH|3|13-393-157-4401|8302.04|AUTOMOBILE|silent deposits sleep quickly. platelets sleep furiously furiously bold ideas. carefully unusua 6561|Customer#000006561|U B1notuUA|24|34-800-483-7728|6486.47|BUILDING|nstructions. regular packages haggle against the bold packages. deposits cajole. blithely unusual pinto beans 6562|Customer#000006562|6XINODN,YblT3W5FrWSo2voo7MeU5kv8hTni|14|24-485-841-2292|6057.40|BUILDING|s accounts. regular, ironic theodolites haggle 6563|Customer#000006563|ckpsWGe2Xt2QnI05rzcbreoFdTEK,OwotWDbccxm|11|21-535-565-6266|-250.80|BUILDING|bold deposits after the ironic, regular asymptotes are regular dolphins. quickly regular requests are bra 6564|Customer#000006564|pqkVXBhs6SV,fGXfelR1l29,yit1uOZ|13|23-518-531-7793|6761.96|MACHINERY|can cajole blithely even foxes. dogged instructions wake after the bold requests. furiously regular pin 6565|Customer#000006565|hqYF09xT hXv3RHhuSgCCfHnZr7Sz, 2Zuy5XfRN|22|32-398-565-3295|5309.19|AUTOMOBILE|furiously express deposits. furiously even asymptotes cajole quickly. final dependencies sleep slyly. 6566|Customer#000006566|i9Aw4WrBV7UKH3JXXjsdz5 W5UKD1t,pSOo0v7r|11|21-404-346-3746|8548.38|FURNITURE|ages haggle furiously regular p 6567|Customer#000006567|R8iEyHwfq7JefvXy7woKlcnqbVN 0TYcPZu|5|15-714-627-6738|-379.87|FURNITURE|l ideas nag furiously bold hockey players. furiously unusual requests brea 6568|Customer#000006568|7KgPG F0nEcyBKpQJqL|22|32-338-335-4860|4514.95|FURNITURE|ly. regular, silent dependencies affix about the idly special excuses. co 6569|Customer#000006569|67iACh32SK|1|11-178-911-3792|2712.91|MACHINERY|al dependencies wake blithely against the ironic pinto beans. final 6570|Customer#000006570|Xvm7kfDpAmTyhISPbrqibCUopcCq1qqNCOE1pzlO|0|10-475-471-1307|9873.25|AUTOMOBILE|wake bravely against the blithely express theodolites. furiously silent deposits a 6571|Customer#000006571|fj4IX5Zk4vvfUuEIC|14|24-892-309-1142|2767.45|FURNITURE| pending accounts sleep above the blithely regul 6572|Customer#000006572|ar ADMGk0y2|19|29-771-561-9164|2540.69|MACHINERY|telets use blithely accounts. pending platelets integrate fur 6573|Customer#000006573|N58H8Hoy0XD216MOSnWysRXUlIsVqAUR6GZ1LTlF|18|28-312-121-4734|8367.22|FURNITURE|lyly slyly pending foxes. blithely final requests cajole slyly after the even, pending deposits. slyly i 6574|Customer#000006574|nzW785SZCqoQLHUqxecq,xzU0EyIxa,bwZ|17|27-392-453-6805|1972.26|AUTOMOBILE|ts above the carefully ironic packages cajole alongside of the sile 6575|Customer#000006575|GARTfwst7rRbB5|22|32-486-660-6159|2769.78|MACHINERY|uriously ironic accounts haggle blithely. ironic pinto beans use carefully furiously final multipliers. fl 6576|Customer#000006576|cWpLaNr2DIuZanI3i|12|22-167-100-5796|8636.08|HOUSEHOLD|ual packages. ironic, special deposits thrash slyl 6577|Customer#000006577|uEQPgw rPzHldPtHfUqU1r4K5|20|30-983-783-3040|-395.07|HOUSEHOLD|egular packages. furiously ironic fo 6578|Customer#000006578| 5L06W67,Mw8G|2|12-946-562-5905|1973.65|AUTOMOBILE| silent accounts haggle blithely blithe ideas. carefully special request 6579|Customer#000006579|tpu9XN6JdsLCO6nnqauXv3|21|31-593-816-5830|4037.08|FURNITURE|about the blithely silent foxes need to sleep ironic, fin 6580|Customer#000006580|AaTSoiFwZUkdYNegqMCCs|6|16-285-854-7551|1193.35|HOUSEHOLD| sleep requests. slyly even asymptotes sleep; sentiments affix according to the waters! final, final fra 6581|Customer#000006581|m7AxxAwDpU173tVX8AryB4bRTv|0|10-473-250-2099|1624.68|FURNITURE|l, special requests after the quickly regular requests are fluffily pending id 6582|Customer#000006582|roC81vpXtYqj6w2ofenW|3|13-543-506-5912|1920.90|BUILDING|st the slyly regular foxes. even 6583|Customer#000006583|xm9DySSRKsU04Oru|15|25-719-747-8483|8469.57|HOUSEHOLD|thely dependencies: quickly ironic tithes wake fluffily special, special requests. regular, spec 6584|Customer#000006584|KVHnlcQNk3RAGL9llr|7|17-892-368-3465|6059.87|MACHINERY|lyly special theodolites cajole fluffily even requests. even pinto beans among the iro 6585|Customer#000006585|FSuVskGB021iRHHJpNyQiBYx2S2eS R1g9|24|34-421-647-5744|8304.87|HOUSEHOLD|ronic packages haggle blithely packages. unusual excuses sleep busily. slyly careful dep 6586|Customer#000006586|rmyIMRGGnIp84hB9APjbpN3l2J4 lDcogPRb|13|23-442-685-1204|9955.66|HOUSEHOLD|oss the slyly regular requests. furiously even requests believe. quickly ex 6587|Customer#000006587|U2vx2k5HvCCh MNEYbIRD3BwkR|17|27-495-708-8832|1888.45|MACHINERY|the ironic, unusual accounts 6588|Customer#000006588|q4ECgmz0iqJlZeKE0U|1|11-899-895-2340|2153.86|FURNITURE|ans. even instructions wake at the slyly even packag 6589|Customer#000006589|OPaay XGwsQ5FPbgRupMO5|2|12-462-422-9223|7820.61|HOUSEHOLD|e ironic pinto beans should boost ironically furiously ironic asymptotes. blithely regular accounts despite the furi 6590|Customer#000006590|aMW6NjpCUVPZxoLFEQ3V75cZ5eVfmeGc|14|24-797-771-6036|106.60|BUILDING|y final requests are. foxes mold carefully regular depos 6591|Customer#000006591|5f8amVgHTYIC9LNg,oJ2358|10|20-937-427-5966|7511.62|FURNITURE|lly ironic deposits integrate regular ideas. packages are quickly. fu 6592|Customer#000006592|ICcURs4cxBOwwDlSm1N0Q3o2gBIhX|21|31-112-360-4445|9860.40|FURNITURE|cuses. furiously bold requests wake above the furiously unusual dinos. ironi 6593|Customer#000006593|bXigSMqStoMDk4bZGp7sSpBFr,KfbgZzZkU6x|19|29-550-277-9067|6893.52|MACHINERY|ake among the fluffily ironic foxes. dependencies sleep orbits 6594|Customer#000006594|2cDc15tGdriYteAK75|18|28-438-658-3673|5410.27|HOUSEHOLD|fully ruthless accounts sleep across the slow instructio 6595|Customer#000006595|uVa27rCZ,a|19|29-292-466-2278|2348.08|FURNITURE|s among the unusual forges integr 6596|Customer#000006596|v05Csj41kqY8c Z|3|13-153-530-7399|5083.03|AUTOMOBILE|ronic decoys. slyly special packages above the slyly regular deposits cajole past the regular, final platelets. 6597|Customer#000006597|xan0fBW83D27pugrU|2|12-820-261-1596|2468.03|BUILDING|y unusual packages. blithely express deposits run slyly. si 6598|Customer#000006598|gbSYPPXD xhYTY|13|23-340-258-3248|8171.34|HOUSEHOLD| blithely final braids; furiously express ideas cajole c 6599|Customer#000006599|zZJjOj,Fl38qicLtaaRFZmXBPrsOPu6K|0|10-297-776-2902|9848.52|HOUSEHOLD|sly ironic ideas sleep fluffily regular instructions. special, pending pinto b 6600|Customer#000006600|m3pLs7ZW2DQGLirHs2KrrsVG|3|13-922-914-9708|5090.20|HOUSEHOLD|ut the furiously ironic realms hang furiously final packages. slyly unusual instructions nod furiously furiously i 6601|Customer#000006601|8CksnofGDe3,eOr yd G1NJvNV8g|3|13-158-904-5841|1921.72|HOUSEHOLD|y against the slyly bold deposits. carefully regular sauternes above the furiously silent deposits 6602|Customer#000006602|xYS0xzAVCHivnaBFSkuHzezVfozYTop|3|13-468-842-3174|252.68|BUILDING|structions. instructions sleep blith 6603|Customer#000006603|G0VkNa06eg5whaAH6XUH|17|27-100-185-6322|9705.48|MACHINERY|ronic, idle foxes. bold, pending warhorses are: dependencies use boldly ironic requests. quickl 6604|Customer#000006604|kmwPS7a1rYHG3d2KI12OKOegpaHNwQitCvRbb|5|15-116-310-7342|-850.55|HOUSEHOLD| use above the final asymptotes. ironic, ironic instructio 6605|Customer#000006605|bJpsYu5HBIgwd3bPpcfTMme|5|15-797-674-4556|2748.15|BUILDING|o beans are after the packages. even pinto beans boost according to the final excuses. fluffil 6606|Customer#000006606|veGv5 O g1eNuMCQ8lbO0X0|18|28-322-896-9125|8130.69|FURNITURE|y special asymptotes. even, even requests above the enticing 6607|Customer#000006607|pm9R99glhFCWZzIEllz428 TqMQQmX|17|27-924-798-8911|8585.44|BUILDING|regular deposits. foxes are blithely carefull 6608|Customer#000006608|WZYAo7ClTq8j|1|11-275-467-1847|6611.72|FURNITURE|s pinto beans. furiously regular platelets according to the furiously regular hockey players slee 6609|Customer#000006609|9XLzC3FZ8xeSgZX2PiZ2JTpZP7uW,KsvvqY|15|25-264-475-4079|-773.01|HOUSEHOLD|theodolites nag quickly idly ironic accounts. ironically even foxes are. blithely ironic platelets c 6610|Customer#000006610| QFFI2olnw1MmJfAqwsm0iw4oNDjU6kITjOa53h|8|18-120-297-4174|7884.20|HOUSEHOLD|ently even packages sleep furiously. carefully even theodolites haggle quickly exc 6611|Customer#000006611|aQKv,MgJ9FK3NjvADMZ3rhl|3|13-452-843-5081|1801.07|HOUSEHOLD|carefully thin, express requests. carefully re 6612|Customer#000006612|gnqac7Ybh2kSoqKa3ASVAHvypm|17|27-106-926-4405|7620.71|AUTOMOBILE|ld asymptotes. carefully final requests haggle fluffily regular, regular platelets 6613|Customer#000006613|zPS5aK 66ca8nD|4|14-453-488-2934|6296.69|AUTOMOBILE|unts. ironic, even instructions cajole blithely a 6614|Customer#000006614|ICoJpB8v3 QU8AfZ|11|21-180-351-9946|6559.73|MACHINERY|eep blithely slyly pending platelets. ironic, express ideas cajole according to the regular ideas. slyly r 6615|Customer#000006615|GkKngc3qcaXlUc0oUGBVh,Ah|22|32-821-576-6664|5407.17|HOUSEHOLD|kindle. always regular accounts thrash furiously ab 6616|Customer#000006616|8ssRl7vqBcxVrQWh xQdt5U1zX34R5ga Txw,|11|21-804-310-8614|5341.58|HOUSEHOLD| silent packages ought to use 6617|Customer#000006617|gkz18C,mxfkSot1U zHcc8E|18|28-955-200-3871|3008.53|MACHINERY|ainst the blithely regular deposits boost carefully carefully unusual ideas. quickly u 6618|Customer#000006618|jUw3FhXzO0qfzz zXxWOhKwfmI3r|11|21-610-354-6624|6411.46|HOUSEHOLD|nal requests affix quickly pending packages. close pint 6619|Customer#000006619|clyImCIDigUVv1edDwAG34tr7MWfI|20|30-409-954-6902|-549.38|AUTOMOBILE|hely final instructions nag; carefully bold accounts maintain after the even requests. carefully unusual saut 6620|Customer#000006620|,paCQ5qIt,Ylr,iREwAefW8ys5k|13|23-766-421-3496|4441.83|FURNITURE| ironic accounts doze carefully according to the blithely regular deposits. slyly silent reques 6621|Customer#000006621|EuR3TpdBnWKm6OMxOW9yAuiT|8|18-801-593-9685|5852.43|MACHINERY|ymptotes sleep quickly regular platelets. furiously ironic fox 6622|Customer#000006622|fidCV6mKheF|16|26-334-959-1721|8847.45|MACHINERY|wake regularly final instructions. deposits are slyly. furiously pending pinto beans boost. care 6623|Customer#000006623|Xjw9Hy5h4Pgb3zcUEWm8|24|34-304-460-3284|-872.31|BUILDING|g the slyly final foxes. even, pending requests boost about the requests. r 6624|Customer#000006624|5VX6OjpQ Bro52jdZej8FlK0OFFtf7R3ESIJPK|1|11-660-742-2374|-7.37|BUILDING|inder blithely? regular platelets about the packages believe instructions. regular fox 6625|Customer#000006625|R3hdYck8WB F4No|0|10-367-922-8280|6076.36|HOUSEHOLD|ly regular requests cajole; blithely pending instructions haggle al 6626|Customer#000006626|ZdfCZEerdNphvEz5|8|18-208-761-1975|-595.84|HOUSEHOLD|have to doze permanent pinto beans. special requests sleep across the furio 6627|Customer#000006627| lMYWGolCqdPZwftuYfusc7pMOGgLUZdrI|6|16-598-408-8591|4678.15|FURNITURE|ckages. ironic pinto beans use carefully-- carefully eve 6628|Customer#000006628|xP MMDAoXhUc,s8N|9|19-364-386-8197|6607.71|HOUSEHOLD|s are. furiously express platelets boost. regular, regular accounts cajole. depo 6629|Customer#000006629|aLIf6koYS8OSSnvRGqhRFqIRnX|6|16-310-265-2013|8761.81|AUTOMOBILE|onic deposits hang slyly enticing packages: slyly regular courts was blithel 6630|Customer#000006630|mrADj6gHX6kvN8H9hV2TTZD|10|20-993-605-4157|844.42|FURNITURE|across the blithely special excuses. express accounts are about the packages. ironic re 6631|Customer#000006631|7iOXTPto,B5|12|22-688-971-9796|89.69|BUILDING|the ironic deposits serve slyly slow, regu 6632|Customer#000006632|WFDE,gGvSeFejvfMg|2|12-979-552-7277|9324.80|MACHINERY|onic theodolites haggle carefully carefully pending excuses! packages boost against the furiously ironic p 6633|Customer#000006633|MXIkYoyLJbpxhc|4|14-157-941-2531|2911.60|FURNITURE|deposits cajole even instructions! slyly final requests integrate enticingly furiously special 6634|Customer#000006634|v40njkdojYnRdVGwAW|5|15-916-327-8851|5261.30|BUILDING|lar instructions. accounts haggle permanently according to th 6635|Customer#000006635|dPKoh,uNPUKABQlR,WDK3pTzCp|24|34-785-680-5961|6618.78|AUTOMOBILE|packages sleep. slyly silent deposits x-ray furiously deposits: bold, e 6636|Customer#000006636|0yf ,8IEF6Ym5JFeZb1HfyYJG CDxU|24|34-625-369-8340|7912.37|BUILDING|ously ironic ideas integrate furiously ironic instructions. careful 6637|Customer#000006637|5RhMx2tf5k8u|13|23-455-236-7135|2317.68|BUILDING|e blithely. furiously even dependencies integrate slyly. car 6638|Customer#000006638|Mm2JAYCCHstTE|16|26-315-337-6748|191.55|FURNITURE|s packages sleep quickly. final ideas boost carefully 6639|Customer#000006639|auyhpm1qOcoflRfR4S35,7nPTFyM ZG04eGAMb,U|13|23-364-243-5030|5269.15|FURNITURE|urts cajole bold, even requests. furiously ironic exc 6640|Customer#000006640|LraQsOeV6d|7|17-233-212-8020|6693.51|AUTOMOBILE|ate after the special packages. carefully regular excuses are quickly quickly even decoys. ironic foxes detect bo 6641|Customer#000006641|gw3LD4q9DzXioZ37chuNxB2|8|18-212-860-9043|9799.19|BUILDING|e furiously according to the theodolites. carefully bold ideas wake fina 6642|Customer#000006642|LoE,WKpwd79h4TGVAJDgTYwzSwDmBfd|2|12-278-337-5916|1026.55|HOUSEHOLD|s. slyly bold excuses play above the furiously ironic requests. furiousl 6643|Customer#000006643|Iuv NTvS6dJ2io6Q76FiZeJI|23|33-173-903-7175|3109.73|HOUSEHOLD|are carefully blithe accounts. carefully unusual pi 6644|Customer#000006644|mf2MRu7b37tyk|21|31-369-661-7739|2257.43|AUTOMOBILE|bout the unusual requests. special, unusual epitaphs haggle furiously final deposits: even requests agains 6645|Customer#000006645|9wuMUJea1LYIsA0MDzoFxKARmIJvMpI|22|32-339-429-1573|-347.93|BUILDING|according to the permanently special packages. pen 6646|Customer#000006646|GioIRa7f673rsFRHt2e|9|19-556-383-9646|2206.85|MACHINERY| furiously special packages haggle busily. s 6647|Customer#000006647|gLBSZ8P6yXMpyGwCirn9HfN,|6|16-349-915-4350|7515.71|AUTOMOBILE|ncies are against the furiously express instructions. deposits above the bold theodolit 6648|Customer#000006648|0HJ,ghgxYfPYPC eCh6nDUgWLalajrwBRY4p|0|10-142-681-9531|1611.85|AUTOMOBILE|y unusual packages. express ideas sleep furiously along the final frets. furiousl 6649|Customer#000006649|q7et5s1SAXMlokVdGW3ZYueHV|20|30-484-404-2575|6300.33|BUILDING|nts. packages integrate blithely around t 6650|Customer#000006650|NxtKIJDfOaBn|16|26-568-235-1339|1611.97|FURNITURE|pite the ironic packages. silent requests across the regular theodolites solve care 6651|Customer#000006651|cAtLrZCfDjtH5vIpaqg,qBs6J|9|19-396-649-7221|8251.81|AUTOMOBILE|al, final pearls wake furiously regular grouches. packages wake bl 6652|Customer#000006652|4mz9d8cQw7Hh,KZYOWhHrH5NpLkhQL|14|24-741-346-4893|9296.36|FURNITURE| above the regular, unusual dependencies. final pinto beans dazzle fluffily. pending 6653|Customer#000006653|4Q4ARPm8n2f|10|20-532-650-2293|8927.69|MACHINERY|riously final packages integrate. ironic, thin accounts cajole among the carefully pending 6654|Customer#000006654|AHYF0lz1LiG,wC1WMH9L9pCe3PdUaO4Q|7|17-933-462-3572|5789.45|MACHINERY|s. blithely regular accounts engage slowly carefully final se 6655|Customer#000006655|FgGQl7KxO2rmmwE0rndJ|24|34-277-845-9539|6372.81|AUTOMOBILE|ermanently regular packages do are carefully express requests. pending accounts sleep. pac 6656|Customer#000006656|jgLvAdS6UQcyaUCSb|17|27-416-436-5518|9824.64|FURNITURE|eas serve furiously pending theodolites. f 6657|Customer#000006657|fDo1gqlFFrkkqjwSb9 9RD7DNbuPI59zt|3|13-538-227-3972|-146.22|BUILDING|ly. blithely stealthy asymptotes alongside of the fu 6658|Customer#000006658|qOYY,NL6MESHOBu1r8jx|21|31-302-918-2979|4384.07|MACHINERY|riously blithe accounts cajole slyly ideas 6659|Customer#000006659|WKPkuTEjDJc|10|20-393-281-4388|3053.98|BUILDING|around the blithely express dependencies. careful 6660|Customer#000006660|Mhf8lV21 BapxeXn9lz9k7b6tBd|23|33-164-539-3410|188.29|BUILDING|s. quickly close requests are quickly. asymptotes are against the carefully slow requests. pending foxes caj 6661|Customer#000006661|bB6EP7Tf3mVskZJ7tLHKQSGDifygpVm2|16|26-168-633-7265|5529.99|AUTOMOBILE|t requests nag. blithely regular courts wake quickly. thinly regular deposits according 6662|Customer#000006662|oZ6a dkzIqrldlL|10|20-378-737-1289|1621.83|AUTOMOBILE|tes use pending, ironic dependencies. even, ironic dependencies boost according to the deposits. regular 6663|Customer#000006663|tFHYxL YrhKpX|13|23-250-528-2581|9077.11|AUTOMOBILE|ckages after the pending, regular dolphins use quickly 6664|Customer#000006664|580X3yq552pkRg5sQwDbltI6XxFJb6c6rb|24|34-426-291-1081|9108.44|BUILDING|unusual dependencies sleep quickly accounts. ironic ideas sleep along the carefully bold accounts: furiousl 6665|Customer#000006665|,Y,EsUyItpokyz9XVi9jf3L7JuOwrQjNE3c1Mlij|0|10-163-117-8909|7628.89|FURNITURE|ickly regular theodolites use blithely among t 6666|Customer#000006666|9zJqKbGbRjPS|18|28-557-833-2670|792.26|AUTOMOBILE|lithely regular requests after t 6667|Customer#000006667|SXoMfAHkfO4b44Yr1Qz|19|29-738-728-6617|3385.51|MACHINERY|ual accounts. final excuses cajole car 6668|Customer#000006668|t,qsLMTcPSMSc I3,7LYW W0EwfqtOu,pmLc,|5|15-364-427-1235|6319.59|FURNITURE|unts boost. final deposits serve furiously according to the carefully regular accounts. b 6669|Customer#000006669|lT6gKsLMZJaBBXCtnGWroOoUkdbqwFyjMU Q|9|19-723-887-5480|6311.20|AUTOMOBILE|carefully even instructions haggle furiously. c 6670|Customer#000006670|naiDNB2uULTBT,321CFY9HYG0jFelpYg|16|26-694-136-5425|8297.83|HOUSEHOLD|gular pinto beans solve quickly carefully bold accounts. express pinto beans promise slyly acr 6671|Customer#000006671|WaqMrlZBfcDiT3n5KvOWt14jgw1m5ZARzqI85fY|18|28-145-184-9679|1603.96|MACHINERY|ans run fluffily blithely unusual dependencies. bold, ironic packages above the quickly pending foxes believe caref 6672|Customer#000006672|TAk1jzQy60fSpBRSLShvpTZae1797Bdve|10|20-897-841-4188|5700.11|AUTOMOBILE|unusual sheaves haggle quickly according to the carefully final som 6673|Customer#000006673|heXKQ2V3L0uxVhdWxCvr42|23|33-166-738-4873|751.90|MACHINERY|packages sleep fluffily final grouches? fur 6674|Customer#000006674|WfEiHyBjekCdDYT2Gbb0infBTvADc27M9R3 BWZ|9|19-629-336-3947|7179.20|BUILDING|y even deposits. accounts cajole furiously about th 6675|Customer#000006675|HAnz4SbD7 1s,9fqBkN8A4L,m8UZ|5|15-317-619-9609|-226.28|MACHINERY|ess accounts. even packages above the quickly regular requests mold a 6676|Customer#000006676|VtWsbzZD4qe8Z T02uSPbSuKiz|10|20-569-115-2865|816.03|FURNITURE|he ironic, bold courts are ac 6677|Customer#000006677|Wh55,5rAIfHnBZbN|0|10-283-641-1486|7035.29|BUILDING|ns cajole furiously beneath the carefully ironic waters. regular requests are according to the furi 6678|Customer#000006678|QDpY,eRLnl5,HsjFuCEmufwZadvV4|3|13-605-424-6382|1201.45|MACHINERY|warthogs. blithely ironic asymptotes 6679|Customer#000006679|VqLS2XSCc0GFgWN5 Ol|19|29-169-326-9045|5642.93|MACHINERY|, even deposits use fluffily regular pi 6680|Customer#000006680|8TEDCB7fdAUhgYRUU7ZfV1Ld3mB00gDQhtNB,oS|17|27-962-923-6320|7673.78|BUILDING|s cajole. fluffily express ideas wake furiously about the ironic 6681|Customer#000006681|th39LRNpVorKObh7TB|6|16-926-344-3556|7321.37|AUTOMOBILE|egrate furiously. slyly ironi 6682|Customer#000006682|47B7Wsx8E7991Y2|19|29-904-610-3132|2230.59|AUTOMOBILE|bout the boldly final requests. slyly special ideas h 6683|Customer#000006683|snrDJsmCFVQ4O3dveQpw5JIDvmtsZXdRtzNmP4O,|10|20-812-694-7415|-549.56|HOUSEHOLD|gular accounts. fluffily express 6684|Customer#000006684|qdqKwrzOb2|5|15-109-442-3321|1412.15|HOUSEHOLD|ily express excuses wake blithely. carefully bold packages sleep above the even, regular pinto beans. 6685|Customer#000006685|BkWlHQusdN1kzNDHvqnOV1SJ9|17|27-406-584-3296|4996.19|BUILDING|ly regular foxes boost blithely bold foxes. e 6686|Customer#000006686|DvvWufVBbpdB3nSVazM7|4|14-799-782-6111|5959.97|HOUSEHOLD|y unusual deposits play final requests. ironi 6687|Customer#000006687|Q0K48 G0mj|17|27-869-237-1381|1157.82|FURNITURE|ress asymptotes. quickly regular deposits haggle. final, close packages snooze fluffily according 6688|Customer#000006688|h61o1Amg8a2wnZYMAqd5gJM16PPB|15|25-369-725-3205|9401.81|AUTOMOBILE|fully regular platelets. ironic ideas hagg 6689|Customer#000006689|x5j,b7K9Irw9KnJmeCur|23|33-787-189-8082|-652.17|MACHINERY|blithely regular packages across the packages poach across the bold platelets. even accounts nag furiously against t 6690|Customer#000006690|B1nB5mFxDsSJQvm5lxn3nkyMl|16|26-671-890-1848|3868.69|FURNITURE|eas try to are fluffily. pending, t 6691|Customer#000006691|NvkXKfvzF9oErd6MzanMySOlVPNUCQ7tcV|5|15-503-903-6718|8999.87|FURNITURE|luffily. unusual dinos doubt slyly. accounts sleep fluffily after 6692|Customer#000006692|HkPGTwsyUV|9|19-145-218-4626|8817.45|AUTOMOBILE|boost permanently final, regular deposits. blithe, ironic requests sleep furiously express dep 6693|Customer#000006693|Bd7U3RV0lbRer WQY|24|34-882-944-4437|925.49|HOUSEHOLD|silent deposits. special foxes nag sometimes permanent, bold deposits. carefully special pinto beans sleep fluffily 6694|Customer#000006694|XyOYqBJqGspMZSdqQq5DvaX9CD|4|14-959-393-1619|9588.19|HOUSEHOLD| against the slyly ironic deposits cajole blithely even pinto beans. final 6695|Customer#000006695|XGxqrdbm4jLBmrqTRmd5dLwIR|18|28-628-769-5605|7779.06|BUILDING|y even dolphins. requests wake. fluffily express asymptotes sleep. slyly final acc 6696|Customer#000006696|xhoBDOYsLR89|21|31-975-638-1125|5667.73|MACHINERY|es. slyly regular accounts integrate! fluffily ruthless accounts wake furiously! quiet accounts 6697|Customer#000006697|0NTbi10hHKSxo|24|34-307-871-2967|6632.75|HOUSEHOLD|re carefully quickly final asymptotes. blithely pending packages cajole quickly along th 6698|Customer#000006698|0QyCW1acGdoAo59FdWV 3pZ|19|29-862-853-4688|620.84|MACHINERY|ges wake blithely. quickly pending braids sleep furiously: carefully regular pinto beans mold after th 6699|Customer#000006699|jGDulmO0cw9lFBN1jlR64OLiBqCJc|2|12-708-798-7379|4317.54|BUILDING|unts-- carefully regular instructions maintain s 6700|Customer#000006700|KUpYErT7tXcOM04gpSlDW4566SCbvBT dA6l|8|18-304-498-5307|6932.01|FURNITURE|structions. blithely final ideas affix. blithely unusual requests haggle slyly fluffily even instructions 6701|Customer#000006701|CVKLM V7ST0Nx,jr0e0gHcduEj5|9|19-659-528-9243|3257.42|AUTOMOBILE|lly final foxes. ironic requests boost carefully du 6702|Customer#000006702|Wi DrmUzjVKPCSBNG6Wok7io9QxVX7kN7Jd|7|17-216-230-3435|9392.14|AUTOMOBILE|t slyly according to the furiously fluffy theodolites. carefully regular requests are carefully int 6703|Customer#000006703|03PTXkGYGGxCThBmmgGy|21|31-804-154-5053|1836.91|BUILDING|. unusual asymptotes cajole carefully against the ironic packages. 6704|Customer#000006704|B1B1Ms3HDpnvU8cDBoMP3T4PkW,mHSWd|19|29-206-376-9599|4862.96|MACHINERY|rmanently even ideas haggle carefully. carefully unusual courts according to the c 6705|Customer#000006705|aDX0WTflMQRyU2JMFFlW|3|13-434-897-7312|-159.03|BUILDING| foxes. furiously silent pinto beans integrate slyly across the grouches. slyly special pinto beans in 6706|Customer#000006706|TBcc48ZL2vZv7afiGoSEhjbM|3|13-134-428-9567|9100.37|AUTOMOBILE| special, even requests are fluffily-- regular accounts a 6707|Customer#000006707|o6Nm3V8 Zj2tt4j3sMiTJ4|6|16-466-364-3185|3977.38|FURNITURE|uriously against the regular, ironic requests. regular packages wake never. quietly pending instructions believe. 6708|Customer#000006708|mUhDV6NCOCJghkAjEX0Jo|12|22-281-780-5651|9900.11|BUILDING|ly. ironic accounts wake carefully at the final pinto beans. slyly regular requests slee 6709|Customer#000006709|BXNfIHT5cfG6DtLlBKePFXUQbBeB4|8|18-207-447-8319|3199.11|BUILDING|ronic, even requests. slyly unusual realms sleep carefully. ironic asymptote 6710|Customer#000006710|CeyxEnNCzAuXmAId1vHI4kN5YuwNvV0rH8NhYz99|12|22-412-672-8270|8494.05|HOUSEHOLD|usly even requests. accounts boost after the bold deposits. slyl 6711|Customer#000006711|z9trQmIO3Y5z60O5ozSv877GQILvo|13|23-886-250-5460|-361.61|AUTOMOBILE|ounts! thinly even theodolites cajole furiousl 6712|Customer#000006712|T QGDjZA2gcd92poV ,1rQrRu9ZB7EzZ|17|27-884-283-7355|3268.82|FURNITURE|ke blithely bold deposits. final requests sleep furiously? blithely specia 6713|Customer#000006713|ynNA7HFYdk0KjYuVxo9R9NK8qw8lXJxDFhA|9|19-945-198-9475|3006.93|FURNITURE|gular asymptotes nag. sometimes final accounts integrate blithely blithely ironic ideas. caref 6714|Customer#000006714|m8zfcfvyi7w PulZPp,g|21|31-407-424-6606|1494.63|MACHINERY| blithely over the furiously regular 6715|Customer#000006715|wEEcGy ed7V nIyY|8|18-303-744-7177|5244.28|BUILDING|ep slyly-- fluffily regular pinto beans use silent accounts: sometimes regular foxes are along the regular, regular 6716|Customer#000006716|qBJ6Dx1ASZOZLrQ7FTreiGRWX|4|14-339-999-9145|6593.08|MACHINERY|ons. blithely bold instructions instead of the blithely even dependencies haggle according t 6717|Customer#000006717|A3TpI60MfBQNedRV 3DcYhf3GUWL|23|33-731-150-6538|4844.87|HOUSEHOLD|ependencies wake except the reques 6718|Customer#000006718|8m61NcNrEZzFo,NmKaBzQXjeX Va|12|22-820-610-1027|3658.95|BUILDING|ely. notornis try to nag. blithely pending package 6719|Customer#000006719|,SJoHSlIYiQZ3ebJw SpwZ8lg|7|17-221-621-5826|3965.07|BUILDING|ronic warthogs after the fluff 6720|Customer#000006720|luGQxDrBGnnftUVgjF|6|16-457-628-5807|843.36|FURNITURE|odolites cajole. ironic packages haggle regular instructions. slyly special 6721|Customer#000006721|OkHnnK98UeBILbX2bjAPwvOdKMBidiW|4|14-701-354-8135|4277.74|AUTOMOBILE|l requests are. furiously quick dependencies cajole. deposits cajole blithely after th 6722|Customer#000006722|1jnPUXu2iBwB7|1|11-451-400-5785|8919.68|MACHINERY|ronic asymptotes use. furiously pending packages boost furiously-- slyly regular deposits wake 6723|Customer#000006723|gAfgW13,GB|20|30-582-567-3761|3070.48|FURNITURE|sts. tithes wake quickly. carefully regular theodolites integrate. carefully final asymptotes 6724|Customer#000006724|bcpvOK,8gUO1|12|22-598-908-4189|2471.21|AUTOMOBILE|s. slyly final pains wake carefully. express, ironic deposits among the furiously final dinos ar 6725|Customer#000006725|if8lplIxBWNs6u2fgJKJLKDrRlfhEjwQcJVooOH|7|17-685-688-6979|6523.60|FURNITURE|n accounts poach silently after the regular 6726|Customer#000006726|hssVbd6,2x8YN7nJDem092|6|16-514-909-5495|3108.77|MACHINERY|es against the carefully regular courts are quickly furiously even th 6727|Customer#000006727|APG0BfGWOYnDkXgbcKdhJXzzJgLtQXYKCoXMPxJA|23|33-373-151-6436|7993.12|BUILDING|riously close foxes. fluffily final accounts haggle quickly across the accounts. bold theodolites engage furiously a 6728|Customer#000006728|v9cVptqbc0tKb9ZT q|8|18-216-645-1385|6449.93|MACHINERY|about the bravely even ideas are carefully ironic accounts. thinly even dependencies haggle 6729|Customer#000006729| NrrDioTuXtkJCh|13|23-575-217-4725|8051.37|AUTOMOBILE| furiously. quickly regular packages above the reg 6730|Customer#000006730|EmtyBtMxazeTZFx1zqBuNju5VmzsqOi930BE|11|21-223-976-8367|6413.26|MACHINERY| carefully final accounts. closely final deposits integrate carefully according to the dogged, regular ideas; platel 6731|Customer#000006731|HpEXXgDmsqIPyVb FXeWbadti|2|12-283-284-3677|7639.55|MACHINERY| foxes. carefully final asymptotes print excuses. furiously ironic waters engage. c 6732|Customer#000006732|sHH4w G8QMTtCVTjbkiEHTvL8bRr5Tu|16|26-779-843-5056|4018.93|FURNITURE|ions haggle quickly against the slyly regular accounts-- bli 6733|Customer#000006733|Kik6R2t1WP|11|21-569-230-8251|4219.13|AUTOMOBILE| boldly across the carefully ironic dugouts; slyly bold pinto beans haggle. 6734|Customer#000006734|yRF8my1jIZM1QuEGXxSuXT0P83w ljVko |14|24-157-448-3086|2971.38|FURNITURE|ly final deposits. excuses along the carefully special packages promise regular theodolites. ca 6735|Customer#000006735|eG5Pt6wlZpFUt0140cHxYox3roKuQU,V|9|19-117-148-6129|8904.94|HOUSEHOLD|onic, express packages wake carefully final asy 6736|Customer#000006736|TwL2M9L6iblvaG,93Efw2HP Glnm8jsH4EQP|17|27-421-117-6858|8363.54|AUTOMOBILE|inal dependencies nag. regular theodolites sleep. blithely unusual foxes boost slyly. final packages nag slyl 6737|Customer#000006737|TnIV3SSmxJFDBYUexZHw2w2m3vIAFgv38lwDY7Rb|4|14-372-941-6618|6015.13|FURNITURE|iously ironic theodolites nag slyly according to the even requests! slow, regular platelets doubt. 6738|Customer#000006738|eAylAy5pHWBnUK6q3v2cXNODweF4|3|13-699-399-9154|4394.53|BUILDING|ending accounts. slyly special pin 6739|Customer#000006739|3lsA2Q8pNR|5|15-493-613-7300|2838.35|BUILDING|uests nag furiously according to the quickly even dinos. ironic, bold requests 6740|Customer#000006740|VZIi3wMuEDjnmgAq4YwDGnf5BK e7Fs2|11|21-751-155-2705|6098.86|MACHINERY|lphins affix quickly alongside of the deposits. slyly pending pearls wake regular, regu 6741|Customer#000006741|ogMwwJa0CY9MZsdv7nCQi5HCOAwuE|7|17-567-242-2914|3514.90|BUILDING| are over the foxes. furiously even requests after the closely r 6742|Customer#000006742|u5lVPzNeS1z2TcfehzgZFHXtHyxNJHU|1|11-643-477-4053|2834.80|BUILDING|bold deposits wake. special deposits about the regular excuses boost furiously furiously final packages. slyly r 6743|Customer#000006743|uXzHSYlqjnoKSFFTrUjwPa4g9oIMRp4D,S,cNM|20|30-195-288-9509|1464.96|FURNITURE| unusual accounts wake slyly about the carefully regular packages: carefully pending dol 6744|Customer#000006744|PGmBpCi8fCF9caRqw|9|19-551-917-8841|1605.91|MACHINERY|ly. quickly special accounts dazzle around the ideas. blithely regular requests sleep quickly carefully regular 6745|Customer#000006745|F4AQlOnBqEtwLi85Fm0RzN3ZmaMdl3|18|28-602-954-3388|5454.54|FURNITURE| final pinto beans sleep pending, ironic deposits. busi 6746|Customer#000006746|M0 KJuw4qc9UBliw9 oD42VXakTKG0,6|18|28-204-291-9356|9415.95|MACHINERY|ges. slyly ironic patterns haggle slowly carefully regular requests. quickly regular t 6747|Customer#000006747|3ntn2pmdvucSKFlb|3|13-243-582-8819|5970.51|AUTOMOBILE|round the slyly bold deposits. slyly unusual dolphins boost bli 6748|Customer#000006748|a8b2paF9T94UrLTiSzHSE38ZmxHL|17|27-580-967-4556|2410.88|AUTOMOBILE| furiously ironic deposits haggle across the slyly silent instructions. quietly ironic 6749|Customer#000006749|sU3,BsMtoVz xNJJX1emkZJxZbOIRc|0|10-774-652-4046|4693.71|AUTOMOBILE|ts. unusual accounts are slyly furiously final accounts! ironic, even pains cajole furiously. 6750|Customer#000006750|V7UkwofaAeXZ7 0rXuHekMbN3mXZLK7koxDO66Tl|4|14-457-948-4761|4227.49|HOUSEHOLD|ding dependencies. carefully regular accounts cajole carefully about the ironic deposits. package 6751|Customer#000006751|Tdxt4GKZYz|7|17-895-936-3672|365.42|FURNITURE|ly for the fluffily silent foxes? slow deposits are. packages doze carefully. quickl 6752|Customer#000006752|80HtfjhjeYU|17|27-487-194-4578|1487.49|AUTOMOBILE|regular accounts. pinto beans sleep express foxes. special instructions wake. quickl 6753|Customer#000006753|yNkYKPtxuhgzqBBm7s6|8|18-860-208-5739|6195.82|HOUSEHOLD|s. bold, ironic packages wake c 6754|Customer#000006754|B6i9QUe1MogaHVLxdQy4QnH3nkb UaY0|7|17-897-648-1827|5143.81|BUILDING|ar ideas doze furiously final 6755|Customer#000006755|aJU5CHd5UZzrd80j1|6|16-237-199-3241|964.67|AUTOMOBILE|nic asymptotes. blithely ironic foxes are. blithely ev 6756|Customer#000006756|sRflkvpvQnc6q8wXS|7|17-105-972-5722|8127.58|HOUSEHOLD|ing accounts. unusual theodolites hag 6757|Customer#000006757|Ot2eNxssE43x0Ivhlzy 2|13|23-725-154-2356|227.89|BUILDING|breach blithely; furiously regular deposits according to the quickly regular packages nag slyly fluffily regular p 6758|Customer#000006758|YhRgjpezuIfxnPM3docKnH9Dr6Zs9dlMPmI r5X|18|28-730-164-7975|15.13|HOUSEHOLD|dolites hang carefully. slyly special accounts nod slyly express theodolites. pending depths according to the regula 6759|Customer#000006759|CyomHt7D6FLzXgN5Rff0fo0a |19|29-276-479-4436|-773.12|FURNITURE|ut the slowly final accounts sleep blithely slyly special foxes. quickly final ac 6760|Customer#000006760|ZXjHellHGmKAh|18|28-609-664-1095|6035.85|FURNITURE|. furiously unusual packages sleep quickly. accounts nod sly 6761|Customer#000006761|v1sTMXm97sEpzFo|1|11-894-413-9156|521.65|BUILDING|tructions cajole carefully against the ironic deposits. blithely final asymptotes wake furiously ironic ac 6762|Customer#000006762|Vha,VM8y w5TD0q7|18|28-388-163-7393|9311.76|BUILDING|against the final requests. p 6763|Customer#000006763|laxf,Ybtd4d2FGE9RCG|18|28-654-592-3290|3895.04|AUTOMOBILE|ts. busy pinto beans cajole carefully since the slow accounts. final pinto beans cajo 6764|Customer#000006764|Ehu6TNlFTGdkY|16|26-720-544-9325|921.28|AUTOMOBILE| forges snooze. carefully pending warhorses cajole carefully upon the carefully regular packages. thinly unusu 6765|Customer#000006765|K8S00oGkfyhG7,A08NTyZTfOItXThIZ|14|24-562-459-8122|4816.14|AUTOMOBILE| regular instructions haggle slyly accor 6766|Customer#000006766|KbP0 fFP7iJCWl4E|4|14-369-503-4420|4814.57|FURNITURE|express, final accounts. furiously u 6767|Customer#000006767|yEB,5rAwZ1Vi,u|21|31-217-839-6340|528.35|BUILDING| ironic requests cajole quickly slyly even foxes. fluffily pending accounts are according to the regul 6768|Customer#000006768|h Kgr0xrW9MkORlqUgwFsGgQZG3,Jks|9|19-647-976-4923|979.30|AUTOMOBILE|furiously idle packages are slyly fluffily quiet dinos. quickly even instructions about the regular reque 6769|Customer#000006769|v5TrRZTAAiD4i1eyVTLNA|20|30-186-784-8328|1029.38|AUTOMOBILE| slyly regular packages nag carefully. furiously bold theodolites along the 6770|Customer#000006770|fhS8YZLcFyBIeZSp2|12|22-300-803-6439|6756.19|BUILDING| express ideas sleep slyly ironic dolphins. u 6771|Customer#000006771|MMWc6i48BJhKrgu9ko Io|24|34-589-527-3110|7287.84|BUILDING|iously alongside of the theodolites! even realms above the furiously special orbits are slyly final ideas. car 6772|Customer#000006772|ggL8d6JSSQUkE9zg6F3|10|20-387-573-8593|-238.33|MACHINERY|eas: carefully regular packages about the furiously silent pains 6773|Customer#000006773|WO 4bouYuu |7|17-627-272-8854|7232.34|BUILDING|tes. furiously bold pains wake care 6774|Customer#000006774|tufe2Xi42Fdq6R4hf7drAFRCg3Rz4i|2|12-800-430-6064|1958.28|HOUSEHOLD|ounts. slyly even theodolites sleep slyly accounts. ironic foxes sleep carefully. blithely pend 6775|Customer#000006775|bzDrDnIGQLCgiExwO6VwqlW|15|25-386-188-6886|9781.70|BUILDING|old, ironic pinto beans wake fluffily. even, regu 6776|Customer#000006776|JXoJqh4JpcdvugREShRwNHCee4ltrxrb3OM1,E7l|18|28-994-492-7101|4054.12|BUILDING|hely among the ideas. slyly unu 6777|Customer#000006777|uT2JCbpTRH9Inj0wgbRuv,|16|26-447-185-5798|1597.51|AUTOMOBILE|cajole. slyly express asymptotes sleep above the final, bold realms. slyly special realms boost on the quickly re 6778|Customer#000006778|ACpbkEbCPYy|4|14-172-893-4202|7333.47|MACHINERY|r platelets affix. carefully ironic accounts arou 6779|Customer#000006779|UeGNbDKhSDW1MkcE,GnxRAiyjHHe0itTSj|18|28-974-892-3856|7630.58|MACHINERY|xcuses might sleep slyly silent, final deposits. regular pinto beans integra 6780|Customer#000006780|R20cDLZCvC,XeXxr3JVgS63kH9IYW41ql9|4|14-601-747-4629|7181.83|MACHINERY|blithely even escapades. even deposits are carefully pending accou 6781|Customer#000006781|hjgOEQCNYiCKQTwECBEJ6pMsKqhNJH1oRj|21|31-556-362-4055|-21.79|AUTOMOBILE|packages nag along the quickly express packages. ironic deposits are along the 6782|Customer#000006782|QfjRd7YtTJRio2K70XUp7w,WY3a9xlo,|18|28-152-315-2630|5711.06|MACHINERY|ss realms. blithely even ideas haggle furiously afte 6783|Customer#000006783|VcFpiOQxpgr2Q5w0,d|4|14-418-290-9278|8724.64|MACHINERY|. blithely regular ideas haggle fluffily against the ironic requests. final deposits sleep. even, regu 6784|Customer#000006784|t7bVQGTfY,PwEmIshCKmrWiNMj|10|20-926-379-4893|3026.77|FURNITURE|ful pinto beans nag final, even accounts. requests 6785|Customer#000006785|Zeb0QWx561VbdmAnNJwFX|5|15-334-822-1897|4849.38|FURNITURE|equests. blithely ironic dependencies about the slyly ironic accounts wake furiously iron 6786|Customer#000006786|y JAZymUDAm19ImpwXrDBhrJ2tkCXImzEU84NXe|14|24-639-124-4027|2564.13|MACHINERY| forges: fluffily bold packages boost about the sometimes final excuses. ironic deposit 6787|Customer#000006787|x3R8xwciGpAldSQtfrfQjKPVA5MK|4|14-221-614-7132|6276.35|BUILDING|ularly silent ideas along the regular, even asymptotes boost fluffily bold theodolites. blithely express packages 6788|Customer#000006788|tkzbRySfDjHBZuJU8xa9XXx4EeZ6L EmX|12|22-769-485-6232|5623.43|MACHINERY|ackages. carefully express platelets haggle blithely 6789|Customer#000006789|wQUHbVSc8YanGHPCDbK0,njoByEX7ThcX7|13|23-254-104-3764|-913.53|MACHINERY|yly: carefully ironic asymptotes are furiously; pending do 6790|Customer#000006790|DCD1tDMXhoUIaKhQPnCUVUKxiLdcGsNK|18|28-476-389-8594|222.98|AUTOMOBILE|rave deposits. regular, even instructions us 6791|Customer#000006791|w7qvbNTA3AUnviiYrUHQ3rrvxg|24|34-796-717-7454|5488.16|BUILDING|g the blithely express packages. u 6792|Customer#000006792|1eyzmigbKdS,uPvd,gTfV82Q0s|7|17-399-754-8075|2503.03|AUTOMOBILE|thely ironic pinto beans. accoun 6793|Customer#000006793|6iyy1xAWokEzS7vzwKebMn8uc9rnD|14|24-645-815-1178|2415.53|AUTOMOBILE|ies sleep slyly along the blithely bold deposits. sly 6794|Customer#000006794|eW4D8D nAkkGhhx|21|31-846-103-1877|5798.75|MACHINERY| integrate about the quickly dogged requests. furiously ironic pinto beans haggle above the never final 6795|Customer#000006795|uHQSwrVKQCUmmUjgTjuUo oR65yPIOzNuZ5j|11|21-520-971-6013|8574.87|MACHINERY|counts. pending packages haggle quickly alongside of the pending requests. ironic fo 6796|Customer#000006796|ry4xotfFJkV|21|31-204-934-2930|4956.27|FURNITURE|beans! furiously even deposits doubt fluffily carefully express pinto beans. furiously pending 6797|Customer#000006797|CbAgm9paksZYcQNM|2|12-443-241-4031|6051.25|FURNITURE|s ideas are carefully across the fur 6798|Customer#000006798|DRAvKxwGdh8qBsv6DlFfvsilS7,QuSOQfO4D|21|31-784-264-5102|6312.50|MACHINERY|kly ironic pinto beans. fluffily final escapades beyond the reg 6799|Customer#000006799|OmuxHQ,MFaA6IKDRJpI7rOq0 2|19|29-861-675-1912|8266.46|MACHINERY|tes. slyly permanent foxes sleep against the carefully regular deposits. ironic, ev 6800|Customer#000006800|dyQqJ sBw9RZiggYPODCddm2|24|34-746-736-6270|2438.35|HOUSEHOLD|al accounts sleep carefully accord 6801|Customer#000006801|h4zMf8BMKsgOf964TpkBtFenTIiHb|7|17-563-914-8922|4350.72|AUTOMOBILE| the furiously express accounts. even foxes sleep fluffily ironic ideas? 6802|Customer#000006802|XwZM0 CSn4C Pe|24|34-621-826-1804|3589.51|MACHINERY|nstructions wake quickly along the bold accounts. ironically ironic tithes sleep blithely unusual 6803|Customer#000006803|6xmvFf,9ifUkUXDtls|6|16-508-741-8182|2880.15|MACHINERY|ully slow asymptotes. special pinto beans use after the final ideas. furiously even ideas hinder. qui 6804|Customer#000006804|y3KkEdC7h4oc6BGM8r xsYpi V7ivtz9|16|26-988-381-6875|4647.66|MACHINERY|ccounts sleep slyly among the regular packages. quickly final ideas sleep furiousl 6805|Customer#000006805|kTi3eJA0j3wmK,PDjWGed8tzM|24|34-956-659-4290|147.84|MACHINERY|eep quickly carefully even courts. 6806|Customer#000006806|63GYGgLROgXoSKxwRPVwKdjdTFx|3|13-535-116-9961|7653.27|HOUSEHOLD|press theodolites. blithely pending accounts according to the quickly regular accounts nag alongside of the unusua 6807|Customer#000006807|zfbTyBPugePY1ea3MbBFWuXrhsT|0|10-318-291-1534|3571.22|BUILDING| blithely ironic deposits wake slyly about the fluffily pending dependencies-- bold, unusual theodolites above the 6808|Customer#000006808|roBxgLQjBCcu7 Yls8BnCT3Oum241ON43b|7|17-840-939-1028|3471.22|MACHINERY|gle idly. ironic instructions sleep carefully accounts. bold, even sentiments haggle. furiously 6809|Customer#000006809|D2OPM0RO7wjQB,y5lQJIhGKRAK|19|29-455-199-9534|5772.30|MACHINERY|efully according to the fluffily silent packages. carefully silent packages use furiously above the carefully even 6810|Customer#000006810|Vav VVRBQ3WYqKNtNTi39x7,kXBcfblF3|13|23-459-864-2903|4116.60|AUTOMOBILE|instructions cajole permanently furiously bold grouc 6811|Customer#000006811|OcI9j59VXpHJy8akRgo9QazL2FBJzr|2|12-527-163-1718|6619.05|FURNITURE|ans besides the carefully ironic pinto beans sleep slyly fluffily pending 6812|Customer#000006812|AhoTwIbKJQn1FD4f59Og02|12|22-522-190-4711|1433.83|HOUSEHOLD| slyly ironic instructions! pending 6813|Customer#000006813|7sU37tY2xghEmgmrmGFw|6|16-378-417-2821|6328.64|FURNITURE|ges. pinto beans are fluffily. blithely 6814|Customer#000006814|xKxM4j61kF6WrHJSYTo|0|10-143-621-7307|4498.28|AUTOMOBILE|ructions. carefully bold Tiresias haggle furiousl 6815|Customer#000006815|SwpQn8U2,7FIzgYG|23|33-553-747-6097|8623.37|BUILDING|dolites sleep. regular theodolites after the express ideas are a 6816|Customer#000006816|l9iMew1ckL2nrx5YiUnd0Bs7Z96,A|23|33-435-484-3984|5981.91|HOUSEHOLD|to beans are alongside of the final 6817|Customer#000006817|0GuFB2pIst2i5ku761SGYJF0YKbemV3oGBxuY|11|21-463-437-2941|6900.32|BUILDING| beans. carefully final theodolites haggle furiously unusual foxes. pin 6818|Customer#000006818|NsWBjqO16kujv3WRtKhhLH28|18|28-865-256-2071|9642.22|HOUSEHOLD|pecial packages sleep carefully furiously regular deposits. even, regular pinto beans cajole slyly about the blith 6819|Customer#000006819|BxmHawhcf5E|19|29-909-925-3216|2833.96|HOUSEHOLD|ts above the regular theodolites cajole at the pending, bold requests! slyly pending asymptotes would haggle. ir 6820|Customer#000006820|i1RJGpP9HqI4s6151jR1z8ZWpXdOU|10|20-589-342-9128|9377.01|AUTOMOBILE|ironic requests against the regular warthogs nag across the fluffily regular deposits: 6821|Customer#000006821|GpFN4sJSskru0JEsjNZLg,dcTKNsIgDO|14|24-956-317-4494|431.91|FURNITURE|l platelets. fluffily regular accounts across the silent pinto beans p 6822|Customer#000006822|bo tbD14X1LH254lRO|18|28-422-199-2677|8027.05|FURNITURE|sly regular ideas wake. fluffily do 6823|Customer#000006823|NxR5mcTtNqZad4WGlWudh PLP1itX0Y,xI4f|5|15-158-466-1756|4742.19|MACHINERY|onic deposits. carefully brave decoys along the blithely ironic asymptotes detect c 6824|Customer#000006824|ots Xj,zI6Lt,5CSMqtm0aoz7UeBs|14|24-918-571-1016|4911.60|HOUSEHOLD|encies cajole at the fluffily even pinto beans. express ep 6825|Customer#000006825|iCyuTflSjsWcDC|12|22-228-819-1874|3300.02|AUTOMOBILE|accounts. furiously unusual ideas are daringly alongside of the carefully unusu 6826|Customer#000006826|i6Gp3F X4bJxVZ,IOFWyvKHBzEWDK7Ao4B|16|26-414-821-5076|-532.24|BUILDING| slyly pending asymptotes! pending foxes nag. slyl 6827|Customer#000006827|pkyUJaeoz7jLGIrQ,kYUVF5loZj|5|15-181-335-1276|6647.51|AUTOMOBILE|press packages cajole carefully blithely bold pinto be 6828|Customer#000006828|i07aofApQ6Sg,gMjYzl ycOlqa sSi|11|21-232-607-3504|6718.53|FURNITURE|ickly fluffily permanent asymptotes. fluffily regular deposits use slyly carefu 6829|Customer#000006829|hLxbhkw8dFbPqPuELUIn39hrnisrNUfJv4F|20|30-413-747-6182|1323.84|BUILDING| the packages. regular dolphins s 6830|Customer#000006830|eVnzLRiUrpOj2zHTaq|6|16-527-593-5356|5679.21|AUTOMOBILE|endencies are slyly against the final, r 6831|Customer#000006831|s ZzU g4nFiJrvNg pE7cN1UGL7z3THVaXiuuY|16|26-690-485-7073|1470.27|MACHINERY|around the accounts integrate slyly according to the special accounts. blithely express deposits wak 6832|Customer#000006832|AggcVEoM1Dw2WfE|10|20-653-330-2367|2821.81|BUILDING|o beans use slyly ironic theodolites. ironic platelets boost carefully into the fina 6833|Customer#000006833|xpXLjnef5qfH2xlHXF9oMMzIjIsuex,F0uw|13|23-153-556-1341|3185.16|FURNITURE| quickly bold instructions. even somas wake blithely. blithely furious theodo 6834|Customer#000006834|2 OGzQbTM4PhDp6|12|22-607-237-9927|4970.18|FURNITURE|y after the notornis. fluffily regu 6835|Customer#000006835| 1Vwc1DHAl|18|28-336-747-3054|8825.71|FURNITURE|ss asymptotes about the bold ideas 6836|Customer#000006836|IPAnUQ7OdyUJ9HlXzSKYEp|9|19-944-807-7519|7724.27|HOUSEHOLD|e regular requests sublate regular platelets. ironic dolphins integrate carefully. slyly even 6837|Customer#000006837|kcS0yon,tgPVfGO7hrxyv5C,Su,gM|4|14-101-991-2132|8823.38|BUILDING|s. pending, express pinto beans w 6838|Customer#000006838|BITaY3dbOQVUe3i5g7,ewHd|23|33-156-718-4716|1578.52|FURNITURE| courts are after the fluffily express requests; c 6839|Customer#000006839|oYnqEjWQ9dZezLjJtHPNIXI8HoxtqANG Z7z zj1|19|29-332-980-9867|7140.01|HOUSEHOLD|tes sleep slyly furiously special requests. carefully even dolphins nod blith 6840|Customer#000006840|8enbm51YZ1P8 9WL56McRjcRLxC|18|28-421-550-7131|1256.89|HOUSEHOLD|ly blithely bold ideas. blithely regular excuses nag slyly even requests. slyly regular forges are packages. pendin 6841|Customer#000006841|g NzADDwQKOQedcgxkP3 vDTJ0OThsh0sqI,vb|9|19-631-468-3008|2459.12|MACHINERY| busily final packages across 6842|Customer#000006842|1xXUXh723lgE0wfY1d9vuOTXELqhEPFo5t|9|19-676-771-4155|1342.45|MACHINERY| above the regular theodolites wake ironically against the bold, regular foxes. unusual requ 6843|Customer#000006843|M5z1DoiXAJCQVVIHKjMbntcT1A3Lfc4XSkXVRY|19|29-764-509-2918|-563.00|FURNITURE|ove the carefully regular instructions. carefully r 6844|Customer#000006844|hgGc5KCVN3QDk0Lci|6|16-763-561-5338|6663.09|AUTOMOBILE|ng to the silently unusual deposits. slyly ironic excuses use around the slyly regular escapades. regul 6845|Customer#000006845|VzpxA2uz,A BwWkAvIJnXqZHGDlhXK8n3IX8|19|29-371-479-7196|4510.06|MACHINERY|g the slyly pending packages wake along the carefully final accounts. regular pin 6846|Customer#000006846|ggBnTO6OZXBMWuflbpz4yVEDFUa9n|20|30-223-579-2199|8524.19|AUTOMOBILE| carefully ironic requests. slyly ironic deposits sleep along the carefully special dep 6847|Customer#000006847|FgCB,v15gQzq6|16|26-489-731-4993|81.08|AUTOMOBILE|across the quickly final theodolites. slyly ironic d 6848|Customer#000006848|XETSPsMa,,KHVAAu|8|18-993-692-5687|7653.55|MACHINERY|. theodolites run carefully quietly even packages. furiously even pinto 6849|Customer#000006849|zwnB9JnyCV0B2O Ue6Jn78KBarQRx2m|19|29-921-924-3641|8217.84|HOUSEHOLD|requests. ironic instructions use blithely blithely special ideas. bold theodolites haggle a 6850|Customer#000006850|A0qvm6saCh|4|14-519-545-3578|-702.43|BUILDING| special ideas. final, regular platelets affix regularly according to the slyly bol 6851|Customer#000006851|hvFJsItQnymk8pxkrjnenWCJ9GAjSNh |12|22-860-161-7304|6230.56|FURNITURE| foxes. blithely enticing accounts above the furiously re 6852|Customer#000006852|C hC7N6xSqjDVmEQzEZ01NYj4SBl5nhu|24|34-681-524-9480|6121.02|HOUSEHOLD|ily silent pinto beans. even, ironic dependenc 6853|Customer#000006853|AbpZYqtK6c5EfpZtpF|10|20-422-853-4017|1986.65|HOUSEHOLD|bout the requests. pending deposits cajole carefully above the regular deposits. requ 6854|Customer#000006854|Ka5iBA43bmKVAjKqFya2HfFkurzVx3pbFn,|5|15-430-821-6024|2568.18|FURNITURE|ts cajole carefully across the regular deposits. furi 6855|Customer#000006855|7yJhlD4Ziy9 IFz4VFzAURNoVWzFj5zyYdKBKT|6|16-876-384-6010|8375.91|BUILDING|s across the slyly unusual requests cajole alongside of 6856|Customer#000006856|CQMnbODOwhI9toZ75vf|23|33-529-574-6366|4984.79|AUTOMOBILE|. blithely ruthless warthogs against the slyly final deposits wake along the furiously fi 6857|Customer#000006857|qnTjfF57SI5LrBzO|23|33-446-990-4978|1409.67|BUILDING|. slyly final packages poach bli 6858|Customer#000006858|pd6Motczl51ondujEyQ7367tYCT4NC6|0|10-799-377-4331|7239.92|BUILDING|olites cajole carefully requests. special, blithe gifts pr 6859|Customer#000006859|1zy,R99p9Dg|17|27-368-450-7892|8452.75|MACHINERY|ular requests. blithely regular requests nag! quickly unusual instructions cajole. carefully ironic ac 6860|Customer#000006860|j0c5d8lwjd X7fYicMKeDycv|4|14-580-483-9140|9414.14|AUTOMOBILE|ithely unusual pinto beans unwind special packages: quickly pending deposits cajo 6861|Customer#000006861|um,8dEPxPOIu4uCO5oF9C os|19|29-269-205-4055|8593.46|BUILDING|ole fluffily against the carefully unusual excuses. special, final accounts 6862|Customer#000006862|WHjtQl UwzncNIkejao3a5W|14|24-682-159-5973|9323.04|FURNITURE|heodolites wake idly against the packages. ironic theodolites c 6863|Customer#000006863|C7xp7Euj8Zatj|1|11-147-882-1449|6353.09|AUTOMOBILE|luffily above the ironic, express requests. sly, careful foxes are about the pending, fl 6864|Customer#000006864|sdyPu0LQ3RkTdoFy|23|33-998-134-8314|8447.64|HOUSEHOLD|ickly even theodolites cajole slyly blithely special pinto beans? slyly unusual depos 6865|Customer#000006865|5JPh7HJJGpVizXF|24|34-198-293-9623|527.61|AUTOMOBILE|l courts. slyly ironic foxes b 6866|Customer#000006866|P7lfQDiROc4qhR3Khflxr|6|16-366-687-4124|-567.46|MACHINERY|ly. slyly regular ideas integrate instructions: dependencies haggle furiously regular, regular dolp 6867|Customer#000006867|t1gpHaWFZqRKhV,raEaxdUJ,MU|14|24-146-780-7698|8539.69|MACHINERY|es doze furiously express accounts. slyly silent requests sleep fluffily fluf 6868|Customer#000006868|R13,6EjlPmdsl|23|33-647-349-2675|7639.96|MACHINERY|al packages. ideas boost slyly above the final requests. regular deposits haggle fur 6869|Customer#000006869|6ZYHNoZX5,Me|14|24-606-630-9319|-84.65|MACHINERY| pinto beans. carefully ironic accounts haggle caref 6870|Customer#000006870|i3GhNxcD0,NDHWYXq778CcKI4Rok57Rk4I50R|16|26-714-569-7040|5292.68|AUTOMOBILE|boost furiously. furiously unusual ideas nag along the ideas. requests cajole carefully among the sly 6871|Customer#000006871|4Aawh3u9VlvfLcp,2|4|14-497-499-6031|6002.37|AUTOMOBILE| regular requests. even, close deposits are. permanen 6872|Customer#000006872|seuInq8wsBrKZU6lBlb ro1lt1gMtWkI L6LEQS|14|24-644-733-1264|9852.24|AUTOMOBILE|y ironic packages after the quickly final accounts haggle foxes. ironic e 6873|Customer#000006873|KXt6OTXQyCYz46Kw5Ynz|0|10-948-162-7136|1599.50|BUILDING|ts boost requests. slyly quiet requests are slyly across th 6874|Customer#000006874|VbQ7fM46whu0cr31adJ|19|29-695-872-3712|7814.06|FURNITURE|en requests mold furiously express, final multipliers. car 6875|Customer#000006875|TjO3DIyAnZ5kX3 KOWj7m7d,3jC1dbZZNy9MoNVg|10|20-934-535-5633|28.31|AUTOMOBILE|lly regular instructions haggle fluffily accounts. carefully regular dependencies wake furiously. 6876|Customer#000006876|AzTnyu945JeYf4PLbjV7cSL3v|21|31-326-600-5503|812.97|MACHINERY|ccounts should nag until the ironic, express packages. ironic forges 6877|Customer#000006877|QR2VDDA94zJHPMmac86RN1Lg3SPxPzIt,u|16|26-767-249-9337|-532.13|BUILDING|rding to the quickly bold instructions? careful ideas cajole furiously against the final packages-- fluffily busy a 6878|Customer#000006878|OyrOaYhd68cPx9maycfwDGq R22|14|24-648-517-8694|8396.86|HOUSEHOLD|ic packages boost around the blithely final deposits. permanent, 6879|Customer#000006879|qGAxDp5CIcJWPNt zGFerq4kFZWQiX|12|22-942-165-1099|5044.93|FURNITURE| to the special foxes? fluffily special theodolites 6880|Customer#000006880|t3um3YPoJ17XQUXC8V3VBIFnoz1D|24|34-961-846-5811|225.45|BUILDING|ts print. unusual, special deposits against the blithely regular accounts serve fluffily after the deposits. even 6881|Customer#000006881|6VfVWZMuCYB8i4b 6dRhgUPFMnWADADr407sLL|18|28-406-320-1387|3156.70|AUTOMOBILE|ets across the ironic pinto beans should boost ironically furiously ironic asymptotes. blithely regular accounts des 6882|Customer#000006882|3lBEzGcwg1vKtSkqtvzMVQwqvwfGBGn|2|12-151-684-6027|-396.82|HOUSEHOLD|he unusual pinto beans. slyly regular accounts haggle furiously express excuses? slyly pending warh 6883|Customer#000006883|,OGuJ9Zt3XpAEONbIpIpal Ux3|11|21-873-538-3754|5579.14|FURNITURE|s frets nod blithely alongside of the deposits. quickly even theodol 6884|Customer#000006884|yRIvbIG72RGuz|11|21-789-993-5944|2597.78|BUILDING|uickly. quickly express deposits sleep slyly? q 6885|Customer#000006885|zgFxaDEtwdOvsa8eku1KI5f0Hq|17|27-266-722-7657|-188.30|AUTOMOBILE|sits wake carefully. ideas are carefully. final, furious packages impress slyly unusual requests. slyly re 6886|Customer#000006886|e1kh m79wZ2sAUFXAzmTAsYsG6bpc2SlJ|5|15-103-327-5461|1158.35|FURNITURE|lly. fluffily regular foxes alo 6887|Customer#000006887|Ts6FP r64sVYHPT|0|10-744-644-3355|7280.37|MACHINERY|lly even accounts. unusual, final foxes cajole express waters. carefully special 6888|Customer#000006888| O40aa4,Ip frcWNEh6XbZL5kktj,QFqRe|5|15-967-617-5600|6207.74|AUTOMOBILE|to beans. regular packages accor 6889|Customer#000006889|SvUWryqsTeYMUI8MoGwe23MxCJcgnk|15|25-961-943-9801|7526.00|FURNITURE|according to the accounts. foxes cajole carefully. ironic packages haggle blithely 6890|Customer#000006890|9h9 InEMYJ5P|7|17-911-310-5010|9514.17|MACHINERY|en packages. express, even inst 6891|Customer#000006891|6KWm7ap8xnKV2zYC7ejZifLwE|12|22-373-997-5030|6815.03|BUILDING| use carefully regular packages. blithely 6892|Customer#000006892|mNuuOriPB9 YdILhmvINOGd38jGBZ|13|23-151-643-6412|6203.78|BUILDING|s sleep carefully foxes. slyly pending requests sleep requests. requests nag furiously carefully 6893|Customer#000006893|IQKKtRNXWAsdbqynSPQ7P5Pd8X uF|3|13-947-844-8526|6929.65|AUTOMOBILE|y special foxes are quickly carefully final requests. unusual packages affix express, regular requests. quickly 6894|Customer#000006894|qP,e3crppnJlKzejVH b2D,|12|22-247-482-7189|7747.33|AUTOMOBILE|al accounts according to the pinto beans doubt among the final braids. blithely fina 6895|Customer#000006895|eYt JN3SRFOg,PHtmhhfqN|20|30-525-577-9213|361.67|BUILDING|kly even theodolites. packages integrate across the slyly pending accou 6896|Customer#000006896|MvVnnxEF29nDVdr WvzbHWkm|24|34-850-657-1872|4576.31|AUTOMOBILE| final, regular waters. final platelets wake furiously silent deposits. regular 6897|Customer#000006897|2y42Nhr2PYA54EHyciLjYpnVHJStojcN|0|10-920-317-5942|101.13|BUILDING| the express dependencies. boldly ironic foxes nag slyly express requests. final 6898|Customer#000006898|Iu VC1aV16HlqfvkfNd8snf4|20|30-234-165-4904|3546.09|MACHINERY|. carefully regular ideas cajole carefully brave, final asymptotes. regular, regular 6899|Customer#000006899|fI2lc9kOUESmGia2kXSJNNI, 5MMioi3cbGg1M97|0|10-535-357-3310|-909.78|BUILDING|e carefully regular ideas. pending warhorses use quickly according to the blithely even accounts. blithely regular 6900|Customer#000006900|XxzSLyP jdrxrFf,iyxf|5|15-836-566-6221|8277.05|HOUSEHOLD|ully final requests sleep carefully enticing, even theodolites. excuses cajole! f 6901|Customer#000006901|MF1Yh6821MDYo|17|27-691-188-7577|4462.87|AUTOMOBILE|sleep quickly even theodolites. furi 6902|Customer#000006902|bDR25t GnLkoK|6|16-378-380-3108|7453.93|HOUSEHOLD|e fluffily above the blithely even dept 6903|Customer#000006903|AtYnMz1ydb2y2yeKlQ6df1txYM1Ibs5u|9|19-471-505-9852|8241.38|MACHINERY|gouts cajole instructions? quickly unusual ideas caj 6904|Customer#000006904|Pm8 tKCreWStvS|22|32-451-759-6848|9979.02|AUTOMOBILE|lyly ironic asymptotes: fluffily ironic accounts cajole blithely pending 6905|Customer#000006905|R,hn e5kStdmzVDhYooDC7Z1tJ7|11|21-390-139-1440|9534.63|MACHINERY|ng requests. carefully silent theodolites 6906|Customer#000006906|NtpZ4oXphusYXOHppCd,DNX g65hon1kdIAGs233|6|16-847-442-4850|9552.15|AUTOMOBILE|inal theodolites wake slyly carefully dogged packages. reque 6907|Customer#000006907|xVxf5 SbjMNLGhAFzmW6|6|16-105-262-6169|7106.46|HOUSEHOLD|ccounts integrate against the even packages; blithely even pac 6908|Customer#000006908|PSFN752zobO,NnrXueh68DsNliR|18|28-352-804-7306|9321.69|HOUSEHOLD|y packages; carefully even theodolites boost carefully after the furiously ironic gifts. foxes kindle slyly. sly 6909|Customer#000006909|wTMW41vDDJ,rDvLBtcP2lAGPortmxNP18h|10|20-480-370-7421|5745.76|AUTOMOBILE|s about the ironic deposits detect blithely bravely ironic orbits. pending packages above the blithely iron 6910|Customer#000006910|wHUTQtKiSMGbGqKcC526p9Pfebuc7r|5|15-722-787-9962|8945.89|FURNITURE|the express, regular escapades. carefully unusual theodolites do wake quickly. quickly 6911|Customer#000006911|lx95MUKc7,CsN WHXRWUoVV|6|16-440-937-2210|3682.25|FURNITURE|riously pending accounts? carefully unusual accounts according to the quickly unusual re 6912|Customer#000006912|eZArhHGNgAZTIIs6m8JEFgvhhsybD77LXI8ScXH|8|18-789-906-8424|-383.69|AUTOMOBILE|quickly against the regular deposits. blithely regu 6913|Customer#000006913|jH27L,gZj9dws4sVD9pjQWjNM9Z3hjJhNEAHd|7|17-424-926-7617|6237.26|HOUSEHOLD|ses nag blithely slyly silent accounts. slyly ironic pinto beans alongside of the 6914|Customer#000006914|6ZnxYihMt7HmIQqiItpeVDq81omFb9S4mul|21|31-622-699-4865|8725.78|MACHINERY|. regular, ironic accounts among the furiously even pe 6915|Customer#000006915| VQo,H2BRkjoa0dYXLAA01mqwUytbBM|19|29-651-328-4337|319.47|BUILDING|lets after the carefully slow packages cajole blithely asy 6916|Customer#000006916|KGSD2t2VRVQ,n3EW4TiQH8i4L8VBRP1P|5|15-509-409-1957|-795.98|AUTOMOBILE|lites. regular pearls solve quickly. carefully regular sheaves against the always 6917|Customer#000006917|SwW3qHYL1ZuBlx17gCKpvcP|6|16-544-374-5520|6932.21|FURNITURE|d sleep. blithely special ideas haggle along the special pinto beans. regular, special warhorses are alongside o 6918|Customer#000006918|0HHnMHiko6CFQTroq5lGeFg0JDtZm6PUhIWU|3|13-658-888-3933|6671.40|HOUSEHOLD| fluffily carefully quick accounts: fluffily regular pinto beans cajole quickly according to the q 6919|Customer#000006919|hUBFeaV6cn41fFHMpvd7nuAMC1Q4|10|20-964-269-3393|1279.94|HOUSEHOLD|deas are carefully. regular, permanent instructions boost slyly. blithely unusual deposits caj 6920|Customer#000006920|DmTeHxRVLSe32KfxJp38lE|0|10-384-129-2868|4968.13|BUILDING|ounts are blithely above the furiously special sauternes-- quickly special 6921|Customer#000006921|nGFGmYctkA9IM0vxf2Y4GBPnT|12|22-642-260-2620|7388.91|HOUSEHOLD|ly unusual requests boost above the final pinto beans. careful 6922|Customer#000006922|3bCCt0S2wd6RQeq|4|14-163-216-8977|4434.20|HOUSEHOLD|kindle carefully. carefully unusual packages d 6923|Customer#000006923|v6BO5dXHqLFmOfPck0|7|17-733-662-9017|9600.79|FURNITURE|t about the pending requests. final, regular theodol 6924|Customer#000006924|z,OREBfNfUkRsj|16|26-289-764-7597|-408.30|BUILDING| the unusual packages. ironic, final requ 6925|Customer#000006925|2Gg4xMqjRAV0nAn|8|18-878-944-5385|-448.55|HOUSEHOLD|sits about the slyly regular accounts x-ray carefully over the unusual, final theodolites. blithel 6926|Customer#000006926|KOx lB nFGsdak3 Sc5zCjiRFoWq1mx5t|22|32-894-701-8844|5160.08|HOUSEHOLD|its haggle quickly. fluffily even packages sleep slyly along the theodolites. express, pending deposits above 6927|Customer#000006927|Ra0BshyuR,4uTcWjvs1PIqswCbIC6U l7A|6|16-135-283-6467|-509.64|FURNITURE|efully even depths. requests should have to w 6928|Customer#000006928|SOf,2mOUgK5PpW,C9US8oH61eNDG4|2|12-944-721-3451|1380.62|AUTOMOBILE|fully after the slyly dogged depos 6929|Customer#000006929|sQEFEgufg0ZT|12|22-151-910-6027|3000.98|HOUSEHOLD|lites haggle final platelets. express, special theodolites wake slyl 6930|Customer#000006930|RpPHMhjHQLhRR|11|21-832-212-8391|386.35|AUTOMOBILE|e blithely even accounts. ironic, even accounts about the furiously final 6931|Customer#000006931|DJcGXg0BrP1ibpV7UCxEaK5OXy yCckOACzF|24|34-282-643-3571|1402.40|HOUSEHOLD|nusual pinto beans across the ironic deposits are blithely alongside of the carefully brave foxes. quic 6932|Customer#000006932|XjlmI08 R3CXJ8JPYGwSihQUDir|19|29-930-104-7406|6064.87|AUTOMOBILE| courts cajole ironic, final ideas. ironic multipliers cajole carefully iron 6933|Customer#000006933|w0D2wXhqohW3rtxjXxIQvcxH75Fh1XAWZ3O|20|30-564-778-5516|4292.61|FURNITURE| accounts above the quickly final warhorses print blithely 6934|Customer#000006934|oVWWov tNz1fn|14|24-121-895-8013|5944.59|MACHINERY|efully express ideas cajole furiously slyly brave packages. closely bold requests haggle 6935|Customer#000006935|eqVuF6zTXrqK4BhmNUBAZQghbjY8uy0co|3|13-497-759-6170|6730.94|FURNITURE|ronic, regular requests after the ca 6936|Customer#000006936|XgPD s4bQqx5|9|19-923-718-1041|1006.58|HOUSEHOLD|he ironically even packages. regular dolphins are slyly by the pending excuses. slyly final requests haggle careful 6937|Customer#000006937|BlUCF0,i2jMXs70wxl3AWvd9frn|24|34-883-594-7672|8496.83|HOUSEHOLD|lly slyly ironic deposits. quickly special excus 6938|Customer#000006938|2ud1eHT7TvCAPub2UZ3bkcKT2Pq8M11eSpd|15|25-974-463-8064|2339.57|HOUSEHOLD|realms are busily carefully express requests. furiously pending foxes try to cajole blithe 6939|Customer#000006939|e2UhEWWPopLOyQ6G|10|20-425-357-4024|5165.17|BUILDING|ut the ironic ideas. regular courts according to the final instructions detect blithely theodolites. even packages 6940|Customer#000006940|37YCc8T1xgurTO|11|21-485-600-2482|8105.71|HOUSEHOLD|y along the carefully regular gifts. final accounts cajole. quickly ironic deposits kindle carefully f 6941|Customer#000006941|63Z3Qe6urA|4|14-210-435-4058|6757.19|MACHINERY|ular instructions cajole above the bold, ironic accounts. ironic ideas about the blithely even re 6942|Customer#000006942|N31XA0a7VOdIiIxuKpbYwE0l|10|20-899-337-6472|2059.91|HOUSEHOLD|ep blithely about the quickly regular dependencies. ironic 6943|Customer#000006943|qjFXg1Ia9FuM|24|34-704-691-5442|1872.41|HOUSEHOLD| carefully regular excuses. ruthlessly unusual instructions was ironic theodolites. pinto beans cajole carefully 6944|Customer#000006944|PlOgCXn qIsC96sqwi7jMKVcv|8|18-677-417-1382|7549.91|AUTOMOBILE|nding accounts. bold dolphins detect bold packages. reques 6945|Customer#000006945|X6nI1uOqfKBA,7V7iEb9PWg6hbM794B1ZmSsgk|10|20-763-130-4896|4363.99|BUILDING|nt accounts sleep quickly. theodol 6946|Customer#000006946|O142a46LKzb75,ggUsnA7yY MHZ7rxNi|24|34-108-937-2099|6626.82|HOUSEHOLD| final packages boost above the carefully bold pinto beans. pending sheaves sleep beneath the sp 6947|Customer#000006947|2q61zt4rUQY6JOVeKxh FcRNRqj|24|34-935-489-1820|-30.39|AUTOMOBILE| packages. furiously close deposits sublate across the final packages. regular, bold requests breach. car 6948|Customer#000006948|GWfdOFWXism9l,PQp8azQJO UPfcdMm2cb,rqU5|3|13-636-530-5983|4150.95|HOUSEHOLD| blithely unusual forges run quickly furiously final accounts. blithely regular accoun 6949|Customer#000006949|88E o,rjHZJVp|0|10-920-156-2564|999.02|AUTOMOBILE| even instructions. asymptotes use. blithe theodolites 6950|Customer#000006950|D7PZ9 FXDeoG9jzsIqiVm8i V|16|26-567-707-8729|2414.22|BUILDING|g to the express platelets. even account 6951|Customer#000006951|dswooF8dKb|21|31-661-344-4562|5751.69|MACHINERY|regular requests. carefully silent theodolites promise quickly regular accounts. platelets sleep among the fina 6952|Customer#000006952|tedolX6bU7a07kS,HSV1Kh2VartW|1|11-964-186-1638|9600.30|HOUSEHOLD|ckages play pending ideas. pending dependencies wake blithely along the unusual accounts. eve 6953|Customer#000006953|T6GNNKYnmp3KhK2a,n|4|14-200-221-1070|2384.73|HOUSEHOLD|ar dependencies boost across the regular, bold 6954|Customer#000006954|rGHTeIqYsbkLG4yOmrZWyvY17zxvWjCgvj|19|29-846-280-8291|5080.67|BUILDING|ross the ironic pinto beans affix after the packages. ironic, express theodolites after the furious 6955|Customer#000006955|7Pl77 AAMou0F56ufvqSGYm2dFZrsBSf|23|33-734-299-2759|6751.07|HOUSEHOLD|is against the carefully bold theodoli 6956|Customer#000006956|k 4M1dYnBY0s9tDIjnpZQ7QSRAsqv,dP4qip CBY|19|29-765-731-1061|9202.51|HOUSEHOLD|ial requests detect carefully express instruct 6957|Customer#000006957|sAt10Pog00qXxf3|1|11-448-447-2941|4672.45|HOUSEHOLD|idly pending pinto beans. carefully pending deposits cajole quickly across the unu 6958|Customer#000006958|pyyjzooPiwi2FUIz|5|15-554-805-5336|9829.41|AUTOMOBILE|ly final pinto beans. carefully express ideas about the furiously unusual pinto beans ha 6959|Customer#000006959|rorENMWClttRXEp|13|23-233-789-3757|2990.45|HOUSEHOLD|gular packages are slyly across the asymptotes? escapades boost blit 6960|Customer#000006960|OUKTB cNG030,aFLzyB|19|29-674-628-3972|-505.65|BUILDING|blithely final waters detect according to the even pinto beans. express 6961|Customer#000006961|T6oeN XNst4bY6QOIxFQAj,WN|16|26-346-690-5410|2474.82|MACHINERY|its use carefully blithely even pinto beans. carefully even acco 6962|Customer#000006962|4G9HL28bwq|5|15-357-936-1344|1281.15|MACHINERY|equests. regular deposits nag. foxes sleep quickly-- carefully regular requests across the deposits wake quick 6963|Customer#000006963|CW0iwpVyNqVJiJf0roU5OAoX|16|26-430-431-1058|3161.07|AUTOMOBILE|le quickly ironic excuses. slyly close ideas wake quickly wit 6964|Customer#000006964|IWYQGMU6rEz4GMdjsQAKCsnQT5|13|23-185-648-1303|7140.70|HOUSEHOLD|lly bold requests use carefully carefully bold 6965|Customer#000006965|d6gerbM9AWmOdDpp|15|25-939-405-4493|1815.68|AUTOMOBILE|ular accounts. carefully close accounts doze. final requests solve furiously furiously ironic requests. special asym 6966|Customer#000006966|Q3yAE1yoj,TKafWnhyfcR4WL322ME Pv9bkNR,FW|22|32-842-357-9534|144.58|FURNITURE|theodolites use. finally special excuses hagg 6967|Customer#000006967|uMPce8nER9v3PCIcsZmNlSrCKcau6tJd4qe|13|23-816-949-8373|7865.21|MACHINERY|r pinto beans. regular multipliers detect carefully. carefully final instructions affix quickly. packages boost af 6968|Customer#000006968|Ez1Rhj5Qi2,10Nug38BsPiacwskhzpT|5|15-858-442-4792|1651.44|AUTOMOBILE|deposits are blithely unusual foxes-- even requests do are to the blit 6969|Customer#000006969|HKsXzhiJwn0oWqic7outvp6ek5|24|34-608-122-1503|727.09|HOUSEHOLD|s. even foxes wake. requests haggle slyly carefully final deposits. even f 6970|Customer#000006970|PEtGiJUdTje1Iag6unPFdev|7|17-244-333-8174|8208.18|MACHINERY| regular accounts grow carefully 6971|Customer#000006971|OMyW3Rc1F9r9ixU|15|25-797-318-1684|1906.06|MACHINERY|ts. carefully even excuses mold blithely carefully final foxes. furiously unusual accounts sleep carefully above 6972|Customer#000006972|MjqRGeXURtvVOEY5u30KbgffSKBvJ2X2OI,Hm6Tj|10|20-340-512-1672|1096.88|FURNITURE|ges wake up the fluffily pending deposits: slow deposits cajole before the slyly regular frets. regular theodolites 6973|Customer#000006973|2kXQXsfJMUjQn|21|31-948-302-3253|8232.51|MACHINERY| the regular, final asymptotes ca 6974|Customer#000006974|TVPV8QjdvgxuewOviKUj8IrMvta yJ|23|33-695-299-2927|3894.63|FURNITURE|ounts above the blithely bold requests sleep furiously across the regular instructions. quickly 6975|Customer#000006975|bSjO03DW yuKgBg2ewRK6e47ixGfAYRuFT,GR9|23|33-652-640-1772|5049.82|BUILDING|ld accounts above the furiously 6976|Customer#000006976|Oj Pipp9GFv8FLDelp82C|9|19-115-502-9616|5348.14|HOUSEHOLD|structions wake carefully. furiously even requests detect blithely. fi 6977|Customer#000006977|IAhFYl42MDQOWl FBdXT8M o|8|18-423-650-1806|3164.56|FURNITURE|usly alongside of the bold accounts. packages haggle after the blithely ironic accounts. ironic pinto beans dete 6978|Customer#000006978| TOk2qS85CJpdrRpGsszF|12|22-707-108-2535|-325.02|MACHINERY|xpress patterns engage fluffily after the deposits. blithely final theodolites nag abo 6979|Customer#000006979|K1aueKP9rhN,OLMh6NB9MF9JC|3|13-179-679-2432|2270.82|BUILDING|ously regular pinto beans wake: ironic requests wak 6980|Customer#000006980|IzbhQ7AaNNSNkTjIGPsH1bZe53WXDupcQm|11|21-352-950-4327|4710.25|HOUSEHOLD|ing instructions: furiously bold dolphins about the furiously ironic instructions haggle furi 6981|Customer#000006981|BDM83,inyQX8VjjAAY8hrSlA8uu9h6zusf|22|32-465-600-2595|7194.73|MACHINERY|le quickly express deposits. final theodolites haggle quickly permanently even pac 6982|Customer#000006982|dTif54yxBO1KSK90YnlVp,j53YYC4D|8|18-481-928-6409|7891.67|BUILDING| alongside of the furiously regular accounts. quick 6983|Customer#000006983|E0AqrX3aIw4ZcINSIWDXD,7qxDykEj5odh8|22|32-796-697-5842|6348.19|BUILDING|ackages. special instructions wake alo 6984|Customer#000006984|ESRBQnnbjKMRydlNyoXn,|2|12-423-137-1791|3020.71|AUTOMOBILE|ckages at the blithely regular ideas wake blithely amon 6985|Customer#000006985|J9qvTE,Ic0Oxv2cbFwAynNxzDW5KY96qVpOQ3q|13|23-652-721-2985|2119.77|BUILDING|efully carefully final deposits: quickly pending dependencies wake al 6986|Customer#000006986|EBY0Sg3HMtzo8CzNhzaUtjiR1bAL|23|33-582-389-4321|6959.44|AUTOMOBILE|nts sleep slyly above the regular packages. slyly unusual packages na 6987|Customer#000006987|lGJOaGFNBmiIy3AIqwf5Az0aGvrXnhfakZBOQawR|17|27-320-420-9595|2282.79|AUTOMOBILE|latelets haggle quickly slyly regular ac 6988|Customer#000006988|PuokzRJ2EY5|3|13-830-329-5880|7733.90|BUILDING|inal platelets affix furiously. blithely regular deposits promise special, unusual requests. fluf 6989|Customer#000006989|VAdDFdOq9BAyUsOP8CaFBvTG8G2K0|23|33-283-282-3404|5712.90|AUTOMOBILE|ular asymptotes use even, regu 6990|Customer#000006990|z3zQwqsS8eg61oVV4e8UgdFDt|7|17-437-544-6391|6746.93|MACHINERY|ackages. express, unusual accounts cajole furiously toward the fluffily final acc 6991|Customer#000006991|xMyl,tcSuNEP L6z5VlZXB69GuDoBz1 FeQxHnb|0|10-948-457-2049|5747.55|AUTOMOBILE|s around the slyly bold deposits wake carefully among the bravely pending asymptotes. quick 6992|Customer#000006992|HvnRKHV1kJ|23|33-830-883-1031|6070.79|MACHINERY|ithely quick foxes cajole fluffily. even accounts 6993|Customer#000006993|VgyZ4hR3o2DDrqX2p9f1ao8GLn7Y el0|11|21-996-879-2346|4754.10|FURNITURE|l pinto beans boost silent pinto beans. carefully even accounts integrate evenly. d 6994|Customer#000006994|rV05MzBGNrLz|0|10-962-702-6979|7109.05|BUILDING|ctions unwind. bold foxes cajole quickly above the regular, i 6995|Customer#000006995|h1Is0,kANMXbCDqVHJav|16|26-594-133-2656|8768.51|HOUSEHOLD|slyly ironic, even accounts. furiou 6996|Customer#000006996|FFH0V1HOhdgfDs,kJV|20|30-658-795-5594|3335.98|MACHINERY|egular asymptotes cajole quickly regular requests. unusual courts use quietly unusual, pending packages. package 6997|Customer#000006997|twNLY6rDvngJvLOEiF|18|28-278-657-2446|9728.39|BUILDING|ep furiously along the grouches. regular, slow accounts use blithely regular, bold foxes. slyly slo 6998|Customer#000006998|R4GseYn1PbPAQkah9p5Bdqfe|15|25-779-557-3473|-870.34|AUTOMOBILE|after the blithely bold deposits impress quickly idle instructions. slyly silent instructions among the bra 6999|Customer#000006999|1T9GzwkkJp,giqi KBNPEXhslG|9|19-345-535-4019|93.74|BUILDING| are above the slowly pending platelets. 7000|Customer#000007000|GabU4EArz2WOSxBImz79QitlufnV,kWbFSrQD|16|26-901-104-2112|469.82|MACHINERY|after the carefully ironic accounts boost slyly regular accounts. express, regular a ================================================ FILE: src/test/regress/data/customer.3.data ================================================ 1001|Customer#000001001|KbWTzGB3ZUymu nNCIuG5eCueaqu|21|31-389-986-4741|7140.81|MACHINERY|ever. fluffily special requests are. slyly final asymptotes are carefully quickly reg 1002|Customer#000001002|98bKmyr3jZWRLEY9WBtyUWOodVd|10|20-973-622-6579|3699.76|MACHINERY|ns. deposits along the ironic, regular packages wake furiously according to the carefully even excuses. slyly qui 1003|Customer#000001003|lE07lPMzVzMhG9CUC54uPwGw3BWO|21|31-716-397-1854|7894.00|AUTOMOBILE|quests sublate blithely blithely special dependencies. excuses use busily express pinto bea 1004|Customer#000001004|mBaNGEJoY2tgXD60V2DEO ajjoM3Zd,Jp|8|18-676-152-4849|1512.46|MACHINERY|ainst the ideas nag fluffily according to 1005|Customer#000001005|cTWPLcTvotjgrrcN3j|13|23-149-373-9093|7790.94|AUTOMOBILE|. furiously ironic accounts affix careful 1006|Customer#000001006|Q46palcsa4KwAMhPS|12|22-364-780-5932|7447.99|BUILDING|equests. regular pinto beans sleep furiously express, ironic accounts. special, 1007|Customer#000001007|PfH0lw8GzD7o|9|19-790-843-5283|7347.90|MACHINERY| theodolites. ironic requests wake. thinly silent 1008|Customer#000001008|AfP6tFNz1Eu4buoUd,HrZAld340 xz2wbQ2|2|12-115-571-7897|8191.74|HOUSEHOLD|press orbits affix furiously pending packages. courts alongsi 1009|Customer#000001009|cWONXs2Vx30bkgYoCkx7LrJH,E|12|22-132-906-1117|594.50|BUILDING|ng to the stealthy, final courts cajole carefully alongside of the gifts? regular ideas above the furiously express 1010|Customer#000001010|uasIK CZZ5|5|15-221-463-3776|1652.78|AUTOMOBILE|ing ideas doze carefully accounts. slyly regular theodolites poach. carefully pending de 1011|Customer#000001011|6m8KP FxT4nnHgoc4CN70TVLW1X5Q|5|15-736-809-3168|1188.94|BUILDING|uriously express asymptotes. u 1012|Customer#000001012|5Zsp rqM6oCmgqqFe|4|14-535-551-6255|4422.45|AUTOMOBILE|s above the carefully express r 1013|Customer#000001013|k5rfeOtchP1 w|15|25-725-599-1183|-951.53|BUILDING|idle packages cajole regular asymptotes. carefully express forg 1014|Customer#000001014|ZsiaboMOOV,aGwWUpfE|11|21-553-425-9152|-392.84|AUTOMOBILE|telets. ironic platelets cajole carefully; bold, special instructions unwind blithely regular somas. carefully 1015|Customer#000001015|RDJWEmcAk4GC8OT8WCsXB|10|20-134-926-5391|6392.00|HOUSEHOLD|g courts nag daringly brave pains. blithely special deposits use blithely carefully c 1016|Customer#000001016|8tzkhPXMFHKgmz|11|21-683-368-2994|2357.54|AUTOMOBILE|ular deposits. special foxes solve quickly idly special ideas. never final asymptotes nag. furiously even deposi 1017|Customer#000001017|OoVPZGR5hUp8oo|16|26-593-941-5690|-913.70|MACHINERY|integrate furiously furiously even pinto beans. ironically pending packages cajole quickly furiously special 1018|Customer#000001018|yldxLZOgQwzrXh3t4yktykZZV8v,vK2c6pVr|18|28-450-764-4871|8341.71|FURNITURE|e quickly around the quick, regular pinto beans. regular, regul 1019|Customer#000001019|VMFs38VlBt01g30PzPyliiAoGHazC4HG74JJ|21|31-502-683-3413|2114.53|FURNITURE|quickly special ideas about the courts use of the pending instructions. furiously final accounts c 1020|Customer#000001020|DHom,LSHKfYSwLSZv39AooYQHlvbaeztefjwR|3|13-692-286-8158|6914.87|MACHINERY|es. blithely unusual asymptotes sleep ironic accounts. fluffily express sheaves haggle fl 1021|Customer#000001021|m h2wQbujQnQOrcf109reW0 o|6|16-469-554-5196|1286.76|MACHINERY|courts could hang quickly express epitaphs. foxes haggle above the carefully unusual requ 1022|Customer#000001022|lP,9H6e6mQwLsWYYr2Y|8|18-733-553-2195|9605.83|AUTOMOBILE|ending packages. dependencies along the slyly pending dependencies wake carefu 1023|Customer#000001023|w8 oxHcOTUiF8dOr,ktZ05pO7qcHZ8ZeH7|17|27-960-306-5136|7188.35|AUTOMOBILE| express requests. slyly ironic asymptotes throughout the ironic, final packages nag careful 1024|Customer#000001024|9wLrRS78uOPy7CHW|11|21-508-779-7822|-425.09|FURNITURE| carefully regular instructions. furiously final deposits across the carefully special ideas cajole furi 1025|Customer#000001025|3T2A1uo8mCqTeO LTW8atjLBLO12nh6lyl|8|18-588-456-4616|3363.46|AUTOMOBILE|to beans sleep according to the fluffily regular instructions. 1026|Customer#000001026|ktKcS9tV2OC8T42KVqMem NjkNO 4pkXmu|17|27-169-221-8173|9699.28|MACHINERY|totes against the stealthy deposits haggle fluffily after the regular, regular deposi 1027|Customer#000001027|GNaw4RXXMr|2|12-278-154-5262|4946.21|BUILDING| final requests haggle. final, even sheaves maintain carefully above the even ac 1028|Customer#000001028|NxmOhIN,w45aogQ1hZSvqoz0 8nrbdkaiZOe|10|20-582-119-3249|1915.53|AUTOMOBILE|ly unusual, even packages. fluffily special foxes across the furiously final asymptotes main 1029|Customer#000001029|D3TLK5s,gc|15|25-602-810-8723|6252.18|BUILDING|arefully furiously final pinto beans. daringly express deposits 1030|Customer#000001030|Xpt1BiB5h9o|8|18-759-877-1870|6359.27|HOUSEHOLD|ding to the slyly unusual accounts. even requests among the evenly 1031|Customer#000001031|dwCYOftUgV5,EwGJc|21|31-946-641-1853|2226.80|AUTOMOBILE| theodolites. even theodolites sleep slyly. special, express excuses cajole among th 1032|Customer#000001032|6yoIzDrw5zLBO|18|28-449-227-3528|1853.64|FURNITURE| sleep quickly even somas. permanently regular grouches cajole blithely furiously ironic ide 1033|Customer#000001033|WOozPuOF8UdYMwjF5|8|18-470-380-2978|81.06|BUILDING| to the quickly final packages. carefully slow accounts use blithely slowly permanent requests. un 1034|Customer#000001034|Fn5qqb64TSKuJWz4f8GpPkF,c3WY3yqjsV,GgHu|5|15-370-179-6631|7349.82|HOUSEHOLD|bout the ironic requests-- packages wake. requests haggle silent, 1035|Customer#000001035|7yTbQ665G3Bi,6BK0EmQPw,Gc7bZOPk4ncXpo|10|20-376-345-3729|7499.36|AUTOMOBILE|ound the fluffily enticing foxes detect slyly at the furiously final deposits. quickly final instru 1036|Customer#000001036|fxujgj8DOFO6oKrH|8|18-791-577-7691|1766.23|BUILDING|ins sleep. slyly express platelets cajol 1037|Customer#000001037|dwgDZPKR5ZuU3HO2sDOS7Ym0oeC8c6Xm|23|33-855-960-2989|4936.25|BUILDING|lly final pinto beans. pending instructions boost careful 1038|Customer#000001038|yQCza56pNgcF9sxDR HCed22GeEq|17|27-511-101-1611|-509.92|AUTOMOBILE|uriously express accounts. even pinto beans wake. slyly regular requests according t 1039|Customer#000001039|F602TgKjElSWrZ|10|20-871-886-9220|7618.54|FURNITURE|nding packages use. blithely regular sheaves doze blithely. fluffi 1040|Customer#000001040|vbJmdHe6U9Pl|11|21-756-109-1482|2860.71|BUILDING|old requests wake slyly! slyly special deposits cajole above the unusua 1041|Customer#000001041|189f n2lA4|11|21-314-290-3052|7993.98|MACHINERY|uctions are busily along the furiously ironic instructions. blithely thin waters cajole slyly. 1042|Customer#000001042|S1sh9gyFn21m4zkb4J95GD5|5|15-215-652-3459|9849.87|FURNITURE|ly bold dugouts! pending asymptotes are blithel 1043|Customer#000001043|HJMn12xn4bl vWC7iVuTRsErYEzlyCO|7|17-266-334-8613|5847.76|HOUSEHOLD|iously ironic deposits cajole slyly busily final account 1044|Customer#000001044|Eh2e8gLyStrLE7A|0|10-451-459-9620|7291.30|BUILDING|ly across the slyly ironic accounts. even requests 1045|Customer#000001045|clvGUnQPLbzX 23hemPp24WS1MEtS4z|20|30-120-992-2121|2942.19|HOUSEHOLD|r deposits cajole blithely along the quickly silent pattern 1046|Customer#000001046|umgqzlyUW3AYz2C39YMhIgf|10|20-890-161-8958|2311.00|BUILDING| accounts. carefully regular theodolites run fluffily carefully e 1047|Customer#000001047|h5iBRMsym,y6LLSQU2DzNftiET qZ|9|19-146-399-4251|8918.99|BUILDING|lar packages haggle theodolites. thinly express deposits 1048|Customer#000001048|Mk0ebiw9SaFBTwoib|19|29-757-642-3735|2583.91|BUILDING|s are slyly regular foxes. slyly final pinto beans wake quickly among the regular, spe 1049|Customer#000001049|bZ1OcFhHaIZ5gMiH|9|19-499-258-2851|8747.99|MACHINERY|uriously according to the furiously silent packages 1050|Customer#000001050|KgVnjN7Y4HCN5f97HEUp7kYNNTrE3 O|11|21-448-313-4374|-517.65|FURNITURE|ely ironic packages. blithely regular foxes sublate furiously. special requests boost furiously agains 1051|Customer#000001051|iHS,UFudVOOe|2|12-869-221-1428|9776.39|HOUSEHOLD|cuses boost furiously silent deposits. quickly silent requests integrate quickly bold asymptotes; slyly regular ide 1052|Customer#000001052|OcXtKS,1Hvf2D0 rPvhw4qXViYOudQ3|13|23-496-475-9040|2837.96|BUILDING|s accounts haggle against the furiously final asymptotes. ironically regular accounts boost. furiously fina 1053|Customer#000001053|wDJTteyausmZswQAFQot|16|26-400-312-6496|-473.85|MACHINERY|efully enticing pinto beans. final pack 1054|Customer#000001054|Xgj6QVy2I9FVoSiIbgLf9LIE8XpWI2RtmbGUx|21|31-915-292-9727|8844.27|HOUSEHOLD|y pending ideas. dogged dependenci 1055|Customer#000001055|Z3AggyEMPME2hqqTfbMC76O0z|7|17-802-131-7180|639.93|HOUSEHOLD|dolphins: furiously ironic pinto beans above the carefully regular foxes nag slyly across th 1056|Customer#000001056|8u1rnDOcvU109|5|15-325-285-5215|6287.12|HOUSEHOLD|leep except the foxes. packages eat enticingly along the requests. even 1057|Customer#000001057|xyV8 FbW4xS,JhkxC0dY527tzcMKxM|24|34-750-735-1314|-377.11|AUTOMOBILE|s. furiously ironic deposits against the carefully bold accounts wake carefully even deposits. fluffily even 1058|Customer#000001058|R0NIEcSVDQ4rNUcCevDrap|19|29-818-620-9637|6807.55|MACHINERY|uctions. slyly express pinto beans are furiously. bold theodolites according to the fur 1059|Customer#000001059|OHwYMiDjmgeIQXhLlNW,8LIwIEr|23|33-683-418-9460|1547.50|HOUSEHOLD| wake carefully. carefully quick excuses cajole ruthlessly among the ideas. bold, ironic braids are 1060|Customer#000001060|aWJkU6JJJOvgaKPOAJJc|8|18-290-794-6133|2840.59|HOUSEHOLD|ter the bold, regular ideas! deposits eat. daringly unusual theodolites sleep alongside of the regular, fina 1061|Customer#000001061|CqLhg io1CpQKhrVHHDhWg1Omrx1hLcpKB6h|4|14-909-417-8324|-258.77|BUILDING|ticing packages maintain doggedly carefully regular instructio 1062|Customer#000001062|3OYrGEJC1YUa9DP|22|32-207-600-8684|4709.92|HOUSEHOLD|ag. carefully regular asymptotes a 1063|Customer#000001063|yHVWD7y1Oe1P|21|31-277-349-9036|1663.28|MACHINERY|ress attainments. furiously regular excu 1064|Customer#000001064|VmFhpV9 aIqPysMHRIWZl|15|25-391-998-4106|1666.07|MACHINERY|gular accounts. thin platelets promise fluffily. carefully express accounts haggle quickly qu 1065|Customer#000001065|qGBa7X0dOMsKLuYBpShpJVwGyU9rh|22|32-605-226-2449|4663.41|HOUSEHOLD|nts. quickly quick dolphins run 1066|Customer#000001066|2Ge 0Nk29FlBs1GuBiY84sLvn38mEkAKnM|0|10-333-463-4472|949.68|MACHINERY|requests. slyly final instructions sleep. fluffily even packages cajole pending, final 1067|Customer#000001067|g25CH,fhra|23|33-764-123-9568|9153.84|FURNITURE|ackages are furiously carefully even dependencies. idly final 1068|Customer#000001068|ElWdGnnKpmo0sA1Au teWwomSVgG,me|18|28-485-984-7299|737.40|AUTOMOBILE|nto beans. dinos sleep fluffily carefully regular sentiments. final, special packages wake blithely. ironic 1069|Customer#000001069|PdWrPGSArhnqWQ km65e|21|31-927-711-6278|5465.29|BUILDING|. ironic excuses after the special hockey pl 1070|Customer#000001070|m0sYmeYs5wLydSS qw542Et32|15|25-894-843-9171|3160.23|FURNITURE| carefully across the express foxes. carefully special accounts ca 1071|Customer#000001071|PgCAYL2LEwE7v7Pk4dYpRe Nn7MN8wVzYbA2qtj|16|26-350-231-6183|4033.56|MACHINERY|ar accounts. quickly regular packages sleep bold ideas. slyly pending asymptotes a 1072|Customer#000001072|HpCr1tM88WoELSld708ByJ|4|14-432-882-6163|7979.48|HOUSEHOLD|ic pinto beans are blithely across the grouches. furiously pending platelets sleep pe 1073|Customer#000001073|KEyFI2gYMZrSVbMMMIf|10|20-774-197-6595|8217.23|BUILDING|fter the blithely regular pinto beans. express asymptotes sleep special pa 1074|Customer#000001074|nG,eR,gjPr|10|20-176-839-1649|122.67|BUILDING|ly final courts haggle quickly boldly express excuses. dependencies eat. slyly even requests boost blithely 1075|Customer#000001075|hTIc2AUg pqhYh2W0yMUTQtrZV1KUutysIb6,nxb|21|31-724-234-4181|2714.50|BUILDING|regular patterns. unusual platelets try to are unusual theod 1076|Customer#000001076|C1gf0FyiU H88P0cpv4UOcdgaPRpVA|10|20-405-710-1902|3509.35|AUTOMOBILE|ainst the silent, silent ideas. sly theodolites use carefully express accounts. regular foxes boost carefully agai 1077|Customer#000001077|sjk1DTHWVMX53kG8AbTtTh1EcMvWeDO8gFDdpQOK|21|31-367-294-4048|8581.78|AUTOMOBILE|lithely regular deposits. carefully pending deposits sleep. quickl 1078|Customer#000001078|ZjRzAz8QbEeIkJxrUI,b|19|29-729-692-6790|70.93|MACHINERY|onic requests wake slyly furiously final attainments. ironic, even accounts cajo 1079|Customer#000001079|cOyd7wsHIQq2LNN|19|29-699-930-2250|2135.91|AUTOMOBILE|ter the express foxes nod slyly excuses. slyly speci 1080|Customer#000001080|Yux,gs14NpneiZEy9Rz|12|22-806-885-5347|3267.19|MACHINERY|ng dolphins cajole across the carefu 1081|Customer#000001081|eGGRjZex7YANvD1jfnPMcBK2JbM|12|22-866-942-1021|8647.42|MACHINERY|pendencies haggle after the quickly special instructions. furio 1082|Customer#000001082|vMX52A1zqDbGNzjfSzgsxSVU0GU6iFmrgiUE|5|15-646-384-2302|3247.92|BUILDING|ly. blithely final packages wake silent ideas. express, special theodo 1083|Customer#000001083|tnrpYmWGxwyaFmJy2Oq0Z|7|17-159-499-3318|3847.29|FURNITURE|luffily. slyly unusual accounts cajole furiously against the ironic asymptotes. slyly reg 1084|Customer#000001084|E091r836A8TPqn5|2|12-378-899-7136|1416.75|FURNITURE|nstructions. fluffily pending pinto beans affix slyly; carefully pending requ 1085|Customer#000001085|pZgtHRGIkUVwiEJLWZXs3KUNi6wLnQzJU1|21|31-831-702-3157|5275.88|BUILDING|he carefully regular courts use special excuses. ironic deposits along the blithely even sauternes nag slyly 1086|Customer#000001086|ECMZrONto2nI2TBv,k|8|18-399-482-6815|9726.83|FURNITURE|ymptotes cajole enticingly furiously silent ideas. furiously pending packages are al 1087|Customer#000001087|ETOH68urIxK839xmKEmfkjc|21|31-334-391-6403|5878.21|MACHINERY|s haggle above the slyly express requests. quickly regular packages after the quickly silent accoun 1088|Customer#000001088|YjXQtOJoM0nhClEy0,WFdNxvJ1g6xpn kL2ommEv|22|32-324-225-2635|2098.62|BUILDING|ly special ideas. slyly unusual requests haggle 1089|Customer#000001089|OO77 pLjaOe7bam1WnH9gtcZNCUlUPI|18|28-164-765-7462|3429.95|BUILDING|o beans affix carefully regular accounts. quickly even ideas sleep. pinto beans haggle fluffy courts. slyly regu 1090|Customer#000001090|P2JDHFVxjU|15|25-711-934-6343|5212.43|AUTOMOBILE|ffily even packages wake quickly 1091|Customer#000001091|4ye7wJ3gU92RZCpwTtDi8Ws,|17|27-336-955-4882|-710.53|BUILDING|y. carefully ironic excuses sleep quickly fluffily even requests. fi 1092|Customer#000001092|,oAq2L60hjb8|15|25-766-175-4580|2004.15|HOUSEHOLD|carefully silent somas can wake carefully aft 1093|Customer#000001093|LO,9qCPIjSXriBqQsAOXLrQKedQ8UO6gb|24|34-931-911-6156|-273.96|HOUSEHOLD|p furiously carefully bold packages. regular escapades breach. blithely unusual ideas integrate across t 1094|Customer#000001094|OFz0eedTmPmXk2 3XM9v9Mcp13NVC0PK|2|12-234-721-9871|2544.49|MACHINERY|tes serve blithely quickly pending foxes. express, quick accounts 1095|Customer#000001095|JtyQvLlCI ZPYQ6ygv,5q|9|19-881-259-2391|6221.26|MACHINERY|foxes. ironic, daring requests sleep regularly across the blithely 1096|Customer#000001096|ldbo6AfnCRjFW8rZnvG6UxbX6o7ISAJRDD7|4|14-368-827-9896|3687.37|FURNITURE|lyly even asymptotes cajole furiously. regular, ironic theodolite 1097|Customer#000001097|a wMc0lQutcHs6cRomoMCGjvM0MwEk4uyrxKI3|6|16-604-758-5574|8651.87|MACHINERY|p carefully. carefully special excuses haggle carefully about th 1098|Customer#000001098|XVJb1HxQeLu9x|22|32-206-732-5183|1009.22|FURNITURE|evenly unusual deposits. slyly even ideas according 1099|Customer#000001099|2ZiU64au LN0 GUxY8|1|11-128-186-5241|8990.07|AUTOMOBILE|ckages: blithely ironic theodolites cajole furiously. f 1100|Customer#000001100|PGXj,,vjAfMNLzd|12|22-880-206-7392|9189.75|BUILDING|ideas. furiously final sheaves integrate. pinto beans haggle slyly according to the furiously ironi 1101|Customer#000001101|h,UOEyoi1ZG4|3|13-528-469-6051|-842.72|MACHINERY|o beans; quickly express accounts slee 1102|Customer#000001102|F9fxZhJJhaR0P4Rgd7SE2PA58x|24|34-103-353-4822|2369.01|HOUSEHOLD|elets. regular requests sleep quickly. express ideas haggle. bold, regular ideas haggle. quickly regular accoun 1103|Customer#000001103|kbYrf d uR|16|26-307-423-8860|4878.10|AUTOMOBILE|n accounts cajole across the even pinto beans. quickly express pat 1104|Customer#000001104|,t,d8FlnmiECPa|8|18-644-507-8095|1230.47|AUTOMOBILE|ages haggle. slyly ironic foxes are idly among the furiously final pearls. slyly unusual reques 1105|Customer#000001105|cZhhOUzv6,Vbaa2bFT|22|32-885-298-6750|9491.46|FURNITURE|y final packages. furiously ironic packages was. fluffily ironic instructions integrate 1106|Customer#000001106|WZEExIU9g2smcowcinj|21|31-214-739-2409|9977.62|HOUSEHOLD|requests nag. fluffily regular packages haggle q 1107|Customer#000001107|yQBP1SLK2uzN4dzgaQ|1|11-720-869-9052|7961.62|AUTOMOBILE| along the final deposits. carefully express ideas wake? quickly regular instructions are furious 1108|Customer#000001108|9sPt6a66R0eCRVYh9QrF8zjxNWFFk8KU|7|17-408-450-8891|4997.35|BUILDING|rding to the final instructions. carefully final accounts wake along the carefully careful pinto beans. re 1109|Customer#000001109|BJCfTYEV9eCDraeyO3v|22|32-194-697-1794|3387.22|FURNITURE|r accounts. bold, final pinto beans wake carefully even Tiresias. quickly busy frays above the blithely ironic de 1110|Customer#000001110|BRnTy8RZ,1oHOg9ly8SsJLIyiuvhv|10|20-777-225-9349|2041.65|HOUSEHOLD|usual platelets along the quickly regul 1111|Customer#000001111|gavpg6eW5lEML|6|16-824-312-3537|2892.21|MACHINERY|s are slyly quiet requests. darin 1112|Customer#000001112|wFf 0nSvdJyk2GqRsqJrcr9 UPr0C3OT5zT|20|30-401-424-6458|9314.59|MACHINERY|fily quickly unusual theodolite 1113|Customer#000001113|jLtKZ0bRJyYL1k|12|22-412-216-1933|7392.30|HOUSEHOLD|ages among the furiously pending packages detect across the blithely unusual accounts. furiously ironic requests sh 1114|Customer#000001114|f7 he8eByBFy6z7vcOajC1gaUKqmRN|14|24-630-988-3843|6446.83|BUILDING|ularly ironic platelets. pinto beans along the slyly express packages wake unusual packa 1115|Customer#000001115|Elvb2a3FinAzxw |5|15-356-145-6356|-178.52|BUILDING|ending instructions thrash blithel 1116|Customer#000001116|aWuLgbu,8HZMbkI|14|24-116-214-4051|592.60|AUTOMOBILE|tes-- final, regular excuses sleep. sly 1117|Customer#000001117|80NfzBRWj5tUUaRdnsFE7Eg|23|33-461-439-5684|2829.07|FURNITURE| ironic deposits need to haggle furiously. furiously bold deposits use among the carefully ironic 1118|Customer#000001118|QHg,DNvEVXaYoCdrywazjAJ|11|21-583-715-8627|4130.18|HOUSEHOLD|y regular requests above the blithely ironic accounts use slyly bold packages: regular pinto beans eat carefully spe 1119|Customer#000001119|ER5vABifV766q5f0FN7l2eN7MIg2lO|20|30-789-716-6850|3971.65|AUTOMOBILE| pinto beans maintain slyly even instructions. regular acc 1120|Customer#000001120|UAG90slCmJS7JOP AhlV12tYD3yUiyB1p2hxZ|2|12-938-579-7156|1543.64|AUTOMOBILE|r theodolites boost. slyly final pinto beans sleep blithely unusual accounts. fluffily even multipliers 1121|Customer#000001121|o2uc3AHYz,m 3vYg8YxBwI0XuG|20|30-197-936-4724|3942.11|MACHINERY|usly? final theodolites are carefully 1122|Customer#000001122|9lxNEW0Rei4DFaT4vX,T551AwBzrZoOXsRNOm|0|10-257-957-3327|45.21|BUILDING|egular, regular instructions are slyly regular requests. deposits despite the regular, pendi 1123|Customer#000001123|pO80QGjK7S0Kmoh46dViD K4OSEVDyiJ53CN|16|26-983-192-5480|9786.36|MACHINERY|s carefully ironic packages. accounts boost boldly fluffily even gifts. slyly final fo 1124|Customer#000001124|EQNw9dNy63,|1|11-709-582-2006|5512.73|BUILDING|ctions wake. packages haggle furiously. express 1125|Customer#000001125|DrHkeaX6wshtuZOI2nLrME|3|13-807-542-3923|8427.55|MACHINERY|counts according to the carefully silent grouches haggl 1126|Customer#000001126|8J bzLWboPqySAWPgHrl4IK4roBvb|8|18-898-994-6389|3905.97|AUTOMOBILE|se carefully asymptotes. unusual accounts use slyly deposits; slyly regular pi 1127|Customer#000001127|nq1w3VhKie4I3ZquEIZuz1 5CWn|10|20-830-875-6204|8631.35|AUTOMOBILE|endencies. express instructions wake about th 1128|Customer#000001128|72XUL0qb4,NLmfyrtzyJlR0eP|0|10-392-200-8982|8123.99|BUILDING|odolites according to the regular courts detect quickly furiously pending foxes? unusual theodolites use p 1129|Customer#000001129|OMEqYv,hhyBAObDjIkoPL03BvuSRw02AuDPVoe|8|18-313-585-9420|6020.02|HOUSEHOLD|pades affix realms. pending courts haggle slowly fluffily final requests. quickly silent deposits are. iro 1130|Customer#000001130|60zzrBpFXjvHzyv0WObH3h8LhYbOaRID58e|22|32-503-721-8203|9519.36|HOUSEHOLD|s requests nag silently carefully special warhorses. special accounts hinder slyly. fluffily enticing 1131|Customer#000001131|KVAvB1lwuN qHWDDPNckenmRGULDFduxYRSBXv|20|30-644-540-9044|6019.10|MACHINERY|er the carefully dogged courts m 1132|Customer#000001132|6dcMOh60XVGcGYyEP|22|32-953-419-6880|4962.12|AUTOMOBILE|ges. final, special requests nag carefully carefully bold deposits. ironic requests boost slyly through th 1133|Customer#000001133|FfA0o cMP02Ylzxtmbq8DCOq|14|24-858-762-2348|5335.36|MACHINERY|g to the pending, ironic pinto beans. furiously blithe packages are fina 1134|Customer#000001134|79TYt94ty a|9|19-832-924-7391|8458.26|HOUSEHOLD|riously across the bold instructions. quickly 1135|Customer#000001135|cONv9cxslXOefPzhUQbGnMeRNKL1x,m2zlVOj|11|21-517-852-3282|3061.78|FURNITURE|regular frays about the bold, regular requests use quickly even pin 1136|Customer#000001136|GHCEiSK0TKsOncuJT3,2zSvlZW4Pz|24|34-440-798-1100|-723.49|FURNITURE|ular pinto beans. slyly special deposits according to the slyly ironic requests maintain quickly 1137|Customer#000001137|LJ3J3i0BlPLrhKi6VabXxNrtpLAGH|16|26-598-565-1269|4210.15|AUTOMOBILE|usly quickly unusual attainments. stealthily unusual requests cajole ironic reques 1138|Customer#000001138|8 9P,dIGWnrrDiVs0S|22|32-236-817-2959|6035.44|BUILDING| instructions cajole thinly ironic requests. regular packages affix. ironic, final pinto beans ac 1139|Customer#000001139|UDGG69rYgUGayNk 9vFytd5q3nZdeRZQNSfL6|22|32-182-662-9475|4604.83|BUILDING|y pending pinto beans haggle blit 1140|Customer#000001140|leG5nToZpjmWNeaOsVv|20|30-331-754-7359|6319.21|AUTOMOBILE| pinto beans. blithely regular packages sleep carefully blithely ironic requests. requests eat blithely aga 1141|Customer#000001141|A6uzuXpgRPp19ek8K8zd5O|22|32-330-618-9020|0.97|MACHINERY| accounts. furiously pending deposits cajole. c 1142|Customer#000001142|b7ytiiX7E9|16|26-191-682-8920|3273.08|AUTOMOBILE|doze slyly. furiously pending deposits cajole fluffily carefully pending packages. boldly regular 1143|Customer#000001143|9tfTGdYHyZXtXbbeboPIXwCT|4|14-568-471-9747|8655.98|AUTOMOBILE|e carefully final packages integrate against the furiously express platelets. ironic ideas wake above the e 1144|Customer#000001144|DGLUWG9evYLNbYhOXVzqZ LdfIMVfBjDf|1|11-336-453-4489|4189.04|BUILDING| ideas. even, regular excuses after the ironic requests cajole blithe 1145|Customer#000001145|6R rPD6SDQPpFuYxxwh,Dv1PeusmP,C6cNcI|2|12-270-756-2968|3249.25|HOUSEHOLD|e. asymptotes sleep fluffily quiet requests. even theodolites among the fluffily regular pinto 1146|Customer#000001146|DRBYvF0iBGsDC3iPNFsPHq3FkU,jCK8LJPX4W|12|22-720-237-1751|4204.36|FURNITURE|final, pending asymptotes. regular requests was 1147|Customer#000001147|AVjlczwVwL CT jO3sgWn|15|25-754-809-7107|7734.64|HOUSEHOLD|eposits. quickly express accounts are idly. slyly final platelets wak 1148|Customer#000001148|7PslyqtS1K2Pabjht 4qgaZ1BbSNFfz6QiK4K|19|29-393-445-2761|7129.84|AUTOMOBILE|c, even deposits. accounts do use. regular accounts haggle blithely special, express courts. blithely 1149|Customer#000001149|5JOAwCy8MD70TUZJDyxgEBMe|3|13-254-242-3889|6287.79|MACHINERY|ress requests haggle carefully across the fluffily regula 1150|Customer#000001150|fUJqzdkQg1|21|31-236-665-8430|-117.31|MACHINERY|usly final dolphins. fluffily bold platelets sleep. slyly unusual attainments lo 1151|Customer#000001151|ratQBQ4rYv TfhWfHe|7|17-948-135-2667|6354.89|BUILDING|l requests. furiously bold orbits after the furiously ironic excuses sleep 1152|Customer#000001152|QRmFl9ZkoBDQ7|12|22-471-341-5516|5680.15|HOUSEHOLD|oost along the quiet, bold foxes. ironic dinos nag fluffily final pinto beans. blithely regular deposit 1153|Customer#000001153|SYG3KMj1fMh7GwvIZ,pY7mGLR1NT6EmNjE|3|13-319-420-5160|6244.03|HOUSEHOLD|s. even packages use fluffily always express packages. regular, even asymptotes about the furiou 1154|Customer#000001154|7RqtNwcSPbaUKaC|19|29-797-132-6916|1498.46|BUILDING|thely. furiously regular accounts above the ironic platelets wake slyly blithely bold pint 1155|Customer#000001155|kEDBn1IQWyHyYjgGGs6FiXfm3|8|18-864-953-3058|3510.25|MACHINERY|ages? fluffily even accounts shall have to boost furiously alongside of the furiously pendin 1156|Customer#000001156|3ShFbt9dTbLOG4lUBvc1AZp0Tam0BNjYS qwTZ|14|24-637-724-1410|1799.67|HOUSEHOLD|ns. carefully regular foxes are quickly. furiously careful accounts accord 1157|Customer#000001157|3rchTZwilGpffMz1MfpnkFfWBtOIdgmvvS1E7sJj|20|30-741-794-9826|6013.09|HOUSEHOLD|equests. deposits cajole quickly slyly spe 1158|Customer#000001158|btAl2dQdvNV9cEzTwVRloTb08sLYKDopV2cK,p|10|20-487-747-8857|3081.79|MACHINERY| theodolites use stealthy asymptotes. frets integrate even instructions. car 1159|Customer#000001159|IAnWq4YFKs7|2|12-269-807-3861|5553.75|HOUSEHOLD|ages sleep fluffily. packages after the carefully express packages nag slyl 1160|Customer#000001160|v65g1aRCGA76ZHySoOBffL31n4vJ0nm,tK,UEA|24|34-103-942-4634|4976.24|AUTOMOBILE| pending, special packages against the blithely unusual packages eat quic 1161|Customer#000001161| QD7s2P6QpCC6g9t2aVzKg7y|19|29-213-663-3342|591.31|MACHINERY|ly alongside of the quickly blithe ideas. quickly ironic accounts haggle regul 1162|Customer#000001162|b5N12h9D6yJemoVx6OQf0uL|2|12-887-115-9986|3139.71|AUTOMOBILE|refully furious packages. furiously ironic ideas against the carefull 1163|Customer#000001163|54fBdElRYOjEH8S|2|12-204-803-1483|90.22|BUILDING|inments. carefully regular instructions haggle carefully slow packages. slyly even packages kindle blithely special 1164|Customer#000001164|XWfNRnO2S5KAW0VodNwaBDixCEtv nKzt2LVFiwm|0|10-828-178-5049|7341.35|HOUSEHOLD| ideas use. unusual packages sleep 1165|Customer#000001165|h7KTXGSqsn0|9|19-766-409-6769|8177.33|MACHINERY|jole slyly beside the quickly final accounts. silent, even requests are stealthily ironic, re 1166|Customer#000001166|W4FAGNPKcJFebzldtNp8SehhH3|17|27-869-223-7506|507.26|MACHINERY| before the platelets! carefully bold ideas lose carefully 1167|Customer#000001167|gNYGOcGkJu3ooSmsCh|19|29-721-479-1548|9510.87|FURNITURE|lly regular ideas grow furiously regular accounts. regular, special requests sleep blithely. slyly bold pla 1168|Customer#000001168|gmAnpwPPg0LX4|17|27-608-883-2632|6194.65|FURNITURE|ses run according to the regular instructions. slyly regular foxes around the furiously ironic theodolites use fl 1169|Customer#000001169|04YQNIYyRRFxUnJsTP36da|4|14-975-169-9356|7503.30|MACHINERY|into beans doubt about the slyly ironic multipliers. carefully regular requests breach theodolites. special packages 1170|Customer#000001170|BNhssVcV36vshEHUAc aPFJ|8|18-670-628-8499|2070.73|HOUSEHOLD|ronic instructions. express pinto beans poach blithely. quickly ironic accounts n 1171|Customer#000001171|GatOC LsLU9MkgyaNMYH|8|18-457-394-2863|7658.97|HOUSEHOLD|c dolphins. accounts are slyl 1172|Customer#000001172|r dreL8m8cRaiIz IqZ83oMo,AVxe2PbsHQzK|14|24-249-588-1969|420.97|FURNITURE|express asymptotes haggle furiously. fluffily special deposits haggle quietly even, special tithes. ironic foxes alo 1173|Customer#000001173|6Abj72jR5Z0GYQMZKBmiQxW|18|28-409-365-6392|182.59|FURNITURE| ironic accounts above the ironic excuses haggle fluffily furiously regular packages-- slyly regular gi 1174|Customer#000001174| b9zecNS,J97qi7Qk5|4|14-131-930-7154|8798.96|AUTOMOBILE| ironic packages. furiously regular excuses sleep about the fluffily unusual pinto beans? regular foxes kind 1175|Customer#000001175|olj7nLYgBZ526MNBg9CV7w LYo6F1uD,Hm54|4|14-756-259-6379|9207.16|FURNITURE| haggle pending requests. carefully regular ideas nag. ruthlessly final packages a 1176|Customer#000001176|V0xc0tXNMmObuUJ0rGARp6YIw4fo84CD|10|20-141-903-5936|5827.59|BUILDING|ven accounts boost after the accounts. slyly silent accounts use fluffily amon 1177|Customer#000001177|hZPNQ8a9QRM ,SYdTdoW9hw|14|24-200-701-8606|9281.72|FURNITURE|ng the quickly bold theodolites cajole carefully around the deposits. furi 1178|Customer#000001178|W,03Nl2iWQ94xYyCo3R8CTMNFhu|9|19-717-739-3103|4966.58|HOUSEHOLD| even requests cajole furiously after the quickly ironic accounts. even re 1179|Customer#000001179|JLtS3n1xDqnNFS5MZc5uZHOjDctAJEl|19|29-311-833-9211|3336.25|AUTOMOBILE|ress, special accounts sleep slyly about the carefully express packages. f 1180|Customer#000001180|jI4QtviiCs0,LOgUPH4aONMnoNt|13|23-188-767-6645|3367.36|BUILDING|uests poach carefully carefully final deposits: ironic, regular deposits are slyly busy excuses. regular 1181|Customer#000001181|ZFFYipzTg0vSjOhPbcBUgPK9se|10|20-330-305-8843|9180.50|AUTOMOBILE|gle about the busily special theodolites. furiously ironic deposits haggle beside the furiously special accounts. 1182|Customer#000001182|pLrF7F1,uoyGaU|6|16-229-473-7194|8814.39|AUTOMOBILE|jole carefully. furiously final pinto beans detect. f 1183|Customer#000001183|qdIqRUfpmvtWo0NGsyi4qyjkwzlImP9,NrSC|1|11-968-244-9275|4455.76|BUILDING|arefully regular dependencies. quick 1184|Customer#000001184|M0dd3R30k0YjIr4GVeo|11|21-661-875-1923|9032.89|BUILDING| excuses nag carefully even accounts. unusual platelets detect carefully bold acco 1185|Customer#000001185|z,dN83fETWpkJkoR|14|24-860-751-6688|2913.52|BUILDING|ndencies against the carefully even accounts cajole carefully quickly regular packages. even fox 1186|Customer#000001186|cj5EeLbJJ6MPdynzposq,Apbj9 2Jm|23|33-500-965-3385|4466.30|BUILDING|ding realms cajole after the even foxes 1187|Customer#000001187|W1GdUKr3vQMVAZIU|10|20-543-260-5157|-932.96|AUTOMOBILE| blithely unusual theodolites detect doggedly. bold dolphins was blithely. pinto beans use carefully at the furiou 1188|Customer#000001188|PtwoF3jNQ9r6 GbPIelt GvbNBuDH|15|25-108-989-8154|3698.86|MACHINERY|ts. quickly unusual ideas affix aft 1189|Customer#000001189|rbnZCbJSTE3qWLl|10|20-746-804-1553|3798.28|HOUSEHOLD|enticingly express platelets wake. regular requests boost even, regular instructions. express dependencies a 1190|Customer#000001190|JwzW9OtxFRXDnVo5hXl8 2A5VxH12|15|25-538-604-9042|2743.63|MACHINERY| regular deposits according to the pending packages wake blithely among the silent inst 1191|Customer#000001191|K9J7dhIXDB2kgubtIVdRC6RP0aF,GQXin|19|29-587-244-2901|9088.54|AUTOMOBILE|ts wake. waters detect fluffily carefully regu 1192|Customer#000001192|8DbtM3KloBZ4OO1dRrF99|20|30-904-936-4914|3231.33|BUILDING|efully final packages use. slyly pend 1193|Customer#000001193|gdKqrIp,yaMaQSFerrGGzc6Kpy|8|18-524-487-2547|-17.10|AUTOMOBILE|accounts sleep carefully. regular accounts wake slyly. carefully regular requests along the quickly pend 1194|Customer#000001194|NzWKbiPK1oFd7PNz|21|31-155-275-3981|7582.29|FURNITURE|lve quickly according to the carefully regu 1195|Customer#000001195|71mmXvaWKf|4|14-355-801-7486|9621.49|FURNITURE|l, regular gifts should have to x-ray blithely fluffily ironic 1196|Customer#000001196|S3sw3q SDWVuUoEFvwv9M|20|30-615-967-7758|6378.67|BUILDING| carefully alongside of the blithely even theodolites. carefully ironic instructions wake after the spec 1197|Customer#000001197|9A1LTDf0KbR|0|10-254-891-7835|9261.05|FURNITURE|ording to the slyly ironic accounts. carefully final instructions haggle. special, unusual foxes haggle enticing 1198|Customer#000001198|r0liwpMwaIBQ9 zQjojGylXkJuKUL|18|28-278-515-1034|9593.35|AUTOMOBILE|intain fluffily ironic instructions. regular requests nag fluffily carefully unu 1199|Customer#000001199|sQJZJRAgYrZY0gPo9fJp6KAbY|16|26-367-212-1737|6503.35|AUTOMOBILE|es. quickly slow foxes boost 1200|Customer#000001200|2PFysvL4pk80l|22|32-890-210-4199|3765.05|HOUSEHOLD|nent frets. blithely bold pearls thrash across the r 1201|Customer#000001201|LfCSVKWozyWOGDW02g9UX,XgH5YU2o5ql1zBrN|10|20-825-400-1187|5165.39|BUILDING|lyly pending packages. special requests sleep-- platelets use blithely after the instructions. sometimes even id 1202|Customer#000001202|xThQDiKdG,0sU IduCCPAgHJfx1PDJwtUQvfU|0|10-788-256-6117|702.73|BUILDING|accounts. fluffily regular requests are. packages among the final deposits haggle carefully arou 1203|Customer#000001203|9pTq4gggfKoSqQetn0yJR|16|26-370-660-6154|5787.69|MACHINERY|osits nag furiously final accounts. silent pack 1204|Customer#000001204|QxpCVhq2x0PwW,zgZ AEuFkgb50ryGM|20|30-117-472-8751|9777.19|HOUSEHOLD|ily final instructions. pending foxes detect doggedly accor 1205|Customer#000001205|1ALD7GN4Iw0Vl5toeM|8|18-185-307-2678|5390.34|HOUSEHOLD|ptotes. silent deposits above the bold warhorses boost 1206|Customer#000001206|dxzjW0gykcG2kJ gN8hZV02q6U5T6uVNfP|20|30-716-117-6066|8437.76|HOUSEHOLD|ng the ironic accounts. regular requests across the quickly bold deposits wake carefully across the 1207|Customer#000001207|tDZe2FlIxGjrf9x,n6N1tu0DbWyUkSSu|3|13-572-474-7362|-556.05|AUTOMOBILE|uriously by the slyly regular packages. fluffily final deposits across the quickly express epitaphs us 1208|Customer#000001208|M uLSFG6IrQkKQxrH5vCbPjglIpB3JC|5|15-789-973-6601|2426.52|BUILDING| blithely bold dependencies detect slyly. carefully silent platelets haggle along the pinto beans 1209|Customer#000001209|PW00geNpQOiug6dftXfBkpwdAfsmRYsve,b44uR8|4|14-664-771-9006|3551.21|HOUSEHOLD|unts. regular dolphins integrate slyly. regular, pending accounts sleep b 1210|Customer#000001210|bUTLW1KIHzzQkuOEwUMwEGCQfTQM7aBmUM0|16|26-202-315-9048|8137.66|AUTOMOBILE|luffily ironic accounts haggle about the regular theod 1211|Customer#000001211|HCACb3Al89h6NqHUJ8qIjhfGFyA4S0c2|18|28-280-785-7324|4723.37|HOUSEHOLD|posits. packages affix carefully after the carefully 1212|Customer#000001212|kjiVLfadsq6sU3A6MYwySu XZnWzgkiWSa9v3K 6|22|32-462-274-7707|7736.03|HOUSEHOLD|e quickly unusual pinto beans. packages need to sleep furiously. regular asymptotes are furiously. final packages 1213|Customer#000001213|4ATLYSTcqLfgAJLxL7U|7|17-548-151-7684|8555.12|HOUSEHOLD|ong the deposits. quickly express deposit 1214|Customer#000001214|EATpN6m rGunAAkWFNSpsqy|4|14-281-851-2904|2935.31|BUILDING|carefully across the carefully ironic asympto 1215|Customer#000001215|oAvLu8VcRg9FNS9sNmoqU9|16|26-405-743-5405|7795.87|MACHINERY|special packages against the slyly final pinto beans wake slyly furiously final 1216|Customer#000001216|CSf1BbFhJhucmvftOwYLQACMEqgXj|3|13-673-633-4561|2155.06|AUTOMOBILE|eposits. slyly ironic dependencies haggle quickly. slyly close orbits above 1217|Customer#000001217|ddk4YC7lmTM,Z3LbX|12|22-191-580-2887|6019.32|FURNITURE|ar instructions. furiously pending 1218|Customer#000001218|JYNvUpFG0dDy7aJNhl2zLyIxUGqZZ35ncUe|22|32-299-871-1751|8801.73|AUTOMOBILE|packages hang against the unusual, unusual accounts 1219|Customer#000001219|kP1xK5be a8tW5JRSq0nwJWgKbO|4|14-364-661-8744|774.23|FURNITURE| the quickly even packages wake fur 1220|Customer#000001220|tbyect2HMX47TzsKy5 ho5|18|28-379-869-1009|8429.33|FURNITURE|sual multipliers. furiously unusual theodolites are. 1221|Customer#000001221|4mmeiymVdRmz|22|32-185-876-3586|816.50|FURNITURE|express, bold pinto beans. packages would detect alongside of the quickly bold 1222|Customer#000001222|hn6SzlP4Dq8F89iOXH0tjIgsz0uBCiBM|11|21-709-519-4959|3883.18|AUTOMOBILE|riously special theodolites nag slyly. slyly special ideas sublate quickly across the slyly un 1223|Customer#000001223|,I0bRfCGE5ssaX4V3|11|21-659-745-8411|-413.03|BUILDING|oxes. bold foxes detect always furiously special platelets. fluffily bold depende 1224|Customer#000001224|PWOwgZKsBoFJQ7py4HJpdJoSO2,|8|18-794-312-9970|8124.15|HOUSEHOLD| furiously regular accounts. slyly regular 1225|Customer#000001225|CgaNokxe s|11|21-839-103-4411|8634.92|MACHINERY|elets. bold packages use blithely special foxes. quick 1226|Customer#000001226|HKR1zog fXW|0|10-251-221-9440|2135.92|FURNITURE|ns. furiously pending packages hinder special accounts. sl 1227|Customer#000001227|GiT5IrOJ1zJTZErbFt1Jy6Gj|23|33-468-642-3107|3335.72|FURNITURE|fily atop the bold, unusual theo 1228|Customer#000001228|fV,ZM6pj1nivvbnfseVaWRkB0UYwKgvv|12|22-778-955-6105|5392.30|AUTOMOBILE|s according to the carefully final ideas ha 1229|Customer#000001229|csvrfGKxtX|9|19-313-452-6076|8328.12|FURNITURE| instructions sleep. carefully regular accounts use furiously. ironic, even foxes wake: busily even deposits caj 1230|Customer#000001230|Pr7yxcRne6NiloD1oR,d28rwVFRnOoTWeYq9|23|33-786-129-3407|4787.85|MACHINERY|ackages cajole furiously quickly pending packages. ironic foxes 1231|Customer#000001231|qJWtxdKmKWcR5XgMDn|9|19-316-348-3289|2326.68|AUTOMOBILE|uickly regular foxes are after t 1232|Customer#000001232|yYXdTto04oLlk04SM|18|28-518-320-7417|8482.51|MACHINERY|. even deposits lose above the even, regular 1233|Customer#000001233|KdmXav1IIIo|15|25-522-912-6255|3649.49|MACHINERY|ymptotes according to the ironic deposits use around the reg 1234|Customer#000001234|B3OhbH0MRJE,F0Lc7Jq0Ttv3|1|11-742-434-6436|-982.32|FURNITURE|y ironic instructions are quickly about the slyly silent pinto beans. quickly final dependenci 1235|Customer#000001235|q 1E JKZqhvUzj48|24|34-549-333-8551|-982.05|BUILDING|ckly. furiously quick dependencie 1236|Customer#000001236|pTEEPYlnQBzi558CN7LJ5UTdvO|11|21-699-526-9355|3600.95|MACHINERY|, pending excuses wake slyly pending accounts. asymptotes wake fluffily against the ironic, bold pack 1237|Customer#000001237| CQEeqR ,cVU7Bby|20|30-415-666-7691|8156.62|BUILDING|the bold accounts wake blithely across the deposits 1238|Customer#000001238|HGCJI27,RIIQcS20,DcJbMQuUmN3Vhzdm|6|16-302-171-7578|4299.22|BUILDING|ly special requests. unusual, special asymptotes according to the blithely express pinto beans wake en 1239|Customer#000001239|,K7wNII9jhC ,|20|30-786-518-4678|6936.72|FURNITURE| are alongside of the requests. s 1240|Customer#000001240|XbvhyAXRkuujtESRmxeD9eQpYSkiCa|4|14-650-555-5310|5439.44|FURNITURE|ans above the slyly regular ideas cajole furiously across the regular, regular decoys. furiously final asymptotes s 1241|Customer#000001241|74mW8ipfvoVPR3PS|3|13-902-876-1609|8654.56|HOUSEHOLD|ructions affix against the evenly ironic packages. slyly regular accounts run carefully. accounts accor 1242|Customer#000001242|mA8bUqB6WqNEe2nsQXlaHqMqaACj|2|12-521-364-1211|2276.15|BUILDING| regular ideas cajole! blithely express excuses b 1243|Customer#000001243|g,qSvyYkgjDcCL cxx5qy8hAwhomRq9cYJRvXZ6|14|24-445-165-9851|6271.38|MACHINERY|s, regular packages through the carefully ruthless theodolites promise quickly blithely final pinto be 1244|Customer#000001244|I3DrbiKwP3dxs1jF0iDwXh|5|15-881-433-2257|-942.80|BUILDING|old deposits alongside of the packages are 1245|Customer#000001245| xLnSgzY70qTKPF753|4|14-500-764-3702|3196.66|FURNITURE|haggle slyly at the carefully pending excuses. slyly pending theodolites use re 1246|Customer#000001246|acguUq5BISzqjHB7Bt|4|14-882-141-9354|260.71|BUILDING|haggle furiously. blithely regular patterns sleep quickly slyly even 1247|Customer#000001247|q5,Og3ezW3jSUtbwK 6qLJPqPwCwdL|0|10-386-173-3167|1696.95|MACHINERY|s against the quickly unusual ideas are blithely packages. accounts sleep quickly. regular 1248|Customer#000001248|f0X68bItSl|8|18-692-669-1536|6539.15|AUTOMOBILE|theodolites for the unusual deposits cajole fluffily final patterns. caref 1249|Customer#000001249|x9ukZnNiUM5pBPXyE3epagewVQBPZzxGYD6sMH|7|17-866-269-1165|448.49|BUILDING| the ironic packages sleep carefully express theodolites. even, final deposits across the even deposits nag after t 1250|Customer#000001250|LBPszo9EVA88sbbdYl7E,MVm7UvoBjmjr|9|19-509-608-4350|780.05|AUTOMOBILE|und the fluffily bold requests. silent, final theodolites solve furiou 1251|Customer#000001251|4AjU4c4D4AMLwQx,lAJGwBIgmT7oSZwYUv0es3J|24|34-741-256-6399|7267.76|MACHINERY|ckly regular accounts affix slyly carefully unusual 1252|Customer#000001252|u2OUDBxaayX4WhrftcM,|12|22-604-782-7617|3279.84|MACHINERY|eans might impress about the bold requests. bl 1253|Customer#000001253|2rEfA2LR6LkUXjoMxIsv58YSHPMjlqr1YXhHSX|1|11-900-587-2067|1222.21|BUILDING|efully regular deposits. bravely ironi 1254|Customer#000001254|wdGz5Cm DrSdF|18|28-832-851-4673|2676.06|HOUSEHOLD|wind fluffily blithely regular pac 1255|Customer#000001255|UC6I32JjBU62t4WgDe e2pDYbuM3VAt4MPM|14|24-359-633-2713|6487.71|FURNITURE|s deposits sleep. blithely ironic dependencies wake. blithely even theodolites sleep. blithely 1256|Customer#000001256|sNx4HcJ35paZik,IN02G7p|15|25-306-342-4782|5012.07|MACHINERY|integrate carefully. blithely e 1257|Customer#000001257|kX6yufw5dfKrgwQPVwWE7|2|12-824-451-8526|8810.83|HOUSEHOLD|hins. furiously unusual foxes about the regular foxes wake blithel 1258|Customer#000001258|zK3TKgKVuFCBdjt|3|13-727-588-7092|-301.75|AUTOMOBILE|ove the carefully ironic asymptotes. quickly final Tiresias wake regular packages. s 1259|Customer#000001259| YQc2RRQJV7kl1zxWg4OiUVU 5GlpB|20|30-930-620-7210|8353.00|HOUSEHOLD|ages sleep blithely regular, final 1260|Customer#000001260|npdrgr5Yqp0znvQt,Cw07j4BS22RNIANcb3t|2|12-742-408-2980|4991.59|MACHINERY|kly express theodolites sleep blithely across the doggedly regular packages. final, unusual instruc 1261|Customer#000001261|mWs6m9QwmTOZ|20|30-372-895-4261|5579.81|BUILDING|uffily final pinto beans. ironic deposits according to th 1262|Customer#000001262|u39WRGDI6AKU|3|13-444-583-3984|2840.36|MACHINERY| final accounts sleep slyly Tiresias. packages are furiously idle platelets. slyly silent requests are acr 1263|Customer#000001263|MXA4v0xQ9Kt |9|19-690-614-5736|6975.90|MACHINERY|ress dependencies are carefully theodolites. blithely ironic foxes among the slyly bold packages sleep blith 1264|Customer#000001264|vC1Yg5q O9 Tt5SM7OF|16|26-617-707-6647|3959.28|MACHINERY|y, regular requests above the fluffily special deposits engage around the furiously 1265|Customer#000001265|CTbTIB ZTYyKUSY42Ksz F33fxKsSG|24|34-945-256-3226|2653.53|MACHINERY|yly pending deposits about the regular accounts pri 1266|Customer#000001266|LW7shrnoCLUjJKQI8EF7SIFofvvIUmiJzpdS|3|13-832-768-3873|1877.05|MACHINERY|egular ideas. blithely regular requests above the deposits unwind on the slyly dogged pinto 1267|Customer#000001267|o3dtauyIeWwFRok2whWam0MLjmOdlG1H|24|34-329-328-2500|8616.12|BUILDING| accounts nag fluffily blithely ironic pinto beans. carefully pending deposits dazzle along 1268|Customer#000001268|SHn6HpO2VXBw3RJFPxjQFGanrsndRwRR2LWdm|24|34-973-735-5374|5152.42|BUILDING|y pending accounts! blithely unusual ideas wake alongside of the regular, f 1269|Customer#000001269|j2hwJHCMprK8HQdK7DpeUx5SG8j4dfuNR|10|20-818-485-8060|2659.68|BUILDING|promise slyly against the carefully ironic deposits. fluffily unusual foxes cajole carefully 1270|Customer#000001270|HPcuKCEtUzP3np7 oDR|13|23-473-283-1422|6170.06|MACHINERY|oost along the unusual, permanent pinto beans. even packages integrate slyly according to the pendin 1271|Customer#000001271|S7fmHdkot3JAv|14|24-698-342-2768|1209.37|HOUSEHOLD|efully after the slyly regular accounts. carefully ironic theo 1272|Customer#000001272|hzMO9cmypW|21|31-659-617-1632|6865.14|FURNITURE|symptotes. carefully regular requests after the pending ideas affix fu 1273|Customer#000001273|6RglRQdIV9mF8Tn6ABFmSQl|12|22-594-567-9307|1499.56|AUTOMOBILE|t final packages. furiously unus 1274|Customer#000001274|eHJnE7ytBm|24|34-152-721-6307|126.97|AUTOMOBILE|nts are enticingly above the furiously 1275|Customer#000001275|KUtV3oFy2Kyuzs4zT DB,S|20|30-410-174-2034|8972.46|MACHINERY|uternes. blithely express accounts detect around the fluffily even theodolites. even, spe 1276|Customer#000001276|c5UAVe71MPvmerPafNlpvTBWCewT|16|26-809-582-2064|-761.70|AUTOMOBILE| the blithely regular packages boost blithely regular excuses. final dep 1277|Customer#000001277|2ETOoQWtvxqp|14|24-502-746-4128|-52.35|BUILDING|oss the packages. packages doze car 1278|Customer#000001278|OB JBXz5fghXsYEaClW8PZpDmxMVZct|5|15-253-270-5149|9038.43|HOUSEHOLD|foxes. even, special theodolites boost. furiously silent packages haggle? furiously 1279|Customer#000001279|fkrzLacsqCnwUwgjjttKmY|14|24-742-587-6985|7915.06|BUILDING|tes are. ideas above the carefully ironic d 1280|Customer#000001280|3AmBFWaqOYt7F|16|26-725-573-7255|3419.66|HOUSEHOLD|eodolites sleep according to the theodolites. slyly pending dolphins among the pending, express platel 1281|Customer#000001281|pekyJqzeIQKGO8TeLvXgH8HR|11|21-124-963-7614|8182.42|AUTOMOBILE|the furiously quick deposits. slyly regul 1282|Customer#000001282|qeYHABkf21,5C5OC5it6q|14|24-750-627-7414|8998.82|FURNITURE| even deposits sleep quickly regular 1283|Customer#000001283|6JeLWEtDERPB,0KzWB,I6Xs8rJXAC8ryFulW5NPC|0|10-203-771-2219|2222.71|AUTOMOBILE| blithely daringly final theodolites. foxes ha 1284|Customer#000001284|sdj PCsILD6mOJfEuIIbrN52hOHTYWwUUPT|18|28-750-346-1442|-911.40|BUILDING|pending packages cajole carefully! furiously final packages wake. special requests sleep along the caref 1285|Customer#000001285|5Hy,ajDzJPtZFeJedRSeLN7XGOJtyUy2FI93|19|29-424-835-1463|3061.58|HOUSEHOLD|ly bold ideas affix blithely about the slyly even pinto beans. slyly regular multip 1286|Customer#000001286|FP3aFvhRMSKfCz3l0h|12|22-374-932-9860|6906.08|MACHINERY|quests. quickly even packages wak 1287|Customer#000001287|8CaksGsCJOK3oUm1kUsQ|15|25-493-734-3918|7461.69|MACHINERY|e quickly silent courts. furiously even packages among the ironic ideas integrate slyl 1288|Customer#000001288|wQDTTCkSGxic2d66|3|13-533-256-9320|6603.43|HOUSEHOLD|equests detect atop the ironic deposits. final requests according to the blithely sp 1289|Customer#000001289|OGb4YMkool8QMr|24|34-409-591-4324|2925.52|MACHINERY|deas haggle carefully alongside of the always even ideas. never unusual as 1290|Customer#000001290|0Q9URl0Y3rJWt9GYZF|24|34-837-161-2672|8108.42|MACHINERY|s use across the express requests-- carefully bold foxes cajole slyly slyly express pinto beans. ironic request 1291|Customer#000001291|dg3hkdHiI9zqk7l3242Q28OLLFy,1vZ,|7|17-693-294-2656|8227.37|BUILDING|, final requests-- furiously careful ideas wake busily ironic, even a 1292|Customer#000001292|QVr2XTDOMzWcLKHtNgrLK|21|31-966-407-1575|5509.11|BUILDING|haggle. special foxes sleep blithely 1293|Customer#000001293|E79dBMCNl5xXBwtnSsjuBLa16VgrLsKz|12|22-517-223-6566|2565.67|HOUSEHOLD|heodolites boost blithely ironic packages. special, even ideas above the asymptotes wake quickly accordin 1294|Customer#000001294|EZVIKislr4L0PrBP8LShL|23|33-506-204-7796|-808.13|AUTOMOBILE|deas boost bravely final ideas. slyly even pearls are furiously 1295|Customer#000001295|kded3b,5e5|24|34-259-484-2624|753.62|BUILDING|slyly final accounts detect blithely regular, bold requests-- blithely final foxes wake blithel 1296|Customer#000001296|cLAyTJcfD3T4hKW52lIU9yk|5|15-130-485-4234|3034.69|MACHINERY|e slyly ironic requests. final requests 1297|Customer#000001297|4QnYEe0KXOP3yridKldXROs7jQdMu9tE|21|31-579-682-9907|6074.01|HOUSEHOLD| pinto beans! furiously regular courts ea 1298|Customer#000001298|ujAPYPBrLW,oIxGpuWmxoTDscSXFOP Tjk|15|25-765-244-1549|3903.54|AUTOMOBILE|er furiously despite the ironic, even ideas. slyly silent ideas boost 1299|Customer#000001299|vm2THnXrMKrn5xvPL88EMT9QntU|11|21-150-179-4763|808.39|HOUSEHOLD|sheaves promise furiously alongside of the slyly pending platelets. pending dolphins at the furi 1300|Customer#000001300|VmW1dNLVaQY0ud6Csa5WHWuV8|23|33-581-399-6027|-370.44|BUILDING| x-ray furiously regular deposits. final, silent theodolites are slyly pending ideas. final dependencies 1301|Customer#000001301|oR0kHfL6GWhF VPD,mM1Jxsd9l3nZEkfDn|10|20-339-347-9046|8966.63|MACHINERY|egular asymptotes along the even, express packages sleep express realms; carefully final packages haggle quickly 1302|Customer#000001302|vyImQ4AVgv,Rn|9|19-316-212-9313|197.90|MACHINERY| platelets engage carefully! furiously express ideas shall have to use. regular 1303|Customer#000001303|MarfB1lCCs2MZ8CWdWqCfb|5|15-658-234-7985|2020.15|MACHINERY|express deposits haggle slyly after the carefully unusual packages-- silently si 1304|Customer#000001304|1sXtodRtFvBd449a2aJ|11|21-638-815-3982|4548.46|AUTOMOBILE|orses boost blithely platelets. fur 1305|Customer#000001305|xHgwqc1p0eLf5F8JkE7zvYXPHhIOP5IgLRJgR|23|33-396-634-9150|4900.66|MACHINERY|efully. furiously ironic packages cajole slyly bold requests. quickly ruthless requests alongside of the iro 1306|Customer#000001306|0YRFIqAc5imIKGi9cEYtn6L|12|22-923-551-9639|6464.77|MACHINERY|le. quickly pending accounts detect furiously. packages 1307|Customer#000001307|L OAVSFQauP87kLdHouM8|3|13-970-299-8199|4344.52|MACHINERY|ts. brave, express packages boost. even, pending instructions nag blithely regular theodolite 1308|Customer#000001308|Ndovi7D9gJ u1gjQwYOkIARup6VzhQFCHHmSMw|18|28-560-833-2066|9290.53|MACHINERY|uickly even dependencies. unus 1309|Customer#000001309|xaOhk73bjekYrVc5zZ36c,GuZUxsMHjo7WH9WVe|10|20-821-905-5952|-922.69|AUTOMOBILE|nusual excuses. ironic deposits are furiously ironic frays. blithely ironic platelets are evenly regular package 1310|Customer#000001310|bN, XpseFnbjZRh3fryWogaudZB|17|27-538-338-3378|204.84|HOUSEHOLD|unts. silently bold ideas against the blithely regular deposits use furiously ironic fo 1311|Customer#000001311|rcff2L75vK5EOUaPK DiDz6atB|13|23-647-279-5735|8713.24|FURNITURE|nd the regularly unusual foxes. regular asympto 1312|Customer#000001312|f5zgMB4MHLMSHaX0tDduHAmVd4|3|13-153-492-9898|9459.50|BUILDING|odolites wake always packages. slyly slow orbits lose. regular depo 1313|Customer#000001313|Ax4TI4jbHvaYUaaFuEUQTiMWQvrjez G|23|33-623-834-3089|889.11|MACHINERY|ely. carefully pending foxes was furiously special, special 1314|Customer#000001314|auN4t99aykk1AlmJl|1|11-290-301-2722|3218.33|HOUSEHOLD|e special theodolites haggle furiously along the even deposits. final accounts haggle. furious 1315|Customer#000001315|5J941XxxkE|10|20-941-614-6433|1447.84|AUTOMOBILE|refully bold packages. final, regular pa 1316|Customer#000001316|nmbpR1rqOdlUDvT6C HXUhm2|5|15-642-801-1329|-158.39|BUILDING| might wake. sometimes unusual requests cajole carefully about the excuses. stealthily final requests wake quick 1317|Customer#000001317|a6M1wdC44LW|14|24-518-294-8197|8925.08|AUTOMOBILE|s haggle furiously slyly final accounts. slyly bold pac 1318|Customer#000001318|yrASJAqw67PQxFYVAVsGU|14|24-524-279-4270|5812.93|BUILDING|lyly blithely final depths. regularly even accounts haggle across the carefu 1319|Customer#000001319|c5M1KcH60UZPYsa|9|19-573-345-3305|4910.48|MACHINERY|se across the dependencies. express, 1320|Customer#000001320|8gman6hzpuKUsX7mKU9katXpP1ia|15|25-116-108-3791|6407.13|MACHINERY|lent, final accounts cajole fluffily special requests. deposits around the fluffily even packages 1321|Customer#000001321|dWd3MhPQY3|10|20-571-787-3958|3589.49|FURNITURE|express foxes are quickly blithely bold 1322|Customer#000001322|35jI39rgIHCI4Pwvpy1beKgL0|13|23-207-256-7245|2621.71|FURNITURE|tes cajole. blithely express request 1323|Customer#000001323|r9R6okXwQID,|23|33-476-768-7390|6006.81|HOUSEHOLD|uffily even packages. dependencies are. excuses cajole furiously regular foxes. special dep 1324|Customer#000001324|6qS1ZDpAYr9aED9Yh ggf8ACJcPi7sp|7|17-415-957-9976|7548.88|AUTOMOBILE|t the busy courts sleep quietly above the ideas. final accounts after the regular, ironic pl 1325|Customer#000001325|Agu uZvi6Xv77 nE7W8|7|17-687-303-1074|9108.61|HOUSEHOLD|n foxes integrate furiously ironic requests. furiously even theodolites use daringly pending deposits; even pl 1326|Customer#000001326|naLuK8XKUP72msE0e|21|31-373-307-4091|-468.49|HOUSEHOLD| pending accounts impress. regular, express accounts cajole ironically express deposits. slyly regular accoun 1327|Customer#000001327|LBVMBxjllZpTQd|12|22-920-576-6295|0.97|MACHINERY| are furiously according to the multipliers. pinto beans are thinly. special deposits haggle quickly express Ti 1328|Customer#000001328|fjKlKFyxTQRJjLeT1Md|10|20-305-428-9878|3264.99|AUTOMOBILE| deposits haggle ironic, bold packages. quickly unusual packages print furiously care 1329|Customer#000001329|Q3 pefFAcrEYPQ6J AC|17|27-945-826-8003|4645.91|BUILDING|quickly silent requests affix blithely slyly bold instructions. furiously even packages dazzle whith 1330|Customer#000001330|MGY4P7QIy3|1|11-353-524-1234|3893.14|BUILDING|to beans doze along the furiously final pinto beans. req 1331|Customer#000001331|mjaArHGsPWg|8|18-140-389-1328|2005.02|BUILDING|ic excuses. requests would promise according to the furiously ironic accounts. slyly final deposi 1332|Customer#000001332|1JMz4nbClfcxmzPyWyJK|18|28-560-351-6594|3323.04|FURNITURE|ing forges. foxes haggle fluffily. express, final excuses sleep slyly blithely express dependencies. 1333|Customer#000001333|o9o6lky2KYgFZ2cSx5lyFQYufM1i1d|2|12-154-975-6824|1330.85|AUTOMOBILE|beans integrate fluffily. carefully final pinto beans wake furiously even pint 1334|Customer#000001334|gZkxQY2Aa3o D6f1O 7nsPdg6BJ3|5|15-493-800-1041|2485.71|BUILDING| unusual dependencies cajole regular, r 1335|Customer#000001335|VeQAJlVqZgl0adTxSpZ6P2ZVIC0kWokJ|10|20-744-779-7057|8341.67|BUILDING|t slyly accounts. slyly express pinto beans nag. 1336|Customer#000001336|E4MeTLnSTIOlWkLDwmG7QPf 9Dq|16|26-350-110-5043|1490.21|AUTOMOBILE|ts after the deposits are quickly deposits! blithely regular theodolites integrate above the slyly pending 1337|Customer#000001337|ACAMJe2Xdw2BCgHrGMd0BX|22|32-528-594-1931|7882.44|MACHINERY|n, express gifts. express, fluffy pinto beans sleep. regula 1338|Customer#000001338|8Nx5v3cKF MK3ejHdMUgcY,FNZZs1|20|30-763-866-5779|5139.00|BUILDING|te quickly above the regular packages. thinly re 1339|Customer#000001339|QGiiQ1iMDmLKLAHsZa L68gZFyPXX18a38IS|4|14-904-963-2452|8167.50|MACHINERY|neath the carefully unusual plat 1340|Customer#000001340|dYRQ2tz0OdH|21|31-872-435-1900|280.29|HOUSEHOLD|against the final theodolites. slyly regular dependencies after the bo 1341|Customer#000001341|n5dnBrBUHnNEnaglCr9jNvONhG tMPb|18|28-701-221-9569|762.69|MACHINERY|nt requests. ironic, even excuse 1342|Customer#000001342|FD6UNqfsYMKkf3ZFZdI4EaYMZ|16|26-340-733-2096|1520.34|MACHINERY|y around the final, special foxes. 1343|Customer#000001343|WtCLJBdycxFOsHyv|18|28-393-594-5247|8303.09|AUTOMOBILE|accounts. blithely pending foxes wake among the carefully express forges. quickly ironic realms wake bl 1344|Customer#000001344|95XSwEZD22AZln3RB|5|15-307-682-9911|2113.32|AUTOMOBILE|after the furiously ironic foxes cajole slyly unusual, pending reques 1345|Customer#000001345|31zcobEB,6Li4YDZbnNX|9|19-913-651-7783|8593.83|BUILDING|of the express, express packages. final requests detect to the regular accou 1346|Customer#000001346|cuwz2Yvj VKYEXjZzfL|10|20-502-685-6183|4524.45|FURNITURE|heodolites after the quickly bold deposits wake according to the regular platelets. ca 1347|Customer#000001347|7oXery7shMx|24|34-956-232-6103|8476.43|HOUSEHOLD|ular accounts. furiously final t 1348|Customer#000001348|CgtcDDYMnvsgI1uozRj|23|33-360-732-3579|459.22|MACHINERY|s cajole furiously among the ironic deposits. carefully bold pinto beans wake carefully against the carefully 1349|Customer#000001349|HvlnFsKOdm39Ge4VPgzE,UN|18|28-950-527-8728|4967.24|AUTOMOBILE|ges. final ideas nag furiously against the fluffily express accounts. 1350|Customer#000001350|fc,TCo2zqB9T3C5IbaGkfV3,hLqLr|3|13-486-903-2349|3339.51|AUTOMOBILE| regular, ironic ideas are carefully against the silent packages. careful, 1351|Customer#000001351|NYMFfkNlCGoTeaDrNO9nn|1|11-916-210-6616|3106.00|FURNITURE| accounts after the final deposits sleep fluffily ironic accoun 1352|Customer#000001352|XW4X8ComPo5mlyrgLn|20|30-631-606-4317|5570.69|FURNITURE|en escapades after the furiously special accounts use slyly regular grouches. fluffily final pinto bean 1353|Customer#000001353|CzscM6Q8vW6|1|11-109-274-1421|3666.51|MACHINERY| quickly ironic packages. regular, bold asymptotes about the foxes haggle carefully regular pa 1354|Customer#000001354|rvGErAt5suIqpuxwtL QPAgN7n7Tyv|13|23-969-619-1363|-897.04|MACHINERY| blithely blithely pending packages. furiously pending accounts use slyly. bl 1355|Customer#000001355|c1r6G98ixzLQkvUV2KphsFwhYvpDo18oToGB|20|30-918-883-1662|2351.10|AUTOMOBILE|anent dependencies are blithely above the quickly silent escapades. requests sleep. final foxes sleep slyl 1356|Customer#000001356|3SLzAiW4PihnFUE243 AHKkwtL1PCj|5|15-656-712-5740|927.39|HOUSEHOLD|fully pending deposits. carefully unusual accounts 1357|Customer#000001357|S1bDHNFkDEi,Gbsc3|15|25-242-146-4223|8627.90|BUILDING|osits boost pending packages. slyly pending deposits along the requ 1358|Customer#000001358|t23gsl4TdVXqTZha DioEHIq5w7y|3|13-264-253-1258|5149.23|BUILDING|sy excuses. slyly express requests detect slyly quic 1359|Customer#000001359|F5XtTR5KeZ,wAL|11|21-124-833-5784|4069.82|FURNITURE|eposits sleep quickly. enticing packages sleep ironic, ironic accounts. daring, regular t 1360|Customer#000001360|xeaT6W6D569UKCKU86iK9b6aUanlra|19|29-574-552-4018|1422.57|MACHINERY|nt packages affix quickly furiously regular foxes; quickly 1361|Customer#000001361|OAHRbO5RS8,yFt16e7glYM4oVEZpf8BefK5DA,7|13|23-104-975-7608|4128.86|HOUSEHOLD| need to promise furiously quickly bold packages. finally express pinto beans alongside of the 1362|Customer#000001362|FKywgbtf4ib|7|17-801-385-5904|3718.92|MACHINERY|gouts. quickly silent foxes affix after the ironic, special accounts. carefully bold d 1363|Customer#000001363|mYa,yAtLmW2mCglfc7cZ8LrPuP0|13|23-964-365-7781|-112.46|HOUSEHOLD|silent packages. blithely regular instructions haggle carefully slyly ironic forges? thin, br 1364|Customer#000001364|INrMv02tUJWFSRMEbBl0oUTtCjry8qUcI8T|19|29-992-959-9626|-181.69|BUILDING|e furiously according to the slyly final ideas. blithely silent excuses cajole s 1365|Customer#000001365|DOifjgJKjlSgnpPJ3cHLl2yi EseDZbg3|17|27-358-301-5393|2207.81|MACHINERY|ironic requests use blithely according to the slyly ironic patterns. carefully regular excuses about the c 1366|Customer#000001366|v3YAa1hq4Qc7FdpLg4Jh0b7xo0soyvq1w,Yrb|20|30-193-707-6924|1634.70|HOUSEHOLD|the furiously final foxes. furiously bold depos 1367|Customer#000001367|gN803k703pZ1YizV5fp6S8|22|32-462-328-6604|5420.32|HOUSEHOLD|riously theodolites. slyly bold excuses thrash slyly final pinto beans. instructions use 1368|Customer#000001368|4PxJqZUIML EhegD7RXkLY8|15|25-801-622-7438|6376.18|BUILDING|iously regular packages wake according to the slyly final deposits. carefully even packages cajole. carefully 1369|Customer#000001369|rXTwOzU0a2ak4Nj5L5b1aLij|10|20-232-617-7418|498.77|AUTOMOBILE|ong the ironic ideas haggle slyly above the courts. packages engage blithely. pend 1370|Customer#000001370|WN7onCgcC,,Lt4dC4C f7SCgnHWSjeTUp|18|28-575-379-5893|9802.04|BUILDING|y across the regular dependencies. fur 1371|Customer#000001371|H,U3MSp1OTLGIQuW2|7|17-492-673-8157|4943.58|BUILDING|es are after the carefully ironic deposits. silent requests alongside of the furiously even dependencies 1372|Customer#000001372|WiWQk7DyBtI,hfP0CIZ|23|33-563-510-6488|1796.09|HOUSEHOLD|l theodolites. regular ideas are around the furiously iron 1373|Customer#000001373|fAfmAacTlPc|13|23-959-476-7310|909.84|FURNITURE|ckages cajole slyly even requests. express 1374|Customer#000001374|vRPteZtcyV|19|29-869-316-1166|-411.43|AUTOMOBILE|ckly permanent accounts wake fluffily regular packages. quickly express foxes cajole. carefully ironic packa 1375|Customer#000001375|lpKhW7g QK7Y13sxKlRvRYI7SItbTbcBxae|14|24-620-497-1489|2011.11|BUILDING|requests! even excuses are furiously express deposits: fluffily ironic 1376|Customer#000001376|VushDntQeYmYLT22vW09rlg5j06B|12|22-972-150-2900|6761.52|HOUSEHOLD|iously unusual ideas. ironic ideas use carefully about the foxes. slyly unusual pinto be 1377|Customer#000001377|P7aUKm47hbe14nVZSrwZ|17|27-398-963-9520|8839.15|MACHINERY|s sublate carefully alongside of the slyly express theodolites. furiously special instructions haggle 1378|Customer#000001378|zDULZOX6KrHF6aL1AMsIg0Ivv4Crz|17|27-806-173-2824|2675.73|HOUSEHOLD|ges haggle slyly alongside of the furiously final excuses. carefully regular foxes boost across the regular, ex 1379|Customer#000001379|rqYSBCMywMKnfcp2DwotVqI|6|16-695-982-9623|1008.26|MACHINERY|sly ironic requests cajole fluffi 1380|Customer#000001380|a,q fKSYFADxRtRQWSppP8YKp|17|27-641-565-1036|3723.53|HOUSEHOLD|lar instructions boost quickly. blithely regular 1381|Customer#000001381|HqKfFUD6Ib9yoFM5cIgMxjXaqdJAyKSN5w Od|22|32-418-900-6494|367.82|BUILDING|foxes thrash slyly express foxes. even th 1382|Customer#000001382|uiTMgqzTPqAPoKQwbnv|10|20-962-576-3853|8898.67|AUTOMOBILE| wake furiously through the pending platelets. furiously pending deposit 1383|Customer#000001383|bSLtrtrAaAky9GZuKhlQqp8BB|15|25-267-778-1591|2092.61|AUTOMOBILE|s. fluffily unusual accounts against the special theodolites print around the special the 1384|Customer#000001384|bQI5haTy6PHM8MyRtKSlvU4ixAUg|8|18-788-299-9227|1534.38|HOUSEHOLD|tes. regularly pending theodolites cajole even sheaves. stealthy, ironic ideas are furiously above the even p 1385|Customer#000001385|4jAtwsWIITPzhTIx7jblhjp9aAzejEGnu|3|13-693-138-5884|2326.75|BUILDING|ithely daring ideas? regular requests wake slyly ag 1386|Customer#000001386|uByG5EoybI5dNNLzU5uD4Ba|11|21-450-191-9064|9643.87|BUILDING|ages above the busily final packages grow blithely alongside of the blithely even f 1387|Customer#000001387|GQhAzCMyKiDoel3|19|29-444-890-8990|8541.87|BUILDING|foxes haggle furiously according to the stealthy ideas: slyly special accounts about the requests use caref 1388|Customer#000001388|WaKdgWEru70hsL8nyLeEkneHyM59Lboo5zfWv|13|23-185-747-9502|291.44|FURNITURE|longside of the ruthlessly regular dugouts. slyly ironic pinto beans wake. dogged de 1389|Customer#000001389|ORf,IQyXsXJ1svlQ,5U|19|29-865-304-6982|1111.02|FURNITURE|posits. accounts are carefully. carefully express deposits cajole-- slyly bold dugouts wake according to 1390|Customer#000001390|fQm,RnwO4Tt PMQIB|11|21-978-977-8988|3931.31|HOUSEHOLD|ress platelets poach carefully above the slyl 1391|Customer#000001391|7MqN5yFijW6Yua7LVU6i7QMjjiyJ2KTZEaQ|23|33-558-545-3053|5510.28|FURNITURE|ges haggle slyly across the carefully pending accounts. slyly regular accounts should sleep. slyly express packages 1392|Customer#000001392|iXmNoe7IBgjc|22|32-561-640-4912|249.82|HOUSEHOLD| sleep fluffily across the final, pendi 1393|Customer#000001393|zVp5Hbhro,9rTwCYys1HUk|24|34-953-819-7858|5672.05|FURNITURE|escapades. fluffily ironic packages nag among the slyly regular dolphins. special asymptot 1394|Customer#000001394|eE8wv lYYKLXB|3|13-580-581-6470|2233.10|MACHINERY|regular accounts cajole never above the even, final instructions. furiously regular foxes wake unusual requests. ca 1395|Customer#000001395|XJoxiYIaBYgEE|23|33-291-909-3901|8733.39|MACHINERY|tions sleep carefully. furiously final requests about the regular excuses a 1396|Customer#000001396|M4dHuyrttFfeBr|3|13-523-516-9742|7149.43|BUILDING| the even theodolites integrate fluffily regular dolphins. blithely 1397|Customer#000001397|1bk KBemIEsKhD3VyXa6IRLx 4GH u|8|18-294-992-6523|5466.83|AUTOMOBILE|onic packages across the bold, regular dolphins boost furiously furious multipliers; furiously specia 1398|Customer#000001398|K1rQq6exc3WcVCcgIjA4SaeqxtK2,HG1|8|18-377-181-4654|7004.90|HOUSEHOLD|the blithely silent dinos. even, special hockey pla 1399|Customer#000001399|FOuY,endAFj|0|10-775-919-7154|7352.14|AUTOMOBILE|foxes across the silent platelets haggle fluffily special requests. unusu 1400|Customer#000001400|BuouRkR7J f|0|10-217-180-5310|2432.73|BUILDING|etect fluffily final courts. carefully special instructions 1401|Customer#000001401|C4vlB8ENikVmaMizX3nH3zgds6|9|19-339-404-7859|8908.63|BUILDING|accounts use furiously unusual pinto bean 1402|Customer#000001402|F7 m0JwiCABmbJLPQpCJ2|6|16-713-144-2780|4396.25|AUTOMOBILE|g the carefully express dolphins: special, pending packages affix after the packa 1403|Customer#000001403|,ql804gtMc3uxTfP,lt4yRBWQ|12|22-458-624-2509|9782.34|HOUSEHOLD|tes are blithely carefully bold pac 1404|Customer#000001404|pIO5i3yjeODChGMHoVvrX,Ctpdj|12|22-320-701-5582|3828.46|MACHINERY|ven platelets use quickly pending requests. busily busy asymptotes sleep slyly across the 1405|Customer#000001405|i9khsGcg17kWI4q5LKTcc8U3aFojf403|16|26-285-488-6682|3987.39|BUILDING|haggle slyly; regular, final excuses are blithe 1406|Customer#000001406|g1xS4snd0fzl4R,JmPHfEzRD|5|15-767-155-6419|2023.59|HOUSEHOLD|uests are furiously carefully express packages. slyly permanen 1407|Customer#000001407|zZsTZ3nI1rG5X|14|24-529-300-1554|7424.99|BUILDING|nding, ironic instructions promise across the quickly regular r 1408|Customer#000001408|NMIb3p1DyU,Z4XOFUS0B,|11|21-901-381-6344|5920.09|HOUSEHOLD|express requests believe. pending, brave deposits sleep furiously. carefully regular deposit 1409|Customer#000001409|jzfaCksWUNlI|11|21-667-401-3780|1959.17|MACHINERY|ously ironic ideas are. unusual packages kindle along the dugouts. unus 1410|Customer#000001410|yEUlreh6mkGmg8SIwKZooUOJ42kuZwAptaR4HAJ|22|32-358-270-1819|2716.95|HOUSEHOLD|lar, express packages cajole bravely permanently final packages. blithely even dolphins nag finally special theo 1411|Customer#000001411|4iLVKtSmtJpU|21|31-898-640-7625|772.14|AUTOMOBILE|sly final ideas. carefully bold pinto beans wake: slyly regular packages must sleep. final platelets inte 1412|Customer#000001412|gfsI6WU i7kYypv09gGIqUFrUod9uhb|10|20-715-510-2804|6368.38|BUILDING|fully regular accounts print ironic, regular platelets. deposits promise slyly. express, ironic ideas 1413|Customer#000001413|9Yh cGpbCbrXZytNfH,dAEwX|15|25-624-816-9010|1387.83|AUTOMOBILE|onic foxes. quickly final dolphins are fluffily. quickly unusual ideas wake carefully. furiously 1414|Customer#000001414|2 HBoqTD0qCyMKtcBPVHbNna|20|30-323-797-7514|3136.15|BUILDING|ackages. blithely unusual accounts a 1415|Customer#000001415|x,hzUUAZ9w7ndksLyH0,fEpMfU|21|31-295-601-1598|6252.12|BUILDING|heodolites. furiously pending requests are above th 1416|Customer#000001416|ovOZcFGL31uxmA2ifIYudX6OuwNDz,B|7|17-306-898-9363|5348.40|MACHINERY|unts wake slyly excuses. bravely even pinto beans across the furiously final de 1417|Customer#000001417|1BDU8AvljnLmkM|1|11-242-612-1339|7543.01|FURNITURE|ag slyly-- furiously final accounts are ironic instruct 1418|Customer#000001418|S5uBtE hDxHcHunowPDXKSxP3csMFnhYt|17|27-773-818-3164|9359.01|AUTOMOBILE|ggle quickly blithely thin excuses. final, even accounts integrate slyly. carefully pending account 1419|Customer#000001419|JM4NV2pq4Ps0xJNRtUtlmQ8uuDvKx|4|14-533-796-5446|5912.72|FURNITURE|. idly express pinto beans sleep above the deposits. excuses use: furiously bold accounts cajole slyly across the ca 1420|Customer#000001420|mjkRUOEzdCWpNdpp5PKOObMmhpufeNGnO1VFdbpK|9|19-433-305-7356|-932.09|AUTOMOBILE|the instructions cajole carefully. slyly final requests nag carefully ac 1421|Customer#000001421|Qx9tZ9yiMo|12|22-139-990-1907|7292.93|MACHINERY|kly even ideas cajole carefully quickly ironic 1422|Customer#000001422|mzXw44ExYC8DAdeKBakiWy0II|17|27-270-833-4320|8389.50|MACHINERY|mong the evenly express asymptotes integrate slyly brave 1423|Customer#000001423|9BcCj8CLsqylKxRj0,lm|21|31-624-875-9135|2411.69|FURNITURE|he carefully express courts. regular instructions haggle. special, express accounts believe? packages cajole slyly. 1424|Customer#000001424| 3QsPgbVLZ|22|32-542-134-6212|7207.70|FURNITURE| instructions against the furiously express accounts doze fluffily unusual d 1425|Customer#000001425|I2UBZAPPdnA9oFKKJTGxaSQZb5QzNzTR4vN6d P|1|11-697-824-4418|5814.72|FURNITURE|uests boost carefully even foxes. accounts along the slyly ironic requests cajole express, final deposits. sl 1426|Customer#000001426|d1Tyzg,0ArKPuBln8CDH, 1Xsukm2nXVl|15|25-629-292-9022|1965.38|HOUSEHOLD|s are slyly according to the express, spe 1427|Customer#000001427|dDsmiig0T4oFKaf9ttFeh1etLvSIc9aV1xF2H|11|21-941-208-2485|8136.53|AUTOMOBILE|es detect accounts. slyly bold theodolites wake ironic, special accounts. instructions cajo 1428|Customer#000001428|3SdWi3lKPXk00UYT,hL|15|25-859-663-3690|7703.84|FURNITURE|iously final platelets sleep slyly ironic instructions. furious 1429|Customer#000001429|K5sID 6zGPrYdfoADUq4kidlPgF3|3|13-119-903-3814|6444.82|AUTOMOBILE|even requests among the blithely regular pinto beans use across the ironic accounts. slyly ironic 1430|Customer#000001430|mv 9MEDwd8yPeQj7N|0|10-209-317-6929|-920.40|BUILDING|nic deposits. bold, even accounts cajole blithely 1431|Customer#000001431|l3LM2d2T1n c7yI4sOfpEbbd540qO66A4MARk|13|23-640-395-7009|5805.81|AUTOMOBILE|eposits up the carefully bold requests mo 1432|Customer#000001432|,pbQM2fi642oAuel|21|31-831-635-9758|6314.25|MACHINERY|ts sleep. regular frets sleep carefully ironically special dolphins. carefully bold pinto beans 1433|Customer#000001433|gK7D76v78U iRA2YI2kxeKLlm4LZMH13,|17|27-558-375-8169|4605.87|BUILDING|ly bold deposits. furiously silent braids alongside 1434|Customer#000001434|V15TQAhSLp7YC3KdjuMwSV3cwg0lp|18|28-749-743-6583|1851.63|MACHINERY|lieve carefully ironic hockey players. special ide 1435|Customer#000001435|s0fqxkVVqLWR IaqibwwOf|20|30-309-437-9265|-729.09|AUTOMOBILE|usly final orbits are? unusual, ironic accounts are slyly. unusual requests sleep after the ironic, 1436|Customer#000001436|kV5m0jkgFEto,|10|20-918-593-1860|9158.91|MACHINERY|theodolites. final Tiresias after the quickly final packages dazzle carefully blithely bold 1437|Customer#000001437|DjHgTwtlzCmcQo|17|27-805-486-5768|8839.32|BUILDING|y. bold pinto beans affix carefully unusual pinto beans. carefully bold Tiresias mold 1438|Customer#000001438|Gz Aey8gzHxIIxtpJaG0lAqd82T|17|27-688-787-7928|5436.81|FURNITURE|odolites cajole slyly. furiously unusual requests boost furiously along the fluffily pending in 1439|Customer#000001439|IGGK4SXvT5ioAeT2fbVYDemsTTqqhsQu6|21|31-433-694-1822|8487.60|AUTOMOBILE|ely after the carefully bold accounts. carefully ironic packages are silent packages. blithely ironic pinto beans ar 1440|Customer#000001440|k3LXBO5QJrG94TBG77adB1HjqQkleDyUf2c|7|17-619-730-9883|1236.36|BUILDING|xpress, even accounts integrate. ironic, special requests doze. carefully express instructions doze furio 1441|Customer#000001441|u0YYZb46w,pwKo5H9vz d6B9zK4BOHhG jx|23|33-681-334-4499|9465.15|BUILDING|nts haggle quietly quickly final accounts. slyly regular accounts among the sl 1442|Customer#000001442|2fTQpX7N2kp31U|16|26-677-746-7145|7917.90|MACHINERY|ess theodolites. furiously express pinto beans alongside of the 1443|Customer#000001443|qtBPSM2NvmJXNePBT Ap3M6UqIZTvaF|10|20-959-383-4792|7141.87|HOUSEHOLD|ickly about the sly foxes. furiously bold patterns sleep regularly across 1444|Customer#000001444|8WcsyfQU5svH9miWvYbSTH9|7|17-107-228-8125|-501.37|HOUSEHOLD|inal platelets. quickly ironic requests do are carefully carefu 1445|Customer#000001445|5y7gtM75FOfTSBKx9gs9c9MkqJt|21|31-151-251-1931|8367.94|HOUSEHOLD|carefully regular accounts after the b 1446|Customer#000001446|p94EVXQW,Q3bhXDyhG1Gp96b5zbaW|23|33-873-120-5388|2981.48|BUILDING|gle ruthlessly. furiously express dolp 1447|Customer#000001447|pHkyNkViDja,dZVNg4bEEbicpoHIVDZvtQi8RPl|17|27-452-251-2941|2718.02|BUILDING|ously around the accounts. packages haggle blithely ironic, idle ac 1448|Customer#000001448|a45QD J55bo35zA4qR3v|24|34-969-612-1458|7756.35|AUTOMOBILE|y ironic instructions? slyly pending platelets hang quickly slyly ironic ideas. blithely ironic instructions a 1449|Customer#000001449|lNFczqF3TjlSO9BuO3jqXY,b|22|32-827-813-3340|9051.75|BUILDING|ructions wake slyly ironic notornis? slyly express courts wake along the slowly 1450|Customer#000001450|z7Pl iXBEstivMNf|3|13-443-688-6724|857.70|MACHINERY|ains; daringly dogged deposits across the furiously regular instructions breach furiously foxes. carefully unusual a 1451|Customer#000001451|Yt69m0Aw1LWZhisHJxL4iGEEzx6y,ehspkes|2|12-590-121-9328|3274.30|FURNITURE|ously regular packages. furiously final deposits boost. slyl 1452|Customer#000001452|51mhHAvPHZACedHYXVU5HXoDIQtBK9,pxuOIlJ|7|17-581-575-4538|7086.97|FURNITURE|silent theodolites. fluffily special 1453|Customer#000001453|FTfWkW1 8jVgOIIR9sMm2HpohiuR1v2278|0|10-852-397-3642|662.67|BUILDING| to wake above the blithely regular foxes. d 1454|Customer#000001454| wwPhUG35PiIVasu88,RvDA|24|34-478-555-5955|3366.61|FURNITURE|ss the blithely ironic deposits. regular deposits after t 1455|Customer#000001455|MrDN0cvhoLZ ioRLZCR hPcY4WvILz2|3|13-839-360-1866|7591.90|MACHINERY|ironic instructions: ironic pinto beans acros 1456|Customer#000001456|zKDB5elqlAQoUQp|13|23-171-834-8997|6123.69|AUTOMOBILE|unusual accounts wake. even, ironic packages wake carefully. regular p 1457|Customer#000001457|qmpteVs7H9WjRow7FDut9a77oFKRDOXxq0JmG|23|33-660-953-7656|2873.49|BUILDING|onic accounts nag blithely among the regular, regular pinto beans. carefully regular 1458|Customer#000001458|vsGifZH3fNgJjlgF6jJbmkSqGc|3|13-392-503-9207|2716.80|AUTOMOBILE|ests sleep bravely ironic accounts. quickly regular accounts cajole 1459|Customer#000001459|2sPwjFNEFf9dN4az|11|21-424-586-6295|9270.88|AUTOMOBILE|ounts use blithely. blithely pending packages use ironic deposits. final accounts boost slyly. care 1460|Customer#000001460|AEgBZGLmuMqe7Gqh1|20|30-151-388-7118|9680.51|BUILDING|accounts. ironic packages cajole furiously; quickly pending requests lose quickly carefully bold deposits. 1461|Customer#000001461|MMmT5l0zAilFCb2ZMqsUm3TXlRH|3|13-393-444-1533|8460.48|AUTOMOBILE|ress, unusual packages affix carefully carefully final ideas. blithely special instructions nag even deposits? f 1462|Customer#000001462|b9Ed,6BCKn5v37q1|17|27-153-195-4457|7305.88|FURNITURE|ously slyly express requests. spec 1463|Customer#000001463|WD3OuRpJ0NVj2qslrTkUPmeJqVx5|7|17-980-394-5868|6039.64|MACHINERY|s. blithely even courts wake quickly: quickly silent pains doubt slyly. slyl 1464|Customer#000001464|5kOAfK9s6goOZabgSzNLgD,CILowRxqC2OLnV|14|24-133-117-1577|9306.20|AUTOMOBILE|iously furiously regular tithes. boldly final requests use carefully at the f 1465|Customer#000001465|tDRaTC7UgFbBX7VF6cVXYQA0|8|18-807-487-1074|9365.93|FURNITURE|s lose blithely ironic, regular packages. regular, final foxes haggle c 1466|Customer#000001466|Fdm3uYarZ0Tnnh9R|17|27-360-496-5041|1268.69|FURNITURE|nts along the blithely bold instructions boost carefully after the unusual depos 1467|Customer#000001467|GE,jQi5oLlkzh4jIUct7r 3C5G|24|34-941-824-8063|2857.19|FURNITURE|. final, bold deposits sleep furiously. unusual instructions are final requests. quickly final sentiments 1468|Customer#000001468|APEd1ssFxDC9fhwosxxeQUul5EhwBczX|12|22-901-280-1023|3826.52|BUILDING|s. slyly regular theodolites aft 1469|Customer#000001469|yLW8qLv25wuMsibRd,1qJe9|7|17-961-583-4658|4329.98|FURNITURE|y even dependencies wake against the regular, final excuses. packages haggle slyly a 1470|Customer#000001470|8ufZxZ5IgwGrUM2CWfxYoRHuBi Vj8rY|17|27-350-836-5521|7033.49|HOUSEHOLD| excuses are slyly after the carefully bold accounts. unusual pinto beans boost. final accounts wak 1471|Customer#000001471|lbRP,tSo,eQT6rDDNNIBx|5|15-230-827-4758|3872.86|AUTOMOBILE|thely according to the carefully ironic foxes; packages according to the quickly special deposits wake fur 1472|Customer#000001472| Eayx9GAqjJEwrGy1Er5 ffNtLL|5|15-464-411-8342|2168.61|MACHINERY| haggle against the carefully bold theodolites. quietly regular ideas haggle. pending pinto beans engage sl 1473|Customer#000001473|UPkONG9dy4VYyGNJGHG|0|10-891-555-7734|2796.93|MACHINERY|uriously. quickly pending multipliers maintain slyly silent excuses. regular requests cajole qui 1474|Customer#000001474|KB83CaaM8DRjvAmEMg1fw|16|26-609-226-4269|2961.79|HOUSEHOLD|kages above the requests sleep furiously packages-- deposits detect fluffily. pending th 1475|Customer#000001475|4tUf4SaYTFV2H7ji|0|10-932-794-2009|1820.28|BUILDING|uctions sleep blithely bold packages. pending, silent deposits after the fluffily final pinto beans ar 1476|Customer#000001476|nsPnedR1dhWK,|16|26-621-638-1459|409.72|BUILDING|across the fluffily final requests. regular forges haggle furiously r 1477|Customer#000001477|nUT6kGEr7tmgpJaPgfFtXY|6|16-407-756-8079|9103.33|MACHINERY|ites nag blithely alongside of the ironic accounts. accounts use. carefully silent deposits 1478|Customer#000001478|x7HDvJDDpR3MqZ5vg2CanfQ1hF0j4|7|17-420-484-5959|9701.54|AUTOMOBILE|ng the furiously bold foxes. even notornis above the unusual 1479|Customer#000001479|KDZMMuMVSWQPkGpoTUE0G 1vXHd3mS4c,A,kFR|16|26-203-849-3685|9793.29|BUILDING|arefully final ideas. unusual accounts sleep. final packages wake. fluffily bold dependencies hang slyly. bl 1480|Customer#000001480|Hzjh65ZXBFSzflrjQgECkrp35gDha,2|7|17-573-775-8796|876.02|HOUSEHOLD|uriously pending courts are. deposits serve quickly blithely final excuses. slyly reg 1481|Customer#000001481|Vp7Um1Vy7MNVJvP 5cqUrz8scGtcaLJB3f5bZDW|12|22-674-694-9039|3204.67|AUTOMOBILE|lithely. idly ruthless packages wake above the bold, quick pinto beans. regular ex 1482|Customer#000001482|kTcr5JgkjFeLKIRcmtnCvOFr1feN59chP7|19|29-452-962-5934|2930.53|FURNITURE|are. slyly regular deposits mold carefully above the blithely regular ideas. carefully r 1483|Customer#000001483|ZjY1C b6cOnY3|7|17-202-113-4814|4409.70|BUILDING|nts sleep around the carefully express theodolites. requests nag 1484|Customer#000001484|WcOint654aJStnQWSgAAtI|18|28-987-505-1842|4883.17|FURNITURE|s against the furiously special packag 1485|Customer#000001485|oR6sZslMa7bPLxtHFhqdJt|24|34-329-123-7678|9412.02|BUILDING|pliers. ironic requests boost slyly carefully express ideas. blithely ironic foxes af 1486|Customer#000001486|7A2MhrNtsA|24|34-559-605-2237|5859.97|BUILDING|lithely ironic dependencies haggle quickly b 1487|Customer#000001487|AJXUi2qFVKfypmmpTEbkmjmz0gPKQ2|17|27-197-562-5547|3589.16|FURNITURE|y final instructions. regular, regular packages boost alongside of the b 1488|Customer#000001488|DtF2uJI8td2wqrumD|4|14-892-461-5341|7929.51|HOUSEHOLD|sits boost quickly fluffily even pinto beans. slyly e 1489|Customer#000001489|yM8biIU5IFKHODCGTCwdkUf|9|19-906-669-4354|4389.66|MACHINERY|ckages play carefully? permanently regular pinto beans 1490|Customer#000001490|vBUkY7eCyWP|20|30-326-598-2437|8997.60|MACHINERY| bold orbits boost slyly according to the carefully ironic accounts. slyly special packages whithout the 1491|Customer#000001491|GjZIP4Fv5lqDt|4|14-931-281-5631|3739.82|HOUSEHOLD|efully silent tithes. even deposits according to the unusual, even platelets haggle furiously a 1492|Customer#000001492|2QNz4Zy0UjjI|1|11-527-949-4092|-875.17|HOUSEHOLD| blithely even accounts. furiously final instructions across the decoys cajo 1493|Customer#000001493|FbV 8Ug9GkSfMde5b|24|34-947-154-7032|7014.12|MACHINERY|carefully quiet requests lose slyly. quickly final pinto beans haggle bli 1494|Customer#000001494|4V71P ku3jrqBfQp|11|21-248-166-9549|8292.21|MACHINERY|arefully furiously special ideas. pending deposits above the blithely regular excuses wake slyly car 1495|Customer#000001495|78w5H7VJSo0Ps,jqeoCWS4Kay17ygM4RtIH|10|20-416-910-7075|6227.55|FURNITURE|osely blithe, ironic foxes. regular dependencies use blithely about 1496|Customer#000001496|ZOyMxutVHpJy|3|13-802-978-9538|-496.49|AUTOMOBILE|counts wake slyly above the instructi 1497|Customer#000001497| D8e2U3gYd57H4grcOr,02|14|24-506-574-8552|2449.57|AUTOMOBILE|gular packages boost foxes. blithely bold escapades wake slyly special pack 1498|Customer#000001498|x XToT5oFi7oIsRG2mgIL3ncvYJoWBsufsQ7N,z|19|29-676-227-6356|5810.56|AUTOMOBILE|ackages are slyly unusual req 1499|Customer#000001499|4,6jWOEqfnuXkwhB7gs0M9TcWJlaJNv4bt|3|13-273-527-9609|9128.69|AUTOMOBILE|ole blithely permanent instructions. carefully even packages 1500|Customer#000001500|4zaoUzuWUTNFiNPbmu43|5|15-200-872-4790|6910.79|MACHINERY|s boost blithely above the fluffily ironic dolphins! ironic accounts 1501|Customer#000001501|tLJmtj5OgXCQM|10|20-489-284-9686|9734.53|BUILDING|longside of the furiously ruthless deposits slee 1502|Customer#000001502|FIsFVFApqxzRHQrRjAlODHWTDZc35,BD0c7CuyVy|13|23-873-733-3833|3361.88|FURNITURE| carefully express requests. quickly even in 1503|Customer#000001503|9fFMPuIIatxmXEDe4XCu4PRea9|2|12-957-226-3187|5164.52|FURNITURE|odolites. express notornis detect blithely unusual, regular d 1504|Customer#000001504|suueZs7bAberaafllLS|0|10-462-929-6039|8151.61|MACHINERY|ess theodolites. blithely speci 1505|Customer#000001505|SFczFxAak1xX,CmWAE|21|31-344-990-8260|8207.39|AUTOMOBILE|yly above the accounts. even deposi 1506|Customer#000001506|RUScjIPOHpz3it|1|11-381-308-9658|-373.24|MACHINERY|s. furiously unusual excuses sleep slyly. blithely unusual packages about the slyly regular accounts are accou 1507|Customer#000001507|KtVNuytlncvuV44YzpoB|10|20-694-294-7077|5801.76|BUILDING|symptotes around the blithely ironic requests may boost blithely during the bold, final ideas. ironic pinto beans sl 1508|Customer#000001508|E7fRkt7uXJHIR8akfmor42eTm5kZH|4|14-740-990-2746|4213.74|AUTOMOBILE| fluffily pending asymptotes. blithely even foxes nag slyly slyly pending platelets. carefully regular req 1509|Customer#000001509|LQY2i,MHY8czRV2Ize|9|19-226-262-5083|328.44|BUILDING|tipliers serve quickly furiously express excuses. furiously unusual deposits slee 1510|Customer#000001510|BVNRoS0TPt7yBxD|1|11-138-490-8934|7100.18|MACHINERY|ng the quickly bold deposits. regular, even deposits use silently. special, regular packages mold slyly regula 1511|Customer#000001511|Lh9VKgOjqeJ5P5veH6NKZG3We|4|14-230-666-6671|2757.29|MACHINERY|xes. ironic, special dolphins against the regular accounts haggle carefully across the f 1512|Customer#000001512| FhwT 40,zugIGQPtYUDkjvXct070xNX4Lze|16|26-502-737-9941|7729.48|HOUSEHOLD|l accounts. final deposits use carefully slyly regular deposit 1513|Customer#000001513|CkEgq3Yvj9kGkHvVeUELT1UP9HBnHwiEIFzRWNTA|10|20-670-367-4252|8434.13|FURNITURE|y regular accounts cajole blithely 1514|Customer#000001514|2dVI195Lf,EUjr1CY37GWxCxb0uUjEa|8|18-602-992-7324|4566.48|FURNITURE|ngage slyly alongside of the f 1515|Customer#000001515|QS USHJ02MP2yd7TIcCNGMyXjyQug0EIShDlUM|12|22-852-688-4287|-179.31|MACHINERY|ily: regular packages cajole furiously at the carefully dogged foxes. final grouches must are. silently daring de 1516|Customer#000001516|VFbEMU7LSQZPCZ3m73dNP2WH0ywr5loATV4r|14|24-797-943-8908|9263.27|FURNITURE|tructions integrate above the regular, regular somas. blithely ironic asymptote 1517|Customer#000001517|hJBcIv8Yc9ukY9Erz96RRKNR8upJ8IBJxgePjf9|23|33-993-734-9681|2875.95|FURNITURE|tes. carefully final packages against the p 1518|Customer#000001518|ulllJKhRl VkFwAIhlb |8|18-242-415-7477|37.80|MACHINERY|l, pending packages boost ironic, final theodolites. fluffy requests are carefully. ironic, regular theodoli 1519|Customer#000001519|ersLKVkITqd,b7yCM1td5h9Y1tQv|14|24-663-396-2927|6172.47|BUILDING|ost from the close accounts. r 1520|Customer#000001520|WuEf6uxQmSgTA1efbl24QhQ60WJoh2166RzzOiV|24|34-364-590-2076|8678.58|HOUSEHOLD|ges after the requests integrate slyly according to the ironic requ 1521|Customer#000001521| UAwhhVG066cebuZN6Wk7s|13|23-168-973-9213|9983.09|AUTOMOBILE|special deposits use quickly according to the furiously express packages. slyly unu 1522|Customer#000001522|aMbkFCcpuqN8YFzn8ctAhm skIIfd|4|14-712-410-2710|948.00|AUTOMOBILE|packages hang slyly alongside of the slyly silent foxes. closely express accounts alongside of the even Tir 1523|Customer#000001523| udI60hnJb0IMvB67xQFfkgamLpP2Bwynf5P|7|17-405-744-1455|2921.13|MACHINERY|oxes. slyly bold ideas snooze blithely. accounts sleep slyly agains 1524|Customer#000001524|nW8RCuzryVNcYMCvE ZKJC7apmhen|22|32-834-498-4224|425.06|BUILDING|ses are regular packages. even asymptotes believe. furiously regular platelets sleep carefully according to t 1525|Customer#000001525|NtS KugGxV4GMBxNAwZdR6wwq02 fd y5,M|0|10-178-851-9228|2915.69|AUTOMOBILE|cial dependencies. ironic accounts integrate regularly across the permanent accounts. quickly 1526|Customer#000001526|pAC6Yj2c5lyOIr5 IQpM0|8|18-679-265-7392|8012.73|MACHINERY|ructions. furiously ironic packages after the evenly express pinto beans 1527|Customer#000001527|486JIEEHa |24|34-219-462-2180|6025.59|FURNITURE|old excuses sleep furiously ironic packages-- final pinto beans cajol 1528|Customer#000001528|fa,9WdvoEW06FtLQ6bpXBYGORjOWt,w|21|31-594-709-9605|4114.07|AUTOMOBILE| bold deposits sleep slyly about the blithely express accounts. final accounts haggle furiously at the 1529|Customer#000001529|NYQrlaZMT2rOQadTbfSpAdPPTQwpWQEEWD|2|12-170-370-8690|8221.42|HOUSEHOLD|each blithely. ironic, even packages sleep slyly alongside of the furiousl 1530|Customer#000001530|KVYYmaQ7fGwFnhgBnot1zTnFa|2|12-845-483-5866|4404.87|MACHINERY|ckly furious deposits. furiously special instructions sleep furiously according to the regula 1531|Customer#000001531|OomxCS69ZBbyC99b6YHXYGvw1Fs|21|31-735-863-6916|728.83|AUTOMOBILE|. deposits use furiously. ironic accounts affix 1532|Customer#000001532|VHjtEO1OwfCkrTIj|1|11-301-550-1539|4311.62|BUILDING|ependencies. fluffily even instruction 1533|Customer#000001533|jVCPod3Ysz|7|17-511-289-3953|6323.48|HOUSEHOLD|cross the ironic, regular ideas. even, regular deposits haggle according to the blithely regular dep 1534|Customer#000001534|EJ1gh5MYQ7R xKH6RfqPU96So94cMHKHgnEVTgy|17|27-975-211-1327|5760.99|FURNITURE|ly final requests; blithely regular pinto beans haggle slyly along the quickly bold notornis. sentiments cajole c 1535|Customer#000001535|2l8xLuwaicTTg5RNA7mwyHhz|23|33-371-530-1740|1041.79|FURNITURE|g the blithely final accounts haggle carefully above the sly, express platelets. excuses eat slyly across the c 1536|Customer#000001536|HRUhB3D7LC6V ydQigaOZ10Y9Be1jN31|13|23-357-877-4041|6388.82|HOUSEHOLD|uriously against the slyly regular ideas. blithely ironic dependencies solve nev 1537|Customer#000001537|Fx1vbSLG90yTE3KF2VGDMOeny|15|25-482-334-6480|4000.55|HOUSEHOLD| courts. quickly regular instructions are blithely silent excuses. slyly special theodolites use finally 1538|Customer#000001538|ohSUJgfMxt2Hq9f0tv,MZaRsombSl,MU1d6,|2|12-766-442-8988|3245.69|FURNITURE| blithely carefully even instructions. blithe 1539|Customer#000001539|EFgodQ9F0u SUYZQcJCNzjDlte5 br0klLU|17|27-544-403-7594|-702.43|MACHINERY|e blithely express pinto beans. carefully regular pack 1540|Customer#000001540|c1kVCV43v2RpUwCoZJ2LBHWYt2BT7|17|27-352-357-7209|6102.98|HOUSEHOLD|ious foxes against the slyly regular deposits boost ideas. regular waters sleep slyly: unusual deposits 1541|Customer#000001541|3HbD4JaolktsAYU,OgPrar2|15|25-786-474-5957|6792.00|AUTOMOBILE|blithely. packages wake final accounts. carefully enticing asymptotes run quietly despite the slyl 1542|Customer#000001542|4whsFeeVSBH7Eq WSu gF5JCsJc|14|24-754-425-8980|4108.68|BUILDING|sual, regular deposits haggle blithely alongside of the quickly ironic foxes. slyly bold pinto beans det 1543|Customer#000001543|IKgaPQRsONfY1vAsPP|18|28-327-662-8527|5653.73|MACHINERY|ckages haggle. idly even deposits according to the regularly even ideas haggle blithely re 1544|Customer#000001544|R,hoHFlkusJ,1Kts,0QEixg|23|33-132-882-2925|2204.41|AUTOMOBILE|ular deposits. final, regular accounts nag carefully quickly regular pinto beans. unusual, unusual theodol 1545|Customer#000001545|08TtvYMUYuq6Hgi1T4IsV2fr1G90cnb D|1|11-287-870-3637|-487.86|BUILDING|ial requests wake. sometimes regular sentiments are. pinto beans use car 1546|Customer#000001546|kFu hXaTK2Vk|20|30-788-120-7833|4488.33|AUTOMOBILE|sts haggle furiously. even, regular packages sleep. idly idle somas affix furiousl 1547|Customer#000001547|RgRcB,v0ZS|4|14-683-809-4484|6387.31|AUTOMOBILE|ts after the accounts are above the fina 1548|Customer#000001548|0uaAwzhbw,1VFL|23|33-610-656-3668|562.75|AUTOMOBILE|eep slyly regular, final instructions. final dependencies engage a 1549|Customer#000001549|Bm8PVyaAYfS0IFPhkXiVGL|24|34-263-284-6757|7050.56|FURNITURE|sts about the quickly unusual dolphins integrate quickly slyly silent platelets. quickly final instructions 1550|Customer#000001550|NgbaaI8wjR|20|30-722-982-9755|4742.59|BUILDING|sits. regular requests cajole boldly after the slyly special ideas. quick t 1551|Customer#000001551|GSs9E1btXLKkSgkCAyLohk1bOLuJ6|14|24-667-589-4141|797.35|BUILDING|ely ironic dependencies. quickly iro 1552|Customer#000001552|eR6My q7YdhYeBH jVxHHC mYpeNFNBDuG10|21|31-902-185-9642|956.02|AUTOMOBILE|regular accounts eat furiously slyly slow ideas. carefully final accounts mold furiously after 1553|Customer#000001553|zS2t71h5ssFkRFiB4EvNtWPqjexC1FaO1MeNutf|1|11-879-323-7032|5853.10|AUTOMOBILE|he slyly unusual packages cajole slyly ab 1554|Customer#000001554|axGq6Zieq8sy|7|17-462-295-6567|8996.02|AUTOMOBILE|latelets cajole furiously final, regular packages. furiously even accounts cajole according to the even 1555|Customer#000001555|7V1UD h0 oKL04nnKVzu7UCmFSL56|5|15-722-660-7220|-740.02|BUILDING|ng deposits alongside of the express, bold deposits cajole blithely deposits. even packa 1556|Customer#000001556|0KThJm1X9rQH3Me,EI2QW8HzrUKnsU,gvw9BwzN|11|21-170-549-6376|1462.44|BUILDING|coys wake slyly along the ironic pinto beans. evenly ironic requests use quickly. final packages are slyly. dolph 1557|Customer#000001557|cbF7Kpmtk4w1vCoqB,3Ev3XNnr|19|29-927-226-6896|3144.14|BUILDING| according to the final, pending account 1558|Customer#000001558|hHKBdZXfRUbMjnlX i8sGWu6|9|19-532-314-9903|8473.41|AUTOMOBILE| thin packages against the even ideas sleep slyly according to the pending instructions: 1559|Customer#000001559|0rOzDCEPki4zpeqXx5nW3ajIGdLN15XHeS|0|10-700-486-1040|4630.52|FURNITURE|ronic sentiments doubt carefully slyly even deposits. e 1560|Customer#000001560|yNFoAP4UcMlluwL1uNYvUmCgrn7GfDiTo3H3mzV|15|25-187-156-3225|9146.01|MACHINERY|furiously ironic requests alongside of the deposits impre 1561|Customer#000001561|11hKNqixtqQsCgZKu3DYu0VEx28g04|13|23-445-875-1233|1083.49|AUTOMOBILE|slyly about the quickly expres 1562|Customer#000001562|Rj0aTQUqnb1u4qOvWzb3|13|23-883-927-3910|3102.27|FURNITURE|eposits cajole. final instructions alongsid 1563|Customer#000001563|cb7 vuR7o4Z5KQqgd5yllan5Evum5|4|14-146-791-5866|8838.33|MACHINERY|ourts-- slyly regular packages sleep car 1564|Customer#000001564| kQ06G,BN4KWou6DYH|5|15-898-126-9264|-184.33|BUILDING|ual foxes wake: theodolites sleep bravely after the furiously 1565|Customer#000001565|EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW|2|12-402-178-2007|1820.03|AUTOMOBILE|ously regular accounts wake slyly ironic idea 1566|Customer#000001566|NfBldfDRJyOWXbZ47UJP2hGn6HF1zOZGJaOa|23|33-480-441-5244|7256.21|FURNITURE| final frets are carefully against th 1567|Customer#000001567|D XMRaJOpRqLttO8yiMZ4tYU1L2nUr|24|34-146-945-2364|7209.94|BUILDING|thely ironic ideas. ruthlessly pending sauternes are furiously enticingly regular pinto 1568|Customer#000001568|uOMsOfJ0raeSGqW9PMPs1sL5D pcO,fUaYsY6|22|32-780-340-3819|-576.58|HOUSEHOLD|ts sublate carefully ironic orbits. final, even accounts sleep toward 1569|Customer#000001569|4vO9w7ixKJ 5od18LqLvr,|6|16-108-793-2841|9416.38|HOUSEHOLD| the unusual packages. even excuses against the fluffily regular idea 1570|Customer#000001570|RMBVeVOCt002J1|13|23-319-685-6601|2106.52|FURNITURE|unusual deposits unwind among the courts. silent packages 1571|Customer#000001571|akbtXy3o6igP3n8C|6|16-661-716-6605|4250.73|BUILDING|silent deposits sleep. silent foxes acro 1572|Customer#000001572|wS4p6kZ8dz8WyKfAbhXeBUO3QJj|5|15-262-124-6233|6070.44|FURNITURE|instructions affix furiously slyly regular foxe 1573|Customer#000001573|pcC2rrIA2bwtSXkcBy8X5eoQBrfoGb7gT|10|20-170-955-7287|9831.27|AUTOMOBILE|ests use of the brave accounts. excuses are 1574|Customer#000001574|hgovcHRlq4 y|8|18-753-101-5745|1204.82|HOUSEHOLD|e regular courts affix after the thinly ironic reque 1575|Customer#000001575|Ntyf,WOVz9hrnESfXT6gBxej1eZjbwgdSEVvmRw|0|10-455-580-7646|7283.99|AUTOMOBILE|dugouts. slyly even deposits about the quickly reg 1576|Customer#000001576|ec9dOjmCD0iicQEc4iIff88zX4kFGHPZUPYX1sBg|22|32-430-540-7796|1006.44|MACHINERY|y bold requests about the ironic, regular theodolites haggle slyly instructions: ironic courts wake carefully. 1577|Customer#000001577|eaMmfsWJ7 USPFwMH|22|32-267-732-1345|6204.00|MACHINERY|dolites boost slyly even deposits. furiously ironic asymptotes ha 1578|Customer#000001578|KGMw1t3in68W4|17|27-348-227-6667|-365.45|AUTOMOBILE|elets. special, express excuses after the accounts promise carefully requests. care 1579|Customer#000001579|TI4GCerFzw2UgqQdzJ94|3|13-255-948-9257|4763.17|AUTOMOBILE|efully across the quickly express deposits. slyly regular deposits sleep above the blithely ironic theodoli 1580|Customer#000001580|Uc5lBMkU8F1zW56P Bo,8fbVlyCKs|13|23-651-166-3240|5587.12|BUILDING|ts before the ironic packages sleep furiously regular, bold dependencies: dolphins 1581|Customer#000001581|fCDyGbFmnkclr,031ny|14|24-603-456-1171|4669.01|AUTOMOBILE|lithely final deposits. quickly express platelets unwind 1582|Customer#000001582|Tw 9wNgPjMmsy1brAYW0|11|21-998-418-6615|7119.80|BUILDING|even accounts. quickly brave deposits haggle. 1583|Customer#000001583|og6OTS,QKN2BidNDSZd0yB,Tn8ls6TGnKUz |13|23-136-310-3804|2540.51|BUILDING|bout the pinto beans. bold, e 1584|Customer#000001584|BWzLMEnPG7tsF54M8kdGVd7zQCxiXniOP|21|31-675-590-3473|5305.86|HOUSEHOLD|. quickly busy deposits haggle carefully under the even, unusual foxes. carefully blithe foxes snooze fluffily? qui 1585|Customer#000001585|kMDzNCvICH1j7sLp8g0CFB8cO12tCS70VTp5wM7|22|32-232-514-3624|7651.05|AUTOMOBILE| carefully regular packages are about the carefully silent foxes. fluffily regular Tiresias wake furiously across th 1586|Customer#000001586|I76G9G7dkkigm162L|2|12-221-668-7869|-808.05|FURNITURE|rs cajole silently. ideas doze furiously! spec 1587|Customer#000001587|ztyGKSLXBi6r,uNDAxxDeWuWWUdfR1WL4maTC|17|27-437-149-3006|2050.48|HOUSEHOLD|closely alongside of the furiously pending foxes. furiously final requests wake about the ironic, ironic depend 1588|Customer#000001588|TOCHdXfBa1nhv26OP|1|11-700-437-5542|8372.34|HOUSEHOLD|onic asymptotes sleep carefully-- furiously regular accounts against the quickly pending pinto beans mold boldl 1589|Customer#000001589|As9UC67KvgdnJcZWfdz,|13|23-189-857-8090|-101.68|HOUSEHOLD|s boost final excuses. slyly ironic deposits wake quickly blithely silent requests. car 1590|Customer#000001590|c9ykZTFqi2xpKpNedlJ5,v03aqCbT|19|29-736-744-4365|5065.00|AUTOMOBILE|nusual instructions sleep fluffily furiously bold realms. regular, quick platelets wake slyly. final asymptote 1591|Customer#000001591|ZLJNTInWmiv9a1|9|19-142-875-7741|7470.70|FURNITURE|yly final foxes should have to use carefully. even, bol 1592|Customer#000001592|Bf Y0y,RTCY4z|9|19-565-127-5247|4042.57|HOUSEHOLD|n foxes. foxes cajole daringly silent deposits. sentiments sleep flu 1593|Customer#000001593|IAhXngV2KlKAbAQh4y6S7Vd|7|17-767-583-7374|5447.72|AUTOMOBILE|he special pinto beans. silent accounts sleep furiously final packages; 1594|Customer#000001594|8No1IYGij7|13|23-416-484-3099|4796.94|FURNITURE| final packages wake idly. quickly regular pack 1595|Customer#000001595|bJ6tl8L3gexrf9rdD,Nn9ojzg92|3|13-153-638-7545|1151.78|MACHINERY|hely final ideas. regularly daring requests sleep. silent excuses s 1596|Customer#000001596|fpSMWvE3a |20|30-259-884-2046|6900.08|AUTOMOBILE|foxes integrate thinly. furious, special packages sleep furiously about the asymptotes. final accounts s 1597|Customer#000001597|6pS2oH twoOdcRPVMT13YQQA YIFu|22|32-621-493-2342|5728.91|HOUSEHOLD|s foxes eat furiously final foxes. slyly unusual packages are never against the deposits 1598|Customer#000001598|bz91jr1NNiJ|9|19-439-414-8308|835.29|AUTOMOBILE|onic packages about the ironic, bold ideas are packages. care 1599|Customer#000001599|DbZoJYdsMvL8hELLlgjAvUZ|11|21-556-967-2607|626.23|MACHINERY|ly unusual gifts. even, regular foxes use slyly along the ironic, regular dependencies. 1600|Customer#000001600|GFgAlTCNWGZU4Gyk9glu8uX2vZ|20|30-563-390-7858|7027.54|BUILDING| sleep blithely along the slyly ironic deposits. blithely permanent accounts nag. fluffily ironic accounts 1601|Customer#000001601|jiy,cXiM41u9yIb1Vy|9|19-152-934-8225|2884.08|MACHINERY|ly ironic, even accounts. special dependencies detect. c 1602|Customer#000001602|Lum76wozwPDwPGgk7yFzLnG|19|29-236-186-6698|4645.67|BUILDING|. blithely even requests use slyly. unusual platelets snooze carefully r 1603|Customer#000001603|JHju hD17jZDMXprwVfC|11|21-880-121-2298|-149.21|HOUSEHOLD|tect slyly quickly regular accounts. daringly bold deposits are blithely blithe 1604|Customer#000001604|DXn5Lr8KjjMebZznHhSsX3n7T6J8UkWdYw|6|16-960-140-9357|9079.75|HOUSEHOLD|odolites. final frets need to grow according to 1605|Customer#000001605|PLOEPrgnofqWl3|15|25-483-103-9669|9396.10|MACHINERY|sly unusual requests. furiously final theodolites are boldly unusual instructions. ironic, permanent acc 1606|Customer#000001606|rIhuh0JIXA caaG|19|29-275-181-3687|2244.46|FURNITURE|y slyly unusual accounts. even foxes print furiously daringly even waters. blithely unusual deposits boo 1607|Customer#000001607|JrtvTEYpFdqK68WSabydH6dz9Opj6X5orhrIYeHY|23|33-529-928-4089|1543.75|FURNITURE|ites sleep furiously quickly pending somas. fluffily special 1608|Customer#000001608|jGjdmzMbF05pXU5STryOYpL9orgJ6F|6|16-897-134-9884|5827.14|BUILDING|oys haggle never. regular ideas solve. express dependencies cajole furiously bold ac 1609|Customer#000001609|YuJ96cGZd lzZ5jo0HI6OAi,7b12GYDC,|20|30-784-105-5546|1710.71|HOUSEHOLD|unts alongside of the regular packages haggle carefull 1610|Customer#000001610|8T,m b4Gwjs9j|18|28-219-755-5479|6861.85|FURNITURE|ainst the slyly ironic instructions. blithely final deposits 1611|Customer#000001611|QX3yB3eqWVsGuy7WetBKk6U6s CXl|18|28-997-908-7044|1148.91|BUILDING|ests. furiously express instructions wake slyly! fluffily express frets haggle. quickly even instructions do 1612|Customer#000001612|oRmhlGYt71UyFdgI4KvPxF|16|26-493-547-6969|2613.29|AUTOMOBILE|oost blithely about the blithely express packages. reg 1613|Customer#000001613|grC4vU,xdQCWgrPJzj|19|29-636-508-4398|7539.75|HOUSEHOLD|onic excuses. regularly unusual deposits sleep. slyly ironic accounts nag carefully. furiousl 1614|Customer#000001614|7BofGHd,3lr2wda7i|1|11-986-549-9647|9609.43|HOUSEHOLD|to beans use blithely. unusual, regular waters along the slyly bold pinto beans wake sly 1615|Customer#000001615|AWqpPsmhK,yirQmha|18|28-449-655-8989|1764.25|MACHINERY|egular packages. carefully express ideas wake fluffily fluffily idle deposits. furiously final dinos n 1616|Customer#000001616|hZ7KvTsImg5hRUWmHXpkZGvhFe|20|30-752-506-2492|4635.15|FURNITURE| pending tithes. furiously blithe instructions boost. thinly pending ideas use careful 1617|Customer#000001617|SfX,PYtDB3h2gdmDD1JMN,gKIIVqo|2|12-677-936-3084|9911.51|AUTOMOBILE|y ironic requests above the regular requests haggle silent asymptotes. daringly special instructions detec 1618|Customer#000001618|efEU9gOnX05FfeJAyNMup|12|22-326-603-9101|7795.47|AUTOMOBILE|refully pending accounts haggle furiously dependencies. 1619|Customer#000001619|qM0OslnGfoXRS4YhYUDaUd6cXDDCPc5Ppke8CU|13|23-371-869-2433|6511.83|BUILDING|bout the slyly silent accounts. fluffily ironic deposits integrate carefully against the ironic, express pinto bea 1620|Customer#000001620|p2BIMDiWvXUWlb FXxIukQZI|5|15-151-404-4005|3324.77|FURNITURE|regular courts. carefully brave deposits 1621|Customer#000001621|5I2xwuWad5n73M5zM,Dj|16|26-615-141-1919|8358.51|AUTOMOBILE|efully silent deposits shall nag blithely a 1622|Customer#000001622|HEfEM41ad6Tar1EH526Q9cxe3Pi|5|15-120-999-7103|9617.34|HOUSEHOLD|e requests-- carefully bold packages are. 1623|Customer#000001623|azlsfbL,uqLb2T1wCn5yQ33YK5KvJ8Fo|23|33-467-523-3238|2746.27|HOUSEHOLD|ress carefully quickly special depend 1624|Customer#000001624|A8VfM,awG8VPydormLPcaw|17|27-822-330-4723|9566.20|AUTOMOBILE|accounts among the brave packages haggle finally careful 1625|Customer#000001625|qQ53P2z9Mnocb2HG9u|9|19-868-381-8072|1189.78|HOUSEHOLD|nts use furiously regular pinto beans. unusual, regular dependencies detect blithely 1626|Customer#000001626|Qqvd9BwVQQ133oxNXb8N1i6V3l9z7eu3A|22|32-751-259-8740|7564.80|FURNITURE|ar accounts haggle never. quietly regular instructions are carefu 1627|Customer#000001627|RV5yXOOv0tjeqxIoRtIw9lKU3UK|10|20-566-949-4093|1673.17|AUTOMOBILE|fully after the blithely regular warhorses-- quickly even 1628|Customer#000001628|xOOuECIqRpweZwxZRgQpb2guNYVE|18|28-241-420-5429|3002.83|MACHINERY|s use fluffily. slyly silent foxes 1629|Customer#000001629|eGVew4ADiILjquTPiTeVS9|18|28-413-295-5895|9601.48|HOUSEHOLD|sly express dependencies abou 1630|Customer#000001630|mFqtTXCA4QaCqP7yXsTlk|24|34-375-163-1478|214.28|FURNITURE|furiously final excuses. accounts cajole blithely ironic pinto 1631|Customer#000001631|TEZnHT8B6dqZw9,OoyNrOJs PlT2QfZKk|13|23-875-411-6115|2401.42|BUILDING|y ironic packages. fluffily express deposits sleep by the slyly ironic requests. regular depos 1632|Customer#000001632|9SoJbgMR23pwWXyE|24|34-183-653-4603|-378.16|AUTOMOBILE|p across the ironic foxes. furiously close dependencies det 1633|Customer#000001633|kO5Tq2Y W,NklARS,|11|21-575-247-9010|162.02|MACHINERY| ruthless pains alongside of the even platele 1634|Customer#000001634|mTRMQ9143TTe5kHsD2FdNE7proZ|24|34-186-980-9064|4030.37|FURNITURE|s are thinly fluffily ironic requests. pinto beans cajole blithely before the regular 1635|Customer#000001635|HjISwY7cr50HVcC81T7MnYJ7byRMXrMgB7RjsV|23|33-974-490-5943|9435.42|HOUSEHOLD| the blithely final dependencies. slyly ironic instructions are furiou 1636|Customer#000001636|t1VhiA5gssjGA4,o5b2e8WJHsaBAmCfm4G|20|30-559-573-5410|3228.88|FURNITURE|to beans are quickly carefully regular dolphins. regular, ironic dolphins de 1637|Customer#000001637|dqGWqAXF4JuL7FcYH7r9dnH3MiT0S08VS7KgD|15|25-177-814-7863|2844.51|HOUSEHOLD|the quickly ironic instructions. packages poach blithely express instructions. even packages haggle sl 1638|Customer#000001638|Gxgb1kyTwOAoVu0fqaRQk4KCyWzkULGTy6tkpdOx|17|27-548-377-6273|-411.15|FURNITURE|lly regular accounts! express attainments maintain carefully. furiously pending pi 1639|Customer#000001639|QWbeop69wQqRFQbySM7WqPGTSd7fW6QMFYIjL|1|11-304-833-1391|6651.19|AUTOMOBILE|nt deposits cajole slyly blithe pinto bea 1640|Customer#000001640|lGeZbMEg03r24lUuK|8|18-270-772-6060|2622.28|BUILDING| regular packages are slyly among th 1641|Customer#000001641|XT5DXFdGy4kjb|12|22-791-967-1788|4611.37|HOUSEHOLD|unts through the slyly pending platelets integrate carefully dependencies? blithely ironic 1642|Customer#000001642|UgLWC4Pw,XZX52b8hcEixGxk,J|9|19-134-303-4344|6248.64|FURNITURE|uick requests. furiously ironi 1643|Customer#000001643|,vdC1qp8aweR4z4 sTdnhujyZn,|13|23-553-752-7340|1982.44|BUILDING|ites. slyly pending accounts across the dogged excuses can are according to the bold packag 1644|Customer#000001644|la3oZuddBtIVanskRXO8|8|18-235-782-9940|8808.00|MACHINERY|thely final dependencies above the deposits boost fluffily above the ironic, express theodolites. ironic theodol 1645|Customer#000001645|2gNNcbkeFHKEgl4WSW7G8XpXL0VW,6MtTc0G|16|26-174-526-8279|7085.79|MACHINERY|ic theodolites. furiously regular theodolites nag furiously carefully pending pinto beans. blithely unusual de 1646|Customer#000001646|RQ,TryFh5loGPxszvCgRncdO5kM daRcgON|24|34-268-537-8282|2854.29|MACHINERY|re according to the ironic, ironic requests. ironic packages wake after the special requests. 1647|Customer#000001647|aLfdvxbHzfKz2CAdiOgKiJ|16|26-784-323-3431|9987.11|HOUSEHOLD| final deposits nag. sometimes final dependencies d 1648|Customer#000001648|e3oTXQ7OOTzwcRFXr|23|33-718-723-5373|2389.14|AUTOMOBILE|d sentiments eat carefully unusual instructions. unusual war 1649|Customer#000001649|7n8CvxEE4tthklLyRNZIMQds5rruRPiQLLdV|4|14-308-532-5953|2271.45|BUILDING|te across the blithely ironic packages. blithely final accounts among the quietl 1650|Customer#000001650|6mpXm8FQzetQ7wA1pzEmuYVcVp5 fnDk|24|34-295-469-8581|4183.71|FURNITURE|express ideas along the regular requests sleep furiously alongside of the fluffily ironic courts. idly final 1651|Customer#000001651|whCw6gMwEuls sCrsaB,DQI0,|3|13-593-198-5028|1614.35|AUTOMOBILE|eans. blithely final requests according to 1652|Customer#000001652|uDJ6cxL10W sEd4,O7,rdoxst2Sp1Ij72Jb1|7|17-670-200-2924|4335.18|MACHINERY|ainst the close pinto beans. furiously final asymptotes across the even deposits 1653|Customer#000001653|PQFMr5tmEgBCF7rww29Vc yMrHY9HJk|8|18-150-322-1853|6338.28|BUILDING|ic packages. platelets along the slyly pending dependencies sleep slyly alongside of the slyly 1654|Customer#000001654|igHSnmh 6yMC3vF|5|15-299-167-7023|1522.54|AUTOMOBILE|thely unusual requests boost slyly special, final requests. quickly exp 1655|Customer#000001655|DjCE7uReh1B,,ikdShz9W3PCfqkJow|5|15-306-607-3769|1214.81|BUILDING|s. special, bold requests haggle fluffily. carefull 1656|Customer#000001656|m3BvBNeQ1O eV1Bnn3y,MkEx7Io8GkfQ|19|29-904-708-2645|-664.93|AUTOMOBILE|furiously pending packages cajole regularly requests. fluffily even foxes wake blithely. furiously bold 1657|Customer#000001657|MSSbpflkYXCciBa|12|22-113-887-8875|9376.24|FURNITURE|ckages nag according to the regular 1658|Customer#000001658|5ZAXRv0hnyOcjObHR1ScVOZ77ncI,0|21|31-949-978-6932|9525.19|BUILDING|press instructions. carefully express requests are against the blithely final 1659|Customer#000001659|6,g4PcDD8cCUdAKNbhAmwyG3lqEKuUbq|11|21-545-972-9730|4982.96|MACHINERY|ake. regular, unusual instructions sleep quickly carefully even foxes. regular sauternes acco 1660|Customer#000001660|ClPcSJym47fEQW78Kt4|21|31-870-788-5315|3581.59|AUTOMOBILE| accounts breach slyly. permanent deposits are furi 1661|Customer#000001661|IRXXgB,YgRc078y2i1C,87 1wZ|0|10-582-676-4365|1760.90|HOUSEHOLD|s. ideas play slyly slyly bold theodolites. furiously ironic packages na 1662|Customer#000001662|fImm0 WZ JU39aNmhsh5WKcnCqXW|13|23-691-593-1242|3333.02|MACHINERY|nic accounts sleep carefully across the fluffily unusual pinto b 1663|Customer#000001663|7PGRXPj1HXVQUbcL S2|16|26-507-387-2886|4085.85|HOUSEHOLD| slyly ironic deposits. daringly regular requests haggle slyly. ironic dependencies cajole furiously 1664|Customer#000001664|LWrtr,G ifu9pwmSc2HknzWQS,o0FOMGucsq7Rdh|12|22-597-130-8584|6912.53|MACHINERY|thely ironic requests haggle slyly. quickly ruthless dolphins are slyly ironic requests. carefully special 1665|Customer#000001665|26NoCK4dbtU7jmEhrXSuq9rtQWM042UYODGFm,|9|19-542-708-2762|5869.13|HOUSEHOLD| wake furiously carefully even id 1666|Customer#000001666|AFCMGLIrCORZavTw7YX1dAVlJIk,aYlH|22|32-587-573-6083|3562.78|MACHINERY|fluffily regular ideas nag furiously. even accounts haggle carefully among the furiously final 1667|Customer#000001667|aEGS4v41BVwqZylqNvPj|16|26-528-257-9769|4649.03|AUTOMOBILE|nments; unusual deposits wake according to the regularly unusual deposits-- fluffily express requests sl 1668|Customer#000001668|CS067JF7eX,ax,vrx8wx|23|33-184-926-5421|1305.72|MACHINERY| ironic platelets must wake fur 1669|Customer#000001669|i38,,EDjrqVpk1UKXsl9cCdAwS,HpcFqPS|6|16-172-628-3560|9180.72|HOUSEHOLD|quickly unusual packages are. furiously express ins 1670|Customer#000001670|YP A2c1cFpn|18|28-571-377-3401|1472.96|AUTOMOBILE|ge silently. packages haggle quickly final packages! ruthless dependencies cajole ruthlessly q 1671|Customer#000001671|6yWHFFBO5YDpHN,YmYEpxulL|2|12-269-842-9419|4030.97|HOUSEHOLD|otes run slyly ironic deposits. carefully silent requests use slyly carefully ironic epita 1672|Customer#000001672|ZqEat15B3nCQI4MaRoxdfhN3WIH96vWpUs80z4|23|33-169-930-6985|8586.04|FURNITURE|sts. thinly final accounts are about the blithely regular foxes. blithely even dolphins promise blithely speci 1673|Customer#000001673|uQi2r9akwS4LNd7XQEa|3|13-713-161-3704|5714.73|HOUSEHOLD| slyly. dolphins wake carefully even deposits. quickly even dolphins grow slyly. express, regular deposits 1674|Customer#000001674|VDSnyhnkkFA1CKYDjdMx4Hpt1QaZ4g,1cy9Fnr|3|13-396-137-7834|4568.47|BUILDING|equests use furiously at the carefully ironic requests. unusual deposits boost slyly carefully even fox 1675|Customer#000001675|YvuT73pnh06wLlgAyTO1ZyZ4w,2e5Wk MGnbFO|24|34-888-827-2907|261.00|AUTOMOBILE|eep carefully. even deposits cajole furiously around the boldly final deposits. dep 1676|Customer#000001676|WgQKmlxIcGQz86n5sQMbWUu8cg7UG7W3r|5|15-612-997-6342|6504.47|AUTOMOBILE|s. even asymptotes boost blithely 1677|Customer#000001677|7PgxZfn 6hX3gSjJzRq|5|15-121-397-5027|650.89|MACHINERY|inal asymptotes haggle carefully packages. slyly ironic req 1678|Customer#000001678|xq R0 eIkV019MjY8yRdj r,Gfd|4|14-624-973-2343|3396.57|MACHINERY|sits. fluffily even instructions are again 1679|Customer#000001679|EBr12ymXS5u3,9Bh6Cd8VCsmJ9cOR8nuS|18|28-730-907-3502|5172.53|HOUSEHOLD|eep quickly. furiously express excuses haggl 1680|Customer#000001680|jYhr0a6R8XTw8RR3XJQ1kToU5H|18|28-465-621-9214|-203.01|HOUSEHOLD|ithely unusual patterns above the unusual ideas cajole blithely according to the fluffily expre 1681|Customer#000001681|QxfVn4jW30|16|26-375-137-8121|6996.10|HOUSEHOLD|tions among the ironically final accounts haggle according to the carefully unusual instructions. fluffi 1682|Customer#000001682|8TtqhjtXrYlzMxQ17N|17|27-438-398-2565|2459.75|MACHINERY|. blithely bold courts sleep carefully except the furiously unusual platelet 1683|Customer#000001683|ecOqgCbaUID9JwYZuvSrFxXH9dIDaV|22|32-209-661-3831|942.28|AUTOMOBILE| carefully unusual foxes doze according to the platelets. bo 1684|Customer#000001684|7t,Vo69PIG3t,ncWkzoLCJ8A,V28nMkK|6|16-197-588-1571|5928.82|AUTOMOBILE|d cajole about the ironic theodolites. fluffily special deposits about the enticingly blithe 1685|Customer#000001685|3Mg0g4AXNPa|8|18-694-638-1767|5621.33|MACHINERY| sheaves. regular packages nag slyly afte 1686|Customer#000001686|EYR2WxcOKG 4rIlcO9wbkAtID7PJOVkcPaC|12|22-241-190-8777|7782.48|HOUSEHOLD|osits sleep furiously about the closely ironic pac 1687|Customer#000001687|lNxhAZMB,t1bbxFz7UXI0gFWhw|23|33-345-542-8289|8151.36|AUTOMOBILE| boost blithely at the furiously special requests. ironic, unusual requests 1688|Customer#000001688|KE2TYjMt08|3|13-420-827-4701|3929.06|BUILDING|ckages use slyly. pinto beans haggle regular instructions. blithely even theodolites cajo 1689|Customer#000001689|oYgoEWSydzBD81VB3q20DEx8TgSUX6qio,zpL83p|23|33-974-102-7427|705.64|MACHINERY|y silent requests according to the slyly final deposits poach ac 1690|Customer#000001690|v2dVbJH3RxbBj5Wk5btJdzv9K35jXAoxhRYVthO|1|11-869-223-3212|-381.24|MACHINERY|riously throughout the slyly express accounts. slyl 1691|Customer#000001691|BvajZGLJDqzvJfZKsuVjdwaixCO|12|22-649-185-6921|3367.22|AUTOMOBILE|side of the ironic packages. fluffily special asymptotes among the slyly unusual foxes haggle slyly b 1692|Customer#000001692|C3n33KUNrCXK|7|17-625-330-9211|6921.50|BUILDING|. quickly unusual courts use furiously carefully ironic grouches. blithely unusual accounts sleep carefu 1693|Customer#000001693|k9j7wuuKPs8gE|12|22-402-777-3279|2747.22|MACHINERY|carefully special packages cajole. blithely bold 1694|Customer#000001694|jCLu0ZDrLdq7wFEJb|4|14-609-696-9902|3535.57|AUTOMOBILE|the pending, special Tiresias. fluffily even deposits wake carefully theo 1695|Customer#000001695|ihGFlPO39VGgr7xRcR7AM1BFKn9pDq3C|16|26-263-151-5237|9258.82|AUTOMOBILE|press, express packages nag. slyly special asymptotes sublate. regular, even 1696|Customer#000001696|HIJoLNtvjJbh5H0PturTaOBtSAQ3T,j7GSqq|16|26-115-967-9585|4978.11|BUILDING|eans across the even attainments haggle carefully above the furiously express req 1697|Customer#000001697|TQj18iiC1gziLOnTileoy|24|34-288-313-5272|-913.24|FURNITURE|nding dependencies wake alongside of the sly 1698|Customer#000001698|Xzkyij4D5OOWYsaWsucYV0|22|32-926-560-9683|5109.01|HOUSEHOLD|uriously ironic packages cajole slyly! regular deposits alon 1699|Customer#000001699|lGzEu5g4oOROn4QvjK8fEd Z,Y9Vn7IV7EnlE5|1|11-655-675-5843|-813.01|HOUSEHOLD|ngage ruthlessly alongside of the carefully 1700|Customer#000001700|DK3nJU9doE2BtBjQTFApwnLxOCSD |21|31-868-665-9539|6683.70|MACHINERY|ound the furiously final hockey pla 1701|Customer#000001701|IbyUBDvH,eRszYTbnEDHGu16B4UsJSbQaA7F |3|13-397-730-2856|9986.13|FURNITURE|gular ideas. deposits about the busily unusual deposits are furiously ironic theodolites. quickly e 1702|Customer#000001702|ZUf5SwR,j3HdY TBel7Mk|19|29-110-823-3729|8048.90|HOUSEHOLD|e quickly pending accounts-- carefully special deposi 1703|Customer#000001703|7qJL pH9GSS4BZ Nc31|19|29-687-882-9664|8889.69|BUILDING|the ironic, final accounts. qui 1704|Customer#000001704|G4lZ0VRWfLKldLDFR3,bA|16|26-425-543-6950|5145.40|HOUSEHOLD|fully express requests about the carefully regular deposits use carefully 1705|Customer#000001705|lvZ9qSNhUMiE0LTOzmU7,NgjBo6VvcGrs|11|21-470-157-6516|5688.31|AUTOMOBILE|s sleep carefully. fluffily regular accounts haggle furious 1706|Customer#000001706|FBx04exFTAFFRA3G,UR9Q2XSM1c8Uopaal2rEFv|2|12-442-364-1024|455.15|HOUSEHOLD| beans after the ironically pending accounts affix furiously quick pla 1707|Customer#000001707|MjHpj4aS20ftMyK5EMEk87p|11|21-859-412-6010|2714.86|AUTOMOBILE|. carefully even requests nag fluffily after the sometimes regular instructions. quickly bold foxes among the idly 1708|Customer#000001708|BdT2freRSXKa31JLnSBliCXmhi8J|2|12-857-766-2851|9702.83|AUTOMOBILE| always. requests are furiously packages. slyly regular tithes c 1709|Customer#000001709|x0 eZbj1iyKLgjF8qImD|11|21-877-394-8667|9588.88|AUTOMOBILE| deposits. blithely regular frets 1710|Customer#000001710|6kwF3TSVbf,JAh3PAjc0NSHHvdEGWjxVf2us|10|20-393-759-9313|8409.91|HOUSEHOLD|gular asymptotes. furiously silent excuses sleep never-- blith 1711|Customer#000001711|Mhg8c9IAFb8G|15|25-302-946-6337|4421.61|MACHINERY|gle carefully. final, even deposi 1712|Customer#000001712|Qdv0r7aA5tmEZw JkozgH|18|28-959-477-6941|7013.72|FURNITURE|le blithely. regular, regular dependencies integrate slowly a 1713|Customer#000001713|saqFezmCXyr2f2sGR3WexM8MSf03S|15|25-566-819-3545|1627.04|AUTOMOBILE|cial deposits. requests alongside of the 1714|Customer#000001714|6xIlLyed lGGxdneig8xdrvHZYHuLldIGeJu|23|33-251-747-7039|8569.50|BUILDING|ar packages. carefully final accounts wake carefully. slyly even ideas above the closely special reque 1715|Customer#000001715|TNwfsu3dIIti|12|22-283-174-5611|2587.36|BUILDING|slyly regular accounts. quickly furious accounts sleep blithely alongside of the carefully special packages. ironic 1716|Customer#000001716|FmPtChYDv7s7KX5Zi8Ug9SGMajKrQYRuv|20|30-299-232-8737|779.07|BUILDING|. final accounts use. furiously ironic asymptotes d 1717|Customer#000001717|HufIAAW0Xbs3qoYpAVxk4KKvm,F|3|13-450-115-4347|1715.40|MACHINERY|s doubt. blithely silent accounts try to haggle: blithely ironic pinto beans boost furiously. carefully unusual a 1718|Customer#000001718|z2SLViCW5QBh8FDiy3|24|34-842-694-7686|8697.88|FURNITURE|s sleep furiously about the carefully pending asymptotes. 1719|Customer#000001719|eOix0jv6gCP34oQBO2i2z1UugzE1hwWx28n7Uog0|9|19-261-141-3893|4257.24|BUILDING|ully furiously ironic courts. even asymptotes sleep! regular, ironic dependencies are. carefully silent theodolit 1720|Customer#000001720|CIRvBtD2pSJ2b2hoqOxhj|7|17-681-485-3576|5496.84|BUILDING|sly even accounts sleep carefully-- pending excuses sublate slyly. slyly unusual foxes al 1721|Customer#000001721|qRNwOW8G9E5fOvG,,W7HDgv|9|19-288-344-3668|5484.48|FURNITURE| furiously. foxes cajole. instructions after the bold instructions 1722|Customer#000001722|XqgWxT4hlg2EjS0HaV6XQwclVGN09kfwUJa iX|1|11-252-319-1249|6718.86|BUILDING|posits. furiously regular requests are fluffily ac 1723|Customer#000001723|riRitYAsJ0nPLmAnd5JX,8QpE2F9iZIKXAg|22|32-609-708-8185|6893.86|AUTOMOBILE|e the carefully unusual theodolites. bold, even deposits alongside of the pending packages s 1724|Customer#000001724|oCZhgKIkixlfOYiqSXqzeiUAV9ctBQSRsW55G6n|11|21-327-332-4432|123.03|AUTOMOBILE|ven packages. fluffily final deposits nag. carefully ironic dependencies unwind b 1725|Customer#000001725|deDnuDKB3fZczOWTxwxutP|6|16-227-800-7867|8778.07|FURNITURE|he furiously stealthy deposits. deposits haggle quickly among the final packages. carefully expres 1726|Customer#000001726|Gi5q77BDziKnHUbMcbOrja22sv|13|23-199-630-2326|-879.29|BUILDING|lithely final foxes around the ev 1727|Customer#000001727|g7iJ1HSEmxE1sd4lWLV XcE64HA2JHYqPEJBA|23|33-952-217-5029|3628.03|AUTOMOBILE|e regular excuses detect carefully spe 1728|Customer#000001728|ZbokjPCGrcUuysUjYtzQRK9gQSSNK CkFo3rh8i1|16|26-560-950-6812|1223.09|AUTOMOBILE|regular pinto beans; bold packages sleep. blithely ironic requests boost carefully furiously fin 1729|Customer#000001729|qc3FBSbJHHDIEkku69mWsz,KO|5|15-361-394-6195|6479.07|AUTOMOBILE|y even depths sleep silent accounts. evenly final epita 1730|Customer#000001730|p8wSGXOVmXO6rJBW3jgEjcFYRKW0jVY|1|11-216-159-3328|2743.27|HOUSEHOLD|efully final somas hang slyly. dependencies over the blithely regular requests haggle carefull 1731|Customer#000001731|YOLyjZXp0eenovcSOSR4cShVAvSLDG84VTtB7|2|12-671-588-4662|3075.42|MACHINERY|sual gifts above the blithely bold packages haggle slyly pending attainmen 1732|Customer#000001732|2ZHJonGizgo0QYzPxNF1PubW|2|12-505-455-1597|8512.55|BUILDING|arefully even deposits would cajole about the sometimes final requests. slyly silent ideas accordi 1733|Customer#000001733|RO7fzBGXbovRqHEQln,fPJza9UAmIOe,Q|21|31-150-385-3780|2568.57|BUILDING| pinto beans nag! furiously pending deposits grow blithely even tithes? pending, ironic requests impress. regular do 1734|Customer#000001734|Hj4hmdNc2sRBgG5YO4mz5q 9|8|18-264-978-2762|3888.89|AUTOMOBILE|inal asymptotes along the final platelets nag above the quickly regular requests. slyly final request 1735|Customer#000001735|59,bPx BNSZ1YJ9thet9N|10|20-343-415-9202|7495.94|HOUSEHOLD|efully along the special realms. ruthlessly brave courts cajole carefully carefully silent reques 1736|Customer#000001736|t3 1Dv36zNjBXauc2HKcSLJk,QRu4t|7|17-572-741-5408|-686.49|HOUSEHOLD|s! slyly sly requests above the expr 1737|Customer#000001737|Mdj2vXS04lbiu1hlBPSpA6XliT0yq,XL|5|15-973-473-2767|-917.33|MACHINERY|wake furiously even deposits. busy, bold grouches integrate q 1738|Customer#000001738|dKNfooI8lr6G0yb19Oug|8|18-263-208-3553|2290.69|AUTOMOBILE|mptotes. courts about the ideas nag requests. quickly unusual multipliers print after the pen 1739|Customer#000001739|lhga8XGNRXPfJTotgUTn5qc4ush|12|22-216-512-8595|8526.70|FURNITURE|ut the fluffily regular foxes. regular accounts haggle. special deposits affix carefully. even, final 1740|Customer#000001740|7SEO3ug3RMgpphEQ6Ozn2N22VGruDJA|19|29-148-722-9708|9323.31|FURNITURE|ronic foxes sleep carefully forges. regular, ironic pinto beans promise carefully across t 1741|Customer#000001741|W9G SogT0038gQBdoLtFsexbHqNl|20|30-605-828-6050|-0.28|FURNITURE|sits wake carefully final Tiresias. carefully unusual requests sleep 1742|Customer#000001742|t3PI5OUN02V0BSuoWRvpEcxhY6qX 3IxBOq|20|30-777-207-9522|4273.99|AUTOMOBILE| regular theodolites. carefully ironic packages are. even, express foxes h 1743|Customer#000001743|F 5AwQdqqG4K q Ra2AZ0DIsKLNwhtIgHVxIr|16|26-867-792-6806|985.87|HOUSEHOLD|riously regular attainments. furiously unusual pinto beans cajole slyly daringly r 1744|Customer#000001744|cUBf1 YMJEgbt2XDeQWD4WinTu4iFIF|17|27-864-312-2867|1436.96|FURNITURE|egularly bold, ironic packages. even theodolites nag. unusual theodolites would sleep furiously express 1745|Customer#000001745|pAo6p8Q,xr4Y|22|32-624-467-3275|3907.29|BUILDING|final instructions affix fluffily alongside of the blithely unusual pinto be 1746|Customer#000001746|f8Ku1TqpFJ 4wU,s6clle8G|17|27-110-285-2511|8774.31|HOUSEHOLD|ions cajole ironic theodolites. slyly even requests wake carefully. blithely regular frets integrate furiously agai 1747|Customer#000001747|xSJopUGOWIGwa2QJA9mbIzcLAA8OM|15|25-493-381-4100|1962.62|MACHINERY|furiously furiously ironic theodolites. stealthily reg 1748|Customer#000001748|4MaVm9Yox5xeu|13|23-496-856-9633|5779.19|AUTOMOBILE|y pending ideas. furiously regular instructions are. quickly ironic depo 1749|Customer#000001749|lgY9SnA8QnkA7x2LAje7MGsVYB|4|14-247-462-3766|9788.66|AUTOMOBILE| carefully ironic packages. d 1750|Customer#000001750|PBWOGiTcFiFucs8kN2h ,ACNS NC|16|26-538-454-1606|110.26|HOUSEHOLD|packages are stealthily final frets. regu 1751|Customer#000001751|8KIBa2IJPXquMkWYLAccUT|7|17-460-999-8173|3151.94|HOUSEHOLD|ithely regular, even accounts. unusual packages lose furiously; asymptotes nag si 1752|Customer#000001752|CZzZOOncqNdRX|5|15-542-683-9661|7585.99|MACHINERY|uriously unusual accounts. carefully unusual dependencies shall have to play sly 1753|Customer#000001753|XlL3uXUfcHDfQ 4unoq pbTPbUzO|11|21-214-570-7769|5787.81|MACHINERY| carefully regular asymptotes a 1754|Customer#000001754|wXsAA1ZgkeofaPGeZaIe|0|10-154-971-3056|759.00|BUILDING|requests. stealthily pending requ 1755|Customer#000001755|emfcQnwjix1Ul0eKbHP|3|13-333-888-8618|5389.05|MACHINERY|detect blithely about the fluffily regular foxes. slyly regular requests wake quickly about the final foxes 1756|Customer#000001756|q8WlGuUrrao7fWFp4pZE|19|29-962-368-3523|8817.44|MACHINERY|press pinto beans. accounts could are blithely carefully express dolphins. blithely bold deposits detect. eve 1757|Customer#000001757|jY0jJh ww3hLh8eLmmJvKhIL47ka2j1uAys|6|16-868-947-1151|830.36|HOUSEHOLD|, bold instructions integrate slyly final accounts! carefully final accounts caj 1758|Customer#000001758|xvGuRiXuKNuXABjdfERW8jl7b5FdFNo|16|26-263-427-8796|5746.38|FURNITURE|old bravely. ironic dugouts are slyly. carefull 1759|Customer#000001759|2hzwTzhbdHJlHbT0jWD6lKTGA|20|30-176-851-5565|8396.37|MACHINERY|tect furiously. permanently even requests ac 1760|Customer#000001760|HcnL7r33sIo1SAwc5|9|19-539-811-6674|7775.38|BUILDING|ts are after the regular theodolites. dar 1761|Customer#000001761|ypll07IxByF9atO nGkJo7R8Tds8Wq |21|31-376-476-3692|9846.97|BUILDING|en requests. accounts sleep quickly around the carefully e 1762|Customer#000001762|LD,Y0Xj io6wJYOZJI|15|25-319-745-1408|1171.92|AUTOMOBILE|ng accounts. thinly regular packages across the furiously ironic accounts wake fluffily accounts 1763|Customer#000001763|Co2ATzqcl61Cd|1|11-738-302-2342|4490.23|AUTOMOBILE|manent accounts use slyly regular requests. carefully special packages after the pending 1764|Customer#000001764|rlpLd5W4dwxpO7aAVKa7YlZHnwOQCziuByOrK|13|23-308-606-7773|5331.37|HOUSEHOLD|sits sleep slyly. even, ironic instructions breach permanently regular foxes. furiously silent platelets abo 1765|Customer#000001765|YYXRIFQJ4V9paDLFCd13|9|19-538-728-6104|7409.60|AUTOMOBILE|final, special deposits sleep carefully furiously special deposits. evenly ironic p 1766|Customer#000001766|n4uulup7FP1ZLXvSRHveY,TXBOTxs0pj67|15|25-155-396-7616|200.00|BUILDING|lly express foxes. blithely final deposits cajole doggedly regular theodol 1767|Customer#000001767|ddqHp2,Ylt8vN8Pf|12|22-421-403-9852|4369.15|AUTOMOBILE|e carefully atop the even theodolites. pending, ironic accounts hinder among the carefully regular foxes. c 1768|Customer#000001768|kId4M 0RG9dW9Po|17|27-746-769-7890|5356.96|AUTOMOBILE|s. carefully even requests are. spe 1769|Customer#000001769|mrche55tm5KGuJb6lGWqCeKKIlZ rj4|10|20-708-644-8998|8457.18|AUTOMOBILE|furiously silent packages. bold, bold theodolites sleep quickly against the quickly ironic theodolites. slyl 1770|Customer#000001770|IonoL5JlAeGYQFKq2k1sKH0QscvwaAYdrpFSxHV|18|28-976-259-9388|6876.37|BUILDING|ole. furiously regular pinto beans affix blithely inside 1771|Customer#000001771|evmQypmt DbfynZ4bXvm0KUNtyvynyDp3zjcXX|11|21-345-763-5234|3151.21|BUILDING|tes wake carefully according to the unusual accounts. fluffily regular theodolites na 1772|Customer#000001772|AQla93nCHVF6jkq,J|13|23-887-525-9315|6394.26|MACHINERY|e slyly ironic requests. furiously bold courts thrash blithely bold foxes. slyly express foxes 1773|Customer#000001773|StUAItIdmWQdpF6Gz|19|29-729-630-8987|7378.35|HOUSEHOLD|are furiously after the carefully unusual dependencies. ideas sleep furiously. slyly bold packages 1774|Customer#000001774|5pstbh4XaxP91 ,wNQFbR|8|18-753-365-9994|2922.61|AUTOMOBILE|furiously theodolites. blithely unusual ideas main 1775|Customer#000001775|RUY1tS8uCKV1DrB|2|12-544-332-4550|3221.15|MACHINERY| the furiously silent packages. pending, final accounts breach. instructions according to the quick 1776|Customer#000001776|,iyGlh4 Wrn2|22|32-425-716-3547|4801.42|BUILDING|l packages. slyly regular requests wake ruthlessly above the blithely bold dependencies. unusual, ironic in 1777|Customer#000001777|54GpDEcWWIDVMRjP3|24|34-469-776-4539|-535.08|MACHINERY|ar packages? blithely regular instructions along the fluffily ironic theodolites believe ironically aro 1778|Customer#000001778|XxR5jsS8OA1xjtocU,KS6F0Pte4Go5|4|14-504-368-5987|2772.31|BUILDING|eans. regular, pending packages haggle blithely regular p 1779|Customer#000001779|3iow2GjE85s8GnxfNO,fnr9T|23|33-986-427-8764|7233.18|AUTOMOBILE|ly pending instructions: carefully regular excuse 1780|Customer#000001780|ZIeOfVh8umRYig|17|27-213-387-3335|5137.48|FURNITURE|patterns use carefully about the slyly even foxes. furiously regular excuses about the furiously careful packages af 1781|Customer#000001781|JYtJY4OTZQUaEQlfDEeVkK4mtO|8|18-350-885-2317|4686.31|AUTOMOBILE|s cajole fluffily. platelets believe quickly. furiously even accounts wa 1782|Customer#000001782|ehV3 VRXVZ9SR3|21|31-246-927-6074|839.66|AUTOMOBILE|beans. requests was about the blithely regular foxes. unusual, final platelets sublate accounts. 1783|Customer#000001783|ey2RVFXAj5c1qisLEFJA43S2|3|13-239-528-2710|8131.81|BUILDING|dolites. ironic excuses shall wake blithely slyly even theodolites. quickly ironic asympt 1784|Customer#000001784|Zs8QpbcHZfcVJ6oujM8g69J|15|25-605-903-3007|5458.37|BUILDING|n frays. blithely ironic theodolites haggle carefully. blithel 1785|Customer#000001785|nnCUQ01AgIgYBDsdHteH4u0na6WiXvBv|23|33-475-488-7723|7805.55|FURNITURE|r packages was. permanent requests are special deposit 1786|Customer#000001786|pZAHd2LxDUbgcS1WQBN4vvoR5BeNpckOkl7DhG9|6|16-436-900-4501|-51.99|AUTOMOBILE|nts sleep among the quickly regular excuses. slyly even 1787|Customer#000001787|l9Rin,i89mx8LxSK0cC0BPi3OBcM3BXp|20|30-258-449-6408|4120.78|HOUSEHOLD|ts above the quickly regular instructions cajole about the attainments. carefully 1788|Customer#000001788|4,Lldd8YNe,DOiCq2dSW4JL0uqjdVpI2yW4fZMDI|24|34-826-793-6480|-45.96|BUILDING|tes promise. carefully final requests use above the blithely express packages. carefully even theodolites na 1789|Customer#000001789|lfHQVe9IPEKad|0|10-582-797-3122|6407.46|FURNITURE|he slyly express requests. silently daring packages sleep express courts! blithely pending foxes w 1790|Customer#000001790|z IQ1wwki0OLw7biMFHxPKNjp1JBOs7|5|15-805-568-6053|9177.60|MACHINERY| sleep along the special, bold requests. even accounts against the dogge 1791|Customer#000001791|nEuCQJ1,wiJUq7k05FED7j6EjZP0QdzriHsUWZ|8|18-937-127-3890|3917.82|HOUSEHOLD|he furiously final instructions sleep regular deposits. blithely ironic accounts boost among the ac 1792|Customer#000001792|IPqTE3D5cvxDmpveKD6WwW7Pb9ymeAgKs,rT|1|11-561-103-5122|9762.91|BUILDING|to the final instructions will wake quickly carefully special 1793|Customer#000001793|B13zlM7kKBKt|19|29-393-128-9130|7329.58|HOUSEHOLD|ringly furiously bold requests. 1794|Customer#000001794|col5dSe1MO8MyVdQ1f09bhAFsVjYwNH8R4I|7|17-670-112-5044|8243.80|HOUSEHOLD| instructions around the furious 1795|Customer#000001795|EaQbFX VR89kRgd6svC3NK8MSivUK8DZ3y|21|31-853-266-7057|6580.13|MACHINERY|; requests wake slyly after the furiously pending pa 1796|Customer#000001796|HGRGXJA8AwhfBNSyDNn8j3JMvbIwPUKjPTcvKBLs|10|20-905-459-3952|7327.39|BUILDING|ironic, regular accounts haggle care 1797|Customer#000001797|5v4QJxOVHNQ3J6NORJE2edRftfg8 HiGu|19|29-805-690-1846|4461.48|FURNITURE|long the accounts are furiously courts. quickly regular exc 1798|Customer#000001798|fOQAhX8wjDFg8tpeOa L ZdgFlOC69bvmZE|7|17-422-203-8428|6072.64|HOUSEHOLD|hely about the carefully bold deposits. quickly ruthless warthogs w 1799|Customer#000001799|07tCCGrmdFwcXYnolQabgAZW9yq|20|30-643-275-8135|2967.77|BUILDING|requests of the slyly ironic instructions believe slyly regular 1800|Customer#000001800|Cc 1QYWD8JeDlRuyLOEffaanH|16|26-566-785-6289|3323.37|FURNITURE|regular pearls. quickly even escapades alon 1801|Customer#000001801|8ZC3HFVDQGf23cjelZL0wa|3|13-994-265-8339|6806.86|MACHINERY|es sleep carefully along the quickly regular accounts. furiously unusual packages must are permanent accou 1802|Customer#000001802|fGUhRVo61nZfOPxAxzZLrp4z|23|33-720-109-4385|897.62|MACHINERY|c requests. slyly final deposits cajole bold packages. ironic deposits wake quickly around the ironic, special i 1803|Customer#000001803|D7E PteFioyOXeI 422,yZZIsEk|0|10-993-780-6774|3318.35|FURNITURE|its eat quickly around the slyly 1804|Customer#000001804|,TDgHBX9y5eC5ycEwVKbO3gJjXChWmj|15|25-332-547-7897|-516.28|HOUSEHOLD|quests cajole fluffily after the pending, special pinto beans. packages try to haggle. braids haggle fina 1805|Customer#000001805|ZERs4Cu5lQTYD|9|19-679-706-1096|-274.75|AUTOMOBILE|ding tithes. slyly ironic packages boost. final, stealthy requests wake. final, re 1806|Customer#000001806|BB6Vr7W,rSIpWKp|9|19-872-322-3433|254.17|MACHINERY|usly blithely regular instructions. slyly regular dugouts sleep carefully. 1807|Customer#000001807|jlGhIS6zaYIfu9tWFHAyDKVQpOvIluJ1RunV3X|6|16-707-959-9348|1720.94|MACHINERY| slyly final pinto beans sleep furious 1808|Customer#000001808|Z Losr9UXEWwm3RgetdFLr6Q|22|32-507-149-7712|2776.30|HOUSEHOLD|oost carefully ironic requests. fluffily regular foxes boost blithely blithely express ideas. car 1809|Customer#000001809|GqhON8SNyRv|20|30-390-140-3365|8306.29|MACHINERY|ld theodolites eat carefully special pinto 1810|Customer#000001810|RI0cwmW3gNVJiSnIGooUzA|0|10-274-496-6960|923.32|MACHINERY| the slyly special ideas. regular platelet 1811|Customer#000001811|GAuB2XYNF6YxAJgUQO1VcXal|14|24-892-235-8707|6104.69|MACHINERY|dazzle blithely. ironic asymptotes breach; carefully iron 1812|Customer#000001812|UqextI Ph0pve58|0|10-876-771-7164|2515.86|BUILDING| final orbits promise fluffily even waters. blithely 1813|Customer#000001813|xEoR4tsV2Bse527UyeFO8aFhmZ|3|13-604-485-5526|9074.92|FURNITURE|luffily ironic requests doze carefully after the express packages. slyly regular excuses sleep slyly carefu 1814|Customer#000001814|wrzLhEh9DAAHPrh19AGxqCxBWQjO52j4qA1fmqw6|13|23-588-705-2608|5215.32|AUTOMOBILE|ully special packages on the bold accounts sleep among the furiously ironic requests. final requests detect quickl 1815|Customer#000001815|wXx75IZkzX3hsqx5H,PBqxf1CB,cjS|13|23-954-607-5326|3940.94|BUILDING|, bold requests. ironically regular accounts sleep blithely carefully 1816|Customer#000001816|WemZ1IDB0sBS9yyqszdSxqRm8YtGGDme6 BW|4|14-307-306-8692|2328.09|BUILDING|e deposits maintain about the furiously thin theodolites. carefully iron 1817|Customer#000001817|,pNXvEI4pkygH 4wjAy,hPxNikXwej29WH8|7|17-246-120-5754|-862.51|FURNITURE|egular packages. carefully even Tiresias haggle. asymptotes haggle furiously across the quickly pending reque 1818|Customer#000001818|wSYuvOMT8YVZqRDS1YCGrALb|22|32-897-843-2620|-331.97|BUILDING|about the blithely regular requests. ironic requests sleep. blithely pending accounts are according to th 1819|Customer#000001819|pfaH03EbQcSE64yOEnKz7mwHHn|18|28-832-601-3029|6435.85|AUTOMOBILE|y even gifts haggle since the slyly bold warthogs. 1820|Customer#000001820|RsTONzasImDok,RBqc2J09pa29w8gSf6JDIuCBx|6|16-496-603-4437|2282.18|BUILDING|ymptotes affix fluffily doggedly express packages. ex 1821|Customer#000001821|QlxYI,DDZRUAyyVfdaF4q nurCpm,3FltjM|11|21-278-374-9593|8593.01|FURNITURE|tegrate slyly carefully final requests. ironic, regular package 1822|Customer#000001822|H79GEJ2,C0JTus4zVXNjEyMqEFe,7RmzxIGvU5|14|24-524-173-1344|1728.52|AUTOMOBILE|regular forges. ironic packages sleep furiously. slyly fina 1823|Customer#000001823|ci7lCqnRGp|19|29-787-260-1556|9240.38|FURNITURE| furiously. blithely final foxes integrate alongside of the instructions. quickly regular 1824|Customer#000001824|aN7aUMe22hd2LnIqVf7TnJLzSI8uggv,YpldbC|10|20-684-209-1084|3177.51|AUTOMOBILE|luffily regular, pending instructions. slyly pending foxes 1825|Customer#000001825|,E8l78G7k,u0eGXu,sGU1fta2Lg|8|18-969-538-4715|8447.18|FURNITURE|ing to the slyly silent deposits. 1826|Customer#000001826|tbWBFvYUZjBY,BH5r5CQsA71GJIQJNx|3|13-389-673-9030|3891.76|BUILDING|e. bold deposits among the slyly regular depths detect slyly a 1827|Customer#000001827|icXm8xlUo0Ca,T9MoUjkWej3|24|34-815-928-1369|800.31|BUILDING|se carefully alongside of the quickly special packages. regular requests run. pinto bean 1828|Customer#000001828|djbEIhlvmo1i8ZUgTNLFT2f1P|13|23-937-592-5811|6673.46|AUTOMOBILE|into beans alongside of the unusual foxes breach blithely above the quickly dog 1829|Customer#000001829|9TWqvF0jyqs5eJ O0OPAprl0chk8WGTf,|0|10-582-390-6176|2886.86|BUILDING|ccounts. boldly bold foxes use slyly even requests. never regular dolphins at th 1830|Customer#000001830|d6ZvjlxbHL6Kq5y,|17|27-148-316-6372|7363.56|AUTOMOBILE|around the fluffily regular notornis integrate above the final deposits. pending requests nag carefully packages 1831|Customer#000001831|DcW02etJU0N2SmSs3UgkmkPRvao7A|13|23-215-897-5399|7448.78|FURNITURE| thin ideas. furiously quiet pinto beans w 1832|Customer#000001832|VffJ90poBdj 9fq7Hpp0NvcEeeuHg |10|20-337-677-6483|9640.29|BUILDING|. furiously express ideas wake furiously. unusual, bold requests nag quickly. slyly regular 1833|Customer#000001833|KluDIF0rSvGj8mjBA vo7db1bA|3|13-425-435-3864|3418.04|BUILDING|y quickly slow requests? ironic, final pinto beans eat furiously. slyly regular deposits sleep 1834|Customer#000001834| oWU4qpGp8v5pjfHDGwIAh7z|24|34-976-916-8306|2985.37|HOUSEHOLD|slyly unusual packages play. unusual, ironic ideas integrate always according to the thin requests. ste 1835|Customer#000001835|XwMDWDz0zgJT7CpiEZ7Mak8kB6m|7|17-285-512-9067|2081.99|AUTOMOBILE|s. carefully unusual packages along the carefully idle pinto beans haggle carefull 1836|Customer#000001836|0oV7OBdPNyXRqIesxfDU0Ol4So|7|17-927-450-7081|9925.39|FURNITURE|ate slyly idle depths. silent, even theodolites are furiously ironic 1837|Customer#000001837|sWO0VNEY4GzuBq L0vJ5p2|14|24-908-704-5003|-886.94|AUTOMOBILE|unts after the bold, regular dependencies breach a 1838|Customer#000001838|4nF9mi2bj3s82Sv8KPgIsS|21|31-194-756-6301|7142.14|AUTOMOBILE| deposits. regular requests nag slyly-- regular accounts are furiously acr 1839|Customer#000001839|QKjHd6wOQJaxJCOqH2E5P n2tZb3Y|5|15-944-305-1436|3909.79|AUTOMOBILE|haggle furiously after the regular packages. ironic, regular foxes haggle 1840|Customer#000001840|cMNMPjKib0Ez0Bl,cesLvkOl39p4oiUH |3|13-833-461-5939|6879.93|BUILDING|the courts sleep blithely according to the slyly permanent ideas. 1841|Customer#000001841|xJLMIrO4OB|10|20-566-527-9390|8053.62|BUILDING|ts snooze furiously final waters. furiousl 1842|Customer#000001842|SYMcJ0iGluYT2pe6NR5jyErQBODPK|10|20-148-955-7830|233.23|BUILDING|y regular theodolites. packages sleep blithely according to the busy, final instructi 1843|Customer#000001843|5Op16OGLtq5hXrYvKBKPPu,B1pB|1|11-977-879-2346|1821.61|HOUSEHOLD| requests after the never bold ideas hang carefully bold sheaves. carefully pending deposits 1844|Customer#000001844|1BOo5no2lQ8Fa CLDWFaRC8j|18|28-705-205-8653|798.99|HOUSEHOLD| about the slyly final foxes. furiously pending asymptotes acco 1845|Customer#000001845|xHga IhnB2a,|10|20-777-704-2095|6567.69|MACHINERY| furiously requests. even instructions haggle. instructions haggle fluffily. requests along the carefu 1846|Customer#000001846|v0hJC6rlPBHdWAUGg7xm7SEUhpI3wWIO2|16|26-860-238-2408|7115.80|BUILDING|riously regular deposits wake regularly above the slyly unusual requests: unusual packages are furiously furiousl 1847|Customer#000001847|g12l6sqk5YNADO,NEnDg, prphdZp|16|26-677-971-4905|1240.90|MACHINERY|. furiously express packages wake quickly. sl 1848|Customer#000001848|g xIZPvP9AkmbUQa6e|4|14-161-419-6384|8684.90|HOUSEHOLD|as. final, regular asymptotes haggle in place of the slyly ironi 1849|Customer#000001849|vRc6mWLRoRf2ElUvz9byqx2Vt|22|32-730-179-7629|6259.04|BUILDING|ckages. bold epitaphs maintain slyly alongside of the carefully exp 1850|Customer#000001850|yY0Ihs9rn4Jt9qzowqGCKcGYUYVqrOpfKVArQK|10|20-451-644-7838|722.28|BUILDING|lly regular accounts. carefully regula 1851|Customer#000001851|n873H6Cz5A9uHamGSt iIs3|3|13-311-225-1271|4290.76|BUILDING|e slyly regular pains wake slyly final requests. quickly 1852|Customer#000001852|LCTu83UaCBLeatTuc|14|24-811-458-3601|7717.57|AUTOMOBILE|ar, final accounts. fluffily bold deposits cajole. ironic deposits above t 1853|Customer#000001853| z1i M6vmUfw|5|15-442-128-2785|6244.70|FURNITURE|ly pending instructions. blithely special pinto beans sleep carefully quickly final packages 1854|Customer#000001854|CYDzQ3P8qyP,o0ZCHt oAxFNfmlkY18|14|24-654-947-6633|1661.15|BUILDING| the fluffily silent deposits. quickly 1855|Customer#000001855|REHQRPduxffKW5vE8Laf|13|23-239-487-1955|9936.71|MACHINERY|nto beans cajole. pending packages nag quickly according to the packages. s 1856|Customer#000001856|FuvHERUFZy4lUPSGX9WzC6rM3dLy4DRPv21scFOR|21|31-242-270-7219|2381.48|AUTOMOBILE|ic requests. furiously silent requests wake slyly according to the never pending packages. slyly iro 1857|Customer#000001857|4BOgUDmgtH,RyMRb4jUc TDYqFB|12|22-974-598-1668|5485.71|HOUSEHOLD|sits doze among the ironic packages. carefully express requests wake blithely alongside of the theodolites. ca 1858|Customer#000001858|jtVS,lDzoPG7|18|28-940-298-3762|5263.13|MACHINERY| unusual theodolites affix according to the silent instructions. furiously darin 1859|Customer#000001859|MV0YXkg6XVwBzeXMljCiLp|1|11-147-516-2776|4508.50|AUTOMOBILE|t ideas. bold deposits across the quickly slow excuses haggle slyly unusual packages. furiously final ideas caj 1860|Customer#000001860|4,u2YoI2nnY|15|25-432-186-2226|4383.03|HOUSEHOLD|ackages wake on the pending sheaves. bold foxes nag slyly slyl 1861|Customer#000001861|E XEUZYUxRnh0Z1MKcU19Ff2kj7|15|25-966-583-8374|7584.92|HOUSEHOLD|blithely regular instructions detect fluffily abou 1862|Customer#000001862|jDYsot4wXtwiJ9W 7XqR,C|7|17-972-634-9470|9848.27|MACHINERY|s are after the blithely even instructions. slyly regular 1863|Customer#000001863|kfPxNgJute862KvopjfTItHJVC9Hu|16|26-777-313-6429|1024.85|FURNITURE|fily final accounts cajole furiously furiously pending accounts. slyly final requests acco 1864|Customer#000001864|ZTAtP8aj97X246rzbsglEXqogjhEcYybWE3W|22|32-582-138-8697|7669.96|FURNITURE|uriously. final packages detect slyly blithely r 1865|Customer#000001865|0wViSaeE08WI09xFxqr58|14|24-208-401-3922|9112.90|BUILDING|luffily regular dolphins. quickly regul 1866|Customer#000001866|bdPJelrGfDMOilIhvlZNmb3|6|16-520-541-7567|5563.66|HOUSEHOLD|ent accounts about the blithely unusual dolphins are carefully along the special deposits! final excuses u 1867|Customer#000001867|72gC1ZTlzFNQy9CvFosWzE|1|11-951-119-6203|6470.11|MACHINERY|final instructions are according to the carefully final accounts. caref 1868|Customer#000001868|kD u32wkAhHa FvvBM,y6oJVpfZQDcuDBQ|2|12-347-925-5872|6150.48|MACHINERY|onic, final ideas haggle blithely excuses. slyly regular ideas across the carefully bold dolphins cajole sly 1869|Customer#000001869|hOzuX9JGbYqPv677zUdWtUN3,LHdGYcEwqva|6|16-750-945-2045|2181.16|BUILDING|nts. special, unusual pinto beans haggle slyly 1870|Customer#000001870|iAG9OC,akOL06jR|1|11-950-597-6395|4739.19|BUILDING| epitaphs according to the slyly regular foxe 1871|Customer#000001871|,B2hG X848Trk9SZ|15|25-329-168-1291|9528.08|AUTOMOBILE|y express excuses. furiously regular warthogs integrate quickly after the regular packag 1872|Customer#000001872|CTHfC,m5kcnglbJawTVx5x1cJx3lynOTa7|22|32-985-585-5168|9479.72|FURNITURE|kages wake carefully after the ironic, special requests: regular foxes from the special asymptote 1873|Customer#000001873|fvkv57 kYzsIjzXH9,csoT7smnh pObla4,|3|13-581-396-9191|-201.93|BUILDING|yly. fluffily unusual theodolites among the ironic ac 1874|Customer#000001874|DVHIWiilJuI|17|27-133-279-5869|3121.93|FURNITURE|s. carefully final requests unwind furiously among the 1875|Customer#000001875|CiwSrmNO0dw3Bvf, v2NL,|10|20-222-437-7307|9319.39|MACHINERY|equests cajole above the slyly final requests. blithely special excuses nag blithely bold dependencies. blithely exp 1876|Customer#000001876|pUsteCJxfoVJ|7|17-965-399-4552|90.46|FURNITURE|s. ironic theodolites around the specia 1877|Customer#000001877|a1Rycy,Cni7sr7VUyLR|23|33-986-738-5501|341.10|MACHINERY|ss instructions. fluffily ironic packages boost alongside of the blithely final patterns. s 1878|Customer#000001878|EJqsZwacKoF5mzgYOzYM5Tt7tcuJ|6|16-201-771-9025|-120.65|MACHINERY|xes. furiously bold platelets sleep quickly among the slyly bold theodolites. special deposi 1879|Customer#000001879|DsH91sbfSoNl3Ohor9NQGBO94rGJeGRkc4|17|27-652-714-8086|5163.22|HOUSEHOLD|theodolites could cajole quickly about the ironically final requests. blithely even asymptotes use theodolit 1880|Customer#000001880|LRh,W2K0FBWZWB|16|26-331-549-2024|8247.29|HOUSEHOLD|aves are ironic, final accounts. regular asymptotes promise pending epitaphs-- ideas 1881|Customer#000001881|eo82SRJdBgn25wzgDGaYETk70Wj Y3NKEat|18|28-135-437-2467|-780.99|HOUSEHOLD|to beans haggle even, unusual packages. fluffily regular de 1882|Customer#000001882|U3c UftfresD4TjzRl8F|3|13-454-520-9987|5785.07|HOUSEHOLD|uickly regular, final accounts. furiously special theodolites cajole 1883|Customer#000001883|vyFP6B54fLxqmV3RkbkgNbb|18|28-359-399-4989|9692.28|MACHINERY|kly express packages haggle slyly according to the furiously even orbits. 1884|Customer#000001884|TFjb8jysqh0cMWSt36NLtlP|0|10-570-500-9765|8261.18|HOUSEHOLD|s over the ironic requests sleep fluffily carefully bold dependencies. 1885|Customer#000001885|WceTHrKwpwDq4AbQiFFO03R88q2|8|18-986-991-1839|1642.14|MACHINERY|. pinto beans wake along the theodolites. caref 1886|Customer#000001886|rEn7jolYiPkRQDuLU8lSbn|13|23-315-177-8462|9333.99|FURNITURE|ar pinto beans sleep even deposits. slyly special requests are. express platelets snooze blithely 1887|Customer#000001887|F9k42yQM7WJEwLS|10|20-704-266-2935|3370.69|AUTOMOBILE|s wake carefully against the carefully ironic deposits. finally regular attainments 1888|Customer#000001888|3psgNE6QfX,Iw2xTWauq9yL9EkSiD1fAUMXfXu8k|5|15-757-359-6861|-861.72|AUTOMOBILE|ntain ironic foxes. unusual accounts sleep quickly. quickly regular accounts are blithely. fin 1889|Customer#000001889|am4T 9xjY,|24|34-941-762-4541|1977.38|FURNITURE| ideas detect blithely. accounts haggle fluffily against the ironic, ironic accounts. slyly p 1890|Customer#000001890|QvxzOMYHMQUUfnfY3Fhydb|9|19-980-717-9488|704.54|BUILDING|nic sauternes. even pinto beans are carefully express requests. carefully express packages across the blithe 1891|Customer#000001891|xdhHnr2OrFVgZSEY190|9|19-323-409-2180|3069.79|HOUSEHOLD|ording to the blithely final requests. requests sleep quickly according to the ironic, even deposits. blithely ir 1892|Customer#000001892|Mm4BlT6Q10AlsxwZBAoJE,rXVDtArlKo3b,Ysk|14|24-677-485-8489|1917.06|AUTOMOBILE|luffily around the slyly ironic platelets. express deposits hang furiously among the r 1893|Customer#000001893|iFCRrPifE A|1|11-796-419-9244|9971.58|HOUSEHOLD|ccounts among the fluffily bold pinto beans 1894|Customer#000001894|X,KINka,nciwGbK|12|22-832-993-2803|5415.55|BUILDING| final deposits. regular frays was blithely carefully regular platelets. even dolphins run among the 1895|Customer#000001895|PAv4VeG04NXcldWKmT|12|22-350-120-2910|3216.87|HOUSEHOLD|onic pinto beans sleep quietly regular Tiresias: slyly regular excuses ha 1896|Customer#000001896|4QX6hX9fgTooTWKoy6jmMOEpNZHoNJ|16|26-210-412-7721|9849.67|MACHINERY|re among the slyly express packages. dolphins affix furiously across t 1897|Customer#000001897|bizbUJTTu896QHClLsGIeQmcIF5K,7pyN|8|18-397-783-2293|2415.01|BUILDING| fluffily final theodolites. blithely regular requests 1898|Customer#000001898|yK530x63pGQpzYZbGjsmt0INzw2f9oKfocxs0b|16|26-214-695-2065|7975.89|MACHINERY|ar requests. slyly pending accounts haggle blithely. busy account 1899|Customer#000001899|XuQb0moba ,XyN|10|20-243-672-4974|2869.40|MACHINERY| even, idle deposits about the busily final foxes 1900|Customer#000001900|lf78mUY8AHVrPld7M8Ysotn9WoXwKoN|21|31-565-138-3230|-57.96|MACHINERY|kly regular accounts. requests use furiously alongside of the carefu 1901|Customer#000001901|TJVv IWFTUUp ihC 7mzH,ru5s0J9Xb|24|34-736-824-5989|2749.93|FURNITURE|s use. express platelets beside 1902|Customer#000001902|03 VVAFfgEADO1Ert upreETF9E9i1QUgy53i|3|13-815-127-1409|5019.36|BUILDING|after the blithely regular packages. carefully regular gi 1903|Customer#000001903|oNu,vFSWInYzj5wIB5khmgObNMx61UfUap|15|25-489-905-9207|-679.30|FURNITURE|slyly regular requests sleep. blithely ironic packages c 1904|Customer#000001904|qtgYwanpIR3LfvbOPexVyJ8XHDWNn p95|24|34-565-694-9621|9913.42|BUILDING|. furiously special requests haggle finally carefully 1905|Customer#000001905|P xPYefatHsYxGtFYbzin3GqmqPWvR31YlXlOt|19|29-697-760-7269|6862.64|BUILDING|ajole about the pending, bold requests. furiously regular platelets above the q 1906|Customer#000001906|pByvzLPeSKa8Zlh5ncJwAuYK2U|5|15-956-730-8661|3448.10|MACHINERY| the fluffily special foxes. carefully pending deposits by the special, final accounts wake care 1907|Customer#000001907|Wjpx2yEVYfTUHPc9TgXzD64PBqW x|24|34-755-117-7537|2145.93|HOUSEHOLD|carefully silent platelets wake fluffily express deposits. forges against the ironic platelets are silent requests. 1908|Customer#000001908|eT6lmLbi11KA|17|27-275-292-5975|6551.65|HOUSEHOLD|ly unusual requests engage carefully blithely express depos 1909|Customer#000001909|wrQOWC 6kE|19|29-108-892-4321|1605.61|HOUSEHOLD| the regular, unusual accounts. fluffily final theodolites d 1910|Customer#000001910|LQ9x01e0v2qFuNr|5|15-112-377-2843|475.44|FURNITURE|. quickly even deposits sleep f 1911|Customer#000001911|F9kOxmOLLFyC|1|11-369-744-8780|2601.29|HOUSEHOLD|y idle requests poach across the quickly final theodolites. slyly ironic re 1912|Customer#000001912|1AHyJO5HDpo5AiczS 8zP9P9x7PUuoU5wwrTLEb|20|30-784-527-9935|3886.95|FURNITURE|uriously above the ironic requests. blithely careful packages use daringly furiously regular tithes. 1913|Customer#000001913|3Ya7Xzjzd,JayN,jN|8|18-831-659-1744|8031.55|BUILDING|y express platelets. quickly ironic requests hinder quickly against the carefully bold deposits. acc 1914|Customer#000001914|JqCaLg2nQOwrdP1DIHNQ9b,Wk|6|16-458-421-7729|3308.34|FURNITURE|ss pinto beans. quickly unusual packages doze carefully furiously final 1915|Customer#000001915|FMtQbAxpu5v3,0NVxEJwU24sV2TgF|16|26-683-446-1665|7268.98|AUTOMOBILE|s. furiously ironic ideas boost furiously among the ironic pinto beans. accounts thrash slyly about the 1916|Customer#000001916|jFGCdeXJVKzv7YDL|5|15-662-459-4512|1768.76|HOUSEHOLD|into beans. even deposits among the fluffily even platelets cajole ab 1917|Customer#000001917|gZW6epQmK,ISulI9 Qt IWytnY|7|17-940-989-5599|3470.08|AUTOMOBILE|silent packages boost quickly. regular, regular instructions nod. idle dependencies are slyly among the slyly i 1918|Customer#000001918|CRBet8MbYBwTgtdPOzX,ipUvTiOrp0vUQjnr|9|19-987-388-4413|8539.61|MACHINERY| ideas alongside of the blithely regular 1919|Customer#000001919|lnfRcSyQXSI|18|28-664-832-4585|6262.18|HOUSEHOLD|as outside the furiously expre 1920|Customer#000001920|FvwmRwSmKgzj7u4disg0ahJOtEfo|8|18-810-300-8541|9476.97|MACHINERY|uctions believe. furiously ironic accounts use fluffily. regula 1921|Customer#000001921|9AE6JuRlhkABh1m8ABQD4AsBk6x5rXlCQ5I|18|28-644-770-2274|8509.33|BUILDING|lar platelets was furiously above the slyly exp 1922|Customer#000001922|qMXL1bzgCtSj1GLMI3AMi1CNV,f, 0iCXt|9|19-532-924-9245|3398.55|MACHINERY|ake carefully even packages. ironic requests against the express, even asymptotes affi 1923|Customer#000001923|m6AVOkvy swPbt,pZ7b9MBfr8zo9vFUvappyyWjv|23|33-781-817-1440|5376.70|HOUSEHOLD|waters detect fluffily carefully regular pinto 1924|Customer#000001924|inc0YxJOJYunXhf|4|14-472-427-4447|-851.49|AUTOMOBILE|gular instructions boost among the blithely s 1925|Customer#000001925|qZ8m0grVeRJN9h6Bn|0|10-814-492-1424|8891.65|HOUSEHOLD|its sleep among the regular packages. fluffily final depende 1926|Customer#000001926|EMRuUDTA2m0|7|17-203-685-7821|4968.09|AUTOMOBILE|ve the even packages. blithely 1927|Customer#000001927|V4zstahq,OaYcbh2IeSkJfiRO7wtMPXOoV|1|11-876-774-5282|6756.49|AUTOMOBILE|side of the furiously final ideas are at the bold patterns. quickly pending requests thrash fluffily 1928|Customer#000001928|Ce02vQsU97|12|22-812-639-3782|1391.69|MACHINERY|t the carefully final instructions wake fluffily f 1929|Customer#000001929|imHqCzuePilpP5h UA2fwTIJMWVdj|20|30-949-561-2278|2157.13|AUTOMOBILE|osits are slyly across the even requests! busily special platelets cajole carefully 1930|Customer#000001930|kJD6LiynLuutDPKOcgIBUIdUQIWb14oGE jb|18|28-793-928-3242|7909.13|HOUSEHOLD|ular packages snooze blithely carefully express packages. quickly regular dolphins sleep about the slyl 1931|Customer#000001931|DUbNQry9jewVHI6GLNIbX0aIBGoWwsqWabbuXPy|19|29-893-435-8218|2762.06|BUILDING|heodolites. final ideas detect slyly furiously regular dugouts. even pinto beans haggle. slyly express foxes na 1932|Customer#000001932|VhBhkEXBrpSzYgxu1jmIJ3 tVomaDEr|15|25-252-625-7296|-55.92|FURNITURE|fully regular theodolites. quickly express requests may sleep carefully among the furio 1933|Customer#000001933|JGeZTkGT1uwbQa1C,sj|0|10-484-847-9516|4003.10|HOUSEHOLD|es the accounts. close, ironic dolphins wake quickly furiously ir 1934|Customer#000001934|REnXv9yHCBrQTv2i7j645dr5iSY5|10|20-656-890-6862|2038.31|HOUSEHOLD|ons. bold foxes haggle furiously blithely ironic ideas. blithely ironic deposits integrat 1935|Customer#000001935|2SwVYEBhbhWTTWCBsGow0Wu|9|19-674-562-9631|1775.61|HOUSEHOLD|ilent epitaphs wake slyly according to the slyly furious deposits. silent, special instructions past t 1936|Customer#000001936|PGu2Wen188BC|0|10-714-881-1338|8592.20|HOUSEHOLD|y regular packages. blithely even accounts are blithely. carefully regular accounts cajole blithely. quickly bol 1937|Customer#000001937|mXJvkSoAwpg1wVW45|3|13-166-456-2309|-866.47|FURNITURE|equests. final, ironic packages haggle: carefully regular packages mold fluffily accord 1938|Customer#000001938|IqaZSFbDxA,8a,xxnV,gHVOT|10|20-789-331-1347|-894.49|AUTOMOBILE|osits affix slow accounts. carefully final ideas according 1939|Customer#000001939|SBEYKDkrUBAt2RNh1z0dtG|0|10-417-450-5341|1306.86|HOUSEHOLD|usly unusual theodolites haggle slyly around the fluffily ironic asymptotes. express instructions cajo 1940|Customer#000001940|DIL GvUsP25WSgsshTdjULCPxy7l|24|34-475-738-8227|6310.63|BUILDING|e the furiously bold dependencies. slyly silent accounts grow carefully according to the 1941|Customer#000001941|,51rsqq9XV|0|10-365-133-1400|9710.06|FURNITURE|fluffily carefully regular foxes. fin 1942|Customer#000001942|5 eTPX2yTp49JPctgc1EeYMUSWm|3|13-500-263-4736|-947.24|MACHINERY| sauternes. quickly express accounts cajole carefully above the regular accounts. pinto beans 1943|Customer#000001943|sH4nc83Lh,6A2xe9ApH6WD|3|13-737-218-4651|5599.71|AUTOMOBILE|ckages above the final, final pinto beans haggle against t 1944|Customer#000001944|LbrsZ4CFBqJZ|14|24-492-922-6990|7279.93|HOUSEHOLD|s play boldly about the slyly iro 1945|Customer#000001945|ghlnLGw i,LV,MAuts8Ii7XE4WaoExoa|10|20-470-267-5365|9880.10|BUILDING|iously bold pinto beans cajole slyly warthogs. silent, special accounts detect evenly across the ironic pl 1946|Customer#000001946|Z3ceD0uA1BJsDmFl4botusq2SBPCJqIqzr9bHQM|20|30-764-188-4293|7851.87|FURNITURE| requests boost fluffily. ideas detect finally. quickly final deposits nag carefully. quick 1947|Customer#000001947|s 1fCYZ4AqqTJ,ZvJsLi5N|20|30-174-661-7603|8385.53|MACHINERY| ought to are furiously alongside of the blithel 1948|Customer#000001948|4aCCe9W1s4,VdYswJWigEOJ2sW6Thh|5|15-227-107-9153|1704.12|HOUSEHOLD|s about the blithely regular deposits cajole blithely 1949|Customer#000001949|fwIfk5BDREkV uapHbKzYrOCWUz|19|29-227-849-7731|6025.80|FURNITURE|nal platelets cajole whithout the silently regular ideas. furiously bold asympto 1950|Customer#000001950|sRX9I79UagRC3 sV9hO0H|6|16-524-582-1728|7698.52|HOUSEHOLD|nal requests. slyly pending pains according 1951|Customer#000001951|NOXYVa52HyF5lmpj9,WR0|0|10-334-921-5346|5002.24|MACHINERY|thely unusual accounts. furiously unusual deposits sleep along the blithely final packages-- f 1952|Customer#000001952|sZvM,CF54wYyQGhSuFVe|18|28-102-243-6615|8620.01|MACHINERY|y express pinto beans sleep carefully carefully regular pinto beans. fluffily final accounts 1953|Customer#000001953|vd2GjYHxhaYoyE0dU46shW|8|18-155-378-1656|4654.64|MACHINERY|ly slow foxes x-ray quickly even pinto beans. slyly permanent packages nag furiously. carefully 1954|Customer#000001954|ec2Cv6JlL3MFSnbBVWrE vljCiPiM6gFr4WE5PW8|22|32-872-341-1949|7575.28|BUILDING|ep furiously about the fluffily regular accounts. 1955|Customer#000001955|INxAk4G6dbQSijuSR,0t|7|17-649-408-5594|1754.99|BUILDING|usly. final packages doze carefully excuses. packages wake slyly according to the slyly ironic pi 1956|Customer#000001956|sOgO,FYO1KNY3Q45sj asSLH33rFk|21|31-426-823-2422|3007.04|HOUSEHOLD|ess platelets. express pinto beans wake furiously according to 1957|Customer#000001957|Hl2JPoWaDAeyGd|21|31-861-837-2140|3366.66|HOUSEHOLD|al platelets. sometimes final accounts 1958|Customer#000001958|P8lsQKRVNyrMTYEgXU7W|4|14-218-444-2269|8425.51|HOUSEHOLD|quests across the accounts need to haggle carefully regular deposits. careful accounts boost carefully fin 1959|Customer#000001959|NMDtn0GA,CLrZYGaMDmwb|7|17-992-958-9639|2608.12|AUTOMOBILE|ronic deposits: quickly bold packages along the fluffily regular depths sleep furiously thinly regular deposits 1960|Customer#000001960|DFG0YX 4bZQ40vdnfLc8zJfgdWY4iARmum|1|11-645-941-2149|8663.17|BUILDING|arefully carefully regular pains. carefully unusual ideas at the furio 1961|Customer#000001961|j7 XCftSKhgKscMoPW|23|33-428-682-4988|3918.30|MACHINERY|ut the fluffily regular theodolites. dependencies among the 1962|Customer#000001962|VxLrW,XnARNIdQgq1J |4|14-913-275-9532|6907.64|FURNITURE|os shall have to eat bold theodolites. regular dependencies sublate blithely alongside of the express, 1963|Customer#000001963|WFokQKiIqXjIiH|2|12-406-701-6501|680.33|BUILDING|ously fluffily express theodolites. ironic, final dolphins snooze regular ac 1964|Customer#000001964|xEoAWLcXGm|11|21-651-389-8060|3323.69|HOUSEHOLD|osits. furiously express ideas integrate 1965|Customer#000001965|Zf89ewXnl,RNe36J86yVxppUfr6VXEeKGYUa|15|25-361-492-1713|1223.48|BUILDING|ronic, even waters use along the final ideas. ruthlessly 1966|Customer#000001966|jPv1 UHra5JLALR5Isci5u0636RoAu7t vH|0|10-973-269-8886|1937.72|BUILDING|the blithely even accounts. final deposits cajole around the blithely final packages. 1967|Customer#000001967|F1wAzk2iYcb|14|24-936-670-3499|5250.63|BUILDING|ut the slyly ironic pearls. re 1968|Customer#000001968|dUAE71eduW|24|34-192-617-7717|3322.72|HOUSEHOLD|uriously regular deposits use car 1969|Customer#000001969|F7fcrNnwmFLr2fXyi58|11|21-978-965-1419|6864.72|FURNITURE|ckages sleep carefully. never bold instructions are even, final ideas. fluffily even id 1970|Customer#000001970|V9aSisZGbj8fo|14|24-949-639-9364|5469.10|MACHINERY|grate furiously ironic orbits? final, pending deposits affix carefully express ideas. special, express pack 1971|Customer#000001971|DZsft5O035OEx6ql2BKC3CHI3R5ZtlFgKsMs XZM|12|22-293-654-8027|1205.73|AUTOMOBILE|arefully. express packages are above the slyl 1972|Customer#000001972|0XaaQuagZ1K9E63qlSw5SBk|7|17-991-532-5136|689.12|AUTOMOBILE|ckly. regular, pending pinto beans solve around the blithely exp 1973|Customer#000001973|J8,WbeY81R4j4nAv5CEczBfn3XPh2dIcxnhLtdf|4|14-828-797-8149|7990.18|HOUSEHOLD|dolites affix bold, final requests. blithely special accounts integrate. bold, iron 1974|Customer#000001974|e4NzvGNJLIOkXfof8RF21qDH1O|24|34-740-820-4310|1007.21|AUTOMOBILE| sleep against the quickly ironic sentiments; ironic packages affix above the carefully final 1975|Customer#000001975|XzyjsmoapfpiWPo fn|5|15-174-926-1370|8125.51|MACHINERY|re slyly against the unusual dependencies. slyly 1976|Customer#000001976|M2WWTo3je0NgpIFOe55SvtJwnyxY8jp8Ikms4|15|25-703-881-1012|9548.13|MACHINERY| slyly regular deposits. fluffily fina 1977|Customer#000001977|TMAohw9bEg3eDJH|2|12-534-854-9706|5529.03|BUILDING| carefully bold requests up the final, speci 1978|Customer#000001978|e4sYh5qmltXfpKhnO fQF3E9dZuWXu,C0E|13|23-513-770-1875|7503.87|MACHINERY|ully regular dolphins. blithely express requests are furiously slyly regular accounts-- caref 1979|Customer#000001979|HSGMB5gHS1ieJ58hBBImFA9OxE|14|24-690-108-2317|631.63|MACHINERY|ns alongside of the bold, silent requests x-ray after the blithely silent p 1980|Customer#000001980|rhVLwo9PmcGj8oWJSAIYmcbPf 2lCekjg6V6|6|16-721-559-9819|9772.49|AUTOMOBILE|ckly regular packages. ironic packages after the special instructions breach fluffil 1981|Customer#000001981|wctyYavWwznYKAs4|13|23-867-283-2614|3283.90|FURNITURE|ding to the blithely bold deposits. ironic packages detect blithely deposits. regular, specia 1982|Customer#000001982|mpS3z3Av8c89RsaJyiet|12|22-891-773-1026|3567.30|HOUSEHOLD|e blithely courts. even, express re 1983|Customer#000001983|KFNKCUFOyS|0|10-305-434-9284|3640.90|FURNITURE|es sleep blithely regular instructions. regular instructions cajole carefully. slyly regular accounts 1984|Customer#000001984|MAqwYLxOBbMoyAWwvjEZK9QYgRMbhtFkdHbiR|13|23-768-636-1831|8661.08|HOUSEHOLD|y unusual requests. furiously ironic deposits haggle quickly a 1985|Customer#000001985|Cy7CMgihIn6lemMjU2d3SKiOmHXi0TrRZK|0|10-248-352-6397|1800.96|HOUSEHOLD| theodolites wake slyly fluffily regular pinto beans. furiously ironic packages according to t 1986|Customer#000001986|3ugRcAUZuXms2oHREuEsprYLg2u|0|10-971-309-8075|5749.20|FURNITURE|fter the slyly even deposits cajole pending accounts. 1987|Customer#000001987|lWwjammTGaKVKq9npAtfs3|19|29-745-306-2348|796.96|BUILDING|s. blithely even pinto beans boost always about the doggedly bold requests. regular requests use. 1988|Customer#000001988|MdIopEKT9C1|9|19-927-922-5628|5414.64|AUTOMOBILE|ve to are furiously bold theodolites. qu 1989|Customer#000001989|,WMIpzbXk5YrV7ZlH23grs5TeGhEtF|10|20-648-105-6440|9859.77|MACHINERY| unusual foxes wake slyly fluffily pending gifts. bold packages across the slyly 1990|Customer#000001990|M9VbFPNnktfGtPgnf3t1ptNYOzS3eCwlKpte|14|24-543-274-3227|7244.87|HOUSEHOLD|usual asymptotes around the packages haggle 1991|Customer#000001991|REzgRDc,2RF4ayaV3b3gZhIdMti9J6b8b90p|9|19-351-767-3188|3517.27|FURNITURE|nag blithely slyly special deposits? express accounts about the dolphins woul 1992|Customer#000001992|FNQZbT88wzGKhU9|1|11-760-815-8444|9679.43|BUILDING|e fluffily regular ideas haggle past the even requests. regular pinto beans nod acc 1993|Customer#000001993|Yi8QRnZDlVqZfW8SHbw,drKwsPBd6qXjavl|0|10-928-142-7350|1209.74|FURNITURE|regular pinto beans. furiously ironic epitaphs after the 1994|Customer#000001994|TRsalXgbwGxicJe1Ogt4th3BwgmrADqgILXd|1|11-384-723-1000|1971.60|HOUSEHOLD|taphs along the furiously regular dependencies must have to run blithely idle dependencies. slyly fi 1995|Customer#000001995|aIGN3hurONjCDOvfE JLx,5pmtqCPnXvgqV7VNy|2|12-339-453-3201|2674.92|BUILDING|ilent instructions sleep quickly quickly bold gifts-- final deposits grow after the final asymptotes. care 1996|Customer#000001996|SkYj5PWHdXD9|17|27-626-208-5273|9379.16|BUILDING|lyly final accounts are blithely final courts! even 1997|Customer#000001997|D7SUlJ,KUdYu|24|34-724-701-3880|3610.05|FURNITURE|accounts wake furiously. silently quick epitaphs are carefully slyly final pinto bea 1998|Customer#000001998|RpAD0CJxRxC2kjR|14|24-529-154-9925|7080.37|FURNITURE|p slyly alongside of the pains. 1999|Customer#000001999|y8mRnn6pJ0V|2|12-967-439-5391|-117.85|AUTOMOBILE|heodolites. furiously ironic excuses boost along the carefully f 2000|Customer#000002000|NabVtyaWao1l9Ss|13|23-122-829-3487|8176.83|AUTOMOBILE|s shall have to sleep quickly ================================================ FILE: src/test/regress/data/datetime_types.csv ================================================ 2000-01-02 04:05:06,1999-01-08 14:05:06+02,2000-01-02,04:05:06,04:00:00 1970-01-01 00:00:00,infinity,-infinity,00:00:00,00:00:00 ================================================ FILE: src/test/regress/data/enum_and_composite_types.csv ================================================ a,"(2,b)" b,"(3,c)" ================================================ FILE: src/test/regress/data/events_table.data ================================================ 2,2017-11-23 12:32:57.851419,3,3,4, 2,2017-11-23 03:35:04.321504,1,0,2, 5,2017-11-22 21:24:22.849224,5,4,1, 6,2017-11-23 14:00:13.20013,3,3,3, 4,2017-11-23 16:20:33.264457,0,0,3, 4,2017-11-23 04:01:12.29256,4,4,3, 2,2017-11-23 17:26:14.563216,1,5,3, 2,2017-11-22 20:16:16.614779,0,1,1, 3,2017-11-22 21:12:24.542921,1,2,5, 3,2017-11-23 02:03:45.966467,4,1,0, 2,2017-11-23 13:27:37.441959,5,2,4, 4,2017-11-23 18:10:21.338399,1,2,4, 5,2017-11-23 09:20:13.303927,3,0,3, 3,2017-11-22 22:05:38.409323,3,2,2, 5,2017-11-23 14:28:51.833214,3,0,1, 3,2017-11-23 07:11:12.987954,3,1,3, 4,2017-11-23 03:20:23.803407,2,3,3, 3,2017-11-22 20:23:46.906523,3,3,2, 3,2017-11-23 03:02:35.021717,1,0,2, 4,2017-11-23 03:29:32.271967,3,2,4, 6,2017-11-22 20:36:09.106561,3,2,1, 5,2017-11-23 16:11:02.929469,4,2,0, 3,2017-11-23 18:08:26.550729,2,4,3, 2,2017-11-22 22:23:25.40611,3,4,4, 1,2017-11-22 23:22:09.957743,1,1,1, 4,2017-11-23 14:19:25.765876,3,1,1, 3,2017-11-22 21:26:21.185134,1,5,3, 3,2017-11-23 14:00:12.884702,3,1,1, 4,2017-11-23 07:19:09.381749,1,0,2, 5,2017-11-22 22:45:17.377228,3,2,0, 3,2017-11-23 04:45:23.548548,1,2,3, 4,2017-11-22 19:00:10.396739,2,1,1, 4,2017-11-23 08:36:53.871919,2,4,3, 6,2017-11-23 00:45:41.784391,2,3,2, 4,2017-11-22 21:05:25.194441,5,4,1, 3,2017-11-23 09:38:45.338008,2,5,0, 3,2017-11-23 01:17:49.040685,2,3,4, 1,2017-11-22 21:06:57.457147,4,3,2, 6,2017-11-22 21:17:09.549341,5,2,5, 4,2017-11-23 00:40:52.517603,1,5,4, 4,2017-11-23 08:14:18.231273,4,3,2, 1,2017-11-22 19:07:03.846437,1,2,5, 2,2017-11-23 14:02:47.738901,1,3,2, 1,2017-11-23 09:33:16.992454,3,4,1, 5,2017-11-23 14:40:40.467511,1,4,1, 2,2017-11-23 01:08:57.24208,2,3,1, 2,2017-11-22 22:56:47.673504,4,1,4, 2,2017-11-22 22:50:33.855696,4,2,3, 5,2017-11-23 06:56:03.019565,2,1,1, 6,2017-11-23 00:01:48.155345,4,2,1, 6,2017-11-23 02:06:53.132461,5,1,1, 2,2017-11-23 05:42:25.859537,2,3,5, 4,2017-11-23 12:58:37.035024,1,2,4, 1,2017-11-23 11:09:38.074595,0,5,1, 5,2017-11-23 14:23:09.889786,3,1,0, 5,2017-11-22 20:45:35.99031,1,2,3, 3,2017-11-23 03:53:44.925122,4,2,3, 1,2017-11-23 05:19:55.213875,0,2,0, 1,2017-11-22 19:03:01.772353,4,1,2, 5,2017-11-23 02:27:02.996588,1,4,1, 2,2017-11-23 07:09:08.06934,1,5,4, 2,2017-11-23 10:14:51.816339,4,2,1, 3,2017-11-23 13:30:04.643655,3,2,1, 2,2017-11-23 02:06:28.579787,3,1,4, 4,2017-11-23 02:59:28.548578,4,2,3, 2,2017-11-22 22:06:12.107108,0,2,5, 2,2017-11-23 10:46:34.177173,3,2,3, 2,2017-11-23 08:26:23.045769,1,1,0, 3,2017-11-23 06:44:50.887182,4,0,4, 6,2017-11-22 22:44:48.458334,1,3,2, 6,2017-11-22 23:15:15.875499,4,3,3, 3,2017-11-23 16:44:41.903713,4,2,2, 5,2017-11-23 13:02:31.055159,0,2,1, 4,2017-11-23 07:32:45.521278,4,5,3, 5,2017-11-23 03:23:09.888244,1,3,2, 1,2017-11-23 09:23:30.994345,3,1,1, 1,2017-11-22 20:56:21.122638,2,4,4, 1,2017-11-23 02:59:23.620864,4,5,4, 2,2017-11-23 10:59:25.787633,3,1,0, 5,2017-11-23 13:26:45.571108,3,3,4, 2,2017-11-23 06:23:43.305403,1,2,1, 4,2017-11-23 04:32:21.986825,4,1,3, 1,2017-11-22 21:47:04.188168,4,2,0, 4,2017-11-23 06:34:15.714083,2,2,2, 2,2017-11-23 04:05:16.217731,1,4,3, 2,2017-11-23 12:17:48.482544,2,3,2, 2,2017-11-23 15:58:49.273421,5,1,2, 2,2017-11-23 12:06:39.464165,4,5,4, 3,2017-11-23 16:31:56.219594,5,1,2, 1,2017-11-23 10:23:27.617726,4,2,5, 3,2017-11-23 10:24:02.530471,2,2,3, 6,2017-11-23 07:27:32.822068,2,2,1, 5,2017-11-23 04:16:29.402814,1,3,3, 3,2017-11-23 13:19:47.138294,2,1,4, 2,2017-11-23 08:07:58.727599,3,1,3, 1,2017-11-23 00:42:37.237615,2,4,3, 3,2017-11-22 22:54:59.938341,2,4,3, 6,2017-11-23 11:16:13.106691,1,1,0, 3,2017-11-22 18:36:16.372893,2,3,4, 1,2017-11-22 18:49:42.327403,3,2,1, 1,2017-11-23 21:54:46.924477,6,4,5, ================================================ FILE: src/test/regress/data/impressions.csv ================================================ 21048,8,474,2017-05-14 16:54:53,http://larkinjacobs.info/nathanael.steuber,,8.215.89.118,"{""location"": ""TN"", ""is_mobile"": false}" 21049,8,474,2017-01-11 19:52:23,http://murraygulgowski.info/kay,,215.126.168.159,"{""location"": ""IQ"", ""is_mobile"": false}" 21050,8,474,2017-01-12 03:38:55,http://bernier.info/alene,,192.183.170.81,"{""location"": ""JM"", ""is_mobile"": false}" 21051,8,474,2017-01-15 01:55:54,http://jast.net/clarabelle,,246.77.195.227,"{""location"": ""GI"", ""is_mobile"": true}" 21052,8,474,2017-03-19 17:17:10,http://becker.com/carolanne_tillman,,241.245.251.129,"{""location"": ""TR"", ""is_mobile"": false}" 21053,8,474,2017-03-11 21:38:30,http://mcdermott.name/monserrat,,58.95.22.196,"{""location"": ""HR"", ""is_mobile"": false}" 21054,8,474,2017-01-05 00:21:05,http://feil.net/natasha,,243.212.193.204,"{""location"": ""GG"", ""is_mobile"": false}" 21055,8,474,2017-03-24 18:36:28,http://jonesmann.io/jace_christiansen,,75.67.156.107,"{""location"": ""CA"", ""is_mobile"": true}" 21056,8,474,2017-02-18 06:27:00,http://faheyhane.info/abbie_eichmann,,115.164.165.4,"{""location"": ""BQ"", ""is_mobile"": true}" 21057,8,474,2017-01-15 18:28:03,http://klockopollich.name/arlie.ritchie,,215.38.238.168,"{""location"": ""GS"", ""is_mobile"": true}" 21058,8,474,2017-01-22 23:41:02,http://kautzercain.biz/brenden_donnelly,,201.221.188.203,"{""location"": ""IR"", ""is_mobile"": true}" 21059,8,474,2017-01-06 22:33:16,http://halvorson.io/dasia,,202.156.170.209,"{""location"": ""AT"", ""is_mobile"": true}" 21060,8,474,2017-02-22 16:34:21,http://mertz.org/buck_lueilwitz,,190.180.45.74,"{""location"": ""YE"", ""is_mobile"": true}" 21061,8,474,2017-01-15 06:31:09,http://ohara.net/kendall,,167.75.123.42,"{""location"": ""AU"", ""is_mobile"": false}" 21062,8,474,2017-05-08 08:07:21,http://shanahan.com/kendra,,66.136.16.94,"{""location"": ""LB"", ""is_mobile"": true}" 21063,8,474,2017-02-01 15:17:13,http://mueller.io/vincenza,,20.151.168.170,"{""location"": ""LS"", ""is_mobile"": false}" 21064,8,474,2017-02-27 17:05:13,http://purdy.biz/tanner_glover,,228.191.66.240,"{""location"": ""GP"", ""is_mobile"": true}" 21065,8,474,2017-04-14 16:03:10,http://collins.co/irma,,26.60.167.220,"{""location"": ""GW"", ""is_mobile"": true}" 21066,8,474,2017-03-15 18:31:01,http://willmsboyle.biz/karl.stamm,,58.61.67.24,"{""location"": ""IR"", ""is_mobile"": true}" 21067,8,474,2017-05-21 16:59:52,http://jacobson.org/maritza.kuhn,,138.84.165.89,"{""location"": ""SI"", ""is_mobile"": false}" 21068,8,474,2017-06-06 00:36:36,http://kihn.net/joaquin.dickens,,89.148.71.17,"{""location"": ""BI"", ""is_mobile"": false}" 21069,8,474,2017-02-11 15:58:53,http://crist.net/dean_cronin,,7.104.218.146,"{""location"": ""AR"", ""is_mobile"": true}" 21070,8,474,2017-03-27 10:04:38,http://wehner.com/grayce,,183.3.86.138,"{""location"": ""MZ"", ""is_mobile"": true}" 21071,8,474,2017-05-04 06:08:52,http://boyerhilpert.biz/sid,,175.16.225.99,"{""location"": ""WS"", ""is_mobile"": false}" 21072,8,474,2017-03-18 04:04:36,http://stark.com/hazel.nikolaus,,254.85.152.70,"{""location"": ""BD"", ""is_mobile"": true}" 21073,8,474,2017-04-29 16:20:24,http://cronin.biz/earl_reichert,,224.161.170.198,"{""location"": ""CI"", ""is_mobile"": true}" 21074,8,474,2017-04-01 09:38:53,http://altenwerthweimann.biz/ruben,,22.178.2.131,"{""location"": ""DZ"", ""is_mobile"": false}" 21075,8,474,2017-05-22 13:31:57,http://gaylord.info/vicky,,205.16.171.137,"{""location"": ""BH"", ""is_mobile"": true}" 21076,8,474,2017-04-04 15:44:32,http://pacochajacobi.com/alvina_hettinger,,241.111.144.230,"{""location"": ""FJ"", ""is_mobile"": true}" 21077,8,474,2016-12-31 21:35:28,http://runolfsdottir.com/queen.king,,73.125.121.41,"{""location"": ""LB"", ""is_mobile"": false}" 21078,8,474,2017-04-29 01:44:05,http://carroll.biz/daniella,,115.63.11.199,"{""location"": ""AG"", ""is_mobile"": true}" 21079,8,474,2016-12-19 02:18:42,http://schmitt.org/otto_waelchi,,196.70.30.165,"{""location"": ""BY"", ""is_mobile"": true}" 21080,8,474,2017-01-22 02:09:47,http://walter.co/rey.jacobs,,209.52.141.190,"{""location"": ""MA"", ""is_mobile"": true}" 21081,8,474,2017-04-21 13:34:56,http://huelsfritsch.net/braeden.bogan,,253.216.207.242,"{""location"": ""SM"", ""is_mobile"": false}" 21082,8,474,2017-05-15 21:28:33,http://bosconader.info/junius,,159.116.91.225,"{""location"": ""PE"", ""is_mobile"": false}" 21083,8,474,2017-02-06 08:32:36,http://stoltenbergweber.org/kristina_gerlach,,34.85.47.54,"{""location"": ""JP"", ""is_mobile"": false}" 21084,8,474,2017-06-02 00:34:18,http://leschgrady.com/stephania.howe,,80.216.156.239,"{""location"": ""PG"", ""is_mobile"": true}" 21085,8,474,2016-12-17 17:22:30,http://gaylordstanton.name/theresia.borer,,47.11.177.24,"{""location"": ""QA"", ""is_mobile"": false}" 21086,8,474,2017-03-18 03:24:02,http://stracke.info/jamar_kulas,,165.248.36.140,"{""location"": ""AU"", ""is_mobile"": false}" 21087,8,474,2017-04-11 18:45:25,http://ferry.co/moie.herman,,62.66.215.251,"{""location"": ""IN"", ""is_mobile"": false}" 21088,8,474,2017-03-12 15:35:39,http://johnson.com/krista,,211.17.120.135,"{""location"": ""MD"", ""is_mobile"": true}" 21089,8,474,2017-03-07 08:29:48,http://grady.info/tyler,,60.43.237.139,"{""location"": ""KI"", ""is_mobile"": false}" 21090,8,474,2017-01-06 02:07:20,http://koeppspencer.com/elizabeth_pacocha,,185.157.242.205,"{""location"": ""SH"", ""is_mobile"": true}" 21091,8,474,2017-02-08 03:40:45,http://sanfordwyman.io/golden,,120.19.63.84,"{""location"": ""NG"", ""is_mobile"": true}" 21092,8,474,2017-02-05 12:28:32,http://bergemccullough.co/deven_schuppe,,103.6.226.229,"{""location"": ""PM"", ""is_mobile"": true}" 21093,8,474,2017-01-20 18:09:10,http://blanda.org/jettie_frami,,47.156.44.66,"{""location"": ""EG"", ""is_mobile"": false}" 21094,8,474,2017-05-21 08:21:28,http://hills.net/leland,,93.163.167.143,"{""location"": ""DK"", ""is_mobile"": true}" 21095,8,474,2016-12-29 16:33:39,http://macejkovic.biz/mallory.denesik,,25.103.198.65,"{""location"": ""BG"", ""is_mobile"": true}" 21096,8,474,2017-02-25 02:36:31,http://johnstonosinski.name/josiane.schamberger,,90.235.249.185,"{""location"": ""PH"", ""is_mobile"": false}" 21097,8,474,2017-02-01 23:15:11,http://ruel.org/lelah_mccullough,,146.91.118.53,"{""location"": ""RU"", ""is_mobile"": true}" 21098,8,474,2017-06-12 00:01:53,http://zulaufbalistreri.org/dimitri_windler,,30.30.52.108,"{""location"": ""QA"", ""is_mobile"": false}" 21099,8,474,2017-05-05 16:40:11,http://kelerbahringer.biz/rory,,120.214.169.120,"{""location"": ""KE"", ""is_mobile"": true}" 21100,8,475,2017-02-14 18:06:39,http://trantow.io/frances_schmidt,,206.187.13.76,"{""location"": ""LC"", ""is_mobile"": true}" 1,1,1,2016-12-28 15:41:12,http://wisozk.org/sonya.herzog,0.3889986856,26.188.235.197,"{""location"": ""AS"", ""is_mobile"": true}" 2,1,1,2017-05-10 07:13:45,http://batzwest.org/reed,0.0638372400,32.158.161.111,"{""location"": ""PG"", ""is_mobile"": true}" 3,1,1,2017-06-01 22:41:32,http://howe.biz/priscilla,0.6467528056,17.104.94.47,"{""location"": ""AS"", ""is_mobile"": true}" 4,1,1,2017-01-10 16:39:34,http://auer.name/dave.johnston,0.7427506486,243.186.9.121,"{""location"": ""YE"", ""is_mobile"": true}" 5,1,1,2017-03-06 22:29:08,http://halvorsonrutherford.name/imani_fahey,0.4283587081,161.226.163.83,"{""location"": ""UG"", ""is_mobile"": true}" 6,1,1,2017-03-16 10:51:52,http://quigley.biz/howard,0.1766760636,102.154.56.47,"{""location"": ""SZ"", ""is_mobile"": false}" 7,1,1,2017-02-05 07:34:31,http://jacobsonklocko.net/willow,0.1532288024,170.152.94.48,"{""location"": ""UG"", ""is_mobile"": true}" 8,1,1,2017-06-04 23:04:25,http://ruel.name/marquise.mertz,0.0606385498,34.205.150.206,"{""location"": ""TM"", ""is_mobile"": false}" 9,1,1,2017-02-04 07:46:27,http://kaulke.net/jeremy,0.4834670365,109.61.30.120,"{""location"": ""SM"", ""is_mobile"": true}" 10,1,1,2017-03-10 11:48:58,http://larkin.biz/sean.hamill,0.7836783542,55.189.154.209,"{""location"": ""ZM"", ""is_mobile"": true}" 11,1,1,2017-02-16 01:12:55,http://stokes.biz/deja,0.0527221629,94.109.124.93,"{""location"": ""MP"", ""is_mobile"": true}" 12,1,1,2017-02-18 16:26:17,http://kulas.net/ervin,0.4500981266,75.214.183.240,"{""location"": ""AL"", ""is_mobile"": true}" 13,1,1,2017-02-20 01:28:24,http://schuster.io/christophe.adams,0.8720638012,85.182.62.188,"{""location"": ""LI"", ""is_mobile"": false}" 14,1,1,2017-01-09 23:41:28,http://bruen.info/fausto.hansen,0.3775360146,162.214.214.66,"{""location"": ""AQ"", ""is_mobile"": false}" 15,1,1,2017-01-11 19:27:10,http://zemlak.info/ocie,0.7637572025,229.102.154.44,"{""location"": ""AE"", ""is_mobile"": false}" 16,1,1,2017-06-08 12:14:07,http://runolfon.com/zechariah,0.0295705185,177.233.153.80,"{""location"": ""VI"", ""is_mobile"": false}" 17,1,1,2017-01-31 02:26:17,http://hoeger.co/marcellus,0.5192454333,136.239.43.229,"{""location"": ""ZW"", ""is_mobile"": true}" 18,1,1,2017-04-17 06:57:41,http://hellerwiegand.biz/darlene.konopelski,0.6917053221,130.70.206.43,"{""location"": ""LC"", ""is_mobile"": false}" 19,1,1,2017-01-17 17:37:17,http://sporer.org/joey,0.4197002462,50.142.252.130,"{""location"": ""LU"", ""is_mobile"": true}" 20,1,1,2017-05-17 17:31:48,http://rau.name/vincenzo,0.8694303306,38.254.5.203,"{""location"": ""JP"", ""is_mobile"": false}" 21,1,1,2017-03-09 19:22:06,http://runolfontromp.co/elisabeth,0.6498390032,43.229.91.171,"{""location"": ""CH"", ""is_mobile"": true}" 22,1,1,2017-02-08 12:13:05,http://erdmanlubowitz.co/german_wolf,0.4904417670,176.239.100.21,"{""location"": ""MY"", ""is_mobile"": false}" 23,1,1,2017-03-23 08:54:50,http://daniel.name/tony,0.8588046862,74.215.183.204,"{""location"": ""NO"", ""is_mobile"": false}" 24,1,1,2017-01-21 12:15:50,http://marquardt.net/ignatius,0.3364057220,24.248.181.46,"{""location"": ""IL"", ""is_mobile"": false}" 25,1,1,2016-12-15 13:53:30,http://hammes.io/rudy.keeling,0.7934070793,179.145.245.93,"{""location"": ""BH"", ""is_mobile"": false}" 26,1,1,2017-04-23 19:43:58,http://walker.co/martine_treutel,0.1573282271,132.27.160.44,"{""location"": ""GB"", ""is_mobile"": false}" 27,1,2,2017-01-17 06:01:02,http://krajcik.name/tia_steuber,0.0079417672,140.121.198.44,"{""location"": ""PW"", ""is_mobile"": false}" 28,1,2,2017-04-20 08:22:00,http://hayes.co/hester,0.5333644539,22.162.27.226,"{""location"": ""AU"", ""is_mobile"": false}" 29,1,2,2017-05-06 23:21:33,http://yost.info/kenneth,0.3397297211,64.59.55.97,"{""location"": ""GR"", ""is_mobile"": false}" 30,1,2,2017-02-03 06:17:48,http://thiel.name/sheila,0.3091657443,79.249.221.62,"{""location"": ""OM"", ""is_mobile"": false}" 31,1,2,2017-04-30 21:18:24,http://sipes.name/golden_harvey,0.2503317316,94.67.39.29,"{""location"": ""NU"", ""is_mobile"": true}" 32,1,2,2017-06-07 06:02:51,http://kilbackbrekke.org/adele.runolfon,0.4153793646,143.109.158.192,"{""location"": ""AM"", ""is_mobile"": false}" 33,1,2,2017-04-11 07:52:48,http://rowe.com/chase,0.6382217585,128.195.119.48,"{""location"": ""LI"", ""is_mobile"": true}" 34,1,2,2017-03-19 03:32:31,http://howeledner.org/abelardo.kertzmann,0.5798386320,111.125.240.232,"{""location"": ""MQ"", ""is_mobile"": false}" 35,1,2,2017-03-01 14:28:50,http://considine.org/eloisa,0.5738457552,84.254.178.198,"{""location"": ""RO"", ""is_mobile"": true}" 36,1,2,2017-05-24 05:51:13,http://gutkowski.net/delphia_grant,0.9937431682,85.89.85.236,"{""location"": ""AW"", ""is_mobile"": false}" 37,1,2,2017-05-10 22:40:20,http://hermanntoy.io/reyes_feil,0.9662424061,171.155.55.77,"{""location"": ""LA"", ""is_mobile"": false}" 38,1,2,2017-01-17 14:58:50,http://hartmannbechtelar.net/marcelle,0.0501072762,142.249.144.160,"{""location"": ""AT"", ""is_mobile"": true}" 39,1,2,2017-02-24 16:00:05,http://torp.biz/adaline.feest,0.7056395420,68.150.202.54,"{""location"": ""CH"", ""is_mobile"": true}" 40,1,2,2017-04-25 15:44:05,http://sanford.io/brent,0.7666997789,214.139.63.172,"{""location"": ""FI"", ""is_mobile"": false}" 41,1,2,2017-05-13 16:32:13,http://mcclure.io/anika_ferry,0.7244277388,52.43.18.91,"{""location"": ""CI"", ""is_mobile"": true}" 42,1,2,2017-04-05 02:09:36,http://fayweimann.io/ernie.bergnaum,0.4478818614,136.239.79.72,"{""location"": ""NI"", ""is_mobile"": false}" 43,1,2,2017-05-10 16:34:47,http://bailey.com/santino.swift,0.1799102658,171.55.242.248,"{""location"": ""LU"", ""is_mobile"": true}" 44,1,2,2017-01-30 08:22:08,http://casperhills.info/reynold.moriette,0.7838694881,222.74.140.227,"{""location"": ""LV"", ""is_mobile"": true}" 45,1,2,2017-01-24 01:16:16,http://rutherford.co/alaina,0.1350277039,40.110.63.158,"{""location"": ""TV"", ""is_mobile"": false}" 46,1,2,2017-02-11 16:56:10,http://connelly.org/clementine.herzog,0.7663320163,64.40.133.96,"{""location"": ""CA"", ""is_mobile"": false}" 47,1,2,2017-03-08 20:26:32,http://mannmckenzie.com/chanelle,0.1063036380,166.148.96.180,"{""location"": ""UG"", ""is_mobile"": false}" 48,1,2,2017-04-21 05:18:56,http://dubuque.io/lelia.steuber,0.9020465502,133.18.180.254,"{""location"": ""MS"", ""is_mobile"": true}" 49,1,2,2017-06-12 04:39:41,http://rippineichmann.co/colleen_howe,0.9121157540,236.83.222.116,"{""location"": ""PT"", ""is_mobile"": true}" 50,1,2,2017-01-03 15:57:54,http://lehner.biz/crystal_bergstrom,0.0969568597,135.90.22.83,"{""location"": ""SX"", ""is_mobile"": false}" 51,1,2,2017-05-26 03:48:58,http://schinner.co/loma,0.7837701570,236.21.11.70,"{""location"": ""HR"", ""is_mobile"": true}" 21101,8,475,2017-05-18 15:20:44,http://marksrempel.net/breanna,,174.182.197.192,"{""location"": ""MZ"", ""is_mobile"": false}" 21102,8,475,2017-04-10 04:19:03,http://will.name/rosemarie_armstrong,,179.154.137.83,"{""location"": ""TN"", ""is_mobile"": false}" 21103,8,475,2017-05-13 23:28:52,http://smitham.info/evelyn,,53.56.169.5,"{""location"": ""KR"", ""is_mobile"": true}" 21104,8,475,2017-06-05 08:17:06,http://weinat.io/branson,,88.227.32.76,"{""location"": ""IT"", ""is_mobile"": true}" 21105,8,475,2017-05-13 01:11:07,http://smith.co/verna,,58.82.148.146,"{""location"": ""AQ"", ""is_mobile"": false}" 21106,8,475,2016-12-22 02:35:50,http://gutmannbins.com/daphney,,253.38.108.124,"{""location"": ""CF"", ""is_mobile"": true}" 21107,8,475,2017-03-01 15:43:39,http://powlowskiharris.co/alexander,,39.127.45.87,"{""location"": ""NG"", ""is_mobile"": false}" 21108,8,475,2017-03-28 12:09:01,http://hackett.name/briana,,22.184.41.162,"{""location"": ""JO"", ""is_mobile"": true}" 21109,8,475,2017-04-05 10:46:40,http://aufderharmclaughlin.com/precious,,166.199.146.250,"{""location"": ""MN"", ""is_mobile"": true}" 21110,8,475,2017-05-31 16:29:56,http://hilpertruel.org/karianne.will,,148.105.96.126,"{""location"": ""LV"", ""is_mobile"": false}" 21111,8,475,2017-03-19 18:00:15,http://metz.org/karen_lueilwitz,,102.175.201.88,"{""location"": ""TL"", ""is_mobile"": true}" 21112,8,475,2017-05-04 16:31:55,http://donnelly.com/della.hackett,,114.133.75.130,"{""location"": ""IN"", ""is_mobile"": false}" 21113,8,475,2017-03-30 11:29:04,http://wolf.io/deanna,,113.249.156.42,"{""location"": ""IR"", ""is_mobile"": true}" 21114,8,475,2017-06-11 14:19:29,http://brakus.name/quentin_okuneva,,53.170.230.9,"{""location"": ""VA"", ""is_mobile"": false}" 21115,8,475,2017-05-23 05:39:58,http://conn.info/karianne,,162.150.191.222,"{""location"": ""TG"", ""is_mobile"": false}" 21116,8,475,2017-02-19 18:31:16,http://greenholt.com/chasity.marks,,143.52.163.5,"{""location"": ""MG"", ""is_mobile"": true}" 21117,8,475,2017-06-06 17:47:52,http://witting.info/tavares.lebsack,,133.212.71.139,"{""location"": ""NZ"", ""is_mobile"": false}" 21118,8,475,2017-04-20 15:38:46,http://weber.co/mitchel.powlowski,,18.66.152.170,"{""location"": ""PT"", ""is_mobile"": true}" 21119,8,475,2017-05-12 20:35:09,http://turnerwalker.io/soledad_kunze,,138.15.129.103,"{""location"": ""HR"", ""is_mobile"": true}" 21120,8,475,2017-04-06 07:03:45,http://wintheiser.biz/gunner.witting,,70.209.101.240,"{""location"": ""MV"", ""is_mobile"": true}" 21121,8,475,2017-06-03 04:26:41,http://hirthe.biz/allan.legros,,168.205.223.223,"{""location"": ""UM"", ""is_mobile"": true}" 21122,8,475,2017-06-07 06:25:06,http://prosacco.net/helen,,159.227.94.130,"{""location"": ""UM"", ""is_mobile"": true}" 21123,8,475,2017-03-09 17:42:22,http://klein.co/callie,,118.119.218.176,"{""location"": ""OM"", ""is_mobile"": true}" 21124,8,475,2017-03-15 07:33:28,http://mohrdubuque.biz/adrianna,,140.171.144.91,"{""location"": ""YT"", ""is_mobile"": true}" 21125,8,475,2017-03-18 12:40:02,http://kozey.io/prudence.murray,,200.53.155.78,"{""location"": ""AI"", ""is_mobile"": false}" 21126,8,475,2017-01-17 14:19:52,http://doyle.net/lexus.rau,,115.42.5.101,"{""location"": ""ET"", ""is_mobile"": true}" 21127,8,475,2017-04-24 16:37:05,http://lemke.name/keyon,,232.67.218.242,"{""location"": ""MG"", ""is_mobile"": false}" 21128,8,475,2017-02-26 05:50:19,http://balistrerimorar.org/alysa,,21.141.212.222,"{""location"": ""BV"", ""is_mobile"": true}" 21129,8,475,2017-02-10 00:51:47,http://wiegand.name/tina,,5.215.6.112,"{""location"": ""JO"", ""is_mobile"": false}" 21130,8,475,2017-03-11 18:08:56,http://windlerkub.biz/nathanael,,122.194.144.161,"{""location"": ""FM"", ""is_mobile"": true}" 21131,8,475,2017-04-21 18:21:40,http://wolffnienow.info/madalyn,,28.132.189.18,"{""location"": ""HU"", ""is_mobile"": false}" 21132,8,475,2017-03-24 23:59:39,http://hoegercrist.org/alvis.mills,,36.53.15.155,"{""location"": ""SZ"", ""is_mobile"": false}" 21133,8,475,2017-04-02 21:28:15,http://danielthompson.org/easter.corwin,,124.90.69.200,"{""location"": ""UZ"", ""is_mobile"": true}" 21134,8,475,2017-01-09 02:52:36,http://hilpert.io/ralph.gutkowski,,53.64.102.201,"{""location"": ""BO"", ""is_mobile"": true}" 21135,8,475,2016-12-21 00:14:03,http://farrellstoltenberg.net/tony_volkman,,86.164.27.64,"{""location"": ""IM"", ""is_mobile"": true}" 21136,8,475,2017-02-21 14:45:30,http://hahn.org/kyla_reynolds,,189.111.122.90,"{""location"": ""LA"", ""is_mobile"": true}" 21137,8,475,2016-12-25 17:40:37,http://gaylord.io/shyanne.zboncak,,17.216.118.190,"{""location"": ""BI"", ""is_mobile"": false}" 21138,8,475,2017-01-13 04:36:31,http://davis.biz/ulises_wiza,,99.112.243.102,"{""location"": ""UG"", ""is_mobile"": true}" 21139,8,475,2017-03-22 09:43:18,http://hegmann.org/hilario,,215.84.141.149,"{""location"": ""CY"", ""is_mobile"": true}" 21140,8,475,2017-01-22 17:20:08,http://eichmann.biz/esperanza.carter,,90.30.26.158,"{""location"": ""UM"", ""is_mobile"": false}" 21141,8,475,2017-02-16 12:54:48,http://westparker.biz/monte,,82.53.210.17,"{""location"": ""TT"", ""is_mobile"": true}" 21142,8,475,2017-04-12 04:19:41,http://thompson.biz/lonzo.kreiger,,50.68.216.75,"{""location"": ""CC"", ""is_mobile"": false}" 21143,8,475,2017-05-14 21:39:08,http://smitham.io/rex.ziemann,,161.145.20.150,"{""location"": ""MU"", ""is_mobile"": false}" 21144,8,475,2017-02-25 00:12:08,http://conroy.net/xavier,,140.110.118.231,"{""location"": ""VU"", ""is_mobile"": true}" 21145,8,475,2017-04-15 04:55:09,http://macgyver.org/jovani,,229.50.233.233,"{""location"": ""DK"", ""is_mobile"": true}" 21146,8,475,2017-02-16 13:10:28,http://wolffmcclure.co/autumn,,187.54.90.82,"{""location"": ""SC"", ""is_mobile"": false}" 21147,8,475,2017-02-03 11:03:47,http://ebert.biz/oral_cronin,,152.190.72.19,"{""location"": ""SX"", ""is_mobile"": false}" 21148,8,475,2016-12-20 12:12:45,http://bradtke.biz/ana.sipes,,85.173.7.67,"{""location"": ""CZ"", ""is_mobile"": false}" 21149,8,475,2017-06-04 12:13:31,http://hoeger.com/aliyah.kunze,,131.177.117.207,"{""location"": ""CG"", ""is_mobile"": true}" 21150,8,475,2016-12-15 12:48:16,http://nienow.co/kattie.zulauf,,123.174.180.110,"{""location"": ""BS"", ""is_mobile"": false}" 21151,8,475,2016-12-29 09:43:51,http://mclaughlin.net/peggie,,238.107.185.241,"{""location"": ""HT"", ""is_mobile"": false}" 21152,8,475,2017-05-02 22:10:46,http://macejkovichoppe.io/kavon,,16.9.86.95,"{""location"": ""NL"", ""is_mobile"": false}" 21153,8,475,2017-04-25 09:07:41,http://donnelly.io/shaniya,,17.118.154.52,"{""location"": ""VU"", ""is_mobile"": false}" 21154,8,475,2017-01-18 14:04:08,http://rohan.net/lorenza,,176.77.82.33,"{""location"": ""SS"", ""is_mobile"": true}" 21155,8,475,2017-05-20 00:35:21,http://jast.org/elva,,142.208.216.94,"{""location"": ""SM"", ""is_mobile"": false}" 52,1,2,2017-06-13 04:55:12,http://tillman.biz/hortense,0.2756487119,170.33.171.155,"{""location"": ""LI"", ""is_mobile"": false}" 53,1,2,2017-02-21 13:00:33,http://marquardt.org/amber,0.9261337557,13.143.25.68,"{""location"": ""IS"", ""is_mobile"": false}" 54,1,2,2017-03-21 04:45:57,http://denesikondricka.name/jailyn_mraz,0.6055894759,148.127.195.70,"{""location"": ""BR"", ""is_mobile"": false}" 55,1,2,2016-12-25 11:50:14,http://carter.com/janice.farrell,0.6343735216,154.91.126.136,"{""location"": ""SC"", ""is_mobile"": false}" 56,1,2,2017-05-06 11:34:17,http://rath.info/earlene,0.0701133420,139.246.186.154,"{""location"": ""JM"", ""is_mobile"": false}" 57,1,2,2017-03-31 20:02:35,http://hammes.biz/dane.macejkovic,0.5892573428,21.159.213.109,"{""location"": ""AI"", ""is_mobile"": true}" 58,1,2,2017-01-22 10:21:05,http://heel.name/shawna_sauer,0.4535145446,182.70.56.55,"{""location"": ""PW"", ""is_mobile"": false}" 59,1,2,2017-04-10 16:34:32,http://reilly.com/pat,0.3653598724,110.200.199.101,"{""location"": ""MW"", ""is_mobile"": false}" 60,1,2,2017-05-27 08:32:17,http://gloverprice.info/jedidiah_simonis,0.3568039311,141.144.138.131,"{""location"": ""LR"", ""is_mobile"": true}" 61,1,2,2017-03-26 03:36:26,http://ernser.info/robbie,0.5785665848,143.32.230.134,"{""location"": ""LU"", ""is_mobile"": true}" 62,1,2,2017-03-04 07:45:20,http://ullrich.biz/travon.fadel,0.5472223873,36.157.40.86,"{""location"": ""GN"", ""is_mobile"": false}" 63,1,2,2017-03-29 16:50:44,http://hamill.co/mackenzie,0.6083412824,184.244.192.150,"{""location"": ""NE"", ""is_mobile"": true}" 64,1,2,2017-04-02 21:06:39,http://kunzekohler.biz/myrtle,0.8276967649,161.192.194.241,"{""location"": ""LY"", ""is_mobile"": false}" 65,1,2,2017-06-09 22:13:22,http://paucek.com/randy,0.4086336508,137.150.186.54,"{""location"": ""NI"", ""is_mobile"": true}" 66,1,3,2017-02-12 07:57:24,http://hahn.biz/fidel_waters,0.3889022593,4.230.51.14,"{""location"": ""LI"", ""is_mobile"": true}" 67,1,3,2017-01-12 11:45:12,http://lowegrimes.com/jana,0.0761463508,218.4.220.11,"{""location"": ""CM"", ""is_mobile"": false}" 68,1,3,2017-03-15 04:36:24,http://mcdermottbuckridge.info/gretchen_kerluke,0.9956589291,3.232.171.243,"{""location"": ""LT"", ""is_mobile"": false}" 69,1,3,2017-02-02 20:42:19,http://gleason.co/rigoberto_schneider,0.1498752570,197.29.133.181,"{""location"": ""MQ"", ""is_mobile"": false}" 70,1,3,2017-02-19 03:53:09,http://connellyparker.org/samson_aufderhar,0.1322172271,2.153.179.122,"{""location"": ""MX"", ""is_mobile"": true}" 71,1,3,2017-05-16 13:51:11,http://block.co/hank.marks,0.0656715468,219.11.70.126,"{""location"": ""MR"", ""is_mobile"": true}" 72,1,3,2017-03-17 17:17:41,http://towne.co/gunnar.larkin,0.0259777458,182.57.71.22,"{""location"": ""GF"", ""is_mobile"": false}" 73,1,3,2016-12-14 09:31:57,http://lefflerkiehn.co/kacey,0.7224987538,72.137.101.165,"{""location"": ""ZW"", ""is_mobile"": true}" 74,1,3,2017-04-03 20:11:07,http://reichel.org/donny.donnelly,0.3498653809,146.132.39.211,"{""location"": ""PR"", ""is_mobile"": false}" 75,1,3,2016-12-21 03:40:01,http://lang.name/neoma,0.4265966274,67.161.17.228,"{""location"": ""PE"", ""is_mobile"": false}" 76,1,3,2017-03-28 04:43:32,http://greenholt.net/laney.tremblay,0.1198275621,224.86.119.31,"{""location"": ""PS"", ""is_mobile"": false}" 77,1,3,2017-03-18 07:51:48,http://metz.name/kenyon.zemlak,0.7292641463,215.238.163.22,"{""location"": ""GA"", ""is_mobile"": false}" 78,1,3,2017-03-22 08:50:23,http://reynoldsferry.com/april_sipes,0.8745492405,184.181.183.61,"{""location"": ""SN"", ""is_mobile"": false}" 79,1,3,2016-12-23 11:55:26,http://schmitt.co/mckenna,0.8740530481,57.211.221.37,"{""location"": ""VN"", ""is_mobile"": false}" 80,1,3,2017-05-21 14:19:51,http://miller.info/thaddeus_crist,0.1973919184,47.122.82.55,"{""location"": ""PH"", ""is_mobile"": true}" 81,1,3,2017-03-07 00:23:47,http://franecki.net/enoch.parker,0.1691501644,188.50.189.129,"{""location"": ""RS"", ""is_mobile"": true}" 82,1,3,2017-02-12 10:17:00,http://muller.io/genoveva.donnelly,0.8380742106,81.174.84.52,"{""location"": ""LT"", ""is_mobile"": false}" 83,1,3,2017-03-13 20:58:31,http://halvorson.biz/arvel_pfeffer,0.4324168422,73.112.169.169,"{""location"": ""AX"", ""is_mobile"": false}" 84,1,3,2017-05-17 03:32:16,http://gradyreichel.io/oceane.herman,0.7048880959,149.247.135.184,"{""location"": ""FK"", ""is_mobile"": false}" 85,1,3,2017-04-05 15:43:40,http://huelsenger.biz/timothy.oconner,0.5912624312,50.145.163.237,"{""location"": ""TJ"", ""is_mobile"": false}" 86,1,3,2017-04-17 06:39:24,http://hodkiewiczthompson.info/audra.paucek,0.8031535367,111.93.176.232,"{""location"": ""CN"", ""is_mobile"": true}" 87,1,3,2017-05-31 06:29:28,http://fay.biz/henderson_ortiz,0.9611721198,17.135.51.21,"{""location"": ""MX"", ""is_mobile"": false}" 88,1,3,2017-01-31 09:53:45,http://feestmorar.name/oceane,0.5488106716,185.64.195.249,"{""location"": ""TH"", ""is_mobile"": false}" 89,1,3,2016-12-20 12:21:28,http://macgyverboyer.net/dorian,0.3399680484,138.123.60.174,"{""location"": ""TN"", ""is_mobile"": false}" 90,1,3,2017-05-17 01:10:02,http://gislason.com/ron,0.8305261708,50.101.163.25,"{""location"": ""PY"", ""is_mobile"": true}" 91,1,3,2017-05-08 20:49:13,http://feeney.org/claudine.gorczany,0.3117818379,132.202.110.37,"{""location"": ""MK"", ""is_mobile"": false}" 92,1,3,2017-01-18 17:27:48,http://king.info/daisha,0.9503684113,21.53.145.74,"{""location"": ""BH"", ""is_mobile"": false}" 93,1,3,2017-04-24 18:21:58,http://stehr.net/ruthe,0.6267859364,139.155.214.115,"{""location"": ""MQ"", ""is_mobile"": false}" 94,1,3,2017-01-14 21:41:43,http://pagac.name/alaina,0.4251381545,113.104.8.31,"{""location"": ""TD"", ""is_mobile"": false}" 95,1,3,2017-03-31 04:15:12,http://manntillman.name/linda,0.1438333256,42.45.97.97,"{""location"": ""MA"", ""is_mobile"": false}" 96,1,3,2017-05-26 18:23:56,http://toylittle.name/sigrid.rippin,0.2939488363,46.105.84.208,"{""location"": ""DK"", ""is_mobile"": true}" 97,1,3,2017-01-05 11:10:56,http://mitchellmarks.co/ardith_schoen,0.3229934275,86.203.137.112,"{""location"": ""US"", ""is_mobile"": false}" 98,1,3,2017-03-15 05:55:12,http://kertzmann.info/carol.kuphal,0.1190198799,137.164.170.240,"{""location"": ""MS"", ""is_mobile"": true}" 99,1,3,2016-12-19 22:13:39,http://ward.co/nelson.quitzon,0.7167377663,212.21.132.231,"{""location"": ""NL"", ""is_mobile"": true}" 100,1,3,2016-12-18 22:36:28,http://veum.io/nestor.lakin,0.1516013059,181.197.250.177,"{""location"": ""LS"", ""is_mobile"": false}" 101,1,3,2017-04-15 01:10:44,http://raumills.co/lukas_friesen,0.8996811047,86.138.241.252,"{""location"": ""PE"", ""is_mobile"": false}" 102,1,3,2017-05-05 22:47:16,http://dubuque.info/chandler_pacocha,0.3647552999,241.17.11.202,"{""location"": ""MF"", ""is_mobile"": false}" 103,1,3,2017-05-24 19:41:25,http://wiegandrice.io/kellen.hackett,0.3783671295,79.251.206.155,"{""location"": ""FK"", ""is_mobile"": true}" 21156,8,475,2017-04-23 05:12:56,http://sanfordrunte.name/korey,,177.64.44.228,"{""location"": ""MF"", ""is_mobile"": true}" 21157,8,475,2017-03-03 13:12:41,http://jenkins.com/johathan.hauck,,241.249.250.141,"{""location"": ""BV"", ""is_mobile"": true}" 21158,8,475,2017-02-25 00:17:19,http://torp.name/bernadette_bauch,,173.158.19.64,"{""location"": ""AQ"", ""is_mobile"": false}" 21159,8,475,2017-01-02 17:38:00,http://bergnaum.net/tania_auer,,106.188.169.172,"{""location"": ""KZ"", ""is_mobile"": true}" 21160,8,475,2017-03-15 23:47:39,http://hahn.co/jefferey_hagenes,,222.160.17.44,"{""location"": ""ZW"", ""is_mobile"": false}" 21161,8,475,2017-01-13 09:05:19,http://schultzfahey.biz/euna,,82.137.159.204,"{""location"": ""TK"", ""is_mobile"": true}" 21162,8,475,2016-12-25 04:07:11,http://bins.info/seamus.beer,,220.182.249.42,"{""location"": ""ES"", ""is_mobile"": true}" 21163,8,476,2017-04-16 04:13:45,http://mcclure.info/dimitri,,202.13.77.86,"{""location"": ""ST"", ""is_mobile"": true}" 21164,8,476,2017-01-02 13:17:51,http://brown.io/walter,,69.156.6.164,"{""location"": ""DO"", ""is_mobile"": true}" 21165,8,476,2017-03-10 17:38:47,http://shieldsabbott.net/tillman,,23.154.52.135,"{""location"": ""SL"", ""is_mobile"": false}" 21166,8,476,2017-03-14 17:07:37,http://gerlach.info/gunnar,,45.204.126.119,"{""location"": ""MX"", ""is_mobile"": false}" 21167,8,476,2017-01-10 12:57:13,http://rogahn.io/alta,,98.213.150.74,"{""location"": ""AW"", ""is_mobile"": false}" 21168,8,476,2017-04-07 22:44:00,http://kuvalis.org/julia.strosin,,79.48.56.120,"{""location"": ""SR"", ""is_mobile"": false}" 21169,8,476,2017-05-14 18:22:32,http://miller.org/te.rutherford,,176.49.189.200,"{""location"": ""CV"", ""is_mobile"": true}" 21170,8,476,2017-05-29 18:55:02,http://turcotte.biz/janae,,146.160.2.66,"{""location"": ""SX"", ""is_mobile"": false}" 21171,8,476,2017-05-24 06:42:55,http://dickinson.org/malachi_wolff,,201.129.45.41,"{""location"": ""KP"", ""is_mobile"": false}" 21172,8,476,2017-03-05 16:17:39,http://kaulke.net/jaunita,,3.137.239.80,"{""location"": ""SZ"", ""is_mobile"": true}" 21173,8,476,2017-05-27 11:16:01,http://davis.io/gerald,,204.246.119.117,"{""location"": ""MD"", ""is_mobile"": true}" 21174,8,476,2017-05-27 13:08:30,http://walshwilderman.name/sierra.reichel,,197.239.245.27,"{""location"": ""TG"", ""is_mobile"": false}" 21175,8,476,2017-04-23 11:49:01,http://grady.net/albin,,27.174.162.11,"{""location"": ""KG"", ""is_mobile"": true}" 21176,8,476,2017-04-01 01:41:55,http://bernier.com/kelsi.braun,,223.49.106.205,"{""location"": ""TC"", ""is_mobile"": true}" 21177,8,476,2017-05-21 10:46:50,http://weinat.com/garland,,128.68.45.113,"{""location"": ""SD"", ""is_mobile"": true}" 21178,8,476,2017-01-24 20:40:53,http://gutkowskiernser.info/jeremy_kozey,,210.14.30.161,"{""location"": ""SK"", ""is_mobile"": false}" 21179,8,476,2017-01-03 13:12:19,http://baumbach.org/mike.hartmann,,177.81.185.83,"{""location"": ""CX"", ""is_mobile"": true}" 21180,8,476,2017-04-13 13:09:09,http://schaefer.io/yesenia.collier,,202.95.16.64,"{""location"": ""NP"", ""is_mobile"": false}" 21181,8,476,2017-05-22 22:19:05,http://buckridge.com/daisy,,34.138.253.129,"{""location"": ""BB"", ""is_mobile"": true}" 21182,8,476,2017-04-28 15:26:40,http://considine.co/yolanda,,224.91.205.183,"{""location"": ""CF"", ""is_mobile"": true}" 21183,8,476,2017-03-06 17:58:42,http://rogahn.net/patrick.rempel,,21.173.187.17,"{""location"": ""AW"", ""is_mobile"": false}" 21184,8,476,2017-05-08 05:01:37,http://jakubowski.io/myrtie_pacocha,,140.163.183.243,"{""location"": ""ML"", ""is_mobile"": true}" 21185,8,476,2017-05-20 16:02:55,http://dooleyoreilly.biz/elya,,110.194.68.89,"{""location"": ""GS"", ""is_mobile"": true}" 21186,8,476,2017-02-11 20:52:58,http://johnson.info/dave,,226.204.234.155,"{""location"": ""MX"", ""is_mobile"": true}" 21187,8,476,2017-01-06 00:46:55,http://okunevawaters.io/albertha,,169.49.49.126,"{""location"": ""TZ"", ""is_mobile"": false}" 21188,8,476,2017-05-14 06:49:21,http://raynor.info/pietro,,180.193.87.240,"{""location"": ""ET"", ""is_mobile"": true}" 21189,8,476,2017-06-14 03:29:49,http://ebert.info/vern,,64.182.151.45,"{""location"": ""US"", ""is_mobile"": true}" 21190,8,476,2017-01-30 16:54:44,http://lakin.com/tevin,,231.163.165.99,"{""location"": ""UG"", ""is_mobile"": false}" 21191,8,476,2017-05-29 06:33:41,http://hammesharvey.org/ignatius,,214.11.74.162,"{""location"": ""CO"", ""is_mobile"": true}" 21192,8,477,2017-06-03 21:49:12,http://breitenberg.info/alfredo,,94.173.155.90,"{""location"": ""SN"", ""is_mobile"": true}" 21193,8,477,2017-02-16 07:26:35,http://funk.name/delfina,,12.180.196.78,"{""location"": ""PS"", ""is_mobile"": true}" 21194,8,477,2017-04-02 15:09:45,http://hoegerconsidine.name/aurelio_howell,,172.40.227.12,"{""location"": ""UG"", ""is_mobile"": true}" 21195,8,477,2017-01-06 20:30:20,http://gusikowskilittel.com/rupert_feest,,103.125.27.164,"{""location"": ""GF"", ""is_mobile"": false}" 21196,8,477,2017-02-07 15:39:00,http://kaulke.io/ubaldo,,144.61.249.95,"{""location"": ""BE"", ""is_mobile"": true}" 21197,8,477,2017-04-23 23:57:13,http://jast.info/keaton_rutherford,,179.12.253.110,"{""location"": ""SD"", ""is_mobile"": true}" 21198,8,477,2017-01-12 20:49:43,http://ankundingarmstrong.io/willow,,42.216.200.136,"{""location"": ""MW"", ""is_mobile"": false}" 21199,8,477,2017-05-21 13:23:12,http://terry.co/ben,,238.97.190.84,"{""location"": ""PM"", ""is_mobile"": true}" 21200,8,477,2017-01-07 16:36:23,http://wymanmoore.info/roscoe_bechtelar,,155.207.200.227,"{""location"": ""PE"", ""is_mobile"": true}" 21201,8,477,2017-03-15 03:13:19,http://morarbraun.co/hayley,,79.82.181.211,"{""location"": ""BB"", ""is_mobile"": true}" 21202,8,477,2017-03-18 08:14:37,http://brekkeblock.name/tad,,98.233.189.77,"{""location"": ""MU"", ""is_mobile"": false}" 21203,8,477,2017-04-02 06:48:43,http://eichmann.org/elliott.gutkowski,,57.158.23.106,"{""location"": ""BZ"", ""is_mobile"": false}" 21204,8,477,2016-12-19 00:54:26,http://okunevablock.org/aurore_lind,,49.111.34.241,"{""location"": ""TW"", ""is_mobile"": false}" 21205,8,477,2017-03-31 07:21:05,http://trompreichel.biz/carmela_berge,,121.63.170.93,"{""location"": ""TV"", ""is_mobile"": true}" 21206,8,477,2017-02-24 01:55:15,http://krajcik.co/jany,,73.7.157.220,"{""location"": ""ET"", ""is_mobile"": true}" 21207,8,477,2017-01-27 07:46:47,http://ratkehintz.name/dasia,,93.234.91.159,"{""location"": ""BT"", ""is_mobile"": true}" 21208,8,477,2017-01-01 12:21:34,http://cole.io/brandy.steuber,,18.18.238.105,"{""location"": ""TR"", ""is_mobile"": false}" 21209,8,477,2017-04-28 18:08:51,http://christiansen.com/chanelle.boyle,,227.63.248.253,"{""location"": ""LT"", ""is_mobile"": true}" 21210,8,477,2017-01-27 07:26:06,http://auer.name/angus_schulist,,147.57.149.240,"{""location"": ""GF"", ""is_mobile"": true}" 21211,8,477,2017-01-13 14:58:29,http://vandervortweber.biz/christopher.kreiger,,222.189.78.121,"{""location"": ""NA"", ""is_mobile"": false}" 104,1,3,2017-02-14 19:36:52,http://thompsonwitting.info/penelope,0.7603798157,42.193.156.225,"{""location"": ""SA"", ""is_mobile"": false}" 105,1,3,2017-06-11 07:27:13,http://pollichconroy.name/lennie,0.3796789796,51.144.40.180,"{""location"": ""LI"", ""is_mobile"": false}" 106,1,3,2017-05-16 09:32:44,http://wintheiser.org/billie,0.5223689934,241.18.20.26,"{""location"": ""MD"", ""is_mobile"": true}" 107,1,3,2017-03-17 13:52:55,http://schumm.name/jayce,0.2390509085,223.37.51.205,"{""location"": ""SS"", ""is_mobile"": true}" 439,1,11,2017-05-15 17:51:19,http://ullrich.io/haven,,115.119.20.15,"{""location"": ""CN"", ""is_mobile"": true}" 108,1,3,2017-03-28 09:51:50,http://dooley.biz/albin_adams,0.9916325338,60.147.21.252,"{""location"": ""PS"", ""is_mobile"": true}" 109,1,3,2017-04-13 23:34:18,http://daughertykemmer.name/jillian,0.7219787891,31.64.221.240,"{""location"": ""NI"", ""is_mobile"": true}" 110,1,3,2017-03-10 02:04:42,http://sanford.info/wellington_hyatt,0.5442953252,83.48.10.205,"{""location"": ""KM"", ""is_mobile"": true}" 111,1,3,2017-05-01 05:29:23,http://heller.info/layla,0.1994001399,63.29.178.139,"{""location"": ""GM"", ""is_mobile"": false}" 112,1,3,2016-12-31 21:51:34,http://hills.info/bailey_smith,0.8265723189,51.5.183.215,"{""location"": ""MH"", ""is_mobile"": false}" 113,1,3,2017-02-16 22:22:21,http://jaskolski.org/kathryn_osinski,0.5635848485,36.234.7.112,"{""location"": ""TT"", ""is_mobile"": false}" 114,1,3,2017-05-25 23:27:43,http://haagmitchell.co/kailee_heller,0.0746966372,139.184.234.253,"{""location"": ""PE"", ""is_mobile"": false}" 115,1,3,2017-01-09 02:20:39,http://effertz.net/melody,0.0752629214,223.175.26.20,"{""location"": ""MK"", ""is_mobile"": true}" 116,1,3,2017-01-10 23:01:37,http://sporer.org/carlie_heidenreich,0.1780834954,95.129.13.105,"{""location"": ""EC"", ""is_mobile"": false}" 117,1,3,2017-04-16 10:09:48,http://schimmelfeeney.net/mariela_lakin,0.3433570122,203.68.235.10,"{""location"": ""ZA"", ""is_mobile"": false}" 118,1,3,2017-04-28 23:43:24,http://stehr.biz/lorine,0.5872423368,84.242.65.72,"{""location"": ""BT"", ""is_mobile"": true}" 119,1,3,2017-02-26 09:36:28,http://beahan.com/marlen,0.0646358215,82.22.103.213,"{""location"": ""PF"", ""is_mobile"": false}" 120,1,4,2017-01-09 12:47:51,http://schambergertreutel.co/jason_stracke,0.4565702586,133.157.245.228,"{""location"": ""GU"", ""is_mobile"": true}" 121,1,4,2017-04-13 23:10:04,http://kautzerdietrich.io/boris,0.6660390786,90.200.195.135,"{""location"": ""UY"", ""is_mobile"": true}" 122,1,4,2017-03-06 21:28:21,http://fay.name/dante_smitham,0.3988895140,86.251.33.174,"{""location"": ""MV"", ""is_mobile"": true}" 123,1,4,2017-01-16 07:08:05,http://moen.io/katheryn.treutel,0.8964232561,120.173.32.33,"{""location"": ""KH"", ""is_mobile"": true}" 124,1,4,2017-03-13 05:44:36,http://nolan.name/mohammad,0.2224835571,190.63.7.97,"{""location"": ""CA"", ""is_mobile"": false}" 125,1,4,2017-02-09 12:59:01,http://feest.com/kenna,0.8122121901,126.97.42.86,"{""location"": ""LA"", ""is_mobile"": false}" 126,1,4,2017-02-24 22:24:23,http://hanewisoky.net/cale,0.3658620474,129.248.247.117,"{""location"": ""DO"", ""is_mobile"": false}" 127,1,4,2017-05-22 22:55:07,http://trantow.name/kip_stanton,0.1241019774,4.45.90.244,"{""location"": ""MC"", ""is_mobile"": false}" 128,1,4,2017-05-10 19:22:46,http://schoen.co/scottie,0.0049093082,171.63.44.91,"{""location"": ""NP"", ""is_mobile"": true}" 129,1,4,2017-02-13 00:05:35,http://townenienow.biz/laila_bailey,0.5457511619,200.143.119.84,"{""location"": ""ES"", ""is_mobile"": true}" 130,1,4,2017-01-24 09:40:48,http://trantowkiehn.net/nya,0.5969854166,144.4.96.9,"{""location"": ""LT"", ""is_mobile"": false}" 131,1,4,2017-01-08 08:56:08,http://mantepredovic.io/mabel_baumbach,0.0456555790,114.93.106.199,"{""location"": ""BG"", ""is_mobile"": false}" 132,1,4,2017-03-04 08:31:16,http://hintz.info/emilia.walter,0.8934218479,224.186.192.228,"{""location"": ""LR"", ""is_mobile"": true}" 133,1,4,2017-04-09 20:00:27,http://marvinjakubowski.co/laurine.aufderhar,0.0084132576,123.108.32.214,"{""location"": ""TN"", ""is_mobile"": false}" 134,1,4,2017-05-09 04:46:30,http://lockman.co/ezekiel,0.0443884515,56.166.226.70,"{""location"": ""CV"", ""is_mobile"": false}" 135,1,4,2017-01-14 01:40:52,http://wunsch.com/xzavier,0.8979338484,19.37.189.101,"{""location"": ""VN"", ""is_mobile"": true}" 136,1,4,2017-01-30 04:58:59,http://runolfsdottir.co/lynn,0.2143528786,84.123.247.153,"{""location"": ""GR"", ""is_mobile"": false}" 137,1,4,2017-06-11 22:00:03,http://goldner.net/john,0.7062060899,243.201.19.68,"{""location"": ""KG"", ""is_mobile"": false}" 138,1,4,2017-04-02 05:56:05,http://mitchell.biz/jazmyn,0.7162014740,249.86.34.119,"{""location"": ""BW"", ""is_mobile"": true}" 139,1,4,2017-02-10 00:39:02,http://jacobistrosin.info/elise,0.4246218770,33.159.103.29,"{""location"": ""VN"", ""is_mobile"": true}" 140,1,4,2017-01-12 03:53:57,http://west.co/roslyn,0.3011598837,124.161.145.47,"{""location"": ""PW"", ""is_mobile"": true}" 141,1,5,2017-01-28 13:35:52,http://barton.biz/kenyon.upton,0.1468870249,87.92.164.74,"{""location"": ""AX"", ""is_mobile"": false}" 142,1,5,2017-06-04 15:01:03,http://rippin.com/dino,0.9960105385,183.195.153.237,"{""location"": ""MS"", ""is_mobile"": false}" 143,1,5,2017-05-05 13:50:11,http://treutel.info/eryn,0.7802999330,227.169.64.170,"{""location"": ""DO"", ""is_mobile"": true}" 144,1,5,2017-04-05 17:21:12,http://reillybailey.co/hazel_veum,0.0237870641,67.158.116.28,"{""location"": ""IE"", ""is_mobile"": false}" 145,1,5,2016-12-19 19:43:52,http://carter.info/blaise_casper,0.5853350326,84.94.163.46,"{""location"": ""BF"", ""is_mobile"": true}" 146,1,5,2017-01-10 17:26:43,http://faheycrooks.com/janis.grady,0.2354234610,9.35.183.96,"{""location"": ""TV"", ""is_mobile"": true}" 147,1,5,2017-05-11 07:44:31,http://farrell.co/frieda,0.7590308840,120.32.60.102,"{""location"": ""ME"", ""is_mobile"": false}" 148,1,5,2016-12-28 12:27:23,http://ebert.io/jolie_carroll,0.3956142731,223.28.209.136,"{""location"": ""JM"", ""is_mobile"": true}" 149,1,5,2017-04-24 19:15:35,http://braun.org/cheyanne,0.1069149872,198.127.91.180,"{""location"": ""LY"", ""is_mobile"": false}" 150,1,5,2017-04-17 00:27:22,http://murray.info/tanner_kertzmann,0.2232039493,126.83.36.114,"{""location"": ""AF"", ""is_mobile"": true}" 151,1,5,2017-06-13 22:51:31,http://weimanndamore.name/aiyana,0.1105747224,248.160.88.37,"{""location"": ""KM"", ""is_mobile"": false}" 152,1,5,2017-03-30 16:29:53,http://lynchcrooks.io/tyrese,0.4941153356,82.103.8.234,"{""location"": ""LA"", ""is_mobile"": true}" 153,1,5,2017-01-03 09:46:42,http://windler.info/king_schuppe,0.3765764061,124.177.220.104,"{""location"": ""IL"", ""is_mobile"": false}" 154,1,5,2017-05-16 16:04:30,http://monahan.org/emma,0.0055531358,133.77.132.127,"{""location"": ""AT"", ""is_mobile"": false}" 155,1,5,2017-02-05 18:07:14,http://thompson.biz/oswaldo,0.5665000194,218.247.157.189,"{""location"": ""PK"", ""is_mobile"": true}" 21212,8,477,2017-01-12 23:47:27,http://murphykirlin.com/jade_brown,,149.212.194.251,"{""location"": ""BQ"", ""is_mobile"": true}" 21213,8,477,2017-03-15 11:47:40,http://howell.com/jace.moriette,,62.92.91.94,"{""location"": ""CM"", ""is_mobile"": true}" 21214,8,477,2017-05-05 05:27:48,http://hettinger.info/dayton_rosenbaum,,43.165.227.233,"{""location"": ""CU"", ""is_mobile"": true}" 21215,8,477,2017-05-11 13:20:08,http://cronin.org/jeffrey_renner,,218.93.65.10,"{""location"": ""SI"", ""is_mobile"": true}" 21216,8,477,2017-04-16 14:06:42,http://heidenreichhauck.name/billy_homenick,,180.177.216.102,"{""location"": ""WF"", ""is_mobile"": true}" 21217,8,477,2017-04-23 14:01:37,http://barrows.info/jan,,217.184.81.237,"{""location"": ""MH"", ""is_mobile"": true}" 21218,8,477,2017-03-09 14:21:44,http://zboncaktoy.io/norval_wyman,,196.30.65.147,"{""location"": ""FJ"", ""is_mobile"": true}" 21219,8,477,2017-04-15 11:35:24,http://mcdermott.name/conor_gaylord,,235.57.94.220,"{""location"": ""CW"", ""is_mobile"": false}" 21220,8,477,2017-05-27 00:12:12,http://macgyverpredovic.com/yadira,,2.179.79.111,"{""location"": ""RE"", ""is_mobile"": false}" 21221,8,477,2017-04-14 12:15:05,http://howe.name/martina,,210.127.78.110,"{""location"": ""HM"", ""is_mobile"": false}" 21222,8,477,2017-04-04 08:42:26,http://mcdermott.co/carli_homenick,,154.141.98.129,"{""location"": ""MN"", ""is_mobile"": true}" 21223,8,477,2017-03-10 07:15:42,http://reillylindgren.org/andre,,166.100.70.136,"{""location"": ""YE"", ""is_mobile"": false}" 21224,8,477,2017-04-15 02:51:30,http://streich.name/donna,,4.61.119.41,"{""location"": ""ET"", ""is_mobile"": false}" 21225,8,477,2017-04-06 22:05:40,http://hirthemills.org/jordi,,87.245.205.182,"{""location"": ""GG"", ""is_mobile"": true}" 21226,8,477,2016-12-22 21:18:06,http://cruickshank.co/ruby.feil,,98.135.245.39,"{""location"": ""MM"", ""is_mobile"": false}" 21227,8,477,2017-04-03 19:30:57,http://conroy.com/ottis.emmerich,,110.226.170.46,"{""location"": ""ZM"", ""is_mobile"": false}" 21228,8,477,2017-02-13 21:33:46,http://runte.io/reva,,84.140.55.60,"{""location"": ""TV"", ""is_mobile"": false}" 21229,8,477,2017-01-06 10:52:02,http://robel.biz/titus,,46.217.50.64,"{""location"": ""MF"", ""is_mobile"": false}" 21230,8,477,2017-01-23 14:19:09,http://metzokuneva.info/eliezer,,102.154.235.9,"{""location"": ""JE"", ""is_mobile"": false}" 21231,8,477,2017-03-29 10:54:05,http://dietrichlebsack.name/bernice_okeefe,,69.88.217.123,"{""location"": ""KR"", ""is_mobile"": true}" 21232,8,477,2017-05-21 09:54:09,http://mueller.biz/selmer.hettinger,,27.214.246.87,"{""location"": ""IN"", ""is_mobile"": true}" 21233,8,477,2017-05-21 19:11:57,http://deckow.org/aimee_frami,,211.181.104.106,"{""location"": ""BA"", ""is_mobile"": true}" 21234,8,477,2017-06-12 08:44:37,http://bernier.io/rogers_shanahan,,154.122.150.4,"{""location"": ""NU"", ""is_mobile"": false}" 21235,8,477,2017-05-22 23:56:49,http://mcclure.co/monique,,16.218.59.75,"{""location"": ""GE"", ""is_mobile"": false}" 21236,8,477,2017-01-19 06:18:03,http://reichel.co/juana.connelly,,169.143.108.81,"{""location"": ""CW"", ""is_mobile"": false}" 21237,8,477,2017-01-28 01:59:49,http://kovacek.net/jack.borer,,221.160.153.169,"{""location"": ""KW"", ""is_mobile"": true}" 21238,8,477,2016-12-23 22:46:17,http://farrellwitting.name/joel,,143.66.12.12,"{""location"": ""TD"", ""is_mobile"": true}" 21239,8,477,2017-03-21 07:03:47,http://framireichel.com/alex.heel,,206.160.108.143,"{""location"": ""PH"", ""is_mobile"": false}" 21240,8,477,2017-04-28 04:16:58,http://stark.io/augusta,,240.195.92.102,"{""location"": ""MH"", ""is_mobile"": false}" 21241,8,477,2017-05-31 05:06:58,http://hahnsimonis.name/esperanza,,107.116.51.177,"{""location"": ""UZ"", ""is_mobile"": true}" 21242,8,477,2016-12-28 13:45:44,http://maggio.co/alta,,42.209.140.171,"{""location"": ""AS"", ""is_mobile"": true}" 21243,8,477,2017-01-07 00:35:38,http://conroy.co/otha_hammes,,56.87.134.220,"{""location"": ""SJ"", ""is_mobile"": true}" 21244,8,477,2017-01-10 01:26:06,http://sipes.co/soledad,,215.236.34.149,"{""location"": ""OM"", ""is_mobile"": false}" 21245,8,477,2017-03-08 21:36:48,http://eichmann.info/marguerite,,43.134.197.239,"{""location"": ""BW"", ""is_mobile"": true}" 21246,8,477,2017-02-24 01:08:40,http://mcdermott.name/terrell,,228.172.130.129,"{""location"": ""BS"", ""is_mobile"": true}" 21247,8,477,2017-05-10 02:15:44,http://torpkautzer.biz/summer,,49.219.212.27,"{""location"": ""TR"", ""is_mobile"": true}" 21248,8,477,2017-05-24 00:45:41,http://bayer.net/keenan_connelly,,217.77.157.145,"{""location"": ""CD"", ""is_mobile"": false}" 21249,8,477,2017-05-30 17:09:33,http://funk.org/jamie,,25.139.98.193,"{""location"": ""GU"", ""is_mobile"": true}" 21250,8,477,2016-12-23 17:43:55,http://rohanpacocha.com/miracle,,87.39.44.253,"{""location"": ""PN"", ""is_mobile"": false}" 21251,8,477,2017-01-27 10:13:15,http://wiza.biz/neoma.hilll,,113.114.218.120,"{""location"": ""SL"", ""is_mobile"": true}" 21252,8,477,2017-05-16 09:32:06,http://gibson.co/tom,,157.242.109.30,"{""location"": ""NR"", ""is_mobile"": true}" 21253,8,477,2017-03-23 04:44:42,http://glover.co/clinton_brekke,,136.65.220.182,"{""location"": ""SC"", ""is_mobile"": true}" 21254,8,477,2016-12-15 08:38:08,http://white.io/candido,,150.27.228.172,"{""location"": ""EE"", ""is_mobile"": false}" 21255,8,477,2017-06-02 11:59:08,http://cain.name/kory.oconnell,,103.211.8.205,"{""location"": ""GY"", ""is_mobile"": false}" 21256,8,477,2017-03-12 15:19:19,http://watershyatt.info/kieran_heel,,196.14.194.73,"{""location"": ""YE"", ""is_mobile"": true}" 21257,8,477,2017-01-29 23:24:35,http://thompson.org/vernie.brakus,,194.140.199.184,"{""location"": ""KE"", ""is_mobile"": false}" 21258,8,477,2017-03-08 11:58:21,http://boehmhintz.co/eugenia.durgan,,22.192.23.84,"{""location"": ""UY"", ""is_mobile"": true}" 21259,8,477,2017-03-26 02:30:29,http://aufderhar.name/tanya.bednar,,89.111.191.11,"{""location"": ""RO"", ""is_mobile"": false}" 21260,8,477,2017-03-01 04:45:11,http://waltersanford.net/gina_conroy,,142.71.188.161,"{""location"": ""MX"", ""is_mobile"": true}" 21261,8,477,2017-01-01 23:34:34,http://auer.io/thea.roob,,48.130.211.130,"{""location"": ""SV"", ""is_mobile"": true}" 21262,8,478,2017-02-02 05:57:48,http://kuhlman.com/daisha,,43.204.36.154,"{""location"": ""KE"", ""is_mobile"": false}" 21263,8,478,2017-03-12 12:00:50,http://haley.com/kareem,,158.65.71.134,"{""location"": ""ZW"", ""is_mobile"": false}" 21264,8,478,2016-12-19 07:36:32,http://kohlermayer.io/demario,,121.33.224.242,"{""location"": ""NR"", ""is_mobile"": true}" 21265,8,478,2017-05-01 12:45:10,http://emmerichquigley.org/landen_kuhic,,127.9.189.61,"{""location"": ""AM"", ""is_mobile"": true}" 21266,8,478,2017-05-12 00:22:18,http://mohrbraun.name/odell_abshire,,245.217.31.155,"{""location"": ""PG"", ""is_mobile"": false}" 156,1,5,2017-02-28 14:01:01,http://farrell.io/adeline,0.0973006459,140.92.208.155,"{""location"": ""CH"", ""is_mobile"": false}" 157,1,5,2017-01-29 06:03:19,http://ortiz.net/emelia.gaylord,0.7117211981,245.10.13.153,"{""location"": ""BD"", ""is_mobile"": false}" 158,1,5,2017-05-24 07:32:09,http://roob.io/sigmund_hyatt,0.7668921151,214.18.177.74,"{""location"": ""CM"", ""is_mobile"": false}" 159,1,5,2017-05-19 10:37:13,http://volkmanswaniawski.name/sophia,0.9076940109,46.164.41.83,"{""location"": ""MS"", ""is_mobile"": false}" 160,1,5,2016-12-14 20:30:03,http://roob.co/katarina,0.1184546810,182.225.238.154,"{""location"": ""GE"", ""is_mobile"": true}" 161,1,5,2017-01-21 21:05:45,http://dibbert.io/taurean.kuhn,0.1564920005,86.81.68.98,"{""location"": ""EE"", ""is_mobile"": true}" 162,1,5,2017-04-29 01:42:08,http://vonrueden.com/tommie.rogahn,0.7513364421,37.222.252.119,"{""location"": ""BS"", ""is_mobile"": true}" 163,1,5,2017-03-10 12:35:06,http://stark.info/afton,0.3932017194,222.252.41.76,"{""location"": ""SO"", ""is_mobile"": true}" 164,1,5,2016-12-16 14:48:39,http://marvin.com/aniya.nikolaus,0.2803402145,107.94.231.160,"{""location"": ""JM"", ""is_mobile"": false}" 165,1,5,2017-04-03 10:13:39,http://danielschuster.biz/haleigh,0.0419757803,58.113.118.10,"{""location"": ""AG"", ""is_mobile"": true}" 166,1,5,2017-02-12 19:38:36,http://schmidt.net/alfonzo_upton,0.3713954095,21.199.217.190,"{""location"": ""TC"", ""is_mobile"": false}" 167,1,5,2017-04-03 18:18:14,http://toy.name/silas_cronin,0.6393181017,71.191.200.33,"{""location"": ""NG"", ""is_mobile"": false}" 168,1,5,2017-03-26 07:58:41,http://boyer.org/octavia_ruel,0.4022229836,50.202.169.211,"{""location"": ""ZW"", ""is_mobile"": false}" 169,1,5,2017-02-04 07:11:19,http://klingmedhurst.org/roie,0.8675705676,195.53.8.185,"{""location"": ""KG"", ""is_mobile"": true}" 170,1,5,2017-01-31 12:30:57,http://conroykuphal.biz/mona_kub,0.6259723123,203.105.14.87,"{""location"": ""AD"", ""is_mobile"": false}" 171,1,5,2017-03-07 01:13:25,http://boehm.name/caitlyn.fahey,0.7856185636,211.168.83.69,"{""location"": ""BD"", ""is_mobile"": true}" 172,1,5,2016-12-21 01:31:31,http://torp.io/dawn,0.5999685263,207.201.119.79,"{""location"": ""ES"", ""is_mobile"": true}" 173,1,5,2017-01-19 01:41:44,http://kuphal.io/abdullah_smith,0.7368600212,108.36.194.181,"{""location"": ""LT"", ""is_mobile"": true}" 174,1,5,2017-06-05 20:32:23,http://lynch.co/helga.bayer,0.5199178973,139.164.108.79,"{""location"": ""RE"", ""is_mobile"": true}" 175,1,5,2017-03-05 07:26:27,http://moen.org/khalil,0.9734379114,56.50.239.44,"{""location"": ""EE"", ""is_mobile"": false}" 176,1,5,2017-05-13 07:14:19,http://abbott.io/adelbert,0.5609720819,221.91.92.84,"{""location"": ""SG"", ""is_mobile"": false}" 177,1,5,2017-02-04 20:36:12,http://schultzpfeffer.biz/allen,0.9632023669,233.81.180.130,"{""location"": ""AL"", ""is_mobile"": false}" 178,1,5,2016-12-17 22:13:20,http://balistreri.info/camryn,0.3752039402,230.175.93.46,"{""location"": ""WF"", ""is_mobile"": true}" 179,1,5,2017-05-10 23:13:02,http://gleason.name/nicholaus,0.2571087249,153.251.99.218,"{""location"": ""DK"", ""is_mobile"": false}" 180,1,5,2017-05-17 01:43:25,http://leannonrunolfsdottir.name/melany,0.3954058751,247.232.194.24,"{""location"": ""SE"", ""is_mobile"": false}" 181,1,5,2017-01-21 17:56:42,http://hodkiewiczlueilwitz.io/destany,0.0586740145,9.239.207.82,"{""location"": ""BG"", ""is_mobile"": true}" 182,1,5,2017-03-15 15:13:04,http://gutkowskihansen.info/preston,0.1595139939,178.191.171.208,"{""location"": ""CN"", ""is_mobile"": true}" 183,1,5,2017-01-01 15:50:44,http://kemmerschowalter.io/camilla_hirthe,0.1647224080,229.202.70.147,"{""location"": ""GF"", ""is_mobile"": true}" 184,1,5,2017-05-28 04:12:32,http://grimes.name/daniela,0.5542240788,215.89.232.207,"{""location"": ""TT"", ""is_mobile"": false}" 185,1,5,2017-05-25 16:51:06,http://dooley.com/ted_harvey,0.1095130625,199.147.84.107,"{""location"": ""NO"", ""is_mobile"": false}" 186,1,5,2017-03-16 21:04:02,http://beier.com/austen.mosciski,0.9352864344,80.205.229.216,"{""location"": ""SH"", ""is_mobile"": false}" 187,1,5,2017-02-25 11:04:02,http://roobconsidine.biz/mark,0.1912498207,5.16.99.162,"{""location"": ""HU"", ""is_mobile"": true}" 188,1,5,2017-03-24 17:55:52,http://balistrerikozey.biz/elise,0.8558493466,90.82.193.37,"{""location"": ""ZM"", ""is_mobile"": false}" 189,1,5,2017-01-27 13:48:59,http://bins.info/melisa_lang,0.1717571191,172.15.39.109,"{""location"": ""KZ"", ""is_mobile"": false}" 190,1,5,2017-04-04 13:50:23,http://kuphal.info/hunter,0.9204948910,243.121.84.137,"{""location"": ""ET"", ""is_mobile"": true}" 191,1,5,2017-01-23 14:39:15,http://wilkinson.net/cathryn.fisher,0.6264778865,8.8.70.158,"{""location"": ""MV"", ""is_mobile"": false}" 192,1,5,2017-02-16 09:08:06,http://walshweber.co/eldora.gutkowski,0.6352322788,224.208.123.62,"{""location"": ""MX"", ""is_mobile"": false}" 193,1,5,2017-01-18 10:12:20,http://hellerherman.name/rubye,0.8023360292,168.81.107.63,"{""location"": ""VN"", ""is_mobile"": true}" 194,1,5,2017-01-31 21:24:32,http://schimmel.com/kade,0.0309756614,163.203.138.182,"{""location"": ""NR"", ""is_mobile"": false}" 195,1,5,2017-02-28 12:50:40,http://hoppe.biz/nicholas,0.3865859501,127.116.151.208,"{""location"": ""BF"", ""is_mobile"": false}" 196,1,6,2017-04-17 13:50:42,http://gutmann.net/cydney.mueller,0.5155177076,253.20.183.162,"{""location"": ""HT"", ""is_mobile"": false}" 197,1,6,2017-04-12 20:39:22,http://willstehr.net/kyra,0.0286517080,32.185.126.197,"{""location"": ""GT"", ""is_mobile"": true}" 198,1,6,2017-06-05 13:58:45,http://franecki.name/daisy,0.4765253859,189.87.70.59,"{""location"": ""GD"", ""is_mobile"": true}" 199,1,6,2017-01-30 02:27:13,http://boehm.io/deonte_bruen,0.6783055797,181.5.178.191,"{""location"": ""GB"", ""is_mobile"": true}" 200,1,6,2017-01-26 13:33:08,http://raynor.net/candida_prohaska,0.5607241009,133.88.178.149,"{""location"": ""MD"", ""is_mobile"": true}" 201,1,6,2016-12-15 05:07:46,http://christiansenhermann.co/aleen_lesch,0.9177731078,32.88.25.183,"{""location"": ""NG"", ""is_mobile"": false}" 202,1,6,2016-12-20 21:03:30,http://heidenreichgreen.co/dock,0.1383376981,244.137.113.126,"{""location"": ""TM"", ""is_mobile"": true}" 203,1,6,2017-04-24 19:49:55,http://kingcasper.info/scot.johnson,0.1853921377,213.187.85.93,"{""location"": ""CX"", ""is_mobile"": false}" 204,1,6,2016-12-16 09:44:43,http://jenkins.biz/baby.bradtke,0.7916128486,105.135.38.101,"{""location"": ""MR"", ""is_mobile"": false}" 205,1,6,2017-06-04 21:28:16,http://ankundingsanford.name/zetta,0.9896897814,88.87.178.197,"{""location"": ""FO"", ""is_mobile"": true}" 206,1,6,2017-01-27 18:30:14,http://dietrich.name/letha,0.0860835674,218.79.225.144,"{""location"": ""FJ"", ""is_mobile"": true}" 207,1,6,2017-04-13 01:02:00,http://jaskolskismith.io/deon,0.5650486488,199.215.37.113,"{""location"": ""GF"", ""is_mobile"": true}" 208,1,6,2017-06-11 12:09:17,http://wisozkking.com/wava_schimmel,0.6031023198,151.66.200.253,"{""location"": ""KW"", ""is_mobile"": true}" 21267,8,478,2016-12-28 16:51:44,http://gutmann.co/bernard,,13.6.182.63,"{""location"": ""YE"", ""is_mobile"": false}" 21268,8,478,2017-05-15 17:06:35,http://predovic.com/nikita,,163.14.145.192,"{""location"": ""IE"", ""is_mobile"": false}" 21269,8,478,2016-12-28 12:09:43,http://walkerharris.io/bradly_langosh,,78.21.194.211,"{""location"": ""FI"", ""is_mobile"": false}" 21270,8,478,2017-03-16 09:48:49,http://kozey.org/jackson,,25.84.226.215,"{""location"": ""VE"", ""is_mobile"": false}" 21271,8,478,2017-05-07 04:21:25,http://abbott.co/kraig,,209.179.145.8,"{""location"": ""TJ"", ""is_mobile"": false}" 21272,8,478,2016-12-17 21:05:13,http://metz.co/ronaldo.fisher,,150.211.179.244,"{""location"": ""SD"", ""is_mobile"": true}" 21273,8,478,2017-03-04 17:07:00,http://ortiz.org/carolyn.will,,78.74.147.109,"{""location"": ""SN"", ""is_mobile"": true}" 21274,8,478,2017-05-06 16:25:04,http://jaskolski.info/wendell.glover,,38.98.92.241,"{""location"": ""MF"", ""is_mobile"": true}" 21275,8,478,2017-03-13 03:54:12,http://kulas.org/orie.kautzer,,150.199.220.23,"{""location"": ""BN"", ""is_mobile"": false}" 21276,8,478,2017-05-15 19:09:30,http://ohara.info/estrella,,96.237.242.104,"{""location"": ""SH"", ""is_mobile"": false}" 21277,8,478,2017-05-20 22:13:23,http://senger.info/bartholome.gerlach,,46.81.135.97,"{""location"": ""SI"", ""is_mobile"": true}" 21278,8,478,2017-06-13 15:01:29,http://walter.com/henry.kuhlman,,191.232.195.220,"{""location"": ""MG"", ""is_mobile"": false}" 21279,8,478,2017-03-24 08:14:52,http://durgannikolaus.biz/sigrid.hauck,,249.206.51.98,"{""location"": ""TC"", ""is_mobile"": true}" 21280,8,478,2017-04-30 08:12:55,http://wizaberge.com/robb,,90.226.20.23,"{""location"": ""ZW"", ""is_mobile"": true}" 21281,8,478,2017-04-14 17:05:56,http://reinger.io/floy,,36.94.122.69,"{""location"": ""PW"", ""is_mobile"": true}" 21282,8,478,2017-05-23 17:02:37,http://beierebert.org/bill,,56.177.71.80,"{""location"": ""WF"", ""is_mobile"": true}" 21283,8,478,2017-03-12 00:42:45,http://wilderman.org/stefanie,,125.194.250.198,"{""location"": ""CV"", ""is_mobile"": false}" 21284,8,478,2017-01-02 18:31:08,http://corkery.io/elisabeth.veum,,102.80.96.27,"{""location"": ""NR"", ""is_mobile"": true}" 21285,8,478,2017-05-12 11:54:03,http://lebsack.io/humberto,,94.215.183.139,"{""location"": ""CH"", ""is_mobile"": false}" 21286,8,478,2017-06-10 03:34:21,http://koepp.com/jo,,25.66.43.252,"{""location"": ""UY"", ""is_mobile"": false}" 21287,8,478,2017-06-12 05:29:21,http://hayes.info/einar,,74.154.211.121,"{""location"": ""GW"", ""is_mobile"": false}" 21288,8,478,2017-03-09 22:21:07,http://willmchulist.info/ro,,253.90.16.69,"{""location"": ""NR"", ""is_mobile"": true}" 21289,8,478,2017-05-03 11:10:51,http://hammesward.net/martine_cronin,,167.94.24.72,"{""location"": ""JP"", ""is_mobile"": true}" 21290,8,478,2017-06-06 20:45:31,http://smith.name/kailey,,155.73.27.242,"{""location"": ""BD"", ""is_mobile"": true}" 21291,8,478,2017-04-15 14:43:20,http://jakubowskicorwin.org/pinkie,,22.204.95.37,"{""location"": ""SV"", ""is_mobile"": false}" 21292,8,478,2017-01-29 15:51:32,http://sawayn.com/xander,,86.46.129.110,"{""location"": ""IQ"", ""is_mobile"": false}" 21293,8,478,2017-03-18 00:57:43,http://wardlind.biz/jamie_zieme,,197.185.179.127,"{""location"": ""BO"", ""is_mobile"": false}" 21294,8,479,2017-02-20 16:47:28,http://botsford.info/adrian.schuppe,,6.129.79.27,"{""location"": ""CD"", ""is_mobile"": true}" 21295,8,479,2017-02-13 00:49:55,http://aufderharlehner.biz/julia_ryan,,247.211.152.196,"{""location"": ""ST"", ""is_mobile"": true}" 21296,8,479,2017-05-09 18:58:25,http://dach.org/ardella,,117.219.55.16,"{""location"": ""NA"", ""is_mobile"": false}" 21297,8,479,2017-04-11 07:46:37,http://von.info/ella,,46.83.48.83,"{""location"": ""VA"", ""is_mobile"": false}" 21298,8,479,2017-04-22 20:48:56,http://reingerrosenbaum.net/janet_pfeffer,,48.249.229.174,"{""location"": ""JO"", ""is_mobile"": true}" 21299,8,479,2017-04-09 11:01:48,http://thiel.biz/alexandro,,75.35.50.27,"{""location"": ""AZ"", ""is_mobile"": false}" 21300,8,479,2017-03-17 01:51:12,http://dare.io/nikki_altenwerth,,48.232.12.135,"{""location"": ""BW"", ""is_mobile"": false}" 21301,8,479,2017-06-11 12:17:01,http://monahan.net/rogelio_feil,,191.101.111.117,"{""location"": ""BR"", ""is_mobile"": true}" 21302,8,479,2017-02-05 17:02:33,http://abernathy.co/lisette.purdy,,128.24.59.150,"{""location"": ""MD"", ""is_mobile"": false}" 21303,8,479,2017-04-23 08:28:39,http://collierhayes.io/sarina,,232.233.145.104,"{""location"": ""SV"", ""is_mobile"": true}" 21304,8,479,2017-06-14 00:20:39,http://beatty.net/kayley.dibbert,,124.53.229.143,"{""location"": ""MR"", ""is_mobile"": false}" 21305,8,479,2017-05-25 19:22:47,http://millersipes.com/shana,,247.131.221.149,"{""location"": ""UM"", ""is_mobile"": false}" 21306,8,479,2017-05-05 09:12:07,http://littelframi.co/linnie_batz,,231.178.99.86,"{""location"": ""NA"", ""is_mobile"": true}" 21307,8,479,2017-03-06 09:50:21,http://ankunding.info/marilie_littel,,227.77.160.223,"{""location"": ""PS"", ""is_mobile"": false}" 21308,8,479,2016-12-18 23:14:23,http://krisgulgowski.co/peter,,120.243.123.208,"{""location"": ""SB"", ""is_mobile"": false}" 21309,8,479,2017-03-17 18:56:10,http://spencertorp.io/bonita_donnelly,,239.121.254.35,"{""location"": ""MS"", ""is_mobile"": true}" 21310,8,479,2017-03-30 01:27:59,http://kertzmann.io/jaleel.abshire,,240.124.109.138,"{""location"": ""TM"", ""is_mobile"": false}" 21311,8,479,2017-05-29 22:38:23,http://carroll.name/retta.walsh,,80.126.62.203,"{""location"": ""GP"", ""is_mobile"": false}" 21312,8,479,2017-01-19 21:58:47,http://dubuque.info/darryl,,126.218.143.193,"{""location"": ""CG"", ""is_mobile"": true}" 21313,8,479,2017-04-02 20:22:26,http://swaniawskihilpert.info/arnoldo_boyer,,109.235.104.62,"{""location"": ""UY"", ""is_mobile"": false}" 21314,8,479,2017-01-22 01:35:03,http://altenwerth.info/donnell,,13.160.72.133,"{""location"": ""HR"", ""is_mobile"": true}" 21315,8,479,2017-04-05 22:53:30,http://kertzmannschroeder.net/luis.cummerata,,117.121.220.44,"{""location"": ""TJ"", ""is_mobile"": true}" 21316,8,479,2016-12-23 19:56:33,http://lindgren.name/baron,,151.122.79.95,"{""location"": ""LR"", ""is_mobile"": false}" 21317,8,479,2017-03-10 22:21:30,http://johns.org/edd,,3.12.27.210,"{""location"": ""SK"", ""is_mobile"": false}" 21318,8,479,2017-03-05 23:44:04,http://jacobsongerhold.io/allie,,191.252.74.88,"{""location"": ""AQ"", ""is_mobile"": true}" 21319,8,479,2017-01-13 16:51:52,http://bins.biz/alda,,230.40.61.172,"{""location"": ""SN"", ""is_mobile"": false}" 21320,8,479,2017-01-01 20:05:39,http://rowe.com/theodore,,187.232.246.117,"{""location"": ""RU"", ""is_mobile"": true}" 21321,8,479,2017-03-09 21:37:45,http://douglas.com/julio_gaylord,,35.69.134.41,"{""location"": ""GQ"", ""is_mobile"": false}" 21322,8,479,2017-02-25 03:25:51,http://ricemacejkovic.org/bertha,,208.149.61.29,"{""location"": ""SN"", ""is_mobile"": false}" 209,1,6,2017-03-18 17:01:21,http://raynordouglas.name/reagan_tromp,0.7501246284,19.68.69.99,"{""location"": ""TG"", ""is_mobile"": false}" 210,1,6,2017-04-16 16:31:07,http://schmelerraynor.com/monserrat_gibson,0.5465721721,228.111.37.47,"{""location"": ""VN"", ""is_mobile"": false}" 211,1,6,2017-03-26 20:04:55,http://wolffbuckridge.co/shanelle,0.5066755219,186.177.147.162,"{""location"": ""KG"", ""is_mobile"": true}" 212,1,6,2017-03-26 09:42:03,http://colekozey.co/camila,0.9769205887,156.44.156.135,"{""location"": ""MQ"", ""is_mobile"": true}" 213,1,6,2017-03-30 00:21:52,http://boehmconn.name/angelita_pagac,0.8910228658,153.142.61.93,"{""location"": ""LA"", ""is_mobile"": true}" 214,1,6,2017-04-20 15:38:23,http://graham.io/jefferey,0.3867140401,27.157.201.85,"{""location"": ""RO"", ""is_mobile"": true}" 1048,1,24,2017-03-26 10:51:38,http://terry.com/lucio,,147.89.43.44,"{""location"": ""NP"", ""is_mobile"": true}" 215,1,6,2017-04-23 09:14:56,http://labadie.com/karl.rodriguez,0.0944978064,50.41.53.48,"{""location"": ""MA"", ""is_mobile"": false}" 216,1,6,2017-04-18 17:15:49,http://stantonraynor.biz/verdie_harris,0.7748436150,100.239.196.216,"{""location"": ""MS"", ""is_mobile"": true}" 217,1,6,2017-03-31 19:17:51,http://okon.name/myriam,0.6210429025,85.212.209.69,"{""location"": ""TW"", ""is_mobile"": true}" 218,1,6,2017-04-09 21:57:11,http://schaefer.biz/alexis.metz,0.1912995859,242.149.190.55,"{""location"": ""MM"", ""is_mobile"": true}" 219,1,6,2017-05-30 16:31:08,http://strosin.org/athena,0.5384474205,214.116.22.251,"{""location"": ""RU"", ""is_mobile"": true}" 220,1,6,2017-03-16 19:27:05,http://bernhardstokes.net/heath,0.7268181081,101.119.111.159,"{""location"": ""VU"", ""is_mobile"": false}" 221,1,6,2017-06-05 07:43:11,http://jacobi.io/robert_steuber,0.4607575021,51.56.141.203,"{""location"": ""WF"", ""is_mobile"": true}" 222,1,6,2017-01-29 12:10:25,http://mooreflatley.net/anastasia.feil,0.9680427466,145.249.163.64,"{""location"": ""KI"", ""is_mobile"": false}" 223,1,6,2017-04-23 21:40:42,http://brakuseffertz.net/lucinda,0.0078797077,97.77.185.183,"{""location"": ""MK"", ""is_mobile"": true}" 224,1,6,2017-01-20 09:36:48,http://windler.org/audra,0.1746856190,135.225.205.59,"{""location"": ""DM"", ""is_mobile"": true}" 225,1,6,2017-06-05 02:21:24,http://emmerichcartwright.biz/calista.mckenzie,0.8087271972,230.149.249.229,"{""location"": ""AT"", ""is_mobile"": true}" 226,1,6,2017-02-27 23:11:06,http://raynor.net/zella,0.9490212741,38.197.253.115,"{""location"": ""BW"", ""is_mobile"": true}" 227,1,6,2017-01-04 14:10:05,http://goyette.biz/major.grant,0.5452318489,18.10.222.243,"{""location"": ""CD"", ""is_mobile"": false}" 228,1,6,2017-01-24 02:04:32,http://ryan.info/vernice,0.9190438041,182.243.172.193,"{""location"": ""LV"", ""is_mobile"": true}" 229,1,6,2017-03-16 12:24:40,http://brekke.com/loma_jerde,0.7985769383,121.26.155.40,"{""location"": ""SI"", ""is_mobile"": true}" 230,1,6,2017-05-26 07:50:22,http://nader.com/mohammed,0.3194378282,121.145.19.247,"{""location"": ""NA"", ""is_mobile"": true}" 231,1,6,2017-05-19 18:32:41,http://botsford.biz/jaylon.stiedemann,0.6751005354,182.150.74.70,"{""location"": ""SJ"", ""is_mobile"": true}" 232,1,6,2017-02-12 13:09:21,http://stracke.net/lolita,0.7501822518,188.219.180.248,"{""location"": ""SX"", ""is_mobile"": false}" 233,1,6,2017-05-30 08:51:52,http://leuschke.co/felix.rowe,0.4612989301,68.29.181.197,"{""location"": ""BQ"", ""is_mobile"": true}" 234,1,6,2017-02-04 07:11:37,http://hilll.info/jazmyn,0.8515188437,195.55.59.163,"{""location"": ""GQ"", ""is_mobile"": true}" 235,1,6,2017-03-04 00:32:13,http://effertzhilpert.org/adriel,0.6796913962,39.88.236.28,"{""location"": ""CV"", ""is_mobile"": false}" 236,1,6,2017-03-19 20:00:35,http://adams.info/dawn,0.2202506216,250.180.63.182,"{""location"": ""EH"", ""is_mobile"": false}" 237,1,6,2017-02-07 13:56:41,http://hintz.io/abe,0.9430528767,81.172.109.71,"{""location"": ""AX"", ""is_mobile"": true}" 238,1,6,2017-03-11 14:31:25,http://bins.info/elody_hackett,0.7404865153,220.221.95.245,"{""location"": ""GF"", ""is_mobile"": true}" 239,1,6,2017-01-08 03:53:16,http://reinger.org/catharine.hahn,0.5051925980,217.170.134.37,"{""location"": ""LY"", ""is_mobile"": false}" 240,1,6,2017-01-10 22:10:27,http://williamson.info/macie_emmerich,0.1841981852,62.128.62.121,"{""location"": ""LY"", ""is_mobile"": false}" 241,1,6,2017-03-08 00:27:14,http://waters.io/suzanne,0.3225056989,64.179.45.203,"{""location"": ""CO"", ""is_mobile"": true}" 242,1,6,2017-03-04 05:52:23,http://yost.com/alison_dicki,0.2189924544,241.47.88.240,"{""location"": ""VN"", ""is_mobile"": true}" 243,1,6,2017-06-09 23:40:52,http://schulistbosco.org/amanda_kozey,0.7364554407,200.205.105.34,"{""location"": ""LK"", ""is_mobile"": true}" 244,1,6,2017-04-29 15:13:02,http://mueller.org/earnestine,0.8886049839,137.51.214.39,"{""location"": ""CV"", ""is_mobile"": false}" 245,1,6,2017-05-30 09:17:10,http://hansen.com/ryan,0.6890976754,126.112.247.243,"{""location"": ""BB"", ""is_mobile"": false}" 246,1,6,2017-03-21 09:10:46,http://bergstrom.com/shana_kautzer,0.1192768589,160.154.45.137,"{""location"": ""BD"", ""is_mobile"": false}" 247,1,6,2017-02-28 04:14:54,http://hudsonquigley.org/adrain,0.2851109589,63.200.15.158,"{""location"": ""BQ"", ""is_mobile"": true}" 248,1,6,2017-02-08 13:48:55,http://von.net/erica_gulgowski,0.5384702868,77.235.89.203,"{""location"": ""BN"", ""is_mobile"": true}" 249,1,6,2017-02-02 11:22:39,http://jones.info/kadin_hyatt,0.3827914721,234.218.36.154,"{""location"": ""FO"", ""is_mobile"": true}" 250,1,6,2017-05-10 15:48:16,http://baileybaumbach.co/jodie.brakus,0.3793901749,43.237.135.95,"{""location"": ""AQ"", ""is_mobile"": false}" 251,1,6,2017-05-19 03:08:38,http://jacobsonmoore.net/gloria_heathcote,0.9626236445,219.63.22.60,"{""location"": ""BO"", ""is_mobile"": false}" 252,1,6,2017-02-16 15:24:40,http://altenwerthkonopelski.name/willis,0.7496539756,23.39.176.58,"{""location"": ""YE"", ""is_mobile"": true}" 253,1,6,2016-12-16 20:03:23,http://jakubowskibeier.io/emmanuel_wyman,0.6223168743,239.151.223.207,"{""location"": ""HR"", ""is_mobile"": false}" 254,1,6,2017-02-28 23:08:42,http://littel.biz/trisha.koepp,0.4587851717,155.73.42.24,"{""location"": ""NP"", ""is_mobile"": true}" 255,1,6,2017-01-06 18:52:48,http://hermiston.co/joanie,0.8136161872,131.122.81.235,"{""location"": ""TG"", ""is_mobile"": true}" 256,1,6,2017-01-29 05:23:43,http://hackett.name/uriel,0.9420652736,51.118.217.30,"{""location"": ""HT"", ""is_mobile"": false}" 257,1,6,2017-01-18 01:27:01,http://skilesromaguera.name/maximo_murazik,0.0886841203,24.92.149.123,"{""location"": ""CA"", ""is_mobile"": false}" 258,1,6,2017-03-31 20:18:43,http://ullrichkub.net/mason,0.4903908107,134.203.215.223,"{""location"": ""GL"", ""is_mobile"": false}" 259,1,6,2016-12-27 06:48:18,http://stokes.io/verla,0.3702651128,149.160.217.153,"{""location"": ""PY"", ""is_mobile"": true}" 21323,8,479,2017-05-24 12:48:33,http://kochmertz.com/rachel,,3.220.12.202,"{""location"": ""MG"", ""is_mobile"": false}" 21324,8,479,2016-12-24 11:53:21,http://zboncak.name/myrna,,137.78.18.181,"{""location"": ""LI"", ""is_mobile"": true}" 21325,8,479,2017-02-01 18:21:39,http://predovicherman.biz/karson,,28.162.133.101,"{""location"": ""MH"", ""is_mobile"": true}" 21326,8,479,2017-01-09 02:45:43,http://moriette.org/westley_gleichner,,177.68.249.139,"{""location"": ""GU"", ""is_mobile"": true}" 21327,8,479,2017-05-25 17:12:49,http://marks.name/brice,,59.223.56.201,"{""location"": ""AE"", ""is_mobile"": false}" 21328,8,479,2017-02-06 14:44:35,http://heller.name/josue,,246.74.124.247,"{""location"": ""BV"", ""is_mobile"": true}" 21329,8,479,2017-03-31 08:50:35,http://cruickshank.co/louisa_deckow,,62.6.240.190,"{""location"": ""MG"", ""is_mobile"": true}" 21330,8,479,2017-02-03 20:30:09,http://jastblock.co/mallie,,198.191.141.146,"{""location"": ""TC"", ""is_mobile"": false}" 21331,8,479,2017-05-10 11:36:44,http://moendickens.info/ted,,109.111.67.238,"{""location"": ""TZ"", ""is_mobile"": false}" 21332,8,479,2017-01-01 14:36:04,http://lindgren.io/raphael.dach,,163.74.100.64,"{""location"": ""VU"", ""is_mobile"": false}" 21333,8,479,2017-01-28 23:45:44,http://johnston.info/earl,,236.138.22.149,"{""location"": ""UA"", ""is_mobile"": true}" 21334,8,479,2017-01-06 22:53:30,http://toy.net/madelynn,,242.218.26.164,"{""location"": ""IE"", ""is_mobile"": true}" 21335,8,479,2017-03-10 23:44:30,http://champlin.org/ken,,210.157.215.166,"{""location"": ""LY"", ""is_mobile"": true}" 21336,8,479,2017-05-17 02:20:53,http://larson.biz/una,,44.136.94.107,"{""location"": ""GD"", ""is_mobile"": true}" 21337,8,479,2017-04-07 16:29:39,http://ullrich.org/isabel_padberg,,200.145.112.62,"{""location"": ""HT"", ""is_mobile"": true}" 21338,8,479,2017-05-08 15:56:23,http://mullerharber.net/kim,,142.85.113.67,"{""location"": ""FM"", ""is_mobile"": false}" 21339,8,479,2017-02-24 23:05:41,http://williamson.info/koby.stroman,,161.233.58.89,"{""location"": ""NP"", ""is_mobile"": false}" 21340,8,479,2017-06-10 12:19:40,http://hoeger.co/trevor.leffler,,227.2.59.3,"{""location"": ""UZ"", ""is_mobile"": true}" 21341,8,479,2017-02-01 16:30:15,http://tremblay.org/madeline_schaden,,222.88.57.202,"{""location"": ""FR"", ""is_mobile"": false}" 21342,8,479,2017-05-03 09:03:26,http://welch.biz/tanya.schuppe,,92.141.135.242,"{""location"": ""NF"", ""is_mobile"": false}" 21343,8,479,2017-06-12 05:06:50,http://volkman.org/edd_cartwright,,237.121.30.5,"{""location"": ""MT"", ""is_mobile"": true}" 21344,8,479,2016-12-15 23:35:20,http://oberbrunnerbradtke.co/jack_konopelski,,210.56.16.230,"{""location"": ""AS"", ""is_mobile"": false}" 21345,8,479,2017-04-21 23:13:39,http://jenkins.org/demarco,,238.198.45.217,"{""location"": ""TR"", ""is_mobile"": false}" 21346,8,479,2017-03-20 12:39:52,http://mclaughlinlemke.com/maximo.dickens,,152.16.186.84,"{""location"": ""UZ"", ""is_mobile"": true}" 21347,8,479,2017-05-02 16:00:07,http://doyle.com/tomas,,40.101.226.188,"{""location"": ""BJ"", ""is_mobile"": true}" 21348,8,479,2017-05-28 03:04:37,http://lakin.info/tristin.huel,,37.100.236.178,"{""location"": ""UA"", ""is_mobile"": false}" 21349,8,479,2017-04-20 00:43:12,http://eichmann.net/mona_wunsch,,217.133.182.65,"{""location"": ""NE"", ""is_mobile"": false}" 21350,8,479,2017-06-03 10:23:05,http://beier.info/everardo.torp,,7.82.160.129,"{""location"": ""RW"", ""is_mobile"": true}" 21351,8,479,2017-03-06 16:30:08,http://bartolettibernier.biz/alisa_fritsch,,165.229.10.146,"{""location"": ""LS"", ""is_mobile"": true}" 21352,8,479,2016-12-29 08:24:11,http://johnson.biz/antonio.fadel,,198.138.194.103,"{""location"": ""BN"", ""is_mobile"": true}" 21353,8,479,2017-01-24 11:01:55,http://hyatthaag.biz/novella,,229.216.76.98,"{""location"": ""UZ"", ""is_mobile"": true}" 21354,8,479,2017-03-27 10:15:22,http://wintheiser.net/rowena_towne,,218.223.167.96,"{""location"": ""RS"", ""is_mobile"": true}" 21355,8,479,2017-06-05 01:56:31,http://wintheiser.org/tyreek,,158.3.7.124,"{""location"": ""IR"", ""is_mobile"": false}" 21356,8,479,2017-04-09 21:12:05,http://watsica.name/britney_brown,,75.218.224.150,"{""location"": ""MR"", ""is_mobile"": true}" 21357,8,479,2017-05-29 17:40:58,http://greenfeldergoyette.io/koby.kirlin,,162.5.47.218,"{""location"": ""BL"", ""is_mobile"": false}" 21358,8,479,2016-12-16 17:18:53,http://leschschuster.biz/keshaun_leuschke,,78.101.173.128,"{""location"": ""FK"", ""is_mobile"": false}" 21359,8,479,2017-03-21 23:06:48,http://gislasonhackett.net/joe,,156.15.171.88,"{""location"": ""LR"", ""is_mobile"": false}" 21360,8,479,2017-02-09 06:24:12,http://schoencartwright.name/maci,,75.133.115.146,"{""location"": ""MR"", ""is_mobile"": false}" 21361,8,479,2017-02-09 22:04:02,http://kuhn.io/angela.heathcote,,165.131.167.110,"{""location"": ""GI"", ""is_mobile"": false}" 21362,8,479,2017-01-01 18:23:07,http://purdy.co/brain,,13.164.126.97,"{""location"": ""NL"", ""is_mobile"": false}" 21363,8,480,2017-03-13 20:12:08,http://jones.name/ruell.hayes,,17.78.76.97,"{""location"": ""ID"", ""is_mobile"": false}" 21364,8,480,2017-05-21 00:58:20,http://barton.info/edison,,157.169.151.145,"{""location"": ""BQ"", ""is_mobile"": true}" 21365,8,480,2017-01-28 08:02:19,http://harveysanford.info/gilbert,,58.185.207.95,"{""location"": ""ER"", ""is_mobile"": true}" 21366,8,480,2017-03-10 15:19:05,http://gerhold.info/max,,66.165.18.86,"{""location"": ""GL"", ""is_mobile"": false}" 21367,8,480,2017-06-07 12:03:12,http://crooks.io/deron.herzog,,129.206.136.167,"{""location"": ""CO"", ""is_mobile"": false}" 21368,8,480,2017-03-27 07:18:19,http://sanford.io/jadyn_kaulke,,204.208.247.42,"{""location"": ""DJ"", ""is_mobile"": true}" 21369,8,480,2017-02-25 10:25:59,http://balistreri.biz/conrad,,55.26.240.155,"{""location"": ""TW"", ""is_mobile"": true}" 21370,8,480,2017-03-21 16:27:11,http://kiehn.net/emmet,,95.146.228.29,"{""location"": ""AI"", ""is_mobile"": false}" 21371,8,480,2017-01-11 20:37:22,http://mcdermott.name/junius.rau,,31.82.170.164,"{""location"": ""NE"", ""is_mobile"": false}" 21372,8,480,2017-03-27 23:30:12,http://starkmoen.biz/shaylee.schuster,,169.23.227.192,"{""location"": ""PH"", ""is_mobile"": true}" 21373,8,480,2017-01-03 04:39:45,http://luettgen.com/tierra,,58.171.169.81,"{""location"": ""GL"", ""is_mobile"": false}" 21374,8,480,2017-05-19 12:25:06,http://willmsfranecki.co/don,,246.221.130.31,"{""location"": ""BB"", ""is_mobile"": true}" 21375,8,480,2017-03-16 11:21:03,http://torp.name/beie.conroy,,97.228.188.234,"{""location"": ""LV"", ""is_mobile"": false}" 21376,8,480,2017-05-10 07:10:58,http://wisokykozey.com/imogene.brown,,81.129.115.27,"{""location"": ""ZA"", ""is_mobile"": true}" 21377,8,480,2016-12-30 01:23:33,http://heidenreich.net/miller.quigley,,112.73.160.217,"{""location"": ""ME"", ""is_mobile"": false}" 260,1,6,2017-04-03 09:43:30,http://larson.org/oscar,0.2864307208,106.187.40.128,"{""location"": ""BM"", ""is_mobile"": true}" 261,1,6,2017-01-03 10:09:12,http://kozey.org/genevieve.hansen,0.7212253501,29.84.70.245,"{""location"": ""VC"", ""is_mobile"": true}" 262,1,6,2017-01-04 19:16:01,http://ryan.io/je,0.2960264203,62.100.252.64,"{""location"": ""YT"", ""is_mobile"": true}" 263,1,6,2017-05-13 12:12:08,http://romaguera.net/crystal.kulas,0.3073693934,12.89.127.107,"{""location"": ""SX"", ""is_mobile"": false}" 264,1,6,2017-01-25 12:16:14,http://heathcote.org/jerad_smith,0.7063256905,171.151.250.61,"{""location"": ""MW"", ""is_mobile"": true}" 265,1,6,2017-05-02 02:41:10,http://lynch.biz/agnes_fay,0.1151494915,242.55.15.141,"{""location"": ""AQ"", ""is_mobile"": false}" 266,1,7,2017-03-15 17:00:41,http://buckridge.biz/eriberto_konopelski,0.7619650945,194.216.49.11,"{""location"": ""JE"", ""is_mobile"": false}" 267,1,7,2017-05-03 13:28:47,http://cristpadberg.io/lia.trantow,0.8814378079,246.198.17.158,"{""location"": ""GF"", ""is_mobile"": false}" 268,1,7,2017-04-22 23:58:00,http://rodrigueztrantow.org/jamey_fay,0.5239271210,190.210.172.134,"{""location"": ""KP"", ""is_mobile"": false}" 269,1,7,2017-02-18 13:58:27,http://hackettbergstrom.io/ryley_franecki,0.5350009438,55.249.104.107,"{""location"": ""PH"", ""is_mobile"": false}" 270,1,7,2017-06-12 00:43:54,http://smithambrown.name/felicity,0.3516943662,39.40.180.181,"{""location"": ""LI"", ""is_mobile"": true}" 271,1,7,2017-03-23 23:50:35,http://luettgencorwin.biz/samson_hegmann,0.3452650771,82.40.130.131,"{""location"": ""TL"", ""is_mobile"": true}" 272,1,7,2017-03-23 16:35:19,http://dietrichzboncak.biz/emil.price,0.2741194903,250.236.187.52,"{""location"": ""SJ"", ""is_mobile"": true}" 273,1,7,2016-12-17 23:57:21,http://bruen.biz/parker.kuhn,0.2805371257,204.49.49.216,"{""location"": ""LR"", ""is_mobile"": true}" 274,1,7,2017-02-20 07:27:41,http://douglas.io/haleigh.botsford,0.8144940696,239.194.17.254,"{""location"": ""GP"", ""is_mobile"": false}" 275,1,7,2016-12-31 09:54:26,http://creminmoore.com/magnus,0.9784108809,243.193.31.49,"{""location"": ""SM"", ""is_mobile"": false}" 276,1,7,2017-01-28 01:00:26,http://bogisich.io/maurice_mayert,0.7996932547,201.163.104.192,"{""location"": ""VI"", ""is_mobile"": true}" 277,1,7,2017-03-31 22:34:10,http://ledner.org/eula,0.9983680004,68.223.168.47,"{""location"": ""IN"", ""is_mobile"": false}" 278,1,7,2016-12-15 12:51:46,http://becker.co/rachel_hirthe,0.5205821927,100.45.224.158,"{""location"": ""CY"", ""is_mobile"": true}" 279,1,7,2017-01-19 10:46:42,http://kozey.net/donna_crooks,0.9312369087,3.172.252.233,"{""location"": ""TW"", ""is_mobile"": false}" 280,1,7,2017-03-19 05:00:47,http://keler.co/serenity.gottlieb,0.0045982871,134.247.219.94,"{""location"": ""DK"", ""is_mobile"": false}" 281,1,7,2017-02-05 01:24:38,http://armstrong.net/jillian,0.2932671090,91.11.189.30,"{""location"": ""HR"", ""is_mobile"": false}" 282,1,7,2017-04-30 02:08:25,http://rogahn.com/cortez.oconner,0.7806050391,137.27.224.89,"{""location"": ""VG"", ""is_mobile"": true}" 283,1,7,2017-03-05 23:55:34,http://trantow.name/je_zboncak,0.3618442111,211.240.227.251,"{""location"": ""SX"", ""is_mobile"": true}" 284,1,7,2017-03-02 22:02:19,http://damore.name/pietro_rice,0.7685661566,242.200.40.75,"{""location"": ""PR"", ""is_mobile"": false}" 285,1,7,2017-05-05 17:31:28,http://steuber.biz/don,0.7519503138,86.198.75.31,"{""location"": ""AT"", ""is_mobile"": true}" 286,1,7,2017-01-01 02:16:45,http://uptonwillms.io/lisandro.konopelski,0.5728517451,109.231.38.130,"{""location"": ""GN"", ""is_mobile"": false}" 287,1,7,2017-03-14 00:46:54,http://brekke.co/belle.lynch,0.8065999620,186.184.245.233,"{""location"": ""NF"", ""is_mobile"": true}" 288,1,7,2017-04-13 14:02:01,http://gleasonleffler.io/elroy,0.9291218574,10.83.80.43,"{""location"": ""RW"", ""is_mobile"": true}" 289,1,7,2017-01-02 19:08:54,http://stokeskris.co/antonetta_vonrueden,0.3180651822,229.37.123.85,"{""location"": ""LT"", ""is_mobile"": false}" 290,1,7,2017-02-26 16:41:08,http://nicolaslubowitz.info/marcella,0.9861733782,9.231.210.104,"{""location"": ""NR"", ""is_mobile"": true}" 291,1,7,2017-01-10 11:02:15,http://macejkovicheidenreich.org/keanu,0.6432571478,12.179.41.223,"{""location"": ""PK"", ""is_mobile"": false}" 292,1,7,2017-04-08 07:23:12,http://greenokeefe.net/salma_baumbach,0.4071050860,166.106.42.91,"{""location"": ""ZW"", ""is_mobile"": true}" 293,1,7,2016-12-24 09:50:23,http://hand.org/allene.leuschke,0.4093570870,96.170.204.167,"{""location"": ""MY"", ""is_mobile"": false}" 294,1,7,2017-04-25 21:01:10,http://osinskijohnston.io/kaley.lubowitz,0.3421712197,172.246.70.245,"{""location"": ""ID"", ""is_mobile"": true}" 295,1,7,2017-01-16 14:39:14,http://harrichuppe.io/marcos,0.9523782445,29.244.43.243,"{""location"": ""MZ"", ""is_mobile"": true}" 296,1,7,2016-12-26 13:14:56,http://koch.name/richard_swift,0.2935921328,177.184.46.13,"{""location"": ""ZA"", ""is_mobile"": true}" 297,1,7,2016-12-28 20:20:22,http://bailey.org/willard_robel,0.1899216399,159.170.35.26,"{""location"": ""GL"", ""is_mobile"": true}" 298,1,7,2017-05-03 16:34:21,http://collinscruickshank.info/jayne,0.6219826289,176.194.184.2,"{""location"": ""AG"", ""is_mobile"": true}" 299,1,7,2017-04-16 01:16:31,http://kuvalis.net/emerson_barton,0.6746315878,177.26.106.161,"{""location"": ""AD"", ""is_mobile"": true}" 300,1,7,2016-12-21 11:47:38,http://hintz.net/delphia,0.9954208743,113.50.173.188,"{""location"": ""NL"", ""is_mobile"": false}" 301,1,7,2017-03-29 16:48:17,http://kirlin.com/aryanna,0.2875143554,153.190.154.47,"{""location"": ""BA"", ""is_mobile"": true}" 302,1,8,2016-12-16 18:15:47,http://weimann.net/ronaldo_haley,,242.140.207.225,"{""location"": ""KP"", ""is_mobile"": true}" 303,1,8,2016-12-13 12:23:57,http://cruickshankschmidt.info/lori_beahan,,243.88.26.72,"{""location"": ""RW"", ""is_mobile"": true}" 304,1,8,2016-12-27 10:32:30,http://franeckiquigley.org/rick_koepp,,34.215.100.8,"{""location"": ""BT"", ""is_mobile"": false}" 305,1,8,2017-01-17 09:38:01,http://lang.info/bianka,,150.150.105.101,"{""location"": ""LB"", ""is_mobile"": true}" 306,1,8,2017-02-16 13:39:00,http://harris.org/dimitri.heaney,,68.155.157.138,"{""location"": ""PH"", ""is_mobile"": false}" 307,1,8,2017-01-19 10:28:13,http://bruen.org/dashawn.smitham,,14.95.222.47,"{""location"": ""MD"", ""is_mobile"": true}" 308,1,8,2017-05-27 19:51:31,http://nikolaus.io/cora,,184.47.164.254,"{""location"": ""GM"", ""is_mobile"": true}" 309,1,8,2017-05-03 06:01:58,http://olson.com/kathlyn,,151.115.36.207,"{""location"": ""WF"", ""is_mobile"": false}" 310,1,8,2017-01-19 21:14:13,http://cormier.info/elisabeth,,171.82.150.188,"{""location"": ""SA"", ""is_mobile"": true}" 311,1,8,2016-12-14 06:40:45,http://boehm.com/kurt.schamberger,,148.141.70.170,"{""location"": ""BO"", ""is_mobile"": true}" 312,1,8,2017-02-23 21:10:55,http://macgyverwilkinson.com/lina.damore,,153.220.141.226,"{""location"": ""BW"", ""is_mobile"": false}" 21378,8,480,2017-01-17 12:51:41,http://mann.co/ebba,,119.17.120.115,"{""location"": ""DK"", ""is_mobile"": true}" 21379,8,480,2017-05-23 09:54:48,http://litteldach.biz/nicholas_eichmann,,117.110.100.53,"{""location"": ""AW"", ""is_mobile"": true}" 21380,8,480,2017-01-09 00:35:17,http://bailey.biz/nikko_keler,,233.101.79.99,"{""location"": ""GR"", ""is_mobile"": false}" 21381,8,480,2017-06-04 04:53:46,http://wymandicki.name/billy_pouros,,203.250.3.132,"{""location"": ""MC"", ""is_mobile"": true}" 21382,8,480,2017-03-05 21:28:14,http://bechtelar.biz/leonardo,,201.21.221.42,"{""location"": ""KZ"", ""is_mobile"": true}" 21383,8,480,2016-12-22 02:40:33,http://pagacwelch.io/ceasar.bernhard,,67.112.121.201,"{""location"": ""TM"", ""is_mobile"": true}" 21384,8,480,2017-03-15 14:24:14,http://hamill.info/elva_bogisich,,34.54.14.154,"{""location"": ""IL"", ""is_mobile"": true}" 21385,8,480,2017-01-08 10:49:24,http://schummrunolfsdottir.net/aunta.rowe,,15.176.19.12,"{""location"": ""SM"", ""is_mobile"": false}" 21386,8,480,2017-01-07 10:23:42,http://dubuque.net/maye,,89.157.157.235,"{""location"": ""CI"", ""is_mobile"": false}" 21387,8,480,2017-05-04 15:37:33,http://bernier.com/madelyn.bednar,,96.245.126.227,"{""location"": ""NZ"", ""is_mobile"": false}" 21388,8,480,2017-02-24 16:52:25,http://reichert.co/toni_hansen,,179.121.83.145,"{""location"": ""VI"", ""is_mobile"": false}" 21389,8,480,2016-12-15 17:26:33,http://oconnell.co/trevion,,208.168.29.4,"{""location"": ""GM"", ""is_mobile"": true}" 21390,8,480,2017-04-30 00:28:49,http://prohaskaebert.net/ila.crooks,,54.85.156.11,"{""location"": ""NA"", ""is_mobile"": true}" 21391,8,480,2017-02-08 10:31:42,http://sipes.com/wyatt,,28.87.37.186,"{""location"": ""SG"", ""is_mobile"": false}" 21392,8,480,2017-02-27 03:29:00,http://stracke.biz/jalen.runolfsdottir,,242.112.200.156,"{""location"": ""PT"", ""is_mobile"": false}" 21393,8,480,2017-05-24 00:52:00,http://schambergerrice.info/katherine.dickinson,,155.122.127.119,"{""location"": ""WF"", ""is_mobile"": true}" 21394,8,480,2017-04-27 08:57:08,http://bashirian.io/wilmer.ondricka,,151.234.37.76,"{""location"": ""VG"", ""is_mobile"": true}" 21395,8,480,2017-05-23 22:23:39,http://morar.co/jordi_eichmann,,205.41.243.230,"{""location"": ""MV"", ""is_mobile"": true}" 21396,8,480,2017-01-03 05:03:59,http://murray.biz/rasheed,,126.137.101.155,"{""location"": ""ML"", ""is_mobile"": true}" 21397,8,480,2016-12-28 09:03:14,http://thiel.org/viviane,,13.56.26.14,"{""location"": ""BS"", ""is_mobile"": true}" 21398,8,480,2017-03-16 01:24:41,http://powlowski.org/lew.crist,,92.171.155.221,"{""location"": ""SM"", ""is_mobile"": false}" 21399,8,480,2017-02-01 00:35:56,http://monahan.org/laura,,135.81.92.53,"{""location"": ""TJ"", ""is_mobile"": true}" 21400,8,480,2017-02-22 18:57:35,http://schmidt.io/benjamin,,64.215.148.145,"{""location"": ""PM"", ""is_mobile"": true}" 21401,8,480,2017-01-16 10:30:47,http://keebler.info/else,,17.179.12.99,"{""location"": ""AW"", ""is_mobile"": true}" 21402,8,480,2017-02-13 01:44:24,http://beierlakin.name/vern,,70.15.140.225,"{""location"": ""TC"", ""is_mobile"": true}" 21403,8,480,2017-05-18 23:51:07,http://funk.com/olin,,236.149.96.50,"{""location"": ""LR"", ""is_mobile"": true}" 21404,8,480,2017-02-14 03:29:40,http://strosin.info/orlando,,213.175.106.74,"{""location"": ""CA"", ""is_mobile"": true}" 21405,8,480,2016-12-27 01:40:15,http://raynordoyle.name/martina,,17.92.55.108,"{""location"": ""IM"", ""is_mobile"": false}" 21406,8,480,2017-01-23 01:23:16,http://stokesvon.org/adolfo.howell,,29.120.77.52,"{""location"": ""LK"", ""is_mobile"": false}" 21407,8,480,2017-02-08 07:46:56,http://pfeffer.org/alice,,175.143.203.182,"{""location"": ""PY"", ""is_mobile"": true}" 21408,8,480,2017-05-30 20:09:36,http://weimann.com/meggie,,135.172.244.213,"{""location"": ""BG"", ""is_mobile"": false}" 21409,8,480,2017-04-19 05:09:55,http://okeefe.co/alex.dach,,22.174.2.171,"{""location"": ""KP"", ""is_mobile"": false}" 21410,8,480,2017-03-25 08:08:13,http://robel.com/jettie,,142.208.114.223,"{""location"": ""SD"", ""is_mobile"": true}" 21411,8,480,2017-01-30 22:08:57,http://hermanmayer.name/delaney,,44.87.203.91,"{""location"": ""BL"", ""is_mobile"": false}" 21412,8,480,2016-12-23 22:50:29,http://steuber.com/adonis.adams,,112.157.126.28,"{""location"": ""NO"", ""is_mobile"": false}" 21413,8,480,2017-04-04 12:24:14,http://ko.com/evangeline,,37.244.92.34,"{""location"": ""YT"", ""is_mobile"": true}" 21414,8,480,2016-12-19 14:28:37,http://bernhard.org/jaylin,,84.90.251.208,"{""location"": ""BG"", ""is_mobile"": false}" 21415,8,481,2017-03-27 04:33:02,http://price.net/mortimer_smith,0.7051708670,171.254.26.164,"{""location"": ""MK"", ""is_mobile"": true}" 21416,8,481,2017-05-11 23:14:45,http://shields.net/braeden,0.2964600864,128.185.162.83,"{""location"": ""PR"", ""is_mobile"": false}" 21417,8,481,2017-03-10 06:56:24,http://spencer.biz/leonor.walter,0.6856056816,197.129.119.150,"{""location"": ""TL"", ""is_mobile"": true}" 21418,8,481,2017-04-25 04:58:47,http://colefadel.net/claudie_turcotte,0.4595871251,59.140.183.157,"{""location"": ""JE"", ""is_mobile"": true}" 21419,8,481,2017-01-13 02:02:33,http://bednar.com/orlo,0.5000198945,54.39.79.168,"{""location"": ""VI"", ""is_mobile"": false}" 21420,8,481,2017-05-17 20:31:56,http://konopelski.name/dudley,0.1520285004,69.10.60.191,"{""location"": ""MG"", ""is_mobile"": false}" 21421,8,481,2017-03-12 15:47:00,http://andersongulgowski.net/nestor,0.4910319367,67.142.20.145,"{""location"": ""DJ"", ""is_mobile"": false}" 21422,8,481,2017-03-02 14:09:10,http://blick.biz/carolina_schneider,0.1628165717,198.74.196.158,"{""location"": ""AR"", ""is_mobile"": true}" 21423,8,481,2017-05-05 19:58:59,http://wilkinson.net/augusta,0.6445749264,100.241.243.235,"{""location"": ""US"", ""is_mobile"": false}" 21424,8,481,2017-01-10 15:03:05,http://kirlindickens.io/dan_parisian,0.2058874908,172.126.171.44,"{""location"": ""MZ"", ""is_mobile"": false}" 21425,8,481,2017-02-26 16:09:41,http://ko.co/shyanne.green,0.0177032475,232.119.131.29,"{""location"": ""LU"", ""is_mobile"": true}" 21426,8,481,2017-04-09 04:53:57,http://renner.net/jaime_dooley,0.6240295143,130.119.199.248,"{""location"": ""LR"", ""is_mobile"": false}" 21427,8,481,2017-04-14 20:25:32,http://davisbogan.name/darren,0.1598364699,85.193.226.34,"{""location"": ""KP"", ""is_mobile"": false}" 21428,8,481,2017-02-14 17:12:34,http://schowaltermoriette.biz/kadin_smitham,0.1867757225,38.253.155.132,"{""location"": ""AM"", ""is_mobile"": true}" 21429,8,481,2017-05-28 04:04:19,http://beahan.com/ellie,0.6610382410,218.83.147.14,"{""location"": ""EE"", ""is_mobile"": true}" 21430,8,481,2017-03-31 17:23:04,http://kuvalis.com/marquise,0.1173893796,73.24.151.182,"{""location"": ""KE"", ""is_mobile"": true}" 21431,8,481,2017-06-13 20:43:41,http://moen.name/myra,0.1079982039,85.65.238.157,"{""location"": ""SC"", ""is_mobile"": false}" 313,1,8,2017-05-10 00:48:15,http://damore.info/lyric_ernser,,30.36.60.121,"{""location"": ""CU"", ""is_mobile"": true}" 314,1,8,2017-03-19 03:32:49,http://quitzon.co/nat_ohara,,159.195.68.147,"{""location"": ""RE"", ""is_mobile"": false}" 315,1,8,2016-12-25 08:29:35,http://cruickshank.info/darwin,,110.77.15.188,"{""location"": ""HM"", ""is_mobile"": true}" 316,1,8,2017-04-18 18:23:30,http://bernier.info/tom_stoltenberg,,74.184.192.76,"{""location"": ""MG"", ""is_mobile"": true}" 317,1,8,2017-04-10 04:08:02,http://cummeratalesch.org/abdul,,100.46.246.37,"{""location"": ""CL"", ""is_mobile"": false}" 318,1,8,2017-05-22 13:47:17,http://crooks.name/clay,,92.49.186.30,"{""location"": ""IL"", ""is_mobile"": true}" 319,1,8,2017-06-12 16:56:22,http://weber.io/icie.ledner,,9.234.14.36,"{""location"": ""BH"", ""is_mobile"": true}" 320,1,8,2017-04-25 17:26:33,http://schmelerterry.net/bonnie.hirthe,,193.231.31.133,"{""location"": ""KI"", ""is_mobile"": true}" 321,1,8,2016-12-23 18:10:58,http://wittingheller.org/remington,,12.120.98.161,"{""location"": ""PM"", ""is_mobile"": false}" 322,1,8,2017-03-11 19:07:33,http://boscopfannerstill.io/jacques,,153.120.103.128,"{""location"": ""CC"", ""is_mobile"": false}" 323,1,8,2017-06-11 04:24:43,http://mcdermott.name/ayden.orn,,110.208.27.149,"{""location"": ""NO"", ""is_mobile"": false}" 324,1,8,2017-01-12 11:15:18,http://hammesheathcote.com/alfonzo_yost,,231.69.190.214,"{""location"": ""DK"", ""is_mobile"": true}" 325,1,8,2017-05-04 20:45:40,http://mayer.biz/fritz,,232.215.103.28,"{""location"": ""KR"", ""is_mobile"": true}" 326,1,8,2017-03-11 13:34:56,http://grimesgusikowski.info/claud,,181.98.80.139,"{""location"": ""LS"", ""is_mobile"": true}" 327,1,8,2017-03-19 00:45:17,http://auer.name/name,,108.22.241.163,"{""location"": ""CN"", ""is_mobile"": true}" 328,1,8,2017-03-18 16:24:07,http://schillergreenfelder.info/valentin_sawayn,,166.82.27.22,"{""location"": ""IQ"", ""is_mobile"": false}" 329,1,8,2017-01-31 10:03:06,http://paucek.org/emmie,,192.46.253.176,"{""location"": ""ST"", ""is_mobile"": true}" 330,1,8,2016-12-24 10:36:23,http://ferrywilliamson.info/cornell,,228.196.155.251,"{""location"": ""VN"", ""is_mobile"": false}" 331,1,8,2017-04-16 08:42:32,http://lockman.net/rodolfo_rohan,,156.96.140.159,"{""location"": ""ZM"", ""is_mobile"": false}" 332,1,8,2017-02-09 18:15:36,http://witting.com/vergie.block,,103.28.212.205,"{""location"": ""AT"", ""is_mobile"": false}" 333,1,8,2017-05-04 11:20:49,http://simonis.org/lexi.kihn,,69.232.104.59,"{""location"": ""NF"", ""is_mobile"": false}" 334,1,8,2017-04-10 21:01:42,http://gerlach.name/miracle,,216.68.204.66,"{""location"": ""LT"", ""is_mobile"": true}" 335,1,8,2017-03-02 11:11:59,http://moore.io/reie_mcglynn,,105.100.130.236,"{""location"": ""DM"", ""is_mobile"": false}" 336,1,8,2017-03-04 18:08:37,http://wilkinsonhettinger.io/elroy,,6.178.181.212,"{""location"": ""GQ"", ""is_mobile"": true}" 337,1,8,2017-05-23 05:40:04,http://gutkowski.name/kenneth,,252.129.190.3,"{""location"": ""WS"", ""is_mobile"": false}" 338,1,8,2017-06-11 14:19:08,http://konopelskimarks.co/arielle,,16.16.227.45,"{""location"": ""BQ"", ""is_mobile"": true}" 339,1,8,2017-02-14 21:18:56,http://turcotte.net/lilliana,,81.147.64.13,"{""location"": ""HM"", ""is_mobile"": true}" 340,1,8,2017-03-24 10:39:09,http://bernhard.org/moshe,,206.167.216.203,"{""location"": ""BZ"", ""is_mobile"": true}" 341,1,9,2017-04-10 05:28:10,http://mcglynn.org/benny.armstrong,,137.65.174.79,"{""location"": ""UM"", ""is_mobile"": false}" 342,1,9,2017-04-24 09:18:06,http://aufderhar.co/lori,,135.201.3.229,"{""location"": ""MR"", ""is_mobile"": false}" 343,1,9,2017-05-20 19:11:19,http://schoenwunsch.co/elisabeth,,189.151.166.242,"{""location"": ""PY"", ""is_mobile"": true}" 344,1,9,2017-03-20 05:37:05,http://danielrowe.org/lupe,,112.8.164.165,"{""location"": ""SR"", ""is_mobile"": false}" 345,1,9,2017-05-14 22:42:41,http://mcculloughwalter.info/evan_hauck,,32.165.179.57,"{""location"": ""BE"", ""is_mobile"": false}" 346,1,9,2017-01-19 00:12:46,http://cole.com/kelton_klocko,,57.116.30.178,"{""location"": ""GL"", ""is_mobile"": false}" 347,1,9,2017-01-27 20:36:01,http://nienow.net/jadon_langworth,,106.241.129.86,"{""location"": ""SG"", ""is_mobile"": true}" 348,1,9,2017-04-28 12:20:53,http://grimetreich.net/jordi,,120.49.22.122,"{""location"": ""TR"", ""is_mobile"": false}" 349,1,9,2017-05-12 12:33:20,http://walter.info/prince,,37.82.182.78,"{""location"": ""ES"", ""is_mobile"": true}" 350,1,9,2017-05-03 12:26:19,http://ruel.info/janiya.adams,,95.5.14.122,"{""location"": ""HM"", ""is_mobile"": true}" 351,1,9,2017-04-01 16:56:20,http://waelchicremin.net/lisette_kuphal,,209.43.122.70,"{""location"": ""AR"", ""is_mobile"": false}" 352,1,9,2017-02-16 03:45:03,http://bergeveum.co/travon,,161.130.165.111,"{""location"": ""NC"", ""is_mobile"": false}" 353,1,9,2017-02-06 08:40:49,http://kirlin.biz/mertie,,21.92.38.138,"{""location"": ""AO"", ""is_mobile"": false}" 354,1,9,2017-06-03 11:41:42,http://murphy.info/waylon,,134.150.30.248,"{""location"": ""BM"", ""is_mobile"": true}" 355,1,9,2017-05-18 20:26:49,http://dibbertmcclure.info/myrl,,67.119.243.225,"{""location"": ""PR"", ""is_mobile"": false}" 356,1,9,2017-01-13 22:31:32,http://pollichconnelly.biz/shad.kuhlman,,155.237.17.84,"{""location"": ""BT"", ""is_mobile"": true}" 357,1,9,2017-04-21 17:39:40,http://reinger.biz/tre,,202.81.250.62,"{""location"": ""DZ"", ""is_mobile"": true}" 358,1,9,2017-06-05 03:18:27,http://breitenberglittle.co/randi.cronin,,218.116.97.120,"{""location"": ""FI"", ""is_mobile"": true}" 359,1,9,2017-04-07 22:52:45,http://hahngleason.com/kellie.mayer,,203.55.86.88,"{""location"": ""MG"", ""is_mobile"": true}" 360,1,9,2017-03-20 17:31:52,http://mills.co/floy.murray,,254.41.109.161,"{""location"": ""MY"", ""is_mobile"": true}" 361,1,9,2017-04-16 20:32:35,http://steuber.biz/vincenza_sanford,,109.219.211.88,"{""location"": ""SY"", ""is_mobile"": false}" 362,1,9,2017-03-19 08:26:27,http://cain.org/abagail,,234.8.35.114,"{""location"": ""DZ"", ""is_mobile"": true}" 363,1,9,2017-04-28 17:57:54,http://murphy.com/herminio.upton,,39.30.116.29,"{""location"": ""IN"", ""is_mobile"": true}" 364,1,9,2017-03-11 02:05:05,http://medhurst.org/tyrel_luettgen,,253.48.216.99,"{""location"": ""SV"", ""is_mobile"": false}" 365,1,9,2017-04-10 23:09:21,http://mclaughlingreenholt.co/tamara.dooley,,217.60.75.15,"{""location"": ""PA"", ""is_mobile"": false}" 366,1,9,2017-01-03 02:06:50,http://mcclure.co/fannie,,36.128.62.29,"{""location"": ""FR"", ""is_mobile"": false}" 367,1,9,2017-01-15 06:07:08,http://koch.io/waino,,212.79.141.113,"{""location"": ""MN"", ""is_mobile"": true}" 368,1,9,2017-03-21 11:51:55,http://reynolds.com/antonia.heathcote,,121.241.196.125,"{""location"": ""CY"", ""is_mobile"": false}" 369,1,9,2017-05-26 14:20:13,http://zboncak.io/marilou,,157.28.57.81,"{""location"": ""KI"", ""is_mobile"": true}" 21432,8,481,2017-06-12 23:14:12,http://wilkinson.name/marlene,0.9886218830,184.102.230.81,"{""location"": ""VG"", ""is_mobile"": false}" 21433,8,481,2017-01-14 07:08:56,http://bradtkehills.com/kevon_hammes,0.7291856400,141.14.61.93,"{""location"": ""CL"", ""is_mobile"": false}" 21434,8,481,2017-05-26 09:41:25,http://uptonhuels.org/destin.wiegand,0.6188809043,252.213.232.3,"{""location"": ""MY"", ""is_mobile"": false}" 21435,8,481,2017-05-08 20:07:01,http://okeefe.io/kelsi_farrell,0.7639908095,186.130.88.81,"{""location"": ""AX"", ""is_mobile"": true}" 21436,8,481,2017-02-04 17:04:41,http://miller.biz/meda,0.9347658989,147.187.29.37,"{""location"": ""KH"", ""is_mobile"": true}" 21437,8,481,2017-04-26 03:56:12,http://douglas.com/wilson.bergstrom,0.7043313229,116.162.65.132,"{""location"": ""GG"", ""is_mobile"": true}" 21438,8,481,2017-04-03 11:46:10,http://schimmelgrant.com/tillman,0.3221038526,87.133.175.145,"{""location"": ""EC"", ""is_mobile"": true}" 21439,8,481,2017-03-08 07:33:30,http://eichmann.name/abe,0.3734542282,45.123.219.183,"{""location"": ""MF"", ""is_mobile"": false}" 21440,8,481,2017-02-24 05:29:55,http://ondricka.info/guadalupe.reichert,0.3015158155,168.253.232.144,"{""location"": ""GQ"", ""is_mobile"": true}" 21441,8,481,2017-02-25 04:09:21,http://murraycrona.info/toy,0.5603720194,98.235.175.161,"{""location"": ""ST"", ""is_mobile"": true}" 21442,8,481,2017-02-19 02:26:50,http://schimmel.org/jeremy,0.0732232134,129.167.7.147,"{""location"": ""CY"", ""is_mobile"": true}" 21443,8,481,2017-02-15 08:08:23,http://roberts.com/deja_ruel,0.2408559357,35.149.3.211,"{""location"": ""MN"", ""is_mobile"": false}" 21444,8,481,2017-04-22 05:54:57,http://gutmann.net/anastasia,0.3571706817,148.27.13.64,"{""location"": ""MP"", ""is_mobile"": true}" 21445,8,481,2017-01-02 16:23:58,http://eichmann.info/andre.cummerata,0.2993549872,152.252.129.36,"{""location"": ""GI"", ""is_mobile"": false}" 21446,8,481,2017-03-30 03:27:42,http://brown.net/kimberly,0.5150267908,239.185.86.71,"{""location"": ""OM"", ""is_mobile"": false}" 21447,8,481,2017-03-02 07:35:41,http://keelinghills.net/adelle.friesen,0.1981348953,172.176.121.188,"{""location"": ""GB"", ""is_mobile"": false}" 21448,8,481,2017-04-27 08:17:50,http://schuster.com/ignacio,0.4544798659,14.161.180.45,"{""location"": ""MW"", ""is_mobile"": false}" 21449,8,481,2017-01-03 07:09:09,http://moore.io/elise_dare,0.1179635025,221.238.51.202,"{""location"": ""MK"", ""is_mobile"": true}" 21450,8,481,2016-12-21 16:31:14,http://will.co/elfrieda,0.1849310077,69.157.99.240,"{""location"": ""MO"", ""is_mobile"": false}" 21451,8,481,2016-12-22 22:07:07,http://considine.net/hayley,0.2721319269,174.9.25.48,"{""location"": ""EE"", ""is_mobile"": false}" 21452,8,481,2017-03-11 09:59:56,http://will.name/golda,0.7751345135,160.37.119.147,"{""location"": ""GE"", ""is_mobile"": true}" 21453,8,481,2017-01-31 11:41:12,http://wardthiel.io/daron_boyle,0.7514174666,178.98.49.155,"{""location"": ""AM"", ""is_mobile"": false}" 21454,8,481,2017-03-19 14:11:10,http://gusikowski.biz/felicity_wintheiser,0.6742491810,98.133.237.176,"{""location"": ""SZ"", ""is_mobile"": true}" 21455,8,481,2017-06-06 00:38:33,http://macejkovic.com/georgianna,0.7163235018,188.149.80.237,"{""location"": ""LR"", ""is_mobile"": true}" 21456,8,481,2017-06-05 18:47:00,http://cain.info/arturo_streich,0.6110775519,191.209.228.129,"{""location"": ""RW"", ""is_mobile"": true}" 21457,8,481,2017-04-18 22:05:47,http://maggiokozey.biz/henriette,0.3905458048,87.221.11.69,"{""location"": ""PA"", ""is_mobile"": true}" 21458,8,481,2017-05-15 05:22:14,http://mitchell.info/chaim,0.1174803264,3.234.84.54,"{""location"": ""NI"", ""is_mobile"": false}" 21459,8,481,2017-01-27 00:30:55,http://turner.co/wellington.bernhard,0.6511589697,252.48.7.197,"{""location"": ""BV"", ""is_mobile"": false}" 21460,8,481,2017-02-21 03:55:26,http://prohaska.io/carlie,0.2442224826,228.35.169.167,"{""location"": ""TO"", ""is_mobile"": false}" 21461,8,481,2017-03-14 01:22:15,http://millsleffler.co/floie,0.1840687797,15.149.185.43,"{""location"": ""MX"", ""is_mobile"": false}" 21462,8,481,2017-02-21 07:06:07,http://berge.net/gertrude,0.8733437662,46.156.211.145,"{""location"": ""SV"", ""is_mobile"": true}" 21463,8,481,2016-12-19 20:09:19,http://dickicummings.info/dawn,0.6164390918,14.36.86.249,"{""location"": ""SK"", ""is_mobile"": false}" 21464,8,481,2016-12-15 19:44:29,http://bogisich.co/ola,0.3511605423,189.64.202.181,"{""location"": ""NU"", ""is_mobile"": true}" 21465,8,481,2017-05-07 14:47:03,http://jenkins.name/odell_kris,0.1667912614,227.143.221.38,"{""location"": ""BW"", ""is_mobile"": false}" 21466,8,481,2017-02-07 00:31:20,http://yundt.org/bennie.beer,0.3253178400,150.78.152.174,"{""location"": ""RU"", ""is_mobile"": false}" 21467,8,481,2017-03-31 15:57:28,http://morar.co/vincenzo,0.5652852570,48.86.49.95,"{""location"": ""DZ"", ""is_mobile"": false}" 21468,8,481,2016-12-13 11:53:58,http://hintzwilderman.name/grant.boehm,0.6908976528,221.250.121.199,"{""location"": ""DK"", ""is_mobile"": true}" 21469,8,481,2017-01-08 02:40:51,http://yosttorp.name/marisa.ritchie,0.0426148595,146.85.159.244,"{""location"": ""GI"", ""is_mobile"": false}" 21470,8,481,2017-03-20 05:43:03,http://langworth.org/georgiana,0.0647868743,149.232.132.57,"{""location"": ""SO"", ""is_mobile"": false}" 21471,8,481,2017-04-02 07:15:05,http://gutkowskibeahan.biz/gloria,0.9133948881,33.13.133.166,"{""location"": ""SH"", ""is_mobile"": true}" 21472,8,481,2017-04-22 02:41:02,http://hane.org/yasmin_ward,0.7600258831,218.52.93.14,"{""location"": ""AZ"", ""is_mobile"": true}" 21473,8,481,2017-03-24 07:56:37,http://torphy.com/patsy_nitzsche,0.2884882990,27.227.120.92,"{""location"": ""CZ"", ""is_mobile"": false}" 21474,8,481,2017-03-22 21:54:50,http://rodriguez.biz/javier,0.3855151322,197.199.225.201,"{""location"": ""SX"", ""is_mobile"": true}" 21475,8,481,2017-02-04 22:58:35,http://funk.net/gideon.frami,0.3410855172,147.57.234.135,"{""location"": ""RU"", ""is_mobile"": true}" 21476,8,481,2016-12-27 16:58:16,http://huelskovacek.org/matt,0.7142886923,149.48.247.98,"{""location"": ""IM"", ""is_mobile"": true}" 21477,8,481,2017-01-19 01:39:11,http://johns.org/janiya.bauch,0.5388145250,229.150.10.235,"{""location"": ""SO"", ""is_mobile"": false}" 21478,8,481,2017-02-14 06:18:30,http://adams.biz/emilio,0.9663540736,36.113.42.69,"{""location"": ""IN"", ""is_mobile"": false}" 21479,8,481,2017-05-26 18:24:26,http://strosindubuque.com/adelbert,0.0753806490,15.254.59.75,"{""location"": ""HM"", ""is_mobile"": false}" 21480,8,481,2017-05-09 11:01:49,http://daugherty.net/kyle,0.7264118568,17.178.107.53,"{""location"": ""GY"", ""is_mobile"": true}" 21481,8,481,2016-12-14 19:04:24,http://tremblaystehr.net/janick,0.8361220957,174.30.49.241,"{""location"": ""SR"", ""is_mobile"": false}" 21482,8,482,2017-03-06 02:51:14,http://kunze.co/candelario.ko,0.4896124765,92.46.58.110,"{""location"": ""DE"", ""is_mobile"": false}" 21483,8,482,2017-03-04 14:30:29,http://haley.org/vickie,0.9298252150,45.58.47.33,"{""location"": ""KP"", ""is_mobile"": false}" 370,1,9,2017-03-09 04:23:48,http://ebert.biz/emil_harris,,42.62.58.11,"{""location"": ""SD"", ""is_mobile"": true}" 371,1,9,2017-01-03 19:34:19,http://kuhiccummings.io/ashlee.quitzon,,111.245.85.209,"{""location"": ""NP"", ""is_mobile"": false}" 372,1,9,2017-06-07 14:48:18,http://schiller.net/ayla,,156.189.55.192,"{""location"": ""MW"", ""is_mobile"": true}" 373,1,9,2016-12-17 11:48:21,http://spencer.biz/rodrick,,158.176.208.80,"{""location"": ""WS"", ""is_mobile"": false}" 374,1,9,2017-04-12 14:30:07,http://wizaheathcote.name/maximus,,10.218.34.162,"{""location"": ""FO"", ""is_mobile"": false}" 375,1,9,2016-12-26 03:26:57,http://bogisichwilliamson.info/kellie.nienow,,213.135.227.69,"{""location"": ""NG"", ""is_mobile"": true}" 376,1,9,2017-05-21 18:16:17,http://keebler.co/lula,,189.42.82.216,"{""location"": ""NF"", ""is_mobile"": false}" 377,1,9,2017-02-26 07:54:58,http://maggio.info/wilhelm,,13.104.3.223,"{""location"": ""KP"", ""is_mobile"": true}" 378,1,9,2017-03-09 06:33:00,http://paucek.io/aisha,,129.102.217.171,"{""location"": ""AD"", ""is_mobile"": true}" 379,1,9,2017-03-31 12:57:54,http://lubowitzhaley.com/lottie,,152.40.31.36,"{""location"": ""NI"", ""is_mobile"": false}" 380,1,9,2016-12-14 01:52:12,http://stokes.biz/caidy.bernier,,48.219.137.149,"{""location"": ""VI"", ""is_mobile"": false}" 381,1,9,2017-02-02 17:42:17,http://wehner.net/taya.adams,,168.239.219.55,"{""location"": ""FO"", ""is_mobile"": false}" 382,1,9,2017-05-19 05:01:22,http://schamberger.org/albin.johnson,,12.60.7.60,"{""location"": ""IR"", ""is_mobile"": true}" 383,1,9,2017-04-17 16:09:37,http://schuppe.org/robb,,165.219.209.120,"{""location"": ""AF"", ""is_mobile"": true}" 384,1,9,2017-05-24 08:09:48,http://kulas.co/deonte,,196.225.67.94,"{""location"": ""TC"", ""is_mobile"": false}" 385,1,9,2017-01-06 16:53:35,http://gottlieb.co/tony,,96.90.31.154,"{""location"": ""BF"", ""is_mobile"": false}" 386,1,9,2017-01-19 11:40:32,http://hintz.net/ettie,,77.195.241.218,"{""location"": ""PK"", ""is_mobile"": false}" 387,1,9,2017-04-01 18:31:52,http://boehm.net/lacey_pollich,,166.191.163.242,"{""location"": ""BG"", ""is_mobile"": false}" 388,1,9,2017-06-04 08:17:11,http://considine.co/dangelo,,99.131.121.248,"{""location"": ""KE"", ""is_mobile"": true}" 389,1,9,2017-02-10 19:33:15,http://bashirianjast.net/joel_jast,,72.118.5.192,"{""location"": ""UZ"", ""is_mobile"": false}" 390,1,9,2017-01-31 03:26:15,http://pagac.co/darrion_damore,,249.48.119.77,"{""location"": ""LB"", ""is_mobile"": true}" 391,1,9,2017-02-24 15:17:00,http://kling.biz/laura,,137.112.49.151,"{""location"": ""NG"", ""is_mobile"": false}" 392,1,9,2017-06-10 15:41:13,http://heathcote.co/vivien,,181.186.120.73,"{""location"": ""KZ"", ""is_mobile"": true}" 393,1,9,2016-12-13 06:27:25,http://altenwerth.org/abbey.botsford,,211.194.44.172,"{""location"": ""GL"", ""is_mobile"": false}" 394,1,9,2017-05-01 04:35:15,http://altenwerth.biz/layla.jerde,,165.79.104.156,"{""location"": ""KE"", ""is_mobile"": false}" 395,1,9,2017-03-18 05:59:54,http://watsicablanda.biz/eulah,,184.89.172.77,"{""location"": ""FK"", ""is_mobile"": false}" 396,1,9,2017-01-22 13:40:34,http://wolf.io/ronny.swift,,35.53.248.124,"{""location"": ""SR"", ""is_mobile"": true}" 397,1,9,2017-01-23 15:11:50,http://boyle.name/orlando_sauer,,166.8.163.27,"{""location"": ""WF"", ""is_mobile"": false}" 398,1,9,2017-06-04 06:42:40,http://hansen.org/summer.kaulke,,197.91.193.64,"{""location"": ""IL"", ""is_mobile"": false}" 399,1,9,2017-03-20 10:04:55,http://cain.com/anastacio_heathcote,,65.110.40.100,"{""location"": ""RU"", ""is_mobile"": false}" 400,1,10,2017-02-28 12:56:30,http://weber.net/nasir,,36.196.102.87,"{""location"": ""SZ"", ""is_mobile"": true}" 401,1,10,2017-02-16 11:44:02,http://ferrycormier.info/kaya.harris,,68.145.84.240,"{""location"": ""TT"", ""is_mobile"": false}" 402,1,10,2016-12-21 21:34:26,http://emard.org/aglae.jenkins,,200.223.114.241,"{""location"": ""MN"", ""is_mobile"": true}" 403,1,10,2017-05-13 08:11:37,http://bergstrom.name/yeenia,,195.200.132.244,"{""location"": ""DO"", ""is_mobile"": false}" 404,1,10,2017-02-23 03:52:21,http://king.net/edwina,,124.152.162.119,"{""location"": ""BH"", ""is_mobile"": false}" 405,1,10,2017-05-11 21:43:14,http://daredickens.io/lina.bode,,57.155.245.154,"{""location"": ""BJ"", ""is_mobile"": false}" 406,1,10,2017-03-09 04:02:02,http://mueller.io/benton,,147.19.223.94,"{""location"": ""PL"", ""is_mobile"": true}" 407,1,10,2017-05-30 16:13:17,http://gaylordconn.info/mitchell,,125.144.122.228,"{""location"": ""VN"", ""is_mobile"": false}" 408,1,10,2017-05-03 07:23:29,http://toy.net/carroll,,187.137.136.251,"{""location"": ""LR"", ""is_mobile"": true}" 409,1,10,2016-12-17 05:25:27,http://bechtelar.net/iva_ferry,,249.133.137.221,"{""location"": ""VG"", ""is_mobile"": false}" 410,1,10,2017-03-20 01:25:34,http://herzog.com/rosemary,,178.168.16.121,"{""location"": ""BN"", ""is_mobile"": false}" 411,1,10,2017-02-14 15:02:28,http://cummerata.biz/caandra_schuster,,81.11.12.35,"{""location"": ""TV"", ""is_mobile"": true}" 412,1,10,2017-05-19 18:52:45,http://nienowbergnaum.co/burley,,50.60.179.168,"{""location"": ""EC"", ""is_mobile"": true}" 413,1,10,2017-02-06 22:06:32,http://lebsack.info/armando_bergnaum,,82.122.182.164,"{""location"": ""TG"", ""is_mobile"": false}" 414,1,10,2017-03-07 13:12:55,http://rogahnlang.info/zella,,169.100.165.128,"{""location"": ""BG"", ""is_mobile"": true}" 415,1,10,2017-02-05 20:38:28,http://schinner.name/andy,,16.143.30.224,"{""location"": ""TF"", ""is_mobile"": true}" 416,1,10,2017-06-08 19:14:28,http://barton.name/larry,,147.116.206.189,"{""location"": ""CL"", ""is_mobile"": true}" 417,1,10,2017-05-13 04:01:44,http://damoredach.io/hayley,,29.167.81.226,"{""location"": ""BB"", ""is_mobile"": false}" 418,1,10,2017-04-05 22:11:38,http://torphyokon.biz/amiya.strosin,,86.94.104.115,"{""location"": ""WF"", ""is_mobile"": false}" 419,1,10,2017-03-26 12:51:11,http://zemlak.net/amir,,11.130.113.197,"{""location"": ""SE"", ""is_mobile"": false}" 420,1,10,2017-01-18 01:09:39,http://hills.net/araceli.dicki,,12.176.42.101,"{""location"": ""PA"", ""is_mobile"": false}" 421,1,10,2017-05-04 06:00:14,http://vandervortmoriette.io/shaylee,,192.193.37.57,"{""location"": ""MC"", ""is_mobile"": true}" 422,1,10,2017-03-01 14:13:46,http://kulas.io/paula_damore,,22.92.235.108,"{""location"": ""DK"", ""is_mobile"": true}" 423,1,10,2017-02-26 16:38:45,http://hilll.name/wyman,,254.67.166.26,"{""location"": ""SR"", ""is_mobile"": true}" 424,1,11,2017-05-02 23:57:24,http://kreiger.org/maribel,,37.62.198.213,"{""location"": ""FO"", ""is_mobile"": true}" 425,1,11,2017-05-05 07:00:31,http://mrazgrady.info/maribel,,202.219.166.111,"{""location"": ""BR"", ""is_mobile"": true}" 426,1,11,2016-12-23 03:00:41,http://corkery.io/taurean_feeney,,136.188.124.12,"{""location"": ""VI"", ""is_mobile"": false}" 21484,8,482,2017-01-30 06:53:27,http://hettinger.co/kaden,0.7178791824,88.23.249.218,"{""location"": ""BB"", ""is_mobile"": true}" 21485,8,482,2017-01-31 12:14:27,http://lehner.name/morton,0.6719325213,27.78.48.248,"{""location"": ""MU"", ""is_mobile"": false}" 21486,8,482,2017-04-28 02:23:58,http://west.biz/cleve.champlin,0.7535044646,186.127.228.108,"{""location"": ""MK"", ""is_mobile"": false}" 21487,8,482,2017-04-19 17:55:53,http://gibsonmraz.co/felton.ko,0.4696008498,205.85.121.163,"{""location"": ""BL"", ""is_mobile"": true}" 21488,8,482,2017-01-15 14:54:16,http://beahan.com/laisha_harber,0.9719216658,45.251.26.249,"{""location"": ""VN"", ""is_mobile"": false}" 21489,8,482,2017-02-27 08:54:03,http://crooks.biz/ines,0.9033331923,135.72.23.210,"{""location"": ""GW"", ""is_mobile"": false}" 21490,8,482,2017-04-23 05:25:46,http://schuster.org/earnest.larson,0.9344914871,184.58.195.139,"{""location"": ""MS"", ""is_mobile"": false}" 21491,8,482,2017-03-30 23:26:03,http://stark.com/janick_stiedemann,0.0454663942,142.60.60.79,"{""location"": ""IM"", ""is_mobile"": true}" 21492,8,482,2017-01-12 20:28:25,http://medhurst.name/adella.bahringer,0.0176105087,144.233.252.168,"{""location"": ""GS"", ""is_mobile"": false}" 21493,8,482,2017-03-30 13:37:41,http://reynolds.com/luisa_legros,0.9266801049,165.150.48.110,"{""location"": ""KR"", ""is_mobile"": true}" 21494,8,482,2017-02-12 06:22:18,http://krajcikvandervort.name/stanton_glover,0.1943651596,134.140.40.19,"{""location"": ""TC"", ""is_mobile"": true}" 21495,8,482,2016-12-27 03:10:39,http://becker.com/patrick,0.6387942446,27.254.153.252,"{""location"": ""AQ"", ""is_mobile"": false}" 21496,8,482,2016-12-29 20:39:09,http://hand.net/kira_nolan,0.3213341194,66.236.239.52,"{""location"": ""IN"", ""is_mobile"": true}" 21497,8,482,2017-06-13 18:47:11,http://conn.com/jeanette,0.2766003865,158.20.151.238,"{""location"": ""JM"", ""is_mobile"": false}" 21498,8,482,2017-06-01 06:29:30,http://barrows.org/kathleen,0.2376837547,230.153.40.134,"{""location"": ""CC"", ""is_mobile"": true}" 21499,8,482,2017-02-20 12:57:06,http://donnelly.co/dangelo,0.2366547163,52.87.141.153,"{""location"": ""TN"", ""is_mobile"": true}" 21500,8,482,2017-05-10 07:50:32,http://haleymayer.co/craig_labadie,0.7644014664,139.253.72.122,"{""location"": ""IO"", ""is_mobile"": false}" 21501,8,482,2017-06-04 21:26:19,http://haagbartoletti.info/lourdes,0.8036379858,90.168.50.195,"{""location"": ""BN"", ""is_mobile"": false}" 21502,8,482,2017-03-31 05:38:48,http://fahey.net/christopher,0.4014651886,173.71.151.152,"{""location"": ""KY"", ""is_mobile"": true}" 21503,8,482,2017-04-05 09:29:15,http://gloverbode.com/theresa_kulas,0.5257911523,182.87.40.212,"{""location"": ""VC"", ""is_mobile"": false}" 21504,8,482,2017-06-02 17:42:58,http://quigley.org/giuseppe,0.7543192362,130.110.178.56,"{""location"": ""PA"", ""is_mobile"": true}" 21505,8,482,2017-04-09 05:19:35,http://gutmannturner.info/jorge.padberg,0.6610327839,230.229.146.240,"{""location"": ""JM"", ""is_mobile"": true}" 21506,8,483,2017-02-18 08:04:36,http://collierkemmer.io/toney.steuber,0.9974149933,110.36.249.189,"{""location"": ""LT"", ""is_mobile"": true}" 21507,8,483,2017-01-25 00:33:21,http://tromp.name/marielle_heathcote,0.2672210832,71.212.219.106,"{""location"": ""EH"", ""is_mobile"": false}" 21508,8,483,2017-03-11 05:03:42,http://sawaynhomenick.name/jayde_mitchell,0.3625100543,187.121.213.136,"{""location"": ""TK"", ""is_mobile"": true}" 21509,8,483,2016-12-25 13:08:50,http://flatley.io/cleveland,0.2215867950,131.139.37.208,"{""location"": ""ID"", ""is_mobile"": false}" 21510,8,483,2017-02-04 03:09:42,http://cristdonnelly.com/amie_aufderhar,0.1286229530,14.198.65.205,"{""location"": ""TN"", ""is_mobile"": true}" 21511,8,483,2017-01-26 22:42:10,http://boylezulauf.biz/wilhelm.kozey,0.2615617623,220.16.164.243,"{""location"": ""CH"", ""is_mobile"": false}" 21512,8,483,2017-03-27 14:30:32,http://kilback.org/amani_ko,0.9461744758,6.237.39.59,"{""location"": ""NL"", ""is_mobile"": false}" 21513,8,483,2017-01-04 04:32:40,http://hettinger.co/annabell.wiegand,0.8596591504,59.135.184.31,"{""location"": ""HU"", ""is_mobile"": true}" 21514,8,483,2017-02-24 06:29:23,http://little.biz/patricia,0.4528267387,108.21.99.155,"{""location"": ""KE"", ""is_mobile"": false}" 21515,8,483,2017-06-11 22:08:13,http://keler.biz/sydney,0.3765745296,171.94.26.6,"{""location"": ""LS"", ""is_mobile"": false}" 21516,8,483,2017-03-27 15:58:16,http://hermanswaniawski.com/aaliyah,0.4955941811,32.168.126.39,"{""location"": ""CN"", ""is_mobile"": false}" 21517,8,483,2017-05-04 13:41:14,http://reynolds.biz/alena,0.5016938180,150.248.76.163,"{""location"": ""PF"", ""is_mobile"": true}" 21518,8,483,2017-04-07 12:56:02,http://parisian.info/gina.oreilly,0.1779087053,199.196.122.217,"{""location"": ""NE"", ""is_mobile"": true}" 21519,8,483,2017-01-09 00:03:03,http://goldner.net/lazaro_feil,0.8788739966,2.107.252.207,"{""location"": ""TF"", ""is_mobile"": true}" 21520,8,483,2017-03-10 16:05:17,http://walsh.com/berneice,0.0699266115,252.94.82.151,"{""location"": ""PW"", ""is_mobile"": false}" 21521,8,483,2017-05-26 11:16:40,http://volkman.info/tiana,0.6661051746,130.110.13.218,"{""location"": ""IO"", ""is_mobile"": true}" 21522,8,483,2017-02-25 18:13:27,http://crona.com/johnnie.nolan,0.1481655402,16.68.145.97,"{""location"": ""DK"", ""is_mobile"": false}" 21523,8,483,2017-03-05 09:35:43,http://spinkaheller.co/lenora_gibson,0.0180225109,194.190.28.160,"{""location"": ""BS"", ""is_mobile"": false}" 21524,8,483,2017-04-16 14:02:06,http://gerlach.io/amari,0.3516609888,119.79.183.138,"{""location"": ""GN"", ""is_mobile"": false}" 21525,8,483,2017-06-06 20:46:01,http://little.com/eliezer,0.2874764068,213.27.189.243,"{""location"": ""QA"", ""is_mobile"": false}" 21526,8,483,2017-03-06 07:09:22,http://stehrzboncak.net/denis,0.0676005767,58.38.35.70,"{""location"": ""WS"", ""is_mobile"": false}" 21527,8,483,2017-03-16 04:44:15,http://halvorsonhuels.com/roel,0.8203649640,43.144.254.189,"{""location"": ""PR"", ""is_mobile"": true}" 21528,8,483,2017-03-28 09:57:01,http://zboncak.info/maverick,0.5161826924,184.47.97.86,"{""location"": ""BG"", ""is_mobile"": false}" 21529,8,483,2017-06-12 09:40:37,http://howell.io/lea,0.2005356822,14.177.134.42,"{""location"": ""NE"", ""is_mobile"": true}" 21530,8,483,2017-06-03 06:56:08,http://connelly.net/odell_hills,0.0896680381,126.35.17.150,"{""location"": ""SE"", ""is_mobile"": false}" 21531,8,483,2017-01-07 03:23:33,http://walter.org/leora,0.7944767684,83.169.207.2,"{""location"": ""JM"", ""is_mobile"": true}" 21532,8,483,2017-01-03 02:38:48,http://bernier.co/jeyca.dare,0.1029929261,30.85.116.103,"{""location"": ""PM"", ""is_mobile"": false}" 21533,8,483,2016-12-28 17:50:04,http://lynch.io/vernon,0.2413923266,248.77.35.70,"{""location"": ""UZ"", ""is_mobile"": true}" 21534,8,483,2017-02-07 06:49:39,http://marvinkunde.io/judd,0.8286400413,245.152.44.251,"{""location"": ""PT"", ""is_mobile"": false}" 427,1,11,2017-01-25 04:46:47,http://halvorson.co/lavonne_feest,,160.130.186.212,"{""location"": ""NR"", ""is_mobile"": false}" 428,1,11,2017-02-21 20:17:34,http://feest.info/alycia.schuster,,236.31.219.124,"{""location"": ""BO"", ""is_mobile"": false}" 429,1,11,2017-06-02 22:10:17,http://medhurst.net/abagail.gerhold,,65.99.122.171,"{""location"": ""FK"", ""is_mobile"": false}" 430,1,11,2017-05-31 06:32:47,http://reilly.name/floyd,,162.220.218.200,"{""location"": ""NI"", ""is_mobile"": true}" 431,1,11,2017-03-27 14:59:54,http://swift.co/magdalena_cronin,,82.201.250.32,"{""location"": ""TW"", ""is_mobile"": true}" 432,1,11,2016-12-29 07:49:14,http://morar.com/hal.towne,,32.189.130.187,"{""location"": ""KG"", ""is_mobile"": false}" 433,1,11,2017-04-24 16:16:11,http://runolfon.io/theresa,,93.129.128.39,"{""location"": ""CN"", ""is_mobile"": true}" 434,1,11,2017-01-05 10:14:34,http://heidenreich.com/vanea,,163.154.107.214,"{""location"": ""TJ"", ""is_mobile"": false}" 435,1,11,2017-05-27 16:07:12,http://cruickshankglover.info/arvilla.schaden,,249.226.33.211,"{""location"": ""SZ"", ""is_mobile"": false}" 436,1,11,2017-01-04 03:33:43,http://wuckertkling.name/sally,,220.114.174.192,"{""location"": ""BB"", ""is_mobile"": true}" 437,1,11,2017-01-11 04:32:49,http://gottlieb.net/della_willms,,124.64.197.182,"{""location"": ""QA"", ""is_mobile"": false}" 438,1,11,2017-04-30 08:27:57,http://roberts.com/micaela_cronin,,240.235.251.182,"{""location"": ""AR"", ""is_mobile"": false}" 440,1,11,2017-03-17 08:22:40,http://ritchie.name/cornell,,178.106.71.42,"{""location"": ""PE"", ""is_mobile"": false}" 441,1,11,2017-02-22 05:18:20,http://larkinebert.name/elian,,237.248.211.187,"{""location"": ""UA"", ""is_mobile"": false}" 442,1,11,2016-12-25 10:22:31,http://reinger.name/geoffrey.ankunding,,45.99.124.85,"{""location"": ""CZ"", ""is_mobile"": false}" 443,1,11,2017-02-03 14:18:10,http://carroll.org/aleia,,76.84.230.187,"{""location"": ""FI"", ""is_mobile"": false}" 444,1,11,2017-04-26 23:27:40,http://priceerdman.net/camille_dach,,190.240.176.67,"{""location"": ""TD"", ""is_mobile"": false}" 445,1,11,2017-06-05 21:00:50,http://carter.info/zander.raynor,,49.207.60.124,"{""location"": ""NC"", ""is_mobile"": true}" 446,1,11,2017-06-01 02:11:02,http://lueilwitzbernier.net/berta.welch,,222.200.162.129,"{""location"": ""AD"", ""is_mobile"": true}" 447,1,11,2017-02-09 01:14:29,http://trantow.com/danial,,226.104.35.83,"{""location"": ""TC"", ""is_mobile"": true}" 448,1,11,2017-03-19 07:49:40,http://brown.co/braeden.robel,,243.239.49.214,"{""location"": ""BR"", ""is_mobile"": true}" 449,1,11,2017-01-25 04:03:56,http://shields.co/dandre.powlowski,,206.94.236.158,"{""location"": ""BQ"", ""is_mobile"": true}" 450,1,11,2017-02-19 04:38:59,http://hettinger.com/angel,,202.117.45.38,"{""location"": ""TZ"", ""is_mobile"": true}" 451,1,11,2017-03-28 17:51:23,http://predovic.com/jeffry,,46.172.72.131,"{""location"": ""VE"", ""is_mobile"": true}" 452,1,11,2017-02-22 09:04:40,http://jonesgulgowski.org/soledad_dickinson,,87.194.5.119,"{""location"": ""GE"", ""is_mobile"": false}" 453,1,11,2017-06-05 19:59:50,http://romaguerahaley.co/fabiola_thompson,,206.24.113.74,"{""location"": ""SR"", ""is_mobile"": false}" 454,1,11,2017-01-20 23:11:51,http://gulgowski.com/lincoln.simonis,,158.148.19.68,"{""location"": ""AU"", ""is_mobile"": false}" 455,1,11,2017-05-11 11:56:54,http://gottlieb.co/jett.willms,,194.157.188.209,"{""location"": ""VN"", ""is_mobile"": false}" 456,1,11,2017-03-27 10:59:22,http://krajcik.com/daniella.altenwerth,,76.108.148.112,"{""location"": ""NE"", ""is_mobile"": false}" 457,1,11,2017-01-10 07:21:49,http://hintzjaskolski.io/kevin,,234.67.164.82,"{""location"": ""CX"", ""is_mobile"": true}" 458,1,11,2017-03-20 13:36:46,http://schoenzieme.com/roselyn,,72.211.4.71,"{""location"": ""TO"", ""is_mobile"": false}" 459,1,11,2017-01-07 20:15:50,http://sanford.co/mattie_kiehn,,74.119.182.242,"{""location"": ""SM"", ""is_mobile"": false}" 460,1,11,2017-05-27 14:14:42,http://goyette.com/naomie,,50.229.211.128,"{""location"": ""CK"", ""is_mobile"": false}" 461,1,11,2017-05-20 07:23:32,http://hintz.net/orlando,,121.19.115.28,"{""location"": ""GT"", ""is_mobile"": true}" 462,1,11,2017-05-13 06:33:17,http://murrayparisian.co/keshaun,,155.52.68.217,"{""location"": ""NU"", ""is_mobile"": false}" 463,1,11,2017-01-12 14:30:21,http://schmeler.org/ania_purdy,,65.183.102.88,"{""location"": ""LS"", ""is_mobile"": false}" 464,1,11,2017-06-06 14:32:33,http://zulauf.co/kaleigh,,36.104.237.195,"{""location"": ""CY"", ""is_mobile"": true}" 465,1,11,2017-02-06 07:26:46,http://rodriguez.biz/andreane,,235.157.40.79,"{""location"": ""BA"", ""is_mobile"": false}" 466,1,11,2017-02-21 08:05:58,http://schulist.biz/victor,,99.6.217.197,"{""location"": ""PR"", ""is_mobile"": false}" 467,1,12,2017-03-13 15:21:17,http://pollich.biz/jasen,,76.161.212.173,"{""location"": ""DJ"", ""is_mobile"": true}" 468,1,12,2017-02-19 14:23:31,http://bayererdman.io/carol_leannon,,146.115.120.184,"{""location"": ""GU"", ""is_mobile"": true}" 469,1,12,2017-01-16 09:37:44,http://hettinger.org/leonie,,123.169.110.75,"{""location"": ""TH"", ""is_mobile"": true}" 470,1,12,2017-02-10 06:13:02,http://wunsch.org/savanah_purdy,,221.7.250.37,"{""location"": ""MP"", ""is_mobile"": true}" 471,1,12,2017-05-16 21:09:36,http://tromp.biz/sophie.pfeffer,,182.87.99.224,"{""location"": ""AS"", ""is_mobile"": false}" 472,1,12,2017-05-18 00:11:36,http://rice.net/elmira_gorczany,,220.209.214.124,"{""location"": ""LR"", ""is_mobile"": false}" 473,1,12,2017-05-26 09:32:47,http://larkin.net/ludwig_lesch,,105.109.93.203,"{""location"": ""PS"", ""is_mobile"": true}" 474,1,12,2016-12-17 11:07:55,http://muller.org/susanna,,15.126.71.232,"{""location"": ""BE"", ""is_mobile"": true}" 475,1,12,2017-04-29 16:58:49,http://hermiston.com/brent.haag,,103.73.204.105,"{""location"": ""MY"", ""is_mobile"": false}" 476,1,12,2017-04-01 05:29:13,http://jast.biz/alexzander,,212.200.19.29,"{""location"": ""AR"", ""is_mobile"": true}" 477,1,12,2016-12-29 20:52:54,http://medhurst.net/otto.gorczany,,234.81.30.230,"{""location"": ""ME"", ""is_mobile"": true}" 478,1,12,2017-05-10 15:16:50,http://prosacco.com/sean,,132.252.152.28,"{""location"": ""TK"", ""is_mobile"": true}" 479,1,12,2017-04-05 15:06:12,http://mueller.com/skylar,,171.186.66.159,"{""location"": ""WS"", ""is_mobile"": true}" 480,1,12,2016-12-22 06:03:27,http://towneko.org/godfrey,,147.116.215.112,"{""location"": ""CD"", ""is_mobile"": true}" 481,1,12,2017-02-03 20:43:43,http://gradyweinat.biz/juanita_ernser,,172.103.23.125,"{""location"": ""DM"", ""is_mobile"": true}" 482,1,12,2017-04-22 12:23:00,http://voneichmann.net/chloe,,212.122.53.48,"{""location"": ""MY"", ""is_mobile"": true}" 483,1,12,2017-05-15 18:03:14,http://damore.net/hershel.kunze,,181.68.238.82,"{""location"": ""CY"", ""is_mobile"": false}" 21535,8,483,2017-02-17 11:28:26,http://mohr.com/maci,0.3094113115,46.57.234.58,"{""location"": ""AZ"", ""is_mobile"": false}" 21536,8,483,2017-04-09 00:44:53,http://rowe.net/matilde,0.3833575420,57.61.227.222,"{""location"": ""TM"", ""is_mobile"": true}" 21537,8,483,2017-01-25 13:42:07,http://waelchi.com/henderson.fahey,0.2663939183,231.126.110.64,"{""location"": ""QA"", ""is_mobile"": false}" 21538,8,483,2016-12-17 23:59:13,http://shields.net/timmothy_cronin,0.0878990993,97.61.37.186,"{""location"": ""MX"", ""is_mobile"": false}" 21539,8,483,2017-05-08 18:56:41,http://schneiderjenkins.biz/sigurd_kuphal,0.1059749590,53.80.206.225,"{""location"": ""DM"", ""is_mobile"": false}" 21540,8,483,2017-05-19 09:09:17,http://mohr.biz/dulce,0.0815258681,131.123.28.145,"{""location"": ""NC"", ""is_mobile"": false}" 21541,8,483,2017-05-12 17:37:32,http://littledonnelly.info/estelle.hermann,0.6343283852,69.44.128.112,"{""location"": ""KN"", ""is_mobile"": true}" 21542,8,483,2017-01-18 17:25:29,http://bergnaumlemke.io/aurelie,0.5026484950,116.124.197.81,"{""location"": ""HK"", ""is_mobile"": false}" 21543,8,483,2017-03-03 14:03:17,http://bogan.io/eunice_corkery,0.4796485543,2.116.204.19,"{""location"": ""EC"", ""is_mobile"": false}" 21544,8,483,2017-04-13 01:32:52,http://muller.co/colt_schmidt,0.7390592173,22.106.23.42,"{""location"": ""MD"", ""is_mobile"": true}" 21545,8,483,2017-01-02 09:08:08,http://lockman.com/blair,0.3209146559,7.229.142.195,"{""location"": ""LS"", ""is_mobile"": true}" 21546,8,483,2017-03-17 01:43:14,http://gibson.com/estel_ullrich,0.6509608651,56.169.13.116,"{""location"": ""CL"", ""is_mobile"": true}" 21547,8,483,2017-01-31 15:00:02,http://upton.org/ford,0.2963979536,198.137.173.251,"{""location"": ""NL"", ""is_mobile"": true}" 21548,8,483,2017-03-30 19:18:22,http://vandervorthuels.biz/pierce_ruecker,0.4869329224,210.213.158.66,"{""location"": ""GM"", ""is_mobile"": true}" 21549,8,483,2017-06-07 16:53:40,http://gaylord.com/chaz,0.1573323542,97.192.165.188,"{""location"": ""SY"", ""is_mobile"": true}" 21550,8,483,2017-02-09 12:22:30,http://mooreankunding.io/katelynn.schuster,0.5803675401,244.181.17.231,"{""location"": ""LB"", ""is_mobile"": true}" 21551,8,483,2017-02-05 01:50:16,http://kingfisher.com/jerald.ernser,0.2264575109,216.163.212.78,"{""location"": ""HU"", ""is_mobile"": true}" 21552,8,483,2017-03-28 14:31:04,http://mertz.info/liam_nikolaus,0.9501513137,14.248.16.134,"{""location"": ""MY"", ""is_mobile"": true}" 21553,8,483,2017-01-27 14:03:46,http://crist.co/doug,0.6245208895,85.224.144.91,"{""location"": ""PR"", ""is_mobile"": false}" 21554,8,483,2017-06-09 04:57:46,http://nicolas.org/ena,0.8245218897,74.125.33.224,"{""location"": ""TV"", ""is_mobile"": false}" 21555,8,483,2017-03-20 05:58:54,http://turcottebernhard.name/velva,0.0176142334,11.139.79.174,"{""location"": ""HN"", ""is_mobile"": false}" 21556,8,483,2017-03-13 02:52:33,http://adams.io/damion,0.9458715898,225.51.96.53,"{""location"": ""AT"", ""is_mobile"": false}" 21557,8,483,2017-03-08 19:37:14,http://connelly.co/davin.swift,0.8194350362,15.177.53.90,"{""location"": ""CM"", ""is_mobile"": false}" 21558,8,483,2017-03-03 05:34:11,http://gleichner.org/tillman.deckow,0.9939032162,201.60.111.171,"{""location"": ""TM"", ""is_mobile"": false}" 21559,8,483,2017-05-26 12:47:09,http://waelchismitham.biz/zelma_beier,0.2913414492,100.67.35.135,"{""location"": ""SR"", ""is_mobile"": false}" 21560,8,483,2017-05-25 10:10:55,http://huels.biz/clemens,0.4868187719,79.6.122.204,"{""location"": ""ES"", ""is_mobile"": true}" 21561,8,483,2017-06-01 10:30:35,http://runte.com/merlin.yundt,0.4837693222,110.132.132.99,"{""location"": ""GG"", ""is_mobile"": true}" 21562,8,483,2017-05-05 01:58:28,http://franecki.com/felton,0.3558535016,86.210.254.115,"{""location"": ""GG"", ""is_mobile"": true}" 21563,8,483,2016-12-30 08:40:51,http://starkgreenholt.biz/kiara_connelly,0.7149379372,97.63.168.166,"{""location"": ""GS"", ""is_mobile"": true}" 21564,8,483,2017-01-14 21:35:05,http://schroeder.com/aleia.schaden,0.0954321972,217.163.112.236,"{""location"": ""FM"", ""is_mobile"": false}" 21565,8,483,2017-04-29 15:26:38,http://hirthe.co/nico,0.0882193261,207.180.195.146,"{""location"": ""MU"", ""is_mobile"": true}" 21566,8,483,2017-06-12 16:14:34,http://mitchell.info/emilio,0.3811874693,57.190.204.217,"{""location"": ""LT"", ""is_mobile"": false}" 21567,8,483,2017-01-15 02:00:02,http://adams.biz/jude,0.1078667972,81.13.168.109,"{""location"": ""VC"", ""is_mobile"": true}" 21568,8,484,2017-02-15 09:59:44,http://west.net/deja,0.0188688845,104.188.185.43,"{""location"": ""AF"", ""is_mobile"": true}" 21569,8,484,2017-01-26 16:38:12,http://naderhuels.org/salma,0.8667917530,196.106.185.93,"{""location"": ""NC"", ""is_mobile"": true}" 21570,8,484,2017-04-13 11:38:51,http://yost.info/mathias,0.5805389507,194.223.148.51,"{""location"": ""KR"", ""is_mobile"": true}" 21571,8,484,2017-03-12 08:22:39,http://funktrantow.info/river_thompson,0.6420174041,227.15.6.103,"{""location"": ""SS"", ""is_mobile"": true}" 21572,8,484,2017-06-05 01:51:47,http://trompdibbert.co/anastasia,0.5762105746,76.186.181.159,"{""location"": ""LK"", ""is_mobile"": true}" 21573,8,484,2017-03-15 00:03:41,http://lebsack.net/santino.jast,0.6704369060,117.59.188.84,"{""location"": ""AR"", ""is_mobile"": false}" 21574,8,484,2016-12-31 01:39:58,http://wehnerlebsack.com/alison_gleason,0.8877749046,64.204.178.136,"{""location"": ""CM"", ""is_mobile"": false}" 21575,8,484,2017-05-19 04:24:51,http://jastschuster.info/ricky.miller,0.0282905050,87.9.171.159,"{""location"": ""NE"", ""is_mobile"": false}" 21576,8,484,2017-06-01 09:06:00,http://morar.info/annabelle,0.5676427342,39.37.223.28,"{""location"": ""SJ"", ""is_mobile"": false}" 21577,8,484,2017-03-14 09:44:28,http://feest.name/zella,0.9756496154,241.205.39.163,"{""location"": ""PF"", ""is_mobile"": true}" 21578,8,484,2017-02-27 04:18:56,http://hand.biz/daron,0.3420753223,160.195.72.172,"{""location"": ""GB"", ""is_mobile"": true}" 21579,8,484,2017-01-17 11:37:07,http://christiansen.info/santa_cummings,0.7848428843,154.241.167.151,"{""location"": ""NF"", ""is_mobile"": false}" 21580,8,484,2017-01-30 18:58:03,http://bartell.name/cale,0.1974509826,202.238.129.187,"{""location"": ""AF"", ""is_mobile"": false}" 21581,8,484,2017-03-01 22:48:25,http://ankundingstamm.name/madison_batz,0.0665106420,248.137.87.103,"{""location"": ""CN"", ""is_mobile"": false}" 21582,8,484,2017-03-25 00:18:01,http://bruen.name/lysanne.king,0.8685346822,5.47.229.224,"{""location"": ""EG"", ""is_mobile"": false}" 21583,8,484,2017-04-18 19:28:47,http://dickinsonvolkman.io/ardith,0.9301972329,202.81.68.250,"{""location"": ""BT"", ""is_mobile"": true}" 21584,8,484,2016-12-15 16:33:09,http://prosacco.info/te_dickinson,0.4374744069,196.55.79.221,"{""location"": ""CO"", ""is_mobile"": false}" 21585,8,484,2017-01-22 23:39:52,http://mertz.biz/dasia.nitzsche,0.3650212279,4.74.124.44,"{""location"": ""TH"", ""is_mobile"": true}" 484,1,12,2016-12-14 15:01:06,http://stark.name/drew_senger,,157.2.253.233,"{""location"": ""SI"", ""is_mobile"": true}" 485,1,12,2017-02-09 19:14:12,http://swaniawski.biz/penelope.metz,,128.154.193.94,"{""location"": ""GT"", ""is_mobile"": true}" 486,1,12,2017-05-29 01:14:44,http://funk.name/casandra.dickens,,237.57.241.226,"{""location"": ""VU"", ""is_mobile"": false}" 487,1,12,2017-02-16 04:01:42,http://strackerogahn.io/alec,,239.196.63.184,"{""location"": ""GG"", ""is_mobile"": true}" 488,1,12,2017-05-14 21:11:17,http://monahan.info/emery_batz,,65.236.231.138,"{""location"": ""PS"", ""is_mobile"": true}" 489,1,12,2017-02-20 11:36:15,http://west.name/destiny,,117.83.166.38,"{""location"": ""TN"", ""is_mobile"": true}" 490,1,12,2017-06-04 23:05:33,http://raynor.co/arch_keeling,,91.245.107.189,"{""location"": ""CR"", ""is_mobile"": true}" 491,1,12,2016-12-29 06:40:13,http://block.info/kaylin.nolan,,163.244.205.143,"{""location"": ""ZA"", ""is_mobile"": true}" 492,1,12,2016-12-27 09:36:00,http://kihn.name/alexandro,,43.157.137.143,"{""location"": ""BG"", ""is_mobile"": true}" 493,1,12,2017-05-31 16:18:13,http://cole.name/greg,,252.214.116.251,"{""location"": ""TC"", ""is_mobile"": false}" 494,1,12,2017-05-16 03:12:36,http://parisian.biz/reginald,,89.249.168.95,"{""location"": ""HU"", ""is_mobile"": true}" 495,1,12,2017-01-15 18:29:18,http://kulaspacocha.net/rosalyn.ko,,72.252.86.197,"{""location"": ""CF"", ""is_mobile"": true}" 496,1,12,2017-03-05 08:51:54,http://reillyeichmann.biz/will,,217.216.188.161,"{""location"": ""AE"", ""is_mobile"": false}" 497,1,12,2017-01-15 16:19:39,http://abbott.io/columbus,,233.148.42.6,"{""location"": ""JM"", ""is_mobile"": true}" 498,1,12,2017-06-07 08:57:07,http://hand.biz/luigi_dare,,117.187.7.13,"{""location"": ""CZ"", ""is_mobile"": true}" 499,1,12,2016-12-22 16:38:31,http://tillman.co/valerie.jacobson,,80.133.105.45,"{""location"": ""KP"", ""is_mobile"": true}" 500,1,12,2017-01-12 18:14:32,http://mcclure.biz/therese.greenfelder,,4.169.197.203,"{""location"": ""TZ"", ""is_mobile"": false}" 501,1,12,2017-03-08 21:53:57,http://zboncakhowell.biz/deron_erdman,,165.174.110.138,"{""location"": ""GU"", ""is_mobile"": true}" 502,1,12,2017-02-07 13:57:39,http://kuhlman.io/nicolas,,14.186.159.254,"{""location"": ""SG"", ""is_mobile"": false}" 503,1,12,2017-02-28 23:26:17,http://lynchaufderhar.com/jayden,,31.9.121.66,"{""location"": ""GG"", ""is_mobile"": true}" 504,1,12,2017-05-02 14:41:36,http://maggio.biz/salvador,,135.55.200.21,"{""location"": ""KW"", ""is_mobile"": true}" 505,1,12,2017-01-17 04:51:03,http://ledner.net/leo,,219.145.104.94,"{""location"": ""UZ"", ""is_mobile"": false}" 506,1,12,2017-01-23 20:51:51,http://kunde.com/frida.herman,,150.105.187.40,"{""location"": ""DZ"", ""is_mobile"": false}" 507,1,12,2017-02-18 12:37:47,http://kihnpagac.info/michelle,,49.246.185.62,"{""location"": ""NL"", ""is_mobile"": true}" 508,1,12,2017-01-19 07:11:26,http://jacobi.co/jorge.walter,,34.131.97.72,"{""location"": ""IT"", ""is_mobile"": true}" 509,1,12,2017-03-27 00:44:48,http://predovicboyer.info/deanna.lueilwitz,,39.250.150.14,"{""location"": ""VA"", ""is_mobile"": true}" 510,1,12,2016-12-17 18:30:41,http://hermannpurdy.co/deonte_schneider,,90.94.179.17,"{""location"": ""CZ"", ""is_mobile"": true}" 511,1,12,2017-01-27 17:59:33,http://mayertveum.net/joel,,180.226.152.59,"{""location"": ""KH"", ""is_mobile"": true}" 512,1,12,2017-03-28 05:36:38,http://breitenberg.name/sonny_toy,,75.111.71.42,"{""location"": ""PL"", ""is_mobile"": false}" 513,1,13,2017-04-08 11:07:46,http://bogisichupton.io/jalyn_becker,,223.92.113.163,"{""location"": ""BD"", ""is_mobile"": false}" 514,1,13,2016-12-20 20:37:05,http://beerharvey.net/otho.green,,136.131.56.188,"{""location"": ""PN"", ""is_mobile"": false}" 515,1,13,2017-01-05 05:46:08,http://reynolds.net/kirk_roberts,,235.231.120.52,"{""location"": ""MZ"", ""is_mobile"": true}" 516,1,13,2017-01-05 06:17:48,http://uptonschamberger.net/ines_tromp,,72.246.34.173,"{""location"": ""TR"", ""is_mobile"": false}" 517,1,13,2017-05-02 02:01:27,http://barrows.info/meggie.cummings,,248.127.144.103,"{""location"": ""OM"", ""is_mobile"": false}" 518,1,13,2017-05-11 09:33:27,http://davisrowe.biz/myah.okon,,14.221.34.85,"{""location"": ""FK"", ""is_mobile"": false}" 519,1,13,2017-03-26 07:54:06,http://schambergerhilll.biz/anabel_wuckert,,155.2.138.41,"{""location"": ""PS"", ""is_mobile"": true}" 520,1,13,2017-05-02 22:28:42,http://ohara.name/thalia,,34.215.94.233,"{""location"": ""TD"", ""is_mobile"": true}" 521,1,13,2017-01-28 13:21:19,http://kerluke.com/khalid,,196.226.101.185,"{""location"": ""DO"", ""is_mobile"": true}" 522,1,13,2017-03-30 00:23:15,http://eichmanncartwright.name/claud.wunsch,,114.3.165.140,"{""location"": ""NR"", ""is_mobile"": false}" 523,1,13,2017-02-19 20:59:55,http://ziemebode.org/pierce_schoen,,19.240.114.57,"{""location"": ""MD"", ""is_mobile"": false}" 524,1,13,2017-01-14 09:31:29,http://lueilwitz.org/kenny,,40.218.151.83,"{""location"": ""VI"", ""is_mobile"": true}" 525,1,13,2016-12-13 15:02:44,http://willfadel.com/elyse,,49.145.31.127,"{""location"": ""CO"", ""is_mobile"": false}" 526,1,13,2017-05-27 15:39:29,http://moriettekautzer.biz/sister,,2.170.154.231,"{""location"": ""TG"", ""is_mobile"": true}" 527,1,13,2017-05-17 01:02:51,http://lubowitz.com/cleta.raynor,,63.116.202.101,"{""location"": ""NO"", ""is_mobile"": true}" 528,1,13,2017-02-19 03:46:51,http://gislasonhand.co/twila,,64.144.116.213,"{""location"": ""TK"", ""is_mobile"": true}" 529,1,13,2017-03-21 02:42:17,http://barrows.biz/chelsea,,241.119.225.38,"{""location"": ""CO"", ""is_mobile"": true}" 530,1,13,2016-12-16 03:26:15,http://fisherfunk.net/pansy,,176.140.76.181,"{""location"": ""KW"", ""is_mobile"": false}" 531,1,13,2017-04-24 08:19:32,http://littel.io/asa,,111.117.12.99,"{""location"": ""SS"", ""is_mobile"": true}" 532,1,13,2017-03-05 04:56:51,http://mcclure.name/kaleb,,130.203.254.128,"{""location"": ""HT"", ""is_mobile"": true}" 533,1,13,2017-01-25 05:45:23,http://kunde.net/quincy_collier,,163.134.206.253,"{""location"": ""NU"", ""is_mobile"": false}" 534,1,13,2017-02-21 03:17:24,http://gradywiza.co/heath,,223.233.177.73,"{""location"": ""MC"", ""is_mobile"": true}" 535,1,13,2017-02-02 18:49:14,http://heaney.org/alene,,203.158.111.152,"{""location"": ""AE"", ""is_mobile"": false}" 536,1,13,2017-05-22 23:23:53,http://boehmbode.com/raegan,,102.249.20.91,"{""location"": ""TD"", ""is_mobile"": false}" 537,1,13,2017-06-12 22:39:56,http://bartolettibotsford.com/wilhelmine_mante,,160.141.210.240,"{""location"": ""GH"", ""is_mobile"": false}" 538,1,13,2017-04-17 00:16:07,http://hane.info/darrell.doyle,,165.219.39.78,"{""location"": ""DM"", ""is_mobile"": false}" 539,1,13,2017-01-10 05:23:05,http://parisian.co/bert_tremblay,,21.58.253.96,"{""location"": ""CM"", ""is_mobile"": false}" 540,1,13,2017-04-10 08:15:00,http://harris.co/reina_ward,,10.86.153.198,"{""location"": ""HT"", ""is_mobile"": true}" 21586,8,484,2017-02-20 12:06:59,http://white.io/gay.lockman,0.2734629959,171.169.219.110,"{""location"": ""ZW"", ""is_mobile"": false}" 21587,8,484,2017-03-06 11:04:52,http://white.io/jerry,0.0779409816,189.227.205.225,"{""location"": ""PG"", ""is_mobile"": false}" 21588,8,484,2017-03-17 16:17:37,http://durgan.name/maybell,0.7921279125,239.178.146.73,"{""location"": ""YE"", ""is_mobile"": false}" 21589,8,484,2017-03-19 04:13:24,http://pfeffer.info/lazaro_koch,0.0467860414,19.251.168.102,"{""location"": ""UM"", ""is_mobile"": false}" 21590,8,484,2017-01-05 00:39:41,http://swift.io/richard,0.7760720083,246.176.65.79,"{""location"": ""TO"", ""is_mobile"": false}" 21591,8,484,2017-03-13 05:36:59,http://predovic.info/antonio,0.5004459570,22.60.44.24,"{""location"": ""MK"", ""is_mobile"": false}" 21592,8,484,2017-03-01 03:57:25,http://keeling.com/laurianne.gibson,0.0362908190,142.2.105.194,"{""location"": ""NU"", ""is_mobile"": false}" 21593,8,484,2017-03-21 06:42:02,http://mante.co/angelo,0.7315154722,101.231.122.42,"{""location"": ""NF"", ""is_mobile"": false}" 21594,8,484,2017-06-02 03:29:00,http://schowalter.info/estelle.bayer,0.6463994151,233.173.182.123,"{""location"": ""PK"", ""is_mobile"": false}" 21595,8,484,2017-03-13 14:23:59,http://waelchi.name/alivia.lueilwitz,0.4544171579,47.148.138.207,"{""location"": ""IQ"", ""is_mobile"": false}" 21596,8,484,2017-02-07 05:46:38,http://schowalter.co/chloe.monahan,0.6220413188,122.25.244.192,"{""location"": ""IQ"", ""is_mobile"": false}" 21597,8,484,2017-03-05 10:26:09,http://haleylind.info/marisol_macejkovic,0.5646170562,239.128.232.101,"{""location"": ""MP"", ""is_mobile"": false}" 21598,8,484,2016-12-18 22:16:16,http://tillman.org/amara,0.9690606315,23.45.137.11,"{""location"": ""ST"", ""is_mobile"": true}" 21599,8,484,2017-04-13 18:49:14,http://hermann.info/johan.torp,0.0483498652,98.135.40.197,"{""location"": ""TH"", ""is_mobile"": false}" 21600,8,484,2017-04-20 08:25:27,http://prohaska.net/stephania,0.5975067518,116.83.106.83,"{""location"": ""PK"", ""is_mobile"": true}" 21601,8,484,2017-04-23 22:29:52,http://stehrcasper.co/natasha_baumbach,0.0716983431,68.134.239.18,"{""location"": ""LB"", ""is_mobile"": false}" 21602,8,484,2016-12-14 16:58:36,http://kovacek.io/dan,0.1845611253,31.207.110.12,"{""location"": ""LB"", ""is_mobile"": true}" 21603,8,484,2017-02-25 06:10:37,http://weimann.biz/virgil,0.8175192608,130.155.127.58,"{""location"": ""AR"", ""is_mobile"": false}" 21604,8,484,2016-12-16 02:16:46,http://harris.info/magdalena,0.6847217847,51.38.220.193,"{""location"": ""TJ"", ""is_mobile"": false}" 21605,8,484,2017-03-24 16:29:28,http://sawaynwolf.com/jewell,0.9826445921,120.234.242.222,"{""location"": ""UA"", ""is_mobile"": false}" 21606,8,484,2016-12-18 16:09:18,http://mosciski.org/theodora_pagac,0.4696616243,42.108.204.214,"{""location"": ""DJ"", ""is_mobile"": true}" 21607,8,484,2017-06-06 22:21:15,http://mueller.name/loyal,0.4831435599,234.17.57.162,"{""location"": ""BN"", ""is_mobile"": true}" 21608,8,484,2017-04-15 13:12:54,http://mosciski.com/oda.lubowitz,0.5449272392,13.90.224.37,"{""location"": ""HM"", ""is_mobile"": false}" 21609,8,484,2017-02-12 14:29:32,http://mann.biz/joanne_johns,0.0019654533,94.233.136.165,"{""location"": ""CC"", ""is_mobile"": false}" 21610,8,484,2017-03-18 19:24:48,http://ferry.org/gonzalo,0.5114070107,144.164.59.136,"{""location"": ""TK"", ""is_mobile"": false}" 21611,8,484,2017-03-17 11:03:29,http://boyer.biz/robb,0.7450941602,97.142.108.234,"{""location"": ""PE"", ""is_mobile"": false}" 21612,8,484,2017-04-01 18:05:26,http://nitzsche.info/estefania.lueilwitz,0.2849088718,127.6.163.64,"{""location"": ""ZW"", ""is_mobile"": false}" 21613,8,484,2017-03-26 08:15:01,http://tillman.com/ashton_bashirian,0.3251828854,89.54.182.225,"{""location"": ""GM"", ""is_mobile"": false}" 21614,8,484,2017-01-18 22:02:01,http://hamill.com/ellie_armstrong,0.5517774293,192.60.176.122,"{""location"": ""BA"", ""is_mobile"": false}" 21615,8,484,2016-12-28 05:25:44,http://gusikowski.io/ara.price,0.2914705297,103.49.240.19,"{""location"": ""LU"", ""is_mobile"": false}" 21616,8,484,2017-06-05 23:05:45,http://tillmanolson.biz/marcia_koelpin,0.4992658872,25.126.229.69,"{""location"": ""AE"", ""is_mobile"": false}" 21617,8,484,2017-05-11 23:16:30,http://west.info/michale.hackett,0.8208222337,168.214.174.152,"{""location"": ""US"", ""is_mobile"": false}" 21618,8,484,2017-01-29 21:25:33,http://breitenbergmaggio.info/jeika,0.8046817819,74.68.54.51,"{""location"": ""HR"", ""is_mobile"": true}" 21619,8,484,2017-04-26 22:51:52,http://heaney.info/ayla,0.6185983150,251.151.138.213,"{""location"": ""NZ"", ""is_mobile"": false}" 21620,8,484,2017-06-05 14:01:01,http://bednar.com/bert,0.7613567955,223.71.5.206,"{""location"": ""MW"", ""is_mobile"": false}" 21621,8,484,2017-02-25 12:23:48,http://halvorson.io/rachelle,0.5487564928,23.39.156.198,"{""location"": ""BD"", ""is_mobile"": true}" 21622,8,484,2017-02-14 23:30:10,http://muller.co/abel.johnson,0.2036564900,239.194.174.33,"{""location"": ""IT"", ""is_mobile"": true}" 21623,8,484,2016-12-30 04:15:57,http://ortiz.net/arielle_heaney,0.2305168868,79.181.143.226,"{""location"": ""NZ"", ""is_mobile"": true}" 21624,8,484,2017-04-10 04:37:35,http://schustercrona.biz/shany,0.0110272785,74.122.185.238,"{""location"": ""HT"", ""is_mobile"": true}" 21625,8,484,2017-04-25 08:54:21,http://white.info/lysanne,0.7857482371,42.122.185.2,"{""location"": ""CM"", ""is_mobile"": false}" 21626,8,484,2017-02-09 13:38:24,http://rath.name/barry,0.8395812454,20.252.251.247,"{""location"": ""TD"", ""is_mobile"": false}" 21627,8,484,2017-02-14 09:41:50,http://krisbeier.org/rickie,0.6164655833,86.204.194.241,"{""location"": ""PA"", ""is_mobile"": true}" 21628,8,484,2017-01-13 10:49:22,http://cartwright.net/margarete,0.0375393362,120.119.22.207,"{""location"": ""SX"", ""is_mobile"": false}" 21629,8,484,2017-03-02 12:58:22,http://ledner.co/loren_block,0.3831794422,104.222.84.232,"{""location"": ""BW"", ""is_mobile"": true}" 21630,8,484,2017-03-05 23:22:37,http://haleyrippin.biz/liza,0.4363591519,91.236.55.95,"{""location"": ""JP"", ""is_mobile"": false}" 21631,8,484,2016-12-29 01:57:49,http://okon.net/tania,0.6806868852,12.222.57.91,"{""location"": ""ET"", ""is_mobile"": true}" 21632,8,484,2017-06-06 10:49:26,http://gerlachwatsica.net/maribel_huels,0.5217159783,205.177.182.6,"{""location"": ""GA"", ""is_mobile"": true}" 21633,8,484,2017-04-19 17:57:35,http://kerlukeschuster.io/anabel.gusikowski,0.5252526615,218.99.47.132,"{""location"": ""AT"", ""is_mobile"": false}" 21634,8,484,2017-02-10 22:11:30,http://smith.biz/zena.buckridge,0.5564113947,134.165.23.109,"{""location"": ""FI"", ""is_mobile"": true}" 21635,8,484,2017-06-13 18:33:41,http://feeneyruecker.io/shayne.nikolaus,0.8393092014,147.95.247.189,"{""location"": ""SA"", ""is_mobile"": false}" 21636,8,484,2017-04-20 22:40:55,http://skiles.io/roie_predovic,0.0420090142,156.170.232.167,"{""location"": ""NF"", ""is_mobile"": true}" 541,1,13,2017-04-30 18:41:27,http://veum.io/vidal,,226.6.126.75,"{""location"": ""BV"", ""is_mobile"": false}" 542,1,13,2017-02-05 19:36:36,http://wunsch.com/anais,,167.179.41.159,"{""location"": ""SS"", ""is_mobile"": true}" 543,1,13,2017-02-13 14:11:43,http://littlemarvin.name/lexus,,135.131.4.218,"{""location"": ""PR"", ""is_mobile"": false}" 544,1,13,2017-04-06 17:52:13,http://zulaufhilll.com/rupert_schoen,,169.42.154.193,"{""location"": ""TH"", ""is_mobile"": true}" 545,1,13,2017-03-15 13:20:41,http://dietrichmoriette.com/domenic,,135.8.175.119,"{""location"": ""BT"", ""is_mobile"": false}" 546,1,13,2017-01-10 16:42:34,http://zulaufkuhlman.io/abdul_hickle,,97.15.210.10,"{""location"": ""KH"", ""is_mobile"": true}" 547,1,13,2017-04-30 16:24:31,http://markscrist.info/lea_cormier,,93.250.29.156,"{""location"": ""CF"", ""is_mobile"": false}" 548,1,13,2017-03-01 17:49:47,http://keler.info/sid,,236.94.217.74,"{""location"": ""SZ"", ""is_mobile"": true}" 549,1,13,2017-03-29 06:11:42,http://cruickshankwehner.co/donavon.west,,209.136.120.80,"{""location"": ""SA"", ""is_mobile"": true}" 550,1,13,2017-06-05 20:47:52,http://stanton.name/rigoberto,,183.239.46.159,"{""location"": ""CR"", ""is_mobile"": true}" 551,1,13,2017-02-03 14:07:22,http://roob.info/bertram,,104.236.80.111,"{""location"": ""LU"", ""is_mobile"": true}" 552,1,13,2017-01-15 07:52:38,http://powlowski.org/floy,,171.131.43.196,"{""location"": ""KE"", ""is_mobile"": false}" 553,1,13,2017-02-07 06:11:53,http://windlermoore.info/roselyn,,203.32.168.121,"{""location"": ""TC"", ""is_mobile"": false}" 554,1,13,2017-03-27 04:54:31,http://kelerheidenreich.io/pinkie,,86.101.189.107,"{""location"": ""SM"", ""is_mobile"": false}" 555,1,13,2016-12-19 05:26:24,http://wardlegros.com/cayla,,100.2.206.179,"{""location"": ""SB"", ""is_mobile"": true}" 556,1,13,2017-04-01 02:55:38,http://considinegerlach.name/josie,,92.29.37.200,"{""location"": ""IQ"", ""is_mobile"": true}" 557,1,13,2017-03-17 23:09:47,http://schimmel.net/olga_gleason,,53.194.85.149,"{""location"": ""FJ"", ""is_mobile"": false}" 558,1,13,2017-05-30 12:16:49,http://connelly.io/adolfo,,125.203.157.34,"{""location"": ""SY"", ""is_mobile"": false}" 559,1,13,2017-05-17 00:34:06,http://millchoen.com/yvonne.bahringer,,64.108.234.232,"{""location"": ""ST"", ""is_mobile"": false}" 560,1,13,2017-01-17 06:51:01,http://kemmer.com/santos_rempel,,85.227.50.187,"{""location"": ""KE"", ""is_mobile"": false}" 561,1,13,2017-05-21 03:52:19,http://goldner.com/augusta,,175.113.113.181,"{""location"": ""LR"", ""is_mobile"": true}" 562,1,13,2017-02-07 23:17:34,http://lockman.co/nicole,,65.211.177.182,"{""location"": ""TW"", ""is_mobile"": false}" 563,1,13,2017-03-20 20:17:05,http://kautzer.info/addison,,238.83.136.31,"{""location"": ""AG"", ""is_mobile"": false}" 564,1,13,2016-12-22 18:15:42,http://rippinankunding.co/keshawn,,191.51.172.248,"{""location"": ""ET"", ""is_mobile"": false}" 565,1,13,2017-01-09 19:21:00,http://barrowsdickinson.com/lue.spinka,,215.175.48.161,"{""location"": ""KM"", ""is_mobile"": true}" 566,1,13,2017-03-19 09:26:27,http://schaefer.io/hiram.bergnaum,,236.228.148.223,"{""location"": ""GY"", ""is_mobile"": true}" 567,1,13,2017-04-22 15:18:37,http://zieme.biz/jayson_fay,,169.131.8.159,"{""location"": ""EG"", ""is_mobile"": true}" 568,1,13,2017-05-15 00:03:51,http://pollich.info/deontae,,48.167.63.144,"{""location"": ""GL"", ""is_mobile"": true}" 569,1,13,2017-01-07 11:50:23,http://quitzon.biz/edna,,122.111.204.246,"{""location"": ""SZ"", ""is_mobile"": false}" 570,1,13,2017-05-30 06:45:52,http://okeefe.org/mikayla,,24.119.66.59,"{""location"": ""GR"", ""is_mobile"": false}" 571,1,14,2017-03-09 15:07:33,http://fay.info/miracle_berge,0.4406991709,83.144.75.221,"{""location"": ""NO"", ""is_mobile"": false}" 572,1,14,2017-03-18 15:40:30,http://ruelgoodwin.biz/andrew.doyle,0.6641132977,92.63.41.202,"{""location"": ""SC"", ""is_mobile"": true}" 573,1,14,2017-02-11 12:04:01,http://feeneyhansen.biz/whitney.wunsch,0.3176705880,191.2.191.33,"{""location"": ""FR"", ""is_mobile"": false}" 574,1,14,2017-04-17 01:06:19,http://keler.org/brett,0.7011144680,90.72.166.13,"{""location"": ""MN"", ""is_mobile"": true}" 575,1,14,2017-05-16 09:07:02,http://hodkiewiczblanda.co/fausto,0.4158055199,241.231.252.113,"{""location"": ""PR"", ""is_mobile"": true}" 576,1,14,2017-01-08 15:34:36,http://cruickshank.info/bobbie_funk,0.8136470929,141.169.70.192,"{""location"": ""IM"", ""is_mobile"": false}" 577,1,14,2017-05-16 09:26:56,http://raynor.co/rhiannon.gleichner,0.8325518538,82.88.163.189,"{""location"": ""NE"", ""is_mobile"": false}" 578,1,14,2017-03-29 04:55:10,http://bogisich.name/kirk.raynor,0.6731242729,66.69.28.142,"{""location"": ""GU"", ""is_mobile"": true}" 579,1,14,2017-05-02 01:45:19,http://wisoky.org/marjolaine_denesik,0.5551845092,4.16.78.229,"{""location"": ""CL"", ""is_mobile"": false}" 580,1,14,2017-03-25 07:10:36,http://wuckert.co/marcella,0.8756977898,56.120.69.245,"{""location"": ""PA"", ""is_mobile"": true}" 581,1,14,2017-06-03 09:53:41,http://gerlach.com/jaida,0.8285406228,216.239.249.207,"{""location"": ""NU"", ""is_mobile"": false}" 582,1,14,2017-03-10 22:08:15,http://feest.biz/makayla,0.4697051394,207.135.241.165,"{""location"": ""LR"", ""is_mobile"": false}" 583,1,14,2017-01-31 11:00:42,http://ko.co/jalen.prohaska,0.2959144758,200.56.121.138,"{""location"": ""SO"", ""is_mobile"": true}" 584,1,14,2017-03-04 01:18:02,http://cremin.net/marcelo,0.6709620928,85.227.30.3,"{""location"": ""MR"", ""is_mobile"": false}" 585,1,14,2017-03-06 17:34:56,http://kutch.org/rachel.cummerata,0.0095729652,215.21.216.125,"{""location"": ""JE"", ""is_mobile"": true}" 586,1,14,2017-05-23 22:54:21,http://beckerosinski.info/hailee_fisher,0.4893956152,19.131.170.85,"{""location"": ""TD"", ""is_mobile"": false}" 587,1,14,2016-12-22 00:08:41,http://will.biz/wendy,0.9942129581,112.219.73.105,"{""location"": ""CH"", ""is_mobile"": true}" 588,1,14,2017-03-08 23:56:28,http://armstrong.name/hettie_mcclure,0.9834721037,242.242.180.33,"{""location"": ""AG"", ""is_mobile"": false}" 589,1,14,2017-01-16 14:24:17,http://littel.com/hellen.keebler,0.7415430626,41.182.248.128,"{""location"": ""EH"", ""is_mobile"": false}" 590,1,14,2017-02-13 19:56:12,http://langworth.biz/emmy_kemmer,0.7283673021,250.96.40.44,"{""location"": ""NO"", ""is_mobile"": false}" 591,1,14,2016-12-24 18:41:17,http://kiehn.org/herta,0.6063709788,45.196.145.124,"{""location"": ""CG"", ""is_mobile"": false}" 592,1,14,2017-01-04 13:25:56,http://doyle.org/kacie.king,0.9666986137,58.237.171.200,"{""location"": ""TL"", ""is_mobile"": false}" 593,1,14,2017-06-04 23:12:01,http://smith.biz/evert,0.4362683140,147.212.227.15,"{""location"": ""CD"", ""is_mobile"": false}" 594,1,14,2017-05-17 13:29:23,http://crona.org/onie_mante,0.9682104593,163.173.151.22,"{""location"": ""CL"", ""is_mobile"": true}" 21637,8,484,2017-01-27 14:00:30,http://hettingerlegros.info/vivianne,0.7803642182,81.78.145.202,"{""location"": ""AF"", ""is_mobile"": false}" 21638,8,485,2017-03-29 16:11:13,http://nolankihn.com/kaden_botsford,0.1790925509,235.137.76.4,"{""location"": ""EC"", ""is_mobile"": false}" 21639,8,485,2017-06-10 12:21:14,http://conroy.name/constance_jenkins,0.8762094793,135.195.160.218,"{""location"": ""NR"", ""is_mobile"": true}" 21640,8,485,2017-05-09 22:28:57,http://thiel.name/giovanny,0.4316531289,28.170.58.79,"{""location"": ""TV"", ""is_mobile"": true}" 21641,8,485,2017-02-25 18:41:11,http://ullrich.com/jaron,0.0691293342,6.103.81.39,"{""location"": ""IS"", ""is_mobile"": true}" 21642,8,485,2017-01-21 03:55:10,http://gleichner.info/jovani.buckridge,0.0137420230,78.59.85.158,"{""location"": ""MV"", ""is_mobile"": true}" 21643,8,485,2016-12-26 16:28:19,http://glover.info/cortney.volkman,0.0777112150,222.174.196.237,"{""location"": ""VE"", ""is_mobile"": true}" 21644,8,485,2017-04-09 15:26:12,http://stehr.co/bernhard,0.2242591705,160.149.42.136,"{""location"": ""VG"", ""is_mobile"": false}" 21645,8,485,2017-04-18 13:38:30,http://veumkunze.biz/lucienne,0.8452676347,121.136.76.22,"{""location"": ""LV"", ""is_mobile"": false}" 21646,8,485,2016-12-24 02:14:23,http://weinat.co/bobbie.haag,0.8515977160,210.63.158.187,"{""location"": ""TZ"", ""is_mobile"": false}" 21647,8,485,2017-03-13 07:11:33,http://treutel.org/raymundo_hilll,0.6630023985,217.59.191.230,"{""location"": ""WF"", ""is_mobile"": true}" 21648,8,485,2017-05-01 09:18:46,http://hermanmohr.co/friedrich.trantow,0.6076692544,164.27.34.213,"{""location"": ""BH"", ""is_mobile"": false}" 21649,8,485,2017-04-23 06:13:31,http://mcglynn.com/ulices,0.2639555470,250.24.144.75,"{""location"": ""IM"", ""is_mobile"": false}" 21650,8,485,2017-02-05 17:56:21,http://wehner.org/stephon,0.4962845743,196.213.227.233,"{""location"": ""VE"", ""is_mobile"": true}" 21651,8,485,2016-12-13 08:04:15,http://barton.biz/merritt,0.7308054826,99.97.183.11,"{""location"": ""DZ"", ""is_mobile"": false}" 21652,8,485,2016-12-13 19:58:59,http://kochlangworth.name/elmo,0.8461257829,160.149.68.213,"{""location"": ""AD"", ""is_mobile"": false}" 21653,8,485,2017-01-03 20:08:08,http://lang.com/wayne.corkery,0.2042719624,125.37.190.162,"{""location"": ""LV"", ""is_mobile"": false}" 21654,8,485,2017-05-24 18:50:00,http://flatleywolff.com/florencio.ferry,0.4976165759,14.33.186.83,"{""location"": ""AI"", ""is_mobile"": false}" 21655,8,485,2017-01-25 23:31:56,http://bartoletti.com/hosea,0.8144539890,49.9.217.152,"{""location"": ""UY"", ""is_mobile"": true}" 21656,8,485,2017-05-25 12:27:58,http://veum.io/quinn_treutel,0.5515213132,233.63.37.19,"{""location"": ""LR"", ""is_mobile"": false}" 21657,8,485,2017-01-07 18:52:47,http://harber.com/tania_reilly,0.1521649732,227.44.4.9,"{""location"": ""HK"", ""is_mobile"": false}" 21658,8,485,2017-04-19 09:46:40,http://gutmannlakin.biz/ruell,0.0467074287,27.27.75.130,"{""location"": ""BY"", ""is_mobile"": false}" 21659,8,485,2017-03-31 01:38:54,http://baileyerdman.io/amelie.kiehn,0.1626992429,253.25.107.105,"{""location"": ""SX"", ""is_mobile"": false}" 21660,8,485,2017-05-23 00:44:44,http://kohler.name/martin,0.3047636488,172.126.3.5,"{""location"": ""TK"", ""is_mobile"": false}" 21661,8,485,2017-02-04 07:24:02,http://kutchhickle.net/kayley,0.2550184922,61.163.248.55,"{""location"": ""ML"", ""is_mobile"": false}" 21662,8,485,2017-05-14 06:13:34,http://smith.biz/giles_green,0.7228108334,85.250.88.109,"{""location"": ""DM"", ""is_mobile"": false}" 21663,8,485,2017-02-25 10:46:28,http://ohara.org/torrey_spinka,0.4689887177,107.23.10.100,"{""location"": ""AM"", ""is_mobile"": true}" 21664,8,485,2017-01-31 20:50:06,http://jaskolskischaefer.com/corine,0.0530013791,226.199.2.182,"{""location"": ""DO"", ""is_mobile"": false}" 21665,8,485,2017-02-20 04:36:42,http://moriette.name/blanca.torp,0.9712469809,123.72.133.64,"{""location"": ""PH"", ""is_mobile"": true}" 21666,8,485,2017-03-19 14:32:08,http://conroy.org/jeica.daugherty,0.6042761325,118.222.157.78,"{""location"": ""ML"", ""is_mobile"": true}" 21667,8,485,2017-05-03 12:52:17,http://kunde.biz/linwood,0.7425004077,79.157.119.171,"{""location"": ""RU"", ""is_mobile"": false}" 21668,8,485,2017-02-02 19:29:26,http://feest.info/winnifred_schumm,0.6118465582,195.124.158.201,"{""location"": ""LK"", ""is_mobile"": false}" 21669,8,485,2017-01-31 00:42:44,http://douglas.io/alysa,0.4281151116,210.148.42.171,"{""location"": ""BR"", ""is_mobile"": true}" 21670,8,485,2016-12-31 12:11:10,http://feest.net/dawn_runte,0.1168304491,93.117.69.182,"{""location"": ""CM"", ""is_mobile"": true}" 21671,8,485,2017-05-14 05:22:50,http://block.biz/durward_koelpin,0.4183887169,160.247.117.233,"{""location"": ""VA"", ""is_mobile"": false}" 21672,8,485,2017-01-25 07:55:44,http://rosenbaum.org/stephon,0.4861377900,184.150.80.14,"{""location"": ""ET"", ""is_mobile"": false}" 21673,8,485,2017-04-14 06:29:42,http://stroman.name/mittie,0.2668798450,15.254.3.232,"{""location"": ""TL"", ""is_mobile"": true}" 21674,8,485,2017-02-21 23:26:05,http://okonlubowitz.org/alford,0.1269332366,169.208.57.97,"{""location"": ""TV"", ""is_mobile"": true}" 21675,8,485,2017-01-18 02:39:24,http://pollich.name/nora_koepp,0.4920038918,175.42.55.71,"{""location"": ""NA"", ""is_mobile"": false}" 21676,8,485,2017-02-07 16:02:14,http://weinat.org/roman.skiles,0.1075989216,223.230.195.78,"{""location"": ""IL"", ""is_mobile"": true}" 21677,8,485,2017-04-30 06:33:40,http://emard.info/trenton,0.2717064282,73.81.81.48,"{""location"": ""AD"", ""is_mobile"": false}" 21678,8,485,2017-05-23 05:31:59,http://collierjerde.info/kelley,0.0536243873,82.175.31.9,"{""location"": ""BW"", ""is_mobile"": false}" 21679,8,485,2017-01-08 19:04:59,http://pfannerstillpfannerstill.org/scarlett,0.9791798050,202.107.172.54,"{""location"": ""CF"", ""is_mobile"": false}" 21680,8,485,2016-12-15 13:56:36,http://fay.name/ayla_stiedemann,0.5324430676,162.34.146.210,"{""location"": ""PW"", ""is_mobile"": true}" 21681,8,485,2017-05-06 14:02:24,http://ratke.name/leon,0.9916864938,14.201.188.4,"{""location"": ""CI"", ""is_mobile"": true}" 21682,8,485,2016-12-29 00:43:18,http://westwalker.co/mariane,0.9314809213,87.9.135.42,"{""location"": ""SA"", ""is_mobile"": false}" 21683,8,485,2017-01-12 01:20:54,http://homenickjohns.org/burdette,0.5441678117,180.22.172.27,"{""location"": ""LC"", ""is_mobile"": false}" 21684,8,485,2017-02-23 08:33:29,http://hartmannlesch.name/robb_mante,0.6640685971,197.19.140.105,"{""location"": ""CK"", ""is_mobile"": false}" 21685,8,485,2016-12-28 07:29:03,http://dare.name/allie,0.7063734397,250.79.148.43,"{""location"": ""VC"", ""is_mobile"": false}" 21686,8,485,2017-04-02 17:37:19,http://heidenreichhyatt.io/rolando.upton,0.6384392639,15.37.168.67,"{""location"": ""LA"", ""is_mobile"": true}" 21687,8,485,2017-05-08 14:34:00,http://dicki.info/braxton.mayert,0.2745501242,56.160.132.48,"{""location"": ""KE"", ""is_mobile"": true}" 21688,8,485,2017-04-10 21:39:44,http://robel.info/trace_sporer,0.2962148749,111.226.166.118,"{""location"": ""IM"", ""is_mobile"": true}" 595,1,14,2017-01-16 19:19:57,http://hoegerlegros.org/tyrel,0.5432075141,91.144.186.14,"{""location"": ""BN"", ""is_mobile"": true}" 596,1,14,2017-02-19 07:17:47,http://altenwerth.name/wendell,0.0659388037,220.187.145.23,"{""location"": ""GA"", ""is_mobile"": false}" 597,1,14,2016-12-27 04:14:54,http://lindgren.name/kendra,0.5089340492,94.165.227.46,"{""location"": ""BR"", ""is_mobile"": false}" 598,1,14,2016-12-25 01:57:41,http://keebler.info/caden.gaylord,0.9146963969,230.86.125.68,"{""location"": ""CF"", ""is_mobile"": true}" 599,1,14,2017-03-14 03:20:49,http://becker.net/americo_hegmann,0.0093533833,249.142.134.86,"{""location"": ""DJ"", ""is_mobile"": true}" 600,1,14,2017-01-30 18:01:58,http://rolfsonferry.org/seamus_feil,0.2775745424,166.244.207.39,"{""location"": ""LA"", ""is_mobile"": true}" 601,1,14,2017-01-24 01:53:12,http://rowe.biz/lavina,0.1462301167,200.144.52.114,"{""location"": ""TG"", ""is_mobile"": true}" 602,1,14,2017-01-09 17:07:09,http://muellerokon.io/laura_leannon,0.7330493598,44.130.254.214,"{""location"": ""MC"", ""is_mobile"": true}" 603,1,14,2017-04-10 23:00:03,http://rauwilkinson.org/charlotte,0.3105306891,77.215.102.213,"{""location"": ""ES"", ""is_mobile"": false}" 604,1,14,2017-06-13 07:32:29,http://jaskolski.co/bernadette.zboncak,0.1769557479,165.208.12.113,"{""location"": ""WF"", ""is_mobile"": false}" 605,1,14,2017-04-16 09:01:51,http://littel.co/hipolito,0.9811855012,49.74.184.165,"{""location"": ""SY"", ""is_mobile"": false}" 606,1,14,2017-05-26 16:54:37,http://reillygoyette.org/sasha_windler,0.8339317455,230.118.80.7,"{""location"": ""OM"", ""is_mobile"": false}" 607,1,14,2017-06-05 08:11:39,http://hyatt.name/deonte,0.0918650207,121.243.80.27,"{""location"": ""UA"", ""is_mobile"": true}" 608,1,14,2016-12-16 02:27:48,http://torpdach.info/lue,0.6739489847,61.125.123.56,"{""location"": ""BZ"", ""is_mobile"": false}" 609,1,14,2016-12-23 23:34:55,http://reichel.co/elia,0.8384398069,76.2.48.134,"{""location"": ""BL"", ""is_mobile"": true}" 610,1,14,2017-04-10 20:10:17,http://kshlerin.net/lilian,0.0310620762,47.75.54.236,"{""location"": ""IS"", ""is_mobile"": true}" 611,1,14,2017-03-21 08:55:09,http://blick.net/ara_wiegand,0.2728816972,150.164.243.184,"{""location"": ""MX"", ""is_mobile"": false}" 612,1,14,2017-05-18 13:54:20,http://haagrowe.net/vernie,0.6654161345,236.203.111.3,"{""location"": ""CN"", ""is_mobile"": false}" 613,1,14,2017-02-01 18:09:55,http://schaefer.org/giovanni_cruickshank,0.2167761636,120.204.39.225,"{""location"": ""GL"", ""is_mobile"": true}" 614,1,14,2017-03-21 00:35:49,http://aufderhar.biz/rosella_bins,0.5570723880,233.62.8.73,"{""location"": ""PW"", ""is_mobile"": false}" 615,1,14,2017-06-09 05:25:49,http://dooley.co/ozella,0.4895660094,128.240.63.237,"{""location"": ""GP"", ""is_mobile"": false}" 616,1,14,2017-04-08 05:57:13,http://mckenzie.info/halle,0.4807887459,200.165.10.157,"{""location"": ""MH"", ""is_mobile"": false}" 617,1,14,2016-12-29 20:07:29,http://boehm.io/margaret_hagenes,0.2275306982,16.18.173.172,"{""location"": ""PE"", ""is_mobile"": false}" 618,1,14,2017-01-04 16:25:02,http://tremblay.io/jordy,0.1394232350,140.212.50.67,"{""location"": ""YT"", ""is_mobile"": true}" 619,1,14,2017-05-23 09:27:28,http://rath.net/vincenza_lakin,0.8313897174,94.206.247.47,"{""location"": ""TG"", ""is_mobile"": false}" 620,1,14,2017-03-04 03:50:57,http://keebler.org/braulio_nader,0.2681330050,180.69.135.140,"{""location"": ""PE"", ""is_mobile"": true}" 621,1,14,2017-04-15 11:03:32,http://osinskiryan.io/ethan,0.4487611654,117.50.194.69,"{""location"": ""BM"", ""is_mobile"": true}" 622,1,14,2016-12-19 04:19:36,http://ziemekunze.biz/roger_vonrueden,0.9174269971,11.62.214.205,"{""location"": ""IN"", ""is_mobile"": false}" 623,1,14,2017-05-26 08:58:00,http://hettinger.io/penelope_hickle,0.5637041815,51.178.143.91,"{""location"": ""WS"", ""is_mobile"": true}" 624,1,14,2017-04-11 17:19:16,http://ondrickarice.co/aidan_padberg,0.6963666691,218.144.154.122,"{""location"": ""ZA"", ""is_mobile"": true}" 625,1,14,2017-03-28 22:21:38,http://torptowne.net/emerson_ohara,0.7768064036,239.140.116.240,"{""location"": ""SO"", ""is_mobile"": true}" 626,1,15,2017-06-11 10:31:39,http://denesik.co/kade.wisoky,0.5051103131,21.115.154.250,"{""location"": ""BB"", ""is_mobile"": false}" 627,1,15,2017-01-21 10:46:59,http://abbott.name/lizzie,0.1422019063,101.66.100.220,"{""location"": ""TL"", ""is_mobile"": true}" 628,1,15,2017-03-25 07:37:21,http://schamberger.org/marcos_fritsch,0.3663524713,178.109.238.51,"{""location"": ""PH"", ""is_mobile"": true}" 629,1,15,2017-01-30 01:48:29,http://kshlerinhackett.biz/kyleigh.moriette,0.7853134503,25.21.127.119,"{""location"": ""NF"", ""is_mobile"": true}" 630,1,15,2017-03-08 12:33:53,http://mohr.co/hank,0.8183540816,223.222.220.239,"{""location"": ""LB"", ""is_mobile"": false}" 631,1,15,2017-02-27 08:47:55,http://kreigerlueilwitz.org/kelli,0.2487063478,242.97.161.94,"{""location"": ""AX"", ""is_mobile"": false}" 632,1,15,2017-05-08 08:20:20,http://wymanlindgren.com/jammie.spinka,0.3412591434,68.3.160.12,"{""location"": ""VG"", ""is_mobile"": false}" 633,1,15,2017-02-17 10:53:19,http://kuphalhagenes.name/maymie.buckridge,0.3996874567,161.84.223.200,"{""location"": ""CY"", ""is_mobile"": false}" 634,1,15,2016-12-26 06:30:24,http://cummeratahuels.io/katheryn.schuster,0.6776608313,203.166.123.231,"{""location"": ""KM"", ""is_mobile"": true}" 635,1,15,2017-03-26 04:49:29,http://flatleygerlach.co/deie_brekke,0.7207512377,254.183.199.157,"{""location"": ""NG"", ""is_mobile"": false}" 636,1,15,2017-01-21 15:44:17,http://hackett.org/muriel_hayes,0.4471580046,141.99.108.10,"{""location"": ""TG"", ""is_mobile"": false}" 637,1,15,2017-01-07 20:17:15,http://brownlueilwitz.biz/earlene_barton,0.9472549191,72.100.22.118,"{""location"": ""GE"", ""is_mobile"": false}" 638,1,15,2017-06-09 12:16:52,http://quigley.com/evelyn,0.6397288046,207.64.164.201,"{""location"": ""FR"", ""is_mobile"": true}" 639,1,15,2017-06-05 10:58:31,http://littlemayert.name/tristin,0.8791812429,227.26.183.26,"{""location"": ""FJ"", ""is_mobile"": false}" 640,1,15,2017-05-18 01:01:24,http://stracke.co/esta,0.3325711879,138.85.254.54,"{""location"": ""VG"", ""is_mobile"": false}" 641,1,15,2017-04-04 10:53:14,http://langworth.co/damon,0.1069602064,244.106.106.236,"{""location"": ""KI"", ""is_mobile"": true}" 642,1,15,2017-04-05 09:39:17,http://rath.co/jonathan,0.0542174405,251.217.158.103,"{""location"": ""BM"", ""is_mobile"": true}" 643,1,15,2017-01-28 23:14:42,http://davislynch.name/carolyn_runolfsdottir,0.1115052349,87.160.127.187,"{""location"": ""HR"", ""is_mobile"": false}" 644,1,15,2017-01-07 09:37:40,http://bauchlittle.biz/donald_keebler,0.7477258304,210.30.115.194,"{""location"": ""HU"", ""is_mobile"": false}" 645,1,15,2017-06-08 10:17:13,http://bergstrom.net/mortimer.oreilly,0.8940896011,211.178.80.176,"{""location"": ""VU"", ""is_mobile"": false}" 646,1,15,2016-12-17 19:08:41,http://mitchellschuster.net/jennyfer,0.5404107081,221.233.25.115,"{""location"": ""PF"", ""is_mobile"": true}" 21689,8,485,2017-02-15 09:41:18,http://littel.name/michelle_metz,0.4264450645,171.230.146.2,"{""location"": ""KH"", ""is_mobile"": false}" 21690,8,485,2017-05-09 17:32:22,http://barton.info/abelardo,0.1581679387,85.235.224.121,"{""location"": ""BN"", ""is_mobile"": true}" 21691,8,485,2017-04-18 07:08:42,http://doylewilkinson.biz/charlotte_stamm,0.1443394564,18.57.231.210,"{""location"": ""SM"", ""is_mobile"": true}" 21692,8,486,2017-05-13 06:59:50,http://homenickdeckow.biz/xavier,0.8921992667,48.20.198.159,"{""location"": ""ES"", ""is_mobile"": true}" 21693,8,486,2017-02-25 05:49:35,http://moenjast.co/olin,0.4779121900,157.7.220.14,"{""location"": ""ER"", ""is_mobile"": false}" 21694,8,486,2017-06-01 12:17:02,http://leannon.name/earlene,0.1818201272,90.131.9.242,"{""location"": ""NA"", ""is_mobile"": true}" 21695,8,486,2017-05-03 21:49:06,http://beahan.com/enola.pouros,0.3547822867,47.110.180.217,"{""location"": ""PF"", ""is_mobile"": true}" 21696,8,486,2017-03-24 23:24:55,http://kleinsenger.io/lenny,0.0116410119,126.108.67.27,"{""location"": ""SM"", ""is_mobile"": false}" 21697,8,486,2016-12-28 12:31:29,http://bogisich.net/cheyenne.crist,0.0816924364,158.5.248.197,"{""location"": ""BQ"", ""is_mobile"": true}" 21698,8,486,2017-02-15 14:27:53,http://beerkovacek.biz/izabella_lang,0.8279827583,94.187.243.75,"{""location"": ""EC"", ""is_mobile"": true}" 21699,8,486,2017-01-22 12:54:48,http://fisher.name/cullen_osinski,0.7246660858,239.217.223.2,"{""location"": ""CR"", ""is_mobile"": false}" 21700,8,486,2017-05-07 23:47:15,http://runolfon.co/dangelo,0.4690782416,156.23.75.123,"{""location"": ""FO"", ""is_mobile"": true}" 21701,8,486,2017-06-12 16:57:49,http://gleichner.biz/janae.auer,0.4191748897,120.87.30.222,"{""location"": ""TW"", ""is_mobile"": true}" 21702,8,486,2017-05-21 22:24:13,http://raynorjohnson.io/shemar.marks,0.4141988553,236.116.188.38,"{""location"": ""NZ"", ""is_mobile"": false}" 21703,8,486,2017-04-25 05:00:07,http://runolfsdottir.org/kade,0.6409177871,186.166.81.215,"{""location"": ""BB"", ""is_mobile"": false}" 21704,8,486,2017-02-08 14:29:45,http://bogan.com/brionna,0.7102126445,68.115.62.5,"{""location"": ""CN"", ""is_mobile"": true}" 21705,8,486,2017-02-13 13:54:28,http://bartoletti.io/reese_dare,0.0634150170,144.194.201.103,"{""location"": ""AD"", ""is_mobile"": true}" 21706,8,486,2017-04-29 10:01:50,http://barton.name/cecile.ledner,0.5832031016,174.63.90.195,"{""location"": ""SL"", ""is_mobile"": true}" 21707,8,486,2017-03-06 06:30:18,http://cruickshankzboncak.com/adolf.ward,0.6110112727,55.4.93.118,"{""location"": ""ER"", ""is_mobile"": true}" 21708,8,486,2017-03-29 23:02:01,http://rodriguez.info/silas,0.5068219802,229.72.21.107,"{""location"": ""PG"", ""is_mobile"": false}" 21709,8,486,2017-01-21 17:26:18,http://rippin.biz/toy_deckow,0.1523969342,89.144.165.135,"{""location"": ""TR"", ""is_mobile"": false}" 21710,8,486,2017-03-13 12:38:22,http://nitzsche.com/aubree,0.6397100353,155.60.103.132,"{""location"": ""PT"", ""is_mobile"": false}" 21711,8,486,2017-03-06 17:14:43,http://bauchfritsch.org/darian.ryan,0.3322842238,16.212.136.104,"{""location"": ""CG"", ""is_mobile"": true}" 21712,8,486,2017-04-11 17:35:58,http://rempel.co/velva.steuber,0.1943471364,94.246.208.105,"{""location"": ""AT"", ""is_mobile"": true}" 21713,8,486,2017-05-19 00:06:14,http://sauer.com/genoveva.casper,0.2891295916,191.204.173.17,"{""location"": ""ZW"", ""is_mobile"": true}" 21714,8,486,2017-03-19 09:47:47,http://weimann.com/linnie,0.5607517309,45.27.73.7,"{""location"": ""TT"", ""is_mobile"": true}" 21715,8,486,2016-12-29 09:25:04,http://sauer.net/trever,0.8734058034,166.97.171.170,"{""location"": ""CU"", ""is_mobile"": true}" 21716,8,486,2017-04-16 00:33:23,http://cormier.io/garrick.hickle,0.6809616187,128.13.43.244,"{""location"": ""BW"", ""is_mobile"": false}" 21717,8,486,2017-01-09 15:18:13,http://harber.com/arturo,0.4932686915,45.223.123.122,"{""location"": ""NU"", ""is_mobile"": false}" 21718,8,486,2017-02-21 19:03:59,http://hayescole.info/ila_sporer,0.1537280711,132.46.203.249,"{""location"": ""OM"", ""is_mobile"": false}" 21719,8,486,2017-04-18 08:34:06,http://emmerichstehr.org/rubie,0.9424516850,81.239.185.12,"{""location"": ""JO"", ""is_mobile"": true}" 21720,8,486,2017-04-19 15:23:19,http://grady.info/amos,0.5807246455,174.2.30.125,"{""location"": ""GF"", ""is_mobile"": true}" 21721,8,486,2017-01-30 01:25:15,http://metz.io/antonietta,0.4564609492,84.88.174.230,"{""location"": ""JO"", ""is_mobile"": false}" 21722,8,486,2017-03-15 03:31:55,http://ritchie.biz/ford,0.8200843336,213.204.170.104,"{""location"": ""LY"", ""is_mobile"": false}" 21723,8,486,2017-01-24 16:34:55,http://von.com/aleandra_quitzon,0.2542551902,114.169.202.150,"{""location"": ""UY"", ""is_mobile"": true}" 21724,8,486,2017-06-04 23:54:56,http://kub.info/bert_robel,0.6004908216,133.77.134.217,"{""location"": ""BT"", ""is_mobile"": false}" 21725,8,486,2016-12-17 19:26:58,http://satterfield.biz/jaleel,0.5027998777,161.121.84.183,"{""location"": ""HU"", ""is_mobile"": false}" 21726,8,486,2017-03-08 16:29:28,http://frami.co/marlon.runte,0.0274555892,36.3.38.46,"{""location"": ""KY"", ""is_mobile"": true}" 21727,8,486,2017-04-24 19:04:18,http://sanford.org/miller,0.6122300013,44.75.73.232,"{""location"": ""ET"", ""is_mobile"": true}" 21728,8,486,2017-02-27 03:05:00,http://buckridgebeier.co/wyatt_cremin,0.6015390341,14.250.45.44,"{""location"": ""UY"", ""is_mobile"": false}" 21729,8,486,2017-01-13 09:26:54,http://kovacekmann.name/jee,0.2899621249,50.49.223.196,"{""location"": ""TG"", ""is_mobile"": false}" 21730,8,486,2017-02-28 22:11:18,http://lebsackbergstrom.info/jalyn,0.5359494010,194.2.176.35,"{""location"": ""OM"", ""is_mobile"": true}" 21731,8,486,2017-01-30 08:56:53,http://fayschimmel.biz/kyleigh,0.6414176020,157.108.156.25,"{""location"": ""AD"", ""is_mobile"": true}" 21732,8,486,2016-12-14 05:06:09,http://windlerharber.io/meda,0.8105419091,220.74.83.110,"{""location"": ""KY"", ""is_mobile"": false}" 21733,8,486,2017-06-08 16:18:42,http://towne.net/axel_kuhn,0.4903286079,192.110.135.56,"{""location"": ""BH"", ""is_mobile"": false}" 21734,8,486,2017-04-24 11:55:25,http://kuhicschroeder.biz/tod,0.0542611465,234.122.140.185,"{""location"": ""NZ"", ""is_mobile"": false}" 21735,8,486,2017-03-02 23:44:29,http://metz.co/leda,0.0947482238,134.27.223.82,"{""location"": ""CI"", ""is_mobile"": true}" 21736,8,486,2017-05-10 01:38:55,http://kundefahey.net/leonardo,0.1213394031,227.70.177.43,"{""location"": ""ET"", ""is_mobile"": false}" 21737,8,486,2017-02-28 05:01:00,http://thiel.name/magdalen_carter,0.1892735298,124.248.243.170,"{""location"": ""DE"", ""is_mobile"": false}" 21738,8,486,2017-06-13 04:11:12,http://ryanmitchell.biz/candido.kuhlman,0.3307187210,211.149.9.27,"{""location"": ""FJ"", ""is_mobile"": false}" 21739,8,486,2017-04-05 18:25:28,http://friesen.com/benjamin_nolan,0.2306927392,175.216.69.180,"{""location"": ""FJ"", ""is_mobile"": false}" 647,1,15,2016-12-27 01:13:20,http://conroy.co/fannie_jones,0.0580697490,144.141.166.77,"{""location"": ""GN"", ""is_mobile"": false}" 648,1,15,2017-05-18 06:05:38,http://keeblerkaulke.name/eula.schroeder,0.6325928777,169.154.222.79,"{""location"": ""JE"", ""is_mobile"": true}" 649,1,15,2017-02-19 05:58:36,http://pollich.info/mathew_macgyver,0.6876000269,197.209.250.16,"{""location"": ""SJ"", ""is_mobile"": false}" 650,1,15,2017-06-03 00:02:28,http://kihn.co/tyrel,0.4427050526,92.34.252.21,"{""location"": ""VA"", ""is_mobile"": false}" 651,1,16,2017-04-02 11:20:57,http://howegutkowski.info/garett,0.6930005556,46.174.128.227,"{""location"": ""BT"", ""is_mobile"": false}" 652,1,16,2017-04-03 01:42:09,http://monahandouglas.org/althea,0.5576192339,65.64.24.12,"{""location"": ""BM"", ""is_mobile"": true}" 653,1,16,2017-02-26 19:09:42,http://hammesdicki.co/josefa,0.9612586901,92.189.186.128,"{""location"": ""CH"", ""is_mobile"": false}" 654,1,16,2017-05-13 13:19:27,http://shanahan.net/dasia,0.5038018738,87.237.35.205,"{""location"": ""KM"", ""is_mobile"": false}" 655,1,16,2017-04-15 08:25:35,http://beatty.name/cordie_baumbach,0.7196666585,99.222.163.122,"{""location"": ""NU"", ""is_mobile"": false}" 656,1,16,2017-04-30 08:19:29,http://halvorson.net/daniela,0.3338070786,111.159.232.223,"{""location"": ""JO"", ""is_mobile"": true}" 657,1,16,2017-03-14 07:52:02,http://dicki.net/noemi_rau,0.1288609802,146.137.129.205,"{""location"": ""PE"", ""is_mobile"": false}" 658,1,16,2017-02-06 16:09:20,http://weinat.info/zaria.predovic,0.7722979702,254.41.155.178,"{""location"": ""MX"", ""is_mobile"": true}" 659,1,16,2017-06-13 23:50:45,http://dickens.biz/pansy_rempel,0.7336186170,115.231.82.120,"{""location"": ""RU"", ""is_mobile"": true}" 660,1,16,2017-06-08 21:17:36,http://murazikpowlowski.net/juana,0.6797698953,46.245.224.45,"{""location"": ""BL"", ""is_mobile"": true}" 661,1,16,2017-01-01 01:44:31,http://kuvalis.io/randy,0.9008435504,162.240.70.16,"{""location"": ""LT"", ""is_mobile"": true}" 662,1,16,2017-01-05 13:30:31,http://muellergoldner.org/icie,0.3427249122,239.166.25.34,"{""location"": ""FR"", ""is_mobile"": false}" 663,1,16,2017-05-06 07:51:18,http://marquardt.org/keely,0.0212565228,180.211.239.105,"{""location"": ""SD"", ""is_mobile"": false}" 664,1,16,2017-06-03 13:34:02,http://botsford.org/vidal.hayes,0.0674131129,239.160.115.136,"{""location"": ""BG"", ""is_mobile"": false}" 665,1,16,2017-03-23 03:50:35,http://harvey.name/margarette,0.4216591311,206.247.121.125,"{""location"": ""SA"", ""is_mobile"": true}" 666,1,16,2017-03-23 05:25:27,http://zieme.com/nickolas_gorczany,0.7138523167,246.18.64.49,"{""location"": ""SA"", ""is_mobile"": true}" 667,1,16,2016-12-20 05:54:22,http://hermann.name/ethyl.hahn,0.3606970978,241.121.25.102,"{""location"": ""IO"", ""is_mobile"": true}" 668,1,16,2016-12-17 21:39:01,http://hilpert.biz/cale,0.1903379742,158.185.230.215,"{""location"": ""BH"", ""is_mobile"": false}" 669,1,16,2017-03-22 19:04:25,http://pouros.net/donny.kuphal,0.8576155915,89.68.156.143,"{""location"": ""AE"", ""is_mobile"": false}" 670,1,16,2017-02-25 22:15:06,http://connellyjacobi.name/ima_hodkiewicz,0.3946469785,28.227.57.197,"{""location"": ""SK"", ""is_mobile"": true}" 671,1,16,2017-02-02 18:38:07,http://daniel.net/adolf,0.6712166817,208.239.95.189,"{""location"": ""NC"", ""is_mobile"": true}" 672,1,16,2017-01-12 07:21:55,http://ziemannspinka.name/foster,0.8757948990,207.192.202.121,"{""location"": ""AO"", ""is_mobile"": true}" 673,1,16,2017-02-06 16:31:12,http://gutkowskigoyette.info/glennie,0.0849041091,146.24.94.204,"{""location"": ""MZ"", ""is_mobile"": true}" 674,1,16,2016-12-20 12:56:24,http://boehm.biz/barbara,0.5807929292,114.217.201.152,"{""location"": ""KN"", ""is_mobile"": false}" 675,1,16,2017-03-20 16:44:34,http://mclaughlin.com/brandon,0.4377246245,137.254.40.244,"{""location"": ""MK"", ""is_mobile"": false}" 676,1,16,2017-02-18 07:08:52,http://kaulke.name/ruell,0.2871905270,115.210.117.95,"{""location"": ""SJ"", ""is_mobile"": true}" 677,1,16,2017-04-17 05:31:52,http://trantowkunde.co/wilfrid_sanford,0.1551226698,62.79.235.130,"{""location"": ""BG"", ""is_mobile"": false}" 678,1,16,2017-04-04 04:14:52,http://reilly.co/eleonore.kohler,0.1741163822,200.251.12.43,"{""location"": ""QA"", ""is_mobile"": true}" 679,1,16,2017-03-05 16:29:21,http://klocko.org/sim,0.5320342371,160.86.178.11,"{""location"": ""NL"", ""is_mobile"": false}" 680,1,16,2017-02-14 12:34:52,http://quigleycormier.info/leonard.barrows,0.9918399512,18.4.135.176,"{""location"": ""MQ"", ""is_mobile"": true}" 681,1,16,2017-03-30 05:13:04,http://huels.co/jada.schmitt,0.3819120076,91.44.8.15,"{""location"": ""CH"", ""is_mobile"": false}" 682,1,16,2017-04-09 01:37:21,http://carroll.org/adolf,0.6355591001,251.123.77.182,"{""location"": ""CY"", ""is_mobile"": false}" 683,1,16,2017-01-10 21:38:56,http://bednarkunde.net/alejandra,0.2563387908,172.245.58.254,"{""location"": ""PT"", ""is_mobile"": true}" 684,1,16,2017-02-10 12:10:17,http://wintheiserhowell.net/elaina,0.4509225291,164.78.228.42,"{""location"": ""UG"", ""is_mobile"": false}" 685,1,16,2017-01-09 09:36:46,http://ryan.info/raphael,0.2556251134,217.94.118.186,"{""location"": ""BI"", ""is_mobile"": false}" 686,1,16,2017-02-17 04:21:41,http://schmidt.name/ella_pfeffer,0.1819068287,196.226.154.33,"{""location"": ""RU"", ""is_mobile"": true}" 687,1,17,2017-03-11 09:01:17,http://schamberger.biz/cielo,0.6388316833,44.242.142.153,"{""location"": ""GH"", ""is_mobile"": true}" 688,1,17,2017-05-02 02:56:57,http://blick.net/asa_mitchell,0.4358244047,194.72.231.243,"{""location"": ""BO"", ""is_mobile"": true}" 689,1,17,2017-05-13 15:04:01,http://schaden.biz/giuseppe,0.5506703269,198.132.202.132,"{""location"": ""WS"", ""is_mobile"": true}" 690,1,17,2017-05-26 21:44:48,http://halvorson.com/janis.fahey,0.5896872540,240.192.149.160,"{""location"": ""RE"", ""is_mobile"": true}" 691,1,17,2017-05-17 14:36:31,http://volkmanmurray.co/jadyn,0.2184602954,151.219.145.97,"{""location"": ""CA"", ""is_mobile"": true}" 692,1,17,2017-04-26 19:13:26,http://heller.biz/jamar_halvorson,0.4983878910,16.187.217.144,"{""location"": ""AX"", ""is_mobile"": false}" 693,1,17,2017-02-04 14:59:22,http://mclaughlindickinson.com/ayana_nikolaus,0.2993980338,40.252.186.230,"{""location"": ""CV"", ""is_mobile"": false}" 694,1,17,2017-03-05 07:22:39,http://schroederwaters.name/lillie,0.9536559892,140.64.204.253,"{""location"": ""BI"", ""is_mobile"": false}" 695,1,17,2017-04-22 07:40:49,http://franecki.net/keagan.nolan,0.3976513512,176.37.254.64,"{""location"": ""MQ"", ""is_mobile"": false}" 696,1,17,2017-01-03 07:51:14,http://hamill.com/gerda.abbott,0.0613937381,3.68.20.223,"{""location"": ""MN"", ""is_mobile"": false}" 697,1,17,2016-12-29 14:02:20,http://abernathy.com/antonio,0.5070615077,192.92.225.49,"{""location"": ""AD"", ""is_mobile"": true}" 698,1,17,2017-01-01 17:35:56,http://ankunding.net/oral.nikolaus,0.3136747089,236.18.205.144,"{""location"": ""EC"", ""is_mobile"": false}" 21740,8,486,2016-12-17 13:11:52,http://oreilly.io/sincere,0.9601836176,182.248.29.81,"{""location"": ""BQ"", ""is_mobile"": true}" 21741,8,486,2017-05-28 20:10:46,http://harberweber.io/violette,0.6283354924,246.54.227.81,"{""location"": ""TW"", ""is_mobile"": false}" 21742,8,486,2017-01-03 07:21:33,http://durgan.info/reyna,0.9898902314,85.67.205.170,"{""location"": ""TH"", ""is_mobile"": true}" 21743,8,486,2017-03-14 14:07:48,http://batz.org/evert.reichel,0.1224614534,85.218.180.83,"{""location"": ""RU"", ""is_mobile"": true}" 21744,8,486,2017-04-05 21:32:22,http://christiansen.org/brent.johns,0.3630375768,209.236.156.35,"{""location"": ""HU"", ""is_mobile"": true}" 21745,8,486,2017-02-27 11:58:29,http://damore.org/gillian_moriette,0.9030868842,206.175.146.183,"{""location"": ""MD"", ""is_mobile"": false}" 21746,8,486,2017-05-16 22:13:16,http://hahnstracke.info/abigale_vonrueden,0.3755267174,121.30.105.142,"{""location"": ""JE"", ""is_mobile"": false}" 21747,8,486,2017-05-25 08:13:10,http://schmidt.name/katrina_stanton,0.3675710071,21.80.33.253,"{""location"": ""AO"", ""is_mobile"": true}" 21748,8,486,2017-01-10 03:35:30,http://schowalterherzog.io/christy_rice,0.1645404389,78.150.100.36,"{""location"": ""TD"", ""is_mobile"": false}" 21749,8,486,2017-03-02 23:00:55,http://trantowmraz.com/milton_marvin,0.2279152498,180.10.194.21,"{""location"": ""MC"", ""is_mobile"": true}" 21750,8,487,2017-04-12 09:57:07,http://friesen.biz/otha,0.0195390988,148.131.181.80,"{""location"": ""PR"", ""is_mobile"": true}" 21751,8,487,2017-02-27 12:22:38,http://carroll.net/diego,0.9348165243,231.174.39.64,"{""location"": ""IO"", ""is_mobile"": false}" 21752,8,487,2016-12-24 01:24:58,http://bogisichprice.org/kenya,0.2022475982,107.163.12.62,"{""location"": ""DE"", ""is_mobile"": false}" 21753,8,487,2017-05-14 04:53:26,http://zulauf.biz/mollie,0.5045467364,144.128.69.117,"{""location"": ""PN"", ""is_mobile"": true}" 21754,8,487,2017-05-01 01:57:33,http://schamberger.org/kaela_kemmer,0.7384094839,188.224.198.161,"{""location"": ""MX"", ""is_mobile"": true}" 21755,8,487,2017-04-04 14:23:42,http://prohaskacartwright.name/rosalinda_von,0.8736868329,134.115.247.156,"{""location"": ""JE"", ""is_mobile"": true}" 21756,8,487,2017-05-18 14:28:13,http://dibbert.com/jason,0.9938585716,4.51.36.51,"{""location"": ""AF"", ""is_mobile"": true}" 21757,8,487,2017-04-11 04:09:05,http://rutherfordwalter.biz/annamae,0.1503358846,115.209.72.11,"{""location"": ""ZM"", ""is_mobile"": true}" 21758,8,487,2017-01-10 05:32:26,http://williamson.io/ayden,0.0983903965,158.223.154.120,"{""location"": ""TL"", ""is_mobile"": false}" 21759,8,487,2017-01-18 13:28:36,http://simonis.io/walter_wiegand,0.7478961176,236.161.233.131,"{""location"": ""BB"", ""is_mobile"": false}" 21760,8,487,2017-02-13 19:09:41,http://towne.org/destin,0.6458293824,26.227.164.249,"{""location"": ""KW"", ""is_mobile"": false}" 21761,8,487,2017-03-10 14:04:13,http://okeefetremblay.net/george_luettgen,0.4349112527,210.107.226.200,"{""location"": ""PN"", ""is_mobile"": true}" 21762,8,487,2016-12-20 22:31:04,http://renner.biz/adella_farrell,0.8303364331,147.151.197.119,"{""location"": ""DK"", ""is_mobile"": true}" 21763,8,487,2017-03-12 12:08:59,http://lindgrendouglas.com/carlo,0.1560037753,49.189.202.195,"{""location"": ""VG"", ""is_mobile"": false}" 21764,8,487,2017-06-01 17:43:45,http://reingerdubuque.org/laney.steuber,0.7804121735,36.11.206.141,"{""location"": ""KP"", ""is_mobile"": false}" 21765,8,487,2017-02-22 09:03:11,http://mertz.co/eliseo_greenfelder,0.8473871245,84.21.49.224,"{""location"": ""TG"", ""is_mobile"": false}" 21766,8,487,2017-03-18 18:22:31,http://vonruedenwatsica.com/devante,0.3634518885,162.242.130.120,"{""location"": ""TC"", ""is_mobile"": false}" 21767,8,487,2017-05-27 21:51:27,http://kohler.com/annetta,0.5641414134,155.249.180.38,"{""location"": ""CV"", ""is_mobile"": true}" 21768,8,487,2017-01-30 14:39:07,http://murazik.org/verla,0.4443445397,227.229.232.188,"{""location"": ""PA"", ""is_mobile"": false}" 21769,8,487,2017-05-20 16:36:56,http://kaulke.net/hudson,0.0459895908,104.157.94.128,"{""location"": ""LK"", ""is_mobile"": false}" 21770,8,487,2017-06-13 09:58:57,http://upton.name/mohammad,0.9326342900,69.108.223.44,"{""location"": ""MC"", ""is_mobile"": false}" 21771,8,487,2017-04-20 18:40:24,http://gerhold.info/reggie.bauch,0.0818195463,59.83.109.239,"{""location"": ""GF"", ""is_mobile"": false}" 21772,8,487,2017-03-26 00:48:25,http://zemlak.net/aurelio,0.5274810200,202.54.238.19,"{""location"": ""VE"", ""is_mobile"": false}" 21773,8,487,2017-01-08 04:34:20,http://abernathy.com/marcelina,0.4078646495,95.143.43.229,"{""location"": ""AW"", ""is_mobile"": true}" 21774,8,487,2017-02-07 00:07:46,http://macgyvermclaughlin.org/santino_huels,0.4235228598,50.101.87.119,"{""location"": ""MA"", ""is_mobile"": true}" 21775,8,487,2017-04-02 04:30:11,http://cummingskovacek.net/caie,0.4842856028,147.59.205.130,"{""location"": ""AU"", ""is_mobile"": false}" 21776,8,487,2017-04-26 21:53:02,http://gradypurdy.name/tyrel.little,0.9612433927,233.70.205.115,"{""location"": ""LB"", ""is_mobile"": true}" 21777,8,487,2017-05-09 16:31:37,http://cronin.io/novella,0.6964588772,200.32.178.56,"{""location"": ""BR"", ""is_mobile"": false}" 21778,8,487,2017-03-25 14:31:25,http://leuschkelueilwitz.com/gregorio,0.0567256574,57.32.14.143,"{""location"": ""TW"", ""is_mobile"": false}" 21779,8,487,2017-01-08 20:43:20,http://dickinsonbogisich.net/kimberly.rutherford,0.2784065390,108.108.213.63,"{""location"": ""FO"", ""is_mobile"": false}" 21780,8,487,2016-12-16 02:09:06,http://mullerreynolds.io/taya,0.1778741308,229.8.167.173,"{""location"": ""MX"", ""is_mobile"": true}" 21781,8,487,2017-01-03 21:31:07,http://dooley.biz/marguerite,0.4039199337,215.132.27.109,"{""location"": ""EE"", ""is_mobile"": true}" 21782,8,487,2017-01-13 08:35:36,http://pagac.com/bruce_gislason,0.1257830008,249.115.243.135,"{""location"": ""BG"", ""is_mobile"": false}" 21783,8,487,2017-04-20 02:22:50,http://wiza.io/hans.jacobi,0.4783631664,10.139.56.8,"{""location"": ""RW"", ""is_mobile"": false}" 21784,8,487,2016-12-13 19:27:32,http://macejkovic.co/zechariah,0.0063537839,140.217.157.96,"{""location"": ""LC"", ""is_mobile"": false}" 21785,8,487,2017-02-19 23:10:47,http://pfeffer.net/wilfredo,0.9502524338,197.178.21.250,"{""location"": ""RW"", ""is_mobile"": false}" 21786,8,487,2017-03-16 00:25:11,http://kiehnemard.info/maggie,0.4513554925,50.59.60.247,"{""location"": ""CZ"", ""is_mobile"": false}" 21787,8,487,2017-05-18 23:57:00,http://jacobi.org/erna_kutch,0.5031997690,51.116.171.120,"{""location"": ""IT"", ""is_mobile"": false}" 21788,8,488,2016-12-21 22:16:34,http://upton.org/alexanne_ferry,0.1581584970,104.212.183.169,"{""location"": ""TH"", ""is_mobile"": true}" 21789,8,488,2017-01-16 06:40:53,http://bergnaum.info/germaine_wilderman,0.4465439206,215.114.142.168,"{""location"": ""TC"", ""is_mobile"": true}" 21790,8,488,2016-12-19 08:37:32,http://mosciski.biz/mathew,0.6820212252,9.236.61.19,"{""location"": ""TL"", ""is_mobile"": true}" 699,1,17,2017-01-06 12:04:52,http://mayertgislason.com/nola,0.6659007426,250.32.221.225,"{""location"": ""SI"", ""is_mobile"": false}" 700,1,17,2016-12-26 18:53:11,http://schambergerbarton.co/felicita.batz,0.3285112133,237.194.207.155,"{""location"": ""GP"", ""is_mobile"": true}" 701,1,17,2017-01-31 04:10:00,http://conroyrowe.name/tina,0.1983847355,114.92.40.163,"{""location"": ""ST"", ""is_mobile"": true}" 702,1,17,2017-05-01 19:38:03,http://ondrickalockman.io/tania,0.5046378015,217.120.35.226,"{""location"": ""ME"", ""is_mobile"": true}" 703,1,17,2017-04-17 11:02:13,http://gutmann.io/claudine_parker,0.1635388210,176.182.188.241,"{""location"": ""PL"", ""is_mobile"": false}" 704,1,17,2017-02-24 15:47:48,http://lindgren.org/luna,0.4657619213,211.132.206.71,"{""location"": ""AX"", ""is_mobile"": false}" 705,1,17,2017-01-21 02:10:58,http://schaefer.co/mariela_terry,0.8209161171,224.105.29.6,"{""location"": ""PL"", ""is_mobile"": true}" 706,1,17,2016-12-19 21:32:14,http://tromp.net/dortha.wyman,0.3949535681,219.24.27.158,"{""location"": ""PW"", ""is_mobile"": false}" 707,1,17,2017-03-22 14:13:59,http://bogan.co/uriel.hauck,0.2967328537,31.22.121.246,"{""location"": ""LU"", ""is_mobile"": true}" 708,1,17,2017-04-26 21:57:28,http://cartwright.co/winona_prosacco,0.9120943701,40.94.205.53,"{""location"": ""PE"", ""is_mobile"": false}" 709,1,17,2017-05-12 11:38:00,http://greenholthagenes.io/ronny,0.2699627694,13.23.182.126,"{""location"": ""HU"", ""is_mobile"": true}" 710,1,17,2017-05-08 04:36:33,http://cole.com/zander,0.0264346793,83.172.13.223,"{""location"": ""AR"", ""is_mobile"": true}" 711,1,17,2017-03-14 22:22:01,http://mertz.biz/adelbert.cormier,0.6527820019,178.48.69.34,"{""location"": ""ME"", ""is_mobile"": false}" 712,1,17,2017-04-27 23:12:00,http://wuckert.org/oswaldo,0.6761603585,58.104.253.225,"{""location"": ""LY"", ""is_mobile"": true}" 713,1,17,2017-06-11 22:04:40,http://rolfsonwintheiser.net/sammie,0.5112922934,57.253.177.169,"{""location"": ""LI"", ""is_mobile"": false}" 714,1,17,2017-02-03 18:54:03,http://haag.co/alexis,0.3490664837,30.181.160.211,"{""location"": ""SO"", ""is_mobile"": true}" 715,1,17,2017-06-02 06:25:51,http://kemmer.com/polly_price,0.6373698062,106.66.243.209,"{""location"": ""VG"", ""is_mobile"": false}" 716,1,17,2017-03-07 11:34:56,http://greenholt.net/verna,0.6727093527,71.46.205.144,"{""location"": ""CZ"", ""is_mobile"": true}" 717,1,17,2017-04-11 06:13:29,http://harveypaucek.co/brooks.okon,0.7390430651,138.119.218.120,"{""location"": ""GS"", ""is_mobile"": true}" 718,1,17,2017-06-02 08:17:16,http://hodkiewiczcarroll.biz/thea_moriette,0.5702686033,11.127.105.64,"{""location"": ""BR"", ""is_mobile"": false}" 719,1,17,2017-02-09 15:05:42,http://blanda.org/bo,0.9986694066,64.142.118.61,"{""location"": ""KG"", ""is_mobile"": true}" 720,1,17,2017-01-08 04:15:36,http://schimmel.info/urban,0.5970538167,205.138.226.161,"{""location"": ""SM"", ""is_mobile"": true}" 721,1,17,2017-05-07 09:44:28,http://swaniawski.info/edwina,0.6319560771,153.107.248.78,"{""location"": ""LK"", ""is_mobile"": false}" 722,1,17,2017-01-30 16:45:56,http://rauschaefer.name/sarina,0.5419149162,129.192.50.219,"{""location"": ""TF"", ""is_mobile"": true}" 723,1,17,2017-05-21 00:15:37,http://conroyhamill.net/florida.hand,0.3416174456,238.245.225.92,"{""location"": ""WF"", ""is_mobile"": true}" 724,1,17,2017-01-06 15:57:03,http://dooleywiegand.org/hazel,0.7137576964,36.95.64.207,"{""location"": ""KM"", ""is_mobile"": false}" 725,1,17,2017-04-07 19:07:52,http://spinka.net/zoila,0.0180465335,177.101.164.145,"{""location"": ""LK"", ""is_mobile"": false}" 726,1,17,2017-02-23 21:02:27,http://hegmann.co/bell,0.8903189511,163.80.93.17,"{""location"": ""GD"", ""is_mobile"": false}" 727,1,17,2017-01-28 06:43:36,http://kemmerbuckridge.info/brenda.von,0.6481964600,240.224.133.151,"{""location"": ""CU"", ""is_mobile"": true}" 728,1,17,2017-02-23 04:59:21,http://bogisichlabadie.org/adelia.gleichner,0.6935115142,15.102.7.201,"{""location"": ""NA"", ""is_mobile"": false}" 729,1,17,2017-05-24 01:40:32,http://gleason.biz/letha,0.5630973309,51.159.204.118,"{""location"": ""KE"", ""is_mobile"": true}" 730,1,17,2017-04-19 21:36:21,http://powlowski.name/alek.runte,0.4717640153,242.40.44.152,"{""location"": ""LB"", ""is_mobile"": true}" 731,1,17,2017-02-11 23:42:57,http://hettingerwaelchi.biz/liliana.kuvalis,0.4403855103,208.85.192.75,"{""location"": ""LK"", ""is_mobile"": false}" 732,1,17,2017-02-17 01:14:45,http://zieme.com/antonietta,0.6635081513,29.219.17.177,"{""location"": ""DO"", ""is_mobile"": false}" 733,1,17,2017-05-08 04:42:11,http://medhurstdeckow.net/zakary,0.0353995083,170.32.69.135,"{""location"": ""FI"", ""is_mobile"": true}" 734,1,18,2017-02-11 12:31:55,http://harris.net/kay.rutherford,0.3248948260,91.90.51.238,"{""location"": ""ZA"", ""is_mobile"": false}" 735,1,18,2017-03-11 07:44:58,http://batz.biz/albertha,0.6930348274,47.223.80.54,"{""location"": ""HT"", ""is_mobile"": false}" 736,1,18,2017-05-09 00:11:36,http://naderondricka.biz/magdalena_bartell,0.1055769758,97.148.9.70,"{""location"": ""LR"", ""is_mobile"": true}" 737,1,18,2017-01-24 02:51:09,http://mcclurehagenes.org/jackeline,0.4425122865,60.204.149.64,"{""location"": ""PH"", ""is_mobile"": true}" 738,1,18,2017-02-22 05:20:54,http://prohaska.org/moriah,0.1943760653,246.205.23.181,"{""location"": ""SX"", ""is_mobile"": true}" 739,1,18,2017-03-26 18:14:21,http://romaguera.com/billy,0.0734000258,163.175.75.164,"{""location"": ""AZ"", ""is_mobile"": false}" 740,1,18,2017-01-13 15:21:44,http://mcdermott.co/dayne.bergnaum,0.2581587922,213.130.58.10,"{""location"": ""BZ"", ""is_mobile"": true}" 741,1,18,2017-03-23 16:26:02,http://kilback.name/tyrese,0.3479709415,210.166.246.190,"{""location"": ""MO"", ""is_mobile"": true}" 742,1,18,2017-02-03 09:03:46,http://romaguerakiehn.biz/ken.dietrich,0.4404779250,23.99.153.111,"{""location"": ""BE"", ""is_mobile"": true}" 743,1,18,2017-06-11 05:51:07,http://connelly.co/rosanna_gibson,0.9291207293,85.197.253.55,"{""location"": ""MD"", ""is_mobile"": true}" 744,1,18,2017-05-09 12:53:15,http://romaguera.biz/malika.kuhn,0.8868278426,106.238.22.198,"{""location"": ""CA"", ""is_mobile"": false}" 745,1,18,2017-02-25 00:45:57,http://breitenberg.biz/harry.kulas,0.5667186124,170.254.38.24,"{""location"": ""NI"", ""is_mobile"": false}" 746,1,18,2017-04-24 11:32:39,http://oconner.name/betsy.miller,0.0302597362,140.177.145.66,"{""location"": ""UA"", ""is_mobile"": true}" 747,1,18,2017-04-04 23:56:23,http://nader.biz/stephan,0.1393401838,160.185.103.118,"{""location"": ""MQ"", ""is_mobile"": false}" 748,1,18,2017-06-04 12:23:01,http://hermistonpaucek.info/bryce.daniel,0.2202493412,52.26.90.142,"{""location"": ""PS"", ""is_mobile"": false}" 749,1,18,2017-03-31 13:00:40,http://gleichnerrohan.io/cameron.watsica,0.1981494054,16.125.216.202,"{""location"": ""PE"", ""is_mobile"": true}" 750,1,18,2017-04-07 14:20:09,http://dickensvon.info/nicolette.reinger,0.1913264392,50.242.162.178,"{""location"": ""CD"", ""is_mobile"": false}" 21791,8,488,2017-01-16 12:33:28,http://ratke.name/mathias.grimes,0.6418457741,181.197.185.219,"{""location"": ""GI"", ""is_mobile"": true}" 21792,8,488,2017-01-13 17:24:10,http://nicolas.co/andy.botsford,0.4402524021,73.156.26.52,"{""location"": ""BQ"", ""is_mobile"": false}" 21793,8,488,2017-02-17 11:04:02,http://lockman.com/mireille.wilderman,0.1332917525,221.144.248.240,"{""location"": ""JP"", ""is_mobile"": true}" 21794,8,488,2017-02-22 12:41:55,http://terrybailey.io/florine,0.3227951404,180.92.209.162,"{""location"": ""BI"", ""is_mobile"": true}" 21795,8,488,2017-03-08 13:32:42,http://kemmer.name/rodrigo_swift,0.2444938758,186.243.16.218,"{""location"": ""FK"", ""is_mobile"": true}" 21796,8,488,2017-02-27 16:57:55,http://runolfsdottir.org/rylee_johnson,0.2834792672,115.248.193.24,"{""location"": ""SX"", ""is_mobile"": false}" 21797,8,488,2017-03-22 11:58:57,http://dibbertwiegand.io/keira,0.7332519670,138.24.141.39,"{""location"": ""MA"", ""is_mobile"": false}" 21798,8,488,2017-05-24 18:59:34,http://daugherty.info/foster,0.2185154888,184.156.152.193,"{""location"": ""KH"", ""is_mobile"": false}" 21799,8,488,2017-03-27 07:02:00,http://okuneva.info/daniela,0.9419276221,87.160.54.29,"{""location"": ""KH"", ""is_mobile"": true}" 21800,8,488,2017-01-16 14:15:21,http://greenfeldertreutel.co/emilie_bernier,0.1394425954,41.215.28.116,"{""location"": ""HN"", ""is_mobile"": false}" 21801,8,488,2017-03-02 10:18:38,http://medhurst.biz/carleton_marks,0.8278235593,82.134.17.167,"{""location"": ""SV"", ""is_mobile"": true}" 21802,8,488,2017-05-08 16:34:00,http://weimann.name/leland,0.3856929690,191.146.19.241,"{""location"": ""BJ"", ""is_mobile"": false}" 21803,8,488,2017-01-12 05:23:42,http://lubowitz.org/jaime.metz,0.5373475400,70.165.40.134,"{""location"": ""TG"", ""is_mobile"": false}" 21804,8,488,2017-05-19 17:38:05,http://boyer.io/josie,0.9876154052,8.206.113.212,"{""location"": ""TM"", ""is_mobile"": true}" 21805,8,488,2017-04-29 20:27:53,http://block.biz/marshall,0.3348429259,30.251.83.117,"{""location"": ""TW"", ""is_mobile"": true}" 21806,8,488,2017-03-11 10:27:44,http://corkerykeler.com/josianne,0.6720744750,49.244.237.220,"{""location"": ""GI"", ""is_mobile"": true}" 21807,8,488,2017-01-05 14:13:57,http://breitenbergdibbert.info/judah,0.7766031453,68.41.60.87,"{""location"": ""VA"", ""is_mobile"": true}" 21808,8,488,2017-01-04 11:04:13,http://bashirian.io/pierre,0.1786664967,146.217.194.82,"{""location"": ""ME"", ""is_mobile"": false}" 21809,8,488,2017-01-09 23:53:16,http://bernhard.org/velva_legros,0.7089982213,227.221.51.229,"{""location"": ""MO"", ""is_mobile"": true}" 21810,8,488,2017-03-26 02:45:43,http://gulgowskigleichner.name/mafalda_schumm,0.2023853675,36.21.241.247,"{""location"": ""FI"", ""is_mobile"": true}" 21811,8,488,2017-05-18 15:50:23,http://kirlin.biz/douglas_klocko,0.3023890230,208.28.75.86,"{""location"": ""HR"", ""is_mobile"": true}" 21812,8,488,2017-01-25 21:11:20,http://rodriguez.name/clare,0.4892635181,252.4.13.13,"{""location"": ""RE"", ""is_mobile"": false}" 21813,8,488,2017-02-13 09:48:47,http://schinnerlabadie.biz/jayson,0.1221528004,218.187.110.85,"{""location"": ""IR"", ""is_mobile"": false}" 21814,8,488,2017-06-08 23:22:06,http://heathcote.net/dimitri_flatley,0.6490576647,11.139.170.56,"{""location"": ""EG"", ""is_mobile"": false}" 21815,8,488,2017-03-19 21:17:21,http://bartonheel.name/austyn,0.4020316482,91.218.91.24,"{""location"": ""BQ"", ""is_mobile"": false}" 21816,8,488,2017-02-01 23:39:13,http://marvin.name/piper_rippin,0.9101164925,250.221.67.133,"{""location"": ""HT"", ""is_mobile"": false}" 21817,8,488,2017-02-14 08:38:51,http://klocko.biz/stefan,0.0822747185,41.85.169.133,"{""location"": ""GL"", ""is_mobile"": false}" 21818,8,488,2017-04-12 18:31:36,http://lehnerkoch.net/arnoldo_witting,0.4666952118,105.92.206.73,"{""location"": ""SX"", ""is_mobile"": false}" 21819,8,488,2017-06-04 22:21:32,http://gaylord.biz/kaley.flatley,0.4092632984,131.79.148.212,"{""location"": ""IO"", ""is_mobile"": false}" 21820,8,488,2016-12-26 01:31:17,http://spinkaebert.net/matilda,0.8126084634,193.160.218.110,"{""location"": ""SX"", ""is_mobile"": false}" 21821,8,488,2017-02-07 06:53:01,http://barton.biz/griffin_kub,0.9745024520,186.41.162.19,"{""location"": ""HU"", ""is_mobile"": true}" 21822,8,488,2017-03-31 05:41:50,http://schulist.com/lacy,0.2727397450,97.50.68.95,"{""location"": ""BO"", ""is_mobile"": false}" 21823,8,488,2017-01-04 15:48:53,http://waterslangworth.org/madelynn.klocko,0.4687375992,56.26.20.45,"{""location"": ""MD"", ""is_mobile"": false}" 21824,8,488,2017-01-18 20:39:45,http://hahn.name/kenton.dach,0.7693017071,61.241.56.65,"{""location"": ""MO"", ""is_mobile"": true}" 21825,8,488,2017-03-26 22:21:37,http://waelchibayer.co/ottis,0.9488593833,132.226.14.163,"{""location"": ""SS"", ""is_mobile"": true}" 21826,8,488,2017-02-04 13:27:52,http://muller.info/edgar_miller,0.5950464935,92.220.201.205,"{""location"": ""IN"", ""is_mobile"": false}" 21827,8,488,2017-06-02 17:54:39,http://wolff.io/nickolas,0.6191351012,30.109.115.43,"{""location"": ""HR"", ""is_mobile"": true}" 21828,8,488,2017-01-23 06:20:10,http://dubuque.info/rylee,0.4682555364,101.134.67.60,"{""location"": ""UA"", ""is_mobile"": true}" 21829,8,488,2017-02-13 09:52:19,http://mueller.co/cordelia,0.5922319079,39.241.82.122,"{""location"": ""LT"", ""is_mobile"": false}" 21830,8,488,2017-04-04 20:37:07,http://kochswift.net/vernice,0.5752751676,10.153.47.146,"{""location"": ""SB"", ""is_mobile"": false}" 21831,8,488,2017-03-04 01:46:41,http://schaeferdonnelly.io/nyah.hoeger,0.4088153361,247.247.47.231,"{""location"": ""NL"", ""is_mobile"": true}" 21832,8,488,2017-04-24 01:53:55,http://osinskirogahn.org/delilah,0.2937071459,153.177.132.225,"{""location"": ""CH"", ""is_mobile"": true}" 21833,8,488,2017-02-28 00:55:04,http://veum.net/christophe.goyette,0.2027813896,25.223.182.214,"{""location"": ""NZ"", ""is_mobile"": true}" 21834,8,488,2017-02-07 15:58:00,http://mitchellhayes.com/sarina,0.0135614644,137.82.243.32,"{""location"": ""SA"", ""is_mobile"": false}" 21835,8,488,2017-03-04 23:31:38,http://moriette.org/jaunita.keler,0.7363914827,146.136.28.41,"{""location"": ""SL"", ""is_mobile"": true}" 21836,8,488,2017-03-10 07:29:47,http://kuphalcronin.name/van_becker,0.3450583617,4.96.68.22,"{""location"": ""BR"", ""is_mobile"": false}" 21837,8,488,2017-05-11 01:04:08,http://kutchkoch.com/emmanuelle.christiansen,0.6879781996,232.106.15.166,"{""location"": ""TT"", ""is_mobile"": true}" 21838,8,488,2017-04-23 19:23:58,http://ruel.org/joan,0.3817558213,108.89.12.177,"{""location"": ""MU"", ""is_mobile"": false}" 21839,8,488,2017-03-19 23:06:42,http://pfeffer.info/eduardo,0.4446939354,98.194.131.45,"{""location"": ""MG"", ""is_mobile"": false}" 21840,8,488,2016-12-14 12:28:54,http://okuneva.co/royce_baumbach,0.3246995558,183.228.151.185,"{""location"": ""KN"", ""is_mobile"": false}" 21841,8,488,2017-05-29 18:38:12,http://gutkowskideckow.io/gregory_gibson,0.2705500712,194.120.140.97,"{""location"": ""KZ"", ""is_mobile"": true}" 751,1,18,2017-02-20 00:04:01,http://wyman.net/eusebio,0.0451662661,81.96.130.197,"{""location"": ""KZ"", ""is_mobile"": true}" 752,1,18,2017-01-01 21:16:09,http://gutmann.name/margaret,0.6256827212,208.81.138.12,"{""location"": ""BY"", ""is_mobile"": false}" 753,1,18,2017-02-07 16:22:50,http://rolfson.co/frederick,0.1635584264,242.152.197.217,"{""location"": ""GH"", ""is_mobile"": false}" 754,1,18,2017-02-05 13:34:54,http://oberbrunnermiller.org/reina,0.2417983787,223.13.50.68,"{""location"": ""MN"", ""is_mobile"": true}" 755,1,18,2017-03-02 22:27:07,http://mayer.co/thomas.christiansen,0.5537420216,215.35.195.153,"{""location"": ""HR"", ""is_mobile"": false}" 756,1,18,2017-03-06 06:44:31,http://thompsonwalsh.io/nelle,0.2420303189,210.159.76.243,"{""location"": ""VA"", ""is_mobile"": false}" 757,1,18,2017-01-21 18:16:20,http://nader.info/camylle,0.5334830759,51.92.70.151,"{""location"": ""SE"", ""is_mobile"": true}" 758,1,18,2017-01-15 22:58:15,http://kirlin.com/afton,0.5876386675,50.110.23.192,"{""location"": ""FO"", ""is_mobile"": true}" 759,1,18,2017-04-17 22:54:09,http://nitzsche.co/corrine,0.1335697103,209.42.140.249,"{""location"": ""YE"", ""is_mobile"": true}" 760,1,18,2016-12-31 21:10:02,http://ritchiemante.com/jayme,0.6446955130,70.11.51.93,"{""location"": ""BY"", ""is_mobile"": true}" 761,1,18,2017-01-13 00:45:58,http://toy.org/arlo_nienow,0.6315342365,176.218.59.240,"{""location"": ""PY"", ""is_mobile"": false}" 762,1,18,2017-03-16 21:08:29,http://beatty.co/rachel.koepp,0.8466019604,136.68.23.53,"{""location"": ""KW"", ""is_mobile"": false}" 763,1,18,2017-04-21 01:19:46,http://breitenbergokuneva.name/kavon.mosciski,0.6582684567,149.220.54.102,"{""location"": ""NP"", ""is_mobile"": true}" 764,1,18,2017-01-28 00:48:43,http://upton.io/bruce.lynch,0.1104383495,153.142.16.172,"{""location"": ""BS"", ""is_mobile"": false}" 765,1,18,2017-03-13 10:53:14,http://emmerich.name/dagmar_hansen,0.0804609547,28.205.158.179,"{""location"": ""VC"", ""is_mobile"": false}" 766,1,18,2017-02-27 15:50:10,http://jacobs.name/abbigail.connelly,0.8983033512,24.185.14.98,"{""location"": ""RE"", ""is_mobile"": true}" 767,1,18,2017-03-06 08:37:19,http://treutel.com/turner,0.9228699549,91.140.183.49,"{""location"": ""CY"", ""is_mobile"": false}" 768,1,18,2017-03-28 00:10:51,http://gibson.com/jaqueline,0.1091751829,6.248.9.100,"{""location"": ""SG"", ""is_mobile"": true}" 769,1,18,2017-04-27 11:47:15,http://runolfon.org/mike_konopelski,0.4336912520,150.234.214.34,"{""location"": ""JO"", ""is_mobile"": false}" 770,1,18,2017-05-13 23:04:41,http://harris.com/linwood.schmitt,0.0522468381,104.70.200.72,"{""location"": ""SA"", ""is_mobile"": false}" 771,1,18,2017-04-22 09:32:25,http://dibbert.co/hillard.mosciski,0.6153340357,246.166.88.200,"{""location"": ""CV"", ""is_mobile"": false}" 772,1,18,2017-03-13 21:48:31,http://king.co/lilliana,0.1118251662,159.186.19.58,"{""location"": ""VG"", ""is_mobile"": false}" 773,1,18,2017-02-28 10:11:23,http://orn.net/antonetta_yost,0.4741321172,120.120.8.15,"{""location"": ""HK"", ""is_mobile"": true}" 774,1,18,2017-01-03 17:45:36,http://witting.org/edwina,0.2653390624,107.96.175.173,"{""location"": ""AW"", ""is_mobile"": false}" 775,1,18,2017-05-02 14:10:57,http://kovacek.name/matilde_wilkinson,0.0737117138,65.180.246.210,"{""location"": ""UA"", ""is_mobile"": false}" 776,1,18,2017-01-28 17:57:53,http://hahn.info/caandra_haley,0.8208549595,243.52.222.170,"{""location"": ""CG"", ""is_mobile"": true}" 777,1,18,2016-12-18 01:16:50,http://glover.info/makayla,0.5830523350,174.11.7.157,"{""location"": ""NU"", ""is_mobile"": false}" 778,1,18,2017-05-04 01:51:52,http://batzdubuque.com/merritt,0.6155353767,126.215.74.135,"{""location"": ""GI"", ""is_mobile"": false}" 779,1,18,2017-02-27 15:32:52,http://schadensteuber.com/caie,0.3121925003,57.4.214.213,"{""location"": ""ME"", ""is_mobile"": true}" 780,1,18,2016-12-29 02:15:05,http://naderkohler.net/jermey,0.9501712048,130.151.254.147,"{""location"": ""BL"", ""is_mobile"": false}" 781,1,18,2017-04-05 00:42:59,http://davis.name/annette_gulgowski,0.6905620332,176.52.103.120,"{""location"": ""SM"", ""is_mobile"": true}" 782,1,18,2016-12-30 23:57:57,http://hahnmckenzie.co/tom,0.3936842107,34.25.77.151,"{""location"": ""EE"", ""is_mobile"": true}" 783,1,18,2017-04-01 05:57:25,http://hermannbalistreri.io/sarina,0.1366392708,228.45.251.38,"{""location"": ""RS"", ""is_mobile"": true}" 784,1,18,2017-02-22 18:10:52,http://steuber.info/coleman,0.9944250901,186.28.6.108,"{""location"": ""UA"", ""is_mobile"": false}" 785,1,18,2017-03-22 23:59:52,http://feil.name/garnett_connelly,0.1955121309,129.231.100.200,"{""location"": ""YE"", ""is_mobile"": true}" 786,1,18,2017-06-03 04:56:51,http://hicklelindgren.org/aimee,0.0729385607,159.180.41.98,"{""location"": ""TD"", ""is_mobile"": true}" 787,1,18,2017-02-01 15:15:00,http://huels.org/breanne.volkman,0.7968237949,43.207.221.234,"{""location"": ""LA"", ""is_mobile"": false}" 788,1,18,2017-03-20 00:11:18,http://gibson.net/keely.blanda,0.0582424962,17.241.247.189,"{""location"": ""PH"", ""is_mobile"": false}" 789,1,18,2016-12-24 08:09:40,http://hayes.io/beulah,0.8458180782,159.46.64.251,"{""location"": ""SV"", ""is_mobile"": true}" 790,1,18,2017-03-03 10:37:37,http://dooley.org/craig_gottlieb,0.8960295729,56.82.97.194,"{""location"": ""IS"", ""is_mobile"": true}" 791,1,18,2017-05-13 00:43:56,http://rueckerdaugherty.com/alexandre,0.7177347752,27.22.143.190,"{""location"": ""CA"", ""is_mobile"": true}" 792,1,18,2017-03-01 18:16:11,http://haley.name/kirk.hamill,0.2563875139,188.216.5.102,"{""location"": ""CC"", ""is_mobile"": true}" 793,1,18,2017-01-23 13:27:35,http://hudsonlittel.io/eric.hackett,0.4302024840,187.93.33.141,"{""location"": ""YE"", ""is_mobile"": true}" 794,1,18,2017-01-13 22:21:53,http://deckow.info/wallace,0.9975462799,102.14.210.204,"{""location"": ""CD"", ""is_mobile"": false}" 795,1,18,2017-02-10 16:18:57,http://trantow.info/wilbert_douglas,0.0970885986,230.195.169.207,"{""location"": ""US"", ""is_mobile"": true}" 796,1,18,2017-04-24 04:20:48,http://ortiz.net/torrance_emard,0.5967184660,139.169.160.40,"{""location"": ""VN"", ""is_mobile"": false}" 797,1,18,2016-12-21 18:20:38,http://johnson.io/darrell,0.6137416592,170.64.228.189,"{""location"": ""IS"", ""is_mobile"": false}" 798,1,18,2017-03-12 08:54:22,http://kilback.io/felicita,0.6944138265,197.90.235.94,"{""location"": ""PN"", ""is_mobile"": true}" 799,1,18,2017-05-13 20:14:49,http://lowe.io/shany,0.3431985301,198.206.153.210,"{""location"": ""BZ"", ""is_mobile"": false}" 800,1,18,2017-04-23 10:50:59,http://schummfeest.co/eleazar,0.0548789691,159.204.152.129,"{""location"": ""BA"", ""is_mobile"": false}" 801,1,19,2017-06-08 23:34:36,http://kingbrekke.biz/juliet,0.3934337329,247.78.212.77,"{""location"": ""GU"", ""is_mobile"": true}" 802,1,19,2017-01-25 17:00:09,http://stokes.info/madilyn,0.7738837079,226.139.2.167,"{""location"": ""GN"", ""is_mobile"": false}" 21842,8,488,2017-02-22 13:19:50,http://walter.co/arne,0.4080410826,3.117.203.170,"{""location"": ""BS"", ""is_mobile"": false}" 21843,8,488,2017-04-20 09:30:33,http://dickinson.biz/curtis_beier,0.4579956626,116.58.137.218,"{""location"": ""MS"", ""is_mobile"": true}" 21844,8,488,2017-04-12 10:06:18,http://marks.net/vincenza_rice,0.3318336670,38.153.211.113,"{""location"": ""AU"", ""is_mobile"": false}" 21845,8,488,2017-03-20 14:16:55,http://cole.net/mireya,0.8093959832,24.76.49.45,"{""location"": ""VE"", ""is_mobile"": false}" 21846,8,488,2017-04-07 09:13:43,http://walter.org/lindsey_heaney,0.9339443366,105.191.18.58,"{""location"": ""ZW"", ""is_mobile"": false}" 21847,8,488,2017-04-06 19:09:20,http://schimmel.info/bianka.bednar,0.0055641447,207.238.22.60,"{""location"": ""IT"", ""is_mobile"": true}" 21848,8,488,2017-03-04 01:47:29,http://hermann.info/rose,0.7948881050,15.69.28.69,"{""location"": ""OM"", ""is_mobile"": true}" 21849,8,488,2017-05-10 08:12:11,http://smitham.com/roxane.prosacco,0.8427825736,101.209.132.21,"{""location"": ""CZ"", ""is_mobile"": true}" 21850,8,488,2016-12-28 03:04:55,http://thompson.org/perry.windler,0.6330362242,128.218.211.202,"{""location"": ""BS"", ""is_mobile"": false}" 21851,8,488,2017-02-17 14:01:24,http://block.name/mya_herman,0.3508962615,47.19.82.198,"{""location"": ""MV"", ""is_mobile"": false}" 21852,8,488,2017-01-23 14:49:02,http://oberbrunner.name/keanu,0.4701324920,182.180.153.121,"{""location"": ""CR"", ""is_mobile"": false}" 21853,8,488,2016-12-22 22:28:47,http://lowe.com/marcus_mills,0.3329101170,217.196.245.53,"{""location"": ""CU"", ""is_mobile"": true}" 21854,8,488,2017-04-06 19:04:51,http://toy.info/hulda.franecki,0.7587740847,117.208.245.249,"{""location"": ""NC"", ""is_mobile"": true}" 21855,8,488,2017-06-02 17:57:36,http://daughertygrant.co/amparo.conn,0.8407182363,202.239.46.178,"{""location"": ""TN"", ""is_mobile"": false}" 21856,8,489,2017-01-25 06:22:57,http://moriette.biz/kelli,0.0140220877,114.241.86.194,"{""location"": ""LS"", ""is_mobile"": false}" 21857,8,489,2016-12-18 01:25:52,http://westcummerata.info/sheila,0.7498768515,17.233.72.217,"{""location"": ""BQ"", ""is_mobile"": true}" 21858,8,489,2017-05-19 23:49:36,http://bartonzemlak.biz/gunnar.schroeder,0.5592423119,167.66.97.162,"{""location"": ""RO"", ""is_mobile"": true}" 21859,8,489,2017-01-20 04:46:57,http://mckenzie.io/breanne_haag,0.0710995074,240.202.54.137,"{""location"": ""MQ"", ""is_mobile"": false}" 21860,8,489,2017-04-08 01:31:31,http://mcdermottkreiger.biz/alvis,0.8268573056,36.215.118.59,"{""location"": ""CL"", ""is_mobile"": true}" 21861,8,489,2017-04-01 18:56:54,http://ritchie.info/quinn.baumbach,0.0980700460,190.78.224.205,"{""location"": ""IM"", ""is_mobile"": false}" 21862,8,489,2017-04-12 08:33:17,http://pouros.org/emelia.romaguera,0.4240368179,225.232.21.107,"{""location"": ""IE"", ""is_mobile"": false}" 21863,8,489,2017-04-29 05:23:49,http://stiedemannbahringer.net/audrey,0.6737351576,101.204.195.109,"{""location"": ""CW"", ""is_mobile"": false}" 21864,8,489,2017-05-22 05:35:39,http://aufderhardicki.info/eusebio,0.7029897514,240.103.44.109,"{""location"": ""PM"", ""is_mobile"": true}" 21865,8,489,2017-04-11 00:22:29,http://blanda.info/devon,0.1195803972,227.54.221.87,"{""location"": ""PY"", ""is_mobile"": true}" 21866,8,489,2017-04-19 10:59:11,http://considine.io/morton.kulas,0.0301423237,39.24.83.179,"{""location"": ""MQ"", ""is_mobile"": true}" 21867,8,489,2017-03-23 07:48:17,http://pacocha.biz/aleandro.conroy,0.9878631766,155.150.103.193,"{""location"": ""CL"", ""is_mobile"": true}" 21868,8,489,2017-04-08 21:57:27,http://schamberger.io/alanis.mills,0.8929679166,47.32.83.147,"{""location"": ""TF"", ""is_mobile"": true}" 21869,8,489,2017-01-30 09:06:16,http://fisher.io/taylor,0.3969409985,101.225.158.22,"{""location"": ""VG"", ""is_mobile"": true}" 21870,8,489,2017-02-13 07:02:24,http://daugherty.biz/furman,0.4926757208,91.99.210.251,"{""location"": ""ET"", ""is_mobile"": false}" 21871,8,489,2017-06-02 11:58:15,http://rath.name/barrett.johns,0.2556695858,239.23.190.63,"{""location"": ""BB"", ""is_mobile"": false}" 21872,8,489,2017-04-20 22:09:59,http://breitenberg.biz/adolf_kreiger,0.7862818294,220.251.97.111,"{""location"": ""IS"", ""is_mobile"": true}" 21873,8,489,2017-04-16 09:56:52,http://shields.io/roscoe,0.9282357653,38.53.139.64,"{""location"": ""DJ"", ""is_mobile"": true}" 21874,8,489,2017-05-25 18:18:52,http://denesik.io/clement_fahey,0.2463799169,45.86.181.5,"{""location"": ""MW"", ""is_mobile"": true}" 21875,8,489,2016-12-17 11:30:26,http://boylelangosh.name/wyatt.zboncak,0.1406781292,242.178.94.91,"{""location"": ""SG"", ""is_mobile"": false}" 21876,8,489,2017-05-18 15:42:14,http://hilll.co/bell,0.9663702052,92.59.55.95,"{""location"": ""DM"", ""is_mobile"": false}" 21877,8,489,2017-01-29 13:13:18,http://ohara.io/melya_schoen,0.8049172951,32.29.147.142,"{""location"": ""LS"", ""is_mobile"": false}" 21989,8,492,2017-01-31 20:48:33,http://bauch.name/percy,,240.206.93.248,"{""location"": ""QA"", ""is_mobile"": false}" 21878,8,489,2017-03-29 17:19:27,http://labadie.com/annie_huels,0.7538627358,109.208.147.203,"{""location"": ""TK"", ""is_mobile"": false}" 21879,8,489,2017-02-11 03:09:37,http://rice.biz/tommie,0.9471123250,56.54.82.25,"{""location"": ""RS"", ""is_mobile"": false}" 21880,8,489,2017-03-27 04:38:05,http://hintz.io/paris,0.7737793713,70.59.213.235,"{""location"": ""CU"", ""is_mobile"": false}" 21881,8,489,2017-03-24 19:44:18,http://swaniawskihaley.name/regan_upton,0.6596115147,30.199.28.122,"{""location"": ""SY"", ""is_mobile"": true}" 21882,8,489,2017-02-13 22:38:53,http://ernsercrona.io/kasandra.glover,0.5885219095,56.231.114.229,"{""location"": ""RS"", ""is_mobile"": true}" 21883,8,489,2017-03-06 05:21:08,http://torphyconn.biz/robb_mccullough,0.9161326618,129.191.243.99,"{""location"": ""IE"", ""is_mobile"": false}" 21884,8,489,2017-02-17 09:20:54,http://pacocha.name/eli.stark,0.6195179867,157.149.235.107,"{""location"": ""BE"", ""is_mobile"": false}" 21885,8,489,2017-05-03 01:47:03,http://labadieherzog.co/carolina.abshire,0.1682292038,95.82.94.59,"{""location"": ""FM"", ""is_mobile"": false}" 21886,8,489,2017-01-03 02:48:31,http://corkeryauer.biz/edgardo_murazik,0.3720517635,144.67.57.163,"{""location"": ""CL"", ""is_mobile"": false}" 21887,8,489,2017-02-21 21:36:19,http://ondrickafeeney.org/cecelia,0.1215024428,68.194.196.190,"{""location"": ""BW"", ""is_mobile"": false}" 21888,8,489,2017-02-03 03:13:54,http://batz.name/connie,0.5185656878,50.235.133.179,"{""location"": ""BL"", ""is_mobile"": false}" 21889,8,489,2017-05-19 03:44:40,http://robel.net/melya.kutch,0.5430641385,89.95.12.111,"{""location"": ""SN"", ""is_mobile"": false}" 21890,8,489,2017-03-19 09:22:09,http://ratke.co/viola.oconner,0.1777644443,164.177.182.194,"{""location"": ""KG"", ""is_mobile"": true}" 21891,8,489,2017-01-15 16:03:31,http://haag.info/alana,0.9844566038,8.41.58.123,"{""location"": ""IN"", ""is_mobile"": true}" 803,1,19,2017-04-13 12:20:15,http://wisozkfarrell.org/eileen,0.1171376036,208.162.29.248,"{""location"": ""TO"", ""is_mobile"": true}" 804,1,19,2016-12-30 08:06:53,http://brownhodkiewicz.com/orie,0.0038368401,105.105.194.185,"{""location"": ""GG"", ""is_mobile"": false}" 805,1,19,2017-04-02 13:21:45,http://wiegand.net/kari.crona,0.4246903807,141.2.183.67,"{""location"": ""SE"", ""is_mobile"": false}" 806,1,19,2017-04-06 02:34:07,http://mcclure.net/ova.dach,0.1150460794,48.242.111.233,"{""location"": ""HU"", ""is_mobile"": true}" 807,1,19,2017-01-27 19:42:09,http://hamill.net/matteo,0.9221379035,127.224.84.183,"{""location"": ""PH"", ""is_mobile"": false}" 808,1,19,2017-04-26 06:36:25,http://damorebode.biz/lelia,0.1675794789,57.198.52.133,"{""location"": ""BO"", ""is_mobile"": false}" 809,1,19,2017-05-25 08:40:48,http://marksmacejkovic.org/eloise,0.6201594615,103.78.135.172,"{""location"": ""NP"", ""is_mobile"": true}" 810,1,19,2017-02-11 21:33:08,http://kuhic.info/ephraim.lesch,0.9769909551,146.143.15.65,"{""location"": ""BA"", ""is_mobile"": false}" 811,1,19,2017-05-14 15:16:08,http://heathcote.io/abbigail,0.7874059671,181.162.252.55,"{""location"": ""FJ"", ""is_mobile"": false}" 812,1,19,2017-03-21 16:45:20,http://white.name/chet.welch,0.3103426479,30.244.232.219,"{""location"": ""LB"", ""is_mobile"": false}" 813,1,19,2016-12-16 08:29:06,http://renner.biz/jerrod_kulas,0.4851849961,160.171.98.152,"{""location"": ""CO"", ""is_mobile"": false}" 814,1,19,2016-12-26 12:38:56,http://bauchlakin.net/brook.schinner,0.9100687141,121.135.52.251,"{""location"": ""LC"", ""is_mobile"": true}" 815,1,19,2017-04-17 18:10:29,http://gerhold.biz/jasmin,0.2489739847,195.156.69.134,"{""location"": ""VC"", ""is_mobile"": true}" 816,1,19,2017-05-25 10:18:50,http://murazik.biz/shyann.raynor,0.5148751574,245.6.187.26,"{""location"": ""ME"", ""is_mobile"": false}" 817,1,19,2017-03-25 06:12:38,http://fritschpfannerstill.biz/dashawn.runte,0.1385088885,190.118.15.105,"{""location"": ""VI"", ""is_mobile"": true}" 818,1,19,2017-05-09 18:33:42,http://mclaughlinterry.info/beth_champlin,0.3268102818,236.157.108.222,"{""location"": ""SZ"", ""is_mobile"": false}" 819,1,19,2016-12-16 22:39:08,http://spencer.co/sam,0.8004854322,230.227.156.95,"{""location"": ""NC"", ""is_mobile"": true}" 820,1,19,2017-02-24 02:55:15,http://kautzer.name/kailey.wintheiser,0.4009802227,46.70.111.40,"{""location"": ""BV"", ""is_mobile"": false}" 821,1,19,2017-02-11 03:56:21,http://marquardt.com/abelardo,0.0467420544,228.239.191.40,"{""location"": ""BA"", ""is_mobile"": true}" 822,1,19,2016-12-28 13:46:31,http://walshgrant.net/duane_hudson,0.8670390488,226.252.246.221,"{""location"": ""DM"", ""is_mobile"": false}" 823,1,19,2017-04-10 00:50:50,http://dooleyfeest.com/mohamed.schroeder,0.8467371604,226.193.71.137,"{""location"": ""DO"", ""is_mobile"": true}" 824,1,20,2017-05-22 16:18:52,http://linderdman.name/narciso,0.2097181402,116.90.57.177,"{""location"": ""FK"", ""is_mobile"": false}" 825,1,20,2017-02-02 11:57:20,http://stamm.biz/keaton,0.0187961612,80.15.233.95,"{""location"": ""CC"", ""is_mobile"": true}" 826,1,20,2016-12-29 04:23:43,http://mccullough.org/flavio.jones,0.1148091326,90.136.10.47,"{""location"": ""NL"", ""is_mobile"": false}" 827,1,20,2017-02-19 18:06:55,http://lesch.com/lucie,0.5494479207,245.112.117.49,"{""location"": ""CC"", ""is_mobile"": true}" 828,1,20,2017-03-23 21:20:33,http://romaguera.net/modesta_gerhold,0.2888483720,236.139.144.223,"{""location"": ""AF"", ""is_mobile"": true}" 829,1,20,2017-03-30 13:00:15,http://stehrbeahan.name/astrid_pfannerstill,0.8905790955,58.24.249.144,"{""location"": ""VA"", ""is_mobile"": false}" 830,1,20,2016-12-13 23:09:47,http://rice.info/vesta,0.4011229174,75.84.251.235,"{""location"": ""NP"", ""is_mobile"": false}" 831,1,20,2017-02-15 13:23:45,http://kertzmann.info/beverly.lebsack,0.4227755444,14.36.229.219,"{""location"": ""PY"", ""is_mobile"": true}" 832,1,20,2017-01-22 00:18:04,http://fritsch.org/devan_crona,0.1649725299,113.240.242.97,"{""location"": ""AM"", ""is_mobile"": false}" 833,1,20,2017-05-25 15:49:31,http://larson.org/maegan_cain,0.2348967356,84.96.199.21,"{""location"": ""FM"", ""is_mobile"": false}" 834,1,20,2017-01-08 15:57:12,http://fadel.info/kurt.kemmer,0.0386258288,64.100.244.8,"{""location"": ""MG"", ""is_mobile"": false}" 835,1,20,2017-04-05 16:39:35,http://roberts.io/ryleigh,0.4347350402,249.77.216.225,"{""location"": ""AM"", ""is_mobile"": true}" 836,1,20,2017-04-14 10:55:19,http://ornerdman.io/clemens,0.8306348925,164.155.175.111,"{""location"": ""NR"", ""is_mobile"": false}" 837,1,20,2017-02-27 10:54:49,http://nolanritchie.name/dixie,0.3079787572,84.9.127.128,"{""location"": ""IT"", ""is_mobile"": true}" 838,1,20,2017-02-23 12:53:22,http://boscowaelchi.com/mireya_streich,0.2183921473,92.62.213.126,"{""location"": ""PH"", ""is_mobile"": false}" 839,1,20,2017-01-22 23:43:28,http://rau.com/abdullah.wolf,0.3966417210,66.203.244.64,"{""location"": ""CG"", ""is_mobile"": false}" 840,1,20,2017-02-03 02:58:22,http://kaulke.com/adelle,0.9115346182,219.30.59.67,"{""location"": ""MO"", ""is_mobile"": true}" 841,1,20,2017-02-28 21:00:01,http://price.co/josue,0.7539135725,76.110.79.197,"{""location"": ""YE"", ""is_mobile"": false}" 842,1,20,2017-06-09 18:52:13,http://mcglynnmoen.org/zackary.zieme,0.6834245460,84.72.155.230,"{""location"": ""MK"", ""is_mobile"": false}" 843,1,20,2017-05-31 20:15:42,http://kulas.biz/drake.abshire,0.5185080765,179.92.167.152,"{""location"": ""GY"", ""is_mobile"": false}" 844,1,20,2017-05-01 02:02:48,http://swaniawskicole.info/yeenia,0.8779527512,28.85.72.84,"{""location"": ""EG"", ""is_mobile"": true}" 845,1,20,2017-05-28 21:00:35,http://kertzmann.com/eugene.wehner,0.1402232253,223.111.17.130,"{""location"": ""TD"", ""is_mobile"": true}" 846,1,20,2017-05-07 21:47:14,http://green.net/marc_casper,0.8514443337,164.114.247.117,"{""location"": ""BR"", ""is_mobile"": false}" 847,1,20,2017-01-03 17:34:28,http://kozey.name/chasity.zulauf,0.3450031712,81.28.120.162,"{""location"": ""TF"", ""is_mobile"": true}" 848,1,20,2017-01-04 15:30:56,http://kaulke.org/walton_okon,0.6617639823,94.229.142.156,"{""location"": ""MD"", ""is_mobile"": false}" 849,1,20,2017-04-20 09:01:07,http://olson.com/velma,0.7203800481,62.249.135.202,"{""location"": ""MR"", ""is_mobile"": false}" 850,1,20,2016-12-22 19:05:27,http://labadie.biz/allene.will,0.0024943608,225.106.68.173,"{""location"": ""BR"", ""is_mobile"": true}" 851,1,20,2017-04-16 05:08:10,http://ward.biz/cindy,0.6024942474,166.124.159.5,"{""location"": ""KM"", ""is_mobile"": false}" 852,1,20,2017-05-19 09:22:25,http://olson.com/carmel,0.1488134424,52.166.125.151,"{""location"": ""SG"", ""is_mobile"": true}" 853,1,20,2017-05-25 21:38:48,http://vandervortrippin.org/ernestine.howe,0.2650837840,214.64.194.46,"{""location"": ""GS"", ""is_mobile"": false}" 854,1,20,2017-05-25 18:20:16,http://stiedemann.biz/mustafa,0.0263490493,209.151.221.78,"{""location"": ""NG"", ""is_mobile"": true}" 21892,8,489,2016-12-17 06:30:08,http://ortiz.com/alda,0.6023445224,58.233.49.18,"{""location"": ""AM"", ""is_mobile"": true}" 21893,8,489,2017-02-23 19:58:32,http://goyette.co/geovanny,0.2007577124,247.227.121.48,"{""location"": ""TV"", ""is_mobile"": false}" 21894,8,489,2017-05-07 20:27:42,http://jaskolski.info/remington.murray,0.8427672193,216.120.140.230,"{""location"": ""UA"", ""is_mobile"": true}" 21895,8,489,2016-12-14 16:15:44,http://pagac.net/chance_stamm,0.5350730661,145.66.152.177,"{""location"": ""GP"", ""is_mobile"": true}" 21896,8,489,2017-03-05 23:11:21,http://grimes.name/stan_mayer,0.4073511901,209.13.85.126,"{""location"": ""ME"", ""is_mobile"": false}" 21897,8,489,2017-04-17 01:42:27,http://hauckhills.io/makenna.parisian,0.7167090787,5.133.87.153,"{""location"": ""DE"", ""is_mobile"": true}" 21898,8,489,2017-02-22 03:19:32,http://auer.biz/shawn.carter,0.4184733219,157.96.77.129,"{""location"": ""NO"", ""is_mobile"": false}" 21899,8,489,2017-02-24 16:32:08,http://mcdermott.net/alycia_witting,0.9910156809,152.58.187.54,"{""location"": ""VI"", ""is_mobile"": true}" 21900,8,489,2017-01-12 23:43:53,http://bogan.info/jadon,0.3330589334,52.235.96.235,"{""location"": ""NL"", ""is_mobile"": false}" 21901,8,489,2017-02-06 21:42:25,http://huels.co/evie.steuber,0.3816273574,224.48.30.177,"{""location"": ""SZ"", ""is_mobile"": false}" 21902,8,489,2016-12-27 23:27:22,http://wisozk.org/cortney.mckenzie,0.5390909976,38.162.67.196,"{""location"": ""NL"", ""is_mobile"": true}" 21903,8,489,2017-05-10 06:12:22,http://huel.name/frieda_schmeler,0.8055094695,37.243.129.95,"{""location"": ""PE"", ""is_mobile"": true}" 21904,8,489,2017-02-19 18:18:10,http://tremblay.com/rosalia,0.0770557598,37.98.139.230,"{""location"": ""CK"", ""is_mobile"": false}" 21905,8,489,2017-03-20 05:48:54,http://gleichnerrohan.biz/tyrell,0.0105190866,163.94.82.60,"{""location"": ""LC"", ""is_mobile"": false}" 21906,8,489,2017-01-18 22:27:03,http://hand.com/annamae_padberg,0.0083998587,228.139.223.102,"{""location"": ""AO"", ""is_mobile"": false}" 21907,8,489,2017-06-12 03:08:48,http://hane.name/jarvis,0.2548298667,141.237.19.173,"{""location"": ""JM"", ""is_mobile"": true}" 21908,8,489,2017-01-11 20:09:50,http://williamsonsteuber.net/darrick,0.1037917908,177.105.138.239,"{""location"": ""MF"", ""is_mobile"": false}" 21909,8,489,2017-01-18 14:07:40,http://lubowitz.info/markus_reichel,0.2059291003,237.145.214.151,"{""location"": ""JO"", ""is_mobile"": false}" 21910,8,489,2017-04-20 12:55:00,http://koelpin.org/carolyne_hayes,0.0381646134,121.96.157.132,"{""location"": ""BO"", ""is_mobile"": false}" 21911,8,489,2017-04-06 23:29:15,http://zemlakhirthe.co/amanda,0.7584330896,114.219.142.97,"{""location"": ""DZ"", ""is_mobile"": false}" 21912,8,489,2016-12-13 11:17:15,http://grimeskilback.biz/selmer.will,0.0357965345,160.224.220.223,"{""location"": ""ZA"", ""is_mobile"": true}" 21913,8,490,2016-12-27 21:21:14,http://bode.io/sunny.langworth,0.3109579430,11.19.6.197,"{""location"": ""IM"", ""is_mobile"": true}" 21914,8,490,2017-02-18 11:21:18,http://greenfelder.net/vicente.kilback,0.6957801000,109.46.5.30,"{""location"": ""TR"", ""is_mobile"": false}" 21915,8,490,2017-03-04 15:29:55,http://larkin.net/rubie_ernser,0.3807046908,60.179.26.13,"{""location"": ""NZ"", ""is_mobile"": true}" 21916,8,490,2017-01-07 11:43:12,http://murphy.name/bennie_bednar,0.5598230178,132.98.39.221,"{""location"": ""KG"", ""is_mobile"": true}" 21917,8,490,2017-01-03 05:21:52,http://damore.co/opal_bogan,0.0406417262,188.109.125.54,"{""location"": ""UA"", ""is_mobile"": true}" 21918,8,490,2017-01-06 15:13:25,http://rogahnwisoky.io/dell_okon,0.7292600498,2.88.140.238,"{""location"": ""NP"", ""is_mobile"": true}" 21919,8,490,2017-04-29 22:42:59,http://torphy.com/tiara_graham,0.8622101786,205.91.89.76,"{""location"": ""BW"", ""is_mobile"": true}" 21920,8,490,2017-02-26 06:39:19,http://ullrich.net/kurtis,0.7291428433,78.202.19.171,"{""location"": ""HU"", ""is_mobile"": false}" 21921,8,490,2017-03-19 00:15:19,http://schmitt.co/toney,0.3567097994,151.101.195.146,"{""location"": ""GA"", ""is_mobile"": true}" 21922,8,490,2017-04-26 15:37:57,http://stoltenbergkohler.biz/orland.crona,0.7370917642,158.36.48.59,"{""location"": ""MA"", ""is_mobile"": false}" 21923,8,490,2017-05-27 17:23:56,http://harvey.co/sherwood,0.2529324571,80.28.183.176,"{""location"": ""AE"", ""is_mobile"": false}" 21924,8,490,2017-02-10 19:26:19,http://harber.biz/irving,0.0035304287,142.83.111.54,"{""location"": ""PS"", ""is_mobile"": false}" 21925,8,490,2017-02-10 01:27:57,http://schuppe.name/mara_reichert,0.6799832781,231.41.200.41,"{""location"": ""IR"", ""is_mobile"": true}" 21926,8,490,2017-01-04 13:40:54,http://jacobi.io/narciso.gibson,0.2341808101,102.175.191.121,"{""location"": ""KW"", ""is_mobile"": true}" 21927,8,490,2016-12-13 19:14:40,http://greenholt.biz/murphy.lind,0.1048775930,208.176.155.69,"{""location"": ""GA"", ""is_mobile"": true}" 21928,8,490,2016-12-31 08:06:01,http://wuckertwiegand.co/trever,0.1052124652,181.34.126.222,"{""location"": ""AZ"", ""is_mobile"": true}" 21929,8,490,2017-03-25 09:27:24,http://macejkovictreutel.co/tia_moriette,0.6995864642,61.82.126.23,"{""location"": ""HN"", ""is_mobile"": false}" 21930,8,490,2017-04-15 16:45:22,http://jast.com/gonzalo,0.9048051221,238.85.151.180,"{""location"": ""AD"", ""is_mobile"": false}" 21931,8,490,2017-03-06 05:23:43,http://smith.name/keeley,0.3379483722,2.171.16.120,"{""location"": ""LY"", ""is_mobile"": false}" 21932,8,490,2016-12-14 00:50:04,http://emard.biz/rosanna_armstrong,0.7517466397,159.199.166.21,"{""location"": ""TH"", ""is_mobile"": false}" 21933,8,490,2017-02-21 15:55:50,http://reynolds.com/mikel.kovacek,0.4558595844,253.158.68.238,"{""location"": ""AQ"", ""is_mobile"": true}" 21934,8,490,2017-04-23 00:28:49,http://fadel.net/alene,0.9781206776,61.99.159.15,"{""location"": ""SO"", ""is_mobile"": true}" 21935,8,490,2017-04-25 03:40:24,http://balistreri.org/ferne,0.8138392533,136.212.201.102,"{""location"": ""IS"", ""is_mobile"": true}" 21936,8,490,2017-05-14 03:59:18,http://steuber.com/burdette,0.2134963163,201.228.116.68,"{""location"": ""DO"", ""is_mobile"": false}" 21937,8,490,2017-01-17 16:29:58,http://smith.co/hilma,0.8588099612,151.55.132.197,"{""location"": ""SZ"", ""is_mobile"": true}" 21938,8,490,2017-03-01 04:40:06,http://wiza.info/corbin,0.9284762098,64.61.36.16,"{""location"": ""CC"", ""is_mobile"": true}" 21939,8,490,2017-04-05 04:38:03,http://stantonmetz.name/anibal,0.6200608947,115.40.239.252,"{""location"": ""FI"", ""is_mobile"": true}" 21940,8,490,2017-02-13 09:48:05,http://purdy.org/korey,0.5614479491,76.185.162.10,"{""location"": ""PT"", ""is_mobile"": true}" 21941,8,490,2017-02-17 20:11:38,http://shields.info/sophia,0.6259998419,203.19.244.226,"{""location"": ""TC"", ""is_mobile"": false}" 21942,8,490,2017-02-09 19:30:12,http://boyerkerluke.name/tyrese_mayer,0.8148552727,254.208.112.78,"{""location"": ""VC"", ""is_mobile"": true}" 855,1,20,2017-04-24 07:18:39,http://kautzerborer.co/vaughn,0.3189389165,198.21.149.180,"{""location"": ""CO"", ""is_mobile"": true}" 856,1,20,2016-12-27 01:38:13,http://welch.net/jeff_heaney,0.9219788862,144.150.139.14,"{""location"": ""ZA"", ""is_mobile"": true}" 857,1,20,2017-06-05 20:20:29,http://terry.io/jada_little,0.6553629451,72.242.165.2,"{""location"": ""TG"", ""is_mobile"": true}" 858,1,20,2016-12-23 12:45:31,http://lubowitz.info/destinee.pacocha,0.6747019281,148.231.191.83,"{""location"": ""CN"", ""is_mobile"": false}" 859,1,20,2017-06-04 05:43:00,http://kerluke.com/olen,0.1600807104,205.55.12.116,"{""location"": ""UY"", ""is_mobile"": false}" 860,1,20,2017-03-22 14:18:40,http://leannon.biz/sammie.stoltenberg,0.5504225740,89.157.24.40,"{""location"": ""FO"", ""is_mobile"": true}" 861,1,20,2017-03-14 15:40:45,http://sengermarvin.com/roxane,0.7807496842,115.188.252.28,"{""location"": ""TK"", ""is_mobile"": false}" 862,1,20,2017-03-22 15:43:22,http://andersonhoeger.org/darwin_rippin,0.5197891442,100.6.191.47,"{""location"": ""PR"", ""is_mobile"": false}" 863,1,20,2017-05-03 13:38:30,http://ziemann.org/adela_murazik,0.3687506406,236.127.245.147,"{""location"": ""MT"", ""is_mobile"": false}" 864,1,20,2017-01-20 05:42:22,http://medhursthaley.biz/joanie,0.5862176499,61.175.181.96,"{""location"": ""US"", ""is_mobile"": false}" 865,1,20,2017-01-11 03:08:34,http://lubowitzoconnell.name/alexis_cummings,0.1274407045,223.59.245.128,"{""location"": ""LC"", ""is_mobile"": true}" 866,1,20,2017-06-06 22:36:57,http://vonruedensanford.io/markus,0.7750341961,168.74.156.71,"{""location"": ""CK"", ""is_mobile"": false}" 867,1,20,2017-05-07 07:50:48,http://bergstrom.biz/dahlia.lehner,0.6143930161,189.187.141.254,"{""location"": ""ES"", ""is_mobile"": false}" 868,1,20,2017-02-12 21:32:37,http://roob.org/joany.hagenes,0.4636330655,136.225.253.135,"{""location"": ""AS"", ""is_mobile"": true}" 869,1,20,2016-12-25 12:16:45,http://pfannerstillmertz.name/marguerite,0.5750866905,107.226.151.230,"{""location"": ""TG"", ""is_mobile"": false}" 870,1,20,2017-06-06 16:56:53,http://jacobi.co/waino,0.2855380353,37.229.91.190,"{""location"": ""PK"", ""is_mobile"": true}" 871,1,20,2017-05-06 16:39:22,http://hodkiewicz.io/addison,0.5566513299,78.89.10.249,"{""location"": ""NP"", ""is_mobile"": false}" 872,1,20,2017-04-28 07:49:51,http://oconnellprosacco.biz/gia,0.1959820837,162.193.169.182,"{""location"": ""RW"", ""is_mobile"": true}" 873,1,20,2017-03-07 17:38:47,http://wilderman.name/citlalli_turner,0.8670163239,72.175.161.3,"{""location"": ""EG"", ""is_mobile"": false}" 874,1,20,2017-01-09 07:47:53,http://kunde.biz/gabriel.volkman,0.8305926303,26.119.129.24,"{""location"": ""DO"", ""is_mobile"": true}" 875,1,20,2017-02-22 16:51:06,http://reinger.co/ari.block,0.9663057902,28.220.159.216,"{""location"": ""LR"", ""is_mobile"": true}" 876,1,20,2017-03-25 14:22:59,http://keebler.io/jarret,0.3390688536,203.102.23.158,"{""location"": ""TR"", ""is_mobile"": false}" 877,1,20,2017-03-12 00:21:54,http://wiza.biz/jaren.zulauf,0.9209901300,169.172.176.27,"{""location"": ""SA"", ""is_mobile"": false}" 878,1,20,2017-02-06 17:42:33,http://yoststamm.biz/coby,0.8376356693,57.130.9.179,"{""location"": ""AR"", ""is_mobile"": true}" 879,1,20,2017-05-23 23:31:22,http://bernhard.co/ashton,0.7039780780,251.123.36.95,"{""location"": ""KW"", ""is_mobile"": true}" 880,1,20,2017-05-19 22:21:58,http://cain.org/tania.nienow,0.5293478831,21.24.240.245,"{""location"": ""CA"", ""is_mobile"": true}" 881,1,20,2017-01-27 10:29:49,http://nienow.org/rosalee.jacobs,0.4032274376,136.180.98.14,"{""location"": ""MH"", ""is_mobile"": false}" 882,1,20,2017-01-04 21:49:06,http://sauer.net/cathryn.berge,0.8179447191,128.227.45.100,"{""location"": ""LI"", ""is_mobile"": true}" 883,1,20,2017-01-21 11:33:22,http://fritsch.net/zelda,0.6736928933,88.221.30.250,"{""location"": ""AG"", ""is_mobile"": true}" 884,1,21,2017-01-30 00:05:30,http://bednar.com/andreanne,,245.227.7.244,"{""location"": ""SY"", ""is_mobile"": true}" 885,1,21,2017-02-17 07:30:35,http://vonrueden.org/alvera,,5.134.42.97,"{""location"": ""GR"", ""is_mobile"": false}" 886,1,21,2017-02-02 13:49:25,http://sipesking.co/paul_emmerich,,225.168.159.171,"{""location"": ""MR"", ""is_mobile"": false}" 887,1,21,2017-03-20 23:01:47,http://murrayzulauf.org/michelle_jaskolski,,91.217.45.213,"{""location"": ""TH"", ""is_mobile"": true}" 888,1,21,2017-01-18 17:03:50,http://roobwillms.com/marion.nikolaus,,19.19.78.170,"{""location"": ""SZ"", ""is_mobile"": false}" 889,1,21,2017-02-24 05:29:22,http://grant.net/meda_romaguera,,147.206.10.180,"{""location"": ""LY"", ""is_mobile"": true}" 890,1,21,2017-02-23 06:17:47,http://ruel.biz/johnny_carroll,,148.87.172.183,"{""location"": ""BZ"", ""is_mobile"": false}" 891,1,21,2017-04-13 03:08:36,http://kaulke.net/genevieve,,224.127.60.22,"{""location"": ""BH"", ""is_mobile"": false}" 892,1,21,2017-01-20 21:49:48,http://runte.net/jerry_bechtelar,,138.74.139.5,"{""location"": ""NP"", ""is_mobile"": true}" 893,1,21,2017-01-07 10:18:29,http://auer.info/lucius.breitenberg,,206.176.71.168,"{""location"": ""LS"", ""is_mobile"": true}" 894,1,21,2016-12-20 17:44:34,http://borer.org/vincenzo_thiel,,244.165.137.186,"{""location"": ""MH"", ""is_mobile"": false}" 895,1,21,2017-01-04 20:44:54,http://feest.co/natasha_mcdermott,,107.178.183.138,"{""location"": ""EE"", ""is_mobile"": false}" 896,1,21,2017-05-04 21:47:25,http://schimmel.net/zakary,,204.109.147.66,"{""location"": ""HU"", ""is_mobile"": true}" 897,1,21,2017-02-18 10:02:10,http://effertz.biz/joy,,197.217.230.96,"{""location"": ""MC"", ""is_mobile"": false}" 898,1,21,2017-03-20 12:30:59,http://waelchi.io/frederic,,222.141.161.25,"{""location"": ""VE"", ""is_mobile"": true}" 899,1,21,2017-03-28 03:39:59,http://nicolaswolf.biz/maximus_spencer,,184.180.61.80,"{""location"": ""PW"", ""is_mobile"": true}" 900,1,21,2017-04-29 01:18:09,http://skilesbosco.net/rosetta_kiehn,,102.151.221.141,"{""location"": ""MW"", ""is_mobile"": true}" 901,1,21,2016-12-27 19:10:41,http://cruickshank.biz/carole.little,,144.111.102.123,"{""location"": ""AZ"", ""is_mobile"": true}" 902,1,21,2017-05-01 15:58:32,http://framikeler.org/jayce.zulauf,,48.131.78.160,"{""location"": ""SD"", ""is_mobile"": true}" 903,1,21,2017-05-27 10:06:35,http://lubowitz.org/odell.sauer,,136.226.141.190,"{""location"": ""BN"", ""is_mobile"": false}" 904,1,21,2017-05-26 12:11:55,http://cormier.info/oran.wilderman,,176.182.15.22,"{""location"": ""NC"", ""is_mobile"": true}" 905,1,21,2017-02-20 11:25:10,http://adams.co/daisy,,116.85.107.14,"{""location"": ""FM"", ""is_mobile"": true}" 906,1,21,2017-01-27 16:57:04,http://towne.io/eldred,,127.134.204.109,"{""location"": ""KM"", ""is_mobile"": true}" 907,1,21,2017-03-05 19:47:31,http://rodriguezluettgen.co/otilia.erdman,,92.63.244.180,"{""location"": ""CL"", ""is_mobile"": false}" 908,1,21,2017-05-23 11:13:38,http://renner.net/melvin.bins,,103.5.249.148,"{""location"": ""LC"", ""is_mobile"": true}" 21943,8,490,2017-04-30 13:14:37,http://heller.io/reva_leuschke,0.6230696998,245.250.106.74,"{""location"": ""CM"", ""is_mobile"": false}" 21944,8,490,2017-03-10 13:46:09,http://abshirecummings.net/breana,0.8880883751,201.173.104.84,"{""location"": ""DZ"", ""is_mobile"": true}" 21945,8,490,2017-05-12 22:36:21,http://prosaccohegmann.org/jeremie.reynolds,0.8660166605,18.52.109.187,"{""location"": ""PK"", ""is_mobile"": false}" 21946,8,490,2017-04-20 02:38:32,http://johns.info/jeika,0.7113250017,81.61.139.203,"{""location"": ""BL"", ""is_mobile"": false}" 21947,8,490,2017-03-27 15:02:14,http://kilback.biz/jaycee.moen,0.3421443151,211.223.91.223,"{""location"": ""SH"", ""is_mobile"": false}" 21948,8,490,2017-01-24 10:18:37,http://raynorhegmann.org/andy,0.1405840203,182.202.228.145,"{""location"": ""MA"", ""is_mobile"": false}" 21949,8,490,2017-03-30 01:16:43,http://vandervort.org/jeffry_strosin,0.9350835244,90.195.161.188,"{""location"": ""PL"", ""is_mobile"": false}" 21950,8,491,2017-03-02 19:15:09,http://champlin.org/deie,,234.8.246.49,"{""location"": ""PK"", ""is_mobile"": false}" 21951,8,491,2017-01-07 12:49:51,http://hand.info/hildegard,,68.125.71.247,"{""location"": ""SC"", ""is_mobile"": false}" 21952,8,491,2017-01-20 23:42:23,http://bashirian.com/trent_batz,,119.154.206.69,"{""location"": ""JE"", ""is_mobile"": false}" 21953,8,491,2016-12-16 19:36:00,http://binstromp.biz/arvid,,115.119.55.236,"{""location"": ""MW"", ""is_mobile"": true}" 21954,8,491,2017-02-03 22:50:08,http://sawayncruickshank.net/bria,,221.179.170.60,"{""location"": ""UZ"", ""is_mobile"": false}" 21955,8,491,2016-12-17 01:05:51,http://schadenmann.com/coralie,,38.86.118.172,"{""location"": ""MC"", ""is_mobile"": false}" 21956,8,491,2017-05-05 21:36:02,http://reichert.org/anabel.spencer,,89.77.173.5,"{""location"": ""MD"", ""is_mobile"": false}" 21957,8,491,2016-12-13 09:28:09,http://donnellylabadie.com/antonietta,,181.6.59.27,"{""location"": ""SB"", ""is_mobile"": false}" 21958,8,491,2017-01-31 10:39:09,http://adamsking.com/abraham_wyman,,19.190.115.115,"{""location"": ""YT"", ""is_mobile"": false}" 21959,8,491,2017-03-21 06:23:19,http://prohaska.biz/vincenza.gutkowski,,54.112.138.141,"{""location"": ""KP"", ""is_mobile"": false}" 21960,8,491,2016-12-15 23:05:37,http://hettinger.com/harrison.hodkiewicz,,36.99.142.163,"{""location"": ""FJ"", ""is_mobile"": true}" 21961,8,491,2016-12-20 15:00:00,http://kozey.net/mandy.prosacco,,32.167.84.95,"{""location"": ""BS"", ""is_mobile"": false}" 21962,8,491,2017-03-19 02:49:38,http://cremin.name/nannie.konopelski,,151.214.16.82,"{""location"": ""BJ"", ""is_mobile"": false}" 21963,8,491,2016-12-28 22:29:23,http://bahringer.info/palma,,133.94.148.190,"{""location"": ""ME"", ""is_mobile"": false}" 21964,8,491,2017-05-29 21:22:47,http://blick.org/graciela_green,,47.195.57.154,"{""location"": ""DK"", ""is_mobile"": true}" 21965,8,491,2017-03-10 01:46:06,http://jakubowski.net/carley,,146.20.27.20,"{""location"": ""SA"", ""is_mobile"": true}" 21966,8,491,2017-04-05 10:42:48,http://bartoletti.biz/giovanni_buckridge,,254.77.2.32,"{""location"": ""CH"", ""is_mobile"": false}" 21967,8,491,2017-02-12 14:31:54,http://grimesboyle.biz/dorothy,,89.124.68.64,"{""location"": ""AD"", ""is_mobile"": false}" 21968,8,491,2016-12-25 21:19:27,http://jastroberts.info/lonzo,,49.106.162.10,"{""location"": ""SG"", ""is_mobile"": true}" 21969,8,491,2017-04-15 19:14:36,http://casper.name/esther.morar,,210.164.116.167,"{""location"": ""KN"", ""is_mobile"": false}" 21970,8,491,2017-03-02 19:28:50,http://thompson.biz/laurine,,222.124.33.186,"{""location"": ""GN"", ""is_mobile"": true}" 21971,8,491,2017-02-05 03:27:21,http://ernser.biz/pearlie_hyatt,,171.25.119.156,"{""location"": ""YE"", ""is_mobile"": false}" 21972,8,491,2017-05-01 21:56:48,http://hintz.org/stacy.oconner,,250.227.210.90,"{""location"": ""VE"", ""is_mobile"": true}" 21973,8,491,2017-03-10 08:19:15,http://oreilly.co/esta,,228.18.5.215,"{""location"": ""SJ"", ""is_mobile"": false}" 21974,8,491,2017-04-19 15:03:34,http://hanebeahan.net/xavier,,107.232.112.244,"{""location"": ""NF"", ""is_mobile"": true}" 21975,8,491,2016-12-17 20:28:19,http://hegmann.biz/jarrod.abbott,,50.13.114.123,"{""location"": ""GW"", ""is_mobile"": true}" 21976,8,491,2016-12-21 18:51:17,http://thiel.biz/norma_schmitt,,106.155.103.236,"{""location"": ""BI"", ""is_mobile"": false}" 21977,8,491,2017-04-29 00:53:28,http://bosco.net/maryam,,103.28.239.126,"{""location"": ""AG"", ""is_mobile"": false}" 21978,8,491,2017-02-06 13:27:52,http://klein.io/wendell,,118.210.40.222,"{""location"": ""FO"", ""is_mobile"": false}" 21979,8,491,2017-02-12 22:51:54,http://leannon.co/laurie,,120.232.60.19,"{""location"": ""LS"", ""is_mobile"": true}" 21980,8,491,2017-03-12 20:57:41,http://mohrlarson.info/henry.emmerich,,38.242.154.219,"{""location"": ""IR"", ""is_mobile"": true}" 21981,8,491,2017-01-19 07:42:06,http://littel.com/kaley,,38.176.207.118,"{""location"": ""AQ"", ""is_mobile"": true}" 21982,8,491,2017-06-11 02:16:09,http://balistreri.org/estrella,,101.60.159.103,"{""location"": ""AW"", ""is_mobile"": false}" 21983,8,491,2017-03-01 03:47:17,http://streich.org/arlo.schmitt,,27.103.205.92,"{""location"": ""ST"", ""is_mobile"": true}" 21984,8,491,2017-05-05 09:15:20,http://runolfon.biz/deborah,,171.136.111.246,"{""location"": ""KN"", ""is_mobile"": false}" 21985,8,491,2017-03-31 02:54:54,http://haley.biz/matt_hermann,,64.206.251.151,"{""location"": ""BL"", ""is_mobile"": false}" 21986,8,492,2017-01-11 00:51:48,http://monahan.co/nat,,243.155.198.230,"{""location"": ""HT"", ""is_mobile"": false}" 21987,8,492,2017-02-01 10:52:23,http://streichreichel.co/marvin.donnelly,,209.3.113.85,"{""location"": ""MG"", ""is_mobile"": false}" 21988,8,492,2016-12-31 11:16:37,http://purdy.net/jamal,,172.159.253.45,"{""location"": ""SC"", ""is_mobile"": true}" 21990,8,492,2017-04-30 00:26:16,http://gerlach.biz/silas.heathcote,,171.85.254.136,"{""location"": ""HN"", ""is_mobile"": false}" 21991,8,492,2017-01-12 02:59:11,http://marks.com/roderick_bernhard,,226.186.156.92,"{""location"": ""BE"", ""is_mobile"": true}" 21992,8,492,2016-12-22 22:57:40,http://leuschke.co/sasha,,112.111.95.177,"{""location"": ""YT"", ""is_mobile"": true}" 21993,8,492,2017-02-09 23:30:26,http://kerlukebrown.org/yvonne_white,,77.201.183.178,"{""location"": ""RU"", ""is_mobile"": true}" 21994,8,492,2016-12-14 07:19:07,http://hammesblanda.co/neal.king,,177.54.237.98,"{""location"": ""BQ"", ""is_mobile"": true}" 21995,8,492,2017-01-24 11:01:48,http://boyle.info/winnifred.schultz,,2.16.168.30,"{""location"": ""CW"", ""is_mobile"": false}" 21996,8,492,2017-03-02 14:00:31,http://mohr.biz/kyle.koch,,191.86.44.17,"{""location"": ""UA"", ""is_mobile"": false}" 21997,8,492,2016-12-22 23:49:15,http://jenkinsmuller.io/leilani_skiles,,157.53.197.53,"{""location"": ""SI"", ""is_mobile"": false}" 21998,8,492,2017-06-08 03:28:27,http://borerkreiger.org/golden,,105.63.17.157,"{""location"": ""BZ"", ""is_mobile"": false}" 909,1,21,2017-03-25 18:19:29,http://thiel.co/aliya,,246.207.196.120,"{""location"": ""VA"", ""is_mobile"": false}" 910,1,21,2017-01-29 18:23:39,http://labadie.info/lavonne,,195.192.253.246,"{""location"": ""TC"", ""is_mobile"": false}" 911,1,21,2017-01-29 19:38:23,http://mayer.io/courtney_batz,,109.45.12.8,"{""location"": ""SO"", ""is_mobile"": true}" 912,1,21,2017-02-17 18:11:04,http://kerlukezemlak.info/jakob.swaniawski,,8.208.25.34,"{""location"": ""WF"", ""is_mobile"": true}" 913,1,21,2017-03-13 01:59:33,http://torphy.org/keon.hills,,67.159.164.144,"{""location"": ""PS"", ""is_mobile"": true}" 914,1,21,2017-02-26 20:33:12,http://glover.com/lois_welch,,62.174.187.200,"{""location"": ""NU"", ""is_mobile"": true}" 915,1,21,2017-06-11 07:38:50,http://von.org/torrey,,229.221.121.156,"{""location"": ""AR"", ""is_mobile"": false}" 916,1,21,2017-03-17 12:40:53,http://thompson.com/jaydon_crist,,37.16.73.69,"{""location"": ""NE"", ""is_mobile"": false}" 917,1,21,2017-05-13 18:45:33,http://funk.biz/kaycee_rolfson,,127.172.41.139,"{""location"": ""SA"", ""is_mobile"": false}" 918,1,21,2017-06-10 02:40:40,http://gerlach.biz/albina_durgan,,208.178.215.93,"{""location"": ""MD"", ""is_mobile"": false}" 919,1,21,2016-12-14 08:52:14,http://halvorsonkoepp.name/nova,,171.231.245.235,"{""location"": ""TZ"", ""is_mobile"": true}" 920,1,21,2017-03-30 14:37:36,http://doylekautzer.biz/adrien.harvey,,241.55.235.183,"{""location"": ""CH"", ""is_mobile"": true}" 921,1,21,2017-04-03 18:02:36,http://stromansenger.info/freddy_williamson,,79.242.82.122,"{""location"": ""SN"", ""is_mobile"": false}" 922,1,21,2017-05-21 17:55:24,http://ratke.com/edmond,,188.106.77.223,"{""location"": ""TN"", ""is_mobile"": false}" 923,1,21,2017-05-15 05:35:59,http://medhurst.org/ruel.robel,,237.34.84.211,"{""location"": ""HK"", ""is_mobile"": true}" 924,1,22,2017-01-11 18:02:14,http://dickinson.biz/geoffrey,,26.169.12.130,"{""location"": ""IR"", ""is_mobile"": false}" 925,1,22,2017-03-18 08:26:23,http://keebler.co/violet,,16.193.193.97,"{""location"": ""CV"", ""is_mobile"": false}" 926,1,22,2017-04-24 02:56:16,http://schaefer.name/vernice.jones,,229.150.65.14,"{""location"": ""ME"", ""is_mobile"": true}" 927,1,22,2017-01-16 13:30:10,http://roberts.org/eloy,,53.179.121.21,"{""location"": ""NU"", ""is_mobile"": true}" 928,1,22,2017-04-16 14:12:55,http://runolfsdottirstehr.io/pansy.lakin,,149.33.246.110,"{""location"": ""VN"", ""is_mobile"": false}" 929,1,22,2017-01-20 22:03:09,http://kreigerfisher.net/sven.lakin,,149.30.232.202,"{""location"": ""MW"", ""is_mobile"": false}" 930,1,22,2017-05-20 08:33:22,http://heller.co/pablo,,100.224.241.216,"{""location"": ""HN"", ""is_mobile"": false}" 931,1,22,2017-02-18 05:56:40,http://ritchie.biz/bernita_hintz,,195.22.12.6,"{""location"": ""WF"", ""is_mobile"": false}" 932,1,22,2017-03-05 00:17:47,http://bruen.com/columbus_schmeler,,95.75.144.22,"{""location"": ""LY"", ""is_mobile"": false}" 933,1,22,2017-03-24 22:58:40,http://wolff.info/claude,,146.154.42.183,"{""location"": ""ES"", ""is_mobile"": false}" 934,1,22,2017-03-10 10:36:11,http://trompkirlin.org/jeffry,,230.151.197.172,"{""location"": ""BI"", ""is_mobile"": false}" 935,1,22,2017-05-19 23:37:48,http://hodkiewicz.name/jo,,229.46.99.4,"{""location"": ""EG"", ""is_mobile"": true}" 936,1,22,2017-04-29 18:49:33,http://hermiston.com/kadin.deckow,,12.29.17.12,"{""location"": ""VA"", ""is_mobile"": false}" 937,1,22,2016-12-31 16:01:59,http://ornnolan.name/antone.ondricka,,167.244.212.68,"{""location"": ""KW"", ""is_mobile"": true}" 938,1,22,2017-05-24 20:59:45,http://ernser.com/amelie.mosciski,,28.232.152.138,"{""location"": ""TO"", ""is_mobile"": false}" 939,1,22,2017-04-09 17:28:15,http://lemke.net/sheila_bogisich,,28.23.97.124,"{""location"": ""NP"", ""is_mobile"": false}" 940,1,22,2017-02-15 19:44:26,http://stokes.org/olaf,,233.248.190.222,"{""location"": ""AF"", ""is_mobile"": false}" 941,1,22,2017-06-08 06:48:05,http://fritsch.biz/natalie,,199.140.68.62,"{""location"": ""GU"", ""is_mobile"": false}" 942,1,22,2017-04-23 21:54:02,http://wyman.net/patrick_lind,,95.170.190.50,"{""location"": ""BH"", ""is_mobile"": true}" 943,1,22,2017-02-03 05:48:09,http://rohan.com/evans,,213.61.68.163,"{""location"": ""UM"", ""is_mobile"": false}" 944,1,22,2017-01-15 18:30:46,http://schinnermcdermott.com/dina,,107.92.242.229,"{""location"": ""TC"", ""is_mobile"": true}" 945,1,22,2017-01-21 11:29:47,http://yundt.biz/okey.nader,,232.130.176.75,"{""location"": ""FK"", ""is_mobile"": true}" 946,1,22,2017-03-25 15:25:42,http://crona.info/gerhard,,38.215.67.157,"{""location"": ""PW"", ""is_mobile"": true}" 947,1,22,2016-12-16 01:43:52,http://schowalter.co/jayde,,65.94.90.72,"{""location"": ""LA"", ""is_mobile"": false}" 948,1,22,2017-04-12 00:52:47,http://dickenskiehn.io/lisette,,43.211.71.223,"{""location"": ""KY"", ""is_mobile"": false}" 949,1,22,2017-04-21 08:31:50,http://homenick.net/darius,,59.193.153.48,"{""location"": ""KE"", ""is_mobile"": true}" 950,1,22,2017-06-14 01:27:26,http://mertz.net/johnnie,,45.16.135.53,"{""location"": ""BR"", ""is_mobile"": true}" 951,1,22,2017-05-21 05:32:20,http://leuschke.biz/torrey.crist,,17.133.156.126,"{""location"": ""TW"", ""is_mobile"": true}" 952,1,23,2017-03-12 11:04:57,http://hoppewhite.info/santino,,218.184.196.22,"{""location"": ""SX"", ""is_mobile"": false}" 953,1,23,2016-12-30 21:35:41,http://kihn.com/novella.simonis,,44.11.84.53,"{""location"": ""KZ"", ""is_mobile"": false}" 954,1,23,2017-01-21 19:27:20,http://faheyturcotte.name/ericka,,144.78.75.75,"{""location"": ""TG"", ""is_mobile"": true}" 955,1,23,2017-06-01 15:29:48,http://grimesmarquardt.org/orlando,,94.190.148.175,"{""location"": ""BB"", ""is_mobile"": true}" 956,1,23,2017-02-23 02:41:52,http://thompsontillman.org/marcelina,,133.220.148.154,"{""location"": ""BJ"", ""is_mobile"": true}" 957,1,23,2017-03-12 20:41:28,http://vonrueden.org/morris.ledner,,187.28.71.118,"{""location"": ""NI"", ""is_mobile"": false}" 958,1,23,2017-05-05 08:03:24,http://nitzsche.net/jakob,,253.121.59.225,"{""location"": ""MP"", ""is_mobile"": true}" 959,1,23,2017-03-24 23:02:06,http://veum.name/priscilla,,211.40.30.248,"{""location"": ""ST"", ""is_mobile"": false}" 960,1,23,2017-02-17 03:29:04,http://adams.io/bernita,,87.142.233.21,"{""location"": ""NL"", ""is_mobile"": false}" 961,1,23,2017-04-20 16:06:13,http://erdmanpowlowski.co/paula_little,,105.194.92.204,"{""location"": ""JM"", ""is_mobile"": false}" 962,1,23,2017-02-06 13:14:33,http://emard.com/paris,,103.16.114.113,"{""location"": ""CG"", ""is_mobile"": false}" 963,1,23,2017-04-20 07:32:30,http://macgyverchamplin.biz/cielo,,218.62.30.246,"{""location"": ""FM"", ""is_mobile"": true}" 964,1,23,2017-05-13 21:00:59,http://fisher.co/gunnar_fritsch,,33.245.152.92,"{""location"": ""KE"", ""is_mobile"": false}" 965,1,23,2017-02-16 06:39:39,http://weimann.co/raoul,,149.174.153.95,"{""location"": ""KI"", ""is_mobile"": false}" 21999,8,492,2017-06-06 11:38:54,http://bechtelar.com/reanna.powlowski,,247.62.67.113,"{""location"": ""LR"", ""is_mobile"": true}" 22000,8,492,2017-01-27 15:49:36,http://ondricka.biz/gabriel,,35.136.67.33,"{""location"": ""MM"", ""is_mobile"": false}" 22001,8,492,2017-03-30 15:13:45,http://rau.name/adriana,,164.4.67.37,"{""location"": ""MM"", ""is_mobile"": true}" 22002,8,492,2017-04-14 07:24:11,http://marks.info/helmer,,186.22.41.30,"{""location"": ""IR"", ""is_mobile"": false}" 22003,8,492,2017-04-25 16:40:19,http://boehmmcdermott.name/theresia.kertzmann,,121.81.248.24,"{""location"": ""SY"", ""is_mobile"": true}" 22004,8,492,2017-04-02 02:16:56,http://frami.org/ahmed_mcglynn,,240.254.239.236,"{""location"": ""BL"", ""is_mobile"": false}" 22005,8,492,2017-01-08 06:52:46,http://kutch.biz/deion_moore,,168.207.235.116,"{""location"": ""JE"", ""is_mobile"": true}" 22006,8,492,2017-03-24 04:42:43,http://jerdeyundt.org/cletus,,232.73.63.137,"{""location"": ""SS"", ""is_mobile"": false}" 22007,8,492,2017-04-07 15:58:39,http://parkereffertz.com/roslyn,,252.167.21.174,"{""location"": ""SC"", ""is_mobile"": false}" 22008,8,492,2017-06-06 02:10:06,http://pacocha.info/easter.zulauf,,200.126.213.154,"{""location"": ""GF"", ""is_mobile"": true}" 22009,8,492,2016-12-29 07:27:57,http://gleichner.io/kelly,,65.170.185.214,"{""location"": ""BZ"", ""is_mobile"": false}" 22010,8,492,2017-05-14 17:03:41,http://simonis.io/giles_farrell,,103.205.212.172,"{""location"": ""SN"", ""is_mobile"": false}" 22011,8,492,2017-04-09 19:57:51,http://luettgen.org/harmony,,84.226.91.111,"{""location"": ""HM"", ""is_mobile"": true}" 22012,8,492,2017-02-11 11:26:52,http://weinat.co/gaetano_okon,,97.149.115.79,"{""location"": ""CV"", ""is_mobile"": true}" 22013,8,492,2017-02-17 08:03:46,http://deckowkunze.name/leilani,,45.120.152.125,"{""location"": ""TC"", ""is_mobile"": false}" 22014,8,492,2017-05-15 07:55:24,http://legros.com/jarrell.hoppe,,32.167.82.157,"{""location"": ""HU"", ""is_mobile"": true}" 22015,8,492,2016-12-24 17:16:41,http://murazik.net/kenyon,,205.188.73.202,"{""location"": ""EC"", ""is_mobile"": true}" 22016,8,492,2017-04-03 11:40:46,http://connelly.com/sofia.crona,,114.166.47.204,"{""location"": ""TD"", ""is_mobile"": false}" 22017,8,492,2017-04-16 06:03:17,http://gottlieb.info/roslyn,,249.178.103.196,"{""location"": ""ME"", ""is_mobile"": true}" 22018,8,492,2017-04-27 21:30:09,http://turnertrantow.name/sister_kertzmann,,161.132.22.220,"{""location"": ""EC"", ""is_mobile"": false}" 22019,8,492,2017-04-16 02:04:50,http://hegmann.co/luisa,,38.18.62.64,"{""location"": ""BV"", ""is_mobile"": true}" 22020,8,492,2016-12-31 09:53:08,http://schulist.info/mackenzie_hermann,,218.193.121.32,"{""location"": ""DE"", ""is_mobile"": true}" 22021,8,493,2017-03-30 13:57:37,http://abbottnicolas.org/jovanny_fritsch,,116.237.233.225,"{""location"": ""HM"", ""is_mobile"": false}" 22022,8,493,2016-12-21 04:18:21,http://toy.net/dedrick,,167.60.43.229,"{""location"": ""AO"", ""is_mobile"": false}" 22023,8,493,2017-04-16 08:18:39,http://funk.io/emmalee,,219.163.49.45,"{""location"": ""SN"", ""is_mobile"": false}" 22024,8,493,2017-01-29 16:03:42,http://dickimosciski.com/tyshawn.denesik,,186.155.110.97,"{""location"": ""SZ"", ""is_mobile"": false}" 22025,8,493,2017-05-29 11:17:37,http://adams.info/marcelle,,37.188.174.192,"{""location"": ""HN"", ""is_mobile"": true}" 22026,8,493,2017-06-12 22:11:35,http://langworth.co/annabel.murazik,,29.112.194.98,"{""location"": ""BG"", ""is_mobile"": true}" 22027,8,493,2017-02-14 02:54:34,http://quitzongerlach.co/lazaro,,111.158.102.106,"{""location"": ""FR"", ""is_mobile"": false}" 22028,8,493,2017-03-23 11:53:05,http://lehnerhuels.net/rosie_senger,,13.57.137.29,"{""location"": ""AG"", ""is_mobile"": true}" 22029,8,493,2017-06-06 03:30:16,http://gerlach.name/raphaelle_herman,,158.221.125.212,"{""location"": ""BG"", ""is_mobile"": true}" 22030,8,493,2017-06-10 03:26:42,http://waelchi.name/efrain_cronin,,222.12.97.124,"{""location"": ""DM"", ""is_mobile"": false}" 22031,8,493,2017-03-04 04:42:15,http://swift.io/raven_dooley,,189.204.49.14,"{""location"": ""LA"", ""is_mobile"": false}" 22032,8,493,2016-12-30 07:04:16,http://hicklebayer.info/clemmie_metz,,49.248.226.107,"{""location"": ""AZ"", ""is_mobile"": false}" 22033,8,493,2017-04-03 23:44:54,http://schoen.io/alyce,,135.251.186.23,"{""location"": ""MX"", ""is_mobile"": false}" 22034,8,493,2017-02-07 20:37:04,http://christiansen.co/oswald_powlowski,,144.46.13.142,"{""location"": ""SZ"", ""is_mobile"": true}" 22035,8,493,2017-01-28 19:50:57,http://schimmel.name/aric.hayes,,206.50.193.114,"{""location"": ""PT"", ""is_mobile"": true}" 22036,8,493,2017-04-16 15:52:39,http://halvorsongoyette.org/adolf_will,,105.216.211.43,"{""location"": ""CY"", ""is_mobile"": true}" 22037,8,493,2017-02-28 21:43:48,http://rolfsonruecker.io/serena,,84.160.224.53,"{""location"": ""AL"", ""is_mobile"": false}" 22038,8,493,2017-02-22 02:26:09,http://nikolauwaniawski.name/fabiola,,41.162.226.178,"{""location"": ""AW"", ""is_mobile"": false}" 22039,8,493,2017-03-25 11:13:14,http://lemke.co/jimmy.olson,,174.54.171.4,"{""location"": ""FJ"", ""is_mobile"": false}" 22040,8,493,2017-02-07 05:52:00,http://mcdermott.net/wendy,,58.117.53.214,"{""location"": ""HK"", ""is_mobile"": false}" 22041,8,493,2017-05-21 16:11:43,http://greenbernhard.info/alaina_bailey,,144.153.229.244,"{""location"": ""LV"", ""is_mobile"": false}" 22042,8,493,2017-05-15 03:08:54,http://emmerich.name/ruell_weber,,99.33.62.55,"{""location"": ""VN"", ""is_mobile"": true}" 22043,8,493,2016-12-17 15:30:58,http://wintheiser.name/frederick,,48.26.139.25,"{""location"": ""RW"", ""is_mobile"": false}" 22044,8,493,2017-02-16 11:02:33,http://weimann.org/anais,,19.234.134.247,"{""location"": ""MS"", ""is_mobile"": false}" 22045,8,493,2017-02-17 01:16:23,http://lakin.org/joyce,,201.19.61.10,"{""location"": ""CY"", ""is_mobile"": true}" 22046,8,493,2017-05-10 11:27:58,http://denesik.info/ottis,,151.45.8.74,"{""location"": ""UG"", ""is_mobile"": false}" 22047,8,493,2017-04-30 16:29:23,http://gottlieb.name/eulalia,,127.65.97.95,"{""location"": ""BA"", ""is_mobile"": false}" 22048,8,493,2017-02-19 08:03:56,http://donnelly.biz/flavio,,180.220.98.82,"{""location"": ""CM"", ""is_mobile"": true}" 22049,8,493,2017-05-17 00:44:10,http://collins.org/rylee,,79.182.78.127,"{""location"": ""VC"", ""is_mobile"": false}" 22050,8,493,2017-01-17 02:34:05,http://bruen.co/davonte_streich,,154.180.132.174,"{""location"": ""HT"", ""is_mobile"": false}" 22051,8,493,2017-03-07 23:04:13,http://welch.com/blake_romaguera,,187.129.71.140,"{""location"": ""SZ"", ""is_mobile"": false}" 22052,8,493,2017-04-26 10:33:47,http://kautzergreen.net/patsy_hoeger,,158.30.59.226,"{""location"": ""CN"", ""is_mobile"": true}" 22053,8,493,2017-03-22 19:08:58,http://ullrichgreenholt.org/wendell,,215.253.125.219,"{""location"": ""BV"", ""is_mobile"": true}" 966,1,23,2016-12-28 00:17:57,http://granthermiston.biz/augustine.hauck,,158.80.65.30,"{""location"": ""SD"", ""is_mobile"": false}" 967,1,23,2017-01-21 04:25:22,http://collier.net/beverly_vonrueden,,160.122.117.59,"{""location"": ""TF"", ""is_mobile"": false}" 968,1,23,2017-03-15 10:42:17,http://bauchcormier.co/morris,,115.24.123.250,"{""location"": ""KY"", ""is_mobile"": false}" 969,1,23,2017-01-21 13:13:13,http://veum.net/alta,,3.174.190.99,"{""location"": ""YT"", ""is_mobile"": false}" 970,1,23,2017-05-24 23:20:58,http://corwin.biz/alayna_fahey,,210.126.89.200,"{""location"": ""YT"", ""is_mobile"": false}" 971,1,23,2017-06-13 22:49:37,http://mcdermott.biz/sandrine,,24.190.88.153,"{""location"": ""LK"", ""is_mobile"": true}" 972,1,23,2017-01-08 09:58:17,http://lockman.org/hans_ullrich,,82.182.187.175,"{""location"": ""CO"", ""is_mobile"": true}" 973,1,23,2017-02-08 16:43:44,http://bartolettikshlerin.net/mavis.steuber,,39.44.143.75,"{""location"": ""PW"", ""is_mobile"": true}" 974,1,23,2016-12-27 05:36:27,http://erdman.info/linwood.mraz,,59.242.87.107,"{""location"": ""YE"", ""is_mobile"": true}" 975,1,23,2016-12-20 14:15:08,http://douglas.name/ashlynn_abbott,,205.254.50.101,"{""location"": ""SE"", ""is_mobile"": true}" 976,1,23,2017-05-08 01:06:32,http://feil.net/ava,,202.110.184.181,"{""location"": ""EE"", ""is_mobile"": true}" 977,1,23,2017-02-20 00:29:59,http://reichel.biz/clemens,,209.9.110.159,"{""location"": ""PR"", ""is_mobile"": true}" 978,1,23,2017-04-05 15:59:00,http://greenfeldertremblay.io/kara,,177.14.178.11,"{""location"": ""CW"", ""is_mobile"": true}" 979,1,23,2017-05-11 08:29:49,http://nikolaus.net/colleen,,31.34.33.182,"{""location"": ""AD"", ""is_mobile"": false}" 980,1,23,2017-05-21 06:07:25,http://gradyjerde.biz/bartholome.bayer,,177.225.19.77,"{""location"": ""LC"", ""is_mobile"": true}" 981,1,23,2017-05-26 10:04:48,http://tillmanjohnson.io/murray,,84.132.209.189,"{""location"": ""CH"", ""is_mobile"": false}" 982,1,23,2017-03-31 21:59:58,http://marvin.info/eliezer_schamberger,,20.188.168.136,"{""location"": ""RU"", ""is_mobile"": true}" 983,1,23,2017-03-30 08:22:39,http://mayert.org/reuben,,213.249.150.53,"{""location"": ""TC"", ""is_mobile"": false}" 984,1,23,2017-05-02 16:22:06,http://romaguerahintz.io/ethel,,232.211.243.199,"{""location"": ""NI"", ""is_mobile"": false}" 985,1,23,2017-01-23 22:56:48,http://terryrempel.org/arvid.heel,,206.50.202.66,"{""location"": ""RU"", ""is_mobile"": false}" 986,1,24,2017-03-12 19:59:56,http://rueckerreichert.biz/cecelia,,202.224.133.252,"{""location"": ""PG"", ""is_mobile"": true}" 987,1,24,2017-06-09 12:35:44,http://nienowzieme.net/aiden.lemke,,70.111.109.173,"{""location"": ""KY"", ""is_mobile"": false}" 988,1,24,2017-05-14 06:46:03,http://gleason.info/violette.mayer,,121.83.83.231,"{""location"": ""JP"", ""is_mobile"": false}" 989,1,24,2017-05-30 02:42:50,http://larkincummings.io/ettie,,119.54.36.9,"{""location"": ""TD"", ""is_mobile"": false}" 990,1,24,2017-04-06 02:21:36,http://lakin.name/lela,,49.20.122.61,"{""location"": ""VE"", ""is_mobile"": true}" 991,1,24,2017-04-03 06:32:53,http://okuneva.io/katherine_steuber,,107.212.42.119,"{""location"": ""CG"", ""is_mobile"": true}" 992,1,24,2017-03-02 18:27:17,http://balistreri.net/ray,,136.31.238.136,"{""location"": ""BJ"", ""is_mobile"": false}" 993,1,24,2017-05-25 18:01:22,http://ward.name/jacques,,58.127.63.118,"{""location"": ""TZ"", ""is_mobile"": true}" 994,1,24,2016-12-31 06:01:41,http://lockmantromp.biz/euna.lang,,152.36.134.85,"{""location"": ""CI"", ""is_mobile"": false}" 995,1,24,2017-02-25 00:57:50,http://shanahan.name/major.cartwright,,104.127.221.154,"{""location"": ""NR"", ""is_mobile"": true}" 996,1,24,2017-01-25 19:02:29,http://terry.name/grayce,,56.216.7.77,"{""location"": ""MX"", ""is_mobile"": false}" 997,1,24,2017-02-03 21:27:29,http://boyerbatz.co/yolanda,,5.109.158.123,"{""location"": ""CG"", ""is_mobile"": false}" 998,1,24,2017-02-23 22:55:57,http://luettgencrist.com/colby_collins,,241.40.6.49,"{""location"": ""MA"", ""is_mobile"": true}" 999,1,24,2017-05-23 13:24:17,http://schneider.io/alta,,172.184.32.195,"{""location"": ""VE"", ""is_mobile"": false}" 1000,1,24,2017-03-16 00:19:30,http://schamberger.co/nikolas.hand,,83.186.216.244,"{""location"": ""LI"", ""is_mobile"": true}" 1001,1,24,2017-05-14 07:30:36,http://pagacmiller.net/cydney.ankunding,,24.252.249.160,"{""location"": ""GF"", ""is_mobile"": true}" 1002,1,24,2017-05-11 12:51:55,http://jacobs.info/braulio.block,,21.63.5.119,"{""location"": ""BD"", ""is_mobile"": false}" 1003,1,24,2016-12-16 23:32:23,http://terry.biz/linwood.bergnaum,,174.210.138.206,"{""location"": ""MK"", ""is_mobile"": false}" 1004,1,24,2017-04-14 16:31:44,http://schmeler.biz/elmira_gaylord,,210.61.12.88,"{""location"": ""WF"", ""is_mobile"": true}" 1005,1,24,2017-03-30 20:11:26,http://mertz.name/mozelle,,49.72.126.82,"{""location"": ""SC"", ""is_mobile"": true}" 1006,1,24,2017-04-08 19:47:17,http://gerhold.biz/graham.ebert,,228.8.176.237,"{""location"": ""BV"", ""is_mobile"": true}" 1007,1,24,2017-02-23 09:00:15,http://greenfelder.net/paolo.sauer,,106.177.249.40,"{""location"": ""EE"", ""is_mobile"": false}" 1008,1,24,2017-04-10 04:26:29,http://willms.net/ismael_hauck,,102.240.56.15,"{""location"": ""MF"", ""is_mobile"": false}" 1009,1,24,2017-04-07 23:06:39,http://prosaccoaltenwerth.org/joana.boyle,,19.51.218.41,"{""location"": ""BL"", ""is_mobile"": true}" 1010,1,24,2017-04-27 21:41:19,http://stammfranecki.biz/eloy.cole,,207.177.59.198,"{""location"": ""EG"", ""is_mobile"": true}" 1011,1,24,2017-01-14 16:14:44,http://white.io/jorge.bauch,,50.162.169.33,"{""location"": ""GY"", ""is_mobile"": true}" 1012,1,24,2016-12-16 05:44:49,http://reillydoyle.com/owen,,202.157.4.188,"{""location"": ""JP"", ""is_mobile"": true}" 1013,1,24,2017-05-26 12:09:48,http://effertz.co/robin.williamson,,8.246.81.65,"{""location"": ""TO"", ""is_mobile"": false}" 1014,1,24,2016-12-30 20:22:54,http://kuphal.com/flavio,,152.125.47.179,"{""location"": ""TJ"", ""is_mobile"": true}" 1015,1,24,2017-05-20 09:03:17,http://wilkinson.co/delbert.johns,,42.113.205.169,"{""location"": ""PS"", ""is_mobile"": true}" 1016,1,24,2017-01-16 11:31:39,http://macejkovic.biz/minerva_moore,,191.15.111.212,"{""location"": ""LA"", ""is_mobile"": false}" 1017,1,24,2017-02-10 15:07:31,http://steuberschumm.info/chasity,,253.253.13.64,"{""location"": ""SE"", ""is_mobile"": true}" 1018,1,24,2017-04-25 19:01:06,http://satterfield.com/forrest,,226.157.148.191,"{""location"": ""AI"", ""is_mobile"": false}" 1019,1,24,2017-05-22 20:43:00,http://kirlin.info/ayden,,96.56.158.67,"{""location"": ""TG"", ""is_mobile"": true}" 1020,1,24,2017-04-10 06:25:45,http://kocherdman.com/kennedi,,224.233.19.130,"{""location"": ""JE"", ""is_mobile"": true}" 1021,1,24,2017-02-09 06:22:07,http://feest.org/tyra_wolff,,174.104.67.8,"{""location"": ""JE"", ""is_mobile"": false}" 22054,8,493,2017-04-09 18:09:33,http://cummeratamuller.net/katherine_hilpert,,133.116.177.155,"{""location"": ""GF"", ""is_mobile"": false}" 22055,8,493,2016-12-27 12:20:53,http://lubowitz.name/lelah,,123.67.83.25,"{""location"": ""BR"", ""is_mobile"": false}" 22056,8,493,2017-01-08 04:09:28,http://beer.org/camila,,47.85.220.57,"{""location"": ""PR"", ""is_mobile"": true}" 22057,8,493,2017-03-26 17:15:37,http://quigley.io/walton,,45.215.17.114,"{""location"": ""GW"", ""is_mobile"": false}" 22058,8,493,2017-04-17 17:08:06,http://thompson.net/treie.cormier,,141.148.39.31,"{""location"": ""DE"", ""is_mobile"": true}" 22059,8,493,2017-03-10 00:29:00,http://hodkiewiczkulas.net/derrick,,140.28.118.101,"{""location"": ""BB"", ""is_mobile"": true}" 22060,8,493,2017-02-18 16:08:32,http://stokes.org/ofelia_jacobson,,177.119.110.86,"{""location"": ""HM"", ""is_mobile"": true}" 22061,8,493,2017-03-29 18:34:27,http://langlowe.biz/brennan,,4.89.154.207,"{""location"": ""AL"", ""is_mobile"": false}" 22062,8,493,2017-04-24 23:38:42,http://stamm.biz/marlin,,204.10.65.127,"{""location"": ""MX"", ""is_mobile"": false}" 22063,8,494,2017-06-09 11:16:40,http://rau.com/jonathon_king,,125.27.45.109,"{""location"": ""ET"", ""is_mobile"": false}" 22064,8,494,2017-04-18 13:08:19,http://walker.com/evangeline,,200.109.172.166,"{""location"": ""GD"", ""is_mobile"": true}" 22065,8,494,2017-05-16 22:07:35,http://welch.name/pinkie_pfannerstill,,248.144.129.126,"{""location"": ""JM"", ""is_mobile"": false}" 22066,8,494,2017-05-22 14:57:54,http://langosh.org/gonzalo,,153.98.128.98,"{""location"": ""AW"", ""is_mobile"": true}" 22067,8,494,2017-03-23 01:18:22,http://thompson.name/lora_okeefe,,28.100.230.78,"{""location"": ""TJ"", ""is_mobile"": true}" 22068,8,494,2016-12-31 07:44:18,http://mitchell.biz/ena.ruecker,,23.126.91.188,"{""location"": ""MC"", ""is_mobile"": false}" 22069,8,494,2017-03-06 21:23:19,http://pagacnitzsche.net/kody,,73.102.136.228,"{""location"": ""VA"", ""is_mobile"": true}" 22070,8,494,2017-02-20 08:03:40,http://mckenzie.org/buford.bednar,,232.127.243.103,"{""location"": ""MV"", ""is_mobile"": true}" 22071,8,494,2017-01-02 06:39:55,http://abshirewhite.org/thora,,136.72.126.15,"{""location"": ""HK"", ""is_mobile"": true}" 22072,8,494,2017-05-17 21:59:28,http://waters.info/granville,,248.230.204.116,"{""location"": ""SI"", ""is_mobile"": true}" 22073,8,494,2016-12-18 15:47:43,http://mante.com/nash.wilderman,,174.33.48.72,"{""location"": ""FI"", ""is_mobile"": true}" 22074,8,494,2017-05-05 04:29:58,http://wilderman.io/tyrell,,136.193.230.240,"{""location"": ""VG"", ""is_mobile"": true}" 22075,8,494,2017-02-15 12:02:27,http://jerde.io/hayden,,180.211.252.58,"{""location"": ""EC"", ""is_mobile"": true}" 22076,8,494,2017-02-13 10:52:09,http://orn.info/patricia_purdy,,185.161.82.118,"{""location"": ""BW"", ""is_mobile"": false}" 22077,8,494,2017-05-09 12:26:50,http://harvey.io/oleta,,224.214.234.239,"{""location"": ""BB"", ""is_mobile"": false}" 22078,8,494,2017-01-24 21:42:13,http://ernsercasper.co/theron_leffler,,94.239.155.149,"{""location"": ""CN"", ""is_mobile"": true}" 22079,8,494,2017-03-08 18:24:15,http://stanton.net/christina,,92.190.147.111,"{""location"": ""GD"", ""is_mobile"": true}" 22080,8,494,2017-02-22 23:55:23,http://gerlach.net/davon_hilpert,,101.97.10.116,"{""location"": ""TL"", ""is_mobile"": true}" 22081,8,494,2017-04-18 01:58:28,http://wehnerrutherford.org/claudie.conroy,,156.176.3.63,"{""location"": ""PY"", ""is_mobile"": true}" 22082,8,494,2017-04-20 15:47:17,http://crooks.name/manley,,134.23.41.57,"{""location"": ""BA"", ""is_mobile"": true}" 22083,8,494,2017-05-11 17:29:37,http://konopelski.co/eliseo,,244.142.71.50,"{""location"": ""AF"", ""is_mobile"": false}" 22084,8,494,2017-03-30 12:06:49,http://haagvolkman.info/brooklyn,,226.3.81.21,"{""location"": ""RS"", ""is_mobile"": true}" 22085,8,494,2017-03-29 16:33:17,http://heaneywelch.co/zola,,199.48.126.55,"{""location"": ""ET"", ""is_mobile"": false}" 22086,8,494,2017-05-07 12:19:13,http://yundt.org/jose,,200.160.27.153,"{""location"": ""BL"", ""is_mobile"": false}" 22087,8,495,2017-01-30 17:38:20,http://abbott.io/kattie.borer,,180.80.103.197,"{""location"": ""MA"", ""is_mobile"": false}" 22088,8,495,2017-02-26 02:19:13,http://ryan.org/sigrid,,71.37.35.178,"{""location"": ""BA"", ""is_mobile"": false}" 22089,8,495,2017-01-17 08:05:42,http://pfeffer.org/lula_towne,,163.156.65.55,"{""location"": ""FO"", ""is_mobile"": false}" 22090,8,495,2017-05-24 08:50:08,http://witting.net/terrell,,113.196.197.220,"{""location"": ""NE"", ""is_mobile"": false}" 22091,8,495,2017-04-03 08:45:20,http://pagac.io/gianni.prohaska,,53.13.26.57,"{""location"": ""BQ"", ""is_mobile"": false}" 22092,8,495,2016-12-20 14:16:21,http://dooleymohr.net/faustino,,205.92.168.88,"{""location"": ""NG"", ""is_mobile"": true}" 22093,8,495,2017-05-24 16:35:22,http://konopelskikuhn.info/reba.mann,,50.148.15.52,"{""location"": ""TL"", ""is_mobile"": true}" 22094,8,495,2017-01-07 15:04:58,http://nikolaus.com/loyce,,2.159.121.233,"{""location"": ""KR"", ""is_mobile"": false}" 22095,8,495,2017-01-13 01:19:49,http://reilly.org/harley,,152.152.206.193,"{""location"": ""DZ"", ""is_mobile"": false}" 22096,8,495,2017-01-18 11:19:50,http://rosenbaumherman.co/electa,,14.49.48.188,"{""location"": ""KY"", ""is_mobile"": false}" 22097,8,495,2017-02-15 14:48:40,http://gutmann.com/kali,,225.214.27.204,"{""location"": ""KW"", ""is_mobile"": true}" 22098,8,495,2016-12-23 05:30:26,http://murphy.io/bartholome_frami,,184.66.214.13,"{""location"": ""GE"", ""is_mobile"": true}" 22099,8,495,2017-04-20 08:33:19,http://cremin.biz/maeve.dicki,,158.23.123.7,"{""location"": ""TK"", ""is_mobile"": true}" 22100,8,495,2017-01-13 20:40:34,http://cartertoy.io/oran.gleason,,2.111.193.126,"{""location"": ""BY"", ""is_mobile"": false}" 22101,8,495,2017-01-16 14:53:45,http://flatley.name/sonya_crist,,220.27.231.203,"{""location"": ""MS"", ""is_mobile"": false}" 22102,8,495,2017-02-24 13:06:42,http://bednar.info/pietro_jacobi,,91.63.25.60,"{""location"": ""VU"", ""is_mobile"": false}" 22103,8,495,2017-05-14 02:11:58,http://komoen.org/isadore,,202.147.206.115,"{""location"": ""MU"", ""is_mobile"": false}" 22104,8,495,2016-12-17 10:18:43,http://balistreri.co/eda,,124.141.229.125,"{""location"": ""MT"", ""is_mobile"": true}" 22105,8,495,2017-06-06 04:36:11,http://yundt.org/flavie,,104.12.54.27,"{""location"": ""DE"", ""is_mobile"": true}" 22106,8,495,2017-02-06 08:14:35,http://borer.biz/eliseo_stiedemann,,233.184.247.233,"{""location"": ""VA"", ""is_mobile"": false}" 22107,8,495,2017-05-10 08:52:24,http://sanford.info/micaela.ankunding,,132.180.43.83,"{""location"": ""VN"", ""is_mobile"": false}" 22108,8,495,2017-01-09 21:21:14,http://mayert.biz/kevon,,147.10.225.235,"{""location"": ""MV"", ""is_mobile"": false}" 22109,8,495,2017-01-31 21:59:21,http://bahringer.info/catharine,,18.201.69.184,"{""location"": ""QA"", ""is_mobile"": false}" 1022,1,24,2017-06-05 07:52:56,http://waters.info/emmanuelle,,170.8.171.20,"{""location"": ""CF"", ""is_mobile"": true}" 1023,1,24,2017-01-05 17:06:18,http://keelingmccullough.biz/leone_robel,,183.96.146.199,"{""location"": ""KZ"", ""is_mobile"": false}" 1024,1,24,2017-02-12 00:39:33,http://pollich.co/ara_collier,,249.227.219.149,"{""location"": ""IM"", ""is_mobile"": false}" 1025,1,24,2017-03-25 16:12:16,http://moriette.info/aglae_kshlerin,,243.156.17.196,"{""location"": ""WF"", ""is_mobile"": false}" 1026,1,24,2017-03-16 22:06:50,http://kemmerleannon.net/cordia_rosenbaum,,31.44.171.174,"{""location"": ""EE"", ""is_mobile"": true}" 1027,1,24,2017-05-10 20:24:27,http://ziemewilliamson.name/orville,,37.110.174.148,"{""location"": ""MY"", ""is_mobile"": true}" 1028,1,24,2017-01-05 18:48:47,http://zboncak.net/hector.ebert,,78.174.166.144,"{""location"": ""NO"", ""is_mobile"": true}" 1029,1,24,2017-05-03 18:11:52,http://zulaufupton.io/roel,,229.47.55.240,"{""location"": ""GP"", ""is_mobile"": true}" 1030,1,24,2017-04-17 01:43:06,http://schinner.io/tobin,,127.137.114.75,"{""location"": ""HT"", ""is_mobile"": false}" 1031,1,24,2017-06-07 08:10:36,http://bayer.co/nadia.collins,,40.81.112.178,"{""location"": ""GY"", ""is_mobile"": true}" 1032,1,24,2017-05-14 00:25:30,http://ondricka.org/kelton.treutel,,249.18.157.249,"{""location"": ""KE"", ""is_mobile"": true}" 1033,1,24,2017-03-08 16:43:09,http://moore.info/camille,,237.20.4.228,"{""location"": ""SR"", ""is_mobile"": false}" 1034,1,24,2017-05-24 03:00:54,http://ornokuneva.info/john,,191.171.213.168,"{""location"": ""TZ"", ""is_mobile"": false}" 1035,1,24,2017-06-08 04:38:07,http://lynchmuller.info/stefan.bechtelar,,227.53.254.2,"{""location"": ""CV"", ""is_mobile"": true}" 1036,1,24,2017-05-10 16:56:30,http://vandervortgibson.info/eula,,124.5.51.33,"{""location"": ""KM"", ""is_mobile"": true}" 1037,1,24,2017-01-08 07:22:37,http://aufderhar.org/clemmie,,224.47.147.31,"{""location"": ""SL"", ""is_mobile"": false}" 1038,1,24,2016-12-21 07:28:50,http://schroeder.com/vincenzo,,142.152.43.81,"{""location"": ""WF"", ""is_mobile"": true}" 1039,1,24,2017-05-10 03:46:23,http://nolanbernhard.io/carson_balistreri,,155.117.81.47,"{""location"": ""KM"", ""is_mobile"": false}" 1040,1,24,2017-03-07 13:14:35,http://bosco.co/derick,,240.101.40.157,"{""location"": ""TK"", ""is_mobile"": true}" 1041,1,24,2017-05-21 20:22:52,http://kochlesch.biz/kenny.armstrong,,90.183.251.193,"{""location"": ""MD"", ""is_mobile"": true}" 1042,1,24,2017-02-19 02:21:12,http://schneider.name/emmet.spencer,,131.48.157.169,"{""location"": ""VN"", ""is_mobile"": true}" 1043,1,24,2017-05-31 19:33:17,http://krajcik.net/eryn.grant,,238.163.78.138,"{""location"": ""IT"", ""is_mobile"": false}" 1044,1,24,2017-01-03 17:14:04,http://botsford.info/nathanial,,57.150.64.12,"{""location"": ""BB"", ""is_mobile"": false}" 1045,1,24,2017-03-05 22:08:23,http://conroy.net/ari_torphy,,220.120.113.158,"{""location"": ""AG"", ""is_mobile"": false}" 1046,1,24,2017-04-20 19:34:23,http://weinat.com/jonathon_bartell,,247.215.136.116,"{""location"": ""TZ"", ""is_mobile"": true}" 1047,1,24,2017-03-23 16:45:46,http://wisoky.net/natalie_beatty,,23.40.252.204,"{""location"": ""KZ"", ""is_mobile"": false}" 1049,1,24,2016-12-17 00:44:41,http://jacobs.biz/isaias.rau,,226.80.88.161,"{""location"": ""FK"", ""is_mobile"": true}" 1050,1,24,2017-05-07 11:33:21,http://rowe.org/francisco.halvorson,,57.118.31.98,"{""location"": ""LA"", ""is_mobile"": true}" 1051,1,24,2017-05-17 12:19:43,http://halvorsonbrown.net/antonio,,24.214.117.177,"{""location"": ""KP"", ""is_mobile"": false}" 1052,1,24,2017-03-27 18:54:08,http://welchjast.biz/tommie,,182.225.138.39,"{""location"": ""BA"", ""is_mobile"": false}" 1053,1,24,2017-03-02 00:55:40,http://howe.co/saul,,190.24.213.254,"{""location"": ""GN"", ""is_mobile"": true}" 1054,1,25,2017-02-14 22:14:42,http://conroy.com/jada,,224.27.231.43,"{""location"": ""SA"", ""is_mobile"": true}" 1055,1,25,2017-01-24 13:27:36,http://hellerbergstrom.info/thomas.turner,,233.69.218.52,"{""location"": ""GU"", ""is_mobile"": false}" 1056,1,25,2017-04-27 06:49:35,http://kutch.co/jordon,,139.46.201.117,"{""location"": ""KP"", ""is_mobile"": false}" 1057,1,25,2017-04-16 21:03:42,http://rice.io/layne.satterfield,,173.198.85.48,"{""location"": ""NP"", ""is_mobile"": false}" 1058,1,25,2017-02-14 23:40:21,http://kingstark.info/albina.pouros,,61.68.220.177,"{""location"": ""GL"", ""is_mobile"": false}" 1059,1,25,2016-12-16 08:51:59,http://pouros.io/mina,,40.96.42.71,"{""location"": ""AO"", ""is_mobile"": false}" 1060,1,25,2017-05-29 09:34:13,http://aufderhar.co/aleandro_corkery,,215.82.122.123,"{""location"": ""AX"", ""is_mobile"": false}" 1061,1,25,2016-12-30 13:23:44,http://okunevastokes.biz/justyn.pollich,,197.137.40.144,"{""location"": ""MG"", ""is_mobile"": true}" 1062,1,25,2017-01-04 15:34:01,http://stroman.org/evie,,80.24.87.252,"{""location"": ""BJ"", ""is_mobile"": true}" 1063,1,25,2017-02-20 10:28:35,http://labadie.biz/milan.mckenzie,,164.233.81.226,"{""location"": ""GL"", ""is_mobile"": false}" 1064,1,25,2017-03-02 08:54:01,http://zemlaksauer.info/bailee.kovacek,,8.91.29.216,"{""location"": ""CL"", ""is_mobile"": true}" 1065,1,25,2017-03-08 14:21:44,http://parisian.io/ines_schmitt,,90.64.200.170,"{""location"": ""BV"", ""is_mobile"": true}" 1066,1,25,2017-04-24 09:24:34,http://langworth.com/ray.marks,,184.86.57.157,"{""location"": ""IM"", ""is_mobile"": false}" 1067,1,25,2017-05-23 10:58:39,http://welchpacocha.name/nedra,,189.173.223.128,"{""location"": ""TO"", ""is_mobile"": true}" 1068,1,25,2017-05-06 15:23:51,http://vonsteuber.name/frieda,,106.43.236.8,"{""location"": ""CU"", ""is_mobile"": false}" 1069,1,25,2016-12-25 20:58:40,http://mcdermottrutherford.org/wellington,,85.175.44.175,"{""location"": ""CD"", ""is_mobile"": false}" 1070,1,25,2017-03-04 05:36:30,http://schmidt.name/estella_boehm,,114.6.217.245,"{""location"": ""MD"", ""is_mobile"": false}" 1071,1,25,2016-12-15 10:25:08,http://kunze.net/percy,,93.103.163.169,"{""location"": ""IQ"", ""is_mobile"": true}" 1072,1,25,2017-03-04 16:31:35,http://volkmanauer.org/amani,,226.150.133.70,"{""location"": ""CO"", ""is_mobile"": false}" 1073,1,25,2017-04-07 20:01:38,http://wilkinson.name/eduardo,,52.221.47.70,"{""location"": ""MQ"", ""is_mobile"": true}" 1074,1,25,2016-12-14 20:17:31,http://lynchemmerich.name/richmond,,236.73.231.238,"{""location"": ""CI"", ""is_mobile"": true}" 1075,1,25,2017-05-18 02:44:26,http://cain.com/javier_hoppe,,179.156.39.189,"{""location"": ""PE"", ""is_mobile"": false}" 1076,1,25,2017-01-24 12:48:45,http://bernhard.info/amelie,,102.226.221.140,"{""location"": ""GM"", ""is_mobile"": true}" 1077,1,25,2017-03-12 12:47:15,http://osinski.io/zackary.jones,,87.42.77.229,"{""location"": ""UA"", ""is_mobile"": true}" 1078,1,25,2016-12-28 19:30:59,http://boylekeler.biz/furman.lind,,73.83.177.118,"{""location"": ""LA"", ""is_mobile"": false}" 22110,8,495,2017-05-15 12:51:36,http://buckridgetorphy.name/javier.gibson,,231.229.104.11,"{""location"": ""FR"", ""is_mobile"": false}" 22111,8,495,2017-03-02 15:42:23,http://predovic.net/ricardo_wilderman,,119.32.137.126,"{""location"": ""CD"", ""is_mobile"": false}" 22112,8,495,2017-04-23 16:25:56,http://marks.com/haan.hammes,,112.227.223.206,"{""location"": ""CA"", ""is_mobile"": true}" 22113,8,495,2017-02-20 17:45:14,http://klocko.io/taurean,,12.183.199.191,"{""location"": ""BW"", ""is_mobile"": false}" 22114,8,495,2016-12-13 09:08:24,http://bechtelarkling.io/keenan,,225.81.53.3,"{""location"": ""SZ"", ""is_mobile"": false}" 22115,8,495,2017-02-14 01:06:46,http://rodriguezbogan.net/aleia.torp,,226.102.109.106,"{""location"": ""ET"", ""is_mobile"": false}" 22116,8,495,2017-01-06 18:04:52,http://balistrerijohnston.name/conner_steuber,,140.183.145.186,"{""location"": ""TG"", ""is_mobile"": true}" 22117,8,495,2017-01-05 23:27:28,http://maggiolynch.biz/milton,,154.5.46.29,"{""location"": ""NU"", ""is_mobile"": true}" 22118,8,495,2017-01-26 01:02:31,http://kuvaliskertzmann.name/camron_wilkinson,,123.143.88.86,"{""location"": ""BG"", ""is_mobile"": true}" 22119,8,495,2017-01-18 17:28:34,http://grimes.co/nathan,,118.95.243.63,"{""location"": ""YE"", ""is_mobile"": true}" 22120,8,495,2017-05-03 09:26:28,http://wilkinson.name/jamey_cremin,,6.13.11.27,"{""location"": ""SZ"", ""is_mobile"": false}" 22121,8,495,2017-06-01 00:48:09,http://windler.net/okey_langosh,,37.72.219.225,"{""location"": ""RO"", ""is_mobile"": true}" 22122,8,495,2017-01-03 00:25:35,http://koch.co/jacques,,10.251.199.216,"{""location"": ""MU"", ""is_mobile"": false}" 22123,8,495,2017-03-01 11:33:55,http://blick.name/devon.aufderhar,,218.120.174.50,"{""location"": ""FJ"", ""is_mobile"": false}" 22124,8,495,2017-05-21 05:47:51,http://kovacekhaag.io/jayce.wehner,,81.89.235.61,"{""location"": ""PH"", ""is_mobile"": true}" 22125,8,495,2017-03-21 04:02:14,http://nolanwitting.net/trycia.feeney,,81.210.71.7,"{""location"": ""SB"", ""is_mobile"": false}" 22126,8,495,2017-01-12 01:31:03,http://stroman.com/edward.tillman,,235.67.170.47,"{""location"": ""SB"", ""is_mobile"": true}" 22127,8,495,2017-01-23 08:49:52,http://mannbeatty.co/sandy.kunde,,181.232.212.78,"{""location"": ""TK"", ""is_mobile"": true}" 22128,8,495,2017-03-10 00:34:35,http://wieganddickinson.co/gene.trantow,,178.161.204.244,"{""location"": ""TL"", ""is_mobile"": false}" 22129,8,495,2017-06-01 18:48:11,http://barrows.org/carole,,36.45.243.203,"{""location"": ""PT"", ""is_mobile"": false}" 22130,8,496,2017-01-01 18:44:09,http://jones.info/marcellus,,151.133.163.88,"{""location"": ""IL"", ""is_mobile"": true}" 22131,8,496,2017-02-01 14:24:37,http://zieme.name/freddie,,203.32.75.137,"{""location"": ""EG"", ""is_mobile"": true}" 22132,8,496,2017-01-23 05:33:33,http://lindoreilly.info/eldred.hartmann,,205.215.102.176,"{""location"": ""SH"", ""is_mobile"": true}" 22133,8,496,2017-01-30 21:22:33,http://schinner.net/iliana,,161.17.155.131,"{""location"": ""ZW"", ""is_mobile"": false}" 22134,8,496,2017-01-03 15:49:29,http://turcottejohnston.com/rachael,,190.86.84.55,"{""location"": ""SN"", ""is_mobile"": true}" 22135,8,496,2016-12-30 14:21:15,http://wardjohnson.biz/donny_weinat,,188.143.242.135,"{""location"": ""UG"", ""is_mobile"": true}" 22136,8,496,2017-03-31 16:33:03,http://schaeferdach.name/heidi,,8.196.41.169,"{""location"": ""LA"", ""is_mobile"": false}" 22137,8,496,2017-01-06 07:35:12,http://kunze.name/lily,,7.49.154.121,"{""location"": ""NZ"", ""is_mobile"": false}" 22138,8,496,2017-04-06 12:05:02,http://rice.biz/ansel.watsica,,189.183.132.85,"{""location"": ""SY"", ""is_mobile"": true}" 22139,8,496,2017-01-08 17:06:27,http://koepp.biz/sam,,78.81.125.120,"{""location"": ""PS"", ""is_mobile"": false}" 22140,8,496,2017-04-15 09:27:33,http://dicki.co/veda,,115.55.102.18,"{""location"": ""KZ"", ""is_mobile"": false}" 22141,8,496,2017-03-09 06:38:40,http://sauer.biz/guy.dietrich,,30.247.46.66,"{""location"": ""MN"", ""is_mobile"": true}" 22142,8,496,2017-03-10 05:46:49,http://wittinggoodwin.net/favian,,155.105.178.213,"{""location"": ""MH"", ""is_mobile"": true}" 22143,8,496,2017-03-25 20:13:57,http://durganritchie.info/kaci_bogisich,,14.27.132.213,"{""location"": ""RU"", ""is_mobile"": true}" 22144,8,496,2017-06-02 10:59:20,http://bergnaumkshlerin.org/marlee,,225.164.126.31,"{""location"": ""DO"", ""is_mobile"": true}" 22145,8,496,2017-03-23 14:14:18,http://osinskihansen.net/kayley,,252.232.227.61,"{""location"": ""KY"", ""is_mobile"": true}" 22146,8,496,2017-06-02 16:51:27,http://hermann.biz/stephon_satterfield,,16.128.19.30,"{""location"": ""KM"", ""is_mobile"": true}" 22147,8,496,2017-05-09 10:52:27,http://botsfordnitzsche.net/kaandra_kuhn,,205.192.251.205,"{""location"": ""PH"", ""is_mobile"": false}" 22148,8,496,2017-05-18 15:11:42,http://stromanthiel.io/tyrel_kub,,217.36.96.190,"{""location"": ""GS"", ""is_mobile"": false}" 22149,8,496,2017-05-06 06:02:10,http://powlowski.com/dillon_torphy,,185.196.173.237,"{""location"": ""NU"", ""is_mobile"": true}" 22150,8,496,2017-04-15 17:48:19,http://jast.co/wellington_spencer,,135.175.216.69,"{""location"": ""BD"", ""is_mobile"": true}" 22151,8,496,2017-04-26 08:40:32,http://monahankulas.org/adrian_anderson,,183.251.123.80,"{""location"": ""HN"", ""is_mobile"": false}" 22152,8,496,2017-03-30 05:39:09,http://willmskirlin.net/bernardo_stracke,,92.213.107.2,"{""location"": ""MF"", ""is_mobile"": false}" 22153,8,496,2016-12-18 09:07:16,http://gislason.net/hertha,,150.167.243.22,"{""location"": ""MR"", ""is_mobile"": false}" 22154,8,496,2017-04-23 23:11:40,http://gaylord.com/erna.satterfield,,229.163.77.17,"{""location"": ""NF"", ""is_mobile"": false}" 22155,8,496,2017-06-05 15:40:16,http://bauch.co/patience,,222.116.19.150,"{""location"": ""GH"", ""is_mobile"": false}" 22156,8,496,2017-04-19 04:44:46,http://schultz.org/jeie,,68.76.212.85,"{""location"": ""MH"", ""is_mobile"": true}" 22157,8,496,2017-04-10 12:27:14,http://runolfon.co/ocie.leffler,,59.179.154.40,"{""location"": ""UZ"", ""is_mobile"": false}" 22158,8,496,2017-01-16 03:29:09,http://weberschroeder.info/annamae,,21.106.134.245,"{""location"": ""RE"", ""is_mobile"": true}" 22159,8,496,2017-06-09 14:12:59,http://okeefe.info/hailie.gibson,,88.55.253.127,"{""location"": ""AI"", ""is_mobile"": true}" 22160,8,496,2017-04-08 08:18:09,http://monahan.org/haie_powlowski,,2.221.159.126,"{""location"": ""MQ"", ""is_mobile"": false}" 22161,8,496,2017-03-13 15:44:24,http://bahringer.co/davin,,95.224.47.241,"{""location"": ""TJ"", ""is_mobile"": false}" 22162,8,496,2017-04-14 10:47:18,http://von.io/jerrod,,163.111.186.158,"{""location"": ""MT"", ""is_mobile"": true}" 22163,8,496,2017-03-30 23:50:38,http://wuckertgibson.net/florence,,134.66.216.195,"{""location"": ""LS"", ""is_mobile"": true}" 22164,8,496,2017-01-03 04:28:33,http://ziemann.biz/marilou,,5.71.243.94,"{""location"": ""TG"", ""is_mobile"": true}" 1079,1,25,2017-03-01 07:44:09,http://lakinwalter.co/maxine.swaniawski,,197.12.249.27,"{""location"": ""BO"", ""is_mobile"": false}" 1080,1,25,2017-04-29 06:19:02,http://gleichnerokuneva.name/carole,,68.69.166.152,"{""location"": ""NP"", ""is_mobile"": false}" 1081,1,25,2017-01-04 03:02:07,http://kovacek.name/jey,,195.232.106.119,"{""location"": ""LV"", ""is_mobile"": true}" 1082,1,25,2017-03-17 16:46:46,http://rogahndonnelly.org/marlee,,156.201.46.80,"{""location"": ""RE"", ""is_mobile"": false}" 1083,1,25,2017-02-19 01:31:41,http://halvorson.com/olaf_schaefer,,226.101.232.100,"{""location"": ""UZ"", ""is_mobile"": true}" 1084,1,25,2017-02-09 04:58:36,http://sengervandervort.com/nannie.marks,,16.16.253.108,"{""location"": ""DJ"", ""is_mobile"": true}" 1085,1,25,2017-03-17 15:18:32,http://schowalter.com/yvonne_rohan,,134.94.10.173,"{""location"": ""KE"", ""is_mobile"": true}" 1086,1,25,2017-03-01 11:31:07,http://parisian.net/ariane,,37.155.178.171,"{""location"": ""CU"", ""is_mobile"": true}" 1087,1,25,2017-04-09 19:12:59,http://beattygislason.biz/erica,,61.35.36.172,"{""location"": ""VN"", ""is_mobile"": true}" 1088,1,25,2017-01-09 01:46:56,http://nolan.net/dewitt.beahan,,166.11.125.91,"{""location"": ""JE"", ""is_mobile"": true}" 1089,1,25,2017-01-23 20:07:46,http://littlesauer.info/sydney.boehm,,253.158.102.3,"{""location"": ""SA"", ""is_mobile"": true}" 1090,1,25,2017-03-06 08:26:00,http://weimannmayert.io/margarette_runte,,17.30.86.76,"{""location"": ""IM"", ""is_mobile"": false}" 1091,1,25,2017-02-18 16:50:13,http://runolfsdottir.io/oswaldo.cremin,,73.228.73.124,"{""location"": ""YT"", ""is_mobile"": true}" 1092,1,25,2017-01-07 12:17:51,http://schmeler.name/amir.welch,,69.32.156.26,"{""location"": ""ER"", ""is_mobile"": true}" 1093,1,25,2017-01-13 07:44:17,http://steuber.net/buford.bahringer,,192.59.220.139,"{""location"": ""IR"", ""is_mobile"": true}" 1094,1,25,2017-03-24 19:10:43,http://miller.co/celestine_treutel,,215.70.175.52,"{""location"": ""AQ"", ""is_mobile"": false}" 1095,1,26,2017-02-21 20:58:43,http://gleason.net/nathen.parisian,,193.128.177.24,"{""location"": ""PF"", ""is_mobile"": true}" 1096,1,26,2017-04-25 06:36:56,http://brown.biz/lois,,45.14.134.67,"{""location"": ""HN"", ""is_mobile"": false}" 1097,1,26,2017-01-20 02:35:52,http://hayesgottlieb.info/maynard.feeney,,167.188.77.108,"{""location"": ""KY"", ""is_mobile"": true}" 1098,1,26,2017-04-03 23:47:47,http://aufderhar.info/muhammad,,235.180.98.160,"{""location"": ""BE"", ""is_mobile"": true}" 1099,1,26,2017-03-06 09:56:16,http://schoen.info/gerard,,144.98.82.192,"{""location"": ""JP"", ""is_mobile"": true}" 1100,1,26,2017-01-21 18:53:29,http://boganrowe.net/merritt,,18.218.175.123,"{""location"": ""GY"", ""is_mobile"": true}" 1101,1,26,2017-06-10 13:47:26,http://schiller.net/shanny,,31.193.88.152,"{""location"": ""MF"", ""is_mobile"": false}" 1102,1,26,2017-02-16 10:28:30,http://quitzon.info/lester,,179.142.252.238,"{""location"": ""TM"", ""is_mobile"": true}" 1103,1,26,2017-04-13 05:02:20,http://ziemannstanton.net/juwan,,178.194.40.165,"{""location"": ""SR"", ""is_mobile"": false}" 1104,1,26,2017-03-08 16:00:20,http://feilhilll.info/sven,,253.9.200.58,"{""location"": ""TV"", ""is_mobile"": true}" 1105,1,26,2017-05-13 03:33:29,http://botsford.info/lacy_corwin,,181.130.152.223,"{""location"": ""KH"", ""is_mobile"": true}" 1106,1,26,2017-06-09 00:00:14,http://hellermcclure.org/kendall,,120.229.175.173,"{""location"": ""SA"", ""is_mobile"": true}" 1107,1,26,2017-05-29 22:14:05,http://becker.biz/laverna_reilly,,94.211.243.83,"{""location"": ""VI"", ""is_mobile"": false}" 1108,1,26,2017-05-26 19:23:46,http://greenfelder.org/delia,,187.233.127.122,"{""location"": ""SE"", ""is_mobile"": true}" 1109,1,26,2017-01-20 01:14:08,http://murazik.biz/warren_mueller,,94.94.33.124,"{""location"": ""BJ"", ""is_mobile"": false}" 1110,1,26,2017-04-25 07:47:15,http://gutmannschneider.io/wilson_ondricka,,112.252.251.13,"{""location"": ""BR"", ""is_mobile"": false}" 1111,1,26,2017-04-03 06:46:32,http://gleichner.info/guillermo,,59.112.58.252,"{""location"": ""AR"", ""is_mobile"": false}" 1112,1,26,2017-04-24 10:40:30,http://medhurstkovacek.io/hyman_lang,,118.40.175.42,"{""location"": ""GT"", ""is_mobile"": false}" 1113,1,26,2016-12-13 12:40:45,http://bruen.info/kamron,,156.60.231.181,"{""location"": ""AS"", ""is_mobile"": true}" 1114,1,26,2017-01-26 12:16:28,http://shields.net/estefania_zemlak,,90.251.103.22,"{""location"": ""GP"", ""is_mobile"": true}" 1115,1,26,2017-04-05 14:52:25,http://weimann.co/tate,,117.79.37.81,"{""location"": ""MU"", ""is_mobile"": false}" 1116,1,26,2017-05-21 20:46:58,http://rodriguez.com/kenna,,3.42.182.95,"{""location"": ""AL"", ""is_mobile"": false}" 1117,1,27,2017-03-18 12:00:29,http://hillslind.com/marguerite,,51.116.178.157,"{""location"": ""BZ"", ""is_mobile"": false}" 1118,1,27,2017-05-10 02:30:30,http://koeppkihn.net/janick,,164.95.139.75,"{""location"": ""TM"", ""is_mobile"": false}" 1119,1,27,2017-01-17 10:33:57,http://oberbrunnerpagac.name/mara,,156.30.147.165,"{""location"": ""GY"", ""is_mobile"": true}" 1120,1,27,2016-12-20 23:31:04,http://rosenbaum.org/sandra,,93.215.215.120,"{""location"": ""ZA"", ""is_mobile"": false}" 1121,1,27,2017-01-18 08:52:38,http://hartmann.info/davon,,57.155.209.253,"{""location"": ""UY"", ""is_mobile"": true}" 1122,1,27,2017-04-26 11:02:59,http://lesch.info/jewell,,109.151.115.86,"{""location"": ""UM"", ""is_mobile"": false}" 1123,1,27,2017-02-23 06:16:29,http://kuvalis.biz/gladyce,,122.94.29.235,"{""location"": ""BB"", ""is_mobile"": false}" 1124,1,27,2016-12-22 07:27:22,http://nolanschiller.org/miles,,254.77.16.189,"{""location"": ""NA"", ""is_mobile"": false}" 1125,1,27,2017-02-14 12:50:27,http://powlowski.co/andy,,12.62.240.21,"{""location"": ""GB"", ""is_mobile"": false}" 1126,1,27,2017-03-15 10:06:43,http://mraz.biz/bridgette,,30.5.138.212,"{""location"": ""FO"", ""is_mobile"": false}" 1127,1,27,2017-02-04 07:03:16,http://ricestoltenberg.info/salma.kshlerin,,76.204.53.87,"{""location"": ""AX"", ""is_mobile"": false}" 1128,1,27,2017-04-14 04:26:26,http://gleason.info/roxanne_walsh,,134.94.37.213,"{""location"": ""DZ"", ""is_mobile"": false}" 1129,1,27,2017-05-03 16:50:59,http://dooleyhand.info/amara,,229.235.225.154,"{""location"": ""AX"", ""is_mobile"": true}" 1130,1,27,2017-02-15 12:26:51,http://stantonkerluke.info/randi,,133.242.185.40,"{""location"": ""ZA"", ""is_mobile"": true}" 1131,1,27,2017-03-27 16:45:43,http://hanechristiansen.io/jeica,,61.156.223.202,"{""location"": ""CX"", ""is_mobile"": true}" 1132,1,27,2016-12-26 07:39:19,http://kihncasper.com/mckayla.zboncak,,149.109.195.155,"{""location"": ""AU"", ""is_mobile"": false}" 1133,1,27,2017-04-25 16:16:08,http://olson.com/alverta_ziemann,,114.112.213.172,"{""location"": ""NR"", ""is_mobile"": true}" 1134,1,27,2017-06-06 09:44:13,http://feeney.org/lyla,,51.146.109.132,"{""location"": ""GN"", ""is_mobile"": false}" 22165,8,496,2017-01-04 12:29:29,http://langworth.info/jee.kozey,,89.174.203.75,"{""location"": ""TH"", ""is_mobile"": true}" 22166,8,496,2017-02-23 10:31:24,http://ward.net/avis,,35.47.170.76,"{""location"": ""LY"", ""is_mobile"": false}" 22167,8,496,2016-12-13 09:36:46,http://spencerdare.io/delores,,220.14.9.97,"{""location"": ""NI"", ""is_mobile"": false}" 22168,8,496,2017-02-01 04:04:52,http://larson.name/ruben,,97.23.180.208,"{""location"": ""SO"", ""is_mobile"": true}" 22169,8,496,2017-03-31 20:51:09,http://kozey.biz/ila.effertz,,197.58.186.217,"{""location"": ""EH"", ""is_mobile"": false}" 22170,8,496,2017-03-08 03:46:26,http://wiegand.name/robin_wunsch,,219.76.131.76,"{""location"": ""TZ"", ""is_mobile"": true}" 22171,8,496,2017-03-27 16:02:08,http://kochrosenbaum.info/kyla,,106.221.13.95,"{""location"": ""VI"", ""is_mobile"": true}" 22172,8,496,2016-12-16 15:19:52,http://windler.name/antone,,206.238.231.191,"{""location"": ""ZW"", ""is_mobile"": true}" 22173,8,496,2017-01-20 08:00:29,http://turner.org/rozella_howell,,131.52.152.74,"{""location"": ""MF"", ""is_mobile"": false}" 22174,8,496,2017-03-09 00:20:49,http://mannfeeney.com/marcia.lebsack,,225.219.39.129,"{""location"": ""SL"", ""is_mobile"": true}" 22175,8,496,2017-03-01 14:30:43,http://nolan.io/caroline_lebsack,,103.27.160.227,"{""location"": ""SJ"", ""is_mobile"": true}" 22176,8,496,2017-02-11 23:56:12,http://auereichmann.co/don,,92.85.87.154,"{""location"": ""PF"", ""is_mobile"": false}" 22177,8,496,2017-06-05 20:51:33,http://marvinabbott.name/lorena_hilll,,204.43.235.108,"{""location"": ""QA"", ""is_mobile"": true}" 22178,8,497,2017-03-08 03:22:02,http://kshlerin.info/godfrey.nitzsche,,3.246.99.180,"{""location"": ""KW"", ""is_mobile"": true}" 22179,8,497,2017-01-22 05:58:32,http://hegmannmayer.info/hope_mosciski,,77.115.39.58,"{""location"": ""MA"", ""is_mobile"": false}" 22180,8,497,2017-04-17 01:19:13,http://gloverarmstrong.net/tevin.hilll,,248.126.131.210,"{""location"": ""IO"", ""is_mobile"": true}" 22181,8,497,2017-03-12 05:42:26,http://mayert.biz/antonetta_buckridge,,69.201.116.22,"{""location"": ""BS"", ""is_mobile"": false}" 22182,8,497,2017-05-25 04:14:34,http://reichel.co/kaci,,93.44.70.144,"{""location"": ""AT"", ""is_mobile"": true}" 22183,8,497,2017-04-07 14:51:06,http://lind.org/vince,,9.33.63.212,"{""location"": ""MY"", ""is_mobile"": false}" 22184,8,497,2017-01-02 20:03:29,http://spencer.name/alexandre,,44.201.234.214,"{""location"": ""LU"", ""is_mobile"": true}" 22185,8,497,2017-06-01 22:06:10,http://dare.co/broderick,,239.125.204.25,"{""location"": ""GQ"", ""is_mobile"": true}" 22186,8,497,2017-04-10 23:10:54,http://lynch.info/violette,,8.5.206.65,"{""location"": ""TF"", ""is_mobile"": false}" 22187,8,497,2017-01-21 15:57:46,http://bogan.org/alison_rau,,164.168.2.49,"{""location"": ""SX"", ""is_mobile"": true}" 22188,8,497,2016-12-25 02:06:27,http://glover.net/bernard.kreiger,,133.225.62.45,"{""location"": ""GN"", ""is_mobile"": false}" 22189,8,497,2016-12-14 13:15:57,http://hamillhaag.net/viva_schiller,,209.184.143.56,"{""location"": ""SV"", ""is_mobile"": false}" 22190,8,497,2017-03-12 10:31:36,http://kunde.co/jadon_spinka,,180.96.105.99,"{""location"": ""MZ"", ""is_mobile"": true}" 22191,8,497,2017-05-06 08:49:57,http://white.info/sven_murray,,131.76.3.71,"{""location"": ""TV"", ""is_mobile"": true}" 22192,8,497,2017-05-19 18:56:23,http://batz.com/lenny,,4.11.63.54,"{""location"": ""JO"", ""is_mobile"": false}" 22193,8,497,2017-06-03 16:09:03,http://carrollmacejkovic.biz/donald,,44.108.145.51,"{""location"": ""TZ"", ""is_mobile"": true}" 22194,8,497,2017-05-17 13:42:58,http://legrosjohnson.net/rudolph.champlin,,233.172.28.108,"{""location"": ""MD"", ""is_mobile"": true}" 22195,8,497,2017-01-07 11:30:17,http://effertz.org/aida.von,,190.99.115.56,"{""location"": ""ST"", ""is_mobile"": true}" 22196,8,497,2017-02-01 21:07:20,http://rutherford.io/leola,,186.106.83.31,"{""location"": ""MZ"", ""is_mobile"": false}" 22197,8,497,2017-05-20 11:47:46,http://rosenbaumbogisich.net/dashawn,,159.44.222.106,"{""location"": ""EE"", ""is_mobile"": false}" 22198,8,497,2017-01-13 19:58:09,http://reichel.co/shanel,,56.174.69.148,"{""location"": ""SG"", ""is_mobile"": true}" 22199,8,497,2017-01-18 17:02:37,http://ondrickasatterfield.net/jeff,,210.193.46.164,"{""location"": ""KN"", ""is_mobile"": false}" 22200,8,497,2017-05-15 18:56:23,http://bailey.info/johanna,,35.176.199.233,"{""location"": ""GB"", ""is_mobile"": true}" 22201,8,497,2017-01-19 01:18:35,http://mohrdibbert.biz/leta,,146.156.212.234,"{""location"": ""SY"", ""is_mobile"": false}" 22202,8,497,2017-04-21 17:03:21,http://hoegerjaskolski.io/charlie,,80.181.212.170,"{""location"": ""YT"", ""is_mobile"": true}" 22203,8,497,2017-03-04 15:21:57,http://langosh.io/orpha,,217.83.95.236,"{""location"": ""EC"", ""is_mobile"": false}" 22204,8,497,2017-02-16 15:18:39,http://torp.com/matilde,,72.89.35.26,"{""location"": ""ES"", ""is_mobile"": false}" 22205,8,497,2017-02-06 23:57:37,http://gerlach.co/jayce,,169.198.78.222,"{""location"": ""BW"", ""is_mobile"": true}" 22206,8,497,2017-03-13 09:21:07,http://lehnerupton.com/rosella,,132.10.59.172,"{""location"": ""AL"", ""is_mobile"": false}" 22207,8,497,2017-06-11 22:19:03,http://mcdermott.biz/gertrude_fay,,67.143.35.3,"{""location"": ""WS"", ""is_mobile"": false}" 22208,8,497,2017-05-29 09:07:29,http://rath.name/elliot_raynor,,102.252.130.43,"{""location"": ""EE"", ""is_mobile"": true}" 22209,8,497,2017-01-19 14:38:39,http://terryblanda.net/madie,,105.238.242.61,"{""location"": ""LS"", ""is_mobile"": true}" 22210,8,497,2017-02-27 15:08:07,http://mcdermott.net/green.gerhold,,94.160.166.245,"{""location"": ""MV"", ""is_mobile"": false}" 22211,8,497,2017-05-18 18:12:23,http://ziemann.org/alberto.block,,103.156.146.50,"{""location"": ""IL"", ""is_mobile"": true}" 22212,8,497,2017-04-27 03:19:28,http://jerde.biz/maximo,,202.107.42.65,"{""location"": ""KY"", ""is_mobile"": true}" 22213,8,497,2017-01-08 22:37:38,http://fahey.biz/enrique_brekke,,16.39.113.108,"{""location"": ""LA"", ""is_mobile"": true}" 22214,8,497,2017-05-22 13:26:54,http://heller.co/destinee_grimes,,218.64.13.242,"{""location"": ""MO"", ""is_mobile"": false}" 22215,8,497,2017-05-11 14:04:59,http://tillmanbartoletti.io/cordie.kulas,,29.120.136.66,"{""location"": ""TR"", ""is_mobile"": false}" 22216,8,497,2017-05-12 01:45:44,http://gleason.name/arlene,,99.160.192.86,"{""location"": ""NP"", ""is_mobile"": false}" 22217,8,497,2016-12-18 02:22:38,http://haag.io/dayana,,132.144.185.125,"{""location"": ""TJ"", ""is_mobile"": false}" 22218,8,497,2016-12-17 02:19:08,http://stantonkirlin.name/anika_bartoletti,,78.142.111.206,"{""location"": ""AZ"", ""is_mobile"": true}" 22448,8,503,2017-05-23 17:50:31,http://mayert.info/sydni,,252.136.82.196,"{""location"": ""CV"", ""is_mobile"": true}" 1135,1,27,2017-03-01 08:52:55,http://emmerich.org/garland.kutch,,189.29.200.198,"{""location"": ""LI"", ""is_mobile"": false}" 1136,1,27,2017-01-05 20:20:25,http://moore.io/peyton.collier,,183.107.225.13,"{""location"": ""HT"", ""is_mobile"": true}" 1137,1,27,2017-04-24 19:31:31,http://treutelabbott.io/craig_gerhold,,67.11.196.61,"{""location"": ""MD"", ""is_mobile"": true}" 1138,1,27,2017-04-26 14:54:59,http://greenblanda.net/xzavier.rolfson,,222.231.147.63,"{""location"": ""CA"", ""is_mobile"": false}" 1139,1,27,2017-02-05 16:29:54,http://jacobsonmayert.biz/milan.nader,,158.160.245.169,"{""location"": ""TJ"", ""is_mobile"": false}" 1140,1,27,2017-01-31 18:15:17,http://kunze.name/kayden_sawayn,,209.185.45.209,"{""location"": ""SI"", ""is_mobile"": true}" 1141,1,27,2017-01-03 01:04:32,http://sawayn.info/brice.kreiger,,190.140.48.208,"{""location"": ""PA"", ""is_mobile"": true}" 1142,1,27,2017-04-15 01:36:49,http://gutmann.info/neal,,247.144.78.25,"{""location"": ""EG"", ""is_mobile"": false}" 1143,1,27,2017-03-03 14:34:35,http://ortiz.name/abigayle_wolff,,144.70.164.114,"{""location"": ""US"", ""is_mobile"": true}" 1144,1,27,2017-01-11 14:20:03,http://monahan.net/jefferey,,154.172.134.4,"{""location"": ""MQ"", ""is_mobile"": false}" 1145,1,27,2017-05-08 17:03:43,http://stoltenberg.net/brycen,,177.243.135.4,"{""location"": ""AT"", ""is_mobile"": false}" 1146,1,27,2017-04-29 02:30:06,http://hellerfunk.net/brandy,,219.23.174.109,"{""location"": ""KN"", ""is_mobile"": false}" 1147,1,27,2017-02-08 23:15:42,http://mcclure.com/ozzie,,86.235.161.124,"{""location"": ""CI"", ""is_mobile"": true}" 1148,1,27,2017-04-28 02:46:35,http://schiller.com/faye.okeefe,,91.165.67.180,"{""location"": ""IE"", ""is_mobile"": true}" 1149,1,27,2017-01-03 07:38:01,http://moen.io/mervin,,83.187.178.148,"{""location"": ""TC"", ""is_mobile"": true}" 1150,1,27,2017-04-20 02:47:11,http://walsh.net/brionna.mann,,127.61.221.16,"{""location"": ""CU"", ""is_mobile"": false}" 1151,1,27,2017-05-22 18:15:59,http://vonhahn.co/lincoln_huel,,97.114.133.203,"{""location"": ""UM"", ""is_mobile"": false}" 1152,1,27,2017-01-30 14:00:10,http://mantevandervort.com/krystina_lueilwitz,,209.20.246.47,"{""location"": ""MR"", ""is_mobile"": false}" 1153,1,27,2017-01-21 10:36:02,http://okon.net/gideon_weinat,,41.133.232.253,"{""location"": ""ME"", ""is_mobile"": false}" 1154,1,27,2017-05-03 15:04:06,http://heller.org/justine,,200.143.192.68,"{""location"": ""LC"", ""is_mobile"": true}" 1155,1,27,2017-02-04 22:45:25,http://rooboreilly.name/jonas,,72.180.80.254,"{""location"": ""MP"", ""is_mobile"": true}" 1156,1,27,2017-01-19 02:37:24,http://barton.name/gina.jast,,160.129.77.114,"{""location"": ""EH"", ""is_mobile"": true}" 1157,1,27,2017-03-21 16:31:26,http://littel.com/dawn,,187.177.133.86,"{""location"": ""PM"", ""is_mobile"": true}" 1158,1,27,2016-12-27 00:51:32,http://doyle.net/clement,,179.41.202.104,"{""location"": ""RW"", ""is_mobile"": false}" 1159,1,27,2017-02-07 04:26:17,http://johnson.net/vance_boyer,,241.21.167.17,"{""location"": ""SG"", ""is_mobile"": false}" 1160,1,27,2017-02-15 04:26:07,http://botsford.org/corrine.abernathy,,161.227.178.169,"{""location"": ""AF"", ""is_mobile"": true}" 1161,1,27,2016-12-30 23:37:29,http://mante.co/noel.schmeler,,48.27.101.2,"{""location"": ""SN"", ""is_mobile"": true}" 1162,1,27,2017-01-30 19:53:32,http://reichert.io/grace.larson,,95.216.171.70,"{""location"": ""EG"", ""is_mobile"": false}" 1848,1,41,2017-05-14 15:56:19,http://mann.name/carroll,,30.180.112.155,"{""location"": ""RO"", ""is_mobile"": true}" 1163,1,27,2017-01-09 04:23:18,http://reynoldswilliamson.net/polly,,98.128.19.230,"{""location"": ""DZ"", ""is_mobile"": true}" 1164,1,27,2017-03-02 23:23:24,http://klocko.net/german,,156.47.8.191,"{""location"": ""LK"", ""is_mobile"": true}" 1165,1,27,2017-05-20 04:18:27,http://willms.org/asa,,227.8.150.87,"{""location"": ""KH"", ""is_mobile"": true}" 1166,1,27,2017-01-20 00:47:18,http://ledner.biz/victoria,,108.133.82.9,"{""location"": ""DO"", ""is_mobile"": false}" 1167,1,27,2017-04-23 18:23:16,http://beer.info/susana,,214.176.240.247,"{""location"": ""IL"", ""is_mobile"": true}" 1168,1,27,2017-04-03 20:14:23,http://pagac.name/jailyn_abshire,,188.204.95.4,"{""location"": ""VN"", ""is_mobile"": true}" 1169,1,27,2017-03-02 11:23:17,http://pollichdibbert.biz/carey,,247.45.150.148,"{""location"": ""HT"", ""is_mobile"": false}" 1170,1,27,2017-03-28 16:12:46,http://ledner.io/marianna_schmeler,,156.103.157.5,"{""location"": ""ER"", ""is_mobile"": true}" 1171,1,27,2016-12-30 11:52:17,http://franecki.com/kathryne,,207.185.94.141,"{""location"": ""MS"", ""is_mobile"": true}" 1172,1,27,2017-04-23 14:25:14,http://quigley.name/kris_robel,,121.85.202.159,"{""location"": ""BT"", ""is_mobile"": false}" 1173,1,27,2017-05-21 15:26:38,http://jacobs.co/zora,,200.121.7.38,"{""location"": ""PR"", ""is_mobile"": true}" 1174,1,27,2017-01-05 12:58:28,http://wisozk.info/ania,,32.129.163.87,"{""location"": ""LT"", ""is_mobile"": true}" 1175,1,27,2017-03-15 08:27:44,http://jaskolski.co/alvena_keler,,35.54.222.168,"{""location"": ""AS"", ""is_mobile"": true}" 1176,1,28,2017-04-04 17:38:17,http://millerrath.name/estrella,,243.229.91.212,"{""location"": ""ST"", ""is_mobile"": true}" 1177,1,28,2017-01-27 06:07:56,http://emard.name/chase,,165.52.4.239,"{""location"": ""MR"", ""is_mobile"": true}" 1178,1,28,2017-03-24 05:06:28,http://anderson.biz/arlene,,220.96.182.181,"{""location"": ""DK"", ""is_mobile"": false}" 1179,1,28,2017-01-15 15:04:30,http://effertz.net/elbert_homenick,,113.29.227.210,"{""location"": ""SX"", ""is_mobile"": true}" 1180,1,28,2016-12-20 05:30:59,http://larson.info/tyreek,,23.187.94.90,"{""location"": ""BW"", ""is_mobile"": false}" 1181,1,28,2017-05-08 05:40:36,http://roweolson.name/ena,,224.61.225.20,"{""location"": ""KN"", ""is_mobile"": true}" 1182,1,28,2017-05-23 02:15:25,http://rath.biz/ryleigh,,43.193.105.58,"{""location"": ""TJ"", ""is_mobile"": true}" 1183,1,28,2017-04-29 00:45:42,http://darehermiston.co/ricky,,102.159.136.211,"{""location"": ""HK"", ""is_mobile"": false}" 1184,1,28,2017-02-06 00:05:57,http://paucek.biz/danika,,62.12.148.28,"{""location"": ""GP"", ""is_mobile"": true}" 1185,1,28,2017-01-07 01:34:33,http://batz.biz/liana.volkman,,25.40.226.161,"{""location"": ""SO"", ""is_mobile"": true}" 1186,1,28,2017-03-22 23:30:26,http://wolf.com/hailey,,134.82.133.17,"{""location"": ""CN"", ""is_mobile"": true}" 1187,1,28,2017-02-07 22:43:44,http://schiller.name/annie,,148.87.9.3,"{""location"": ""TN"", ""is_mobile"": false}" 1188,1,28,2017-04-28 10:46:12,http://boyer.biz/mariela.dach,,10.70.167.198,"{""location"": ""PK"", ""is_mobile"": true}" 1189,1,28,2017-04-06 00:52:44,http://pacocha.biz/keegan,,101.33.83.152,"{""location"": ""LC"", ""is_mobile"": true}" 22219,8,498,2017-03-06 23:16:19,http://lemke.name/davonte,,180.96.146.109,"{""location"": ""EE"", ""is_mobile"": false}" 22220,8,498,2017-02-10 11:01:00,http://ruel.info/phoebe,,80.209.110.39,"{""location"": ""NI"", ""is_mobile"": false}" 22221,8,498,2016-12-18 22:22:59,http://grantsawayn.com/aiyana,,68.85.179.176,"{""location"": ""KH"", ""is_mobile"": false}" 22222,8,498,2017-04-04 08:24:39,http://schusteremmerich.name/clifford,,101.206.36.193,"{""location"": ""RW"", ""is_mobile"": true}" 22223,8,498,2017-04-29 03:30:32,http://bogan.name/brandy_glover,,227.7.48.60,"{""location"": ""TL"", ""is_mobile"": false}" 22224,8,498,2017-01-15 00:58:41,http://durganheidenreich.biz/neva,,190.220.201.63,"{""location"": ""SS"", ""is_mobile"": false}" 22225,8,498,2017-01-04 14:10:45,http://schultztremblay.io/kelli,,174.49.197.252,"{""location"": ""BY"", ""is_mobile"": false}" 22226,8,498,2017-05-17 10:10:28,http://sauerhickle.org/rudy_nienow,,62.73.253.25,"{""location"": ""SB"", ""is_mobile"": false}" 22227,8,498,2017-06-02 13:17:34,http://windlergulgowski.biz/doug,,152.135.25.163,"{""location"": ""RO"", ""is_mobile"": false}" 22228,8,498,2017-03-17 08:46:21,http://renner.info/ashtyn.torphy,,89.115.186.75,"{""location"": ""KM"", ""is_mobile"": false}" 22229,8,498,2016-12-27 12:09:21,http://toy.net/ruby,,201.9.73.58,"{""location"": ""IT"", ""is_mobile"": false}" 22230,8,498,2016-12-26 07:08:33,http://cummings.org/claudine.schinner,,81.100.108.136,"{""location"": ""VI"", ""is_mobile"": true}" 22231,8,498,2017-02-04 20:58:00,http://franecki.co/clifford,,227.41.228.104,"{""location"": ""IE"", ""is_mobile"": true}" 22232,8,498,2017-02-03 03:32:46,http://denesik.name/darrel,,226.151.168.25,"{""location"": ""MM"", ""is_mobile"": false}" 22233,8,498,2017-06-09 08:45:27,http://ryanboyer.co/axel,,250.29.14.36,"{""location"": ""MH"", ""is_mobile"": true}" 22234,8,498,2017-03-17 12:19:00,http://mayertfunk.info/cayla,,203.33.26.88,"{""location"": ""FJ"", ""is_mobile"": true}" 22235,8,498,2017-05-09 12:00:51,http://yundt.co/america,,238.144.240.188,"{""location"": ""OM"", ""is_mobile"": false}" 22236,8,498,2016-12-31 10:37:04,http://runteortiz.name/osvaldo_homenick,,251.217.101.246,"{""location"": ""DJ"", ""is_mobile"": false}" 22237,8,498,2017-02-21 16:34:27,http://gutkowskicole.net/eusebio,,194.74.153.160,"{""location"": ""DJ"", ""is_mobile"": true}" 22238,8,498,2017-01-29 03:10:40,http://volkman.co/ramona_kerluke,,201.45.25.163,"{""location"": ""UZ"", ""is_mobile"": false}" 22239,8,498,2016-12-15 05:11:57,http://robel.io/alisa,,175.162.104.35,"{""location"": ""GG"", ""is_mobile"": false}" 22240,8,498,2017-03-12 20:38:00,http://ondrickarolfson.net/odell_quitzon,,109.201.99.209,"{""location"": ""BN"", ""is_mobile"": true}" 22241,8,498,2017-05-01 12:12:41,http://jastweinat.io/maria_bailey,,44.249.30.194,"{""location"": ""LB"", ""is_mobile"": false}" 22242,8,498,2017-05-13 18:45:24,http://schuster.io/jimmie,,83.194.176.3,"{""location"": ""VG"", ""is_mobile"": false}" 22243,8,498,2017-05-04 01:00:41,http://hayes.co/emanuel_predovic,,10.92.91.13,"{""location"": ""FR"", ""is_mobile"": false}" 22244,8,498,2017-04-07 03:12:55,http://cristkutch.com/jerad,,111.24.49.202,"{""location"": ""JM"", ""is_mobile"": false}" 22245,8,498,2017-05-28 07:00:47,http://cartermcdermott.info/brady.green,,95.159.175.124,"{""location"": ""CX"", ""is_mobile"": false}" 22246,8,498,2017-02-12 20:34:56,http://moriettehilll.co/collin_christiansen,,150.77.190.238,"{""location"": ""LS"", ""is_mobile"": false}" 22247,8,498,2017-02-01 18:37:07,http://johnston.info/marisa.wolff,,148.25.82.24,"{""location"": ""GL"", ""is_mobile"": true}" 22248,8,498,2017-05-21 11:50:16,http://emard.biz/jake,,253.213.224.186,"{""location"": ""IS"", ""is_mobile"": true}" 22249,8,498,2017-01-07 03:20:28,http://cronin.name/domenic_morar,,239.138.98.156,"{""location"": ""SI"", ""is_mobile"": false}" 22250,8,498,2017-03-24 01:18:10,http://heaney.name/isac.gulgowski,,72.143.244.9,"{""location"": ""PA"", ""is_mobile"": false}" 22251,8,498,2017-03-25 14:21:11,http://runte.net/lillian.yost,,165.79.165.123,"{""location"": ""JO"", ""is_mobile"": true}" 22252,8,498,2017-05-16 07:00:43,http://tillman.io/amos.cronin,,67.53.27.124,"{""location"": ""AZ"", ""is_mobile"": true}" 22253,8,498,2017-01-11 07:08:31,http://fadel.co/terrence,,34.14.236.196,"{""location"": ""KY"", ""is_mobile"": true}" 22254,8,498,2016-12-22 07:24:30,http://ricerice.info/keven,,146.72.44.181,"{""location"": ""GI"", ""is_mobile"": true}" 22255,8,498,2017-05-30 00:07:20,http://friesen.info/seamus_deckow,,87.87.150.89,"{""location"": ""KG"", ""is_mobile"": false}" 22256,8,498,2017-01-09 14:47:23,http://fisherswaniawski.biz/amari,,194.16.124.206,"{""location"": ""BB"", ""is_mobile"": false}" 22257,8,498,2017-06-08 06:12:05,http://dooley.co/kameron,,34.18.87.195,"{""location"": ""SS"", ""is_mobile"": false}" 22258,8,498,2017-02-16 20:47:10,http://feeney.name/abdiel.gleichner,,90.92.147.228,"{""location"": ""BE"", ""is_mobile"": false}" 22259,8,498,2017-01-27 22:36:00,http://volkmantremblay.co/deborah,,221.123.138.137,"{""location"": ""IS"", ""is_mobile"": false}" 22260,8,498,2017-01-14 22:31:55,http://sauer.net/nash,,62.180.159.142,"{""location"": ""TF"", ""is_mobile"": true}" 22261,8,498,2017-02-13 12:46:29,http://roob.org/francisco,,4.159.120.159,"{""location"": ""BH"", ""is_mobile"": true}" 22262,8,498,2017-05-16 10:15:21,http://walsh.biz/alyson_boyer,,42.168.113.171,"{""location"": ""KM"", ""is_mobile"": false}" 22263,8,498,2017-03-03 05:29:47,http://kilback.co/misty_miller,,68.127.146.69,"{""location"": ""BJ"", ""is_mobile"": false}" 22264,8,498,2017-05-15 05:21:29,http://wisozk.org/angelo,,253.97.208.115,"{""location"": ""LS"", ""is_mobile"": false}" 22265,8,498,2017-06-04 09:52:37,http://muller.biz/lurline,,147.108.181.194,"{""location"": ""MG"", ""is_mobile"": false}" 22266,8,498,2017-04-07 05:23:31,http://darerath.biz/ivory_raynor,,69.43.203.250,"{""location"": ""LC"", ""is_mobile"": true}" 22267,8,498,2017-05-28 15:54:36,http://lefflerfahey.org/florida_spinka,,221.35.111.129,"{""location"": ""US"", ""is_mobile"": false}" 22268,8,498,2017-05-23 01:46:14,http://johnston.name/julianne,,221.132.16.196,"{""location"": ""PA"", ""is_mobile"": false}" 22269,8,498,2017-03-18 04:20:46,http://rutherfordaltenwerth.co/melvina_zieme,,141.148.210.167,"{""location"": ""BT"", ""is_mobile"": true}" 22270,8,498,2017-05-17 11:36:22,http://heller.co/keven_hyatt,,3.81.14.119,"{""location"": ""BD"", ""is_mobile"": true}" 22271,8,498,2016-12-22 01:11:57,http://mooreklein.info/taurean.herzog,,128.189.159.118,"{""location"": ""GT"", ""is_mobile"": false}" 22272,8,498,2017-04-26 05:29:44,http://thiel.com/jovanny.schumm,,106.52.179.97,"{""location"": ""LV"", ""is_mobile"": false}" 22273,8,498,2017-06-01 19:33:16,http://haag.io/jazmin_ondricka,,50.230.233.98,"{""location"": ""VA"", ""is_mobile"": true}" 22274,8,498,2016-12-21 16:32:59,http://hegmannmertz.info/reid,,215.172.85.51,"{""location"": ""BN"", ""is_mobile"": false}" 1190,1,28,2017-06-04 01:46:14,http://champlin.com/dell.williamson,,176.221.215.144,"{""location"": ""VA"", ""is_mobile"": true}" 1191,1,28,2017-03-02 21:41:16,http://wunschschneider.net/norma.spencer,,91.220.71.182,"{""location"": ""MR"", ""is_mobile"": true}" 1192,1,28,2017-05-18 16:19:16,http://okuneva.biz/marilou,,49.164.92.24,"{""location"": ""NI"", ""is_mobile"": true}" 1193,1,28,2017-02-11 22:15:32,http://toybins.com/dewitt.fisher,,26.38.105.239,"{""location"": ""ST"", ""is_mobile"": true}" 1194,1,28,2017-01-26 02:11:38,http://feil.com/mariane.howe,,64.13.177.132,"{""location"": ""PN"", ""is_mobile"": false}" 1195,1,28,2017-04-03 09:18:17,http://kingreichel.com/alyson.lakin,,224.208.95.87,"{""location"": ""GL"", ""is_mobile"": true}" 1196,1,28,2017-01-08 00:16:15,http://cremin.info/elinor_schmidt,,133.110.140.80,"{""location"": ""MO"", ""is_mobile"": false}" 1197,1,28,2017-01-25 18:15:10,http://torphy.net/emery,,128.75.239.167,"{""location"": ""PN"", ""is_mobile"": false}" 1198,1,28,2016-12-17 05:22:26,http://hintz.name/melia,,150.190.130.215,"{""location"": ""AR"", ""is_mobile"": false}" 1199,1,28,2017-05-24 18:42:42,http://runolfon.name/june.price,,139.203.155.122,"{""location"": ""CK"", ""is_mobile"": true}" 1200,1,28,2017-05-18 13:29:30,http://boehm.com/murray,,124.117.174.170,"{""location"": ""CR"", ""is_mobile"": false}" 1201,1,28,2017-04-04 22:05:28,http://willeichmann.org/barrett_yost,,50.204.14.194,"{""location"": ""ZW"", ""is_mobile"": false}" 1202,1,28,2017-06-13 14:44:39,http://lockman.net/lottie_ratke,,166.49.53.174,"{""location"": ""LS"", ""is_mobile"": true}" 1203,1,28,2017-03-09 22:50:23,http://schamberger.info/aliza_torphy,,55.159.89.17,"{""location"": ""SI"", ""is_mobile"": false}" 1204,1,28,2017-04-11 05:24:28,http://schinnerweber.name/paxton,,219.125.8.28,"{""location"": ""IR"", ""is_mobile"": true}" 1205,1,28,2017-02-23 04:57:07,http://nicolas.co/claudie.padberg,,10.84.61.153,"{""location"": ""AE"", ""is_mobile"": true}" 1206,1,28,2017-02-12 06:41:48,http://wisoky.io/bernardo_cummerata,,143.122.44.85,"{""location"": ""SG"", ""is_mobile"": false}" 1207,1,28,2017-04-24 12:37:34,http://kovacek.co/melia.rice,,147.248.183.26,"{""location"": ""RE"", ""is_mobile"": true}" 1208,1,28,2017-05-08 13:55:45,http://hamill.info/buck.hermann,,57.222.179.220,"{""location"": ""CC"", ""is_mobile"": false}" 1209,1,28,2017-02-02 10:22:12,http://heathcotewisozk.co/jerome_strosin,,124.133.10.145,"{""location"": ""BD"", ""is_mobile"": true}" 1210,1,28,2017-01-26 01:25:17,http://wolfharris.com/austyn.rohan,,49.248.140.206,"{""location"": ""VG"", ""is_mobile"": false}" 1211,1,28,2017-02-12 15:35:58,http://lueilwitz.co/ruby,,150.247.4.216,"{""location"": ""LY"", ""is_mobile"": true}" 1212,1,28,2017-01-05 10:40:34,http://cole.io/orval_schmitt,,176.142.193.117,"{""location"": ""LK"", ""is_mobile"": true}" 1213,1,28,2017-01-18 05:34:14,http://grimesrunolfon.io/nia.larson,,231.47.40.47,"{""location"": ""ZM"", ""is_mobile"": true}" 1214,1,28,2017-02-14 14:37:06,http://christiansensatterfield.org/wilson,,94.79.175.221,"{""location"": ""BO"", ""is_mobile"": true}" 1215,1,28,2017-03-27 12:18:25,http://bogan.co/effie.erdman,,194.23.177.199,"{""location"": ""CU"", ""is_mobile"": false}" 1216,1,28,2017-04-25 04:59:34,http://olsonward.info/brando,,55.124.216.30,"{""location"": ""JM"", ""is_mobile"": false}" 1217,1,28,2016-12-19 14:37:16,http://bartolettilarson.co/emmalee.miller,,142.247.220.32,"{""location"": ""HU"", ""is_mobile"": false}" 1218,1,28,2017-04-17 04:50:26,http://bradtke.info/candido,,236.195.190.135,"{""location"": ""PH"", ""is_mobile"": true}" 1219,1,28,2017-03-06 12:55:55,http://windlerdouglas.net/taryn.lebsack,,254.64.57.168,"{""location"": ""DM"", ""is_mobile"": true}" 2020,1,45,2017-03-29 21:49:25,http://torp.co/uriah,,30.166.27.103,"{""location"": ""LT"", ""is_mobile"": false}" 1220,1,28,2017-01-27 19:12:24,http://weimannokuneva.biz/jermaine,,93.236.134.190,"{""location"": ""FO"", ""is_mobile"": true}" 1221,1,28,2017-02-11 04:59:19,http://littledenesik.name/carmen,,24.247.44.172,"{""location"": ""RO"", ""is_mobile"": false}" 1222,1,28,2017-03-17 18:24:01,http://sporer.info/emmitt,,131.233.240.210,"{""location"": ""NU"", ""is_mobile"": false}" 1223,1,28,2017-05-28 14:33:22,http://millsvandervort.co/carli,,34.241.167.250,"{""location"": ""LV"", ""is_mobile"": true}" 1224,1,28,2017-05-12 07:28:37,http://batz.info/andres_sipes,,172.23.139.169,"{""location"": ""VU"", ""is_mobile"": true}" 1225,1,28,2017-03-22 18:10:21,http://grant.org/jettie_strosin,,191.232.38.9,"{""location"": ""AL"", ""is_mobile"": false}" 1226,1,28,2017-02-13 16:33:53,http://frami.org/marcia_gulgowski,,245.73.125.245,"{""location"": ""AG"", ""is_mobile"": true}" 1227,1,28,2017-02-19 17:59:25,http://sawaynlockman.info/mafalda.cummerata,,228.179.121.23,"{""location"": ""TM"", ""is_mobile"": true}" 1228,1,28,2017-05-07 14:26:02,http://cremin.co/telly.pacocha,,129.58.96.161,"{""location"": ""DZ"", ""is_mobile"": false}" 1229,1,28,2016-12-29 17:23:43,http://schuppebrekke.info/maegan,,53.151.219.245,"{""location"": ""CY"", ""is_mobile"": false}" 1230,1,28,2017-03-16 09:25:48,http://crist.com/carrie,,71.105.238.129,"{""location"": ""LI"", ""is_mobile"": false}" 1231,1,28,2017-01-11 23:54:11,http://schultz.biz/dorcas.wolf,,105.58.166.56,"{""location"": ""LC"", ""is_mobile"": false}" 1232,1,28,2017-04-26 22:47:03,http://krishuels.net/karlie_tremblay,,221.177.179.17,"{""location"": ""NZ"", ""is_mobile"": true}" 1233,1,28,2017-04-08 22:38:30,http://west.org/wanda.fritsch,,197.138.25.179,"{""location"": ""SJ"", ""is_mobile"": false}" 1234,1,28,2017-04-16 17:59:33,http://prohaska.io/imelda_grady,,224.73.235.159,"{""location"": ""HK"", ""is_mobile"": false}" 1235,1,28,2016-12-30 21:23:59,http://hageneswyman.biz/tito.wilkinson,,6.223.202.165,"{""location"": ""DZ"", ""is_mobile"": false}" 1236,1,28,2017-03-27 14:42:24,http://jenkins.name/allie,,233.223.245.18,"{""location"": ""TH"", ""is_mobile"": false}" 1237,1,28,2017-03-05 16:27:54,http://braunbernier.name/brigitte,,71.209.50.134,"{""location"": ""SH"", ""is_mobile"": false}" 1238,1,28,2017-03-19 12:19:30,http://jaskolski.io/ivy,,198.156.71.25,"{""location"": ""ER"", ""is_mobile"": false}" 1239,1,28,2017-06-07 08:33:04,http://renner.io/jaleel,,182.139.11.212,"{""location"": ""LU"", ""is_mobile"": true}" 1240,1,28,2017-05-22 02:55:26,http://bruen.com/garett_abshire,,220.175.156.92,"{""location"": ""BL"", ""is_mobile"": false}" 1241,1,28,2017-02-19 15:18:41,http://smithcrooks.name/merl,,161.235.159.158,"{""location"": ""KY"", ""is_mobile"": true}" 1242,1,28,2017-02-27 17:22:20,http://rutherford.org/jey,,207.125.189.19,"{""location"": ""SH"", ""is_mobile"": false}" 1243,1,28,2017-03-06 15:47:46,http://stracke.io/alfred,,113.183.117.236,"{""location"": ""GG"", ""is_mobile"": false}" 1244,1,28,2016-12-19 01:46:43,http://johnston.io/casper_pfeffer,,156.104.141.214,"{""location"": ""YE"", ""is_mobile"": true}" 22275,8,498,2017-01-25 12:04:30,http://cummerata.org/damaris_heidenreich,,137.140.72.118,"{""location"": ""NO"", ""is_mobile"": false}" 22276,8,498,2017-06-13 03:49:36,http://heel.com/kareem_feeney,,51.128.208.13,"{""location"": ""TF"", ""is_mobile"": true}" 22277,8,498,2016-12-27 02:51:54,http://homenick.co/wallace_barton,,174.86.15.42,"{""location"": ""MP"", ""is_mobile"": false}" 22278,8,499,2016-12-22 19:01:18,http://mraz.co/vicente_beer,,28.22.93.50,"{""location"": ""HU"", ""is_mobile"": false}" 22279,8,499,2017-02-10 04:41:01,http://lubowitz.net/otto,,39.179.28.218,"{""location"": ""GE"", ""is_mobile"": false}" 22280,8,499,2017-03-30 13:16:57,http://predovic.biz/angeline_mcglynn,,107.112.138.132,"{""location"": ""LR"", ""is_mobile"": true}" 22281,8,499,2017-03-21 22:07:09,http://jacobiprosacco.io/je.smith,,173.3.76.5,"{""location"": ""MC"", ""is_mobile"": false}" 22282,8,499,2017-02-08 15:53:44,http://feil.co/doris_brekke,,47.38.55.10,"{""location"": ""IE"", ""is_mobile"": true}" 22283,8,499,2017-01-18 11:56:29,http://howearmstrong.name/dangelo,,72.8.176.141,"{""location"": ""PG"", ""is_mobile"": true}" 22284,8,499,2017-05-16 07:42:40,http://balistrerifriesen.info/marge.mohr,,140.12.147.54,"{""location"": ""AD"", ""is_mobile"": true}" 22285,8,499,2017-05-21 11:26:52,http://turnersatterfield.com/hiram,,25.51.111.82,"{""location"": ""MN"", ""is_mobile"": true}" 22286,8,499,2017-06-13 17:46:36,http://stromanconsidine.name/mariela_stiedemann,,44.21.20.24,"{""location"": ""SR"", ""is_mobile"": true}" 22287,8,499,2017-04-11 12:31:43,http://friesenrowe.name/gay,,230.166.52.253,"{""location"": ""TV"", ""is_mobile"": true}" 22288,8,499,2017-01-22 12:26:32,http://steuber.com/isaiah,,121.114.78.34,"{""location"": ""GH"", ""is_mobile"": false}" 22289,8,499,2017-05-25 05:53:08,http://stamm.org/freda.damore,,28.211.16.120,"{""location"": ""ZA"", ""is_mobile"": true}" 22290,8,499,2017-04-30 00:43:40,http://oberbrunnermayert.io/keaton,,44.174.222.194,"{""location"": ""SA"", ""is_mobile"": false}" 22291,8,499,2017-04-03 12:53:57,http://keler.biz/noe_farrell,,105.135.49.203,"{""location"": ""PE"", ""is_mobile"": false}" 22292,8,499,2017-05-08 20:11:18,http://beerboyle.info/jeff_konopelski,,202.93.42.79,"{""location"": ""SA"", ""is_mobile"": true}" 22293,8,499,2017-05-14 22:10:08,http://beatty.com/althea_schaefer,,150.176.60.78,"{""location"": ""OM"", ""is_mobile"": false}" 22294,8,499,2017-03-12 08:06:51,http://kaulkebotsford.biz/odell.murray,,215.108.199.94,"{""location"": ""PY"", ""is_mobile"": true}" 22295,8,499,2017-01-22 15:31:11,http://kuvalis.org/uriah,,7.244.88.34,"{""location"": ""CG"", ""is_mobile"": true}" 22296,8,499,2017-02-09 08:44:07,http://ankundingjenkins.info/dannie,,61.12.179.59,"{""location"": ""HR"", ""is_mobile"": false}" 22297,8,499,2017-03-18 19:11:19,http://altenwerth.biz/kip,,118.237.87.8,"{""location"": ""EE"", ""is_mobile"": false}" 22298,8,499,2017-05-06 05:42:10,http://goldnermarquardt.co/brice,,49.77.147.238,"{""location"": ""TV"", ""is_mobile"": true}" 22299,8,499,2017-06-06 00:13:13,http://howell.name/violette,,32.90.117.35,"{""location"": ""DK"", ""is_mobile"": true}" 22300,8,499,2017-04-16 06:00:11,http://gibson.name/beatrice,,237.232.88.241,"{""location"": ""UA"", ""is_mobile"": false}" 22301,8,499,2017-03-29 21:51:32,http://hauckjacobs.org/alicia_dicki,,23.133.106.27,"{""location"": ""EH"", ""is_mobile"": true}" 22302,8,499,2017-03-28 01:22:47,http://colewillms.com/armand,,99.129.193.186,"{""location"": ""UY"", ""is_mobile"": true}" 22303,8,500,2017-05-23 11:16:52,http://moriettelind.name/modesto,,221.162.224.141,"{""location"": ""GH"", ""is_mobile"": true}" 22304,8,500,2017-03-24 16:16:58,http://nitzsche.io/cyril_friesen,,156.90.88.188,"{""location"": ""SK"", ""is_mobile"": true}" 22305,8,500,2016-12-29 13:32:09,http://pacocha.biz/meghan.hudson,,35.47.7.85,"{""location"": ""BR"", ""is_mobile"": false}" 22306,8,500,2017-01-01 14:36:51,http://schoen.biz/justina,,138.186.206.213,"{""location"": ""MX"", ""is_mobile"": false}" 22307,8,500,2017-03-06 22:59:45,http://marvin.info/orlando_ferry,,193.4.4.92,"{""location"": ""RS"", ""is_mobile"": true}" 22308,8,500,2017-01-18 09:31:09,http://johnstonheller.name/lourdes_skiles,,232.175.153.129,"{""location"": ""UG"", ""is_mobile"": true}" 22309,8,500,2017-06-10 06:22:37,http://carroll.co/nyah,,109.145.192.101,"{""location"": ""AX"", ""is_mobile"": true}" 22310,8,500,2017-05-25 14:47:57,http://veum.info/margie,,5.221.125.117,"{""location"": ""RW"", ""is_mobile"": true}" 22311,8,500,2016-12-28 06:08:47,http://hartmann.org/brendon,,2.47.51.155,"{""location"": ""ZW"", ""is_mobile"": false}" 22312,8,500,2017-06-08 00:39:20,http://casper.net/adelle,,60.67.87.198,"{""location"": ""BQ"", ""is_mobile"": true}" 22313,8,500,2017-01-10 03:05:40,http://willweinat.biz/skylar_watsica,,182.161.206.247,"{""location"": ""GI"", ""is_mobile"": true}" 22314,8,500,2017-04-13 15:06:05,http://leuschkeko.co/gillian,,46.81.58.21,"{""location"": ""CY"", ""is_mobile"": true}" 22315,8,500,2017-02-02 04:23:36,http://kemmer.name/bradly,,212.101.68.253,"{""location"": ""GG"", ""is_mobile"": true}" 22316,8,500,2016-12-31 14:53:42,http://schaden.name/napoleon_brekke,,156.49.166.31,"{""location"": ""PS"", ""is_mobile"": true}" 22317,8,500,2017-04-07 06:16:50,http://white.name/tamia,,49.68.161.238,"{""location"": ""AR"", ""is_mobile"": true}" 22318,8,500,2017-02-12 01:56:08,http://batz.io/araceli,,26.165.131.163,"{""location"": ""IR"", ""is_mobile"": true}" 22319,8,500,2017-06-05 18:43:47,http://gibson.co/zachariah.donnelly,,108.242.205.194,"{""location"": ""KY"", ""is_mobile"": false}" 22320,8,500,2017-03-06 01:44:51,http://hermann.biz/kenya,,13.150.27.243,"{""location"": ""ZW"", ""is_mobile"": true}" 22321,8,500,2017-05-10 13:30:14,http://stroman.net/florian,,161.53.74.27,"{""location"": ""MN"", ""is_mobile"": false}" 22322,8,500,2017-04-03 04:19:55,http://hilpert.io/kale,,171.28.121.19,"{""location"": ""SO"", ""is_mobile"": true}" 22323,8,500,2017-04-26 19:00:44,http://mclaughlin.com/berniece_rodriguez,,220.124.197.209,"{""location"": ""IR"", ""is_mobile"": true}" 22324,8,500,2017-01-12 07:50:02,http://weber.name/kellie_mckenzie,,217.183.147.241,"{""location"": ""US"", ""is_mobile"": false}" 22325,8,500,2017-02-02 06:46:29,http://schowalterconroy.biz/freddie,,107.34.54.148,"{""location"": ""BQ"", ""is_mobile"": true}" 22326,8,500,2017-01-28 22:29:56,http://gusikowski.co/jarred,,116.69.131.253,"{""location"": ""BQ"", ""is_mobile"": false}" 22327,8,500,2017-05-30 03:41:07,http://adams.io/jairo.rosenbaum,,5.116.111.52,"{""location"": ""IR"", ""is_mobile"": true}" 22328,8,500,2017-01-29 02:49:57,http://lubowitz.co/amber_murray,,149.128.85.53,"{""location"": ""NP"", ""is_mobile"": false}" 22329,8,500,2017-03-07 03:04:33,http://ziemezemlak.org/claudia,,56.93.151.67,"{""location"": ""PG"", ""is_mobile"": false}" 1245,1,28,2017-03-26 00:58:13,http://hudsonoreilly.co/tyshawn,,32.43.141.48,"{""location"": ""CN"", ""is_mobile"": false}" 1246,1,29,2017-01-25 14:08:04,http://ullrich.net/jey.klein,,134.140.169.82,"{""location"": ""WS"", ""is_mobile"": true}" 1247,1,29,2017-04-23 16:06:34,http://schroeder.co/pearline,,72.8.93.22,"{""location"": ""TD"", ""is_mobile"": true}" 1248,1,29,2017-01-18 00:01:52,http://west.name/mauricio,,138.186.247.130,"{""location"": ""BG"", ""is_mobile"": false}" 1249,1,29,2017-03-25 23:07:47,http://mcglynn.co/floyd_schmeler,,251.80.146.45,"{""location"": ""BE"", ""is_mobile"": false}" 1250,1,29,2017-05-19 03:57:44,http://legros.co/iva_mann,,180.115.38.18,"{""location"": ""YT"", ""is_mobile"": true}" 1251,1,29,2017-05-20 22:10:33,http://rippin.co/estrella,,100.133.243.42,"{""location"": ""AG"", ""is_mobile"": false}" 1252,1,29,2017-03-28 19:37:37,http://murazikkautzer.com/christopher,,211.51.69.190,"{""location"": ""MG"", ""is_mobile"": false}" 1253,1,29,2017-06-09 13:49:26,http://maggio.org/leopoldo_auer,,60.224.111.123,"{""location"": ""SX"", ""is_mobile"": true}" 1254,1,29,2017-02-09 04:22:20,http://yost.io/lucy,,77.145.75.184,"{""location"": ""TL"", ""is_mobile"": false}" 1255,1,29,2017-03-10 01:56:02,http://kautzer.org/kiana,,34.85.171.165,"{""location"": ""MG"", ""is_mobile"": false}" 1256,1,29,2017-02-12 09:58:54,http://wunsch.info/treva.toy,,136.178.108.27,"{""location"": ""FI"", ""is_mobile"": false}" 1257,1,29,2017-03-08 01:45:19,http://lockman.info/betsy.rolfson,,138.77.216.163,"{""location"": ""CD"", ""is_mobile"": false}" 1258,1,29,2017-02-16 07:23:11,http://hickle.name/amely,,18.188.161.28,"{""location"": ""GY"", ""is_mobile"": true}" 1259,1,29,2017-01-07 12:59:45,http://nienow.info/jaclyn.ferry,,228.202.191.33,"{""location"": ""SR"", ""is_mobile"": false}" 1260,1,29,2017-02-11 03:21:45,http://schulist.io/ofelia,,106.95.229.5,"{""location"": ""MQ"", ""is_mobile"": false}" 1261,1,29,2017-02-10 16:36:18,http://jerde.info/shaylee_gibson,,167.78.31.61,"{""location"": ""IO"", ""is_mobile"": false}" 1262,1,29,2017-02-12 09:15:42,http://schroedertremblay.info/bradly,,117.207.50.111,"{""location"": ""NO"", ""is_mobile"": false}" 1263,1,29,2017-01-02 14:32:37,http://hermistonrau.io/sadye.barrows,,95.164.109.118,"{""location"": ""MG"", ""is_mobile"": true}" 1264,1,29,2017-03-18 11:07:43,http://kuphalherman.biz/ernestine,,27.99.147.44,"{""location"": ""GY"", ""is_mobile"": true}" 1265,1,29,2017-02-22 17:00:10,http://strosin.io/johnson.stiedemann,,245.38.130.246,"{""location"": ""LR"", ""is_mobile"": true}" 1266,1,29,2017-05-12 22:24:18,http://goodwinhuels.org/merle,,72.234.226.191,"{""location"": ""YT"", ""is_mobile"": false}" 1267,1,29,2017-01-23 21:45:45,http://boganweimann.net/eric,,151.122.12.247,"{""location"": ""FK"", ""is_mobile"": true}" 1268,1,29,2017-03-16 10:37:46,http://boyer.biz/zella.ortiz,,207.170.151.108,"{""location"": ""RO"", ""is_mobile"": false}" 1269,1,29,2017-01-21 13:29:38,http://pollich.com/merritt_kunze,,178.110.155.124,"{""location"": ""RS"", ""is_mobile"": true}" 1270,1,29,2016-12-20 17:13:24,http://hegmann.net/eloisa,,165.140.81.119,"{""location"": ""TR"", ""is_mobile"": false}" 1271,1,29,2017-02-19 12:17:45,http://bayer.io/torrey,,151.13.53.59,"{""location"": ""NC"", ""is_mobile"": false}" 1272,1,29,2017-04-13 12:02:56,http://schneider.org/rubye.strosin,,55.66.181.77,"{""location"": ""NC"", ""is_mobile"": true}" 1273,1,29,2017-01-31 11:43:30,http://oconnell.info/cathryn.mayert,,221.115.95.86,"{""location"": ""GG"", ""is_mobile"": true}" 1274,1,29,2017-01-28 22:05:19,http://cummeratacole.name/cary_johns,,122.186.23.127,"{""location"": ""BS"", ""is_mobile"": true}" 1275,1,29,2017-02-24 20:04:31,http://upton.com/bethany,,186.28.111.145,"{""location"": ""KZ"", ""is_mobile"": false}" 1276,1,29,2017-01-31 13:10:44,http://rowe.name/howell,,148.250.53.232,"{""location"": ""UZ"", ""is_mobile"": true}" 2136,1,47,2016-12-23 16:54:09,http://toy.net/celine,,158.67.223.96,"{""location"": ""MD"", ""is_mobile"": true}" 1277,1,29,2017-03-05 09:49:07,http://mohr.co/maia.brown,,54.43.33.125,"{""location"": ""TF"", ""is_mobile"": true}" 1278,1,29,2017-05-31 15:13:59,http://carterbalistreri.info/dwight,,224.245.95.149,"{""location"": ""PY"", ""is_mobile"": true}" 1279,1,29,2017-03-16 01:08:25,http://gottliebwilliamson.io/mireya,,199.171.229.42,"{""location"": ""TW"", ""is_mobile"": false}" 1280,1,29,2017-06-03 02:14:15,http://kilback.org/aileen,,110.49.51.222,"{""location"": ""GQ"", ""is_mobile"": false}" 1281,1,29,2017-05-13 12:13:31,http://daugherty.org/ed_stracke,,66.181.246.81,"{""location"": ""SL"", ""is_mobile"": true}" 1282,1,29,2016-12-31 18:53:28,http://leffler.co/jason_sporer,,148.102.86.198,"{""location"": ""AL"", ""is_mobile"": true}" 1283,1,29,2016-12-22 11:54:56,http://terry.com/hal.grant,,153.229.202.34,"{""location"": ""DM"", ""is_mobile"": true}" 1284,1,29,2017-01-06 18:20:17,http://hamill.io/ernest,,203.29.226.90,"{""location"": ""AD"", ""is_mobile"": false}" 1285,1,29,2016-12-15 15:20:38,http://ebertstark.net/bonnie.gusikowski,,166.2.85.71,"{""location"": ""LB"", ""is_mobile"": true}" 1286,1,30,2017-02-08 02:23:53,http://gusikowskicruickshank.net/eula,,50.43.62.62,"{""location"": ""ER"", ""is_mobile"": true}" 1287,1,30,2017-03-18 06:29:50,http://beer.biz/otto.leuschke,,190.13.147.126,"{""location"": ""GF"", ""is_mobile"": false}" 1288,1,30,2017-05-28 12:44:01,http://yundtgerlach.net/meta,,154.230.128.46,"{""location"": ""BJ"", ""is_mobile"": true}" 1289,1,30,2017-01-30 05:23:18,http://cummerata.biz/braden_rolfson,,47.247.117.193,"{""location"": ""MP"", ""is_mobile"": true}" 1290,1,30,2017-06-01 07:50:37,http://reilly.co/brenda.becker,,207.250.159.205,"{""location"": ""TK"", ""is_mobile"": false}" 1291,1,30,2017-06-14 00:12:10,http://gerlachthiel.org/roel,,125.246.138.12,"{""location"": ""DJ"", ""is_mobile"": true}" 1292,1,30,2017-04-17 09:04:52,http://kuhlmankonopelski.net/rolando,,109.103.130.121,"{""location"": ""AM"", ""is_mobile"": true}" 1293,1,30,2017-06-07 00:28:11,http://goldner.com/kathleen,,192.99.18.166,"{""location"": ""ES"", ""is_mobile"": true}" 1294,1,30,2016-12-15 10:58:08,http://kuvalis.io/kiara.schuppe,,215.91.116.164,"{""location"": ""TV"", ""is_mobile"": false}" 1295,1,30,2017-04-24 02:21:26,http://ankunding.name/tyreek.weber,,19.37.180.218,"{""location"": ""GN"", ""is_mobile"": false}" 1296,1,30,2017-02-03 02:00:07,http://millsdubuque.org/francesco,,97.163.62.139,"{""location"": ""QA"", ""is_mobile"": true}" 1297,1,30,2017-01-23 00:04:45,http://dickens.biz/tomasa,,153.93.13.248,"{""location"": ""PN"", ""is_mobile"": true}" 1298,1,30,2017-05-24 05:48:42,http://windlerziemann.org/mateo,,213.195.40.94,"{""location"": ""PE"", ""is_mobile"": true}" 1299,1,30,2017-02-11 10:03:37,http://volkmanmuller.io/bailee_schuster,,242.76.28.245,"{""location"": ""MA"", ""is_mobile"": true}" 22330,8,500,2017-05-02 20:45:51,http://wiza.io/destini,,87.56.79.239,"{""location"": ""BB"", ""is_mobile"": true}" 22331,8,500,2017-02-21 07:06:15,http://stantonbashirian.info/aleandro_ruecker,,70.4.225.11,"{""location"": ""CM"", ""is_mobile"": false}" 22332,8,500,2017-01-27 19:50:46,http://green.com/karianne,,125.135.97.48,"{""location"": ""LI"", ""is_mobile"": false}" 22333,8,500,2017-01-06 01:44:00,http://hermanabshire.com/jamey_will,,145.125.14.43,"{""location"": ""GN"", ""is_mobile"": true}" 22334,8,501,2017-05-24 16:40:15,http://greenfelder.io/gerhard,,57.202.16.55,"{""location"": ""GM"", ""is_mobile"": false}" 22335,8,501,2017-04-19 15:24:37,http://heller.net/keagan.christiansen,,68.218.167.58,"{""location"": ""TT"", ""is_mobile"": false}" 22336,8,501,2017-04-04 02:22:39,http://rueckerdenesik.org/eula_dickens,,19.37.222.251,"{""location"": ""KP"", ""is_mobile"": true}" 22337,8,501,2017-03-31 10:49:44,http://kunde.com/ilene,,190.126.214.240,"{""location"": ""NL"", ""is_mobile"": true}" 22338,8,501,2016-12-25 11:28:33,http://mayercollier.net/alexie,,205.167.180.36,"{""location"": ""BQ"", ""is_mobile"": true}" 22339,8,501,2017-01-10 03:59:27,http://bergstrom.co/aleen.leannon,,114.4.171.72,"{""location"": ""SB"", ""is_mobile"": true}" 22340,8,501,2017-01-10 15:50:34,http://walsh.org/wilson.king,,217.129.209.185,"{""location"": ""CN"", ""is_mobile"": true}" 22341,8,501,2017-01-23 22:40:20,http://bauchrohan.biz/delmer,,83.188.129.224,"{""location"": ""IR"", ""is_mobile"": true}" 22342,8,501,2017-02-06 01:52:07,http://marks.org/shyanne.metz,,38.84.121.152,"{""location"": ""MY"", ""is_mobile"": true}" 22343,8,501,2017-04-07 00:33:36,http://witting.biz/valentine_mclaughlin,,243.87.227.177,"{""location"": ""CC"", ""is_mobile"": true}" 22344,8,501,2017-01-05 16:34:31,http://shields.net/mayra.huel,,16.49.212.67,"{""location"": ""US"", ""is_mobile"": true}" 22345,8,501,2017-05-18 15:00:50,http://kuhn.com/larry_jerde,,188.23.253.18,"{""location"": ""MT"", ""is_mobile"": true}" 22346,8,501,2017-03-16 04:22:36,http://stoltenbergdach.info/jeyca,,30.35.124.38,"{""location"": ""CI"", ""is_mobile"": true}" 22347,8,501,2017-05-05 23:34:24,http://corkery.biz/jermey,,195.96.13.44,"{""location"": ""KW"", ""is_mobile"": false}" 22348,8,501,2017-04-15 07:39:52,http://paucek.org/treva.spencer,,26.54.145.98,"{""location"": ""IO"", ""is_mobile"": true}" 22349,8,501,2017-01-05 06:20:41,http://beier.info/price_nikolaus,,85.86.158.15,"{""location"": ""PL"", ""is_mobile"": true}" 22350,8,501,2017-03-31 06:19:41,http://shields.name/nichole_crist,,212.26.149.191,"{""location"": ""LY"", ""is_mobile"": false}" 22351,8,501,2017-03-10 09:07:44,http://stehr.io/hilario,,109.62.56.22,"{""location"": ""AX"", ""is_mobile"": true}" 22352,8,501,2017-02-07 16:00:40,http://blockrempel.net/judson.reichel,,40.206.204.51,"{""location"": ""SN"", ""is_mobile"": false}" 22353,8,501,2017-04-26 17:43:19,http://kilback.name/percy,,181.243.83.174,"{""location"": ""SB"", ""is_mobile"": false}" 22354,8,501,2017-01-17 18:45:58,http://goodwin.io/hilton_thompson,,212.208.106.83,"{""location"": ""PY"", ""is_mobile"": true}" 22355,8,501,2017-04-05 04:55:03,http://gottlieb.io/susie.adams,,112.202.105.251,"{""location"": ""PT"", ""is_mobile"": true}" 22356,8,501,2016-12-18 05:08:13,http://abernathy.io/rosetta,,158.103.42.87,"{""location"": ""GI"", ""is_mobile"": false}" 22357,8,501,2017-03-11 08:54:15,http://creminzboncak.org/rafael,,170.110.203.31,"{""location"": ""EC"", ""is_mobile"": false}" 22358,8,501,2017-02-28 23:54:34,http://mcdermott.com/tomas.weinat,,111.3.235.28,"{""location"": ""VE"", ""is_mobile"": false}" 22359,8,501,2017-04-02 12:45:52,http://prohaska.com/lazaro,,222.45.38.98,"{""location"": ""SR"", ""is_mobile"": true}" 22360,8,501,2017-02-21 23:37:22,http://klocko.biz/sonny,,89.61.36.12,"{""location"": ""PH"", ""is_mobile"": false}" 22361,8,501,2017-05-14 00:23:12,http://romaguerathompson.com/aunta.sawayn,,91.93.4.127,"{""location"": ""DZ"", ""is_mobile"": true}" 22362,8,501,2017-05-27 02:57:46,http://walter.name/erna_flatley,,102.240.83.106,"{""location"": ""SH"", ""is_mobile"": false}" 22363,8,501,2016-12-16 02:58:26,http://deckowarmstrong.io/kelvin,,188.239.36.198,"{""location"": ""PH"", ""is_mobile"": true}" 22364,8,501,2017-05-05 14:04:37,http://corkery.co/caidy,,29.157.164.227,"{""location"": ""VN"", ""is_mobile"": true}" 22365,8,501,2017-01-18 06:07:38,http://vandervort.name/johnnie,,173.106.121.103,"{""location"": ""TG"", ""is_mobile"": false}" 22366,8,501,2017-01-05 20:47:23,http://watsicapacocha.net/hailee.reilly,,205.127.51.193,"{""location"": ""SE"", ""is_mobile"": false}" 22367,8,501,2016-12-13 11:06:50,http://dickimcdermott.info/gabe_kaulke,,60.216.116.216,"{""location"": ""DK"", ""is_mobile"": false}" 22368,8,501,2017-05-31 23:38:34,http://goodwin.com/napoleon,,209.148.231.195,"{""location"": ""CR"", ""is_mobile"": false}" 22369,8,501,2016-12-25 11:58:28,http://ruecker.io/anastacio_altenwerth,,14.40.145.114,"{""location"": ""MS"", ""is_mobile"": false}" 22370,8,501,2017-01-11 09:17:02,http://okuneva.io/guillermo_haley,,185.96.108.33,"{""location"": ""KH"", ""is_mobile"": false}" 22371,8,501,2017-02-04 11:06:08,http://barrows.co/brayan_fay,,8.79.145.110,"{""location"": ""TM"", ""is_mobile"": true}" 22372,8,501,2017-04-16 19:34:45,http://waelchi.co/edmond_donnelly,,101.22.159.74,"{""location"": ""BY"", ""is_mobile"": true}" 22373,8,501,2017-05-07 00:53:51,http://rowe.info/quincy_kulas,,112.24.21.173,"{""location"": ""DE"", ""is_mobile"": true}" 22374,8,501,2017-05-21 03:11:39,http://beier.net/myrtis.stark,,219.96.81.51,"{""location"": ""FI"", ""is_mobile"": false}" 22375,8,501,2017-02-24 22:10:21,http://huels.io/olga,,22.94.165.240,"{""location"": ""PG"", ""is_mobile"": false}" 22376,8,501,2017-04-23 23:02:06,http://schimmel.net/skyla_altenwerth,,153.136.231.87,"{""location"": ""SO"", ""is_mobile"": false}" 22377,8,501,2016-12-24 23:37:37,http://oberbrunner.io/maritza.eichmann,,82.139.149.64,"{""location"": ""AF"", ""is_mobile"": true}" 22378,8,501,2017-02-08 18:29:07,http://cummerataglover.name/alvah.hintz,,228.46.142.103,"{""location"": ""TZ"", ""is_mobile"": false}" 22379,8,501,2017-02-17 00:52:23,http://macgyverhoeger.net/jordy_paucek,,92.139.51.191,"{""location"": ""JO"", ""is_mobile"": false}" 22380,8,501,2017-01-18 13:56:39,http://considinegraham.org/aleandra.armstrong,,66.100.91.108,"{""location"": ""TV"", ""is_mobile"": true}" 22381,8,501,2017-04-24 01:44:44,http://boylepollich.com/daphney_kuhic,,40.4.67.236,"{""location"": ""AE"", ""is_mobile"": false}" 22382,8,502,2016-12-25 21:21:12,http://reichert.info/ronny,,253.124.40.119,"{""location"": ""MM"", ""is_mobile"": false}" 22383,8,502,2017-01-29 00:15:47,http://roberts.io/lyric,,69.244.238.38,"{""location"": ""DK"", ""is_mobile"": true}" 22384,8,502,2017-06-13 16:12:12,http://cruickshank.name/otho,,250.224.130.84,"{""location"": ""MM"", ""is_mobile"": true}" 1300,1,30,2017-03-14 12:07:44,http://bogan.org/gilberto.okeefe,,59.198.69.140,"{""location"": ""KN"", ""is_mobile"": true}" 1301,1,30,2017-02-24 14:19:48,http://cole.name/isidro_bauch,,101.109.62.59,"{""location"": ""PE"", ""is_mobile"": true}" 1302,1,30,2017-03-07 02:10:29,http://schowalterkoch.biz/mandy_hayes,,83.172.190.76,"{""location"": ""CO"", ""is_mobile"": true}" 1303,1,30,2017-01-26 12:29:20,http://romaguera.name/fabiola,,111.36.12.238,"{""location"": ""SE"", ""is_mobile"": true}" 1304,1,30,2017-03-12 12:50:22,http://carter.net/kelley.spencer,,3.93.178.87,"{""location"": ""AF"", ""is_mobile"": true}" 1305,1,30,2017-03-23 20:52:22,http://fahey.com/dan_hermann,,162.24.201.77,"{""location"": ""BG"", ""is_mobile"": true}" 1306,1,30,2017-02-15 20:44:17,http://goldnerhand.io/rachael,,68.219.202.82,"{""location"": ""NO"", ""is_mobile"": true}" 1307,1,30,2017-05-20 17:58:43,http://mckenzie.org/laurianne,,68.17.173.171,"{""location"": ""US"", ""is_mobile"": false}" 1308,1,30,2017-05-29 10:57:33,http://conroy.info/katelin,,201.71.208.114,"{""location"": ""MR"", ""is_mobile"": true}" 1309,1,30,2017-02-07 08:30:27,http://ernser.io/marques,,150.58.64.180,"{""location"": ""MS"", ""is_mobile"": true}" 1310,1,30,2016-12-21 05:47:10,http://faheyfadel.io/kip,,25.147.67.91,"{""location"": ""SS"", ""is_mobile"": true}" 1311,1,30,2017-02-06 03:15:24,http://corkery.info/geovany,,168.194.154.31,"{""location"": ""TV"", ""is_mobile"": false}" 1312,1,30,2017-05-27 10:49:39,http://collier.co/gardner_skiles,,112.232.179.65,"{""location"": ""LV"", ""is_mobile"": false}" 1313,1,30,2017-04-02 03:11:08,http://mckenzie.com/liana_hauck,,71.212.189.254,"{""location"": ""UG"", ""is_mobile"": true}" 1314,1,30,2017-02-23 14:25:44,http://barton.co/robb,,92.221.170.184,"{""location"": ""UY"", ""is_mobile"": false}" 1315,1,30,2017-01-12 16:38:40,http://swaniawskidouglas.io/annamae,,230.236.77.86,"{""location"": ""BW"", ""is_mobile"": false}" 1316,1,30,2017-05-05 07:07:44,http://marquardt.io/mary,,93.10.94.76,"{""location"": ""IR"", ""is_mobile"": true}" 1317,1,30,2017-05-03 23:37:29,http://vonrueden.biz/caandra_ferry,,129.232.240.171,"{""location"": ""TH"", ""is_mobile"": true}" 1318,1,30,2017-02-23 23:12:34,http://huels.biz/maximillian,,199.239.81.207,"{""location"": ""BN"", ""is_mobile"": true}" 1319,1,30,2017-06-02 07:55:56,http://wunsch.name/sophie,,151.216.94.209,"{""location"": ""BN"", ""is_mobile"": false}" 1320,1,30,2017-04-04 01:00:55,http://pfannerstillebert.biz/ted,,175.215.206.114,"{""location"": ""MC"", ""is_mobile"": true}" 1321,1,30,2017-05-16 03:41:40,http://harrisaltenwerth.name/kristopher,,26.13.246.244,"{""location"": ""GW"", ""is_mobile"": true}" 1322,1,30,2017-06-06 02:05:52,http://bartell.info/erna.powlowski,,22.228.134.116,"{""location"": ""LY"", ""is_mobile"": false}" 1323,1,30,2017-01-20 14:07:53,http://corkerykeler.io/aliza.erdman,,22.126.169.61,"{""location"": ""CK"", ""is_mobile"": true}" 1324,1,30,2017-05-09 07:06:14,http://kirlin.co/rigoberto,,59.83.44.91,"{""location"": ""FO"", ""is_mobile"": false}" 1325,1,30,2017-03-16 00:42:12,http://gutkowski.name/rubye,,86.169.136.43,"{""location"": ""IR"", ""is_mobile"": false}" 1326,1,30,2017-02-09 04:40:25,http://leschstrosin.org/ethelyn,,178.4.191.27,"{""location"": ""US"", ""is_mobile"": false}" 1327,1,30,2017-06-08 22:32:12,http://lebsack.co/tia,,155.236.15.168,"{""location"": ""BZ"", ""is_mobile"": true}" 1328,1,31,2017-03-10 20:57:07,http://boyle.info/rigoberto_heel,,83.7.164.104,"{""location"": ""MG"", ""is_mobile"": true}" 1329,1,31,2017-05-16 00:35:24,http://boyer.name/rita.kunde,,66.114.82.210,"{""location"": ""SE"", ""is_mobile"": false}" 1330,1,31,2017-03-08 17:56:01,http://satterfieldkuhic.io/mack_adams,,130.61.20.55,"{""location"": ""BF"", ""is_mobile"": true}" 1331,1,31,2017-01-26 11:46:35,http://oconnell.biz/sherman,,155.159.129.149,"{""location"": ""LB"", ""is_mobile"": true}" 1332,1,31,2017-01-10 20:21:20,http://roberts.info/guido_gorczany,,227.86.15.230,"{""location"": ""FJ"", ""is_mobile"": false}" 1333,1,31,2017-03-22 00:41:46,http://tillman.biz/lucile_bode,,179.127.165.179,"{""location"": ""IM"", ""is_mobile"": false}" 1334,1,31,2017-01-15 17:01:48,http://zboncak.biz/heaven.bergnaum,,140.145.4.121,"{""location"": ""AD"", ""is_mobile"": true}" 1335,1,31,2016-12-24 04:30:44,http://walterkuvalis.co/ayden_rice,,92.218.87.31,"{""location"": ""MS"", ""is_mobile"": false}" 1336,1,31,2017-02-17 07:44:30,http://kuvalis.com/erwin,,173.183.191.184,"{""location"": ""VA"", ""is_mobile"": false}" 1337,1,31,2017-04-17 16:52:37,http://vonruedenlehner.biz/keely,,43.181.36.245,"{""location"": ""BW"", ""is_mobile"": true}" 1338,1,31,2016-12-22 20:22:13,http://greenholt.name/maudie.borer,,27.140.46.68,"{""location"": ""VU"", ""is_mobile"": true}" 1339,1,31,2017-05-24 03:01:01,http://metz.io/michele,,110.114.184.251,"{""location"": ""ER"", ""is_mobile"": false}" 1340,1,31,2017-05-02 12:05:25,http://bogisich.io/audra,,166.104.70.24,"{""location"": ""GB"", ""is_mobile"": true}" 1341,1,31,2016-12-26 14:38:24,http://schumm.com/favian_larson,,33.26.244.205,"{""location"": ""YE"", ""is_mobile"": true}" 1342,1,31,2017-04-21 15:01:27,http://dibbert.co/isai,,206.47.229.124,"{""location"": ""HR"", ""is_mobile"": true}" 1343,1,31,2017-03-01 08:21:35,http://ward.com/carter,,49.134.70.222,"{""location"": ""TH"", ""is_mobile"": false}" 1344,1,31,2017-04-17 09:12:47,http://okon.net/alden_thompson,,175.198.246.114,"{""location"": ""IL"", ""is_mobile"": false}" 1345,1,31,2016-12-29 12:27:49,http://bahringer.io/bridie.wilderman,,178.172.207.186,"{""location"": ""BJ"", ""is_mobile"": true}" 1346,1,31,2017-01-07 15:46:42,http://krajcik.info/elinore_trantow,,184.247.82.85,"{""location"": ""TT"", ""is_mobile"": false}" 1347,1,31,2017-03-23 22:06:27,http://hammes.co/owen,,240.160.128.69,"{""location"": ""SZ"", ""is_mobile"": false}" 1348,1,31,2017-02-08 18:59:23,http://johnston.name/saul_sipes,,100.136.151.101,"{""location"": ""LK"", ""is_mobile"": true}" 1349,1,31,2017-06-02 17:02:35,http://kaulkemayer.net/carmelo_casper,,39.83.56.45,"{""location"": ""GG"", ""is_mobile"": true}" 1350,1,31,2017-03-15 11:44:12,http://kovacek.biz/precious.davis,,162.2.73.90,"{""location"": ""SZ"", ""is_mobile"": true}" 1351,1,31,2017-03-13 20:47:11,http://homenickfarrell.org/isac,,38.161.92.191,"{""location"": ""CV"", ""is_mobile"": false}" 1352,1,31,2017-02-17 12:11:09,http://schillermills.biz/melisa,,194.239.129.140,"{""location"": ""BO"", ""is_mobile"": false}" 1353,1,31,2017-04-25 10:09:47,http://welch.com/karley,,120.202.239.96,"{""location"": ""IS"", ""is_mobile"": true}" 1354,1,31,2017-06-04 23:59:08,http://gerholdschmitt.com/maxime.lowe,,71.52.145.60,"{""location"": ""ER"", ""is_mobile"": true}" 1355,1,31,2017-02-22 11:20:28,http://hansenlowe.info/glen,,128.28.22.198,"{""location"": ""OM"", ""is_mobile"": false}" 1356,1,31,2017-02-22 22:44:36,http://strosindietrich.biz/catalina,,162.93.159.166,"{""location"": ""SZ"", ""is_mobile"": true}" 22385,8,502,2017-02-21 19:23:20,http://luettgencorwin.org/pete,,216.31.51.190,"{""location"": ""BD"", ""is_mobile"": false}" 22386,8,502,2017-03-22 05:35:48,http://johnston.org/anderson,,153.82.132.22,"{""location"": ""CI"", ""is_mobile"": false}" 22387,8,502,2017-01-23 05:37:27,http://kemmer.name/malachi.dicki,,201.139.12.224,"{""location"": ""NE"", ""is_mobile"": true}" 22388,8,502,2017-02-17 17:22:40,http://connelly.info/meagan.reichert,,145.232.233.79,"{""location"": ""GM"", ""is_mobile"": false}" 22389,8,502,2017-03-09 15:00:28,http://feil.co/delta_denesik,,115.11.28.82,"{""location"": ""GG"", ""is_mobile"": false}" 22390,8,502,2017-06-12 02:31:32,http://stracke.net/montana_senger,,239.228.248.21,"{""location"": ""BJ"", ""is_mobile"": false}" 22391,8,502,2017-02-02 15:05:47,http://bins.info/laurie,,33.115.171.236,"{""location"": ""AM"", ""is_mobile"": true}" 22392,8,502,2017-05-29 00:57:54,http://swift.name/jalon,,29.141.133.193,"{""location"": ""MG"", ""is_mobile"": true}" 22393,8,502,2017-06-03 17:45:59,http://cummings.io/emil.gleichner,,197.62.201.212,"{""location"": ""CO"", ""is_mobile"": true}" 22394,8,502,2017-03-18 16:10:32,http://simoniscorwin.biz/bette,,84.219.198.239,"{""location"": ""EE"", ""is_mobile"": false}" 22395,8,502,2017-04-14 08:39:10,http://goldner.org/alphonso,,57.92.235.109,"{""location"": ""TJ"", ""is_mobile"": true}" 22396,8,502,2017-02-15 18:07:00,http://kling.io/arnoldo,,174.66.134.25,"{""location"": ""KN"", ""is_mobile"": true}" 22397,8,502,2016-12-23 02:24:05,http://croninbergstrom.io/ashleigh_funk,,150.203.234.232,"{""location"": ""BB"", ""is_mobile"": true}" 22398,8,502,2017-03-02 13:59:25,http://cummeratawunsch.biz/christelle,,220.131.205.251,"{""location"": ""JE"", ""is_mobile"": true}" 22399,8,502,2017-03-16 19:55:16,http://howe.name/osborne,,48.235.28.19,"{""location"": ""CU"", ""is_mobile"": false}" 22400,8,502,2017-05-11 06:01:50,http://lesch.io/marlin,,172.86.5.133,"{""location"": ""MF"", ""is_mobile"": false}" 22401,8,502,2017-03-10 10:03:03,http://zemlakjast.org/tiffany_heidenreich,,183.64.51.38,"{""location"": ""WF"", ""is_mobile"": true}" 22402,8,502,2017-03-31 13:29:47,http://okon.biz/lemuel,,87.105.71.27,"{""location"": ""SH"", ""is_mobile"": true}" 22403,8,502,2017-03-24 08:21:11,http://schumm.name/vivianne.zulauf,,252.125.220.43,"{""location"": ""HK"", ""is_mobile"": false}" 22404,8,502,2017-04-02 03:04:17,http://fadeldamore.io/matt,,8.222.6.140,"{""location"": ""AE"", ""is_mobile"": true}" 22405,8,502,2017-01-13 18:56:33,http://kemmer.org/maegan,,41.48.73.119,"{""location"": ""BF"", ""is_mobile"": false}" 22406,8,502,2017-02-07 15:32:36,http://legros.info/merritt,,125.222.106.79,"{""location"": ""MS"", ""is_mobile"": true}" 22407,8,503,2016-12-20 21:15:04,http://marquardt.biz/reggie,,82.89.118.227,"{""location"": ""LC"", ""is_mobile"": true}" 22408,8,503,2017-04-21 19:51:38,http://mitchell.io/darlene.stark,,25.21.90.244,"{""location"": ""AO"", ""is_mobile"": false}" 22409,8,503,2017-04-21 10:28:10,http://gorczany.co/jaylan,,87.115.108.106,"{""location"": ""SL"", ""is_mobile"": true}" 22410,8,503,2016-12-18 13:51:21,http://quitzonschaden.org/avery,,206.11.123.123,"{""location"": ""MP"", ""is_mobile"": false}" 22411,8,503,2017-05-04 01:52:08,http://nolan.net/kellie.effertz,,150.35.92.59,"{""location"": ""PR"", ""is_mobile"": true}" 22412,8,503,2017-01-15 08:26:42,http://lemkecrist.biz/austin,,104.88.240.60,"{""location"": ""BB"", ""is_mobile"": false}" 22413,8,503,2016-12-21 20:22:24,http://gradygreenfelder.biz/hulda,,150.79.29.147,"{""location"": ""PG"", ""is_mobile"": false}" 22414,8,503,2017-03-31 12:51:11,http://windlerrunolfon.com/loraine_rice,,246.160.80.126,"{""location"": ""AO"", ""is_mobile"": true}" 22415,8,503,2017-03-17 04:03:57,http://jaskolski.biz/lily,,201.175.247.60,"{""location"": ""SO"", ""is_mobile"": false}" 22416,8,503,2016-12-15 11:35:37,http://bayer.net/camila,,88.70.251.62,"{""location"": ""PK"", ""is_mobile"": false}" 22417,8,503,2016-12-16 20:49:28,http://bradtke.org/gunnar_heller,,40.2.240.157,"{""location"": ""VC"", ""is_mobile"": false}" 22418,8,503,2017-01-22 05:38:25,http://streich.co/adah,,203.190.37.161,"{""location"": ""BY"", ""is_mobile"": false}" 22419,8,503,2017-02-14 01:14:04,http://greenwitting.biz/amie,,218.51.12.97,"{""location"": ""GR"", ""is_mobile"": true}" 22420,8,503,2017-01-30 12:37:53,http://langosh.name/estel,,239.50.212.76,"{""location"": ""QA"", ""is_mobile"": false}" 22421,8,503,2016-12-14 07:15:28,http://volkman.info/bo.macejkovic,,39.240.149.70,"{""location"": ""SL"", ""is_mobile"": false}" 22422,8,503,2017-04-27 11:23:04,http://muller.com/brooke,,175.191.73.243,"{""location"": ""IN"", ""is_mobile"": true}" 22423,8,503,2017-01-04 23:10:06,http://becker.biz/treie_ruecker,,156.133.33.151,"{""location"": ""BN"", ""is_mobile"": false}" 22424,8,503,2016-12-27 16:16:10,http://howellgottlieb.net/jarvis,,124.178.3.191,"{""location"": ""MW"", ""is_mobile"": true}" 22425,8,503,2017-03-23 06:38:33,http://hahn.org/estell_mcdermott,,65.143.16.248,"{""location"": ""VI"", ""is_mobile"": true}" 22426,8,503,2017-05-07 20:41:47,http://corwinhansen.co/tremayne.halvorson,,180.95.110.122,"{""location"": ""BJ"", ""is_mobile"": true}" 22427,8,503,2017-05-13 07:23:14,http://tremblay.co/betsy,,250.132.214.191,"{""location"": ""AD"", ""is_mobile"": true}" 22428,8,503,2017-06-03 21:19:17,http://cronin.name/aryanna_gerlach,,218.3.166.102,"{""location"": ""CY"", ""is_mobile"": true}" 22429,8,503,2017-03-30 21:59:47,http://cain.biz/karl_stroman,,26.127.228.35,"{""location"": ""RE"", ""is_mobile"": true}" 22430,8,503,2017-01-15 14:25:33,http://cole.org/reid_dietrich,,14.253.125.60,"{""location"": ""IS"", ""is_mobile"": false}" 22431,8,503,2017-04-22 08:34:16,http://hermiston.com/bradley_hermiston,,128.105.29.122,"{""location"": ""LS"", ""is_mobile"": true}" 22432,8,503,2017-03-16 16:43:09,http://borerkertzmann.org/patsy,,212.55.202.24,"{""location"": ""PW"", ""is_mobile"": false}" 22433,8,503,2017-02-03 01:52:20,http://leffler.co/jannie,,53.139.220.13,"{""location"": ""NR"", ""is_mobile"": false}" 22434,8,503,2017-04-13 01:33:57,http://weimann.org/weston.heidenreich,,192.59.89.254,"{""location"": ""PK"", ""is_mobile"": false}" 22435,8,503,2017-05-22 17:35:46,http://lindgrengutkowski.org/geo.frami,,25.150.90.157,"{""location"": ""TV"", ""is_mobile"": true}" 22436,8,503,2017-04-28 13:08:14,http://satterfield.io/aliza_walker,,105.83.102.199,"{""location"": ""AR"", ""is_mobile"": false}" 22437,8,503,2017-05-22 15:09:50,http://prosaccobauch.co/kimberly_fahey,,210.35.39.189,"{""location"": ""KI"", ""is_mobile"": false}" 22438,8,503,2017-05-29 12:24:15,http://huels.biz/greta,,246.199.47.185,"{""location"": ""LT"", ""is_mobile"": true}" 22439,8,503,2017-05-25 00:09:00,http://padberg.co/rebeka,,239.158.39.115,"{""location"": ""LK"", ""is_mobile"": true}" 22440,8,503,2017-01-06 15:56:40,http://bahringer.com/winifred,,251.248.245.191,"{""location"": ""FM"", ""is_mobile"": false}" 1357,1,31,2017-04-20 01:05:31,http://cremin.biz/ewell.price,,206.23.43.118,"{""location"": ""BS"", ""is_mobile"": true}" 1358,1,31,2017-04-24 14:41:23,http://jenkinskunde.org/hardy.batz,,211.96.214.86,"{""location"": ""WF"", ""is_mobile"": false}" 1359,1,31,2017-02-01 00:55:27,http://quitzonschaden.org/godfrey_block,,130.29.22.242,"{""location"": ""PY"", ""is_mobile"": false}" 1360,1,31,2017-01-01 20:58:45,http://harber.name/kamron_spinka,,194.98.183.247,"{""location"": ""BV"", ""is_mobile"": false}" 1361,1,31,2017-06-04 07:37:53,http://deckow.co/amely_lindgren,,79.227.136.48,"{""location"": ""ER"", ""is_mobile"": true}" 1362,1,31,2017-06-05 05:33:14,http://goyette.info/kaitlin.brekke,,169.121.11.85,"{""location"": ""FO"", ""is_mobile"": false}" 1363,1,31,2017-03-17 19:47:11,http://pagac.com/nona,,102.141.231.15,"{""location"": ""TL"", ""is_mobile"": true}" 1364,1,31,2016-12-24 00:40:44,http://wolffkovacek.com/judah.brakus,,235.219.7.200,"{""location"": ""DM"", ""is_mobile"": true}" 1365,1,31,2017-03-21 06:00:31,http://jerde.co/johnnie_moriette,,65.172.58.170,"{""location"": ""NE"", ""is_mobile"": true}" 1366,1,31,2017-02-01 22:01:03,http://bednar.net/marianna_marvin,,52.21.13.48,"{""location"": ""TZ"", ""is_mobile"": true}" 1367,1,31,2017-01-18 02:14:46,http://ohara.org/veronica,,64.32.247.171,"{""location"": ""KG"", ""is_mobile"": false}" 1368,1,31,2017-03-20 03:52:23,http://auer.co/earl,,171.197.88.39,"{""location"": ""ES"", ""is_mobile"": false}" 1369,1,31,2016-12-29 13:25:25,http://simonis.com/immanuel.hirthe,,14.174.196.47,"{""location"": ""BE"", ""is_mobile"": true}" 1370,1,31,2017-03-01 07:55:38,http://okeefe.co/tara.schinner,,212.177.203.49,"{""location"": ""SA"", ""is_mobile"": true}" 1371,1,31,2016-12-26 04:44:47,http://langbailey.biz/chelsey_macgyver,,17.159.83.239,"{""location"": ""CF"", ""is_mobile"": false}" 1372,1,31,2017-01-07 08:24:11,http://osinskispencer.org/dulce,,134.17.250.131,"{""location"": ""AX"", ""is_mobile"": false}" 1373,1,31,2017-05-29 03:24:37,http://hermann.biz/keshawn_graham,,136.125.159.147,"{""location"": ""BQ"", ""is_mobile"": true}" 1374,1,31,2017-02-16 01:35:30,http://bergstrom.name/jerrold_cole,,105.186.28.200,"{""location"": ""BH"", ""is_mobile"": true}" 1375,1,31,2017-01-22 01:51:48,http://windlerromaguera.io/lambert.jones,,168.60.230.132,"{""location"": ""GF"", ""is_mobile"": false}" 1376,1,31,2017-03-18 13:36:21,http://shanahan.biz/leonora_bosco,,165.177.179.169,"{""location"": ""IL"", ""is_mobile"": true}" 1377,1,32,2017-02-03 10:35:24,http://bradtke.io/quincy,,134.236.50.140,"{""location"": ""PT"", ""is_mobile"": true}" 1378,1,32,2017-04-05 03:03:29,http://auer.name/bulah,,118.174.96.212,"{""location"": ""MF"", ""is_mobile"": false}" 1379,1,32,2017-05-30 19:33:50,http://friesen.biz/daphne,,28.53.104.225,"{""location"": ""HK"", ""is_mobile"": true}" 1380,1,32,2017-02-23 16:18:42,http://dibbert.name/heloise_breitenberg,,228.195.75.204,"{""location"": ""CX"", ""is_mobile"": false}" 1381,1,32,2017-04-03 15:13:09,http://ryanklocko.io/augustus.beier,,145.113.64.126,"{""location"": ""NL"", ""is_mobile"": false}" 1382,1,32,2017-02-04 10:41:57,http://schadenmedhurst.net/frederique,,144.152.69.220,"{""location"": ""LU"", ""is_mobile"": true}" 1383,1,32,2017-04-08 00:13:12,http://rau.org/damaris_gleichner,,126.141.48.90,"{""location"": ""GD"", ""is_mobile"": true}" 1384,1,32,2017-05-18 03:53:51,http://lynch.biz/ottilie.jakubowski,,53.138.205.6,"{""location"": ""BL"", ""is_mobile"": true}" 1385,1,32,2017-02-17 09:38:32,http://pauceksenger.biz/cody,,141.227.254.7,"{""location"": ""PA"", ""is_mobile"": false}" 1386,1,32,2017-04-08 10:41:37,http://johnsonosinski.net/benedict.padberg,,233.142.95.125,"{""location"": ""BL"", ""is_mobile"": true}" 1387,1,32,2017-05-08 14:13:28,http://schoen.biz/katelynn_jast,,199.80.210.50,"{""location"": ""CK"", ""is_mobile"": false}" 1388,1,32,2016-12-19 01:07:46,http://torphy.net/camila,,107.124.94.75,"{""location"": ""SS"", ""is_mobile"": true}" 1389,1,32,2017-05-12 04:34:12,http://greenfelder.io/maegan_schmitt,,173.30.182.126,"{""location"": ""TR"", ""is_mobile"": true}" 1390,1,32,2017-02-25 11:50:13,http://kaulke.net/ezequiel,,181.23.74.127,"{""location"": ""SM"", ""is_mobile"": true}" 1391,1,32,2017-06-11 15:52:25,http://brown.name/verlie_kunze,,114.49.140.83,"{""location"": ""DZ"", ""is_mobile"": true}" 1392,1,32,2017-05-20 10:26:13,http://kovacekkeler.co/elvie_oconnell,,248.14.251.188,"{""location"": ""UG"", ""is_mobile"": true}" 1393,1,32,2017-03-07 00:01:46,http://mertzmurray.io/alexandria_beer,,147.86.154.248,"{""location"": ""GI"", ""is_mobile"": true}" 1394,1,32,2017-04-11 07:23:12,http://langoshhills.co/idella,,113.58.165.151,"{""location"": ""NA"", ""is_mobile"": false}" 1395,1,32,2017-02-03 02:34:25,http://sanford.io/presley,,11.200.134.64,"{""location"": ""SK"", ""is_mobile"": true}" 1396,1,32,2017-02-22 09:33:50,http://streich.io/samir,,229.250.195.180,"{""location"": ""TF"", ""is_mobile"": false}" 1397,1,32,2017-03-09 22:05:01,http://bauch.org/roselyn.maggio,,102.58.57.204,"{""location"": ""SH"", ""is_mobile"": true}" 1398,1,32,2017-02-13 03:54:20,http://franecki.io/kendrick_schneider,,63.41.90.139,"{""location"": ""PT"", ""is_mobile"": false}" 1399,1,33,2017-03-17 08:03:25,http://weberrunolfon.info/cordelia,,38.158.171.248,"{""location"": ""SJ"", ""is_mobile"": false}" 1400,1,33,2016-12-26 11:36:46,http://cummerata.com/aimee.smith,,167.57.67.145,"{""location"": ""CM"", ""is_mobile"": false}" 1401,1,33,2017-03-23 09:42:14,http://wuckert.com/tania,,214.230.233.180,"{""location"": ""CD"", ""is_mobile"": true}" 1402,1,33,2017-04-25 17:13:55,http://dubuque.biz/kayleigh.bergnaum,,186.185.73.29,"{""location"": ""BI"", ""is_mobile"": true}" 1403,1,33,2017-03-08 20:36:34,http://bodeward.info/charity,,174.97.58.44,"{""location"": ""FR"", ""is_mobile"": false}" 1404,1,33,2017-02-23 08:39:13,http://raugleason.com/charley,,226.132.94.73,"{""location"": ""EE"", ""is_mobile"": false}" 1405,1,33,2017-01-23 18:52:38,http://farrell.biz/darrion,,249.70.231.71,"{""location"": ""HR"", ""is_mobile"": true}" 1406,1,33,2017-04-22 16:45:11,http://rice.net/bertram_batz,,197.152.251.64,"{""location"": ""OM"", ""is_mobile"": false}" 1407,1,33,2017-01-27 09:58:21,http://dibbert.biz/waldo.kuhn,,137.230.90.181,"{""location"": ""SO"", ""is_mobile"": false}" 1408,1,33,2017-02-13 22:18:49,http://runte.io/kayley,,129.40.240.171,"{""location"": ""VA"", ""is_mobile"": false}" 1409,1,33,2017-01-31 07:38:49,http://beattyledner.biz/anika,,20.34.171.93,"{""location"": ""JO"", ""is_mobile"": true}" 1410,1,33,2017-04-26 15:34:45,http://damore.name/sam,,26.171.30.115,"{""location"": ""SN"", ""is_mobile"": false}" 1411,1,33,2017-02-20 08:01:31,http://kuhic.info/gavin.batz,,84.41.50.23,"{""location"": ""LC"", ""is_mobile"": false}" 1412,1,33,2017-03-04 13:59:11,http://hamillhintz.org/curt,,65.8.49.106,"{""location"": ""KW"", ""is_mobile"": false}" 22441,8,503,2017-02-14 16:12:32,http://emard.com/jane.muller,,40.180.25.181,"{""location"": ""PK"", ""is_mobile"": true}" 22442,8,503,2017-01-05 08:16:31,http://jacobson.name/verdie,,85.56.238.200,"{""location"": ""VA"", ""is_mobile"": true}" 22443,8,503,2017-03-08 00:05:40,http://welchschroeder.io/jordyn_heidenreich,,117.62.167.113,"{""location"": ""TF"", ""is_mobile"": false}" 22444,8,503,2017-03-25 13:51:49,http://kohlerfarrell.name/velma,,154.118.199.112,"{""location"": ""GH"", ""is_mobile"": true}" 22445,8,503,2017-01-24 04:38:03,http://ohara.net/celia.davis,,132.153.143.205,"{""location"": ""IQ"", ""is_mobile"": true}" 22446,8,503,2017-01-30 21:28:40,http://mertz.co/ellis,,120.204.92.235,"{""location"": ""TM"", ""is_mobile"": false}" 22447,8,503,2017-02-24 03:14:50,http://hane.name/melya,,225.197.66.224,"{""location"": ""IM"", ""is_mobile"": false}" 22449,8,503,2017-03-24 16:19:27,http://lemke.co/easter,,2.118.81.162,"{""location"": ""NI"", ""is_mobile"": false}" 22450,8,504,2017-04-11 10:01:27,http://rau.info/jee,,12.181.158.137,"{""location"": ""NC"", ""is_mobile"": false}" 22451,8,504,2017-02-17 01:34:34,http://torp.net/chase,,24.194.253.49,"{""location"": ""IE"", ""is_mobile"": false}" 22452,8,504,2017-01-27 21:02:16,http://oconnell.net/lukas_pfannerstill,,59.39.49.228,"{""location"": ""MQ"", ""is_mobile"": false}" 22453,8,504,2017-01-07 11:17:53,http://haag.net/michel_mertz,,231.216.240.92,"{""location"": ""NU"", ""is_mobile"": true}" 22454,8,504,2017-02-02 10:58:14,http://hodkiewiczmetz.info/jairo_considine,,205.220.64.63,"{""location"": ""FR"", ""is_mobile"": false}" 22455,8,504,2016-12-19 09:34:52,http://ullrichking.com/ned.becker,,169.65.36.19,"{""location"": ""ME"", ""is_mobile"": true}" 22456,8,504,2017-03-05 08:17:31,http://erdman.info/earnestine_walter,,78.95.121.30,"{""location"": ""AO"", ""is_mobile"": false}" 22457,8,504,2017-05-22 11:57:52,http://toy.info/lavinia_ruel,,74.72.71.50,"{""location"": ""NU"", ""is_mobile"": false}" 22458,8,504,2017-05-04 03:46:28,http://walter.io/ike.bauch,,133.17.58.29,"{""location"": ""CF"", ""is_mobile"": false}" 22459,8,504,2016-12-26 11:43:09,http://hartmanndach.net/blair.huel,,86.63.23.45,"{""location"": ""EC"", ""is_mobile"": false}" 22460,8,504,2017-05-17 23:57:47,http://macgyver.name/glenda,,232.214.87.247,"{""location"": ""SG"", ""is_mobile"": true}" 22461,8,504,2017-05-03 12:59:12,http://littelstiedemann.com/newell,,77.62.103.85,"{""location"": ""MX"", ""is_mobile"": true}" 22462,8,504,2017-04-13 17:04:51,http://breitenberg.com/osborne,,177.64.101.153,"{""location"": ""FM"", ""is_mobile"": true}" 22463,8,504,2016-12-20 14:04:29,http://rowe.info/winona_wilkinson,,187.194.91.107,"{""location"": ""UA"", ""is_mobile"": false}" 22464,8,504,2016-12-31 15:48:36,http://schoen.co/wilfrid.adams,,6.179.177.119,"{""location"": ""BS"", ""is_mobile"": false}" 22465,8,504,2016-12-20 11:23:59,http://koelpin.io/bertha,,126.145.167.36,"{""location"": ""PH"", ""is_mobile"": true}" 22466,8,504,2016-12-21 12:23:28,http://koch.com/demarco_bailey,,87.238.190.57,"{""location"": ""DZ"", ""is_mobile"": true}" 22467,8,504,2016-12-25 14:28:32,http://ornoberbrunner.com/ferne.reynolds,,72.57.194.73,"{""location"": ""PW"", ""is_mobile"": false}" 22468,8,504,2017-05-29 06:03:30,http://davis.io/woodrow.eichmann,,104.129.206.47,"{""location"": ""NP"", ""is_mobile"": true}" 22469,8,504,2017-05-02 02:15:39,http://volkman.net/tatyana.stark,,41.103.93.196,"{""location"": ""LR"", ""is_mobile"": false}" 22470,8,504,2017-03-20 09:03:51,http://wiegand.net/terrill,,139.80.104.44,"{""location"": ""KW"", ""is_mobile"": false}" 22471,8,504,2017-04-10 01:45:53,http://raynor.co/wyman,,130.253.54.245,"{""location"": ""DM"", ""is_mobile"": true}" 22472,8,504,2017-01-27 17:02:56,http://rohan.info/demario.legros,,10.186.194.140,"{""location"": ""VE"", ""is_mobile"": false}" 22473,8,504,2017-05-03 21:12:41,http://bernhardbailey.net/hildegard_reilly,,245.7.14.64,"{""location"": ""SO"", ""is_mobile"": true}" 22474,8,504,2017-03-10 18:00:29,http://daniel.name/amya_kiehn,,80.17.246.39,"{""location"": ""LA"", ""is_mobile"": true}" 22475,8,504,2017-04-08 05:13:09,http://robel.info/franz,,48.113.74.188,"{""location"": ""NR"", ""is_mobile"": false}" 22476,8,504,2017-05-11 23:38:03,http://harberschmeler.com/ellie,,66.139.246.238,"{""location"": ""MS"", ""is_mobile"": false}" 22477,8,504,2016-12-21 08:54:28,http://gerholdschneider.biz/glenna.goyette,,237.103.46.232,"{""location"": ""RW"", ""is_mobile"": true}" 22478,8,504,2017-06-13 05:18:57,http://bauch.com/kaley,,187.41.200.173,"{""location"": ""RO"", ""is_mobile"": false}" 22479,8,504,2017-01-26 14:25:44,http://rolfsonlangworth.com/josue,,171.108.72.121,"{""location"": ""MD"", ""is_mobile"": true}" 22480,8,504,2017-05-01 23:16:34,http://haag.io/darius_rogahn,,202.54.239.65,"{""location"": ""MM"", ""is_mobile"": false}" 22481,8,504,2017-02-04 19:13:05,http://lemke.com/meredith.prohaska,,160.200.65.115,"{""location"": ""SM"", ""is_mobile"": false}" 22482,8,504,2017-02-08 15:06:12,http://kirlinnolan.co/cortney_waelchi,,38.28.192.164,"{""location"": ""IS"", ""is_mobile"": false}" 22483,8,504,2017-04-06 21:13:30,http://hegmannshanahan.name/wilber,,235.160.213.208,"{""location"": ""SJ"", ""is_mobile"": true}" 22484,8,504,2017-02-15 16:03:11,http://purdy.org/keira.adams,,11.195.217.210,"{""location"": ""NL"", ""is_mobile"": false}" 22485,8,504,2017-02-17 11:37:32,http://smith.name/emmanuel,,60.134.45.63,"{""location"": ""BS"", ""is_mobile"": false}" 22486,8,504,2017-01-05 13:17:26,http://gerhold.io/ted,,117.68.23.220,"{""location"": ""NU"", ""is_mobile"": false}" 22487,8,504,2017-05-24 12:33:26,http://dooleysmitham.info/taylor,,87.20.136.26,"{""location"": ""TG"", ""is_mobile"": false}" 22488,8,504,2017-02-24 12:44:40,http://grady.co/madisyn.hoppe,,21.74.144.174,"{""location"": ""MW"", ""is_mobile"": true}" 22489,8,504,2016-12-21 04:46:43,http://andersonbednar.info/esther.wehner,,114.36.92.103,"{""location"": ""KG"", ""is_mobile"": false}" 22490,8,504,2017-04-03 22:08:53,http://oharaterry.name/kenneth_dibbert,,161.58.103.230,"{""location"": ""PF"", ""is_mobile"": true}" 22491,8,504,2017-06-12 19:04:24,http://gulgowskischuster.org/ford.windler,,94.100.72.157,"{""location"": ""NG"", ""is_mobile"": false}" 22492,8,504,2017-06-08 07:31:30,http://cain.com/audra.cartwright,,164.144.30.9,"{""location"": ""FI"", ""is_mobile"": true}" 22493,8,504,2017-06-13 13:29:56,http://maggiokeeling.info/sabina.lynch,,200.169.186.167,"{""location"": ""SZ"", ""is_mobile"": false}" 22494,8,504,2017-01-09 23:57:05,http://cummerata.biz/zula,,97.197.178.250,"{""location"": ""JP"", ""is_mobile"": true}" 22495,8,504,2017-01-19 16:50:08,http://ortizstark.org/syble.osinski,,156.21.239.180,"{""location"": ""HT"", ""is_mobile"": false}" 22496,8,504,2017-06-06 12:21:01,http://hackett.com/bret,,72.93.124.208,"{""location"": ""UZ"", ""is_mobile"": false}" 1413,1,33,2017-02-25 07:32:27,http://heidenreichwatsica.io/robb.heller,,20.238.209.72,"{""location"": ""IO"", ""is_mobile"": false}" 1414,1,33,2017-02-15 17:47:21,http://gutmann.info/zoey.abbott,,237.197.199.108,"{""location"": ""PG"", ""is_mobile"": false}" 1415,1,33,2017-04-26 04:44:00,http://bahringer.com/gunner.howell,,224.172.67.65,"{""location"": ""SG"", ""is_mobile"": true}" 1416,1,33,2017-03-19 11:18:50,http://morarruecker.co/schuyler,,183.92.82.238,"{""location"": ""IQ"", ""is_mobile"": false}" 1417,1,33,2017-03-22 18:22:01,http://funk.name/alvah,,20.50.171.57,"{""location"": ""KZ"", ""is_mobile"": true}" 1418,1,33,2017-02-23 23:05:32,http://daugherty.name/nathan,,102.11.214.137,"{""location"": ""HN"", ""is_mobile"": false}" 1419,1,33,2017-03-15 23:15:22,http://dubuque.com/gracie,,242.41.36.136,"{""location"": ""GY"", ""is_mobile"": false}" 1420,1,33,2017-06-09 12:19:59,http://nicolas.info/saul.king,,213.184.223.78,"{""location"": ""GL"", ""is_mobile"": true}" 1421,1,33,2017-02-03 01:13:12,http://graham.biz/maurine.dickinson,,17.77.158.166,"{""location"": ""DE"", ""is_mobile"": true}" 1422,1,33,2017-03-01 03:31:59,http://wizafeil.biz/luisa,,214.100.37.152,"{""location"": ""GY"", ""is_mobile"": true}" 1423,1,33,2017-05-17 17:38:40,http://hettinger.org/chet,,93.204.194.204,"{""location"": ""GQ"", ""is_mobile"": true}" 1424,1,33,2016-12-31 06:00:30,http://reichel.biz/myles,,100.199.206.163,"{""location"": ""CX"", ""is_mobile"": false}" 1425,1,33,2017-04-27 08:22:55,http://nienow.biz/name,,185.250.65.140,"{""location"": ""SM"", ""is_mobile"": true}" 1426,1,33,2017-04-29 20:00:37,http://schuppe.io/cheyenne,,182.104.156.48,"{""location"": ""HT"", ""is_mobile"": true}" 1427,1,33,2016-12-14 07:26:49,http://spinka.info/lorenz,,142.3.214.240,"{""location"": ""PT"", ""is_mobile"": false}" 1428,1,33,2017-04-22 18:36:40,http://crist.org/easter,,217.176.103.127,"{""location"": ""PA"", ""is_mobile"": false}" 1429,1,33,2017-02-11 19:21:58,http://howe.name/fannie,,57.4.56.71,"{""location"": ""AI"", ""is_mobile"": false}" 1430,1,33,2017-01-30 20:30:54,http://king.name/pascale,,242.18.166.5,"{""location"": ""FJ"", ""is_mobile"": false}" 1431,1,33,2017-01-22 18:06:35,http://wilderman.name/tatum,,185.200.107.62,"{""location"": ""DZ"", ""is_mobile"": true}" 1432,1,33,2016-12-15 00:20:47,http://okunevanolan.co/jensen.lueilwitz,,196.148.102.109,"{""location"": ""TW"", ""is_mobile"": true}" 1433,1,33,2016-12-16 15:12:06,http://beier.co/hilton,,15.246.148.181,"{""location"": ""GR"", ""is_mobile"": false}" 1434,1,33,2017-06-08 03:13:27,http://gislason.io/bonnie,,172.107.39.232,"{""location"": ""NZ"", ""is_mobile"": false}" 1435,1,33,2016-12-30 22:59:12,http://dibbert.info/beie,,220.251.125.233,"{""location"": ""TR"", ""is_mobile"": true}" 1436,1,33,2017-02-14 01:02:06,http://roobkrajcik.org/jarret_erdman,,93.41.207.99,"{""location"": ""RE"", ""is_mobile"": false}" 1437,1,33,2017-03-14 17:09:55,http://swift.io/ayden,,30.66.113.200,"{""location"": ""KR"", ""is_mobile"": true}" 1438,1,33,2017-02-02 06:44:35,http://fay.name/emilie.heidenreich,,75.199.208.248,"{""location"": ""QA"", ""is_mobile"": false}" 1439,1,33,2017-05-04 08:03:59,http://prosaccoheidenreich.com/libby,,250.57.118.155,"{""location"": ""GT"", ""is_mobile"": true}" 1440,1,33,2017-04-12 01:51:51,http://hoegerswaniawski.org/rebekah,,25.218.6.179,"{""location"": ""GA"", ""is_mobile"": true}" 1441,1,33,2017-05-30 04:59:35,http://prosacco.net/kiera.breitenberg,,232.112.15.46,"{""location"": ""MS"", ""is_mobile"": true}" 1442,1,33,2017-05-06 06:42:46,http://ruel.com/wilfredo,,243.62.142.102,"{""location"": ""SB"", ""is_mobile"": true}" 1443,1,33,2017-01-20 06:11:45,http://klingbahringer.org/hailie_willms,,103.37.48.119,"{""location"": ""BT"", ""is_mobile"": true}" 1444,1,33,2017-02-21 09:05:13,http://balistreri.info/jovani,,155.144.162.32,"{""location"": ""GF"", ""is_mobile"": true}" 1445,1,33,2017-01-09 23:57:36,http://shieldskreiger.io/judd,,70.169.233.73,"{""location"": ""JP"", ""is_mobile"": false}" 1446,1,33,2017-03-18 03:01:39,http://hartmannsmitham.co/emily,,84.110.191.115,"{""location"": ""SY"", ""is_mobile"": true}" 1447,1,33,2017-02-28 16:09:46,http://mayer.net/alford,,33.59.73.15,"{""location"": ""KG"", ""is_mobile"": true}" 1448,1,33,2017-06-05 23:56:20,http://wilkinson.name/delta.brown,,95.17.231.156,"{""location"": ""JP"", ""is_mobile"": false}" 1449,1,33,2017-05-18 09:15:09,http://upton.com/vidal.mcglynn,,225.163.182.69,"{""location"": ""BR"", ""is_mobile"": false}" 1450,1,33,2017-04-09 18:53:56,http://schmidtokon.biz/michale_swift,,123.157.129.112,"{""location"": ""SK"", ""is_mobile"": true}" 1451,1,33,2017-05-06 11:29:45,http://damore.name/pierre,,119.197.56.110,"{""location"": ""BG"", ""is_mobile"": false}" 1452,1,33,2017-01-04 04:47:25,http://thompson.net/lula.littel,,38.230.23.42,"{""location"": ""TN"", ""is_mobile"": false}" 1453,1,33,2017-05-06 15:18:01,http://mcdermott.info/albin.bogan,,114.40.180.171,"{""location"": ""NI"", ""is_mobile"": true}" 1454,1,33,2017-02-12 15:35:15,http://toy.biz/everett,,164.185.53.64,"{""location"": ""RS"", ""is_mobile"": false}" 1455,1,33,2017-03-18 06:22:52,http://hauck.io/ava_treutel,,63.56.20.77,"{""location"": ""AT"", ""is_mobile"": false}" 1456,1,33,2017-01-03 19:25:06,http://reingershields.biz/gaylord.schuppe,,216.230.179.184,"{""location"": ""LS"", ""is_mobile"": false}" 1457,1,33,2017-05-12 12:41:49,http://hickledaniel.net/mallory_grady,,163.60.88.62,"{""location"": ""FM"", ""is_mobile"": true}" 1458,1,33,2017-03-14 05:18:13,http://hellerharvey.name/esta_crooks,,236.121.189.13,"{""location"": ""RS"", ""is_mobile"": true}" 1459,1,33,2017-02-02 00:35:37,http://goodwinziemann.net/berenice_white,,162.98.49.33,"{""location"": ""TH"", ""is_mobile"": false}" 1460,1,33,2017-03-22 22:26:11,http://gorczany.info/johnny,,7.68.103.8,"{""location"": ""AD"", ""is_mobile"": false}" 1461,1,33,2017-04-09 11:36:10,http://kuvalis.io/willis.ratke,,144.220.157.121,"{""location"": ""EG"", ""is_mobile"": true}" 1462,1,33,2017-01-08 23:23:10,http://howell.biz/jensen,,4.88.150.176,"{""location"": ""KG"", ""is_mobile"": false}" 1463,1,33,2016-12-24 06:12:49,http://greenholtgulgowski.biz/elsa.leuschke,,30.212.30.100,"{""location"": ""KP"", ""is_mobile"": true}" 1464,1,33,2017-05-18 09:44:50,http://flatleylang.io/ruben,,82.36.17.199,"{""location"": ""GG"", ""is_mobile"": false}" 1465,1,33,2017-04-18 06:18:17,http://willms.biz/theron.crona,,203.18.20.8,"{""location"": ""HN"", ""is_mobile"": false}" 1466,1,33,2017-01-26 11:28:07,http://steuberspinka.name/delphine,,149.51.104.181,"{""location"": ""LC"", ""is_mobile"": true}" 1467,1,33,2017-06-03 10:49:38,http://jakubowski.net/diamond_tremblay,,201.19.214.183,"{""location"": ""OM"", ""is_mobile"": true}" 1468,1,34,2017-03-16 14:42:11,http://borer.info/mavis_legros,,24.167.182.197,"{""location"": ""IT"", ""is_mobile"": true}" 22497,8,504,2017-04-06 17:06:07,http://abbott.name/shanelle,,226.84.167.188,"{""location"": ""NC"", ""is_mobile"": true}" 22498,8,504,2017-02-07 15:01:58,http://bartell.co/cesar,,43.69.203.245,"{""location"": ""NC"", ""is_mobile"": false}" 22499,8,504,2017-02-18 11:44:10,http://cummingtrosin.biz/damion,,56.221.94.164,"{""location"": ""NZ"", ""is_mobile"": true}" 22500,8,504,2017-02-02 09:18:36,http://cartwrightking.io/dante,,23.127.19.52,"{""location"": ""BF"", ""is_mobile"": true}" 22501,8,504,2017-03-30 13:09:10,http://parkerebert.com/royce_moen,,40.196.114.41,"{""location"": ""BA"", ""is_mobile"": true}" 22502,8,504,2017-05-21 09:46:58,http://morarkuhlman.info/baylee,,179.147.104.34,"{""location"": ""AS"", ""is_mobile"": false}" 22503,8,504,2017-01-11 03:46:36,http://kutch.name/alberto.mcglynn,,97.15.105.90,"{""location"": ""ES"", ""is_mobile"": true}" 22504,8,504,2017-02-21 12:44:42,http://grantbradtke.io/soledad,,112.64.31.152,"{""location"": ""CL"", ""is_mobile"": false}" 22505,8,504,2017-03-05 02:38:28,http://oconner.biz/ebony_jacobi,,36.139.5.162,"{""location"": ""GU"", ""is_mobile"": true}" 22506,8,504,2017-01-07 09:40:56,http://auerflatley.name/odea_ruel,,155.86.59.112,"{""location"": ""CD"", ""is_mobile"": false}" 22507,8,504,2017-03-29 03:08:08,http://powlowski.org/kailee.haag,,39.30.188.139,"{""location"": ""FJ"", ""is_mobile"": false}" 22508,8,505,2017-03-07 06:39:04,http://bernierstracke.org/allan_baumbach,,128.50.138.165,"{""location"": ""CY"", ""is_mobile"": false}" 22509,8,505,2017-06-01 06:11:30,http://torpprohaska.biz/alanna,,132.209.245.7,"{""location"": ""BN"", ""is_mobile"": false}" 22510,8,505,2017-01-30 21:29:13,http://schmidt.com/valentina,,8.183.77.145,"{""location"": ""RU"", ""is_mobile"": false}" 22511,8,505,2017-04-01 15:16:50,http://yundt.org/evie,,165.84.221.153,"{""location"": ""TO"", ""is_mobile"": false}" 22512,8,505,2017-04-12 08:43:00,http://fayhammes.org/jose.walker,,7.95.10.121,"{""location"": ""QA"", ""is_mobile"": true}" 22513,8,505,2017-01-02 09:05:15,http://cronastoltenberg.biz/scottie,,228.70.208.88,"{""location"": ""PL"", ""is_mobile"": false}" 22514,8,505,2017-04-01 02:26:37,http://roberts.info/jerry_nader,,40.247.41.148,"{""location"": ""TG"", ""is_mobile"": false}" 22515,8,505,2017-02-25 02:08:54,http://harris.org/benny.marvin,,45.209.243.200,"{""location"": ""TG"", ""is_mobile"": true}" 22516,8,505,2017-05-22 21:33:25,http://schulist.info/destin,,23.193.27.34,"{""location"": ""KZ"", ""is_mobile"": true}" 22517,8,505,2017-03-05 02:15:05,http://donnellyullrich.co/coty,,228.117.188.3,"{""location"": ""YE"", ""is_mobile"": false}" 22518,8,505,2017-01-15 15:39:28,http://rodriguez.co/gia.adams,,103.192.192.170,"{""location"": ""AL"", ""is_mobile"": false}" 22519,8,505,2017-01-31 10:05:20,http://beer.net/antoinette,,208.148.5.201,"{""location"": ""NU"", ""is_mobile"": true}" 22520,8,505,2017-02-24 12:25:32,http://kling.info/noel.hudson,,60.224.137.248,"{""location"": ""CH"", ""is_mobile"": true}" 22521,8,505,2017-05-17 23:33:08,http://jaskolski.biz/kaia,,181.210.55.214,"{""location"": ""TL"", ""is_mobile"": true}" 22522,8,505,2017-02-27 00:25:48,http://kunde.org/margarette.franecki,,46.134.167.246,"{""location"": ""PW"", ""is_mobile"": true}" 22523,8,505,2017-01-18 11:15:34,http://huels.info/friedrich.ohara,,228.248.23.207,"{""location"": ""BQ"", ""is_mobile"": true}" 22524,8,505,2017-03-21 01:41:23,http://friesen.biz/angus,,34.223.4.58,"{""location"": ""NU"", ""is_mobile"": true}" 22525,8,505,2017-01-09 18:37:17,http://dickinson.co/edwin,,28.184.74.102,"{""location"": ""IE"", ""is_mobile"": false}" 22526,8,505,2017-02-03 06:15:25,http://block.com/hunter,,162.78.174.133,"{""location"": ""CD"", ""is_mobile"": true}" 22527,8,505,2017-05-25 19:28:47,http://tromp.net/nestor,,207.237.12.122,"{""location"": ""SM"", ""is_mobile"": false}" 22528,8,506,2017-01-01 14:47:45,http://reichert.co/valentine.terry,,231.66.221.4,"{""location"": ""MA"", ""is_mobile"": false}" 22529,8,506,2017-06-01 23:04:53,http://pauceklangworth.co/susanna.weimann,,143.251.35.180,"{""location"": ""AQ"", ""is_mobile"": true}" 22530,8,506,2017-03-13 18:40:22,http://abbott.io/noelia.larson,,245.159.235.116,"{""location"": ""GL"", ""is_mobile"": false}" 22531,8,506,2017-01-05 02:54:07,http://farrellgutkowski.info/mikel,,168.150.197.249,"{""location"": ""UA"", ""is_mobile"": false}" 22532,8,506,2017-05-17 22:21:26,http://leannonzboncak.com/fletcher.pacocha,,115.216.224.43,"{""location"": ""GT"", ""is_mobile"": true}" 22533,8,506,2017-04-08 07:01:25,http://bayer.name/mose.ward,,42.100.126.145,"{""location"": ""PA"", ""is_mobile"": false}" 22534,8,506,2016-12-28 21:05:53,http://hegmannemard.net/reese.greenfelder,,101.115.152.43,"{""location"": ""MU"", ""is_mobile"": false}" 22535,8,506,2017-02-13 14:56:42,http://runolfsdottir.info/britney.herman,,175.71.145.244,"{""location"": ""HM"", ""is_mobile"": true}" 22536,8,506,2017-05-09 08:46:01,http://mertznitzsche.io/raheem_stokes,,54.167.226.30,"{""location"": ""BV"", ""is_mobile"": false}" 22537,8,506,2017-05-22 07:06:25,http://keelingbergnaum.com/kareem,,63.92.148.44,"{""location"": ""ET"", ""is_mobile"": false}" 22538,8,506,2017-03-04 15:49:40,http://welch.com/lilian,,22.238.27.40,"{""location"": ""ZA"", ""is_mobile"": true}" 22539,8,506,2017-05-23 14:58:47,http://oreillyhirthe.biz/veda,,155.111.169.43,"{""location"": ""MR"", ""is_mobile"": false}" 22540,8,506,2017-05-26 12:37:01,http://fahey.com/haan.herman,,16.42.129.190,"{""location"": ""MQ"", ""is_mobile"": true}" 22541,8,506,2017-01-19 21:37:31,http://connmckenzie.co/amari,,56.99.39.128,"{""location"": ""RS"", ""is_mobile"": true}" 22542,8,506,2017-04-09 11:18:01,http://schinner.com/aurore.borer,,97.109.173.56,"{""location"": ""VU"", ""is_mobile"": true}" 22543,8,506,2017-06-01 06:13:06,http://beahan.info/eulalia,,190.81.28.166,"{""location"": ""BQ"", ""is_mobile"": true}" 22544,8,506,2017-04-13 14:49:51,http://armstrongcronin.name/kari.ruecker,,188.27.198.120,"{""location"": ""NU"", ""is_mobile"": true}" 22545,8,506,2017-06-12 14:35:04,http://lind.name/izaiah_west,,15.227.215.159,"{""location"": ""AF"", ""is_mobile"": false}" 22546,8,506,2017-02-20 23:45:44,http://ward.org/deborah,,39.125.61.134,"{""location"": ""GP"", ""is_mobile"": true}" 22547,8,506,2017-01-07 12:32:59,http://smithammoore.com/keeley_okuneva,,31.14.121.73,"{""location"": ""AL"", ""is_mobile"": false}" 22548,8,506,2017-06-01 05:50:10,http://wuckert.org/kristofer,,100.153.104.82,"{""location"": ""BA"", ""is_mobile"": true}" 22549,8,506,2017-05-16 01:08:26,http://carterreichel.com/stan.aufderhar,,112.12.46.168,"{""location"": ""BB"", ""is_mobile"": true}" 22550,8,506,2017-04-24 17:39:24,http://cainjohnson.org/alexander,,81.140.215.254,"{""location"": ""LC"", ""is_mobile"": false}" 22551,8,506,2017-01-01 13:08:45,http://hane.com/maegan,,60.225.87.70,"{""location"": ""AE"", ""is_mobile"": true}" 1469,1,34,2017-04-19 02:26:53,http://romaguera.com/karson,,91.10.113.168,"{""location"": ""CZ"", ""is_mobile"": true}" 1470,1,34,2017-03-02 13:23:51,http://shields.info/frederique_wyman,,250.135.29.35,"{""location"": ""SY"", ""is_mobile"": false}" 1471,1,34,2017-02-02 11:15:52,http://lemke.io/kellie,,120.182.131.56,"{""location"": ""DO"", ""is_mobile"": true}" 1472,1,34,2017-01-16 07:20:14,http://ferry.info/norwood_hartmann,,58.203.32.244,"{""location"": ""BN"", ""is_mobile"": false}" 1473,1,34,2017-03-15 07:00:34,http://davis.com/eve,,41.171.223.37,"{""location"": ""CH"", ""is_mobile"": false}" 1474,1,34,2017-02-23 22:59:58,http://kub.name/saul,,186.91.19.22,"{""location"": ""GI"", ""is_mobile"": true}" 1475,1,34,2017-01-05 02:10:41,http://rodrigueznikolaus.name/mariah_fay,,210.125.100.109,"{""location"": ""MF"", ""is_mobile"": true}" 1476,1,34,2017-01-16 20:43:20,http://koelpinwunsch.co/angela_smitham,,103.169.127.143,"{""location"": ""NR"", ""is_mobile"": false}" 1477,1,34,2016-12-16 21:09:27,http://ebert.com/abby,,219.15.243.224,"{""location"": ""KH"", ""is_mobile"": true}" 1478,1,34,2017-06-01 13:30:30,http://murazik.biz/reinhold,,34.32.100.22,"{""location"": ""LY"", ""is_mobile"": false}" 1479,1,34,2017-05-23 15:31:45,http://barton.io/brendon,,241.95.24.143,"{""location"": ""NE"", ""is_mobile"": true}" 1480,1,34,2017-01-23 09:58:52,http://kundehaley.net/sarah,,148.70.250.49,"{""location"": ""IT"", ""is_mobile"": true}" 1481,1,34,2017-01-12 03:54:04,http://mcclure.net/gilda_baumbach,,128.77.11.21,"{""location"": ""MO"", ""is_mobile"": false}" 1482,1,34,2017-02-15 00:34:49,http://rodriguezluettgen.name/sienna,,238.232.229.17,"{""location"": ""PS"", ""is_mobile"": true}" 1483,1,34,2017-03-25 06:10:33,http://franecki.io/may,,87.209.168.181,"{""location"": ""ST"", ""is_mobile"": false}" 1484,1,34,2017-04-18 14:22:42,http://dachjenkins.biz/matilda,,4.116.189.3,"{""location"": ""GT"", ""is_mobile"": true}" 1485,1,34,2017-01-16 22:37:12,http://rolfson.info/susanna,,149.232.190.144,"{""location"": ""MA"", ""is_mobile"": true}" 1486,1,34,2016-12-29 19:45:12,http://jaskolski.co/lazaro,,119.211.251.221,"{""location"": ""CN"", ""is_mobile"": true}" 1487,1,34,2017-04-04 14:11:46,http://effertzdooley.biz/kathryn_weinat,,51.246.130.100,"{""location"": ""CY"", ""is_mobile"": true}" 1488,1,34,2017-05-09 20:37:24,http://runolfsdottir.org/meaghan.mosciski,,227.203.120.251,"{""location"": ""SE"", ""is_mobile"": false}" 1489,1,34,2017-03-18 19:39:35,http://rice.name/nelle,,202.12.123.84,"{""location"": ""TD"", ""is_mobile"": false}" 1490,1,34,2017-04-29 10:11:00,http://greenholtvolkman.io/manuel,,137.107.37.200,"{""location"": ""AF"", ""is_mobile"": true}" 1491,1,34,2017-03-07 04:20:36,http://leannon.info/george,,95.152.139.55,"{""location"": ""VA"", ""is_mobile"": false}" 1492,1,34,2017-01-11 12:10:28,http://watsica.name/rashad.shanahan,,203.184.153.242,"{""location"": ""CO"", ""is_mobile"": true}" 1493,1,34,2017-05-14 03:14:36,http://bernierschimmel.com/arvilla_altenwerth,,31.183.225.20,"{""location"": ""TR"", ""is_mobile"": true}" 1494,1,34,2017-01-21 02:11:38,http://champlin.info/riley,,56.194.191.194,"{""location"": ""CM"", ""is_mobile"": false}" 1495,1,34,2017-06-02 14:38:26,http://balistrerischulist.net/jimmie,,13.160.33.212,"{""location"": ""UA"", ""is_mobile"": true}" 1496,1,34,2017-02-22 11:17:08,http://morar.com/oral,,89.240.47.230,"{""location"": ""PR"", ""is_mobile"": false}" 1497,1,34,2017-01-06 00:37:38,http://oconnell.net/cecilia,,10.248.130.230,"{""location"": ""NL"", ""is_mobile"": true}" 1498,1,34,2017-04-30 14:06:45,http://corwin.org/chance,,98.189.166.127,"{""location"": ""MF"", ""is_mobile"": true}" 1499,1,34,2017-02-12 12:30:03,http://cronin.com/darien.smith,,181.9.60.102,"{""location"": ""SM"", ""is_mobile"": true}" 1500,1,34,2017-03-21 08:19:28,http://boehm.name/maybell,,67.4.175.61,"{""location"": ""OM"", ""is_mobile"": true}" 1501,1,34,2017-05-26 13:06:19,http://labadiecarter.co/santa.sauer,,237.33.90.154,"{""location"": ""UA"", ""is_mobile"": false}" 1502,1,34,2016-12-22 10:51:01,http://kohler.biz/vena.greenholt,,131.232.30.138,"{""location"": ""HR"", ""is_mobile"": true}" 1503,1,34,2016-12-29 03:45:27,http://oreillylehner.biz/abigayle.bode,,36.240.177.220,"{""location"": ""MK"", ""is_mobile"": true}" 1504,1,34,2017-02-16 23:50:48,http://hansen.co/kaylah,,50.222.59.171,"{""location"": ""CU"", ""is_mobile"": true}" 1505,1,35,2017-03-17 17:58:59,http://emardking.net/vada_bergstrom,,39.84.81.231,"{""location"": ""TC"", ""is_mobile"": false}" 1506,1,35,2017-05-15 19:11:43,http://nolangutmann.name/donny_turner,,99.153.96.83,"{""location"": ""LA"", ""is_mobile"": true}" 1507,1,35,2017-01-13 09:33:23,http://strackebergnaum.name/precious,,251.168.134.141,"{""location"": ""VN"", ""is_mobile"": true}" 1508,1,35,2017-01-10 00:42:51,http://kleinherman.io/precious_schimmel,,68.71.141.115,"{""location"": ""PE"", ""is_mobile"": true}" 1509,1,35,2017-02-21 17:09:50,http://christiansenreinger.io/jasen,,143.210.138.177,"{""location"": ""WS"", ""is_mobile"": true}" 1510,1,35,2017-01-01 01:36:35,http://gerhold.net/pasquale,,216.236.132.252,"{""location"": ""GW"", ""is_mobile"": false}" 1511,1,35,2017-05-23 18:08:12,http://mitchell.io/arnoldo.ko,,249.159.173.104,"{""location"": ""GP"", ""is_mobile"": false}" 1512,1,35,2017-02-24 18:26:50,http://wittingschimmel.info/reynold,,174.36.113.3,"{""location"": ""GF"", ""is_mobile"": true}" 1513,1,35,2017-01-25 00:13:37,http://buckridge.info/raphaelle_hermann,,102.82.46.181,"{""location"": ""UY"", ""is_mobile"": true}" 1514,1,35,2017-05-23 08:34:52,http://ortiz.name/ubaldo,,179.35.48.142,"{""location"": ""WF"", ""is_mobile"": true}" 1515,1,35,2017-02-14 19:53:39,http://sawayn.com/zaria,,46.7.180.229,"{""location"": ""SM"", ""is_mobile"": true}" 1516,1,35,2017-06-09 14:53:56,http://stark.io/julie,,244.184.42.65,"{""location"": ""UA"", ""is_mobile"": true}" 1517,1,35,2017-02-09 17:54:52,http://beahanbosco.io/alfonso,,102.172.29.239,"{""location"": ""ML"", ""is_mobile"": true}" 1518,1,35,2017-03-19 19:12:40,http://weinattrantow.co/kobe,,246.77.30.185,"{""location"": ""LS"", ""is_mobile"": true}" 1519,1,35,2017-03-17 05:07:37,http://stammmckenzie.co/hilma,,85.248.193.179,"{""location"": ""IQ"", ""is_mobile"": false}" 1520,1,35,2017-05-18 04:40:08,http://bechtelarsimonis.io/landen.hagenes,,63.167.80.73,"{""location"": ""TV"", ""is_mobile"": true}" 1521,1,35,2016-12-13 20:24:42,http://towne.info/jailyn,,238.241.195.187,"{""location"": ""VI"", ""is_mobile"": true}" 1522,1,35,2017-04-16 16:50:11,http://wolf.co/perry_balistreri,,99.118.40.160,"{""location"": ""PT"", ""is_mobile"": true}" 1523,1,35,2016-12-20 10:03:58,http://mosciski.name/ransom.corwin,,176.34.147.135,"{""location"": ""SC"", ""is_mobile"": true}" 1524,1,35,2017-05-07 00:57:22,http://dicki.com/mike,,36.196.227.90,"{""location"": ""IN"", ""is_mobile"": false}" 22552,8,506,2017-02-03 11:44:27,http://farrellgleichner.co/vern.hand,,157.20.199.91,"{""location"": ""GL"", ""is_mobile"": false}" 22553,8,506,2017-03-13 08:16:21,http://nolan.name/xander_nolan,,85.140.253.70,"{""location"": ""UM"", ""is_mobile"": false}" 22554,8,506,2017-04-08 07:13:17,http://pacochasporer.net/kathryn.kemmer,,46.106.181.218,"{""location"": ""AR"", ""is_mobile"": false}" 22555,8,506,2017-05-13 05:20:47,http://yundt.info/robert,,14.63.135.209,"{""location"": ""DJ"", ""is_mobile"": false}" 22556,8,506,2017-04-11 08:44:49,http://swaniawskifeil.biz/raegan.larkin,,236.143.131.215,"{""location"": ""TK"", ""is_mobile"": false}" 22557,8,506,2017-06-07 18:25:10,http://turcotte.org/lisa_yundt,,169.81.94.52,"{""location"": ""SN"", ""is_mobile"": true}" 22558,8,506,2017-01-13 17:51:02,http://crona.info/kaela.morar,,174.21.5.161,"{""location"": ""MZ"", ""is_mobile"": false}" 22559,8,506,2017-03-30 17:31:33,http://corwinmertz.info/marjory.little,,107.72.194.31,"{""location"": ""KG"", ""is_mobile"": false}" 22560,8,506,2016-12-16 22:06:08,http://greenwisoky.biz/mitchel,,96.156.8.12,"{""location"": ""VU"", ""is_mobile"": false}" 22561,8,506,2017-03-13 21:24:05,http://vonrueden.io/dean,,79.225.185.127,"{""location"": ""BQ"", ""is_mobile"": false}" 22562,8,506,2017-03-14 15:57:00,http://okon.info/electa.kautzer,,102.222.189.236,"{""location"": ""VC"", ""is_mobile"": true}" 22563,8,506,2017-03-31 07:15:37,http://murphyzieme.com/bernadine,,203.159.186.8,"{""location"": ""BR"", ""is_mobile"": false}" 22564,8,506,2017-02-20 10:48:23,http://schaefer.net/columbus,,230.177.254.10,"{""location"": ""RO"", ""is_mobile"": false}" 22565,8,506,2017-05-12 00:35:15,http://damore.co/brian_mosciski,,158.90.189.144,"{""location"": ""RW"", ""is_mobile"": false}" 22566,8,506,2017-01-13 08:35:16,http://baumbachberge.info/sammy,,223.170.104.114,"{""location"": ""AX"", ""is_mobile"": false}" 22567,8,506,2017-06-11 16:35:38,http://batz.com/armani,,31.160.31.211,"{""location"": ""LY"", ""is_mobile"": false}" 22568,8,507,2016-12-25 02:34:52,http://lubowitz.name/rachelle,,89.218.139.92,"{""location"": ""PR"", ""is_mobile"": false}" 22569,8,507,2017-01-20 04:10:00,http://schuster.info/alvera,,27.200.80.234,"{""location"": ""TZ"", ""is_mobile"": false}" 22570,8,507,2017-05-21 13:47:04,http://kihn.name/annetta,,217.41.193.16,"{""location"": ""CF"", ""is_mobile"": false}" 22571,8,507,2017-03-03 03:03:48,http://dickenswillms.net/rudy_toy,,222.173.40.128,"{""location"": ""AU"", ""is_mobile"": true}" 22572,8,507,2017-04-16 08:24:49,http://carter.biz/fannie,,86.181.239.144,"{""location"": ""MY"", ""is_mobile"": true}" 22573,8,507,2017-04-18 11:36:43,http://buckridge.biz/ena_veum,,227.23.89.193,"{""location"": ""ZM"", ""is_mobile"": true}" 22574,8,507,2016-12-19 14:57:14,http://murrayebert.co/bonita,,217.208.104.148,"{""location"": ""MT"", ""is_mobile"": true}" 22575,8,507,2016-12-20 03:48:01,http://greenbode.biz/pinkie_fahey,,137.26.53.220,"{""location"": ""TT"", ""is_mobile"": true}" 22576,8,507,2017-02-24 23:00:40,http://schmittcorkery.org/clinton,,243.162.79.100,"{""location"": ""DO"", ""is_mobile"": true}" 22577,8,507,2017-03-21 05:24:14,http://harber.name/milan,,130.60.27.188,"{""location"": ""MW"", ""is_mobile"": true}" 22578,8,507,2017-06-07 05:18:29,http://lesch.org/lizeth,,182.18.121.191,"{""location"": ""CF"", ""is_mobile"": false}" 22579,8,507,2017-06-01 02:41:08,http://shields.org/kelsie_kunde,,160.195.230.82,"{""location"": ""SB"", ""is_mobile"": false}" 22580,8,507,2017-06-12 04:20:53,http://prosacco.com/eddie.cole,,132.241.217.118,"{""location"": ""TF"", ""is_mobile"": false}" 22581,8,507,2017-01-02 05:52:42,http://stokes.info/dee.orn,,8.46.98.170,"{""location"": ""MP"", ""is_mobile"": false}" 22582,8,507,2016-12-17 04:29:26,http://hilll.info/taurean_gleason,,166.220.188.139,"{""location"": ""KE"", ""is_mobile"": true}" 22583,8,507,2016-12-25 01:25:32,http://effertz.name/ora_ko,,3.244.111.19,"{""location"": ""ME"", ""is_mobile"": true}" 22584,8,507,2017-06-05 02:06:44,http://stokes.org/markus,,141.203.12.242,"{""location"": ""EC"", ""is_mobile"": false}" 22585,8,507,2017-02-11 00:54:50,http://zboncak.org/dayne,,14.23.61.213,"{""location"": ""DK"", ""is_mobile"": true}" 22586,8,507,2017-04-16 00:42:39,http://jastkertzmann.co/tanner,,151.18.124.141,"{""location"": ""AO"", ""is_mobile"": true}" 22587,8,507,2017-01-23 03:02:54,http://pfannerstillhermann.org/rodger,,92.143.45.72,"{""location"": ""TM"", ""is_mobile"": true}" 22588,8,507,2017-04-03 20:57:46,http://wisozk.io/rita.hartmann,,44.17.83.177,"{""location"": ""GF"", ""is_mobile"": true}" 22589,8,507,2017-04-30 09:09:52,http://lubowitz.com/cody,,103.133.128.65,"{""location"": ""LT"", ""is_mobile"": false}" 22590,8,507,2017-05-20 23:15:13,http://bode.net/vida,,51.136.130.83,"{""location"": ""BG"", ""is_mobile"": true}" 22591,8,507,2017-06-06 06:37:33,http://buckridge.info/dejah.simonis,,104.135.94.253,"{""location"": ""PR"", ""is_mobile"": true}" 22592,8,507,2017-04-22 11:46:00,http://millchaden.com/ludwig_metz,,211.240.105.187,"{""location"": ""IT"", ""is_mobile"": false}" 22593,8,507,2017-01-14 22:25:28,http://mitchellbreitenberg.biz/faustino.corkery,,153.13.8.186,"{""location"": ""CM"", ""is_mobile"": true}" 22594,8,507,2017-04-11 06:05:16,http://bednar.biz/bettye,,200.244.225.34,"{""location"": ""PG"", ""is_mobile"": false}" 22595,8,507,2017-06-10 07:30:40,http://predovicdooley.info/lura_carroll,,45.143.76.179,"{""location"": ""GD"", ""is_mobile"": true}" 22596,8,507,2017-02-21 18:06:38,http://swiftsatterfield.org/ruthe_zulauf,,33.136.132.45,"{""location"": ""BM"", ""is_mobile"": false}" 22597,8,507,2017-04-26 22:31:34,http://welch.co/cathryn_schiller,,242.72.241.178,"{""location"": ""CZ"", ""is_mobile"": false}" 22598,8,507,2017-02-21 04:20:47,http://spinkasmitham.com/dante,,165.67.161.193,"{""location"": ""FJ"", ""is_mobile"": true}" 22599,8,507,2017-03-04 14:58:40,http://reinger.info/quinten,,59.230.31.31,"{""location"": ""LI"", ""is_mobile"": true}" 22600,8,507,2017-04-25 02:48:30,http://beier.net/lyric_borer,,189.27.96.109,"{""location"": ""TW"", ""is_mobile"": true}" 22601,8,507,2017-03-01 18:54:25,http://mcglynnblanda.net/bridgette.aufderhar,,126.225.16.217,"{""location"": ""MV"", ""is_mobile"": true}" 22602,8,507,2017-04-08 18:17:44,http://mann.info/chester.hettinger,,161.102.121.155,"{""location"": ""MZ"", ""is_mobile"": false}" 22603,8,507,2017-04-02 06:13:51,http://krishansen.info/lavinia,,27.199.35.11,"{""location"": ""UA"", ""is_mobile"": false}" 22604,8,507,2017-04-21 06:29:11,http://osinski.org/angela,,31.125.175.198,"{""location"": ""BE"", ""is_mobile"": true}" 22605,8,507,2017-02-10 09:35:38,http://zulauf.io/liam,,42.84.231.207,"{""location"": ""FI"", ""is_mobile"": false}" 22606,8,507,2017-01-20 02:20:46,http://lynch.org/elda_weimann,,44.69.118.160,"{""location"": ""GE"", ""is_mobile"": true}" 1525,1,35,2017-01-14 19:31:35,http://lakin.name/ardith_eichmann,,38.6.122.139,"{""location"": ""PA"", ""is_mobile"": false}" 1526,1,35,2017-02-22 23:08:43,http://ledner.co/sasha,,25.175.35.192,"{""location"": ""TT"", ""is_mobile"": false}" 1527,1,35,2017-04-12 15:48:58,http://schmitt.name/keira,,231.165.172.14,"{""location"": ""TN"", ""is_mobile"": true}" 1528,1,35,2016-12-25 21:00:58,http://cain.co/arthur_brekke,,80.82.62.121,"{""location"": ""KM"", ""is_mobile"": false}" 1529,1,35,2017-01-24 17:44:44,http://huels.co/noemi.durgan,,172.45.230.123,"{""location"": ""NG"", ""is_mobile"": false}" 1530,1,35,2017-01-25 09:33:09,http://brown.name/ed,,244.60.158.144,"{""location"": ""CI"", ""is_mobile"": false}" 1531,1,35,2017-04-21 09:47:48,http://feeney.org/malinda_borer,,169.10.91.161,"{""location"": ""GR"", ""is_mobile"": true}" 1532,1,35,2017-03-08 11:57:52,http://wisozk.io/ottis_stroman,,31.207.75.56,"{""location"": ""JO"", ""is_mobile"": true}" 1533,1,35,2017-01-24 12:07:16,http://mcdermott.com/adolf,,13.227.143.104,"{""location"": ""QA"", ""is_mobile"": true}" 1534,1,35,2017-03-08 16:28:58,http://hoppe.info/marlee,,32.168.107.16,"{""location"": ""CL"", ""is_mobile"": true}" 1535,1,35,2017-02-13 06:07:44,http://feeney.com/delfina,,105.139.238.92,"{""location"": ""NE"", ""is_mobile"": false}" 1536,1,35,2016-12-20 10:17:38,http://balistreri.io/rodrigo_kunze,,215.166.57.77,"{""location"": ""KN"", ""is_mobile"": false}" 1537,1,35,2017-04-24 03:06:40,http://vonhyatt.name/adelle,,164.10.254.12,"{""location"": ""GH"", ""is_mobile"": true}" 1538,1,35,2017-05-28 23:41:11,http://rippin.co/elliot,,106.149.104.121,"{""location"": ""VN"", ""is_mobile"": true}" 1539,1,35,2017-03-01 14:53:26,http://oreilly.info/hailey.emmerich,,196.60.106.150,"{""location"": ""AR"", ""is_mobile"": false}" 1540,1,35,2017-01-25 17:23:30,http://morar.io/rosella.waters,,134.98.158.210,"{""location"": ""SH"", ""is_mobile"": false}" 1541,1,35,2017-05-20 03:41:55,http://breitenberggulgowski.org/wade.keeling,,226.181.92.151,"{""location"": ""MT"", ""is_mobile"": true}" 1542,1,35,2017-01-31 14:44:08,http://botsford.net/anna,,88.222.229.250,"{""location"": ""PS"", ""is_mobile"": false}" 1543,1,35,2017-03-04 02:46:22,http://oreillyprosacco.org/tristin,,173.140.171.156,"{""location"": ""SR"", ""is_mobile"": true}" 1544,1,35,2017-05-30 05:16:38,http://toy.info/nicolas,,65.82.110.40,"{""location"": ""LI"", ""is_mobile"": true}" 1545,1,35,2017-03-25 12:21:43,http://mcclure.info/jaylon_gerlach,,156.208.33.73,"{""location"": ""KR"", ""is_mobile"": true}" 1546,1,35,2017-05-25 02:02:54,http://reichertturcotte.org/isabell_ziemann,,9.146.140.5,"{""location"": ""EH"", ""is_mobile"": true}" 1547,1,35,2017-01-06 07:51:28,http://gibson.com/camille,,132.208.163.89,"{""location"": ""BM"", ""is_mobile"": false}" 1548,1,35,2017-06-06 15:50:16,http://davis.io/tara.lemke,,23.23.163.7,"{""location"": ""JO"", ""is_mobile"": false}" 1549,1,35,2016-12-31 16:32:07,http://zboncak.name/jermey,,54.138.214.199,"{""location"": ""KR"", ""is_mobile"": true}" 1550,1,35,2017-01-21 07:26:27,http://oconnellboehm.io/roger_nienow,,159.178.234.152,"{""location"": ""CL"", ""is_mobile"": false}" 1551,1,35,2017-02-24 17:42:48,http://sipes.name/kaitlin,,167.7.78.91,"{""location"": ""JP"", ""is_mobile"": false}" 1552,1,35,2016-12-22 22:23:46,http://crooks.com/keely.larkin,,75.64.153.33,"{""location"": ""GH"", ""is_mobile"": false}" 1553,1,35,2017-02-23 19:30:22,http://nicolas.co/kayley,,179.85.2.123,"{""location"": ""MF"", ""is_mobile"": false}" 1554,1,35,2016-12-29 08:59:41,http://bergstrom.info/gaston.denesik,,246.198.249.104,"{""location"": ""GP"", ""is_mobile"": false}" 1555,1,35,2017-06-06 13:14:22,http://huelswitting.co/thalia,,2.77.57.153,"{""location"": ""NE"", ""is_mobile"": false}" 1556,1,35,2017-01-04 22:21:21,http://mraz.info/haie.okuneva,,12.184.41.134,"{""location"": ""BD"", ""is_mobile"": false}" 1557,1,35,2017-03-12 01:56:42,http://casper.name/lane,,84.78.118.133,"{""location"": ""PM"", ""is_mobile"": false}" 1558,1,35,2017-06-13 10:00:17,http://metzcartwright.info/carter,,144.184.167.41,"{""location"": ""MY"", ""is_mobile"": true}" 1559,1,35,2016-12-17 13:42:05,http://spinka.name/olga,,170.104.13.85,"{""location"": ""MP"", ""is_mobile"": false}" 1560,1,35,2017-04-13 06:25:17,http://stoltenbergmoen.co/mallory_christiansen,,109.64.114.155,"{""location"": ""BF"", ""is_mobile"": true}" 1561,1,35,2017-04-19 13:49:43,http://bechtelarschmidt.info/aditya.ruel,,131.166.218.154,"{""location"": ""LI"", ""is_mobile"": true}" 1562,1,35,2017-01-08 15:11:11,http://schroedermraz.com/amelie,,153.181.135.183,"{""location"": ""BA"", ""is_mobile"": false}" 1563,1,35,2017-02-28 09:06:05,http://wiza.biz/tommie_kunde,,185.46.170.156,"{""location"": ""MA"", ""is_mobile"": false}" 1564,1,35,2017-04-21 10:00:15,http://ruecker.name/katarina.terry,,115.12.245.186,"{""location"": ""KG"", ""is_mobile"": true}" 1565,1,35,2017-01-06 03:54:31,http://grimes.biz/kenny,,12.181.173.112,"{""location"": ""BO"", ""is_mobile"": true}" 1566,1,36,2017-05-22 18:47:15,http://glovergaylord.biz/madaline,,98.122.152.188,"{""location"": ""BS"", ""is_mobile"": false}" 1567,1,36,2017-02-18 21:09:35,http://runte.info/bennett,,89.12.21.48,"{""location"": ""JM"", ""is_mobile"": true}" 1568,1,36,2017-04-14 05:48:51,http://osinski.biz/jeanne_wiza,,35.128.175.41,"{""location"": ""MA"", ""is_mobile"": true}" 1569,1,36,2017-05-29 19:39:44,http://medhurstharvey.name/may_cormier,,167.101.31.248,"{""location"": ""IS"", ""is_mobile"": true}" 1570,1,36,2017-01-29 16:52:38,http://mertz.com/birdie_glover,,249.50.93.179,"{""location"": ""BY"", ""is_mobile"": false}" 1571,1,36,2017-01-29 13:27:41,http://hickleratke.io/minerva_murazik,,180.191.169.118,"{""location"": ""SC"", ""is_mobile"": true}" 1572,1,36,2017-05-21 11:46:31,http://mckenzie.biz/rylee_kihn,,248.203.213.119,"{""location"": ""TW"", ""is_mobile"": true}" 1573,1,36,2017-01-21 06:27:06,http://huelscorwin.info/marc.labadie,,202.111.180.93,"{""location"": ""SO"", ""is_mobile"": false}" 1574,1,36,2017-02-08 00:56:36,http://herzog.info/gerry.schuppe,,30.233.56.144,"{""location"": ""PK"", ""is_mobile"": false}" 1575,1,36,2017-04-30 19:51:59,http://markschamplin.info/cielo.hickle,,141.94.47.121,"{""location"": ""AZ"", ""is_mobile"": true}" 1576,1,36,2017-01-16 17:49:07,http://kaulke.co/aurelia,,63.192.126.114,"{""location"": ""CF"", ""is_mobile"": false}" 1577,1,36,2017-02-25 10:36:12,http://faheyjohns.com/wiley,,240.155.78.191,"{""location"": ""FR"", ""is_mobile"": false}" 1578,1,36,2017-02-01 15:13:27,http://dare.com/therese,,79.149.238.56,"{""location"": ""AG"", ""is_mobile"": false}" 1579,1,36,2017-05-07 19:50:52,http://huels.co/ceasar,,39.234.233.164,"{""location"": ""BI"", ""is_mobile"": true}" 1580,1,36,2017-06-03 11:28:49,http://hayes.net/ayla.gutmann,,158.72.146.86,"{""location"": ""IL"", ""is_mobile"": true}" 22607,8,507,2017-05-26 03:48:14,http://farrell.info/ariel.weber,,172.6.231.34,"{""location"": ""VA"", ""is_mobile"": true}" 22608,8,507,2017-01-18 23:04:13,http://feest.com/telly,,9.104.152.83,"{""location"": ""PA"", ""is_mobile"": true}" 22609,8,507,2017-03-10 05:09:17,http://hamill.name/brandi_weimann,,13.8.58.130,"{""location"": ""LU"", ""is_mobile"": false}" 22610,8,507,2017-04-12 03:37:34,http://swaniawski.name/haylee_brekke,,209.75.62.185,"{""location"": ""HN"", ""is_mobile"": false}" 22611,8,507,2017-01-10 21:33:00,http://jastmohr.co/anastasia_bailey,,231.228.122.123,"{""location"": ""TN"", ""is_mobile"": true}" 22612,8,507,2016-12-15 20:00:15,http://wiegandwaters.name/jermey_rogahn,,129.234.67.168,"{""location"": ""SI"", ""is_mobile"": false}" 22613,8,507,2017-04-11 00:04:57,http://frami.io/alaina.flatley,,115.245.183.81,"{""location"": ""MU"", ""is_mobile"": false}" 22614,8,507,2017-01-28 10:31:32,http://wuckertwill.net/buster.barrows,,239.209.219.246,"{""location"": ""MV"", ""is_mobile"": false}" 22615,8,507,2017-04-18 22:08:57,http://gislason.biz/peggie_wolf,,243.55.88.86,"{""location"": ""TD"", ""is_mobile"": false}" 22616,8,507,2017-04-15 08:59:53,http://hilll.org/ulices_morar,,37.105.206.134,"{""location"": ""MS"", ""is_mobile"": false}" 22617,8,508,2017-03-15 13:52:38,http://harber.co/zelma.orn,,52.157.180.241,"{""location"": ""MX"", ""is_mobile"": false}" 22618,8,508,2017-03-03 17:12:45,http://blick.name/clinton,,59.149.243.74,"{""location"": ""UG"", ""is_mobile"": false}" 22619,8,508,2017-06-06 04:35:01,http://deckowbarrows.org/abigayle,,2.199.42.207,"{""location"": ""LA"", ""is_mobile"": true}" 22620,8,508,2017-02-08 12:25:39,http://flatleyquitzon.name/colten,,110.248.157.63,"{""location"": ""IN"", ""is_mobile"": true}" 22621,8,508,2017-04-22 06:20:33,http://altenwerthjerde.name/jan,,174.180.129.200,"{""location"": ""LC"", ""is_mobile"": false}" 22622,8,508,2017-03-21 05:03:05,http://strosintreutel.info/dejon.keebler,,68.210.118.62,"{""location"": ""AW"", ""is_mobile"": true}" 22623,8,508,2017-01-19 16:59:33,http://ankundingbalistreri.co/frankie_hegmann,,201.54.95.234,"{""location"": ""BV"", ""is_mobile"": true}" 22624,8,508,2016-12-21 14:08:03,http://connellyhuel.com/jovany,,38.230.11.103,"{""location"": ""MP"", ""is_mobile"": true}" 22625,8,508,2017-06-10 14:19:11,http://hyattdurgan.info/celestino.zboncak,,74.4.130.34,"{""location"": ""MP"", ""is_mobile"": true}" 22626,8,508,2017-01-12 20:35:41,http://wuckert.com/abagail,,81.153.98.138,"{""location"": ""KH"", ""is_mobile"": false}" 22627,8,508,2017-01-18 20:16:31,http://kolangosh.co/twila,,7.52.100.23,"{""location"": ""TV"", ""is_mobile"": false}" 22628,8,508,2017-02-20 13:11:56,http://hodkiewicz.info/florencio,,231.214.41.92,"{""location"": ""CV"", ""is_mobile"": true}" 22629,8,508,2017-03-07 03:00:08,http://pagac.io/johann.kovacek,,115.117.124.54,"{""location"": ""GB"", ""is_mobile"": true}" 22630,8,508,2017-02-08 18:28:48,http://borer.co/harmony_jakubowski,,180.72.132.35,"{""location"": ""TG"", ""is_mobile"": false}" 22631,8,508,2017-03-22 14:22:17,http://mckenzie.biz/laurianne_kuvalis,,40.49.34.225,"{""location"": ""GT"", ""is_mobile"": false}" 22632,8,508,2017-05-02 13:22:46,http://rutherford.org/jakob,,153.61.220.103,"{""location"": ""CK"", ""is_mobile"": true}" 22633,8,508,2017-05-08 02:04:23,http://douglas.info/kaycee.cremin,,230.4.51.36,"{""location"": ""IT"", ""is_mobile"": false}" 22634,8,508,2017-04-22 00:31:51,http://sporer.com/arnold,,213.56.182.198,"{""location"": ""NL"", ""is_mobile"": false}" 22635,8,508,2016-12-14 01:04:54,http://kirlinschimmel.io/lew.pouros,,48.110.238.104,"{""location"": ""TR"", ""is_mobile"": true}" 22636,8,508,2016-12-19 15:52:47,http://corkery.io/coralie.collins,,74.144.107.2,"{""location"": ""SX"", ""is_mobile"": true}" 22637,8,508,2017-04-10 08:42:46,http://barton.co/marguerite,,217.42.181.225,"{""location"": ""CV"", ""is_mobile"": false}" 22638,8,508,2017-01-25 15:28:56,http://grahamwehner.info/genoveva_pfannerstill,,130.224.119.145,"{""location"": ""CK"", ""is_mobile"": false}" 22639,8,508,2016-12-23 21:52:21,http://cartwright.com/deangelo.crona,,21.85.166.245,"{""location"": ""FM"", ""is_mobile"": true}" 22640,8,508,2017-05-17 12:41:43,http://jacobsreichel.io/lurline.armstrong,,169.154.199.148,"{""location"": ""LI"", ""is_mobile"": false}" 22641,8,508,2016-12-20 12:15:51,http://douglasoconnell.com/chaim,,215.244.88.108,"{""location"": ""SR"", ""is_mobile"": true}" 22642,8,508,2017-03-23 05:58:23,http://jacobsonrosenbaum.biz/wilford_dietrich,,78.57.246.246,"{""location"": ""BI"", ""is_mobile"": false}" 22643,8,508,2017-05-28 07:55:42,http://bradtke.net/guadalupe_welch,,143.23.155.73,"{""location"": ""AG"", ""is_mobile"": false}" 22644,8,508,2016-12-24 09:49:38,http://greendickens.info/kennedi_wyman,,207.31.96.195,"{""location"": ""JM"", ""is_mobile"": true}" 22645,8,508,2017-02-15 12:55:30,http://bins.org/linda.ritchie,,142.100.198.220,"{""location"": ""SL"", ""is_mobile"": false}" 22646,8,508,2017-05-02 22:14:22,http://douglas.co/dante,,84.186.131.235,"{""location"": ""SE"", ""is_mobile"": true}" 22647,8,508,2017-01-19 03:43:53,http://wolffroob.biz/aniya_kozey,,162.51.121.180,"{""location"": ""CI"", ""is_mobile"": true}" 22648,8,508,2017-01-20 23:23:30,http://beer.com/marcelle.zieme,,113.210.246.44,"{""location"": ""GB"", ""is_mobile"": false}" 22649,8,508,2017-05-31 03:18:18,http://keeblerbogisich.net/trenton,,94.92.179.195,"{""location"": ""PF"", ""is_mobile"": false}" 22650,8,508,2017-05-02 22:03:15,http://bayerzulauf.org/filomena_lubowitz,,180.161.190.22,"{""location"": ""FO"", ""is_mobile"": false}" 22651,8,508,2017-06-08 07:50:43,http://oberbrunner.co/giles.casper,,236.18.107.12,"{""location"": ""KZ"", ""is_mobile"": true}" 22652,8,508,2016-12-21 01:49:42,http://franeckijakubowski.biz/elsa_klocko,,109.68.236.159,"{""location"": ""CU"", ""is_mobile"": false}" 22653,8,508,2017-01-06 05:58:31,http://nolan.com/davon,,101.168.55.116,"{""location"": ""DO"", ""is_mobile"": true}" 22654,8,508,2016-12-24 15:39:12,http://mertzdavis.name/lance_romaguera,,48.33.116.180,"{""location"": ""MU"", ""is_mobile"": true}" 22655,8,508,2017-03-11 16:11:53,http://abbott.org/cecil_hamill,,55.58.193.115,"{""location"": ""AU"", ""is_mobile"": true}" 22656,8,508,2017-03-04 00:42:05,http://reillynolan.org/leanna,,203.126.4.192,"{""location"": ""BM"", ""is_mobile"": false}" 22657,8,508,2017-05-01 17:34:57,http://toy.org/jaleel.kerluke,,138.166.68.53,"{""location"": ""BV"", ""is_mobile"": true}" 22658,8,508,2017-03-16 08:34:29,http://terry.name/hubert_maggio,,117.122.24.92,"{""location"": ""MC"", ""is_mobile"": true}" 22659,8,508,2017-02-26 06:40:41,http://bernierbeer.com/guie.braun,,39.86.47.114,"{""location"": ""MM"", ""is_mobile"": false}" 22660,8,508,2017-04-30 19:21:45,http://sanford.co/melya_hoppe,,219.84.236.155,"{""location"": ""CD"", ""is_mobile"": false}" 22661,8,508,2017-03-31 16:35:20,http://beer.org/neal,,88.195.196.251,"{""location"": ""DZ"", ""is_mobile"": false}" 1581,1,36,2017-04-30 08:47:22,http://hodkiewicz.org/maximillia_schuster,,85.121.215.213,"{""location"": ""DO"", ""is_mobile"": true}" 1582,1,36,2017-04-13 05:36:59,http://bartellchristiansen.net/maverick.fay,,64.152.24.50,"{""location"": ""PM"", ""is_mobile"": true}" 1583,1,36,2017-04-01 03:54:48,http://franeckigrady.biz/delilah,,58.45.21.207,"{""location"": ""DE"", ""is_mobile"": false}" 1584,1,36,2017-01-12 05:36:13,http://batz.co/sonya,,114.194.148.69,"{""location"": ""TL"", ""is_mobile"": false}" 1585,1,36,2017-05-23 07:03:03,http://huel.co/fabian,,220.115.126.67,"{""location"": ""UY"", ""is_mobile"": false}" 1586,1,36,2017-02-20 12:40:25,http://huels.com/demond_schowalter,,240.34.139.215,"{""location"": ""IO"", ""is_mobile"": true}" 1587,1,36,2017-05-30 08:13:43,http://predovic.org/dexter.ziemann,,237.223.135.138,"{""location"": ""WF"", ""is_mobile"": true}" 1588,1,36,2017-02-21 09:31:48,http://ledner.biz/fritz.beahan,,249.132.86.10,"{""location"": ""PF"", ""is_mobile"": false}" 1589,1,36,2017-03-15 20:13:53,http://cummerata.info/noemie_ledner,,114.221.34.202,"{""location"": ""NZ"", ""is_mobile"": false}" 1590,1,36,2017-05-27 19:21:42,http://dach.name/margot,,39.240.94.9,"{""location"": ""BG"", ""is_mobile"": true}" 1591,1,36,2017-05-19 14:07:37,http://ondricka.com/shanel.lebsack,,121.137.251.47,"{""location"": ""AZ"", ""is_mobile"": false}" 1592,1,36,2017-01-17 15:11:17,http://sipescremin.biz/stuart,,173.216.168.73,"{""location"": ""GQ"", ""is_mobile"": true}" 1593,1,36,2017-04-05 12:23:29,http://grantbauch.name/madilyn,,226.178.229.219,"{""location"": ""LU"", ""is_mobile"": true}" 1594,1,36,2017-03-31 10:28:31,http://mckenzie.net/mable_crist,,130.72.99.26,"{""location"": ""BJ"", ""is_mobile"": true}" 1595,1,36,2017-03-06 18:36:25,http://schmidt.com/margie_welch,,29.36.15.186,"{""location"": ""SX"", ""is_mobile"": false}" 1596,1,36,2017-05-20 19:12:36,http://walkerlegros.name/ernesto,,5.219.171.61,"{""location"": ""SJ"", ""is_mobile"": true}" 1597,1,36,2017-06-04 04:37:27,http://paucek.io/floy,,140.168.68.241,"{""location"": ""VA"", ""is_mobile"": false}" 1598,1,36,2017-01-13 06:17:35,http://bartolettiledner.biz/dayna.bailey,,249.191.28.23,"{""location"": ""TK"", ""is_mobile"": false}" 1599,1,36,2016-12-15 08:43:40,http://terry.info/alvina_gulgowski,,17.55.250.249,"{""location"": ""EC"", ""is_mobile"": true}" 1600,1,36,2017-01-01 05:29:55,http://hermiston.net/rosetta_hermiston,,240.182.153.72,"{""location"": ""SC"", ""is_mobile"": true}" 1601,1,36,2017-01-15 14:27:16,http://bartell.biz/jamal.quigley,,60.50.110.155,"{""location"": ""DZ"", ""is_mobile"": true}" 1602,1,36,2017-03-14 07:07:12,http://crist.info/amalia.frami,,178.33.130.165,"{""location"": ""WF"", ""is_mobile"": false}" 1603,1,36,2017-06-13 19:04:47,http://boehm.net/horace,,237.251.225.144,"{""location"": ""GY"", ""is_mobile"": true}" 1604,1,36,2017-04-17 13:19:51,http://stehr.name/oleta_pagac,,232.211.218.181,"{""location"": ""ID"", ""is_mobile"": false}" 1605,1,36,2017-03-28 22:58:05,http://pollich.org/cleo_emard,,194.55.8.74,"{""location"": ""PW"", ""is_mobile"": false}" 1606,1,36,2017-04-13 17:24:14,http://mcglynn.com/cecelia.jast,,249.115.7.173,"{""location"": ""LY"", ""is_mobile"": false}" 1607,1,36,2017-01-18 21:02:32,http://ernser.io/blanche,,26.57.15.169,"{""location"": ""SL"", ""is_mobile"": true}" 1608,1,36,2017-01-13 18:46:55,http://kuhnharris.io/alayna,,156.132.49.194,"{""location"": ""KN"", ""is_mobile"": false}" 1609,1,36,2017-04-24 02:13:47,http://fritsch.org/jayne.schulist,,159.20.150.203,"{""location"": ""SE"", ""is_mobile"": false}" 1610,1,36,2017-02-14 06:07:48,http://kling.biz/tyree_white,,153.215.149.248,"{""location"": ""BQ"", ""is_mobile"": false}" 1611,1,36,2017-03-08 15:27:10,http://nitzscheharber.name/keara.feil,,115.37.239.81,"{""location"": ""CM"", ""is_mobile"": true}" 1612,1,36,2017-01-31 14:41:33,http://considine.info/nikolas_cruickshank,,193.170.26.184,"{""location"": ""NA"", ""is_mobile"": false}" 1613,1,36,2017-04-20 02:14:38,http://rowemante.co/tony_pacocha,,25.251.205.155,"{""location"": ""BQ"", ""is_mobile"": false}" 1614,1,36,2017-06-01 22:17:18,http://langosh.org/miouri_ko,,234.199.239.10,"{""location"": ""TR"", ""is_mobile"": true}" 1615,1,36,2017-04-18 09:35:50,http://haleyferry.biz/kian,,227.204.36.7,"{""location"": ""MO"", ""is_mobile"": false}" 1616,1,36,2017-06-02 08:30:07,http://rempel.io/abigale_deckow,,160.194.58.164,"{""location"": ""NR"", ""is_mobile"": true}" 1617,1,36,2017-02-28 07:06:44,http://terry.info/lillie,,167.56.225.32,"{""location"": ""MK"", ""is_mobile"": true}" 1618,1,37,2017-03-05 00:32:55,http://swaniawski.biz/gilbert,,26.212.89.25,"{""location"": ""PT"", ""is_mobile"": false}" 1619,1,37,2017-01-01 01:54:12,http://keler.net/camryn_hermann,,234.171.151.219,"{""location"": ""PE"", ""is_mobile"": true}" 1620,1,37,2017-01-22 17:23:04,http://okuneva.io/agustin_williamson,,216.141.12.173,"{""location"": ""AZ"", ""is_mobile"": true}" 1621,1,37,2017-04-25 01:56:56,http://murazik.io/fanny.kunze,,10.211.29.57,"{""location"": ""QA"", ""is_mobile"": true}" 1622,1,37,2017-06-06 17:36:01,http://pollich.co/jan_kohler,,188.167.254.19,"{""location"": ""CK"", ""is_mobile"": false}" 1623,1,37,2017-04-30 00:22:43,http://keeling.biz/ila,,232.99.92.5,"{""location"": ""CC"", ""is_mobile"": true}" 1624,1,37,2017-03-04 12:57:34,http://boyleschroeder.biz/diamond.deckow,,56.49.231.110,"{""location"": ""GY"", ""is_mobile"": false}" 1625,1,37,2017-05-28 09:36:50,http://pagac.biz/lelah_bruen,,41.17.250.50,"{""location"": ""SJ"", ""is_mobile"": true}" 1626,1,37,2017-05-02 00:25:21,http://kerluke.net/heidi,,74.202.143.227,"{""location"": ""BV"", ""is_mobile"": true}" 1627,1,37,2017-04-24 09:26:48,http://herzogwelch.name/guy_parisian,,201.166.75.157,"{""location"": ""BZ"", ""is_mobile"": false}" 1628,1,37,2017-01-07 11:37:30,http://stanton.net/bo_smitham,,252.15.14.127,"{""location"": ""VU"", ""is_mobile"": true}" 1629,1,37,2017-05-15 15:17:08,http://kunze.net/bud,,21.174.145.178,"{""location"": ""RO"", ""is_mobile"": false}" 1630,1,37,2017-01-12 07:50:02,http://shanahan.co/ettie_gleason,,186.245.134.61,"{""location"": ""KP"", ""is_mobile"": false}" 1631,1,37,2017-03-05 11:46:56,http://baumbachbogisich.io/olga,,33.12.117.48,"{""location"": ""GN"", ""is_mobile"": false}" 1632,1,37,2017-02-09 11:39:41,http://faheyharber.com/delbert_schulist,,234.166.230.252,"{""location"": ""TZ"", ""is_mobile"": false}" 1633,1,37,2017-03-09 16:16:42,http://kub.org/gielle,,206.68.69.216,"{""location"": ""SV"", ""is_mobile"": false}" 1634,1,37,2017-02-09 19:22:28,http://willmsrau.info/alphonso.mraz,,220.118.95.7,"{""location"": ""ML"", ""is_mobile"": true}" 1635,1,37,2017-04-08 21:15:45,http://kertzmannsatterfield.name/ashtyn,,82.167.23.184,"{""location"": ""PW"", ""is_mobile"": false}" 1636,1,37,2017-05-31 00:40:53,http://okonernser.co/kaandra_rutherford,,167.40.30.133,"{""location"": ""MR"", ""is_mobile"": true}" 22662,8,508,2017-04-09 22:16:48,http://maggio.biz/darron,,82.61.130.9,"{""location"": ""OM"", ""is_mobile"": true}" 22663,8,508,2017-05-12 20:44:56,http://watsicaleannon.io/gianni,,192.156.227.238,"{""location"": ""CD"", ""is_mobile"": false}" 22664,8,508,2017-04-27 10:31:01,http://jaskolski.io/deontae.jones,,58.195.206.119,"{""location"": ""GI"", ""is_mobile"": false}" 22665,8,508,2017-05-18 11:52:40,http://veum.co/je_mohr,,152.37.71.77,"{""location"": ""HT"", ""is_mobile"": false}" 22666,8,509,2017-03-26 11:50:50,http://brekke.net/elian,,156.171.87.54,"{""location"": ""BB"", ""is_mobile"": true}" 22667,8,509,2017-03-14 00:16:52,http://gaylordshields.info/marquise.wehner,,119.164.118.105,"{""location"": ""ME"", ""is_mobile"": true}" 22668,8,509,2017-03-20 18:26:01,http://howe.info/aurelie.quigley,,206.62.188.110,"{""location"": ""UZ"", ""is_mobile"": false}" 22669,8,509,2017-03-12 08:25:55,http://rowe.name/stevie.hackett,,206.238.79.143,"{""location"": ""MQ"", ""is_mobile"": false}" 22670,8,509,2017-03-16 20:34:48,http://dickens.co/kattie,,210.186.197.143,"{""location"": ""UA"", ""is_mobile"": false}" 22671,8,509,2017-01-29 05:13:24,http://daugherty.io/lauryn.moore,,24.155.195.118,"{""location"": ""GA"", ""is_mobile"": true}" 22672,8,509,2017-05-23 20:38:37,http://gulgowskihegmann.io/antonetta,,192.15.26.2,"{""location"": ""LS"", ""is_mobile"": true}" 22673,8,509,2017-05-02 05:54:33,http://stokes.info/karianne,,33.50.22.113,"{""location"": ""MH"", ""is_mobile"": false}" 22674,8,509,2017-03-02 09:22:42,http://wolff.io/boris.hermiston,,121.230.162.93,"{""location"": ""SS"", ""is_mobile"": false}" 22675,8,509,2017-02-22 04:43:18,http://wildermanhahn.biz/josephine,,125.185.117.169,"{""location"": ""GQ"", ""is_mobile"": true}" 23018,8,516,2016-12-15 15:43:06,http://stracke.name/noe,,153.150.32.32,"{""location"": ""DE"", ""is_mobile"": true}" 22676,8,509,2017-02-14 14:20:56,http://reynoldsgleichner.io/kip_klein,,120.84.146.170,"{""location"": ""AR"", ""is_mobile"": false}" 22677,8,509,2017-05-21 13:14:01,http://white.org/keira,,163.65.70.68,"{""location"": ""SJ"", ""is_mobile"": false}" 22678,8,509,2017-03-27 10:06:42,http://quigleykeeling.net/cielo.mitchell,,99.232.219.18,"{""location"": ""MU"", ""is_mobile"": false}" 22679,8,509,2017-05-09 02:32:36,http://carterstrosin.net/anne,,227.132.51.224,"{""location"": ""MF"", ""is_mobile"": false}" 22680,8,509,2017-03-03 14:58:52,http://beatty.co/brennan,,27.120.204.102,"{""location"": ""TW"", ""is_mobile"": true}" 22681,8,509,2016-12-26 05:30:08,http://schimmel.info/clyde,,241.154.83.152,"{""location"": ""SX"", ""is_mobile"": false}" 22682,8,509,2017-05-31 02:04:50,http://heaneycruickshank.name/hilario,,105.39.150.128,"{""location"": ""JO"", ""is_mobile"": false}" 22683,8,509,2017-04-25 00:55:37,http://bednar.co/leilani_durgan,,170.29.133.76,"{""location"": ""AQ"", ""is_mobile"": true}" 22684,8,509,2017-02-08 10:15:58,http://wisozkjerde.co/kieran_turcotte,,236.239.104.216,"{""location"": ""EH"", ""is_mobile"": true}" 22685,8,509,2017-03-30 21:33:41,http://von.name/laurence,,88.40.187.153,"{""location"": ""BV"", ""is_mobile"": false}" 22686,8,509,2017-02-09 06:16:10,http://wintheiser.biz/maryse.leannon,,254.144.210.100,"{""location"": ""SS"", ""is_mobile"": true}" 22687,8,509,2017-01-08 08:56:02,http://hamill.com/rachelle,,231.204.88.249,"{""location"": ""PM"", ""is_mobile"": false}" 22688,8,509,2016-12-24 22:48:47,http://muller.io/demond.weimann,,159.120.119.97,"{""location"": ""TZ"", ""is_mobile"": false}" 22689,8,509,2017-01-13 04:34:38,http://gutkowski.info/julianne_mertz,,82.131.224.192,"{""location"": ""SX"", ""is_mobile"": true}" 22690,8,509,2016-12-26 05:55:27,http://nicolas.co/ryan_stracke,,157.145.196.154,"{""location"": ""IR"", ""is_mobile"": true}" 22691,8,509,2017-05-22 22:17:50,http://hansen.com/emely,,155.158.130.126,"{""location"": ""ER"", ""is_mobile"": true}" 22692,8,509,2017-04-27 01:48:44,http://bernhard.co/westley,,50.11.235.171,"{""location"": ""AS"", ""is_mobile"": false}" 22693,8,509,2017-04-22 02:46:52,http://douglas.org/mateo_murphy,,54.254.68.220,"{""location"": ""EC"", ""is_mobile"": true}" 22694,8,509,2017-06-08 22:12:38,http://bayer.net/gregory,,185.78.162.198,"{""location"": ""JO"", ""is_mobile"": true}" 22695,8,509,2017-03-04 12:47:28,http://koelpin.info/elisha,,201.40.208.243,"{""location"": ""HM"", ""is_mobile"": true}" 22696,8,509,2017-02-28 19:33:35,http://swaniawskistamm.com/corrine.kuphal,,170.144.120.147,"{""location"": ""TO"", ""is_mobile"": false}" 22697,8,509,2016-12-18 02:54:33,http://walker.co/blanca_boehm,,64.18.196.54,"{""location"": ""ME"", ""is_mobile"": true}" 22698,8,509,2017-02-09 11:23:30,http://toyfisher.org/ruthie,,157.178.32.92,"{""location"": ""CM"", ""is_mobile"": false}" 22699,8,509,2017-05-08 00:42:28,http://walsh.io/jordon_conn,,18.145.175.193,"{""location"": ""BM"", ""is_mobile"": false}" 22700,8,509,2017-04-11 00:35:32,http://koelpin.org/dorothea,,182.145.4.235,"{""location"": ""BR"", ""is_mobile"": false}" 22701,8,509,2017-05-18 06:27:58,http://goyettecollins.co/dave,,160.130.107.6,"{""location"": ""AW"", ""is_mobile"": false}" 22702,8,509,2017-06-01 10:53:10,http://sporer.co/elise,,166.64.42.69,"{""location"": ""VN"", ""is_mobile"": true}" 22703,8,509,2016-12-17 07:59:48,http://windlermaggio.biz/kiarra_daugherty,,105.231.135.187,"{""location"": ""CV"", ""is_mobile"": false}" 22704,8,509,2017-02-19 09:22:46,http://toy.name/zella,,104.179.64.163,"{""location"": ""EE"", ""is_mobile"": true}" 22705,8,509,2017-01-24 11:12:08,http://ondricka.org/noble_dubuque,,54.187.239.84,"{""location"": ""SH"", ""is_mobile"": false}" 22706,8,509,2017-04-16 03:36:20,http://terry.info/levi,,104.125.199.231,"{""location"": ""AR"", ""is_mobile"": true}" 22707,8,509,2017-02-08 08:11:45,http://anderson.info/tristin.hamill,,59.254.238.82,"{""location"": ""CU"", ""is_mobile"": true}" 22708,8,509,2017-06-09 02:05:55,http://greenholt.com/nia,,128.54.32.152,"{""location"": ""NE"", ""is_mobile"": true}" 22709,8,509,2017-01-14 22:19:21,http://simonis.com/jodie,,223.224.42.170,"{""location"": ""BS"", ""is_mobile"": true}" 22710,8,509,2017-03-06 04:51:20,http://runolfsdottir.info/jakob,,166.160.24.180,"{""location"": ""DM"", ""is_mobile"": false}" 22711,8,509,2016-12-13 15:55:23,http://quitzonswaniawski.name/quinton_murray,,224.9.209.183,"{""location"": ""CI"", ""is_mobile"": true}" 22712,8,509,2017-05-29 06:50:34,http://nicolashagenes.com/fidel_yundt,,93.18.230.133,"{""location"": ""PR"", ""is_mobile"": false}" 22713,8,509,2017-02-26 17:40:12,http://buckridgefeest.name/julius,,177.164.210.234,"{""location"": ""PY"", ""is_mobile"": false}" 22714,8,509,2017-03-27 17:10:50,http://yostschmeler.info/keenan.kris,,42.102.188.97,"{""location"": ""HU"", ""is_mobile"": false}" 22715,8,509,2017-02-05 01:22:04,http://carroll.info/corrine,,130.18.240.112,"{""location"": ""PS"", ""is_mobile"": true}" 1637,1,37,2017-01-23 15:36:56,http://blick.name/aiden_kuhlman,,252.211.46.206,"{""location"": ""LU"", ""is_mobile"": false}" 1638,1,37,2016-12-29 18:38:32,http://ruecker.io/rolando,,128.124.105.187,"{""location"": ""BQ"", ""is_mobile"": false}" 1639,1,37,2017-02-23 11:34:46,http://feeney.biz/jamar,,120.216.249.120,"{""location"": ""NC"", ""is_mobile"": true}" 1640,1,37,2017-05-30 06:20:26,http://heaney.co/maxwell,,220.254.79.181,"{""location"": ""PN"", ""is_mobile"": true}" 1641,1,37,2017-02-02 03:03:12,http://ornosinski.net/je,,36.62.120.142,"{""location"": ""CG"", ""is_mobile"": true}" 1642,1,37,2017-04-02 18:29:27,http://schmitt.io/sandra,,39.139.172.248,"{""location"": ""AF"", ""is_mobile"": false}" 1643,1,37,2017-03-12 21:51:45,http://okeefe.io/wilfredo.emmerich,,155.154.130.200,"{""location"": ""VI"", ""is_mobile"": false}" 1644,1,37,2017-03-15 07:03:12,http://weimann.co/orville_kertzmann,,6.213.52.55,"{""location"": ""RU"", ""is_mobile"": false}" 1645,1,37,2017-02-21 18:08:52,http://rice.io/mandy_erdman,,151.127.128.203,"{""location"": ""NE"", ""is_mobile"": true}" 1646,1,37,2017-04-14 14:45:35,http://quitzon.info/je,,238.138.229.86,"{""location"": ""BL"", ""is_mobile"": true}" 1647,1,37,2017-05-21 18:11:34,http://bednar.net/electa,,160.120.96.162,"{""location"": ""EG"", ""is_mobile"": false}" 1648,1,37,2017-02-12 00:43:12,http://ko.name/amy,,61.202.4.235,"{""location"": ""JE"", ""is_mobile"": true}" 1649,1,37,2017-05-24 20:28:07,http://moore.io/camren,,138.155.133.229,"{""location"": ""TH"", ""is_mobile"": true}" 1650,1,37,2017-04-24 12:37:22,http://spinka.name/jose,,234.52.151.227,"{""location"": ""PS"", ""is_mobile"": false}" 1651,1,37,2017-04-06 12:52:39,http://gottlieb.org/vincenzo,,152.26.241.60,"{""location"": ""QA"", ""is_mobile"": false}" 1652,1,37,2017-06-12 05:44:46,http://sporerbrekke.name/gust.hansen,,116.117.160.141,"{""location"": ""CY"", ""is_mobile"": false}" 1653,1,37,2017-03-03 14:00:15,http://gorczanyhansen.io/carmel,,95.242.94.65,"{""location"": ""ET"", ""is_mobile"": false}" 1654,1,37,2016-12-18 20:40:54,http://faheywalsh.net/alva,,246.123.199.185,"{""location"": ""NO"", ""is_mobile"": true}" 1655,1,37,2017-04-11 22:01:39,http://powlowskiohara.org/kari.nader,,215.124.163.195,"{""location"": ""TH"", ""is_mobile"": true}" 1656,1,37,2017-04-09 21:40:35,http://bauch.net/adonis_hermiston,,186.109.222.26,"{""location"": ""GF"", ""is_mobile"": false}" 1657,1,37,2017-03-22 16:59:59,http://connstroman.info/magnolia,,233.2.156.103,"{""location"": ""LA"", ""is_mobile"": true}" 1658,1,37,2017-04-03 09:26:28,http://quitzonlebsack.name/joy,,195.237.251.231,"{""location"": ""BB"", ""is_mobile"": false}" 1659,1,37,2016-12-30 12:00:31,http://turnernader.com/tevin_ullrich,,101.126.139.178,"{""location"": ""NG"", ""is_mobile"": true}" 1660,1,37,2016-12-23 18:03:57,http://hudsonlegros.info/raleigh,,3.159.228.202,"{""location"": ""MO"", ""is_mobile"": true}" 1661,1,37,2017-01-21 12:06:56,http://kunze.info/cody_hudson,,90.226.190.118,"{""location"": ""EC"", ""is_mobile"": false}" 1662,1,37,2017-03-22 01:10:41,http://kozey.name/astrid.wisoky,,208.147.117.99,"{""location"": ""LR"", ""is_mobile"": true}" 1663,1,37,2016-12-20 09:09:39,http://cole.co/noemi,,182.222.145.182,"{""location"": ""TC"", ""is_mobile"": true}" 1664,1,37,2016-12-29 15:50:37,http://cummings.org/martine,,182.224.132.125,"{""location"": ""DM"", ""is_mobile"": true}" 1665,1,37,2017-02-10 04:38:36,http://schmitt.info/geoffrey,,111.9.12.233,"{""location"": ""AD"", ""is_mobile"": true}" 1666,1,37,2017-04-03 17:27:20,http://moriettejacobs.name/joanny.jones,,226.168.192.144,"{""location"": ""BW"", ""is_mobile"": false}" 1667,1,37,2017-03-01 04:23:12,http://crookslangosh.info/eduardo_douglas,,168.142.80.16,"{""location"": ""CW"", ""is_mobile"": false}" 1668,1,37,2017-01-02 05:44:20,http://ziemann.org/houston,,148.174.237.154,"{""location"": ""TZ"", ""is_mobile"": false}" 1669,1,37,2017-01-08 15:53:40,http://vandervort.co/talia_smith,,31.123.33.178,"{""location"": ""GY"", ""is_mobile"": true}" 1670,1,37,2017-04-04 16:13:03,http://kunde.net/adriana_batz,,114.89.163.241,"{""location"": ""SE"", ""is_mobile"": true}" 1671,1,37,2017-05-13 08:13:32,http://lynch.com/mozelle.rowe,,44.170.192.101,"{""location"": ""ET"", ""is_mobile"": true}" 1672,1,37,2017-06-14 02:30:53,http://lehnerkautzer.info/isabel,,23.242.41.92,"{""location"": ""ES"", ""is_mobile"": false}" 1673,1,37,2017-02-20 22:31:50,http://mayerleuschke.co/shane.damore,,11.207.89.170,"{""location"": ""HN"", ""is_mobile"": true}" 1674,1,37,2017-03-04 05:03:49,http://lynchheidenreich.co/keely.dare,,176.44.174.166,"{""location"": ""ML"", ""is_mobile"": true}" 1675,1,37,2017-05-30 09:06:50,http://goldner.info/zechariah.abernathy,,174.17.69.226,"{""location"": ""HT"", ""is_mobile"": true}" 1676,1,37,2017-04-14 20:08:58,http://wolfadams.info/chester_connelly,,34.189.155.19,"{""location"": ""DZ"", ""is_mobile"": true}" 2194,1,49,2017-02-15 12:20:31,http://oconnell.io/glen,,54.179.196.206,"{""location"": ""WS"", ""is_mobile"": true}" 1677,1,37,2016-12-17 05:08:44,http://pfefferhowe.name/zula.lockman,,196.91.105.198,"{""location"": ""FJ"", ""is_mobile"": false}" 1678,1,37,2017-01-22 10:24:28,http://eichmann.org/birdie.hilpert,,79.70.16.122,"{""location"": ""GF"", ""is_mobile"": false}" 1679,1,38,2017-05-06 09:46:44,http://conroy.com/alexis_haag,,15.115.69.239,"{""location"": ""GQ"", ""is_mobile"": false}" 1680,1,38,2016-12-13 06:41:28,http://jones.info/sophia.bradtke,,177.19.141.108,"{""location"": ""GG"", ""is_mobile"": true}" 1681,1,38,2017-04-23 01:39:20,http://schuppeheel.biz/lambert_feest,,166.96.234.171,"{""location"": ""AT"", ""is_mobile"": true}" 1682,1,38,2017-04-18 18:57:30,http://bradtke.io/vesta,,106.238.29.62,"{""location"": ""VE"", ""is_mobile"": false}" 1683,1,38,2017-03-09 14:19:14,http://vonrueden.co/eryn,,171.219.151.17,"{""location"": ""LT"", ""is_mobile"": false}" 1684,1,38,2017-02-05 22:27:46,http://hoeger.io/gabe,,63.240.153.165,"{""location"": ""LI"", ""is_mobile"": true}" 1685,1,38,2017-04-27 18:07:31,http://schroeder.io/ayden.barton,,202.214.134.60,"{""location"": ""BQ"", ""is_mobile"": false}" 1686,1,38,2017-05-01 10:45:53,http://spencer.net/elmira.king,,34.18.85.99,"{""location"": ""TT"", ""is_mobile"": false}" 1687,1,38,2017-04-24 16:37:55,http://sipeskertzmann.info/savanna,,12.80.104.197,"{""location"": ""TH"", ""is_mobile"": true}" 1688,1,38,2017-01-16 03:45:27,http://mohr.info/lawrence,,152.28.230.227,"{""location"": ""CG"", ""is_mobile"": false}" 1689,1,38,2017-02-10 00:14:06,http://ward.info/jaydon,,30.180.206.96,"{""location"": ""GG"", ""is_mobile"": true}" 1690,1,38,2017-05-25 03:28:21,http://westdibbert.biz/elinore,,252.220.104.2,"{""location"": ""SJ"", ""is_mobile"": false}" 1691,1,38,2016-12-26 21:02:17,http://bartell.info/marianna,,138.70.254.156,"{""location"": ""PF"", ""is_mobile"": false}" 1692,1,38,2017-02-13 10:33:39,http://okeefe.io/evan_ebert,,113.161.151.223,"{""location"": ""VA"", ""is_mobile"": true}" 22716,8,509,2017-03-02 07:21:22,http://keebler.info/karson,,200.42.157.40,"{""location"": ""SB"", ""is_mobile"": false}" 22717,8,509,2017-03-10 17:19:20,http://olson.net/rogers,,201.38.143.97,"{""location"": ""CK"", ""is_mobile"": false}" 22718,8,509,2017-02-03 02:55:27,http://shanahankaulke.name/kianna.walter,,127.79.28.32,"{""location"": ""ES"", ""is_mobile"": true}" 22719,8,509,2017-05-10 14:40:07,http://binsnienow.info/shea,,146.170.196.197,"{""location"": ""LK"", ""is_mobile"": true}" 22720,8,509,2016-12-31 17:15:43,http://reilly.org/jeromy,,193.35.92.138,"{""location"": ""SD"", ""is_mobile"": true}" 22721,8,510,2017-01-17 09:52:08,http://feil.name/coy_schaden,,143.228.38.33,"{""location"": ""ZA"", ""is_mobile"": false}" 22722,8,510,2017-01-24 17:19:14,http://kovacek.name/loyce_mcclure,,195.172.227.194,"{""location"": ""BQ"", ""is_mobile"": true}" 22723,8,510,2017-04-20 03:51:29,http://breitenberg.net/viviane,,193.242.39.249,"{""location"": ""GI"", ""is_mobile"": false}" 22724,8,510,2017-05-18 07:01:08,http://macgyver.biz/angeline,,70.65.132.21,"{""location"": ""JE"", ""is_mobile"": true}" 22725,8,510,2017-05-17 13:15:29,http://mayertromaguera.co/jonathon_leffler,,110.183.187.53,"{""location"": ""TK"", ""is_mobile"": true}" 22726,8,510,2017-02-07 02:57:15,http://dickenslarson.info/reina,,99.204.235.81,"{""location"": ""KZ"", ""is_mobile"": false}" 22727,8,510,2017-03-02 22:25:32,http://torphybartoletti.io/merlin,,209.34.215.245,"{""location"": ""IR"", ""is_mobile"": false}" 22728,8,510,2017-05-28 21:35:41,http://bartoletti.net/darrion.rice,,110.129.16.120,"{""location"": ""WS"", ""is_mobile"": true}" 22729,8,510,2017-06-05 19:23:51,http://murazikmckenzie.co/tiara,,69.79.79.117,"{""location"": ""BW"", ""is_mobile"": false}" 22730,8,510,2017-01-06 08:38:37,http://kulasmcglynn.org/ethan,,121.157.60.9,"{""location"": ""FJ"", ""is_mobile"": true}" 22731,8,510,2017-01-12 02:02:14,http://doyle.io/kenton_heel,,9.153.186.160,"{""location"": ""UZ"", ""is_mobile"": false}" 22732,8,510,2017-01-22 14:39:35,http://larkinhammes.com/desmond,,3.72.175.29,"{""location"": ""IS"", ""is_mobile"": false}" 22733,8,510,2017-05-24 03:06:47,http://sauer.com/schuyler.hartmann,,132.86.6.42,"{""location"": ""UZ"", ""is_mobile"": true}" 22734,8,510,2016-12-24 21:43:43,http://reichert.org/ashlee.fadel,,202.126.2.97,"{""location"": ""NG"", ""is_mobile"": false}" 22735,8,510,2017-03-02 20:18:58,http://dickens.org/aliya,,128.80.57.154,"{""location"": ""KW"", ""is_mobile"": true}" 22736,8,510,2017-03-30 10:27:47,http://smitham.net/jalyn_green,,9.65.73.234,"{""location"": ""AM"", ""is_mobile"": false}" 22737,8,510,2017-05-06 22:41:47,http://sporerreynolds.io/chaim,,66.129.121.40,"{""location"": ""VE"", ""is_mobile"": false}" 22738,8,510,2017-04-03 08:53:04,http://daniel.io/mackenzie.lakin,,32.92.65.58,"{""location"": ""MO"", ""is_mobile"": true}" 22739,8,510,2017-02-26 00:41:02,http://mrazemmerich.net/darian_hills,,26.132.144.225,"{""location"": ""US"", ""is_mobile"": false}" 22740,8,510,2017-01-11 07:53:10,http://nitzsche.biz/isobel.emmerich,,239.119.228.198,"{""location"": ""LC"", ""is_mobile"": false}" 22741,8,510,2017-02-10 12:56:04,http://lueilwitz.name/vernie.torphy,,169.126.195.34,"{""location"": ""GU"", ""is_mobile"": true}" 22742,8,510,2016-12-23 08:56:26,http://kovacek.biz/brielle_mcdermott,,188.71.204.42,"{""location"": ""SL"", ""is_mobile"": false}" 22743,8,510,2017-01-01 10:38:56,http://moen.co/claudia,,229.250.73.115,"{""location"": ""LA"", ""is_mobile"": false}" 22744,8,510,2017-01-09 00:29:28,http://schimmel.co/aliza_donnelly,,165.172.91.34,"{""location"": ""BQ"", ""is_mobile"": false}" 22745,8,510,2017-03-13 20:50:39,http://stoltenberg.co/stan,,19.138.242.34,"{""location"": ""NG"", ""is_mobile"": true}" 22746,8,510,2017-01-14 08:22:46,http://eichmann.biz/audra,,166.245.55.127,"{""location"": ""FR"", ""is_mobile"": false}" 22747,8,510,2017-05-19 08:56:38,http://von.co/guy_schaefer,,77.46.158.126,"{""location"": ""ZA"", ""is_mobile"": false}" 22748,8,510,2017-05-21 20:51:05,http://glover.biz/layne.kihn,,254.150.95.100,"{""location"": ""FJ"", ""is_mobile"": true}" 22749,8,510,2016-12-27 16:34:09,http://dare.com/kaia,,43.230.185.98,"{""location"": ""MH"", ""is_mobile"": true}" 22750,8,510,2017-03-28 05:23:00,http://ferry.io/abdullah,,86.65.30.194,"{""location"": ""SK"", ""is_mobile"": true}" 22751,8,510,2017-01-22 03:29:16,http://gerlachwhite.com/samantha.mcclure,,47.239.211.15,"{""location"": ""SR"", ""is_mobile"": false}" 22752,8,510,2017-01-27 07:13:15,http://greenfelder.info/august,,215.239.198.204,"{""location"": ""NC"", ""is_mobile"": true}" 22753,8,510,2017-05-11 21:32:36,http://hahngrant.info/margarette,,4.192.136.23,"{""location"": ""GH"", ""is_mobile"": false}" 22754,8,510,2017-03-26 06:02:18,http://mcclurenicolas.biz/gerard.greenfelder,,40.211.215.132,"{""location"": ""SG"", ""is_mobile"": true}" 22755,8,510,2017-06-01 17:29:32,http://daughertydare.org/reinhold.ondricka,,131.168.62.32,"{""location"": ""TW"", ""is_mobile"": true}" 22756,8,510,2017-04-28 18:17:31,http://gerholdupton.net/mckenna,,244.179.223.109,"{""location"": ""LT"", ""is_mobile"": false}" 22757,8,510,2016-12-24 05:43:19,http://price.name/nikita,,33.137.235.152,"{""location"": ""SH"", ""is_mobile"": false}" 22758,8,510,2016-12-13 18:55:19,http://romaguera.info/barry,,27.134.243.121,"{""location"": ""GU"", ""is_mobile"": true}" 22759,8,510,2017-01-17 11:57:36,http://mccullough.io/renee_luettgen,,11.11.124.190,"{""location"": ""AF"", ""is_mobile"": false}" 22760,8,510,2016-12-19 08:35:03,http://jaskolskiquitzon.biz/brad,,202.180.129.4,"{""location"": ""PN"", ""is_mobile"": true}" 22761,8,510,2017-01-03 20:20:12,http://grimes.name/izaiah,,247.101.98.148,"{""location"": ""EG"", ""is_mobile"": true}" 22762,8,510,2017-01-25 18:55:39,http://koelpinhowell.name/jerome,,92.252.38.3,"{""location"": ""PH"", ""is_mobile"": false}" 22763,8,510,2017-01-08 04:27:27,http://stanton.com/erna,,202.105.102.90,"{""location"": ""TC"", ""is_mobile"": true}" 22764,8,510,2017-03-31 04:59:03,http://wolff.info/christopher,,157.103.117.149,"{""location"": ""CO"", ""is_mobile"": false}" 22765,8,510,2017-05-09 15:00:35,http://kuhlman.net/neal,,132.202.118.38,"{""location"": ""NL"", ""is_mobile"": false}" 22766,8,510,2017-06-14 01:58:32,http://stamm.org/fredy_prosacco,,89.183.251.251,"{""location"": ""RU"", ""is_mobile"": true}" 22767,8,510,2017-02-19 18:23:30,http://bartolettiemmerich.co/berta,,112.41.228.119,"{""location"": ""BH"", ""is_mobile"": false}" 22768,8,510,2017-02-24 21:39:38,http://davis.io/meta,,124.104.55.226,"{""location"": ""AF"", ""is_mobile"": true}" 22769,8,510,2017-04-12 15:00:04,http://gottliebschmidt.io/zita_deckow,,156.154.42.98,"{""location"": ""GG"", ""is_mobile"": false}" 22770,8,510,2016-12-28 04:29:35,http://runolfsdottir.io/katarina,,209.13.116.199,"{""location"": ""PG"", ""is_mobile"": false}" 1693,1,38,2017-02-05 16:21:18,http://mclaughlin.com/joseph.mclaughlin,,250.149.219.44,"{""location"": ""LK"", ""is_mobile"": true}" 1694,1,38,2017-02-22 20:07:58,http://mckenzieruel.name/hazle,,48.5.169.248,"{""location"": ""HR"", ""is_mobile"": false}" 1695,1,38,2017-02-06 09:59:24,http://ziemann.org/alanna,,113.214.152.158,"{""location"": ""KP"", ""is_mobile"": false}" 1696,1,38,2016-12-16 09:21:17,http://wintheiserkozey.biz/gustave.hintz,,187.65.209.4,"{""location"": ""RE"", ""is_mobile"": false}" 1697,1,38,2017-01-27 15:56:04,http://halvorson.io/zella_pfannerstill,,74.151.69.61,"{""location"": ""AQ"", ""is_mobile"": true}" 1698,1,38,2017-05-16 02:10:07,http://krajcik.io/erick,,239.83.99.228,"{""location"": ""TO"", ""is_mobile"": true}" 1699,1,38,2016-12-22 18:09:18,http://walter.org/myrtis,,45.226.216.61,"{""location"": ""SY"", ""is_mobile"": false}" 1700,1,38,2017-05-16 19:50:04,http://ritchie.com/isabella.beer,,162.171.203.235,"{""location"": ""MY"", ""is_mobile"": true}" 1701,1,38,2017-02-13 11:38:57,http://shields.biz/winfield,,86.133.191.205,"{""location"": ""SY"", ""is_mobile"": true}" 1702,1,38,2017-05-15 18:04:42,http://johnson.org/eliane,,113.90.66.27,"{""location"": ""MN"", ""is_mobile"": false}" 1703,1,38,2017-03-01 20:46:42,http://abshire.io/bernita.bergnaum,,107.165.58.51,"{""location"": ""TG"", ""is_mobile"": true}" 1704,1,38,2017-05-07 21:35:59,http://pacocha.biz/rubie,,59.108.206.188,"{""location"": ""RO"", ""is_mobile"": false}" 1705,1,38,2017-03-25 17:36:17,http://wolff.biz/foster.abernathy,,112.62.173.165,"{""location"": ""DJ"", ""is_mobile"": true}" 1706,1,38,2017-03-30 16:35:11,http://spencer.co/eden.gislason,,161.245.79.42,"{""location"": ""FI"", ""is_mobile"": false}" 1707,1,38,2017-02-08 06:36:23,http://koch.com/chasity_fay,,244.50.225.37,"{""location"": ""GM"", ""is_mobile"": false}" 1708,1,38,2017-06-01 03:15:52,http://baumbach.io/candace_towne,,57.87.102.78,"{""location"": ""TH"", ""is_mobile"": false}" 1709,1,38,2017-01-23 17:13:39,http://yundtledner.biz/mason,,44.245.131.40,"{""location"": ""MN"", ""is_mobile"": true}" 1710,1,38,2017-03-06 00:44:36,http://bogisich.co/kurt,,109.87.233.76,"{""location"": ""AX"", ""is_mobile"": true}" 1711,1,38,2016-12-17 01:30:30,http://watsicalesch.org/alexandria,,134.152.3.2,"{""location"": ""LT"", ""is_mobile"": false}" 1712,1,38,2017-02-21 18:03:08,http://sporer.biz/adelia.langworth,,167.167.150.5,"{""location"": ""PL"", ""is_mobile"": true}" 1713,1,38,2017-02-07 09:19:43,http://beier.info/emerson,,252.87.50.11,"{""location"": ""FK"", ""is_mobile"": true}" 1714,1,38,2017-06-13 23:16:23,http://auer.co/kiel_mcglynn,,19.20.34.76,"{""location"": ""GI"", ""is_mobile"": false}" 1715,1,38,2017-05-18 10:42:58,http://cronanitzsche.name/dangelo_heidenreich,,5.132.209.196,"{""location"": ""IN"", ""is_mobile"": false}" 1716,1,38,2017-03-11 19:39:07,http://weinatpfannerstill.biz/opal,,158.227.10.220,"{""location"": ""MS"", ""is_mobile"": true}" 1717,1,38,2017-01-25 17:41:35,http://carter.biz/reggie,,65.206.165.53,"{""location"": ""GI"", ""is_mobile"": true}" 1718,1,38,2017-02-16 02:58:13,http://green.net/crystel_durgan,,12.221.20.21,"{""location"": ""BR"", ""is_mobile"": true}" 1719,1,38,2017-03-26 02:52:39,http://bergnaumhowe.biz/mathilde,,11.236.155.231,"{""location"": ""TZ"", ""is_mobile"": false}" 1720,1,38,2017-04-29 12:49:22,http://mills.info/daija,,61.85.194.30,"{""location"": ""VC"", ""is_mobile"": true}" 1721,1,39,2017-02-20 21:09:49,http://crooks.net/roderick_hermann,,228.194.188.82,"{""location"": ""ST"", ""is_mobile"": false}" 1722,1,39,2017-02-13 21:07:29,http://douglas.co/jarod,,247.205.97.160,"{""location"": ""EC"", ""is_mobile"": true}" 1723,1,39,2017-03-06 23:46:09,http://botsfordgoldner.info/abner_luettgen,,113.10.129.189,"{""location"": ""GT"", ""is_mobile"": true}" 1724,1,39,2017-04-12 01:13:29,http://rodriguezorn.net/violette_larkin,,68.174.129.211,"{""location"": ""MK"", ""is_mobile"": true}" 1725,1,39,2017-05-01 23:51:32,http://wisoky.name/shaylee,,162.110.174.3,"{""location"": ""TK"", ""is_mobile"": true}" 1726,1,39,2017-05-24 07:45:25,http://hauck.info/constance,,42.221.227.216,"{""location"": ""MO"", ""is_mobile"": true}" 1727,1,39,2017-01-08 06:41:23,http://hahn.name/maxine,,117.138.246.76,"{""location"": ""MZ"", ""is_mobile"": false}" 1728,1,39,2017-04-25 01:17:23,http://larkinturcotte.io/donny.glover,,21.183.171.80,"{""location"": ""ZM"", ""is_mobile"": false}" 1729,1,39,2017-01-22 09:41:38,http://franecki.org/abbey.ratke,,60.150.72.124,"{""location"": ""AX"", ""is_mobile"": false}" 1730,1,39,2017-04-12 16:22:05,http://haley.biz/kiarra,,181.52.126.101,"{""location"": ""PL"", ""is_mobile"": true}" 1731,1,39,2016-12-26 12:30:29,http://bednardavis.co/christ,,47.253.43.240,"{""location"": ""MG"", ""is_mobile"": false}" 1732,1,39,2016-12-16 17:37:01,http://abshire.co/brittany.pfeffer,,91.50.70.140,"{""location"": ""TR"", ""is_mobile"": true}" 1733,1,39,2017-02-22 23:39:26,http://boscosteuber.name/rasheed_heidenreich,,34.2.67.252,"{""location"": ""LC"", ""is_mobile"": true}" 1734,1,39,2017-02-09 09:25:45,http://schamberger.co/selina,,127.110.237.58,"{""location"": ""HK"", ""is_mobile"": true}" 1735,1,39,2017-05-03 01:37:59,http://wilkinson.co/lia.blick,,174.150.29.86,"{""location"": ""MY"", ""is_mobile"": true}" 1736,1,39,2017-01-02 17:34:47,http://spinka.org/geovanny,,39.208.79.68,"{""location"": ""FM"", ""is_mobile"": false}" 1737,1,39,2017-01-13 10:49:04,http://dickinson.name/glennie,,174.66.71.52,"{""location"": ""KM"", ""is_mobile"": false}" 1738,1,39,2017-03-27 10:37:53,http://oreilly.com/otilia.cruickshank,,36.57.44.59,"{""location"": ""CF"", ""is_mobile"": true}" 1739,1,39,2017-04-13 01:26:42,http://rath.info/theresa,,47.225.120.84,"{""location"": ""EH"", ""is_mobile"": false}" 1740,1,39,2017-01-01 15:20:35,http://kunze.info/chris,,218.60.19.33,"{""location"": ""CK"", ""is_mobile"": false}" 1741,1,39,2017-05-07 22:19:18,http://hoppe.biz/electa.yost,,252.13.179.103,"{""location"": ""MU"", ""is_mobile"": false}" 1742,1,39,2017-06-08 04:29:08,http://padberg.biz/trea.kertzmann,,147.200.23.57,"{""location"": ""TF"", ""is_mobile"": true}" 1743,1,39,2017-05-08 12:22:05,http://stoltenberg.info/magdalena.satterfield,,253.233.19.253,"{""location"": ""LB"", ""is_mobile"": true}" 1744,1,39,2017-01-04 06:35:13,http://schamberger.org/frederique.parker,,166.177.188.141,"{""location"": ""UZ"", ""is_mobile"": true}" 1745,1,39,2017-03-29 15:25:41,http://farrell.co/mandy_damore,,12.157.113.232,"{""location"": ""BW"", ""is_mobile"": true}" 1746,1,39,2016-12-28 14:56:25,http://bruen.co/stacey,,92.227.208.137,"{""location"": ""JP"", ""is_mobile"": false}" 1747,1,39,2017-02-24 07:51:57,http://stokesheel.io/elias,,2.243.15.174,"{""location"": ""GF"", ""is_mobile"": true}" 1748,1,39,2017-04-17 06:44:35,http://olson.com/tyler_steuber,,252.26.182.51,"{""location"": ""CF"", ""is_mobile"": true}" 22771,8,510,2017-03-06 03:43:50,http://toy.io/manley_bruen,,69.196.231.172,"{""location"": ""GU"", ""is_mobile"": false}" 22772,8,510,2017-04-25 22:35:50,http://ebert.name/josefina.pollich,,123.66.216.247,"{""location"": ""SL"", ""is_mobile"": false}" 22773,8,510,2017-05-22 14:32:32,http://cristzieme.com/heath_mccullough,,89.240.169.16,"{""location"": ""UA"", ""is_mobile"": true}" 22774,8,510,2017-01-26 09:49:22,http://wilkinson.info/antwan.franecki,,94.112.9.139,"{""location"": ""TO"", ""is_mobile"": false}" 22775,8,510,2017-03-10 02:06:05,http://bechtelar.co/jacynthe_schimmel,,227.173.141.106,"{""location"": ""CL"", ""is_mobile"": false}" 22776,8,510,2017-05-27 14:25:01,http://rathmarvin.co/allison,,42.244.198.57,"{""location"": ""GR"", ""is_mobile"": true}" 22777,8,510,2017-06-06 20:32:20,http://gaylord.io/franco_tremblay,,228.185.173.31,"{""location"": ""LR"", ""is_mobile"": true}" 22778,8,510,2017-05-30 02:02:36,http://konopelski.co/davin_friesen,,103.197.251.11,"{""location"": ""CL"", ""is_mobile"": false}" 22779,8,510,2017-04-02 11:44:37,http://gerhold.org/hollie,,251.38.254.200,"{""location"": ""BT"", ""is_mobile"": false}" 22780,8,510,2016-12-21 00:12:48,http://franecki.co/fay.heel,,54.180.117.115,"{""location"": ""TV"", ""is_mobile"": true}" 22781,8,510,2016-12-13 07:50:30,http://gerhold.com/jedidiah_pfannerstill,,249.105.15.78,"{""location"": ""CW"", ""is_mobile"": true}" 22782,8,510,2016-12-31 14:03:07,http://walker.name/garfield_mayert,,51.235.110.190,"{""location"": ""AT"", ""is_mobile"": false}" 22783,8,510,2017-06-05 10:03:40,http://parker.info/gia_macejkovic,,222.213.125.249,"{""location"": ""TG"", ""is_mobile"": false}" 22784,8,510,2017-02-13 15:45:37,http://cummings.net/hertha_schuppe,,251.223.78.96,"{""location"": ""AZ"", ""is_mobile"": false}" 22785,8,510,2017-04-27 19:56:25,http://wolfschamberger.name/alda.mcclure,,150.19.247.22,"{""location"": ""FI"", ""is_mobile"": true}" 22786,8,510,2017-01-05 06:35:23,http://herzog.co/electa.schultz,,43.24.55.228,"{""location"": ""CW"", ""is_mobile"": true}" 22787,8,510,2016-12-22 11:21:35,http://hoppe.name/magdalen,,235.178.36.105,"{""location"": ""WS"", ""is_mobile"": true}" 22788,8,510,2017-05-09 13:26:58,http://bauch.org/stone_huels,,212.214.154.57,"{""location"": ""LS"", ""is_mobile"": false}" 22789,8,510,2017-03-20 02:22:20,http://schuster.name/zoila,,249.208.8.59,"{""location"": ""PN"", ""is_mobile"": true}" 22790,8,511,2017-04-07 18:48:21,http://hackett.org/pascale_watsica,,24.243.195.154,"{""location"": ""CO"", ""is_mobile"": true}" 22791,8,511,2017-04-13 23:49:21,http://mosciski.co/chaz_dickinson,,26.80.109.112,"{""location"": ""CN"", ""is_mobile"": false}" 22792,8,511,2016-12-17 03:22:37,http://macgyver.org/elwin,,22.201.147.221,"{""location"": ""MT"", ""is_mobile"": true}" 22793,8,511,2017-01-01 11:48:04,http://kovacek.biz/creola,,37.187.179.54,"{""location"": ""DM"", ""is_mobile"": false}" 22794,8,511,2017-02-27 14:09:02,http://dickenswiegand.com/emerald_funk,,78.175.97.57,"{""location"": ""BY"", ""is_mobile"": true}" 22795,8,511,2017-01-20 16:32:19,http://hegmann.info/obie_stamm,,163.218.202.120,"{""location"": ""CF"", ""is_mobile"": true}" 22796,8,511,2017-02-20 01:34:17,http://lind.org/veda,,226.122.143.88,"{""location"": ""TO"", ""is_mobile"": true}" 22797,8,511,2017-05-30 17:28:47,http://ondricka.net/katheryn,,47.133.200.234,"{""location"": ""CN"", ""is_mobile"": true}" 22798,8,511,2017-06-01 11:41:15,http://terrypowlowski.io/gerard,,245.119.140.198,"{""location"": ""BR"", ""is_mobile"": false}" 22799,8,511,2017-02-22 03:04:10,http://parisianwolff.io/cortney.sauer,,31.44.151.165,"{""location"": ""ZM"", ""is_mobile"": false}" 22800,8,511,2017-01-15 20:32:09,http://bosco.co/francisca.feeney,,7.74.130.75,"{""location"": ""NE"", ""is_mobile"": true}" 22801,8,511,2017-06-12 13:04:15,http://johnsglover.name/maritza,,154.164.173.120,"{""location"": ""BQ"", ""is_mobile"": false}" 22802,8,511,2017-05-05 23:40:24,http://mcculloughgreenholt.com/ila,,28.16.71.211,"{""location"": ""TN"", ""is_mobile"": true}" 22803,8,511,2017-05-12 12:14:53,http://shanahan.name/kirsten,,144.157.152.250,"{""location"": ""SI"", ""is_mobile"": false}" 22804,8,511,2017-03-26 23:38:56,http://haag.io/wade_turcotte,,24.54.57.9,"{""location"": ""MU"", ""is_mobile"": true}" 22805,8,511,2017-02-23 07:33:43,http://yostbayer.net/hazel_marks,,73.162.84.75,"{""location"": ""IM"", ""is_mobile"": true}" 22806,8,511,2017-04-19 11:15:54,http://hand.net/preston,,172.185.89.125,"{""location"": ""WF"", ""is_mobile"": false}" 22807,8,511,2017-01-22 19:59:02,http://quitzonbernier.org/zoila,,67.71.55.242,"{""location"": ""IQ"", ""is_mobile"": true}" 22808,8,511,2017-03-17 00:16:57,http://bahringer.org/adam,,19.36.64.197,"{""location"": ""GP"", ""is_mobile"": false}" 22809,8,511,2017-04-11 06:05:38,http://wolff.com/hattie,,152.239.172.48,"{""location"": ""DZ"", ""is_mobile"": true}" 22810,8,511,2017-03-09 12:55:00,http://spencer.co/jaeden.ortiz,,21.182.77.220,"{""location"": ""QA"", ""is_mobile"": true}" 22811,8,511,2016-12-20 05:52:40,http://schroederschimmel.name/makayla,,29.172.202.188,"{""location"": ""BZ"", ""is_mobile"": false}" 22812,8,511,2016-12-26 07:05:18,http://ohara.co/elaina,,81.173.245.167,"{""location"": ""VA"", ""is_mobile"": true}" 22813,8,511,2017-02-15 16:24:32,http://orn.org/kathryne_pacocha,,52.65.13.48,"{""location"": ""SY"", ""is_mobile"": false}" 22814,8,511,2017-01-16 18:44:10,http://johnston.info/eleanora.durgan,,70.118.81.151,"{""location"": ""SL"", ""is_mobile"": false}" 22815,8,511,2016-12-15 23:08:02,http://jacobi.biz/paige,,158.172.98.19,"{""location"": ""JP"", ""is_mobile"": false}" 22816,8,511,2017-05-01 12:08:13,http://borerwillms.com/blanca.schiller,,123.199.60.188,"{""location"": ""TT"", ""is_mobile"": true}" 22817,8,511,2017-03-09 18:48:02,http://schaden.name/davion_marvin,,228.56.177.198,"{""location"": ""SL"", ""is_mobile"": false}" 22818,8,511,2017-02-25 16:56:41,http://gislasonborer.com/kailyn_hermiston,,61.45.235.112,"{""location"": ""PK"", ""is_mobile"": false}" 22819,8,511,2017-03-05 10:50:44,http://kovacekwillms.name/favian_wehner,,56.189.223.156,"{""location"": ""DJ"", ""is_mobile"": true}" 22820,8,511,2017-03-06 02:32:35,http://wolffschroeder.net/gwendolyn_marquardt,,211.125.234.128,"{""location"": ""BT"", ""is_mobile"": true}" 22821,8,511,2017-02-16 00:12:03,http://mills.co/vesta,,141.172.56.32,"{""location"": ""AS"", ""is_mobile"": false}" 22822,8,511,2017-02-01 01:03:41,http://vonrueden.net/akeem,,195.145.245.134,"{""location"": ""SY"", ""is_mobile"": false}" 22823,8,511,2017-01-26 11:54:16,http://torphy.name/dayton_okon,,54.167.226.79,"{""location"": ""AM"", ""is_mobile"": true}" 22824,8,511,2017-05-17 10:14:45,http://gleichner.info/nakia_vonrueden,,14.213.25.13,"{""location"": ""GG"", ""is_mobile"": false}" 22825,8,511,2017-04-27 01:45:41,http://volkman.info/genevieve.rohan,,46.218.83.95,"{""location"": ""AU"", ""is_mobile"": false}" 1749,1,39,2017-04-01 11:20:22,http://casper.org/vernice_kaulke,,237.149.207.239,"{""location"": ""AD"", ""is_mobile"": false}" 1750,1,39,2017-03-04 07:52:29,http://marks.name/bertrand.oberbrunner,,96.85.73.213,"{""location"": ""HR"", ""is_mobile"": false}" 1751,1,39,2017-05-08 01:19:25,http://toybraun.org/deie.zulauf,,81.52.192.165,"{""location"": ""BN"", ""is_mobile"": true}" 1752,1,39,2017-04-20 07:01:46,http://hansen.io/guy.ryan,,66.119.167.53,"{""location"": ""PT"", ""is_mobile"": true}" 1753,1,39,2017-02-13 22:19:27,http://kohlerstark.biz/giovani_rosenbaum,,162.199.32.177,"{""location"": ""MD"", ""is_mobile"": false}" 1754,1,39,2017-04-15 18:21:34,http://lehner.biz/florencio_waelchi,,233.116.13.174,"{""location"": ""AR"", ""is_mobile"": false}" 1755,1,39,2017-04-01 02:41:44,http://kemmer.net/kian,,119.109.217.133,"{""location"": ""KN"", ""is_mobile"": false}" 1756,1,39,2017-04-04 07:47:51,http://lakinpowlowski.net/edmond.gulgowski,,47.223.20.100,"{""location"": ""UZ"", ""is_mobile"": true}" 1757,1,39,2017-05-06 17:59:10,http://schmidtjaskolski.info/dorian,,170.138.176.134,"{""location"": ""GY"", ""is_mobile"": true}" 1758,1,39,2017-01-04 21:47:35,http://darekohler.info/burnice,,193.145.247.209,"{""location"": ""SX"", ""is_mobile"": true}" 1759,1,39,2017-05-03 20:55:57,http://daugherty.io/elvis_goodwin,,197.160.115.195,"{""location"": ""GE"", ""is_mobile"": true}" 1760,1,39,2017-06-07 11:16:46,http://king.name/oda,,80.224.146.110,"{""location"": ""SB"", ""is_mobile"": true}" 1761,1,39,2017-05-20 01:33:20,http://kuhnquigley.name/clare,,49.222.204.13,"{""location"": ""RE"", ""is_mobile"": true}" 1762,1,39,2017-02-08 08:23:07,http://howezulauf.info/breana_hayes,,15.57.28.86,"{""location"": ""SM"", ""is_mobile"": false}" 1763,1,39,2017-01-16 01:59:44,http://reilly.biz/oleta.schneider,,111.89.198.226,"{""location"": ""NI"", ""is_mobile"": true}" 1764,1,39,2017-01-19 16:31:07,http://shields.io/maximillian,,254.168.250.26,"{""location"": ""PY"", ""is_mobile"": false}" 1765,1,39,2017-05-07 16:16:56,http://fadelreilly.io/anderson,,31.52.164.60,"{""location"": ""VE"", ""is_mobile"": true}" 1766,1,39,2017-05-30 09:58:49,http://wiza.name/karolann,,34.127.16.132,"{""location"": ""CG"", ""is_mobile"": false}" 1767,1,39,2017-04-04 19:58:31,http://balistreriwindler.org/gladys,,69.133.66.2,"{""location"": ""MQ"", ""is_mobile"": true}" 1768,1,39,2017-02-23 12:36:42,http://okonterry.info/annetta.casper,,209.152.46.186,"{""location"": ""KE"", ""is_mobile"": true}" 1769,1,39,2017-01-30 07:19:27,http://stamm.org/marjorie.kerluke,,118.201.90.182,"{""location"": ""GG"", ""is_mobile"": false}" 1770,1,39,2017-06-03 22:53:51,http://effertz.io/bertha_haley,,227.201.212.243,"{""location"": ""LK"", ""is_mobile"": false}" 1771,1,39,2017-04-06 09:52:02,http://strosin.io/bette_marquardt,,147.2.229.8,"{""location"": ""FO"", ""is_mobile"": true}" 1772,1,39,2017-05-06 03:57:24,http://hamillwitting.info/antonia.halvorson,,192.216.194.228,"{""location"": ""PN"", ""is_mobile"": true}" 1773,1,39,2017-01-31 04:25:23,http://reilly.co/terry,,198.75.11.239,"{""location"": ""VG"", ""is_mobile"": false}" 1774,1,39,2017-05-22 08:12:08,http://barrows.co/clint,,39.106.67.170,"{""location"": ""VG"", ""is_mobile"": true}" 1775,1,39,2017-02-10 02:38:45,http://cole.io/wilburn.haley,,253.8.79.81,"{""location"": ""PM"", ""is_mobile"": false}" 1776,1,39,2017-05-11 10:00:09,http://gulgowski.name/clifton.cummings,,21.244.140.148,"{""location"": ""VN"", ""is_mobile"": false}" 1777,1,39,2017-01-15 22:50:48,http://breitenberg.co/carlie,,183.246.46.144,"{""location"": ""BQ"", ""is_mobile"": true}" 1778,1,39,2017-02-04 06:35:03,http://volkman.co/elwin,,20.123.171.129,"{""location"": ""ET"", ""is_mobile"": true}" 1779,1,40,2017-02-03 01:46:56,http://hettinger.com/emmett.hyatt,,30.137.215.216,"{""location"": ""WF"", ""is_mobile"": false}" 1780,1,40,2017-02-04 07:17:11,http://hayes.info/coby.labadie,,67.103.41.185,"{""location"": ""CK"", ""is_mobile"": true}" 1781,1,40,2017-01-31 06:01:10,http://fritschparker.name/liliane,,182.223.196.96,"{""location"": ""LC"", ""is_mobile"": false}" 1782,1,40,2017-01-17 18:32:57,http://casper.biz/jeramy,,121.129.14.142,"{""location"": ""VA"", ""is_mobile"": false}" 1783,1,40,2017-01-14 19:07:40,http://mills.com/christelle.gibson,,9.54.162.142,"{""location"": ""BV"", ""is_mobile"": false}" 1784,1,40,2017-04-25 20:46:41,http://williamson.name/gertrude,,49.219.207.89,"{""location"": ""PN"", ""is_mobile"": true}" 1785,1,40,2017-02-17 04:47:01,http://nienowoconnell.org/katlyn.roberts,,232.20.173.136,"{""location"": ""VA"", ""is_mobile"": false}" 1786,1,40,2017-03-07 06:05:34,http://waters.org/wallace.damore,,88.38.94.71,"{""location"": ""ML"", ""is_mobile"": true}" 1787,1,40,2017-05-05 19:48:00,http://spencer.com/cheyenne.simonis,,197.82.252.197,"{""location"": ""UM"", ""is_mobile"": true}" 1788,1,40,2017-03-18 07:56:55,http://kinggutkowski.org/pietro_okuneva,,52.121.203.32,"{""location"": ""SV"", ""is_mobile"": true}" 1789,1,40,2017-04-12 21:38:19,http://rippin.net/corene,,101.201.248.66,"{""location"": ""CD"", ""is_mobile"": false}" 1790,1,40,2017-03-28 02:37:10,http://croninmayert.co/hilma,,48.128.169.240,"{""location"": ""RS"", ""is_mobile"": true}" 1791,1,40,2017-04-30 16:17:35,http://marquardtpagac.net/kiera,,51.225.69.228,"{""location"": ""PH"", ""is_mobile"": true}" 1792,1,40,2017-02-09 22:23:45,http://sporer.io/timmothy,,110.250.8.32,"{""location"": ""NO"", ""is_mobile"": true}" 1793,1,40,2017-05-25 11:20:15,http://hoppegutmann.name/destany_kautzer,,31.104.220.240,"{""location"": ""SL"", ""is_mobile"": true}" 1794,1,40,2017-04-16 17:29:17,http://gulgowskismith.com/lillie.volkman,,121.227.133.94,"{""location"": ""CN"", ""is_mobile"": true}" 1795,1,40,2017-01-22 16:46:31,http://daughertyschowalter.org/carmela,,172.142.33.55,"{""location"": ""GW"", ""is_mobile"": true}" 1796,1,40,2017-04-30 01:54:54,http://berge.name/salma,,111.82.211.76,"{""location"": ""SR"", ""is_mobile"": true}" 1797,1,40,2017-03-09 22:36:54,http://schultz.com/gertrude,,44.167.249.27,"{""location"": ""RO"", ""is_mobile"": false}" 1798,1,40,2017-06-06 08:33:05,http://effertzkeeling.org/braden_will,,26.143.105.6,"{""location"": ""VN"", ""is_mobile"": true}" 1799,1,40,2016-12-31 23:52:26,http://kreiger.net/ebony,,154.2.167.176,"{""location"": ""PR"", ""is_mobile"": false}" 1800,1,40,2017-03-12 05:45:43,http://strosinfeeney.co/loren.dooley,,160.122.32.112,"{""location"": ""AE"", ""is_mobile"": true}" 1801,1,40,2017-04-09 16:33:06,http://damoreoberbrunner.name/braden,,26.18.246.114,"{""location"": ""VI"", ""is_mobile"": false}" 1802,1,40,2017-05-20 10:42:07,http://koeppsimonis.net/frieda,,60.46.31.106,"{""location"": ""US"", ""is_mobile"": true}" 1803,1,40,2016-12-29 17:53:58,http://denesik.biz/stephanie,,142.173.86.11,"{""location"": ""PF"", ""is_mobile"": false}" 22826,8,512,2017-04-29 11:35:57,http://batztorp.name/maymie,,151.164.248.221,"{""location"": ""TL"", ""is_mobile"": false}" 22827,8,512,2017-03-14 08:44:51,http://mayerbotsford.info/dexter.heathcote,,249.183.159.193,"{""location"": ""JE"", ""is_mobile"": true}" 22828,8,512,2017-02-26 12:42:09,http://metz.co/gilberto_pouros,,163.115.226.245,"{""location"": ""BT"", ""is_mobile"": true}" 22829,8,512,2016-12-23 13:48:31,http://hills.info/augustine,,97.184.49.244,"{""location"": ""MT"", ""is_mobile"": true}" 22830,8,512,2017-01-30 21:09:19,http://legros.org/trinity,,215.113.115.227,"{""location"": ""RE"", ""is_mobile"": false}" 22831,8,512,2017-06-03 11:00:23,http://greenfeldertowne.org/jay.kozey,,132.52.96.115,"{""location"": ""MY"", ""is_mobile"": false}" 22832,8,512,2017-06-06 10:45:04,http://cronin.com/malinda,,248.54.45.21,"{""location"": ""PE"", ""is_mobile"": true}" 22833,8,512,2017-01-17 13:17:42,http://stroman.com/mortimer,,238.164.250.53,"{""location"": ""NO"", ""is_mobile"": false}" 22834,8,512,2016-12-17 17:45:58,http://okon.com/bernadine.moen,,223.231.138.91,"{""location"": ""SV"", ""is_mobile"": false}" 22835,8,512,2017-01-01 04:16:01,http://satterfieldschneider.net/jamal.dietrich,,39.59.162.155,"{""location"": ""NA"", ""is_mobile"": false}" 22836,8,512,2017-04-30 05:35:18,http://mclaughlin.org/abigale,,215.147.111.160,"{""location"": ""BE"", ""is_mobile"": true}" 22837,8,512,2017-04-14 11:33:46,http://schummstiedemann.info/lia,,35.211.111.113,"{""location"": ""RU"", ""is_mobile"": true}" 22838,8,512,2017-03-06 16:49:31,http://boehm.org/kyra,,218.23.247.79,"{""location"": ""PR"", ""is_mobile"": false}" 22839,8,512,2016-12-13 06:29:54,http://yundt.name/domingo,,20.175.210.139,"{""location"": ""PH"", ""is_mobile"": false}" 22840,8,512,2017-02-28 12:28:18,http://swiftkrajcik.net/hudson.wiegand,,184.59.248.58,"{""location"": ""UA"", ""is_mobile"": false}" 22841,8,512,2017-03-24 06:42:57,http://schmidtrau.biz/sabrina,,172.50.119.105,"{""location"": ""AL"", ""is_mobile"": false}" 22842,8,512,2017-02-12 10:26:41,http://davis.io/hosea,,129.248.183.230,"{""location"": ""SI"", ""is_mobile"": false}" 22843,8,512,2017-05-21 17:27:39,http://reichel.net/lawrence_carroll,,199.45.168.149,"{""location"": ""SS"", ""is_mobile"": true}" 22844,8,512,2017-01-15 09:14:53,http://bayer.co/lexus.beer,,179.156.226.18,"{""location"": ""FI"", ""is_mobile"": true}" 22845,8,512,2017-06-08 09:23:00,http://senger.name/jeanne_oberbrunner,,248.152.32.115,"{""location"": ""IQ"", ""is_mobile"": true}" 22846,8,512,2017-02-18 06:27:15,http://barrows.name/dominic.tromp,,232.88.107.183,"{""location"": ""BS"", ""is_mobile"": false}" 22847,8,512,2017-02-04 21:02:13,http://anderson.io/clarabelle,,199.189.131.135,"{""location"": ""EC"", ""is_mobile"": false}" 22848,8,512,2016-12-16 03:14:29,http://parisian.org/tobin_keeling,,161.87.104.73,"{""location"": ""AU"", ""is_mobile"": false}" 22849,8,512,2017-04-08 08:25:41,http://nicolas.biz/brandyn,,62.70.214.228,"{""location"": ""MD"", ""is_mobile"": false}" 22850,8,512,2017-05-10 06:42:30,http://turnerabshire.org/penelope,,202.162.48.94,"{""location"": ""IQ"", ""is_mobile"": false}" 22851,8,512,2017-01-15 13:20:58,http://ullrich.org/viva,,204.51.217.33,"{""location"": ""KI"", ""is_mobile"": true}" 22852,8,512,2017-02-25 05:17:56,http://wunschstracke.net/vladimir.hoeger,,64.127.216.31,"{""location"": ""HM"", ""is_mobile"": false}" 22853,8,512,2017-06-05 20:05:03,http://greenholt.net/jaquan_fahey,,202.150.249.114,"{""location"": ""LB"", ""is_mobile"": true}" 22854,8,512,2016-12-20 11:14:40,http://rueckerkuphal.name/furman,,48.5.209.159,"{""location"": ""AE"", ""is_mobile"": true}" 22855,8,512,2017-01-14 18:15:12,http://huels.net/yeenia,,89.138.40.22,"{""location"": ""RU"", ""is_mobile"": false}" 22856,8,512,2017-06-10 04:54:44,http://robertsquigley.com/jamal_luettgen,,237.99.130.154,"{""location"": ""CC"", ""is_mobile"": false}" 22857,8,512,2017-04-26 21:30:18,http://kovacek.biz/davion.cronin,,8.125.64.24,"{""location"": ""FM"", ""is_mobile"": false}" 22858,8,512,2017-05-07 03:48:37,http://damore.biz/wilfred_bergstrom,,237.186.66.71,"{""location"": ""GH"", ""is_mobile"": true}" 22859,8,512,2016-12-14 08:30:12,http://glover.com/jamar,,5.126.85.253,"{""location"": ""SI"", ""is_mobile"": true}" 22860,8,512,2017-05-19 23:35:46,http://schowalter.io/talon_ullrich,,13.162.125.248,"{""location"": ""GP"", ""is_mobile"": true}" 22861,8,512,2017-06-02 17:34:02,http://lednercrist.com/gordon_keebler,,137.213.121.48,"{""location"": ""BL"", ""is_mobile"": true}" 22862,8,512,2017-03-24 16:04:12,http://homenick.co/greta,,180.15.230.163,"{""location"": ""UY"", ""is_mobile"": false}" 22863,8,512,2017-04-01 00:54:14,http://swaniawski.co/taylor,,167.81.87.6,"{""location"": ""UZ"", ""is_mobile"": false}" 22864,8,513,2017-04-07 08:31:14,http://homenicklangworth.info/zoey_schultz,,96.33.181.204,"{""location"": ""GR"", ""is_mobile"": true}" 22865,8,513,2017-01-03 01:32:19,http://abernathy.com/jarred.paucek,,242.187.226.14,"{""location"": ""US"", ""is_mobile"": false}" 22866,8,513,2017-01-13 15:16:28,http://funk.org/richie,,215.107.28.111,"{""location"": ""MM"", ""is_mobile"": true}" 22867,8,513,2017-01-15 18:19:59,http://greenfelder.biz/orlo,,16.158.48.90,"{""location"": ""HR"", ""is_mobile"": true}" 22868,8,513,2017-01-30 17:48:15,http://ryanlangosh.io/jaron,,147.194.23.27,"{""location"": ""UY"", ""is_mobile"": true}" 22869,8,513,2017-01-25 12:59:25,http://leuschke.info/nolan_durgan,,177.69.105.76,"{""location"": ""CH"", ""is_mobile"": true}" 22870,8,513,2017-03-02 23:27:54,http://hamillrutherford.io/zachariah,,241.222.181.241,"{""location"": ""EH"", ""is_mobile"": true}" 22871,8,513,2017-01-13 05:45:17,http://doyle.co/karen.cremin,,70.125.133.238,"{""location"": ""BA"", ""is_mobile"": false}" 22872,8,513,2017-05-11 17:14:33,http://weber.net/isidro,,87.167.87.229,"{""location"": ""VC"", ""is_mobile"": true}" 22873,8,513,2016-12-31 04:20:37,http://stracke.net/elbert_ratke,,95.72.172.97,"{""location"": ""LS"", ""is_mobile"": false}" 22874,8,513,2017-06-08 14:40:19,http://kuhicwalker.io/juston,,59.247.44.99,"{""location"": ""FK"", ""is_mobile"": false}" 22875,8,513,2017-01-04 15:23:03,http://willgulgowski.name/furman,,13.207.73.76,"{""location"": ""DZ"", ""is_mobile"": false}" 22876,8,513,2017-02-21 00:56:03,http://huels.co/elinor_walker,,117.84.139.30,"{""location"": ""MK"", ""is_mobile"": false}" 22877,8,513,2017-04-05 03:45:03,http://mayer.biz/keith.bradtke,,16.197.66.192,"{""location"": ""CI"", ""is_mobile"": false}" 22878,8,513,2017-04-10 08:34:17,http://ratke.info/amber,,136.100.4.153,"{""location"": ""MN"", ""is_mobile"": true}" 22879,8,513,2017-01-05 20:22:19,http://stanton.net/noemie,,208.84.129.84,"{""location"": ""NI"", ""is_mobile"": true}" 22880,8,513,2017-04-10 10:06:25,http://eichmann.com/stanton,,239.53.249.168,"{""location"": ""TH"", ""is_mobile"": false}" 22881,8,513,2017-03-21 08:49:48,http://streich.io/jovani,,111.247.110.127,"{""location"": ""PW"", ""is_mobile"": false}" 1804,1,40,2017-04-12 10:26:21,http://keler.co/harley.lueilwitz,,44.76.228.249,"{""location"": ""IM"", ""is_mobile"": false}" 1805,1,40,2017-02-16 07:38:45,http://von.io/dane_toy,,86.52.213.41,"{""location"": ""AO"", ""is_mobile"": false}" 1806,1,40,2016-12-23 06:14:22,http://mcglynn.org/turner,,181.58.226.169,"{""location"": ""FK"", ""is_mobile"": true}" 1807,1,40,2017-04-29 11:42:01,http://wiza.co/jarvis,,4.109.94.164,"{""location"": ""GI"", ""is_mobile"": true}" 1808,1,40,2017-03-28 17:23:37,http://dicki.name/kelley.breitenberg,,200.109.87.172,"{""location"": ""TT"", ""is_mobile"": true}" 1809,1,40,2017-03-30 21:14:58,http://bergebuckridge.net/donna,,153.219.200.173,"{""location"": ""LT"", ""is_mobile"": true}" 1810,1,40,2017-04-19 21:49:35,http://pouros.biz/lucy,,210.200.160.154,"{""location"": ""FK"", ""is_mobile"": false}" 1811,1,40,2017-01-03 03:09:24,http://fadel.name/claria_borer,,33.54.28.219,"{""location"": ""SB"", ""is_mobile"": true}" 1812,1,40,2017-01-02 02:52:49,http://jacobs.net/sylvan,,184.66.77.233,"{""location"": ""CN"", ""is_mobile"": true}" 1813,1,40,2017-05-05 14:51:46,http://medhurstrolfson.biz/jabari_trantow,,144.3.73.60,"{""location"": ""QA"", ""is_mobile"": true}" 1814,1,40,2017-05-01 03:34:49,http://hagenes.com/nettie.beier,,45.159.227.230,"{""location"": ""RS"", ""is_mobile"": false}" 1815,1,40,2017-01-25 22:31:54,http://rolfson.co/bridgette,,198.46.173.105,"{""location"": ""EC"", ""is_mobile"": true}" 1816,1,40,2017-03-02 08:26:39,http://bergnaum.co/anais,,225.110.57.203,"{""location"": ""VU"", ""is_mobile"": false}" 1817,1,40,2017-04-10 10:36:12,http://gaylordrunolfon.biz/tiara_runolfsdottir,,136.198.252.218,"{""location"": ""AF"", ""is_mobile"": false}" 1818,1,40,2017-01-25 23:18:48,http://corwin.info/danielle,,168.135.31.4,"{""location"": ""JO"", ""is_mobile"": false}" 1819,1,40,2017-03-21 04:03:13,http://flatley.io/vito,,215.223.159.56,"{""location"": ""CD"", ""is_mobile"": true}" 1820,1,40,2017-01-27 15:15:43,http://mcdermottgislason.info/verlie,,59.21.138.91,"{""location"": ""GM"", ""is_mobile"": false}" 1821,1,40,2017-03-03 15:15:03,http://cain.co/mara,,214.173.198.15,"{""location"": ""GP"", ""is_mobile"": false}" 1822,1,40,2016-12-24 12:23:19,http://anderson.biz/estell,,58.235.157.179,"{""location"": ""BF"", ""is_mobile"": false}" 1823,1,40,2017-01-14 06:23:17,http://ortizerdman.info/hardy,,32.168.154.112,"{""location"": ""ML"", ""is_mobile"": false}" 1824,1,40,2016-12-14 17:51:31,http://bartell.com/kaia,,168.56.27.198,"{""location"": ""MV"", ""is_mobile"": true}" 1825,1,40,2017-06-13 02:42:33,http://beahan.io/elya.wisoky,,26.121.117.125,"{""location"": ""SA"", ""is_mobile"": false}" 1826,1,40,2017-02-24 05:53:03,http://funk.co/ima.considine,,163.240.134.178,"{""location"": ""GP"", ""is_mobile"": true}" 1827,1,40,2016-12-29 20:28:36,http://hackettmacejkovic.info/bernice_reinger,,94.93.106.109,"{""location"": ""PG"", ""is_mobile"": false}" 1828,1,40,2017-05-13 22:04:27,http://cruickshank.com/lonny_walker,,101.178.234.126,"{""location"": ""AU"", ""is_mobile"": false}" 1829,1,40,2017-04-14 05:36:37,http://nienow.info/gilberto,,13.251.241.167,"{""location"": ""RS"", ""is_mobile"": false}" 1830,1,40,2017-05-23 02:33:30,http://kohler.io/conrad.luettgen,,212.173.27.232,"{""location"": ""LK"", ""is_mobile"": false}" 1831,1,40,2017-03-06 14:49:20,http://shields.net/lisette,,112.202.19.166,"{""location"": ""EE"", ""is_mobile"": false}" 1832,1,40,2017-02-11 13:30:32,http://millerstiedemann.org/lucio,,72.90.105.209,"{""location"": ""AI"", ""is_mobile"": true}" 1833,1,41,2017-02-23 02:46:23,http://dickinsonrobel.name/althea,,231.101.125.119,"{""location"": ""DJ"", ""is_mobile"": false}" 1834,1,41,2017-02-12 10:24:04,http://gradybechtelar.org/bianka,,95.146.242.87,"{""location"": ""GL"", ""is_mobile"": false}" 1835,1,41,2017-03-24 16:59:34,http://okuneva.org/krystel,,32.184.194.61,"{""location"": ""FK"", ""is_mobile"": true}" 1836,1,41,2017-02-22 03:18:01,http://turner.info/jade_buckridge,,207.74.227.24,"{""location"": ""MT"", ""is_mobile"": false}" 1837,1,41,2016-12-23 10:30:59,http://bosco.io/vada,,139.242.246.98,"{""location"": ""TT"", ""is_mobile"": false}" 1838,1,41,2017-06-11 04:31:39,http://pouros.io/scottie_keeling,,183.126.97.177,"{""location"": ""SR"", ""is_mobile"": false}" 1839,1,41,2017-06-08 04:47:42,http://turner.io/jordan,,242.13.82.58,"{""location"": ""UA"", ""is_mobile"": true}" 1840,1,41,2017-04-11 06:56:16,http://larkinpadberg.biz/dannie_wuckert,,205.116.68.43,"{""location"": ""CH"", ""is_mobile"": false}" 1841,1,41,2017-04-19 19:09:35,http://turnervandervort.com/filiberto,,7.106.87.71,"{""location"": ""ER"", ""is_mobile"": true}" 1842,1,41,2017-01-15 23:26:16,http://mcculloughmiller.org/morton.osinski,,184.21.173.157,"{""location"": ""ME"", ""is_mobile"": true}" 1843,1,41,2017-04-16 03:01:47,http://nitzscheauer.com/diana,,26.45.17.193,"{""location"": ""DJ"", ""is_mobile"": false}" 1844,1,41,2017-03-14 14:31:59,http://gusikowski.io/edna_will,,131.12.157.247,"{""location"": ""SV"", ""is_mobile"": true}" 1845,1,41,2017-05-13 09:51:45,http://mohrhilll.info/derek.mcglynn,,87.63.111.202,"{""location"": ""TJ"", ""is_mobile"": false}" 1846,1,41,2016-12-17 12:52:45,http://gottlieb.net/marilyne_mayert,,96.223.133.191,"{""location"": ""KP"", ""is_mobile"": true}" 1847,1,41,2017-01-16 11:23:36,http://zemlak.name/august_gibson,,46.90.215.23,"{""location"": ""MG"", ""is_mobile"": true}" 1849,1,41,2017-02-02 22:23:25,http://grimes.io/carli_mcglynn,,211.218.92.248,"{""location"": ""MZ"", ""is_mobile"": true}" 1850,1,41,2017-02-23 04:19:53,http://murazikmurphy.co/shawna,,78.216.178.44,"{""location"": ""TN"", ""is_mobile"": false}" 1851,1,41,2017-01-15 23:32:05,http://pacochaschamberger.org/berry,,145.54.172.165,"{""location"": ""TT"", ""is_mobile"": false}" 1852,1,41,2017-04-06 05:31:59,http://hermann.biz/horacio,,128.90.254.98,"{""location"": ""AZ"", ""is_mobile"": false}" 1853,1,41,2017-04-07 01:02:39,http://ryan.co/trent,,220.29.84.139,"{""location"": ""PR"", ""is_mobile"": true}" 1854,1,41,2017-02-17 15:34:37,http://stracke.com/christophe,,23.101.170.232,"{""location"": ""PK"", ""is_mobile"": true}" 1855,1,41,2017-04-17 13:18:23,http://ortiz.com/keith,,56.108.34.197,"{""location"": ""TG"", ""is_mobile"": true}" 1856,1,41,2017-02-10 08:14:30,http://hermann.info/reuben,,65.186.154.109,"{""location"": ""MR"", ""is_mobile"": true}" 1857,1,41,2017-04-21 02:48:14,http://dietrichking.info/naomie_doyle,,71.164.70.176,"{""location"": ""BT"", ""is_mobile"": true}" 1858,1,41,2017-01-15 13:08:31,http://hansen.name/manuela_okon,,247.149.120.50,"{""location"": ""SD"", ""is_mobile"": true}" 1859,1,41,2017-04-05 10:01:59,http://franecki.io/dayne,,194.29.36.164,"{""location"": ""TK"", ""is_mobile"": false}" 1860,1,41,2016-12-28 10:53:05,http://littlewaelchi.info/teagan.stiedemann,,30.242.73.115,"{""location"": ""TL"", ""is_mobile"": false}" 22882,8,513,2017-03-31 19:03:08,http://strosin.co/monica.parisian,,95.9.215.49,"{""location"": ""KI"", ""is_mobile"": false}" 22883,8,513,2017-02-11 09:23:36,http://larsonspinka.biz/jailyn,,12.243.11.234,"{""location"": ""BJ"", ""is_mobile"": false}" 22884,8,513,2016-12-20 10:48:45,http://yostwolf.com/gielle,,103.154.5.162,"{""location"": ""FO"", ""is_mobile"": true}" 22885,8,513,2017-05-07 15:34:08,http://heelpfeffer.com/jeie_luettgen,,143.224.189.42,"{""location"": ""NL"", ""is_mobile"": true}" 22886,8,513,2017-05-16 01:26:42,http://mitchell.info/clotilde,,89.15.32.166,"{""location"": ""ZA"", ""is_mobile"": true}" 22887,8,513,2016-12-13 10:42:52,http://harber.co/candelario_hettinger,,218.95.13.43,"{""location"": ""BF"", ""is_mobile"": true}" 22888,8,513,2017-05-28 13:18:54,http://stromanfahey.org/claude.dickens,,182.227.103.16,"{""location"": ""NE"", ""is_mobile"": true}" 22889,8,513,2017-03-28 03:58:03,http://goyette.biz/ania,,212.56.197.211,"{""location"": ""MG"", ""is_mobile"": true}" 22890,8,513,2017-06-01 14:21:26,http://beer.name/tito,,50.164.151.38,"{""location"": ""MU"", ""is_mobile"": true}" 22891,8,513,2017-05-22 21:59:40,http://johnson.co/judson_gaylord,,155.96.33.232,"{""location"": ""GI"", ""is_mobile"": true}" 22892,8,513,2017-02-28 17:21:01,http://adams.co/laurie,,190.15.201.169,"{""location"": ""VA"", ""is_mobile"": false}" 22893,8,513,2017-04-05 17:13:11,http://dachabbott.co/irma_zulauf,,216.26.69.148,"{""location"": ""MN"", ""is_mobile"": false}" 22894,8,513,2017-04-03 01:03:51,http://murraystoltenberg.com/kavon,,94.63.2.189,"{""location"": ""TD"", ""is_mobile"": false}" 22895,8,513,2017-02-08 19:25:10,http://lakinbergnaum.name/cary_franecki,,134.120.113.148,"{""location"": ""PT"", ""is_mobile"": true}" 22896,8,513,2017-04-08 15:16:36,http://townehaley.info/laura_brown,,66.158.250.205,"{""location"": ""AL"", ""is_mobile"": false}" 22897,8,513,2017-03-26 05:05:28,http://trompdooley.name/hilma_orn,,94.112.236.230,"{""location"": ""IQ"", ""is_mobile"": true}" 22898,8,513,2017-04-02 22:44:30,http://prohaskakertzmann.com/marcella.lemke,,20.183.215.115,"{""location"": ""AS"", ""is_mobile"": false}" 22899,8,513,2017-01-25 17:57:36,http://bernhardokeefe.info/evert_brekke,,111.154.176.242,"{""location"": ""UZ"", ""is_mobile"": true}" 22900,8,513,2017-06-10 03:06:08,http://hand.name/gladyce_turcotte,,208.232.116.74,"{""location"": ""SC"", ""is_mobile"": true}" 22901,8,514,2017-04-19 02:07:47,http://donnelly.info/reta_hintz,,75.222.242.128,"{""location"": ""TD"", ""is_mobile"": false}" 22902,8,514,2017-02-21 08:15:51,http://borer.com/mozell,,164.104.140.137,"{""location"": ""NU"", ""is_mobile"": true}" 22903,8,514,2016-12-25 06:57:44,http://mckenzie.info/herminia.nitzsche,,85.170.17.202,"{""location"": ""PY"", ""is_mobile"": false}" 22904,8,514,2017-04-20 08:05:00,http://macejkovic.org/josefa.hyatt,,195.246.116.251,"{""location"": ""LU"", ""is_mobile"": true}" 22905,8,514,2017-03-18 04:38:58,http://altenwerth.net/hettie,,251.41.177.114,"{""location"": ""NG"", ""is_mobile"": false}" 22906,8,514,2017-03-27 20:18:16,http://schuppe.org/dortha,,115.184.64.41,"{""location"": ""TK"", ""is_mobile"": true}" 22907,8,514,2017-02-02 14:06:42,http://koch.com/ruel,,224.186.108.60,"{""location"": ""JO"", ""is_mobile"": false}" 22908,8,514,2017-03-20 05:30:46,http://kundewitting.co/patrick.hegmann,,37.64.79.10,"{""location"": ""CX"", ""is_mobile"": false}" 22909,8,514,2016-12-31 20:11:25,http://ullrich.co/friedrich_moriette,,19.158.134.237,"{""location"": ""PG"", ""is_mobile"": false}" 22910,8,514,2017-06-03 10:54:19,http://zulauffay.biz/erling,,40.150.40.68,"{""location"": ""NO"", ""is_mobile"": false}" 22911,8,514,2017-02-16 03:03:05,http://braun.biz/lilla,,170.200.81.94,"{""location"": ""HU"", ""is_mobile"": false}" 22912,8,514,2017-04-02 03:22:49,http://borer.com/micah,,124.176.27.49,"{""location"": ""AE"", ""is_mobile"": true}" 22913,8,514,2017-04-06 20:25:34,http://olson.io/orion.hahn,,103.122.43.174,"{""location"": ""BH"", ""is_mobile"": false}" 22914,8,514,2017-04-09 16:09:54,http://mayer.info/blanca_schiller,,41.9.37.197,"{""location"": ""IE"", ""is_mobile"": true}" 22915,8,514,2017-06-11 17:21:57,http://cristmueller.biz/george,,38.215.49.114,"{""location"": ""AX"", ""is_mobile"": true}" 22916,8,514,2017-03-08 19:44:47,http://koelpinshanahan.info/axel_parisian,,33.70.215.48,"{""location"": ""YE"", ""is_mobile"": true}" 22917,8,514,2017-06-01 16:03:30,http://hamill.io/maxwell.tillman,,248.112.67.30,"{""location"": ""HU"", ""is_mobile"": false}" 22918,8,514,2017-05-14 13:04:40,http://dickens.io/annetta,,137.207.5.241,"{""location"": ""CC"", ""is_mobile"": false}" 22919,8,514,2017-05-22 20:06:30,http://kshlerincorwin.co/antonia.shields,,82.70.24.237,"{""location"": ""SK"", ""is_mobile"": true}" 22920,8,514,2017-03-27 11:38:55,http://hermann.org/maci.mayert,,38.248.186.116,"{""location"": ""MH"", ""is_mobile"": false}" 22921,8,514,2017-01-10 07:54:41,http://ziemannhilpert.com/jade_upton,,184.236.113.239,"{""location"": ""CN"", ""is_mobile"": false}" 22922,8,514,2017-05-16 21:36:19,http://mitchell.org/nikolas,,157.6.48.124,"{""location"": ""CZ"", ""is_mobile"": false}" 22923,8,514,2017-01-01 12:29:37,http://bergstrombarton.co/jeanne,,245.67.213.92,"{""location"": ""RS"", ""is_mobile"": true}" 22924,8,514,2017-02-26 04:41:59,http://eichmann.net/alda,,196.79.7.30,"{""location"": ""PM"", ""is_mobile"": true}" 22925,8,514,2017-06-11 00:31:59,http://murphy.info/ray,,51.218.89.51,"{""location"": ""HN"", ""is_mobile"": true}" 22926,8,514,2017-05-21 14:25:02,http://towne.biz/jarret.huel,,252.240.229.100,"{""location"": ""CY"", ""is_mobile"": false}" 22927,8,514,2017-05-29 04:28:36,http://willmmith.net/jason,,42.103.103.187,"{""location"": ""AT"", ""is_mobile"": false}" 22928,8,514,2017-05-30 06:02:58,http://kris.biz/deie.pfannerstill,,239.170.174.78,"{""location"": ""IR"", ""is_mobile"": true}" 22929,8,514,2017-03-28 07:10:50,http://spencer.name/daisha,,94.170.182.250,"{""location"": ""AI"", ""is_mobile"": false}" 22930,8,514,2017-02-28 22:02:11,http://hahnbarton.com/michel.haley,,139.75.82.166,"{""location"": ""CC"", ""is_mobile"": true}" 22931,8,515,2017-05-02 22:21:40,http://hermannrippin.name/libbie.stoltenberg,,198.88.21.187,"{""location"": ""IO"", ""is_mobile"": false}" 22932,8,515,2016-12-24 17:22:25,http://quigleydietrich.info/dasia,,18.138.126.84,"{""location"": ""WF"", ""is_mobile"": true}" 22933,8,515,2017-05-07 03:10:06,http://king.co/deja.hintz,,25.200.119.190,"{""location"": ""CN"", ""is_mobile"": true}" 22934,8,515,2017-01-30 10:59:20,http://stracke.biz/ezequiel.waters,,250.211.35.174,"{""location"": ""PN"", ""is_mobile"": true}" 22935,8,515,2017-05-04 03:06:27,http://walkerhuels.biz/luciano,,114.128.124.152,"{""location"": ""LC"", ""is_mobile"": true}" 22936,8,515,2017-03-06 03:24:33,http://bradtke.io/aaron,,50.200.159.214,"{""location"": ""NR"", ""is_mobile"": false}" 1861,1,41,2017-05-13 23:47:14,http://wisoky.biz/emmy_durgan,,188.241.7.151,"{""location"": ""IR"", ""is_mobile"": true}" 1862,1,41,2017-04-03 02:43:57,http://nader.io/clara.schmitt,,223.104.63.131,"{""location"": ""CV"", ""is_mobile"": false}" 1863,1,41,2017-04-15 13:18:11,http://greensipes.com/trycia.wolff,,133.114.194.137,"{""location"": ""SE"", ""is_mobile"": false}" 1864,1,41,2017-04-24 06:21:21,http://browngreen.biz/savanah,,136.145.190.206,"{""location"": ""BY"", ""is_mobile"": false}" 1865,1,41,2017-06-12 10:46:08,http://moriette.net/norene,,129.36.12.213,"{""location"": ""NU"", ""is_mobile"": false}" 1866,1,41,2017-05-24 07:48:57,http://ondricka.info/shawna,,217.97.129.16,"{""location"": ""RS"", ""is_mobile"": false}" 1867,1,41,2016-12-25 17:33:33,http://treutelbosco.biz/nash,,252.68.9.244,"{""location"": ""DO"", ""is_mobile"": true}" 1868,1,41,2016-12-16 09:00:34,http://spinkaconnelly.co/shaina.moriette,,185.225.30.205,"{""location"": ""GB"", ""is_mobile"": false}" 1869,1,41,2017-02-15 17:50:40,http://von.org/adolf,,135.213.175.42,"{""location"": ""AD"", ""is_mobile"": false}" 1870,1,41,2017-06-02 12:15:21,http://feil.net/betsy.block,,176.93.119.205,"{""location"": ""BA"", ""is_mobile"": true}" 1871,1,41,2017-03-24 08:46:20,http://homenick.info/annalise_murray,,219.206.75.9,"{""location"": ""HR"", ""is_mobile"": true}" 1872,1,41,2017-06-05 22:40:40,http://emard.org/evans.huel,,183.30.46.80,"{""location"": ""MZ"", ""is_mobile"": true}" 1873,1,41,2017-01-11 17:07:01,http://townekaulke.info/delbert,,163.75.125.76,"{""location"": ""TD"", ""is_mobile"": true}" 1874,1,41,2017-05-17 08:12:53,http://bergewelch.net/april.becker,,152.157.216.39,"{""location"": ""BM"", ""is_mobile"": true}" 1875,1,41,2017-05-09 03:26:40,http://little.net/lizzie_zemlak,,244.72.11.128,"{""location"": ""CZ"", ""is_mobile"": false}" 1876,1,41,2017-04-19 15:41:20,http://gleasonsatterfield.org/maximus_metz,,222.204.110.248,"{""location"": ""SE"", ""is_mobile"": false}" 1877,1,41,2017-01-30 09:17:40,http://welchcarroll.co/ruby,,148.178.76.76,"{""location"": ""CZ"", ""is_mobile"": false}" 1878,1,41,2017-03-18 07:52:43,http://powlowski.info/raphael,,159.208.62.116,"{""location"": ""LT"", ""is_mobile"": false}" 1879,1,41,2017-01-23 13:44:36,http://homenick.biz/kaci,,94.170.12.216,"{""location"": ""RU"", ""is_mobile"": true}" 1880,1,41,2017-05-28 21:11:15,http://hartmann.name/eveline.shanahan,,125.135.192.150,"{""location"": ""SX"", ""is_mobile"": true}" 1881,1,41,2017-04-28 14:15:28,http://bashirian.biz/allene_jenkins,,114.197.165.79,"{""location"": ""MC"", ""is_mobile"": true}" 1882,1,42,2017-05-19 00:10:36,http://larkin.org/sandrine.mckenzie,,93.63.207.120,"{""location"": ""ST"", ""is_mobile"": false}" 1883,1,42,2017-01-06 00:01:33,http://oconnell.name/kadin,,3.137.201.248,"{""location"": ""MD"", ""is_mobile"": false}" 1884,1,42,2016-12-30 14:37:46,http://lindgren.io/eusebio,,84.105.217.197,"{""location"": ""US"", ""is_mobile"": true}" 1885,1,42,2017-05-14 23:10:57,http://heel.io/amparo_wyman,,87.225.152.248,"{""location"": ""BJ"", ""is_mobile"": false}" 1886,1,42,2016-12-15 15:31:51,http://moen.biz/mohamed,,174.199.4.142,"{""location"": ""BG"", ""is_mobile"": false}" 1887,1,42,2017-03-15 04:23:04,http://caspermetz.io/kendall,,105.200.94.86,"{""location"": ""GE"", ""is_mobile"": true}" 1888,1,42,2017-01-07 12:08:40,http://mohrkohler.io/torrey_barton,,40.245.155.197,"{""location"": ""AG"", ""is_mobile"": true}" 1889,1,42,2017-02-23 15:24:54,http://feeney.info/blake.jast,,228.198.42.131,"{""location"": ""CG"", ""is_mobile"": true}" 1890,1,42,2017-04-08 05:50:27,http://heaney.com/dorothy.braun,,88.178.177.89,"{""location"": ""MO"", ""is_mobile"": false}" 1891,1,42,2017-03-27 06:39:38,http://monahan.net/zane_sauer,,172.109.78.180,"{""location"": ""SK"", ""is_mobile"": false}" 1892,1,42,2016-12-21 17:22:11,http://wildermanbruen.org/emanuel.nolan,,140.249.60.242,"{""location"": ""PT"", ""is_mobile"": true}" 1893,1,42,2017-03-29 04:46:54,http://lesch.info/ardella,,188.167.100.12,"{""location"": ""MQ"", ""is_mobile"": true}" 1894,1,42,2017-01-02 06:20:33,http://lindgrenhackett.co/tanya,,202.245.197.237,"{""location"": ""TV"", ""is_mobile"": false}" 1895,1,42,2017-01-12 23:56:35,http://pauceklegros.co/emelia,,139.122.54.11,"{""location"": ""BS"", ""is_mobile"": true}" 1896,1,42,2017-01-26 01:20:52,http://dach.org/valentine_goldner,,135.93.243.224,"{""location"": ""IE"", ""is_mobile"": false}" 1897,1,42,2017-03-29 23:37:28,http://wolff.name/henri,,143.240.218.162,"{""location"": ""UZ"", ""is_mobile"": false}" 1898,1,42,2017-05-23 21:30:54,http://altenwerthconn.com/kenneth,,250.164.69.92,"{""location"": ""CZ"", ""is_mobile"": true}" 1899,1,42,2017-03-17 23:17:16,http://paucek.name/scotty,,137.2.120.35,"{""location"": ""YT"", ""is_mobile"": false}" 1900,1,42,2017-03-24 05:56:18,http://fadelmiller.io/moriah_ratke,,75.197.230.219,"{""location"": ""FO"", ""is_mobile"": true}" 1901,1,42,2017-04-18 12:06:34,http://luettgen.org/berniece,,149.155.175.144,"{""location"": ""FI"", ""is_mobile"": false}" 1902,1,42,2017-05-28 08:27:53,http://ortiz.info/jennings.douglas,,10.175.194.225,"{""location"": ""HU"", ""is_mobile"": true}" 1903,1,42,2016-12-22 14:53:53,http://will.name/alphonso.stoltenberg,,56.112.40.181,"{""location"": ""NO"", ""is_mobile"": false}" 1904,1,42,2016-12-16 03:39:43,http://rodriguez.info/winfield,,151.117.61.206,"{""location"": ""KY"", ""is_mobile"": false}" 1905,1,42,2017-02-20 15:36:26,http://smitham.info/devin.hayes,,207.127.20.136,"{""location"": ""EC"", ""is_mobile"": false}" 1906,1,42,2017-03-06 09:59:48,http://keelingluettgen.org/marquise_satterfield,,202.204.68.250,"{""location"": ""GY"", ""is_mobile"": true}" 1907,1,42,2017-05-20 12:28:00,http://weber.com/elvie.schamberger,,68.26.44.142,"{""location"": ""CG"", ""is_mobile"": true}" 1908,1,42,2017-04-04 08:51:36,http://cummings.info/amber.aufderhar,,162.206.47.14,"{""location"": ""IL"", ""is_mobile"": true}" 1909,1,42,2017-03-09 16:11:49,http://hackettwehner.com/viva,,155.63.197.231,"{""location"": ""AX"", ""is_mobile"": true}" 1910,1,42,2016-12-13 17:34:46,http://weimannschuster.net/valentin,,59.50.152.73,"{""location"": ""SH"", ""is_mobile"": true}" 1911,1,42,2017-05-06 17:31:18,http://ryanbergstrom.info/litzy,,19.121.56.122,"{""location"": ""KR"", ""is_mobile"": true}" 1912,1,42,2017-01-05 04:19:12,http://wolff.co/vergie,,149.49.154.13,"{""location"": ""TW"", ""is_mobile"": false}" 1913,1,42,2017-03-15 00:07:58,http://reichel.name/verner,,237.239.192.70,"{""location"": ""CW"", ""is_mobile"": false}" 1914,1,42,2017-03-11 11:06:21,http://wunsch.co/janae,,29.145.26.79,"{""location"": ""RS"", ""is_mobile"": true}" 1915,1,42,2017-03-26 01:13:04,http://gusikowskijohns.io/houston_bernhard,,177.206.2.140,"{""location"": ""BQ"", ""is_mobile"": false}" 1916,1,42,2017-03-30 06:45:13,http://marks.org/ashleigh.kub,,210.52.18.111,"{""location"": ""AS"", ""is_mobile"": false}" 22937,8,515,2017-03-06 17:27:10,http://jones.org/danny,,160.133.36.98,"{""location"": ""HR"", ""is_mobile"": true}" 22938,8,515,2017-03-06 05:26:16,http://upton.org/myrtie,,123.228.13.84,"{""location"": ""PT"", ""is_mobile"": false}" 22939,8,515,2017-01-09 21:54:26,http://strosinsauer.com/elizabeth.schultz,,103.244.213.81,"{""location"": ""SX"", ""is_mobile"": false}" 22940,8,515,2017-05-06 02:45:59,http://hilllglover.biz/lelah_emmerich,,68.74.33.69,"{""location"": ""GD"", ""is_mobile"": true}" 22941,8,515,2017-05-20 17:30:14,http://christiansen.name/bettye,,124.204.209.57,"{""location"": ""AZ"", ""is_mobile"": true}" 22942,8,515,2017-02-23 01:04:08,http://kohler.info/haylee.strosin,,77.119.56.86,"{""location"": ""SN"", ""is_mobile"": true}" 22943,8,515,2017-02-13 04:45:27,http://padberg.name/dee.ruecker,,181.125.243.61,"{""location"": ""VI"", ""is_mobile"": true}" 22944,8,515,2016-12-28 16:53:48,http://reinger.co/lon.beier,,219.249.114.77,"{""location"": ""BH"", ""is_mobile"": false}" 22945,8,515,2017-01-19 08:58:59,http://kreiger.biz/karley.schaefer,,134.149.41.34,"{""location"": ""UA"", ""is_mobile"": true}" 22946,8,515,2017-01-08 15:48:29,http://gusikowskihettinger.io/emmitt,,46.61.175.170,"{""location"": ""BD"", ""is_mobile"": false}" 22947,8,515,2017-03-12 18:02:14,http://brown.name/mireille_stehr,,249.196.146.85,"{""location"": ""GF"", ""is_mobile"": true}" 22948,8,515,2017-01-22 20:11:23,http://gusikowskihomenick.info/asha,,17.70.183.172,"{""location"": ""PM"", ""is_mobile"": false}" 22949,8,515,2017-01-08 11:30:30,http://kozeyohara.co/rusty,,56.209.24.155,"{""location"": ""ML"", ""is_mobile"": false}" 22950,8,515,2017-01-27 12:48:53,http://olson.net/tyreek.king,,89.234.9.219,"{""location"": ""GY"", ""is_mobile"": false}" 22951,8,515,2017-03-20 12:00:29,http://hermiston.com/irwin.murphy,,213.198.159.158,"{""location"": ""SC"", ""is_mobile"": false}" 22952,8,515,2017-04-29 06:16:05,http://bechtelarwaelchi.co/bernita,,114.195.49.188,"{""location"": ""IL"", ""is_mobile"": true}" 22953,8,515,2016-12-18 15:28:52,http://bartell.co/kathryn,,237.39.47.158,"{""location"": ""AD"", ""is_mobile"": false}" 22954,8,515,2017-02-21 05:05:56,http://schinner.net/emiliano_schultz,,85.246.76.138,"{""location"": ""AS"", ""is_mobile"": true}" 22955,8,515,2017-03-20 01:56:01,http://bashirianullrich.net/alexys,,112.61.33.248,"{""location"": ""MY"", ""is_mobile"": true}" 22956,8,515,2017-03-21 01:15:16,http://hamill.com/josh,,126.143.167.122,"{""location"": ""EE"", ""is_mobile"": false}" 22957,8,515,2017-01-18 21:38:33,http://schambergermann.net/jayda,,215.174.225.164,"{""location"": ""GE"", ""is_mobile"": false}" 22958,8,515,2017-04-25 09:00:00,http://schoenbotsford.org/jasmin,,23.35.195.156,"{""location"": ""GU"", ""is_mobile"": false}" 22959,8,515,2017-05-07 10:50:02,http://flatley.name/gardner_keler,,185.88.9.153,"{""location"": ""MD"", ""is_mobile"": false}" 22960,8,515,2017-05-30 23:01:21,http://goyette.io/ashly,,168.109.190.124,"{""location"": ""SI"", ""is_mobile"": false}" 22961,8,515,2017-04-09 13:49:16,http://toy.net/robbie.adams,,83.76.99.203,"{""location"": ""TF"", ""is_mobile"": true}" 22962,8,515,2017-06-08 10:19:59,http://conroyswaniawski.co/skyla,,232.95.193.11,"{""location"": ""SB"", ""is_mobile"": false}" 22963,8,515,2017-02-17 12:07:10,http://grimes.name/keanu_feeney,,223.240.95.176,"{""location"": ""NA"", ""is_mobile"": true}" 22964,8,515,2017-01-27 01:08:33,http://leannon.org/rosemary,,57.190.29.135,"{""location"": ""NA"", ""is_mobile"": true}" 22965,8,515,2017-05-01 07:17:49,http://schowalter.name/buddy,,99.195.28.156,"{""location"": ""LV"", ""is_mobile"": true}" 22966,8,515,2017-03-07 01:21:16,http://legrosbauch.org/camryn,,26.78.191.106,"{""location"": ""JM"", ""is_mobile"": true}" 22967,8,515,2017-05-27 02:40:14,http://wiegand.com/deshaun,,46.235.144.251,"{""location"": ""AS"", ""is_mobile"": false}" 22968,8,515,2016-12-22 03:43:55,http://parisianshields.io/lazaro.mayert,,216.158.121.162,"{""location"": ""SL"", ""is_mobile"": true}" 22969,8,515,2017-03-03 00:03:58,http://nienow.net/adriana_ward,,86.207.170.211,"{""location"": ""EG"", ""is_mobile"": false}" 22970,8,515,2017-03-10 12:15:06,http://kundemcglynn.net/ruth,,119.232.138.87,"{""location"": ""ER"", ""is_mobile"": true}" 22971,8,515,2017-03-29 12:45:02,http://simonis.co/darryl_welch,,233.64.81.102,"{""location"": ""NC"", ""is_mobile"": true}" 22972,8,515,2017-01-16 07:51:06,http://stehr.com/mandy.marquardt,,37.224.153.133,"{""location"": ""CC"", ""is_mobile"": false}" 22973,8,515,2017-01-15 12:15:32,http://corkery.com/rodrick_turner,,111.71.128.163,"{""location"": ""MT"", ""is_mobile"": false}" 22974,8,515,2017-04-24 09:38:34,http://reichert.biz/ila,,23.227.55.154,"{""location"": ""AG"", ""is_mobile"": true}" 22975,8,515,2017-04-19 01:26:52,http://bashirian.net/britney,,248.213.187.190,"{""location"": ""GR"", ""is_mobile"": true}" 22976,8,515,2016-12-13 13:38:06,http://gulgowski.name/paula,,153.230.167.162,"{""location"": ""KR"", ""is_mobile"": false}" 22977,8,515,2016-12-22 02:57:06,http://ratke.biz/amiya,,108.233.204.95,"{""location"": ""MH"", ""is_mobile"": true}" 22978,8,515,2017-03-15 10:27:57,http://hoeger.biz/iliana_swaniawski,,35.147.18.14,"{""location"": ""VN"", ""is_mobile"": true}" 22979,8,515,2017-01-12 23:58:00,http://jaskolski.co/kristin,,243.155.231.110,"{""location"": ""LT"", ""is_mobile"": true}" 22980,8,515,2017-04-14 12:06:19,http://mayer.net/sherwood,,161.218.110.48,"{""location"": ""SS"", ""is_mobile"": true}" 22981,8,515,2017-01-13 09:57:58,http://schummborer.com/herminio_upton,,46.153.76.237,"{""location"": ""TD"", ""is_mobile"": false}" 22982,8,515,2017-01-07 21:25:31,http://greenholt.co/hattie,,105.42.172.116,"{""location"": ""AE"", ""is_mobile"": true}" 22983,8,516,2017-04-17 07:53:40,http://huelkoepp.net/rhea.runolfon,,134.133.218.172,"{""location"": ""MD"", ""is_mobile"": true}" 22984,8,516,2017-03-15 14:17:37,http://swaniawski.net/dale.schamberger,,42.114.246.249,"{""location"": ""IT"", ""is_mobile"": true}" 22985,8,516,2017-05-07 11:26:26,http://schmidt.co/margret,,86.180.4.123,"{""location"": ""BO"", ""is_mobile"": true}" 22986,8,516,2017-06-12 07:48:57,http://langosh.org/abraham,,201.174.228.176,"{""location"": ""SC"", ""is_mobile"": true}" 22987,8,516,2017-06-02 10:18:33,http://ledner.net/zaria,,37.99.217.118,"{""location"": ""NO"", ""is_mobile"": true}" 22988,8,516,2017-06-10 22:48:42,http://green.biz/elody_koch,,203.28.140.229,"{""location"": ""MO"", ""is_mobile"": false}" 22989,8,516,2017-04-14 02:55:13,http://willmsfeil.co/desiree,,202.228.22.110,"{""location"": ""NC"", ""is_mobile"": true}" 22990,8,516,2017-05-30 23:24:38,http://langworth.io/robbie.jakubowski,,121.250.109.224,"{""location"": ""GF"", ""is_mobile"": false}" 22991,8,516,2017-02-04 07:29:22,http://medhurst.io/emmanuel.yundt,,111.107.214.250,"{""location"": ""KZ"", ""is_mobile"": true}" 1917,1,42,2017-06-13 21:27:02,http://ullrich.info/vickie.roberts,,188.230.166.163,"{""location"": ""UY"", ""is_mobile"": true}" 1918,1,42,2017-01-01 22:42:44,http://lebsackcollins.co/wellington,,164.153.178.188,"{""location"": ""LV"", ""is_mobile"": false}" 1919,1,42,2017-02-18 18:30:22,http://luettgen.co/belle_deckow,,126.103.46.135,"{""location"": ""GL"", ""is_mobile"": true}" 1920,1,42,2017-03-18 03:45:09,http://mraz.net/clyde,,126.140.153.86,"{""location"": ""UM"", ""is_mobile"": true}" 1921,1,42,2017-06-09 09:18:15,http://macgyver.name/hulda,,124.44.208.166,"{""location"": ""MD"", ""is_mobile"": true}" 1922,1,42,2017-01-03 17:42:59,http://gorczany.info/henderson.williamson,,65.36.113.97,"{""location"": ""GP"", ""is_mobile"": false}" 1923,1,42,2017-03-17 15:22:34,http://strosin.io/oliver_smith,,226.90.19.47,"{""location"": ""AS"", ""is_mobile"": true}" 1924,1,42,2017-06-04 08:52:36,http://mosciskiwatsica.org/diamond,,237.50.15.127,"{""location"": ""CW"", ""is_mobile"": false}" 1925,1,42,2017-03-01 15:28:23,http://beer.org/tyrese_boyer,,35.137.90.72,"{""location"": ""AM"", ""is_mobile"": true}" 1926,1,42,2017-06-11 20:35:45,http://douglasherzog.io/augustine_larson,,94.225.180.162,"{""location"": ""NO"", ""is_mobile"": false}" 1927,1,42,2017-02-21 09:15:56,http://deckow.info/anya.romaguera,,171.79.239.208,"{""location"": ""PE"", ""is_mobile"": true}" 1928,1,42,2017-03-08 19:09:35,http://weimann.io/ambrose,,9.133.138.50,"{""location"": ""DM"", ""is_mobile"": true}" 1929,1,42,2017-01-25 23:47:47,http://raynorzboncak.net/zack_goodwin,,123.184.109.149,"{""location"": ""AZ"", ""is_mobile"": true}" 1930,1,42,2017-03-24 06:25:33,http://littel.info/toy,,243.54.10.53,"{""location"": ""IR"", ""is_mobile"": false}" 1931,1,42,2017-04-03 10:03:06,http://wuckerttreutel.io/cornell_crist,,25.246.96.139,"{""location"": ""PA"", ""is_mobile"": true}" 1932,1,42,2017-05-25 09:57:07,http://hegmann.name/hosea,,69.8.236.80,"{""location"": ""GR"", ""is_mobile"": false}" 1933,1,42,2017-02-06 12:46:50,http://mills.org/emmitt.lind,,92.222.50.199,"{""location"": ""DO"", ""is_mobile"": false}" 1934,1,42,2017-06-05 08:47:53,http://ernser.co/kaela,,253.112.100.14,"{""location"": ""MD"", ""is_mobile"": false}" 1935,1,42,2017-04-14 01:59:19,http://bergebradtke.net/bernice,,146.13.186.192,"{""location"": ""CK"", ""is_mobile"": false}" 1936,1,42,2017-01-13 02:26:36,http://ryan.com/willie,,15.140.161.249,"{""location"": ""FR"", ""is_mobile"": true}" 1937,1,42,2017-01-09 02:01:09,http://beahan.co/rickie.nitzsche,,196.29.136.199,"{""location"": ""MQ"", ""is_mobile"": true}" 1938,1,43,2016-12-27 03:57:53,http://predovic.com/rae_monahan,,126.85.73.47,"{""location"": ""BD"", ""is_mobile"": true}" 1939,1,43,2017-01-24 09:59:32,http://corwin.co/calista_harris,,5.137.54.118,"{""location"": ""FM"", ""is_mobile"": true}" 1940,1,43,2017-01-13 13:35:47,http://corwin.net/federico,,9.117.217.177,"{""location"": ""TH"", ""is_mobile"": false}" 1941,1,43,2017-06-02 14:23:09,http://adams.com/alvina_jones,,246.183.199.117,"{""location"": ""NG"", ""is_mobile"": false}" 1942,1,43,2017-02-25 18:22:09,http://marquardtgrant.net/elvis,,151.189.159.101,"{""location"": ""KG"", ""is_mobile"": true}" 1943,1,43,2017-05-06 15:23:20,http://mann.org/wayne,,153.2.21.19,"{""location"": ""CN"", ""is_mobile"": false}" 1944,1,43,2017-01-09 14:23:30,http://ratkecummerata.name/gene_jerde,,119.233.173.26,"{""location"": ""GW"", ""is_mobile"": true}" 1945,1,43,2017-03-07 15:20:56,http://rohan.biz/monserrate,,252.104.87.25,"{""location"": ""CF"", ""is_mobile"": false}" 1946,1,43,2017-01-04 13:52:57,http://cartercrona.name/alden_kling,,2.93.184.215,"{""location"": ""ST"", ""is_mobile"": false}" 1947,1,43,2017-02-03 09:07:53,http://gottlieb.com/bethel,,231.73.190.41,"{""location"": ""CD"", ""is_mobile"": true}" 1948,1,43,2017-04-21 22:00:32,http://waelchi.name/isac,,65.183.19.8,"{""location"": ""CC"", ""is_mobile"": false}" 1949,1,43,2017-06-08 06:52:03,http://bechtelar.co/clifford,,215.214.220.23,"{""location"": ""TV"", ""is_mobile"": true}" 1950,1,43,2017-01-18 20:08:50,http://kirlin.org/leopold,,202.6.167.140,"{""location"": ""KM"", ""is_mobile"": false}" 1951,1,43,2017-03-02 19:34:50,http://bartoletti.org/stewart_ko,,171.239.212.6,"{""location"": ""BR"", ""is_mobile"": true}" 1952,1,43,2017-01-07 16:40:44,http://pollich.io/kaleb.davis,,22.12.155.110,"{""location"": ""HR"", ""is_mobile"": false}" 1953,1,43,2017-02-12 06:19:14,http://swaniawski.org/lulu_shanahan,,20.47.164.202,"{""location"": ""ME"", ""is_mobile"": true}" 1954,1,43,2017-04-25 12:37:50,http://vandervort.net/alberta,,217.13.145.150,"{""location"": ""BT"", ""is_mobile"": true}" 1955,1,43,2017-01-29 12:14:45,http://brown.biz/pablo,,25.48.125.126,"{""location"": ""DE"", ""is_mobile"": true}" 1956,1,43,2017-01-10 23:47:16,http://kuhn.net/hank_schneider,,151.32.201.177,"{""location"": ""MN"", ""is_mobile"": true}" 1957,1,43,2017-05-23 22:48:30,http://bernhard.com/king,,231.210.250.32,"{""location"": ""HN"", ""is_mobile"": true}" 1958,1,43,2016-12-21 16:30:51,http://pagacwalter.info/moises_nitzsche,,124.58.150.18,"{""location"": ""TG"", ""is_mobile"": true}" 1959,1,43,2016-12-29 15:14:28,http://eichmann.net/madalyn,,159.179.243.220,"{""location"": ""EG"", ""is_mobile"": false}" 1960,1,43,2017-01-02 22:14:23,http://zemlakjakubowski.io/horace,,103.213.239.60,"{""location"": ""PE"", ""is_mobile"": false}" 1961,1,43,2017-01-07 20:00:58,http://terryrath.info/norval_lebsack,,234.6.213.166,"{""location"": ""BF"", ""is_mobile"": true}" 1962,1,43,2017-03-11 16:18:21,http://kleinstrosin.net/caitlyn_thompson,,5.121.151.172,"{""location"": ""CH"", ""is_mobile"": true}" 1963,1,43,2017-04-12 15:56:27,http://balistreri.net/willard_kaulke,,150.189.10.12,"{""location"": ""MM"", ""is_mobile"": false}" 1964,1,43,2017-01-12 02:30:49,http://emardschiller.org/lois,,12.204.6.109,"{""location"": ""SV"", ""is_mobile"": false}" 1965,1,43,2017-01-19 14:07:24,http://yundt.name/nelson,,143.3.169.189,"{""location"": ""EH"", ""is_mobile"": false}" 1966,1,43,2017-02-01 10:47:00,http://aufderhar.name/odie,,80.39.15.222,"{""location"": ""MF"", ""is_mobile"": false}" 1967,1,44,2017-02-02 00:25:27,http://mante.io/saul,,219.200.251.254,"{""location"": ""TJ"", ""is_mobile"": true}" 1968,1,44,2017-01-10 20:35:20,http://murphy.co/christina,,7.2.37.203,"{""location"": ""SG"", ""is_mobile"": true}" 1969,1,44,2016-12-26 21:37:05,http://will.name/vergie,,220.98.114.218,"{""location"": ""LV"", ""is_mobile"": false}" 1970,1,44,2016-12-28 06:18:17,http://kovacekgaylord.co/violette,,215.89.149.198,"{""location"": ""US"", ""is_mobile"": false}" 1971,1,44,2017-03-11 12:05:41,http://beatty.net/yesenia,,234.40.19.160,"{""location"": ""DM"", ""is_mobile"": false}" 1972,1,44,2017-04-19 18:28:57,http://mckenziefadel.name/coby,,145.94.166.181,"{""location"": ""AT"", ""is_mobile"": true}" 1973,1,44,2017-01-21 11:32:30,http://wisozkcrooks.io/julien_corkery,,252.175.221.18,"{""location"": ""RU"", ""is_mobile"": false}" 22992,8,516,2017-05-15 19:37:28,http://conroy.io/serena.steuber,,20.50.39.25,"{""location"": ""AU"", ""is_mobile"": true}" 22993,8,516,2017-01-22 07:58:28,http://hodkiewicz.org/rey_cole,,173.139.113.126,"{""location"": ""LC"", ""is_mobile"": true}" 22994,8,516,2016-12-24 23:25:31,http://hudsonhartmann.io/lemuel.stehr,,193.238.143.68,"{""location"": ""RE"", ""is_mobile"": false}" 22995,8,516,2017-03-18 06:55:11,http://weinat.info/bonita,,7.222.172.223,"{""location"": ""SE"", ""is_mobile"": true}" 22996,8,516,2017-05-18 00:02:32,http://grant.com/lila,,210.178.231.66,"{""location"": ""MM"", ""is_mobile"": true}" 22997,8,516,2017-01-19 07:27:40,http://monahan.net/shemar,,56.64.142.138,"{""location"": ""NI"", ""is_mobile"": true}" 22998,8,516,2017-02-20 15:19:56,http://considine.io/tevin,,149.150.242.45,"{""location"": ""ZM"", ""is_mobile"": false}" 22999,8,516,2016-12-20 00:09:34,http://mosciski.biz/rosella_littel,,200.96.180.237,"{""location"": ""RW"", ""is_mobile"": true}" 23000,8,516,2017-05-12 14:55:17,http://smith.biz/marcella,,185.146.59.101,"{""location"": ""KI"", ""is_mobile"": true}" 23001,8,516,2017-05-17 00:27:18,http://swiftdicki.com/aiden_west,,145.46.40.193,"{""location"": ""LU"", ""is_mobile"": true}" 23002,8,516,2017-03-21 04:30:37,http://kuhlman.name/madaline,,254.156.221.77,"{""location"": ""JO"", ""is_mobile"": true}" 23003,8,516,2017-03-13 12:39:43,http://deckowsanford.name/toney,,27.226.114.134,"{""location"": ""KE"", ""is_mobile"": false}" 23004,8,516,2017-05-02 06:40:16,http://reichel.com/deonte.swaniawski,,219.157.156.28,"{""location"": ""GN"", ""is_mobile"": true}" 23005,8,516,2017-01-27 19:02:40,http://fay.biz/johnson,,234.134.175.231,"{""location"": ""EH"", ""is_mobile"": false}" 23006,8,516,2017-04-30 06:20:43,http://gerhold.info/donny,,200.123.146.113,"{""location"": ""MV"", ""is_mobile"": false}" 23007,8,516,2017-03-31 02:57:11,http://roob.net/alexie,,98.144.154.144,"{""location"": ""PN"", ""is_mobile"": false}" 23008,8,516,2017-03-25 05:43:38,http://bartell.name/edwin_greenfelder,,235.16.232.71,"{""location"": ""EE"", ""is_mobile"": true}" 23009,8,516,2017-01-20 01:11:09,http://mayer.io/jamaal,,157.110.245.225,"{""location"": ""SN"", ""is_mobile"": true}" 23010,8,516,2017-02-14 22:05:23,http://crist.name/sandrine_trantow,,26.176.138.128,"{""location"": ""JM"", ""is_mobile"": false}" 23011,8,516,2016-12-27 16:49:04,http://rau.name/harold.hettinger,,199.170.54.250,"{""location"": ""CO"", ""is_mobile"": true}" 23012,8,516,2017-05-25 17:16:20,http://kilbacklang.co/pearline,,252.38.213.230,"{""location"": ""IE"", ""is_mobile"": false}" 23013,8,516,2017-04-26 01:09:52,http://rauframi.info/river_upton,,126.178.138.159,"{""location"": ""TZ"", ""is_mobile"": true}" 23014,8,516,2017-02-02 18:08:16,http://bahringer.biz/luis,,189.208.194.177,"{""location"": ""IE"", ""is_mobile"": false}" 23015,8,516,2017-05-26 08:45:05,http://reynolds.info/isobel,,39.40.37.124,"{""location"": ""NE"", ""is_mobile"": true}" 23016,8,516,2017-03-24 19:08:25,http://predovic.com/rachel_stamm,,135.240.208.22,"{""location"": ""PN"", ""is_mobile"": false}" 23017,8,516,2017-04-11 19:51:03,http://mcdermottwill.biz/caleb_mccullough,,140.127.197.14,"{""location"": ""VG"", ""is_mobile"": false}" 23019,8,516,2017-04-23 20:14:10,http://casper.co/pinkie,,251.126.92.153,"{""location"": ""HN"", ""is_mobile"": false}" 23020,8,516,2017-02-23 18:03:54,http://wildermanrosenbaum.co/effie,,62.2.141.199,"{""location"": ""TT"", ""is_mobile"": false}" 23021,8,516,2017-03-08 02:07:17,http://ullrich.net/beryl.bergstrom,,33.246.121.161,"{""location"": ""DO"", ""is_mobile"": false}" 23022,8,516,2017-06-07 16:13:47,http://kozey.co/maurine_shields,,237.178.141.13,"{""location"": ""TN"", ""is_mobile"": true}" 23023,8,516,2017-04-09 18:08:11,http://volkman.name/sherman_macejkovic,,182.57.226.21,"{""location"": ""BY"", ""is_mobile"": true}" 23024,8,516,2016-12-23 12:03:17,http://monahan.io/mafalda,,250.71.64.42,"{""location"": ""GF"", ""is_mobile"": true}" 23025,8,516,2017-02-12 23:00:41,http://steuberchristiansen.biz/jordy,,144.158.34.52,"{""location"": ""CH"", ""is_mobile"": false}" 23026,8,516,2016-12-14 10:01:59,http://franecki.co/sadie.leannon,,28.101.114.173,"{""location"": ""AR"", ""is_mobile"": false}" 23027,8,516,2016-12-16 13:52:07,http://champlinmraz.io/harley_simonis,,68.60.182.243,"{""location"": ""PM"", ""is_mobile"": false}" 23028,8,517,2017-02-14 13:16:30,http://koelpinsipes.biz/geovanni.eichmann,,252.175.214.144,"{""location"": ""VE"", ""is_mobile"": true}" 23029,8,517,2017-02-15 19:19:11,http://wehnerbogan.co/louie,,141.173.69.59,"{""location"": ""MM"", ""is_mobile"": true}" 23030,8,517,2017-02-04 00:45:02,http://oreilly.com/turner,,51.176.238.12,"{""location"": ""UY"", ""is_mobile"": true}" 23031,8,517,2017-03-07 18:52:58,http://reingermcglynn.info/rickie_mayer,,69.84.36.156,"{""location"": ""VC"", ""is_mobile"": false}" 23032,8,517,2017-05-14 01:19:24,http://lind.info/hank.sanford,,252.199.25.201,"{""location"": ""SG"", ""is_mobile"": false}" 23033,8,517,2017-03-25 12:21:28,http://lynch.co/lonzo,,146.224.87.36,"{""location"": ""UY"", ""is_mobile"": true}" 23034,8,517,2017-03-15 23:20:14,http://cormierweber.co/zoey,,170.60.105.205,"{""location"": ""CZ"", ""is_mobile"": false}" 23035,8,517,2017-05-22 16:30:33,http://christiansencollier.com/rahul.terry,,202.128.107.213,"{""location"": ""PA"", ""is_mobile"": false}" 23036,8,517,2017-04-28 04:12:50,http://fay.org/alf,,218.177.246.177,"{""location"": ""BJ"", ""is_mobile"": true}" 23037,8,517,2017-05-08 23:28:23,http://okeefe.biz/maxine_keebler,,21.137.234.226,"{""location"": ""MG"", ""is_mobile"": false}" 23038,8,517,2016-12-27 20:08:48,http://gusikowski.biz/tamara.deckow,,73.163.60.198,"{""location"": ""RE"", ""is_mobile"": true}" 23039,8,517,2017-01-28 13:40:36,http://weimannhudson.com/jeromy,,141.116.47.59,"{""location"": ""NA"", ""is_mobile"": true}" 23040,8,517,2017-03-24 20:09:10,http://kihn.net/estelle.kuhn,,6.132.64.231,"{""location"": ""TN"", ""is_mobile"": false}" 23041,8,517,2017-05-10 01:06:28,http://schuster.name/jacques.bosco,,162.134.106.85,"{""location"": ""BZ"", ""is_mobile"": false}" 23042,8,517,2017-04-09 21:37:37,http://runteoconnell.info/beryl,,160.146.224.207,"{""location"": ""CF"", ""is_mobile"": false}" 23043,8,517,2017-03-24 19:08:24,http://green.name/darian,,177.87.36.75,"{""location"": ""NO"", ""is_mobile"": true}" 23044,8,517,2017-03-17 13:45:46,http://feest.org/lucile_hilpert,,39.114.39.139,"{""location"": ""BQ"", ""is_mobile"": true}" 23045,8,517,2017-01-10 00:15:03,http://white.org/billy,,149.7.54.5,"{""location"": ""JE"", ""is_mobile"": true}" 23046,8,517,2017-04-30 02:48:39,http://littel.net/aleen,,196.44.135.226,"{""location"": ""FI"", ""is_mobile"": false}" 23047,8,517,2017-03-11 06:12:02,http://gleichnerjerde.com/haley,,240.141.36.245,"{""location"": ""BI"", ""is_mobile"": true}" 23048,8,517,2017-02-08 10:07:21,http://hilll.org/madaline.kling,,206.33.87.196,"{""location"": ""MR"", ""is_mobile"": true}" 1974,1,44,2017-01-27 01:52:14,http://faybosco.name/leonardo.schoen,,131.64.9.245,"{""location"": ""BZ"", ""is_mobile"": false}" 1975,1,44,2017-03-23 22:47:35,http://hilpert.name/estelle_lakin,,250.218.161.86,"{""location"": ""GE"", ""is_mobile"": false}" 1976,1,44,2017-02-10 11:32:17,http://larkin.net/francesco,,59.170.76.193,"{""location"": ""SC"", ""is_mobile"": true}" 1977,1,44,2017-01-17 15:22:56,http://beerwehner.biz/maybell.spinka,,91.118.140.162,"{""location"": ""ZW"", ""is_mobile"": true}" 1978,1,44,2017-01-16 01:51:28,http://reichertlubowitz.org/mckayla,,150.180.98.209,"{""location"": ""FM"", ""is_mobile"": true}" 1979,1,44,2017-05-18 22:22:44,http://pfannerstill.co/agustina,,129.117.3.120,"{""location"": ""LB"", ""is_mobile"": false}" 1980,1,44,2017-05-25 23:29:14,http://hane.io/melvin,,76.188.74.31,"{""location"": ""LB"", ""is_mobile"": false}" 1981,1,44,2017-05-28 13:21:38,http://balistrerimayer.info/lowell_abbott,,27.76.138.248,"{""location"": ""FR"", ""is_mobile"": true}" 1982,1,44,2017-03-19 06:16:46,http://quitzon.co/trevion,,179.232.32.244,"{""location"": ""BB"", ""is_mobile"": true}" 1983,1,44,2017-02-13 18:44:04,http://framigibson.name/jamil,,179.172.206.46,"{""location"": ""SA"", ""is_mobile"": false}" 1984,1,44,2017-04-08 23:52:16,http://tromp.org/noelia_wolff,,21.26.53.45,"{""location"": ""AO"", ""is_mobile"": true}" 1985,1,44,2017-06-13 06:27:28,http://hane.com/ahmad,,232.177.200.115,"{""location"": ""ML"", ""is_mobile"": true}" 1986,1,44,2017-05-07 10:06:11,http://bergnaumbaumbach.co/haylee.toy,,13.29.123.63,"{""location"": ""TN"", ""is_mobile"": false}" 1987,1,44,2017-03-26 14:57:28,http://maggiomosciski.com/itzel_glover,,183.245.173.220,"{""location"": ""RW"", ""is_mobile"": true}" 1988,1,44,2017-04-14 03:53:13,http://johns.info/martin,,244.240.228.67,"{""location"": ""ET"", ""is_mobile"": true}" 1989,1,44,2017-05-29 04:13:16,http://harberframi.com/brock.hills,,225.117.26.112,"{""location"": ""BN"", ""is_mobile"": false}" 1990,1,44,2017-05-16 13:02:59,http://crona.org/ada_schowalter,,72.166.120.241,"{""location"": ""MQ"", ""is_mobile"": true}" 1991,1,44,2017-01-20 03:11:43,http://kubhauck.io/reina,,229.28.139.242,"{""location"": ""HR"", ""is_mobile"": false}" 1992,1,44,2017-03-03 09:59:33,http://harber.net/brock.kutch,,160.200.78.19,"{""location"": ""IQ"", ""is_mobile"": true}" 1993,1,44,2017-04-19 16:16:56,http://towneemard.net/faustino_grant,,7.59.32.185,"{""location"": ""KY"", ""is_mobile"": false}" 1994,1,44,2017-03-26 13:58:13,http://millerkozey.io/justyn,,210.235.156.78,"{""location"": ""DZ"", ""is_mobile"": true}" 1995,1,44,2017-03-18 02:07:18,http://schneider.info/susie,,110.41.200.20,"{""location"": ""ME"", ""is_mobile"": true}" 1996,1,44,2017-04-16 03:59:09,http://wilkinson.com/mortimer,,33.52.130.118,"{""location"": ""KE"", ""is_mobile"": false}" 1997,1,44,2017-01-26 23:54:36,http://schimmelschimmel.com/bonita,,122.119.37.198,"{""location"": ""SK"", ""is_mobile"": false}" 1998,1,44,2016-12-23 17:29:58,http://purdybeier.io/daryl_hintz,,242.169.31.198,"{""location"": ""TO"", ""is_mobile"": false}" 1999,1,45,2017-05-15 05:32:53,http://koch.io/ebony,,168.35.49.194,"{""location"": ""GE"", ""is_mobile"": true}" 2000,1,45,2017-04-07 08:24:14,http://mclaughlin.biz/connor_mertz,,50.242.196.128,"{""location"": ""IM"", ""is_mobile"": false}" 2001,1,45,2017-03-25 07:49:28,http://gerhold.net/domenic,,67.156.123.73,"{""location"": ""CW"", ""is_mobile"": true}" 2002,1,45,2017-05-02 20:24:14,http://hegmannkohler.name/john.kautzer,,187.22.126.51,"{""location"": ""ZA"", ""is_mobile"": true}" 2003,1,45,2017-03-16 19:56:26,http://watsica.org/burnice_cole,,147.41.156.123,"{""location"": ""PY"", ""is_mobile"": false}" 2004,1,45,2017-03-22 09:03:37,http://douglas.co/lupe_yundt,,15.89.103.157,"{""location"": ""DZ"", ""is_mobile"": true}" 2005,1,45,2017-05-28 19:11:05,http://ricechristiansen.org/alvis_bednar,,111.159.71.184,"{""location"": ""GF"", ""is_mobile"": false}" 2006,1,45,2017-04-11 07:24:05,http://walsh.name/chandler_pagac,,15.49.128.21,"{""location"": ""MV"", ""is_mobile"": true}" 2007,1,45,2017-04-12 08:01:15,http://prosacco.io/torrey,,134.65.51.101,"{""location"": ""PF"", ""is_mobile"": true}" 2008,1,45,2017-04-23 23:36:23,http://armstrong.info/johnson_auer,,80.96.163.145,"{""location"": ""MW"", ""is_mobile"": false}" 2009,1,45,2017-01-06 16:23:13,http://pollich.com/norbert,,27.85.215.23,"{""location"": ""CL"", ""is_mobile"": true}" 2010,1,45,2017-03-17 14:42:17,http://schambergerreilly.io/zachary,,205.125.77.136,"{""location"": ""IT"", ""is_mobile"": false}" 2011,1,45,2017-03-15 20:40:24,http://legroscorwin.net/fleta,,163.16.118.211,"{""location"": ""PE"", ""is_mobile"": false}" 2012,1,45,2017-01-08 00:26:13,http://quigley.info/joaquin,,22.4.216.11,"{""location"": ""MZ"", ""is_mobile"": true}" 2013,1,45,2017-03-02 07:18:48,http://cain.io/gilda,,28.251.174.219,"{""location"": ""JE"", ""is_mobile"": false}" 2014,1,45,2017-02-26 00:25:28,http://keebler.co/maximo.flatley,,63.161.204.198,"{""location"": ""AG"", ""is_mobile"": true}" 2015,1,45,2017-03-06 00:40:27,http://breitenberg.net/miguel_fay,,152.107.172.120,"{""location"": ""TH"", ""is_mobile"": true}" 2016,1,45,2017-04-06 10:08:12,http://dickinsonkreiger.name/emilia,,220.205.192.6,"{""location"": ""AL"", ""is_mobile"": false}" 2017,1,45,2017-01-30 00:29:06,http://pfannerstillrippin.name/grover.smitham,,72.155.240.217,"{""location"": ""CD"", ""is_mobile"": false}" 2018,1,45,2017-01-03 10:57:16,http://kozeydenesik.net/jerry.hermann,,147.4.16.90,"{""location"": ""KN"", ""is_mobile"": false}" 2019,1,45,2017-01-09 13:23:54,http://ruelaltenwerth.co/silas,,163.31.170.30,"{""location"": ""TG"", ""is_mobile"": false}" 2021,1,45,2017-05-27 09:15:40,http://heidenreich.com/curtis,,52.80.152.139,"{""location"": ""FK"", ""is_mobile"": true}" 2022,1,45,2017-04-28 10:20:52,http://harvey.com/garrett_boyle,,58.12.41.87,"{""location"": ""IL"", ""is_mobile"": false}" 2023,1,45,2017-03-17 06:13:22,http://schuppe.co/scottie,,98.156.168.195,"{""location"": ""CM"", ""is_mobile"": true}" 2024,1,45,2017-05-14 13:01:23,http://johns.org/raegan,,164.241.75.15,"{""location"": ""LI"", ""is_mobile"": true}" 2025,1,45,2017-02-19 17:28:37,http://gottlieb.io/susie,,217.70.183.95,"{""location"": ""NL"", ""is_mobile"": false}" 2026,1,45,2017-01-29 20:26:02,http://oreilly.io/janae_welch,,39.23.44.142,"{""location"": ""OM"", ""is_mobile"": false}" 2027,1,45,2017-01-28 10:10:53,http://white.name/jeramie.will,,94.41.7.28,"{""location"": ""PN"", ""is_mobile"": true}" 2028,1,45,2017-02-28 07:53:02,http://hyatt.com/carleton_ritchie,,111.192.110.72,"{""location"": ""BZ"", ""is_mobile"": true}" 2029,1,45,2017-03-10 13:30:11,http://lebsack.name/henri,,145.254.109.159,"{""location"": ""CH"", ""is_mobile"": true}" 2030,1,45,2017-05-23 07:21:04,http://johnsonbechtelar.io/annamarie.jenkins,,47.37.132.204,"{""location"": ""GT"", ""is_mobile"": true}" 23049,8,517,2017-02-27 20:30:43,http://mccullough.org/may.lehner,,72.219.131.115,"{""location"": ""MM"", ""is_mobile"": true}" 23050,8,517,2017-01-06 09:47:19,http://sawayn.net/louie_heel,,50.35.154.93,"{""location"": ""AW"", ""is_mobile"": false}" 23051,8,517,2017-02-21 05:21:17,http://stanton.io/ivy.maggio,,170.196.100.105,"{""location"": ""WS"", ""is_mobile"": false}" 23052,8,517,2017-03-31 20:21:13,http://tremblay.net/lila,,201.159.32.108,"{""location"": ""LB"", ""is_mobile"": true}" 23053,8,517,2017-05-25 03:29:07,http://ferry.info/alexandria,,45.167.163.254,"{""location"": ""SG"", ""is_mobile"": false}" 23054,8,517,2017-04-28 15:32:26,http://haag.io/corine.kulas,,109.217.108.14,"{""location"": ""CG"", ""is_mobile"": true}" 23055,8,517,2017-04-28 08:23:41,http://wizakunze.name/dorian_hudson,,205.8.163.70,"{""location"": ""LK"", ""is_mobile"": true}" 23056,8,517,2017-01-26 18:15:36,http://little.io/aurelie_kulas,,73.207.190.109,"{""location"": ""IE"", ""is_mobile"": true}" 23057,8,517,2017-01-13 18:45:30,http://collier.io/margarett,,243.60.167.132,"{""location"": ""CG"", ""is_mobile"": true}" 23058,8,517,2016-12-30 07:32:08,http://vandervortweimann.biz/jazmyne,,139.58.140.142,"{""location"": ""GR"", ""is_mobile"": false}" 23059,8,517,2017-01-28 06:17:16,http://rempelsanford.co/lorenz,,151.170.219.5,"{""location"": ""YE"", ""is_mobile"": true}" 23060,8,517,2017-02-25 18:34:19,http://macgyver.co/wilfrid.jacobi,,222.75.42.163,"{""location"": ""MD"", ""is_mobile"": false}" 23061,8,517,2016-12-31 10:19:10,http://moen.io/macie_hagenes,,204.252.122.219,"{""location"": ""CN"", ""is_mobile"": true}" 23062,8,517,2017-03-29 19:24:04,http://gutkowski.com/etha.bauch,,154.196.142.87,"{""location"": ""NL"", ""is_mobile"": true}" 23063,8,517,2017-02-07 22:06:19,http://ritchiecarter.info/alejandrin,,53.240.156.251,"{""location"": ""KE"", ""is_mobile"": true}" 23064,8,517,2017-01-08 14:05:20,http://von.co/eli,,163.72.158.20,"{""location"": ""DJ"", ""is_mobile"": false}" 23065,8,517,2016-12-18 01:09:54,http://kunze.com/maryam.romaguera,,3.242.233.40,"{""location"": ""HK"", ""is_mobile"": true}" 23066,8,517,2017-06-02 16:09:58,http://blick.com/lorenzo,,218.113.161.40,"{""location"": ""KY"", ""is_mobile"": true}" 23067,8,517,2017-04-19 15:32:34,http://boyerhilpert.name/kaylee.miller,,221.227.14.213,"{""location"": ""ID"", ""is_mobile"": true}" 23068,8,517,2017-03-17 18:00:43,http://stromanstamm.biz/meaghan_ferry,,249.20.199.202,"{""location"": ""GY"", ""is_mobile"": false}" 23069,8,517,2017-02-25 22:36:12,http://kuvalis.co/shaun_parker,,57.78.168.108,"{""location"": ""SK"", ""is_mobile"": true}" 23070,8,517,2017-01-03 10:22:52,http://homenick.net/rachel_hauck,,203.108.200.214,"{""location"": ""IN"", ""is_mobile"": true}" 23071,8,517,2016-12-30 08:30:58,http://beckerhackett.info/ofelia,,154.17.208.27,"{""location"": ""GY"", ""is_mobile"": true}" 23072,8,517,2017-02-06 01:47:34,http://wuckert.org/max_cole,,27.190.37.96,"{""location"": ""IT"", ""is_mobile"": true}" 23073,8,517,2017-04-20 10:32:28,http://ondricka.com/danial,,37.17.238.179,"{""location"": ""MA"", ""is_mobile"": true}" 23074,8,517,2017-04-01 21:59:06,http://barrowscronin.biz/lucile.frami,,30.125.90.45,"{""location"": ""AT"", ""is_mobile"": false}" 23075,8,517,2017-04-08 17:16:05,http://dubuque.net/darren,,104.226.24.26,"{""location"": ""MD"", ""is_mobile"": false}" 23076,8,517,2017-04-25 04:57:36,http://hodkiewicz.info/adolph,,147.192.15.54,"{""location"": ""LT"", ""is_mobile"": true}" 23077,8,517,2017-03-20 19:31:06,http://hodkiewiczkaulke.net/jacynthe.gibson,,93.119.23.8,"{""location"": ""GD"", ""is_mobile"": false}" 23078,8,517,2016-12-19 01:19:11,http://abshirejacobs.org/dexter_hintz,,34.195.129.101,"{""location"": ""KE"", ""is_mobile"": false}" 23079,8,517,2017-01-04 23:25:20,http://gottlieb.name/anabelle.denesik,,57.152.64.49,"{""location"": ""BI"", ""is_mobile"": false}" 23080,8,517,2017-05-13 19:34:20,http://okon.io/rosie_emmerich,,147.194.225.190,"{""location"": ""LU"", ""is_mobile"": false}" 23081,8,517,2017-02-04 14:57:47,http://greendickinson.io/jade,,242.42.100.132,"{""location"": ""TV"", ""is_mobile"": false}" 23082,8,517,2017-05-02 23:20:29,http://stehrhayes.com/thaddeus,,76.129.41.252,"{""location"": ""BI"", ""is_mobile"": false}" 23083,8,517,2017-03-15 09:08:03,http://zboncakhauck.io/horacio.rolfson,,49.248.197.209,"{""location"": ""KH"", ""is_mobile"": true}" 23084,8,517,2017-02-24 02:55:10,http://prosacco.com/rosie.kihn,,48.86.132.153,"{""location"": ""GP"", ""is_mobile"": true}" 23085,8,517,2017-05-08 00:01:56,http://herman.biz/fannie_reichert,,52.239.105.222,"{""location"": ""AG"", ""is_mobile"": true}" 23086,8,517,2017-04-06 04:10:32,http://predovic.io/orlo_abshire,,198.186.181.74,"{""location"": ""ES"", ""is_mobile"": false}" 23087,8,517,2017-05-06 18:51:10,http://mohr.biz/cruz,,87.142.212.85,"{""location"": ""SO"", ""is_mobile"": true}" 23088,8,517,2016-12-25 14:57:20,http://stroman.biz/oda,,220.83.176.126,"{""location"": ""MA"", ""is_mobile"": false}" 23089,8,517,2017-03-25 09:51:18,http://ferry.info/skye_kovacek,,238.203.62.222,"{""location"": ""BY"", ""is_mobile"": true}" 23090,8,517,2017-05-11 01:44:23,http://hartmann.net/kenna,,30.204.31.207,"{""location"": ""IE"", ""is_mobile"": false}" 23091,8,517,2016-12-23 12:32:19,http://harvey.io/modesta,,108.239.91.107,"{""location"": ""IS"", ""is_mobile"": true}" 23092,8,517,2017-04-05 19:38:46,http://heathcote.net/green.schultz,,253.47.7.25,"{""location"": ""IE"", ""is_mobile"": true}" 23093,8,517,2017-05-05 01:33:21,http://cronin.biz/imani,,127.232.122.71,"{""location"": ""TK"", ""is_mobile"": true}" 23094,8,517,2017-02-22 21:30:45,http://reillyrosenbaum.com/isidro.legros,,156.216.219.110,"{""location"": ""SM"", ""is_mobile"": false}" 23095,8,517,2017-02-21 18:47:14,http://wisoky.io/rosemary.ferry,,158.215.167.88,"{""location"": ""ZA"", ""is_mobile"": true}" 23096,8,518,2017-05-04 02:12:47,http://bergnaumkuvalis.info/grady_mcglynn,,172.117.220.99,"{""location"": ""VI"", ""is_mobile"": false}" 23097,8,518,2017-04-28 10:24:08,http://kleinjohnston.info/kaylah,,56.203.118.231,"{""location"": ""SR"", ""is_mobile"": true}" 23098,8,518,2017-02-09 21:26:54,http://larsonrunolfsdottir.info/curt,,148.223.201.124,"{""location"": ""LV"", ""is_mobile"": false}" 23099,8,518,2017-04-13 16:56:49,http://powlowski.com/dillan_zemlak,,18.154.163.112,"{""location"": ""HT"", ""is_mobile"": false}" 23100,8,518,2017-01-07 01:27:37,http://luettgen.net/elna.bogisich,,165.10.204.214,"{""location"": ""JO"", ""is_mobile"": true}" 23101,8,518,2017-06-09 08:14:55,http://ortiz.info/alice_raynor,,185.130.74.92,"{""location"": ""MD"", ""is_mobile"": false}" 23102,8,518,2017-04-15 11:53:13,http://hartmann.info/johann,,189.66.110.233,"{""location"": ""SK"", ""is_mobile"": false}" 23103,8,518,2017-03-22 07:48:59,http://hintz.com/miouri,,25.135.139.35,"{""location"": ""RO"", ""is_mobile"": false}" 2031,1,45,2016-12-15 23:12:13,http://dickidibbert.com/coleman,,29.137.83.69,"{""location"": ""SY"", ""is_mobile"": false}" 2032,1,45,2016-12-15 12:30:40,http://deckow.org/damon,,229.48.127.178,"{""location"": ""MH"", ""is_mobile"": false}" 2033,1,45,2017-01-30 06:06:04,http://vandervort.org/roberto_macejkovic,,73.12.79.101,"{""location"": ""SL"", ""is_mobile"": false}" 2034,1,45,2017-02-04 14:21:57,http://braun.name/forest_zemlak,,232.222.230.31,"{""location"": ""PK"", ""is_mobile"": false}" 2035,1,46,2016-12-16 08:54:04,http://walsh.net/annamarie.fay,,36.241.75.95,"{""location"": ""AG"", ""is_mobile"": true}" 2036,1,46,2017-06-11 18:42:24,http://homenickhodkiewicz.info/olin.konopelski,,16.47.209.64,"{""location"": ""TK"", ""is_mobile"": false}" 2037,1,46,2017-01-31 17:55:52,http://haley.name/terrill.lemke,,53.125.188.61,"{""location"": ""MQ"", ""is_mobile"": false}" 2038,1,46,2017-05-01 10:42:01,http://vandervortstreich.co/raegan.barton,,82.245.95.132,"{""location"": ""LY"", ""is_mobile"": false}" 2039,1,46,2017-01-31 10:20:08,http://walter.org/rosendo.koepp,,169.253.237.198,"{""location"": ""NA"", ""is_mobile"": true}" 2040,1,46,2016-12-30 15:00:04,http://quigley.co/nadia_sauer,,254.209.239.47,"{""location"": ""SV"", ""is_mobile"": true}" 2041,1,46,2017-02-01 12:27:30,http://satterfield.biz/jaleel.kreiger,,225.249.166.47,"{""location"": ""VC"", ""is_mobile"": false}" 2042,1,46,2016-12-20 23:36:14,http://friesen.co/arvel_mclaughlin,,32.186.8.184,"{""location"": ""JM"", ""is_mobile"": true}" 2043,1,46,2017-02-26 17:48:15,http://gutkowskirice.io/shaniya,,77.158.221.104,"{""location"": ""PL"", ""is_mobile"": false}" 2044,1,46,2017-03-31 16:17:46,http://ullrich.io/donny_ohara,,28.225.210.94,"{""location"": ""GR"", ""is_mobile"": false}" 2045,1,46,2017-03-09 02:38:21,http://raynorbartell.com/anthony,,217.70.215.91,"{""location"": ""ER"", ""is_mobile"": true}" 2046,1,46,2017-04-21 19:51:16,http://buckridge.com/anabelle_bergnaum,,22.131.41.39,"{""location"": ""LB"", ""is_mobile"": false}" 2047,1,46,2017-05-10 12:47:13,http://schroeder.name/ari,,235.49.207.188,"{""location"": ""SG"", ""is_mobile"": true}" 2048,1,46,2016-12-27 14:40:08,http://dareabshire.name/robert,,199.183.199.166,"{""location"": ""JO"", ""is_mobile"": true}" 2049,1,46,2017-02-03 08:43:57,http://osinski.org/luisa.bode,,108.102.64.78,"{""location"": ""AT"", ""is_mobile"": false}" 2050,1,46,2017-03-07 20:05:21,http://smitham.io/malika,,2.239.41.138,"{""location"": ""GH"", ""is_mobile"": false}" 2051,1,46,2017-04-16 12:47:39,http://cummings.net/enoch.oreilly,,97.31.4.88,"{""location"": ""MG"", ""is_mobile"": false}" 2052,1,46,2017-04-08 07:31:02,http://dickens.name/brady_schumm,,239.21.93.52,"{""location"": ""BV"", ""is_mobile"": true}" 2053,1,46,2017-03-18 16:55:09,http://smitham.net/pablo,,41.140.108.64,"{""location"": ""MQ"", ""is_mobile"": true}" 2054,1,46,2017-01-10 16:03:42,http://fayhilpert.net/rusty.morar,,72.233.9.17,"{""location"": ""TL"", ""is_mobile"": false}" 2055,1,46,2017-02-02 02:24:44,http://gerlach.biz/kaitlin,,110.56.99.166,"{""location"": ""SS"", ""is_mobile"": true}" 2056,1,46,2017-03-24 05:58:51,http://vonruedengoyette.info/nathaniel,,45.89.180.220,"{""location"": ""LU"", ""is_mobile"": false}" 2057,1,46,2017-02-20 21:20:45,http://nicolas.co/ricky_nikolaus,,149.14.39.157,"{""location"": ""TF"", ""is_mobile"": false}" 2058,1,46,2017-05-20 06:38:46,http://rosenbaum.name/abraham,,71.99.48.98,"{""location"": ""BI"", ""is_mobile"": true}" 2059,1,46,2017-01-20 17:52:22,http://altenwerthgreenfelder.io/garrett,,149.187.205.167,"{""location"": ""FI"", ""is_mobile"": true}" 2060,1,46,2017-03-09 10:17:55,http://homenickward.name/arvilla,,137.158.20.217,"{""location"": ""KY"", ""is_mobile"": true}" 2061,1,46,2017-05-17 03:19:56,http://marksdoyle.org/pierre_will,,142.64.7.186,"{""location"": ""SV"", ""is_mobile"": true}" 2062,1,46,2017-01-19 18:08:37,http://haag.biz/frida_kilback,,239.190.41.136,"{""location"": ""TC"", ""is_mobile"": true}" 2063,1,46,2017-01-18 19:10:19,http://kris.biz/ken.leffler,,101.45.9.175,"{""location"": ""GR"", ""is_mobile"": false}" 2064,1,46,2017-04-27 14:26:41,http://nicolas.name/velma,,115.178.90.142,"{""location"": ""BT"", ""is_mobile"": false}" 2065,1,46,2016-12-28 22:31:00,http://conn.net/vivianne,,61.71.217.229,"{""location"": ""MO"", ""is_mobile"": true}" 2066,1,46,2017-02-06 20:03:17,http://goodwin.com/mohamed_lowe,,186.180.76.152,"{""location"": ""NA"", ""is_mobile"": true}" 2067,1,46,2017-03-29 13:16:24,http://skilesbergstrom.co/casimir,,253.73.27.194,"{""location"": ""AE"", ""is_mobile"": true}" 2068,1,46,2016-12-20 20:18:07,http://hartmann.co/bette.mclaughlin,,244.105.239.47,"{""location"": ""TG"", ""is_mobile"": true}" 2069,1,46,2017-04-24 00:15:38,http://weimannshields.com/megane.grimes,,47.79.126.156,"{""location"": ""AI"", ""is_mobile"": true}" 2070,1,46,2017-03-09 15:44:20,http://hoppestark.io/ozzie,,222.167.152.44,"{""location"": ""OM"", ""is_mobile"": true}" 2071,1,46,2017-04-02 20:55:37,http://smitham.org/sienna_kling,,181.93.223.42,"{""location"": ""CG"", ""is_mobile"": false}" 2072,1,46,2017-02-23 22:24:04,http://harris.info/trinity,,13.73.92.22,"{""location"": ""KH"", ""is_mobile"": true}" 2073,1,46,2016-12-23 23:58:59,http://jenkins.name/braxton.tillman,,217.169.84.77,"{""location"": ""RU"", ""is_mobile"": false}" 2074,1,46,2017-01-01 07:09:13,http://wiegandlockman.name/juliet,,22.247.15.145,"{""location"": ""NR"", ""is_mobile"": false}" 2075,1,46,2017-03-22 23:36:03,http://bradtke.com/bradly.beer,,26.204.77.123,"{""location"": ""NI"", ""is_mobile"": false}" 2076,1,46,2017-02-16 11:59:16,http://ratke.info/adonis.carter,,102.203.230.178,"{""location"": ""IS"", ""is_mobile"": true}" 2077,1,46,2017-03-25 12:17:14,http://spencer.com/william,,89.202.107.154,"{""location"": ""HM"", ""is_mobile"": false}" 2078,1,46,2017-05-03 01:01:34,http://jastpaucek.io/myrtle.kemmer,,169.197.38.244,"{""location"": ""PR"", ""is_mobile"": true}" 2079,1,46,2017-03-11 02:39:48,http://lockman.net/emile,,117.21.111.67,"{""location"": ""LC"", ""is_mobile"": true}" 2080,1,46,2017-02-17 18:06:11,http://hudson.biz/aiyana,,121.222.127.94,"{""location"": ""BY"", ""is_mobile"": false}" 2081,1,46,2017-02-26 10:58:09,http://sipes.org/laney,,144.32.110.162,"{""location"": ""MM"", ""is_mobile"": false}" 2082,1,46,2017-02-01 06:58:17,http://gradyherzog.com/toney,,108.51.80.7,"{""location"": ""SC"", ""is_mobile"": true}" 2083,1,46,2017-02-20 22:31:29,http://littel.co/talon_hirthe,,107.116.246.23,"{""location"": ""GI"", ""is_mobile"": true}" 2084,1,46,2017-06-11 06:56:28,http://lubowitz.com/elza,,242.36.59.99,"{""location"": ""SV"", ""is_mobile"": false}" 2085,1,46,2017-02-12 04:56:44,http://kreiger.info/agustin.murazik,,109.14.140.39,"{""location"": ""BI"", ""is_mobile"": false}" 2086,1,46,2017-05-21 15:13:47,http://buckridgerempel.org/orland,,68.187.177.175,"{""location"": ""CM"", ""is_mobile"": false}" 23104,8,518,2017-03-05 18:40:45,http://greenholt.biz/terrill,,133.93.192.241,"{""location"": ""IQ"", ""is_mobile"": false}" 23105,8,518,2017-04-15 03:06:54,http://nienow.biz/emmett,,52.137.228.85,"{""location"": ""CN"", ""is_mobile"": true}" 23106,8,518,2017-04-25 07:35:01,http://labadie.info/monte,,2.102.108.125,"{""location"": ""UG"", ""is_mobile"": true}" 23107,8,518,2017-02-01 02:06:21,http://weinat.io/willa,,186.3.78.70,"{""location"": ""SB"", ""is_mobile"": false}" 23108,8,518,2017-05-10 04:27:07,http://labadie.org/andres.konopelski,,108.93.183.185,"{""location"": ""KZ"", ""is_mobile"": false}" 23109,8,518,2016-12-26 14:38:46,http://herman.com/whitney,,220.31.147.9,"{""location"": ""MC"", ""is_mobile"": true}" 23110,8,518,2017-06-04 19:50:25,http://gislason.biz/stanton.kshlerin,,224.248.44.79,"{""location"": ""EG"", ""is_mobile"": true}" 23111,8,518,2017-02-05 18:28:36,http://weinat.info/waldo.rodriguez,,93.234.90.62,"{""location"": ""GE"", ""is_mobile"": false}" 23112,8,518,2017-05-11 18:03:45,http://kaulkekeeling.net/brett_ernser,,170.165.20.17,"{""location"": ""UA"", ""is_mobile"": true}" 23113,8,518,2017-02-07 12:59:00,http://tremblaymills.io/providenci,,28.75.64.118,"{""location"": ""AX"", ""is_mobile"": false}" 23114,8,518,2017-02-10 21:03:18,http://botsfordprohaska.info/cleora,,143.43.238.43,"{""location"": ""AZ"", ""is_mobile"": true}" 23115,8,518,2017-03-14 19:47:28,http://turnerfisher.org/kaandra,,94.227.207.165,"{""location"": ""UM"", ""is_mobile"": false}" 23116,8,518,2017-05-03 23:21:41,http://feil.biz/dayna_mohr,,183.124.96.242,"{""location"": ""GS"", ""is_mobile"": false}" 23117,8,518,2017-01-05 15:15:03,http://christiansenwelch.net/kade.larkin,,134.228.29.134,"{""location"": ""MU"", ""is_mobile"": false}" 23118,8,518,2016-12-27 09:19:13,http://parker.biz/maye,,109.187.66.58,"{""location"": ""KN"", ""is_mobile"": true}" 23119,8,518,2017-06-12 19:57:35,http://hand.name/conor_raynor,,153.153.111.22,"{""location"": ""KM"", ""is_mobile"": false}" 23120,8,518,2017-03-16 16:57:56,http://wildermanlittel.net/daija,,180.14.175.133,"{""location"": ""QA"", ""is_mobile"": false}" 23121,8,518,2017-06-06 23:07:57,http://walternikolaus.com/zelda,,56.251.219.82,"{""location"": ""ML"", ""is_mobile"": false}" 23122,8,518,2017-05-16 10:23:48,http://hettingergoodwin.info/napoleon,,16.213.51.116,"{""location"": ""GG"", ""is_mobile"": false}" 23123,8,518,2016-12-27 16:31:43,http://bartoletti.io/kaela.schmitt,,66.23.234.68,"{""location"": ""HT"", ""is_mobile"": false}" 23124,8,518,2017-05-02 19:19:25,http://bahringer.org/walker,,61.28.148.247,"{""location"": ""BZ"", ""is_mobile"": true}" 23125,8,518,2017-04-18 01:03:46,http://rippinrice.name/lamont.oconner,,102.197.6.91,"{""location"": ""FI"", ""is_mobile"": false}" 23126,8,518,2017-04-10 07:24:05,http://dachokon.co/rico.kaulke,,39.114.227.186,"{""location"": ""KG"", ""is_mobile"": true}" 23127,8,518,2017-02-04 02:55:12,http://glovercorkery.io/viola,,117.221.173.205,"{""location"": ""MW"", ""is_mobile"": false}" 23128,8,518,2017-03-08 13:59:50,http://kiehn.com/milton.schaefer,,50.234.203.127,"{""location"": ""VE"", ""is_mobile"": true}" 23129,8,518,2016-12-29 05:15:46,http://reilly.info/kara,,205.93.229.162,"{""location"": ""CH"", ""is_mobile"": false}" 23130,8,518,2017-01-23 04:20:23,http://stiedemann.biz/bridget,,207.16.222.156,"{""location"": ""SC"", ""is_mobile"": true}" 23131,8,518,2016-12-20 01:46:27,http://crona.name/darlene,,200.251.208.217,"{""location"": ""MD"", ""is_mobile"": false}" 23132,8,518,2017-05-03 13:56:27,http://schoen.info/sunny_cain,,135.167.61.142,"{""location"": ""ST"", ""is_mobile"": true}" 23133,8,518,2017-03-19 04:29:16,http://spinkahowell.info/ansley,,84.39.80.227,"{""location"": ""FJ"", ""is_mobile"": false}" 23134,8,518,2017-02-02 11:40:56,http://okon.com/fletcher,,54.23.217.181,"{""location"": ""AR"", ""is_mobile"": false}" 23135,8,518,2017-01-26 02:54:55,http://vandervort.com/gavin,,84.53.243.112,"{""location"": ""MM"", ""is_mobile"": false}" 23136,8,518,2017-02-17 04:05:32,http://schmidt.biz/kaylie.larkin,,225.238.38.163,"{""location"": ""MC"", ""is_mobile"": true}" 23137,8,518,2017-02-28 06:42:54,http://eichmann.info/claire,,234.61.111.170,"{""location"": ""TT"", ""is_mobile"": false}" 23138,8,518,2017-01-15 02:01:58,http://leuschke.net/citlalli.legros,,82.207.15.65,"{""location"": ""PS"", ""is_mobile"": false}" 23139,8,518,2017-06-13 17:14:00,http://wolff.org/margarett.altenwerth,,114.130.239.128,"{""location"": ""CV"", ""is_mobile"": true}" 23140,8,518,2017-04-08 02:15:44,http://conroy.io/moses.strosin,,135.203.174.108,"{""location"": ""CK"", ""is_mobile"": false}" 23141,8,518,2017-04-23 02:32:48,http://stehr.info/royce,,22.98.94.58,"{""location"": ""EE"", ""is_mobile"": false}" 23142,8,518,2017-03-17 19:49:28,http://abernathy.biz/clare,,74.127.145.24,"{""location"": ""HT"", ""is_mobile"": true}" 23143,8,518,2017-05-04 02:43:17,http://hintzlittel.net/gielle,,127.238.61.15,"{""location"": ""ST"", ""is_mobile"": true}" 23144,8,518,2017-01-26 11:19:52,http://fahey.info/katarina,,16.51.137.49,"{""location"": ""AF"", ""is_mobile"": false}" 23145,8,518,2017-03-28 05:49:14,http://macgyverframi.com/eugenia.von,,103.215.78.137,"{""location"": ""MC"", ""is_mobile"": false}" 23146,8,518,2017-04-21 08:42:50,http://fadel.info/leopold.sawayn,,189.45.213.142,"{""location"": ""LC"", ""is_mobile"": false}" 23147,8,518,2017-05-02 00:32:26,http://friesen.net/libby.hayes,,102.39.170.37,"{""location"": ""BG"", ""is_mobile"": false}" 23148,8,519,2017-05-06 18:13:38,http://caspermedhurst.name/evert.sanford,,208.179.61.210,"{""location"": ""FO"", ""is_mobile"": true}" 23149,8,519,2017-02-26 16:24:17,http://hyatt.name/caandra.hahn,,88.189.114.114,"{""location"": ""ZW"", ""is_mobile"": false}" 23150,8,519,2016-12-19 04:12:47,http://hintz.co/kurt,,212.119.22.175,"{""location"": ""VE"", ""is_mobile"": true}" 23151,8,519,2017-03-02 14:16:48,http://gradytillman.org/violette_robel,,73.161.118.180,"{""location"": ""RU"", ""is_mobile"": true}" 23152,8,519,2017-01-04 06:53:05,http://reichel.net/amari_johns,,238.123.47.53,"{""location"": ""FJ"", ""is_mobile"": true}" 23153,8,519,2017-02-25 13:10:21,http://hintz.info/juliana.gutkowski,,61.250.147.53,"{""location"": ""PY"", ""is_mobile"": false}" 23154,8,519,2017-02-12 06:07:05,http://wymanjohnson.biz/judson,,122.77.78.119,"{""location"": ""FI"", ""is_mobile"": true}" 23155,8,519,2017-04-25 20:53:39,http://schusterkub.name/angel_shields,,127.197.105.202,"{""location"": ""NC"", ""is_mobile"": false}" 23156,8,519,2017-01-31 14:44:45,http://hickle.name/dorcas,,11.168.219.220,"{""location"": ""RE"", ""is_mobile"": true}" 23157,8,519,2017-05-21 07:45:38,http://armstrongbogisich.io/guie.king,,241.72.246.112,"{""location"": ""CD"", ""is_mobile"": true}" 23158,8,519,2017-01-13 11:12:52,http://denesik.org/holden.koelpin,,216.80.107.160,"{""location"": ""PA"", ""is_mobile"": true}" 2087,1,46,2017-04-03 13:58:11,http://shieldsquigley.biz/porter,,27.66.96.39,"{""location"": ""NR"", ""is_mobile"": true}" 2088,1,46,2017-02-05 19:13:22,http://ricegrimes.info/fannie,,85.65.152.97,"{""location"": ""CG"", ""is_mobile"": false}" 2089,1,46,2017-04-11 11:32:52,http://tromp.info/leilani,,174.60.119.30,"{""location"": ""AT"", ""is_mobile"": true}" 2090,1,46,2017-05-06 00:14:19,http://bernierfarrell.io/letha_walsh,,216.192.109.140,"{""location"": ""BV"", ""is_mobile"": false}" 2091,1,46,2017-02-20 17:59:24,http://hauckluettgen.co/lavonne,,97.144.75.108,"{""location"": ""MS"", ""is_mobile"": false}" 2092,1,46,2017-04-16 18:29:34,http://windler.io/julio,,63.46.160.112,"{""location"": ""TG"", ""is_mobile"": false}" 2093,1,46,2017-01-13 04:59:37,http://stamm.name/tanya,,235.253.194.103,"{""location"": ""SV"", ""is_mobile"": false}" 2094,1,46,2017-01-19 05:04:53,http://tremblay.info/augusta,,45.4.23.241,"{""location"": ""LA"", ""is_mobile"": true}" 2095,1,46,2017-02-26 10:34:51,http://wildermanleannon.com/karley,,76.192.137.63,"{""location"": ""BJ"", ""is_mobile"": false}" 2096,1,46,2016-12-16 22:08:19,http://lowe.org/jules_gulgowski,,196.76.254.201,"{""location"": ""BV"", ""is_mobile"": false}" 2097,1,46,2017-03-05 08:17:51,http://johnston.io/althea_reinger,,34.118.4.28,"{""location"": ""BT"", ""is_mobile"": true}" 2098,1,46,2017-02-20 04:24:30,http://haag.info/adam,,245.143.218.131,"{""location"": ""BW"", ""is_mobile"": false}" 2099,1,46,2017-05-15 07:10:57,http://farrell.biz/ona_herman,,215.222.243.242,"{""location"": ""MN"", ""is_mobile"": true}" 2100,1,46,2016-12-18 16:32:08,http://gradyhackett.co/sabryna,,208.179.239.61,"{""location"": ""PM"", ""is_mobile"": false}" 2101,1,46,2017-03-10 10:04:02,http://sauermoen.info/lawrence,,175.235.76.151,"{""location"": ""IE"", ""is_mobile"": true}" 2102,1,46,2017-01-07 12:01:18,http://gerlach.com/colleen,,108.212.161.210,"{""location"": ""NP"", ""is_mobile"": true}" 2103,1,46,2017-02-22 14:41:36,http://ferryhodkiewicz.info/katheryn,,195.6.90.229,"{""location"": ""MQ"", ""is_mobile"": false}" 2104,1,47,2017-04-12 01:36:42,http://crooks.net/kolby,,211.116.101.159,"{""location"": ""CC"", ""is_mobile"": true}" 2105,1,47,2017-03-07 00:00:48,http://mcculloughschuppe.info/icie,,223.216.55.230,"{""location"": ""EH"", ""is_mobile"": false}" 2106,1,47,2017-01-10 18:09:47,http://walter.io/paula,,106.29.11.141,"{""location"": ""GH"", ""is_mobile"": false}" 2107,1,47,2017-03-06 08:07:29,http://orn.name/marianna,,108.138.171.147,"{""location"": ""TL"", ""is_mobile"": false}" 2108,1,47,2017-04-12 02:35:44,http://williamsonmacejkovic.io/hayley_ohara,,210.173.9.112,"{""location"": ""ES"", ""is_mobile"": false}" 2109,1,47,2016-12-20 21:07:25,http://kemmer.net/rhea,,112.173.147.187,"{""location"": ""IE"", ""is_mobile"": false}" 2110,1,47,2017-03-15 04:42:45,http://oconner.org/herbert,,33.57.85.102,"{""location"": ""CC"", ""is_mobile"": true}" 2111,1,47,2017-05-25 19:54:12,http://beatty.name/hailie,,139.236.34.83,"{""location"": ""OM"", ""is_mobile"": false}" 2112,1,47,2017-05-05 23:38:54,http://batz.info/rhiannon.wyman,,107.60.236.134,"{""location"": ""EH"", ""is_mobile"": false}" 2113,1,47,2017-02-10 07:03:02,http://bergnaum.co/belle,,111.158.82.102,"{""location"": ""GY"", ""is_mobile"": false}" 2114,1,47,2017-04-26 06:25:29,http://harristowne.biz/myah,,203.226.193.39,"{""location"": ""GD"", ""is_mobile"": true}" 2115,1,47,2017-05-06 17:30:11,http://torp.biz/leann,,9.180.156.17,"{""location"": ""BW"", ""is_mobile"": true}" 2116,1,47,2016-12-13 09:10:10,http://barrows.biz/delphia.pagac,,242.69.124.236,"{""location"": ""CU"", ""is_mobile"": false}" 2117,1,47,2017-02-22 15:46:48,http://stehr.name/pierre,,61.26.136.125,"{""location"": ""LU"", ""is_mobile"": false}" 2118,1,47,2017-05-27 14:30:54,http://berge.org/shanel.rolfson,,210.80.4.33,"{""location"": ""SJ"", ""is_mobile"": true}" 2119,1,47,2017-03-23 21:24:59,http://beier.co/brice,,151.239.191.6,"{""location"": ""NI"", ""is_mobile"": true}" 2120,1,47,2017-04-14 14:07:02,http://dubuquewindler.io/ali.lockman,,119.136.117.172,"{""location"": ""ZA"", ""is_mobile"": false}" 2121,1,47,2017-04-22 11:18:47,http://jakubowskifisher.com/jon_erdman,,232.210.207.210,"{""location"": ""US"", ""is_mobile"": true}" 2122,1,47,2017-04-10 07:31:50,http://zulauf.name/philip,,45.111.200.250,"{""location"": ""VG"", ""is_mobile"": true}" 2123,1,47,2017-04-10 01:22:07,http://walker.net/hailie,,32.78.17.74,"{""location"": ""BQ"", ""is_mobile"": true}" 2124,1,47,2017-04-03 12:29:10,http://macejkovic.biz/ewell.mccullough,,91.192.131.103,"{""location"": ""CL"", ""is_mobile"": true}" 2125,1,47,2017-02-28 11:13:37,http://lakin.net/melisa,,2.65.4.127,"{""location"": ""GM"", ""is_mobile"": false}" 2126,1,47,2017-05-30 23:03:06,http://bins.info/lupe,,202.141.227.28,"{""location"": ""CW"", ""is_mobile"": true}" 2127,1,47,2017-04-17 03:25:44,http://schowalter.net/bella_wehner,,43.113.251.128,"{""location"": ""CV"", ""is_mobile"": false}" 2128,1,47,2016-12-18 08:22:34,http://beertillman.co/kenna,,124.231.85.155,"{""location"": ""CH"", ""is_mobile"": true}" 2129,1,47,2017-04-27 03:06:49,http://kshlerin.org/sonya,,10.95.109.113,"{""location"": ""KH"", ""is_mobile"": true}" 2130,1,47,2017-01-30 02:57:25,http://hudsonhahn.name/jalyn,,16.89.50.142,"{""location"": ""LB"", ""is_mobile"": false}" 2131,1,47,2017-02-25 07:26:52,http://schoen.info/frieda.schowalter,,4.187.140.162,"{""location"": ""BH"", ""is_mobile"": false}" 2132,1,47,2017-05-31 15:46:18,http://murray.info/garnet_rogahn,,90.6.94.190,"{""location"": ""UA"", ""is_mobile"": false}" 2133,1,47,2017-04-06 08:39:31,http://paucek.com/johan_moen,,182.54.36.148,"{""location"": ""SK"", ""is_mobile"": false}" 2134,1,47,2016-12-17 03:38:47,http://graham.co/ashley,,33.83.36.150,"{""location"": ""SL"", ""is_mobile"": true}" 2135,1,47,2017-04-30 10:06:38,http://rosenbaum.info/kaitlin,,6.29.157.116,"{""location"": ""KR"", ""is_mobile"": false}" 2137,1,47,2017-04-17 11:55:06,http://stanton.co/milan.harvey,,58.77.6.241,"{""location"": ""RO"", ""is_mobile"": true}" 2138,1,47,2016-12-21 00:14:10,http://turcottebartell.co/rene,,71.45.221.159,"{""location"": ""IT"", ""is_mobile"": true}" 2139,1,47,2017-06-04 11:38:49,http://konopelski.org/bud,,123.182.198.128,"{""location"": ""CU"", ""is_mobile"": false}" 2140,1,48,2017-05-08 00:17:37,http://dooleycarroll.co/lester,,108.20.202.152,"{""location"": ""KG"", ""is_mobile"": false}" 2141,1,48,2017-02-11 10:57:24,http://bernier.com/lee,,223.71.167.12,"{""location"": ""GN"", ""is_mobile"": false}" 2142,1,48,2017-03-24 21:33:54,http://simonis.org/chet.towne,,142.248.150.53,"{""location"": ""IQ"", ""is_mobile"": false}" 2143,1,48,2017-05-15 09:53:35,http://greenholt.com/loren.heel,,19.203.46.182,"{""location"": ""IQ"", ""is_mobile"": true}" 23159,8,519,2017-04-23 05:03:31,http://mertzdoyle.org/jovani_ward,,121.95.253.61,"{""location"": ""IO"", ""is_mobile"": true}" 23160,8,519,2017-03-01 10:14:39,http://hodkiewiczwintheiser.co/terrell_jenkins,,148.88.216.16,"{""location"": ""GF"", ""is_mobile"": false}" 23161,8,519,2017-04-13 14:08:55,http://cain.org/virginia,,83.85.211.54,"{""location"": ""VG"", ""is_mobile"": true}" 23162,8,519,2017-04-15 17:47:20,http://mohr.info/hershel_mcclure,,167.222.246.35,"{""location"": ""VG"", ""is_mobile"": true}" 23163,8,519,2017-01-03 17:36:27,http://davis.net/kane_mann,,229.146.160.40,"{""location"": ""MU"", ""is_mobile"": true}" 23164,8,519,2017-01-22 14:10:39,http://binswiza.biz/deontae.harris,,253.127.232.38,"{""location"": ""OM"", ""is_mobile"": false}" 23165,8,519,2016-12-17 18:04:44,http://gerhold.net/alta,,247.151.26.47,"{""location"": ""NC"", ""is_mobile"": true}" 23166,8,519,2017-01-17 21:38:27,http://deckow.name/mackenzie,,199.92.225.99,"{""location"": ""IO"", ""is_mobile"": true}" 23167,8,519,2017-06-13 17:32:33,http://weber.co/demetrius,,139.6.4.59,"{""location"": ""MU"", ""is_mobile"": true}" 23168,8,519,2017-06-01 17:37:51,http://macgyverhoppe.io/viva_balistreri,,104.85.190.132,"{""location"": ""PS"", ""is_mobile"": false}" 23169,8,519,2017-04-02 00:01:53,http://roweconroy.co/brandyn,,237.177.206.127,"{""location"": ""NC"", ""is_mobile"": true}" 23170,8,519,2017-03-06 20:48:45,http://pourosko.org/gunnar.donnelly,,108.73.57.254,"{""location"": ""NC"", ""is_mobile"": true}" 23171,8,520,2017-01-09 13:03:35,http://padberg.biz/brown.maggio,,140.19.214.145,"{""location"": ""CR"", ""is_mobile"": false}" 23172,8,520,2017-02-13 19:00:19,http://pfeffer.io/waldo,,191.27.144.174,"{""location"": ""MV"", ""is_mobile"": false}" 23173,8,520,2017-06-10 21:16:30,http://pollichwyman.org/janice_rohan,,160.55.113.28,"{""location"": ""BN"", ""is_mobile"": true}" 23174,8,520,2017-05-13 13:47:43,http://kerluke.co/margie,,135.140.133.25,"{""location"": ""BY"", ""is_mobile"": false}" 23175,8,520,2017-03-16 23:10:18,http://flatley.co/lynn,,178.99.85.175,"{""location"": ""SS"", ""is_mobile"": false}" 23176,8,520,2017-02-22 09:00:12,http://heidenreichrunolfon.org/tevin.goodwin,,224.27.174.248,"{""location"": ""MV"", ""is_mobile"": true}" 23177,8,520,2017-02-21 12:14:21,http://lindgren.biz/kaylin.haley,,211.253.39.18,"{""location"": ""FM"", ""is_mobile"": true}" 23178,8,520,2017-03-13 13:54:38,http://sauer.co/jada,,50.253.244.194,"{""location"": ""CA"", ""is_mobile"": false}" 23179,8,520,2017-02-28 01:39:10,http://reynolds.name/dudley,,252.170.102.79,"{""location"": ""SA"", ""is_mobile"": false}" 23180,8,520,2017-02-12 21:30:56,http://huels.info/lera.johnston,,56.86.72.163,"{""location"": ""CF"", ""is_mobile"": true}" 23181,8,520,2017-01-08 02:51:04,http://walsh.biz/kolby,,254.57.60.209,"{""location"": ""VA"", ""is_mobile"": false}" 23182,8,520,2017-04-10 17:36:53,http://cristwalker.net/berneice_larson,,238.59.122.33,"{""location"": ""NZ"", ""is_mobile"": false}" 23183,8,520,2017-05-13 17:57:33,http://kautzer.name/kenny,,21.64.110.127,"{""location"": ""CR"", ""is_mobile"": true}" 23184,8,520,2017-04-03 20:29:39,http://kochkoch.info/onie,,189.88.182.39,"{""location"": ""SI"", ""is_mobile"": true}" 23185,8,520,2017-02-19 07:52:55,http://quitzon.info/reilly,,70.24.33.193,"{""location"": ""BQ"", ""is_mobile"": true}" 23186,8,520,2017-05-02 20:21:24,http://howell.co/vergie,,212.55.138.226,"{""location"": ""GY"", ""is_mobile"": true}" 23187,8,520,2017-05-19 16:30:07,http://kuhlmanmarquardt.com/ernie_schmidt,,181.246.155.132,"{""location"": ""SV"", ""is_mobile"": false}" 23188,8,520,2017-04-12 15:40:22,http://zemlakspinka.io/wayne,,4.170.250.119,"{""location"": ""NR"", ""is_mobile"": false}" 23189,8,520,2017-03-09 11:00:49,http://sauer.net/alicia,,172.237.162.43,"{""location"": ""YE"", ""is_mobile"": true}" 23190,8,520,2017-05-04 04:45:18,http://davis.info/melody.lind,,130.169.225.5,"{""location"": ""NP"", ""is_mobile"": false}" 23191,8,520,2017-01-29 17:53:11,http://lesch.name/katarina,,201.235.81.26,"{""location"": ""EH"", ""is_mobile"": true}" 23192,8,520,2016-12-27 18:25:57,http://mante.net/keon_hintz,,215.19.102.8,"{""location"": ""NL"", ""is_mobile"": true}" 23193,8,520,2017-04-10 04:06:33,http://bauch.info/mavis,,164.31.6.177,"{""location"": ""AU"", ""is_mobile"": false}" 23194,8,520,2017-02-21 12:44:50,http://ryanbahringer.net/adolfo,,168.167.90.58,"{""location"": ""WF"", ""is_mobile"": true}" 23195,8,521,2017-03-27 00:28:45,http://heathcote.net/ivy.paucek,,104.249.227.121,"{""location"": ""MF"", ""is_mobile"": false}" 23196,8,521,2016-12-29 03:10:45,http://gerlach.io/garett.lakin,,94.35.220.128,"{""location"": ""TM"", ""is_mobile"": true}" 23197,8,521,2017-04-25 12:36:03,http://botsford.name/seamus,,127.180.148.138,"{""location"": ""LT"", ""is_mobile"": false}" 23198,8,521,2017-01-09 16:02:01,http://heidenreich.name/tabitha,,167.178.159.254,"{""location"": ""BW"", ""is_mobile"": false}" 23199,8,521,2017-02-19 23:25:46,http://towne.co/sasha_ernser,,26.46.200.18,"{""location"": ""AZ"", ""is_mobile"": true}" 23200,8,521,2017-03-03 00:38:51,http://schinner.name/janice.lynch,,215.38.174.41,"{""location"": ""AL"", ""is_mobile"": true}" 23201,8,521,2017-06-09 03:29:42,http://hoeger.biz/elliot,,157.4.157.185,"{""location"": ""PR"", ""is_mobile"": false}" 23202,8,521,2017-03-30 04:15:04,http://littel.info/adela.tromp,,112.143.156.30,"{""location"": ""NF"", ""is_mobile"": false}" 23203,8,521,2017-05-09 23:44:39,http://yundt.co/augusta,,94.40.149.237,"{""location"": ""BB"", ""is_mobile"": true}" 23204,8,521,2017-01-03 07:16:54,http://pfannerstillrohan.net/kirk,,214.135.175.118,"{""location"": ""MV"", ""is_mobile"": true}" 23205,8,521,2017-05-16 22:18:59,http://vandervortwisozk.com/maeve_lockman,,199.83.16.81,"{""location"": ""CV"", ""is_mobile"": false}" 23206,8,521,2017-03-11 03:39:30,http://denesikdonnelly.net/curtis,,35.251.37.107,"{""location"": ""LY"", ""is_mobile"": true}" 23207,8,521,2017-05-21 09:40:12,http://pollichwalsh.net/angie,,150.30.83.102,"{""location"": ""CG"", ""is_mobile"": true}" 23208,8,521,2017-01-29 02:59:02,http://koch.name/beatrice,,67.121.248.175,"{""location"": ""LI"", ""is_mobile"": false}" 23209,8,521,2017-06-10 09:18:17,http://murphybreitenberg.name/deontae,,218.240.22.159,"{""location"": ""AR"", ""is_mobile"": false}" 23210,8,521,2017-04-19 07:22:05,http://stokesreinger.biz/brenda_reilly,,150.112.216.138,"{""location"": ""CN"", ""is_mobile"": false}" 23211,8,521,2017-06-04 15:56:42,http://barton.net/breanna.dickinson,,66.240.3.109,"{""location"": ""GG"", ""is_mobile"": true}" 23212,8,521,2016-12-29 00:40:23,http://waelchischmitt.biz/baylee,,254.60.83.210,"{""location"": ""NI"", ""is_mobile"": true}" 23213,8,521,2016-12-24 13:48:26,http://parker.co/elna,,173.42.196.106,"{""location"": ""ZM"", ""is_mobile"": true}" 2144,1,48,2017-05-04 22:35:41,http://yundt.info/lorenzo,,210.78.2.46,"{""location"": ""MC"", ""is_mobile"": true}" 2145,1,48,2017-03-10 18:21:47,http://oconner.net/justyn,,39.62.104.234,"{""location"": ""ME"", ""is_mobile"": false}" 2146,1,48,2017-03-12 04:40:41,http://gutmannspencer.info/federico_dickens,,214.65.2.249,"{""location"": ""JE"", ""is_mobile"": true}" 2147,1,48,2017-06-08 09:45:40,http://toy.com/janelle.bosco,,153.247.92.146,"{""location"": ""FI"", ""is_mobile"": true}" 2148,1,48,2017-03-07 06:19:27,http://mueller.info/tiffany.sporer,,69.33.142.106,"{""location"": ""CV"", ""is_mobile"": true}" 2149,1,48,2017-04-20 11:52:45,http://kuhnwisoky.biz/jedediah,,100.182.156.253,"{""location"": ""PF"", ""is_mobile"": true}" 2150,1,48,2017-05-09 22:24:54,http://reichel.co/vladimir,,185.216.27.188,"{""location"": ""KI"", ""is_mobile"": true}" 2151,1,48,2017-03-11 10:15:21,http://sengernolan.biz/wayne,,153.14.123.14,"{""location"": ""LU"", ""is_mobile"": true}" 2152,1,48,2017-05-06 07:11:15,http://stehr.com/lesly.connelly,,133.39.231.42,"{""location"": ""CF"", ""is_mobile"": true}" 2153,1,48,2017-04-18 21:42:28,http://olsonterry.info/roger_swaniawski,,241.5.202.237,"{""location"": ""GT"", ""is_mobile"": false}" 2154,1,48,2017-02-27 04:09:44,http://terry.com/keenan_berge,,27.206.37.129,"{""location"": ""RE"", ""is_mobile"": false}" 2155,1,48,2016-12-22 18:24:33,http://satterfield.com/einar.runolfon,,189.26.134.53,"{""location"": ""SX"", ""is_mobile"": false}" 2156,1,48,2017-04-10 17:21:32,http://abbott.co/myles,,15.38.141.242,"{""location"": ""VC"", ""is_mobile"": true}" 2157,1,48,2017-02-05 08:18:32,http://aufderhar.org/lamont.mayert,,243.147.79.178,"{""location"": ""MU"", ""is_mobile"": false}" 2158,1,48,2017-05-15 13:04:23,http://schmidtko.com/pietro,,175.132.147.49,"{""location"": ""SJ"", ""is_mobile"": false}" 2159,1,48,2017-05-27 16:24:32,http://schulistcorkery.co/darron.lakin,,195.142.199.210,"{""location"": ""UM"", ""is_mobile"": true}" 2160,1,48,2017-03-24 19:25:27,http://sipesquitzon.net/agustina_nader,,148.13.88.206,"{""location"": ""KI"", ""is_mobile"": false}" 2161,1,48,2017-03-02 03:08:15,http://stokes.info/keaton_jones,,162.247.71.193,"{""location"": ""TL"", ""is_mobile"": false}" 2162,1,48,2017-02-10 06:27:56,http://douglas.co/hilton.paucek,,85.28.100.120,"{""location"": ""GW"", ""is_mobile"": false}" 2163,1,48,2017-03-19 20:50:18,http://haleyschroeder.co/kirsten,,44.148.211.113,"{""location"": ""AL"", ""is_mobile"": true}" 2164,1,48,2016-12-23 13:26:11,http://haag.name/soledad_lemke,,2.188.128.203,"{""location"": ""TV"", ""is_mobile"": true}" 2165,1,48,2017-06-02 22:14:52,http://wymancollins.io/taurean.stoltenberg,,178.146.96.253,"{""location"": ""GN"", ""is_mobile"": true}" 2166,1,48,2017-02-02 21:31:17,http://schroeder.co/nicole_lowe,,228.141.57.79,"{""location"": ""AX"", ""is_mobile"": true}" 2167,1,48,2017-04-26 16:51:18,http://dooley.net/ottis_spencer,,135.253.202.163,"{""location"": ""EE"", ""is_mobile"": false}" 2168,1,48,2017-02-04 09:06:47,http://will.name/elenora,,226.215.110.65,"{""location"": ""SO"", ""is_mobile"": false}" 2169,1,48,2017-04-27 10:49:22,http://lehner.biz/bella_bosco,,50.64.10.129,"{""location"": ""LT"", ""is_mobile"": true}" 2170,1,48,2017-02-07 07:13:17,http://thompson.biz/oleta_olson,,172.74.244.32,"{""location"": ""RU"", ""is_mobile"": false}" 2171,1,48,2017-06-13 14:13:14,http://schmitt.com/sofia.hauck,,65.220.179.31,"{""location"": ""SS"", ""is_mobile"": true}" 2172,1,48,2017-05-28 02:08:19,http://rippinsimonis.io/coty,,167.120.190.240,"{""location"": ""PW"", ""is_mobile"": false}" 2173,1,49,2017-01-24 12:32:31,http://halvorsonohara.net/nya,,65.206.201.211,"{""location"": ""JM"", ""is_mobile"": false}" 2174,1,49,2017-02-19 03:07:20,http://roobbernhard.com/lew,,75.98.107.48,"{""location"": ""TD"", ""is_mobile"": false}" 2175,1,49,2017-01-05 03:04:16,http://stiedemann.io/grover_mann,,236.91.112.202,"{""location"": ""MO"", ""is_mobile"": true}" 2176,1,49,2017-05-13 22:46:42,http://kozey.org/jaqueline_berge,,217.173.252.118,"{""location"": ""BO"", ""is_mobile"": false}" 2177,1,49,2017-03-01 08:07:30,http://cummings.io/candace,,31.224.5.50,"{""location"": ""FO"", ""is_mobile"": true}" 2178,1,49,2017-04-24 15:27:06,http://mullerromaguera.info/lauryn,,79.51.94.198,"{""location"": ""MU"", ""is_mobile"": false}" 2179,1,49,2017-03-13 09:15:15,http://bernier.io/river.weinat,,84.27.149.171,"{""location"": ""CA"", ""is_mobile"": false}" 2180,1,49,2017-03-05 23:48:38,http://ferry.co/franco.krajcik,,161.128.115.231,"{""location"": ""MQ"", ""is_mobile"": true}" 2181,1,49,2017-01-21 04:04:50,http://schoenlindgren.info/anjali.grady,,5.56.3.132,"{""location"": ""TT"", ""is_mobile"": true}" 2182,1,49,2017-04-18 09:04:17,http://wehner.biz/abbey,,131.133.65.142,"{""location"": ""BQ"", ""is_mobile"": true}" 2183,1,49,2017-03-09 00:53:20,http://price.org/rosalia,,237.73.4.189,"{""location"": ""AU"", ""is_mobile"": false}" 2184,1,49,2017-03-02 02:59:16,http://walsh.biz/norwood.wiza,,19.130.237.13,"{""location"": ""UY"", ""is_mobile"": false}" 2185,1,49,2017-05-29 18:48:57,http://mullerdickinson.net/audra.kuhn,,119.177.103.227,"{""location"": ""VC"", ""is_mobile"": false}" 2186,1,49,2017-04-08 01:02:08,http://halvorson.net/hollie,,116.220.91.155,"{""location"": ""ES"", ""is_mobile"": true}" 2187,1,49,2017-06-01 18:07:35,http://gleason.biz/frances_shields,,196.102.251.120,"{""location"": ""KE"", ""is_mobile"": false}" 2188,1,49,2017-04-30 14:30:50,http://kuhicmitchell.biz/mireya_tremblay,,93.144.74.239,"{""location"": ""PW"", ""is_mobile"": true}" 2189,1,49,2017-04-14 07:31:55,http://gutkowskibayer.net/maymie_becker,,131.110.43.20,"{""location"": ""LU"", ""is_mobile"": false}" 2190,1,49,2017-01-03 12:44:12,http://mcclureswaniawski.info/victor_dooley,,130.236.239.30,"{""location"": ""SD"", ""is_mobile"": true}" 2191,1,49,2017-03-06 03:55:35,http://feil.io/claudie.rath,,187.85.241.170,"{""location"": ""PA"", ""is_mobile"": true}" 2192,1,49,2017-03-13 11:21:50,http://hyatt.io/friedrich.durgan,,122.98.64.136,"{""location"": ""VA"", ""is_mobile"": false}" 2193,1,49,2017-01-16 13:53:47,http://murraybosco.org/izabella.macejkovic,,251.113.159.10,"{""location"": ""TM"", ""is_mobile"": true}" 2195,1,49,2017-02-08 14:37:39,http://pouros.biz/mackenzie,,30.35.243.145,"{""location"": ""TH"", ""is_mobile"": false}" 2196,1,49,2016-12-19 08:57:42,http://weimann.info/cyril,,184.24.238.15,"{""location"": ""ML"", ""is_mobile"": true}" 2197,1,49,2017-01-30 16:59:50,http://cummerata.name/mina.tromp,,87.214.248.81,"{""location"": ""IT"", ""is_mobile"": true}" 2198,1,49,2017-05-20 03:00:12,http://mosciskipaucek.name/amber_beahan,,39.115.23.224,"{""location"": ""BW"", ""is_mobile"": true}" 2199,1,49,2017-03-21 14:49:28,http://hellerfunk.co/kenyatta.stoltenberg,,146.17.145.183,"{""location"": ""SR"", ""is_mobile"": false}" 2200,1,49,2017-03-31 07:44:05,http://homenick.info/felipe.keler,,103.105.163.101,"{""location"": ""GE"", ""is_mobile"": false}" 23214,8,521,2017-03-01 07:50:46,http://spinka.io/london,,177.114.95.198,"{""location"": ""MD"", ""is_mobile"": false}" 23215,8,521,2017-01-20 19:22:29,http://ernser.co/waino_carroll,,151.45.228.141,"{""location"": ""GP"", ""is_mobile"": true}" 23216,8,521,2017-06-04 09:57:29,http://hauckfunk.com/garrison,,57.161.13.3,"{""location"": ""PL"", ""is_mobile"": false}" 23217,8,521,2017-05-02 01:16:18,http://vonrueden.co/tyson,,245.132.147.222,"{""location"": ""AW"", ""is_mobile"": false}" 23218,8,521,2017-02-15 12:06:28,http://stiedemann.org/rylee,,113.252.139.69,"{""location"": ""NL"", ""is_mobile"": false}" 23219,8,521,2017-06-12 12:44:07,http://tromp.com/jovani,,112.28.174.151,"{""location"": ""SX"", ""is_mobile"": false}" 23220,8,521,2017-01-06 15:59:20,http://lehner.biz/elvera.blick,,67.36.34.241,"{""location"": ""UA"", ""is_mobile"": true}" 23221,8,521,2017-03-19 11:59:52,http://blickmayert.name/leon,,186.243.42.129,"{""location"": ""CA"", ""is_mobile"": false}" 23222,8,521,2017-04-18 07:20:39,http://ratke.co/sabryna,,195.96.78.227,"{""location"": ""MS"", ""is_mobile"": true}" 23223,8,521,2017-05-12 14:19:13,http://keeling.io/susie_senger,,28.127.185.193,"{""location"": ""IS"", ""is_mobile"": false}" 23224,8,521,2017-04-24 17:50:39,http://kovacek.info/meredith.bechtelar,,20.73.172.106,"{""location"": ""UG"", ""is_mobile"": false}" 23225,8,521,2017-01-26 15:45:08,http://balistrerimccullough.biz/sage,,173.190.12.187,"{""location"": ""KR"", ""is_mobile"": true}" 23226,8,521,2017-05-12 08:24:17,http://goodwin.net/eloy,,41.49.176.156,"{""location"": ""LB"", ""is_mobile"": true}" 23227,8,521,2017-05-22 03:32:57,http://wilkinson.io/alexa_glover,,65.69.103.99,"{""location"": ""GU"", ""is_mobile"": false}" 23228,8,521,2017-05-19 20:03:05,http://haleyledner.biz/casimir,,112.25.153.20,"{""location"": ""BV"", ""is_mobile"": true}" 23229,8,522,2017-01-25 19:09:02,http://stantonwitting.name/dorthy,,139.156.167.59,"{""location"": ""JO"", ""is_mobile"": true}" 23230,8,522,2016-12-13 18:24:52,http://keeblerruel.org/imelda_spinka,,206.79.92.28,"{""location"": ""GQ"", ""is_mobile"": true}" 23231,8,522,2017-02-18 09:31:55,http://feilmoen.net/renee,,165.43.140.213,"{""location"": ""AO"", ""is_mobile"": true}" 23232,8,522,2017-03-29 02:15:26,http://wehner.net/palma_medhurst,,120.228.42.188,"{""location"": ""BQ"", ""is_mobile"": false}" 23233,8,522,2016-12-22 20:57:13,http://littleconsidine.io/hal,,30.231.40.211,"{""location"": ""GR"", ""is_mobile"": false}" 23234,8,522,2017-02-12 12:34:48,http://vonrueden.net/keanu,,20.105.143.105,"{""location"": ""NU"", ""is_mobile"": true}" 23235,8,522,2017-05-10 04:13:46,http://lynch.net/coby,,67.180.137.157,"{""location"": ""AO"", ""is_mobile"": true}" 23236,8,522,2017-05-21 20:32:16,http://lindgren.info/mellie,,155.224.153.4,"{""location"": ""DO"", ""is_mobile"": true}" 23237,8,522,2017-05-28 17:38:49,http://cremin.com/ahmed.monahan,,78.189.68.62,"{""location"": ""IQ"", ""is_mobile"": true}" 23238,8,522,2017-05-22 02:00:22,http://romagueradaniel.biz/francis,,8.154.34.87,"{""location"": ""ET"", ""is_mobile"": false}" 23239,8,522,2017-02-14 18:33:19,http://hirthe.net/claud_berge,,168.172.24.124,"{""location"": ""ER"", ""is_mobile"": true}" 23240,8,522,2017-05-05 02:17:56,http://lebsack.io/colleen,,159.90.253.62,"{""location"": ""BA"", ""is_mobile"": false}" 23241,8,522,2017-03-10 13:22:29,http://lind.com/markus_wilkinson,,60.71.131.224,"{""location"": ""KZ"", ""is_mobile"": false}" 23242,8,522,2017-04-14 01:57:21,http://kuhn.name/kelton,,38.103.16.149,"{""location"": ""GU"", ""is_mobile"": true}" 23243,8,522,2017-03-12 08:28:48,http://ernser.org/dane_bartell,,3.230.40.41,"{""location"": ""IN"", ""is_mobile"": false}" 23244,8,522,2017-01-13 09:04:37,http://schimmelmcdermott.com/benjamin,,47.46.206.57,"{""location"": ""BZ"", ""is_mobile"": true}" 23245,8,522,2017-01-18 21:35:11,http://koepp.org/aleandro.haag,,175.76.107.30,"{""location"": ""BG"", ""is_mobile"": true}" 23246,8,522,2017-04-10 09:39:20,http://davismohr.io/lurline.medhurst,,204.224.173.195,"{""location"": ""BY"", ""is_mobile"": false}" 23247,8,522,2016-12-30 12:21:43,http://jakubowski.name/florine,,126.109.229.81,"{""location"": ""IO"", ""is_mobile"": true}" 23248,8,522,2017-01-16 18:52:14,http://heidenreichstiedemann.com/gudrun,,198.62.31.88,"{""location"": ""SH"", ""is_mobile"": false}" 23249,8,522,2017-01-09 15:40:01,http://oharastroman.biz/arnulfo_konopelski,,99.245.217.209,"{""location"": ""GE"", ""is_mobile"": true}" 23250,8,522,2017-02-06 17:18:24,http://morar.org/rosa.stoltenberg,,201.241.244.223,"{""location"": ""NR"", ""is_mobile"": true}" 23251,8,522,2017-03-18 07:49:59,http://macejkoviclarkin.com/kane_harris,,239.208.109.56,"{""location"": ""GN"", ""is_mobile"": false}" 23252,8,522,2016-12-16 03:45:11,http://waters.net/edison,,13.99.240.125,"{""location"": ""AS"", ""is_mobile"": true}" 23253,8,522,2017-05-06 05:56:07,http://effertzwolff.io/price,,123.64.75.133,"{""location"": ""AS"", ""is_mobile"": true}" 23254,8,522,2017-03-20 17:51:59,http://volkman.info/cleora,,38.197.224.103,"{""location"": ""AZ"", ""is_mobile"": false}" 23255,8,522,2017-03-23 16:54:51,http://feeney.net/leta_jones,,134.18.32.116,"{""location"": ""JM"", ""is_mobile"": true}" 23256,8,522,2017-01-13 13:54:28,http://beahanmckenzie.org/kenny_marvin,,71.172.75.240,"{""location"": ""CU"", ""is_mobile"": false}" 23257,8,522,2017-04-05 15:04:24,http://schroeder.info/athena.satterfield,,208.31.160.41,"{""location"": ""BE"", ""is_mobile"": true}" 23258,8,522,2016-12-31 16:46:58,http://oconner.com/nickolas.hills,,11.29.123.149,"{""location"": ""AF"", ""is_mobile"": false}" 23259,8,522,2017-01-13 21:53:15,http://jacobson.name/santiago,,43.169.198.35,"{""location"": ""VA"", ""is_mobile"": false}" 23260,8,522,2017-05-10 01:19:06,http://zemlak.name/dock,,66.11.218.115,"{""location"": ""DO"", ""is_mobile"": false}" 23261,8,522,2017-02-20 18:14:39,http://sporer.net/kylee,,207.38.197.159,"{""location"": ""BO"", ""is_mobile"": false}" 23262,8,522,2017-02-19 14:49:24,http://shanahan.com/otis,,10.246.159.22,"{""location"": ""CY"", ""is_mobile"": true}" 23263,8,522,2017-04-13 23:09:58,http://hirtheheller.name/marjory,,37.64.86.197,"{""location"": ""BM"", ""is_mobile"": false}" 23264,8,522,2017-03-10 10:04:51,http://beiermills.info/chaya,,194.165.229.5,"{""location"": ""LC"", ""is_mobile"": true}" 23265,8,522,2017-04-11 18:49:16,http://botsfordhammes.biz/gayle.funk,,174.11.189.16,"{""location"": ""CD"", ""is_mobile"": true}" 23266,8,522,2017-04-26 15:32:46,http://smitham.info/caandra.mills,,166.3.84.165,"{""location"": ""LC"", ""is_mobile"": false}" 23267,8,522,2017-02-23 14:10:01,http://okonwitting.org/haleigh_mraz,,27.113.167.114,"{""location"": ""SG"", ""is_mobile"": false}" 23268,8,523,2017-03-19 00:20:57,http://toy.name/bell,,16.148.120.113,"{""location"": ""NZ"", ""is_mobile"": true}" 23269,8,523,2017-05-09 09:43:25,http://crona.net/raymundo_bartoletti,,17.32.209.95,"{""location"": ""KM"", ""is_mobile"": false}" 2201,1,49,2017-05-05 08:37:14,http://larkinroob.biz/macy,,97.48.174.141,"{""location"": ""KM"", ""is_mobile"": false}" 2202,1,49,2017-04-28 16:05:01,http://stehr.biz/lambert,,50.31.57.246,"{""location"": ""TV"", ""is_mobile"": false}" 2203,1,49,2017-04-06 13:10:12,http://gleichnerstehr.info/estrella.ledner,,2.82.162.233,"{""location"": ""CL"", ""is_mobile"": true}" 2204,1,49,2017-01-01 11:02:08,http://walkerkuhic.net/dexter.kuphal,,75.154.69.230,"{""location"": ""FJ"", ""is_mobile"": true}" 2205,1,49,2017-05-29 09:12:03,http://rempel.net/sidney_block,,93.95.162.181,"{""location"": ""PE"", ""is_mobile"": true}" 2206,1,49,2017-04-28 23:10:16,http://trompbraun.com/hank.moore,,159.90.92.254,"{""location"": ""MS"", ""is_mobile"": true}" 2207,1,49,2017-04-14 23:56:40,http://feil.name/ike,,105.120.52.68,"{""location"": ""CM"", ""is_mobile"": true}" 2208,1,49,2017-02-20 14:30:01,http://blick.name/joany,,73.158.223.8,"{""location"": ""RW"", ""is_mobile"": false}" 2209,1,49,2017-04-17 04:36:11,http://christiansenosinski.co/aniyah,,23.112.109.132,"{""location"": ""MU"", ""is_mobile"": true}" 2210,1,50,2017-05-28 14:29:08,http://sauer.info/natasha,,50.230.142.31,"{""location"": ""NP"", ""is_mobile"": true}" 2211,1,50,2017-04-03 15:30:10,http://johnsonframi.com/marielle.rohan,,159.23.141.243,"{""location"": ""IT"", ""is_mobile"": false}" 2212,1,50,2017-06-11 22:04:13,http://tromp.co/drew_wisozk,,117.59.12.156,"{""location"": ""IR"", ""is_mobile"": true}" 2213,1,50,2017-05-08 19:38:21,http://mclaughlinpacocha.io/wilton,,158.188.228.242,"{""location"": ""NP"", ""is_mobile"": false}" 2214,1,50,2017-05-28 07:25:19,http://toygrimes.com/brandon,,194.128.166.252,"{""location"": ""SR"", ""is_mobile"": false}" 2215,1,50,2017-06-11 03:39:20,http://bruen.biz/theodora,,63.230.229.171,"{""location"": ""TF"", ""is_mobile"": true}" 2216,1,50,2017-03-28 00:39:53,http://hettingerstreich.biz/freda_wilkinson,,40.153.184.107,"{""location"": ""NE"", ""is_mobile"": false}" 2217,1,50,2017-05-13 20:46:57,http://sawayn.net/rosa,,213.73.93.132,"{""location"": ""BO"", ""is_mobile"": false}" 2218,1,50,2017-04-16 11:26:55,http://kowalker.biz/breanna,,65.246.43.178,"{""location"": ""RU"", ""is_mobile"": false}" 2219,1,50,2017-01-01 03:03:33,http://rodriguez.co/delmer,,126.177.144.38,"{""location"": ""QA"", ""is_mobile"": false}" 2220,1,50,2016-12-20 01:50:10,http://ledner.com/clifton,,254.96.41.204,"{""location"": ""SB"", ""is_mobile"": true}" 2221,1,50,2017-03-15 12:29:17,http://bechtelar.info/ellsworth,,121.245.121.85,"{""location"": ""MR"", ""is_mobile"": false}" 2222,1,50,2016-12-16 15:46:54,http://leschwalker.io/armand,,99.190.62.92,"{""location"": ""CR"", ""is_mobile"": false}" 2223,1,50,2017-04-26 21:11:22,http://haaggusikowski.co/brigitte.ebert,,92.158.93.53,"{""location"": ""ES"", ""is_mobile"": true}" 2224,1,50,2017-04-28 09:15:23,http://oberbrunner.co/caesar,,172.69.195.245,"{""location"": ""GG"", ""is_mobile"": true}" 2225,1,50,2017-04-18 10:27:18,http://mcglynnoreilly.com/korey,,48.119.247.78,"{""location"": ""AG"", ""is_mobile"": true}" 2226,1,50,2017-02-20 22:39:24,http://gusikowskiweber.org/tiara_dibbert,,3.118.180.184,"{""location"": ""BI"", ""is_mobile"": true}" 2227,1,50,2017-04-07 21:58:03,http://prohaskadubuque.name/alivia.halvorson,,78.209.141.104,"{""location"": ""ID"", ""is_mobile"": false}" 2228,1,50,2017-02-05 23:59:28,http://naderveum.io/rashawn,,40.72.244.253,"{""location"": ""ER"", ""is_mobile"": true}" 2229,1,50,2017-03-26 04:21:32,http://powlowski.info/vada,,121.237.90.228,"{""location"": ""HK"", ""is_mobile"": false}" 2230,1,50,2017-02-13 23:34:47,http://damorehegmann.org/laron,,203.251.254.161,"{""location"": ""GU"", ""is_mobile"": true}" 2231,1,50,2017-01-22 10:34:50,http://mooreyost.io/hardy_maggio,,143.162.59.16,"{""location"": ""PR"", ""is_mobile"": false}" 2232,1,50,2017-05-16 08:37:02,http://brownshanahan.io/dena.grady,,133.216.251.27,"{""location"": ""GB"", ""is_mobile"": false}" 2233,1,50,2017-01-31 18:23:01,http://klockodach.info/gregoria,,12.80.7.164,"{""location"": ""LK"", ""is_mobile"": false}" 2234,1,50,2017-02-06 20:26:25,http://turner.net/hal_baumbach,,220.143.13.133,"{""location"": ""AQ"", ""is_mobile"": true}" 2235,1,50,2017-05-19 19:12:56,http://nienowhansen.name/kayley,,90.140.238.94,"{""location"": ""AS"", ""is_mobile"": false}" 2236,1,50,2017-01-17 23:57:40,http://gutkowski.org/felicita,,210.2.188.8,"{""location"": ""BA"", ""is_mobile"": true}" 2237,1,50,2017-06-05 23:27:45,http://stracke.biz/lizeth_murphy,,84.189.179.242,"{""location"": ""GE"", ""is_mobile"": false}" 2238,1,50,2017-06-12 15:33:24,http://fadelsporer.io/aidan,,107.15.48.31,"{""location"": ""RS"", ""is_mobile"": true}" 2239,1,50,2017-04-07 20:43:43,http://ruel.com/amara.wuckert,,94.23.28.11,"{""location"": ""MO"", ""is_mobile"": true}" 2240,1,50,2017-04-03 17:13:02,http://balistreri.name/gloria,,6.191.134.63,"{""location"": ""MZ"", ""is_mobile"": false}" 2241,1,50,2017-01-23 11:28:28,http://emmerich.com/sam_rempel,,185.141.101.46,"{""location"": ""KG"", ""is_mobile"": false}" 2242,1,50,2017-02-19 04:14:25,http://hermiston.net/fleta.hane,,113.20.127.102,"{""location"": ""BB"", ""is_mobile"": false}" 2243,1,50,2017-01-17 18:16:04,http://crooksherzog.co/albina_okon,,123.252.146.234,"{""location"": ""NC"", ""is_mobile"": false}" 2244,1,50,2017-04-15 13:08:49,http://oharareynolds.io/eugene,,102.170.88.73,"{""location"": ""TO"", ""is_mobile"": false}" 2245,1,50,2017-03-15 18:50:21,http://ryan.io/jarod_jones,,197.223.148.124,"{""location"": ""ST"", ""is_mobile"": false}" 2246,1,50,2017-06-01 16:50:13,http://kilback.name/kyler.wiegand,,199.104.24.86,"{""location"": ""NO"", ""is_mobile"": true}" 2247,1,50,2017-01-29 17:58:52,http://mante.info/delphia.homenick,,221.232.93.151,"{""location"": ""CG"", ""is_mobile"": true}" 2248,1,50,2017-05-17 04:43:40,http://hartmannzulauf.net/velda_ritchie,,180.26.208.183,"{""location"": ""AF"", ""is_mobile"": false}" 2249,1,50,2017-05-30 02:54:02,http://shields.net/maximo.heller,,9.177.70.2,"{""location"": ""KE"", ""is_mobile"": false}" 2250,1,51,2017-05-27 00:31:31,http://legros.name/raegan,,121.67.189.190,"{""location"": ""KW"", ""is_mobile"": false}" 2251,1,51,2017-03-28 10:12:00,http://willmstromp.net/brett,,129.70.79.141,"{""location"": ""DK"", ""is_mobile"": true}" 2252,1,51,2017-01-31 23:38:08,http://bailey.io/natalie,,120.174.100.15,"{""location"": ""MQ"", ""is_mobile"": false}" 2253,1,51,2017-02-10 11:31:17,http://johnson.info/ryder,,242.233.190.211,"{""location"": ""MY"", ""is_mobile"": true}" 2254,1,51,2017-05-20 16:22:32,http://ullrich.net/mekhi,,34.149.108.120,"{""location"": ""RO"", ""is_mobile"": false}" 2255,1,51,2017-03-03 12:47:27,http://wolf.name/phyllis.pacocha,,147.163.6.188,"{""location"": ""RS"", ""is_mobile"": false}" 2256,1,51,2017-05-14 02:16:22,http://swift.com/shayna_moen,,91.180.67.239,"{""location"": ""PN"", ""is_mobile"": false}" 23270,8,523,2017-04-29 10:25:49,http://rogahnheidenreich.net/sylvia_krajcik,,216.46.254.175,"{""location"": ""KG"", ""is_mobile"": true}" 23271,8,523,2017-04-25 20:25:48,http://rutherford.info/stuart,,129.3.207.134,"{""location"": ""KY"", ""is_mobile"": true}" 23272,8,523,2017-05-26 06:34:33,http://schinnerziemann.biz/danny_upton,,85.72.31.23,"{""location"": ""SX"", ""is_mobile"": false}" 23273,8,523,2017-03-27 08:30:38,http://heathcote.com/drake.graham,,56.44.228.44,"{""location"": ""KE"", ""is_mobile"": true}" 23274,8,523,2016-12-27 14:23:04,http://renner.org/walker,,56.207.54.88,"{""location"": ""DM"", ""is_mobile"": false}" 23275,8,523,2017-01-05 18:28:15,http://runolfonnikolaus.info/maiya,,200.132.58.26,"{""location"": ""IO"", ""is_mobile"": false}" 23276,8,523,2017-01-17 02:09:17,http://bartelllarson.info/buck_corkery,,82.105.29.196,"{""location"": ""CC"", ""is_mobile"": true}" 23277,8,523,2016-12-21 19:16:14,http://raynor.io/abdullah,,206.95.187.53,"{""location"": ""TC"", ""is_mobile"": true}" 23278,8,523,2017-02-19 21:10:12,http://heaney.net/elvie,,198.190.238.131,"{""location"": ""IL"", ""is_mobile"": false}" 23279,8,523,2017-05-20 09:08:14,http://reichert.net/leora,,243.136.88.60,"{""location"": ""BQ"", ""is_mobile"": true}" 23280,8,523,2017-04-28 22:36:33,http://kuhlman.com/alex_upton,,184.240.153.114,"{""location"": ""LY"", ""is_mobile"": true}" 23281,8,523,2017-03-16 04:45:29,http://mann.com/adela,,78.16.254.234,"{""location"": ""AX"", ""is_mobile"": true}" 23282,8,523,2017-03-21 07:58:51,http://halvorsoncummerata.name/jeff,,162.12.236.144,"{""location"": ""AQ"", ""is_mobile"": false}" 23283,8,523,2017-01-19 19:51:04,http://roweokon.org/johnny,,6.191.211.121,"{""location"": ""PK"", ""is_mobile"": true}" 23284,8,523,2017-03-18 03:43:03,http://armstrong.io/duane.hirthe,,203.128.170.222,"{""location"": ""SO"", ""is_mobile"": false}" 23285,8,523,2017-03-16 16:29:14,http://rutherford.com/eveline_rau,,32.209.188.34,"{""location"": ""CL"", ""is_mobile"": false}" 23286,8,523,2017-03-09 17:15:48,http://kuphalkshlerin.com/terence,,150.236.110.18,"{""location"": ""KM"", ""is_mobile"": true}" 23287,8,523,2017-05-23 13:29:17,http://gerhold.org/kenton_kling,,39.238.93.215,"{""location"": ""AR"", ""is_mobile"": false}" 23288,8,523,2017-02-22 06:29:23,http://kerluke.io/dejuan,,28.98.68.157,"{""location"": ""FO"", ""is_mobile"": true}" 23289,8,523,2017-03-05 13:22:44,http://schamberger.net/deven,,31.61.75.201,"{""location"": ""GD"", ""is_mobile"": false}" 23290,8,523,2017-04-29 16:28:43,http://morar.biz/eriberto_kautzer,,145.175.209.224,"{""location"": ""GU"", ""is_mobile"": false}" 23291,8,523,2017-04-27 09:00:46,http://croninlangworth.name/erick.waters,,7.57.32.246,"{""location"": ""DK"", ""is_mobile"": false}" 23292,8,523,2017-06-09 07:31:57,http://bartoletti.info/trinity_jacobson,,164.12.207.122,"{""location"": ""JP"", ""is_mobile"": false}" 23293,8,523,2017-05-12 03:56:27,http://yundt.io/carmela_gislason,,241.159.195.79,"{""location"": ""IM"", ""is_mobile"": true}" 23294,8,523,2017-04-27 13:53:24,http://thiel.net/deja,,70.182.196.176,"{""location"": ""LC"", ""is_mobile"": false}" 23295,8,523,2017-01-24 05:01:39,http://wilkinsonbeier.net/susana_klein,,188.65.2.211,"{""location"": ""ZW"", ""is_mobile"": true}" 23296,8,523,2017-03-05 01:08:35,http://greenfelder.info/wilmer,,231.52.232.169,"{""location"": ""ZW"", ""is_mobile"": true}" 23297,8,523,2017-06-09 22:28:09,http://batzcummings.com/madison,,42.135.75.252,"{""location"": ""CM"", ""is_mobile"": true}" 23298,8,523,2017-04-30 14:42:14,http://howell.com/elenora,,99.36.48.29,"{""location"": ""SH"", ""is_mobile"": false}" 23299,8,523,2017-04-20 04:18:03,http://brakus.org/major,,66.56.237.93,"{""location"": ""MF"", ""is_mobile"": true}" 23300,8,523,2017-05-18 01:49:32,http://cruickshank.name/bailey.ruel,,16.134.14.192,"{""location"": ""BN"", ""is_mobile"": false}" 23301,8,523,2017-05-10 12:20:55,http://runolfsdottir.org/adrianna.dooley,,231.204.203.207,"{""location"": ""TH"", ""is_mobile"": true}" 23302,8,523,2017-02-13 11:05:03,http://bauch.com/lora,,78.153.72.186,"{""location"": ""MN"", ""is_mobile"": false}" 23303,8,523,2017-04-26 08:26:48,http://murazikstanton.co/pete,,16.243.64.189,"{""location"": ""CI"", ""is_mobile"": true}" 23304,8,523,2017-05-06 03:26:27,http://bechtelar.co/kayley.hermiston,,191.148.160.120,"{""location"": ""LS"", ""is_mobile"": true}" 23305,8,523,2017-01-31 13:58:25,http://bayer.net/jaden.oberbrunner,,110.48.152.205,"{""location"": ""HM"", ""is_mobile"": true}" 23306,8,523,2017-01-17 12:35:43,http://grant.co/mohammed.kshlerin,,220.203.26.89,"{""location"": ""DK"", ""is_mobile"": true}" 23307,8,523,2017-02-06 16:07:55,http://sawayntrantow.io/zetta.erdman,,201.68.6.219,"{""location"": ""CC"", ""is_mobile"": true}" 23308,8,523,2017-02-10 03:57:16,http://osinski.co/allan.bradtke,,81.189.105.49,"{""location"": ""PN"", ""is_mobile"": true}" 23309,8,523,2016-12-20 18:36:11,http://stamm.name/marietta,,67.138.53.26,"{""location"": ""TT"", ""is_mobile"": false}" 23310,8,523,2017-03-07 15:03:10,http://effertzharvey.info/trey,,200.192.220.153,"{""location"": ""MT"", ""is_mobile"": false}" 23311,8,523,2017-02-28 03:59:38,http://weimann.io/makenzie,,31.187.189.60,"{""location"": ""JE"", ""is_mobile"": true}" 23312,8,523,2017-02-04 15:25:13,http://haley.com/miles,,197.4.26.3,"{""location"": ""GT"", ""is_mobile"": false}" 23313,8,523,2017-02-22 09:41:38,http://gorczany.name/jennings,,251.29.70.58,"{""location"": ""SO"", ""is_mobile"": true}" 23314,8,523,2017-04-12 00:39:52,http://dach.co/nathan.macgyver,,192.243.92.143,"{""location"": ""BE"", ""is_mobile"": false}" 23315,8,523,2017-01-17 08:45:37,http://boehm.biz/tracy.murray,,204.19.172.245,"{""location"": ""BT"", ""is_mobile"": false}" 23316,8,523,2017-01-16 10:38:20,http://hamill.net/yazmin,,81.66.219.242,"{""location"": ""PM"", ""is_mobile"": true}" 23317,8,523,2017-05-07 00:08:12,http://beatty.org/nasir,,252.22.80.89,"{""location"": ""CX"", ""is_mobile"": true}" 23318,8,523,2017-06-07 09:53:28,http://bayer.biz/wilfrid.bayer,,116.140.95.123,"{""location"": ""KE"", ""is_mobile"": true}" 23319,8,523,2017-05-28 07:15:13,http://zemlak.org/erik_schimmel,,30.168.245.207,"{""location"": ""AM"", ""is_mobile"": false}" 23320,8,523,2017-04-10 13:56:42,http://farrellhackett.net/casandra,,94.194.2.81,"{""location"": ""MU"", ""is_mobile"": true}" 23321,8,523,2017-05-26 20:26:28,http://trantow.info/dayton_shanahan,,160.128.53.211,"{""location"": ""NE"", ""is_mobile"": false}" 23322,8,523,2016-12-30 04:43:25,http://kerluke.co/lizzie.rosenbaum,,222.202.132.35,"{""location"": ""MZ"", ""is_mobile"": true}" 23323,8,523,2017-05-04 00:17:03,http://klocko.io/ryann,,253.135.82.41,"{""location"": ""SV"", ""is_mobile"": true}" 23324,8,524,2017-03-01 01:00:39,http://feilkemmer.io/joshuah,,206.9.16.34,"{""location"": ""GT"", ""is_mobile"": true}" 2257,1,51,2017-05-05 16:31:19,http://spinka.info/ashlee,,199.122.58.194,"{""location"": ""VN"", ""is_mobile"": true}" 2258,1,51,2017-03-07 05:17:16,http://hellerwilliamson.net/emmet,,40.149.134.142,"{""location"": ""MC"", ""is_mobile"": false}" 2259,1,51,2017-05-15 23:08:29,http://kuhlman.info/rodger_wilderman,,253.197.214.204,"{""location"": ""ML"", ""is_mobile"": true}" 2260,1,51,2016-12-18 11:50:03,http://wiza.biz/troy_moen,,51.69.14.19,"{""location"": ""ET"", ""is_mobile"": true}" 2261,1,51,2017-03-16 23:17:55,http://huel.io/jacinto,,247.82.230.66,"{""location"": ""UG"", ""is_mobile"": true}" 2262,1,51,2017-06-12 19:30:09,http://goldner.info/roxanne,,158.117.156.85,"{""location"": ""BF"", ""is_mobile"": false}" 2263,1,51,2017-04-08 23:05:38,http://kohler.co/pamela.nader,,72.72.82.183,"{""location"": ""GL"", ""is_mobile"": true}" 2264,1,51,2017-03-01 00:58:48,http://jones.co/owen,,166.97.131.115,"{""location"": ""US"", ""is_mobile"": false}" 2265,1,51,2017-03-26 16:57:53,http://torphy.name/kaia,,225.195.71.71,"{""location"": ""IQ"", ""is_mobile"": true}" 2266,1,51,2017-03-15 02:07:48,http://gleason.net/haskell.price,,57.161.84.200,"{""location"": ""DK"", ""is_mobile"": false}" 2267,1,51,2017-04-20 17:37:41,http://ortiz.com/jaycee_raynor,,2.183.83.75,"{""location"": ""CO"", ""is_mobile"": false}" 2268,1,51,2017-06-11 11:51:07,http://dach.org/jenifer,,24.203.187.225,"{""location"": ""DK"", ""is_mobile"": true}" 2269,1,51,2017-04-25 00:33:50,http://oberbrunner.io/karson_marks,,150.95.216.116,"{""location"": ""CG"", ""is_mobile"": false}" 2270,1,51,2016-12-29 23:50:11,http://jerde.io/hailee,,30.51.7.189,"{""location"": ""VG"", ""is_mobile"": false}" 2271,1,51,2017-02-27 16:15:12,http://von.org/elisa,,139.157.8.195,"{""location"": ""CY"", ""is_mobile"": false}" 2272,1,51,2016-12-29 13:12:31,http://goyettebruen.name/arlie_prosacco,,153.26.6.83,"{""location"": ""IQ"", ""is_mobile"": false}" 2273,1,51,2017-05-11 20:19:10,http://mills.com/paula.mante,,176.21.59.91,"{""location"": ""SS"", ""is_mobile"": true}" 2274,1,51,2017-01-02 09:45:07,http://stehr.info/isaiah.hoppe,,168.88.252.15,"{""location"": ""CK"", ""is_mobile"": true}" 2275,1,51,2017-02-20 08:00:31,http://rosenbaum.io/santa,,94.65.69.195,"{""location"": ""LA"", ""is_mobile"": true}" 2276,1,51,2017-01-26 00:25:45,http://stiedemann.com/randal.bode,,57.189.233.113,"{""location"": ""MQ"", ""is_mobile"": false}" 2277,1,51,2017-03-29 20:03:11,http://sawaynryan.net/drake,,37.120.93.155,"{""location"": ""GD"", ""is_mobile"": true}" 2278,1,51,2017-02-13 22:50:00,http://larsonhalvorson.biz/eliza.bernhard,,248.77.218.220,"{""location"": ""FO"", ""is_mobile"": true}" 2279,1,51,2017-06-04 21:27:00,http://mills.com/lucy.wiegand,,124.146.182.197,"{""location"": ""PG"", ""is_mobile"": true}" 2280,1,51,2017-05-22 04:08:18,http://kautzer.org/elliot.spinka,,233.46.236.169,"{""location"": ""PH"", ""is_mobile"": false}" 2281,1,51,2017-04-14 07:54:05,http://wilderman.info/hector,,137.238.73.181,"{""location"": ""EG"", ""is_mobile"": true}" 2282,1,51,2017-03-28 16:34:45,http://satterfield.info/halie.hane,,106.49.189.153,"{""location"": ""AF"", ""is_mobile"": false}" 2283,1,51,2017-06-08 07:56:38,http://dickensbeahan.name/tate,,175.51.80.186,"{""location"": ""MT"", ""is_mobile"": false}" 2284,1,51,2017-01-08 11:43:02,http://buckridge.net/letitia_ward,,51.220.74.131,"{""location"": ""NG"", ""is_mobile"": false}" 2285,1,51,2017-02-01 04:54:00,http://welchmayert.net/keely,,193.225.26.2,"{""location"": ""GB"", ""is_mobile"": true}" 2286,1,51,2017-02-14 14:26:40,http://langworthstokes.co/reid,,197.246.249.59,"{""location"": ""MH"", ""is_mobile"": true}" 2287,1,51,2017-01-30 22:16:53,http://cain.com/hosea,,232.186.107.5,"{""location"": ""BY"", ""is_mobile"": true}" 2288,1,51,2017-04-07 09:20:35,http://grahammiller.io/citlalli_lubowitz,,100.3.144.164,"{""location"": ""TV"", ""is_mobile"": false}" 2289,1,51,2016-12-13 12:34:56,http://gerlach.co/haskell,,143.92.242.126,"{""location"": ""DZ"", ""is_mobile"": false}" 2290,1,51,2017-03-26 23:19:12,http://kunde.com/libbie.treutel,,239.236.57.224,"{""location"": ""GA"", ""is_mobile"": true}" 2291,1,51,2017-06-08 03:47:07,http://kub.com/emelie.fahey,,176.193.129.250,"{""location"": ""HN"", ""is_mobile"": false}" 2292,1,51,2017-04-05 09:02:48,http://farrellleuschke.org/fatima_hegmann,,190.126.74.58,"{""location"": ""LI"", ""is_mobile"": true}" 2293,1,51,2017-03-06 03:43:23,http://wiegandkeebler.biz/garry_leffler,,182.41.107.72,"{""location"": ""VA"", ""is_mobile"": false}" 2294,1,51,2017-02-18 19:51:08,http://ledner.co/alan,,245.222.82.39,"{""location"": ""AE"", ""is_mobile"": true}" 2295,1,51,2017-02-19 05:16:51,http://bosco.com/filiberto_kris,,14.215.65.198,"{""location"": ""ZM"", ""is_mobile"": false}" 2296,1,51,2017-04-30 20:39:46,http://von.com/sebastian,,163.159.82.173,"{""location"": ""BG"", ""is_mobile"": true}" 2297,1,51,2017-04-22 08:38:41,http://purdy.org/chelsea,,108.124.109.172,"{""location"": ""CV"", ""is_mobile"": false}" 2298,1,51,2017-03-17 04:06:24,http://schinner.net/dorothea_swaniawski,,139.32.147.156,"{""location"": ""AR"", ""is_mobile"": true}" 2299,1,51,2016-12-31 01:27:38,http://littel.co/arielle,,67.228.162.194,"{""location"": ""NO"", ""is_mobile"": false}" 2300,1,51,2016-12-31 18:01:29,http://bailey.co/oceane_beer,,113.242.214.133,"{""location"": ""CK"", ""is_mobile"": false}" 2301,1,51,2017-01-27 08:04:31,http://wisokyluettgen.name/madelyn,,194.27.248.41,"{""location"": ""MH"", ""is_mobile"": false}" 2302,1,51,2017-02-25 05:55:41,http://gaylordrice.info/tatum_strosin,,220.209.99.105,"{""location"": ""MV"", ""is_mobile"": true}" 2303,1,51,2017-04-10 03:12:59,http://trantow.biz/delphia,,251.225.176.253,"{""location"": ""SL"", ""is_mobile"": false}" 2304,1,51,2017-02-24 00:45:49,http://stiedemann.org/guiseppe,,116.64.107.43,"{""location"": ""FO"", ""is_mobile"": true}" 2305,1,51,2017-03-22 17:21:07,http://wuckert.biz/cicero_hauck,,28.39.159.223,"{""location"": ""TO"", ""is_mobile"": true}" 2306,1,51,2017-03-20 14:25:36,http://robertscruickshank.biz/minerva_dicki,,6.23.119.119,"{""location"": ""SJ"", ""is_mobile"": false}" 2307,1,51,2017-04-02 14:14:07,http://rempel.org/josue_conroy,,244.231.121.250,"{""location"": ""GU"", ""is_mobile"": false}" 2308,1,51,2017-05-01 18:14:12,http://legroslarkin.io/nash_bins,,234.250.89.249,"{""location"": ""SD"", ""is_mobile"": true}" 2309,1,51,2017-02-11 08:29:37,http://baumbach.io/thea_reichel,,159.78.11.64,"{""location"": ""GD"", ""is_mobile"": false}" 2310,1,51,2017-06-04 19:09:57,http://hayes.net/aiden_ziemann,,198.114.252.62,"{""location"": ""LS"", ""is_mobile"": false}" 2311,1,51,2017-03-24 10:47:47,http://roob.name/nyah,,190.6.52.191,"{""location"": ""PK"", ""is_mobile"": false}" 2312,1,51,2017-05-01 19:47:03,http://lakin.name/chasity.zieme,,208.42.15.4,"{""location"": ""RU"", ""is_mobile"": false}" 23325,8,524,2017-04-15 02:54:03,http://beier.com/karson,,125.224.197.142,"{""location"": ""AQ"", ""is_mobile"": false}" 23326,8,524,2017-03-04 08:30:48,http://boehmoreilly.io/lesley_fritsch,,123.116.57.212,"{""location"": ""SY"", ""is_mobile"": false}" 23327,8,524,2017-06-10 18:44:38,http://waelchi.co/reese,,26.199.118.88,"{""location"": ""LS"", ""is_mobile"": true}" 23328,8,524,2016-12-19 02:22:50,http://prohaskawhite.org/esther,,136.119.21.245,"{""location"": ""MW"", ""is_mobile"": false}" 23329,8,524,2017-03-28 18:29:27,http://hegmannhoeger.info/lyla,,240.4.88.7,"{""location"": ""GE"", ""is_mobile"": true}" 23330,8,524,2017-03-08 15:17:07,http://rutherfordschuster.org/abbigail,,190.217.76.170,"{""location"": ""AG"", ""is_mobile"": true}" 23331,8,524,2017-05-15 20:04:38,http://ziememclaughlin.net/alene,,202.183.212.51,"{""location"": ""GN"", ""is_mobile"": true}" 23332,8,524,2017-06-01 08:31:11,http://krichimmel.name/garfield.wehner,,133.72.113.20,"{""location"": ""MS"", ""is_mobile"": false}" 23333,8,524,2017-04-18 01:40:33,http://kshlerinjacobson.info/christelle,,136.26.76.119,"{""location"": ""BV"", ""is_mobile"": false}" 23334,8,524,2017-05-26 10:19:55,http://thiel.co/mallie_ernser,,87.28.175.56,"{""location"": ""AQ"", ""is_mobile"": false}" 23335,8,524,2017-06-10 19:20:14,http://rath.name/elisabeth,,61.32.102.199,"{""location"": ""PL"", ""is_mobile"": false}" 23336,8,524,2017-01-17 06:46:09,http://brakus.info/retta,,106.88.91.79,"{""location"": ""MR"", ""is_mobile"": true}" 23337,8,524,2017-06-08 15:15:10,http://kling.io/noah,,138.224.40.130,"{""location"": ""AO"", ""is_mobile"": false}" 23338,8,524,2017-05-19 07:45:45,http://treutelleannon.net/vallie,,45.19.95.236,"{""location"": ""BQ"", ""is_mobile"": true}" 23339,8,524,2017-01-15 20:51:01,http://breitenberg.biz/keshaun.lesch,,243.213.68.197,"{""location"": ""EE"", ""is_mobile"": true}" 23340,8,524,2016-12-29 22:42:36,http://heidenreichrowe.io/mariane.klocko,,153.8.120.28,"{""location"": ""MA"", ""is_mobile"": false}" 23341,8,524,2017-01-27 12:51:47,http://zulauf.net/benedict.daniel,,24.181.190.147,"{""location"": ""EC"", ""is_mobile"": false}" 23342,8,524,2017-02-15 04:43:55,http://romaguera.io/darien_windler,,122.149.77.244,"{""location"": ""BG"", ""is_mobile"": true}" 23343,8,524,2017-01-05 06:24:21,http://flatleyfarrell.io/charity,,70.131.79.131,"{""location"": ""MV"", ""is_mobile"": false}" 23344,8,524,2017-04-03 17:37:02,http://heathcotekeler.net/camilla,,158.234.240.15,"{""location"": ""US"", ""is_mobile"": true}" 23345,8,524,2017-05-09 23:19:51,http://denesiklowe.io/zoe,,230.97.254.22,"{""location"": ""MA"", ""is_mobile"": true}" 23346,8,524,2017-03-27 06:46:34,http://rempel.biz/polly_bergstrom,,190.11.176.20,"{""location"": ""AD"", ""is_mobile"": true}" 23347,8,524,2017-01-09 15:01:32,http://kling.co/cordia,,87.250.146.56,"{""location"": ""CK"", ""is_mobile"": true}" 23348,8,524,2016-12-25 21:20:17,http://kirlin.co/marcella,,38.193.185.126,"{""location"": ""KE"", ""is_mobile"": true}" 23349,8,524,2017-03-13 01:55:38,http://brown.biz/abe,,29.234.235.120,"{""location"": ""TH"", ""is_mobile"": true}" 23350,8,524,2017-06-13 04:46:05,http://rodriguezdaniel.name/savanah,,58.251.198.183,"{""location"": ""RW"", ""is_mobile"": false}" 23351,8,524,2016-12-31 16:53:33,http://wintheiserhegmann.io/linwood,,183.120.116.149,"{""location"": ""GB"", ""is_mobile"": false}" 23352,8,524,2016-12-29 11:52:26,http://mccluremurphy.io/dorris.trantow,,211.28.34.68,"{""location"": ""BA"", ""is_mobile"": false}" 23353,8,524,2017-03-09 20:08:17,http://flatleybartoletti.org/catharine,,125.7.31.28,"{""location"": ""CY"", ""is_mobile"": false}" 23354,8,524,2017-04-26 15:49:29,http://langhudson.name/abraham,,119.114.64.221,"{""location"": ""BB"", ""is_mobile"": false}" 23355,8,524,2017-06-01 08:37:00,http://goldnerhilll.io/kayla.cormier,,17.173.35.122,"{""location"": ""ME"", ""is_mobile"": false}" 23356,8,524,2017-03-04 14:16:31,http://hauck.io/adele.schoen,,191.73.238.3,"{""location"": ""AT"", ""is_mobile"": true}" 23357,8,524,2017-03-12 13:42:59,http://wiza.com/athena.rutherford,,83.220.229.8,"{""location"": ""IT"", ""is_mobile"": false}" 23358,8,524,2017-01-14 23:52:02,http://kilback.biz/bella.kaulke,,221.148.111.125,"{""location"": ""FK"", ""is_mobile"": false}" 23359,8,524,2017-03-29 21:52:35,http://halvorsonhansen.com/domenica,,20.156.33.217,"{""location"": ""CL"", ""is_mobile"": false}" 23360,8,524,2017-04-19 13:16:20,http://welch.name/arnold.mcglynn,,81.74.143.164,"{""location"": ""CX"", ""is_mobile"": false}" 23361,8,524,2017-06-07 05:12:28,http://volkman.net/tyreek_nienow,,159.157.253.62,"{""location"": ""SL"", ""is_mobile"": false}" 23362,8,524,2017-05-05 19:54:53,http://ebertstreich.co/regan_olson,,48.122.176.111,"{""location"": ""TH"", ""is_mobile"": true}" 23363,8,524,2017-02-21 13:29:08,http://robel.io/maeve_maggio,,208.27.148.121,"{""location"": ""BA"", ""is_mobile"": true}" 23364,8,524,2017-05-09 11:22:42,http://auerwuckert.io/pearline,,65.227.240.150,"{""location"": ""UY"", ""is_mobile"": true}" 23365,8,524,2017-04-16 19:30:56,http://schmidt.io/jeie,,11.132.48.249,"{""location"": ""PN"", ""is_mobile"": false}" 23366,8,524,2017-01-02 10:59:18,http://schultzweinat.org/kitty,,37.125.208.186,"{""location"": ""NR"", ""is_mobile"": false}" 23367,8,524,2017-01-23 01:17:41,http://prosaccovandervort.co/winnifred,,131.94.131.28,"{""location"": ""SI"", ""is_mobile"": true}" 23368,8,524,2017-03-15 04:11:59,http://moriette.info/scot_streich,,187.26.16.84,"{""location"": ""ZA"", ""is_mobile"": true}" 23369,8,524,2017-01-21 21:59:22,http://kunze.io/providenci,,179.19.22.68,"{""location"": ""SE"", ""is_mobile"": true}" 23370,8,524,2017-03-16 01:20:42,http://terry.io/bridget,,35.26.169.124,"{""location"": ""TN"", ""is_mobile"": true}" 23371,8,524,2017-04-15 07:37:42,http://boehmhuel.co/libbie_stamm,,106.95.17.168,"{""location"": ""LK"", ""is_mobile"": true}" 23372,8,524,2017-03-23 21:15:16,http://durgan.biz/justen,,87.90.185.8,"{""location"": ""PR"", ""is_mobile"": true}" 23373,8,524,2016-12-22 20:19:35,http://mayert.info/albert,,61.33.222.198,"{""location"": ""LR"", ""is_mobile"": false}" 23374,8,524,2017-01-27 18:58:03,http://okuneva.io/cara,,193.242.155.29,"{""location"": ""HM"", ""is_mobile"": true}" 23375,8,524,2016-12-22 03:37:56,http://willms.biz/corrine,,34.45.86.45,"{""location"": ""TC"", ""is_mobile"": true}" 23376,8,524,2017-02-12 23:40:29,http://weinatschiller.io/baby_moen,,92.207.193.52,"{""location"": ""BW"", ""is_mobile"": true}" 23377,8,524,2017-04-02 11:13:35,http://mann.org/mittie,,104.53.79.102,"{""location"": ""BY"", ""is_mobile"": false}" 23378,8,524,2017-01-23 03:52:20,http://ruecker.net/ignatius,,54.81.28.204,"{""location"": ""TG"", ""is_mobile"": false}" 23379,8,524,2017-02-21 07:10:21,http://oberbrunnerwilliamson.name/maxine.pollich,,244.48.199.27,"{""location"": ""NR"", ""is_mobile"": true}" 23380,8,524,2017-02-27 13:32:20,http://macgyver.net/beau_shanahan,,252.152.61.65,"{""location"": ""RU"", ""is_mobile"": true}" 2313,1,51,2017-05-07 09:35:22,http://strosin.co/krystal_hettinger,,102.165.139.208,"{""location"": ""GN"", ""is_mobile"": true}" 2314,1,51,2017-04-12 19:46:15,http://labadie.name/laurel_feest,,45.6.114.18,"{""location"": ""AL"", ""is_mobile"": false}" 23381,8,524,2017-01-13 13:38:42,http://casper.info/wilson.jacobson,,53.141.103.49,"{""location"": ""AM"", ""is_mobile"": false}" 23382,8,524,2017-01-25 07:23:13,http://bailey.biz/roberta_schinner,,243.61.96.168,"{""location"": ""NU"", ""is_mobile"": true}" 23383,8,524,2017-06-07 06:39:04,http://pfeffer.io/hilbert,,196.11.5.11,"{""location"": ""IR"", ""is_mobile"": false}" 23384,8,524,2017-03-05 11:26:04,http://feeneysteuber.biz/janea.leffler,,47.42.109.179,"{""location"": ""DM"", ""is_mobile"": false}" 23385,8,524,2017-04-29 00:09:51,http://morar.com/lue,,10.177.7.87,"{""location"": ""NP"", ""is_mobile"": false}" 23386,8,524,2017-03-16 22:37:47,http://kling.info/teagan.flatley,,204.54.186.123,"{""location"": ""VA"", ""is_mobile"": true}" 23387,8,524,2017-04-14 04:54:16,http://larson.io/bradly_mertz,,190.216.84.246,"{""location"": ""VA"", ""is_mobile"": true}" 23388,8,524,2017-01-10 15:22:57,http://powlowski.info/rudolph.schulist,,97.5.3.247,"{""location"": ""GS"", ""is_mobile"": false}" 23389,8,524,2016-12-29 12:08:15,http://schmelerwatsica.net/gaylord_beahan,,205.101.60.49,"{""location"": ""NU"", ""is_mobile"": true}" 23390,8,525,2017-05-24 01:05:40,http://hayes.info/lorenza,,5.87.127.204,"{""location"": ""MC"", ""is_mobile"": false}" 23391,8,525,2017-06-09 06:38:51,http://beier.biz/mallie,,178.83.232.95,"{""location"": ""MZ"", ""is_mobile"": false}" 23392,8,525,2017-04-28 09:25:14,http://towne.io/tiana,,236.238.186.183,"{""location"": ""ET"", ""is_mobile"": false}" 23393,8,525,2017-04-11 15:39:03,http://goldnerstark.com/heath.schumm,,202.213.244.3,"{""location"": ""DM"", ""is_mobile"": false}" 23394,8,525,2017-01-11 08:30:32,http://stracke.biz/myrtice,,15.139.143.124,"{""location"": ""SC"", ""is_mobile"": false}" 23395,8,525,2017-03-09 03:27:04,http://sauerwaelchi.co/kaelyn,,142.8.167.50,"{""location"": ""ST"", ""is_mobile"": false}" 23396,8,525,2017-06-09 07:34:35,http://romaguera.name/reina.mcdermott,,88.201.24.118,"{""location"": ""US"", ""is_mobile"": true}" 23397,8,525,2017-04-05 03:34:17,http://kiehn.biz/jaquan_borer,,96.235.40.50,"{""location"": ""LV"", ""is_mobile"": true}" 23398,8,525,2017-01-27 22:11:48,http://larkin.biz/leora,,69.244.233.186,"{""location"": ""TN"", ""is_mobile"": true}" 23399,8,525,2017-04-08 18:55:58,http://gaylordhirthe.net/kody,,92.71.95.187,"{""location"": ""TD"", ""is_mobile"": true}" 23400,8,525,2017-03-16 11:06:34,http://rogahn.net/leland_erdman,,209.52.177.100,"{""location"": ""JO"", ""is_mobile"": false}" 23401,8,525,2017-01-18 05:05:47,http://homenick.biz/cruz,,31.246.112.67,"{""location"": ""MR"", ""is_mobile"": true}" 23402,8,525,2017-05-05 20:17:46,http://ratkemcdermott.io/declan_rowe,,99.114.110.242,"{""location"": ""PT"", ""is_mobile"": true}" 23403,8,525,2017-03-09 02:17:46,http://king.name/shaniya_bayer,,254.120.144.54,"{""location"": ""GG"", ""is_mobile"": true}" 23404,8,525,2017-03-26 02:15:19,http://cremin.org/shaun.zemlak,,230.19.126.143,"{""location"": ""VC"", ""is_mobile"": false}" 23405,8,525,2017-05-26 22:56:52,http://wyman.net/lorenza.fadel,,72.95.20.59,"{""location"": ""MT"", ""is_mobile"": true}" 23406,8,525,2017-06-09 02:20:24,http://gleichner.com/weston.bailey,,16.147.70.245,"{""location"": ""TO"", ""is_mobile"": true}" 23407,8,525,2017-01-04 12:09:10,http://hansenankunding.co/clara_pollich,,28.120.233.113,"{""location"": ""GW"", ""is_mobile"": true}" 23408,8,525,2017-05-06 05:33:26,http://quigley.io/trey_renner,,129.240.107.243,"{""location"": ""WF"", ""is_mobile"": false}" 23409,8,525,2017-05-28 19:12:08,http://predovic.name/sage.kunze,,58.220.56.115,"{""location"": ""SY"", ""is_mobile"": false}" 23410,8,525,2017-03-28 19:13:53,http://wisokybernhard.org/krystal.kiehn,,80.128.15.208,"{""location"": ""IR"", ""is_mobile"": false}" 23411,8,525,2017-05-16 10:35:48,http://beer.info/marcelle,,44.199.136.70,"{""location"": ""LA"", ""is_mobile"": false}" 23412,8,525,2017-05-31 06:55:27,http://goldnerromaguera.net/armani_schiller,,99.238.214.4,"{""location"": ""UY"", ""is_mobile"": true}" 23413,8,525,2017-04-30 07:56:14,http://windler.net/lina,,102.169.14.222,"{""location"": ""NR"", ""is_mobile"": true}" 23414,8,525,2017-05-06 01:30:50,http://king.name/darrion_muller,,155.65.92.216,"{""location"": ""AG"", ""is_mobile"": true}" 23415,8,525,2017-03-02 15:22:51,http://kuhn.biz/javonte,,122.27.215.14,"{""location"": ""TJ"", ""is_mobile"": false}" 23416,8,525,2017-05-12 09:27:26,http://langosh.co/riley.heaney,,173.151.26.146,"{""location"": ""LB"", ""is_mobile"": true}" 23417,8,525,2017-01-22 13:36:36,http://mcglynn.org/autumn,,29.177.244.60,"{""location"": ""IT"", ""is_mobile"": false}" 23418,8,525,2017-04-03 14:45:04,http://hammes.org/aimee.bednar,,157.85.81.83,"{""location"": ""YE"", ""is_mobile"": false}" 23419,8,525,2016-12-30 14:44:55,http://kubrau.io/monserrate,,28.121.96.231,"{""location"": ""BJ"", ""is_mobile"": true}" 23420,8,525,2017-03-10 03:14:58,http://hahn.org/desiree_ko,,114.92.218.12,"{""location"": ""CA"", ""is_mobile"": false}" 23421,8,525,2017-04-07 17:36:11,http://padbergtillman.org/rosalia,,158.66.141.195,"{""location"": ""LC"", ""is_mobile"": true}" 23422,8,525,2017-06-04 13:45:18,http://farrellerdman.io/claude,,11.80.77.223,"{""location"": ""IO"", ""is_mobile"": false}" 23423,8,525,2017-01-27 10:06:15,http://lind.com/orpha,,112.57.92.35,"{""location"": ""TO"", ""is_mobile"": false}" 23424,8,525,2017-03-01 14:41:38,http://hilll.name/dewayne.pfannerstill,,169.157.187.50,"{""location"": ""MA"", ""is_mobile"": true}" 23425,8,525,2017-03-11 07:16:43,http://naderbeer.name/henry,,196.111.229.92,"{""location"": ""SD"", ""is_mobile"": false}" 23426,8,525,2017-02-23 20:56:28,http://rippin.biz/samara,,58.126.57.250,"{""location"": ""IM"", ""is_mobile"": true}" 23427,8,525,2017-02-09 13:14:03,http://gusikowskicruickshank.info/harrison,,203.76.189.145,"{""location"": ""AO"", ""is_mobile"": false}" 23428,8,525,2017-01-01 20:26:53,http://zemlakwisozk.co/marie,,151.246.24.32,"{""location"": ""AO"", ""is_mobile"": true}" 23429,8,525,2017-05-12 20:13:48,http://schmeler.co/pascale,,11.236.179.154,"{""location"": ""GN"", ""is_mobile"": true}" 23430,8,525,2017-03-11 20:06:56,http://borerpouros.org/thaddeus,,175.137.20.32,"{""location"": ""CZ"", ""is_mobile"": false}" 23431,8,525,2016-12-30 16:45:57,http://schimmelweber.biz/kevin,,195.136.52.22,"{""location"": ""AZ"", ""is_mobile"": false}" 23432,8,525,2017-06-04 02:34:44,http://larson.io/keaton.lesch,,165.156.9.200,"{""location"": ""GU"", ""is_mobile"": false}" 23433,8,525,2017-03-08 01:43:15,http://stroman.biz/ibrahim_tillman,,58.161.174.52,"{""location"": ""NL"", ""is_mobile"": true}" 23434,8,525,2017-03-14 13:56:06,http://waelchipollich.org/keanu.reinger,,209.243.141.250,"{""location"": ""GB"", ""is_mobile"": false}" 23435,8,525,2016-12-23 05:29:32,http://hettingerlueilwitz.co/idell,,240.86.238.206,"{""location"": ""DK"", ""is_mobile"": true}" 23436,8,525,2017-04-02 20:16:11,http://murazikcollier.info/pattie_heathcote,,63.221.249.5,"{""location"": ""CH"", ""is_mobile"": false}" 23437,8,525,2017-02-21 00:33:14,http://rice.net/brennan,,167.89.69.206,"{""location"": ""TL"", ""is_mobile"": false}" 23438,8,525,2017-01-11 17:23:52,http://ziemann.co/mohammed.heidenreich,,187.103.85.37,"{""location"": ""TH"", ""is_mobile"": true}" 23439,8,525,2017-02-05 05:02:27,http://hansen.org/aglae_sawayn,,110.22.215.110,"{""location"": ""BJ"", ""is_mobile"": false}" 23440,8,525,2017-06-04 09:59:24,http://kingwillms.co/westley_olson,,158.31.123.246,"{""location"": ""SH"", ""is_mobile"": false}" 23441,8,525,2017-03-24 18:50:42,http://kozeymccullough.net/margarita,,244.200.101.176,"{""location"": ""EC"", ""is_mobile"": false}" 23442,8,525,2017-04-30 15:48:29,http://stokes.name/halle_brakus,,101.21.135.126,"{""location"": ""PK"", ""is_mobile"": false}" 23443,8,525,2017-01-13 01:14:56,http://thompson.com/briana.schmidt,,68.134.254.162,"{""location"": ""IM"", ""is_mobile"": true}" 23444,8,525,2017-05-17 12:52:31,http://miller.net/garth.braun,,79.8.18.104,"{""location"": ""AM"", ""is_mobile"": true}" 23445,8,525,2017-01-19 00:33:22,http://smitham.com/heidi_smitham,,30.173.211.164,"{""location"": ""UZ"", ""is_mobile"": false}" 23446,8,525,2017-01-22 15:11:00,http://haley.co/larue.greenfelder,,201.206.145.36,"{""location"": ""IR"", ""is_mobile"": true}" 23447,8,525,2017-05-14 08:53:59,http://friesenkuhic.name/tyrique_kulas,,213.81.133.114,"{""location"": ""ME"", ""is_mobile"": false}" 23448,8,525,2017-03-29 02:31:15,http://rohantillman.biz/elnora.ernser,,85.51.162.10,"{""location"": ""AX"", ""is_mobile"": true}" 8683,4,195,2017-06-03 21:19:16,http://kelerlarson.org/everardo,0.7837852015,57.116.161.131,"{""location"": ""UA"", ""is_mobile"": false}" 8684,4,195,2017-02-11 22:45:01,http://ward.co/julia,0.6341351753,13.103.93.167,"{""location"": ""TC"", ""is_mobile"": true}" 8685,4,195,2017-04-14 02:15:19,http://dietrichquitzon.io/paxton.erdman,0.4488990600,165.159.3.148,"{""location"": ""CA"", ""is_mobile"": false}" 8686,4,195,2017-05-08 03:26:40,http://hand.com/xzavier,0.1773083712,254.198.137.54,"{""location"": ""PL"", ""is_mobile"": false}" 8687,4,195,2016-12-14 20:19:49,http://king.io/dayana,0.5574280453,7.223.122.147,"{""location"": ""NR"", ""is_mobile"": false}" 8688,4,195,2017-05-15 22:26:32,http://abbott.biz/elta,0.0224180187,249.118.198.27,"{""location"": ""NP"", ""is_mobile"": true}" 8689,4,195,2017-06-12 15:01:39,http://treutelpadberg.co/may,0.6318311955,188.195.187.121,"{""location"": ""UY"", ""is_mobile"": false}" 8690,4,195,2017-05-11 14:35:42,http://auer.biz/jude,0.3261076292,217.171.48.109,"{""location"": ""TJ"", ""is_mobile"": false}" 8691,4,195,2017-03-19 15:51:52,http://brakus.biz/te_cremin,0.5271661267,37.112.254.131,"{""location"": ""SY"", ""is_mobile"": true}" 8692,4,195,2017-02-03 17:41:06,http://funkkuhic.io/jaren_stoltenberg,0.9053255429,238.14.207.110,"{""location"": ""GL"", ""is_mobile"": false}" 8693,4,195,2017-05-12 20:29:42,http://purdy.co/jon,0.5588179133,92.62.116.82,"{""location"": ""KN"", ""is_mobile"": false}" 8694,4,195,2016-12-17 12:13:26,http://abshirekuvalis.co/shayna,0.5505128310,211.101.168.143,"{""location"": ""CR"", ""is_mobile"": false}" 8695,4,195,2017-04-21 06:31:21,http://rowe.net/broderick_jacobson,0.9659758026,58.198.251.128,"{""location"": ""MG"", ""is_mobile"": false}" 8696,4,195,2017-05-23 09:15:35,http://aufderhargibson.io/mariane,0.7967229161,243.246.226.152,"{""location"": ""SO"", ""is_mobile"": false}" 8697,4,195,2017-01-25 22:30:28,http://rippinleuschke.com/emilie.bashirian,0.6919455416,34.113.66.191,"{""location"": ""HR"", ""is_mobile"": false}" 8698,4,195,2017-05-09 03:55:34,http://miller.name/sammy_hamill,0.3681866896,88.233.168.141,"{""location"": ""SE"", ""is_mobile"": false}" 8699,4,195,2017-01-02 11:49:43,http://yundtschoen.net/walker_walter,0.9930442368,95.209.77.90,"{""location"": ""MF"", ""is_mobile"": true}" 8700,4,195,2016-12-24 05:08:52,http://conroy.biz/raymond_witting,0.7068929068,166.121.146.37,"{""location"": ""VU"", ""is_mobile"": true}" 8701,4,195,2017-04-12 08:33:19,http://king.name/markus_corkery,0.9497535706,167.190.69.96,"{""location"": ""BQ"", ""is_mobile"": false}" 8702,4,195,2017-05-25 01:39:50,http://dicki.co/keith_effertz,0.5056025923,199.236.182.189,"{""location"": ""MF"", ""is_mobile"": true}" 8703,4,195,2017-05-10 19:21:03,http://ziemeullrich.info/emile,0.9636015368,131.98.129.195,"{""location"": ""SK"", ""is_mobile"": false}" 8704,4,195,2017-05-13 09:22:22,http://terry.co/murphy,0.7034442891,39.215.202.176,"{""location"": ""CY"", ""is_mobile"": false}" 8705,4,195,2017-05-17 02:24:11,http://mayert.biz/cristopher_dickinson,0.6471544390,161.169.146.228,"{""location"": ""TL"", ""is_mobile"": true}" 8706,4,195,2017-03-09 09:53:19,http://cole.co/wade,0.0215397393,254.89.248.128,"{""location"": ""MC"", ""is_mobile"": false}" 8707,4,195,2017-03-27 18:11:47,http://hahnkilback.com/everardo,0.2530588511,138.89.229.175,"{""location"": ""SJ"", ""is_mobile"": false}" 8708,4,195,2017-02-21 02:21:40,http://wiza.org/logan.hamill,0.5923070116,125.32.249.100,"{""location"": ""GY"", ""is_mobile"": false}" 8709,4,195,2017-06-09 22:28:51,http://daugherty.name/roscoe_cremin,0.0084498931,164.89.129.229,"{""location"": ""PT"", ""is_mobile"": true}" 8710,4,195,2017-05-01 23:07:04,http://greenpredovic.co/jillian.nikolaus,0.4677807676,80.208.188.88,"{""location"": ""UM"", ""is_mobile"": false}" 8711,4,195,2017-06-11 14:08:42,http://wisoky.net/bell_schimmel,0.4752186679,250.247.83.99,"{""location"": ""BB"", ""is_mobile"": false}" 8712,4,195,2016-12-22 12:46:03,http://nader.info/vern,0.8469564257,142.114.234.228,"{""location"": ""MZ"", ""is_mobile"": false}" 8713,4,196,2017-04-10 18:34:40,http://ankunding.co/stone,0.7167377991,3.12.254.189,"{""location"": ""TJ"", ""is_mobile"": true}" 8714,4,196,2016-12-30 17:12:41,http://mcclurethompson.co/lamar.keebler,0.3186774939,101.143.159.85,"{""location"": ""PE"", ""is_mobile"": true}" 8715,4,196,2017-02-09 09:48:54,http://robel.biz/gracie.toy,0.2396363835,102.83.150.184,"{""location"": ""OM"", ""is_mobile"": false}" 8716,4,196,2017-04-18 06:50:17,http://hicklecummerata.io/norwood.mann,0.0151560679,8.65.160.100,"{""location"": ""CO"", ""is_mobile"": true}" 8717,4,196,2017-02-24 16:11:59,http://schowalterbauch.org/nyasia_hintz,0.5030379481,125.124.207.74,"{""location"": ""MX"", ""is_mobile"": false}" 8718,4,196,2017-05-11 06:49:51,http://schroeder.com/claudine,0.5015744150,79.178.212.146,"{""location"": ""AZ"", ""is_mobile"": false}" 8719,4,196,2017-01-12 11:05:18,http://harberlind.info/maxwell.mckenzie,0.9985890050,4.44.212.37,"{""location"": ""KZ"", ""is_mobile"": true}" 8720,4,196,2017-03-30 14:46:22,http://williamson.org/shanna.davis,0.2034361840,117.116.248.134,"{""location"": ""SM"", ""is_mobile"": false}" 8721,4,196,2016-12-26 16:43:02,http://walker.io/adam.goyette,0.9304590185,201.120.97.36,"{""location"": ""LA"", ""is_mobile"": true}" 8722,4,196,2017-01-15 02:25:00,http://brekke.name/nikita_kuvalis,0.8621046820,164.206.112.162,"{""location"": ""BO"", ""is_mobile"": true}" 8723,4,196,2017-05-23 20:13:35,http://moenwisoky.info/vernon.buckridge,0.8134010853,245.92.213.112,"{""location"": ""BZ"", ""is_mobile"": false}" 8724,4,196,2017-02-28 05:29:25,http://blick.net/kailey.walsh,0.0537161785,151.229.248.6,"{""location"": ""AZ"", ""is_mobile"": true}" 8725,4,196,2017-06-10 12:48:14,http://gerlach.info/emery_nicolas,0.2196112943,124.168.227.143,"{""location"": ""HU"", ""is_mobile"": false}" 8726,4,196,2017-04-30 15:44:43,http://cain.biz/lupe.kiehn,0.1688795558,240.168.178.83,"{""location"": ""US"", ""is_mobile"": false}" 8727,4,196,2017-03-28 12:14:43,http://howecruickshank.com/kyler,0.4800721541,89.14.30.56,"{""location"": ""MF"", ""is_mobile"": true}" 8728,4,196,2017-02-04 03:12:50,http://andersonkling.com/elmore.nolan,0.2465674377,26.180.250.3,"{""location"": ""ZW"", ""is_mobile"": true}" 8729,4,196,2017-06-13 21:31:39,http://gaylord.org/sydney_sanford,0.4774269048,31.17.96.120,"{""location"": ""TT"", ""is_mobile"": true}" 8730,4,196,2017-01-08 19:02:05,http://rippin.net/paige_halvorson,0.9619041212,241.186.27.105,"{""location"": ""BB"", ""is_mobile"": true}" 8731,4,196,2017-03-01 19:34:53,http://hyatt.io/claudine,0.2889963249,211.199.198.206,"{""location"": ""IS"", ""is_mobile"": true}" 8732,4,196,2017-03-21 00:07:31,http://dickinsonleuschke.net/wilhelm,0.2895888913,5.132.186.63,"{""location"": ""CW"", ""is_mobile"": true}" 8733,4,196,2017-03-22 00:39:38,http://barton.net/geraldine.lynch,0.5521089322,228.184.233.221,"{""location"": ""TG"", ""is_mobile"": false}" 8734,4,196,2017-02-28 06:32:45,http://kohler.co/shanna,0.8735604275,55.39.184.171,"{""location"": ""AQ"", ""is_mobile"": false}" 8735,4,196,2017-01-28 07:27:15,http://stammpurdy.net/esther_kuhlman,0.6144055588,109.115.200.191,"{""location"": ""PT"", ""is_mobile"": true}" 8736,4,196,2017-05-21 13:54:35,http://terrymraz.com/rogelio,0.1453577152,35.191.107.147,"{""location"": ""AD"", ""is_mobile"": true}" 8737,4,196,2017-01-10 02:53:55,http://schusterschultz.biz/susie,0.2226834152,32.209.103.6,"{""location"": ""IT"", ""is_mobile"": true}" 8738,4,196,2017-05-17 07:04:52,http://abshire.biz/nikolas,0.7368007009,60.68.72.96,"{""location"": ""BE"", ""is_mobile"": true}" 8739,4,196,2016-12-16 03:50:21,http://reilly.biz/dulce,0.5882471274,191.150.166.252,"{""location"": ""SY"", ""is_mobile"": true}" 8740,4,196,2016-12-21 21:50:16,http://schinner.co/orlo,0.5687910592,105.177.106.188,"{""location"": ""VE"", ""is_mobile"": true}" 8741,4,196,2017-04-22 05:06:02,http://will.net/emmett,0.3029667457,162.235.68.99,"{""location"": ""IT"", ""is_mobile"": true}" 8742,4,196,2017-03-26 22:05:53,http://nikolaus.name/wanda_veum,0.2171931378,169.11.136.242,"{""location"": ""FK"", ""is_mobile"": false}" 8743,4,196,2017-03-10 03:13:30,http://shanahanmcglynn.net/manuel_upton,0.8908734344,239.156.154.196,"{""location"": ""UZ"", ""is_mobile"": true}" 8744,4,196,2017-06-02 01:19:20,http://friesenwehner.net/frieda,0.9800004458,106.94.189.223,"{""location"": ""PN"", ""is_mobile"": true}" 8745,4,196,2017-04-05 11:33:00,http://franecki.info/salvatore,0.7239601332,234.173.181.106,"{""location"": ""KI"", ""is_mobile"": false}" 8746,4,196,2017-06-04 19:13:00,http://muller.co/dylan,0.3236419281,241.170.80.216,"{""location"": ""CW"", ""is_mobile"": false}" 8747,4,196,2017-03-30 01:10:10,http://marquardt.com/kade,0.4982056566,37.20.206.34,"{""location"": ""BQ"", ""is_mobile"": false}" 8748,4,196,2017-03-13 13:28:04,http://berge.biz/kyra.blick,0.0645575607,198.205.228.229,"{""location"": ""CU"", ""is_mobile"": false}" 8749,4,196,2017-04-29 02:49:22,http://dooleyconn.com/juwan,0.1440836982,56.65.32.218,"{""location"": ""ET"", ""is_mobile"": false}" 8750,4,196,2016-12-30 02:10:06,http://deckow.info/franz,0.5056517775,100.210.213.240,"{""location"": ""AL"", ""is_mobile"": false}" 8751,4,196,2017-04-23 00:27:26,http://bauchconsidine.info/ruth.trantow,0.9829378082,238.45.96.191,"{""location"": ""RU"", ""is_mobile"": false}" 8752,4,196,2017-04-08 21:01:30,http://zieme.com/waldo,0.0115969148,176.9.213.94,"{""location"": ""KH"", ""is_mobile"": true}" 8753,4,196,2017-04-04 17:24:52,http://roberts.com/kiel,0.2218761841,50.91.64.55,"{""location"": ""LK"", ""is_mobile"": true}" 8754,4,196,2017-05-14 22:13:36,http://tillmanfisher.biz/sydni.cartwright,0.3414799995,110.139.219.90,"{""location"": ""AQ"", ""is_mobile"": false}" 8755,4,196,2017-06-05 07:49:00,http://leannonvonrueden.co/london_mitchell,0.5397942421,50.209.84.58,"{""location"": ""KI"", ""is_mobile"": false}" 8756,4,196,2017-01-03 03:03:19,http://uptonsmith.io/stevie.hudson,0.3326452450,139.174.152.110,"{""location"": ""EE"", ""is_mobile"": false}" 8757,4,196,2017-05-20 04:12:33,http://terry.net/clara_mann,0.3054421766,137.46.162.198,"{""location"": ""SH"", ""is_mobile"": false}" 8758,4,196,2016-12-24 04:58:09,http://hageneswaters.net/lorna,0.3017815319,213.234.252.61,"{""location"": ""TJ"", ""is_mobile"": false}" 8759,4,196,2017-03-23 18:12:53,http://blandasatterfield.com/cali,0.6014862279,125.119.128.8,"{""location"": ""RU"", ""is_mobile"": false}" 8760,4,196,2016-12-16 02:47:22,http://morar.io/marcia.volkman,0.9932661765,60.52.43.234,"{""location"": ""BE"", ""is_mobile"": true}" 8761,4,196,2017-02-03 01:02:22,http://cartwright.biz/jerald,0.2602474415,189.192.148.254,"{""location"": ""NI"", ""is_mobile"": false}" 8762,4,196,2017-03-24 23:00:04,http://padbergnader.org/wade,0.8908535753,198.110.181.41,"{""location"": ""VG"", ""is_mobile"": true}" 8763,4,196,2017-01-16 17:07:09,http://welch.net/skye_spencer,0.9444863735,79.248.127.190,"{""location"": ""IQ"", ""is_mobile"": false}" 8764,4,196,2017-03-10 04:04:37,http://weimann.co/thora.ledner,0.2645013143,14.237.197.30,"{""location"": ""HU"", ""is_mobile"": true}" 8765,4,196,2017-05-05 15:13:47,http://terry.co/gustave.hettinger,0.1946649441,145.31.59.226,"{""location"": ""CR"", ""is_mobile"": false}" 8766,4,197,2017-02-24 20:56:31,http://schiller.co/tiffany,0.8525530358,196.37.102.196,"{""location"": ""HU"", ""is_mobile"": true}" 8767,4,197,2017-03-25 16:16:46,http://durgan.com/saul.kunze,0.8548617241,191.184.199.225,"{""location"": ""KE"", ""is_mobile"": false}" 8768,4,197,2017-03-29 03:28:51,http://turnerhagenes.info/adelia_west,0.9370212503,182.102.244.75,"{""location"": ""IL"", ""is_mobile"": true}" 8769,4,197,2017-03-06 15:21:31,http://harveydaniel.name/casimir.renner,0.8567609227,22.120.49.68,"{""location"": ""KP"", ""is_mobile"": true}" 8770,4,197,2016-12-23 19:49:25,http://reichertflatley.org/joel.hermann,0.9432064212,238.211.137.29,"{""location"": ""AL"", ""is_mobile"": true}" 8771,4,197,2017-04-07 20:18:12,http://harveybashirian.biz/bernita.gutkowski,0.1023805070,228.245.225.43,"{""location"": ""QA"", ""is_mobile"": false}" 8772,4,197,2017-06-08 20:11:59,http://johnstonrippin.name/rogelio.grady,0.1207089154,230.254.236.236,"{""location"": ""TO"", ""is_mobile"": true}" 8773,4,197,2017-04-15 23:25:24,http://white.com/anika.anderson,0.2536446078,207.226.132.49,"{""location"": ""FI"", ""is_mobile"": false}" 8774,4,197,2017-02-19 05:59:26,http://ullrich.net/pete_mante,0.8754261180,234.213.106.79,"{""location"": ""MR"", ""is_mobile"": false}" 8775,4,197,2017-02-02 04:41:54,http://deckow.net/anika_borer,0.6959540940,40.61.18.182,"{""location"": ""GU"", ""is_mobile"": false}" 8776,4,197,2017-04-04 17:44:21,http://gusikowski.biz/nakia_lang,0.6876525379,79.35.93.167,"{""location"": ""FI"", ""is_mobile"": false}" 8777,4,197,2017-03-11 14:09:45,http://koelpinprohaska.co/danny_cremin,0.1630662334,195.113.91.191,"{""location"": ""BS"", ""is_mobile"": false}" 8778,4,197,2016-12-15 18:53:18,http://koch.com/london,0.8343991080,206.221.57.82,"{""location"": ""ES"", ""is_mobile"": true}" 8779,4,197,2017-02-27 19:21:16,http://howe.biz/emerald,0.9553694777,241.61.109.196,"{""location"": ""BV"", ""is_mobile"": true}" 8780,4,197,2017-02-11 23:46:18,http://ratke.name/keyon,0.9723278236,33.149.53.106,"{""location"": ""MX"", ""is_mobile"": true}" 8781,4,197,2017-02-25 23:02:28,http://beier.co/hector,0.0198749615,179.6.180.6,"{""location"": ""PS"", ""is_mobile"": false}" 8782,4,197,2017-03-10 01:19:09,http://leschwiza.org/donavon,0.6151547357,176.200.133.227,"{""location"": ""IM"", ""is_mobile"": true}" 8783,4,197,2017-04-25 00:49:43,http://harveyhaag.com/frances,0.0186454341,73.177.3.38,"{""location"": ""IR"", ""is_mobile"": false}" 8784,4,197,2017-05-27 02:37:09,http://sanford.co/hosea,0.8863437358,103.222.165.178,"{""location"": ""VI"", ""is_mobile"": false}" 8785,4,197,2016-12-30 23:07:13,http://greenfelder.biz/anabelle,0.7799739026,21.223.45.122,"{""location"": ""BD"", ""is_mobile"": false}" 8786,4,197,2017-02-20 20:06:34,http://koelpin.io/zora,0.7092584126,163.101.102.150,"{""location"": ""FI"", ""is_mobile"": false}" 8787,4,198,2017-04-24 21:55:05,http://champlin.name/lia.bashirian,0.1610006191,241.47.31.114,"{""location"": ""NI"", ""is_mobile"": true}" 8788,4,198,2017-05-01 20:31:21,http://willms.com/cody,0.8698253476,108.179.170.239,"{""location"": ""UZ"", ""is_mobile"": true}" 8789,4,198,2017-04-26 11:36:09,http://little.biz/dortha,0.8525511280,53.178.15.131,"{""location"": ""SB"", ""is_mobile"": true}" 8790,4,198,2017-03-19 19:52:09,http://strosin.info/claude,0.2872581408,199.151.107.74,"{""location"": ""BL"", ""is_mobile"": true}" 8791,4,198,2017-01-31 20:24:01,http://labadie.biz/maryam,0.9370233241,185.200.145.13,"{""location"": ""PH"", ""is_mobile"": true}" 8792,4,198,2017-05-18 00:00:46,http://kutch.com/vivienne.monahan,0.4895155120,132.70.114.73,"{""location"": ""MU"", ""is_mobile"": false}" 8793,4,198,2017-04-23 16:09:53,http://schmelerbruen.name/dianna,0.7047241343,107.113.74.15,"{""location"": ""MY"", ""is_mobile"": false}" 8794,4,198,2017-02-07 01:21:52,http://corwinfriesen.com/ryleigh,0.0568916640,222.156.240.16,"{""location"": ""MD"", ""is_mobile"": true}" 8795,4,198,2017-01-24 09:34:33,http://frami.biz/joseph_keeling,0.0306896389,132.115.163.188,"{""location"": ""TV"", ""is_mobile"": true}" 8796,4,198,2017-01-23 15:26:43,http://wintheiser.info/rosalinda.harber,0.9774626407,186.217.229.44,"{""location"": ""VU"", ""is_mobile"": true}" 8797,4,198,2017-05-03 03:29:16,http://weinat.info/joey.collins,0.1712909351,48.80.18.38,"{""location"": ""AL"", ""is_mobile"": false}" 8798,4,198,2017-04-04 08:02:42,http://stammgleason.org/donnell.abernathy,0.5596023177,202.132.44.211,"{""location"": ""KH"", ""is_mobile"": true}" 8799,4,198,2017-03-27 07:16:16,http://grimes.name/eloise_frami,0.8900954596,102.32.3.213,"{""location"": ""ST"", ""is_mobile"": true}" 8800,4,198,2017-04-22 03:08:08,http://weber.biz/max,0.1198896721,143.246.188.216,"{""location"": ""MR"", ""is_mobile"": false}" 8801,4,198,2017-03-23 23:26:27,http://naderdeckow.com/emmanuelle.mcglynn,0.2920895247,119.90.46.113,"{""location"": ""BM"", ""is_mobile"": true}" 8802,4,198,2017-02-04 13:54:46,http://mohr.org/shanel,0.4336557161,71.81.48.150,"{""location"": ""EC"", ""is_mobile"": true}" 8803,4,198,2017-05-16 12:46:08,http://gerlach.io/elena,0.8421889151,191.109.191.241,"{""location"": ""HN"", ""is_mobile"": true}" 8804,4,198,2017-02-03 12:21:21,http://leannonhyatt.net/natalie_kshlerin,0.0409433998,132.194.206.142,"{""location"": ""BG"", ""is_mobile"": false}" 8805,4,198,2017-02-13 02:01:29,http://kuhic.name/charlene_mcclure,0.4034373868,58.7.240.130,"{""location"": ""BO"", ""is_mobile"": true}" 8806,4,198,2017-04-08 09:10:44,http://hintz.net/humberto,0.5279324083,96.22.177.61,"{""location"": ""AO"", ""is_mobile"": false}" 8807,4,198,2017-04-22 04:48:28,http://emmerichfarrell.biz/furman_gislason,0.8604471129,9.246.5.210,"{""location"": ""CA"", ""is_mobile"": true}" 8808,4,198,2017-05-31 20:12:08,http://blanda.name/heather,0.6547761520,78.65.196.192,"{""location"": ""MY"", ""is_mobile"": false}" 8809,4,198,2016-12-17 15:30:38,http://kuvalis.org/herminio,0.7237387998,199.107.71.86,"{""location"": ""MT"", ""is_mobile"": true}" 8810,4,198,2017-01-07 21:09:28,http://pouros.io/vinnie_moen,0.5282389066,91.200.210.215,"{""location"": ""CR"", ""is_mobile"": false}" 8811,4,198,2017-02-08 14:20:49,http://pouros.com/troy,0.9972482239,210.61.183.157,"{""location"": ""UM"", ""is_mobile"": true}" 8812,4,198,2017-02-13 00:06:04,http://bernhardlebsack.org/emilio.jerde,0.4443593866,15.245.216.224,"{""location"": ""LA"", ""is_mobile"": true}" 8813,4,198,2017-02-21 22:57:08,http://torp.com/marilou_lehner,0.1664114066,93.117.48.177,"{""location"": ""AM"", ""is_mobile"": true}" 8814,4,198,2017-05-26 22:46:51,http://mannfarrell.name/rebecca.dach,0.6492696285,177.245.202.81,"{""location"": ""GL"", ""is_mobile"": true}" 8815,4,198,2017-04-07 23:36:18,http://mosciski.biz/brown.fadel,0.5192736514,95.44.107.25,"{""location"": ""DO"", ""is_mobile"": true}" 8816,4,198,2017-01-03 18:15:42,http://cruickshankpredovic.biz/lina,0.0932487451,162.104.85.245,"{""location"": ""MZ"", ""is_mobile"": false}" 8817,4,198,2017-06-05 23:56:45,http://dooley.info/cletus_wilderman,0.2304722827,64.112.110.232,"{""location"": ""LS"", ""is_mobile"": true}" 8818,4,198,2017-06-05 11:02:51,http://kozeywolf.biz/esteban_bashirian,0.8628739886,65.79.41.167,"{""location"": ""JM"", ""is_mobile"": true}" 8819,4,198,2017-03-12 00:43:11,http://effertz.com/filiberto,0.5978230981,75.85.100.180,"{""location"": ""LC"", ""is_mobile"": true}" 8820,4,198,2017-05-28 13:46:58,http://fadel.com/jamar.gibson,0.4717492853,213.5.217.192,"{""location"": ""FM"", ""is_mobile"": true}" 8821,4,198,2017-04-30 17:52:45,http://feeney.biz/pink,0.0020606172,218.84.219.79,"{""location"": ""MC"", ""is_mobile"": true}" 8822,4,198,2017-03-26 08:26:31,http://tremblay.io/jany,0.9786546798,142.134.200.223,"{""location"": ""BS"", ""is_mobile"": true}" 8823,4,198,2017-03-22 15:14:20,http://windlerframi.info/aliza,0.4910358123,218.96.245.187,"{""location"": ""NP"", ""is_mobile"": false}" 8824,4,198,2017-01-16 17:03:20,http://keeling.name/akeem.walker,0.6586889301,146.189.109.114,"{""location"": ""HR"", ""is_mobile"": false}" 8825,4,198,2017-05-04 11:00:50,http://hammes.com/sylvester.cormier,0.6127640742,81.164.253.15,"{""location"": ""KE"", ""is_mobile"": false}" 8826,4,198,2017-04-05 04:29:26,http://schuppe.net/jamal_walker,0.8486946242,16.241.119.214,"{""location"": ""SH"", ""is_mobile"": true}" 8827,4,198,2017-04-16 16:25:23,http://kuphalziemann.net/orion,0.7549960793,243.217.134.67,"{""location"": ""FM"", ""is_mobile"": true}" 8828,4,198,2017-04-19 17:39:08,http://boyle.org/neoma_renner,0.0623859320,191.57.121.162,"{""location"": ""MF"", ""is_mobile"": true}" 8829,4,198,2017-06-12 17:16:51,http://jones.com/mitchel,0.0947111947,37.213.241.95,"{""location"": ""SR"", ""is_mobile"": true}" 8830,4,198,2017-03-03 13:01:16,http://treutel.io/nettie_murphy,0.5876643815,197.219.33.13,"{""location"": ""MH"", ""is_mobile"": true}" 8831,4,198,2017-04-06 12:56:06,http://bodecole.name/idella.wiza,0.1300575205,137.136.178.232,"{""location"": ""HU"", ""is_mobile"": false}" 8832,4,198,2017-06-04 01:43:43,http://zulauf.io/kris.medhurst,0.2224550573,89.117.76.206,"{""location"": ""KH"", ""is_mobile"": true}" 8833,4,198,2017-06-11 10:07:42,http://ziemannleuschke.biz/savion,0.3792597564,108.73.190.232,"{""location"": ""US"", ""is_mobile"": true}" 8834,4,198,2017-04-08 10:50:08,http://pagac.com/jack,0.1507565899,77.32.202.169,"{""location"": ""MM"", ""is_mobile"": true}" 5761,3,126,2017-02-11 01:37:39,http://spinkaritchie.info/jaquan.schuppe,,19.241.180.52,"{""location"": ""PR"", ""is_mobile"": true}" 5762,3,126,2017-05-19 07:49:02,http://roob.name/jarvis,,137.150.246.101,"{""location"": ""CK"", ""is_mobile"": true}" 5763,3,126,2017-04-02 05:54:42,http://graham.io/uriel,,68.18.167.224,"{""location"": ""FO"", ""is_mobile"": true}" 5764,3,126,2017-01-30 07:01:16,http://willspencer.co/dane,,250.215.10.145,"{""location"": ""DK"", ""is_mobile"": true}" 5765,3,126,2017-02-07 08:05:07,http://little.co/ethel_kirlin,,52.153.156.78,"{""location"": ""SE"", ""is_mobile"": true}" 5766,3,126,2017-04-18 02:05:43,http://leschbednar.com/elva_schaefer,,130.73.251.200,"{""location"": ""HU"", ""is_mobile"": true}" 5767,3,126,2017-04-02 15:26:15,http://kihnsimonis.io/ted,,230.245.96.46,"{""location"": ""GW"", ""is_mobile"": false}" 5768,3,126,2017-03-29 22:16:14,http://bayer.com/myah,,151.63.44.212,"{""location"": ""KI"", ""is_mobile"": false}" 5769,3,126,2016-12-16 06:52:23,http://kovacek.info/waldo,,111.219.40.19,"{""location"": ""CD"", ""is_mobile"": false}" 5770,3,126,2017-04-20 10:47:47,http://franecki.io/priscilla_zboncak,,250.206.28.76,"{""location"": ""SL"", ""is_mobile"": true}" 5771,3,126,2017-05-24 01:36:28,http://schultzerdman.biz/sydni,,9.46.197.157,"{""location"": ""SO"", ""is_mobile"": false}" 5772,3,126,2017-06-04 08:16:00,http://mcglynn.io/juana_kutch,,235.26.46.82,"{""location"": ""GP"", ""is_mobile"": true}" 5773,3,126,2017-03-31 11:45:03,http://mcdermottmcglynn.org/edward,,217.67.28.93,"{""location"": ""GS"", ""is_mobile"": false}" 5774,3,126,2017-05-26 01:02:43,http://effertz.co/charity,,203.50.150.86,"{""location"": ""EG"", ""is_mobile"": false}" 5775,3,126,2017-03-22 05:32:49,http://breitenberg.name/neoma.graham,,101.45.6.114,"{""location"": ""GU"", ""is_mobile"": false}" 5776,3,126,2017-01-16 20:01:42,http://macgyver.com/sheila,,243.115.6.81,"{""location"": ""VC"", ""is_mobile"": true}" 5777,3,126,2017-03-10 08:10:21,http://jacobsheidenreich.info/mateo.olson,,216.191.47.120,"{""location"": ""JE"", ""is_mobile"": true}" 5778,3,126,2017-02-16 22:00:30,http://wilkinson.com/tania_nienow,,233.233.31.156,"{""location"": ""ET"", ""is_mobile"": false}" 5779,3,126,2017-05-30 18:17:50,http://bogisich.info/mustafa,,188.232.224.103,"{""location"": ""SH"", ""is_mobile"": false}" 5780,3,126,2017-05-07 01:31:09,http://nitzsche.io/ara_kemmer,,18.97.64.164,"{""location"": ""GI"", ""is_mobile"": false}" 5781,3,126,2016-12-28 05:29:02,http://hintzhodkiewicz.org/rhiannon,,195.60.79.164,"{""location"": ""GB"", ""is_mobile"": false}" 5782,3,126,2017-03-08 12:58:42,http://kreiger.com/leann_balistreri,,20.140.219.112,"{""location"": ""RU"", ""is_mobile"": false}" 5783,3,126,2017-04-30 01:07:30,http://turcotte.io/enoch,,95.54.125.220,"{""location"": ""CD"", ""is_mobile"": true}" 5784,3,126,2017-01-13 04:15:00,http://schambergerhettinger.biz/izaiah,,200.32.60.207,"{""location"": ""CM"", ""is_mobile"": true}" 5785,3,126,2017-05-24 03:33:32,http://sanfordrohan.net/arnulfo.vandervort,,247.254.202.254,"{""location"": ""MO"", ""is_mobile"": false}" 5786,3,126,2017-02-22 15:27:05,http://mitchell.io/libby,,64.61.36.182,"{""location"": ""AT"", ""is_mobile"": true}" 5787,3,126,2017-05-18 08:59:04,http://ullrichferry.co/sasha,,131.125.219.48,"{""location"": ""MM"", ""is_mobile"": false}" 5788,3,126,2017-01-05 05:36:52,http://schusterkautzer.net/kathryne.jacobi,,175.117.79.20,"{""location"": ""GQ"", ""is_mobile"": true}" 5789,3,126,2017-05-31 00:53:08,http://nitzscherunolfsdottir.net/margarete.metz,,52.2.218.176,"{""location"": ""SS"", ""is_mobile"": true}" 5790,3,126,2017-04-18 10:36:00,http://carrolljerde.org/roie_schuppe,,166.134.156.162,"{""location"": ""LK"", ""is_mobile"": false}" 5791,3,126,2017-04-08 15:05:29,http://nikolauskerluke.info/malinda,,98.192.86.187,"{""location"": ""MZ"", ""is_mobile"": true}" 5792,3,126,2017-05-29 00:14:09,http://leannondaugherty.net/tanya,,177.179.132.132,"{""location"": ""DK"", ""is_mobile"": false}" 5793,3,126,2017-03-18 23:16:20,http://swiftschultz.net/kurt,,185.75.187.200,"{""location"": ""CK"", ""is_mobile"": true}" 5794,3,126,2017-04-13 08:25:15,http://rueckerdurgan.io/aglae,,69.40.47.79,"{""location"": ""IO"", ""is_mobile"": false}" 5795,3,126,2017-02-01 23:03:35,http://stoltenbergblanda.net/luisa.bailey,,25.14.201.92,"{""location"": ""MF"", ""is_mobile"": true}" 5796,3,126,2017-02-22 18:16:41,http://millercronin.info/tabitha,,34.161.44.60,"{""location"": ""SH"", ""is_mobile"": true}" 5797,3,126,2017-04-11 01:06:19,http://hilpertgoyette.net/abdul,,9.141.48.95,"{""location"": ""AU"", ""is_mobile"": true}" 5798,3,126,2017-03-14 07:47:56,http://brekkedietrich.com/ramona,,195.195.73.210,"{""location"": ""AI"", ""is_mobile"": true}" 5799,3,126,2017-03-08 09:02:25,http://satterfield.name/mable_bechtelar,,65.2.206.237,"{""location"": ""SX"", ""is_mobile"": false}" 5800,3,126,2017-05-06 15:28:37,http://franecki.com/madyson.kub,,182.2.159.252,"{""location"": ""PH"", ""is_mobile"": false}" 5801,3,126,2017-05-17 03:00:56,http://roob.co/ivy,,101.203.244.98,"{""location"": ""PS"", ""is_mobile"": true}" 5802,3,126,2017-05-12 04:02:40,http://wunsch.name/cristina,,140.125.81.247,"{""location"": ""WF"", ""is_mobile"": false}" 5803,3,126,2017-02-14 09:39:51,http://gerlach.io/pete,,141.93.138.232,"{""location"": ""IM"", ""is_mobile"": true}" 5804,3,126,2017-05-12 07:06:24,http://bergejohns.biz/brian,,194.238.112.173,"{""location"": ""CF"", ""is_mobile"": false}" 5805,3,126,2017-02-02 02:45:58,http://buckridgemertz.info/roscoe_waelchi,,7.77.133.61,"{""location"": ""YT"", ""is_mobile"": false}" 5806,3,126,2017-03-01 02:16:29,http://walsh.com/isidro,,191.102.236.76,"{""location"": ""SC"", ""is_mobile"": false}" 5807,3,126,2017-06-08 01:47:23,http://stracke.name/allison,,94.230.23.13,"{""location"": ""AM"", ""is_mobile"": true}" 5808,3,126,2016-12-18 05:10:10,http://gerhold.biz/concepcion,,253.181.17.28,"{""location"": ""OM"", ""is_mobile"": false}" 5809,3,126,2016-12-22 23:29:19,http://strosinsimonis.biz/maximillian_lemke,,33.237.59.193,"{""location"": ""NI"", ""is_mobile"": false}" 5810,3,126,2017-02-20 09:43:06,http://kertzmann.net/alyce,,214.81.8.20,"{""location"": ""CF"", ""is_mobile"": false}" 5811,3,126,2017-03-17 09:28:45,http://willsteuber.name/queen,,10.25.5.44,"{""location"": ""IT"", ""is_mobile"": true}" 5812,3,126,2017-01-24 19:24:54,http://bodeskiles.info/domenic_krajcik,,115.91.253.61,"{""location"": ""LU"", ""is_mobile"": true}" 5813,3,126,2017-01-20 14:50:27,http://skiles.org/jee,,142.43.172.147,"{""location"": ""UZ"", ""is_mobile"": true}" 5814,3,126,2017-03-27 22:17:50,http://conn.net/raleigh,,4.211.141.25,"{""location"": ""AM"", ""is_mobile"": false}" 8835,4,198,2016-12-20 22:41:15,http://ryan.co/cara,0.5597857971,123.73.11.206,"{""location"": ""MO"", ""is_mobile"": true}" 8836,4,198,2016-12-22 11:02:10,http://hodkiewicz.name/agnes,0.4273521017,241.13.64.198,"{""location"": ""KP"", ""is_mobile"": true}" 8837,4,198,2017-01-25 02:17:22,http://hand.net/ursula_altenwerth,0.7412839932,217.56.198.79,"{""location"": ""SL"", ""is_mobile"": true}" 8838,4,198,2017-03-03 18:18:28,http://legrosrunte.co/agustin.purdy,0.2312579653,107.184.213.54,"{""location"": ""GQ"", ""is_mobile"": true}" 8839,4,198,2017-05-19 04:56:08,http://grant.net/kara.buckridge,0.1595464850,78.47.100.123,"{""location"": ""NA"", ""is_mobile"": true}" 8840,4,198,2017-06-13 23:11:15,http://dach.co/bette.harvey,0.8751103728,199.181.57.136,"{""location"": ""AT"", ""is_mobile"": true}" 8841,4,198,2017-04-30 08:57:49,http://cruickshank.com/ima.little,0.1409160647,30.80.207.158,"{""location"": ""GB"", ""is_mobile"": true}" 8842,4,198,2017-05-24 22:14:15,http://jacobs.com/yasmeen,0.2400374820,51.164.133.145,"{""location"": ""IM"", ""is_mobile"": false}" 8843,4,198,2017-04-13 19:09:19,http://schumm.biz/ima,0.7108763836,33.248.213.4,"{""location"": ""HM"", ""is_mobile"": true}" 8844,4,198,2017-03-24 19:15:50,http://rogahn.co/jaydon.schmitt,0.5650419068,25.16.34.79,"{""location"": ""TO"", ""is_mobile"": true}" 8845,4,198,2017-02-23 08:30:11,http://boyle.com/makenna,0.3737284898,59.190.213.74,"{""location"": ""NU"", ""is_mobile"": false}" 8846,4,198,2017-03-09 02:47:01,http://mraz.co/eleanora,0.9467390478,46.147.247.254,"{""location"": ""SD"", ""is_mobile"": false}" 8847,4,199,2017-01-17 13:15:21,http://reillygusikowski.com/norene_gulgowski,0.9757982228,116.120.227.21,"{""location"": ""PS"", ""is_mobile"": true}" 8848,4,199,2016-12-31 02:00:48,http://becker.com/jo_lubowitz,0.5733575374,202.103.11.2,"{""location"": ""SV"", ""is_mobile"": false}" 8849,4,199,2017-04-08 05:00:24,http://reilly.org/santiago_murray,0.8998182259,113.108.127.189,"{""location"": ""MU"", ""is_mobile"": true}" 8850,4,199,2017-01-28 10:38:24,http://block.biz/carter,0.6884710172,245.172.199.155,"{""location"": ""BQ"", ""is_mobile"": false}" 8851,4,199,2017-04-04 17:40:43,http://leuschke.net/juston,0.0950574058,68.38.78.199,"{""location"": ""GE"", ""is_mobile"": true}" 8852,4,199,2017-06-06 11:37:06,http://lang.info/willard,0.6666275966,196.152.57.120,"{""location"": ""IM"", ""is_mobile"": true}" 8853,4,199,2017-05-13 20:32:19,http://deckow.org/ashton.donnelly,0.3069954647,165.103.51.65,"{""location"": ""SA"", ""is_mobile"": false}" 8854,4,199,2017-01-14 19:52:59,http://durgan.name/magnolia_gleason,0.5743377291,39.120.76.185,"{""location"": ""AM"", ""is_mobile"": true}" 8855,4,199,2017-06-13 06:58:46,http://runolfon.co/marcella,0.8904960201,233.10.142.125,"{""location"": ""FR"", ""is_mobile"": false}" 8856,4,199,2017-03-22 05:50:58,http://nadersanford.co/paige,0.2439292823,83.184.222.153,"{""location"": ""SJ"", ""is_mobile"": false}" 8857,4,199,2017-01-28 23:04:51,http://oconner.io/gregg.heel,0.3512053496,147.77.56.158,"{""location"": ""KG"", ""is_mobile"": true}" 8858,4,199,2017-04-13 09:31:03,http://ernserryan.info/orrin_heidenreich,0.4265410236,232.15.22.205,"{""location"": ""DM"", ""is_mobile"": false}" 8859,4,199,2017-02-21 18:42:37,http://priceparker.name/miguel,0.7252021458,247.79.225.110,"{""location"": ""LT"", ""is_mobile"": true}" 8860,4,199,2017-03-25 16:58:58,http://rogahn.net/ada,0.1964984097,127.190.222.154,"{""location"": ""GN"", ""is_mobile"": true}" 8861,4,199,2017-03-13 13:52:29,http://wisozkrunolfon.org/gladys,0.2452635266,103.15.69.175,"{""location"": ""JP"", ""is_mobile"": true}" 8862,4,199,2017-05-27 23:16:35,http://sanfordweimann.net/neha.gerlach,0.3878320428,154.191.213.74,"{""location"": ""RU"", ""is_mobile"": false}" 8863,4,199,2017-05-07 14:35:55,http://marksherman.net/alek,0.3495561763,105.12.97.69,"{""location"": ""MD"", ""is_mobile"": false}" 8864,4,199,2017-04-01 07:45:23,http://carroll.io/dion.green,0.7871974859,46.33.72.25,"{""location"": ""GA"", ""is_mobile"": false}" 8865,4,199,2017-01-17 05:02:06,http://yundt.name/anais_reichel,0.5399770214,157.7.119.219,"{""location"": ""MN"", ""is_mobile"": true}" 8866,4,199,2017-01-06 23:48:03,http://konopelski.name/marion,0.7501745993,41.103.127.87,"{""location"": ""MU"", ""is_mobile"": true}" 8867,4,199,2017-05-24 06:43:00,http://okon.com/toni,0.7998795361,104.70.37.183,"{""location"": ""PR"", ""is_mobile"": true}" 8868,4,199,2017-04-30 13:48:44,http://hilpert.io/alycia,0.8425533516,144.110.218.89,"{""location"": ""FR"", ""is_mobile"": true}" 8869,4,199,2017-02-10 15:11:46,http://krisreichert.io/catalina.barrows,0.5081939500,117.46.49.110,"{""location"": ""PE"", ""is_mobile"": false}" 8870,4,199,2016-12-27 03:16:03,http://kshlerinlockman.info/melyna_upton,0.0952862521,220.209.133.38,"{""location"": ""GB"", ""is_mobile"": true}" 8871,4,199,2017-06-13 04:10:23,http://hudsonthiel.info/kaylee_thiel,0.7483013854,144.65.57.204,"{""location"": ""RW"", ""is_mobile"": true}" 8872,4,199,2017-05-06 15:41:22,http://okonmarks.org/hans.ondricka,0.3289508272,204.183.95.137,"{""location"": ""SC"", ""is_mobile"": true}" 8873,4,199,2017-03-03 04:02:50,http://ruecker.co/alford.schamberger,0.4900606503,57.216.89.30,"{""location"": ""SK"", ""is_mobile"": true}" 8874,4,199,2017-04-16 11:35:37,http://mcdermottwuckert.net/ocie.rowe,0.2898629585,164.52.200.150,"{""location"": ""JP"", ""is_mobile"": true}" 8875,4,199,2017-03-12 14:42:26,http://daughertyhirthe.com/narciso,0.8320872702,86.135.45.14,"{""location"": ""EC"", ""is_mobile"": false}" 8876,4,199,2017-02-11 08:48:15,http://lang.co/dustin_mcdermott,0.4426876779,8.5.137.78,"{""location"": ""AO"", ""is_mobile"": false}" 8877,4,199,2017-02-19 06:45:49,http://cruickshank.biz/april,0.8704595244,149.225.225.21,"{""location"": ""TF"", ""is_mobile"": true}" 8878,4,199,2017-01-19 06:23:10,http://stokes.co/jolie,0.9187108653,122.55.182.222,"{""location"": ""CW"", ""is_mobile"": true}" 8879,4,199,2017-04-19 08:24:02,http://miller.biz/merlin,0.1996435795,84.231.14.79,"{""location"": ""BY"", ""is_mobile"": true}" 8880,4,199,2017-04-06 22:14:06,http://runtehintz.name/kenton_jerde,0.7733691640,160.182.241.62,"{""location"": ""CM"", ""is_mobile"": true}" 8881,4,199,2017-03-07 08:09:38,http://hilpert.biz/tate,0.4247030224,162.203.127.132,"{""location"": ""VU"", ""is_mobile"": true}" 8882,4,199,2017-05-04 05:07:46,http://hartmann.name/ezra,0.8284693998,120.133.67.103,"{""location"": ""LB"", ""is_mobile"": true}" 8883,4,199,2016-12-15 14:31:15,http://jerderowe.info/jordon_reynolds,0.2388870643,191.70.97.49,"{""location"": ""TJ"", ""is_mobile"": true}" 8884,4,199,2017-01-25 07:44:03,http://hauck.biz/ryann,0.6717933339,73.39.10.121,"{""location"": ""CN"", ""is_mobile"": false}" 8885,4,199,2017-04-15 06:06:05,http://mckenzie.name/orin,0.6378697137,80.97.138.232,"{""location"": ""CN"", ""is_mobile"": true}" 8886,4,199,2017-02-13 01:11:45,http://stanton.com/bette_towne,0.2023286210,94.250.126.20,"{""location"": ""WF"", ""is_mobile"": true}" 5815,3,126,2017-02-08 16:26:02,http://satterfield.co/holly_ohara,,134.152.103.122,"{""location"": ""MF"", ""is_mobile"": true}" 5816,3,126,2017-03-28 17:31:32,http://torphyauer.biz/helmer.heller,,49.86.57.56,"{""location"": ""CF"", ""is_mobile"": true}" 5817,3,126,2017-02-20 21:48:45,http://cummerata.com/nels,,101.189.149.200,"{""location"": ""CM"", ""is_mobile"": false}" 5818,3,126,2017-01-16 10:46:05,http://thompson.io/melia,,65.24.189.52,"{""location"": ""AS"", ""is_mobile"": true}" 5819,3,127,2017-02-26 12:06:13,http://feil.info/floyd_aufderhar,,62.175.124.96,"{""location"": ""ML"", ""is_mobile"": false}" 5820,3,127,2017-04-09 22:42:32,http://bergstromabbott.com/kaia,,61.224.132.213,"{""location"": ""VI"", ""is_mobile"": true}" 5821,3,127,2017-01-27 00:45:24,http://pricespencer.info/eli.moore,,85.89.146.10,"{""location"": ""KE"", ""is_mobile"": false}" 5822,3,127,2017-04-17 16:14:16,http://gaylordherzog.name/ryleigh.kunze,,75.89.4.17,"{""location"": ""VG"", ""is_mobile"": true}" 5823,3,127,2017-04-14 13:30:08,http://brown.com/maximilian.stanton,,202.174.202.86,"{""location"": ""IT"", ""is_mobile"": false}" 5824,3,127,2017-02-15 03:41:36,http://stoltenbergnader.io/hannah,,172.206.48.97,"{""location"": ""BI"", ""is_mobile"": true}" 5825,3,127,2016-12-26 00:47:49,http://kub.net/alek_ernser,,166.206.100.123,"{""location"": ""BI"", ""is_mobile"": false}" 5826,3,127,2016-12-28 18:37:24,http://adams.name/alanis,,116.220.245.254,"{""location"": ""NG"", ""is_mobile"": true}" 5827,3,127,2016-12-28 11:00:48,http://mcglynn.biz/elsa_carroll,,144.33.5.234,"{""location"": ""GN"", ""is_mobile"": true}" 5828,3,127,2017-02-17 01:46:00,http://hettinger.net/leonardo,,122.127.173.190,"{""location"": ""WS"", ""is_mobile"": false}" 5829,3,127,2016-12-15 22:31:07,http://waters.org/margaretta.jacobson,,246.250.199.6,"{""location"": ""PY"", ""is_mobile"": false}" 5830,3,127,2017-02-25 03:37:25,http://frami.com/jaunita.kuphal,,94.122.28.209,"{""location"": ""BT"", ""is_mobile"": true}" 5831,3,127,2017-05-07 23:11:24,http://kihnfranecki.co/omari_altenwerth,,56.16.52.212,"{""location"": ""CM"", ""is_mobile"": true}" 5832,3,127,2017-02-14 23:53:40,http://brownpowlowski.net/dedrick.tillman,,78.88.95.87,"{""location"": ""LA"", ""is_mobile"": true}" 5833,3,127,2017-04-17 12:48:19,http://grady.org/buddy,,27.7.210.206,"{""location"": ""SJ"", ""is_mobile"": true}" 5834,3,127,2017-05-08 13:14:20,http://carrollgutmann.co/antonetta.cartwright,,249.69.73.186,"{""location"": ""ET"", ""is_mobile"": false}" 5835,3,127,2017-06-06 09:46:28,http://dachmante.com/rachel_hauck,,81.186.139.160,"{""location"": ""SR"", ""is_mobile"": false}" 5836,3,127,2017-01-02 05:08:39,http://ohara.co/shayne,,130.251.188.145,"{""location"": ""MQ"", ""is_mobile"": true}" 5837,3,127,2017-06-06 10:09:21,http://rowe.io/ford,,111.209.128.75,"{""location"": ""SC"", ""is_mobile"": true}" 5838,3,127,2016-12-23 11:46:29,http://littel.io/aniya.casper,,174.254.240.70,"{""location"": ""HT"", ""is_mobile"": true}" 5839,3,127,2017-04-21 16:04:32,http://homenicksauer.io/dillan.rippin,,173.239.52.75,"{""location"": ""TK"", ""is_mobile"": true}" 5840,3,127,2017-01-08 11:16:19,http://erdmanolson.name/eino_turner,,16.85.248.223,"{""location"": ""BD"", ""is_mobile"": false}" 5841,3,128,2017-06-08 15:21:48,http://reichelarmstrong.co/carter,,72.207.42.42,"{""location"": ""JO"", ""is_mobile"": true}" 5842,3,128,2017-02-12 03:53:30,http://kunde.org/lucas_pollich,,106.207.6.9,"{""location"": ""MU"", ""is_mobile"": true}" 5843,3,128,2016-12-30 20:21:24,http://hahntorphy.io/leon,,169.6.103.14,"{""location"": ""KZ"", ""is_mobile"": true}" 5844,3,128,2016-12-26 05:15:14,http://hayesflatley.name/rico,,172.221.172.252,"{""location"": ""SY"", ""is_mobile"": true}" 5845,3,128,2017-01-16 02:15:05,http://sengerlang.com/cletus_beatty,,195.239.32.166,"{""location"": ""BM"", ""is_mobile"": false}" 5846,3,128,2017-01-10 05:10:02,http://hand.info/daphne,,108.127.192.85,"{""location"": ""IQ"", ""is_mobile"": true}" 5847,3,128,2017-01-12 08:51:11,http://casper.name/freeman,,229.47.145.227,"{""location"": ""TM"", ""is_mobile"": false}" 5848,3,128,2017-01-09 16:35:25,http://schimmel.io/fritz.schuppe,,157.79.200.222,"{""location"": ""EC"", ""is_mobile"": true}" 5849,3,128,2017-05-23 04:18:38,http://lindgrenaltenwerth.co/george.ullrich,,106.235.120.160,"{""location"": ""UY"", ""is_mobile"": false}" 5850,3,128,2017-01-07 06:19:31,http://jones.org/alejandra_okuneva,,25.160.219.138,"{""location"": ""GA"", ""is_mobile"": true}" 5851,3,128,2017-03-17 05:22:59,http://bartell.org/andreane,,20.149.122.52,"{""location"": ""TN"", ""is_mobile"": false}" 5852,3,128,2016-12-17 04:54:48,http://hackett.net/anthony,,131.238.49.210,"{""location"": ""GW"", ""is_mobile"": false}" 5853,3,128,2017-03-21 18:43:03,http://townehintz.biz/amelie_wiegand,,85.70.210.56,"{""location"": ""MC"", ""is_mobile"": false}" 5854,3,128,2017-04-13 01:48:34,http://swaniawski.info/marilie.adams,,51.3.62.226,"{""location"": ""BR"", ""is_mobile"": false}" 5855,3,128,2016-12-19 07:26:44,http://veum.net/brant,,63.133.153.111,"{""location"": ""AX"", ""is_mobile"": false}" 5856,3,128,2017-03-11 04:22:17,http://brauncormier.io/braulio_king,,159.44.162.224,"{""location"": ""GD"", ""is_mobile"": false}" 5857,3,128,2017-06-01 00:03:56,http://kundehaag.name/nina,,145.169.253.252,"{""location"": ""NL"", ""is_mobile"": false}" 5858,3,128,2017-05-10 15:32:34,http://green.net/claudia,,21.125.20.220,"{""location"": ""SS"", ""is_mobile"": true}" 5859,3,128,2017-01-03 15:30:00,http://cormier.biz/elmore,,19.45.61.239,"{""location"": ""GR"", ""is_mobile"": false}" 5860,3,128,2017-01-05 21:24:18,http://stiedemann.io/gail,,240.228.74.226,"{""location"": ""AG"", ""is_mobile"": true}" 5861,3,128,2016-12-17 13:27:07,http://rowe.info/francisca,,163.148.75.75,"{""location"": ""SC"", ""is_mobile"": false}" 5862,3,128,2017-06-07 08:12:45,http://lehner.io/ed,,102.46.128.223,"{""location"": ""NL"", ""is_mobile"": false}" 5863,3,128,2017-02-16 16:44:13,http://welch.net/alvera.schaden,,203.237.152.66,"{""location"": ""BR"", ""is_mobile"": true}" 5864,3,128,2017-05-01 06:42:37,http://turner.biz/rita,,17.136.117.214,"{""location"": ""AO"", ""is_mobile"": false}" 5865,3,128,2017-03-23 02:43:08,http://reingerwalter.com/constantin,,46.205.141.143,"{""location"": ""KP"", ""is_mobile"": false}" 5866,3,128,2017-01-31 11:30:27,http://rath.net/shania,,139.248.152.86,"{""location"": ""ER"", ""is_mobile"": false}" 5867,3,128,2016-12-31 00:21:17,http://jaskolski.com/rickey,,98.13.92.24,"{""location"": ""UG"", ""is_mobile"": false}" 5868,3,128,2017-06-08 13:28:33,http://bechtelargoodwin.biz/deondre.prosacco,,210.82.194.231,"{""location"": ""TK"", ""is_mobile"": false}" 5869,3,128,2017-02-10 16:53:16,http://collier.biz/tre_wehner,,40.75.59.173,"{""location"": ""MY"", ""is_mobile"": true}" 11753,5,263,2017-06-11 19:06:30,http://bahringer.biz/rae,0.2091751182,60.114.254.10,"{""location"": ""TJ"", ""is_mobile"": true}" 11754,5,263,2017-02-11 07:21:11,http://renner.name/otilia,0.6408695181,30.106.42.26,"{""location"": ""NG"", ""is_mobile"": false}" 11755,5,263,2016-12-24 22:38:09,http://danieltowne.co/angie_wintheiser,0.1623754006,209.80.103.24,"{""location"": ""GP"", ""is_mobile"": false}" 11756,5,263,2016-12-22 15:57:10,http://walshnader.co/billy,0.3004525220,74.2.168.151,"{""location"": ""EH"", ""is_mobile"": true}" 11757,5,263,2017-02-10 07:03:12,http://konopelskinienow.com/peggie_heel,0.2372708509,233.225.189.146,"{""location"": ""RS"", ""is_mobile"": false}" 11758,5,263,2017-01-14 03:38:01,http://shanahankoch.net/wade.connelly,0.8361292759,237.150.180.22,"{""location"": ""BM"", ""is_mobile"": true}" 11759,5,263,2017-04-04 06:32:06,http://wiza.name/lynn,0.4176487362,6.155.218.162,"{""location"": ""MO"", ""is_mobile"": true}" 11760,5,263,2016-12-18 20:08:28,http://reilly.com/samanta,0.1374549331,161.171.184.95,"{""location"": ""GW"", ""is_mobile"": true}" 11761,5,263,2017-03-26 14:11:30,http://runtemoore.co/rodolfo,0.3678270642,191.162.136.31,"{""location"": ""TT"", ""is_mobile"": false}" 11762,5,263,2017-05-16 22:47:36,http://lehner.net/felicia.jacobi,0.8765909621,81.33.244.45,"{""location"": ""YE"", ""is_mobile"": true}" 11763,5,263,2017-04-11 21:43:42,http://jacobson.net/devante,0.6588732520,250.9.18.245,"{""location"": ""GH"", ""is_mobile"": false}" 11764,5,263,2017-02-04 03:04:15,http://breitenbergweber.name/charley_wisoky,0.2655681962,71.93.221.238,"{""location"": ""PK"", ""is_mobile"": true}" 11765,5,263,2016-12-20 02:24:07,http://rolfson.biz/jarrod,0.9641338458,72.126.67.66,"{""location"": ""GQ"", ""is_mobile"": true}" 11766,5,263,2016-12-15 07:26:19,http://harris.biz/juliet.kuhlman,0.6301629881,167.149.114.171,"{""location"": ""KZ"", ""is_mobile"": false}" 11767,5,263,2017-04-15 19:37:19,http://mraz.net/shana_hackett,0.9526687514,144.216.117.206,"{""location"": ""KR"", ""is_mobile"": false}" 11768,5,263,2017-04-19 05:04:49,http://homenick.com/kim,0.1186251824,209.207.26.239,"{""location"": ""IE"", ""is_mobile"": false}" 11769,5,263,2017-02-06 10:28:57,http://ferry.io/celia,0.2869380626,93.133.66.251,"{""location"": ""MW"", ""is_mobile"": false}" 11770,5,263,2017-06-04 14:46:20,http://ward.biz/dovie_heathcote,0.2719403218,206.129.103.236,"{""location"": ""FI"", ""is_mobile"": false}" 11771,5,263,2017-02-21 07:58:11,http://lemke.info/amanda_gibson,0.9896862550,225.204.43.217,"{""location"": ""CD"", ""is_mobile"": true}" 11772,5,263,2017-03-21 15:36:11,http://will.com/samara,0.2176258570,87.227.33.129,"{""location"": ""PT"", ""is_mobile"": true}" 11773,5,263,2017-04-18 21:39:03,http://schowalter.name/zaria_eichmann,0.5786358946,86.36.218.36,"{""location"": ""ER"", ""is_mobile"": true}" 11774,5,264,2017-04-20 07:36:20,http://flatley.org/sienna_erdman,0.3083172227,163.225.201.157,"{""location"": ""BA"", ""is_mobile"": true}" 11775,5,264,2017-02-20 16:25:05,http://rauconnelly.info/margaretta,0.9783480849,161.220.78.78,"{""location"": ""WF"", ""is_mobile"": true}" 11776,5,264,2017-05-28 17:43:59,http://creminstoltenberg.co/brendan_rowe,0.7458118886,198.219.252.123,"{""location"": ""HN"", ""is_mobile"": true}" 11777,5,264,2017-04-20 10:58:04,http://hegmann.biz/rene_hansen,0.7922647543,38.119.71.221,"{""location"": ""FK"", ""is_mobile"": true}" 11778,5,264,2017-04-14 19:11:04,http://kuhn.io/edmund,0.3975225880,100.54.155.9,"{""location"": ""MM"", ""is_mobile"": false}" 11779,5,264,2017-04-06 21:44:49,http://balistreri.info/hershel_gerhold,0.5550865955,67.183.26.175,"{""location"": ""SC"", ""is_mobile"": false}" 11780,5,264,2017-05-09 14:59:19,http://flatley.com/lewis,0.4312329361,165.251.22.166,"{""location"": ""BB"", ""is_mobile"": true}" 11781,5,264,2017-04-19 16:11:28,http://gleason.co/max,0.8814850552,8.212.9.27,"{""location"": ""ET"", ""is_mobile"": true}" 11782,5,264,2017-03-17 04:17:22,http://lynch.io/aiden.corkery,0.0660631373,14.178.243.6,"{""location"": ""AL"", ""is_mobile"": false}" 11783,5,264,2017-05-19 23:08:02,http://hoeger.name/opal,0.5941297618,44.46.221.46,"{""location"": ""SJ"", ""is_mobile"": false}" 11784,5,264,2017-06-07 18:19:47,http://ryan.co/susanna,0.4880950590,27.28.215.36,"{""location"": ""MH"", ""is_mobile"": false}" 11785,5,264,2017-06-02 02:02:24,http://gutkowskistanton.net/brady.king,0.9731355309,228.132.38.128,"{""location"": ""MY"", ""is_mobile"": true}" 11786,5,264,2016-12-24 11:38:35,http://stracke.net/selina,0.8405589635,247.164.239.194,"{""location"": ""GI"", ""is_mobile"": true}" 11787,5,264,2017-06-05 00:50:21,http://bogisichterry.co/ray,0.2937510124,107.186.66.115,"{""location"": ""KZ"", ""is_mobile"": true}" 11788,5,264,2016-12-14 07:03:18,http://sawayn.info/margie_kling,0.8357591555,23.180.121.89,"{""location"": ""ID"", ""is_mobile"": true}" 11789,5,264,2017-04-14 02:58:20,http://green.org/emory_mann,0.2629312041,143.125.220.36,"{""location"": ""US"", ""is_mobile"": false}" 11790,5,264,2017-03-10 14:55:24,http://schumm.name/mia.pagac,0.5566922607,131.24.20.215,"{""location"": ""DM"", ""is_mobile"": true}" 11791,5,264,2017-03-16 09:39:23,http://mraz.io/lucienne,0.9698309033,115.4.109.246,"{""location"": ""PK"", ""is_mobile"": false}" 11792,5,264,2017-04-14 23:26:25,http://oconner.name/hershel_mccullough,0.6218460271,173.169.207.82,"{""location"": ""MW"", ""is_mobile"": true}" 11793,5,264,2016-12-19 18:31:04,http://bednardibbert.org/rhea.barton,0.5442219433,19.231.145.123,"{""location"": ""LR"", ""is_mobile"": false}" 11794,5,264,2017-02-14 17:07:03,http://wiegand.org/aurelie,0.2730209832,118.87.122.94,"{""location"": ""KZ"", ""is_mobile"": true}" 11795,5,264,2017-04-08 01:44:44,http://terry.biz/geovanni,0.1498978854,77.63.173.180,"{""location"": ""KN"", ""is_mobile"": false}" 11796,5,264,2017-02-25 18:38:39,http://moenaltenwerth.name/sarai.grady,0.0525185271,32.188.243.216,"{""location"": ""LS"", ""is_mobile"": false}" 11797,5,264,2017-02-15 00:59:53,http://ratke.biz/carley,0.9792004753,207.61.54.2,"{""location"": ""SV"", ""is_mobile"": true}" 11798,5,264,2017-04-13 18:52:20,http://hacketttillman.info/megane,0.2000283080,169.220.12.34,"{""location"": ""HR"", ""is_mobile"": true}" 11799,5,264,2017-04-15 16:19:25,http://johns.biz/abbigail.bruen,0.7005317652,233.223.123.246,"{""location"": ""CF"", ""is_mobile"": false}" 11800,5,264,2016-12-22 18:35:27,http://yostgusikowski.info/steve,0.7034506332,40.185.6.234,"{""location"": ""CH"", ""is_mobile"": false}" 11801,5,264,2017-02-02 07:10:02,http://reynolds.info/pedro,0.0604570209,3.79.237.102,"{""location"": ""GD"", ""is_mobile"": true}" 11802,5,264,2017-04-26 04:47:54,http://boehm.co/virgil,0.6700351048,70.86.169.17,"{""location"": ""KW"", ""is_mobile"": true}" 8887,4,199,2017-01-06 10:54:33,http://collierkohler.com/eduardo.jakubowski,0.5452140762,246.28.87.203,"{""location"": ""CL"", ""is_mobile"": true}" 8888,4,199,2017-05-10 11:08:56,http://johnsongaylord.name/hilario,0.4697952740,66.143.187.56,"{""location"": ""TM"", ""is_mobile"": true}" 8889,4,199,2016-12-28 17:19:47,http://yundtgreenholt.io/marianna,0.3223297646,77.181.56.188,"{""location"": ""BD"", ""is_mobile"": true}" 8890,4,199,2017-01-06 18:16:24,http://kris.org/al,0.4301509114,180.15.213.245,"{""location"": ""BR"", ""is_mobile"": false}" 8891,4,199,2017-04-07 00:42:48,http://oconnellsteuber.biz/richie_skiles,0.8376457942,24.110.195.135,"{""location"": ""IM"", ""is_mobile"": true}" 8892,4,199,2016-12-25 11:12:27,http://trantow.name/erika.will,0.7093578694,120.113.129.207,"{""location"": ""NL"", ""is_mobile"": false}" 8893,4,199,2017-02-02 09:59:44,http://erdmanlang.co/roel_borer,0.1034211088,147.77.59.23,"{""location"": ""GL"", ""is_mobile"": false}" 8894,4,199,2017-01-25 21:58:39,http://shields.org/gregory,0.6686169953,227.105.88.72,"{""location"": ""KY"", ""is_mobile"": true}" 8895,4,199,2017-01-15 00:19:20,http://trantow.com/palma_roberts,0.0251613110,171.246.183.67,"{""location"": ""ZA"", ""is_mobile"": true}" 8896,4,199,2017-06-08 17:27:34,http://donnellydavis.com/hattie,0.5330141072,218.226.76.251,"{""location"": ""MY"", ""is_mobile"": false}" 8897,4,199,2017-06-09 21:47:36,http://smithamabshire.io/danika.kaulke,0.1851461362,218.14.232.187,"{""location"": ""CR"", ""is_mobile"": false}" 8898,4,199,2017-03-01 23:28:33,http://wizaspinka.name/roma_grimes,0.8940611531,139.177.33.126,"{""location"": ""KN"", ""is_mobile"": true}" 8899,4,199,2017-01-31 11:44:24,http://pollich.org/cyril,0.2417822554,183.159.67.75,"{""location"": ""SI"", ""is_mobile"": false}" 8900,4,199,2017-05-05 19:09:12,http://stantonnolan.co/cara,0.6459126273,108.23.223.139,"{""location"": ""BL"", ""is_mobile"": true}" 8901,4,199,2017-04-12 11:51:13,http://smitham.net/josh_hills,0.6931682517,68.145.103.219,"{""location"": ""TO"", ""is_mobile"": false}" 8902,4,199,2016-12-15 03:15:43,http://maggioschneider.org/consuelo_dooley,0.0036287370,174.55.87.141,"{""location"": ""AM"", ""is_mobile"": false}" 8903,4,199,2016-12-27 20:08:35,http://dachnikolaus.net/roy,0.7517413810,208.191.88.60,"{""location"": ""MV"", ""is_mobile"": true}" 8904,4,199,2017-03-25 04:17:08,http://crist.info/willis,0.3230633267,53.144.51.195,"{""location"": ""KY"", ""is_mobile"": false}" 8905,4,199,2017-04-16 12:51:14,http://labadie.com/augustus,0.8987353695,77.201.33.166,"{""location"": ""AT"", ""is_mobile"": false}" 8906,4,199,2016-12-27 23:57:19,http://luettgen.io/elizabeth,0.9752222880,108.86.52.112,"{""location"": ""CR"", ""is_mobile"": true}" 8907,4,199,2017-02-22 08:00:18,http://emmerichankunding.io/colleen,0.1672225502,159.134.205.223,"{""location"": ""VI"", ""is_mobile"": true}" 8908,4,199,2016-12-17 10:01:24,http://lind.biz/lavinia_franecki,0.9052647685,247.112.186.173,"{""location"": ""LC"", ""is_mobile"": false}" 8909,4,199,2017-04-17 09:56:14,http://king.info/krystina_fay,0.9400359907,196.143.235.33,"{""location"": ""KM"", ""is_mobile"": true}" 8910,4,199,2017-04-26 14:15:42,http://schuppe.net/jeanette,0.4429472720,247.41.63.144,"{""location"": ""TF"", ""is_mobile"": false}" 8911,4,199,2017-04-12 04:20:34,http://lefflermohr.biz/dallas,0.0758749258,100.89.16.241,"{""location"": ""RE"", ""is_mobile"": false}" 8912,4,199,2017-03-03 11:29:47,http://sauer.org/claude,0.5007537769,188.134.250.193,"{""location"": ""MT"", ""is_mobile"": false}" 8913,4,200,2017-02-11 11:37:40,http://krisfunk.com/chyna,0.9868017792,192.159.111.249,"{""location"": ""BM"", ""is_mobile"": true}" 8914,4,200,2017-01-08 16:22:43,http://considine.biz/newell,0.8540858332,108.30.221.95,"{""location"": ""CR"", ""is_mobile"": true}" 8915,4,200,2016-12-29 04:05:09,http://wintheiser.net/mary_stiedemann,0.2027538603,127.82.134.118,"{""location"": ""LA"", ""is_mobile"": true}" 8916,4,200,2017-04-18 04:14:02,http://corwin.com/eliezer,0.4859105104,98.195.48.220,"{""location"": ""BJ"", ""is_mobile"": true}" 8917,4,200,2017-04-24 14:26:51,http://dickenshintz.net/aniyah,0.8364392966,219.24.251.168,"{""location"": ""TJ"", ""is_mobile"": true}" 8918,4,200,2017-03-28 18:07:17,http://cremin.name/noemi,0.5397364043,220.50.231.4,"{""location"": ""GT"", ""is_mobile"": true}" 8919,4,200,2017-01-28 06:09:26,http://dach.co/catharine.tillman,0.7892810243,30.193.74.133,"{""location"": ""ML"", ""is_mobile"": true}" 8920,4,200,2017-04-30 05:14:36,http://raukuhlman.com/trenton_ko,0.3466791421,33.141.66.180,"{""location"": ""MF"", ""is_mobile"": false}" 8921,4,200,2017-05-16 13:42:28,http://mcglynn.net/natalie_pacocha,0.1242810546,152.5.64.235,"{""location"": ""IE"", ""is_mobile"": true}" 8922,4,200,2017-05-05 07:27:09,http://hudson.org/julianne_rowe,0.4562617110,206.60.110.60,"{""location"": ""VC"", ""is_mobile"": true}" 8923,4,200,2016-12-18 15:28:29,http://buckridge.info/vivien,0.0159612831,114.75.252.61,"{""location"": ""CH"", ""is_mobile"": false}" 8924,4,200,2016-12-25 19:55:27,http://romaguera.biz/porter.casper,0.0103996868,120.41.67.251,"{""location"": ""SB"", ""is_mobile"": true}" 8925,4,200,2017-03-29 22:20:21,http://rolfsonorn.biz/germaine_willms,0.2905568949,242.252.182.29,"{""location"": ""IM"", ""is_mobile"": false}" 8926,4,200,2017-03-22 05:44:48,http://sanford.io/kolby.feil,0.9276146908,174.140.105.146,"{""location"": ""BD"", ""is_mobile"": false}" 8927,4,200,2016-12-13 13:24:48,http://kiehngottlieb.name/alexis.greenholt,0.6472984636,231.193.250.6,"{""location"": ""JE"", ""is_mobile"": true}" 8928,4,200,2017-02-26 14:39:46,http://muellertreutel.co/meagan_feeney,0.7247556074,231.164.141.225,"{""location"": ""GT"", ""is_mobile"": false}" 8929,4,200,2017-05-10 13:44:49,http://hegmann.info/lila.nader,0.6366875602,225.235.102.239,"{""location"": ""ID"", ""is_mobile"": false}" 8930,4,200,2017-04-20 05:52:08,http://mannreynolds.org/eve,0.8859466110,154.131.187.176,"{""location"": ""CH"", ""is_mobile"": true}" 8931,4,200,2017-02-26 13:17:28,http://von.co/lulu.altenwerth,0.2840902850,71.188.176.77,"{""location"": ""UA"", ""is_mobile"": false}" 8932,4,200,2017-02-01 04:31:33,http://johnson.co/derick_kunde,0.0076819992,213.117.4.22,"{""location"": ""LY"", ""is_mobile"": true}" 8933,4,200,2017-03-02 01:29:02,http://muller.name/krystel,0.6000433657,207.171.234.228,"{""location"": ""RW"", ""is_mobile"": false}" 8934,4,200,2017-01-06 15:50:41,http://hellerhowell.info/rickey,0.1538772415,219.12.167.150,"{""location"": ""BL"", ""is_mobile"": false}" 8935,4,200,2017-05-24 00:38:01,http://jerdequigley.io/oda,0.4239317426,106.13.251.95,"{""location"": ""GP"", ""is_mobile"": false}" 8936,4,200,2017-04-28 03:29:50,http://kozey.io/carmine_lesch,0.2452819805,9.42.162.59,"{""location"": ""LK"", ""is_mobile"": false}" 8937,4,200,2017-02-11 17:39:05,http://wunschoconnell.com/pearline_cartwright,0.1373539715,141.231.11.96,"{""location"": ""HU"", ""is_mobile"": true}" 5870,3,128,2017-06-13 10:47:14,http://kuvalitrosin.com/chadrick_murphy,,108.241.47.103,"{""location"": ""BG"", ""is_mobile"": false}" 5871,3,128,2016-12-17 18:06:56,http://heidenreichgulgowski.co/alta,,193.168.113.226,"{""location"": ""GG"", ""is_mobile"": true}" 5872,3,128,2017-06-11 08:45:01,http://greenfelder.biz/kathryn,,94.77.246.234,"{""location"": ""FJ"", ""is_mobile"": false}" 5873,3,128,2017-06-04 15:35:29,http://predovic.biz/gielle.romaguera,,45.3.203.95,"{""location"": ""BR"", ""is_mobile"": true}" 5874,3,128,2017-06-06 17:25:08,http://treutelprosacco.net/devin,,134.91.23.110,"{""location"": ""RO"", ""is_mobile"": true}" 5875,3,128,2017-04-09 09:51:12,http://schamberger.co/wilhelmine.wuckert,,72.156.224.97,"{""location"": ""ET"", ""is_mobile"": true}" 5876,3,128,2017-05-30 23:08:00,http://mayertlueilwitz.name/domenico,,85.42.140.121,"{""location"": ""EE"", ""is_mobile"": false}" 5877,3,128,2017-02-26 19:43:10,http://gutmann.co/joy_balistreri,,112.95.130.243,"{""location"": ""KR"", ""is_mobile"": false}" 5878,3,128,2017-05-31 00:19:23,http://hilll.io/zena,,84.13.126.226,"{""location"": ""CY"", ""is_mobile"": true}" 5879,3,128,2017-02-06 20:37:46,http://schultzmcdermott.info/kaleigh,,116.110.252.226,"{""location"": ""BG"", ""is_mobile"": true}" 5880,3,128,2017-05-25 18:23:13,http://hermistongrimes.info/noelia,,205.190.199.233,"{""location"": ""LA"", ""is_mobile"": true}" 5881,3,128,2017-06-09 13:16:59,http://greenfeldercartwright.io/loyal_goldner,,212.29.184.225,"{""location"": ""KM"", ""is_mobile"": false}" 5882,3,128,2017-01-18 20:48:36,http://sporer.co/granville,,115.193.43.30,"{""location"": ""CA"", ""is_mobile"": true}" 5883,3,128,2017-03-25 23:40:06,http://schuppe.com/rusty.hirthe,,29.151.81.45,"{""location"": ""AS"", ""is_mobile"": true}" 5884,3,128,2016-12-30 01:24:32,http://kautzer.name/wava_hudson,,183.193.41.143,"{""location"": ""KW"", ""is_mobile"": false}" 5885,3,128,2017-05-25 13:31:31,http://blockmarvin.biz/suzanne_beatty,,83.151.250.236,"{""location"": ""AX"", ""is_mobile"": false}" 5886,3,128,2017-04-24 21:33:42,http://mrazvandervort.net/raphaelle,,175.202.240.56,"{""location"": ""TT"", ""is_mobile"": true}" 5887,3,128,2017-05-29 00:43:22,http://kiehn.io/elta,,110.31.172.95,"{""location"": ""FR"", ""is_mobile"": true}" 5888,3,128,2017-05-24 15:31:21,http://hansenpredovic.co/jordan.jacobs,,75.68.168.130,"{""location"": ""AG"", ""is_mobile"": false}" 5889,3,128,2017-05-17 06:42:21,http://powlowskiblock.org/don,,223.150.237.97,"{""location"": ""VC"", ""is_mobile"": true}" 5890,3,128,2017-05-12 23:52:25,http://mante.info/kaandra,,181.104.73.80,"{""location"": ""MV"", ""is_mobile"": true}" 5891,3,128,2017-03-27 09:00:25,http://schoenmetz.net/elinor,,185.10.237.250,"{""location"": ""CF"", ""is_mobile"": true}" 5892,3,128,2017-05-14 01:26:53,http://dibbertjohnston.net/te,,51.29.254.165,"{""location"": ""LV"", ""is_mobile"": true}" 5893,3,128,2017-05-21 22:38:08,http://torp.io/kayden,,11.81.193.58,"{""location"": ""JE"", ""is_mobile"": false}" 5894,3,128,2016-12-23 13:28:28,http://ruel.name/bonnie,,19.9.173.178,"{""location"": ""AU"", ""is_mobile"": false}" 5895,3,128,2017-06-11 11:57:16,http://gibson.biz/arianna_huel,,77.121.122.153,"{""location"": ""GM"", ""is_mobile"": true}" 5896,3,128,2017-02-24 06:35:50,http://macejkovic.name/malcolm_koch,,14.110.130.206,"{""location"": ""VU"", ""is_mobile"": true}" 5897,3,128,2017-01-20 03:43:18,http://pfannerstill.org/coleman,,152.100.138.79,"{""location"": ""KE"", ""is_mobile"": true}" 5898,3,128,2017-02-22 11:59:42,http://witting.org/vernie_denesik,,141.170.22.82,"{""location"": ""TZ"", ""is_mobile"": true}" 5899,3,128,2017-01-23 12:25:31,http://bechtelar.co/wilhelmine_haag,,243.224.167.41,"{""location"": ""LV"", ""is_mobile"": true}" 5900,3,128,2017-05-27 17:15:42,http://bergstrom.name/ally,,163.57.126.241,"{""location"": ""IO"", ""is_mobile"": true}" 5901,3,128,2017-05-19 04:16:53,http://reichertgoyette.net/cory,,229.174.124.122,"{""location"": ""GQ"", ""is_mobile"": true}" 5902,3,128,2017-02-19 02:18:41,http://homenick.net/mabel.batz,,20.90.216.209,"{""location"": ""ET"", ""is_mobile"": true}" 5903,3,128,2017-01-14 19:23:26,http://quitzon.org/lucinda,,187.8.177.13,"{""location"": ""SE"", ""is_mobile"": false}" 5904,3,128,2017-01-22 10:46:50,http://marvin.info/moses,,85.181.39.92,"{""location"": ""IN"", ""is_mobile"": false}" 5905,3,128,2016-12-14 23:34:01,http://oreilly.org/emelie,,194.119.193.228,"{""location"": ""TV"", ""is_mobile"": true}" 5906,3,128,2017-04-07 21:29:52,http://breitenberg.com/savanah,,176.100.50.231,"{""location"": ""UZ"", ""is_mobile"": true}" 5907,3,128,2017-05-03 09:09:41,http://crist.io/evangeline,,174.162.17.6,"{""location"": ""CH"", ""is_mobile"": false}" 5908,3,128,2017-04-27 19:28:05,http://kuvalis.com/tanya,,122.166.148.207,"{""location"": ""ZM"", ""is_mobile"": false}" 5909,3,128,2017-04-18 10:29:02,http://abshire.net/miles,,177.5.74.58,"{""location"": ""HN"", ""is_mobile"": false}" 5910,3,128,2017-02-09 19:34:23,http://dubuque.org/dawson,,201.193.212.93,"{""location"": ""UZ"", ""is_mobile"": false}" 5911,3,129,2017-03-16 21:28:21,http://kunde.co/jacques,,250.86.25.38,"{""location"": ""CM"", ""is_mobile"": false}" 5912,3,129,2017-02-15 05:22:18,http://gibson.org/verlie_crooks,,134.107.127.32,"{""location"": ""VG"", ""is_mobile"": false}" 5913,3,129,2017-04-12 09:57:18,http://gislasonleuschke.co/dayna,,112.52.111.246,"{""location"": ""AQ"", ""is_mobile"": true}" 5914,3,129,2017-05-09 13:09:07,http://satterfield.net/briana.metz,,145.178.27.26,"{""location"": ""IM"", ""is_mobile"": true}" 5915,3,129,2017-01-02 23:13:15,http://emmerich.org/arlo_rodriguez,,190.229.212.181,"{""location"": ""AG"", ""is_mobile"": false}" 5916,3,129,2017-03-19 02:26:18,http://gerhold.name/declan.hilpert,,113.85.38.56,"{""location"": ""MY"", ""is_mobile"": true}" 5917,3,129,2017-01-10 12:20:06,http://kutch.name/dianna,,63.10.23.107,"{""location"": ""SJ"", ""is_mobile"": true}" 5918,3,129,2017-03-09 23:21:38,http://gulgowskidurgan.co/madeline,,149.190.130.23,"{""location"": ""BS"", ""is_mobile"": true}" 5919,3,129,2017-04-20 12:19:31,http://welchhansen.name/dannie,,14.126.56.193,"{""location"": ""TF"", ""is_mobile"": false}" 5920,3,129,2017-04-15 00:23:19,http://lesch.info/leola.wiegand,,53.111.2.40,"{""location"": ""AL"", ""is_mobile"": false}" 5921,3,129,2017-04-05 19:52:53,http://stark.name/consuelo.fadel,,140.31.60.102,"{""location"": ""YE"", ""is_mobile"": false}" 5922,3,129,2017-04-04 10:26:54,http://weber.com/abigail,,219.44.117.88,"{""location"": ""PA"", ""is_mobile"": false}" 5923,3,129,2017-06-02 02:10:40,http://schambergermueller.net/zora.roob,,49.179.12.51,"{""location"": ""CU"", ""is_mobile"": true}" 5924,3,129,2017-05-01 09:42:41,http://spencer.biz/mya,,178.73.190.134,"{""location"": ""CK"", ""is_mobile"": false}" 5925,3,129,2017-02-06 06:32:14,http://jakubowski.name/drew,,73.237.205.159,"{""location"": ""CA"", ""is_mobile"": true}" 11803,5,264,2017-04-07 01:15:52,http://collins.org/marta,0.5092112511,170.109.126.166,"{""location"": ""AT"", ""is_mobile"": false}" 11804,5,264,2017-04-09 17:07:13,http://daniel.net/rodolfo,0.4705072877,149.76.60.11,"{""location"": ""GF"", ""is_mobile"": true}" 11805,5,264,2017-05-31 04:12:51,http://carter.org/queen,0.8897180847,118.12.139.105,"{""location"": ""CF"", ""is_mobile"": false}" 11806,5,264,2016-12-25 00:47:05,http://hackettokon.info/easter.pfannerstill,0.7276331323,109.91.82.84,"{""location"": ""CG"", ""is_mobile"": true}" 11807,5,264,2017-01-24 17:21:29,http://moenfriesen.com/twila.turcotte,0.6990072158,157.62.29.240,"{""location"": ""BT"", ""is_mobile"": true}" 11808,5,264,2016-12-21 21:19:53,http://feest.biz/sasha_rosenbaum,0.9242724067,150.140.204.252,"{""location"": ""SN"", ""is_mobile"": true}" 11809,5,264,2017-06-14 02:07:37,http://fritsch.io/bernie,0.3992487994,60.173.24.178,"{""location"": ""PL"", ""is_mobile"": false}" 11810,5,264,2017-03-05 19:10:33,http://oconner.io/myah,0.3722569298,18.218.41.139,"{""location"": ""BT"", ""is_mobile"": false}" 11811,5,264,2017-04-26 18:02:56,http://ebert.org/van,0.2569907847,224.142.76.83,"{""location"": ""GS"", ""is_mobile"": true}" 11812,5,264,2017-02-18 01:01:37,http://mckenzie.co/magdalena_halvorson,0.4464662152,2.147.211.206,"{""location"": ""WS"", ""is_mobile"": false}" 11813,5,264,2017-01-17 06:19:29,http://balistreridietrich.com/ernest,0.1178537319,244.70.160.134,"{""location"": ""IR"", ""is_mobile"": false}" 11814,5,264,2017-05-02 19:21:31,http://gibson.io/karl.conroy,0.5524511255,133.137.114.114,"{""location"": ""NI"", ""is_mobile"": true}" 11815,5,264,2017-03-03 03:56:30,http://mcclure.net/toni_abernathy,0.9636599314,144.135.145.156,"{""location"": ""IQ"", ""is_mobile"": true}" 11816,5,264,2017-05-13 03:55:02,http://ratke.org/terrence,0.9665444428,50.169.5.126,"{""location"": ""DM"", ""is_mobile"": true}" 11817,5,264,2017-04-20 18:38:49,http://huelrosenbaum.com/matteo_herzog,0.3740746129,33.40.134.16,"{""location"": ""MA"", ""is_mobile"": false}" 11818,5,264,2017-05-17 18:34:43,http://mayerjohnson.com/pierre.schneider,0.0888158790,111.4.230.141,"{""location"": ""BS"", ""is_mobile"": true}" 11819,5,264,2017-05-17 09:19:56,http://yostfeil.name/vesta,0.3206051294,55.218.132.231,"{""location"": ""MH"", ""is_mobile"": true}" 11820,5,264,2016-12-28 16:27:22,http://marquardt.io/jarrell,0.9625616449,90.251.190.148,"{""location"": ""HU"", ""is_mobile"": false}" 11821,5,264,2017-06-08 19:55:38,http://mayer.net/cecelia.turner,0.4121764044,185.194.43.148,"{""location"": ""TC"", ""is_mobile"": true}" 11822,5,264,2016-12-14 15:52:30,http://dickinsongutkowski.io/burnice_kulas,0.9288447098,210.100.159.19,"{""location"": ""GQ"", ""is_mobile"": false}" 11823,5,264,2017-02-10 11:00:39,http://farrellbechtelar.name/hipolito_cain,0.6572692414,70.43.250.102,"{""location"": ""TT"", ""is_mobile"": true}" 11824,5,264,2017-01-13 03:01:45,http://bradtkerippin.org/general.kuhlman,0.0499739102,70.28.161.218,"{""location"": ""GP"", ""is_mobile"": true}" 11825,5,264,2017-03-10 00:47:42,http://oconnercummerata.org/shirley.predovic,0.3286439957,217.53.167.46,"{""location"": ""LV"", ""is_mobile"": true}" 11826,5,264,2017-05-14 06:08:05,http://kohler.co/callie,0.5543270117,44.232.50.34,"{""location"": ""DJ"", ""is_mobile"": false}" 11827,5,264,2017-04-24 22:37:18,http://jacobi.com/paxton.shields,0.3844232620,24.24.199.209,"{""location"": ""TZ"", ""is_mobile"": false}" 11828,5,264,2017-02-11 01:37:56,http://pouroswindler.io/elmore.reichel,0.9084581637,67.155.2.48,"{""location"": ""KR"", ""is_mobile"": true}" 11829,5,264,2017-04-12 09:22:50,http://mitchellcollins.info/rudolph_roberts,0.8678939228,72.110.68.102,"{""location"": ""BV"", ""is_mobile"": false}" 11830,5,264,2017-04-09 16:57:53,http://crona.io/kale_breitenberg,0.7337803856,23.114.236.213,"{""location"": ""MO"", ""is_mobile"": true}" 11831,5,264,2017-01-07 17:14:40,http://bogisich.com/jerald,0.6105300849,228.26.122.51,"{""location"": ""RW"", ""is_mobile"": true}" 11832,5,264,2016-12-14 23:42:50,http://carrollstroman.org/dasia,0.1750003859,110.162.166.184,"{""location"": ""HN"", ""is_mobile"": false}" 11833,5,264,2017-02-09 20:35:02,http://nicolasankunding.biz/dudley_larson,0.3962373414,97.129.202.7,"{""location"": ""MR"", ""is_mobile"": false}" 11834,5,264,2017-01-03 09:50:42,http://dach.io/adelle,0.5130153850,235.31.153.237,"{""location"": ""ML"", ""is_mobile"": false}" 11835,5,264,2017-01-01 18:13:44,http://grady.info/marcia.walsh,0.3271548493,83.66.140.53,"{""location"": ""PT"", ""is_mobile"": true}" 11836,5,264,2017-02-26 06:05:39,http://hettinger.io/orland,0.6142208273,250.165.63.96,"{""location"": ""LY"", ""is_mobile"": false}" 11837,5,264,2017-03-12 05:52:31,http://reichel.biz/magnolia.hackett,0.8056922867,76.236.74.235,"{""location"": ""PG"", ""is_mobile"": true}" 11838,5,264,2017-05-24 01:47:56,http://veum.co/darby,0.2873974556,10.151.64.229,"{""location"": ""BL"", ""is_mobile"": false}" 11839,5,264,2017-03-08 20:14:12,http://herzog.net/alexa_weinat,0.7614394208,189.232.130.11,"{""location"": ""MS"", ""is_mobile"": true}" 11840,5,264,2017-01-28 10:48:43,http://boscookuneva.biz/terry,0.0771578786,242.35.119.54,"{""location"": ""SH"", ""is_mobile"": false}" 11841,5,264,2017-03-30 05:06:33,http://gaylordkohler.org/pascale_predovic,0.2769083246,17.99.247.192,"{""location"": ""BT"", ""is_mobile"": false}" 11842,5,265,2017-04-12 16:02:35,http://gerhold.net/dejuan.anderson,0.8444568189,69.146.179.63,"{""location"": ""ME"", ""is_mobile"": true}" 11843,5,265,2017-02-01 12:45:02,http://connheel.info/hazle,0.0430238197,102.93.51.251,"{""location"": ""TC"", ""is_mobile"": false}" 11844,5,265,2017-03-27 00:22:55,http://robertslockman.io/matteo.heller,0.8495660591,67.45.139.87,"{""location"": ""AU"", ""is_mobile"": true}" 11845,5,265,2016-12-28 21:00:57,http://larkinborer.org/ladarius,0.1428910138,168.59.186.35,"{""location"": ""CX"", ""is_mobile"": true}" 11846,5,265,2017-03-09 09:35:40,http://rowe.name/nichole_weber,0.2415414949,184.108.93.53,"{""location"": ""ID"", ""is_mobile"": true}" 11847,5,265,2017-02-20 23:02:03,http://ornhand.co/laron,0.0977885918,152.48.109.120,"{""location"": ""MQ"", ""is_mobile"": true}" 11848,5,265,2017-06-13 01:58:42,http://schuster.org/patricia_schimmel,0.8064989147,189.87.216.55,"{""location"": ""MV"", ""is_mobile"": true}" 11849,5,265,2017-01-17 13:05:19,http://reichel.name/morton_quigley,0.1882777781,250.74.246.155,"{""location"": ""BR"", ""is_mobile"": false}" 11850,5,265,2017-04-22 00:00:02,http://wisoky.io/rudolph_smitham,0.1331254577,106.77.219.204,"{""location"": ""PF"", ""is_mobile"": true}" 11851,5,265,2017-05-06 21:18:27,http://gleason.info/francis_morar,0.4049434841,93.83.249.37,"{""location"": ""FR"", ""is_mobile"": false}" 11852,5,265,2017-01-24 07:14:44,http://torp.com/thad,0.5691784122,241.195.93.72,"{""location"": ""GY"", ""is_mobile"": true}" 8938,4,200,2017-06-09 22:53:12,http://okeefe.biz/fritz_stiedemann,0.9042114293,105.68.181.199,"{""location"": ""BD"", ""is_mobile"": true}" 8939,4,200,2017-04-18 11:43:50,http://wolf.co/rowan.damore,0.4099718946,159.74.22.68,"{""location"": ""YE"", ""is_mobile"": false}" 8940,4,201,2017-02-21 20:17:31,http://koepp.co/junius,0.3361907255,223.242.146.98,"{""location"": ""PW"", ""is_mobile"": false}" 8941,4,201,2017-02-01 21:25:31,http://mann.co/keely_huels,0.2578809864,245.168.56.77,"{""location"": ""SY"", ""is_mobile"": false}" 8942,4,201,2017-04-01 03:35:37,http://lakin.name/alfreda,0.4026363870,160.55.82.42,"{""location"": ""SG"", ""is_mobile"": false}" 8943,4,201,2017-01-19 21:16:47,http://sauer.biz/lazaro,0.7132542221,99.73.153.187,"{""location"": ""PT"", ""is_mobile"": true}" 8944,4,201,2017-03-16 06:01:21,http://hanewitting.info/lizeth.orn,0.6391945721,207.164.114.203,"{""location"": ""SM"", ""is_mobile"": false}" 8945,4,201,2016-12-23 14:30:13,http://howellhermiston.net/alexandre,0.9959124371,47.123.238.76,"{""location"": ""MA"", ""is_mobile"": false}" 8946,4,201,2017-01-17 13:28:38,http://johnston.co/akeem,0.5668473804,48.174.74.249,"{""location"": ""LV"", ""is_mobile"": true}" 8947,4,201,2017-04-09 21:00:46,http://kshlerin.net/timmy,0.9359064478,236.67.185.69,"{""location"": ""TJ"", ""is_mobile"": true}" 8948,4,201,2017-03-14 11:00:59,http://turner.net/donato.hamill,0.6776376900,149.30.19.195,"{""location"": ""FI"", ""is_mobile"": false}" 8949,4,201,2017-01-16 22:28:33,http://hauck.biz/lon.funk,0.6269355054,131.193.32.179,"{""location"": ""MH"", ""is_mobile"": true}" 8950,4,201,2017-03-15 07:36:33,http://fritsch.name/deron_green,0.9810399711,163.178.126.180,"{""location"": ""ID"", ""is_mobile"": false}" 8951,4,201,2017-05-30 06:43:21,http://zboncak.com/ernie_hane,0.6465781846,179.211.41.200,"{""location"": ""BO"", ""is_mobile"": true}" 8952,4,201,2017-02-23 15:33:02,http://hoeger.co/mohammad_adams,0.2439910650,124.133.40.247,"{""location"": ""PK"", ""is_mobile"": true}" 8953,4,201,2017-05-05 05:08:10,http://langworthjohnston.biz/sheldon,0.5329562055,98.62.193.252,"{""location"": ""CA"", ""is_mobile"": false}" 8954,4,201,2016-12-25 20:54:16,http://mcclurekuphal.info/aracely_smitham,0.8850597086,16.143.76.246,"{""location"": ""PA"", ""is_mobile"": false}" 8955,4,201,2017-03-09 03:52:20,http://langosh.com/jedidiah_carter,0.3057327534,232.246.85.10,"{""location"": ""SH"", ""is_mobile"": false}" 8956,4,201,2017-01-09 19:31:31,http://thiel.net/brent,0.9988888548,228.215.116.150,"{""location"": ""PL"", ""is_mobile"": true}" 8957,4,201,2017-02-19 00:34:39,http://klingbrekke.co/aleen_kiehn,0.4870008507,31.83.139.103,"{""location"": ""MX"", ""is_mobile"": true}" 8958,4,201,2017-01-22 17:17:16,http://satterfield.info/antonetta.stroman,0.3101512536,221.223.70.108,"{""location"": ""BZ"", ""is_mobile"": true}" 8959,4,201,2017-04-01 07:19:22,http://lesch.info/kale_spinka,0.9587893650,73.234.233.211,"{""location"": ""RO"", ""is_mobile"": true}" 8960,4,201,2017-02-11 09:38:56,http://ward.com/kelli_stehr,0.6901307704,135.122.226.127,"{""location"": ""AL"", ""is_mobile"": true}" 8961,4,201,2017-03-18 09:03:36,http://gerholdcasper.net/ethan_ruecker,0.9397766450,252.154.122.236,"{""location"": ""PK"", ""is_mobile"": false}" 8962,4,201,2017-01-12 14:13:12,http://predovic.net/oleta,0.8902798469,215.112.21.228,"{""location"": ""KZ"", ""is_mobile"": true}" 8963,4,201,2017-02-22 15:42:07,http://marvin.name/tony_franecki,0.0130136231,51.19.65.215,"{""location"": ""KI"", ""is_mobile"": true}" 8964,4,201,2017-05-16 04:38:23,http://blick.info/jabari,0.9294544341,233.151.143.221,"{""location"": ""CR"", ""is_mobile"": true}" 8965,4,201,2017-05-19 14:48:42,http://shanahan.org/imani.willms,0.4976632911,166.65.241.232,"{""location"": ""UY"", ""is_mobile"": true}" 8966,4,201,2017-05-18 09:56:53,http://parisian.name/stephania.hayes,0.9032666234,146.145.15.135,"{""location"": ""TG"", ""is_mobile"": false}" 8967,4,202,2017-06-05 23:02:19,http://berge.biz/amanda,0.9302212510,252.24.157.213,"{""location"": ""TZ"", ""is_mobile"": true}" 8968,4,202,2017-02-20 17:18:08,http://gleichner.org/bryce,0.3455450368,77.232.63.43,"{""location"": ""GL"", ""is_mobile"": true}" 8969,4,202,2017-06-11 04:09:56,http://marks.name/justen,0.7528374472,113.8.75.187,"{""location"": ""SL"", ""is_mobile"": true}" 8970,4,202,2017-06-12 02:15:53,http://nikolaus.name/devonte,0.6114518275,38.139.205.5,"{""location"": ""TJ"", ""is_mobile"": false}" 8971,4,202,2017-02-06 05:19:25,http://bosco.info/germaine,0.8981378032,135.176.11.67,"{""location"": ""HU"", ""is_mobile"": false}" 8972,4,202,2017-05-23 11:56:37,http://waelchi.name/bertrand,0.5610399436,182.136.215.53,"{""location"": ""LR"", ""is_mobile"": false}" 8973,4,202,2017-05-25 01:43:11,http://sanford.name/jaylin_mraz,0.8082886509,71.97.225.156,"{""location"": ""CR"", ""is_mobile"": false}" 8974,4,202,2017-03-07 14:03:52,http://koelpin.co/susanna.gaylord,0.4098374336,243.148.78.208,"{""location"": ""US"", ""is_mobile"": true}" 8975,4,202,2017-02-13 10:58:54,http://abbott.co/israel_klein,0.2053049629,116.219.76.223,"{""location"": ""ES"", ""is_mobile"": true}" 8976,4,202,2017-05-05 18:21:38,http://abernathy.biz/araceli,0.0174805890,51.208.122.154,"{""location"": ""PF"", ""is_mobile"": false}" 8977,4,202,2017-05-17 14:12:14,http://lindcorkery.info/rusty,0.3171729814,209.34.115.234,"{""location"": ""AF"", ""is_mobile"": false}" 8978,4,202,2017-02-17 03:33:11,http://farrellwintheiser.co/santino_ledner,0.2093625735,150.127.36.235,"{""location"": ""TR"", ""is_mobile"": true}" 8979,4,202,2017-04-19 23:04:52,http://abshire.org/lucio_blanda,0.6023417124,11.9.88.139,"{""location"": ""NI"", ""is_mobile"": false}" 8980,4,202,2017-05-08 12:59:16,http://lehnerwaelchi.name/heidi,0.9199542922,227.128.142.98,"{""location"": ""TW"", ""is_mobile"": false}" 8981,4,202,2017-04-18 12:50:44,http://connelly.name/leopold_smitham,0.5440987643,9.74.45.159,"{""location"": ""IM"", ""is_mobile"": true}" 8982,4,202,2017-02-01 04:07:07,http://yostnitzsche.io/kamille,0.3938259813,113.40.175.251,"{""location"": ""WS"", ""is_mobile"": false}" 8983,4,202,2017-02-07 21:49:45,http://jacobson.com/trystan.wiegand,0.9542054127,231.175.167.138,"{""location"": ""NO"", ""is_mobile"": true}" 8984,4,202,2017-01-11 02:35:30,http://barton.biz/yvonne.dooley,0.2493037519,169.76.160.254,"{""location"": ""IL"", ""is_mobile"": false}" 8985,4,202,2017-03-17 10:11:01,http://hicklekemmer.net/deshawn,0.6294770458,152.38.142.162,"{""location"": ""GH"", ""is_mobile"": true}" 8986,4,202,2016-12-15 16:39:09,http://caspercollier.com/nicholaus_gorczany,0.2120287345,141.52.9.223,"{""location"": ""KR"", ""is_mobile"": true}" 8987,4,202,2017-04-04 06:40:19,http://stark.name/damien,0.1271790707,252.161.143.225,"{""location"": ""TZ"", ""is_mobile"": false}" 8988,4,202,2017-02-04 11:19:35,http://macgyver.name/elroy,0.0521492665,74.222.229.69,"{""location"": ""CU"", ""is_mobile"": true}" 8989,4,202,2017-04-13 05:06:24,http://lakin.net/frank_emard,0.6224848020,246.179.166.135,"{""location"": ""PH"", ""is_mobile"": false}" 5926,3,129,2017-06-03 06:10:05,http://brakus.biz/keven,,198.2.7.95,"{""location"": ""MT"", ""is_mobile"": true}" 5927,3,129,2017-05-15 12:32:25,http://oberbrunner.name/abigail_kunze,,197.53.209.107,"{""location"": ""TO"", ""is_mobile"": false}" 5928,3,129,2017-01-09 12:56:44,http://dooley.org/austyn,,158.168.104.53,"{""location"": ""CL"", ""is_mobile"": true}" 5929,3,129,2017-03-13 11:03:43,http://rice.co/anastasia,,78.190.208.75,"{""location"": ""IM"", ""is_mobile"": true}" 5930,3,129,2017-01-29 20:43:59,http://gutkowskiupton.net/hayden_hermiston,,98.245.165.244,"{""location"": ""ID"", ""is_mobile"": false}" 5931,3,129,2017-03-08 23:38:30,http://tromp.org/lew,,100.48.125.72,"{""location"": ""SL"", ""is_mobile"": true}" 5932,3,129,2017-04-22 17:38:39,http://stiedemannkling.io/candido.kunde,,61.212.99.144,"{""location"": ""LI"", ""is_mobile"": true}" 5933,3,129,2017-03-05 06:08:02,http://armstrong.io/nicholas_cremin,,152.191.40.195,"{""location"": ""SM"", ""is_mobile"": true}" 5934,3,129,2017-01-30 05:07:18,http://bernierruel.io/fredrick,,144.142.204.169,"{""location"": ""SD"", ""is_mobile"": true}" 5935,3,129,2017-06-01 06:07:41,http://kirlin.org/winfield,,172.232.148.208,"{""location"": ""CY"", ""is_mobile"": true}" 5936,3,129,2017-04-15 09:20:35,http://cainharris.io/cali.miller,,38.33.26.96,"{""location"": ""PY"", ""is_mobile"": false}" 5937,3,129,2017-02-12 18:17:59,http://carroll.biz/mitchell,,131.54.76.24,"{""location"": ""IR"", ""is_mobile"": true}" 5938,3,129,2016-12-30 21:51:19,http://hoeger.net/markus,,24.138.154.4,"{""location"": ""JM"", ""is_mobile"": true}" 5939,3,129,2017-04-08 05:19:45,http://labadieklocko.org/ubaldo.nader,,149.230.191.118,"{""location"": ""BT"", ""is_mobile"": true}" 5940,3,129,2017-04-22 13:05:49,http://price.name/giovanni_buckridge,,231.217.5.229,"{""location"": ""AZ"", ""is_mobile"": true}" 5941,3,129,2017-01-12 22:49:55,http://baumbachcorwin.co/alfonzo,,32.179.33.4,"{""location"": ""HT"", ""is_mobile"": false}" 5942,3,129,2017-01-24 23:36:34,http://boyer.name/kamren.zieme,,215.203.184.70,"{""location"": ""LI"", ""is_mobile"": false}" 5943,3,129,2017-03-03 21:11:06,http://hickle.org/bria,,62.73.42.210,"{""location"": ""NA"", ""is_mobile"": false}" 5944,3,129,2017-06-01 20:53:39,http://kleinschimmel.io/hugh.gusikowski,,165.213.138.97,"{""location"": ""CH"", ""is_mobile"": true}" 5945,3,129,2017-04-15 23:53:20,http://sawayn.biz/hans_medhurst,,142.236.151.91,"{""location"": ""GB"", ""is_mobile"": true}" 5946,3,129,2017-01-08 21:40:02,http://lindgren.com/cayla,,254.123.30.132,"{""location"": ""AM"", ""is_mobile"": true}" 5947,3,129,2017-01-19 02:37:03,http://schadendoyle.io/wilmer.wilkinson,,75.29.4.229,"{""location"": ""ZA"", ""is_mobile"": true}" 5948,3,129,2017-06-08 12:41:25,http://larkin.com/max_sauer,,78.94.13.117,"{""location"": ""MR"", ""is_mobile"": false}" 5949,3,129,2017-03-15 13:12:10,http://champlinabernathy.biz/arthur,,60.249.150.31,"{""location"": ""LS"", ""is_mobile"": false}" 5950,3,129,2017-04-17 22:24:07,http://ortiz.net/idella,,2.50.57.212,"{""location"": ""BR"", ""is_mobile"": true}" 5951,3,129,2017-04-22 12:00:42,http://kohler.info/golda,,238.211.80.219,"{""location"": ""VA"", ""is_mobile"": true}" 5952,3,129,2017-03-17 14:03:47,http://barton.io/jeanie,,239.215.176.125,"{""location"": ""JM"", ""is_mobile"": false}" 5953,3,129,2017-03-28 16:20:22,http://smith.name/ozella,,27.219.165.15,"{""location"": ""HU"", ""is_mobile"": false}" 5954,3,129,2017-05-23 18:24:29,http://adams.io/gerry,,72.83.41.29,"{""location"": ""DE"", ""is_mobile"": false}" 5955,3,129,2017-05-29 06:55:40,http://cristokuneva.name/kiara,,125.246.196.90,"{""location"": ""TJ"", ""is_mobile"": true}" 5956,3,129,2017-03-17 06:26:04,http://berge.info/delia,,160.126.4.21,"{""location"": ""ZA"", ""is_mobile"": true}" 5957,3,129,2017-01-05 13:37:41,http://torp.com/rodolfo.dicki,,81.237.52.84,"{""location"": ""MY"", ""is_mobile"": true}" 5958,3,129,2017-06-05 07:49:10,http://braun.biz/scot_hane,,63.105.24.208,"{""location"": ""CN"", ""is_mobile"": false}" 5959,3,129,2016-12-13 08:22:14,http://kuhn.io/jovanny_konopelski,,143.3.135.235,"{""location"": ""ST"", ""is_mobile"": false}" 5960,3,129,2017-05-05 03:51:47,http://morar.biz/angeline,,51.184.6.138,"{""location"": ""RS"", ""is_mobile"": true}" 5961,3,129,2017-01-11 17:36:29,http://windler.co/arden.oberbrunner,,14.171.59.219,"{""location"": ""CA"", ""is_mobile"": false}" 5962,3,129,2017-06-01 20:56:14,http://kirlindach.name/jarret_sipes,,17.223.39.132,"{""location"": ""KM"", ""is_mobile"": true}" 5963,3,129,2016-12-16 04:23:42,http://jacobi.org/leora,,47.114.178.124,"{""location"": ""GW"", ""is_mobile"": false}" 5964,3,129,2017-03-14 16:35:21,http://pagacgreen.biz/jaydon.boyer,,243.130.209.108,"{""location"": ""BT"", ""is_mobile"": false}" 5965,3,129,2017-06-13 10:49:52,http://wolff.net/emmet,,183.53.168.11,"{""location"": ""MV"", ""is_mobile"": true}" 5966,3,129,2017-04-23 01:35:34,http://gibsonhermiston.co/marquise_corwin,,226.67.77.219,"{""location"": ""SO"", ""is_mobile"": false}" 5967,3,129,2017-02-25 03:25:14,http://west.net/naomie,,239.228.79.152,"{""location"": ""FJ"", ""is_mobile"": true}" 5968,3,129,2017-02-28 07:35:11,http://rippin.co/demond,,113.250.78.227,"{""location"": ""AR"", ""is_mobile"": true}" 5969,3,129,2017-05-03 11:34:25,http://wintheiser.name/chris_kilback,,85.233.184.237,"{""location"": ""MC"", ""is_mobile"": false}" 5970,3,129,2017-05-29 13:08:43,http://koepp.net/cora,,146.168.191.142,"{""location"": ""IM"", ""is_mobile"": false}" 5971,3,129,2017-03-03 21:16:53,http://kaulkewill.com/jerel,,156.17.19.231,"{""location"": ""PA"", ""is_mobile"": false}" 5972,3,129,2017-05-11 09:55:05,http://gottlieb.info/christy,,205.98.206.237,"{""location"": ""BT"", ""is_mobile"": true}" 5973,3,129,2017-05-26 16:15:51,http://fisherorn.io/kevon.little,,103.199.48.8,"{""location"": ""LA"", ""is_mobile"": true}" 5974,3,129,2017-01-22 01:53:11,http://mante.net/christian_reinger,,119.41.194.44,"{""location"": ""ES"", ""is_mobile"": false}" 5975,3,129,2017-03-19 09:04:02,http://dietrich.info/cornell_lesch,,152.253.178.190,"{""location"": ""TJ"", ""is_mobile"": false}" 5976,3,129,2017-02-02 06:39:54,http://hansen.name/raphael,,165.198.5.172,"{""location"": ""KI"", ""is_mobile"": false}" 5977,3,129,2017-05-22 20:48:30,http://hermistongulgowski.biz/sebastian,,27.235.235.131,"{""location"": ""JE"", ""is_mobile"": false}" 5978,3,129,2017-05-01 01:27:45,http://maggiolehner.org/tyrese,,214.72.36.110,"{""location"": ""YT"", ""is_mobile"": true}" 5979,3,130,2017-03-21 21:25:39,http://harris.org/lucienne.skiles,,88.152.130.202,"{""location"": ""IN"", ""is_mobile"": true}" 5980,3,130,2017-02-28 05:59:19,http://mertzpfannerstill.io/michele,,41.159.202.35,"{""location"": ""SD"", ""is_mobile"": false}" 5981,3,130,2017-02-09 08:12:06,http://wolfrath.net/joel_kuvalis,,164.69.201.39,"{""location"": ""MA"", ""is_mobile"": false}" 11853,5,265,2017-02-19 09:50:27,http://pacochawaelchi.info/briana.durgan,0.5202608552,124.94.41.14,"{""location"": ""KW"", ""is_mobile"": false}" 11854,5,265,2017-03-04 19:17:08,http://mannferry.info/boyd_leffler,0.1254577793,210.101.180.90,"{""location"": ""SK"", ""is_mobile"": true}" 11855,5,265,2017-02-03 09:47:31,http://bradtkewelch.org/sherwood_ohara,0.4146630130,88.53.3.32,"{""location"": ""GM"", ""is_mobile"": false}" 11856,5,265,2017-03-17 22:23:01,http://heaney.biz/nannie.bashirian,0.8082636923,89.146.186.107,"{""location"": ""GE"", ""is_mobile"": false}" 11857,5,265,2017-05-16 03:39:50,http://trompjohns.biz/evie,0.7063900754,206.207.38.226,"{""location"": ""MQ"", ""is_mobile"": true}" 11858,5,265,2017-01-04 09:14:34,http://reilly.biz/benton,0.2344035831,71.218.16.77,"{""location"": ""SR"", ""is_mobile"": false}" 11859,5,265,2017-04-03 17:18:09,http://pfannerstillcruickshank.name/monserrat_larson,0.8732617190,171.237.41.61,"{""location"": ""WF"", ""is_mobile"": true}" 11860,5,265,2016-12-25 21:07:54,http://gerhold.net/ola,0.3029357011,235.134.252.46,"{""location"": ""CI"", ""is_mobile"": false}" 11861,5,265,2016-12-25 16:21:38,http://kiehn.biz/bernadine,0.5869571235,161.233.38.14,"{""location"": ""EH"", ""is_mobile"": false}" 11862,5,265,2017-04-14 07:01:29,http://kuvaliskertzmann.net/aryanna,0.2781810805,33.71.216.109,"{""location"": ""CF"", ""is_mobile"": true}" 11863,5,265,2017-01-18 02:03:12,http://lindreichel.co/jaida,0.3332937234,54.110.216.11,"{""location"": ""TL"", ""is_mobile"": false}" 11864,5,265,2017-01-16 23:19:54,http://kris.com/madalyn,0.0184447562,156.178.99.112,"{""location"": ""MX"", ""is_mobile"": true}" 11865,5,265,2017-05-08 07:59:01,http://rathbahringer.co/dayton,0.5825414017,120.109.185.19,"{""location"": ""CN"", ""is_mobile"": true}" 11866,5,265,2017-03-06 04:11:09,http://runolfon.org/bridget,0.2113803616,249.34.95.56,"{""location"": ""TM"", ""is_mobile"": true}" 11867,5,265,2017-01-17 07:17:16,http://kunze.net/rebekah.zulauf,0.0540986531,147.184.38.206,"{""location"": ""FR"", ""is_mobile"": true}" 11868,5,265,2017-06-04 18:38:20,http://stiedemann.org/ivah_hintz,0.5909273565,79.92.202.124,"{""location"": ""GU"", ""is_mobile"": true}" 11869,5,265,2017-01-16 22:20:12,http://macejkovic.info/adelia,0.2741205832,78.33.99.52,"{""location"": ""LC"", ""is_mobile"": true}" 11870,5,265,2017-02-21 04:16:13,http://weimann.name/blaze,0.3272367807,126.246.183.11,"{""location"": ""NA"", ""is_mobile"": true}" 11871,5,265,2017-03-25 09:47:37,http://casperschmitt.org/asha_schultz,0.1036787605,202.222.54.153,"{""location"": ""MT"", ""is_mobile"": false}" 11872,5,266,2017-05-24 04:55:09,http://bechtelar.net/jalon,0.3533988179,54.100.79.148,"{""location"": ""UY"", ""is_mobile"": true}" 11873,5,266,2016-12-24 15:22:32,http://hintz.co/kamille,0.5197254803,205.218.20.118,"{""location"": ""KR"", ""is_mobile"": false}" 11874,5,266,2017-05-17 09:05:22,http://spencerwalsh.info/maximillian,0.9826143939,102.168.210.52,"{""location"": ""TR"", ""is_mobile"": false}" 11875,5,266,2017-04-26 01:07:01,http://tremblay.co/chadd,0.1106894504,109.231.171.166,"{""location"": ""AU"", ""is_mobile"": false}" 11876,5,266,2017-03-07 05:20:57,http://simonis.io/yoshiko,0.0842173598,245.242.9.195,"{""location"": ""KP"", ""is_mobile"": false}" 11877,5,266,2017-01-28 09:31:53,http://auer.org/anais_moore,0.1652019584,177.243.231.136,"{""location"": ""NL"", ""is_mobile"": false}" 11878,5,266,2017-01-20 13:59:31,http://bins.co/libby,0.9688937562,197.143.127.108,"{""location"": ""ST"", ""is_mobile"": false}" 11879,5,266,2017-03-07 10:17:09,http://bruen.info/pinkie,0.9117557075,24.145.147.50,"{""location"": ""SE"", ""is_mobile"": false}" 11880,5,266,2017-01-30 04:00:59,http://fay.net/bonita,0.1626124429,230.206.230.241,"{""location"": ""RW"", ""is_mobile"": false}" 11881,5,266,2017-04-24 08:40:05,http://blickrodriguez.co/andrew,0.2173005821,185.33.211.161,"{""location"": ""UZ"", ""is_mobile"": true}" 11882,5,266,2017-01-27 19:22:15,http://hintz.org/danielle.wolf,0.6722116571,203.234.173.82,"{""location"": ""CG"", ""is_mobile"": false}" 11883,5,266,2017-04-29 17:12:31,http://ruecker.co/amos_shanahan,0.5494990482,196.135.94.71,"{""location"": ""GB"", ""is_mobile"": false}" 11884,5,266,2017-01-20 12:09:57,http://volkmanmurphy.co/merritt,0.2760339899,135.106.101.25,"{""location"": ""KE"", ""is_mobile"": true}" 11885,5,266,2017-01-29 19:16:52,http://lebsack.com/douglas,0.5645589625,42.173.5.212,"{""location"": ""IQ"", ""is_mobile"": true}" 11886,5,266,2017-01-15 03:15:04,http://schamberger.net/karli,0.2529970953,71.41.213.81,"{""location"": ""JP"", ""is_mobile"": false}" 11887,5,266,2017-05-06 02:33:13,http://hackett.info/magdalen,0.9760795290,146.31.33.247,"{""location"": ""SG"", ""is_mobile"": false}" 11888,5,266,2017-05-31 04:07:38,http://cartwright.net/shaun,0.4154398525,248.38.79.91,"{""location"": ""AZ"", ""is_mobile"": true}" 11889,5,266,2016-12-18 10:31:51,http://bernierpagac.com/lenny,0.6750198749,14.17.205.72,"{""location"": ""RE"", ""is_mobile"": true}" 11890,5,266,2017-02-25 08:27:55,http://labadiemaggio.biz/tom_dibbert,0.2625525453,53.134.183.87,"{""location"": ""HN"", ""is_mobile"": false}" 11891,5,266,2017-04-14 16:20:17,http://ziemannkoch.biz/shad_gusikowski,0.9207830607,82.91.128.159,"{""location"": ""PL"", ""is_mobile"": false}" 11892,5,266,2017-01-25 07:08:18,http://littleskiles.name/paris,0.3513102021,169.60.254.52,"{""location"": ""PG"", ""is_mobile"": false}" 11893,5,266,2017-06-01 13:30:14,http://beahan.net/pablo_stehr,0.1118792376,63.110.183.74,"{""location"": ""TZ"", ""is_mobile"": true}" 11894,5,266,2017-04-13 13:59:48,http://harbergerlach.org/tanya,0.7964352896,254.12.31.69,"{""location"": ""WS"", ""is_mobile"": true}" 11895,5,266,2017-06-13 00:16:47,http://mcdermott.net/taurean.jacobi,0.3393810357,203.205.207.228,"{""location"": ""PK"", ""is_mobile"": false}" 11896,5,266,2017-05-12 05:23:22,http://pollich.name/jermaine.bosco,0.3619880089,46.147.21.116,"{""location"": ""YE"", ""is_mobile"": false}" 11897,5,266,2017-06-10 21:56:45,http://emmerich.org/dejon,0.8222048029,191.153.239.203,"{""location"": ""SO"", ""is_mobile"": false}" 11898,5,266,2016-12-25 11:35:55,http://armstrong.com/eden,0.7952813383,82.94.196.146,"{""location"": ""TZ"", ""is_mobile"": true}" 11899,5,266,2017-06-01 13:51:30,http://cainwisozk.info/mireya.koelpin,0.1580573980,196.120.226.78,"{""location"": ""TV"", ""is_mobile"": false}" 11900,5,266,2016-12-26 07:33:07,http://schaefer.co/bernie,0.3730441288,118.72.240.110,"{""location"": ""SA"", ""is_mobile"": false}" 11901,5,266,2017-05-10 16:50:06,http://franecki.info/shanny,0.3363766351,4.240.102.26,"{""location"": ""YE"", ""is_mobile"": false}" 11902,5,266,2017-02-14 03:53:27,http://kilback.co/jeramy.fadel,0.5191199105,57.240.224.120,"{""location"": ""GQ"", ""is_mobile"": true}" 11903,5,266,2017-05-01 17:19:32,http://macgyver.info/dasia_boyle,0.9510133266,81.208.5.237,"{""location"": ""KH"", ""is_mobile"": false}" 11904,5,266,2017-03-27 22:54:17,http://leuschkezemlak.org/arjun,0.8991342255,72.99.228.35,"{""location"": ""CI"", ""is_mobile"": false}" 8990,4,202,2017-06-05 05:56:59,http://willmsratke.name/roslyn,0.0858122439,120.193.145.38,"{""location"": ""MC"", ""is_mobile"": false}" 8991,4,202,2017-02-03 11:23:05,http://prosaccokreiger.com/carmine,0.6440021014,37.164.144.118,"{""location"": ""CM"", ""is_mobile"": true}" 8992,4,202,2016-12-16 20:38:38,http://williamsonflatley.biz/syble,0.4136741186,55.41.79.194,"{""location"": ""AO"", ""is_mobile"": false}" 8993,4,202,2017-06-05 12:49:55,http://abernathy.com/lela,0.5590893278,20.209.176.179,"{""location"": ""SN"", ""is_mobile"": false}" 8994,4,202,2016-12-25 14:17:50,http://parisianhintz.com/jaylan.hackett,0.7871529418,36.124.65.42,"{""location"": ""MR"", ""is_mobile"": true}" 8995,4,202,2017-03-04 17:42:18,http://hamilltreutel.com/ahmad,0.2901478944,201.11.237.191,"{""location"": ""ID"", ""is_mobile"": false}" 8996,4,202,2017-06-05 08:59:12,http://veum.com/reid,0.0919036992,8.22.66.158,"{""location"": ""TT"", ""is_mobile"": false}" 8997,4,202,2016-12-15 04:58:57,http://kerlukeprosacco.io/erika_vandervort,0.7384921110,68.134.104.32,"{""location"": ""TJ"", ""is_mobile"": false}" 8998,4,202,2017-03-05 19:29:26,http://treutelmills.net/caie_purdy,0.2232696100,151.84.82.96,"{""location"": ""AG"", ""is_mobile"": false}" 8999,4,202,2017-05-12 11:28:06,http://goodwin.net/asia.herman,0.0418368089,254.122.53.200,"{""location"": ""VA"", ""is_mobile"": true}" 9000,4,202,2017-01-18 08:22:30,http://bechtelarlockman.io/maximillia,0.6891395862,136.105.209.153,"{""location"": ""IQ"", ""is_mobile"": true}" 9001,4,202,2017-04-08 07:54:55,http://strosin.co/modesto,0.0051938671,195.179.227.238,"{""location"": ""EG"", ""is_mobile"": false}" 9002,4,202,2017-02-09 16:48:55,http://oreilly.co/ebony.dare,0.6468116459,157.122.6.156,"{""location"": ""PN"", ""is_mobile"": true}" 9003,4,202,2017-01-05 19:39:36,http://spinka.name/rubye,0.9334324632,215.196.126.164,"{""location"": ""FJ"", ""is_mobile"": true}" 9004,4,202,2016-12-20 08:49:46,http://sporerfeeney.name/margot.fritsch,0.8036648590,167.166.83.232,"{""location"": ""RS"", ""is_mobile"": false}" 9005,4,202,2017-04-08 16:16:05,http://larson.info/sadie.conroy,0.0298865630,199.209.162.73,"{""location"": ""SS"", ""is_mobile"": true}" 9549,4,214,2017-02-17 13:18:14,http://boehm.info/cortez,,91.177.191.144,"{""location"": ""BE"", ""is_mobile"": true}" 9006,4,202,2017-04-22 09:01:24,http://howebreitenberg.com/helga,0.1746734646,50.162.125.177,"{""location"": ""ET"", ""is_mobile"": false}" 9007,4,202,2017-05-27 04:17:17,http://medhurstadams.net/tamara.kemmer,0.6822859470,119.35.170.116,"{""location"": ""BR"", ""is_mobile"": false}" 9008,4,202,2017-02-06 13:25:16,http://mclaughlin.io/myrtis.legros,0.2875759054,77.115.5.240,"{""location"": ""MD"", ""is_mobile"": true}" 9009,4,202,2017-01-09 21:35:13,http://oreilly.name/akeem.oconner,0.1857670898,194.81.146.60,"{""location"": ""TH"", ""is_mobile"": false}" 9010,4,202,2017-06-12 15:46:07,http://zemlakhoppe.co/stephany,0.3621109135,70.133.69.213,"{""location"": ""ZA"", ""is_mobile"": true}" 9011,4,202,2017-04-30 07:28:53,http://reynolds.com/virgil.streich,0.4175791383,112.5.72.68,"{""location"": ""MN"", ""is_mobile"": false}" 9012,4,202,2016-12-13 12:06:52,http://conn.info/cristal,0.3976183111,31.253.54.32,"{""location"": ""MD"", ""is_mobile"": true}" 9013,4,202,2016-12-18 00:17:51,http://grant.biz/tyrese,0.5287466133,122.148.66.164,"{""location"": ""KG"", ""is_mobile"": true}" 9014,4,202,2017-06-11 20:59:03,http://oberbrunner.io/garret_bins,0.9446460917,25.125.125.54,"{""location"": ""SN"", ""is_mobile"": true}" 9015,4,202,2017-03-28 16:28:47,http://wunsch.io/iva.reichert,0.4994488150,86.49.144.137,"{""location"": ""CO"", ""is_mobile"": false}" 9016,4,202,2017-05-13 02:38:00,http://rosenbaumstamm.io/aliya,0.2171280144,109.238.197.52,"{""location"": ""NE"", ""is_mobile"": true}" 9017,4,202,2017-03-24 03:13:26,http://bauch.biz/mary_schroeder,0.2750063718,115.83.8.124,"{""location"": ""NC"", ""is_mobile"": false}" 9018,4,202,2016-12-17 02:48:08,http://mertz.io/durward,0.9037435748,226.23.244.9,"{""location"": ""MX"", ""is_mobile"": false}" 9019,4,202,2017-02-19 12:59:30,http://kirlincartwright.com/alta,0.6815282558,8.85.62.57,"{""location"": ""KR"", ""is_mobile"": true}" 9020,4,202,2017-06-08 18:01:51,http://mertz.io/felicia,0.3851121654,68.123.243.250,"{""location"": ""AO"", ""is_mobile"": true}" 9021,4,202,2017-04-20 07:44:27,http://hahn.biz/modesto,0.6118891850,66.221.244.132,"{""location"": ""TF"", ""is_mobile"": false}" 9022,4,202,2017-01-04 10:26:45,http://ortiz.info/tanner,0.9417757011,63.247.223.146,"{""location"": ""MO"", ""is_mobile"": true}" 9023,4,202,2017-01-09 09:41:11,http://trompkemmer.biz/margarita_barton,0.7242031121,22.192.134.146,"{""location"": ""MQ"", ""is_mobile"": true}" 9024,4,202,2017-04-23 05:25:57,http://wilderman.name/hope.yost,0.2887292067,210.45.105.128,"{""location"": ""AX"", ""is_mobile"": false}" 9025,4,202,2017-01-27 23:41:17,http://willmsbayer.io/noe_witting,0.8025460902,111.204.222.63,"{""location"": ""HK"", ""is_mobile"": false}" 9026,4,202,2017-06-01 00:21:20,http://upton.info/gerard_mcclure,0.4922519929,163.16.42.209,"{""location"": ""WF"", ""is_mobile"": false}" 9027,4,202,2017-04-08 14:01:38,http://wilkinsonshanahan.org/jerrell,0.9900998673,197.147.220.50,"{""location"": ""CR"", ""is_mobile"": true}" 9028,4,202,2017-01-16 07:19:45,http://pacochatorphy.biz/stephon,0.1649935122,205.73.160.229,"{""location"": ""AW"", ""is_mobile"": false}" 9029,4,202,2016-12-19 19:38:22,http://ondricka.org/benedict_grant,0.7482612622,24.71.70.149,"{""location"": ""DM"", ""is_mobile"": false}" 9030,4,202,2017-01-28 14:21:10,http://haley.org/idell_kuhlman,0.6853021437,222.232.104.202,"{""location"": ""KH"", ""is_mobile"": true}" 9031,4,202,2016-12-27 01:38:28,http://zieme.name/rozella,0.6932023612,198.99.85.131,"{""location"": ""SG"", ""is_mobile"": true}" 9032,4,203,2017-05-16 08:04:05,http://veum.io/ryann.rau,0.0167704913,195.94.217.217,"{""location"": ""SL"", ""is_mobile"": false}" 9033,4,203,2017-02-09 01:49:00,http://leuschke.io/obie_rutherford,0.2399798459,75.35.151.64,"{""location"": ""CK"", ""is_mobile"": false}" 9034,4,203,2017-01-31 00:03:57,http://strosin.biz/jaron,0.1635135243,190.103.31.203,"{""location"": ""GD"", ""is_mobile"": false}" 9035,4,203,2016-12-29 23:56:33,http://schaefer.info/jaqueline,0.8630221451,196.116.253.85,"{""location"": ""TW"", ""is_mobile"": false}" 9036,4,203,2017-02-04 01:54:32,http://gulgowski.org/aurelie.harris,0.1782974629,156.9.64.18,"{""location"": ""RU"", ""is_mobile"": false}" 9037,4,203,2017-01-06 20:04:49,http://stamm.io/joanne_bins,0.8916993249,125.192.192.22,"{""location"": ""SC"", ""is_mobile"": false}" 9038,4,203,2017-06-08 02:36:22,http://bednar.com/rylee_dubuque,0.1266266732,249.150.237.139,"{""location"": ""VI"", ""is_mobile"": false}" 9039,4,203,2017-02-19 02:31:04,http://wehner.io/coty,0.7731078093,69.36.104.219,"{""location"": ""BS"", ""is_mobile"": false}" 5982,3,130,2017-05-09 22:58:19,http://braun.info/elmer,,48.30.229.215,"{""location"": ""NO"", ""is_mobile"": false}" 5983,3,130,2017-05-30 20:14:40,http://bahringer.net/ayden,,158.131.49.93,"{""location"": ""UZ"", ""is_mobile"": false}" 5984,3,130,2017-01-02 14:50:35,http://wiegandstokes.name/alejandra,,177.54.112.219,"{""location"": ""LC"", ""is_mobile"": true}" 5985,3,130,2017-02-25 03:44:53,http://monahan.net/lillie.flatley,,80.54.26.152,"{""location"": ""TF"", ""is_mobile"": true}" 5986,3,130,2017-05-11 13:39:27,http://dibbertdaugherty.biz/nathanael,,232.176.156.206,"{""location"": ""SE"", ""is_mobile"": true}" 5987,3,130,2017-01-19 20:52:07,http://howellyost.name/roselyn.greenholt,,213.158.140.209,"{""location"": ""KI"", ""is_mobile"": false}" 5988,3,130,2017-03-19 15:14:38,http://moorebode.io/bobbie,,159.225.19.24,"{""location"": ""YE"", ""is_mobile"": true}" 5989,3,130,2017-01-18 22:19:59,http://zemlakweber.info/houston,,216.34.229.173,"{""location"": ""ZM"", ""is_mobile"": true}" 5990,3,130,2017-04-27 08:39:22,http://strosin.info/dominique,,254.173.171.74,"{""location"": ""LC"", ""is_mobile"": true}" 5991,3,130,2017-04-03 21:43:01,http://goyetteruel.biz/broderick,,115.204.176.89,"{""location"": ""AR"", ""is_mobile"": true}" 5992,3,130,2017-02-16 10:07:24,http://bergstromschmeler.io/van,,118.200.46.133,"{""location"": ""ID"", ""is_mobile"": false}" 5993,3,130,2017-02-19 17:49:21,http://roberts.info/bert,,198.224.5.151,"{""location"": ""PH"", ""is_mobile"": false}" 5994,3,130,2017-01-11 12:44:12,http://harrishyatt.name/wyman,,211.114.109.56,"{""location"": ""DJ"", ""is_mobile"": true}" 5995,3,130,2017-05-11 10:07:47,http://wittingdamore.name/bud_rath,,207.183.44.217,"{""location"": ""YT"", ""is_mobile"": false}" 5996,3,130,2016-12-28 03:26:08,http://schaden.io/reanna.renner,,242.52.221.244,"{""location"": ""FR"", ""is_mobile"": false}" 5997,3,130,2017-01-24 08:55:52,http://mckenziewintheiser.net/euna.grant,,41.141.26.249,"{""location"": ""AI"", ""is_mobile"": true}" 5998,3,130,2017-06-06 17:55:49,http://schuppekozey.info/hollie.kling,,178.222.46.111,"{""location"": ""NI"", ""is_mobile"": true}" 5999,3,130,2017-05-01 12:23:57,http://wuckert.co/phoebe,,219.243.74.128,"{""location"": ""WS"", ""is_mobile"": false}" 6000,3,130,2017-05-08 02:17:16,http://barrowhields.name/avis,,210.218.224.93,"{""location"": ""AI"", ""is_mobile"": false}" 6001,3,130,2017-03-24 23:30:36,http://cummingsroob.co/mireya.rodriguez,,196.107.162.232,"{""location"": ""CN"", ""is_mobile"": false}" 6002,3,130,2017-06-06 19:32:38,http://whitekeler.biz/estevan.block,,174.241.253.18,"{""location"": ""UA"", ""is_mobile"": false}" 6003,3,130,2017-03-06 19:33:10,http://predovicgottlieb.biz/claude.king,,221.29.144.166,"{""location"": ""FJ"", ""is_mobile"": true}" 6004,3,130,2017-02-24 12:22:52,http://leuschke.biz/reese,,34.145.27.166,"{""location"": ""BD"", ""is_mobile"": false}" 6005,3,130,2017-05-10 07:27:24,http://wehner.org/samara_kris,,132.140.148.97,"{""location"": ""PK"", ""is_mobile"": false}" 6006,3,130,2017-04-04 12:22:50,http://hicklevolkman.biz/ottis,,218.205.249.198,"{""location"": ""KY"", ""is_mobile"": false}" 6007,3,130,2017-03-24 22:51:25,http://upton.io/leonie.gerlach,,142.115.219.164,"{""location"": ""HT"", ""is_mobile"": false}" 6008,3,130,2017-04-26 23:32:40,http://hoppe.org/dolores,,36.102.37.137,"{""location"": ""MK"", ""is_mobile"": false}" 6009,3,130,2017-05-21 07:10:55,http://dicki.name/emmie_beahan,,98.231.68.157,"{""location"": ""BB"", ""is_mobile"": false}" 6010,3,130,2017-03-17 21:39:27,http://turner.io/angelica,,159.126.57.66,"{""location"": ""KW"", ""is_mobile"": true}" 6011,3,130,2017-01-22 12:08:48,http://harvey.name/roger,,173.39.137.85,"{""location"": ""ML"", ""is_mobile"": false}" 6012,3,130,2017-02-03 21:27:36,http://boscomurray.biz/moshe,,79.129.36.170,"{""location"": ""ML"", ""is_mobile"": false}" 6013,3,130,2017-04-20 01:50:35,http://bechtelarwalker.org/velda,,217.36.85.81,"{""location"": ""OM"", ""is_mobile"": true}" 6014,3,130,2017-03-30 18:38:31,http://dach.com/grace,,239.188.218.193,"{""location"": ""MS"", ""is_mobile"": true}" 6015,3,131,2017-03-11 05:11:49,http://treutelkoch.biz/adolphus,,112.209.69.63,"{""location"": ""PH"", ""is_mobile"": false}" 6016,3,131,2017-01-29 03:50:35,http://abernathy.io/billie,,43.87.66.46,"{""location"": ""VE"", ""is_mobile"": false}" 6017,3,131,2017-05-11 19:43:19,http://lind.biz/bradley,,181.92.136.30,"{""location"": ""MW"", ""is_mobile"": false}" 6018,3,131,2017-05-14 14:43:12,http://faydicki.name/jodie.davis,,23.6.196.63,"{""location"": ""HN"", ""is_mobile"": false}" 6019,3,131,2017-01-28 07:14:51,http://harvey.org/cristina,,149.44.193.213,"{""location"": ""BB"", ""is_mobile"": false}" 6020,3,131,2017-03-26 21:25:13,http://boehm.io/quinn_frami,,190.111.202.213,"{""location"": ""NP"", ""is_mobile"": true}" 6021,3,131,2017-03-15 08:14:18,http://stiedemannhomenick.net/guy,,56.67.3.98,"{""location"": ""GF"", ""is_mobile"": false}" 6022,3,131,2017-02-02 21:08:48,http://marks.com/alanis,,98.229.159.211,"{""location"": ""NU"", ""is_mobile"": false}" 6023,3,131,2017-05-08 02:22:09,http://labadie.net/jaylin.pacocha,,197.94.159.119,"{""location"": ""TG"", ""is_mobile"": false}" 6024,3,131,2017-03-13 07:13:49,http://donnelly.io/tyrese,,102.47.143.39,"{""location"": ""DJ"", ""is_mobile"": false}" 6025,3,131,2017-05-10 21:04:05,http://turcotte.name/tamara_koch,,23.86.247.148,"{""location"": ""ME"", ""is_mobile"": false}" 6026,3,131,2017-04-24 05:12:19,http://will.info/arlene,,239.12.118.148,"{""location"": ""BJ"", ""is_mobile"": true}" 6027,3,131,2017-01-11 22:19:55,http://hickleanderson.co/abigail.kreiger,,73.50.23.80,"{""location"": ""GL"", ""is_mobile"": false}" 6028,3,131,2017-06-12 02:18:28,http://cremin.com/celestino,,181.185.154.6,"{""location"": ""NE"", ""is_mobile"": false}" 6029,3,131,2017-06-05 14:57:21,http://johnson.info/tyshawn.watsica,,87.68.195.35,"{""location"": ""IL"", ""is_mobile"": true}" 6030,3,131,2017-05-04 23:38:55,http://price.co/henri_emard,,249.141.117.124,"{""location"": ""GI"", ""is_mobile"": true}" 6031,3,131,2017-03-08 18:18:44,http://hand.net/katherine_gaylord,,149.133.43.209,"{""location"": ""SX"", ""is_mobile"": false}" 6032,3,131,2017-04-17 06:23:25,http://nienow.net/archibald.altenwerth,,150.231.205.225,"{""location"": ""NR"", ""is_mobile"": true}" 6033,3,131,2017-05-31 22:18:56,http://senger.net/hadley_carroll,,167.225.155.74,"{""location"": ""YT"", ""is_mobile"": true}" 6034,3,131,2017-05-06 20:13:51,http://balistrerilockman.net/ruby,,161.155.3.236,"{""location"": ""FK"", ""is_mobile"": false}" 6035,3,131,2017-05-08 13:19:44,http://schmelerconn.co/buddy.prohaska,,113.97.29.2,"{""location"": ""TN"", ""is_mobile"": false}" 6036,3,131,2017-02-17 00:05:54,http://kuhn.net/emory.oreilly,,15.212.43.21,"{""location"": ""AQ"", ""is_mobile"": true}" 6037,3,131,2017-01-09 21:26:42,http://tremblay.biz/herbert,,109.12.47.27,"{""location"": ""BN"", ""is_mobile"": true}" 11905,5,266,2017-02-04 05:14:26,http://stroman.biz/tyree,0.4045663378,51.161.154.182,"{""location"": ""EC"", ""is_mobile"": false}" 11906,5,266,2017-04-13 11:27:43,http://oreilly.co/claude,0.7786988546,232.221.160.235,"{""location"": ""NG"", ""is_mobile"": false}" 11907,5,266,2017-05-18 16:09:00,http://dibbert.biz/aliza,0.9521876302,51.175.69.208,"{""location"": ""LC"", ""is_mobile"": true}" 11908,5,266,2017-04-24 18:41:39,http://doyle.com/norval,0.7110095699,95.198.42.52,"{""location"": ""AQ"", ""is_mobile"": false}" 11909,5,266,2017-03-29 05:05:07,http://prosacco.co/levi_ledner,0.6947770379,37.145.96.48,"{""location"": ""GM"", ""is_mobile"": true}" 11910,5,266,2017-04-01 11:00:16,http://smith.co/josue,0.5399709413,74.205.48.10,"{""location"": ""NZ"", ""is_mobile"": false}" 11911,5,266,2017-02-20 14:43:17,http://lindgrenskiles.org/murphy.keeling,0.6076513774,244.79.250.44,"{""location"": ""MX"", ""is_mobile"": false}" 11912,5,266,2017-05-14 01:09:25,http://collier.co/rachelle,0.1157078916,71.57.237.135,"{""location"": ""GS"", ""is_mobile"": true}" 11913,5,266,2017-04-30 08:02:06,http://kris.name/westley.spencer,0.7217221606,145.27.239.118,"{""location"": ""SK"", ""is_mobile"": true}" 11914,5,266,2017-06-12 07:11:46,http://rice.biz/elton,0.0920499357,232.28.151.86,"{""location"": ""FR"", ""is_mobile"": true}" 11915,5,266,2017-06-05 00:07:28,http://kiehn.net/kadin.ritchie,0.9591546487,138.234.191.248,"{""location"": ""AX"", ""is_mobile"": false}" 11916,5,266,2017-04-17 16:16:16,http://raynor.name/francesco,0.4782681370,199.165.46.2,"{""location"": ""AO"", ""is_mobile"": false}" 11917,5,266,2017-03-23 02:04:59,http://dach.org/brycen,0.1930651800,207.197.140.212,"{""location"": ""IL"", ""is_mobile"": true}" 11918,5,266,2017-01-22 18:52:34,http://conroylueilwitz.name/chase.reilly,0.9098380054,196.64.129.124,"{""location"": ""CL"", ""is_mobile"": true}" 11919,5,266,2016-12-15 19:39:02,http://cummerata.net/idell_blick,0.4035263418,244.33.73.219,"{""location"": ""CL"", ""is_mobile"": false}" 11920,5,266,2017-03-02 20:40:24,http://kautzer.net/margarita.moen,0.6065635639,160.243.58.252,"{""location"": ""BR"", ""is_mobile"": true}" 11921,5,266,2017-04-21 18:33:25,http://kris.io/marguerite,0.9405963569,44.10.109.34,"{""location"": ""LT"", ""is_mobile"": true}" 11922,5,266,2017-02-21 13:46:43,http://buckridge.io/myron.herman,0.7893794467,253.244.254.239,"{""location"": ""PA"", ""is_mobile"": false}" 11923,5,266,2017-03-31 08:17:59,http://hudson.name/aunta_moore,0.4768288601,245.249.188.101,"{""location"": ""SI"", ""is_mobile"": false}" 11924,5,266,2017-01-26 02:55:43,http://hyatt.biz/sage_prosacco,0.5445600675,145.130.57.251,"{""location"": ""IT"", ""is_mobile"": false}" 11925,5,266,2017-04-08 06:51:32,http://okondickens.io/mauricio,0.1569363952,135.228.131.182,"{""location"": ""BG"", ""is_mobile"": false}" 11926,5,266,2017-05-09 00:00:11,http://mills.name/jeramie_stroman,0.0789664822,23.194.56.72,"{""location"": ""HR"", ""is_mobile"": true}" 11927,5,266,2017-02-16 10:18:43,http://jones.org/yoshiko_jacobi,0.1742015917,203.11.183.77,"{""location"": ""IN"", ""is_mobile"": true}" 11928,5,267,2017-02-03 05:09:13,http://halvorsonspencer.co/leonora.yost,0.8921496120,239.229.238.147,"{""location"": ""UZ"", ""is_mobile"": true}" 11929,5,267,2017-04-13 10:08:13,http://runtelebsack.co/geraldine_hintz,0.4763687591,76.239.27.202,"{""location"": ""TF"", ""is_mobile"": true}" 11930,5,267,2017-04-28 14:11:20,http://prosacco.org/roxane,0.9412549963,173.229.124.123,"{""location"": ""AW"", ""is_mobile"": false}" 11931,5,267,2017-05-17 05:46:17,http://treutelheel.net/teagan_heaney,0.4586043847,71.32.183.177,"{""location"": ""IT"", ""is_mobile"": true}" 11932,5,267,2017-03-13 13:30:25,http://ratke.com/craig,0.4269398300,142.152.216.151,"{""location"": ""BJ"", ""is_mobile"": true}" 11933,5,267,2016-12-23 23:04:42,http://sipes.org/madisen,0.7973064512,174.8.178.55,"{""location"": ""BN"", ""is_mobile"": false}" 11934,5,267,2017-05-06 14:09:35,http://hauck.org/arvid.shields,0.6349922132,3.13.50.113,"{""location"": ""GU"", ""is_mobile"": true}" 11935,5,267,2017-03-05 14:41:37,http://friesen.io/krystel.hoppe,0.8471922139,102.186.233.219,"{""location"": ""MA"", ""is_mobile"": true}" 11936,5,267,2017-04-07 03:30:22,http://conroy.co/caandra,0.1791867072,101.186.77.9,"{""location"": ""AX"", ""is_mobile"": true}" 11937,5,267,2017-06-03 20:14:08,http://wisoky.io/anastacio.stroman,0.9789593226,10.196.137.58,"{""location"": ""NU"", ""is_mobile"": true}" 11938,5,267,2017-04-06 15:33:19,http://turnerlowe.org/braeden,0.7245668430,63.221.218.5,"{""location"": ""RU"", ""is_mobile"": false}" 11939,5,267,2017-02-26 02:46:18,http://anderson.com/litzy_johnson,0.1997374733,151.137.144.58,"{""location"": ""UZ"", ""is_mobile"": true}" 11940,5,267,2016-12-22 00:00:59,http://ritchie.info/alayna.block,0.5915255548,39.207.185.84,"{""location"": ""VG"", ""is_mobile"": true}" 11941,5,267,2017-02-28 10:14:52,http://reilly.net/efren.marvin,0.2072807494,241.196.40.160,"{""location"": ""LS"", ""is_mobile"": false}" 11942,5,267,2017-05-15 15:30:07,http://mayert.co/elliott,0.3824977492,124.10.213.9,"{""location"": ""NC"", ""is_mobile"": true}" 11943,5,267,2017-01-22 02:01:29,http://rueckerherzog.co/gregoria,0.6320496105,43.213.198.8,"{""location"": ""EG"", ""is_mobile"": false}" 11944,5,267,2017-02-13 08:48:27,http://beier.name/marlin_lakin,0.7347539524,153.184.7.68,"{""location"": ""SV"", ""is_mobile"": true}" 11945,5,267,2017-03-28 00:57:24,http://stromanschumm.biz/lois,0.9074942253,199.173.49.102,"{""location"": ""SK"", ""is_mobile"": false}" 11946,5,267,2017-01-08 00:42:08,http://moen.net/brandt.jacobson,0.5533901845,187.171.13.52,"{""location"": ""NR"", ""is_mobile"": false}" 11947,5,267,2017-01-18 10:09:04,http://gaylord.biz/evans_howe,0.1409655073,242.222.132.144,"{""location"": ""FM"", ""is_mobile"": false}" 11948,5,267,2017-05-17 15:24:29,http://romaguera.co/shane,0.0164909587,142.57.163.122,"{""location"": ""ST"", ""is_mobile"": false}" 11949,5,267,2017-01-08 12:03:23,http://bartell.io/audie,0.1887320540,123.180.44.101,"{""location"": ""AE"", ""is_mobile"": true}" 11950,5,267,2017-03-26 00:29:36,http://gibson.info/faye,0.9753343094,228.49.157.27,"{""location"": ""GY"", ""is_mobile"": true}" 11951,5,267,2017-03-28 02:40:17,http://turcotte.com/shawn.kreiger,0.9961716671,78.32.121.92,"{""location"": ""PR"", ""is_mobile"": false}" 11952,5,267,2017-03-13 07:16:44,http://jaskolski.com/jacques.johnson,0.7470105142,169.186.189.126,"{""location"": ""ER"", ""is_mobile"": true}" 11953,5,267,2017-04-07 23:33:08,http://deckow.io/irwin.renner,0.2890123246,11.10.197.244,"{""location"": ""HN"", ""is_mobile"": false}" 11954,5,267,2017-01-26 08:54:59,http://gibson.io/eldridge,0.8031064807,202.213.19.154,"{""location"": ""AW"", ""is_mobile"": true}" 11955,5,267,2017-06-04 09:58:22,http://jaskolski.co/brianne,0.8054397129,72.131.222.133,"{""location"": ""LC"", ""is_mobile"": true}" 14889,6,336,2017-04-15 13:26:32,http://stanton.biz/boris.bartell,,105.242.129.53,"{""location"": ""KM"", ""is_mobile"": false}" 14890,6,336,2017-05-30 12:27:10,http://beerhaley.co/jermaine.goldner,,82.155.185.170,"{""location"": ""UG"", ""is_mobile"": false}" 14891,6,336,2017-01-09 04:15:55,http://mcclure.io/merle.metz,,34.112.197.238,"{""location"": ""GW"", ""is_mobile"": true}" 14892,6,336,2017-06-10 06:45:28,http://cummerata.org/lloyd_zemlak,,190.7.39.130,"{""location"": ""PL"", ""is_mobile"": true}" 14893,6,336,2016-12-13 19:54:04,http://lind.org/mariela.kirlin,,5.17.96.252,"{""location"": ""VE"", ""is_mobile"": false}" 14894,6,336,2017-02-06 02:01:25,http://damoreklocko.com/quinn,,193.159.77.30,"{""location"": ""NA"", ""is_mobile"": false}" 14895,6,336,2017-03-17 13:33:31,http://murazik.io/nikolas.fay,,82.86.120.243,"{""location"": ""GA"", ""is_mobile"": true}" 14896,6,336,2016-12-21 19:50:51,http://sipes.biz/cyrus.ferry,,232.84.16.228,"{""location"": ""MN"", ""is_mobile"": false}" 14897,6,336,2016-12-29 09:14:50,http://stroman.io/rita.corkery,,137.116.120.224,"{""location"": ""DM"", ""is_mobile"": true}" 14898,6,336,2017-03-11 13:40:07,http://bartolettijohnston.com/anibal,,106.75.204.236,"{""location"": ""BW"", ""is_mobile"": true}" 14899,6,336,2017-05-02 08:25:53,http://turcotte.info/darien_green,,238.200.70.151,"{""location"": ""TF"", ""is_mobile"": true}" 14900,6,336,2017-04-09 08:45:09,http://medhurst.org/maya,,10.58.48.207,"{""location"": ""FI"", ""is_mobile"": true}" 14901,6,336,2017-02-03 00:56:44,http://johnsonheller.com/leann.olson,,69.158.190.142,"{""location"": ""SN"", ""is_mobile"": true}" 14902,6,336,2017-03-03 06:08:22,http://lueilwitz.biz/jaren.ritchie,,90.169.173.150,"{""location"": ""GM"", ""is_mobile"": true}" 14903,6,336,2017-01-10 12:16:02,http://corkery.com/jayden_bradtke,,246.175.41.133,"{""location"": ""TC"", ""is_mobile"": true}" 14904,6,336,2017-04-29 15:28:32,http://jakubowski.com/jeramy_schinner,,73.215.40.65,"{""location"": ""SA"", ""is_mobile"": false}" 14905,6,336,2017-01-10 11:16:08,http://rogahn.io/eunice,,28.156.35.132,"{""location"": ""CG"", ""is_mobile"": false}" 14906,6,336,2017-03-22 22:41:55,http://ferry.net/rosamond_kohler,,136.84.110.233,"{""location"": ""CU"", ""is_mobile"": true}" 14907,6,336,2017-01-19 15:03:22,http://eichmann.net/houston_murphy,,10.66.14.141,"{""location"": ""MZ"", ""is_mobile"": false}" 14908,6,336,2017-02-20 04:05:16,http://metz.name/leland_lind,,15.107.229.107,"{""location"": ""AR"", ""is_mobile"": false}" 14909,6,336,2017-02-07 12:27:35,http://collinsking.net/willard,,246.105.249.175,"{""location"": ""TR"", ""is_mobile"": false}" 14910,6,336,2017-02-07 22:02:47,http://greenholtbruen.co/jovany,,160.44.92.179,"{""location"": ""MA"", ""is_mobile"": true}" 14911,6,336,2017-03-28 16:30:48,http://leuschke.com/obie,,86.144.147.211,"{""location"": ""SK"", ""is_mobile"": true}" 14912,6,336,2017-02-13 12:42:25,http://borer.org/joany,,200.199.31.82,"{""location"": ""KY"", ""is_mobile"": true}" 14913,6,336,2017-05-08 20:22:47,http://hahn.io/rahsaan,,179.155.67.59,"{""location"": ""BZ"", ""is_mobile"": false}" 14914,6,336,2017-01-27 16:05:09,http://lowe.biz/dominique.langworth,,54.95.178.82,"{""location"": ""VU"", ""is_mobile"": true}" 14915,6,336,2017-02-27 08:57:44,http://lehner.name/mack_goodwin,,203.121.66.30,"{""location"": ""MG"", ""is_mobile"": true}" 14916,6,336,2017-04-23 15:53:08,http://rathdickinson.biz/ethelyn_conn,,20.202.82.220,"{""location"": ""CF"", ""is_mobile"": true}" 14917,6,336,2017-02-17 00:48:25,http://kochrempel.net/earnestine_brown,,201.139.191.5,"{""location"": ""AS"", ""is_mobile"": false}" 14918,6,336,2017-06-13 14:18:25,http://ankunding.biz/chris,,108.22.217.15,"{""location"": ""GW"", ""is_mobile"": false}" 14919,6,336,2017-02-14 11:53:35,http://gutmann.co/rosemary.parker,,147.124.128.139,"{""location"": ""GW"", ""is_mobile"": false}" 14920,6,336,2017-01-29 12:27:46,http://balistreri.com/dorian,,55.163.221.208,"{""location"": ""CK"", ""is_mobile"": true}" 14921,6,336,2017-01-05 00:46:38,http://strosinhickle.org/tyree.homenick,,25.201.118.141,"{""location"": ""CR"", ""is_mobile"": true}" 14922,6,336,2017-01-11 14:42:14,http://jenkinscremin.name/geo_marvin,,148.95.178.231,"{""location"": ""TG"", ""is_mobile"": false}" 14923,6,336,2017-03-06 22:47:38,http://howe.org/rosalinda,,91.115.59.17,"{""location"": ""KN"", ""is_mobile"": true}" 14924,6,336,2016-12-28 06:14:20,http://stromanhartmann.org/sanford,,16.17.98.76,"{""location"": ""GY"", ""is_mobile"": false}" 14925,6,336,2017-05-31 23:15:30,http://leuschke.net/dayne_jakubowski,,134.119.158.66,"{""location"": ""KE"", ""is_mobile"": false}" 14926,6,336,2017-03-15 14:30:54,http://kris.io/durward_ullrich,,103.9.6.105,"{""location"": ""GH"", ""is_mobile"": false}" 14927,6,336,2017-02-07 17:03:10,http://howelind.org/francis_kreiger,,165.24.177.62,"{""location"": ""AU"", ""is_mobile"": true}" 14928,6,336,2017-01-19 20:10:42,http://dubuquemayert.co/chris,,2.74.115.161,"{""location"": ""SY"", ""is_mobile"": true}" 14929,6,336,2017-02-20 23:44:26,http://bins.net/kay_kshlerin,,65.82.158.198,"{""location"": ""GR"", ""is_mobile"": false}" 14930,6,336,2017-01-20 00:30:59,http://flatley.co/torrance,,34.69.125.148,"{""location"": ""LS"", ""is_mobile"": true}" 14931,6,336,2017-05-30 12:33:14,http://gerlach.biz/adrain_reichert,,48.177.20.228,"{""location"": ""RO"", ""is_mobile"": true}" 14932,6,336,2017-04-11 15:59:41,http://keelingrippin.org/katlynn.auer,,13.145.2.214,"{""location"": ""VC"", ""is_mobile"": false}" 14933,6,336,2017-06-07 14:32:51,http://rolfson.net/reie,,208.88.82.249,"{""location"": ""CH"", ""is_mobile"": false}" 14934,6,336,2017-04-04 05:12:16,http://bradtke.org/beaulah,,222.162.136.158,"{""location"": ""TT"", ""is_mobile"": false}" 14935,6,336,2017-05-24 13:53:53,http://okonvon.biz/wyatt.bruen,,12.102.135.160,"{""location"": ""FR"", ""is_mobile"": true}" 14936,6,336,2017-01-23 16:37:38,http://homenickroob.co/bryon,,29.70.32.71,"{""location"": ""PA"", ""is_mobile"": false}" 14937,6,336,2017-02-02 14:06:46,http://stokekiles.name/claudia.sauer,,102.141.217.79,"{""location"": ""GH"", ""is_mobile"": true}" 14938,6,336,2017-04-26 12:40:10,http://hansen.io/chaim.ortiz,,14.129.6.193,"{""location"": ""TL"", ""is_mobile"": false}" 14939,6,336,2017-04-26 07:56:32,http://steuber.org/kiera_weber,,112.166.246.56,"{""location"": ""IQ"", ""is_mobile"": false}" 14940,6,336,2017-01-20 16:13:32,http://kaulke.info/josephine,,74.69.141.245,"{""location"": ""AI"", ""is_mobile"": false}" 14941,6,336,2017-02-20 09:13:16,http://vonhermiston.co/johann.purdy,,83.101.112.141,"{""location"": ""SH"", ""is_mobile"": false}" 9040,4,203,2017-05-09 01:04:26,http://altenwerthwindler.info/mark,0.8181794576,215.30.139.66,"{""location"": ""GT"", ""is_mobile"": true}" 9041,4,203,2017-02-16 17:22:18,http://feest.co/viviane,0.0166207917,197.167.52.253,"{""location"": ""BJ"", ""is_mobile"": true}" 9042,4,203,2017-03-05 13:29:16,http://osinski.co/kaylah,0.4091500876,118.8.62.45,"{""location"": ""MR"", ""is_mobile"": false}" 9043,4,203,2017-01-13 08:45:25,http://kshlerinstracke.com/taya_considine,0.6496999597,211.234.58.206,"{""location"": ""DZ"", ""is_mobile"": true}" 9044,4,203,2017-05-29 23:07:17,http://strosin.info/leora,0.9612903181,120.249.175.183,"{""location"": ""BN"", ""is_mobile"": false}" 9045,4,203,2016-12-31 01:02:55,http://kozey.info/jacinto.christiansen,0.9578217278,198.67.60.180,"{""location"": ""SG"", ""is_mobile"": true}" 9046,4,203,2017-01-07 00:27:36,http://ankunding.com/fredrick,0.5998710009,94.159.242.20,"{""location"": ""SM"", ""is_mobile"": false}" 9047,4,203,2017-05-19 05:55:22,http://simonis.net/ansel,0.3120550466,140.87.192.190,"{""location"": ""CY"", ""is_mobile"": false}" 9048,4,203,2016-12-28 07:20:55,http://pagac.io/lionel,0.0089949262,193.144.24.48,"{""location"": ""SR"", ""is_mobile"": false}" 9049,4,203,2017-02-24 17:20:52,http://larsontoy.com/mya_cormier,0.7929195627,241.54.248.154,"{""location"": ""RS"", ""is_mobile"": true}" 9050,4,203,2017-01-20 21:25:36,http://howell.io/rubye,0.0307491492,222.155.82.151,"{""location"": ""TG"", ""is_mobile"": true}" 9051,4,203,2017-03-23 16:46:41,http://wisozkfritsch.com/eda.hodkiewicz,0.0580375398,96.61.169.31,"{""location"": ""RE"", ""is_mobile"": true}" 9052,4,203,2017-05-27 17:23:53,http://cruickshankcormier.net/timothy,0.4655233226,55.114.87.35,"{""location"": ""SX"", ""is_mobile"": false}" 9053,4,203,2017-03-21 04:47:04,http://zboncak.com/ellen.walsh,0.3254719706,69.29.36.212,"{""location"": ""SI"", ""is_mobile"": false}" 9054,4,203,2017-04-24 03:50:11,http://predovic.name/lucile.reichel,0.8369299046,80.124.123.50,"{""location"": ""SJ"", ""is_mobile"": false}" 9055,4,203,2017-01-25 05:31:54,http://bruen.info/lorena_jacobson,0.7872795308,147.68.87.110,"{""location"": ""GE"", ""is_mobile"": false}" 9056,4,203,2017-06-08 12:47:28,http://yundt.com/laurine.hackett,0.4236863199,40.50.204.51,"{""location"": ""SN"", ""is_mobile"": true}" 9057,4,203,2017-02-24 22:33:56,http://torphystark.io/jamison.rau,0.2140555521,162.183.155.88,"{""location"": ""MR"", ""is_mobile"": false}" 9058,4,203,2017-02-27 16:27:24,http://ryan.name/hershel.cole,0.7994463846,55.95.142.237,"{""location"": ""KR"", ""is_mobile"": true}" 9059,4,203,2017-03-31 01:00:07,http://wilderman.info/lexi_rogahn,0.0580235824,125.177.16.114,"{""location"": ""SE"", ""is_mobile"": true}" 9060,4,203,2016-12-17 07:51:16,http://feestleuschke.name/eleazar,0.1139732523,189.78.239.163,"{""location"": ""TW"", ""is_mobile"": true}" 9061,4,203,2017-06-05 01:24:05,http://padberg.biz/kiarra_beahan,0.4975870937,15.220.82.24,"{""location"": ""FR"", ""is_mobile"": false}" 9062,4,203,2017-04-06 12:16:52,http://feeney.org/roberta.bogisich,0.8732157700,71.197.209.133,"{""location"": ""GF"", ""is_mobile"": false}" 9063,4,203,2017-01-12 23:42:49,http://kuhnkris.com/dudley,0.1320846054,80.42.164.251,"{""location"": ""CX"", ""is_mobile"": false}" 9064,4,203,2017-04-21 20:19:37,http://donnelly.org/julianne,0.2578789095,142.117.231.16,"{""location"": ""KW"", ""is_mobile"": false}" 9065,4,203,2017-01-20 08:00:36,http://renner.org/theresa,0.2933961033,213.104.46.181,"{""location"": ""CG"", ""is_mobile"": false}" 9066,4,203,2017-04-02 18:35:05,http://ratke.co/mariam.block,0.8631010956,46.53.37.127,"{""location"": ""CM"", ""is_mobile"": true}" 9067,4,203,2017-02-04 15:05:46,http://wymanschmeler.io/cleveland_vonrueden,0.0334704511,59.52.156.28,"{""location"": ""DJ"", ""is_mobile"": false}" 9068,4,203,2017-05-03 20:20:03,http://hettingercollins.name/myra,0.3985607251,61.155.4.103,"{""location"": ""ID"", ""is_mobile"": false}" 9069,4,203,2017-03-23 03:59:48,http://murray.co/lura,0.9012938095,169.105.99.206,"{""location"": ""PR"", ""is_mobile"": false}" 9070,4,203,2017-05-05 00:26:56,http://oconnellschmeler.co/lafayette,0.7466290768,126.151.12.17,"{""location"": ""CI"", ""is_mobile"": false}" 9071,4,203,2017-03-09 08:00:09,http://weber.biz/mckenna,0.8483811166,76.72.20.29,"{""location"": ""CC"", ""is_mobile"": false}" 9072,4,203,2017-02-22 05:48:32,http://daniel.info/verona.anderson,0.0900736265,159.40.126.234,"{""location"": ""MH"", ""is_mobile"": true}" 9073,4,203,2017-03-28 04:41:46,http://steuber.biz/valentine.hintz,0.6159139134,254.74.27.27,"{""location"": ""FR"", ""is_mobile"": true}" 9074,4,204,2017-04-15 01:36:13,http://bernier.info/kendra,0.2860127345,10.117.67.181,"{""location"": ""SR"", ""is_mobile"": true}" 9075,4,204,2016-12-17 13:07:11,http://lemke.io/dusty.larson,0.7302770880,69.93.61.85,"{""location"": ""NE"", ""is_mobile"": true}" 9076,4,204,2017-01-17 07:16:28,http://breitenbergswaniawski.net/freda,0.2200828297,134.4.200.238,"{""location"": ""BL"", ""is_mobile"": true}" 9077,4,204,2017-05-24 01:18:12,http://simonis.com/tamara_volkman,0.5736650801,203.156.247.238,"{""location"": ""LK"", ""is_mobile"": true}" 9078,4,204,2017-02-14 07:06:36,http://schinner.biz/wallace,0.9625829253,179.73.62.129,"{""location"": ""MR"", ""is_mobile"": false}" 9079,4,204,2017-01-02 06:31:54,http://parker.info/delores.smitham,0.3940273666,166.218.187.169,"{""location"": ""MR"", ""is_mobile"": false}" 9080,4,204,2017-06-06 06:10:07,http://lesch.net/jensen,0.8243936713,60.104.164.253,"{""location"": ""KZ"", ""is_mobile"": true}" 9081,4,204,2017-06-07 22:11:25,http://oconnellmante.io/angelica,0.7041920274,91.39.171.16,"{""location"": ""LA"", ""is_mobile"": false}" 9082,4,204,2017-01-28 02:11:08,http://braunbotsford.biz/carter_reichel,0.5084487523,239.44.141.144,"{""location"": ""AT"", ""is_mobile"": false}" 9083,4,204,2017-05-19 19:05:08,http://olson.net/nadia_keeling,0.5601176911,33.204.199.252,"{""location"": ""GE"", ""is_mobile"": false}" 9084,4,204,2017-03-20 16:20:40,http://wildermanlowe.io/erna,0.0486222039,211.166.75.178,"{""location"": ""AZ"", ""is_mobile"": true}" 9085,4,204,2017-05-25 17:33:26,http://buckridge.co/burdette.ruel,0.2434911752,38.110.26.169,"{""location"": ""TO"", ""is_mobile"": true}" 9086,4,204,2017-04-24 05:00:13,http://torplehner.info/glen_hagenes,0.5138480049,136.215.154.172,"{""location"": ""AM"", ""is_mobile"": false}" 9087,4,204,2017-01-02 12:13:37,http://miller.name/lenore_tromp,0.6522095470,174.168.48.115,"{""location"": ""CW"", ""is_mobile"": true}" 9088,4,204,2017-03-20 07:14:28,http://wuckert.org/oran.hoeger,0.0765314606,217.210.59.117,"{""location"": ""BT"", ""is_mobile"": true}" 9089,4,204,2017-02-05 10:30:47,http://keeling.io/ava,0.1361255966,88.161.204.85,"{""location"": ""AS"", ""is_mobile"": true}" 9090,4,204,2017-05-08 05:45:01,http://jenkins.com/joy_bradtke,0.5579061913,40.214.137.148,"{""location"": ""WF"", ""is_mobile"": true}" 9091,4,204,2017-01-22 02:51:47,http://hudson.net/irving.gulgowski,0.7225069755,164.191.186.213,"{""location"": ""NZ"", ""is_mobile"": false}" 6038,3,131,2017-01-29 22:12:26,http://homenick.net/gunner,,116.12.85.150,"{""location"": ""UG"", ""is_mobile"": true}" 6039,3,131,2016-12-31 09:43:51,http://ritchie.biz/della,,216.238.223.10,"{""location"": ""GL"", ""is_mobile"": false}" 6040,3,131,2017-04-10 00:17:53,http://batzhuels.io/kirsten,,251.21.5.33,"{""location"": ""SR"", ""is_mobile"": true}" 6041,3,131,2017-02-06 23:53:42,http://walker.name/modesto,,150.200.120.68,"{""location"": ""SS"", ""is_mobile"": true}" 6042,3,131,2017-05-01 09:21:07,http://friesen.net/willy,,180.62.163.87,"{""location"": ""IN"", ""is_mobile"": true}" 6043,3,131,2016-12-27 18:16:49,http://mohr.com/jose.brekke,,129.130.226.79,"{""location"": ""EG"", ""is_mobile"": true}" 6044,3,131,2017-03-04 15:23:19,http://schoen.name/oran,,241.197.116.25,"{""location"": ""GE"", ""is_mobile"": true}" 6045,3,132,2017-01-22 22:23:31,http://lehnertorphy.com/kip_wintheiser,,15.90.144.147,"{""location"": ""RO"", ""is_mobile"": false}" 6046,3,132,2017-04-06 23:15:26,http://terryfisher.info/marge.hills,,147.125.135.121,"{""location"": ""JO"", ""is_mobile"": false}" 6933,3,152,2017-05-27 12:47:08,http://reichert.io/tyson,,179.120.56.49,"{""location"": ""AD"", ""is_mobile"": false}" 6047,3,132,2017-05-15 12:33:43,http://wintheisercorwin.com/elinore.kunze,,19.174.147.206,"{""location"": ""NU"", ""is_mobile"": true}" 6048,3,132,2017-05-13 17:11:36,http://davis.info/taryn,,13.172.230.117,"{""location"": ""LK"", ""is_mobile"": false}" 6049,3,132,2017-03-20 06:06:34,http://schusterdibbert.co/dock,,147.5.185.166,"{""location"": ""NE"", ""is_mobile"": true}" 6050,3,132,2017-05-14 10:20:58,http://durgannienow.net/althea.schaefer,,170.28.127.189,"{""location"": ""AO"", ""is_mobile"": false}" 6051,3,132,2017-03-31 07:04:53,http://lindgrenstoltenberg.io/estel,,244.20.228.10,"{""location"": ""GI"", ""is_mobile"": false}" 6052,3,132,2017-03-17 17:49:30,http://wilkinson.net/gerald_hegmann,,230.197.111.98,"{""location"": ""AI"", ""is_mobile"": false}" 6053,3,132,2017-04-20 02:37:48,http://kris.info/vince,,226.21.103.158,"{""location"": ""EC"", ""is_mobile"": false}" 6054,3,132,2017-04-06 23:36:12,http://reichert.io/ernestine.kuphal,,152.124.87.219,"{""location"": ""AX"", ""is_mobile"": true}" 6055,3,132,2017-01-31 17:50:23,http://oberbrunner.info/nick.littel,,65.210.70.30,"{""location"": ""UM"", ""is_mobile"": false}" 6056,3,132,2017-04-25 13:42:38,http://mante.info/hardy,,244.87.104.61,"{""location"": ""KE"", ""is_mobile"": true}" 6057,3,132,2017-04-02 13:14:39,http://schinnerhaag.co/georgianna.tillman,,154.241.193.238,"{""location"": ""SE"", ""is_mobile"": false}" 6058,3,132,2017-02-08 08:18:53,http://hand.com/abbey.lang,,203.81.229.155,"{""location"": ""SB"", ""is_mobile"": false}" 6059,3,132,2017-06-05 11:14:41,http://ritchie.io/vernon_gibson,,28.121.43.56,"{""location"": ""CF"", ""is_mobile"": true}" 6060,3,132,2017-01-22 17:18:21,http://kihnmills.org/cathryn,,14.22.143.75,"{""location"": ""SN"", ""is_mobile"": false}" 6061,3,132,2017-03-19 16:54:57,http://wuckert.com/carlee,,223.12.197.32,"{""location"": ""GD"", ""is_mobile"": true}" 6062,3,132,2017-06-11 19:55:44,http://cain.org/kamron,,242.244.43.243,"{""location"": ""FM"", ""is_mobile"": true}" 6063,3,132,2017-03-24 20:48:58,http://jacobshirthe.org/kaylin,,156.231.203.92,"{""location"": ""GU"", ""is_mobile"": true}" 6064,3,132,2016-12-28 05:01:18,http://lockman.org/dimitri,,122.65.27.233,"{""location"": ""GA"", ""is_mobile"": true}" 6065,3,132,2017-02-09 10:19:49,http://hintz.io/gwendolyn_donnelly,,195.232.235.149,"{""location"": ""CK"", ""is_mobile"": true}" 6066,3,132,2017-05-22 12:47:57,http://schroeder.net/susanna,,118.142.125.236,"{""location"": ""LU"", ""is_mobile"": false}" 6067,3,132,2017-02-28 01:53:22,http://ziemann.org/dedric_hilll,,254.31.43.238,"{""location"": ""TH"", ""is_mobile"": false}" 6068,3,132,2017-01-12 07:20:02,http://metz.net/prudence_kozey,,167.120.210.65,"{""location"": ""SL"", ""is_mobile"": false}" 6069,3,132,2016-12-18 16:09:17,http://wehner.co/roberta,,10.143.72.213,"{""location"": ""BJ"", ""is_mobile"": true}" 6070,3,132,2017-06-10 07:04:06,http://reilly.org/madeline,,58.151.111.93,"{""location"": ""ZM"", ""is_mobile"": false}" 6071,3,132,2017-05-18 14:40:07,http://wilkinson.org/jewell,,60.175.124.234,"{""location"": ""FJ"", ""is_mobile"": true}" 6072,3,132,2017-05-25 16:55:49,http://lemke.info/flo,,160.146.184.69,"{""location"": ""AT"", ""is_mobile"": false}" 6073,3,132,2017-05-20 16:22:08,http://romaguera.org/kayli,,219.240.174.202,"{""location"": ""HK"", ""is_mobile"": false}" 6074,3,132,2017-05-24 17:18:45,http://franecki.com/jannie,,121.3.65.168,"{""location"": ""VN"", ""is_mobile"": true}" 6075,3,132,2017-03-10 14:13:59,http://steuber.net/verona.kirlin,,29.58.13.131,"{""location"": ""NC"", ""is_mobile"": false}" 6076,3,132,2017-01-19 18:15:32,http://douglas.net/trycia,,10.190.144.20,"{""location"": ""SI"", ""is_mobile"": false}" 6077,3,132,2017-04-21 01:20:20,http://eichmannauer.biz/mckenna,,195.153.22.202,"{""location"": ""BW"", ""is_mobile"": false}" 6078,3,132,2017-01-16 10:19:15,http://homenick.io/cristobal,,34.232.224.27,"{""location"": ""KW"", ""is_mobile"": false}" 6079,3,132,2017-01-18 06:38:13,http://nicolas.io/mireya,,35.221.116.46,"{""location"": ""LS"", ""is_mobile"": false}" 6080,3,132,2017-02-10 02:42:22,http://weinat.co/lenore.lesch,,60.153.67.233,"{""location"": ""OM"", ""is_mobile"": false}" 6081,3,132,2017-04-04 15:09:49,http://king.biz/david_leuschke,,214.2.167.47,"{""location"": ""KG"", ""is_mobile"": false}" 6082,3,132,2017-03-23 04:43:24,http://ankundinggutkowski.biz/otis_schinner,,169.248.240.41,"{""location"": ""NA"", ""is_mobile"": true}" 6083,3,132,2017-01-20 16:41:21,http://pagaclynch.org/jermaine.reinger,,172.189.129.42,"{""location"": ""ZA"", ""is_mobile"": false}" 6084,3,132,2017-05-22 15:46:14,http://simonis.com/dahlia,,133.99.155.25,"{""location"": ""IQ"", ""is_mobile"": false}" 6085,3,132,2017-02-19 12:55:52,http://hickle.name/thea,,200.118.97.11,"{""location"": ""NE"", ""is_mobile"": false}" 6086,3,132,2016-12-19 20:21:23,http://torpvonrueden.org/paul_blick,,8.99.40.208,"{""location"": ""HN"", ""is_mobile"": true}" 6087,3,132,2017-01-31 03:35:20,http://kunze.name/macey.frami,,230.214.121.244,"{""location"": ""AG"", ""is_mobile"": false}" 6088,3,132,2017-03-25 03:25:42,http://purdy.io/idell,,75.178.150.198,"{""location"": ""IR"", ""is_mobile"": true}" 6089,3,132,2017-01-06 08:01:16,http://bartell.biz/dianna.wilderman,,190.125.253.253,"{""location"": ""SI"", ""is_mobile"": false}" 6090,3,132,2017-04-01 20:14:04,http://bartoletti.biz/brady,,48.90.5.92,"{""location"": ""MO"", ""is_mobile"": true}" 6091,3,132,2017-02-05 09:33:57,http://funk.name/lelia,,88.16.180.119,"{""location"": ""NI"", ""is_mobile"": true}" 6092,3,132,2016-12-29 03:16:05,http://conroy.co/dan.stanton,,189.27.148.16,"{""location"": ""TV"", ""is_mobile"": false}" 11956,5,267,2017-01-22 01:47:58,http://dooley.co/matilde,0.9826193419,21.188.188.183,"{""location"": ""DJ"", ""is_mobile"": true}" 11957,5,267,2017-02-26 13:07:20,http://schaden.org/adella_macgyver,0.8226360078,174.141.211.178,"{""location"": ""MC"", ""is_mobile"": false}" 11958,5,267,2017-04-14 05:39:40,http://cronatillman.info/richard.hermiston,0.4415385242,159.69.221.135,"{""location"": ""LC"", ""is_mobile"": false}" 11959,5,267,2017-02-08 16:03:03,http://borerkemmer.io/rozella.larkin,0.1989662219,154.32.27.198,"{""location"": ""LS"", ""is_mobile"": true}" 11960,5,267,2017-01-10 02:51:17,http://abshire.io/beie,0.4832993235,131.57.97.68,"{""location"": ""KH"", ""is_mobile"": false}" 11961,5,267,2017-03-19 11:38:14,http://mohrkautzer.org/mallory,0.7266332543,146.109.146.104,"{""location"": ""IM"", ""is_mobile"": true}" 11962,5,267,2016-12-16 05:40:47,http://pfannerstillpouros.net/lane,0.6262636080,74.171.125.104,"{""location"": ""PN"", ""is_mobile"": false}" 11963,5,267,2017-02-09 22:36:51,http://kreigerturner.co/kelsie_haag,0.9632597957,28.67.98.100,"{""location"": ""KW"", ""is_mobile"": true}" 11964,5,267,2017-01-24 18:30:42,http://gulgowski.info/sandy,0.6272052952,127.3.129.243,"{""location"": ""CI"", ""is_mobile"": false}" 11965,5,267,2017-03-19 12:36:21,http://krajcik.name/sandrine,0.6771677990,117.113.49.230,"{""location"": ""KE"", ""is_mobile"": false}" 11966,5,267,2016-12-16 06:28:50,http://luettgen.biz/marty,0.9445689779,218.112.150.211,"{""location"": ""FR"", ""is_mobile"": false}" 11967,5,267,2017-03-19 10:48:27,http://cronin.com/jamison_abshire,0.7348573267,176.81.48.70,"{""location"": ""UA"", ""is_mobile"": true}" 11968,5,267,2017-03-30 00:07:26,http://cummings.io/nolan_mraz,0.1786904787,176.59.246.112,"{""location"": ""NI"", ""is_mobile"": false}" 11969,5,268,2017-03-17 10:23:09,http://wolfbartell.com/elmer,0.3978016248,69.42.150.203,"{""location"": ""MX"", ""is_mobile"": false}" 11970,5,268,2017-03-18 12:02:00,http://bartonleffler.io/tyra_kihn,0.7274953699,87.70.168.129,"{""location"": ""PA"", ""is_mobile"": false}" 11971,5,268,2017-02-04 10:01:44,http://romaguera.io/aidan,0.4182211296,202.95.114.14,"{""location"": ""FM"", ""is_mobile"": false}" 11972,5,268,2017-05-31 14:08:33,http://robel.com/xzavier_kling,0.6691326127,246.159.81.165,"{""location"": ""FI"", ""is_mobile"": false}" 11973,5,268,2017-05-08 11:01:42,http://thiel.com/marilou,0.8612565932,25.44.125.143,"{""location"": ""MP"", ""is_mobile"": false}" 11974,5,268,2017-04-25 06:05:19,http://yostmacgyver.net/franz,0.0550424418,35.21.171.83,"{""location"": ""PW"", ""is_mobile"": true}" 11975,5,268,2017-05-11 02:37:40,http://towne.com/candelario.mohr,0.1853853084,238.78.93.113,"{""location"": ""GE"", ""is_mobile"": true}" 11976,5,268,2017-03-18 17:13:09,http://pacocha.name/guiseppe_casper,0.9686103812,253.49.5.160,"{""location"": ""SJ"", ""is_mobile"": false}" 11977,5,268,2017-02-09 01:53:14,http://douglas.io/marta_okon,0.5742725811,75.225.68.38,"{""location"": ""GS"", ""is_mobile"": false}" 11978,5,268,2017-05-24 09:56:51,http://feest.net/baron,0.5179085506,83.165.9.127,"{""location"": ""KN"", ""is_mobile"": true}" 11979,5,268,2017-05-30 20:43:29,http://tromp.com/dawson,0.2441047687,241.190.86.236,"{""location"": ""DM"", ""is_mobile"": false}" 11980,5,268,2017-06-08 06:35:13,http://beerkonopelski.io/aidan_oconner,0.4846495398,24.140.127.63,"{""location"": ""FK"", ""is_mobile"": false}" 11981,5,268,2017-02-17 14:46:58,http://crona.io/craig_lakin,0.6916640257,12.194.22.124,"{""location"": ""MP"", ""is_mobile"": false}" 11982,5,268,2017-02-13 23:17:50,http://wolff.io/helene,0.2431830556,228.103.253.176,"{""location"": ""AQ"", ""is_mobile"": true}" 11983,5,268,2017-04-14 05:08:31,http://lemkebeer.io/helena_lehner,0.5978222649,137.12.21.86,"{""location"": ""SX"", ""is_mobile"": true}" 11984,5,268,2017-01-29 00:55:00,http://bauchconroy.io/shayna,0.6175000440,219.141.196.205,"{""location"": ""PN"", ""is_mobile"": false}" 11985,5,268,2017-06-01 16:50:37,http://lubowitz.com/madelynn_stamm,0.0692788105,12.240.209.35,"{""location"": ""TO"", ""is_mobile"": true}" 11986,5,268,2017-05-19 18:59:46,http://kertzmann.name/elnora.bergstrom,0.7415936805,15.209.191.43,"{""location"": ""CN"", ""is_mobile"": true}" 11987,5,268,2017-04-23 13:05:15,http://goodwinstanton.net/ali,0.8436591364,127.142.118.201,"{""location"": ""BD"", ""is_mobile"": false}" 11988,5,268,2017-04-22 16:29:56,http://markslueilwitz.co/imogene,0.6398817647,163.107.166.14,"{""location"": ""GB"", ""is_mobile"": false}" 11989,5,268,2017-02-24 10:25:52,http://will.co/aaliyah,0.7705110494,79.85.74.33,"{""location"": ""MG"", ""is_mobile"": false}" 11990,5,268,2017-01-16 09:02:40,http://shanahan.biz/haie,0.9210062106,229.212.213.55,"{""location"": ""GH"", ""is_mobile"": true}" 11991,5,268,2016-12-18 18:12:23,http://gerhold.info/amina,0.5868686893,224.74.10.112,"{""location"": ""CO"", ""is_mobile"": true}" 11992,5,268,2017-05-17 18:51:31,http://hettinger.biz/jerrod_mann,0.0383565053,67.141.69.50,"{""location"": ""IO"", ""is_mobile"": true}" 11993,5,268,2016-12-23 22:10:36,http://wehner.name/felipe,0.1440324429,15.112.231.73,"{""location"": ""PL"", ""is_mobile"": false}" 11994,5,268,2017-04-26 22:40:42,http://rosenbaum.name/davion,0.9336001123,166.172.58.116,"{""location"": ""SX"", ""is_mobile"": true}" 11995,5,268,2016-12-30 02:12:21,http://bruenglover.net/allene,0.9291949150,68.92.33.234,"{""location"": ""AQ"", ""is_mobile"": false}" 11996,5,268,2017-01-22 05:03:15,http://mraz.io/lon,0.7699133113,116.154.218.71,"{""location"": ""PG"", ""is_mobile"": false}" 11997,5,268,2017-02-18 13:28:16,http://littel.info/michael,0.9195926877,25.208.128.40,"{""location"": ""RE"", ""is_mobile"": true}" 11998,5,269,2017-01-17 17:02:31,http://runolfsdottir.info/milo,0.7529212630,23.225.230.54,"{""location"": ""AT"", ""is_mobile"": false}" 11999,5,269,2016-12-22 12:46:58,http://langoshruecker.org/lester.langworth,0.8221729192,195.125.10.242,"{""location"": ""FO"", ""is_mobile"": true}" 12000,5,269,2017-06-08 19:31:49,http://fisher.info/billy,0.8688481259,173.132.56.241,"{""location"": ""TR"", ""is_mobile"": true}" 12001,5,269,2017-06-04 19:32:17,http://lesch.org/dahlia_kunze,0.8372441708,218.74.22.48,"{""location"": ""MO"", ""is_mobile"": false}" 12002,5,269,2017-01-14 17:53:48,http://johns.com/matilde,0.9238419313,72.196.133.76,"{""location"": ""CD"", ""is_mobile"": false}" 12003,5,269,2017-01-10 22:55:18,http://mertz.io/ramona.beatty,0.4049512991,4.179.59.177,"{""location"": ""VG"", ""is_mobile"": true}" 12004,5,269,2017-04-26 23:02:48,http://schoenmcglynn.co/jamil,0.2602687884,119.2.186.141,"{""location"": ""VN"", ""is_mobile"": false}" 12005,5,269,2017-05-15 11:56:15,http://heller.io/ozzie,0.5490753550,202.6.96.125,"{""location"": ""BJ"", ""is_mobile"": true}" 12006,5,269,2017-06-05 08:03:18,http://hegmann.biz/jermaine,0.7685844344,156.92.65.122,"{""location"": ""VU"", ""is_mobile"": true}" 12007,5,269,2017-03-27 01:07:32,http://ferry.name/orpha,0.2710002316,53.236.106.39,"{""location"": ""PK"", ""is_mobile"": true}" 14942,6,336,2016-12-17 00:13:29,http://moendicki.biz/jodie_skiles,,76.225.15.123,"{""location"": ""SZ"", ""is_mobile"": false}" 14943,6,336,2017-05-11 15:23:20,http://mayertwiza.name/donavon_crist,,217.114.94.64,"{""location"": ""MW"", ""is_mobile"": true}" 14944,6,336,2017-01-30 12:44:57,http://collins.name/llewellyn.legros,,66.94.137.90,"{""location"": ""TT"", ""is_mobile"": false}" 14945,6,336,2016-12-21 14:37:19,http://osinski.info/reid_blanda,,29.56.188.198,"{""location"": ""SE"", ""is_mobile"": true}" 14946,6,337,2017-02-17 23:39:17,http://sporer.co/rudolph,,22.9.194.150,"{""location"": ""VU"", ""is_mobile"": true}" 14947,6,337,2017-05-01 10:45:07,http://vandervort.name/edwardo_brakus,,107.22.147.15,"{""location"": ""QA"", ""is_mobile"": false}" 14948,6,337,2017-01-15 10:31:42,http://treutel.org/jacinthe.barton,,104.29.148.136,"{""location"": ""CW"", ""is_mobile"": true}" 14949,6,337,2017-05-10 13:00:08,http://bauch.org/leonora.sipes,,183.173.230.194,"{""location"": ""ST"", ""is_mobile"": false}" 14950,6,337,2017-02-11 01:53:58,http://walter.com/geoffrey_wisozk,,119.200.102.113,"{""location"": ""MD"", ""is_mobile"": true}" 14951,6,337,2017-03-04 21:59:28,http://schuppe.name/deon.rau,,139.226.167.94,"{""location"": ""MX"", ""is_mobile"": true}" 14952,6,337,2017-06-13 18:46:23,http://casperkohler.com/laurie,,68.5.154.218,"{""location"": ""PS"", ""is_mobile"": false}" 14953,6,337,2017-01-29 10:41:36,http://kozey.org/laron,,24.225.111.96,"{""location"": ""MF"", ""is_mobile"": true}" 14954,6,337,2017-04-27 04:56:10,http://champlin.org/garland,,208.43.141.165,"{""location"": ""HN"", ""is_mobile"": true}" 14955,6,337,2017-04-04 15:23:11,http://wiegand.com/lindsay.feest,,159.113.188.86,"{""location"": ""TD"", ""is_mobile"": false}" 14956,6,337,2017-03-27 04:41:47,http://tremblayhaag.com/armand,,60.127.85.236,"{""location"": ""PN"", ""is_mobile"": true}" 14957,6,337,2017-01-27 16:18:26,http://jast.biz/darlene,,64.253.200.7,"{""location"": ""CW"", ""is_mobile"": false}" 14958,6,337,2017-01-18 02:03:58,http://beahanhowe.co/laron,,51.144.41.155,"{""location"": ""NF"", ""is_mobile"": false}" 14959,6,337,2017-04-27 10:07:35,http://robel.biz/drake,,48.251.51.225,"{""location"": ""RW"", ""is_mobile"": false}" 14960,6,337,2017-04-29 00:13:59,http://beahandubuque.com/stan,,11.72.68.150,"{""location"": ""CK"", ""is_mobile"": false}" 14961,6,337,2017-02-12 19:56:30,http://bode.net/rosario.nolan,,81.239.245.6,"{""location"": ""VA"", ""is_mobile"": true}" 14962,6,337,2017-02-09 19:58:39,http://douglaskertzmann.org/marielle,,37.98.139.198,"{""location"": ""CM"", ""is_mobile"": true}" 14963,6,337,2017-05-06 11:51:24,http://cole.co/kylee,,69.44.127.33,"{""location"": ""KE"", ""is_mobile"": false}" 14964,6,337,2017-03-16 08:23:33,http://reingertillman.co/kristoffer.barrows,,98.115.3.171,"{""location"": ""GE"", ""is_mobile"": true}" 14965,6,337,2017-05-16 14:57:23,http://luettgen.com/lucio.crooks,,143.99.128.124,"{""location"": ""AW"", ""is_mobile"": false}" 14966,6,337,2017-03-02 05:40:08,http://hammes.name/uriah_langworth,,101.116.157.65,"{""location"": ""MT"", ""is_mobile"": true}" 14967,6,337,2017-06-03 08:39:35,http://cummerata.name/trea,,233.149.215.39,"{""location"": ""BQ"", ""is_mobile"": true}" 14968,6,337,2017-05-31 21:52:15,http://brekke.net/garett_volkman,,77.225.204.163,"{""location"": ""CU"", ""is_mobile"": false}" 14969,6,337,2017-01-21 19:25:25,http://christiansenokuneva.io/markus,,124.71.99.130,"{""location"": ""CY"", ""is_mobile"": true}" 14970,6,337,2017-02-21 03:16:14,http://ko.co/demarco,,152.160.127.138,"{""location"": ""IM"", ""is_mobile"": false}" 14971,6,337,2017-02-13 13:44:37,http://bauchkshlerin.info/aiyana,,54.128.236.96,"{""location"": ""VU"", ""is_mobile"": false}" 14972,6,337,2017-05-09 01:19:47,http://stehrfahey.com/cecil,,20.139.215.121,"{""location"": ""BG"", ""is_mobile"": false}" 14973,6,337,2017-02-28 14:47:22,http://corkery.org/kirk,,199.47.122.159,"{""location"": ""BA"", ""is_mobile"": false}" 14974,6,337,2016-12-20 10:08:49,http://windler.net/rusty_gleichner,,155.165.194.105,"{""location"": ""GH"", ""is_mobile"": true}" 14975,6,337,2017-03-24 14:51:37,http://walter.info/nya_tillman,,225.160.161.240,"{""location"": ""NO"", ""is_mobile"": true}" 14976,6,337,2017-05-28 13:53:08,http://vonrueden.net/alaina,,45.244.117.161,"{""location"": ""LI"", ""is_mobile"": false}" 14977,6,337,2017-02-18 19:17:07,http://kuhic.com/kyler,,172.79.241.100,"{""location"": ""TW"", ""is_mobile"": false}" 14978,6,337,2017-01-16 02:54:31,http://harberhermiston.info/wilbert,,247.50.171.193,"{""location"": ""MV"", ""is_mobile"": true}" 14979,6,337,2017-03-03 23:12:56,http://bernhardchristiansen.io/corene,,132.200.224.252,"{""location"": ""GY"", ""is_mobile"": true}" 14980,6,337,2017-03-16 00:46:04,http://kshlerinbarton.com/travon_crooks,,82.7.42.227,"{""location"": ""RO"", ""is_mobile"": false}" 14981,6,337,2017-02-26 07:04:53,http://wilderman.io/araceli.koepp,,194.224.206.197,"{""location"": ""KN"", ""is_mobile"": false}" 14982,6,337,2017-05-04 15:26:45,http://braun.biz/erin,,205.156.188.36,"{""location"": ""KP"", ""is_mobile"": true}" 14983,6,337,2017-02-16 11:09:59,http://wehner.biz/hollis.schmidt,,231.239.222.163,"{""location"": ""SZ"", ""is_mobile"": false}" 14984,6,337,2017-04-28 02:38:53,http://rutherford.biz/maiya,,122.100.157.197,"{""location"": ""AE"", ""is_mobile"": false}" 14985,6,337,2017-02-27 12:54:33,http://larkinlindgren.biz/rita,,130.218.3.247,"{""location"": ""NR"", ""is_mobile"": true}" 14986,6,337,2016-12-16 15:43:58,http://gislason.com/haven,,31.130.7.87,"{""location"": ""PM"", ""is_mobile"": false}" 14987,6,337,2017-02-23 01:35:48,http://wisozkfadel.biz/claire_kunze,,54.182.66.107,"{""location"": ""MA"", ""is_mobile"": false}" 14988,6,337,2017-02-10 00:56:38,http://kaulkekirlin.net/kitty_buckridge,,119.130.21.153,"{""location"": ""BB"", ""is_mobile"": false}" 14989,6,337,2016-12-20 04:00:33,http://gislasonroberts.net/kaleigh,,220.111.167.51,"{""location"": ""DZ"", ""is_mobile"": false}" 14990,6,337,2017-04-09 18:42:14,http://davis.com/carmen,,47.70.106.42,"{""location"": ""FK"", ""is_mobile"": true}" 14991,6,337,2017-03-27 04:15:45,http://altenwerthrenner.net/blair.price,,137.157.196.202,"{""location"": ""FK"", ""is_mobile"": true}" 14992,6,337,2016-12-19 23:46:29,http://ortiz.biz/lizzie,,36.223.183.130,"{""location"": ""JP"", ""is_mobile"": false}" 14993,6,337,2017-01-19 14:33:40,http://stoltenberg.biz/napoleon.flatley,,58.75.96.120,"{""location"": ""KR"", ""is_mobile"": false}" 14994,6,337,2017-01-08 03:03:42,http://pouros.co/kacey,,3.177.96.178,"{""location"": ""SL"", ""is_mobile"": false}" 14995,6,337,2017-05-18 19:35:16,http://shanahantremblay.name/domingo.lowe,,52.237.13.100,"{""location"": ""DJ"", ""is_mobile"": true}" 14996,6,337,2017-05-04 20:41:25,http://barton.info/raymundo_bergnaum,,181.149.95.247,"{""location"": ""HT"", ""is_mobile"": true}" 9092,4,204,2017-05-19 11:15:12,http://goyettemaggio.info/wyatt,0.7328989314,47.236.19.172,"{""location"": ""ML"", ""is_mobile"": false}" 9093,4,204,2017-01-15 13:19:49,http://jaskolski.biz/verdie,0.9902895891,2.48.241.59,"{""location"": ""KP"", ""is_mobile"": true}" 9094,4,204,2017-03-16 05:01:00,http://wyman.co/donato,0.6453332985,194.117.227.253,"{""location"": ""PN"", ""is_mobile"": false}" 9095,4,204,2017-05-26 20:42:21,http://rolfsonkuhic.io/aleandra.spinka,0.0745380998,55.56.204.83,"{""location"": ""GQ"", ""is_mobile"": false}" 9096,4,204,2017-05-28 01:59:40,http://rueckertrantow.io/cicero,0.3649383382,206.33.100.79,"{""location"": ""AM"", ""is_mobile"": false}" 9097,4,204,2017-01-24 23:52:11,http://cronajast.biz/price_cummerata,0.6995824441,154.145.129.85,"{""location"": ""LA"", ""is_mobile"": false}" 9098,4,204,2017-03-10 00:25:37,http://schuster.io/dortha.halvorson,0.5355483849,52.166.185.20,"{""location"": ""SG"", ""is_mobile"": true}" 9099,4,204,2017-01-10 05:28:28,http://nicolas.name/dane,0.1501250828,40.9.56.68,"{""location"": ""BG"", ""is_mobile"": false}" 9100,4,204,2016-12-20 06:22:07,http://grimes.name/joanne_runolfon,0.2873279056,176.155.114.104,"{""location"": ""CY"", ""is_mobile"": false}" 9101,4,204,2017-06-13 06:13:07,http://medhurst.co/reagan,0.8320412793,139.211.192.242,"{""location"": ""BR"", ""is_mobile"": true}" 9102,4,204,2016-12-19 16:22:27,http://friesenmarquardt.com/daniela.nitzsche,0.9209833617,143.231.142.64,"{""location"": ""GN"", ""is_mobile"": true}" 9103,4,204,2017-02-19 18:28:29,http://thiel.biz/noemy,0.6742106266,218.46.216.232,"{""location"": ""AW"", ""is_mobile"": false}" 9104,4,204,2017-04-27 01:01:50,http://oreilly.biz/ellsworth.ferry,0.2251240793,52.195.234.77,"{""location"": ""CN"", ""is_mobile"": false}" 9105,4,204,2017-04-09 17:05:07,http://doyle.info/conor.hayes,0.8176178279,143.220.44.158,"{""location"": ""SH"", ""is_mobile"": true}" 9106,4,204,2016-12-28 05:32:18,http://quitzonokeefe.net/emmanuel,0.1876087604,45.35.5.200,"{""location"": ""IL"", ""is_mobile"": false}" 9107,4,204,2017-06-11 15:12:57,http://schoen.co/douglas,0.1378070040,207.84.226.223,"{""location"": ""ER"", ""is_mobile"": true}" 9108,4,204,2017-04-08 16:06:47,http://steuberfeeney.info/glenna,0.8528889640,93.248.14.183,"{""location"": ""AU"", ""is_mobile"": true}" 9109,4,204,2017-01-11 17:00:35,http://willms.net/haley,0.6451649344,204.79.79.27,"{""location"": ""BN"", ""is_mobile"": false}" 9110,4,204,2016-12-19 15:26:01,http://turcotteconsidine.biz/casimer.veum,0.6253840699,28.229.244.45,"{""location"": ""HU"", ""is_mobile"": true}" 9111,4,204,2016-12-31 12:08:17,http://kozey.io/crystel_hermiston,0.5209262356,84.155.43.138,"{""location"": ""PG"", ""is_mobile"": false}" 9112,4,204,2016-12-30 05:03:54,http://weimann.info/isadore,0.4748274417,143.96.52.223,"{""location"": ""UG"", ""is_mobile"": false}" 10759,4,241,2017-05-11 16:46:25,http://boehm.co/hillary,,26.241.152.218,"{""location"": ""NO"", ""is_mobile"": true}" 9113,4,204,2017-04-06 13:48:11,http://leannonjenkins.co/korey,0.2447610148,180.253.223.29,"{""location"": ""ST"", ""is_mobile"": false}" 9114,4,204,2017-05-30 18:43:06,http://goldnerreinger.org/tate.boehm,0.8376861122,56.157.116.143,"{""location"": ""HT"", ""is_mobile"": true}" 9115,4,204,2017-01-21 21:10:15,http://bednargraham.info/janea_hettinger,0.3473730567,30.122.208.222,"{""location"": ""MR"", ""is_mobile"": false}" 9116,4,204,2017-03-09 11:26:48,http://grahamthompson.co/ally,0.2243270024,247.132.216.127,"{""location"": ""VE"", ""is_mobile"": false}" 9117,4,204,2017-05-04 05:14:42,http://beatty.net/shyann,0.0799632838,251.126.113.82,"{""location"": ""AI"", ""is_mobile"": true}" 9118,4,204,2016-12-22 23:12:14,http://hilll.net/horace_lockman,0.5890702029,131.76.104.121,"{""location"": ""LU"", ""is_mobile"": true}" 9119,4,204,2017-05-06 16:33:13,http://lind.io/deondre,0.4124258251,223.123.60.135,"{""location"": ""PG"", ""is_mobile"": false}" 9120,4,204,2017-05-22 06:19:13,http://mcculloughschuppe.org/ursula,0.2139264179,163.44.4.55,"{""location"": ""NA"", ""is_mobile"": false}" 9121,4,204,2017-04-03 08:39:50,http://mcglynn.co/cole,0.8164851359,34.117.113.83,"{""location"": ""CW"", ""is_mobile"": false}" 9122,4,204,2017-01-29 07:26:22,http://mosciski.name/chelsea.kuphal,0.0146069227,33.136.169.253,"{""location"": ""AE"", ""is_mobile"": true}" 9123,4,204,2017-01-09 08:36:17,http://eichmann.net/howell,0.3949973363,250.203.152.38,"{""location"": ""FI"", ""is_mobile"": false}" 9124,4,204,2017-01-27 17:00:34,http://hicklejerde.co/delphine,0.3867595613,53.207.124.240,"{""location"": ""BG"", ""is_mobile"": false}" 9125,4,204,2017-06-07 13:09:40,http://harvey.info/harley_durgan,0.6820768489,13.213.188.189,"{""location"": ""KP"", ""is_mobile"": true}" 9126,4,204,2017-02-13 14:08:57,http://spinka.net/taryn,0.6623359844,216.82.141.238,"{""location"": ""PM"", ""is_mobile"": false}" 9127,4,204,2017-05-27 09:07:26,http://kuhnbahringer.co/elia,0.5511253862,197.4.46.55,"{""location"": ""MP"", ""is_mobile"": true}" 9128,4,204,2017-06-02 03:38:17,http://kochlowe.biz/sterling,0.1485038484,23.213.114.203,"{""location"": ""PF"", ""is_mobile"": true}" 9129,4,204,2017-05-23 20:48:41,http://hilll.org/weldon,0.7037941575,143.63.77.121,"{""location"": ""SJ"", ""is_mobile"": false}" 9130,4,204,2017-05-27 05:24:51,http://wuckert.biz/jerome,0.6226746770,200.30.108.167,"{""location"": ""UZ"", ""is_mobile"": true}" 9131,4,204,2017-03-08 11:10:52,http://senger.biz/kelley,0.8783279862,83.35.232.21,"{""location"": ""AE"", ""is_mobile"": true}" 9132,4,204,2017-04-01 08:27:48,http://dickirogahn.org/kariane.prohaska,0.4187568201,67.116.84.95,"{""location"": ""CF"", ""is_mobile"": true}" 9133,4,204,2016-12-18 01:31:54,http://medhurst.co/micheal.donnelly,0.8638965760,160.180.191.214,"{""location"": ""GS"", ""is_mobile"": true}" 9134,4,204,2017-02-17 23:23:08,http://hayes.biz/rosina,0.9905281770,134.9.55.74,"{""location"": ""SX"", ""is_mobile"": true}" 9135,4,204,2017-02-12 22:51:33,http://rice.org/afton.ruecker,0.2215904570,109.73.20.220,"{""location"": ""GE"", ""is_mobile"": true}" 9136,4,204,2017-05-05 13:32:33,http://gulgowskicollins.org/aleandro_legros,0.3413409497,151.222.108.12,"{""location"": ""PE"", ""is_mobile"": true}" 9137,4,204,2017-06-14 04:00:23,http://kozey.io/uriel,0.5283781357,85.183.101.105,"{""location"": ""KH"", ""is_mobile"": true}" 9138,4,204,2017-03-13 08:20:29,http://huelsmurray.biz/fernando_senger,0.4423401693,140.113.215.34,"{""location"": ""CH"", ""is_mobile"": false}" 9139,4,204,2016-12-20 16:45:45,http://schoen.org/leone_torphy,0.4349613211,32.203.42.227,"{""location"": ""IQ"", ""is_mobile"": true}" 9140,4,205,2017-05-15 09:46:02,http://green.info/bettye.terry,0.8928266414,82.130.187.195,"{""location"": ""SI"", ""is_mobile"": false}" 9141,4,205,2017-06-11 01:12:03,http://hintz.info/april,0.5060247482,153.113.118.140,"{""location"": ""ST"", ""is_mobile"": true}" 9142,4,205,2017-03-06 07:39:48,http://ernser.co/lina,0.0040075619,243.117.179.33,"{""location"": ""CX"", ""is_mobile"": false}" 6093,3,132,2017-05-16 08:12:34,http://blandakilback.name/ephraim,,114.80.71.42,"{""location"": ""AG"", ""is_mobile"": true}" 6094,3,132,2017-04-28 00:42:32,http://kaulkeko.info/abby,,45.252.254.246,"{""location"": ""TZ"", ""is_mobile"": true}" 6095,3,132,2017-05-17 04:31:04,http://olsonmcclure.biz/gwendolyn_ruel,,35.198.64.144,"{""location"": ""NU"", ""is_mobile"": true}" 6096,3,132,2017-05-03 16:44:29,http://ferry.com/jude_ondricka,,152.118.62.27,"{""location"": ""MY"", ""is_mobile"": false}" 6097,3,132,2017-01-05 16:21:45,http://hilpertmarvin.org/alena_emmerich,,166.53.86.25,"{""location"": ""TH"", ""is_mobile"": true}" 6098,3,132,2017-02-14 06:03:57,http://dietrichjerde.org/lane.stroman,,87.68.10.254,"{""location"": ""LI"", ""is_mobile"": false}" 6099,3,132,2017-03-01 19:55:16,http://kris.co/maudie,,117.62.93.218,"{""location"": ""CV"", ""is_mobile"": false}" 6100,3,132,2017-06-13 20:50:58,http://heathcote.com/adrianna_feeney,,215.131.78.117,"{""location"": ""SI"", ""is_mobile"": true}" 6101,3,132,2017-04-19 23:28:00,http://runolfsdottir.io/joel,,208.19.231.234,"{""location"": ""TM"", ""is_mobile"": true}" 6102,3,132,2017-06-08 05:52:54,http://oharanicolas.net/deon_cormier,,234.61.199.61,"{""location"": ""MD"", ""is_mobile"": true}" 6103,3,132,2017-05-27 20:15:41,http://weberbrown.org/cathrine_fay,,47.156.195.251,"{""location"": ""IT"", ""is_mobile"": false}" 6104,3,132,2017-05-26 12:33:28,http://gislasonauer.name/jenifer,,133.250.164.234,"{""location"": ""BH"", ""is_mobile"": false}" 6105,3,132,2017-05-28 06:25:24,http://ratke.info/jadon,,241.188.163.245,"{""location"": ""GQ"", ""is_mobile"": true}" 6106,3,132,2017-01-28 14:29:53,http://schultzpaucek.name/florence.kutch,,157.44.206.94,"{""location"": ""AI"", ""is_mobile"": true}" 6107,3,132,2017-05-19 21:26:31,http://heel.com/odell.kunze,,98.184.77.50,"{""location"": ""CN"", ""is_mobile"": false}" 6108,3,132,2017-03-02 17:00:22,http://connellyyost.info/colt_moriette,,183.113.73.101,"{""location"": ""AE"", ""is_mobile"": true}" 6109,3,133,2017-05-01 12:18:16,http://hartmannfeest.name/oswaldo_carter,,226.38.228.210,"{""location"": ""IS"", ""is_mobile"": true}" 6110,3,133,2017-02-01 03:41:43,http://hyattgoodwin.org/aylin_langworth,,173.10.158.75,"{""location"": ""SN"", ""is_mobile"": true}" 6111,3,133,2017-04-22 21:33:47,http://homenickkovacek.io/coleman.abernathy,,235.193.87.49,"{""location"": ""LI"", ""is_mobile"": false}" 6112,3,133,2017-06-13 15:13:41,http://rauboehm.co/kailee,,25.42.223.85,"{""location"": ""SV"", ""is_mobile"": false}" 6113,3,133,2016-12-17 22:53:32,http://bode.info/norwood_gulgowski,,161.127.175.132,"{""location"": ""AF"", ""is_mobile"": true}" 6114,3,133,2017-01-05 19:02:45,http://walsh.com/earnestine,,234.7.156.33,"{""location"": ""BB"", ""is_mobile"": true}" 6115,3,133,2016-12-30 15:06:35,http://markswalker.io/larry_hauck,,244.149.84.120,"{""location"": ""NA"", ""is_mobile"": false}" 6116,3,133,2017-04-09 11:38:17,http://dubuque.biz/emely_macejkovic,,112.229.95.220,"{""location"": ""BY"", ""is_mobile"": true}" 6117,3,133,2017-05-07 09:19:24,http://wiza.biz/andre.predovic,,135.214.189.148,"{""location"": ""HU"", ""is_mobile"": false}" 6118,3,133,2017-02-09 13:36:31,http://sporer.name/derrick,,30.240.175.164,"{""location"": ""BQ"", ""is_mobile"": false}" 6119,3,133,2017-04-09 23:23:20,http://strosin.co/delphia,,146.51.178.26,"{""location"": ""MZ"", ""is_mobile"": true}" 6120,3,133,2017-05-13 14:51:32,http://macejkovic.io/stone,,186.142.82.112,"{""location"": ""CG"", ""is_mobile"": true}" 6121,3,133,2017-02-02 15:56:31,http://faheykozey.org/naomie,,199.36.136.61,"{""location"": ""NR"", ""is_mobile"": true}" 6122,3,133,2017-02-07 13:54:50,http://klein.org/vance,,51.92.210.113,"{""location"": ""KN"", ""is_mobile"": true}" 6123,3,133,2016-12-17 04:13:00,http://braunbauch.name/walton,,12.249.173.224,"{""location"": ""PN"", ""is_mobile"": true}" 6124,3,133,2017-01-10 21:43:17,http://schoen.net/jensen,,176.155.137.9,"{""location"": ""NP"", ""is_mobile"": true}" 6125,3,133,2017-06-10 10:04:57,http://hansenfranecki.io/bridgette_franecki,,165.45.150.245,"{""location"": ""GE"", ""is_mobile"": false}" 6126,3,133,2017-04-01 20:56:54,http://hickle.net/horacio.littel,,88.242.103.199,"{""location"": ""KR"", ""is_mobile"": true}" 6127,3,133,2017-01-01 19:12:20,http://koelpin.info/elwin,,8.124.44.22,"{""location"": ""MP"", ""is_mobile"": true}" 6128,3,133,2017-05-06 04:32:30,http://durgan.co/audra,,241.19.231.187,"{""location"": ""PL"", ""is_mobile"": true}" 6129,3,133,2017-02-18 03:21:26,http://torp.net/gene_farrell,,92.97.189.7,"{""location"": ""TT"", ""is_mobile"": false}" 6130,3,133,2017-03-13 09:36:51,http://homenick.io/kay.yost,,171.161.91.160,"{""location"": ""AX"", ""is_mobile"": false}" 6131,3,133,2017-01-06 19:26:53,http://runteheaney.co/hilda.hahn,,233.99.173.23,"{""location"": ""MY"", ""is_mobile"": true}" 6132,3,133,2016-12-26 12:05:00,http://hudson.io/maurine_greenfelder,,19.66.162.11,"{""location"": ""TD"", ""is_mobile"": false}" 6133,3,133,2017-05-28 07:33:32,http://block.info/josephine_murazik,,62.103.137.223,"{""location"": ""CD"", ""is_mobile"": true}" 6134,3,133,2016-12-21 16:32:05,http://weinat.com/shane,,139.216.229.150,"{""location"": ""MH"", ""is_mobile"": false}" 6135,3,133,2017-02-28 22:06:51,http://upton.name/nels_koelpin,,233.134.114.66,"{""location"": ""SH"", ""is_mobile"": false}" 6136,3,133,2017-05-26 14:52:37,http://jast.info/bennett,,92.49.204.226,"{""location"": ""BY"", ""is_mobile"": true}" 6137,3,133,2017-03-26 19:29:35,http://lubowitz.org/angelita_murazik,,103.80.106.182,"{""location"": ""PT"", ""is_mobile"": true}" 6138,3,133,2017-04-28 07:45:21,http://rau.net/harold_pacocha,,196.154.160.243,"{""location"": ""LT"", ""is_mobile"": false}" 6139,3,133,2017-03-19 03:11:42,http://gusikowski.co/triston_witting,,243.253.15.156,"{""location"": ""SE"", ""is_mobile"": false}" 6140,3,133,2017-05-20 16:36:23,http://smith.org/saul.glover,,219.239.243.109,"{""location"": ""BI"", ""is_mobile"": true}" 6141,3,133,2017-02-21 11:42:12,http://ward.info/al_balistreri,,22.39.239.144,"{""location"": ""LC"", ""is_mobile"": false}" 6142,3,133,2017-01-30 03:51:17,http://blick.org/charlotte_kunde,,60.6.175.130,"{""location"": ""CM"", ""is_mobile"": true}" 6143,3,133,2016-12-16 21:49:25,http://kertzmann.co/jewell_mayer,,6.50.44.114,"{""location"": ""KW"", ""is_mobile"": true}" 6144,3,133,2017-02-12 23:46:31,http://boyle.info/deja,,2.66.252.22,"{""location"": ""ZA"", ""is_mobile"": false}" 6145,3,134,2017-06-13 20:38:15,http://rodriguez.info/lorenza_botsford,,76.214.60.18,"{""location"": ""NF"", ""is_mobile"": true}" 6146,3,134,2017-01-12 07:52:08,http://rosenbaum.name/leopold.dietrich,,72.124.59.45,"{""location"": ""GA"", ""is_mobile"": true}" 6147,3,134,2017-02-05 18:55:11,http://kerlukesauer.com/roselyn,,161.107.204.3,"{""location"": ""IE"", ""is_mobile"": true}" 12008,5,269,2017-02-11 15:57:11,http://kiehn.com/moshe_ritchie,0.2543979137,131.200.17.140,"{""location"": ""US"", ""is_mobile"": true}" 12009,5,269,2017-01-18 21:59:39,http://tillman.com/scotty.wilderman,0.5149678327,122.206.98.226,"{""location"": ""MF"", ""is_mobile"": false}" 12010,5,269,2017-06-08 18:19:06,http://heidenreich.name/rosa,0.1490686367,10.28.193.13,"{""location"": ""KY"", ""is_mobile"": false}" 12011,5,269,2017-05-21 02:43:15,http://ryan.io/zora_marquardt,0.1009912442,29.253.101.128,"{""location"": ""LA"", ""is_mobile"": true}" 12012,5,269,2017-03-09 02:18:22,http://kunzeschaden.biz/jaclyn.schaefer,0.5978221576,249.189.197.31,"{""location"": ""BY"", ""is_mobile"": false}" 12013,5,269,2017-02-21 13:54:13,http://daniel.org/gia_strosin,0.5639718745,31.41.210.140,"{""location"": ""CM"", ""is_mobile"": true}" 12014,5,269,2017-05-05 06:45:00,http://cummingsmayer.io/paula,0.3270419902,214.160.97.230,"{""location"": ""CM"", ""is_mobile"": true}" 12015,5,269,2017-04-13 11:33:23,http://cainwisoky.co/nova,0.0981890187,126.176.212.65,"{""location"": ""SL"", ""is_mobile"": false}" 12016,5,269,2016-12-23 00:26:30,http://zboncak.biz/jennyfer_bergnaum,0.0487299878,45.242.230.126,"{""location"": ""SG"", ""is_mobile"": true}" 12017,5,269,2017-02-01 08:19:28,http://lockman.co/hector.corkery,0.0078371218,239.159.232.181,"{""location"": ""NR"", ""is_mobile"": true}" 12018,5,269,2017-05-13 04:06:16,http://murazik.com/kennedy,0.8326550255,77.58.47.73,"{""location"": ""CI"", ""is_mobile"": true}" 12019,5,269,2017-03-23 17:47:37,http://cain.io/demarcus,0.2960890938,98.31.9.45,"{""location"": ""BM"", ""is_mobile"": true}" 12020,5,269,2017-03-12 09:19:12,http://kub.info/nicola.kozey,0.6201079113,71.81.205.96,"{""location"": ""FJ"", ""is_mobile"": false}" 12021,5,269,2017-05-14 03:02:28,http://jast.name/macey,0.4924928829,35.241.171.12,"{""location"": ""TL"", ""is_mobile"": false}" 12022,5,269,2017-02-14 01:21:05,http://feeney.net/jaclyn,0.1797621070,253.128.217.182,"{""location"": ""KW"", ""is_mobile"": true}" 12023,5,269,2017-03-12 21:36:08,http://runolfsdottir.info/deangelo,0.1952823653,252.205.153.190,"{""location"": ""AF"", ""is_mobile"": true}" 12024,5,269,2017-02-11 06:48:15,http://tremblay.org/katheryn,0.5578453583,129.84.35.213,"{""location"": ""AO"", ""is_mobile"": true}" 12025,5,269,2017-04-16 20:12:53,http://damore.net/paxton_rempel,0.8417161320,38.152.50.205,"{""location"": ""VU"", ""is_mobile"": false}" 12026,5,269,2017-01-12 12:04:19,http://harris.net/eldred_schmidt,0.8647438520,99.35.226.59,"{""location"": ""ZA"", ""is_mobile"": true}" 12027,5,269,2017-06-12 09:21:44,http://donnelly.co/delores,0.5918832636,150.77.205.227,"{""location"": ""GL"", ""is_mobile"": true}" 12028,5,269,2017-02-11 20:57:32,http://macgyver.name/brayan,0.0232875349,238.57.203.153,"{""location"": ""BY"", ""is_mobile"": true}" 12029,5,269,2017-05-20 04:55:28,http://mckenzie.org/enrique,0.3235847542,198.109.200.199,"{""location"": ""IL"", ""is_mobile"": true}" 12030,5,269,2017-01-19 22:08:17,http://mueller.biz/beth.denesik,0.6415754576,232.142.36.63,"{""location"": ""CF"", ""is_mobile"": false}" 12031,5,269,2017-04-18 11:36:52,http://nienow.co/charles.wehner,0.4728163508,249.18.30.32,"{""location"": ""MO"", ""is_mobile"": false}" 12032,5,269,2017-03-01 19:30:52,http://gerlach.com/jairo,0.2542180712,250.105.132.237,"{""location"": ""GM"", ""is_mobile"": false}" 12033,5,269,2017-03-01 09:06:47,http://heidenreich.info/bradford,0.1404943559,193.198.155.100,"{""location"": ""UY"", ""is_mobile"": true}" 12034,5,269,2017-01-03 12:00:29,http://stiedemann.biz/delmer,0.7344922997,225.41.132.162,"{""location"": ""YE"", ""is_mobile"": false}" 12035,5,269,2017-01-03 04:06:27,http://rowe.net/britney,0.9992014842,123.29.203.90,"{""location"": ""MH"", ""is_mobile"": false}" 12036,5,269,2017-01-02 06:47:44,http://rutherford.name/crystal_dubuque,0.2552458318,220.181.231.72,"{""location"": ""CC"", ""is_mobile"": true}" 12037,5,269,2017-04-30 21:44:52,http://bernhard.net/roy,0.9728514541,192.238.199.150,"{""location"": ""TT"", ""is_mobile"": true}" 12038,5,269,2017-03-03 15:11:03,http://jerdevandervort.biz/providenci,0.7962400217,164.211.90.32,"{""location"": ""BV"", ""is_mobile"": false}" 12039,5,269,2017-02-23 07:41:47,http://wilkinson.org/ellen_yost,0.7300726250,104.127.48.11,"{""location"": ""EH"", ""is_mobile"": true}" 12040,5,269,2017-04-16 09:04:14,http://zemlakstokes.com/trisha,0.4945305453,174.216.141.60,"{""location"": ""LB"", ""is_mobile"": false}" 12041,5,269,2017-01-20 18:57:08,http://stehr.org/amanda.vandervort,0.8990267437,101.243.115.11,"{""location"": ""UY"", ""is_mobile"": true}" 12042,5,269,2017-02-04 08:12:20,http://white.co/kamron,0.7592601812,34.59.181.170,"{""location"": ""LR"", ""is_mobile"": false}" 12043,5,269,2017-02-10 18:02:09,http://flatley.org/clotilde,0.7576953982,60.59.114.168,"{""location"": ""TR"", ""is_mobile"": false}" 12044,5,269,2017-06-05 11:59:13,http://krajcik.org/lysanne,0.0515529037,25.143.50.128,"{""location"": ""AL"", ""is_mobile"": false}" 12045,5,269,2017-04-04 17:23:14,http://ward.name/tad.okuneva,0.3121969329,38.9.220.8,"{""location"": ""BJ"", ""is_mobile"": true}" 12046,5,269,2017-01-01 12:32:49,http://stracke.name/freida_bayer,0.6963837850,37.115.199.108,"{""location"": ""BA"", ""is_mobile"": false}" 12047,5,269,2017-03-21 13:22:31,http://ondrickareilly.org/caria_schoen,0.8335077512,81.98.214.147,"{""location"": ""BW"", ""is_mobile"": true}" 12048,5,269,2017-05-13 19:14:05,http://west.info/sibyl_cummerata,0.5105976112,50.67.30.126,"{""location"": ""MS"", ""is_mobile"": true}" 12049,5,269,2017-01-06 09:14:13,http://brakuspfeffer.name/lavada.graham,0.9282154430,215.198.35.11,"{""location"": ""LV"", ""is_mobile"": true}" 12050,5,269,2017-01-06 05:57:19,http://spinka.net/earline,0.5674510161,161.9.133.164,"{""location"": ""PH"", ""is_mobile"": false}" 12051,5,269,2017-01-01 20:09:50,http://hoegerlakin.biz/damion.kuhlman,0.0548341439,140.223.161.238,"{""location"": ""IT"", ""is_mobile"": true}" 12052,5,269,2017-03-28 20:01:53,http://vonrueden.net/rosalind_bernier,0.6713182914,60.128.216.210,"{""location"": ""UZ"", ""is_mobile"": true}" 12053,5,269,2017-02-17 09:01:39,http://stark.net/savion,0.3978664781,235.226.98.225,"{""location"": ""CV"", ""is_mobile"": false}" 12054,5,269,2017-03-11 09:21:10,http://keebler.name/estell,0.4396447672,171.35.172.248,"{""location"": ""PN"", ""is_mobile"": false}" 12055,5,269,2017-02-10 23:04:58,http://mclaughlinquigley.biz/breanna.larson,0.8226971841,212.250.16.152,"{""location"": ""IN"", ""is_mobile"": true}" 12056,5,269,2017-01-04 21:25:52,http://yost.com/kacie,0.6667122349,34.128.104.106,"{""location"": ""BG"", ""is_mobile"": false}" 12057,5,269,2017-04-24 17:22:36,http://mills.com/emerson,0.4601424190,120.194.97.90,"{""location"": ""BF"", ""is_mobile"": false}" 12058,5,269,2017-04-04 18:23:08,http://ward.org/catherine_dach,0.7773580455,60.199.118.135,"{""location"": ""FR"", ""is_mobile"": false}" 14997,6,337,2016-12-30 00:51:17,http://daniel.com/waylon,,138.243.222.160,"{""location"": ""BH"", ""is_mobile"": false}" 14998,6,337,2017-02-19 17:29:09,http://walter.net/lesley,,23.189.98.129,"{""location"": ""MH"", ""is_mobile"": false}" 14999,6,337,2017-05-28 01:31:29,http://zboncak.name/toby_dickinson,,214.138.103.32,"{""location"": ""PR"", ""is_mobile"": true}" 15000,6,337,2017-04-25 03:28:10,http://boyer.com/parker_hintz,,24.82.150.179,"{""location"": ""SO"", ""is_mobile"": true}" 15001,6,337,2017-04-25 09:28:42,http://schowalter.info/ole,,60.142.121.14,"{""location"": ""SK"", ""is_mobile"": true}" 15002,6,337,2017-01-17 19:58:48,http://abshire.biz/antonietta,,195.161.66.124,"{""location"": ""LI"", ""is_mobile"": false}" 15003,6,338,2017-05-14 09:47:47,http://weber.org/rita,,215.125.245.40,"{""location"": ""SN"", ""is_mobile"": false}" 15004,6,338,2017-04-08 10:40:08,http://hilpertgoyette.org/estelle.heller,,83.247.236.89,"{""location"": ""PW"", ""is_mobile"": false}" 15005,6,338,2017-03-26 05:32:03,http://bogisich.org/gilda,,116.133.126.113,"{""location"": ""IN"", ""is_mobile"": true}" 15006,6,338,2016-12-19 18:36:32,http://hauckhickle.co/zoe,,66.219.202.240,"{""location"": ""SK"", ""is_mobile"": true}" 15007,6,338,2017-01-24 02:57:37,http://eichmann.name/zaria,,173.134.88.170,"{""location"": ""BD"", ""is_mobile"": false}" 15008,6,338,2017-06-06 16:28:05,http://davis.org/tiana,,249.110.176.162,"{""location"": ""EG"", ""is_mobile"": true}" 15009,6,338,2017-03-16 05:52:08,http://nienow.biz/carlie.cummerata,,107.23.69.223,"{""location"": ""PR"", ""is_mobile"": false}" 15010,6,338,2017-02-09 12:21:28,http://handgoodwin.biz/allie,,224.168.32.19,"{""location"": ""VE"", ""is_mobile"": true}" 15011,6,338,2017-01-05 15:42:22,http://robel.co/delilah,,28.178.118.74,"{""location"": ""JO"", ""is_mobile"": false}" 15012,6,338,2017-04-04 19:02:46,http://feest.com/haan_white,,238.78.253.135,"{""location"": ""CK"", ""is_mobile"": false}" 15013,6,338,2017-01-05 11:45:56,http://rolfson.com/meda.lakin,,92.151.149.17,"{""location"": ""SO"", ""is_mobile"": true}" 15014,6,338,2017-05-26 08:51:38,http://reynolds.com/tyler,,45.56.195.83,"{""location"": ""SC"", ""is_mobile"": true}" 15015,6,338,2017-01-14 17:25:20,http://klocko.org/thora_rogahn,,4.87.145.205,"{""location"": ""MQ"", ""is_mobile"": true}" 15016,6,338,2017-04-08 23:56:14,http://baumbachbernier.biz/valentina,,57.59.88.123,"{""location"": ""TJ"", ""is_mobile"": true}" 15017,6,338,2017-01-11 12:18:10,http://roob.name/tiara,,222.86.110.184,"{""location"": ""SM"", ""is_mobile"": false}" 15018,6,338,2017-05-14 16:09:02,http://streich.biz/charlotte.vonrueden,,84.90.175.173,"{""location"": ""PH"", ""is_mobile"": true}" 15019,6,338,2017-04-03 05:19:46,http://gleason.co/shawna,,207.92.131.115,"{""location"": ""MU"", ""is_mobile"": true}" 15020,6,338,2017-03-27 16:40:27,http://stracketowne.info/emerald_hilll,,94.238.131.59,"{""location"": ""YE"", ""is_mobile"": true}" 15021,6,338,2017-01-28 20:18:39,http://welchweber.name/jonatan,,224.203.231.120,"{""location"": ""EG"", ""is_mobile"": false}" 15022,6,338,2017-01-18 09:08:26,http://hagenes.org/kurt,,239.114.192.4,"{""location"": ""LR"", ""is_mobile"": true}" 15023,6,338,2017-03-06 12:01:27,http://beahanferry.org/kayla,,53.53.30.37,"{""location"": ""BB"", ""is_mobile"": false}" 15024,6,338,2017-03-07 19:01:00,http://windler.io/lucienne_kerluke,,168.193.107.244,"{""location"": ""NP"", ""is_mobile"": true}" 15025,6,338,2017-03-12 01:06:23,http://waelchi.name/efrain,,212.104.95.156,"{""location"": ""AG"", ""is_mobile"": false}" 15026,6,338,2017-04-20 16:57:59,http://connelly.biz/chesley_dibbert,,235.146.251.250,"{""location"": ""LS"", ""is_mobile"": true}" 15027,6,338,2017-04-23 22:01:26,http://runolfonrodriguez.name/uriah,,72.109.165.202,"{""location"": ""MM"", ""is_mobile"": true}" 15028,6,338,2017-04-23 08:22:48,http://mcglynn.io/lorenzo,,69.106.73.91,"{""location"": ""MG"", ""is_mobile"": true}" 15029,6,338,2017-02-13 20:01:54,http://sipes.io/shawn,,238.68.118.192,"{""location"": ""KY"", ""is_mobile"": false}" 15030,6,338,2017-02-18 13:19:24,http://gottlieb.name/otho,,22.217.184.147,"{""location"": ""LR"", ""is_mobile"": false}" 15031,6,338,2017-04-16 22:46:05,http://marquardtcain.io/trycia_flatley,,156.122.132.173,"{""location"": ""CV"", ""is_mobile"": false}" 15032,6,338,2017-03-17 21:16:51,http://braunterry.io/kasey.reilly,,124.3.57.132,"{""location"": ""LU"", ""is_mobile"": true}" 15033,6,338,2017-03-13 13:16:57,http://corwin.org/axel_hayes,,76.51.193.17,"{""location"": ""MN"", ""is_mobile"": true}" 15034,6,338,2017-04-08 08:40:46,http://vandervort.biz/hallie,,46.70.129.74,"{""location"": ""KP"", ""is_mobile"": false}" 15035,6,338,2017-01-13 14:30:26,http://murazikstoltenberg.com/gertrude_crona,,250.192.216.91,"{""location"": ""SV"", ""is_mobile"": true}" 15036,6,338,2017-01-20 17:41:05,http://feestfeeney.name/aylin.yost,,76.35.199.180,"{""location"": ""RW"", ""is_mobile"": false}" 15037,6,338,2017-04-02 14:36:16,http://champlinschowalter.org/gabe,,167.212.161.36,"{""location"": ""MC"", ""is_mobile"": false}" 15038,6,338,2016-12-31 18:09:59,http://bergnaumdicki.com/hardy.jaskolski,,96.41.181.214,"{""location"": ""IT"", ""is_mobile"": true}" 15039,6,338,2017-04-08 12:29:34,http://willms.org/jolie,,29.55.175.207,"{""location"": ""KW"", ""is_mobile"": false}" 15040,6,338,2017-04-06 19:21:49,http://ruel.com/leonard,,80.34.224.191,"{""location"": ""DK"", ""is_mobile"": true}" 15041,6,338,2017-05-04 03:59:33,http://lehner.biz/reymundo,,219.129.191.19,"{""location"": ""CR"", ""is_mobile"": true}" 15042,6,338,2016-12-30 04:33:49,http://lang.co/amanda.blick,,37.79.215.75,"{""location"": ""BW"", ""is_mobile"": false}" 15043,6,338,2017-02-06 14:36:22,http://strosin.biz/roberto_gorczany,,155.52.227.214,"{""location"": ""SC"", ""is_mobile"": true}" 15044,6,338,2016-12-17 16:17:22,http://satterfield.info/dolores_mckenzie,,68.49.46.180,"{""location"": ""CR"", ""is_mobile"": false}" 15045,6,338,2017-04-23 05:02:54,http://adamsbruen.io/roma.block,,216.36.32.89,"{""location"": ""ME"", ""is_mobile"": false}" 15046,6,338,2017-01-18 04:13:39,http://blanda.net/dereck,,161.238.93.222,"{""location"": ""TT"", ""is_mobile"": false}" 15047,6,338,2017-04-19 09:15:51,http://botsford.org/selina.gutkowski,,49.253.56.215,"{""location"": ""BM"", ""is_mobile"": false}" 15048,6,338,2017-03-30 09:02:20,http://colesipes.org/effie,,211.212.159.250,"{""location"": ""AE"", ""is_mobile"": true}" 15049,6,338,2017-02-20 20:49:29,http://jaskolskitillman.name/tianna,,161.3.127.35,"{""location"": ""CI"", ""is_mobile"": false}" 15050,6,338,2017-05-28 07:18:46,http://hegmann.info/joanie_auer,,117.129.152.32,"{""location"": ""PM"", ""is_mobile"": false}" 15051,6,338,2017-02-25 22:10:22,http://hyatt.name/esteban,,150.133.11.84,"{""location"": ""BY"", ""is_mobile"": false}" 15052,6,338,2017-05-29 01:25:39,http://keelingcremin.io/elsie_prosacco,,191.244.190.189,"{""location"": ""CV"", ""is_mobile"": false}" 9143,4,205,2017-04-22 06:40:29,http://bailey.name/reta,0.2256821006,102.165.138.69,"{""location"": ""NR"", ""is_mobile"": true}" 9144,4,205,2017-03-23 22:28:59,http://boehmbins.org/angela,0.9112830698,180.151.97.85,"{""location"": ""TK"", ""is_mobile"": true}" 9145,4,205,2017-05-08 20:40:47,http://goodwinhackett.name/christina.hills,0.4902191709,143.133.119.33,"{""location"": ""AE"", ""is_mobile"": true}" 9146,4,205,2017-01-21 01:07:52,http://kozey.name/alexa,0.5379756666,217.84.110.196,"{""location"": ""ML"", ""is_mobile"": false}" 9147,4,205,2017-04-06 10:43:38,http://grantcummerata.biz/river.stracke,0.0355948960,223.120.106.205,"{""location"": ""YE"", ""is_mobile"": false}" 9148,4,205,2016-12-22 21:29:10,http://klein.io/roma,0.7196241445,227.63.22.113,"{""location"": ""SN"", ""is_mobile"": false}" 9149,4,205,2017-01-29 15:45:40,http://dare.info/nikita,0.3442478471,173.196.195.187,"{""location"": ""CZ"", ""is_mobile"": false}" 9150,4,205,2016-12-16 06:11:47,http://dickinson.name/viviane,0.8937007900,226.210.217.85,"{""location"": ""RU"", ""is_mobile"": true}" 9151,4,205,2017-03-06 14:13:00,http://hegmannfisher.com/mya.rowe,0.2616255212,252.60.199.198,"{""location"": ""NZ"", ""is_mobile"": false}" 9152,4,205,2017-05-18 05:17:08,http://wisokysimonis.io/taryn_wolf,0.2010558656,214.209.16.46,"{""location"": ""TM"", ""is_mobile"": false}" 9153,4,205,2017-03-31 13:09:06,http://skileswuckert.io/trycia,0.9094021604,168.229.34.19,"{""location"": ""AQ"", ""is_mobile"": true}" 9154,4,205,2017-05-18 23:28:38,http://stanton.net/roie,0.3947653885,73.10.89.249,"{""location"": ""PS"", ""is_mobile"": true}" 9155,4,205,2017-04-04 11:43:15,http://homenicklynch.info/berneice_barton,0.5035305484,24.232.142.238,"{""location"": ""VE"", ""is_mobile"": true}" 9156,4,205,2017-01-18 11:27:24,http://satterfield.net/stefanie_nitzsche,0.5346291665,40.43.220.201,"{""location"": ""AO"", ""is_mobile"": true}" 9157,4,205,2017-01-12 21:38:16,http://crona.org/mike_west,0.5673669563,2.61.36.2,"{""location"": ""TR"", ""is_mobile"": false}" 9158,4,205,2017-02-14 17:02:28,http://oreilly.info/luigi_ebert,0.9677572493,226.195.218.215,"{""location"": ""MG"", ""is_mobile"": false}" 9159,4,205,2017-01-14 16:16:40,http://white.name/delbert,0.4220793030,111.253.221.113,"{""location"": ""BE"", ""is_mobile"": false}" 9160,4,205,2017-05-13 02:35:26,http://west.com/emmanuelle,0.7299551041,230.6.126.36,"{""location"": ""SM"", ""is_mobile"": false}" 9161,4,205,2017-03-10 16:58:40,http://pfeffersimonis.co/weston,0.5072743161,190.93.100.228,"{""location"": ""YT"", ""is_mobile"": true}" 9162,4,205,2017-05-26 10:04:43,http://okunevabeer.name/roie,0.8913576936,2.126.180.63,"{""location"": ""HR"", ""is_mobile"": false}" 9163,4,205,2017-05-14 07:26:14,http://welchheathcote.com/celestine,0.6944279246,131.77.152.139,"{""location"": ""NO"", ""is_mobile"": true}" 9164,4,205,2017-01-28 19:39:38,http://glover.biz/gabriel,0.2268501227,163.193.78.32,"{""location"": ""GT"", ""is_mobile"": false}" 9165,4,205,2017-03-09 06:59:01,http://franecki.io/vena.torp,0.7127921445,4.162.234.111,"{""location"": ""SM"", ""is_mobile"": false}" 9166,4,205,2017-05-11 14:06:26,http://powlowski.net/meagan,0.3337514132,58.9.14.89,"{""location"": ""GG"", ""is_mobile"": false}" 9167,4,205,2017-03-09 11:13:08,http://swaniawski.co/leola,0.2640261639,181.222.176.91,"{""location"": ""CY"", ""is_mobile"": true}" 9168,4,205,2017-03-01 00:31:04,http://feest.name/amya_jast,0.4031073256,169.170.182.250,"{""location"": ""GS"", ""is_mobile"": true}" 9169,4,205,2017-02-10 14:28:05,http://damore.co/marilou,0.6047272861,226.46.222.104,"{""location"": ""UY"", ""is_mobile"": false}" 9170,4,205,2017-05-09 08:51:40,http://dubuque.org/tristian_strosin,0.3059447626,245.93.4.238,"{""location"": ""UZ"", ""is_mobile"": true}" 9171,4,205,2017-06-06 04:52:05,http://jacobi.org/antonetta_ankunding,0.9318954889,93.254.55.34,"{""location"": ""GD"", ""is_mobile"": true}" 9172,4,205,2017-04-04 00:53:15,http://swaniawski.co/gaylord,0.1223763208,127.252.37.161,"{""location"": ""CD"", ""is_mobile"": true}" 9173,4,205,2017-05-08 11:05:13,http://schuppe.net/addison.bayer,0.8534238626,163.51.39.46,"{""location"": ""BM"", ""is_mobile"": true}" 9174,4,205,2017-04-20 02:33:01,http://murphy.name/felton.torp,0.2017467171,93.241.235.18,"{""location"": ""LI"", ""is_mobile"": false}" 9175,4,205,2017-05-22 23:32:52,http://feil.net/elisabeth,0.1549648509,170.80.119.84,"{""location"": ""GL"", ""is_mobile"": false}" 9176,4,205,2017-04-10 04:31:06,http://pfannerstill.name/lila_waters,0.6935064244,208.251.201.201,"{""location"": ""ET"", ""is_mobile"": false}" 9177,4,205,2017-05-29 21:21:16,http://purdynicolas.info/reece,0.0111927293,104.130.95.44,"{""location"": ""GA"", ""is_mobile"": false}" 9178,4,205,2017-03-13 14:47:00,http://bayer.org/mark,0.4648782469,237.95.120.142,"{""location"": ""RW"", ""is_mobile"": true}" 9179,4,205,2017-05-12 03:40:15,http://fahey.io/clovis,0.8989951313,124.97.233.109,"{""location"": ""CK"", ""is_mobile"": true}" 9180,4,205,2017-03-12 05:18:33,http://eichmannhalvorson.info/ike,0.8817087621,98.204.29.122,"{""location"": ""GI"", ""is_mobile"": true}" 9181,4,205,2017-01-18 17:25:55,http://jaskolski.io/wilford,0.8523479727,187.62.42.46,"{""location"": ""TZ"", ""is_mobile"": true}" 9182,4,205,2017-04-26 22:53:25,http://hagenes.net/lisandro_ferry,0.0084174318,105.231.142.219,"{""location"": ""AI"", ""is_mobile"": false}" 9183,4,205,2017-05-03 02:40:17,http://glover.name/annabel,0.5708733287,100.227.204.83,"{""location"": ""CV"", ""is_mobile"": true}" 9184,4,205,2017-04-29 11:24:08,http://schuster.org/corine,0.7316064770,218.70.232.69,"{""location"": ""TD"", ""is_mobile"": true}" 9185,4,205,2017-05-30 03:55:54,http://wuckert.org/bria_glover,0.5574463229,191.111.102.57,"{""location"": ""NC"", ""is_mobile"": false}" 9186,4,205,2017-02-19 23:35:53,http://kirlincruickshank.io/misael_schmidt,0.9549867210,79.128.120.213,"{""location"": ""GI"", ""is_mobile"": true}" 9187,4,205,2017-03-25 04:47:56,http://moore.name/ottilie_kuphal,0.3397509773,168.250.142.125,"{""location"": ""NI"", ""is_mobile"": true}" 9188,4,205,2017-01-14 03:24:01,http://tillman.co/susanna,0.3418296875,243.105.135.35,"{""location"": ""ZW"", ""is_mobile"": false}" 9189,4,205,2017-04-26 00:07:29,http://buckridge.info/lonny,0.8326977532,151.220.80.145,"{""location"": ""FJ"", ""is_mobile"": false}" 9190,4,205,2017-01-03 06:56:01,http://torp.org/jadon.monahan,0.5547797549,62.23.74.99,"{""location"": ""PY"", ""is_mobile"": true}" 9191,4,205,2017-02-08 02:24:35,http://schoen.io/demond,0.3486094667,121.68.40.233,"{""location"": ""AU"", ""is_mobile"": false}" 9192,4,205,2017-02-27 12:04:21,http://mclaughlin.net/johnathon_franecki,0.8613185552,43.13.196.73,"{""location"": ""AT"", ""is_mobile"": true}" 9193,4,205,2017-05-22 02:31:57,http://balistreri.io/jamil,0.3936402323,244.21.210.121,"{""location"": ""LB"", ""is_mobile"": true}" 6148,3,134,2017-04-18 10:56:49,http://leffler.net/eryn,,251.193.74.234,"{""location"": ""SI"", ""is_mobile"": true}" 6149,3,134,2017-04-10 23:29:42,http://abshire.info/furman.harris,,150.93.166.161,"{""location"": ""IL"", ""is_mobile"": true}" 6150,3,134,2017-04-09 00:52:19,http://hartmannweimann.biz/maryjane,,35.68.76.169,"{""location"": ""UG"", ""is_mobile"": false}" 6151,3,134,2017-02-13 22:22:41,http://keeblerhowe.org/lavada.trantow,,26.72.87.214,"{""location"": ""MH"", ""is_mobile"": false}" 6152,3,134,2017-04-05 00:44:44,http://king.io/carmine.lang,,72.196.31.196,"{""location"": ""PN"", ""is_mobile"": false}" 6153,3,134,2017-03-15 12:37:22,http://kunde.co/verla,,36.76.93.58,"{""location"": ""GW"", ""is_mobile"": true}" 6154,3,134,2017-02-07 01:27:11,http://connjaskolski.com/amiya.fahey,,77.244.174.21,"{""location"": ""AZ"", ""is_mobile"": true}" 6155,3,134,2016-12-27 06:01:08,http://nicolasbashirian.name/telly,,82.102.185.136,"{""location"": ""PK"", ""is_mobile"": false}" 6156,3,134,2017-03-01 19:38:05,http://fisherfadel.co/nathen_koch,,251.135.88.120,"{""location"": ""IT"", ""is_mobile"": false}" 6157,3,134,2017-06-06 14:01:47,http://schuppe.net/linwood_gottlieb,,25.149.251.252,"{""location"": ""GG"", ""is_mobile"": true}" 6158,3,134,2017-01-17 02:19:58,http://yost.io/addie,,46.43.193.176,"{""location"": ""TW"", ""is_mobile"": true}" 6159,3,134,2017-04-26 12:08:35,http://heathcote.io/irwin.schroeder,,76.226.101.50,"{""location"": ""CV"", ""is_mobile"": false}" 6160,3,134,2017-04-28 05:16:51,http://hicklethompson.name/hattie,,144.25.165.86,"{""location"": ""TT"", ""is_mobile"": false}" 6161,3,134,2017-02-02 12:23:54,http://spinka.biz/waylon.heller,,183.46.42.136,"{""location"": ""JE"", ""is_mobile"": true}" 6162,3,134,2017-05-23 15:51:29,http://ortiz.org/brandyn.vonrueden,,147.31.204.47,"{""location"": ""FK"", ""is_mobile"": true}" 6163,3,134,2017-01-10 03:56:47,http://runolfon.io/bette.emmerich,,22.122.7.161,"{""location"": ""GD"", ""is_mobile"": false}" 6164,3,134,2017-04-19 00:21:15,http://spinkafadel.net/tyra_prohaska,,149.27.84.61,"{""location"": ""TF"", ""is_mobile"": true}" 6165,3,134,2017-01-16 22:13:02,http://thompson.io/ted_halvorson,,110.119.144.137,"{""location"": ""TR"", ""is_mobile"": true}" 6166,3,134,2017-05-20 18:55:13,http://wilkinson.name/maiya.bosco,,124.189.71.202,"{""location"": ""RO"", ""is_mobile"": false}" 6167,3,134,2017-05-14 00:52:56,http://brekkewintheiser.com/colby.torphy,,63.5.69.149,"{""location"": ""MX"", ""is_mobile"": true}" 6168,3,134,2016-12-16 07:34:33,http://medhurstbartell.com/michael,,82.218.105.28,"{""location"": ""BA"", ""is_mobile"": true}" 6169,3,134,2017-04-13 13:29:06,http://dicki.biz/isom,,92.189.185.184,"{""location"": ""PM"", ""is_mobile"": false}" 6170,3,134,2017-01-31 10:06:04,http://moriette.com/jennings,,118.192.159.250,"{""location"": ""CD"", ""is_mobile"": false}" 6171,3,134,2016-12-15 03:45:52,http://weinat.name/eliza_bailey,,149.140.3.63,"{""location"": ""ZM"", ""is_mobile"": true}" 6172,3,134,2017-02-01 10:59:04,http://oreillyoreilly.name/dale,,138.238.61.210,"{""location"": ""PK"", ""is_mobile"": true}" 6173,3,134,2017-02-01 18:00:49,http://heller.org/antonio.kunze,,142.45.15.54,"{""location"": ""GM"", ""is_mobile"": true}" 6174,3,135,2017-01-23 11:22:50,http://wolff.info/patricia_mcglynn,,179.177.143.216,"{""location"": ""US"", ""is_mobile"": false}" 6175,3,135,2017-04-06 13:04:42,http://thompson.info/elian.wiegand,,156.45.134.146,"{""location"": ""ST"", ""is_mobile"": true}" 6176,3,135,2017-01-01 18:13:00,http://vandervorthaley.net/geoffrey,,72.213.67.28,"{""location"": ""CY"", ""is_mobile"": true}" 6177,3,135,2016-12-30 15:12:25,http://damoreheller.name/wilton,,97.172.15.81,"{""location"": ""SI"", ""is_mobile"": false}" 6178,3,135,2017-05-17 14:35:31,http://mills.co/aleen,,133.129.245.237,"{""location"": ""IQ"", ""is_mobile"": true}" 6179,3,135,2017-02-14 01:33:19,http://padbergruecker.info/celestino.runolfon,,15.206.99.104,"{""location"": ""BE"", ""is_mobile"": true}" 6180,3,135,2017-03-15 00:00:12,http://balistrerigrant.net/rylan_kirlin,,231.33.184.169,"{""location"": ""CH"", ""is_mobile"": true}" 6181,3,135,2017-03-20 21:30:40,http://robel.co/sylvia,,16.194.79.116,"{""location"": ""GT"", ""is_mobile"": false}" 6182,3,135,2017-03-29 13:14:11,http://hickle.net/rocio.dickens,,113.133.35.134,"{""location"": ""DJ"", ""is_mobile"": true}" 6183,3,135,2017-03-18 22:11:27,http://borersanford.io/francesco,,58.208.184.97,"{""location"": ""KR"", ""is_mobile"": true}" 6184,3,135,2016-12-14 03:44:59,http://kautzerboyle.com/elda,,40.99.85.7,"{""location"": ""VN"", ""is_mobile"": false}" 6185,3,135,2017-06-03 10:14:26,http://jakubowskiwilkinson.net/tania,,205.25.49.49,"{""location"": ""LY"", ""is_mobile"": false}" 6186,3,135,2017-05-20 01:40:27,http://block.info/joana.gislason,,60.42.165.209,"{""location"": ""BW"", ""is_mobile"": false}" 6187,3,135,2017-02-06 23:25:19,http://volkman.com/barrett,,163.200.208.151,"{""location"": ""EH"", ""is_mobile"": true}" 6188,3,135,2017-04-06 08:39:18,http://schummmckenzie.io/rickie,,251.48.109.66,"{""location"": ""KR"", ""is_mobile"": true}" 6189,3,135,2017-01-10 00:46:01,http://okonfunk.name/jazmyne,,28.61.26.192,"{""location"": ""MC"", ""is_mobile"": false}" 6190,3,135,2017-01-21 06:33:44,http://hegmann.com/granville.stanton,,75.194.71.113,"{""location"": ""SJ"", ""is_mobile"": false}" 6191,3,135,2017-06-01 05:51:31,http://abbott.com/weston_bauch,,53.101.140.128,"{""location"": ""MQ"", ""is_mobile"": false}" 6192,3,135,2017-01-13 17:42:40,http://gerlachwalter.biz/waino.bogan,,137.131.135.188,"{""location"": ""BA"", ""is_mobile"": true}" 6193,3,135,2017-05-09 08:19:43,http://roberts.biz/jovanny_cummerata,,37.44.113.30,"{""location"": ""EC"", ""is_mobile"": true}" 6194,3,135,2017-04-09 03:28:17,http://kemmerjohns.name/orin,,5.213.129.40,"{""location"": ""MV"", ""is_mobile"": false}" 6195,3,135,2017-02-15 18:50:07,http://wiza.org/desiree_grant,,45.80.89.195,"{""location"": ""NC"", ""is_mobile"": true}" 6196,3,135,2017-04-27 22:19:48,http://keeblerhammes.net/jadyn,,235.64.171.198,"{""location"": ""MM"", ""is_mobile"": false}" 6197,3,135,2017-02-26 16:38:25,http://nader.name/theron,,230.104.104.239,"{""location"": ""SR"", ""is_mobile"": false}" 6198,3,135,2016-12-26 10:21:50,http://gerholdberge.org/emile,,65.246.132.136,"{""location"": ""ZA"", ""is_mobile"": false}" 6199,3,135,2017-05-09 09:39:02,http://goodwin.com/marilou,,68.153.28.220,"{""location"": ""BF"", ""is_mobile"": false}" 6200,3,135,2017-03-15 23:09:15,http://dubuque.info/enola,,241.86.229.87,"{""location"": ""CH"", ""is_mobile"": false}" 6201,3,135,2016-12-26 19:51:05,http://schneider.info/leone,,202.142.114.130,"{""location"": ""KH"", ""is_mobile"": true}" 6202,3,135,2017-01-04 03:59:52,http://grantrau.org/wellington.padberg,,232.211.117.41,"{""location"": ""EH"", ""is_mobile"": false}" 6203,3,135,2016-12-19 13:35:12,http://marquardthauck.co/viva,,28.157.162.171,"{""location"": ""RO"", ""is_mobile"": true}" 12059,5,269,2017-06-02 05:30:13,http://aufderhar.io/kaitlyn,0.4445752241,55.55.83.6,"{""location"": ""BA"", ""is_mobile"": false}" 12060,5,269,2017-01-09 02:33:22,http://sporer.co/alana_bashirian,0.9942746574,102.116.37.77,"{""location"": ""PA"", ""is_mobile"": true}" 12061,5,269,2017-04-13 10:10:26,http://kautzer.org/vance.predovic,0.1076890855,173.143.204.228,"{""location"": ""SG"", ""is_mobile"": true}" 12062,5,270,2017-04-03 22:31:21,http://rodriguez.co/kimberly.welch,0.6231263565,114.8.90.61,"{""location"": ""SL"", ""is_mobile"": true}" 12063,5,270,2017-02-03 16:21:46,http://swaniawski.biz/karolann.jast,0.6740172015,232.252.212.26,"{""location"": ""KE"", ""is_mobile"": true}" 12064,5,270,2017-04-16 11:52:32,http://feil.net/alexandro.schaden,0.4500720812,247.76.27.171,"{""location"": ""GA"", ""is_mobile"": true}" 12065,5,270,2017-06-12 22:57:25,http://kautzer.org/hermann_strosin,0.3812269844,206.243.245.4,"{""location"": ""RS"", ""is_mobile"": false}" 12066,5,270,2017-06-14 01:41:01,http://luettgen.name/amaya,0.8312571527,138.46.61.216,"{""location"": ""TV"", ""is_mobile"": false}" 12067,5,270,2017-02-14 12:57:44,http://boehmwunsch.io/herbert,0.9890807699,29.124.229.155,"{""location"": ""LC"", ""is_mobile"": false}" 12068,5,270,2017-06-07 10:33:46,http://price.com/emelia_welch,0.4842501111,79.160.41.151,"{""location"": ""SS"", ""is_mobile"": false}" 12069,5,270,2017-06-10 23:57:01,http://beattypurdy.com/clementine_ward,0.8696854050,90.179.241.239,"{""location"": ""RS"", ""is_mobile"": true}" 12070,5,270,2017-04-28 01:09:01,http://haagkiehn.org/winona,0.1514734737,68.139.213.70,"{""location"": ""IM"", ""is_mobile"": false}" 12071,5,270,2017-02-18 14:40:07,http://lind.name/elia.kertzmann,0.9314016705,195.164.135.231,"{""location"": ""IS"", ""is_mobile"": true}" 12072,5,270,2017-01-07 05:59:38,http://schumm.net/pattie,0.2848337342,149.37.223.19,"{""location"": ""LB"", ""is_mobile"": false}" 12073,5,270,2017-02-07 08:19:05,http://braun.com/marjory,0.5272372853,231.235.122.107,"{""location"": ""LI"", ""is_mobile"": true}" 12074,5,270,2017-03-02 01:47:27,http://hilll.co/charles.braun,0.9322564333,109.9.41.132,"{""location"": ""ME"", ""is_mobile"": true}" 12075,5,270,2017-04-16 01:40:11,http://dach.net/magnolia_koelpin,0.3298946993,13.11.14.232,"{""location"": ""LK"", ""is_mobile"": false}" 12076,5,270,2017-04-24 03:48:33,http://bogan.com/jadon.ernser,0.8375385266,91.30.35.67,"{""location"": ""PW"", ""is_mobile"": true}" 12077,5,270,2017-05-16 02:03:07,http://williamson.biz/minerva_bailey,0.9760023623,121.194.147.186,"{""location"": ""KM"", ""is_mobile"": true}" 12078,5,270,2017-02-17 11:56:32,http://stromanhamill.io/kelly,0.2857649178,233.222.241.135,"{""location"": ""LC"", ""is_mobile"": true}" 12079,5,270,2017-04-12 18:19:40,http://nienow.name/ashley,0.8513482430,162.111.74.31,"{""location"": ""UY"", ""is_mobile"": true}" 12080,5,270,2017-01-18 06:29:33,http://pagacmetz.info/jasmin_kris,0.1452375785,66.102.174.43,"{""location"": ""SA"", ""is_mobile"": true}" 12081,5,270,2017-01-17 23:08:52,http://hansen.net/lafayette_rice,0.2495088824,26.191.140.134,"{""location"": ""CR"", ""is_mobile"": true}" 12082,5,270,2017-03-21 05:15:13,http://stokesondricka.co/karlie,0.2121652510,151.142.195.50,"{""location"": ""PA"", ""is_mobile"": true}" 12083,5,270,2017-01-30 02:56:30,http://daniel.info/reva,0.0470262413,173.103.7.68,"{""location"": ""CW"", ""is_mobile"": false}" 12084,5,270,2016-12-27 18:25:42,http://nikolauskeler.com/zakary,0.2862411638,47.11.169.121,"{""location"": ""LR"", ""is_mobile"": true}" 12085,5,270,2017-03-20 01:36:57,http://rosenbaum.name/adela_grant,0.1434473061,157.10.80.246,"{""location"": ""PW"", ""is_mobile"": false}" 12086,5,270,2017-05-18 08:05:50,http://herman.name/ivory_wuckert,0.5275954853,185.24.10.8,"{""location"": ""KH"", ""is_mobile"": true}" 12087,5,270,2017-01-10 15:34:33,http://gerhold.info/kitty_gusikowski,0.9524461317,186.206.209.48,"{""location"": ""AG"", ""is_mobile"": false}" 12088,5,270,2016-12-29 10:10:09,http://flatleymoen.name/janelle,0.2466624080,6.3.79.182,"{""location"": ""DZ"", ""is_mobile"": true}" 12089,5,270,2017-02-27 12:17:18,http://koepp.name/rolando,0.5116147936,254.170.172.160,"{""location"": ""CO"", ""is_mobile"": false}" 12090,5,270,2017-01-21 14:58:28,http://damore.co/conor_monahan,0.8557456064,185.190.207.243,"{""location"": ""NL"", ""is_mobile"": false}" 12091,5,270,2017-04-12 14:04:15,http://stiedemanngrady.org/markus.jast,0.3681156658,95.155.168.112,"{""location"": ""LC"", ""is_mobile"": false}" 12092,5,270,2017-05-01 13:13:43,http://greenfelderjacobson.org/carole.will,0.1190346987,85.149.138.212,"{""location"": ""HT"", ""is_mobile"": true}" 12093,5,270,2017-02-20 17:27:18,http://mcdermottmurphy.org/rachelle_kihn,0.8878591510,71.167.240.38,"{""location"": ""GT"", ""is_mobile"": true}" 12094,5,270,2017-02-27 00:23:22,http://konopelskigleichner.net/kaelyn.grant,0.1105416168,148.18.4.191,"{""location"": ""NZ"", ""is_mobile"": false}" 12095,5,270,2017-05-31 03:48:49,http://swifthamill.io/lora.howell,0.1640670758,19.37.177.140,"{""location"": ""MY"", ""is_mobile"": true}" 12096,5,270,2017-02-16 23:44:06,http://gutmann.org/elva_white,0.0988592039,78.34.23.74,"{""location"": ""LV"", ""is_mobile"": true}" 12097,5,270,2017-03-23 13:31:51,http://kreigervolkman.name/monica,0.4454310415,145.202.212.70,"{""location"": ""VN"", ""is_mobile"": true}" 12098,5,270,2017-03-04 00:31:40,http://buckridge.io/floy_predovic,0.5559864616,246.219.250.80,"{""location"": ""TZ"", ""is_mobile"": true}" 12099,5,270,2017-02-17 15:10:24,http://daugherty.io/landen,0.4714608421,165.215.104.161,"{""location"": ""MN"", ""is_mobile"": false}" 12100,5,270,2017-04-07 05:35:44,http://stehrmcdermott.io/keyon,0.1595940489,239.45.129.234,"{""location"": ""PS"", ""is_mobile"": true}" 12101,5,270,2017-01-04 12:56:44,http://rutherford.com/halie,0.2240296337,219.191.51.208,"{""location"": ""UA"", ""is_mobile"": false}" 12102,5,270,2017-01-21 23:22:47,http://hahn.io/gretchen,0.6105054904,95.221.96.47,"{""location"": ""BJ"", ""is_mobile"": true}" 12103,5,270,2017-04-11 18:44:08,http://mueller.io/jordan_walter,0.7324863717,49.140.209.131,"{""location"": ""FJ"", ""is_mobile"": false}" 12104,5,270,2017-02-26 16:51:23,http://beiernicolas.io/kailey,0.2604616583,187.122.171.233,"{""location"": ""SX"", ""is_mobile"": false}" 12105,5,270,2017-05-26 16:56:43,http://lehnerankunding.io/johann,0.2765160865,93.187.241.71,"{""location"": ""UM"", ""is_mobile"": true}" 12106,5,270,2017-05-22 07:21:59,http://lehner.io/florida,0.9421040274,199.48.220.63,"{""location"": ""TH"", ""is_mobile"": false}" 12107,5,270,2017-01-05 23:47:57,http://haag.net/triston,0.7969915163,99.86.245.182,"{""location"": ""LU"", ""is_mobile"": true}" 12108,5,270,2017-03-16 23:40:26,http://goyette.info/vincent,0.4964221716,188.108.13.17,"{""location"": ""CY"", ""is_mobile"": false}" 12109,5,270,2017-04-07 17:56:45,http://sauer.co/candelario_mohr,0.3437343330,204.207.133.239,"{""location"": ""BR"", ""is_mobile"": true}" 15053,6,338,2017-05-12 21:28:39,http://beahan.org/odie.kub,,176.208.75.184,"{""location"": ""TT"", ""is_mobile"": true}" 15054,6,338,2017-04-28 16:22:13,http://kuhn.io/juanita,,27.190.63.173,"{""location"": ""SG"", ""is_mobile"": false}" 15055,6,338,2017-05-12 11:43:17,http://mcclure.co/mabel_hudson,,201.189.225.16,"{""location"": ""IT"", ""is_mobile"": true}" 15056,6,338,2017-06-12 00:11:21,http://moriette.net/frederick_huel,,200.78.99.221,"{""location"": ""IE"", ""is_mobile"": false}" 15057,6,338,2017-03-31 16:41:50,http://douglas.co/talia_ward,,100.21.4.201,"{""location"": ""AQ"", ""is_mobile"": false}" 15058,6,338,2017-03-23 03:08:40,http://creminwatsica.co/darion_ruel,,146.167.138.108,"{""location"": ""SK"", ""is_mobile"": true}" 15059,6,338,2017-02-23 18:01:51,http://schaden.biz/margarett,,6.231.103.121,"{""location"": ""PG"", ""is_mobile"": false}" 15060,6,338,2017-03-26 00:21:23,http://kovacekkeler.name/manuel_yundt,,106.233.50.244,"{""location"": ""VI"", ""is_mobile"": false}" 15061,6,338,2017-03-25 11:17:03,http://heidenreich.biz/rubye_moore,,229.167.185.30,"{""location"": ""UA"", ""is_mobile"": false}" 15062,6,338,2017-04-06 23:20:41,http://swifthagenes.org/bonita_dooley,,196.106.178.189,"{""location"": ""IM"", ""is_mobile"": true}" 15063,6,338,2017-03-19 18:19:00,http://collinsmacejkovic.biz/glenda.klein,,65.76.121.41,"{""location"": ""BQ"", ""is_mobile"": true}" 15064,6,338,2017-02-26 00:52:41,http://spinka.co/kariane.lubowitz,,195.4.114.220,"{""location"": ""MD"", ""is_mobile"": false}" 15065,6,338,2017-01-22 18:36:16,http://oberbrunnerabernathy.info/garland_heathcote,,121.225.185.96,"{""location"": ""AW"", ""is_mobile"": false}" 15066,6,338,2017-05-21 23:47:53,http://mueller.org/rachael_bayer,,55.18.165.205,"{""location"": ""KM"", ""is_mobile"": true}" 15067,6,338,2017-05-11 14:07:19,http://roberts.net/sid.roberts,,86.25.84.63,"{""location"": ""AU"", ""is_mobile"": true}" 15068,6,338,2017-02-11 16:00:32,http://oreilly.info/aurelie_keler,,192.101.49.49,"{""location"": ""WF"", ""is_mobile"": true}" 15069,6,338,2017-03-24 16:09:35,http://nadercartwright.name/kathlyn,,99.87.164.159,"{""location"": ""SI"", ""is_mobile"": true}" 15070,6,338,2017-03-28 15:07:12,http://hicklelangworth.org/marcus_halvorson,,39.147.133.203,"{""location"": ""VN"", ""is_mobile"": false}" 15071,6,338,2017-03-11 16:31:52,http://ortizankunding.biz/fannie,,68.39.248.236,"{""location"": ""WS"", ""is_mobile"": false}" 15072,6,339,2017-04-06 18:15:17,http://waelchibecker.net/kitty_brown,,217.89.104.96,"{""location"": ""TT"", ""is_mobile"": false}" 15073,6,339,2017-02-16 14:09:10,http://schimmel.info/jarrod.halvorson,,62.181.9.205,"{""location"": ""MX"", ""is_mobile"": true}" 15074,6,339,2017-04-30 03:23:13,http://jerde.io/clemmie,,221.253.169.162,"{""location"": ""KZ"", ""is_mobile"": false}" 15075,6,339,2017-03-14 18:16:22,http://green.io/rosa_wuckert,,22.175.78.239,"{""location"": ""JP"", ""is_mobile"": true}" 15076,6,339,2017-06-02 18:13:52,http://schamberger.net/ivory.schiller,,81.36.125.167,"{""location"": ""IO"", ""is_mobile"": false}" 15077,6,339,2017-03-17 21:17:13,http://abshire.net/sadie,,47.122.216.239,"{""location"": ""ID"", ""is_mobile"": false}" 15078,6,339,2017-01-10 18:29:31,http://funk.name/bernard.fadel,,41.137.170.185,"{""location"": ""MU"", ""is_mobile"": true}" 15079,6,339,2017-02-06 10:35:09,http://erdman.org/wallace,,110.136.59.136,"{""location"": ""BQ"", ""is_mobile"": true}" 15080,6,339,2017-05-08 04:53:58,http://williamson.com/lelia,,74.114.162.167,"{""location"": ""QA"", ""is_mobile"": true}" 15081,6,339,2017-05-28 01:31:21,http://kub.org/harry_wyman,,174.12.58.141,"{""location"": ""CM"", ""is_mobile"": false}" 15082,6,339,2017-02-03 04:11:13,http://ruel.name/coty.streich,,120.98.214.109,"{""location"": ""NL"", ""is_mobile"": false}" 15083,6,339,2016-12-20 04:41:04,http://ratke.net/louvenia.kuhlman,,59.43.79.20,"{""location"": ""CG"", ""is_mobile"": false}" 15084,6,339,2017-03-19 03:30:39,http://keler.co/hector,,63.121.195.204,"{""location"": ""BQ"", ""is_mobile"": true}" 15085,6,339,2016-12-24 13:31:10,http://rolfson.co/miguel,,21.219.214.74,"{""location"": ""KH"", ""is_mobile"": false}" 15086,6,339,2017-03-22 22:14:51,http://flatley.io/elisabeth,,202.69.45.238,"{""location"": ""PR"", ""is_mobile"": true}" 15087,6,339,2017-01-19 15:52:42,http://veum.net/leta,,132.174.18.48,"{""location"": ""AI"", ""is_mobile"": true}" 15088,6,339,2017-03-16 20:30:21,http://schmidt.io/theresa,,93.184.237.166,"{""location"": ""ET"", ""is_mobile"": false}" 15089,6,339,2016-12-17 03:23:55,http://eichmann.com/landen,,91.134.108.172,"{""location"": ""BH"", ""is_mobile"": true}" 15090,6,339,2017-03-03 19:27:18,http://davis.biz/freddy,,180.80.125.27,"{""location"": ""HU"", ""is_mobile"": false}" 15091,6,339,2017-02-24 16:24:17,http://torpbuckridge.name/sigmund,,50.131.195.111,"{""location"": ""US"", ""is_mobile"": true}" 15092,6,339,2017-06-11 14:32:13,http://sanford.biz/americo,,247.75.145.197,"{""location"": ""DZ"", ""is_mobile"": true}" 15093,6,339,2017-02-03 15:14:30,http://balistreribatz.name/mable,,204.201.243.201,"{""location"": ""TK"", ""is_mobile"": true}" 15094,6,339,2017-02-17 01:09:34,http://lubowitz.com/litzy_fritsch,,173.180.171.212,"{""location"": ""NO"", ""is_mobile"": true}" 15095,6,339,2017-02-08 10:55:50,http://abshireshanahan.co/jameson,,171.141.53.240,"{""location"": ""CG"", ""is_mobile"": true}" 15096,6,339,2017-05-14 02:46:53,http://keebler.co/evalyn_lemke,,232.124.77.217,"{""location"": ""AU"", ""is_mobile"": false}" 15097,6,339,2017-06-07 06:59:58,http://willms.com/irwin,,224.232.131.105,"{""location"": ""TL"", ""is_mobile"": false}" 15098,6,339,2017-04-04 10:00:45,http://flatleyhamill.net/jaren,,40.174.201.133,"{""location"": ""SB"", ""is_mobile"": true}" 15099,6,339,2017-01-22 21:41:36,http://parisian.org/mabelle,,199.37.86.85,"{""location"": ""SY"", ""is_mobile"": false}" 15100,6,339,2017-01-01 15:50:26,http://blockkling.info/jayne,,102.217.29.43,"{""location"": ""KR"", ""is_mobile"": true}" 15101,6,339,2017-04-14 17:47:23,http://brakus.info/dorthy,,240.143.14.156,"{""location"": ""ER"", ""is_mobile"": true}" 15102,6,339,2017-01-19 00:04:43,http://corkery.info/fredy.white,,70.38.88.146,"{""location"": ""SA"", ""is_mobile"": false}" 15103,6,339,2017-03-22 07:53:01,http://mayerlarkin.com/cristina.weimann,,107.30.246.84,"{""location"": ""PA"", ""is_mobile"": false}" 15104,6,339,2017-02-18 11:15:40,http://watsica.name/bo,,210.146.209.100,"{""location"": ""BE"", ""is_mobile"": true}" 15105,6,339,2017-02-12 15:59:37,http://pfeffer.io/miouri,,218.35.62.250,"{""location"": ""MV"", ""is_mobile"": true}" 15106,6,339,2017-02-14 02:11:30,http://greenholtbatz.biz/edwardo.hickle,,197.70.193.243,"{""location"": ""RO"", ""is_mobile"": true}" 15107,6,339,2017-01-29 14:45:39,http://mann.org/demond,,189.128.228.13,"{""location"": ""WF"", ""is_mobile"": false}" 9194,4,205,2017-05-24 17:59:15,http://hillsjenkins.co/nickolas.stark,0.1486469762,129.164.77.136,"{""location"": ""VG"", ""is_mobile"": true}" 9195,4,205,2017-01-12 20:34:57,http://kingrunolfon.biz/ewald_bins,0.8858589040,158.181.107.35,"{""location"": ""BZ"", ""is_mobile"": true}" 9196,4,205,2017-01-31 16:44:54,http://mitchellschamberger.info/elna.gutmann,0.1441584937,91.209.231.155,"{""location"": ""KZ"", ""is_mobile"": false}" 9197,4,205,2017-05-11 16:07:53,http://gleichner.co/jovanny_ullrich,0.2194563581,69.35.34.102,"{""location"": ""LK"", ""is_mobile"": false}" 9198,4,205,2017-03-26 17:08:58,http://rolfson.io/jacklyn,0.5587352141,194.207.193.41,"{""location"": ""CY"", ""is_mobile"": false}" 9199,4,205,2017-04-22 07:44:10,http://erdman.io/bart,0.6339333540,57.96.64.216,"{""location"": ""UG"", ""is_mobile"": false}" 9200,4,205,2017-02-07 02:00:53,http://ritchietillman.info/noelia.lebsack,0.2251079585,95.107.43.205,"{""location"": ""DO"", ""is_mobile"": false}" 9201,4,206,2016-12-16 14:59:52,http://miller.io/aracely,0.9100050953,167.144.49.13,"{""location"": ""JE"", ""is_mobile"": true}" 9202,4,206,2017-03-19 08:11:05,http://cartwrightwitting.io/harmon,0.0799805408,21.254.99.67,"{""location"": ""SK"", ""is_mobile"": true}" 9203,4,206,2017-05-13 22:38:02,http://macgyver.info/josue.kling,0.8319751394,48.176.96.15,"{""location"": ""US"", ""is_mobile"": true}" 9204,4,206,2017-03-05 17:54:06,http://kris.biz/caitlyn,0.4852922526,58.51.227.200,"{""location"": ""LK"", ""is_mobile"": true}" 9205,4,206,2016-12-13 18:22:00,http://steuber.co/meagan.stracke,0.3655498089,225.195.167.216,"{""location"": ""NI"", ""is_mobile"": false}" 9206,4,206,2017-02-20 04:13:30,http://yundt.co/derick,0.9014575402,233.33.79.154,"{""location"": ""LI"", ""is_mobile"": true}" 9207,4,206,2017-01-24 18:52:54,http://greenfelderkoch.net/hermann.homenick,0.0755470803,93.80.57.237,"{""location"": ""DJ"", ""is_mobile"": false}" 9208,4,206,2017-05-29 03:50:30,http://beckermonahan.biz/andreane,0.7576713509,114.222.238.8,"{""location"": ""SH"", ""is_mobile"": true}" 9209,4,206,2017-05-15 21:48:36,http://cole.info/jany,0.3436693031,35.53.190.5,"{""location"": ""NG"", ""is_mobile"": true}" 9210,4,206,2017-01-12 00:09:34,http://priceadams.co/miles,0.9425489426,238.54.171.200,"{""location"": ""GQ"", ""is_mobile"": true}" 9211,4,206,2017-01-28 16:31:05,http://kreigersmith.net/miles,0.6706872392,113.128.85.13,"{""location"": ""MK"", ""is_mobile"": false}" 9212,4,206,2017-04-23 05:13:32,http://tremblayhudson.info/modesta,0.2026104572,219.68.72.45,"{""location"": ""ER"", ""is_mobile"": false}" 9213,4,206,2016-12-28 15:56:16,http://volkman.org/avis,0.6246514592,107.212.137.51,"{""location"": ""BT"", ""is_mobile"": true}" 9214,4,206,2017-05-28 02:00:24,http://collierrenner.info/nathan.torp,0.2951615887,248.89.213.31,"{""location"": ""MN"", ""is_mobile"": true}" 9215,4,206,2017-06-07 17:23:24,http://wittingblick.name/sonny,0.7895017671,32.165.243.221,"{""location"": ""GF"", ""is_mobile"": true}" 9216,4,206,2017-04-21 09:33:03,http://kuvalis.com/bud,0.4437604608,63.226.154.52,"{""location"": ""SN"", ""is_mobile"": true}" 9217,4,206,2017-06-03 17:31:56,http://johnhields.org/gunnar_thompson,0.6667069416,133.254.227.186,"{""location"": ""SJ"", ""is_mobile"": false}" 9218,4,206,2017-01-24 14:45:20,http://bogisich.co/kaycee.will,0.5524306225,25.52.186.215,"{""location"": ""BE"", ""is_mobile"": true}" 9219,4,206,2016-12-15 21:54:15,http://zieme.net/mckenna.ward,0.9415597634,118.55.84.164,"{""location"": ""CR"", ""is_mobile"": true}" 9220,4,206,2017-05-23 12:25:39,http://hartmannheaney.co/isaias,0.4321246880,137.198.146.116,"{""location"": ""BZ"", ""is_mobile"": true}" 9221,4,206,2017-05-09 16:02:50,http://jacobson.co/name.kihn,0.7730106533,172.178.22.171,"{""location"": ""GF"", ""is_mobile"": false}" 9222,4,206,2017-05-27 03:00:36,http://mayert.biz/kara.zemlak,0.5256156880,212.207.242.163,"{""location"": ""ZM"", ""is_mobile"": false}" 9223,4,206,2017-05-23 00:45:59,http://shanahan.co/shakira,0.7948530899,159.195.199.84,"{""location"": ""EG"", ""is_mobile"": false}" 9224,4,206,2017-03-25 13:58:58,http://feest.co/marjolaine.kuphal,0.9568539712,65.124.44.219,"{""location"": ""SY"", ""is_mobile"": true}" 9225,4,206,2017-05-11 01:03:05,http://hodkiewiczconsidine.info/antwon,0.3871411351,252.156.34.33,"{""location"": ""TV"", ""is_mobile"": true}" 9226,4,206,2016-12-23 21:07:01,http://crookskertzmann.biz/giovanny_friesen,0.1711776252,18.58.60.23,"{""location"": ""KZ"", ""is_mobile"": false}" 9227,4,206,2016-12-23 03:44:46,http://dickidicki.com/maximo,0.3413620981,125.68.4.92,"{""location"": ""AQ"", ""is_mobile"": true}" 9228,4,206,2017-01-01 23:20:32,http://harber.org/adan_king,0.1525092676,64.112.151.213,"{""location"": ""CO"", ""is_mobile"": true}" 9229,4,206,2017-05-18 07:47:22,http://hudson.biz/shane,0.0600363110,10.9.61.18,"{""location"": ""BD"", ""is_mobile"": false}" 9230,4,206,2017-01-05 09:45:43,http://waters.io/valentine,0.3482018189,250.106.177.14,"{""location"": ""TK"", ""is_mobile"": false}" 9231,4,206,2017-03-03 05:10:43,http://tillman.com/naomie.gaylord,0.1910742377,134.108.47.219,"{""location"": ""ZA"", ""is_mobile"": false}" 9232,4,206,2017-02-05 22:38:26,http://pfannerstill.io/francisca,0.8297523765,74.201.243.117,"{""location"": ""TW"", ""is_mobile"": false}" 9233,4,206,2017-01-30 18:12:31,http://gorczany.org/luis.pollich,0.1865280105,252.87.153.156,"{""location"": ""GD"", ""is_mobile"": true}" 9234,4,206,2017-02-21 14:48:23,http://okon.org/alexander.turcotte,0.2085916382,5.193.174.36,"{""location"": ""UG"", ""is_mobile"": true}" 9235,4,207,2017-04-28 11:23:58,http://gerlach.org/charity,0.1862029814,137.201.114.46,"{""location"": ""AG"", ""is_mobile"": true}" 9236,4,207,2017-05-15 00:08:11,http://harrislindgren.com/matt,0.4754738060,246.39.107.39,"{""location"": ""TO"", ""is_mobile"": false}" 9237,4,207,2017-04-14 19:38:55,http://orn.biz/ophelia_boyle,0.3337471096,189.194.193.154,"{""location"": ""BO"", ""is_mobile"": false}" 9238,4,207,2017-03-30 22:40:57,http://raynor.net/ines,0.1749852478,117.101.78.218,"{""location"": ""MS"", ""is_mobile"": true}" 9239,4,207,2016-12-31 12:41:49,http://marvin.io/adolfo_rice,0.6309963848,178.19.239.187,"{""location"": ""TG"", ""is_mobile"": true}" 9240,4,207,2017-03-22 19:03:34,http://feest.io/emmy,0.2347868019,80.77.138.242,"{""location"": ""BV"", ""is_mobile"": false}" 9241,4,207,2017-03-26 16:32:33,http://strosin.info/alfred,0.6264832332,45.179.239.10,"{""location"": ""MU"", ""is_mobile"": true}" 9242,4,207,2017-01-13 13:22:46,http://lowe.com/eleanora.donnelly,0.3862410861,233.182.138.89,"{""location"": ""HN"", ""is_mobile"": true}" 9243,4,207,2017-05-12 10:54:07,http://jakubowskidach.org/esperanza,0.1933427601,198.145.11.235,"{""location"": ""YT"", ""is_mobile"": true}" 9244,4,207,2017-02-28 09:16:13,http://howe.co/ambrose,0.1973236241,216.20.9.235,"{""location"": ""SG"", ""is_mobile"": true}" 9245,4,207,2017-01-23 04:20:06,http://effertzhermann.io/alexanne,0.4733063123,131.182.203.249,"{""location"": ""CO"", ""is_mobile"": true}" 6204,3,135,2017-02-10 09:13:06,http://rempellangworth.name/angelina,,32.163.161.14,"{""location"": ""GE"", ""is_mobile"": true}" 6205,3,135,2017-01-16 20:24:16,http://haneblock.com/loy_spinka,,26.140.124.189,"{""location"": ""IO"", ""is_mobile"": true}" 6206,3,135,2017-03-09 23:50:54,http://mckenzie.io/rocky.dicki,,209.185.139.80,"{""location"": ""AO"", ""is_mobile"": false}" 6207,3,135,2017-02-10 11:00:58,http://stammcrooks.info/don.douglas,,86.160.216.76,"{""location"": ""PW"", ""is_mobile"": false}" 6208,3,135,2016-12-26 17:47:57,http://rowe.co/raquel,,203.235.140.142,"{""location"": ""TO"", ""is_mobile"": false}" 6209,3,135,2017-05-26 14:22:39,http://haag.io/beulah_turcotte,,181.222.93.101,"{""location"": ""AG"", ""is_mobile"": true}" 6210,3,135,2017-03-12 13:49:53,http://von.org/marcia,,2.153.247.217,"{""location"": ""TW"", ""is_mobile"": false}" 6211,3,135,2016-12-27 13:42:39,http://grimes.info/bridie.connelly,,68.217.236.97,"{""location"": ""TG"", ""is_mobile"": true}" 6212,3,135,2017-04-25 15:50:00,http://simonis.net/brendon,,8.226.40.193,"{""location"": ""BJ"", ""is_mobile"": true}" 6213,3,136,2017-05-30 04:57:41,http://tremblay.info/gillian_carter,0.1304169010,223.243.221.129,"{""location"": ""VE"", ""is_mobile"": true}" 6214,3,136,2017-05-03 02:38:40,http://koelpin.org/rosetta,0.6987075191,41.153.232.8,"{""location"": ""BR"", ""is_mobile"": true}" 6215,3,136,2017-03-25 20:14:59,http://beatty.com/josephine,0.3103418293,132.83.204.12,"{""location"": ""SY"", ""is_mobile"": true}" 6216,3,136,2017-02-07 04:18:05,http://altenwerthcummerata.name/magdalena.wyman,0.9120022161,110.214.11.140,"{""location"": ""FO"", ""is_mobile"": false}" 6217,3,136,2017-03-20 12:53:59,http://hills.org/cale_gerlach,0.8935410738,170.118.170.86,"{""location"": ""CC"", ""is_mobile"": false}" 6218,3,136,2017-05-23 21:04:30,http://buckridge.co/dora_botsford,0.9321871171,89.20.13.104,"{""location"": ""TZ"", ""is_mobile"": false}" 6219,3,136,2017-05-18 17:44:50,http://hamill.co/gilda,0.6287615392,217.158.167.186,"{""location"": ""HT"", ""is_mobile"": false}" 6220,3,136,2017-04-24 18:38:02,http://auer.io/gabriel.kutch,0.7846586075,176.92.17.250,"{""location"": ""ZW"", ""is_mobile"": false}" 6221,3,136,2017-03-20 06:05:10,http://ko.com/holden.quitzon,0.6074397445,118.153.251.55,"{""location"": ""BZ"", ""is_mobile"": true}" 6222,3,136,2017-02-02 19:14:22,http://lednerswift.com/jasper,0.6236198541,205.99.98.166,"{""location"": ""PE"", ""is_mobile"": false}" 6223,3,136,2017-01-20 14:09:37,http://stiedemannflatley.net/magnolia_boehm,0.9124009477,232.96.201.173,"{""location"": ""KR"", ""is_mobile"": true}" 6224,3,136,2017-05-09 13:58:39,http://shields.org/natalie,0.3432271085,165.143.73.226,"{""location"": ""BI"", ""is_mobile"": true}" 6225,3,136,2017-01-22 11:01:07,http://grant.org/trudie,0.2218677671,198.25.4.91,"{""location"": ""MP"", ""is_mobile"": false}" 6226,3,136,2017-02-19 05:52:05,http://kuhlman.com/name_green,0.9436802366,40.129.46.136,"{""location"": ""US"", ""is_mobile"": false}" 6227,3,136,2017-05-20 20:55:41,http://gorczanymoore.net/lonzo,0.5418630111,65.59.91.16,"{""location"": ""LU"", ""is_mobile"": true}" 6228,3,136,2016-12-22 05:47:51,http://batzprosacco.biz/timothy,0.8773651602,22.37.134.195,"{""location"": ""CR"", ""is_mobile"": false}" 6229,3,136,2016-12-23 04:50:41,http://cummerata.co/fredy.kaulke,0.3287255280,238.171.200.238,"{""location"": ""TC"", ""is_mobile"": true}" 6230,3,136,2017-02-28 21:14:39,http://gutmann.net/bianka_goodwin,0.5555421206,28.197.106.63,"{""location"": ""BI"", ""is_mobile"": false}" 6231,3,136,2017-06-11 15:28:58,http://runolfsdottir.name/buddy_botsford,0.1932598364,222.39.85.46,"{""location"": ""CO"", ""is_mobile"": false}" 6232,3,136,2017-01-14 21:39:43,http://carrollflatley.com/blanche.kuhic,0.8936742240,41.98.104.223,"{""location"": ""SZ"", ""is_mobile"": false}" 6233,3,136,2016-12-23 09:42:16,http://bashirian.org/rosina,0.1217784851,43.198.114.153,"{""location"": ""BO"", ""is_mobile"": true}" 6234,3,136,2017-02-26 06:24:53,http://huel.com/edgardo,0.5919622926,27.98.183.95,"{""location"": ""HK"", ""is_mobile"": false}" 6235,3,136,2017-04-03 02:47:36,http://hudson.co/loyal,0.4696667372,11.138.106.168,"{""location"": ""MX"", ""is_mobile"": false}" 6236,3,136,2017-01-22 22:20:49,http://abernathy.name/rickie_maggio,0.6517122502,104.70.38.174,"{""location"": ""TO"", ""is_mobile"": false}" 6237,3,136,2017-05-02 11:16:04,http://graham.biz/jada,0.5078953375,97.83.164.244,"{""location"": ""HM"", ""is_mobile"": false}" 6238,3,136,2017-05-14 00:07:05,http://olsonhudson.info/earline,0.4456315306,44.230.21.127,"{""location"": ""VU"", ""is_mobile"": true}" 6239,3,137,2017-03-14 21:28:05,http://langworthbatz.biz/jacinthe,0.6026301897,179.252.234.224,"{""location"": ""LT"", ""is_mobile"": false}" 6240,3,137,2017-03-22 15:55:39,http://sawaynmuller.net/letha,0.9239881055,3.18.37.204,"{""location"": ""PF"", ""is_mobile"": true}" 6241,3,137,2017-01-19 07:59:58,http://graham.info/aric,0.4572211326,93.181.103.195,"{""location"": ""LY"", ""is_mobile"": true}" 6242,3,137,2017-03-18 08:55:31,http://larson.biz/cameron.schaefer,0.8049743126,31.235.136.141,"{""location"": ""CW"", ""is_mobile"": false}" 6243,3,137,2017-04-28 10:18:36,http://gleasonbayer.com/stacy.auer,0.4727663590,49.101.199.203,"{""location"": ""JE"", ""is_mobile"": true}" 6244,3,137,2017-05-06 20:26:51,http://steuber.org/estel.labadie,0.3439952014,241.245.168.212,"{""location"": ""ME"", ""is_mobile"": false}" 6245,3,137,2017-02-08 19:03:51,http://dibbert.net/michel,0.6141984872,188.68.183.101,"{""location"": ""GH"", ""is_mobile"": false}" 6246,3,137,2017-05-15 21:30:20,http://hillljerde.co/wade_hintz,0.8529427565,59.21.195.94,"{""location"": ""NC"", ""is_mobile"": false}" 6247,3,137,2017-03-10 10:53:59,http://roobgoodwin.biz/antwan,0.6093570111,228.233.212.125,"{""location"": ""PM"", ""is_mobile"": false}" 6248,3,137,2017-02-02 06:08:42,http://parisian.name/dax,0.1382939850,228.183.39.172,"{""location"": ""MO"", ""is_mobile"": false}" 6249,3,137,2017-05-24 03:42:48,http://monahan.co/isabella,0.8997006660,170.112.63.150,"{""location"": ""IO"", ""is_mobile"": true}" 6250,3,137,2017-03-05 16:14:59,http://kaulkegottlieb.name/dorothy_casper,0.0167946557,103.245.8.82,"{""location"": ""MF"", ""is_mobile"": true}" 6251,3,137,2017-06-08 18:01:14,http://howe.info/ruel_koch,0.6079230112,83.176.18.153,"{""location"": ""MM"", ""is_mobile"": true}" 6252,3,137,2017-05-16 07:17:12,http://gusikowski.name/eino,0.9678693641,178.42.171.45,"{""location"": ""GN"", ""is_mobile"": false}" 6253,3,137,2017-05-04 09:22:12,http://mclaughlinprice.net/samson,0.3165578042,215.155.114.161,"{""location"": ""IR"", ""is_mobile"": true}" 6254,3,137,2017-04-17 18:05:19,http://stokes.io/ryley,0.5300459882,202.86.130.173,"{""location"": ""MR"", ""is_mobile"": true}" 6255,3,137,2017-05-12 18:49:21,http://douglas.co/meggie.hintz,0.3575868941,244.60.96.4,"{""location"": ""CU"", ""is_mobile"": false}" 12110,5,270,2017-06-09 03:58:30,http://swaniawski.biz/reie,0.5112924218,218.63.36.171,"{""location"": ""JE"", ""is_mobile"": false}" 12111,5,270,2017-04-26 17:17:41,http://eichmann.biz/ludwig,0.9980377398,118.249.37.6,"{""location"": ""TO"", ""is_mobile"": true}" 12112,5,270,2017-01-23 16:38:40,http://shanahan.co/lavonne.dibbert,0.7424350835,29.247.244.117,"{""location"": ""SZ"", ""is_mobile"": false}" 12113,5,270,2017-06-01 19:39:54,http://goodwin.net/lori,0.2069524489,231.239.72.131,"{""location"": ""GD"", ""is_mobile"": true}" 12114,5,270,2017-04-10 06:23:47,http://rohan.net/merritt_blick,0.5913165126,189.152.70.164,"{""location"": ""MG"", ""is_mobile"": true}" 12115,5,270,2017-01-19 20:01:19,http://baumbachfisher.com/janea,0.0198536664,225.209.162.142,"{""location"": ""MP"", ""is_mobile"": false}" 12116,5,270,2017-05-27 14:04:48,http://jerdewyman.org/selmer.hermann,0.2244732613,39.228.71.132,"{""location"": ""BI"", ""is_mobile"": true}" 12117,5,270,2017-05-02 06:49:24,http://reichel.org/mireya,0.3948823616,71.221.78.187,"{""location"": ""GY"", ""is_mobile"": false}" 12118,5,270,2017-04-25 06:47:30,http://littel.info/estelle_nikolaus,0.6883469080,130.185.101.87,"{""location"": ""WS"", ""is_mobile"": false}" 12119,5,270,2016-12-27 17:54:04,http://langwilliamson.co/alfred,0.1089695199,223.173.200.161,"{""location"": ""SS"", ""is_mobile"": true}" 12120,5,270,2017-04-27 00:54:32,http://hand.co/alene,0.5766178798,98.163.46.103,"{""location"": ""HM"", ""is_mobile"": true}" 12121,5,270,2017-04-16 08:34:21,http://gusikowski.info/okey_carroll,0.8552397876,69.150.86.120,"{""location"": ""CY"", ""is_mobile"": false}" 12122,5,270,2017-05-15 21:08:20,http://carter.com/rhiannon,0.7649971136,132.19.6.34,"{""location"": ""BG"", ""is_mobile"": true}" 12123,5,270,2017-02-17 13:25:38,http://beahan.org/everett,0.1261063826,78.172.69.164,"{""location"": ""IS"", ""is_mobile"": true}" 12124,5,270,2017-04-09 16:58:29,http://johnson.com/keanu,0.1705002015,236.70.28.115,"{""location"": ""GN"", ""is_mobile"": false}" 12125,5,270,2017-06-08 00:50:48,http://okuneva.name/bernhard.hoppe,0.8756187518,164.141.144.152,"{""location"": ""CG"", ""is_mobile"": false}" 12126,5,270,2017-05-18 15:36:08,http://windler.info/camylle,0.7560108979,141.90.227.202,"{""location"": ""GS"", ""is_mobile"": true}" 12127,5,270,2017-04-29 15:29:26,http://walsh.com/brant,0.5252656327,3.192.78.41,"{""location"": ""LV"", ""is_mobile"": true}" 12128,5,270,2017-04-20 07:11:54,http://auerzboncak.io/kenya_kilback,0.3878928500,8.71.168.188,"{""location"": ""SD"", ""is_mobile"": true}" 12129,5,270,2017-04-22 04:18:29,http://braun.net/abby_flatley,0.1338790860,108.118.26.190,"{""location"": ""US"", ""is_mobile"": false}" 12130,5,270,2017-05-24 19:50:50,http://altenwerth.biz/eunice,0.4391788519,78.155.74.210,"{""location"": ""SH"", ""is_mobile"": false}" 12131,5,270,2017-01-13 05:39:27,http://larson.net/chaz,0.7577919437,124.138.142.92,"{""location"": ""IM"", ""is_mobile"": true}" 12132,5,271,2017-01-01 22:56:31,http://pfefferklein.biz/berneice_gusikowski,0.1528351116,119.135.236.215,"{""location"": ""BN"", ""is_mobile"": false}" 12133,5,271,2016-12-22 10:37:31,http://boscolittel.net/diamond_goodwin,0.8140434586,68.247.177.215,"{""location"": ""HN"", ""is_mobile"": true}" 12134,5,271,2017-01-09 03:16:09,http://cummings.co/marion.connelly,0.8805606478,145.231.100.117,"{""location"": ""HT"", ""is_mobile"": false}" 12135,5,271,2017-04-30 12:31:02,http://skiles.io/winston,0.0958300270,244.200.152.138,"{""location"": ""DO"", ""is_mobile"": true}" 12136,5,271,2017-05-27 00:45:44,http://jacobs.co/kellen,0.0501203629,70.72.206.58,"{""location"": ""NG"", ""is_mobile"": true}" 12137,5,271,2017-02-16 10:05:57,http://langosh.biz/vince,0.7827857300,44.61.231.129,"{""location"": ""BY"", ""is_mobile"": true}" 12138,5,271,2017-03-27 21:00:32,http://sanford.com/marjory.mraz,0.7837173731,4.126.205.55,"{""location"": ""IT"", ""is_mobile"": false}" 12139,5,271,2017-04-17 08:13:26,http://rempel.name/vella.gibson,0.0916990792,197.217.162.175,"{""location"": ""DO"", ""is_mobile"": true}" 12140,5,271,2017-04-24 20:22:58,http://ankunding.co/meagan,0.1272529807,102.177.96.148,"{""location"": ""IQ"", ""is_mobile"": true}" 12141,5,271,2017-04-24 03:33:35,http://oconnell.info/alexander.stoltenberg,0.4186582783,82.102.176.35,"{""location"": ""TV"", ""is_mobile"": false}" 12142,5,271,2016-12-23 22:15:01,http://mcglynn.info/colt_dubuque,0.5228071822,223.143.53.205,"{""location"": ""TC"", ""is_mobile"": false}" 12143,5,271,2017-04-01 07:52:41,http://mcculloughhudson.com/wilhelmine_prohaska,0.7683406035,64.34.85.172,"{""location"": ""MA"", ""is_mobile"": false}" 12144,5,271,2017-03-01 14:26:58,http://goyette.net/nikita_leffler,0.6256323693,204.145.129.117,"{""location"": ""ZM"", ""is_mobile"": true}" 12145,5,271,2017-04-13 18:29:23,http://jacobson.co/julie,0.3090947700,17.26.27.78,"{""location"": ""AG"", ""is_mobile"": true}" 12146,5,271,2017-04-19 09:46:44,http://oconnerdaniel.io/alice.lesch,0.3026562252,190.123.230.177,"{""location"": ""BT"", ""is_mobile"": true}" 12147,5,271,2017-01-23 05:40:18,http://ernser.io/alek.wehner,0.3374379186,12.236.206.196,"{""location"": ""ZW"", ""is_mobile"": true}" 12148,5,271,2017-05-22 06:07:05,http://nicolas.co/alvera.fadel,0.2390870799,54.202.126.47,"{""location"": ""MG"", ""is_mobile"": false}" 12149,5,271,2017-04-30 16:08:32,http://rutherford.net/bo,0.5254076580,95.194.234.39,"{""location"": ""AF"", ""is_mobile"": false}" 12150,5,271,2017-03-23 15:30:20,http://lockman.com/natalie,0.6999005549,141.197.79.200,"{""location"": ""LC"", ""is_mobile"": true}" 12151,5,271,2017-05-02 21:58:40,http://lebsack.io/rogers_abernathy,0.3167431577,206.17.195.99,"{""location"": ""SH"", ""is_mobile"": true}" 12152,5,271,2017-01-25 13:06:19,http://veumlehner.co/yasmeen,0.5760805995,239.9.174.168,"{""location"": ""CI"", ""is_mobile"": false}" 12153,5,271,2017-01-03 15:21:00,http://lockman.net/lonny_price,0.8140926397,139.93.208.204,"{""location"": ""CD"", ""is_mobile"": true}" 12154,5,271,2017-04-11 21:04:38,http://strosinhettinger.co/cory_pagac,0.6472540787,21.247.62.123,"{""location"": ""AS"", ""is_mobile"": true}" 12155,5,271,2017-06-09 15:02:39,http://emard.com/ashlee,0.1136928866,208.137.16.102,"{""location"": ""BB"", ""is_mobile"": true}" 12156,5,271,2017-05-28 04:47:28,http://denesikberge.com/mac,0.6682052332,98.31.178.107,"{""location"": ""PA"", ""is_mobile"": false}" 12157,5,271,2017-01-20 01:44:09,http://heller.org/darian,0.5804666592,93.41.73.124,"{""location"": ""VU"", ""is_mobile"": false}" 12158,5,271,2017-02-05 04:00:57,http://mullerdicki.info/hazel,0.9811452595,20.156.140.101,"{""location"": ""TD"", ""is_mobile"": true}" 12159,5,271,2017-04-17 14:25:28,http://trantow.name/stephania,0.7461452680,28.190.150.226,"{""location"": ""GH"", ""is_mobile"": false}" 12160,5,271,2017-01-12 09:06:37,http://goyettemetz.io/hermina,0.1317994816,20.161.82.117,"{""location"": ""RE"", ""is_mobile"": true}" 12161,5,271,2017-03-16 09:57:17,http://weimann.org/montana,0.8721054668,124.230.70.87,"{""location"": ""IT"", ""is_mobile"": false}" 15108,6,339,2017-06-13 02:13:15,http://quitzonschimmel.io/rowland,,146.199.33.183,"{""location"": ""LU"", ""is_mobile"": true}" 15109,6,339,2017-05-22 05:27:12,http://leschwisozk.biz/ericka,,39.136.149.120,"{""location"": ""DM"", ""is_mobile"": false}" 15110,6,339,2017-04-30 09:55:48,http://kirlin.co/nina,,28.12.44.105,"{""location"": ""SE"", ""is_mobile"": false}" 15111,6,339,2017-05-11 05:00:54,http://monahanweinat.com/demarco,,113.74.91.192,"{""location"": ""CM"", ""is_mobile"": false}" 15112,6,339,2017-06-09 00:06:59,http://nicolashudson.info/vinnie_heaney,,181.125.226.16,"{""location"": ""CW"", ""is_mobile"": true}" 15113,6,340,2017-05-18 23:27:46,http://bernier.biz/kolby_mertz,,97.133.161.136,"{""location"": ""LC"", ""is_mobile"": false}" 15114,6,340,2016-12-31 20:21:57,http://yundt.info/estell,,135.192.108.85,"{""location"": ""CX"", ""is_mobile"": true}" 15115,6,340,2017-02-06 01:23:19,http://lowe.io/kirsten_schoen,,138.63.173.4,"{""location"": ""CO"", ""is_mobile"": true}" 15116,6,340,2017-03-22 05:42:59,http://ziemann.io/korbin,,115.148.95.14,"{""location"": ""FR"", ""is_mobile"": false}" 15117,6,340,2017-02-03 01:03:35,http://lockman.co/erin,,210.171.222.157,"{""location"": ""LT"", ""is_mobile"": false}" 15118,6,340,2017-03-30 16:23:43,http://lang.com/liliana,,224.55.72.32,"{""location"": ""AM"", ""is_mobile"": false}" 15119,6,340,2016-12-30 15:38:43,http://weberwilliamson.biz/cleo_kaulke,,90.178.59.232,"{""location"": ""RE"", ""is_mobile"": false}" 15120,6,340,2017-02-18 17:30:08,http://whitekuvalis.com/mariano.kemmer,,163.70.211.154,"{""location"": ""PS"", ""is_mobile"": false}" 15121,6,340,2017-05-19 22:51:38,http://schowalter.io/damon.sauer,,219.19.152.111,"{""location"": ""UG"", ""is_mobile"": false}" 15122,6,340,2017-04-22 20:41:24,http://larkin.co/brycen_okon,,175.76.12.59,"{""location"": ""GE"", ""is_mobile"": true}" 15123,6,340,2017-01-06 20:25:58,http://shields.co/norval,,81.23.211.183,"{""location"": ""EH"", ""is_mobile"": false}" 15124,6,340,2017-03-19 17:41:09,http://gusikowskiframi.biz/terrill,,176.84.63.22,"{""location"": ""GE"", ""is_mobile"": false}" 15125,6,340,2017-06-10 10:55:49,http://vandervort.org/ransom.oconnell,,99.140.192.94,"{""location"": ""EC"", ""is_mobile"": true}" 15126,6,340,2017-01-19 05:02:28,http://legrosklein.info/marjory_crona,,3.104.49.86,"{""location"": ""GE"", ""is_mobile"": true}" 15127,6,340,2017-05-13 04:16:42,http://pfannerstill.info/electa,,176.249.205.19,"{""location"": ""AQ"", ""is_mobile"": true}" 15128,6,340,2017-04-26 04:35:03,http://heelabbott.io/annabelle,,103.248.9.169,"{""location"": ""HR"", ""is_mobile"": true}" 15129,6,340,2017-04-25 08:34:33,http://braunkeler.biz/kacey.farrell,,70.50.196.188,"{""location"": ""SR"", ""is_mobile"": false}" 15130,6,340,2017-02-14 16:53:10,http://dubuquebruen.biz/mertie,,139.161.188.119,"{""location"": ""ZA"", ""is_mobile"": true}" 15131,6,340,2017-03-02 09:29:25,http://oconnerbailey.co/darrin,,21.152.34.176,"{""location"": ""HN"", ""is_mobile"": false}" 15132,6,340,2017-01-02 09:36:11,http://stracke.io/lorna,,91.124.202.160,"{""location"": ""NI"", ""is_mobile"": true}" 15133,6,340,2016-12-14 07:41:02,http://spinka.biz/kenyon,,13.68.238.204,"{""location"": ""CA"", ""is_mobile"": true}" 15134,6,340,2017-01-11 16:12:28,http://gaylordmills.net/colton.eichmann,,76.19.80.126,"{""location"": ""BE"", ""is_mobile"": true}" 15135,6,340,2016-12-22 09:29:56,http://ricewilkinson.org/norris,,239.184.204.152,"{""location"": ""GT"", ""is_mobile"": false}" 15136,6,340,2017-04-06 17:15:12,http://stracke.biz/braulio.okeefe,,12.192.6.161,"{""location"": ""AT"", ""is_mobile"": false}" 15137,6,340,2017-05-12 14:48:56,http://greenfelder.net/savanna.schultz,,23.173.41.147,"{""location"": ""DK"", ""is_mobile"": true}" 15138,6,340,2017-05-10 19:03:31,http://kiehnerdman.net/pablo,,140.126.228.70,"{""location"": ""LT"", ""is_mobile"": false}" 15139,6,340,2017-03-08 07:03:19,http://jast.net/asia,,229.30.51.56,"{""location"": ""CF"", ""is_mobile"": true}" 15140,6,340,2017-02-27 16:58:46,http://schuster.biz/carson,,246.223.23.129,"{""location"": ""BE"", ""is_mobile"": false}" 15141,6,340,2017-01-15 20:37:54,http://vonruedenhane.com/angie,,165.79.69.159,"{""location"": ""PE"", ""is_mobile"": true}" 15142,6,340,2016-12-20 22:25:07,http://hanemccullough.name/stacy,,118.62.236.50,"{""location"": ""MA"", ""is_mobile"": false}" 15143,6,340,2017-05-06 20:12:41,http://rennerheaney.io/sydni.bins,,107.35.89.161,"{""location"": ""JO"", ""is_mobile"": false}" 15144,6,340,2017-04-11 04:19:14,http://lesch.info/amos,,96.218.220.60,"{""location"": ""GQ"", ""is_mobile"": true}" 15145,6,340,2017-04-25 10:24:45,http://bergnaum.info/lynn.aufderhar,,170.204.134.103,"{""location"": ""ZA"", ""is_mobile"": true}" 15146,6,340,2017-04-27 23:47:24,http://schumm.name/arely,,190.41.93.33,"{""location"": ""RS"", ""is_mobile"": true}" 15147,6,340,2017-04-22 18:45:06,http://okon.info/danielle,,136.191.169.39,"{""location"": ""TO"", ""is_mobile"": true}" 15148,6,340,2017-04-30 13:18:17,http://pacocha.co/darrin,,216.240.62.253,"{""location"": ""NF"", ""is_mobile"": true}" 15149,6,340,2017-04-14 23:00:33,http://schimmel.info/madisyn,,212.217.109.12,"{""location"": ""TG"", ""is_mobile"": false}" 15150,6,340,2017-02-22 18:59:23,http://berge.org/kadin,,71.187.206.241,"{""location"": ""TC"", ""is_mobile"": true}" 15151,6,340,2016-12-21 00:44:38,http://lehnermoore.net/savannah.schroeder,,90.249.144.90,"{""location"": ""LI"", ""is_mobile"": true}" 15152,6,340,2017-06-10 00:56:07,http://schowalter.com/penelope.wiza,,83.5.172.29,"{""location"": ""SZ"", ""is_mobile"": true}" 15153,6,340,2017-04-29 10:53:11,http://bradtke.org/roxanne,,246.233.96.143,"{""location"": ""AD"", ""is_mobile"": false}" 15154,6,340,2017-05-25 10:43:29,http://heidenreich.org/tyler_barrows,,81.41.111.37,"{""location"": ""NG"", ""is_mobile"": false}" 15155,6,340,2017-01-30 17:31:52,http://jerde.info/beau.toy,,7.183.18.90,"{""location"": ""NL"", ""is_mobile"": false}" 15156,6,340,2017-03-10 11:08:23,http://smith.info/verona_emmerich,,62.9.23.16,"{""location"": ""HU"", ""is_mobile"": true}" 15157,6,340,2017-03-12 11:40:22,http://mulleryost.biz/iliana,,202.177.213.74,"{""location"": ""NU"", ""is_mobile"": false}" 15158,6,340,2017-04-11 00:33:37,http://hartmanncarter.org/ernestina.ratke,,211.137.153.6,"{""location"": ""PH"", ""is_mobile"": false}" 15159,6,340,2017-01-07 04:16:55,http://daniel.com/asa,,222.174.254.194,"{""location"": ""PH"", ""is_mobile"": false}" 15160,6,341,2017-04-02 23:34:09,http://pagac.io/aric_lindgren,,169.118.182.220,"{""location"": ""MA"", ""is_mobile"": false}" 15161,6,341,2017-03-15 00:01:13,http://barrowsmarquardt.org/idella,,50.77.220.152,"{""location"": ""MH"", ""is_mobile"": true}" 15162,6,341,2017-06-11 01:01:21,http://grimes.biz/samara.von,,194.114.126.236,"{""location"": ""BW"", ""is_mobile"": true}" 15163,6,341,2016-12-29 14:53:43,http://block.org/cristina_schiller,,43.143.64.78,"{""location"": ""ET"", ""is_mobile"": false}" 2315,2,52,2017-03-28 03:00:59,http://bechtelarcain.info/annetta,0.8117562816,94.27.121.20,"{""location"": ""TJ"", ""is_mobile"": true}" 2316,2,52,2017-03-22 15:11:20,http://larson.info/lolita.blanda,0.0864011113,166.9.120.174,"{""location"": ""BB"", ""is_mobile"": false}" 2317,2,52,2017-04-09 16:21:28,http://runte.com/lucile,0.5552759115,136.114.18.117,"{""location"": ""BI"", ""is_mobile"": false}" 2318,2,52,2017-03-28 11:19:40,http://huels.co/tamia_reilly,0.4273676662,206.193.206.127,"{""location"": ""CG"", ""is_mobile"": true}" 2319,2,52,2016-12-20 12:47:52,http://boehm.co/bradly,0.4063026458,31.151.2.100,"{""location"": ""CH"", ""is_mobile"": false}" 2320,2,52,2016-12-24 10:38:34,http://grantkoepp.com/peggie.fahey,0.7660398481,40.51.228.228,"{""location"": ""FJ"", ""is_mobile"": true}" 2321,2,52,2017-04-15 19:51:51,http://krisbecker.info/roxane.fay,0.0729742021,120.82.183.116,"{""location"": ""SS"", ""is_mobile"": false}" 2322,2,52,2017-05-10 19:22:39,http://gusikowski.org/evangeline,0.0457805507,142.244.229.31,"{""location"": ""CC"", ""is_mobile"": true}" 2323,2,52,2017-02-18 21:54:40,http://marquardt.biz/alisha,0.6443649442,105.179.115.94,"{""location"": ""NL"", ""is_mobile"": true}" 2324,2,52,2016-12-26 14:57:22,http://ortiz.name/hank,0.2442663746,248.29.27.188,"{""location"": ""FR"", ""is_mobile"": false}" 2325,2,52,2016-12-30 15:36:54,http://goldner.org/reilly_maggio,0.8042181519,181.199.189.153,"{""location"": ""UM"", ""is_mobile"": true}" 2326,2,52,2017-02-17 12:59:58,http://casper.net/kristopher,0.2699786469,23.138.31.248,"{""location"": ""IE"", ""is_mobile"": true}" 2327,2,52,2017-02-10 02:42:45,http://watsicaschuppe.com/edd,0.2284343090,64.101.58.210,"{""location"": ""NC"", ""is_mobile"": true}" 2328,2,52,2017-01-07 22:10:25,http://gutkowski.biz/amanda,0.0311637941,221.128.65.186,"{""location"": ""IE"", ""is_mobile"": true}" 2329,2,52,2017-02-08 19:01:58,http://johns.name/piper.schmitt,0.0032607542,74.147.205.245,"{""location"": ""LR"", ""is_mobile"": true}" 2330,2,52,2016-12-25 19:53:41,http://ruecker.net/alana.effertz,0.1777038242,152.5.39.152,"{""location"": ""MT"", ""is_mobile"": false}" 2331,2,52,2017-04-21 11:32:28,http://klein.biz/reyes,0.8044369694,37.5.108.165,"{""location"": ""BM"", ""is_mobile"": true}" 2332,2,52,2016-12-28 05:56:18,http://lynchmurray.co/alexane,0.2049284918,113.204.192.139,"{""location"": ""NR"", ""is_mobile"": false}" 2333,2,52,2016-12-25 17:56:08,http://kuphal.biz/aurelia,0.1870730105,156.83.117.89,"{""location"": ""CR"", ""is_mobile"": true}" 2334,2,52,2017-01-26 22:26:05,http://daregreen.net/kelley.beahan,0.5246625391,89.44.42.42,"{""location"": ""HN"", ""is_mobile"": false}" 2335,2,52,2017-05-02 19:26:11,http://kovacek.info/maximillian_lang,0.1232737121,243.24.5.206,"{""location"": ""VC"", ""is_mobile"": false}" 2336,2,52,2017-01-25 22:42:35,http://zulaufmills.biz/chloe,0.4194587873,29.242.81.115,"{""location"": ""PY"", ""is_mobile"": true}" 2337,2,52,2017-03-24 07:54:29,http://kertzmann.com/andreanne_witting,0.3723470282,149.29.115.136,"{""location"": ""RO"", ""is_mobile"": false}" 2338,2,52,2017-03-29 05:27:05,http://walter.info/nona_weber,0.9891523491,153.64.109.208,"{""location"": ""MR"", ""is_mobile"": false}" 2339,2,52,2017-02-17 15:54:26,http://bergnaummraz.co/devon.wyman,0.0708524469,142.122.34.62,"{""location"": ""IT"", ""is_mobile"": false}" 2340,2,52,2016-12-28 23:26:58,http://langworth.net/shaniya,0.6008142440,98.24.131.218,"{""location"": ""SB"", ""is_mobile"": true}" 2341,2,52,2016-12-14 18:33:06,http://kertzmann.co/zita_tillman,0.5359397516,99.191.68.193,"{""location"": ""TL"", ""is_mobile"": false}" 2342,2,52,2017-05-10 02:56:30,http://grimes.info/alyce,0.4276145713,211.106.96.176,"{""location"": ""NU"", ""is_mobile"": true}" 2343,2,52,2017-02-25 06:00:35,http://davis.info/trea,0.4742620606,18.168.220.134,"{""location"": ""HK"", ""is_mobile"": true}" 2344,2,52,2017-03-31 18:13:52,http://conroy.name/denis,0.8156568666,200.66.223.134,"{""location"": ""FO"", ""is_mobile"": true}" 2345,2,52,2017-01-04 21:58:20,http://becker.io/jalen,0.0056497214,12.66.135.32,"{""location"": ""KI"", ""is_mobile"": false}" 2346,2,52,2017-06-06 22:56:03,http://wehner.biz/noe,0.7771645561,102.229.136.44,"{""location"": ""BR"", ""is_mobile"": true}" 2347,2,52,2017-05-27 19:44:33,http://haley.name/jed,0.2199990683,246.22.72.58,"{""location"": ""MY"", ""is_mobile"": false}" 2348,2,52,2017-03-10 14:53:14,http://effertz.io/hortense.okon,0.3712002844,116.234.223.235,"{""location"": ""ZM"", ""is_mobile"": false}" 2349,2,52,2017-01-11 21:42:01,http://gottlieb.net/lucy_champlin,0.3727873244,150.99.219.56,"{""location"": ""BF"", ""is_mobile"": true}" 2350,2,53,2017-03-25 19:08:20,http://ruecker.net/saige_glover,0.5164576592,52.19.130.63,"{""location"": ""JE"", ""is_mobile"": true}" 2351,2,53,2017-05-02 11:34:12,http://armstrong.io/mellie,0.5096708646,27.100.193.245,"{""location"": ""TC"", ""is_mobile"": false}" 2352,2,53,2017-04-26 03:59:20,http://ankunding.biz/kody.brakus,0.8617573774,170.31.17.118,"{""location"": ""MG"", ""is_mobile"": true}" 2353,2,53,2017-03-15 13:08:17,http://kub.org/henry.ratke,0.3484772765,189.37.198.112,"{""location"": ""SL"", ""is_mobile"": false}" 2354,2,53,2017-05-18 19:33:14,http://marquardtrunte.co/adriana,0.1508065710,178.54.214.115,"{""location"": ""GL"", ""is_mobile"": true}" 2355,2,53,2017-01-16 15:30:56,http://ruelmarquardt.biz/kaylin_hyatt,0.7883620728,162.21.226.246,"{""location"": ""KZ"", ""is_mobile"": true}" 2356,2,53,2016-12-21 02:57:42,http://ernserfadel.org/kailey,0.3706178185,212.119.250.114,"{""location"": ""BY"", ""is_mobile"": false}" 2357,2,53,2017-01-15 02:16:52,http://hansenschultz.org/lloyd_rosenbaum,0.0349892231,60.234.145.107,"{""location"": ""LB"", ""is_mobile"": false}" 2358,2,53,2017-03-01 07:25:31,http://veumstrosin.com/vernice_hartmann,0.1261609038,68.23.54.201,"{""location"": ""JM"", ""is_mobile"": true}" 2359,2,53,2017-04-15 07:59:47,http://murazikstehr.co/timmy,0.8189687834,163.208.25.209,"{""location"": ""GT"", ""is_mobile"": false}" 2360,2,53,2017-01-13 21:50:51,http://conn.net/elta.white,0.4690406864,126.105.241.108,"{""location"": ""QA"", ""is_mobile"": true}" 2361,2,53,2017-03-17 07:48:54,http://oberbrunnermuller.name/libbie,0.7521856388,167.185.68.138,"{""location"": ""TF"", ""is_mobile"": true}" 2362,2,53,2017-03-24 23:03:03,http://ziemeemard.io/addie,0.6090183262,80.138.96.214,"{""location"": ""KM"", ""is_mobile"": false}" 2363,2,53,2017-02-09 09:10:21,http://hettinger.co/shanon_shields,0.5964716013,248.83.118.80,"{""location"": ""CV"", ""is_mobile"": true}" 2364,2,53,2017-05-02 18:27:10,http://graham.biz/drew,0.5118672575,215.91.251.12,"{""location"": ""MV"", ""is_mobile"": false}" 9246,4,207,2017-06-14 04:47:00,http://bins.info/cleta_dietrich,0.2631021989,189.187.76.41,"{""location"": ""PF"", ""is_mobile"": false}" 9247,4,207,2016-12-30 17:27:27,http://muller.net/westley,0.5409273462,105.238.222.248,"{""location"": ""IO"", ""is_mobile"": true}" 9248,4,207,2017-02-15 19:05:50,http://faheywyman.com/telly_predovic,0.5960631414,57.27.41.129,"{""location"": ""ME"", ""is_mobile"": false}" 9249,4,207,2017-05-30 18:40:55,http://croninarmstrong.info/bartholome_rath,0.8393735103,37.124.63.219,"{""location"": ""MW"", ""is_mobile"": true}" 9250,4,207,2017-03-22 13:36:42,http://mccullough.org/elody.lebsack,0.7581915253,246.164.117.245,"{""location"": ""AO"", ""is_mobile"": true}" 9251,4,207,2017-02-02 16:37:55,http://hammes.info/anastasia_conn,0.5398227014,186.192.110.250,"{""location"": ""SK"", ""is_mobile"": false}" 9252,4,207,2017-05-12 22:36:31,http://leannon.io/austen,0.2575185081,132.16.25.211,"{""location"": ""SH"", ""is_mobile"": true}" 9253,4,207,2017-03-30 14:53:47,http://roberts.net/beverly,0.0689722918,179.115.250.70,"{""location"": ""CR"", ""is_mobile"": false}" 9254,4,207,2017-04-30 15:56:32,http://willms.com/retha_moore,0.9440775669,248.67.129.253,"{""location"": ""MH"", ""is_mobile"": true}" 9255,4,207,2016-12-16 07:48:03,http://steuber.info/elfrieda,0.7630692993,215.46.50.189,"{""location"": ""AF"", ""is_mobile"": false}" 9256,4,207,2016-12-21 18:31:48,http://champlinschaefer.org/ebba_wuckert,0.0005517334,63.134.92.186,"{""location"": ""UG"", ""is_mobile"": false}" 9257,4,207,2017-03-28 18:10:23,http://turner.name/andrew.schimmel,0.9356970245,41.96.86.233,"{""location"": ""MH"", ""is_mobile"": false}" 9258,4,207,2017-01-04 11:10:24,http://dicki.org/darlene_strosin,0.5237157390,212.50.230.230,"{""location"": ""HU"", ""is_mobile"": true}" 9259,4,207,2017-02-16 19:48:08,http://beatty.biz/jaquelin,0.6996001955,110.220.132.148,"{""location"": ""BY"", ""is_mobile"": false}" 9260,4,207,2017-05-26 22:59:56,http://greenfelder.co/dena.nader,0.8238089952,109.55.35.18,"{""location"": ""GQ"", ""is_mobile"": true}" 9261,4,207,2017-05-18 19:54:40,http://herman.net/reanna,0.7237817657,227.23.150.53,"{""location"": ""PA"", ""is_mobile"": false}" 9262,4,207,2017-01-31 02:04:01,http://blandamayert.biz/dario.ko,0.7458458958,212.17.221.38,"{""location"": ""GW"", ""is_mobile"": false}" 9263,4,207,2017-04-25 21:14:35,http://turner.co/josefina.nikolaus,0.2416807078,138.196.203.219,"{""location"": ""KG"", ""is_mobile"": false}" 9264,4,207,2017-05-27 05:10:44,http://dietrich.com/tyreek.bauch,0.4698139780,186.67.198.89,"{""location"": ""GU"", ""is_mobile"": true}" 9265,4,207,2017-04-25 15:32:03,http://rosenbaum.name/rene_koepp,0.6125750907,119.99.124.112,"{""location"": ""IL"", ""is_mobile"": false}" 9266,4,207,2017-05-31 09:37:58,http://beatty.info/crystal,0.1021023838,124.122.99.151,"{""location"": ""LT"", ""is_mobile"": false}" 9267,4,208,2017-05-03 00:48:13,http://buckridgevonrueden.org/mercedes_hayes,0.1183813167,249.238.28.243,"{""location"": ""VG"", ""is_mobile"": true}" 9268,4,208,2017-03-07 11:54:50,http://kohlerwunsch.name/marcos,0.7167534405,29.215.249.127,"{""location"": ""GF"", ""is_mobile"": true}" 9269,4,208,2017-04-16 05:58:18,http://jerde.org/bartholome_gerlach,0.2646265080,58.84.198.97,"{""location"": ""CR"", ""is_mobile"": false}" 9270,4,208,2017-02-09 17:39:16,http://wiza.net/cleora,0.3902235589,105.50.207.112,"{""location"": ""AS"", ""is_mobile"": true}" 9271,4,208,2017-03-02 20:24:57,http://nicolas.name/paula,0.4779355366,242.99.155.6,"{""location"": ""AG"", ""is_mobile"": false}" 9272,4,208,2017-06-03 18:52:03,http://considine.name/geraldine.rogahn,0.5257825220,208.215.227.234,"{""location"": ""LA"", ""is_mobile"": true}" 9273,4,208,2017-03-08 02:55:13,http://veum.name/rigoberto_buckridge,0.0987337405,232.180.204.222,"{""location"": ""BN"", ""is_mobile"": true}" 9274,4,208,2017-05-23 07:10:33,http://moen.info/annette,0.3740418999,32.108.189.123,"{""location"": ""MV"", ""is_mobile"": false}" 9275,4,208,2017-01-26 06:19:22,http://abshire.net/carolina,0.7237835254,18.22.152.92,"{""location"": ""BQ"", ""is_mobile"": true}" 9276,4,208,2017-04-29 04:28:20,http://runtehalvorson.com/creola,0.0785254861,190.64.16.191,"{""location"": ""AG"", ""is_mobile"": false}" 9277,4,208,2017-05-21 06:41:55,http://walker.net/vivianne,0.4224005545,25.140.118.103,"{""location"": ""MY"", ""is_mobile"": true}" 9278,4,208,2017-04-11 21:12:32,http://harber.com/nathaniel.larkin,0.3894485898,243.106.225.186,"{""location"": ""SS"", ""is_mobile"": true}" 9279,4,208,2017-04-04 17:07:04,http://leuschke.co/jennyfer.weimann,0.2077013213,10.203.221.132,"{""location"": ""CU"", ""is_mobile"": true}" 9280,4,208,2017-02-13 16:49:02,http://rippin.info/otilia,0.4329094838,107.121.46.49,"{""location"": ""GM"", ""is_mobile"": true}" 9281,4,208,2017-01-12 20:28:23,http://nicolas.net/sally_dicki,0.7407448081,252.47.127.198,"{""location"": ""CY"", ""is_mobile"": false}" 9282,4,208,2017-05-30 07:46:11,http://hanefeest.info/darrick.schuppe,0.6651645634,15.183.77.93,"{""location"": ""KG"", ""is_mobile"": false}" 9283,4,208,2017-03-05 02:42:43,http://boganmayert.net/leon,0.9191018167,118.89.229.131,"{""location"": ""VA"", ""is_mobile"": false}" 9284,4,208,2016-12-14 02:32:41,http://armstrong.name/lucienne.rowe,0.5926753287,125.220.90.233,"{""location"": ""NO"", ""is_mobile"": false}" 9285,4,208,2017-05-02 09:04:33,http://boylejaskolski.com/gregg.terry,0.7420721660,108.57.239.78,"{""location"": ""MS"", ""is_mobile"": false}" 9286,4,208,2017-05-29 19:42:16,http://green.com/jamir,0.1356057665,174.21.207.58,"{""location"": ""CY"", ""is_mobile"": true}" 9287,4,208,2017-02-21 17:55:01,http://zieme.info/zetta_bruen,0.1907284576,224.31.216.135,"{""location"": ""LU"", ""is_mobile"": true}" 9288,4,208,2017-03-07 07:38:36,http://schadenglover.io/stevie,0.6452982011,171.60.41.143,"{""location"": ""LC"", ""is_mobile"": false}" 9289,4,208,2017-05-14 01:51:53,http://purdy.biz/providenci,0.4299172987,204.94.15.23,"{""location"": ""FR"", ""is_mobile"": false}" 9290,4,208,2017-01-15 07:58:29,http://kuhlmanjacobson.org/brandt,0.6836529775,233.242.49.177,"{""location"": ""MY"", ""is_mobile"": false}" 9291,4,208,2017-04-01 00:37:05,http://smith.name/johnnie,0.2417132898,219.150.93.145,"{""location"": ""GE"", ""is_mobile"": false}" 9292,4,208,2017-05-31 11:58:12,http://schultzwilkinson.name/estefania,0.2319053558,228.102.183.26,"{""location"": ""TG"", ""is_mobile"": true}" 9293,4,208,2017-04-22 07:12:06,http://hettinger.biz/foster,0.7954642393,149.232.31.71,"{""location"": ""TD"", ""is_mobile"": false}" 9294,4,208,2017-05-06 14:26:41,http://johnsonhand.name/aurelia,0.5564617573,128.99.246.92,"{""location"": ""SL"", ""is_mobile"": true}" 9295,4,208,2017-01-08 12:47:27,http://veum.com/rozella_schimmel,0.2940808970,27.13.15.206,"{""location"": ""SH"", ""is_mobile"": false}" 9296,4,208,2017-05-17 15:24:07,http://zemlak.info/christine,0.0023034185,164.163.167.147,"{""location"": ""GS"", ""is_mobile"": true}" 6256,3,137,2017-04-17 08:15:30,http://schillergleichner.com/cierra_davis,0.3444077548,161.45.112.132,"{""location"": ""JP"", ""is_mobile"": false}" 6257,3,137,2017-03-30 20:21:27,http://faheyhaag.net/ruthe.macejkovic,0.3291536209,29.2.189.70,"{""location"": ""GP"", ""is_mobile"": false}" 6258,3,137,2017-05-09 18:23:39,http://emardraynor.co/greg.langworth,0.3854922903,191.188.108.136,"{""location"": ""CZ"", ""is_mobile"": false}" 6259,3,137,2017-06-11 14:33:03,http://funk.com/sophie,0.9211509777,137.234.71.24,"{""location"": ""MZ"", ""is_mobile"": false}" 6260,3,137,2017-02-08 10:50:26,http://dibbertcollier.net/ludie_monahan,0.0002704200,56.79.186.246,"{""location"": ""SK"", ""is_mobile"": false}" 6261,3,137,2017-04-17 14:08:04,http://turnerkuvalis.name/haven,0.9274661383,68.71.125.223,"{""location"": ""GT"", ""is_mobile"": true}" 6262,3,137,2017-03-12 22:23:09,http://dickischuppe.org/aryanna,0.0865578861,108.104.94.84,"{""location"": ""SR"", ""is_mobile"": false}" 6263,3,137,2017-03-04 17:10:58,http://cronacrooks.name/elias_spencer,0.1299530524,7.185.163.158,"{""location"": ""DE"", ""is_mobile"": false}" 6264,3,137,2017-04-23 01:16:07,http://pagac.com/mariela,0.3814399773,158.114.28.213,"{""location"": ""PF"", ""is_mobile"": true}" 6265,3,137,2017-01-23 19:35:39,http://rohanthiel.info/edward_cormier,0.5069687449,169.164.161.190,"{""location"": ""GP"", ""is_mobile"": true}" 6266,3,137,2017-02-20 14:28:28,http://konopelski.name/delmer,0.6576658097,247.60.209.112,"{""location"": ""BQ"", ""is_mobile"": false}" 6267,3,137,2016-12-22 11:16:04,http://stracke.org/nicolette.wintheiser,0.5536527877,177.101.26.33,"{""location"": ""GW"", ""is_mobile"": false}" 6268,3,137,2017-02-17 05:45:28,http://ullrich.org/otis,0.5446096613,129.220.162.148,"{""location"": ""SH"", ""is_mobile"": false}" 6269,3,137,2017-05-17 17:52:59,http://beattyschmitt.co/jeremie_torphy,0.6997734032,139.179.80.164,"{""location"": ""KG"", ""is_mobile"": false}" 6270,3,137,2017-04-16 19:12:47,http://ziemann.org/jamison_langosh,0.3558893123,196.178.48.201,"{""location"": ""ER"", ""is_mobile"": true}" 6271,3,137,2017-05-10 20:02:26,http://ankunding.biz/malinda,0.4119439340,18.241.41.194,"{""location"": ""BW"", ""is_mobile"": false}" 6272,3,137,2017-01-28 23:05:09,http://satterfield.net/marcos.marvin,0.5904091713,66.248.240.114,"{""location"": ""BF"", ""is_mobile"": false}" 6273,3,137,2017-03-12 17:04:40,http://jaskolskiwhite.biz/mabelle.kihn,0.8506620112,135.153.151.69,"{""location"": ""ZW"", ""is_mobile"": false}" 6274,3,137,2017-03-20 05:37:50,http://heelromaguera.co/brenna.terry,0.1043294138,133.55.116.145,"{""location"": ""DZ"", ""is_mobile"": true}" 6275,3,137,2017-01-17 17:36:53,http://coledouglas.info/noah,0.5002019950,222.128.20.73,"{""location"": ""YE"", ""is_mobile"": true}" 6276,3,137,2016-12-17 00:57:44,http://yundtlehner.com/katlyn,0.5099815493,58.17.63.129,"{""location"": ""MR"", ""is_mobile"": false}" 6277,3,137,2017-03-10 03:20:43,http://schumm.org/kayden.collins,0.9535713041,225.241.103.55,"{""location"": ""TO"", ""is_mobile"": true}" 6278,3,137,2017-03-25 09:02:52,http://stamm.org/hobart.emard,0.3547567871,70.36.170.76,"{""location"": ""DZ"", ""is_mobile"": true}" 6279,3,137,2017-05-07 04:34:14,http://stokes.com/bret_langworth,0.2248224318,98.10.143.153,"{""location"": ""KG"", ""is_mobile"": false}" 6280,3,137,2017-06-02 01:11:44,http://colebechtelar.name/zane.kozey,0.1504423568,229.2.94.68,"{""location"": ""GI"", ""is_mobile"": true}" 6281,3,137,2017-03-26 09:55:57,http://grahammertz.co/katharina,0.9658975774,187.130.225.65,"{""location"": ""WF"", ""is_mobile"": true}" 6282,3,137,2017-03-01 01:03:45,http://heller.co/maritza,0.8618922175,214.213.189.161,"{""location"": ""MR"", ""is_mobile"": true}" 6283,3,137,2017-06-09 01:20:25,http://reillybruen.io/oceane_ritchie,0.4929228842,77.180.228.211,"{""location"": ""LS"", ""is_mobile"": false}" 6284,3,137,2017-06-06 18:48:56,http://millskulas.io/willow.koelpin,0.3434163902,4.212.16.167,"{""location"": ""FR"", ""is_mobile"": false}" 6285,3,137,2016-12-23 00:58:44,http://little.name/lawrence,0.7815114377,10.238.244.4,"{""location"": ""LV"", ""is_mobile"": true}" 6286,3,137,2017-02-16 12:47:15,http://schillerkonopelski.org/myrtis,0.2057812266,13.12.171.114,"{""location"": ""CF"", ""is_mobile"": true}" 6287,3,137,2017-03-02 18:57:09,http://maggiostokes.name/calista,0.9175527536,23.242.122.55,"{""location"": ""TT"", ""is_mobile"": true}" 6288,3,137,2017-05-08 21:06:57,http://volkman.info/freda.bauch,0.0659148056,252.147.120.169,"{""location"": ""TD"", ""is_mobile"": false}" 6289,3,137,2017-05-29 08:51:33,http://hartmannrenner.name/melyna_fahey,0.4559406338,104.196.82.62,"{""location"": ""PH"", ""is_mobile"": true}" 6290,3,137,2017-05-25 17:21:18,http://mueller.info/alfreda.rogahn,0.2921272151,102.99.4.242,"{""location"": ""PM"", ""is_mobile"": false}" 6291,3,137,2017-05-18 22:21:18,http://collinporer.info/paul.gutmann,0.7008044921,124.137.231.49,"{""location"": ""NE"", ""is_mobile"": true}" 6292,3,137,2017-04-26 16:53:33,http://gislason.net/retta,0.8635893812,69.213.114.123,"{""location"": ""PR"", ""is_mobile"": true}" 6293,3,137,2017-05-14 22:05:24,http://olson.net/alfreda,0.9498455135,58.83.100.247,"{""location"": ""GP"", ""is_mobile"": false}" 6294,3,137,2017-04-16 00:13:26,http://wolf.info/stefanie_kirlin,0.5415256470,111.71.111.41,"{""location"": ""IN"", ""is_mobile"": false}" 6295,3,137,2017-05-21 07:39:04,http://okuneva.info/imani,0.1476843409,5.17.212.54,"{""location"": ""TM"", ""is_mobile"": false}" 6296,3,137,2017-04-25 23:52:50,http://bogisich.net/jerome,0.0893724673,12.141.130.212,"{""location"": ""CL"", ""is_mobile"": true}" 6297,3,137,2017-02-01 06:50:01,http://douglas.com/jensen.fritsch,0.8859816902,57.198.62.233,"{""location"": ""SX"", ""is_mobile"": false}" 6298,3,137,2017-01-16 23:36:15,http://bechtelar.org/kameron_douglas,0.0955677983,153.211.78.123,"{""location"": ""SX"", ""is_mobile"": true}" 6299,3,137,2017-03-29 22:01:01,http://dickensrogahn.biz/lolita,0.5846314723,163.252.195.73,"{""location"": ""CV"", ""is_mobile"": false}" 6300,3,138,2017-06-12 09:09:17,http://gerhold.com/meredith,0.6334303105,162.137.185.77,"{""location"": ""LI"", ""is_mobile"": true}" 6301,3,138,2017-03-16 00:38:45,http://hettinger.io/madyson.boehm,0.5764337266,66.253.78.69,"{""location"": ""KH"", ""is_mobile"": true}" 6302,3,138,2017-01-15 10:39:20,http://wittingschiller.info/dorian,0.6731242524,49.243.85.121,"{""location"": ""ES"", ""is_mobile"": false}" 6303,3,138,2017-03-20 19:21:39,http://gerlachwisoky.org/destin,0.8032572660,73.243.212.38,"{""location"": ""GM"", ""is_mobile"": false}" 6304,3,138,2017-01-05 15:24:50,http://rowedouglas.info/verna,0.1295793538,129.74.9.24,"{""location"": ""PF"", ""is_mobile"": true}" 6305,3,138,2016-12-27 09:58:26,http://doyle.io/magdalen.damore,0.2432063415,220.168.187.49,"{""location"": ""MQ"", ""is_mobile"": false}" 6306,3,138,2017-04-14 12:23:51,http://schmidt.co/glennie_sipes,0.9389140290,92.13.111.249,"{""location"": ""EC"", ""is_mobile"": true}" 12162,5,271,2017-01-23 17:31:21,http://halvorson.biz/eden.romaguera,0.9152243315,154.169.230.5,"{""location"": ""FM"", ""is_mobile"": false}" 12163,5,271,2017-03-24 03:11:13,http://keler.net/miracle,0.4523198428,77.30.59.138,"{""location"": ""KG"", ""is_mobile"": false}" 12164,5,271,2017-03-25 13:44:55,http://lindgrencormier.name/talon,0.2941577176,18.138.174.22,"{""location"": ""CG"", ""is_mobile"": true}" 12165,5,271,2016-12-26 22:38:21,http://collins.org/herbert.sauer,0.3437208237,32.38.198.224,"{""location"": ""WF"", ""is_mobile"": true}" 12166,5,271,2017-04-25 10:05:12,http://runolfoneffertz.com/gardner,0.5347472070,126.115.246.73,"{""location"": ""TW"", ""is_mobile"": true}" 12167,5,271,2017-03-10 09:13:23,http://prohaska.org/aiden,0.0395803779,47.154.100.116,"{""location"": ""NP"", ""is_mobile"": false}" 12168,5,271,2017-05-21 21:24:56,http://little.name/katlynn,0.6627255584,165.58.71.202,"{""location"": ""IN"", ""is_mobile"": false}" 12169,5,271,2017-05-23 18:28:10,http://wuckert.com/keira.mayer,0.3581529267,33.215.107.112,"{""location"": ""GA"", ""is_mobile"": true}" 12170,5,271,2017-04-11 07:47:30,http://langworthmurazik.co/agustin.kunze,0.5502703345,104.49.202.144,"{""location"": ""HK"", ""is_mobile"": true}" 12171,5,271,2017-03-26 15:17:48,http://considine.net/noemi.orn,0.5803195671,208.108.180.235,"{""location"": ""MK"", ""is_mobile"": true}" 12172,5,271,2017-03-30 06:42:38,http://gaylordbarton.net/dora.ullrich,0.3028631099,49.45.48.45,"{""location"": ""PH"", ""is_mobile"": false}" 12173,5,272,2017-01-06 02:59:11,http://smithamdurgan.co/della,0.7392904654,92.160.227.167,"{""location"": ""TN"", ""is_mobile"": true}" 12174,5,272,2017-01-28 01:04:41,http://osinski.io/zoey,0.3231424598,19.84.135.166,"{""location"": ""ST"", ""is_mobile"": true}" 12175,5,272,2017-04-21 06:53:01,http://turner.biz/lina_kub,0.4225825896,224.252.189.112,"{""location"": ""QA"", ""is_mobile"": false}" 12176,5,272,2017-04-27 06:30:10,http://huel.co/alvah.smitham,0.5780021262,14.112.244.125,"{""location"": ""AZ"", ""is_mobile"": false}" 12177,5,272,2016-12-15 11:58:40,http://rogahn.name/erna,0.1400310900,164.123.133.170,"{""location"": ""MW"", ""is_mobile"": true}" 12178,5,272,2017-01-22 12:17:02,http://cummerata.name/estell,0.5946274022,113.233.12.196,"{""location"": ""GM"", ""is_mobile"": true}" 12179,5,272,2016-12-27 15:47:05,http://hueltehr.name/beaulah.kemmer,0.5386903027,76.43.98.199,"{""location"": ""LK"", ""is_mobile"": true}" 12180,5,272,2016-12-31 07:57:57,http://nader.co/haie,0.3698294525,105.162.19.160,"{""location"": ""AS"", ""is_mobile"": false}" 12181,5,272,2016-12-22 18:24:23,http://macgyver.co/adonis,0.6230459711,26.179.106.237,"{""location"": ""TV"", ""is_mobile"": false}" 12182,5,272,2017-02-24 13:27:02,http://schneider.org/ania,0.3245742819,167.123.205.173,"{""location"": ""MK"", ""is_mobile"": false}" 12183,5,272,2017-03-30 18:53:55,http://lubowitzlind.co/mavis,0.1589947128,92.123.61.63,"{""location"": ""BD"", ""is_mobile"": false}" 12184,5,272,2017-06-06 13:08:33,http://gleasonhegmann.co/torey_lehner,0.7753999823,247.73.150.51,"{""location"": ""DK"", ""is_mobile"": true}" 12185,5,272,2017-01-28 21:43:57,http://reinger.info/felicita_kunze,0.1524308345,131.79.57.93,"{""location"": ""VU"", ""is_mobile"": false}" 12186,5,272,2017-03-25 03:31:02,http://halvorson.org/vivien,0.3292610349,34.66.95.108,"{""location"": ""BW"", ""is_mobile"": false}" 12187,5,272,2017-02-09 08:25:51,http://casper.co/nathanial,0.1307309404,118.168.130.80,"{""location"": ""PG"", ""is_mobile"": true}" 12188,5,272,2017-03-23 09:03:58,http://greenfelder.net/beau,0.1135911565,95.52.75.143,"{""location"": ""LV"", ""is_mobile"": true}" 12189,5,272,2017-02-18 20:59:15,http://wolf.org/alize.balistreri,0.7754545229,112.196.145.147,"{""location"": ""GG"", ""is_mobile"": true}" 12190,5,272,2017-01-02 16:46:18,http://kerluketreutel.org/enrique_mertz,0.1628890943,59.88.41.158,"{""location"": ""CR"", ""is_mobile"": false}" 12191,5,272,2017-01-07 01:59:28,http://quitzon.co/libbie.hartmann,0.5713274612,198.119.226.196,"{""location"": ""TR"", ""is_mobile"": false}" 12192,5,272,2016-12-28 01:03:02,http://walter.com/mohammed,0.5828578729,196.144.198.15,"{""location"": ""IS"", ""is_mobile"": false}" 12193,5,272,2017-05-28 05:31:27,http://huels.co/myah,0.5868853343,202.5.5.129,"{""location"": ""SB"", ""is_mobile"": true}" 12194,5,272,2017-01-29 06:49:26,http://nicolas.net/pansy,0.9304664227,110.187.190.55,"{""location"": ""BQ"", ""is_mobile"": false}" 12195,5,272,2017-03-25 01:50:27,http://maggio.net/jaydon.conn,0.0498080181,63.73.80.48,"{""location"": ""LS"", ""is_mobile"": false}" 12196,5,272,2017-02-21 03:32:45,http://donnelly.biz/keyshawn,0.0115978437,52.46.109.141,"{""location"": ""MU"", ""is_mobile"": true}" 12197,5,272,2017-04-05 03:42:15,http://lueilwitz.org/gardner_prosacco,0.0345924742,100.247.119.222,"{""location"": ""IL"", ""is_mobile"": false}" 12198,5,272,2017-01-02 15:14:56,http://beerschulist.name/milo.jacobi,0.5278339549,3.249.91.192,"{""location"": ""MZ"", ""is_mobile"": false}" 12199,5,272,2017-04-25 14:02:35,http://hilpertondricka.co/jennyfer.brekke,0.9814302872,167.37.95.54,"{""location"": ""CR"", ""is_mobile"": false}" 12200,5,272,2017-04-21 09:08:04,http://homenick.biz/norris,0.4516604227,195.35.251.138,"{""location"": ""MO"", ""is_mobile"": false}" 12201,5,272,2017-03-30 03:06:04,http://ritchie.com/elroy_batz,0.9243214607,23.239.37.68,"{""location"": ""PK"", ""is_mobile"": true}" 12202,5,272,2017-05-21 05:58:34,http://cummerata.net/frederic,0.8955865897,188.155.44.250,"{""location"": ""LI"", ""is_mobile"": false}" 12203,5,272,2017-04-07 19:29:53,http://ruel.info/hudson,0.8549933199,199.167.132.32,"{""location"": ""PS"", ""is_mobile"": true}" 12204,5,272,2017-05-15 11:59:43,http://kautzer.biz/sebastian,0.5383722068,249.125.93.21,"{""location"": ""FK"", ""is_mobile"": true}" 12205,5,272,2017-06-05 04:22:06,http://senger.net/nyah,0.7066919282,215.51.15.37,"{""location"": ""GY"", ""is_mobile"": false}" 12206,5,272,2017-06-03 19:16:41,http://bechtelar.net/jakob_predovic,0.8195985815,30.155.143.140,"{""location"": ""GL"", ""is_mobile"": false}" 12207,5,272,2017-03-09 01:49:14,http://mayer.org/kasey,0.9179704231,201.106.133.69,"{""location"": ""GI"", ""is_mobile"": false}" 12208,5,272,2017-06-04 04:43:17,http://hermannwaters.name/novella,0.8506033850,54.2.104.130,"{""location"": ""GD"", ""is_mobile"": true}" 12209,5,272,2017-02-14 23:47:34,http://kautzer.biz/aniya,0.1529949030,144.138.86.136,"{""location"": ""UM"", ""is_mobile"": false}" 12210,5,273,2017-05-12 23:16:02,http://borershields.org/ryan,0.7492205087,191.128.143.44,"{""location"": ""KW"", ""is_mobile"": false}" 12211,5,273,2016-12-27 09:49:37,http://okonlynch.biz/teresa,0.3576184174,14.13.142.170,"{""location"": ""KW"", ""is_mobile"": true}" 12212,5,273,2016-12-20 11:04:06,http://gutkowskikonopelski.com/reina,0.8463747150,125.129.41.4,"{""location"": ""AI"", ""is_mobile"": true}" 15164,6,341,2017-03-18 12:52:44,http://kshlerin.co/nikolas.ebert,,166.131.154.212,"{""location"": ""MU"", ""is_mobile"": true}" 15165,6,341,2017-01-31 07:56:25,http://christiansendaugherty.info/maxine.padberg,,7.194.57.104,"{""location"": ""GI"", ""is_mobile"": false}" 15166,6,341,2017-03-22 21:49:07,http://mueller.biz/evalyn_wehner,,97.54.208.220,"{""location"": ""TK"", ""is_mobile"": true}" 15167,6,341,2017-01-23 00:10:45,http://herman.io/eddie_little,,166.99.243.200,"{""location"": ""HM"", ""is_mobile"": false}" 15168,6,341,2017-06-09 18:32:49,http://becker.net/fidel,,143.160.25.202,"{""location"": ""TN"", ""is_mobile"": false}" 15169,6,341,2017-05-26 16:44:06,http://bechtelar.net/alivia,,86.86.70.8,"{""location"": ""MX"", ""is_mobile"": false}" 15170,6,341,2017-01-11 01:35:58,http://bernhard.co/jose,,65.151.119.65,"{""location"": ""ZW"", ""is_mobile"": true}" 15171,6,341,2017-01-30 14:23:42,http://weber.net/chesley.yost,,77.57.231.165,"{""location"": ""AW"", ""is_mobile"": false}" 15172,6,341,2017-06-13 14:46:19,http://emard.name/ruthie,,186.42.186.169,"{""location"": ""UY"", ""is_mobile"": false}" 15173,6,341,2017-05-05 20:45:14,http://howe.co/rachelle,,44.44.249.197,"{""location"": ""TZ"", ""is_mobile"": false}" 15288,6,344,2017-04-12 09:25:44,http://flatley.org/ahmed,,27.41.53.187,"{""location"": ""BF"", ""is_mobile"": true}" 15174,6,341,2017-02-06 16:52:00,http://lockman.name/jarvis.lakin,,230.224.152.216,"{""location"": ""TK"", ""is_mobile"": true}" 15175,6,341,2017-06-11 17:29:49,http://medhurst.co/dora,,120.185.159.218,"{""location"": ""AL"", ""is_mobile"": false}" 15176,6,341,2016-12-17 18:34:32,http://homenick.biz/ben.howell,,93.18.205.221,"{""location"": ""UZ"", ""is_mobile"": false}" 15177,6,341,2017-04-17 00:23:58,http://feil.com/montana.mayer,,21.77.184.141,"{""location"": ""TF"", ""is_mobile"": false}" 15178,6,341,2017-02-15 20:48:22,http://brownlang.io/otto,,148.52.47.128,"{""location"": ""TW"", ""is_mobile"": false}" 15179,6,341,2017-06-02 20:56:11,http://schneiderglover.name/millie.hartmann,,51.90.230.89,"{""location"": ""IL"", ""is_mobile"": false}" 15180,6,341,2017-05-23 17:37:49,http://moriette.co/megane,,242.121.218.183,"{""location"": ""RS"", ""is_mobile"": false}" 15181,6,341,2017-03-06 09:49:25,http://conroy.biz/josephine,,22.210.218.206,"{""location"": ""CI"", ""is_mobile"": false}" 15182,6,341,2017-02-09 00:19:18,http://wuckert.biz/norwood_ledner,,27.250.6.93,"{""location"": ""AQ"", ""is_mobile"": false}" 15183,6,341,2017-04-24 01:59:28,http://adams.biz/elenor,,79.191.134.89,"{""location"": ""ZM"", ""is_mobile"": true}" 15184,6,341,2017-01-10 19:22:45,http://schiller.com/raina.langosh,,45.49.141.61,"{""location"": ""FK"", ""is_mobile"": false}" 15185,6,341,2017-03-27 08:35:27,http://ziemann.org/carol.walsh,,164.86.156.89,"{""location"": ""AS"", ""is_mobile"": false}" 15186,6,341,2017-01-03 11:08:37,http://blanda.co/rudolph.kemmer,,65.188.53.110,"{""location"": ""ZA"", ""is_mobile"": false}" 15187,6,341,2017-01-30 17:17:12,http://rosenbaum.com/cynthia.windler,,204.134.197.214,"{""location"": ""QA"", ""is_mobile"": true}" 15188,6,341,2017-03-25 11:36:36,http://collierpollich.org/virgie.boehm,,241.20.229.176,"{""location"": ""DE"", ""is_mobile"": true}" 15189,6,341,2017-03-02 00:32:43,http://oconner.name/zita.sanford,,129.183.172.93,"{""location"": ""ID"", ""is_mobile"": true}" 15190,6,341,2017-02-18 14:57:36,http://aufderharlarson.org/nikki.nicolas,,42.36.164.25,"{""location"": ""KP"", ""is_mobile"": false}" 15191,6,342,2016-12-31 10:07:01,http://rippin.org/riley,,77.162.191.36,"{""location"": ""BJ"", ""is_mobile"": true}" 15192,6,342,2017-02-09 17:44:10,http://pagac.name/margarita_walker,,246.170.185.24,"{""location"": ""VU"", ""is_mobile"": true}" 15193,6,342,2017-02-28 11:17:34,http://ruel.com/gay_balistreri,,88.196.190.139,"{""location"": ""BQ"", ""is_mobile"": false}" 15194,6,342,2017-05-23 12:28:19,http://robertshodkiewicz.org/lou,,66.213.167.93,"{""location"": ""LI"", ""is_mobile"": false}" 15195,6,342,2017-05-01 09:11:29,http://koelpin.biz/freeman.wolf,,67.37.79.217,"{""location"": ""ID"", ""is_mobile"": false}" 15196,6,342,2017-03-22 04:13:42,http://cormier.io/consuelo.littel,,170.72.46.154,"{""location"": ""PF"", ""is_mobile"": true}" 15197,6,342,2017-04-01 17:08:53,http://muellermante.co/lamar_farrell,,207.120.236.19,"{""location"": ""AO"", ""is_mobile"": false}" 15198,6,342,2017-05-17 06:35:16,http://bashirian.biz/dean,,117.38.154.98,"{""location"": ""FM"", ""is_mobile"": true}" 15199,6,342,2017-02-15 23:33:46,http://romaguera.com/melvin_littel,,252.165.249.12,"{""location"": ""SA"", ""is_mobile"": false}" 15200,6,342,2017-05-21 05:33:36,http://oconner.org/gerald_schultz,,24.179.128.211,"{""location"": ""HT"", ""is_mobile"": false}" 15201,6,342,2017-04-19 03:57:05,http://leannon.biz/london,,250.79.130.13,"{""location"": ""FJ"", ""is_mobile"": true}" 15202,6,342,2016-12-25 22:23:43,http://schuppe.org/madonna,,126.87.242.73,"{""location"": ""TT"", ""is_mobile"": true}" 15203,6,342,2017-01-28 21:02:12,http://kovacek.biz/zion.rippin,,112.141.244.249,"{""location"": ""GF"", ""is_mobile"": false}" 15204,6,342,2017-01-12 19:42:05,http://beckerarmstrong.info/lloyd_steuber,,49.18.31.254,"{""location"": ""PH"", ""is_mobile"": true}" 15205,6,342,2017-04-03 08:45:01,http://morarernser.com/lottie,,28.92.44.213,"{""location"": ""PL"", ""is_mobile"": true}" 15206,6,342,2017-05-18 07:32:45,http://little.io/marilyne,,167.35.249.35,"{""location"": ""SX"", ""is_mobile"": true}" 15207,6,342,2016-12-16 22:24:13,http://williamson.io/jany,,85.179.173.152,"{""location"": ""BL"", ""is_mobile"": false}" 15208,6,342,2017-05-25 02:31:01,http://schumm.name/janea,,27.8.253.51,"{""location"": ""BO"", ""is_mobile"": true}" 15209,6,342,2017-04-05 02:59:56,http://mann.net/kale,,201.26.66.182,"{""location"": ""MP"", ""is_mobile"": false}" 15210,6,342,2017-03-05 04:39:07,http://murray.info/emmy,,227.177.2.177,"{""location"": ""RU"", ""is_mobile"": false}" 15211,6,342,2017-02-14 23:19:16,http://okeefekirlin.co/daphney,,223.96.179.218,"{""location"": ""CC"", ""is_mobile"": false}" 15212,6,342,2016-12-22 05:13:59,http://ebertosinski.com/dino_connelly,,117.4.224.47,"{""location"": ""AO"", ""is_mobile"": false}" 15213,6,342,2017-04-28 17:54:46,http://yostoreilly.info/mark,,107.78.190.252,"{""location"": ""NU"", ""is_mobile"": false}" 15214,6,342,2017-03-26 17:51:01,http://swaniawski.org/sim.bogan,,94.214.52.117,"{""location"": ""DK"", ""is_mobile"": false}" 15215,6,342,2017-01-23 09:09:44,http://osinskigottlieb.co/deja,,34.210.21.97,"{""location"": ""LT"", ""is_mobile"": false}" 15216,6,342,2017-04-26 06:49:49,http://brakusjenkins.com/jadon,,2.219.119.7,"{""location"": ""TR"", ""is_mobile"": false}" 15217,6,342,2017-01-07 09:15:04,http://abshire.io/colt,,138.97.219.174,"{""location"": ""CA"", ""is_mobile"": true}" 2365,2,53,2017-03-20 17:37:27,http://hane.com/gerson.schinner,0.1946556893,154.155.250.88,"{""location"": ""SX"", ""is_mobile"": true}" 2366,2,53,2017-04-20 12:57:07,http://feeney.org/fatima_flatley,0.8662892300,61.151.212.88,"{""location"": ""MQ"", ""is_mobile"": true}" 2367,2,53,2016-12-13 22:16:03,http://yostfarrell.info/arvid,0.4864209150,79.249.176.54,"{""location"": ""NA"", ""is_mobile"": false}" 2368,2,53,2017-04-24 14:23:18,http://blick.co/susan.kautzer,0.8258063698,36.221.240.119,"{""location"": ""RU"", ""is_mobile"": true}" 2369,2,53,2017-05-14 04:06:05,http://koelpinarmstrong.io/pamela,0.5978810584,9.29.198.173,"{""location"": ""JP"", ""is_mobile"": true}" 2370,2,53,2017-05-22 17:27:43,http://senger.com/lucy_berge,0.0268709428,22.180.145.161,"{""location"": ""PA"", ""is_mobile"": false}" 2371,2,53,2017-02-27 07:25:56,http://zboncak.net/ricky_torp,0.3816027039,137.57.47.138,"{""location"": ""AW"", ""is_mobile"": false}" 2372,2,53,2016-12-30 15:40:30,http://nikolaus.io/kitty_damore,0.6807434403,27.9.80.207,"{""location"": ""YE"", ""is_mobile"": true}" 2373,2,53,2017-03-13 00:58:42,http://wehnermarks.biz/laury_stroman,0.9948558469,104.17.7.100,"{""location"": ""LR"", ""is_mobile"": false}" 2374,2,53,2017-02-26 21:35:49,http://hilll.info/donnell,0.2329052533,182.206.139.166,"{""location"": ""SS"", ""is_mobile"": true}" 2375,2,53,2017-02-23 14:40:44,http://reichel.name/bernard,0.8797106997,91.17.226.131,"{""location"": ""CV"", ""is_mobile"": false}" 2376,2,53,2017-01-11 14:59:17,http://welch.biz/paolo_reichert,0.5211365624,8.130.223.123,"{""location"": ""LR"", ""is_mobile"": true}" 2377,2,53,2017-05-26 22:25:36,http://schimmel.org/danny,0.6611775519,148.4.226.136,"{""location"": ""ID"", ""is_mobile"": false}" 2378,2,53,2017-03-06 07:55:47,http://mueller.net/tremayne,0.9946802045,108.243.121.244,"{""location"": ""SM"", ""is_mobile"": true}" 2379,2,53,2017-03-28 17:39:45,http://gleichner.biz/baron,0.1557417745,158.195.56.201,"{""location"": ""AD"", ""is_mobile"": true}" 2380,2,53,2017-05-10 23:18:45,http://wyman.org/ibrahim.tremblay,0.5578915306,3.220.131.177,"{""location"": ""AT"", ""is_mobile"": false}" 2381,2,53,2017-03-28 11:20:18,http://ziemeryan.net/layne_zboncak,0.6882189772,86.139.152.246,"{""location"": ""GL"", ""is_mobile"": true}" 2382,2,53,2017-05-21 08:04:45,http://yost.net/concepcion_padberg,0.0076692451,117.25.178.30,"{""location"": ""AO"", ""is_mobile"": true}" 2383,2,53,2017-03-04 09:59:51,http://ruel.com/cecelia,0.9087857899,219.148.100.14,"{""location"": ""EH"", ""is_mobile"": false}" 2384,2,53,2016-12-28 22:53:27,http://emmerich.org/avis,0.6129427694,130.65.140.114,"{""location"": ""LU"", ""is_mobile"": false}" 2385,2,53,2017-02-26 11:13:28,http://cormier.io/evans.ziemann,0.3169762101,118.209.12.178,"{""location"": ""CN"", ""is_mobile"": true}" 2386,2,53,2017-03-31 16:07:49,http://hermann.co/damien,0.5544336640,234.54.110.28,"{""location"": ""LB"", ""is_mobile"": false}" 2387,2,53,2017-04-25 01:18:15,http://oconnellhomenick.biz/sasha,0.3456076971,153.82.156.109,"{""location"": ""KE"", ""is_mobile"": true}" 2388,2,53,2017-04-03 23:02:26,http://oconner.io/avis,0.1863634178,52.191.156.122,"{""location"": ""AF"", ""is_mobile"": false}" 2389,2,53,2017-05-20 15:52:53,http://bechtelar.net/reginald.pouros,0.5096195793,47.147.141.157,"{""location"": ""FO"", ""is_mobile"": true}" 2390,2,53,2017-04-25 23:37:27,http://johnsongraham.io/gabe_kunde,0.1904917103,236.108.58.85,"{""location"": ""BS"", ""is_mobile"": true}" 2391,2,53,2017-05-05 03:09:36,http://brakusblanda.name/jovanny,0.0045222434,124.95.214.23,"{""location"": ""BE"", ""is_mobile"": true}" 2392,2,53,2017-03-19 19:42:05,http://von.com/trace,0.9574226105,124.18.236.164,"{""location"": ""GM"", ""is_mobile"": true}" 2393,2,53,2017-03-12 08:58:00,http://vonkoepp.info/aletha.pagac,0.5147524947,155.100.133.85,"{""location"": ""LK"", ""is_mobile"": true}" 2394,2,53,2017-01-13 00:51:26,http://hegmannorn.co/cielo,0.5546058194,102.185.160.154,"{""location"": ""CI"", ""is_mobile"": false}" 2395,2,53,2017-03-18 09:38:17,http://frami.com/domenico_kuhic,0.2898686436,169.242.229.225,"{""location"": ""HK"", ""is_mobile"": false}" 2396,2,53,2017-01-06 06:10:44,http://hamilllindgren.biz/janae,0.1807431239,49.123.88.218,"{""location"": ""FK"", ""is_mobile"": true}" 2397,2,53,2017-06-01 21:03:21,http://wisozk.com/mozell.marks,0.0922068186,139.20.27.172,"{""location"": ""KM"", ""is_mobile"": true}" 2398,2,53,2017-01-02 02:27:10,http://beer.io/paige,0.4546792421,10.164.35.24,"{""location"": ""CD"", ""is_mobile"": false}" 2399,2,53,2016-12-23 14:20:56,http://deckow.name/dean_thiel,0.2863342202,155.244.137.252,"{""location"": ""SR"", ""is_mobile"": true}" 2400,2,53,2017-01-03 22:52:58,http://schowaltercummings.name/marilie.carter,0.6989122034,170.65.49.23,"{""location"": ""GB"", ""is_mobile"": true}" 2401,2,53,2017-02-24 15:11:45,http://cronin.io/rusty,0.7269303713,141.27.31.37,"{""location"": ""VA"", ""is_mobile"": true}" 2402,2,53,2017-01-18 14:47:00,http://bogisichschuster.org/marianne,0.7551204529,29.211.12.178,"{""location"": ""PW"", ""is_mobile"": true}" 2403,2,53,2017-02-26 19:53:05,http://weber.com/cullen,0.3934441122,66.5.98.60,"{""location"": ""MU"", ""is_mobile"": false}" 2404,2,53,2017-01-23 04:10:55,http://herman.name/gabrielle_ledner,0.8301137780,34.42.155.254,"{""location"": ""NL"", ""is_mobile"": false}" 2405,2,53,2017-03-12 07:14:44,http://schuster.co/julius,0.0751106836,6.55.229.147,"{""location"": ""CV"", ""is_mobile"": false}" 2406,2,53,2017-02-27 21:23:43,http://jast.info/donna,0.9917694026,128.163.186.212,"{""location"": ""AZ"", ""is_mobile"": true}" 2407,2,53,2017-04-07 06:42:30,http://langosh.org/pinkie,0.7839362987,51.128.118.193,"{""location"": ""TJ"", ""is_mobile"": true}" 2408,2,53,2017-04-04 20:47:42,http://predovic.io/nikki_denesik,0.1674089481,200.243.237.22,"{""location"": ""TH"", ""is_mobile"": true}" 2409,2,53,2017-01-26 19:29:05,http://zieme.co/arno.zemlak,0.2630875453,214.120.203.223,"{""location"": ""GT"", ""is_mobile"": false}" 2410,2,53,2017-06-03 18:27:32,http://schulistbotsford.name/thelma.rosenbaum,0.4146018123,30.133.46.15,"{""location"": ""VN"", ""is_mobile"": true}" 2411,2,53,2017-02-07 15:54:41,http://ledner.net/erika,0.9836448789,203.102.45.104,"{""location"": ""HU"", ""is_mobile"": false}" 2412,2,54,2017-05-26 12:08:06,http://friesenleannon.com/lee,0.7718756286,61.163.241.193,"{""location"": ""AD"", ""is_mobile"": false}" 2413,2,54,2017-03-21 02:48:04,http://schaefer.info/pat,0.7506664376,205.251.105.189,"{""location"": ""UZ"", ""is_mobile"": true}" 2414,2,54,2017-04-30 07:54:07,http://mueller.biz/brendan.walker,0.3487401390,207.35.54.121,"{""location"": ""VG"", ""is_mobile"": false}" 2415,2,54,2017-01-24 16:21:38,http://walker.org/adrain_schuster,0.4100866152,151.243.92.203,"{""location"": ""BV"", ""is_mobile"": true}" 2416,2,54,2017-04-03 13:50:24,http://wolfwilliamson.co/janea,0.2243629478,87.119.30.141,"{""location"": ""BN"", ""is_mobile"": true}" 9297,4,208,2017-04-28 22:17:59,http://herzog.name/amiya,0.3617453118,145.3.236.50,"{""location"": ""MR"", ""is_mobile"": true}" 9298,4,208,2017-04-12 19:14:20,http://boehm.com/adrien,0.4922574993,58.17.35.195,"{""location"": ""AW"", ""is_mobile"": true}" 9299,4,208,2016-12-29 01:09:38,http://kreigerpadberg.info/rosanna,0.8732172657,61.36.82.158,"{""location"": ""UG"", ""is_mobile"": true}" 9300,4,208,2016-12-16 14:25:49,http://bergeromaguera.org/ro,0.4023517801,68.40.224.132,"{""location"": ""BZ"", ""is_mobile"": true}" 9301,4,208,2017-03-08 09:50:03,http://pricerobel.biz/elisabeth.rice,0.6523231493,32.58.52.77,"{""location"": ""IE"", ""is_mobile"": false}" 9302,4,208,2017-03-12 09:01:37,http://hilpertlangosh.org/corine,0.0097227823,183.13.183.196,"{""location"": ""HT"", ""is_mobile"": false}" 9303,4,208,2017-03-03 15:17:07,http://beer.com/halle,0.8441910445,229.28.67.249,"{""location"": ""GR"", ""is_mobile"": true}" 9304,4,208,2016-12-14 11:53:25,http://mcclure.org/noemy,0.5593226362,59.40.4.88,"{""location"": ""BO"", ""is_mobile"": true}" 9305,4,208,2017-02-11 01:23:32,http://bruen.name/harold.mitchell,0.3115333649,223.75.34.147,"{""location"": ""MV"", ""is_mobile"": false}" 9306,4,209,2017-01-10 03:19:38,http://kirlin.info/braeden_nikolaus,0.2939818517,94.17.198.95,"{""location"": ""AT"", ""is_mobile"": true}" 9307,4,209,2017-04-23 23:07:06,http://powlowski.net/desmond.wiegand,0.6426840429,204.167.193.56,"{""location"": ""KI"", ""is_mobile"": false}" 9308,4,209,2016-12-17 08:05:53,http://lubowitz.org/claudia,0.4463073494,34.4.226.25,"{""location"": ""MQ"", ""is_mobile"": false}" 9309,4,209,2017-02-11 01:16:41,http://kohlerschneider.co/cecilia,0.1965727258,2.133.54.169,"{""location"": ""KH"", ""is_mobile"": false}" 9310,4,209,2017-01-30 04:22:22,http://bechtelar.com/allie_ohara,0.2089018490,13.214.30.3,"{""location"": ""RU"", ""is_mobile"": true}" 9311,4,209,2017-01-22 04:29:30,http://hilpertbaumbach.io/gunner,0.1994833758,69.173.235.242,"{""location"": ""YE"", ""is_mobile"": false}" 9312,4,209,2017-06-13 05:56:34,http://lueilwitzjacobi.io/ru.goyette,0.5203954536,59.25.127.231,"{""location"": ""KG"", ""is_mobile"": false}" 9313,4,209,2017-06-01 03:38:06,http://rath.info/macey,0.7931764981,253.130.237.228,"{""location"": ""BV"", ""is_mobile"": true}" 9314,4,209,2017-03-30 09:20:18,http://weinat.net/mackenzie,0.5799655389,182.226.108.182,"{""location"": ""AR"", ""is_mobile"": true}" 9315,4,209,2017-05-03 02:45:24,http://kundeswaniawski.net/korbin,0.9251474172,247.140.191.181,"{""location"": ""DO"", ""is_mobile"": true}" 9316,4,209,2017-02-25 19:43:37,http://gislason.org/oral.lockman,0.4853284240,90.195.162.224,"{""location"": ""CY"", ""is_mobile"": true}" 9317,4,209,2017-05-07 00:30:29,http://bechtelar.io/erling_bode,0.9852079326,70.149.226.165,"{""location"": ""KY"", ""is_mobile"": true}" 9318,4,209,2017-04-30 19:23:25,http://gottlieb.com/daphnee,0.5883836057,93.37.178.189,"{""location"": ""NI"", ""is_mobile"": true}" 9319,4,209,2017-03-23 09:59:44,http://jerde.com/conor,0.0716072786,108.178.58.121,"{""location"": ""PK"", ""is_mobile"": false}" 9320,4,209,2017-01-30 18:28:23,http://hamill.co/monroe_waelchi,0.4827818183,83.232.84.97,"{""location"": ""DK"", ""is_mobile"": false}" 9321,4,209,2017-04-08 14:25:56,http://schambergerbeahan.biz/luisa_donnelly,0.1195958839,31.187.218.221,"{""location"": ""NI"", ""is_mobile"": true}" 9322,4,209,2017-02-21 08:22:13,http://schinner.net/maynard.glover,0.1641035738,26.246.158.97,"{""location"": ""KZ"", ""is_mobile"": false}" 9323,4,209,2017-03-10 12:37:51,http://swift.org/donato_halvorson,0.3624143793,200.134.46.97,"{""location"": ""TZ"", ""is_mobile"": true}" 9324,4,209,2017-05-21 10:19:30,http://sauer.io/ephraim,0.9675188699,235.8.205.107,"{""location"": ""FR"", ""is_mobile"": true}" 9325,4,209,2016-12-28 08:21:53,http://rohan.net/dulce.brown,0.3339167234,180.157.101.148,"{""location"": ""CR"", ""is_mobile"": false}" 9326,4,209,2016-12-24 00:36:00,http://schinner.com/simone,0.0418594864,20.194.69.79,"{""location"": ""LR"", ""is_mobile"": true}" 9327,4,209,2017-02-16 21:14:36,http://lindgrenbecker.info/broderick_hintz,0.9286739805,253.177.28.189,"{""location"": ""CO"", ""is_mobile"": false}" 9328,4,209,2017-02-23 23:18:47,http://kautzerrobel.io/althea,0.7082975171,223.56.89.18,"{""location"": ""IN"", ""is_mobile"": false}" 9329,4,209,2017-05-12 10:17:36,http://torphy.com/kade_nikolaus,0.6471889543,7.9.99.173,"{""location"": ""TG"", ""is_mobile"": true}" 9330,4,210,2016-12-24 06:32:20,http://auerkuhn.co/wilfrid_schaefer,0.6992155072,147.35.43.46,"{""location"": ""EG"", ""is_mobile"": false}" 9331,4,210,2017-02-20 17:23:09,http://koelpin.co/brenda_padberg,0.6829044983,25.232.254.65,"{""location"": ""PS"", ""is_mobile"": false}" 9332,4,210,2017-04-29 08:12:28,http://boyer.info/velda,0.3222011743,5.233.27.3,"{""location"": ""NA"", ""is_mobile"": true}" 9333,4,210,2017-03-27 13:46:45,http://bergstrom.info/tom.jacobs,0.4206463208,52.252.7.158,"{""location"": ""GW"", ""is_mobile"": false}" 9334,4,210,2017-05-08 06:01:11,http://zemlak.org/toni.weinat,0.4906737024,21.134.224.149,"{""location"": ""VC"", ""is_mobile"": false}" 9335,4,210,2017-03-24 08:57:40,http://simonispurdy.io/vickie,0.5801079878,63.61.178.8,"{""location"": ""KI"", ""is_mobile"": true}" 9336,4,210,2017-01-10 05:19:47,http://thompsonvandervort.biz/haylie.schultz,0.0322686404,228.40.141.183,"{""location"": ""IR"", ""is_mobile"": true}" 9337,4,210,2017-05-16 08:44:53,http://batz.co/stanford.keler,0.1113232106,83.142.12.254,"{""location"": ""BS"", ""is_mobile"": true}" 9338,4,210,2017-05-19 19:33:05,http://johnston.co/esmeralda,0.8470147087,84.8.145.78,"{""location"": ""MW"", ""is_mobile"": true}" 9339,4,210,2017-03-10 08:58:56,http://watersfriesen.com/alia.heathcote,0.1324550507,190.174.3.158,"{""location"": ""MT"", ""is_mobile"": true}" 9340,4,210,2017-02-14 01:37:17,http://collins.org/gust,0.1706196846,222.193.41.58,"{""location"": ""LK"", ""is_mobile"": true}" 9341,4,210,2017-05-03 01:10:20,http://murray.name/logan_bayer,0.4009189854,145.122.75.58,"{""location"": ""HN"", ""is_mobile"": true}" 9342,4,210,2017-01-31 14:54:30,http://dickens.info/raina.kunde,0.4930651234,204.152.212.28,"{""location"": ""DE"", ""is_mobile"": true}" 9343,4,210,2016-12-15 12:41:39,http://ledner.com/zakary.schumm,0.6423810855,2.81.45.238,"{""location"": ""LV"", ""is_mobile"": false}" 9344,4,210,2017-01-27 19:46:06,http://weinat.com/giles_greenfelder,0.0084849894,192.178.175.80,"{""location"": ""NF"", ""is_mobile"": true}" 9345,4,210,2017-05-13 22:53:26,http://goodwin.net/javon_bode,0.0155760676,246.88.79.129,"{""location"": ""JP"", ""is_mobile"": false}" 9346,4,210,2017-03-10 15:20:22,http://reichel.org/lilyan,0.2880157081,202.127.99.170,"{""location"": ""CU"", ""is_mobile"": false}" 9347,4,210,2017-05-20 14:30:03,http://gleichner.info/noble,0.4924746802,205.247.13.144,"{""location"": ""EG"", ""is_mobile"": false}" 9348,4,210,2017-01-25 05:14:24,http://casperveum.net/tyrique.olson,0.9891424539,194.125.76.134,"{""location"": ""LS"", ""is_mobile"": false}" 6307,3,138,2017-05-27 11:45:26,http://hills.name/elroy.marvin,0.5359448729,167.73.142.11,"{""location"": ""NE"", ""is_mobile"": false}" 6308,3,138,2017-05-22 11:14:05,http://mayer.io/kayley,0.1844877744,224.64.46.225,"{""location"": ""MF"", ""is_mobile"": false}" 6309,3,138,2017-03-11 11:16:21,http://schowalterbeatty.io/jakob.dickens,0.3248997970,27.64.240.246,"{""location"": ""DM"", ""is_mobile"": false}" 6310,3,138,2017-05-16 21:16:15,http://stoltenberg.name/devante_vandervort,0.4393021806,211.194.88.75,"{""location"": ""IM"", ""is_mobile"": false}" 6311,3,138,2017-02-07 08:26:41,http://colegreenholt.com/robin,0.5496566502,101.56.227.209,"{""location"": ""DE"", ""is_mobile"": true}" 6312,3,138,2017-06-13 06:58:47,http://rowe.co/alexandre,0.0341347219,147.158.103.22,"{""location"": ""SG"", ""is_mobile"": true}" 6313,3,138,2017-05-16 08:15:21,http://marvin.name/john,0.8582179729,237.171.32.116,"{""location"": ""SZ"", ""is_mobile"": false}" 6314,3,138,2016-12-18 05:14:21,http://mannbeatty.net/elsie,0.6494612987,66.24.223.143,"{""location"": ""IT"", ""is_mobile"": false}" 6315,3,138,2017-02-22 03:13:57,http://langledner.info/delilah,0.9809692158,73.15.155.179,"{""location"": ""EC"", ""is_mobile"": false}" 6316,3,138,2017-05-09 18:32:51,http://schulist.info/ephraim,0.9823130473,151.37.178.222,"{""location"": ""BJ"", ""is_mobile"": false}" 6317,3,138,2017-04-06 23:51:58,http://marquardt.info/omari.haley,0.5644039664,218.93.158.104,"{""location"": ""JM"", ""is_mobile"": true}" 6318,3,138,2017-02-13 23:02:28,http://nolan.co/darryl,0.1583780770,3.252.78.199,"{""location"": ""NG"", ""is_mobile"": false}" 6319,3,138,2016-12-15 17:05:23,http://muller.name/deie.kovacek,0.6887579739,190.248.162.127,"{""location"": ""CY"", ""is_mobile"": true}" 6320,3,138,2017-04-06 04:17:04,http://terry.co/keaton,0.7026601684,3.62.27.129,"{""location"": ""GE"", ""is_mobile"": false}" 6321,3,138,2017-06-11 10:29:29,http://ondricka.info/verdie,0.7295113699,44.236.11.221,"{""location"": ""OM"", ""is_mobile"": true}" 6322,3,138,2017-03-28 04:59:30,http://labadie.biz/shawn,0.0193518133,224.25.128.155,"{""location"": ""CI"", ""is_mobile"": true}" 6323,3,138,2017-03-13 19:39:44,http://bernhard.info/melia.abbott,0.2836191943,18.50.9.106,"{""location"": ""RE"", ""is_mobile"": true}" 6324,3,138,2017-01-16 03:40:11,http://howewalsh.info/nikolas.gaylord,0.5835706050,26.81.188.149,"{""location"": ""NA"", ""is_mobile"": true}" 6325,3,138,2017-04-09 11:09:03,http://ryan.io/enrique,0.8467580809,40.71.216.180,"{""location"": ""CU"", ""is_mobile"": true}" 6326,3,138,2016-12-14 22:38:53,http://jakubowski.org/ines_greenholt,0.6760293597,182.31.192.224,"{""location"": ""BQ"", ""is_mobile"": true}" 6327,3,138,2016-12-13 06:10:06,http://beatty.net/rachael_harris,0.1114272091,130.145.167.132,"{""location"": ""NE"", ""is_mobile"": true}" 6328,3,138,2017-02-23 22:01:42,http://grant.io/marisa_stokes,0.7931276438,23.130.47.165,"{""location"": ""EG"", ""is_mobile"": true}" 6329,3,138,2017-03-29 10:05:36,http://hackett.name/gilbert,0.0164761156,35.108.225.78,"{""location"": ""ST"", ""is_mobile"": true}" 6330,3,138,2017-01-25 03:36:41,http://reilly.info/bridgette,0.4730522526,206.43.189.16,"{""location"": ""AX"", ""is_mobile"": false}" 6331,3,138,2016-12-21 21:24:06,http://flatley.name/brandi_gislason,0.4247980046,63.95.75.66,"{""location"": ""BO"", ""is_mobile"": false}" 6332,3,138,2017-04-29 22:02:36,http://strackerunolfon.biz/nels.gerhold,0.0135598763,25.153.162.172,"{""location"": ""VA"", ""is_mobile"": true}" 6333,3,138,2017-01-11 02:57:50,http://warddenesik.io/harmon,0.0751959541,173.233.106.253,"{""location"": ""CC"", ""is_mobile"": false}" 6334,3,138,2017-04-07 22:04:20,http://parker.name/monica.lindgren,0.4560840691,26.147.241.94,"{""location"": ""NP"", ""is_mobile"": false}" 6335,3,138,2017-05-07 14:24:29,http://thiel.co/dianna,0.9697225762,236.144.35.23,"{""location"": ""IL"", ""is_mobile"": false}" 6336,3,138,2017-05-25 15:49:05,http://starkglover.net/sandra,0.2329547709,24.137.80.224,"{""location"": ""SE"", ""is_mobile"": false}" 6337,3,138,2017-03-14 20:23:29,http://ziemannblanda.com/sedrick.dickens,0.2927034680,214.180.42.251,"{""location"": ""MO"", ""is_mobile"": true}" 6338,3,138,2017-01-02 20:31:34,http://reilly.co/ana,0.8134123507,56.83.29.48,"{""location"": ""GF"", ""is_mobile"": true}" 6339,3,138,2017-03-19 07:17:14,http://okeefestanton.net/lora.schoen,0.7080689063,6.246.90.42,"{""location"": ""SX"", ""is_mobile"": true}" 6340,3,138,2017-02-16 23:34:55,http://boyle.info/shaina,0.9300706167,246.82.168.85,"{""location"": ""TN"", ""is_mobile"": false}" 6341,3,138,2017-01-29 15:11:01,http://abshirelubowitz.net/aliza,0.7728521100,196.138.204.131,"{""location"": ""CH"", ""is_mobile"": false}" 6342,3,138,2017-05-09 18:53:51,http://schimmelframi.org/shakira,0.6390740627,110.72.46.100,"{""location"": ""HT"", ""is_mobile"": true}" 6343,3,138,2017-02-26 09:04:08,http://douglas.co/nona_fisher,0.4234561343,98.200.165.225,"{""location"": ""KY"", ""is_mobile"": true}" 6344,3,138,2017-04-22 02:00:24,http://swaniawskicain.name/charley,0.1832450485,6.129.125.182,"{""location"": ""TK"", ""is_mobile"": true}" 6345,3,138,2017-05-20 03:16:16,http://effertz.com/celia_zboncak,0.9990616036,147.214.112.102,"{""location"": ""MF"", ""is_mobile"": true}" 6346,3,138,2017-01-03 04:50:25,http://flatley.co/trey.reinger,0.0878801283,146.130.106.149,"{""location"": ""PT"", ""is_mobile"": false}" 6347,3,138,2016-12-14 23:15:32,http://tremblay.info/davon,0.3495742930,189.153.123.118,"{""location"": ""PT"", ""is_mobile"": true}" 6348,3,138,2017-03-23 18:12:03,http://abbott.co/vergie.oconnell,0.3958848621,145.137.48.220,"{""location"": ""KZ"", ""is_mobile"": false}" 6349,3,138,2017-02-01 17:28:34,http://weimannmurazik.com/damien,0.8566588608,92.221.21.97,"{""location"": ""NG"", ""is_mobile"": false}" 6350,3,138,2017-05-23 00:20:25,http://adams.co/roma_pagac,0.5051341258,250.147.200.102,"{""location"": ""SL"", ""is_mobile"": false}" 6351,3,138,2017-05-08 17:16:55,http://bernier.io/peggie,0.3125895216,47.171.129.83,"{""location"": ""PN"", ""is_mobile"": true}" 6352,3,138,2017-04-08 20:54:06,http://turcotte.io/mittie,0.8960375233,187.237.216.121,"{""location"": ""LR"", ""is_mobile"": true}" 6353,3,138,2017-05-18 05:15:38,http://crist.com/myriam,0.3233817283,244.118.23.10,"{""location"": ""BW"", ""is_mobile"": false}" 6354,3,138,2017-01-22 06:03:43,http://robel.biz/tobin.rodriguez,0.2764347278,253.210.93.56,"{""location"": ""BR"", ""is_mobile"": false}" 6355,3,138,2017-04-26 12:27:18,http://torphy.co/solon,0.6585900637,185.179.5.62,"{""location"": ""NF"", ""is_mobile"": true}" 6356,3,138,2017-01-08 10:12:15,http://padberg.net/billie.bayer,0.9378437829,60.209.174.84,"{""location"": ""EE"", ""is_mobile"": false}" 6357,3,138,2017-04-08 22:11:08,http://macejkovic.net/aleen,0.2020010888,30.80.17.206,"{""location"": ""UG"", ""is_mobile"": false}" 6358,3,139,2017-03-01 07:42:35,http://emard.net/beulah,0.3430519153,181.104.56.250,"{""location"": ""LI"", ""is_mobile"": false}" 12213,5,273,2017-05-19 09:45:16,http://murazikernser.name/wendell,0.0667729723,32.146.231.144,"{""location"": ""MK"", ""is_mobile"": false}" 12214,5,273,2016-12-17 19:01:33,http://streichcole.name/kamron,0.4978375530,253.160.71.45,"{""location"": ""DE"", ""is_mobile"": false}" 12215,5,273,2017-03-30 05:01:04,http://zemlak.net/rebecca_borer,0.2682215873,102.144.41.244,"{""location"": ""CR"", ""is_mobile"": false}" 12216,5,273,2017-01-25 04:15:58,http://hoppe.com/adalberto.sauer,0.5538525469,163.129.218.37,"{""location"": ""MV"", ""is_mobile"": false}" 12217,5,273,2017-04-27 10:47:01,http://marvin.info/florian,0.8302553606,218.82.232.155,"{""location"": ""AT"", ""is_mobile"": true}" 12218,5,273,2017-02-05 16:02:49,http://jacobimonis.org/jovani,0.4353532766,173.50.82.173,"{""location"": ""CR"", ""is_mobile"": true}" 12219,5,273,2017-01-06 18:51:43,http://kilbackkoch.co/janiya,0.0939441661,51.221.201.188,"{""location"": ""RU"", ""is_mobile"": false}" 12220,5,273,2017-03-22 09:38:05,http://rutherfordledner.io/kay,0.6416748851,207.54.3.169,"{""location"": ""BT"", ""is_mobile"": false}" 12221,5,273,2017-01-10 09:58:33,http://mertz.co/bradford.schneider,0.7443481067,234.211.126.100,"{""location"": ""IN"", ""is_mobile"": false}" 12222,5,273,2017-01-01 04:55:09,http://schillermitchell.biz/leora_altenwerth,0.6451414231,137.186.117.235,"{""location"": ""GD"", ""is_mobile"": false}" 12223,5,273,2017-02-25 18:54:13,http://weinat.biz/manuela,0.9408462506,94.241.248.233,"{""location"": ""CH"", ""is_mobile"": true}" 12224,5,273,2017-03-29 21:23:29,http://carter.name/robb.gusikowski,0.6550517656,202.137.232.251,"{""location"": ""SK"", ""is_mobile"": false}" 12225,5,273,2017-02-05 03:48:02,http://kutchcummings.biz/beulah,0.1212898404,13.23.119.23,"{""location"": ""TO"", ""is_mobile"": false}" 12226,5,273,2016-12-28 15:11:46,http://nader.biz/guadalupe_wuckert,0.4177440442,8.169.131.82,"{""location"": ""AZ"", ""is_mobile"": false}" 12227,5,273,2017-06-11 23:23:27,http://maggiostoltenberg.net/verna,0.0554142642,101.95.137.101,"{""location"": ""AT"", ""is_mobile"": false}" 12228,5,273,2017-03-19 06:18:12,http://ebert.com/ollie.hayes,0.7879363452,47.162.122.132,"{""location"": ""MU"", ""is_mobile"": true}" 12229,5,273,2017-03-11 03:08:39,http://corkery.org/bernadette,0.6405496132,222.243.219.127,"{""location"": ""VU"", ""is_mobile"": false}" 12230,5,273,2017-04-28 14:56:44,http://okuneva.biz/josiah_ruel,0.5900461251,130.145.175.184,"{""location"": ""ZW"", ""is_mobile"": true}" 12231,5,273,2016-12-20 12:44:41,http://ryan.io/beau_little,0.7445352551,38.19.69.145,"{""location"": ""YT"", ""is_mobile"": true}" 12232,5,273,2017-02-28 15:15:12,http://bernhardkub.name/aimee,0.1403402443,66.240.50.168,"{""location"": ""TJ"", ""is_mobile"": false}" 12233,5,273,2016-12-31 06:27:28,http://larkinkunze.co/amber,0.9130957596,170.71.141.120,"{""location"": ""GE"", ""is_mobile"": true}" 12234,5,273,2017-04-18 02:43:57,http://jacobs.org/cordie.schimmel,0.5193229192,23.95.77.175,"{""location"": ""LK"", ""is_mobile"": true}" 12235,5,273,2017-01-23 06:28:20,http://durgan.name/erica.hickle,0.7790974268,128.49.219.148,"{""location"": ""NF"", ""is_mobile"": false}" 12236,5,273,2017-03-29 16:15:52,http://roweheidenreich.biz/michele,0.1157401907,50.43.79.16,"{""location"": ""ST"", ""is_mobile"": false}" 12237,5,273,2016-12-16 06:42:06,http://dietrich.biz/van,0.6211244260,69.168.161.114,"{""location"": ""TD"", ""is_mobile"": true}" 12238,5,273,2017-04-02 14:13:23,http://goodwin.name/kenny_goyette,0.5072768580,191.198.137.226,"{""location"": ""AF"", ""is_mobile"": true}" 12239,5,273,2017-01-06 02:15:02,http://lynch.co/taryn.becker,0.1229846940,185.169.197.236,"{""location"": ""BI"", ""is_mobile"": true}" 12240,5,273,2017-03-11 17:30:22,http://luettgenkuhn.com/arne.zulauf,0.7777745309,40.153.108.93,"{""location"": ""EC"", ""is_mobile"": true}" 12241,5,274,2016-12-29 07:22:05,http://lowe.biz/coty_bins,0.9970221510,235.139.211.201,"{""location"": ""HN"", ""is_mobile"": false}" 12242,5,274,2017-06-14 01:19:29,http://pollich.co/colten,0.8202675679,50.65.24.236,"{""location"": ""VI"", ""is_mobile"": true}" 12243,5,274,2016-12-23 23:10:35,http://hirthefadel.com/iva_corkery,0.9102462320,72.135.55.12,"{""location"": ""MA"", ""is_mobile"": false}" 12244,5,274,2016-12-15 16:55:57,http://goyettehudson.net/marcel,0.3482114405,212.247.162.115,"{""location"": ""AM"", ""is_mobile"": true}" 12245,5,274,2017-01-10 02:21:07,http://mayerhodkiewicz.name/bertrand.baumbach,0.3057380597,222.21.10.40,"{""location"": ""PE"", ""is_mobile"": false}" 12246,5,274,2017-03-14 00:31:12,http://keebler.net/jeramie.runolfon,0.5478349799,235.5.229.191,"{""location"": ""CD"", ""is_mobile"": true}" 12247,5,274,2017-06-10 16:18:20,http://kertzmann.com/velda,0.7072329976,34.201.195.185,"{""location"": ""MF"", ""is_mobile"": false}" 12248,5,274,2017-03-04 09:01:32,http://fahey.co/lea,0.0219728267,171.163.199.18,"{""location"": ""PH"", ""is_mobile"": false}" 12249,5,274,2017-03-01 07:52:39,http://goodwin.biz/kobe,0.5877336945,137.232.143.73,"{""location"": ""BH"", ""is_mobile"": false}" 12250,5,274,2017-03-19 23:22:10,http://williamson.org/maeve.heathcote,0.6796373995,95.56.222.222,"{""location"": ""MQ"", ""is_mobile"": false}" 12251,5,274,2017-06-10 11:10:57,http://kuhlman.co/pinkie.homenick,0.5762023369,162.207.132.125,"{""location"": ""SV"", ""is_mobile"": false}" 12252,5,274,2017-04-16 01:25:36,http://hintz.biz/harrison_hintz,0.5260598576,22.101.187.92,"{""location"": ""ER"", ""is_mobile"": true}" 12253,5,274,2017-01-02 00:49:10,http://wilderman.name/joaquin_reichert,0.8620059332,28.26.135.23,"{""location"": ""MO"", ""is_mobile"": false}" 12254,5,274,2017-01-21 01:30:08,http://murazik.info/valentina_koepp,0.3642075661,190.23.65.136,"{""location"": ""FO"", ""is_mobile"": true}" 12255,5,274,2017-01-30 08:40:40,http://spinkaosinski.org/frida,0.0344494613,159.251.72.204,"{""location"": ""GD"", ""is_mobile"": false}" 12256,5,274,2017-05-17 09:10:48,http://schillerbrakus.io/ruth,0.9663932457,26.110.97.213,"{""location"": ""IQ"", ""is_mobile"": false}" 12257,5,274,2017-04-14 17:44:11,http://cummeratabins.io/jewel,0.4274322182,219.66.231.107,"{""location"": ""CC"", ""is_mobile"": true}" 12258,5,274,2016-12-25 11:28:29,http://schustertorphy.io/ashton_labadie,0.0143100111,145.168.199.52,"{""location"": ""ZA"", ""is_mobile"": false}" 12259,5,274,2017-02-20 14:54:14,http://murphy.name/melvin,0.1290073591,249.161.9.23,"{""location"": ""VA"", ""is_mobile"": true}" 12260,5,274,2017-06-12 04:30:20,http://framischmitt.info/autumn,0.2499451737,22.241.214.179,"{""location"": ""CV"", ""is_mobile"": false}" 12261,5,274,2017-03-16 07:25:41,http://moen.name/floy,0.4307568928,122.64.118.125,"{""location"": ""IR"", ""is_mobile"": false}" 12262,5,274,2017-04-13 22:55:45,http://corkery.name/octavia,0.1715488252,105.56.178.152,"{""location"": ""SS"", ""is_mobile"": true}" 12263,5,275,2017-04-11 09:55:50,http://westcummings.com/evie_barton,0.7185387225,193.129.105.68,"{""location"": ""PY"", ""is_mobile"": false}" 15218,6,342,2017-04-18 22:17:21,http://langworth.info/hoyt.reilly,,6.212.158.214,"{""location"": ""SE"", ""is_mobile"": true}" 15219,6,342,2017-05-14 02:33:17,http://wilderman.org/ted,,92.26.166.132,"{""location"": ""GN"", ""is_mobile"": true}" 15220,6,342,2017-04-28 00:09:59,http://conroy.co/zula,,85.168.18.161,"{""location"": ""PE"", ""is_mobile"": false}" 15221,6,342,2017-01-15 19:07:25,http://hand.info/valentina,,157.116.232.52,"{""location"": ""PK"", ""is_mobile"": true}" 15222,6,342,2017-01-23 14:46:45,http://vandervort.name/leta.harber,,51.27.231.86,"{""location"": ""NU"", ""is_mobile"": false}" 15223,6,342,2017-04-22 00:15:15,http://oberbrunner.org/titus.morar,,106.165.159.41,"{""location"": ""SS"", ""is_mobile"": false}" 15224,6,342,2017-02-13 02:41:00,http://kuhnthiel.name/madyson.grant,,17.9.54.202,"{""location"": ""MW"", ""is_mobile"": true}" 15225,6,342,2017-03-02 16:03:28,http://schoennolan.io/emmanuelle,,68.158.111.161,"{""location"": ""MG"", ""is_mobile"": true}" 15226,6,342,2017-01-06 16:55:43,http://breitenberglakin.name/ara_wilderman,,177.237.22.74,"{""location"": ""VA"", ""is_mobile"": true}" 15227,6,342,2017-01-22 07:42:52,http://stroman.name/keenan,,84.248.76.7,"{""location"": ""CM"", ""is_mobile"": true}" 15228,6,342,2017-04-29 09:42:02,http://mertzvon.info/angie,,218.124.208.67,"{""location"": ""HK"", ""is_mobile"": false}" 15229,6,342,2017-04-01 15:04:01,http://hoppe.co/bernadine_emmerich,,11.140.84.138,"{""location"": ""KW"", ""is_mobile"": false}" 15230,6,342,2017-02-22 11:33:17,http://franeckicrist.io/johann.kilback,,71.15.233.186,"{""location"": ""BQ"", ""is_mobile"": false}" 15231,6,342,2017-02-19 13:02:58,http://westschmidt.name/rebeka_beer,,137.149.210.28,"{""location"": ""GF"", ""is_mobile"": true}" 15232,6,342,2017-05-17 06:14:37,http://lyncholson.co/isaiah_lubowitz,,77.45.118.89,"{""location"": ""GS"", ""is_mobile"": true}" 15233,6,342,2017-01-14 13:23:36,http://wittingwindler.org/gerard,,185.254.74.44,"{""location"": ""AS"", ""is_mobile"": false}" 15234,6,342,2017-02-17 09:49:33,http://rice.io/donny_okeefe,,86.129.188.27,"{""location"": ""ML"", ""is_mobile"": false}" 15235,6,342,2017-01-08 21:26:16,http://pouros.info/randy.morar,,229.33.151.71,"{""location"": ""HM"", ""is_mobile"": false}" 15236,6,342,2017-03-19 12:58:55,http://schillerkertzmann.net/raphael,,212.104.153.167,"{""location"": ""HN"", ""is_mobile"": true}" 15237,6,342,2017-04-19 08:01:04,http://jenkins.co/pedro_friesen,,220.142.130.138,"{""location"": ""AW"", ""is_mobile"": false}" 15238,6,342,2017-03-11 14:40:41,http://lynchrowe.org/jeffery_strosin,,114.138.38.112,"{""location"": ""ML"", ""is_mobile"": false}" 15239,6,342,2017-01-29 02:39:22,http://morar.name/pascale,,11.33.45.198,"{""location"": ""UY"", ""is_mobile"": true}" 15240,6,342,2017-01-20 21:23:45,http://osinskibogisich.co/mabel_breitenberg,,188.219.2.23,"{""location"": ""TM"", ""is_mobile"": true}" 15241,6,342,2017-03-29 21:35:59,http://mcclure.net/dominic_johns,,178.87.232.164,"{""location"": ""GN"", ""is_mobile"": true}" 15242,6,342,2017-03-30 08:37:11,http://brown.net/naomi,,176.235.123.59,"{""location"": ""LR"", ""is_mobile"": true}" 15243,6,342,2017-01-18 06:20:20,http://corwin.io/wilton,,123.93.88.222,"{""location"": ""SC"", ""is_mobile"": false}" 15244,6,342,2017-04-08 22:55:32,http://schmidt.biz/dante,,85.211.141.58,"{""location"": ""SY"", ""is_mobile"": false}" 15245,6,342,2016-12-17 18:37:19,http://mills.net/esta.jacobson,,101.205.244.168,"{""location"": ""PN"", ""is_mobile"": true}" 15246,6,342,2017-04-24 15:36:51,http://parker.net/jackie,,235.115.28.76,"{""location"": ""UA"", ""is_mobile"": false}" 15247,6,342,2017-02-16 05:17:09,http://vandervortmedhurst.net/nicolas_dickens,,224.40.96.34,"{""location"": ""GG"", ""is_mobile"": true}" 15248,6,342,2017-05-11 04:53:58,http://schowalter.biz/khalil,,214.73.183.60,"{""location"": ""RO"", ""is_mobile"": false}" 15249,6,342,2017-05-09 03:28:48,http://mckenzie.net/jevon.daniel,,247.225.30.182,"{""location"": ""MW"", ""is_mobile"": true}" 15250,6,342,2017-03-07 06:22:22,http://boehmvandervort.name/jailyn,,75.21.36.112,"{""location"": ""SZ"", ""is_mobile"": false}" 15251,6,342,2017-03-06 13:35:00,http://stark.info/caterina,,155.217.36.202,"{""location"": ""BQ"", ""is_mobile"": false}" 15252,6,342,2017-05-30 03:47:51,http://smithcain.io/kameron,,162.152.116.58,"{""location"": ""BY"", ""is_mobile"": false}" 15253,6,342,2017-04-21 09:32:16,http://bahringer.name/will,,77.233.188.225,"{""location"": ""CM"", ""is_mobile"": true}" 15254,6,343,2016-12-28 20:18:52,http://rogahn.net/leo,,114.231.252.231,"{""location"": ""ZW"", ""is_mobile"": true}" 15255,6,343,2017-05-04 21:09:25,http://legros.com/skyla_romaguera,,231.95.133.117,"{""location"": ""BQ"", ""is_mobile"": true}" 15256,6,343,2016-12-27 04:58:59,http://bahringer.com/jonathon_pouros,,46.32.128.121,"{""location"": ""GF"", ""is_mobile"": false}" 15257,6,343,2017-03-08 08:45:11,http://goodwin.net/ben.kunze,,6.4.207.46,"{""location"": ""GE"", ""is_mobile"": false}" 15258,6,343,2017-01-18 08:28:45,http://greenholtgerlach.io/savanna,,16.118.136.145,"{""location"": ""AW"", ""is_mobile"": false}" 15259,6,343,2017-03-02 08:48:26,http://kleingerhold.name/aurelia.barton,,193.100.164.16,"{""location"": ""MS"", ""is_mobile"": false}" 15260,6,343,2016-12-15 06:10:08,http://hoegerfahey.name/ryleigh_dickinson,,213.8.96.150,"{""location"": ""DK"", ""is_mobile"": true}" 15261,6,343,2017-02-26 03:02:53,http://parisianrau.org/rosalind.baumbach,,246.132.133.210,"{""location"": ""GA"", ""is_mobile"": true}" 15262,6,343,2017-02-06 01:44:16,http://kulas.org/santiago.schamberger,,135.228.135.202,"{""location"": ""CK"", ""is_mobile"": true}" 15263,6,343,2017-04-05 14:33:46,http://emmerich.io/collin,,13.24.63.165,"{""location"": ""RU"", ""is_mobile"": true}" 15264,6,343,2017-06-14 00:48:15,http://hyatt.info/verner.rosenbaum,,50.126.41.36,"{""location"": ""WS"", ""is_mobile"": false}" 15265,6,343,2017-01-09 21:56:30,http://medhurst.biz/alene_rau,,43.238.52.108,"{""location"": ""TC"", ""is_mobile"": true}" 15266,6,343,2016-12-19 15:58:50,http://douglachuppe.name/luther,,239.195.143.62,"{""location"": ""BD"", ""is_mobile"": true}" 15267,6,343,2016-12-30 12:25:13,http://larson.com/theron_lehner,,58.167.29.138,"{""location"": ""TD"", ""is_mobile"": false}" 15268,6,343,2017-05-30 12:40:34,http://baumbach.info/genesis_farrell,,99.78.189.126,"{""location"": ""FI"", ""is_mobile"": false}" 15269,6,343,2017-05-25 07:20:44,http://feeney.co/annamarie,,234.59.90.103,"{""location"": ""CF"", ""is_mobile"": false}" 15270,6,343,2017-05-22 10:35:59,http://treuteljaskolski.co/keith,,132.54.169.115,"{""location"": ""SZ"", ""is_mobile"": false}" 15271,6,343,2017-02-24 10:50:29,http://watsicastroman.net/marty,,254.246.142.182,"{""location"": ""GU"", ""is_mobile"": true}" 15272,6,343,2017-04-04 16:45:35,http://bahringer.biz/amara.mohr,,151.81.251.17,"{""location"": ""YE"", ""is_mobile"": true}" 2417,2,54,2017-04-18 09:00:09,http://gorczanyoconnell.co/noah_bechtelar,0.4291460190,9.35.68.134,"{""location"": ""PG"", ""is_mobile"": true}" 2418,2,54,2017-01-20 20:22:58,http://wiegand.biz/graciela.block,0.3967234432,14.179.65.191,"{""location"": ""BZ"", ""is_mobile"": false}" 2419,2,54,2017-01-28 08:43:28,http://wunsch.com/julio_runolfsdottir,0.3607383094,2.215.162.124,"{""location"": ""SM"", ""is_mobile"": false}" 2420,2,54,2017-01-11 23:25:13,http://blick.biz/vicente_durgan,0.4973748376,239.92.19.204,"{""location"": ""TR"", ""is_mobile"": true}" 2421,2,54,2017-05-22 09:34:15,http://hagenechuppe.name/cierra,0.1136271324,121.67.38.77,"{""location"": ""KN"", ""is_mobile"": true}" 2422,2,54,2017-01-22 09:14:28,http://satterfield.com/reinhold,0.2485791683,163.203.181.245,"{""location"": ""TK"", ""is_mobile"": true}" 2423,2,54,2017-01-20 03:39:19,http://abbottkoepp.name/rebeca_ohara,0.6832795418,187.117.100.140,"{""location"": ""LT"", ""is_mobile"": false}" 2424,2,54,2017-02-20 04:33:42,http://moenmclaughlin.biz/beth,0.5520260967,215.229.164.166,"{""location"": ""HM"", ""is_mobile"": false}" 2425,2,54,2017-04-24 12:42:55,http://kertzmannconn.org/isabella_heel,0.8859386931,149.34.122.94,"{""location"": ""GF"", ""is_mobile"": true}" 2426,2,54,2017-06-04 22:14:01,http://ruel.biz/christopher_yost,0.4832150396,155.193.72.123,"{""location"": ""FJ"", ""is_mobile"": true}" 2427,2,54,2017-02-02 11:50:37,http://swift.net/natasha_moriette,0.2342283745,55.149.77.197,"{""location"": ""BN"", ""is_mobile"": false}" 2428,2,54,2017-01-20 05:24:09,http://robel.io/andreanne,0.3969909273,126.161.4.228,"{""location"": ""MW"", ""is_mobile"": true}" 2429,2,54,2017-04-08 16:38:54,http://shanahan.com/mathew,0.9098223635,233.179.133.134,"{""location"": ""AQ"", ""is_mobile"": true}" 2430,2,54,2017-04-19 08:08:34,http://kaulkeconnelly.name/rey,0.6410266447,245.194.234.143,"{""location"": ""SN"", ""is_mobile"": true}" 2431,2,54,2017-02-05 09:04:57,http://oberbrunner.com/felicity.parisian,0.8120353815,155.57.185.212,"{""location"": ""GY"", ""is_mobile"": true}" 2432,2,54,2017-06-02 23:01:00,http://bayer.net/jerad.lind,0.6635199745,111.144.2.124,"{""location"": ""NR"", ""is_mobile"": false}" 2433,2,54,2017-01-22 15:09:26,http://bruen.com/ted_macejkovic,0.0360239262,226.10.181.228,"{""location"": ""FO"", ""is_mobile"": true}" 2434,2,54,2017-04-01 11:29:30,http://wilkinson.info/ally.conn,0.1461150533,16.201.211.205,"{""location"": ""JE"", ""is_mobile"": false}" 2435,2,54,2017-03-11 21:30:42,http://gulgowski.info/raul_sporer,0.9735253309,231.154.31.89,"{""location"": ""MZ"", ""is_mobile"": true}" 2436,2,54,2016-12-15 17:55:42,http://lednerkunze.net/ardella_hagenes,0.5632065474,10.46.149.223,"{""location"": ""RU"", ""is_mobile"": true}" 2437,2,54,2017-02-25 16:17:48,http://baumbachkirlin.name/danny,0.3288653827,81.212.43.93,"{""location"": ""UA"", ""is_mobile"": false}" 2438,2,54,2017-03-23 15:58:05,http://halvorson.name/johann_jacobs,0.1610952090,122.39.244.212,"{""location"": ""GM"", ""is_mobile"": true}" 2439,2,54,2017-05-04 20:52:26,http://kochfriesen.name/efren,0.3005845556,97.161.44.52,"{""location"": ""TM"", ""is_mobile"": false}" 2440,2,54,2017-01-27 23:58:10,http://greenholtlesch.net/karley,0.7330898391,237.234.72.222,"{""location"": ""MY"", ""is_mobile"": true}" 2441,2,54,2017-04-30 16:01:59,http://hilpert.info/cindy.abbott,0.6901110667,176.165.162.38,"{""location"": ""KP"", ""is_mobile"": false}" 2442,2,54,2017-02-27 08:45:27,http://upton.info/kim_lindgren,0.0489630406,227.210.70.4,"{""location"": ""JO"", ""is_mobile"": true}" 2443,2,54,2017-05-08 06:17:09,http://braun.info/scarlett.sporer,0.5954730654,182.64.229.61,"{""location"": ""IR"", ""is_mobile"": true}" 2444,2,54,2017-02-27 10:32:53,http://kub.info/paige.gislason,0.3080395874,226.141.241.3,"{""location"": ""ZA"", ""is_mobile"": false}" 2445,2,54,2017-03-16 17:52:02,http://bernhard.com/liza.nader,0.6442912473,8.130.239.219,"{""location"": ""GD"", ""is_mobile"": false}" 2446,2,54,2017-03-24 19:58:39,http://paucekokeefe.info/loren,0.7926050422,140.74.58.119,"{""location"": ""SI"", ""is_mobile"": false}" 2447,2,54,2017-03-28 19:52:57,http://williamson.co/lilyan.kunze,0.4062501810,225.89.126.31,"{""location"": ""BD"", ""is_mobile"": true}" 2448,2,54,2017-01-28 05:49:40,http://blandahagenes.co/grady,0.8910244725,139.182.176.39,"{""location"": ""CW"", ""is_mobile"": false}" 2449,2,54,2017-04-25 23:40:20,http://walter.org/halie.conroy,0.0115670398,69.114.18.194,"{""location"": ""TR"", ""is_mobile"": false}" 2450,2,54,2017-03-30 07:17:29,http://pfeffer.io/nels.lindgren,0.0414388291,160.183.60.239,"{""location"": ""MT"", ""is_mobile"": false}" 2451,2,54,2017-03-20 22:30:30,http://hilpert.biz/estevan,0.2504798341,145.41.149.184,"{""location"": ""CI"", ""is_mobile"": false}" 2452,2,54,2017-05-03 08:26:52,http://koelpinwilderman.biz/edd_krajcik,0.2432213460,139.117.48.145,"{""location"": ""MH"", ""is_mobile"": false}" 2453,2,54,2017-03-06 06:18:11,http://walker.com/tod.hermiston,0.9057980964,76.192.16.71,"{""location"": ""PS"", ""is_mobile"": true}" 2454,2,54,2017-02-02 15:00:36,http://keeling.co/stefanie_bernhard,0.7444520398,97.126.3.108,"{""location"": ""BL"", ""is_mobile"": true}" 2455,2,54,2017-03-26 22:06:48,http://blick.net/tierra_will,0.0564215964,191.201.245.36,"{""location"": ""RE"", ""is_mobile"": true}" 2456,2,54,2016-12-18 00:47:22,http://steuber.net/kris,0.1182580799,54.98.68.8,"{""location"": ""VU"", ""is_mobile"": true}" 2457,2,54,2017-02-24 23:35:50,http://rohan.io/darian_beatty,0.2912406688,200.210.13.245,"{""location"": ""PG"", ""is_mobile"": true}" 2458,2,54,2017-01-06 02:48:59,http://collier.io/tomasa,0.1305194071,172.59.237.208,"{""location"": ""AO"", ""is_mobile"": true}" 2459,2,54,2017-05-04 10:13:55,http://kuphalgulgowski.name/dannie.glover,0.1544406108,205.111.111.43,"{""location"": ""GD"", ""is_mobile"": true}" 2460,2,54,2017-04-26 03:03:43,http://oharadooley.co/emmie.schulist,0.6623398932,13.221.60.141,"{""location"": ""DZ"", ""is_mobile"": false}" 2461,2,54,2017-06-09 02:57:49,http://quigley.com/nolan,0.1161044528,234.12.65.85,"{""location"": ""BQ"", ""is_mobile"": false}" 2462,2,54,2017-01-29 08:49:07,http://doyle.com/elnora,0.2221848784,69.131.41.238,"{""location"": ""BY"", ""is_mobile"": false}" 2463,2,54,2017-03-01 15:14:45,http://blockparisian.org/lauren.renner,0.5798114963,48.101.219.211,"{""location"": ""NL"", ""is_mobile"": true}" 2464,2,54,2017-06-05 18:54:49,http://kerlukejones.info/danial.kirlin,0.7999860184,235.77.87.53,"{""location"": ""CG"", ""is_mobile"": false}" 2465,2,54,2017-03-30 01:43:17,http://donnelly.co/dulce.mills,0.3761852577,110.54.126.83,"{""location"": ""CW"", ""is_mobile"": true}" 2466,2,54,2017-03-09 18:01:28,http://abshire.org/raleigh_hammes,0.1358647694,191.26.204.232,"{""location"": ""SH"", ""is_mobile"": false}" 2467,2,54,2017-02-17 03:29:58,http://mcclure.info/milan.weinat,0.5421756206,212.166.50.244,"{""location"": ""FM"", ""is_mobile"": true}" 2468,2,54,2016-12-28 21:10:16,http://cummingswintheiser.biz/anne,0.9552554912,199.34.17.239,"{""location"": ""PE"", ""is_mobile"": true}" 9349,4,210,2016-12-30 08:19:10,http://reilly.info/zoie_parisian,0.4409377672,253.177.140.112,"{""location"": ""LV"", ""is_mobile"": false}" 9350,4,210,2017-01-29 06:23:43,http://williamsonpurdy.com/rachel.corwin,0.9660761513,222.192.133.201,"{""location"": ""RO"", ""is_mobile"": true}" 9351,4,210,2017-01-14 16:00:15,http://dubuque.com/genoveva_mills,0.9669760605,52.169.124.88,"{""location"": ""TL"", ""is_mobile"": false}" 9352,4,210,2017-03-23 06:42:41,http://skiles.org/ariane,0.5235652846,50.44.174.147,"{""location"": ""GM"", ""is_mobile"": false}" 9353,4,210,2017-04-07 03:59:36,http://adams.org/camden,0.9939310365,21.193.46.199,"{""location"": ""AD"", ""is_mobile"": true}" 9354,4,210,2017-01-18 10:19:45,http://harrisvolkman.net/tremayne,0.6777456391,23.237.37.128,"{""location"": ""RO"", ""is_mobile"": true}" 9355,4,210,2017-05-25 20:02:10,http://aufderhar.name/toney,0.7059451732,20.6.129.51,"{""location"": ""SA"", ""is_mobile"": false}" 9356,4,210,2017-02-11 22:44:29,http://gerholdcartwright.com/john,0.1662921222,180.15.73.104,"{""location"": ""IR"", ""is_mobile"": true}" 9357,4,211,2017-01-26 11:30:19,http://marks.co/taurean_franecki,0.6213545286,32.84.66.200,"{""location"": ""ZA"", ""is_mobile"": true}" 9358,4,211,2017-05-06 12:36:29,http://leuschkehaag.com/hertha,0.5273060323,205.93.226.17,"{""location"": ""DK"", ""is_mobile"": false}" 9359,4,211,2017-05-07 06:28:11,http://spencer.org/tamia.maggio,0.0476171740,70.116.235.68,"{""location"": ""SA"", ""is_mobile"": true}" 9360,4,211,2017-03-10 11:35:48,http://simonisgorczany.name/estella_harris,0.9435498028,214.58.229.244,"{""location"": ""BL"", ""is_mobile"": true}" 9361,4,211,2016-12-19 10:09:25,http://gerhold.co/jayden,0.4168374282,236.84.68.14,"{""location"": ""WS"", ""is_mobile"": true}" 9362,4,211,2017-05-22 10:39:06,http://volkman.biz/katharina.considine,0.5264327981,91.108.95.114,"{""location"": ""NC"", ""is_mobile"": true}" 9363,4,211,2017-03-17 14:19:07,http://hirthe.name/ari,0.7665950570,194.142.86.134,"{""location"": ""MM"", ""is_mobile"": true}" 9364,4,211,2017-01-24 06:32:50,http://botsford.name/daphnee_stamm,0.4336679465,134.45.218.204,"{""location"": ""UY"", ""is_mobile"": false}" 9365,4,211,2017-01-20 19:13:24,http://collins.org/gideon,0.5647984649,85.110.181.180,"{""location"": ""VE"", ""is_mobile"": false}" 9366,4,211,2017-05-15 00:00:26,http://schmitt.co/cheyanne,0.6328360450,169.5.177.5,"{""location"": ""TK"", ""is_mobile"": false}" 9367,4,211,2017-03-07 01:13:12,http://ferrybergnaum.net/glenna_aufderhar,0.6856979457,210.210.198.48,"{""location"": ""IE"", ""is_mobile"": true}" 9368,4,211,2017-02-03 17:33:34,http://kirlintreutel.name/josiane,0.2965349912,167.111.252.179,"{""location"": ""CW"", ""is_mobile"": true}" 9369,4,211,2017-05-09 17:57:19,http://leannon.com/rubie,0.3151328549,92.112.89.227,"{""location"": ""DZ"", ""is_mobile"": true}" 9370,4,211,2017-04-08 20:29:00,http://tremblaylubowitz.biz/frieda.wisoky,0.3998291300,87.114.64.240,"{""location"": ""GH"", ""is_mobile"": false}" 9371,4,211,2017-01-05 21:17:00,http://oconnell.com/arielle,0.5437939266,212.85.159.102,"{""location"": ""NI"", ""is_mobile"": false}" 9372,4,211,2016-12-27 07:18:10,http://waelchi.biz/jerald,0.5419435361,216.42.168.13,"{""location"": ""UM"", ""is_mobile"": true}" 9373,4,211,2017-03-30 21:41:10,http://friesen.com/arely,0.7077259468,44.226.17.197,"{""location"": ""LC"", ""is_mobile"": false}" 9374,4,211,2017-01-15 15:51:54,http://wiza.net/kamryn.wintheiser,0.6271060189,51.22.62.82,"{""location"": ""CO"", ""is_mobile"": false}" 9375,4,211,2017-03-17 08:03:16,http://schamberger.com/faye,0.0573184025,35.174.8.227,"{""location"": ""VU"", ""is_mobile"": false}" 9376,4,211,2017-03-03 11:02:38,http://gerhold.co/gerry_kozey,0.3933131091,133.31.205.36,"{""location"": ""OM"", ""is_mobile"": true}" 9377,4,211,2017-03-11 20:07:04,http://kuhn.com/tyra_stanton,0.5513489342,162.210.149.220,"{""location"": ""GF"", ""is_mobile"": false}" 9378,4,211,2017-05-11 13:37:22,http://heidenreichwatsica.org/giuseppe_sipes,0.3053096320,63.181.235.66,"{""location"": ""TM"", ""is_mobile"": true}" 9379,4,211,2017-05-16 10:11:10,http://green.biz/isabell_pouros,0.6090670802,74.165.206.30,"{""location"": ""ID"", ""is_mobile"": true}" 9380,4,211,2017-01-16 15:30:07,http://abshire.net/stefanie.raynor,0.3667866503,182.252.170.233,"{""location"": ""MF"", ""is_mobile"": false}" 11500,4,256,2017-06-10 04:48:54,http://kihn.biz/suzanne,,109.106.250.197,"{""location"": ""MA"", ""is_mobile"": false}" 9381,4,211,2017-05-04 08:14:45,http://herman.info/arvel.crona,0.3171763558,111.12.123.212,"{""location"": ""AI"", ""is_mobile"": true}" 9382,4,211,2016-12-17 08:38:11,http://moore.io/raven.labadie,0.4885376574,136.85.232.116,"{""location"": ""HN"", ""is_mobile"": false}" 9383,4,211,2017-02-19 06:18:27,http://watsicabaumbach.info/amina,0.9446527603,86.198.10.91,"{""location"": ""MW"", ""is_mobile"": true}" 9384,4,211,2017-03-13 21:09:10,http://stokes.net/jasmin,0.5844571943,184.178.53.136,"{""location"": ""HU"", ""is_mobile"": false}" 9385,4,211,2017-01-03 17:05:03,http://collier.io/reina.price,0.6239919180,138.143.114.43,"{""location"": ""BA"", ""is_mobile"": false}" 9386,4,211,2017-03-16 22:38:11,http://mohr.name/darron_wiza,0.3485517382,189.23.47.127,"{""location"": ""PY"", ""is_mobile"": true}" 9387,4,211,2017-03-10 11:41:42,http://koepp.com/sylvia.aufderhar,0.9589477348,228.240.218.137,"{""location"": ""GN"", ""is_mobile"": false}" 9388,4,211,2016-12-22 10:33:56,http://hilllschamberger.net/elton,0.8173880504,188.146.243.39,"{""location"": ""FK"", ""is_mobile"": true}" 9389,4,211,2017-03-02 20:33:47,http://oreilly.info/heaven.ebert,0.1248879651,63.84.148.7,"{""location"": ""CW"", ""is_mobile"": true}" 9390,4,211,2017-05-11 07:27:19,http://rippingoldner.net/maritza,0.9731227528,2.62.187.28,"{""location"": ""SC"", ""is_mobile"": true}" 9391,4,211,2017-02-16 12:35:24,http://labadie.biz/sage.crona,0.2524034989,248.80.17.81,"{""location"": ""ZA"", ""is_mobile"": true}" 9392,4,211,2017-02-17 04:35:47,http://kautzer.co/nicolas,0.7585233979,130.188.61.193,"{""location"": ""RW"", ""is_mobile"": true}" 9393,4,211,2017-04-16 08:13:35,http://kovacek.io/reggie_leuschke,0.2940170536,215.101.116.54,"{""location"": ""TD"", ""is_mobile"": false}" 9394,4,211,2017-04-17 01:41:26,http://schroeder.co/leta,0.0014175598,247.228.45.45,"{""location"": ""HM"", ""is_mobile"": false}" 9395,4,211,2017-05-16 10:15:50,http://schmitt.net/maybell.veum,0.1812880349,91.132.213.216,"{""location"": ""TF"", ""is_mobile"": false}" 9396,4,211,2017-01-15 02:04:44,http://nienowtromp.io/jerald.zemlak,0.9661257015,163.212.164.191,"{""location"": ""BH"", ""is_mobile"": false}" 9397,4,211,2017-01-21 00:16:57,http://rolfson.biz/abagail,0.1012646651,114.133.99.126,"{""location"": ""MG"", ""is_mobile"": true}" 9398,4,211,2017-05-22 04:56:14,http://okeefewilliamson.io/alberta,0.0298791773,151.45.200.112,"{""location"": ""IE"", ""is_mobile"": true}" 6359,3,139,2017-06-13 16:07:05,http://krajcik.org/judge_beier,0.9768025919,76.26.50.137,"{""location"": ""NL"", ""is_mobile"": true}" 6360,3,139,2017-06-12 09:58:06,http://bartell.co/josue.konopelski,0.5886286776,114.253.39.71,"{""location"": ""LK"", ""is_mobile"": false}" 6361,3,139,2017-01-29 11:41:17,http://trantow.co/kaylah.kihn,0.8445051948,115.92.205.12,"{""location"": ""BA"", ""is_mobile"": false}" 6362,3,139,2017-01-10 07:00:04,http://kub.org/buddy.klocko,0.8088140049,177.213.198.197,"{""location"": ""DJ"", ""is_mobile"": false}" 6363,3,139,2017-06-04 23:08:17,http://damorerobel.net/karlie.schmitt,0.4369161297,97.112.227.249,"{""location"": ""GW"", ""is_mobile"": true}" 6364,3,139,2017-05-30 23:16:31,http://bergstrom.name/wilfred_runolfon,0.5525087658,141.81.10.60,"{""location"": ""AT"", ""is_mobile"": true}" 6365,3,139,2016-12-26 18:27:03,http://borer.org/keegan,0.6675013230,227.153.58.120,"{""location"": ""SH"", ""is_mobile"": true}" 6366,3,139,2017-03-09 01:21:49,http://wyman.co/dolly,0.1512464473,26.216.72.33,"{""location"": ""CI"", ""is_mobile"": true}" 6367,3,139,2017-06-13 11:08:34,http://collins.co/dominic,0.5157626616,105.100.108.156,"{""location"": ""FM"", ""is_mobile"": true}" 6368,3,139,2017-05-31 07:22:49,http://morar.org/demetris,0.3118847648,108.49.52.182,"{""location"": ""JE"", ""is_mobile"": false}" 6369,3,139,2017-01-26 17:56:58,http://dickinsonbogan.co/laurianne,0.2343032712,67.247.201.127,"{""location"": ""LU"", ""is_mobile"": true}" 6370,3,139,2017-02-15 00:55:25,http://veum.info/marie_huel,0.8789436790,55.253.23.56,"{""location"": ""MD"", ""is_mobile"": true}" 6371,3,139,2017-06-01 05:03:28,http://bechtelar.io/arely_feest,0.6073036038,2.92.33.173,"{""location"": ""RW"", ""is_mobile"": false}" 6372,3,139,2017-04-29 03:01:04,http://johns.io/brisa_pagac,0.6490538059,138.21.173.250,"{""location"": ""CU"", ""is_mobile"": true}" 6373,3,139,2017-03-18 10:29:32,http://kozey.org/adriana_hoeger,0.6041005561,61.58.80.107,"{""location"": ""SJ"", ""is_mobile"": false}" 6374,3,139,2017-01-04 03:07:41,http://hand.info/renee.boyle,0.4767194682,122.18.30.234,"{""location"": ""MM"", ""is_mobile"": true}" 6375,3,139,2017-03-28 01:49:41,http://rogahn.co/ambrose.renner,0.2625564565,40.84.39.99,"{""location"": ""PM"", ""is_mobile"": false}" 6376,3,139,2016-12-23 21:41:06,http://reinger.co/pauline.willms,0.1704846240,186.124.152.28,"{""location"": ""IE"", ""is_mobile"": true}" 6377,3,139,2017-02-23 09:26:44,http://okeefecummings.info/arturo.cremin,0.9487255376,141.7.200.126,"{""location"": ""SO"", ""is_mobile"": true}" 6378,3,139,2017-06-06 21:21:01,http://reingermraz.net/laney,0.2618966039,31.221.48.247,"{""location"": ""QA"", ""is_mobile"": false}" 6379,3,139,2017-03-01 16:30:58,http://abshirelehner.biz/alexane.hamill,0.4346032247,162.93.236.100,"{""location"": ""BI"", ""is_mobile"": false}" 6380,3,139,2017-05-30 02:09:31,http://zboncak.com/forrest_yundt,0.1013567634,114.175.223.80,"{""location"": ""AX"", ""is_mobile"": true}" 6381,3,139,2017-06-13 22:26:40,http://donnelly.com/cale_greenholt,0.0338284174,187.205.183.252,"{""location"": ""SY"", ""is_mobile"": false}" 6382,3,139,2017-02-14 19:09:08,http://ledner.com/price_jakubowski,0.5872349034,47.184.221.2,"{""location"": ""UM"", ""is_mobile"": false}" 6383,3,139,2016-12-28 11:43:16,http://gutmann.net/willie.erdman,0.7881227786,53.30.253.142,"{""location"": ""AO"", ""is_mobile"": false}" 6384,3,139,2017-01-13 15:21:34,http://rosenbaum.co/alysha,0.6484327361,194.32.186.242,"{""location"": ""BY"", ""is_mobile"": false}" 6385,3,139,2017-03-28 11:12:05,http://olsonboyer.io/lavada,0.1197242961,157.209.189.47,"{""location"": ""BH"", ""is_mobile"": false}" 6386,3,139,2017-02-12 05:46:38,http://schmittbogisich.co/dusty,0.4827288270,101.85.228.20,"{""location"": ""TK"", ""is_mobile"": false}" 6387,3,139,2017-01-24 00:05:30,http://weinat.co/javier_ruecker,0.2149831602,153.54.102.84,"{""location"": ""DZ"", ""is_mobile"": true}" 6388,3,139,2017-05-21 07:26:08,http://beatty.org/maximilian,0.4786258747,12.82.33.156,"{""location"": ""OM"", ""is_mobile"": false}" 6389,3,139,2017-01-17 01:56:47,http://johns.name/vita,0.8447178769,112.80.237.101,"{""location"": ""NU"", ""is_mobile"": true}" 6390,3,139,2017-03-06 18:55:50,http://mcglynnbaumbach.co/marilyne,0.2000496615,162.246.70.24,"{""location"": ""TT"", ""is_mobile"": true}" 6391,3,139,2016-12-24 04:36:58,http://lubowitz.co/alexandra,0.5054985245,82.68.210.127,"{""location"": ""BQ"", ""is_mobile"": false}" 6392,3,139,2017-06-12 18:51:56,http://grimes.co/logan.kuhn,0.5562429488,214.164.229.134,"{""location"": ""AF"", ""is_mobile"": true}" 6393,3,139,2017-05-11 22:16:02,http://weimannveum.co/aileen,0.9535645480,162.248.230.161,"{""location"": ""BT"", ""is_mobile"": false}" 6394,3,140,2017-01-24 05:26:35,http://mcglynn.net/cruz,0.0545569090,145.220.60.248,"{""location"": ""IQ"", ""is_mobile"": false}" 6395,3,140,2017-04-16 12:28:23,http://collins.com/charlene,0.8821985081,237.140.171.225,"{""location"": ""AW"", ""is_mobile"": false}" 6396,3,140,2017-01-29 06:33:29,http://homenickkuvalis.name/hector.bechtelar,0.4011843730,119.198.56.114,"{""location"": ""PS"", ""is_mobile"": true}" 6397,3,140,2017-03-07 13:37:08,http://barton.info/sherman,0.8800024164,168.36.254.21,"{""location"": ""HM"", ""is_mobile"": false}" 6398,3,140,2017-02-21 06:11:32,http://bergnaum.net/geovany.miller,0.4137851761,62.195.110.220,"{""location"": ""TN"", ""is_mobile"": true}" 6399,3,140,2017-01-28 23:36:01,http://mosciski.com/gunnar,0.6189960182,33.83.57.243,"{""location"": ""MU"", ""is_mobile"": true}" 6400,3,140,2017-05-06 00:04:20,http://lindgrenjerde.org/unique,0.9859982223,66.45.117.224,"{""location"": ""HM"", ""is_mobile"": false}" 6401,3,140,2017-06-03 18:28:50,http://grahamkiehn.org/stefanie,0.2531287665,230.13.248.159,"{""location"": ""PW"", ""is_mobile"": false}" 6402,3,140,2017-06-11 14:22:55,http://herman.name/hildegard.robel,0.4580445433,65.69.88.102,"{""location"": ""QA"", ""is_mobile"": true}" 6403,3,140,2017-06-11 20:45:44,http://tremblay.co/nicole.blanda,0.2308204977,241.137.235.157,"{""location"": ""SG"", ""is_mobile"": true}" 6404,3,140,2017-04-28 09:11:08,http://shanahanhane.net/rhoda_zieme,0.1290331321,142.55.162.227,"{""location"": ""NA"", ""is_mobile"": true}" 6405,3,140,2017-05-26 13:11:52,http://gleichner.biz/verla,0.6662069960,217.175.183.37,"{""location"": ""GN"", ""is_mobile"": false}" 6406,3,140,2017-02-23 13:42:58,http://wunschlockman.net/blanche_lueilwitz,0.6493628477,45.191.221.26,"{""location"": ""MX"", ""is_mobile"": true}" 6407,3,140,2017-05-31 01:10:24,http://mosciski.io/hoyt,0.8283926030,21.144.64.99,"{""location"": ""PT"", ""is_mobile"": false}" 6408,3,140,2017-03-24 13:01:27,http://beier.com/pamela,0.8381995955,176.72.90.225,"{""location"": ""JE"", ""is_mobile"": false}" 6409,3,140,2017-01-25 18:58:27,http://blickondricka.biz/chelsea,0.1076236514,65.130.85.36,"{""location"": ""SA"", ""is_mobile"": true}" 12264,5,275,2017-01-21 03:02:53,http://abbottquitzon.info/remington.corkery,0.3360508046,138.245.131.247,"{""location"": ""HU"", ""is_mobile"": true}" 12265,5,275,2017-03-08 19:28:31,http://dickinson.co/lurline.lehner,0.5566131469,49.163.153.27,"{""location"": ""NR"", ""is_mobile"": false}" 12266,5,275,2017-03-24 19:24:33,http://hoeger.biz/dax,0.0368716295,133.58.212.40,"{""location"": ""UA"", ""is_mobile"": true}" 12267,5,275,2016-12-13 12:21:56,http://kertzmann.net/roberto,0.7619531571,58.241.52.253,"{""location"": ""BT"", ""is_mobile"": false}" 12268,5,275,2016-12-25 05:28:47,http://wisoky.info/adela,0.9074700084,106.132.9.70,"{""location"": ""BE"", ""is_mobile"": true}" 12269,5,275,2016-12-22 00:41:12,http://bergstrom.net/isabel_keler,0.3521403076,203.178.229.58,"{""location"": ""UG"", ""is_mobile"": false}" 12270,5,275,2017-04-20 13:51:47,http://hermannwitting.biz/wellington,0.0844487688,158.135.128.109,"{""location"": ""LB"", ""is_mobile"": true}" 12271,5,275,2017-03-06 21:47:04,http://walter.biz/david_maggio,0.7075108705,77.164.248.60,"{""location"": ""ST"", ""is_mobile"": false}" 12272,5,275,2017-03-10 03:36:10,http://moriettelind.io/william_ernser,0.9417105766,211.42.187.89,"{""location"": ""EE"", ""is_mobile"": false}" 12273,5,275,2017-04-19 21:43:47,http://prosacco.com/hanna.crist,0.4546798121,142.218.128.69,"{""location"": ""BY"", ""is_mobile"": true}" 12274,5,275,2017-06-07 14:59:56,http://macejkovic.biz/levi,0.9619941203,170.156.59.49,"{""location"": ""LV"", ""is_mobile"": true}" 12275,5,275,2017-05-15 09:10:02,http://connelly.com/terrance_murazik,0.3199102955,153.37.42.185,"{""location"": ""BQ"", ""is_mobile"": false}" 12276,5,275,2016-12-28 13:07:57,http://smithtillman.net/flavie,0.1938164742,58.156.82.158,"{""location"": ""NU"", ""is_mobile"": true}" 12277,5,275,2017-05-20 14:44:32,http://blanda.co/myrna_ledner,0.0319988476,153.113.79.70,"{""location"": ""CL"", ""is_mobile"": true}" 12278,5,275,2017-04-29 00:38:22,http://roberts.co/timmy.trantow,0.9341448052,38.62.136.78,"{""location"": ""BB"", ""is_mobile"": true}" 12279,5,275,2017-03-05 11:41:19,http://rath.net/edna,0.3080957963,154.56.178.137,"{""location"": ""GU"", ""is_mobile"": false}" 12280,5,275,2017-01-25 21:52:24,http://fay.co/lina.rau,0.6199755258,160.124.175.163,"{""location"": ""RE"", ""is_mobile"": false}" 12281,5,275,2017-01-19 02:59:56,http://monahan.co/trisha,0.2159740112,189.70.249.202,"{""location"": ""BD"", ""is_mobile"": true}" 12282,5,275,2017-02-05 06:51:00,http://kingzieme.org/aiden.rolfson,0.7594094241,132.233.50.220,"{""location"": ""TK"", ""is_mobile"": true}" 12283,5,275,2017-05-30 09:20:43,http://tremblaycummings.biz/carley_ryan,0.8662728946,86.224.154.166,"{""location"": ""IR"", ""is_mobile"": true}" 12284,5,275,2017-03-09 22:04:09,http://uptonboyer.co/rex,0.1611521137,195.64.10.59,"{""location"": ""ID"", ""is_mobile"": false}" 12285,5,275,2017-04-15 03:30:50,http://balistreri.name/mattie_rolfson,0.3323063187,2.222.63.163,"{""location"": ""RW"", ""is_mobile"": true}" 12286,5,275,2017-02-11 00:05:31,http://jacobs.com/vicky,0.0885012252,191.120.248.64,"{""location"": ""DK"", ""is_mobile"": false}" 12287,5,275,2017-05-30 22:11:47,http://gulgowski.net/hermann.oconner,0.6808919251,150.136.21.39,"{""location"": ""AU"", ""is_mobile"": true}" 12288,5,275,2017-04-02 15:29:01,http://kilbackmoore.co/frederick,0.2369582049,15.224.71.71,"{""location"": ""CC"", ""is_mobile"": false}" 12289,5,275,2017-01-01 19:00:11,http://monahan.co/derek,0.7570727199,250.59.85.100,"{""location"": ""AT"", ""is_mobile"": false}" 12290,5,275,2017-04-23 11:02:18,http://bosco.biz/shakira.torphy,0.0993611696,26.77.105.238,"{""location"": ""PY"", ""is_mobile"": true}" 12291,5,275,2017-06-11 05:34:02,http://homenick.com/anais.feil,0.8647500981,128.46.151.159,"{""location"": ""IO"", ""is_mobile"": false}" 12292,5,275,2017-02-10 03:14:50,http://stammschimmel.com/margaretta_kilback,0.6482379851,177.18.93.38,"{""location"": ""NZ"", ""is_mobile"": true}" 12293,5,275,2017-03-17 20:37:03,http://padberg.com/noemie,0.6394385123,3.113.251.249,"{""location"": ""AO"", ""is_mobile"": false}" 12294,5,275,2017-05-21 20:39:23,http://kozey.com/eddie,0.4267610022,131.77.27.240,"{""location"": ""IS"", ""is_mobile"": true}" 12295,5,275,2017-01-09 00:00:31,http://cremin.com/alden_stoltenberg,0.5668973249,11.219.175.94,"{""location"": ""FO"", ""is_mobile"": false}" 12296,5,275,2017-02-09 05:43:41,http://beattyhirthe.com/denis,0.5835324289,180.95.226.137,"{""location"": ""EE"", ""is_mobile"": false}" 12297,5,275,2017-01-27 03:56:14,http://johnsonfarrell.name/sydnee,0.9460944867,237.126.220.154,"{""location"": ""NL"", ""is_mobile"": false}" 12298,5,275,2017-02-24 05:16:51,http://thiel.name/wellington,0.7201660010,78.217.254.23,"{""location"": ""SO"", ""is_mobile"": true}" 12299,5,275,2017-05-03 02:03:12,http://hagenes.io/wilma,0.6527421520,134.207.8.76,"{""location"": ""GQ"", ""is_mobile"": false}" 12300,5,275,2016-12-31 11:01:57,http://lubowitz.biz/mallie,0.9505579418,108.114.45.30,"{""location"": ""CX"", ""is_mobile"": false}" 12301,5,275,2016-12-23 19:22:35,http://mcglynn.com/margarett,0.2703175220,242.158.48.122,"{""location"": ""NF"", ""is_mobile"": true}" 12302,5,275,2017-06-04 04:53:18,http://christiansendare.com/helmer.ferry,0.3395245536,215.160.50.7,"{""location"": ""CC"", ""is_mobile"": true}" 12303,5,275,2017-03-29 06:37:56,http://hoppe.com/lennie_lemke,0.7517829154,70.123.214.102,"{""location"": ""PA"", ""is_mobile"": true}" 12304,5,275,2017-03-23 15:39:44,http://lemkepacocha.org/rickie,0.7810242527,155.35.4.29,"{""location"": ""AQ"", ""is_mobile"": false}" 12305,5,275,2016-12-24 03:17:50,http://johnson.com/mckayla.schmitt,0.8823610745,84.75.39.253,"{""location"": ""BN"", ""is_mobile"": false}" 12306,5,275,2017-03-07 20:45:31,http://konopelski.info/mohammad_jacobi,0.7539223855,124.220.169.62,"{""location"": ""TV"", ""is_mobile"": true}" 12307,5,275,2017-02-03 12:34:11,http://macejkovicfriesen.co/gladys_emard,0.9932231861,65.36.182.161,"{""location"": ""MK"", ""is_mobile"": false}" 12308,5,275,2017-02-21 12:41:03,http://barton.biz/antonina,0.5258819225,177.58.212.30,"{""location"": ""GG"", ""is_mobile"": true}" 12309,5,275,2017-05-28 03:22:22,http://wintheiser.co/cleve_nienow,0.7332386759,180.251.2.246,"{""location"": ""VG"", ""is_mobile"": false}" 12310,5,275,2017-05-01 04:01:00,http://denesikhauck.io/stephanie,0.5985991975,162.205.111.14,"{""location"": ""BI"", ""is_mobile"": true}" 12311,5,275,2017-05-07 17:25:14,http://reynolds.info/tomas.rempel,0.2183152721,50.231.87.72,"{""location"": ""AQ"", ""is_mobile"": false}" 12312,5,276,2017-01-05 07:16:06,http://zboncak.name/michale,0.2001611054,81.142.184.236,"{""location"": ""MP"", ""is_mobile"": true}" 12313,5,276,2017-03-29 11:52:50,http://fadel.io/ron.altenwerth,0.7816472955,139.205.87.162,"{""location"": ""MV"", ""is_mobile"": true}" 12314,5,276,2016-12-16 06:12:20,http://padberg.biz/maida,0.4919462451,213.216.244.111,"{""location"": ""TM"", ""is_mobile"": true}" 15273,6,343,2017-01-15 15:41:02,http://murray.info/elmer_huel,,201.197.242.127,"{""location"": ""QA"", ""is_mobile"": false}" 15274,6,343,2017-06-09 22:48:47,http://mccullough.com/clay,,160.155.140.244,"{""location"": ""SO"", ""is_mobile"": false}" 15275,6,343,2017-02-13 02:11:39,http://wisokyupton.net/tevin_hickle,,124.125.58.226,"{""location"": ""FI"", ""is_mobile"": false}" 15276,6,343,2016-12-31 09:35:02,http://dietrichjacobi.com/david,,62.52.102.9,"{""location"": ""CR"", ""is_mobile"": true}" 15277,6,343,2017-02-07 10:23:44,http://littel.org/woodrow,,114.60.183.85,"{""location"": ""IN"", ""is_mobile"": false}" 15278,6,343,2017-05-01 06:07:36,http://reynolds.co/tyler_haag,,116.141.65.21,"{""location"": ""CY"", ""is_mobile"": false}" 15279,6,343,2017-03-30 21:42:00,http://dach.info/guiseppe,,249.246.204.35,"{""location"": ""KG"", ""is_mobile"": false}" 15280,6,343,2017-04-04 04:10:47,http://donnelly.biz/rhoda.bernhard,,100.236.49.92,"{""location"": ""BE"", ""is_mobile"": true}" 15281,6,344,2017-02-28 04:31:42,http://kubbode.org/ebba_jakubowski,,44.168.99.26,"{""location"": ""LI"", ""is_mobile"": false}" 15282,6,344,2017-04-08 20:00:52,http://harvey.net/clifford,,231.157.13.17,"{""location"": ""NZ"", ""is_mobile"": false}" 15283,6,344,2017-01-01 16:21:08,http://cummings.info/eleazar,,88.2.247.68,"{""location"": ""GY"", ""is_mobile"": false}" 15284,6,344,2017-03-19 21:53:17,http://renner.org/dawn,,243.53.204.55,"{""location"": ""GW"", ""is_mobile"": false}" 15285,6,344,2017-02-24 17:15:18,http://hermannolan.com/robyn_howe,,43.211.179.210,"{""location"": ""GP"", ""is_mobile"": true}" 15286,6,344,2017-04-01 14:17:51,http://stammlangworth.biz/mohammad,,153.231.102.192,"{""location"": ""MC"", ""is_mobile"": false}" 15287,6,344,2017-03-01 08:45:35,http://emmerich.co/haley.blick,,240.99.62.164,"{""location"": ""VA"", ""is_mobile"": false}" 15289,6,344,2017-01-18 17:24:29,http://hammesheathcote.com/landen,,137.221.154.122,"{""location"": ""CR"", ""is_mobile"": true}" 15290,6,344,2017-02-11 14:12:50,http://eichmann.io/hayley,,53.252.70.148,"{""location"": ""GI"", ""is_mobile"": false}" 15291,6,344,2017-06-06 18:24:30,http://wiza.biz/shakira_volkman,,58.162.149.22,"{""location"": ""EH"", ""is_mobile"": false}" 15292,6,344,2017-02-18 20:36:54,http://schuppevon.co/tyrique.rohan,,206.45.144.15,"{""location"": ""LT"", ""is_mobile"": true}" 15293,6,344,2017-05-23 01:35:03,http://rolfson.biz/nat,,160.131.66.36,"{""location"": ""GW"", ""is_mobile"": true}" 15294,6,344,2016-12-15 02:21:43,http://rodriguez.com/tania,,8.69.9.172,"{""location"": ""LK"", ""is_mobile"": false}" 15295,6,344,2017-02-18 01:11:14,http://kaulkehowell.name/amelia_osinski,,107.91.196.222,"{""location"": ""AO"", ""is_mobile"": true}" 15296,6,344,2017-01-16 09:10:12,http://abbott.org/marielle,,99.14.73.155,"{""location"": ""GF"", ""is_mobile"": true}" 15297,6,344,2017-03-12 22:36:07,http://legros.org/triston.runolfsdottir,,63.189.73.225,"{""location"": ""LV"", ""is_mobile"": true}" 15298,6,344,2017-01-08 06:19:07,http://kling.co/serenity.oreilly,,210.128.42.223,"{""location"": ""MW"", ""is_mobile"": false}" 15299,6,344,2016-12-18 10:20:07,http://barrowswunsch.org/hanna_bernhard,,5.187.11.100,"{""location"": ""PG"", ""is_mobile"": true}" 15300,6,344,2017-06-01 22:05:18,http://greenfelderdickinson.info/jordon.heathcote,,15.254.24.23,"{""location"": ""LB"", ""is_mobile"": true}" 15301,6,344,2017-04-11 07:21:31,http://keler.info/hilda,,182.113.119.84,"{""location"": ""MS"", ""is_mobile"": false}" 15302,6,344,2017-04-24 12:24:33,http://wildermanfeest.co/harvey,,191.78.190.180,"{""location"": ""GH"", ""is_mobile"": true}" 15303,6,344,2017-02-17 16:38:34,http://ledner.io/jadyn,,84.160.109.203,"{""location"": ""AR"", ""is_mobile"": true}" 15304,6,344,2017-02-22 00:23:36,http://kohler.com/hillard.hudson,,24.43.129.186,"{""location"": ""OM"", ""is_mobile"": true}" 15305,6,344,2017-04-20 13:06:30,http://stroman.name/trea.moriette,,105.91.51.3,"{""location"": ""CF"", ""is_mobile"": true}" 15306,6,344,2017-03-15 04:26:44,http://parker.net/toy,,152.113.3.51,"{""location"": ""AW"", ""is_mobile"": true}" 15307,6,344,2017-04-28 20:31:52,http://howe.io/teagan.crona,,201.9.207.3,"{""location"": ""AG"", ""is_mobile"": true}" 15308,6,344,2017-05-17 16:48:45,http://jacobsonharvey.co/janelle,,110.74.152.236,"{""location"": ""MV"", ""is_mobile"": true}" 15309,6,344,2017-04-13 14:24:21,http://harriauer.org/lyda.daniel,,108.32.5.196,"{""location"": ""JP"", ""is_mobile"": false}" 15310,6,344,2017-04-06 06:35:42,http://emard.name/briana_shields,,133.112.128.54,"{""location"": ""KR"", ""is_mobile"": false}" 15311,6,344,2017-02-03 00:11:31,http://walker.org/jefferey,,119.117.229.85,"{""location"": ""BF"", ""is_mobile"": true}" 15312,6,344,2017-02-28 20:40:02,http://jacobi.io/casey,,47.185.82.141,"{""location"": ""ID"", ""is_mobile"": false}" 15313,6,344,2017-03-25 08:06:59,http://schuppe.io/narciso,,39.207.238.18,"{""location"": ""MR"", ""is_mobile"": false}" 15314,6,344,2017-05-01 15:52:07,http://wintheiser.info/germaine,,64.34.186.176,"{""location"": ""AM"", ""is_mobile"": true}" 15315,6,344,2017-05-24 02:51:09,http://danielschowalter.net/jada,,84.171.219.189,"{""location"": ""CR"", ""is_mobile"": false}" 15316,6,344,2017-05-26 02:03:30,http://davisheller.com/genoveva_littel,,24.143.174.247,"{""location"": ""CM"", ""is_mobile"": false}" 15317,6,344,2017-04-02 23:05:13,http://moriette.co/aubrey,,199.79.254.202,"{""location"": ""IS"", ""is_mobile"": true}" 15318,6,344,2017-06-11 20:57:37,http://veumlangworth.name/emelie_price,,171.176.221.244,"{""location"": ""RO"", ""is_mobile"": true}" 15319,6,344,2017-02-01 12:25:13,http://moen.org/marietta,,163.244.44.104,"{""location"": ""LS"", ""is_mobile"": false}" 15320,6,344,2017-03-26 14:42:32,http://lynchkshlerin.name/brooklyn,,43.28.224.119,"{""location"": ""YE"", ""is_mobile"": true}" 15321,6,344,2017-01-10 11:57:30,http://damorefay.net/vance_dickinson,,15.152.50.151,"{""location"": ""IO"", ""is_mobile"": true}" 15322,6,344,2017-05-17 15:58:27,http://abbottweinat.biz/owen,,25.105.175.119,"{""location"": ""LS"", ""is_mobile"": true}" 15323,6,344,2017-04-21 21:14:30,http://leannonbrown.info/morton.gusikowski,,63.185.137.129,"{""location"": ""GY"", ""is_mobile"": true}" 15324,6,344,2016-12-16 15:18:59,http://mueller.org/dario,,141.130.113.240,"{""location"": ""PE"", ""is_mobile"": true}" 15325,6,344,2017-05-08 21:02:37,http://lang.biz/marques.kovacek,,103.42.188.248,"{""location"": ""TC"", ""is_mobile"": false}" 15326,6,344,2017-06-08 21:48:21,http://murray.biz/norene,,250.15.107.36,"{""location"": ""DE"", ""is_mobile"": true}" 15327,6,344,2017-01-13 15:59:07,http://mueller.info/angelina_kunde,,29.238.101.129,"{""location"": ""DZ"", ""is_mobile"": true}" 15328,6,344,2017-03-18 00:37:43,http://shields.name/lindsey_kertzmann,,50.129.179.169,"{""location"": ""IE"", ""is_mobile"": false}" 2469,2,54,2017-06-13 16:45:47,http://roobpowlowski.co/keshaun_pacocha,0.1376652059,159.80.152.126,"{""location"": ""GY"", ""is_mobile"": false}" 2470,2,54,2016-12-16 22:19:54,http://steuber.biz/daisha,0.6553158244,166.212.44.94,"{""location"": ""TM"", ""is_mobile"": false}" 2471,2,54,2017-05-28 19:03:41,http://hermann.name/dean,0.0998805735,120.112.191.40,"{""location"": ""GE"", ""is_mobile"": false}" 2472,2,54,2017-02-25 11:13:27,http://rogahnanderson.info/annette,0.1487252361,213.51.147.155,"{""location"": ""CL"", ""is_mobile"": false}" 2473,2,54,2017-04-01 23:05:16,http://murray.co/iliana.ryan,0.8899319728,74.49.252.55,"{""location"": ""IO"", ""is_mobile"": false}" 2474,2,54,2017-02-23 14:20:12,http://crona.io/garret_yost,0.4080043172,77.103.12.113,"{""location"": ""MW"", ""is_mobile"": true}" 2475,2,54,2017-03-15 19:26:06,http://ledner.net/kelsie.daugherty,0.4644809780,112.40.248.83,"{""location"": ""TZ"", ""is_mobile"": true}" 2476,2,54,2017-01-19 16:00:33,http://rau.io/ro.vonrueden,0.0434729480,128.28.242.53,"{""location"": ""PK"", ""is_mobile"": false}" 2477,2,54,2017-05-30 03:07:26,http://bins.biz/jakayla_jacobson,0.5296388744,78.234.87.148,"{""location"": ""PL"", ""is_mobile"": false}" 2478,2,54,2017-01-23 01:12:46,http://hauck.info/amir,0.1001439741,175.38.218.73,"{""location"": ""SN"", ""is_mobile"": true}" 2479,2,54,2017-01-02 00:16:12,http://friesen.biz/kristina,0.6982916759,202.26.170.92,"{""location"": ""AL"", ""is_mobile"": true}" 2480,2,55,2017-01-29 05:00:01,http://beer.biz/alphonso.wiza,0.2933637260,71.81.245.244,"{""location"": ""LC"", ""is_mobile"": false}" 2481,2,55,2017-05-02 05:51:55,http://gutkowski.net/nathaniel.upton,0.6982740163,35.107.151.36,"{""location"": ""LI"", ""is_mobile"": true}" 2482,2,55,2017-05-30 18:56:30,http://gottliebpollich.info/marlen.gleichner,0.7685743326,79.83.80.125,"{""location"": ""PY"", ""is_mobile"": true}" 2483,2,55,2017-04-30 12:46:49,http://mcclurepowlowski.com/adella,0.0784691170,2.224.118.87,"{""location"": ""DE"", ""is_mobile"": true}" 2484,2,55,2017-04-24 17:40:17,http://crona.io/lucinda,0.6413251726,115.81.85.62,"{""location"": ""PY"", ""is_mobile"": true}" 2485,2,55,2017-03-27 03:11:10,http://andersonfay.co/kaylie_kihn,0.7402888559,21.203.253.252,"{""location"": ""IO"", ""is_mobile"": false}" 2486,2,55,2017-02-28 13:11:39,http://mertz.com/ena,0.8391321516,179.192.21.80,"{""location"": ""QA"", ""is_mobile"": false}" 2487,2,55,2017-06-06 02:34:48,http://volkman.name/cora.ankunding,0.9616650348,19.233.159.95,"{""location"": ""EG"", ""is_mobile"": false}" 2488,2,55,2017-02-06 09:38:03,http://lang.co/katlynn_kirlin,0.6222331055,102.219.13.73,"{""location"": ""GR"", ""is_mobile"": false}" 2489,2,55,2017-05-15 13:11:04,http://schaden.co/darien.heaney,0.7065338815,113.233.184.148,"{""location"": ""TV"", ""is_mobile"": true}" 2490,2,55,2017-01-16 13:22:00,http://westdaugherty.com/destany,0.0898102048,136.122.235.101,"{""location"": ""GI"", ""is_mobile"": false}" 2491,2,55,2017-04-02 13:12:13,http://oharapurdy.com/evalyn,0.8124353259,13.203.60.147,"{""location"": ""MQ"", ""is_mobile"": false}" 2492,2,55,2017-03-26 07:27:50,http://mcdermott.co/marlen,0.4265812321,235.205.45.56,"{""location"": ""VN"", ""is_mobile"": true}" 2493,2,55,2017-03-02 18:54:42,http://halvorson.com/christop,0.3876591823,228.32.155.102,"{""location"": ""CK"", ""is_mobile"": false}" 2494,2,55,2016-12-20 12:36:13,http://johnsbrekke.biz/tyra,0.6168654309,96.49.233.114,"{""location"": ""BH"", ""is_mobile"": false}" 2495,2,55,2017-05-31 14:47:00,http://farrell.net/nathanial.ernser,0.7017816544,62.202.141.168,"{""location"": ""CN"", ""is_mobile"": false}" 2496,2,55,2017-02-10 07:49:41,http://beerabbott.name/elia,0.2959021392,40.29.50.212,"{""location"": ""LY"", ""is_mobile"": true}" 2497,2,55,2017-03-21 09:32:34,http://wintheiserprice.name/johnny.wintheiser,0.2247293141,99.229.130.181,"{""location"": ""MM"", ""is_mobile"": true}" 2498,2,55,2017-03-04 05:50:57,http://fisheroconner.co/mae,0.1329079173,121.31.213.165,"{""location"": ""UG"", ""is_mobile"": true}" 2499,2,55,2017-05-02 21:43:09,http://collierschaden.biz/elisa.schaden,0.4299326549,130.242.45.204,"{""location"": ""VE"", ""is_mobile"": false}" 2500,2,55,2016-12-21 10:12:51,http://konopelskiborer.io/carson.schulist,0.8249208574,53.183.133.28,"{""location"": ""TT"", ""is_mobile"": true}" 2501,2,55,2017-01-31 05:50:26,http://kunze.info/tyrese,0.6774383913,116.46.237.244,"{""location"": ""CC"", ""is_mobile"": true}" 2502,2,55,2017-03-14 03:36:51,http://bechtelarsimonis.com/jenifer,0.4854301452,23.28.156.234,"{""location"": ""PG"", ""is_mobile"": true}" 2503,2,55,2017-05-13 18:13:46,http://bailey.co/geovanny_hansen,0.6224936895,9.77.4.131,"{""location"": ""PF"", ""is_mobile"": false}" 2504,2,55,2017-06-12 19:40:27,http://kihn.io/aurore.sauer,0.4651812668,165.3.152.168,"{""location"": ""IQ"", ""is_mobile"": false}" 2505,2,55,2017-03-21 11:46:17,http://jaskolskimurphy.org/zakary,0.4325778088,81.173.210.4,"{""location"": ""GL"", ""is_mobile"": true}" 2506,2,55,2017-02-06 17:40:10,http://morar.co/paul,0.5537675962,75.84.233.211,"{""location"": ""BE"", ""is_mobile"": true}" 2507,2,55,2016-12-25 04:43:50,http://moen.com/leanna,0.8428602755,68.236.8.72,"{""location"": ""MG"", ""is_mobile"": true}" 2508,2,55,2017-06-05 13:57:27,http://macejkovic.name/josianne.mueller,0.5623264176,97.100.174.40,"{""location"": ""GF"", ""is_mobile"": false}" 2509,2,55,2016-12-27 01:53:08,http://kris.info/isabell_thompson,0.8242909863,173.223.150.174,"{""location"": ""KR"", ""is_mobile"": true}" 2510,2,55,2017-04-11 01:16:23,http://heidenreich.com/shayne,0.5579806403,46.191.180.85,"{""location"": ""CA"", ""is_mobile"": false}" 2511,2,55,2017-03-02 09:35:56,http://hand.io/kelvin,0.0933199378,80.239.4.33,"{""location"": ""MA"", ""is_mobile"": true}" 2512,2,55,2017-04-29 06:01:52,http://kuphalmckenzie.com/nelson_cartwright,0.2074446034,164.116.62.102,"{""location"": ""VA"", ""is_mobile"": false}" 2513,2,55,2017-04-12 01:54:17,http://waters.co/dandre,0.8209743581,138.70.54.232,"{""location"": ""KG"", ""is_mobile"": true}" 2514,2,55,2017-01-07 04:29:37,http://daniel.com/richmond,0.2543039085,216.9.4.237,"{""location"": ""CK"", ""is_mobile"": false}" 2515,2,55,2016-12-15 04:30:31,http://osinski.biz/brody,0.6562397367,183.79.115.81,"{""location"": ""EG"", ""is_mobile"": false}" 2516,2,55,2017-05-07 09:29:16,http://spencer.io/alf.kub,0.6931936520,195.49.154.47,"{""location"": ""BS"", ""is_mobile"": true}" 2517,2,55,2017-02-10 12:21:05,http://thompsondicki.org/marcia.keler,0.3465529883,16.111.192.151,"{""location"": ""GS"", ""is_mobile"": false}" 2518,2,55,2017-02-06 22:33:40,http://erdman.io/birdie_parisian,0.0569547118,163.247.190.103,"{""location"": ""SI"", ""is_mobile"": true}" 2519,2,55,2017-05-26 19:36:18,http://shanahan.name/oleta.streich,0.5992936409,13.240.44.214,"{""location"": ""BT"", ""is_mobile"": true}" 9399,4,211,2017-06-11 18:32:24,http://christiansenpfeffer.org/kaylin_haag,0.7329149630,150.102.172.170,"{""location"": ""NF"", ""is_mobile"": true}" 9400,4,211,2017-06-12 17:55:25,http://marquardt.com/isabelle,0.2733610399,66.227.9.196,"{""location"": ""NF"", ""is_mobile"": true}" 9401,4,211,2017-06-08 16:24:11,http://haag.io/jeramie.ward,0.0692280223,95.215.140.202,"{""location"": ""KP"", ""is_mobile"": true}" 9402,4,211,2017-04-02 15:24:35,http://damore.net/herman_champlin,0.4622915705,134.156.118.183,"{""location"": ""GG"", ""is_mobile"": true}" 9403,4,211,2017-01-17 01:21:06,http://leuschkeswaniawski.info/bernardo,0.6154881653,12.51.42.24,"{""location"": ""PW"", ""is_mobile"": false}" 9404,4,211,2017-01-02 21:58:18,http://reichertcormier.info/isabelle_ferry,0.2269357814,218.213.15.188,"{""location"": ""IL"", ""is_mobile"": false}" 9405,4,211,2017-01-08 12:32:40,http://jacobs.name/giuseppe,0.4705715981,180.39.210.35,"{""location"": ""LI"", ""is_mobile"": true}" 9406,4,211,2017-06-10 18:31:53,http://stanton.info/caterina.pagac,0.6715041217,59.199.36.112,"{""location"": ""SL"", ""is_mobile"": true}" 9407,4,211,2017-01-31 17:27:42,http://hahn.info/kurtis_osinski,0.7132647084,12.149.25.183,"{""location"": ""BS"", ""is_mobile"": false}" 9408,4,211,2017-03-31 02:42:59,http://konopelski.com/herbert,0.2071476949,9.65.116.125,"{""location"": ""SH"", ""is_mobile"": false}" 9409,4,211,2017-05-10 03:32:14,http://mclaughlin.org/mariah,0.1170145298,115.167.202.165,"{""location"": ""BN"", ""is_mobile"": false}" 9410,4,211,2017-01-28 05:26:24,http://marquardtkozey.io/douglas_spencer,0.5444746912,225.27.38.197,"{""location"": ""ZM"", ""is_mobile"": true}" 9411,4,211,2017-03-15 13:53:03,http://greenholt.biz/garry,0.5884721635,198.60.237.130,"{""location"": ""VA"", ""is_mobile"": false}" 9412,4,211,2017-05-02 18:16:04,http://hegmann.name/henry.doyle,0.6682490290,125.98.105.192,"{""location"": ""MF"", ""is_mobile"": true}" 9413,4,211,2017-06-14 03:11:40,http://parisian.io/connie,0.7446083749,249.25.118.174,"{""location"": ""CM"", ""is_mobile"": true}" 9414,4,211,2017-01-01 10:54:18,http://lockmannitzsche.org/tina_romaguera,0.0208218114,250.12.230.165,"{""location"": ""GQ"", ""is_mobile"": true}" 9415,4,211,2017-03-20 15:24:34,http://lemke.org/shanelle_braun,0.7212142886,225.195.28.138,"{""location"": ""KW"", ""is_mobile"": true}" 9416,4,211,2017-03-06 15:18:45,http://jakubowski.com/jaleel,0.9687462391,31.140.119.12,"{""location"": ""GF"", ""is_mobile"": false}" 9417,4,211,2017-05-05 03:17:16,http://klingreilly.biz/lauriane.legros,0.7197915302,218.49.172.193,"{""location"": ""CG"", ""is_mobile"": false}" 9418,4,211,2017-05-23 12:28:52,http://mayer.org/junior.farrell,0.2983501204,130.74.195.217,"{""location"": ""TO"", ""is_mobile"": false}" 9419,4,211,2017-01-05 18:38:34,http://hanecollier.com/sigmund,0.7322762465,56.151.236.17,"{""location"": ""BS"", ""is_mobile"": true}" 9420,4,211,2017-05-30 22:33:19,http://rodriguez.co/arno.pouros,0.0204592383,173.60.76.34,"{""location"": ""VG"", ""is_mobile"": false}" 9421,4,211,2017-01-02 16:35:17,http://nolan.co/sister_jacobson,0.1546756897,109.121.228.164,"{""location"": ""HT"", ""is_mobile"": true}" 9422,4,211,2017-02-12 20:20:57,http://rutherford.biz/lloyd,0.9095044179,193.125.41.234,"{""location"": ""BF"", ""is_mobile"": false}" 9423,4,211,2017-03-17 21:03:49,http://kerluke.org/lawrence,0.2883967101,216.236.114.141,"{""location"": ""VA"", ""is_mobile"": true}" 9424,4,211,2017-03-21 16:13:52,http://reinger.net/laron,0.9805945881,214.191.146.200,"{""location"": ""MW"", ""is_mobile"": true}" 9425,4,212,2017-01-13 16:51:26,http://klein.biz/cornelius.mcdermott,,103.66.254.112,"{""location"": ""AL"", ""is_mobile"": false}" 9426,4,212,2017-03-22 18:24:38,http://kshlerin.net/verdie,,135.8.87.8,"{""location"": ""EG"", ""is_mobile"": false}" 9427,4,212,2017-01-12 17:03:07,http://franecki.biz/dashawn,,122.170.79.8,"{""location"": ""PM"", ""is_mobile"": true}" 9428,4,212,2017-01-24 10:02:50,http://kautzerosinski.com/nels,,117.11.177.112,"{""location"": ""GT"", ""is_mobile"": false}" 9429,4,212,2017-06-12 19:09:09,http://mohrmetz.org/korbin.swaniawski,,84.69.102.32,"{""location"": ""CR"", ""is_mobile"": true}" 9430,4,212,2017-03-28 00:49:03,http://trompbode.net/thora,,199.103.143.101,"{""location"": ""MK"", ""is_mobile"": true}" 9431,4,212,2017-06-07 06:51:45,http://davis.biz/marlin,,251.199.32.241,"{""location"": ""CF"", ""is_mobile"": false}" 9432,4,212,2017-05-25 22:09:37,http://goldneryost.org/jedediah.erdman,,50.198.119.113,"{""location"": ""TM"", ""is_mobile"": false}" 9433,4,212,2017-01-10 05:05:27,http://nitzsche.info/lora.stracke,,97.38.166.248,"{""location"": ""RU"", ""is_mobile"": false}" 9434,4,212,2017-04-11 20:23:02,http://bartonschulist.com/ralph_anderson,,85.84.254.247,"{""location"": ""TC"", ""is_mobile"": true}" 9435,4,212,2017-03-31 12:50:20,http://jacobi.org/jailyn_casper,,159.164.199.157,"{""location"": ""UY"", ""is_mobile"": false}" 9436,4,212,2017-04-18 16:35:33,http://hudsonconsidine.info/adeline,,124.175.214.104,"{""location"": ""FM"", ""is_mobile"": false}" 9437,4,212,2016-12-13 11:31:40,http://witting.io/sanford_huels,,16.84.207.169,"{""location"": ""ME"", ""is_mobile"": false}" 9438,4,212,2017-02-26 16:16:52,http://kautzerhilll.info/mona_bins,,142.133.25.149,"{""location"": ""GH"", ""is_mobile"": true}" 9439,4,212,2017-04-18 03:27:09,http://skilesquigley.info/kale,,41.123.124.18,"{""location"": ""GE"", ""is_mobile"": true}" 9440,4,212,2017-05-29 14:08:00,http://schowalter.biz/chad.braun,,115.3.222.2,"{""location"": ""ET"", ""is_mobile"": false}" 9441,4,212,2017-02-22 02:43:39,http://mullermiller.com/jarrett_wiza,,168.110.80.196,"{""location"": ""LB"", ""is_mobile"": false}" 9442,4,212,2017-04-16 20:35:17,http://tillman.org/rahsaan,,127.31.200.157,"{""location"": ""ST"", ""is_mobile"": false}" 9443,4,212,2017-04-11 08:48:11,http://moriette.org/obie.altenwerth,,88.117.235.163,"{""location"": ""CC"", ""is_mobile"": true}" 9444,4,212,2017-03-06 07:28:00,http://oreilly.name/liam_zboncak,,89.111.166.114,"{""location"": ""VC"", ""is_mobile"": true}" 9445,4,212,2017-04-02 00:40:17,http://adamchmidt.info/danyka,,254.87.210.193,"{""location"": ""TN"", ""is_mobile"": false}" 9446,4,212,2017-01-07 11:51:48,http://shanahan.biz/tanner,,66.73.29.100,"{""location"": ""AG"", ""is_mobile"": false}" 9447,4,212,2017-02-05 02:19:37,http://kris.co/lucile,,153.2.254.61,"{""location"": ""KW"", ""is_mobile"": false}" 9448,4,212,2017-05-04 01:12:55,http://rodriguez.com/evan,,228.122.241.141,"{""location"": ""LB"", ""is_mobile"": false}" 9449,4,212,2017-06-10 23:15:22,http://durgan.io/nathanael.abernathy,,45.145.234.214,"{""location"": ""WF"", ""is_mobile"": true}" 9450,4,212,2017-01-15 08:50:36,http://medhurst.info/alene_emmerich,,18.8.251.184,"{""location"": ""FJ"", ""is_mobile"": true}" 9451,4,212,2017-05-08 06:12:34,http://pfefferpacocha.info/amara,,196.156.91.231,"{""location"": ""MH"", ""is_mobile"": true}" 6410,3,140,2017-02-07 18:52:26,http://mclaughlin.biz/shanon_pollich,0.6068526101,193.176.106.211,"{""location"": ""MY"", ""is_mobile"": false}" 6411,3,140,2017-04-06 11:17:03,http://fay.org/greyson,0.4311413296,234.141.113.132,"{""location"": ""PH"", ""is_mobile"": true}" 6412,3,140,2017-06-06 15:26:54,http://abshire.net/elliot,0.1775242058,110.241.101.65,"{""location"": ""BJ"", ""is_mobile"": false}" 6413,3,140,2017-04-17 13:46:11,http://torphyschaden.net/loyal,0.6346625091,236.20.167.68,"{""location"": ""GW"", ""is_mobile"": false}" 6414,3,140,2017-02-09 21:40:17,http://heaneybode.io/tremaine_herzog,0.4352150545,241.77.122.199,"{""location"": ""BO"", ""is_mobile"": false}" 6415,3,140,2017-01-08 07:17:11,http://wilkinson.co/milford,0.1880575295,240.235.134.138,"{""location"": ""MK"", ""is_mobile"": true}" 6416,3,140,2017-06-12 23:36:17,http://windler.org/drew_cummings,0.5167461771,147.76.247.214,"{""location"": ""ML"", ""is_mobile"": true}" 6417,3,140,2017-05-16 01:29:02,http://abshireyost.com/ken,0.3493025812,130.10.24.120,"{""location"": ""PT"", ""is_mobile"": false}" 6418,3,140,2017-04-07 06:46:44,http://satterfield.info/jana,0.6257156864,22.24.177.25,"{""location"": ""HN"", ""is_mobile"": true}" 6419,3,140,2017-03-07 14:24:36,http://bednar.com/jabari,0.5950558764,119.252.39.219,"{""location"": ""VN"", ""is_mobile"": true}" 6420,3,140,2017-05-10 21:36:21,http://weber.org/johnnie.mcglynn,0.9555044164,237.189.193.242,"{""location"": ""BZ"", ""is_mobile"": false}" 6421,3,140,2017-04-08 13:33:06,http://oconnellparker.info/roderick,0.0705178582,74.177.96.65,"{""location"": ""BR"", ""is_mobile"": true}" 6422,3,140,2017-03-02 04:45:42,http://barrowshirthe.info/cooper,0.1732032916,57.248.13.251,"{""location"": ""DE"", ""is_mobile"": true}" 6423,3,140,2017-03-18 09:53:22,http://oharasmith.net/fay,0.2861797084,90.134.130.96,"{""location"": ""AL"", ""is_mobile"": true}" 6424,3,140,2017-06-14 04:55:58,http://sipes.net/leif,0.4671150186,56.59.220.7,"{""location"": ""SJ"", ""is_mobile"": false}" 6425,3,140,2016-12-22 20:47:00,http://gorczany.info/francesca,0.5067994423,197.97.220.141,"{""location"": ""MW"", ""is_mobile"": false}" 6426,3,140,2017-06-05 14:42:45,http://nader.co/immanuel,0.2523675524,210.19.42.193,"{""location"": ""GH"", ""is_mobile"": true}" 6427,3,140,2016-12-20 19:33:18,http://conroyhahn.com/johann,0.0484292042,210.239.105.68,"{""location"": ""BF"", ""is_mobile"": false}" 6428,3,140,2017-03-14 16:41:05,http://cruickshank.io/kaandra_padberg,0.7218668952,254.180.106.158,"{""location"": ""CK"", ""is_mobile"": true}" 6429,3,140,2016-12-21 13:29:50,http://schmeler.net/jaquan,0.1601703674,120.84.176.39,"{""location"": ""PL"", ""is_mobile"": false}" 6430,3,140,2016-12-28 03:58:44,http://schmittzulauf.biz/myriam.lueilwitz,0.1908061177,46.197.65.48,"{""location"": ""SK"", ""is_mobile"": true}" 6431,3,140,2017-01-31 16:40:41,http://dare.name/dino.vonrueden,0.2058627319,102.251.52.3,"{""location"": ""ES"", ""is_mobile"": true}" 6432,3,140,2017-02-20 08:36:30,http://kubkuhn.name/norwood,0.7202211069,224.250.106.186,"{""location"": ""MZ"", ""is_mobile"": false}" 6433,3,140,2017-04-13 13:16:25,http://witting.name/justyn.mann,0.2010426384,90.59.213.244,"{""location"": ""JM"", ""is_mobile"": true}" 6434,3,140,2017-02-19 01:22:15,http://ziemann.info/travis,0.1295173813,172.126.212.23,"{""location"": ""CF"", ""is_mobile"": true}" 6435,3,140,2017-03-12 08:11:00,http://ebert.biz/ludie_cronin,0.1332296773,2.109.77.79,"{""location"": ""CM"", ""is_mobile"": true}" 6436,3,140,2017-03-04 19:28:01,http://gutmann.io/jordon.dickens,0.7937100156,128.241.168.21,"{""location"": ""CH"", ""is_mobile"": true}" 6437,3,140,2017-01-10 05:50:45,http://dietrich.com/rosella,0.9592443277,47.3.209.151,"{""location"": ""FI"", ""is_mobile"": true}" 6438,3,140,2016-12-28 10:29:46,http://kutchking.net/effie.bailey,0.7082703601,143.13.183.246,"{""location"": ""SB"", ""is_mobile"": true}" 6439,3,140,2017-05-14 22:36:11,http://corwinherman.name/minnie.cain,0.8023136747,88.157.112.226,"{""location"": ""MU"", ""is_mobile"": false}" 6440,3,141,2017-04-22 11:39:14,http://spinka.biz/nathanial.brown,0.7834279349,198.219.229.235,"{""location"": ""GG"", ""is_mobile"": false}" 6441,3,141,2017-04-06 23:16:45,http://kertzmann.biz/ericka,0.1179760095,182.8.123.112,"{""location"": ""RU"", ""is_mobile"": false}" 6442,3,141,2017-04-20 03:31:17,http://hermanglover.net/rico,0.5147940181,247.232.117.173,"{""location"": ""GN"", ""is_mobile"": false}" 6443,3,141,2016-12-14 10:46:52,http://beerwindler.io/micah,0.7163490741,160.182.6.66,"{""location"": ""BN"", ""is_mobile"": false}" 6444,3,141,2017-02-24 23:23:01,http://bogisichdonnelly.info/cornell,0.4190616994,73.72.251.19,"{""location"": ""RW"", ""is_mobile"": false}" 6445,3,141,2017-03-02 05:12:04,http://wintheiserdickinson.io/christy,0.8244214012,12.237.182.244,"{""location"": ""CY"", ""is_mobile"": true}" 6446,3,141,2017-01-12 23:59:24,http://moriette.info/jeyca,0.9629920400,22.33.159.96,"{""location"": ""MS"", ""is_mobile"": false}" 6447,3,141,2017-03-24 05:05:43,http://cronin.io/weldon_hamill,0.1303221086,240.27.12.111,"{""location"": ""DM"", ""is_mobile"": false}" 6448,3,141,2016-12-19 17:39:11,http://lynch.name/kareem.abbott,0.7937626992,85.59.183.164,"{""location"": ""KE"", ""is_mobile"": true}" 6449,3,141,2017-03-11 02:01:15,http://kertzmannlegros.name/lukas.gerhold,0.4414455521,156.69.164.236,"{""location"": ""VA"", ""is_mobile"": false}" 6450,3,141,2017-05-01 05:47:27,http://luettgenbernhard.biz/lori,0.2162075829,85.115.106.71,"{""location"": ""CN"", ""is_mobile"": false}" 6451,3,141,2017-01-18 04:59:59,http://heaneyjaskolski.net/alba_fahey,0.4086008375,137.217.5.121,"{""location"": ""GT"", ""is_mobile"": true}" 6452,3,141,2016-12-28 01:36:24,http://hintzondricka.org/elroy,0.4204390014,16.53.30.99,"{""location"": ""MP"", ""is_mobile"": false}" 6453,3,141,2017-05-03 03:58:19,http://zieme.net/tyler,0.4510168509,165.195.138.120,"{""location"": ""VA"", ""is_mobile"": false}" 6454,3,141,2017-04-17 05:05:29,http://ledner.co/leonard_terry,0.5904854657,11.172.201.132,"{""location"": ""IN"", ""is_mobile"": false}" 6455,3,141,2017-04-23 01:56:39,http://weimann.org/dashawn,0.2565429494,230.231.166.186,"{""location"": ""CH"", ""is_mobile"": true}" 6456,3,141,2017-05-16 12:09:30,http://harvey.biz/elbert,0.2563860021,112.106.39.108,"{""location"": ""KZ"", ""is_mobile"": true}" 6457,3,141,2017-05-10 09:34:06,http://cormier.biz/emmett_gislason,0.2863691887,158.147.149.125,"{""location"": ""RW"", ""is_mobile"": true}" 6458,3,141,2017-06-11 00:59:40,http://hegmannkiehn.net/ola,0.8514715575,100.184.180.4,"{""location"": ""CH"", ""is_mobile"": false}" 6459,3,141,2017-01-31 09:59:15,http://feeney.info/sheila.stroman,0.8037566920,23.197.86.155,"{""location"": ""LB"", ""is_mobile"": false}" 6460,3,141,2017-04-05 16:33:24,http://christiansen.org/rick,0.5610079912,112.134.55.242,"{""location"": ""CD"", ""is_mobile"": true}" 6461,3,141,2017-05-07 10:53:26,http://dickens.com/citlalli,0.4747050192,221.191.43.72,"{""location"": ""SS"", ""is_mobile"": false}" 12315,5,276,2017-04-04 05:17:39,http://thompson.net/luz,0.1245938421,211.230.108.182,"{""location"": ""MP"", ""is_mobile"": false}" 12316,5,276,2017-02-25 04:06:24,http://cronin.net/jamil,0.8229863779,143.125.56.99,"{""location"": ""BR"", ""is_mobile"": true}" 12317,5,276,2017-03-03 09:25:56,http://creminkling.co/krystal,0.0154400923,45.64.218.155,"{""location"": ""PN"", ""is_mobile"": true}" 12318,5,276,2016-12-14 18:04:55,http://willmsrunte.io/mable.kling,0.2340428431,179.86.60.63,"{""location"": ""TJ"", ""is_mobile"": true}" 12319,5,276,2017-05-07 01:38:33,http://turcottegorczany.co/megane_kirlin,0.6282171495,27.25.210.93,"{""location"": ""GP"", ""is_mobile"": false}" 12320,5,276,2017-05-29 13:03:59,http://pagac.io/ramiro.mueller,0.4997321873,56.30.17.27,"{""location"": ""DJ"", ""is_mobile"": false}" 12321,5,276,2017-03-10 20:49:15,http://ruecker.com/helmer_wunsch,0.9977485969,115.253.68.239,"{""location"": ""CW"", ""is_mobile"": false}" 12322,5,276,2017-03-31 11:26:57,http://lindgrenwest.info/wiley_wolf,0.6368795697,34.5.253.114,"{""location"": ""IQ"", ""is_mobile"": true}" 12323,5,276,2017-06-11 22:35:05,http://rice.name/elliott,0.9047116281,197.152.220.219,"{""location"": ""ME"", ""is_mobile"": true}" 12324,5,276,2017-02-10 17:21:42,http://wyman.biz/dustin.hansen,0.8834770029,142.15.167.57,"{""location"": ""ME"", ""is_mobile"": true}" 12325,5,276,2017-04-16 17:10:04,http://littleemard.org/humberto,0.7759275370,243.249.207.30,"{""location"": ""CY"", ""is_mobile"": false}" 12326,5,276,2017-06-10 11:44:20,http://gleason.name/alvena_volkman,0.1433168133,240.156.78.150,"{""location"": ""CG"", ""is_mobile"": false}" 12327,5,276,2017-03-24 05:39:07,http://friesengibson.name/garland,0.5872954834,180.170.236.243,"{""location"": ""IN"", ""is_mobile"": true}" 12328,5,276,2017-03-31 04:42:13,http://grimes.com/jermain,0.9996232106,169.63.238.123,"{""location"": ""SJ"", ""is_mobile"": true}" 12329,5,276,2017-04-26 03:14:02,http://goldner.name/sydnee_botsford,0.8585202820,174.95.6.228,"{""location"": ""MN"", ""is_mobile"": true}" 12330,5,276,2017-05-23 16:33:56,http://fritsch.io/floyd_fay,0.7111542601,80.74.8.143,"{""location"": ""SZ"", ""is_mobile"": false}" 12331,5,276,2017-05-15 05:21:18,http://funkaltenwerth.info/giovanny,0.9565265199,44.21.122.109,"{""location"": ""CN"", ""is_mobile"": true}" 12332,5,276,2017-01-27 15:03:14,http://ward.info/jena.bergnaum,0.6214597970,70.93.232.41,"{""location"": ""NA"", ""is_mobile"": false}" 12333,5,276,2017-02-13 23:47:14,http://boyerdach.net/kaylah_halvorson,0.3935029746,214.179.117.215,"{""location"": ""UZ"", ""is_mobile"": true}" 12334,5,276,2017-01-28 00:17:01,http://raynor.net/johann_kertzmann,0.3017178698,208.253.95.175,"{""location"": ""MW"", ""is_mobile"": true}" 12335,5,276,2017-01-09 11:46:19,http://kreigerhand.io/meda,0.7318144963,168.154.53.119,"{""location"": ""DK"", ""is_mobile"": false}" 12336,5,276,2017-01-16 09:21:16,http://goldner.info/damion,0.1878417901,184.16.41.44,"{""location"": ""MV"", ""is_mobile"": false}" 12337,5,276,2017-01-17 07:02:54,http://nicolas.name/leone,0.4983207905,83.8.126.207,"{""location"": ""BZ"", ""is_mobile"": true}" 12338,5,276,2017-05-11 22:54:06,http://dachgottlieb.info/bart,0.7617898608,228.187.33.173,"{""location"": ""GA"", ""is_mobile"": false}" 12339,5,276,2017-05-21 15:34:11,http://klockowehner.com/pierce,0.1420916514,65.250.233.20,"{""location"": ""GY"", ""is_mobile"": false}" 12340,5,276,2016-12-27 16:42:27,http://paucek.co/jee.ledner,0.0966197770,35.219.67.86,"{""location"": ""GW"", ""is_mobile"": true}" 12341,5,276,2017-01-28 23:44:36,http://loweschmitt.net/francesco,0.5049618842,121.174.27.166,"{""location"": ""ZW"", ""is_mobile"": false}" 12342,5,276,2017-05-16 20:38:43,http://kihnmckenzie.com/ari,0.0246283131,238.219.106.239,"{""location"": ""BQ"", ""is_mobile"": true}" 12343,5,276,2017-03-15 12:43:53,http://deckowframi.io/ismael,0.5620613241,239.39.141.104,"{""location"": ""ID"", ""is_mobile"": true}" 12344,5,276,2017-03-01 05:44:11,http://bailey.info/jeffery_mckenzie,0.8964999629,102.84.32.116,"{""location"": ""WF"", ""is_mobile"": true}" 12345,5,276,2017-04-26 15:37:30,http://bernhard.org/coby,0.8076975884,48.152.146.10,"{""location"": ""JM"", ""is_mobile"": false}" 12346,5,276,2016-12-21 17:22:20,http://ziemann.org/aileen_bailey,0.5939816179,113.138.196.82,"{""location"": ""AZ"", ""is_mobile"": true}" 12347,5,276,2017-01-21 01:13:58,http://beahanschuster.biz/karley,0.2953783356,141.7.176.167,"{""location"": ""IQ"", ""is_mobile"": false}" 12348,5,276,2016-12-16 12:15:30,http://pagacgrady.name/urban,0.8142533469,26.250.96.115,"{""location"": ""FI"", ""is_mobile"": true}" 12349,5,276,2017-02-23 23:07:39,http://whiteratke.name/antwon,0.7234452439,2.131.100.247,"{""location"": ""ID"", ""is_mobile"": true}" 12350,5,276,2016-12-23 06:35:53,http://johnson.info/marques_littel,0.8627129725,241.103.22.220,"{""location"": ""BG"", ""is_mobile"": true}" 12351,5,276,2017-02-27 14:30:39,http://rempelcrist.info/mauricio,0.3729186422,121.126.252.250,"{""location"": ""EH"", ""is_mobile"": false}" 12352,5,276,2016-12-30 16:40:40,http://weimann.com/jayden,0.8062340566,247.229.37.25,"{""location"": ""TR"", ""is_mobile"": false}" 12353,5,276,2017-05-26 21:44:59,http://weimann.biz/kolby_osinski,0.5062131192,72.47.128.72,"{""location"": ""HK"", ""is_mobile"": false}" 12354,5,276,2016-12-23 04:01:02,http://mcclure.co/ahmad,0.7069956173,199.30.74.218,"{""location"": ""TC"", ""is_mobile"": true}" 12355,5,276,2017-05-16 18:46:16,http://kozeylarkin.co/burnice_osinski,0.4435905820,18.25.44.27,"{""location"": ""DZ"", ""is_mobile"": true}" 12356,5,276,2017-03-10 02:14:11,http://feest.co/burley,0.3399967225,163.128.54.47,"{""location"": ""KI"", ""is_mobile"": true}" 12357,5,276,2017-05-09 11:38:19,http://balistreri.net/barney_keler,0.7357131973,216.15.108.19,"{""location"": ""SV"", ""is_mobile"": true}" 12358,5,276,2017-01-20 13:50:47,http://oreillyharris.info/veda.langosh,0.8694034082,150.168.122.206,"{""location"": ""JO"", ""is_mobile"": false}" 12359,5,276,2017-05-10 06:17:43,http://thompsonhills.biz/gianni,0.8675569459,19.64.167.76,"{""location"": ""LS"", ""is_mobile"": false}" 12360,5,276,2017-05-20 08:39:33,http://mcclure.biz/sibyl,0.1562443413,187.174.119.12,"{""location"": ""GH"", ""is_mobile"": true}" 12361,5,276,2017-01-16 18:50:17,http://lynch.com/cortney,0.1901213851,26.37.97.179,"{""location"": ""EE"", ""is_mobile"": true}" 12362,5,276,2017-05-05 04:34:35,http://schuppe.net/marley_baumbach,0.5467797189,110.34.122.187,"{""location"": ""FK"", ""is_mobile"": false}" 12363,5,276,2017-03-11 14:10:00,http://bogisich.io/addie,0.1624575752,204.53.17.190,"{""location"": ""MQ"", ""is_mobile"": false}" 12364,5,276,2017-04-03 09:50:53,http://mann.com/derek.abernathy,0.6455735569,121.129.174.75,"{""location"": ""MT"", ""is_mobile"": true}" 12365,5,276,2017-02-13 20:58:32,http://zemlakblick.com/mireya,0.0773085477,238.124.142.171,"{""location"": ""PL"", ""is_mobile"": false}" 12366,5,276,2016-12-22 18:11:26,http://rolfsonwitting.org/annabel,0.7385035228,99.133.27.73,"{""location"": ""GI"", ""is_mobile"": true}" 15329,6,344,2017-02-22 20:21:51,http://heathcotehilll.org/ellis,,132.131.124.229,"{""location"": ""BF"", ""is_mobile"": false}" 15330,6,344,2017-03-08 16:55:14,http://mertzvonrueden.com/tavares.lubowitz,,216.62.239.204,"{""location"": ""KE"", ""is_mobile"": true}" 15331,6,344,2017-02-06 11:43:23,http://grahamcole.io/suzanne,,135.20.14.63,"{""location"": ""CZ"", ""is_mobile"": true}" 15332,6,344,2017-05-27 12:33:17,http://franecki.co/yesenia,,213.225.97.172,"{""location"": ""AL"", ""is_mobile"": true}" 15333,6,345,2017-01-02 17:59:01,http://wilkinson.biz/valentine_hudson,0.5516121172,183.59.239.200,"{""location"": ""ST"", ""is_mobile"": true}" 15334,6,345,2017-02-09 10:25:31,http://hammesokuneva.co/ludwig_mills,0.0569908049,128.108.28.56,"{""location"": ""BL"", ""is_mobile"": true}" 15335,6,345,2016-12-24 14:55:05,http://koelpinspencer.name/cordell,0.7006015060,250.189.130.101,"{""location"": ""BD"", ""is_mobile"": true}" 15336,6,345,2017-05-21 17:42:59,http://pfannerstillwisozk.co/francesco_yost,0.8640271524,56.201.171.216,"{""location"": ""GD"", ""is_mobile"": false}" 15337,6,345,2017-01-23 14:07:53,http://brakus.info/danielle_huel,0.9618580883,46.95.94.127,"{""location"": ""AD"", ""is_mobile"": false}" 15338,6,345,2017-05-17 12:33:31,http://damorebreitenberg.io/arvid,0.8638478389,39.51.165.36,"{""location"": ""VE"", ""is_mobile"": true}" 15339,6,345,2017-06-05 09:14:38,http://kingking.biz/camden.thiel,0.2538111727,199.59.97.148,"{""location"": ""FK"", ""is_mobile"": true}" 15340,6,345,2017-04-04 04:53:28,http://lueilwitzwitting.name/cornelius.daugherty,0.0196035505,61.98.133.143,"{""location"": ""BO"", ""is_mobile"": false}" 15341,6,345,2017-02-02 21:45:44,http://turcotte.net/nettie,0.3667576158,228.97.82.50,"{""location"": ""CK"", ""is_mobile"": false}" 15342,6,345,2017-03-07 16:46:21,http://quigley.biz/maureen.herman,0.5492664271,114.237.191.214,"{""location"": ""KZ"", ""is_mobile"": false}" 15343,6,345,2017-02-03 08:11:21,http://parker.io/ona,0.7313932587,197.247.21.91,"{""location"": ""BQ"", ""is_mobile"": true}" 15344,6,345,2017-05-15 19:15:57,http://douglas.com/hillard_jones,0.1290171924,12.44.35.147,"{""location"": ""RO"", ""is_mobile"": true}" 15345,6,345,2017-03-13 00:46:29,http://hayesgreenholt.io/nathanial,0.6558660680,20.178.74.132,"{""location"": ""SM"", ""is_mobile"": true}" 15346,6,345,2017-04-18 01:41:05,http://schinnerklocko.org/judah.schuster,0.5318601960,3.26.125.153,"{""location"": ""UA"", ""is_mobile"": true}" 15347,6,345,2017-02-27 04:35:27,http://oberbrunnerhaley.co/june_powlowski,0.0912056261,122.50.144.169,"{""location"": ""IL"", ""is_mobile"": true}" 15348,6,345,2017-01-21 15:10:49,http://larkin.org/arielle.schroeder,0.2103725543,151.202.152.39,"{""location"": ""NF"", ""is_mobile"": false}" 15349,6,345,2017-03-10 09:59:44,http://millwift.biz/adrain,0.7126190731,151.252.121.87,"{""location"": ""LI"", ""is_mobile"": true}" 15350,6,345,2017-05-18 04:25:48,http://roberts.biz/maurine.wolff,0.5071947403,244.120.245.232,"{""location"": ""KM"", ""is_mobile"": true}" 15351,6,345,2017-05-30 15:12:07,http://leffler.co/rodolfo.goldner,0.1196561373,136.221.217.150,"{""location"": ""NL"", ""is_mobile"": true}" 15352,6,345,2017-05-05 19:27:50,http://hagenesbrekke.name/elwyn,0.2832094646,174.10.29.71,"{""location"": ""TJ"", ""is_mobile"": false}" 15353,6,345,2017-03-18 00:42:00,http://mclaughlingoldner.io/yvonne_jakubowski,0.7204790107,80.141.252.148,"{""location"": ""US"", ""is_mobile"": true}" 15354,6,345,2017-02-14 04:56:51,http://trantow.net/mafalda,0.8854696204,223.71.53.4,"{""location"": ""KH"", ""is_mobile"": true}" 15355,6,345,2016-12-20 07:13:02,http://dooley.info/trevion,0.9176201221,106.187.77.31,"{""location"": ""AF"", ""is_mobile"": true}" 15356,6,345,2017-01-07 10:40:35,http://sawayn.com/davin,0.0476147458,94.241.138.212,"{""location"": ""UZ"", ""is_mobile"": true}" 15357,6,345,2017-02-17 19:05:22,http://witting.info/charlene,0.6609751449,41.147.128.148,"{""location"": ""AM"", ""is_mobile"": true}" 15358,6,345,2017-03-29 17:00:55,http://trantow.net/sam.bernier,0.6469723282,241.50.165.194,"{""location"": ""SO"", ""is_mobile"": false}" 15359,6,345,2017-06-06 21:03:44,http://fadeljohns.com/marco,0.9677392148,196.208.143.198,"{""location"": ""LA"", ""is_mobile"": false}" 15360,6,345,2017-02-11 04:34:59,http://zboncakgleichner.io/obie.beier,0.9958544291,51.88.61.5,"{""location"": ""AL"", ""is_mobile"": true}" 15361,6,345,2017-02-21 02:00:32,http://leffler.biz/dax,0.8906229433,149.163.134.63,"{""location"": ""PA"", ""is_mobile"": false}" 15362,6,345,2017-05-31 14:23:06,http://gleichner.org/dillon,0.8558024150,20.105.72.170,"{""location"": ""CG"", ""is_mobile"": false}" 15363,6,345,2017-05-01 07:15:03,http://bartell.name/linnea_sawayn,0.9206186141,87.53.138.68,"{""location"": ""ES"", ""is_mobile"": false}" 15364,6,345,2017-04-06 02:58:13,http://jenkinsnolan.co/vada.anderson,0.5602940669,237.241.60.201,"{""location"": ""PN"", ""is_mobile"": true}" 15365,6,345,2017-05-02 11:32:00,http://framigraham.biz/ezekiel,0.4765118850,148.24.41.228,"{""location"": ""YT"", ""is_mobile"": true}" 15366,6,345,2017-04-17 00:59:37,http://homenick.name/marilou,0.4158873247,152.206.11.77,"{""location"": ""VN"", ""is_mobile"": false}" 15367,6,345,2017-03-22 05:37:10,http://daughertycruickshank.name/abagail_willms,0.8896438446,3.240.136.72,"{""location"": ""CO"", ""is_mobile"": false}" 15368,6,345,2016-12-23 20:51:11,http://kihn.name/adan,0.4425875974,13.46.244.12,"{""location"": ""SZ"", ""is_mobile"": true}" 15369,6,345,2017-03-16 07:57:28,http://gulgowski.co/fred.schinner,0.0440594323,173.24.86.224,"{""location"": ""MR"", ""is_mobile"": false}" 15370,6,345,2017-02-06 03:42:48,http://rowe.com/emerald.zieme,0.7183463469,232.20.155.70,"{""location"": ""ML"", ""is_mobile"": true}" 15371,6,345,2016-12-30 03:09:34,http://cain.info/meggie_greenfelder,0.9139643588,59.107.129.98,"{""location"": ""MW"", ""is_mobile"": true}" 15372,6,345,2016-12-27 05:33:02,http://reilly.co/maurine,0.4107003480,62.54.75.185,"{""location"": ""GQ"", ""is_mobile"": true}" 15373,6,346,2017-03-11 02:15:53,http://cruickshank.org/leda.thiel,0.7479382715,6.172.13.127,"{""location"": ""SS"", ""is_mobile"": false}" 15374,6,346,2017-05-28 07:53:33,http://bartoletti.net/lysanne_gusikowski,0.1830214792,253.169.108.223,"{""location"": ""BH"", ""is_mobile"": false}" 15375,6,346,2017-05-25 14:40:34,http://block.co/garrison,0.1138116066,221.19.95.227,"{""location"": ""PL"", ""is_mobile"": true}" 15376,6,346,2017-05-17 06:54:19,http://pacocharempel.net/kali,0.0526155784,82.76.165.194,"{""location"": ""AQ"", ""is_mobile"": true}" 15377,6,346,2017-05-07 17:05:51,http://davis.com/geraldine,0.3205749386,26.250.100.216,"{""location"": ""GL"", ""is_mobile"": false}" 15378,6,346,2017-03-08 13:24:23,http://batz.co/kraig,0.2796871906,20.86.145.50,"{""location"": ""BV"", ""is_mobile"": true}" 15379,6,346,2017-01-21 08:44:19,http://stiedemann.net/nigel.little,0.5489254835,80.112.160.98,"{""location"": ""MO"", ""is_mobile"": false}" 2520,2,55,2017-03-04 19:46:05,http://carter.name/sarina_zboncak,0.2406291522,17.106.125.161,"{""location"": ""BG"", ""is_mobile"": true}" 2521,2,56,2017-05-21 00:00:04,http://fay.io/millie_franecki,0.5040639485,148.94.90.128,"{""location"": ""JE"", ""is_mobile"": true}" 2522,2,56,2017-01-18 10:58:28,http://leannon.org/mariam_bednar,0.1610338434,236.33.10.218,"{""location"": ""AG"", ""is_mobile"": true}" 2523,2,56,2017-01-06 21:03:14,http://leuschke.org/tillman,0.5081438527,5.145.156.239,"{""location"": ""PT"", ""is_mobile"": true}" 2524,2,56,2017-04-03 01:09:50,http://senger.name/gennaro_williamson,0.8301891924,15.110.67.28,"{""location"": ""YE"", ""is_mobile"": false}" 2525,2,56,2017-01-23 09:09:24,http://schinner.name/diamond,0.7062934890,225.249.29.250,"{""location"": ""AL"", ""is_mobile"": false}" 2526,2,56,2017-01-10 08:54:32,http://rowe.biz/parker,0.5997955306,92.60.162.210,"{""location"": ""TG"", ""is_mobile"": false}" 2527,2,56,2017-04-30 20:44:03,http://langworth.org/sallie,0.9525906620,49.210.91.197,"{""location"": ""UZ"", ""is_mobile"": true}" 2528,2,56,2016-12-25 12:50:17,http://hansen.com/marcia.willms,0.0808373436,87.230.51.63,"{""location"": ""FO"", ""is_mobile"": false}" 2529,2,56,2017-02-26 05:48:43,http://spinkaratke.net/damon_armstrong,0.7515305557,145.216.246.154,"{""location"": ""NP"", ""is_mobile"": false}" 2530,2,56,2017-03-24 16:50:42,http://anderson.net/lora.kunze,0.5269732887,212.224.19.102,"{""location"": ""ZA"", ""is_mobile"": true}" 2531,2,56,2016-12-30 05:54:16,http://moriette.co/reagan,0.5713368906,112.166.165.120,"{""location"": ""QA"", ""is_mobile"": true}" 2532,2,56,2016-12-29 17:19:31,http://mitchell.co/sienna,0.6557021565,140.127.13.81,"{""location"": ""MF"", ""is_mobile"": true}" 2533,2,56,2017-05-05 14:52:05,http://wehner.net/bertha_goodwin,0.2720637580,204.81.240.251,"{""location"": ""TW"", ""is_mobile"": true}" 2534,2,56,2017-03-16 13:16:41,http://rath.com/rachael,0.9533892620,38.42.224.180,"{""location"": ""TH"", ""is_mobile"": true}" 2535,2,56,2017-01-25 06:50:16,http://ritchie.net/julian,0.5826187889,107.236.142.203,"{""location"": ""PL"", ""is_mobile"": false}" 2536,2,56,2016-12-25 20:24:19,http://bauch.info/antwan,0.0924447667,204.183.167.167,"{""location"": ""CK"", ""is_mobile"": false}" 2537,2,56,2017-01-15 20:47:44,http://hackett.org/celestino_bosco,0.3328585183,97.92.146.190,"{""location"": ""LB"", ""is_mobile"": false}" 2538,2,56,2016-12-29 03:24:27,http://ohara.com/fernando_bogisich,0.2182006607,27.151.60.110,"{""location"": ""BQ"", ""is_mobile"": true}" 2539,2,56,2017-02-10 03:09:17,http://moencrooks.biz/eliza.flatley,0.5927647505,87.176.125.140,"{""location"": ""TZ"", ""is_mobile"": true}" 2540,2,56,2017-02-14 02:06:18,http://block.org/delia.lakin,0.3161621762,234.114.161.249,"{""location"": ""SS"", ""is_mobile"": false}" 2541,2,56,2017-03-24 20:06:27,http://medhurst.com/guadalupe,0.6531447096,209.205.172.191,"{""location"": ""PT"", ""is_mobile"": false}" 2542,2,56,2017-02-26 17:45:33,http://walsh.com/sam_reichel,0.5056337570,108.25.72.163,"{""location"": ""KY"", ""is_mobile"": true}" 2543,2,56,2017-03-12 10:19:41,http://leffler.name/adeline_feeney,0.5546199902,169.163.4.89,"{""location"": ""BL"", ""is_mobile"": true}" 2544,2,56,2017-02-23 01:24:32,http://pollich.biz/thelma_stoltenberg,0.8758554759,250.74.227.36,"{""location"": ""RU"", ""is_mobile"": false}" 2545,2,57,2017-01-12 00:03:25,http://bashirian.biz/kane_macejkovic,0.6843203102,136.138.120.247,"{""location"": ""TK"", ""is_mobile"": true}" 2546,2,57,2017-04-26 17:41:19,http://halvorson.biz/alisha_waelchi,0.2114965996,187.85.2.166,"{""location"": ""ME"", ""is_mobile"": false}" 2547,2,57,2017-03-01 08:33:46,http://corwin.org/elvie,0.0119174423,124.202.6.207,"{""location"": ""GY"", ""is_mobile"": false}" 2548,2,57,2017-02-04 04:28:00,http://rowepurdy.org/shaina,0.5256406987,236.119.141.212,"{""location"": ""KR"", ""is_mobile"": false}" 2549,2,57,2016-12-26 03:08:50,http://bradtke.name/hudson_mcclure,0.7114100948,220.7.222.82,"{""location"": ""CC"", ""is_mobile"": true}" 2550,2,57,2017-04-12 03:26:03,http://wintheisermorar.net/fatima,0.1141829077,71.206.157.248,"{""location"": ""ER"", ""is_mobile"": true}" 2551,2,57,2017-05-10 10:42:11,http://kreigerschmeler.net/ava.langworth,0.6146830864,12.6.165.74,"{""location"": ""SH"", ""is_mobile"": false}" 2552,2,57,2017-04-29 12:50:27,http://bauch.com/jeika.pouros,0.7990823296,219.206.157.41,"{""location"": ""CL"", ""is_mobile"": true}" 2553,2,57,2017-02-20 02:14:32,http://windlerhettinger.biz/stevie_mueller,0.8656687665,106.121.171.40,"{""location"": ""CF"", ""is_mobile"": true}" 2554,2,57,2017-06-03 04:26:13,http://blick.net/donnie_baumbach,0.7949682684,212.187.163.5,"{""location"": ""SJ"", ""is_mobile"": true}" 2555,2,57,2017-02-19 02:25:12,http://pourosbauch.com/cayla,0.3666714402,178.183.84.241,"{""location"": ""MC"", ""is_mobile"": false}" 2556,2,57,2017-06-06 13:33:58,http://trantow.co/uriah,0.9949590361,93.221.169.234,"{""location"": ""PE"", ""is_mobile"": true}" 2557,2,57,2017-03-15 04:20:55,http://baumbach.biz/minnie.kozey,0.6661542148,165.92.188.6,"{""location"": ""AQ"", ""is_mobile"": false}" 2558,2,57,2017-05-02 18:56:49,http://hagenesbernhard.name/karianne,0.0095368078,251.149.181.216,"{""location"": ""MW"", ""is_mobile"": false}" 2559,2,57,2017-04-08 12:25:04,http://adams.io/justice,0.3288718439,220.247.64.234,"{""location"": ""BG"", ""is_mobile"": false}" 2560,2,57,2017-02-11 13:14:06,http://senger.name/major_jones,0.9550624501,5.86.125.201,"{""location"": ""MX"", ""is_mobile"": true}" 2561,2,57,2017-06-08 04:00:29,http://willms.name/magnolia_bartell,0.3234512572,32.190.229.192,"{""location"": ""NL"", ""is_mobile"": true}" 2562,2,57,2017-04-07 10:33:08,http://beahan.com/marisol,0.4165447750,140.199.32.176,"{""location"": ""MA"", ""is_mobile"": false}" 2563,2,57,2017-05-02 02:07:33,http://armstrong.net/peggie,0.8441234924,56.35.72.163,"{""location"": ""HK"", ""is_mobile"": false}" 2564,2,57,2017-04-29 11:45:44,http://altenwerth.name/rashad,0.2448623564,117.128.207.122,"{""location"": ""CH"", ""is_mobile"": true}" 2565,2,57,2017-02-11 13:49:13,http://greenholt.name/pinkie.kiehn,0.5522003463,117.167.189.119,"{""location"": ""PF"", ""is_mobile"": false}" 2566,2,57,2017-04-23 02:42:54,http://johnson.co/tyler.cormier,0.4777561568,65.229.169.90,"{""location"": ""EE"", ""is_mobile"": false}" 2567,2,57,2017-03-30 07:07:25,http://watsicaconsidine.name/breana,0.5501082580,12.167.241.67,"{""location"": ""SE"", ""is_mobile"": false}" 2568,2,57,2017-01-26 16:16:54,http://bartell.com/omari.langworth,0.2932564554,125.53.222.178,"{""location"": ""NE"", ""is_mobile"": true}" 2569,2,57,2017-05-19 12:01:05,http://erdman.io/efrain,0.1168755859,206.68.55.89,"{""location"": ""AE"", ""is_mobile"": true}" 2570,2,57,2017-03-07 12:46:26,http://bayerhuels.net/scarlett,0.9181115474,143.176.3.11,"{""location"": ""BW"", ""is_mobile"": false}" 2571,2,57,2016-12-14 03:14:03,http://streich.net/jan_frami,0.7667705961,16.233.198.180,"{""location"": ""GN"", ""is_mobile"": false}" 9452,4,212,2017-02-26 07:22:21,http://emmerichflatley.info/sallie,,41.31.155.236,"{""location"": ""CV"", ""is_mobile"": false}" 9453,4,212,2017-04-08 04:10:18,http://langworthstroman.org/lavonne.beier,,210.98.27.97,"{""location"": ""RS"", ""is_mobile"": false}" 9454,4,212,2016-12-28 02:13:50,http://mclaughlin.info/eladio_dietrich,,219.216.26.193,"{""location"": ""CG"", ""is_mobile"": false}" 9455,4,212,2017-06-04 10:42:17,http://beerhuels.name/colten,,8.6.85.145,"{""location"": ""QA"", ""is_mobile"": false}" 9456,4,212,2017-05-01 10:23:55,http://rosenbaum.com/mercedes.gerlach,,8.18.230.45,"{""location"": ""MV"", ""is_mobile"": false}" 9457,4,212,2017-04-11 11:57:32,http://willms.com/winona.parker,,33.142.251.111,"{""location"": ""CW"", ""is_mobile"": false}" 9458,4,212,2017-02-24 00:00:10,http://parisian.biz/deondre_klein,,248.90.202.29,"{""location"": ""SV"", ""is_mobile"": false}" 9459,4,212,2017-02-11 10:41:53,http://marquardt.net/betty_bins,,242.47.112.130,"{""location"": ""BR"", ""is_mobile"": false}" 9460,4,212,2017-06-11 14:44:20,http://macgyveranderson.com/deven.ziemann,,230.168.186.38,"{""location"": ""GB"", ""is_mobile"": false}" 9461,4,212,2017-04-09 11:23:20,http://townestreich.biz/raul,,49.110.34.116,"{""location"": ""PS"", ""is_mobile"": true}" 9462,4,212,2016-12-25 00:09:23,http://weinatbins.net/marjolaine.barrows,,154.161.242.65,"{""location"": ""AT"", ""is_mobile"": true}" 9463,4,212,2017-02-09 00:58:02,http://paucekkovacek.biz/arlo.brekke,,197.236.77.212,"{""location"": ""CF"", ""is_mobile"": false}" 9464,4,212,2017-02-13 19:21:32,http://cartwrighthudson.io/scot,,155.209.48.171,"{""location"": ""GD"", ""is_mobile"": true}" 9465,4,212,2017-04-12 02:37:42,http://swift.co/marcelino,,127.144.83.222,"{""location"": ""IN"", ""is_mobile"": true}" 9466,4,212,2017-01-09 05:25:55,http://price.co/annette_kertzmann,,248.82.12.48,"{""location"": ""VG"", ""is_mobile"": true}" 9467,4,212,2017-06-09 08:12:31,http://harvey.io/skyla,,243.209.69.128,"{""location"": ""NG"", ""is_mobile"": true}" 9468,4,212,2017-05-27 04:30:45,http://ondrickaleannon.com/jalen_torphy,,36.48.213.180,"{""location"": ""TK"", ""is_mobile"": true}" 9469,4,212,2017-01-13 09:56:27,http://hudson.org/ardella.hintz,,87.132.130.115,"{""location"": ""FI"", ""is_mobile"": false}" 9470,4,212,2017-03-29 21:28:42,http://pouros.org/constantin,,178.157.33.253,"{""location"": ""BJ"", ""is_mobile"": true}" 9471,4,212,2017-02-25 00:26:36,http://parkerprohaska.info/jasmin_oreilly,,106.200.221.74,"{""location"": ""LI"", ""is_mobile"": false}" 9472,4,212,2017-05-23 17:18:14,http://witting.io/avis,,67.70.216.64,"{""location"": ""VI"", ""is_mobile"": true}" 9473,4,212,2017-01-25 00:49:23,http://conroy.org/osbaldo.waters,,216.201.24.159,"{""location"": ""AD"", ""is_mobile"": false}" 9474,4,212,2017-06-11 23:15:08,http://stark.org/ariane.robel,,209.93.33.141,"{""location"": ""NP"", ""is_mobile"": false}" 9475,4,212,2017-05-15 15:51:56,http://daugherty.net/antoinette,,41.49.196.32,"{""location"": ""IM"", ""is_mobile"": true}" 9476,4,212,2017-05-25 17:52:29,http://larkin.net/elouise.carroll,,215.112.38.182,"{""location"": ""QA"", ""is_mobile"": true}" 9477,4,212,2017-02-12 03:10:23,http://kreigerwalsh.com/gladys,,198.119.95.2,"{""location"": ""CU"", ""is_mobile"": true}" 9478,4,212,2017-01-12 23:43:33,http://hilpertdamore.com/violet,,89.16.184.158,"{""location"": ""SL"", ""is_mobile"": true}" 9479,4,212,2016-12-23 05:21:38,http://kerlukebartell.com/jerry,,204.162.126.26,"{""location"": ""BV"", ""is_mobile"": true}" 9480,4,212,2017-02-04 03:06:15,http://anderson.org/damian,,222.198.143.105,"{""location"": ""AU"", ""is_mobile"": false}" 9481,4,212,2017-01-14 08:02:19,http://schumm.com/makenna.feil,,183.185.199.193,"{""location"": ""SJ"", ""is_mobile"": true}" 9482,4,212,2017-03-30 19:28:18,http://koepp.org/bianka.langworth,,110.36.180.213,"{""location"": ""FM"", ""is_mobile"": false}" 9483,4,212,2017-05-10 17:59:12,http://bahringer.co/erica_stracke,,217.211.219.9,"{""location"": ""VN"", ""is_mobile"": true}" 9484,4,212,2017-01-07 00:33:57,http://champlin.co/wava,,200.213.183.179,"{""location"": ""AW"", ""is_mobile"": true}" 9485,4,212,2017-05-01 20:48:51,http://donnelly.com/stanford_kreiger,,219.111.118.42,"{""location"": ""TZ"", ""is_mobile"": false}" 9486,4,213,2017-01-01 19:34:46,http://heidenreich.name/giovani,,6.84.139.104,"{""location"": ""PY"", ""is_mobile"": true}" 9487,4,213,2017-01-29 19:10:06,http://herman.info/rosemary.medhurst,,231.205.60.155,"{""location"": ""WF"", ""is_mobile"": true}" 9488,4,213,2017-06-07 14:46:48,http://zboncak.biz/jamaal,,37.139.160.114,"{""location"": ""PR"", ""is_mobile"": true}" 9489,4,213,2017-02-22 07:50:28,http://bauchjacobs.co/larue,,246.53.115.195,"{""location"": ""IL"", ""is_mobile"": true}" 9490,4,213,2016-12-18 23:49:38,http://turner.biz/jeffry.farrell,,6.179.198.119,"{""location"": ""DZ"", ""is_mobile"": true}" 9491,4,213,2017-05-08 02:09:32,http://muellersawayn.biz/boris_ortiz,,172.10.131.15,"{""location"": ""US"", ""is_mobile"": true}" 9492,4,213,2017-04-11 17:49:55,http://aufderhar.name/tyrell_heller,,232.219.155.126,"{""location"": ""KZ"", ""is_mobile"": false}" 9493,4,213,2016-12-29 08:04:26,http://oberbrunnerschaden.info/marina_wuckert,,154.161.115.38,"{""location"": ""MS"", ""is_mobile"": false}" 9494,4,213,2017-05-18 13:52:44,http://windler.com/curt_gaylord,,9.175.66.252,"{""location"": ""SB"", ""is_mobile"": false}" 9495,4,213,2017-01-05 10:01:53,http://toy.biz/gudrun_streich,,58.178.48.181,"{""location"": ""CF"", ""is_mobile"": false}" 9496,4,213,2017-04-13 17:21:24,http://purdy.net/juwan.stehr,,149.206.236.119,"{""location"": ""SS"", ""is_mobile"": true}" 9497,4,213,2017-03-08 12:21:52,http://framigislason.com/chance.greenholt,,54.127.147.84,"{""location"": ""KI"", ""is_mobile"": true}" 9498,4,213,2017-03-11 19:32:29,http://schinner.biz/willis.reichel,,20.32.216.62,"{""location"": ""RE"", ""is_mobile"": true}" 9499,4,213,2017-01-18 20:49:41,http://mcglynn.com/deontae_collins,,202.11.115.198,"{""location"": ""BV"", ""is_mobile"": false}" 9500,4,213,2017-02-19 00:13:57,http://lebsackluettgen.com/makenzie.gottlieb,,161.51.16.46,"{""location"": ""TF"", ""is_mobile"": false}" 9501,4,213,2017-03-17 06:39:05,http://cormier.com/orie,,35.195.203.140,"{""location"": ""FR"", ""is_mobile"": true}" 9502,4,213,2017-01-16 04:05:11,http://feeneybreitenberg.com/cletus.boyer,,244.17.163.200,"{""location"": ""HM"", ""is_mobile"": true}" 9503,4,213,2017-03-29 10:54:12,http://fadelankunding.org/myrtie.sanford,,23.70.102.210,"{""location"": ""HN"", ""is_mobile"": false}" 9504,4,213,2017-06-02 00:20:17,http://borer.biz/austin_larson,,207.5.84.31,"{""location"": ""VA"", ""is_mobile"": true}" 9505,4,213,2017-01-15 20:51:31,http://hagenesrogahn.net/weldon,,196.177.194.63,"{""location"": ""BW"", ""is_mobile"": true}" 9506,4,213,2017-03-05 06:27:21,http://wyman.net/brian_oconner,,88.109.168.167,"{""location"": ""MX"", ""is_mobile"": true}" 6462,3,141,2017-03-09 12:47:52,http://rath.org/mitchell_macgyver,0.0095553997,188.90.94.100,"{""location"": ""GD"", ""is_mobile"": true}" 6463,3,141,2016-12-26 19:06:10,http://fisher.info/jane,0.3421866398,61.19.8.222,"{""location"": ""QA"", ""is_mobile"": true}" 6464,3,141,2017-04-01 16:18:55,http://eichmann.co/jace.wilkinson,0.0595269944,18.81.110.253,"{""location"": ""UA"", ""is_mobile"": true}" 6465,3,141,2017-04-15 18:03:50,http://gloverdickinson.com/myah,0.1733263447,13.190.57.32,"{""location"": ""GA"", ""is_mobile"": true}" 6466,3,141,2017-03-19 23:43:49,http://romaguera.biz/matilda,0.2075737801,21.52.81.158,"{""location"": ""GY"", ""is_mobile"": false}" 6467,3,141,2017-04-05 23:26:59,http://west.org/alva.langosh,0.8352090228,60.193.83.165,"{""location"": ""FJ"", ""is_mobile"": true}" 6468,3,141,2017-05-17 14:25:04,http://aufderhar.biz/chasity.hintz,0.4904130174,203.136.168.28,"{""location"": ""NC"", ""is_mobile"": false}" 6469,3,141,2017-04-30 11:40:50,http://langworth.io/lauryn_nikolaus,0.6055070704,122.236.149.196,"{""location"": ""GD"", ""is_mobile"": true}" 6470,3,141,2017-03-08 14:34:35,http://fritsch.info/celestine_johnston,0.3173265371,45.164.167.192,"{""location"": ""GF"", ""is_mobile"": true}" 6471,3,141,2016-12-13 16:49:41,http://kreigerfeeney.org/patrick,0.5738068434,186.244.68.68,"{""location"": ""SZ"", ""is_mobile"": false}" 6472,3,141,2017-02-17 16:36:33,http://mann.biz/violet,0.7873676103,106.95.98.186,"{""location"": ""GA"", ""is_mobile"": false}" 6473,3,141,2017-06-05 20:52:39,http://corwin.name/enos.reynolds,0.4819027215,56.247.139.240,"{""location"": ""UM"", ""is_mobile"": true}" 6474,3,141,2017-03-21 20:50:41,http://wisozkparker.co/lorena_lueilwitz,0.5594959506,4.82.252.5,"{""location"": ""DO"", ""is_mobile"": true}" 6475,3,141,2017-01-08 02:13:27,http://ondrickawolf.co/sigurd.collins,0.4235609464,211.248.201.229,"{""location"": ""RO"", ""is_mobile"": false}" 6476,3,141,2017-05-05 16:07:28,http://lockmanlegros.org/jeremy,0.3236024355,235.174.13.211,"{""location"": ""JO"", ""is_mobile"": false}" 6477,3,141,2017-02-28 07:19:50,http://rolfson.io/kennith,0.1691977830,230.162.153.96,"{""location"": ""SG"", ""is_mobile"": true}" 6478,3,141,2017-04-06 14:12:24,http://homenick.biz/lyda.grady,0.4023058753,172.241.137.191,"{""location"": ""MQ"", ""is_mobile"": true}" 6479,3,141,2017-04-16 23:03:05,http://adams.co/olin,0.8211132361,47.16.199.73,"{""location"": ""BL"", ""is_mobile"": false}" 6480,3,141,2017-03-01 11:21:53,http://adams.org/kitty_rohan,0.4220758821,4.13.2.19,"{""location"": ""PY"", ""is_mobile"": true}" 6481,3,141,2016-12-22 11:55:17,http://lind.com/rae_wehner,0.8677715908,177.147.69.205,"{""location"": ""GI"", ""is_mobile"": true}" 6482,3,141,2016-12-27 06:44:02,http://millsmcclure.info/zola_thiel,0.4847160222,248.180.108.194,"{""location"": ""VI"", ""is_mobile"": false}" 6483,3,141,2017-05-13 01:28:12,http://lang.name/joseph_hansen,0.6752448703,213.246.63.143,"{""location"": ""NP"", ""is_mobile"": false}" 6484,3,141,2017-02-27 02:51:17,http://considinespinka.com/maude.ullrich,0.1137719418,188.233.22.3,"{""location"": ""GU"", ""is_mobile"": false}" 6485,3,141,2017-02-15 22:16:34,http://mitchell.co/elta.bayer,0.2850086517,89.55.46.54,"{""location"": ""BL"", ""is_mobile"": true}" 6486,3,141,2017-05-13 14:04:30,http://hickledavis.info/hollis.dicki,0.2623858482,179.184.163.169,"{""location"": ""NC"", ""is_mobile"": false}" 6487,3,141,2017-03-25 09:58:49,http://herman.biz/brittany_muller,0.4291851335,247.227.168.35,"{""location"": ""ZA"", ""is_mobile"": false}" 6488,3,141,2017-02-05 15:30:00,http://darebode.org/keara.grimes,0.4131658600,110.252.160.126,"{""location"": ""YE"", ""is_mobile"": true}" 6489,3,141,2017-02-22 21:35:02,http://schowalter.info/conrad_lemke,0.7694219596,123.19.97.99,"{""location"": ""CL"", ""is_mobile"": true}" 6490,3,141,2017-02-15 21:24:50,http://predovic.co/bailey,0.2324889811,204.171.34.189,"{""location"": ""HT"", ""is_mobile"": true}" 6491,3,141,2017-05-12 15:31:30,http://vandervort.io/berry_deckow,0.1739193478,171.7.83.248,"{""location"": ""CZ"", ""is_mobile"": true}" 6492,3,141,2017-04-09 22:06:17,http://kirlinmohr.net/hettie_hagenes,0.9987492282,34.161.17.72,"{""location"": ""SS"", ""is_mobile"": false}" 6493,3,141,2017-04-27 07:55:01,http://murray.info/katrina.rosenbaum,0.8562649399,176.196.64.168,"{""location"": ""MU"", ""is_mobile"": false}" 6494,3,141,2017-05-18 09:43:34,http://dare.biz/mona,0.1221599869,194.60.109.242,"{""location"": ""KN"", ""is_mobile"": true}" 6495,3,141,2017-04-18 17:58:12,http://jenkins.name/finn_kreiger,0.4611103864,170.164.249.104,"{""location"": ""SD"", ""is_mobile"": true}" 6496,3,141,2017-06-08 12:34:15,http://botsfordquitzon.info/aglae.nikolaus,0.1212544032,148.242.141.89,"{""location"": ""SD"", ""is_mobile"": false}" 6497,3,141,2017-04-19 14:25:36,http://dickens.name/dedric_marks,0.4090285530,191.108.231.124,"{""location"": ""GI"", ""is_mobile"": false}" 6498,3,141,2017-06-05 19:36:01,http://jaskolskischiller.name/name_spencer,0.8432471125,233.230.95.45,"{""location"": ""AD"", ""is_mobile"": true}" 6499,3,141,2017-02-24 03:10:31,http://heaney.io/reva.kunde,0.3502500201,92.63.103.77,"{""location"": ""BN"", ""is_mobile"": false}" 6500,3,141,2017-02-22 00:52:54,http://shanahangislason.biz/nia_hintz,0.1567743189,23.167.199.102,"{""location"": ""KZ"", ""is_mobile"": true}" 6501,3,141,2017-02-13 02:15:51,http://heller.com/armando_quigley,0.6989783624,231.30.245.113,"{""location"": ""DJ"", ""is_mobile"": true}" 6502,3,141,2017-04-11 23:09:11,http://erdmanstark.org/pablo.ratke,0.1061977172,173.218.9.7,"{""location"": ""CK"", ""is_mobile"": true}" 6503,3,141,2017-04-30 04:27:44,http://lockmancormier.biz/clovis,0.8674163079,67.88.170.174,"{""location"": ""MN"", ""is_mobile"": true}" 6504,3,141,2017-05-31 10:11:49,http://vonaltenwerth.info/deja,0.5013648602,241.59.183.181,"{""location"": ""KP"", ""is_mobile"": false}" 6505,3,142,2016-12-15 23:23:24,http://sporer.biz/talon,0.0545928940,243.93.219.161,"{""location"": ""PK"", ""is_mobile"": true}" 6506,3,142,2017-01-23 23:02:01,http://yundt.co/kayla.schamberger,0.1530779717,242.58.242.14,"{""location"": ""NR"", ""is_mobile"": false}" 6507,3,142,2017-03-01 14:16:26,http://johns.io/jesus_wiegand,0.6581167082,13.144.217.154,"{""location"": ""GI"", ""is_mobile"": false}" 6508,3,142,2017-01-28 16:07:46,http://swift.name/adalberto.boyer,0.8407273584,153.91.130.68,"{""location"": ""MR"", ""is_mobile"": false}" 6509,3,142,2017-05-16 19:13:55,http://farrell.org/dayne,0.7228775913,126.165.160.59,"{""location"": ""ET"", ""is_mobile"": true}" 6510,3,142,2017-03-10 00:15:32,http://bayer.net/wilfredo,0.8568567877,117.250.177.204,"{""location"": ""GR"", ""is_mobile"": false}" 6511,3,142,2017-02-23 15:52:39,http://prosacco.co/hugh,0.4623412759,53.97.212.167,"{""location"": ""ID"", ""is_mobile"": false}" 6512,3,142,2017-02-22 05:34:06,http://smith.com/bernadine,0.0751614975,201.242.194.117,"{""location"": ""RU"", ""is_mobile"": false}" 12367,5,277,2017-03-14 10:23:09,http://deckow.info/heber_ward,0.6188413720,53.193.228.191,"{""location"": ""TM"", ""is_mobile"": true}" 12368,5,277,2016-12-13 16:44:22,http://goldner.net/stephany.cain,0.5797786736,206.29.80.132,"{""location"": ""MN"", ""is_mobile"": false}" 12369,5,277,2017-01-08 17:57:19,http://mueller.net/richmond_spinka,0.6344151885,62.55.72.138,"{""location"": ""GL"", ""is_mobile"": true}" 12370,5,277,2017-02-07 06:59:17,http://hillsgraham.com/jensen,0.8250679292,193.120.147.15,"{""location"": ""KI"", ""is_mobile"": false}" 12371,5,277,2017-04-15 23:29:53,http://gottlieb.biz/benedict_sawayn,0.2179542885,211.131.196.132,"{""location"": ""FR"", ""is_mobile"": true}" 12372,5,277,2017-04-02 16:33:07,http://cremincrooks.net/myrtis,0.3742482103,44.66.119.63,"{""location"": ""RU"", ""is_mobile"": true}" 12373,5,277,2017-01-14 14:07:15,http://hillsroob.net/whitney,0.2279878559,42.199.115.226,"{""location"": ""MN"", ""is_mobile"": true}" 12374,5,277,2017-05-11 05:40:42,http://hellerlabadie.co/tremaine_weimann,0.2458975885,79.193.229.149,"{""location"": ""TF"", ""is_mobile"": false}" 12375,5,277,2017-06-06 22:44:05,http://ruecker.biz/alvera.okon,0.2035548425,236.124.166.149,"{""location"": ""GG"", ""is_mobile"": false}" 12376,5,277,2017-03-31 22:57:04,http://ferry.name/trisha_runolfsdottir,0.1658654415,229.51.113.84,"{""location"": ""RS"", ""is_mobile"": true}" 12377,5,277,2017-04-08 01:55:56,http://murazik.net/carolyn,0.0817002070,146.125.220.155,"{""location"": ""EC"", ""is_mobile"": true}" 12378,5,277,2017-03-13 09:33:34,http://adams.info/lonnie_kuvalis,0.8794112754,242.19.25.219,"{""location"": ""MX"", ""is_mobile"": true}" 12379,5,277,2017-04-24 22:24:51,http://leuschke.info/vivienne_howe,0.0215698207,221.116.75.165,"{""location"": ""IT"", ""is_mobile"": false}" 12380,5,277,2017-04-28 03:26:51,http://wilderman.biz/georgiana_reynolds,0.4427542452,171.223.103.21,"{""location"": ""SX"", ""is_mobile"": false}" 12381,5,277,2017-05-01 10:20:55,http://marks.info/susana,0.5237403277,130.90.194.202,"{""location"": ""KW"", ""is_mobile"": false}" 12382,5,277,2017-01-22 15:33:14,http://torphy.org/dallin.lehner,0.5063891223,55.160.84.241,"{""location"": ""CG"", ""is_mobile"": true}" 12383,5,277,2017-04-26 11:43:07,http://wildermanschoen.biz/delta,0.1625101596,226.54.21.49,"{""location"": ""GF"", ""is_mobile"": true}" 12384,5,277,2017-05-17 12:39:30,http://bergstrom.name/angel,0.3329893455,206.14.196.180,"{""location"": ""CY"", ""is_mobile"": true}" 12385,5,277,2017-02-15 02:52:47,http://rogahnmarvin.io/sterling.block,0.7461423786,206.219.70.243,"{""location"": ""HN"", ""is_mobile"": true}" 12386,5,277,2017-03-05 12:18:01,http://boyermaggio.io/reyna_muller,0.6897209806,209.178.191.2,"{""location"": ""VU"", ""is_mobile"": false}" 12387,5,277,2017-03-12 09:08:54,http://boylebeatty.org/aliyah,0.0638395781,94.124.145.239,"{""location"": ""TK"", ""is_mobile"": true}" 12388,5,277,2017-02-25 06:18:51,http://wisoky.org/alverta,0.1786807373,48.54.39.178,"{""location"": ""MO"", ""is_mobile"": true}" 12389,5,277,2017-02-16 11:39:50,http://sawayn.org/jayden_heidenreich,0.1825154538,95.224.22.18,"{""location"": ""LA"", ""is_mobile"": false}" 12390,5,277,2017-01-24 10:12:44,http://gottliebzulauf.info/karli,0.4067333206,82.94.109.75,"{""location"": ""GR"", ""is_mobile"": false}" 12391,5,278,2017-06-01 06:45:18,http://durganleannon.org/casandra,0.6029347181,247.78.66.42,"{""location"": ""LS"", ""is_mobile"": true}" 12392,5,278,2017-02-06 00:32:17,http://pagacgreenholt.io/hilda_hermiston,0.8725936540,18.53.220.172,"{""location"": ""TR"", ""is_mobile"": false}" 12393,5,278,2017-01-01 18:10:58,http://jerdedoyle.net/iliana.homenick,0.3992647544,115.72.224.226,"{""location"": ""DE"", ""is_mobile"": false}" 12394,5,278,2017-03-18 20:04:40,http://harvey.co/zakary_bernier,0.2877629809,23.211.92.100,"{""location"": ""LC"", ""is_mobile"": true}" 12395,5,278,2017-01-24 22:21:30,http://gerholdturner.net/haven.nikolaus,0.9788648616,246.251.143.42,"{""location"": ""MM"", ""is_mobile"": false}" 12396,5,278,2016-12-22 00:44:19,http://ward.name/daisha.schultz,0.8518152583,146.10.183.220,"{""location"": ""ES"", ""is_mobile"": false}" 12397,5,278,2017-01-20 16:41:18,http://schaden.org/emmanuel_okuneva,0.8783758018,122.195.77.72,"{""location"": ""VG"", ""is_mobile"": true}" 12398,5,278,2017-05-20 13:22:39,http://paucek.biz/rae,0.0850013473,28.34.213.153,"{""location"": ""FK"", ""is_mobile"": true}" 12399,5,278,2017-03-06 15:31:35,http://vandervort.info/gerald,0.5727095515,182.205.196.250,"{""location"": ""NI"", ""is_mobile"": false}" 12400,5,278,2017-05-14 12:19:02,http://pagacdibbert.net/jamison,0.2397249686,62.248.48.67,"{""location"": ""MQ"", ""is_mobile"": false}" 12401,5,278,2017-03-30 16:19:36,http://conroykovacek.io/antwon,0.5661140597,49.213.170.169,"{""location"": ""YE"", ""is_mobile"": true}" 12402,5,278,2016-12-25 23:36:54,http://schmitt.co/ava,0.4114977835,81.251.23.109,"{""location"": ""MD"", ""is_mobile"": true}" 12403,5,278,2017-02-15 05:50:21,http://rodriguez.info/reed,0.6145236708,65.40.27.7,"{""location"": ""TK"", ""is_mobile"": true}" 12404,5,278,2017-05-20 08:30:22,http://stokes.io/claudine_blick,0.4919722412,175.159.169.159,"{""location"": ""CU"", ""is_mobile"": false}" 12405,5,278,2016-12-19 21:15:36,http://wolf.io/daniela.hoeger,0.9437493702,109.109.135.182,"{""location"": ""CA"", ""is_mobile"": true}" 12406,5,278,2017-03-10 18:51:21,http://collier.biz/tommie,0.1773666446,97.181.46.245,"{""location"": ""CX"", ""is_mobile"": false}" 12407,5,278,2016-12-21 03:20:05,http://schaden.biz/savannah_herman,0.2686811719,21.110.79.39,"{""location"": ""CL"", ""is_mobile"": false}" 12408,5,278,2017-01-23 15:28:00,http://krajcik.info/delta.becker,0.9050136013,95.26.152.2,"{""location"": ""GM"", ""is_mobile"": false}" 12409,5,278,2017-03-23 06:06:09,http://shanahan.net/lillian_sanford,0.6091156172,113.135.190.237,"{""location"": ""OM"", ""is_mobile"": false}" 12410,5,278,2017-02-23 08:37:39,http://lebsack.name/felipe,0.6284852772,175.159.20.77,"{""location"": ""LV"", ""is_mobile"": false}" 12411,5,278,2017-02-28 21:28:22,http://gleason.net/isac_cremin,0.5171061003,250.87.116.220,"{""location"": ""LS"", ""is_mobile"": true}" 12412,5,278,2016-12-27 14:18:18,http://wardcorkery.io/connor,0.3560251774,12.86.209.203,"{""location"": ""GT"", ""is_mobile"": true}" 12413,5,278,2017-03-30 14:53:48,http://blanda.biz/beaulah,0.1012625911,121.230.19.226,"{""location"": ""KG"", ""is_mobile"": true}" 12414,5,278,2017-02-12 03:14:35,http://langosh.co/garry,0.4003607200,158.94.206.14,"{""location"": ""NC"", ""is_mobile"": true}" 12415,5,278,2017-02-16 07:10:05,http://brakuserdman.net/ashly,0.5833876146,117.169.141.12,"{""location"": ""NL"", ""is_mobile"": true}" 12416,5,278,2017-02-25 01:54:32,http://brakus.co/zakary_runte,0.8340396410,63.104.137.168,"{""location"": ""ET"", ""is_mobile"": false}" 12417,5,278,2017-02-10 13:41:13,http://okon.name/lorenz_abshire,0.2329338354,3.190.88.38,"{""location"": ""GI"", ""is_mobile"": true}" 15380,6,346,2016-12-25 04:20:01,http://effertz.co/german.marks,0.2075568221,107.179.169.168,"{""location"": ""MN"", ""is_mobile"": true}" 15381,6,346,2017-04-23 20:21:15,http://price.com/rodrick_ferry,0.1066343061,70.188.3.186,"{""location"": ""TL"", ""is_mobile"": false}" 15382,6,346,2017-01-24 03:05:16,http://cummings.name/ana,0.0518414579,24.52.93.245,"{""location"": ""VG"", ""is_mobile"": false}" 15383,6,346,2017-04-28 17:59:03,http://kunde.co/taya_tromp,0.7840980486,91.184.177.129,"{""location"": ""SO"", ""is_mobile"": false}" 15384,6,346,2016-12-26 14:19:21,http://corkeryhalvorson.info/malcolm,0.8465100123,131.91.84.47,"{""location"": ""CI"", ""is_mobile"": false}" 15385,6,346,2017-05-31 12:52:08,http://whiteveum.net/romaine,0.4042632890,3.198.195.203,"{""location"": ""PN"", ""is_mobile"": true}" 15386,6,346,2017-04-07 15:09:50,http://grimeshomenick.info/antone,0.4636847374,114.180.188.68,"{""location"": ""JM"", ""is_mobile"": false}" 15387,6,346,2017-02-27 12:08:54,http://handledner.co/lucy_gislason,0.3377969347,4.202.27.246,"{""location"": ""NG"", ""is_mobile"": false}" 15388,6,346,2017-06-05 06:15:59,http://botsford.org/ericka,0.7114482597,11.193.148.164,"{""location"": ""LT"", ""is_mobile"": true}" 15389,6,346,2016-12-29 00:45:32,http://legros.com/daisy.quitzon,0.7964926146,76.127.124.83,"{""location"": ""GA"", ""is_mobile"": false}" 15390,6,346,2017-04-04 06:33:49,http://rohan.biz/clifford.feeney,0.2549817971,241.194.72.86,"{""location"": ""EE"", ""is_mobile"": true}" 15391,6,346,2017-02-06 21:02:53,http://gorczany.io/tod.jacobi,0.3015180241,138.112.16.199,"{""location"": ""LI"", ""is_mobile"": false}" 15392,6,346,2017-04-18 14:55:45,http://parker.net/magnolia.frami,0.7522752114,122.67.223.56,"{""location"": ""IE"", ""is_mobile"": true}" 15393,6,346,2017-01-31 03:45:02,http://haagharber.org/kaleigh.wolff,0.4615015731,118.4.15.119,"{""location"": ""SV"", ""is_mobile"": false}" 15394,6,346,2016-12-20 12:08:26,http://nikolaus.co/efrain,0.1885623482,228.95.40.62,"{""location"": ""MO"", ""is_mobile"": false}" 15395,6,347,2017-06-01 23:53:29,http://baumbach.org/kaylie.nienow,0.9524400845,12.176.96.249,"{""location"": ""VU"", ""is_mobile"": true}" 15396,6,347,2017-02-24 13:16:39,http://kihntrantow.com/aileen,0.5500079118,45.236.44.173,"{""location"": ""AE"", ""is_mobile"": true}" 15397,6,347,2017-02-09 17:14:01,http://muller.net/ava,0.4314798103,20.114.23.242,"{""location"": ""CN"", ""is_mobile"": true}" 15615,6,351,2017-05-24 03:07:10,http://conn.com/vivianne,,219.90.248.254,"{""location"": ""QA"", ""is_mobile"": false}" 15398,6,347,2017-05-09 12:38:02,http://dickinson.net/macey_okuneva,0.4434698849,244.124.233.31,"{""location"": ""CX"", ""is_mobile"": true}" 15399,6,347,2017-03-12 14:06:42,http://zboncak.com/giles.emmerich,0.3320638596,26.171.242.147,"{""location"": ""PT"", ""is_mobile"": true}" 15400,6,347,2017-05-06 03:03:28,http://torphy.net/orlando,0.0574609239,190.88.184.70,"{""location"": ""CC"", ""is_mobile"": true}" 15401,6,347,2017-04-12 22:28:20,http://strosinwisoky.info/zora.purdy,0.8903193009,105.73.95.10,"{""location"": ""GY"", ""is_mobile"": false}" 15402,6,347,2017-05-08 07:15:35,http://pouroshudson.info/jerad,0.8666881874,220.212.55.80,"{""location"": ""OM"", ""is_mobile"": true}" 15403,6,347,2017-01-25 00:02:29,http://carrolllebsack.org/mauricio,0.8899328170,5.16.207.251,"{""location"": ""LK"", ""is_mobile"": true}" 15404,6,347,2017-05-01 17:22:54,http://homenickabshire.com/allison.king,0.5856855933,202.58.25.39,"{""location"": ""HT"", ""is_mobile"": false}" 15405,6,347,2016-12-28 16:15:53,http://tillman.biz/andreanne.jacobi,0.2250158381,148.100.243.197,"{""location"": ""HM"", ""is_mobile"": false}" 15406,6,347,2017-03-09 23:51:11,http://mraz.name/alison,0.6570918192,33.90.203.88,"{""location"": ""FK"", ""is_mobile"": false}" 15407,6,347,2017-04-06 04:28:06,http://mosciskihuels.com/dustin,0.4694390979,176.209.195.131,"{""location"": ""BQ"", ""is_mobile"": true}" 15408,6,347,2017-03-05 23:15:38,http://haley.net/bria_terry,0.8606799023,212.253.194.103,"{""location"": ""NP"", ""is_mobile"": false}" 15409,6,347,2017-03-24 22:17:37,http://wymanemard.com/jerod,0.2646811268,10.27.169.36,"{""location"": ""MC"", ""is_mobile"": true}" 15410,6,347,2017-01-09 13:44:57,http://thiel.io/deven,0.3180440044,134.253.98.147,"{""location"": ""BR"", ""is_mobile"": true}" 15411,6,347,2017-04-10 03:06:28,http://abernathy.io/cordie,0.1369883552,148.195.108.189,"{""location"": ""BB"", ""is_mobile"": true}" 15412,6,347,2017-01-20 03:43:44,http://muller.co/marietta,0.8754454408,230.205.237.179,"{""location"": ""ZM"", ""is_mobile"": true}" 15413,6,347,2017-03-16 14:11:51,http://swaniawski.org/birdie,0.9140335871,233.146.192.57,"{""location"": ""TC"", ""is_mobile"": false}" 15414,6,347,2017-03-31 04:07:28,http://bosco.co/brianne.davis,0.6841297308,105.224.210.61,"{""location"": ""AF"", ""is_mobile"": true}" 15415,6,347,2017-03-04 18:37:22,http://willms.biz/demetrius_king,0.7694036109,212.169.57.211,"{""location"": ""AE"", ""is_mobile"": true}" 15416,6,347,2017-06-01 04:37:01,http://feestvolkman.io/carlos,0.3082332581,159.74.226.187,"{""location"": ""EC"", ""is_mobile"": false}" 15417,6,347,2017-05-13 16:18:48,http://trantow.name/alta,0.8011993262,163.166.35.110,"{""location"": ""HM"", ""is_mobile"": true}" 15418,6,347,2017-03-01 09:52:55,http://dickinson.co/jazmyn.senger,0.6873833191,223.138.15.52,"{""location"": ""DZ"", ""is_mobile"": false}" 15419,6,347,2017-02-02 22:59:50,http://oconnell.name/patsy_dooley,0.1413283964,159.24.11.245,"{""location"": ""CF"", ""is_mobile"": false}" 15420,6,347,2017-04-28 11:46:31,http://watsica.org/jordy,0.2280335048,181.212.201.113,"{""location"": ""NZ"", ""is_mobile"": true}" 15421,6,347,2016-12-13 14:51:57,http://weimannschulist.io/brayan.turner,0.6103585661,76.117.206.129,"{""location"": ""KZ"", ""is_mobile"": false}" 15422,6,347,2017-05-30 01:28:17,http://koelpin.co/harley,0.0418357533,17.192.163.141,"{""location"": ""AO"", ""is_mobile"": true}" 15423,6,347,2017-02-08 18:24:42,http://braunmckenzie.com/augustine.turner,0.8849326377,186.202.48.224,"{""location"": ""ST"", ""is_mobile"": true}" 15424,6,347,2017-05-13 00:19:35,http://von.org/duncan_mertz,0.8476366035,225.85.96.113,"{""location"": ""AL"", ""is_mobile"": false}" 15425,6,347,2017-06-08 04:16:07,http://kozey.name/eden,0.4933382473,34.101.122.97,"{""location"": ""IM"", ""is_mobile"": true}" 15426,6,347,2017-05-31 18:59:38,http://leannon.io/ernesto,0.9114987561,97.74.53.3,"{""location"": ""KE"", ""is_mobile"": true}" 15427,6,347,2016-12-28 04:57:06,http://kaulke.io/deshawn,0.6611986901,73.141.243.249,"{""location"": ""GG"", ""is_mobile"": true}" 15428,6,347,2017-01-17 16:34:49,http://brakus.net/luigi_morar,0.8426907128,55.234.99.10,"{""location"": ""SX"", ""is_mobile"": true}" 15429,6,347,2017-02-03 22:40:46,http://lockmanlakin.io/mohammed,0.8712739595,130.18.238.123,"{""location"": ""TK"", ""is_mobile"": false}" 15430,6,347,2017-02-02 11:22:47,http://rutherford.org/barton,0.9585057088,125.161.205.69,"{""location"": ""MM"", ""is_mobile"": true}" 2572,2,57,2017-04-05 00:59:48,http://lakinwiegand.name/teie,0.5224903377,190.233.15.235,"{""location"": ""GR"", ""is_mobile"": true}" 2573,2,57,2017-01-05 16:48:10,http://west.com/torrance.moore,0.6885023240,217.137.34.209,"{""location"": ""SJ"", ""is_mobile"": false}" 2574,2,57,2017-04-22 03:22:11,http://watsicahane.io/agnes.beier,0.4387159604,171.78.150.213,"{""location"": ""CH"", ""is_mobile"": false}" 2575,2,57,2017-05-30 08:23:57,http://hegmann.co/jakayla,0.8247869480,181.60.199.93,"{""location"": ""DK"", ""is_mobile"": false}" 2576,2,57,2017-05-19 17:06:53,http://wilderman.com/jayce,0.5582098526,126.212.145.230,"{""location"": ""NC"", ""is_mobile"": false}" 2577,2,57,2017-04-15 05:57:16,http://volkman.net/kenton,0.2564190876,113.197.235.122,"{""location"": ""FO"", ""is_mobile"": true}" 2578,2,57,2017-06-04 19:34:08,http://durgan.co/monte.stanton,0.8444813335,32.84.103.142,"{""location"": ""GT"", ""is_mobile"": true}" 2579,2,57,2017-01-26 19:34:01,http://boehmgaylord.biz/kaandra_prosacco,0.8554819541,241.238.39.110,"{""location"": ""PE"", ""is_mobile"": true}" 2580,2,57,2017-02-10 20:31:43,http://macejkovic.net/jairo_franecki,0.4007724653,31.214.162.229,"{""location"": ""KR"", ""is_mobile"": false}" 2581,2,57,2017-02-04 05:38:47,http://feiljohns.com/skye.steuber,0.8868847022,209.205.21.214,"{""location"": ""IS"", ""is_mobile"": true}" 2582,2,58,2017-01-25 10:50:05,http://schaden.co/allie_quitzon,,94.156.106.213,"{""location"": ""BV"", ""is_mobile"": false}" 2583,2,58,2017-02-20 05:39:29,http://cronakunde.net/alva.willms,,179.33.220.180,"{""location"": ""PF"", ""is_mobile"": true}" 2584,2,58,2017-05-09 09:23:30,http://bogisich.biz/amalia,,246.161.45.49,"{""location"": ""SK"", ""is_mobile"": false}" 2585,2,58,2017-05-26 12:01:07,http://blanda.co/brice,,214.146.125.106,"{""location"": ""KM"", ""is_mobile"": true}" 2586,2,58,2017-04-04 04:34:38,http://medhurstkeebler.name/stella,,167.246.217.149,"{""location"": ""RU"", ""is_mobile"": true}" 2587,2,58,2017-03-20 07:07:28,http://hermiston.biz/asa.kautzer,,196.204.157.4,"{""location"": ""CX"", ""is_mobile"": false}" 2588,2,58,2017-05-11 15:17:24,http://ferry.biz/alvina,,226.53.232.85,"{""location"": ""SI"", ""is_mobile"": false}" 2589,2,58,2017-01-07 04:51:05,http://mayerupton.co/maynard_legros,,144.144.178.215,"{""location"": ""CA"", ""is_mobile"": false}" 2590,2,58,2017-04-30 09:44:48,http://flatley.biz/fae,,199.37.24.78,"{""location"": ""AE"", ""is_mobile"": true}" 2591,2,58,2017-02-17 15:09:15,http://nitzsche.org/randal,,22.140.173.127,"{""location"": ""SI"", ""is_mobile"": true}" 2592,2,58,2016-12-22 17:06:07,http://olson.co/eliane,,64.5.129.44,"{""location"": ""CY"", ""is_mobile"": false}" 2593,2,58,2017-05-05 15:01:12,http://kovacektorphy.com/yasmeen,,245.101.143.202,"{""location"": ""AG"", ""is_mobile"": true}" 2594,2,58,2017-03-15 01:56:26,http://mosciski.co/brenden_ohara,,169.71.77.27,"{""location"": ""AO"", ""is_mobile"": true}" 2595,2,58,2016-12-31 01:52:50,http://volkman.com/keara.frami,,229.96.209.147,"{""location"": ""CN"", ""is_mobile"": true}" 2596,2,58,2017-02-04 01:26:34,http://dicki.co/deron,,12.27.205.41,"{""location"": ""KI"", ""is_mobile"": true}" 2597,2,58,2017-05-01 20:05:27,http://kelerquigley.com/vernon,,101.135.11.241,"{""location"": ""SN"", ""is_mobile"": false}" 2598,2,58,2016-12-22 17:38:50,http://koepp.name/robyn,,184.197.231.99,"{""location"": ""AG"", ""is_mobile"": false}" 2599,2,58,2017-05-18 10:40:57,http://west.biz/nikki_schmeler,,86.52.144.33,"{""location"": ""TK"", ""is_mobile"": true}" 2600,2,58,2017-02-12 02:52:03,http://rempel.org/jovani.wiza,,160.151.161.122,"{""location"": ""HR"", ""is_mobile"": true}" 2601,2,58,2017-03-17 07:54:51,http://carter.com/reese_casper,,84.16.233.225,"{""location"": ""AZ"", ""is_mobile"": true}" 2602,2,58,2017-06-01 01:57:20,http://larsonprosacco.com/josefina,,175.227.49.187,"{""location"": ""MN"", ""is_mobile"": true}" 2603,2,58,2017-05-29 11:33:56,http://smith.com/andreane,,16.42.179.107,"{""location"": ""KW"", ""is_mobile"": false}" 2604,2,58,2017-06-13 14:22:09,http://satterfield.net/arno.tremblay,,77.167.238.156,"{""location"": ""SJ"", ""is_mobile"": true}" 2605,2,58,2017-01-27 19:33:48,http://hayes.biz/clemens,,165.54.181.100,"{""location"": ""LC"", ""is_mobile"": false}" 2606,2,58,2017-01-02 03:59:25,http://mclaughlin.io/hailee,,10.31.92.156,"{""location"": ""ID"", ""is_mobile"": false}" 2607,2,58,2017-01-28 19:04:52,http://bergstromjast.info/tristin,,112.66.129.2,"{""location"": ""BJ"", ""is_mobile"": true}" 2608,2,58,2017-02-07 09:21:03,http://terry.net/orpha.bartoletti,,105.20.204.115,"{""location"": ""SE"", ""is_mobile"": false}" 2609,2,58,2017-01-05 13:53:57,http://robel.biz/alia,,142.56.92.238,"{""location"": ""SO"", ""is_mobile"": false}" 2610,2,58,2017-05-07 05:38:30,http://emmerich.com/keon,,168.251.213.116,"{""location"": ""SK"", ""is_mobile"": false}" 2611,2,58,2017-02-15 21:56:27,http://mitchellbergstrom.org/twila,,229.194.39.21,"{""location"": ""MU"", ""is_mobile"": true}" 2612,2,58,2017-02-12 20:33:33,http://fahey.io/lexus_hammes,,219.44.50.195,"{""location"": ""BM"", ""is_mobile"": true}" 2613,2,58,2017-03-21 14:23:34,http://lowe.com/mittie,,245.106.12.211,"{""location"": ""NL"", ""is_mobile"": true}" 2614,2,58,2017-05-03 13:00:57,http://lindgrenabernathy.net/lowell,,125.235.41.194,"{""location"": ""BQ"", ""is_mobile"": false}" 2615,2,58,2017-02-22 16:33:13,http://kuphalcremin.org/dahlia,,225.10.238.111,"{""location"": ""PN"", ""is_mobile"": true}" 2616,2,58,2017-01-05 10:29:14,http://thompsonabshire.org/harmon,,10.225.39.38,"{""location"": ""CK"", ""is_mobile"": true}" 2617,2,58,2017-05-23 00:38:36,http://parisian.org/jana,,158.181.23.159,"{""location"": ""SJ"", ""is_mobile"": false}" 2618,2,58,2017-02-27 12:28:42,http://kovacekschumm.net/vicente.hermiston,,206.217.158.24,"{""location"": ""AE"", ""is_mobile"": false}" 2619,2,58,2017-05-14 08:35:41,http://medhurst.com/dalton,,163.236.146.112,"{""location"": ""CY"", ""is_mobile"": false}" 2620,2,58,2017-03-06 04:28:28,http://swaniawskidach.info/velma,,111.132.22.114,"{""location"": ""PA"", ""is_mobile"": false}" 2621,2,58,2017-06-12 03:26:50,http://bailey.io/ewell_king,,86.102.228.198,"{""location"": ""IQ"", ""is_mobile"": true}" 2622,2,58,2017-02-15 23:09:44,http://yundt.name/pedro.renner,,119.171.110.112,"{""location"": ""GS"", ""is_mobile"": false}" 2623,2,58,2017-05-20 12:57:19,http://daugherty.biz/karine_flatley,,155.12.135.97,"{""location"": ""BO"", ""is_mobile"": true}" 2624,2,58,2017-03-06 09:21:42,http://paucekarmstrong.org/dario,,118.86.227.27,"{""location"": ""SL"", ""is_mobile"": true}" 2625,2,58,2017-04-26 05:53:45,http://kuhlman.com/leonel,,39.51.35.245,"{""location"": ""TK"", ""is_mobile"": false}" 2626,2,58,2017-02-16 23:24:56,http://jacobi.co/kay,,200.45.206.98,"{""location"": ""RS"", ""is_mobile"": true}" 2627,2,58,2017-01-29 07:27:22,http://mertz.io/wallace.abbott,,99.137.36.65,"{""location"": ""BA"", ""is_mobile"": false}" 9507,4,213,2017-01-05 20:45:59,http://dubuquehickle.net/willy,,211.41.38.98,"{""location"": ""ZW"", ""is_mobile"": true}" 9508,4,213,2017-01-30 02:58:01,http://ziemann.co/alan,,7.77.19.202,"{""location"": ""AX"", ""is_mobile"": false}" 9509,4,213,2017-03-09 21:19:07,http://douglasgusikowski.com/jaylon,,48.87.204.37,"{""location"": ""KG"", ""is_mobile"": false}" 9510,4,213,2017-01-04 14:14:09,http://gorczany.info/dedric_schuppe,,246.179.239.249,"{""location"": ""NU"", ""is_mobile"": true}" 9511,4,213,2017-02-20 00:44:46,http://hilpert.name/krystel_jakubowski,,128.48.113.116,"{""location"": ""KH"", ""is_mobile"": true}" 9512,4,213,2017-02-12 06:48:26,http://rosenbaumhamill.net/ocie_nolan,,55.252.89.11,"{""location"": ""ZW"", ""is_mobile"": true}" 9513,4,213,2017-04-13 09:46:15,http://orn.info/mose,,140.50.8.76,"{""location"": ""KZ"", ""is_mobile"": true}" 9514,4,213,2017-03-05 02:42:34,http://oconner.info/ismael,,66.63.18.222,"{""location"": ""CI"", ""is_mobile"": false}" 9515,4,213,2017-02-06 12:25:12,http://lednercormier.info/melisa,,121.130.223.77,"{""location"": ""CD"", ""is_mobile"": true}" 9516,4,213,2017-04-12 17:33:22,http://herzogreinger.com/curtis_oconnell,,156.112.44.165,"{""location"": ""NZ"", ""is_mobile"": true}" 9517,4,213,2017-01-29 16:06:31,http://spinka.org/arne.smitham,,139.9.164.202,"{""location"": ""PA"", ""is_mobile"": false}" 9518,4,213,2017-03-12 02:21:29,http://aufderhar.io/jake.mckenzie,,206.12.154.142,"{""location"": ""PN"", ""is_mobile"": false}" 9519,4,213,2017-03-21 21:12:27,http://predoviccain.net/cara.hansen,,118.64.29.194,"{""location"": ""MZ"", ""is_mobile"": false}" 9520,4,213,2017-03-01 15:21:27,http://kling.co/maverick,,248.108.26.68,"{""location"": ""PL"", ""is_mobile"": false}" 9521,4,213,2017-03-06 15:51:28,http://powlowskistracke.co/isadore.skiles,,109.175.7.48,"{""location"": ""PA"", ""is_mobile"": false}" 9522,4,213,2017-06-03 05:22:13,http://gleason.name/julien,,181.112.160.171,"{""location"": ""MP"", ""is_mobile"": false}" 9523,4,213,2017-01-05 01:30:36,http://daugherty.io/dixie,,242.87.102.216,"{""location"": ""AR"", ""is_mobile"": false}" 9524,4,213,2017-03-19 10:33:44,http://connellymcclure.co/antonetta,,159.187.164.66,"{""location"": ""CG"", ""is_mobile"": false}" 9525,4,213,2017-05-27 12:46:21,http://raynor.com/thalia,,149.21.148.172,"{""location"": ""TO"", ""is_mobile"": false}" 9526,4,213,2017-06-12 08:17:34,http://kovacekhagenes.org/gwen,,30.175.12.78,"{""location"": ""AM"", ""is_mobile"": true}" 9527,4,213,2016-12-20 07:49:08,http://bogan.co/mozelle,,20.165.76.72,"{""location"": ""VA"", ""is_mobile"": true}" 9528,4,213,2017-05-14 11:05:16,http://simonitroman.io/jeica,,23.127.42.184,"{""location"": ""TC"", ""is_mobile"": true}" 9529,4,213,2017-03-29 20:18:29,http://corwin.biz/alanna_wolff,,33.59.24.71,"{""location"": ""LT"", ""is_mobile"": false}" 9530,4,213,2017-06-11 17:27:27,http://jacobsonbins.name/allene,,11.145.79.39,"{""location"": ""BJ"", ""is_mobile"": true}" 9531,4,213,2017-02-18 00:37:08,http://yost.org/brook_white,,230.10.176.179,"{""location"": ""PS"", ""is_mobile"": false}" 9532,4,213,2016-12-21 08:59:34,http://veum.io/theodore,,173.155.252.67,"{""location"": ""YT"", ""is_mobile"": true}" 9533,4,213,2017-01-27 10:00:29,http://walsh.name/benny,,238.176.122.110,"{""location"": ""IO"", ""is_mobile"": true}" 9534,4,213,2017-03-08 19:48:31,http://okunevadeckow.io/ophelia_beer,,250.214.253.157,"{""location"": ""HM"", ""is_mobile"": false}" 9535,4,213,2017-04-16 15:22:18,http://rempel.name/mozelle_upton,,127.61.189.108,"{""location"": ""NI"", ""is_mobile"": false}" 9536,4,213,2017-03-29 15:17:50,http://osinskikeeling.info/jorge,,226.235.226.174,"{""location"": ""GR"", ""is_mobile"": false}" 9537,4,213,2016-12-19 02:05:01,http://sanford.net/brandon,,240.87.27.101,"{""location"": ""EG"", ""is_mobile"": true}" 9538,4,213,2017-05-02 13:39:06,http://schuster.io/taylor.rohan,,69.74.90.23,"{""location"": ""EH"", ""is_mobile"": true}" 9539,4,213,2017-06-10 03:15:09,http://wintheiser.com/leann.crooks,,39.68.228.176,"{""location"": ""IT"", ""is_mobile"": true}" 9540,4,214,2017-05-25 15:06:03,http://bergstrom.org/armando,,31.237.61.15,"{""location"": ""MM"", ""is_mobile"": false}" 9541,4,214,2017-05-06 18:40:16,http://balistreri.biz/alexandrea.armstrong,,38.203.43.22,"{""location"": ""BM"", ""is_mobile"": false}" 9542,4,214,2017-01-24 18:27:39,http://mueller.com/miracle.robel,,138.10.143.15,"{""location"": ""MA"", ""is_mobile"": false}" 9543,4,214,2017-01-08 07:43:37,http://dickens.biz/abby.hoppe,,75.98.219.76,"{""location"": ""UZ"", ""is_mobile"": false}" 9544,4,214,2017-04-07 05:50:13,http://gusikowski.io/cheyenne.lakin,,129.47.189.48,"{""location"": ""PH"", ""is_mobile"": false}" 9545,4,214,2016-12-24 04:38:36,http://brakus.com/zelma_bernier,,122.90.127.171,"{""location"": ""MQ"", ""is_mobile"": true}" 9546,4,214,2016-12-13 14:04:05,http://legrosconroy.biz/darrick.anderson,,14.110.53.57,"{""location"": ""LU"", ""is_mobile"": false}" 9547,4,214,2017-04-12 22:41:10,http://considinehirthe.name/royce,,87.137.71.80,"{""location"": ""CN"", ""is_mobile"": true}" 9548,4,214,2017-06-04 10:38:53,http://hyattcrist.net/larue.bruen,,221.170.106.83,"{""location"": ""GI"", ""is_mobile"": false}" 9550,4,214,2017-05-18 00:10:10,http://brown.info/major.ebert,,29.238.195.29,"{""location"": ""NR"", ""is_mobile"": true}" 9551,4,214,2017-05-18 21:01:35,http://strosin.com/muhammad,,247.223.105.55,"{""location"": ""IT"", ""is_mobile"": true}" 9552,4,214,2017-03-20 11:10:14,http://boylehansen.name/morris,,29.207.133.27,"{""location"": ""PN"", ""is_mobile"": false}" 9553,4,214,2017-05-03 18:42:52,http://carrollwuckert.co/astrid.hagenes,,189.181.113.169,"{""location"": ""PE"", ""is_mobile"": true}" 9554,4,214,2017-04-15 02:24:48,http://zboncak.co/marjory,,51.29.209.47,"{""location"": ""SG"", ""is_mobile"": true}" 9555,4,214,2017-02-05 13:03:00,http://gleasonschuster.net/vernon,,239.73.183.60,"{""location"": ""TH"", ""is_mobile"": true}" 9556,4,214,2017-05-22 20:34:41,http://walter.org/deangelo,,175.139.158.28,"{""location"": ""BZ"", ""is_mobile"": false}" 9557,4,214,2017-06-09 20:35:44,http://caspertreutel.name/lula_rau,,200.54.2.39,"{""location"": ""CX"", ""is_mobile"": false}" 9558,4,214,2017-06-13 20:08:42,http://kubhaley.biz/merritt_harris,,252.47.76.166,"{""location"": ""PL"", ""is_mobile"": true}" 9559,4,214,2017-04-26 17:23:38,http://parisian.net/erin,,147.39.230.217,"{""location"": ""AU"", ""is_mobile"": true}" 9560,4,214,2017-03-18 15:37:26,http://baumbach.biz/treva,,238.175.158.187,"{""location"": ""GW"", ""is_mobile"": false}" 9561,4,214,2017-06-09 04:45:56,http://conroy.org/mattie,,116.61.181.68,"{""location"": ""AW"", ""is_mobile"": true}" 9562,4,214,2017-06-08 16:53:17,http://fay.org/solon.welch,,68.56.104.2,"{""location"": ""FM"", ""is_mobile"": false}" 9563,4,214,2017-05-11 17:13:12,http://murazik.biz/tamara.connelly,,243.160.250.138,"{""location"": ""PH"", ""is_mobile"": false}" 6513,3,142,2017-04-09 16:23:15,http://greenreilly.name/nicola_bogan,0.9800229940,206.113.84.254,"{""location"": ""CR"", ""is_mobile"": false}" 6514,3,142,2017-01-01 01:27:19,http://considine.org/randi_bogan,0.0272615037,67.47.44.98,"{""location"": ""CI"", ""is_mobile"": true}" 6515,3,142,2017-01-08 10:56:07,http://stoltenberg.com/reie,0.4633431105,116.78.115.248,"{""location"": ""ID"", ""is_mobile"": false}" 6516,3,142,2017-02-06 21:49:23,http://pacocha.org/brandy.harvey,0.7075524987,202.199.203.119,"{""location"": ""IE"", ""is_mobile"": false}" 6517,3,142,2017-03-09 08:46:27,http://braun.com/darrion.gerlach,0.8931621040,120.246.124.59,"{""location"": ""JP"", ""is_mobile"": true}" 6518,3,142,2017-03-06 14:21:52,http://macejkovicparker.biz/boris.luettgen,0.8714782226,250.158.35.221,"{""location"": ""NC"", ""is_mobile"": false}" 6519,3,142,2017-01-27 05:14:15,http://swaniawski.biz/efrain_leffler,0.1795823392,68.211.166.220,"{""location"": ""TL"", ""is_mobile"": true}" 6520,3,142,2017-01-30 11:13:39,http://nader.org/cleora,0.2562371143,199.175.252.98,"{""location"": ""PR"", ""is_mobile"": true}" 6521,3,142,2017-04-27 00:54:33,http://jacobson.biz/hal_stark,0.3378514436,144.172.144.38,"{""location"": ""PG"", ""is_mobile"": true}" 6522,3,142,2016-12-22 01:55:14,http://boylewest.co/penelope_crooks,0.9434379410,210.115.211.192,"{""location"": ""AQ"", ""is_mobile"": false}" 6523,3,142,2017-05-08 02:58:25,http://breitenbergauer.net/evie.klein,0.1019453860,90.3.125.9,"{""location"": ""KP"", ""is_mobile"": true}" 6524,3,142,2017-03-28 11:40:16,http://emmerich.org/oral.bernhard,0.5372194847,67.202.84.251,"{""location"": ""NO"", ""is_mobile"": true}" 6525,3,142,2017-02-15 14:52:04,http://yundt.net/ivah,0.3488151095,47.66.119.45,"{""location"": ""MZ"", ""is_mobile"": false}" 6526,3,142,2017-04-18 16:18:43,http://runolfsdottirvandervort.io/marlon,0.3527407710,231.194.119.146,"{""location"": ""SJ"", ""is_mobile"": false}" 6527,3,142,2017-03-03 17:07:03,http://wisoky.com/pattie,0.2138910501,199.143.239.74,"{""location"": ""CC"", ""is_mobile"": false}" 6528,3,142,2016-12-28 04:18:56,http://langosh.biz/ashleigh,0.4420210850,190.232.3.154,"{""location"": ""BI"", ""is_mobile"": true}" 6529,3,142,2016-12-14 11:01:51,http://framibechtelar.org/jaylan,0.7946070682,43.33.161.220,"{""location"": ""PL"", ""is_mobile"": true}" 6530,3,142,2017-04-03 01:46:01,http://schuster.name/beth.paucek,0.8305308010,35.35.198.93,"{""location"": ""SH"", ""is_mobile"": true}" 6531,3,142,2016-12-14 15:07:11,http://bauch.net/bryce,0.2291212002,136.32.52.88,"{""location"": ""KH"", ""is_mobile"": true}" 6532,3,142,2017-02-11 23:14:59,http://schroeder.biz/destany.hand,0.8156219987,139.129.88.156,"{""location"": ""PN"", ""is_mobile"": true}" 6533,3,142,2017-01-23 08:37:14,http://purdy.net/johnathon.kautzer,0.5492407052,149.106.233.227,"{""location"": ""MN"", ""is_mobile"": false}" 6534,3,142,2017-01-06 01:41:16,http://sengerthiel.name/patricia,0.8300098828,57.186.67.143,"{""location"": ""AL"", ""is_mobile"": true}" 6535,3,142,2017-04-05 01:31:34,http://labadie.co/alfredo,0.0948036718,155.159.125.123,"{""location"": ""MY"", ""is_mobile"": false}" 6536,3,142,2017-01-13 06:20:47,http://parisian.biz/berenice.price,0.0054255541,5.179.225.204,"{""location"": ""MH"", ""is_mobile"": true}" 6537,3,142,2017-05-01 19:15:46,http://barrows.info/maybell_wisozk,0.3841912727,27.240.143.70,"{""location"": ""IR"", ""is_mobile"": true}" 6538,3,142,2017-03-22 14:06:53,http://schuppe.biz/ryley,0.7006116772,155.174.69.232,"{""location"": ""YT"", ""is_mobile"": true}" 6539,3,142,2017-02-25 09:32:30,http://walsh.biz/green,0.0783315551,119.120.148.227,"{""location"": ""ZA"", ""is_mobile"": false}" 6540,3,142,2017-04-24 15:53:06,http://grimes.co/madelyn,0.0572643655,138.62.115.78,"{""location"": ""AF"", ""is_mobile"": true}" 6541,3,142,2016-12-23 04:02:03,http://cole.name/clare_bednar,0.6269816588,134.72.108.216,"{""location"": ""KN"", ""is_mobile"": true}" 6542,3,142,2017-04-13 08:18:03,http://schoen.co/sadie,0.2992819606,194.212.13.105,"{""location"": ""KW"", ""is_mobile"": false}" 6543,3,142,2017-04-04 08:10:29,http://gislasonmacgyver.com/felix,0.8682777494,109.158.172.173,"{""location"": ""NL"", ""is_mobile"": false}" 6544,3,142,2016-12-30 05:41:07,http://ortiz.biz/vance,0.3302972464,79.158.210.228,"{""location"": ""VE"", ""is_mobile"": true}" 6545,3,142,2016-12-23 23:59:08,http://satterfieldgusikowski.info/carole.lehner,0.8281299488,45.56.157.240,"{""location"": ""NF"", ""is_mobile"": false}" 6546,3,142,2017-03-14 19:02:23,http://romaguera.com/marques.howell,0.9602289271,142.166.63.24,"{""location"": ""MS"", ""is_mobile"": true}" 6547,3,142,2017-06-08 06:31:15,http://aufderhar.org/franz_rolfson,0.0221862535,157.78.5.179,"{""location"": ""DK"", ""is_mobile"": false}" 6548,3,142,2016-12-23 06:03:20,http://kuhic.com/hope,0.7752699546,143.142.11.192,"{""location"": ""MA"", ""is_mobile"": false}" 6549,3,142,2016-12-23 21:18:56,http://jacobson.com/noemie_medhurst,0.7916418967,210.167.202.35,"{""location"": ""LC"", ""is_mobile"": true}" 6550,3,142,2017-03-20 00:07:15,http://waelchiklocko.name/willard,0.7811822657,155.225.113.27,"{""location"": ""CH"", ""is_mobile"": true}" 6551,3,142,2017-05-09 15:07:50,http://beatty.name/charley_hartmann,0.3814397295,98.204.2.208,"{""location"": ""IL"", ""is_mobile"": false}" 6552,3,142,2017-02-12 11:04:14,http://walsh.org/rachael,0.4716689416,86.156.199.20,"{""location"": ""PN"", ""is_mobile"": false}" 6553,3,142,2016-12-28 21:36:42,http://stokes.com/napoleon,0.2396320103,203.10.249.16,"{""location"": ""EC"", ""is_mobile"": false}" 6554,3,142,2017-05-14 01:33:28,http://hudsonbeier.org/gregoria.cronin,0.7430821017,97.93.151.51,"{""location"": ""RU"", ""is_mobile"": true}" 6555,3,142,2017-02-22 21:33:28,http://stracke.com/lydia,0.2831647543,223.116.24.71,"{""location"": ""DE"", ""is_mobile"": false}" 6556,3,142,2017-05-07 19:41:01,http://welch.org/israel.auer,0.9938440036,14.226.22.154,"{""location"": ""PE"", ""is_mobile"": true}" 6557,3,142,2017-01-23 05:05:47,http://hyatt.io/haylee,0.1815453485,243.16.29.75,"{""location"": ""PM"", ""is_mobile"": true}" 6558,3,142,2017-01-18 04:11:55,http://wiegand.io/marcelo,0.7679505077,214.197.190.12,"{""location"": ""VI"", ""is_mobile"": true}" 6559,3,142,2017-03-25 03:32:31,http://thiel.io/estefania,0.2516111779,173.141.239.77,"{""location"": ""AO"", ""is_mobile"": true}" 6560,3,142,2017-01-27 13:28:33,http://wildermanabbott.com/braeden_orn,0.0974296592,227.177.74.52,"{""location"": ""CC"", ""is_mobile"": false}" 6561,3,142,2017-02-10 14:51:01,http://bernierrippin.io/eliezer,0.9832251748,90.9.167.50,"{""location"": ""MU"", ""is_mobile"": false}" 6562,3,142,2017-02-28 02:20:37,http://cummingsrenner.co/carlie,0.6679053674,40.35.71.121,"{""location"": ""VA"", ""is_mobile"": false}" 6563,3,142,2017-05-13 19:25:26,http://mcclure.com/cleo,0.3370384441,127.66.228.142,"{""location"": ""GS"", ""is_mobile"": false}" 12418,5,278,2017-04-16 20:40:24,http://keebler.biz/baylee.glover,0.2350792117,79.211.13.90,"{""location"": ""NI"", ""is_mobile"": false}" 12419,5,278,2017-04-03 00:41:49,http://zulauf.io/silas.shields,0.6827203720,200.149.28.132,"{""location"": ""LY"", ""is_mobile"": true}" 12420,5,278,2017-05-18 18:11:52,http://runtejohnston.com/kitty.bechtelar,0.4103684471,104.27.155.13,"{""location"": ""MC"", ""is_mobile"": false}" 12421,5,278,2016-12-22 09:26:46,http://stehrwitting.co/ottilie_jenkins,0.5364121831,177.56.220.222,"{""location"": ""SC"", ""is_mobile"": false}" 12422,5,278,2017-02-25 22:36:24,http://cainrodriguez.info/carmen.gorczany,0.4315975547,12.37.184.178,"{""location"": ""AI"", ""is_mobile"": false}" 12423,5,278,2017-04-16 04:58:56,http://bauch.io/merlin_spinka,0.9014341070,252.252.243.125,"{""location"": ""SO"", ""is_mobile"": true}" 12424,5,278,2017-01-12 06:39:35,http://wilkinsonhaag.net/yazmin,0.5812948276,131.65.86.101,"{""location"": ""ZW"", ""is_mobile"": true}" 12425,5,278,2017-04-05 15:04:44,http://trantow.co/brooks,0.5032742786,14.38.170.5,"{""location"": ""TT"", ""is_mobile"": true}" 12426,5,278,2017-03-31 00:13:48,http://klein.net/sylvan_bosco,0.9513140021,90.159.215.114,"{""location"": ""VU"", ""is_mobile"": true}" 12427,5,279,2017-01-07 05:23:54,http://schumm.co/judy.watsica,,9.33.219.40,"{""location"": ""GS"", ""is_mobile"": false}" 12428,5,279,2017-05-10 23:56:58,http://murray.co/charlie,,173.41.54.59,"{""location"": ""HN"", ""is_mobile"": true}" 12429,5,279,2017-01-21 12:00:00,http://zulaufchristiansen.biz/angelita,,26.62.188.134,"{""location"": ""CV"", ""is_mobile"": false}" 12430,5,279,2017-04-08 09:29:16,http://ohara.biz/savanah,,96.135.96.100,"{""location"": ""GT"", ""is_mobile"": true}" 12431,5,279,2017-04-16 22:32:00,http://stiedemannherzog.com/alexandro.stark,,82.91.227.158,"{""location"": ""HU"", ""is_mobile"": false}" 12432,5,279,2017-06-05 07:08:26,http://stiedemannsenger.name/madonna_murphy,,71.198.120.188,"{""location"": ""FJ"", ""is_mobile"": false}" 12433,5,279,2017-05-29 09:17:41,http://rutherford.name/moshe.wuckert,,254.35.161.238,"{""location"": ""BH"", ""is_mobile"": true}" 12434,5,279,2017-03-29 21:30:15,http://lind.info/alford_krajcik,,54.2.152.160,"{""location"": ""VE"", ""is_mobile"": false}" 12435,5,279,2017-05-01 20:18:25,http://larkinlesch.org/natalia,,18.67.167.8,"{""location"": ""HM"", ""is_mobile"": true}" 12436,5,279,2017-03-13 06:01:30,http://pourosharvey.biz/carlos_hoppe,,28.4.164.209,"{""location"": ""DJ"", ""is_mobile"": true}" 12437,5,279,2017-04-09 14:52:41,http://casper.net/torrey,,203.90.224.57,"{""location"": ""BH"", ""is_mobile"": false}" 12438,5,279,2017-03-15 02:43:43,http://weber.info/jamal_carroll,,172.108.115.199,"{""location"": ""IE"", ""is_mobile"": true}" 12439,5,279,2016-12-20 13:20:48,http://harris.com/lucas.ritchie,,4.148.103.230,"{""location"": ""LS"", ""is_mobile"": true}" 12440,5,279,2017-05-14 18:22:35,http://hane.name/eva,,224.181.199.226,"{""location"": ""AM"", ""is_mobile"": true}" 12441,5,279,2016-12-27 02:06:40,http://connellyfeeney.info/emma_durgan,,131.242.134.225,"{""location"": ""TM"", ""is_mobile"": true}" 12442,5,279,2017-05-14 02:22:06,http://altenwerth.io/oda,,177.82.213.141,"{""location"": ""SL"", ""is_mobile"": true}" 12443,5,279,2017-02-11 10:20:27,http://zieme.name/norris,,157.94.107.30,"{""location"": ""WS"", ""is_mobile"": false}" 12444,5,279,2017-06-10 11:54:35,http://macejkovickeebler.co/keely,,230.56.107.187,"{""location"": ""KP"", ""is_mobile"": true}" 12445,5,279,2017-03-26 03:28:30,http://lebsack.name/saul,,176.8.46.228,"{""location"": ""BQ"", ""is_mobile"": true}" 12446,5,279,2017-05-25 01:26:04,http://hayes.info/kay,,187.24.52.167,"{""location"": ""KP"", ""is_mobile"": false}" 12447,5,279,2017-04-04 01:04:13,http://damore.name/sarah.volkman,,222.65.138.203,"{""location"": ""ER"", ""is_mobile"": false}" 12448,5,279,2017-03-27 01:01:56,http://schaefer.org/lucius_rempel,,237.90.53.241,"{""location"": ""EC"", ""is_mobile"": true}" 12449,5,279,2017-03-04 15:30:38,http://ankunding.biz/una.schowalter,,155.46.102.132,"{""location"": ""SG"", ""is_mobile"": true}" 12450,5,279,2017-01-24 04:45:08,http://weimann.info/murray_schmeler,,181.217.2.242,"{""location"": ""DK"", ""is_mobile"": false}" 12451,5,279,2017-04-30 02:53:07,http://okon.name/maddison_sipes,,92.54.245.36,"{""location"": ""GF"", ""is_mobile"": false}" 12452,5,279,2017-06-14 00:28:54,http://volkman.net/clair,,69.52.82.86,"{""location"": ""GN"", ""is_mobile"": true}" 12453,5,279,2017-02-08 19:19:34,http://hartmannkozey.net/bridgette,,195.115.230.30,"{""location"": ""BG"", ""is_mobile"": false}" 12454,5,279,2016-12-19 11:25:53,http://kleinlabadie.biz/eric,,176.68.147.103,"{""location"": ""LU"", ""is_mobile"": true}" 12455,5,279,2017-04-28 23:31:17,http://stroman.com/alva.bailey,,154.180.216.50,"{""location"": ""GS"", ""is_mobile"": true}" 12456,5,279,2017-01-04 22:23:13,http://gleichner.io/vivianne.casper,,163.69.130.238,"{""location"": ""NE"", ""is_mobile"": false}" 12457,5,279,2016-12-24 04:51:36,http://jacobson.name/dedrick,,60.98.123.200,"{""location"": ""FO"", ""is_mobile"": true}" 12458,5,279,2017-03-11 04:08:31,http://kemmerondricka.io/eldora.prosacco,,243.163.30.42,"{""location"": ""NR"", ""is_mobile"": false}" 12459,5,279,2017-02-11 18:46:04,http://cain.name/matilda,,82.195.168.12,"{""location"": ""SC"", ""is_mobile"": true}" 12460,5,279,2017-02-27 05:53:17,http://auervolkman.com/eli_goyette,,188.124.17.148,"{""location"": ""CD"", ""is_mobile"": false}" 12461,5,279,2017-03-12 08:30:41,http://hayesmitchell.info/aleia,,155.109.80.41,"{""location"": ""GI"", ""is_mobile"": true}" 12462,5,279,2017-05-30 13:51:56,http://block.co/eriberto_grady,,174.235.115.207,"{""location"": ""CK"", ""is_mobile"": true}" 12463,5,279,2017-01-22 08:27:23,http://kuhicpadberg.co/claudia,,209.38.236.44,"{""location"": ""TL"", ""is_mobile"": false}" 12464,5,279,2017-03-12 08:26:11,http://hand.name/fay.schneider,,39.14.168.159,"{""location"": ""BL"", ""is_mobile"": false}" 12465,5,279,2017-03-27 14:29:02,http://leffler.org/katelin_erdman,,222.184.105.59,"{""location"": ""MU"", ""is_mobile"": true}" 12466,5,279,2017-05-20 06:06:49,http://koepp.biz/elouise,,195.65.170.162,"{""location"": ""UG"", ""is_mobile"": true}" 12467,5,279,2017-03-19 10:35:57,http://reichel.info/keegan.gorczany,,204.128.195.47,"{""location"": ""MF"", ""is_mobile"": true}" 12468,5,279,2017-04-07 15:11:06,http://lindgrenkoepp.co/deon,,210.69.115.23,"{""location"": ""MY"", ""is_mobile"": true}" 12469,5,279,2017-03-08 09:41:06,http://davis.biz/nestor.homenick,,99.72.147.242,"{""location"": ""CI"", ""is_mobile"": false}" 12470,5,279,2017-01-25 08:14:53,http://erdmanbeier.name/delores,,153.244.128.33,"{""location"": ""YT"", ""is_mobile"": true}" 12471,5,279,2017-05-01 00:21:55,http://greenholtdenesik.com/jayson,,142.130.41.209,"{""location"": ""TN"", ""is_mobile"": false}" 15431,6,347,2017-04-19 03:52:41,http://beahanbechtelar.name/amelie,0.1752103028,48.36.85.217,"{""location"": ""PN"", ""is_mobile"": true}" 15432,6,347,2017-05-31 02:35:54,http://rice.net/earline,0.8706524028,96.202.9.248,"{""location"": ""AT"", ""is_mobile"": false}" 15433,6,347,2017-06-05 03:02:55,http://bergnaumohara.io/madelyn,0.6730532335,41.92.198.26,"{""location"": ""KY"", ""is_mobile"": true}" 15434,6,347,2017-03-28 17:01:02,http://reichert.name/cornell,0.9219355011,28.162.137.55,"{""location"": ""BQ"", ""is_mobile"": false}" 15435,6,347,2016-12-31 18:48:27,http://collinszboncak.name/maximilian,0.4653461043,77.35.71.171,"{""location"": ""VG"", ""is_mobile"": true}" 15436,6,348,2017-05-31 19:29:35,http://gerhold.net/freddie_bednar,0.2338442227,211.15.109.232,"{""location"": ""VI"", ""is_mobile"": true}" 15437,6,348,2017-05-01 06:23:15,http://crooks.biz/korey.reinger,0.0041134345,127.93.32.243,"{""location"": ""CF"", ""is_mobile"": false}" 15438,6,348,2017-05-08 23:11:51,http://hamill.io/dana_ritchie,0.5323271545,46.130.163.103,"{""location"": ""US"", ""is_mobile"": false}" 15439,6,348,2017-03-01 21:03:35,http://colethompson.io/oceane_kub,0.7161216476,95.72.112.81,"{""location"": ""BM"", ""is_mobile"": false}" 15440,6,348,2017-04-05 01:21:55,http://macejkovic.net/milford.runte,0.1053956183,142.106.95.24,"{""location"": ""TW"", ""is_mobile"": true}" 15441,6,348,2017-03-23 22:31:59,http://kihn.io/heber_bradtke,0.7954978758,61.9.193.197,"{""location"": ""IQ"", ""is_mobile"": true}" 15442,6,348,2017-04-08 12:34:53,http://spencer.biz/oran,0.5587005181,97.132.101.108,"{""location"": ""BV"", ""is_mobile"": true}" 15443,6,348,2017-02-22 12:27:41,http://hintz.io/brendan,0.3890339005,13.6.8.201,"{""location"": ""MF"", ""is_mobile"": false}" 15444,6,348,2017-04-08 02:54:32,http://schneider.org/lucie,0.1802019941,13.121.53.84,"{""location"": ""PS"", ""is_mobile"": true}" 15445,6,348,2017-05-27 20:01:21,http://kirlinwaelchi.com/dovie.altenwerth,0.5754985492,16.204.65.86,"{""location"": ""LK"", ""is_mobile"": true}" 15446,6,348,2017-01-18 06:24:13,http://jerde.biz/briana,0.0192508262,117.109.23.227,"{""location"": ""GQ"", ""is_mobile"": false}" 15447,6,348,2017-03-27 20:02:09,http://dooley.com/chelsea.bernhard,0.2175125282,175.242.27.194,"{""location"": ""BM"", ""is_mobile"": false}" 15448,6,348,2017-04-15 19:16:53,http://hayes.io/elisabeth,0.6257551232,30.15.192.40,"{""location"": ""CZ"", ""is_mobile"": true}" 15449,6,348,2016-12-22 03:33:18,http://bartoletti.co/kellen,0.9404426152,228.64.243.52,"{""location"": ""MN"", ""is_mobile"": true}" 15450,6,348,2017-05-15 15:18:04,http://millsrowe.biz/jaqueline_simonis,0.7940012030,239.114.18.131,"{""location"": ""LB"", ""is_mobile"": false}" 15451,6,348,2017-05-12 09:34:14,http://goyetteblock.org/cole,0.4810776967,77.70.232.108,"{""location"": ""SX"", ""is_mobile"": false}" 15452,6,348,2017-06-07 15:34:25,http://kertzmann.com/jarred_waters,0.3483627498,126.242.39.182,"{""location"": ""LS"", ""is_mobile"": true}" 15453,6,348,2017-01-15 20:21:53,http://blanda.org/trinity_bosco,0.5829822641,69.49.70.28,"{""location"": ""FO"", ""is_mobile"": true}" 15454,6,348,2017-06-02 17:59:14,http://gleichner.biz/michelle,0.9370479837,77.75.146.69,"{""location"": ""KZ"", ""is_mobile"": false}" 15455,6,348,2017-04-24 06:38:38,http://keebler.info/vince_bode,0.1047879880,116.162.72.114,"{""location"": ""IM"", ""is_mobile"": true}" 15456,6,348,2017-03-05 11:09:35,http://rodriguezreynolds.biz/royce,0.0700696186,202.9.134.132,"{""location"": ""KP"", ""is_mobile"": false}" 15457,6,348,2017-05-25 11:25:39,http://macejkovic.net/marlon.jerde,0.4649247807,192.181.61.171,"{""location"": ""UA"", ""is_mobile"": true}" 15458,6,348,2017-05-04 15:24:19,http://mitchellgorczany.com/rolando_shanahan,0.8467351890,172.149.232.28,"{""location"": ""CR"", ""is_mobile"": true}" 15459,6,348,2017-04-16 14:29:05,http://lueilwitz.net/leonardo_glover,0.8562019759,199.199.100.80,"{""location"": ""TD"", ""is_mobile"": true}" 15460,6,348,2017-01-19 13:28:53,http://senger.name/vinnie_koch,0.4966059169,93.16.92.120,"{""location"": ""BG"", ""is_mobile"": false}" 15461,6,348,2017-05-29 10:36:44,http://beer.io/dee_ankunding,0.8180170168,72.137.209.182,"{""location"": ""KR"", ""is_mobile"": false}" 15462,6,348,2017-01-09 07:21:00,http://green.com/lori,0.1209700815,243.184.204.37,"{""location"": ""MG"", ""is_mobile"": false}" 15463,6,348,2017-05-22 05:58:12,http://sawayn.info/amani,0.4909453677,37.224.46.26,"{""location"": ""GB"", ""is_mobile"": true}" 15464,6,348,2017-03-13 08:01:54,http://metzcain.info/chance,0.1141870467,52.119.130.85,"{""location"": ""BG"", ""is_mobile"": false}" 15465,6,348,2017-02-14 05:39:14,http://hermiston.biz/kira,0.9183425449,245.25.7.193,"{""location"": ""CU"", ""is_mobile"": true}" 15466,6,348,2017-04-30 07:22:32,http://schoen.net/ramona,0.7874701979,80.245.120.158,"{""location"": ""LT"", ""is_mobile"": false}" 15467,6,348,2017-03-26 13:10:43,http://zulaufdickinson.co/gilda,0.5391390816,123.42.171.145,"{""location"": ""BF"", ""is_mobile"": false}" 15468,6,348,2017-01-05 21:49:12,http://jaskolski.info/ellen_senger,0.0252048218,2.198.15.101,"{""location"": ""SO"", ""is_mobile"": true}" 15469,6,348,2016-12-14 15:50:28,http://bartell.name/ottilie_bode,0.7395370324,227.110.81.92,"{""location"": ""GS"", ""is_mobile"": false}" 15470,6,348,2017-06-05 12:14:46,http://dickinson.co/lula,0.8599456534,220.37.54.60,"{""location"": ""BT"", ""is_mobile"": false}" 15471,6,348,2017-02-28 01:24:08,http://schaden.info/emmie.block,0.6411947264,235.105.237.71,"{""location"": ""CV"", ""is_mobile"": false}" 15472,6,348,2017-02-12 17:36:00,http://feeneyweber.co/savannah,0.1452810262,43.215.60.60,"{""location"": ""SZ"", ""is_mobile"": true}" 15473,6,348,2017-04-29 02:34:26,http://wisoky.io/nicole.ziemann,0.0106055210,35.175.107.184,"{""location"": ""NI"", ""is_mobile"": false}" 15474,6,348,2017-05-30 19:31:08,http://bodeabshire.com/brielle,0.4151619287,155.231.72.214,"{""location"": ""DJ"", ""is_mobile"": true}" 15475,6,348,2017-03-30 16:34:12,http://lesch.com/dagmar_mccullough,0.3635830278,165.113.185.41,"{""location"": ""AO"", ""is_mobile"": false}" 15476,6,348,2017-01-29 01:09:23,http://zemlak.com/marge.moen,0.9724559593,225.37.141.228,"{""location"": ""PW"", ""is_mobile"": true}" 15477,6,348,2017-04-20 06:32:27,http://considine.org/daron_greenholt,0.5054400552,33.221.110.209,"{""location"": ""GL"", ""is_mobile"": true}" 15478,6,348,2017-02-09 15:47:07,http://schneidertillman.name/morris_keler,0.1047567810,207.33.66.96,"{""location"": ""UA"", ""is_mobile"": false}" 15479,6,348,2016-12-15 12:17:31,http://smithamkunze.org/lucas,0.4375805357,21.198.20.28,"{""location"": ""IL"", ""is_mobile"": false}" 15480,6,348,2017-05-05 04:17:27,http://reilly.name/keegan,0.9134892282,41.201.136.133,"{""location"": ""UG"", ""is_mobile"": false}" 15481,6,348,2017-06-11 01:35:15,http://volkman.org/eldora,0.6044429743,71.89.129.158,"{""location"": ""BI"", ""is_mobile"": false}" 2628,2,58,2016-12-26 06:22:54,http://hirthe.co/westley,,35.92.64.230,"{""location"": ""EG"", ""is_mobile"": false}" 2629,2,58,2016-12-21 00:25:24,http://hand.io/wava_adams,,147.153.234.70,"{""location"": ""CF"", ""is_mobile"": false}" 2630,2,58,2017-03-03 00:09:05,http://davis.name/sydnee_roob,,83.90.119.171,"{""location"": ""ME"", ""is_mobile"": true}" 2631,2,58,2017-02-09 12:58:32,http://schaeferdubuque.name/cecilia,,126.31.149.124,"{""location"": ""CC"", ""is_mobile"": false}" 2632,2,58,2017-01-03 19:01:39,http://cruickshank.info/shirley.stark,,53.100.118.198,"{""location"": ""NU"", ""is_mobile"": true}" 2633,2,58,2017-03-22 20:58:47,http://bartonjacobs.name/kelsi,,133.75.15.204,"{""location"": ""RO"", ""is_mobile"": true}" 2634,2,58,2017-01-08 14:29:24,http://lockman.info/conner,,81.215.4.206,"{""location"": ""BB"", ""is_mobile"": true}" 2635,2,58,2017-04-29 13:34:18,http://glover.com/rosalyn_goodwin,,27.45.88.246,"{""location"": ""MF"", ""is_mobile"": false}" 2636,2,58,2017-01-01 08:41:02,http://sawaynbatz.name/amelia,,150.74.229.45,"{""location"": ""ZW"", ""is_mobile"": false}" 2637,2,58,2017-02-05 07:53:43,http://tillman.io/alene,,201.240.196.145,"{""location"": ""BH"", ""is_mobile"": true}" 2638,2,58,2017-03-21 13:14:08,http://kub.org/dorian,,96.19.177.18,"{""location"": ""BH"", ""is_mobile"": false}" 2639,2,58,2017-03-25 20:01:30,http://willms.biz/nona_jenkins,,13.120.163.207,"{""location"": ""PK"", ""is_mobile"": false}" 2640,2,58,2017-01-25 21:45:17,http://dubuque.net/sherwood,,201.55.22.240,"{""location"": ""IS"", ""is_mobile"": false}" 2641,2,58,2017-05-18 10:39:50,http://macgyver.net/irma_tillman,,139.67.94.50,"{""location"": ""GT"", ""is_mobile"": false}" 2642,2,58,2017-05-13 01:48:25,http://block.com/gino,,242.207.134.237,"{""location"": ""BI"", ""is_mobile"": true}" 2643,2,58,2017-06-07 15:37:25,http://bosco.io/jedidiah_yost,,111.218.156.72,"{""location"": ""TD"", ""is_mobile"": true}" 2644,2,58,2017-03-16 17:30:43,http://littel.org/euna_bernhard,,72.171.81.230,"{""location"": ""LS"", ""is_mobile"": false}" 2645,2,58,2017-04-12 04:45:45,http://boehmwillms.io/rebecca,,62.108.215.205,"{""location"": ""BT"", ""is_mobile"": false}" 2646,2,58,2017-03-05 18:15:50,http://kertzmann.org/layne,,87.140.122.185,"{""location"": ""GU"", ""is_mobile"": false}" 2647,2,58,2017-02-10 04:19:06,http://durgan.net/herbert_bahringer,,29.249.33.119,"{""location"": ""TR"", ""is_mobile"": false}" 2648,2,58,2017-05-02 15:23:03,http://kuhic.io/fern.kilback,,124.219.172.225,"{""location"": ""PS"", ""is_mobile"": true}" 2649,2,59,2017-02-18 06:43:07,http://ferry.io/keven,,153.152.76.145,"{""location"": ""LB"", ""is_mobile"": false}" 2650,2,59,2017-01-04 12:16:40,http://kreigercrooks.info/van,,22.84.143.61,"{""location"": ""HT"", ""is_mobile"": false}" 2651,2,59,2017-04-16 18:31:32,http://bahringer.info/laverne.hills,,199.166.115.191,"{""location"": ""SB"", ""is_mobile"": true}" 2652,2,59,2017-03-07 14:46:16,http://schuppekirlin.name/julio.hirthe,,28.31.190.212,"{""location"": ""IR"", ""is_mobile"": false}" 2653,2,59,2017-05-01 08:27:23,http://ward.name/destiny_dibbert,,244.200.82.145,"{""location"": ""MM"", ""is_mobile"": true}" 2654,2,59,2017-02-06 14:02:09,http://reichert.info/jocelyn.hahn,,138.100.61.74,"{""location"": ""NR"", ""is_mobile"": false}" 2655,2,59,2017-01-16 12:17:57,http://koelpineichmann.name/casper_abernathy,,138.202.238.93,"{""location"": ""AO"", ""is_mobile"": true}" 2656,2,59,2017-05-28 08:33:46,http://cartwright.name/aiyana.swaniawski,,221.72.94.93,"{""location"": ""SH"", ""is_mobile"": false}" 2657,2,59,2016-12-27 16:09:53,http://gutkowski.biz/retha,,116.150.39.104,"{""location"": ""LR"", ""is_mobile"": false}" 2658,2,59,2017-03-11 13:09:03,http://farrell.co/samantha,,152.204.208.147,"{""location"": ""FR"", ""is_mobile"": false}" 2659,2,59,2017-03-25 15:07:47,http://cole.com/keon,,92.223.190.118,"{""location"": ""JO"", ""is_mobile"": false}" 2660,2,59,2017-02-08 07:08:24,http://cruickshank.com/harmon.wilderman,,16.156.186.150,"{""location"": ""LR"", ""is_mobile"": true}" 2661,2,59,2017-04-11 12:44:49,http://wuckertleannon.com/taryn.gleichner,,84.25.202.195,"{""location"": ""CD"", ""is_mobile"": true}" 2662,2,59,2017-04-14 03:01:35,http://hartmann.co/percy,,154.121.12.174,"{""location"": ""BH"", ""is_mobile"": true}" 2663,2,59,2016-12-30 04:02:49,http://schuppe.biz/kyla,,232.114.84.90,"{""location"": ""UZ"", ""is_mobile"": false}" 2664,2,59,2017-01-08 14:37:19,http://runolfsdottir.net/jace,,55.220.179.97,"{""location"": ""UY"", ""is_mobile"": false}" 2665,2,59,2017-01-17 21:37:41,http://witting.com/davion,,186.244.28.18,"{""location"": ""SJ"", ""is_mobile"": true}" 2666,2,59,2017-05-26 02:53:00,http://torphy.info/lacey_fadel,,159.12.195.89,"{""location"": ""MK"", ""is_mobile"": true}" 2667,2,59,2017-05-14 05:14:26,http://fahey.net/hans,,93.120.166.215,"{""location"": ""GU"", ""is_mobile"": true}" 2668,2,59,2017-06-05 02:31:52,http://quigley.info/curtis_bernhard,,151.69.37.49,"{""location"": ""BO"", ""is_mobile"": false}" 2669,2,59,2017-05-05 15:39:35,http://haley.co/leo,,23.102.40.250,"{""location"": ""DM"", ""is_mobile"": false}" 2670,2,59,2017-03-21 16:59:24,http://altenwerth.io/esther,,59.130.205.221,"{""location"": ""AE"", ""is_mobile"": false}" 2671,2,59,2017-02-05 06:52:37,http://purdy.net/carrie,,121.22.189.85,"{""location"": ""BB"", ""is_mobile"": false}" 2672,2,59,2017-05-08 20:54:09,http://bahringer.io/alanna,,172.163.104.227,"{""location"": ""AZ"", ""is_mobile"": false}" 2673,2,59,2017-04-17 15:29:16,http://hillskuhic.name/augustine,,101.251.122.149,"{""location"": ""KM"", ""is_mobile"": false}" 2674,2,59,2017-02-20 00:46:14,http://schaefer.com/luz,,104.254.5.188,"{""location"": ""ML"", ""is_mobile"": false}" 2675,2,59,2017-04-01 07:50:10,http://torphyjast.co/shyanne_block,,41.41.207.79,"{""location"": ""SM"", ""is_mobile"": false}" 2676,2,59,2017-01-17 00:02:12,http://aufderhar.name/freeman_bechtelar,,63.129.199.223,"{""location"": ""GF"", ""is_mobile"": false}" 2677,2,59,2017-01-25 10:55:54,http://gradydenesik.co/brennon,,219.190.90.95,"{""location"": ""GR"", ""is_mobile"": false}" 2678,2,59,2017-04-11 07:24:52,http://effertz.co/gabe_mann,,225.102.94.62,"{""location"": ""NG"", ""is_mobile"": true}" 2679,2,59,2017-05-10 16:46:26,http://ziemann.info/weston.medhurst,,192.79.239.69,"{""location"": ""VE"", ""is_mobile"": true}" 2680,2,59,2017-04-05 02:33:45,http://gleichnerkemmer.biz/roman,,198.16.210.216,"{""location"": ""VE"", ""is_mobile"": false}" 2681,2,59,2016-12-30 18:51:01,http://huels.net/emmie.welch,,71.214.115.155,"{""location"": ""AS"", ""is_mobile"": false}" 2682,2,59,2017-01-26 11:47:42,http://hayes.org/lilly,,27.116.96.130,"{""location"": ""TV"", ""is_mobile"": false}" 2683,2,59,2016-12-25 20:01:39,http://toy.name/chaz.eichmann,,211.26.148.153,"{""location"": ""GG"", ""is_mobile"": false}" 9564,4,214,2017-01-28 15:12:47,http://paucek.net/emelia_schulist,,86.74.84.147,"{""location"": ""NI"", ""is_mobile"": false}" 9565,4,214,2017-03-17 03:23:01,http://haley.info/simeon.lemke,,49.81.143.132,"{""location"": ""KZ"", ""is_mobile"": false}" 9566,4,214,2017-04-22 04:54:43,http://lebsackjohnson.info/justus.price,,67.119.5.99,"{""location"": ""BY"", ""is_mobile"": true}" 9567,4,215,2016-12-21 14:11:00,http://spinka.biz/tyson.steuber,,252.88.249.48,"{""location"": ""SI"", ""is_mobile"": false}" 9568,4,215,2017-01-17 08:05:45,http://emard.com/quentin,,208.81.79.152,"{""location"": ""GE"", ""is_mobile"": true}" 9569,4,215,2017-04-19 18:14:24,http://manndibbert.net/alize,,38.212.251.239,"{""location"": ""BS"", ""is_mobile"": false}" 9570,4,215,2017-03-13 16:54:47,http://murazikconnelly.biz/nora,,71.216.225.242,"{""location"": ""YT"", ""is_mobile"": false}" 9571,4,215,2017-06-10 21:56:15,http://roberts.net/elliott,,251.204.39.99,"{""location"": ""PM"", ""is_mobile"": false}" 9572,4,215,2017-05-12 18:22:40,http://deckowbeatty.name/jamey_thompson,,148.252.87.139,"{""location"": ""FK"", ""is_mobile"": false}" 9573,4,215,2017-02-25 17:13:00,http://goyettecruickshank.org/ramiro_pouros,,162.115.248.24,"{""location"": ""UA"", ""is_mobile"": true}" 9574,4,215,2017-01-30 20:46:28,http://abernathygleason.org/meredith,,116.9.119.127,"{""location"": ""JP"", ""is_mobile"": true}" 9575,4,215,2017-06-09 10:45:38,http://uptonhammes.co/efrain_denesik,,98.237.191.216,"{""location"": ""MM"", ""is_mobile"": false}" 9576,4,215,2017-04-26 08:31:12,http://larkin.net/lisette,,176.222.113.206,"{""location"": ""BV"", ""is_mobile"": false}" 9577,4,215,2017-05-22 14:44:01,http://casper.com/alyson,,219.94.81.251,"{""location"": ""ML"", ""is_mobile"": true}" 9578,4,215,2017-01-01 15:59:48,http://huel.info/mozelle,,129.138.84.124,"{""location"": ""AQ"", ""is_mobile"": false}" 9579,4,215,2017-02-04 02:53:38,http://lowe.io/nina_lakin,,177.183.198.109,"{""location"": ""LB"", ""is_mobile"": true}" 9580,4,215,2017-05-11 06:13:45,http://kilbackblick.biz/maudie_johnston,,50.30.52.57,"{""location"": ""KG"", ""is_mobile"": false}" 9581,4,215,2017-05-30 06:12:45,http://schinner.com/ewell.mccullough,,139.248.244.45,"{""location"": ""MM"", ""is_mobile"": true}" 9582,4,215,2017-02-27 19:31:05,http://sawayn.co/peggie,,27.30.235.42,"{""location"": ""EG"", ""is_mobile"": true}" 9583,4,215,2017-02-13 17:33:22,http://ritchie.biz/georgianna,,249.195.92.247,"{""location"": ""BI"", ""is_mobile"": true}" 9584,4,215,2017-03-26 22:04:39,http://beer.io/kaitlyn,,89.39.110.11,"{""location"": ""IM"", ""is_mobile"": false}" 9585,4,215,2017-06-01 16:34:13,http://larson.net/ruell.ratke,,87.22.109.19,"{""location"": ""NZ"", ""is_mobile"": true}" 9586,4,215,2016-12-28 12:29:20,http://douglas.org/nicolas,,170.248.204.149,"{""location"": ""BQ"", ""is_mobile"": true}" 9587,4,215,2017-03-04 23:49:41,http://hintz.biz/faustino,,161.220.165.191,"{""location"": ""SJ"", ""is_mobile"": false}" 9588,4,215,2017-03-04 00:56:49,http://mayerstrosin.com/fletcher_wyman,,245.214.227.200,"{""location"": ""NI"", ""is_mobile"": true}" 9589,4,215,2017-04-12 11:04:51,http://fahey.net/jaylan_boehm,,149.60.22.154,"{""location"": ""BI"", ""is_mobile"": true}" 9590,4,215,2017-02-09 06:11:41,http://west.io/celia,,163.165.170.106,"{""location"": ""AQ"", ""is_mobile"": false}" 9591,4,215,2017-02-19 09:30:53,http://pouros.info/kavon,,9.101.128.103,"{""location"": ""AO"", ""is_mobile"": true}" 9592,4,215,2017-02-06 17:25:55,http://corkery.io/brice,,56.79.215.231,"{""location"": ""BI"", ""is_mobile"": true}" 9593,4,215,2017-04-28 22:19:59,http://klein.name/gladyce,,251.194.109.108,"{""location"": ""MD"", ""is_mobile"": true}" 9594,4,215,2017-05-01 10:03:29,http://price.biz/norma,,139.153.99.67,"{""location"": ""MK"", ""is_mobile"": false}" 9595,4,215,2017-01-14 22:10:58,http://beer.io/jamar.nolan,,35.32.162.195,"{""location"": ""IL"", ""is_mobile"": true}" 9596,4,215,2016-12-16 14:44:51,http://bergstrom.org/carolyne.turcotte,,166.118.86.108,"{""location"": ""VC"", ""is_mobile"": false}" 9597,4,215,2017-04-05 23:04:57,http://hudson.org/ava_mann,,199.223.103.65,"{""location"": ""RE"", ""is_mobile"": false}" 9598,4,215,2017-06-03 17:51:55,http://runolfon.co/della,,225.3.50.122,"{""location"": ""AR"", ""is_mobile"": false}" 9599,4,215,2017-01-05 12:59:46,http://kozey.net/jovanny,,60.45.202.222,"{""location"": ""DO"", ""is_mobile"": true}" 9600,4,215,2016-12-27 04:55:17,http://macejkovicsipes.net/lorena,,68.251.69.122,"{""location"": ""VI"", ""is_mobile"": true}" 9601,4,215,2016-12-23 04:44:08,http://millchulist.io/cindy,,84.204.125.112,"{""location"": ""SO"", ""is_mobile"": true}" 9602,4,215,2017-04-14 05:32:15,http://legros.net/cristian.kilback,,44.111.45.109,"{""location"": ""SM"", ""is_mobile"": true}" 9603,4,215,2017-01-09 22:03:59,http://swiftjohnston.name/wilfrid,,73.31.205.199,"{""location"": ""NE"", ""is_mobile"": false}" 9604,4,215,2017-02-09 08:25:09,http://torphy.co/moses,,224.251.13.104,"{""location"": ""TT"", ""is_mobile"": false}" 9605,4,215,2017-05-10 12:09:37,http://okeefe.biz/victor,,95.23.154.240,"{""location"": ""UM"", ""is_mobile"": false}" 9606,4,215,2017-01-09 02:00:47,http://berge.biz/cole.rolfson,,98.83.99.151,"{""location"": ""GH"", ""is_mobile"": true}" 9607,4,215,2017-04-11 23:22:45,http://casper.org/tyree,,53.31.12.88,"{""location"": ""FJ"", ""is_mobile"": false}" 9608,4,215,2017-05-15 07:01:44,http://spinkaking.co/jakayla,,129.232.247.17,"{""location"": ""PG"", ""is_mobile"": true}" 9609,4,215,2017-02-03 23:50:45,http://goyette.biz/rosemarie.bernhard,,184.190.125.125,"{""location"": ""CM"", ""is_mobile"": true}" 9610,4,215,2016-12-14 21:48:44,http://lindgren.org/christian,,41.154.55.36,"{""location"": ""NZ"", ""is_mobile"": true}" 9611,4,215,2016-12-15 14:42:22,http://kreiger.net/arvel_mcdermott,,17.123.134.207,"{""location"": ""EG"", ""is_mobile"": false}" 9612,4,215,2017-02-22 10:18:21,http://green.biz/laurine,,148.96.10.218,"{""location"": ""AO"", ""is_mobile"": false}" 9613,4,215,2017-05-30 04:25:42,http://mraz.name/zetta_casper,,162.13.130.18,"{""location"": ""DZ"", ""is_mobile"": false}" 9614,4,215,2017-03-08 11:31:33,http://hilpert.com/aleandro.dickinson,,31.28.125.154,"{""location"": ""JP"", ""is_mobile"": false}" 9615,4,215,2017-06-11 04:33:25,http://konopelski.biz/carmella,,222.211.43.162,"{""location"": ""CX"", ""is_mobile"": true}" 9616,4,215,2017-05-28 21:23:43,http://heathcote.biz/marcus.kiehn,,225.15.44.249,"{""location"": ""AD"", ""is_mobile"": true}" 9617,4,215,2017-04-11 05:29:50,http://townehuel.biz/nathanial_white,,224.105.234.73,"{""location"": ""LI"", ""is_mobile"": true}" 9618,4,215,2017-05-16 16:02:23,http://aufderharschmeler.org/jamil,,115.195.139.117,"{""location"": ""US"", ""is_mobile"": true}" 9619,4,216,2017-04-15 08:59:10,http://rutherford.biz/neva_goldner,,55.167.246.241,"{""location"": ""ER"", ""is_mobile"": false}" 6564,3,142,2017-06-11 13:01:20,http://wilderman.com/elmore_mckenzie,0.6741451587,78.81.157.196,"{""location"": ""CX"", ""is_mobile"": true}" 6565,3,142,2017-03-17 22:19:41,http://kulas.org/alexie.bergnaum,0.3194787899,223.86.151.193,"{""location"": ""BQ"", ""is_mobile"": true}" 6566,3,142,2017-02-27 13:42:36,http://kaulke.biz/willa,0.5028555408,224.133.163.156,"{""location"": ""IO"", ""is_mobile"": true}" 6567,3,142,2017-02-16 04:38:25,http://hand.co/adolfo.lowe,0.2608082344,63.37.38.249,"{""location"": ""ET"", ""is_mobile"": true}" 6568,3,142,2017-01-06 20:51:48,http://zulauf.info/marie.oconnell,0.6214164336,36.87.162.118,"{""location"": ""NC"", ""is_mobile"": true}" 6569,3,142,2017-01-26 06:31:21,http://dickihartmann.info/sydnie,0.0373839159,9.119.251.136,"{""location"": ""GY"", ""is_mobile"": false}" 6570,3,143,2017-02-25 21:02:20,http://berge.com/leonie,0.7542596844,17.145.64.72,"{""location"": ""CN"", ""is_mobile"": true}" 6571,3,143,2017-01-22 23:58:18,http://oreilly.com/adolph_kutch,0.6013033900,47.227.100.177,"{""location"": ""ID"", ""is_mobile"": false}" 6572,3,143,2017-04-26 10:30:10,http://torphy.com/woodrow.ruel,0.7849814094,242.114.185.226,"{""location"": ""PF"", ""is_mobile"": false}" 6573,3,143,2017-03-20 23:11:15,http://ledner.co/kaylin,0.1842973642,241.73.9.188,"{""location"": ""CN"", ""is_mobile"": false}" 6574,3,143,2017-04-06 02:46:39,http://schmeler.co/berneice_hegmann,0.6438856256,59.106.143.7,"{""location"": ""PG"", ""is_mobile"": true}" 6575,3,143,2017-02-12 01:48:16,http://abbott.name/lazaro.boyer,0.2504664185,131.59.77.29,"{""location"": ""CY"", ""is_mobile"": false}" 6576,3,143,2017-01-20 07:05:52,http://collins.info/carmella,0.3013626199,2.37.212.219,"{""location"": ""GG"", ""is_mobile"": true}" 6577,3,143,2017-05-25 21:05:04,http://wisozk.co/olaf,0.4341056705,223.206.210.61,"{""location"": ""GH"", ""is_mobile"": true}" 6578,3,143,2017-03-12 21:40:20,http://jacobson.co/neva_lowe,0.1885951323,87.238.11.250,"{""location"": ""TF"", ""is_mobile"": false}" 6579,3,143,2017-04-23 05:05:03,http://bruen.co/curt_pagac,0.7979621105,228.220.44.193,"{""location"": ""EG"", ""is_mobile"": false}" 6580,3,143,2017-01-05 22:24:44,http://bruenhahn.name/brennan.waelchi,0.3908039371,118.145.122.145,"{""location"": ""MH"", ""is_mobile"": false}" 6581,3,143,2017-04-06 20:23:04,http://hoeger.com/hardy,0.8173891216,177.91.63.75,"{""location"": ""IM"", ""is_mobile"": false}" 6582,3,143,2017-03-29 17:55:46,http://rath.net/margarett_hamill,0.5522251673,180.38.225.235,"{""location"": ""BN"", ""is_mobile"": true}" 6583,3,143,2017-01-24 19:06:04,http://wolff.io/diamond,0.3938183283,102.175.47.172,"{""location"": ""SA"", ""is_mobile"": false}" 6584,3,143,2017-03-13 06:47:40,http://von.io/addison,0.4439737765,224.77.100.220,"{""location"": ""PL"", ""is_mobile"": false}" 6585,3,143,2017-04-05 20:56:08,http://schneiderhettinger.org/cary_dach,0.6230306827,195.120.149.215,"{""location"": ""GN"", ""is_mobile"": true}" 6586,3,143,2017-05-11 03:25:56,http://lynch.biz/shirley.mills,0.8004484277,203.184.138.175,"{""location"": ""TM"", ""is_mobile"": true}" 6587,3,143,2017-04-14 13:43:44,http://conn.name/kathleen_hermiston,0.2647540543,93.177.3.106,"{""location"": ""MA"", ""is_mobile"": false}" 6588,3,143,2017-05-14 10:12:21,http://stromantrantow.biz/abe_cole,0.0670506782,106.142.69.168,"{""location"": ""SN"", ""is_mobile"": false}" 6589,3,143,2017-02-03 07:31:25,http://okon.org/kenyon_smitham,0.0739444786,111.201.62.171,"{""location"": ""TN"", ""is_mobile"": false}" 6590,3,143,2016-12-15 17:45:48,http://marvincartwright.org/tillman,0.7437269897,203.72.136.83,"{""location"": ""CM"", ""is_mobile"": false}" 6591,3,143,2017-04-19 08:06:28,http://orn.net/lonny.pollich,0.7110749188,19.130.188.23,"{""location"": ""PS"", ""is_mobile"": true}" 6592,3,143,2017-05-05 16:25:42,http://leffler.biz/avery,0.5219298087,224.2.226.197,"{""location"": ""EC"", ""is_mobile"": false}" 6593,3,143,2017-04-18 09:37:07,http://little.net/ova,0.7026690496,85.141.11.253,"{""location"": ""UM"", ""is_mobile"": false}" 6594,3,143,2017-03-04 03:04:38,http://ricefunk.net/ronaldo,0.9495107167,186.129.190.156,"{""location"": ""VE"", ""is_mobile"": true}" 6595,3,143,2017-05-01 12:34:16,http://marks.info/julianne,0.6705335646,147.116.185.10,"{""location"": ""CA"", ""is_mobile"": false}" 6596,3,143,2017-01-27 15:45:11,http://reichert.com/gregoria.hirthe,0.0653634066,218.18.52.179,"{""location"": ""SR"", ""is_mobile"": true}" 6597,3,143,2017-01-12 23:55:58,http://kulachowalter.org/elwin.rempel,0.5565461945,107.175.34.173,"{""location"": ""GF"", ""is_mobile"": false}" 6598,3,143,2017-02-20 01:56:16,http://bayer.info/jordon,0.2778506371,36.16.53.210,"{""location"": ""WF"", ""is_mobile"": true}" 6599,3,143,2017-01-11 20:44:21,http://pacochayost.info/naomie.zemlak,0.9841250917,205.77.139.152,"{""location"": ""TT"", ""is_mobile"": false}" 6600,3,143,2017-05-29 08:42:39,http://jacobs.name/macy,0.0114181093,170.46.183.52,"{""location"": ""ID"", ""is_mobile"": true}" 6601,3,143,2017-01-28 20:31:45,http://grant.name/lillie_dooley,0.9710361587,164.243.152.132,"{""location"": ""TJ"", ""is_mobile"": false}" 6602,3,143,2017-04-05 02:56:58,http://bruen.info/harrison_lang,0.0529181649,78.55.34.107,"{""location"": ""FR"", ""is_mobile"": true}" 6603,3,143,2017-04-13 12:05:52,http://aufderharmacejkovic.net/rubie.bergnaum,0.6704035988,12.83.179.7,"{""location"": ""PL"", ""is_mobile"": false}" 6604,3,143,2017-05-31 18:43:48,http://considine.info/kyle.barton,0.9186188554,216.75.228.76,"{""location"": ""ID"", ""is_mobile"": true}" 6605,3,143,2017-05-26 12:11:34,http://windler.io/david.mckenzie,0.1889592017,215.89.43.23,"{""location"": ""MZ"", ""is_mobile"": false}" 6606,3,143,2017-04-09 20:33:09,http://grady.biz/kaitlin.rempel,0.6907391628,137.129.91.160,"{""location"": ""EE"", ""is_mobile"": false}" 6607,3,143,2016-12-28 18:23:37,http://baumbachward.io/jorge,0.8896273114,153.163.195.26,"{""location"": ""KY"", ""is_mobile"": true}" 6608,3,143,2017-06-09 21:09:21,http://brekke.info/jayson,0.1261721288,30.38.194.55,"{""location"": ""TG"", ""is_mobile"": false}" 6609,3,143,2017-01-01 09:27:16,http://osinskirippin.biz/orval,0.7091148550,254.102.177.250,"{""location"": ""LV"", ""is_mobile"": false}" 6610,3,143,2017-01-12 01:14:15,http://kuvalisdavis.info/calista,0.7565339066,185.26.172.39,"{""location"": ""BN"", ""is_mobile"": true}" 6611,3,143,2017-01-04 01:27:18,http://cain.co/jarod,0.2100995224,151.117.106.249,"{""location"": ""PK"", ""is_mobile"": true}" 6612,3,143,2017-02-02 15:27:14,http://mosciski.com/cordie,0.3837210121,139.162.228.139,"{""location"": ""YT"", ""is_mobile"": false}" 6613,3,143,2016-12-22 17:33:12,http://labadie.biz/renee.doyle,0.0981821118,126.155.135.150,"{""location"": ""CY"", ""is_mobile"": false}" 6614,3,143,2017-06-07 07:02:35,http://stamm.io/carolyne,0.4166966130,124.142.52.86,"{""location"": ""MH"", ""is_mobile"": true}" 6615,3,143,2017-04-16 06:42:32,http://hansen.net/samir_schamberger,0.3377165885,72.10.19.35,"{""location"": ""GE"", ""is_mobile"": false}" 12472,5,279,2017-02-11 00:26:43,http://brown.net/miracle,,143.180.56.207,"{""location"": ""ML"", ""is_mobile"": true}" 12473,5,279,2017-03-11 09:18:56,http://batzstiedemann.org/lia,,61.225.124.35,"{""location"": ""KH"", ""is_mobile"": true}" 12474,5,279,2017-05-04 06:29:14,http://hyatt.org/deven.rice,,80.23.113.32,"{""location"": ""AM"", ""is_mobile"": true}" 12475,5,279,2017-02-04 13:49:24,http://halvorson.biz/reilly,,133.250.57.130,"{""location"": ""BY"", ""is_mobile"": false}" 12476,5,279,2016-12-13 13:04:30,http://kunze.net/brianne.gerlach,,228.118.191.225,"{""location"": ""GQ"", ""is_mobile"": false}" 12477,5,279,2017-02-28 21:08:22,http://balistrerikertzmann.co/josue,,225.171.244.148,"{""location"": ""TO"", ""is_mobile"": false}" 12478,5,279,2017-04-22 22:54:13,http://strosin.net/cynthia,,199.251.184.30,"{""location"": ""SJ"", ""is_mobile"": false}" 12479,5,279,2017-06-06 17:21:59,http://hermistonbreitenberg.biz/laura,,99.187.63.223,"{""location"": ""RO"", ""is_mobile"": false}" 12480,5,279,2017-04-15 13:29:06,http://hettingerruel.biz/athena,,146.101.190.112,"{""location"": ""SC"", ""is_mobile"": true}" 12481,5,279,2017-02-27 20:40:26,http://purdyko.org/ron,,102.157.97.78,"{""location"": ""CL"", ""is_mobile"": true}" 12482,5,279,2017-03-15 16:39:58,http://oconnell.com/misty,,94.23.72.44,"{""location"": ""MH"", ""is_mobile"": false}" 12483,5,279,2017-06-10 10:04:50,http://powlowskiwolff.info/marjory_hermann,,20.240.131.119,"{""location"": ""BT"", ""is_mobile"": true}" 12484,5,279,2017-05-09 00:12:48,http://kunzebalistreri.org/bruce,,142.12.249.77,"{""location"": ""ES"", ""is_mobile"": false}" 12485,5,279,2016-12-17 05:26:53,http://franecki.org/marian,,217.173.149.17,"{""location"": ""SR"", ""is_mobile"": true}" 12486,5,279,2017-03-26 02:51:04,http://langosh.info/leanna,,178.8.182.166,"{""location"": ""SY"", ""is_mobile"": true}" 12487,5,279,2017-03-22 02:45:43,http://effertzbayer.com/jason,,153.112.182.190,"{""location"": ""BG"", ""is_mobile"": true}" 12488,5,279,2017-03-20 01:32:04,http://runte.io/ambrose,,56.168.185.113,"{""location"": ""KZ"", ""is_mobile"": false}" 12489,5,280,2017-01-13 13:18:36,http://schulist.io/lea.beatty,,82.130.150.59,"{""location"": ""YT"", ""is_mobile"": true}" 12490,5,280,2017-06-01 11:20:27,http://koconn.io/alena_wilderman,,170.217.57.93,"{""location"": ""SY"", ""is_mobile"": false}" 12491,5,280,2017-04-01 09:47:29,http://kirlinveum.org/floy_kling,,38.161.163.190,"{""location"": ""BM"", ""is_mobile"": false}" 12492,5,280,2017-05-10 21:35:25,http://gleason.net/reece,,171.217.125.91,"{""location"": ""CW"", ""is_mobile"": false}" 12493,5,280,2017-05-03 00:51:47,http://johnson.co/lilian_friesen,,227.249.155.170,"{""location"": ""RO"", ""is_mobile"": false}" 12494,5,280,2017-01-29 08:09:42,http://bahringer.net/gilbert,,187.160.248.28,"{""location"": ""IQ"", ""is_mobile"": true}" 12495,5,280,2016-12-20 21:24:45,http://ricequitzon.info/ramona_parker,,113.147.131.198,"{""location"": ""SC"", ""is_mobile"": false}" 12496,5,280,2017-04-10 16:17:25,http://cartwright.io/cicero,,245.144.119.56,"{""location"": ""SL"", ""is_mobile"": true}" 12497,5,280,2016-12-17 07:14:37,http://metz.biz/dena_cole,,56.235.224.35,"{""location"": ""CC"", ""is_mobile"": false}" 12498,5,280,2017-04-14 05:05:02,http://wisoky.co/maya,,24.105.84.48,"{""location"": ""SC"", ""is_mobile"": true}" 12499,5,280,2017-03-14 12:14:48,http://rolfson.net/hayley,,64.184.136.34,"{""location"": ""ID"", ""is_mobile"": true}" 12500,5,280,2017-05-10 22:44:54,http://mraz.net/horace.huels,,206.175.200.218,"{""location"": ""TL"", ""is_mobile"": false}" 12501,5,280,2016-12-31 12:53:37,http://toyjaskolski.co/isabell,,178.187.23.20,"{""location"": ""TN"", ""is_mobile"": false}" 12502,5,280,2017-01-24 12:38:21,http://mitchell.com/camille_lehner,,186.71.251.67,"{""location"": ""ES"", ""is_mobile"": false}" 12503,5,280,2017-03-02 18:40:42,http://skiles.info/terrill,,204.135.100.31,"{""location"": ""KY"", ""is_mobile"": true}" 12504,5,280,2017-03-22 05:28:18,http://strackemann.net/sienna,,218.53.150.5,"{""location"": ""BQ"", ""is_mobile"": false}" 12505,5,280,2017-01-20 09:23:39,http://schambergerwolff.net/camden,,63.122.38.204,"{""location"": ""TT"", ""is_mobile"": false}" 12506,5,280,2017-03-08 18:56:24,http://osinskihaley.name/andrew,,48.231.185.204,"{""location"": ""IO"", ""is_mobile"": false}" 12507,5,280,2017-04-09 00:08:35,http://hansen.name/maggie_auer,,225.236.114.8,"{""location"": ""AW"", ""is_mobile"": true}" 12508,5,280,2017-02-01 18:33:24,http://monahan.info/granville.weimann,,62.189.191.144,"{""location"": ""TV"", ""is_mobile"": false}" 12509,5,280,2017-06-09 14:50:02,http://considinekris.org/bertha,,170.191.154.203,"{""location"": ""IL"", ""is_mobile"": false}" 12510,5,280,2016-12-14 09:31:57,http://considine.com/avis.mclaughlin,,43.242.19.27,"{""location"": ""KM"", ""is_mobile"": true}" 12511,5,280,2016-12-20 07:58:14,http://wintheiserrutherford.name/brennon,,143.234.139.111,"{""location"": ""DJ"", ""is_mobile"": true}" 12512,5,280,2017-01-31 17:50:09,http://oconnell.org/aleia_hudson,,173.95.193.173,"{""location"": ""SL"", ""is_mobile"": true}" 12513,5,280,2017-03-18 17:43:56,http://wilderman.org/jarrett_crooks,,140.87.235.80,"{""location"": ""DJ"", ""is_mobile"": false}" 12514,5,280,2017-06-12 12:52:02,http://schmidtbruen.com/nat_doyle,,229.43.231.209,"{""location"": ""BQ"", ""is_mobile"": false}" 12515,5,280,2017-02-11 07:18:27,http://macejkovicschultz.io/ike,,252.70.232.225,"{""location"": ""AR"", ""is_mobile"": true}" 12516,5,280,2017-03-20 04:51:55,http://marksberge.com/mona,,75.11.33.22,"{""location"": ""NR"", ""is_mobile"": false}" 12517,5,280,2017-02-14 17:35:52,http://anderson.biz/tyrel,,50.239.224.75,"{""location"": ""IQ"", ""is_mobile"": true}" 12518,5,280,2017-01-22 23:57:10,http://barrows.name/adaline,,60.164.242.47,"{""location"": ""YE"", ""is_mobile"": true}" 12519,5,280,2017-05-09 21:42:16,http://langworth.io/immanuel_stanton,,111.235.217.176,"{""location"": ""BJ"", ""is_mobile"": false}" 12520,5,280,2017-02-04 11:25:42,http://stracke.co/jaydon,,159.47.104.75,"{""location"": ""AF"", ""is_mobile"": false}" 12521,5,280,2017-05-06 20:19:44,http://nitzsche.biz/maude_kreiger,,218.177.174.36,"{""location"": ""SJ"", ""is_mobile"": true}" 12522,5,280,2017-06-14 04:39:09,http://sengerwalker.com/noemi.kunde,,163.226.246.54,"{""location"": ""IL"", ""is_mobile"": true}" 12523,5,280,2017-06-11 12:17:50,http://nolan.io/casimir,,11.139.95.36,"{""location"": ""BI"", ""is_mobile"": true}" 12524,5,280,2017-03-31 20:58:35,http://cormier.org/emery,,206.118.177.199,"{""location"": ""AL"", ""is_mobile"": true}" 12525,5,280,2017-03-14 03:20:20,http://kunde.name/marisol_dietrich,,3.128.186.157,"{""location"": ""IO"", ""is_mobile"": false}" 12526,5,280,2017-02-12 20:33:57,http://hillsmiller.name/cheyenne_collier,,250.246.112.215,"{""location"": ""PA"", ""is_mobile"": false}" 15482,6,348,2017-03-19 18:16:19,http://reichel.co/aimee.tillman,0.2963933829,96.184.40.51,"{""location"": ""GE"", ""is_mobile"": true}" 15483,6,348,2017-02-10 03:48:28,http://huels.biz/ozella,0.3568982700,136.25.114.54,"{""location"": ""GH"", ""is_mobile"": true}" 15484,6,348,2017-04-26 05:41:19,http://casper.org/jamar,0.1206179609,237.54.195.64,"{""location"": ""IT"", ""is_mobile"": true}" 15485,6,348,2017-04-03 16:11:43,http://windler.io/wyatt_hand,0.3555207366,81.161.194.233,"{""location"": ""BY"", ""is_mobile"": false}" 15486,6,348,2017-06-01 11:15:26,http://kris.name/emilie,0.9901904373,179.57.157.84,"{""location"": ""MT"", ""is_mobile"": true}" 15487,6,348,2017-03-06 21:31:53,http://bartoletti.io/aurore,0.2588076915,176.31.194.105,"{""location"": ""MS"", ""is_mobile"": true}" 15488,6,348,2017-02-01 21:16:22,http://konopelskigibson.info/ian,0.1924671808,123.169.76.98,"{""location"": ""CA"", ""is_mobile"": false}" 15489,6,348,2017-06-03 08:57:15,http://haag.com/eladio,0.3921073801,49.138.212.30,"{""location"": ""BJ"", ""is_mobile"": false}" 15490,6,348,2017-06-02 09:58:47,http://kunzelynch.org/miguel,0.5793877021,131.219.171.196,"{""location"": ""SZ"", ""is_mobile"": false}" 15491,6,348,2017-01-08 07:41:59,http://moen.biz/tamara.mosciski,0.1917166811,23.205.139.201,"{""location"": ""PS"", ""is_mobile"": false}" 15492,6,348,2017-02-26 23:39:26,http://marvinadams.com/eve.dubuque,0.6821636183,89.5.219.108,"{""location"": ""YT"", ""is_mobile"": false}" 15493,6,348,2017-01-07 03:29:10,http://bergstrom.org/edgar,0.9302279036,227.216.82.202,"{""location"": ""GN"", ""is_mobile"": false}" 15494,6,348,2017-06-07 07:21:02,http://adams.biz/ocie_nolan,0.2561259714,87.23.114.79,"{""location"": ""CG"", ""is_mobile"": true}" 15495,6,348,2017-06-01 19:10:09,http://hills.biz/ayla,0.7588906462,246.241.164.122,"{""location"": ""AL"", ""is_mobile"": false}" 15496,6,348,2017-05-30 14:48:32,http://powlowski.net/theodore,0.3802896132,16.175.238.153,"{""location"": ""GH"", ""is_mobile"": false}" 15497,6,348,2017-03-12 10:47:53,http://runte.info/bennett,0.3968527314,126.173.168.14,"{""location"": ""FK"", ""is_mobile"": false}" 15498,6,348,2017-04-07 20:26:32,http://littel.co/malachi_connelly,0.9147388075,148.148.96.164,"{""location"": ""MW"", ""is_mobile"": true}" 15499,6,348,2017-05-09 17:32:17,http://mantestamm.org/german,0.7817669277,13.157.103.107,"{""location"": ""PM"", ""is_mobile"": false}" 15500,6,348,2017-03-29 23:35:51,http://champlin.info/anibal_fay,0.6789675870,245.53.34.155,"{""location"": ""KM"", ""is_mobile"": false}" 15501,6,348,2017-03-16 12:55:16,http://hermiston.info/maybell,0.3151012880,14.18.34.238,"{""location"": ""GY"", ""is_mobile"": true}" 15502,6,348,2016-12-15 06:58:33,http://beier.co/roxane_harber,0.3844250160,185.79.40.136,"{""location"": ""NU"", ""is_mobile"": false}" 15503,6,348,2017-04-05 23:42:47,http://gulgowski.com/cordia_paucek,0.4762554555,191.63.157.82,"{""location"": ""LB"", ""is_mobile"": true}" 15504,6,348,2017-03-16 05:21:55,http://pfannerstill.com/rigoberto_hodkiewicz,0.1221494372,193.120.155.208,"{""location"": ""AL"", ""is_mobile"": false}" 15505,6,348,2017-05-31 12:52:28,http://sauer.io/mallory,0.4546239966,138.72.197.138,"{""location"": ""BY"", ""is_mobile"": false}" 15506,6,349,2017-01-03 10:51:17,http://altenwerth.org/dereck.predovic,0.5998374718,236.215.150.112,"{""location"": ""AD"", ""is_mobile"": true}" 15507,6,349,2016-12-24 18:57:43,http://yost.io/elisa,0.1465831770,97.108.38.150,"{""location"": ""YE"", ""is_mobile"": true}" 15508,6,349,2017-05-15 01:33:05,http://spencergreenholt.io/keon,0.3065167272,208.5.105.197,"{""location"": ""DM"", ""is_mobile"": false}" 15509,6,349,2017-04-07 11:16:30,http://mckenziemiller.info/reggie,0.9745659576,64.209.191.246,"{""location"": ""GF"", ""is_mobile"": true}" 15510,6,349,2017-04-16 09:44:30,http://auerkoelpin.com/frederic_hudson,0.1880741817,237.196.42.65,"{""location"": ""PK"", ""is_mobile"": false}" 15511,6,349,2017-03-26 15:37:01,http://hilllbashirian.name/kolby,0.8120521941,133.3.241.188,"{""location"": ""SS"", ""is_mobile"": true}" 15512,6,349,2017-05-01 03:00:08,http://rau.name/josue,0.8628040379,241.107.17.35,"{""location"": ""MC"", ""is_mobile"": false}" 15513,6,349,2017-01-03 18:49:05,http://lubowitz.net/mozell_legros,0.9040671911,79.89.70.245,"{""location"": ""IE"", ""is_mobile"": false}" 15514,6,349,2017-01-28 06:56:54,http://nader.net/rahsaan.king,0.4733137609,252.157.62.19,"{""location"": ""TK"", ""is_mobile"": false}" 15515,6,349,2016-12-17 22:33:12,http://schuppe.co/albin,0.9247782407,169.223.39.246,"{""location"": ""VI"", ""is_mobile"": true}" 15516,6,349,2017-05-22 21:05:23,http://emardwatsica.org/nannie,0.7651647468,159.184.24.130,"{""location"": ""VI"", ""is_mobile"": false}" 15517,6,349,2017-04-11 19:45:21,http://blanda.info/demarco_hoeger,0.7323298720,23.70.78.97,"{""location"": ""CW"", ""is_mobile"": true}" 15518,6,349,2017-02-19 10:01:28,http://ritchiegaylord.info/liana.von,0.5610327624,135.124.28.180,"{""location"": ""ZW"", ""is_mobile"": false}" 15519,6,349,2017-04-19 12:29:38,http://rau.com/shirley.frami,0.5829079430,81.40.10.45,"{""location"": ""PA"", ""is_mobile"": false}" 15520,6,349,2017-04-21 21:05:55,http://schuppe.org/ludwig,0.8381578576,221.231.224.130,"{""location"": ""BF"", ""is_mobile"": false}" 15521,6,349,2017-03-21 19:17:38,http://rueljohnston.io/ricky,0.6688429986,87.3.102.228,"{""location"": ""OM"", ""is_mobile"": true}" 15522,6,349,2017-05-01 09:39:13,http://zieme.io/orion,0.6314982464,220.91.43.96,"{""location"": ""CN"", ""is_mobile"": true}" 15523,6,349,2017-04-11 07:46:32,http://anderson.co/marie,0.7680719609,43.150.72.16,"{""location"": ""BJ"", ""is_mobile"": true}" 15524,6,349,2017-03-24 06:47:31,http://cruickshankmarks.info/jarvis_ruel,0.1418163929,165.240.74.183,"{""location"": ""DJ"", ""is_mobile"": true}" 15525,6,349,2017-04-17 00:02:32,http://ruel.net/noemy,0.1379164449,22.213.243.54,"{""location"": ""AR"", ""is_mobile"": true}" 15526,6,349,2016-12-15 13:15:10,http://kuhn.io/stacey_ondricka,0.5209680561,19.87.116.197,"{""location"": ""TC"", ""is_mobile"": true}" 15527,6,349,2017-01-23 22:01:59,http://reinger.info/devon,0.8173415017,37.234.45.167,"{""location"": ""KP"", ""is_mobile"": false}" 15528,6,349,2017-01-05 10:18:38,http://bartolettipaucek.io/dallin,0.0328396956,181.104.108.139,"{""location"": ""CK"", ""is_mobile"": false}" 15529,6,349,2017-04-20 22:09:39,http://hilll.org/abelardo.towne,0.7992551162,121.69.235.203,"{""location"": ""RS"", ""is_mobile"": true}" 15530,6,349,2017-02-10 23:58:58,http://hettinger.co/thora_gottlieb,0.5002487361,175.6.97.187,"{""location"": ""DO"", ""is_mobile"": false}" 15531,6,349,2016-12-21 07:57:28,http://prohaska.name/vladimir,0.2856866049,204.112.150.36,"{""location"": ""TN"", ""is_mobile"": true}" 15532,6,349,2017-03-21 19:51:43,http://hermann.com/kaden_jakubowski,0.4003162190,248.251.107.180,"{""location"": ""IO"", ""is_mobile"": false}" 15533,6,349,2017-03-19 10:53:57,http://hane.com/bernadette.hermiston,0.6000633312,5.159.121.63,"{""location"": ""FO"", ""is_mobile"": true}" 2684,2,59,2016-12-20 04:08:26,http://doyle.co/deanna_roberts,,173.30.250.192,"{""location"": ""JO"", ""is_mobile"": false}" 2685,2,59,2017-03-01 15:09:11,http://pacochadare.org/robb,,24.86.213.81,"{""location"": ""KH"", ""is_mobile"": true}" 2686,2,59,2016-12-18 10:41:50,http://gleason.org/deanna.carter,,83.239.91.219,"{""location"": ""MM"", ""is_mobile"": true}" 2687,2,59,2017-02-24 18:21:30,http://harris.com/zula.kerluke,,194.86.46.220,"{""location"": ""LS"", ""is_mobile"": true}" 2688,2,59,2016-12-25 19:34:36,http://hermiston.name/carolina_haag,,207.137.90.249,"{""location"": ""BH"", ""is_mobile"": false}" 2689,2,59,2016-12-23 17:58:27,http://padbergschuster.info/golda,,49.112.63.229,"{""location"": ""NO"", ""is_mobile"": false}" 2690,2,59,2017-04-29 15:56:15,http://watsicajacobs.info/garth,,210.23.187.150,"{""location"": ""AW"", ""is_mobile"": false}" 2691,2,59,2016-12-30 09:27:00,http://raynor.com/golda.heathcote,,250.222.89.231,"{""location"": ""CO"", ""is_mobile"": true}" 2692,2,60,2017-04-25 11:47:08,http://deckowstiedemann.biz/alford_emmerich,,92.178.81.132,"{""location"": ""VA"", ""is_mobile"": false}" 2693,2,60,2017-02-05 23:16:21,http://hettinger.io/rosa_kunde,,119.128.203.134,"{""location"": ""HU"", ""is_mobile"": false}" 2694,2,60,2017-02-08 16:42:49,http://rogahnkerluke.info/immanuel_durgan,,147.75.101.58,"{""location"": ""CU"", ""is_mobile"": false}" 2695,2,60,2017-02-21 18:46:51,http://strackestroman.info/don,,44.15.247.143,"{""location"": ""GQ"", ""is_mobile"": false}" 2696,2,60,2017-04-14 18:14:38,http://murphy.info/margret,,69.115.177.41,"{""location"": ""IR"", ""is_mobile"": true}" 2697,2,60,2017-01-15 15:52:15,http://stiedemann.name/judah,,224.174.203.199,"{""location"": ""PG"", ""is_mobile"": true}" 2698,2,60,2017-01-19 18:23:52,http://okeefe.co/daisy,,246.243.203.99,"{""location"": ""NU"", ""is_mobile"": false}" 2699,2,60,2017-03-03 05:47:58,http://gerlach.net/macey,,184.30.137.212,"{""location"": ""GR"", ""is_mobile"": false}" 2700,2,60,2017-04-01 09:55:43,http://handortiz.info/flavio_west,,90.72.196.73,"{""location"": ""KY"", ""is_mobile"": true}" 2701,2,60,2017-06-13 07:33:44,http://senger.name/fanny,,61.54.202.148,"{""location"": ""QA"", ""is_mobile"": false}" 2702,2,60,2017-05-01 02:42:37,http://halvorsonfeil.io/yasmeen_kling,,119.159.28.252,"{""location"": ""CI"", ""is_mobile"": true}" 2703,2,60,2017-02-23 06:39:46,http://ernserabernathy.biz/ilene_howe,,89.141.187.61,"{""location"": ""MN"", ""is_mobile"": false}" 2704,2,60,2017-01-19 00:21:11,http://quigley.info/wilbert.wintheiser,,180.39.162.145,"{""location"": ""GF"", ""is_mobile"": false}" 2705,2,60,2017-02-14 22:30:39,http://denesik.info/alphonso,,233.89.180.76,"{""location"": ""SA"", ""is_mobile"": true}" 2706,2,60,2017-03-01 17:57:57,http://volkmannader.io/mathew,,77.195.222.55,"{""location"": ""GN"", ""is_mobile"": true}" 2707,2,60,2016-12-16 17:28:15,http://king.org/lizeth,,76.240.156.165,"{""location"": ""CC"", ""is_mobile"": true}" 2708,2,60,2016-12-16 21:38:41,http://raynor.name/alford,,134.131.157.137,"{""location"": ""GH"", ""is_mobile"": false}" 2709,2,60,2017-05-07 21:09:07,http://toygoyette.com/christa.schiller,,174.40.54.65,"{""location"": ""RW"", ""is_mobile"": true}" 2710,2,60,2017-03-19 03:46:44,http://kaulke.biz/manuel_bernier,,7.98.147.74,"{""location"": ""GQ"", ""is_mobile"": false}" 2711,2,60,2017-04-24 05:27:04,http://goyettesenger.org/sylvia,,67.238.182.159,"{""location"": ""LB"", ""is_mobile"": true}" 2712,2,60,2016-12-25 18:03:01,http://krajcikjakubowski.org/sigrid_spencer,,173.247.199.65,"{""location"": ""BS"", ""is_mobile"": false}" 2713,2,60,2017-06-05 12:35:03,http://wiza.co/karen_walker,,57.47.220.20,"{""location"": ""CH"", ""is_mobile"": true}" 2714,2,60,2016-12-26 20:16:45,http://langworthcruickshank.org/loren.herman,,234.127.243.184,"{""location"": ""TJ"", ""is_mobile"": false}" 2715,2,60,2017-05-11 12:28:08,http://swiftcarroll.biz/adelbert_keler,,152.186.29.247,"{""location"": ""PG"", ""is_mobile"": true}" 2716,2,60,2017-04-24 22:13:23,http://dach.co/deshawn,,214.111.179.2,"{""location"": ""SY"", ""is_mobile"": false}" 2717,2,60,2017-01-24 04:45:43,http://wyman.io/ena_batz,,52.29.8.188,"{""location"": ""NG"", ""is_mobile"": true}" 2718,2,60,2017-03-02 22:38:57,http://rippin.name/jana.brown,,3.114.63.183,"{""location"": ""SX"", ""is_mobile"": false}" 2719,2,60,2017-01-29 17:56:03,http://mclaughlin.com/isaiah.hartmann,,69.26.205.226,"{""location"": ""AT"", ""is_mobile"": false}" 2720,2,60,2017-06-04 15:35:49,http://zulauf.name/lavern,,59.208.251.112,"{""location"": ""LK"", ""is_mobile"": true}" 2721,2,60,2017-01-25 07:34:01,http://funkmitchell.name/reed_hammes,,10.106.237.234,"{""location"": ""AG"", ""is_mobile"": true}" 2722,2,60,2017-01-01 22:06:37,http://heaney.com/cierra_schuster,,149.115.183.169,"{""location"": ""NZ"", ""is_mobile"": false}" 2723,2,60,2017-01-11 13:08:32,http://lang.org/jacinto,,108.14.141.49,"{""location"": ""SS"", ""is_mobile"": true}" 2724,2,60,2017-05-10 21:16:46,http://langosh.info/colton,,220.92.197.207,"{""location"": ""TH"", ""is_mobile"": false}" 2725,2,60,2017-03-05 09:12:03,http://oreillymuller.org/kelvin,,63.46.28.44,"{""location"": ""SO"", ""is_mobile"": true}" 2726,2,60,2017-04-08 04:45:04,http://pouros.name/jaclyn,,14.48.201.18,"{""location"": ""CN"", ""is_mobile"": true}" 2727,2,60,2017-04-13 03:07:33,http://will.io/gail_cole,,96.19.40.83,"{""location"": ""BQ"", ""is_mobile"": true}" 2728,2,60,2017-04-05 00:50:32,http://blick.net/bryce,,158.34.113.242,"{""location"": ""FM"", ""is_mobile"": true}" 2729,2,60,2017-03-10 22:21:58,http://champlin.io/dexter,,193.245.123.127,"{""location"": ""VC"", ""is_mobile"": false}" 2730,2,60,2017-04-16 18:19:25,http://johnston.org/sasha.kemmer,,68.204.71.174,"{""location"": ""CZ"", ""is_mobile"": false}" 2731,2,60,2016-12-21 05:09:31,http://leuschke.io/norene,,209.66.63.27,"{""location"": ""JE"", ""is_mobile"": false}" 2732,2,60,2017-01-21 01:04:16,http://hudson.com/preston,,3.246.82.71,"{""location"": ""NC"", ""is_mobile"": false}" 2733,2,60,2017-02-07 01:47:02,http://osinski.name/darwin.brekke,,127.89.214.17,"{""location"": ""BQ"", ""is_mobile"": false}" 2734,2,60,2017-02-28 23:14:16,http://okunevaabernathy.org/maximillia_collins,,23.153.223.179,"{""location"": ""SE"", ""is_mobile"": true}" 2735,2,60,2017-03-03 13:22:28,http://kohler.info/leda.legros,,7.74.129.183,"{""location"": ""HR"", ""is_mobile"": true}" 2736,2,60,2017-03-15 01:47:57,http://mccullough.io/vallie_hermiston,,170.30.236.163,"{""location"": ""FK"", ""is_mobile"": true}" 2737,2,60,2017-05-31 03:37:41,http://blocksenger.co/travon,,146.49.72.101,"{""location"": ""NA"", ""is_mobile"": true}" 2738,2,60,2017-05-30 11:23:20,http://zieme.org/eliseo_herman,,154.120.174.62,"{""location"": ""TO"", ""is_mobile"": false}" 2739,2,60,2017-04-14 04:24:49,http://crona.biz/ida_kub,,239.63.203.83,"{""location"": ""NE"", ""is_mobile"": true}" 9620,4,216,2017-06-13 05:25:32,http://pfannerstill.io/kelvin,,13.88.46.99,"{""location"": ""SI"", ""is_mobile"": true}" 9621,4,216,2016-12-13 20:59:24,http://baumbachmedhurst.biz/kaandra,,157.175.167.109,"{""location"": ""PT"", ""is_mobile"": true}" 9622,4,216,2016-12-29 14:04:35,http://connelly.com/delpha,,78.101.26.168,"{""location"": ""MP"", ""is_mobile"": true}" 9623,4,216,2017-02-24 21:05:26,http://walsh.co/emerson.rippin,,13.251.77.22,"{""location"": ""MQ"", ""is_mobile"": false}" 9624,4,216,2017-03-07 07:05:17,http://zulaufhills.com/shayne_wyman,,160.174.110.156,"{""location"": ""JM"", ""is_mobile"": false}" 9625,4,216,2017-04-05 16:50:51,http://jerdeziemann.io/colt.toy,,27.75.127.232,"{""location"": ""KM"", ""is_mobile"": false}" 9626,4,216,2017-03-24 18:38:08,http://friesen.com/tracey.mante,,81.155.212.204,"{""location"": ""IN"", ""is_mobile"": true}" 9627,4,216,2017-03-20 05:39:48,http://sanford.io/davon,,37.10.115.136,"{""location"": ""MM"", ""is_mobile"": true}" 9628,4,216,2017-04-25 09:28:48,http://harvey.info/brenna,,126.217.21.129,"{""location"": ""DM"", ""is_mobile"": false}" 9629,4,216,2017-02-27 17:53:20,http://hilll.co/marianne_marks,,74.207.13.134,"{""location"": ""PL"", ""is_mobile"": false}" 9630,4,216,2016-12-14 09:45:50,http://kubpfannerstill.io/rozella.langworth,,173.104.252.234,"{""location"": ""LR"", ""is_mobile"": true}" 9631,4,216,2017-01-16 02:19:29,http://haley.org/nya,,105.166.206.121,"{""location"": ""VG"", ""is_mobile"": true}" 9632,4,216,2017-04-22 12:36:33,http://gaylord.org/kayden,,189.205.60.140,"{""location"": ""SJ"", ""is_mobile"": true}" 9633,4,216,2017-05-19 15:19:27,http://leannon.com/liana.mills,,254.92.217.51,"{""location"": ""BT"", ""is_mobile"": true}" 9634,4,216,2017-05-21 17:44:47,http://cain.net/forest,,54.23.124.170,"{""location"": ""BS"", ""is_mobile"": false}" 9635,4,216,2017-01-21 04:59:19,http://dach.org/ena.blick,,188.63.109.193,"{""location"": ""WF"", ""is_mobile"": false}" 9636,4,216,2016-12-16 04:27:07,http://rogahn.biz/nichole_jast,,215.47.100.221,"{""location"": ""DE"", ""is_mobile"": true}" 9637,4,216,2017-04-01 10:49:25,http://schillergusikowski.biz/tillman,,111.224.62.253,"{""location"": ""IO"", ""is_mobile"": true}" 9638,4,216,2016-12-17 21:56:26,http://kautzergusikowski.biz/elya.klocko,,190.50.70.146,"{""location"": ""LU"", ""is_mobile"": false}" 9639,4,216,2016-12-23 10:29:47,http://lemke.io/julian,,50.155.198.12,"{""location"": ""LU"", ""is_mobile"": false}" 9640,4,216,2016-12-29 05:09:31,http://brakusraynor.net/amara_weimann,,156.92.135.61,"{""location"": ""AW"", ""is_mobile"": false}" 9641,4,216,2017-02-14 23:50:39,http://cain.biz/cristian,,147.118.133.165,"{""location"": ""MQ"", ""is_mobile"": true}" 9642,4,216,2017-04-03 19:13:34,http://leffler.io/eulah,,189.61.253.56,"{""location"": ""ES"", ""is_mobile"": true}" 9643,4,216,2017-01-14 16:45:10,http://carternienow.co/lelia_bergstrom,,121.172.136.252,"{""location"": ""BV"", ""is_mobile"": false}" 9644,4,216,2017-01-09 04:04:00,http://lubowitz.net/price.carter,,238.253.134.173,"{""location"": ""AD"", ""is_mobile"": false}" 9645,4,216,2017-05-06 12:28:00,http://reillygreenfelder.co/merlin,,60.122.162.101,"{""location"": ""ZA"", ""is_mobile"": true}" 9646,4,216,2017-02-26 01:47:16,http://feil.biz/rowena.friesen,,131.33.151.65,"{""location"": ""GA"", ""is_mobile"": false}" 9647,4,216,2017-05-12 12:34:19,http://dietrich.io/maybell_corkery,,182.93.87.61,"{""location"": ""VC"", ""is_mobile"": false}" 9648,4,216,2017-04-24 05:02:34,http://yundt.name/antonina,,56.241.217.82,"{""location"": ""TL"", ""is_mobile"": true}" 9649,4,216,2017-02-18 15:20:37,http://olson.biz/rolando,,36.237.195.72,"{""location"": ""CC"", ""is_mobile"": false}" 9650,4,216,2017-01-29 05:27:50,http://hartmann.info/lois.bradtke,,77.69.80.70,"{""location"": ""LS"", ""is_mobile"": false}" 9651,4,217,2017-06-01 06:54:43,http://howell.name/mercedes_upton,,250.185.179.123,"{""location"": ""MC"", ""is_mobile"": false}" 9652,4,217,2017-04-27 03:19:22,http://paucek.io/ambrose,,41.207.75.156,"{""location"": ""DO"", ""is_mobile"": true}" 9653,4,217,2017-03-18 01:31:12,http://cain.com/kole,,162.60.55.213,"{""location"": ""CF"", ""is_mobile"": true}" 9654,4,217,2017-03-22 16:44:01,http://gislasonmaggio.biz/kayla,,156.30.46.8,"{""location"": ""ES"", ""is_mobile"": false}" 9655,4,217,2017-01-27 13:54:06,http://schoenshanahan.name/joana_botsford,,145.227.194.224,"{""location"": ""TF"", ""is_mobile"": true}" 9656,4,217,2017-04-19 12:11:15,http://vandervortskiles.co/providenci.senger,,208.120.8.149,"{""location"": ""NR"", ""is_mobile"": false}" 9657,4,217,2017-04-17 10:44:34,http://reichertberge.org/julianne_ritchie,,197.60.178.150,"{""location"": ""LA"", ""is_mobile"": false}" 9658,4,217,2017-06-07 10:54:40,http://borer.co/bernadine_willms,,41.222.57.51,"{""location"": ""PA"", ""is_mobile"": false}" 9659,4,217,2017-01-12 15:28:36,http://dietrich.name/jeff,,218.233.77.163,"{""location"": ""ST"", ""is_mobile"": true}" 9660,4,217,2017-04-10 23:25:42,http://huels.co/holly,,16.165.139.168,"{""location"": ""KI"", ""is_mobile"": true}" 9661,4,217,2017-03-09 23:10:17,http://welchquigley.net/mavis_cole,,240.88.223.177,"{""location"": ""TV"", ""is_mobile"": true}" 9662,4,217,2017-04-30 09:46:07,http://bruen.biz/gilda_hand,,233.131.210.26,"{""location"": ""HT"", ""is_mobile"": false}" 9663,4,217,2017-04-01 18:19:04,http://hettinger.info/eloise.mclaughlin,,134.214.230.205,"{""location"": ""NC"", ""is_mobile"": true}" 9664,4,217,2016-12-14 01:05:42,http://metz.com/thalia,,31.87.228.153,"{""location"": ""IQ"", ""is_mobile"": true}" 9665,4,217,2017-04-28 19:54:38,http://runtebrown.info/maya.haag,,210.181.213.99,"{""location"": ""CU"", ""is_mobile"": true}" 9666,4,217,2017-04-12 16:13:31,http://bechtelar.name/celestine_hintz,,148.35.88.121,"{""location"": ""ST"", ""is_mobile"": true}" 9667,4,217,2017-05-11 19:57:07,http://williamson.name/herbert_reilly,,98.164.111.225,"{""location"": ""IL"", ""is_mobile"": true}" 9668,4,217,2017-01-17 02:57:31,http://mitchellconsidine.info/wallace,,57.210.119.150,"{""location"": ""CU"", ""is_mobile"": true}" 9669,4,217,2017-01-15 08:15:59,http://moennicolas.org/miguel_konopelski,,127.137.202.140,"{""location"": ""HN"", ""is_mobile"": true}" 9670,4,217,2016-12-29 18:18:59,http://koch.io/arturo.gottlieb,,247.207.56.237,"{""location"": ""JO"", ""is_mobile"": true}" 9671,4,217,2017-01-27 09:14:33,http://waters.com/rosetta.rohan,,64.195.192.53,"{""location"": ""TV"", ""is_mobile"": false}" 9672,4,217,2017-02-03 01:21:47,http://wardgreenfelder.org/caria,,194.179.68.197,"{""location"": ""GQ"", ""is_mobile"": true}" 9673,4,217,2017-03-30 05:05:51,http://schroedergottlieb.org/ulises,,188.29.35.27,"{""location"": ""OM"", ""is_mobile"": false}" 9674,4,217,2017-05-24 07:43:51,http://bergnaum.info/keely.leffler,,129.122.187.140,"{""location"": ""FR"", ""is_mobile"": false}" 6616,3,143,2016-12-23 08:11:25,http://koelpin.info/nia,0.3844630263,60.46.248.244,"{""location"": ""DZ"", ""is_mobile"": true}" 6617,3,143,2017-03-07 17:13:39,http://torp.org/morgan,0.9385344047,168.11.250.221,"{""location"": ""NG"", ""is_mobile"": true}" 6618,3,143,2017-04-17 21:20:16,http://carter.name/cristal_strosin,0.1928578701,203.58.154.118,"{""location"": ""CI"", ""is_mobile"": false}" 6619,3,143,2017-01-15 06:38:53,http://walkerlarson.com/gertrude,0.2845837043,200.19.25.102,"{""location"": ""CD"", ""is_mobile"": false}" 6620,3,143,2017-04-28 18:52:07,http://turcotte.biz/johnnie_sawayn,0.0165610671,195.5.26.71,"{""location"": ""AZ"", ""is_mobile"": false}" 6621,3,144,2017-05-13 09:10:40,http://ritchie.net/angel,0.1618870814,58.119.240.164,"{""location"": ""GG"", ""is_mobile"": true}" 6622,3,144,2017-04-29 10:39:24,http://kertzmannlebsack.name/felton.purdy,0.0445990740,3.165.19.19,"{""location"": ""IQ"", ""is_mobile"": false}" 6623,3,144,2017-02-04 22:44:38,http://harberheaney.net/erica,0.8706823444,198.10.71.41,"{""location"": ""SE"", ""is_mobile"": false}" 6624,3,144,2016-12-14 06:07:26,http://lang.org/alexandria,0.6867206713,220.239.71.129,"{""location"": ""TL"", ""is_mobile"": false}" 6625,3,144,2017-02-25 04:51:52,http://cronin.info/maryam_oconner,0.2750111440,182.158.61.49,"{""location"": ""BI"", ""is_mobile"": true}" 6626,3,144,2017-02-12 11:43:00,http://crona.io/sydnie,0.7335374390,86.39.214.16,"{""location"": ""DK"", ""is_mobile"": true}" 6627,3,144,2017-04-01 15:03:46,http://dickens.name/brittany,0.9224148443,195.73.152.202,"{""location"": ""VC"", ""is_mobile"": false}" 6628,3,144,2017-04-19 17:15:42,http://kihn.org/blanche,0.2203803436,104.85.57.71,"{""location"": ""CK"", ""is_mobile"": false}" 6629,3,144,2017-05-14 06:33:37,http://kelerkunze.co/lola_larkin,0.6038576567,230.141.47.235,"{""location"": ""AX"", ""is_mobile"": true}" 6630,3,144,2017-04-21 01:16:12,http://macejkovicgreenholt.net/santino,0.6924042411,35.79.81.26,"{""location"": ""IS"", ""is_mobile"": false}" 6631,3,144,2017-02-02 16:58:14,http://zboncakondricka.net/zack,0.1599982152,108.49.165.57,"{""location"": ""CM"", ""is_mobile"": false}" 6632,3,144,2017-06-02 14:55:19,http://schoen.com/nathanial,0.5161636441,48.223.77.21,"{""location"": ""NE"", ""is_mobile"": true}" 6633,3,144,2017-04-09 08:22:06,http://manncole.com/efrain,0.3037949951,59.154.106.34,"{""location"": ""BT"", ""is_mobile"": true}" 6634,3,144,2017-01-19 10:35:15,http://kshlerin.info/carmel,0.5467559139,72.228.125.199,"{""location"": ""UZ"", ""is_mobile"": true}" 6635,3,144,2017-05-23 04:23:18,http://doyle.com/sydni,0.9311953274,113.101.65.244,"{""location"": ""GM"", ""is_mobile"": true}" 6636,3,144,2017-04-14 10:13:21,http://oconnell.io/matilde_jenkins,0.6690599839,205.155.177.227,"{""location"": ""PM"", ""is_mobile"": true}" 6637,3,144,2017-01-17 14:55:14,http://mccullough.name/marielle_breitenberg,0.1094189471,12.217.129.67,"{""location"": ""ER"", ""is_mobile"": false}" 6638,3,144,2017-04-17 06:48:51,http://mcglynn.com/earline,0.0708567131,176.187.27.46,"{""location"": ""CL"", ""is_mobile"": true}" 6639,3,144,2017-02-24 18:49:19,http://darebauch.io/destiny.baumbach,0.2542724202,56.230.163.86,"{""location"": ""BT"", ""is_mobile"": false}" 6640,3,144,2017-01-17 20:50:57,http://kutchkuhn.org/elza,0.4079738422,41.191.191.174,"{""location"": ""LV"", ""is_mobile"": false}" 6641,3,144,2017-01-11 13:24:47,http://watsica.org/torrance,0.9512958894,189.68.60.103,"{""location"": ""NA"", ""is_mobile"": true}" 6642,3,144,2017-04-26 19:33:50,http://walterpredovic.biz/dangelo,0.9411036089,2.153.161.54,"{""location"": ""KG"", ""is_mobile"": false}" 6643,3,144,2017-04-27 08:24:46,http://schowalterdonnelly.co/tyrel,0.8244558683,206.183.106.187,"{""location"": ""CM"", ""is_mobile"": true}" 6644,3,144,2017-01-22 03:36:17,http://connelly.com/aliza_dietrich,0.8745942777,51.126.100.120,"{""location"": ""FK"", ""is_mobile"": true}" 6645,3,145,2017-05-04 14:41:36,http://harbernicolas.biz/easton,0.0387648209,137.138.74.186,"{""location"": ""GQ"", ""is_mobile"": false}" 6646,3,145,2017-01-21 10:51:47,http://hermannschmeler.biz/tristin,0.0654599151,65.111.173.221,"{""location"": ""MF"", ""is_mobile"": true}" 6647,3,145,2016-12-25 06:36:54,http://okonruecker.name/helena,0.2489352568,127.177.20.5,"{""location"": ""GT"", ""is_mobile"": true}" 6648,3,145,2017-05-06 04:21:39,http://dickens.com/maia.emmerich,0.8166643130,200.62.233.200,"{""location"": ""BG"", ""is_mobile"": false}" 6649,3,145,2017-04-08 07:50:30,http://lindconnelly.info/ezequiel.nader,0.5778004120,197.116.6.248,"{""location"": ""NU"", ""is_mobile"": true}" 6650,3,145,2017-02-10 02:27:17,http://kemmer.org/celestino,0.4232851047,234.49.209.78,"{""location"": ""NU"", ""is_mobile"": true}" 6651,3,145,2017-02-13 22:52:56,http://crooks.io/maritza,0.6641849413,190.96.163.188,"{""location"": ""MD"", ""is_mobile"": true}" 6652,3,145,2017-06-03 11:53:56,http://goldner.co/bethel.krajcik,0.8628365967,101.222.73.108,"{""location"": ""CK"", ""is_mobile"": true}" 6653,3,145,2017-03-27 08:37:44,http://ziemeoconner.org/johan,0.8574671927,81.22.125.253,"{""location"": ""HR"", ""is_mobile"": true}" 6654,3,145,2017-03-17 11:53:30,http://douglaswisoky.co/gavin.hane,0.2849749160,32.251.191.74,"{""location"": ""AT"", ""is_mobile"": true}" 6655,3,145,2017-05-22 17:43:52,http://abshiremurray.net/francesca,0.2225947285,204.209.181.47,"{""location"": ""JP"", ""is_mobile"": true}" 6656,3,145,2017-05-15 22:28:10,http://cronin.name/buck,0.0953272161,214.167.237.39,"{""location"": ""SM"", ""is_mobile"": true}" 6657,3,145,2017-01-22 01:33:49,http://hegmann.biz/horace.oconnell,0.5125026500,133.37.233.236,"{""location"": ""MQ"", ""is_mobile"": false}" 6658,3,145,2017-03-13 16:00:53,http://haag.com/fannie.cruickshank,0.5726887119,251.110.79.249,"{""location"": ""QA"", ""is_mobile"": false}" 6659,3,145,2017-01-17 16:40:28,http://williamsonhintz.org/leila_hane,0.6097342789,106.49.243.37,"{""location"": ""GQ"", ""is_mobile"": false}" 6660,3,145,2017-05-03 01:16:13,http://schroeder.com/jayne,0.6067090805,213.15.10.84,"{""location"": ""AX"", ""is_mobile"": true}" 6661,3,145,2017-04-16 22:32:58,http://batzmiller.info/peyton,0.1481871247,232.145.165.225,"{""location"": ""NA"", ""is_mobile"": false}" 6662,3,145,2017-03-28 02:59:51,http://weimann.info/jackeline_rogahn,0.8464249725,131.60.227.228,"{""location"": ""LV"", ""is_mobile"": false}" 6663,3,145,2017-02-15 01:45:15,http://beahanmaggio.info/erwin.orn,0.9434261224,134.152.59.173,"{""location"": ""VG"", ""is_mobile"": true}" 6664,3,145,2017-05-22 17:39:49,http://homenickconn.org/bradly,0.3085766595,182.56.21.128,"{""location"": ""KZ"", ""is_mobile"": true}" 6665,3,145,2017-03-12 21:11:48,http://powlowski.com/ernie_mills,0.2776174995,122.113.96.42,"{""location"": ""TF"", ""is_mobile"": true}" 6666,3,145,2017-04-10 16:54:24,http://marks.co/wilson_west,0.2983725679,145.147.153.103,"{""location"": ""HM"", ""is_mobile"": false}" 6667,3,145,2017-01-14 05:25:23,http://crist.net/lurline,0.5795780862,218.177.141.241,"{""location"": ""FJ"", ""is_mobile"": true}" 12527,5,280,2016-12-29 10:42:51,http://konopelski.info/reta,,226.97.86.66,"{""location"": ""FM"", ""is_mobile"": true}" 12528,5,280,2017-03-26 23:35:49,http://steuber.com/tanya.balistreri,,181.231.111.41,"{""location"": ""SB"", ""is_mobile"": true}" 12529,5,280,2017-01-11 03:38:11,http://gradyfisher.info/abigayle,,5.79.102.34,"{""location"": ""SM"", ""is_mobile"": true}" 12530,5,280,2017-05-26 15:33:49,http://daniel.net/sarah,,79.21.181.192,"{""location"": ""UZ"", ""is_mobile"": false}" 12531,5,280,2017-05-12 18:09:23,http://mcclureheidenreich.info/bart_schamberger,,124.109.148.203,"{""location"": ""GS"", ""is_mobile"": false}" 12532,5,280,2016-12-22 15:25:27,http://pfannerstill.name/ardella.mueller,,52.127.181.221,"{""location"": ""IO"", ""is_mobile"": true}" 12533,5,280,2017-02-19 18:38:54,http://shields.net/tobin,,134.24.206.31,"{""location"": ""CZ"", ""is_mobile"": false}" 12534,5,280,2017-01-29 20:41:32,http://collins.co/theo_johnson,,252.203.191.117,"{""location"": ""PE"", ""is_mobile"": false}" 12535,5,281,2017-01-24 10:48:03,http://stracke.org/cathrine.mayer,,143.99.228.203,"{""location"": ""BG"", ""is_mobile"": false}" 12536,5,281,2017-04-03 21:09:52,http://frami.io/ron.predovic,,49.10.205.15,"{""location"": ""MU"", ""is_mobile"": false}" 12537,5,281,2017-02-06 13:44:34,http://danielpfannerstill.biz/morton,,67.3.70.115,"{""location"": ""IE"", ""is_mobile"": false}" 12538,5,281,2017-01-24 20:34:46,http://wintheiser.name/arielle,,29.73.29.240,"{""location"": ""PG"", ""is_mobile"": true}" 12539,5,281,2017-05-30 00:25:53,http://kilback.io/emerald.schoen,,35.179.16.138,"{""location"": ""IL"", ""is_mobile"": true}" 12540,5,281,2017-03-02 17:41:04,http://yundtmcglynn.co/kelli,,199.92.77.107,"{""location"": ""HM"", ""is_mobile"": false}" 12541,5,281,2017-03-24 07:20:19,http://quigley.io/marcella,,231.77.84.67,"{""location"": ""CA"", ""is_mobile"": false}" 12542,5,281,2017-01-19 16:09:18,http://raynor.net/ivah_waters,,94.132.156.170,"{""location"": ""MU"", ""is_mobile"": true}" 12543,5,281,2017-04-10 11:05:24,http://white.io/ben,,6.58.143.99,"{""location"": ""CM"", ""is_mobile"": true}" 12544,5,281,2017-01-09 07:40:38,http://rath.io/elda_jacobs,,166.184.61.214,"{""location"": ""RS"", ""is_mobile"": false}" 12545,5,281,2016-12-17 06:05:10,http://gerlachdooley.info/dagmar.romaguera,,99.20.17.69,"{""location"": ""TH"", ""is_mobile"": true}" 12546,5,281,2017-01-01 01:25:18,http://langoshbradtke.name/magnus,,238.174.98.16,"{""location"": ""AL"", ""is_mobile"": true}" 12547,5,281,2017-01-18 21:37:43,http://rau.org/asha.crona,,98.14.63.13,"{""location"": ""TC"", ""is_mobile"": true}" 12548,5,281,2017-02-26 06:18:18,http://keeling.io/gudrun.heidenreich,,23.117.139.94,"{""location"": ""RO"", ""is_mobile"": false}" 12549,5,281,2017-03-10 22:27:22,http://weber.com/lew.powlowski,,130.198.128.98,"{""location"": ""UM"", ""is_mobile"": false}" 12550,5,281,2017-05-06 10:46:41,http://roberts.org/ethyl,,134.5.192.109,"{""location"": ""KM"", ""is_mobile"": false}" 12551,5,281,2017-04-17 19:16:14,http://douglas.org/steve.schumm,,111.171.82.195,"{""location"": ""KW"", ""is_mobile"": true}" 12552,5,281,2017-04-08 11:25:54,http://yundt.org/clint,,111.41.230.229,"{""location"": ""BL"", ""is_mobile"": false}" 12553,5,281,2017-01-09 08:45:53,http://koelpinhauck.biz/darrel,,160.133.247.12,"{""location"": ""SV"", ""is_mobile"": true}" 12554,5,281,2017-02-20 04:59:58,http://cremin.biz/rosario_bode,,23.72.43.61,"{""location"": ""BG"", ""is_mobile"": false}" 12555,5,281,2017-02-23 02:31:16,http://blockbosco.co/monique,,241.143.86.194,"{""location"": ""GA"", ""is_mobile"": true}" 12556,5,281,2017-03-31 01:34:31,http://kuhic.com/shyann,,216.120.118.50,"{""location"": ""AU"", ""is_mobile"": false}" 12557,5,281,2017-01-18 07:15:01,http://ondricka.net/ettie,,165.240.158.76,"{""location"": ""BL"", ""is_mobile"": true}" 12558,5,281,2017-06-07 01:55:58,http://okuneva.org/aliya,,250.181.116.96,"{""location"": ""CH"", ""is_mobile"": true}" 12559,5,281,2017-04-27 18:47:44,http://schultzbauch.info/shayna,,100.141.11.238,"{""location"": ""MQ"", ""is_mobile"": false}" 12560,5,281,2017-06-03 15:33:56,http://mosciski.name/marcella.hand,,189.248.20.179,"{""location"": ""CO"", ""is_mobile"": false}" 12561,5,281,2016-12-15 09:43:53,http://gleason.info/al,,48.200.52.178,"{""location"": ""AO"", ""is_mobile"": true}" 12562,5,281,2017-06-11 22:02:24,http://langoshwillms.net/madeline.balistreri,,122.2.4.102,"{""location"": ""MD"", ""is_mobile"": false}" 12563,5,281,2017-03-17 05:40:07,http://nienow.com/kitty_koepp,,192.3.80.205,"{""location"": ""MN"", ""is_mobile"": true}" 12564,5,281,2017-02-13 06:56:30,http://ferry.co/shad.nienow,,38.125.211.247,"{""location"": ""SH"", ""is_mobile"": false}" 12565,5,281,2017-01-02 10:39:21,http://blickjohns.co/javonte_fadel,,59.23.228.251,"{""location"": ""AX"", ""is_mobile"": false}" 12566,5,281,2017-06-08 20:50:06,http://buckridgepfannerstill.io/dereck_bode,,121.228.123.252,"{""location"": ""CR"", ""is_mobile"": true}" 12567,5,281,2017-04-03 11:01:48,http://mcclurebrakus.net/gudrun,,10.207.123.253,"{""location"": ""MP"", ""is_mobile"": true}" 12568,5,281,2017-05-20 03:20:25,http://hermiston.co/morris,,56.150.184.212,"{""location"": ""QA"", ""is_mobile"": true}" 12569,5,281,2017-05-17 02:58:44,http://jakubowski.biz/brandy,,118.146.210.201,"{""location"": ""ZW"", ""is_mobile"": false}" 12570,5,281,2017-02-10 16:09:11,http://grimetracke.biz/kobe,,132.68.214.248,"{""location"": ""NG"", ""is_mobile"": true}" 12571,5,281,2017-01-28 00:20:57,http://ziemanncollins.name/hester_howell,,144.223.8.198,"{""location"": ""JO"", ""is_mobile"": true}" 12572,5,281,2017-02-03 09:21:17,http://konopelskikirlin.net/alphonso.kaulke,,249.232.145.84,"{""location"": ""FM"", ""is_mobile"": false}" 12573,5,281,2017-06-02 17:40:01,http://powlowski.io/emory_streich,,46.227.175.89,"{""location"": ""EG"", ""is_mobile"": true}" 12574,5,281,2016-12-18 19:34:14,http://langosh.info/amari.brekke,,5.2.128.94,"{""location"": ""UY"", ""is_mobile"": true}" 12575,5,281,2017-02-03 10:22:40,http://schuppe.info/casey_fay,,18.245.124.217,"{""location"": ""AS"", ""is_mobile"": true}" 12576,5,281,2016-12-17 06:15:31,http://hermanemmerich.net/annamarie_hagenes,,254.203.236.245,"{""location"": ""JO"", ""is_mobile"": false}" 12577,5,281,2017-05-25 15:32:46,http://spencer.biz/jett,,25.246.245.55,"{""location"": ""CN"", ""is_mobile"": false}" 12578,5,281,2017-01-06 18:28:34,http://legros.net/maiya_doyle,,226.33.83.105,"{""location"": ""JO"", ""is_mobile"": true}" 12579,5,281,2017-03-22 15:49:10,http://hirthe.io/reed.schaden,,126.112.164.57,"{""location"": ""SI"", ""is_mobile"": false}" 12580,5,281,2017-05-28 04:47:57,http://price.io/viva,,164.123.171.152,"{""location"": ""BB"", ""is_mobile"": false}" 12581,5,281,2017-02-10 03:12:01,http://romagueragrant.info/arlo_crona,,124.189.230.148,"{""location"": ""LR"", ""is_mobile"": false}" 12582,5,281,2017-03-17 05:46:30,http://hirtheschmidt.biz/madison,,180.245.72.72,"{""location"": ""AD"", ""is_mobile"": true}" 15534,6,349,2017-04-05 21:46:19,http://klein.co/gennaro,0.5580615877,140.6.31.213,"{""location"": ""SZ"", ""is_mobile"": false}" 15535,6,349,2017-01-10 09:18:47,http://kuhic.name/amelia_blanda,0.7318310513,84.13.85.219,"{""location"": ""SM"", ""is_mobile"": true}" 15536,6,349,2017-01-01 15:47:33,http://hand.com/myrtle,0.6863038324,224.172.56.218,"{""location"": ""QA"", ""is_mobile"": false}" 15537,6,349,2017-02-10 08:55:24,http://walterlynch.info/felicia.jacobson,0.2771948797,20.82.74.240,"{""location"": ""MV"", ""is_mobile"": false}" 15538,6,349,2017-01-02 09:29:59,http://torphy.io/darian,0.4637163134,133.139.17.163,"{""location"": ""KE"", ""is_mobile"": false}" 15539,6,349,2017-05-14 00:24:17,http://rau.net/leslie,0.3436317685,125.159.82.214,"{""location"": ""DM"", ""is_mobile"": true}" 15540,6,349,2016-12-24 09:46:42,http://colezieme.io/kayli.stracke,0.2245251956,11.152.181.10,"{""location"": ""TW"", ""is_mobile"": false}" 15541,6,349,2017-05-13 21:16:56,http://schuster.info/dante.conn,0.1849811381,67.244.213.151,"{""location"": ""JM"", ""is_mobile"": false}" 15542,6,349,2017-02-20 23:20:37,http://jacobswilliamson.biz/cheyanne_braun,0.4461908160,129.21.102.206,"{""location"": ""KZ"", ""is_mobile"": true}" 15543,6,349,2017-04-12 09:34:51,http://heaney.com/jerad,0.5561262621,36.142.109.165,"{""location"": ""BB"", ""is_mobile"": false}" 15544,6,349,2017-02-05 06:50:54,http://yost.net/gay,0.5009781839,30.134.170.19,"{""location"": ""GY"", ""is_mobile"": false}" 15545,6,349,2017-04-30 10:05:02,http://waters.net/domenick_hane,0.0214952193,46.107.6.134,"{""location"": ""AE"", ""is_mobile"": true}" 15546,6,349,2016-12-14 03:29:33,http://bruen.com/jordan.hilpert,0.4299121915,13.250.161.230,"{""location"": ""ST"", ""is_mobile"": false}" 15547,6,349,2017-06-01 19:14:53,http://erdmanwalter.co/ottilie_braun,0.3788584718,128.50.172.111,"{""location"": ""LS"", ""is_mobile"": true}" 15548,6,349,2017-01-02 11:02:14,http://damore.net/junior,0.4588030545,223.217.55.166,"{""location"": ""TM"", ""is_mobile"": false}" 15549,6,349,2017-05-30 21:39:29,http://mohrwiza.info/jayda,0.1796893586,116.54.64.97,"{""location"": ""DO"", ""is_mobile"": false}" 15550,6,349,2017-04-10 14:50:57,http://schuppe.biz/georgette,0.6812837887,168.50.112.244,"{""location"": ""PF"", ""is_mobile"": true}" 15551,6,349,2016-12-28 05:10:13,http://stoltenberg.name/mauricio.pagac,0.9919936520,190.98.4.211,"{""location"": ""PE"", ""is_mobile"": true}" 15552,6,349,2017-05-03 12:50:06,http://bayer.name/kris_rau,0.8629439953,57.142.19.90,"{""location"": ""IE"", ""is_mobile"": false}" 15553,6,349,2017-01-05 05:30:16,http://willmhields.name/mckenzie,0.3806188393,10.82.28.101,"{""location"": ""AG"", ""is_mobile"": true}" 15554,6,349,2017-01-08 15:47:39,http://rolfson.net/ezra,0.7136891866,224.8.100.63,"{""location"": ""MO"", ""is_mobile"": false}" 15555,6,349,2017-06-01 05:33:37,http://oconnellkoch.name/ubaldo.macgyver,0.8908210155,228.177.78.130,"{""location"": ""VN"", ""is_mobile"": true}" 15556,6,349,2017-02-16 23:48:35,http://damore.org/clementine,0.9220386329,38.172.230.110,"{""location"": ""DO"", ""is_mobile"": true}" 15557,6,349,2017-03-18 01:05:43,http://weinatschinner.org/petra,0.7874413588,250.208.4.101,"{""location"": ""ST"", ""is_mobile"": true}" 15558,6,350,2017-05-12 14:01:48,http://boscometz.com/jaunita,0.9961574866,199.212.123.95,"{""location"": ""KZ"", ""is_mobile"": true}" 15559,6,350,2017-02-09 11:28:59,http://schillerfisher.io/elna,0.4830711816,205.13.95.42,"{""location"": ""DO"", ""is_mobile"": true}" 15560,6,350,2017-03-13 04:03:51,http://ryan.org/lois_prohaska,0.3426683149,176.111.151.52,"{""location"": ""CA"", ""is_mobile"": true}" 15561,6,350,2017-01-04 05:21:04,http://weimann.biz/alexis_blanda,0.7671297124,179.78.232.172,"{""location"": ""BF"", ""is_mobile"": false}" 15562,6,350,2017-02-16 16:44:58,http://mcglynn.com/spencer.schneider,0.2825364054,23.87.149.216,"{""location"": ""AM"", ""is_mobile"": false}" 15563,6,350,2017-06-06 08:40:49,http://lakin.org/elta_legros,0.0589057194,101.129.119.221,"{""location"": ""KZ"", ""is_mobile"": false}" 15564,6,350,2017-02-03 09:27:04,http://balistrerikozey.com/name,0.5543355010,146.134.42.11,"{""location"": ""CX"", ""is_mobile"": true}" 15565,6,350,2017-05-27 10:27:20,http://keelingbrown.name/llewellyn.legros,0.5455010757,95.121.234.220,"{""location"": ""ST"", ""is_mobile"": true}" 15566,6,350,2017-03-02 10:14:42,http://reilly.net/tianna,0.3311596295,151.15.248.149,"{""location"": ""KP"", ""is_mobile"": true}" 15567,6,350,2017-02-18 05:27:01,http://homenicksipes.io/marina,0.4047344522,226.137.29.37,"{""location"": ""DK"", ""is_mobile"": false}" 15568,6,350,2017-02-18 04:43:30,http://okuneva.io/naomie.lakin,0.7082532615,160.53.31.165,"{""location"": ""KN"", ""is_mobile"": false}" 15569,6,350,2017-02-28 06:33:29,http://strosin.name/pierce_borer,0.0650436409,145.123.169.227,"{""location"": ""CU"", ""is_mobile"": true}" 15570,6,350,2017-06-11 15:25:15,http://jerdeklein.name/darrel,0.8759895258,71.119.82.21,"{""location"": ""TM"", ""is_mobile"": true}" 15571,6,350,2017-01-26 19:19:03,http://dibbert.info/garfield.dooley,0.8143404711,129.89.3.209,"{""location"": ""CC"", ""is_mobile"": false}" 15572,6,350,2017-05-13 15:17:54,http://johnson.org/brody,0.0707917550,124.123.64.153,"{""location"": ""TF"", ""is_mobile"": true}" 15573,6,350,2017-01-09 00:07:15,http://hackett.io/emmet_reilly,0.0487651569,68.16.181.120,"{""location"": ""NI"", ""is_mobile"": true}" 15574,6,350,2017-03-02 04:04:58,http://emmerich.co/dominique_johns,0.1236202720,139.150.100.157,"{""location"": ""AD"", ""is_mobile"": false}" 15575,6,350,2017-01-01 21:08:53,http://hoppe.name/mariana.hilll,0.0461736478,127.63.130.77,"{""location"": ""AL"", ""is_mobile"": true}" 15576,6,350,2017-06-03 08:47:04,http://prohaska.info/zoey,0.4754812287,95.4.184.2,"{""location"": ""NG"", ""is_mobile"": false}" 15577,6,350,2017-02-26 08:49:45,http://kilback.com/jerry,0.3968286383,58.110.106.190,"{""location"": ""LI"", ""is_mobile"": true}" 15578,6,350,2017-03-29 20:29:47,http://brekkestreich.org/hailee,0.6179057588,190.165.53.217,"{""location"": ""AI"", ""is_mobile"": true}" 15579,6,350,2017-02-23 02:33:32,http://huel.biz/izabella,0.8130352320,40.115.95.189,"{""location"": ""CM"", ""is_mobile"": false}" 15580,6,350,2017-01-24 20:35:01,http://ferry.info/adonis.wunsch,0.9979639497,64.224.242.215,"{""location"": ""SD"", ""is_mobile"": true}" 15581,6,350,2017-05-01 06:40:06,http://harvey.info/payton,0.6961932886,197.87.169.233,"{""location"": ""MW"", ""is_mobile"": false}" 15582,6,350,2017-06-13 15:42:59,http://larkin.co/deron_wolf,0.5145331095,209.235.61.210,"{""location"": ""QA"", ""is_mobile"": true}" 15583,6,350,2017-04-07 03:04:34,http://schamberger.net/furman,0.3325312333,166.121.117.222,"{""location"": ""SE"", ""is_mobile"": true}" 15584,6,350,2017-02-09 02:05:21,http://hahn.info/janea,0.4238055755,126.222.42.27,"{""location"": ""FO"", ""is_mobile"": false}" 2740,2,60,2017-02-15 09:25:39,http://balistreri.co/efrain.weber,,240.69.237.77,"{""location"": ""AG"", ""is_mobile"": false}" 2741,2,60,2017-02-06 16:54:40,http://hackett.io/adrien,,254.28.227.205,"{""location"": ""AG"", ""is_mobile"": false}" 2742,2,60,2017-03-12 13:46:28,http://renner.net/candido.raynor,,82.8.227.48,"{""location"": ""BM"", ""is_mobile"": true}" 2743,2,60,2017-01-24 15:01:58,http://nadernicolas.io/michale,,49.10.200.6,"{""location"": ""BH"", ""is_mobile"": true}" 2744,2,60,2016-12-14 09:50:32,http://kiehnshields.name/alvah,,112.237.199.112,"{""location"": ""TK"", ""is_mobile"": false}" 2745,2,60,2017-03-19 01:45:20,http://zboncakstanton.org/muriel,,120.52.202.141,"{""location"": ""ME"", ""is_mobile"": true}" 2746,2,61,2017-02-09 03:35:14,http://rippin.org/lavina_stanton,,155.213.4.84,"{""location"": ""TN"", ""is_mobile"": true}" 2747,2,61,2017-04-02 11:06:47,http://leuschke.io/gustave,,250.250.253.117,"{""location"": ""CH"", ""is_mobile"": true}" 2748,2,61,2017-03-27 10:44:38,http://collins.info/melvina,,177.100.105.235,"{""location"": ""SM"", ""is_mobile"": true}" 2749,2,61,2016-12-27 16:45:39,http://gulgowski.co/herminia,,155.157.69.198,"{""location"": ""TF"", ""is_mobile"": false}" 2750,2,61,2016-12-22 12:25:45,http://lakin.com/wyatt.lindgren,,94.106.236.199,"{""location"": ""AI"", ""is_mobile"": true}" 2751,2,61,2017-03-08 04:29:25,http://turcottekemmer.net/garnett.wunsch,,28.224.240.211,"{""location"": ""ES"", ""is_mobile"": true}" 2752,2,61,2017-03-08 13:51:50,http://mann.net/kenyon,,180.30.86.113,"{""location"": ""BD"", ""is_mobile"": false}" 2753,2,61,2016-12-22 19:15:53,http://becker.org/orie,,158.81.9.76,"{""location"": ""EC"", ""is_mobile"": true}" 2754,2,61,2017-02-22 07:58:06,http://hills.net/nicklaus_wolff,,91.203.193.144,"{""location"": ""SJ"", ""is_mobile"": true}" 2755,2,61,2017-05-12 06:51:14,http://gleasonhodkiewicz.biz/clement,,99.158.37.81,"{""location"": ""BE"", ""is_mobile"": true}" 2756,2,61,2017-03-30 12:30:35,http://smith.name/rosalee,,79.2.100.161,"{""location"": ""AT"", ""is_mobile"": false}" 2757,2,61,2017-03-26 19:23:47,http://mitchell.net/heather_hickle,,72.224.151.246,"{""location"": ""TO"", ""is_mobile"": false}" 2758,2,61,2017-04-02 11:26:20,http://lemke.biz/marjolaine_hand,,12.39.11.185,"{""location"": ""WS"", ""is_mobile"": false}" 2759,2,61,2017-04-17 06:52:11,http://jacobizboncak.com/kenny_bailey,,223.246.197.171,"{""location"": ""FI"", ""is_mobile"": false}" 2760,2,61,2017-03-11 09:39:35,http://hermannschaden.com/benton_upton,,91.78.37.177,"{""location"": ""NL"", ""is_mobile"": false}" 2761,2,61,2017-03-26 14:09:21,http://lockman.biz/hardy_lehner,,131.100.35.199,"{""location"": ""GL"", ""is_mobile"": true}" 2762,2,61,2017-05-15 00:53:02,http://mclaughlinrogahn.com/merle,,142.57.34.201,"{""location"": ""NE"", ""is_mobile"": true}" 2763,2,61,2017-06-05 06:42:10,http://hettinger.io/jeffry,,163.93.212.151,"{""location"": ""BB"", ""is_mobile"": false}" 2764,2,61,2017-03-11 00:20:40,http://kris.co/elna,,94.136.128.61,"{""location"": ""MU"", ""is_mobile"": false}" 2765,2,61,2017-04-29 12:38:55,http://abbottboyle.biz/marcelina,,203.114.254.17,"{""location"": ""PG"", ""is_mobile"": false}" 2766,2,61,2017-03-31 20:33:05,http://erdman.io/raleigh,,46.162.112.217,"{""location"": ""NE"", ""is_mobile"": false}" 2767,2,61,2016-12-31 03:58:31,http://ortiz.name/elizabeth_larson,,79.100.195.200,"{""location"": ""BZ"", ""is_mobile"": true}" 2768,2,61,2017-01-29 12:56:42,http://will.info/bill_terry,,79.22.164.163,"{""location"": ""KN"", ""is_mobile"": true}" 2769,2,61,2017-01-23 13:47:31,http://handhalvorson.io/gunnar,,156.98.167.25,"{""location"": ""SL"", ""is_mobile"": true}" 2770,2,61,2017-02-24 05:24:37,http://predovicdurgan.co/janet.heidenreich,,230.52.68.2,"{""location"": ""JE"", ""is_mobile"": false}" 2771,2,61,2017-05-24 14:34:03,http://abshireberge.com/gino.tillman,,53.76.16.191,"{""location"": ""CI"", ""is_mobile"": false}" 2772,2,61,2017-04-23 10:59:12,http://wisozkreinger.org/theresia.balistreri,,186.254.44.16,"{""location"": ""ET"", ""is_mobile"": true}" 2773,2,61,2017-05-21 20:01:57,http://batzgerlach.net/jairo_kunze,,38.108.20.216,"{""location"": ""ES"", ""is_mobile"": true}" 2774,2,61,2017-04-27 23:56:08,http://little.co/tremaine,,173.156.110.200,"{""location"": ""PE"", ""is_mobile"": false}" 2775,2,61,2017-03-15 07:30:05,http://waters.info/evalyn.wiza,,9.191.58.206,"{""location"": ""KR"", ""is_mobile"": false}" 2776,2,61,2017-01-14 22:35:04,http://fritschwolf.info/watson.jakubowski,,25.206.38.9,"{""location"": ""GI"", ""is_mobile"": true}" 2777,2,61,2017-02-24 01:00:01,http://medhurst.info/brayan,,24.253.90.187,"{""location"": ""SY"", ""is_mobile"": true}" 2778,2,61,2017-04-03 14:44:04,http://bechtelar.co/norberto_luettgen,,103.144.240.51,"{""location"": ""PL"", ""is_mobile"": true}" 2779,2,61,2017-03-19 09:07:04,http://keebler.org/abraham,,65.12.35.25,"{""location"": ""MM"", ""is_mobile"": true}" 2780,2,61,2017-05-24 18:48:23,http://oconnell.biz/justen,,69.81.176.194,"{""location"": ""MD"", ""is_mobile"": true}" 2781,2,61,2017-06-03 14:21:16,http://wilkinson.net/aron,,156.120.221.155,"{""location"": ""NZ"", ""is_mobile"": true}" 2782,2,61,2017-03-28 23:34:40,http://labadie.com/titus.schinner,,122.63.103.244,"{""location"": ""SB"", ""is_mobile"": true}" 2783,2,61,2017-04-28 09:05:55,http://douglas.net/ed.roberts,,101.204.59.163,"{""location"": ""SK"", ""is_mobile"": false}" 2784,2,61,2017-02-23 06:36:13,http://zboncak.biz/cristina,,83.250.41.8,"{""location"": ""BB"", ""is_mobile"": false}" 2785,2,61,2017-06-03 12:21:03,http://mccullough.com/rosendo_nikolaus,,53.161.200.88,"{""location"": ""ID"", ""is_mobile"": false}" 2786,2,61,2017-01-08 20:24:08,http://mayer.name/ronaldo_quitzon,,182.9.234.123,"{""location"": ""PY"", ""is_mobile"": false}" 2787,2,61,2017-06-04 19:49:34,http://pfannerstill.co/sammie_parker,,201.70.19.128,"{""location"": ""MC"", ""is_mobile"": false}" 2788,2,61,2017-03-21 04:39:26,http://franecki.name/ethel,,48.80.145.180,"{""location"": ""DJ"", ""is_mobile"": true}" 2789,2,61,2017-03-21 12:52:00,http://kuhic.org/buford,,171.54.17.247,"{""location"": ""IR"", ""is_mobile"": true}" 2790,2,61,2017-02-11 01:44:56,http://bogisichbauch.info/perry,,192.217.235.63,"{""location"": ""LY"", ""is_mobile"": false}" 2791,2,61,2017-01-06 06:05:12,http://hartmann.net/monica.abshire,,22.212.54.75,"{""location"": ""BH"", ""is_mobile"": false}" 2792,2,61,2016-12-20 02:10:09,http://nienow.io/duncan_altenwerth,,234.252.146.35,"{""location"": ""PE"", ""is_mobile"": true}" 2793,2,61,2017-01-25 05:21:20,http://beahan.info/jamar.gaylord,,228.119.145.31,"{""location"": ""LI"", ""is_mobile"": false}" 2794,2,61,2017-01-01 18:19:48,http://rennerrolfson.org/emmie.green,,8.165.11.63,"{""location"": ""BF"", ""is_mobile"": false}" 2795,2,61,2017-01-11 06:38:58,http://feeney.org/adonis,,64.141.74.181,"{""location"": ""MA"", ""is_mobile"": true}" 9675,4,217,2017-04-19 16:54:25,http://volkmanerdman.net/iva_klein,,160.118.41.61,"{""location"": ""DM"", ""is_mobile"": true}" 9676,4,217,2016-12-17 07:40:32,http://feilkohler.net/gaetano_ankunding,,45.7.105.133,"{""location"": ""ZM"", ""is_mobile"": false}" 9677,4,217,2017-02-01 09:10:13,http://will.net/coralie_franecki,,130.221.57.134,"{""location"": ""AD"", ""is_mobile"": true}" 9678,4,217,2016-12-20 21:55:13,http://lehnerweimann.net/jena,,140.107.202.98,"{""location"": ""GU"", ""is_mobile"": true}" 9679,4,217,2016-12-16 07:52:57,http://zboncakwunsch.biz/emelie_braun,,99.231.11.136,"{""location"": ""JO"", ""is_mobile"": false}" 9680,4,217,2016-12-23 05:38:47,http://huelerdman.co/theresia,,251.89.164.244,"{""location"": ""PS"", ""is_mobile"": true}" 9681,4,217,2017-03-11 01:38:00,http://donnellykub.co/ruell.bins,,252.199.37.105,"{""location"": ""MC"", ""is_mobile"": false}" 9682,4,217,2017-05-15 10:53:58,http://altenwerth.net/angeline.ferry,,158.136.176.207,"{""location"": ""GQ"", ""is_mobile"": true}" 9683,4,217,2017-01-01 22:59:53,http://bashirian.co/nathaniel.lesch,,116.237.55.241,"{""location"": ""KZ"", ""is_mobile"": false}" 9684,4,217,2017-01-11 15:06:36,http://hand.co/jovan,,136.5.79.121,"{""location"": ""BJ"", ""is_mobile"": true}" 9685,4,217,2017-01-19 07:56:33,http://bayer.io/alberta,,102.151.213.111,"{""location"": ""NE"", ""is_mobile"": true}" 9686,4,217,2017-05-09 10:00:40,http://thiel.io/janiya.towne,,127.25.243.165,"{""location"": ""PE"", ""is_mobile"": false}" 9687,4,217,2017-05-18 22:36:13,http://reynolds.com/hettie,,166.59.27.23,"{""location"": ""DJ"", ""is_mobile"": false}" 9688,4,217,2017-03-10 01:25:16,http://brekke.com/henderson_daugherty,,49.81.41.33,"{""location"": ""HU"", ""is_mobile"": true}" 9689,4,217,2017-05-27 03:57:38,http://quigley.biz/avis.klocko,,141.134.13.187,"{""location"": ""KE"", ""is_mobile"": false}" 9690,4,217,2016-12-19 15:56:52,http://leschnolan.biz/maud,,148.127.95.113,"{""location"": ""BZ"", ""is_mobile"": true}" 9691,4,217,2017-03-27 07:23:35,http://carroll.co/santos,,163.50.229.108,"{""location"": ""BJ"", ""is_mobile"": false}" 9692,4,217,2016-12-23 23:12:31,http://ernserklocko.name/montana_conroy,,142.8.220.203,"{""location"": ""KY"", ""is_mobile"": false}" 9693,4,217,2017-04-13 16:35:08,http://paucek.net/ashley_howe,,87.242.73.145,"{""location"": ""SO"", ""is_mobile"": false}" 9694,4,217,2016-12-29 15:31:47,http://turcotte.biz/ashly_goodwin,,103.174.26.198,"{""location"": ""AG"", ""is_mobile"": true}" 9695,4,217,2016-12-25 22:47:34,http://runolfon.net/mallory_hettinger,,143.228.30.6,"{""location"": ""CR"", ""is_mobile"": false}" 9696,4,217,2017-04-13 04:28:56,http://mertz.biz/maureen,,5.24.128.253,"{""location"": ""BG"", ""is_mobile"": true}" 9697,4,217,2017-02-05 03:22:28,http://bahringer.name/alysha,,254.60.80.91,"{""location"": ""RW"", ""is_mobile"": false}" 9698,4,217,2017-03-24 20:15:34,http://predovicprice.org/philip,,19.136.161.22,"{""location"": ""SM"", ""is_mobile"": true}" 9699,4,217,2016-12-27 05:18:03,http://schimmelmclaughlin.com/jamie_treutel,,219.62.83.55,"{""location"": ""CO"", ""is_mobile"": true}" 9700,4,217,2016-12-27 01:27:56,http://carrollshanahan.com/gardner,,70.150.203.237,"{""location"": ""CY"", ""is_mobile"": true}" 9701,4,217,2017-01-25 16:38:02,http://fay.com/brock,,173.3.16.168,"{""location"": ""NZ"", ""is_mobile"": false}" 9702,4,217,2017-04-03 16:10:04,http://dibbert.io/kyle_willms,,75.68.222.104,"{""location"": ""NR"", ""is_mobile"": true}" 9703,4,217,2017-04-24 07:45:47,http://bayer.com/finn,,37.18.191.135,"{""location"": ""TO"", ""is_mobile"": false}" 9704,4,217,2017-05-23 00:00:54,http://considine.name/nora,,164.26.212.58,"{""location"": ""GB"", ""is_mobile"": false}" 9705,4,217,2017-04-29 09:26:04,http://hilllbechtelar.net/dominic_crona,,69.233.20.45,"{""location"": ""WS"", ""is_mobile"": false}" 9706,4,217,2017-03-20 12:23:51,http://lakin.org/brice_carter,,151.229.77.177,"{""location"": ""KG"", ""is_mobile"": true}" 9707,4,217,2017-05-11 04:34:28,http://larkin.net/annie_leannon,,141.135.168.206,"{""location"": ""TH"", ""is_mobile"": true}" 9708,4,217,2016-12-17 08:27:13,http://beer.com/dario,,41.77.60.118,"{""location"": ""AI"", ""is_mobile"": true}" 9709,4,217,2017-01-10 19:35:46,http://strackeschaden.name/shaina,,29.120.204.72,"{""location"": ""PH"", ""is_mobile"": false}" 9710,4,217,2017-03-17 01:42:10,http://rosenbaum.name/tania,,166.87.69.68,"{""location"": ""TL"", ""is_mobile"": false}" 9711,4,218,2017-01-03 23:49:56,http://uptonjohnston.info/rollin,,203.173.14.127,"{""location"": ""DM"", ""is_mobile"": true}" 9712,4,218,2017-04-27 11:02:25,http://kohler.biz/billie,,72.17.87.177,"{""location"": ""UA"", ""is_mobile"": false}" 9713,4,218,2017-01-16 16:18:29,http://nolan.io/travis,,121.89.238.220,"{""location"": ""NA"", ""is_mobile"": true}" 9714,4,218,2017-01-02 06:14:25,http://carter.net/bobbie_kuhlman,,232.87.36.208,"{""location"": ""GA"", ""is_mobile"": false}" 9715,4,218,2017-03-20 03:00:11,http://walsh.biz/drake.nienow,,82.123.194.73,"{""location"": ""KP"", ""is_mobile"": true}" 9716,4,218,2016-12-15 15:31:07,http://rohanwilkinson.info/stacey.torphy,,65.227.39.119,"{""location"": ""GU"", ""is_mobile"": false}" 9717,4,218,2017-01-24 21:35:23,http://heaney.name/trystan,,120.207.179.210,"{""location"": ""PS"", ""is_mobile"": false}" 9718,4,218,2017-02-21 05:08:54,http://carter.co/helen.beier,,60.150.108.54,"{""location"": ""AS"", ""is_mobile"": true}" 9719,4,218,2016-12-19 09:48:01,http://herman.biz/alexandria_ryan,,249.30.31.239,"{""location"": ""BL"", ""is_mobile"": true}" 9720,4,218,2017-04-01 16:41:56,http://wolfhickle.org/genoveva,,215.80.28.183,"{""location"": ""GT"", ""is_mobile"": false}" 9721,4,218,2017-01-12 13:03:12,http://collier.com/holden.mueller,,180.228.58.53,"{""location"": ""GE"", ""is_mobile"": false}" 9722,4,218,2016-12-29 22:01:58,http://gleichner.name/abbie,,98.71.108.108,"{""location"": ""SB"", ""is_mobile"": true}" 9723,4,218,2017-01-15 23:23:07,http://hermiston.name/judy_friesen,,189.201.133.188,"{""location"": ""MA"", ""is_mobile"": false}" 9724,4,218,2017-05-26 10:34:00,http://funk.name/katharina.dickinson,,63.159.211.208,"{""location"": ""DJ"", ""is_mobile"": false}" 9725,4,218,2017-05-16 23:04:45,http://leschjones.io/maurine_rau,,205.30.208.95,"{""location"": ""AF"", ""is_mobile"": true}" 9726,4,218,2016-12-29 14:08:14,http://krajciklabadie.com/nasir,,118.108.138.79,"{""location"": ""BG"", ""is_mobile"": false}" 9727,4,218,2017-01-05 23:05:03,http://marquardt.com/ila_baumbach,,30.112.236.197,"{""location"": ""ET"", ""is_mobile"": false}" 9728,4,218,2017-06-08 04:15:29,http://baumbachlebsack.io/rachel,,148.193.229.224,"{""location"": ""GD"", ""is_mobile"": false}" 9729,4,218,2016-12-21 08:41:02,http://funk.name/dangelo.runolfsdottir,,176.200.33.242,"{""location"": ""PF"", ""is_mobile"": false}" 9730,4,218,2017-04-29 23:58:28,http://jastschultz.name/lillie_leannon,,74.54.65.176,"{""location"": ""AW"", ""is_mobile"": true}" 6668,3,145,2017-02-28 09:55:35,http://rueckerabernathy.info/noe.mills,0.4560321096,154.116.95.164,"{""location"": ""KZ"", ""is_mobile"": false}" 6669,3,145,2016-12-22 00:09:26,http://hudson.biz/benedict,0.6738025853,209.175.238.219,"{""location"": ""BH"", ""is_mobile"": true}" 6670,3,145,2017-06-04 08:02:12,http://bergnaum.biz/elroy,0.2338821147,157.98.99.169,"{""location"": ""PH"", ""is_mobile"": false}" 6671,3,145,2017-01-12 06:29:12,http://dicki.io/jamie.blick,0.6181160679,17.56.199.174,"{""location"": ""FI"", ""is_mobile"": true}" 6672,3,146,2016-12-16 14:31:59,http://kreiger.io/catalina,,193.173.244.166,"{""location"": ""BQ"", ""is_mobile"": false}" 6673,3,146,2017-05-27 00:01:06,http://gleichner.net/trystan.dickinson,,11.244.249.59,"{""location"": ""CV"", ""is_mobile"": false}" 6674,3,146,2017-01-27 20:47:58,http://robel.info/ransom,,161.17.247.92,"{""location"": ""BL"", ""is_mobile"": false}" 6675,3,146,2017-02-18 13:57:12,http://hoppe.org/bailey,,36.54.185.14,"{""location"": ""SX"", ""is_mobile"": false}" 6676,3,146,2017-04-17 15:55:30,http://hansenmayert.co/julianne,,195.181.131.253,"{""location"": ""VE"", ""is_mobile"": false}" 6677,3,146,2017-02-06 14:17:24,http://oconnerflatley.name/brittany.jakubowski,,157.34.111.46,"{""location"": ""SS"", ""is_mobile"": true}" 6678,3,146,2017-02-16 05:22:54,http://beahanhoeger.name/larry.bernier,,33.50.161.158,"{""location"": ""TO"", ""is_mobile"": true}" 6679,3,146,2017-01-21 04:09:48,http://mertzconnelly.net/danika,,45.37.55.45,"{""location"": ""DZ"", ""is_mobile"": false}" 6680,3,146,2017-05-21 22:53:11,http://koepp.net/laurine,,214.154.12.135,"{""location"": ""MR"", ""is_mobile"": true}" 6681,3,146,2017-03-25 06:03:47,http://hettingerratke.biz/kellie_rolfson,,131.112.7.107,"{""location"": ""GD"", ""is_mobile"": false}" 6682,3,146,2017-04-04 03:55:07,http://thielcrist.com/frederik.cormier,,177.167.80.18,"{""location"": ""BN"", ""is_mobile"": true}" 6683,3,146,2017-06-10 06:05:55,http://lakin.net/raheem,,102.237.37.50,"{""location"": ""CV"", ""is_mobile"": true}" 6684,3,146,2017-01-23 23:32:43,http://kozey.org/berniece_lubowitz,,56.209.13.7,"{""location"": ""US"", ""is_mobile"": true}" 6685,3,146,2017-01-08 11:05:32,http://weimann.co/tanya.sipes,,81.251.139.107,"{""location"": ""TM"", ""is_mobile"": false}" 6686,3,146,2017-01-23 14:34:37,http://adams.net/ethel,,225.207.5.162,"{""location"": ""SV"", ""is_mobile"": false}" 6687,3,146,2017-01-13 15:30:58,http://wolf.net/alvis,,249.143.153.247,"{""location"": ""KY"", ""is_mobile"": false}" 6688,3,146,2016-12-26 01:38:16,http://boyerkuvalis.net/mallory,,130.54.216.161,"{""location"": ""AD"", ""is_mobile"": true}" 6689,3,146,2017-04-11 10:42:24,http://schmidt.co/myrtie.hoeger,,70.196.126.205,"{""location"": ""MM"", ""is_mobile"": false}" 6690,3,146,2017-03-17 01:58:38,http://vonhahn.com/marianne_hills,,185.64.126.37,"{""location"": ""GR"", ""is_mobile"": true}" 6691,3,146,2017-01-16 21:40:28,http://runteward.io/robb,,71.141.37.234,"{""location"": ""GF"", ""is_mobile"": false}" 6692,3,146,2017-04-05 00:46:46,http://hudson.co/kamryn,,18.16.230.208,"{""location"": ""LT"", ""is_mobile"": true}" 6693,3,146,2017-05-08 00:26:02,http://smitham.net/leann_leuschke,,135.174.163.189,"{""location"": ""MA"", ""is_mobile"": true}" 6694,3,147,2017-03-10 16:15:03,http://macejkovic.biz/nils,,32.164.129.36,"{""location"": ""CN"", ""is_mobile"": true}" 6695,3,147,2017-01-03 22:24:13,http://frami.co/america,,228.2.127.67,"{""location"": ""GR"", ""is_mobile"": false}" 6696,3,147,2017-05-15 23:52:59,http://haley.co/beau,,164.99.219.8,"{""location"": ""GF"", ""is_mobile"": false}" 6697,3,147,2017-05-27 10:21:30,http://thompsonmurray.name/sophia,,201.48.128.142,"{""location"": ""PE"", ""is_mobile"": true}" 6698,3,147,2017-05-09 20:37:23,http://hansenward.info/mya,,110.244.225.61,"{""location"": ""BD"", ""is_mobile"": false}" 6699,3,147,2017-05-14 17:19:50,http://parkerhoppe.biz/keyshawn,,124.113.42.5,"{""location"": ""UZ"", ""is_mobile"": false}" 6700,3,147,2017-06-08 12:01:35,http://willruel.co/madilyn,,105.11.57.94,"{""location"": ""LV"", ""is_mobile"": true}" 6701,3,147,2016-12-21 12:38:49,http://kuvalis.name/brice,,15.157.134.140,"{""location"": ""AI"", ""is_mobile"": true}" 6702,3,147,2017-06-12 20:43:34,http://bogan.net/itzel,,70.3.239.205,"{""location"": ""UZ"", ""is_mobile"": false}" 6703,3,147,2016-12-18 17:46:39,http://schustergorczany.info/hettie.thiel,,151.208.176.224,"{""location"": ""UZ"", ""is_mobile"": true}" 6704,3,147,2017-04-02 12:16:35,http://schroederflatley.net/bianka_considine,,127.73.173.173,"{""location"": ""AG"", ""is_mobile"": false}" 6705,3,147,2017-04-02 23:29:15,http://lowe.org/danial.hills,,20.58.36.150,"{""location"": ""JP"", ""is_mobile"": true}" 6706,3,147,2017-03-10 19:03:47,http://langmcdermott.biz/benny_champlin,,18.27.85.200,"{""location"": ""FJ"", ""is_mobile"": true}" 6707,3,147,2017-01-03 15:26:07,http://wisoky.info/tommie.bergstrom,,55.12.14.140,"{""location"": ""BS"", ""is_mobile"": true}" 6708,3,147,2017-03-23 06:54:17,http://lebsackkutch.biz/blaise_ward,,156.21.243.252,"{""location"": ""LK"", ""is_mobile"": false}" 6709,3,147,2017-01-14 20:20:17,http://deckow.net/nels,,152.206.105.41,"{""location"": ""ES"", ""is_mobile"": true}" 6710,3,147,2017-05-13 23:50:33,http://berge.io/owen_homenick,,181.251.186.125,"{""location"": ""PS"", ""is_mobile"": true}" 6711,3,147,2017-03-14 10:59:54,http://lubowitz.info/delia,,155.236.132.222,"{""location"": ""OM"", ""is_mobile"": false}" 6712,3,147,2017-02-03 16:26:11,http://weinatcarter.name/annamarie,,174.2.172.149,"{""location"": ""EH"", ""is_mobile"": false}" 6713,3,147,2017-05-18 14:28:41,http://daugherty.info/delores,,14.113.87.18,"{""location"": ""BL"", ""is_mobile"": false}" 6714,3,147,2017-04-20 12:17:06,http://marvin.co/vernice_klocko,,108.242.67.67,"{""location"": ""PN"", ""is_mobile"": true}" 6715,3,147,2017-01-23 22:18:29,http://ruelwalker.info/matteo_harvey,,25.66.241.87,"{""location"": ""VA"", ""is_mobile"": false}" 6716,3,147,2017-04-28 09:16:59,http://littel.io/kira.west,,224.176.19.105,"{""location"": ""KM"", ""is_mobile"": true}" 6717,3,147,2017-05-28 04:51:47,http://moore.org/muriel,,88.130.214.81,"{""location"": ""SS"", ""is_mobile"": false}" 6718,3,147,2016-12-23 15:53:01,http://ferry.io/carolyne_beier,,216.125.86.63,"{""location"": ""FK"", ""is_mobile"": false}" 6719,3,147,2017-01-30 09:51:14,http://mayert.name/conner,,100.45.73.112,"{""location"": ""KM"", ""is_mobile"": false}" 6720,3,147,2017-06-08 18:55:02,http://johnsmayert.name/magdalena.miller,,72.197.188.126,"{""location"": ""MW"", ""is_mobile"": false}" 6721,3,147,2017-06-03 10:53:23,http://shanahan.org/mackenzie.skiles,,230.167.68.47,"{""location"": ""KM"", ""is_mobile"": false}" 6722,3,147,2017-01-20 02:45:07,http://schaefer.com/ethan_murphy,,3.193.91.104,"{""location"": ""GF"", ""is_mobile"": true}" 12583,5,281,2017-04-09 21:30:44,http://sanford.com/berenice_lakin,,148.85.57.76,"{""location"": ""CC"", ""is_mobile"": false}" 12584,5,281,2017-05-26 06:53:26,http://harvey.io/america.cremin,,61.227.125.222,"{""location"": ""JP"", ""is_mobile"": true}" 12585,5,281,2017-03-03 19:00:23,http://pollich.org/laurianne,,223.239.140.178,"{""location"": ""BG"", ""is_mobile"": true}" 12586,5,281,2017-04-01 15:22:20,http://lednerdaniel.org/clyde.will,,189.72.228.177,"{""location"": ""CD"", ""is_mobile"": false}" 12587,5,281,2017-01-07 01:42:42,http://murray.com/burley,,100.121.92.12,"{""location"": ""AZ"", ""is_mobile"": true}" 12588,5,281,2017-05-04 23:01:42,http://grady.org/mitchell,,56.242.110.59,"{""location"": ""CO"", ""is_mobile"": true}" 12589,5,281,2017-04-26 10:15:15,http://homenick.info/maryam,,248.55.221.114,"{""location"": ""BT"", ""is_mobile"": true}" 12590,5,281,2017-03-26 21:03:27,http://fay.biz/elwyn,,213.108.113.199,"{""location"": ""LU"", ""is_mobile"": false}" 12591,5,281,2017-03-03 12:24:10,http://west.biz/joelle_murray,,18.224.73.248,"{""location"": ""GT"", ""is_mobile"": false}" 12592,5,281,2017-01-23 20:18:44,http://pacocha.co/elyse.howe,,78.107.17.250,"{""location"": ""SJ"", ""is_mobile"": true}" 12593,5,281,2017-02-17 12:46:07,http://okeefe.co/haley.corkery,,167.145.39.10,"{""location"": ""PE"", ""is_mobile"": true}" 12594,5,282,2017-02-04 19:02:37,http://fadel.io/frances,,249.220.233.162,"{""location"": ""BN"", ""is_mobile"": false}" 12595,5,282,2017-01-13 05:57:31,http://cormierboehm.com/august,,149.87.21.238,"{""location"": ""ML"", ""is_mobile"": false}" 12596,5,282,2017-04-20 06:24:02,http://willpowlowski.io/rosanna,,52.31.99.195,"{""location"": ""KG"", ""is_mobile"": false}" 12597,5,282,2017-03-07 23:09:25,http://schultz.net/bailee.ryan,,173.88.171.198,"{""location"": ""NG"", ""is_mobile"": true}" 12598,5,282,2017-01-03 04:29:22,http://bernhard.name/sid_hamill,,176.13.223.219,"{""location"": ""NG"", ""is_mobile"": true}" 12599,5,282,2017-03-24 04:05:56,http://walterwehner.info/davion,,123.201.114.135,"{""location"": ""TG"", ""is_mobile"": true}" 12600,5,282,2017-04-09 01:06:20,http://corwin.co/arnold_bergstrom,,254.22.129.167,"{""location"": ""DE"", ""is_mobile"": true}" 12601,5,282,2017-03-11 00:21:33,http://littel.com/leilani,,246.98.20.200,"{""location"": ""ME"", ""is_mobile"": false}" 12602,5,282,2016-12-29 14:57:31,http://mohrheller.biz/tommie.spencer,,82.60.243.207,"{""location"": ""VU"", ""is_mobile"": false}" 12603,5,282,2017-04-04 08:01:11,http://huel.net/marcos,,107.16.130.216,"{""location"": ""NO"", ""is_mobile"": false}" 12604,5,282,2017-03-27 01:00:05,http://gutmannharvey.info/miguel,,46.193.34.91,"{""location"": ""IR"", ""is_mobile"": false}" 12605,5,282,2017-02-11 13:38:42,http://dooley.org/denis.becker,,229.156.139.180,"{""location"": ""KZ"", ""is_mobile"": false}" 12606,5,282,2017-04-16 23:06:56,http://hegmann.com/izaiah,,193.184.242.177,"{""location"": ""CF"", ""is_mobile"": true}" 12607,5,282,2017-01-23 08:42:53,http://zulauf.io/geo,,57.28.194.234,"{""location"": ""HT"", ""is_mobile"": true}" 12608,5,282,2017-01-21 18:13:25,http://legros.org/verner_rempel,,161.224.203.16,"{""location"": ""NZ"", ""is_mobile"": true}" 12609,5,282,2017-02-04 05:32:01,http://ernser.co/philip.dicki,,160.76.232.94,"{""location"": ""IN"", ""is_mobile"": false}" 12610,5,282,2017-02-27 21:53:30,http://hayescronin.net/gonzalo_borer,,172.236.22.32,"{""location"": ""SJ"", ""is_mobile"": true}" 12611,5,282,2017-06-09 16:24:32,http://schimmel.info/garret.walsh,,108.246.52.89,"{""location"": ""BF"", ""is_mobile"": true}" 12612,5,282,2017-04-04 03:48:42,http://turnerwalsh.io/albert,,18.122.100.181,"{""location"": ""KY"", ""is_mobile"": true}" 12613,5,282,2017-04-20 17:27:16,http://murphy.io/josefina_rohan,,165.207.238.29,"{""location"": ""MF"", ""is_mobile"": false}" 12614,5,282,2017-02-07 16:37:49,http://moen.net/griffin,,241.199.39.5,"{""location"": ""LC"", ""is_mobile"": true}" 12615,5,282,2017-05-03 23:33:52,http://rippin.name/christa,,2.4.188.175,"{""location"": ""JP"", ""is_mobile"": false}" 12616,5,282,2017-01-03 21:22:37,http://parisianwaelchi.org/willard_greenholt,,109.67.57.144,"{""location"": ""MG"", ""is_mobile"": false}" 12617,5,282,2017-05-13 13:02:56,http://schneider.io/margot_lynch,,216.15.213.97,"{""location"": ""BN"", ""is_mobile"": true}" 12618,5,282,2017-01-16 23:07:00,http://conroyherzog.io/margarita_bogan,,8.219.8.192,"{""location"": ""ML"", ""is_mobile"": true}" 12619,5,282,2017-05-02 08:15:50,http://ebert.biz/marco.stiedemann,,71.149.78.205,"{""location"": ""SX"", ""is_mobile"": true}" 12620,5,282,2017-01-20 22:43:38,http://haagsanford.net/bobbie.lubowitz,,169.148.187.213,"{""location"": ""AT"", ""is_mobile"": false}" 12621,5,282,2017-05-10 07:30:43,http://reynoldskiehn.io/amparo,,222.51.58.142,"{""location"": ""ML"", ""is_mobile"": false}" 12622,5,282,2017-03-30 00:45:23,http://runolfsdottir.org/salma.volkman,,49.83.118.33,"{""location"": ""BD"", ""is_mobile"": false}" 12623,5,282,2017-03-31 12:33:11,http://romaguera.com/patsy,,28.143.189.196,"{""location"": ""MF"", ""is_mobile"": false}" 12624,5,282,2017-02-14 06:13:24,http://williamson.biz/talia.hyatt,,146.135.225.236,"{""location"": ""KG"", ""is_mobile"": false}" 12625,5,282,2017-02-11 00:47:26,http://aufderhar.com/ernestine,,75.233.40.126,"{""location"": ""AR"", ""is_mobile"": false}" 12626,5,282,2017-06-03 18:52:39,http://torp.co/lonnie,,38.89.34.210,"{""location"": ""CW"", ""is_mobile"": false}" 12627,5,282,2017-01-28 20:00:59,http://hoppe.biz/darrion,,98.129.170.155,"{""location"": ""PA"", ""is_mobile"": false}" 12628,5,282,2017-01-30 00:48:44,http://dare.co/yoshiko,,98.144.160.86,"{""location"": ""MH"", ""is_mobile"": false}" 12629,5,282,2017-02-16 05:59:48,http://hartmann.net/jazlyn.kilback,,203.207.214.143,"{""location"": ""WS"", ""is_mobile"": false}" 12630,5,282,2017-03-26 23:50:33,http://heidenreich.biz/virgil,,187.114.18.112,"{""location"": ""GB"", ""is_mobile"": false}" 12631,5,282,2017-05-24 14:00:09,http://raynor.biz/antwon.stroman,,146.228.161.200,"{""location"": ""DO"", ""is_mobile"": true}" 12632,5,282,2016-12-18 20:17:55,http://durgan.io/henry_dare,,72.209.114.16,"{""location"": ""KI"", ""is_mobile"": false}" 12633,5,282,2017-02-21 04:44:58,http://mayert.info/fannie_gutkowski,,124.148.11.238,"{""location"": ""PS"", ""is_mobile"": false}" 12634,5,282,2017-02-25 22:17:47,http://williamsonziemann.co/keven.rowe,,117.221.204.186,"{""location"": ""PW"", ""is_mobile"": false}" 12635,5,282,2016-12-19 01:36:33,http://price.name/evan,,125.142.138.74,"{""location"": ""BZ"", ""is_mobile"": true}" 12636,5,282,2017-04-16 06:09:24,http://maggio.biz/myriam,,220.193.145.128,"{""location"": ""TG"", ""is_mobile"": true}" 12637,5,282,2017-03-07 12:43:52,http://nienowharber.io/vincenzo,,30.249.237.103,"{""location"": ""GM"", ""is_mobile"": false}" 15585,6,350,2016-12-16 20:26:20,http://connsimonis.biz/dolly.marquardt,0.5958027443,75.37.100.169,"{""location"": ""BD"", ""is_mobile"": true}" 15586,6,350,2017-02-09 21:06:00,http://wolff.org/kayden,0.3545751719,63.184.150.98,"{""location"": ""NR"", ""is_mobile"": true}" 15587,6,350,2017-06-04 22:14:09,http://adamsheller.io/julie_rippin,0.9632084924,63.69.119.110,"{""location"": ""LB"", ""is_mobile"": true}" 15588,6,350,2017-03-30 19:06:06,http://sipesbreitenberg.name/carlie,0.1117152522,223.77.99.93,"{""location"": ""CM"", ""is_mobile"": true}" 15589,6,350,2017-01-05 13:56:29,http://champlinkertzmann.org/laron,0.6917430925,68.249.199.33,"{""location"": ""SM"", ""is_mobile"": false}" 15590,6,350,2017-03-04 15:39:38,http://kilback.net/vesta.jerde,0.8532566458,104.161.107.85,"{""location"": ""TC"", ""is_mobile"": true}" 15591,6,350,2016-12-19 15:53:50,http://farrelldubuque.name/florine_lang,0.7479824168,227.231.46.96,"{""location"": ""LY"", ""is_mobile"": true}" 15592,6,350,2017-01-26 11:44:08,http://roberts.name/nikita_labadie,0.5523686205,131.82.33.85,"{""location"": ""GY"", ""is_mobile"": true}" 15593,6,350,2017-01-05 05:48:01,http://schulistleannon.net/dangelo_murray,0.7505460364,202.168.14.27,"{""location"": ""GM"", ""is_mobile"": true}" 15594,6,350,2017-05-16 06:35:41,http://heidenreichcremin.org/charley,0.5069874346,85.124.177.43,"{""location"": ""JE"", ""is_mobile"": true}" 15595,6,350,2017-05-14 04:04:59,http://skilesruecker.net/leola,0.9897049271,88.251.253.120,"{""location"": ""TD"", ""is_mobile"": false}" 15596,6,350,2017-05-06 01:24:29,http://rauwolf.name/ora_emard,0.3174065626,45.167.160.30,"{""location"": ""HT"", ""is_mobile"": false}" 15597,6,351,2017-02-26 22:35:28,http://becker.biz/dariana_lueilwitz,,71.198.14.54,"{""location"": ""GG"", ""is_mobile"": true}" 15598,6,351,2017-05-18 04:20:01,http://cummerata.org/nicholaus,,67.178.230.251,"{""location"": ""MG"", ""is_mobile"": true}" 15599,6,351,2017-02-28 05:45:55,http://kuphalebert.co/dewitt,,231.67.170.244,"{""location"": ""HT"", ""is_mobile"": false}" 15600,6,351,2017-04-01 01:24:39,http://lesch.name/karlie_hansen,,108.116.5.223,"{""location"": ""BM"", ""is_mobile"": false}" 15601,6,351,2017-04-13 07:58:49,http://bode.info/chyna.hamill,,149.86.219.17,"{""location"": ""VC"", ""is_mobile"": true}" 15602,6,351,2017-01-08 08:12:53,http://beatty.info/skye_langosh,,220.151.211.12,"{""location"": ""CA"", ""is_mobile"": true}" 15603,6,351,2017-05-08 02:36:55,http://connpaucek.io/ulises_hegmann,,160.135.132.228,"{""location"": ""KI"", ""is_mobile"": true}" 15604,6,351,2017-02-27 00:08:25,http://bergnaum.info/winston.cruickshank,,24.252.121.85,"{""location"": ""UZ"", ""is_mobile"": false}" 15605,6,351,2017-06-11 14:57:46,http://leannon.org/lori,,125.228.177.176,"{""location"": ""UM"", ""is_mobile"": true}" 15606,6,351,2017-03-12 13:58:31,http://ratke.io/mireille_cummings,,207.95.206.103,"{""location"": ""BI"", ""is_mobile"": true}" 15607,6,351,2017-05-11 08:52:07,http://cronin.name/asha,,220.250.70.88,"{""location"": ""TW"", ""is_mobile"": false}" 15608,6,351,2017-03-28 02:16:17,http://dickensmacejkovic.biz/leonardo,,166.95.195.114,"{""location"": ""BN"", ""is_mobile"": true}" 15609,6,351,2016-12-15 13:23:08,http://bartoletti.io/tito_oreilly,,124.126.215.54,"{""location"": ""IE"", ""is_mobile"": true}" 15610,6,351,2017-02-20 06:32:58,http://stiedemann.name/shane,,252.41.65.181,"{""location"": ""NC"", ""is_mobile"": false}" 15611,6,351,2017-03-25 22:23:56,http://davis.co/harrison,,141.193.103.232,"{""location"": ""CL"", ""is_mobile"": true}" 15612,6,351,2017-05-31 12:45:35,http://kozey.co/morris,,67.74.122.61,"{""location"": ""BQ"", ""is_mobile"": false}" 15613,6,351,2017-03-02 14:44:52,http://quigley.com/lorena,,236.50.24.17,"{""location"": ""BN"", ""is_mobile"": true}" 15614,6,351,2017-04-20 19:47:31,http://orn.com/leonora_hand,,182.5.132.26,"{""location"": ""TO"", ""is_mobile"": true}" 15616,6,351,2017-05-05 22:44:42,http://buckridge.net/deshaun.lindgren,,199.184.254.233,"{""location"": ""HN"", ""is_mobile"": true}" 15617,6,351,2017-01-18 15:39:42,http://ruel.co/dale.waelchi,,159.226.80.132,"{""location"": ""BI"", ""is_mobile"": false}" 15618,6,351,2017-06-07 13:24:00,http://dickinson.com/abbey,,23.62.9.20,"{""location"": ""AR"", ""is_mobile"": true}" 15619,6,351,2016-12-26 23:29:14,http://watsica.net/armand_west,,61.36.246.124,"{""location"": ""FO"", ""is_mobile"": false}" 15620,6,351,2016-12-31 07:49:55,http://simonis.info/sabryna,,107.231.236.15,"{""location"": ""MF"", ""is_mobile"": false}" 15621,6,352,2017-03-18 23:16:48,http://bechtelar.org/jordi_wisozk,,192.32.64.108,"{""location"": ""GS"", ""is_mobile"": true}" 15622,6,352,2017-05-26 03:56:08,http://kulascormier.name/cleta,,122.61.207.160,"{""location"": ""CW"", ""is_mobile"": false}" 15623,6,352,2017-01-30 14:04:07,http://bosco.biz/joanne,,237.216.15.95,"{""location"": ""TH"", ""is_mobile"": true}" 15624,6,352,2017-04-09 16:43:35,http://quitzon.co/kristofer_frami,,133.52.224.17,"{""location"": ""MQ"", ""is_mobile"": true}" 15625,6,352,2017-04-26 02:18:32,http://jastgrimes.co/vincenzo.kirlin,,47.96.115.132,"{""location"": ""CX"", ""is_mobile"": false}" 15626,6,352,2017-05-23 12:15:59,http://brownstoltenberg.com/maud,,64.116.29.38,"{""location"": ""KR"", ""is_mobile"": false}" 15627,6,352,2017-03-24 22:36:12,http://wisokykonopelski.io/aisha.brakus,,92.139.250.123,"{""location"": ""PK"", ""is_mobile"": false}" 15628,6,352,2017-04-04 19:52:16,http://erdman.name/gaston,,153.170.167.221,"{""location"": ""PW"", ""is_mobile"": false}" 15629,6,352,2017-06-13 14:21:29,http://osinski.biz/cletus,,227.85.87.41,"{""location"": ""JP"", ""is_mobile"": false}" 15630,6,352,2016-12-15 07:12:34,http://wisoky.co/cleta.rodriguez,,169.209.26.112,"{""location"": ""GG"", ""is_mobile"": false}" 15631,6,352,2017-02-14 06:20:18,http://emard.info/adonis.fahey,,61.9.22.49,"{""location"": ""VC"", ""is_mobile"": false}" 15632,6,352,2017-03-01 09:51:44,http://sporer.info/eusebio,,96.41.122.209,"{""location"": ""EC"", ""is_mobile"": true}" 15633,6,352,2017-03-04 16:35:29,http://faheyfunk.net/carol_hauck,,8.6.173.37,"{""location"": ""FR"", ""is_mobile"": false}" 15634,6,352,2016-12-20 06:29:21,http://hansenschimmel.biz/shanna,,24.235.129.178,"{""location"": ""RE"", ""is_mobile"": false}" 15635,6,352,2017-06-09 01:34:37,http://kihn.io/gwendolyn.gislason,,144.196.138.46,"{""location"": ""SN"", ""is_mobile"": false}" 15636,6,352,2017-04-13 16:29:59,http://ernser.com/forrest.zemlak,,31.88.68.86,"{""location"": ""DZ"", ""is_mobile"": false}" 15637,6,352,2017-02-01 15:37:46,http://veumparisian.io/lemuel,,14.202.165.74,"{""location"": ""VG"", ""is_mobile"": true}" 15638,6,352,2017-04-29 02:02:33,http://roob.org/nicholas.toy,,231.165.14.223,"{""location"": ""ZA"", ""is_mobile"": true}" 15639,6,352,2017-02-27 22:13:23,http://langshanahan.org/ellen_stiedemann,,100.145.172.71,"{""location"": ""LU"", ""is_mobile"": false}" 15640,6,352,2017-06-03 23:29:48,http://ward.biz/emilia,,241.240.230.78,"{""location"": ""KI"", ""is_mobile"": false}" 2796,2,61,2017-04-23 21:29:46,http://rippinkuhn.com/christelle,,233.77.35.106,"{""location"": ""TV"", ""is_mobile"": true}" 2797,2,61,2017-03-25 08:41:17,http://lindschuppe.org/ellie,,19.110.87.176,"{""location"": ""SV"", ""is_mobile"": true}" 2798,2,61,2017-02-21 07:19:37,http://rodriguezborer.org/kaleigh.wyman,,87.50.189.21,"{""location"": ""BL"", ""is_mobile"": true}" 2799,2,61,2017-05-20 09:52:36,http://beier.name/lupe.hauck,,188.228.37.149,"{""location"": ""SG"", ""is_mobile"": false}" 2800,2,61,2017-02-19 02:22:40,http://cummings.com/jordan_ebert,,174.251.252.109,"{""location"": ""BR"", ""is_mobile"": true}" 2801,2,61,2016-12-13 22:53:20,http://durgan.name/matt,,22.216.243.81,"{""location"": ""OM"", ""is_mobile"": true}" 2802,2,61,2017-05-05 08:51:05,http://walsh.name/dina,,31.91.111.210,"{""location"": ""RE"", ""is_mobile"": false}" 2803,2,61,2017-02-19 06:16:07,http://cole.biz/forrest_monahan,,173.195.224.251,"{""location"": ""PF"", ""is_mobile"": false}" 2804,2,61,2017-06-13 18:47:50,http://lehner.org/tara,,92.228.108.198,"{""location"": ""MW"", ""is_mobile"": true}" 2805,2,61,2017-05-06 02:07:41,http://bradtke.net/theodore,,45.132.239.87,"{""location"": ""BO"", ""is_mobile"": false}" 2806,2,61,2016-12-14 04:10:36,http://nolan.io/stephen,,104.133.94.89,"{""location"": ""BF"", ""is_mobile"": true}" 2807,2,61,2017-02-05 09:05:38,http://hintz.info/aylin,,8.23.147.198,"{""location"": ""PW"", ""is_mobile"": false}" 2808,2,61,2017-01-15 23:05:09,http://spencerabshire.co/gudrun,,7.52.185.108,"{""location"": ""AZ"", ""is_mobile"": true}" 2809,2,61,2017-04-10 12:01:54,http://kutch.io/dahlia,,121.210.92.67,"{""location"": ""LU"", ""is_mobile"": true}" 2810,2,61,2017-05-24 21:27:58,http://borerstehr.net/eileen,,163.249.173.29,"{""location"": ""MV"", ""is_mobile"": false}" 2811,2,62,2017-03-23 12:09:59,http://lang.com/odea,,155.120.25.93,"{""location"": ""AL"", ""is_mobile"": true}" 2812,2,62,2017-03-28 01:38:04,http://nadermccullough.io/rafael.murphy,,184.231.65.140,"{""location"": ""TD"", ""is_mobile"": true}" 2813,2,62,2016-12-14 12:00:51,http://adams.org/markus,,248.87.115.35,"{""location"": ""FK"", ""is_mobile"": true}" 2814,2,62,2017-02-04 01:07:26,http://ruellegros.com/nora_roob,,39.216.55.243,"{""location"": ""IR"", ""is_mobile"": false}" 2815,2,62,2017-02-27 15:41:26,http://volkman.biz/lauriane,,243.227.233.16,"{""location"": ""SH"", ""is_mobile"": false}" 2816,2,62,2017-03-22 23:12:20,http://prohaskakonopelski.biz/zachariah_gutkowski,,42.118.248.238,"{""location"": ""ZW"", ""is_mobile"": true}" 2817,2,62,2017-01-30 08:25:31,http://brekke.info/reagan_macgyver,,24.34.9.200,"{""location"": ""VE"", ""is_mobile"": false}" 2818,2,62,2017-05-12 04:52:01,http://kozeypouros.info/zakary,,214.179.65.36,"{""location"": ""SN"", ""is_mobile"": true}" 2819,2,62,2017-04-12 10:21:40,http://white.name/cade,,21.42.67.127,"{""location"": ""WS"", ""is_mobile"": false}" 2820,2,62,2017-03-09 21:32:38,http://rodriguez.org/adan_kirlin,,18.237.6.24,"{""location"": ""SO"", ""is_mobile"": false}" 2821,2,62,2017-02-07 17:43:50,http://rempel.co/annabelle,,103.86.224.168,"{""location"": ""LY"", ""is_mobile"": false}" 2822,2,62,2017-03-03 16:58:28,http://champlinkrajcik.info/robb_bode,,188.165.18.25,"{""location"": ""JE"", ""is_mobile"": false}" 2823,2,62,2017-01-18 10:44:31,http://runolfsdottir.co/mario.turcotte,,122.208.117.82,"{""location"": ""NO"", ""is_mobile"": true}" 2824,2,62,2017-06-05 17:59:00,http://oreillydouglas.io/crystel.stehr,,244.115.21.195,"{""location"": ""PE"", ""is_mobile"": true}" 2825,2,62,2017-02-02 07:50:39,http://erdman.name/manuela,,116.193.40.175,"{""location"": ""WS"", ""is_mobile"": true}" 2826,2,62,2017-01-01 07:31:27,http://zboncak.name/effie,,178.140.120.50,"{""location"": ""TH"", ""is_mobile"": true}" 2827,2,62,2017-02-27 00:22:45,http://smithklein.com/clement_haley,,51.81.101.193,"{""location"": ""KP"", ""is_mobile"": true}" 2828,2,62,2017-02-01 00:54:06,http://ziemelehner.name/heidi_little,,181.228.6.50,"{""location"": ""GB"", ""is_mobile"": false}" 2829,2,62,2017-04-02 07:50:38,http://trompschamberger.io/evert,,75.47.13.180,"{""location"": ""GM"", ""is_mobile"": true}" 2830,2,62,2017-05-14 02:39:33,http://marquardtkuvalis.biz/cayla,,131.51.63.53,"{""location"": ""YT"", ""is_mobile"": true}" 2831,2,62,2017-01-25 20:12:52,http://roob.name/ezra.bergstrom,,227.244.64.46,"{""location"": ""SC"", ""is_mobile"": true}" 2832,2,62,2017-02-06 09:41:44,http://witting.info/jordy.carroll,,181.68.200.91,"{""location"": ""CM"", ""is_mobile"": true}" 2833,2,62,2017-01-08 08:35:13,http://wolff.net/laisha_lindgren,,24.34.24.210,"{""location"": ""GR"", ""is_mobile"": true}" 2834,2,62,2017-03-11 03:30:54,http://zieme.co/violet_schuster,,236.184.6.169,"{""location"": ""GE"", ""is_mobile"": true}" 2835,2,62,2017-04-13 02:24:45,http://emard.biz/omari.kilback,,113.117.86.5,"{""location"": ""BQ"", ""is_mobile"": false}" 2836,2,62,2017-04-20 10:38:19,http://schambergerchristiansen.co/dylan_sporer,,217.197.136.180,"{""location"": ""GR"", ""is_mobile"": false}" 2837,2,62,2016-12-26 11:07:17,http://stehr.co/anibal,,70.176.204.226,"{""location"": ""DK"", ""is_mobile"": true}" 2838,2,62,2017-05-31 22:42:55,http://graham.info/victor_kirlin,,150.227.210.227,"{""location"": ""FJ"", ""is_mobile"": true}" 2839,2,62,2016-12-21 09:13:40,http://mayertko.io/kane,,40.203.231.253,"{""location"": ""DZ"", ""is_mobile"": false}" 2840,2,62,2017-02-13 03:30:53,http://faheywolff.org/asa,,36.99.3.21,"{""location"": ""PH"", ""is_mobile"": false}" 2841,2,62,2017-05-01 06:20:59,http://hintz.biz/una.cremin,,238.104.50.63,"{""location"": ""BD"", ""is_mobile"": true}" 2842,2,63,2017-03-23 23:47:16,http://hettingercrona.biz/hanna,,172.56.141.44,"{""location"": ""WF"", ""is_mobile"": false}" 2843,2,63,2017-03-09 15:35:43,http://torphy.org/jackeline.ledner,,84.82.22.124,"{""location"": ""TR"", ""is_mobile"": true}" 2844,2,63,2017-04-09 12:31:37,http://mcdermott.io/maeve_stanton,,186.17.116.177,"{""location"": ""FM"", ""is_mobile"": false}" 2845,2,63,2017-03-02 01:11:49,http://hermiston.io/harrison_wisozk,,241.192.199.92,"{""location"": ""TN"", ""is_mobile"": true}" 2846,2,63,2017-01-29 14:11:53,http://simonis.net/ryder,,15.109.252.126,"{""location"": ""LR"", ""is_mobile"": false}" 2847,2,63,2016-12-23 12:41:35,http://durganrunolfsdottir.name/winifred_graham,,175.189.90.107,"{""location"": ""LA"", ""is_mobile"": false}" 2848,2,63,2017-06-03 19:06:25,http://berge.com/roselyn.crooks,,251.125.157.62,"{""location"": ""TJ"", ""is_mobile"": false}" 2849,2,63,2017-01-18 17:15:05,http://macgyverbechtelar.io/keeley,,90.157.196.215,"{""location"": ""IO"", ""is_mobile"": false}" 2850,2,63,2017-06-09 08:04:21,http://kling.com/laurence_mclaughlin,,67.121.6.233,"{""location"": ""FM"", ""is_mobile"": false}" 2851,2,63,2017-01-02 02:18:55,http://jastbahringer.info/lisette.gulgowski,,202.35.200.231,"{""location"": ""GF"", ""is_mobile"": false}" 9731,4,218,2017-02-20 15:20:35,http://kelerquitzon.io/kayli,,249.7.45.131,"{""location"": ""VG"", ""is_mobile"": true}" 9732,4,218,2017-05-28 20:53:30,http://oconnell.com/kathryne,,81.107.182.147,"{""location"": ""SR"", ""is_mobile"": false}" 9733,4,218,2017-04-13 00:09:10,http://hansen.net/shaina,,20.149.24.126,"{""location"": ""IR"", ""is_mobile"": true}" 9734,4,218,2017-01-23 12:50:49,http://swaniawski.io/jackson,,203.48.193.226,"{""location"": ""GH"", ""is_mobile"": true}" 9735,4,218,2017-03-16 08:52:56,http://hirthestracke.org/floyd.dicki,,128.107.31.184,"{""location"": ""KW"", ""is_mobile"": true}" 9736,4,218,2016-12-22 04:43:35,http://raynor.co/sydni,,92.218.108.81,"{""location"": ""MY"", ""is_mobile"": true}" 9737,4,218,2017-02-19 23:30:08,http://hettinger.co/aglae,,19.247.198.94,"{""location"": ""NU"", ""is_mobile"": false}" 9738,4,218,2017-05-25 04:43:04,http://champlin.net/eladio.kilback,,43.27.196.25,"{""location"": ""KP"", ""is_mobile"": false}" 9739,4,218,2017-01-09 18:19:19,http://weberoconnell.co/simone.schmeler,,24.34.178.172,"{""location"": ""UG"", ""is_mobile"": true}" 9740,4,218,2017-05-25 05:39:56,http://kirlin.info/otis_ko,,66.136.74.113,"{""location"": ""WS"", ""is_mobile"": true}" 9741,4,218,2017-02-24 05:59:25,http://mcglynnrowe.com/andy.strosin,,31.65.217.107,"{""location"": ""KG"", ""is_mobile"": true}" 9742,4,218,2017-01-06 16:16:40,http://dubuque.io/melany.wolf,,202.205.78.105,"{""location"": ""BN"", ""is_mobile"": true}" 9743,4,218,2017-03-17 09:34:30,http://lebsack.info/maritza,,57.215.91.163,"{""location"": ""NE"", ""is_mobile"": true}" 9744,4,218,2017-04-19 20:07:55,http://johnson.info/allie,,189.58.26.216,"{""location"": ""QA"", ""is_mobile"": false}" 9745,4,218,2016-12-17 13:55:57,http://pfeffer.net/jaquan,,120.66.103.103,"{""location"": ""KI"", ""is_mobile"": false}" 9746,4,218,2017-01-19 04:23:39,http://kunde.name/milan,,196.51.164.53,"{""location"": ""HT"", ""is_mobile"": false}" 9747,4,218,2017-05-03 18:56:28,http://nienowturcotte.info/erna,,9.128.185.50,"{""location"": ""PH"", ""is_mobile"": false}" 9748,4,218,2016-12-15 08:08:24,http://huel.io/shanon.gulgowski,,81.8.139.66,"{""location"": ""MY"", ""is_mobile"": false}" 9749,4,218,2017-02-17 16:19:30,http://wilkinson.name/andre.emard,,141.134.111.19,"{""location"": ""JO"", ""is_mobile"": false}" 9750,4,218,2017-04-01 19:31:36,http://yostspinka.io/earnestine.sanford,,113.79.136.7,"{""location"": ""LR"", ""is_mobile"": false}" 9751,4,218,2017-05-31 18:40:01,http://hilpert.info/rachelle.bogisich,,247.15.205.41,"{""location"": ""AT"", ""is_mobile"": true}" 9752,4,218,2016-12-17 19:21:41,http://gorczany.name/marjolaine.howe,,100.39.190.196,"{""location"": ""GD"", ""is_mobile"": false}" 9753,4,218,2017-03-09 08:17:42,http://becker.com/jaydon.paucek,,155.6.2.107,"{""location"": ""BB"", ""is_mobile"": false}" 9754,4,218,2017-03-19 05:44:44,http://hoeger.co/caleigh,,138.237.13.241,"{""location"": ""RO"", ""is_mobile"": true}" 9755,4,219,2017-02-03 14:23:07,http://kohlerwalter.co/chanelle,,76.196.161.249,"{""location"": ""GI"", ""is_mobile"": false}" 9756,4,219,2017-05-14 01:43:40,http://moriettehirthe.name/raleigh.strosin,,19.193.4.175,"{""location"": ""CC"", ""is_mobile"": false}" 9757,4,219,2017-03-03 06:43:26,http://darewolff.info/dylan,,203.84.214.156,"{""location"": ""SO"", ""is_mobile"": false}" 9758,4,219,2017-05-19 00:05:10,http://miller.com/leila.kihn,,11.193.75.77,"{""location"": ""PM"", ""is_mobile"": true}" 9759,4,219,2017-05-24 09:30:11,http://hane.io/oscar,,118.86.33.244,"{""location"": ""TM"", ""is_mobile"": false}" 9760,4,219,2017-04-04 04:13:34,http://cristschmeler.org/savannah_schoen,,208.137.247.4,"{""location"": ""BD"", ""is_mobile"": false}" 9761,4,219,2017-06-02 10:14:37,http://marquardt.biz/lia_schowalter,,2.214.214.169,"{""location"": ""BV"", ""is_mobile"": false}" 9762,4,219,2017-06-02 02:52:58,http://simonisgoldner.biz/annabelle,,237.139.208.193,"{""location"": ""SJ"", ""is_mobile"": false}" 9763,4,219,2017-02-17 20:57:51,http://parisiankunde.biz/amie_nitzsche,,86.76.161.175,"{""location"": ""NL"", ""is_mobile"": true}" 9764,4,219,2016-12-22 08:47:38,http://schmidt.net/trent.mohr,,190.209.99.249,"{""location"": ""US"", ""is_mobile"": true}" 9765,4,219,2017-02-22 03:47:15,http://goldner.io/gerald,,90.245.133.181,"{""location"": ""SC"", ""is_mobile"": false}" 9766,4,219,2017-04-21 07:17:25,http://king.co/jackeline,,188.212.229.61,"{""location"": ""IM"", ""is_mobile"": true}" 9767,4,219,2017-02-13 01:56:29,http://gorczanysporer.name/dena.jast,,177.74.17.14,"{""location"": ""SO"", ""is_mobile"": false}" 9768,4,219,2017-01-03 00:26:54,http://dickinson.org/mateo.reynolds,,50.92.4.247,"{""location"": ""MZ"", ""is_mobile"": false}" 9769,4,219,2017-03-20 20:58:53,http://jast.name/herbert_kreiger,,155.142.70.158,"{""location"": ""GS"", ""is_mobile"": false}" 9770,4,219,2017-03-14 11:20:58,http://vonrueden.net/sister_jacobi,,34.107.41.203,"{""location"": ""GB"", ""is_mobile"": false}" 9771,4,219,2017-04-30 02:30:33,http://schmitt.io/randy,,247.178.124.222,"{""location"": ""HN"", ""is_mobile"": true}" 9772,4,219,2017-02-16 06:19:33,http://bednarnolan.io/linnie,,162.236.80.172,"{""location"": ""TH"", ""is_mobile"": false}" 9773,4,219,2017-03-05 13:20:52,http://boehm.org/susanna,,11.8.182.191,"{""location"": ""TN"", ""is_mobile"": true}" 9774,4,219,2017-06-06 14:18:05,http://mosciski.name/constantin.lang,,88.194.247.242,"{""location"": ""HN"", ""is_mobile"": false}" 9775,4,219,2017-06-11 02:06:09,http://mante.info/ebba.quigley,,249.165.104.26,"{""location"": ""SZ"", ""is_mobile"": false}" 9776,4,219,2016-12-26 04:07:35,http://reynolds.name/stephan,,204.179.220.164,"{""location"": ""BQ"", ""is_mobile"": true}" 9777,4,219,2017-01-25 07:39:06,http://medhurst.name/andres_schimmel,,207.118.20.154,"{""location"": ""TG"", ""is_mobile"": true}" 9778,4,219,2017-05-26 15:55:03,http://brekke.org/walton,,145.69.75.200,"{""location"": ""MY"", ""is_mobile"": true}" 9779,4,219,2017-06-03 23:22:40,http://andersonkreiger.co/eden,,118.107.132.48,"{""location"": ""YE"", ""is_mobile"": false}" 9780,4,219,2017-04-19 20:16:50,http://crona.com/pasquale_padberg,,233.247.128.87,"{""location"": ""SD"", ""is_mobile"": true}" 9781,4,219,2017-01-02 17:51:32,http://bergstrom.biz/gregoria,,13.71.148.214,"{""location"": ""GA"", ""is_mobile"": true}" 9782,4,219,2017-05-08 14:38:23,http://nicolas.net/cedrick,,99.10.4.164,"{""location"": ""MT"", ""is_mobile"": false}" 9783,4,219,2017-02-14 12:12:11,http://tremblay.info/tom_becker,,56.108.233.49,"{""location"": ""PF"", ""is_mobile"": true}" 9784,4,219,2017-05-31 15:22:00,http://johnsonheller.com/katrina,,97.73.19.224,"{""location"": ""QA"", ""is_mobile"": true}" 9785,4,219,2017-05-07 12:15:46,http://bauch.org/therese,,111.11.247.57,"{""location"": ""KG"", ""is_mobile"": true}" 9786,4,219,2017-06-08 09:41:46,http://hermann.com/odea,,16.122.216.230,"{""location"": ""PM"", ""is_mobile"": true}" 6723,3,147,2017-05-02 20:30:54,http://marquardt.co/elta_wilderman,,100.5.6.39,"{""location"": ""KY"", ""is_mobile"": true}" 6724,3,147,2017-01-27 23:43:00,http://swaniawskikreiger.co/colt,,120.10.214.240,"{""location"": ""AT"", ""is_mobile"": false}" 6725,3,147,2017-06-08 10:34:30,http://ritchie.net/frederik_goyette,,6.56.116.172,"{""location"": ""AI"", ""is_mobile"": true}" 6726,3,147,2017-06-01 22:52:46,http://daniel.org/mallory_howell,,169.132.94.199,"{""location"": ""BR"", ""is_mobile"": false}" 6727,3,147,2017-04-17 20:20:09,http://schoen.info/joe,,99.14.71.238,"{""location"": ""GM"", ""is_mobile"": true}" 6728,3,147,2017-02-21 13:00:41,http://ratkefarrell.org/esteban.green,,146.51.46.172,"{""location"": ""PE"", ""is_mobile"": false}" 6729,3,147,2017-03-18 08:26:36,http://blandadurgan.co/kyleigh.mueller,,107.48.176.117,"{""location"": ""MS"", ""is_mobile"": false}" 6730,3,147,2016-12-29 08:33:57,http://mcglynn.biz/una,,87.187.56.190,"{""location"": ""MU"", ""is_mobile"": false}" 6731,3,147,2017-01-07 08:49:08,http://swaniawski.name/pascale.hickle,,75.17.204.225,"{""location"": ""OM"", ""is_mobile"": false}" 6732,3,147,2017-04-03 23:27:38,http://bruen.info/angeline,,157.199.101.185,"{""location"": ""ZM"", ""is_mobile"": false}" 6733,3,147,2017-05-25 10:30:45,http://kulas.io/ervin,,251.169.204.70,"{""location"": ""CO"", ""is_mobile"": true}" 6734,3,147,2017-01-02 13:36:05,http://rau.com/aunta_fisher,,186.165.123.59,"{""location"": ""BA"", ""is_mobile"": true}" 6735,3,147,2017-01-09 16:56:27,http://vonruedenrunolfon.co/lilly,,98.95.114.242,"{""location"": ""MP"", ""is_mobile"": false}" 6736,3,147,2017-03-01 21:17:11,http://prohaskawiegand.io/tia_dibbert,,112.137.155.197,"{""location"": ""AL"", ""is_mobile"": true}" 6737,3,147,2017-05-11 14:48:46,http://mann.biz/rosendo.zulauf,,51.229.191.121,"{""location"": ""SK"", ""is_mobile"": true}" 6738,3,147,2017-01-18 15:22:14,http://wittinghaley.co/lauriane,,120.49.182.187,"{""location"": ""NA"", ""is_mobile"": true}" 6739,3,147,2017-04-14 07:51:58,http://eichmann.biz/trystan,,95.185.83.16,"{""location"": ""LB"", ""is_mobile"": false}" 6740,3,147,2017-05-11 14:48:57,http://carterbechtelar.net/clementine,,247.177.236.63,"{""location"": ""LK"", ""is_mobile"": true}" 6741,3,147,2017-02-01 05:19:24,http://conn.name/wanda_oreilly,,52.46.63.254,"{""location"": ""AD"", ""is_mobile"": true}" 6742,3,147,2017-03-13 17:04:23,http://vandervort.net/cicero_murazik,,252.42.193.221,"{""location"": ""PY"", ""is_mobile"": true}" 6743,3,148,2017-05-26 15:38:57,http://beer.co/roma,,72.38.185.225,"{""location"": ""AW"", ""is_mobile"": false}" 6744,3,148,2017-06-09 23:55:58,http://stoltenberg.biz/jedediah,,76.102.30.212,"{""location"": ""IQ"", ""is_mobile"": false}" 6745,3,148,2017-02-24 16:10:47,http://yundtzboncak.co/quentin_jast,,68.149.191.13,"{""location"": ""SH"", ""is_mobile"": true}" 6746,3,148,2016-12-18 10:53:03,http://morar.info/margarette,,204.12.250.211,"{""location"": ""MR"", ""is_mobile"": false}" 6747,3,148,2017-05-05 03:35:35,http://roobherzog.org/jacinthe_skiles,,116.229.20.177,"{""location"": ""MV"", ""is_mobile"": true}" 6748,3,148,2017-03-29 08:58:09,http://oreilly.co/elmira,,199.199.149.8,"{""location"": ""SB"", ""is_mobile"": true}" 6749,3,148,2017-03-20 03:33:08,http://harber.biz/merle,,135.95.233.79,"{""location"": ""MN"", ""is_mobile"": true}" 6750,3,148,2017-01-22 09:49:13,http://dietrichjaskolski.info/aniyah,,131.232.24.32,"{""location"": ""NC"", ""is_mobile"": true}" 6751,3,148,2017-04-01 07:41:05,http://mannankunding.info/geovany,,168.231.7.48,"{""location"": ""AT"", ""is_mobile"": true}" 6752,3,148,2017-03-21 12:47:30,http://kuhlmandoyle.name/uriel,,43.37.125.232,"{""location"": ""SV"", ""is_mobile"": false}" 6753,3,148,2017-01-03 22:43:38,http://hintz.com/emerson,,222.104.239.61,"{""location"": ""GB"", ""is_mobile"": false}" 6754,3,148,2017-01-27 20:36:01,http://mckenzietorphy.biz/ellis_nicolas,,233.162.94.143,"{""location"": ""KI"", ""is_mobile"": true}" 6755,3,148,2017-03-27 11:28:52,http://bogisich.biz/providenci.mosciski,,108.193.82.226,"{""location"": ""KN"", ""is_mobile"": true}" 6756,3,148,2017-04-21 17:14:20,http://gutmann.io/alf.hyatt,,47.127.220.253,"{""location"": ""NA"", ""is_mobile"": true}" 6757,3,148,2017-05-08 04:45:33,http://bosco.biz/reanna,,171.225.232.132,"{""location"": ""NL"", ""is_mobile"": false}" 6758,3,148,2017-03-03 10:37:45,http://feest.com/martine,,214.11.26.12,"{""location"": ""KN"", ""is_mobile"": true}" 6759,3,148,2017-01-03 10:33:26,http://lednerkoepp.com/william,,200.151.193.199,"{""location"": ""PG"", ""is_mobile"": true}" 6760,3,148,2017-05-13 16:37:57,http://grant.biz/ivah,,187.135.90.64,"{""location"": ""NG"", ""is_mobile"": false}" 6761,3,148,2017-03-06 17:00:26,http://kovacek.biz/kory.reinger,,99.105.53.174,"{""location"": ""ID"", ""is_mobile"": false}" 6762,3,148,2017-03-26 01:21:36,http://mcculloughschuppe.biz/jordane,,22.108.121.156,"{""location"": ""PF"", ""is_mobile"": true}" 6763,3,148,2016-12-22 23:29:16,http://oconnell.info/kallie.kling,,238.204.144.63,"{""location"": ""IS"", ""is_mobile"": false}" 6764,3,148,2017-03-02 09:17:57,http://reichel.com/gage_hackett,,87.135.193.55,"{""location"": ""SE"", ""is_mobile"": false}" 6765,3,148,2017-04-09 07:03:25,http://west.co/elya,,193.51.20.129,"{""location"": ""EH"", ""is_mobile"": true}" 6766,3,148,2017-03-28 01:16:40,http://eichmann.info/jeremie,,52.224.116.32,"{""location"": ""GE"", ""is_mobile"": false}" 6767,3,148,2017-03-31 17:43:50,http://haagbradtke.com/amelia.little,,214.90.190.56,"{""location"": ""SK"", ""is_mobile"": false}" 6768,3,148,2017-01-11 14:13:30,http://wunschreichel.org/bennett_kling,,97.249.138.92,"{""location"": ""MR"", ""is_mobile"": false}" 6769,3,148,2017-03-15 10:19:52,http://rosenbaumanderson.org/mckenna.price,,241.133.119.119,"{""location"": ""IN"", ""is_mobile"": false}" 6770,3,148,2017-02-06 00:43:13,http://rempel.biz/noemi.cartwright,,5.170.191.33,"{""location"": ""CA"", ""is_mobile"": true}" 6771,3,148,2017-04-04 15:09:27,http://sauer.io/eleonore.hyatt,,113.178.136.243,"{""location"": ""AQ"", ""is_mobile"": false}" 6772,3,148,2017-05-09 11:50:36,http://hickle.co/enid,,62.94.241.136,"{""location"": ""LC"", ""is_mobile"": false}" 6773,3,148,2017-01-03 02:46:33,http://farrelltorp.com/twila,,93.207.172.158,"{""location"": ""CX"", ""is_mobile"": true}" 6774,3,149,2017-05-23 13:55:29,http://spencer.name/whitney.hayes,,93.179.68.107,"{""location"": ""NR"", ""is_mobile"": false}" 6775,3,149,2017-02-25 10:30:08,http://smith.com/claire_kunde,,176.115.213.182,"{""location"": ""NL"", ""is_mobile"": true}" 6776,3,149,2017-02-25 03:56:55,http://yundt.info/tracy,,58.226.242.141,"{""location"": ""EC"", ""is_mobile"": false}" 6777,3,149,2017-05-01 20:58:22,http://block.co/anthony,,126.203.182.209,"{""location"": ""TF"", ""is_mobile"": false}" 6778,3,149,2017-02-25 21:29:58,http://fadelhamill.co/greta,,50.60.81.24,"{""location"": ""HM"", ""is_mobile"": false}" 12638,5,282,2017-04-21 03:05:51,http://beerharvey.com/quentin,,60.60.26.234,"{""location"": ""TO"", ""is_mobile"": false}" 12639,5,282,2017-05-23 06:14:50,http://donnellyshanahan.com/roscoe_nitzsche,,151.60.190.123,"{""location"": ""AL"", ""is_mobile"": false}" 12640,5,282,2017-03-06 12:06:48,http://will.io/emilio.berge,,158.106.77.204,"{""location"": ""LB"", ""is_mobile"": true}" 12641,5,282,2017-03-21 12:02:28,http://klocko.net/daniella,,155.30.101.205,"{""location"": ""TV"", ""is_mobile"": false}" 12642,5,282,2017-04-26 05:09:00,http://osinskiratke.com/christa,,23.183.82.67,"{""location"": ""DM"", ""is_mobile"": true}" 12643,5,282,2017-03-01 23:49:44,http://kshlerin.org/abner,,22.120.95.2,"{""location"": ""VA"", ""is_mobile"": false}" 12644,5,282,2017-02-16 19:53:40,http://willms.io/garrison,,94.198.58.175,"{""location"": ""TD"", ""is_mobile"": true}" 12645,5,282,2017-03-01 10:51:08,http://lind.net/josie,,254.78.210.48,"{""location"": ""TF"", ""is_mobile"": false}" 12646,5,282,2017-05-10 08:33:03,http://wisoky.biz/dalton.kuvalis,,27.140.115.93,"{""location"": ""MO"", ""is_mobile"": false}" 12647,5,282,2017-03-15 14:23:33,http://reinger.net/lenna_mann,,233.82.92.26,"{""location"": ""VN"", ""is_mobile"": false}" 12648,5,282,2016-12-18 12:45:22,http://ryan.info/selina,,221.108.41.20,"{""location"": ""FO"", ""is_mobile"": true}" 12649,5,282,2017-04-11 02:02:19,http://okeefe.io/vito.rosenbaum,,46.89.78.144,"{""location"": ""CZ"", ""is_mobile"": false}" 12650,5,282,2017-04-07 22:15:18,http://blanda.info/zoie_dickinson,,137.176.82.202,"{""location"": ""RU"", ""is_mobile"": true}" 12651,5,282,2017-01-05 02:31:03,http://macgyver.co/otho_olson,,229.225.163.190,"{""location"": ""ES"", ""is_mobile"": false}" 12652,5,282,2017-01-29 15:22:42,http://kotreutel.org/alford.zieme,,8.45.8.117,"{""location"": ""EC"", ""is_mobile"": false}" 12653,5,282,2017-04-01 06:15:14,http://rowemcclure.org/leonard,,211.238.236.157,"{""location"": ""SV"", ""is_mobile"": true}" 12654,5,282,2017-01-02 18:24:25,http://weimann.name/westley_hahn,,105.203.42.5,"{""location"": ""AZ"", ""is_mobile"": false}" 12655,5,282,2017-01-09 15:37:34,http://reillyhagenes.com/chelsey,,204.244.110.169,"{""location"": ""VI"", ""is_mobile"": true}" 12656,5,282,2017-06-13 19:18:39,http://hermiston.info/enos,,17.42.233.119,"{""location"": ""MV"", ""is_mobile"": false}" 12657,5,282,2017-04-12 22:51:25,http://auerdibbert.biz/kris,,166.100.133.65,"{""location"": ""LU"", ""is_mobile"": true}" 12658,5,283,2017-05-22 14:45:52,http://sanford.biz/itzel_abernathy,,98.80.16.74,"{""location"": ""ST"", ""is_mobile"": false}" 12659,5,283,2017-01-05 07:58:12,http://schulist.biz/ezequiel,,137.20.123.216,"{""location"": ""PY"", ""is_mobile"": false}" 12660,5,283,2017-05-14 07:27:05,http://jaskolski.biz/jaylon,,48.77.203.18,"{""location"": ""GW"", ""is_mobile"": false}" 12661,5,283,2017-03-28 10:18:06,http://rosenbaum.name/reyna,,12.151.195.196,"{""location"": ""NL"", ""is_mobile"": true}" 12662,5,283,2017-06-05 10:45:40,http://cremin.info/bernadette,,208.148.198.189,"{""location"": ""BN"", ""is_mobile"": true}" 12663,5,283,2017-01-13 11:02:45,http://batz.com/payton_herzog,,193.6.151.153,"{""location"": ""ME"", ""is_mobile"": false}" 12664,5,283,2017-06-12 08:20:21,http://cummings.info/aliyah.ohara,,95.2.166.158,"{""location"": ""SB"", ""is_mobile"": true}" 12665,5,283,2017-04-03 18:55:41,http://koepp.com/emory,,202.198.162.108,"{""location"": ""RU"", ""is_mobile"": true}" 12666,5,283,2017-04-16 05:24:51,http://schmeler.net/velda,,195.231.187.59,"{""location"": ""LB"", ""is_mobile"": false}" 12667,5,283,2017-05-15 14:44:25,http://turnerstokes.biz/elfrieda,,168.186.153.194,"{""location"": ""GN"", ""is_mobile"": true}" 12668,5,283,2017-06-08 15:51:59,http://mckenzie.com/valentina_hettinger,,209.149.121.82,"{""location"": ""KR"", ""is_mobile"": false}" 12669,5,283,2017-02-06 10:40:12,http://simonisreynolds.net/margarete_frami,,104.20.43.197,"{""location"": ""IL"", ""is_mobile"": true}" 12670,5,283,2016-12-16 15:49:16,http://bruen.info/alice_hauck,,94.203.37.149,"{""location"": ""IN"", ""is_mobile"": true}" 12671,5,283,2017-02-13 15:21:59,http://blockberge.io/jordi,,156.128.206.209,"{""location"": ""KN"", ""is_mobile"": false}" 12672,5,283,2017-06-10 19:01:19,http://lehner.com/jewel,,118.76.109.3,"{""location"": ""BA"", ""is_mobile"": false}" 12673,5,283,2017-06-05 10:33:02,http://koepp.name/jamey,,132.68.78.153,"{""location"": ""TC"", ""is_mobile"": true}" 12674,5,283,2017-05-12 18:25:45,http://olsonbechtelar.org/jennings,,33.213.140.97,"{""location"": ""NG"", ""is_mobile"": false}" 12675,5,283,2016-12-28 20:15:03,http://green.biz/keven.hodkiewicz,,145.101.241.28,"{""location"": ""KG"", ""is_mobile"": true}" 12676,5,283,2017-05-31 12:28:05,http://gutmann.com/jonatan_hyatt,,67.117.132.176,"{""location"": ""SY"", ""is_mobile"": true}" 12677,5,283,2017-02-16 07:09:34,http://cremin.org/christop,,185.44.78.142,"{""location"": ""CO"", ""is_mobile"": false}" 12678,5,283,2017-01-25 04:14:44,http://grimes.info/maude,,213.211.65.222,"{""location"": ""JP"", ""is_mobile"": true}" 12679,5,283,2017-05-28 18:02:45,http://roob.net/gregorio.nolan,,44.16.234.13,"{""location"": ""SY"", ""is_mobile"": false}" 12680,5,283,2017-04-22 18:50:30,http://ryanschiller.info/guiseppe,,9.59.160.216,"{""location"": ""AS"", ""is_mobile"": false}" 12681,5,283,2017-03-30 10:58:20,http://kemmerbosco.biz/layla.jerde,,199.90.150.138,"{""location"": ""LB"", ""is_mobile"": true}" 12682,5,283,2017-02-16 02:23:24,http://wunsch.biz/cary_kilback,,231.26.193.67,"{""location"": ""PM"", ""is_mobile"": true}" 12683,5,283,2017-01-01 17:42:00,http://okeefe.biz/milo,,181.175.110.237,"{""location"": ""VC"", ""is_mobile"": false}" 12684,5,283,2017-05-23 13:08:14,http://lehner.biz/susanna.leuschke,,58.191.91.176,"{""location"": ""TT"", ""is_mobile"": true}" 12685,5,283,2017-05-17 15:38:04,http://dietrich.co/lenore.macgyver,,56.132.157.83,"{""location"": ""EG"", ""is_mobile"": true}" 12686,5,283,2017-06-12 05:03:55,http://smitham.org/barton.homenick,,119.167.138.171,"{""location"": ""EG"", ""is_mobile"": true}" 12687,5,283,2017-05-31 04:14:03,http://morar.info/jeramy,,181.226.56.58,"{""location"": ""PF"", ""is_mobile"": false}" 12688,5,283,2017-06-06 10:10:34,http://johnson.io/darian_fay,,54.68.129.242,"{""location"": ""GR"", ""is_mobile"": true}" 12689,5,283,2017-02-08 05:32:48,http://heller.net/stephan,,89.32.150.92,"{""location"": ""BM"", ""is_mobile"": false}" 12690,5,283,2016-12-25 03:32:46,http://mckenzie.co/merle.moore,,162.252.223.108,"{""location"": ""GT"", ""is_mobile"": false}" 12691,5,283,2016-12-28 06:33:48,http://lind.name/casimir_anderson,,137.232.61.205,"{""location"": ""MO"", ""is_mobile"": true}" 12692,5,283,2017-05-13 05:03:19,http://mraz.io/sigmund,,3.197.215.104,"{""location"": ""MK"", ""is_mobile"": true}" 12693,5,283,2017-04-04 06:45:58,http://nolanrutherford.com/jazmyne,,137.12.182.206,"{""location"": ""IS"", ""is_mobile"": true}" 15641,6,352,2017-03-22 10:26:14,http://pollich.net/pat,,188.61.63.58,"{""location"": ""VN"", ""is_mobile"": false}" 15642,6,352,2017-04-03 01:36:56,http://thiel.net/reid,,82.41.221.49,"{""location"": ""AW"", ""is_mobile"": true}" 15643,6,352,2017-01-07 09:02:14,http://tillmansawayn.com/marisol_herman,,61.138.6.2,"{""location"": ""KE"", ""is_mobile"": false}" 15644,6,352,2017-05-06 06:38:37,http://dickens.com/ashton,,238.156.179.135,"{""location"": ""AM"", ""is_mobile"": true}" 15645,6,352,2017-05-03 08:23:11,http://greenholt.biz/burley_conroy,,183.32.196.171,"{""location"": ""AW"", ""is_mobile"": true}" 15646,6,352,2017-04-22 10:27:56,http://bernhard.biz/duane.schmeler,,35.146.182.251,"{""location"": ""PY"", ""is_mobile"": false}" 15647,6,352,2017-01-31 14:55:50,http://gulgowski.io/sibyl.hagenes,,81.149.244.71,"{""location"": ""AX"", ""is_mobile"": false}" 15648,6,352,2017-01-07 05:57:35,http://will.net/anabelle_turner,,86.253.233.107,"{""location"": ""GN"", ""is_mobile"": false}" 15649,6,352,2017-02-01 22:59:19,http://oconner.org/loy,,84.32.77.20,"{""location"": ""TH"", ""is_mobile"": true}" 15650,6,352,2017-01-20 17:20:24,http://koelpinullrich.org/jaylin,,214.101.176.117,"{""location"": ""QA"", ""is_mobile"": false}" 15651,6,352,2017-01-17 20:14:21,http://bradtke.biz/jaron,,164.62.73.241,"{""location"": ""BV"", ""is_mobile"": false}" 15652,6,352,2017-03-15 18:48:10,http://reilly.org/trenton,,139.23.59.244,"{""location"": ""HT"", ""is_mobile"": true}" 15653,6,352,2017-04-08 05:51:56,http://wunsch.name/amie.pfannerstill,,66.159.139.208,"{""location"": ""RU"", ""is_mobile"": true}" 15654,6,352,2017-05-05 15:52:06,http://mertz.info/vincenza.greenholt,,253.71.131.50,"{""location"": ""PG"", ""is_mobile"": true}" 15655,6,352,2017-01-08 08:53:09,http://andersonstamm.net/kaitlyn,,3.132.182.72,"{""location"": ""MU"", ""is_mobile"": false}" 15656,6,352,2017-04-03 07:11:55,http://kulas.io/kailee,,115.173.180.185,"{""location"": ""VA"", ""is_mobile"": true}" 15657,6,352,2017-01-17 00:54:22,http://bergstrom.name/justyn,,201.28.119.175,"{""location"": ""TJ"", ""is_mobile"": true}" 15658,6,352,2017-01-15 23:49:10,http://hintzratke.net/liza,,54.112.118.94,"{""location"": ""PY"", ""is_mobile"": false}" 15659,6,352,2017-04-16 17:58:27,http://kunze.io/colby,,192.26.52.234,"{""location"": ""IS"", ""is_mobile"": false}" 15660,6,352,2017-03-31 23:08:13,http://lind.biz/annabelle,,241.131.137.60,"{""location"": ""NC"", ""is_mobile"": false}" 15661,6,352,2017-01-25 02:19:19,http://kling.name/carlos.cruickshank,,241.50.16.106,"{""location"": ""EC"", ""is_mobile"": false}" 15662,6,352,2017-01-31 12:16:46,http://lueilwitz.biz/cullen.runolfon,,47.137.58.230,"{""location"": ""CO"", ""is_mobile"": false}" 15663,6,352,2017-05-23 16:26:24,http://trantowmoore.name/nathanial,,213.67.228.221,"{""location"": ""PY"", ""is_mobile"": false}" 15664,6,352,2017-01-18 18:28:50,http://kozeybeer.name/birdie,,178.20.86.119,"{""location"": ""CD"", ""is_mobile"": true}" 15665,6,352,2017-05-12 00:56:49,http://kaulke.com/rahsaan,,125.2.8.27,"{""location"": ""SS"", ""is_mobile"": false}" 15666,6,352,2017-02-24 05:55:42,http://kirlin.net/elvis.marquardt,,186.165.101.115,"{""location"": ""NO"", ""is_mobile"": true}" 15667,6,352,2017-02-03 05:04:27,http://fahey.name/johnny.mante,,10.240.151.250,"{""location"": ""CY"", ""is_mobile"": true}" 15668,6,352,2017-02-14 16:24:24,http://hauck.net/eleanore,,34.229.111.226,"{""location"": ""BY"", ""is_mobile"": false}" 15669,6,352,2017-05-14 00:45:14,http://johnsonkuhic.name/raina.koch,,85.104.57.7,"{""location"": ""RW"", ""is_mobile"": false}" 15670,6,352,2017-04-18 02:16:53,http://senger.io/ezequiel.schmitt,,91.101.183.20,"{""location"": ""SA"", ""is_mobile"": true}" 15671,6,352,2017-03-21 15:45:42,http://stromanschmeler.info/tyshawn.kemmer,,48.14.11.177,"{""location"": ""CZ"", ""is_mobile"": false}" 15672,6,353,2017-03-27 03:13:57,http://williamson.co/porter,,232.168.138.89,"{""location"": ""JE"", ""is_mobile"": true}" 15673,6,353,2017-01-23 03:28:11,http://purdyveum.co/luz_buckridge,,6.191.233.7,"{""location"": ""IQ"", ""is_mobile"": false}" 15674,6,353,2017-04-02 23:28:36,http://erdman.co/elizabeth,,189.132.15.229,"{""location"": ""ID"", ""is_mobile"": true}" 15675,6,353,2017-03-31 13:17:52,http://roobrohan.info/nels_kertzmann,,174.58.222.85,"{""location"": ""CC"", ""is_mobile"": false}" 15676,6,353,2016-12-25 17:02:46,http://bogisichrippin.biz/vita,,174.11.252.227,"{""location"": ""TL"", ""is_mobile"": false}" 15677,6,353,2017-02-27 09:32:12,http://rodriguez.org/arlo_schulist,,44.249.250.113,"{""location"": ""DE"", ""is_mobile"": false}" 15678,6,353,2017-03-09 14:21:28,http://block.co/leie_dickinson,,75.34.253.21,"{""location"": ""NL"", ""is_mobile"": true}" 15679,6,353,2017-05-11 23:38:26,http://bernhardsporer.org/vance,,6.54.240.224,"{""location"": ""PK"", ""is_mobile"": true}" 15680,6,353,2017-04-21 07:27:57,http://smitham.info/lilly_king,,184.129.215.151,"{""location"": ""ZA"", ""is_mobile"": false}" 15681,6,353,2017-01-28 00:24:16,http://mcglynn.co/mariam,,216.10.233.29,"{""location"": ""WS"", ""is_mobile"": false}" 15682,6,353,2017-05-27 20:05:52,http://bogan.io/yeenia_abshire,,158.156.106.161,"{""location"": ""TN"", ""is_mobile"": false}" 15683,6,353,2017-04-21 10:31:14,http://schaden.info/joel,,133.107.127.198,"{""location"": ""CO"", ""is_mobile"": true}" 15684,6,353,2017-04-15 04:28:18,http://brakuskuvalis.org/verna.larson,,184.123.37.237,"{""location"": ""TO"", ""is_mobile"": true}" 15685,6,353,2017-05-06 13:12:22,http://johnson.io/sincere_dickinson,,147.64.57.3,"{""location"": ""MZ"", ""is_mobile"": true}" 15686,6,353,2017-03-16 18:54:02,http://lueilwitz.com/viola.kshlerin,,242.76.140.154,"{""location"": ""BS"", ""is_mobile"": true}" 15687,6,353,2017-03-14 00:51:03,http://mckenzie.org/destiny,,206.4.201.72,"{""location"": ""NR"", ""is_mobile"": true}" 15688,6,353,2017-03-22 00:51:49,http://johnston.info/delmer.abshire,,86.210.119.199,"{""location"": ""SX"", ""is_mobile"": false}" 15689,6,353,2017-01-04 10:27:58,http://ohara.co/shannon,,155.80.197.166,"{""location"": ""PA"", ""is_mobile"": true}" 15690,6,353,2017-03-21 13:14:39,http://larkin.info/marquise,,36.219.71.30,"{""location"": ""SV"", ""is_mobile"": true}" 15691,6,353,2017-03-05 21:21:28,http://olsontreutel.info/katelin_blick,,191.186.170.228,"{""location"": ""AO"", ""is_mobile"": false}" 15692,6,353,2017-04-03 02:44:55,http://romaguera.io/lucile,,16.114.194.185,"{""location"": ""YT"", ""is_mobile"": true}" 15693,6,353,2016-12-21 18:25:54,http://crooks.net/marlee.wolff,,251.140.116.66,"{""location"": ""BL"", ""is_mobile"": false}" 15694,6,353,2017-03-08 16:50:10,http://collier.co/zoe.kris,,186.197.46.47,"{""location"": ""IN"", ""is_mobile"": false}" 15695,6,353,2017-02-27 17:22:45,http://schuster.org/kolby_ortiz,,104.254.189.238,"{""location"": ""VG"", ""is_mobile"": true}" 2852,2,63,2017-04-01 13:54:04,http://franecki.com/katarina_ernser,,168.130.238.203,"{""location"": ""CV"", ""is_mobile"": true}" 2853,2,63,2017-03-30 17:42:07,http://schumm.net/keira,,99.50.111.133,"{""location"": ""BN"", ""is_mobile"": false}" 2854,2,63,2017-04-26 01:24:46,http://quigley.com/lorine_kuvalis,,35.171.116.38,"{""location"": ""CY"", ""is_mobile"": false}" 2855,2,63,2017-04-17 13:46:59,http://murphyturner.com/danielle_mohr,,205.228.242.109,"{""location"": ""TT"", ""is_mobile"": false}" 2856,2,63,2017-01-29 16:16:18,http://strosin.name/zoey,,41.52.203.28,"{""location"": ""LK"", ""is_mobile"": true}" 2857,2,63,2017-05-11 21:41:32,http://stroman.io/kira_gislason,,11.208.13.63,"{""location"": ""SK"", ""is_mobile"": false}" 2858,2,63,2017-05-30 15:58:53,http://weimann.co/lisette_christiansen,,49.67.189.15,"{""location"": ""KN"", ""is_mobile"": true}" 2859,2,63,2017-02-02 20:15:28,http://champlinferry.org/allie,,144.254.120.150,"{""location"": ""MP"", ""is_mobile"": false}" 2860,2,63,2017-04-10 11:01:19,http://runolfon.biz/karlee_rohan,,42.221.164.55,"{""location"": ""CL"", ""is_mobile"": true}" 2861,2,63,2017-05-11 22:07:51,http://mccullough.com/nicolette_armstrong,,95.242.164.113,"{""location"": ""MK"", ""is_mobile"": true}" 2862,2,63,2016-12-29 19:01:10,http://wolf.io/lewis,,146.49.105.154,"{""location"": ""IQ"", ""is_mobile"": false}" 2863,2,63,2017-04-02 02:58:03,http://baumbach.info/haley.graham,,43.169.61.202,"{""location"": ""KP"", ""is_mobile"": false}" 2864,2,63,2017-01-09 09:22:03,http://pfeffer.name/fleta,,75.76.113.194,"{""location"": ""BY"", ""is_mobile"": true}" 2865,2,63,2017-06-02 02:22:22,http://gislason.info/isom,,234.140.250.140,"{""location"": ""IQ"", ""is_mobile"": true}" 2866,2,63,2017-01-19 09:57:54,http://trantowgleason.com/harold,,150.18.21.66,"{""location"": ""NZ"", ""is_mobile"": false}" 2867,2,63,2016-12-25 15:59:57,http://emmerich.org/thaddeus.hilpert,,117.50.4.91,"{""location"": ""LA"", ""is_mobile"": true}" 2868,2,63,2016-12-30 08:43:02,http://volkman.io/mack,,79.170.25.174,"{""location"": ""NI"", ""is_mobile"": false}" 2869,2,63,2017-01-11 10:53:44,http://dickens.co/ben,,68.69.218.19,"{""location"": ""EE"", ""is_mobile"": false}" 2870,2,63,2016-12-19 08:02:03,http://jacobson.name/tomas,,63.112.155.212,"{""location"": ""KZ"", ""is_mobile"": true}" 2871,2,63,2017-03-05 20:05:31,http://danieltillman.org/rasheed.ferry,,133.147.230.215,"{""location"": ""CK"", ""is_mobile"": true}" 2872,2,63,2017-04-29 22:50:08,http://ullrich.io/ayana.ruecker,,156.155.170.145,"{""location"": ""UY"", ""is_mobile"": false}" 2873,2,63,2017-06-01 11:31:21,http://hilpert.com/lewis.wolff,,86.245.54.89,"{""location"": ""ZA"", ""is_mobile"": true}" 2874,2,63,2017-04-03 20:12:37,http://lockmanmayer.net/josiah,,91.184.146.232,"{""location"": ""NL"", ""is_mobile"": true}" 2875,2,64,2017-03-18 19:41:57,http://beatty.com/dawn.cronin,0.9511796056,183.134.179.232,"{""location"": ""BH"", ""is_mobile"": true}" 2876,2,64,2017-06-10 08:46:59,http://aufderhar.io/myriam,0.6346225665,209.22.192.110,"{""location"": ""CI"", ""is_mobile"": true}" 2877,2,64,2017-04-26 01:14:04,http://wisozk.info/timmothy.kunze,0.1928551209,2.9.181.44,"{""location"": ""NF"", ""is_mobile"": true}" 2878,2,64,2017-03-18 07:22:24,http://mitchell.org/alayna,0.6019735012,129.108.166.80,"{""location"": ""AO"", ""is_mobile"": true}" 2879,2,64,2017-04-19 03:40:44,http://murazik.net/henderson_toy,0.2634694294,34.67.13.63,"{""location"": ""AG"", ""is_mobile"": false}" 2880,2,64,2017-05-06 07:46:28,http://fisher.biz/selina,0.7121637419,16.204.208.214,"{""location"": ""RO"", ""is_mobile"": false}" 2881,2,64,2017-02-07 19:24:15,http://jacobs.name/krystel.kovacek,0.8676960906,184.71.50.153,"{""location"": ""MY"", ""is_mobile"": true}" 2882,2,64,2016-12-27 06:03:20,http://deckowreynolds.name/celine.deckow,0.8177789743,179.166.30.183,"{""location"": ""RW"", ""is_mobile"": false}" 2883,2,64,2017-04-06 01:19:21,http://schulist.net/earnest,0.5083965950,83.230.236.96,"{""location"": ""MK"", ""is_mobile"": false}" 2884,2,64,2017-05-03 19:36:02,http://swaniawskiheel.com/valentin.williamson,0.9581201732,228.41.244.207,"{""location"": ""LT"", ""is_mobile"": true}" 2885,2,64,2017-03-04 18:21:57,http://eichmann.com/chyna.veum,0.9601646946,198.89.119.146,"{""location"": ""BH"", ""is_mobile"": false}" 2886,2,64,2017-06-14 01:01:00,http://bogisichrippin.name/kaya_flatley,0.5277688758,190.78.123.52,"{""location"": ""KR"", ""is_mobile"": true}" 2887,2,64,2017-02-28 15:50:23,http://trantow.info/ruel,0.0960232109,244.25.47.214,"{""location"": ""MF"", ""is_mobile"": false}" 2888,2,64,2017-03-07 14:12:18,http://carter.io/seth_watsica,0.7066932917,126.198.162.216,"{""location"": ""CV"", ""is_mobile"": true}" 2889,2,64,2017-04-04 10:44:36,http://rippin.name/emmitt.quigley,0.0534928588,249.141.63.22,"{""location"": ""RW"", ""is_mobile"": false}" 2890,2,64,2017-04-13 04:33:25,http://zulauf.io/joseph,0.7075664303,98.44.114.157,"{""location"": ""AZ"", ""is_mobile"": false}" 2891,2,64,2017-03-04 16:06:25,http://fisherbogisich.co/helga_grant,0.7161153806,91.104.240.81,"{""location"": ""SH"", ""is_mobile"": true}" 2892,2,64,2017-01-09 00:40:36,http://handrutherford.net/mireya_wilderman,0.5680583550,150.87.105.193,"{""location"": ""GL"", ""is_mobile"": true}" 2893,2,64,2017-05-17 20:59:28,http://swaniawski.io/jacques_corwin,0.2410868668,230.98.180.68,"{""location"": ""WF"", ""is_mobile"": true}" 2894,2,64,2017-01-20 03:54:49,http://kuhn.org/alexandro,0.1077098089,129.62.119.213,"{""location"": ""NR"", ""is_mobile"": false}" 2895,2,64,2017-05-08 05:03:21,http://conroy.io/kyla_sporer,0.2012890936,215.16.67.251,"{""location"": ""MG"", ""is_mobile"": true}" 2896,2,64,2017-05-28 00:19:27,http://leannon.com/laverne,0.9760099187,16.145.181.55,"{""location"": ""TF"", ""is_mobile"": true}" 2897,2,64,2017-01-22 05:00:05,http://heel.com/kay_strosin,0.0065256895,115.208.79.242,"{""location"": ""AX"", ""is_mobile"": false}" 2898,2,64,2017-02-12 04:37:47,http://zboncakkunze.info/sedrick.brekke,0.6712320237,72.37.223.252,"{""location"": ""NI"", ""is_mobile"": false}" 2899,2,64,2017-03-10 18:43:41,http://stehr.net/darius.volkman,0.6126397169,114.221.170.35,"{""location"": ""MK"", ""is_mobile"": false}" 2900,2,64,2017-03-07 12:48:02,http://purdy.io/alf,0.0183792302,209.122.170.47,"{""location"": ""PM"", ""is_mobile"": true}" 2901,2,64,2017-04-17 14:53:17,http://ziemanndibbert.net/mariam.denesik,0.3137354145,228.199.64.234,"{""location"": ""AF"", ""is_mobile"": true}" 2902,2,64,2017-05-27 01:44:39,http://purdyhoppe.co/nathan.emard,0.7654613637,189.58.231.216,"{""location"": ""NG"", ""is_mobile"": false}" 2903,2,64,2017-04-22 00:20:02,http://rolfson.io/juana,0.7987053689,125.250.53.68,"{""location"": ""ZM"", ""is_mobile"": false}" 2904,2,64,2017-03-09 02:18:53,http://weinatweber.org/darion,0.1256875556,19.231.13.113,"{""location"": ""VN"", ""is_mobile"": true}" 2905,2,64,2017-04-24 07:44:05,http://nader.net/amelie_bauch,0.3906543467,229.28.63.116,"{""location"": ""TM"", ""is_mobile"": true}" 9787,4,219,2017-05-21 11:05:17,http://ankunding.info/bethel.volkman,,121.253.224.147,"{""location"": ""MH"", ""is_mobile"": true}" 9788,4,219,2017-02-21 20:07:39,http://kohler.org/ada,,180.151.36.119,"{""location"": ""PA"", ""is_mobile"": true}" 9789,4,219,2017-02-28 17:12:04,http://kihnhayes.name/nannie,,130.19.138.30,"{""location"": ""HU"", ""is_mobile"": true}" 9790,4,219,2017-02-05 15:16:45,http://fishermetz.org/guy.kunze,,193.171.16.81,"{""location"": ""MO"", ""is_mobile"": true}" 9791,4,219,2017-01-06 04:27:06,http://armstrong.org/ona.lubowitz,,252.23.176.149,"{""location"": ""NU"", ""is_mobile"": false}" 9792,4,219,2017-05-23 20:33:24,http://mayertbosco.org/rebeka.gutkowski,,254.134.20.12,"{""location"": ""VU"", ""is_mobile"": true}" 9793,4,219,2017-04-13 10:23:16,http://erdman.io/aditya.spinka,,129.194.170.96,"{""location"": ""MU"", ""is_mobile"": false}" 9794,4,220,2017-03-10 01:37:56,http://torp.com/geovany_pfeffer,,178.100.230.142,"{""location"": ""SA"", ""is_mobile"": true}" 9795,4,220,2016-12-14 10:13:40,http://schneiderkemmer.com/adella,,72.193.27.60,"{""location"": ""PG"", ""is_mobile"": true}" 9796,4,220,2017-03-11 19:29:42,http://veumsauer.io/estel_kohler,,120.73.169.23,"{""location"": ""KI"", ""is_mobile"": false}" 9797,4,220,2017-01-25 04:26:57,http://wyman.info/barrett_denesik,,147.223.45.241,"{""location"": ""GF"", ""is_mobile"": true}" 9798,4,220,2017-05-22 00:24:35,http://von.net/alexane_mante,,210.170.123.249,"{""location"": ""BQ"", ""is_mobile"": true}" 9799,4,220,2017-02-21 04:28:49,http://kiehn.org/kelley,,247.92.243.251,"{""location"": ""GQ"", ""is_mobile"": true}" 9800,4,220,2017-04-11 21:33:15,http://schiller.org/rodrigo,,116.251.11.162,"{""location"": ""SO"", ""is_mobile"": false}" 9801,4,220,2016-12-14 20:25:04,http://dare.biz/wilhelm.green,,48.104.161.205,"{""location"": ""GF"", ""is_mobile"": false}" 9802,4,220,2017-04-10 17:24:54,http://starkmorar.info/lupe_kling,,117.4.174.49,"{""location"": ""LY"", ""is_mobile"": false}" 9803,4,220,2017-06-02 04:02:36,http://altenwerthgislason.name/ramon,,149.164.60.180,"{""location"": ""ST"", ""is_mobile"": false}" 9804,4,220,2017-05-21 09:43:25,http://larson.info/austin.armstrong,,151.7.149.149,"{""location"": ""MD"", ""is_mobile"": false}" 9805,4,220,2017-03-22 16:52:57,http://purdy.co/bernie,,56.29.166.119,"{""location"": ""CD"", ""is_mobile"": false}" 9806,4,220,2017-05-30 02:47:06,http://hickle.biz/sammy,,170.9.100.241,"{""location"": ""AU"", ""is_mobile"": false}" 9807,4,220,2017-06-05 23:41:37,http://ziemann.name/bernadine,,123.87.61.148,"{""location"": ""BE"", ""is_mobile"": true}" 9808,4,220,2017-02-07 20:57:06,http://schaeferweinat.com/everette,,54.21.199.171,"{""location"": ""TW"", ""is_mobile"": false}" 9809,4,220,2017-03-04 04:15:29,http://schuster.info/leta,,116.167.146.100,"{""location"": ""GI"", ""is_mobile"": true}" 9810,4,220,2017-01-22 08:04:42,http://hintz.co/sabrina,,217.162.223.130,"{""location"": ""YE"", ""is_mobile"": true}" 9811,4,220,2017-06-05 22:56:46,http://will.biz/meda,,50.178.28.120,"{""location"": ""SG"", ""is_mobile"": false}" 9812,4,220,2017-01-03 12:07:18,http://schaefer.info/cornelius,,232.180.229.135,"{""location"": ""IM"", ""is_mobile"": true}" 9813,4,220,2017-03-29 06:21:49,http://terrystreich.co/madyson,,87.182.47.244,"{""location"": ""FR"", ""is_mobile"": true}" 9814,4,220,2017-05-21 13:09:38,http://stoltenberg.net/marlee.bashirian,,154.218.202.208,"{""location"": ""BW"", ""is_mobile"": true}" 9815,4,220,2016-12-26 00:58:22,http://dubuque.biz/ottilie_hamill,,141.34.196.98,"{""location"": ""RU"", ""is_mobile"": true}" 9816,4,220,2017-05-11 17:22:49,http://wiegand.biz/nash,,83.40.33.72,"{""location"": ""LT"", ""is_mobile"": false}" 9817,4,220,2017-03-23 05:00:04,http://damore.info/lionel,,95.137.20.200,"{""location"": ""MN"", ""is_mobile"": true}" 9818,4,220,2017-04-10 11:21:03,http://lindgrentromp.name/rosemarie.effertz,,190.114.214.165,"{""location"": ""FO"", ""is_mobile"": false}" 9819,4,220,2017-04-04 13:30:18,http://schillercummings.co/velda.wolff,,85.64.25.180,"{""location"": ""SL"", ""is_mobile"": true}" 9820,4,220,2017-05-25 14:54:12,http://conroybuckridge.com/fabian,,85.78.138.82,"{""location"": ""ER"", ""is_mobile"": false}" 9821,4,220,2017-06-14 00:48:45,http://buckridge.name/minnie_olson,,200.90.28.64,"{""location"": ""IS"", ""is_mobile"": true}" 9822,4,220,2017-04-27 17:32:07,http://kohaag.co/montana,,133.42.115.179,"{""location"": ""TL"", ""is_mobile"": true}" 9823,4,220,2017-01-01 07:38:52,http://langoshhermiston.biz/kailee,,109.207.74.6,"{""location"": ""KW"", ""is_mobile"": true}" 9824,4,220,2017-05-27 23:25:37,http://torphy.info/leopoldo,,35.81.229.70,"{""location"": ""LK"", ""is_mobile"": true}" 9825,4,220,2017-05-29 20:58:43,http://senger.info/jordane,,196.216.145.156,"{""location"": ""IR"", ""is_mobile"": true}" 9826,4,220,2017-04-29 17:57:04,http://torphy.name/cyrus,,132.42.25.179,"{""location"": ""KM"", ""is_mobile"": true}" 9827,4,220,2017-02-17 04:52:13,http://harber.co/aliya.wuckert,,98.159.169.140,"{""location"": ""GI"", ""is_mobile"": true}" 9828,4,220,2017-04-14 00:16:11,http://sporer.org/johnson_kutch,,7.109.126.162,"{""location"": ""BE"", ""is_mobile"": false}" 9829,4,220,2017-04-05 19:13:46,http://skileshintz.net/casper.ritchie,,164.86.75.232,"{""location"": ""UY"", ""is_mobile"": true}" 9830,4,220,2017-05-19 08:58:36,http://baumbach.co/bret,,54.102.182.23,"{""location"": ""HR"", ""is_mobile"": true}" 9831,4,220,2017-05-03 22:46:13,http://pacocha.org/sam.reilly,,7.248.42.64,"{""location"": ""GF"", ""is_mobile"": true}" 9832,4,220,2016-12-20 16:08:24,http://volkman.net/darwin_adams,,171.158.76.147,"{""location"": ""YE"", ""is_mobile"": true}" 9833,4,220,2017-04-30 19:32:42,http://gerlach.name/percy,,237.242.8.87,"{""location"": ""TJ"", ""is_mobile"": true}" 9834,4,220,2017-06-01 21:10:40,http://gerhold.com/dahlia_hermiston,,110.225.61.246,"{""location"": ""AS"", ""is_mobile"": false}" 9835,4,221,2017-04-13 06:43:46,http://farrell.org/shad_stiedemann,0.8544780865,28.32.117.170,"{""location"": ""WS"", ""is_mobile"": false}" 9836,4,221,2017-05-26 17:46:58,http://streich.name/reid,0.9731786518,116.97.8.178,"{""location"": ""BQ"", ""is_mobile"": false}" 9837,4,221,2017-05-30 23:53:03,http://weimanngrimes.com/jillian,0.7283078522,48.89.110.11,"{""location"": ""PK"", ""is_mobile"": true}" 9838,4,221,2017-01-14 01:13:26,http://kunzeveum.org/deie,0.8985458574,87.247.67.242,"{""location"": ""AF"", ""is_mobile"": false}" 9839,4,221,2016-12-27 22:39:04,http://sanfordpowlowski.biz/rafaela.gleason,0.0682227551,71.50.65.10,"{""location"": ""KY"", ""is_mobile"": false}" 9840,4,221,2017-05-19 00:45:41,http://lesch.biz/leone_hilpert,0.9640156136,134.82.91.203,"{""location"": ""BE"", ""is_mobile"": true}" 9841,4,221,2017-01-17 20:15:10,http://hilpertjakubowski.com/karina.boyle,0.4458298816,229.32.247.114,"{""location"": ""AZ"", ""is_mobile"": false}" 6779,3,149,2017-06-10 20:33:17,http://rathbartoletti.net/hershel_lubowitz,,167.210.66.181,"{""location"": ""TW"", ""is_mobile"": false}" 6780,3,149,2017-04-09 20:27:54,http://raynor.net/roger_dickens,,173.209.5.186,"{""location"": ""DJ"", ""is_mobile"": true}" 6781,3,149,2017-01-24 01:04:56,http://upton.info/emory,,218.210.103.222,"{""location"": ""MA"", ""is_mobile"": true}" 6782,3,149,2016-12-30 18:21:42,http://rutherford.com/helena_zulauf,,93.5.156.22,"{""location"": ""MF"", ""is_mobile"": false}" 6783,3,149,2017-04-17 20:41:54,http://haneshanahan.name/lew,,35.75.11.17,"{""location"": ""AM"", ""is_mobile"": false}" 6784,3,149,2017-01-29 14:22:08,http://lind.org/burnice_hintz,,124.241.129.145,"{""location"": ""YE"", ""is_mobile"": true}" 6785,3,149,2017-04-17 19:06:40,http://kshlerin.name/katrina.corkery,,10.192.17.149,"{""location"": ""ES"", ""is_mobile"": false}" 6786,3,149,2017-06-06 06:19:57,http://kleinokon.name/ewell_kertzmann,,200.213.239.170,"{""location"": ""SS"", ""is_mobile"": true}" 6787,3,149,2017-04-05 08:25:00,http://reilly.info/abner,,26.109.83.89,"{""location"": ""CR"", ""is_mobile"": true}" 6788,3,149,2017-01-29 13:04:50,http://hoeger.org/beryl.johnston,,113.6.74.3,"{""location"": ""PT"", ""is_mobile"": false}" 6789,3,149,2017-02-01 20:23:29,http://littleortiz.info/dangelo.torphy,,242.166.82.223,"{""location"": ""NG"", ""is_mobile"": true}" 6790,3,149,2017-01-06 10:02:09,http://schuster.name/herminia_quitzon,,190.41.77.148,"{""location"": ""GB"", ""is_mobile"": false}" 6791,3,149,2017-02-10 12:01:38,http://kubryan.io/melya_crona,,147.21.105.83,"{""location"": ""GY"", ""is_mobile"": true}" 6792,3,149,2017-04-01 00:46:37,http://schadenhane.biz/federico.grimes,,133.24.148.53,"{""location"": ""VC"", ""is_mobile"": false}" 6793,3,149,2017-01-27 14:40:23,http://gerlach.info/tanner,,30.43.179.196,"{""location"": ""AG"", ""is_mobile"": true}" 6794,3,149,2017-05-23 19:37:33,http://johnston.org/audrey,,235.114.253.124,"{""location"": ""AX"", ""is_mobile"": false}" 6795,3,149,2017-04-09 08:18:05,http://kohler.biz/heath,,39.19.70.113,"{""location"": ""ES"", ""is_mobile"": true}" 6796,3,149,2017-01-17 08:17:13,http://trantowjones.net/hiram,,247.87.97.102,"{""location"": ""JM"", ""is_mobile"": false}" 6797,3,149,2017-05-15 15:17:23,http://lueilwitz.org/ashleigh.botsford,,159.26.188.243,"{""location"": ""PR"", ""is_mobile"": false}" 6798,3,149,2017-06-12 19:16:15,http://gutmann.org/khalid.lehner,,222.84.136.244,"{""location"": ""IM"", ""is_mobile"": false}" 6799,3,149,2016-12-29 12:53:47,http://bartolettifahey.biz/chance.stoltenberg,,237.253.192.101,"{""location"": ""RU"", ""is_mobile"": false}" 6800,3,149,2017-02-10 08:57:42,http://ziemannlakin.info/lavina,,2.74.223.192,"{""location"": ""GT"", ""is_mobile"": true}" 6801,3,149,2017-06-07 01:28:10,http://predovic.org/nya,,221.242.98.102,"{""location"": ""TV"", ""is_mobile"": true}" 6802,3,149,2017-04-25 16:02:26,http://nienow.net/lempi,,245.250.96.180,"{""location"": ""JM"", ""is_mobile"": true}" 6803,3,149,2017-04-24 17:48:54,http://friesenwest.org/tabitha.haag,,64.144.38.230,"{""location"": ""WF"", ""is_mobile"": false}" 6804,3,149,2017-01-08 08:41:15,http://jacobi.biz/samantha,,135.250.246.99,"{""location"": ""PM"", ""is_mobile"": false}" 6805,3,149,2017-02-28 10:04:27,http://shanahangrant.org/nathanial.borer,,189.66.65.55,"{""location"": ""MN"", ""is_mobile"": false}" 6806,3,149,2017-05-30 19:14:52,http://renner.co/ali,,91.246.6.246,"{""location"": ""TV"", ""is_mobile"": false}" 6807,3,149,2017-01-10 20:29:59,http://kiehnwalker.org/jaunita_jacobson,,177.94.77.168,"{""location"": ""HR"", ""is_mobile"": true}" 6808,3,149,2017-01-11 22:14:17,http://rau.org/isobel.conroy,,49.34.164.18,"{""location"": ""JP"", ""is_mobile"": false}" 6809,3,150,2017-02-11 10:06:06,http://legros.org/ardith,,76.57.126.14,"{""location"": ""ML"", ""is_mobile"": false}" 6810,3,150,2017-05-08 01:09:33,http://lemkekovacek.biz/zackary.gleichner,,179.223.68.220,"{""location"": ""CK"", ""is_mobile"": true}" 6811,3,150,2017-03-27 17:34:44,http://bosco.name/dean.padberg,,83.32.58.228,"{""location"": ""VA"", ""is_mobile"": true}" 6812,3,150,2017-02-16 16:31:15,http://walkerbruen.org/montana.dooley,,148.242.132.119,"{""location"": ""KM"", ""is_mobile"": false}" 6813,3,150,2017-03-20 02:42:20,http://kiehn.com/luigi,,165.135.17.17,"{""location"": ""JE"", ""is_mobile"": true}" 6814,3,150,2017-04-15 18:35:33,http://rohan.io/angel,,202.196.134.178,"{""location"": ""GW"", ""is_mobile"": true}" 6815,3,150,2017-02-18 02:37:01,http://rennerstanton.name/maymie_boehm,,168.166.177.214,"{""location"": ""GF"", ""is_mobile"": true}" 6816,3,150,2017-06-13 15:30:26,http://dickinson.net/lizeth,,248.75.80.68,"{""location"": ""AX"", ""is_mobile"": false}" 6817,3,150,2017-04-01 15:28:41,http://lynch.org/sarah,,183.245.226.127,"{""location"": ""TV"", ""is_mobile"": true}" 6818,3,150,2017-05-16 01:46:51,http://wuckert.name/rozella,,130.243.234.14,"{""location"": ""AZ"", ""is_mobile"": false}" 6819,3,150,2016-12-28 10:33:38,http://harbermccullough.biz/bailee,,229.99.29.28,"{""location"": ""BN"", ""is_mobile"": true}" 6820,3,150,2017-04-03 10:45:45,http://daniel.name/kane_greenfelder,,117.106.195.135,"{""location"": ""TJ"", ""is_mobile"": false}" 6821,3,150,2017-05-19 09:24:16,http://goyette.biz/maia,,114.231.176.161,"{""location"": ""LR"", ""is_mobile"": true}" 6822,3,150,2017-04-24 09:09:53,http://nitzsche.net/vickie,,43.235.34.28,"{""location"": ""CI"", ""is_mobile"": false}" 6823,3,150,2017-05-17 21:42:57,http://stiedemann.org/henri_tremblay,,163.204.165.183,"{""location"": ""TF"", ""is_mobile"": false}" 6824,3,150,2016-12-24 11:19:52,http://farrell.biz/victoria_hilll,,186.168.204.160,"{""location"": ""WF"", ""is_mobile"": true}" 6825,3,150,2017-02-01 07:43:20,http://batzcrona.net/billie,,86.222.40.88,"{""location"": ""EH"", ""is_mobile"": true}" 6826,3,150,2017-01-16 09:49:49,http://larsonoberbrunner.org/cielo_mraz,,230.188.98.105,"{""location"": ""LB"", ""is_mobile"": false}" 6827,3,150,2016-12-26 18:51:51,http://leannon.co/mylene_heidenreich,,175.77.116.66,"{""location"": ""LK"", ""is_mobile"": true}" 6828,3,150,2017-01-12 22:09:32,http://klockofisher.com/lew.durgan,,15.92.158.38,"{""location"": ""CA"", ""is_mobile"": true}" 6829,3,150,2017-03-06 04:53:19,http://feest.co/theodora_murray,,182.89.210.50,"{""location"": ""ZA"", ""is_mobile"": false}" 6830,3,150,2017-01-07 21:36:49,http://ratkethompson.name/raymond,,170.147.127.38,"{""location"": ""BS"", ""is_mobile"": false}" 6831,3,150,2017-05-01 09:10:52,http://spinka.info/delilah,,195.86.51.203,"{""location"": ""BB"", ""is_mobile"": true}" 6832,3,150,2017-03-27 12:19:29,http://moen.io/candida.barrows,,174.150.106.7,"{""location"": ""CF"", ""is_mobile"": true}" 6833,3,150,2017-05-03 15:44:09,http://herzog.info/caie,,251.237.120.131,"{""location"": ""BR"", ""is_mobile"": true}" 12694,5,283,2017-04-07 06:00:38,http://jerdenikolaus.com/robyn.vonrueden,,78.87.44.139,"{""location"": ""NZ"", ""is_mobile"": false}" 12695,5,283,2017-01-01 01:26:37,http://gaylord.net/rosie.schaefer,,41.48.247.92,"{""location"": ""MW"", ""is_mobile"": true}" 12696,5,283,2017-01-07 12:44:46,http://nitzsche.name/meredith,,45.58.131.230,"{""location"": ""SD"", ""is_mobile"": true}" 12697,5,284,2017-06-03 23:04:04,http://rueckertorphy.biz/arvel,,180.239.187.156,"{""location"": ""LT"", ""is_mobile"": false}" 12698,5,284,2017-03-07 01:10:22,http://wolf.org/herminia_durgan,,79.57.57.101,"{""location"": ""GT"", ""is_mobile"": false}" 12699,5,284,2017-04-24 19:24:06,http://emard.io/fannie,,110.93.210.109,"{""location"": ""SH"", ""is_mobile"": false}" 12700,5,284,2017-01-16 17:45:54,http://stark.name/leslie_pollich,,182.132.2.48,"{""location"": ""GQ"", ""is_mobile"": true}" 12701,5,284,2017-01-10 05:15:40,http://pfannerstill.org/jonatan_effertz,,204.10.215.194,"{""location"": ""MY"", ""is_mobile"": false}" 12702,5,284,2017-04-09 23:52:37,http://hermanbaumbach.name/christelle_borer,,81.189.156.237,"{""location"": ""TV"", ""is_mobile"": true}" 12703,5,284,2017-05-27 10:29:51,http://rogahn.co/ubaldo_king,,65.117.210.236,"{""location"": ""BG"", ""is_mobile"": true}" 12704,5,284,2017-03-25 08:24:01,http://crona.name/madilyn_cremin,,145.248.86.77,"{""location"": ""CN"", ""is_mobile"": false}" 12705,5,284,2016-12-26 08:35:35,http://metz.name/jaleel,,241.25.101.119,"{""location"": ""WF"", ""is_mobile"": false}" 12706,5,284,2017-01-04 21:40:17,http://lakinstoltenberg.io/amir_leuschke,,116.87.43.78,"{""location"": ""SH"", ""is_mobile"": true}" 12707,5,284,2017-05-05 16:04:29,http://buckridge.name/christina.weimann,,209.81.247.43,"{""location"": ""GY"", ""is_mobile"": false}" 12708,5,284,2017-05-07 16:08:35,http://runolfon.io/ania_armstrong,,4.91.230.57,"{""location"": ""SB"", ""is_mobile"": false}" 12709,5,284,2017-05-09 05:20:01,http://hettingerconsidine.info/cornell,,80.214.27.179,"{""location"": ""VN"", ""is_mobile"": true}" 12710,5,284,2017-04-27 14:37:06,http://keeling.io/flo.wintheiser,,62.33.109.158,"{""location"": ""LR"", ""is_mobile"": true}" 12711,5,284,2017-03-22 05:16:17,http://bergstromhegmann.org/evelyn.koch,,252.219.12.217,"{""location"": ""PT"", ""is_mobile"": true}" 12712,5,284,2017-01-15 19:11:31,http://hermanmertz.info/vesta,,178.200.77.234,"{""location"": ""CO"", ""is_mobile"": true}" 12713,5,284,2017-04-04 00:46:19,http://kutch.co/teresa_stehr,,210.195.16.193,"{""location"": ""SG"", ""is_mobile"": false}" 12714,5,284,2017-03-01 23:57:12,http://stark.net/luther,,251.38.54.58,"{""location"": ""VN"", ""is_mobile"": true}" 12715,5,284,2017-04-11 23:03:23,http://barrows.net/garrison,,107.29.25.5,"{""location"": ""PA"", ""is_mobile"": true}" 12716,5,284,2017-05-27 19:19:12,http://ryanwitting.co/gina.quigley,,216.240.118.135,"{""location"": ""CL"", ""is_mobile"": true}" 12717,5,284,2017-03-16 21:49:55,http://durgan.org/tyrique,,53.134.246.2,"{""location"": ""AI"", ""is_mobile"": true}" 12718,5,284,2017-02-07 03:38:32,http://kub.org/nolan_blick,,181.68.115.26,"{""location"": ""AQ"", ""is_mobile"": false}" 12719,5,284,2017-02-23 03:30:21,http://gleichner.biz/kathleen,,39.168.111.15,"{""location"": ""EC"", ""is_mobile"": false}" 12720,5,284,2017-05-27 08:59:10,http://emmerich.net/braxton,,190.65.246.24,"{""location"": ""NP"", ""is_mobile"": false}" 12721,5,284,2017-01-22 12:46:55,http://connelly.co/karl.ebert,,98.48.141.142,"{""location"": ""TC"", ""is_mobile"": true}" 12722,5,284,2016-12-21 14:58:26,http://ledner.name/arne,,196.130.249.163,"{""location"": ""YT"", ""is_mobile"": false}" 12723,5,284,2017-01-25 03:30:21,http://reichel.info/april,,194.115.239.249,"{""location"": ""TD"", ""is_mobile"": false}" 12724,5,284,2016-12-17 07:25:05,http://olson.name/lon_krajcik,,132.14.193.216,"{""location"": ""LU"", ""is_mobile"": true}" 12725,5,284,2017-01-18 05:17:10,http://jacobson.io/baron_abernathy,,59.77.38.54,"{""location"": ""LV"", ""is_mobile"": true}" 12726,5,284,2017-05-05 09:15:12,http://lueilwitz.name/zena,,147.145.179.195,"{""location"": ""ES"", ""is_mobile"": true}" 12727,5,284,2017-03-23 15:38:23,http://ryan.info/kaylin_lubowitz,,114.46.5.45,"{""location"": ""DK"", ""is_mobile"": false}" 12728,5,284,2017-06-07 06:53:15,http://conn.io/alize_wisozk,,227.105.28.41,"{""location"": ""PL"", ""is_mobile"": false}" 12729,5,284,2017-02-09 01:39:01,http://hagenes.org/joesph_toy,,168.164.190.77,"{""location"": ""CZ"", ""is_mobile"": false}" 12730,5,284,2017-05-22 22:14:30,http://frami.info/salvatore.predovic,,75.66.157.8,"{""location"": ""SM"", ""is_mobile"": true}" 12731,5,284,2017-01-30 08:22:05,http://reichel.io/faye.ebert,,185.66.78.50,"{""location"": ""BI"", ""is_mobile"": false}" 12732,5,284,2017-06-04 09:20:57,http://olsonleffler.com/lula,,157.20.108.119,"{""location"": ""ES"", ""is_mobile"": false}" 12733,5,284,2017-05-25 02:44:13,http://grahamgreenholt.com/erick.rice,,227.26.87.145,"{""location"": ""DO"", ""is_mobile"": true}" 12734,5,284,2017-05-29 19:19:21,http://herzogcole.info/mya,,210.204.225.63,"{""location"": ""TC"", ""is_mobile"": false}" 12735,5,284,2017-02-06 02:42:56,http://cormier.info/destany,,16.217.63.45,"{""location"": ""SX"", ""is_mobile"": false}" 12736,5,284,2017-03-12 21:40:36,http://abernathy.co/conor,,254.179.149.164,"{""location"": ""PF"", ""is_mobile"": true}" 12737,5,284,2017-05-13 14:00:41,http://muller.org/jared,,153.178.149.120,"{""location"": ""RS"", ""is_mobile"": false}" 12738,5,284,2017-03-31 02:18:06,http://king.biz/casimir_prosacco,,205.108.222.189,"{""location"": ""AF"", ""is_mobile"": false}" 12739,5,284,2017-05-28 15:04:46,http://lynch.biz/norene,,146.60.252.228,"{""location"": ""MV"", ""is_mobile"": false}" 12740,5,284,2016-12-25 20:42:20,http://mante.io/telly.beahan,,252.100.196.37,"{""location"": ""TH"", ""is_mobile"": true}" 12741,5,284,2017-05-30 03:33:21,http://johnston.info/asha.conroy,,226.223.48.219,"{""location"": ""PK"", ""is_mobile"": false}" 12742,5,284,2017-01-05 18:49:47,http://leannon.org/zelda,,148.121.224.57,"{""location"": ""NI"", ""is_mobile"": false}" 12743,5,284,2017-03-18 15:29:35,http://crooks.org/laney,,188.40.130.234,"{""location"": ""LY"", ""is_mobile"": false}" 12744,5,284,2016-12-21 09:50:43,http://wolff.co/ed_schiller,,233.43.146.190,"{""location"": ""RU"", ""is_mobile"": false}" 12745,5,284,2017-01-30 07:09:28,http://rutherfordkemmer.net/josue_morar,,10.140.184.122,"{""location"": ""NE"", ""is_mobile"": false}" 12746,5,284,2017-04-09 09:09:43,http://funk.name/rollin.willms,,122.231.56.70,"{""location"": ""AO"", ""is_mobile"": true}" 12747,5,284,2017-05-06 06:47:06,http://schowalterjacobson.info/isaiah_kuhlman,,64.69.193.202,"{""location"": ""AU"", ""is_mobile"": false}" 12748,5,284,2017-03-24 06:38:22,http://stroman.biz/lizeth,,178.163.200.122,"{""location"": ""CV"", ""is_mobile"": true}" 15696,6,353,2017-05-22 10:47:22,http://baumbachheel.net/eliza,,127.100.30.220,"{""location"": ""FJ"", ""is_mobile"": true}" 15697,6,353,2017-05-14 03:05:51,http://greenholt.info/charlotte.moriette,,105.184.220.42,"{""location"": ""TO"", ""is_mobile"": false}" 15698,6,353,2017-05-02 06:18:28,http://zboncakrodriguez.biz/jefferey.zulauf,,113.85.182.19,"{""location"": ""VN"", ""is_mobile"": true}" 15699,6,353,2017-01-04 19:24:37,http://feeney.org/janie_krajcik,,117.19.200.153,"{""location"": ""JP"", ""is_mobile"": true}" 15700,6,353,2017-04-02 03:08:33,http://dickens.io/helga,,106.100.10.163,"{""location"": ""IM"", ""is_mobile"": false}" 15701,6,353,2017-05-07 16:30:32,http://donnelly.biz/lila.marvin,,162.222.181.231,"{""location"": ""AR"", ""is_mobile"": true}" 15702,6,353,2017-01-06 07:49:09,http://reichel.biz/broderick,,157.182.38.225,"{""location"": ""TM"", ""is_mobile"": true}" 15703,6,353,2017-04-15 14:02:57,http://bode.name/violette_breitenberg,,23.78.188.86,"{""location"": ""SX"", ""is_mobile"": false}" 15704,6,353,2016-12-19 09:49:10,http://glover.co/nikko.koepp,,49.175.186.58,"{""location"": ""NI"", ""is_mobile"": false}" 15705,6,353,2017-02-20 17:16:30,http://morarcorkery.org/garth_kunde,,176.121.115.254,"{""location"": ""SR"", ""is_mobile"": true}" 15706,6,353,2017-03-04 19:44:17,http://macgyverhodkiewicz.name/daniela_predovic,,47.16.143.51,"{""location"": ""SJ"", ""is_mobile"": true}" 15707,6,353,2017-06-13 07:08:37,http://collins.org/dovie,,129.186.128.250,"{""location"": ""MZ"", ""is_mobile"": true}" 15708,6,353,2017-01-15 12:11:37,http://cruickshankheaney.com/harrison,,162.125.123.52,"{""location"": ""AR"", ""is_mobile"": true}" 15709,6,353,2017-04-13 00:51:31,http://mrazbergnaum.com/karlee,,242.92.58.103,"{""location"": ""RS"", ""is_mobile"": true}" 15710,6,353,2017-02-03 19:28:00,http://croninhyatt.io/stephon.brakus,,215.175.179.222,"{""location"": ""BB"", ""is_mobile"": false}" 15711,6,353,2017-02-09 15:37:29,http://hayes.name/zane_mckenzie,,238.27.43.92,"{""location"": ""KZ"", ""is_mobile"": false}" 15712,6,353,2017-03-20 00:40:04,http://gorczanycole.com/hattie.predovic,,228.44.116.14,"{""location"": ""GI"", ""is_mobile"": true}" 15713,6,353,2017-01-11 06:31:18,http://kihn.org/jaylin.koch,,44.75.221.36,"{""location"": ""MH"", ""is_mobile"": false}" 15714,6,353,2017-03-18 10:34:22,http://lynch.biz/carlos,,47.12.2.107,"{""location"": ""VG"", ""is_mobile"": true}" 15715,6,353,2017-05-21 20:44:40,http://brownyost.org/jaden,,153.28.219.46,"{""location"": ""PT"", ""is_mobile"": false}" 15716,6,354,2017-04-26 03:53:38,http://smithamvolkman.info/darlene,,114.249.14.190,"{""location"": ""BT"", ""is_mobile"": false}" 15717,6,354,2017-05-10 00:43:56,http://hilpert.info/eduardo.huel,,250.246.189.176,"{""location"": ""MZ"", ""is_mobile"": true}" 15718,6,354,2017-01-25 17:36:32,http://deckow.name/gregory_dach,,143.48.10.190,"{""location"": ""VA"", ""is_mobile"": true}" 15719,6,354,2017-01-15 17:54:10,http://beckersimonis.info/kaitlyn.price,,24.238.240.203,"{""location"": ""TD"", ""is_mobile"": true}" 15720,6,354,2017-05-23 05:29:45,http://murphygraham.io/christ,,34.162.31.235,"{""location"": ""EC"", ""is_mobile"": false}" 15721,6,354,2017-04-10 10:59:31,http://moenpfannerstill.co/estella,,139.154.41.40,"{""location"": ""VU"", ""is_mobile"": false}" 15722,6,354,2017-03-18 07:10:52,http://zemlak.name/mekhi_jacobson,,141.96.210.243,"{""location"": ""UZ"", ""is_mobile"": true}" 15723,6,354,2016-12-28 13:10:41,http://parker.name/kolby.bartoletti,,91.49.32.195,"{""location"": ""KP"", ""is_mobile"": false}" 15724,6,354,2017-04-29 04:36:11,http://ferry.biz/kareem,,192.48.31.130,"{""location"": ""HN"", ""is_mobile"": false}" 15725,6,354,2017-01-22 14:40:20,http://cronincronin.biz/carlo,,243.170.77.201,"{""location"": ""ML"", ""is_mobile"": false}" 15726,6,354,2017-03-18 17:02:12,http://grant.com/kristy_zulauf,,7.74.228.158,"{""location"": ""IR"", ""is_mobile"": false}" 15727,6,354,2017-02-08 10:54:55,http://hickle.biz/vivien_moore,,120.106.219.207,"{""location"": ""PN"", ""is_mobile"": false}" 15728,6,354,2017-03-25 22:18:08,http://halvorson.biz/rocky,,135.149.214.156,"{""location"": ""BQ"", ""is_mobile"": false}" 15729,6,354,2017-02-28 23:27:14,http://boyle.co/dianna,,229.184.45.244,"{""location"": ""GS"", ""is_mobile"": true}" 15730,6,354,2017-04-15 01:32:19,http://funk.com/veda,,115.237.185.209,"{""location"": ""GW"", ""is_mobile"": true}" 15731,6,354,2017-01-23 21:29:21,http://conn.co/blaze_rowe,,124.230.186.48,"{""location"": ""MD"", ""is_mobile"": false}" 15732,6,354,2017-01-01 08:10:15,http://rohan.info/emilie,,215.203.46.89,"{""location"": ""PM"", ""is_mobile"": true}" 15733,6,354,2017-03-02 17:49:48,http://quigleymcdermott.io/jamil.rath,,166.149.126.24,"{""location"": ""IS"", ""is_mobile"": true}" 15734,6,354,2017-03-16 22:10:07,http://greenholtkemmer.name/sedrick,,25.133.174.149,"{""location"": ""BQ"", ""is_mobile"": true}" 15735,6,354,2017-06-08 19:48:34,http://denesiknolan.net/sammie_mueller,,13.93.40.40,"{""location"": ""BJ"", ""is_mobile"": false}" 15736,6,354,2017-06-05 05:37:32,http://lefflerbrakus.io/donny_stiedemann,,78.192.24.69,"{""location"": ""PF"", ""is_mobile"": false}" 15737,6,354,2017-03-19 17:52:30,http://jenkins.biz/jaquan.hauck,,111.2.165.153,"{""location"": ""MS"", ""is_mobile"": false}" 15738,6,354,2017-06-06 06:55:26,http://bergnaum.io/diamond,,228.216.182.122,"{""location"": ""SI"", ""is_mobile"": true}" 15739,6,354,2016-12-19 06:09:57,http://legros.co/josiane_runte,,183.22.101.159,"{""location"": ""GB"", ""is_mobile"": true}" 15740,6,354,2017-03-20 03:00:41,http://williamson.com/wilburn,,52.187.18.176,"{""location"": ""KP"", ""is_mobile"": true}" 15741,6,354,2017-01-12 06:44:05,http://okuneva.org/manuela,,100.145.243.123,"{""location"": ""TR"", ""is_mobile"": false}" 15742,6,354,2017-05-09 17:41:47,http://stracke.info/mateo,,239.106.231.25,"{""location"": ""MW"", ""is_mobile"": true}" 15743,6,354,2017-04-01 16:13:12,http://heathcotekeeling.io/felipe,,83.151.102.119,"{""location"": ""TC"", ""is_mobile"": true}" 15744,6,354,2017-03-19 11:21:03,http://kiehn.co/sandra_kihn,,127.83.249.103,"{""location"": ""CU"", ""is_mobile"": true}" 15745,6,354,2017-05-23 10:44:45,http://homenick.co/heaven_ko,,149.92.15.183,"{""location"": ""TM"", ""is_mobile"": false}" 15746,6,354,2017-02-01 13:19:02,http://bernier.org/mazie.kozey,,235.221.31.64,"{""location"": ""NR"", ""is_mobile"": true}" 15747,6,354,2017-03-20 03:36:02,http://mcclure.net/herminia_bergnaum,,44.41.188.198,"{""location"": ""TN"", ""is_mobile"": false}" 15748,6,354,2017-01-04 18:56:23,http://blanda.name/fermin,,82.210.228.141,"{""location"": ""MM"", ""is_mobile"": true}" 15749,6,354,2016-12-13 19:15:53,http://kulas.name/lavern_turner,,72.131.7.147,"{""location"": ""LK"", ""is_mobile"": true}" 15750,6,354,2017-01-02 07:48:37,http://kelerankunding.com/candace_tremblay,,49.199.186.207,"{""location"": ""EG"", ""is_mobile"": false}" 2906,2,64,2017-05-27 12:49:43,http://denesik.net/stacey.aufderhar,0.1648262502,165.162.213.186,"{""location"": ""GB"", ""is_mobile"": true}" 2907,2,64,2017-01-18 07:55:05,http://strackejohns.co/emile,0.3817178047,153.217.88.142,"{""location"": ""YE"", ""is_mobile"": false}" 2908,2,64,2017-05-02 00:59:34,http://king.org/josiane,0.6800251921,183.201.252.57,"{""location"": ""MX"", ""is_mobile"": true}" 2909,2,64,2017-04-01 16:09:29,http://witting.net/deshawn_doyle,0.1010081191,92.58.199.27,"{""location"": ""BQ"", ""is_mobile"": false}" 2910,2,64,2017-06-02 05:16:41,http://mohr.net/casimir.fahey,0.9212890942,47.207.70.235,"{""location"": ""NP"", ""is_mobile"": true}" 2911,2,64,2017-06-07 00:57:06,http://crooktoltenberg.co/bradford,0.6480223567,56.80.219.35,"{""location"": ""BQ"", ""is_mobile"": false}" 2912,2,64,2017-04-01 19:47:45,http://wehnerpollich.io/adrain,0.5353433482,53.117.205.71,"{""location"": ""SO"", ""is_mobile"": true}" 2913,2,64,2017-01-05 17:29:35,http://heel.com/greta,0.4274482932,250.114.7.204,"{""location"": ""FI"", ""is_mobile"": true}" 2914,2,64,2017-02-24 21:51:58,http://reichert.io/ronny.wiza,0.8570592901,186.118.57.169,"{""location"": ""TG"", ""is_mobile"": false}" 2915,2,64,2017-03-31 01:18:24,http://leannonstanton.co/angelica,0.0976372904,150.69.7.114,"{""location"": ""CK"", ""is_mobile"": true}" 2916,2,64,2017-01-04 23:17:57,http://schowalter.com/lenora_wyman,0.3583426950,35.3.69.120,"{""location"": ""GR"", ""is_mobile"": false}" 2917,2,64,2017-04-27 20:06:22,http://farrell.biz/rupert,0.6116453719,58.221.209.218,"{""location"": ""TL"", ""is_mobile"": false}" 2918,2,64,2017-04-13 01:25:43,http://halvorsonchamplin.co/kali,0.6340286841,16.182.225.78,"{""location"": ""CD"", ""is_mobile"": false}" 2919,2,64,2017-03-21 15:48:35,http://breitenberg.com/porter,0.1676519262,99.121.35.16,"{""location"": ""UZ"", ""is_mobile"": false}" 2920,2,64,2017-01-11 18:58:10,http://walter.io/reed.vandervort,0.5001945653,221.178.231.120,"{""location"": ""DK"", ""is_mobile"": true}" 2921,2,64,2017-05-23 19:39:49,http://douglas.co/precious,0.5072863712,69.133.147.85,"{""location"": ""KN"", ""is_mobile"": true}" 2922,2,64,2017-03-24 16:09:12,http://glover.com/waino_kutch,0.8574326949,116.147.221.157,"{""location"": ""AS"", ""is_mobile"": false}" 2923,2,65,2017-02-19 14:34:43,http://jenkins.co/florence.moriette,0.8109989174,188.56.226.240,"{""location"": ""OM"", ""is_mobile"": true}" 2924,2,65,2017-01-09 18:21:24,http://rathhagenes.info/aurore,0.5023311389,31.7.89.67,"{""location"": ""PF"", ""is_mobile"": false}" 2925,2,65,2017-02-17 10:42:51,http://kovacek.net/ezekiel,0.7814605669,162.19.98.187,"{""location"": ""MH"", ""is_mobile"": true}" 2926,2,65,2017-03-01 18:25:59,http://murphy.org/roderick,0.6242767281,202.26.141.123,"{""location"": ""EH"", ""is_mobile"": true}" 2927,2,65,2016-12-19 02:44:46,http://hettinger.co/arturo,0.8853867871,24.23.11.130,"{""location"": ""GP"", ""is_mobile"": false}" 2928,2,65,2016-12-16 09:35:47,http://kerluke.net/jacynthe.oberbrunner,0.3334095711,241.219.223.146,"{""location"": ""VI"", ""is_mobile"": true}" 2929,2,65,2017-04-13 18:36:46,http://stamm.co/anabel,0.3163032951,246.149.177.204,"{""location"": ""TR"", ""is_mobile"": true}" 2930,2,65,2017-01-06 19:28:58,http://hills.biz/braulio.hintz,0.6874250717,8.61.147.252,"{""location"": ""TC"", ""is_mobile"": true}" 2931,2,65,2016-12-24 06:39:37,http://schiller.io/kaden.zboncak,0.4237815431,184.214.125.122,"{""location"": ""SV"", ""is_mobile"": true}" 2932,2,65,2017-01-14 15:06:46,http://morarhowell.net/isaias,0.5509876293,135.243.235.239,"{""location"": ""NP"", ""is_mobile"": false}" 2933,2,65,2017-01-23 04:02:44,http://zulaufschmeler.org/korbin_smitham,0.5605076506,227.196.246.229,"{""location"": ""ML"", ""is_mobile"": true}" 2934,2,65,2017-03-05 12:01:19,http://windler.org/cortney_kemmer,0.9480535579,159.170.15.131,"{""location"": ""HK"", ""is_mobile"": true}" 2935,2,65,2017-01-07 00:10:36,http://spencerkemmer.name/dakota.swaniawski,0.4326836503,125.112.231.143,"{""location"": ""FO"", ""is_mobile"": false}" 2936,2,65,2017-01-09 20:53:06,http://kulas.org/harmon.weinat,0.2532339087,54.3.235.205,"{""location"": ""SK"", ""is_mobile"": false}" 2937,2,65,2017-04-01 08:00:28,http://beahan.com/christophe,0.9496827760,120.252.253.231,"{""location"": ""TZ"", ""is_mobile"": true}" 2938,2,65,2017-01-04 21:02:23,http://krajcik.biz/rebeka_donnelly,0.9560487469,253.66.186.198,"{""location"": ""KW"", ""is_mobile"": true}" 2939,2,65,2017-05-12 02:40:02,http://bode.co/camille,0.0153785099,113.36.100.73,"{""location"": ""SJ"", ""is_mobile"": true}" 2940,2,65,2017-01-21 09:27:48,http://jacobson.biz/celia_schultz,0.9784070527,247.243.15.189,"{""location"": ""LI"", ""is_mobile"": false}" 2941,2,65,2017-04-26 10:35:21,http://quitzon.com/quincy.schumm,0.1667851298,100.147.136.125,"{""location"": ""GM"", ""is_mobile"": true}" 2942,2,65,2017-05-23 08:20:29,http://gaylord.info/caandre,0.7223754919,63.226.170.93,"{""location"": ""MR"", ""is_mobile"": true}" 2943,2,65,2017-01-31 20:09:58,http://hudson.co/reina_ernser,0.5436318485,191.147.59.61,"{""location"": ""ET"", ""is_mobile"": true}" 2944,2,65,2017-05-17 15:46:36,http://powlowskicain.biz/velda_zemlak,0.6176269495,164.238.63.36,"{""location"": ""JM"", ""is_mobile"": false}" 2945,2,65,2017-02-02 15:18:22,http://lind.name/stanley_ondricka,0.7626547180,147.241.227.130,"{""location"": ""AD"", ""is_mobile"": true}" 2946,2,65,2017-03-08 12:32:21,http://littlenikolaus.net/haskell.bartell,0.4908110678,119.180.104.123,"{""location"": ""CN"", ""is_mobile"": true}" 2947,2,65,2017-03-12 21:21:25,http://kirlinbarrows.info/gerda,0.8842974958,11.224.199.237,"{""location"": ""LI"", ""is_mobile"": false}" 2948,2,65,2017-04-26 19:21:44,http://sipes.com/gerardo_daugherty,0.3192392049,69.116.65.45,"{""location"": ""WF"", ""is_mobile"": true}" 2949,2,65,2017-03-07 20:21:44,http://eichmann.net/nelda_heller,0.4568917456,5.73.8.68,"{""location"": ""GP"", ""is_mobile"": true}" 2950,2,65,2017-02-22 07:48:00,http://romaguera.com/seamus_zulauf,0.2964098804,184.101.74.37,"{""location"": ""BS"", ""is_mobile"": true}" 2951,2,65,2017-06-13 18:40:49,http://gaylordharvey.io/duane_larkin,0.9359039759,227.184.101.246,"{""location"": ""RU"", ""is_mobile"": false}" 2952,2,65,2017-02-17 20:52:17,http://wilderman.info/sam,0.7266130418,66.252.143.149,"{""location"": ""FK"", ""is_mobile"": false}" 2953,2,65,2017-04-12 17:29:10,http://walshsmitham.net/braeden,0.5946515427,31.98.148.11,"{""location"": ""UY"", ""is_mobile"": true}" 2954,2,65,2016-12-21 16:29:01,http://mayert.com/mathias,0.4921715151,210.202.17.77,"{""location"": ""SD"", ""is_mobile"": false}" 2955,2,65,2016-12-30 07:25:36,http://westcummings.net/susanna,0.9634033114,247.24.25.108,"{""location"": ""SC"", ""is_mobile"": false}" 2956,2,65,2017-03-09 12:18:13,http://pollich.info/malvina,0.5346158652,205.234.129.115,"{""location"": ""IN"", ""is_mobile"": true}" 9842,4,221,2017-02-07 14:17:14,http://ricekaulke.biz/kolby_macgyver,0.3742259453,247.44.138.18,"{""location"": ""JP"", ""is_mobile"": true}" 9843,4,221,2017-02-06 12:09:25,http://walker.biz/allie.fadel,0.0099313869,87.213.67.151,"{""location"": ""KH"", ""is_mobile"": true}" 9844,4,221,2017-04-02 13:58:15,http://walter.biz/ivory_hamill,0.8696352059,19.232.160.215,"{""location"": ""TR"", ""is_mobile"": true}" 9845,4,221,2017-03-17 07:10:39,http://reinger.net/astrid,0.6127495328,228.127.160.254,"{""location"": ""JM"", ""is_mobile"": false}" 9846,4,221,2017-05-02 08:11:02,http://willms.org/armani_dare,0.0552043017,65.217.165.79,"{""location"": ""GN"", ""is_mobile"": false}" 9847,4,221,2017-05-15 04:35:25,http://oreilly.io/brain,0.1222123055,11.36.131.26,"{""location"": ""JO"", ""is_mobile"": true}" 9848,4,221,2017-03-05 21:52:12,http://buckridge.io/susanna,0.2736269425,107.15.181.2,"{""location"": ""BH"", ""is_mobile"": true}" 9849,4,221,2017-04-25 21:13:17,http://walsh.co/hayden.kshlerin,0.7117761415,166.101.113.238,"{""location"": ""LA"", ""is_mobile"": true}" 9850,4,221,2017-05-08 06:15:18,http://keler.biz/lisa_ullrich,0.2012082672,83.251.41.160,"{""location"": ""FJ"", ""is_mobile"": false}" 9851,4,221,2017-02-03 05:42:38,http://jerde.com/adrienne,0.3289415038,111.177.236.201,"{""location"": ""MR"", ""is_mobile"": true}" 9852,4,221,2017-02-17 08:44:41,http://greenfelder.name/olaf.hagenes,0.8169530901,16.176.13.54,"{""location"": ""VN"", ""is_mobile"": true}" 9853,4,221,2017-05-17 12:01:29,http://quitzon.org/jordi,0.3274574223,20.186.155.67,"{""location"": ""JE"", ""is_mobile"": false}" 9854,4,221,2017-03-18 19:41:20,http://wyman.name/triston,0.6037084992,141.62.129.115,"{""location"": ""NF"", ""is_mobile"": true}" 9855,4,221,2017-06-08 19:14:54,http://daughertyschneider.info/milford.schinner,0.7066142558,189.242.60.141,"{""location"": ""NO"", ""is_mobile"": false}" 9856,4,221,2016-12-21 22:49:29,http://gibson.name/rosalia.keler,0.2682220915,240.169.18.162,"{""location"": ""MR"", ""is_mobile"": true}" 9857,4,221,2017-06-02 21:47:58,http://homenick.org/muriel,0.7829910711,116.196.75.217,"{""location"": ""IL"", ""is_mobile"": true}" 9858,4,221,2017-02-15 07:23:02,http://torpkoch.name/jeanette,0.3655956294,16.60.185.210,"{""location"": ""VC"", ""is_mobile"": true}" 9859,4,221,2017-02-18 06:29:29,http://auer.co/wyman.tillman,0.1419698842,60.225.141.186,"{""location"": ""MQ"", ""is_mobile"": false}" 9860,4,221,2017-04-19 11:49:09,http://gusikowskihermann.info/darius_lemke,0.6759285914,217.181.134.102,"{""location"": ""SE"", ""is_mobile"": false}" 9861,4,221,2017-01-28 03:26:28,http://sanforddietrich.com/anjali_botsford,0.3710781372,244.25.34.97,"{""location"": ""BV"", ""is_mobile"": false}" 9862,4,221,2017-02-04 05:35:47,http://beiergleason.co/santiago.schowalter,0.2602600645,99.146.18.244,"{""location"": ""MR"", ""is_mobile"": true}" 9863,4,221,2017-03-26 02:53:35,http://blick.org/jerrell,0.5067720743,55.190.136.253,"{""location"": ""KG"", ""is_mobile"": false}" 9864,4,221,2017-05-17 20:59:47,http://predovic.info/ryder,0.9431469738,120.168.53.249,"{""location"": ""UG"", ""is_mobile"": false}" 9865,4,221,2017-05-23 14:09:55,http://brown.co/damon,0.4317209422,224.39.206.167,"{""location"": ""IM"", ""is_mobile"": true}" 9866,4,221,2016-12-23 17:07:29,http://volkman.org/clifton.ebert,0.1331418719,177.219.95.69,"{""location"": ""BI"", ""is_mobile"": false}" 9867,4,221,2017-01-19 10:20:30,http://bechtelar.info/ebony.stoltenberg,0.7268938428,88.221.106.40,"{""location"": ""SC"", ""is_mobile"": false}" 9868,4,221,2017-03-09 17:41:46,http://croninschmeler.co/felicia,0.9097963626,110.150.38.31,"{""location"": ""VA"", ""is_mobile"": true}" 9869,4,221,2017-04-08 19:44:20,http://schoen.info/michel_mclaughlin,0.6658248494,52.87.90.107,"{""location"": ""LV"", ""is_mobile"": true}" 9870,4,221,2017-02-26 12:24:37,http://hodkiewicz.org/nannie.kovacek,0.4858070880,84.248.117.24,"{""location"": ""TK"", ""is_mobile"": true}" 9871,4,221,2017-01-09 01:32:07,http://stanton.biz/adolfo.gaylord,0.2497390271,89.26.206.100,"{""location"": ""JO"", ""is_mobile"": true}" 9872,4,221,2017-05-11 20:28:33,http://romaguera.biz/brandt,0.0138754435,31.14.106.187,"{""location"": ""TC"", ""is_mobile"": false}" 9873,4,221,2017-01-09 21:35:48,http://ernser.co/pascale,0.7032806041,147.174.140.234,"{""location"": ""MO"", ""is_mobile"": false}" 9874,4,221,2017-05-05 15:25:05,http://vonruedenhintz.io/caidy.sporer,0.4515276316,183.157.137.231,"{""location"": ""IE"", ""is_mobile"": true}" 9875,4,221,2017-04-16 16:35:40,http://schoen.io/ryder.prohaska,0.9535643159,141.210.9.183,"{""location"": ""NO"", ""is_mobile"": false}" 9876,4,221,2017-04-14 20:57:00,http://dickinson.com/agnes_wiza,0.5705275428,203.217.175.38,"{""location"": ""GB"", ""is_mobile"": false}" 9877,4,221,2017-04-16 16:16:59,http://gusikowskijast.io/albina.renner,0.5441168734,247.58.85.31,"{""location"": ""BZ"", ""is_mobile"": false}" 9878,4,221,2017-04-22 03:01:22,http://moen.name/judah_davis,0.7314554402,6.135.178.12,"{""location"": ""AE"", ""is_mobile"": true}" 9879,4,221,2017-04-04 13:50:19,http://lefflerkling.name/maya,0.6689526769,194.122.31.144,"{""location"": ""SZ"", ""is_mobile"": false}" 9880,4,221,2017-03-23 04:08:03,http://bins.name/norval,0.5403204964,93.34.77.183,"{""location"": ""AQ"", ""is_mobile"": true}" 9881,4,221,2017-02-13 14:13:39,http://kling.co/abagail.farrell,0.6046073060,120.197.180.67,"{""location"": ""SJ"", ""is_mobile"": true}" 9882,4,221,2017-02-02 04:53:43,http://kilbackmueller.biz/kenyon_borer,0.7485072967,40.95.215.150,"{""location"": ""ZM"", ""is_mobile"": true}" 9883,4,221,2017-04-26 01:56:39,http://rodriguezsipes.org/adele,0.1634595831,85.170.45.197,"{""location"": ""DM"", ""is_mobile"": false}" 9884,4,221,2017-03-17 07:39:54,http://kuvalis.com/margie,0.9623555284,29.2.162.201,"{""location"": ""MZ"", ""is_mobile"": false}" 9885,4,221,2016-12-14 18:04:00,http://larkin.name/hester_stamm,0.4952940447,50.24.78.75,"{""location"": ""GT"", ""is_mobile"": false}" 9886,4,221,2017-01-11 09:52:22,http://kovacek.biz/nicholaus.hills,0.7957641336,193.68.62.252,"{""location"": ""KE"", ""is_mobile"": false}" 9887,4,221,2017-02-22 08:39:25,http://zboncak.info/gertrude_rolfson,0.1720415080,74.43.148.141,"{""location"": ""NR"", ""is_mobile"": false}" 9888,4,221,2017-05-09 19:25:12,http://bauch.co/liliana,0.9288063303,46.207.39.66,"{""location"": ""GB"", ""is_mobile"": false}" 9889,4,221,2017-03-26 22:59:36,http://torp.name/bridie,0.5942013457,113.246.130.170,"{""location"": ""BN"", ""is_mobile"": true}" 9890,4,221,2017-01-12 08:53:11,http://gerlach.biz/sasha,0.3079783255,78.99.80.212,"{""location"": ""HU"", ""is_mobile"": false}" 9891,4,221,2017-05-31 13:32:33,http://nikolausokuneva.name/aditya,0.0728848938,220.222.106.70,"{""location"": ""AI"", ""is_mobile"": false}" 9892,4,221,2017-02-09 19:25:02,http://flatley.org/laurence.oconnell,0.7800609424,235.102.60.161,"{""location"": ""BW"", ""is_mobile"": false}" 6834,3,150,2017-01-03 00:14:17,http://watsica.org/ernestine,,52.48.194.115,"{""location"": ""CW"", ""is_mobile"": true}" 6835,3,150,2016-12-25 01:23:01,http://hickle.biz/chester,,125.180.51.200,"{""location"": ""BS"", ""is_mobile"": false}" 6836,3,150,2017-02-15 15:42:33,http://littel.com/jason,,213.18.90.77,"{""location"": ""ST"", ""is_mobile"": true}" 6837,3,150,2017-05-13 17:00:59,http://schamberger.com/mohammad,,200.77.18.194,"{""location"": ""KZ"", ""is_mobile"": true}" 6838,3,150,2017-06-11 17:41:58,http://prohaska.org/tyshawn_effertz,,222.32.67.65,"{""location"": ""HT"", ""is_mobile"": true}" 6839,3,150,2017-04-28 12:07:16,http://senger.biz/jeie,,138.152.246.78,"{""location"": ""KM"", ""is_mobile"": false}" 6840,3,150,2017-05-25 18:32:43,http://krajcik.name/toy_reinger,,227.228.122.187,"{""location"": ""SK"", ""is_mobile"": true}" 6841,3,150,2017-04-02 21:23:23,http://brekkefeil.com/eugenia.greenfelder,,250.155.8.46,"{""location"": ""GE"", ""is_mobile"": true}" 6842,3,150,2017-05-08 02:45:10,http://gaylord.info/kamron,,26.209.130.247,"{""location"": ""PT"", ""is_mobile"": true}" 6843,3,150,2017-02-22 00:10:41,http://dietrich.com/kasey,,166.252.36.44,"{""location"": ""GN"", ""is_mobile"": false}" 6844,3,150,2017-01-10 00:59:24,http://mayertmohr.biz/isidro,,215.214.146.49,"{""location"": ""NU"", ""is_mobile"": false}" 6845,3,150,2017-03-01 14:33:31,http://greenfelderkulas.biz/brennan,,231.221.136.217,"{""location"": ""QA"", ""is_mobile"": false}" 6846,3,151,2017-05-30 15:47:26,http://kerluke.info/yoshiko,,132.46.129.145,"{""location"": ""JE"", ""is_mobile"": true}" 6847,3,151,2017-04-01 12:36:07,http://swift.org/leann,,177.71.237.133,"{""location"": ""AF"", ""is_mobile"": false}" 6848,3,151,2017-04-04 05:58:29,http://okeefe.info/leon,,116.30.114.127,"{""location"": ""SY"", ""is_mobile"": true}" 6849,3,151,2017-03-07 14:25:22,http://rowe.name/willa,,34.162.136.144,"{""location"": ""LA"", ""is_mobile"": false}" 6850,3,151,2017-01-21 02:40:03,http://schowalter.co/lea.goodwin,,182.239.102.207,"{""location"": ""PY"", ""is_mobile"": true}" 6851,3,151,2017-02-06 21:32:52,http://moriette.biz/ettie_weber,,22.115.73.77,"{""location"": ""TZ"", ""is_mobile"": true}" 6852,3,151,2017-02-15 20:49:52,http://stehrhayes.info/jazmyn,,209.178.246.219,"{""location"": ""NU"", ""is_mobile"": true}" 6853,3,151,2017-01-23 21:57:21,http://waters.net/cristal_kovacek,,142.5.190.203,"{""location"": ""MD"", ""is_mobile"": false}" 6854,3,151,2016-12-24 14:54:36,http://lakinkovacek.info/donnell,,25.144.202.211,"{""location"": ""AI"", ""is_mobile"": true}" 6855,3,151,2017-01-12 10:10:38,http://bednar.co/melisa_gerlach,,248.48.157.68,"{""location"": ""ID"", ""is_mobile"": true}" 6856,3,151,2017-03-12 02:54:28,http://schuppe.net/isaias,,86.250.147.55,"{""location"": ""BN"", ""is_mobile"": false}" 6857,3,151,2017-04-17 08:44:44,http://braun.com/mateo_mosciski,,9.95.33.187,"{""location"": ""GI"", ""is_mobile"": false}" 6858,3,151,2016-12-25 13:44:44,http://torpgraham.info/allison.treutel,,189.120.93.243,"{""location"": ""UG"", ""is_mobile"": false}" 6859,3,151,2017-04-11 08:13:36,http://larkin.org/rylan,,219.217.113.27,"{""location"": ""HT"", ""is_mobile"": false}" 6860,3,151,2017-05-13 14:45:00,http://hirthekoch.org/francesca_goyette,,228.137.19.75,"{""location"": ""LY"", ""is_mobile"": false}" 6861,3,151,2017-05-25 15:55:31,http://ferry.net/kyla,,120.31.213.91,"{""location"": ""MR"", ""is_mobile"": true}" 6862,3,151,2016-12-21 04:52:08,http://larson.name/fatima,,39.11.220.2,"{""location"": ""BG"", ""is_mobile"": false}" 6863,3,151,2017-03-08 14:39:12,http://jacobi.net/shirley_dicki,,82.155.184.149,"{""location"": ""CL"", ""is_mobile"": false}" 6864,3,151,2017-04-05 21:48:43,http://jakubowskideckow.name/ronaldo.hermiston,,254.106.164.79,"{""location"": ""LA"", ""is_mobile"": true}" 6865,3,151,2017-04-27 03:20:28,http://hagenesmorar.com/vincent_hayes,,183.111.130.148,"{""location"": ""SG"", ""is_mobile"": false}" 6866,3,151,2017-01-24 15:43:00,http://vonrueden.co/tanner.barrows,,228.11.33.253,"{""location"": ""CI"", ""is_mobile"": true}" 6867,3,151,2017-02-10 20:23:40,http://morar.io/demetris_dickinson,,237.176.148.136,"{""location"": ""BZ"", ""is_mobile"": false}" 6868,3,151,2017-04-16 18:10:13,http://stark.biz/ethyl_wintheiser,,236.14.50.200,"{""location"": ""PL"", ""is_mobile"": true}" 6869,3,151,2017-02-16 01:04:04,http://schamberger.name/dayna,,156.37.254.110,"{""location"": ""BO"", ""is_mobile"": true}" 6870,3,151,2017-03-05 11:08:36,http://howe.name/cecile_dare,,5.72.233.21,"{""location"": ""AZ"", ""is_mobile"": false}" 6871,3,151,2017-04-24 18:34:12,http://kozey.io/savanah,,64.216.61.247,"{""location"": ""BA"", ""is_mobile"": false}" 6872,3,151,2017-03-15 02:19:11,http://jakubowskiokeefe.info/krystina_blick,,175.97.90.246,"{""location"": ""AX"", ""is_mobile"": false}" 6873,3,151,2017-04-08 11:01:49,http://gibson.com/catherine_mclaughlin,,139.215.121.63,"{""location"": ""EC"", ""is_mobile"": true}" 6874,3,151,2016-12-23 05:04:57,http://walterrenner.co/micheal,,23.221.86.17,"{""location"": ""BJ"", ""is_mobile"": false}" 6875,3,151,2017-03-26 18:35:26,http://ward.co/sam,,111.101.40.92,"{""location"": ""TW"", ""is_mobile"": true}" 6876,3,151,2017-05-06 05:05:50,http://johnston.name/emerson,,10.100.21.225,"{""location"": ""AU"", ""is_mobile"": true}" 6877,3,151,2017-06-02 10:02:25,http://lebsackmckenzie.io/dariana,,210.180.29.174,"{""location"": ""SR"", ""is_mobile"": false}" 6878,3,151,2017-02-04 07:13:07,http://stiedemannsatterfield.org/wilfred,,212.157.53.80,"{""location"": ""BW"", ""is_mobile"": true}" 6879,3,151,2017-03-04 10:01:02,http://tromp.com/raphael_mcglynn,,167.195.191.167,"{""location"": ""SE"", ""is_mobile"": true}" 6880,3,151,2017-05-29 09:22:10,http://feeney.name/virgil.dooley,,40.98.75.211,"{""location"": ""LC"", ""is_mobile"": true}" 6881,3,151,2016-12-28 14:17:17,http://ohara.com/eulah.ward,,221.36.219.129,"{""location"": ""AR"", ""is_mobile"": false}" 6882,3,151,2017-01-17 13:44:41,http://heller.name/hermina.kuhic,,72.163.211.194,"{""location"": ""VE"", ""is_mobile"": true}" 6883,3,151,2017-04-13 20:04:57,http://schiller.io/katelin,,148.254.14.220,"{""location"": ""CV"", ""is_mobile"": true}" 6884,3,151,2017-02-04 19:39:51,http://ernser.com/aiyana_yundt,,220.72.218.194,"{""location"": ""NE"", ""is_mobile"": false}" 6885,3,151,2017-01-20 01:20:04,http://kutch.name/bryce.becker,,59.3.121.185,"{""location"": ""SO"", ""is_mobile"": true}" 6886,3,151,2017-02-08 14:45:28,http://brekkeschiller.net/bennett,,108.100.237.46,"{""location"": ""NI"", ""is_mobile"": true}" 6887,3,151,2017-03-12 08:55:00,http://marvin.net/sydnee_schuster,,149.226.212.13,"{""location"": ""MH"", ""is_mobile"": true}" 6888,3,151,2017-01-29 06:26:00,http://durganhowe.co/keenan,,230.131.158.155,"{""location"": ""GH"", ""is_mobile"": true}" 6889,3,151,2017-05-10 12:28:01,http://goyette.biz/josue,,68.68.189.118,"{""location"": ""GP"", ""is_mobile"": true}" 12749,5,285,2017-03-15 05:23:42,http://upton.biz/montana,,230.250.33.186,"{""location"": ""FK"", ""is_mobile"": false}" 12750,5,285,2017-02-16 00:28:51,http://schoen.io/savanna,,101.97.208.80,"{""location"": ""AD"", ""is_mobile"": true}" 12751,5,285,2017-05-13 06:06:28,http://berge.info/jamil,,150.101.194.117,"{""location"": ""NG"", ""is_mobile"": false}" 12752,5,285,2017-04-12 14:08:24,http://kemmer.co/lillie,,129.31.92.208,"{""location"": ""NF"", ""is_mobile"": false}" 12753,5,285,2017-01-12 03:54:57,http://auer.net/damian_abernathy,,195.230.37.176,"{""location"": ""SE"", ""is_mobile"": true}" 12754,5,285,2017-03-11 18:06:54,http://schiller.info/hobart,,52.148.159.65,"{""location"": ""NG"", ""is_mobile"": false}" 12755,5,285,2017-03-01 19:39:13,http://emmerichlesch.info/fabian.nikolaus,,12.178.198.24,"{""location"": ""KY"", ""is_mobile"": false}" 12756,5,285,2017-02-09 08:30:33,http://kshlerinbecker.name/everardo,,149.227.36.107,"{""location"": ""US"", ""is_mobile"": false}" 12757,5,285,2016-12-17 17:17:49,http://macejkovic.biz/gabrielle,,80.186.156.140,"{""location"": ""NF"", ""is_mobile"": false}" 12758,5,285,2017-06-11 02:04:43,http://walshullrich.org/pansy.crona,,194.27.2.106,"{""location"": ""CA"", ""is_mobile"": true}" 12759,5,285,2017-05-12 00:06:38,http://gerlachschmidt.info/retha_luettgen,,211.44.230.230,"{""location"": ""TW"", ""is_mobile"": true}" 12760,5,285,2017-05-18 13:33:11,http://larkin.com/vella_orn,,126.186.8.94,"{""location"": ""DM"", ""is_mobile"": false}" 12761,5,285,2017-05-19 21:38:12,http://keeling.io/nikki.champlin,,148.156.111.158,"{""location"": ""FI"", ""is_mobile"": true}" 12762,5,285,2017-01-02 05:19:56,http://ondrickaziemann.co/raina.murphy,,61.51.62.212,"{""location"": ""UG"", ""is_mobile"": true}" 12763,5,285,2017-02-27 15:34:34,http://jaskolskileuschke.net/justice_johns,,238.221.135.205,"{""location"": ""PL"", ""is_mobile"": false}" 12764,5,285,2017-04-16 17:08:24,http://satterfield.biz/caroline,,19.93.150.105,"{""location"": ""ZA"", ""is_mobile"": false}" 12765,5,285,2017-05-20 17:41:02,http://hills.co/gene,,88.132.210.72,"{""location"": ""TO"", ""is_mobile"": true}" 12766,5,285,2017-01-31 04:37:54,http://durgan.co/brandon,,156.243.215.26,"{""location"": ""CY"", ""is_mobile"": true}" 12767,5,285,2017-03-14 09:44:06,http://west.org/rico_jacobson,,222.149.47.172,"{""location"": ""TH"", ""is_mobile"": true}" 12768,5,285,2017-06-12 07:53:43,http://hand.com/lucienne.crist,,96.68.43.200,"{""location"": ""LA"", ""is_mobile"": true}" 12769,5,285,2017-03-17 09:33:37,http://schultz.biz/theresia,,52.229.93.242,"{""location"": ""FR"", ""is_mobile"": true}" 12770,5,285,2017-01-03 15:28:32,http://oreilly.io/jaylan,,95.232.121.161,"{""location"": ""ES"", ""is_mobile"": true}" 12771,5,285,2017-01-17 01:18:41,http://tillman.co/krystal,,37.216.128.176,"{""location"": ""CO"", ""is_mobile"": false}" 12772,5,285,2017-06-04 21:33:38,http://beahan.co/joel_conn,,26.167.74.20,"{""location"": ""CF"", ""is_mobile"": true}" 12773,5,285,2017-06-07 08:12:17,http://macejkovic.co/dangelo_metz,,58.72.157.222,"{""location"": ""LA"", ""is_mobile"": true}" 12774,5,285,2017-04-10 01:00:01,http://schumm.net/elian,,192.167.72.153,"{""location"": ""CM"", ""is_mobile"": true}" 12775,5,285,2017-03-10 23:47:31,http://torphyfriesen.name/therese_huel,,123.9.155.43,"{""location"": ""NP"", ""is_mobile"": false}" 12776,5,285,2017-04-26 01:01:41,http://abbott.net/donato.oconner,,225.165.173.76,"{""location"": ""SJ"", ""is_mobile"": false}" 12777,5,285,2017-03-06 18:18:03,http://littelkeeling.org/makenzie,,35.81.38.126,"{""location"": ""MP"", ""is_mobile"": false}" 12778,5,285,2017-01-07 10:54:08,http://wolfdicki.co/patricia_prosacco,,78.7.254.239,"{""location"": ""GQ"", ""is_mobile"": false}" 12779,5,285,2017-03-07 08:16:12,http://mante.net/janiya,,104.20.168.93,"{""location"": ""KH"", ""is_mobile"": true}" 12780,5,285,2016-12-17 17:50:10,http://brown.io/harry_kihn,,100.90.200.253,"{""location"": ""MG"", ""is_mobile"": false}" 12781,5,285,2017-05-09 08:41:52,http://colekirlin.info/obie,,171.171.155.151,"{""location"": ""DZ"", ""is_mobile"": true}" 12782,5,285,2017-02-02 04:29:30,http://leuschkegrimes.org/alize.kuphal,,21.72.181.40,"{""location"": ""LB"", ""is_mobile"": false}" 12783,5,285,2017-02-09 19:05:29,http://moriette.com/nat_fadel,,165.110.206.114,"{""location"": ""ET"", ""is_mobile"": false}" 12784,5,285,2017-06-10 19:58:37,http://hand.co/danyka,,246.247.188.185,"{""location"": ""HU"", ""is_mobile"": true}" 12785,5,285,2017-05-19 22:10:54,http://bednar.info/ariel,,50.208.166.135,"{""location"": ""SY"", ""is_mobile"": true}" 12786,5,285,2017-01-06 16:14:22,http://kreiger.net/suzanne_kozey,,31.62.17.192,"{""location"": ""GY"", ""is_mobile"": false}" 12787,5,285,2016-12-13 09:45:27,http://jaskolski.net/juvenal,,164.134.61.80,"{""location"": ""AS"", ""is_mobile"": false}" 12788,5,285,2017-06-13 14:59:36,http://graham.info/reid,,123.130.243.176,"{""location"": ""PH"", ""is_mobile"": true}" 12789,5,285,2017-03-24 13:30:44,http://vandervort.net/elisha.dibbert,,24.234.80.157,"{""location"": ""GA"", ""is_mobile"": false}" 12790,5,285,2017-02-20 08:33:49,http://muller.co/pattie,,129.200.46.203,"{""location"": ""AR"", ""is_mobile"": true}" 12791,5,285,2017-04-11 09:45:11,http://jacobi.co/consuelo,,231.7.59.93,"{""location"": ""AX"", ""is_mobile"": true}" 12792,5,285,2017-01-31 08:56:53,http://schroeder.co/malachi_carroll,,141.99.2.227,"{""location"": ""FO"", ""is_mobile"": false}" 12793,5,285,2017-01-17 23:45:50,http://kuphal.io/ismael.mayert,,41.217.211.182,"{""location"": ""EC"", ""is_mobile"": false}" 12794,5,285,2017-06-10 05:40:47,http://larson.com/dudley.daugherty,,5.129.6.6,"{""location"": ""LK"", ""is_mobile"": true}" 12795,5,285,2017-05-01 00:03:20,http://heel.co/antwan,,179.142.111.3,"{""location"": ""NA"", ""is_mobile"": true}" 12796,5,285,2017-02-21 21:20:47,http://dickens.io/sid.cummings,,206.156.12.6,"{""location"": ""BD"", ""is_mobile"": true}" 12797,5,285,2017-04-20 04:24:02,http://wiegand.info/janie,,11.76.150.65,"{""location"": ""MM"", ""is_mobile"": false}" 12798,5,285,2016-12-24 19:59:20,http://macejkovic.net/deshawn.rutherford,,74.152.223.128,"{""location"": ""PT"", ""is_mobile"": false}" 12799,5,285,2017-01-26 02:02:25,http://brownjacobs.io/abagail,,197.138.45.83,"{""location"": ""GR"", ""is_mobile"": false}" 12800,5,285,2017-04-18 13:40:36,http://torptorphy.net/frederic,,12.138.41.85,"{""location"": ""GY"", ""is_mobile"": true}" 12801,5,285,2017-04-21 13:27:05,http://zboncak.io/frances,,2.78.159.187,"{""location"": ""SH"", ""is_mobile"": true}" 12802,5,286,2017-05-22 13:05:53,http://ohara.com/rhiannon,,199.147.153.205,"{""location"": ""KY"", ""is_mobile"": true}" 12803,5,286,2017-05-10 17:29:40,http://torp.org/darlene.king,,68.222.173.147,"{""location"": ""SJ"", ""is_mobile"": true}" 12804,5,286,2017-05-10 23:43:16,http://marksfranecki.org/stacy.predovic,,134.112.187.41,"{""location"": ""PN"", ""is_mobile"": false}" 15751,6,354,2017-05-14 01:38:14,http://bartell.biz/katlyn,,203.72.96.171,"{""location"": ""SN"", ""is_mobile"": false}" 15752,6,354,2017-05-29 21:57:09,http://predovickrajcik.io/kelsi,,150.202.94.20,"{""location"": ""GQ"", ""is_mobile"": true}" 15753,6,354,2017-02-02 17:42:59,http://kertzmannwindler.org/daren,,228.221.80.187,"{""location"": ""HR"", ""is_mobile"": false}" 15754,6,354,2017-01-23 08:25:20,http://hermiston.info/demarco_jacobi,,84.193.26.213,"{""location"": ""SV"", ""is_mobile"": true}" 15755,6,354,2017-04-26 14:38:42,http://huelbins.biz/carmel_koch,,161.45.146.122,"{""location"": ""DK"", ""is_mobile"": true}" 15756,6,354,2016-12-25 19:34:30,http://bartolettischowalter.biz/quinn.reynolds,,76.169.67.207,"{""location"": ""GP"", ""is_mobile"": false}" 15757,6,354,2017-03-26 15:16:54,http://rutherford.org/stephon,,23.61.236.247,"{""location"": ""IT"", ""is_mobile"": true}" 15758,6,354,2017-06-10 03:52:37,http://walter.com/guie_kris,,197.39.165.190,"{""location"": ""SD"", ""is_mobile"": false}" 15759,6,354,2017-01-05 22:45:27,http://wuckert.org/eloisa.kreiger,,59.37.124.103,"{""location"": ""PS"", ""is_mobile"": false}" 15760,6,354,2017-03-19 20:56:23,http://kiehn.io/jay_conroy,,37.61.132.219,"{""location"": ""IT"", ""is_mobile"": false}" 15761,6,354,2017-03-26 02:40:52,http://andersondenesik.name/myles_ernser,,105.28.247.200,"{""location"": ""SC"", ""is_mobile"": true}" 15762,6,354,2017-06-06 01:01:05,http://nicolas.biz/raymond_eichmann,,22.35.189.232,"{""location"": ""DZ"", ""is_mobile"": false}" 15763,6,354,2017-02-16 02:36:50,http://hilpert.name/jaida.sawayn,,117.174.127.178,"{""location"": ""UY"", ""is_mobile"": false}" 15764,6,354,2017-06-13 21:07:58,http://wisozkgreenfelder.info/orland,,158.196.251.49,"{""location"": ""TH"", ""is_mobile"": true}" 15765,6,354,2017-01-05 06:17:53,http://starkokuneva.io/sonya.rutherford,,55.152.124.60,"{""location"": ""CI"", ""is_mobile"": false}" 15766,6,354,2017-02-07 14:28:03,http://blanda.co/hilbert,,71.85.15.39,"{""location"": ""BB"", ""is_mobile"": true}" 15767,6,354,2017-04-18 17:19:45,http://kautzercole.info/elnora.cormier,,144.188.165.128,"{""location"": ""YT"", ""is_mobile"": true}" 15768,6,354,2017-02-05 23:30:51,http://glover.org/noemi,,236.98.101.204,"{""location"": ""QA"", ""is_mobile"": true}" 15769,6,354,2017-06-06 22:59:58,http://gleason.org/lupe_schuster,,107.74.114.160,"{""location"": ""PG"", ""is_mobile"": false}" 15770,6,354,2017-05-09 21:04:32,http://boyer.name/freida.hackett,,71.106.140.79,"{""location"": ""FJ"", ""is_mobile"": false}" 15771,6,354,2016-12-29 03:42:42,http://block.name/milan_ledner,,90.88.87.48,"{""location"": ""NG"", ""is_mobile"": false}" 15772,6,355,2017-03-29 07:03:23,http://collier.biz/shanie,,113.24.223.92,"{""location"": ""FR"", ""is_mobile"": true}" 15773,6,355,2017-02-27 13:07:20,http://kub.info/sienna,,78.157.219.9,"{""location"": ""IR"", ""is_mobile"": true}" 15774,6,355,2017-04-08 00:24:35,http://toylemke.net/pedro,,166.90.45.203,"{""location"": ""DJ"", ""is_mobile"": false}" 15775,6,355,2017-06-10 15:27:16,http://rogahn.com/nona,,222.245.242.119,"{""location"": ""PH"", ""is_mobile"": true}" 15776,6,355,2017-01-01 09:24:21,http://quitzon.org/oswald,,107.169.230.172,"{""location"": ""SV"", ""is_mobile"": true}" 15777,6,355,2017-02-13 11:35:21,http://boyer.name/zelda,,201.125.143.5,"{""location"": ""SI"", ""is_mobile"": false}" 15778,6,355,2017-02-07 02:22:46,http://wolf.co/quincy_predovic,,84.177.162.219,"{""location"": ""GF"", ""is_mobile"": true}" 15779,6,355,2017-04-12 02:29:59,http://kling.info/warren.schmeler,,182.52.38.34,"{""location"": ""TZ"", ""is_mobile"": false}" 15780,6,355,2017-05-26 20:11:50,http://thielkunze.net/vella,,103.132.253.49,"{""location"": ""GE"", ""is_mobile"": true}" 15781,6,355,2017-04-07 19:30:39,http://connelly.info/aryanna.satterfield,,112.79.129.252,"{""location"": ""BG"", ""is_mobile"": true}" 15782,6,355,2016-12-31 22:26:29,http://raynorhackett.biz/kaelyn,,16.214.75.241,"{""location"": ""ZW"", ""is_mobile"": true}" 15783,6,355,2017-06-06 05:32:40,http://hermanspinka.name/monica_cruickshank,,96.96.188.233,"{""location"": ""SB"", ""is_mobile"": true}" 15784,6,355,2017-01-29 16:39:43,http://nicolas.biz/herman_mante,,124.73.179.221,"{""location"": ""MG"", ""is_mobile"": true}" 15785,6,355,2017-01-07 18:48:57,http://dickipollich.biz/maryam,,177.125.76.234,"{""location"": ""MN"", ""is_mobile"": true}" 15786,6,355,2017-04-20 15:58:13,http://oconner.net/teresa.dooley,,221.147.18.16,"{""location"": ""SI"", ""is_mobile"": true}" 15787,6,355,2017-03-08 13:42:20,http://littel.info/cicero,,6.197.99.129,"{""location"": ""FI"", ""is_mobile"": true}" 15788,6,355,2017-01-01 12:06:30,http://tromp.io/frieda.dach,,35.54.157.86,"{""location"": ""FR"", ""is_mobile"": true}" 15789,6,355,2017-02-01 23:54:37,http://schmidt.info/titus,,187.51.14.191,"{""location"": ""YE"", ""is_mobile"": false}" 15790,6,355,2017-05-15 17:50:14,http://runolfon.biz/mabel_moen,,195.90.34.116,"{""location"": ""VN"", ""is_mobile"": true}" 15791,6,355,2017-03-09 19:25:56,http://prosacco.io/alia.mccullough,,38.77.168.21,"{""location"": ""BM"", ""is_mobile"": false}" 15792,6,355,2017-06-13 03:55:26,http://barrows.com/marcos,,181.159.83.74,"{""location"": ""NG"", ""is_mobile"": true}" 15793,6,355,2017-01-08 17:47:23,http://towne.name/rose_glover,,177.14.77.246,"{""location"": ""VG"", ""is_mobile"": false}" 15794,6,355,2017-01-28 06:08:01,http://klingmayer.com/jake,,42.150.58.145,"{""location"": ""LR"", ""is_mobile"": false}" 15795,6,355,2017-05-26 17:38:16,http://sipesrunte.com/albertha.morar,,72.231.184.123,"{""location"": ""TV"", ""is_mobile"": true}" 15796,6,355,2017-05-29 22:33:30,http://wittinggreenholt.biz/abdul,,209.99.242.152,"{""location"": ""MN"", ""is_mobile"": true}" 15797,6,355,2017-04-12 02:21:15,http://hills.biz/dawn,,251.99.139.54,"{""location"": ""RO"", ""is_mobile"": false}" 15798,6,355,2017-06-11 11:01:08,http://bahringerrutherford.com/martina_anderson,,129.97.212.115,"{""location"": ""PM"", ""is_mobile"": true}" 15799,6,355,2017-05-08 02:51:06,http://boscoryan.info/larue,,18.142.69.216,"{""location"": ""TZ"", ""is_mobile"": false}" 15800,6,355,2017-05-30 07:48:22,http://veumrobel.info/hyman,,37.25.57.126,"{""location"": ""FK"", ""is_mobile"": false}" 15801,6,355,2017-02-16 01:38:19,http://stanton.info/newell.nienow,,32.102.165.207,"{""location"": ""UZ"", ""is_mobile"": false}" 15802,6,355,2017-02-14 22:23:25,http://sanford.biz/jayme.gusikowski,,142.220.42.13,"{""location"": ""TH"", ""is_mobile"": false}" 15803,6,355,2017-02-28 19:16:03,http://wolfadams.org/adan_halvorson,,109.69.242.210,"{""location"": ""IL"", ""is_mobile"": true}" 15804,6,355,2017-03-11 02:08:14,http://ondrickavon.name/araceli.smith,,217.205.215.61,"{""location"": ""LC"", ""is_mobile"": false}" 15805,6,355,2017-01-06 19:59:42,http://cartwrightmetz.net/juliet_keebler,,201.106.40.35,"{""location"": ""GD"", ""is_mobile"": true}" 2957,2,65,2017-05-10 14:25:29,http://funk.biz/ryan.oconner,0.4598884545,39.39.181.3,"{""location"": ""MC"", ""is_mobile"": true}" 2958,2,65,2017-06-09 22:51:34,http://murray.name/thora_mccullough,0.9545603204,224.92.149.187,"{""location"": ""SI"", ""is_mobile"": false}" 2959,2,65,2016-12-25 02:10:14,http://rolfson.net/oma.boehm,0.7263601725,206.191.214.130,"{""location"": ""VN"", ""is_mobile"": true}" 2960,2,65,2017-03-16 12:49:41,http://stoltenberg.info/dejuan,0.9851600068,61.194.244.186,"{""location"": ""MT"", ""is_mobile"": false}" 2961,2,65,2016-12-21 06:37:22,http://schaeferorn.co/shawna,0.8438723357,142.52.5.133,"{""location"": ""SV"", ""is_mobile"": false}" 2962,2,65,2017-05-28 08:05:35,http://strosin.info/vito.kris,0.7085063828,125.86.54.175,"{""location"": ""NC"", ""is_mobile"": true}" 2963,2,66,2017-03-22 22:46:01,http://rogahn.info/elouise,0.9048305324,65.210.62.23,"{""location"": ""NP"", ""is_mobile"": false}" 2964,2,66,2016-12-25 03:15:38,http://von.net/kelsie_mclaughlin,0.1989095692,2.95.231.51,"{""location"": ""GN"", ""is_mobile"": true}" 2965,2,66,2017-06-06 14:25:48,http://bernhard.com/fernando.doyle,0.6860244887,244.118.230.56,"{""location"": ""FJ"", ""is_mobile"": false}" 2966,2,66,2016-12-18 19:59:17,http://hane.com/alf,0.9810337320,213.94.223.147,"{""location"": ""MY"", ""is_mobile"": true}" 2967,2,66,2017-04-22 20:43:42,http://bechtelar.name/zena.nitzsche,0.8785353873,29.147.253.112,"{""location"": ""RU"", ""is_mobile"": false}" 2968,2,66,2017-05-31 15:20:19,http://padberg.info/johnpaul,0.3705462125,29.147.115.131,"{""location"": ""CK"", ""is_mobile"": true}" 2969,2,66,2017-03-12 07:40:15,http://mueller.biz/bernard,0.1321717799,251.115.229.91,"{""location"": ""MO"", ""is_mobile"": false}" 2970,2,66,2017-04-15 11:56:14,http://berge.biz/madelyn,0.6632042283,192.120.175.167,"{""location"": ""NF"", ""is_mobile"": true}" 2971,2,66,2017-06-11 05:31:31,http://gusikowskipowlowski.info/dan,0.8257899771,77.83.85.72,"{""location"": ""SL"", ""is_mobile"": true}" 2972,2,66,2017-04-11 18:01:11,http://damore.org/green,0.4497904166,152.231.57.202,"{""location"": ""ZW"", ""is_mobile"": true}" 2973,2,66,2017-02-07 17:50:37,http://strackebecker.info/hertha.fritsch,0.2365714273,30.242.101.215,"{""location"": ""TW"", ""is_mobile"": true}" 2974,2,66,2017-02-13 10:12:12,http://schinner.org/eloy,0.9443732849,198.159.6.235,"{""location"": ""HN"", ""is_mobile"": false}" 2975,2,66,2017-04-21 05:59:29,http://nikolausterry.info/elenora.kerluke,0.2105318936,149.145.55.215,"{""location"": ""GW"", ""is_mobile"": true}" 2976,2,66,2017-02-11 10:03:11,http://prohaska.com/amelia_kuphal,0.7121725580,119.194.184.32,"{""location"": ""GH"", ""is_mobile"": false}" 3946,2,87,2017-01-09 13:42:45,http://larkin.biz/bruce,,236.17.229.221,"{""location"": ""IM"", ""is_mobile"": true}" 2977,2,66,2017-02-06 15:22:03,http://weimann.net/josiane_lubowitz,0.9060446351,71.22.90.168,"{""location"": ""SI"", ""is_mobile"": true}" 2978,2,66,2017-01-25 02:40:05,http://hamilllubowitz.com/lambert.turner,0.6888431909,11.29.95.60,"{""location"": ""IO"", ""is_mobile"": false}" 2979,2,66,2017-02-12 13:20:57,http://naderkshlerin.name/joany_schulist,0.9559052130,146.211.117.92,"{""location"": ""ER"", ""is_mobile"": true}" 2980,2,66,2017-04-13 17:39:17,http://little.org/freeda.berge,0.4453407911,202.137.198.167,"{""location"": ""BE"", ""is_mobile"": false}" 2981,2,66,2017-02-06 11:03:53,http://mckenzie.co/providenci,0.3175380580,133.33.158.61,"{""location"": ""BV"", ""is_mobile"": true}" 2982,2,66,2017-03-24 06:53:40,http://sauer.com/jacinto,0.6318277114,254.235.133.73,"{""location"": ""KN"", ""is_mobile"": false}" 2983,2,66,2017-02-01 03:24:03,http://oberbrunnerkonopelski.net/alfredo,0.9653703150,142.74.218.61,"{""location"": ""BQ"", ""is_mobile"": false}" 2984,2,66,2017-03-20 22:46:05,http://moen.biz/darrin_white,0.2553838715,153.16.135.189,"{""location"": ""GQ"", ""is_mobile"": true}" 2985,2,66,2017-01-25 19:01:46,http://moore.name/bernita_hoppe,0.1828467353,179.234.124.35,"{""location"": ""AQ"", ""is_mobile"": true}" 2986,2,66,2017-05-30 11:25:09,http://morarrowe.net/mina,0.2323580328,116.145.52.152,"{""location"": ""UY"", ""is_mobile"": true}" 2987,2,66,2017-02-06 11:17:07,http://larsonfeeney.io/hertha_jerde,0.7669827216,66.143.193.213,"{""location"": ""VC"", ""is_mobile"": true}" 2988,2,66,2017-05-13 06:28:43,http://boehmturcotte.info/karen,0.1697935864,251.151.128.147,"{""location"": ""SH"", ""is_mobile"": true}" 2989,2,66,2017-01-04 21:37:10,http://gibsonrowe.biz/omer,0.9791108276,26.248.249.185,"{""location"": ""PS"", ""is_mobile"": false}" 2990,2,66,2017-06-03 15:15:20,http://altenwerth.name/filiberto_hegmann,0.7634864120,36.88.99.83,"{""location"": ""TF"", ""is_mobile"": false}" 2991,2,66,2017-04-12 05:01:30,http://erdman.info/desmond_leuschke,0.7755646847,228.110.223.107,"{""location"": ""TJ"", ""is_mobile"": false}" 2992,2,66,2017-01-02 09:53:41,http://denesik.biz/kathryn.champlin,0.9921455659,94.6.133.44,"{""location"": ""NF"", ""is_mobile"": false}" 2993,2,66,2017-03-22 02:32:10,http://klinglubowitz.com/augustus_rempel,0.9848990854,248.221.106.235,"{""location"": ""AU"", ""is_mobile"": true}" 2994,2,66,2017-02-21 13:54:11,http://kubframi.name/wilton_turcotte,0.0825520141,58.198.245.22,"{""location"": ""UM"", ""is_mobile"": false}" 2995,2,66,2017-04-10 04:37:39,http://jenkins.co/jermain,0.5959374033,252.21.248.109,"{""location"": ""CF"", ""is_mobile"": true}" 2996,2,66,2017-06-13 07:57:54,http://krajcikkshlerin.net/lysanne,0.7657235355,176.236.165.140,"{""location"": ""MF"", ""is_mobile"": true}" 2997,2,66,2017-03-20 19:11:14,http://skiles.net/kareem_rogahn,0.9188463793,44.237.204.210,"{""location"": ""SD"", ""is_mobile"": true}" 2998,2,66,2017-05-05 12:13:50,http://simonis.info/enos,0.5870025946,155.3.223.104,"{""location"": ""TG"", ""is_mobile"": true}" 2999,2,66,2016-12-19 22:02:29,http://feest.biz/brain,0.3884351561,56.244.6.107,"{""location"": ""TH"", ""is_mobile"": true}" 3000,2,66,2017-05-22 01:28:18,http://ryanreichel.name/laurianne.durgan,0.3051801624,77.219.129.58,"{""location"": ""EC"", ""is_mobile"": false}" 3001,2,66,2016-12-29 19:59:57,http://lockman.biz/lamont,0.4292839274,89.120.122.249,"{""location"": ""BF"", ""is_mobile"": false}" 3002,2,66,2017-01-25 22:27:48,http://abshire.net/amely.blanda,0.0660953820,85.82.85.112,"{""location"": ""SK"", ""is_mobile"": false}" 3003,2,66,2017-01-30 21:03:48,http://binshartmann.org/providenci.kunze,0.2334200402,94.139.245.40,"{""location"": ""GN"", ""is_mobile"": true}" 3004,2,66,2017-04-25 16:15:18,http://torphy.co/arjun,0.3727160113,137.233.151.138,"{""location"": ""PL"", ""is_mobile"": true}" 3005,2,66,2017-04-16 05:43:20,http://labadie.co/bertram_moriette,0.1648455782,122.158.147.236,"{""location"": ""GI"", ""is_mobile"": false}" 3006,2,66,2017-06-10 02:27:16,http://beahan.co/jaeden,0.6726252622,122.33.165.51,"{""location"": ""NP"", ""is_mobile"": true}" 3007,2,66,2017-03-12 03:45:25,http://jones.co/chanel,0.0763117607,40.69.10.136,"{""location"": ""SY"", ""is_mobile"": false}" 9893,4,221,2017-06-09 14:06:30,http://howe.name/elva_littel,0.9960789505,68.155.14.95,"{""location"": ""HT"", ""is_mobile"": true}" 9894,4,221,2017-05-27 04:42:40,http://stehr.com/vickie,0.2653502529,43.199.219.141,"{""location"": ""BS"", ""is_mobile"": false}" 9895,4,221,2017-04-24 00:14:23,http://keler.org/caterina_prohaska,0.8508095690,64.189.128.171,"{""location"": ""GD"", ""is_mobile"": false}" 9896,4,221,2017-06-14 01:04:30,http://gerlach.name/barton,0.3212696929,84.4.173.119,"{""location"": ""BB"", ""is_mobile"": false}" 9897,4,221,2017-04-28 04:14:43,http://moore.io/camilla,0.6867684723,135.249.86.31,"{""location"": ""CO"", ""is_mobile"": false}" 9898,4,221,2017-01-19 11:18:09,http://okeefe.com/jaunita.von,0.8567044926,249.109.214.30,"{""location"": ""TL"", ""is_mobile"": false}" 9899,4,221,2017-03-06 06:46:41,http://darehyatt.org/nora.moriette,0.7092761715,152.172.120.197,"{""location"": ""MK"", ""is_mobile"": false}" 9900,4,221,2017-06-01 11:07:14,http://kundecasper.net/ernie,0.7294184274,117.222.154.17,"{""location"": ""BG"", ""is_mobile"": false}" 9901,4,221,2016-12-16 10:09:22,http://daniel.info/anna,0.5186599459,116.16.243.80,"{""location"": ""GF"", ""is_mobile"": true}" 9902,4,222,2017-03-24 03:07:58,http://heaney.biz/diego,0.8655071324,221.192.161.229,"{""location"": ""NE"", ""is_mobile"": true}" 9903,4,222,2017-04-02 10:28:29,http://ryan.info/easton,0.0102671841,209.49.218.82,"{""location"": ""PH"", ""is_mobile"": false}" 9904,4,222,2017-06-10 16:43:02,http://deckow.info/maynard,0.4657222618,28.44.31.215,"{""location"": ""FI"", ""is_mobile"": true}" 9905,4,222,2016-12-19 15:08:15,http://bartellborer.co/ruben,0.5136799851,220.111.248.216,"{""location"": ""LS"", ""is_mobile"": false}" 9906,4,222,2017-03-08 03:37:12,http://murphy.info/kaden,0.1278335126,238.211.202.211,"{""location"": ""LK"", ""is_mobile"": false}" 9907,4,222,2017-03-11 09:44:03,http://gleason.org/marian.pagac,0.1298642076,103.247.6.215,"{""location"": ""PA"", ""is_mobile"": false}" 9908,4,222,2017-05-31 16:10:38,http://kelerjacobi.io/geoffrey.rodriguez,0.0282220485,184.95.104.134,"{""location"": ""MG"", ""is_mobile"": true}" 9909,4,222,2017-03-02 06:23:48,http://reillyernser.org/guy_becker,0.7981495528,83.207.95.86,"{""location"": ""BY"", ""is_mobile"": true}" 9910,4,222,2017-04-14 12:48:28,http://wolffullrich.com/deshaun_goodwin,0.4252387221,154.184.95.229,"{""location"": ""WF"", ""is_mobile"": true}" 9911,4,222,2017-02-23 20:04:24,http://langworth.co/carolyn.kohler,0.6644933621,76.50.179.226,"{""location"": ""EH"", ""is_mobile"": false}" 9912,4,222,2017-06-05 05:55:21,http://douglasroob.info/frank,0.8580511222,234.116.153.6,"{""location"": ""FR"", ""is_mobile"": true}" 9913,4,222,2017-01-14 15:16:20,http://gottliebreilly.info/edison.buckridge,0.9873040120,69.181.6.230,"{""location"": ""CH"", ""is_mobile"": true}" 9914,4,222,2016-12-14 00:52:26,http://lynch.org/stone_becker,0.6274415243,54.51.187.136,"{""location"": ""CU"", ""is_mobile"": false}" 9915,4,222,2017-02-10 15:39:15,http://ferry.info/kenya,0.6103079753,205.15.73.109,"{""location"": ""AX"", ""is_mobile"": false}" 9916,4,222,2017-01-26 15:00:04,http://kertzmann.net/joelle,0.1879752067,77.216.217.185,"{""location"": ""BQ"", ""is_mobile"": true}" 9917,4,222,2017-02-11 09:40:42,http://gutmanntrantow.org/yesenia,0.8999576168,164.25.201.105,"{""location"": ""MH"", ""is_mobile"": false}" 9918,4,222,2017-02-11 07:33:38,http://spinkabarton.co/lexus,0.0972669064,105.190.70.160,"{""location"": ""EH"", ""is_mobile"": true}" 9919,4,222,2017-05-02 06:43:15,http://adams.com/julio.mills,0.4950991137,98.63.222.43,"{""location"": ""EG"", ""is_mobile"": true}" 9920,4,222,2017-05-12 04:11:09,http://ziemestark.net/mohamed,0.2543832217,170.78.156.208,"{""location"": ""ST"", ""is_mobile"": false}" 9921,4,222,2017-02-03 20:19:30,http://simonis.co/stevie,0.4043849745,254.159.254.107,"{""location"": ""IQ"", ""is_mobile"": false}" 9922,4,222,2017-02-11 09:50:25,http://ankunding.co/wyatt.rempel,0.2679693583,44.36.123.40,"{""location"": ""CU"", ""is_mobile"": true}" 9923,4,222,2017-01-30 01:00:42,http://kirlin.co/daisha,0.7674543857,187.120.193.63,"{""location"": ""KN"", ""is_mobile"": true}" 9924,4,222,2017-02-23 03:24:45,http://bode.biz/garrett.mayer,0.2497501477,128.22.29.2,"{""location"": ""AU"", ""is_mobile"": true}" 9925,4,222,2016-12-20 09:15:59,http://friesenkuhlman.name/millie,0.6171785043,225.91.191.149,"{""location"": ""PR"", ""is_mobile"": false}" 9926,4,222,2017-04-30 01:47:26,http://champlinlang.com/dedrick,0.3992283042,203.81.44.2,"{""location"": ""MY"", ""is_mobile"": true}" 9927,4,222,2017-04-13 13:06:26,http://lehnerkuphal.info/keegan_sanford,0.8585245644,104.234.159.8,"{""location"": ""GP"", ""is_mobile"": true}" 9928,4,222,2017-02-20 19:54:59,http://herzogquitzon.biz/valentin,0.1572329922,105.65.99.96,"{""location"": ""LI"", ""is_mobile"": false}" 9929,4,222,2017-04-16 19:56:56,http://heathcote.net/doug,0.5846206944,252.194.173.209,"{""location"": ""LA"", ""is_mobile"": false}" 9930,4,222,2016-12-22 11:47:03,http://kuhiccole.co/jettie_von,0.1279437551,250.119.252.2,"{""location"": ""SA"", ""is_mobile"": true}" 9931,4,222,2017-04-17 10:24:47,http://schmitt.info/giovanna_mclaughlin,0.4299213108,33.48.139.151,"{""location"": ""AG"", ""is_mobile"": false}" 9932,4,222,2017-06-03 01:53:44,http://keebler.io/leola,0.9049610680,146.30.172.177,"{""location"": ""BE"", ""is_mobile"": false}" 9933,4,222,2017-03-05 00:33:29,http://rolfsonthiel.info/cayla.nolan,0.6333016458,66.13.252.63,"{""location"": ""BI"", ""is_mobile"": false}" 9934,4,222,2017-06-12 01:49:38,http://beahan.com/alberta.carroll,0.7913128933,96.14.5.129,"{""location"": ""TR"", ""is_mobile"": true}" 9935,4,222,2017-02-13 18:14:00,http://johnsko.info/robert_feeney,0.1717907512,2.60.238.102,"{""location"": ""MM"", ""is_mobile"": true}" 9936,4,222,2016-12-23 01:21:25,http://oberbrunnerhermann.io/ernesto,0.6817795429,118.222.176.193,"{""location"": ""HM"", ""is_mobile"": true}" 9937,4,222,2017-02-28 11:30:26,http://goyette.co/eugene.leannon,0.2698759457,147.124.4.230,"{""location"": ""BG"", ""is_mobile"": true}" 9938,4,222,2017-05-18 12:23:47,http://brakus.biz/vidal,0.1500199384,124.136.225.37,"{""location"": ""LA"", ""is_mobile"": false}" 9939,4,222,2017-03-08 00:19:50,http://gerlach.name/keegan.okuneva,0.3891908024,23.33.58.74,"{""location"": ""SK"", ""is_mobile"": false}" 9940,4,222,2017-04-19 23:09:24,http://pagac.info/jabari_ratke,0.2221759482,246.87.133.187,"{""location"": ""WF"", ""is_mobile"": false}" 9941,4,222,2017-06-12 23:42:54,http://dietrichgleichner.biz/hans_bednar,0.6254960501,49.73.140.252,"{""location"": ""HR"", ""is_mobile"": true}" 9942,4,223,2017-03-24 06:15:53,http://hand.biz/aleandra.shanahan,0.1325028515,209.137.5.210,"{""location"": ""BT"", ""is_mobile"": true}" 9943,4,223,2017-01-12 23:08:52,http://keebler.co/lora_schmitt,0.7249031819,171.91.64.210,"{""location"": ""SC"", ""is_mobile"": true}" 9944,4,223,2017-05-02 13:56:37,http://nitzsche.name/selena,0.4627372954,147.212.182.23,"{""location"": ""BW"", ""is_mobile"": true}" 6890,3,151,2017-01-28 22:40:08,http://dach.io/olen.mills,,187.69.35.112,"{""location"": ""MO"", ""is_mobile"": false}" 6891,3,151,2017-03-26 09:06:10,http://lednerjerde.name/urban,,221.120.162.48,"{""location"": ""MA"", ""is_mobile"": false}" 6892,3,152,2017-05-28 21:49:50,http://maggio.com/jace.jones,,84.105.36.87,"{""location"": ""LB"", ""is_mobile"": false}" 6893,3,152,2017-03-17 19:12:26,http://gulgowski.net/ludie,,110.13.70.91,"{""location"": ""PF"", ""is_mobile"": true}" 6894,3,152,2017-01-31 16:41:25,http://fay.net/nolan.gleichner,,226.83.253.204,"{""location"": ""ID"", ""is_mobile"": true}" 6895,3,152,2017-03-01 11:33:12,http://sawayn.biz/bryce.howe,,47.150.160.185,"{""location"": ""SN"", ""is_mobile"": false}" 6896,3,152,2017-03-31 03:34:21,http://runolfsdottirharvey.org/jayne_corkery,,28.37.209.92,"{""location"": ""JP"", ""is_mobile"": true}" 6897,3,152,2017-05-25 10:26:30,http://wehnerbogisich.info/keeley.runolfsdottir,,59.55.70.145,"{""location"": ""TO"", ""is_mobile"": false}" 6898,3,152,2017-02-01 09:34:16,http://walker.co/alvah,,94.121.167.207,"{""location"": ""DJ"", ""is_mobile"": true}" 6899,3,152,2017-04-06 12:18:07,http://rauzboncak.biz/elna,,179.195.168.149,"{""location"": ""GI"", ""is_mobile"": true}" 6900,3,152,2017-01-02 06:08:00,http://heelmoriette.info/kasandra.dietrich,,67.2.173.101,"{""location"": ""CL"", ""is_mobile"": false}" 6901,3,152,2017-04-04 14:17:31,http://lemke.co/kiera.prosacco,,216.147.251.11,"{""location"": ""DJ"", ""is_mobile"": false}" 6902,3,152,2017-03-11 18:45:27,http://douglas.com/kamryn,,94.125.79.142,"{""location"": ""MP"", ""is_mobile"": false}" 6903,3,152,2016-12-29 08:49:11,http://mullerdare.net/toby.feeney,,93.117.252.212,"{""location"": ""CA"", ""is_mobile"": true}" 6904,3,152,2017-04-23 06:28:59,http://goyettemarks.net/calista.daniel,,9.240.152.89,"{""location"": ""KR"", ""is_mobile"": false}" 6905,3,152,2017-05-21 12:50:35,http://hermannnicolas.com/crystal,,193.203.206.62,"{""location"": ""NF"", ""is_mobile"": false}" 6906,3,152,2016-12-24 20:34:08,http://kreiger.org/annabell_adams,,190.193.26.231,"{""location"": ""SB"", ""is_mobile"": false}" 6907,3,152,2017-05-27 23:38:55,http://bauch.io/lucile,,118.100.70.12,"{""location"": ""PL"", ""is_mobile"": false}" 6908,3,152,2017-01-04 20:14:54,http://volkmanquitzon.org/jordan,,15.70.183.109,"{""location"": ""SN"", ""is_mobile"": true}" 6909,3,152,2017-01-14 15:13:20,http://kuhn.org/deon,,246.66.6.8,"{""location"": ""LB"", ""is_mobile"": true}" 6910,3,152,2017-06-13 05:15:13,http://abshire.co/christy.gerhold,,70.12.71.230,"{""location"": ""TZ"", ""is_mobile"": false}" 6911,3,152,2017-04-17 23:44:47,http://harbertromp.net/camila,,54.87.226.70,"{""location"": ""GL"", ""is_mobile"": true}" 6912,3,152,2017-03-26 08:01:05,http://bechtelarhaag.info/gretchen.heel,,62.65.9.85,"{""location"": ""SD"", ""is_mobile"": true}" 6913,3,152,2017-03-26 12:49:17,http://casper.info/belle_eichmann,,180.110.102.48,"{""location"": ""ZM"", ""is_mobile"": false}" 6914,3,152,2017-01-15 20:43:31,http://funk.net/eugene_cain,,87.46.189.63,"{""location"": ""CZ"", ""is_mobile"": false}" 6915,3,152,2017-01-17 23:13:46,http://doyleharris.com/anibal.sanford,,31.189.231.184,"{""location"": ""LY"", ""is_mobile"": false}" 6916,3,152,2017-02-26 15:02:58,http://toy.biz/jalen_heel,,59.99.135.21,"{""location"": ""GH"", ""is_mobile"": false}" 6917,3,152,2017-02-22 02:18:46,http://harveygoyette.io/autumn,,252.224.155.123,"{""location"": ""PA"", ""is_mobile"": true}" 6918,3,152,2017-05-30 09:30:24,http://sipes.net/christiana_davis,,17.128.237.152,"{""location"": ""HR"", ""is_mobile"": false}" 6919,3,152,2017-01-11 10:43:10,http://doyle.net/christiana,,92.83.129.48,"{""location"": ""CW"", ""is_mobile"": false}" 6920,3,152,2017-01-09 09:03:17,http://okeefe.io/madie,,112.80.244.168,"{""location"": ""VE"", ""is_mobile"": true}" 6921,3,152,2017-06-05 02:07:49,http://mannabernathy.biz/georgianna_rosenbaum,,104.115.217.83,"{""location"": ""SZ"", ""is_mobile"": false}" 6922,3,152,2017-04-26 23:20:40,http://swaniawski.com/granville_upton,,166.121.66.220,"{""location"": ""SK"", ""is_mobile"": false}" 6923,3,152,2017-02-11 10:25:51,http://kling.info/chandler.dibbert,,196.224.116.200,"{""location"": ""CC"", ""is_mobile"": true}" 6924,3,152,2017-03-12 07:23:38,http://pacochakoch.net/keon_osinski,,29.232.98.162,"{""location"": ""CH"", ""is_mobile"": false}" 6925,3,152,2017-02-13 21:28:05,http://kuhn.info/estelle,,163.60.121.195,"{""location"": ""BG"", ""is_mobile"": false}" 6926,3,152,2017-05-30 15:04:47,http://walshjast.io/francis,,219.135.220.118,"{""location"": ""AR"", ""is_mobile"": false}" 6927,3,152,2017-04-25 02:28:00,http://hegmann.name/jaida,,59.61.135.141,"{""location"": ""IR"", ""is_mobile"": true}" 6928,3,152,2016-12-28 22:25:16,http://rogahn.name/hank.gulgowski,,249.191.25.57,"{""location"": ""BT"", ""is_mobile"": false}" 6929,3,152,2017-02-23 01:49:55,http://von.name/quinten.gutkowski,,201.101.168.4,"{""location"": ""CH"", ""is_mobile"": false}" 6930,3,152,2017-01-01 16:29:43,http://murphy.name/barry,,138.43.205.91,"{""location"": ""ZW"", ""is_mobile"": false}" 6931,3,152,2017-01-21 20:38:28,http://kunde.co/francesca.heidenreich,,153.40.233.220,"{""location"": ""AI"", ""is_mobile"": false}" 6932,3,152,2017-02-11 11:51:32,http://parisian.co/clare,,177.24.173.181,"{""location"": ""KR"", ""is_mobile"": false}" 6934,3,152,2017-03-01 14:56:39,http://feeneybecker.info/bethany,,24.113.203.65,"{""location"": ""DZ"", ""is_mobile"": true}" 6935,3,152,2017-01-17 04:41:09,http://littel.co/viva,,36.252.127.127,"{""location"": ""ZM"", ""is_mobile"": false}" 6936,3,152,2017-05-17 10:27:49,http://boganbauch.net/jodie.nolan,,81.122.235.221,"{""location"": ""BG"", ""is_mobile"": false}" 6937,3,152,2017-04-05 23:11:12,http://gottlieb.info/tiffany.tromp,,82.154.179.169,"{""location"": ""KI"", ""is_mobile"": false}" 6938,3,152,2017-03-06 12:20:18,http://williamson.net/erick,,15.238.96.175,"{""location"": ""TN"", ""is_mobile"": false}" 6939,3,152,2017-05-04 08:37:44,http://denesikharvey.io/casandra,,116.238.20.97,"{""location"": ""BW"", ""is_mobile"": true}" 6940,3,152,2017-06-04 14:36:38,http://prohaskanienow.info/giuseppe,,67.102.112.75,"{""location"": ""NU"", ""is_mobile"": true}" 6941,3,152,2017-01-12 05:08:55,http://jacobsoncummerata.io/valentine.gutmann,,219.82.181.181,"{""location"": ""LC"", ""is_mobile"": true}" 6942,3,152,2016-12-19 01:55:38,http://kris.org/kattie,,159.122.97.227,"{""location"": ""GH"", ""is_mobile"": true}" 6943,3,152,2017-02-12 03:10:21,http://okuneva.org/kayli,,170.251.126.98,"{""location"": ""KN"", ""is_mobile"": false}" 6944,3,152,2017-04-17 09:15:07,http://walker.io/joshua_rohan,,166.68.19.112,"{""location"": ""CI"", ""is_mobile"": true}" 6945,3,152,2017-04-21 12:59:13,http://buckridge.io/eulah,,237.159.111.94,"{""location"": ""MH"", ""is_mobile"": false}" 6946,3,153,2017-06-04 19:14:24,http://welchkaulke.biz/elliot,,252.251.126.218,"{""location"": ""ML"", ""is_mobile"": false}" 12805,5,286,2017-01-03 17:16:53,http://bergstrom.net/cali.murphy,,172.114.9.108,"{""location"": ""SI"", ""is_mobile"": true}" 12806,5,286,2017-03-21 12:03:44,http://murray.net/dulce,,82.66.163.94,"{""location"": ""DZ"", ""is_mobile"": false}" 12807,5,286,2017-02-19 10:08:16,http://muller.biz/jordy,,47.227.130.25,"{""location"": ""MS"", ""is_mobile"": true}" 12808,5,286,2016-12-14 01:10:59,http://satterfieldhackett.org/ivy,,110.92.177.150,"{""location"": ""PN"", ""is_mobile"": false}" 12809,5,286,2017-01-08 18:13:27,http://koelpinruecker.name/lorena,,71.111.201.164,"{""location"": ""TW"", ""is_mobile"": true}" 12810,5,286,2017-01-20 22:59:55,http://walkermante.io/elliott,,106.242.80.105,"{""location"": ""VN"", ""is_mobile"": true}" 12811,5,286,2017-04-16 15:10:54,http://block.name/tierra,,61.20.96.210,"{""location"": ""MP"", ""is_mobile"": true}" 12812,5,286,2017-04-30 05:42:18,http://hartmannaufderhar.info/alvera.ruecker,,10.35.155.236,"{""location"": ""FM"", ""is_mobile"": false}" 12813,5,286,2017-04-06 07:59:32,http://runolfsdottir.net/brice_streich,,38.64.228.179,"{""location"": ""DZ"", ""is_mobile"": true}" 12814,5,286,2017-01-20 09:12:26,http://dickens.com/clara,,218.159.55.151,"{""location"": ""WF"", ""is_mobile"": true}" 12815,5,286,2016-12-16 07:16:22,http://von.biz/ludie_ohara,,26.209.79.94,"{""location"": ""PH"", ""is_mobile"": false}" 12816,5,286,2017-02-09 19:39:56,http://kling.biz/alexie,,190.76.86.8,"{""location"": ""HK"", ""is_mobile"": true}" 12817,5,286,2017-05-05 16:17:00,http://keler.net/thurman_runolfon,,200.191.34.127,"{""location"": ""PG"", ""is_mobile"": false}" 12818,5,286,2017-05-07 21:47:52,http://granthamill.biz/ewald,,181.201.48.72,"{""location"": ""CM"", ""is_mobile"": false}" 12819,5,286,2017-03-31 07:23:38,http://ortizruecker.io/forest.purdy,,126.220.183.132,"{""location"": ""BS"", ""is_mobile"": false}" 12820,5,286,2017-06-04 23:16:01,http://jaskolskizieme.biz/mireille_armstrong,,127.165.31.103,"{""location"": ""AZ"", ""is_mobile"": false}" 12821,5,286,2017-06-11 23:46:36,http://labadie.org/floy_dare,,221.238.153.37,"{""location"": ""AL"", ""is_mobile"": false}" 12822,5,286,2017-01-24 14:54:51,http://hills.name/carter.koelpin,,61.7.17.163,"{""location"": ""JO"", ""is_mobile"": true}" 12823,5,286,2017-05-25 12:35:42,http://hartmann.co/terrance_weber,,136.232.187.203,"{""location"": ""UM"", ""is_mobile"": true}" 12824,5,286,2017-03-16 19:55:57,http://spencer.com/hettie,,116.226.240.199,"{""location"": ""CY"", ""is_mobile"": false}" 12825,5,286,2017-04-19 13:02:40,http://bayer.info/jeffery,,201.246.224.5,"{""location"": ""EG"", ""is_mobile"": false}" 12826,5,286,2017-01-28 19:48:40,http://kingshanahan.name/josianne,,112.253.181.164,"{""location"": ""DZ"", ""is_mobile"": true}" 12827,5,286,2017-02-20 13:36:52,http://kohlermante.io/lloyd.west,,214.228.48.49,"{""location"": ""SL"", ""is_mobile"": true}" 12828,5,286,2017-05-15 21:34:21,http://abshire.com/marcelina,,58.72.47.44,"{""location"": ""LY"", ""is_mobile"": true}" 12829,5,286,2017-04-18 20:00:02,http://jerdecain.info/vicky,,227.111.149.147,"{""location"": ""CR"", ""is_mobile"": true}" 12830,5,286,2017-06-13 06:39:00,http://terry.biz/ethan,,118.61.50.152,"{""location"": ""PM"", ""is_mobile"": true}" 12831,5,286,2016-12-28 07:37:58,http://funk.com/catharine,,91.98.182.83,"{""location"": ""PT"", ""is_mobile"": true}" 12832,5,286,2017-03-31 13:43:07,http://harber.com/alex_durgan,,133.20.26.212,"{""location"": ""CH"", ""is_mobile"": false}" 12833,5,286,2017-04-04 01:42:05,http://litteljast.co/ruben,,20.189.26.160,"{""location"": ""NR"", ""is_mobile"": true}" 12834,5,286,2017-01-27 00:10:18,http://mueller.io/idell_daniel,,237.244.109.236,"{""location"": ""NE"", ""is_mobile"": true}" 12835,5,286,2017-06-08 17:35:34,http://kertzmannhand.com/amanda,,50.125.19.114,"{""location"": ""SC"", ""is_mobile"": false}" 12836,5,286,2017-05-08 05:17:53,http://wintheisercrist.org/elody,,194.232.36.216,"{""location"": ""CR"", ""is_mobile"": false}" 12837,5,286,2017-05-26 02:10:53,http://torp.info/abbigail,,234.208.236.207,"{""location"": ""ZM"", ""is_mobile"": true}" 12838,5,286,2016-12-18 21:18:58,http://wilkinson.com/berenice.hagenes,,244.26.133.105,"{""location"": ""ZA"", ""is_mobile"": true}" 12839,5,286,2017-05-01 06:30:00,http://hamill.biz/edyth.aufderhar,,169.175.161.20,"{""location"": ""VI"", ""is_mobile"": false}" 12840,5,286,2017-03-06 18:13:27,http://mcdermottskiles.biz/mariane.steuber,,56.156.192.142,"{""location"": ""VN"", ""is_mobile"": true}" 12841,5,286,2017-01-09 08:02:00,http://kreiger.org/mina,,35.163.31.79,"{""location"": ""MY"", ""is_mobile"": false}" 12842,5,286,2017-05-07 00:49:08,http://koch.co/ella,,110.143.21.230,"{""location"": ""MX"", ""is_mobile"": false}" 12843,5,286,2016-12-18 07:54:00,http://huels.biz/griffin,,237.15.242.136,"{""location"": ""YT"", ""is_mobile"": true}" 12844,5,286,2017-02-11 13:10:57,http://herzog.biz/bell,,100.192.115.44,"{""location"": ""LU"", ""is_mobile"": true}" 12845,5,287,2017-02-24 16:59:19,http://zemlak.name/victoria,,44.62.2.232,"{""location"": ""JP"", ""is_mobile"": false}" 12846,5,287,2017-06-01 01:43:04,http://jerdezemlak.info/lacy,,62.82.128.55,"{""location"": ""VC"", ""is_mobile"": true}" 12847,5,287,2017-01-18 09:07:19,http://bauchrolfson.net/benjamin.quigley,,223.202.227.95,"{""location"": ""BY"", ""is_mobile"": true}" 12848,5,287,2017-05-21 01:47:02,http://turner.co/amani_johnston,,50.130.174.129,"{""location"": ""CG"", ""is_mobile"": false}" 12849,5,287,2017-01-21 15:19:25,http://ondricka.biz/berry.lind,,126.242.159.40,"{""location"": ""DO"", ""is_mobile"": true}" 12850,5,287,2017-04-18 03:07:07,http://schuster.org/queenie.barton,,188.245.250.54,"{""location"": ""BV"", ""is_mobile"": true}" 12851,5,287,2017-05-10 17:36:10,http://gerlachstanton.io/jermain.kerluke,,253.95.51.108,"{""location"": ""EG"", ""is_mobile"": false}" 12852,5,287,2017-05-05 22:36:23,http://cristtillman.net/gideon_nikolaus,,234.222.163.101,"{""location"": ""GT"", ""is_mobile"": true}" 12853,5,287,2017-02-17 18:29:20,http://watsica.biz/hilma_wolf,,101.149.200.115,"{""location"": ""MT"", ""is_mobile"": false}" 12854,5,287,2017-03-13 02:23:34,http://osinskiparisian.co/gunnar,,229.84.243.10,"{""location"": ""NR"", ""is_mobile"": false}" 12855,5,287,2017-01-01 00:47:55,http://lehner.biz/august,,195.43.167.177,"{""location"": ""PT"", ""is_mobile"": true}" 12856,5,287,2017-01-22 11:49:32,http://fahey.net/ophelia.schneider,,197.84.253.105,"{""location"": ""VG"", ""is_mobile"": true}" 12857,5,287,2017-06-12 16:59:11,http://runolfon.org/alyson,,65.138.229.208,"{""location"": ""KE"", ""is_mobile"": true}" 12858,5,287,2017-06-06 04:17:40,http://heathcotejohnston.co/amelia,,210.85.165.70,"{""location"": ""BG"", ""is_mobile"": true}" 12859,5,287,2016-12-23 18:49:51,http://spinka.biz/donna,,19.52.66.25,"{""location"": ""CG"", ""is_mobile"": false}" 15806,6,355,2016-12-25 04:08:29,http://mosciski.biz/ilene_rice,,208.129.24.189,"{""location"": ""MH"", ""is_mobile"": false}" 15807,6,355,2017-03-13 23:08:26,http://bodestracke.net/nora,,216.44.115.200,"{""location"": ""AM"", ""is_mobile"": false}" 15808,6,356,2017-02-28 09:16:53,http://welch.info/jillian,,110.168.68.218,"{""location"": ""JO"", ""is_mobile"": true}" 15809,6,356,2017-05-20 17:46:06,http://rolfsonstamm.co/abdullah_bosco,,133.117.55.153,"{""location"": ""TL"", ""is_mobile"": false}" 15810,6,356,2016-12-28 03:29:13,http://aufderhar.org/dashawn,,22.89.182.252,"{""location"": ""EG"", ""is_mobile"": true}" 15811,6,356,2017-04-06 07:43:03,http://gleichnerko.co/jamel.hills,,29.31.227.37,"{""location"": ""BS"", ""is_mobile"": true}" 15812,6,356,2017-02-20 17:55:44,http://toylegros.name/max.wuckert,,213.105.81.147,"{""location"": ""LK"", ""is_mobile"": true}" 15813,6,356,2017-03-27 23:20:23,http://douglas.org/casandra,,187.137.116.109,"{""location"": ""CI"", ""is_mobile"": false}" 15814,6,356,2017-06-09 11:39:03,http://leuschke.co/yeenia_hyatt,,112.7.174.40,"{""location"": ""BA"", ""is_mobile"": true}" 15815,6,356,2017-02-17 02:52:12,http://kunde.org/israel.lindgren,,216.26.118.77,"{""location"": ""PK"", ""is_mobile"": true}" 15816,6,356,2017-03-11 04:40:09,http://murazik.org/lindsay,,23.154.251.251,"{""location"": ""ET"", ""is_mobile"": true}" 15817,6,356,2017-03-29 23:32:47,http://hagenes.co/eulah,,141.34.67.178,"{""location"": ""LR"", ""is_mobile"": true}" 15818,6,356,2017-01-08 19:12:52,http://nader.io/robin.hagenes,,47.249.196.93,"{""location"": ""GI"", ""is_mobile"": true}" 15819,6,356,2017-03-30 05:25:17,http://hand.name/deon,,166.136.132.185,"{""location"": ""GW"", ""is_mobile"": false}" 15820,6,356,2017-04-07 04:22:06,http://bartoletti.biz/grayce_mante,,110.45.241.229,"{""location"": ""CU"", ""is_mobile"": true}" 15821,6,356,2017-05-22 09:06:36,http://konopelskiframi.biz/liliana,,220.98.2.146,"{""location"": ""TO"", ""is_mobile"": false}" 15822,6,356,2017-05-21 08:27:03,http://murphy.org/lucious,,31.248.90.149,"{""location"": ""NE"", ""is_mobile"": true}" 15823,6,356,2017-05-06 17:29:55,http://krishand.info/kaycee_becker,,26.2.62.204,"{""location"": ""SH"", ""is_mobile"": false}" 15824,6,356,2017-04-03 02:44:55,http://blick.org/geovanny_borer,,141.17.108.66,"{""location"": ""EG"", ""is_mobile"": true}" 15825,6,356,2017-03-08 08:15:37,http://mayertgrimes.net/melia.altenwerth,,249.47.158.185,"{""location"": ""VE"", ""is_mobile"": true}" 15826,6,356,2017-01-11 22:40:41,http://heathcotehowell.biz/terry_wiza,,122.124.179.166,"{""location"": ""TZ"", ""is_mobile"": false}" 15827,6,356,2017-03-23 19:54:40,http://abernathypagac.info/maximillian,,8.132.205.204,"{""location"": ""WS"", ""is_mobile"": true}" 15828,6,356,2017-04-24 23:48:58,http://lowe.io/jimmy_walter,,78.100.18.96,"{""location"": ""BH"", ""is_mobile"": false}" 15829,6,356,2017-01-09 16:44:39,http://boganhammes.co/khalil,,98.191.172.34,"{""location"": ""EH"", ""is_mobile"": false}" 15830,6,356,2017-02-09 00:30:09,http://kohler.co/candace.hills,,233.188.146.34,"{""location"": ""MF"", ""is_mobile"": false}" 15831,6,356,2017-01-14 15:44:56,http://lebsack.org/moises_schulist,,103.40.103.158,"{""location"": ""BR"", ""is_mobile"": true}" 15832,6,356,2017-03-24 15:50:42,http://schoen.info/jarrell,,77.176.218.106,"{""location"": ""MY"", ""is_mobile"": true}" 15833,6,356,2016-12-30 02:36:09,http://lowe.biz/abdullah,,234.154.210.211,"{""location"": ""RS"", ""is_mobile"": true}" 15834,6,356,2016-12-20 16:06:03,http://terry.biz/amy,,75.211.88.47,"{""location"": ""GT"", ""is_mobile"": false}" 15835,6,356,2017-05-11 07:23:12,http://schustermante.com/jovan_fisher,,178.210.221.181,"{""location"": ""PY"", ""is_mobile"": true}" 15836,6,356,2016-12-17 02:29:18,http://vandervort.net/stanton,,64.31.36.126,"{""location"": ""IE"", ""is_mobile"": false}" 15837,6,356,2017-05-25 13:41:43,http://terry.biz/benjamin_mccullough,,54.83.19.126,"{""location"": ""DJ"", ""is_mobile"": true}" 15838,6,356,2017-04-03 02:27:03,http://miller.co/chadrick,,248.157.100.104,"{""location"": ""RU"", ""is_mobile"": false}" 15839,6,356,2017-05-06 11:04:20,http://gloverortiz.net/alysha_harris,,178.70.112.50,"{""location"": ""TC"", ""is_mobile"": true}" 15840,6,356,2017-02-01 23:49:16,http://flatley.com/felipe.crooks,,75.160.34.182,"{""location"": ""FO"", ""is_mobile"": false}" 15841,6,356,2017-03-28 16:40:04,http://purdyblock.biz/lula_gaylord,,44.43.47.123,"{""location"": ""KP"", ""is_mobile"": false}" 15842,6,356,2017-02-22 21:01:56,http://kihn.co/joanne,,113.157.142.16,"{""location"": ""ZW"", ""is_mobile"": false}" 15843,6,356,2017-06-02 21:04:29,http://prohaska.org/sedrick_cain,,71.74.246.192,"{""location"": ""MM"", ""is_mobile"": false}" 15844,6,356,2017-04-29 19:28:22,http://roobschinner.net/gerson.schaden,,254.217.247.187,"{""location"": ""GE"", ""is_mobile"": false}" 15845,6,356,2017-05-04 07:21:35,http://wintheiser.name/francisco,,201.164.148.133,"{""location"": ""TH"", ""is_mobile"": false}" 15846,6,356,2017-04-12 23:33:39,http://parker.name/forest_feil,,151.109.253.110,"{""location"": ""AL"", ""is_mobile"": true}" 15847,6,356,2017-02-06 19:31:17,http://prosaccoaufderhar.com/elza.effertz,,225.25.158.87,"{""location"": ""SB"", ""is_mobile"": true}" 15848,6,356,2017-01-23 07:58:32,http://emmerichdaugherty.com/janelle,,19.129.230.152,"{""location"": ""BZ"", ""is_mobile"": false}" 15849,6,356,2017-05-29 03:32:05,http://ernserbosco.biz/filomena,,66.199.32.228,"{""location"": ""TD"", ""is_mobile"": false}" 15850,6,356,2017-03-10 13:28:45,http://beatty.net/willa,,176.150.106.231,"{""location"": ""TF"", ""is_mobile"": true}" 15851,6,356,2017-02-20 12:31:30,http://larson.net/corine_grant,,150.13.195.66,"{""location"": ""CZ"", ""is_mobile"": false}" 15852,6,356,2017-02-25 15:58:30,http://lueilwitz.io/ryan,,27.190.206.215,"{""location"": ""ZW"", ""is_mobile"": false}" 15853,6,356,2017-04-03 20:38:57,http://cummings.net/oma,,138.150.187.129,"{""location"": ""FM"", ""is_mobile"": false}" 15854,6,356,2016-12-29 01:54:02,http://wolff.org/silas,,50.207.103.206,"{""location"": ""BE"", ""is_mobile"": false}" 15855,6,356,2017-01-22 07:02:58,http://quigley.org/selina,,42.126.110.89,"{""location"": ""ZW"", ""is_mobile"": true}" 15856,6,356,2017-03-30 02:38:59,http://oberbrunner.co/natalia,,162.254.41.35,"{""location"": ""AQ"", ""is_mobile"": true}" 15857,6,356,2017-02-05 20:27:36,http://monahanmurphy.biz/dominique_bartell,,218.87.200.114,"{""location"": ""SE"", ""is_mobile"": true}" 15858,6,356,2017-01-07 14:36:49,http://carroll.io/hailie_cronin,,210.103.94.162,"{""location"": ""UG"", ""is_mobile"": true}" 15859,6,356,2017-02-18 20:46:45,http://erdmanrau.name/alvina,,63.178.80.51,"{""location"": ""AQ"", ""is_mobile"": false}" 15860,6,356,2016-12-26 00:35:29,http://haleyhilll.co/jada.veum,,237.119.140.87,"{""location"": ""TL"", ""is_mobile"": true}" 3008,2,66,2017-03-30 12:03:09,http://sporerhickle.net/america.dare,0.3283648343,226.74.219.121,"{""location"": ""BO"", ""is_mobile"": false}" 3009,2,66,2017-06-03 22:00:53,http://macgyver.co/frederick_kuvalis,0.9103569328,176.134.139.198,"{""location"": ""KR"", ""is_mobile"": true}" 3010,2,66,2017-03-25 13:20:31,http://bogan.biz/aric,0.4612352990,117.96.213.96,"{""location"": ""EH"", ""is_mobile"": false}" 3011,2,66,2017-04-25 04:56:04,http://pagac.name/adella,0.5499951986,214.209.201.142,"{""location"": ""CX"", ""is_mobile"": true}" 3012,2,66,2017-03-22 10:57:58,http://upton.org/lysanne,0.4207174868,249.165.157.59,"{""location"": ""MC"", ""is_mobile"": true}" 3013,2,66,2017-02-05 13:55:34,http://orn.co/myrtle,0.2625692102,118.116.21.133,"{""location"": ""HK"", ""is_mobile"": false}" 3014,2,66,2017-05-18 22:38:22,http://bernier.co/piper_effertz,0.4949108400,144.61.245.183,"{""location"": ""NR"", ""is_mobile"": true}" 3015,2,66,2017-02-07 01:06:16,http://wardbins.com/reilly,0.8417543620,135.45.156.144,"{""location"": ""TV"", ""is_mobile"": false}" 3016,2,66,2017-06-06 13:16:40,http://framidaniel.info/kraig,0.4584310559,200.238.173.8,"{""location"": ""HK"", ""is_mobile"": true}" 3017,2,66,2017-01-23 22:07:45,http://windler.info/elza,0.9858953043,230.39.63.31,"{""location"": ""CW"", ""is_mobile"": false}" 3018,2,66,2017-04-03 17:55:28,http://erdman.name/alex.gutkowski,0.1428229556,57.253.194.30,"{""location"": ""AS"", ""is_mobile"": true}" 3019,2,66,2017-01-02 22:48:13,http://hintz.com/estel_bogisich,0.9664657952,156.246.37.181,"{""location"": ""GG"", ""is_mobile"": false}" 3020,2,66,2017-05-18 15:37:57,http://stantonpacocha.net/carlotta,0.7941341497,229.245.85.80,"{""location"": ""FJ"", ""is_mobile"": false}" 3021,2,66,2017-03-16 03:48:26,http://mannbrakus.biz/otto,0.5628489351,81.148.186.70,"{""location"": ""YE"", ""is_mobile"": true}" 3022,2,66,2017-01-15 09:10:34,http://berge.io/margaretta.rodriguez,0.7975393241,236.63.140.167,"{""location"": ""UY"", ""is_mobile"": true}" 3023,2,66,2017-03-23 14:41:44,http://erdmankreiger.org/thea_graham,0.0511527373,197.21.220.182,"{""location"": ""AU"", ""is_mobile"": false}" 3024,2,66,2017-03-09 12:42:59,http://lindgreenfelder.io/graham.steuber,0.5863194511,37.29.217.224,"{""location"": ""SI"", ""is_mobile"": false}" 3025,2,66,2017-01-28 22:05:34,http://fisher.co/armand_lebsack,0.6187797718,206.166.82.132,"{""location"": ""CO"", ""is_mobile"": true}" 3026,2,66,2017-04-26 01:47:14,http://prosaccoherzog.info/wava.roberts,0.1244857225,219.99.30.4,"{""location"": ""MQ"", ""is_mobile"": true}" 3027,2,66,2017-05-21 15:41:34,http://bosco.biz/abner,0.3120541704,249.212.95.250,"{""location"": ""SS"", ""is_mobile"": false}" 3028,2,66,2017-02-18 09:06:51,http://anderson.info/celestine.marquardt,0.3742896222,68.116.83.120,"{""location"": ""ME"", ""is_mobile"": true}" 3029,2,66,2017-03-30 11:12:47,http://zemlakmarks.info/catharine,0.3660466952,68.83.220.89,"{""location"": ""BB"", ""is_mobile"": false}" 3030,2,66,2017-03-11 18:03:26,http://skiles.com/kaleb,0.1424076583,230.143.84.11,"{""location"": ""TZ"", ""is_mobile"": false}" 3031,2,66,2017-03-25 16:20:28,http://dibbert.biz/alfonso.gutkowski,0.5790093665,178.230.246.143,"{""location"": ""TM"", ""is_mobile"": true}" 3032,2,67,2016-12-26 16:11:08,http://hartmann.co/keshaun,0.3731311886,8.122.124.190,"{""location"": ""BE"", ""is_mobile"": false}" 3033,2,67,2016-12-21 02:06:24,http://rippingerlach.co/ned.huel,0.5341077110,102.45.227.30,"{""location"": ""SE"", ""is_mobile"": true}" 3034,2,67,2016-12-24 22:11:16,http://lowebeatty.org/frida,0.8777716334,176.234.178.137,"{""location"": ""CZ"", ""is_mobile"": true}" 3035,2,67,2017-02-17 03:55:22,http://raynorjenkins.name/gladys_braun,0.0803604485,156.196.9.19,"{""location"": ""MR"", ""is_mobile"": false}" 3036,2,67,2017-03-20 21:33:37,http://feeney.io/edmond_cummerata,0.9128735293,252.63.65.230,"{""location"": ""MG"", ""is_mobile"": true}" 3037,2,67,2016-12-16 13:46:46,http://kreiger.name/pinkie,0.4912933073,183.77.25.246,"{""location"": ""AX"", ""is_mobile"": true}" 3038,2,67,2017-01-14 11:09:35,http://kemmer.org/jakob_tremblay,0.3391633804,107.91.99.119,"{""location"": ""BV"", ""is_mobile"": true}" 3039,2,67,2017-04-21 05:00:33,http://schmeler.org/odea,0.0857790696,177.15.254.124,"{""location"": ""CK"", ""is_mobile"": true}" 3040,2,67,2017-06-07 16:37:16,http://haley.biz/lorenzo,0.7017768962,82.5.123.131,"{""location"": ""AM"", ""is_mobile"": true}" 3041,2,67,2017-02-21 23:14:16,http://douglaskris.io/ashleigh,0.8891327685,246.88.81.74,"{""location"": ""BE"", ""is_mobile"": true}" 3042,2,67,2017-01-04 10:01:03,http://mckenziesipes.info/roberta.hermann,0.7302447964,54.73.235.92,"{""location"": ""UZ"", ""is_mobile"": false}" 3043,2,67,2017-01-03 11:04:04,http://harber.name/ryleigh,0.3016545887,159.52.28.219,"{""location"": ""KH"", ""is_mobile"": false}" 3044,2,67,2017-03-08 20:07:48,http://schuppe.net/susan_durgan,0.8988463304,244.87.97.239,"{""location"": ""PL"", ""is_mobile"": true}" 3045,2,67,2017-04-09 05:07:06,http://weinat.io/kayla,0.9491274007,173.236.51.134,"{""location"": ""MM"", ""is_mobile"": true}" 3046,2,67,2017-05-23 07:07:50,http://quitzonkunde.co/glenna_dietrich,0.0074697193,94.152.80.191,"{""location"": ""DO"", ""is_mobile"": false}" 3047,2,67,2017-03-07 11:32:02,http://cormierprice.name/boris,0.8103363758,161.72.201.87,"{""location"": ""GH"", ""is_mobile"": false}" 3048,2,67,2016-12-24 17:15:38,http://fay.biz/wade,0.6269527414,159.25.67.90,"{""location"": ""VI"", ""is_mobile"": true}" 3049,2,67,2017-06-03 12:52:34,http://walter.name/gerda_predovic,0.1856226982,181.65.210.206,"{""location"": ""LY"", ""is_mobile"": false}" 3050,2,67,2017-05-05 21:07:19,http://considine.io/kamron.spinka,0.7511434170,10.83.203.13,"{""location"": ""AF"", ""is_mobile"": true}" 3051,2,67,2017-01-14 21:25:26,http://ruel.co/micaela_mccullough,0.9971675376,221.92.206.5,"{""location"": ""OM"", ""is_mobile"": false}" 3052,2,67,2017-01-29 03:16:34,http://kirlincorwin.net/halie.hegmann,0.6613248013,194.230.14.145,"{""location"": ""JP"", ""is_mobile"": true}" 3053,2,67,2017-04-13 18:32:27,http://trantow.com/ahmad.wehner,0.6868047757,100.53.13.67,"{""location"": ""AM"", ""is_mobile"": false}" 3054,2,67,2017-03-03 02:19:38,http://jenkinsmacejkovic.info/leon,0.3702052801,110.115.161.22,"{""location"": ""JM"", ""is_mobile"": false}" 3055,2,67,2017-01-28 13:17:35,http://dibbert.net/eve.legros,0.4927245752,94.182.201.101,"{""location"": ""VI"", ""is_mobile"": true}" 3056,2,67,2017-05-21 16:57:28,http://boscocrona.co/eli_aufderhar,0.8121811029,44.16.202.177,"{""location"": ""ST"", ""is_mobile"": true}" 3057,2,67,2017-05-15 04:26:29,http://thompson.info/rachael.rodriguez,0.2974642401,114.76.98.56,"{""location"": ""NO"", ""is_mobile"": true}" 3058,2,67,2017-05-16 23:14:35,http://gulgowski.info/mafalda,0.2550798173,246.111.228.247,"{""location"": ""IN"", ""is_mobile"": true}" 3059,2,67,2017-05-20 03:01:37,http://daugherty.info/christop,0.5640715361,245.110.26.134,"{""location"": ""TT"", ""is_mobile"": true}" 9945,4,223,2016-12-28 17:56:33,http://bartonluettgen.net/linwood_erdman,0.6676393242,110.103.42.180,"{""location"": ""CL"", ""is_mobile"": true}" 9946,4,223,2017-05-13 16:19:06,http://vonruedenoberbrunner.org/domingo,0.0312599221,143.90.107.160,"{""location"": ""DM"", ""is_mobile"": true}" 9947,4,223,2017-04-16 10:02:57,http://kemmer.biz/wyatt,0.9541460641,60.144.64.198,"{""location"": ""DM"", ""is_mobile"": true}" 9948,4,223,2017-03-29 01:55:50,http://hegmann.net/lou.kuphal,0.4039689280,151.242.30.163,"{""location"": ""AG"", ""is_mobile"": false}" 9949,4,223,2017-03-04 09:43:30,http://feeneycarroll.org/murphy.kutch,0.9157664444,189.238.147.184,"{""location"": ""KY"", ""is_mobile"": false}" 9950,4,223,2017-01-29 17:17:23,http://okuneva.net/margret.lubowitz,0.6254031044,244.70.156.109,"{""location"": ""IR"", ""is_mobile"": true}" 9951,4,223,2016-12-23 21:10:39,http://nikolaus.org/lia,0.6835859605,91.225.166.203,"{""location"": ""GM"", ""is_mobile"": true}" 9952,4,223,2017-02-10 17:40:40,http://smithweinat.co/tina_gutkowski,0.5343808740,8.195.157.154,"{""location"": ""AG"", ""is_mobile"": true}" 9953,4,223,2017-01-09 22:01:26,http://wilderman.io/queen,0.4040836962,238.90.3.214,"{""location"": ""BY"", ""is_mobile"": false}" 9954,4,223,2017-01-11 20:11:58,http://treutelgoodwin.net/cordie,0.2247655053,55.124.23.219,"{""location"": ""BI"", ""is_mobile"": false}" 9955,4,223,2017-02-25 13:54:34,http://schinnerwest.io/tamara,0.4195395606,147.114.192.59,"{""location"": ""VI"", ""is_mobile"": true}" 9956,4,223,2016-12-23 20:41:02,http://zulauf.com/brenna_goldner,0.3522003279,209.29.221.107,"{""location"": ""BG"", ""is_mobile"": false}" 9957,4,223,2017-02-15 04:01:03,http://quigleykonopelski.co/caandre,0.2052398580,226.220.92.246,"{""location"": ""DK"", ""is_mobile"": false}" 9958,4,223,2017-05-11 16:39:58,http://stokes.org/keagan_howe,0.4636249267,144.229.236.137,"{""location"": ""SN"", ""is_mobile"": true}" 9959,4,223,2017-04-28 15:18:19,http://rosenbaum.info/bonita,0.1770827803,178.154.92.128,"{""location"": ""CA"", ""is_mobile"": false}" 9960,4,223,2017-04-12 01:19:30,http://bayer.co/imogene,0.3849046257,142.216.51.240,"{""location"": ""BH"", ""is_mobile"": false}" 9961,4,223,2016-12-22 22:21:20,http://breitenbergroberts.org/justina.stokes,0.8753023837,67.85.40.13,"{""location"": ""PG"", ""is_mobile"": false}" 9962,4,223,2017-06-07 23:16:29,http://berge.info/tierra_gleichner,0.4455484062,211.7.132.108,"{""location"": ""CM"", ""is_mobile"": true}" 9963,4,223,2017-03-09 15:29:17,http://bernier.biz/shyann_mccullough,0.0260969728,28.165.40.76,"{""location"": ""SV"", ""is_mobile"": false}" 9964,4,223,2017-01-31 11:10:31,http://corkery.com/sigmund_jerde,0.0856309933,20.121.112.186,"{""location"": ""SI"", ""is_mobile"": false}" 9965,4,223,2017-01-27 09:53:49,http://millersawayn.name/kayden,0.6371018193,195.40.206.61,"{""location"": ""KR"", ""is_mobile"": false}" 9966,4,223,2017-05-03 12:06:18,http://hickle.com/monty_schumm,0.2485320767,5.121.60.192,"{""location"": ""TK"", ""is_mobile"": true}" 9967,4,223,2017-02-14 05:24:12,http://ebert.net/tyra_runte,0.0960862540,113.16.230.252,"{""location"": ""KM"", ""is_mobile"": false}" 9968,4,223,2017-04-01 12:33:05,http://bednar.biz/eloy.gleason,0.9805692589,196.60.176.22,"{""location"": ""CI"", ""is_mobile"": true}" 9969,4,223,2017-05-05 12:54:23,http://bruen.net/sidney,0.9282815899,115.239.158.223,"{""location"": ""TF"", ""is_mobile"": false}" 9970,4,223,2017-04-07 10:47:16,http://rath.org/emie,0.9929037398,187.113.14.41,"{""location"": ""TF"", ""is_mobile"": false}" 9971,4,223,2017-03-25 11:10:14,http://krajcik.info/aiyana,0.9301638389,180.179.119.203,"{""location"": ""IM"", ""is_mobile"": true}" 9972,4,223,2017-01-19 12:32:38,http://farrell.io/linda,0.5165077297,97.229.172.252,"{""location"": ""LT"", ""is_mobile"": false}" 9973,4,223,2017-02-24 23:14:06,http://fay.com/ellen,0.2160714421,102.157.85.223,"{""location"": ""ZM"", ""is_mobile"": true}" 9974,4,223,2017-01-26 10:15:19,http://beatty.info/nova.labadie,0.4601833123,169.112.208.218,"{""location"": ""LS"", ""is_mobile"": false}" 9975,4,223,2017-05-20 17:07:33,http://upton.com/stephen,0.7412656474,71.153.111.220,"{""location"": ""BH"", ""is_mobile"": false}" 9976,4,223,2017-02-28 02:54:07,http://okunevacarroll.name/damaris,0.5931668933,85.227.146.164,"{""location"": ""GU"", ""is_mobile"": false}" 9977,4,223,2017-05-22 13:48:48,http://bashirian.name/drew,0.3627805645,21.81.210.142,"{""location"": ""GU"", ""is_mobile"": true}" 9978,4,223,2017-01-26 10:37:46,http://braunjenkins.org/morton.thiel,0.3657094415,20.23.137.3,"{""location"": ""CG"", ""is_mobile"": true}" 9979,4,223,2017-05-06 15:30:51,http://green.net/andre,0.8152582598,80.84.50.239,"{""location"": ""CM"", ""is_mobile"": true}" 9980,4,223,2017-05-05 17:25:25,http://weimann.io/juliana.blanda,0.2129642866,13.68.242.188,"{""location"": ""NE"", ""is_mobile"": true}" 9981,4,223,2017-04-02 19:37:02,http://ritchie.io/matt.ferry,0.2135305360,170.167.181.24,"{""location"": ""PL"", ""is_mobile"": true}" 9982,4,223,2017-01-09 03:23:44,http://altenwerthgulgowski.info/ervin,0.5326790224,38.26.105.108,"{""location"": ""UZ"", ""is_mobile"": true}" 9983,4,223,2017-06-10 17:20:40,http://nienow.org/mallie,0.4930635926,48.188.204.40,"{""location"": ""NU"", ""is_mobile"": true}" 9984,4,223,2017-05-20 00:57:06,http://kaulke.net/jamel,0.4598325005,204.40.52.225,"{""location"": ""PK"", ""is_mobile"": true}" 9985,4,223,2017-02-10 05:08:36,http://kuhlman.com/gerardo.rippin,0.0906940136,149.81.223.211,"{""location"": ""GM"", ""is_mobile"": false}" 9986,4,223,2017-05-30 13:34:51,http://oberbrunnerveum.biz/rosetta,0.3472847485,98.124.58.230,"{""location"": ""CM"", ""is_mobile"": true}" 9987,4,223,2017-06-09 08:56:59,http://hettingerschowalter.name/sabina.bode,0.4082582901,117.190.179.76,"{""location"": ""GB"", ""is_mobile"": false}" 9988,4,223,2017-04-30 19:25:37,http://kuvalismaggio.com/river,0.0139497046,124.174.246.38,"{""location"": ""GM"", ""is_mobile"": false}" 9989,4,223,2016-12-23 04:25:38,http://champlin.info/clarabelle,0.9773780236,158.121.13.9,"{""location"": ""BR"", ""is_mobile"": false}" 9990,4,223,2017-03-01 20:00:41,http://cruickshankjaskolski.com/otilia.kris,0.0025326056,118.168.135.138,"{""location"": ""BA"", ""is_mobile"": false}" 9991,4,223,2017-05-13 20:08:11,http://walterlegros.io/valentin_lueilwitz,0.2092987679,249.194.244.99,"{""location"": ""ZM"", ""is_mobile"": true}" 9992,4,223,2017-04-30 11:28:59,http://rosenbaum.name/queen,0.7928851507,60.72.7.71,"{""location"": ""MG"", ""is_mobile"": true}" 9993,4,223,2017-05-12 15:47:21,http://pouros.biz/adolph.lang,0.4385363771,127.21.233.84,"{""location"": ""US"", ""is_mobile"": true}" 9994,4,223,2017-01-25 02:48:19,http://kovacek.info/cheyenne_hermann,0.1651114554,54.152.97.206,"{""location"": ""AG"", ""is_mobile"": true}" 9995,4,223,2017-01-25 01:39:45,http://mcdermott.info/breanne_bergnaum,0.3033566916,172.39.200.225,"{""location"": ""TH"", ""is_mobile"": false}" 6947,3,153,2017-04-15 09:49:04,http://kuhn.com/reyes,,221.195.49.112,"{""location"": ""AR"", ""is_mobile"": false}" 6948,3,153,2017-02-03 07:01:23,http://predovic.name/percival,,231.75.84.206,"{""location"": ""LK"", ""is_mobile"": true}" 6949,3,153,2017-05-27 23:30:47,http://bayer.io/abagail.heidenreich,,166.169.160.232,"{""location"": ""SB"", ""is_mobile"": false}" 6950,3,153,2017-01-08 14:48:30,http://sipes.com/samanta,,6.71.193.159,"{""location"": ""EC"", ""is_mobile"": true}" 6951,3,153,2017-03-06 00:55:55,http://shieldsernser.org/ken.hauck,,24.85.73.123,"{""location"": ""BA"", ""is_mobile"": false}" 6952,3,153,2017-02-27 16:30:01,http://zulauf.io/rhianna.hoppe,,12.215.33.155,"{""location"": ""DE"", ""is_mobile"": false}" 6953,3,153,2017-05-24 10:54:52,http://quitzonquigley.info/ayden,,158.60.130.137,"{""location"": ""SZ"", ""is_mobile"": false}" 6954,3,153,2017-01-10 14:08:20,http://boscokoch.net/leopold.smitham,,77.185.225.192,"{""location"": ""VE"", ""is_mobile"": true}" 6955,3,153,2017-06-02 03:51:54,http://jonesmiller.net/armando.hettinger,,254.36.84.241,"{""location"": ""KZ"", ""is_mobile"": true}" 6956,3,153,2017-02-17 14:58:53,http://jenkins.name/kirsten.cremin,,171.20.170.28,"{""location"": ""CI"", ""is_mobile"": true}" 6957,3,153,2017-01-27 14:42:49,http://brekkesawayn.biz/jarod,,220.231.17.3,"{""location"": ""ID"", ""is_mobile"": false}" 6958,3,153,2017-04-03 13:05:35,http://block.biz/icie_denesik,,128.77.34.65,"{""location"": ""AO"", ""is_mobile"": true}" 6959,3,153,2017-01-17 20:25:24,http://turcotteschumm.com/jarrell.ko,,232.217.36.48,"{""location"": ""SY"", ""is_mobile"": false}" 6960,3,153,2016-12-24 14:05:43,http://mitchell.info/gust,,117.39.198.118,"{""location"": ""AD"", ""is_mobile"": true}" 6961,3,153,2017-03-31 07:29:22,http://metz.biz/tyrel,,250.33.166.196,"{""location"": ""CD"", ""is_mobile"": false}" 6962,3,153,2017-03-31 13:33:59,http://mann.name/delia_deckow,,108.139.19.191,"{""location"": ""MS"", ""is_mobile"": true}" 6963,3,153,2017-02-18 19:52:55,http://watersmckenzie.com/selena_rath,,211.118.64.157,"{""location"": ""PH"", ""is_mobile"": true}" 6964,3,153,2017-01-21 15:41:54,http://cummeratabauch.info/monique.harber,,149.223.177.198,"{""location"": ""MS"", ""is_mobile"": true}" 6965,3,153,2017-04-13 02:50:37,http://smitham.com/elia,,13.30.186.101,"{""location"": ""HN"", ""is_mobile"": true}" 6966,3,153,2017-06-06 09:19:27,http://bartoletti.biz/haan.zieme,,143.201.146.131,"{""location"": ""BI"", ""is_mobile"": true}" 6967,3,153,2017-03-30 07:19:41,http://runolfonbailey.info/sammy.terry,,34.252.231.220,"{""location"": ""CI"", ""is_mobile"": true}" 6968,3,153,2016-12-13 06:33:03,http://quitzon.info/kenny.kiehn,,44.88.248.239,"{""location"": ""FI"", ""is_mobile"": true}" 6969,3,153,2016-12-23 18:32:28,http://heaney.org/elijah,,137.246.95.23,"{""location"": ""SO"", ""is_mobile"": true}" 6970,3,153,2017-01-18 20:08:30,http://osinski.biz/tremayne_cartwright,,153.171.131.152,"{""location"": ""GG"", ""is_mobile"": true}" 6971,3,153,2016-12-26 10:33:28,http://heathcotetrantow.biz/krystal,,110.225.147.177,"{""location"": ""LR"", ""is_mobile"": true}" 6972,3,153,2017-04-15 19:21:20,http://wymanmcdermott.co/juliana,,242.77.227.3,"{""location"": ""PA"", ""is_mobile"": true}" 6973,3,153,2017-03-06 07:59:36,http://eichmann.info/rodrigo_turcotte,,4.27.69.43,"{""location"": ""KN"", ""is_mobile"": false}" 6974,3,153,2017-04-13 22:27:37,http://walshupton.co/gonzalo,,189.250.82.2,"{""location"": ""BA"", ""is_mobile"": false}" 6975,3,153,2017-04-11 23:40:53,http://hirthehuels.biz/floy.wyman,,6.202.48.23,"{""location"": ""SR"", ""is_mobile"": false}" 6976,3,153,2016-12-20 05:35:31,http://veum.name/glenna,,145.192.26.181,"{""location"": ""HN"", ""is_mobile"": false}" 6977,3,153,2017-03-23 09:50:26,http://jacobson.org/ryleigh,,252.65.60.195,"{""location"": ""OM"", ""is_mobile"": true}" 6978,3,153,2017-06-12 07:43:24,http://schmidt.com/danika.towne,,55.52.20.216,"{""location"": ""PR"", ""is_mobile"": false}" 6979,3,153,2017-02-22 00:36:37,http://rathmills.info/ruthie.fritsch,,105.57.42.183,"{""location"": ""KP"", ""is_mobile"": false}" 6980,3,153,2017-05-27 09:39:09,http://anderson.io/norberto,,44.251.223.79,"{""location"": ""CX"", ""is_mobile"": true}" 6981,3,154,2017-05-10 06:00:30,http://hilpertthompson.co/kayla.cremin,,182.5.45.92,"{""location"": ""HT"", ""is_mobile"": true}" 6982,3,154,2017-02-18 14:51:09,http://wizahowe.net/kathryn.durgan,,16.44.105.10,"{""location"": ""BF"", ""is_mobile"": true}" 6983,3,154,2017-02-07 05:28:44,http://schoenrenner.co/felipe_thompson,,51.103.233.65,"{""location"": ""LV"", ""is_mobile"": true}" 6984,3,154,2017-01-26 14:37:23,http://carterbahringer.co/kody.beier,,134.112.214.202,"{""location"": ""TT"", ""is_mobile"": true}" 6985,3,154,2017-05-30 19:42:29,http://zulauf.org/felipe.murray,,11.61.181.144,"{""location"": ""GB"", ""is_mobile"": true}" 6986,3,154,2017-02-15 00:06:52,http://predovic.net/roie,,206.243.44.99,"{""location"": ""VA"", ""is_mobile"": false}" 6987,3,154,2017-04-22 00:54:14,http://stammkuphal.biz/ken,,120.210.36.75,"{""location"": ""KP"", ""is_mobile"": true}" 6988,3,154,2017-01-27 00:51:21,http://bashirian.io/gonzalo_gibson,,104.171.134.99,"{""location"": ""MS"", ""is_mobile"": false}" 6989,3,154,2017-05-04 15:56:58,http://tremblay.co/litzy,,55.70.151.144,"{""location"": ""FM"", ""is_mobile"": false}" 6990,3,154,2017-01-18 16:51:04,http://ziemann.net/cameron,,96.14.38.239,"{""location"": ""BL"", ""is_mobile"": true}" 6991,3,154,2017-01-29 13:01:58,http://walker.biz/lucienne,,201.162.172.211,"{""location"": ""CL"", ""is_mobile"": false}" 6992,3,154,2017-03-15 09:38:33,http://bahringer.io/marcella,,238.132.183.2,"{""location"": ""TN"", ""is_mobile"": true}" 6993,3,154,2017-03-23 20:54:58,http://dubuque.info/alisa,,6.159.169.196,"{""location"": ""GT"", ""is_mobile"": false}" 6994,3,154,2017-02-13 11:45:04,http://murazik.io/kristin.metz,,81.16.62.70,"{""location"": ""VU"", ""is_mobile"": false}" 6995,3,154,2017-05-20 04:47:08,http://langworth.com/josh,,110.253.18.210,"{""location"": ""SZ"", ""is_mobile"": false}" 6996,3,154,2017-03-27 17:18:20,http://brekke.name/tia.runolfon,,109.238.103.60,"{""location"": ""WS"", ""is_mobile"": false}" 6997,3,154,2017-02-16 13:23:11,http://johns.biz/garnet_moriette,,36.43.123.187,"{""location"": ""FK"", ""is_mobile"": false}" 6998,3,154,2017-06-08 09:08:59,http://mclaughlinschuster.biz/sigrid_maggio,,13.121.160.133,"{""location"": ""CD"", ""is_mobile"": true}" 6999,3,154,2017-04-23 18:08:34,http://heidenreich.io/austin_nikolaus,,177.215.76.193,"{""location"": ""SA"", ""is_mobile"": false}" 7000,3,154,2017-05-26 21:11:52,http://walterhamill.name/kasey.pfannerstill,,69.66.93.12,"{""location"": ""AM"", ""is_mobile"": true}" 7001,3,154,2016-12-31 00:55:47,http://gerlach.info/karelle,,36.254.227.43,"{""location"": ""CI"", ""is_mobile"": true}" 12860,5,287,2017-05-07 05:00:43,http://murazikbraun.co/waylon.nicolas,,244.36.73.37,"{""location"": ""FM"", ""is_mobile"": true}" 12861,5,287,2017-01-11 04:03:05,http://trantowgibson.name/aryanna_conn,,141.36.167.152,"{""location"": ""GR"", ""is_mobile"": true}" 12862,5,287,2017-03-06 02:38:35,http://balistreri.co/braxton.collins,,245.252.23.220,"{""location"": ""SJ"", ""is_mobile"": false}" 12863,5,287,2017-03-09 13:09:48,http://bartell.com/julie,,55.41.33.237,"{""location"": ""VA"", ""is_mobile"": true}" 12864,5,287,2017-03-13 04:41:33,http://dietrich.net/brady_huel,,171.124.16.63,"{""location"": ""LU"", ""is_mobile"": false}" 12865,5,287,2017-04-13 09:09:37,http://streich.info/laurine_harber,,224.194.105.247,"{""location"": ""US"", ""is_mobile"": false}" 12866,5,287,2017-02-04 01:38:05,http://heidenreichcasper.biz/zelda,,174.236.208.162,"{""location"": ""FK"", ""is_mobile"": true}" 12867,5,287,2016-12-20 13:21:29,http://wuckert.name/makenzie,,253.15.83.44,"{""location"": ""BQ"", ""is_mobile"": false}" 12868,5,287,2017-01-01 18:43:59,http://cartwrightbruen.com/fredy,,194.166.105.218,"{""location"": ""HM"", ""is_mobile"": false}" 12869,5,287,2017-02-07 13:57:18,http://moen.name/kiel.rolfson,,128.250.166.176,"{""location"": ""DM"", ""is_mobile"": true}" 12870,5,287,2016-12-25 22:20:09,http://herman.info/roger,,182.190.149.209,"{""location"": ""MQ"", ""is_mobile"": false}" 12871,5,287,2017-04-01 17:21:09,http://trantow.org/jeromy_toy,,202.28.151.93,"{""location"": ""DE"", ""is_mobile"": false}" 12872,5,287,2017-02-24 10:47:09,http://hamillweber.biz/maci,,13.214.27.143,"{""location"": ""GM"", ""is_mobile"": true}" 12873,5,287,2017-06-02 03:16:17,http://sauerpacocha.org/bertram,,167.169.45.70,"{""location"": ""ET"", ""is_mobile"": false}" 12874,5,287,2017-02-22 22:40:09,http://torphycartwright.org/foster,,131.127.182.64,"{""location"": ""BH"", ""is_mobile"": true}" 12875,5,287,2017-04-05 23:43:06,http://ryankeeling.co/emma_hyatt,,130.151.142.208,"{""location"": ""VC"", ""is_mobile"": true}" 12876,5,287,2017-01-16 14:37:02,http://schmeler.name/geovanny,,7.240.44.44,"{""location"": ""JE"", ""is_mobile"": false}" 12877,5,287,2017-02-03 16:24:56,http://hansen.biz/efrain,,251.3.93.86,"{""location"": ""NU"", ""is_mobile"": false}" 12878,5,287,2017-01-09 05:05:37,http://greenholt.net/emie,,84.8.239.117,"{""location"": ""PF"", ""is_mobile"": true}" 12879,5,287,2017-01-05 11:54:00,http://cummings.io/brandi.heidenreich,,48.182.42.112,"{""location"": ""CM"", ""is_mobile"": false}" 12880,5,287,2017-03-03 01:16:10,http://schimmel.info/melia,,45.29.109.181,"{""location"": ""AE"", ""is_mobile"": true}" 12881,5,288,2017-01-04 22:45:00,http://hermann.com/modesto,0.7787327495,120.76.49.124,"{""location"": ""CN"", ""is_mobile"": true}" 12882,5,288,2017-05-03 01:05:50,http://rogahn.org/michael.bednar,0.7351705188,125.14.37.24,"{""location"": ""PM"", ""is_mobile"": false}" 12883,5,288,2017-03-10 02:51:35,http://wolf.io/charlotte,0.1138953315,206.76.96.74,"{""location"": ""JE"", ""is_mobile"": false}" 12884,5,288,2016-12-17 18:29:45,http://mcclurekeebler.co/belle_wilderman,0.4644237301,33.165.198.199,"{""location"": ""AX"", ""is_mobile"": false}" 12885,5,288,2017-03-03 11:12:20,http://williamsongaylord.io/rubye,0.5695371739,44.250.113.125,"{""location"": ""FO"", ""is_mobile"": true}" 12886,5,288,2016-12-23 02:16:07,http://champlinhaag.org/liana,0.6578666124,119.52.214.167,"{""location"": ""CK"", ""is_mobile"": false}" 12887,5,288,2017-01-16 07:12:40,http://lehner.net/nico_walker,0.6491775219,105.181.169.129,"{""location"": ""KY"", ""is_mobile"": true}" 12888,5,288,2017-05-08 13:26:48,http://schimmel.io/connie,0.4650732160,17.129.97.248,"{""location"": ""SJ"", ""is_mobile"": true}" 12889,5,288,2017-03-14 17:41:14,http://murray.info/fred_mckenzie,0.7114877466,100.142.27.116,"{""location"": ""SH"", ""is_mobile"": true}" 12890,5,288,2017-06-11 05:29:21,http://collierschuppe.co/araceli,0.8883118933,93.172.40.146,"{""location"": ""TM"", ""is_mobile"": true}" 12891,5,288,2017-05-11 02:33:04,http://nolansporer.com/jude,0.3452559839,146.10.241.14,"{""location"": ""JO"", ""is_mobile"": true}" 12892,5,288,2017-02-23 07:09:55,http://beer.biz/erica,0.3027656393,135.21.18.105,"{""location"": ""SY"", ""is_mobile"": true}" 12893,5,288,2017-03-27 18:53:37,http://brownkshlerin.org/rhoda,0.3978771191,83.79.21.19,"{""location"": ""NO"", ""is_mobile"": false}" 12894,5,288,2017-06-07 21:41:01,http://wehner.co/jeramy,0.1194254282,109.63.98.158,"{""location"": ""AO"", ""is_mobile"": false}" 12895,5,288,2017-01-26 22:46:19,http://hettinger.org/marilyne,0.0977744560,27.154.71.32,"{""location"": ""BV"", ""is_mobile"": true}" 12896,5,288,2017-05-24 14:16:48,http://lakin.net/krystel.ferry,0.1668884006,97.21.182.248,"{""location"": ""IR"", ""is_mobile"": false}" 12897,5,288,2017-01-21 23:03:33,http://schimmel.io/jacklyn.davis,0.4410425647,184.11.4.193,"{""location"": ""GM"", ""is_mobile"": false}" 12898,5,288,2017-05-29 12:37:27,http://hellertreutel.io/dena.lowe,0.7313394271,174.63.17.95,"{""location"": ""AF"", ""is_mobile"": true}" 12899,5,288,2017-06-11 19:51:13,http://howe.com/darren,0.0398333868,105.17.77.86,"{""location"": ""HN"", ""is_mobile"": false}" 12900,5,288,2017-04-28 15:42:58,http://spencerwolff.info/vincent.wisozk,0.8039928458,170.72.207.199,"{""location"": ""NA"", ""is_mobile"": true}" 12901,5,288,2017-01-19 22:52:37,http://shanahan.io/aurore,0.5763521189,84.241.8.126,"{""location"": ""SE"", ""is_mobile"": true}" 12902,5,289,2017-03-09 16:41:31,http://parisian.biz/napoleon_cummings,0.5637876083,75.89.116.243,"{""location"": ""BA"", ""is_mobile"": false}" 12903,5,289,2017-06-11 16:02:44,http://labadiegreen.com/seth,0.1790093412,9.209.222.245,"{""location"": ""GB"", ""is_mobile"": true}" 12904,5,289,2017-06-01 07:32:47,http://boehmhintz.co/rogelio_rohan,0.4565257791,173.231.24.48,"{""location"": ""SO"", ""is_mobile"": false}" 12905,5,289,2017-02-10 19:06:23,http://steuberpagac.name/addison,0.1377349129,205.221.199.227,"{""location"": ""DJ"", ""is_mobile"": true}" 12906,5,289,2016-12-31 06:00:35,http://stracke.org/destany,0.2053571853,50.155.99.238,"{""location"": ""GM"", ""is_mobile"": true}" 12907,5,289,2017-05-26 23:20:04,http://bednarmccullough.info/moie.lehner,0.6865716784,72.135.123.24,"{""location"": ""UZ"", ""is_mobile"": false}" 12908,5,289,2017-05-11 20:01:34,http://zemlak.io/theodore,0.0289214345,8.116.198.235,"{""location"": ""CU"", ""is_mobile"": false}" 12909,5,289,2017-05-18 15:33:39,http://ullrich.biz/dax,0.4620727060,203.227.93.228,"{""location"": ""ID"", ""is_mobile"": true}" 12910,5,289,2016-12-24 01:06:47,http://weber.org/mae.erdman,0.8089092620,177.143.249.15,"{""location"": ""PG"", ""is_mobile"": true}" 12911,5,289,2017-04-20 11:22:44,http://conn.biz/jerome,0.8191011852,132.239.184.152,"{""location"": ""TM"", ""is_mobile"": false}" 12912,5,289,2017-03-27 18:52:54,http://fisher.io/akeem,0.1322306280,137.111.78.254,"{""location"": ""NG"", ""is_mobile"": false}" 15861,6,356,2017-05-24 07:59:20,http://kihnlynch.com/afton,,107.44.203.29,"{""location"": ""AG"", ""is_mobile"": true}" 15862,6,356,2017-04-24 23:05:44,http://kuhic.net/maxwell,,253.97.215.99,"{""location"": ""AG"", ""is_mobile"": false}" 15863,6,356,2017-03-24 19:36:31,http://gutmann.io/cecil,,159.143.98.11,"{""location"": ""LS"", ""is_mobile"": false}" 15864,6,356,2017-01-30 20:59:41,http://shanahanlueilwitz.co/rene.macgyver,,151.100.231.121,"{""location"": ""BT"", ""is_mobile"": false}" 15865,6,356,2017-06-01 04:00:23,http://stromanbahringer.com/heloise,,163.244.203.204,"{""location"": ""SM"", ""is_mobile"": false}" 15866,6,356,2017-03-25 02:46:18,http://schulistfahey.org/graham,,140.165.32.102,"{""location"": ""BQ"", ""is_mobile"": true}" 15867,6,356,2017-04-20 17:53:17,http://lang.net/george,,68.240.107.119,"{""location"": ""KR"", ""is_mobile"": false}" 15868,6,356,2017-04-27 01:51:49,http://labadie.com/idella_gerhold,,209.157.32.14,"{""location"": ""IS"", ""is_mobile"": false}" 15869,6,356,2016-12-13 06:46:53,http://gleichnerbins.com/kirsten_gleichner,,194.172.47.42,"{""location"": ""NZ"", ""is_mobile"": true}" 15870,6,356,2017-03-23 20:12:18,http://kiehnwatsica.name/carli_breitenberg,,137.218.200.53,"{""location"": ""CY"", ""is_mobile"": true}" 15871,6,356,2017-03-27 11:36:37,http://huels.org/ezra,,186.63.237.33,"{""location"": ""LC"", ""is_mobile"": false}" 15872,6,356,2017-03-07 05:15:34,http://carter.info/chase.stoltenberg,,166.194.26.61,"{""location"": ""DO"", ""is_mobile"": true}" 15873,6,356,2017-01-11 02:53:14,http://dach.io/damien,,103.237.169.183,"{""location"": ""SS"", ""is_mobile"": true}" 15874,6,356,2017-03-16 00:57:20,http://rolfson.org/lottie,,85.22.174.92,"{""location"": ""SG"", ""is_mobile"": false}" 15875,6,356,2016-12-18 21:31:00,http://kertzmann.name/jedidiah,,157.202.214.73,"{""location"": ""NL"", ""is_mobile"": true}" 15876,6,356,2017-05-28 13:22:07,http://goyetteconroy.info/mertie.boyle,,210.137.247.249,"{""location"": ""KN"", ""is_mobile"": true}" 15877,6,356,2017-01-04 16:11:18,http://blick.com/april_kovacek,,241.66.22.188,"{""location"": ""BB"", ""is_mobile"": true}" 15878,6,357,2017-03-27 17:13:54,http://gorczany.io/carey.koelpin,,119.161.38.176,"{""location"": ""BQ"", ""is_mobile"": false}" 15879,6,357,2017-04-29 05:39:11,http://jast.net/dereck_nitzsche,,35.27.131.178,"{""location"": ""BT"", ""is_mobile"": false}" 15880,6,357,2017-03-04 05:24:30,http://auer.name/madelynn_metz,,181.19.216.209,"{""location"": ""AR"", ""is_mobile"": false}" 15881,6,357,2017-02-13 07:19:51,http://ernser.com/jaydon.bednar,,192.120.183.37,"{""location"": ""DE"", ""is_mobile"": true}" 15882,6,357,2017-05-14 01:07:09,http://hansen.co/alford,,9.250.149.58,"{""location"": ""SV"", ""is_mobile"": true}" 15883,6,357,2016-12-14 01:45:11,http://mitchell.com/pierce_pollich,,6.165.125.133,"{""location"": ""NC"", ""is_mobile"": true}" 15884,6,357,2017-03-30 15:24:04,http://schneider.biz/darian,,28.113.123.24,"{""location"": ""AS"", ""is_mobile"": false}" 15885,6,357,2017-01-27 11:18:47,http://howe.info/grover,,149.237.66.14,"{""location"": ""AD"", ""is_mobile"": false}" 15886,6,357,2017-02-28 11:55:24,http://gulgowskigraham.info/mckenzie,,193.75.183.57,"{""location"": ""MF"", ""is_mobile"": false}" 15887,6,357,2017-05-30 13:55:22,http://hettinger.name/abby,,31.48.195.198,"{""location"": ""TD"", ""is_mobile"": true}" 15888,6,357,2017-02-07 16:51:59,http://heaney.org/josephine_brekke,,58.108.96.216,"{""location"": ""CG"", ""is_mobile"": true}" 15889,6,357,2017-02-22 09:54:58,http://rutherfordschuster.org/amanda.johnson,,61.213.239.33,"{""location"": ""IR"", ""is_mobile"": true}" 15890,6,357,2017-02-27 17:01:38,http://funk.co/alexane.frami,,201.212.10.254,"{""location"": ""CI"", ""is_mobile"": true}" 15891,6,357,2016-12-30 16:06:52,http://senger.co/jo,,65.203.234.226,"{""location"": ""CM"", ""is_mobile"": false}" 15892,6,357,2016-12-21 00:11:28,http://wisoky.io/kaleigh.champlin,,105.191.204.52,"{""location"": ""GE"", ""is_mobile"": false}" 15893,6,357,2017-01-12 10:23:03,http://hageneawayn.biz/janice,,32.85.12.192,"{""location"": ""GW"", ""is_mobile"": false}" 15894,6,357,2017-04-28 14:54:13,http://donnellygutkowski.io/ashlynn.davis,,205.20.117.45,"{""location"": ""CC"", ""is_mobile"": false}" 15895,6,357,2017-05-07 07:09:44,http://harber.io/sasha_halvorson,,25.161.234.35,"{""location"": ""UZ"", ""is_mobile"": true}" 15896,6,357,2017-04-25 02:41:18,http://oreillyfritsch.com/toy,,23.61.182.47,"{""location"": ""KY"", ""is_mobile"": true}" 15897,6,357,2017-02-15 22:50:55,http://wuckert.name/federico_huels,,195.114.174.18,"{""location"": ""LI"", ""is_mobile"": true}" 15898,6,357,2017-02-22 20:18:54,http://brakus.info/jaiden.mcdermott,,139.151.40.116,"{""location"": ""BS"", ""is_mobile"": true}" 15899,6,357,2017-03-01 04:01:31,http://huelstoltenberg.net/salma.shanahan,,214.34.91.115,"{""location"": ""MW"", ""is_mobile"": true}" 15900,6,357,2017-01-07 19:47:25,http://streich.biz/ivy,,7.26.170.57,"{""location"": ""MA"", ""is_mobile"": false}" 15901,6,357,2017-06-12 04:56:09,http://kuhic.org/hobart,,68.42.139.79,"{""location"": ""HT"", ""is_mobile"": true}" 15902,6,357,2017-01-14 19:05:49,http://schamberger.com/eudora,,91.13.86.170,"{""location"": ""AL"", ""is_mobile"": false}" 15903,6,357,2017-04-26 22:12:21,http://lubowitz.net/te,,184.219.79.234,"{""location"": ""KW"", ""is_mobile"": false}" 15904,6,357,2017-02-28 05:05:15,http://zieme.biz/elody,,111.34.217.211,"{""location"": ""WS"", ""is_mobile"": false}" 15905,6,357,2017-01-07 19:33:44,http://rodriguezwiza.co/terrence.luettgen,,179.247.144.123,"{""location"": ""NZ"", ""is_mobile"": true}" 15906,6,357,2017-03-12 04:29:02,http://purdy.com/dominic.mann,,153.48.54.234,"{""location"": ""ME"", ""is_mobile"": true}" 15907,6,358,2017-03-28 15:20:18,http://runolfonkoelpin.biz/mark.mohr,,23.66.82.177,"{""location"": ""NL"", ""is_mobile"": false}" 15908,6,358,2017-01-29 20:12:05,http://macejkovic.biz/timmothy_windler,,166.159.227.210,"{""location"": ""AR"", ""is_mobile"": false}" 15909,6,358,2017-02-16 09:04:21,http://mann.co/bettye_cain,,252.53.183.52,"{""location"": ""VN"", ""is_mobile"": true}" 15910,6,358,2017-02-26 18:10:24,http://emmerichgreenholt.info/mara,,235.85.225.137,"{""location"": ""BL"", ""is_mobile"": true}" 15911,6,358,2017-01-11 18:56:52,http://heidenreichkilback.co/ken,,111.192.151.229,"{""location"": ""LA"", ""is_mobile"": false}" 15912,6,358,2016-12-21 08:04:51,http://schneider.com/troy,,209.162.235.57,"{""location"": ""TG"", ""is_mobile"": false}" 15913,6,358,2017-01-24 06:43:23,http://lockmanbecker.com/martine,,213.232.26.245,"{""location"": ""BA"", ""is_mobile"": false}" 15914,6,358,2017-01-04 10:55:15,http://sawaynbatz.co/desmond.keeling,,42.36.58.199,"{""location"": ""KG"", ""is_mobile"": true}" 15915,6,358,2017-04-13 19:21:39,http://harberstoltenberg.co/merlin,,199.32.194.69,"{""location"": ""GT"", ""is_mobile"": false}" 3060,2,67,2017-06-04 07:47:35,http://daniel.com/andreanne,0.5081365347,104.223.204.123,"{""location"": ""KN"", ""is_mobile"": true}" 3061,2,67,2017-05-27 17:18:34,http://mertz.biz/patrick_leffler,0.6428793500,121.240.69.73,"{""location"": ""TZ"", ""is_mobile"": true}" 3062,2,67,2017-02-17 23:52:39,http://steuber.com/emelia.wintheiser,0.7345841760,92.254.7.121,"{""location"": ""CF"", ""is_mobile"": true}" 3063,2,67,2016-12-14 22:16:58,http://walter.io/modesto.simonis,0.8903366154,71.15.55.197,"{""location"": ""GW"", ""is_mobile"": false}" 3064,2,67,2016-12-17 11:54:24,http://labadie.org/grover.waters,0.4764158963,86.59.142.135,"{""location"": ""CW"", ""is_mobile"": false}" 3065,2,67,2017-01-24 04:07:58,http://ferry.com/blaze,0.7393647915,105.241.26.140,"{""location"": ""GE"", ""is_mobile"": true}" 3066,2,67,2017-02-18 21:22:03,http://wolff.org/junior,0.1373549526,71.77.175.140,"{""location"": ""NU"", ""is_mobile"": false}" 3067,2,67,2017-02-17 05:42:22,http://rohanking.info/desmond.stroman,0.1648052082,242.240.82.43,"{""location"": ""NC"", ""is_mobile"": true}" 3068,2,67,2017-03-19 01:44:47,http://weinat.com/lue_nader,0.9962787196,179.89.185.38,"{""location"": ""KM"", ""is_mobile"": true}" 3069,2,67,2017-02-06 12:54:10,http://cormier.net/jordane,0.5503633857,116.65.94.19,"{""location"": ""AL"", ""is_mobile"": true}" 3070,2,67,2017-06-10 13:05:05,http://wilkinsonsimonis.biz/marcellus.schiller,0.1873058066,144.148.223.18,"{""location"": ""TT"", ""is_mobile"": true}" 3071,2,67,2017-04-13 03:27:52,http://bergnaumwill.info/laverna,0.6004717392,109.5.131.233,"{""location"": ""SC"", ""is_mobile"": false}" 3072,2,67,2017-02-15 03:20:19,http://wintheiser.io/letitia.bednar,0.3552216278,106.76.177.183,"{""location"": ""FM"", ""is_mobile"": false}" 3073,2,67,2017-02-10 21:07:43,http://runolfsdottir.info/lavinia,0.1145646945,222.220.31.158,"{""location"": ""CC"", ""is_mobile"": true}" 3074,2,67,2017-05-11 05:43:21,http://kuhnwindler.org/rebeca_pacocha,0.2933389308,125.29.242.243,"{""location"": ""VU"", ""is_mobile"": false}" 3075,2,67,2017-03-31 16:14:08,http://goodwin.net/america_lemke,0.4858740781,88.2.149.68,"{""location"": ""BY"", ""is_mobile"": false}" 3076,2,67,2017-05-11 02:33:47,http://herzog.name/maude,0.0106117873,117.116.205.202,"{""location"": ""LK"", ""is_mobile"": false}" 3077,2,67,2017-03-20 12:44:47,http://stehr.co/gardner,0.6639282114,125.130.47.156,"{""location"": ""BI"", ""is_mobile"": true}" 3078,2,67,2017-01-31 22:26:43,http://pouros.co/alysha,0.0699426450,40.210.144.196,"{""location"": ""MV"", ""is_mobile"": true}" 3079,2,67,2017-04-17 02:51:31,http://wilkinson.io/brenna,0.4833066038,135.152.82.60,"{""location"": ""KZ"", ""is_mobile"": false}" 3080,2,67,2017-05-28 09:50:00,http://herman.name/athena_keebler,0.1843461595,119.177.168.105,"{""location"": ""CY"", ""is_mobile"": true}" 3081,2,67,2017-02-06 23:46:11,http://shanahan.net/lina_hauck,0.7073889898,39.113.105.115,"{""location"": ""KP"", ""is_mobile"": false}" 3082,2,67,2017-04-01 14:31:37,http://hand.org/leola.friesen,0.6395754157,84.181.10.190,"{""location"": ""FJ"", ""is_mobile"": true}" 3083,2,67,2017-05-08 18:25:58,http://mosciski.org/ava_carter,0.5432194911,86.42.28.124,"{""location"": ""BW"", ""is_mobile"": true}" 3084,2,67,2016-12-27 18:21:35,http://rodriguez.org/danika,0.0195413358,65.215.10.227,"{""location"": ""PK"", ""is_mobile"": false}" 3085,2,67,2017-04-27 20:33:11,http://trompcorkery.biz/catharine_hammes,0.7785582599,99.201.77.119,"{""location"": ""ST"", ""is_mobile"": false}" 3086,2,67,2017-04-03 20:41:54,http://klocko.org/laurie.hauck,0.4295465037,219.61.171.236,"{""location"": ""TG"", ""is_mobile"": false}" 3087,2,67,2017-04-10 15:45:10,http://prosacco.io/jakob,0.9702963902,174.206.54.249,"{""location"": ""DE"", ""is_mobile"": false}" 3088,2,67,2017-05-23 13:52:02,http://altenwerth.io/hayley,0.5522858634,120.27.168.75,"{""location"": ""IN"", ""is_mobile"": true}" 3089,2,67,2017-05-16 09:17:54,http://lind.biz/carroll.hettinger,0.1695288634,131.4.38.150,"{""location"": ""CK"", ""is_mobile"": true}" 3090,2,67,2017-01-08 02:13:33,http://lemke.name/einar.botsford,0.7323684522,22.82.38.73,"{""location"": ""FR"", ""is_mobile"": false}" 3091,2,68,2017-02-08 04:02:21,http://okon.net/john,0.0228505400,30.135.190.47,"{""location"": ""FO"", ""is_mobile"": false}" 3092,2,68,2017-05-19 02:53:14,http://wisozk.io/blair_pfannerstill,0.1881665641,229.218.106.143,"{""location"": ""SV"", ""is_mobile"": false}" 3093,2,68,2017-04-05 03:53:39,http://wuckert.net/kathryn,0.8955266268,238.250.188.195,"{""location"": ""GM"", ""is_mobile"": false}" 3094,2,68,2017-05-09 17:00:40,http://heaney.com/rachael,0.8634943041,240.32.102.200,"{""location"": ""MQ"", ""is_mobile"": true}" 3095,2,68,2017-05-17 11:28:08,http://heller.info/vince,0.8918104107,221.11.132.24,"{""location"": ""DM"", ""is_mobile"": false}" 3096,2,68,2017-04-16 04:13:18,http://corwineichmann.org/shayna_mueller,0.0393817586,93.121.136.9,"{""location"": ""BY"", ""is_mobile"": false}" 3097,2,68,2017-02-10 18:17:20,http://padberg.io/kyleigh,0.0130914409,128.249.116.15,"{""location"": ""SX"", ""is_mobile"": true}" 3098,2,68,2017-01-31 16:23:52,http://parker.name/magdalena_price,0.8521141740,165.206.27.33,"{""location"": ""AQ"", ""is_mobile"": true}" 3099,2,68,2017-02-17 05:34:23,http://daniel.com/rebeka,0.9185710806,108.6.26.27,"{""location"": ""BN"", ""is_mobile"": true}" 3100,2,68,2016-12-17 16:48:25,http://okeefe.biz/justen.kuphal,0.0015395372,67.158.191.197,"{""location"": ""LB"", ""is_mobile"": false}" 3101,2,68,2017-05-29 14:58:25,http://bechtelar.io/helene,0.2010510408,59.67.157.18,"{""location"": ""MZ"", ""is_mobile"": false}" 3102,2,68,2016-12-29 13:46:09,http://funk.org/ervin,0.1494269200,165.92.59.199,"{""location"": ""MX"", ""is_mobile"": false}" 3103,2,68,2017-05-15 01:06:45,http://kohler.biz/alayna,0.1601838411,202.56.15.120,"{""location"": ""KI"", ""is_mobile"": true}" 3104,2,68,2017-04-18 03:56:28,http://west.io/francisca.boehm,0.8928508989,132.8.246.232,"{""location"": ""AW"", ""is_mobile"": true}" 3105,2,68,2017-02-03 04:05:15,http://jones.info/trystan,0.7573981124,247.99.21.246,"{""location"": ""AS"", ""is_mobile"": false}" 3106,2,68,2017-05-12 19:12:40,http://homenick.biz/jonathon_stark,0.2784887883,147.213.122.245,"{""location"": ""MX"", ""is_mobile"": true}" 3107,2,68,2016-12-15 18:31:18,http://mitchell.biz/peyton,0.8696738815,230.224.124.99,"{""location"": ""TJ"", ""is_mobile"": false}" 3108,2,68,2017-05-06 23:29:35,http://dickinson.org/ivory,0.7421303781,176.135.115.90,"{""location"": ""SI"", ""is_mobile"": true}" 3109,2,68,2017-04-16 17:15:08,http://breitenberg.biz/jerod,0.2523042135,49.241.168.221,"{""location"": ""MU"", ""is_mobile"": false}" 3110,2,68,2017-05-28 20:49:30,http://oconnell.net/charles.bernhard,0.9487590323,92.102.143.131,"{""location"": ""OM"", ""is_mobile"": true}" 3111,2,68,2017-03-01 05:43:41,http://weinatcollins.net/lexus.cole,0.8671190870,94.164.38.213,"{""location"": ""KZ"", ""is_mobile"": false}" 9996,4,223,2017-05-24 00:54:35,http://feil.net/zoey,0.0671630238,29.111.198.129,"{""location"": ""ES"", ""is_mobile"": false}" 9997,4,223,2017-05-18 05:01:25,http://botsford.biz/helga.funk,0.4426849024,216.58.213.142,"{""location"": ""RO"", ""is_mobile"": false}" 9998,4,223,2017-05-15 09:14:12,http://nitzsche.org/mitchel.lindgren,0.3630798485,111.109.233.70,"{""location"": ""DM"", ""is_mobile"": true}" 9999,4,223,2017-01-31 14:03:52,http://schroeder.org/alexandria,0.6756549808,140.31.192.86,"{""location"": ""SR"", ""is_mobile"": false}" 10000,4,223,2017-02-19 11:51:24,http://ledner.com/trevion,0.6515260766,211.94.7.49,"{""location"": ""SJ"", ""is_mobile"": true}" 10001,4,223,2017-01-24 10:09:06,http://leffler.io/melyna,0.5911099904,198.24.142.72,"{""location"": ""KH"", ""is_mobile"": true}" 10002,4,223,2016-12-26 08:33:13,http://moriette.name/briana_moriette,0.2514496626,125.92.115.87,"{""location"": ""JP"", ""is_mobile"": false}" 10003,4,223,2017-03-21 14:34:44,http://bartoletti.com/devon,0.7213022701,41.171.106.92,"{""location"": ""VE"", ""is_mobile"": true}" 10004,4,223,2017-05-19 03:01:42,http://thompson.net/noah.sporer,0.1148729238,226.219.139.117,"{""location"": ""KM"", ""is_mobile"": false}" 10005,4,223,2017-06-14 02:02:28,http://okonstrosin.org/petra.kirlin,0.2770214283,36.130.43.106,"{""location"": ""PA"", ""is_mobile"": true}" 10006,4,223,2017-02-24 09:32:16,http://abshire.org/hermina.williamson,0.7333105519,16.244.77.96,"{""location"": ""PM"", ""is_mobile"": false}" 10007,4,223,2017-02-02 16:45:52,http://cummingswaters.info/alvina,0.1406641602,227.134.146.8,"{""location"": ""GQ"", ""is_mobile"": true}" 10008,4,224,2017-04-03 23:26:57,http://armstrong.biz/erwin,0.8019035669,65.168.211.225,"{""location"": ""VU"", ""is_mobile"": true}" 10009,4,224,2017-05-20 14:57:42,http://ullrich.org/alexandrine_cronin,0.1422289452,15.29.136.104,"{""location"": ""NC"", ""is_mobile"": false}" 10010,4,224,2017-06-01 04:24:00,http://boyle.info/alexandre,0.0295243911,130.124.173.170,"{""location"": ""AF"", ""is_mobile"": true}" 10011,4,224,2016-12-14 02:44:50,http://bayerpouros.biz/garett,0.3821132978,63.7.43.223,"{""location"": ""FI"", ""is_mobile"": false}" 10012,4,224,2017-02-28 22:45:43,http://muller.com/julianne_mosciski,0.4137981111,148.109.110.112,"{""location"": ""TN"", ""is_mobile"": true}" 10013,4,224,2017-04-15 10:06:01,http://cormier.biz/morgan,0.3683053537,77.22.107.118,"{""location"": ""SE"", ""is_mobile"": false}" 10014,4,224,2017-04-14 09:08:57,http://gerholdgraham.name/guie.kozey,0.8481704464,168.224.90.232,"{""location"": ""GA"", ""is_mobile"": true}" 10015,4,224,2017-01-09 13:42:49,http://okuneva.name/delaney_crist,0.0240244726,101.165.72.81,"{""location"": ""MK"", ""is_mobile"": true}" 10016,4,224,2017-05-10 12:00:16,http://eberthilll.co/mark_roob,0.7175282874,160.246.220.164,"{""location"": ""MV"", ""is_mobile"": false}" 10017,4,224,2017-05-01 14:49:38,http://conroymoriette.net/destiney,0.5453984019,49.99.14.141,"{""location"": ""MD"", ""is_mobile"": true}" 10018,4,224,2016-12-26 20:00:57,http://lefflerkihn.io/cora_pollich,0.9887016783,62.79.53.147,"{""location"": ""LR"", ""is_mobile"": false}" 10019,4,224,2017-04-14 20:26:30,http://schmeler.org/antonina,0.3667238933,95.185.2.152,"{""location"": ""KW"", ""is_mobile"": true}" 10020,4,224,2017-05-13 08:41:03,http://sipes.org/ruby.mertz,0.1748238044,147.26.16.186,"{""location"": ""ER"", ""is_mobile"": false}" 10021,4,224,2017-04-02 00:28:59,http://schambergersanford.net/avis_oconner,0.5084078587,218.5.243.169,"{""location"": ""CH"", ""is_mobile"": true}" 10022,4,224,2017-03-24 08:28:29,http://langoshgutkowski.com/paula.welch,0.3080244138,185.73.8.217,"{""location"": ""SC"", ""is_mobile"": true}" 10023,4,224,2017-03-08 13:41:42,http://hettingertorp.biz/duane,0.6301798479,104.188.159.90,"{""location"": ""BY"", ""is_mobile"": true}" 10024,4,224,2017-03-05 05:48:55,http://ziemannhahn.io/annetta.vonrueden,0.0424738026,174.220.128.60,"{""location"": ""IS"", ""is_mobile"": false}" 10025,4,224,2017-01-02 13:17:13,http://veumschiller.co/eric_sauer,0.1563862592,251.147.253.47,"{""location"": ""LC"", ""is_mobile"": true}" 10026,4,224,2016-12-14 10:35:30,http://lang.co/lonzo,0.7270488260,64.12.38.28,"{""location"": ""MC"", ""is_mobile"": true}" 10027,4,224,2017-01-30 12:56:05,http://jakubowskiwaelchi.com/jose,0.9428672359,166.131.46.80,"{""location"": ""GM"", ""is_mobile"": false}" 10028,4,224,2017-05-03 00:12:16,http://keler.name/evert,0.5552977545,249.162.216.13,"{""location"": ""CG"", ""is_mobile"": false}" 10029,4,224,2017-04-04 00:23:12,http://kemmerlangosh.com/sally,0.4814987407,169.254.190.222,"{""location"": ""ZW"", ""is_mobile"": false}" 10030,4,224,2017-04-07 20:03:26,http://fahey.net/richard_conroy,0.2796135147,27.231.241.88,"{""location"": ""IS"", ""is_mobile"": false}" 10031,4,224,2017-06-05 09:30:17,http://marksgleichner.name/torrey_langworth,0.4976732834,191.105.162.159,"{""location"": ""GN"", ""is_mobile"": true}" 10032,4,224,2017-02-24 11:47:48,http://kohler.org/carmella_yundt,0.9604905292,5.37.29.202,"{""location"": ""ME"", ""is_mobile"": false}" 10033,4,224,2016-12-24 12:10:15,http://emmerich.org/carmine.reichert,0.5902192977,141.92.249.219,"{""location"": ""GG"", ""is_mobile"": false}" 10034,4,224,2017-01-08 04:09:33,http://olson.name/royal,0.7248193964,224.183.27.18,"{""location"": ""AW"", ""is_mobile"": true}" 10035,4,224,2017-05-06 15:39:10,http://harvey.co/eugenia,0.8089411981,123.208.61.68,"{""location"": ""AU"", ""is_mobile"": true}" 10036,4,224,2017-01-29 12:20:25,http://stehr.com/juliana.turcotte,0.5435663782,164.213.232.135,"{""location"": ""TJ"", ""is_mobile"": true}" 10037,4,224,2017-01-21 17:04:07,http://kuvalis.io/samanta_schuppe,0.4193934774,54.48.69.180,"{""location"": ""JP"", ""is_mobile"": false}" 10038,4,224,2017-01-03 09:03:49,http://ankundingrohan.co/maye_reinger,0.2258893031,29.34.43.114,"{""location"": ""MR"", ""is_mobile"": true}" 10039,4,225,2017-05-06 03:45:19,http://jones.net/osborne,0.7582668016,243.133.33.144,"{""location"": ""SO"", ""is_mobile"": true}" 10040,4,225,2017-05-13 15:52:59,http://bergebergnaum.net/ethyl,0.9929988940,55.110.223.125,"{""location"": ""HK"", ""is_mobile"": false}" 10041,4,225,2017-02-22 16:56:30,http://gutkowskibailey.org/carolanne,0.7890077182,66.85.250.254,"{""location"": ""BS"", ""is_mobile"": false}" 10042,4,225,2017-04-22 21:39:25,http://hintz.io/neal.reichert,0.8373338188,32.223.22.102,"{""location"": ""BB"", ""is_mobile"": true}" 10043,4,225,2017-01-24 13:09:58,http://lindbotsford.com/mazie_will,0.4117967747,4.250.87.248,"{""location"": ""ID"", ""is_mobile"": false}" 10044,4,225,2017-01-10 02:49:34,http://kertzmann.biz/loyal.koelpin,0.8807088268,172.84.6.80,"{""location"": ""SO"", ""is_mobile"": false}" 10045,4,225,2017-03-05 04:09:18,http://parkerschmitt.io/sigrid.kulas,0.5817756627,27.189.74.50,"{""location"": ""EE"", ""is_mobile"": false}" 10046,4,225,2017-04-07 08:36:57,http://mayert.org/melisa_ruel,0.1234101914,25.226.250.94,"{""location"": ""PR"", ""is_mobile"": true}" 7002,3,154,2017-02-14 12:01:42,http://fahey.info/zachariah.swaniawski,,211.170.128.100,"{""location"": ""KI"", ""is_mobile"": true}" 7003,3,154,2017-03-10 08:47:40,http://raynor.co/hal,,22.85.79.65,"{""location"": ""GN"", ""is_mobile"": false}" 7004,3,154,2017-04-21 15:22:38,http://stehr.info/elbert,,97.128.190.41,"{""location"": ""LV"", ""is_mobile"": true}" 7005,3,154,2016-12-19 14:15:47,http://ricekulas.co/daniela,,45.31.134.130,"{""location"": ""BT"", ""is_mobile"": false}" 7006,3,154,2017-04-29 02:59:02,http://stracke.info/albert,,2.41.53.66,"{""location"": ""SK"", ""is_mobile"": false}" 7007,3,154,2017-05-23 11:59:53,http://rutherford.co/camden,,217.211.109.38,"{""location"": ""TH"", ""is_mobile"": true}" 7008,3,154,2016-12-26 07:05:42,http://orn.net/keara_dickens,,211.206.237.149,"{""location"": ""LA"", ""is_mobile"": false}" 7009,3,154,2017-03-31 02:27:32,http://schaeferlakin.name/mervin_brekke,,134.161.159.135,"{""location"": ""NR"", ""is_mobile"": true}" 7010,3,155,2017-06-03 21:42:47,http://cole.co/jaren_dare,,94.136.25.229,"{""location"": ""AW"", ""is_mobile"": true}" 7011,3,155,2017-05-09 22:55:31,http://orn.name/phyllis,,187.142.243.111,"{""location"": ""TL"", ""is_mobile"": true}" 7012,3,155,2017-01-18 18:23:53,http://lueilwitz.co/kathryn,,122.215.189.45,"{""location"": ""PY"", ""is_mobile"": true}" 7013,3,155,2017-05-02 16:57:03,http://gloverhuels.com/jacklyn,,109.81.80.173,"{""location"": ""OM"", ""is_mobile"": true}" 7014,3,155,2017-02-13 16:09:35,http://wolf.net/caleb.deckow,,10.14.152.236,"{""location"": ""VU"", ""is_mobile"": false}" 7015,3,155,2017-06-06 15:00:49,http://ondricka.com/judson,,143.220.194.91,"{""location"": ""EE"", ""is_mobile"": true}" 7016,3,155,2017-02-08 22:25:40,http://roberts.biz/shawn_baumbach,,227.55.162.132,"{""location"": ""KI"", ""is_mobile"": true}" 7017,3,155,2017-02-13 13:12:33,http://wilkinsonkuphal.org/alycia_emard,,23.84.126.158,"{""location"": ""KE"", ""is_mobile"": false}" 7018,3,155,2017-03-07 02:12:01,http://jerde.info/pattie.spinka,,232.7.60.76,"{""location"": ""ET"", ""is_mobile"": true}" 7019,3,155,2017-05-06 12:09:13,http://franecki.com/clare,,61.136.149.40,"{""location"": ""TH"", ""is_mobile"": true}" 7020,3,155,2017-01-06 22:26:16,http://weber.co/baylee,,44.17.111.129,"{""location"": ""SA"", ""is_mobile"": true}" 7021,3,155,2017-03-24 10:51:43,http://metz.com/unique_kirlin,,86.110.236.217,"{""location"": ""MK"", ""is_mobile"": true}" 7022,3,155,2017-01-04 23:09:12,http://kovacek.info/yazmin,,223.102.4.37,"{""location"": ""LV"", ""is_mobile"": true}" 7023,3,155,2017-01-12 07:03:06,http://grady.net/marques,,19.130.46.202,"{""location"": ""LS"", ""is_mobile"": false}" 7024,3,155,2017-04-26 07:29:32,http://loweokeefe.name/jeanne.langworth,,11.205.230.155,"{""location"": ""HM"", ""is_mobile"": false}" 7025,3,155,2017-02-27 21:14:12,http://rutherfordhammes.net/noel_sipes,,23.235.70.35,"{""location"": ""MA"", ""is_mobile"": false}" 7026,3,155,2017-04-18 17:37:43,http://andersonsmitham.co/frederic,,216.192.31.125,"{""location"": ""AU"", ""is_mobile"": false}" 7027,3,155,2017-03-27 22:02:07,http://leannondoyle.name/vladimir.pfannerstill,,2.57.161.73,"{""location"": ""YT"", ""is_mobile"": false}" 7028,3,155,2017-04-05 06:59:51,http://rutherford.com/kaia.herzog,,139.28.174.246,"{""location"": ""BJ"", ""is_mobile"": false}" 7029,3,155,2017-02-01 01:16:49,http://sanford.com/lester,,45.216.242.107,"{""location"": ""SM"", ""is_mobile"": true}" 7030,3,155,2017-03-08 07:13:01,http://kozeykuhn.name/elza.willms,,196.74.108.219,"{""location"": ""CR"", ""is_mobile"": true}" 7031,3,155,2016-12-21 08:10:53,http://hane.name/evans,,191.218.81.226,"{""location"": ""MR"", ""is_mobile"": true}" 7032,3,155,2017-03-01 21:01:43,http://breitenberg.co/kadin,,117.132.181.154,"{""location"": ""AG"", ""is_mobile"": false}" 7033,3,155,2017-04-12 12:57:56,http://schaefer.net/eladio,,109.134.13.18,"{""location"": ""PN"", ""is_mobile"": true}" 7034,3,155,2017-05-30 23:57:46,http://farrell.name/vincenzo,,229.241.150.244,"{""location"": ""RU"", ""is_mobile"": false}" 7035,3,155,2017-04-12 21:27:17,http://becker.biz/carlos.grimes,,8.150.199.87,"{""location"": ""PE"", ""is_mobile"": true}" 7036,3,156,2017-04-05 11:40:54,http://skiletoltenberg.io/green.quigley,0.0218777881,78.193.43.33,"{""location"": ""HN"", ""is_mobile"": true}" 7037,3,156,2017-04-26 07:45:22,http://bernier.biz/maribel,0.0711979253,70.171.31.171,"{""location"": ""AE"", ""is_mobile"": false}" 7038,3,156,2017-04-29 15:29:54,http://murrayfisher.biz/trevor,0.0349001465,73.140.102.5,"{""location"": ""SM"", ""is_mobile"": true}" 7039,3,156,2017-04-06 21:46:31,http://doyle.io/lela,0.7815179423,246.51.236.43,"{""location"": ""SO"", ""is_mobile"": true}" 7040,3,156,2017-01-13 00:27:10,http://goodwinwiegand.info/tate_boyer,0.1959538141,241.253.46.200,"{""location"": ""CD"", ""is_mobile"": false}" 7041,3,156,2017-01-07 18:54:28,http://beahan.org/daisha.weimann,0.0547582120,155.53.24.179,"{""location"": ""MT"", ""is_mobile"": false}" 7042,3,156,2017-03-05 19:22:16,http://ricegibson.org/pamela_rath,0.1491594551,95.175.73.174,"{""location"": ""LA"", ""is_mobile"": false}" 7043,3,156,2017-04-25 20:48:33,http://anderson.net/abelardo,0.2684333554,231.86.193.190,"{""location"": ""BR"", ""is_mobile"": false}" 7044,3,156,2017-01-28 03:52:07,http://grantgrady.info/erica,0.4401895081,9.253.16.14,"{""location"": ""HR"", ""is_mobile"": false}" 7045,3,156,2017-01-08 00:34:31,http://mitchellhills.info/seamus_reichert,0.1986207852,180.180.208.228,"{""location"": ""DE"", ""is_mobile"": false}" 7046,3,156,2017-05-08 23:52:23,http://nicolasbrekke.com/brionna.vonrueden,0.2702300079,178.197.31.87,"{""location"": ""GR"", ""is_mobile"": false}" 7047,3,156,2017-03-19 00:20:22,http://waelchimccullough.name/rosalee_hagenes,0.7767875117,249.62.162.122,"{""location"": ""DJ"", ""is_mobile"": true}" 7048,3,156,2017-03-24 08:00:42,http://ko.com/monserrat,0.7187589391,117.165.248.45,"{""location"": ""BZ"", ""is_mobile"": false}" 7049,3,156,2017-04-02 12:20:10,http://leffler.net/gracie,0.6450532900,84.200.94.200,"{""location"": ""ST"", ""is_mobile"": false}" 7050,3,156,2017-03-01 07:22:16,http://bradtke.com/maggie,0.8463542005,204.12.70.135,"{""location"": ""AM"", ""is_mobile"": true}" 7051,3,156,2017-01-20 09:41:54,http://heel.io/garrett_goldner,0.6585828317,149.93.36.108,"{""location"": ""GB"", ""is_mobile"": true}" 7052,3,156,2017-05-11 13:51:35,http://frami.net/elta.stehr,0.7272995671,152.71.16.54,"{""location"": ""NF"", ""is_mobile"": true}" 7053,3,156,2016-12-31 14:03:50,http://kilback.info/hillary_kunde,0.2482418354,55.72.83.242,"{""location"": ""PE"", ""is_mobile"": true}" 7054,3,156,2017-06-05 04:06:10,http://collier.io/sonya,0.1024340705,201.230.70.102,"{""location"": ""SA"", ""is_mobile"": true}" 7055,3,156,2017-04-20 02:45:36,http://goldner.net/vilma.bartoletti,0.7415573045,31.176.226.89,"{""location"": ""GM"", ""is_mobile"": true}" 12913,5,289,2017-05-12 18:19:06,http://marquardtkling.io/garnet.marks,0.2114571526,64.128.154.207,"{""location"": ""LV"", ""is_mobile"": true}" 12914,5,289,2017-03-09 06:17:24,http://welchhauck.name/hudson_oconnell,0.3579188433,107.140.161.117,"{""location"": ""JE"", ""is_mobile"": true}" 12915,5,289,2017-04-05 14:51:42,http://bednar.info/chasity,0.7221942403,210.144.212.154,"{""location"": ""TT"", ""is_mobile"": false}" 12916,5,289,2017-03-07 04:12:56,http://reichel.io/jamison.kling,0.3657567777,68.208.60.85,"{""location"": ""CK"", ""is_mobile"": true}" 12917,5,289,2017-04-28 14:04:06,http://becker.com/donald,0.0535041910,246.119.99.113,"{""location"": ""NG"", ""is_mobile"": true}" 12918,5,289,2017-06-07 18:37:33,http://greenholt.co/iva,0.5310092216,21.39.253.57,"{""location"": ""BN"", ""is_mobile"": false}" 12919,5,289,2017-03-05 21:59:38,http://runolfsdottir.biz/cletus.hayes,0.4710642801,163.26.204.138,"{""location"": ""VG"", ""is_mobile"": false}" 12920,5,289,2017-04-15 18:07:23,http://cronin.co/maybell,0.3359936474,180.212.94.163,"{""location"": ""RU"", ""is_mobile"": false}" 12921,5,289,2017-05-27 15:50:18,http://fahey.com/lacey,0.9771584125,28.145.201.223,"{""location"": ""TN"", ""is_mobile"": false}" 12922,5,289,2017-03-13 04:21:22,http://torp.info/isadore_herman,0.3819657892,15.201.30.205,"{""location"": ""CK"", ""is_mobile"": true}" 12923,5,289,2017-03-18 12:12:23,http://rath.net/ruby.koepp,0.8512739068,34.145.244.196,"{""location"": ""KZ"", ""is_mobile"": false}" 12924,5,289,2017-02-21 16:44:32,http://champlin.name/myles_lang,0.0453148654,51.63.233.62,"{""location"": ""LC"", ""is_mobile"": false}" 12925,5,289,2017-03-23 13:59:19,http://powlowski.org/jaylin_padberg,0.6919893587,177.183.171.117,"{""location"": ""LI"", ""is_mobile"": true}" 12926,5,289,2017-06-09 01:03:12,http://kreigerbergnaum.biz/norberto,0.0742084578,12.148.102.227,"{""location"": ""CG"", ""is_mobile"": true}" 12927,5,289,2017-02-27 17:04:11,http://hane.org/rae,0.6256220353,139.147.142.66,"{""location"": ""SG"", ""is_mobile"": true}" 12928,5,289,2016-12-23 20:04:30,http://hintzhilll.org/cathrine,0.8752426985,138.229.120.99,"{""location"": ""NU"", ""is_mobile"": true}" 12929,5,289,2017-04-29 12:07:45,http://lind.name/yasmin,0.9627893802,252.28.136.77,"{""location"": ""OM"", ""is_mobile"": false}" 12930,5,289,2017-02-26 05:38:57,http://larson.net/markus_casper,0.6463302032,145.244.89.45,"{""location"": ""BO"", ""is_mobile"": true}" 12931,5,289,2017-06-08 19:42:28,http://flatley.name/abe,0.2061320792,216.190.73.156,"{""location"": ""NA"", ""is_mobile"": false}" 12932,5,289,2017-03-23 23:30:38,http://bogisich.biz/hayden,0.4216103311,44.206.232.19,"{""location"": ""UG"", ""is_mobile"": false}" 12933,5,289,2017-05-17 22:55:54,http://dooley.info/monty,0.9298374874,227.123.154.22,"{""location"": ""NP"", ""is_mobile"": true}" 12934,5,289,2017-02-16 01:44:35,http://reynolds.net/brant.stracke,0.2862261264,107.236.24.58,"{""location"": ""TH"", ""is_mobile"": true}" 12935,5,289,2017-05-11 00:48:34,http://lowerosenbaum.biz/tania.lubowitz,0.1804552158,58.73.244.7,"{""location"": ""BA"", ""is_mobile"": false}" 12936,5,289,2017-04-29 04:37:15,http://labadie.com/palma.metz,0.9864401177,167.88.98.68,"{""location"": ""KE"", ""is_mobile"": true}" 12937,5,289,2017-01-13 02:19:58,http://spinkarobel.com/audrey,0.2958251714,69.18.3.21,"{""location"": ""YT"", ""is_mobile"": false}" 12938,5,289,2017-02-24 12:14:52,http://hackett.info/ollie,0.9631013858,173.91.151.16,"{""location"": ""LI"", ""is_mobile"": false}" 12939,5,289,2017-03-21 17:10:46,http://blanda.io/harmony_lesch,0.3545326717,54.52.51.136,"{""location"": ""CW"", ""is_mobile"": false}" 12940,5,289,2017-04-27 07:08:17,http://kemmertillman.io/coty,0.2285528420,210.133.203.245,"{""location"": ""CL"", ""is_mobile"": false}" 12941,5,289,2017-04-23 07:24:05,http://beahan.biz/dell,0.2815222932,217.50.92.125,"{""location"": ""NE"", ""is_mobile"": true}" 12942,5,289,2017-03-13 17:32:55,http://schmittblick.org/norval_veum,0.6730696121,20.166.222.67,"{""location"": ""PR"", ""is_mobile"": false}" 12943,5,289,2017-03-31 09:15:19,http://gaylord.net/jey_schaefer,0.0993606289,210.160.19.106,"{""location"": ""BI"", ""is_mobile"": true}" 12944,5,289,2017-01-03 06:01:53,http://feeney.biz/ford_hand,0.2517836048,95.79.195.236,"{""location"": ""SA"", ""is_mobile"": true}" 12945,5,289,2016-12-25 04:17:28,http://gutmann.biz/zoila,0.5404774759,39.162.126.181,"{""location"": ""IL"", ""is_mobile"": true}" 12946,5,289,2017-04-27 09:15:20,http://kunze.name/jarrod_rosenbaum,0.7540004096,124.49.241.220,"{""location"": ""AO"", ""is_mobile"": true}" 12947,5,289,2017-06-12 03:23:53,http://luettgen.com/lori_fahey,0.6781993300,96.123.158.170,"{""location"": ""MG"", ""is_mobile"": true}" 12948,5,289,2016-12-20 13:16:04,http://kuhic.com/raquel,0.9492554153,160.127.174.7,"{""location"": ""TZ"", ""is_mobile"": false}" 12949,5,289,2017-03-30 12:44:21,http://lynchpollich.co/jensen,0.7626915552,42.226.115.229,"{""location"": ""AU"", ""is_mobile"": false}" 12950,5,289,2017-04-19 07:58:36,http://wiza.com/lamont,0.2380596080,28.70.136.96,"{""location"": ""CW"", ""is_mobile"": false}" 12951,5,290,2017-02-09 09:53:06,http://moriette.net/shane,0.5720359253,46.207.70.183,"{""location"": ""GA"", ""is_mobile"": true}" 12952,5,290,2017-02-04 11:15:57,http://zemlak.name/obie,0.7425776544,167.209.138.234,"{""location"": ""GM"", ""is_mobile"": false}" 12953,5,290,2017-06-01 14:35:32,http://price.io/domenico_monahan,0.6072363176,224.90.243.143,"{""location"": ""HK"", ""is_mobile"": false}" 12954,5,290,2017-06-09 23:45:03,http://marvin.co/abigail,0.5257599185,156.217.240.44,"{""location"": ""BQ"", ""is_mobile"": false}" 12955,5,290,2017-01-10 00:26:11,http://wardwolff.net/chelsie_dickinson,0.4880739788,115.96.57.162,"{""location"": ""CL"", ""is_mobile"": true}" 12956,5,290,2016-12-31 05:43:36,http://crookshuel.info/mollie_hammes,0.1974298713,109.186.10.23,"{""location"": ""FR"", ""is_mobile"": true}" 12957,5,290,2017-04-07 19:06:29,http://west.io/mallory_mitchell,0.1910741705,11.223.58.33,"{""location"": ""DM"", ""is_mobile"": true}" 12958,5,290,2017-03-22 07:32:20,http://carrolltillman.co/luisa,0.1249702950,244.244.32.253,"{""location"": ""CY"", ""is_mobile"": true}" 12959,5,290,2017-02-28 04:16:13,http://reichel.biz/jacky,0.3684256531,217.72.83.73,"{""location"": ""GP"", ""is_mobile"": false}" 12960,5,290,2017-06-13 06:45:31,http://bernhardklocko.org/delta_nolan,0.4919868023,172.15.243.206,"{""location"": ""GN"", ""is_mobile"": false}" 12961,5,290,2017-02-22 05:07:59,http://nolanfisher.net/stefan_bartoletti,0.0055156304,242.149.248.251,"{""location"": ""TD"", ""is_mobile"": false}" 12962,5,290,2017-04-20 16:28:14,http://leuschkecummings.net/margarette.reynolds,0.8307134822,173.197.73.230,"{""location"": ""AI"", ""is_mobile"": false}" 12963,5,290,2017-05-15 04:33:27,http://mayert.org/elian,0.3434131551,244.96.214.5,"{""location"": ""BW"", ""is_mobile"": false}" 15916,6,358,2017-04-06 19:34:07,http://witting.io/davin.wehner,,22.75.102.111,"{""location"": ""JE"", ""is_mobile"": true}" 15917,6,358,2017-04-21 07:17:19,http://krajcik.biz/ada.leffler,,108.234.35.24,"{""location"": ""SS"", ""is_mobile"": false}" 15918,6,358,2017-02-07 15:04:36,http://jaskolski.org/abraham,,122.103.94.178,"{""location"": ""SB"", ""is_mobile"": true}" 15919,6,358,2017-01-05 18:01:50,http://stokes.biz/brian_beahan,,43.117.27.186,"{""location"": ""CM"", ""is_mobile"": false}" 15920,6,358,2017-05-08 09:49:12,http://kreiger.info/katherine,,252.218.42.147,"{""location"": ""ML"", ""is_mobile"": false}" 15921,6,358,2017-02-27 14:32:52,http://gorczany.info/cheyenne_schmidt,,203.166.131.110,"{""location"": ""CR"", ""is_mobile"": false}" 15922,6,358,2017-05-12 17:26:59,http://simonis.io/esperanza,,182.228.182.53,"{""location"": ""SX"", ""is_mobile"": true}" 15923,6,358,2016-12-15 03:24:19,http://feil.co/nya,,141.245.87.54,"{""location"": ""CY"", ""is_mobile"": false}" 15924,6,358,2017-03-27 13:47:16,http://feeney.net/lexi_boyle,,108.207.85.139,"{""location"": ""PA"", ""is_mobile"": true}" 15925,6,358,2017-02-21 22:54:59,http://beatty.info/vena.wehner,,193.99.65.214,"{""location"": ""MG"", ""is_mobile"": true}" 15926,6,358,2017-01-01 20:49:42,http://waelchi.org/hillard,,176.20.240.216,"{""location"": ""BI"", ""is_mobile"": true}" 15927,6,358,2017-03-25 09:37:18,http://ziemann.info/kathryn,,195.203.155.147,"{""location"": ""BB"", ""is_mobile"": true}" 15928,6,358,2017-05-11 14:27:21,http://price.org/laron.brakus,,126.24.60.176,"{""location"": ""ES"", ""is_mobile"": false}" 15929,6,358,2017-04-02 22:02:35,http://nicolas.co/melyna.zemlak,,230.131.61.44,"{""location"": ""ES"", ""is_mobile"": true}" 15930,6,358,2017-01-22 10:51:49,http://mayerhintz.info/rylee,,59.94.207.175,"{""location"": ""GW"", ""is_mobile"": true}" 15931,6,359,2017-02-02 12:52:05,http://dickens.net/cicero_paucek,,161.89.122.59,"{""location"": ""CV"", ""is_mobile"": false}" 15932,6,359,2017-05-31 13:30:51,http://hansen.net/christian,,114.187.179.174,"{""location"": ""WS"", ""is_mobile"": false}" 15933,6,359,2017-03-21 15:20:53,http://osinski.org/filomena,,233.70.95.120,"{""location"": ""BZ"", ""is_mobile"": false}" 15934,6,359,2017-03-13 22:46:15,http://kshlerin.name/alvah.sawayn,,52.253.213.211,"{""location"": ""TV"", ""is_mobile"": false}" 15935,6,359,2017-01-31 07:35:52,http://larkin.co/giles_damore,,244.117.113.110,"{""location"": ""OM"", ""is_mobile"": true}" 15936,6,359,2017-04-23 04:51:54,http://hettinger.name/beverly.kuhic,,125.245.159.127,"{""location"": ""UY"", ""is_mobile"": false}" 15937,6,359,2017-05-08 07:56:08,http://bechtelar.info/lelia.stracke,,44.234.183.96,"{""location"": ""ZA"", ""is_mobile"": false}" 15938,6,359,2017-05-19 12:26:38,http://yundt.co/kaci_bosco,,97.190.65.99,"{""location"": ""GI"", ""is_mobile"": true}" 15939,6,359,2016-12-15 12:56:49,http://lindbins.io/elwyn,,175.140.157.129,"{""location"": ""GT"", ""is_mobile"": true}" 15940,6,359,2016-12-27 23:53:43,http://blockkilback.io/obie,,109.51.186.148,"{""location"": ""PL"", ""is_mobile"": false}" 15941,6,359,2017-03-07 19:37:37,http://maggiojerde.info/rebeka,,195.242.236.6,"{""location"": ""GL"", ""is_mobile"": false}" 15942,6,359,2017-01-27 08:17:39,http://dare.com/chris,,129.176.17.70,"{""location"": ""ZW"", ""is_mobile"": true}" 15943,6,359,2017-06-01 02:06:24,http://lowe.org/courtney,,230.200.168.23,"{""location"": ""CN"", ""is_mobile"": false}" 15944,6,359,2016-12-20 13:06:46,http://hilll.co/vada,,236.80.224.225,"{""location"": ""SY"", ""is_mobile"": false}" 15945,6,359,2016-12-31 01:18:09,http://hyattmayer.biz/brandt_jacobs,,241.153.73.249,"{""location"": ""CW"", ""is_mobile"": true}" 15946,6,359,2017-04-07 04:40:42,http://mannschaefer.info/laury,,122.40.100.193,"{""location"": ""LK"", ""is_mobile"": true}" 15947,6,359,2017-05-10 14:25:42,http://mueller.name/kali_nienow,,213.169.214.134,"{""location"": ""SN"", ""is_mobile"": true}" 15948,6,359,2017-04-14 05:55:45,http://cormier.com/jarod_konopelski,,69.123.116.123,"{""location"": ""SO"", ""is_mobile"": true}" 15949,6,359,2017-06-10 07:28:51,http://hahn.net/milo,,206.153.176.254,"{""location"": ""FK"", ""is_mobile"": true}" 15950,6,359,2017-06-03 01:44:56,http://cronafisher.io/lempi_jacobi,,2.222.158.104,"{""location"": ""UG"", ""is_mobile"": false}" 15951,6,359,2017-04-06 19:13:06,http://kunde.com/tate,,230.77.99.186,"{""location"": ""PR"", ""is_mobile"": true}" 15952,6,359,2017-05-08 13:55:42,http://senger.name/brooke_sauer,,194.201.180.56,"{""location"": ""GQ"", ""is_mobile"": true}" 15953,6,359,2017-06-09 08:29:30,http://swift.io/carmen.stokes,,164.174.146.226,"{""location"": ""BY"", ""is_mobile"": false}" 15954,6,359,2017-05-28 22:35:28,http://swaniawski.io/imani_paucek,,133.117.50.250,"{""location"": ""NC"", ""is_mobile"": false}" 15955,6,359,2017-02-17 13:00:33,http://dietrich.info/estelle,,53.227.232.95,"{""location"": ""BV"", ""is_mobile"": true}" 15956,6,359,2017-02-07 03:22:54,http://walsh.name/meredith,,201.101.180.194,"{""location"": ""MZ"", ""is_mobile"": true}" 15957,6,359,2017-04-29 03:33:08,http://greenfelder.biz/aida.zulauf,,83.152.129.93,"{""location"": ""BE"", ""is_mobile"": false}" 17071,6,386,2017-01-09 03:31:15,http://spinka.org/cecile,,160.230.207.86,"{""location"": ""UM"", ""is_mobile"": true}" 15958,6,359,2017-01-06 02:14:53,http://schimmelcrona.name/monserrat,,164.71.57.119,"{""location"": ""AT"", ""is_mobile"": true}" 15959,6,359,2017-01-09 20:31:20,http://hackett.biz/carmelo.jast,,254.166.241.238,"{""location"": ""TJ"", ""is_mobile"": true}" 15960,6,359,2017-01-17 09:04:17,http://stamm.co/demond.schinner,,128.68.101.25,"{""location"": ""SD"", ""is_mobile"": true}" 15961,6,359,2017-02-22 22:05:13,http://mohrnikolaus.net/stewart,,212.57.235.159,"{""location"": ""BA"", ""is_mobile"": true}" 15962,6,359,2017-02-06 12:46:35,http://lemke.biz/beryl,,217.164.133.8,"{""location"": ""AM"", ""is_mobile"": true}" 15963,6,359,2017-03-03 14:30:23,http://tillman.co/loyce,,81.170.33.137,"{""location"": ""BG"", ""is_mobile"": false}" 15964,6,359,2017-01-07 14:04:56,http://cremin.com/tiffany,,67.2.188.11,"{""location"": ""FO"", ""is_mobile"": true}" 15965,6,359,2017-03-16 17:58:40,http://kutch.biz/carole.yost,,229.57.133.124,"{""location"": ""BY"", ""is_mobile"": false}" 15966,6,359,2017-02-07 21:22:17,http://hegmann.com/tamara,,56.184.178.76,"{""location"": ""HK"", ""is_mobile"": true}" 15967,6,359,2017-04-17 02:02:23,http://breitenberg.org/tiara,,93.141.29.110,"{""location"": ""CX"", ""is_mobile"": true}" 15968,6,359,2017-01-23 02:24:13,http://wyman.info/clara,,77.13.203.20,"{""location"": ""KG"", ""is_mobile"": true}" 15969,6,359,2017-02-28 12:11:10,http://stiedemann.io/floyd,,171.17.187.179,"{""location"": ""GH"", ""is_mobile"": true}" 15970,6,359,2017-01-27 18:00:32,http://stiedemann.net/jake,,139.163.27.73,"{""location"": ""NA"", ""is_mobile"": false}" 3112,2,68,2017-04-12 17:13:19,http://denesik.info/kristopher.bogan,0.5974878075,203.21.50.243,"{""location"": ""IO"", ""is_mobile"": true}" 3113,2,68,2017-04-23 10:02:43,http://breitenbergweinat.name/domenick,0.1911644382,53.237.245.250,"{""location"": ""FI"", ""is_mobile"": true}" 3114,2,69,2017-03-25 15:13:24,http://hanerolfson.info/kaleb,0.9996665033,157.212.243.59,"{""location"": ""PW"", ""is_mobile"": false}" 3115,2,69,2016-12-31 06:57:29,http://walkeroconner.net/aleandra,0.6722299459,184.12.169.26,"{""location"": ""SV"", ""is_mobile"": false}" 3116,2,69,2017-03-04 09:46:26,http://reinger.co/johathan.yost,0.7854829853,148.151.165.11,"{""location"": ""MV"", ""is_mobile"": false}" 3117,2,69,2017-02-03 16:50:50,http://hansen.io/carlie_hamill,0.0863582474,67.177.89.42,"{""location"": ""TV"", ""is_mobile"": false}" 3118,2,69,2017-02-25 23:47:55,http://fay.info/jaylon_senger,0.5206267978,67.32.159.143,"{""location"": ""FO"", ""is_mobile"": false}" 3119,2,69,2016-12-26 08:38:43,http://gleichnerlangworth.biz/tamara,0.3397541567,230.109.143.229,"{""location"": ""MR"", ""is_mobile"": false}" 3120,2,69,2016-12-19 07:28:38,http://mantestiedemann.com/deja_kshlerin,0.4346794280,109.44.215.139,"{""location"": ""CR"", ""is_mobile"": false}" 3121,2,69,2017-06-01 03:55:19,http://bradtke.biz/garry,0.5734775743,47.175.65.247,"{""location"": ""HU"", ""is_mobile"": false}" 3122,2,69,2017-05-18 02:11:03,http://johnson.net/daren,0.4229125511,43.225.250.117,"{""location"": ""PY"", ""is_mobile"": false}" 3123,2,69,2017-06-01 09:19:24,http://lakin.net/jasper,0.9772882885,220.197.155.202,"{""location"": ""AE"", ""is_mobile"": false}" 3124,2,69,2017-03-24 19:10:38,http://hansenbrekke.org/boris,0.5008544086,2.157.174.200,"{""location"": ""PR"", ""is_mobile"": false}" 3125,2,69,2017-02-14 13:08:00,http://macejkovic.io/johan_herman,0.2307843963,165.142.45.3,"{""location"": ""BF"", ""is_mobile"": false}" 3126,2,69,2017-06-04 08:27:48,http://champlin.name/elroy_ernser,0.9662507808,22.163.39.128,"{""location"": ""FI"", ""is_mobile"": false}" 3127,2,69,2017-02-17 05:02:39,http://spinkagrimes.biz/dillan,0.6342897693,186.204.52.137,"{""location"": ""RO"", ""is_mobile"": true}" 3128,2,69,2017-02-25 02:25:36,http://mrazkoelpin.net/norene,0.2252231497,125.6.30.134,"{""location"": ""BH"", ""is_mobile"": false}" 3129,2,69,2017-03-10 03:58:07,http://wisozk.info/hermina,0.5488085094,147.139.208.176,"{""location"": ""AL"", ""is_mobile"": true}" 3130,2,69,2017-03-27 14:34:00,http://stroman.co/luciano,0.8465012278,132.187.19.90,"{""location"": ""PM"", ""is_mobile"": true}" 3131,2,69,2017-05-27 21:36:05,http://huels.co/weston.swaniawski,0.9839823085,33.72.161.166,"{""location"": ""AO"", ""is_mobile"": true}" 3132,2,69,2017-01-09 14:30:55,http://skilesbahringer.name/scarlett_lueilwitz,0.4501433862,224.189.14.6,"{""location"": ""SV"", ""is_mobile"": true}" 3133,2,69,2016-12-20 15:39:33,http://klein.io/paris,0.8083949107,155.252.250.109,"{""location"": ""LT"", ""is_mobile"": true}" 3134,2,69,2017-02-09 04:40:27,http://beier.com/nils_walsh,0.8604351129,34.93.24.185,"{""location"": ""SI"", ""is_mobile"": true}" 3135,2,69,2017-02-15 11:01:15,http://kulasfranecki.info/lavon,0.3847863653,57.134.131.21,"{""location"": ""FO"", ""is_mobile"": false}" 3136,2,69,2017-06-05 18:05:55,http://kihngerhold.com/margarete_strosin,0.7769132799,169.224.54.146,"{""location"": ""CH"", ""is_mobile"": true}" 3137,2,69,2016-12-17 03:52:07,http://murazik.com/garnet,0.5502466655,52.174.40.82,"{""location"": ""SA"", ""is_mobile"": false}" 3138,2,69,2016-12-30 14:35:54,http://deckow.org/kenny_gislason,0.7484658726,251.243.174.45,"{""location"": ""TC"", ""is_mobile"": true}" 3139,2,69,2017-04-28 22:48:00,http://gerlach.biz/briana_cole,0.0539582506,11.81.87.54,"{""location"": ""ME"", ""is_mobile"": false}" 3140,2,69,2017-02-04 05:53:35,http://purdy.net/paul,0.9650374810,166.70.119.217,"{""location"": ""SO"", ""is_mobile"": true}" 3141,2,69,2017-03-10 18:51:51,http://mayermcdermott.co/kathlyn_bernier,0.8205558098,24.87.22.103,"{""location"": ""CN"", ""is_mobile"": false}" 3142,2,69,2017-05-05 16:22:54,http://hermiston.co/myrtie,0.9125203508,183.120.139.243,"{""location"": ""SY"", ""is_mobile"": false}" 3143,2,69,2017-02-08 13:53:45,http://hyatt.org/reece.pfeffer,0.3757934771,201.229.159.89,"{""location"": ""GH"", ""is_mobile"": false}" 3144,2,69,2017-04-28 05:48:18,http://carter.com/eliza,0.6419280306,109.144.72.23,"{""location"": ""JE"", ""is_mobile"": false}" 3145,2,69,2016-12-25 02:38:11,http://waelchi.io/robbie_aufderhar,0.8109047527,35.147.23.97,"{""location"": ""LB"", ""is_mobile"": false}" 3146,2,69,2016-12-23 22:59:32,http://framiabshire.net/alfonzo,0.9589649718,170.164.231.237,"{""location"": ""KW"", ""is_mobile"": true}" 3147,2,69,2017-05-21 03:25:40,http://harvey.name/cornell,0.0195763306,15.155.75.103,"{""location"": ""MY"", ""is_mobile"": false}" 3148,2,69,2017-01-24 15:26:30,http://klocko.io/stephan_wolff,0.8668169186,108.15.249.17,"{""location"": ""MT"", ""is_mobile"": false}" 3149,2,69,2016-12-22 19:04:25,http://hayes.io/gilbert.roberts,0.8626886040,235.68.212.180,"{""location"": ""AM"", ""is_mobile"": true}" 3150,2,69,2017-01-16 06:48:25,http://damore.name/madie.legros,0.8404784869,243.191.251.89,"{""location"": ""NP"", ""is_mobile"": true}" 3151,2,69,2017-05-18 21:40:57,http://hamill.info/george,0.3961158672,2.121.81.64,"{""location"": ""AF"", ""is_mobile"": false}" 3152,2,69,2017-04-17 15:39:15,http://runteschuppe.biz/gretchen.roberts,0.5894416392,29.178.248.109,"{""location"": ""KZ"", ""is_mobile"": false}" 3153,2,69,2017-02-26 07:40:31,http://mills.com/myles,0.6909378060,143.103.225.98,"{""location"": ""SS"", ""is_mobile"": true}" 3154,2,69,2017-03-15 12:35:55,http://feest.io/norbert,0.1219484846,101.250.60.181,"{""location"": ""DE"", ""is_mobile"": false}" 3155,2,69,2016-12-27 04:03:09,http://sawayn.name/walker,0.5429478769,164.56.114.216,"{""location"": ""AW"", ""is_mobile"": true}" 3156,2,69,2017-06-04 15:53:42,http://murray.io/reece_koepp,0.1305136170,152.151.43.91,"{""location"": ""BR"", ""is_mobile"": true}" 3157,2,69,2017-01-10 20:11:58,http://ryan.io/colt_vandervort,0.5289830917,8.145.178.242,"{""location"": ""NL"", ""is_mobile"": false}" 3158,2,69,2017-02-07 00:52:35,http://beahan.com/kareem,0.9830014039,106.125.96.44,"{""location"": ""VU"", ""is_mobile"": false}" 3159,2,69,2017-03-26 14:33:01,http://cruickshank.info/wyatt,0.1436772183,15.24.196.54,"{""location"": ""MF"", ""is_mobile"": false}" 3160,2,70,2017-02-24 03:19:59,http://kilback.info/wallace,0.0501336203,166.17.141.53,"{""location"": ""MO"", ""is_mobile"": false}" 3161,2,70,2017-05-07 15:07:47,http://champlinwhite.name/maximillia,0.9703294405,157.221.235.125,"{""location"": ""QA"", ""is_mobile"": false}" 3162,2,70,2017-01-08 04:38:40,http://runolfon.com/austen_johns,0.3816768533,168.104.140.225,"{""location"": ""NC"", ""is_mobile"": false}" 3163,2,70,2017-01-22 03:23:42,http://kuphal.co/raheem,0.3158470806,223.251.55.233,"{""location"": ""QA"", ""is_mobile"": true}" 10047,4,225,2017-05-02 16:06:59,http://armstrong.co/ephraim_kuphal,0.8963865325,55.29.160.210,"{""location"": ""GH"", ""is_mobile"": true}" 10048,4,225,2017-04-22 03:13:51,http://schaden.com/jonathan_bayer,0.4939406247,47.225.162.87,"{""location"": ""ET"", ""is_mobile"": false}" 10049,4,225,2017-03-28 12:10:03,http://morarpredovic.org/braeden,0.3607602004,20.254.56.59,"{""location"": ""JP"", ""is_mobile"": false}" 10050,4,225,2017-05-24 02:09:54,http://purdy.io/lavonne,0.6901845225,89.210.76.176,"{""location"": ""BH"", ""is_mobile"": true}" 10051,4,225,2017-04-11 15:36:49,http://brakus.biz/khalid,0.5376315138,120.122.178.58,"{""location"": ""SX"", ""is_mobile"": true}" 10052,4,225,2017-03-27 11:39:24,http://dibbert.io/johan,0.4527326973,142.150.27.28,"{""location"": ""HR"", ""is_mobile"": true}" 10053,4,225,2017-03-30 01:48:12,http://pouros.info/pete_hand,0.8423703205,196.143.119.199,"{""location"": ""HU"", ""is_mobile"": true}" 10054,4,225,2017-05-07 08:17:13,http://schimmel.com/melisa_gorczany,0.1614912784,194.213.156.176,"{""location"": ""QA"", ""is_mobile"": false}" 10055,4,225,2017-01-16 23:32:39,http://barrows.co/domenico.kunze,0.0590944245,3.153.18.135,"{""location"": ""AZ"", ""is_mobile"": false}" 10056,4,225,2017-01-02 04:04:14,http://lueilwitz.com/drake_von,0.9083508914,128.214.219.78,"{""location"": ""FO"", ""is_mobile"": true}" 10057,4,225,2017-04-20 21:48:30,http://howell.co/eladio_trantow,0.4691959655,147.163.45.139,"{""location"": ""PL"", ""is_mobile"": false}" 10058,4,225,2017-03-27 11:09:43,http://rohan.biz/kaylie,0.6507741491,225.199.50.127,"{""location"": ""KH"", ""is_mobile"": false}" 10059,4,225,2017-05-10 01:35:40,http://swaniawski.biz/anya,0.0159485417,72.217.140.93,"{""location"": ""MO"", ""is_mobile"": false}" 10060,4,225,2017-06-05 09:06:22,http://moore.org/deshawn,0.6421662535,95.242.60.145,"{""location"": ""ZA"", ""is_mobile"": false}" 10061,4,225,2017-02-14 21:06:35,http://aufderharwiegand.com/craig,0.1884342827,109.252.215.28,"{""location"": ""SB"", ""is_mobile"": false}" 10062,4,225,2017-01-18 19:55:45,http://faheyanderson.biz/aleandro,0.9759867905,158.154.39.83,"{""location"": ""RW"", ""is_mobile"": false}" 10063,4,225,2017-02-12 22:49:37,http://auer.name/maxwell,0.6447699111,6.100.99.33,"{""location"": ""LI"", ""is_mobile"": false}" 10064,4,225,2017-04-01 06:51:53,http://rolfson.net/audie,0.3148405542,202.52.235.66,"{""location"": ""PY"", ""is_mobile"": false}" 10065,4,225,2017-06-02 10:57:37,http://bernhard.org/bryon.casper,0.6136907033,153.38.140.20,"{""location"": ""ME"", ""is_mobile"": true}" 10066,4,225,2017-03-18 16:19:24,http://cormier.io/adella.auer,0.5284071008,129.238.108.210,"{""location"": ""IO"", ""is_mobile"": true}" 10067,4,225,2017-01-20 21:07:30,http://prohaskatowne.info/merlin.abernathy,0.1465674256,68.62.21.239,"{""location"": ""TT"", ""is_mobile"": false}" 10068,4,225,2017-05-13 10:08:32,http://jenkinsdubuque.net/delfina,0.9093268155,128.95.59.90,"{""location"": ""TR"", ""is_mobile"": true}" 10069,4,226,2017-02-21 21:56:36,http://gusikowski.org/caden,0.0927987991,206.208.143.37,"{""location"": ""SH"", ""is_mobile"": true}" 10070,4,226,2017-01-21 15:55:02,http://goodwin.name/lillie_hintz,0.8965432268,81.105.163.248,"{""location"": ""KY"", ""is_mobile"": true}" 10071,4,226,2017-01-09 20:12:35,http://barton.co/griffin,0.2903004413,151.78.107.195,"{""location"": ""CF"", ""is_mobile"": true}" 10072,4,226,2017-02-24 05:02:18,http://hyattbogisich.biz/clotilde,0.8055224888,14.133.6.216,"{""location"": ""GW"", ""is_mobile"": false}" 10073,4,226,2017-05-03 18:15:53,http://binshoppe.info/annabel,0.5319501797,214.57.87.129,"{""location"": ""PT"", ""is_mobile"": true}" 10074,4,226,2017-01-26 14:44:14,http://torphyblock.info/urban.kohler,0.7789587729,30.79.141.97,"{""location"": ""GL"", ""is_mobile"": false}" 10075,4,226,2017-05-29 10:11:00,http://cormier.name/ruby,0.0202851091,48.107.226.5,"{""location"": ""BA"", ""is_mobile"": true}" 10076,4,226,2017-05-07 19:16:44,http://pollich.biz/gudrun,0.1514022032,165.39.79.87,"{""location"": ""KI"", ""is_mobile"": false}" 10077,4,226,2017-02-26 01:13:50,http://bogisichgutkowski.org/jose,0.4710720278,184.124.59.236,"{""location"": ""AL"", ""is_mobile"": true}" 10078,4,226,2017-01-25 02:22:23,http://mraz.name/sigmund,0.3581254307,49.8.148.227,"{""location"": ""AM"", ""is_mobile"": false}" 10079,4,226,2016-12-23 03:41:56,http://hermiston.name/toni.runte,0.9604485637,246.82.150.158,"{""location"": ""KZ"", ""is_mobile"": false}" 10080,4,226,2017-03-21 11:17:38,http://bruenschuppe.biz/adah,0.1126660183,242.129.254.204,"{""location"": ""CR"", ""is_mobile"": false}" 10081,4,226,2017-03-02 03:55:33,http://kelerfay.com/gerardo.herman,0.1366062928,50.252.49.178,"{""location"": ""LT"", ""is_mobile"": true}" 10082,4,226,2017-03-18 07:20:19,http://langosh.io/camilla,0.6457041416,79.163.207.160,"{""location"": ""PY"", ""is_mobile"": false}" 10083,4,226,2017-01-06 06:43:07,http://thompson.name/linnea_grady,0.1530135510,183.94.30.97,"{""location"": ""JP"", ""is_mobile"": false}" 10084,4,226,2017-02-17 06:02:39,http://tillmandare.biz/tyson,0.1753918850,3.40.30.179,"{""location"": ""KW"", ""is_mobile"": true}" 10085,4,226,2017-05-16 07:07:18,http://gibsonkuphal.name/sofia,0.8340643466,199.231.218.121,"{""location"": ""GU"", ""is_mobile"": true}" 10086,4,226,2017-02-02 12:01:19,http://christiansenkutch.org/greyson.casper,0.1945193331,59.132.144.74,"{""location"": ""BO"", ""is_mobile"": true}" 10087,4,226,2017-04-09 17:18:35,http://davis.org/tyler,0.4832460840,245.71.135.108,"{""location"": ""RU"", ""is_mobile"": true}" 10088,4,226,2017-01-30 20:48:09,http://gulgowski.net/marguerite,0.2826123820,20.139.118.20,"{""location"": ""DO"", ""is_mobile"": true}" 10089,4,226,2017-01-14 23:49:06,http://swaniawski.name/mina.johnson,0.6488654431,195.38.210.177,"{""location"": ""SN"", ""is_mobile"": true}" 10090,4,226,2017-05-31 20:50:05,http://satterfield.name/tracey,0.6287612171,133.247.159.43,"{""location"": ""TT"", ""is_mobile"": true}" 10091,4,226,2017-01-09 05:25:55,http://lowe.co/enid_graham,0.5150062857,148.129.167.159,"{""location"": ""RS"", ""is_mobile"": true}" 10092,4,226,2017-02-02 04:48:26,http://cummings.org/carolanne,0.9387358654,230.199.31.232,"{""location"": ""MM"", ""is_mobile"": true}" 10093,4,226,2017-02-25 20:45:04,http://terry.net/erna.mann,0.7581702070,126.173.180.19,"{""location"": ""PE"", ""is_mobile"": false}" 10094,4,227,2016-12-22 01:14:28,http://cartwrightschroeder.net/meaghan_lynch,0.3109966382,108.74.88.239,"{""location"": ""WS"", ""is_mobile"": true}" 10095,4,227,2017-05-10 07:51:25,http://mertz.net/rex,0.4515466504,32.76.133.90,"{""location"": ""TZ"", ""is_mobile"": true}" 10096,4,227,2017-02-09 20:19:43,http://bins.net/bonita,0.1717030152,197.20.225.164,"{""location"": ""ER"", ""is_mobile"": false}" 10097,4,227,2017-02-08 10:56:21,http://gislason.info/claud,0.7552811053,67.155.122.181,"{""location"": ""KR"", ""is_mobile"": false}" 7056,3,156,2017-05-09 16:04:03,http://ricebatz.com/flavie,0.5967866412,152.146.215.47,"{""location"": ""CV"", ""is_mobile"": true}" 7057,3,156,2017-02-01 08:36:27,http://gleasonweinat.com/bernice.murphy,0.0333287868,16.172.80.162,"{""location"": ""MH"", ""is_mobile"": true}" 7058,3,157,2017-05-25 23:48:55,http://walker.net/aleen,0.0567311750,251.102.64.231,"{""location"": ""PY"", ""is_mobile"": true}" 7059,3,157,2017-04-27 09:44:07,http://considine.biz/anibal,0.9366841954,176.177.160.109,"{""location"": ""PL"", ""is_mobile"": false}" 7060,3,157,2017-03-14 18:01:54,http://reichel.net/hilbert,0.1029688468,154.121.226.131,"{""location"": ""GI"", ""is_mobile"": false}" 7061,3,157,2017-06-09 03:49:09,http://davis.biz/ansel,0.5609859823,187.250.139.67,"{""location"": ""BJ"", ""is_mobile"": false}" 7062,3,157,2016-12-28 02:19:02,http://kub.name/kaylin_strosin,0.0128193576,42.213.211.116,"{""location"": ""MG"", ""is_mobile"": false}" 7063,3,157,2017-05-02 03:00:49,http://hoegerbeatty.net/briana,0.1550153762,12.86.16.136,"{""location"": ""TF"", ""is_mobile"": false}" 7064,3,157,2017-01-27 12:48:48,http://powlowskiwitting.org/raleigh,0.4915111190,96.95.158.162,"{""location"": ""RO"", ""is_mobile"": true}" 7065,3,157,2017-04-25 18:58:56,http://schulist.io/ardella,0.6182895478,113.241.166.113,"{""location"": ""LV"", ""is_mobile"": true}" 7066,3,157,2017-01-01 18:14:19,http://weber.io/jermain.nolan,0.3777212918,160.135.157.73,"{""location"": ""GQ"", ""is_mobile"": true}" 7067,3,157,2016-12-30 22:15:33,http://lednersporer.biz/casper,0.8325869696,17.23.38.98,"{""location"": ""TJ"", ""is_mobile"": true}" 7068,3,157,2017-02-19 13:32:30,http://effertz.net/deangelo_pollich,0.5471904599,88.244.35.187,"{""location"": ""NC"", ""is_mobile"": false}" 7069,3,157,2017-02-08 01:44:55,http://champlin.info/addie,0.9712519353,96.30.140.194,"{""location"": ""IQ"", ""is_mobile"": false}" 7070,3,157,2017-05-07 06:34:27,http://ward.biz/everardo,0.0230003880,142.234.112.130,"{""location"": ""BD"", ""is_mobile"": false}" 7071,3,157,2017-04-04 08:44:04,http://frami.com/ottilie,0.8194871738,74.199.209.38,"{""location"": ""SB"", ""is_mobile"": false}" 7072,3,157,2017-03-21 11:22:02,http://muellerhettinger.info/felton,0.1512753199,23.139.119.35,"{""location"": ""SG"", ""is_mobile"": true}" 7073,3,157,2016-12-26 20:30:02,http://orn.info/clement,0.2130948479,165.12.116.6,"{""location"": ""TZ"", ""is_mobile"": false}" 7074,3,157,2017-04-16 10:10:56,http://collins.co/ron,0.0885804718,160.19.204.239,"{""location"": ""TC"", ""is_mobile"": false}" 7075,3,157,2017-05-14 02:40:40,http://schultz.co/estella.osinski,0.6037884710,65.130.186.183,"{""location"": ""BS"", ""is_mobile"": true}" 7076,3,157,2017-05-22 19:42:26,http://jast.name/maximilian_herzog,0.2378929184,206.39.33.102,"{""location"": ""PM"", ""is_mobile"": true}" 7077,3,157,2017-03-04 09:45:03,http://veum.org/dannie.hilll,0.1994470863,25.168.12.106,"{""location"": ""VU"", ""is_mobile"": true}" 7078,3,157,2017-03-19 16:42:14,http://mckenzie.net/madison,0.7149027762,223.158.100.241,"{""location"": ""KN"", ""is_mobile"": false}" 7079,3,157,2017-04-15 20:45:08,http://goyette.io/bernhard.will,0.2188010941,98.154.194.120,"{""location"": ""AM"", ""is_mobile"": false}" 7080,3,157,2017-05-25 06:24:36,http://hanewitting.biz/lew,0.4440808125,94.186.215.96,"{""location"": ""ML"", ""is_mobile"": true}" 7081,3,157,2017-05-17 12:49:41,http://reynoldtark.com/breanna.lockman,0.5481628532,220.132.154.202,"{""location"": ""GQ"", ""is_mobile"": true}" 7082,3,157,2017-03-02 17:29:53,http://conroy.co/keira,0.4594788193,4.49.48.55,"{""location"": ""SJ"", ""is_mobile"": true}" 7083,3,157,2017-04-23 12:02:01,http://purdyokon.biz/ettie.hintz,0.2319666502,179.92.14.9,"{""location"": ""GI"", ""is_mobile"": true}" 7084,3,157,2017-05-23 15:57:15,http://breitenberg.name/rebecca,0.5736828474,90.62.72.176,"{""location"": ""NP"", ""is_mobile"": true}" 7085,3,157,2017-06-03 22:40:05,http://hodkiewiczstoltenberg.io/rod,0.9617909479,43.103.222.200,"{""location"": ""BQ"", ""is_mobile"": false}" 7086,3,157,2017-04-28 11:41:52,http://feeney.co/florencio,0.7898441842,41.63.9.58,"{""location"": ""AS"", ""is_mobile"": false}" 7087,3,157,2017-01-18 07:52:22,http://ziemann.org/cory.langworth,0.7160168862,194.95.100.100,"{""location"": ""MH"", ""is_mobile"": true}" 7088,3,158,2017-05-20 09:19:50,http://denesik.co/hiram,0.5076915781,94.56.147.59,"{""location"": ""BD"", ""is_mobile"": true}" 7089,3,158,2017-03-13 12:42:50,http://kemmerhand.biz/damian,0.8946720664,126.49.115.48,"{""location"": ""UY"", ""is_mobile"": false}" 7090,3,158,2017-02-22 16:26:33,http://gerlach.name/lexi.keebler,0.3129572937,7.92.185.93,"{""location"": ""VU"", ""is_mobile"": true}" 7091,3,158,2017-02-08 11:00:30,http://fadel.io/linwood,0.1666701422,130.107.137.135,"{""location"": ""LY"", ""is_mobile"": true}" 7092,3,158,2017-04-02 08:42:00,http://hansen.co/sophie_moriette,0.4384174488,122.12.115.139,"{""location"": ""BH"", ""is_mobile"": false}" 7093,3,158,2017-02-11 07:57:51,http://daniel.info/zelda.schinner,0.7196864813,158.81.223.15,"{""location"": ""IO"", ""is_mobile"": true}" 7094,3,158,2017-02-22 09:44:23,http://metz.biz/emelie,0.1879753913,114.241.6.76,"{""location"": ""CO"", ""is_mobile"": false}" 7095,3,158,2017-03-29 00:19:18,http://zieme.name/howell_mitchell,0.3571206246,166.53.191.169,"{""location"": ""DM"", ""is_mobile"": false}" 7096,3,158,2017-02-07 08:42:11,http://kilback.co/elody,0.1070852815,51.203.90.105,"{""location"": ""AF"", ""is_mobile"": true}" 7097,3,158,2017-05-25 18:57:18,http://gutmann.com/may_donnelly,0.8081765585,112.229.215.173,"{""location"": ""SZ"", ""is_mobile"": true}" 7098,3,158,2017-01-21 18:46:27,http://doyle.org/vallie,0.2105464417,156.217.163.52,"{""location"": ""BD"", ""is_mobile"": true}" 7099,3,158,2016-12-19 14:21:02,http://boehm.biz/rebekah,0.6513311080,179.60.152.250,"{""location"": ""SD"", ""is_mobile"": true}" 7100,3,158,2017-05-22 18:18:58,http://jacobshoppe.net/hillard.koelpin,0.7822400916,216.103.188.13,"{""location"": ""TT"", ""is_mobile"": false}" 7101,3,158,2017-06-10 11:44:28,http://ruel.info/muriel.runte,0.2390961175,72.46.79.24,"{""location"": ""TM"", ""is_mobile"": false}" 7102,3,158,2017-03-02 02:24:51,http://brekke.biz/beth,0.8653283838,80.54.198.157,"{""location"": ""YT"", ""is_mobile"": true}" 7103,3,158,2017-01-28 12:55:22,http://brown.biz/adrain,0.6063683983,103.183.250.184,"{""location"": ""TJ"", ""is_mobile"": true}" 7104,3,158,2017-06-02 07:14:18,http://reynoldsko.name/brandt.littel,0.3558468345,36.179.152.180,"{""location"": ""SY"", ""is_mobile"": false}" 7105,3,158,2016-12-31 00:24:55,http://schiller.name/ahmed,0.6433073994,220.141.110.186,"{""location"": ""MF"", ""is_mobile"": true}" 7106,3,158,2017-02-01 13:26:51,http://feil.com/carolina_hahn,0.0453928478,219.209.241.127,"{""location"": ""ZW"", ""is_mobile"": true}" 7107,3,158,2017-01-31 20:46:02,http://kohler.co/orion,0.0830145086,121.102.204.182,"{""location"": ""HN"", ""is_mobile"": false}" 12964,5,290,2017-02-25 04:37:49,http://rempel.net/ernestine_tremblay,0.9370939059,86.3.77.69,"{""location"": ""BJ"", ""is_mobile"": false}" 12965,5,290,2017-06-12 04:56:05,http://okuneva.com/coy.nikolaus,0.2069425362,117.205.37.251,"{""location"": ""PL"", ""is_mobile"": true}" 12966,5,290,2017-04-17 21:44:31,http://ryan.com/micah,0.8543146560,143.242.33.158,"{""location"": ""FJ"", ""is_mobile"": false}" 12967,5,290,2017-01-22 14:13:39,http://weinat.org/camden,0.7648103820,3.232.48.85,"{""location"": ""GH"", ""is_mobile"": false}" 12968,5,290,2017-01-23 13:09:57,http://feil.org/hope,0.7705368696,80.98.128.128,"{""location"": ""SE"", ""is_mobile"": true}" 12969,5,290,2017-04-08 07:05:00,http://hackettmueller.name/garland,0.1323472432,190.243.88.181,"{""location"": ""SN"", ""is_mobile"": true}" 12970,5,290,2017-04-26 00:19:54,http://von.net/lilly,0.4373598389,152.174.36.181,"{""location"": ""FO"", ""is_mobile"": false}" 12971,5,290,2017-04-26 08:23:49,http://hillanford.co/kennith_hilll,0.5403922576,68.56.225.48,"{""location"": ""LT"", ""is_mobile"": true}" 12972,5,290,2017-02-06 13:32:38,http://erdmanhowell.io/elsie.anderson,0.0835851416,75.161.153.147,"{""location"": ""FJ"", ""is_mobile"": false}" 12973,5,291,2017-01-20 04:28:33,http://rolfson.info/bernardo,0.7182776696,82.211.7.6,"{""location"": ""NZ"", ""is_mobile"": true}" 12974,5,291,2017-03-04 17:57:52,http://bradtke.io/reece.ritchie,0.7306997775,115.206.160.244,"{""location"": ""JM"", ""is_mobile"": true}" 12975,5,291,2016-12-29 21:09:22,http://volkmankulas.name/rhea,0.1278940541,177.110.69.212,"{""location"": ""KR"", ""is_mobile"": false}" 12976,5,291,2017-04-07 08:07:05,http://mayert.net/emmalee,0.1709884575,248.115.111.243,"{""location"": ""HM"", ""is_mobile"": true}" 12977,5,291,2017-06-06 09:57:58,http://dubuquegrant.io/jennings.kuhlman,0.2704914706,36.103.76.86,"{""location"": ""ML"", ""is_mobile"": true}" 12978,5,291,2017-01-02 04:19:38,http://stoltenberg.org/sister,0.8889207650,54.89.191.229,"{""location"": ""AR"", ""is_mobile"": false}" 12979,5,291,2016-12-30 17:16:44,http://murazik.name/archibald,0.5748712531,81.152.148.52,"{""location"": ""RO"", ""is_mobile"": true}" 12980,5,291,2017-05-05 15:34:14,http://wisokybecker.com/cornell,0.6870723165,136.32.181.216,"{""location"": ""GI"", ""is_mobile"": false}" 12981,5,291,2017-01-20 19:51:29,http://hammes.biz/kallie,0.7803801234,119.226.193.149,"{""location"": ""MR"", ""is_mobile"": false}" 12982,5,291,2016-12-28 03:31:53,http://shields.name/easter,0.0275044990,73.241.207.190,"{""location"": ""CM"", ""is_mobile"": true}" 12983,5,291,2017-04-15 18:00:20,http://prohaskapadberg.co/kaela_mosciski,0.6975186117,49.190.225.203,"{""location"": ""CA"", ""is_mobile"": false}" 12984,5,291,2017-03-21 14:30:19,http://batz.com/marcella,0.3275682393,141.68.20.248,"{""location"": ""TD"", ""is_mobile"": true}" 12985,5,291,2017-06-11 03:54:13,http://dooleybechtelar.com/kaden.pagac,0.2337586066,66.147.173.199,"{""location"": ""AX"", ""is_mobile"": false}" 12986,5,291,2017-05-26 22:04:51,http://strosin.biz/judd,0.2687380016,240.131.156.147,"{""location"": ""MU"", ""is_mobile"": true}" 12987,5,291,2016-12-15 05:39:03,http://jacobsziemann.com/katlyn,0.4534504027,113.101.64.252,"{""location"": ""BZ"", ""is_mobile"": true}" 12988,5,291,2017-02-26 12:11:18,http://miller.name/derrick.herzog,0.0577833451,221.111.173.139,"{""location"": ""MX"", ""is_mobile"": true}" 12989,5,291,2017-02-08 01:27:18,http://graham.name/travon,0.2924272255,114.78.210.65,"{""location"": ""SI"", ""is_mobile"": false}" 12990,5,291,2017-01-31 07:50:18,http://millshauck.io/mike.bailey,0.1967106309,195.122.235.106,"{""location"": ""HU"", ""is_mobile"": false}" 12991,5,291,2017-01-10 20:07:54,http://runolfon.biz/kelton,0.2450870992,149.53.20.197,"{""location"": ""NI"", ""is_mobile"": false}" 12992,5,291,2017-02-05 10:31:30,http://lemke.org/rowland.runolfsdottir,0.6484380855,137.112.210.47,"{""location"": ""TF"", ""is_mobile"": true}" 12993,5,291,2017-02-25 14:02:26,http://cummings.net/lera,0.9569843695,134.45.57.187,"{""location"": ""LA"", ""is_mobile"": true}" 12994,5,291,2017-05-02 21:57:36,http://handkunze.io/philip.ankunding,0.7866881664,170.59.45.150,"{""location"": ""VC"", ""is_mobile"": false}" 12995,5,291,2016-12-28 03:52:34,http://bogisich.net/elyse,0.1908533126,177.247.240.36,"{""location"": ""UZ"", ""is_mobile"": true}" 12996,5,291,2017-05-05 22:12:02,http://adams.io/novella,0.9289694406,72.44.77.246,"{""location"": ""MX"", ""is_mobile"": false}" 12997,5,291,2017-04-03 13:32:45,http://bosco.net/florida_adams,0.9398514625,242.80.225.224,"{""location"": ""MS"", ""is_mobile"": true}" 12998,5,291,2016-12-13 08:57:37,http://nolanwalsh.org/shea.volkman,0.9808276364,142.240.122.74,"{""location"": ""CN"", ""is_mobile"": false}" 12999,5,292,2017-01-12 14:45:50,http://torphykeeling.com/golden.champlin,0.0878710696,2.54.57.52,"{""location"": ""CC"", ""is_mobile"": true}" 13000,5,292,2017-05-03 06:15:42,http://bartoletti.co/vickie_bauch,0.6314044983,89.142.146.102,"{""location"": ""CV"", ""is_mobile"": true}" 13001,5,292,2017-02-16 08:05:39,http://heller.com/vivian,0.3405306538,227.160.175.94,"{""location"": ""AQ"", ""is_mobile"": false}" 13002,5,292,2017-05-11 19:53:39,http://lindgren.net/clair.nolan,0.4701617972,69.136.248.126,"{""location"": ""BR"", ""is_mobile"": false}" 13003,5,292,2017-03-13 06:10:27,http://gerhold.info/hilton,0.4689550291,245.117.206.3,"{""location"": ""KP"", ""is_mobile"": true}" 13004,5,292,2017-01-03 16:15:15,http://stiedemannkuvalis.org/kavon_farrell,0.2092244725,75.228.108.115,"{""location"": ""VI"", ""is_mobile"": false}" 13005,5,292,2017-01-19 09:57:16,http://feest.net/dangelo,0.0946881106,101.188.23.49,"{""location"": ""AQ"", ""is_mobile"": false}" 13006,5,292,2017-01-26 05:19:14,http://mitchell.co/cara.graham,0.5972926003,68.103.238.184,"{""location"": ""SB"", ""is_mobile"": false}" 13007,5,292,2017-04-16 16:03:56,http://hodkiewicz.com/lexie_gislason,0.9448041806,97.42.245.27,"{""location"": ""IS"", ""is_mobile"": false}" 13008,5,292,2016-12-29 07:34:53,http://kilback.biz/lottie.sawayn,0.5688799455,235.218.92.43,"{""location"": ""MA"", ""is_mobile"": false}" 13009,5,292,2017-02-21 09:07:03,http://okuneva.com/davin.anderson,0.0179929344,142.125.230.76,"{""location"": ""SN"", ""is_mobile"": true}" 13010,5,292,2016-12-24 14:34:57,http://kovacek.co/jennie,0.8189158677,250.60.82.214,"{""location"": ""CF"", ""is_mobile"": true}" 13011,5,292,2017-03-26 08:24:52,http://littelberge.name/yasmeen.ledner,0.9280171973,90.243.144.239,"{""location"": ""GY"", ""is_mobile"": false}" 13012,5,292,2017-03-29 19:21:31,http://harris.biz/sylvester,0.2673627990,21.72.97.213,"{""location"": ""IR"", ""is_mobile"": false}" 13013,5,292,2017-04-05 19:57:41,http://padbergcasper.org/jaqueline,0.5038247888,125.28.41.161,"{""location"": ""GH"", ""is_mobile"": true}" 13014,5,292,2017-04-11 16:39:26,http://feest.io/armand,0.6134931194,106.145.147.229,"{""location"": ""PW"", ""is_mobile"": false}" 13015,5,292,2017-04-27 23:42:14,http://conroy.io/corine.weinat,0.8841835339,132.114.29.92,"{""location"": ""OM"", ""is_mobile"": true}" 15971,6,359,2017-04-25 23:22:05,http://damore.net/hudson_hahn,,205.240.209.108,"{""location"": ""FJ"", ""is_mobile"": true}" 15972,6,359,2017-01-09 16:49:27,http://luettgen.io/nathanial.west,,16.54.201.107,"{""location"": ""SD"", ""is_mobile"": false}" 15973,6,359,2017-05-24 10:25:41,http://lebsackkris.name/laila.feest,,159.96.74.12,"{""location"": ""WS"", ""is_mobile"": true}" 15974,6,359,2017-01-31 13:29:36,http://kihn.io/mallie_kuhlman,,78.190.119.43,"{""location"": ""AZ"", ""is_mobile"": false}" 15975,6,359,2017-01-23 00:51:52,http://ratkeweimann.info/esperanza,,41.28.238.239,"{""location"": ""SV"", ""is_mobile"": true}" 15976,6,359,2016-12-22 06:32:21,http://beatty.biz/bobby,,87.32.75.75,"{""location"": ""BH"", ""is_mobile"": false}" 15977,6,359,2017-01-10 00:55:21,http://tromp.com/remington.daniel,,80.213.30.149,"{""location"": ""FK"", ""is_mobile"": false}" 15978,6,359,2017-01-29 11:31:16,http://corwinadams.name/randall.mckenzie,,53.95.49.170,"{""location"": ""BW"", ""is_mobile"": true}" 15979,6,359,2017-02-01 06:20:34,http://reillybreitenberg.co/aric,,144.166.70.227,"{""location"": ""BB"", ""is_mobile"": false}" 15980,6,359,2017-01-28 13:02:41,http://larson.co/edmond_herzog,,154.42.41.126,"{""location"": ""MW"", ""is_mobile"": true}" 15981,6,359,2017-01-13 04:29:18,http://rice.biz/lilian,,31.82.173.106,"{""location"": ""CH"", ""is_mobile"": false}" 15982,6,359,2017-06-11 09:46:57,http://graham.com/kenneth,,149.188.56.94,"{""location"": ""TC"", ""is_mobile"": true}" 15983,6,359,2017-05-19 12:18:44,http://kuvalis.co/herman,,109.108.140.142,"{""location"": ""BL"", ""is_mobile"": false}" 15984,6,359,2017-05-17 07:58:55,http://schneider.biz/winston_muller,,98.17.232.11,"{""location"": ""JP"", ""is_mobile"": false}" 15985,6,360,2017-03-21 12:43:09,http://lowe.name/norbert_koelpin,,66.161.88.167,"{""location"": ""LK"", ""is_mobile"": true}" 15986,6,360,2016-12-22 05:48:44,http://waters.io/citlalli,,232.3.90.160,"{""location"": ""MO"", ""is_mobile"": true}" 15987,6,360,2017-03-22 20:59:30,http://mills.com/cecile,,105.178.172.157,"{""location"": ""BM"", ""is_mobile"": true}" 15988,6,360,2016-12-29 10:19:50,http://yostgibson.com/clint_schaden,,164.11.232.164,"{""location"": ""DJ"", ""is_mobile"": true}" 15989,6,360,2017-04-11 02:38:15,http://collins.biz/lester_fadel,,241.181.61.146,"{""location"": ""TG"", ""is_mobile"": true}" 15990,6,360,2017-02-21 03:09:59,http://bashirianblick.name/reynold.schmidt,,102.44.196.132,"{""location"": ""SS"", ""is_mobile"": false}" 15991,6,360,2017-02-13 05:37:27,http://doylegislason.info/edgardo_kuhn,,123.72.18.64,"{""location"": ""RO"", ""is_mobile"": true}" 15992,6,360,2017-02-06 14:00:38,http://robelkemmer.name/althea,,146.74.179.33,"{""location"": ""KN"", ""is_mobile"": true}" 15993,6,360,2017-04-14 02:24:16,http://skiles.info/bobbie.raynor,,204.114.214.93,"{""location"": ""RO"", ""is_mobile"": false}" 15994,6,360,2017-06-02 13:54:38,http://christiansengerlach.name/webster_luettgen,,248.194.192.127,"{""location"": ""MM"", ""is_mobile"": false}" 15995,6,360,2017-04-22 10:03:09,http://gleichner.com/alycia,,193.98.37.209,"{""location"": ""NP"", ""is_mobile"": false}" 15996,6,360,2017-04-21 03:54:09,http://boehm.io/erik_huel,,111.98.79.129,"{""location"": ""AM"", ""is_mobile"": true}" 15997,6,360,2017-01-28 03:45:26,http://mitchell.biz/selena,,253.27.225.206,"{""location"": ""FI"", ""is_mobile"": false}" 15998,6,360,2017-01-18 20:22:24,http://hintzkutch.org/selina,,37.206.229.95,"{""location"": ""VN"", ""is_mobile"": false}" 15999,6,360,2017-02-16 04:37:55,http://veum.info/sydney.wilderman,,19.156.178.77,"{""location"": ""TT"", ""is_mobile"": true}" 16000,6,360,2017-01-02 20:17:12,http://bailey.io/caterina,,149.154.177.101,"{""location"": ""HM"", ""is_mobile"": true}" 16001,6,360,2017-02-07 08:49:25,http://rath.net/dexter_kozey,,88.30.62.175,"{""location"": ""CV"", ""is_mobile"": true}" 16002,6,360,2017-01-18 16:05:30,http://emardkonopelski.com/angel_hoppe,,115.244.98.133,"{""location"": ""MZ"", ""is_mobile"": false}" 16003,6,360,2017-05-13 01:35:55,http://oreilly.info/mitchel,,235.184.4.115,"{""location"": ""NP"", ""is_mobile"": true}" 16004,6,360,2017-05-12 10:16:25,http://wyman.info/betty,,7.116.195.118,"{""location"": ""AR"", ""is_mobile"": false}" 16005,6,360,2017-02-23 09:18:24,http://hudsonjakubowski.org/brandon,,85.116.149.16,"{""location"": ""NA"", ""is_mobile"": true}" 16006,6,360,2017-05-15 19:37:32,http://miller.info/holly,,140.55.200.40,"{""location"": ""BF"", ""is_mobile"": true}" 16007,6,360,2017-03-12 13:53:28,http://pricegraham.org/stella,,17.248.63.10,"{""location"": ""UA"", ""is_mobile"": false}" 16008,6,361,2017-01-17 13:06:45,http://weinat.biz/yasmin_champlin,,194.36.184.196,"{""location"": ""MM"", ""is_mobile"": true}" 16009,6,361,2017-01-08 11:41:24,http://thiel.co/mack_lockman,,24.163.26.147,"{""location"": ""MA"", ""is_mobile"": false}" 16010,6,361,2017-05-14 23:15:59,http://daniel.info/zita,,131.228.237.182,"{""location"": ""TM"", ""is_mobile"": true}" 16011,6,361,2017-01-01 15:28:38,http://thompsonortiz.co/milan.torp,,101.170.243.21,"{""location"": ""GR"", ""is_mobile"": false}" 16012,6,361,2017-01-21 05:11:51,http://vonrueden.biz/magdalena.buckridge,,159.193.225.83,"{""location"": ""TC"", ""is_mobile"": true}" 16013,6,361,2017-02-15 19:58:42,http://lehner.co/cristobal.cremin,,51.105.118.172,"{""location"": ""MQ"", ""is_mobile"": true}" 16014,6,361,2017-06-07 05:30:36,http://kunzebashirian.co/jose,,56.71.98.136,"{""location"": ""TJ"", ""is_mobile"": true}" 16015,6,361,2017-05-18 22:03:35,http://buckridge.co/lucius,,242.195.35.51,"{""location"": ""BF"", ""is_mobile"": false}" 16016,6,361,2017-02-18 23:54:43,http://stokesadams.net/adriana.kilback,,25.66.41.143,"{""location"": ""CD"", ""is_mobile"": false}" 16017,6,361,2017-04-05 19:27:47,http://reynoldskautzer.com/lolita,,123.49.27.110,"{""location"": ""TG"", ""is_mobile"": true}" 16018,6,361,2017-03-30 20:50:14,http://wunsch.com/edison.thiel,,120.181.159.198,"{""location"": ""TM"", ""is_mobile"": true}" 16019,6,361,2017-01-08 14:01:29,http://towneebert.io/dayne,,54.188.235.36,"{""location"": ""TJ"", ""is_mobile"": false}" 16020,6,361,2017-05-12 08:35:36,http://dubuque.net/liliane,,151.198.36.114,"{""location"": ""UZ"", ""is_mobile"": true}" 16021,6,361,2017-05-14 05:53:55,http://sipespadberg.name/daniela.satterfield,,231.178.182.150,"{""location"": ""HK"", ""is_mobile"": false}" 16022,6,361,2017-05-22 19:42:11,http://grahamweimann.org/frances,,175.145.161.162,"{""location"": ""KR"", ""is_mobile"": true}" 16023,6,361,2016-12-29 02:30:04,http://ryanadams.info/telly,,130.18.105.155,"{""location"": ""BB"", ""is_mobile"": true}" 16024,6,361,2017-01-10 23:17:20,http://runolfon.co/jefferey,,113.42.137.147,"{""location"": ""PL"", ""is_mobile"": true}" 16025,6,361,2017-01-01 22:05:48,http://bartoletti.co/cleve_mcglynn,,103.219.217.232,"{""location"": ""PK"", ""is_mobile"": true}" 3164,2,70,2017-04-26 12:18:50,http://langosh.com/dylan.satterfield,0.3995446815,172.210.245.150,"{""location"": ""MD"", ""is_mobile"": true}" 3165,2,70,2016-12-14 19:51:12,http://leannonfeil.co/crawford,0.7269254292,188.231.166.31,"{""location"": ""MS"", ""is_mobile"": false}" 3166,2,70,2017-01-11 06:20:16,http://herman.com/nadia,0.4981931865,123.213.225.198,"{""location"": ""SE"", ""is_mobile"": false}" 3167,2,70,2017-04-26 10:36:11,http://okuneva.io/aisha.kuvalis,0.2036350992,207.146.228.114,"{""location"": ""VG"", ""is_mobile"": true}" 3168,2,70,2017-06-04 11:40:42,http://wintheiser.name/ali,0.7791044149,233.154.134.69,"{""location"": ""BH"", ""is_mobile"": false}" 3169,2,70,2017-06-02 14:16:16,http://windler.biz/enos,0.5576621281,65.4.54.208,"{""location"": ""ZW"", ""is_mobile"": false}" 3170,2,70,2017-01-19 05:40:15,http://kertzmann.co/precious_greenholt,0.2306446487,192.19.12.108,"{""location"": ""BL"", ""is_mobile"": false}" 3171,2,70,2017-04-29 18:39:53,http://reichertconsidine.info/napoleon.klocko,0.8635961808,150.100.82.111,"{""location"": ""CU"", ""is_mobile"": false}" 3172,2,70,2017-06-01 08:14:49,http://champlinlindgren.name/astrid,0.0891855067,132.24.183.209,"{""location"": ""ES"", ""is_mobile"": false}" 3173,2,70,2017-04-10 15:56:48,http://koelpinbeahan.net/jamarcus,0.8422969521,89.131.49.226,"{""location"": ""SH"", ""is_mobile"": true}" 3174,2,70,2017-02-18 20:46:50,http://bergstromeffertz.info/ova,0.6214309409,189.134.57.14,"{""location"": ""SV"", ""is_mobile"": false}" 3175,2,70,2017-02-02 13:45:12,http://grantkunde.net/savanna,0.1519146681,130.80.86.90,"{""location"": ""LY"", ""is_mobile"": false}" 3176,2,70,2016-12-31 17:17:19,http://zemlak.biz/blaise.smitham,0.6224488311,102.101.232.246,"{""location"": ""VN"", ""is_mobile"": false}" 3177,2,70,2017-03-16 16:18:28,http://mcdermott.org/hilario,0.2703748848,96.197.109.219,"{""location"": ""UG"", ""is_mobile"": false}" 3178,2,70,2017-03-03 18:02:20,http://effertzleuschke.co/adam,0.9819698129,73.114.16.85,"{""location"": ""AU"", ""is_mobile"": true}" 3179,2,70,2017-03-29 01:04:49,http://paucek.name/horace,0.6134758915,185.70.176.48,"{""location"": ""MX"", ""is_mobile"": false}" 3180,2,70,2017-02-09 19:42:28,http://schmitt.co/evan.becker,0.6190268694,76.112.171.178,"{""location"": ""KH"", ""is_mobile"": false}" 3181,2,70,2017-03-09 10:02:36,http://smith.io/nelson,0.7606227337,180.95.190.165,"{""location"": ""SK"", ""is_mobile"": false}" 3182,2,70,2017-01-05 00:18:40,http://crist.com/grayson_conn,0.6191406598,128.76.254.135,"{""location"": ""ET"", ""is_mobile"": true}" 3183,2,70,2017-01-14 04:12:44,http://crooks.info/luisa,0.2900826882,153.209.189.137,"{""location"": ""CF"", ""is_mobile"": false}" 3184,2,70,2017-05-06 02:29:34,http://mooremarquardt.org/patience,0.7598943188,101.131.189.15,"{""location"": ""SX"", ""is_mobile"": false}" 3185,2,70,2017-05-09 16:42:10,http://hermanbogisich.name/carlo.hamill,0.4921300632,104.146.249.218,"{""location"": ""UZ"", ""is_mobile"": false}" 3186,2,70,2017-01-31 12:59:00,http://hicklefadel.io/jovani,0.6312898753,218.39.179.88,"{""location"": ""AE"", ""is_mobile"": true}" 3187,2,70,2017-03-30 02:28:51,http://kulas.biz/magali.gaylord,0.2207516186,164.193.215.151,"{""location"": ""PH"", ""is_mobile"": false}" 3188,2,70,2016-12-18 16:17:45,http://powlowski.co/sydni_kiehn,0.5645217244,166.48.231.4,"{""location"": ""BQ"", ""is_mobile"": false}" 3189,2,70,2017-02-05 21:55:21,http://macejkovic.net/annabel,0.2988220242,99.108.27.110,"{""location"": ""VI"", ""is_mobile"": true}" 3190,2,70,2017-03-07 03:33:43,http://zboncaknicolas.io/johnnie,0.5294670252,212.120.106.73,"{""location"": ""NU"", ""is_mobile"": true}" 3191,2,70,2017-01-06 23:06:39,http://kuhnokon.co/jettie.mante,0.5736509639,47.26.103.228,"{""location"": ""LA"", ""is_mobile"": false}" 3192,2,70,2017-02-02 02:19:25,http://kutch.org/merlin.skiles,0.7365899188,122.165.144.251,"{""location"": ""VC"", ""is_mobile"": false}" 3193,2,70,2017-01-26 20:59:51,http://franecki.name/mathew_considine,0.8382310223,245.132.194.115,"{""location"": ""VC"", ""is_mobile"": true}" 3194,2,70,2017-01-31 13:07:13,http://heathcote.org/immanuel.dooley,0.0491573526,71.200.107.59,"{""location"": ""CH"", ""is_mobile"": true}" 3195,2,70,2017-01-12 20:33:14,http://pricehuel.net/carolanne_homenick,0.4668266403,105.217.201.171,"{""location"": ""CX"", ""is_mobile"": false}" 3196,2,70,2017-03-08 18:48:54,http://schimmel.info/bonnie,0.1052624965,5.4.136.143,"{""location"": ""CA"", ""is_mobile"": false}" 3197,2,70,2017-03-01 02:55:57,http://kaulkeflatley.name/vince_sauer,0.4389680120,203.169.218.38,"{""location"": ""RU"", ""is_mobile"": false}" 3198,2,70,2017-03-01 20:43:49,http://vonruedenbreitenberg.com/creola,0.4113556699,111.98.128.184,"{""location"": ""BJ"", ""is_mobile"": false}" 3199,2,70,2017-01-06 02:00:45,http://kilback.biz/dion,0.8085387191,29.253.170.54,"{""location"": ""CH"", ""is_mobile"": true}" 3200,2,70,2017-06-12 00:33:54,http://keeling.info/kendrick,0.5736482370,16.253.231.236,"{""location"": ""PT"", ""is_mobile"": false}" 3201,2,70,2017-02-15 11:35:22,http://sengermacgyver.org/casimir,0.8246168954,84.243.6.7,"{""location"": ""ME"", ""is_mobile"": true}" 3202,2,70,2017-01-13 07:58:38,http://zieme.biz/jaqueline,0.0657853727,225.64.240.186,"{""location"": ""SR"", ""is_mobile"": true}" 3203,2,70,2017-06-03 01:09:35,http://jerdebauch.net/reta.watsica,0.3588749459,223.26.150.99,"{""location"": ""GB"", ""is_mobile"": false}" 3204,2,70,2016-12-21 13:32:24,http://abbott.com/jena,0.0143137107,240.239.78.233,"{""location"": ""MC"", ""is_mobile"": false}" 3205,2,70,2017-06-13 01:36:31,http://runolfsdottir.biz/glenna,0.1816033279,160.173.211.112,"{""location"": ""GR"", ""is_mobile"": false}" 3206,2,70,2017-05-03 09:55:05,http://reillyquitzon.net/jovany.deckow,0.1485986436,28.188.144.199,"{""location"": ""CN"", ""is_mobile"": false}" 3207,2,70,2017-01-07 03:18:40,http://mraz.name/wayne.hermann,0.5027297235,77.145.165.197,"{""location"": ""GQ"", ""is_mobile"": true}" 3208,2,70,2017-02-20 23:21:56,http://kojakubowski.io/hilario,0.5652418826,79.77.43.82,"{""location"": ""RE"", ""is_mobile"": false}" 3209,2,70,2017-05-05 05:13:35,http://hillsmills.org/gregory_schuster,0.1086386041,84.129.4.97,"{""location"": ""TK"", ""is_mobile"": false}" 3210,2,70,2017-01-20 12:25:45,http://dickinson.com/benny,0.8201960793,21.233.79.43,"{""location"": ""VA"", ""is_mobile"": false}" 3211,2,70,2017-05-22 04:32:36,http://larkinkrajcik.name/osbaldo_rempel,0.3163130446,153.239.53.108,"{""location"": ""NL"", ""is_mobile"": true}" 3212,2,70,2016-12-19 04:40:27,http://cormiervonrueden.co/reie,0.2893209751,190.177.211.100,"{""location"": ""CH"", ""is_mobile"": false}" 3213,2,70,2017-02-19 16:50:54,http://bosco.co/ona.predovic,0.7843253866,22.152.2.235,"{""location"": ""LK"", ""is_mobile"": false}" 3214,2,70,2017-05-22 21:01:22,http://turcotte.biz/ansley_herman,0.2523834349,149.39.49.152,"{""location"": ""GU"", ""is_mobile"": true}" 10098,4,227,2017-01-08 15:10:02,http://gerhold.name/jasmin.witting,0.2999711425,77.176.235.3,"{""location"": ""GG"", ""is_mobile"": false}" 10099,4,227,2017-04-26 03:01:31,http://collier.io/alexander_cremin,0.5574552123,131.49.74.42,"{""location"": ""MU"", ""is_mobile"": true}" 10100,4,227,2017-01-16 21:22:30,http://sauer.net/stefan_haag,0.7188778358,245.69.23.52,"{""location"": ""CM"", ""is_mobile"": false}" 10101,4,227,2017-04-14 03:06:57,http://moen.info/maddison,0.2989466310,46.184.215.189,"{""location"": ""BT"", ""is_mobile"": false}" 10102,4,227,2017-02-20 10:59:17,http://mertz.net/antonetta.bauch,0.7650339055,246.63.8.85,"{""location"": ""EE"", ""is_mobile"": false}" 10103,4,227,2017-03-02 07:51:02,http://kuhn.org/arnulfo.wintheiser,0.3047712608,17.160.214.15,"{""location"": ""NC"", ""is_mobile"": false}" 10104,4,227,2017-04-21 04:16:02,http://davis.com/joseph.lebsack,0.5922916891,126.91.56.141,"{""location"": ""MH"", ""is_mobile"": false}" 10105,4,227,2017-03-13 07:19:30,http://will.net/brody.schneider,0.1235586767,180.23.189.98,"{""location"": ""SM"", ""is_mobile"": false}" 10106,4,227,2017-01-30 15:49:17,http://nitzsche.co/ernesto,0.9741504274,51.206.97.121,"{""location"": ""AI"", ""is_mobile"": true}" 10107,4,227,2017-02-05 16:33:31,http://stammdonnelly.name/santino.wuckert,0.1763606904,22.139.247.42,"{""location"": ""AU"", ""is_mobile"": false}" 10108,4,227,2017-04-21 03:51:37,http://oberbrunnerjaskolski.co/rosie.christiansen,0.4362991037,198.48.137.124,"{""location"": ""SY"", ""is_mobile"": true}" 10109,4,227,2017-02-12 00:23:27,http://balistreri.net/ronny,0.2229095207,44.33.102.47,"{""location"": ""AL"", ""is_mobile"": true}" 10110,4,227,2017-03-25 17:26:04,http://koepp.info/amani,0.1154796555,245.114.243.230,"{""location"": ""JM"", ""is_mobile"": true}" 10111,4,227,2017-05-10 08:46:29,http://hermannhahn.co/conner,0.0488825977,183.250.107.64,"{""location"": ""TV"", ""is_mobile"": true}" 10112,4,227,2017-04-18 09:57:01,http://cormiermccullough.name/kristin.dooley,0.2531532267,158.226.222.7,"{""location"": ""LU"", ""is_mobile"": false}" 10113,4,227,2017-01-27 06:18:44,http://witting.io/maritza,0.2873185384,50.147.198.27,"{""location"": ""FR"", ""is_mobile"": true}" 10114,4,227,2016-12-22 21:20:16,http://ritchie.io/virginia,0.8764654319,47.222.223.132,"{""location"": ""IT"", ""is_mobile"": false}" 10115,4,227,2017-01-24 10:28:50,http://bogisichbrekke.name/reinhold.jones,0.4474255691,245.246.103.142,"{""location"": ""SD"", ""is_mobile"": true}" 10116,4,227,2017-05-04 01:32:33,http://mckenzie.com/meaghan_goyette,0.9005084192,148.215.128.77,"{""location"": ""BW"", ""is_mobile"": true}" 10117,4,227,2017-04-14 14:30:34,http://gorczanybrown.io/eloy.brown,0.9527298232,74.96.135.57,"{""location"": ""CI"", ""is_mobile"": false}" 10118,4,227,2017-01-09 00:50:40,http://lemketremblay.info/kelsi_mohr,0.3243366193,229.172.214.86,"{""location"": ""ET"", ""is_mobile"": true}" 10119,4,227,2017-02-28 21:34:33,http://vonreilly.co/shanel_renner,0.4897581340,41.11.126.91,"{""location"": ""TO"", ""is_mobile"": true}" 10120,4,227,2017-02-17 01:42:30,http://bradtke.info/dennis,0.4495669875,181.146.163.200,"{""location"": ""WS"", ""is_mobile"": true}" 10121,4,227,2017-04-20 17:56:38,http://wehner.info/edwardo.grady,0.4919298034,220.238.115.187,"{""location"": ""BQ"", ""is_mobile"": false}" 10122,4,227,2017-01-20 17:31:11,http://doyle.co/jerrod.heaney,0.0245842196,81.116.152.27,"{""location"": ""KP"", ""is_mobile"": true}" 10123,4,227,2017-06-02 05:50:34,http://conn.io/horacio.murphy,0.3301172545,34.5.50.248,"{""location"": ""WF"", ""is_mobile"": true}" 10124,4,227,2017-05-25 18:59:57,http://fisher.org/nicolette,0.8309284190,71.242.165.69,"{""location"": ""SB"", ""is_mobile"": true}" 10125,4,227,2017-06-10 18:42:17,http://wisokyschinner.info/carmelo.kreiger,0.5230115363,9.56.239.154,"{""location"": ""BI"", ""is_mobile"": false}" 10126,4,228,2017-05-17 03:01:15,http://padberggleichner.name/mafalda_williamson,0.6538482406,163.250.79.221,"{""location"": ""MT"", ""is_mobile"": false}" 10127,4,228,2016-12-15 22:49:26,http://huel.net/jordane_zemlak,0.8296645972,48.111.121.64,"{""location"": ""CN"", ""is_mobile"": true}" 10128,4,228,2017-04-22 07:41:49,http://heidenreich.org/floy,0.2961226237,10.134.51.58,"{""location"": ""QA"", ""is_mobile"": false}" 10129,4,228,2017-01-15 14:55:59,http://sipesblock.org/garett,0.3157907186,95.92.39.178,"{""location"": ""ES"", ""is_mobile"": true}" 10130,4,228,2017-04-06 22:45:30,http://kautzermedhurst.io/ole.will,0.7712411500,58.160.48.245,"{""location"": ""NU"", ""is_mobile"": false}" 10131,4,228,2017-04-05 02:46:55,http://doyleleffler.name/ramiro.flatley,0.2231594701,246.182.162.33,"{""location"": ""NU"", ""is_mobile"": true}" 10132,4,228,2017-02-07 16:09:24,http://bechtelarbauch.info/terrill,0.4000776033,53.145.162.254,"{""location"": ""MY"", ""is_mobile"": true}" 10133,4,228,2016-12-28 20:24:20,http://reynolds.info/clotilde.kohler,0.4283429046,251.44.46.110,"{""location"": ""TN"", ""is_mobile"": true}" 10134,4,228,2017-05-28 14:09:51,http://sporer.io/yvette.cummerata,0.1270343523,110.242.169.98,"{""location"": ""NC"", ""is_mobile"": true}" 10135,4,228,2017-05-18 23:46:48,http://corkerygoyette.org/lonny.rath,0.8190526142,222.120.98.114,"{""location"": ""CV"", ""is_mobile"": true}" 10136,4,228,2017-03-23 17:42:03,http://donnellyarmstrong.co/isabell,0.0450598976,232.240.137.160,"{""location"": ""BT"", ""is_mobile"": false}" 10137,4,228,2017-01-05 08:01:26,http://wittingstrosin.co/bert,0.9140051563,159.101.236.197,"{""location"": ""BI"", ""is_mobile"": true}" 10138,4,228,2017-01-28 12:13:59,http://monahan.net/margarett_daniel,0.5072758715,150.187.42.142,"{""location"": ""KZ"", ""is_mobile"": true}" 10139,4,228,2017-02-18 07:49:01,http://dickensjohns.name/deven_rolfson,0.4814953619,141.203.61.168,"{""location"": ""SL"", ""is_mobile"": false}" 10140,4,228,2017-01-10 09:31:41,http://effertzkunde.info/axel,0.4930035481,131.101.120.43,"{""location"": ""YE"", ""is_mobile"": true}" 10141,4,228,2017-04-10 08:30:11,http://dach.info/arielle,0.6100998913,57.165.203.115,"{""location"": ""EG"", ""is_mobile"": true}" 10142,4,228,2017-02-11 02:54:01,http://emmerich.name/delpha,0.3229747228,214.213.208.50,"{""location"": ""SX"", ""is_mobile"": false}" 10143,4,228,2017-05-20 05:04:09,http://stiedemann.name/antonette.bradtke,0.0256426523,111.211.84.190,"{""location"": ""RE"", ""is_mobile"": true}" 10144,4,228,2016-12-14 19:18:52,http://bartoletti.com/christelle,0.4099673203,245.103.95.41,"{""location"": ""CN"", ""is_mobile"": true}" 10145,4,228,2017-03-20 04:49:36,http://paucek.io/clementine,0.6394117659,185.56.162.184,"{""location"": ""KP"", ""is_mobile"": false}" 10146,4,228,2017-06-02 04:25:53,http://jenkinsmckenzie.biz/mateo_blick,0.7486199186,63.136.10.198,"{""location"": ""BR"", ""is_mobile"": false}" 10147,4,228,2017-02-15 05:06:35,http://lind.name/kurt,0.1414203785,25.237.47.111,"{""location"": ""NP"", ""is_mobile"": false}" 10148,4,228,2017-03-21 00:05:42,http://waelchikuhic.co/daron,0.7758615428,152.201.210.190,"{""location"": ""TF"", ""is_mobile"": true}" 7108,3,158,2017-01-03 02:03:41,http://davisnitzsche.io/mavis,0.0259288807,228.220.91.80,"{""location"": ""BY"", ""is_mobile"": true}" 7109,3,158,2017-05-19 14:21:07,http://conn.biz/perry,0.7184517048,182.95.162.11,"{""location"": ""DK"", ""is_mobile"": false}" 7110,3,158,2017-03-20 06:05:31,http://jacobson.biz/hayley_waters,0.0165299534,151.232.106.203,"{""location"": ""ET"", ""is_mobile"": true}" 7111,3,158,2017-05-14 10:44:15,http://klingruel.biz/stefan,0.4625819741,238.88.200.42,"{""location"": ""LA"", ""is_mobile"": true}" 7112,3,158,2017-01-01 20:20:46,http://bednargerlach.info/hailee_mraz,0.5826191778,147.2.160.122,"{""location"": ""RU"", ""is_mobile"": true}" 7113,3,158,2017-06-02 21:58:44,http://gottlieb.net/preston,0.6180902773,125.195.140.231,"{""location"": ""BR"", ""is_mobile"": false}" 7114,3,158,2017-02-28 09:15:00,http://witting.info/janick,0.9757653506,206.56.146.88,"{""location"": ""BS"", ""is_mobile"": true}" 7115,3,158,2017-03-28 21:11:04,http://rolfson.net/eugene,0.4090121515,78.13.179.34,"{""location"": ""LY"", ""is_mobile"": false}" 7116,3,158,2017-05-28 08:47:14,http://rowecorwin.io/lavina,0.3948104941,195.149.150.85,"{""location"": ""ME"", ""is_mobile"": false}" 7117,3,158,2016-12-20 06:00:09,http://ko.org/ivah.spinka,0.9299602623,117.138.164.147,"{""location"": ""RE"", ""is_mobile"": false}" 7118,3,158,2017-05-14 03:03:34,http://haley.io/morris,0.3753767857,70.28.66.201,"{""location"": ""AS"", ""is_mobile"": true}" 7119,3,158,2017-01-24 19:53:57,http://kerluke.biz/stuart,0.8332994322,5.206.33.96,"{""location"": ""LC"", ""is_mobile"": false}" 7120,3,158,2017-05-16 05:26:43,http://blockhomenick.net/letitia,0.7670838278,121.31.16.249,"{""location"": ""BV"", ""is_mobile"": true}" 7121,3,158,2017-01-23 19:07:54,http://kuhlman.info/virginia_mohr,0.4017290327,15.187.203.82,"{""location"": ""RS"", ""is_mobile"": true}" 7122,3,158,2017-05-04 10:10:25,http://nitzsche.biz/dena,0.1333217604,189.74.159.65,"{""location"": ""BO"", ""is_mobile"": true}" 7123,3,158,2017-05-29 06:35:05,http://brakuchmidt.name/angel_glover,0.1793286629,113.58.93.42,"{""location"": ""PW"", ""is_mobile"": false}" 7124,3,158,2017-03-17 10:07:01,http://dickens.name/evan_hauck,0.6652268726,204.50.82.205,"{""location"": ""JP"", ""is_mobile"": false}" 7125,3,158,2017-03-17 15:08:01,http://lebsackmcclure.co/ramona.west,0.2982106747,240.228.86.9,"{""location"": ""AG"", ""is_mobile"": false}" 7126,3,158,2016-12-24 18:13:42,http://davis.net/candido,0.0137252945,140.222.21.197,"{""location"": ""MA"", ""is_mobile"": false}" 7127,3,158,2016-12-23 01:33:33,http://feeney.com/ila.pfeffer,0.4194038406,88.201.217.167,"{""location"": ""GH"", ""is_mobile"": true}" 7128,3,158,2017-03-25 10:12:43,http://hermistonstrosin.name/don,0.3467404801,55.4.145.65,"{""location"": ""AF"", ""is_mobile"": false}" 7129,3,158,2017-05-24 04:46:52,http://batz.io/lilla_corwin,0.0958603401,170.200.18.214,"{""location"": ""JE"", ""is_mobile"": false}" 7130,3,158,2016-12-27 01:30:47,http://ankundingheidenreich.info/wilber,0.6118997205,68.123.249.77,"{""location"": ""IO"", ""is_mobile"": true}" 7131,3,158,2017-03-10 21:54:32,http://jacobi.co/haie_braun,0.2822770438,119.100.42.23,"{""location"": ""HM"", ""is_mobile"": false}" 7132,3,158,2017-03-26 08:42:34,http://murazikkling.co/jordyn_kuhic,0.4926158024,28.205.104.28,"{""location"": ""SY"", ""is_mobile"": true}" 7133,3,158,2017-03-31 17:56:33,http://pourosbogan.io/jerald,0.7046601528,68.201.83.41,"{""location"": ""VN"", ""is_mobile"": false}" 7134,3,158,2017-01-19 01:17:14,http://keebler.biz/clarabelle,0.9689898710,45.151.93.62,"{""location"": ""KZ"", ""is_mobile"": true}" 7135,3,158,2017-02-17 00:49:08,http://ondricka.org/reanna_armstrong,0.9817095524,94.223.141.129,"{""location"": ""MA"", ""is_mobile"": false}" 7136,3,158,2017-03-09 01:19:00,http://cainmitchell.com/pietro,0.7321598889,194.165.79.71,"{""location"": ""MO"", ""is_mobile"": false}" 7137,3,158,2017-03-30 07:58:10,http://ziemeklocko.org/emmanuelle,0.2885898450,132.121.224.79,"{""location"": ""BA"", ""is_mobile"": false}" 7138,3,158,2017-03-25 15:52:48,http://wyman.org/bill,0.8165699443,84.101.88.27,"{""location"": ""GB"", ""is_mobile"": true}" 7139,3,159,2017-03-01 22:26:01,http://andersonzboncak.biz/ozella_konopelski,0.5507918025,209.192.169.208,"{""location"": ""PK"", ""is_mobile"": true}" 7140,3,159,2017-04-28 17:00:43,http://wolfweber.info/chadrick,0.5450484023,45.42.135.151,"{""location"": ""TT"", ""is_mobile"": false}" 7141,3,159,2017-04-19 03:59:06,http://funkrath.co/forest,0.5885048465,211.159.181.109,"{""location"": ""NI"", ""is_mobile"": true}" 7142,3,159,2016-12-16 07:33:06,http://zemlak.com/furman.kerluke,0.5784893009,78.172.31.234,"{""location"": ""SK"", ""is_mobile"": false}" 7143,3,159,2017-02-12 19:11:40,http://mayerstokes.co/augustus,0.5120345166,54.101.67.116,"{""location"": ""MY"", ""is_mobile"": true}" 7144,3,159,2017-02-03 22:44:26,http://kozey.com/estell,0.1143227688,145.3.91.108,"{""location"": ""AQ"", ""is_mobile"": false}" 7145,3,159,2017-05-05 11:56:53,http://senger.com/jayda.stanton,0.0661511933,140.20.163.54,"{""location"": ""VA"", ""is_mobile"": false}" 7146,3,159,2017-06-09 03:56:39,http://cartwright.biz/antone.bartell,0.2337101414,176.23.60.253,"{""location"": ""IS"", ""is_mobile"": true}" 7147,3,159,2017-03-18 06:22:32,http://daugherty.com/asa_senger,0.5627915219,219.2.229.222,"{""location"": ""BQ"", ""is_mobile"": true}" 7148,3,159,2017-03-21 22:51:05,http://bradtke.com/layne,0.0160742247,110.63.88.147,"{""location"": ""NR"", ""is_mobile"": false}" 7149,3,159,2017-01-12 17:11:41,http://walker.net/roma,0.7488417764,87.230.20.153,"{""location"": ""SC"", ""is_mobile"": true}" 7150,3,159,2017-02-02 21:28:25,http://friesenkovacek.com/te,0.2926401315,230.45.87.235,"{""location"": ""SY"", ""is_mobile"": true}" 7151,3,159,2017-03-15 06:47:39,http://boyerhowell.org/marty_kovacek,0.5767038461,95.62.75.194,"{""location"": ""IL"", ""is_mobile"": true}" 7152,3,159,2017-06-02 06:42:29,http://schmeler.io/mabelle.price,0.5840588052,13.147.60.175,"{""location"": ""CG"", ""is_mobile"": true}" 7153,3,159,2017-06-02 21:18:54,http://langworth.org/daron,0.1259956049,191.125.245.202,"{""location"": ""BD"", ""is_mobile"": false}" 7154,3,159,2017-01-25 07:13:35,http://gradyschneider.info/kip,0.8104220538,34.147.183.234,"{""location"": ""LK"", ""is_mobile"": true}" 7155,3,159,2016-12-20 08:23:30,http://cummingsrempel.name/reva,0.5176506628,213.153.168.176,"{""location"": ""KM"", ""is_mobile"": false}" 7156,3,159,2017-02-19 15:48:23,http://heathcote.info/arch_douglas,0.7194358241,127.160.238.250,"{""location"": ""BJ"", ""is_mobile"": false}" 7157,3,159,2017-04-10 08:42:17,http://grant.org/osvaldo.white,0.2793506395,229.98.47.183,"{""location"": ""UG"", ""is_mobile"": true}" 7158,3,159,2017-04-28 23:14:42,http://cremin.name/autumn,0.3433644069,181.24.71.166,"{""location"": ""CU"", ""is_mobile"": false}" 7159,3,159,2017-04-30 19:27:53,http://nienow.io/zoe,0.6521688503,2.12.240.43,"{""location"": ""IL"", ""is_mobile"": false}" 13016,5,292,2017-02-20 18:17:22,http://spencer.info/albin_bailey,0.2943082138,242.38.159.86,"{""location"": ""BJ"", ""is_mobile"": true}" 13017,5,292,2017-03-19 23:16:41,http://lowe.info/seth,0.8280911811,197.23.182.254,"{""location"": ""TV"", ""is_mobile"": true}" 13018,5,292,2017-02-19 18:37:51,http://bernhardschulist.io/cielo_reichert,0.2072642508,7.78.111.93,"{""location"": ""EE"", ""is_mobile"": false}" 13019,5,292,2017-05-09 10:20:57,http://haley.com/julien,0.3318970043,189.225.113.202,"{""location"": ""PF"", ""is_mobile"": false}" 13020,5,292,2017-01-24 20:19:14,http://gaylordhayes.info/juwan_cain,0.1983777353,21.126.94.172,"{""location"": ""LR"", ""is_mobile"": false}" 13021,5,292,2017-01-16 01:31:31,http://dicki.info/ilene,0.4271826278,169.74.237.163,"{""location"": ""MC"", ""is_mobile"": false}" 13022,5,292,2017-02-17 13:05:29,http://williamson.com/margarett.pollich,0.0028463334,174.84.69.55,"{""location"": ""MS"", ""is_mobile"": true}" 13023,5,292,2017-02-16 02:22:35,http://rippinbruen.io/damian,0.9648393564,100.56.166.220,"{""location"": ""AL"", ""is_mobile"": false}" 13024,5,292,2017-01-25 04:02:21,http://fayboyer.io/rupert,0.3821288254,24.196.166.98,"{""location"": ""HT"", ""is_mobile"": true}" 13025,5,292,2017-05-20 17:28:35,http://hansen.com/madisyn.shanahan,0.4364290279,232.210.198.224,"{""location"": ""NA"", ""is_mobile"": false}" 13026,5,292,2017-04-21 20:31:30,http://kihn.biz/lance_kozey,0.5051495200,217.248.98.50,"{""location"": ""MW"", ""is_mobile"": false}" 13027,5,292,2017-03-12 12:50:56,http://labadie.net/misty.halvorson,0.4331065695,112.113.236.163,"{""location"": ""DK"", ""is_mobile"": false}" 13028,5,292,2017-03-25 03:27:35,http://quitzon.co/amos.hahn,0.2705957444,98.224.36.142,"{""location"": ""CV"", ""is_mobile"": true}" 13029,5,292,2017-01-21 23:24:01,http://heller.name/rowan,0.9804042040,203.207.12.89,"{""location"": ""SB"", ""is_mobile"": true}" 13030,5,292,2017-02-07 06:41:26,http://howeoberbrunner.org/linwood.paucek,0.7902376272,179.146.66.171,"{""location"": ""KW"", ""is_mobile"": false}" 13031,5,292,2017-05-30 16:53:58,http://raynor.co/carolyn_runolfon,0.3923046594,213.195.110.211,"{""location"": ""GB"", ""is_mobile"": false}" 13032,5,292,2016-12-24 06:06:31,http://boyle.name/barney,0.9038312169,88.73.184.227,"{""location"": ""GE"", ""is_mobile"": false}" 13033,5,292,2017-05-04 21:23:22,http://walkerchamplin.info/abbey_hansen,0.6775122425,54.155.185.238,"{""location"": ""AU"", ""is_mobile"": true}" 13034,5,292,2017-01-20 05:04:31,http://mertz.name/gregg,0.7409434311,18.8.133.214,"{""location"": ""SN"", ""is_mobile"": true}" 13035,5,292,2017-04-01 05:01:09,http://bernhard.co/otis_harber,0.8469362508,178.103.139.31,"{""location"": ""AZ"", ""is_mobile"": false}" 13036,5,292,2017-04-10 08:06:41,http://schaden.co/darrin,0.9443850890,75.188.235.6,"{""location"": ""CM"", ""is_mobile"": true}" 13037,5,292,2017-03-17 19:18:43,http://roobbeahan.net/beryl_friesen,0.7490845098,90.15.82.19,"{""location"": ""GW"", ""is_mobile"": true}" 13038,5,292,2017-02-21 03:15:33,http://hartmannmckenzie.co/deron_glover,0.1000497029,164.7.77.213,"{""location"": ""JP"", ""is_mobile"": false}" 13039,5,292,2017-05-23 22:28:11,http://schroeder.org/lizzie,0.2962046598,212.195.95.153,"{""location"": ""YT"", ""is_mobile"": true}" 13040,5,292,2017-04-05 09:43:55,http://mayerteichmann.net/mark.conroy,0.5631642105,164.70.16.27,"{""location"": ""GW"", ""is_mobile"": true}" 13041,5,292,2017-04-22 13:40:42,http://kohler.info/bailee_hegmann,0.1486229725,25.235.71.189,"{""location"": ""BZ"", ""is_mobile"": false}" 13042,5,292,2017-04-19 23:27:26,http://schultz.info/jonatan,0.2694945418,254.248.251.226,"{""location"": ""IM"", ""is_mobile"": false}" 13043,5,292,2017-04-29 21:22:44,http://macejkovicbernier.org/genoveva,0.3296009372,78.66.80.130,"{""location"": ""BS"", ""is_mobile"": false}" 13044,5,292,2017-03-15 18:35:24,http://hilll.net/jamaal.rohan,0.1817401536,227.210.146.214,"{""location"": ""KZ"", ""is_mobile"": false}" 13045,5,292,2017-06-07 15:31:00,http://wehner.name/nicklaus,0.1779788538,188.169.92.181,"{""location"": ""AX"", ""is_mobile"": false}" 13046,5,292,2017-06-01 03:44:18,http://homenick.biz/bill,0.5606071360,73.68.78.122,"{""location"": ""KI"", ""is_mobile"": true}" 13047,5,292,2017-04-27 10:01:42,http://kihn.co/judge_satterfield,0.5413843636,218.214.30.140,"{""location"": ""JE"", ""is_mobile"": true}" 13048,5,292,2017-04-28 14:04:48,http://franecki.info/virginie,0.9511857796,42.58.41.21,"{""location"": ""GR"", ""is_mobile"": false}" 13049,5,292,2017-06-04 15:44:37,http://bartoletti.io/michale.cormier,0.1938801298,51.57.159.193,"{""location"": ""LB"", ""is_mobile"": false}" 13050,5,292,2017-05-21 21:36:45,http://haag.name/ola_hahn,0.0381020946,105.166.194.111,"{""location"": ""AI"", ""is_mobile"": false}" 13051,5,292,2017-02-07 00:41:12,http://beatty.io/justine,0.3867543718,249.115.165.72,"{""location"": ""VG"", ""is_mobile"": true}" 13052,5,292,2017-03-24 06:17:02,http://farrellreichert.com/caie.bartell,0.3193526202,181.116.158.232,"{""location"": ""LS"", ""is_mobile"": false}" 13053,5,292,2017-05-29 22:44:51,http://hayes.biz/tina,0.2285871054,71.80.232.252,"{""location"": ""SA"", ""is_mobile"": true}" 13054,5,292,2017-03-18 05:53:26,http://zulauf.co/misty,0.8093256506,168.74.127.49,"{""location"": ""BE"", ""is_mobile"": true}" 13055,5,292,2017-04-18 15:45:34,http://feilhuels.com/gust,0.9804736886,123.243.222.121,"{""location"": ""DM"", ""is_mobile"": false}" 13056,5,293,2017-02-17 22:51:59,http://heelwintheiser.io/stacey.rodriguez,0.4441187659,169.24.129.166,"{""location"": ""DE"", ""is_mobile"": true}" 13057,5,293,2017-03-26 18:15:23,http://simonis.name/albert.oberbrunner,0.5918567905,46.109.171.211,"{""location"": ""MO"", ""is_mobile"": false}" 13058,5,293,2017-03-18 15:18:46,http://cruickshank.biz/neoma,0.1064824851,70.200.246.5,"{""location"": ""TV"", ""is_mobile"": false}" 13059,5,293,2017-02-06 20:51:12,http://considine.info/juwan,0.5526028717,89.83.155.162,"{""location"": ""VG"", ""is_mobile"": true}" 13060,5,293,2017-05-25 01:53:21,http://upton.org/dangelo_casper,0.5829071748,227.100.110.219,"{""location"": ""PW"", ""is_mobile"": false}" 13061,5,293,2017-01-27 06:25:46,http://parisian.name/jey,0.2481821642,18.179.200.114,"{""location"": ""SK"", ""is_mobile"": true}" 13062,5,293,2017-04-21 21:05:53,http://roberts.net/chance_funk,0.1141835795,199.250.222.102,"{""location"": ""FI"", ""is_mobile"": true}" 13063,5,293,2017-02-27 22:15:55,http://okeefe.name/kaitlin,0.4495180944,74.177.236.162,"{""location"": ""GU"", ""is_mobile"": false}" 13064,5,293,2017-02-21 05:18:33,http://wehner.net/ramona_bechtelar,0.3925710094,124.52.68.180,"{""location"": ""BZ"", ""is_mobile"": true}" 13065,5,293,2017-03-30 05:45:47,http://mertz.info/don.williamson,0.8612276067,108.248.85.147,"{""location"": ""IS"", ""is_mobile"": true}" 13066,5,293,2017-01-09 17:53:51,http://fay.io/glenna,0.2656969860,173.155.14.67,"{""location"": ""MK"", ""is_mobile"": false}" 16026,6,361,2016-12-24 12:09:05,http://kertzmanncasper.net/arturo.harris,,68.254.58.157,"{""location"": ""CF"", ""is_mobile"": true}" 16027,6,361,2017-02-26 20:15:56,http://mosciski.name/nelson,,161.131.116.199,"{""location"": ""BS"", ""is_mobile"": false}" 16028,6,361,2017-04-25 18:12:21,http://wehner.org/carey.dare,,19.208.58.37,"{""location"": ""GD"", ""is_mobile"": true}" 16029,6,361,2017-01-20 18:54:30,http://parisiangaylord.name/cordie,,212.42.107.135,"{""location"": ""TV"", ""is_mobile"": true}" 16030,6,361,2017-01-19 11:11:06,http://ohara.io/paris,,42.24.50.112,"{""location"": ""VA"", ""is_mobile"": true}" 16031,6,361,2017-02-18 01:35:02,http://conn.io/bill,,36.111.218.159,"{""location"": ""GE"", ""is_mobile"": false}" 16032,6,361,2016-12-28 00:18:32,http://west.org/barrett_hilpert,,244.133.70.218,"{""location"": ""GE"", ""is_mobile"": true}" 16033,6,361,2017-05-04 18:21:37,http://marquardtschaefer.com/madisen.hackett,,233.134.209.182,"{""location"": ""FO"", ""is_mobile"": true}" 16034,6,361,2017-04-01 13:11:51,http://mueller.org/elwyn.gibson,,14.179.128.90,"{""location"": ""BG"", ""is_mobile"": true}" 16035,6,362,2017-01-29 11:43:38,http://brakusmetz.com/coleman,,22.179.155.235,"{""location"": ""DZ"", ""is_mobile"": false}" 16036,6,362,2017-03-27 23:47:41,http://hodkiewicz.name/eriberto,,164.113.234.175,"{""location"": ""MX"", ""is_mobile"": true}" 16037,6,362,2016-12-25 14:21:52,http://trantowgutkowski.org/denis.mann,,101.73.124.101,"{""location"": ""WS"", ""is_mobile"": false}" 16038,6,362,2017-01-14 04:28:36,http://fahey.io/lourdes_goldner,,178.104.109.185,"{""location"": ""TD"", ""is_mobile"": false}" 16039,6,362,2017-04-04 09:49:42,http://daniel.name/daisy_dickens,,95.146.188.205,"{""location"": ""RE"", ""is_mobile"": false}" 16040,6,362,2017-01-09 13:12:36,http://dachschulist.biz/amara_jones,,61.99.151.30,"{""location"": ""TC"", ""is_mobile"": false}" 16041,6,362,2017-03-02 07:31:44,http://greenfelder.name/thea.dibbert,,18.195.88.121,"{""location"": ""AD"", ""is_mobile"": false}" 16042,6,362,2017-02-06 11:39:14,http://prohaska.io/monty.swaniawski,,79.123.241.58,"{""location"": ""IE"", ""is_mobile"": false}" 16043,6,362,2017-05-26 19:04:35,http://witting.co/georgianna,,109.112.206.59,"{""location"": ""CY"", ""is_mobile"": false}" 16044,6,362,2017-06-07 06:52:58,http://oconnellmcclure.org/kelsie_haag,,184.143.135.114,"{""location"": ""MY"", ""is_mobile"": false}" 16045,6,362,2017-04-06 06:50:56,http://mcclure.com/ernest,,208.233.186.102,"{""location"": ""LS"", ""is_mobile"": false}" 16046,6,362,2017-05-29 07:56:56,http://klocko.biz/sedrick_stracke,,187.138.58.219,"{""location"": ""CM"", ""is_mobile"": false}" 16047,6,362,2016-12-16 23:21:53,http://lakinjast.org/prince_friesen,,192.161.241.153,"{""location"": ""YT"", ""is_mobile"": false}" 16048,6,362,2017-06-13 17:21:41,http://hilll.io/geovanny.pacocha,,96.251.211.60,"{""location"": ""CM"", ""is_mobile"": true}" 16049,6,362,2017-04-13 14:39:55,http://witting.net/haleigh_farrell,,209.149.148.250,"{""location"": ""MQ"", ""is_mobile"": false}" 16050,6,362,2017-02-28 06:24:03,http://reinger.io/kristy,,3.149.187.130,"{""location"": ""SD"", ""is_mobile"": false}" 16051,6,362,2017-06-06 17:08:25,http://kuvalis.com/norwood,,228.233.113.6,"{""location"": ""GY"", ""is_mobile"": true}" 16052,6,362,2016-12-22 17:12:01,http://ferry.net/annamae.renner,,62.82.184.3,"{""location"": ""BL"", ""is_mobile"": false}" 16053,6,362,2017-02-03 10:42:05,http://rogahn.org/william,,10.84.202.177,"{""location"": ""WF"", ""is_mobile"": false}" 16054,6,362,2017-04-10 15:47:57,http://boyle.com/marta,,127.248.215.175,"{""location"": ""GA"", ""is_mobile"": true}" 16055,6,362,2016-12-27 12:05:10,http://borer.org/gerard.glover,,151.112.147.227,"{""location"": ""GT"", ""is_mobile"": true}" 16056,6,362,2017-06-09 07:35:02,http://rosenbaum.com/lavern_gusikowski,,70.217.74.156,"{""location"": ""VN"", ""is_mobile"": false}" 16057,6,363,2017-04-15 01:58:14,http://greenkuhn.co/tom,,134.8.110.230,"{""location"": ""PH"", ""is_mobile"": true}" 16058,6,363,2016-12-15 10:19:37,http://mcdermott.com/floie_heaney,,145.20.107.44,"{""location"": ""ZM"", ""is_mobile"": true}" 16059,6,363,2017-02-25 17:43:48,http://ward.com/sydni_kub,,72.249.214.41,"{""location"": ""HR"", ""is_mobile"": false}" 16060,6,363,2017-03-20 14:58:16,http://vonruedenabbott.info/lonny,,108.28.24.204,"{""location"": ""PN"", ""is_mobile"": true}" 16061,6,363,2017-01-18 10:30:11,http://walker.co/eldridge,,199.191.169.77,"{""location"": ""PK"", ""is_mobile"": false}" 16062,6,363,2017-05-18 23:23:58,http://mayer.net/heidi_wuckert,,65.12.166.72,"{""location"": ""BY"", ""is_mobile"": true}" 16063,6,363,2017-03-06 12:37:07,http://cartwright.info/robb_olson,,117.33.147.202,"{""location"": ""VC"", ""is_mobile"": true}" 16064,6,363,2016-12-21 08:17:49,http://strosin.co/caandre,,38.110.110.108,"{""location"": ""US"", ""is_mobile"": false}" 16065,6,363,2017-01-16 18:20:12,http://schamberger.io/adan,,8.9.215.61,"{""location"": ""SI"", ""is_mobile"": true}" 16066,6,363,2017-05-25 18:10:24,http://schoen.co/zack_wunsch,,116.193.180.201,"{""location"": ""NP"", ""is_mobile"": true}" 16067,6,363,2017-03-15 12:18:47,http://zulauf.io/dejuan_jacobi,,18.185.175.133,"{""location"": ""TT"", ""is_mobile"": false}" 16068,6,363,2017-05-18 19:54:17,http://gleichner.org/eric,,207.18.157.43,"{""location"": ""TK"", ""is_mobile"": true}" 16069,6,363,2017-03-04 16:04:29,http://damorekuhic.org/catharine_johnston,,37.155.155.243,"{""location"": ""KR"", ""is_mobile"": false}" 16070,6,363,2017-04-20 15:36:59,http://fisher.co/darlene,,57.58.112.7,"{""location"": ""MG"", ""is_mobile"": false}" 16071,6,363,2017-05-31 02:20:08,http://conroy.info/brenda.harvey,,100.63.206.87,"{""location"": ""NF"", ""is_mobile"": true}" 16072,6,363,2017-05-16 12:41:36,http://swaniawskisanford.com/alexie,,177.41.161.59,"{""location"": ""LY"", ""is_mobile"": true}" 16073,6,363,2016-12-15 23:51:11,http://macgyver.io/chanel.schimmel,,29.175.119.115,"{""location"": ""MA"", ""is_mobile"": false}" 16074,6,363,2017-03-15 16:33:56,http://strosin.com/ruby.schinner,,36.35.38.128,"{""location"": ""LR"", ""is_mobile"": false}" 16075,6,363,2017-06-10 09:03:32,http://leffler.net/alexandra.kulas,,99.85.112.98,"{""location"": ""TH"", ""is_mobile"": true}" 16076,6,363,2017-04-11 20:16:08,http://cronin.io/thora,,152.210.235.21,"{""location"": ""DE"", ""is_mobile"": true}" 16077,6,363,2017-03-21 02:50:36,http://quitzon.name/henri.keeling,,41.236.179.232,"{""location"": ""CD"", ""is_mobile"": false}" 16078,6,363,2017-01-10 12:24:17,http://kilback.com/lottie.kozey,,41.228.94.148,"{""location"": ""UZ"", ""is_mobile"": false}" 16079,6,363,2016-12-21 13:33:23,http://padberg.com/eva,,209.66.113.241,"{""location"": ""TR"", ""is_mobile"": false}" 16080,6,363,2017-06-05 07:38:27,http://kohler.com/christ_emard,,31.217.197.53,"{""location"": ""MV"", ""is_mobile"": true}" 16081,6,363,2017-01-16 23:43:17,http://gusikowskishields.biz/alexzander.kohler,,4.172.224.221,"{""location"": ""CZ"", ""is_mobile"": false}" 3215,2,70,2017-01-26 09:23:10,http://caingorczany.io/lawson.kerluke,0.6710299662,168.233.129.212,"{""location"": ""CZ"", ""is_mobile"": false}" 3216,2,70,2017-02-23 11:03:43,http://schultz.biz/hattie_kuhn,0.1095510261,188.86.81.231,"{""location"": ""AM"", ""is_mobile"": true}" 3217,2,70,2016-12-17 16:02:30,http://herzoggottlieb.io/jaiden,0.6279733399,30.213.78.148,"{""location"": ""GG"", ""is_mobile"": false}" 3218,2,70,2017-05-05 15:42:37,http://rempel.net/malvina,0.5794572574,127.238.105.202,"{""location"": ""MG"", ""is_mobile"": true}" 3219,2,70,2017-01-09 23:48:43,http://maggio.info/waylon,0.4083953369,252.158.185.237,"{""location"": ""TH"", ""is_mobile"": false}" 3220,2,71,2016-12-27 23:51:13,http://sauerhackett.name/cody.hodkiewicz,0.5017354407,98.109.132.216,"{""location"": ""CN"", ""is_mobile"": false}" 3221,2,71,2017-04-04 22:05:27,http://wilkinson.co/valentine.hane,0.6171185063,83.47.192.141,"{""location"": ""JE"", ""is_mobile"": true}" 3222,2,71,2017-03-26 02:36:52,http://cremin.org/audrey.kirlin,0.7640025030,107.254.95.192,"{""location"": ""FJ"", ""is_mobile"": false}" 3223,2,71,2017-06-07 19:19:07,http://nadermayert.com/verla_marquardt,0.2587349674,71.235.222.120,"{""location"": ""ZM"", ""is_mobile"": false}" 3224,2,71,2016-12-31 03:33:16,http://ohara.com/gino,0.1150013208,190.203.67.98,"{""location"": ""AD"", ""is_mobile"": false}" 3225,2,71,2017-03-23 23:52:48,http://vandervort.co/liana.waelchi,0.0635390583,238.232.237.79,"{""location"": ""JE"", ""is_mobile"": true}" 3226,2,71,2017-04-08 18:11:47,http://stroman.name/marley_kunze,0.3694944091,118.86.196.186,"{""location"": ""WF"", ""is_mobile"": false}" 3227,2,71,2017-04-04 08:06:42,http://halvorson.net/cynthia,0.8831656825,96.27.20.68,"{""location"": ""AE"", ""is_mobile"": true}" 3228,2,71,2017-02-03 09:01:55,http://nikolaus.org/liam,0.4989752548,98.122.138.223,"{""location"": ""EC"", ""is_mobile"": true}" 3229,2,71,2017-03-15 22:53:14,http://blandamonahan.io/orville,0.5240664809,190.75.24.135,"{""location"": ""SG"", ""is_mobile"": true}" 3230,2,71,2017-05-26 15:24:40,http://johnston.com/walker_collier,0.7856001078,41.8.196.152,"{""location"": ""FO"", ""is_mobile"": true}" 3231,2,71,2017-04-29 12:56:32,http://moen.io/herman.olson,0.2390327218,179.231.186.232,"{""location"": ""GD"", ""is_mobile"": true}" 3232,2,71,2017-05-15 16:51:20,http://rutherfordwilliamson.io/sierra_runolfon,0.9108688192,172.240.203.42,"{""location"": ""GG"", ""is_mobile"": false}" 3233,2,71,2017-04-30 19:10:03,http://berge.io/kirstin,0.5207004752,64.32.201.145,"{""location"": ""HM"", ""is_mobile"": false}" 3234,2,71,2017-05-22 04:55:46,http://mcglynn.co/kathryne_oconner,0.9044288915,47.134.182.205,"{""location"": ""CR"", ""is_mobile"": true}" 3235,2,71,2017-03-03 04:01:32,http://gottlieb.io/dashawn,0.5924953658,96.16.109.58,"{""location"": ""NI"", ""is_mobile"": true}" 3236,2,71,2017-04-04 02:37:41,http://stokes.info/effie,0.1326918827,230.83.208.194,"{""location"": ""GR"", ""is_mobile"": true}" 3237,2,71,2017-03-14 19:04:58,http://cainkoelpin.biz/omer,0.5205022193,216.60.151.110,"{""location"": ""MD"", ""is_mobile"": true}" 3238,2,71,2017-02-06 03:00:57,http://kerluke.co/korbin,0.2115641982,126.49.144.57,"{""location"": ""AW"", ""is_mobile"": false}" 3239,2,71,2017-01-10 08:06:56,http://mitchell.name/maxie,0.1299830812,150.102.58.84,"{""location"": ""CI"", ""is_mobile"": true}" 3240,2,71,2017-05-27 11:26:23,http://oreilly.net/samir_bashirian,0.8362010460,226.160.65.59,"{""location"": ""NF"", ""is_mobile"": true}" 3241,2,71,2017-01-19 00:15:17,http://okeefepurdy.org/anthony.eichmann,0.5766176293,197.23.219.26,"{""location"": ""VE"", ""is_mobile"": false}" 3242,2,71,2017-03-27 15:48:20,http://lehnerhodkiewicz.com/forest,0.6647326079,4.196.209.70,"{""location"": ""HK"", ""is_mobile"": false}" 3243,2,71,2017-03-27 00:30:57,http://crooksboehm.info/charlotte,0.3098527141,52.84.112.250,"{""location"": ""SL"", ""is_mobile"": false}" 3244,2,71,2017-02-12 03:19:34,http://hermann.org/dallin,0.9309141605,85.141.240.236,"{""location"": ""TK"", ""is_mobile"": true}" 4175,2,91,2017-02-25 15:42:48,http://volkman.biz/anahi,,99.119.209.241,"{""location"": ""SY"", ""is_mobile"": true}" 3245,2,71,2017-06-01 21:57:20,http://beckerherman.co/josh.kuhn,0.4660185347,18.198.145.27,"{""location"": ""MK"", ""is_mobile"": false}" 3246,2,71,2017-01-19 19:32:09,http://keeblerschulist.io/donavon.hoppe,0.2276658571,118.144.205.164,"{""location"": ""DE"", ""is_mobile"": false}" 3247,2,71,2017-01-22 14:49:07,http://champlin.biz/caterina_okon,0.1620862099,81.39.43.128,"{""location"": ""YE"", ""is_mobile"": true}" 3248,2,71,2017-04-15 00:37:42,http://considine.io/paige.eichmann,0.9456620378,156.68.107.19,"{""location"": ""IE"", ""is_mobile"": true}" 3249,2,71,2017-05-25 23:06:00,http://rolfsonborer.co/alia,0.0477833516,99.74.204.105,"{""location"": ""HK"", ""is_mobile"": true}" 3250,2,71,2017-02-15 09:02:28,http://auer.co/claudine,0.3661682972,133.50.220.71,"{""location"": ""MN"", ""is_mobile"": false}" 3251,2,71,2017-01-14 11:09:19,http://johns.info/camren_schmeler,0.1504238984,162.223.193.42,"{""location"": ""JO"", ""is_mobile"": true}" 3252,2,71,2017-04-21 07:18:37,http://feeney.org/trent.tremblay,0.1978312241,69.55.203.19,"{""location"": ""ET"", ""is_mobile"": true}" 3253,2,71,2016-12-22 07:07:42,http://kuphal.net/pamela,0.2927983392,52.79.219.177,"{""location"": ""TN"", ""is_mobile"": true}" 3254,2,71,2017-06-13 14:07:19,http://mayerbogisich.io/harvey,0.8264181827,254.108.128.176,"{""location"": ""TW"", ""is_mobile"": true}" 3255,2,71,2017-03-11 18:04:27,http://howe.io/myriam.thiel,0.6233006005,187.77.145.16,"{""location"": ""EC"", ""is_mobile"": false}" 3256,2,71,2017-06-04 17:31:23,http://schmidtokuneva.name/blair.cole,0.3115824973,80.136.32.35,"{""location"": ""PM"", ""is_mobile"": true}" 3257,2,71,2017-03-10 18:50:08,http://adams.biz/ronny.lubowitz,0.8430832072,162.84.252.232,"{""location"": ""GA"", ""is_mobile"": false}" 3258,2,71,2017-04-19 15:17:52,http://murray.net/mazie,0.9586781246,114.55.229.35,"{""location"": ""PH"", ""is_mobile"": true}" 3259,2,71,2017-02-18 09:05:42,http://bogan.net/agnes,0.2396434865,120.112.133.63,"{""location"": ""SS"", ""is_mobile"": true}" 3260,2,71,2017-06-08 09:02:51,http://hodkiewicz.name/clair,0.4347797367,211.158.26.121,"{""location"": ""MP"", ""is_mobile"": true}" 3261,2,71,2017-02-17 19:30:41,http://nikolaus.net/lyla,0.0486440341,126.152.178.159,"{""location"": ""SY"", ""is_mobile"": false}" 3262,2,71,2017-01-12 19:01:23,http://wymangrimes.io/janet_brown,0.5517823673,222.153.206.82,"{""location"": ""MD"", ""is_mobile"": true}" 3263,2,71,2017-01-26 14:55:15,http://howe.name/mitchel,0.5127387826,90.17.223.189,"{""location"": ""BJ"", ""is_mobile"": false}" 3264,2,71,2017-03-10 03:22:56,http://robelfunk.io/chanel_wilkinson,0.7647385105,132.96.173.204,"{""location"": ""NU"", ""is_mobile"": true}" 3265,2,71,2017-06-06 17:18:17,http://hirthe.name/deon_block,0.8125045726,71.197.76.182,"{""location"": ""BO"", ""is_mobile"": false}" 10149,4,228,2017-01-03 18:34:04,http://welchwisozk.net/annabell.legros,0.7298383411,78.54.193.53,"{""location"": ""MX"", ""is_mobile"": false}" 10150,4,228,2017-04-10 09:05:31,http://moen.net/cole.smitham,0.0279983687,19.167.40.193,"{""location"": ""MQ"", ""is_mobile"": true}" 10151,4,228,2017-02-12 23:29:27,http://bodebernier.name/robin,0.1175456608,177.90.253.110,"{""location"": ""MY"", ""is_mobile"": true}" 10152,4,228,2017-06-08 06:53:24,http://conroy.co/ruth,0.2812274434,249.161.218.225,"{""location"": ""ML"", ""is_mobile"": false}" 10153,4,228,2017-04-29 19:30:47,http://nitzsche.org/pietro.kohler,0.6247987616,24.9.56.162,"{""location"": ""RW"", ""is_mobile"": false}" 10154,4,228,2017-04-10 06:49:44,http://hills.name/guiseppe,0.7074445237,190.111.201.46,"{""location"": ""ZW"", ""is_mobile"": true}" 10155,4,228,2016-12-31 02:20:50,http://friesencummerata.com/flavie,0.9729885963,188.14.83.232,"{""location"": ""CM"", ""is_mobile"": true}" 10156,4,228,2017-01-12 20:30:55,http://marvinstark.co/orlo,0.9051712551,24.192.177.72,"{""location"": ""GI"", ""is_mobile"": false}" 10157,4,228,2017-02-14 20:39:10,http://streichfeeney.biz/esmeralda_fahey,0.1902308829,102.163.59.43,"{""location"": ""VE"", ""is_mobile"": false}" 10158,4,228,2017-05-20 00:45:16,http://littel.biz/marcos,0.0369681729,215.134.136.177,"{""location"": ""FR"", ""is_mobile"": false}" 10159,4,228,2017-04-19 04:28:16,http://smitham.name/monty_heathcote,0.4633078601,191.23.250.248,"{""location"": ""FK"", ""is_mobile"": true}" 10160,4,228,2017-03-05 10:35:19,http://bechtelar.io/roxane,0.5337310195,242.226.139.68,"{""location"": ""JP"", ""is_mobile"": false}" 10161,4,228,2017-05-25 06:50:48,http://lakin.biz/maybell,0.4334823649,233.185.78.206,"{""location"": ""CL"", ""is_mobile"": true}" 10162,4,229,2017-04-24 09:22:03,http://schmitt.name/elian_mccullough,0.8144799267,221.210.92.150,"{""location"": ""PE"", ""is_mobile"": true}" 10163,4,229,2017-06-02 17:42:01,http://fay.org/marjolaine.cole,0.2334119532,126.82.110.195,"{""location"": ""ME"", ""is_mobile"": true}" 10164,4,229,2017-04-20 07:34:49,http://senger.biz/grover,0.1988847070,98.16.192.34,"{""location"": ""NU"", ""is_mobile"": true}" 10165,4,229,2017-04-25 15:19:51,http://wintheiser.name/haven,0.1697069636,41.170.25.227,"{""location"": ""BF"", ""is_mobile"": true}" 10166,4,229,2017-04-18 01:08:30,http://runolfsdottirstamm.co/carter,0.5805796231,173.46.94.108,"{""location"": ""AZ"", ""is_mobile"": true}" 10167,4,229,2017-04-12 04:22:15,http://carroll.name/stephanie.kiehn,0.7707336185,43.137.171.195,"{""location"": ""GH"", ""is_mobile"": true}" 10168,4,229,2017-01-04 00:38:15,http://handhaag.info/tavares.prosacco,0.0390443831,36.197.20.145,"{""location"": ""CD"", ""is_mobile"": false}" 10169,4,229,2017-05-29 05:57:38,http://carroll.io/carolina.bartell,0.8175035244,116.41.45.142,"{""location"": ""SC"", ""is_mobile"": false}" 10170,4,229,2017-01-10 06:49:28,http://runte.com/nella_kunde,0.0529022773,204.124.177.41,"{""location"": ""AU"", ""is_mobile"": true}" 10171,4,229,2016-12-19 14:16:27,http://rutherford.co/sharon,0.1259322576,44.48.67.227,"{""location"": ""GA"", ""is_mobile"": true}" 10172,4,229,2016-12-28 00:22:48,http://hyatt.io/nedra.ortiz,0.8455079937,152.19.60.222,"{""location"": ""CA"", ""is_mobile"": false}" 10173,4,229,2017-01-19 13:07:50,http://howell.net/claude.brown,0.9856821626,140.61.77.121,"{""location"": ""AX"", ""is_mobile"": true}" 10174,4,229,2017-03-21 14:42:25,http://riceking.net/murray.leannon,0.4212328349,183.250.112.84,"{""location"": ""BT"", ""is_mobile"": false}" 10175,4,229,2016-12-25 13:18:11,http://schowalter.biz/kasandra_predovic,0.8581680472,203.165.52.42,"{""location"": ""TW"", ""is_mobile"": false}" 10176,4,229,2017-03-16 00:52:14,http://gislason.co/octavia,0.1393364495,222.45.21.236,"{""location"": ""EH"", ""is_mobile"": false}" 10177,4,229,2017-02-27 21:12:21,http://brakusmarquardt.info/tatum,0.8365316956,189.7.191.97,"{""location"": ""VA"", ""is_mobile"": true}" 10178,4,229,2017-05-17 03:09:31,http://prosaccocummings.net/prudence,0.5655096047,48.178.230.192,"{""location"": ""IT"", ""is_mobile"": true}" 10179,4,229,2017-04-01 20:46:50,http://wiegand.net/alvah.eichmann,0.8684992353,222.57.158.87,"{""location"": ""BR"", ""is_mobile"": true}" 10180,4,229,2017-06-06 03:28:47,http://kohler.name/zoey,0.1839769661,46.19.254.9,"{""location"": ""AL"", ""is_mobile"": true}" 10181,4,229,2017-04-11 03:33:54,http://gaylorddamore.name/maximillia,0.2770437907,145.5.157.27,"{""location"": ""PK"", ""is_mobile"": false}" 10182,4,229,2017-06-10 09:11:21,http://goldner.info/muhammad_little,0.5261174075,194.206.135.247,"{""location"": ""KY"", ""is_mobile"": true}" 10183,4,229,2017-03-08 20:53:04,http://will.org/ahmad,0.2028454119,146.63.12.246,"{""location"": ""TG"", ""is_mobile"": true}" 10184,4,229,2017-05-22 00:11:37,http://okuneva.name/cordie.stoltenberg,0.6293621060,2.144.111.107,"{""location"": ""GH"", ""is_mobile"": true}" 10185,4,229,2017-01-18 05:38:10,http://lindgrenhayes.co/chesley_reinger,0.4992063040,134.98.190.242,"{""location"": ""SL"", ""is_mobile"": true}" 10186,4,229,2017-06-05 02:55:26,http://prosacco.name/nikki.padberg,0.3508857279,133.39.72.190,"{""location"": ""DE"", ""is_mobile"": true}" 10187,4,229,2017-05-22 16:15:59,http://gleichner.org/garrison,0.3230640977,31.119.228.223,"{""location"": ""IM"", ""is_mobile"": true}" 10188,4,229,2017-05-20 06:00:33,http://jastbotsford.name/elody,0.4988829771,136.222.20.139,"{""location"": ""US"", ""is_mobile"": false}" 10189,4,229,2016-12-26 07:59:31,http://ryan.com/angelica,0.8377964741,19.43.5.92,"{""location"": ""TR"", ""is_mobile"": false}" 10190,4,229,2017-05-02 12:27:35,http://ebert.com/gianni.oconner,0.0714369048,132.15.42.200,"{""location"": ""GF"", ""is_mobile"": true}" 10191,4,229,2017-05-26 09:40:22,http://mueller.io/doug_feeney,0.6918321415,120.2.15.157,"{""location"": ""BJ"", ""is_mobile"": false}" 10192,4,229,2017-03-23 16:22:28,http://farrell.biz/corene_medhurst,0.1550340937,147.2.241.125,"{""location"": ""KY"", ""is_mobile"": true}" 10193,4,229,2017-06-08 08:21:46,http://carterblock.net/kurtis,0.7826291896,5.40.155.163,"{""location"": ""KP"", ""is_mobile"": true}" 10194,4,229,2017-02-21 15:09:34,http://hamill.biz/cordia,0.2283176811,117.57.207.240,"{""location"": ""KN"", ""is_mobile"": false}" 10195,4,229,2017-02-07 10:01:23,http://windlerschinner.biz/cordell_jerde,0.0195087870,61.229.252.83,"{""location"": ""CR"", ""is_mobile"": true}" 10196,4,229,2017-03-17 14:54:37,http://rueckerfay.io/judd_mcclure,0.4042943187,235.215.198.129,"{""location"": ""HM"", ""is_mobile"": false}" 10197,4,229,2017-03-09 03:47:46,http://reynolds.org/filomena_bode,0.2389422022,99.166.31.125,"{""location"": ""SH"", ""is_mobile"": false}" 10198,4,229,2017-02-23 04:40:36,http://jakubowski.org/margret,0.3497965016,103.63.22.112,"{""location"": ""HM"", ""is_mobile"": false}" 10199,4,229,2017-06-02 11:48:46,http://bogan.net/shany,0.7711693009,156.202.35.163,"{""location"": ""NI"", ""is_mobile"": true}" 7160,3,159,2017-02-13 21:07:47,http://bruenhuel.name/leopoldo_windler,0.1097070402,180.231.118.221,"{""location"": ""BH"", ""is_mobile"": true}" 7161,3,159,2017-05-06 07:10:06,http://wardtorp.co/pietro_glover,0.6763080544,95.90.80.81,"{""location"": ""PG"", ""is_mobile"": false}" 7162,3,159,2017-03-06 11:47:08,http://ebertbeatty.net/macie,0.7291401707,37.97.40.195,"{""location"": ""BL"", ""is_mobile"": false}" 7163,3,159,2017-02-11 23:26:15,http://rath.name/bridgette,0.4823116446,241.22.212.192,"{""location"": ""CU"", ""is_mobile"": false}" 7164,3,159,2016-12-27 17:15:26,http://hillsheller.io/coby.watsica,0.9534398448,221.121.232.235,"{""location"": ""TN"", ""is_mobile"": true}" 7165,3,159,2017-05-14 12:39:51,http://hilpert.co/christiana,0.6167481923,178.25.94.38,"{""location"": ""MF"", ""is_mobile"": false}" 7166,3,159,2017-03-23 22:16:16,http://steubergoldner.org/jedidiah_wisozk,0.2700860470,221.162.58.50,"{""location"": ""CG"", ""is_mobile"": false}" 7167,3,159,2017-06-02 10:31:46,http://legros.org/dillon,0.6922706699,162.162.39.6,"{""location"": ""GS"", ""is_mobile"": false}" 7168,3,159,2017-02-21 22:53:15,http://bernier.net/leon,0.2497714046,80.3.239.112,"{""location"": ""BQ"", ""is_mobile"": true}" 7169,3,159,2017-01-03 02:53:16,http://hills.co/tabitha.ankunding,0.6055786532,121.41.208.49,"{""location"": ""SG"", ""is_mobile"": false}" 7170,3,159,2017-04-06 16:35:25,http://dickinson.biz/annabell,0.8899835480,154.185.248.67,"{""location"": ""TH"", ""is_mobile"": false}" 7171,3,159,2017-01-06 13:46:52,http://smith.org/moshe,0.6392432823,228.12.216.110,"{""location"": ""CD"", ""is_mobile"": false}" 7172,3,159,2017-04-26 18:18:45,http://tillmanreichert.co/jeramie,0.2353707539,170.109.231.13,"{""location"": ""CD"", ""is_mobile"": true}" 7173,3,159,2017-01-04 10:09:53,http://schulistromaguera.name/garland_wiza,0.5798975538,165.215.98.227,"{""location"": ""AI"", ""is_mobile"": true}" 7174,3,159,2017-04-26 23:18:13,http://brownjohnston.info/talon,0.9461407825,4.46.180.16,"{""location"": ""SR"", ""is_mobile"": true}" 7175,3,159,2017-05-15 07:17:20,http://reichert.co/hank,0.5445757128,228.163.73.67,"{""location"": ""BA"", ""is_mobile"": true}" 7176,3,159,2017-03-27 08:03:26,http://huel.co/salvatore_oreilly,0.5898897901,104.24.167.187,"{""location"": ""MX"", ""is_mobile"": false}" 7177,3,159,2017-05-04 23:11:12,http://nienow.org/marcel,0.8826730329,20.123.11.227,"{""location"": ""SG"", ""is_mobile"": true}" 7178,3,159,2017-03-29 11:34:32,http://rodriguez.com/karina,0.1190058829,40.98.135.229,"{""location"": ""VC"", ""is_mobile"": false}" 7179,3,159,2017-02-18 16:20:39,http://monahanmills.net/jeremie,0.5932551902,165.19.74.123,"{""location"": ""VE"", ""is_mobile"": false}" 7180,3,159,2017-01-20 20:40:47,http://schmidt.name/johathan.zulauf,0.6676961550,216.235.52.17,"{""location"": ""NP"", ""is_mobile"": false}" 7181,3,159,2017-05-14 20:02:00,http://borerkling.org/edgardo,0.3092586253,46.179.97.4,"{""location"": ""PN"", ""is_mobile"": false}" 7182,3,159,2017-03-12 03:41:01,http://gislason.org/mateo,0.5062100076,90.245.97.92,"{""location"": ""TH"", ""is_mobile"": true}" 7183,3,159,2017-05-09 20:34:26,http://grimes.co/lura.yundt,0.1372168524,144.15.204.224,"{""location"": ""TM"", ""is_mobile"": true}" 7184,3,159,2017-03-18 22:35:48,http://langosh.io/lyda.schmitt,0.9423699509,134.223.88.177,"{""location"": ""ZW"", ""is_mobile"": false}" 7185,3,160,2017-05-11 22:04:13,http://wehner.com/anita,0.0859580585,129.198.79.158,"{""location"": ""FJ"", ""is_mobile"": true}" 7186,3,160,2017-03-03 08:02:34,http://rempeldietrich.info/carolyn,0.2671684347,92.14.23.15,"{""location"": ""LY"", ""is_mobile"": true}" 7187,3,160,2017-05-19 16:04:38,http://kling.co/aida.hackett,0.2715481893,213.166.114.9,"{""location"": ""BS"", ""is_mobile"": true}" 7188,3,160,2017-05-31 20:10:32,http://ankunding.net/jovani,0.9118092220,193.16.177.10,"{""location"": ""ZW"", ""is_mobile"": true}" 7189,3,160,2017-01-13 12:58:02,http://halvorsonrempel.net/abel,0.4098406340,206.224.109.175,"{""location"": ""CY"", ""is_mobile"": true}" 7190,3,160,2016-12-25 15:56:07,http://crona.io/camilla,0.8173357970,125.109.209.76,"{""location"": ""BQ"", ""is_mobile"": true}" 7191,3,160,2017-03-17 11:08:52,http://kautzer.name/major.altenwerth,0.2484386938,189.243.247.56,"{""location"": ""LC"", ""is_mobile"": false}" 7192,3,160,2017-05-22 13:44:44,http://nolanhaag.net/marianne.robel,0.8908008564,152.117.88.220,"{""location"": ""HR"", ""is_mobile"": true}" 7193,3,160,2017-03-29 22:09:44,http://schimmel.org/caleb_crona,0.0477542464,100.144.184.32,"{""location"": ""NL"", ""is_mobile"": false}" 7194,3,160,2017-04-03 21:11:10,http://lueilwitz.io/beaulah_weber,0.8721864004,69.26.196.182,"{""location"": ""KR"", ""is_mobile"": true}" 7195,3,160,2017-03-08 22:34:53,http://bahringer.biz/rodger,0.5342937769,209.152.25.104,"{""location"": ""TO"", ""is_mobile"": false}" 7196,3,160,2017-04-22 13:03:02,http://marvin.biz/dane_mann,0.7306232231,53.28.189.253,"{""location"": ""JO"", ""is_mobile"": false}" 7197,3,160,2016-12-26 05:24:58,http://ferrybergnaum.name/jaeden,0.3370518861,75.58.247.29,"{""location"": ""PM"", ""is_mobile"": true}" 7198,3,160,2017-03-27 01:53:10,http://mclaughlin.info/earnestine,0.5568099396,11.65.223.117,"{""location"": ""GF"", ""is_mobile"": false}" 7199,3,160,2016-12-29 12:52:19,http://gloverdietrich.net/warren.lockman,0.5295855118,170.135.246.41,"{""location"": ""GI"", ""is_mobile"": false}" 7200,3,160,2017-04-27 18:39:53,http://labadiecorwin.info/luella_herzog,0.6763142366,150.127.227.5,"{""location"": ""SC"", ""is_mobile"": false}" 7201,3,160,2017-02-25 09:11:46,http://auer.com/selena,0.6596328654,56.15.33.20,"{""location"": ""LU"", ""is_mobile"": false}" 7202,3,160,2017-05-28 14:08:46,http://lubowitzlangworth.com/coby,0.5063715352,157.151.207.254,"{""location"": ""ZM"", ""is_mobile"": false}" 7203,3,160,2017-04-08 10:47:45,http://simonis.org/moshe.kunze,0.4257943447,155.171.227.238,"{""location"": ""JO"", ""is_mobile"": true}" 7204,3,160,2017-03-27 22:15:52,http://gleasonvon.info/maxwell,0.8763815474,99.86.84.22,"{""location"": ""AF"", ""is_mobile"": false}" 7205,3,160,2017-02-25 00:37:59,http://ryanrodriguez.com/kenton_bode,0.4506237499,16.133.164.251,"{""location"": ""SD"", ""is_mobile"": false}" 7206,3,160,2017-05-11 19:19:42,http://reichellowe.com/hellen_raynor,0.6598220828,173.151.206.205,"{""location"": ""VU"", ""is_mobile"": false}" 7207,3,160,2017-03-30 01:05:09,http://bergnaum.com/lucio,0.9793735429,247.200.149.130,"{""location"": ""AL"", ""is_mobile"": true}" 7208,3,160,2017-02-10 18:52:30,http://daniel.info/fidel_mitchell,0.8286475997,66.51.28.173,"{""location"": ""SR"", ""is_mobile"": true}" 7209,3,160,2017-01-10 12:14:15,http://millawayn.info/ernie_maggio,0.0227986185,221.168.99.80,"{""location"": ""RS"", ""is_mobile"": false}" 7210,3,160,2016-12-24 09:46:15,http://hirthe.net/alexys.torphy,0.1247856988,35.92.168.141,"{""location"": ""GH"", ""is_mobile"": true}" 7211,3,160,2017-02-15 07:47:54,http://zboncakwalker.biz/margot,0.8092878106,118.198.76.78,"{""location"": ""AX"", ""is_mobile"": true}" 13067,5,293,2017-02-13 09:55:19,http://harris.org/alex.blanda,0.0984958663,129.75.103.157,"{""location"": ""MY"", ""is_mobile"": false}" 13068,5,293,2017-05-20 10:10:48,http://langworthfahey.org/treie,0.1884070382,33.92.169.122,"{""location"": ""NE"", ""is_mobile"": true}" 13069,5,293,2017-04-24 02:26:46,http://krajciktorp.net/emma,0.7473057686,59.92.114.137,"{""location"": ""NG"", ""is_mobile"": false}" 13070,5,293,2017-01-26 23:35:01,http://botsford.io/jonathon.wiegand,0.9215855254,152.220.149.82,"{""location"": ""PN"", ""is_mobile"": true}" 13071,5,293,2017-06-11 20:22:51,http://legros.info/norbert,0.0387347448,245.219.64.215,"{""location"": ""HM"", ""is_mobile"": false}" 13072,5,293,2017-05-06 08:11:24,http://hoppe.co/taryn_johns,0.2515641707,254.224.157.85,"{""location"": ""BR"", ""is_mobile"": false}" 13073,5,293,2017-03-29 14:06:45,http://morarkshlerin.org/larue,0.1772712246,184.204.98.232,"{""location"": ""PA"", ""is_mobile"": true}" 13074,5,293,2017-05-09 11:50:01,http://bruen.org/frieda,0.9339929151,252.165.244.200,"{""location"": ""NL"", ""is_mobile"": false}" 13075,5,293,2017-03-20 17:03:46,http://casper.com/marcelo,0.9484389505,51.191.73.143,"{""location"": ""BW"", ""is_mobile"": false}" 13076,5,293,2017-06-10 20:58:46,http://ratkekub.name/pasquale.roob,0.5509251266,223.107.107.16,"{""location"": ""CY"", ""is_mobile"": true}" 13077,5,293,2017-05-05 17:50:42,http://lebsack.net/emilia_lubowitz,0.5796893394,42.52.107.46,"{""location"": ""ID"", ""is_mobile"": false}" 13078,5,293,2017-05-28 16:19:15,http://hegmannrosenbaum.net/delpha,0.5868423358,100.143.224.236,"{""location"": ""HK"", ""is_mobile"": true}" 13079,5,293,2017-04-05 04:49:10,http://zemlak.biz/alexandrine,0.3163519602,215.67.143.53,"{""location"": ""LY"", ""is_mobile"": false}" 13080,5,293,2017-03-16 22:38:28,http://rosenbaum.name/melia_torp,0.5505986042,162.21.90.8,"{""location"": ""TO"", ""is_mobile"": false}" 13081,5,293,2017-03-31 07:14:55,http://wiza.io/bridget.boyle,0.1493724429,110.144.181.149,"{""location"": ""CW"", ""is_mobile"": false}" 13082,5,293,2017-03-13 01:29:54,http://muellerrolfson.co/arlene,0.8404533560,44.26.85.118,"{""location"": ""BA"", ""is_mobile"": false}" 13083,5,293,2017-01-12 03:48:31,http://corwinkunde.biz/oscar,0.9629320205,249.32.210.19,"{""location"": ""IS"", ""is_mobile"": true}" 13084,5,293,2017-03-24 08:52:34,http://greenholt.co/misael.stroman,0.4937035013,165.230.123.64,"{""location"": ""HT"", ""is_mobile"": false}" 13085,5,293,2017-03-20 06:20:07,http://mayer.net/gina,0.9699452488,78.51.223.31,"{""location"": ""CU"", ""is_mobile"": false}" 13086,5,293,2017-04-03 00:37:10,http://halvorsonmacejkovic.io/murray_kozey,0.3501627276,78.24.244.164,"{""location"": ""EG"", ""is_mobile"": false}" 13087,5,293,2017-03-24 20:07:00,http://nolan.net/florencio,0.4262637829,181.167.95.58,"{""location"": ""BM"", ""is_mobile"": true}" 13088,5,294,2017-03-20 05:55:58,http://stark.info/shanel,0.3105251338,236.82.50.21,"{""location"": ""SN"", ""is_mobile"": false}" 13089,5,294,2017-04-18 00:15:54,http://schroeder.co/lindsey_keebler,0.8185599990,142.225.175.106,"{""location"": ""AD"", ""is_mobile"": true}" 13090,5,294,2017-02-11 10:35:55,http://hamillraynor.org/phoebe_wolff,0.0049951172,50.95.213.184,"{""location"": ""TW"", ""is_mobile"": false}" 13091,5,294,2017-04-03 15:45:47,http://goodwinledner.org/minerva,0.8630669141,211.67.57.21,"{""location"": ""FK"", ""is_mobile"": true}" 13092,5,294,2017-04-13 13:46:57,http://swift.io/jaqueline,0.5878224067,181.193.198.91,"{""location"": ""IN"", ""is_mobile"": true}" 13093,5,294,2017-01-29 15:43:06,http://tromp.org/aiyana.pfannerstill,0.3836138661,200.118.117.191,"{""location"": ""YT"", ""is_mobile"": true}" 13094,5,294,2017-01-10 09:52:15,http://satterfieldtillman.io/kiara,0.7365877219,172.14.64.196,"{""location"": ""BO"", ""is_mobile"": false}" 13095,5,294,2017-06-09 03:29:33,http://pouroshalvorson.biz/christina,0.3711972680,87.173.144.63,"{""location"": ""BD"", ""is_mobile"": true}" 13096,5,294,2017-02-24 09:40:07,http://weimannschoen.info/wayne,0.9160825282,250.172.221.126,"{""location"": ""GU"", ""is_mobile"": true}" 13097,5,294,2017-05-04 12:47:56,http://mayer.org/will_fahey,0.4709717897,224.215.61.186,"{""location"": ""PM"", ""is_mobile"": false}" 13098,5,294,2017-03-19 20:09:34,http://dibbertblock.org/lolita.rath,0.3350012719,115.73.167.61,"{""location"": ""PN"", ""is_mobile"": true}" 13099,5,294,2017-01-11 09:27:45,http://kiehn.com/giovanna,0.7916575896,174.71.138.126,"{""location"": ""BA"", ""is_mobile"": true}" 13100,5,294,2017-03-05 02:46:52,http://hills.io/forest.kiehn,0.9407850408,154.145.141.76,"{""location"": ""MH"", ""is_mobile"": false}" 13101,5,294,2017-03-16 15:30:18,http://hayes.org/laurie,0.3943755142,228.106.216.18,"{""location"": ""BM"", ""is_mobile"": true}" 13102,5,294,2017-03-15 07:13:45,http://hansenrunolfon.name/anna,0.3905482201,68.113.202.8,"{""location"": ""MW"", ""is_mobile"": true}" 13103,5,294,2017-02-12 02:06:24,http://boyer.name/hipolito.koepp,0.0481377739,43.163.48.25,"{""location"": ""PS"", ""is_mobile"": true}" 13104,5,294,2016-12-26 03:12:45,http://frami.co/dario_bednar,0.1943778112,59.8.98.88,"{""location"": ""FK"", ""is_mobile"": false}" 13105,5,294,2017-02-02 16:22:06,http://christiansen.com/alvah_king,0.6604624182,156.75.214.250,"{""location"": ""NE"", ""is_mobile"": false}" 13106,5,294,2016-12-18 09:33:19,http://gorczanyhuel.com/emilio.champlin,0.9628269960,250.155.133.59,"{""location"": ""SY"", ""is_mobile"": false}" 13107,5,294,2017-02-15 21:10:33,http://emmerichkoelpin.com/danyka_romaguera,0.6068931713,252.160.213.92,"{""location"": ""KR"", ""is_mobile"": true}" 13108,5,294,2016-12-25 14:46:03,http://jonesmoriette.org/brandyn_barrows,0.6937975194,170.81.198.145,"{""location"": ""SZ"", ""is_mobile"": false}" 13109,5,294,2017-04-18 17:21:43,http://bailey.io/marisa.bins,0.9222917131,51.225.109.83,"{""location"": ""AD"", ""is_mobile"": false}" 13110,5,294,2017-05-06 15:33:46,http://kutch.info/marcel_hauck,0.7630032412,100.100.30.254,"{""location"": ""BO"", ""is_mobile"": true}" 13111,5,294,2017-05-09 14:32:53,http://klein.net/brooke_morar,0.1937616715,32.37.65.138,"{""location"": ""WF"", ""is_mobile"": true}" 13112,5,294,2017-02-23 10:35:21,http://thiel.info/jace,0.4289262914,181.87.235.163,"{""location"": ""PY"", ""is_mobile"": false}" 13113,5,294,2017-06-02 17:53:19,http://shieldsgrant.io/samara,0.0259831335,177.201.92.215,"{""location"": ""GB"", ""is_mobile"": false}" 13114,5,294,2016-12-27 21:27:01,http://crona.com/jamel.hodkiewicz,0.7980229606,33.190.152.130,"{""location"": ""CL"", ""is_mobile"": false}" 13115,5,294,2017-01-30 12:09:34,http://hellerpagac.org/tobin,0.9813192684,147.210.6.16,"{""location"": ""AZ"", ""is_mobile"": true}" 13116,5,294,2017-04-05 23:25:50,http://dibbert.name/carolina,0.8965045517,230.212.214.129,"{""location"": ""NI"", ""is_mobile"": false}" 13117,5,294,2017-01-08 23:00:03,http://feest.io/iac_daugherty,0.5950305740,196.35.215.186,"{""location"": ""UG"", ""is_mobile"": false}" 16082,6,363,2016-12-28 04:55:09,http://runte.com/jamey_reichert,,81.247.13.39,"{""location"": ""CU"", ""is_mobile"": true}" 16083,6,363,2017-03-02 02:21:50,http://stammprohaska.org/jason_nikolaus,,39.140.54.240,"{""location"": ""IL"", ""is_mobile"": false}" 16084,6,363,2017-03-23 06:39:29,http://kochwolff.co/francesco_beatty,,23.154.30.32,"{""location"": ""SX"", ""is_mobile"": false}" 16085,6,363,2017-03-06 17:41:57,http://bahringer.biz/tristian,,153.157.229.85,"{""location"": ""TM"", ""is_mobile"": true}" 16086,6,363,2017-06-05 13:43:47,http://simonis.biz/kellen.altenwerth,,74.38.34.190,"{""location"": ""CF"", ""is_mobile"": true}" 16087,6,363,2017-05-19 02:40:24,http://gusikowski.co/jeramie.kling,,141.168.198.2,"{""location"": ""MG"", ""is_mobile"": true}" 16088,6,363,2017-01-27 11:58:03,http://bogansipes.biz/treva_stokes,,109.64.133.53,"{""location"": ""CK"", ""is_mobile"": true}" 16089,6,363,2016-12-13 11:02:27,http://webermacgyver.biz/ariel,,131.242.241.35,"{""location"": ""IQ"", ""is_mobile"": true}" 16090,6,363,2017-02-17 00:13:36,http://watsicasanford.name/everette.kertzmann,,42.96.151.136,"{""location"": ""IR"", ""is_mobile"": false}" 16091,6,363,2017-01-26 18:15:40,http://murrayfeil.biz/marlin.spinka,,90.219.235.249,"{""location"": ""AM"", ""is_mobile"": false}" 16092,6,364,2017-03-13 09:46:48,http://towne.biz/ryleigh,,219.3.30.186,"{""location"": ""CI"", ""is_mobile"": false}" 16093,6,364,2017-06-03 06:43:48,http://harvey.net/rylee,,187.208.155.201,"{""location"": ""KW"", ""is_mobile"": true}" 16094,6,364,2016-12-22 23:58:22,http://gutkowski.org/candace_lemke,,78.134.125.206,"{""location"": ""BR"", ""is_mobile"": true}" 16095,6,364,2017-01-10 05:53:58,http://monahan.net/andre.rogahn,,103.111.119.196,"{""location"": ""WS"", ""is_mobile"": true}" 16096,6,364,2017-05-09 14:51:43,http://luettgen.name/levi.runolfon,,3.91.225.165,"{""location"": ""MA"", ""is_mobile"": false}" 16097,6,364,2017-01-08 15:00:28,http://shanahan.info/theodora_nicolas,,26.32.5.220,"{""location"": ""KN"", ""is_mobile"": true}" 16098,6,364,2017-01-18 02:28:19,http://kutch.info/kaia,,67.252.76.9,"{""location"": ""GN"", ""is_mobile"": false}" 16099,6,364,2017-04-07 08:14:23,http://rodriguez.io/erich,,190.243.72.46,"{""location"": ""RU"", ""is_mobile"": false}" 16100,6,364,2016-12-17 06:37:32,http://mertzzieme.biz/lonnie_kris,,120.34.12.185,"{""location"": ""GE"", ""is_mobile"": false}" 16101,6,364,2017-05-28 02:05:46,http://heidenreich.name/andre,,81.25.134.232,"{""location"": ""CF"", ""is_mobile"": true}" 16102,6,364,2017-02-19 15:10:00,http://barrows.org/ahmed,,21.98.194.172,"{""location"": ""IQ"", ""is_mobile"": false}" 16103,6,364,2017-03-26 06:38:28,http://jones.co/tyreek,,44.213.127.23,"{""location"": ""CU"", ""is_mobile"": true}" 16104,6,364,2017-01-04 21:50:43,http://swift.com/albertha,,101.14.86.95,"{""location"": ""HR"", ""is_mobile"": true}" 16105,6,364,2017-04-27 21:44:42,http://heidenreichoconnell.io/daniella,,200.74.75.208,"{""location"": ""SB"", ""is_mobile"": true}" 16106,6,364,2017-01-24 21:14:58,http://huel.co/brayan_torphy,,216.110.55.154,"{""location"": ""CC"", ""is_mobile"": false}" 16107,6,364,2017-04-25 00:15:42,http://auer.info/dorcas.hagenes,,100.143.13.90,"{""location"": ""VA"", ""is_mobile"": false}" 16108,6,364,2017-03-30 13:29:03,http://kshlerinwillms.net/erna_bechtelar,,105.27.173.123,"{""location"": ""BT"", ""is_mobile"": false}" 16109,6,364,2017-02-10 09:07:22,http://mcglynn.info/jazlyn_lynch,,75.237.135.247,"{""location"": ""MD"", ""is_mobile"": true}" 16110,6,364,2017-01-13 08:59:46,http://westfeest.info/quinten_kris,,14.81.42.56,"{""location"": ""MP"", ""is_mobile"": false}" 16111,6,364,2017-04-23 20:21:13,http://pollich.name/kade_dietrich,,142.192.159.100,"{""location"": ""CG"", ""is_mobile"": false}" 16112,6,364,2017-03-25 18:38:52,http://smitham.co/pink_bayer,,220.119.183.240,"{""location"": ""MV"", ""is_mobile"": true}" 16113,6,364,2017-05-16 01:35:45,http://greenholtchristiansen.net/imogene_huels,,242.92.134.170,"{""location"": ""FR"", ""is_mobile"": true}" 16114,6,364,2017-03-19 21:36:33,http://altenwerth.co/david_prosacco,,128.106.149.160,"{""location"": ""UG"", ""is_mobile"": true}" 16115,6,364,2017-04-12 14:29:48,http://nicolaenger.info/justine_kautzer,,183.171.103.52,"{""location"": ""SB"", ""is_mobile"": false}" 16116,6,364,2016-12-20 10:05:49,http://beier.co/dulce.adams,,69.94.231.114,"{""location"": ""CF"", ""is_mobile"": false}" 16117,6,364,2017-02-06 00:54:09,http://rutherford.com/brendan,,218.240.214.90,"{""location"": ""MU"", ""is_mobile"": true}" 16118,6,364,2017-04-14 23:30:11,http://rohan.io/elliott,,40.171.166.104,"{""location"": ""LS"", ""is_mobile"": true}" 16119,6,364,2017-06-10 03:05:34,http://sporer.org/meagan.mitchell,,118.89.232.159,"{""location"": ""TF"", ""is_mobile"": true}" 16120,6,364,2017-05-19 02:32:35,http://price.info/destinee_schuster,,23.160.156.106,"{""location"": ""NR"", ""is_mobile"": false}" 16121,6,364,2016-12-23 14:39:34,http://swift.info/ethel_kiehn,,7.84.193.193,"{""location"": ""KY"", ""is_mobile"": false}" 16122,6,364,2016-12-18 21:57:01,http://kertzmannhammes.io/alexzander,,209.109.230.161,"{""location"": ""SJ"", ""is_mobile"": false}" 16123,6,364,2017-01-16 21:47:20,http://moore.org/zion,,155.223.21.174,"{""location"": ""PN"", ""is_mobile"": true}" 16124,6,364,2017-05-26 07:41:25,http://marquardt.name/addison,,55.130.143.188,"{""location"": ""AT"", ""is_mobile"": true}" 16125,6,364,2017-06-02 06:25:33,http://kirlin.name/floy,,55.9.135.106,"{""location"": ""PS"", ""is_mobile"": true}" 16126,6,364,2017-04-09 22:58:15,http://renner.org/sydnie.schamberger,,220.218.130.3,"{""location"": ""MS"", ""is_mobile"": true}" 16127,6,364,2017-05-07 20:31:10,http://schuppeluettgen.io/donald,,26.74.145.131,"{""location"": ""IL"", ""is_mobile"": true}" 16128,6,364,2017-04-15 14:41:12,http://coleterry.org/amelie,,25.238.224.137,"{""location"": ""SZ"", ""is_mobile"": false}" 16129,6,364,2017-01-05 12:35:46,http://dach.co/sheila.labadie,,143.178.179.209,"{""location"": ""LK"", ""is_mobile"": false}" 16130,6,364,2017-05-27 10:00:07,http://huel.co/alexandro.auer,,184.158.128.75,"{""location"": ""BF"", ""is_mobile"": true}" 16131,6,364,2017-01-02 02:05:02,http://metz.name/jamaal_emard,,162.68.37.75,"{""location"": ""MQ"", ""is_mobile"": false}" 16132,6,364,2017-03-21 10:32:03,http://murazik.info/jasper,,188.132.163.190,"{""location"": ""BQ"", ""is_mobile"": false}" 16133,6,364,2017-01-11 20:50:19,http://murazik.co/geraldine,,237.20.139.15,"{""location"": ""SJ"", ""is_mobile"": true}" 16134,6,364,2017-01-29 21:50:33,http://herzog.org/adriel.weinat,,56.176.124.91,"{""location"": ""IO"", ""is_mobile"": false}" 16135,6,364,2016-12-15 21:02:00,http://kulas.org/pansy.mohr,,12.40.189.30,"{""location"": ""PS"", ""is_mobile"": true}" 16136,6,364,2017-01-18 00:01:26,http://senger.io/carole.krajcik,,49.176.236.126,"{""location"": ""SR"", ""is_mobile"": false}" 3266,2,71,2017-03-28 02:04:16,http://rempel.co/eunice.windler,0.6391566473,12.28.231.182,"{""location"": ""MW"", ""is_mobile"": false}" 3267,2,71,2017-05-28 18:16:19,http://zieme.info/malcolm,0.0336052790,88.7.201.103,"{""location"": ""GU"", ""is_mobile"": true}" 3268,2,71,2017-01-23 05:49:45,http://jenkins.org/tamia,0.1427517969,125.6.35.238,"{""location"": ""SN"", ""is_mobile"": true}" 3269,2,71,2017-02-10 08:42:56,http://schoen.name/king,0.3924669421,9.155.84.49,"{""location"": ""WF"", ""is_mobile"": false}" 3270,2,72,2017-01-18 04:43:34,http://schmitt.biz/magdalena.reynolds,0.5146584949,100.153.162.194,"{""location"": ""NL"", ""is_mobile"": true}" 3271,2,72,2017-04-15 18:04:14,http://hauck.io/gus,0.0933673965,157.29.190.27,"{""location"": ""DO"", ""is_mobile"": false}" 3272,2,72,2017-03-17 20:52:54,http://kuvalis.org/anna,0.4406808695,36.176.159.210,"{""location"": ""BM"", ""is_mobile"": true}" 3273,2,72,2017-05-09 14:32:57,http://buckridge.org/gayle,0.6397725698,80.111.77.70,"{""location"": ""MS"", ""is_mobile"": true}" 3274,2,72,2017-05-21 20:47:44,http://sawayn.io/elisabeth.schuster,0.7477722450,28.12.228.164,"{""location"": ""ME"", ""is_mobile"": true}" 3275,2,72,2016-12-25 04:02:14,http://jacobson.name/ruth,0.0867302231,169.244.74.103,"{""location"": ""MC"", ""is_mobile"": true}" 3276,2,72,2017-05-31 10:19:29,http://labadie.io/chelsie,0.5137322247,206.136.224.222,"{""location"": ""CK"", ""is_mobile"": true}" 3277,2,72,2017-02-14 10:16:42,http://beahankulas.co/bradford,0.1128547083,175.51.62.198,"{""location"": ""LC"", ""is_mobile"": false}" 3278,2,72,2017-03-07 04:14:30,http://hilll.net/ova.green,0.5244510176,188.176.130.171,"{""location"": ""GN"", ""is_mobile"": true}" 3279,2,72,2016-12-15 08:00:35,http://barton.io/leila,0.9723739081,83.181.47.51,"{""location"": ""BS"", ""is_mobile"": true}" 3280,2,72,2017-02-04 14:30:08,http://walshbruen.com/gaylord_bradtke,0.1836845997,146.211.71.174,"{""location"": ""KH"", ""is_mobile"": true}" 3281,2,72,2017-01-10 20:20:02,http://harris.name/kyleigh,0.9938335481,171.15.188.220,"{""location"": ""MX"", ""is_mobile"": false}" 3282,2,72,2017-02-10 13:15:11,http://ward.net/charlotte.bergstrom,0.3086160407,173.32.180.4,"{""location"": ""NR"", ""is_mobile"": true}" 3283,2,72,2017-02-26 07:07:22,http://macgyvermurphy.org/darren,0.6509816301,151.53.226.48,"{""location"": ""SG"", ""is_mobile"": true}" 3284,2,72,2017-01-31 12:45:19,http://kunde.org/garry_larson,0.4068417133,14.29.229.209,"{""location"": ""MU"", ""is_mobile"": false}" 3285,2,72,2017-05-13 01:18:45,http://runolfon.biz/nicole,0.0522700384,98.24.249.54,"{""location"": ""BZ"", ""is_mobile"": true}" 3286,2,72,2017-04-09 02:09:58,http://haag.biz/kory_runolfsdottir,0.1137819421,180.223.78.82,"{""location"": ""GF"", ""is_mobile"": false}" 3287,2,72,2017-06-07 07:03:42,http://bahringerkaulke.net/lucy,0.2120947589,55.69.120.12,"{""location"": ""LI"", ""is_mobile"": false}" 3288,2,72,2017-01-14 06:01:46,http://trantow.co/julia,0.6252401982,89.187.209.82,"{""location"": ""MT"", ""is_mobile"": false}" 3289,2,72,2017-01-28 22:10:23,http://vandervort.info/sincere,0.9594850765,167.7.40.183,"{""location"": ""BS"", ""is_mobile"": true}" 3290,2,73,2017-02-22 13:45:02,http://bernier.io/margie,0.4402075526,127.62.141.89,"{""location"": ""PK"", ""is_mobile"": true}" 3291,2,73,2017-04-27 03:33:20,http://lowe.net/cayla_boehm,0.5802552046,23.228.236.126,"{""location"": ""TF"", ""is_mobile"": true}" 3292,2,73,2017-02-16 07:35:05,http://lubowitzmurazik.com/yolanda,0.7643475197,251.14.2.216,"{""location"": ""PW"", ""is_mobile"": false}" 3293,2,73,2017-03-22 15:20:18,http://batz.co/xzavier.heathcote,0.9593511680,32.179.175.95,"{""location"": ""LI"", ""is_mobile"": false}" 3294,2,73,2016-12-20 02:10:08,http://cummerata.biz/beth,0.2271449926,98.43.74.238,"{""location"": ""DM"", ""is_mobile"": true}" 3295,2,73,2017-04-05 21:37:30,http://ferrykiehn.net/alisha,0.0178964756,103.94.247.167,"{""location"": ""UA"", ""is_mobile"": true}" 3296,2,73,2017-01-27 18:43:03,http://turnercollier.org/casandra,0.0694885468,6.48.155.115,"{""location"": ""FO"", ""is_mobile"": false}" 3297,2,73,2016-12-26 13:25:03,http://dietrichweinat.co/stephen_strosin,0.8349198175,91.143.97.106,"{""location"": ""DO"", ""is_mobile"": false}" 3298,2,73,2017-02-07 21:56:48,http://feil.info/raphael,0.6614312058,247.72.7.165,"{""location"": ""MD"", ""is_mobile"": true}" 3299,2,73,2016-12-28 18:27:18,http://orn.name/pietro.goodwin,0.4649633682,79.170.234.66,"{""location"": ""MG"", ""is_mobile"": true}" 3300,2,73,2016-12-16 14:47:00,http://stroman.net/lula.wiza,0.3136502613,179.151.32.168,"{""location"": ""SR"", ""is_mobile"": true}" 3301,2,73,2017-02-21 09:01:20,http://predovic.org/mekhi.parisian,0.8899581913,171.86.227.52,"{""location"": ""FK"", ""is_mobile"": true}" 3302,2,73,2017-03-17 02:42:37,http://pfeffer.name/richie,0.0113097917,111.133.203.167,"{""location"": ""PY"", ""is_mobile"": true}" 3303,2,73,2017-03-27 12:06:31,http://damoreadams.com/rowland.beier,0.8226942708,201.189.136.201,"{""location"": ""MS"", ""is_mobile"": true}" 3304,2,73,2017-05-06 06:10:53,http://lemkeboyle.net/bettie,0.1657922807,200.75.60.70,"{""location"": ""GM"", ""is_mobile"": false}" 3305,2,73,2017-05-02 00:44:01,http://johns.biz/adrien_olson,0.0178767661,239.211.230.200,"{""location"": ""IO"", ""is_mobile"": false}" 3306,2,73,2016-12-17 21:12:24,http://wisozkwilkinson.name/abby_schaefer,0.1171075303,221.210.178.65,"{""location"": ""MZ"", ""is_mobile"": true}" 3307,2,73,2017-03-23 06:19:55,http://hammes.io/katelin,0.8455369558,6.131.100.251,"{""location"": ""EC"", ""is_mobile"": true}" 3308,2,73,2017-05-02 11:17:45,http://swaniawski.io/shemar.veum,0.3896119467,48.131.138.226,"{""location"": ""MH"", ""is_mobile"": true}" 3309,2,73,2017-01-12 00:04:16,http://schultzwelch.biz/madisen.armstrong,0.6011708012,117.194.101.39,"{""location"": ""MZ"", ""is_mobile"": true}" 3310,2,73,2017-02-04 16:52:51,http://reingerkemmer.com/kelley,0.8961612974,114.44.50.224,"{""location"": ""NE"", ""is_mobile"": false}" 3311,2,73,2017-04-18 14:04:57,http://douglas.name/jillian,0.2516802338,105.127.178.98,"{""location"": ""GY"", ""is_mobile"": false}" 3312,2,73,2016-12-26 09:31:37,http://boehm.org/cristian.pfeffer,0.2530844474,165.154.29.116,"{""location"": ""BM"", ""is_mobile"": true}" 3313,2,73,2017-04-08 10:45:05,http://morar.name/hailee.rolfson,0.6116291765,119.77.12.204,"{""location"": ""AF"", ""is_mobile"": true}" 3314,2,73,2017-06-03 02:47:07,http://hellerpacocha.com/antone.brakus,0.3337944704,107.137.127.179,"{""location"": ""SK"", ""is_mobile"": true}" 3315,2,73,2017-03-09 15:39:35,http://zemlakschowalter.biz/brisa.pouros,0.4950607822,115.87.142.171,"{""location"": ""SM"", ""is_mobile"": true}" 3316,2,73,2017-05-06 11:13:44,http://johnsoncarroll.info/rylee_altenwerth,0.2855759377,84.57.92.120,"{""location"": ""PF"", ""is_mobile"": false}" 3317,2,73,2017-04-24 02:12:14,http://kleinzemlak.net/casandra,0.3156966336,147.197.80.69,"{""location"": ""TG"", ""is_mobile"": true}" 10200,4,229,2017-01-26 06:32:18,http://jast.info/francisco,0.1741305127,66.68.174.125,"{""location"": ""CW"", ""is_mobile"": true}" 10201,4,229,2017-01-20 09:24:43,http://kohler.name/tyshawn,0.2718992467,50.250.244.47,"{""location"": ""FJ"", ""is_mobile"": false}" 10202,4,229,2017-04-23 14:25:30,http://wiegand.io/julio,0.0430457183,138.95.218.34,"{""location"": ""GS"", ""is_mobile"": false}" 10203,4,229,2017-04-07 19:17:38,http://lueilwitz.org/elliot.douglas,0.5340850875,105.23.3.145,"{""location"": ""LB"", ""is_mobile"": true}" 10204,4,229,2017-04-24 23:43:53,http://farrell.io/arnoldo,0.7018283964,110.42.5.237,"{""location"": ""DM"", ""is_mobile"": false}" 10205,4,229,2017-01-29 04:02:31,http://jacobson.co/anastacio_carter,0.5890158237,133.76.237.100,"{""location"": ""UM"", ""is_mobile"": false}" 10206,4,229,2017-02-02 17:08:05,http://schummschulist.info/piper,0.2709407495,239.210.142.107,"{""location"": ""SS"", ""is_mobile"": true}" 10207,4,229,2017-02-06 01:02:15,http://nicolaskris.name/savion,0.2522551951,58.207.9.251,"{""location"": ""RE"", ""is_mobile"": false}" 10208,4,229,2017-06-06 05:39:13,http://bergnaum.biz/terrence.oconner,0.1134816208,23.33.211.21,"{""location"": ""JP"", ""is_mobile"": true}" 10209,4,229,2017-03-12 22:53:59,http://binspacocha.net/beulah,0.6774651173,150.178.251.183,"{""location"": ""CK"", ""is_mobile"": false}" 10210,4,230,2017-06-02 15:56:13,http://barrows.info/imogene_koelpin,0.7798749067,156.8.113.40,"{""location"": ""KH"", ""is_mobile"": false}" 20872,7,469,2017-05-23 15:58:01,http://moen.org/vivian,,90.132.141.249,"{""location"": ""UM"", ""is_mobile"": true}" 10211,4,230,2017-05-26 02:30:26,http://grant.net/jarrett_nikolaus,0.2230757995,197.162.190.93,"{""location"": ""FO"", ""is_mobile"": true}" 10212,4,230,2017-02-18 09:40:45,http://windler.biz/elias,0.6824598870,127.100.141.72,"{""location"": ""TL"", ""is_mobile"": false}" 10213,4,230,2017-04-22 23:08:24,http://bartell.net/zoey.fay,0.0381699580,211.160.65.125,"{""location"": ""EG"", ""is_mobile"": true}" 10214,4,230,2017-06-07 04:39:12,http://kuhlman.biz/jayne,0.0193422758,155.133.194.136,"{""location"": ""GM"", ""is_mobile"": true}" 10215,4,230,2017-03-27 00:32:21,http://ornwunsch.org/alvena,0.3715409956,97.20.112.91,"{""location"": ""KH"", ""is_mobile"": true}" 10216,4,230,2016-12-28 23:22:34,http://lakin.org/addison,0.3622046372,89.231.226.131,"{""location"": ""CM"", ""is_mobile"": false}" 10217,4,230,2017-03-12 09:44:40,http://stroman.biz/edmond,0.1784426963,173.84.203.156,"{""location"": ""NZ"", ""is_mobile"": false}" 10218,4,230,2017-04-16 07:26:16,http://kulasgulgowski.org/collin_gottlieb,0.6692173860,254.67.12.95,"{""location"": ""ME"", ""is_mobile"": true}" 10219,4,230,2017-03-03 00:09:15,http://little.net/hipolito,0.8529434057,246.74.81.148,"{""location"": ""UG"", ""is_mobile"": true}" 10220,4,230,2017-03-07 11:54:27,http://jacobson.net/lance_howell,0.4083775344,242.166.239.65,"{""location"": ""LT"", ""is_mobile"": true}" 10221,4,230,2017-04-22 00:21:16,http://wyman.io/paige.kemmer,0.2905651718,85.235.200.5,"{""location"": ""PE"", ""is_mobile"": false}" 10222,4,230,2017-06-09 03:59:02,http://murray.org/violet,0.8919752332,250.116.210.220,"{""location"": ""AR"", ""is_mobile"": true}" 10223,4,230,2016-12-26 14:22:23,http://yundtharris.co/elda,0.9849185153,160.143.224.50,"{""location"": ""MA"", ""is_mobile"": true}" 10224,4,230,2016-12-17 12:08:07,http://ruecker.info/alyce,0.3545076803,162.115.223.126,"{""location"": ""BA"", ""is_mobile"": false}" 10225,4,230,2017-04-18 02:49:06,http://marks.net/clifford,0.6911274923,91.111.160.15,"{""location"": ""TD"", ""is_mobile"": true}" 10226,4,230,2017-01-01 22:24:27,http://kuhnstark.com/katelynn,0.0012205319,156.43.161.215,"{""location"": ""ML"", ""is_mobile"": false}" 10227,4,230,2017-04-17 00:46:52,http://borer.name/shad,0.7690708637,113.2.154.190,"{""location"": ""GD"", ""is_mobile"": true}" 10228,4,230,2017-03-09 09:52:39,http://cummerataparker.biz/ella,0.2591016210,176.146.8.145,"{""location"": ""ET"", ""is_mobile"": true}" 10229,4,230,2016-12-24 10:44:51,http://zemlakbins.name/johnathan,0.0490166439,249.45.19.68,"{""location"": ""PW"", ""is_mobile"": true}" 10230,4,230,2017-03-09 20:35:37,http://breitenberg.io/ryann_kling,0.9360933206,197.197.70.46,"{""location"": ""VE"", ""is_mobile"": false}" 10231,4,230,2017-01-23 15:15:42,http://bahringerlebsack.org/mazie,0.0210043523,4.94.207.219,"{""location"": ""HM"", ""is_mobile"": false}" 10232,4,230,2017-03-31 12:19:34,http://welchbuckridge.info/tania,0.6583608785,79.248.64.250,"{""location"": ""ZM"", ""is_mobile"": false}" 10233,4,230,2017-04-08 21:39:23,http://schmelerpadberg.org/sherman,0.4406905517,140.199.78.225,"{""location"": ""GM"", ""is_mobile"": false}" 10234,4,230,2017-05-31 18:17:59,http://lefflerrath.io/selena_huels,0.9065213498,42.144.172.62,"{""location"": ""GW"", ""is_mobile"": false}" 10235,4,230,2017-02-19 05:18:48,http://pagac.biz/vickie_gleason,0.3308059198,147.192.118.125,"{""location"": ""AZ"", ""is_mobile"": false}" 10236,4,230,2017-04-05 10:54:20,http://bodeveum.org/sydney_dicki,0.8490904798,34.82.230.28,"{""location"": ""FR"", ""is_mobile"": false}" 10237,4,230,2017-04-13 14:23:41,http://bednarlockman.info/kaycee,0.8028305025,28.130.178.233,"{""location"": ""CX"", ""is_mobile"": false}" 10238,4,230,2017-02-23 09:44:22,http://swift.co/declan,0.6632006793,117.186.222.52,"{""location"": ""BQ"", ""is_mobile"": true}" 10239,4,230,2017-04-11 20:01:35,http://schulist.org/stephania_lynch,0.2528724275,12.213.154.195,"{""location"": ""BT"", ""is_mobile"": true}" 10240,4,230,2017-06-12 05:00:07,http://raynorbernier.org/rebekah.runolfon,0.3122378582,250.66.35.113,"{""location"": ""BW"", ""is_mobile"": true}" 10241,4,230,2017-03-02 09:35:36,http://conroy.co/kiera_sawayn,0.4890252655,74.20.103.28,"{""location"": ""SB"", ""is_mobile"": false}" 10242,4,230,2017-05-30 02:35:02,http://cartermckenzie.co/alexandrine.kautzer,0.3295967331,129.251.141.187,"{""location"": ""PH"", ""is_mobile"": true}" 10243,4,230,2017-02-15 04:24:13,http://hirthe.org/jacey,0.7885282345,2.190.89.157,"{""location"": ""AF"", ""is_mobile"": true}" 10244,4,231,2017-02-24 05:12:24,http://feest.net/hoyt.kshlerin,0.2949218997,49.166.254.149,"{""location"": ""DM"", ""is_mobile"": true}" 10245,4,231,2017-04-09 04:55:57,http://lednermills.name/dora,0.4072513167,64.156.153.54,"{""location"": ""LS"", ""is_mobile"": true}" 10246,4,231,2017-03-14 18:41:58,http://wisozk.info/theodora,0.8454667686,38.118.119.90,"{""location"": ""BV"", ""is_mobile"": false}" 10247,4,231,2017-04-23 21:57:15,http://schuppe.org/hettie_larson,0.6427273204,67.234.183.196,"{""location"": ""IT"", ""is_mobile"": false}" 10248,4,231,2017-02-08 13:30:11,http://bins.info/skye,0.5986265650,176.58.240.168,"{""location"": ""CH"", ""is_mobile"": true}" 10249,4,231,2017-01-04 05:29:25,http://wunschoconner.name/lacy,0.4839622595,135.150.195.208,"{""location"": ""GT"", ""is_mobile"": true}" 7212,3,160,2017-05-09 15:30:35,http://jacobs.biz/maynard.pfeffer,0.3597311400,102.140.58.164,"{""location"": ""CY"", ""is_mobile"": false}" 7213,3,160,2017-05-23 04:13:20,http://lang.com/dan,0.7050599590,199.139.118.41,"{""location"": ""DO"", ""is_mobile"": false}" 7214,3,160,2017-03-24 10:46:32,http://mcglynn.biz/frieda,0.2764999604,225.196.97.25,"{""location"": ""BS"", ""is_mobile"": true}" 7215,3,160,2017-02-21 23:58:31,http://rice.info/jason,0.2883737204,167.94.80.149,"{""location"": ""FJ"", ""is_mobile"": false}" 7216,3,160,2017-04-04 23:00:57,http://wisozk.biz/shawn,0.5024763017,87.172.193.137,"{""location"": ""AX"", ""is_mobile"": true}" 7217,3,160,2017-06-07 20:21:09,http://grantweimann.name/pauline_okuneva,0.6485795140,237.73.179.200,"{""location"": ""LT"", ""is_mobile"": true}" 7218,3,160,2017-06-05 02:49:38,http://roob.info/petra.dach,0.2186605065,110.88.208.230,"{""location"": ""MQ"", ""is_mobile"": true}" 7219,3,160,2017-05-28 13:06:54,http://kihn.org/felix_bernier,0.0719621123,95.243.206.87,"{""location"": ""SM"", ""is_mobile"": true}" 7220,3,160,2017-01-29 05:00:56,http://skiles.org/mariane,0.0256635760,237.164.56.60,"{""location"": ""FO"", ""is_mobile"": false}" 7221,3,160,2017-04-22 09:13:15,http://kriscormier.org/flavio,0.0322540759,128.84.206.175,"{""location"": ""CA"", ""is_mobile"": false}" 7222,3,160,2017-03-03 12:32:00,http://osinski.info/aleen,0.2322720326,86.28.213.59,"{""location"": ""NC"", ""is_mobile"": true}" 7223,3,160,2017-04-28 17:57:17,http://lebsack.org/jacey,0.9540180941,244.53.31.25,"{""location"": ""MR"", ""is_mobile"": true}" 7224,3,160,2017-03-24 22:48:19,http://feeney.biz/martina.deckow,0.0153660401,202.163.172.238,"{""location"": ""MW"", ""is_mobile"": false}" 7225,3,160,2017-04-29 07:13:14,http://purdy.co/providenci,0.4966021377,154.100.5.135,"{""location"": ""LU"", ""is_mobile"": false}" 7226,3,160,2017-03-26 11:44:09,http://weinat.com/nat.fisher,0.1158819019,8.122.34.206,"{""location"": ""EH"", ""is_mobile"": true}" 7227,3,160,2017-03-01 17:45:04,http://hayes.biz/nasir,0.8534083027,79.191.6.111,"{""location"": ""CM"", ""is_mobile"": false}" 7228,3,160,2017-06-03 18:15:19,http://kozeygislason.name/elmer_auer,0.7647156352,94.68.116.131,"{""location"": ""FI"", ""is_mobile"": false}" 7229,3,160,2017-04-27 14:20:27,http://grant.org/gaston,0.5205816062,249.124.41.231,"{""location"": ""QA"", ""is_mobile"": true}" 7230,3,160,2017-05-17 04:05:47,http://oconnerbuckridge.co/forrest,0.0424716245,157.114.49.171,"{""location"": ""MR"", ""is_mobile"": true}" 7231,3,160,2017-01-04 03:33:31,http://parker.co/talon.waters,0.8882526399,135.172.87.15,"{""location"": ""NR"", ""is_mobile"": false}" 7232,3,160,2017-05-24 23:36:01,http://murraynienow.co/everardo_stanton,0.3773239432,242.125.171.195,"{""location"": ""YE"", ""is_mobile"": false}" 7233,3,160,2016-12-29 23:57:43,http://hermann.com/ivory.crooks,0.7807960854,59.118.101.153,"{""location"": ""TC"", ""is_mobile"": false}" 7234,3,160,2017-01-03 12:12:07,http://manteleuschke.io/isabelle.sipes,0.8652077085,175.5.204.173,"{""location"": ""CM"", ""is_mobile"": false}" 7235,3,160,2017-02-02 14:07:25,http://tremblay.co/harrison,0.8091720675,81.133.85.234,"{""location"": ""ID"", ""is_mobile"": false}" 7236,3,160,2017-01-13 09:59:58,http://gislasonconn.io/andy_strosin,0.6897593688,204.138.123.2,"{""location"": ""SZ"", ""is_mobile"": false}" 7237,3,160,2017-03-14 21:11:41,http://barrows.info/triston,0.5148206784,20.133.93.109,"{""location"": ""NO"", ""is_mobile"": true}" 7238,3,160,2016-12-31 09:53:12,http://altenwerth.name/marty,0.4154112922,95.249.205.134,"{""location"": ""GR"", ""is_mobile"": false}" 7239,3,160,2017-04-27 23:05:24,http://marks.biz/lizeth.lubowitz,0.2340357948,76.99.53.219,"{""location"": ""TC"", ""is_mobile"": true}" 7240,3,160,2017-01-03 07:04:11,http://kerluke.biz/verner.brekke,0.4208857998,40.37.204.128,"{""location"": ""UY"", ""is_mobile"": false}" 7241,3,160,2017-03-27 14:41:03,http://ebert.org/anais_mosciski,0.7532544265,15.56.250.213,"{""location"": ""SO"", ""is_mobile"": false}" 7242,3,160,2017-01-10 05:13:55,http://waelchi.co/mathilde_raynor,0.5638954264,133.244.13.12,"{""location"": ""VC"", ""is_mobile"": false}" 7243,3,160,2016-12-29 08:27:50,http://abshirepouros.biz/wendy,0.6920799070,10.130.54.19,"{""location"": ""FK"", ""is_mobile"": false}" 7244,3,160,2017-04-14 05:16:19,http://andersonthompson.biz/cornell.kling,0.5368915854,80.49.122.161,"{""location"": ""SH"", ""is_mobile"": false}" 7245,3,160,2017-05-09 12:41:05,http://stokes.net/peter,0.2712604449,115.133.121.56,"{""location"": ""MZ"", ""is_mobile"": true}" 7246,3,160,2017-02-12 12:47:20,http://tromprempel.biz/jesus,0.1979286348,158.201.152.122,"{""location"": ""BS"", ""is_mobile"": false}" 7247,3,161,2017-06-05 01:08:07,http://borer.biz/nils.nitzsche,0.3749918622,33.58.58.4,"{""location"": ""ME"", ""is_mobile"": true}" 7248,3,161,2017-04-23 00:52:05,http://goodwin.co/freeman,0.7390166648,57.209.43.186,"{""location"": ""FJ"", ""is_mobile"": true}" 7249,3,161,2017-05-29 22:34:30,http://yundtjast.biz/marisol_johnston,0.2106895314,44.3.77.104,"{""location"": ""ER"", ""is_mobile"": true}" 7250,3,161,2017-01-20 04:25:03,http://wisoky.com/kevin,0.8543972732,74.102.125.152,"{""location"": ""CM"", ""is_mobile"": true}" 7251,3,161,2017-02-22 15:40:30,http://kutch.com/kale,0.1853705488,74.145.69.62,"{""location"": ""AT"", ""is_mobile"": true}" 7252,3,161,2017-05-21 07:43:06,http://sporer.co/cyrus_simonis,0.8633512922,29.199.210.102,"{""location"": ""KW"", ""is_mobile"": true}" 7253,3,161,2017-02-03 23:50:04,http://sporermcdermott.info/peter,0.5044924462,24.208.238.205,"{""location"": ""BQ"", ""is_mobile"": false}" 7254,3,161,2017-02-01 22:23:11,http://daniel.biz/agustina,0.9450692001,195.217.192.58,"{""location"": ""SY"", ""is_mobile"": false}" 7255,3,161,2017-03-23 14:00:33,http://grimes.com/van.wuckert,0.8101462776,191.94.224.152,"{""location"": ""ZA"", ""is_mobile"": true}" 7256,3,161,2017-04-29 20:51:18,http://ernserreichel.co/jeika,0.6061784273,72.193.16.219,"{""location"": ""PR"", ""is_mobile"": true}" 7257,3,161,2017-05-26 06:24:58,http://lockman.net/sharon_collier,0.0252690386,127.51.194.62,"{""location"": ""CR"", ""is_mobile"": true}" 7258,3,161,2017-05-16 20:56:01,http://strosinkling.com/brett_walker,0.2324779106,232.85.29.237,"{""location"": ""ES"", ""is_mobile"": true}" 7259,3,161,2017-04-25 16:20:50,http://terry.io/hiram,0.3811084138,222.55.190.36,"{""location"": ""CM"", ""is_mobile"": false}" 7260,3,161,2017-04-26 10:06:47,http://friesen.com/peggie_mcglynn,0.6045645532,81.199.12.157,"{""location"": ""SK"", ""is_mobile"": true}" 7261,3,161,2017-02-08 07:53:59,http://leschcasper.co/sherwood.bartoletti,0.4746615700,86.154.186.168,"{""location"": ""ZW"", ""is_mobile"": true}" 7262,3,161,2017-04-04 03:52:16,http://harber.io/merritt_borer,0.3552821742,30.113.186.110,"{""location"": ""SH"", ""is_mobile"": false}" 13118,5,294,2016-12-18 18:19:12,http://bins.net/lisette,0.1851633740,227.79.148.3,"{""location"": ""TZ"", ""is_mobile"": false}" 13119,5,294,2017-06-05 18:20:00,http://wizaframi.net/alisa.kilback,0.3033689630,114.144.71.173,"{""location"": ""GH"", ""is_mobile"": true}" 13120,5,294,2017-01-08 03:00:46,http://fritsch.info/rashawn,0.9782695112,152.185.247.240,"{""location"": ""MR"", ""is_mobile"": true}" 13121,5,294,2017-04-11 07:30:09,http://von.co/haskell.raynor,0.4098467227,252.42.67.183,"{""location"": ""AD"", ""is_mobile"": true}" 13122,5,294,2017-03-15 18:22:58,http://crooks.co/jaylon,0.0258377265,159.126.54.107,"{""location"": ""GT"", ""is_mobile"": false}" 13123,5,294,2017-05-18 18:20:57,http://maggio.co/augustus,0.1032491087,198.72.97.10,"{""location"": ""MM"", ""is_mobile"": false}" 13124,5,294,2017-05-07 19:30:17,http://hartmann.name/oliver,0.5149978408,198.95.193.54,"{""location"": ""AX"", ""is_mobile"": true}" 13125,5,294,2017-04-28 04:28:56,http://bartellhyatt.info/gage,0.7303327798,165.114.175.56,"{""location"": ""SZ"", ""is_mobile"": false}" 13126,5,295,2017-04-12 06:38:32,http://abbott.biz/carmelo.hane,,121.65.204.71,"{""location"": ""TG"", ""is_mobile"": false}" 13127,5,295,2017-06-01 12:03:34,http://bauch.co/donavon_adams,,94.84.142.215,"{""location"": ""CD"", ""is_mobile"": true}" 13128,5,295,2017-01-30 01:23:05,http://jacobilindgren.biz/chauncey_hyatt,,97.67.121.30,"{""location"": ""NU"", ""is_mobile"": false}" 13129,5,295,2016-12-30 10:26:25,http://gerhold.name/austen.lubowitz,,9.26.183.188,"{""location"": ""GI"", ""is_mobile"": false}" 13130,5,295,2017-02-04 21:19:58,http://keler.co/amparo_becker,,163.130.75.254,"{""location"": ""BD"", ""is_mobile"": true}" 13131,5,295,2017-02-12 20:21:08,http://keeblerabbott.org/furman,,241.221.93.23,"{""location"": ""FR"", ""is_mobile"": true}" 13132,5,295,2017-02-18 09:25:43,http://veumankunding.io/bennie,,6.132.90.68,"{""location"": ""SE"", ""is_mobile"": false}" 13133,5,295,2017-03-15 21:58:26,http://krajcik.name/felicity,,38.44.81.86,"{""location"": ""CX"", ""is_mobile"": true}" 13134,5,295,2017-01-21 08:03:35,http://torphy.biz/percival,,116.119.172.221,"{""location"": ""EG"", ""is_mobile"": true}" 13135,5,295,2017-01-24 14:00:20,http://millergraham.co/akeem,,9.219.126.149,"{""location"": ""PG"", ""is_mobile"": false}" 13136,5,295,2017-01-05 12:43:19,http://rathgerlach.name/barton,,118.82.249.44,"{""location"": ""SK"", ""is_mobile"": true}" 13137,5,295,2017-04-02 20:15:25,http://gutkowskikris.net/graciela,,222.173.244.21,"{""location"": ""HR"", ""is_mobile"": false}" 13138,5,295,2017-05-15 00:39:51,http://koeppspinka.info/jayne_heel,,113.202.114.115,"{""location"": ""VC"", ""is_mobile"": false}" 13139,5,295,2017-06-11 04:08:50,http://schmeler.net/natalie.schumm,,95.106.37.172,"{""location"": ""SB"", ""is_mobile"": true}" 13140,5,295,2017-05-04 02:18:29,http://nienow.biz/karl.bauch,,182.53.128.60,"{""location"": ""NI"", ""is_mobile"": true}" 13141,5,295,2016-12-15 19:46:25,http://funk.biz/caria,,204.241.160.146,"{""location"": ""BH"", ""is_mobile"": true}" 13142,5,295,2017-06-01 06:58:35,http://dietrich.name/lucas.windler,,116.196.119.192,"{""location"": ""SX"", ""is_mobile"": false}" 13143,5,295,2017-05-31 16:12:18,http://hermiston.info/jolie,,178.162.117.88,"{""location"": ""GY"", ""is_mobile"": false}" 13144,5,295,2017-02-01 17:00:57,http://bernier.com/stuart,,72.172.8.6,"{""location"": ""AE"", ""is_mobile"": false}" 13145,5,295,2017-04-27 20:04:29,http://labadiehintz.org/mayra,,216.48.105.246,"{""location"": ""KZ"", ""is_mobile"": true}" 13146,5,295,2017-02-19 00:34:25,http://terrymraz.biz/casper_keler,,111.78.22.98,"{""location"": ""ZW"", ""is_mobile"": false}" 13147,5,295,2017-02-08 17:54:03,http://glover.io/jakayla_hand,,192.35.15.96,"{""location"": ""KW"", ""is_mobile"": false}" 13148,5,295,2017-04-09 19:48:01,http://reichertdaniel.biz/josh.will,,181.15.99.237,"{""location"": ""CY"", ""is_mobile"": true}" 13149,5,295,2017-05-30 19:19:28,http://ebert.net/dell.powlowski,,118.53.125.39,"{""location"": ""HU"", ""is_mobile"": false}" 13150,5,295,2017-05-27 00:13:12,http://nolan.net/bartholome_abbott,,53.159.47.226,"{""location"": ""UZ"", ""is_mobile"": false}" 13151,5,295,2017-05-22 01:37:16,http://mcglynn.io/kelsi_schmitt,,27.98.58.234,"{""location"": ""MD"", ""is_mobile"": true}" 13152,5,295,2017-04-11 07:40:57,http://ernser.org/milford,,228.143.171.78,"{""location"": ""CW"", ""is_mobile"": false}" 13153,5,295,2017-04-17 06:43:45,http://breitenberg.io/gregg,,116.84.12.251,"{""location"": ""SX"", ""is_mobile"": true}" 13154,5,295,2017-03-25 01:56:03,http://dicki.co/emiliano.zieme,,178.211.35.164,"{""location"": ""LT"", ""is_mobile"": true}" 13155,5,295,2017-01-07 11:04:28,http://conroy.org/esther,,191.166.183.215,"{""location"": ""AM"", ""is_mobile"": true}" 13156,5,295,2017-05-02 09:47:36,http://wisozk.name/sam,,183.239.185.104,"{""location"": ""DE"", ""is_mobile"": false}" 13157,5,295,2017-02-20 03:39:37,http://lakin.org/schuyler_langworth,,190.170.241.79,"{""location"": ""CY"", ""is_mobile"": false}" 13158,5,295,2017-06-11 17:05:44,http://goyettenolan.net/rickie.hayes,,24.148.116.113,"{""location"": ""ML"", ""is_mobile"": false}" 13159,5,295,2017-03-10 21:35:36,http://moen.org/ayden.fay,,241.53.162.46,"{""location"": ""MM"", ""is_mobile"": true}" 13160,5,296,2017-02-06 15:36:26,http://hartmannbins.net/max.hilll,,179.69.233.241,"{""location"": ""KY"", ""is_mobile"": true}" 13161,5,296,2017-02-14 16:55:13,http://weber.co/evalyn,,20.161.92.229,"{""location"": ""IE"", ""is_mobile"": true}" 13162,5,296,2017-04-23 01:00:57,http://boscodamore.io/agustina,,214.44.186.187,"{""location"": ""LB"", ""is_mobile"": false}" 13163,5,296,2017-05-15 18:58:48,http://stammnikolaus.name/newton.oreilly,,72.218.111.159,"{""location"": ""SO"", ""is_mobile"": true}" 13164,5,296,2017-02-24 22:05:40,http://runolfsdottir.com/kian.fahey,,244.21.233.152,"{""location"": ""MH"", ""is_mobile"": true}" 13165,5,296,2017-04-23 20:44:19,http://boyerstroman.com/kristian,,126.157.201.64,"{""location"": ""GY"", ""is_mobile"": true}" 13166,5,296,2017-01-04 18:26:00,http://harvey.name/dolly_anderson,,96.36.151.96,"{""location"": ""KW"", ""is_mobile"": false}" 13167,5,296,2017-03-14 19:07:59,http://smithamkling.co/salvatore.douglas,,186.121.114.185,"{""location"": ""CU"", ""is_mobile"": true}" 13168,5,296,2017-01-03 01:20:17,http://romaguera.io/hilma,,22.187.217.70,"{""location"": ""BZ"", ""is_mobile"": false}" 13169,5,296,2017-03-16 21:43:36,http://cummings.io/marlee,,128.189.197.59,"{""location"": ""DZ"", ""is_mobile"": false}" 13170,5,296,2016-12-27 00:07:49,http://pollich.co/lempi,,15.120.65.72,"{""location"": ""IL"", ""is_mobile"": true}" 13171,5,296,2017-02-07 21:40:38,http://kovacek.com/lottie.medhurst,,31.110.237.224,"{""location"": ""LV"", ""is_mobile"": false}" 13172,5,296,2017-04-03 11:28:39,http://skiles.info/elbert.ryan,,77.229.114.213,"{""location"": ""MO"", ""is_mobile"": true}" 16137,6,364,2017-02-01 06:36:09,http://greenholtturner.io/ernest_wilkinson,,233.71.182.243,"{""location"": ""BJ"", ""is_mobile"": false}" 16138,6,364,2016-12-27 22:08:41,http://mayert.info/pat,,88.246.53.133,"{""location"": ""BY"", ""is_mobile"": false}" 16139,6,364,2017-05-16 18:22:29,http://halvorsonroob.io/norwood.lebsack,,61.191.104.240,"{""location"": ""NA"", ""is_mobile"": true}" 16140,6,364,2017-01-15 09:49:14,http://bosco.io/conner_bergnaum,,30.251.10.66,"{""location"": ""MU"", ""is_mobile"": true}" 16141,6,364,2017-03-02 00:17:12,http://macgyverfarrell.com/laney_cole,,81.175.98.7,"{""location"": ""NG"", ""is_mobile"": true}" 16142,6,364,2017-04-22 22:51:56,http://abshire.org/theo.dibbert,,130.117.67.238,"{""location"": ""SG"", ""is_mobile"": true}" 16143,6,364,2017-02-28 06:21:16,http://douglasleffler.biz/lyda.hand,,190.81.89.96,"{""location"": ""AQ"", ""is_mobile"": true}" 16144,6,364,2016-12-16 22:56:10,http://kautzerzemlak.name/jaclyn,,147.53.244.58,"{""location"": ""RE"", ""is_mobile"": false}" 16145,6,364,2017-03-18 17:37:13,http://douglas.biz/jasmin,,96.38.85.186,"{""location"": ""CF"", ""is_mobile"": true}" 16146,6,364,2017-01-21 18:00:40,http://kaulke.com/ettie.ritchie,,150.134.130.54,"{""location"": ""JE"", ""is_mobile"": true}" 16147,6,364,2017-06-02 09:46:06,http://graham.com/maya,,80.163.133.145,"{""location"": ""KM"", ""is_mobile"": false}" 16148,6,364,2017-04-11 09:12:27,http://jacobi.biz/alberto_bartoletti,,199.85.39.224,"{""location"": ""BQ"", ""is_mobile"": true}" 16149,6,364,2017-06-09 21:39:42,http://schoen.info/laverna_wisoky,,244.65.159.248,"{""location"": ""CA"", ""is_mobile"": false}" 16150,6,364,2017-05-27 05:32:38,http://nader.biz/delpha.haley,,150.93.28.18,"{""location"": ""PL"", ""is_mobile"": false}" 16151,6,364,2017-02-25 18:19:40,http://cormier.org/neal_denesik,,5.245.85.144,"{""location"": ""TF"", ""is_mobile"": true}" 16152,6,364,2017-04-06 12:42:42,http://spinka.info/janie,,101.24.42.174,"{""location"": ""IT"", ""is_mobile"": true}" 16153,6,364,2017-02-07 17:10:46,http://little.io/elody,,121.204.99.230,"{""location"": ""DJ"", ""is_mobile"": false}" 16154,6,365,2016-12-18 14:36:12,http://breitenbergbahringer.net/gay.bartoletti,,84.94.210.235,"{""location"": ""DJ"", ""is_mobile"": true}" 16155,6,365,2017-05-24 11:25:00,http://lemkeeffertz.net/francis,,23.86.126.13,"{""location"": ""BS"", ""is_mobile"": true}" 16156,6,365,2017-01-15 23:30:49,http://gutkowskikuphal.net/marcellus,,248.179.9.90,"{""location"": ""IQ"", ""is_mobile"": false}" 16157,6,365,2017-02-01 08:40:31,http://colewalsh.net/elvera,,13.6.229.148,"{""location"": ""DZ"", ""is_mobile"": false}" 16158,6,365,2017-05-21 15:23:57,http://olson.name/judah_hintz,,115.42.5.250,"{""location"": ""KW"", ""is_mobile"": false}" 16159,6,365,2017-02-06 04:10:38,http://hackett.name/hermann.kerluke,,83.210.225.71,"{""location"": ""SM"", ""is_mobile"": true}" 16160,6,365,2017-02-02 06:21:26,http://connellydouglas.info/crystel.sporer,,243.173.212.133,"{""location"": ""GT"", ""is_mobile"": false}" 16161,6,365,2017-01-16 21:18:35,http://douglas.io/nella_jerde,,21.98.137.176,"{""location"": ""CG"", ""is_mobile"": true}" 16162,6,365,2017-01-25 08:07:53,http://boyer.info/orrin_moen,,178.43.254.232,"{""location"": ""RU"", ""is_mobile"": true}" 16163,6,365,2017-02-28 22:36:36,http://gislason.org/nicola.farrell,,218.125.53.176,"{""location"": ""NC"", ""is_mobile"": false}" 16164,6,365,2017-01-28 22:44:13,http://rohanwisozk.org/claudine,,247.161.125.237,"{""location"": ""MK"", ""is_mobile"": false}" 16165,6,365,2017-01-02 08:44:09,http://kuhlman.biz/porter,,253.154.172.233,"{""location"": ""VE"", ""is_mobile"": false}" 16166,6,365,2017-05-08 10:46:43,http://king.com/alden,,66.29.29.220,"{""location"": ""VC"", ""is_mobile"": true}" 16167,6,365,2017-04-24 23:20:34,http://harvey.info/julien,,82.40.131.219,"{""location"": ""LY"", ""is_mobile"": true}" 16168,6,365,2017-01-18 11:13:31,http://murphy.biz/reyna_waters,,18.164.162.8,"{""location"": ""IO"", ""is_mobile"": true}" 16169,6,365,2017-04-17 19:38:49,http://dickens.com/jaquelin_kautzer,,143.148.3.64,"{""location"": ""DZ"", ""is_mobile"": true}" 16170,6,365,2017-03-14 07:42:03,http://bradtke.name/amelie,,209.5.78.115,"{""location"": ""AD"", ""is_mobile"": true}" 16171,6,365,2017-03-25 19:28:58,http://leffler.name/meaghan,,210.41.142.214,"{""location"": ""JO"", ""is_mobile"": false}" 16172,6,365,2017-04-11 21:15:45,http://gottlieb.io/thea,,97.210.210.77,"{""location"": ""TT"", ""is_mobile"": false}" 16173,6,365,2017-05-18 16:30:12,http://haag.info/eldred.stokes,,55.3.80.194,"{""location"": ""KH"", ""is_mobile"": false}" 16174,6,365,2016-12-24 21:21:07,http://stiedemann.name/keshawn,,83.86.222.238,"{""location"": ""TF"", ""is_mobile"": true}" 16175,6,365,2017-05-13 02:47:33,http://bernhardlarson.info/valentin_roob,,106.129.240.139,"{""location"": ""TT"", ""is_mobile"": false}" 16176,6,365,2017-02-23 22:57:26,http://rowe.co/dorris.emard,,111.126.74.76,"{""location"": ""LI"", ""is_mobile"": true}" 16177,6,365,2017-05-29 11:42:04,http://mclaughlinschmeler.org/jaycee,,57.112.53.86,"{""location"": ""KN"", ""is_mobile"": false}" 16178,6,365,2016-12-21 05:55:04,http://langworth.net/micah,,37.254.51.13,"{""location"": ""ST"", ""is_mobile"": true}" 16179,6,365,2016-12-26 03:23:16,http://millchiller.name/gielle,,142.148.163.137,"{""location"": ""SY"", ""is_mobile"": true}" 16180,6,365,2017-02-06 06:02:40,http://fahey.com/emely_fay,,6.111.238.117,"{""location"": ""SC"", ""is_mobile"": true}" 16181,6,365,2017-02-03 10:26:51,http://kiehnhoppe.com/billie,,179.46.188.252,"{""location"": ""IQ"", ""is_mobile"": true}" 16182,6,365,2017-01-23 12:25:55,http://quitzon.net/stanford.feeney,,43.177.22.84,"{""location"": ""ZW"", ""is_mobile"": false}" 16183,6,365,2017-04-07 23:16:26,http://abernathyhuel.name/letha,,173.239.52.89,"{""location"": ""RE"", ""is_mobile"": true}" 16184,6,365,2016-12-16 22:35:09,http://halvorson.name/evalyn_feeney,,122.56.179.221,"{""location"": ""VN"", ""is_mobile"": false}" 16185,6,365,2017-04-25 00:55:53,http://volkman.co/jaleel.beer,,83.74.102.236,"{""location"": ""HK"", ""is_mobile"": false}" 16186,6,366,2017-04-20 05:46:19,http://mannleffler.co/obie.mckenzie,,146.22.20.245,"{""location"": ""BN"", ""is_mobile"": false}" 16187,6,366,2017-05-15 08:58:03,http://torphy.name/claudie,,48.185.210.204,"{""location"": ""IE"", ""is_mobile"": true}" 16188,6,366,2017-02-27 04:38:21,http://greenholt.com/clinton,,221.101.133.192,"{""location"": ""NP"", ""is_mobile"": false}" 16189,6,366,2017-02-03 16:14:54,http://barrows.io/tierra,,75.73.154.217,"{""location"": ""SE"", ""is_mobile"": true}" 16190,6,366,2017-02-21 15:55:54,http://collier.net/tyra.hickle,,248.55.146.26,"{""location"": ""DZ"", ""is_mobile"": true}" 16191,6,366,2017-06-08 02:58:10,http://haag.name/mozell.senger,,121.116.105.103,"{""location"": ""SN"", ""is_mobile"": false}" 3318,2,73,2017-01-24 15:35:49,http://oberbrunnerkuvalis.info/rosie.breitenberg,0.7276529732,83.146.165.57,"{""location"": ""VN"", ""is_mobile"": false}" 3319,2,73,2017-01-29 13:30:44,http://windlerrath.com/yvette_boyer,0.0833660566,140.137.206.238,"{""location"": ""LT"", ""is_mobile"": false}" 3320,2,73,2017-01-08 08:31:52,http://cruickshankhegmann.biz/alexis_sauer,0.4260702140,143.111.162.235,"{""location"": ""SL"", ""is_mobile"": true}" 3321,2,73,2017-01-03 10:18:36,http://huelsbartell.name/carolyne,0.4110839752,15.54.66.167,"{""location"": ""TK"", ""is_mobile"": false}" 3322,2,73,2017-02-22 05:33:03,http://bechtelar.name/kristofer_hane,0.5930202235,220.181.36.176,"{""location"": ""AQ"", ""is_mobile"": false}" 3323,2,73,2017-02-01 22:35:10,http://torphy.name/sigmund,0.6040979851,204.22.203.208,"{""location"": ""VC"", ""is_mobile"": false}" 3324,2,73,2017-04-04 07:48:51,http://feest.net/abel.west,0.1718269634,46.31.39.55,"{""location"": ""SD"", ""is_mobile"": false}" 3325,2,73,2017-06-01 03:18:36,http://nolan.com/jovan_ohara,0.0427052714,104.43.217.89,"{""location"": ""JM"", ""is_mobile"": true}" 3326,2,73,2017-04-10 17:54:45,http://okeefe.com/omer,0.5052237576,13.141.56.52,"{""location"": ""MM"", ""is_mobile"": false}" 3327,2,73,2017-02-07 19:58:41,http://cronaklocko.biz/angeline,0.6725601288,56.49.220.151,"{""location"": ""IO"", ""is_mobile"": true}" 3328,2,73,2016-12-16 10:12:47,http://ankunding.info/vilma.spencer,0.3950093244,29.123.226.64,"{""location"": ""PK"", ""is_mobile"": false}" 3329,2,73,2017-02-17 09:07:07,http://cummeratagutmann.info/wyatt,0.2592343962,186.190.179.110,"{""location"": ""CV"", ""is_mobile"": true}" 3330,2,73,2017-02-07 23:25:52,http://abbottrobel.co/kristina,0.8371929420,201.148.200.63,"{""location"": ""SY"", ""is_mobile"": false}" 3331,2,73,2017-06-11 18:38:36,http://leannon.info/enrico,0.3770720906,15.112.196.68,"{""location"": ""KE"", ""is_mobile"": true}" 3332,2,73,2017-01-01 05:37:02,http://flatley.biz/keith,0.9220624744,177.245.101.110,"{""location"": ""HN"", ""is_mobile"": true}" 3333,2,73,2017-01-25 18:39:06,http://hauck.org/jocelyn.schneider,0.2473830123,105.203.17.116,"{""location"": ""SK"", ""is_mobile"": false}" 3334,2,74,2016-12-24 15:07:07,http://fadel.info/juanita,0.6310982422,65.29.76.164,"{""location"": ""MA"", ""is_mobile"": true}" 3335,2,74,2017-02-27 23:39:00,http://purdy.net/pearl.von,0.0911642441,88.151.67.5,"{""location"": ""PH"", ""is_mobile"": false}" 3336,2,74,2017-05-21 06:52:33,http://sanford.com/dayne.dooley,0.0752830254,34.67.228.116,"{""location"": ""PT"", ""is_mobile"": true}" 3337,2,74,2017-05-12 21:54:56,http://hackett.info/demarco_labadie,0.7975416159,119.247.161.123,"{""location"": ""GE"", ""is_mobile"": true}" 3338,2,74,2017-06-07 19:29:31,http://ebert.info/jedediah,0.4155092593,108.107.121.161,"{""location"": ""LI"", ""is_mobile"": false}" 3339,2,74,2017-03-23 14:36:25,http://balistreri.name/winfield,0.5599428936,95.214.40.90,"{""location"": ""SA"", ""is_mobile"": true}" 3340,2,74,2017-03-18 20:13:44,http://mclaughlin.info/francisco_beahan,0.3199011332,109.253.252.77,"{""location"": ""ET"", ""is_mobile"": false}" 3341,2,74,2017-04-07 17:20:41,http://gorczany.biz/milford_schneider,0.0468558707,191.248.198.234,"{""location"": ""SG"", ""is_mobile"": false}" 3342,2,74,2017-04-18 22:17:43,http://corwin.net/darrin,0.2577090757,3.150.124.36,"{""location"": ""FJ"", ""is_mobile"": true}" 3343,2,74,2017-02-19 08:49:02,http://beckerheaney.net/henri,0.2544168691,146.97.55.119,"{""location"": ""BM"", ""is_mobile"": false}" 3344,2,74,2017-01-01 23:52:15,http://haag.co/cicero,0.2682738094,193.207.10.220,"{""location"": ""WS"", ""is_mobile"": true}" 3345,2,74,2017-03-09 14:25:15,http://keebler.name/patrick_kub,0.6952813459,154.87.3.92,"{""location"": ""RS"", ""is_mobile"": true}" 3346,2,74,2016-12-28 16:53:20,http://bauchrodriguez.info/alfreda,0.8467694945,122.79.181.229,"{""location"": ""PM"", ""is_mobile"": true}" 3347,2,74,2017-05-13 17:24:34,http://nolanhilll.com/jennie.haley,0.6307927771,21.69.149.160,"{""location"": ""MK"", ""is_mobile"": true}" 3348,2,74,2017-06-06 06:36:24,http://schulistpadberg.co/ladarius,0.9144426528,193.17.143.99,"{""location"": ""NL"", ""is_mobile"": false}" 3349,2,74,2017-02-12 08:35:08,http://jaskolski.com/chanel.goodwin,0.1054820153,124.62.249.97,"{""location"": ""YT"", ""is_mobile"": false}" 3350,2,74,2017-03-15 01:57:43,http://senger.io/ashtyn,0.2515146309,174.101.246.178,"{""location"": ""AO"", ""is_mobile"": false}" 3351,2,74,2017-02-18 17:55:34,http://abernathybeier.org/vivienne.kub,0.2434417855,177.61.32.107,"{""location"": ""AQ"", ""is_mobile"": false}" 3352,2,74,2017-05-13 19:21:06,http://mckenzie.info/haven_kuhlman,0.6673771423,73.226.38.69,"{""location"": ""NA"", ""is_mobile"": true}" 3353,2,74,2017-02-23 10:52:29,http://boyerkonopelski.name/dexter,0.2088725613,77.23.61.206,"{""location"": ""AI"", ""is_mobile"": false}" 3354,2,74,2017-04-23 18:47:50,http://volkman.biz/lexie,0.2429459894,221.173.46.105,"{""location"": ""SG"", ""is_mobile"": true}" 3355,2,74,2017-04-10 05:49:43,http://larson.info/abbie,0.3743725848,61.26.6.126,"{""location"": ""KZ"", ""is_mobile"": true}" 3356,2,74,2017-03-14 10:22:54,http://kilbackdavis.info/kendall.walter,0.4478114516,163.88.116.128,"{""location"": ""IM"", ""is_mobile"": true}" 3357,2,74,2017-03-14 06:27:46,http://koepp.io/florine,0.3409988534,72.238.66.31,"{""location"": ""NZ"", ""is_mobile"": true}" 3358,2,74,2017-04-16 20:07:08,http://rempelerdman.co/dan,0.9724749047,159.198.183.13,"{""location"": ""GT"", ""is_mobile"": true}" 3359,2,74,2017-03-05 14:41:33,http://hagenesrohan.io/salvador,0.0299427016,112.58.59.207,"{""location"": ""MZ"", ""is_mobile"": true}" 3360,2,74,2017-05-30 02:53:54,http://torp.info/hilario_sawayn,0.3424102316,32.219.151.162,"{""location"": ""HK"", ""is_mobile"": false}" 3361,2,75,2017-02-18 13:05:24,http://raynor.name/brooklyn,0.8552383741,153.67.229.59,"{""location"": ""AZ"", ""is_mobile"": false}" 3362,2,75,2017-01-06 03:13:55,http://hirthe.io/paolo_jerde,0.9585967615,74.114.154.97,"{""location"": ""ZW"", ""is_mobile"": true}" 3363,2,75,2017-03-10 02:46:33,http://macgyverbeahan.name/lucius,0.3031828772,110.134.245.74,"{""location"": ""NA"", ""is_mobile"": false}" 3364,2,75,2017-04-11 18:54:01,http://glover.biz/bridgette.beier,0.4084725848,236.245.114.66,"{""location"": ""GL"", ""is_mobile"": false}" 3365,2,75,2017-01-06 13:16:06,http://heidenreichboyer.co/mikayla.hane,0.3301250576,186.7.95.226,"{""location"": ""ZM"", ""is_mobile"": false}" 3366,2,75,2017-05-31 19:53:33,http://rogahn.com/eliza,0.0284332440,148.140.154.179,"{""location"": ""PM"", ""is_mobile"": false}" 3367,2,75,2017-03-19 07:06:11,http://walker.name/vicenta,0.9498869898,128.130.182.6,"{""location"": ""ME"", ""is_mobile"": true}" 3368,2,75,2017-04-03 05:54:16,http://cain.info/orlando,0.9675430658,158.134.185.169,"{""location"": ""AU"", ""is_mobile"": false}" 3369,2,75,2017-01-22 15:36:48,http://andersonconsidine.co/rosella,0.7374516434,2.66.153.14,"{""location"": ""KH"", ""is_mobile"": true}" 10250,4,231,2017-05-17 08:15:22,http://cainbradtke.com/chyna,0.2596368566,249.165.132.20,"{""location"": ""MW"", ""is_mobile"": false}" 10251,4,231,2017-04-01 04:03:40,http://farrellwyman.org/marlin,0.5402574527,63.245.221.231,"{""location"": ""DO"", ""is_mobile"": false}" 10252,4,231,2017-03-02 03:27:54,http://reynolds.org/waylon_wuckert,0.3275844421,166.96.93.244,"{""location"": ""TW"", ""is_mobile"": false}" 10253,4,231,2017-03-18 22:11:01,http://gusikowski.com/kadin.streich,0.4105330944,36.81.11.69,"{""location"": ""RU"", ""is_mobile"": false}" 10254,4,231,2016-12-24 06:03:38,http://huel.biz/consuelo.lemke,0.3097520921,52.213.69.7,"{""location"": ""BI"", ""is_mobile"": true}" 10255,4,231,2017-02-23 21:59:36,http://mills.io/lilian,0.7972534040,215.105.17.233,"{""location"": ""HM"", ""is_mobile"": false}" 10256,4,231,2017-02-18 19:21:13,http://cummerata.co/alvis,0.6382327641,30.100.83.152,"{""location"": ""VE"", ""is_mobile"": false}" 10257,4,231,2017-05-10 16:27:56,http://rosenbaum.name/dayna.abernathy,0.1550157712,84.73.126.185,"{""location"": ""NI"", ""is_mobile"": true}" 10258,4,231,2017-04-14 05:12:57,http://legrosryan.info/felicia.oberbrunner,0.3295082861,158.238.104.109,"{""location"": ""UY"", ""is_mobile"": false}" 10259,4,231,2017-01-17 03:45:50,http://dooleymertz.net/damion,0.0535820789,137.48.219.212,"{""location"": ""GP"", ""is_mobile"": false}" 10260,4,231,2017-01-24 11:30:40,http://kutch.info/jaida_kulas,0.1296176018,139.198.157.74,"{""location"": ""GW"", ""is_mobile"": false}" 10261,4,231,2017-02-17 01:44:10,http://zulauf.com/hulda,0.3461819944,41.51.150.94,"{""location"": ""VG"", ""is_mobile"": false}" 10262,4,231,2017-01-17 10:58:34,http://stanton.org/maximillia.barton,0.9334106981,126.91.187.68,"{""location"": ""UA"", ""is_mobile"": true}" 10263,4,231,2017-02-28 03:28:25,http://gislason.name/virgie,0.4259577908,206.148.170.72,"{""location"": ""TK"", ""is_mobile"": false}" 10264,4,231,2017-05-28 21:55:10,http://farrell.biz/cleora.vonrueden,0.6022779389,56.46.123.86,"{""location"": ""JE"", ""is_mobile"": false}" 10265,4,231,2017-06-07 11:41:12,http://raynor.io/trey_langworth,0.4203812266,119.84.235.163,"{""location"": ""GE"", ""is_mobile"": true}" 10266,4,231,2017-02-25 20:48:16,http://pouros.io/kory,0.5231721367,73.90.31.223,"{""location"": ""WS"", ""is_mobile"": true}" 10267,4,231,2017-05-09 13:21:44,http://hilpertkeebler.io/elva,0.7347521751,247.227.178.171,"{""location"": ""TR"", ""is_mobile"": true}" 10268,4,231,2017-01-14 14:01:48,http://robertswatsica.io/elian.tillman,0.0691424662,35.150.108.209,"{""location"": ""PS"", ""is_mobile"": false}" 10269,4,231,2017-01-30 02:34:22,http://oharavon.org/leta.nikolaus,0.2877276154,180.242.75.171,"{""location"": ""RE"", ""is_mobile"": true}" 10270,4,231,2017-05-11 20:52:22,http://rolfsonorn.name/jeica.zulauf,0.9192271474,141.43.140.136,"{""location"": ""BL"", ""is_mobile"": true}" 10271,4,231,2017-01-13 21:31:17,http://kovaceksmitham.net/rylan,0.6832663659,140.130.154.240,"{""location"": ""UA"", ""is_mobile"": false}" 10272,4,231,2017-01-29 08:34:01,http://west.biz/wallace_bechtelar,0.1841709577,211.104.97.117,"{""location"": ""SR"", ""is_mobile"": true}" 10273,4,231,2017-04-11 10:58:50,http://connelly.org/krista,0.8574019595,47.105.102.19,"{""location"": ""BD"", ""is_mobile"": false}" 10274,4,231,2017-03-20 16:21:44,http://murray.io/cecilia_anderson,0.9396774481,212.108.124.190,"{""location"": ""ZA"", ""is_mobile"": true}" 10275,4,231,2016-12-29 06:49:26,http://reichert.io/luciano,0.3817683029,187.113.132.77,"{""location"": ""GU"", ""is_mobile"": false}" 10276,4,231,2016-12-18 04:23:48,http://robel.net/zella,0.3312385128,213.164.144.32,"{""location"": ""CW"", ""is_mobile"": true}" 10277,4,231,2017-04-20 01:23:02,http://schamberger.co/rubie.ortiz,0.8441995049,234.115.69.241,"{""location"": ""BJ"", ""is_mobile"": true}" 10278,4,231,2017-04-17 14:38:35,http://olson.biz/josh_murphy,0.0807495815,90.190.55.195,"{""location"": ""AW"", ""is_mobile"": false}" 10279,4,231,2017-04-22 15:41:48,http://schinner.io/lauretta,0.0201766096,112.127.81.170,"{""location"": ""LB"", ""is_mobile"": true}" 10280,4,231,2017-02-07 23:18:48,http://robelheidenreich.name/emmanuelle,0.1476162038,52.182.72.78,"{""location"": ""UG"", ""is_mobile"": false}" 10281,4,231,2017-01-15 07:31:02,http://schamberger.info/shaina,0.9339568098,21.101.57.24,"{""location"": ""SB"", ""is_mobile"": true}" 10282,4,231,2017-02-16 01:17:18,http://morar.io/abbigail.schulist,0.2544911302,251.123.76.65,"{""location"": ""DM"", ""is_mobile"": true}" 10283,4,231,2016-12-22 23:04:49,http://ruecker.com/sydnee_towne,0.8075260468,232.169.246.165,"{""location"": ""DM"", ""is_mobile"": true}" 10284,4,231,2017-04-21 19:52:09,http://haag.info/garry.ullrich,0.8770736781,133.240.80.145,"{""location"": ""TK"", ""is_mobile"": false}" 10285,4,231,2017-05-15 11:16:44,http://rippin.name/gabriel.von,0.9849806523,179.48.56.20,"{""location"": ""PM"", ""is_mobile"": false}" 10286,4,231,2017-05-02 18:50:49,http://herzogmonahan.net/queen,0.3307911247,41.112.183.27,"{""location"": ""MQ"", ""is_mobile"": false}" 10287,4,231,2017-04-06 16:17:36,http://sauer.co/andrew.ritchie,0.4558812383,236.251.20.72,"{""location"": ""VI"", ""is_mobile"": true}" 10288,4,231,2017-03-02 02:45:08,http://yost.io/toby.rolfson,0.8796943905,234.76.106.91,"{""location"": ""GU"", ""is_mobile"": false}" 10289,4,231,2017-06-05 12:23:18,http://mosciski.co/celine_hettinger,0.8884712918,144.43.100.129,"{""location"": ""SJ"", ""is_mobile"": false}" 10290,4,231,2016-12-17 13:50:34,http://windler.biz/dorthy_yost,0.4624625495,96.236.242.136,"{""location"": ""CA"", ""is_mobile"": true}" 10291,4,231,2017-05-26 02:20:49,http://shanahan.net/judge.harris,0.2885144078,30.71.153.132,"{""location"": ""FK"", ""is_mobile"": true}" 10292,4,232,2017-05-14 07:07:45,http://johnston.org/shana,0.3728541019,159.193.225.113,"{""location"": ""BD"", ""is_mobile"": true}" 10293,4,232,2017-03-16 16:59:56,http://hermistonpfannerstill.net/bryon,0.3670304748,190.109.227.162,"{""location"": ""TR"", ""is_mobile"": true}" 10294,4,232,2017-06-10 08:36:39,http://cruickshank.io/vivienne,0.3600121407,120.209.17.102,"{""location"": ""PR"", ""is_mobile"": false}" 10295,4,232,2016-12-31 06:32:25,http://erdman.name/lacey_brown,0.9979101792,14.68.124.42,"{""location"": ""OM"", ""is_mobile"": false}" 10296,4,232,2017-04-01 00:34:05,http://hudsonkoch.net/hayley,0.5307143342,230.237.191.118,"{""location"": ""BH"", ""is_mobile"": true}" 10297,4,232,2017-04-12 12:06:43,http://kovacek.name/keagan_lowe,0.5402884838,85.20.34.254,"{""location"": ""IR"", ""is_mobile"": true}" 10298,4,232,2017-02-17 04:18:07,http://zulauf.io/mertie,0.0305559281,85.162.56.12,"{""location"": ""CZ"", ""is_mobile"": true}" 10299,4,232,2017-01-17 02:42:25,http://feeney.com/juston,0.2050076896,52.144.139.196,"{""location"": ""JM"", ""is_mobile"": true}" 10300,4,232,2017-04-17 12:58:15,http://kemmer.net/heidi,0.5435829148,210.34.74.19,"{""location"": ""NE"", ""is_mobile"": false}" 10301,4,232,2017-05-07 08:36:42,http://jakubowski.io/may,0.2368814972,224.216.127.176,"{""location"": ""GW"", ""is_mobile"": true}" 7263,3,161,2017-03-16 12:38:39,http://berge.biz/josie.bosco,0.0601504283,161.16.55.248,"{""location"": ""DZ"", ""is_mobile"": false}" 7264,3,161,2017-01-17 23:24:11,http://boscohamill.info/major,0.0853269010,98.54.54.180,"{""location"": ""UY"", ""is_mobile"": false}" 7265,3,161,2016-12-28 00:29:27,http://weinat.io/wilfred_beer,0.4554543793,126.187.134.229,"{""location"": ""GR"", ""is_mobile"": true}" 7266,3,161,2017-04-18 10:12:45,http://yundt.biz/luis,0.8856571261,97.150.130.96,"{""location"": ""LK"", ""is_mobile"": true}" 7267,3,161,2017-05-30 14:26:08,http://grant.co/brent_effertz,0.8008623843,50.227.82.148,"{""location"": ""FO"", ""is_mobile"": false}" 7268,3,161,2017-02-18 22:11:00,http://bechtelarkuvalis.info/martin,0.0131400124,4.196.63.137,"{""location"": ""BI"", ""is_mobile"": true}" 7269,3,161,2017-04-05 17:49:45,http://heel.net/daren,0.2872192993,3.205.225.243,"{""location"": ""GB"", ""is_mobile"": false}" 7270,3,161,2017-01-08 14:06:54,http://sporer.net/giovanni,0.8565914423,53.238.175.80,"{""location"": ""BN"", ""is_mobile"": false}" 7271,3,161,2017-01-14 19:44:07,http://johnston.co/elza.larson,0.2577679711,52.102.235.126,"{""location"": ""ML"", ""is_mobile"": false}" 7272,3,161,2017-01-22 12:17:36,http://toy.name/hal_beer,0.7722237089,227.232.29.60,"{""location"": ""GT"", ""is_mobile"": false}" 7273,3,161,2017-02-11 05:08:41,http://stroman.co/merritt,0.6641424277,172.86.169.97,"{""location"": ""CR"", ""is_mobile"": false}" 7274,3,161,2017-04-12 10:56:16,http://schaefergoyette.io/colten,0.1726948151,188.96.101.219,"{""location"": ""IN"", ""is_mobile"": true}" 7275,3,161,2017-02-25 13:20:42,http://gorczany.net/mallory.sipes,0.4555251542,45.65.120.164,"{""location"": ""SA"", ""is_mobile"": true}" 7276,3,161,2017-03-28 09:32:32,http://weber.name/macy,0.7134317647,80.242.104.20,"{""location"": ""GW"", ""is_mobile"": true}" 7277,3,161,2017-06-03 22:54:13,http://conn.co/roxane,0.3962254580,193.46.7.235,"{""location"": ""PF"", ""is_mobile"": false}" 7278,3,161,2017-02-27 02:14:34,http://littel.net/daphne,0.9186295396,219.214.24.110,"{""location"": ""AF"", ""is_mobile"": true}" 7279,3,161,2017-02-23 02:20:28,http://nader.net/viva.prosacco,0.9550941310,224.83.80.100,"{""location"": ""CO"", ""is_mobile"": true}" 7280,3,161,2017-02-24 18:06:37,http://klocko.co/corrine,0.6801428928,75.101.23.44,"{""location"": ""NU"", ""is_mobile"": true}" 7281,3,161,2017-06-06 11:04:36,http://millsfranecki.co/lavern.bogan,0.7730865065,75.225.65.168,"{""location"": ""ER"", ""is_mobile"": true}" 7282,3,161,2017-03-16 06:34:16,http://johnson.io/adriel,0.0351813452,30.166.154.224,"{""location"": ""RU"", ""is_mobile"": true}" 7283,3,161,2017-04-24 14:36:31,http://okuneva.name/alexis.langworth,0.5025736064,252.210.107.28,"{""location"": ""FO"", ""is_mobile"": true}" 7284,3,161,2017-06-13 10:12:40,http://hintzschroeder.name/marion,0.1926216576,110.31.8.88,"{""location"": ""NO"", ""is_mobile"": true}" 7285,3,161,2017-05-30 13:00:01,http://pollich.io/adrienne,0.8300253782,95.149.46.45,"{""location"": ""PY"", ""is_mobile"": false}" 7286,3,161,2016-12-29 00:24:24,http://herman.net/abraham,0.6749699775,27.28.188.110,"{""location"": ""FI"", ""is_mobile"": true}" 7287,3,162,2017-02-20 08:43:41,http://kreiger.net/raymond,,119.101.41.157,"{""location"": ""PA"", ""is_mobile"": false}" 7288,3,162,2017-04-25 07:10:15,http://doyleharvey.net/derrick,,164.189.56.41,"{""location"": ""IM"", ""is_mobile"": true}" 7289,3,162,2017-03-04 13:34:38,http://hahnhermann.io/shanna,,3.231.229.232,"{""location"": ""AE"", ""is_mobile"": false}" 7290,3,162,2017-06-07 20:15:36,http://moriettehuel.net/earnestine,,147.221.226.58,"{""location"": ""AI"", ""is_mobile"": false}" 7291,3,162,2017-02-15 02:02:04,http://klein.info/kelli.fisher,,252.114.159.180,"{""location"": ""BO"", ""is_mobile"": false}" 7292,3,162,2017-02-07 09:13:45,http://bernhard.name/dillon.jerde,,205.203.70.213,"{""location"": ""MS"", ""is_mobile"": true}" 7293,3,162,2017-01-06 22:38:25,http://mraz.biz/mckenna,,8.179.17.103,"{""location"": ""MT"", ""is_mobile"": false}" 7294,3,162,2017-03-28 05:38:55,http://leuschke.org/consuelo.muller,,119.205.39.42,"{""location"": ""AQ"", ""is_mobile"": false}" 7295,3,162,2017-02-08 01:04:03,http://wunschbergstrom.io/grant,,190.47.76.88,"{""location"": ""AW"", ""is_mobile"": false}" 7296,3,162,2017-01-28 14:41:23,http://morar.org/esta,,87.218.37.175,"{""location"": ""EC"", ""is_mobile"": true}" 7297,3,162,2017-05-23 07:24:12,http://rogahnkilback.info/simeon,,188.215.68.85,"{""location"": ""SY"", ""is_mobile"": true}" 7298,3,162,2017-03-25 18:56:36,http://gottliebbergstrom.com/adan_durgan,,145.227.242.116,"{""location"": ""EE"", ""is_mobile"": true}" 7299,3,162,2017-05-11 19:08:49,http://bogisichratke.com/karlie.bernier,,167.75.92.132,"{""location"": ""CR"", ""is_mobile"": true}" 7300,3,162,2017-03-20 01:38:01,http://moore.co/carmella_swaniawski,,34.153.222.176,"{""location"": ""HR"", ""is_mobile"": true}" 7301,3,162,2017-06-13 16:13:19,http://dibbert.co/tina,,131.16.163.55,"{""location"": ""MS"", ""is_mobile"": true}" 7302,3,162,2017-02-23 18:34:49,http://carter.co/benjamin_white,,168.67.199.230,"{""location"": ""GG"", ""is_mobile"": false}" 7303,3,162,2017-05-08 05:19:36,http://schroeder.biz/zion_smitham,,130.70.160.100,"{""location"": ""SH"", ""is_mobile"": true}" 7304,3,162,2017-05-11 05:37:28,http://gleason.io/austyn_mosciski,,184.53.196.208,"{""location"": ""MS"", ""is_mobile"": true}" 7305,3,162,2017-03-25 14:52:21,http://breitenbergabbott.info/ansley_toy,,110.220.30.231,"{""location"": ""GQ"", ""is_mobile"": false}" 7306,3,162,2017-02-19 04:27:22,http://satterfieldklein.biz/izaiah,,202.34.201.253,"{""location"": ""HN"", ""is_mobile"": false}" 7307,3,162,2017-01-25 22:54:41,http://beier.info/hayley,,77.209.46.23,"{""location"": ""NR"", ""is_mobile"": true}" 7308,3,162,2017-04-30 19:49:29,http://krisgrady.name/adelle.schaefer,,214.166.228.197,"{""location"": ""QA"", ""is_mobile"": true}" 7309,3,162,2017-03-13 18:21:29,http://zemlak.name/loyal,,52.78.238.173,"{""location"": ""SI"", ""is_mobile"": false}" 7310,3,162,2017-05-28 14:54:26,http://ritchie.biz/helga,,233.64.115.87,"{""location"": ""MT"", ""is_mobile"": true}" 7311,3,162,2017-01-26 22:05:05,http://borer.co/kasandra,,91.234.39.236,"{""location"": ""CF"", ""is_mobile"": false}" 7312,3,162,2017-04-14 15:38:59,http://yundtbecker.biz/annabelle.hayes,,95.116.218.115,"{""location"": ""AD"", ""is_mobile"": true}" 7313,3,162,2017-05-02 06:17:52,http://fahey.net/colton_wolff,,91.70.192.95,"{""location"": ""NZ"", ""is_mobile"": false}" 7314,3,162,2016-12-28 09:31:05,http://johnsjerde.info/meggie_hoeger,,92.208.165.37,"{""location"": ""NZ"", ""is_mobile"": false}" 7315,3,162,2017-06-14 04:47:18,http://grant.name/amber,,184.250.46.226,"{""location"": ""GS"", ""is_mobile"": false}" 7316,3,162,2017-05-19 07:30:35,http://grady.io/porter_cummerata,,149.52.7.237,"{""location"": ""YT"", ""is_mobile"": false}" 7317,3,163,2017-05-12 06:02:06,http://walkerkiehn.co/jacques.rogahn,,208.216.224.12,"{""location"": ""CL"", ""is_mobile"": true}" 13173,5,296,2017-05-15 09:27:04,http://kilback.biz/ima_thiel,,226.32.56.149,"{""location"": ""CH"", ""is_mobile"": true}" 13174,5,296,2017-05-11 21:04:52,http://schultzjacobs.net/eulah,,170.222.233.158,"{""location"": ""SO"", ""is_mobile"": false}" 13175,5,296,2017-05-16 16:20:40,http://buckridge.biz/soledad,,146.15.131.187,"{""location"": ""IS"", ""is_mobile"": true}" 13176,5,296,2017-04-19 03:24:39,http://toy.biz/duane_renner,,105.190.248.175,"{""location"": ""IT"", ""is_mobile"": true}" 13177,5,296,2017-05-28 10:29:44,http://jenkins.net/electa,,28.50.195.158,"{""location"": ""KG"", ""is_mobile"": true}" 13178,5,296,2017-01-07 13:56:02,http://farrell.net/terence,,6.216.45.212,"{""location"": ""TO"", ""is_mobile"": true}" 13179,5,296,2017-05-19 17:10:48,http://zieme.name/antone,,14.27.148.250,"{""location"": ""GF"", ""is_mobile"": false}" 13180,5,296,2017-03-10 09:35:06,http://ullrich.biz/vergie_pagac,,82.20.81.238,"{""location"": ""ID"", ""is_mobile"": true}" 13181,5,296,2017-05-25 03:56:26,http://okon.org/alexane,,168.244.136.27,"{""location"": ""GM"", ""is_mobile"": true}" 13182,5,296,2017-06-03 16:35:49,http://crona.com/courtney_conroy,,106.53.18.68,"{""location"": ""GG"", ""is_mobile"": false}" 13183,5,296,2017-01-18 02:31:27,http://legros.name/amy_douglas,,232.250.41.85,"{""location"": ""GR"", ""is_mobile"": false}" 13184,5,296,2017-01-20 08:31:47,http://mertz.org/favian.oconner,,235.185.233.26,"{""location"": ""HT"", ""is_mobile"": true}" 13185,5,296,2017-03-16 21:32:41,http://okeefewaelchi.com/brandt,,120.244.152.76,"{""location"": ""PK"", ""is_mobile"": false}" 13186,5,296,2017-03-07 04:29:19,http://howell.name/kim_lindgren,,21.142.120.228,"{""location"": ""NO"", ""is_mobile"": false}" 13187,5,296,2017-01-10 06:28:06,http://osinskizemlak.io/oral.sporer,,96.173.171.205,"{""location"": ""LK"", ""is_mobile"": false}" 13188,5,296,2017-01-14 19:24:27,http://ryan.biz/george_bashirian,,114.183.7.114,"{""location"": ""BO"", ""is_mobile"": false}" 13189,5,296,2017-01-06 20:03:42,http://leffler.info/daphnee_willms,,80.119.198.142,"{""location"": ""TV"", ""is_mobile"": true}" 13190,5,296,2017-03-13 02:06:31,http://considine.biz/halle.quitzon,,80.17.69.83,"{""location"": ""TV"", ""is_mobile"": false}" 13191,5,297,2017-03-24 13:05:41,http://zieme.info/estel,,64.231.24.208,"{""location"": ""PY"", ""is_mobile"": true}" 13192,5,297,2017-02-22 21:48:38,http://harvey.info/maria.adams,,252.198.83.7,"{""location"": ""TC"", ""is_mobile"": true}" 13193,5,297,2017-05-07 11:55:34,http://cronin.name/bell_baumbach,,101.91.191.33,"{""location"": ""CH"", ""is_mobile"": true}" 13194,5,297,2017-04-14 04:54:56,http://rath.net/osbaldo,,38.124.100.43,"{""location"": ""SD"", ""is_mobile"": true}" 13195,5,297,2017-03-20 02:52:26,http://markskulas.org/chandler.abbott,,117.47.209.90,"{""location"": ""DO"", ""is_mobile"": true}" 13196,5,297,2017-06-01 06:14:26,http://keebler.net/samanta.dicki,,7.13.247.206,"{""location"": ""KZ"", ""is_mobile"": true}" 13197,5,297,2017-03-11 11:27:34,http://pollich.com/alexis_morar,,130.127.28.156,"{""location"": ""HR"", ""is_mobile"": false}" 13198,5,297,2017-05-16 02:21:46,http://carroll.name/camron.oreilly,,62.132.245.171,"{""location"": ""HR"", ""is_mobile"": false}" 13199,5,297,2017-01-30 22:32:13,http://jenkins.io/ethel.jacobi,,90.13.56.198,"{""location"": ""KP"", ""is_mobile"": false}" 13200,5,297,2017-02-27 06:38:17,http://westmcclure.biz/lea,,9.95.125.19,"{""location"": ""GL"", ""is_mobile"": true}" 13201,5,297,2017-04-13 05:11:23,http://kris.org/dagmar,,169.67.61.23,"{""location"": ""BF"", ""is_mobile"": false}" 13202,5,297,2017-04-22 12:49:54,http://hettingermcglynn.info/jorge,,12.3.148.9,"{""location"": ""ES"", ""is_mobile"": false}" 13203,5,297,2017-05-28 22:38:38,http://zulaufledner.io/connie,,219.204.37.55,"{""location"": ""GR"", ""is_mobile"": true}" 13204,5,297,2017-02-04 21:54:25,http://ziemeboyer.io/emile_cormier,,242.136.58.116,"{""location"": ""AW"", ""is_mobile"": true}" 13205,5,297,2017-03-07 11:22:06,http://vandervort.info/patience,,196.217.253.83,"{""location"": ""TG"", ""is_mobile"": false}" 13206,5,297,2017-03-20 14:38:31,http://parisianwaters.com/kiana.greenfelder,,18.234.216.135,"{""location"": ""GF"", ""is_mobile"": false}" 13207,5,297,2017-03-15 05:44:26,http://block.name/euna.waters,,116.191.2.177,"{""location"": ""MX"", ""is_mobile"": true}" 13208,5,297,2017-01-17 20:20:38,http://goyettehuels.name/cale,,210.185.143.108,"{""location"": ""BB"", ""is_mobile"": false}" 13209,5,297,2017-03-14 19:32:39,http://kochtorp.name/al_bode,,7.253.93.145,"{""location"": ""LT"", ""is_mobile"": true}" 13210,5,297,2017-02-21 11:15:40,http://mclaughlin.biz/karlee,,47.150.217.203,"{""location"": ""PL"", ""is_mobile"": true}" 13211,5,298,2017-04-09 11:16:13,http://baumbachstehr.biz/ali_brakus,,134.96.33.185,"{""location"": ""IQ"", ""is_mobile"": false}" 13212,5,298,2017-02-19 23:08:23,http://johnsonwunsch.co/theo,,116.96.102.165,"{""location"": ""ES"", ""is_mobile"": false}" 13213,5,298,2017-02-19 22:49:33,http://cormier.name/jamey.howe,,55.129.197.224,"{""location"": ""SI"", ""is_mobile"": false}" 13214,5,298,2017-03-24 18:01:11,http://carroll.name/wiley,,12.211.168.56,"{""location"": ""LR"", ""is_mobile"": false}" 13215,5,298,2017-03-07 23:28:36,http://kulas.com/chasity_mitchell,,98.118.79.235,"{""location"": ""ST"", ""is_mobile"": false}" 13216,5,298,2017-01-12 20:38:07,http://howe.org/fatima.von,,80.173.2.25,"{""location"": ""BW"", ""is_mobile"": true}" 13217,5,298,2017-01-15 12:39:07,http://hackett.net/felipe_hilll,,200.116.16.32,"{""location"": ""TR"", ""is_mobile"": false}" 13218,5,298,2017-05-28 05:08:34,http://reichel.name/roxanne,,64.25.209.147,"{""location"": ""SE"", ""is_mobile"": false}" 13219,5,298,2017-03-23 13:59:52,http://cormier.io/delpha_medhurst,,69.223.254.57,"{""location"": ""PR"", ""is_mobile"": true}" 13220,5,298,2017-05-15 02:15:34,http://bauch.biz/frederic.cartwright,,129.47.209.228,"{""location"": ""LK"", ""is_mobile"": true}" 13221,5,298,2017-05-10 19:00:44,http://jerdeschuppe.biz/esther_boyle,,20.205.43.63,"{""location"": ""CC"", ""is_mobile"": true}" 13222,5,298,2017-06-02 09:45:34,http://davislarson.biz/bartholome,,175.58.13.106,"{""location"": ""BN"", ""is_mobile"": false}" 13223,5,298,2017-04-05 09:07:52,http://halvorson.name/ahmad.stehr,,215.196.23.206,"{""location"": ""LB"", ""is_mobile"": false}" 13224,5,298,2016-12-25 09:32:27,http://doylebrown.com/leopold,,21.168.108.87,"{""location"": ""MZ"", ""is_mobile"": false}" 13225,5,298,2017-02-02 16:07:25,http://collins.info/abel.berge,,26.250.121.45,"{""location"": ""MN"", ""is_mobile"": false}" 13226,5,298,2017-01-05 06:52:08,http://champlinskiles.io/mark_predovic,,102.37.158.158,"{""location"": ""SN"", ""is_mobile"": true}" 13227,5,298,2017-05-08 22:37:08,http://moriettenader.info/amely,,8.45.102.28,"{""location"": ""NA"", ""is_mobile"": false}" 16192,6,366,2017-04-12 05:14:31,http://schulist.org/geovany_durgan,,2.60.154.3,"{""location"": ""MW"", ""is_mobile"": false}" 16193,6,366,2017-02-08 01:50:11,http://ryancrooks.co/nathanial_prohaska,,38.135.84.133,"{""location"": ""ZA"", ""is_mobile"": true}" 16194,6,366,2017-02-12 17:37:11,http://ortiz.biz/dorris.blanda,,31.38.39.35,"{""location"": ""TV"", ""is_mobile"": false}" 16195,6,366,2017-05-22 01:01:54,http://boehmlindgren.net/ashton_hettinger,,195.48.66.144,"{""location"": ""MM"", ""is_mobile"": true}" 16196,6,366,2017-05-21 03:17:10,http://brown.org/jordan_weimann,,160.128.119.243,"{""location"": ""NZ"", ""is_mobile"": true}" 16197,6,366,2017-02-10 03:48:01,http://considinekunde.biz/amir,,240.40.135.193,"{""location"": ""TH"", ""is_mobile"": false}" 16198,6,366,2016-12-29 05:02:30,http://macgyvergoodwin.io/aiyana,,165.68.96.20,"{""location"": ""MK"", ""is_mobile"": true}" 16199,6,366,2017-04-25 15:34:03,http://harvey.io/kiley_lang,,106.202.41.84,"{""location"": ""AQ"", ""is_mobile"": true}" 16200,6,366,2017-05-07 13:49:52,http://will.name/timothy_kihn,,148.43.87.34,"{""location"": ""SN"", ""is_mobile"": true}" 16201,6,366,2017-05-15 13:43:48,http://daniel.co/katherine,,47.209.161.88,"{""location"": ""BF"", ""is_mobile"": true}" 16202,6,366,2017-01-01 10:43:50,http://williamson.org/izabella_kemmer,,161.63.170.43,"{""location"": ""MQ"", ""is_mobile"": true}" 16203,6,366,2017-05-24 17:03:58,http://pricetillman.net/cheyenne,,188.165.160.16,"{""location"": ""VG"", ""is_mobile"": true}" 16204,6,366,2017-05-30 17:00:06,http://mayerthodkiewicz.name/mina,,32.140.40.4,"{""location"": ""NG"", ""is_mobile"": true}" 16205,6,366,2017-02-16 14:00:42,http://cummings.info/alice_treutel,,41.253.244.176,"{""location"": ""CF"", ""is_mobile"": false}" 16206,6,366,2017-04-10 16:38:35,http://legros.net/camilla,,107.215.232.15,"{""location"": ""SR"", ""is_mobile"": true}" 16207,6,366,2017-02-13 10:24:34,http://kleinbradtke.biz/sophia,,240.64.165.79,"{""location"": ""WF"", ""is_mobile"": true}" 16208,6,366,2017-06-03 21:32:42,http://cremin.org/abel_hermann,,63.32.28.218,"{""location"": ""CM"", ""is_mobile"": false}" 16209,6,366,2017-03-25 20:46:48,http://langgutmann.biz/carolina,,9.94.207.92,"{""location"": ""UY"", ""is_mobile"": false}" 16210,6,366,2017-04-27 02:41:52,http://aufderharhegmann.net/fatima.ruecker,,6.109.239.98,"{""location"": ""MQ"", ""is_mobile"": false}" 16211,6,366,2017-01-01 19:13:40,http://rodriguez.io/tiffany,,145.23.209.175,"{""location"": ""AQ"", ""is_mobile"": true}" 16212,6,366,2017-02-17 06:56:59,http://wilderman.info/gerson_schuppe,,73.22.57.77,"{""location"": ""VC"", ""is_mobile"": false}" 16213,6,367,2016-12-31 00:42:38,http://hartmann.net/ellie,0.2692487466,79.176.47.13,"{""location"": ""BL"", ""is_mobile"": false}" 16214,6,367,2017-01-11 14:31:10,http://kemmerauer.name/floie,0.5859551846,222.183.93.154,"{""location"": ""GA"", ""is_mobile"": false}" 16215,6,367,2017-01-23 14:31:18,http://schultz.info/hailee,0.1031484475,156.249.79.12,"{""location"": ""GY"", ""is_mobile"": true}" 16216,6,367,2017-05-23 02:53:32,http://hyattkuhn.io/kaela,0.6045939188,191.202.44.213,"{""location"": ""AZ"", ""is_mobile"": false}" 16217,6,367,2017-02-01 23:11:00,http://bradtkesteuber.biz/richard.medhurst,0.4428672649,172.83.17.37,"{""location"": ""PM"", ""is_mobile"": false}" 16218,6,367,2017-02-01 22:21:53,http://franecki.co/nathan,0.6766504339,117.52.152.186,"{""location"": ""AL"", ""is_mobile"": true}" 16219,6,367,2017-05-30 20:58:51,http://kubdonnelly.biz/maryse.keebler,0.8113703997,108.49.95.211,"{""location"": ""VI"", ""is_mobile"": true}" 16220,6,367,2017-05-31 07:29:06,http://halvorson.biz/isabel,0.0903848355,229.203.73.132,"{""location"": ""YE"", ""is_mobile"": true}" 16221,6,367,2017-04-16 14:05:31,http://emmerichweber.io/terrell,0.4802804429,53.96.23.40,"{""location"": ""CO"", ""is_mobile"": true}" 16222,6,367,2017-04-08 14:48:23,http://bruenschumm.org/rory,0.9250272977,29.158.244.71,"{""location"": ""TZ"", ""is_mobile"": true}" 16223,6,367,2016-12-30 04:09:19,http://hartmann.net/jeanette_kuhic,0.2203697234,220.168.125.206,"{""location"": ""DE"", ""is_mobile"": false}" 16224,6,367,2017-01-18 05:15:17,http://jenkins.com/dexter,0.1801383451,88.57.124.91,"{""location"": ""CR"", ""is_mobile"": true}" 16225,6,367,2017-03-19 16:56:16,http://miller.org/humberto_heathcote,0.6116325775,161.177.132.29,"{""location"": ""GB"", ""is_mobile"": true}" 16226,6,367,2017-06-12 07:07:19,http://welch.biz/coleman_denesik,0.0689314905,208.97.112.181,"{""location"": ""ZA"", ""is_mobile"": false}" 16227,6,367,2017-04-09 23:32:45,http://predovic.com/jameson_harris,0.8073426437,92.239.73.102,"{""location"": ""UZ"", ""is_mobile"": false}" 16228,6,367,2016-12-26 11:53:59,http://dickenserdman.net/nettie,0.9852326582,241.39.246.7,"{""location"": ""NL"", ""is_mobile"": false}" 16229,6,367,2017-04-06 11:50:22,http://friesenupton.biz/chyna,0.6861870883,204.17.17.27,"{""location"": ""AS"", ""is_mobile"": false}" 16230,6,367,2016-12-22 04:40:33,http://schulisthilpert.io/myrtle_reilly,0.3677931654,253.76.65.31,"{""location"": ""NL"", ""is_mobile"": false}" 16231,6,367,2017-03-30 16:42:19,http://langworth.biz/eve.koepp,0.2350714299,225.150.43.192,"{""location"": ""GT"", ""is_mobile"": true}" 16232,6,367,2017-02-11 14:27:29,http://bashirian.info/nathan,0.8061285148,83.141.117.26,"{""location"": ""CZ"", ""is_mobile"": false}" 16233,6,367,2016-12-26 12:58:29,http://jacobson.net/pascale,0.8695064620,2.216.162.28,"{""location"": ""GQ"", ""is_mobile"": true}" 16234,6,367,2017-02-07 12:16:27,http://ledner.com/hermann,0.6117577502,64.43.152.203,"{""location"": ""TR"", ""is_mobile"": true}" 16235,6,367,2017-03-19 13:23:43,http://danielweinat.com/norberto,0.3989781794,98.233.243.154,"{""location"": ""TW"", ""is_mobile"": true}" 16236,6,367,2017-05-14 08:47:22,http://pagac.name/darrion_leannon,0.1079601705,17.129.65.165,"{""location"": ""NZ"", ""is_mobile"": false}" 16237,6,367,2017-03-13 06:26:10,http://pagac.com/zachery,0.0097162892,43.233.157.245,"{""location"": ""GF"", ""is_mobile"": true}" 16238,6,367,2017-02-15 21:05:50,http://dooleydaugherty.biz/merlin,0.1535962850,33.62.74.156,"{""location"": ""SY"", ""is_mobile"": true}" 16239,6,367,2017-01-24 01:54:19,http://hane.com/lucy,0.4935618705,145.25.23.221,"{""location"": ""KP"", ""is_mobile"": false}" 16240,6,368,2017-01-19 13:18:24,http://stanton.io/myron,0.0623187218,131.154.239.251,"{""location"": ""HK"", ""is_mobile"": true}" 16241,6,368,2017-06-14 02:48:49,http://wisozk.name/selmer,0.9338420034,95.159.156.158,"{""location"": ""SR"", ""is_mobile"": true}" 16242,6,368,2017-05-28 17:26:21,http://kuhic.io/burley,0.5808360322,123.229.69.201,"{""location"": ""SK"", ""is_mobile"": true}" 16243,6,368,2017-05-31 06:01:46,http://kovacek.info/frida_ritchie,0.9116740531,143.181.3.95,"{""location"": ""AX"", ""is_mobile"": true}" 16244,6,368,2017-03-19 20:39:12,http://oberbrunner.io/shanie,0.6303565479,3.167.186.94,"{""location"": ""MK"", ""is_mobile"": true}" 3370,2,75,2017-04-17 13:08:10,http://deckowwehner.info/ebony,0.1469519343,8.45.253.98,"{""location"": ""AS"", ""is_mobile"": false}" 3371,2,75,2017-03-21 20:02:18,http://hyatt.org/bobby,0.4744092855,233.87.213.218,"{""location"": ""PM"", ""is_mobile"": false}" 3372,2,75,2017-04-19 17:03:47,http://schultz.co/mariah,0.9903107904,248.175.37.5,"{""location"": ""VE"", ""is_mobile"": false}" 3373,2,75,2016-12-14 21:25:11,http://lynch.org/angelita,0.5126503763,228.11.20.148,"{""location"": ""MA"", ""is_mobile"": false}" 3374,2,75,2017-01-02 19:16:26,http://hintzgrady.co/orion,0.0017227968,254.246.141.154,"{""location"": ""GL"", ""is_mobile"": true}" 3375,2,75,2017-05-26 10:28:40,http://kerluke.co/brice,0.4613338446,132.211.35.180,"{""location"": ""SJ"", ""is_mobile"": true}" 3376,2,75,2017-05-11 14:04:13,http://weimann.io/hailee.goldner,0.5096386726,15.190.62.57,"{""location"": ""SJ"", ""is_mobile"": true}" 3377,2,75,2017-01-10 16:06:23,http://strosin.com/ulises.stracke,0.6477960298,105.248.164.10,"{""location"": ""PK"", ""is_mobile"": true}" 3378,2,75,2017-03-20 00:06:28,http://white.com/rahul,0.3314935286,29.186.96.3,"{""location"": ""BZ"", ""is_mobile"": false}" 3379,2,75,2017-05-26 23:26:29,http://bahringer.net/mackenzie,0.3641569231,179.246.105.51,"{""location"": ""JM"", ""is_mobile"": true}" 3380,2,75,2017-04-04 08:20:25,http://brown.info/cecilia_abshire,0.4929411665,114.150.141.108,"{""location"": ""VG"", ""is_mobile"": true}" 3381,2,75,2017-01-06 18:38:36,http://blanda.name/ruth,0.9543421790,171.164.218.202,"{""location"": ""JM"", ""is_mobile"": false}" 3382,2,75,2017-01-17 10:56:18,http://oreillypollich.co/jeica,0.6938371394,35.209.17.113,"{""location"": ""AZ"", ""is_mobile"": true}" 3383,2,75,2017-01-21 04:36:47,http://boyle.net/matteo_wintheiser,0.2772141060,192.29.43.24,"{""location"": ""BW"", ""is_mobile"": false}" 3384,2,75,2016-12-28 10:17:08,http://gulgowskihuels.io/leonora,0.5667267704,15.137.160.63,"{""location"": ""BZ"", ""is_mobile"": false}" 3385,2,75,2017-02-09 17:01:38,http://turner.net/fritz,0.6711068506,247.211.93.5,"{""location"": ""LT"", ""is_mobile"": false}" 3386,2,75,2017-05-07 05:46:18,http://luettgenbayer.org/dejuan,0.4758818277,252.128.254.173,"{""location"": ""CL"", ""is_mobile"": true}" 3387,2,75,2017-01-14 23:23:36,http://bahringer.co/clint.hoppe,0.8600543533,6.155.191.169,"{""location"": ""TK"", ""is_mobile"": true}" 3388,2,75,2017-04-07 05:31:38,http://wunsch.org/mauricio.aufderhar,0.3442423088,51.188.151.8,"{""location"": ""MU"", ""is_mobile"": true}" 3389,2,75,2017-05-08 09:02:58,http://weberbayer.name/maxwell.rogahn,0.6128888803,201.190.134.26,"{""location"": ""NR"", ""is_mobile"": true}" 3390,2,75,2017-04-26 08:51:34,http://kutch.info/brennan_bergstrom,0.5683008262,198.168.103.74,"{""location"": ""DE"", ""is_mobile"": true}" 3391,2,75,2017-04-25 01:42:27,http://welch.net/lexi.schmitt,0.1952069273,87.49.90.152,"{""location"": ""BL"", ""is_mobile"": true}" 3392,2,75,2017-03-08 19:18:59,http://cainparker.io/fannie,0.2908365463,87.137.232.101,"{""location"": ""UG"", ""is_mobile"": false}" 3393,2,75,2017-04-03 16:12:41,http://erdman.io/lon.crist,0.7183839901,166.220.98.211,"{""location"": ""KW"", ""is_mobile"": false}" 3394,2,75,2017-04-04 00:47:40,http://haleycruickshank.net/talia_jacobs,0.4320038066,54.158.50.99,"{""location"": ""ER"", ""is_mobile"": true}" 3395,2,75,2017-01-19 04:45:38,http://kutch.biz/alek,0.1289284683,219.62.109.66,"{""location"": ""JM"", ""is_mobile"": false}" 3396,2,75,2017-03-25 11:07:19,http://mann.com/marian,0.6163136485,43.242.52.177,"{""location"": ""AF"", ""is_mobile"": true}" 3397,2,75,2017-01-02 08:44:39,http://schiller.biz/shanna,0.4999624490,128.43.151.249,"{""location"": ""CF"", ""is_mobile"": false}" 3398,2,75,2017-01-21 00:02:31,http://rutherford.co/alfonzo_okeefe,0.8018986088,226.2.193.103,"{""location"": ""PY"", ""is_mobile"": false}" 3399,2,75,2017-04-21 21:46:41,http://farrell.biz/eloy,0.8175547102,167.94.46.190,"{""location"": ""SE"", ""is_mobile"": false}" 3400,2,75,2017-01-28 17:43:24,http://kilback.com/brandi,0.8880545793,165.205.135.224,"{""location"": ""TM"", ""is_mobile"": true}" 3401,2,75,2017-05-01 19:36:52,http://terry.name/davonte,0.4886291504,126.137.95.80,"{""location"": ""AQ"", ""is_mobile"": false}" 3402,2,75,2017-01-08 20:40:45,http://reichert.com/lura,0.9979804867,230.175.226.193,"{""location"": ""WS"", ""is_mobile"": false}" 3403,2,75,2017-05-18 19:47:27,http://marks.co/carli,0.4168628322,134.83.37.40,"{""location"": ""TW"", ""is_mobile"": false}" 3404,2,75,2017-01-24 00:20:12,http://williamsonstehr.org/prudence,0.3603267049,149.206.197.82,"{""location"": ""RE"", ""is_mobile"": false}" 3405,2,75,2017-04-26 01:02:17,http://bogisichveum.net/alex_gutkowski,0.3095471375,56.97.83.82,"{""location"": ""JM"", ""is_mobile"": false}" 3406,2,75,2017-02-09 22:26:10,http://sauer.io/michelle.heidenreich,0.0042031375,118.131.210.58,"{""location"": ""CG"", ""is_mobile"": true}" 3407,2,75,2017-04-14 22:39:03,http://oberbrunnerkuhlman.org/else,0.1513546666,27.201.189.83,"{""location"": ""MZ"", ""is_mobile"": false}" 3408,2,75,2017-05-19 17:07:58,http://king.co/jadyn,0.0951166032,224.105.166.230,"{""location"": ""GH"", ""is_mobile"": false}" 3409,2,75,2017-02-19 07:38:18,http://grady.biz/itzel,0.9850280318,223.214.57.185,"{""location"": ""NZ"", ""is_mobile"": true}" 3410,2,75,2017-03-21 07:18:35,http://roberts.biz/avery,0.5407842795,188.37.175.109,"{""location"": ""ZW"", ""is_mobile"": true}" 3411,2,75,2017-02-20 14:50:31,http://beier.co/delmer.abbott,0.2605207416,48.210.93.176,"{""location"": ""GQ"", ""is_mobile"": false}" 3412,2,75,2017-03-14 23:16:25,http://jacobsgerhold.biz/nickolas.fadel,0.6147114645,124.103.55.48,"{""location"": ""GD"", ""is_mobile"": true}" 3413,2,75,2017-03-16 06:27:37,http://schneider.com/freddie,0.6997591781,36.72.202.80,"{""location"": ""NZ"", ""is_mobile"": true}" 3414,2,75,2017-03-25 02:25:21,http://durgan.name/celine_anderson,0.6713381605,110.222.175.38,"{""location"": ""KR"", ""is_mobile"": false}" 3415,2,75,2016-12-28 17:36:35,http://schroederwalker.org/rosie_king,0.1994014263,241.122.62.91,"{""location"": ""PA"", ""is_mobile"": false}" 3416,2,75,2017-04-30 08:05:18,http://glover.io/beverly,0.9307711344,230.109.16.19,"{""location"": ""MD"", ""is_mobile"": false}" 3417,2,75,2017-02-12 00:49:25,http://sipes.com/cary,0.3787116741,47.101.67.125,"{""location"": ""MN"", ""is_mobile"": false}" 3418,2,75,2017-01-25 21:43:27,http://purdy.name/davon_kaulke,0.4997361024,253.188.172.157,"{""location"": ""TV"", ""is_mobile"": true}" 3419,2,75,2017-02-05 03:04:35,http://hirthe.org/arnold,0.3729473352,155.98.202.102,"{""location"": ""TC"", ""is_mobile"": true}" 3420,2,75,2017-03-24 02:31:26,http://simonisheaney.co/ashton.koepp,0.1093954466,175.55.133.105,"{""location"": ""CR"", ""is_mobile"": true}" 3421,2,75,2017-01-12 05:55:28,http://monahanwest.co/giovanni_blick,0.2944820783,18.147.79.194,"{""location"": ""KM"", ""is_mobile"": true}" 10302,4,232,2017-04-24 05:05:48,http://connelly.net/johnathan.pacocha,0.5766852254,116.218.67.69,"{""location"": ""TR"", ""is_mobile"": true}" 10303,4,232,2017-02-06 09:14:27,http://wunsch.co/delpha_treutel,0.8754796255,141.105.242.77,"{""location"": ""TC"", ""is_mobile"": true}" 10304,4,232,2017-04-17 04:57:29,http://ebert.info/giuseppe_marquardt,0.9666389303,62.151.196.222,"{""location"": ""BQ"", ""is_mobile"": true}" 10305,4,232,2017-02-14 17:43:34,http://waelchi.co/aliyah,0.5816607737,193.52.132.243,"{""location"": ""DM"", ""is_mobile"": false}" 10306,4,232,2017-05-23 23:27:06,http://hellercain.org/terry.champlin,0.6977502523,130.133.164.54,"{""location"": ""BN"", ""is_mobile"": true}" 10307,4,232,2017-05-24 15:27:36,http://boyer.co/trent_stracke,0.3737416558,124.89.95.119,"{""location"": ""VI"", ""is_mobile"": false}" 10308,4,232,2017-04-09 14:24:38,http://pollichfeeney.name/reyna_maggio,0.5165004498,156.89.211.80,"{""location"": ""HM"", ""is_mobile"": true}" 10309,4,232,2017-05-28 01:09:11,http://gibson.co/bailee,0.5418506277,211.13.140.192,"{""location"": ""ER"", ""is_mobile"": true}" 10310,4,232,2016-12-26 09:08:46,http://ankundingaltenwerth.biz/ceasar.prohaska,0.1003636675,125.16.9.117,"{""location"": ""AI"", ""is_mobile"": false}" 10311,4,232,2017-05-17 12:22:40,http://adams.biz/nathen.miller,0.2299884632,127.195.248.179,"{""location"": ""JE"", ""is_mobile"": false}" 10312,4,232,2016-12-26 03:26:09,http://goldnerking.co/esperanza,0.7098713578,117.158.162.5,"{""location"": ""MS"", ""is_mobile"": true}" 10313,4,232,2017-05-06 22:12:51,http://emmerich.net/isabel,0.4491685494,27.226.29.239,"{""location"": ""PF"", ""is_mobile"": false}" 10314,4,232,2017-03-12 08:29:54,http://monahanhirthe.io/javonte.nolan,0.4315295130,72.156.122.52,"{""location"": ""IO"", ""is_mobile"": false}" 10315,4,232,2017-03-10 15:43:33,http://lueilwitz.io/arlie.torphy,0.6968920862,38.92.131.122,"{""location"": ""BI"", ""is_mobile"": false}" 10316,4,232,2017-05-09 08:42:08,http://champlin.org/ana_kirlin,0.6166241244,130.221.86.105,"{""location"": ""CV"", ""is_mobile"": false}" 10317,4,232,2017-05-14 12:56:57,http://gusikowski.com/jorge,0.7884550821,156.14.220.246,"{""location"": ""AL"", ""is_mobile"": false}" 10318,4,232,2017-05-24 05:30:14,http://windler.net/cesar,0.1915814112,84.68.221.81,"{""location"": ""MG"", ""is_mobile"": false}" 10319,4,232,2017-04-22 23:51:58,http://mcglynn.biz/kaela_mitchell,0.7476739414,47.128.193.97,"{""location"": ""CU"", ""is_mobile"": false}" 10320,4,233,2016-12-15 09:54:08,http://schimmel.biz/kaci,0.5313860183,145.142.190.60,"{""location"": ""FO"", ""is_mobile"": true}" 10321,4,233,2017-06-13 21:07:41,http://wizabode.net/annamarie.rogahn,0.5830613367,160.113.28.151,"{""location"": ""CG"", ""is_mobile"": true}" 10322,4,233,2017-06-12 17:39:35,http://oconnell.io/luna,0.9171928629,9.50.189.116,"{""location"": ""TF"", ""is_mobile"": false}" 10323,4,233,2017-06-13 13:31:37,http://ebert.org/branson,0.4159324368,171.202.165.119,"{""location"": ""ZM"", ""is_mobile"": false}" 10324,4,233,2017-04-12 00:45:56,http://lang.org/christiana.gulgowski,0.4169997180,159.5.72.75,"{""location"": ""HR"", ""is_mobile"": false}" 10325,4,233,2017-06-09 18:20:49,http://terry.io/wilhelm.hermiston,0.5714368566,138.8.47.238,"{""location"": ""PR"", ""is_mobile"": false}" 10326,4,233,2017-04-13 05:08:48,http://hagenes.co/eladio_macejkovic,0.7727668863,144.17.154.234,"{""location"": ""KY"", ""is_mobile"": false}" 10327,4,233,2017-01-08 19:39:11,http://metz.org/imani.jacobs,0.4223402053,208.229.222.223,"{""location"": ""BG"", ""is_mobile"": false}" 10328,4,233,2017-02-23 13:42:39,http://beerkonopelski.biz/lawson_becker,0.9797869022,32.207.6.95,"{""location"": ""SI"", ""is_mobile"": true}" 10329,4,233,2017-03-07 20:21:46,http://bauch.info/roxane.wyman,0.5430061440,109.15.115.241,"{""location"": ""SX"", ""is_mobile"": true}" 10330,4,233,2017-04-12 21:39:13,http://murray.name/samson,0.7243304301,203.10.65.9,"{""location"": ""GP"", ""is_mobile"": false}" 10331,4,233,2017-02-24 02:27:55,http://crist.co/zella.leffler,0.8001750229,18.5.127.184,"{""location"": ""KR"", ""is_mobile"": false}" 10332,4,233,2017-03-02 07:06:01,http://schuster.org/david_collier,0.0699062022,28.108.73.63,"{""location"": ""KY"", ""is_mobile"": false}" 10333,4,233,2017-02-11 17:31:54,http://hermistonoconnell.info/arno,0.6807600965,21.156.71.218,"{""location"": ""MD"", ""is_mobile"": false}" 10334,4,233,2017-03-24 02:16:56,http://bosco.co/retta,0.0589956718,145.82.54.97,"{""location"": ""HM"", ""is_mobile"": false}" 10335,4,233,2017-04-13 01:19:31,http://okon.info/emile,0.7686083101,150.39.40.42,"{""location"": ""BG"", ""is_mobile"": false}" 10336,4,233,2017-06-12 12:12:50,http://ebertprohaska.name/myron_mcdermott,0.4785455068,44.191.139.5,"{""location"": ""GE"", ""is_mobile"": true}" 10337,4,233,2017-06-06 05:10:53,http://koepp.org/sherwood.mosciski,0.4359782585,207.21.103.89,"{""location"": ""SX"", ""is_mobile"": true}" 10338,4,233,2017-06-13 10:20:02,http://rath.info/nella.stamm,0.9933801499,85.55.40.230,"{""location"": ""JE"", ""is_mobile"": false}" 10339,4,233,2017-02-23 13:37:29,http://feest.co/alexander,0.5582695376,60.33.84.162,"{""location"": ""JO"", ""is_mobile"": false}" 10340,4,233,2017-04-16 18:57:08,http://hahn.name/bette_rodriguez,0.5256977540,99.198.60.40,"{""location"": ""ML"", ""is_mobile"": true}" 10341,4,233,2017-04-03 02:17:15,http://croninframi.net/peter.johnson,0.6670274192,12.138.82.121,"{""location"": ""SN"", ""is_mobile"": true}" 10342,4,233,2017-04-17 21:21:24,http://quitzon.biz/dana_rippin,0.9143234167,62.148.139.173,"{""location"": ""ET"", ""is_mobile"": true}" 10343,4,233,2017-01-12 04:13:21,http://ebert.biz/bruce,0.8995920803,27.55.144.248,"{""location"": ""FO"", ""is_mobile"": false}" 10344,4,233,2017-06-08 11:27:58,http://fadel.com/lura,0.7067520331,31.79.227.213,"{""location"": ""FK"", ""is_mobile"": false}" 10345,4,233,2017-02-12 13:16:40,http://tromp.com/petra,0.3668838269,193.56.62.17,"{""location"": ""RU"", ""is_mobile"": true}" 10346,4,233,2017-03-30 14:00:29,http://bruen.co/eli.hansen,0.5698399915,221.65.252.13,"{""location"": ""GY"", ""is_mobile"": true}" 10347,4,233,2017-04-11 23:56:23,http://schiller.net/pierre,0.7834002952,173.57.246.85,"{""location"": ""UY"", ""is_mobile"": true}" 10348,4,233,2017-03-08 11:18:01,http://willmskerluke.org/tremaine,0.0247925768,220.245.227.174,"{""location"": ""CO"", ""is_mobile"": true}" 10349,4,233,2016-12-27 02:08:56,http://wuckert.name/alden,0.1347947862,61.122.32.131,"{""location"": ""CD"", ""is_mobile"": true}" 10350,4,233,2016-12-28 01:28:06,http://skilescartwright.org/furman.cruickshank,0.0855490202,134.113.6.11,"{""location"": ""BV"", ""is_mobile"": false}" 10351,4,233,2017-04-06 03:18:55,http://pouroslindgren.io/janis,0.0636159031,79.165.96.33,"{""location"": ""TO"", ""is_mobile"": false}" 10352,4,233,2017-03-29 15:42:15,http://stiedemann.io/quinn,0.1727632817,121.223.105.159,"{""location"": ""IL"", ""is_mobile"": false}" 7318,3,163,2017-05-16 19:06:40,http://gaylord.biz/audie,,33.70.52.131,"{""location"": ""YT"", ""is_mobile"": false}" 7319,3,163,2017-02-27 15:00:25,http://jakubowski.biz/jolie_waters,,151.55.16.129,"{""location"": ""PH"", ""is_mobile"": true}" 7320,3,163,2017-03-07 02:41:10,http://cartwrightschroeder.co/aleandra.parisian,,129.46.139.115,"{""location"": ""TN"", ""is_mobile"": true}" 7321,3,163,2017-01-26 07:56:08,http://veumheidenreich.info/jeika,,34.4.63.123,"{""location"": ""SJ"", ""is_mobile"": false}" 7322,3,163,2017-05-26 03:07:27,http://kohler.net/rodger.ko,,169.123.41.3,"{""location"": ""BT"", ""is_mobile"": true}" 7323,3,163,2017-02-15 11:11:21,http://stracke.info/treva,,42.154.126.55,"{""location"": ""DO"", ""is_mobile"": true}" 7324,3,163,2017-04-20 15:01:12,http://hettinger.com/heath,,141.84.205.106,"{""location"": ""LT"", ""is_mobile"": true}" 7325,3,163,2017-03-19 03:05:04,http://okeefeanderson.com/madeline,,75.195.242.86,"{""location"": ""FK"", ""is_mobile"": false}" 7326,3,163,2017-04-02 11:56:04,http://treutel.name/tanya.walsh,,7.101.58.148,"{""location"": ""JM"", ""is_mobile"": true}" 7327,3,163,2017-05-25 20:59:10,http://prosacco.info/evert,,128.59.209.129,"{""location"": ""PE"", ""is_mobile"": true}" 7328,3,163,2017-04-06 23:11:59,http://greenholtlueilwitz.info/luna,,62.110.170.69,"{""location"": ""MU"", ""is_mobile"": false}" 7329,3,163,2017-01-06 09:18:26,http://casper.org/myles_schmidt,,155.200.102.84,"{""location"": ""CV"", ""is_mobile"": true}" 7330,3,163,2016-12-19 21:39:32,http://huelswolf.info/pascale_howell,,130.54.245.243,"{""location"": ""SB"", ""is_mobile"": false}" 7331,3,163,2017-03-06 12:55:11,http://gorczany.org/hillard.harvey,,114.67.44.174,"{""location"": ""ER"", ""is_mobile"": true}" 7332,3,163,2017-04-17 14:29:30,http://trantow.com/helena,,39.66.47.158,"{""location"": ""WF"", ""is_mobile"": false}" 7333,3,163,2017-02-04 04:35:11,http://conn.org/delia_cummings,,190.92.184.27,"{""location"": ""TV"", ""is_mobile"": true}" 7334,3,163,2017-03-22 03:41:37,http://boscoko.info/addison,,214.245.208.137,"{""location"": ""TL"", ""is_mobile"": false}" 7335,3,163,2017-01-28 09:33:47,http://mayer.com/arden,,229.177.4.211,"{""location"": ""SE"", ""is_mobile"": true}" 7336,3,163,2017-06-04 03:09:33,http://bernhard.com/isaias.champlin,,240.10.8.62,"{""location"": ""GD"", ""is_mobile"": true}" 7337,3,163,2017-02-26 08:39:38,http://ondrickako.co/elenor_volkman,,170.79.145.43,"{""location"": ""WF"", ""is_mobile"": false}" 7338,3,163,2017-03-06 19:43:12,http://breitenberg.co/immanuel,,237.65.167.244,"{""location"": ""ET"", ""is_mobile"": false}" 7339,3,163,2017-01-09 14:24:11,http://mannconroy.info/jacques,,251.190.124.42,"{""location"": ""KE"", ""is_mobile"": false}" 7340,3,163,2017-02-27 01:46:04,http://bartell.co/juana.jacobs,,126.24.135.108,"{""location"": ""JO"", ""is_mobile"": true}" 7341,3,163,2017-03-11 09:22:56,http://streich.info/kendall,,253.176.227.153,"{""location"": ""MH"", ""is_mobile"": false}" 7342,3,163,2017-01-09 00:09:31,http://smithdenesik.biz/tyra.bayer,,18.211.118.150,"{""location"": ""ZM"", ""is_mobile"": true}" 7343,3,163,2017-03-07 16:28:14,http://macgyver.io/maxime.emard,,78.29.208.186,"{""location"": ""CG"", ""is_mobile"": true}" 7344,3,163,2017-04-27 14:46:13,http://schmitt.org/addie_wunsch,,87.193.175.243,"{""location"": ""CW"", ""is_mobile"": true}" 7345,3,163,2017-02-27 09:24:48,http://johnsonmclaughlin.name/ezequiel,,33.11.63.116,"{""location"": ""GB"", ""is_mobile"": true}" 7346,3,163,2017-04-11 10:24:16,http://gerhold.com/yeenia.sipes,,153.154.98.160,"{""location"": ""BL"", ""is_mobile"": true}" 7347,3,163,2017-04-04 04:54:52,http://trantow.org/tyrese,,214.149.222.235,"{""location"": ""TH"", ""is_mobile"": false}" 7348,3,163,2017-03-22 21:03:33,http://schowalter.org/kyla,,147.90.242.220,"{""location"": ""BT"", ""is_mobile"": true}" 7349,3,163,2017-01-05 17:12:25,http://considine.name/jaime,,212.72.40.93,"{""location"": ""TC"", ""is_mobile"": true}" 7350,3,163,2017-01-01 17:28:19,http://ferry.net/shirley,,206.83.251.237,"{""location"": ""NF"", ""is_mobile"": true}" 7351,3,163,2017-06-03 19:53:50,http://mayermarks.io/edmund_altenwerth,,72.58.35.112,"{""location"": ""EE"", ""is_mobile"": false}" 7352,3,163,2017-02-21 18:09:32,http://baileyfahey.io/jayne.heller,,7.96.10.46,"{""location"": ""TC"", ""is_mobile"": true}" 7353,3,163,2017-03-22 11:18:47,http://hintz.name/joanne_casper,,44.83.191.9,"{""location"": ""HR"", ""is_mobile"": true}" 7354,3,163,2017-02-19 08:01:27,http://prosacco.name/elwin.metz,,187.126.67.2,"{""location"": ""AR"", ""is_mobile"": false}" 7355,3,163,2016-12-21 03:31:48,http://hermannkerluke.org/elias,,111.44.204.120,"{""location"": ""TF"", ""is_mobile"": true}" 7356,3,163,2017-03-04 15:33:45,http://torp.net/caidy,,241.191.164.53,"{""location"": ""CR"", ""is_mobile"": false}" 7357,3,163,2017-04-17 13:14:18,http://rosenbaum.co/lorine_schultz,,248.117.131.21,"{""location"": ""SG"", ""is_mobile"": false}" 7358,3,163,2017-01-08 09:13:58,http://maggio.org/bettye,,65.162.87.226,"{""location"": ""CN"", ""is_mobile"": true}" 7359,3,163,2017-04-19 03:04:14,http://simonis.name/keshawn,,165.151.107.87,"{""location"": ""MY"", ""is_mobile"": true}" 7360,3,163,2017-06-11 06:24:51,http://hermannfranecki.name/myrtie_koelpin,,221.85.186.107,"{""location"": ""GE"", ""is_mobile"": true}" 7361,3,163,2017-01-19 09:37:04,http://oberbrunnerbins.net/anais,,70.93.67.137,"{""location"": ""AZ"", ""is_mobile"": true}" 7362,3,163,2017-05-30 22:14:01,http://schowalter.co/elenora,,57.140.166.174,"{""location"": ""ZW"", ""is_mobile"": false}" 7363,3,163,2017-04-11 15:43:01,http://kleinwhite.net/muhammad,,150.247.16.119,"{""location"": ""GS"", ""is_mobile"": true}" 7364,3,163,2017-03-25 19:54:06,http://howe.name/casandra.kling,,12.13.63.138,"{""location"": ""DK"", ""is_mobile"": true}" 7365,3,163,2017-05-24 11:07:43,http://funk.name/evie,,159.179.127.151,"{""location"": ""AD"", ""is_mobile"": true}" 7366,3,163,2017-02-09 00:04:25,http://conn.io/andres,,144.98.95.92,"{""location"": ""LS"", ""is_mobile"": false}" 7367,3,163,2017-01-20 18:36:30,http://hansen.co/kaitlin_rempel,,212.94.11.215,"{""location"": ""IM"", ""is_mobile"": true}" 7368,3,163,2017-04-07 10:48:45,http://klein.com/teresa,,251.207.254.166,"{""location"": ""SI"", ""is_mobile"": false}" 7369,3,163,2017-03-02 10:19:54,http://schmeler.name/lorna,,72.175.102.16,"{""location"": ""BW"", ""is_mobile"": false}" 7370,3,163,2017-02-16 03:49:20,http://steuberkonopelski.net/kareem,,209.125.167.89,"{""location"": ""DM"", ""is_mobile"": true}" 7371,3,163,2017-01-12 11:41:28,http://nienowbosco.io/jasen.balistreri,,57.216.97.173,"{""location"": ""SR"", ""is_mobile"": true}" 7372,3,163,2017-06-02 23:27:02,http://quigley.biz/declan,,159.42.25.103,"{""location"": ""SH"", ""is_mobile"": true}" 13228,5,298,2017-04-08 00:08:11,http://moen.info/dorothea,,243.230.53.181,"{""location"": ""TK"", ""is_mobile"": false}" 13229,5,298,2017-05-29 23:15:21,http://harber.biz/jakob.lynch,,223.71.101.19,"{""location"": ""AW"", ""is_mobile"": false}" 13230,5,298,2017-02-24 08:57:55,http://schinner.biz/odea.crona,,99.236.248.233,"{""location"": ""IS"", ""is_mobile"": true}" 13231,5,298,2017-04-04 14:04:03,http://metz.info/kyler.stokes,,111.17.65.104,"{""location"": ""FO"", ""is_mobile"": false}" 13232,5,298,2017-01-21 17:20:19,http://barton.com/charles_harvey,,19.15.46.46,"{""location"": ""CY"", ""is_mobile"": true}" 13233,5,298,2017-01-08 12:28:48,http://denesikgorczany.co/tevin.muller,,94.165.35.5,"{""location"": ""MX"", ""is_mobile"": true}" 13234,5,298,2017-04-14 18:05:49,http://brekkeswaniawski.info/alyce,,124.230.83.23,"{""location"": ""TT"", ""is_mobile"": false}" 13235,5,298,2017-05-05 19:27:13,http://stroman.org/casandra,,66.92.253.159,"{""location"": ""OM"", ""is_mobile"": true}" 13236,5,298,2017-04-10 05:19:23,http://harrisking.co/norene.rempel,,191.175.249.246,"{""location"": ""CK"", ""is_mobile"": true}" 13237,5,298,2017-03-14 05:32:36,http://okon.org/violette.bahringer,,215.107.121.149,"{""location"": ""KW"", ""is_mobile"": true}" 13238,5,298,2017-02-19 13:05:06,http://harris.biz/lonzo.mertz,,163.247.161.134,"{""location"": ""AW"", ""is_mobile"": false}" 13239,5,298,2017-03-13 23:44:28,http://crist.net/conner_gerhold,,227.192.221.23,"{""location"": ""ES"", ""is_mobile"": true}" 13240,5,298,2017-02-05 12:11:07,http://osinski.co/barry,,93.23.237.129,"{""location"": ""ZM"", ""is_mobile"": true}" 13241,5,298,2017-03-01 04:01:20,http://powlowski.co/alverta,,8.220.70.253,"{""location"": ""HM"", ""is_mobile"": true}" 13242,5,298,2017-04-14 02:21:34,http://kemmer.name/warren,,121.140.162.122,"{""location"": ""LS"", ""is_mobile"": false}" 13243,5,298,2017-02-27 09:16:45,http://lueilwitz.info/paxton.wolff,,8.44.11.148,"{""location"": ""IR"", ""is_mobile"": false}" 13244,5,298,2017-01-25 22:40:20,http://runolfsdottir.info/walker.murazik,,54.39.37.101,"{""location"": ""DJ"", ""is_mobile"": false}" 13245,5,298,2017-05-04 07:03:57,http://starkwalker.info/adriel,,105.212.94.223,"{""location"": ""SD"", ""is_mobile"": true}" 13246,5,298,2017-02-25 14:06:47,http://kshlerinbruen.com/craig.mante,,195.149.125.111,"{""location"": ""DE"", ""is_mobile"": false}" 13247,5,298,2016-12-20 23:08:12,http://kohler.com/boris,,143.58.114.143,"{""location"": ""SB"", ""is_mobile"": false}" 13248,5,298,2017-03-07 17:01:17,http://hudson.co/tommie_renner,,82.154.205.164,"{""location"": ""TN"", ""is_mobile"": false}" 13249,5,298,2017-01-04 20:39:29,http://romaguera.name/baron_powlowski,,190.42.157.162,"{""location"": ""SY"", ""is_mobile"": true}" 13250,5,298,2017-01-31 19:38:21,http://boyer.net/deshawn,,247.28.182.217,"{""location"": ""AF"", ""is_mobile"": true}" 13251,5,298,2017-04-30 12:23:13,http://greenholtwiegand.co/keagan,,94.201.129.23,"{""location"": ""DZ"", ""is_mobile"": true}" 13252,5,298,2017-04-10 07:58:13,http://spinkaconnelly.biz/brando.orn,,73.13.141.246,"{""location"": ""IN"", ""is_mobile"": false}" 13253,5,298,2017-02-11 06:51:26,http://pfefferjenkins.com/johnathan.effertz,,69.229.228.59,"{""location"": ""AI"", ""is_mobile"": false}" 13254,5,298,2017-03-08 06:33:14,http://dibbert.co/keenan_greenfelder,,166.207.42.169,"{""location"": ""KZ"", ""is_mobile"": true}" 13255,5,298,2017-04-18 17:58:31,http://stantonmertz.info/dock_nolan,,233.244.53.103,"{""location"": ""TJ"", ""is_mobile"": true}" 13256,5,298,2016-12-29 21:05:10,http://graham.name/rudolph,,20.151.225.193,"{""location"": ""MG"", ""is_mobile"": false}" 13257,5,298,2017-04-27 14:05:04,http://moen.org/jorge_ko,,98.105.127.223,"{""location"": ""BS"", ""is_mobile"": true}" 13258,5,298,2017-02-07 17:28:07,http://towne.biz/baylee,,10.151.108.162,"{""location"": ""KI"", ""is_mobile"": false}" 13259,5,298,2017-05-22 09:43:22,http://emard.biz/zoe,,182.205.252.146,"{""location"": ""SA"", ""is_mobile"": false}" 13260,5,298,2017-05-05 12:07:44,http://goyette.biz/theron,,50.29.163.62,"{""location"": ""MC"", ""is_mobile"": false}" 13261,5,298,2017-02-23 08:17:47,http://kovacek.name/eric_halvorson,,123.122.237.66,"{""location"": ""KM"", ""is_mobile"": false}" 13262,5,298,2017-06-07 14:30:07,http://farrell.name/garnet,,38.157.181.209,"{""location"": ""IM"", ""is_mobile"": true}" 13263,5,298,2017-02-17 08:27:32,http://legros.net/bethany_funk,,96.32.30.21,"{""location"": ""CY"", ""is_mobile"": false}" 13264,5,298,2017-05-08 19:18:57,http://bauch.biz/morris,,159.27.192.127,"{""location"": ""NR"", ""is_mobile"": true}" 13265,5,298,2017-04-20 21:04:03,http://connpowlowski.org/nelle_keler,,167.131.98.27,"{""location"": ""MW"", ""is_mobile"": true}" 13266,5,298,2017-02-21 13:57:54,http://waelchi.io/damien.wyman,,144.167.203.141,"{""location"": ""KE"", ""is_mobile"": true}" 13267,5,298,2017-01-07 12:45:56,http://walsh.biz/hellen,,43.81.65.165,"{""location"": ""CM"", ""is_mobile"": true}" 13268,5,298,2016-12-24 08:13:51,http://stammpfeffer.info/deron_gutmann,,216.136.174.49,"{""location"": ""AS"", ""is_mobile"": false}" 13269,5,298,2017-05-25 04:09:31,http://nolan.com/jacynthe,,178.248.140.187,"{""location"": ""TD"", ""is_mobile"": false}" 13270,5,298,2017-01-06 13:50:17,http://crist.com/georgiana_schimmel,,23.114.112.76,"{""location"": ""GT"", ""is_mobile"": false}" 13271,5,299,2017-06-02 19:58:52,http://reichel.io/ralph.gislason,,212.243.170.229,"{""location"": ""UZ"", ""is_mobile"": true}" 13272,5,299,2017-02-03 18:23:53,http://hettingerschmeler.org/pamela.jerde,,206.165.235.136,"{""location"": ""BA"", ""is_mobile"": true}" 13273,5,299,2017-05-26 23:48:46,http://bartoletti.org/kameron,,218.78.254.136,"{""location"": ""CR"", ""is_mobile"": true}" 13274,5,299,2017-02-10 11:39:16,http://wolff.org/citlalli.west,,164.36.238.52,"{""location"": ""CI"", ""is_mobile"": false}" 13275,5,299,2017-02-21 06:24:27,http://anderson.info/carson,,77.80.22.226,"{""location"": ""TV"", ""is_mobile"": true}" 13276,5,299,2017-05-01 19:11:49,http://bailey.com/murray,,128.239.117.117,"{""location"": ""ZM"", ""is_mobile"": true}" 13277,5,299,2017-04-04 21:24:42,http://marquardtjones.name/melyna,,8.177.90.237,"{""location"": ""FO"", ""is_mobile"": false}" 13278,5,299,2017-02-19 09:54:25,http://lehner.io/charity,,244.128.98.196,"{""location"": ""NR"", ""is_mobile"": true}" 13279,5,299,2017-05-19 10:41:52,http://orn.com/frida,,84.13.127.115,"{""location"": ""RU"", ""is_mobile"": true}" 13280,5,299,2017-05-25 03:22:55,http://jakubowskiquitzon.info/melisa,,93.211.177.187,"{""location"": ""HK"", ""is_mobile"": false}" 13281,5,299,2017-05-15 07:10:57,http://pfeffer.name/lukas_little,,175.108.65.88,"{""location"": ""BZ"", ""is_mobile"": false}" 13282,5,299,2017-03-23 09:38:32,http://doyle.net/maci_spinka,,113.251.200.230,"{""location"": ""GT"", ""is_mobile"": true}" 13283,5,299,2017-05-05 14:06:00,http://schultz.name/shanna,,247.88.152.171,"{""location"": ""NL"", ""is_mobile"": true}" 16245,6,368,2017-01-15 13:30:44,http://okunevaebert.io/haan_block,0.9873102921,38.238.168.135,"{""location"": ""TD"", ""is_mobile"": true}" 16246,6,368,2017-02-28 01:23:41,http://uptonlarson.org/aubrey_mitchell,0.9463942891,95.191.88.166,"{""location"": ""BW"", ""is_mobile"": true}" 16247,6,368,2017-04-29 07:07:14,http://kris.net/ruel_gulgowski,0.9389584342,40.207.202.158,"{""location"": ""BQ"", ""is_mobile"": true}" 16248,6,368,2016-12-25 02:46:48,http://wymanmarks.name/cade_goodwin,0.8300843886,46.247.210.238,"{""location"": ""MS"", ""is_mobile"": true}" 16249,6,368,2017-05-09 08:45:26,http://franecki.net/ford_king,0.2234202148,106.94.34.17,"{""location"": ""HN"", ""is_mobile"": false}" 16250,6,368,2017-03-08 05:44:37,http://batzmohr.name/armando.hauck,0.9120070570,63.242.161.113,"{""location"": ""KI"", ""is_mobile"": false}" 16251,6,368,2016-12-14 10:43:08,http://wolffsmitham.io/deja,0.9168017643,79.191.198.77,"{""location"": ""BT"", ""is_mobile"": true}" 16252,6,368,2017-03-05 19:03:26,http://davisgerlach.co/travon,0.4698140909,17.18.210.241,"{""location"": ""CW"", ""is_mobile"": false}" 16253,6,368,2017-06-13 12:02:37,http://durganabbott.io/santos.stark,0.4194204107,113.240.52.180,"{""location"": ""MU"", ""is_mobile"": false}" 16254,6,368,2017-03-20 04:49:01,http://kreiger.io/myrtis_romaguera,0.3903464819,132.242.188.17,"{""location"": ""JM"", ""is_mobile"": false}" 16255,6,368,2017-01-23 17:39:02,http://marquardtboyer.net/jewell,0.3145347043,14.199.221.217,"{""location"": ""ST"", ""is_mobile"": true}" 16256,6,368,2017-04-03 16:01:41,http://pagac.biz/lester.farrell,0.1280220352,180.231.106.123,"{""location"": ""LT"", ""is_mobile"": true}" 16257,6,368,2017-06-03 05:35:57,http://kirlin.co/foster,0.6597787251,109.117.98.56,"{""location"": ""VU"", ""is_mobile"": false}" 16258,6,368,2017-02-01 03:55:06,http://johnston.name/maximus,0.0701680996,42.73.35.25,"{""location"": ""EG"", ""is_mobile"": false}" 16259,6,368,2017-01-10 10:07:38,http://douglasbraun.biz/justyn_hills,0.6346064770,9.168.186.131,"{""location"": ""ER"", ""is_mobile"": true}" 16260,6,369,2017-04-28 04:15:53,http://zieme.net/jonathon,0.5754439774,73.162.4.239,"{""location"": ""IR"", ""is_mobile"": false}" 16261,6,369,2017-01-30 10:14:29,http://hettingergrant.io/ashly,0.0337277039,176.242.108.11,"{""location"": ""IO"", ""is_mobile"": true}" 16262,6,369,2017-06-01 02:16:59,http://koch.name/lawrence,0.4724159794,151.96.71.161,"{""location"": ""GH"", ""is_mobile"": true}" 16263,6,369,2017-04-09 12:08:26,http://rempelhammes.name/watson.rippin,0.3835698840,158.243.97.236,"{""location"": ""EE"", ""is_mobile"": true}" 16264,6,369,2017-06-08 16:30:52,http://durgan.io/lyda.quitzon,0.2700187305,126.250.198.58,"{""location"": ""TK"", ""is_mobile"": true}" 16265,6,369,2017-04-15 22:01:58,http://green.co/brain_aufderhar,0.0113180537,248.148.233.47,"{""location"": ""LV"", ""is_mobile"": false}" 16266,6,369,2017-03-25 03:00:02,http://dietrich.biz/osborne_jast,0.1454255094,142.172.194.56,"{""location"": ""GR"", ""is_mobile"": false}" 16267,6,369,2017-02-03 19:45:33,http://von.info/imogene_fay,0.4464987838,86.80.74.138,"{""location"": ""BV"", ""is_mobile"": true}" 16268,6,369,2017-05-05 23:30:51,http://donnellyluettgen.biz/dedrick,0.5072231450,70.181.252.29,"{""location"": ""KY"", ""is_mobile"": false}" 16269,6,369,2017-05-19 19:43:10,http://rohan.io/shannon.kulas,0.9281805122,32.26.184.232,"{""location"": ""VG"", ""is_mobile"": false}" 16270,6,369,2017-04-14 14:56:17,http://gulgowski.co/miguel_stroman,0.4453845040,152.133.46.226,"{""location"": ""BL"", ""is_mobile"": true}" 16271,6,369,2017-05-11 11:40:41,http://champlinherman.biz/alyce,0.6091485431,114.234.105.227,"{""location"": ""VG"", ""is_mobile"": false}" 16272,6,369,2017-02-09 01:54:41,http://olsonschuster.net/jalen,0.0990943742,14.54.135.24,"{""location"": ""SM"", ""is_mobile"": false}" 16273,6,369,2016-12-24 14:34:12,http://buckridge.biz/letitia,0.7071240623,216.55.139.160,"{""location"": ""AX"", ""is_mobile"": true}" 16274,6,369,2017-05-18 11:43:56,http://sipes.co/celestino.dietrich,0.6597806414,21.210.239.237,"{""location"": ""RE"", ""is_mobile"": true}" 16275,6,369,2017-05-16 16:28:44,http://buckridgedooley.name/rahsaan.damore,0.3605297457,64.160.110.192,"{""location"": ""ZA"", ""is_mobile"": false}" 16276,6,369,2017-01-14 22:10:42,http://schulist.com/duane.runolfsdottir,0.1473741185,69.246.12.226,"{""location"": ""NZ"", ""is_mobile"": false}" 16277,6,369,2017-05-19 15:57:08,http://leuschkemurazik.info/virgil_bosco,0.6482213889,87.153.64.178,"{""location"": ""BI"", ""is_mobile"": true}" 16278,6,369,2017-01-10 18:53:04,http://mante.org/jennyfer.farrell,0.2032017631,193.43.187.146,"{""location"": ""SM"", ""is_mobile"": true}" 16279,6,369,2016-12-27 21:25:47,http://douglas.name/kameron_connelly,0.1370963645,4.8.209.139,"{""location"": ""BI"", ""is_mobile"": false}" 16280,6,369,2017-02-18 05:45:08,http://trantow.biz/octavia,0.3486724951,160.228.220.133,"{""location"": ""CF"", ""is_mobile"": true}" 16281,6,369,2017-02-07 04:01:54,http://blockcarter.name/emmanuel_hilll,0.4935620530,26.120.214.46,"{""location"": ""BE"", ""is_mobile"": true}" 16282,6,369,2017-04-03 10:55:34,http://osinski.biz/maximillian_bradtke,0.9567287730,219.170.102.57,"{""location"": ""YE"", ""is_mobile"": false}" 16283,6,369,2017-02-20 10:42:36,http://bayer.biz/crawford,0.4517583520,83.102.37.189,"{""location"": ""VU"", ""is_mobile"": true}" 16284,6,369,2017-04-16 02:16:22,http://klocko.biz/newton_moore,0.7263436141,5.114.134.155,"{""location"": ""AZ"", ""is_mobile"": true}" 16285,6,369,2017-06-06 22:58:32,http://hermanntorphy.com/sandy.erdman,0.3447338173,66.203.37.51,"{""location"": ""MY"", ""is_mobile"": false}" 16286,6,369,2017-05-07 08:05:38,http://hamillnitzsche.info/aimee_goodwin,0.5688188248,180.239.29.43,"{""location"": ""PT"", ""is_mobile"": false}" 16287,6,369,2017-04-26 14:54:32,http://blick.org/charley,0.1654159243,45.84.112.9,"{""location"": ""YT"", ""is_mobile"": false}" 16288,6,369,2017-05-09 14:43:27,http://mckenzie.net/ena.cartwright,0.1180147275,160.170.252.134,"{""location"": ""HK"", ""is_mobile"": true}" 16289,6,369,2017-04-15 12:18:07,http://fritsch.org/jacky,0.9635111083,248.167.159.21,"{""location"": ""CY"", ""is_mobile"": true}" 16290,6,369,2017-05-04 04:21:45,http://ward.com/burley,0.3126057709,179.138.48.9,"{""location"": ""IE"", ""is_mobile"": false}" 16291,6,369,2017-04-08 15:53:46,http://braun.org/lorenzo,0.0739755123,150.252.100.87,"{""location"": ""GF"", ""is_mobile"": true}" 16292,6,369,2017-02-28 13:44:30,http://considine.name/reagan_watsica,0.3183597835,100.154.164.54,"{""location"": ""PG"", ""is_mobile"": true}" 16293,6,369,2017-05-11 18:12:11,http://zboncak.info/chauncey,0.4118755984,70.224.122.93,"{""location"": ""NE"", ""is_mobile"": true}" 16294,6,369,2017-01-02 13:22:30,http://connelly.io/franz_bernhard,0.9940413354,185.169.68.52,"{""location"": ""SM"", ""is_mobile"": true}" 16295,6,369,2017-04-23 20:03:01,http://osinski.org/ford,0.1315041406,49.40.117.213,"{""location"": ""IT"", ""is_mobile"": false}" 3422,2,75,2017-02-23 23:38:37,http://koepp.info/felipa_grant,0.3790825185,81.172.227.32,"{""location"": ""VU"", ""is_mobile"": false}" 3423,2,75,2017-03-08 15:55:39,http://daniel.co/joanne,0.6407551796,42.127.115.130,"{""location"": ""SV"", ""is_mobile"": false}" 3424,2,75,2016-12-14 23:25:15,http://schustersawayn.org/verda,0.3184199098,89.204.60.235,"{""location"": ""CZ"", ""is_mobile"": true}" 3425,2,75,2017-05-07 21:35:57,http://ferry.co/ned,0.2626383275,118.9.92.137,"{""location"": ""SD"", ""is_mobile"": true}" 3426,2,75,2017-05-25 15:30:07,http://lindgren.info/alisa_kuhic,0.8839290630,159.192.142.217,"{""location"": ""SH"", ""is_mobile"": true}" 3427,2,75,2017-02-06 05:19:30,http://pricedietrich.info/adolphus_mraz,0.9491762344,6.19.183.245,"{""location"": ""CA"", ""is_mobile"": true}" 3428,2,75,2016-12-29 23:21:02,http://medhurst.io/imani.trantow,0.6055260922,21.239.210.51,"{""location"": ""TH"", ""is_mobile"": true}" 3429,2,75,2017-02-28 15:07:15,http://rohan.io/mohamed.osinski,0.4424948208,245.251.195.55,"{""location"": ""SI"", ""is_mobile"": true}" 3430,2,75,2017-03-23 21:28:23,http://runte.io/floie.windler,0.5805570890,200.92.243.37,"{""location"": ""VN"", ""is_mobile"": false}" 3431,2,76,2017-02-22 08:09:47,http://hanesmitham.io/emily.reilly,0.9226101750,237.157.126.238,"{""location"": ""SG"", ""is_mobile"": false}" 3432,2,76,2017-04-01 04:45:13,http://johns.org/viviane,0.7864294932,27.178.106.221,"{""location"": ""MO"", ""is_mobile"": false}" 3433,2,76,2017-02-07 03:26:30,http://mckenzieritchie.info/sigurd,0.5753753195,227.250.120.216,"{""location"": ""FR"", ""is_mobile"": true}" 3434,2,76,2017-04-28 18:31:45,http://wiegand.org/gertrude.schinner,0.6983172262,183.3.215.229,"{""location"": ""NR"", ""is_mobile"": true}" 3435,2,76,2017-03-01 14:02:05,http://bailey.biz/verdie.brown,0.7047169300,205.157.141.233,"{""location"": ""BJ"", ""is_mobile"": false}" 3436,2,76,2017-04-02 09:38:07,http://tremblay.org/christine.mayert,0.3366553677,46.98.20.40,"{""location"": ""TR"", ""is_mobile"": true}" 3437,2,76,2017-05-30 17:06:49,http://rippin.info/elnora_weber,0.9609882074,186.56.183.11,"{""location"": ""LU"", ""is_mobile"": true}" 3438,2,76,2017-02-20 13:26:40,http://treutel.com/lawson,0.9559764240,208.195.59.250,"{""location"": ""AR"", ""is_mobile"": true}" 3439,2,76,2017-05-23 02:32:28,http://cartertorphy.info/art.runolfsdottir,0.3098590675,252.130.45.113,"{""location"": ""PW"", ""is_mobile"": true}" 3440,2,76,2017-04-07 16:36:58,http://raupredovic.info/zion,0.1779518690,148.231.120.51,"{""location"": ""GU"", ""is_mobile"": false}" 3441,2,76,2017-04-19 16:27:27,http://konopelskischimmel.io/alisa,0.5441564397,34.91.3.29,"{""location"": ""FJ"", ""is_mobile"": false}" 3442,2,76,2017-01-26 00:02:40,http://wehner.org/wayne.wehner,0.9183848694,164.152.16.5,"{""location"": ""CF"", ""is_mobile"": false}" 3443,2,76,2016-12-24 00:06:38,http://weinat.com/gertrude,0.2711227337,180.69.102.158,"{""location"": ""SX"", ""is_mobile"": true}" 3444,2,76,2017-03-14 22:59:22,http://hickle.io/lambert,0.2294650904,243.89.235.64,"{""location"": ""LV"", ""is_mobile"": false}" 3445,2,76,2017-05-25 03:18:19,http://kuphal.biz/kenyatta.frami,0.2359243707,201.46.216.193,"{""location"": ""IM"", ""is_mobile"": true}" 3446,2,76,2017-03-13 17:39:55,http://lindfadel.name/bella,0.2325263309,153.111.253.194,"{""location"": ""US"", ""is_mobile"": false}" 3447,2,76,2017-04-02 04:53:17,http://shanahanlittel.co/claire.bergstrom,0.4329497801,61.138.31.215,"{""location"": ""MH"", ""is_mobile"": true}" 3448,2,76,2017-04-11 13:36:02,http://ko.info/kurt,0.7719690706,75.39.25.148,"{""location"": ""PG"", ""is_mobile"": false}" 3449,2,76,2017-03-04 18:45:25,http://wisoky.info/cordia,0.2241097954,137.6.143.20,"{""location"": ""GL"", ""is_mobile"": false}" 3450,2,76,2017-04-18 13:24:03,http://mayercremin.com/fritz.lowe,0.3311455758,85.178.152.24,"{""location"": ""SD"", ""is_mobile"": false}" 3451,2,76,2017-04-27 01:39:02,http://effertz.com/ivy.torp,0.4186087980,152.218.84.8,"{""location"": ""DM"", ""is_mobile"": false}" 3452,2,76,2017-06-01 20:13:07,http://wardmcglynn.name/winnifred_kuhlman,0.0715451990,50.206.41.30,"{""location"": ""FM"", ""is_mobile"": false}" 3453,2,76,2017-01-05 16:10:52,http://rohanlehner.name/shawn,0.7155945608,250.35.245.45,"{""location"": ""MY"", ""is_mobile"": true}" 3454,2,76,2017-01-28 08:56:29,http://wintheiser.io/adrian,0.7955898943,78.175.231.8,"{""location"": ""DK"", ""is_mobile"": true}" 3455,2,76,2017-01-21 16:55:22,http://larson.com/eliezer_pouros,0.2565055680,38.237.216.237,"{""location"": ""CM"", ""is_mobile"": true}" 3456,2,76,2017-03-11 18:17:55,http://okuneva.biz/izabella.boehm,0.2585407048,97.149.213.222,"{""location"": ""LS"", ""is_mobile"": true}" 3457,2,76,2017-04-25 17:06:41,http://weber.info/lon,0.0031570732,7.26.151.182,"{""location"": ""OM"", ""is_mobile"": false}" 3458,2,76,2017-03-28 22:10:26,http://vonrueden.name/saige,0.9540906608,134.193.75.114,"{""location"": ""TW"", ""is_mobile"": true}" 3459,2,76,2017-05-16 02:55:02,http://wolff.biz/dina_bradtke,0.2825102295,13.125.157.200,"{""location"": ""DO"", ""is_mobile"": true}" 3460,2,76,2017-02-14 06:14:13,http://heathcote.io/mose,0.7278688947,191.218.247.240,"{""location"": ""RU"", ""is_mobile"": true}" 3461,2,76,2017-06-03 23:52:21,http://olson.io/mariana,0.3593013493,243.224.149.185,"{""location"": ""TK"", ""is_mobile"": true}" 3462,2,76,2017-06-03 15:06:46,http://heathcotedeckow.biz/autumn.runolfsdottir,0.5345384645,142.113.5.240,"{""location"": ""MA"", ""is_mobile"": true}" 3463,2,76,2017-04-15 03:21:51,http://hellerstehr.io/nola.bergnaum,0.6920942458,102.212.47.146,"{""location"": ""GM"", ""is_mobile"": true}" 3464,2,76,2017-02-19 08:41:54,http://kaulkehermann.co/flavio_parker,0.1038661566,95.99.240.153,"{""location"": ""BZ"", ""is_mobile"": true}" 3465,2,76,2017-04-28 09:04:24,http://reinger.name/watson_stamm,0.2810152866,169.69.93.64,"{""location"": ""MU"", ""is_mobile"": false}" 3466,2,76,2017-04-21 18:35:42,http://shanahan.org/lorenz,0.9606444690,10.178.221.98,"{""location"": ""HM"", ""is_mobile"": true}" 3467,2,76,2017-01-17 12:53:17,http://torp.info/vernie_hoeger,0.0187218003,156.115.90.141,"{""location"": ""BS"", ""is_mobile"": true}" 3468,2,76,2017-03-14 08:42:20,http://bednarokeefe.biz/brendon,0.2241540253,25.56.62.8,"{""location"": ""GF"", ""is_mobile"": true}" 3469,2,76,2017-01-14 03:08:49,http://pollichreichert.net/marley,0.9168778879,95.206.31.58,"{""location"": ""BQ"", ""is_mobile"": false}" 3470,2,76,2017-05-18 02:49:31,http://gerlachmurray.io/pascale,0.8558500037,5.90.248.133,"{""location"": ""NC"", ""is_mobile"": true}" 3471,2,76,2017-05-22 19:22:08,http://cruickshank.io/giuseppe.weimann,0.7523806443,49.54.80.234,"{""location"": ""RE"", ""is_mobile"": false}" 3472,2,76,2017-05-25 11:57:53,http://boyerjenkins.co/americo.sanford,0.6849335270,184.36.195.239,"{""location"": ""AW"", ""is_mobile"": false}" 3473,2,76,2017-02-15 16:26:51,http://ondricka.net/astrid.padberg,0.8012557887,199.94.181.155,"{""location"": ""MY"", ""is_mobile"": false}" 10353,4,233,2017-05-14 00:36:31,http://blanda.org/bethany,0.8891134476,16.225.191.208,"{""location"": ""BE"", ""is_mobile"": true}" 10354,4,233,2016-12-29 17:37:34,http://hudson.name/hayley_herzog,0.1036093910,134.197.188.103,"{""location"": ""KZ"", ""is_mobile"": false}" 10355,4,233,2017-06-02 23:30:50,http://toy.co/laurie.kreiger,0.7091896805,69.187.234.146,"{""location"": ""BQ"", ""is_mobile"": false}" 10356,4,233,2017-04-03 18:38:20,http://wisozk.net/vivian,0.5835490360,68.188.126.182,"{""location"": ""SB"", ""is_mobile"": false}" 10357,4,233,2017-03-25 02:44:38,http://bednar.io/marina,0.0754743037,242.20.131.50,"{""location"": ""CC"", ""is_mobile"": false}" 10358,4,233,2017-02-11 10:46:49,http://armstrong.name/danyka,0.9823391762,100.253.33.212,"{""location"": ""CL"", ""is_mobile"": true}" 10359,4,233,2017-05-28 08:25:02,http://spencer.co/aubrey.schowalter,0.4974512335,198.228.86.207,"{""location"": ""AT"", ""is_mobile"": false}" 10360,4,233,2017-02-15 23:00:04,http://schulistdaugherty.com/frederic,0.2931384880,43.138.86.218,"{""location"": ""AT"", ""is_mobile"": false}" 10361,4,233,2017-02-07 05:26:43,http://larkinkub.io/bernie.bahringer,0.6887152543,191.203.12.181,"{""location"": ""PN"", ""is_mobile"": true}" 10362,4,233,2017-06-07 15:38:35,http://sawayn.io/eleanora_dickens,0.4463504250,33.197.111.5,"{""location"": ""CG"", ""is_mobile"": false}" 10363,4,233,2016-12-21 16:20:54,http://schaefer.net/chase.lind,0.8341924702,252.183.180.125,"{""location"": ""CW"", ""is_mobile"": true}" 10364,4,233,2017-03-01 06:56:06,http://gleason.com/devan_runolfon,0.1390004634,48.60.144.104,"{""location"": ""HT"", ""is_mobile"": false}" 10365,4,233,2017-06-10 03:13:44,http://mohr.info/talon,0.4604288957,101.31.252.254,"{""location"": ""CW"", ""is_mobile"": false}" 10366,4,233,2017-01-09 01:55:09,http://heller.co/elody_mcglynn,0.5798585361,200.163.24.254,"{""location"": ""CY"", ""is_mobile"": true}" 10367,4,233,2017-03-30 11:21:39,http://baileycrist.net/rhea,0.4634556661,129.251.191.17,"{""location"": ""DJ"", ""is_mobile"": true}" 10368,4,233,2017-04-28 02:29:02,http://johnson.biz/willa,0.2977830818,33.226.56.126,"{""location"": ""HM"", ""is_mobile"": true}" 10369,4,233,2017-02-04 18:22:59,http://robel.info/donavon.hamill,0.2965782686,120.68.73.233,"{""location"": ""TN"", ""is_mobile"": true}" 10370,4,233,2017-02-18 21:00:12,http://hand.biz/americo,0.2108010811,244.135.113.8,"{""location"": ""SL"", ""is_mobile"": false}" 10371,4,233,2017-04-17 21:00:44,http://jaskolski.name/tyrese.predovic,0.2390435823,29.28.122.170,"{""location"": ""UM"", ""is_mobile"": false}" 10372,4,233,2017-03-03 13:52:45,http://hilpert.co/noel.lubowitz,0.2338709626,48.141.147.8,"{""location"": ""BV"", ""is_mobile"": true}" 10373,4,233,2017-06-11 18:41:13,http://kuhic.info/rosalyn,0.2297831935,161.189.56.44,"{""location"": ""TJ"", ""is_mobile"": true}" 10374,4,233,2016-12-22 07:37:53,http://batz.info/hunter,0.5087692074,102.101.138.64,"{""location"": ""SV"", ""is_mobile"": false}" 10375,4,233,2017-03-12 20:05:48,http://collier.biz/dedrick,0.8118078616,117.93.201.46,"{""location"": ""LR"", ""is_mobile"": true}" 10376,4,233,2017-05-19 08:00:40,http://hansen.com/vida,0.6980505815,95.93.29.68,"{""location"": ""MV"", ""is_mobile"": true}" 10377,4,233,2016-12-31 23:20:41,http://feestgerhold.co/tiara_rodriguez,0.9234647050,20.227.53.242,"{""location"": ""SV"", ""is_mobile"": false}" 10378,4,233,2017-05-28 03:43:39,http://hermannupton.info/brittany,0.7765468099,214.187.94.68,"{""location"": ""GN"", ""is_mobile"": true}" 10379,4,233,2017-02-19 20:13:46,http://hahn.org/susie_reynolds,0.9260299324,11.87.228.168,"{""location"": ""LV"", ""is_mobile"": true}" 10380,4,233,2017-03-07 05:54:15,http://purdy.co/kamille_baumbach,0.8778112100,148.10.212.81,"{""location"": ""BH"", ""is_mobile"": true}" 10381,4,233,2016-12-31 14:31:52,http://schroederchamplin.org/casper,0.7207356286,184.211.111.62,"{""location"": ""CK"", ""is_mobile"": false}" 10382,4,233,2017-05-01 06:02:13,http://kozeyparisian.name/chandler.reichert,0.6722432179,112.231.130.111,"{""location"": ""GW"", ""is_mobile"": false}" 10383,4,233,2017-04-06 05:24:58,http://hoppewyman.net/moie.torphy,0.6828329433,19.35.221.253,"{""location"": ""SC"", ""is_mobile"": true}" 10384,4,233,2016-12-19 23:48:53,http://legros.com/mireille,0.2136537032,81.164.174.34,"{""location"": ""VI"", ""is_mobile"": false}" 10385,4,233,2016-12-13 15:10:55,http://kuvalis.info/emmanuelle,0.9889925310,102.244.53.231,"{""location"": ""ZM"", ""is_mobile"": false}" 10386,4,233,2017-03-20 15:02:08,http://grimehields.com/nikolas.greenholt,0.1086702870,16.50.225.114,"{""location"": ""TJ"", ""is_mobile"": true}" 10387,4,234,2017-03-21 21:25:36,http://little.org/laurie.stamm,0.6573493538,127.194.72.51,"{""location"": ""PR"", ""is_mobile"": true}" 10388,4,234,2017-05-05 08:38:55,http://orn.com/holden,0.6484413405,56.103.111.38,"{""location"": ""RU"", ""is_mobile"": true}" 10389,4,234,2017-03-10 04:33:13,http://mueller.info/titus,0.2374538896,110.217.157.206,"{""location"": ""MF"", ""is_mobile"": true}" 10390,4,234,2017-02-03 00:51:56,http://doyle.org/kayli.nader,0.8973808488,90.246.16.234,"{""location"": ""RW"", ""is_mobile"": false}" 10391,4,234,2017-02-05 10:16:18,http://gislason.co/morton.mcdermott,0.8458858184,149.169.115.190,"{""location"": ""TC"", ""is_mobile"": false}" 10392,4,234,2016-12-21 11:38:41,http://maggio.net/gregg,0.0284452825,85.78.17.219,"{""location"": ""MC"", ""is_mobile"": true}" 10393,4,234,2017-03-02 00:09:21,http://vandervort.io/baylee,0.6817406640,31.49.30.143,"{""location"": ""AX"", ""is_mobile"": true}" 10394,4,234,2017-04-22 23:38:03,http://franecki.name/tavares,0.4982464665,73.180.95.129,"{""location"": ""GB"", ""is_mobile"": false}" 10395,4,234,2017-05-20 21:28:17,http://bailey.info/augustus,0.0717753045,71.161.155.96,"{""location"": ""ET"", ""is_mobile"": false}" 10396,4,234,2017-04-15 01:00:26,http://konopelski.com/theresa_johnston,0.7155839449,73.17.47.28,"{""location"": ""PL"", ""is_mobile"": true}" 10397,4,234,2017-04-17 11:37:11,http://hane.name/mittie_hackett,0.3430854160,107.191.172.133,"{""location"": ""DJ"", ""is_mobile"": true}" 10398,4,234,2017-01-04 10:36:49,http://jerde.io/cleve,0.7402261959,240.40.43.191,"{""location"": ""UG"", ""is_mobile"": false}" 10399,4,234,2017-03-03 10:42:34,http://beatty.co/lizeth,0.5228685974,171.206.135.227,"{""location"": ""HM"", ""is_mobile"": true}" 10400,4,234,2017-01-17 21:02:00,http://moen.biz/santina_anderson,0.1239439254,115.42.204.142,"{""location"": ""KY"", ""is_mobile"": false}" 10401,4,234,2017-03-21 05:18:03,http://lebsack.co/treva_spencer,0.0171621802,11.217.42.229,"{""location"": ""SA"", ""is_mobile"": true}" 10402,4,234,2017-04-02 11:13:27,http://vandervortmorar.com/mina.purdy,0.1553133322,60.233.152.92,"{""location"": ""MQ"", ""is_mobile"": true}" 10403,4,234,2017-04-02 19:54:50,http://vonruedenkrajcik.net/sarai,0.4341828697,187.222.40.9,"{""location"": ""CN"", ""is_mobile"": false}" 7373,3,164,2017-06-05 16:35:06,http://rogahnmurray.info/rashawn_renner,,251.245.207.122,"{""location"": ""RS"", ""is_mobile"": true}" 7374,3,164,2017-04-18 04:30:12,http://morar.info/vernie,,16.89.34.116,"{""location"": ""NE"", ""is_mobile"": false}" 7375,3,164,2017-02-27 00:17:46,http://bergstrommayer.org/romaine.ruel,,99.172.128.31,"{""location"": ""BL"", ""is_mobile"": false}" 7376,3,164,2017-04-25 03:28:11,http://kreiger.info/ethan,,215.132.105.25,"{""location"": ""NF"", ""is_mobile"": true}" 7377,3,164,2017-06-05 02:58:15,http://waelchi.name/edison,,33.252.95.9,"{""location"": ""CH"", ""is_mobile"": false}" 7378,3,164,2017-01-08 06:23:12,http://huelcain.co/rosemary_blanda,,69.35.157.93,"{""location"": ""JP"", ""is_mobile"": true}" 7379,3,164,2017-01-14 16:40:38,http://hand.net/chaim.hermiston,,240.168.93.169,"{""location"": ""LI"", ""is_mobile"": true}" 7380,3,164,2017-04-29 19:07:54,http://hoeger.biz/glenna,,238.179.37.42,"{""location"": ""SY"", ""is_mobile"": false}" 7381,3,164,2017-04-02 18:52:21,http://ullrich.name/marge.barton,,238.230.164.233,"{""location"": ""UG"", ""is_mobile"": true}" 7382,3,164,2017-03-31 14:21:24,http://graham.co/estelle.yost,,83.234.153.133,"{""location"": ""AF"", ""is_mobile"": false}" 7383,3,164,2017-04-06 08:47:27,http://erdmanschmeler.com/opal_lowe,,195.160.220.132,"{""location"": ""GF"", ""is_mobile"": false}" 7384,3,164,2017-02-27 20:28:21,http://goldnerprohaska.io/wyatt,,4.69.251.237,"{""location"": ""UA"", ""is_mobile"": false}" 7385,3,164,2017-05-31 07:16:20,http://kunze.io/raleigh_kulas,,180.74.45.113,"{""location"": ""EH"", ""is_mobile"": true}" 7386,3,164,2017-01-24 19:29:30,http://anderson.biz/oswaldo.gorczany,,39.85.131.21,"{""location"": ""LV"", ""is_mobile"": false}" 7387,3,164,2017-06-05 07:37:45,http://gutmanndaniel.biz/vivien.schoen,,249.151.13.109,"{""location"": ""CO"", ""is_mobile"": true}" 7388,3,164,2017-05-16 08:08:08,http://ebert.io/autumn_fahey,,253.251.177.33,"{""location"": ""DZ"", ""is_mobile"": true}" 7389,3,164,2017-06-01 19:23:32,http://murrayschmeler.com/queen.sauer,,178.201.244.103,"{""location"": ""LC"", ""is_mobile"": true}" 7390,3,164,2017-03-11 03:40:35,http://smith.org/summer_beer,,120.45.4.114,"{""location"": ""SJ"", ""is_mobile"": true}" 7391,3,164,2017-02-01 08:17:58,http://greenholt.name/fernando,,15.123.110.132,"{""location"": ""DJ"", ""is_mobile"": true}" 7392,3,164,2017-02-28 22:57:59,http://frami.net/mark,,50.19.163.98,"{""location"": ""TN"", ""is_mobile"": true}" 7393,3,164,2017-01-21 03:26:19,http://mcculloughmayert.biz/roxane_sanford,,138.27.196.40,"{""location"": ""PK"", ""is_mobile"": true}" 7394,3,164,2017-03-10 23:23:33,http://donnellyernser.io/rylan.hoppe,,248.5.148.91,"{""location"": ""MM"", ""is_mobile"": true}" 7395,3,164,2017-02-27 05:38:44,http://nolan.co/amalia,,212.202.158.111,"{""location"": ""RS"", ""is_mobile"": false}" 7396,3,164,2017-01-29 00:45:41,http://barrows.co/evangeline_price,,231.116.243.147,"{""location"": ""CA"", ""is_mobile"": false}" 7397,3,164,2017-05-12 14:05:09,http://cole.io/adalberto,,32.157.88.107,"{""location"": ""SA"", ""is_mobile"": false}" 7398,3,164,2017-05-27 01:45:14,http://rath.name/lowell.terry,,109.64.172.103,"{""location"": ""ZA"", ""is_mobile"": true}" 7399,3,164,2017-03-30 16:52:08,http://gaylord.org/enola,,250.33.231.108,"{""location"": ""HT"", ""is_mobile"": false}" 7400,3,164,2017-05-10 04:03:38,http://langworth.io/angel.bernhard,,75.127.129.85,"{""location"": ""MT"", ""is_mobile"": true}" 7401,3,164,2017-04-04 15:09:40,http://corkery.net/deonte,,199.121.60.179,"{""location"": ""CH"", ""is_mobile"": true}" 7402,3,164,2017-06-12 19:13:19,http://boehmlesch.net/torey,,35.146.35.23,"{""location"": ""FI"", ""is_mobile"": false}" 7403,3,164,2017-06-10 21:16:09,http://dachkemmer.co/reagan,,70.126.125.26,"{""location"": ""BW"", ""is_mobile"": true}" 7404,3,164,2017-04-15 03:07:31,http://eichmannlehner.biz/lilly.metz,,12.119.54.120,"{""location"": ""IM"", ""is_mobile"": true}" 7405,3,164,2017-04-17 20:41:10,http://botsford.org/meagan.raynor,,82.200.24.112,"{""location"": ""CZ"", ""is_mobile"": false}" 7406,3,164,2017-03-25 22:31:57,http://ankundingbartell.org/erick,,129.194.217.48,"{""location"": ""MM"", ""is_mobile"": true}" 7407,3,164,2017-01-23 02:16:08,http://altenwerth.net/ena,,179.213.223.57,"{""location"": ""MS"", ""is_mobile"": true}" 7408,3,164,2017-05-28 01:31:35,http://schneiderkohler.net/ansel,,183.149.18.11,"{""location"": ""RO"", ""is_mobile"": false}" 7409,3,164,2017-01-20 12:42:21,http://rogahnko.org/gideon,,38.82.141.82,"{""location"": ""MN"", ""is_mobile"": true}" 7410,3,164,2017-06-08 13:45:21,http://balistrerisawayn.info/precious_lehner,,53.187.89.39,"{""location"": ""AZ"", ""is_mobile"": false}" 7411,3,164,2017-05-05 22:34:49,http://erdman.biz/kirsten,,40.237.71.145,"{""location"": ""DZ"", ""is_mobile"": false}" 7412,3,164,2017-02-03 08:49:25,http://mraz.org/gladyce_baumbach,,156.183.47.110,"{""location"": ""TF"", ""is_mobile"": false}" 7413,3,164,2017-05-22 05:09:19,http://kutchgerhold.name/teie_ankunding,,221.115.143.167,"{""location"": ""CZ"", ""is_mobile"": false}" 7414,3,164,2017-02-27 11:29:10,http://dietrich.io/horace,,84.177.90.90,"{""location"": ""NF"", ""is_mobile"": false}" 7415,3,164,2017-01-20 18:52:57,http://boderolfson.net/grace.kuhic,,37.80.142.147,"{""location"": ""CF"", ""is_mobile"": true}" 7416,3,164,2016-12-25 18:21:53,http://friesen.biz/esperanza,,157.41.236.58,"{""location"": ""KE"", ""is_mobile"": false}" 7417,3,164,2017-06-11 02:09:14,http://rolfson.co/van.armstrong,,146.36.41.85,"{""location"": ""NA"", ""is_mobile"": false}" 7418,3,165,2017-04-09 12:51:11,http://ferry.com/adam,,80.117.47.28,"{""location"": ""GG"", ""is_mobile"": false}" 7419,3,165,2017-01-12 02:30:01,http://sawayn.com/martin_schuster,,190.136.100.151,"{""location"": ""HT"", ""is_mobile"": false}" 7420,3,165,2017-04-28 09:32:32,http://windler.net/isai,,185.21.168.171,"{""location"": ""RE"", ""is_mobile"": false}" 7421,3,165,2017-01-19 12:28:27,http://gottliebparisian.net/caandre,,147.114.142.229,"{""location"": ""TR"", ""is_mobile"": false}" 7422,3,165,2017-04-20 14:40:35,http://dare.net/clementine.maggio,,209.109.173.103,"{""location"": ""LR"", ""is_mobile"": false}" 7423,3,165,2017-06-09 07:09:30,http://schuppe.name/marvin_renner,,199.224.104.7,"{""location"": ""LA"", ""is_mobile"": true}" 7424,3,165,2017-02-10 01:56:52,http://ryan.com/bryce.gutkowski,,106.12.61.222,"{""location"": ""SK"", ""is_mobile"": true}" 7425,3,165,2017-05-21 14:46:13,http://leschhintz.name/angela,,219.47.9.242,"{""location"": ""PS"", ""is_mobile"": false}" 7426,3,165,2017-01-10 11:48:31,http://leannon.org/jonathon.hammes,,231.109.183.27,"{""location"": ""CL"", ""is_mobile"": true}" 7427,3,165,2017-04-26 01:49:06,http://weber.org/lorenzo_kovacek,,138.99.80.196,"{""location"": ""ML"", ""is_mobile"": true}" 7428,3,165,2017-04-21 04:36:48,http://hayescollier.info/walter_dare,,135.208.26.173,"{""location"": ""PL"", ""is_mobile"": true}" 13284,5,299,2017-04-02 22:07:43,http://tillman.io/beryl,,236.61.90.107,"{""location"": ""GU"", ""is_mobile"": false}" 13285,5,299,2016-12-17 20:35:11,http://corkeryritchie.com/darrick.willms,,59.254.79.114,"{""location"": ""TR"", ""is_mobile"": true}" 13286,5,299,2017-04-06 05:38:05,http://mcclure.net/emmie.moore,,196.192.121.57,"{""location"": ""TH"", ""is_mobile"": false}" 13287,5,299,2017-06-04 13:32:37,http://rau.biz/haan,,58.168.204.203,"{""location"": ""CI"", ""is_mobile"": false}" 13288,5,299,2016-12-29 15:16:03,http://renner.com/marco.ankunding,,231.42.91.182,"{""location"": ""IQ"", ""is_mobile"": false}" 13289,5,299,2017-02-06 18:57:31,http://friesen.info/monica,,13.66.203.3,"{""location"": ""GE"", ""is_mobile"": true}" 13290,5,299,2017-03-23 15:52:20,http://heathcoteschultz.io/jeffry,,105.239.37.97,"{""location"": ""DM"", ""is_mobile"": false}" 13291,5,299,2017-02-14 15:18:02,http://whitemosciski.info/gene,,248.9.149.56,"{""location"": ""EC"", ""is_mobile"": false}" 13292,5,299,2016-12-27 23:57:06,http://vandervort.info/pamela_dickens,,148.216.81.254,"{""location"": ""GE"", ""is_mobile"": true}" 13293,5,299,2017-02-28 00:31:53,http://sawayn.org/verda_leuschke,,123.215.90.26,"{""location"": ""LV"", ""is_mobile"": true}" 13294,5,300,2016-12-31 22:51:43,http://donnellytrantow.biz/audrey,,146.216.178.144,"{""location"": ""TJ"", ""is_mobile"": false}" 13295,5,300,2016-12-31 05:27:02,http://ruel.com/benjamin_watsica,,82.91.188.47,"{""location"": ""UY"", ""is_mobile"": false}" 13296,5,300,2017-05-05 02:19:20,http://rippinbaumbach.name/heaven,,114.208.155.129,"{""location"": ""PE"", ""is_mobile"": true}" 13297,5,300,2017-01-04 21:36:42,http://fritsch.io/gilda.hayes,,180.164.31.64,"{""location"": ""WS"", ""is_mobile"": false}" 13298,5,300,2016-12-28 05:06:20,http://macejkovic.org/marietta_west,,34.14.24.165,"{""location"": ""NO"", ""is_mobile"": false}" 13299,5,300,2017-04-05 22:13:44,http://trantowbrekke.com/dashawn.goyette,,213.33.18.163,"{""location"": ""KM"", ""is_mobile"": true}" 13300,5,300,2017-05-31 10:25:17,http://toyzboncak.name/trey,,97.49.210.167,"{""location"": ""KI"", ""is_mobile"": false}" 13301,5,300,2017-05-22 01:22:27,http://champlin.co/dock_morar,,220.123.95.114,"{""location"": ""GD"", ""is_mobile"": false}" 13302,5,300,2017-05-12 13:37:43,http://ortiz.co/emory.bernhard,,59.164.14.191,"{""location"": ""AZ"", ""is_mobile"": true}" 13303,5,300,2017-04-10 14:46:41,http://collierbernier.io/yasmeen_hagenes,,171.128.25.193,"{""location"": ""UA"", ""is_mobile"": true}" 13304,5,300,2017-06-01 13:51:08,http://rempel.co/toby,,32.183.82.95,"{""location"": ""PN"", ""is_mobile"": false}" 13305,5,300,2017-05-08 15:31:10,http://aufderhar.info/amelie,,156.206.6.173,"{""location"": ""MT"", ""is_mobile"": false}" 13306,5,300,2016-12-15 10:13:06,http://ruecker.info/dawn.champlin,,234.206.233.166,"{""location"": ""SE"", ""is_mobile"": true}" 13307,5,300,2017-02-15 10:38:02,http://schmelerrempel.io/harley.deckow,,46.96.133.141,"{""location"": ""PK"", ""is_mobile"": true}" 13308,5,300,2017-04-27 21:16:41,http://keeblerrowe.net/grady.hayes,,249.211.98.105,"{""location"": ""UG"", ""is_mobile"": true}" 13309,5,300,2017-02-25 00:44:09,http://nitzsche.io/preston.beier,,101.55.142.175,"{""location"": ""CR"", ""is_mobile"": false}" 13310,5,300,2017-02-11 04:33:01,http://labadie.biz/rafael,,50.165.48.5,"{""location"": ""TL"", ""is_mobile"": false}" 13311,5,300,2017-04-11 21:45:54,http://schroeder.info/andre,,232.94.185.113,"{""location"": ""YT"", ""is_mobile"": true}" 13312,5,300,2017-02-28 21:08:20,http://schoenhintz.biz/pearlie,,210.118.125.212,"{""location"": ""GY"", ""is_mobile"": false}" 13313,5,300,2017-04-06 06:39:50,http://mann.com/karine.kirlin,,155.208.168.243,"{""location"": ""SI"", ""is_mobile"": true}" 13314,5,300,2017-05-03 12:02:44,http://boehmcarter.biz/antonina.mohr,,183.174.159.65,"{""location"": ""BE"", ""is_mobile"": false}" 13315,5,300,2017-01-26 18:53:00,http://beatty.org/elda.rohan,,22.122.49.159,"{""location"": ""GM"", ""is_mobile"": false}" 13316,5,300,2016-12-23 18:25:57,http://kirlin.com/orion,,179.46.245.152,"{""location"": ""BZ"", ""is_mobile"": false}" 13317,5,300,2017-03-16 12:46:01,http://king.net/mina_crist,,184.227.85.254,"{""location"": ""SH"", ""is_mobile"": false}" 13318,5,300,2017-05-09 04:19:31,http://dubuque.biz/jaron,,226.39.251.101,"{""location"": ""KI"", ""is_mobile"": true}" 13319,5,300,2016-12-29 08:20:50,http://dooley.co/hilbert,,51.160.133.163,"{""location"": ""CM"", ""is_mobile"": true}" 13320,5,300,2017-02-16 07:30:51,http://waelchi.org/kenton,,150.218.85.186,"{""location"": ""RS"", ""is_mobile"": true}" 13321,5,300,2017-01-07 16:21:56,http://fisher.com/geoffrey,,111.114.141.80,"{""location"": ""BN"", ""is_mobile"": true}" 13322,5,300,2017-05-19 11:32:54,http://gleichner.com/rocky,,4.114.127.112,"{""location"": ""TN"", ""is_mobile"": false}" 13323,5,300,2017-02-06 05:19:34,http://ritchiesawayn.info/conrad_oreilly,,118.174.113.248,"{""location"": ""NF"", ""is_mobile"": true}" 13324,5,300,2017-06-04 23:54:38,http://okeefe.net/yvonne.blanda,,223.204.123.248,"{""location"": ""BD"", ""is_mobile"": false}" 13325,5,300,2017-01-19 11:18:19,http://paucek.info/zakary.boyer,,59.31.76.22,"{""location"": ""GB"", ""is_mobile"": true}" 13326,5,300,2017-03-09 23:35:47,http://shieldsbayer.info/dayana.buckridge,,106.170.227.176,"{""location"": ""BQ"", ""is_mobile"": false}" 13327,5,300,2017-03-05 07:48:35,http://cain.net/jaylan,,179.93.178.207,"{""location"": ""GQ"", ""is_mobile"": true}" 13328,5,300,2017-05-13 13:14:19,http://osinski.org/koby.dickens,,21.156.89.215,"{""location"": ""BJ"", ""is_mobile"": false}" 13329,5,300,2017-06-13 02:25:39,http://yost.org/dana_cremin,,61.214.99.152,"{""location"": ""HM"", ""is_mobile"": true}" 13330,5,300,2017-04-01 17:47:30,http://kunde.name/vilma_goodwin,,27.189.24.24,"{""location"": ""RS"", ""is_mobile"": true}" 13331,5,300,2016-12-19 17:26:39,http://grant.info/megane.vonrueden,,93.123.216.46,"{""location"": ""TN"", ""is_mobile"": false}" 13332,5,300,2016-12-17 07:23:33,http://cronin.name/alejandra,,75.46.177.44,"{""location"": ""GW"", ""is_mobile"": false}" 13333,5,300,2017-02-13 07:00:26,http://leuschke.io/alison_zboncak,,35.90.178.90,"{""location"": ""SB"", ""is_mobile"": true}" 13334,5,300,2017-06-13 04:43:31,http://armstrong.org/kennith,,216.17.47.157,"{""location"": ""BE"", ""is_mobile"": false}" 13335,5,300,2017-02-11 06:20:36,http://okuneva.biz/winona,,139.87.209.140,"{""location"": ""UZ"", ""is_mobile"": false}" 13336,5,300,2017-05-29 05:16:47,http://wiegandbrown.io/johnny.tromp,,245.179.16.9,"{""location"": ""AT"", ""is_mobile"": false}" 13337,5,300,2017-04-01 20:08:22,http://croninfeil.io/laurianne,,102.194.220.122,"{""location"": ""BD"", ""is_mobile"": false}" 13338,5,300,2016-12-31 17:23:56,http://daugherty.co/lyda,,68.8.220.4,"{""location"": ""GA"", ""is_mobile"": true}" 16296,6,369,2017-02-04 16:41:26,http://ondrickadubuque.name/ola.dubuque,0.1082320092,254.153.64.167,"{""location"": ""VA"", ""is_mobile"": true}" 16297,6,369,2017-01-16 12:49:08,http://larkin.name/milo.torp,0.1235837877,150.48.41.74,"{""location"": ""IM"", ""is_mobile"": false}" 16298,6,369,2017-01-10 09:52:58,http://heaney.com/janis,0.4792710186,118.25.175.170,"{""location"": ""IT"", ""is_mobile"": false}" 16299,6,369,2017-04-11 21:29:28,http://klocko.org/nyasia_donnelly,0.0311419012,8.240.63.214,"{""location"": ""MR"", ""is_mobile"": false}" 16300,6,369,2016-12-31 12:28:00,http://ruecker.biz/amelia.doyle,0.2831392465,74.185.171.46,"{""location"": ""KW"", ""is_mobile"": true}" 16301,6,369,2017-02-25 04:57:04,http://boscowalter.com/reid,0.9391230313,168.180.127.181,"{""location"": ""PY"", ""is_mobile"": true}" 16302,6,369,2016-12-27 18:01:59,http://lebsack.net/nora,0.9364728851,161.186.182.150,"{""location"": ""BT"", ""is_mobile"": true}" 16303,6,370,2016-12-15 01:15:44,http://kreiger.biz/wallace_schulist,0.3823492868,129.199.101.124,"{""location"": ""KW"", ""is_mobile"": false}" 16304,6,370,2017-03-26 03:52:40,http://daniel.name/gerson.wolff,0.8348129382,40.213.209.186,"{""location"": ""MK"", ""is_mobile"": true}" 16305,6,370,2017-05-13 06:56:09,http://heaney.org/lila,0.0761149900,160.193.113.7,"{""location"": ""NA"", ""is_mobile"": false}" 16306,6,370,2017-06-11 17:15:39,http://cartwright.org/deangelo_gerlach,0.3020162453,148.55.138.190,"{""location"": ""LC"", ""is_mobile"": true}" 16307,6,370,2017-01-07 07:21:37,http://ondrickahegmann.co/dax.gutkowski,0.3639463700,136.91.137.92,"{""location"": ""SC"", ""is_mobile"": false}" 16308,6,370,2017-04-18 08:50:50,http://funk.info/keyon,0.7261787769,252.248.83.59,"{""location"": ""IL"", ""is_mobile"": false}" 16309,6,370,2017-02-17 10:16:44,http://tillmanblick.com/cynthia.frami,0.3151682452,91.133.146.118,"{""location"": ""IT"", ""is_mobile"": true}" 16310,6,370,2017-05-21 23:32:55,http://jerde.io/priscilla,0.2985300502,241.150.61.44,"{""location"": ""IR"", ""is_mobile"": true}" 16311,6,370,2017-02-25 20:39:59,http://mohr.io/zackery,0.2929681076,62.167.247.121,"{""location"": ""UG"", ""is_mobile"": true}" 16312,6,370,2017-05-26 03:48:59,http://lueilwitz.io/dejah,0.8435892980,250.136.212.238,"{""location"": ""NO"", ""is_mobile"": true}" 16313,6,370,2017-03-05 03:11:05,http://wisozkmertz.io/tomasa,0.2458996276,8.114.52.81,"{""location"": ""SE"", ""is_mobile"": true}" 16314,6,370,2017-05-21 08:12:34,http://dickinsonbauch.name/earnest.bednar,0.4948641724,40.91.177.144,"{""location"": ""VI"", ""is_mobile"": false}" 16315,6,370,2017-05-26 21:31:05,http://wiegand.com/andrew,0.2509022458,119.115.170.115,"{""location"": ""PR"", ""is_mobile"": true}" 16316,6,370,2017-06-09 15:55:10,http://rodriguez.info/milton.parker,0.8104413653,195.103.234.224,"{""location"": ""NR"", ""is_mobile"": false}" 16317,6,370,2017-05-06 07:34:58,http://jerde.net/nathanial,0.9148858402,81.109.183.91,"{""location"": ""HK"", ""is_mobile"": false}" 16318,6,370,2017-03-13 22:30:11,http://schiller.io/jalyn,0.7003597290,92.42.136.108,"{""location"": ""GN"", ""is_mobile"": false}" 16319,6,370,2017-02-15 01:45:07,http://goldnerhalvorson.biz/ivy_fadel,0.2055940152,77.69.57.14,"{""location"": ""VN"", ""is_mobile"": true}" 16320,6,370,2017-03-11 09:45:25,http://mohr.com/vivianne_spencer,0.6310179374,130.41.116.197,"{""location"": ""CU"", ""is_mobile"": true}" 16321,6,370,2017-01-22 15:29:50,http://powlowski.com/hayley_wiza,0.9230116362,231.7.250.179,"{""location"": ""SZ"", ""is_mobile"": true}" 16322,6,370,2017-01-21 01:55:39,http://conn.name/kaylee_runolfon,0.1749968831,240.107.126.73,"{""location"": ""MK"", ""is_mobile"": false}" 16323,6,370,2016-12-21 13:35:47,http://parkerrosenbaum.com/ladarius.johns,0.4304615760,133.110.247.5,"{""location"": ""TT"", ""is_mobile"": true}" 16324,6,370,2017-03-17 20:23:52,http://cain.biz/kristofer,0.4500653456,194.166.182.10,"{""location"": ""BI"", ""is_mobile"": false}" 16325,6,370,2017-04-19 06:07:12,http://hahncummings.co/sincere,0.3971096422,134.231.239.142,"{""location"": ""BW"", ""is_mobile"": true}" 16326,6,370,2017-02-21 22:15:42,http://bogan.io/dameon,0.3686123870,5.96.17.150,"{""location"": ""CH"", ""is_mobile"": false}" 16327,6,370,2016-12-13 08:35:44,http://grady.info/august_gottlieb,0.6555042974,31.187.176.101,"{""location"": ""MR"", ""is_mobile"": false}" 16328,6,370,2017-02-13 17:48:54,http://wuckertlegros.io/percival_rosenbaum,0.1657302147,174.144.133.13,"{""location"": ""PA"", ""is_mobile"": false}" 16329,6,370,2017-05-06 15:00:21,http://larsonsmith.org/maxwell.bechtelar,0.8835049836,73.131.23.152,"{""location"": ""IL"", ""is_mobile"": true}" 16330,6,370,2017-01-16 10:10:34,http://rippinziemann.net/wilfred_schneider,0.7559936338,66.158.68.81,"{""location"": ""PK"", ""is_mobile"": true}" 16331,6,370,2016-12-17 00:13:37,http://sanfordpaucek.co/candido_herzog,0.7124732749,121.163.204.228,"{""location"": ""NG"", ""is_mobile"": true}" 16332,6,370,2017-05-25 04:17:25,http://rohan.co/marie,0.7106250558,7.130.55.106,"{""location"": ""UZ"", ""is_mobile"": false}" 16333,6,370,2017-04-26 01:37:07,http://spencer.co/flavio,0.6823717638,237.65.98.138,"{""location"": ""IS"", ""is_mobile"": true}" 16334,6,370,2017-02-05 07:52:21,http://hand.co/verner.anderson,0.9383441181,94.188.57.158,"{""location"": ""CW"", ""is_mobile"": true}" 16335,6,370,2016-12-14 13:49:27,http://smithambednar.org/lila_hahn,0.5348912094,96.91.14.213,"{""location"": ""CR"", ""is_mobile"": true}" 16336,6,370,2017-02-03 15:11:04,http://gorczanygottlieb.net/jose.heathcote,0.5761590696,243.70.116.161,"{""location"": ""TT"", ""is_mobile"": true}" 16337,6,370,2017-04-05 16:32:23,http://adamsnikolaus.name/kurtis_lind,0.1596249044,116.125.229.32,"{""location"": ""SC"", ""is_mobile"": true}" 16338,6,370,2017-02-06 02:38:36,http://pfefferblick.co/darron_bergnaum,0.7311297591,159.132.171.207,"{""location"": ""VC"", ""is_mobile"": false}" 16339,6,370,2017-06-13 14:38:12,http://konopelski.name/lexi,0.2657541790,245.63.225.67,"{""location"": ""AX"", ""is_mobile"": true}" 16340,6,370,2017-05-17 12:22:06,http://sauer.biz/herminia,0.6733656989,83.159.11.75,"{""location"": ""DM"", ""is_mobile"": false}" 16341,6,370,2016-12-21 09:24:11,http://brekke.info/fidel,0.4242227500,241.106.43.138,"{""location"": ""MS"", ""is_mobile"": true}" 16342,6,370,2017-04-01 01:17:03,http://ruecker.net/trudie,0.6467240818,153.13.103.227,"{""location"": ""AU"", ""is_mobile"": false}" 16343,6,370,2017-02-19 22:19:07,http://labadiemaggio.co/meta_murray,0.5435278819,51.62.201.62,"{""location"": ""TD"", ""is_mobile"": true}" 16344,6,370,2017-06-01 02:54:28,http://lemke.info/alda,0.9784477979,11.195.16.49,"{""location"": ""SS"", ""is_mobile"": true}" 16345,6,370,2017-01-30 03:46:12,http://moriette.com/berniece.howell,0.7427948740,227.81.183.133,"{""location"": ""SA"", ""is_mobile"": false}" 16346,6,370,2017-01-06 23:18:21,http://buckridge.name/lorena,0.5354354988,35.90.64.111,"{""location"": ""BA"", ""is_mobile"": true}" 3474,2,76,2017-01-01 09:39:39,http://beahan.info/lexus,0.5073137971,125.88.117.251,"{""location"": ""AX"", ""is_mobile"": true}" 3475,2,76,2017-02-07 19:40:51,http://rutherford.com/helen,0.9146112184,155.140.150.41,"{""location"": ""NZ"", ""is_mobile"": false}" 3476,2,76,2016-12-18 17:20:09,http://kerluke.com/cecilia,0.4010907309,180.236.57.183,"{""location"": ""RU"", ""is_mobile"": false}" 3477,2,76,2017-04-22 22:44:20,http://kilback.io/hector,0.1334181834,29.37.251.159,"{""location"": ""PS"", ""is_mobile"": false}" 3478,2,76,2017-02-28 02:25:25,http://schinnertorp.io/rocky.kling,0.6045549905,107.195.136.143,"{""location"": ""TH"", ""is_mobile"": true}" 3479,2,76,2017-03-10 18:38:00,http://pfannerstill.name/oswald.hagenes,0.4082453107,183.238.115.188,"{""location"": ""CK"", ""is_mobile"": true}" 3480,2,76,2017-05-22 09:21:58,http://boehm.name/imani.wiegand,0.4740647269,6.219.178.66,"{""location"": ""GR"", ""is_mobile"": true}" 3481,2,76,2017-06-04 16:09:47,http://harber.info/immanuel,0.7213410551,228.145.130.158,"{""location"": ""GG"", ""is_mobile"": true}" 3482,2,76,2017-04-15 08:54:35,http://littlemertz.co/johathan,0.6995928355,179.2.142.51,"{""location"": ""PM"", ""is_mobile"": true}" 3483,2,76,2017-04-12 09:13:49,http://hammes.biz/maye,0.1710449997,174.227.110.134,"{""location"": ""BJ"", ""is_mobile"": false}" 3484,2,76,2017-01-21 09:26:04,http://ryanschowalter.info/alda_rolfson,0.4621735265,212.201.88.26,"{""location"": ""GP"", ""is_mobile"": false}" 3485,2,76,2017-05-27 03:42:11,http://wiegand.biz/kendall.okuneva,0.2661528277,113.249.3.81,"{""location"": ""GI"", ""is_mobile"": false}" 3486,2,77,2017-01-15 15:44:43,http://leuschkesmitham.org/kristopher.maggio,0.7607770825,7.179.197.8,"{""location"": ""CY"", ""is_mobile"": true}" 3487,2,77,2017-04-14 21:30:31,http://oconner.co/verona_kreiger,0.8388445286,74.172.202.118,"{""location"": ""MU"", ""is_mobile"": false}" 3488,2,77,2017-05-10 19:06:12,http://wuckert.biz/sigmund.christiansen,0.7766338310,9.244.228.30,"{""location"": ""BF"", ""is_mobile"": true}" 3489,2,77,2017-02-07 07:45:08,http://ward.net/brayan,0.6086022330,65.152.45.229,"{""location"": ""AG"", ""is_mobile"": false}" 3490,2,77,2017-01-05 21:54:46,http://connelly.com/alyce,0.6389652674,216.201.20.182,"{""location"": ""NR"", ""is_mobile"": true}" 3491,2,77,2017-05-11 07:56:48,http://keebler.name/wellington.lebsack,0.3654820078,242.35.238.135,"{""location"": ""MX"", ""is_mobile"": true}" 3492,2,77,2017-05-23 17:28:17,http://koleffler.net/sebastian_turcotte,0.8544604177,37.43.21.31,"{""location"": ""KP"", ""is_mobile"": true}" 3493,2,77,2017-01-21 03:09:22,http://reichert.name/rickie,0.0783224145,78.89.104.213,"{""location"": ""TG"", ""is_mobile"": false}" 3494,2,77,2016-12-15 10:42:55,http://wildermanpadberg.com/bettie,0.5106544425,109.239.31.146,"{""location"": ""MQ"", ""is_mobile"": true}" 3495,2,77,2017-01-21 18:40:08,http://zieme.co/ona_langworth,0.2503709905,177.107.31.32,"{""location"": ""SR"", ""is_mobile"": true}" 3496,2,77,2017-01-02 15:38:24,http://rice.co/geovanni_herman,0.5178194873,224.134.48.131,"{""location"": ""MN"", ""is_mobile"": false}" 3497,2,77,2017-01-18 14:13:42,http://vandervortlowe.org/annamarie,0.0138910584,120.95.245.239,"{""location"": ""DE"", ""is_mobile"": true}" 3498,2,77,2017-05-01 07:23:02,http://baumbach.org/jeramie,0.5862554567,239.74.37.213,"{""location"": ""LV"", ""is_mobile"": false}" 3499,2,77,2017-01-26 00:44:12,http://powlowskibogisich.io/liana.ebert,0.3963297344,227.112.35.97,"{""location"": ""WS"", ""is_mobile"": false}" 3500,2,77,2017-05-26 15:30:42,http://fay.io/nelda,0.4356159077,214.189.54.210,"{""location"": ""FI"", ""is_mobile"": true}" 3501,2,77,2017-02-20 17:40:36,http://davisjast.org/chandler_schaden,0.4592299522,250.215.205.112,"{""location"": ""PA"", ""is_mobile"": false}" 3502,2,77,2017-02-02 22:35:16,http://davis.io/tierra,0.7544524170,222.217.196.181,"{""location"": ""MA"", ""is_mobile"": false}" 3503,2,77,2017-06-08 15:30:21,http://collins.name/vincenza,0.9922112984,14.247.226.13,"{""location"": ""BF"", ""is_mobile"": false}" 3504,2,77,2017-02-21 22:46:36,http://mclaughlin.biz/kayli.cronin,0.2454114312,179.17.134.144,"{""location"": ""GN"", ""is_mobile"": true}" 3505,2,77,2017-06-13 15:28:20,http://hermann.com/tyson.hartmann,0.6165013451,3.181.224.198,"{""location"": ""FI"", ""is_mobile"": true}" 3506,2,77,2017-05-14 03:28:16,http://bogan.com/isabella,0.1140352509,47.94.170.89,"{""location"": ""MM"", ""is_mobile"": true}" 3507,2,77,2017-01-02 17:04:00,http://cremin.org/esta,0.5577890246,218.217.26.231,"{""location"": ""BA"", ""is_mobile"": false}" 3508,2,77,2017-03-24 14:53:20,http://cain.io/burley,0.9701615874,205.40.30.246,"{""location"": ""RW"", ""is_mobile"": true}" 3509,2,77,2016-12-30 00:21:20,http://lemke.biz/annamarie_gleichner,0.3712402170,64.158.60.235,"{""location"": ""SO"", ""is_mobile"": true}" 3510,2,77,2017-05-11 23:40:33,http://purdystehr.net/isaias,0.0244238508,221.186.64.15,"{""location"": ""MH"", ""is_mobile"": false}" 3511,2,77,2017-04-16 06:56:15,http://moriette.org/watson_gislason,0.0312024572,239.196.168.244,"{""location"": ""TZ"", ""is_mobile"": false}" 3512,2,77,2017-03-23 22:31:44,http://weinat.biz/cordelia_rippin,0.4258899193,157.199.53.135,"{""location"": ""MC"", ""is_mobile"": true}" 3513,2,77,2017-05-08 09:41:01,http://ruecker.name/paul,0.7453022127,208.107.128.129,"{""location"": ""BD"", ""is_mobile"": true}" 3514,2,77,2017-02-21 04:54:59,http://schmeler.name/kenny_lockman,0.9523867887,159.47.175.251,"{""location"": ""SZ"", ""is_mobile"": false}" 3515,2,77,2017-03-24 11:06:54,http://smithcummings.org/shanny,0.0989161825,28.227.225.155,"{""location"": ""WF"", ""is_mobile"": true}" 3516,2,77,2017-03-15 10:07:18,http://jaskolskischumm.name/braeden,0.3120749362,160.179.212.168,"{""location"": ""KG"", ""is_mobile"": false}" 3517,2,77,2017-03-26 23:12:20,http://skilesgulgowski.io/hubert_douglas,0.8290865725,157.79.194.14,"{""location"": ""BD"", ""is_mobile"": true}" 3518,2,77,2017-05-12 03:30:55,http://cruickshankturner.co/nathanial.green,0.9614707468,207.136.59.5,"{""location"": ""RU"", ""is_mobile"": false}" 3519,2,77,2017-01-28 04:05:52,http://hettinger.io/mayra.trantow,0.8469165715,41.239.99.15,"{""location"": ""AI"", ""is_mobile"": false}" 3520,2,77,2017-03-25 18:15:37,http://ward.biz/stevie,0.1751824372,212.141.137.181,"{""location"": ""HU"", ""is_mobile"": true}" 3521,2,77,2017-05-06 01:13:18,http://kuhlman.info/jenifer,0.9143773921,39.191.246.108,"{""location"": ""GY"", ""is_mobile"": true}" 3522,2,77,2017-01-03 11:07:16,http://langoshbeahan.net/myrtie.smitham,0.7006306102,106.88.65.104,"{""location"": ""TN"", ""is_mobile"": false}" 3523,2,77,2017-02-25 11:13:02,http://schinner.io/jada.hegmann,0.5656580614,190.144.244.207,"{""location"": ""AG"", ""is_mobile"": true}" 3524,2,77,2017-02-02 10:13:13,http://mooreokon.name/waldo,0.8967128783,204.150.200.95,"{""location"": ""GR"", ""is_mobile"": true}" 10404,4,234,2017-05-07 22:39:51,http://reicheljohns.info/jamel_oreilly,0.5688983177,233.94.20.173,"{""location"": ""VU"", ""is_mobile"": false}" 10405,4,234,2017-06-02 08:32:39,http://stokes.org/ro,0.7911434364,230.67.50.111,"{""location"": ""FI"", ""is_mobile"": true}" 10406,4,234,2017-05-13 19:00:32,http://turcottereichel.org/antonette_barton,0.9668983179,182.246.149.13,"{""location"": ""GH"", ""is_mobile"": true}" 10407,4,234,2017-01-25 12:43:40,http://nienowdeckow.io/murphy.stanton,0.9821918100,16.81.104.185,"{""location"": ""MF"", ""is_mobile"": false}" 10408,4,234,2017-04-09 14:54:02,http://wolff.org/paolo_batz,0.7092666670,5.208.16.230,"{""location"": ""NA"", ""is_mobile"": true}" 10409,4,234,2017-05-27 07:50:21,http://bernhard.com/niko,0.9850529654,5.135.161.237,"{""location"": ""ML"", ""is_mobile"": true}" 10410,4,234,2017-01-29 05:02:46,http://lind.io/domenick,0.0171860910,35.13.22.253,"{""location"": ""CD"", ""is_mobile"": false}" 10411,4,234,2017-04-26 17:38:06,http://sipes.com/zaria.considine,0.7116760851,186.176.233.251,"{""location"": ""MM"", ""is_mobile"": true}" 10412,4,234,2017-02-18 07:57:32,http://windlerhoeger.name/mac,0.1197273722,231.9.70.244,"{""location"": ""JM"", ""is_mobile"": false}" 10413,4,234,2017-02-03 19:14:25,http://stamm.co/sydnee_hahn,0.0789239169,139.42.154.27,"{""location"": ""BL"", ""is_mobile"": true}" 10414,4,234,2017-05-22 08:06:32,http://eichmannklein.co/alana_feest,0.7110976753,190.38.22.92,"{""location"": ""SR"", ""is_mobile"": false}" 10415,4,234,2017-04-28 04:14:53,http://schuster.net/billie_emmerich,0.7180308734,88.89.2.31,"{""location"": ""PK"", ""is_mobile"": false}" 10416,4,234,2017-06-02 02:38:13,http://skiles.net/danielle,0.2278176787,74.253.87.135,"{""location"": ""AZ"", ""is_mobile"": true}" 10417,4,234,2017-05-21 03:37:04,http://erdman.co/vena,0.3321244917,131.113.139.201,"{""location"": ""AM"", ""is_mobile"": false}" 10418,4,234,2017-04-15 11:04:58,http://skilehanahan.io/devonte_parker,0.0023951782,177.230.104.184,"{""location"": ""NU"", ""is_mobile"": true}" 10419,4,234,2017-03-03 01:47:54,http://mitchell.org/rogers,0.8425463117,174.208.240.55,"{""location"": ""AF"", ""is_mobile"": true}" 10420,4,234,2017-06-09 01:24:45,http://murraylowe.com/melia_cartwright,0.2395018968,138.131.52.179,"{""location"": ""IS"", ""is_mobile"": false}" 10421,4,234,2017-05-08 04:02:57,http://cainblanda.net/olen_wilderman,0.5524719408,110.183.168.3,"{""location"": ""TO"", ""is_mobile"": true}" 10422,4,234,2017-02-12 06:40:26,http://lind.biz/kathryne,0.6743823322,239.158.182.177,"{""location"": ""GA"", ""is_mobile"": false}" 10423,4,234,2017-05-09 07:56:25,http://abernathyhane.net/gonzalo_goyette,0.3590141637,17.213.86.165,"{""location"": ""CA"", ""is_mobile"": false}" 10424,4,234,2017-04-10 18:21:34,http://ferry.co/bettie,0.2723317347,169.248.209.241,"{""location"": ""YT"", ""is_mobile"": true}" 10425,4,234,2017-01-10 09:11:40,http://breitenberghilpert.biz/gerson,0.1667641882,242.251.2.248,"{""location"": ""GF"", ""is_mobile"": true}" 10426,4,234,2017-01-16 14:24:26,http://dubuqueschroeder.com/antwan.collier,0.9669955526,97.141.210.160,"{""location"": ""FK"", ""is_mobile"": false}" 10427,4,234,2017-03-25 01:16:09,http://monahan.info/monty,0.9547918394,182.248.80.8,"{""location"": ""SN"", ""is_mobile"": true}" 10428,4,234,2017-02-01 18:55:30,http://haag.name/albert,0.1954546776,14.167.35.203,"{""location"": ""MQ"", ""is_mobile"": true}" 10429,4,235,2017-02-05 16:00:37,http://lynch.net/toni_oreilly,0.8463385411,149.127.132.51,"{""location"": ""LB"", ""is_mobile"": true}" 10430,4,235,2017-04-21 13:31:32,http://murazik.net/krystel,0.3764827972,246.223.228.55,"{""location"": ""IS"", ""is_mobile"": false}" 10431,4,235,2017-02-20 19:22:17,http://abshiregusikowski.com/alba_dicki,0.5848952225,121.214.66.253,"{""location"": ""VG"", ""is_mobile"": false}" 10432,4,235,2016-12-30 14:41:00,http://pagac.org/nestor,0.8598227959,175.4.77.30,"{""location"": ""GU"", ""is_mobile"": false}" 10433,4,235,2017-06-08 19:35:38,http://parker.name/norbert_collins,0.6612779253,231.119.36.223,"{""location"": ""HT"", ""is_mobile"": true}" 10434,4,235,2017-01-15 17:26:38,http://jacobideckow.name/terry,0.0320799136,33.124.201.137,"{""location"": ""NU"", ""is_mobile"": true}" 10435,4,235,2017-02-24 14:27:13,http://windler.co/jules,0.8004334986,221.80.235.66,"{""location"": ""AU"", ""is_mobile"": true}" 10436,4,235,2017-02-24 13:06:04,http://hackett.net/hallie_fahey,0.6497129494,174.72.201.138,"{""location"": ""IL"", ""is_mobile"": false}" 10437,4,235,2017-04-30 07:03:31,http://armstrong.com/whitney.bahringer,0.7932158146,115.40.70.171,"{""location"": ""GD"", ""is_mobile"": true}" 10438,4,235,2017-05-14 12:12:04,http://treutel.co/mayra,0.2869364412,63.71.98.67,"{""location"": ""CH"", ""is_mobile"": true}" 10439,4,235,2017-01-13 02:50:52,http://reilly.io/belle.mante,0.0458380195,19.158.14.62,"{""location"": ""SL"", ""is_mobile"": true}" 10440,4,235,2016-12-25 07:14:48,http://robelschaden.net/josie,0.0107134376,83.123.29.211,"{""location"": ""FI"", ""is_mobile"": false}" 10441,4,235,2017-04-27 03:31:01,http://mckenzie.info/bart,0.9334165177,86.43.43.164,"{""location"": ""CY"", ""is_mobile"": false}" 10442,4,235,2016-12-20 19:19:14,http://kris.co/brad,0.1977877267,112.94.156.208,"{""location"": ""CY"", ""is_mobile"": true}" 10443,4,235,2016-12-29 22:07:01,http://schimmel.info/gabriella,0.9680778849,51.225.56.243,"{""location"": ""AG"", ""is_mobile"": false}" 10444,4,235,2017-04-14 02:18:48,http://collier.info/christina,0.4340239801,11.221.30.182,"{""location"": ""IR"", ""is_mobile"": true}" 10445,4,235,2017-04-14 20:33:28,http://powlowski.co/miguel,0.7185863782,64.125.162.102,"{""location"": ""BI"", ""is_mobile"": true}" 10446,4,235,2017-02-16 00:37:43,http://hamill.co/ellsworth_zulauf,0.7735244505,210.17.15.88,"{""location"": ""TH"", ""is_mobile"": false}" 10447,4,235,2017-04-26 04:28:56,http://doyle.net/maegan,0.4558510314,26.186.216.47,"{""location"": ""UG"", ""is_mobile"": false}" 10448,4,235,2017-01-18 03:45:14,http://johnson.info/justen,0.4837986418,62.46.125.242,"{""location"": ""BB"", ""is_mobile"": true}" 10449,4,235,2017-04-18 23:42:37,http://gulgowski.com/ryann_hilpert,0.7050118694,74.32.185.166,"{""location"": ""DE"", ""is_mobile"": true}" 10450,4,235,2017-03-10 19:46:57,http://nolan.info/gardner,0.8341803926,36.169.230.32,"{""location"": ""SC"", ""is_mobile"": false}" 10451,4,235,2017-01-26 07:11:38,http://howebecker.org/laurianne,0.3446242048,249.99.159.51,"{""location"": ""BW"", ""is_mobile"": true}" 10452,4,235,2017-02-28 08:33:06,http://wisozkfeest.org/hellen_rogahn,0.3186207922,195.21.99.160,"{""location"": ""IM"", ""is_mobile"": false}" 10453,4,235,2016-12-14 00:37:22,http://cummingswalter.name/albert,0.0930160052,114.230.232.190,"{""location"": ""ST"", ""is_mobile"": false}" 10454,4,235,2017-06-04 22:04:27,http://moriettegottlieb.name/rafaela.mcglynn,0.8687719195,253.232.58.110,"{""location"": ""NI"", ""is_mobile"": true}" 7429,3,165,2017-01-19 00:23:25,http://boyerpfeffer.com/brooklyn.paucek,,147.129.221.72,"{""location"": ""YT"", ""is_mobile"": false}" 7430,3,165,2017-03-27 02:50:20,http://goldner.name/stacy.bechtelar,,120.232.50.37,"{""location"": ""BL"", ""is_mobile"": false}" 7431,3,165,2017-05-09 22:24:45,http://weinatrohan.io/grace,,66.57.115.208,"{""location"": ""ME"", ""is_mobile"": false}" 7432,3,165,2017-03-01 15:22:16,http://shanahangibson.net/katherine,,251.248.128.42,"{""location"": ""GD"", ""is_mobile"": false}" 7433,3,165,2017-01-05 06:49:48,http://deckow.com/waino,,171.182.144.239,"{""location"": ""MD"", ""is_mobile"": true}" 7434,3,165,2017-03-24 23:53:23,http://konopelskiruecker.biz/anika,,84.75.228.221,"{""location"": ""UY"", ""is_mobile"": false}" 7435,3,165,2017-03-26 11:51:41,http://greenholt.co/hosea.waters,,86.238.154.34,"{""location"": ""MF"", ""is_mobile"": true}" 7436,3,165,2017-06-02 04:05:30,http://pacochakoepp.info/alejandra,,114.42.156.7,"{""location"": ""AR"", ""is_mobile"": false}" 7437,3,165,2016-12-25 21:55:36,http://kleinschimmel.info/sabrina.sanford,,118.120.223.11,"{""location"": ""NA"", ""is_mobile"": false}" 7438,3,165,2017-05-24 13:10:59,http://runolfsdottirokuneva.com/delphia,,229.150.50.36,"{""location"": ""KY"", ""is_mobile"": true}" 7439,3,165,2017-06-05 13:51:17,http://lindgren.co/enoch,,104.11.151.6,"{""location"": ""KE"", ""is_mobile"": false}" 7440,3,165,2017-05-24 21:57:06,http://streichherman.io/jimmy,,36.105.180.83,"{""location"": ""KE"", ""is_mobile"": true}" 7441,3,165,2017-01-27 16:50:18,http://abbott.com/clint,,122.36.244.17,"{""location"": ""UM"", ""is_mobile"": true}" 7442,3,165,2017-04-11 20:40:41,http://kertzmann.name/hipolito.runolfsdottir,,92.55.172.98,"{""location"": ""UZ"", ""is_mobile"": false}" 7443,3,165,2017-04-20 22:04:41,http://robertshettinger.biz/astrid.metz,,43.74.174.178,"{""location"": ""MZ"", ""is_mobile"": false}" 7444,3,165,2017-01-02 20:03:23,http://schimmel.biz/renee_stokes,,48.12.239.78,"{""location"": ""GR"", ""is_mobile"": true}" 7445,3,165,2017-01-06 12:24:48,http://mitchelllang.org/philip_collier,,88.161.57.125,"{""location"": ""SB"", ""is_mobile"": true}" 7446,3,165,2017-01-18 23:51:52,http://stiedemann.net/burnice_cormier,,99.204.121.117,"{""location"": ""TK"", ""is_mobile"": false}" 7447,3,165,2017-03-02 20:41:57,http://hamill.info/emmalee,,41.150.251.28,"{""location"": ""AF"", ""is_mobile"": true}" 7448,3,165,2017-03-18 20:58:55,http://emard.name/kendall_halvorson,,178.112.224.240,"{""location"": ""RW"", ""is_mobile"": false}" 7449,3,165,2016-12-29 18:58:09,http://jerde.org/marlon,,179.161.147.217,"{""location"": ""MK"", ""is_mobile"": false}" 7450,3,165,2017-05-19 08:44:01,http://mayer.com/maiya,,115.143.104.79,"{""location"": ""SG"", ""is_mobile"": false}" 7451,3,165,2017-05-15 19:36:14,http://weinatschamberger.co/valentina,,181.147.47.238,"{""location"": ""BW"", ""is_mobile"": false}" 7452,3,165,2017-05-30 07:16:10,http://trompbecker.com/madilyn_barrows,,134.83.129.211,"{""location"": ""AI"", ""is_mobile"": true}" 7453,3,165,2016-12-29 11:30:51,http://rosenbaum.com/ahmed,,197.27.173.173,"{""location"": ""AE"", ""is_mobile"": true}" 7454,3,165,2017-06-13 09:02:56,http://jacobiprohaska.info/kiarra,,72.77.63.192,"{""location"": ""NA"", ""is_mobile"": false}" 7455,3,165,2017-01-24 13:11:57,http://lockman.co/ashley_erdman,,156.181.175.116,"{""location"": ""NE"", ""is_mobile"": true}" 7456,3,165,2017-03-17 20:58:32,http://eichmanncrona.name/leanna.jacobi,,185.114.251.4,"{""location"": ""MU"", ""is_mobile"": true}" 7457,3,165,2017-04-11 02:58:06,http://hoppe.name/rogers,,188.233.90.228,"{""location"": ""IO"", ""is_mobile"": false}" 7458,3,165,2017-03-28 15:38:08,http://herzoghermann.name/carlo,,184.77.162.159,"{""location"": ""BY"", ""is_mobile"": true}" 7459,3,165,2017-05-24 14:44:46,http://ritchie.org/rasheed,,165.222.202.175,"{""location"": ""AU"", ""is_mobile"": false}" 7460,3,165,2017-06-11 12:24:47,http://friesenerdman.net/marianna,,196.83.253.183,"{""location"": ""UG"", ""is_mobile"": false}" 7461,3,165,2017-01-04 07:11:17,http://prohaska.org/jabari_feeney,,57.119.34.239,"{""location"": ""VI"", ""is_mobile"": false}" 7462,3,165,2016-12-14 21:01:22,http://kerlukestamm.info/adolf,,109.230.102.157,"{""location"": ""GR"", ""is_mobile"": false}" 7463,3,165,2017-03-03 09:48:19,http://schroeder.org/colten,,78.131.156.52,"{""location"": ""AU"", ""is_mobile"": true}" 7464,3,165,2017-05-25 03:53:49,http://spinkawilliamson.info/jed.rutherford,,161.119.238.233,"{""location"": ""BB"", ""is_mobile"": false}" 7465,3,165,2017-03-25 11:44:52,http://barrows.co/wilber,,164.185.21.232,"{""location"": ""BF"", ""is_mobile"": false}" 7466,3,165,2017-04-25 18:58:17,http://schinnerokon.biz/shanna,,244.102.176.246,"{""location"": ""GQ"", ""is_mobile"": false}" 7467,3,165,2017-03-31 21:42:35,http://hermannharvey.name/ervin_weber,,79.23.200.53,"{""location"": ""AQ"", ""is_mobile"": false}" 7468,3,165,2017-01-23 16:49:45,http://christiansen.com/kacie,,120.205.147.44,"{""location"": ""NE"", ""is_mobile"": true}" 7469,3,165,2017-06-04 07:56:07,http://mcglynn.name/terrence,,37.7.185.151,"{""location"": ""SH"", ""is_mobile"": false}" 7470,3,165,2017-05-06 22:26:02,http://haleyrobel.com/karen.rohan,,250.143.250.186,"{""location"": ""TG"", ""is_mobile"": true}" 7471,3,165,2017-04-25 00:15:12,http://hirthe.org/perry_hauck,,162.151.173.241,"{""location"": ""ID"", ""is_mobile"": false}" 7472,3,165,2017-02-05 16:01:25,http://grimes.io/bryana,,102.185.8.215,"{""location"": ""IS"", ""is_mobile"": true}" 7473,3,165,2017-02-11 10:55:50,http://connsawayn.com/lamont.satterfield,,232.161.134.192,"{""location"": ""MK"", ""is_mobile"": false}" 7474,3,165,2017-01-09 09:47:29,http://mayert.net/jimmie.stroman,,118.156.160.155,"{""location"": ""MK"", ""is_mobile"": true}" 7475,3,165,2017-01-07 16:31:54,http://bednar.com/bonita,,56.12.154.166,"{""location"": ""GB"", ""is_mobile"": true}" 7476,3,165,2016-12-31 17:51:52,http://kulaswelch.name/lucinda_mcglynn,,89.65.137.103,"{""location"": ""GR"", ""is_mobile"": false}" 7477,3,165,2017-01-01 12:53:18,http://green.io/baby,,163.114.193.225,"{""location"": ""BR"", ""is_mobile"": true}" 7478,3,165,2017-03-03 07:36:51,http://mohr.org/creola_pagac,,126.147.104.230,"{""location"": ""IR"", ""is_mobile"": true}" 7479,3,165,2017-05-24 16:48:46,http://mohr.name/jordyn,,71.10.207.97,"{""location"": ""IR"", ""is_mobile"": true}" 7480,3,165,2017-03-23 03:09:08,http://gerholdbeer.org/laverna,,130.77.216.212,"{""location"": ""LA"", ""is_mobile"": false}" 7481,3,165,2017-01-13 11:17:31,http://watsicarau.net/deon,,222.76.167.147,"{""location"": ""BA"", ""is_mobile"": false}" 7482,3,166,2017-01-28 00:11:44,http://parisian.com/reinhold,,147.187.162.115,"{""location"": ""NI"", ""is_mobile"": false}" 7483,3,166,2017-05-13 20:39:05,http://prosacco.co/kattie,,86.147.250.116,"{""location"": ""LS"", ""is_mobile"": true}" 13339,5,300,2017-02-26 16:16:44,http://metz.biz/kirsten,,17.200.91.209,"{""location"": ""SE"", ""is_mobile"": true}" 13340,5,300,2017-04-20 18:15:47,http://hyattrodriguez.co/gayle,,237.68.253.55,"{""location"": ""ME"", ""is_mobile"": false}" 13341,5,300,2017-03-03 22:27:23,http://ziemann.biz/kiarra,,136.231.20.114,"{""location"": ""SZ"", ""is_mobile"": false}" 13342,5,300,2017-03-09 16:53:12,http://klein.io/emelia,,55.52.166.132,"{""location"": ""DM"", ""is_mobile"": true}" 13343,5,300,2017-02-13 21:16:13,http://pourosrunolfsdottir.io/dolly,,105.17.82.102,"{""location"": ""GD"", ""is_mobile"": true}" 13344,5,300,2017-04-30 23:29:35,http://johnson.org/stella,,31.187.189.152,"{""location"": ""VI"", ""is_mobile"": true}" 13345,5,300,2017-05-25 22:52:48,http://wardjohns.io/kamron,,33.122.9.81,"{""location"": ""VG"", ""is_mobile"": true}" 13346,5,300,2016-12-19 20:27:44,http://barrows.name/alejandra_padberg,,189.114.137.118,"{""location"": ""SC"", ""is_mobile"": false}" 13347,5,300,2017-01-12 15:05:16,http://renner.co/loraine,,112.23.124.112,"{""location"": ""DO"", ""is_mobile"": false}" 13348,5,300,2017-06-05 20:20:42,http://carter.biz/jevon,,87.98.213.91,"{""location"": ""EG"", ""is_mobile"": true}" 13349,5,300,2016-12-23 07:43:08,http://oconnelllarkin.name/peggie,,157.8.31.198,"{""location"": ""ID"", ""is_mobile"": false}" 13350,5,300,2017-05-08 09:41:00,http://tromp.com/wendy_lockman,,121.253.38.13,"{""location"": ""KI"", ""is_mobile"": false}" 13351,5,301,2017-02-04 06:47:09,http://keelinglind.biz/nikko,,152.238.51.36,"{""location"": ""VE"", ""is_mobile"": false}" 13352,5,301,2017-03-12 19:45:06,http://auer.co/reece.roob,,125.98.169.120,"{""location"": ""SD"", ""is_mobile"": true}" 13353,5,301,2017-02-26 08:50:42,http://schamberger.biz/leonard,,95.78.118.89,"{""location"": ""KW"", ""is_mobile"": false}" 13354,5,301,2017-02-03 23:20:40,http://conroy.info/cortez.goodwin,,170.47.68.126,"{""location"": ""HT"", ""is_mobile"": false}" 13355,5,301,2016-12-27 15:25:36,http://mills.net/lorenz,,241.198.125.180,"{""location"": ""BZ"", ""is_mobile"": true}" 13356,5,301,2017-02-05 00:36:29,http://fadelmorar.org/devonte,,46.120.105.35,"{""location"": ""NA"", ""is_mobile"": true}" 13357,5,301,2017-01-02 17:00:22,http://ratke.biz/cleveland,,139.10.232.168,"{""location"": ""ES"", ""is_mobile"": false}" 13358,5,301,2017-01-03 17:02:32,http://sipes.com/jacquelyn.klein,,189.58.226.122,"{""location"": ""AI"", ""is_mobile"": false}" 13359,5,301,2017-01-11 21:48:30,http://brauntorphy.io/loren,,237.229.79.68,"{""location"": ""PH"", ""is_mobile"": false}" 13360,5,301,2017-04-17 12:42:54,http://little.com/ernestine,,107.197.189.98,"{""location"": ""SG"", ""is_mobile"": false}" 13361,5,301,2016-12-22 00:18:16,http://jerdedibbert.net/isaias,,236.142.57.245,"{""location"": ""CN"", ""is_mobile"": true}" 13362,5,301,2017-03-27 05:45:37,http://nikolaus.co/marquise,,206.250.76.84,"{""location"": ""JE"", ""is_mobile"": true}" 13363,5,301,2017-05-31 08:18:27,http://durgan.io/rowan_hirthe,,73.88.183.19,"{""location"": ""LA"", ""is_mobile"": true}" 13364,5,301,2017-03-08 18:50:24,http://reichelyost.io/giovanny.little,,90.205.247.202,"{""location"": ""AM"", ""is_mobile"": false}" 13365,5,301,2017-03-19 11:57:03,http://rosenbaumleffler.biz/olen,,218.223.55.129,"{""location"": ""DK"", ""is_mobile"": true}" 13366,5,301,2017-03-30 12:19:37,http://weinat.info/reyna,,53.47.134.55,"{""location"": ""IR"", ""is_mobile"": true}" 13367,5,301,2017-01-30 22:14:01,http://tillman.biz/karine,,52.64.160.222,"{""location"": ""VC"", ""is_mobile"": true}" 13368,5,301,2017-02-17 13:28:54,http://boehmjacobs.io/yeenia.barrows,,2.190.195.54,"{""location"": ""GY"", ""is_mobile"": false}" 13369,5,301,2017-04-25 11:09:28,http://turcottehilll.co/anika.rowe,,117.90.152.194,"{""location"": ""LI"", ""is_mobile"": false}" 13370,5,301,2017-06-10 17:05:35,http://white.co/alfreda,,24.83.53.102,"{""location"": ""MS"", ""is_mobile"": false}" 13371,5,301,2016-12-16 14:29:04,http://flatleychristiansen.io/jonathan,,65.203.102.168,"{""location"": ""SJ"", ""is_mobile"": false}" 13372,5,301,2016-12-31 18:20:42,http://stokesullrich.com/leanne,,240.150.35.229,"{""location"": ""CA"", ""is_mobile"": true}" 13373,5,301,2017-05-21 15:50:58,http://rau.co/lawrence,,67.161.31.249,"{""location"": ""VE"", ""is_mobile"": false}" 13374,5,301,2016-12-29 04:51:26,http://waters.co/americo,,171.8.67.243,"{""location"": ""GA"", ""is_mobile"": true}" 13375,5,301,2017-01-23 01:50:28,http://streichkoch.co/chanel_pollich,,31.94.30.11,"{""location"": ""KR"", ""is_mobile"": false}" 13376,5,301,2017-01-09 10:32:41,http://gulgowski.net/samson_lynch,,147.115.189.153,"{""location"": ""ER"", ""is_mobile"": false}" 13377,5,301,2017-02-16 14:27:55,http://torphy.info/casey,,55.120.205.233,"{""location"": ""MU"", ""is_mobile"": true}" 13378,5,301,2017-01-29 14:04:01,http://brownoberbrunner.org/mia,,150.189.17.149,"{""location"": ""TM"", ""is_mobile"": true}" 13379,5,301,2017-01-22 04:22:58,http://rogahn.com/theodora_schowalter,,118.55.246.109,"{""location"": ""SY"", ""is_mobile"": true}" 13380,5,301,2017-04-05 09:49:59,http://heathcote.co/lillian.beier,,199.71.97.222,"{""location"": ""IM"", ""is_mobile"": false}" 13381,5,301,2017-01-24 08:39:21,http://greenholt.net/kathryn,,155.79.34.5,"{""location"": ""QA"", ""is_mobile"": false}" 13382,5,301,2017-02-26 07:48:36,http://greenholtweimann.name/krista,,229.48.203.181,"{""location"": ""CA"", ""is_mobile"": true}" 13383,5,301,2017-04-29 15:15:21,http://raustrosin.org/jordan_torphy,,14.9.219.148,"{""location"": ""DM"", ""is_mobile"": false}" 13384,5,301,2017-05-31 11:39:14,http://wisozk.biz/benjamin,,88.56.173.205,"{""location"": ""EH"", ""is_mobile"": false}" 13385,5,301,2017-02-09 01:55:06,http://ebert.com/raphaelle,,20.162.192.212,"{""location"": ""BE"", ""is_mobile"": true}" 13386,5,301,2017-02-23 04:10:49,http://kundehyatt.com/nathanael_yost,,171.85.131.208,"{""location"": ""BE"", ""is_mobile"": false}" 13387,5,301,2017-02-07 08:34:25,http://schinner.io/keith.smitham,,191.140.44.87,"{""location"": ""LA"", ""is_mobile"": true}" 13388,5,301,2017-03-29 17:41:25,http://hermann.co/patience_bernhard,,104.85.118.11,"{""location"": ""ZW"", ""is_mobile"": true}" 13389,5,301,2017-01-05 22:00:48,http://colemraz.net/laverna.kuhn,,111.141.28.253,"{""location"": ""MG"", ""is_mobile"": false}" 13390,5,301,2017-01-27 20:46:04,http://schusterconroy.name/fanny_rolfson,,6.45.212.205,"{""location"": ""ZW"", ""is_mobile"": true}" 13391,5,301,2017-03-07 17:00:44,http://mcdermott.name/rocio,,246.33.7.162,"{""location"": ""FO"", ""is_mobile"": true}" 13392,5,301,2017-03-03 13:34:52,http://miller.net/leone,,128.128.135.49,"{""location"": ""IQ"", ""is_mobile"": true}" 13393,5,301,2017-06-08 04:47:36,http://berge.io/fritz,,164.204.232.52,"{""location"": ""SM"", ""is_mobile"": true}" 13394,5,301,2017-05-16 21:10:12,http://pagac.co/milford,,238.242.69.62,"{""location"": ""KG"", ""is_mobile"": true}" 16347,6,370,2016-12-29 22:12:41,http://cummings.co/alf,0.9392782598,83.95.104.249,"{""location"": ""KG"", ""is_mobile"": true}" 16348,6,370,2017-01-23 05:30:55,http://connelly.name/rhiannon.fisher,0.8577447533,237.190.119.147,"{""location"": ""CU"", ""is_mobile"": false}" 16349,6,370,2017-04-06 21:00:29,http://leschhane.io/saige,0.3867775915,88.160.122.184,"{""location"": ""ER"", ""is_mobile"": true}" 16350,6,370,2017-04-05 22:57:27,http://marvintoy.io/kaandra,0.3779063708,163.125.126.141,"{""location"": ""SN"", ""is_mobile"": false}" 16351,6,370,2017-02-22 07:37:09,http://johnsmoen.co/malcolm,0.2594580189,192.236.3.152,"{""location"": ""DO"", ""is_mobile"": true}" 16352,6,370,2017-05-02 17:16:17,http://bartolettimaggio.net/kimberly,0.1229465548,51.110.66.27,"{""location"": ""BQ"", ""is_mobile"": false}" 16353,6,370,2017-05-06 14:49:00,http://starkrolfson.info/juwan,0.1936098527,102.109.148.43,"{""location"": ""GB"", ""is_mobile"": false}" 16354,6,370,2017-06-12 19:36:51,http://harris.biz/korbin.kutch,0.3714085406,153.217.77.86,"{""location"": ""KR"", ""is_mobile"": false}" 16355,6,370,2017-05-07 10:58:00,http://prosacco.name/jody,0.6422231779,124.96.94.150,"{""location"": ""LB"", ""is_mobile"": false}" 16356,6,370,2017-03-08 19:10:50,http://langworth.name/abdul.gutmann,0.3179840192,212.235.39.90,"{""location"": ""DJ"", ""is_mobile"": true}" 16357,6,370,2017-02-01 07:51:14,http://simoniscummerata.info/philip,0.9756520859,16.127.126.28,"{""location"": ""GM"", ""is_mobile"": false}" 16358,6,370,2017-01-11 16:36:36,http://kuhlman.net/guy,0.9505561941,10.126.249.212,"{""location"": ""PE"", ""is_mobile"": false}" 16359,6,370,2017-05-08 16:29:44,http://pfannerstillschmitt.name/wilhelm_kuphal,0.2526571959,119.182.164.211,"{""location"": ""GM"", ""is_mobile"": true}" 16360,6,371,2017-05-08 02:38:51,http://dach.com/quincy,0.6713239686,229.219.134.140,"{""location"": ""VG"", ""is_mobile"": true}" 16361,6,371,2017-05-14 10:58:11,http://langworthhand.com/webster_kiehn,0.4714293771,96.21.118.17,"{""location"": ""BD"", ""is_mobile"": true}" 16362,6,371,2017-05-23 08:23:27,http://weimann.name/guy,0.8522895659,71.233.115.130,"{""location"": ""NP"", ""is_mobile"": true}" 16363,6,371,2017-04-28 02:01:38,http://bergnaum.com/alysha,0.9928476210,173.74.104.252,"{""location"": ""TW"", ""is_mobile"": true}" 16364,6,371,2017-05-02 02:43:01,http://haag.net/mellie,0.8358821013,51.209.109.30,"{""location"": ""LK"", ""is_mobile"": true}" 16365,6,371,2017-03-24 17:37:08,http://fay.co/daren,0.4846032874,141.180.240.169,"{""location"": ""PR"", ""is_mobile"": true}" 16366,6,371,2017-02-23 02:35:17,http://schmitt.biz/dawn.langosh,0.4103010959,39.136.242.86,"{""location"": ""AL"", ""is_mobile"": false}" 16367,6,371,2017-05-04 15:50:17,http://stiedemannchamplin.com/queenie.stokes,0.9944715661,163.102.206.234,"{""location"": ""SN"", ""is_mobile"": false}" 16368,6,371,2017-05-19 19:47:21,http://monahan.io/nicholas_murazik,0.5991129038,4.72.34.163,"{""location"": ""TM"", ""is_mobile"": true}" 16369,6,371,2017-02-17 13:38:28,http://mcglynn.name/heather.carroll,0.2994231871,90.41.103.251,"{""location"": ""CG"", ""is_mobile"": true}" 16370,6,371,2017-04-24 12:04:46,http://berge.org/kimberly,0.6616318221,175.106.225.122,"{""location"": ""GA"", ""is_mobile"": false}" 16371,6,371,2016-12-26 02:23:10,http://jenkinsmarquardt.biz/westley.wilderman,0.2240959061,59.91.18.4,"{""location"": ""AZ"", ""is_mobile"": false}" 16372,6,371,2017-03-26 03:43:21,http://bauch.info/mariano,0.3114070546,201.10.106.214,"{""location"": ""KY"", ""is_mobile"": true}" 16373,6,371,2017-05-31 08:16:38,http://vonaltenwerth.co/luigi,0.9771082932,42.252.157.93,"{""location"": ""US"", ""is_mobile"": false}" 16374,6,371,2017-05-18 05:51:18,http://simonisbatz.org/avery.lang,0.7040879919,208.43.216.250,"{""location"": ""KZ"", ""is_mobile"": true}" 16375,6,371,2017-01-04 10:14:28,http://kshlerin.co/eldon.maggio,0.5649297556,77.168.254.23,"{""location"": ""KI"", ""is_mobile"": true}" 16376,6,371,2017-01-29 23:30:25,http://grahambauch.com/camryn,0.3918487075,34.8.230.143,"{""location"": ""SZ"", ""is_mobile"": false}" 16377,6,371,2016-12-24 21:52:22,http://stiedemann.org/keaton.lubowitz,0.2864161980,241.141.229.81,"{""location"": ""ML"", ""is_mobile"": true}" 16378,6,371,2017-05-08 11:49:06,http://spencer.biz/cara_wilderman,0.3497677859,35.181.187.137,"{""location"": ""TF"", ""is_mobile"": true}" 16379,6,371,2017-03-06 03:19:11,http://kozey.org/cheyenne.schultz,0.4765532724,143.236.24.183,"{""location"": ""AO"", ""is_mobile"": true}" 16380,6,371,2017-06-02 06:19:37,http://grahamfunk.co/horacio,0.0988377897,99.129.129.243,"{""location"": ""ET"", ""is_mobile"": false}" 16381,6,371,2017-03-11 03:29:17,http://jerderolfson.name/verner,0.4666825641,180.52.148.248,"{""location"": ""CR"", ""is_mobile"": true}" 16382,6,371,2017-02-01 02:39:37,http://welchkulas.info/steve,0.1345339292,66.134.190.138,"{""location"": ""AU"", ""is_mobile"": false}" 16383,6,371,2017-02-05 07:10:51,http://runtehegmann.biz/juliana,0.3006660486,104.95.220.70,"{""location"": ""DZ"", ""is_mobile"": false}" 16384,6,371,2017-01-08 19:13:41,http://murray.info/monte.donnelly,0.5379703473,231.34.223.191,"{""location"": ""PH"", ""is_mobile"": true}" 16385,6,371,2017-04-15 19:42:51,http://kohler.org/valentine,0.4083965299,137.107.180.153,"{""location"": ""GU"", ""is_mobile"": false}" 16386,6,371,2017-01-16 02:46:30,http://grantmoen.com/lexie,0.2463123588,191.81.206.246,"{""location"": ""BW"", ""is_mobile"": false}" 16387,6,371,2017-06-04 03:11:43,http://rempel.net/verna,0.5828606327,170.201.248.109,"{""location"": ""TC"", ""is_mobile"": true}" 16388,6,371,2017-04-27 06:54:00,http://grantboyer.org/vicky,0.4477090816,119.7.226.11,"{""location"": ""CO"", ""is_mobile"": false}" 16389,6,371,2017-03-16 07:10:03,http://schuppemonahan.io/gaetano.schiller,0.8474319353,167.223.38.128,"{""location"": ""MM"", ""is_mobile"": false}" 16390,6,371,2017-03-26 10:10:01,http://pfannerstillcollier.com/archibald.dibbert,0.6715038430,71.70.81.49,"{""location"": ""GD"", ""is_mobile"": true}" 16391,6,371,2016-12-19 14:35:20,http://willms.io/coby,0.7673755570,129.55.198.209,"{""location"": ""KW"", ""is_mobile"": false}" 16392,6,371,2017-04-16 01:48:00,http://heathcote.info/sydney_wyman,0.0845630446,42.203.246.244,"{""location"": ""ET"", ""is_mobile"": true}" 16393,6,371,2017-03-13 21:21:14,http://rowerogahn.name/pansy_west,0.4532845679,196.108.9.99,"{""location"": ""VG"", ""is_mobile"": true}" 16394,6,371,2017-02-03 14:42:18,http://boyer.name/brian_lemke,0.8030681905,119.94.155.222,"{""location"": ""CL"", ""is_mobile"": false}" 16395,6,371,2017-02-23 00:45:05,http://frami.co/alena,0.9786662685,26.16.22.152,"{""location"": ""BN"", ""is_mobile"": false}" 16396,6,372,2016-12-26 13:00:47,http://stokehields.org/howell,0.2942215462,249.237.198.99,"{""location"": ""CZ"", ""is_mobile"": false}" 16397,6,372,2017-04-08 13:07:49,http://feestbeahan.net/coleman.schmidt,0.3976201655,236.211.33.136,"{""location"": ""MA"", ""is_mobile"": false}" 3525,2,77,2017-06-09 17:05:01,http://reichertjaskolski.net/nathan.littel,0.8646689543,227.188.18.36,"{""location"": ""LC"", ""is_mobile"": false}" 3526,2,77,2017-04-28 19:41:52,http://nicolas.name/natasha,0.2104158513,107.252.68.168,"{""location"": ""SE"", ""is_mobile"": true}" 3527,2,77,2016-12-17 01:22:03,http://harberrohan.biz/carlos.oconner,0.3867501898,159.157.60.109,"{""location"": ""JE"", ""is_mobile"": true}" 3528,2,77,2017-04-21 06:08:33,http://schiller.info/kiara,0.6127111111,31.196.245.111,"{""location"": ""HM"", ""is_mobile"": false}" 3529,2,77,2017-01-31 12:17:11,http://gulgowskitillman.org/chandler,0.6236046562,226.180.14.210,"{""location"": ""PH"", ""is_mobile"": true}" 3530,2,77,2017-06-03 21:46:28,http://kemmer.biz/mozell_collins,0.8042414686,219.136.95.133,"{""location"": ""BS"", ""is_mobile"": true}" 3531,2,77,2017-04-07 07:46:07,http://effertz.io/guie,0.9617592151,67.162.129.97,"{""location"": ""CC"", ""is_mobile"": false}" 3532,2,77,2017-05-31 22:30:33,http://okeefe.biz/guiseppe_marks,0.0438050593,61.220.21.42,"{""location"": ""CF"", ""is_mobile"": true}" 3533,2,77,2017-02-24 21:23:43,http://gutmann.co/arnaldo,0.7312482272,191.7.207.69,"{""location"": ""GH"", ""is_mobile"": false}" 3534,2,77,2017-03-08 20:54:11,http://stiedemannankunding.biz/blanca.tromp,0.8118066682,236.128.180.12,"{""location"": ""ST"", ""is_mobile"": false}" 3535,2,77,2017-01-24 22:58:21,http://beerdooley.net/jerald,0.7538317543,216.40.218.124,"{""location"": ""UM"", ""is_mobile"": true}" 3536,2,77,2017-02-14 21:06:43,http://waterskiehn.net/emmanuelle.mcglynn,0.0766171818,75.85.207.248,"{""location"": ""BO"", ""is_mobile"": false}" 3537,2,77,2017-04-21 14:12:59,http://rowe.info/oliver,0.8325669403,228.195.95.20,"{""location"": ""DJ"", ""is_mobile"": false}" 3538,2,77,2017-02-27 16:09:27,http://moen.io/rodger.nitzsche,0.4487657110,23.240.244.244,"{""location"": ""TL"", ""is_mobile"": false}" 3539,2,77,2017-02-24 21:06:25,http://cain.info/rocky,0.1144254233,198.67.77.138,"{""location"": ""LB"", ""is_mobile"": false}" 3540,2,77,2016-12-28 08:59:08,http://wintheiser.net/harmon.ryan,0.6079171977,183.45.232.252,"{""location"": ""PF"", ""is_mobile"": false}" 3541,2,77,2017-05-12 11:25:12,http://kerlukecarter.net/willow_luettgen,0.3401192250,58.72.155.236,"{""location"": ""VU"", ""is_mobile"": false}" 3542,2,77,2017-01-30 07:25:26,http://witting.org/gayle.welch,0.5360338563,2.99.77.32,"{""location"": ""NG"", ""is_mobile"": false}" 3543,2,77,2017-01-29 00:12:01,http://doyle.com/eliane,0.2077790720,106.23.20.108,"{""location"": ""AO"", ""is_mobile"": true}" 3544,2,78,2017-04-08 22:46:20,http://huel.co/beaulah,0.4554331367,3.231.173.86,"{""location"": ""PL"", ""is_mobile"": true}" 3545,2,78,2017-03-05 17:59:00,http://ratkebreitenberg.name/tillman.kerluke,0.2146511401,192.194.29.53,"{""location"": ""DZ"", ""is_mobile"": true}" 3546,2,78,2017-05-17 03:51:54,http://kuvalis.net/ebony,0.2857327956,55.33.83.230,"{""location"": ""MP"", ""is_mobile"": false}" 3547,2,78,2016-12-26 01:15:53,http://littel.net/jayden,0.9504714400,226.142.113.46,"{""location"": ""BS"", ""is_mobile"": true}" 3548,2,78,2017-03-22 19:02:06,http://reilly.org/skyla.haley,0.0503013805,159.174.244.117,"{""location"": ""DJ"", ""is_mobile"": false}" 3549,2,78,2017-05-06 04:25:34,http://pagac.net/idella,0.2488798866,238.88.221.121,"{""location"": ""MT"", ""is_mobile"": false}" 3550,2,78,2017-02-11 20:16:57,http://green.com/stella.kohler,0.0972691888,205.116.48.135,"{""location"": ""MW"", ""is_mobile"": true}" 3551,2,78,2017-03-13 22:57:24,http://westhilpert.info/vincenzo.wisoky,0.8488775564,249.215.195.153,"{""location"": ""AX"", ""is_mobile"": false}" 3552,2,78,2017-02-19 15:51:00,http://leschschamberger.io/maureen_klein,0.9440256971,145.216.103.95,"{""location"": ""IS"", ""is_mobile"": false}" 3553,2,78,2017-03-15 14:41:00,http://block.co/nyasia.hahn,0.7671362305,210.141.90.219,"{""location"": ""PL"", ""is_mobile"": true}" 3554,2,78,2017-02-07 03:33:57,http://hartmann.org/mafalda.nader,0.7822364623,177.127.247.132,"{""location"": ""ML"", ""is_mobile"": true}" 3555,2,78,2017-01-06 09:57:44,http://bergstrom.com/maxime_hayes,0.0869469526,154.135.77.149,"{""location"": ""MA"", ""is_mobile"": false}" 3556,2,78,2017-05-04 15:49:16,http://lubowitz.com/ron.grimes,0.7574265973,136.11.157.244,"{""location"": ""CM"", ""is_mobile"": true}" 3557,2,78,2017-03-24 08:24:06,http://cruickshank.org/freda,0.5609586225,138.241.135.217,"{""location"": ""TG"", ""is_mobile"": false}" 3558,2,78,2017-04-01 08:58:17,http://carter.co/karli_schmidt,0.4374409776,154.30.107.65,"{""location"": ""DE"", ""is_mobile"": false}" 3559,2,78,2016-12-30 23:13:22,http://oconnell.biz/era_borer,0.0011911789,125.108.187.57,"{""location"": ""MX"", ""is_mobile"": true}" 3560,2,78,2017-04-26 11:00:33,http://littel.net/marcelina_mayer,0.1881197207,117.125.174.170,"{""location"": ""BA"", ""is_mobile"": true}" 3561,2,78,2017-03-03 23:27:56,http://eichmann.io/guy,0.9895087895,234.123.37.216,"{""location"": ""SN"", ""is_mobile"": true}" 3562,2,78,2017-04-04 07:01:47,http://oreilly.net/emery.keeling,0.1047191591,141.3.209.210,"{""location"": ""YE"", ""is_mobile"": true}" 3563,2,78,2017-02-24 19:41:21,http://grant.org/allison,0.6140782687,100.142.144.65,"{""location"": ""CH"", ""is_mobile"": false}" 3564,2,78,2017-03-15 12:09:40,http://wilkinson.info/edmund,0.5211968015,254.43.217.211,"{""location"": ""SH"", ""is_mobile"": true}" 3565,2,78,2017-04-10 19:30:22,http://haagrogahn.org/bennett_vandervort,0.2949713385,218.83.38.201,"{""location"": ""GH"", ""is_mobile"": false}" 3566,2,78,2017-01-03 17:25:32,http://haley.io/jacklyn.kuphal,0.4572064984,60.219.185.33,"{""location"": ""MV"", ""is_mobile"": false}" 5092,2,112,2017-01-09 21:04:58,http://mann.info/kade,,253.44.209.73,"{""location"": ""JE"", ""is_mobile"": true}" 3567,2,78,2017-04-11 08:18:54,http://eichmannmcdermott.info/lois,0.0172292793,9.26.190.54,"{""location"": ""KN"", ""is_mobile"": false}" 3568,2,78,2017-05-02 18:52:56,http://gutmann.org/nina_hodkiewicz,0.1043503759,139.93.148.152,"{""location"": ""HM"", ""is_mobile"": true}" 3569,2,78,2017-03-30 08:59:31,http://gottliebbrekke.biz/claudie_jakubowski,0.3856217561,131.224.51.242,"{""location"": ""KY"", ""is_mobile"": false}" 3570,2,78,2017-02-08 05:31:26,http://leffler.io/dorothy,0.0364641697,90.8.170.178,"{""location"": ""KH"", ""is_mobile"": true}" 3571,2,78,2017-04-27 23:34:21,http://mitchell.biz/robert,0.1353822738,62.222.245.40,"{""location"": ""CK"", ""is_mobile"": false}" 3572,2,78,2017-03-08 01:13:51,http://pouros.name/willow.franecki,0.0208733173,203.103.10.142,"{""location"": ""MG"", ""is_mobile"": false}" 3573,2,78,2017-05-30 22:36:11,http://hackettherzog.co/lenora_weinat,0.4635001035,233.234.9.191,"{""location"": ""DE"", ""is_mobile"": true}" 3574,2,78,2017-04-04 12:51:24,http://beatty.net/lisette_swaniawski,0.5947414712,194.53.191.240,"{""location"": ""VG"", ""is_mobile"": false}" 3575,2,78,2017-04-19 02:46:57,http://ratkerunolfsdottir.biz/adolphus,0.8256513761,35.97.30.106,"{""location"": ""SZ"", ""is_mobile"": false}" 10455,4,235,2016-12-21 01:05:05,http://harriswiegand.org/meggie.abernathy,0.9693606419,231.3.201.166,"{""location"": ""BG"", ""is_mobile"": false}" 10456,4,235,2016-12-28 23:57:43,http://mertz.com/wilburn,0.8264564967,146.67.150.190,"{""location"": ""GW"", ""is_mobile"": true}" 10457,4,235,2016-12-23 03:19:38,http://lang.io/odell.kuphal,0.1719213082,107.144.39.193,"{""location"": ""BQ"", ""is_mobile"": false}" 10458,4,235,2017-04-25 03:22:43,http://beermcclure.com/kaela.gislason,0.0443778766,27.165.86.6,"{""location"": ""DK"", ""is_mobile"": true}" 10459,4,235,2017-03-22 08:07:04,http://heidenreich.co/christian_collins,0.3230310248,212.137.189.215,"{""location"": ""NG"", ""is_mobile"": false}" 10460,4,235,2017-03-08 19:03:25,http://heaney.biz/adelia,0.9470726970,45.48.187.21,"{""location"": ""YE"", ""is_mobile"": false}" 10461,4,235,2016-12-19 09:23:10,http://kuhickoch.co/dallin,0.1162802783,34.207.19.185,"{""location"": ""SV"", ""is_mobile"": false}" 10462,4,235,2017-04-23 20:22:19,http://sengershanahan.com/daphnee.ortiz,0.8032905184,251.207.235.65,"{""location"": ""LR"", ""is_mobile"": false}" 10463,4,235,2017-05-08 08:48:51,http://gerhold.io/lemuel,0.4918276388,105.188.192.78,"{""location"": ""TV"", ""is_mobile"": false}" 10464,4,235,2017-03-04 14:55:13,http://weinat.io/karlee.keeling,0.6070637305,188.87.221.40,"{""location"": ""SJ"", ""is_mobile"": true}" 10465,4,235,2017-02-24 04:16:37,http://mcdermott.net/marcelo.wintheiser,0.6697459142,20.241.156.163,"{""location"": ""BL"", ""is_mobile"": true}" 10466,4,235,2017-03-20 08:49:48,http://harrispollich.biz/cleo.klein,0.7022408021,100.235.217.55,"{""location"": ""SO"", ""is_mobile"": false}" 10467,4,235,2017-02-07 13:08:01,http://hamill.name/alysa,0.4104049618,89.180.231.131,"{""location"": ""VN"", ""is_mobile"": false}" 10468,4,235,2017-01-18 03:43:36,http://mueller.net/idella,0.8401533589,197.213.157.14,"{""location"": ""SL"", ""is_mobile"": false}" 10469,4,235,2017-01-12 03:31:42,http://beer.name/reese,0.1363918534,160.74.23.167,"{""location"": ""FJ"", ""is_mobile"": false}" 10470,4,235,2017-02-10 06:14:13,http://vonabbott.co/lenora,0.5569875980,156.60.24.130,"{""location"": ""CA"", ""is_mobile"": false}" 10471,4,235,2017-01-01 11:44:16,http://dickens.net/agustin,0.7226600862,40.7.158.85,"{""location"": ""AF"", ""is_mobile"": false}" 10472,4,235,2017-03-07 18:50:59,http://cristveum.com/maci.bernier,0.3112403492,162.247.214.22,"{""location"": ""LC"", ""is_mobile"": true}" 10473,4,235,2017-06-07 18:59:35,http://huels.org/colby,0.3971920870,201.243.236.196,"{""location"": ""VI"", ""is_mobile"": false}" 10474,4,235,2017-06-03 13:48:34,http://mcglynn.net/gertrude_sanford,0.1286912343,110.126.192.17,"{""location"": ""GT"", ""is_mobile"": true}" 10475,4,235,2017-02-11 12:37:27,http://connellykihn.info/prince,0.5799262610,174.157.220.17,"{""location"": ""CR"", ""is_mobile"": false}" 10476,4,235,2017-04-15 16:34:05,http://doyle.biz/myriam,0.6533878646,99.72.233.59,"{""location"": ""AT"", ""is_mobile"": true}" 10477,4,235,2017-05-05 06:55:16,http://blickbalistreri.net/elfrieda.marquardt,0.5345555233,47.42.197.135,"{""location"": ""AM"", ""is_mobile"": true}" 10478,4,235,2017-03-06 22:18:41,http://bergnaumdooley.com/dashawn,0.0956727523,50.147.199.22,"{""location"": ""CF"", ""is_mobile"": false}" 10479,4,235,2017-02-11 15:20:55,http://anderson.name/stanford_collier,0.4157616880,114.58.165.7,"{""location"": ""DO"", ""is_mobile"": true}" 10480,4,235,2017-02-25 06:40:36,http://bednar.io/maritza,0.7957961496,89.34.116.152,"{""location"": ""KI"", ""is_mobile"": false}" 10481,4,235,2017-02-05 23:56:37,http://kunze.info/felix_ziemann,0.7467728283,179.98.214.229,"{""location"": ""SK"", ""is_mobile"": true}" 10482,4,235,2017-05-19 02:29:56,http://kshleringutmann.net/boris,0.4164079174,211.141.32.212,"{""location"": ""SD"", ""is_mobile"": false}" 10483,4,235,2017-04-26 11:03:21,http://corkery.net/vivian,0.8206122476,21.99.105.42,"{""location"": ""GW"", ""is_mobile"": true}" 10484,4,235,2017-01-24 08:10:20,http://walkergreenfelder.info/gage_wisozk,0.8904756572,112.197.162.4,"{""location"": ""KR"", ""is_mobile"": true}" 10485,4,235,2017-01-04 00:45:18,http://predovicaufderhar.info/janiya_bruen,0.9720644648,182.184.207.165,"{""location"": ""HK"", ""is_mobile"": false}" 10486,4,235,2017-02-14 06:52:49,http://emmerichheaney.biz/rolando,0.4915439858,155.21.178.24,"{""location"": ""GB"", ""is_mobile"": false}" 10487,4,235,2017-01-12 03:58:19,http://lehnerroob.biz/eileen_huels,0.3230373007,82.72.218.253,"{""location"": ""KG"", ""is_mobile"": false}" 10488,4,236,2017-01-12 10:41:58,http://bosco.info/juanita,0.8215787053,133.204.12.119,"{""location"": ""MM"", ""is_mobile"": true}" 10489,4,236,2017-03-07 10:35:05,http://bednarfeil.org/adelbert,0.6908538357,147.89.109.81,"{""location"": ""AT"", ""is_mobile"": true}" 10490,4,236,2017-05-02 20:15:55,http://ebert.io/theron_veum,0.4907814716,118.180.197.210,"{""location"": ""KN"", ""is_mobile"": true}" 10491,4,236,2017-03-03 10:32:50,http://roob.net/santa,0.5856656290,126.15.188.234,"{""location"": ""BR"", ""is_mobile"": true}" 10492,4,236,2017-02-05 07:45:04,http://schummharvey.org/martine,0.8861414911,203.226.163.254,"{""location"": ""GQ"", ""is_mobile"": true}" 10493,4,236,2017-03-06 17:16:12,http://gorczany.net/kendall_boehm,0.3041111055,198.11.64.28,"{""location"": ""BQ"", ""is_mobile"": true}" 10494,4,236,2017-04-28 23:47:52,http://koeppnader.info/ariane.runolfon,0.4240416080,108.30.188.89,"{""location"": ""MW"", ""is_mobile"": true}" 10495,4,236,2017-05-01 19:26:02,http://shields.name/baylee,0.4990114562,200.203.135.164,"{""location"": ""EC"", ""is_mobile"": true}" 10496,4,236,2017-05-13 23:01:19,http://hoppekuhic.io/rachel.ruel,0.7579896293,193.241.231.242,"{""location"": ""PT"", ""is_mobile"": false}" 10497,4,236,2017-03-20 03:54:39,http://rau.io/myra_smitham,0.8070058865,78.245.69.36,"{""location"": ""GD"", ""is_mobile"": true}" 10498,4,236,2017-02-16 00:18:23,http://bernier.net/lauryn,0.0479388910,175.51.241.162,"{""location"": ""BO"", ""is_mobile"": true}" 10499,4,236,2017-02-27 03:17:03,http://rennerheidenreich.org/eleanora,0.4933401575,68.17.37.221,"{""location"": ""GB"", ""is_mobile"": true}" 10500,4,236,2017-01-15 12:46:58,http://rauharris.org/sandrine,0.9667560721,179.50.246.63,"{""location"": ""CU"", ""is_mobile"": true}" 10501,4,236,2017-04-03 08:04:32,http://mayer.name/dolores,0.7951249153,216.61.151.27,"{""location"": ""VG"", ""is_mobile"": false}" 10502,4,236,2017-04-16 10:32:44,http://larsonturner.biz/laurianne,0.8963280140,144.6.189.169,"{""location"": ""BD"", ""is_mobile"": false}" 10503,4,236,2017-01-14 01:21:39,http://willwolf.biz/noelia.kerluke,0.8580412727,80.220.233.18,"{""location"": ""BN"", ""is_mobile"": false}" 10504,4,236,2017-01-02 18:12:18,http://balistrerisenger.biz/gustave,0.8909889183,73.35.75.178,"{""location"": ""GD"", ""is_mobile"": false}" 10505,4,236,2017-04-29 02:53:44,http://barton.info/jeanne,0.3107550533,228.58.118.108,"{""location"": ""EG"", ""is_mobile"": true}" 10506,4,236,2017-02-15 13:25:18,http://pfannerstill.org/delores,0.7102466642,12.75.118.162,"{""location"": ""AT"", ""is_mobile"": true}" 7484,3,166,2017-03-14 06:01:38,http://flatleyhilpert.name/demarcus,,253.183.175.158,"{""location"": ""MG"", ""is_mobile"": false}" 7485,3,166,2017-03-10 03:27:58,http://nitzschesatterfield.info/gardner.hansen,,151.77.161.77,"{""location"": ""BL"", ""is_mobile"": false}" 7486,3,166,2017-05-02 11:01:46,http://lockman.biz/omer.morar,,141.15.182.72,"{""location"": ""KW"", ""is_mobile"": true}" 7487,3,166,2017-01-14 22:50:37,http://cronawisozk.biz/ian_cremin,,128.93.19.151,"{""location"": ""SC"", ""is_mobile"": false}" 7488,3,166,2017-03-15 18:06:30,http://flatley.name/may_wintheiser,,15.86.58.222,"{""location"": ""SH"", ""is_mobile"": false}" 7489,3,166,2017-06-09 15:53:11,http://smitham.info/jarrod,,225.123.59.120,"{""location"": ""MF"", ""is_mobile"": true}" 7490,3,166,2017-01-17 12:36:18,http://cain.io/leanna,,37.141.118.248,"{""location"": ""UM"", ""is_mobile"": false}" 7491,3,166,2017-03-18 02:35:38,http://dickiwalsh.name/dashawn_dickinson,,240.108.81.123,"{""location"": ""KM"", ""is_mobile"": false}" 7492,3,166,2017-03-18 21:09:21,http://goldnerschowalter.org/winifred,,145.107.177.235,"{""location"": ""HR"", ""is_mobile"": false}" 7493,3,166,2017-02-09 20:48:53,http://schmidtebert.io/delfina,,25.33.97.99,"{""location"": ""YT"", ""is_mobile"": false}" 7494,3,166,2017-05-27 19:35:01,http://kautzer.com/rodolfo_botsford,,80.251.190.183,"{""location"": ""GB"", ""is_mobile"": true}" 7495,3,166,2017-01-17 18:30:21,http://heidenreichwelch.name/mariela,,51.212.156.235,"{""location"": ""TG"", ""is_mobile"": true}" 7496,3,166,2017-04-23 07:51:12,http://glover.info/amos_farrell,,219.148.94.141,"{""location"": ""BB"", ""is_mobile"": true}" 7497,3,166,2017-05-19 06:05:34,http://kemmer.info/malinda_lemke,,209.252.8.153,"{""location"": ""GB"", ""is_mobile"": true}" 7498,3,166,2017-06-07 13:24:19,http://volkmankemmer.com/zakary_denesik,,169.170.121.22,"{""location"": ""MK"", ""is_mobile"": false}" 7499,3,166,2017-02-08 18:12:03,http://welch.io/delbert,,30.229.213.70,"{""location"": ""CI"", ""is_mobile"": true}" 7500,3,166,2017-01-10 05:37:50,http://abbottgrant.io/vesta_mohr,,44.169.79.194,"{""location"": ""MH"", ""is_mobile"": true}" 7501,3,166,2017-06-10 08:29:05,http://block.name/glenda.jenkins,,200.136.139.72,"{""location"": ""PG"", ""is_mobile"": true}" 7502,3,166,2017-06-04 20:08:54,http://robel.net/dahlia_boyle,,8.100.143.28,"{""location"": ""PH"", ""is_mobile"": false}" 7503,3,166,2017-01-02 22:45:35,http://daniel.net/jillian.kuhn,,182.39.254.94,"{""location"": ""TN"", ""is_mobile"": false}" 7504,3,166,2017-04-29 18:56:14,http://larkin.net/twila_sanford,,114.101.173.37,"{""location"": ""GG"", ""is_mobile"": true}" 7505,3,166,2017-05-12 11:54:16,http://robertsbatz.name/rashawn_monahan,,20.2.185.221,"{""location"": ""PM"", ""is_mobile"": false}" 7506,3,166,2017-03-31 10:45:03,http://keeblerhane.name/americo,,46.249.99.10,"{""location"": ""GM"", ""is_mobile"": true}" 7507,3,166,2017-04-26 11:09:14,http://oconner.net/boyd.lebsack,,119.65.127.193,"{""location"": ""GA"", ""is_mobile"": true}" 7508,3,166,2017-01-19 07:28:30,http://champlin.com/aurelie,,188.71.46.203,"{""location"": ""SJ"", ""is_mobile"": false}" 7509,3,166,2016-12-19 02:15:17,http://wilkinsongulgowski.name/flavio,,34.135.26.77,"{""location"": ""ST"", ""is_mobile"": true}" 7510,3,166,2017-01-26 21:11:24,http://fritsch.name/zoila,,252.91.225.120,"{""location"": ""BJ"", ""is_mobile"": false}" 7511,3,166,2017-01-13 14:02:00,http://miller.org/kieran,,177.65.233.201,"{""location"": ""BH"", ""is_mobile"": false}" 7512,3,166,2017-04-05 14:39:49,http://fadelwilkinson.io/penelope.price,,213.139.170.201,"{""location"": ""HM"", ""is_mobile"": false}" 7513,3,166,2016-12-16 08:43:40,http://adams.org/olaf.reichert,,215.45.128.193,"{""location"": ""MV"", ""is_mobile"": false}" 7514,3,166,2017-04-17 05:52:40,http://sawayn.co/gerard_swaniawski,,68.216.126.88,"{""location"": ""MX"", ""is_mobile"": true}" 7515,3,166,2017-02-04 08:12:22,http://vandervort.biz/valerie,,71.25.167.182,"{""location"": ""BI"", ""is_mobile"": true}" 7516,3,166,2017-01-11 12:08:38,http://damore.org/maida,,134.29.49.223,"{""location"": ""TV"", ""is_mobile"": true}" 7517,3,166,2017-02-25 18:23:17,http://wilkinsonswaniawski.co/judge.bashirian,,175.13.235.72,"{""location"": ""SS"", ""is_mobile"": false}" 7518,3,166,2017-01-26 04:57:28,http://cronamarks.org/margot,,185.13.133.198,"{""location"": ""GS"", ""is_mobile"": true}" 7519,3,166,2017-05-09 18:29:36,http://littel.name/edward,,62.221.238.152,"{""location"": ""MT"", ""is_mobile"": true}" 7520,3,166,2017-05-27 00:03:53,http://mertzthiel.net/ken.bailey,,248.188.232.25,"{""location"": ""AO"", ""is_mobile"": true}" 7521,3,166,2017-04-08 21:26:47,http://kundelowe.info/roxane.schiller,,204.217.176.130,"{""location"": ""HK"", ""is_mobile"": true}" 7522,3,166,2017-01-30 10:56:11,http://hahnwalker.net/betty.schmitt,,237.207.229.119,"{""location"": ""PS"", ""is_mobile"": false}" 7523,3,167,2017-04-21 03:44:16,http://moen.org/ahmed,,241.206.93.229,"{""location"": ""TR"", ""is_mobile"": true}" 7524,3,167,2017-03-29 19:06:18,http://vonstracke.name/retta,,136.93.29.97,"{""location"": ""MF"", ""is_mobile"": false}" 7525,3,167,2017-03-22 03:53:41,http://lynch.info/stephanie,,226.30.73.11,"{""location"": ""LR"", ""is_mobile"": true}" 7526,3,167,2017-01-25 13:12:58,http://gleichnerhomenick.com/raheem.boyer,,240.68.185.58,"{""location"": ""CN"", ""is_mobile"": true}" 7527,3,167,2017-05-24 11:23:55,http://dachbarrows.org/kelli.schmidt,,88.189.217.38,"{""location"": ""ST"", ""is_mobile"": false}" 7528,3,167,2016-12-23 10:16:08,http://ondricka.io/bernice.gerhold,,26.162.170.152,"{""location"": ""KI"", ""is_mobile"": false}" 7529,3,167,2017-03-14 07:01:40,http://haley.co/zola_sanford,,34.254.190.181,"{""location"": ""GY"", ""is_mobile"": false}" 7530,3,167,2017-03-26 10:38:18,http://wuckert.info/kacey_fritsch,,9.206.129.18,"{""location"": ""LU"", ""is_mobile"": false}" 7531,3,167,2017-03-22 12:09:20,http://ratke.co/lucas,,93.174.219.193,"{""location"": ""SS"", ""is_mobile"": true}" 7532,3,167,2016-12-22 15:04:26,http://bechtelarmosciski.co/hosea,,117.230.202.130,"{""location"": ""GT"", ""is_mobile"": false}" 7533,3,167,2017-06-12 09:35:24,http://parisian.com/rico,,239.228.207.245,"{""location"": ""CX"", ""is_mobile"": true}" 7534,3,167,2017-04-13 03:10:56,http://stromanwintheiser.name/vladimir_nienow,,55.46.151.20,"{""location"": ""PW"", ""is_mobile"": false}" 7535,3,167,2017-01-03 04:36:26,http://douglashomenick.biz/brittany_abernathy,,124.102.179.138,"{""location"": ""JP"", ""is_mobile"": false}" 7536,3,167,2017-03-22 10:26:05,http://koepp.name/brycen,,203.158.48.26,"{""location"": ""FO"", ""is_mobile"": false}" 7537,3,167,2017-05-02 01:48:38,http://prosacco.biz/claudie_kutch,,105.254.251.179,"{""location"": ""PG"", ""is_mobile"": true}" 7538,3,167,2017-01-24 21:10:29,http://collier.net/josephine,,20.138.13.140,"{""location"": ""AO"", ""is_mobile"": false}" 13395,5,301,2017-02-07 13:57:28,http://zulaufmckenzie.org/lisa_dietrich,,39.64.120.218,"{""location"": ""MW"", ""is_mobile"": true}" 13396,5,301,2017-03-31 12:20:21,http://senger.org/jon.pacocha,,189.163.141.182,"{""location"": ""KE"", ""is_mobile"": false}" 13397,5,301,2017-05-01 10:09:14,http://blick.co/manley,,197.143.77.204,"{""location"": ""GM"", ""is_mobile"": false}" 13398,5,301,2017-01-12 06:26:11,http://lockman.org/edythe,,119.116.82.21,"{""location"": ""TK"", ""is_mobile"": false}" 13399,5,301,2017-04-14 01:41:39,http://dickialtenwerth.io/braxton,,52.195.149.232,"{""location"": ""WF"", ""is_mobile"": true}" 13400,5,301,2017-03-04 02:05:26,http://schinner.io/colt,,232.218.107.211,"{""location"": ""MQ"", ""is_mobile"": true}" 13401,5,301,2016-12-15 23:30:47,http://kuphalcummerata.com/coralie.flatley,,241.62.60.91,"{""location"": ""TO"", ""is_mobile"": true}" 13402,5,301,2017-03-17 21:33:57,http://emmerich.io/tommie,,249.235.4.68,"{""location"": ""GA"", ""is_mobile"": false}" 13403,5,301,2016-12-31 01:23:17,http://beatty.io/roxane,,44.212.154.101,"{""location"": ""MA"", ""is_mobile"": false}" 13404,5,301,2017-03-11 18:30:35,http://buckridgewuckert.biz/marcel,,207.68.154.46,"{""location"": ""MP"", ""is_mobile"": true}" 13405,5,301,2017-05-17 03:08:53,http://berge.org/kim_bernier,,175.2.50.220,"{""location"": ""GP"", ""is_mobile"": false}" 13406,5,302,2017-04-26 13:13:01,http://kunde.io/camille,,205.232.87.113,"{""location"": ""TC"", ""is_mobile"": false}" 13407,5,302,2017-05-21 20:35:14,http://goldnerheidenreich.biz/asha,,30.209.52.244,"{""location"": ""CD"", ""is_mobile"": true}" 13408,5,302,2017-03-21 16:22:09,http://kautzer.info/lavern.predovic,,163.5.212.90,"{""location"": ""GG"", ""is_mobile"": false}" 13409,5,302,2017-05-17 12:41:11,http://wittingshanahan.name/harmony_collier,,22.171.133.174,"{""location"": ""SR"", ""is_mobile"": true}" 13410,5,302,2016-12-22 13:15:32,http://kuhic.com/toy,,40.11.167.202,"{""location"": ""RS"", ""is_mobile"": false}" 13411,5,302,2017-01-15 14:10:28,http://huel.co/dante_green,,68.156.189.107,"{""location"": ""CM"", ""is_mobile"": false}" 13412,5,302,2017-02-19 11:22:21,http://bauchbashirian.name/marcellus.adams,,31.7.25.153,"{""location"": ""CF"", ""is_mobile"": false}" 13413,5,302,2017-06-03 22:15:44,http://corkery.net/marjorie,,119.187.42.231,"{""location"": ""BA"", ""is_mobile"": true}" 13414,5,302,2017-05-29 18:09:24,http://kocarroll.biz/sister.beier,,222.243.58.254,"{""location"": ""GI"", ""is_mobile"": false}" 13415,5,302,2017-05-21 13:17:04,http://runolfon.biz/obie.west,,247.252.77.254,"{""location"": ""FK"", ""is_mobile"": false}" 13416,5,302,2017-02-07 23:24:28,http://ko.name/oswald,,166.235.24.217,"{""location"": ""CR"", ""is_mobile"": true}" 13417,5,302,2017-04-01 16:05:21,http://hackett.org/johanna,,33.244.16.234,"{""location"": ""NP"", ""is_mobile"": true}" 13418,5,302,2017-05-24 08:06:43,http://kutch.org/courtney,,161.120.20.37,"{""location"": ""UM"", ""is_mobile"": true}" 13419,5,302,2017-04-11 20:35:29,http://davis.co/bonnie_botsford,,225.32.204.191,"{""location"": ""NE"", ""is_mobile"": true}" 13420,5,302,2017-03-17 15:35:10,http://hand.name/ricardo.kertzmann,,2.7.138.139,"{""location"": ""FJ"", ""is_mobile"": true}" 13421,5,302,2016-12-28 03:36:23,http://raunicolas.net/ronaldo,,160.178.55.148,"{""location"": ""WS"", ""is_mobile"": true}" 13422,5,302,2017-03-25 08:49:27,http://mcclure.io/napoleon_ruel,,200.24.220.4,"{""location"": ""NC"", ""is_mobile"": true}" 13423,5,302,2017-06-04 05:50:19,http://christiansenbrown.com/kyle,,159.122.245.60,"{""location"": ""AS"", ""is_mobile"": false}" 13424,5,302,2017-04-25 00:32:49,http://hilll.biz/shaun.johnston,,98.154.86.139,"{""location"": ""MZ"", ""is_mobile"": true}" 13425,5,302,2017-02-28 15:48:26,http://gerlach.org/guido,,61.243.252.45,"{""location"": ""RW"", ""is_mobile"": true}" 13426,5,302,2017-01-11 16:58:40,http://barton.com/clemens.osinski,,126.42.98.250,"{""location"": ""FK"", ""is_mobile"": false}" 13427,5,302,2017-03-31 10:30:59,http://wilkinson.com/leola.hoppe,,50.90.241.91,"{""location"": ""ZM"", ""is_mobile"": true}" 13428,5,302,2017-05-01 05:03:21,http://gutmannnader.com/rodrick.roob,,61.48.105.112,"{""location"": ""SB"", ""is_mobile"": false}" 13429,5,302,2017-01-01 03:02:36,http://pagac.co/berniece.windler,,167.192.118.31,"{""location"": ""IL"", ""is_mobile"": true}" 13430,5,302,2017-06-02 05:16:35,http://parisianullrich.name/serena,,20.174.57.44,"{""location"": ""NI"", ""is_mobile"": true}" 13431,5,302,2017-05-12 00:57:29,http://casper.com/lisette.smith,,117.62.135.104,"{""location"": ""SK"", ""is_mobile"": false}" 13432,5,302,2017-06-09 15:11:44,http://bayer.co/francisca_kub,,146.53.179.223,"{""location"": ""BN"", ""is_mobile"": false}" 13433,5,302,2017-02-03 04:34:49,http://collins.name/annetta.wisozk,,235.24.196.225,"{""location"": ""NP"", ""is_mobile"": false}" 13434,5,302,2017-05-13 03:42:19,http://faheybednar.co/lilian,,145.175.121.143,"{""location"": ""US"", ""is_mobile"": false}" 13435,5,302,2017-05-30 17:26:00,http://schowalterschumm.net/mariela,,105.74.133.177,"{""location"": ""TW"", ""is_mobile"": true}" 13436,5,302,2017-02-05 16:34:50,http://fritsch.name/mia_dooley,,74.27.207.67,"{""location"": ""LK"", ""is_mobile"": false}" 13437,5,302,2016-12-26 03:00:21,http://weinat.com/dee.ruecker,,252.122.26.147,"{""location"": ""BE"", ""is_mobile"": true}" 13438,5,302,2017-03-02 16:50:34,http://mckenzie.com/maddison.waelchi,,169.116.81.44,"{""location"": ""BS"", ""is_mobile"": false}" 13439,5,302,2017-05-16 21:42:12,http://smithstroman.biz/edyth.gibson,,103.11.7.180,"{""location"": ""AZ"", ""is_mobile"": false}" 13440,5,302,2017-03-31 05:55:31,http://treutel.name/madison.padberg,,72.101.12.164,"{""location"": ""SK"", ""is_mobile"": false}" 13441,5,302,2017-03-15 13:05:54,http://stoltenbergmayert.net/elmo,,146.57.149.238,"{""location"": ""TH"", ""is_mobile"": true}" 13442,5,302,2017-03-29 02:04:19,http://konopelski.net/kali,,206.100.227.58,"{""location"": ""TJ"", ""is_mobile"": true}" 13443,5,302,2017-03-17 08:55:54,http://windler.io/alexandra.jacobson,,183.55.7.56,"{""location"": ""CI"", ""is_mobile"": false}" 13444,5,302,2017-06-13 19:03:37,http://bednarbednar.net/letha.corwin,,3.174.22.17,"{""location"": ""LV"", ""is_mobile"": false}" 13445,5,302,2017-03-05 04:20:14,http://pfannerstill.org/cordie.heel,,169.100.18.247,"{""location"": ""BY"", ""is_mobile"": true}" 13446,5,302,2017-03-20 04:11:41,http://parisian.io/jerrod_hartmann,,3.135.200.210,"{""location"": ""TC"", ""is_mobile"": true}" 13447,5,302,2017-05-17 12:32:10,http://daughertyferry.biz/joseph.torp,,189.55.249.144,"{""location"": ""HN"", ""is_mobile"": false}" 13448,5,302,2017-05-16 12:13:05,http://feesteichmann.io/bradly_schumm,,119.208.222.217,"{""location"": ""BN"", ""is_mobile"": true}" 13449,5,302,2016-12-22 20:51:08,http://johnstonmacejkovic.co/jerrod_lynch,,171.219.45.99,"{""location"": ""AL"", ""is_mobile"": true}" 16398,6,372,2017-03-29 07:47:11,http://hermiston.biz/agnes_block,0.1251294066,26.111.147.46,"{""location"": ""DK"", ""is_mobile"": false}" 16399,6,372,2017-01-09 11:13:52,http://prosaccofranecki.name/etha,0.8027375769,119.25.96.133,"{""location"": ""YE"", ""is_mobile"": false}" 16400,6,372,2017-01-20 04:35:35,http://kirlin.biz/cameron_abernathy,0.1366641467,88.60.28.181,"{""location"": ""GR"", ""is_mobile"": true}" 16401,6,372,2017-04-16 05:27:19,http://moriette.name/camille,0.6796051945,147.35.158.196,"{""location"": ""GR"", ""is_mobile"": false}" 16402,6,372,2017-05-12 19:30:24,http://kuhlman.biz/vernice,0.9534140100,223.222.115.48,"{""location"": ""TN"", ""is_mobile"": false}" 16403,6,372,2017-03-15 23:02:35,http://aufderharluettgen.org/laria_rodriguez,0.1768131380,162.51.6.78,"{""location"": ""CA"", ""is_mobile"": false}" 16404,6,372,2017-03-11 07:35:46,http://adams.co/ena.oconnell,0.7475443611,233.65.173.122,"{""location"": ""SV"", ""is_mobile"": true}" 16405,6,372,2017-03-30 17:44:08,http://gaylordbogisich.name/fredrick,0.2895152905,212.108.187.247,"{""location"": ""CX"", ""is_mobile"": true}" 16406,6,372,2017-03-22 01:10:18,http://ziemekaulke.com/maybelle.kutch,0.9330083294,97.61.8.190,"{""location"": ""BO"", ""is_mobile"": true}" 16407,6,372,2017-04-22 13:07:49,http://rodriguez.info/nyasia.lockman,0.3340102074,185.230.224.199,"{""location"": ""CL"", ""is_mobile"": true}" 16408,6,372,2017-05-16 16:23:54,http://jaskolskiokon.biz/douglas,0.3438057692,72.68.14.59,"{""location"": ""ET"", ""is_mobile"": true}" 16409,6,372,2016-12-23 11:54:20,http://parkerprosacco.org/darius,0.1974158276,117.163.252.126,"{""location"": ""JP"", ""is_mobile"": true}" 16410,6,372,2017-02-27 18:58:58,http://heaneyorn.net/marlene,0.4106341421,163.139.81.63,"{""location"": ""VI"", ""is_mobile"": true}" 16411,6,372,2017-05-19 04:18:55,http://jacobiko.net/tiana.gorczany,0.7702360372,222.159.174.142,"{""location"": ""LI"", ""is_mobile"": false}" 16412,6,372,2016-12-29 22:55:03,http://kub.io/bertram_jaskolski,0.0653712018,249.56.138.245,"{""location"": ""CN"", ""is_mobile"": true}" 16413,6,372,2017-03-03 08:47:43,http://reichelkuhn.io/arvel_schneider,0.7262177865,115.4.119.239,"{""location"": ""KE"", ""is_mobile"": true}" 16414,6,372,2017-04-20 03:07:29,http://zboncak.com/jarod.funk,0.7239719221,38.224.209.54,"{""location"": ""VN"", ""is_mobile"": false}" 16415,6,372,2017-02-24 18:07:54,http://monahan.io/mallory.conroy,0.0426637946,98.37.205.89,"{""location"": ""AZ"", ""is_mobile"": true}" 16416,6,372,2017-05-13 13:12:36,http://rippinondricka.biz/eliezer,0.8816189796,194.55.40.40,"{""location"": ""MW"", ""is_mobile"": false}" 16417,6,372,2017-05-11 03:19:06,http://morar.co/wilfrid_emmerich,0.1699453918,129.195.32.24,"{""location"": ""TK"", ""is_mobile"": true}" 16418,6,372,2017-03-13 19:03:25,http://shanahan.net/duane_swift,0.0619036335,192.24.150.72,"{""location"": ""CY"", ""is_mobile"": true}" 16419,6,372,2016-12-25 05:50:37,http://vonrueden.info/ida_veum,0.6345671305,94.35.183.201,"{""location"": ""BD"", ""is_mobile"": false}" 16420,6,372,2016-12-15 18:58:48,http://macgyver.co/jose,0.7812324820,219.5.168.125,"{""location"": ""IN"", ""is_mobile"": true}" 16421,6,372,2017-03-24 17:11:28,http://leffler.org/maggie_quitzon,0.7008069996,229.65.13.131,"{""location"": ""LA"", ""is_mobile"": true}" 16422,6,372,2017-06-05 01:42:58,http://robelryan.biz/jacynthe.auer,0.8802381139,27.99.240.38,"{""location"": ""DE"", ""is_mobile"": false}" 16423,6,372,2017-03-28 14:28:35,http://hammes.co/travon,0.7156870522,48.77.211.133,"{""location"": ""CF"", ""is_mobile"": true}" 16424,6,372,2016-12-17 04:58:56,http://von.info/caitlyn,0.4957561939,56.188.104.107,"{""location"": ""JM"", ""is_mobile"": true}" 16425,6,372,2017-02-20 06:21:54,http://quigley.name/dock,0.3595924243,34.223.92.68,"{""location"": ""KY"", ""is_mobile"": false}" 16426,6,372,2017-02-13 13:19:38,http://muller.org/alize.daugherty,0.1746717026,200.167.253.165,"{""location"": ""DJ"", ""is_mobile"": true}" 16427,6,372,2017-06-06 03:05:21,http://kub.org/annie.kunde,0.4328178215,111.117.65.39,"{""location"": ""AT"", ""is_mobile"": true}" 16428,6,372,2017-03-15 11:18:03,http://hillshilll.com/claudia,0.3160476002,208.212.47.183,"{""location"": ""GT"", ""is_mobile"": true}" 16429,6,372,2017-02-18 12:27:31,http://carroll.info/viviane,0.8305244732,43.106.182.249,"{""location"": ""FM"", ""is_mobile"": true}" 16430,6,372,2017-03-27 12:53:53,http://skilesheidenreich.org/kiarra,0.8520098704,103.28.116.22,"{""location"": ""NZ"", ""is_mobile"": false}" 16431,6,372,2017-02-18 06:54:21,http://conroy.net/fabiola,0.3322204443,21.197.178.204,"{""location"": ""LR"", ""is_mobile"": true}" 16432,6,372,2017-01-13 02:23:52,http://eichmann.net/hilton.lowe,0.5071588830,57.85.4.220,"{""location"": ""US"", ""is_mobile"": true}" 16433,6,372,2017-02-11 09:44:00,http://reilly.name/fred,0.2999466254,56.218.107.116,"{""location"": ""KE"", ""is_mobile"": true}" 16434,6,372,2016-12-31 20:12:21,http://prosacco.info/betsy_davis,0.8719771885,221.38.73.244,"{""location"": ""MP"", ""is_mobile"": true}" 16435,6,372,2017-03-21 14:01:14,http://effertz.info/lexus.konopelski,0.9529029325,140.112.133.106,"{""location"": ""CN"", ""is_mobile"": true}" 16436,6,372,2017-02-02 18:10:41,http://harveymccullough.co/rollin.gulgowski,0.0003648808,215.177.197.129,"{""location"": ""GP"", ""is_mobile"": false}" 16437,6,372,2017-01-14 16:03:59,http://herzogkertzmann.name/rico,0.2950396543,28.141.136.35,"{""location"": ""BQ"", ""is_mobile"": false}" 16438,6,372,2017-02-21 17:51:29,http://wehner.com/zoila,0.0115972739,168.77.220.253,"{""location"": ""MP"", ""is_mobile"": true}" 16439,6,372,2016-12-21 23:46:24,http://gorczany.info/emilia,0.9197509855,5.209.209.216,"{""location"": ""SG"", ""is_mobile"": true}" 16440,6,372,2017-01-03 10:05:37,http://metz.io/verda.rohan,0.4085709892,22.133.81.234,"{""location"": ""LA"", ""is_mobile"": true}" 16441,6,372,2017-04-02 21:05:29,http://schimmel.co/rowland_hilll,0.9718457163,202.16.16.140,"{""location"": ""PW"", ""is_mobile"": false}" 16442,6,372,2017-04-17 20:09:49,http://mclaughlincronin.org/nicola_fadel,0.2946698773,228.85.166.21,"{""location"": ""BG"", ""is_mobile"": true}" 16443,6,372,2017-03-05 19:45:52,http://balistreri.biz/vidal_feest,0.2746982211,183.161.143.11,"{""location"": ""SY"", ""is_mobile"": false}" 16444,6,372,2017-01-31 22:53:52,http://sanford.org/juwan,0.7636329492,43.135.177.211,"{""location"": ""ST"", ""is_mobile"": true}" 16445,6,372,2016-12-27 07:34:12,http://runolfonaufderhar.info/corene,0.6519548338,13.70.110.211,"{""location"": ""DE"", ""is_mobile"": true}" 16446,6,372,2017-04-04 13:59:39,http://sawayn.co/chyna_schaefer,0.8836918461,124.230.94.101,"{""location"": ""BH"", ""is_mobile"": true}" 16447,6,372,2016-12-25 22:42:22,http://zboncakmcclure.io/evert,0.0844935369,122.237.77.146,"{""location"": ""RS"", ""is_mobile"": true}" 16448,6,372,2017-04-01 17:12:46,http://gutkowski.info/stephanie.aufderhar,0.9587922765,189.234.225.230,"{""location"": ""BO"", ""is_mobile"": false}" 3576,2,78,2017-04-03 15:00:30,http://kuvalis.co/imelda_dooley,0.9513122524,160.145.236.93,"{""location"": ""JO"", ""is_mobile"": false}" 3577,2,78,2017-05-12 00:15:23,http://dachhills.io/andreane.jacobson,0.4646776912,19.139.212.124,"{""location"": ""AL"", ""is_mobile"": false}" 3578,2,78,2017-03-19 21:55:32,http://breitenberg.co/domenico,0.8568929204,41.116.63.175,"{""location"": ""MD"", ""is_mobile"": true}" 3579,2,78,2017-01-31 05:17:16,http://stiedemann.info/ali,0.9001517863,123.220.214.32,"{""location"": ""MP"", ""is_mobile"": false}" 3580,2,78,2017-01-18 07:59:38,http://kingernser.biz/rita,0.9801864135,189.102.139.184,"{""location"": ""CV"", ""is_mobile"": true}" 3581,2,78,2017-02-14 05:42:18,http://johnston.net/justyn,0.0908603178,12.175.19.52,"{""location"": ""SC"", ""is_mobile"": true}" 3582,2,78,2017-01-13 19:32:24,http://kautzer.com/jarod_pollich,0.4904784161,9.206.220.41,"{""location"": ""NP"", ""is_mobile"": false}" 3583,2,78,2017-03-14 09:12:09,http://marquardt.biz/liam.mills,0.0764839083,7.39.185.147,"{""location"": ""GY"", ""is_mobile"": false}" 3584,2,78,2017-02-07 01:15:41,http://armstrong.com/augustine_thompson,0.6219164065,82.252.36.10,"{""location"": ""LR"", ""is_mobile"": true}" 3585,2,78,2016-12-20 00:34:03,http://bogisich.name/isabell,0.7597031035,39.207.194.120,"{""location"": ""DK"", ""is_mobile"": false}" 3586,2,78,2016-12-25 00:54:39,http://sporer.name/cathryn_beer,0.3494657758,31.43.150.207,"{""location"": ""AR"", ""is_mobile"": false}" 3587,2,78,2017-05-18 22:45:18,http://rice.net/jaden.beier,0.8690038546,238.126.151.224,"{""location"": ""FK"", ""is_mobile"": true}" 3588,2,78,2017-05-15 11:07:12,http://moore.io/edgar_paucek,0.7622942359,152.147.186.208,"{""location"": ""IS"", ""is_mobile"": true}" 3589,2,78,2017-05-05 18:03:36,http://marvinmurphy.org/valentine.buckridge,0.7924143528,137.136.228.28,"{""location"": ""IL"", ""is_mobile"": true}" 3590,2,78,2016-12-15 22:25:23,http://yundt.biz/dixie.weinat,0.2856726376,30.183.44.48,"{""location"": ""RO"", ""is_mobile"": false}" 3591,2,78,2017-01-25 13:44:13,http://rohancole.io/cydney,0.6165229552,103.221.90.201,"{""location"": ""PE"", ""is_mobile"": true}" 3592,2,78,2017-05-18 06:32:21,http://wolffmcdermott.co/brenda,0.0943098887,194.225.252.247,"{""location"": ""PN"", ""is_mobile"": false}" 3593,2,78,2017-04-12 13:07:54,http://stoltenberg.org/jena_brakus,0.0973377316,40.161.157.42,"{""location"": ""TW"", ""is_mobile"": false}" 3594,2,78,2017-01-25 14:01:57,http://batz.io/freddie_streich,0.0089402871,139.234.47.42,"{""location"": ""ID"", ""is_mobile"": false}" 3595,2,78,2017-05-24 02:58:13,http://feil.com/vallie.mayer,0.2172398835,117.10.186.121,"{""location"": ""LB"", ""is_mobile"": false}" 3596,2,78,2017-02-18 23:43:33,http://okonlueilwitz.com/cory.stanton,0.4150208213,222.165.33.19,"{""location"": ""JP"", ""is_mobile"": true}" 3597,2,78,2017-05-13 09:48:09,http://ledner.info/katelynn.wyman,0.1625432812,48.9.243.84,"{""location"": ""LS"", ""is_mobile"": false}" 3598,2,78,2017-03-05 08:03:21,http://fay.com/simeon,0.2633522831,146.252.155.201,"{""location"": ""PA"", ""is_mobile"": false}" 3599,2,79,2017-04-17 13:53:10,http://herman.biz/blanca,0.1521463905,48.206.254.168,"{""location"": ""BL"", ""is_mobile"": false}" 3600,2,79,2017-06-08 22:19:34,http://kemmer.name/sammie.king,0.1008228036,166.101.124.54,"{""location"": ""DO"", ""is_mobile"": false}" 3601,2,79,2017-02-21 13:43:45,http://heller.org/webster,0.5002912926,248.22.19.229,"{""location"": ""VG"", ""is_mobile"": true}" 3602,2,79,2017-03-15 06:03:38,http://lakin.net/luella,0.1469201658,93.154.167.78,"{""location"": ""WS"", ""is_mobile"": false}" 3603,2,79,2017-01-24 01:35:32,http://schuster.co/eric.sawayn,0.4941361290,225.143.52.14,"{""location"": ""BN"", ""is_mobile"": true}" 3604,2,79,2017-02-16 23:08:53,http://bechtelar.name/zoe,0.9550003363,212.132.67.78,"{""location"": ""SG"", ""is_mobile"": true}" 3605,2,79,2017-01-26 22:34:36,http://yostgrimes.org/justina.rolfson,0.9142675474,107.233.177.55,"{""location"": ""TN"", ""is_mobile"": false}" 3606,2,79,2017-01-29 12:52:04,http://schillerziemann.com/haven.monahan,0.8873877151,76.14.215.158,"{""location"": ""GN"", ""is_mobile"": false}" 3607,2,79,2017-03-15 14:17:09,http://yost.io/dane_miller,0.1102816271,182.103.66.111,"{""location"": ""PA"", ""is_mobile"": false}" 3608,2,79,2016-12-28 17:25:32,http://lehner.net/darrel,0.8827334065,191.99.253.155,"{""location"": ""TW"", ""is_mobile"": true}" 3609,2,79,2017-03-08 14:57:01,http://raynor.info/buddy.stehr,0.5616336583,128.234.75.248,"{""location"": ""SG"", ""is_mobile"": false}" 3610,2,79,2017-02-25 01:20:05,http://dickinson.com/annalise,0.6406560648,22.95.101.215,"{""location"": ""NI"", ""is_mobile"": true}" 3611,2,79,2017-01-30 18:24:55,http://ernser.net/demetrius,0.0511146229,48.130.92.134,"{""location"": ""LB"", ""is_mobile"": false}" 3612,2,79,2017-01-28 07:27:53,http://kris.com/ransom,0.7012343285,146.133.18.212,"{""location"": ""MO"", ""is_mobile"": false}" 3613,2,79,2017-06-08 19:17:40,http://cristblanda.com/angeline_gaylord,0.7385760602,8.137.172.246,"{""location"": ""AE"", ""is_mobile"": false}" 3614,2,79,2017-05-06 07:29:24,http://considinehilpert.name/duane_ruel,0.0873713664,176.76.241.79,"{""location"": ""CK"", ""is_mobile"": false}" 3615,2,79,2017-05-01 22:02:50,http://boyeroconner.com/mike,0.2064206664,167.77.167.229,"{""location"": ""RU"", ""is_mobile"": true}" 3616,2,79,2017-03-14 00:22:17,http://prohaska.biz/jeff,0.1428393160,141.191.236.178,"{""location"": ""LR"", ""is_mobile"": false}" 3617,2,79,2017-04-27 21:48:09,http://schultz.info/zoe_gulgowski,0.6046281189,174.113.39.170,"{""location"": ""MN"", ""is_mobile"": true}" 3618,2,79,2017-01-05 09:12:17,http://okuneva.co/bethany_jacobson,0.8969673267,253.66.70.35,"{""location"": ""SY"", ""is_mobile"": false}" 3619,2,79,2017-04-09 11:28:44,http://hicklestiedemann.org/abbigail,0.7665452472,124.138.218.4,"{""location"": ""FO"", ""is_mobile"": true}" 3620,2,79,2017-03-26 06:26:22,http://boyermoriette.name/jeanette,0.3316051296,140.245.156.121,"{""location"": ""SZ"", ""is_mobile"": false}" 3621,2,79,2017-05-25 03:22:17,http://ziemanntromp.io/lucy,0.8176562502,82.80.89.38,"{""location"": ""BE"", ""is_mobile"": false}" 3622,2,79,2017-05-15 10:07:43,http://wittingbode.biz/ferne.carroll,0.0739457834,139.243.83.26,"{""location"": ""IE"", ""is_mobile"": true}" 3623,2,79,2017-05-12 20:26:53,http://gulgowskizulauf.com/jettie_kreiger,0.9327555255,63.238.24.176,"{""location"": ""GA"", ""is_mobile"": false}" 3624,2,79,2017-02-14 09:48:02,http://weinat.name/fatima,0.9276834610,21.105.201.100,"{""location"": ""SL"", ""is_mobile"": false}" 3625,2,79,2017-06-13 17:54:38,http://ritchiegleichner.org/betty,0.6594940822,69.223.111.127,"{""location"": ""BY"", ""is_mobile"": true}" 3626,2,79,2017-04-20 09:58:37,http://howell.com/lamont,0.9814672343,249.78.51.25,"{""location"": ""GH"", ""is_mobile"": false}" 3627,2,79,2017-04-28 22:21:45,http://beer.name/sigrid,0.1406056101,55.151.151.132,"{""location"": ""JE"", ""is_mobile"": true}" 10507,4,236,2017-03-08 00:27:33,http://denesik.org/earnest,0.4804708479,151.155.96.84,"{""location"": ""ST"", ""is_mobile"": false}" 10508,4,236,2017-02-04 22:45:01,http://pollich.biz/nestor,0.8781342456,213.21.224.57,"{""location"": ""PF"", ""is_mobile"": true}" 10509,4,236,2017-01-05 17:22:40,http://hackett.info/theresia,0.3041773127,138.214.58.176,"{""location"": ""DM"", ""is_mobile"": true}" 10510,4,236,2017-03-11 06:29:58,http://ward.io/stone_kunde,0.7171746772,169.42.171.177,"{""location"": ""KM"", ""is_mobile"": false}" 10511,4,236,2017-04-21 22:16:45,http://pouros.biz/baylee_kris,0.3358408031,136.64.113.242,"{""location"": ""ET"", ""is_mobile"": true}" 10512,4,236,2017-05-12 17:22:31,http://weinat.org/audrey,0.0395049489,64.120.197.228,"{""location"": ""AM"", ""is_mobile"": true}" 10513,4,236,2017-03-21 03:39:15,http://dooley.info/arne.dicki,0.8566855584,77.48.56.33,"{""location"": ""GR"", ""is_mobile"": false}" 10514,4,236,2017-01-15 13:45:14,http://heaney.co/salma.towne,0.5998052326,37.114.76.236,"{""location"": ""MU"", ""is_mobile"": false}" 10515,4,236,2017-01-11 00:52:23,http://vandervort.org/serenity,0.4032090139,145.99.68.230,"{""location"": ""SI"", ""is_mobile"": false}" 10516,4,236,2017-04-24 05:50:34,http://nicolas.info/madalyn.simonis,0.5747144927,71.129.31.189,"{""location"": ""PS"", ""is_mobile"": false}" 10517,4,236,2017-01-25 02:41:05,http://koelpin.info/jameson.veum,0.1142909061,59.170.6.150,"{""location"": ""CM"", ""is_mobile"": false}" 10518,4,236,2017-02-11 01:50:42,http://westkozey.org/sage_braun,0.5506491675,11.238.124.121,"{""location"": ""ZA"", ""is_mobile"": false}" 10519,4,236,2017-01-28 20:19:29,http://howe.info/bert_boyer,0.2586218915,240.12.8.151,"{""location"": ""YT"", ""is_mobile"": true}" 10520,4,236,2017-01-12 04:13:49,http://anderson.info/wilburn,0.2555767503,27.51.186.72,"{""location"": ""GE"", ""is_mobile"": true}" 10521,4,236,2017-06-07 12:07:35,http://jaskolski.biz/lacey.balistreri,0.1809298892,209.47.237.171,"{""location"": ""KE"", ""is_mobile"": true}" 10522,4,236,2017-01-21 23:24:32,http://ernserfritsch.info/brigitte_berge,0.6323257516,212.242.45.211,"{""location"": ""LU"", ""is_mobile"": false}" 10523,4,236,2017-03-18 06:14:30,http://kohlermohr.info/gloria_hoeger,0.5609849873,185.153.147.209,"{""location"": ""GY"", ""is_mobile"": true}" 10524,4,236,2017-03-26 22:34:25,http://hackettharris.org/sienna_lehner,0.8158671470,119.25.119.4,"{""location"": ""RE"", ""is_mobile"": false}" 10525,4,236,2017-01-04 14:57:52,http://huelcorwin.name/price_huel,0.2985459190,92.77.115.111,"{""location"": ""PE"", ""is_mobile"": true}" 10526,4,236,2016-12-20 21:01:48,http://lehner.name/ryder.auer,0.3975976496,136.151.222.224,"{""location"": ""GN"", ""is_mobile"": true}" 10527,4,236,2017-04-01 18:48:12,http://koeppmiller.com/makenna,0.3918116529,69.139.138.79,"{""location"": ""MX"", ""is_mobile"": true}" 10528,4,236,2017-02-11 09:59:16,http://rippin.co/johanna,0.6075352802,148.195.106.87,"{""location"": ""ST"", ""is_mobile"": false}" 10529,4,236,2017-04-15 17:59:47,http://farrell.org/gladyce,0.7422126487,147.162.155.188,"{""location"": ""IN"", ""is_mobile"": true}" 10530,4,236,2017-01-09 07:19:03,http://donnelly.net/piper_effertz,0.5163013935,157.186.65.105,"{""location"": ""TN"", ""is_mobile"": false}" 10531,4,236,2017-03-29 12:21:18,http://kuvalichiller.co/montana,0.2173230074,222.171.193.205,"{""location"": ""MQ"", ""is_mobile"": true}" 10532,4,236,2017-01-09 16:31:39,http://bayer.biz/ronaldo,0.6960075351,245.242.225.5,"{""location"": ""CR"", ""is_mobile"": false}" 10533,4,236,2017-04-20 20:04:54,http://klockorowe.io/dino,0.5428317097,34.109.65.150,"{""location"": ""UM"", ""is_mobile"": false}" 10534,4,236,2017-01-04 01:59:46,http://lind.com/zoie.hane,0.9167601343,93.34.170.157,"{""location"": ""ID"", ""is_mobile"": false}" 10535,4,236,2017-04-16 07:10:29,http://adams.info/pierre_langosh,0.2239015300,109.109.27.169,"{""location"": ""WS"", ""is_mobile"": true}" 10536,4,236,2017-04-06 05:59:07,http://rice.net/hillard.ledner,0.4194894456,249.245.252.140,"{""location"": ""NU"", ""is_mobile"": true}" 10537,4,236,2017-02-04 11:38:09,http://kaulke.io/maegan,0.8894278319,250.219.34.205,"{""location"": ""SD"", ""is_mobile"": false}" 10538,4,236,2017-03-25 00:41:30,http://bashiriankulas.net/domingo.aufderhar,0.3949766793,43.30.240.245,"{""location"": ""SK"", ""is_mobile"": false}" 10539,4,236,2017-04-10 22:42:14,http://schultzdare.io/gilda,0.9564966603,60.39.235.29,"{""location"": ""AO"", ""is_mobile"": true}" 10540,4,236,2016-12-26 01:28:21,http://mann.info/demetris,0.2244504561,115.130.111.243,"{""location"": ""HK"", ""is_mobile"": false}" 10541,4,236,2017-02-16 02:44:35,http://herman.org/rosendo,0.3690045413,107.249.200.220,"{""location"": ""CL"", ""is_mobile"": false}" 10542,4,236,2017-02-06 01:44:52,http://fritsch.biz/eldon.harber,0.9712747596,132.48.4.249,"{""location"": ""BJ"", ""is_mobile"": true}" 10543,4,236,2017-04-09 04:24:17,http://muller.org/dora,0.3793870202,44.229.64.145,"{""location"": ""TW"", ""is_mobile"": false}" 10544,4,236,2017-01-30 10:32:04,http://wolff.info/deonte.gutkowski,0.7486952481,59.228.176.39,"{""location"": ""BA"", ""is_mobile"": false}" 10545,4,236,2016-12-18 18:34:49,http://kubwilderman.io/ole.homenick,0.2491658069,224.250.243.190,"{""location"": ""NR"", ""is_mobile"": false}" 10546,4,236,2017-03-14 15:10:31,http://pfannerstill.name/velma.kuphal,0.5391647786,6.231.235.216,"{""location"": ""BY"", ""is_mobile"": true}" 10547,4,236,2017-02-03 20:32:00,http://waelchi.name/thaddeus_christiansen,0.0753712235,151.227.97.111,"{""location"": ""MO"", ""is_mobile"": false}" 10548,4,236,2017-05-23 02:33:14,http://windlerwintheiser.net/grayce,0.6054031355,163.22.233.39,"{""location"": ""MP"", ""is_mobile"": true}" 10549,4,236,2017-01-04 08:38:37,http://brownhintz.io/erna,0.9204305397,171.219.161.233,"{""location"": ""NA"", ""is_mobile"": false}" 10550,4,236,2017-04-03 07:28:26,http://tromp.co/christine_quitzon,0.6780338284,234.20.224.247,"{""location"": ""AL"", ""is_mobile"": false}" 10551,4,237,2017-01-09 02:06:28,http://prosacco.biz/tyson_prohaska,0.6117945758,198.85.138.85,"{""location"": ""LB"", ""is_mobile"": false}" 10552,4,237,2017-02-19 05:26:03,http://osinskirice.biz/adalberto_feeney,0.9402731208,135.122.91.40,"{""location"": ""DJ"", ""is_mobile"": true}" 10553,4,237,2017-06-10 07:10:39,http://mccullough.io/shaniya.okon,0.4881826648,15.86.169.226,"{""location"": ""MT"", ""is_mobile"": false}" 10554,4,237,2017-06-13 05:16:41,http://hintz.biz/tina,0.4732681860,137.214.194.122,"{""location"": ""TJ"", ""is_mobile"": true}" 10555,4,237,2017-04-09 18:36:40,http://bernier.name/brook_barrows,0.4339356897,14.38.132.158,"{""location"": ""PS"", ""is_mobile"": true}" 10556,4,237,2017-03-01 18:01:16,http://ricegerlach.co/rosa,0.3613574377,129.241.254.190,"{""location"": ""FR"", ""is_mobile"": true}" 10557,4,237,2016-12-22 21:28:12,http://daughertysteuber.net/tatyana,0.1559138895,91.219.228.156,"{""location"": ""AG"", ""is_mobile"": false}" 7539,3,167,2017-03-21 10:47:56,http://legrosnitzsche.net/frederique_renner,,241.192.123.113,"{""location"": ""AW"", ""is_mobile"": true}" 7540,3,167,2017-03-03 21:33:26,http://danielruel.com/edgar,,154.104.114.243,"{""location"": ""DM"", ""is_mobile"": false}" 7541,3,167,2017-05-05 21:15:21,http://vonrueden.biz/kimberly,,244.37.6.95,"{""location"": ""GT"", ""is_mobile"": false}" 7542,3,167,2017-02-19 21:13:58,http://moen.name/norwood,,131.46.109.88,"{""location"": ""PF"", ""is_mobile"": true}" 7543,3,167,2017-01-10 07:31:32,http://kunzerunolfsdottir.net/cale.schoen,,86.147.185.199,"{""location"": ""KE"", ""is_mobile"": true}" 7544,3,167,2017-03-20 19:35:33,http://kuhlmanmertz.info/rhianna,,18.16.22.26,"{""location"": ""IS"", ""is_mobile"": false}" 7545,3,167,2017-04-02 12:36:56,http://gleichnergreenfelder.net/ruth_frami,,175.5.69.70,"{""location"": ""YE"", ""is_mobile"": true}" 7546,3,167,2017-02-04 22:22:23,http://hirthe.io/chet,,186.250.243.202,"{""location"": ""PY"", ""is_mobile"": true}" 7547,3,167,2017-01-22 01:02:30,http://paucekschulist.net/dan,,90.238.102.183,"{""location"": ""NE"", ""is_mobile"": false}" 7548,3,167,2017-06-13 18:32:46,http://murphyvolkman.com/chasity,,185.142.116.47,"{""location"": ""ID"", ""is_mobile"": true}" 7549,3,167,2017-05-06 04:19:11,http://douglas.info/thea,,79.55.231.21,"{""location"": ""SX"", ""is_mobile"": false}" 7550,3,167,2017-02-02 01:14:13,http://schoen.biz/elody_hilpert,,93.160.89.195,"{""location"": ""NL"", ""is_mobile"": false}" 7551,3,167,2017-03-15 06:26:54,http://sporerfisher.info/benny,,18.131.50.106,"{""location"": ""SM"", ""is_mobile"": false}" 7552,3,167,2017-06-10 13:27:56,http://pfannerstill.co/agnes.brakus,,157.118.245.133,"{""location"": ""AX"", ""is_mobile"": false}" 7553,3,167,2017-01-14 21:57:56,http://lednerhane.com/annamarie,,146.172.232.54,"{""location"": ""LA"", ""is_mobile"": true}" 7554,3,167,2017-05-04 17:09:31,http://mcclure.io/trystan,,157.141.240.185,"{""location"": ""AT"", ""is_mobile"": false}" 7555,3,167,2016-12-15 06:40:08,http://kshlerin.net/hallie,,183.20.61.70,"{""location"": ""SK"", ""is_mobile"": false}" 7556,3,167,2017-01-09 16:54:29,http://murphy.io/gladyce,,96.151.34.15,"{""location"": ""SM"", ""is_mobile"": false}" 7557,3,167,2017-03-15 11:30:04,http://dietrichkreiger.com/pansy,,254.235.69.78,"{""location"": ""UZ"", ""is_mobile"": false}" 7558,3,167,2017-05-12 08:27:44,http://kshlerinstehr.biz/brock,,108.35.59.223,"{""location"": ""CA"", ""is_mobile"": false}" 7559,3,167,2016-12-23 19:06:31,http://littelryan.io/ron,,120.127.145.182,"{""location"": ""MZ"", ""is_mobile"": true}" 7560,3,167,2017-04-10 05:30:18,http://kris.co/viva,,85.184.244.18,"{""location"": ""TJ"", ""is_mobile"": false}" 7561,3,167,2017-02-02 10:30:11,http://vonbartell.info/kirstin,,47.107.105.164,"{""location"": ""TV"", ""is_mobile"": true}" 7562,3,167,2016-12-16 01:41:49,http://moen.io/santa,,85.165.172.9,"{""location"": ""DZ"", ""is_mobile"": false}" 7563,3,167,2017-04-26 02:53:49,http://rolfson.co/henry.dicki,,43.72.19.9,"{""location"": ""PK"", ""is_mobile"": false}" 7564,3,167,2017-01-19 02:16:51,http://roberts.net/olga.schulist,,191.185.150.88,"{""location"": ""VG"", ""is_mobile"": false}" 7565,3,167,2017-03-17 17:22:42,http://monahanfisher.biz/joelle,,43.67.150.140,"{""location"": ""CG"", ""is_mobile"": false}" 7566,3,167,2017-01-25 04:17:26,http://schneider.biz/vivien,,116.118.21.149,"{""location"": ""BA"", ""is_mobile"": false}" 7567,3,167,2017-06-11 03:02:42,http://lehner.biz/claire,,26.154.160.107,"{""location"": ""EC"", ""is_mobile"": false}" 7568,3,167,2017-03-31 00:39:38,http://reinger.biz/darrel,,23.7.7.78,"{""location"": ""LU"", ""is_mobile"": false}" 7569,3,167,2017-04-01 21:55:46,http://hettinger.net/deontae,,159.8.116.11,"{""location"": ""PK"", ""is_mobile"": false}" 7570,3,167,2017-04-03 11:06:15,http://smitham.net/koby_cain,,40.179.192.232,"{""location"": ""KN"", ""is_mobile"": false}" 7571,3,167,2017-06-01 00:26:02,http://leuschkeleannon.info/johnathan_smitham,,138.97.60.253,"{""location"": ""UM"", ""is_mobile"": true}" 7572,3,167,2017-02-09 01:51:50,http://emard.com/gwendolyn.gleason,,149.116.226.218,"{""location"": ""KG"", ""is_mobile"": true}" 7573,3,167,2017-03-02 10:21:46,http://fisherkoepp.org/wilson_schmeler,,12.45.224.103,"{""location"": ""QA"", ""is_mobile"": false}" 7574,3,167,2017-02-06 07:52:56,http://block.io/nova.shanahan,,118.2.124.206,"{""location"": ""ZA"", ""is_mobile"": true}" 7575,3,167,2017-04-07 07:31:41,http://beckersawayn.net/clint,,216.141.214.21,"{""location"": ""TD"", ""is_mobile"": false}" 7576,3,167,2016-12-26 17:39:24,http://beahan.biz/otho,,129.81.49.54,"{""location"": ""BT"", ""is_mobile"": true}" 7577,3,167,2017-03-31 20:43:49,http://stehrspencer.info/emmalee.mraz,,113.86.72.120,"{""location"": ""BE"", ""is_mobile"": true}" 7578,3,167,2017-05-17 12:46:07,http://swaniawski.name/luna_stanton,,156.27.85.27,"{""location"": ""ML"", ""is_mobile"": false}" 7579,3,167,2017-05-15 02:25:33,http://lebsack.io/ally,,102.90.75.104,"{""location"": ""MV"", ""is_mobile"": true}" 7580,3,167,2017-05-12 17:07:46,http://rutherford.net/haylie,,100.238.132.169,"{""location"": ""MF"", ""is_mobile"": false}" 7581,3,167,2017-06-02 07:50:53,http://erdman.org/freddy.gulgowski,,59.91.247.217,"{""location"": ""MK"", ""is_mobile"": true}" 7582,3,167,2017-01-27 02:05:52,http://cummerataernser.net/leslie_towne,,101.43.137.200,"{""location"": ""BG"", ""is_mobile"": false}" 7583,3,167,2017-02-28 01:51:59,http://parker.org/melvin.kuhlman,,40.19.53.149,"{""location"": ""CI"", ""is_mobile"": true}" 7584,3,167,2017-04-30 03:02:36,http://beerboyle.biz/jackie,,15.127.67.229,"{""location"": ""SD"", ""is_mobile"": false}" 7585,3,167,2017-03-12 11:01:20,http://lefflerhermiston.com/alexanne.crooks,,42.6.226.168,"{""location"": ""NI"", ""is_mobile"": false}" 7586,3,167,2017-04-05 20:23:00,http://gloverhoppe.info/araceli,,252.42.29.165,"{""location"": ""HU"", ""is_mobile"": true}" 7587,3,167,2017-04-29 06:45:15,http://waelchi.io/ena.metz,,135.85.32.29,"{""location"": ""LK"", ""is_mobile"": false}" 7588,3,167,2017-01-09 02:38:44,http://framifarrell.biz/tara_ebert,,13.191.206.226,"{""location"": ""CD"", ""is_mobile"": true}" 7589,3,167,2017-03-10 22:12:28,http://bernhard.co/michaela,,179.3.184.72,"{""location"": ""ER"", ""is_mobile"": false}" 7590,3,167,2017-03-17 19:13:04,http://schulist.org/marlene.hayes,,238.158.183.12,"{""location"": ""IO"", ""is_mobile"": false}" 7591,3,168,2017-01-21 16:35:20,http://corkeryschmidt.com/isom,,227.163.227.236,"{""location"": ""BB"", ""is_mobile"": true}" 7592,3,168,2017-04-07 05:39:20,http://hammes.info/danika,,23.122.221.36,"{""location"": ""SL"", ""is_mobile"": false}" 7593,3,168,2017-03-06 03:46:24,http://flatley.co/sabrina.stehr,,5.119.158.233,"{""location"": ""KM"", ""is_mobile"": true}" 7594,3,168,2016-12-15 16:52:49,http://grady.biz/fredrick,,51.4.89.107,"{""location"": ""GM"", ""is_mobile"": true}" 13450,5,302,2016-12-31 19:05:17,http://ruel.com/thurman.kautzer,,102.47.222.92,"{""location"": ""CM"", ""is_mobile"": false}" 13451,5,302,2017-01-11 14:48:49,http://muellerweimann.name/webster.hudson,,18.31.19.122,"{""location"": ""PS"", ""is_mobile"": true}" 13452,5,302,2017-01-08 01:02:35,http://marks.com/lilyan_grant,,144.251.128.15,"{""location"": ""BN"", ""is_mobile"": true}" 13453,5,302,2017-03-04 06:24:36,http://luettgen.org/vickie_bahringer,,150.17.50.16,"{""location"": ""CO"", ""is_mobile"": true}" 13454,5,302,2017-01-13 04:14:19,http://baileyjohnston.co/duane.trantow,,22.186.208.123,"{""location"": ""UM"", ""is_mobile"": false}" 13455,5,302,2017-01-23 23:55:13,http://funkhartmann.com/london,,109.65.129.209,"{""location"": ""LV"", ""is_mobile"": true}" 13456,5,302,2017-04-17 01:38:54,http://rolfsonterry.info/arden,,68.98.57.227,"{""location"": ""MU"", ""is_mobile"": false}" 13457,5,302,2017-04-12 13:38:45,http://lang.info/lucienne,,174.173.94.154,"{""location"": ""NC"", ""is_mobile"": false}" 13458,5,302,2016-12-28 09:25:43,http://skilesritchie.io/brannon_keeling,,172.251.86.153,"{""location"": ""GN"", ""is_mobile"": false}" 13459,5,302,2017-03-01 02:37:08,http://hirthe.co/diamond,,136.99.43.123,"{""location"": ""MH"", ""is_mobile"": false}" 13460,5,302,2016-12-29 17:11:02,http://hahn.org/luz,,106.189.146.206,"{""location"": ""CH"", ""is_mobile"": false}" 13461,5,302,2017-05-04 00:53:45,http://grimes.name/reva,,212.159.124.108,"{""location"": ""WF"", ""is_mobile"": false}" 13462,5,302,2017-01-22 15:36:53,http://johnsonankunding.co/uriah,,162.66.164.70,"{""location"": ""MH"", ""is_mobile"": true}" 13463,5,302,2016-12-14 14:10:19,http://shields.org/estrella,,188.133.214.109,"{""location"": ""GW"", ""is_mobile"": true}" 13464,5,302,2016-12-20 21:40:51,http://pfefferhansen.co/garfield,,153.150.158.131,"{""location"": ""DK"", ""is_mobile"": true}" 13465,5,302,2017-04-06 00:52:48,http://smith.info/dakota,,172.18.19.141,"{""location"": ""MD"", ""is_mobile"": false}" 13466,5,302,2017-04-15 06:37:32,http://gutmannherzog.net/antwan,,61.205.74.92,"{""location"": ""SM"", ""is_mobile"": true}" 13467,5,302,2016-12-19 08:16:12,http://pfannerstillgibson.info/jamison,,4.56.111.66,"{""location"": ""LT"", ""is_mobile"": false}" 13468,5,302,2017-04-25 07:22:04,http://daniel.info/robbie,,167.14.99.139,"{""location"": ""EH"", ""is_mobile"": false}" 13469,5,302,2016-12-14 19:34:54,http://schneider.io/kasandra.homenick,,28.233.200.67,"{""location"": ""LI"", ""is_mobile"": true}" 13470,5,302,2016-12-17 16:24:31,http://gusikowskirice.info/vidal,,127.247.209.207,"{""location"": ""UA"", ""is_mobile"": false}" 13471,5,302,2017-04-16 04:10:53,http://koch.biz/oswald,,165.32.192.176,"{""location"": ""LT"", ""is_mobile"": false}" 13472,5,302,2017-04-29 07:22:55,http://metz.biz/trycia,,245.56.34.204,"{""location"": ""LB"", ""is_mobile"": false}" 13473,5,302,2017-01-04 18:37:15,http://romaguera.co/mackenzie_kreiger,,3.119.182.223,"{""location"": ""RE"", ""is_mobile"": false}" 13474,5,303,2017-05-21 18:38:16,http://mcclure.com/al_kuhn,,149.50.71.231,"{""location"": ""PS"", ""is_mobile"": false}" 13475,5,303,2016-12-25 15:59:04,http://schroeder.biz/hermann,,3.61.80.38,"{""location"": ""VA"", ""is_mobile"": false}" 13476,5,303,2017-02-22 17:56:44,http://kuphal.io/paige,,106.113.121.215,"{""location"": ""TK"", ""is_mobile"": false}" 13477,5,303,2017-03-11 08:02:02,http://watsicaluettgen.com/ceasar,,108.78.81.221,"{""location"": ""MO"", ""is_mobile"": true}" 13478,5,303,2017-04-14 01:29:22,http://balistrerihauck.net/jonathan,,31.112.84.164,"{""location"": ""MP"", ""is_mobile"": false}" 13479,5,303,2017-02-26 23:36:04,http://skiles.org/rick,,224.90.239.171,"{""location"": ""TW"", ""is_mobile"": false}" 13480,5,303,2016-12-30 15:04:12,http://stracke.co/maia_stroman,,166.26.105.80,"{""location"": ""MH"", ""is_mobile"": false}" 13481,5,303,2017-05-21 04:36:58,http://medhurst.co/mina.johnston,,142.39.246.161,"{""location"": ""DK"", ""is_mobile"": false}" 13482,5,303,2017-06-06 06:50:52,http://steuber.info/ena_heel,,197.230.37.101,"{""location"": ""GI"", ""is_mobile"": true}" 13483,5,303,2017-01-04 12:24:07,http://conn.com/ed,,180.72.78.65,"{""location"": ""MR"", ""is_mobile"": false}" 13484,5,303,2017-03-16 19:00:51,http://hirthe.biz/carlo.conroy,,94.254.194.208,"{""location"": ""HT"", ""is_mobile"": false}" 13485,5,303,2017-01-31 12:22:27,http://klein.net/fredrick.spinka,,64.175.131.167,"{""location"": ""UM"", ""is_mobile"": false}" 13486,5,303,2017-04-25 18:17:28,http://roberts.net/evan.auer,,106.25.55.179,"{""location"": ""DK"", ""is_mobile"": true}" 13487,5,303,2017-05-06 03:15:05,http://naderhalvorson.co/hilbert,,2.34.91.187,"{""location"": ""DJ"", ""is_mobile"": false}" 13488,5,303,2017-01-11 15:15:31,http://gislason.org/yeenia.paucek,,169.145.8.241,"{""location"": ""MX"", ""is_mobile"": true}" 13489,5,303,2017-03-03 00:37:17,http://sawaynruel.name/erick.shields,,31.146.142.235,"{""location"": ""BJ"", ""is_mobile"": false}" 13490,5,303,2017-01-12 14:49:38,http://olson.co/kamren,,3.224.139.96,"{""location"": ""SH"", ""is_mobile"": true}" 13491,5,303,2017-01-21 18:41:47,http://langworth.io/lonzo,,61.219.112.200,"{""location"": ""SG"", ""is_mobile"": true}" 13492,5,303,2017-01-29 09:01:01,http://sengerbergstrom.biz/maynard,,65.236.46.38,"{""location"": ""TG"", ""is_mobile"": false}" 13493,5,303,2017-01-28 00:20:32,http://turcottedooley.biz/opal,,195.218.102.28,"{""location"": ""TK"", ""is_mobile"": true}" 13494,5,303,2017-05-23 01:22:52,http://langosh.com/verna.sawayn,,109.227.89.243,"{""location"": ""BH"", ""is_mobile"": false}" 13495,5,303,2017-05-03 09:42:27,http://hills.net/camryn,,154.232.119.192,"{""location"": ""RO"", ""is_mobile"": true}" 13496,5,303,2017-02-11 02:13:27,http://stoltenberg.name/susana.ruel,,226.22.52.77,"{""location"": ""IM"", ""is_mobile"": true}" 13497,5,303,2017-02-19 09:17:14,http://thiel.org/bradly,,83.153.176.64,"{""location"": ""MR"", ""is_mobile"": true}" 13498,5,303,2017-02-18 20:49:48,http://mayerbednar.info/sabina,,75.10.176.51,"{""location"": ""IQ"", ""is_mobile"": false}" 13499,5,303,2017-05-15 06:15:26,http://frami.com/norbert,,150.108.112.226,"{""location"": ""GH"", ""is_mobile"": true}" 13500,5,303,2017-02-15 15:39:56,http://kozeycummerata.org/emil_rodriguez,,133.118.156.237,"{""location"": ""MQ"", ""is_mobile"": true}" 13501,5,303,2016-12-27 21:00:41,http://kuhn.name/tad_schowalter,,147.41.134.210,"{""location"": ""DK"", ""is_mobile"": false}" 13502,5,303,2017-01-26 16:26:54,http://king.net/lucinda_jenkins,,239.33.209.182,"{""location"": ""GM"", ""is_mobile"": false}" 13503,5,303,2017-01-23 12:06:15,http://jerde.org/christopher_ward,,145.2.63.170,"{""location"": ""YE"", ""is_mobile"": true}" 13504,5,303,2017-04-30 16:18:22,http://harris.org/tremaine_raynor,,101.182.143.244,"{""location"": ""PM"", ""is_mobile"": false}" 16449,6,372,2017-06-02 06:25:17,http://muller.com/quinton,0.1973752570,116.37.229.156,"{""location"": ""QA"", ""is_mobile"": false}" 16450,6,372,2017-03-06 04:37:14,http://wintheiserbrown.co/vincenzo,0.2030002086,111.16.62.83,"{""location"": ""TZ"", ""is_mobile"": true}" 16451,6,372,2017-04-10 08:03:47,http://kutch.org/sabina,0.2000365490,68.158.176.63,"{""location"": ""KE"", ""is_mobile"": false}" 16452,6,372,2017-03-16 10:50:53,http://quitzoncollier.org/pete,0.9621480611,75.149.41.33,"{""location"": ""BR"", ""is_mobile"": true}" 16453,6,372,2017-04-22 09:34:23,http://cartwrightwalter.info/wilber,0.0266869107,86.125.125.209,"{""location"": ""MG"", ""is_mobile"": true}" 16454,6,372,2017-05-18 12:19:10,http://emard.biz/lemuel,0.8823735958,254.48.59.33,"{""location"": ""MH"", ""is_mobile"": true}" 17072,6,386,2017-02-04 18:36:36,http://strosin.io/wayne,,216.248.126.173,"{""location"": ""ES"", ""is_mobile"": true}" 16455,6,372,2016-12-15 17:58:23,http://torp.org/morgan.kozey,0.7551455952,60.63.129.211,"{""location"": ""BB"", ""is_mobile"": false}" 16456,6,372,2017-03-18 08:58:39,http://macgyverchristiansen.name/alvah,0.2544162446,101.164.193.116,"{""location"": ""PN"", ""is_mobile"": true}" 16457,6,372,2017-01-18 01:12:40,http://jastwalker.net/kaitlyn,0.5104458740,81.252.22.64,"{""location"": ""NF"", ""is_mobile"": true}" 16458,6,372,2017-03-19 15:06:56,http://zulauf.co/jamir.rath,0.9617416213,116.168.139.36,"{""location"": ""RE"", ""is_mobile"": true}" 16459,6,372,2017-02-13 08:16:59,http://waterscremin.io/jackson,0.6614676721,184.40.190.106,"{""location"": ""FK"", ""is_mobile"": true}" 16460,6,372,2017-01-17 18:02:48,http://spencer.net/dedric_smitham,0.4800381135,103.7.149.107,"{""location"": ""BE"", ""is_mobile"": true}" 16461,6,372,2017-03-19 04:08:34,http://wolfleffler.co/christelle,0.4860522215,246.43.63.148,"{""location"": ""PL"", ""is_mobile"": false}" 16462,6,373,2017-03-12 06:09:02,http://ricekiehn.com/travon,0.1851442789,86.254.205.75,"{""location"": ""TZ"", ""is_mobile"": false}" 16463,6,373,2017-06-10 10:59:15,http://kshlerin.net/aisha,0.1888351377,74.160.71.71,"{""location"": ""NZ"", ""is_mobile"": false}" 16464,6,373,2017-02-14 12:58:00,http://blockmurray.co/carmine.volkman,0.1098374658,193.124.100.184,"{""location"": ""NC"", ""is_mobile"": true}" 16465,6,373,2017-04-02 03:22:30,http://dietrich.info/ruben.schumm,0.0120726040,134.175.156.205,"{""location"": ""NL"", ""is_mobile"": true}" 16466,6,373,2017-03-02 04:21:00,http://boehmschmidt.org/carole.sporer,0.3128258021,98.179.220.253,"{""location"": ""MV"", ""is_mobile"": false}" 16467,6,373,2017-03-09 11:50:21,http://fritschharris.co/felipa.durgan,0.2001753947,98.192.84.134,"{""location"": ""LR"", ""is_mobile"": false}" 16468,6,373,2017-04-30 16:35:03,http://beatty.io/griffin,0.9737877151,104.196.245.74,"{""location"": ""TK"", ""is_mobile"": false}" 16469,6,373,2017-03-19 09:32:40,http://wolff.net/sidney.runolfsdottir,0.8808831126,78.45.164.164,"{""location"": ""GB"", ""is_mobile"": true}" 16470,6,373,2017-05-03 19:29:26,http://torphyohara.net/demetris.veum,0.4468978187,202.225.174.118,"{""location"": ""GM"", ""is_mobile"": true}" 16471,6,373,2017-03-30 09:20:48,http://trompstanton.net/cruz_shields,0.7088947760,210.117.156.13,"{""location"": ""GI"", ""is_mobile"": true}" 16472,6,373,2016-12-30 14:43:46,http://willdaugherty.com/keshawn_bogan,0.5734123140,61.219.209.200,"{""location"": ""JP"", ""is_mobile"": true}" 16473,6,373,2017-03-16 01:29:47,http://faheyschinner.io/diamond,0.9750494630,193.26.213.185,"{""location"": ""GM"", ""is_mobile"": true}" 16474,6,373,2016-12-14 01:49:50,http://hauck.com/jeanie.greenfelder,0.8317289243,133.196.210.186,"{""location"": ""MF"", ""is_mobile"": false}" 16475,6,373,2017-01-10 20:47:45,http://mcglynn.io/wyatt.haley,0.7854649868,167.86.132.219,"{""location"": ""JE"", ""is_mobile"": false}" 16476,6,373,2017-06-06 21:05:38,http://schroeder.com/kory.schulist,0.3510878952,158.9.61.87,"{""location"": ""CX"", ""is_mobile"": false}" 16477,6,373,2017-05-31 18:55:09,http://kuhlman.com/jordane_mante,0.8611408105,160.243.154.144,"{""location"": ""VE"", ""is_mobile"": false}" 16478,6,373,2017-04-25 06:31:58,http://kuphal.com/tyler,0.8873420106,77.240.25.129,"{""location"": ""UM"", ""is_mobile"": false}" 16479,6,373,2017-03-15 20:51:34,http://vandervortwisozk.com/katelyn.torphy,0.0599523671,55.63.82.188,"{""location"": ""NL"", ""is_mobile"": false}" 16480,6,373,2017-01-04 07:54:29,http://cronin.co/grayce,0.7908632368,201.205.184.104,"{""location"": ""OM"", ""is_mobile"": false}" 16481,6,373,2017-06-12 22:37:22,http://senger.info/newell_metz,0.3048645868,250.63.49.39,"{""location"": ""GQ"", ""is_mobile"": true}" 16482,6,373,2017-04-21 23:21:41,http://grimes.net/amelia,0.1316567862,177.53.93.40,"{""location"": ""PG"", ""is_mobile"": true}" 16483,6,373,2017-01-14 18:33:05,http://pagacoconnell.com/arnaldo.gislason,0.7483362846,16.122.75.4,"{""location"": ""KW"", ""is_mobile"": false}" 16484,6,373,2017-02-07 08:13:49,http://cummerata.io/manuela.kunze,0.0023618376,159.75.33.117,"{""location"": ""SD"", ""is_mobile"": true}" 16485,6,373,2017-03-18 01:49:43,http://gaylord.net/shaina,0.0789131261,210.123.127.103,"{""location"": ""ID"", ""is_mobile"": true}" 16486,6,373,2017-01-02 04:15:53,http://tromp.net/tod.fay,0.8889491983,134.109.102.168,"{""location"": ""DZ"", ""is_mobile"": false}" 16487,6,373,2017-01-05 06:50:52,http://kohler.io/jerry,0.6844462692,253.134.126.231,"{""location"": ""GF"", ""is_mobile"": false}" 16488,6,373,2017-02-21 17:37:48,http://paucekhalvorson.biz/rollin,0.3516019262,225.145.155.239,"{""location"": ""PK"", ""is_mobile"": false}" 16489,6,373,2017-01-18 10:59:46,http://kovacek.co/caitlyn_lemke,0.5204691914,161.10.210.85,"{""location"": ""TT"", ""is_mobile"": false}" 16490,6,373,2017-04-09 04:07:36,http://lynchparisian.info/bart_purdy,0.1013379108,25.227.106.158,"{""location"": ""RE"", ""is_mobile"": true}" 16491,6,373,2017-01-20 05:38:50,http://murray.org/neoma.klein,0.1695164730,123.194.12.145,"{""location"": ""MY"", ""is_mobile"": true}" 16492,6,373,2017-01-20 07:29:48,http://mcclurerogahn.biz/perry.skiles,0.9867254774,222.216.220.76,"{""location"": ""PG"", ""is_mobile"": false}" 16493,6,373,2017-05-28 11:52:52,http://luettgen.co/alphonso,0.4440297784,231.105.185.213,"{""location"": ""BD"", ""is_mobile"": false}" 16494,6,373,2017-03-08 06:51:38,http://gislasonquitzon.name/sally,0.3633105596,117.188.38.90,"{""location"": ""BH"", ""is_mobile"": true}" 16495,6,374,2017-01-09 17:20:44,http://danielhauck.net/cicero,0.4490735170,38.245.146.125,"{""location"": ""NR"", ""is_mobile"": false}" 16496,6,374,2017-01-22 14:18:17,http://heathcote.io/yeenia,0.4122300320,97.197.28.238,"{""location"": ""EC"", ""is_mobile"": false}" 16497,6,374,2017-03-03 03:31:48,http://cummings.net/adrianna_klein,0.5954648125,251.90.115.227,"{""location"": ""NU"", ""is_mobile"": true}" 16498,6,374,2017-01-10 20:05:24,http://little.com/cloyd.walter,0.5211980380,65.177.90.215,"{""location"": ""ZA"", ""is_mobile"": false}" 3628,2,79,2016-12-21 05:59:25,http://crist.info/connie_bins,0.9138911387,129.203.143.170,"{""location"": ""CA"", ""is_mobile"": false}" 3629,2,80,2017-04-16 15:54:08,http://stokes.co/roman,0.8297012614,31.185.243.211,"{""location"": ""ME"", ""is_mobile"": true}" 3630,2,80,2017-02-18 14:32:47,http://binchneider.net/gerda,0.7488640118,211.249.152.246,"{""location"": ""AT"", ""is_mobile"": true}" 3631,2,80,2017-04-20 22:18:03,http://powlowski.com/lukas_kemmer,0.3134077379,34.228.237.7,"{""location"": ""CH"", ""is_mobile"": false}" 3632,2,80,2016-12-21 23:34:08,http://skilespredovic.name/matilde,0.5065128173,222.251.187.139,"{""location"": ""SD"", ""is_mobile"": true}" 3633,2,80,2016-12-28 10:28:58,http://feest.name/etha,0.0786364232,230.157.20.194,"{""location"": ""CF"", ""is_mobile"": true}" 3634,2,80,2017-05-20 00:26:37,http://mckenzie.org/chaya.douglas,0.3290803862,14.48.181.30,"{""location"": ""ZW"", ""is_mobile"": true}" 3635,2,80,2017-03-11 13:01:47,http://kautzer.org/kody.miller,0.4858843772,185.162.200.48,"{""location"": ""ST"", ""is_mobile"": false}" 3636,2,80,2017-01-29 22:41:27,http://douglas.net/hans,0.8825739187,6.136.229.84,"{""location"": ""ET"", ""is_mobile"": false}" 3637,2,80,2016-12-29 20:54:59,http://wolffrempel.net/kirsten,0.0850205734,115.32.178.142,"{""location"": ""SS"", ""is_mobile"": true}" 3638,2,80,2016-12-21 15:40:16,http://hoppe.io/greg,0.9871615284,97.36.171.192,"{""location"": ""BA"", ""is_mobile"": false}" 3639,2,80,2017-02-23 03:52:46,http://oreilly.net/henderson.ondricka,0.9905054879,165.117.60.32,"{""location"": ""PL"", ""is_mobile"": true}" 3640,2,80,2017-02-20 00:58:15,http://hartmann.io/mariam,0.0749720015,234.228.81.84,"{""location"": ""NE"", ""is_mobile"": true}" 3641,2,80,2017-02-20 17:05:15,http://lind.name/jerry.hoeger,0.0431920351,113.203.212.209,"{""location"": ""BY"", ""is_mobile"": true}" 3642,2,80,2017-04-26 09:48:00,http://littelhilpert.com/emmy,0.5781848495,136.204.143.202,"{""location"": ""TM"", ""is_mobile"": false}" 3643,2,80,2017-06-13 04:38:08,http://schuppe.name/chesley_hyatt,0.1105078752,201.175.6.125,"{""location"": ""OM"", ""is_mobile"": true}" 3644,2,80,2017-04-13 12:02:10,http://yost.co/kellen_cole,0.4360119497,214.137.244.231,"{""location"": ""TZ"", ""is_mobile"": true}" 3645,2,80,2016-12-16 04:40:18,http://trantowrobel.info/amina_schumm,0.6903764688,171.248.53.24,"{""location"": ""LK"", ""is_mobile"": true}" 3646,2,80,2017-05-28 09:53:28,http://wehner.net/drake.mraz,0.1656301198,125.116.106.27,"{""location"": ""MM"", ""is_mobile"": false}" 3647,2,80,2017-03-04 14:33:26,http://mckenzie.com/lauren.johnson,0.5171414844,108.125.176.153,"{""location"": ""BR"", ""is_mobile"": true}" 3648,2,80,2017-01-19 12:16:12,http://botsford.org/anastasia,0.8976783584,103.126.177.58,"{""location"": ""BI"", ""is_mobile"": false}" 3649,2,80,2017-02-18 13:27:46,http://simonis.io/greg,0.1548457723,190.186.225.126,"{""location"": ""AZ"", ""is_mobile"": true}" 3650,2,80,2017-06-12 00:27:29,http://olson.name/eino_cummings,0.5379417945,251.94.19.192,"{""location"": ""SS"", ""is_mobile"": true}" 3651,2,80,2017-03-29 20:14:02,http://reichelbahringer.biz/helmer,0.9349966692,67.245.90.212,"{""location"": ""ID"", ""is_mobile"": false}" 3652,2,80,2017-03-31 16:20:09,http://raynor.com/matilda,0.9159226752,150.194.123.30,"{""location"": ""TO"", ""is_mobile"": true}" 3653,2,81,2016-12-31 04:03:11,http://greenfelder.io/lorenz,0.3109537616,214.190.116.131,"{""location"": ""BH"", ""is_mobile"": true}" 3654,2,81,2017-05-27 06:34:44,http://welch.io/hermann_spencer,0.6976900727,193.76.117.149,"{""location"": ""SX"", ""is_mobile"": false}" 3655,2,81,2017-04-02 20:45:56,http://rempel.org/lucile.kuhn,0.8990942647,45.129.117.129,"{""location"": ""MY"", ""is_mobile"": false}" 3656,2,81,2017-02-16 06:48:33,http://ziemann.io/jorge_bogisich,0.2123913256,83.95.129.176,"{""location"": ""MT"", ""is_mobile"": false}" 3657,2,81,2017-06-10 04:41:37,http://cummerata.net/allison_luettgen,0.5312368882,205.141.11.174,"{""location"": ""PY"", ""is_mobile"": true}" 3658,2,81,2017-02-24 08:02:08,http://turcotte.org/alford,0.5048272775,169.56.233.245,"{""location"": ""TV"", ""is_mobile"": false}" 3659,2,81,2017-03-25 09:53:59,http://muller.com/abbigail,0.1188537265,23.248.133.58,"{""location"": ""QA"", ""is_mobile"": true}" 3660,2,81,2016-12-21 05:32:37,http://glover.io/lamont,0.6565661835,82.133.88.242,"{""location"": ""BE"", ""is_mobile"": false}" 3661,2,81,2017-03-04 20:03:26,http://ankunding.co/estel_johns,0.8308224698,196.234.50.12,"{""location"": ""FM"", ""is_mobile"": false}" 3662,2,81,2017-02-11 09:23:15,http://heel.info/abdiel,0.3500677187,253.33.85.151,"{""location"": ""CN"", ""is_mobile"": true}" 3663,2,81,2017-01-19 00:34:48,http://oharahuels.org/carli,0.9001636608,68.144.111.18,"{""location"": ""FR"", ""is_mobile"": false}" 3664,2,81,2017-06-10 18:07:29,http://yost.co/constantin.stokes,0.5806744132,84.225.212.246,"{""location"": ""KW"", ""is_mobile"": false}" 3665,2,81,2016-12-21 17:42:52,http://hodkiewiczwatsica.org/stewart_pacocha,0.2733941284,110.212.241.216,"{""location"": ""KN"", ""is_mobile"": false}" 3666,2,81,2017-05-12 01:19:35,http://caspergorczany.info/christa,0.6989731268,106.36.35.70,"{""location"": ""SG"", ""is_mobile"": true}" 3667,2,81,2017-01-09 23:14:02,http://bayer.info/kelsie_oconnell,0.5609915011,197.98.30.207,"{""location"": ""HU"", ""is_mobile"": false}" 3668,2,81,2017-01-18 03:01:51,http://jakubowskicarroll.io/fabian,0.8868192423,103.247.229.185,"{""location"": ""RS"", ""is_mobile"": false}" 3669,2,81,2017-03-23 07:02:10,http://bartontremblay.com/linnea,0.7641608996,11.175.219.221,"{""location"": ""NA"", ""is_mobile"": false}" 3670,2,81,2017-01-09 07:40:22,http://padberg.io/jovanny,0.8688972377,15.104.75.241,"{""location"": ""NL"", ""is_mobile"": false}" 3671,2,81,2016-12-25 11:27:47,http://kihn.biz/tamia,0.2865774740,59.176.168.208,"{""location"": ""SY"", ""is_mobile"": true}" 3672,2,81,2017-02-06 20:49:16,http://marvin.org/elfrieda_schowalter,0.9149511090,111.181.49.52,"{""location"": ""TR"", ""is_mobile"": false}" 3673,2,81,2017-04-01 02:03:51,http://reichert.co/bryon_mueller,0.7846001947,6.20.223.235,"{""location"": ""EC"", ""is_mobile"": false}" 3674,2,81,2017-03-02 03:00:01,http://lakin.name/tristin_bosco,0.1090421002,105.72.251.206,"{""location"": ""AR"", ""is_mobile"": false}" 3675,2,81,2017-06-09 16:49:54,http://swaniawski.co/laverne_haag,0.1925328005,77.71.116.74,"{""location"": ""NG"", ""is_mobile"": true}" 3676,2,81,2017-04-26 13:52:12,http://volkmanrempel.org/dedrick_pagac,0.1823318305,34.124.53.111,"{""location"": ""UY"", ""is_mobile"": false}" 3677,2,81,2017-05-12 08:15:08,http://heelbechtelar.info/alvina,0.8544303043,63.155.139.177,"{""location"": ""HN"", ""is_mobile"": false}" 3678,2,81,2017-05-15 01:01:58,http://grimes.net/herman.wiza,0.2631234503,223.227.109.37,"{""location"": ""BO"", ""is_mobile"": true}" 10558,4,237,2017-04-01 11:31:54,http://olsonzboncak.io/lew,0.8222770938,251.123.172.114,"{""location"": ""SR"", ""is_mobile"": false}" 10559,4,237,2017-02-26 06:30:17,http://effertz.co/willow.trantow,0.6116384231,218.196.156.54,"{""location"": ""GI"", ""is_mobile"": false}" 10560,4,237,2017-05-10 01:58:00,http://gorczany.name/golden_leuschke,0.2125930432,246.194.177.106,"{""location"": ""FI"", ""is_mobile"": true}" 10561,4,237,2017-03-07 12:16:09,http://dietrich.net/ellsworth.kub,0.7639739247,244.147.214.160,"{""location"": ""LA"", ""is_mobile"": true}" 10562,4,237,2017-01-21 09:55:28,http://grady.io/dee,0.5625695579,98.247.192.205,"{""location"": ""MK"", ""is_mobile"": false}" 10563,4,237,2017-06-05 22:28:51,http://doyle.com/madyson,0.2010966497,194.83.95.133,"{""location"": ""LC"", ""is_mobile"": true}" 10564,4,237,2017-03-04 11:49:07,http://aufderhar.co/verlie.krajcik,0.9460950671,220.107.146.216,"{""location"": ""GB"", ""is_mobile"": false}" 10565,4,237,2017-03-02 03:08:30,http://mann.info/isaac_bogisich,0.0078230196,243.72.243.229,"{""location"": ""GF"", ""is_mobile"": true}" 10566,4,237,2017-03-04 15:57:55,http://balistreri.co/cletus_sauer,0.0858922776,39.223.36.80,"{""location"": ""MW"", ""is_mobile"": true}" 10567,4,237,2017-02-03 00:02:56,http://pouros.io/tanner,0.8202174981,31.57.9.70,"{""location"": ""SB"", ""is_mobile"": false}" 10568,4,237,2017-02-21 05:37:39,http://shanahanmoore.com/lacey,0.2211712623,36.14.72.76,"{""location"": ""LA"", ""is_mobile"": false}" 10569,4,237,2017-02-02 22:13:13,http://dubuque.info/andreanne,0.0587440729,207.223.215.91,"{""location"": ""DJ"", ""is_mobile"": false}" 10570,4,237,2017-06-12 16:09:44,http://runolfon.biz/domenica.mueller,0.0696271687,33.139.127.18,"{""location"": ""MC"", ""is_mobile"": true}" 10571,4,237,2017-01-27 16:03:16,http://tillman.name/dimitri.ritchie,0.0771560471,129.230.176.146,"{""location"": ""TR"", ""is_mobile"": true}" 10572,4,237,2017-06-07 15:57:20,http://wiegand.name/bridget_heidenreich,0.9598645716,218.156.78.114,"{""location"": ""KI"", ""is_mobile"": false}" 10573,4,237,2017-05-10 12:21:26,http://nicolas.co/adeline,0.7137310064,243.249.65.147,"{""location"": ""ZW"", ""is_mobile"": true}" 10574,4,237,2017-03-28 19:13:14,http://hintzmueller.net/rose,0.0165286823,89.84.66.60,"{""location"": ""ST"", ""is_mobile"": false}" 10575,4,237,2016-12-22 09:07:25,http://roob.co/jerrold.von,0.7307312732,9.218.204.106,"{""location"": ""SM"", ""is_mobile"": true}" 10576,4,237,2017-03-10 10:00:59,http://toyskiles.biz/delmer,0.2528395374,139.75.26.253,"{""location"": ""KN"", ""is_mobile"": true}" 10577,4,237,2017-04-03 02:08:52,http://kuhlman.name/roberto,0.1232882079,112.118.248.53,"{""location"": ""BJ"", ""is_mobile"": false}" 10578,4,237,2017-05-11 01:07:39,http://lowefisher.io/jacquelyn,0.9416824502,33.147.133.71,"{""location"": ""MU"", ""is_mobile"": true}" 10579,4,237,2017-01-31 11:47:25,http://von.info/bailey_pouros,0.5309082330,36.29.198.61,"{""location"": ""GB"", ""is_mobile"": true}" 10580,4,237,2017-03-22 01:50:37,http://feeneylindgren.name/wilburn,0.9433520767,160.93.216.150,"{""location"": ""MA"", ""is_mobile"": false}" 10581,4,237,2017-04-12 05:15:29,http://oconnell.io/august_murray,0.5542342988,215.249.198.174,"{""location"": ""SC"", ""is_mobile"": false}" 10582,4,237,2017-06-09 11:42:00,http://huel.info/jannie,0.3171781344,76.203.228.34,"{""location"": ""BT"", ""is_mobile"": false}" 10583,4,237,2017-05-20 09:29:18,http://hilllschumm.io/ervin,0.1037897024,190.137.111.196,"{""location"": ""LB"", ""is_mobile"": true}" 10584,4,237,2017-01-28 10:04:08,http://waelchischaden.biz/braden,0.4151032787,113.173.8.6,"{""location"": ""WS"", ""is_mobile"": false}" 10585,4,237,2017-02-07 01:38:56,http://cummerata.net/easton,0.2455397445,161.42.227.226,"{""location"": ""TT"", ""is_mobile"": true}" 10586,4,237,2017-01-15 03:04:19,http://gutkowski.info/kaia,0.1476172723,251.77.228.52,"{""location"": ""TL"", ""is_mobile"": false}" 10587,4,237,2017-05-31 22:57:20,http://goodwinschultz.name/savion,0.4381016055,55.170.53.178,"{""location"": ""NZ"", ""is_mobile"": true}" 10588,4,237,2017-02-03 06:23:32,http://kuvalis.co/ronaldo,0.8467151677,24.87.25.103,"{""location"": ""SV"", ""is_mobile"": false}" 10589,4,237,2017-03-02 16:13:54,http://schimmelhyatt.name/lauriane,0.1312543562,228.157.247.103,"{""location"": ""OM"", ""is_mobile"": true}" 10590,4,237,2017-01-31 06:54:49,http://pfannerstill.name/giuseppe_abshire,0.7492461879,237.188.7.164,"{""location"": ""WF"", ""is_mobile"": false}" 10591,4,237,2017-01-31 21:51:07,http://watsica.com/elisha.dach,0.0195667740,42.213.228.25,"{""location"": ""AW"", ""is_mobile"": true}" 10592,4,237,2016-12-18 02:08:14,http://pricewaelchi.co/zola,0.1238586174,42.242.219.150,"{""location"": ""BT"", ""is_mobile"": true}" 10593,4,237,2017-02-12 18:30:58,http://vandervortschinner.net/ray,0.3887518998,208.100.238.20,"{""location"": ""NL"", ""is_mobile"": true}" 10594,4,237,2017-02-06 04:49:53,http://johnstonwilderman.biz/claud,0.6679085619,21.65.124.58,"{""location"": ""TJ"", ""is_mobile"": false}" 10595,4,238,2017-02-15 23:46:58,http://litteleichmann.co/garfield,,202.183.105.168,"{""location"": ""SX"", ""is_mobile"": true}" 10596,4,238,2017-02-06 21:08:05,http://cronincain.info/adela.ankunding,,230.133.250.64,"{""location"": ""BT"", ""is_mobile"": true}" 10597,4,238,2017-05-19 00:50:42,http://cronin.net/cristal.murray,,143.41.137.241,"{""location"": ""DK"", ""is_mobile"": true}" 10598,4,238,2017-04-15 12:14:48,http://crist.co/leonora,,178.162.113.54,"{""location"": ""MS"", ""is_mobile"": true}" 10599,4,238,2017-05-12 13:47:30,http://crona.co/yesenia,,158.88.103.227,"{""location"": ""BH"", ""is_mobile"": false}" 10600,4,238,2017-04-04 10:48:23,http://waelchiprohaska.info/lafayette_greenfelder,,144.236.7.79,"{""location"": ""GN"", ""is_mobile"": false}" 10601,4,238,2017-05-28 13:38:49,http://padbergabbott.com/fletcher.kohler,,69.153.6.118,"{""location"": ""RS"", ""is_mobile"": true}" 10602,4,238,2017-01-30 16:21:53,http://smithamcummings.io/isaias_rodriguez,,14.202.162.135,"{""location"": ""NZ"", ""is_mobile"": false}" 10603,4,238,2017-05-14 03:05:20,http://boehm.net/ella.ferry,,189.35.4.10,"{""location"": ""SR"", ""is_mobile"": true}" 10604,4,238,2016-12-30 11:25:07,http://runolfon.biz/ernestina,,206.161.86.238,"{""location"": ""EG"", ""is_mobile"": false}" 10605,4,238,2017-03-09 23:07:42,http://bradtkemedhurst.info/reyna.hilll,,125.236.184.52,"{""location"": ""TO"", ""is_mobile"": true}" 10606,4,238,2017-05-13 06:07:24,http://ko.com/olaf_batz,,131.221.63.43,"{""location"": ""RE"", ""is_mobile"": true}" 10607,4,238,2017-01-23 07:46:39,http://jacobson.biz/gunner,,254.241.234.6,"{""location"": ""HR"", ""is_mobile"": true}" 10608,4,238,2016-12-14 22:07:28,http://monahan.name/jaquelin_murazik,,253.249.54.130,"{""location"": ""MW"", ""is_mobile"": true}" 10609,4,238,2017-05-05 12:35:54,http://framiswaniawski.co/leila.schaefer,,67.94.120.51,"{""location"": ""MC"", ""is_mobile"": true}" 7595,3,168,2017-05-01 21:08:53,http://lowe.name/eduardo,,205.33.231.167,"{""location"": ""CX"", ""is_mobile"": true}" 7596,3,168,2017-05-25 23:16:11,http://batz.org/ludwig.wyman,,7.10.241.248,"{""location"": ""JP"", ""is_mobile"": true}" 7597,3,168,2017-03-06 22:18:29,http://boyer.org/sabina.lind,,84.171.241.138,"{""location"": ""EE"", ""is_mobile"": false}" 7598,3,168,2017-03-29 02:56:11,http://ratkelueilwitz.info/makayla,,207.61.100.178,"{""location"": ""AX"", ""is_mobile"": true}" 7599,3,168,2017-01-13 08:19:28,http://weber.net/colin.hansen,,15.164.162.151,"{""location"": ""TM"", ""is_mobile"": true}" 7600,3,168,2017-06-12 04:22:15,http://kuhlmangoldner.io/wilson,,215.151.203.178,"{""location"": ""BN"", ""is_mobile"": true}" 7601,3,168,2017-05-10 11:32:27,http://leffler.biz/dayton_crooks,,78.52.113.58,"{""location"": ""AF"", ""is_mobile"": false}" 7602,3,168,2017-02-20 00:47:08,http://runteschmitt.com/melba,,96.57.32.124,"{""location"": ""RE"", ""is_mobile"": false}" 7603,3,168,2017-04-08 21:53:00,http://mcclure.io/jedidiah,,148.98.111.144,"{""location"": ""PS"", ""is_mobile"": false}" 7604,3,168,2017-03-27 19:39:10,http://hagenes.net/dedric.moen,,178.113.109.208,"{""location"": ""AZ"", ""is_mobile"": true}" 7605,3,168,2017-06-04 10:01:44,http://gleasonrutherford.io/zoie_erdman,,131.211.124.244,"{""location"": ""KE"", ""is_mobile"": true}" 7606,3,168,2017-05-07 09:24:33,http://metz.com/nels,,82.209.180.47,"{""location"": ""PL"", ""is_mobile"": true}" 7607,3,168,2017-04-21 21:58:43,http://sipes.io/keegan,,249.13.182.34,"{""location"": ""HR"", ""is_mobile"": false}" 7608,3,168,2017-04-19 14:19:37,http://wintheiser.com/linda_lueilwitz,,169.135.85.242,"{""location"": ""TM"", ""is_mobile"": true}" 7609,3,168,2017-03-11 17:36:54,http://champlin.biz/graciela.gulgowski,,29.153.168.144,"{""location"": ""MD"", ""is_mobile"": true}" 7610,3,168,2017-01-05 13:08:23,http://kling.com/colton,,65.64.193.222,"{""location"": ""LR"", ""is_mobile"": false}" 7611,3,168,2017-04-19 16:35:46,http://morietteromaguera.biz/kenneth,,240.244.33.88,"{""location"": ""SD"", ""is_mobile"": false}" 7612,3,168,2017-03-12 08:39:45,http://reichelmorar.name/myrna,,53.123.124.236,"{""location"": ""HN"", ""is_mobile"": true}" 7613,3,168,2016-12-26 16:16:33,http://mills.biz/lauriane,,103.116.139.95,"{""location"": ""PT"", ""is_mobile"": true}" 7614,3,168,2017-02-26 02:55:14,http://dooley.name/thea.hettinger,,2.49.22.175,"{""location"": ""HU"", ""is_mobile"": true}" 7615,3,168,2017-03-02 16:51:28,http://schaefer.biz/tracy,,52.237.89.142,"{""location"": ""VA"", ""is_mobile"": false}" 7616,3,168,2017-05-09 11:43:45,http://bogansipes.name/lera,,135.171.3.235,"{""location"": ""ST"", ""is_mobile"": false}" 7617,3,168,2017-05-18 22:37:58,http://abbottschinner.co/johnpaul.oconner,,17.163.115.204,"{""location"": ""KR"", ""is_mobile"": false}" 7618,3,168,2017-02-19 11:09:30,http://littel.co/laverna_bernier,,49.99.126.109,"{""location"": ""IE"", ""is_mobile"": true}" 7619,3,168,2017-05-30 13:31:20,http://muller.org/earlene,,177.251.216.217,"{""location"": ""CH"", ""is_mobile"": true}" 7620,3,169,2017-05-29 13:06:25,http://mcdermottmetz.org/casimir_crist,,7.164.243.4,"{""location"": ""MA"", ""is_mobile"": true}" 7621,3,169,2017-05-01 17:03:46,http://senger.biz/lee_goyette,,41.33.6.146,"{""location"": ""CO"", ""is_mobile"": false}" 7622,3,169,2016-12-29 18:55:09,http://daniel.net/ruby_pouros,,135.232.193.209,"{""location"": ""SY"", ""is_mobile"": true}" 7623,3,169,2017-03-15 14:50:14,http://feil.biz/stephany,,201.158.49.46,"{""location"": ""GD"", ""is_mobile"": true}" 7624,3,169,2017-04-06 17:49:20,http://wilderman.name/darrel,,93.243.184.91,"{""location"": ""SX"", ""is_mobile"": false}" 7625,3,169,2017-04-06 16:15:32,http://padberg.name/modesta,,153.59.82.121,"{""location"": ""GG"", ""is_mobile"": true}" 7626,3,169,2017-05-04 02:35:41,http://vonruedenjohnston.biz/maida_reichel,,234.167.187.70,"{""location"": ""TV"", ""is_mobile"": true}" 7627,3,169,2017-01-07 14:54:58,http://davis.com/duane,,218.35.153.195,"{""location"": ""GB"", ""is_mobile"": false}" 7628,3,169,2017-04-19 13:39:48,http://carterklocko.name/rolando_crooks,,232.216.70.191,"{""location"": ""BG"", ""is_mobile"": false}" 7629,3,169,2017-05-11 22:29:55,http://schaden.info/ben,,219.18.149.153,"{""location"": ""GE"", ""is_mobile"": true}" 7630,3,169,2017-01-11 07:17:11,http://pollichkovacek.co/keagan,,183.108.90.45,"{""location"": ""RU"", ""is_mobile"": false}" 7631,3,169,2017-02-01 14:46:21,http://bergnaum.com/shaina_hudson,,127.121.136.112,"{""location"": ""MF"", ""is_mobile"": false}" 7632,3,169,2017-02-18 05:35:11,http://langworth.co/lonzo,,118.2.205.33,"{""location"": ""AI"", ""is_mobile"": true}" 7633,3,169,2017-04-12 10:35:27,http://mannjohnson.biz/harold.heaney,,127.73.186.4,"{""location"": ""EG"", ""is_mobile"": false}" 7634,3,169,2017-02-08 22:23:02,http://corkeryhahn.net/hollie,,150.221.157.190,"{""location"": ""BR"", ""is_mobile"": true}" 7635,3,169,2017-06-05 21:46:22,http://fisher.name/kailyn_crooks,,41.32.97.219,"{""location"": ""SC"", ""is_mobile"": false}" 7636,3,169,2017-02-06 15:23:55,http://robertsconn.io/dejah,,91.110.233.103,"{""location"": ""DJ"", ""is_mobile"": false}" 7637,3,169,2017-06-08 04:30:21,http://hudson.info/winifred.wilkinson,,237.19.168.232,"{""location"": ""CF"", ""is_mobile"": false}" 7638,3,169,2017-02-24 23:15:21,http://strackehackett.name/pearlie_daugherty,,111.239.122.183,"{""location"": ""VA"", ""is_mobile"": false}" 7639,3,169,2017-01-09 17:37:57,http://hillsbarrows.net/erik,,18.43.43.153,"{""location"": ""BZ"", ""is_mobile"": true}" 7640,3,169,2016-12-29 13:24:48,http://dickinsonrunolfon.info/luther_lynch,,49.145.250.109,"{""location"": ""LC"", ""is_mobile"": true}" 7641,3,169,2017-01-03 17:28:01,http://luettgen.org/breana.monahan,,205.31.25.123,"{""location"": ""GP"", ""is_mobile"": false}" 7642,3,169,2016-12-13 10:07:37,http://yundt.biz/ladarius,,149.74.23.152,"{""location"": ""SZ"", ""is_mobile"": false}" 7643,3,169,2017-06-07 16:29:59,http://price.io/granville.schimmel,,151.87.162.219,"{""location"": ""AQ"", ""is_mobile"": false}" 7644,3,169,2017-03-24 20:06:30,http://mooreshields.info/earl_ratke,,65.51.126.157,"{""location"": ""MM"", ""is_mobile"": false}" 7645,3,169,2017-05-01 14:44:13,http://homenick.com/stephon_kuhn,,201.252.10.179,"{""location"": ""CR"", ""is_mobile"": false}" 7646,3,169,2017-03-13 07:07:32,http://ondricka.com/deion,,73.5.114.234,"{""location"": ""SC"", ""is_mobile"": true}" 7647,3,169,2017-03-30 12:50:06,http://schultz.io/luella,,75.137.25.166,"{""location"": ""KH"", ""is_mobile"": false}" 7648,3,169,2017-04-20 15:52:10,http://bayerkeebler.org/tracy.vonrueden,,31.142.214.200,"{""location"": ""LR"", ""is_mobile"": true}" 7649,3,169,2017-03-23 01:16:06,http://fritschbatz.co/branson.kreiger,,143.226.90.64,"{""location"": ""AF"", ""is_mobile"": false}" 13505,5,303,2017-05-12 23:14:55,http://weimann.net/lemuel,,2.115.154.218,"{""location"": ""AL"", ""is_mobile"": true}" 13506,5,303,2017-02-11 18:06:40,http://braun.name/tiana,,56.37.189.234,"{""location"": ""GD"", ""is_mobile"": false}" 13507,5,303,2017-01-24 20:34:50,http://senger.org/araceli_osinski,,174.235.249.189,"{""location"": ""BR"", ""is_mobile"": true}" 13508,5,303,2017-05-27 08:08:51,http://rippin.net/alisha,,6.85.63.36,"{""location"": ""CV"", ""is_mobile"": false}" 13509,5,303,2017-04-27 21:57:31,http://lakin.io/ines,,129.14.112.98,"{""location"": ""BJ"", ""is_mobile"": true}" 13510,5,303,2017-04-26 04:45:31,http://bayerlehner.co/dedrick.legros,,2.125.52.92,"{""location"": ""NR"", ""is_mobile"": false}" 13511,5,303,2017-04-03 14:28:10,http://padberg.com/kira,,91.120.125.88,"{""location"": ""MF"", ""is_mobile"": false}" 13512,5,303,2017-05-24 13:26:40,http://quigley.name/scotty_ullrich,,242.247.202.131,"{""location"": ""MU"", ""is_mobile"": true}" 13513,5,303,2017-02-27 15:44:59,http://wisozk.net/roie,,2.72.95.52,"{""location"": ""NF"", ""is_mobile"": false}" 13514,5,303,2016-12-20 03:00:31,http://zemlakcormier.org/carlee_hyatt,,225.98.45.73,"{""location"": ""FO"", ""is_mobile"": false}" 13515,5,303,2017-04-06 22:01:22,http://ruecker.org/nils,,17.72.132.110,"{""location"": ""CY"", ""is_mobile"": true}" 13516,5,303,2017-05-23 10:03:40,http://dare.biz/aileen.schultz,,111.87.166.177,"{""location"": ""KN"", ""is_mobile"": true}" 13517,5,303,2017-03-26 15:58:33,http://schusterbarton.com/brielle.bashirian,,214.34.143.230,"{""location"": ""LY"", ""is_mobile"": true}" 13518,5,303,2017-01-18 00:33:01,http://collier.co/makenna_zulauf,,172.3.128.56,"{""location"": ""GI"", ""is_mobile"": true}" 13519,5,303,2017-02-27 06:27:46,http://pagaclehner.biz/roxanne,,13.24.66.217,"{""location"": ""UA"", ""is_mobile"": true}" 13520,5,303,2017-04-30 13:22:24,http://kutchbechtelar.io/calista.jakubowski,,156.39.70.5,"{""location"": ""IM"", ""is_mobile"": true}" 13521,5,303,2017-02-03 20:42:11,http://huelscasper.name/freeda,,101.166.35.38,"{""location"": ""KY"", ""is_mobile"": true}" 13522,5,303,2017-05-20 14:47:11,http://johnstonpowlowski.co/gail_pacocha,,253.13.132.11,"{""location"": ""UM"", ""is_mobile"": true}" 13523,5,303,2017-04-08 17:19:40,http://hermiston.biz/camden,,168.35.137.233,"{""location"": ""PR"", ""is_mobile"": true}" 13524,5,303,2017-01-09 10:04:50,http://hoppe.name/beulah_price,,75.147.233.125,"{""location"": ""LB"", ""is_mobile"": false}" 13525,5,303,2017-05-07 07:30:11,http://nader.com/rachael_heaney,,114.192.21.175,"{""location"": ""IT"", ""is_mobile"": false}" 13526,5,303,2017-05-23 13:37:57,http://casperjerde.name/johnny_jacobs,,150.219.5.105,"{""location"": ""IO"", ""is_mobile"": false}" 13527,5,303,2017-01-07 01:09:18,http://auer.io/golda,,229.168.64.36,"{""location"": ""MM"", ""is_mobile"": false}" 13528,5,303,2017-04-16 16:47:11,http://johns.net/curt,,26.147.45.196,"{""location"": ""IL"", ""is_mobile"": true}" 13529,5,304,2017-05-31 09:35:52,http://gaylord.com/titus,,64.61.241.81,"{""location"": ""PG"", ""is_mobile"": false}" 13530,5,304,2017-03-21 20:22:15,http://turnergleason.org/edmond,,161.66.146.85,"{""location"": ""IT"", ""is_mobile"": true}" 13531,5,304,2017-02-18 18:36:02,http://whitemoen.com/anne_aufderhar,,130.144.160.58,"{""location"": ""WS"", ""is_mobile"": true}" 13532,5,304,2017-04-17 19:03:12,http://vandervort.org/jordy.thiel,,237.137.80.136,"{""location"": ""MT"", ""is_mobile"": true}" 13533,5,304,2017-03-30 11:21:02,http://heidenreichrempel.info/ian,,32.62.3.126,"{""location"": ""SE"", ""is_mobile"": false}" 13534,5,304,2017-03-13 19:57:13,http://bergstrom.io/zane,,252.161.25.204,"{""location"": ""SV"", ""is_mobile"": true}" 13535,5,304,2017-03-17 14:26:09,http://howegoldner.io/serenity_jacobson,,184.202.245.135,"{""location"": ""BB"", ""is_mobile"": false}" 13536,5,304,2017-05-04 06:30:52,http://orn.net/celia.rosenbaum,,213.80.60.137,"{""location"": ""MA"", ""is_mobile"": true}" 13537,5,304,2017-01-26 06:43:55,http://gutmanntremblay.co/jeromy,,132.129.247.61,"{""location"": ""MM"", ""is_mobile"": false}" 13538,5,304,2016-12-29 10:44:43,http://fisher.biz/beulah,,101.162.148.197,"{""location"": ""UG"", ""is_mobile"": true}" 13539,5,304,2017-04-04 22:56:06,http://mertzconn.net/emilia,,242.213.13.113,"{""location"": ""ZW"", ""is_mobile"": false}" 13540,5,304,2017-01-26 13:23:39,http://schmeler.net/taryn.stokes,,101.153.214.157,"{""location"": ""BY"", ""is_mobile"": true}" 13541,5,304,2017-03-21 17:22:35,http://heaney.io/henderson.pagac,,11.9.116.213,"{""location"": ""BQ"", ""is_mobile"": true}" 13542,5,304,2017-04-17 14:03:34,http://watsicafay.net/troy,,3.22.202.114,"{""location"": ""PH"", ""is_mobile"": false}" 13543,5,304,2017-05-09 13:34:50,http://swaniawski.com/ervin,,161.2.91.161,"{""location"": ""BA"", ""is_mobile"": true}" 13544,5,304,2017-05-15 19:17:57,http://okon.net/dorothy_mosciski,,142.156.185.218,"{""location"": ""MD"", ""is_mobile"": false}" 13545,5,304,2017-01-02 00:03:00,http://haag.biz/quinn.bergnaum,,218.32.46.141,"{""location"": ""BE"", ""is_mobile"": false}" 13546,5,304,2017-01-24 04:50:57,http://oconner.io/brock_nolan,,155.67.248.87,"{""location"": ""BV"", ""is_mobile"": true}" 13547,5,304,2017-01-19 16:59:18,http://hermanrodriguez.name/camilla,,49.10.85.132,"{""location"": ""TG"", ""is_mobile"": true}" 13548,5,304,2016-12-18 02:05:56,http://nicolas.info/velda,,29.91.143.56,"{""location"": ""KY"", ""is_mobile"": false}" 13549,5,304,2017-02-10 14:12:44,http://schiller.co/astrid,,34.230.78.35,"{""location"": ""ME"", ""is_mobile"": true}" 13550,5,304,2017-03-03 21:53:13,http://oreilly.com/emmanuelle,,46.151.157.136,"{""location"": ""MF"", ""is_mobile"": false}" 13551,5,304,2017-05-31 05:21:55,http://botsford.com/lisandro.ward,,9.3.55.10,"{""location"": ""AL"", ""is_mobile"": true}" 13552,5,304,2017-01-20 15:58:02,http://goyette.biz/justina,,188.65.162.58,"{""location"": ""NU"", ""is_mobile"": false}" 13553,5,304,2017-01-04 21:07:36,http://ziemannjohns.info/golden.nikolaus,,246.191.60.228,"{""location"": ""KN"", ""is_mobile"": false}" 13554,5,304,2017-04-03 12:40:09,http://heel.biz/malvina_lynch,,84.74.65.128,"{""location"": ""WF"", ""is_mobile"": false}" 13555,5,304,2017-01-04 04:33:36,http://hamill.info/kasey.lueilwitz,,194.223.180.4,"{""location"": ""GU"", ""is_mobile"": true}" 13556,5,304,2017-05-12 04:06:22,http://collierruecker.org/caidy,,48.190.33.162,"{""location"": ""TT"", ""is_mobile"": true}" 13557,5,304,2017-05-14 16:46:10,http://littlehowe.co/reva,,243.189.172.101,"{""location"": ""PE"", ""is_mobile"": false}" 13558,5,305,2017-01-05 09:49:20,http://yundt.info/johathan.vandervort,,73.35.225.122,"{""location"": ""BD"", ""is_mobile"": false}" 13559,5,305,2017-02-13 07:31:42,http://cronindickinson.com/alicia,,47.233.43.59,"{""location"": ""NU"", ""is_mobile"": false}" 13560,5,305,2017-05-22 16:59:53,http://lemke.co/fernando,,174.198.36.134,"{""location"": ""AM"", ""is_mobile"": false}" 16499,6,374,2017-04-27 23:55:30,http://hermiston.io/jaron,0.0876729279,213.101.222.174,"{""location"": ""GI"", ""is_mobile"": false}" 16500,6,374,2017-02-09 23:54:15,http://ullrichklocko.io/dovie_toy,0.2344138990,125.213.150.214,"{""location"": ""UZ"", ""is_mobile"": false}" 16501,6,374,2017-02-27 08:24:29,http://mccullough.net/elias,0.3315874214,247.155.60.113,"{""location"": ""UZ"", ""is_mobile"": false}" 16502,6,374,2017-03-14 00:17:26,http://tromp.co/pearlie,0.2625691552,189.181.231.111,"{""location"": ""SV"", ""is_mobile"": false}" 16503,6,374,2017-05-16 14:02:28,http://schultzbednar.co/alphonso.feil,0.7289912067,195.94.192.168,"{""location"": ""SH"", ""is_mobile"": false}" 16504,6,374,2016-12-21 23:47:15,http://bosco.com/ida.considine,0.9071153985,99.126.178.101,"{""location"": ""MT"", ""is_mobile"": false}" 16505,6,374,2017-01-29 18:19:03,http://mantelueilwitz.info/emmanuel.lang,0.7134897352,231.30.155.178,"{""location"": ""IR"", ""is_mobile"": false}" 16506,6,374,2017-02-05 22:53:35,http://reichertmiller.net/max,0.7043530245,147.89.104.121,"{""location"": ""LU"", ""is_mobile"": false}" 16507,6,374,2017-05-13 20:26:39,http://schiller.org/vincent_stehr,0.9298747852,121.89.65.160,"{""location"": ""QA"", ""is_mobile"": true}" 16508,6,374,2017-03-19 03:20:17,http://zboncak.info/roosevelt,0.9145210902,6.38.134.150,"{""location"": ""VU"", ""is_mobile"": true}" 16509,6,374,2016-12-18 11:00:14,http://rauswift.io/lesley_rohan,0.7559308640,76.88.233.211,"{""location"": ""MU"", ""is_mobile"": false}" 16510,6,374,2016-12-23 16:24:10,http://marvinmedhurst.co/tina,0.9541546718,139.9.18.125,"{""location"": ""US"", ""is_mobile"": false}" 16511,6,374,2017-02-17 01:41:13,http://hills.org/ephraim,0.3125044949,178.104.32.117,"{""location"": ""HR"", ""is_mobile"": false}" 16512,6,374,2017-04-12 12:52:13,http://okuneva.com/cristal,0.0660640862,9.51.128.156,"{""location"": ""BT"", ""is_mobile"": false}" 16513,6,374,2017-02-28 10:53:04,http://mckenziemoriette.io/juliet,0.7980906704,187.253.193.190,"{""location"": ""CU"", ""is_mobile"": true}" 16514,6,374,2017-05-20 10:03:58,http://bartoletti.com/maiya.lebsack,0.2118090336,242.58.193.148,"{""location"": ""PE"", ""is_mobile"": false}" 16515,6,374,2017-06-04 23:43:50,http://davis.biz/monty,0.8455773382,91.71.243.168,"{""location"": ""ES"", ""is_mobile"": false}" 16516,6,374,2017-01-05 04:50:51,http://sawaynmcglynn.name/jefferey.rosenbaum,0.1260010574,176.228.97.200,"{""location"": ""PN"", ""is_mobile"": true}" 16517,6,374,2017-03-09 04:55:31,http://boyle.biz/elizabeth,0.1496360318,17.69.47.18,"{""location"": ""ST"", ""is_mobile"": false}" 16518,6,374,2017-01-20 13:14:47,http://fay.biz/jacques.rice,0.4549677955,124.11.164.193,"{""location"": ""EE"", ""is_mobile"": true}" 16519,6,374,2016-12-31 17:07:32,http://murazik.net/wilmer_dooley,0.2604744036,134.82.214.48,"{""location"": ""IL"", ""is_mobile"": false}" 16520,6,374,2016-12-18 18:39:50,http://reichel.biz/athena_johnson,0.0497031707,29.199.203.155,"{""location"": ""MH"", ""is_mobile"": false}" 16521,6,374,2017-03-08 03:17:15,http://rennerdonnelly.name/lulu.mertz,0.0678041039,84.69.141.161,"{""location"": ""ZA"", ""is_mobile"": false}" 16522,6,374,2017-02-24 12:45:19,http://torpfahey.co/evelyn,0.7294135695,235.148.239.162,"{""location"": ""GW"", ""is_mobile"": true}" 16523,6,374,2017-04-06 03:05:58,http://tillman.name/dixie_mueller,0.0860107908,251.144.75.232,"{""location"": ""ZW"", ""is_mobile"": true}" 16524,6,374,2017-03-05 07:07:49,http://hoegerhegmann.name/ettie_sporer,0.1880112615,119.107.173.207,"{""location"": ""SX"", ""is_mobile"": false}" 16525,6,374,2017-05-22 04:49:53,http://mullergreenholt.co/lottie_johnston,0.6013065789,112.122.48.61,"{""location"": ""IQ"", ""is_mobile"": false}" 16526,6,374,2017-01-14 11:27:09,http://tremblaynienow.net/catalina,0.6638085820,41.51.177.173,"{""location"": ""BV"", ""is_mobile"": true}" 16527,6,374,2017-05-15 04:45:24,http://feeneycarroll.org/lupe,0.4426522710,162.151.62.126,"{""location"": ""WS"", ""is_mobile"": true}" 16528,6,374,2016-12-22 13:44:49,http://funk.name/sarai,0.2758144877,211.108.212.185,"{""location"": ""FR"", ""is_mobile"": false}" 16529,6,374,2017-01-18 05:35:22,http://farrellyost.info/eloise.schulist,0.0030401876,90.117.246.214,"{""location"": ""ES"", ""is_mobile"": false}" 16530,6,374,2017-01-30 05:38:42,http://denesik.net/winifred,0.7855551479,190.249.246.60,"{""location"": ""BV"", ""is_mobile"": false}" 16531,6,374,2017-01-29 06:19:59,http://jacobson.biz/lillie,0.7882509878,161.215.15.145,"{""location"": ""GY"", ""is_mobile"": true}" 16532,6,374,2017-01-01 16:53:53,http://creminlehner.info/piper,0.2485135678,13.143.254.38,"{""location"": ""CL"", ""is_mobile"": true}" 16533,6,374,2017-04-06 13:31:23,http://hills.com/archibald.moore,0.8336688566,177.62.14.78,"{""location"": ""YE"", ""is_mobile"": false}" 16534,6,374,2017-05-02 16:32:08,http://wuckert.net/marie_olson,0.1472104422,50.146.250.117,"{""location"": ""BZ"", ""is_mobile"": false}" 16535,6,374,2017-04-22 22:09:19,http://wisozk.co/clementina,0.3517094800,72.150.232.182,"{""location"": ""ES"", ""is_mobile"": false}" 16536,6,374,2017-03-24 10:29:22,http://grimes.co/gaylord.jones,0.6857956060,74.75.235.156,"{""location"": ""KZ"", ""is_mobile"": false}" 16537,6,374,2016-12-26 10:13:55,http://prohaska.name/matteo_sporer,0.1080580447,122.170.158.72,"{""location"": ""DM"", ""is_mobile"": true}" 16538,6,374,2017-04-05 05:19:53,http://altenwerthroberts.co/haley,0.5313464386,9.188.67.139,"{""location"": ""EE"", ""is_mobile"": false}" 16539,6,374,2017-01-31 02:01:14,http://beatty.org/solon.lowe,0.0554982072,167.45.25.137,"{""location"": ""KH"", ""is_mobile"": false}" 16540,6,374,2017-01-06 14:13:17,http://white.co/mazie_turcotte,0.0009740604,240.65.49.44,"{""location"": ""GP"", ""is_mobile"": false}" 16541,6,374,2017-01-30 00:39:29,http://kiehnharber.biz/rene_rowe,0.5933451805,182.243.39.207,"{""location"": ""GW"", ""is_mobile"": false}" 16542,6,374,2017-01-10 15:30:55,http://ryan.name/kenyatta,0.2548738447,21.128.34.229,"{""location"": ""YT"", ""is_mobile"": false}" 16543,6,375,2017-01-09 21:10:14,http://will.org/jane.kutch,0.6700710379,208.175.62.97,"{""location"": ""CD"", ""is_mobile"": true}" 16544,6,375,2017-05-27 20:45:00,http://crist.io/jaylen_padberg,0.2601721021,102.124.109.56,"{""location"": ""KP"", ""is_mobile"": true}" 16545,6,375,2017-06-01 04:17:54,http://hegmann.info/cortez_kuhlman,0.1056598516,67.92.181.179,"{""location"": ""QA"", ""is_mobile"": true}" 16546,6,375,2017-04-02 02:23:36,http://kuhic.io/graham_jerde,0.3594908148,101.125.27.56,"{""location"": ""KM"", ""is_mobile"": true}" 16547,6,375,2017-04-11 14:43:11,http://trompdavis.org/maximilian,0.7078409207,159.215.200.168,"{""location"": ""TH"", ""is_mobile"": true}" 16548,6,375,2016-12-15 17:04:56,http://nitzsche.com/krystal,0.4217207367,107.235.56.88,"{""location"": ""FK"", ""is_mobile"": false}" 16549,6,375,2017-03-14 03:28:34,http://von.net/leann,0.4828579090,22.230.157.37,"{""location"": ""LC"", ""is_mobile"": true}" 3679,2,81,2017-06-08 12:10:40,http://jaskolski.com/adrian,0.6636959060,253.119.68.171,"{""location"": ""FM"", ""is_mobile"": false}" 3680,2,81,2017-04-23 22:01:33,http://gerhold.biz/christina,0.8248568875,47.195.125.176,"{""location"": ""DM"", ""is_mobile"": true}" 3681,2,81,2017-03-27 10:46:43,http://aufderhar.co/willy.langosh,0.9806572447,40.147.133.86,"{""location"": ""MW"", ""is_mobile"": false}" 3682,2,81,2017-04-01 07:29:18,http://bergstrom.info/brielle,0.7166249107,52.109.23.118,"{""location"": ""CO"", ""is_mobile"": false}" 3683,2,81,2016-12-26 00:49:13,http://stanton.net/christiana.kautzer,0.2887667627,169.156.231.169,"{""location"": ""ID"", ""is_mobile"": false}" 3684,2,81,2017-04-18 18:42:25,http://crist.name/hubert.dibbert,0.6098747128,6.14.46.107,"{""location"": ""AW"", ""is_mobile"": false}" 3685,2,81,2017-06-09 02:17:53,http://larsonkeeling.net/alanna,0.3617874399,94.204.114.194,"{""location"": ""ME"", ""is_mobile"": false}" 3686,2,81,2017-02-22 08:18:34,http://keebler.org/natalia,0.4698671039,62.202.203.135,"{""location"": ""PL"", ""is_mobile"": true}" 3687,2,81,2017-05-30 01:57:46,http://lindgrenrau.name/cleveland,0.0099738994,249.46.48.41,"{""location"": ""VI"", ""is_mobile"": false}" 3688,2,81,2017-03-03 10:58:50,http://pfeffer.co/jayde_stracke,0.6456791993,185.233.7.47,"{""location"": ""VN"", ""is_mobile"": false}" 3689,2,81,2017-04-12 10:51:35,http://parisianko.com/patience,0.9995092826,81.147.112.87,"{""location"": ""MT"", ""is_mobile"": false}" 3690,2,81,2017-04-30 09:14:30,http://heidenreich.io/jerald.hudson,0.5765593944,87.25.220.177,"{""location"": ""DO"", ""is_mobile"": false}" 3691,2,81,2017-03-24 03:28:09,http://boscopfannerstill.name/larue,0.5158344647,193.90.163.180,"{""location"": ""PS"", ""is_mobile"": false}" 3692,2,81,2017-05-25 06:08:23,http://stehr.biz/abel,0.8240833087,165.24.241.170,"{""location"": ""ML"", ""is_mobile"": true}" 3693,2,81,2017-04-27 07:57:28,http://jaskolski.com/lonie,0.0411070100,92.214.73.156,"{""location"": ""GB"", ""is_mobile"": true}" 3694,2,81,2017-04-28 02:06:48,http://moen.io/jerad.bartell,0.2430045730,108.72.240.11,"{""location"": ""NO"", ""is_mobile"": false}" 3695,2,81,2017-04-03 03:21:47,http://carter.info/claire_botsford,0.1110989982,199.96.136.230,"{""location"": ""AZ"", ""is_mobile"": true}" 3696,2,81,2016-12-23 00:15:14,http://morar.name/reinhold.reynolds,0.3323657707,83.106.236.129,"{""location"": ""TW"", ""is_mobile"": false}" 3697,2,81,2016-12-23 01:01:26,http://reingerpfeffer.info/ibrahim_graham,0.0348730015,99.100.90.179,"{""location"": ""RE"", ""is_mobile"": true}" 3698,2,81,2017-02-09 12:48:04,http://sanfordbahringer.net/mitchel,0.3860624331,244.13.11.157,"{""location"": ""EC"", ""is_mobile"": true}" 3699,2,82,2016-12-16 13:45:03,http://beattylynch.name/ruthe.tremblay,0.1719507383,81.137.65.14,"{""location"": ""AR"", ""is_mobile"": false}" 3700,2,82,2017-01-03 19:56:06,http://schulist.io/kari,0.4425700728,220.109.167.26,"{""location"": ""CX"", ""is_mobile"": false}" 3701,2,82,2017-02-04 02:53:08,http://powlowskilynch.io/georgette.windler,0.8548436078,203.17.100.84,"{""location"": ""IS"", ""is_mobile"": true}" 3702,2,82,2017-04-19 04:31:39,http://ratkeconn.name/rey,0.0940351632,19.150.14.235,"{""location"": ""VI"", ""is_mobile"": true}" 3703,2,82,2017-01-07 13:02:59,http://greenfelder.net/krista.osinski,0.1918241783,83.133.110.13,"{""location"": ""MZ"", ""is_mobile"": false}" 3704,2,82,2017-05-21 18:52:51,http://reingernolan.biz/santiago_blick,0.2396111637,55.38.230.171,"{""location"": ""AX"", ""is_mobile"": false}" 3705,2,82,2017-05-21 05:11:35,http://bernier.biz/cheyanne,0.9317791857,211.230.123.239,"{""location"": ""DK"", ""is_mobile"": false}" 3706,2,82,2016-12-15 13:07:57,http://waters.info/patience_crooks,0.5890127490,120.183.84.176,"{""location"": ""ST"", ""is_mobile"": false}" 3707,2,82,2017-05-30 15:18:27,http://aufderharrenner.biz/isabel_gutmann,0.7017492607,81.46.107.133,"{""location"": ""CL"", ""is_mobile"": false}" 3708,2,82,2017-05-11 17:48:19,http://lemke.net/leanna,0.3671928975,57.13.107.77,"{""location"": ""IR"", ""is_mobile"": true}" 3709,2,82,2017-06-10 07:36:14,http://spinkalesch.com/jacynthe_medhurst,0.1079739084,122.181.65.241,"{""location"": ""PF"", ""is_mobile"": true}" 3710,2,82,2017-04-14 00:29:55,http://kuphaltorphy.biz/dorian.connelly,0.0918454085,220.253.201.233,"{""location"": ""DE"", ""is_mobile"": true}" 3711,2,82,2017-06-03 12:10:41,http://abbott.io/holden_mertz,0.1237994727,159.177.245.70,"{""location"": ""SV"", ""is_mobile"": false}" 3712,2,82,2017-02-07 10:10:34,http://medhurst.io/fabian,0.4427489996,174.120.229.157,"{""location"": ""TT"", ""is_mobile"": true}" 3713,2,82,2017-01-24 17:03:02,http://steuber.net/madeline,0.0094943570,241.234.221.128,"{""location"": ""IR"", ""is_mobile"": false}" 3714,2,82,2017-05-01 11:49:47,http://bernierbeer.org/dolores_becker,0.7420093295,126.174.135.119,"{""location"": ""BT"", ""is_mobile"": true}" 3715,2,82,2017-01-03 01:44:10,http://feeney.org/sierra_murazik,0.1390614430,252.181.89.24,"{""location"": ""PM"", ""is_mobile"": true}" 3716,2,82,2017-05-07 00:06:24,http://zboncak.net/julie_rolfson,0.8305547022,254.131.6.150,"{""location"": ""AQ"", ""is_mobile"": true}" 3717,2,82,2017-04-29 13:18:02,http://glover.biz/izaiah.dooley,0.8526831156,67.59.82.212,"{""location"": ""AI"", ""is_mobile"": true}" 3718,2,82,2017-05-23 14:17:44,http://kuhlman.name/mohammed,0.9961869913,219.232.200.106,"{""location"": ""AW"", ""is_mobile"": true}" 3719,2,82,2017-04-22 19:48:11,http://wisoky.name/roselyn,0.7049974084,96.231.141.80,"{""location"": ""MN"", ""is_mobile"": true}" 3720,2,82,2017-04-28 12:27:47,http://boyer.net/amira,0.8448352625,129.226.19.84,"{""location"": ""VE"", ""is_mobile"": true}" 3721,2,82,2016-12-28 17:03:48,http://will.co/saige.wilkinson,0.5892998827,245.164.150.76,"{""location"": ""BH"", ""is_mobile"": true}" 3722,2,82,2017-01-06 10:59:24,http://ohara.co/regan.langworth,0.9677411832,245.168.149.68,"{""location"": ""TV"", ""is_mobile"": true}" 3723,2,82,2017-02-08 13:42:44,http://ankunding.com/ila,0.2499449915,131.66.14.157,"{""location"": ""YT"", ""is_mobile"": false}" 3724,2,82,2017-06-12 11:04:14,http://koelpin.biz/freda_waelchi,0.4451940815,176.156.215.200,"{""location"": ""UA"", ""is_mobile"": false}" 3725,2,82,2017-01-04 12:35:08,http://fadel.name/nina,0.4290173728,150.254.39.12,"{""location"": ""SK"", ""is_mobile"": false}" 3726,2,82,2017-04-16 01:58:54,http://reichert.org/jana.murphy,0.5330636222,83.113.167.38,"{""location"": ""GT"", ""is_mobile"": false}" 3727,2,82,2017-04-24 21:10:15,http://ondrickadamore.com/noemi,0.0051330045,201.228.157.173,"{""location"": ""CX"", ""is_mobile"": true}" 5265,2,116,2017-03-03 09:06:03,http://blanda.com/amaya,,223.141.162.48,"{""location"": ""LY"", ""is_mobile"": true}" 3728,2,82,2017-02-20 19:39:37,http://howellbruen.biz/verla,0.4051799151,158.34.47.131,"{""location"": ""LU"", ""is_mobile"": false}" 3729,2,82,2017-04-24 02:09:05,http://streich.io/jules,0.7551628554,231.38.118.251,"{""location"": ""AL"", ""is_mobile"": false}" 10610,4,238,2017-04-14 22:29:21,http://kuhlmanweinat.biz/alberto,,100.58.122.219,"{""location"": ""BQ"", ""is_mobile"": false}" 10611,4,238,2017-06-03 00:34:24,http://ruel.io/laury_crooks,,35.117.72.56,"{""location"": ""PE"", ""is_mobile"": false}" 10612,4,238,2017-06-03 17:03:35,http://gleasonemard.co/dion,,53.85.52.117,"{""location"": ""CC"", ""is_mobile"": true}" 10613,4,238,2016-12-14 23:47:05,http://jacobson.biz/juana.reynolds,,78.70.63.178,"{""location"": ""ZW"", ""is_mobile"": true}" 10614,4,238,2017-01-02 20:11:50,http://mills.name/margarita,,5.131.150.220,"{""location"": ""HT"", ""is_mobile"": true}" 10615,4,238,2017-03-26 03:05:52,http://krajcikboehm.co/eldred_pagac,,134.205.139.54,"{""location"": ""KZ"", ""is_mobile"": false}" 10616,4,238,2017-01-19 18:20:05,http://powlowski.io/travon,,234.139.222.204,"{""location"": ""ME"", ""is_mobile"": false}" 10617,4,238,2017-04-27 14:21:40,http://feeney.biz/lia_price,,228.254.49.98,"{""location"": ""MY"", ""is_mobile"": false}" 10618,4,238,2017-03-03 12:54:43,http://runolfsdottir.biz/diana.wisozk,,195.189.92.24,"{""location"": ""AO"", ""is_mobile"": true}" 10619,4,238,2017-01-11 03:35:02,http://kuvalis.io/marcellus,,147.217.116.92,"{""location"": ""TH"", ""is_mobile"": false}" 10620,4,238,2017-05-21 22:21:28,http://gaylord.net/katheryn,,140.184.39.28,"{""location"": ""TW"", ""is_mobile"": true}" 10621,4,238,2017-02-04 07:15:12,http://ernserlebsack.io/ward_hilpert,,116.126.152.61,"{""location"": ""BT"", ""is_mobile"": false}" 10622,4,238,2017-05-07 19:02:43,http://bartolettimraz.info/sidney,,16.153.101.76,"{""location"": ""TT"", ""is_mobile"": false}" 10623,4,238,2017-03-27 14:45:00,http://olson.io/adeline_blick,,163.62.134.65,"{""location"": ""IS"", ""is_mobile"": false}" 10624,4,238,2017-04-02 04:39:14,http://howeluettgen.biz/theresia,,151.163.108.84,"{""location"": ""ER"", ""is_mobile"": true}" 10625,4,238,2017-02-03 22:02:38,http://spinka.info/stephania.fahey,,117.92.194.14,"{""location"": ""US"", ""is_mobile"": true}" 10626,4,238,2017-06-08 13:37:19,http://herzoggraham.org/alexandria.heaney,,239.39.112.60,"{""location"": ""BO"", ""is_mobile"": true}" 10627,4,238,2017-04-09 03:33:34,http://brekke.co/piper_davis,,50.79.101.237,"{""location"": ""IS"", ""is_mobile"": true}" 10628,4,238,2017-03-24 22:13:15,http://pfannerstill.biz/annetta,,253.156.6.145,"{""location"": ""KZ"", ""is_mobile"": true}" 10629,4,238,2017-04-04 16:37:31,http://leannon.com/abbie,,168.99.230.237,"{""location"": ""CH"", ""is_mobile"": true}" 10630,4,238,2017-02-18 04:25:20,http://langosh.info/aisha.brekke,,228.62.99.73,"{""location"": ""PN"", ""is_mobile"": false}" 10631,4,238,2017-01-23 13:41:40,http://hodkiewiczschulist.io/cade_welch,,66.123.152.8,"{""location"": ""CU"", ""is_mobile"": false}" 10632,4,238,2017-04-23 13:17:33,http://dibbert.io/armando,,148.157.173.24,"{""location"": ""CC"", ""is_mobile"": false}" 10633,4,238,2017-03-05 03:48:28,http://upton.biz/angie.dietrich,,96.178.48.191,"{""location"": ""NC"", ""is_mobile"": true}" 10634,4,238,2017-03-05 11:28:17,http://cruickshank.com/kelly,,144.178.240.83,"{""location"": ""PW"", ""is_mobile"": false}" 10635,4,238,2017-02-13 21:15:07,http://mante.net/dan,,180.241.27.97,"{""location"": ""BJ"", ""is_mobile"": true}" 10636,4,238,2017-06-06 00:05:55,http://padberg.biz/ryder,,73.155.220.147,"{""location"": ""CN"", ""is_mobile"": false}" 10637,4,238,2017-02-11 12:04:03,http://lynch.net/bernie,,188.45.194.140,"{""location"": ""GL"", ""is_mobile"": true}" 10638,4,238,2017-04-01 06:21:33,http://lemke.com/hudson,,8.246.106.76,"{""location"": ""CI"", ""is_mobile"": true}" 10639,4,238,2016-12-30 18:19:08,http://becker.com/crystel.herman,,41.102.81.148,"{""location"": ""GR"", ""is_mobile"": false}" 10640,4,238,2017-01-02 22:35:28,http://zulauf.co/georgette,,171.70.63.217,"{""location"": ""TW"", ""is_mobile"": false}" 10641,4,238,2017-02-11 02:12:00,http://howecorkery.com/lurline.jacobs,,213.173.204.154,"{""location"": ""BM"", ""is_mobile"": true}" 10642,4,238,2017-02-15 22:06:24,http://goldnerkulas.info/abby,,16.111.64.63,"{""location"": ""KW"", ""is_mobile"": true}" 10643,4,238,2017-04-16 07:01:01,http://block.name/kevon.rutherford,,93.27.224.254,"{""location"": ""TM"", ""is_mobile"": false}" 10644,4,238,2017-03-03 20:51:47,http://doyle.com/hermann.block,,151.27.140.202,"{""location"": ""RU"", ""is_mobile"": true}" 10645,4,238,2017-05-19 05:11:07,http://kerluke.io/juanita_orn,,121.56.112.245,"{""location"": ""SY"", ""is_mobile"": false}" 10646,4,238,2017-05-14 09:32:05,http://howe.co/felix_sauer,,85.225.157.142,"{""location"": ""CH"", ""is_mobile"": true}" 10647,4,238,2017-03-25 03:50:38,http://romaguera.net/solon,,135.175.106.238,"{""location"": ""MV"", ""is_mobile"": true}" 10648,4,238,2017-05-04 21:22:33,http://strosin.biz/ruben.gutkowski,,59.89.103.219,"{""location"": ""MY"", ""is_mobile"": true}" 10649,4,238,2016-12-31 21:33:05,http://ferry.net/beth_prohaska,,164.237.65.65,"{""location"": ""TC"", ""is_mobile"": true}" 10650,4,238,2017-03-14 22:17:40,http://heel.net/celine_gaylord,,238.115.74.184,"{""location"": ""VA"", ""is_mobile"": true}" 10651,4,238,2017-04-12 06:37:01,http://lowelockman.name/dahlia.emard,,139.27.224.68,"{""location"": ""SR"", ""is_mobile"": false}" 10652,4,238,2017-02-01 04:51:24,http://larsonshanahan.io/americo.osinski,,118.12.14.55,"{""location"": ""GT"", ""is_mobile"": false}" 10653,4,238,2016-12-19 21:45:53,http://rodriguez.com/ignatius,,153.2.223.79,"{""location"": ""GM"", ""is_mobile"": true}" 10654,4,238,2017-01-27 07:06:38,http://senger.biz/sibyl.hoppe,,192.210.208.147,"{""location"": ""AO"", ""is_mobile"": false}" 10655,4,238,2017-01-26 20:05:49,http://weberrogahn.name/gus_lebsack,,183.128.67.113,"{""location"": ""WF"", ""is_mobile"": false}" 10656,4,238,2017-04-26 23:15:34,http://ratke.org/eli_fay,,9.240.131.8,"{""location"": ""FR"", ""is_mobile"": true}" 10657,4,238,2017-01-31 20:35:57,http://padberg.com/marjorie_kemmer,,161.153.81.151,"{""location"": ""TJ"", ""is_mobile"": false}" 10658,4,239,2017-01-13 19:57:50,http://daviwaniawski.net/samara,,160.151.59.129,"{""location"": ""NE"", ""is_mobile"": false}" 10659,4,239,2017-06-01 10:44:45,http://bergejones.com/rashad.heller,,128.242.48.89,"{""location"": ""BS"", ""is_mobile"": false}" 10660,4,239,2017-04-03 15:55:58,http://osinski.org/carroll,,22.125.124.247,"{""location"": ""GL"", ""is_mobile"": false}" 10661,4,239,2017-03-06 20:26:05,http://goldnerkuhic.org/saul_wisozk,,214.103.183.130,"{""location"": ""MM"", ""is_mobile"": false}" 10662,4,239,2017-05-20 11:24:18,http://ruel.org/merritt.oreilly,,225.218.9.173,"{""location"": ""TW"", ""is_mobile"": true}" 10663,4,239,2017-01-17 15:47:00,http://cole.biz/aubrey,,203.243.114.60,"{""location"": ""BG"", ""is_mobile"": false}" 10664,4,239,2016-12-25 22:49:52,http://rosenbaum.name/jason.kohler,,83.190.3.127,"{""location"": ""MQ"", ""is_mobile"": true}" 7650,3,169,2017-04-27 01:00:45,http://maggio.io/astrid_schimmel,,211.160.242.182,"{""location"": ""NR"", ""is_mobile"": true}" 7651,3,169,2017-01-14 07:38:56,http://spinka.net/neha_bailey,,187.73.129.211,"{""location"": ""BR"", ""is_mobile"": false}" 7652,3,169,2017-01-28 03:55:36,http://berniertreutel.info/randi_ondricka,,141.248.114.82,"{""location"": ""DO"", ""is_mobile"": true}" 7653,3,169,2017-04-21 11:20:20,http://kuhlman.co/may.ratke,,141.152.184.15,"{""location"": ""IN"", ""is_mobile"": false}" 7654,3,170,2017-02-28 11:03:44,http://kihn.org/madeline.breitenberg,,115.154.5.123,"{""location"": ""MS"", ""is_mobile"": false}" 7655,3,170,2017-01-12 05:54:14,http://emardstehr.biz/daisy,,161.117.232.18,"{""location"": ""IM"", ""is_mobile"": false}" 7656,3,170,2017-03-31 07:54:01,http://kozey.name/anya,,101.189.148.63,"{""location"": ""GB"", ""is_mobile"": false}" 7657,3,170,2017-01-21 19:38:00,http://schuppewatsica.info/kyla.donnelly,,189.217.81.24,"{""location"": ""UY"", ""is_mobile"": true}" 7658,3,170,2017-02-10 02:16:39,http://nienow.net/grover,,214.140.210.139,"{""location"": ""ZM"", ""is_mobile"": true}" 7659,3,170,2017-06-06 12:31:14,http://flatley.com/dana.denesik,,15.60.113.156,"{""location"": ""MD"", ""is_mobile"": false}" 7660,3,170,2017-01-29 21:03:08,http://connhermann.net/dorothy_ankunding,,125.33.7.93,"{""location"": ""KM"", ""is_mobile"": true}" 7661,3,170,2017-04-05 10:05:08,http://simonisharvey.com/liliana.miller,,36.122.85.128,"{""location"": ""VU"", ""is_mobile"": true}" 7662,3,170,2017-03-15 17:28:34,http://wuckertcasper.co/grant_turcotte,,84.9.231.6,"{""location"": ""AQ"", ""is_mobile"": false}" 7663,3,170,2017-04-07 03:41:25,http://mante.net/haley,,200.125.142.17,"{""location"": ""GU"", ""is_mobile"": false}" 7664,3,170,2017-05-16 17:38:37,http://hodkiewicz.com/lamont,,27.211.206.153,"{""location"": ""BJ"", ""is_mobile"": false}" 7665,3,170,2017-02-07 03:35:18,http://volkman.com/albin,,161.14.206.54,"{""location"": ""AS"", ""is_mobile"": false}" 7666,3,170,2017-02-26 11:40:35,http://waters.info/walton,,17.117.107.78,"{""location"": ""AZ"", ""is_mobile"": true}" 7667,3,170,2017-05-14 11:23:28,http://nitzsche.biz/laron,,249.15.212.149,"{""location"": ""KG"", ""is_mobile"": true}" 7668,3,170,2017-01-01 23:06:35,http://grahamrobel.biz/chris_willms,,35.3.218.86,"{""location"": ""QA"", ""is_mobile"": true}" 7669,3,170,2017-06-06 00:46:54,http://shields.io/yesenia,,115.247.96.101,"{""location"": ""TO"", ""is_mobile"": true}" 7670,3,170,2017-05-18 06:36:29,http://marvin.io/elton.willms,,21.12.45.68,"{""location"": ""NP"", ""is_mobile"": true}" 7671,3,170,2017-04-05 07:38:21,http://bailey.biz/anjali,,58.175.252.175,"{""location"": ""KG"", ""is_mobile"": false}" 7672,3,170,2017-02-28 04:06:31,http://farrell.name/tamia,,126.183.2.180,"{""location"": ""WF"", ""is_mobile"": false}" 7673,3,170,2017-05-24 05:12:09,http://littel.org/lenna,,8.106.214.184,"{""location"": ""US"", ""is_mobile"": false}" 7674,3,170,2017-01-17 02:54:47,http://rueckersmitham.co/thea_cole,,237.232.68.126,"{""location"": ""SM"", ""is_mobile"": false}" 7675,3,170,2017-02-17 13:29:19,http://kleinwehner.name/johanna.mayert,,203.21.123.70,"{""location"": ""MT"", ""is_mobile"": true}" 7676,3,170,2017-04-17 11:12:26,http://gloverhaag.net/amir.corkery,,90.52.44.135,"{""location"": ""CO"", ""is_mobile"": true}" 7677,3,170,2017-05-09 17:52:08,http://rice.biz/eula,,32.191.224.142,"{""location"": ""BM"", ""is_mobile"": false}" 7678,3,170,2017-05-10 10:56:26,http://schultz.biz/ania,,133.166.29.226,"{""location"": ""LK"", ""is_mobile"": false}" 7679,3,170,2017-03-30 14:09:16,http://mohrdoyle.com/rhoda,,189.143.166.202,"{""location"": ""SD"", ""is_mobile"": false}" 7680,3,170,2017-05-25 22:44:01,http://oberbrunnerwilkinson.biz/keira,,81.58.57.4,"{""location"": ""US"", ""is_mobile"": false}" 7681,3,170,2017-04-01 20:11:21,http://metz.info/virginie,,11.214.149.8,"{""location"": ""HK"", ""is_mobile"": false}" 7682,3,170,2017-01-16 11:03:53,http://aufderhar.name/aaron_terry,,213.242.172.31,"{""location"": ""CW"", ""is_mobile"": false}" 7683,3,170,2017-06-06 18:28:08,http://vonrueden.io/deion,,146.54.49.3,"{""location"": ""KP"", ""is_mobile"": false}" 7684,3,170,2017-05-27 23:44:36,http://nienow.co/camryn,,193.179.94.227,"{""location"": ""NR"", ""is_mobile"": true}" 7685,3,170,2017-05-04 05:27:12,http://wisozk.org/ima,,42.77.122.204,"{""location"": ""CZ"", ""is_mobile"": false}" 7686,3,170,2017-04-11 14:25:48,http://price.co/glennie,,94.119.46.5,"{""location"": ""AO"", ""is_mobile"": false}" 7687,3,170,2017-05-03 00:07:34,http://metzbatz.biz/frieda,,75.98.47.196,"{""location"": ""LC"", ""is_mobile"": true}" 7688,3,170,2016-12-31 18:55:58,http://reynolds.biz/mack,,75.78.239.129,"{""location"": ""HM"", ""is_mobile"": false}" 7689,3,170,2017-05-30 22:49:09,http://langworth.org/jermain.stamm,,135.114.138.137,"{""location"": ""KG"", ""is_mobile"": true}" 7690,3,170,2017-03-05 07:39:27,http://bartell.biz/kailee,,154.79.208.136,"{""location"": ""MG"", ""is_mobile"": true}" 7691,3,170,2016-12-19 22:42:26,http://kuhn.com/carter.cummings,,50.118.53.244,"{""location"": ""ET"", ""is_mobile"": true}" 7692,3,170,2017-03-18 09:38:06,http://cole.co/cleo.towne,,80.67.156.124,"{""location"": ""BO"", ""is_mobile"": true}" 7693,3,171,2017-03-25 03:50:30,http://olson.biz/yadira.vonrueden,,235.81.97.112,"{""location"": ""EG"", ""is_mobile"": true}" 7694,3,171,2017-02-04 12:37:57,http://friesenjones.io/alexandrea_koepp,,120.86.147.38,"{""location"": ""DK"", ""is_mobile"": true}" 7695,3,171,2017-06-05 23:33:56,http://senger.org/violette_trantow,,128.81.248.7,"{""location"": ""RE"", ""is_mobile"": false}" 7696,3,171,2017-03-26 06:15:32,http://trantowstrosin.name/ashleigh,,175.101.249.28,"{""location"": ""HT"", ""is_mobile"": true}" 7697,3,171,2017-05-06 22:24:32,http://rempel.com/clara,,181.235.41.245,"{""location"": ""SL"", ""is_mobile"": true}" 7698,3,171,2017-03-27 03:51:01,http://mantebins.net/richard_lakin,,40.110.159.35,"{""location"": ""SZ"", ""is_mobile"": true}" 7699,3,171,2016-12-26 02:10:56,http://reinger.com/verla.schuster,,60.177.67.67,"{""location"": ""LV"", ""is_mobile"": false}" 7700,3,171,2017-05-19 20:51:59,http://skiles.io/travon,,155.72.125.67,"{""location"": ""LA"", ""is_mobile"": false}" 7701,3,171,2017-05-20 04:51:09,http://williamson.io/margot.collins,,181.97.49.244,"{""location"": ""FR"", ""is_mobile"": false}" 7702,3,171,2016-12-24 23:38:53,http://blockbreitenberg.org/geovanni_corkery,,51.202.203.202,"{""location"": ""VC"", ""is_mobile"": true}" 7703,3,171,2017-01-10 08:51:31,http://balistreri.info/bertha_kilback,,95.49.216.169,"{""location"": ""CR"", ""is_mobile"": true}" 7704,3,171,2017-04-26 14:38:49,http://brekke.net/favian_mayer,,244.136.227.99,"{""location"": ""MU"", ""is_mobile"": true}" 7705,3,171,2017-01-09 17:29:27,http://murphy.name/kelsie.king,,241.64.73.8,"{""location"": ""AE"", ""is_mobile"": false}" 13561,5,305,2017-03-01 12:34:56,http://bruen.biz/ashlynn,,168.162.91.28,"{""location"": ""GD"", ""is_mobile"": false}" 13562,5,305,2017-06-02 10:49:37,http://wardwhite.io/kadin.volkman,,233.122.97.16,"{""location"": ""PN"", ""is_mobile"": true}" 13563,5,305,2017-04-04 20:31:53,http://swaniawskivandervort.co/lilly,,145.242.113.133,"{""location"": ""GD"", ""is_mobile"": true}" 13564,5,305,2017-02-28 10:43:23,http://lueilwitzcorkery.io/einar,,49.67.235.50,"{""location"": ""SR"", ""is_mobile"": false}" 13565,5,305,2017-05-24 08:07:34,http://moen.biz/lea.schneider,,5.201.84.92,"{""location"": ""GP"", ""is_mobile"": true}" 13566,5,305,2017-03-11 21:43:52,http://wintheiser.biz/art.casper,,225.249.21.41,"{""location"": ""GS"", ""is_mobile"": false}" 13567,5,305,2017-01-10 04:29:21,http://murray.net/dagmar,,93.204.193.142,"{""location"": ""FO"", ""is_mobile"": true}" 13568,5,305,2017-03-23 23:09:19,http://upton.net/myrtis.romaguera,,253.136.40.228,"{""location"": ""MQ"", ""is_mobile"": true}" 13569,5,305,2017-05-28 09:34:32,http://gislason.biz/nella_strosin,,160.136.222.206,"{""location"": ""NU"", ""is_mobile"": true}" 13570,5,305,2017-02-18 14:02:10,http://hamillgislason.biz/sarah.leannon,,155.213.205.74,"{""location"": ""CY"", ""is_mobile"": true}" 13571,5,305,2017-04-22 04:25:26,http://darebechtelar.name/vada.hoppe,,247.215.201.114,"{""location"": ""PE"", ""is_mobile"": false}" 13572,5,305,2017-01-05 13:44:58,http://shanahan.info/ewald_hagenes,,158.15.137.153,"{""location"": ""ZM"", ""is_mobile"": false}" 13573,5,305,2017-01-24 10:41:52,http://mosciski.info/lilliana.mayert,,149.62.50.236,"{""location"": ""SI"", ""is_mobile"": true}" 13574,5,305,2017-05-10 20:34:52,http://jenkinchuster.name/giles_larson,,30.155.10.163,"{""location"": ""TG"", ""is_mobile"": false}" 13575,5,305,2017-02-21 15:22:49,http://kub.net/genesis_armstrong,,19.34.137.223,"{""location"": ""CN"", ""is_mobile"": false}" 13576,5,305,2017-05-20 18:31:21,http://cain.com/izaiah,,43.97.73.171,"{""location"": ""HU"", ""is_mobile"": true}" 13577,5,305,2017-01-05 04:34:47,http://zieme.biz/enid,,106.250.195.79,"{""location"": ""CV"", ""is_mobile"": true}" 13578,5,305,2017-05-03 05:09:26,http://goldner.net/timothy.heaney,,219.12.25.143,"{""location"": ""SL"", ""is_mobile"": true}" 13579,5,305,2017-05-09 03:14:29,http://lebsackwhite.io/amaya.eichmann,,229.152.180.220,"{""location"": ""AG"", ""is_mobile"": false}" 13580,5,305,2017-03-05 08:30:39,http://mosciski.com/kennith_borer,,226.148.206.200,"{""location"": ""KP"", ""is_mobile"": true}" 13581,5,305,2017-03-03 14:08:05,http://marquardt.net/peggie,,208.40.239.144,"{""location"": ""BR"", ""is_mobile"": false}" 13582,5,305,2017-01-15 10:12:43,http://west.co/tyrel.howell,,211.220.134.254,"{""location"": ""MY"", ""is_mobile"": true}" 13583,5,305,2017-01-30 04:45:15,http://murphy.net/gertrude,,104.191.162.248,"{""location"": ""BF"", ""is_mobile"": false}" 13584,5,306,2017-04-16 16:42:33,http://gerhold.com/vesta,,104.242.137.169,"{""location"": ""GI"", ""is_mobile"": false}" 13585,5,306,2017-03-25 07:49:31,http://pfeffer.io/verda,,174.164.91.183,"{""location"": ""BG"", ""is_mobile"": false}" 13586,5,306,2017-03-08 21:01:29,http://champlincronin.io/margarete_sporer,,252.51.52.242,"{""location"": ""OM"", ""is_mobile"": false}" 13587,5,306,2017-04-30 02:47:38,http://shields.net/lazaro_green,,130.213.115.198,"{""location"": ""BI"", ""is_mobile"": false}" 13588,5,306,2017-04-16 14:40:13,http://johns.net/delilah_kris,,186.173.128.95,"{""location"": ""BH"", ""is_mobile"": true}" 13589,5,306,2017-05-14 21:31:56,http://considine.net/stanford,,205.114.155.193,"{""location"": ""AT"", ""is_mobile"": false}" 13590,5,306,2017-02-17 13:43:59,http://thompson.biz/tommie,,228.240.120.86,"{""location"": ""FK"", ""is_mobile"": false}" 13591,5,306,2017-04-20 08:08:15,http://bogan.com/ryley_kilback,,40.24.36.133,"{""location"": ""CZ"", ""is_mobile"": true}" 13592,5,306,2017-05-13 15:34:22,http://bins.co/adam,,169.11.76.113,"{""location"": ""LV"", ""is_mobile"": true}" 13593,5,306,2017-01-08 18:55:44,http://bahringerwintheiser.name/reta_spinka,,109.187.30.240,"{""location"": ""NL"", ""is_mobile"": true}" 13594,5,306,2016-12-16 22:03:33,http://bartoletti.com/parker,,2.211.150.59,"{""location"": ""GM"", ""is_mobile"": false}" 13595,5,306,2017-03-28 23:08:18,http://collier.info/imelda,,118.161.24.254,"{""location"": ""TF"", ""is_mobile"": true}" 13596,5,306,2017-01-03 18:55:32,http://lesch.biz/moses,,25.146.250.101,"{""location"": ""CU"", ""is_mobile"": false}" 13597,5,306,2017-04-02 22:52:20,http://smitham.net/parker.damore,,139.34.26.128,"{""location"": ""EC"", ""is_mobile"": true}" 13598,5,306,2017-01-13 12:57:23,http://little.name/heidi,,181.46.209.17,"{""location"": ""LU"", ""is_mobile"": false}" 13599,5,306,2017-05-04 22:28:34,http://armstrong.biz/serenity.nader,,147.3.254.156,"{""location"": ""HT"", ""is_mobile"": false}" 13600,5,306,2017-01-03 06:16:09,http://kubdickinson.net/carlo_skiles,,2.97.8.185,"{""location"": ""MK"", ""is_mobile"": false}" 13601,5,306,2016-12-17 07:36:27,http://kreiger.org/jeanne_gerlach,,77.42.79.33,"{""location"": ""CW"", ""is_mobile"": false}" 13602,5,306,2017-01-31 02:25:37,http://hackettbechtelar.info/melody_wintheiser,,164.139.252.151,"{""location"": ""BV"", ""is_mobile"": false}" 13603,5,306,2017-04-19 07:38:29,http://kihn.net/ernie,,155.217.149.42,"{""location"": ""LA"", ""is_mobile"": true}" 13604,5,306,2016-12-31 22:35:54,http://leuschkefranecki.biz/elinor,,127.95.203.24,"{""location"": ""HN"", ""is_mobile"": true}" 13605,5,307,2017-04-04 06:13:04,http://schmidtkihn.name/zetta_schoen,,181.189.167.23,"{""location"": ""NA"", ""is_mobile"": false}" 13606,5,307,2016-12-29 22:46:51,http://little.net/patsy,,76.240.150.115,"{""location"": ""KR"", ""is_mobile"": true}" 13607,5,307,2017-05-20 12:21:18,http://champlin.biz/roosevelt.heaney,,24.134.120.18,"{""location"": ""ZW"", ""is_mobile"": false}" 13608,5,307,2017-01-15 09:16:29,http://larson.name/jacinto.oreilly,,245.200.201.38,"{""location"": ""CX"", ""is_mobile"": true}" 13609,5,307,2017-05-06 05:32:47,http://bernhardtrantow.info/edgar.luettgen,,59.107.32.107,"{""location"": ""CL"", ""is_mobile"": false}" 13610,5,307,2017-05-04 18:33:40,http://huel.org/wilmer,,101.113.33.145,"{""location"": ""UZ"", ""is_mobile"": true}" 13611,5,307,2017-03-28 22:05:07,http://baileyupton.name/duncan_kuphal,,99.96.185.159,"{""location"": ""LS"", ""is_mobile"": false}" 13612,5,307,2017-01-11 09:47:32,http://streich.name/warren_schultz,,109.95.113.89,"{""location"": ""MM"", ""is_mobile"": false}" 13613,5,307,2017-01-23 13:06:46,http://williamsonkoelpin.org/joe,,24.199.124.147,"{""location"": ""DO"", ""is_mobile"": true}" 13614,5,307,2017-01-30 15:46:34,http://conroygorczany.com/maggie,,247.19.75.205,"{""location"": ""AW"", ""is_mobile"": true}" 13615,5,307,2017-01-26 00:40:52,http://gislason.io/daron,,23.30.205.48,"{""location"": ""SR"", ""is_mobile"": false}" 16550,6,375,2017-02-05 09:21:46,http://robel.com/eloisa.volkman,0.6949929000,193.85.197.233,"{""location"": ""HR"", ""is_mobile"": false}" 16551,6,375,2017-05-21 19:40:07,http://boehmking.co/emmie,0.2638383181,116.22.112.60,"{""location"": ""LK"", ""is_mobile"": false}" 16552,6,375,2017-01-05 01:41:10,http://renner.co/arno,0.5445909148,234.117.122.4,"{""location"": ""HM"", ""is_mobile"": false}" 16553,6,375,2017-03-29 12:48:56,http://larsonlueilwitz.co/dana,0.0607361288,8.203.136.101,"{""location"": ""KR"", ""is_mobile"": false}" 16554,6,375,2017-04-24 01:44:43,http://oreilly.co/lennie_tremblay,0.4115837413,76.126.140.208,"{""location"": ""KZ"", ""is_mobile"": false}" 16555,6,375,2017-01-07 20:04:25,http://champlinrath.org/noemie_morar,0.7261834535,47.11.11.244,"{""location"": ""MC"", ""is_mobile"": true}" 16556,6,375,2017-03-20 19:15:18,http://blockemmerich.io/janice,0.7297880856,244.188.36.115,"{""location"": ""CC"", ""is_mobile"": true}" 16557,6,375,2017-04-12 01:03:59,http://mayert.co/claire,0.5893664110,252.54.210.190,"{""location"": ""GB"", ""is_mobile"": false}" 16558,6,375,2017-03-23 20:10:15,http://ward.io/desiree,0.1995249701,121.48.239.26,"{""location"": ""CF"", ""is_mobile"": true}" 16559,6,375,2017-02-24 14:57:16,http://robel.org/eleonore,0.0981679080,121.226.95.251,"{""location"": ""MG"", ""is_mobile"": false}" 16560,6,375,2017-01-22 05:30:17,http://zemlakmohr.info/jeyca.weinat,0.6726298085,9.162.233.210,"{""location"": ""FR"", ""is_mobile"": false}" 16561,6,375,2017-02-26 10:48:11,http://hudson.net/jovan,0.9269268113,93.66.169.211,"{""location"": ""AI"", ""is_mobile"": true}" 16562,6,375,2016-12-25 04:59:12,http://gerlach.info/dudley,0.5294814726,206.164.246.143,"{""location"": ""NU"", ""is_mobile"": true}" 16563,6,375,2017-02-13 17:52:48,http://jacobsvandervort.name/tyrese.leuschke,0.5935772800,226.127.5.143,"{""location"": ""NR"", ""is_mobile"": false}" 16564,6,375,2017-02-03 05:50:28,http://tromphermiston.biz/linda,0.4299662640,153.151.84.207,"{""location"": ""CD"", ""is_mobile"": false}" 16565,6,375,2017-04-16 23:35:55,http://ohara.name/sydney,0.1784514638,45.195.177.173,"{""location"": ""GN"", ""is_mobile"": true}" 16566,6,375,2017-03-11 15:31:38,http://rosenbaum.com/shanie,0.5172540134,78.48.161.217,"{""location"": ""AD"", ""is_mobile"": true}" 16567,6,375,2017-02-24 07:25:37,http://dibbertpollich.net/marietta,0.6741563282,246.42.69.240,"{""location"": ""PM"", ""is_mobile"": false}" 16568,6,375,2017-05-13 18:46:20,http://legros.org/adaline.grady,0.3895963991,25.64.179.226,"{""location"": ""MQ"", ""is_mobile"": true}" 16569,6,376,2017-01-20 09:49:12,http://rohan.org/kay,0.1809315447,87.58.12.110,"{""location"": ""FJ"", ""is_mobile"": true}" 16570,6,376,2017-01-13 22:20:22,http://lemkerodriguez.co/kellen.casper,0.0504526179,232.199.147.174,"{""location"": ""ID"", ""is_mobile"": false}" 16571,6,376,2016-12-16 17:31:44,http://trantow.biz/sven_maggio,0.4401615469,37.122.119.204,"{""location"": ""PA"", ""is_mobile"": true}" 16572,6,376,2017-02-05 08:08:35,http://schustermosciski.info/zack,0.6202902262,184.22.111.106,"{""location"": ""HR"", ""is_mobile"": false}" 16573,6,376,2017-05-25 17:21:37,http://cremin.io/westley,0.4146574479,58.146.26.23,"{""location"": ""JO"", ""is_mobile"": false}" 16574,6,376,2017-03-20 13:11:46,http://wunschwilkinson.net/rene,0.6886023629,110.251.141.94,"{""location"": ""HN"", ""is_mobile"": false}" 16575,6,376,2017-02-12 18:36:36,http://reinger.io/giovanna,0.8972720373,59.188.168.12,"{""location"": ""LA"", ""is_mobile"": true}" 16576,6,376,2017-04-29 19:31:48,http://spencer.biz/kamron,0.3230641352,191.174.110.5,"{""location"": ""NE"", ""is_mobile"": true}" 16577,6,376,2016-12-16 16:59:44,http://westspencer.name/nasir,0.9636904835,87.57.214.97,"{""location"": ""SS"", ""is_mobile"": false}" 16578,6,376,2017-04-11 02:21:45,http://boganmcdermott.name/helga_moore,0.2502849076,207.239.125.186,"{""location"": ""IO"", ""is_mobile"": false}" 16579,6,376,2017-05-22 13:36:33,http://gulgowskiwiza.info/dee_kulas,0.8315659239,128.35.147.238,"{""location"": ""VU"", ""is_mobile"": true}" 16580,6,376,2016-12-23 07:46:38,http://herman.com/richard.towne,0.6425510365,195.111.207.238,"{""location"": ""LI"", ""is_mobile"": false}" 16581,6,376,2017-04-19 04:25:37,http://jacobson.com/sabrina,0.5782485286,52.171.169.181,"{""location"": ""MS"", ""is_mobile"": false}" 16582,6,376,2017-02-12 15:19:59,http://bradtkeroob.com/karianne_rutherford,0.6000834283,249.110.63.63,"{""location"": ""IO"", ""is_mobile"": true}" 16583,6,376,2017-05-04 13:50:29,http://luettgen.net/demarcus.connelly,0.9955280651,161.251.32.13,"{""location"": ""CO"", ""is_mobile"": false}" 16584,6,376,2017-04-08 13:17:46,http://erdman.name/elenora,0.8318231835,49.116.58.146,"{""location"": ""BR"", ""is_mobile"": false}" 16585,6,376,2017-06-12 17:37:15,http://batz.net/idella,0.9257205453,104.28.10.74,"{""location"": ""LC"", ""is_mobile"": false}" 16586,6,376,2017-05-05 19:15:47,http://greenholt.co/gunnar.schuppe,0.9934659527,67.205.205.202,"{""location"": ""BQ"", ""is_mobile"": true}" 16587,6,376,2017-02-18 00:06:51,http://langworthklein.io/stone,0.5751141785,203.134.136.130,"{""location"": ""PR"", ""is_mobile"": true}" 16588,6,376,2017-03-11 09:43:31,http://wolfkuhlman.biz/rachael,0.0841935518,215.131.208.181,"{""location"": ""SX"", ""is_mobile"": false}" 16589,6,376,2017-03-22 13:57:07,http://homenickmonahan.co/eloy.okon,0.7589594296,46.90.124.123,"{""location"": ""MC"", ""is_mobile"": true}" 16590,6,376,2017-01-18 06:56:30,http://blick.info/wava,0.3006569116,113.153.212.35,"{""location"": ""BR"", ""is_mobile"": true}" 16591,6,376,2017-02-08 09:58:52,http://schuppe.info/leonard,0.8832744749,57.140.217.4,"{""location"": ""BO"", ""is_mobile"": false}" 16592,6,376,2017-03-03 20:05:02,http://kuhlmanfritsch.info/carey,0.2515468007,36.171.124.160,"{""location"": ""AT"", ""is_mobile"": false}" 16593,6,376,2017-04-20 12:59:02,http://hudson.biz/gertrude,0.9893260249,43.61.107.76,"{""location"": ""GW"", ""is_mobile"": true}" 16594,6,376,2017-02-04 13:35:27,http://ortizhoppe.com/shirley_altenwerth,0.2920871690,205.37.58.23,"{""location"": ""AS"", ""is_mobile"": false}" 16595,6,376,2017-01-02 16:34:38,http://hartmann.name/enola.powlowski,0.5935290543,203.132.178.215,"{""location"": ""FK"", ""is_mobile"": true}" 16596,6,376,2016-12-20 09:11:12,http://ankunding.biz/miles_ortiz,0.7730332284,88.93.147.180,"{""location"": ""TZ"", ""is_mobile"": true}" 16597,6,376,2017-01-20 15:00:49,http://ratkefriesen.com/hermina,0.0099469422,146.54.158.216,"{""location"": ""IL"", ""is_mobile"": false}" 16598,6,376,2017-02-26 09:28:01,http://weimann.co/elisa.orn,0.0039764457,113.172.135.187,"{""location"": ""CU"", ""is_mobile"": true}" 16599,6,376,2016-12-19 01:37:42,http://bergnaum.io/darlene,0.7238795796,135.170.28.251,"{""location"": ""SJ"", ""is_mobile"": false}" 16600,6,376,2017-03-13 14:44:19,http://cormier.net/sebastian_pollich,0.9447776791,35.18.214.32,"{""location"": ""NE"", ""is_mobile"": true}" 3730,2,82,2017-01-23 15:59:38,http://krajcik.io/desiree_nienow,0.4445342905,250.235.178.87,"{""location"": ""DE"", ""is_mobile"": false}" 3731,2,82,2017-05-25 21:40:34,http://batz.net/monique.smitham,0.0245300165,53.171.123.125,"{""location"": ""GY"", ""is_mobile"": true}" 3732,2,82,2016-12-25 21:13:48,http://mcculloughtillman.io/greyson_nader,0.8107643054,224.20.242.107,"{""location"": ""VA"", ""is_mobile"": false}" 3733,2,82,2017-05-29 17:11:42,http://trantow.info/corrine,0.3126949361,11.172.68.4,"{""location"": ""CH"", ""is_mobile"": false}" 3734,2,82,2017-01-28 19:51:01,http://stiedemann.biz/gabrielle,0.1411261311,248.15.89.222,"{""location"": ""SL"", ""is_mobile"": false}" 3735,2,82,2017-05-18 02:24:44,http://leannon.com/ottis,0.0324509181,106.121.152.218,"{""location"": ""VG"", ""is_mobile"": false}" 3736,2,82,2017-05-28 16:06:17,http://kautzer.biz/haylee,0.6244162394,123.249.141.128,"{""location"": ""MH"", ""is_mobile"": true}" 3737,2,82,2017-04-09 19:23:49,http://oberbrunner.net/reagan_durgan,0.9772428084,224.209.238.37,"{""location"": ""BZ"", ""is_mobile"": false}" 3738,2,82,2017-04-24 02:36:58,http://gerhold.biz/joe_ortiz,0.7578866000,114.50.241.242,"{""location"": ""NZ"", ""is_mobile"": true}" 3739,2,82,2017-04-14 12:12:09,http://hirthespinka.co/luigi,0.2498373962,46.162.244.21,"{""location"": ""FK"", ""is_mobile"": true}" 3740,2,82,2017-02-08 23:53:23,http://deckow.net/jadyn_orn,0.7306647390,242.187.108.183,"{""location"": ""NG"", ""is_mobile"": false}" 3741,2,82,2017-02-25 01:17:29,http://klein.net/bertrand,0.2235401421,67.218.205.86,"{""location"": ""US"", ""is_mobile"": true}" 3742,2,82,2017-04-29 15:27:08,http://bernhardcorkery.com/kaelyn_boyle,0.8041746759,67.183.92.72,"{""location"": ""KM"", ""is_mobile"": true}" 3743,2,82,2017-01-04 19:36:16,http://baileyschneider.net/jacklyn_blanda,0.3777350757,10.95.202.127,"{""location"": ""PH"", ""is_mobile"": true}" 3744,2,82,2016-12-13 14:24:52,http://beahan.net/bret,0.3751634681,8.149.220.175,"{""location"": ""PR"", ""is_mobile"": true}" 3745,2,82,2017-04-30 14:08:39,http://conn.name/guy_mckenzie,0.5590909361,3.29.83.248,"{""location"": ""MU"", ""is_mobile"": false}" 3746,2,82,2017-06-06 21:31:45,http://robel.info/kennedy_abernathy,0.2610399275,240.58.98.163,"{""location"": ""SZ"", ""is_mobile"": false}" 3747,2,82,2017-04-30 06:06:50,http://grimesarmstrong.io/alysa,0.1555998387,234.184.176.72,"{""location"": ""BI"", ""is_mobile"": false}" 3748,2,82,2016-12-28 11:24:51,http://bosco.info/sven_gleason,0.6114911391,81.105.114.93,"{""location"": ""NO"", ""is_mobile"": false}" 3749,2,82,2017-01-19 17:05:14,http://treutel.net/lulu.herzog,0.1282911193,65.7.89.122,"{""location"": ""VC"", ""is_mobile"": false}" 3750,2,82,2017-05-10 21:14:19,http://hicklegrady.info/randy,0.1158412531,206.10.123.35,"{""location"": ""CV"", ""is_mobile"": false}" 3751,2,82,2017-03-22 18:02:00,http://heidenreich.net/samanta,0.7563317070,22.218.154.143,"{""location"": ""GB"", ""is_mobile"": false}" 3752,2,82,2016-12-26 03:06:24,http://armstrong.com/cory,0.1902196335,195.152.177.137,"{""location"": ""QA"", ""is_mobile"": false}" 3753,2,82,2017-04-21 16:23:45,http://lueilwitzkuhlman.net/helena,0.7396126422,253.177.46.22,"{""location"": ""UA"", ""is_mobile"": false}" 3754,2,82,2017-04-06 15:50:02,http://jacobswalter.co/araceli,0.1059263279,145.73.92.242,"{""location"": ""JP"", ""is_mobile"": false}" 3755,2,82,2017-01-10 08:41:49,http://bartoncarter.info/paxton_greenholt,0.6855307198,232.203.165.228,"{""location"": ""MS"", ""is_mobile"": false}" 3756,2,82,2017-05-12 09:44:23,http://carroll.biz/serenity.nolan,0.0720488790,202.138.61.166,"{""location"": ""AU"", ""is_mobile"": false}" 3757,2,82,2016-12-13 20:44:12,http://walter.net/landen,0.5364589920,241.158.162.38,"{""location"": ""SJ"", ""is_mobile"": false}" 3758,2,82,2017-05-23 14:14:23,http://leffler.co/stefanie_littel,0.0844675819,19.13.127.148,"{""location"": ""PK"", ""is_mobile"": true}" 3759,2,82,2017-03-11 17:17:54,http://ohara.com/keaton.sanford,0.4500484821,94.210.132.57,"{""location"": ""DM"", ""is_mobile"": false}" 3760,2,82,2017-01-22 03:13:30,http://runtehills.org/ashlynn,0.3145458912,138.119.218.26,"{""location"": ""ML"", ""is_mobile"": true}" 3761,2,82,2017-04-04 07:05:03,http://macgyverblick.info/tyshawn,0.4754459346,41.152.243.37,"{""location"": ""NP"", ""is_mobile"": false}" 3762,2,82,2017-04-12 19:27:51,http://harber.org/cathrine,0.3886921842,86.175.174.200,"{""location"": ""KP"", ""is_mobile"": true}" 3763,2,82,2017-05-13 08:43:00,http://simonis.com/alta_jenkins,0.2373924440,114.185.149.191,"{""location"": ""TJ"", ""is_mobile"": false}" 3764,2,82,2017-02-20 08:06:28,http://millsdoyle.biz/jerald,0.4185823068,142.67.115.10,"{""location"": ""JE"", ""is_mobile"": true}" 3765,2,82,2017-02-11 12:59:00,http://runte.io/lucinda,0.3251611265,168.27.177.91,"{""location"": ""SJ"", ""is_mobile"": false}" 3766,2,82,2016-12-17 00:33:42,http://wolff.co/jefferey,0.6383166665,53.135.40.230,"{""location"": ""MG"", ""is_mobile"": true}" 3767,2,82,2016-12-21 05:15:02,http://kerluke.name/sister,0.2665727269,222.212.69.4,"{""location"": ""VG"", ""is_mobile"": true}" 3768,2,82,2017-03-22 12:13:44,http://prohaska.io/katheryn.cartwright,0.0858944588,238.207.94.186,"{""location"": ""KG"", ""is_mobile"": true}" 3769,2,83,2017-02-07 20:29:16,http://collins.net/demetris_kuhlman,0.6913464708,4.224.208.148,"{""location"": ""DE"", ""is_mobile"": false}" 3770,2,83,2017-02-07 11:42:18,http://mraz.com/christiana,0.4491614033,241.141.27.241,"{""location"": ""GY"", ""is_mobile"": true}" 3771,2,83,2017-04-27 07:03:09,http://farrell.name/misty.kub,0.2532366731,16.154.50.81,"{""location"": ""HM"", ""is_mobile"": true}" 3772,2,83,2017-05-30 11:33:32,http://crookshilll.name/shea.kihn,0.0923430092,216.148.228.159,"{""location"": ""PK"", ""is_mobile"": true}" 3773,2,83,2017-03-02 22:42:50,http://stanton.com/jude.gaylord,0.1618154374,102.118.250.23,"{""location"": ""AF"", ""is_mobile"": false}" 3774,2,83,2017-01-15 23:47:30,http://okeefeward.com/judd_bogan,0.7488847529,237.114.138.14,"{""location"": ""CX"", ""is_mobile"": true}" 3775,2,83,2017-05-06 22:28:56,http://miller.com/allison,0.7724482953,86.129.120.19,"{""location"": ""CL"", ""is_mobile"": false}" 3776,2,83,2016-12-21 00:01:35,http://jast.name/aileen,0.2876425937,56.158.197.195,"{""location"": ""VG"", ""is_mobile"": false}" 3777,2,83,2016-12-25 05:23:24,http://schinner.com/cole,0.5613038915,170.27.248.159,"{""location"": ""AG"", ""is_mobile"": false}" 3778,2,83,2017-03-11 08:15:55,http://white.com/jerrod_kirlin,0.8321620892,60.23.47.195,"{""location"": ""SE"", ""is_mobile"": false}" 3779,2,83,2017-06-06 21:57:23,http://conn.biz/else.kiehn,0.4787418351,44.13.86.138,"{""location"": ""TJ"", ""is_mobile"": true}" 3780,2,83,2017-06-02 01:38:31,http://schulist.com/donavon_lang,0.8682351409,243.234.227.66,"{""location"": ""PA"", ""is_mobile"": false}" 3781,2,83,2017-05-11 04:37:03,http://harris.name/aileen,0.4740015994,142.34.3.49,"{""location"": ""BW"", ""is_mobile"": false}" 10665,4,239,2017-04-09 19:26:32,http://mosciski.org/deontae_reynolds,,208.136.131.78,"{""location"": ""FJ"", ""is_mobile"": false}" 10666,4,239,2017-01-31 21:57:11,http://bosco.net/zane_gutkowski,,241.8.239.156,"{""location"": ""KW"", ""is_mobile"": false}" 10667,4,239,2017-05-06 06:30:21,http://gerhold.info/arturo_kaulke,,226.119.127.170,"{""location"": ""AM"", ""is_mobile"": true}" 10668,4,239,2016-12-14 01:59:34,http://gerlachbatz.co/wilfred_leuschke,,221.131.28.128,"{""location"": ""SB"", ""is_mobile"": false}" 10669,4,239,2017-03-10 14:28:04,http://olsonwaters.org/paige_bartell,,45.202.49.62,"{""location"": ""ID"", ""is_mobile"": true}" 10670,4,239,2017-02-14 23:08:03,http://purdy.net/caria,,37.149.87.164,"{""location"": ""SA"", ""is_mobile"": true}" 10671,4,239,2017-01-15 21:40:38,http://vonmoen.biz/columbus.wunsch,,76.248.120.37,"{""location"": ""CY"", ""is_mobile"": true}" 10672,4,239,2017-02-26 04:08:24,http://labadie.name/murl.swaniawski,,107.110.55.195,"{""location"": ""LK"", ""is_mobile"": true}" 10673,4,239,2017-05-31 22:39:32,http://bogisich.net/haie,,66.66.177.242,"{""location"": ""PG"", ""is_mobile"": false}" 10674,4,239,2016-12-26 16:43:45,http://jones.io/mona.bogan,,137.196.16.251,"{""location"": ""EC"", ""is_mobile"": false}" 10675,4,239,2017-05-31 17:44:57,http://murazikko.biz/josefina.reynolds,,160.39.239.196,"{""location"": ""KG"", ""is_mobile"": false}" 10676,4,239,2017-05-11 02:20:15,http://hartmann.org/jee,,184.54.120.190,"{""location"": ""MM"", ""is_mobile"": true}" 10677,4,239,2017-02-10 00:41:56,http://haleygrant.co/pansy,,184.200.118.20,"{""location"": ""KZ"", ""is_mobile"": true}" 10678,4,239,2017-04-13 18:18:59,http://towneerdman.net/fidel.tremblay,,22.212.2.38,"{""location"": ""TC"", ""is_mobile"": true}" 10679,4,239,2017-05-20 16:01:33,http://fadel.net/nayeli,,75.205.40.131,"{""location"": ""SO"", ""is_mobile"": false}" 10680,4,239,2017-05-04 01:11:15,http://grant.com/greyson,,122.197.189.40,"{""location"": ""GP"", ""is_mobile"": true}" 10681,4,239,2017-04-16 01:49:05,http://stiedemann.name/retha_grimes,,57.88.3.177,"{""location"": ""DM"", ""is_mobile"": true}" 10682,4,239,2017-01-29 02:07:09,http://rohan.com/juwan.hand,,152.203.9.136,"{""location"": ""AS"", ""is_mobile"": true}" 10683,4,239,2017-03-18 09:01:07,http://goyette.co/pasquale,,17.173.152.38,"{""location"": ""BS"", ""is_mobile"": false}" 10684,4,239,2017-02-20 22:38:38,http://conn.name/augustus,,22.18.56.141,"{""location"": ""SE"", ""is_mobile"": true}" 10685,4,239,2017-01-14 13:36:58,http://greenholt.biz/flavio.collier,,247.110.15.105,"{""location"": ""AG"", ""is_mobile"": true}" 10686,4,239,2017-01-03 14:17:36,http://botsford.co/garnet.medhurst,,236.154.184.195,"{""location"": ""HM"", ""is_mobile"": false}" 10687,4,239,2017-01-03 20:20:54,http://naderwilliamson.co/mylene,,142.211.2.213,"{""location"": ""AL"", ""is_mobile"": false}" 10688,4,239,2016-12-29 02:23:05,http://lindgrenschinner.name/isabelle.buckridge,,79.135.235.126,"{""location"": ""BZ"", ""is_mobile"": true}" 10689,4,239,2017-05-18 22:06:28,http://whitebogan.biz/german.huels,,66.121.191.68,"{""location"": ""EG"", ""is_mobile"": false}" 10690,4,239,2017-03-03 00:03:56,http://aufderhar.net/darrell_harvey,,185.83.56.228,"{""location"": ""AG"", ""is_mobile"": false}" 10691,4,240,2017-05-11 06:32:51,http://labadie.io/jason,,150.67.201.69,"{""location"": ""US"", ""is_mobile"": true}" 10692,4,240,2017-04-27 05:42:43,http://kopouros.biz/arvilla.morar,,132.148.246.208,"{""location"": ""KN"", ""is_mobile"": false}" 10693,4,240,2017-02-16 17:30:28,http://bernhardlakin.co/felix,,89.225.173.167,"{""location"": ""FI"", ""is_mobile"": true}" 10694,4,240,2017-06-12 20:11:37,http://williamson.name/ike_ryan,,217.88.170.211,"{""location"": ""OM"", ""is_mobile"": true}" 10695,4,240,2017-03-01 00:55:34,http://tromp.biz/vicenta_doyle,,16.41.2.248,"{""location"": ""IE"", ""is_mobile"": false}" 10696,4,240,2017-01-05 15:27:27,http://towne.com/chanel,,245.69.107.232,"{""location"": ""AD"", ""is_mobile"": false}" 10697,4,240,2017-05-13 16:10:35,http://daniel.name/nelle.huels,,108.37.75.203,"{""location"": ""CH"", ""is_mobile"": true}" 10698,4,240,2017-06-12 23:01:33,http://thompson.name/demond.brown,,3.166.16.136,"{""location"": ""PW"", ""is_mobile"": false}" 10699,4,240,2017-02-09 10:43:54,http://rohan.co/margarett.wiza,,19.201.89.107,"{""location"": ""TO"", ""is_mobile"": true}" 10700,4,240,2017-05-24 19:05:47,http://runolfsdottir.co/morris_trantow,,240.229.185.206,"{""location"": ""IN"", ""is_mobile"": false}" 10701,4,240,2016-12-14 14:43:35,http://hahndickens.co/giuseppe.parisian,,73.121.43.132,"{""location"": ""AO"", ""is_mobile"": false}" 10702,4,240,2017-05-20 13:34:58,http://cremin.org/denis.trantow,,79.252.199.27,"{""location"": ""MO"", ""is_mobile"": true}" 10703,4,240,2017-04-05 06:12:29,http://dickiankunding.info/kylee_eichmann,,50.73.119.200,"{""location"": ""LB"", ""is_mobile"": true}" 10704,4,240,2017-04-01 10:02:55,http://collier.io/johnson.beatty,,93.166.110.162,"{""location"": ""TO"", ""is_mobile"": true}" 10705,4,240,2016-12-23 19:44:33,http://cronin.io/bertha_mclaughlin,,124.57.113.209,"{""location"": ""TZ"", ""is_mobile"": true}" 10706,4,240,2017-02-22 14:36:25,http://armstrong.org/rachael,,237.6.217.215,"{""location"": ""DJ"", ""is_mobile"": true}" 10707,4,240,2017-02-08 09:03:34,http://lind.net/armand.kreiger,,170.143.9.85,"{""location"": ""KR"", ""is_mobile"": false}" 10708,4,240,2017-04-21 22:39:02,http://hoeger.net/dorothy.beahan,,145.8.251.62,"{""location"": ""GE"", ""is_mobile"": true}" 10709,4,240,2017-04-12 04:14:09,http://spencer.co/donnie,,65.78.101.66,"{""location"": ""VE"", ""is_mobile"": false}" 10710,4,240,2017-05-01 08:23:42,http://cruickshank.com/felix_gottlieb,,192.180.15.230,"{""location"": ""NI"", ""is_mobile"": false}" 10711,4,240,2017-06-08 02:37:21,http://reynolds.io/trenton.dietrich,,160.239.187.241,"{""location"": ""KP"", ""is_mobile"": false}" 10712,4,240,2016-12-25 01:16:43,http://zulauf.org/tyrell.bode,,134.113.222.17,"{""location"": ""AO"", ""is_mobile"": false}" 10713,4,240,2017-01-11 10:35:34,http://schuppe.co/maddison,,160.57.196.163,"{""location"": ""CW"", ""is_mobile"": false}" 10714,4,240,2016-12-19 06:40:37,http://mclaughlin.io/alvina_shanahan,,107.208.88.35,"{""location"": ""GH"", ""is_mobile"": false}" 10715,4,240,2017-05-03 22:49:30,http://rauokon.info/tierra,,251.24.43.105,"{""location"": ""NG"", ""is_mobile"": false}" 10716,4,240,2017-05-29 03:36:36,http://dickieffertz.com/jules,,58.135.181.26,"{""location"": ""KH"", ""is_mobile"": false}" 10717,4,240,2017-02-16 04:34:53,http://kulas.co/everette_berge,,100.219.149.56,"{""location"": ""CG"", ""is_mobile"": false}" 10718,4,240,2017-03-05 05:35:14,http://feil.co/orlando,,146.200.96.145,"{""location"": ""VI"", ""is_mobile"": false}" 10719,4,240,2017-02-27 16:24:37,http://hackettcruickshank.com/ramona_greenholt,,124.234.185.114,"{""location"": ""AX"", ""is_mobile"": true}" 7706,3,171,2016-12-20 21:09:07,http://zemlakvandervort.net/marlin.langosh,,56.218.155.211,"{""location"": ""UZ"", ""is_mobile"": false}" 7707,3,171,2017-05-11 21:11:49,http://kautzer.org/dayana_oconner,,196.93.197.135,"{""location"": ""HM"", ""is_mobile"": true}" 7708,3,171,2017-06-12 11:24:13,http://dooley.biz/king,,215.163.214.208,"{""location"": ""GE"", ""is_mobile"": false}" 7709,3,171,2017-05-18 07:25:25,http://walter.org/marcella_shanahan,,123.37.244.138,"{""location"": ""NR"", ""is_mobile"": true}" 7710,3,171,2017-01-22 09:34:21,http://wisozk.name/deangelo_schmitt,,77.95.210.10,"{""location"": ""AR"", ""is_mobile"": true}" 7711,3,171,2017-01-07 01:11:45,http://jacobs.io/crystal_hettinger,,163.44.76.179,"{""location"": ""GD"", ""is_mobile"": true}" 7712,3,171,2017-06-02 23:18:27,http://hoppe.org/eleanora,,209.193.50.72,"{""location"": ""AI"", ""is_mobile"": false}" 7713,3,171,2017-04-17 17:37:02,http://rutherfordtreutel.biz/nicole,,49.36.238.185,"{""location"": ""BV"", ""is_mobile"": true}" 7714,3,171,2016-12-20 11:31:44,http://lakin.biz/marilie,,251.149.155.4,"{""location"": ""BO"", ""is_mobile"": true}" 7715,3,171,2017-05-30 09:12:22,http://raynor.biz/terrill,,158.37.25.160,"{""location"": ""KP"", ""is_mobile"": true}" 7716,3,171,2017-04-16 07:12:02,http://friesensmith.net/citlalli,,125.76.91.205,"{""location"": ""CX"", ""is_mobile"": false}" 7717,3,171,2017-04-08 10:32:59,http://fisher.name/hobart_boehm,,142.143.14.114,"{""location"": ""PY"", ""is_mobile"": false}" 7718,3,172,2017-03-20 01:09:07,http://damore.org/dolly.will,0.9870148939,16.90.140.233,"{""location"": ""MR"", ""is_mobile"": false}" 7719,3,172,2017-04-19 02:34:08,http://boyle.net/audrey,0.0981862012,161.103.20.231,"{""location"": ""BW"", ""is_mobile"": true}" 7720,3,172,2017-04-14 08:44:05,http://welchemard.co/sylvester.gerhold,0.2991805303,220.164.224.199,"{""location"": ""BB"", ""is_mobile"": true}" 7721,3,172,2017-06-03 00:36:45,http://hansen.io/dixie,0.4252261409,8.232.244.37,"{""location"": ""CW"", ""is_mobile"": false}" 7722,3,172,2016-12-14 04:31:39,http://bartolettirohan.org/shayna_hauck,0.2662086494,233.84.63.35,"{""location"": ""ER"", ""is_mobile"": false}" 7723,3,172,2016-12-15 02:06:01,http://johnson.org/abagail.moriette,0.2696360854,48.226.142.138,"{""location"": ""KZ"", ""is_mobile"": false}" 7724,3,172,2017-05-21 23:24:46,http://mcclure.info/olen_morar,0.7801248300,62.247.67.150,"{""location"": ""NE"", ""is_mobile"": true}" 7725,3,172,2017-04-12 16:58:33,http://kihngoldner.info/aunta.daniel,0.3234418532,225.137.96.98,"{""location"": ""LB"", ""is_mobile"": true}" 7726,3,172,2017-03-25 07:27:17,http://mcculloughbednar.org/horacio,0.7912951602,46.125.166.70,"{""location"": ""SB"", ""is_mobile"": true}" 7727,3,172,2017-01-15 12:45:21,http://kuphal.net/vella,0.0203223484,174.99.39.192,"{""location"": ""GW"", ""is_mobile"": true}" 7728,3,172,2017-05-29 00:44:29,http://treutelherzog.co/bria,0.2754350707,172.30.173.94,"{""location"": ""KG"", ""is_mobile"": true}" 7729,3,172,2016-12-15 13:34:21,http://gibson.net/trever.ernser,0.2247154880,114.210.54.95,"{""location"": ""GP"", ""is_mobile"": false}" 7730,3,172,2017-03-12 04:01:05,http://reingergerlach.org/ward.haag,0.9585523972,203.210.198.119,"{""location"": ""GQ"", ""is_mobile"": true}" 7731,3,172,2017-03-03 16:58:41,http://littelmetz.info/waino.wehner,0.0395603348,244.127.63.226,"{""location"": ""TH"", ""is_mobile"": false}" 7732,3,172,2017-03-21 17:48:40,http://deckow.info/vickie,0.8080118317,2.108.226.2,"{""location"": ""NA"", ""is_mobile"": true}" 7733,3,172,2017-04-18 11:45:12,http://bogan.com/adelle,0.7814021989,84.117.250.176,"{""location"": ""TO"", ""is_mobile"": true}" 7734,3,172,2016-12-22 04:12:14,http://heathcote.io/yasmine_zemlak,0.9524736155,70.222.144.67,"{""location"": ""PW"", ""is_mobile"": false}" 7735,3,172,2017-02-18 04:51:54,http://tromp.io/hillard.graham,0.8427736778,2.125.164.73,"{""location"": ""SB"", ""is_mobile"": false}" 7736,3,172,2017-02-03 20:01:14,http://heidenreichyundt.net/ulises_weimann,0.6632473895,213.81.15.139,"{""location"": ""HK"", ""is_mobile"": false}" 7737,3,172,2017-02-18 00:47:50,http://cristfriesen.io/brian_pollich,0.7213159779,158.118.245.201,"{""location"": ""NA"", ""is_mobile"": true}" 7738,3,172,2017-02-06 05:22:59,http://olson.org/dayton,0.4909528422,196.251.184.169,"{""location"": ""GU"", ""is_mobile"": false}" 7739,3,172,2017-02-11 18:19:13,http://kirlinmacgyver.co/ola_leannon,0.1246779852,28.61.19.35,"{""location"": ""ZM"", ""is_mobile"": true}" 7740,3,172,2017-02-08 09:44:19,http://mcglynn.info/ezra,0.4471496276,226.38.207.137,"{""location"": ""TC"", ""is_mobile"": false}" 7741,3,172,2016-12-30 10:10:51,http://mclaughlin.info/kaelyn.oconner,0.0706284163,99.169.104.62,"{""location"": ""BQ"", ""is_mobile"": true}" 7742,3,172,2017-05-02 05:02:33,http://bashirian.com/carlos,0.7031612968,149.116.36.21,"{""location"": ""YT"", ""is_mobile"": true}" 7743,3,172,2017-03-22 15:41:00,http://berge.io/stephania,0.8192396451,45.156.201.75,"{""location"": ""LV"", ""is_mobile"": false}" 7744,3,172,2017-05-10 15:34:51,http://casper.org/oran_ohara,0.5670537300,118.43.83.138,"{""location"": ""GN"", ""is_mobile"": true}" 7745,3,172,2017-06-02 11:31:56,http://sanford.info/name,0.2036572521,184.151.141.125,"{""location"": ""VE"", ""is_mobile"": true}" 7746,3,172,2016-12-13 23:49:06,http://kirlin.org/mozell.becker,0.0014899413,181.13.105.85,"{""location"": ""HT"", ""is_mobile"": true}" 7747,3,172,2017-01-20 06:42:24,http://connelly.name/madge.monahan,0.7065613316,164.97.28.240,"{""location"": ""MO"", ""is_mobile"": false}" 7748,3,172,2017-05-07 15:56:43,http://ankundingmaggio.com/bo,0.0912106695,205.12.171.173,"{""location"": ""DO"", ""is_mobile"": true}" 7749,3,172,2016-12-15 05:58:50,http://douglasklocko.biz/rowena_harber,0.6029303275,150.77.23.190,"{""location"": ""GA"", ""is_mobile"": true}" 7750,3,172,2017-01-04 07:10:57,http://greenfelder.co/charlotte,0.0206210600,28.100.46.197,"{""location"": ""WS"", ""is_mobile"": true}" 7751,3,172,2017-06-03 17:44:45,http://carroll.name/elvis,0.3298061607,94.7.142.7,"{""location"": ""KH"", ""is_mobile"": true}" 7752,3,172,2017-02-09 00:37:18,http://witting.io/alfonso_hodkiewicz,0.5386554706,160.145.67.18,"{""location"": ""UM"", ""is_mobile"": true}" 7753,3,172,2017-06-08 03:47:18,http://nitzsche.net/nolan.buckridge,0.7242867838,29.206.242.36,"{""location"": ""VG"", ""is_mobile"": true}" 7754,3,172,2017-02-28 05:20:51,http://vonrueden.biz/aglae.schroeder,0.5086554700,15.254.15.89,"{""location"": ""CZ"", ""is_mobile"": true}" 7755,3,172,2017-03-04 00:13:42,http://fadel.info/effie.ko,0.0855208025,196.179.250.110,"{""location"": ""JO"", ""is_mobile"": true}" 7756,3,172,2016-12-23 08:56:51,http://block.info/tristin,0.5773946599,69.246.16.90,"{""location"": ""SO"", ""is_mobile"": true}" 7757,3,172,2017-01-19 17:47:13,http://jacobidietrich.biz/jackeline,0.8952473707,113.114.83.247,"{""location"": ""TH"", ""is_mobile"": true}" 7758,3,172,2017-02-24 19:01:41,http://eichmann.co/audra.auer,0.3185523449,180.186.112.47,"{""location"": ""NP"", ""is_mobile"": false}" 13616,5,307,2017-04-03 00:48:24,http://hodkiewicz.info/ellsworth,,145.28.110.156,"{""location"": ""CL"", ""is_mobile"": true}" 13617,5,307,2017-01-16 08:11:04,http://ondricka.name/jodie_hickle,,147.146.74.210,"{""location"": ""TM"", ""is_mobile"": true}" 13618,5,307,2017-04-24 07:57:46,http://bergstrompollich.org/sterling_rempel,,117.11.3.146,"{""location"": ""CR"", ""is_mobile"": true}" 13619,5,307,2017-02-10 13:12:46,http://mcglynn.com/khalil.kulas,,246.149.147.238,"{""location"": ""QA"", ""is_mobile"": true}" 13620,5,307,2017-01-20 15:12:52,http://corkery.name/anastasia,,72.208.40.12,"{""location"": ""KI"", ""is_mobile"": false}" 13621,5,307,2017-04-29 17:18:18,http://ruel.name/dimitri,,72.46.29.169,"{""location"": ""YT"", ""is_mobile"": false}" 13622,5,307,2017-01-12 05:59:59,http://block.org/kraig_schinner,,179.200.154.141,"{""location"": ""KZ"", ""is_mobile"": false}" 13623,5,307,2017-01-18 13:43:28,http://kuhlman.com/claudine.prohaska,,155.66.94.57,"{""location"": ""PT"", ""is_mobile"": true}" 13624,5,307,2016-12-14 08:31:37,http://schoenferry.io/percival,,212.196.169.61,"{""location"": ""GA"", ""is_mobile"": true}" 13625,5,307,2017-01-23 10:36:56,http://breitenberg.co/kip,,22.102.127.253,"{""location"": ""SL"", ""is_mobile"": true}" 13626,5,307,2017-01-18 04:40:47,http://deckow.org/meda,,229.85.243.98,"{""location"": ""SJ"", ""is_mobile"": true}" 13627,5,307,2017-04-05 15:27:15,http://okeefeheidenreich.net/germaine,,134.49.93.203,"{""location"": ""TR"", ""is_mobile"": true}" 13628,5,307,2017-01-08 11:23:57,http://rowemills.info/haleigh,,129.108.9.125,"{""location"": ""KM"", ""is_mobile"": false}" 13629,5,307,2017-03-06 23:28:49,http://feilfritsch.info/bernie,,86.103.84.164,"{""location"": ""DO"", ""is_mobile"": true}" 13630,5,307,2017-02-23 10:46:15,http://brakus.org/greg,,26.48.174.45,"{""location"": ""CU"", ""is_mobile"": false}" 13631,5,307,2017-03-11 07:02:45,http://abbottullrich.net/orion.stanton,,30.122.199.195,"{""location"": ""AE"", ""is_mobile"": false}" 13632,5,307,2017-01-07 14:28:48,http://lemkefay.biz/odell.morar,,109.153.192.34,"{""location"": ""TL"", ""is_mobile"": false}" 13633,5,307,2017-03-12 20:51:39,http://muller.com/sister.satterfield,,206.104.253.105,"{""location"": ""NI"", ""is_mobile"": true}" 13634,5,307,2017-04-01 18:52:28,http://rohanjacobson.biz/kristoffer,,113.17.225.153,"{""location"": ""PT"", ""is_mobile"": true}" 13635,5,307,2017-05-31 09:36:55,http://watersborer.net/abel,,78.45.198.52,"{""location"": ""FJ"", ""is_mobile"": true}" 13636,5,307,2017-02-06 01:03:10,http://macejkovic.net/darlene_hirthe,,36.56.188.201,"{""location"": ""PH"", ""is_mobile"": true}" 13637,5,307,2017-02-24 00:19:46,http://crona.co/merle_nikolaus,,242.70.22.199,"{""location"": ""BN"", ""is_mobile"": false}" 13638,5,307,2017-01-20 22:35:05,http://rosenbaumstracke.io/esmeralda,,182.114.71.228,"{""location"": ""PS"", ""is_mobile"": false}" 13639,5,307,2017-06-07 15:32:05,http://mills.info/mariane_labadie,,107.93.173.70,"{""location"": ""SG"", ""is_mobile"": true}" 13640,5,307,2017-05-04 21:33:29,http://quitzon.com/newell.wilderman,,159.240.113.78,"{""location"": ""BQ"", ""is_mobile"": false}" 13641,5,307,2017-01-19 20:09:39,http://leschparisian.net/shea.connelly,,216.139.119.47,"{""location"": ""TD"", ""is_mobile"": true}" 13642,5,307,2017-06-03 15:09:38,http://yundtmclaughlin.biz/leo,,93.89.193.243,"{""location"": ""GW"", ""is_mobile"": true}" 13643,5,307,2017-02-05 00:11:27,http://kaulke.org/ethelyn,,45.25.183.83,"{""location"": ""IO"", ""is_mobile"": false}" 13644,5,307,2017-05-04 21:30:49,http://binsgutmann.co/orlo,,39.57.139.233,"{""location"": ""PR"", ""is_mobile"": true}" 13645,5,307,2017-04-13 19:54:02,http://lebsack.name/fabiola_erdman,,120.195.102.7,"{""location"": ""AO"", ""is_mobile"": false}" 13646,5,307,2017-01-18 06:41:09,http://kiehn.biz/keyon,,14.115.5.49,"{""location"": ""BQ"", ""is_mobile"": true}" 13647,5,307,2017-03-28 08:01:40,http://ziemannkuphal.io/ellen_parisian,,195.78.26.80,"{""location"": ""CL"", ""is_mobile"": false}" 13648,5,307,2017-04-22 02:06:57,http://moriettemills.co/unique.waters,,133.77.147.34,"{""location"": ""SX"", ""is_mobile"": false}" 13649,5,307,2017-03-30 00:07:00,http://cruickshank.io/cristopher,,110.5.192.241,"{""location"": ""KP"", ""is_mobile"": true}" 13650,5,307,2017-04-18 05:56:10,http://smithhamill.info/morton,,56.93.218.217,"{""location"": ""MP"", ""is_mobile"": true}" 13651,5,307,2017-03-23 12:41:57,http://gerhold.com/alyson,,153.29.171.84,"{""location"": ""SJ"", ""is_mobile"": false}" 13652,5,307,2017-02-07 13:52:25,http://rohan.biz/adela.cummings,,82.133.65.80,"{""location"": ""FI"", ""is_mobile"": false}" 13653,5,307,2017-05-10 23:09:11,http://barton.org/libby.grady,,114.141.228.158,"{""location"": ""KI"", ""is_mobile"": false}" 13654,5,307,2017-03-30 23:19:01,http://medhurst.org/patrick.kiehn,,188.141.126.48,"{""location"": ""NL"", ""is_mobile"": false}" 13655,5,307,2017-03-08 01:04:17,http://mohr.org/savanna.herman,,120.187.148.18,"{""location"": ""CW"", ""is_mobile"": true}" 13656,5,307,2017-03-23 21:08:51,http://marks.com/brandt,,72.173.35.34,"{""location"": ""IE"", ""is_mobile"": false}" 13657,5,307,2017-01-24 22:25:48,http://wehner.name/karelle,,59.179.131.110,"{""location"": ""GF"", ""is_mobile"": false}" 13658,5,307,2017-02-28 15:38:26,http://heathcote.biz/major.keeling,,11.88.155.229,"{""location"": ""GE"", ""is_mobile"": true}" 13659,5,307,2017-01-11 12:08:19,http://breitenberg.io/brenna.schoen,,141.63.30.203,"{""location"": ""IN"", ""is_mobile"": false}" 13660,5,307,2017-01-01 01:31:54,http://emard.org/taurean.dibbert,,217.181.72.36,"{""location"": ""MO"", ""is_mobile"": false}" 13661,5,307,2017-06-11 03:59:10,http://jaskolski.info/elia.mitchell,,232.185.47.39,"{""location"": ""RS"", ""is_mobile"": true}" 13662,5,307,2017-01-27 13:19:15,http://fritschpredovic.info/colten,,14.46.197.240,"{""location"": ""SB"", ""is_mobile"": false}" 13663,5,307,2017-04-24 15:57:47,http://rempel.com/jade,,80.208.39.28,"{""location"": ""TD"", ""is_mobile"": true}" 13664,5,307,2017-04-08 00:00:36,http://koepphoeger.biz/ashley,,77.34.215.126,"{""location"": ""KZ"", ""is_mobile"": true}" 13665,5,307,2017-01-07 20:07:27,http://bode.io/asha,,210.20.93.23,"{""location"": ""BY"", ""is_mobile"": true}" 13666,5,308,2017-04-06 19:49:04,http://gutkowski.net/juwan,,228.204.53.130,"{""location"": ""FO"", ""is_mobile"": true}" 13667,5,308,2017-05-02 05:12:16,http://oberbrunner.net/orrin_fritsch,,181.4.155.30,"{""location"": ""WS"", ""is_mobile"": false}" 13668,5,308,2017-01-24 19:46:59,http://goyette.net/aric_beahan,,82.127.32.178,"{""location"": ""CR"", ""is_mobile"": true}" 13669,5,308,2017-06-12 11:51:03,http://ferrygreen.info/marlon,,213.115.57.36,"{""location"": ""BQ"", ""is_mobile"": true}" 13670,5,308,2017-05-21 05:23:43,http://gaylord.com/micheal.haag,,39.243.185.240,"{""location"": ""CW"", ""is_mobile"": true}" 16601,6,376,2017-02-12 23:30:11,http://rennerbosco.co/candace,0.6305074180,48.249.32.16,"{""location"": ""QA"", ""is_mobile"": true}" 16602,6,376,2017-05-06 17:22:27,http://ondricka.co/brent.wehner,0.7518886484,117.99.110.203,"{""location"": ""YT"", ""is_mobile"": false}" 16603,6,376,2017-05-16 05:08:37,http://rowe.com/blair,0.0326024815,164.52.141.199,"{""location"": ""SG"", ""is_mobile"": true}" 16604,6,376,2017-05-25 03:50:18,http://abshire.biz/macie,0.3861400726,27.201.64.196,"{""location"": ""CK"", ""is_mobile"": true}" 16605,6,376,2017-06-14 00:23:16,http://hodkiewiczcain.net/simone.strosin,0.5079105061,89.208.171.143,"{""location"": ""PF"", ""is_mobile"": true}" 16606,6,376,2017-03-15 13:59:27,http://eichmannrodriguez.io/mauricio,0.4547583888,213.202.248.41,"{""location"": ""CC"", ""is_mobile"": false}" 16607,6,376,2017-01-31 21:14:43,http://stamm.name/gustave,0.9940792418,53.56.26.82,"{""location"": ""FJ"", ""is_mobile"": false}" 16608,6,376,2017-01-04 19:39:12,http://donnellyroob.io/halie,0.1842309505,159.153.7.79,"{""location"": ""NG"", ""is_mobile"": false}" 16609,6,376,2017-04-12 10:30:54,http://bode.net/sienna,0.3116145987,202.144.215.132,"{""location"": ""JO"", ""is_mobile"": false}" 16610,6,376,2017-05-11 17:20:08,http://lynch.co/robert.klein,0.2554555624,155.66.95.216,"{""location"": ""CL"", ""is_mobile"": false}" 16611,6,376,2017-02-02 23:39:38,http://oreilly.biz/naomi,0.9345552899,229.212.200.231,"{""location"": ""CV"", ""is_mobile"": true}" 16612,6,377,2016-12-30 00:59:07,http://price.biz/rico,,68.237.245.100,"{""location"": ""NI"", ""is_mobile"": true}" 16613,6,377,2017-01-06 12:22:07,http://roberts.io/emory,,12.4.64.162,"{""location"": ""LK"", ""is_mobile"": true}" 16614,6,377,2017-01-08 22:23:27,http://cain.com/osvaldo.abbott,,206.117.134.220,"{""location"": ""BA"", ""is_mobile"": true}" 16615,6,377,2016-12-21 06:11:14,http://zulaufstracke.co/noemy,,42.13.89.123,"{""location"": ""MN"", ""is_mobile"": true}" 16616,6,377,2017-03-27 11:11:04,http://hayesryan.net/giles_langworth,,191.210.227.47,"{""location"": ""CG"", ""is_mobile"": false}" 16617,6,377,2017-04-28 15:59:35,http://pollich.com/elijah.leuschke,,251.62.129.41,"{""location"": ""UZ"", ""is_mobile"": false}" 16618,6,377,2017-04-12 03:11:11,http://kerluke.net/kurt,,72.192.24.163,"{""location"": ""SE"", ""is_mobile"": false}" 16619,6,377,2017-05-20 02:24:36,http://deckowhartmann.com/bradley,,214.239.177.101,"{""location"": ""SJ"", ""is_mobile"": true}" 16620,6,377,2017-03-05 01:58:58,http://kuhiccain.com/leola.effertz,,24.175.126.217,"{""location"": ""TC"", ""is_mobile"": true}" 16621,6,377,2017-04-11 02:31:50,http://mcdermott.org/hillard,,85.79.214.58,"{""location"": ""HM"", ""is_mobile"": false}" 16622,6,377,2017-06-09 14:13:49,http://flatleymcdermott.co/kiara.stanton,,240.149.116.106,"{""location"": ""SR"", ""is_mobile"": false}" 16623,6,377,2017-05-06 08:14:24,http://farrell.net/anika_bernhard,,26.43.161.176,"{""location"": ""CY"", ""is_mobile"": true}" 16624,6,377,2017-03-06 13:29:04,http://lind.info/stefanie,,164.48.129.249,"{""location"": ""AU"", ""is_mobile"": true}" 16625,6,377,2017-05-20 16:55:17,http://mooreblanda.org/dulce.bernier,,205.74.83.146,"{""location"": ""GM"", ""is_mobile"": true}" 16626,6,377,2017-02-04 00:07:49,http://koch.biz/alvera_purdy,,43.224.57.117,"{""location"": ""ER"", ""is_mobile"": true}" 16627,6,377,2017-05-17 16:36:31,http://schuster.info/kim.pouros,,68.250.220.36,"{""location"": ""OM"", ""is_mobile"": true}" 16628,6,377,2017-04-28 12:33:52,http://emmerichruel.name/irwin.christiansen,,76.178.6.134,"{""location"": ""MS"", ""is_mobile"": false}" 16629,6,377,2017-05-15 17:31:40,http://mcdermott.org/lyda,,72.59.187.223,"{""location"": ""CA"", ""is_mobile"": false}" 16630,6,377,2017-04-18 00:08:18,http://ryan.biz/karine,,97.14.190.90,"{""location"": ""AG"", ""is_mobile"": true}" 16631,6,377,2017-04-02 08:53:14,http://bergstrom.info/cristian,,181.38.31.28,"{""location"": ""BW"", ""is_mobile"": false}" 16632,6,377,2017-03-18 22:26:11,http://oconnell.org/julia_quigley,,104.167.152.113,"{""location"": ""PK"", ""is_mobile"": false}" 16633,6,377,2017-04-28 04:54:56,http://muller.co/jerald,,164.219.46.211,"{""location"": ""BD"", ""is_mobile"": true}" 16634,6,377,2017-06-07 03:12:48,http://bradtke.org/jerrod,,157.79.228.146,"{""location"": ""GP"", ""is_mobile"": true}" 16635,6,377,2017-05-05 00:29:28,http://brown.name/kali,,144.175.220.124,"{""location"": ""SI"", ""is_mobile"": false}" 16636,6,377,2017-02-11 13:32:37,http://funk.net/neva,,176.97.213.159,"{""location"": ""AD"", ""is_mobile"": false}" 16637,6,377,2017-03-10 08:40:03,http://robelrosenbaum.io/markus_hyatt,,203.209.205.141,"{""location"": ""LA"", ""is_mobile"": true}" 16638,6,377,2017-01-14 14:00:39,http://kochcronin.io/nellie_thompson,,164.95.37.4,"{""location"": ""MG"", ""is_mobile"": false}" 16639,6,377,2017-04-29 22:03:26,http://johnseichmann.name/larue_mcclure,,78.75.126.34,"{""location"": ""CR"", ""is_mobile"": false}" 16640,6,377,2017-02-23 21:07:25,http://ryanhartmann.name/arnoldo,,218.153.195.97,"{""location"": ""BE"", ""is_mobile"": false}" 16641,6,377,2016-12-27 13:18:18,http://kostanton.biz/wendy,,123.75.60.130,"{""location"": ""VE"", ""is_mobile"": true}" 16642,6,377,2017-04-17 13:55:12,http://collins.io/lorine,,132.194.171.157,"{""location"": ""SC"", ""is_mobile"": false}" 16643,6,377,2016-12-18 06:40:59,http://bergnaum.co/jaleel,,97.53.6.147,"{""location"": ""KZ"", ""is_mobile"": false}" 16644,6,377,2017-03-30 08:56:16,http://millsmann.biz/conner.bosco,,164.135.32.130,"{""location"": ""SI"", ""is_mobile"": false}" 16645,6,377,2017-01-05 19:21:16,http://gutmann.net/bart.cronin,,231.102.234.53,"{""location"": ""TD"", ""is_mobile"": false}" 16646,6,377,2017-03-13 19:15:39,http://fishercarroll.com/presley.kirlin,,127.242.77.58,"{""location"": ""ML"", ""is_mobile"": true}" 16647,6,377,2017-06-10 19:25:13,http://lynchwatsica.info/maude,,242.192.107.186,"{""location"": ""AO"", ""is_mobile"": false}" 16648,6,377,2016-12-17 00:35:11,http://mosciski.net/marlene,,19.247.189.113,"{""location"": ""GI"", ""is_mobile"": true}" 16649,6,377,2017-03-16 21:07:48,http://strosin.org/anderson,,99.107.122.135,"{""location"": ""HM"", ""is_mobile"": true}" 16650,6,377,2017-05-11 23:55:59,http://jakubowski.io/yoshiko.spinka,,223.90.84.243,"{""location"": ""SH"", ""is_mobile"": true}" 16651,6,377,2017-04-07 00:41:47,http://reichelkuphal.org/jensen_welch,,80.106.249.67,"{""location"": ""KH"", ""is_mobile"": false}" 16652,6,377,2017-01-03 22:00:52,http://mayertschmeler.co/misael,,241.198.30.132,"{""location"": ""BV"", ""is_mobile"": false}" 16653,6,377,2017-03-05 01:47:56,http://lakin.com/jerrod.grant,,59.83.172.185,"{""location"": ""CO"", ""is_mobile"": false}" 16654,6,377,2017-03-24 04:58:04,http://cartwright.co/chad.brown,,111.104.81.220,"{""location"": ""AX"", ""is_mobile"": false}" 3782,2,83,2017-05-16 03:28:38,http://eichmann.net/deondre,0.2811298498,124.118.167.78,"{""location"": ""GF"", ""is_mobile"": false}" 3783,2,83,2017-03-11 23:19:04,http://farrell.net/briana,0.9871184350,235.130.80.92,"{""location"": ""FO"", ""is_mobile"": true}" 3784,2,83,2017-06-13 11:49:02,http://hansen.com/vernon_jones,0.3613562241,168.108.42.155,"{""location"": ""IO"", ""is_mobile"": true}" 3785,2,83,2017-02-14 11:34:09,http://schroeder.biz/josianne,0.0451508652,77.95.72.13,"{""location"": ""BF"", ""is_mobile"": false}" 3786,2,83,2017-03-29 17:06:34,http://goldner.com/guido,0.7425097458,185.151.237.90,"{""location"": ""MV"", ""is_mobile"": true}" 3787,2,83,2017-03-27 07:22:08,http://jacobson.info/margarete.blanda,0.1330591735,164.188.33.97,"{""location"": ""FK"", ""is_mobile"": true}" 3788,2,83,2017-04-14 15:46:10,http://purdywillms.org/francisco,0.4405247874,212.162.129.108,"{""location"": ""TO"", ""is_mobile"": true}" 3789,2,83,2017-03-13 03:18:23,http://block.org/bulah_kulas,0.2605070241,108.72.122.196,"{""location"": ""RU"", ""is_mobile"": false}" 3790,2,83,2017-05-29 13:52:05,http://dietrichschroeder.info/stephon_rolfson,0.1153439895,28.154.201.115,"{""location"": ""KM"", ""is_mobile"": false}" 3791,2,83,2017-01-17 10:45:34,http://zieme.info/gielle,0.3008681070,224.62.179.116,"{""location"": ""FR"", ""is_mobile"": false}" 3792,2,83,2017-03-04 12:50:59,http://herzog.co/luz.hilll,0.9207124836,224.135.107.113,"{""location"": ""MM"", ""is_mobile"": false}" 3793,2,83,2017-04-01 03:32:26,http://zulaufquigley.org/chasity,0.4004565104,251.182.195.226,"{""location"": ""BO"", ""is_mobile"": false}" 3794,2,83,2017-04-23 06:51:08,http://ferry.name/johan,0.9533464021,87.122.213.114,"{""location"": ""PW"", ""is_mobile"": true}" 3795,2,83,2017-05-12 01:04:00,http://koelpinharvey.com/efren_nolan,0.4994297520,215.178.172.234,"{""location"": ""JM"", ""is_mobile"": false}" 3796,2,83,2017-02-24 19:10:55,http://tillman.io/octavia,0.7370758426,214.249.65.114,"{""location"": ""PL"", ""is_mobile"": false}" 3797,2,83,2016-12-29 12:16:43,http://medhurstcronin.co/loyce_harber,0.9640302644,101.45.121.173,"{""location"": ""SO"", ""is_mobile"": true}" 3798,2,83,2016-12-22 13:07:48,http://gislasonklocko.co/henderson,0.2415487959,139.188.55.252,"{""location"": ""CW"", ""is_mobile"": true}" 3799,2,83,2017-03-04 05:33:11,http://renner.io/alvis,0.4650356651,108.45.40.135,"{""location"": ""TG"", ""is_mobile"": true}" 3800,2,83,2016-12-17 14:36:09,http://fahey.com/ida,0.2364755247,96.161.35.27,"{""location"": ""CU"", ""is_mobile"": true}" 3801,2,83,2017-04-23 07:46:15,http://vandervort.com/marcelina,0.7997106266,174.175.120.200,"{""location"": ""PY"", ""is_mobile"": false}" 3802,2,83,2017-04-15 21:47:39,http://bailey.name/veda.blick,0.8077206358,62.202.210.229,"{""location"": ""MZ"", ""is_mobile"": true}" 3803,2,83,2017-01-25 08:36:28,http://gutmann.name/courtney,0.7634836408,119.183.71.178,"{""location"": ""FK"", ""is_mobile"": true}" 3804,2,83,2017-05-07 19:03:04,http://huel.co/alexanne.zulauf,0.2686630039,32.115.60.244,"{""location"": ""JM"", ""is_mobile"": false}" 3805,2,83,2017-05-08 13:16:41,http://stroman.info/fletcher,0.5522063347,139.192.3.90,"{""location"": ""GB"", ""is_mobile"": true}" 3806,2,83,2017-03-18 05:37:08,http://frami.name/holden_gleichner,0.1982739151,112.69.170.146,"{""location"": ""GW"", ""is_mobile"": false}" 3807,2,83,2017-03-23 00:28:34,http://stark.org/angelica,0.9128288526,247.234.117.183,"{""location"": ""MZ"", ""is_mobile"": true}" 3808,2,83,2017-05-28 15:19:17,http://koelpin.io/luna,0.8984262259,224.252.121.77,"{""location"": ""ID"", ""is_mobile"": false}" 3809,2,83,2017-02-17 22:33:36,http://romagueracain.com/austin,0.6408306886,69.228.99.193,"{""location"": ""PH"", ""is_mobile"": true}" 3810,2,83,2017-04-04 23:38:53,http://gutmann.info/magdalena.gaylord,0.9504748502,126.190.166.45,"{""location"": ""BI"", ""is_mobile"": false}" 3811,2,83,2017-02-18 10:26:19,http://gutmannmarks.org/taylor.white,0.2659960982,177.21.101.38,"{""location"": ""FJ"", ""is_mobile"": true}" 3812,2,83,2017-01-16 21:15:21,http://bernhard.net/karine.hudson,0.2666801845,29.242.220.247,"{""location"": ""PL"", ""is_mobile"": false}" 3813,2,83,2017-01-25 18:38:55,http://glover.info/shannon.bechtelar,0.5664703481,152.125.108.240,"{""location"": ""FJ"", ""is_mobile"": true}" 3814,2,83,2017-05-12 08:06:41,http://hartmannluettgen.name/paris_wiza,0.9345912830,184.243.93.133,"{""location"": ""BW"", ""is_mobile"": true}" 3815,2,83,2016-12-17 02:42:20,http://waelchi.com/stefanie,0.8615195812,100.127.164.230,"{""location"": ""OM"", ""is_mobile"": false}" 3816,2,83,2017-06-13 00:57:01,http://bradtke.name/jayson.schmeler,0.6962484339,143.168.233.147,"{""location"": ""BZ"", ""is_mobile"": false}" 3817,2,83,2017-03-03 08:27:44,http://upton.io/sarina.torphy,0.5202475538,97.87.86.52,"{""location"": ""HR"", ""is_mobile"": true}" 3818,2,83,2017-03-13 21:55:30,http://blockklocko.org/orie,0.4178897008,176.22.63.153,"{""location"": ""AU"", ""is_mobile"": false}" 3819,2,83,2017-03-10 10:55:42,http://ward.com/patricia,0.3888814560,95.183.30.244,"{""location"": ""MC"", ""is_mobile"": false}" 3820,2,84,2016-12-27 18:09:03,http://lebsack.co/vivian,0.2130481939,231.215.221.2,"{""location"": ""LR"", ""is_mobile"": true}" 3821,2,84,2017-05-19 20:35:32,http://reinger.name/gabriella,0.4563738950,180.149.236.215,"{""location"": ""TW"", ""is_mobile"": true}" 3822,2,84,2017-04-10 12:45:02,http://kreiger.co/elwin_sipes,0.8368338506,164.78.106.242,"{""location"": ""TJ"", ""is_mobile"": true}" 3823,2,84,2017-03-15 02:18:36,http://wittingheidenreich.com/lamont,0.6477546244,239.254.169.39,"{""location"": ""BT"", ""is_mobile"": false}" 3824,2,84,2016-12-15 05:27:03,http://christiansen.com/demond,0.7365750633,30.242.253.84,"{""location"": ""KR"", ""is_mobile"": false}" 3825,2,84,2017-02-26 00:49:15,http://littel.biz/elisha.mcdermott,0.3408252847,226.36.17.151,"{""location"": ""TC"", ""is_mobile"": false}" 3826,2,84,2017-01-14 08:26:43,http://brown.co/sanford,0.2673639591,162.178.157.137,"{""location"": ""NR"", ""is_mobile"": false}" 3827,2,84,2016-12-31 23:18:29,http://gorczany.io/barrett,0.0604014761,151.128.231.115,"{""location"": ""IM"", ""is_mobile"": false}" 3828,2,84,2017-03-07 00:38:19,http://hansen.name/rollin.murazik,0.6278163967,67.186.237.75,"{""location"": ""OM"", ""is_mobile"": false}" 3829,2,84,2017-03-04 18:07:30,http://rowemoen.info/brayan,0.6065882612,168.138.93.245,"{""location"": ""MO"", ""is_mobile"": true}" 3830,2,84,2017-02-20 13:16:23,http://mertz.biz/demetrius,0.9482747680,224.145.61.97,"{""location"": ""BZ"", ""is_mobile"": false}" 3831,2,84,2017-03-08 00:04:04,http://leannonsenger.co/lea,0.6436282555,31.26.122.90,"{""location"": ""FO"", ""is_mobile"": true}" 3832,2,84,2017-02-06 15:50:17,http://mayer.co/shaina_lynch,0.0639014367,170.141.206.23,"{""location"": ""PY"", ""is_mobile"": false}" 3833,2,84,2017-05-15 16:29:56,http://parisian.com/raegan,0.4475628069,119.32.92.96,"{""location"": ""ME"", ""is_mobile"": true}" 10720,4,240,2016-12-25 09:46:44,http://dibbert.org/samanta.stehr,,224.112.88.44,"{""location"": ""MC"", ""is_mobile"": true}" 10721,4,240,2017-06-07 06:55:49,http://huel.name/pearline.fay,,235.95.22.227,"{""location"": ""IL"", ""is_mobile"": true}" 10722,4,240,2017-01-09 23:11:23,http://lueilwitz.io/bernice,,52.237.151.15,"{""location"": ""BM"", ""is_mobile"": true}" 10723,4,240,2017-04-10 22:23:21,http://cummeratakemmer.info/arvilla.stanton,,194.97.164.119,"{""location"": ""BV"", ""is_mobile"": false}" 10724,4,240,2017-01-29 21:03:59,http://hyatt.com/benedict_collier,,168.133.69.19,"{""location"": ""SX"", ""is_mobile"": true}" 10725,4,240,2017-02-07 14:45:21,http://raynor.org/merl_rogahn,,64.83.27.43,"{""location"": ""SL"", ""is_mobile"": false}" 10726,4,240,2017-04-10 08:24:56,http://mueller.info/beryl.hayes,,199.184.148.30,"{""location"": ""LC"", ""is_mobile"": false}" 10727,4,240,2017-02-18 08:40:22,http://harrisdietrich.name/delilah,,152.96.209.2,"{""location"": ""AQ"", ""is_mobile"": false}" 10728,4,240,2017-06-14 01:03:44,http://von.name/muhammad.smith,,233.166.96.209,"{""location"": ""AE"", ""is_mobile"": true}" 10729,4,240,2016-12-16 16:47:18,http://bashirian.name/fabian,,37.7.172.189,"{""location"": ""UA"", ""is_mobile"": true}" 10730,4,240,2017-02-07 17:45:42,http://klocko.com/niko.rodriguez,,97.204.55.226,"{""location"": ""MG"", ""is_mobile"": false}" 10731,4,240,2017-03-14 01:30:28,http://welchwunsch.com/delmer,,32.218.31.244,"{""location"": ""NG"", ""is_mobile"": true}" 10732,4,240,2017-03-01 15:28:27,http://halvorson.co/reynold,,148.227.178.231,"{""location"": ""CR"", ""is_mobile"": false}" 10733,4,240,2017-01-06 02:01:22,http://emmerich.io/darius,,114.158.65.130,"{""location"": ""CA"", ""is_mobile"": true}" 10734,4,240,2017-02-22 19:53:27,http://hegmann.info/shakira_pacocha,,39.96.78.207,"{""location"": ""NU"", ""is_mobile"": true}" 10735,4,240,2016-12-24 01:43:25,http://kihn.co/dallas.blick,,142.248.33.237,"{""location"": ""BL"", ""is_mobile"": false}" 10736,4,240,2017-04-13 07:54:49,http://dare.biz/april,,124.119.77.230,"{""location"": ""CY"", ""is_mobile"": false}" 10737,4,240,2016-12-16 01:07:12,http://willdonnelly.net/scarlett.wunsch,,254.32.25.227,"{""location"": ""BQ"", ""is_mobile"": true}" 10738,4,240,2016-12-29 00:32:02,http://halvorson.com/misael,,98.100.237.76,"{""location"": ""CH"", ""is_mobile"": false}" 10739,4,240,2017-01-29 06:07:23,http://wilderman.co/clair_hills,,220.62.96.225,"{""location"": ""NA"", ""is_mobile"": false}" 10740,4,240,2016-12-30 09:00:33,http://reichel.name/arturo.conroy,,196.131.14.148,"{""location"": ""NU"", ""is_mobile"": true}" 10741,4,240,2017-01-07 17:34:31,http://schimmel.com/bennett,,168.159.144.224,"{""location"": ""SJ"", ""is_mobile"": false}" 10742,4,240,2017-03-03 22:40:47,http://feil.org/catharine,,10.29.69.78,"{""location"": ""MX"", ""is_mobile"": false}" 10743,4,240,2017-01-19 03:16:40,http://fritschstark.info/dawn,,148.69.190.28,"{""location"": ""AO"", ""is_mobile"": true}" 10744,4,241,2017-05-26 03:26:53,http://mrazkeebler.io/bridget,,100.218.90.231,"{""location"": ""AD"", ""is_mobile"": false}" 10745,4,241,2017-05-26 23:47:38,http://osinskireinger.net/obie,,137.121.148.124,"{""location"": ""PL"", ""is_mobile"": false}" 10746,4,241,2017-01-11 14:12:45,http://wisozk.com/kieran,,99.100.199.46,"{""location"": ""KE"", ""is_mobile"": true}" 10747,4,241,2017-03-18 08:37:53,http://altenwerth.com/camden_johnson,,19.114.83.173,"{""location"": ""US"", ""is_mobile"": false}" 10748,4,241,2017-03-29 20:02:35,http://pollich.io/thurman_reynolds,,213.57.52.180,"{""location"": ""TC"", ""is_mobile"": false}" 10749,4,241,2016-12-23 23:21:54,http://west.com/felix.kling,,208.217.126.246,"{""location"": ""LB"", ""is_mobile"": true}" 10750,4,241,2016-12-21 21:15:45,http://rempel.org/evan,,2.210.249.99,"{""location"": ""JM"", ""is_mobile"": true}" 10751,4,241,2017-03-24 19:39:32,http://grimes.net/makenna_tromp,,189.100.172.30,"{""location"": ""MF"", ""is_mobile"": false}" 10752,4,241,2017-05-05 03:49:24,http://runolfonglover.co/emil_hills,,222.42.235.145,"{""location"": ""TC"", ""is_mobile"": false}" 10753,4,241,2017-01-31 19:28:34,http://kunze.info/hilma,,127.127.139.92,"{""location"": ""SR"", ""is_mobile"": false}" 10754,4,241,2017-05-01 09:17:45,http://schneider.co/davin,,212.169.12.25,"{""location"": ""KE"", ""is_mobile"": false}" 10755,4,241,2017-05-08 06:31:04,http://bahringer.info/josianne,,145.164.54.26,"{""location"": ""MO"", ""is_mobile"": false}" 10756,4,241,2017-04-12 15:36:04,http://dicki.info/lenora,,57.145.161.196,"{""location"": ""SN"", ""is_mobile"": true}" 10757,4,241,2017-02-04 12:20:22,http://rempel.org/mae_parker,,88.202.208.153,"{""location"": ""PY"", ""is_mobile"": true}" 10758,4,241,2017-05-23 21:18:18,http://schaden.co/katrine_zboncak,,176.3.192.62,"{""location"": ""MM"", ""is_mobile"": true}" 10760,4,241,2017-01-30 09:11:11,http://vandervortlubowitz.name/don.larson,,119.82.45.120,"{""location"": ""RE"", ""is_mobile"": false}" 10761,4,241,2017-02-08 02:23:13,http://stehrraynor.org/velma,,169.80.254.157,"{""location"": ""AX"", ""is_mobile"": false}" 10762,4,241,2017-06-09 00:23:32,http://baumbach.io/garnett,,239.243.176.194,"{""location"": ""LY"", ""is_mobile"": false}" 10763,4,241,2017-03-12 11:59:00,http://gorczany.io/madeline.olson,,124.245.233.192,"{""location"": ""UA"", ""is_mobile"": true}" 10764,4,241,2017-04-02 09:42:38,http://spencer.info/dusty,,93.163.13.89,"{""location"": ""BV"", ""is_mobile"": false}" 10765,4,241,2016-12-25 11:11:30,http://pfefferhowell.net/amina_willms,,116.67.33.224,"{""location"": ""RO"", ""is_mobile"": false}" 10766,4,241,2017-01-30 09:21:06,http://hettinger.net/dixie_nader,,183.191.114.202,"{""location"": ""RU"", ""is_mobile"": true}" 10767,4,241,2016-12-15 23:57:21,http://marks.co/alf_gulgowski,,89.13.50.41,"{""location"": ""HR"", ""is_mobile"": true}" 10768,4,241,2017-03-12 08:53:35,http://ziemecorkery.io/crawford,,188.87.76.124,"{""location"": ""LB"", ""is_mobile"": false}" 10769,4,241,2017-01-18 01:41:31,http://hilll.info/whitney_kris,,45.147.210.135,"{""location"": ""GS"", ""is_mobile"": false}" 10770,4,241,2017-01-03 20:44:00,http://eichmannmann.org/kamren,,18.171.7.177,"{""location"": ""BB"", ""is_mobile"": true}" 10771,4,241,2016-12-24 09:36:56,http://marvin.io/burley_wilkinson,,110.46.12.135,"{""location"": ""KY"", ""is_mobile"": true}" 10772,4,241,2017-06-11 10:48:31,http://cartwright.net/michale_dach,,120.173.136.182,"{""location"": ""LA"", ""is_mobile"": false}" 10773,4,241,2017-03-07 09:30:12,http://grady.co/narciso.stiedemann,,253.98.183.20,"{""location"": ""CM"", ""is_mobile"": false}" 10774,4,241,2017-03-07 10:53:29,http://kub.org/nikko.nienow,,30.70.124.24,"{""location"": ""SN"", ""is_mobile"": true}" 10775,4,241,2016-12-15 07:16:02,http://nikolaus.org/ardith,,105.230.179.76,"{""location"": ""CA"", ""is_mobile"": false}" 7759,3,172,2016-12-24 05:59:48,http://millerziemann.com/janelle.schmidt,0.5758763139,42.205.125.91,"{""location"": ""SV"", ""is_mobile"": false}" 7760,3,172,2017-05-16 16:57:34,http://welchreynolds.biz/tre.schneider,0.7518322846,95.9.247.29,"{""location"": ""PG"", ""is_mobile"": true}" 7761,3,172,2017-01-21 09:11:52,http://carter.biz/shad.kemmer,0.4955654949,231.69.251.32,"{""location"": ""IM"", ""is_mobile"": false}" 7762,3,172,2016-12-29 21:06:08,http://anderson.io/haylee.ferry,0.3888264939,52.54.176.52,"{""location"": ""LB"", ""is_mobile"": true}" 7763,3,172,2017-01-13 13:02:49,http://hahnortiz.biz/shania_kris,0.8901944864,41.147.38.230,"{""location"": ""IO"", ""is_mobile"": true}" 7764,3,172,2016-12-28 18:34:38,http://dickinson.info/willis_rodriguez,0.7382470133,119.174.170.123,"{""location"": ""SL"", ""is_mobile"": true}" 7765,3,172,2017-04-18 05:35:25,http://mcclure.com/yvette,0.5087866791,238.133.170.132,"{""location"": ""SV"", ""is_mobile"": false}" 7766,3,172,2017-05-09 13:09:32,http://hirtheprice.net/sofia.hackett,0.1417928001,135.59.145.90,"{""location"": ""IQ"", ""is_mobile"": false}" 7767,3,172,2017-03-07 18:26:11,http://huels.net/demarco.wiza,0.9916159953,221.190.13.38,"{""location"": ""PA"", ""is_mobile"": false}" 7768,3,172,2016-12-20 23:19:04,http://larson.io/fern_runte,0.4479922119,190.46.90.86,"{""location"": ""RU"", ""is_mobile"": true}" 7769,3,172,2017-05-13 01:44:07,http://mcculloughkoch.io/charley.thiel,0.6984428169,98.239.253.232,"{""location"": ""PW"", ""is_mobile"": false}" 7770,3,172,2017-06-01 05:24:27,http://fadel.io/raleigh.jenkins,0.4123349670,111.169.188.245,"{""location"": ""DJ"", ""is_mobile"": true}" 7771,3,172,2017-05-15 07:48:37,http://rueckermacgyver.org/icie.kunze,0.4819117509,152.238.95.22,"{""location"": ""MF"", ""is_mobile"": true}" 7772,3,172,2017-05-15 05:08:12,http://boehm.com/alanis,0.7340079976,185.6.240.58,"{""location"": ""MG"", ""is_mobile"": true}" 7773,3,172,2017-05-22 20:32:21,http://brown.co/camila,0.0565760925,136.103.69.65,"{""location"": ""GM"", ""is_mobile"": false}" 7774,3,172,2017-02-02 05:34:09,http://ruel.name/edgardo,0.0956523220,121.79.225.52,"{""location"": ""PA"", ""is_mobile"": false}" 7775,3,172,2017-02-19 21:26:06,http://ankunding.biz/imani_kreiger,0.8953782780,115.44.57.101,"{""location"": ""MQ"", ""is_mobile"": true}" 7776,3,172,2017-01-05 00:50:31,http://morietterodriguez.name/dovie.steuber,0.9101376646,143.23.113.166,"{""location"": ""CC"", ""is_mobile"": false}" 7777,3,172,2017-02-25 20:14:43,http://ritchiedeckow.org/laisha,0.6118238942,159.11.29.42,"{""location"": ""BN"", ""is_mobile"": false}" 7778,3,172,2017-06-09 23:21:50,http://weberwolf.com/corine,0.9205468842,202.18.244.244,"{""location"": ""RE"", ""is_mobile"": false}" 7779,3,172,2017-01-10 05:04:08,http://heathcote.com/maxine,0.1717773907,199.13.242.55,"{""location"": ""GP"", ""is_mobile"": false}" 7780,3,172,2017-01-19 20:54:00,http://bogisich.net/tara_crona,0.6029252307,216.27.180.138,"{""location"": ""BN"", ""is_mobile"": false}" 7781,3,172,2017-04-22 18:28:48,http://kilback.org/thurman,0.0642268519,12.28.194.120,"{""location"": ""KP"", ""is_mobile"": false}" 7782,3,172,2017-03-11 01:45:43,http://langoshkunze.name/lenora,0.7746693127,136.3.64.145,"{""location"": ""GB"", ""is_mobile"": false}" 7783,3,172,2017-03-31 07:48:10,http://lang.co/baylee_lehner,0.4785013558,97.206.141.105,"{""location"": ""BN"", ""is_mobile"": false}" 7784,3,173,2017-04-12 20:26:03,http://schiller.name/macie_gibson,0.6091162429,37.215.249.234,"{""location"": ""MT"", ""is_mobile"": true}" 7785,3,173,2017-03-12 12:49:46,http://erdman.co/caandra,0.3598751538,221.92.90.54,"{""location"": ""NP"", ""is_mobile"": true}" 7786,3,173,2017-06-01 08:17:02,http://keebler.info/trea.white,0.5116434742,79.90.237.25,"{""location"": ""AL"", ""is_mobile"": false}" 7787,3,173,2017-04-15 04:20:25,http://cormiergreen.biz/riley.keebler,0.4937220829,99.119.45.241,"{""location"": ""GG"", ""is_mobile"": true}" 7788,3,173,2017-01-05 21:26:30,http://goodwin.biz/ferne.okuneva,0.2380092060,245.124.174.127,"{""location"": ""SH"", ""is_mobile"": false}" 7789,3,173,2017-02-17 16:38:24,http://okeefe.org/dortha,0.5662159104,182.82.129.204,"{""location"": ""BR"", ""is_mobile"": false}" 7790,3,173,2017-02-17 01:44:08,http://daniel.info/bud_ernser,0.8756343066,29.252.155.45,"{""location"": ""SL"", ""is_mobile"": true}" 7791,3,173,2017-04-09 02:20:20,http://kuphal.biz/estel_ledner,0.3028546513,107.191.225.79,"{""location"": ""VG"", ""is_mobile"": true}" 7792,3,173,2017-06-04 17:41:04,http://lemke.name/buford.will,0.9401803852,25.26.88.41,"{""location"": ""GI"", ""is_mobile"": true}" 7793,3,173,2017-06-06 00:28:01,http://runolfon.net/giuseppe,0.6439888565,127.238.79.205,"{""location"": ""PG"", ""is_mobile"": false}" 7794,3,173,2017-04-05 03:43:59,http://connelly.com/alda,0.2138651856,11.96.90.195,"{""location"": ""SD"", ""is_mobile"": true}" 7795,3,173,2017-05-07 23:02:49,http://handhalvorson.name/dovie,0.3671982941,97.72.110.191,"{""location"": ""UM"", ""is_mobile"": false}" 7796,3,173,2016-12-17 05:13:57,http://mohrfunk.biz/don,0.1072522227,195.76.66.65,"{""location"": ""TK"", ""is_mobile"": false}" 7797,3,173,2017-04-07 04:52:36,http://schaden.com/leanne,0.4248062550,203.20.186.214,"{""location"": ""EG"", ""is_mobile"": false}" 7798,3,173,2017-03-04 07:15:00,http://harrispollich.com/omer,0.2782420652,178.108.87.113,"{""location"": ""KE"", ""is_mobile"": true}" 7799,3,173,2017-02-04 05:49:22,http://dibbert.name/adrien,0.8636266594,114.80.90.81,"{""location"": ""CF"", ""is_mobile"": false}" 7800,3,173,2017-05-18 13:42:54,http://schowalter.io/eda,0.2671007991,240.48.138.131,"{""location"": ""KM"", ""is_mobile"": true}" 7801,3,173,2017-03-12 08:03:58,http://mcdermottboyer.io/delphine,0.6620544501,225.14.93.40,"{""location"": ""PY"", ""is_mobile"": true}" 7802,3,173,2017-06-10 02:31:41,http://krishansen.info/eugenia_goyette,0.3839357143,45.82.54.113,"{""location"": ""CK"", ""is_mobile"": true}" 7803,3,173,2017-06-13 17:24:22,http://mann.org/rita,0.8376311053,89.91.183.159,"{""location"": ""RU"", ""is_mobile"": false}" 7804,3,173,2017-04-02 15:50:25,http://keebler.net/eddie,0.3019285908,187.58.136.106,"{""location"": ""MX"", ""is_mobile"": false}" 7805,3,173,2017-06-10 18:05:22,http://kertzmannhowell.io/kevin,0.0432030824,191.239.27.161,"{""location"": ""NP"", ""is_mobile"": false}" 7806,3,173,2017-04-01 07:11:29,http://mcdermott.info/derrick.king,0.8398729680,109.183.131.254,"{""location"": ""SG"", ""is_mobile"": false}" 7807,3,173,2017-04-11 00:08:53,http://jacobs.name/maude.buckridge,0.5571221681,126.136.40.134,"{""location"": ""CI"", ""is_mobile"": true}" 7808,3,173,2017-06-04 01:23:06,http://ledner.io/margie_beatty,0.7499801952,129.121.218.177,"{""location"": ""AE"", ""is_mobile"": true}" 7809,3,173,2017-04-24 10:40:53,http://bashirianschroeder.org/jo,0.2691613624,54.77.114.157,"{""location"": ""VI"", ""is_mobile"": false}" 13671,5,308,2017-02-15 12:21:35,http://tillman.org/christian,,223.79.155.232,"{""location"": ""RO"", ""is_mobile"": false}" 13672,5,308,2017-04-08 22:37:24,http://parisiangutmann.com/adah_buckridge,,248.19.20.36,"{""location"": ""NR"", ""is_mobile"": false}" 13673,5,308,2017-06-02 02:39:52,http://johnspredovic.co/clinton,,223.165.46.77,"{""location"": ""AD"", ""is_mobile"": true}" 13674,5,308,2017-02-17 11:13:53,http://ko.biz/lucius,,158.176.35.222,"{""location"": ""KI"", ""is_mobile"": true}" 13675,5,308,2016-12-13 14:59:23,http://gislasonmedhurst.io/chris,,146.239.38.69,"{""location"": ""HT"", ""is_mobile"": true}" 13676,5,308,2017-01-27 12:26:21,http://williamson.info/jo,,23.57.34.165,"{""location"": ""TL"", ""is_mobile"": false}" 13677,5,308,2017-01-06 07:14:47,http://rempelbernhard.org/pinkie,,209.62.32.30,"{""location"": ""KZ"", ""is_mobile"": true}" 13678,5,308,2017-05-10 17:40:34,http://hyatt.com/dolores,,148.24.51.124,"{""location"": ""MR"", ""is_mobile"": true}" 13679,5,308,2017-05-04 08:02:24,http://hamill.info/sam.reynolds,,204.29.74.7,"{""location"": ""AX"", ""is_mobile"": true}" 13680,5,308,2016-12-29 19:55:30,http://kelermarvin.net/esteban,,204.204.35.248,"{""location"": ""CG"", ""is_mobile"": false}" 13681,5,308,2017-03-14 14:06:33,http://osinskiwaelchi.org/marlen.beier,,180.83.128.230,"{""location"": ""BY"", ""is_mobile"": false}" 13682,5,308,2017-05-03 12:51:43,http://bogan.co/hayden,,172.21.113.226,"{""location"": ""LS"", ""is_mobile"": false}" 13683,5,308,2017-02-17 00:19:39,http://olsonoreilly.com/kailey,,65.130.190.58,"{""location"": ""PY"", ""is_mobile"": true}" 13684,5,308,2017-02-05 04:15:21,http://lockman.org/rogelio.bailey,,16.234.19.115,"{""location"": ""NU"", ""is_mobile"": true}" 13685,5,308,2017-02-11 09:15:42,http://hagenesgrant.org/teagan_mante,,244.212.102.140,"{""location"": ""BA"", ""is_mobile"": false}" 13686,5,308,2016-12-16 11:42:24,http://shieldsfranecki.name/arden,,119.243.212.37,"{""location"": ""ZM"", ""is_mobile"": false}" 13687,5,308,2017-01-20 22:51:53,http://strackeherman.org/keshawn,,165.250.127.212,"{""location"": ""TV"", ""is_mobile"": false}" 13688,5,308,2017-04-14 20:17:21,http://bayerbalistreri.org/ahmad,,225.100.252.128,"{""location"": ""TL"", ""is_mobile"": true}" 13689,5,308,2017-03-16 02:26:53,http://cole.net/izabella.ernser,,55.104.141.157,"{""location"": ""DO"", ""is_mobile"": false}" 13690,5,308,2017-04-11 16:21:10,http://dicki.net/kianna_gutmann,,79.158.6.201,"{""location"": ""NG"", ""is_mobile"": false}" 13691,5,308,2017-05-06 03:07:46,http://baumbach.biz/alex_hoeger,,225.78.33.141,"{""location"": ""LA"", ""is_mobile"": false}" 13692,5,308,2017-05-06 15:30:08,http://larkin.biz/kailyn_harber,,26.223.170.144,"{""location"": ""FI"", ""is_mobile"": false}" 13693,5,308,2017-03-03 20:43:00,http://breitenberg.co/deshawn,,46.10.66.229,"{""location"": ""NA"", ""is_mobile"": true}" 13694,5,308,2016-12-18 22:00:05,http://berge.io/samson,,14.178.171.106,"{""location"": ""CI"", ""is_mobile"": true}" 13695,5,308,2017-03-31 04:44:58,http://ryan.io/dylan,,206.141.43.196,"{""location"": ""JO"", ""is_mobile"": false}" 13696,5,308,2017-06-11 11:01:14,http://rosenbaum.name/brooklyn_hoppe,,227.248.110.53,"{""location"": ""SV"", ""is_mobile"": false}" 13697,5,308,2016-12-25 16:53:19,http://shanahan.net/maximo.nienow,,171.200.229.132,"{""location"": ""KI"", ""is_mobile"": true}" 13698,5,308,2016-12-14 03:17:45,http://rosenbaum.info/florian,,148.140.92.213,"{""location"": ""TF"", ""is_mobile"": false}" 13699,5,308,2017-03-13 01:49:53,http://king.info/vanea_dickinson,,152.202.49.34,"{""location"": ""DO"", ""is_mobile"": true}" 13700,5,308,2017-05-09 19:13:04,http://kuhn.org/jerrell.thompson,,68.185.2.101,"{""location"": ""WS"", ""is_mobile"": false}" 13701,5,308,2017-02-28 07:35:59,http://abshireshanahan.io/conrad.kling,,220.114.164.237,"{""location"": ""BV"", ""is_mobile"": true}" 13702,5,308,2016-12-22 17:40:19,http://handrobel.net/cordelia,,203.127.68.91,"{""location"": ""FM"", ""is_mobile"": false}" 13703,5,308,2017-02-20 06:43:24,http://rogahnhaag.com/nicholas.beatty,,219.90.113.55,"{""location"": ""CH"", ""is_mobile"": false}" 13704,5,308,2017-03-21 03:56:22,http://gulgowski.co/gerry,,233.157.79.53,"{""location"": ""LV"", ""is_mobile"": true}" 13705,5,308,2017-04-01 19:01:13,http://nikolaus.co/mauricio,,154.176.160.176,"{""location"": ""TF"", ""is_mobile"": false}" 13706,5,308,2017-05-03 09:45:13,http://hermann.org/rhoda,,65.231.223.68,"{""location"": ""MO"", ""is_mobile"": true}" 13707,5,308,2017-01-19 02:37:24,http://dare.io/mikayla.kshlerin,,142.50.156.215,"{""location"": ""BG"", ""is_mobile"": false}" 13708,5,308,2017-06-10 04:53:12,http://daviswatsica.name/saul.oconner,,71.240.100.89,"{""location"": ""SI"", ""is_mobile"": false}" 13709,5,308,2017-01-30 01:59:48,http://gleichner.co/melvin,,92.25.233.164,"{""location"": ""YE"", ""is_mobile"": false}" 13710,5,308,2017-05-23 19:52:21,http://ornkaulke.name/dee.white,,254.105.44.38,"{""location"": ""KY"", ""is_mobile"": false}" 13711,5,308,2017-02-08 18:32:04,http://fayshields.com/lawson_ziemann,,43.164.190.254,"{""location"": ""FR"", ""is_mobile"": false}" 13712,5,308,2017-01-10 07:10:17,http://littlegrimes.info/krystina.rutherford,,111.139.236.231,"{""location"": ""NC"", ""is_mobile"": false}" 13713,5,308,2017-06-10 14:14:34,http://crookstremblay.info/laury_bauch,,226.145.123.169,"{""location"": ""PR"", ""is_mobile"": true}" 13714,5,308,2017-05-25 16:27:55,http://marksdickinson.org/woodrow_zieme,,60.54.90.19,"{""location"": ""TG"", ""is_mobile"": false}" 13715,5,308,2016-12-24 10:14:25,http://swaniawski.biz/paige_gislason,,29.227.13.109,"{""location"": ""CO"", ""is_mobile"": false}" 13716,5,308,2016-12-23 20:32:40,http://ziemannboyer.co/alford,,184.234.75.195,"{""location"": ""CH"", ""is_mobile"": true}" 13717,5,308,2017-03-23 20:40:17,http://nolan.com/heather,,161.3.123.178,"{""location"": ""IR"", ""is_mobile"": true}" 13718,5,308,2017-05-19 12:56:10,http://haley.org/neoma.klocko,,110.19.164.243,"{""location"": ""KE"", ""is_mobile"": false}" 13719,5,308,2017-01-12 06:46:11,http://kihn.net/madilyn_wiegand,,246.234.45.139,"{""location"": ""MK"", ""is_mobile"": true}" 13720,5,308,2017-02-21 04:17:21,http://fahey.com/korey,,195.94.176.2,"{""location"": ""IL"", ""is_mobile"": true}" 13721,5,309,2016-12-24 17:30:12,http://turcotte.name/tate_powlowski,,15.23.122.216,"{""location"": ""YT"", ""is_mobile"": false}" 13722,5,309,2017-05-04 15:22:58,http://vandervortwolf.co/laurel,,61.215.132.118,"{""location"": ""CK"", ""is_mobile"": false}" 13723,5,309,2017-05-01 03:13:07,http://raynorlowe.org/arlo.witting,,51.212.46.11,"{""location"": ""ZW"", ""is_mobile"": true}" 13724,5,309,2016-12-16 10:47:10,http://kaulke.net/vicenta.sauer,,170.216.74.133,"{""location"": ""BM"", ""is_mobile"": true}" 13725,5,309,2017-01-23 16:04:40,http://okunevako.com/lon.yost,,66.120.220.210,"{""location"": ""GH"", ""is_mobile"": false}" 16655,6,378,2017-06-07 06:16:21,http://stark.biz/jennyfer,,185.119.59.117,"{""location"": ""PK"", ""is_mobile"": false}" 16656,6,378,2017-02-21 19:56:33,http://lockman.name/esmeralda_kuphal,,77.53.78.10,"{""location"": ""GY"", ""is_mobile"": false}" 16657,6,378,2017-02-19 04:29:06,http://hackett.net/paxton.nitzsche,,79.231.15.168,"{""location"": ""CZ"", ""is_mobile"": false}" 16658,6,378,2017-02-23 16:38:54,http://wisozk.co/lelah,,3.92.168.138,"{""location"": ""GW"", ""is_mobile"": true}" 16659,6,378,2017-03-27 18:21:01,http://darewuckert.io/llewellyn.mcclure,,39.38.89.147,"{""location"": ""FR"", ""is_mobile"": true}" 16660,6,378,2017-05-01 04:16:24,http://christiansenreynolds.name/jody,,116.31.124.184,"{""location"": ""SK"", ""is_mobile"": false}" 16661,6,378,2017-01-13 02:07:33,http://marks.info/ethan,,135.142.50.101,"{""location"": ""DO"", ""is_mobile"": true}" 16662,6,378,2017-04-11 15:00:58,http://stammryan.io/henri,,227.228.230.46,"{""location"": ""IN"", ""is_mobile"": false}" 16663,6,378,2017-02-09 07:55:49,http://schneider.org/erwin,,55.252.95.214,"{""location"": ""NF"", ""is_mobile"": true}" 16664,6,378,2017-01-31 23:53:07,http://weimannborer.info/hardy_spencer,,238.154.195.248,"{""location"": ""LY"", ""is_mobile"": false}" 16665,6,378,2017-01-30 11:43:57,http://thompson.org/beth.ernser,,205.162.28.167,"{""location"": ""BN"", ""is_mobile"": true}" 16666,6,378,2016-12-19 17:10:38,http://jakubowski.io/janea.hills,,145.226.14.249,"{""location"": ""GL"", ""is_mobile"": false}" 16667,6,378,2017-05-03 03:19:44,http://ward.org/gustave,,222.152.153.249,"{""location"": ""CM"", ""is_mobile"": false}" 16668,6,378,2017-06-12 14:05:18,http://beier.biz/mariane,,196.180.19.159,"{""location"": ""MX"", ""is_mobile"": true}" 16669,6,378,2017-03-18 09:43:43,http://hills.info/trisha,,88.150.157.61,"{""location"": ""BA"", ""is_mobile"": true}" 16670,6,378,2017-05-14 10:14:35,http://cain.com/consuelo,,72.7.55.156,"{""location"": ""EG"", ""is_mobile"": false}" 16671,6,378,2017-01-11 18:58:17,http://hartmannrempel.net/pinkie_prosacco,,225.148.181.2,"{""location"": ""PY"", ""is_mobile"": true}" 16672,6,378,2017-05-01 02:20:19,http://tillman.net/jeff,,235.250.16.96,"{""location"": ""TT"", ""is_mobile"": false}" 16673,6,378,2017-06-07 21:29:32,http://swaniawski.net/elena_gleichner,,46.128.67.203,"{""location"": ""MU"", ""is_mobile"": true}" 16674,6,378,2017-05-07 15:01:10,http://upton.com/keyon,,128.218.66.233,"{""location"": ""PR"", ""is_mobile"": false}" 16675,6,378,2017-04-17 14:50:44,http://ratke.net/alfonzo.kiehn,,97.85.138.229,"{""location"": ""BG"", ""is_mobile"": true}" 16676,6,378,2017-04-23 01:30:57,http://armstrongmetz.name/hazle,,244.128.74.108,"{""location"": ""KY"", ""is_mobile"": false}" 16677,6,378,2017-02-26 17:52:28,http://donnelly.biz/daniela,,250.100.210.111,"{""location"": ""DK"", ""is_mobile"": false}" 16678,6,378,2017-03-29 19:15:17,http://lednerprosacco.com/libby,,226.147.134.134,"{""location"": ""SZ"", ""is_mobile"": false}" 16679,6,378,2017-01-20 15:14:44,http://leannon.biz/savanna.cole,,66.100.228.129,"{""location"": ""KY"", ""is_mobile"": false}" 16680,6,378,2017-01-25 21:26:24,http://stokes.net/pink,,39.159.52.106,"{""location"": ""PT"", ""is_mobile"": false}" 16681,6,378,2017-02-03 06:20:01,http://gibson.org/rubie,,131.129.66.170,"{""location"": ""CD"", ""is_mobile"": true}" 16682,6,378,2017-03-06 19:48:54,http://dibbertnikolaus.com/jeremie.dubuque,,165.122.226.230,"{""location"": ""BH"", ""is_mobile"": false}" 16683,6,378,2017-05-25 11:13:59,http://howe.org/julianne,,38.227.183.223,"{""location"": ""TR"", ""is_mobile"": false}" 16684,6,378,2017-02-06 07:27:20,http://oberbrunner.io/kayla,,191.220.49.139,"{""location"": ""GY"", ""is_mobile"": false}" 16685,6,378,2017-05-26 10:33:14,http://rutherfordchamplin.net/sylvan.nolan,,139.76.202.131,"{""location"": ""PS"", ""is_mobile"": false}" 16686,6,378,2017-03-15 20:26:44,http://kaulke.biz/melia,,203.217.115.252,"{""location"": ""SX"", ""is_mobile"": false}" 16687,6,378,2017-04-10 04:49:32,http://volkman.biz/myrtice.ferry,,253.120.147.129,"{""location"": ""HK"", ""is_mobile"": true}" 16688,6,378,2017-05-10 02:35:43,http://pagac.io/burnice,,217.176.27.76,"{""location"": ""IL"", ""is_mobile"": true}" 16689,6,378,2017-02-13 04:17:51,http://wolfflegros.com/natalie.ondricka,,91.239.5.247,"{""location"": ""NA"", ""is_mobile"": false}" 16690,6,378,2017-04-28 00:03:14,http://sawaynwilkinson.com/trace,,164.242.212.159,"{""location"": ""LV"", ""is_mobile"": false}" 16691,6,378,2017-05-13 12:19:48,http://osinski.biz/kris,,26.81.191.252,"{""location"": ""KE"", ""is_mobile"": true}" 16692,6,378,2016-12-30 03:31:26,http://reichel.org/davion,,34.85.189.228,"{""location"": ""HT"", ""is_mobile"": true}" 16693,6,378,2017-05-13 22:09:40,http://armstrongsmith.io/leatha_walsh,,119.144.200.146,"{""location"": ""DK"", ""is_mobile"": false}" 16694,6,378,2016-12-17 03:08:12,http://schumm.net/miguel,,171.210.235.182,"{""location"": ""GG"", ""is_mobile"": true}" 16695,6,378,2017-01-22 06:45:37,http://ritchieruecker.co/nat_mcglynn,,61.38.230.204,"{""location"": ""BI"", ""is_mobile"": true}" 16696,6,378,2017-01-24 23:32:44,http://littlereynolds.info/briana,,165.152.107.186,"{""location"": ""TM"", ""is_mobile"": true}" 16697,6,378,2017-06-02 08:43:16,http://oreilly.biz/lulu.skiles,,239.194.230.91,"{""location"": ""BZ"", ""is_mobile"": true}" 16698,6,378,2017-05-14 21:32:54,http://waelchi.com/jeika_rowe,,211.250.153.194,"{""location"": ""PL"", ""is_mobile"": true}" 16699,6,378,2017-05-07 01:26:35,http://bernhardcrooks.com/linnea_cormier,,21.58.183.30,"{""location"": ""MP"", ""is_mobile"": true}" 16700,6,378,2017-01-06 16:25:35,http://heaney.info/jordan,,83.176.119.99,"{""location"": ""MX"", ""is_mobile"": false}" 16701,6,378,2017-02-20 06:25:32,http://balistreri.net/kenya_abbott,,98.59.45.14,"{""location"": ""AU"", ""is_mobile"": true}" 16702,6,378,2017-04-05 01:41:05,http://jonesernser.biz/matteo.hauck,,26.183.163.117,"{""location"": ""MQ"", ""is_mobile"": true}" 16703,6,378,2017-03-18 20:02:07,http://sengerherman.biz/geovanni,,98.37.198.30,"{""location"": ""TV"", ""is_mobile"": true}" 16704,6,378,2017-04-19 16:45:36,http://dach.net/laisha_bernier,,160.136.164.155,"{""location"": ""SZ"", ""is_mobile"": true}" 16705,6,378,2017-05-10 08:51:08,http://collins.io/piper.schultz,,6.96.160.164,"{""location"": ""MO"", ""is_mobile"": false}" 16706,6,378,2017-02-06 03:31:50,http://walshvandervort.io/patience.gerhold,,174.127.213.100,"{""location"": ""DJ"", ""is_mobile"": false}" 16707,6,379,2017-06-09 08:17:48,http://ohara.co/hulda_heathcote,,51.20.83.73,"{""location"": ""RO"", ""is_mobile"": false}" 16708,6,379,2016-12-16 13:22:15,http://goyette.io/allene,,93.6.178.119,"{""location"": ""NR"", ""is_mobile"": false}" 16709,6,379,2017-06-13 22:36:01,http://fay.com/marguerite_barrows,,118.241.170.25,"{""location"": ""TK"", ""is_mobile"": false}" 16710,6,379,2016-12-13 23:57:32,http://dickijones.net/reid,,252.161.103.86,"{""location"": ""BD"", ""is_mobile"": false}" 3834,2,84,2017-02-22 03:04:18,http://coleziemann.info/efren_leuschke,0.5952692137,166.214.34.203,"{""location"": ""PK"", ""is_mobile"": true}" 3835,2,84,2017-04-17 12:27:57,http://brakuslang.org/hector,0.0609658362,222.148.231.158,"{""location"": ""BQ"", ""is_mobile"": false}" 3836,2,84,2017-01-04 00:40:43,http://bednargulgowski.co/hunter,0.0918979353,76.198.117.136,"{""location"": ""CX"", ""is_mobile"": true}" 3837,2,84,2017-06-07 20:41:08,http://feest.org/gertrude_kris,0.7863564891,7.15.107.191,"{""location"": ""KM"", ""is_mobile"": false}" 3838,2,84,2017-01-02 08:47:20,http://kihn.com/vanea.wilderman,0.9080873176,58.190.103.240,"{""location"": ""JP"", ""is_mobile"": false}" 3839,2,84,2017-05-25 21:07:39,http://oconner.io/elinor,0.1389007248,160.230.106.243,"{""location"": ""JM"", ""is_mobile"": false}" 3840,2,84,2017-03-24 13:40:32,http://hauck.org/queenie_langosh,0.8452131340,50.166.209.233,"{""location"": ""ZA"", ""is_mobile"": false}" 3841,2,84,2017-01-29 14:24:33,http://ferry.name/ardith_schimmel,0.0496780388,96.104.241.149,"{""location"": ""BA"", ""is_mobile"": false}" 3842,2,84,2017-01-18 14:47:11,http://wehnerko.name/lexi_hickle,0.9428979289,77.225.113.46,"{""location"": ""VU"", ""is_mobile"": false}" 3843,2,84,2017-06-06 05:48:15,http://bradtke.org/emile,0.0338051448,191.45.74.237,"{""location"": ""PS"", ""is_mobile"": false}" 3844,2,84,2017-04-13 20:39:09,http://littlehyatt.biz/rosemarie,0.3127367836,122.21.239.74,"{""location"": ""AT"", ""is_mobile"": true}" 3845,2,84,2017-06-05 07:39:26,http://bartell.net/franz,0.8907152072,222.94.42.116,"{""location"": ""UG"", ""is_mobile"": true}" 3846,2,84,2017-03-18 04:38:22,http://mcglynn.org/estrella.cummerata,0.0546735276,195.177.176.90,"{""location"": ""AW"", ""is_mobile"": false}" 3847,2,84,2017-06-12 03:35:15,http://blanda.net/skye,0.8363039517,178.167.248.96,"{""location"": ""TW"", ""is_mobile"": true}" 3848,2,84,2017-01-03 17:34:46,http://halvorsonhermiston.co/marlene,0.8330775186,147.89.37.40,"{""location"": ""CW"", ""is_mobile"": false}" 3849,2,84,2017-03-01 09:23:14,http://harris.co/alfreda,0.9523583343,83.208.214.181,"{""location"": ""HM"", ""is_mobile"": false}" 3850,2,84,2017-06-12 16:08:40,http://kovacek.info/jermey.larkin,0.6288034264,204.224.156.105,"{""location"": ""SJ"", ""is_mobile"": false}" 3851,2,84,2017-03-11 10:36:21,http://hahn.name/emilio,0.9029883644,62.221.145.130,"{""location"": ""LV"", ""is_mobile"": false}" 3852,2,84,2017-03-18 14:07:36,http://huelbartell.net/audie,0.0098023043,29.37.216.138,"{""location"": ""FR"", ""is_mobile"": false}" 3853,2,84,2016-12-25 08:46:37,http://schamberger.io/yesenia,0.8532166072,3.215.19.95,"{""location"": ""JO"", ""is_mobile"": true}" 3854,2,84,2017-03-04 13:22:26,http://brekke.info/mohammed_gulgowski,0.1368232795,100.172.47.167,"{""location"": ""GU"", ""is_mobile"": true}" 3855,2,84,2017-01-16 17:10:14,http://bosconitzsche.com/phyllis,0.0109074617,150.96.187.2,"{""location"": ""PM"", ""is_mobile"": false}" 3856,2,84,2017-01-09 17:41:28,http://quigleykoch.name/wallace,0.9658007683,60.181.204.137,"{""location"": ""PM"", ""is_mobile"": true}" 3857,2,84,2017-03-03 08:31:35,http://frami.name/etha,0.2908532885,146.68.15.242,"{""location"": ""KN"", ""is_mobile"": false}" 3858,2,84,2017-03-16 12:17:32,http://lubowitzkris.org/wilton.reichert,0.3897538653,205.88.62.97,"{""location"": ""HK"", ""is_mobile"": true}" 3859,2,84,2017-04-05 12:54:35,http://upton.name/kayleigh_herman,0.2035934388,149.80.149.221,"{""location"": ""DJ"", ""is_mobile"": true}" 3860,2,84,2017-04-01 23:30:13,http://leannon.io/nathen,0.1347799211,236.132.253.98,"{""location"": ""BW"", ""is_mobile"": true}" 3861,2,84,2017-04-30 20:04:00,http://sengerconsidine.biz/giovanny.smith,0.8535512916,168.72.54.45,"{""location"": ""SE"", ""is_mobile"": true}" 3862,2,84,2017-02-22 12:46:08,http://nolan.biz/adalberto,0.2871367699,213.81.95.172,"{""location"": ""VU"", ""is_mobile"": false}" 3863,2,84,2017-03-03 15:45:12,http://walter.com/hettie,0.8775993932,38.65.86.212,"{""location"": ""GB"", ""is_mobile"": false}" 3864,2,84,2017-06-13 10:02:13,http://littel.name/shayne,0.2570876400,90.76.205.173,"{""location"": ""ER"", ""is_mobile"": false}" 3865,2,84,2017-06-10 18:34:48,http://hoppebarrows.io/agnes,0.0260311825,243.87.236.62,"{""location"": ""KI"", ""is_mobile"": false}" 3866,2,84,2017-03-22 00:42:27,http://kutch.co/sanford,0.0518801126,32.88.11.246,"{""location"": ""MX"", ""is_mobile"": false}" 3867,2,85,2017-01-12 19:53:45,http://bayer.com/antonio_champlin,0.4509257915,3.206.239.196,"{""location"": ""DM"", ""is_mobile"": false}" 3868,2,85,2016-12-29 01:30:16,http://zboncak.net/precious,0.8903190243,200.250.191.104,"{""location"": ""MY"", ""is_mobile"": true}" 3869,2,85,2017-04-05 21:49:36,http://bergnaum.com/omari.ritchie,0.0897596170,165.183.116.98,"{""location"": ""LC"", ""is_mobile"": false}" 3870,2,85,2017-02-05 03:37:42,http://baileystamm.name/jayden.mann,0.2535172972,84.167.123.216,"{""location"": ""NL"", ""is_mobile"": false}" 3871,2,85,2017-01-09 11:59:11,http://fadel.org/ibrahim.klein,0.0604008711,11.150.147.197,"{""location"": ""SZ"", ""is_mobile"": false}" 3872,2,85,2017-05-02 11:16:43,http://mohr.biz/ed,0.9041609089,206.97.48.217,"{""location"": ""MQ"", ""is_mobile"": false}" 3873,2,85,2017-04-16 01:45:13,http://spencer.io/aileen.ratke,0.1811435615,83.149.27.220,"{""location"": ""SE"", ""is_mobile"": true}" 3874,2,85,2017-03-17 15:55:25,http://murray.net/harrison_hamill,0.0789080930,221.209.15.115,"{""location"": ""MX"", ""is_mobile"": true}" 3875,2,85,2017-02-03 23:51:24,http://larsonrunolfon.info/germaine.auer,0.6277004745,120.23.102.74,"{""location"": ""CY"", ""is_mobile"": false}" 3876,2,85,2017-05-16 15:08:51,http://abbottsawayn.co/chloe_bednar,0.0972372793,243.186.123.180,"{""location"": ""TM"", ""is_mobile"": true}" 3877,2,85,2017-05-31 21:10:57,http://zemlakkshlerin.name/jazmyne,0.0259444961,107.41.113.247,"{""location"": ""CF"", ""is_mobile"": false}" 3878,2,85,2017-01-09 08:19:07,http://leffler.name/willa.wilderman,0.3769691608,61.103.181.167,"{""location"": ""DJ"", ""is_mobile"": true}" 3879,2,85,2016-12-13 17:56:17,http://dickenshyatt.net/consuelo,0.6665946563,246.24.162.97,"{""location"": ""MG"", ""is_mobile"": true}" 3880,2,85,2017-02-24 01:05:03,http://toy.io/charlene,0.9159693093,49.214.228.50,"{""location"": ""SD"", ""is_mobile"": false}" 3881,2,85,2017-02-21 23:45:38,http://friesenstroman.name/kiarra_fisher,0.4004157319,48.220.178.219,"{""location"": ""KN"", ""is_mobile"": true}" 3882,2,85,2017-01-11 23:20:59,http://jacobson.io/waylon_mayer,0.1073032887,154.245.83.100,"{""location"": ""MM"", ""is_mobile"": true}" 3883,2,85,2017-04-09 03:14:02,http://gleichner.info/geovanni_dietrich,0.3509593339,19.87.198.44,"{""location"": ""BH"", ""is_mobile"": false}" 3884,2,85,2017-05-02 02:17:19,http://jakubowski.org/marty.gibson,0.2413180487,233.203.106.135,"{""location"": ""EC"", ""is_mobile"": false}" 10776,4,241,2017-03-11 16:17:07,http://sanford.info/petra.schaden,,168.22.243.125,"{""location"": ""GM"", ""is_mobile"": true}" 10777,4,241,2017-03-09 19:30:25,http://gulgowski.io/ericka,,238.64.10.225,"{""location"": ""GI"", ""is_mobile"": true}" 10778,4,241,2016-12-24 14:49:18,http://brekke.co/verner,,15.249.96.57,"{""location"": ""VI"", ""is_mobile"": true}" 10779,4,241,2017-06-10 14:11:10,http://donnelly.com/tom_mertz,,21.95.180.211,"{""location"": ""IS"", ""is_mobile"": true}" 10780,4,241,2017-02-14 20:38:57,http://balistreri.co/lafayette.collins,,55.115.20.60,"{""location"": ""LB"", ""is_mobile"": false}" 10781,4,241,2017-04-18 16:20:53,http://brakushills.com/dax_smitham,,228.162.55.179,"{""location"": ""KE"", ""is_mobile"": true}" 10782,4,241,2017-03-30 06:15:22,http://steuber.net/dorthy_kihn,,24.30.172.16,"{""location"": ""FJ"", ""is_mobile"": true}" 10783,4,241,2016-12-17 00:47:55,http://gutkowski.net/kurtis_hoppe,,111.3.136.185,"{""location"": ""IN"", ""is_mobile"": false}" 10784,4,241,2017-02-05 10:42:57,http://ondrickadavis.co/ludwig_donnelly,,194.243.73.245,"{""location"": ""BL"", ""is_mobile"": true}" 10785,4,241,2017-03-27 23:26:19,http://ruecker.biz/kianna.heel,,187.43.221.151,"{""location"": ""SY"", ""is_mobile"": true}" 10786,4,241,2017-01-06 19:49:30,http://moriette.net/kitty,,199.23.183.201,"{""location"": ""RE"", ""is_mobile"": true}" 10787,4,241,2017-01-11 00:26:13,http://rennerturner.name/earnest,,126.245.140.197,"{""location"": ""CN"", ""is_mobile"": true}" 10788,4,241,2017-05-19 04:06:21,http://mclaughlinzieme.net/rosalinda,,177.240.87.224,"{""location"": ""DZ"", ""is_mobile"": true}" 10789,4,241,2017-03-15 12:20:19,http://beatty.io/gerson,,242.246.127.135,"{""location"": ""AQ"", ""is_mobile"": true}" 10790,4,241,2017-04-20 12:07:29,http://pricegoldner.com/fanny_jast,,101.92.123.52,"{""location"": ""LU"", ""is_mobile"": true}" 10791,4,241,2017-03-11 08:06:12,http://abernathy.net/frederik_bogan,,89.110.220.154,"{""location"": ""FR"", ""is_mobile"": true}" 10792,4,241,2017-01-16 13:08:16,http://williamson.co/evangeline,,223.34.84.244,"{""location"": ""VN"", ""is_mobile"": false}" 10793,4,241,2017-05-03 02:25:30,http://rosenbaum.net/ulises,,52.90.40.196,"{""location"": ""SM"", ""is_mobile"": true}" 10794,4,241,2017-02-28 21:14:39,http://mayertcorwin.name/jacklyn_windler,,7.132.42.172,"{""location"": ""LA"", ""is_mobile"": false}" 10795,4,241,2017-01-04 05:20:26,http://strosin.biz/susan,,202.136.121.59,"{""location"": ""ZA"", ""is_mobile"": true}" 10796,4,241,2017-06-08 22:43:31,http://greenholt.com/rodrick_gerhold,,109.196.13.109,"{""location"": ""GH"", ""is_mobile"": false}" 10797,4,241,2017-01-11 14:06:27,http://feest.com/ofelia,,66.10.94.107,"{""location"": ""NF"", ""is_mobile"": false}" 10798,4,241,2017-03-13 11:31:08,http://pfannerstill.net/manuel,,60.104.50.163,"{""location"": ""CK"", ""is_mobile"": false}" 10799,4,241,2017-01-16 05:28:37,http://hilpert.name/lesly_wunsch,,75.12.203.131,"{""location"": ""GQ"", ""is_mobile"": false}" 10800,4,241,2017-06-09 19:06:58,http://hackett.co/trace,,162.178.191.48,"{""location"": ""EC"", ""is_mobile"": false}" 10801,4,241,2017-04-28 13:17:54,http://spencerbradtke.name/sandrine,,89.168.43.6,"{""location"": ""ZW"", ""is_mobile"": true}" 10802,4,241,2017-04-24 17:01:08,http://howellklocko.org/bryana,,132.165.74.139,"{""location"": ""NR"", ""is_mobile"": true}" 10803,4,241,2017-06-01 05:59:14,http://zieme.io/kaylee.rempel,,151.219.191.13,"{""location"": ""AD"", ""is_mobile"": false}" 10804,4,242,2017-03-23 08:39:58,http://schinner.co/florian,,13.148.154.114,"{""location"": ""NL"", ""is_mobile"": false}" 10805,4,242,2017-02-25 17:55:29,http://heidenreich.biz/pamela,,144.243.165.110,"{""location"": ""MN"", ""is_mobile"": false}" 10806,4,242,2017-04-15 18:04:12,http://smitham.biz/mallory_kulas,,221.179.80.172,"{""location"": ""VU"", ""is_mobile"": true}" 10807,4,242,2017-03-14 01:42:55,http://sauer.net/laura_kling,,112.253.72.86,"{""location"": ""KM"", ""is_mobile"": true}" 10808,4,242,2016-12-22 02:39:29,http://legros.biz/carolanne_boyle,,247.178.21.2,"{""location"": ""PT"", ""is_mobile"": true}" 10809,4,242,2017-01-08 20:19:33,http://lindgrenkirlin.name/amina_orn,,187.32.84.33,"{""location"": ""ZA"", ""is_mobile"": true}" 10810,4,242,2016-12-24 11:03:22,http://kuhlmanmarks.info/virginia,,213.208.7.233,"{""location"": ""DE"", ""is_mobile"": false}" 10811,4,242,2017-05-26 13:11:15,http://lynch.net/alvera,,144.188.14.171,"{""location"": ""LS"", ""is_mobile"": false}" 10812,4,242,2017-03-08 15:18:42,http://ohara.net/nona,,240.34.215.186,"{""location"": ""GH"", ""is_mobile"": true}" 10813,4,242,2017-03-02 02:50:41,http://friesen.net/zane,,217.96.206.155,"{""location"": ""TT"", ""is_mobile"": false}" 10814,4,242,2017-02-22 20:40:16,http://hintz.io/dagmar,,92.12.223.39,"{""location"": ""MW"", ""is_mobile"": true}" 10815,4,242,2017-03-17 21:04:19,http://turner.net/gregorio,,122.100.187.77,"{""location"": ""NZ"", ""is_mobile"": false}" 10816,4,242,2017-01-28 09:04:35,http://kubmann.biz/ruel,,91.133.22.118,"{""location"": ""PS"", ""is_mobile"": true}" 10817,4,242,2016-12-21 17:12:41,http://cummingsmitchell.co/malachi_schultz,,192.213.246.234,"{""location"": ""ZA"", ""is_mobile"": true}" 10818,4,242,2017-06-07 18:55:23,http://ankunding.net/antonetta.jast,,10.48.144.243,"{""location"": ""MQ"", ""is_mobile"": false}" 10819,4,242,2017-03-03 17:00:03,http://moen.name/kory,,225.122.233.182,"{""location"": ""LU"", ""is_mobile"": true}" 10820,4,242,2017-06-03 19:52:09,http://gottlieb.name/vita_labadie,,217.148.35.121,"{""location"": ""BI"", ""is_mobile"": true}" 10821,4,242,2017-03-18 18:36:51,http://deckow.io/donnell_beatty,,192.124.232.229,"{""location"": ""LT"", ""is_mobile"": false}" 10822,4,242,2016-12-22 07:05:27,http://quitzon.com/sarai,,242.2.216.29,"{""location"": ""JE"", ""is_mobile"": true}" 10823,4,242,2017-03-12 01:41:45,http://zemlakhills.net/joel.kuhlman,,124.194.110.91,"{""location"": ""BM"", ""is_mobile"": true}" 10824,4,242,2017-05-09 10:17:32,http://gusikowskischuppe.io/duane.will,,222.9.148.114,"{""location"": ""SX"", ""is_mobile"": false}" 10825,4,242,2016-12-31 16:51:19,http://crist.biz/granville.bergstrom,,200.11.126.215,"{""location"": ""SO"", ""is_mobile"": false}" 10826,4,242,2016-12-18 19:30:48,http://reingerboyle.org/heidi,,204.226.156.172,"{""location"": ""PT"", ""is_mobile"": true}" 10827,4,242,2017-03-15 20:28:22,http://streich.biz/freddie_heaney,,115.173.224.88,"{""location"": ""CY"", ""is_mobile"": false}" 10828,4,242,2017-01-02 09:37:18,http://reilly.org/william_witting,,220.245.139.52,"{""location"": ""CM"", ""is_mobile"": true}" 10829,4,242,2017-02-26 20:06:47,http://robel.biz/alysha,,68.187.181.13,"{""location"": ""SG"", ""is_mobile"": false}" 10830,4,242,2017-04-23 17:50:28,http://stantonkautzer.info/luciano,,110.75.206.221,"{""location"": ""CR"", ""is_mobile"": true}" 10831,4,242,2017-03-18 16:23:20,http://kaulkebatz.com/taya,,71.193.235.239,"{""location"": ""KW"", ""is_mobile"": true}" 7810,3,173,2017-04-30 08:50:49,http://kemmer.biz/brielle.wuckert,0.6554664378,203.11.233.16,"{""location"": ""LI"", ""is_mobile"": true}" 7811,3,173,2017-04-19 00:24:44,http://davisgerlach.name/frederik.kemmer,0.4002814152,47.160.114.146,"{""location"": ""IN"", ""is_mobile"": false}" 7812,3,173,2017-03-23 03:07:34,http://rosenbaumwest.io/pamela,0.5444390891,197.154.169.202,"{""location"": ""SO"", ""is_mobile"": false}" 7813,3,173,2017-05-27 09:35:13,http://lindmccullough.name/dedric.welch,0.6264647403,152.232.213.197,"{""location"": ""RU"", ""is_mobile"": true}" 7814,3,173,2017-04-02 15:17:28,http://rempelsporer.org/davion_johns,0.9711488311,51.251.80.251,"{""location"": ""UM"", ""is_mobile"": true}" 7815,3,173,2017-02-22 23:03:19,http://ullrichschultz.name/lulu,0.6083168207,60.181.231.18,"{""location"": ""CO"", ""is_mobile"": false}" 7816,3,173,2016-12-15 14:20:35,http://wuckert.com/bella_frami,0.1759115185,90.89.222.62,"{""location"": ""PF"", ""is_mobile"": false}" 7817,3,173,2017-04-09 09:32:22,http://schroeder.net/elva,0.3615197798,175.80.158.228,"{""location"": ""SM"", ""is_mobile"": false}" 7818,3,173,2016-12-15 00:18:52,http://schneider.name/sydni,0.7357132063,110.194.23.221,"{""location"": ""ZW"", ""is_mobile"": false}" 7819,3,173,2017-04-13 05:00:31,http://ohara.biz/destany,0.5305625345,25.203.160.53,"{""location"": ""CK"", ""is_mobile"": true}" 7820,3,173,2017-05-21 11:54:33,http://stracke.biz/lauretta.braun,0.5866356307,250.47.188.9,"{""location"": ""SG"", ""is_mobile"": false}" 7821,3,173,2017-02-20 22:05:02,http://waelchi.net/ladarius,0.0319090361,32.244.101.69,"{""location"": ""TD"", ""is_mobile"": true}" 7822,3,173,2017-03-24 20:31:47,http://vonrueden.net/wanda_herman,0.1650674686,128.135.161.15,"{""location"": ""NO"", ""is_mobile"": false}" 7823,3,173,2017-04-26 16:44:53,http://gorczany.com/dario_waelchi,0.8345326160,235.233.248.70,"{""location"": ""BR"", ""is_mobile"": false}" 7824,3,173,2017-05-30 02:39:14,http://predovic.io/colleen_rodriguez,0.4009573320,9.164.226.90,"{""location"": ""HK"", ""is_mobile"": false}" 7825,3,173,2016-12-17 00:15:12,http://armstrong.co/erich,0.2320792064,155.53.254.126,"{""location"": ""ZW"", ""is_mobile"": false}" 7826,3,173,2017-05-03 20:31:03,http://brown.net/jaunita,0.2399424689,151.148.237.37,"{""location"": ""MH"", ""is_mobile"": false}" 7827,3,173,2017-01-10 03:41:19,http://baumbach.org/guy,0.4346414049,223.33.22.35,"{""location"": ""TG"", ""is_mobile"": true}" 7828,3,173,2017-03-17 00:25:42,http://vandervortgorczany.com/wilma,0.7570314042,48.227.90.218,"{""location"": ""TW"", ""is_mobile"": false}" 7829,3,173,2017-01-29 17:42:39,http://ullrich.io/raphael,0.3605711494,191.83.226.228,"{""location"": ""LR"", ""is_mobile"": true}" 7830,3,173,2017-06-03 05:50:00,http://treutel.info/zachery,0.5195696527,217.198.96.247,"{""location"": ""GP"", ""is_mobile"": false}" 7831,3,173,2017-05-25 09:46:43,http://fayhickle.com/jamil.harvey,0.7769349310,105.145.145.179,"{""location"": ""BF"", ""is_mobile"": true}" 7832,3,173,2017-03-15 08:17:43,http://labadiewalter.name/dorothea_heaney,0.7675033131,69.220.175.116,"{""location"": ""IM"", ""is_mobile"": true}" 7833,3,173,2017-06-06 23:09:24,http://langworth.com/joaquin_mohr,0.4412650839,250.155.251.46,"{""location"": ""IT"", ""is_mobile"": true}" 7834,3,173,2017-06-10 08:56:52,http://hand.co/keely_marvin,0.9706078542,28.245.31.159,"{""location"": ""NR"", ""is_mobile"": false}" 7835,3,173,2017-01-22 23:05:10,http://ryan.com/jadon,0.5721037849,134.35.23.56,"{""location"": ""SH"", ""is_mobile"": true}" 7836,3,173,2017-02-14 19:45:35,http://morar.io/nayeli,0.4236049669,203.13.245.221,"{""location"": ""ZA"", ""is_mobile"": true}" 7837,3,173,2016-12-23 19:25:44,http://gibson.org/lenore,0.5715791549,144.175.99.125,"{""location"": ""VU"", ""is_mobile"": true}" 7838,3,173,2017-03-21 23:18:54,http://prosacco.io/jerod.reynolds,0.1434894918,252.169.88.99,"{""location"": ""SN"", ""is_mobile"": false}" 7839,3,173,2017-03-06 06:23:33,http://erdmanschmidt.name/elsie.kilback,0.2500313649,67.154.187.100,"{""location"": ""IM"", ""is_mobile"": true}" 7840,3,173,2017-04-30 01:18:01,http://breitenbergreynolds.com/eldora,0.2742778727,40.230.209.226,"{""location"": ""UM"", ""is_mobile"": false}" 7841,3,173,2017-05-09 23:43:54,http://johnsonsauer.io/jaylen,0.8250444788,245.29.141.60,"{""location"": ""IM"", ""is_mobile"": true}" 7842,3,173,2017-02-14 07:27:00,http://marvin.com/holly,0.0951856308,69.23.233.126,"{""location"": ""AD"", ""is_mobile"": true}" 7843,3,173,2017-01-07 21:57:46,http://millskunze.com/cristian,0.5453320169,103.68.43.173,"{""location"": ""AR"", ""is_mobile"": true}" 7844,3,173,2017-04-10 05:58:38,http://herzogshanahan.net/dudley,0.3071566885,107.70.154.162,"{""location"": ""WF"", ""is_mobile"": true}" 7845,3,173,2017-03-25 00:36:03,http://barrows.org/amya.berge,0.5022741081,44.44.246.201,"{""location"": ""BW"", ""is_mobile"": true}" 7846,3,173,2017-01-09 23:34:54,http://hilll.net/ezra,0.3445473513,97.54.30.217,"{""location"": ""LC"", ""is_mobile"": true}" 7847,3,173,2016-12-25 03:22:26,http://gusikowski.co/lila,0.5884212918,59.24.69.74,"{""location"": ""DE"", ""is_mobile"": false}" 7848,3,173,2016-12-27 10:37:39,http://kuhlmanerdman.net/betsy_littel,0.9792943798,93.21.246.209,"{""location"": ""HM"", ""is_mobile"": true}" 7849,3,173,2017-03-31 13:11:24,http://berniergerhold.name/lauryn.langosh,0.6094809820,37.232.225.5,"{""location"": ""NU"", ""is_mobile"": false}" 7850,3,173,2017-05-10 00:23:00,http://oharacrooks.com/magdalena,0.9342530156,179.136.250.175,"{""location"": ""JP"", ""is_mobile"": false}" 7851,3,173,2017-05-21 16:45:02,http://murphy.com/morgan.jenkins,0.4212109045,154.182.54.151,"{""location"": ""LU"", ""is_mobile"": false}" 7852,3,173,2017-01-28 17:03:05,http://bailey.com/brionna_king,0.5380773224,10.183.3.29,"{""location"": ""ZW"", ""is_mobile"": false}" 7853,3,174,2017-01-25 00:29:15,http://kertzmann.com/jaida,0.5045877755,174.227.96.58,"{""location"": ""GW"", ""is_mobile"": true}" 7854,3,174,2017-02-21 21:44:51,http://botsford.com/peyton,0.5989844639,212.3.24.80,"{""location"": ""GE"", ""is_mobile"": true}" 7855,3,174,2017-05-06 18:45:53,http://stamm.biz/mya_dietrich,0.6828866631,217.150.243.20,"{""location"": ""BO"", ""is_mobile"": true}" 7856,3,174,2017-04-04 01:20:17,http://okeefecasper.info/nels_schoen,0.6233104010,204.249.122.201,"{""location"": ""SB"", ""is_mobile"": true}" 7857,3,174,2017-02-26 08:55:31,http://konopelski.net/eldred_krajcik,0.6051368373,241.36.174.109,"{""location"": ""CC"", ""is_mobile"": true}" 7858,3,174,2017-04-19 13:24:04,http://gottlieb.name/neva,0.9741511839,192.207.208.175,"{""location"": ""FK"", ""is_mobile"": false}" 7859,3,174,2017-05-22 10:46:03,http://murazik.org/davonte,0.5356288784,89.159.134.56,"{""location"": ""TG"", ""is_mobile"": false}" 7860,3,174,2017-01-18 09:46:55,http://medhurst.io/vladimir,0.2812396216,217.94.171.156,"{""location"": ""GT"", ""is_mobile"": true}" 7861,3,174,2017-04-13 05:51:35,http://watsica.name/kiera_prohaska,0.5287688641,100.137.113.49,"{""location"": ""MN"", ""is_mobile"": true}" 13726,5,309,2017-05-26 22:09:22,http://hermiston.com/colten,,199.88.58.78,"{""location"": ""GP"", ""is_mobile"": false}" 13727,5,309,2017-06-09 09:57:53,http://parisianoreilly.name/leanne.gottlieb,,209.137.196.114,"{""location"": ""SM"", ""is_mobile"": true}" 13728,5,309,2017-03-02 14:26:52,http://gaylordgoyette.biz/nicklaus,,46.84.147.73,"{""location"": ""AS"", ""is_mobile"": false}" 13729,5,309,2017-06-10 02:05:52,http://johns.co/waylon,,173.237.32.71,"{""location"": ""NZ"", ""is_mobile"": false}" 13730,5,309,2017-03-31 12:58:40,http://beckerhodkiewicz.net/myrtis_quitzon,,169.242.191.144,"{""location"": ""AQ"", ""is_mobile"": true}" 13731,5,309,2017-03-15 19:01:07,http://christiansen.name/kaelyn.konopelski,,116.30.254.25,"{""location"": ""FO"", ""is_mobile"": false}" 13732,5,309,2016-12-18 23:28:02,http://block.io/edwin,,115.196.244.77,"{""location"": ""GT"", ""is_mobile"": true}" 13733,5,309,2017-04-12 12:26:27,http://bergnaum.biz/nikita_schmeler,,218.157.134.210,"{""location"": ""PK"", ""is_mobile"": false}" 13734,5,309,2017-04-09 16:47:51,http://mckenzie.com/jordyn,,53.180.198.38,"{""location"": ""NE"", ""is_mobile"": false}" 13735,5,309,2017-03-05 04:43:06,http://treutelspencer.io/beaulah.harris,,232.16.175.77,"{""location"": ""BZ"", ""is_mobile"": false}" 13736,5,309,2017-03-06 07:00:24,http://ward.co/macie_renner,,235.104.251.148,"{""location"": ""SZ"", ""is_mobile"": true}" 13737,5,309,2017-05-20 03:39:14,http://stokes.co/eula_waelchi,,134.135.39.123,"{""location"": ""TO"", ""is_mobile"": false}" 13738,5,309,2017-05-14 14:14:59,http://wuckert.com/leonor,,149.78.179.5,"{""location"": ""KH"", ""is_mobile"": true}" 13739,5,309,2017-04-16 09:44:23,http://gutkowskikris.name/isaias,,134.95.181.138,"{""location"": ""ET"", ""is_mobile"": true}" 13740,5,309,2017-02-06 10:34:22,http://grant.co/darlene,,48.186.114.168,"{""location"": ""CD"", ""is_mobile"": false}" 13741,5,309,2017-01-03 06:09:24,http://jacobson.biz/chester.weimann,,247.127.167.181,"{""location"": ""NO"", ""is_mobile"": false}" 13742,5,309,2017-02-28 06:32:07,http://hansen.net/sid_stiedemann,,189.112.117.97,"{""location"": ""DZ"", ""is_mobile"": false}" 13743,5,309,2017-02-06 14:07:52,http://balistreri.info/august.bruen,,130.97.93.115,"{""location"": ""JE"", ""is_mobile"": false}" 13744,5,309,2017-05-13 19:54:53,http://kaulkeboehm.name/shannon,,22.239.160.88,"{""location"": ""CR"", ""is_mobile"": false}" 13745,5,309,2017-02-28 02:46:15,http://fahey.com/davion,,44.250.137.156,"{""location"": ""PN"", ""is_mobile"": false}" 13746,5,309,2017-05-30 06:26:02,http://denesik.name/caandre,,16.29.182.205,"{""location"": ""DM"", ""is_mobile"": false}" 13747,5,309,2017-01-08 01:57:37,http://pfannerstill.com/aurelio_schulist,,205.161.81.175,"{""location"": ""CK"", ""is_mobile"": true}" 13748,5,309,2017-01-18 23:58:22,http://marquardt.name/tyrel_bogan,,226.235.5.199,"{""location"": ""US"", ""is_mobile"": false}" 13749,5,309,2017-02-14 21:19:51,http://hackettblanda.info/hilario_rempel,,245.199.43.23,"{""location"": ""KP"", ""is_mobile"": false}" 13750,5,309,2017-02-10 19:59:34,http://donnellylebsack.io/alberto,,7.2.97.100,"{""location"": ""TM"", ""is_mobile"": true}" 13751,5,309,2017-06-10 14:04:22,http://parker.net/sarai,,122.71.148.235,"{""location"": ""AE"", ""is_mobile"": true}" 13752,5,309,2017-03-12 00:27:50,http://schinner.org/jonathan.baumbach,,155.47.71.240,"{""location"": ""SD"", ""is_mobile"": false}" 13753,5,309,2017-01-18 08:29:15,http://hansenkaulke.biz/shayne,,72.84.217.93,"{""location"": ""CF"", ""is_mobile"": false}" 13754,5,309,2017-02-05 02:42:07,http://casperbins.net/garth,,36.33.86.217,"{""location"": ""BO"", ""is_mobile"": true}" 13755,5,309,2016-12-25 21:54:58,http://huelshills.com/mattie_kunde,,134.45.138.21,"{""location"": ""AR"", ""is_mobile"": true}" 13756,5,309,2017-01-07 04:42:10,http://schoen.net/bartholome.leffler,,108.39.237.111,"{""location"": ""SL"", ""is_mobile"": true}" 13757,5,309,2017-03-14 05:25:06,http://kohler.biz/elna,,251.22.69.96,"{""location"": ""TC"", ""is_mobile"": true}" 13758,5,309,2016-12-22 06:07:04,http://walshlindgren.org/ambrose.erdman,,120.229.173.149,"{""location"": ""AF"", ""is_mobile"": true}" 13759,5,309,2017-05-12 11:26:47,http://hand.co/wayne,,221.235.140.114,"{""location"": ""KN"", ""is_mobile"": true}" 13760,5,309,2017-01-25 05:15:08,http://rodriguez.io/macy.white,,150.53.77.137,"{""location"": ""BS"", ""is_mobile"": false}" 13761,5,309,2017-01-22 01:19:59,http://goyette.com/arvel.tremblay,,13.62.126.175,"{""location"": ""AO"", ""is_mobile"": false}" 13762,5,309,2016-12-26 03:00:31,http://kub.name/dennis,,126.97.74.68,"{""location"": ""DO"", ""is_mobile"": false}" 13763,5,309,2017-05-14 15:49:13,http://larson.com/kirk_lehner,,163.165.182.23,"{""location"": ""GB"", ""is_mobile"": true}" 13764,5,309,2017-01-19 02:28:53,http://hettinger.name/vincenzo,,127.77.245.8,"{""location"": ""RO"", ""is_mobile"": true}" 13765,5,309,2017-01-06 07:01:33,http://blickkutch.name/odell,,2.252.216.46,"{""location"": ""IL"", ""is_mobile"": false}" 13766,5,309,2017-02-16 07:26:09,http://howell.net/virgil.kutch,,120.151.59.215,"{""location"": ""BR"", ""is_mobile"": false}" 13767,5,309,2017-02-15 19:46:30,http://kerluke.co/toy,,254.103.120.151,"{""location"": ""MR"", ""is_mobile"": true}" 13768,5,309,2017-05-19 11:49:04,http://kuvalis.io/lauriane.gottlieb,,220.219.252.196,"{""location"": ""SA"", ""is_mobile"": false}" 13769,5,309,2017-03-01 05:32:06,http://reynoldsebert.io/zachariah,,210.249.183.58,"{""location"": ""GN"", ""is_mobile"": true}" 13770,5,309,2017-02-22 12:10:15,http://muller.biz/josiane,,180.150.46.228,"{""location"": ""SZ"", ""is_mobile"": true}" 13771,5,309,2016-12-26 17:40:25,http://nikolauslynch.name/eryn,,185.26.74.220,"{""location"": ""KI"", ""is_mobile"": false}" 13772,5,309,2017-03-28 01:55:56,http://schuster.name/caie,,32.223.176.82,"{""location"": ""ES"", ""is_mobile"": true}" 13773,5,309,2017-04-04 06:24:01,http://schmidtkoepp.io/ania,,111.157.230.206,"{""location"": ""GU"", ""is_mobile"": true}" 13774,5,309,2017-04-21 10:17:46,http://hirtheconnelly.info/cordell,,21.230.65.134,"{""location"": ""BZ"", ""is_mobile"": true}" 13775,5,309,2017-05-11 13:35:09,http://ondricka.net/kristofer_nolan,,150.144.35.195,"{""location"": ""SZ"", ""is_mobile"": false}" 13776,5,310,2017-04-03 15:13:39,http://kirlin.io/frankie.bode,,48.41.127.223,"{""location"": ""LI"", ""is_mobile"": false}" 13777,5,310,2017-03-07 01:28:07,http://runte.name/kylie_kunde,,46.152.183.34,"{""location"": ""RO"", ""is_mobile"": false}" 13778,5,310,2017-05-28 11:06:48,http://von.io/alta.hammes,,152.185.65.72,"{""location"": ""HU"", ""is_mobile"": true}" 13779,5,310,2017-06-07 10:29:05,http://hintz.net/golden_mohr,,4.22.215.107,"{""location"": ""IT"", ""is_mobile"": true}" 13780,5,310,2017-05-26 16:52:23,http://halvorson.info/karlee.kris,,76.167.156.65,"{""location"": ""CK"", ""is_mobile"": false}" 16711,6,379,2017-01-04 04:05:32,http://mrazbernhard.info/addie.dicki,,193.74.59.60,"{""location"": ""TR"", ""is_mobile"": false}" 16712,6,379,2017-01-18 02:06:14,http://mohrmiller.net/wilhelmine.durgan,,88.46.43.163,"{""location"": ""GS"", ""is_mobile"": false}" 16713,6,379,2017-05-02 11:05:22,http://lowemaggio.io/edna,,82.73.174.87,"{""location"": ""PL"", ""is_mobile"": false}" 16714,6,379,2017-04-13 07:48:16,http://strosintowne.org/kirk.gerhold,,76.163.216.3,"{""location"": ""NO"", ""is_mobile"": false}" 16715,6,379,2017-02-23 16:33:20,http://yost.org/arne_lynch,,63.77.93.46,"{""location"": ""TR"", ""is_mobile"": false}" 16716,6,379,2017-04-09 23:45:56,http://streichgoyette.com/dusty.runolfsdottir,,135.41.170.2,"{""location"": ""SS"", ""is_mobile"": true}" 16717,6,379,2017-03-09 14:07:29,http://damore.name/eve,,170.26.74.163,"{""location"": ""CW"", ""is_mobile"": false}" 16718,6,379,2017-02-16 03:33:47,http://homenick.com/jacquelyn,,20.161.191.193,"{""location"": ""JE"", ""is_mobile"": true}" 16719,6,379,2017-02-03 01:12:07,http://mclaughlinratke.name/cameron_effertz,,193.169.41.213,"{""location"": ""NO"", ""is_mobile"": false}" 16720,6,379,2017-03-15 07:56:57,http://runte.com/hertha,,81.159.9.49,"{""location"": ""AD"", ""is_mobile"": true}" 16721,6,379,2017-05-07 18:45:47,http://kunzebradtke.net/natalie.predovic,,94.242.199.27,"{""location"": ""BW"", ""is_mobile"": true}" 16722,6,379,2017-04-10 10:36:01,http://gaylordjones.biz/maxine,,121.198.196.112,"{""location"": ""MC"", ""is_mobile"": true}" 16723,6,379,2017-06-03 20:34:43,http://kaulkekuhic.net/myah,,163.67.118.207,"{""location"": ""CN"", ""is_mobile"": false}" 16724,6,379,2017-05-15 07:00:04,http://parker.net/erica.haag,,56.212.56.23,"{""location"": ""WS"", ""is_mobile"": true}" 16725,6,379,2017-01-11 08:32:55,http://orn.name/lacey,,18.5.228.92,"{""location"": ""NF"", ""is_mobile"": false}" 16726,6,379,2017-01-30 12:45:17,http://hintz.info/van,,163.205.63.239,"{""location"": ""EH"", ""is_mobile"": false}" 16727,6,379,2016-12-22 08:31:09,http://koelpinkirlin.io/ryley_altenwerth,,104.151.128.82,"{""location"": ""IL"", ""is_mobile"": true}" 16728,6,379,2017-01-02 01:34:06,http://grant.name/florida.kertzmann,,73.240.182.46,"{""location"": ""CY"", ""is_mobile"": false}" 16729,6,379,2017-06-08 08:41:41,http://frami.io/lexie,,19.196.24.156,"{""location"": ""SE"", ""is_mobile"": false}" 16730,6,379,2017-04-23 17:43:26,http://larsonswaniawski.org/clair_mcclure,,85.188.33.178,"{""location"": ""IR"", ""is_mobile"": true}" 16731,6,379,2017-02-20 08:04:29,http://deckowkertzmann.info/okey_huels,,101.129.143.169,"{""location"": ""TW"", ""is_mobile"": false}" 16732,6,379,2017-01-24 04:42:57,http://roobgrimes.biz/dolores,,180.196.118.15,"{""location"": ""HK"", ""is_mobile"": false}" 16733,6,379,2016-12-27 03:12:46,http://corkery.biz/kenneth.schmeler,,145.226.130.133,"{""location"": ""HT"", ""is_mobile"": true}" 16734,6,379,2017-03-12 10:11:34,http://donnellytreutel.info/prince,,236.254.110.209,"{""location"": ""SX"", ""is_mobile"": false}" 16735,6,379,2017-05-18 21:10:21,http://schinner.org/ronny.hackett,,89.192.216.14,"{""location"": ""KE"", ""is_mobile"": true}" 16736,6,379,2017-01-22 00:49:39,http://fay.name/sebastian,,125.156.192.214,"{""location"": ""DM"", ""is_mobile"": true}" 16737,6,379,2017-05-26 11:15:34,http://lehner.biz/leopold,,106.47.203.2,"{""location"": ""RW"", ""is_mobile"": false}" 16738,6,379,2017-01-10 10:42:38,http://wintheiser.com/hilda_stroman,,193.197.141.165,"{""location"": ""DJ"", ""is_mobile"": false}" 16739,6,379,2017-05-02 06:22:28,http://feil.net/brendan_halvorson,,21.245.191.164,"{""location"": ""FM"", ""is_mobile"": false}" 16740,6,379,2017-02-16 23:23:09,http://schowalter.name/jacinto_dare,,129.102.188.231,"{""location"": ""KM"", ""is_mobile"": true}" 16741,6,379,2017-01-22 15:16:59,http://luettgenzemlak.co/perry.mccullough,,108.147.134.145,"{""location"": ""NI"", ""is_mobile"": true}" 16742,6,379,2017-06-03 13:47:21,http://hamill.name/vella,,102.214.68.89,"{""location"": ""GU"", ""is_mobile"": false}" 16743,6,379,2016-12-27 15:07:56,http://cain.info/bruce_terry,,108.208.35.34,"{""location"": ""FJ"", ""is_mobile"": true}" 16744,6,379,2017-03-16 02:26:06,http://stoltenberg.com/madonna,,79.191.16.63,"{""location"": ""IE"", ""is_mobile"": true}" 16745,6,379,2016-12-30 02:03:38,http://huel.biz/vladimir_dietrich,,150.27.112.175,"{""location"": ""PT"", ""is_mobile"": false}" 16746,6,379,2017-01-02 09:07:41,http://reinger.org/camryn,,113.86.136.248,"{""location"": ""PA"", ""is_mobile"": true}" 16747,6,379,2016-12-21 14:52:13,http://hackett.net/consuelo_haley,,202.141.228.78,"{""location"": ""AZ"", ""is_mobile"": false}" 16748,6,379,2017-02-04 18:56:09,http://shields.co/eric.runte,,171.47.14.181,"{""location"": ""JE"", ""is_mobile"": true}" 16749,6,379,2017-01-23 17:48:44,http://balistrericain.net/chelsie,,104.190.247.163,"{""location"": ""PR"", ""is_mobile"": true}" 16750,6,379,2017-05-21 11:21:41,http://koeppwiza.com/kendall.torphy,,199.164.237.173,"{""location"": ""FK"", ""is_mobile"": false}" 16751,6,379,2017-05-01 16:57:37,http://hartmannvandervort.io/carolyne_paucek,,17.173.113.218,"{""location"": ""NZ"", ""is_mobile"": false}" 16752,6,379,2017-04-07 02:59:38,http://mertz.info/grayce.ledner,,28.70.104.167,"{""location"": ""MS"", ""is_mobile"": false}" 16753,6,379,2017-02-10 02:36:58,http://volkman.info/virginie.mclaughlin,,72.24.13.215,"{""location"": ""TK"", ""is_mobile"": true}" 16754,6,379,2017-05-12 22:34:32,http://block.biz/justine.west,,171.126.139.159,"{""location"": ""TO"", ""is_mobile"": true}" 16755,6,379,2017-03-06 20:39:28,http://weimannmacejkovic.name/sarai_cain,,136.104.82.93,"{""location"": ""FJ"", ""is_mobile"": false}" 16756,6,379,2017-05-28 08:15:11,http://kerlukesanford.info/alanna,,7.151.24.67,"{""location"": ""CI"", ""is_mobile"": true}" 16757,6,379,2017-03-15 17:56:29,http://glover.name/king,,139.193.137.23,"{""location"": ""GY"", ""is_mobile"": false}" 16758,6,379,2016-12-28 13:03:39,http://pfannerstill.net/maye,,143.98.150.69,"{""location"": ""DO"", ""is_mobile"": false}" 16759,6,379,2017-01-31 16:31:06,http://streichryan.net/brent_brakus,,217.47.241.62,"{""location"": ""VN"", ""is_mobile"": false}" 16760,6,380,2017-06-02 16:46:43,http://hilpert.biz/walter_schamberger,,40.179.48.138,"{""location"": ""EE"", ""is_mobile"": true}" 16761,6,380,2017-01-23 01:00:06,http://terrykunde.biz/annalise,,107.235.57.144,"{""location"": ""MO"", ""is_mobile"": true}" 16762,6,380,2017-02-06 15:41:19,http://luettgen.info/chelsey.ledner,,59.198.139.140,"{""location"": ""ES"", ""is_mobile"": true}" 16763,6,380,2017-03-17 03:01:23,http://dareziemann.org/kenya.thompson,,62.2.185.202,"{""location"": ""ZA"", ""is_mobile"": false}" 16764,6,380,2017-03-11 19:00:16,http://batz.io/shanie.hoeger,,15.249.75.71,"{""location"": ""VI"", ""is_mobile"": true}" 3885,2,85,2017-01-15 13:44:33,http://swaniawskischuster.info/jee.adams,0.5837225407,12.140.162.216,"{""location"": ""RS"", ""is_mobile"": false}" 3886,2,85,2017-03-20 11:29:04,http://murphy.org/benedict.stroman,0.1641855878,74.82.132.170,"{""location"": ""LY"", ""is_mobile"": true}" 3887,2,85,2017-03-19 22:25:51,http://handsanford.biz/mariela.bins,0.9454433415,93.231.202.131,"{""location"": ""KN"", ""is_mobile"": false}" 3888,2,85,2017-02-04 02:15:15,http://hermankovacek.info/heaven,0.4433155936,56.66.21.247,"{""location"": ""PH"", ""is_mobile"": false}" 3889,2,85,2017-02-04 10:55:04,http://greenfelder.biz/marilou,0.7655142543,214.6.214.59,"{""location"": ""CC"", ""is_mobile"": false}" 3890,2,85,2017-02-17 17:39:37,http://macejkovic.net/burnice.rice,0.4680082249,52.74.185.124,"{""location"": ""SO"", ""is_mobile"": false}" 3891,2,85,2017-03-08 04:11:43,http://johnson.name/julio,0.1214440570,220.77.183.36,"{""location"": ""NP"", ""is_mobile"": true}" 3892,2,86,2017-05-30 04:07:01,http://kuhn.net/zula_smitham,,149.127.137.197,"{""location"": ""AL"", ""is_mobile"": false}" 3893,2,86,2017-02-21 12:29:39,http://predovicmurray.org/rylan.rolfson,,86.63.237.103,"{""location"": ""KZ"", ""is_mobile"": true}" 3894,2,86,2017-02-27 04:13:17,http://millskautzer.biz/jolie_parker,,130.18.212.56,"{""location"": ""SA"", ""is_mobile"": false}" 3895,2,86,2017-02-03 17:48:40,http://sanford.com/coralie,,254.144.81.128,"{""location"": ""HT"", ""is_mobile"": false}" 3896,2,86,2017-06-08 11:12:19,http://mayer.name/dianna,,10.159.57.209,"{""location"": ""BY"", ""is_mobile"": false}" 3897,2,86,2017-05-29 23:45:57,http://millerjakubowski.com/lesly.walsh,,206.155.181.227,"{""location"": ""ZW"", ""is_mobile"": false}" 3898,2,86,2017-01-02 15:13:33,http://baumbach.com/jennifer,,180.214.204.12,"{""location"": ""BR"", ""is_mobile"": false}" 3899,2,86,2017-01-31 20:39:20,http://jenkins.io/deie_wuckert,,181.136.101.166,"{""location"": ""YE"", ""is_mobile"": true}" 3900,2,86,2017-04-26 11:51:30,http://connelly.co/chaya.hauck,,40.65.62.242,"{""location"": ""CX"", ""is_mobile"": false}" 3901,2,86,2017-02-12 21:50:54,http://hills.info/krystal.hamill,,84.19.249.202,"{""location"": ""FI"", ""is_mobile"": true}" 3902,2,86,2017-04-26 19:25:34,http://stokes.name/tod,,189.193.170.194,"{""location"": ""VI"", ""is_mobile"": true}" 3903,2,86,2017-06-10 08:30:30,http://bednarpacocha.biz/rahsaan.rodriguez,,50.117.196.84,"{""location"": ""CO"", ""is_mobile"": true}" 3904,2,86,2017-03-28 12:06:53,http://howellkonopelski.net/maurine,,90.22.175.21,"{""location"": ""MS"", ""is_mobile"": false}" 3905,2,86,2017-04-17 15:48:41,http://emmerich.biz/jeika_flatley,,236.165.74.220,"{""location"": ""IQ"", ""is_mobile"": true}" 3906,2,86,2016-12-23 21:08:40,http://dooley.io/tiara_jaskolski,,116.62.56.243,"{""location"": ""LI"", ""is_mobile"": true}" 3907,2,86,2017-04-22 00:27:27,http://balistreri.io/gina,,196.16.28.18,"{""location"": ""UM"", ""is_mobile"": false}" 3908,2,86,2017-01-01 10:45:55,http://gleichnerhowe.io/mallie,,122.41.193.69,"{""location"": ""MG"", ""is_mobile"": true}" 3909,2,86,2017-03-17 17:40:21,http://botsfordgorczany.com/arlo,,34.122.77.249,"{""location"": ""MZ"", ""is_mobile"": false}" 3910,2,86,2017-02-09 01:20:33,http://wiegand.name/kristoffer,,227.147.37.40,"{""location"": ""LY"", ""is_mobile"": false}" 3911,2,86,2017-03-23 11:32:37,http://keeling.net/domingo.white,,196.8.150.24,"{""location"": ""VU"", ""is_mobile"": false}" 3912,2,86,2017-01-02 19:27:56,http://emmerichlehner.com/isabell,,133.184.201.182,"{""location"": ""PH"", ""is_mobile"": true}" 3913,2,86,2017-06-03 17:02:24,http://grant.name/marlen,,123.66.122.150,"{""location"": ""FR"", ""is_mobile"": false}" 3914,2,86,2017-03-08 08:55:10,http://boyle.info/sydnie_schinner,,142.142.194.200,"{""location"": ""MA"", ""is_mobile"": true}" 3915,2,86,2017-02-26 02:28:28,http://donnellykulas.name/theo,,248.69.127.31,"{""location"": ""BY"", ""is_mobile"": false}" 3916,2,86,2017-03-17 11:24:18,http://johns.co/mozelle_emmerich,,55.217.114.183,"{""location"": ""FK"", ""is_mobile"": false}" 3917,2,86,2017-03-13 06:09:01,http://gerlach.net/moses_green,,82.245.199.148,"{""location"": ""SJ"", ""is_mobile"": true}" 3918,2,86,2017-01-01 03:54:35,http://hodkiewiczgorczany.io/tristin.emard,,141.169.193.160,"{""location"": ""NR"", ""is_mobile"": false}" 3919,2,86,2016-12-22 20:22:48,http://braungutmann.com/mariano,,195.209.223.133,"{""location"": ""GD"", ""is_mobile"": true}" 3920,2,86,2017-02-27 02:02:27,http://howegreen.name/beth,,199.18.213.11,"{""location"": ""KZ"", ""is_mobile"": false}" 3921,2,86,2016-12-14 17:08:22,http://stoltenberggleichner.com/reed,,12.71.225.33,"{""location"": ""KY"", ""is_mobile"": false}" 3922,2,86,2017-05-30 23:19:20,http://schuster.name/alf,,198.121.124.151,"{""location"": ""CC"", ""is_mobile"": true}" 3923,2,86,2017-06-02 17:18:02,http://ebert.net/eunice_sporer,,228.24.106.219,"{""location"": ""BI"", ""is_mobile"": false}" 3924,2,86,2017-02-10 10:39:40,http://hills.name/bettie.hagenes,,28.71.88.77,"{""location"": ""LT"", ""is_mobile"": false}" 3925,2,86,2017-04-28 19:51:09,http://mayer.info/kane,,80.198.145.118,"{""location"": ""MW"", ""is_mobile"": false}" 3926,2,86,2016-12-20 09:26:04,http://boehm.co/lyla.dickens,,139.123.63.109,"{""location"": ""AL"", ""is_mobile"": false}" 3927,2,86,2017-05-19 15:46:40,http://pollich.io/mariano_lemke,,63.178.153.23,"{""location"": ""ST"", ""is_mobile"": false}" 3928,2,86,2017-06-14 02:40:47,http://roberts.info/morgan_emard,,184.120.220.219,"{""location"": ""CX"", ""is_mobile"": true}" 3929,2,86,2017-06-13 04:00:51,http://connelly.com/retha,,177.91.253.152,"{""location"": ""TM"", ""is_mobile"": false}" 3930,2,86,2017-05-16 18:47:46,http://lefflerschaefer.info/kyleigh.sanford,,184.208.101.97,"{""location"": ""BE"", ""is_mobile"": true}" 3931,2,86,2017-05-02 22:48:02,http://stroman.info/sunny_torp,,94.162.40.103,"{""location"": ""SE"", ""is_mobile"": true}" 3932,2,86,2017-04-01 02:36:16,http://sawayn.name/samson_bernier,,8.108.212.142,"{""location"": ""RU"", ""is_mobile"": true}" 3933,2,86,2017-03-15 22:02:52,http://hilll.info/demetris,,169.73.106.180,"{""location"": ""LU"", ""is_mobile"": false}" 3934,2,86,2017-01-03 19:34:41,http://stroman.io/edwin_bogan,,37.75.10.8,"{""location"": ""UM"", ""is_mobile"": true}" 3935,2,86,2016-12-14 00:29:28,http://wunsch.org/manuela_franecki,,32.64.187.252,"{""location"": ""AU"", ""is_mobile"": true}" 3936,2,86,2017-01-22 08:24:06,http://larkinblock.com/fay,,253.77.104.48,"{""location"": ""NO"", ""is_mobile"": true}" 3937,2,86,2017-05-11 02:56:41,http://fisher.info/amelia,,235.168.113.227,"{""location"": ""AZ"", ""is_mobile"": true}" 3938,2,86,2017-02-15 21:45:31,http://leffler.name/damaris.weimann,,174.210.226.121,"{""location"": ""NZ"", ""is_mobile"": false}" 3939,2,86,2017-02-07 18:03:36,http://wintheiser.info/roselyn.emmerich,,67.109.69.133,"{""location"": ""NO"", ""is_mobile"": false}" 3940,2,86,2017-03-27 23:03:27,http://doyle.io/jensen_schimmel,,173.212.122.73,"{""location"": ""GH"", ""is_mobile"": true}" 10832,4,242,2017-05-02 07:46:28,http://sipesryan.co/mortimer.klocko,,165.154.119.74,"{""location"": ""SH"", ""is_mobile"": false}" 10833,4,242,2017-03-16 21:37:35,http://cristreinger.com/norris_gulgowski,,200.86.75.29,"{""location"": ""TJ"", ""is_mobile"": true}" 10834,4,242,2017-01-31 12:50:56,http://schiller.co/buddy,,138.4.85.237,"{""location"": ""AM"", ""is_mobile"": false}" 10835,4,242,2016-12-17 16:01:56,http://greenfelderbarton.biz/camren.wintheiser,,37.211.161.67,"{""location"": ""LK"", ""is_mobile"": false}" 10836,4,242,2017-06-08 01:44:43,http://wisoky.name/deshaun.jones,,124.231.249.197,"{""location"": ""BN"", ""is_mobile"": false}" 10837,4,242,2017-01-23 14:06:46,http://collins.name/chester,,177.240.248.243,"{""location"": ""HU"", ""is_mobile"": false}" 10838,4,242,2017-01-29 19:22:56,http://kemmer.name/dandre,,204.69.46.138,"{""location"": ""UA"", ""is_mobile"": true}" 10839,4,242,2017-01-07 07:32:57,http://turnermcdermott.com/rafaela.thiel,,144.141.213.141,"{""location"": ""IT"", ""is_mobile"": false}" 10840,4,242,2017-02-20 19:38:49,http://marquardtfadel.net/lance.hodkiewicz,,138.175.254.207,"{""location"": ""ES"", ""is_mobile"": false}" 10841,4,242,2017-02-16 06:09:47,http://gusikowskifeil.org/lilian_zieme,,106.186.5.106,"{""location"": ""TT"", ""is_mobile"": false}" 10842,4,242,2017-02-28 11:22:40,http://lockmanmoen.biz/maurice,,148.201.68.103,"{""location"": ""TJ"", ""is_mobile"": true}" 10843,4,242,2017-05-20 20:57:02,http://rowenicolas.io/jena_herzog,,74.102.174.118,"{""location"": ""WS"", ""is_mobile"": false}" 10844,4,242,2017-06-08 22:04:25,http://lakin.info/sam,,78.63.238.183,"{""location"": ""BT"", ""is_mobile"": true}" 10845,4,242,2017-01-06 10:40:29,http://kaulkekris.info/axel_wolff,,108.92.185.91,"{""location"": ""NI"", ""is_mobile"": false}" 10846,4,242,2017-01-18 05:31:51,http://okeefemaggio.com/madaline.hintz,,72.134.178.132,"{""location"": ""PE"", ""is_mobile"": false}" 10847,4,242,2017-05-01 22:57:51,http://aufderhar.com/gilbert_goldner,,189.81.147.78,"{""location"": ""AT"", ""is_mobile"": true}" 10848,4,242,2017-03-04 02:11:48,http://mayer.co/carmel_littel,,230.188.100.36,"{""location"": ""ZM"", ""is_mobile"": true}" 10849,4,242,2017-02-09 19:30:55,http://dibbert.biz/adriel_skiles,,114.91.91.118,"{""location"": ""TN"", ""is_mobile"": true}" 10850,4,242,2017-06-09 18:44:40,http://effertzlegros.info/ashlee_pouros,,26.9.69.8,"{""location"": ""AR"", ""is_mobile"": true}" 10851,4,242,2017-05-01 19:29:17,http://jenkinsaltenwerth.com/murl.olson,,206.199.17.178,"{""location"": ""KH"", ""is_mobile"": true}" 10852,4,242,2017-03-16 10:13:50,http://manterunte.io/leonie,,165.170.225.4,"{""location"": ""TZ"", ""is_mobile"": true}" 10853,4,242,2017-05-12 14:35:06,http://heaney.biz/ena,,125.149.171.213,"{""location"": ""LK"", ""is_mobile"": true}" 10854,4,242,2017-02-27 05:13:02,http://bruenruecker.net/jabari,,165.197.186.196,"{""location"": ""PA"", ""is_mobile"": true}" 10855,4,242,2017-04-17 20:29:37,http://shanahanlarson.org/roosevelt.langworth,,82.145.177.227,"{""location"": ""NL"", ""is_mobile"": false}" 10856,4,242,2017-02-06 22:42:11,http://swaniawski.info/ed,,220.31.18.130,"{""location"": ""IM"", ""is_mobile"": true}" 10857,4,242,2016-12-15 11:41:59,http://heidenreichbartoletti.info/viva,,43.69.7.73,"{""location"": ""RO"", ""is_mobile"": true}" 10858,4,242,2017-03-28 22:51:23,http://dicki.com/samson.abernathy,,223.22.182.16,"{""location"": ""TL"", ""is_mobile"": false}" 10859,4,243,2017-04-10 22:40:03,http://macgyverkovacek.net/brown,,133.135.175.185,"{""location"": ""GW"", ""is_mobile"": true}" 10860,4,243,2017-03-23 05:43:21,http://schadenstamm.io/juanita,,4.129.251.240,"{""location"": ""PA"", ""is_mobile"": false}" 10861,4,243,2017-01-26 15:23:40,http://mcdermottbotsford.biz/rickie.oreilly,,240.153.202.130,"{""location"": ""IL"", ""is_mobile"": false}" 10862,4,243,2017-02-12 19:12:19,http://romaguera.net/lenora_krajcik,,91.58.92.252,"{""location"": ""CR"", ""is_mobile"": true}" 10863,4,243,2017-01-05 11:49:04,http://stehrhaley.io/reggie,,29.168.68.167,"{""location"": ""US"", ""is_mobile"": false}" 10864,4,243,2017-03-29 08:21:43,http://veum.org/vicenta,,64.206.94.126,"{""location"": ""GS"", ""is_mobile"": true}" 10865,4,243,2017-03-02 02:09:00,http://mohrbecker.com/taryn_leuschke,,188.252.19.78,"{""location"": ""GT"", ""is_mobile"": true}" 10866,4,243,2017-06-01 02:12:09,http://ebert.net/johnnie,,115.239.254.59,"{""location"": ""LI"", ""is_mobile"": false}" 10867,4,243,2017-01-31 08:14:11,http://farrell.biz/marquis.mckenzie,,127.224.129.237,"{""location"": ""LV"", ""is_mobile"": true}" 10868,4,243,2017-05-11 23:42:45,http://ruel.io/molly.block,,198.177.190.44,"{""location"": ""FM"", ""is_mobile"": true}" 10869,4,243,2017-03-19 03:22:36,http://weimann.co/eileen_feil,,243.199.124.241,"{""location"": ""LY"", ""is_mobile"": false}" 10870,4,243,2016-12-23 07:08:58,http://robertshuels.com/stuart,,218.253.92.200,"{""location"": ""IT"", ""is_mobile"": true}" 10871,4,243,2017-01-28 00:19:23,http://dietrich.io/alejandra_reichel,,66.166.34.156,"{""location"": ""HK"", ""is_mobile"": true}" 10872,4,243,2017-02-24 18:47:29,http://schimmel.info/austyn,,210.225.221.75,"{""location"": ""FK"", ""is_mobile"": false}" 10873,4,243,2017-02-26 05:03:27,http://millerrogahn.name/orland,,2.77.250.43,"{""location"": ""ER"", ""is_mobile"": false}" 10874,4,243,2017-02-18 01:58:53,http://ratkedurgan.com/rubye,,39.76.194.52,"{""location"": ""LA"", ""is_mobile"": false}" 10875,4,243,2017-04-21 01:01:51,http://moen.net/karen.dibbert,,15.59.233.16,"{""location"": ""BO"", ""is_mobile"": true}" 10876,4,243,2017-04-02 18:23:42,http://jaskolski.info/oda.keler,,117.236.60.114,"{""location"": ""RS"", ""is_mobile"": true}" 10877,4,243,2016-12-23 04:21:50,http://jastgrimes.name/steve,,229.94.227.171,"{""location"": ""SA"", ""is_mobile"": true}" 10878,4,243,2017-06-11 16:58:44,http://kub.co/jace,,10.249.220.31,"{""location"": ""VG"", ""is_mobile"": true}" 10879,4,243,2017-05-09 22:34:57,http://denesikwalsh.net/orie,,241.140.57.90,"{""location"": ""FO"", ""is_mobile"": true}" 10880,4,243,2017-01-24 12:14:05,http://ledner.io/marley.eichmann,,239.80.82.52,"{""location"": ""AW"", ""is_mobile"": false}" 10881,4,243,2017-06-01 14:28:55,http://howell.info/kaitlin.mclaughlin,,218.185.106.178,"{""location"": ""BF"", ""is_mobile"": true}" 10882,4,243,2017-01-30 15:39:48,http://block.biz/ella.kertzmann,,14.90.66.224,"{""location"": ""BL"", ""is_mobile"": false}" 10883,4,243,2017-03-11 14:53:37,http://spinka.info/daphnee.bauch,,50.178.140.233,"{""location"": ""NA"", ""is_mobile"": true}" 10884,4,243,2017-04-22 10:20:58,http://wolf.name/emmalee,,150.216.10.213,"{""location"": ""BQ"", ""is_mobile"": false}" 10885,4,243,2016-12-27 09:55:28,http://breitenbergflatley.io/aric_olson,,112.232.27.191,"{""location"": ""GW"", ""is_mobile"": false}" 7862,3,174,2017-04-22 18:35:35,http://littel.com/reinhold,0.9166469728,57.22.38.55,"{""location"": ""CN"", ""is_mobile"": false}" 7863,3,174,2017-06-05 01:53:52,http://hamill.io/destin,0.5139975903,206.98.217.31,"{""location"": ""TL"", ""is_mobile"": true}" 7864,3,174,2017-05-13 00:33:09,http://bechtelar.biz/armand,0.4308718317,52.192.215.239,"{""location"": ""KZ"", ""is_mobile"": false}" 7865,3,174,2017-01-16 22:16:39,http://botsfordboehm.net/sylvan,0.7321272876,20.30.121.71,"{""location"": ""BS"", ""is_mobile"": true}" 7866,3,174,2017-02-09 23:59:39,http://trantow.biz/kylie,0.8402582197,114.31.219.127,"{""location"": ""KW"", ""is_mobile"": false}" 7867,3,174,2017-04-24 10:07:29,http://pagac.co/blanca,0.1732603353,25.119.69.229,"{""location"": ""MW"", ""is_mobile"": true}" 7868,3,174,2017-06-04 05:13:47,http://dare.io/wade_conn,0.7658330798,191.117.74.198,"{""location"": ""IQ"", ""is_mobile"": false}" 7869,3,174,2017-04-07 12:04:54,http://moriette.com/moses,0.9340995287,167.173.60.84,"{""location"": ""AD"", ""is_mobile"": false}" 7870,3,174,2017-03-24 15:08:33,http://lakin.io/brent,0.1908519741,224.247.115.57,"{""location"": ""MX"", ""is_mobile"": true}" 7871,3,174,2017-05-10 04:39:13,http://mcculloughgerhold.info/iliana.becker,0.9521869380,33.44.42.254,"{""location"": ""CI"", ""is_mobile"": true}" 7872,3,174,2016-12-31 00:40:09,http://dietrich.info/moie,0.0605965968,206.27.161.125,"{""location"": ""KI"", ""is_mobile"": false}" 7873,3,174,2017-03-13 02:09:50,http://crookskunze.org/rebeka_rippin,0.9697851546,129.46.156.83,"{""location"": ""SH"", ""is_mobile"": false}" 7874,3,174,2017-03-18 07:10:50,http://oconnellgusikowski.io/brenden.weinat,0.6075556957,72.6.100.45,"{""location"": ""SL"", ""is_mobile"": false}" 7875,3,174,2017-01-04 20:14:06,http://simonis.net/andreanne_waelchi,0.1809189700,125.65.185.210,"{""location"": ""SC"", ""is_mobile"": true}" 7876,3,174,2017-03-17 19:57:39,http://fritsch.biz/hudson_mosciski,0.7601568441,76.223.23.60,"{""location"": ""DJ"", ""is_mobile"": false}" 7877,3,174,2017-06-05 17:42:32,http://barrowchneider.io/granville_auer,0.4149225709,36.92.82.160,"{""location"": ""SX"", ""is_mobile"": true}" 7878,3,174,2017-04-17 20:44:43,http://stehrmckenzie.co/liliane.vandervort,0.7229513675,72.237.185.132,"{""location"": ""CW"", ""is_mobile"": false}" 7879,3,174,2017-02-15 02:56:22,http://weinat.co/donny,0.6759521361,39.240.20.41,"{""location"": ""EC"", ""is_mobile"": false}" 7880,3,174,2017-01-04 13:10:59,http://larsonhills.name/zoey,0.5781600811,231.219.100.128,"{""location"": ""NU"", ""is_mobile"": false}" 7881,3,174,2017-04-12 18:01:31,http://cain.co/arjun,0.8445436672,11.199.147.102,"{""location"": ""MK"", ""is_mobile"": false}" 7882,3,174,2017-01-17 10:00:28,http://olson.org/casandra_mueller,0.5241645541,185.6.81.18,"{""location"": ""NI"", ""is_mobile"": true}" 7883,3,174,2017-03-02 08:11:13,http://reinger.org/abigale,0.2680718869,51.84.12.35,"{""location"": ""KI"", ""is_mobile"": false}" 7884,3,174,2017-05-16 19:56:15,http://lynch.co/eryn,0.3442996792,153.190.139.140,"{""location"": ""IT"", ""is_mobile"": true}" 7885,3,174,2017-06-01 03:21:30,http://johns.name/jayden,0.6608596201,151.3.164.170,"{""location"": ""ES"", ""is_mobile"": false}" 7886,3,174,2017-05-22 06:13:17,http://gulgowski.biz/kaci,0.9235670678,149.102.34.127,"{""location"": ""BM"", ""is_mobile"": false}" 7887,3,174,2017-02-10 07:59:28,http://keler.org/jazmyn.reynolds,0.5639187562,238.102.115.248,"{""location"": ""AQ"", ""is_mobile"": false}" 7888,3,174,2017-03-09 12:57:38,http://barton.name/mattie_maggio,0.9321865052,92.69.212.216,"{""location"": ""NO"", ""is_mobile"": false}" 7889,3,174,2017-01-06 23:51:00,http://sporer.info/easter_olson,0.9161850057,228.240.3.142,"{""location"": ""JM"", ""is_mobile"": false}" 7890,3,174,2017-04-22 21:31:07,http://yost.org/ericka.mayert,0.8149270274,93.97.116.144,"{""location"": ""CZ"", ""is_mobile"": false}" 7891,3,174,2017-03-17 16:02:12,http://yundtvolkman.com/amiya,0.3288158562,129.123.94.198,"{""location"": ""CZ"", ""is_mobile"": true}" 7892,3,174,2017-02-08 00:21:57,http://padbergdavis.biz/durward.schroeder,0.9779064640,132.17.122.242,"{""location"": ""TV"", ""is_mobile"": true}" 7893,3,174,2017-06-10 22:25:53,http://bechtelar.co/johnson_haley,0.8828865510,125.87.4.2,"{""location"": ""BN"", ""is_mobile"": false}" 7894,3,174,2017-05-21 11:21:18,http://jacobszboncak.com/luisa_cummerata,0.8589780302,210.20.59.96,"{""location"": ""GS"", ""is_mobile"": true}" 7895,3,174,2017-02-26 20:15:42,http://borerkulas.name/durward.greenholt,0.6696621739,58.222.99.153,"{""location"": ""MK"", ""is_mobile"": true}" 7896,3,174,2017-04-02 04:02:16,http://greenholtkeebler.net/william,0.2188867782,49.48.81.211,"{""location"": ""VE"", ""is_mobile"": true}" 7897,3,174,2017-06-07 20:42:10,http://bernhard.net/beatrice_hahn,0.7196042458,36.98.105.9,"{""location"": ""SZ"", ""is_mobile"": false}" 7898,3,174,2017-05-23 18:35:31,http://gutmann.com/hazel,0.7550118031,200.241.99.129,"{""location"": ""AQ"", ""is_mobile"": true}" 7899,3,174,2017-05-24 21:49:11,http://braun.info/ruth.windler,0.1194146827,63.97.152.157,"{""location"": ""CY"", ""is_mobile"": false}" 7900,3,174,2017-02-20 00:48:51,http://schmidthudson.biz/wyman.heaney,0.7563418325,197.141.244.39,"{""location"": ""TD"", ""is_mobile"": true}" 7901,3,174,2017-01-21 09:31:10,http://leschleuschke.net/jeika.torp,0.1410776356,152.127.24.184,"{""location"": ""WF"", ""is_mobile"": true}" 7902,3,174,2017-01-29 03:51:16,http://bauchanderson.io/alexane.jast,0.0836223956,31.74.177.59,"{""location"": ""TN"", ""is_mobile"": false}" 7903,3,174,2017-05-17 05:26:17,http://prohaska.net/aylin,0.2025571370,202.253.165.209,"{""location"": ""FR"", ""is_mobile"": false}" 7904,3,174,2017-02-21 06:28:03,http://vandervort.org/alanis,0.8775173629,166.17.171.249,"{""location"": ""KP"", ""is_mobile"": false}" 7905,3,174,2017-02-13 00:36:11,http://beattygreen.org/christina,0.9563345548,102.165.93.56,"{""location"": ""FJ"", ""is_mobile"": true}" 7906,3,174,2017-06-06 18:03:56,http://swift.name/marcel.reichert,0.2314760320,145.216.242.12,"{""location"": ""DM"", ""is_mobile"": false}" 7907,3,174,2017-04-30 22:06:40,http://leuschkewitting.com/michael.mayert,0.9766764608,33.6.160.254,"{""location"": ""TG"", ""is_mobile"": false}" 7908,3,174,2016-12-15 00:15:24,http://cruickshank.biz/flo_hane,0.7444315740,22.198.131.82,"{""location"": ""SR"", ""is_mobile"": false}" 7909,3,174,2017-01-10 11:03:11,http://williamson.io/velva,0.5210392107,150.106.166.66,"{""location"": ""PS"", ""is_mobile"": false}" 7910,3,174,2017-02-14 22:51:23,http://powlowskikihn.net/nola,0.2023901062,194.242.117.85,"{""location"": ""IO"", ""is_mobile"": true}" 7911,3,174,2017-05-23 09:00:34,http://rohan.name/mayra_dubuque,0.2161409555,80.52.237.78,"{""location"": ""CW"", ""is_mobile"": false}" 7912,3,175,2017-04-20 02:00:36,http://medhurst.com/iliana,0.9698854914,50.139.184.84,"{""location"": ""SC"", ""is_mobile"": true}" 13781,5,310,2017-03-17 20:55:10,http://stracke.name/elenor_corwin,,99.106.25.162,"{""location"": ""TK"", ""is_mobile"": false}" 13782,5,310,2016-12-18 16:28:40,http://hettinger.biz/vaughn,,208.73.247.220,"{""location"": ""CC"", ""is_mobile"": false}" 13783,5,310,2016-12-16 22:29:35,http://harveytrantow.org/ricardo.batz,,227.213.134.43,"{""location"": ""RE"", ""is_mobile"": true}" 13784,5,310,2017-02-19 17:53:08,http://bernhard.net/daisha.leannon,,166.20.145.10,"{""location"": ""PN"", ""is_mobile"": false}" 13785,5,310,2017-02-04 23:09:18,http://harristhiel.io/pansy,,252.101.56.170,"{""location"": ""ES"", ""is_mobile"": true}" 13786,5,310,2017-03-26 03:01:48,http://smith.io/jeika,,135.31.161.32,"{""location"": ""LR"", ""is_mobile"": true}" 13787,5,310,2017-02-13 11:22:55,http://kaulkegreen.net/rosendo,,217.93.35.231,"{""location"": ""PH"", ""is_mobile"": true}" 13788,5,310,2017-01-20 10:49:06,http://thiel.biz/armani,,156.217.245.187,"{""location"": ""MG"", ""is_mobile"": false}" 13789,5,310,2017-03-31 03:08:57,http://yundt.io/daphney,,153.79.56.48,"{""location"": ""MD"", ""is_mobile"": false}" 13790,5,310,2016-12-29 19:16:40,http://daremurray.io/hector,,79.35.93.135,"{""location"": ""AI"", ""is_mobile"": true}" 13791,5,310,2017-03-01 22:10:06,http://senger.co/myles,,11.161.42.201,"{""location"": ""FO"", ""is_mobile"": false}" 13792,5,310,2017-01-13 12:54:18,http://handjaskolski.net/ignacio_cremin,,42.155.137.230,"{""location"": ""NE"", ""is_mobile"": false}" 13793,5,310,2017-05-22 05:56:13,http://bosco.net/tomas_kuvalis,,62.225.131.172,"{""location"": ""PY"", ""is_mobile"": false}" 13794,5,310,2017-02-01 23:06:55,http://simonishartmann.name/marques_pfeffer,,107.165.97.64,"{""location"": ""KW"", ""is_mobile"": false}" 13795,5,310,2017-04-11 15:35:58,http://wuckert.name/grover,,30.52.32.175,"{""location"": ""CU"", ""is_mobile"": true}" 13796,5,310,2016-12-23 06:34:17,http://feil.co/vivianne_murray,,27.203.185.207,"{""location"": ""MG"", ""is_mobile"": false}" 13797,5,310,2017-03-27 18:13:07,http://macejkovic.biz/ashton,,227.101.43.131,"{""location"": ""BT"", ""is_mobile"": true}" 13798,5,310,2017-03-02 16:07:46,http://murazik.com/arielle.welch,,27.74.84.75,"{""location"": ""TO"", ""is_mobile"": true}" 13799,5,310,2017-05-28 00:14:59,http://larson.info/judd,,229.71.177.61,"{""location"": ""DE"", ""is_mobile"": false}" 13800,5,310,2017-03-08 14:08:15,http://stanton.io/dayna,,113.151.61.191,"{""location"": ""CX"", ""is_mobile"": false}" 13801,5,310,2017-01-24 01:59:38,http://stokechuppe.io/anthony,,31.113.176.92,"{""location"": ""VC"", ""is_mobile"": true}" 13802,5,310,2016-12-13 06:45:23,http://reichel.co/jane,,190.143.92.236,"{""location"": ""ZM"", ""is_mobile"": true}" 13803,5,310,2017-02-19 03:32:30,http://ortiz.org/jamal,,30.51.233.210,"{""location"": ""GR"", ""is_mobile"": false}" 13804,5,310,2017-04-01 11:17:16,http://oconnell.io/meagan,,168.135.195.8,"{""location"": ""KN"", ""is_mobile"": true}" 13805,5,310,2017-06-13 17:32:12,http://thompson.net/kathryn_leannon,,45.31.8.15,"{""location"": ""BQ"", ""is_mobile"": false}" 13806,5,310,2017-02-10 16:12:02,http://jacobs.net/nora.weimann,,133.234.249.184,"{""location"": ""WF"", ""is_mobile"": true}" 13807,5,310,2017-01-12 20:35:54,http://gislasonjohnson.info/stephen.beer,,45.56.150.52,"{""location"": ""ZM"", ""is_mobile"": true}" 13808,5,310,2016-12-18 08:12:58,http://okeefe.org/malinda.torphy,,91.5.242.158,"{""location"": ""LI"", ""is_mobile"": false}" 13809,5,310,2016-12-22 18:42:44,http://oberbrunnerbayer.com/maxine,,183.109.147.219,"{""location"": ""NI"", ""is_mobile"": false}" 13810,5,310,2017-06-13 15:06:36,http://welch.com/izabella_ferry,,220.78.217.47,"{""location"": ""GH"", ""is_mobile"": false}" 13811,5,310,2017-06-13 17:56:54,http://leannon.info/giovani,,69.201.100.81,"{""location"": ""CK"", ""is_mobile"": false}" 13812,5,310,2016-12-31 08:30:12,http://barton.name/christelle_gutkowski,,33.223.11.250,"{""location"": ""AQ"", ""is_mobile"": false}" 13813,5,310,2017-02-27 17:36:07,http://lindtorp.co/deja,,42.247.206.5,"{""location"": ""PH"", ""is_mobile"": false}" 13814,5,310,2017-03-06 18:16:52,http://ohara.biz/tatyana,,13.230.151.60,"{""location"": ""IQ"", ""is_mobile"": true}" 13815,5,310,2017-01-05 19:35:13,http://heaneytremblay.name/koby_macejkovic,,52.169.55.212,"{""location"": ""SV"", ""is_mobile"": false}" 13816,5,310,2017-01-02 20:14:36,http://pouros.net/ladarius,,117.233.240.135,"{""location"": ""KE"", ""is_mobile"": false}" 13817,5,310,2017-04-10 07:10:35,http://johnsonschuster.name/layne.nikolaus,,136.22.110.68,"{""location"": ""NF"", ""is_mobile"": true}" 13818,5,310,2017-06-08 01:02:28,http://hammes.info/willy.gislason,,141.142.252.93,"{""location"": ""JP"", ""is_mobile"": true}" 13819,5,310,2016-12-21 14:02:05,http://heel.net/regan,,243.51.90.144,"{""location"": ""IQ"", ""is_mobile"": false}" 13820,5,310,2017-03-26 22:34:15,http://hermistonrobel.net/claudine.weinat,,185.228.193.157,"{""location"": ""GQ"", ""is_mobile"": false}" 13821,5,310,2017-01-10 08:15:13,http://marvin.biz/virgie,,207.14.146.142,"{""location"": ""IQ"", ""is_mobile"": false}" 13822,5,310,2017-01-09 20:38:28,http://gaylordschuster.biz/wilmer,,104.191.182.169,"{""location"": ""KE"", ""is_mobile"": true}" 13823,5,310,2017-06-08 17:51:45,http://reichert.io/eleanora,,135.187.203.38,"{""location"": ""ST"", ""is_mobile"": true}" 13824,5,310,2017-01-23 07:47:58,http://feeneycrist.info/monserrate.kemmer,,57.106.184.8,"{""location"": ""CG"", ""is_mobile"": true}" 13825,5,310,2017-05-31 04:36:25,http://senger.io/birdie_schmeler,,181.170.134.26,"{""location"": ""VE"", ""is_mobile"": true}" 13826,5,310,2017-01-26 04:16:19,http://rosenbaum.name/concepcion_gislason,,168.148.74.253,"{""location"": ""ET"", ""is_mobile"": true}" 13827,5,310,2017-04-24 03:53:58,http://hicklegoodwin.io/anderson_cummerata,,9.116.5.205,"{""location"": ""BI"", ""is_mobile"": false}" 13828,5,310,2017-06-11 20:04:57,http://hegmann.info/newton_reynolds,,209.220.26.251,"{""location"": ""YE"", ""is_mobile"": true}" 13829,5,310,2016-12-25 14:16:15,http://powlowski.biz/emelia.koelpin,,80.44.152.62,"{""location"": ""PL"", ""is_mobile"": true}" 13830,5,310,2017-01-17 16:58:46,http://moen.io/dorris,,197.113.253.165,"{""location"": ""SX"", ""is_mobile"": false}" 13831,5,310,2016-12-28 20:08:31,http://mclaughlin.net/bettie,,148.6.188.228,"{""location"": ""AS"", ""is_mobile"": false}" 13832,5,310,2017-04-18 13:14:32,http://daugherty.biz/marshall,,9.145.53.59,"{""location"": ""GR"", ""is_mobile"": true}" 13833,5,310,2017-05-23 15:10:18,http://harveysmitham.org/maeve.heel,,147.10.64.238,"{""location"": ""TF"", ""is_mobile"": true}" 13834,5,310,2017-03-11 12:31:59,http://hilll.net/peyton,,102.219.180.75,"{""location"": ""SV"", ""is_mobile"": true}" 13835,5,310,2017-05-26 19:29:25,http://klocko.com/ashley.padberg,,191.234.224.32,"{""location"": ""PL"", ""is_mobile"": false}" 16765,6,380,2017-03-01 20:48:00,http://mitchell.name/lelia.thiel,,142.43.245.150,"{""location"": ""IN"", ""is_mobile"": true}" 16766,6,380,2017-06-11 05:13:15,http://herzog.net/nayeli,,207.207.157.229,"{""location"": ""MV"", ""is_mobile"": true}" 16767,6,380,2017-04-16 10:46:24,http://dicki.org/jayda.fisher,,36.69.59.76,"{""location"": ""NP"", ""is_mobile"": true}" 16768,6,380,2017-03-26 07:25:49,http://fisherkuhlman.co/eleazar.rohan,,71.63.233.177,"{""location"": ""GB"", ""is_mobile"": true}" 16769,6,380,2017-02-20 19:38:53,http://rice.net/golda_klein,,214.49.204.120,"{""location"": ""BT"", ""is_mobile"": false}" 16770,6,380,2016-12-17 07:23:15,http://gottlieb.co/alfred,,100.69.12.69,"{""location"": ""PS"", ""is_mobile"": false}" 16771,6,380,2017-04-20 01:08:03,http://hackettmuller.org/francisca.mraz,,87.234.227.103,"{""location"": ""MC"", ""is_mobile"": false}" 16772,6,380,2017-02-02 20:02:35,http://kulas.org/garnet_toy,,73.137.177.250,"{""location"": ""FI"", ""is_mobile"": true}" 16773,6,380,2017-04-02 20:06:04,http://hilll.net/dante,,145.4.254.99,"{""location"": ""NO"", ""is_mobile"": false}" 16774,6,380,2017-04-01 11:48:55,http://ohara.com/nicolette.beer,,133.74.230.134,"{""location"": ""RO"", ""is_mobile"": false}" 16775,6,380,2017-02-18 07:49:10,http://hegmann.name/nels.ritchie,,238.198.118.16,"{""location"": ""DZ"", ""is_mobile"": false}" 16776,6,380,2017-04-16 02:59:06,http://goldner.com/janea.botsford,,48.132.39.166,"{""location"": ""VN"", ""is_mobile"": false}" 16777,6,380,2017-05-01 17:16:13,http://denesik.io/luella_koepp,,165.173.131.204,"{""location"": ""BO"", ""is_mobile"": true}" 16778,6,380,2017-02-01 01:32:59,http://balistreri.org/taurean,,129.132.40.152,"{""location"": ""SZ"", ""is_mobile"": true}" 16779,6,380,2017-03-10 02:51:41,http://graham.name/theresa,,162.125.42.177,"{""location"": ""AW"", ""is_mobile"": true}" 16780,6,380,2017-05-16 12:51:58,http://waters.net/jammie.shields,,12.22.82.105,"{""location"": ""JO"", ""is_mobile"": true}" 16781,6,380,2017-03-17 03:53:07,http://kunde.io/freida,,142.100.100.125,"{""location"": ""LU"", ""is_mobile"": true}" 16782,6,380,2017-01-21 14:35:12,http://denesik.io/mellie_abshire,,115.192.30.111,"{""location"": ""SD"", ""is_mobile"": true}" 16783,6,380,2016-12-26 14:41:58,http://wisoky.name/may.prosacco,,18.91.129.204,"{""location"": ""IL"", ""is_mobile"": true}" 16784,6,380,2016-12-25 04:32:06,http://reingerbalistreri.net/madge,,123.150.112.102,"{""location"": ""TJ"", ""is_mobile"": false}" 16785,6,380,2017-04-20 09:50:12,http://jerdefay.io/fredrick_prosacco,,168.23.2.26,"{""location"": ""CU"", ""is_mobile"": false}" 17073,6,386,2017-03-03 00:56:19,http://roberts.org/te,,9.248.14.79,"{""location"": ""IL"", ""is_mobile"": false}" 16786,6,380,2017-02-15 15:54:10,http://kuhlman.co/hanna_romaguera,,203.215.193.171,"{""location"": ""LV"", ""is_mobile"": true}" 16787,6,380,2017-03-08 04:45:59,http://harrisgrady.info/montana_pagac,,78.176.22.238,"{""location"": ""KI"", ""is_mobile"": false}" 16788,6,380,2017-04-12 17:16:11,http://glover.io/pattie,,91.48.9.167,"{""location"": ""VE"", ""is_mobile"": true}" 16789,6,380,2017-02-27 07:09:44,http://raynorweimann.co/rasheed,,87.119.42.111,"{""location"": ""DK"", ""is_mobile"": true}" 16790,6,380,2017-03-13 03:46:19,http://adamsmoriette.net/ignacio_abshire,,51.85.223.41,"{""location"": ""MY"", ""is_mobile"": false}" 16791,6,380,2017-06-09 13:31:09,http://schimmel.name/waino.grady,,4.169.135.232,"{""location"": ""KI"", ""is_mobile"": true}" 16792,6,380,2017-03-05 09:20:43,http://stromanwalter.info/rolando,,131.177.37.136,"{""location"": ""MR"", ""is_mobile"": false}" 16793,6,380,2017-03-05 20:49:20,http://monahanaufderhar.net/ana.feeney,,183.135.184.72,"{""location"": ""CN"", ""is_mobile"": false}" 16794,6,380,2017-04-29 03:00:06,http://zboncak.net/vinnie_stoltenberg,,170.164.228.186,"{""location"": ""TD"", ""is_mobile"": true}" 16795,6,380,2017-04-06 16:59:33,http://sawayn.info/madelyn.metz,,101.162.98.37,"{""location"": ""MA"", ""is_mobile"": true}" 16796,6,380,2017-03-22 16:37:23,http://stiedemann.name/magali.effertz,,88.240.243.222,"{""location"": ""SY"", ""is_mobile"": false}" 16797,6,380,2017-01-29 04:14:28,http://danielstiedemann.info/parker.schimmel,,53.78.87.171,"{""location"": ""RW"", ""is_mobile"": false}" 16798,6,380,2017-01-15 03:33:49,http://darechamplin.com/verner,,100.52.56.252,"{""location"": ""BV"", ""is_mobile"": true}" 16799,6,380,2017-06-06 19:10:03,http://donnellyoberbrunner.biz/rudy.wiza,,91.220.75.99,"{""location"": ""MP"", ""is_mobile"": false}" 16800,6,380,2016-12-27 13:16:54,http://sawaynbahringer.org/donald_ortiz,,27.197.133.3,"{""location"": ""MU"", ""is_mobile"": true}" 16801,6,380,2017-05-10 12:52:39,http://kiehnwelch.name/samir,,170.179.9.21,"{""location"": ""ES"", ""is_mobile"": false}" 16802,6,380,2017-03-11 13:15:07,http://towneschimmel.io/maryjane_littel,,189.31.231.213,"{""location"": ""MY"", ""is_mobile"": true}" 16803,6,380,2017-05-06 23:19:17,http://white.io/jaquan.anderson,,161.36.66.187,"{""location"": ""UY"", ""is_mobile"": false}" 16804,6,380,2017-06-01 22:01:10,http://hahn.name/sidney_zieme,,151.99.126.86,"{""location"": ""GA"", ""is_mobile"": true}" 16805,6,380,2017-06-10 15:57:10,http://bogisich.io/laurel_kerluke,,116.193.243.128,"{""location"": ""SR"", ""is_mobile"": false}" 16806,6,380,2017-05-23 22:31:45,http://hegmann.name/xavier,,25.212.195.241,"{""location"": ""UZ"", ""is_mobile"": true}" 16807,6,381,2017-01-07 01:53:55,http://kulasparker.org/linda.okon,,228.209.168.208,"{""location"": ""IS"", ""is_mobile"": true}" 16808,6,381,2016-12-15 06:43:58,http://stanton.co/dedrick.feeney,,33.222.79.128,"{""location"": ""LA"", ""is_mobile"": true}" 16809,6,381,2016-12-14 01:16:55,http://rennerankunding.info/duncan,,56.141.152.134,"{""location"": ""MX"", ""is_mobile"": false}" 16810,6,381,2017-03-31 06:53:31,http://kaulke.net/dillan,,139.122.93.161,"{""location"": ""IN"", ""is_mobile"": false}" 16811,6,381,2017-04-22 02:48:06,http://hamilldouglas.name/rhianna,,177.132.180.80,"{""location"": ""GQ"", ""is_mobile"": false}" 16812,6,381,2017-04-29 11:08:03,http://keler.io/tomas.schuppe,,4.42.93.217,"{""location"": ""BM"", ""is_mobile"": false}" 16813,6,381,2017-02-04 21:10:46,http://lueilwitz.name/virginie,,170.183.60.26,"{""location"": ""MP"", ""is_mobile"": false}" 16814,6,381,2017-02-13 18:40:41,http://gutmann.io/julianne,,93.182.149.77,"{""location"": ""KP"", ""is_mobile"": true}" 16815,6,381,2016-12-26 10:33:29,http://schultzreilly.biz/allie,,192.236.49.80,"{""location"": ""NC"", ""is_mobile"": false}" 16816,6,381,2017-05-01 01:49:01,http://stroman.info/annalise,,239.201.224.26,"{""location"": ""SC"", ""is_mobile"": false}" 16817,6,381,2017-04-15 04:48:09,http://tremblaygaylord.io/randal_stoltenberg,,3.92.37.111,"{""location"": ""EG"", ""is_mobile"": true}" 16818,6,381,2017-01-04 10:00:21,http://hartmann.biz/katharina,,7.170.141.152,"{""location"": ""SJ"", ""is_mobile"": false}" 3941,2,87,2017-01-12 09:02:41,http://littel.name/fabiola.mcclure,,113.149.212.41,"{""location"": ""GS"", ""is_mobile"": true}" 3942,2,87,2017-04-09 15:55:30,http://hoeger.biz/ruth.torp,,166.190.229.63,"{""location"": ""SG"", ""is_mobile"": false}" 3943,2,87,2017-02-13 22:26:20,http://marvin.com/ken,,153.117.184.67,"{""location"": ""TT"", ""is_mobile"": false}" 3944,2,87,2017-04-01 09:03:12,http://hilll.name/montana_cole,,116.111.186.184,"{""location"": ""SX"", ""is_mobile"": true}" 3945,2,87,2017-05-12 06:21:37,http://hermiston.net/mozell,,118.30.111.171,"{""location"": ""SN"", ""is_mobile"": true}" 3947,2,87,2017-02-24 17:51:54,http://herzog.com/roger.nader,,254.168.211.56,"{""location"": ""AS"", ""is_mobile"": true}" 3948,2,87,2017-03-13 08:20:04,http://gloverjaskolski.biz/fidel.kulas,,30.200.85.179,"{""location"": ""LV"", ""is_mobile"": true}" 3949,2,87,2017-04-11 18:30:43,http://gerlach.name/rebeka,,24.211.147.247,"{""location"": ""CG"", ""is_mobile"": true}" 3950,2,87,2016-12-20 07:32:54,http://breitenberg.name/felicia.littel,,41.238.159.226,"{""location"": ""SV"", ""is_mobile"": true}" 3951,2,87,2016-12-20 06:20:37,http://moorelowe.biz/julianne,,10.139.233.123,"{""location"": ""GE"", ""is_mobile"": false}" 3952,2,87,2017-05-31 11:20:33,http://dach.name/cornelius.price,,148.167.42.17,"{""location"": ""SR"", ""is_mobile"": false}" 3953,2,87,2017-03-08 17:57:24,http://goyette.com/eduardo,,74.157.5.121,"{""location"": ""VN"", ""is_mobile"": false}" 3954,2,87,2017-01-18 07:10:37,http://farrell.org/rosalind,,98.212.105.119,"{""location"": ""ER"", ""is_mobile"": true}" 3955,2,87,2017-04-28 09:04:38,http://cristroberts.net/dannie,,143.17.83.27,"{""location"": ""OM"", ""is_mobile"": false}" 3956,2,87,2017-03-27 04:35:04,http://crooks.io/nicolas.hackett,,151.253.107.219,"{""location"": ""HT"", ""is_mobile"": true}" 3957,2,87,2017-04-14 12:15:05,http://schaefer.name/lora,,199.100.125.208,"{""location"": ""KY"", ""is_mobile"": true}" 3958,2,87,2017-06-07 00:57:31,http://bernieremmerich.biz/ben.barrows,,48.27.147.177,"{""location"": ""ST"", ""is_mobile"": true}" 3959,2,87,2017-02-13 03:04:23,http://mclaughlingislason.co/wiley_nitzsche,,106.50.238.72,"{""location"": ""PR"", ""is_mobile"": true}" 3960,2,87,2017-01-15 03:50:37,http://gibsonschaefer.biz/montana.gleason,,47.54.102.214,"{""location"": ""AL"", ""is_mobile"": false}" 3961,2,87,2017-03-07 21:29:08,http://brekkethiel.co/paula.hayes,,54.20.54.36,"{""location"": ""TR"", ""is_mobile"": true}" 3962,2,87,2017-01-06 21:33:18,http://botsford.info/rolando_lakin,,104.47.214.103,"{""location"": ""OM"", ""is_mobile"": true}" 3963,2,87,2017-05-16 19:11:30,http://okeefe.co/emilie,,203.239.102.226,"{""location"": ""GE"", ""is_mobile"": false}" 3964,2,87,2017-05-27 11:55:59,http://emarddurgan.co/edgardo.haley,,224.227.110.58,"{""location"": ""SV"", ""is_mobile"": true}" 3965,2,87,2017-01-17 13:03:41,http://konopelskikiehn.org/dangelo_gutmann,,175.159.231.51,"{""location"": ""PS"", ""is_mobile"": true}" 3966,2,87,2017-01-22 20:05:38,http://beckerlang.net/dashawn,,151.114.201.55,"{""location"": ""PG"", ""is_mobile"": false}" 3967,2,87,2017-04-21 01:58:56,http://gislason.info/deon_kshlerin,,211.241.95.191,"{""location"": ""MK"", ""is_mobile"": true}" 3968,2,87,2017-02-17 07:27:45,http://hegmannhegmann.info/dane,,211.252.119.217,"{""location"": ""CO"", ""is_mobile"": true}" 3969,2,87,2016-12-19 23:56:36,http://schillerernser.com/nikki,,202.217.95.193,"{""location"": ""BQ"", ""is_mobile"": false}" 3970,2,87,2017-01-06 08:40:40,http://medhurst.biz/ryley.keeling,,123.218.150.224,"{""location"": ""CO"", ""is_mobile"": false}" 3971,2,87,2016-12-23 01:58:13,http://pollichharvey.info/monique.lebsack,,233.143.177.76,"{""location"": ""DO"", ""is_mobile"": false}" 3972,2,87,2016-12-29 12:33:07,http://crooks.org/maud.rogahn,,211.157.203.31,"{""location"": ""GY"", ""is_mobile"": true}" 3973,2,87,2017-03-21 18:05:10,http://riceraynor.net/lenny.smitham,,130.66.186.222,"{""location"": ""DM"", ""is_mobile"": false}" 3974,2,87,2016-12-19 11:29:18,http://keelingreynolds.co/mireille,,136.114.15.201,"{""location"": ""ET"", ""is_mobile"": true}" 3975,2,87,2017-06-10 03:02:34,http://donnellymedhurst.biz/francesco.beer,,116.232.169.162,"{""location"": ""CX"", ""is_mobile"": true}" 3976,2,87,2017-02-02 05:50:54,http://terry.com/finn,,179.85.80.137,"{""location"": ""NZ"", ""is_mobile"": true}" 3977,2,87,2017-05-16 09:31:58,http://ryan.org/clementina,,251.12.91.85,"{""location"": ""LU"", ""is_mobile"": false}" 3978,2,87,2017-01-30 05:56:46,http://reinger.org/carlo,,92.136.199.109,"{""location"": ""TD"", ""is_mobile"": true}" 3979,2,87,2017-06-08 21:14:01,http://gleason.org/judy,,181.193.143.200,"{""location"": ""HM"", ""is_mobile"": true}" 3980,2,87,2017-03-29 11:37:32,http://schultz.org/roy.borer,,39.196.224.172,"{""location"": ""AX"", ""is_mobile"": false}" 3981,2,87,2017-02-25 07:16:19,http://sawayn.net/kenny.hamill,,98.61.112.116,"{""location"": ""HN"", ""is_mobile"": true}" 3982,2,87,2017-02-07 20:17:41,http://swift.net/lucile,,3.70.186.178,"{""location"": ""BT"", ""is_mobile"": false}" 3983,2,87,2017-02-03 04:54:09,http://pfannerstillreichel.io/camryn,,186.14.109.21,"{""location"": ""GS"", ""is_mobile"": false}" 3984,2,87,2016-12-16 13:52:21,http://bechtelarwest.io/addison.armstrong,,21.13.55.80,"{""location"": ""HN"", ""is_mobile"": false}" 3985,2,87,2017-01-10 05:07:06,http://naderrosenbaum.net/gina_schmitt,,223.177.161.49,"{""location"": ""PM"", ""is_mobile"": false}" 3986,2,87,2017-02-10 14:26:27,http://cronapacocha.io/eldora,,251.170.246.50,"{""location"": ""CA"", ""is_mobile"": false}" 3987,2,87,2017-01-26 03:39:29,http://oconnelltrantow.name/vinnie,,168.37.120.85,"{""location"": ""BQ"", ""is_mobile"": false}" 3988,2,87,2017-02-02 18:17:06,http://hermistondonnelly.biz/stephen.konopelski,,22.238.207.105,"{""location"": ""TC"", ""is_mobile"": true}" 3989,2,87,2017-05-05 22:24:34,http://kling.co/presley,,159.197.245.204,"{""location"": ""GA"", ""is_mobile"": false}" 3990,2,87,2017-05-18 20:45:00,http://hansen.org/cierra,,109.194.115.30,"{""location"": ""MX"", ""is_mobile"": false}" 3991,2,87,2017-06-12 08:23:34,http://durgan.name/jaycee,,33.60.96.138,"{""location"": ""GY"", ""is_mobile"": false}" 3992,2,87,2017-03-22 19:45:28,http://padbergankunding.com/beryl_mcglynn,,102.252.253.174,"{""location"": ""YE"", ""is_mobile"": false}" 3993,2,87,2017-06-03 01:10:06,http://ullrich.io/trevor,,2.71.124.234,"{""location"": ""JM"", ""is_mobile"": false}" 3994,2,87,2017-05-07 03:07:13,http://gottliebsteuber.io/eric_bode,,71.155.69.201,"{""location"": ""KH"", ""is_mobile"": true}" 3995,2,87,2017-04-09 10:04:49,http://vonschoen.io/alvis,,139.25.38.160,"{""location"": ""BV"", ""is_mobile"": false}" 3996,2,87,2017-03-13 16:11:11,http://doyleruel.info/kacie,,238.183.14.155,"{""location"": ""BA"", ""is_mobile"": false}" 10886,4,243,2017-03-06 05:44:35,http://dachstanton.net/joy,,177.243.25.147,"{""location"": ""IR"", ""is_mobile"": false}" 10887,4,243,2017-02-14 09:09:51,http://ornmoen.com/maximilian,,223.142.233.14,"{""location"": ""SI"", ""is_mobile"": false}" 10888,4,243,2017-03-03 01:19:11,http://jacobson.co/maryjane,,53.154.63.124,"{""location"": ""LB"", ""is_mobile"": false}" 10889,4,243,2017-03-23 14:39:16,http://hopperunolfon.com/zander,,230.105.148.155,"{""location"": ""PW"", ""is_mobile"": true}" 10890,4,243,2017-01-03 03:24:21,http://williamson.org/valentin,,193.231.143.170,"{""location"": ""MY"", ""is_mobile"": true}" 10891,4,243,2017-03-20 11:27:36,http://kaulkeflatley.co/tristian,,196.61.70.213,"{""location"": ""BD"", ""is_mobile"": true}" 10892,4,243,2017-04-13 06:06:25,http://zulaufsanford.io/cali.osinski,,41.164.16.231,"{""location"": ""TT"", ""is_mobile"": true}" 10893,4,243,2017-02-14 12:34:36,http://ondrickadibbert.biz/birdie,,247.48.231.215,"{""location"": ""LK"", ""is_mobile"": false}" 10894,4,243,2017-04-03 08:34:17,http://kuhlman.co/giovani.berge,,47.232.140.124,"{""location"": ""MY"", ""is_mobile"": true}" 10895,4,243,2017-03-30 17:27:41,http://kautzer.biz/jacynthe_barrows,,33.135.206.204,"{""location"": ""GP"", ""is_mobile"": true}" 10896,4,243,2017-03-24 01:28:12,http://senger.net/casimir,,179.249.170.158,"{""location"": ""DZ"", ""is_mobile"": true}" 10897,4,243,2017-04-28 04:31:07,http://witting.io/nora,,145.191.40.22,"{""location"": ""TT"", ""is_mobile"": true}" 10898,4,243,2017-04-08 12:21:15,http://schmelerbauch.org/abe,,119.37.208.223,"{""location"": ""AD"", ""is_mobile"": false}" 10899,4,243,2017-03-17 09:04:22,http://mohr.name/jamison,,124.199.26.247,"{""location"": ""NF"", ""is_mobile"": true}" 10900,4,243,2017-05-23 08:43:19,http://oreilly.co/delores_kunze,,28.56.119.28,"{""location"": ""MX"", ""is_mobile"": false}" 10901,4,243,2017-01-01 16:06:21,http://herzog.com/demarcus,,6.58.10.144,"{""location"": ""YT"", ""is_mobile"": false}" 10902,4,243,2017-01-31 10:58:48,http://hammes.co/yasmine.rowe,,106.151.70.136,"{""location"": ""NZ"", ""is_mobile"": false}" 10903,4,243,2017-04-01 22:57:24,http://weinatstreich.biz/tremayne,,121.217.127.79,"{""location"": ""FO"", ""is_mobile"": true}" 10904,4,243,2016-12-28 05:48:32,http://brakus.info/howard,,45.232.111.81,"{""location"": ""LI"", ""is_mobile"": true}" 10905,4,243,2017-04-29 13:35:56,http://lockman.info/jennings,,104.178.196.99,"{""location"": ""KY"", ""is_mobile"": false}" 10906,4,243,2017-03-14 14:17:32,http://hamill.org/elya_emmerich,,28.116.35.66,"{""location"": ""SL"", ""is_mobile"": true}" 10907,4,244,2017-01-05 22:32:19,http://beer.co/oda,,174.173.37.48,"{""location"": ""MM"", ""is_mobile"": false}" 10908,4,244,2016-12-16 04:56:46,http://mann.co/earnest,,51.223.198.254,"{""location"": ""HM"", ""is_mobile"": false}" 10909,4,244,2017-03-31 10:49:58,http://hoppe.biz/paris,,219.181.81.173,"{""location"": ""NR"", ""is_mobile"": true}" 10910,4,244,2017-01-07 18:48:25,http://oreillycole.io/glennie,,133.60.160.163,"{""location"": ""TF"", ""is_mobile"": false}" 10911,4,244,2017-05-29 06:01:54,http://lindgren.biz/will,,11.251.82.73,"{""location"": ""FK"", ""is_mobile"": false}" 10912,4,244,2017-01-24 10:03:41,http://spencerullrich.net/leif_schuppe,,147.95.188.205,"{""location"": ""AO"", ""is_mobile"": false}" 10913,4,244,2017-04-06 07:54:42,http://heathcoteward.io/geraldine.prohaska,,149.128.45.97,"{""location"": ""HM"", ""is_mobile"": false}" 10914,4,244,2017-04-08 16:03:30,http://vandervort.info/alva,,51.122.14.128,"{""location"": ""BD"", ""is_mobile"": true}" 10915,4,244,2017-02-19 05:37:26,http://boehm.io/korbin_barton,,158.229.76.223,"{""location"": ""UG"", ""is_mobile"": false}" 10916,4,244,2017-01-12 21:58:23,http://pfannerstillhammes.biz/melisa_terry,,105.205.60.166,"{""location"": ""PN"", ""is_mobile"": true}" 10917,4,244,2017-05-18 14:33:41,http://heathcote.biz/keven.flatley,,247.12.221.27,"{""location"": ""MN"", ""is_mobile"": true}" 10918,4,244,2017-01-05 02:14:45,http://dachrodriguez.info/sierra,,249.67.31.233,"{""location"": ""LI"", ""is_mobile"": true}" 10919,4,244,2017-06-13 10:56:58,http://sawayn.name/gerda_langworth,,47.41.168.16,"{""location"": ""SB"", ""is_mobile"": false}" 10920,4,244,2017-04-06 14:57:24,http://wymantorp.info/aniyah.wyman,,197.71.117.83,"{""location"": ""YE"", ""is_mobile"": true}" 10921,4,244,2017-01-26 18:53:36,http://considinebailey.net/edgar,,150.192.203.42,"{""location"": ""GY"", ""is_mobile"": false}" 10922,4,244,2017-04-25 02:56:28,http://berge.co/alan,,154.73.60.143,"{""location"": ""LI"", ""is_mobile"": true}" 10923,4,244,2017-04-16 23:29:33,http://mante.name/denis.beer,,8.15.8.95,"{""location"": ""GH"", ""is_mobile"": false}" 10924,4,244,2017-03-13 00:27:11,http://ankunding.net/kirk,,223.66.18.111,"{""location"": ""NO"", ""is_mobile"": false}" 10925,4,244,2017-06-03 10:23:49,http://swaniawski.co/quinton,,93.72.41.104,"{""location"": ""MH"", ""is_mobile"": false}" 10926,4,244,2016-12-18 14:33:31,http://dachzieme.org/madaline,,44.54.6.83,"{""location"": ""KZ"", ""is_mobile"": false}" 10927,4,244,2017-05-21 20:43:30,http://rutherford.co/claudia_renner,,98.147.61.116,"{""location"": ""SX"", ""is_mobile"": false}" 10928,4,244,2017-03-30 20:28:14,http://kshlerinhammes.biz/richmond_auer,,190.19.237.229,"{""location"": ""FO"", ""is_mobile"": true}" 10929,4,244,2017-06-03 00:44:20,http://kovacekprohaska.net/janie_kub,,39.183.146.107,"{""location"": ""NP"", ""is_mobile"": false}" 10930,4,244,2017-02-25 01:49:28,http://mertzhermann.biz/kody,,33.102.225.62,"{""location"": ""LC"", ""is_mobile"": true}" 10931,4,244,2017-05-07 21:38:35,http://cartwrightveum.info/lilly,,162.118.194.66,"{""location"": ""EC"", ""is_mobile"": false}" 10932,4,244,2016-12-27 08:15:42,http://mcdermott.co/sincere_barrows,,146.38.188.55,"{""location"": ""LC"", ""is_mobile"": false}" 10933,4,244,2017-02-25 06:56:40,http://leffler.org/cedrick,,167.65.152.20,"{""location"": ""CA"", ""is_mobile"": true}" 10934,4,244,2017-01-29 08:48:12,http://lueilwitzwest.net/mariela_zemlak,,138.249.106.216,"{""location"": ""GD"", ""is_mobile"": true}" 10935,4,244,2017-02-25 23:29:04,http://schmidt.org/durward.mcclure,,182.34.249.82,"{""location"": ""IR"", ""is_mobile"": true}" 10936,4,244,2017-06-08 00:06:23,http://hirthebins.co/maud,,61.254.30.159,"{""location"": ""NI"", ""is_mobile"": false}" 10937,4,244,2017-02-21 15:29:43,http://daremetz.co/eino.kunze,,7.111.143.241,"{""location"": ""YT"", ""is_mobile"": true}" 10938,4,244,2017-05-21 13:26:11,http://vonkling.info/johnny,,206.123.245.180,"{""location"": ""FO"", ""is_mobile"": true}" 10939,4,244,2017-05-24 21:31:24,http://olson.co/clementina,,79.222.214.242,"{""location"": ""ME"", ""is_mobile"": false}" 10940,4,244,2017-02-24 00:43:59,http://gerlach.biz/genevieve,,117.66.211.5,"{""location"": ""CW"", ""is_mobile"": false}" 10941,4,245,2016-12-16 22:30:59,http://howe.com/delphine,,193.108.58.69,"{""location"": ""ID"", ""is_mobile"": false}" 7913,3,175,2017-04-30 17:15:51,http://torp.io/mozelle,0.3218307734,226.234.218.148,"{""location"": ""CH"", ""is_mobile"": true}" 7914,3,175,2017-05-10 00:09:42,http://walshbrakus.net/margret.ward,0.3656563740,2.46.70.28,"{""location"": ""AO"", ""is_mobile"": false}" 7915,3,175,2017-04-17 09:38:03,http://muellerhayes.co/eli.renner,0.3715112770,19.126.74.142,"{""location"": ""PH"", ""is_mobile"": true}" 7916,3,175,2017-06-11 20:14:29,http://bauch.org/effie_ferry,0.5110993096,10.252.187.189,"{""location"": ""SK"", ""is_mobile"": false}" 7917,3,175,2017-01-23 05:44:20,http://littleschuppe.biz/kaley,0.2745430581,21.21.205.205,"{""location"": ""MZ"", ""is_mobile"": false}" 7918,3,175,2017-02-26 21:29:48,http://mccullough.biz/giovani,0.8515915258,102.91.124.219,"{""location"": ""TD"", ""is_mobile"": true}" 7919,3,175,2017-02-28 06:44:56,http://hagenes.io/sabina,0.6833825453,127.17.54.218,"{""location"": ""SS"", ""is_mobile"": true}" 7920,3,175,2017-04-30 08:43:21,http://homenick.net/sallie.gerhold,0.2235630439,47.179.75.44,"{""location"": ""MT"", ""is_mobile"": false}" 7921,3,175,2017-03-30 14:45:36,http://blandadare.info/john,0.3101675885,76.129.201.145,"{""location"": ""ET"", ""is_mobile"": false}" 7922,3,175,2017-01-28 05:16:27,http://kirlin.info/saige_ryan,0.0761294075,138.223.173.84,"{""location"": ""PY"", ""is_mobile"": true}" 7923,3,175,2017-04-10 19:26:05,http://zieme.com/bonita.abernathy,0.7457542776,121.140.20.206,"{""location"": ""HM"", ""is_mobile"": false}" 7924,3,175,2017-04-19 00:35:28,http://turner.io/madyson,0.4710870377,202.101.61.244,"{""location"": ""GW"", ""is_mobile"": false}" 7925,3,175,2016-12-18 02:31:51,http://zulaufhermann.co/luna,0.2842030498,10.76.35.217,"{""location"": ""CL"", ""is_mobile"": true}" 7926,3,175,2017-01-19 09:58:35,http://parisian.biz/jaclyn,0.3976442552,237.19.237.31,"{""location"": ""IT"", ""is_mobile"": false}" 7927,3,175,2017-02-28 20:58:44,http://baumbachboyle.io/camille,0.3875905380,72.91.131.77,"{""location"": ""AX"", ""is_mobile"": true}" 7928,3,175,2017-03-12 21:12:02,http://gerholdaltenwerth.name/jordy_bayer,0.2769767452,155.180.124.46,"{""location"": ""TM"", ""is_mobile"": true}" 7929,3,175,2017-05-28 05:34:44,http://rippin.org/muhammad,0.3732024103,205.207.76.98,"{""location"": ""CO"", ""is_mobile"": false}" 7930,3,175,2017-01-13 05:07:10,http://veum.name/martine.leannon,0.0469703859,55.97.136.201,"{""location"": ""AS"", ""is_mobile"": false}" 7931,3,175,2017-02-15 04:17:14,http://adams.info/garnett.mayer,0.8142884857,158.123.112.107,"{""location"": ""SN"", ""is_mobile"": true}" 7932,3,175,2017-01-27 09:53:27,http://mayert.co/dereck,0.7222127642,150.146.39.77,"{""location"": ""NE"", ""is_mobile"": true}" 7933,3,175,2017-02-12 09:31:45,http://skiles.co/ferne,0.2403680375,79.243.41.141,"{""location"": ""BY"", ""is_mobile"": true}" 7934,3,175,2017-02-27 19:53:40,http://lang.com/eve,0.1007440226,237.34.253.185,"{""location"": ""WS"", ""is_mobile"": true}" 7935,3,175,2017-03-24 12:49:15,http://bruenbernier.info/margarette_daniel,0.9745281211,201.158.222.177,"{""location"": ""CD"", ""is_mobile"": false}" 7936,3,175,2017-01-12 00:00:55,http://ondrickakertzmann.net/theresa.paucek,0.0949857445,146.133.138.228,"{""location"": ""CN"", ""is_mobile"": false}" 7937,3,175,2017-04-20 23:22:37,http://christiansen.org/emerson_welch,0.6864013637,39.189.232.106,"{""location"": ""HT"", ""is_mobile"": false}" 7938,3,175,2017-04-10 02:52:25,http://gerhold.info/elvera,0.4767182222,225.199.53.36,"{""location"": ""LV"", ""is_mobile"": false}" 7939,3,175,2017-04-23 23:30:01,http://mckenzieoberbrunner.org/elsa,0.7492948181,166.124.201.69,"{""location"": ""GP"", ""is_mobile"": true}" 7940,3,175,2017-03-27 01:34:04,http://jaskolski.name/rickie,0.7093146406,71.157.11.215,"{""location"": ""MH"", ""is_mobile"": false}" 7941,3,175,2016-12-29 07:08:47,http://bednar.org/lacy,0.5873346596,189.166.235.146,"{""location"": ""MZ"", ""is_mobile"": true}" 7942,3,175,2016-12-19 12:25:50,http://beierthompson.io/richard.rolfson,0.3115867502,140.56.65.219,"{""location"": ""GP"", ""is_mobile"": false}" 7943,3,175,2017-06-12 00:08:58,http://hermiston.com/aniya_beahan,0.3270498454,143.16.239.71,"{""location"": ""MX"", ""is_mobile"": true}" 7944,3,175,2017-02-14 11:30:02,http://streichbartoletti.io/antone_yost,0.7069025425,188.219.175.219,"{""location"": ""NP"", ""is_mobile"": false}" 7945,3,175,2017-01-19 14:19:14,http://schuppe.net/pearline.dickens,0.5217813262,49.197.236.25,"{""location"": ""FM"", ""is_mobile"": true}" 7946,3,175,2017-05-12 15:59:27,http://pacochaernser.org/amari,0.8538477292,12.145.58.13,"{""location"": ""GY"", ""is_mobile"": false}" 7947,3,175,2017-05-31 08:02:03,http://halvorson.biz/macie,0.8437397515,21.63.37.182,"{""location"": ""MO"", ""is_mobile"": false}" 7948,3,175,2017-06-07 00:53:18,http://heaneyokuneva.info/bettye,0.8177091000,218.229.209.3,"{""location"": ""AO"", ""is_mobile"": true}" 7949,3,175,2017-04-14 22:05:12,http://cronablanda.co/tiana.bednar,0.7816068691,9.40.8.139,"{""location"": ""TH"", ""is_mobile"": false}" 7950,3,175,2017-02-14 16:20:34,http://brekke.io/stacey,0.9718075345,37.254.209.43,"{""location"": ""AW"", ""is_mobile"": false}" 7951,3,175,2016-12-22 06:11:23,http://langworth.org/stevie.vonrueden,0.7441396908,130.133.228.113,"{""location"": ""SK"", ""is_mobile"": false}" 7952,3,175,2017-02-13 01:43:20,http://dicki.name/ettie_nienow,0.7136953485,112.111.14.138,"{""location"": ""KY"", ""is_mobile"": false}" 7953,3,175,2017-01-21 09:40:10,http://weberberge.org/tobin,0.8083029008,62.29.23.214,"{""location"": ""MA"", ""is_mobile"": true}" 7954,3,175,2017-02-28 03:18:01,http://oreilly.biz/ellie,0.1888093971,231.251.115.109,"{""location"": ""PK"", ""is_mobile"": false}" 7955,3,175,2017-04-05 21:13:59,http://heeljohnson.co/carolanne,0.3254085960,232.43.66.150,"{""location"": ""RS"", ""is_mobile"": false}" 7956,3,175,2017-05-18 16:04:12,http://mcculloughmacejkovic.io/leland_crooks,0.9597548536,65.91.245.238,"{""location"": ""GT"", ""is_mobile"": true}" 7957,3,175,2017-02-26 19:12:51,http://purdyyundt.net/corine,0.9433513776,32.184.6.89,"{""location"": ""YT"", ""is_mobile"": false}" 7958,3,175,2017-04-12 02:43:42,http://romaguera.info/alexandrea_jacobs,0.8394344721,182.79.229.58,"{""location"": ""ER"", ""is_mobile"": false}" 7959,3,175,2017-05-11 19:55:29,http://willms.info/monique_friesen,0.4298751191,239.10.38.20,"{""location"": ""SY"", ""is_mobile"": false}" 7960,3,175,2017-02-23 11:00:01,http://hegmann.co/alanis,0.8771821686,155.111.220.65,"{""location"": ""KY"", ""is_mobile"": false}" 7961,3,175,2017-01-24 00:58:58,http://christiansen.info/zion.kuhlman,0.4425200993,75.27.98.211,"{""location"": ""RW"", ""is_mobile"": false}" 7962,3,175,2016-12-28 21:50:10,http://tillmanprohaska.io/kathleen,0.0401880158,204.122.131.140,"{""location"": ""PH"", ""is_mobile"": false}" 7963,3,175,2017-02-26 22:41:45,http://grady.co/daisy,0.4622889768,98.175.204.63,"{""location"": ""CD"", ""is_mobile"": false}" 13836,5,310,2017-05-17 00:59:22,http://brekke.com/alyon,,168.99.41.63,"{""location"": ""GG"", ""is_mobile"": true}" 13837,5,310,2017-04-05 01:36:21,http://grahamkeebler.co/antonetta_luettgen,,114.157.59.205,"{""location"": ""BR"", ""is_mobile"": true}" 13838,5,310,2017-01-05 21:38:10,http://damore.co/andy.conroy,,243.236.167.116,"{""location"": ""SM"", ""is_mobile"": true}" 13839,5,310,2017-05-02 02:22:26,http://haleyspinka.org/aleandro.beahan,,111.89.190.34,"{""location"": ""BQ"", ""is_mobile"": false}" 13840,5,310,2017-04-08 03:29:53,http://lakin.io/otto_heller,,199.72.216.147,"{""location"": ""CK"", ""is_mobile"": false}" 13841,5,310,2016-12-26 18:39:23,http://turcotte.io/keenan,,61.191.231.179,"{""location"": ""SN"", ""is_mobile"": true}" 13842,5,310,2017-04-02 09:10:45,http://hermann.name/christop.kertzmann,,212.92.82.71,"{""location"": ""MO"", ""is_mobile"": false}" 13843,5,311,2016-12-27 22:16:06,http://grimes.co/kacie,,209.54.113.206,"{""location"": ""MC"", ""is_mobile"": false}" 13844,5,311,2017-01-09 23:26:05,http://haag.org/oral.hartmann,,134.74.197.167,"{""location"": ""DK"", ""is_mobile"": true}" 13845,5,311,2017-04-07 11:30:45,http://hills.io/pearlie_smith,,119.95.254.215,"{""location"": ""MY"", ""is_mobile"": true}" 13846,5,311,2017-02-28 12:48:53,http://kerluke.com/maggie.ankunding,,195.214.60.242,"{""location"": ""SX"", ""is_mobile"": false}" 13847,5,311,2017-04-15 10:24:47,http://cole.org/roderick,,237.80.231.186,"{""location"": ""AW"", ""is_mobile"": false}" 13848,5,311,2017-02-27 00:37:08,http://robel.name/maci_carter,,147.49.182.217,"{""location"": ""TM"", ""is_mobile"": false}" 13849,5,311,2017-05-29 01:37:08,http://gaylordgerlach.biz/lexi.tremblay,,90.34.133.225,"{""location"": ""GR"", ""is_mobile"": true}" 13850,5,311,2017-01-24 16:59:27,http://bins.org/elian,,207.60.10.72,"{""location"": ""MP"", ""is_mobile"": false}" 13851,5,311,2017-04-13 21:11:58,http://koelpin.name/sierra.simonis,,92.71.35.205,"{""location"": ""SY"", ""is_mobile"": false}" 13852,5,311,2017-03-29 19:55:29,http://greenfelder.org/buck.schneider,,25.215.240.220,"{""location"": ""MY"", ""is_mobile"": true}" 13853,5,311,2017-02-20 07:57:51,http://hansen.io/trisha_walker,,108.95.117.241,"{""location"": ""FI"", ""is_mobile"": true}" 13854,5,311,2017-04-19 06:28:48,http://bradtke.co/manley_littel,,98.128.133.45,"{""location"": ""UY"", ""is_mobile"": true}" 13855,5,311,2017-03-23 23:23:18,http://binsmarks.com/frederic,,221.203.38.179,"{""location"": ""FM"", ""is_mobile"": false}" 13856,5,311,2016-12-13 08:47:22,http://littelhilpert.io/tobin,,252.169.196.207,"{""location"": ""FI"", ""is_mobile"": false}" 13857,5,311,2017-04-30 08:06:25,http://willmskoelpin.net/reagan,,99.212.50.151,"{""location"": ""CI"", ""is_mobile"": false}" 13858,5,311,2017-05-07 11:47:28,http://rolfsonkoepp.com/noemy.heaney,,126.184.245.209,"{""location"": ""GF"", ""is_mobile"": true}" 13859,5,311,2017-01-27 18:09:49,http://gislason.io/jennings_corwin,,177.85.39.67,"{""location"": ""LA"", ""is_mobile"": false}" 13860,5,311,2017-01-23 13:32:03,http://kirlincormier.biz/bart.klein,,244.207.131.111,"{""location"": ""CR"", ""is_mobile"": true}" 13861,5,311,2017-05-03 08:11:08,http://parisian.net/tracy,,211.62.180.19,"{""location"": ""KY"", ""is_mobile"": false}" 13862,5,311,2016-12-13 06:17:44,http://jakubowskibuckridge.co/camryn,,183.31.115.61,"{""location"": ""SY"", ""is_mobile"": false}" 13863,5,311,2017-03-24 00:23:04,http://jerde.name/claud.grimes,,197.229.64.172,"{""location"": ""CD"", ""is_mobile"": false}" 13864,5,311,2017-06-07 00:38:07,http://adamsbahringer.info/bryon.simonis,,39.42.252.90,"{""location"": ""MR"", ""is_mobile"": true}" 13865,5,311,2017-02-27 16:27:12,http://marvinbode.name/keeley,,245.139.196.162,"{""location"": ""MA"", ""is_mobile"": false}" 13866,5,311,2017-04-04 04:12:51,http://purdy.co/francisco,,169.197.54.245,"{""location"": ""AF"", ""is_mobile"": true}" 13867,5,311,2017-04-14 07:58:01,http://swaniawski.info/dewayne,,225.113.185.23,"{""location"": ""GP"", ""is_mobile"": true}" 13868,5,311,2017-05-02 05:37:26,http://kiehn.org/salvatore,,238.125.24.158,"{""location"": ""MZ"", ""is_mobile"": true}" 13869,5,311,2017-01-17 23:14:03,http://robertsaufderhar.info/rolando.lubowitz,,158.205.17.114,"{""location"": ""CV"", ""is_mobile"": true}" 13870,5,311,2017-01-03 01:22:43,http://pollich.io/sonny,,58.46.64.6,"{""location"": ""BY"", ""is_mobile"": true}" 13871,5,311,2017-02-11 23:03:19,http://konopelski.biz/marc_runolfsdottir,,121.63.102.237,"{""location"": ""MW"", ""is_mobile"": false}" 13872,5,311,2017-04-28 19:18:32,http://metz.org/velda.cole,,131.240.170.6,"{""location"": ""NI"", ""is_mobile"": true}" 13873,5,311,2016-12-25 10:51:15,http://dibbert.co/suzanne.rohan,,210.171.215.66,"{""location"": ""LK"", ""is_mobile"": true}" 13874,5,311,2017-05-17 01:40:25,http://lehner.com/neal,,247.128.53.5,"{""location"": ""DK"", ""is_mobile"": false}" 13875,5,311,2017-04-13 01:18:29,http://larson.net/damon.dibbert,,137.85.63.14,"{""location"": ""WF"", ""is_mobile"": true}" 13876,5,311,2017-04-02 19:00:09,http://lueilwitz.net/nettie_schuppe,,158.165.229.155,"{""location"": ""LC"", ""is_mobile"": false}" 13877,5,311,2017-04-06 19:28:28,http://wardeichmann.biz/yasmeen,,107.48.108.152,"{""location"": ""AO"", ""is_mobile"": true}" 13878,5,311,2017-04-24 09:51:14,http://gutkowskischimmel.biz/andre.pouros,,118.16.91.4,"{""location"": ""TO"", ""is_mobile"": true}" 13879,5,311,2017-05-31 06:29:24,http://monahangrimes.co/maya_goldner,,46.43.2.227,"{""location"": ""BQ"", ""is_mobile"": true}" 13880,5,311,2017-06-06 08:46:44,http://runolfon.org/ole,,91.59.238.178,"{""location"": ""UM"", ""is_mobile"": false}" 13881,5,311,2017-01-19 16:31:01,http://kundeconroy.co/doug_gleason,,150.65.136.97,"{""location"": ""NZ"", ""is_mobile"": false}" 13882,5,311,2017-06-13 17:23:17,http://stehr.io/madilyn,,76.160.234.104,"{""location"": ""CG"", ""is_mobile"": false}" 13883,5,311,2017-03-05 21:05:50,http://kutchhirthe.co/yeenia.hauck,,81.97.122.196,"{""location"": ""KE"", ""is_mobile"": false}" 13884,5,311,2017-01-20 11:32:11,http://emard.name/roxane,,23.244.107.138,"{""location"": ""CY"", ""is_mobile"": false}" 13885,5,311,2017-01-07 11:33:04,http://konopelskiwisozk.name/alexie_schinner,,58.161.5.253,"{""location"": ""AG"", ""is_mobile"": false}" 13886,5,311,2017-04-26 02:22:38,http://bergstromreilly.com/mavis,,100.185.104.239,"{""location"": ""BM"", ""is_mobile"": true}" 13887,5,311,2017-05-30 16:01:35,http://botsford.net/jean,,29.135.220.50,"{""location"": ""TC"", ""is_mobile"": true}" 13888,5,311,2017-01-03 06:29:37,http://wilkinsonharris.io/vidal,,127.134.143.249,"{""location"": ""LB"", ""is_mobile"": false}" 13889,5,311,2017-03-04 11:57:56,http://baumbach.org/filiberto_sipes,,178.253.126.252,"{""location"": ""AI"", ""is_mobile"": true}" 13890,5,311,2017-04-22 04:06:28,http://williamson.biz/luigi,,133.105.195.237,"{""location"": ""KG"", ""is_mobile"": true}" 16819,6,381,2017-01-01 23:39:22,http://johnston.io/elwin,,140.180.254.223,"{""location"": ""NU"", ""is_mobile"": false}" 16820,6,381,2017-06-08 17:19:46,http://mueller.name/ewald,,100.140.72.46,"{""location"": ""TO"", ""is_mobile"": true}" 16821,6,381,2017-06-02 05:39:43,http://damore.com/ahmed_heathcote,,126.250.51.133,"{""location"": ""TZ"", ""is_mobile"": true}" 16822,6,381,2017-02-10 01:29:39,http://trompfarrell.info/roslyn,,252.108.6.15,"{""location"": ""FJ"", ""is_mobile"": false}" 16823,6,381,2017-05-10 10:55:17,http://stiedemann.org/helena,,163.173.230.167,"{""location"": ""AU"", ""is_mobile"": true}" 16824,6,381,2017-03-11 21:52:53,http://turcotte.com/oda_bradtke,,146.121.3.195,"{""location"": ""GL"", ""is_mobile"": true}" 16825,6,381,2017-05-26 03:17:16,http://bartoletti.io/kaia_lindgren,,10.121.203.16,"{""location"": ""CC"", ""is_mobile"": false}" 16826,6,381,2017-06-01 04:45:49,http://macejkovic.io/boyd,,208.168.176.70,"{""location"": ""CN"", ""is_mobile"": true}" 16827,6,381,2016-12-19 01:49:38,http://collier.io/shanon,,84.147.138.110,"{""location"": ""TK"", ""is_mobile"": true}" 16828,6,381,2017-06-08 16:26:08,http://nikolauwift.net/ernie_purdy,,240.208.231.59,"{""location"": ""AG"", ""is_mobile"": false}" 16829,6,381,2016-12-24 19:32:40,http://vandervort.io/rose,,143.217.249.127,"{""location"": ""MX"", ""is_mobile"": false}" 16830,6,381,2017-04-15 12:17:42,http://koch.biz/haven,,140.117.5.22,"{""location"": ""CU"", ""is_mobile"": false}" 16831,6,381,2017-01-25 13:24:51,http://barton.net/beatrice_koelpin,,33.70.250.151,"{""location"": ""PE"", ""is_mobile"": false}" 16832,6,381,2017-02-08 09:08:24,http://wunsch.net/nathaniel,,142.235.238.96,"{""location"": ""TN"", ""is_mobile"": true}" 16833,6,381,2017-03-26 21:30:33,http://bergnaumadams.info/joanie_murazik,,254.11.114.27,"{""location"": ""ST"", ""is_mobile"": false}" 16834,6,381,2016-12-21 15:54:07,http://dooley.net/gennaro_ritchie,,131.58.212.137,"{""location"": ""GF"", ""is_mobile"": true}" 16835,6,381,2017-04-28 11:03:42,http://bahringerbrown.name/michael.daniel,,145.235.182.232,"{""location"": ""MD"", ""is_mobile"": true}" 16836,6,381,2016-12-25 06:50:04,http://powlowski.com/raoul.romaguera,,208.226.169.116,"{""location"": ""VU"", ""is_mobile"": false}" 16837,6,381,2017-01-02 06:29:29,http://klingkreiger.biz/blaise,,84.178.27.230,"{""location"": ""JP"", ""is_mobile"": true}" 16838,6,381,2017-05-19 03:45:46,http://leschgrady.name/connie,,202.249.72.25,"{""location"": ""CL"", ""is_mobile"": true}" 16839,6,381,2017-03-31 07:49:57,http://harber.info/jan,,175.159.232.26,"{""location"": ""ES"", ""is_mobile"": false}" 16840,6,381,2017-04-05 00:49:04,http://oconnelldeckow.co/rashawn_johnson,,241.16.249.170,"{""location"": ""CD"", ""is_mobile"": true}" 16841,6,381,2017-04-14 09:32:59,http://goyette.name/uriel_witting,,236.124.66.189,"{""location"": ""JO"", ""is_mobile"": true}" 17302,6,390,2017-03-05 05:03:55,http://terry.org/stephan,,160.61.146.23,"{""location"": ""SX"", ""is_mobile"": true}" 16842,6,381,2017-04-19 23:40:05,http://trompmaggio.biz/lizeth_heidenreich,,163.145.65.202,"{""location"": ""NL"", ""is_mobile"": true}" 16843,6,381,2017-04-08 04:05:47,http://koeppsteuber.info/grace,,93.138.65.168,"{""location"": ""LA"", ""is_mobile"": false}" 16844,6,381,2017-02-09 05:11:49,http://luettgencummings.info/anthony.bashirian,,250.196.193.136,"{""location"": ""TD"", ""is_mobile"": false}" 16845,6,381,2017-05-31 19:23:32,http://howe.name/herminio_west,,27.148.146.69,"{""location"": ""NC"", ""is_mobile"": false}" 16846,6,381,2017-06-11 21:08:44,http://hayeszboncak.com/al.kohler,,94.23.139.139,"{""location"": ""BB"", ""is_mobile"": true}" 16847,6,381,2017-04-05 00:09:30,http://okongrady.org/morgan.zemlak,,118.44.139.15,"{""location"": ""KH"", ""is_mobile"": false}" 16848,6,381,2017-03-19 23:02:02,http://weimannullrich.net/ramona,,172.242.30.79,"{""location"": ""CX"", ""is_mobile"": false}" 16849,6,381,2017-02-20 23:25:01,http://wolffritchie.org/leilani.kutch,,168.252.100.55,"{""location"": ""JE"", ""is_mobile"": false}" 16850,6,381,2017-02-02 08:06:24,http://beckerstracke.co/titus,,119.113.98.116,"{""location"": ""BG"", ""is_mobile"": true}" 16851,6,381,2017-03-08 15:27:31,http://prohaska.org/derick,,65.62.171.179,"{""location"": ""PA"", ""is_mobile"": false}" 16852,6,381,2016-12-15 09:44:48,http://cruickshank.net/annabelle,,93.86.154.118,"{""location"": ""CY"", ""is_mobile"": true}" 16853,6,381,2016-12-13 19:35:24,http://torphy.name/kristopher.legros,,123.90.253.128,"{""location"": ""BZ"", ""is_mobile"": true}" 16854,6,381,2017-02-03 03:24:32,http://schinner.co/idella.hirthe,,49.194.32.8,"{""location"": ""LV"", ""is_mobile"": true}" 16855,6,381,2017-04-12 09:32:38,http://altenwerth.co/martin,,158.92.146.164,"{""location"": ""LC"", ""is_mobile"": true}" 16856,6,381,2017-05-05 13:51:20,http://stoltenbergquitzon.com/opal_wilkinson,,28.185.194.211,"{""location"": ""CW"", ""is_mobile"": false}" 16857,6,381,2017-03-06 12:55:55,http://hammes.org/marcella,,17.189.113.94,"{""location"": ""DK"", ""is_mobile"": false}" 16858,6,381,2017-02-06 05:14:15,http://pacocha.info/heber.oberbrunner,,35.240.218.239,"{""location"": ""SN"", ""is_mobile"": false}" 16859,6,381,2017-03-05 04:14:03,http://runolfsdottir.info/josh,,157.58.8.24,"{""location"": ""BZ"", ""is_mobile"": true}" 16860,6,381,2017-05-19 00:45:35,http://oreilly.net/lora.greenfelder,,160.214.206.15,"{""location"": ""PE"", ""is_mobile"": true}" 16861,6,382,2017-01-03 03:01:17,http://marks.com/clare_bernhard,,169.220.219.73,"{""location"": ""NC"", ""is_mobile"": true}" 16862,6,382,2017-01-22 23:50:00,http://denesik.info/bo,,59.63.78.9,"{""location"": ""HU"", ""is_mobile"": true}" 16863,6,382,2016-12-30 16:26:42,http://welch.co/santos_beer,,116.34.46.213,"{""location"": ""GP"", ""is_mobile"": true}" 16864,6,382,2017-05-11 09:09:33,http://mueller.org/genevieve_powlowski,,248.113.46.35,"{""location"": ""EG"", ""is_mobile"": true}" 16865,6,382,2017-01-12 14:11:55,http://kuhn.org/osbaldo,,139.201.11.225,"{""location"": ""KP"", ""is_mobile"": false}" 16866,6,382,2017-05-05 14:27:06,http://hudsonbauch.biz/etha,,121.4.95.161,"{""location"": ""AL"", ""is_mobile"": false}" 16867,6,382,2017-04-03 04:57:18,http://schaden.net/mackenzie,,13.151.193.202,"{""location"": ""PW"", ""is_mobile"": false}" 16868,6,382,2017-06-08 03:35:43,http://homenick.co/rebecca.herzog,,41.11.45.162,"{""location"": ""UA"", ""is_mobile"": false}" 16869,6,382,2017-03-01 16:50:36,http://leuschke.biz/connie.lind,,157.247.38.20,"{""location"": ""NE"", ""is_mobile"": true}" 16870,6,382,2017-05-19 18:44:45,http://walker.info/leta_bauch,,193.220.126.8,"{""location"": ""IN"", ""is_mobile"": true}" 16871,6,382,2017-05-27 09:44:51,http://predovic.io/fatima_schowalter,,118.28.208.136,"{""location"": ""MM"", ""is_mobile"": true}" 16872,6,382,2017-05-09 14:10:18,http://wehner.info/brandon,,98.207.157.23,"{""location"": ""AE"", ""is_mobile"": false}" 3997,2,87,2017-04-12 03:42:37,http://medhurst.co/dana.ryan,,238.123.161.103,"{""location"": ""KI"", ""is_mobile"": false}" 3998,2,87,2017-01-12 08:34:00,http://keler.biz/helena_ruecker,,174.144.164.52,"{""location"": ""NC"", ""is_mobile"": true}" 3999,2,87,2016-12-27 06:03:54,http://wiza.biz/enid_feeney,,159.156.152.78,"{""location"": ""JO"", ""is_mobile"": false}" 4000,2,87,2017-02-01 18:01:16,http://moriette.io/emilio.marvin,,94.190.61.141,"{""location"": ""PS"", ""is_mobile"": true}" 4001,2,87,2017-02-24 13:53:27,http://rogahntremblay.biz/jasper.kreiger,,22.245.80.143,"{""location"": ""HT"", ""is_mobile"": true}" 4002,2,87,2017-03-28 03:42:54,http://rempelmcclure.com/edmund,,111.85.152.4,"{""location"": ""BS"", ""is_mobile"": false}" 4003,2,87,2017-04-08 20:29:53,http://barton.co/estell.lakin,,132.244.18.176,"{""location"": ""MN"", ""is_mobile"": true}" 4004,2,87,2017-02-07 10:19:57,http://maggio.biz/letha.dickinson,,70.164.94.149,"{""location"": ""CU"", ""is_mobile"": false}" 4005,2,87,2017-03-28 11:01:39,http://bernhardluettgen.com/braden.stokes,,2.18.240.18,"{""location"": ""KG"", ""is_mobile"": true}" 4006,2,88,2017-03-28 09:53:25,http://flatley.info/celine,,26.167.49.106,"{""location"": ""AZ"", ""is_mobile"": true}" 4007,2,88,2017-05-11 16:58:35,http://cummings.name/adalberto,,227.16.79.227,"{""location"": ""KW"", ""is_mobile"": true}" 4008,2,88,2017-02-21 08:04:48,http://willms.org/delilah,,241.38.37.111,"{""location"": ""GE"", ""is_mobile"": false}" 4009,2,88,2017-03-20 13:46:49,http://luettgenprice.com/miouri,,155.244.57.179,"{""location"": ""ZM"", ""is_mobile"": false}" 4010,2,88,2017-05-29 08:16:26,http://schmelerschmeler.co/virgie.beatty,,89.159.225.34,"{""location"": ""HU"", ""is_mobile"": false}" 4011,2,88,2016-12-28 19:49:23,http://stoltenberg.biz/tyson.abbott,,236.23.214.136,"{""location"": ""TZ"", ""is_mobile"": false}" 4012,2,88,2017-03-25 18:42:50,http://schaefer.name/jevon.littel,,189.198.252.221,"{""location"": ""GM"", ""is_mobile"": false}" 4013,2,88,2017-06-10 21:36:39,http://towne.io/arlene.morar,,15.73.100.75,"{""location"": ""LU"", ""is_mobile"": false}" 4014,2,88,2017-01-31 14:59:20,http://wunschwhite.org/jamal,,241.117.228.211,"{""location"": ""CZ"", ""is_mobile"": true}" 4015,2,88,2017-05-27 09:55:23,http://bahringer.net/maryjane.keeling,,63.115.114.65,"{""location"": ""GT"", ""is_mobile"": true}" 4016,2,88,2017-03-03 18:40:24,http://mosciski.io/ethel,,155.118.28.172,"{""location"": ""LT"", ""is_mobile"": false}" 4017,2,88,2017-01-19 10:00:11,http://tillman.com/blanca_fadel,,140.131.220.48,"{""location"": ""MH"", ""is_mobile"": true}" 4018,2,88,2017-05-28 09:38:27,http://hudson.io/lula_pagac,,200.74.236.8,"{""location"": ""BQ"", ""is_mobile"": false}" 4019,2,88,2017-05-17 00:00:52,http://wehnerkoch.io/ima_johnston,,189.232.98.188,"{""location"": ""BI"", ""is_mobile"": false}" 4020,2,88,2017-03-02 14:15:44,http://wyman.org/finn,,186.223.134.157,"{""location"": ""PH"", ""is_mobile"": false}" 4021,2,88,2017-06-05 22:42:51,http://corwin.name/liam,,37.157.121.235,"{""location"": ""MC"", ""is_mobile"": true}" 4022,2,88,2017-05-14 13:56:04,http://mann.io/daphnee_jacobs,,187.197.188.246,"{""location"": ""GS"", ""is_mobile"": false}" 4023,2,88,2017-02-12 22:06:24,http://waelchi.name/natasha,,239.56.63.78,"{""location"": ""CZ"", ""is_mobile"": false}" 4024,2,88,2017-05-18 16:02:46,http://kuhlman.biz/tabitha.hickle,,23.49.115.169,"{""location"": ""FM"", ""is_mobile"": false}" 4025,2,88,2017-04-21 15:11:02,http://kleingoyette.biz/felipe,,144.8.239.4,"{""location"": ""FK"", ""is_mobile"": true}" 4026,2,88,2017-04-04 08:18:45,http://schusterbalistreri.info/dannie.turner,,203.92.36.77,"{""location"": ""CF"", ""is_mobile"": true}" 4027,2,88,2016-12-14 17:19:15,http://williamsonfeest.net/maci.ledner,,44.11.181.154,"{""location"": ""BZ"", ""is_mobile"": true}" 4028,2,88,2017-03-20 13:35:16,http://rodriguez.co/braeden.anderson,,19.204.208.228,"{""location"": ""TV"", ""is_mobile"": true}" 4029,2,88,2017-04-04 02:04:29,http://brakus.biz/laila.gleason,,254.194.189.12,"{""location"": ""FI"", ""is_mobile"": true}" 4030,2,88,2017-06-07 14:44:32,http://rathkeeling.com/alexandro.pacocha,,224.55.66.95,"{""location"": ""ZW"", ""is_mobile"": false}" 4031,2,88,2017-05-17 15:26:36,http://walsh.net/edna.flatley,,214.250.168.89,"{""location"": ""PW"", ""is_mobile"": false}" 4032,2,88,2017-02-08 14:51:51,http://koepp.name/niko.lemke,,166.54.154.216,"{""location"": ""BJ"", ""is_mobile"": true}" 4033,2,88,2016-12-31 06:12:49,http://conn.org/evalyn,,86.187.55.244,"{""location"": ""MN"", ""is_mobile"": true}" 4034,2,88,2017-03-20 15:28:49,http://connreilly.io/flo,,138.2.175.22,"{""location"": ""BV"", ""is_mobile"": true}" 4035,2,88,2017-05-14 05:15:59,http://kunde.org/dolly,,245.175.173.217,"{""location"": ""ZM"", ""is_mobile"": false}" 4036,2,88,2017-02-17 17:26:20,http://mclaughlinhahn.org/bernadette_bartoletti,,108.78.231.12,"{""location"": ""BM"", ""is_mobile"": true}" 4037,2,88,2017-03-25 07:35:43,http://friesen.org/neal,,201.65.68.167,"{""location"": ""EC"", ""is_mobile"": false}" 4038,2,88,2017-04-15 16:38:11,http://lemke.info/justice_schmidt,,158.62.43.42,"{""location"": ""HR"", ""is_mobile"": true}" 4039,2,88,2017-05-30 02:46:20,http://mayer.com/grayce.bernier,,43.177.56.247,"{""location"": ""SB"", ""is_mobile"": true}" 4040,2,89,2017-06-09 18:13:24,http://eichmann.co/cierra,,120.68.44.201,"{""location"": ""FJ"", ""is_mobile"": false}" 4041,2,89,2017-04-27 02:33:06,http://mccullough.name/aliza_gerhold,,124.165.245.174,"{""location"": ""GT"", ""is_mobile"": false}" 4042,2,89,2016-12-16 22:14:28,http://langosh.biz/esther,,186.217.227.50,"{""location"": ""SO"", ""is_mobile"": false}" 4043,2,89,2017-05-03 18:32:34,http://welchcruickshank.info/lauriane,,148.94.83.48,"{""location"": ""CF"", ""is_mobile"": false}" 4044,2,89,2017-05-12 11:37:34,http://ko.io/will_grady,,118.18.39.87,"{""location"": ""SR"", ""is_mobile"": false}" 4045,2,89,2017-03-20 23:54:27,http://quitzon.org/meredith,,26.246.150.114,"{""location"": ""HU"", ""is_mobile"": false}" 4046,2,89,2017-02-03 06:50:41,http://larson.co/kiara.lemke,,70.50.11.193,"{""location"": ""MA"", ""is_mobile"": false}" 4047,2,89,2017-02-07 05:40:42,http://mayer.io/joana.walsh,,186.161.124.161,"{""location"": ""PR"", ""is_mobile"": true}" 4048,2,89,2017-04-08 16:34:40,http://kemmerterry.co/merl,,222.45.75.242,"{""location"": ""MO"", ""is_mobile"": false}" 4049,2,89,2017-05-20 21:01:40,http://kihndicki.biz/isom.wilkinson,,221.226.34.68,"{""location"": ""ME"", ""is_mobile"": true}" 4050,2,89,2016-12-30 00:07:25,http://cartwright.com/braden_johnson,,181.241.59.98,"{""location"": ""AI"", ""is_mobile"": true}" 4051,2,89,2017-01-27 00:27:01,http://wehner.io/cody,,223.149.19.139,"{""location"": ""LB"", ""is_mobile"": true}" 4052,2,89,2016-12-27 07:06:47,http://ruel.net/arvel,,235.162.183.209,"{""location"": ""CD"", ""is_mobile"": true}" 10942,4,245,2017-03-30 16:28:31,http://prosacco.net/rylee,,146.72.32.222,"{""location"": ""GQ"", ""is_mobile"": false}" 10943,4,245,2017-02-22 12:32:49,http://rempel.org/lucio,,38.44.61.56,"{""location"": ""UY"", ""is_mobile"": false}" 10944,4,245,2016-12-29 08:26:02,http://labadie.com/justina.erdman,,44.115.143.233,"{""location"": ""AI"", ""is_mobile"": false}" 10945,4,245,2017-01-11 20:21:36,http://thompson.biz/jacklyn_rohan,,150.12.119.162,"{""location"": ""AG"", ""is_mobile"": false}" 10946,4,245,2017-03-28 16:57:55,http://kutch.com/kara.quitzon,,212.74.247.82,"{""location"": ""BJ"", ""is_mobile"": true}" 10947,4,245,2017-04-18 13:46:11,http://kohlerwalker.co/presley,,175.59.52.63,"{""location"": ""QA"", ""is_mobile"": false}" 10948,4,245,2017-02-26 23:45:00,http://rodriguezemard.co/samantha.barton,,33.111.92.164,"{""location"": ""UZ"", ""is_mobile"": false}" 10949,4,245,2017-05-15 05:24:20,http://mohr.com/odell_wolff,,150.116.181.126,"{""location"": ""LA"", ""is_mobile"": false}" 10950,4,245,2017-06-11 02:49:48,http://romagueradouglas.io/ebba_terry,,6.46.229.183,"{""location"": ""KY"", ""is_mobile"": false}" 10951,4,245,2017-04-29 10:02:44,http://vandervortspencer.name/odell,,21.18.109.190,"{""location"": ""KY"", ""is_mobile"": true}" 10952,4,245,2017-06-02 22:35:29,http://nader.org/hellen,,171.138.152.67,"{""location"": ""TW"", ""is_mobile"": false}" 10953,4,245,2017-04-08 12:46:43,http://gleichner.info/katharina.nolan,,235.12.102.192,"{""location"": ""CN"", ""is_mobile"": false}" 10954,4,245,2017-02-03 09:30:14,http://strosin.co/aaliyah.vonrueden,,162.80.206.71,"{""location"": ""SL"", ""is_mobile"": true}" 10955,4,245,2016-12-26 13:23:38,http://corkery.net/mariano,,224.163.203.58,"{""location"": ""SC"", ""is_mobile"": false}" 10956,4,245,2017-05-12 01:40:12,http://lynch.name/rebekah_howell,,235.216.69.222,"{""location"": ""AL"", ""is_mobile"": true}" 10957,4,245,2016-12-18 13:08:18,http://labadie.name/freddie_schulist,,80.140.107.53,"{""location"": ""TV"", ""is_mobile"": false}" 10958,4,245,2016-12-15 04:30:43,http://mayertmueller.org/allen.torp,,172.180.71.49,"{""location"": ""NF"", ""is_mobile"": false}" 10959,4,245,2017-05-10 07:10:02,http://brekkespencer.co/kari,,180.210.179.161,"{""location"": ""RS"", ""is_mobile"": true}" 10960,4,245,2017-03-02 22:14:49,http://funk.org/natalie,,96.232.55.208,"{""location"": ""JO"", ""is_mobile"": false}" 10961,4,245,2017-03-10 00:55:25,http://grimesgleason.io/alyce.casper,,104.56.40.224,"{""location"": ""UA"", ""is_mobile"": true}" 10962,4,245,2017-04-17 02:01:30,http://corwin.io/brooks_gleason,,224.163.224.71,"{""location"": ""NR"", ""is_mobile"": true}" 10963,4,245,2017-01-16 11:05:04,http://kris.com/marcella,,8.152.186.247,"{""location"": ""BQ"", ""is_mobile"": false}" 10964,4,245,2017-03-20 14:20:25,http://schumm.name/patsy,,142.113.2.82,"{""location"": ""TD"", ""is_mobile"": true}" 10965,4,245,2017-01-24 17:01:54,http://little.net/eldon,,74.12.72.196,"{""location"": ""KE"", ""is_mobile"": true}" 10966,4,245,2016-12-29 06:31:13,http://krajcik.co/jayson.oberbrunner,,211.224.238.104,"{""location"": ""QA"", ""is_mobile"": false}" 10967,4,245,2017-06-12 19:06:43,http://johnston.info/jordy,,38.94.198.114,"{""location"": ""SC"", ""is_mobile"": false}" 10968,4,245,2017-03-01 10:37:11,http://morar.io/lily,,31.113.206.244,"{""location"": ""BF"", ""is_mobile"": true}" 10969,4,245,2017-04-21 14:46:40,http://klocko.net/taylor,,18.173.144.212,"{""location"": ""ZA"", ""is_mobile"": false}" 10970,4,245,2017-02-28 03:16:20,http://ortiz.name/jesus_heel,,220.175.217.88,"{""location"": ""UY"", ""is_mobile"": false}" 10971,4,245,2017-06-11 06:42:52,http://stanton.co/marlee_hills,,137.52.46.177,"{""location"": ""TF"", ""is_mobile"": true}" 10972,4,245,2017-04-15 11:18:08,http://cummerata.com/lucious,,108.216.253.160,"{""location"": ""FO"", ""is_mobile"": false}" 10973,4,245,2017-01-08 05:47:27,http://murphystreich.net/devan.paucek,,155.28.215.176,"{""location"": ""JM"", ""is_mobile"": false}" 10974,4,245,2017-02-21 12:41:14,http://vandervort.com/jonathan_walsh,,69.40.237.38,"{""location"": ""BN"", ""is_mobile"": false}" 10975,4,245,2017-02-18 20:41:13,http://denesik.name/kraig,,204.83.223.83,"{""location"": ""GR"", ""is_mobile"": true}" 10976,4,245,2017-03-17 00:24:00,http://lehner.biz/julian_kulas,,78.41.221.4,"{""location"": ""SI"", ""is_mobile"": false}" 10977,4,245,2017-04-09 17:33:58,http://altenwerth.org/andreanne_schultz,,100.151.222.153,"{""location"": ""MW"", ""is_mobile"": false}" 10978,4,245,2017-05-13 23:15:33,http://orn.net/jacquelyn_mcdermott,,173.31.9.140,"{""location"": ""BR"", ""is_mobile"": false}" 10979,4,245,2017-02-07 05:59:44,http://medhurstaltenwerth.org/icie_ziemann,,152.142.182.138,"{""location"": ""GS"", ""is_mobile"": true}" 10980,4,245,2017-05-16 14:11:16,http://crookslang.co/roscoe.stoltenberg,,106.96.204.158,"{""location"": ""ES"", ""is_mobile"": true}" 10981,4,245,2017-01-18 16:22:30,http://nolanlarkin.com/june,,56.53.163.114,"{""location"": ""GG"", ""is_mobile"": true}" 10982,4,245,2017-04-26 07:31:53,http://hoeger.io/cora.hyatt,,116.208.10.177,"{""location"": ""IS"", ""is_mobile"": true}" 10983,4,245,2016-12-26 23:03:31,http://sipes.co/green,,5.95.163.74,"{""location"": ""LS"", ""is_mobile"": false}" 10984,4,245,2017-04-28 03:15:45,http://kiehn.name/lera.murazik,,114.144.85.39,"{""location"": ""TZ"", ""is_mobile"": false}" 10985,4,245,2017-02-06 17:29:33,http://whitepaucek.co/marlen,,216.241.60.159,"{""location"": ""LY"", ""is_mobile"": false}" 10986,4,245,2016-12-14 00:25:45,http://emmerich.name/angie.fahey,,173.134.231.74,"{""location"": ""BM"", ""is_mobile"": true}" 10987,4,245,2017-05-12 22:49:26,http://lockman.info/rae.stracke,,247.254.250.64,"{""location"": ""BT"", ""is_mobile"": true}" 10988,4,246,2017-05-04 18:39:22,http://bechtelaraltenwerth.info/haven.schowalter,,252.60.12.251,"{""location"": ""IQ"", ""is_mobile"": false}" 10989,4,246,2017-03-01 02:34:26,http://simonis.net/merl_denesik,,200.92.155.40,"{""location"": ""GW"", ""is_mobile"": true}" 10990,4,246,2017-02-19 02:56:19,http://lesch.info/leora_tremblay,,187.10.50.39,"{""location"": ""NO"", ""is_mobile"": true}" 10991,4,246,2017-01-02 22:27:07,http://mcclure.io/alfred_kilback,,174.38.20.197,"{""location"": ""CK"", ""is_mobile"": true}" 10992,4,246,2017-04-28 17:15:56,http://dibbert.name/orrin,,67.100.216.167,"{""location"": ""SA"", ""is_mobile"": false}" 10993,4,246,2017-06-10 19:51:58,http://schoen.name/amiya,,131.225.90.141,"{""location"": ""LR"", ""is_mobile"": true}" 10994,4,246,2017-05-09 18:19:00,http://ondricka.co/max_oconner,,230.242.94.79,"{""location"": ""AE"", ""is_mobile"": false}" 10995,4,246,2017-03-03 01:12:39,http://doylegibson.io/anderson.vandervort,,136.189.75.12,"{""location"": ""AO"", ""is_mobile"": true}" 10996,4,246,2017-02-16 05:23:09,http://gerhold.name/allen_mraz,,101.220.226.220,"{""location"": ""BA"", ""is_mobile"": true}" 7964,3,175,2017-03-26 06:37:28,http://halvorsonbartell.info/reynold.ruecker,0.1770922293,242.56.123.124,"{""location"": ""MH"", ""is_mobile"": false}" 7965,3,175,2017-02-21 20:56:09,http://armstrong.io/claria,0.3949878181,62.55.198.161,"{""location"": ""US"", ""is_mobile"": true}" 7966,3,175,2017-02-18 07:45:41,http://heel.net/laney,0.0015151864,211.142.76.235,"{""location"": ""LK"", ""is_mobile"": true}" 7967,3,175,2017-02-03 22:48:28,http://ryan.io/arjun_marvin,0.5891185421,189.251.172.209,"{""location"": ""GA"", ""is_mobile"": true}" 7968,3,175,2017-01-18 11:04:06,http://price.name/meggie_goyette,0.8782751706,202.202.135.239,"{""location"": ""GP"", ""is_mobile"": true}" 7969,3,175,2017-02-17 14:46:48,http://langworth.org/fausto,0.2026356405,217.125.64.197,"{""location"": ""DO"", ""is_mobile"": true}" 7970,3,175,2017-05-22 22:28:45,http://whitewehner.info/jailyn.herzog,0.7340996480,42.162.156.198,"{""location"": ""BM"", ""is_mobile"": false}" 7971,3,175,2017-02-24 10:12:32,http://donnelly.biz/roger_kihn,0.8375209830,82.53.121.148,"{""location"": ""MR"", ""is_mobile"": true}" 7972,3,176,2017-05-12 04:44:16,http://keler.net/keyon.oconnell,0.8242732446,81.123.33.134,"{""location"": ""LI"", ""is_mobile"": true}" 7973,3,176,2017-02-14 21:02:00,http://hyattarmstrong.net/camden,0.7097888392,153.101.35.206,"{""location"": ""BZ"", ""is_mobile"": false}" 7974,3,176,2017-02-27 12:02:59,http://crooksbraun.net/eldridge,0.1403471327,90.243.14.23,"{""location"": ""AX"", ""is_mobile"": false}" 7975,3,176,2017-02-06 16:07:29,http://kilback.co/armando,0.8880322062,163.132.63.225,"{""location"": ""AS"", ""is_mobile"": false}" 7976,3,176,2017-02-12 11:17:06,http://kirlin.com/elva.grimes,0.2882909818,243.216.143.95,"{""location"": ""FK"", ""is_mobile"": true}" 7977,3,176,2017-02-22 20:09:20,http://wintheisercronin.info/adrienne_runolfsdottir,0.3190290468,187.132.16.224,"{""location"": ""SX"", ""is_mobile"": true}" 7978,3,176,2017-02-12 05:41:24,http://romaguera.co/mathilde,0.9978610297,134.69.223.37,"{""location"": ""CK"", ""is_mobile"": false}" 7979,3,176,2017-04-18 11:08:15,http://sawayn.co/luisa,0.9029364119,52.81.229.164,"{""location"": ""SE"", ""is_mobile"": false}" 7980,3,176,2016-12-21 00:41:59,http://maggiokoelpin.biz/broderick,0.0314982963,58.232.42.8,"{""location"": ""PW"", ""is_mobile"": true}" 7981,3,176,2016-12-17 04:34:40,http://bartell.info/rylee_kihn,0.5857787742,195.181.193.147,"{""location"": ""SM"", ""is_mobile"": true}" 7982,3,176,2017-02-19 05:59:47,http://hilll.biz/benton.stiedemann,0.1728682252,146.4.164.29,"{""location"": ""GW"", ""is_mobile"": true}" 7983,3,176,2017-06-09 10:01:56,http://borer.net/aaron_kunze,0.9053722242,29.116.32.144,"{""location"": ""ZM"", ""is_mobile"": false}" 7984,3,176,2017-03-31 23:42:12,http://robel.net/vincenza,0.7119945219,145.47.126.102,"{""location"": ""TD"", ""is_mobile"": true}" 7985,3,176,2016-12-24 00:53:12,http://terryjenkins.co/archibald_turcotte,0.5183296490,224.241.24.9,"{""location"": ""BV"", ""is_mobile"": false}" 7986,3,176,2017-06-06 05:00:40,http://halvorsonmorar.info/albert,0.9113281081,70.188.75.108,"{""location"": ""NO"", ""is_mobile"": false}" 7987,3,176,2017-05-22 22:54:20,http://willthiel.com/joanny,0.5353973714,6.42.83.159,"{""location"": ""HU"", ""is_mobile"": true}" 7988,3,176,2016-12-26 15:10:16,http://okeefe.co/louie,0.8175263848,250.223.15.215,"{""location"": ""MC"", ""is_mobile"": false}" 7989,3,176,2017-04-10 20:53:02,http://hammesherman.biz/raymundo_romaguera,0.3561497743,144.253.24.111,"{""location"": ""AR"", ""is_mobile"": false}" 7990,3,176,2017-01-12 09:28:09,http://veum.org/erling_greenholt,0.1661255751,159.152.10.56,"{""location"": ""TK"", ""is_mobile"": true}" 7991,3,176,2017-05-18 03:31:38,http://krajciklubowitz.org/tyra,0.1536691130,234.197.75.33,"{""location"": ""MY"", ""is_mobile"": false}" 7992,3,176,2017-01-17 18:52:52,http://walker.org/luigi,0.2574998039,198.25.58.172,"{""location"": ""SH"", ""is_mobile"": true}" 7993,3,176,2017-02-23 11:14:00,http://swaniawskicain.biz/alexys_deckow,0.4698242346,249.136.15.48,"{""location"": ""RE"", ""is_mobile"": false}" 7994,3,176,2017-05-27 20:04:57,http://hahn.name/darron.hegmann,0.6833504349,179.68.51.187,"{""location"": ""CC"", ""is_mobile"": false}" 7995,3,176,2017-04-30 21:25:34,http://altenwerth.co/berry_pouros,0.3840477417,41.148.12.75,"{""location"": ""MD"", ""is_mobile"": true}" 7996,3,176,2017-05-24 06:37:21,http://larsonschaden.io/clyde,0.8700829022,88.128.80.5,"{""location"": ""ML"", ""is_mobile"": true}" 7997,3,176,2017-01-08 16:56:43,http://mccullough.info/roie,0.5410355983,200.235.125.157,"{""location"": ""HK"", ""is_mobile"": false}" 7998,3,176,2017-02-16 15:36:19,http://streichgleason.org/erik.lindgren,0.6796937281,103.17.82.70,"{""location"": ""FK"", ""is_mobile"": false}" 7999,3,176,2017-06-08 16:38:50,http://torphaag.info/jared,0.8477595758,106.195.199.145,"{""location"": ""GL"", ""is_mobile"": false}" 8000,3,176,2017-01-29 07:20:37,http://keler.biz/libby.herman,0.2792333661,46.86.149.136,"{""location"": ""TV"", ""is_mobile"": false}" 8001,3,176,2017-03-31 17:07:45,http://braun.io/terrell_johns,0.8383616576,152.217.34.46,"{""location"": ""MR"", ""is_mobile"": true}" 8002,3,176,2017-04-02 22:21:30,http://kris.com/jairo_casper,0.4148156934,209.168.41.244,"{""location"": ""CA"", ""is_mobile"": false}" 8003,3,176,2017-05-17 18:07:34,http://kreiger.net/alysa,0.1652377375,36.85.157.182,"{""location"": ""CY"", ""is_mobile"": false}" 8004,3,177,2017-05-03 08:35:11,http://powlowskiconsidine.net/hilbert.rowe,0.2100720049,154.161.18.90,"{""location"": ""DE"", ""is_mobile"": true}" 8005,3,177,2017-02-24 15:03:00,http://aufderhar.com/nyah,0.0413644111,211.84.43.167,"{""location"": ""DJ"", ""is_mobile"": true}" 8006,3,177,2017-06-03 20:39:46,http://dooley.com/kailey,0.5118417278,156.152.208.245,"{""location"": ""MO"", ""is_mobile"": false}" 8007,3,177,2017-04-27 13:34:22,http://casper.biz/jordane_nitzsche,0.4774063520,134.230.5.152,"{""location"": ""VE"", ""is_mobile"": true}" 8008,3,177,2017-04-05 13:00:23,http://gislasonullrich.org/titus_gibson,0.3042262956,170.247.149.161,"{""location"": ""RO"", ""is_mobile"": true}" 8009,3,177,2017-03-25 02:42:19,http://gutkowskifritsch.org/tony.metz,0.5591993881,91.166.86.104,"{""location"": ""KH"", ""is_mobile"": true}" 8010,3,177,2017-01-28 21:36:22,http://hackettgerlach.org/danika,0.7915168587,193.135.91.134,"{""location"": ""BJ"", ""is_mobile"": false}" 8011,3,177,2017-06-05 04:59:56,http://abernathyhamill.org/emile_hackett,0.0641388188,14.149.191.249,"{""location"": ""AZ"", ""is_mobile"": false}" 8012,3,177,2017-03-05 02:16:34,http://moenhickle.biz/katrine_wisozk,0.2494156840,36.5.126.97,"{""location"": ""MD"", ""is_mobile"": false}" 8013,3,177,2017-03-14 20:49:34,http://mcdermottwolf.info/alvena,0.5946420682,217.161.168.235,"{""location"": ""GE"", ""is_mobile"": true}" 8014,3,177,2017-06-08 09:56:36,http://walker.biz/bradford,0.5582160877,227.231.98.150,"{""location"": ""BT"", ""is_mobile"": true}" 8015,3,177,2016-12-22 12:24:07,http://bruen.co/dan,0.8877522429,33.19.107.34,"{""location"": ""MS"", ""is_mobile"": false}" 13891,5,311,2017-01-29 13:11:25,http://hammespurdy.io/preston,,234.132.209.83,"{""location"": ""VC"", ""is_mobile"": false}" 13892,5,311,2017-02-27 16:51:41,http://schulistheathcote.io/ebony.johns,,59.246.191.125,"{""location"": ""YT"", ""is_mobile"": true}" 13893,5,311,2017-03-15 13:04:15,http://rippin.info/brendon,,226.125.228.8,"{""location"": ""BI"", ""is_mobile"": true}" 13894,5,311,2017-02-04 13:55:35,http://gottliebpagac.co/trudie,,27.218.40.46,"{""location"": ""BT"", ""is_mobile"": true}" 13895,5,311,2017-04-18 07:38:50,http://abernathy.com/elenor,,176.123.127.234,"{""location"": ""JE"", ""is_mobile"": false}" 13896,5,311,2017-03-12 09:42:10,http://cummingsbergstrom.com/arvid.torp,,210.198.231.42,"{""location"": ""PK"", ""is_mobile"": true}" 13897,5,311,2016-12-19 23:23:42,http://runolfsdottir.io/jamir,,237.41.124.34,"{""location"": ""TD"", ""is_mobile"": false}" 13898,5,311,2017-01-28 21:51:58,http://jenkins.info/ryann,,138.121.64.200,"{""location"": ""FJ"", ""is_mobile"": true}" 13899,5,311,2017-05-20 11:00:55,http://conroy.com/wilburn_gerlach,,107.27.195.112,"{""location"": ""HR"", ""is_mobile"": false}" 13900,5,311,2017-04-05 09:01:46,http://murray.io/lafayette_wyman,,147.150.101.16,"{""location"": ""AE"", ""is_mobile"": true}" 13901,5,311,2017-05-14 04:47:58,http://gislason.org/marian_dubuque,,74.214.64.66,"{""location"": ""LS"", ""is_mobile"": false}" 13902,5,312,2017-06-01 03:34:05,http://schneider.org/roel_ledner,,111.162.233.145,"{""location"": ""CM"", ""is_mobile"": false}" 13903,5,312,2017-06-01 23:30:14,http://skiles.name/dora_rosenbaum,,5.84.89.45,"{""location"": ""KZ"", ""is_mobile"": true}" 13904,5,312,2017-02-25 02:17:58,http://dickihudson.io/tyra,,134.251.6.249,"{""location"": ""GL"", ""is_mobile"": false}" 13905,5,312,2017-04-26 09:17:21,http://lindgren.com/carlos_hamill,,180.141.207.73,"{""location"": ""TF"", ""is_mobile"": true}" 13906,5,312,2017-04-06 20:30:33,http://keeling.info/cecil_huel,,144.95.54.214,"{""location"": ""MM"", ""is_mobile"": true}" 13907,5,312,2017-04-28 08:05:02,http://goldner.org/aron,,7.10.76.29,"{""location"": ""TO"", ""is_mobile"": false}" 13908,5,312,2017-05-16 18:14:01,http://boylerosenbaum.org/roy.kuvalis,,229.57.45.120,"{""location"": ""PS"", ""is_mobile"": true}" 13909,5,312,2017-03-23 00:05:32,http://jakubowskipredovic.info/dylan,,201.91.154.139,"{""location"": ""UG"", ""is_mobile"": false}" 13910,5,312,2017-01-17 03:47:40,http://stark.co/beulah,,211.123.251.78,"{""location"": ""GT"", ""is_mobile"": false}" 13911,5,312,2017-01-27 00:34:31,http://howe.net/camila.satterfield,,84.248.62.228,"{""location"": ""LR"", ""is_mobile"": false}" 13912,5,312,2017-01-23 05:45:46,http://yundt.co/logan,,201.116.228.245,"{""location"": ""MK"", ""is_mobile"": true}" 13913,5,312,2017-05-18 10:09:38,http://lakinstamm.io/ephraim,,95.104.218.23,"{""location"": ""AR"", ""is_mobile"": false}" 13914,5,312,2017-03-30 07:44:53,http://roob.net/summer,,52.223.223.85,"{""location"": ""LK"", ""is_mobile"": false}" 13915,5,312,2017-05-11 02:07:31,http://kling.info/dylan_reinger,,148.117.17.109,"{""location"": ""MX"", ""is_mobile"": true}" 13916,5,312,2017-01-01 10:50:06,http://beer.org/aniyah,,170.212.130.152,"{""location"": ""KH"", ""is_mobile"": false}" 13917,5,312,2017-04-19 23:45:25,http://wunsch.com/zula_schroeder,,249.15.234.74,"{""location"": ""LR"", ""is_mobile"": true}" 13918,5,312,2017-01-05 08:57:11,http://yundt.biz/leif,,121.117.189.123,"{""location"": ""BF"", ""is_mobile"": false}" 13919,5,312,2017-06-14 00:03:04,http://hilperthagenes.io/eileen_bahringer,,146.199.168.246,"{""location"": ""KE"", ""is_mobile"": true}" 13920,5,312,2017-03-17 07:09:27,http://bergstrom.com/bennie.osinski,,31.222.154.154,"{""location"": ""FM"", ""is_mobile"": false}" 13921,5,312,2017-03-11 20:08:00,http://koelpin.org/yasmine_west,,198.187.196.251,"{""location"": ""SB"", ""is_mobile"": false}" 13922,5,312,2017-05-23 20:42:44,http://hintz.info/ellie_witting,,215.53.222.204,"{""location"": ""KM"", ""is_mobile"": true}" 13923,5,312,2017-05-26 21:20:37,http://mckenzie.info/vincent.rolfson,,50.58.82.236,"{""location"": ""YT"", ""is_mobile"": true}" 13924,5,312,2017-04-24 23:44:45,http://dubuquekerluke.co/foster,,225.194.228.235,"{""location"": ""PG"", ""is_mobile"": false}" 13925,5,312,2017-04-10 11:48:05,http://eichmann.co/reese,,68.234.73.187,"{""location"": ""VA"", ""is_mobile"": false}" 13926,5,312,2017-03-10 07:26:13,http://padberg.name/lizzie_brown,,185.134.203.171,"{""location"": ""PE"", ""is_mobile"": false}" 13927,5,312,2017-01-05 21:45:06,http://zemlak.info/timmothy.hegmann,,30.34.225.59,"{""location"": ""UG"", ""is_mobile"": false}" 13928,5,312,2017-01-17 14:59:46,http://windler.co/addie,,10.44.111.85,"{""location"": ""GB"", ""is_mobile"": true}" 13929,5,312,2017-02-25 21:28:05,http://homenick.net/hillard.flatley,,200.15.72.193,"{""location"": ""BV"", ""is_mobile"": false}" 13930,5,312,2017-05-18 14:15:34,http://tremblaylesch.com/adah.kuhic,,48.91.107.202,"{""location"": ""IT"", ""is_mobile"": true}" 13931,5,312,2017-04-11 05:43:16,http://wilderman.com/cyril,,245.220.7.163,"{""location"": ""HN"", ""is_mobile"": true}" 13932,5,312,2017-04-09 15:56:38,http://monahan.io/noelia,,60.227.120.72,"{""location"": ""TM"", ""is_mobile"": true}" 13933,5,312,2017-01-08 19:27:47,http://wiegandkuhic.name/laverna,,248.86.85.146,"{""location"": ""TK"", ""is_mobile"": false}" 13934,5,312,2017-01-16 18:15:29,http://pfeffer.io/leopold.flatley,,169.114.238.6,"{""location"": ""LB"", ""is_mobile"": true}" 13935,5,312,2017-06-07 11:57:40,http://graham.com/will_toy,,232.87.160.149,"{""location"": ""SL"", ""is_mobile"": false}" 13936,5,312,2016-12-26 02:04:27,http://prosacco.biz/lela.ratke,,105.7.144.49,"{""location"": ""BH"", ""is_mobile"": false}" 13937,5,312,2017-04-05 21:13:04,http://connellysawayn.co/rodrigo_rowe,,41.173.175.228,"{""location"": ""IS"", ""is_mobile"": false}" 13938,5,312,2017-06-12 23:12:07,http://bogan.biz/bonita,,167.164.54.165,"{""location"": ""LV"", ""is_mobile"": false}" 13939,5,312,2017-02-04 11:14:10,http://schuster.net/sidney.wyman,,99.88.69.219,"{""location"": ""KR"", ""is_mobile"": false}" 13940,5,312,2016-12-31 22:04:10,http://murrayfunk.com/monty.gutmann,,24.228.214.17,"{""location"": ""RU"", ""is_mobile"": false}" 13941,5,312,2017-02-14 15:17:08,http://marks.com/aliza.grady,,234.200.61.127,"{""location"": ""IN"", ""is_mobile"": true}" 13942,5,312,2017-03-12 16:03:46,http://rolfson.biz/tre_hane,,215.253.184.227,"{""location"": ""CC"", ""is_mobile"": false}" 13943,5,312,2017-02-21 09:27:49,http://parisian.biz/jennyfer,,154.102.13.34,"{""location"": ""GQ"", ""is_mobile"": true}" 13944,5,312,2017-02-24 23:55:42,http://pollich.co/pedro,,125.77.73.133,"{""location"": ""YT"", ""is_mobile"": false}" 13945,5,312,2017-05-09 03:52:47,http://boehm.info/joanie,,2.52.96.59,"{""location"": ""CU"", ""is_mobile"": true}" 13946,5,312,2017-02-14 09:17:16,http://lockman.info/carolyn,,159.189.66.122,"{""location"": ""GR"", ""is_mobile"": true}" 16873,6,382,2017-02-08 20:39:22,http://toy.co/felix,,33.149.77.90,"{""location"": ""FK"", ""is_mobile"": true}" 16874,6,382,2017-05-04 05:32:48,http://creminbrakus.io/wilber_bayer,,64.206.121.83,"{""location"": ""MA"", ""is_mobile"": false}" 16875,6,382,2017-04-15 13:20:48,http://binsmaggio.net/melvina.hamill,,57.225.199.201,"{""location"": ""MG"", ""is_mobile"": true}" 16876,6,382,2017-05-30 15:58:55,http://gradyrippin.com/kevon,,137.52.171.215,"{""location"": ""AQ"", ""is_mobile"": false}" 16877,6,382,2017-02-19 02:50:25,http://graham.info/mark_kunde,,49.230.127.71,"{""location"": ""KE"", ""is_mobile"": false}" 16878,6,382,2017-03-06 00:00:54,http://haley.io/daphney.deckow,,228.249.10.254,"{""location"": ""CV"", ""is_mobile"": true}" 16879,6,382,2017-06-03 12:29:12,http://macejkovic.biz/nellie,,245.110.112.195,"{""location"": ""FR"", ""is_mobile"": false}" 16880,6,382,2017-04-15 10:25:09,http://crist.name/vita,,71.185.164.53,"{""location"": ""HR"", ""is_mobile"": true}" 16881,6,382,2016-12-23 04:49:50,http://koelpin.com/tania.kilback,,111.178.236.200,"{""location"": ""HN"", ""is_mobile"": true}" 16882,6,382,2016-12-13 13:38:41,http://kozeyromaguera.io/henri.macejkovic,,168.53.160.125,"{""location"": ""HR"", ""is_mobile"": false}" 16883,6,382,2017-04-25 05:38:40,http://altenwerthcrona.name/lucious,,54.171.3.121,"{""location"": ""FO"", ""is_mobile"": false}" 16884,6,382,2017-01-20 05:59:05,http://leuschke.info/alphonso,,61.207.151.108,"{""location"": ""WS"", ""is_mobile"": true}" 16885,6,382,2017-05-23 14:57:00,http://denesik.org/bertha,,25.162.63.36,"{""location"": ""IE"", ""is_mobile"": true}" 16886,6,382,2017-04-09 11:07:22,http://kunze.net/america,,154.100.162.186,"{""location"": ""AM"", ""is_mobile"": true}" 16887,6,382,2017-01-12 18:56:04,http://wizaflatley.co/beulah_langworth,,250.197.75.121,"{""location"": ""UZ"", ""is_mobile"": true}" 16888,6,382,2016-12-14 06:36:05,http://haneoreilly.biz/dasia,,23.145.16.174,"{""location"": ""YE"", ""is_mobile"": false}" 16889,6,382,2017-02-13 08:00:08,http://armstrong.net/floyd_hudson,,218.131.217.98,"{""location"": ""AD"", ""is_mobile"": true}" 16890,6,382,2017-02-26 11:25:06,http://doyle.net/marjolaine.dach,,129.161.192.114,"{""location"": ""CN"", ""is_mobile"": true}" 16891,6,382,2017-01-22 02:22:44,http://armstrongtorp.co/tod_brakus,,159.250.110.220,"{""location"": ""OM"", ""is_mobile"": false}" 16892,6,382,2017-01-09 22:03:44,http://uptonfahey.io/amie,,104.167.146.220,"{""location"": ""CY"", ""is_mobile"": true}" 16893,6,382,2017-03-12 16:05:19,http://heathcote.biz/fletcher,,91.107.198.222,"{""location"": ""EC"", ""is_mobile"": true}" 16894,6,382,2017-03-04 21:02:42,http://stantonjast.net/beie.schmitt,,151.102.123.185,"{""location"": ""AW"", ""is_mobile"": true}" 16895,6,382,2017-04-22 01:02:16,http://haaggaylord.co/kale,,137.53.162.46,"{""location"": ""JM"", ""is_mobile"": false}" 16896,6,382,2017-06-02 09:27:53,http://wiza.org/connie.bode,,23.15.18.131,"{""location"": ""FI"", ""is_mobile"": false}" 16897,6,382,2017-05-05 06:48:49,http://wiza.org/noemi_hintz,,65.78.46.66,"{""location"": ""CG"", ""is_mobile"": true}" 16898,6,382,2017-03-14 10:51:58,http://mertz.io/gennaro.ernser,,102.199.88.40,"{""location"": ""CH"", ""is_mobile"": true}" 16899,6,382,2017-05-31 12:59:41,http://herzogweinat.co/jeika,,231.34.206.111,"{""location"": ""HT"", ""is_mobile"": false}" 16900,6,382,2017-05-14 13:35:20,http://bartonhodkiewicz.io/casimer.schuppe,,190.21.60.193,"{""location"": ""HK"", ""is_mobile"": true}" 16901,6,382,2017-04-10 16:08:22,http://hyattwillms.com/david,,178.87.96.110,"{""location"": ""TN"", ""is_mobile"": false}" 16902,6,382,2017-01-08 03:59:49,http://moriette.biz/mario,,148.13.15.157,"{""location"": ""SG"", ""is_mobile"": false}" 16903,6,382,2016-12-30 04:13:02,http://hand.com/loy,,122.245.222.228,"{""location"": ""GI"", ""is_mobile"": true}" 16904,6,382,2017-03-29 14:54:57,http://kuphal.io/tad,,137.22.25.230,"{""location"": ""TV"", ""is_mobile"": false}" 16905,6,382,2016-12-15 02:14:28,http://wisoky.com/hilbert,,14.202.56.184,"{""location"": ""KR"", ""is_mobile"": true}" 16906,6,382,2017-05-07 20:18:38,http://champlin.name/giuseppe.mcglynn,,249.54.233.87,"{""location"": ""CI"", ""is_mobile"": true}" 16907,6,383,2017-02-02 02:00:25,http://ondrickaschuppe.com/christiana_conn,,52.117.195.242,"{""location"": ""PF"", ""is_mobile"": true}" 16908,6,383,2017-03-03 04:47:31,http://ruelmcdermott.net/efrain,,74.59.146.157,"{""location"": ""SJ"", ""is_mobile"": false}" 16909,6,383,2017-05-27 00:01:56,http://leuschke.co/kaycee,,179.100.117.101,"{""location"": ""MP"", ""is_mobile"": false}" 16910,6,383,2017-04-04 17:23:05,http://keebler.org/dayana.crooks,,57.103.242.253,"{""location"": ""KH"", ""is_mobile"": true}" 16911,6,383,2017-03-26 02:14:50,http://douglaimonis.co/jackeline,,180.24.246.186,"{""location"": ""VE"", ""is_mobile"": true}" 16912,6,383,2017-01-20 01:43:42,http://turnerbednar.net/ru_lind,,7.58.42.107,"{""location"": ""MA"", ""is_mobile"": true}" 16913,6,383,2017-04-11 16:47:03,http://gleasonrippin.name/leilani,,66.183.29.224,"{""location"": ""SY"", ""is_mobile"": true}" 16914,6,383,2017-03-29 07:37:14,http://nader.info/wilbert,,187.38.83.97,"{""location"": ""DO"", ""is_mobile"": true}" 16915,6,383,2017-01-04 07:20:36,http://heaney.com/linwood,,50.226.123.5,"{""location"": ""DE"", ""is_mobile"": false}" 16916,6,383,2016-12-29 21:12:06,http://grant.info/ismael,,135.246.76.113,"{""location"": ""FR"", ""is_mobile"": true}" 16917,6,383,2017-03-14 01:59:25,http://hegmann.name/keven.bosco,,227.250.17.14,"{""location"": ""BI"", ""is_mobile"": false}" 16918,6,383,2017-01-13 19:48:22,http://pfannerstillbeer.org/marques.watsica,,194.236.200.243,"{""location"": ""MH"", ""is_mobile"": true}" 16919,6,383,2017-05-26 02:08:52,http://donnelly.io/kaden.roob,,178.118.219.145,"{""location"": ""BS"", ""is_mobile"": false}" 16920,6,383,2016-12-25 17:34:33,http://kub.io/armani,,17.151.209.212,"{""location"": ""NE"", ""is_mobile"": false}" 16921,6,383,2017-05-14 11:55:50,http://jacobi.org/cleo.paucek,,239.2.66.232,"{""location"": ""LT"", ""is_mobile"": true}" 16922,6,383,2017-03-13 18:23:09,http://corwin.io/jada.hegmann,,171.167.189.146,"{""location"": ""IQ"", ""is_mobile"": false}" 16923,6,383,2017-05-27 21:34:48,http://kuhic.name/reie_mayer,,183.218.202.31,"{""location"": ""ME"", ""is_mobile"": false}" 16924,6,383,2017-05-27 04:04:32,http://swift.name/zena.walsh,,47.89.107.189,"{""location"": ""JM"", ""is_mobile"": false}" 16925,6,383,2017-03-09 19:46:01,http://carrollschulist.biz/lyla,,101.102.211.158,"{""location"": ""MU"", ""is_mobile"": false}" 16926,6,383,2017-02-18 19:58:49,http://hellerschaefer.name/raymundo.ondricka,,93.10.67.204,"{""location"": ""MT"", ""is_mobile"": false}" 16927,6,383,2017-03-16 07:33:38,http://gislason.info/gene,,188.98.52.166,"{""location"": ""FK"", ""is_mobile"": true}" 16928,6,383,2017-02-27 14:32:55,http://marksterry.info/raul,,123.119.221.223,"{""location"": ""BF"", ""is_mobile"": false}" 4053,2,89,2017-04-20 06:21:03,http://westwilderman.co/weston,,231.49.65.15,"{""location"": ""TF"", ""is_mobile"": true}" 4054,2,89,2017-01-25 11:45:54,http://effertz.io/jennifer,,176.54.111.32,"{""location"": ""MO"", ""is_mobile"": true}" 4055,2,89,2017-03-04 01:07:34,http://dietrich.net/golda,,254.186.94.37,"{""location"": ""MZ"", ""is_mobile"": false}" 4056,2,89,2017-01-06 08:40:50,http://lemkehudson.net/emmitt,,115.242.87.74,"{""location"": ""SJ"", ""is_mobile"": false}" 4057,2,89,2017-06-03 15:16:15,http://kuhnritchie.co/audrey.damore,,159.232.91.182,"{""location"": ""NL"", ""is_mobile"": true}" 4058,2,89,2017-04-13 09:56:49,http://bins.name/melody.koch,,19.29.89.112,"{""location"": ""AR"", ""is_mobile"": true}" 4059,2,89,2017-04-23 20:50:12,http://feeney.biz/mae,,139.47.157.59,"{""location"": ""GN"", ""is_mobile"": false}" 4060,2,89,2017-06-11 22:57:33,http://dare.info/dejuan.marquardt,,35.151.210.31,"{""location"": ""MQ"", ""is_mobile"": true}" 4061,2,89,2017-06-10 16:10:39,http://satterfieldbergstrom.net/arely,,216.79.49.153,"{""location"": ""MR"", ""is_mobile"": true}" 4062,2,89,2017-04-06 02:26:49,http://ebert.net/shaina,,242.19.179.130,"{""location"": ""LT"", ""is_mobile"": true}" 4063,2,89,2017-05-18 15:10:56,http://connelly.co/jadyn_leannon,,155.239.159.111,"{""location"": ""DJ"", ""is_mobile"": true}" 4064,2,89,2017-06-06 10:21:49,http://gusikowski.com/everett_kris,,24.206.76.172,"{""location"": ""MZ"", ""is_mobile"": true}" 4065,2,89,2017-03-27 22:20:54,http://bernierdicki.net/anjali,,149.221.201.179,"{""location"": ""EE"", ""is_mobile"": false}" 4066,2,89,2017-04-26 09:46:57,http://quitzon.info/demarco,,12.205.145.114,"{""location"": ""PS"", ""is_mobile"": false}" 4067,2,89,2017-01-05 15:30:56,http://kling.org/elna,,184.221.126.98,"{""location"": ""MT"", ""is_mobile"": true}" 4068,2,89,2017-02-15 19:44:49,http://hayes.net/jeremie.huels,,139.157.66.68,"{""location"": ""UZ"", ""is_mobile"": false}" 4069,2,89,2017-02-02 14:07:12,http://langworth.org/pablo,,80.218.213.90,"{""location"": ""IL"", ""is_mobile"": true}" 4070,2,89,2017-03-18 23:09:13,http://renner.info/kendall,,5.105.250.223,"{""location"": ""SD"", ""is_mobile"": true}" 4071,2,89,2017-04-10 04:02:59,http://cole.name/floy_smitham,,154.134.174.160,"{""location"": ""ZW"", ""is_mobile"": true}" 4072,2,89,2017-03-04 17:43:38,http://kautzer.com/alaina_zemlak,,17.124.90.231,"{""location"": ""AI"", ""is_mobile"": true}" 4073,2,89,2017-03-22 08:41:40,http://nader.net/callie,,182.130.134.114,"{""location"": ""CZ"", ""is_mobile"": false}" 4074,2,89,2016-12-30 20:14:17,http://bednarullrich.com/alfred,,8.111.204.89,"{""location"": ""VC"", ""is_mobile"": true}" 4075,2,89,2017-04-29 07:08:50,http://goldner.biz/victoria,,186.196.104.128,"{""location"": ""KM"", ""is_mobile"": false}" 4076,2,89,2017-04-04 23:17:49,http://collierwaters.name/susanna_romaguera,,145.149.212.63,"{""location"": ""NZ"", ""is_mobile"": true}" 4077,2,89,2017-01-22 01:41:27,http://armstrong.co/madeline.haley,,138.158.135.167,"{""location"": ""SH"", ""is_mobile"": false}" 4078,2,89,2017-04-20 11:06:28,http://millerkutch.name/lila,,192.248.5.162,"{""location"": ""CX"", ""is_mobile"": false}" 4079,2,89,2017-06-02 20:05:35,http://lehnerbreitenberg.info/faye,,254.37.214.124,"{""location"": ""DM"", ""is_mobile"": false}" 4080,2,89,2017-02-23 00:39:39,http://oconnerjenkins.name/sophie_fisher,,248.234.122.65,"{""location"": ""GT"", ""is_mobile"": false}" 4081,2,89,2017-01-16 11:06:59,http://romaguerawest.net/bridie,,19.10.233.189,"{""location"": ""MT"", ""is_mobile"": true}" 4082,2,89,2017-01-18 03:30:01,http://cummings.info/paige_moriette,,165.80.9.216,"{""location"": ""IR"", ""is_mobile"": false}" 4083,2,89,2017-05-04 20:58:01,http://cronin.com/barrett,,175.139.148.178,"{""location"": ""YE"", ""is_mobile"": true}" 4084,2,89,2017-02-18 20:42:19,http://kshlerin.io/harmon,,111.216.168.4,"{""location"": ""NU"", ""is_mobile"": false}" 4085,2,89,2017-05-05 17:19:09,http://hintzschultz.co/lois.stroman,,176.245.159.58,"{""location"": ""TM"", ""is_mobile"": false}" 4086,2,89,2016-12-14 23:15:32,http://jones.net/gertrude_stroman,,170.93.42.95,"{""location"": ""PS"", ""is_mobile"": true}" 4087,2,89,2017-03-21 18:09:54,http://hammes.org/lelia,,27.177.51.226,"{""location"": ""PY"", ""is_mobile"": true}" 4088,2,89,2017-05-07 20:46:00,http://schulist.com/hudson.muller,,53.241.25.155,"{""location"": ""SX"", ""is_mobile"": true}" 4089,2,89,2016-12-29 16:27:35,http://hoppe.io/robb_wyman,,132.253.140.28,"{""location"": ""CR"", ""is_mobile"": false}" 4090,2,89,2016-12-30 14:57:40,http://weimann.net/cydney_spencer,,47.208.33.91,"{""location"": ""GE"", ""is_mobile"": false}" 4091,2,89,2017-01-01 20:33:10,http://orn.org/sadie.shanahan,,115.71.51.194,"{""location"": ""DJ"", ""is_mobile"": true}" 4092,2,89,2017-01-26 13:48:33,http://lockman.net/khalil,,105.228.101.180,"{""location"": ""MA"", ""is_mobile"": true}" 4093,2,89,2017-04-08 15:39:46,http://ruel.com/maya,,128.243.100.101,"{""location"": ""MU"", ""is_mobile"": true}" 4094,2,89,2016-12-24 18:18:28,http://kuhn.net/sharon.treutel,,174.98.94.89,"{""location"": ""AX"", ""is_mobile"": false}" 4095,2,89,2017-05-20 12:37:25,http://quitzon.com/chelsey,,117.41.81.118,"{""location"": ""SB"", ""is_mobile"": true}" 4096,2,89,2017-02-19 04:14:27,http://reichelgusikowski.co/forest_jaskolski,,201.224.182.241,"{""location"": ""VE"", ""is_mobile"": false}" 4097,2,89,2016-12-13 16:29:49,http://cremin.biz/jada,,109.105.98.164,"{""location"": ""RW"", ""is_mobile"": true}" 4098,2,89,2017-03-03 07:55:46,http://stanton.co/kacey.torp,,74.105.36.138,"{""location"": ""HR"", ""is_mobile"": false}" 4099,2,90,2017-04-02 17:17:09,http://hackettgoyette.name/lourdes_sauer,,129.66.68.4,"{""location"": ""FJ"", ""is_mobile"": false}" 4100,2,90,2017-02-05 13:22:07,http://pfeffer.com/rosetta.casper,,125.114.51.228,"{""location"": ""KN"", ""is_mobile"": true}" 4101,2,90,2017-02-15 08:34:26,http://padberg.info/zachery_mohr,,165.45.129.254,"{""location"": ""CN"", ""is_mobile"": false}" 4102,2,90,2017-01-05 01:07:57,http://jaskolskilueilwitz.name/amanda.davis,,141.89.8.48,"{""location"": ""HK"", ""is_mobile"": true}" 4103,2,90,2017-03-24 18:30:05,http://steuber.com/mariam,,81.142.111.166,"{""location"": ""ML"", ""is_mobile"": false}" 4104,2,90,2017-04-11 11:13:15,http://harber.org/keyon.gutmann,,65.156.109.18,"{""location"": ""UZ"", ""is_mobile"": true}" 4105,2,90,2017-03-09 11:45:26,http://kovacek.co/idella.runolfsdottir,,60.133.71.106,"{""location"": ""VC"", ""is_mobile"": true}" 4106,2,90,2017-04-11 18:35:29,http://ruecker.biz/danika,,201.157.227.95,"{""location"": ""IT"", ""is_mobile"": false}" 4107,2,90,2017-02-04 07:41:29,http://barrows.co/ariel,,117.151.9.61,"{""location"": ""MC"", ""is_mobile"": false}" 4108,2,90,2017-01-26 01:56:41,http://gloverkozey.org/joey,,53.45.193.180,"{""location"": ""GB"", ""is_mobile"": true}" 10997,4,246,2016-12-22 05:55:58,http://weber.info/lily,,60.244.111.169,"{""location"": ""FM"", ""is_mobile"": true}" 10998,4,246,2016-12-20 07:17:13,http://bauchdooley.biz/vilma,,140.181.79.20,"{""location"": ""LT"", ""is_mobile"": true}" 10999,4,246,2017-03-18 08:32:53,http://kovacek.name/iliana,,46.246.183.58,"{""location"": ""GW"", ""is_mobile"": false}" 11000,4,246,2017-03-14 00:42:39,http://wilkinson.io/elda_wisozk,,14.5.125.162,"{""location"": ""SD"", ""is_mobile"": true}" 11001,4,246,2017-04-26 01:56:34,http://labadie.io/devin,,12.95.38.212,"{""location"": ""SH"", ""is_mobile"": true}" 11002,4,246,2016-12-22 04:05:14,http://kristorphy.co/tracey.wisozk,,93.36.29.82,"{""location"": ""FM"", ""is_mobile"": true}" 11003,4,246,2017-03-25 12:40:45,http://skiles.io/moses,,189.62.218.71,"{""location"": ""VN"", ""is_mobile"": false}" 11004,4,246,2017-01-06 21:19:53,http://wunsch.biz/shaun,,119.125.232.67,"{""location"": ""KZ"", ""is_mobile"": false}" 11005,4,246,2017-04-18 20:13:49,http://smithtillman.biz/dexter,,234.141.31.128,"{""location"": ""IO"", ""is_mobile"": false}" 11006,4,246,2017-03-06 01:15:10,http://jenkinsrempel.net/antone.bechtelar,,14.119.186.243,"{""location"": ""MY"", ""is_mobile"": true}" 11007,4,246,2017-03-18 08:43:16,http://heel.com/jairo.fritsch,,218.176.59.198,"{""location"": ""GH"", ""is_mobile"": true}" 11008,4,246,2017-03-19 22:21:59,http://breitenberg.info/ethan,,124.190.173.202,"{""location"": ""LT"", ""is_mobile"": false}" 11009,4,246,2017-01-20 12:29:36,http://boyer.com/sim,,50.124.219.63,"{""location"": ""YE"", ""is_mobile"": false}" 11010,4,246,2016-12-17 22:54:24,http://barton.io/isadore_ankunding,,10.252.208.143,"{""location"": ""KZ"", ""is_mobile"": true}" 11011,4,246,2017-02-18 15:45:22,http://breitenberg.name/rolando,,160.213.114.59,"{""location"": ""EG"", ""is_mobile"": false}" 11012,4,246,2016-12-30 02:04:43,http://faheyzieme.org/keaton,,144.21.146.132,"{""location"": ""KN"", ""is_mobile"": false}" 11013,4,246,2017-06-01 01:42:08,http://kuvalis.com/otilia,,132.196.210.161,"{""location"": ""RO"", ""is_mobile"": true}" 11014,4,246,2017-03-02 09:00:06,http://barton.com/marie.thompson,,151.113.139.107,"{""location"": ""BH"", ""is_mobile"": false}" 11015,4,246,2017-02-07 09:27:58,http://pagac.io/conner_moriette,,207.240.11.103,"{""location"": ""FI"", ""is_mobile"": true}" 11016,4,246,2016-12-14 21:58:55,http://bahringer.com/rhett.ortiz,,145.55.140.71,"{""location"": ""MZ"", ""is_mobile"": true}" 11017,4,246,2017-05-30 13:58:50,http://schowalterdavis.co/dulce,,251.44.169.51,"{""location"": ""EG"", ""is_mobile"": true}" 11018,4,246,2017-05-14 12:11:44,http://larson.info/william,,170.180.239.30,"{""location"": ""NF"", ""is_mobile"": false}" 11019,4,246,2016-12-21 22:12:04,http://jacobi.io/golden,,18.229.253.153,"{""location"": ""MO"", ""is_mobile"": true}" 11020,4,246,2017-04-15 04:00:24,http://kreigermarquardt.co/alysa,,61.226.131.19,"{""location"": ""BV"", ""is_mobile"": false}" 11021,4,246,2017-05-19 14:29:27,http://lehner.info/dolores.howell,,203.106.36.222,"{""location"": ""AI"", ""is_mobile"": true}" 11022,4,246,2017-04-02 00:32:22,http://mcdermott.io/lori,,98.206.159.78,"{""location"": ""NO"", ""is_mobile"": false}" 11023,4,246,2016-12-16 11:57:21,http://rodriguezleuschke.org/asha,,153.206.85.55,"{""location"": ""GB"", ""is_mobile"": true}" 11024,4,246,2017-01-24 16:00:15,http://murphydietrich.name/lulu.kohler,,170.193.62.230,"{""location"": ""TO"", ""is_mobile"": true}" 11025,4,246,2017-04-24 18:23:47,http://adams.name/anastasia,,183.17.13.139,"{""location"": ""PW"", ""is_mobile"": true}" 11026,4,246,2017-02-01 12:04:46,http://stark.io/arno.upton,,37.238.26.37,"{""location"": ""GG"", ""is_mobile"": true}" 11027,4,246,2017-01-19 23:45:05,http://maggio.biz/bradley,,195.68.22.108,"{""location"": ""FI"", ""is_mobile"": false}" 11028,4,246,2016-12-30 23:29:12,http://heaney.name/xzavier,,57.235.205.189,"{""location"": ""TM"", ""is_mobile"": true}" 11029,4,246,2017-04-19 06:32:06,http://schulist.org/jammie,,224.47.110.169,"{""location"": ""ET"", ""is_mobile"": false}" 11030,4,247,2017-04-05 09:02:49,http://gulgowski.co/danielle.muller,,169.21.181.181,"{""location"": ""ZA"", ""is_mobile"": true}" 11031,4,247,2017-02-21 21:57:32,http://schusterwindler.info/guiseppe,,151.210.87.183,"{""location"": ""TJ"", ""is_mobile"": true}" 11032,4,247,2017-03-05 18:43:15,http://leannonstreich.io/nedra_lowe,,219.173.254.208,"{""location"": ""KE"", ""is_mobile"": false}" 11033,4,247,2017-03-13 11:28:46,http://jacobi.net/vallie,,47.160.103.201,"{""location"": ""YT"", ""is_mobile"": true}" 11034,4,247,2017-03-27 15:52:46,http://gutmann.info/astrid.rolfson,,34.197.145.118,"{""location"": ""JM"", ""is_mobile"": true}" 11035,4,247,2017-01-22 16:22:31,http://larson.biz/efren_kris,,97.18.34.197,"{""location"": ""NU"", ""is_mobile"": false}" 11036,4,247,2017-02-17 10:08:54,http://muller.io/bridie,,180.59.17.98,"{""location"": ""TF"", ""is_mobile"": false}" 11037,4,247,2017-03-11 07:13:54,http://glover.name/agustin,,58.134.69.89,"{""location"": ""AE"", ""is_mobile"": false}" 11038,4,247,2017-06-06 13:41:10,http://mckenzie.com/leonie,,104.106.164.86,"{""location"": ""ER"", ""is_mobile"": false}" 11039,4,247,2017-02-16 01:57:13,http://gutmannrosenbaum.org/neal.herzog,,250.105.90.228,"{""location"": ""TN"", ""is_mobile"": true}" 11040,4,247,2017-02-02 05:16:06,http://batz.info/elena,,231.142.3.145,"{""location"": ""MH"", ""is_mobile"": false}" 11041,4,247,2017-05-17 11:49:57,http://paucekfritsch.biz/lindsey,,138.10.16.232,"{""location"": ""BM"", ""is_mobile"": true}" 11042,4,247,2017-06-07 08:25:30,http://torphylindgren.biz/douglas.hermiston,,210.58.36.154,"{""location"": ""CO"", ""is_mobile"": true}" 11043,4,247,2016-12-13 21:58:25,http://lind.io/christelle_pagac,,200.128.211.223,"{""location"": ""ES"", ""is_mobile"": false}" 20873,7,469,2017-02-11 15:32:11,http://mann.info/savanah,,92.197.113.79,"{""location"": ""LB"", ""is_mobile"": true}" 11044,4,247,2017-06-03 02:34:01,http://satterfieldvon.com/dedrick_eichmann,,171.138.88.195,"{""location"": ""HU"", ""is_mobile"": true}" 11045,4,247,2017-04-20 04:01:00,http://okuneva.biz/erika.nikolaus,,14.85.43.242,"{""location"": ""AM"", ""is_mobile"": true}" 11046,4,247,2017-05-18 02:01:20,http://kshlerin.co/ruby_barton,,164.224.189.55,"{""location"": ""BG"", ""is_mobile"": false}" 11047,4,247,2017-02-23 20:54:06,http://heaney.com/lazaro,,190.128.238.8,"{""location"": ""AM"", ""is_mobile"": false}" 11048,4,247,2017-02-26 07:12:09,http://schmelernicolas.name/amaya,,164.19.5.169,"{""location"": ""BS"", ""is_mobile"": false}" 11049,4,247,2017-06-12 02:37:54,http://stehr.name/dakota,,41.165.53.173,"{""location"": ""FI"", ""is_mobile"": false}" 11050,4,247,2017-05-03 05:00:37,http://ondrickakuhic.info/antonetta.wunsch,,113.86.213.121,"{""location"": ""HR"", ""is_mobile"": true}" 11051,4,247,2017-03-27 22:49:16,http://kautzer.net/clint,,113.216.32.162,"{""location"": ""RO"", ""is_mobile"": false}" 8016,3,177,2017-02-13 16:10:12,http://tremblay.io/kevin.mitchell,0.7810997006,20.173.121.193,"{""location"": ""PS"", ""is_mobile"": false}" 8017,3,177,2017-02-06 18:59:04,http://schamberger.co/arnulfo,0.7099294106,46.253.149.132,"{""location"": ""BO"", ""is_mobile"": false}" 8018,3,177,2017-03-19 00:31:30,http://rowe.org/vincenzo,0.9436079594,234.128.11.31,"{""location"": ""AS"", ""is_mobile"": true}" 8019,3,177,2017-05-13 21:29:12,http://west.com/jasen,0.7213813267,107.147.38.173,"{""location"": ""AZ"", ""is_mobile"": false}" 8020,3,177,2016-12-28 17:47:45,http://kulashartmann.info/wyman,0.4970999526,166.241.117.205,"{""location"": ""NL"", ""is_mobile"": true}" 8021,3,177,2017-02-16 16:44:00,http://marquardt.info/unique,0.9964751140,250.112.137.143,"{""location"": ""SG"", ""is_mobile"": true}" 8022,3,177,2017-01-09 12:48:57,http://sipesfriesen.biz/miouri,0.3630616840,136.59.208.126,"{""location"": ""NA"", ""is_mobile"": true}" 8023,3,177,2017-03-26 20:15:26,http://zboncak.biz/constance,0.1826403295,253.135.95.89,"{""location"": ""SN"", ""is_mobile"": false}" 8024,3,177,2017-05-28 23:03:41,http://heathcote.com/zetta,0.7635537038,79.121.168.215,"{""location"": ""GH"", ""is_mobile"": false}" 8025,3,177,2016-12-19 20:11:28,http://ritchie.info/alvah_mcglynn,0.8832455742,231.5.75.235,"{""location"": ""FK"", ""is_mobile"": false}" 8026,3,177,2017-06-12 17:47:36,http://framialtenwerth.org/price,0.3869795801,104.168.198.37,"{""location"": ""ML"", ""is_mobile"": true}" 8027,3,177,2016-12-26 10:24:12,http://langworthkuhn.info/bryon,0.5038616223,73.95.91.132,"{""location"": ""JP"", ""is_mobile"": true}" 8028,3,177,2016-12-27 00:05:21,http://durgan.net/benton.mann,0.5662410665,187.195.123.188,"{""location"": ""OM"", ""is_mobile"": true}" 8029,3,177,2016-12-29 01:39:36,http://hahnklein.biz/deshaun.cole,0.3156903187,148.75.124.8,"{""location"": ""RW"", ""is_mobile"": true}" 8030,3,177,2017-05-01 13:58:40,http://cainpredovic.name/burley,0.0157745820,126.253.202.177,"{""location"": ""CU"", ""is_mobile"": false}" 8031,3,177,2016-12-14 11:02:11,http://cummerata.info/priscilla_heathcote,0.7312232683,176.47.168.219,"{""location"": ""CR"", ""is_mobile"": true}" 8032,3,177,2017-06-05 08:02:34,http://friesen.co/anahi,0.6738535086,190.176.70.107,"{""location"": ""BI"", ""is_mobile"": false}" 8033,3,177,2017-02-01 20:11:50,http://kautzer.info/hope,0.0454966590,99.14.253.57,"{""location"": ""SJ"", ""is_mobile"": false}" 8034,3,177,2017-06-02 16:24:59,http://gleason.com/patience_mosciski,0.5459525637,137.188.154.193,"{""location"": ""SD"", ""is_mobile"": false}" 8035,3,177,2017-05-10 13:15:19,http://jaskolski.net/gregoria,0.2487646364,111.79.97.241,"{""location"": ""MV"", ""is_mobile"": true}" 8036,3,177,2017-03-23 17:12:31,http://quitzon.com/xander_schultz,0.6674486887,132.20.128.217,"{""location"": ""UZ"", ""is_mobile"": true}" 8037,3,177,2017-02-27 06:07:41,http://klein.io/henri_dickinson,0.8924257247,188.124.153.130,"{""location"": ""KH"", ""is_mobile"": false}" 8038,3,177,2016-12-15 21:56:09,http://stromanorn.co/jadyn.fisher,0.9498596757,69.175.152.201,"{""location"": ""GS"", ""is_mobile"": true}" 8039,3,177,2017-02-27 07:55:23,http://towne.io/francisco.schinner,0.7018704199,157.176.17.234,"{""location"": ""ST"", ""is_mobile"": false}" 8040,3,177,2017-02-21 02:44:19,http://reinger.io/korey,0.9411329892,88.207.202.166,"{""location"": ""VG"", ""is_mobile"": false}" 8041,3,177,2017-03-19 16:29:04,http://nitzsche.info/ubaldo_herman,0.5273322996,168.169.110.137,"{""location"": ""SN"", ""is_mobile"": true}" 8042,3,177,2017-02-28 15:39:27,http://miller.biz/lulu_ebert,0.3420143253,196.6.118.32,"{""location"": ""GF"", ""is_mobile"": true}" 8043,3,177,2017-05-04 23:01:42,http://collier.net/brandi,0.5612238369,138.4.85.44,"{""location"": ""RO"", ""is_mobile"": true}" 8044,3,177,2017-02-28 10:50:35,http://heidenreichgoodwin.biz/bridie_shields,0.1416802464,138.22.36.5,"{""location"": ""TM"", ""is_mobile"": false}" 8045,3,177,2017-06-03 22:28:03,http://walker.biz/idell,0.6302482235,211.209.232.100,"{""location"": ""BG"", ""is_mobile"": true}" 8046,3,177,2017-01-21 14:30:29,http://mosciskikutch.co/thaddeus_wolf,0.1719313361,198.120.240.169,"{""location"": ""SS"", ""is_mobile"": true}" 8047,3,177,2017-02-10 19:25:20,http://tremblaydooley.name/adrien_predovic,0.4279252972,21.121.4.192,"{""location"": ""ME"", ""is_mobile"": true}" 8048,3,177,2017-02-03 09:16:46,http://murphybotsford.co/jalen,0.9963803364,79.73.208.138,"{""location"": ""NO"", ""is_mobile"": true}" 8049,3,177,2017-06-11 11:56:29,http://ankunding.com/sylvan_fay,0.1971399844,184.253.4.89,"{""location"": ""RW"", ""is_mobile"": false}" 8050,3,177,2017-04-13 23:13:36,http://wiza.info/maddison.nader,0.9019514855,49.238.41.68,"{""location"": ""SX"", ""is_mobile"": false}" 8051,3,177,2017-04-22 15:35:43,http://rauhermann.info/tyrique,0.3842554954,21.10.96.115,"{""location"": ""ST"", ""is_mobile"": true}" 8052,3,177,2016-12-17 06:15:38,http://wilderman.io/camren,0.0673269346,41.95.143.148,"{""location"": ""IT"", ""is_mobile"": true}" 8053,3,177,2017-03-17 19:42:47,http://klockomaggio.info/lempi,0.2429432111,3.180.10.25,"{""location"": ""GF"", ""is_mobile"": true}" 8054,3,177,2017-03-12 23:07:40,http://johnsonheaney.name/leon,0.3679755471,137.55.229.238,"{""location"": ""PG"", ""is_mobile"": false}" 8055,3,177,2017-03-29 03:34:43,http://durgan.biz/guadalupe,0.3795920586,225.235.98.37,"{""location"": ""PK"", ""is_mobile"": false}" 8056,3,177,2017-04-17 03:38:46,http://shanahan.info/shaniya_becker,0.5844643821,93.248.140.240,"{""location"": ""SV"", ""is_mobile"": true}" 8057,3,178,2017-04-15 05:08:44,http://ward.name/kirk_hayes,0.5855662791,203.75.98.30,"{""location"": ""CX"", ""is_mobile"": false}" 8058,3,178,2017-01-14 21:32:32,http://crooks.co/otilia,0.4926851585,68.104.67.230,"{""location"": ""RS"", ""is_mobile"": false}" 8059,3,178,2017-03-06 09:19:29,http://rodriguezstark.co/ted.marquardt,0.9685450889,203.12.71.156,"{""location"": ""ZW"", ""is_mobile"": false}" 8060,3,178,2017-05-02 05:07:01,http://creminkaulke.info/lauren.ferry,0.6621673616,57.100.147.177,"{""location"": ""KY"", ""is_mobile"": false}" 8061,3,178,2017-04-28 08:23:15,http://zemlak.com/chester,0.6646025622,105.52.247.56,"{""location"": ""GB"", ""is_mobile"": true}" 8062,3,178,2017-06-09 23:40:54,http://damore.io/fern,0.1608152039,59.137.96.51,"{""location"": ""IT"", ""is_mobile"": true}" 8063,3,178,2017-01-18 19:03:30,http://mrazsteuber.biz/cleo,0.2142916299,252.65.48.141,"{""location"": ""TD"", ""is_mobile"": false}" 8064,3,178,2017-04-28 11:11:46,http://johns.io/reagan,0.9246977884,208.230.251.128,"{""location"": ""CY"", ""is_mobile"": false}" 8065,3,178,2017-01-21 22:09:36,http://hyattdenesik.com/rhoda,0.2511539917,238.70.206.247,"{""location"": ""JO"", ""is_mobile"": true}" 8066,3,178,2017-06-03 00:26:44,http://bins.co/caesar.olson,0.8350315986,252.238.108.160,"{""location"": ""TR"", ""is_mobile"": false}" 13947,5,312,2017-05-02 10:48:36,http://runolfon.net/josefa,,125.23.207.227,"{""location"": ""AL"", ""is_mobile"": true}" 13948,5,312,2017-04-10 13:10:52,http://spinka.net/christopher_fay,,173.220.218.78,"{""location"": ""BD"", ""is_mobile"": true}" 13949,5,312,2016-12-28 06:24:34,http://willmskertzmann.biz/fannie.casper,,104.174.18.203,"{""location"": ""UZ"", ""is_mobile"": false}" 13950,5,312,2017-03-31 03:33:26,http://keeblerondricka.biz/joshuah,,140.30.188.202,"{""location"": ""CN"", ""is_mobile"": false}" 13951,5,312,2017-02-05 14:00:54,http://johns.co/fredy.schmitt,,31.36.222.239,"{""location"": ""TF"", ""is_mobile"": true}" 13952,5,313,2017-04-23 03:26:12,http://metzrempel.co/thelma_gulgowski,,251.148.250.17,"{""location"": ""AF"", ""is_mobile"": true}" 13953,5,313,2017-05-31 18:11:10,http://nienow.com/frederik.gottlieb,,168.248.197.4,"{""location"": ""DZ"", ""is_mobile"": false}" 13954,5,313,2017-05-06 16:43:32,http://effertz.biz/june,,91.101.65.54,"{""location"": ""LI"", ""is_mobile"": true}" 13955,5,313,2017-01-21 19:41:30,http://langoshmitchell.io/frederik_bailey,,123.111.160.37,"{""location"": ""PW"", ""is_mobile"": true}" 13956,5,313,2017-06-08 06:09:48,http://borer.co/zander.wolff,,189.219.80.216,"{""location"": ""GR"", ""is_mobile"": true}" 13957,5,313,2017-03-23 23:18:31,http://langworth.org/jaeden_brekke,,165.112.226.194,"{""location"": ""CI"", ""is_mobile"": true}" 13958,5,313,2016-12-30 18:58:17,http://heelwisozk.net/sarah,,9.188.200.241,"{""location"": ""PY"", ""is_mobile"": true}" 13959,5,313,2016-12-22 17:24:21,http://stoltenberg.info/santina.muller,,153.207.2.207,"{""location"": ""AT"", ""is_mobile"": true}" 13960,5,313,2017-03-22 22:50:24,http://nikolaus.net/mafalda_smith,,83.74.88.254,"{""location"": ""CD"", ""is_mobile"": false}" 13961,5,313,2017-04-06 02:02:34,http://rau.name/zola.schoen,,11.215.86.248,"{""location"": ""SH"", ""is_mobile"": false}" 13962,5,313,2017-01-28 09:29:59,http://gulgowski.org/rolando_metz,,42.254.181.152,"{""location"": ""GL"", ""is_mobile"": true}" 13963,5,313,2017-04-22 10:11:41,http://kshlerin.name/abelardo_schaefer,,62.216.181.147,"{""location"": ""JO"", ""is_mobile"": false}" 13964,5,313,2017-06-12 00:29:10,http://gottlieb.net/aliya_kohler,,220.159.164.235,"{""location"": ""KG"", ""is_mobile"": true}" 13965,5,313,2017-04-25 13:38:54,http://hackett.io/kayley,,144.110.86.213,"{""location"": ""SA"", ""is_mobile"": false}" 13966,5,313,2017-01-15 18:23:49,http://corwinraynor.name/vivian_schmitt,,114.126.99.3,"{""location"": ""GD"", ""is_mobile"": true}" 13967,5,313,2017-04-28 14:05:53,http://ziemannwolff.org/ubaldo,,120.99.143.40,"{""location"": ""GE"", ""is_mobile"": false}" 13968,5,313,2017-02-15 14:27:12,http://ortiz.co/celestine,,15.247.232.93,"{""location"": ""PW"", ""is_mobile"": true}" 13969,5,313,2017-05-25 05:36:44,http://powlowski.name/veronica.roob,,13.214.94.82,"{""location"": ""BJ"", ""is_mobile"": false}" 13970,5,313,2017-03-23 07:39:01,http://hamillhuels.name/evalyn,,198.151.165.211,"{""location"": ""BS"", ""is_mobile"": false}" 13971,5,313,2017-02-17 13:40:08,http://howebruen.biz/wilhelmine,,39.74.85.124,"{""location"": ""GF"", ""is_mobile"": true}" 13972,5,313,2017-02-03 01:06:55,http://smith.io/mark_mills,,89.21.221.233,"{""location"": ""AF"", ""is_mobile"": false}" 13973,5,313,2016-12-23 08:48:07,http://hettingerluettgen.info/wyman,,224.145.67.129,"{""location"": ""ZM"", ""is_mobile"": true}" 13974,5,313,2017-04-05 09:55:27,http://cronindamore.name/irwin,,251.252.147.117,"{""location"": ""DJ"", ""is_mobile"": false}" 13975,5,313,2017-05-11 23:25:54,http://nolan.co/barry,,151.236.105.104,"{""location"": ""VE"", ""is_mobile"": true}" 13976,5,313,2017-01-29 00:53:16,http://klingparker.com/coy,,124.211.252.187,"{""location"": ""BZ"", ""is_mobile"": true}" 13977,5,313,2017-03-10 10:55:27,http://wehner.org/blair_nolan,,145.182.181.66,"{""location"": ""VA"", ""is_mobile"": true}" 13978,5,313,2017-06-07 20:39:14,http://uptonhalvorson.info/melyna,,174.62.79.230,"{""location"": ""SO"", ""is_mobile"": false}" 13979,5,313,2017-01-08 06:40:35,http://vandervort.biz/okey,,150.181.205.67,"{""location"": ""GU"", ""is_mobile"": true}" 13980,5,313,2017-04-08 06:26:02,http://purdy.com/romaine,,7.46.251.158,"{""location"": ""SJ"", ""is_mobile"": false}" 13981,5,313,2017-04-12 02:48:34,http://buckridge.com/efrain,,214.108.223.144,"{""location"": ""PA"", ""is_mobile"": true}" 13982,5,313,2016-12-17 11:38:17,http://mayer.io/genesis.waters,,99.173.26.17,"{""location"": ""PT"", ""is_mobile"": false}" 13983,5,314,2017-05-24 22:44:38,http://tillman.net/tyshawn_schuster,,92.59.35.61,"{""location"": ""DK"", ""is_mobile"": true}" 13984,5,314,2017-03-31 13:10:26,http://homenickgerlach.com/garnet,,243.15.11.14,"{""location"": ""PK"", ""is_mobile"": false}" 13985,5,314,2017-04-12 19:09:31,http://boylegrady.co/anibal.rolfson,,101.217.133.204,"{""location"": ""BT"", ""is_mobile"": false}" 13986,5,314,2017-06-11 21:02:03,http://howellabernathy.com/kieran.weinat,,86.108.218.156,"{""location"": ""BV"", ""is_mobile"": true}" 13987,5,314,2017-03-28 23:44:16,http://stokes.net/aurelio.altenwerth,,62.152.177.178,"{""location"": ""KW"", ""is_mobile"": false}" 13988,5,314,2017-01-26 12:14:26,http://murazik.info/alex,,188.195.17.149,"{""location"": ""EH"", ""is_mobile"": true}" 13989,5,314,2017-05-22 00:41:49,http://spencerhirthe.com/jevon_cain,,126.84.42.222,"{""location"": ""PR"", ""is_mobile"": true}" 13990,5,314,2017-05-18 19:39:46,http://wilderman.com/lilla,,44.166.157.78,"{""location"": ""BW"", ""is_mobile"": true}" 13991,5,314,2017-05-18 12:36:10,http://langworth.org/logan.larson,,34.177.174.221,"{""location"": ""BI"", ""is_mobile"": false}" 13992,5,314,2017-04-29 02:44:35,http://beahan.name/kay,,166.14.40.136,"{""location"": ""BL"", ""is_mobile"": true}" 13993,5,314,2017-04-27 17:55:02,http://kovacek.net/patsy,,141.130.145.39,"{""location"": ""PA"", ""is_mobile"": true}" 13994,5,314,2017-05-06 12:04:02,http://millerking.io/thalia.volkman,,49.172.254.207,"{""location"": ""IL"", ""is_mobile"": false}" 13995,5,314,2016-12-23 11:49:05,http://gutmannrobel.info/bernita,,41.176.249.59,"{""location"": ""MK"", ""is_mobile"": true}" 13996,5,314,2017-03-06 16:34:33,http://pacochaheaney.net/adam.wyman,,212.238.182.147,"{""location"": ""MS"", ""is_mobile"": false}" 13997,5,314,2017-05-20 10:51:06,http://upton.name/milan.boyer,,8.174.95.48,"{""location"": ""AF"", ""is_mobile"": false}" 13998,5,314,2017-04-30 08:21:32,http://shanahan.org/darron_trantow,,165.90.198.24,"{""location"": ""MG"", ""is_mobile"": true}" 13999,5,314,2017-04-18 22:18:59,http://kozey.net/devin,,2.88.185.182,"{""location"": ""AZ"", ""is_mobile"": false}" 14000,5,314,2017-04-15 14:46:01,http://thompson.org/tobin,,115.47.88.17,"{""location"": ""UG"", ""is_mobile"": false}" 14001,5,314,2016-12-22 16:04:54,http://stokes.co/jedediah_satterfield,,37.249.189.203,"{""location"": ""MM"", ""is_mobile"": false}" 16929,6,383,2017-02-20 15:30:39,http://powlowskiraynor.info/liza,,82.64.51.171,"{""location"": ""CY"", ""is_mobile"": false}" 16930,6,383,2016-12-15 21:11:09,http://mcclure.org/twila_gleichner,,124.186.106.186,"{""location"": ""ZW"", ""is_mobile"": true}" 16931,6,383,2017-05-03 16:20:39,http://altenwerth.org/bret_borer,,39.135.150.245,"{""location"": ""LB"", ""is_mobile"": true}" 16932,6,383,2017-02-20 01:42:11,http://schuster.biz/april,,5.239.130.40,"{""location"": ""GQ"", ""is_mobile"": true}" 16933,6,383,2017-05-22 13:03:07,http://fay.io/reid,,130.61.83.187,"{""location"": ""US"", ""is_mobile"": false}" 16934,6,383,2017-05-27 20:06:31,http://larkin.io/dimitri,,15.61.30.232,"{""location"": ""CG"", ""is_mobile"": false}" 16935,6,383,2017-05-11 03:00:33,http://hermann.biz/lizeth,,134.197.65.103,"{""location"": ""TV"", ""is_mobile"": false}" 16936,6,383,2016-12-25 09:09:16,http://connelly.name/allison_berge,,112.136.30.182,"{""location"": ""CN"", ""is_mobile"": false}" 16937,6,383,2017-02-11 01:20:52,http://keeblerdooley.net/ashly,,131.250.139.33,"{""location"": ""CH"", ""is_mobile"": true}" 16938,6,383,2017-02-08 11:28:56,http://mcglynn.co/marian,,60.237.195.179,"{""location"": ""PY"", ""is_mobile"": false}" 16939,6,383,2016-12-26 02:21:12,http://kertzmann.co/edgardo,,112.151.116.170,"{""location"": ""BA"", ""is_mobile"": true}" 16940,6,383,2017-04-19 18:38:26,http://schmidt.info/randal.greenholt,,169.246.109.11,"{""location"": ""TO"", ""is_mobile"": true}" 16941,6,383,2017-04-07 02:25:53,http://rath.io/jeanette,,52.9.176.239,"{""location"": ""TV"", ""is_mobile"": true}" 16942,6,383,2017-05-13 08:05:32,http://pfeffer.biz/grady.harvey,,19.37.129.118,"{""location"": ""TR"", ""is_mobile"": true}" 16943,6,383,2017-03-29 22:26:42,http://sanford.biz/amely.leffler,,250.163.214.64,"{""location"": ""MW"", ""is_mobile"": false}" 16944,6,383,2017-01-24 02:47:54,http://cummings.io/elna.mann,,198.137.60.125,"{""location"": ""GN"", ""is_mobile"": false}" 16945,6,383,2017-03-22 00:57:35,http://dare.co/deondre_batz,,104.41.223.63,"{""location"": ""NC"", ""is_mobile"": true}" 16946,6,383,2017-05-11 05:53:10,http://turnerkshlerin.biz/evan_larkin,,89.145.248.53,"{""location"": ""NL"", ""is_mobile"": true}" 16947,6,383,2017-03-31 20:10:35,http://mcglynn.biz/cody,,183.246.221.76,"{""location"": ""ST"", ""is_mobile"": false}" 16948,6,383,2017-03-03 17:57:40,http://veum.org/janea,,121.202.167.152,"{""location"": ""KZ"", ""is_mobile"": true}" 16949,6,383,2017-06-01 18:07:58,http://wittingcronin.net/ole.grant,,132.176.138.139,"{""location"": ""BY"", ""is_mobile"": true}" 16950,6,383,2017-02-28 16:33:57,http://klockoaltenwerth.info/camilla,,206.180.130.97,"{""location"": ""UA"", ""is_mobile"": true}" 16951,6,383,2017-02-12 21:16:21,http://bruen.io/heloise.murphy,,242.158.208.71,"{""location"": ""IS"", ""is_mobile"": true}" 16952,6,383,2017-02-24 11:46:19,http://heidenreich.org/eloise_ebert,,165.140.154.109,"{""location"": ""NI"", ""is_mobile"": false}" 16953,6,383,2017-05-31 22:07:37,http://keler.org/edna,,236.20.7.210,"{""location"": ""GW"", ""is_mobile"": true}" 16954,6,383,2017-03-18 16:17:29,http://abbottjenkins.io/vella_crooks,,192.222.38.195,"{""location"": ""ST"", ""is_mobile"": false}" 16955,6,383,2017-04-14 11:03:59,http://king.com/gabriel_hane,,63.142.82.166,"{""location"": ""TD"", ""is_mobile"": false}" 17417,6,392,2017-04-28 17:02:00,http://bauch.io/devonte,,241.252.118.14,"{""location"": ""AI"", ""is_mobile"": true}" 16956,6,383,2017-05-12 18:27:12,http://corkery.info/juanita.schowalter,,211.253.58.232,"{""location"": ""TZ"", ""is_mobile"": true}" 16957,6,383,2017-01-17 17:00:53,http://kuvalis.co/breanna,,143.138.29.216,"{""location"": ""JM"", ""is_mobile"": true}" 16958,6,383,2017-03-12 05:16:30,http://bednardeckow.org/dawson_ullrich,,214.103.72.78,"{""location"": ""TK"", ""is_mobile"": true}" 16959,6,383,2017-05-07 09:25:42,http://kerlukedurgan.name/adolf_stokes,,72.151.24.161,"{""location"": ""KZ"", ""is_mobile"": false}" 16960,6,383,2016-12-14 01:00:29,http://mckenzie.info/audra_oconner,,66.247.38.125,"{""location"": ""DO"", ""is_mobile"": true}" 16961,6,383,2017-05-16 18:29:37,http://bashirian.org/iac,,62.76.235.245,"{""location"": ""PM"", ""is_mobile"": true}" 16962,6,383,2017-02-04 02:22:30,http://dibberthegmann.net/jamel,,24.109.196.57,"{""location"": ""LV"", ""is_mobile"": false}" 16963,6,383,2017-02-16 02:20:39,http://hilpert.co/cary,,250.239.129.96,"{""location"": ""MV"", ""is_mobile"": true}" 16964,6,383,2017-02-26 13:01:22,http://fisher.name/effie.daniel,,185.192.212.152,"{""location"": ""VA"", ""is_mobile"": false}" 16965,6,383,2017-05-15 00:51:58,http://wildermanlindgren.info/dameon.quitzon,,69.87.87.55,"{""location"": ""AF"", ""is_mobile"": true}" 16966,6,384,2017-04-11 07:08:19,http://champlin.co/abraham,,99.128.151.235,"{""location"": ""AM"", ""is_mobile"": true}" 16967,6,384,2017-01-09 09:33:25,http://spencer.com/zetta,,226.51.41.7,"{""location"": ""KG"", ""is_mobile"": false}" 16968,6,384,2017-06-12 10:23:48,http://barrows.org/heather_davis,,197.213.155.164,"{""location"": ""RU"", ""is_mobile"": false}" 16969,6,384,2017-05-25 03:00:12,http://fahey.name/silas,,95.111.65.21,"{""location"": ""GS"", ""is_mobile"": false}" 16970,6,384,2017-05-21 18:52:10,http://parker.name/lonnie_krajcik,,227.125.167.123,"{""location"": ""GT"", ""is_mobile"": true}" 16971,6,384,2017-02-20 08:37:48,http://osinski.name/lisette,,79.100.84.127,"{""location"": ""LT"", ""is_mobile"": false}" 16972,6,384,2017-03-20 00:32:18,http://botsfordprohaska.net/leo,,89.175.30.76,"{""location"": ""SO"", ""is_mobile"": false}" 16973,6,384,2017-01-24 21:47:25,http://nicolas.biz/lea,,85.166.123.10,"{""location"": ""MC"", ""is_mobile"": false}" 16974,6,384,2016-12-31 17:03:14,http://oberbrunner.com/christelle,,154.89.92.55,"{""location"": ""GY"", ""is_mobile"": true}" 16975,6,384,2017-03-26 03:16:12,http://dibbert.com/linda,,109.78.66.42,"{""location"": ""GB"", ""is_mobile"": true}" 16976,6,384,2017-03-29 21:39:30,http://robel.io/lola,,173.51.48.27,"{""location"": ""GD"", ""is_mobile"": false}" 16977,6,384,2016-12-16 02:40:34,http://prosaccotillman.name/alicia_bauch,,56.18.244.156,"{""location"": ""SX"", ""is_mobile"": true}" 16978,6,384,2017-05-28 16:11:27,http://cormiergusikowski.co/lottie,,225.176.247.61,"{""location"": ""BM"", ""is_mobile"": true}" 16979,6,384,2017-03-13 16:12:20,http://rau.com/alfred_kutch,,38.110.12.98,"{""location"": ""SM"", ""is_mobile"": false}" 16980,6,384,2017-02-20 22:10:05,http://dooley.io/verdie,,88.198.21.90,"{""location"": ""MT"", ""is_mobile"": true}" 16981,6,384,2017-02-26 10:29:51,http://hilll.io/judah,,39.92.92.86,"{""location"": ""KH"", ""is_mobile"": true}" 16982,6,384,2017-02-28 10:12:52,http://townehagenes.info/ilene,,61.100.67.163,"{""location"": ""SK"", ""is_mobile"": false}" 4109,2,90,2017-03-27 11:32:59,http://lehner.io/mohammed.feeney,,192.43.231.3,"{""location"": ""DZ"", ""is_mobile"": true}" 4110,2,90,2017-03-25 10:11:05,http://okeefe.biz/jonathan.gleason,,94.89.176.180,"{""location"": ""HU"", ""is_mobile"": false}" 4111,2,90,2017-02-12 21:36:39,http://emmerichmayert.info/ignacio.beatty,,129.234.55.96,"{""location"": ""PN"", ""is_mobile"": false}" 4112,2,90,2017-04-21 13:21:23,http://ondricka.co/evelyn,,80.211.244.253,"{""location"": ""AW"", ""is_mobile"": true}" 4113,2,90,2016-12-28 14:17:57,http://lindwilderman.info/roderick.daugherty,,207.95.162.76,"{""location"": ""CD"", ""is_mobile"": true}" 4114,2,90,2017-04-30 22:08:56,http://hyatt.io/berneice,,61.251.148.37,"{""location"": ""LA"", ""is_mobile"": true}" 4115,2,90,2017-04-15 15:26:07,http://lemke.biz/otha.mante,,4.70.159.55,"{""location"": ""ES"", ""is_mobile"": false}" 4116,2,90,2016-12-29 09:51:22,http://sporerschimmel.net/monserrate_deckow,,219.220.68.170,"{""location"": ""FO"", ""is_mobile"": false}" 4117,2,90,2017-03-15 00:39:40,http://toy.io/aimee,,227.196.65.194,"{""location"": ""SB"", ""is_mobile"": false}" 4118,2,90,2017-05-03 23:58:54,http://feil.info/edwardo.upton,,206.239.236.147,"{""location"": ""SD"", ""is_mobile"": false}" 4119,2,90,2017-05-30 07:24:03,http://larsonlesch.org/dedrick_kshlerin,,114.108.101.60,"{""location"": ""HM"", ""is_mobile"": false}" 4120,2,90,2017-04-30 17:59:18,http://schuppe.org/kacie,,29.215.211.121,"{""location"": ""BG"", ""is_mobile"": true}" 4121,2,90,2017-01-19 06:19:15,http://gleason.net/megane_stoltenberg,,214.84.142.113,"{""location"": ""NU"", ""is_mobile"": false}" 4122,2,90,2017-02-12 06:51:20,http://dubuque.biz/torrey,,247.52.174.184,"{""location"": ""SK"", ""is_mobile"": false}" 4123,2,90,2017-04-08 10:33:33,http://rennerharris.org/tyree.hilll,,233.156.181.124,"{""location"": ""DM"", ""is_mobile"": false}" 4124,2,90,2017-04-02 13:15:28,http://fahey.com/mariam_pacocha,,167.254.172.198,"{""location"": ""BV"", ""is_mobile"": true}" 4125,2,90,2017-03-29 20:23:32,http://hyattupton.io/fredrick.labadie,,101.127.239.213,"{""location"": ""QA"", ""is_mobile"": false}" 4126,2,90,2017-03-02 14:25:06,http://pacocha.name/walton.jenkins,,222.237.19.26,"{""location"": ""DJ"", ""is_mobile"": false}" 4127,2,90,2017-04-26 16:32:31,http://mclaughlin.net/juwan.gusikowski,,197.209.84.221,"{""location"": ""AM"", ""is_mobile"": false}" 4128,2,90,2017-03-01 11:10:50,http://corkery.io/rosella.kutch,,251.221.119.168,"{""location"": ""GM"", ""is_mobile"": true}" 4129,2,90,2017-05-28 12:31:51,http://schinner.io/david_raynor,,207.157.213.208,"{""location"": ""MR"", ""is_mobile"": true}" 4130,2,90,2017-03-06 03:56:18,http://kertzmann.io/adolph,,3.179.179.118,"{""location"": ""GQ"", ""is_mobile"": true}" 4131,2,90,2016-12-21 09:02:09,http://wunschkris.biz/scotty,,181.104.65.80,"{""location"": ""PF"", ""is_mobile"": false}" 4132,2,90,2017-01-10 09:33:37,http://nadergrant.co/isadore.stamm,,204.122.13.99,"{""location"": ""DJ"", ""is_mobile"": true}" 4133,2,90,2017-02-14 13:08:12,http://schusterharris.org/regan,,83.237.153.195,"{""location"": ""VA"", ""is_mobile"": false}" 4134,2,90,2017-02-28 22:18:13,http://emard.net/lambert.yundt,,249.17.63.191,"{""location"": ""HM"", ""is_mobile"": true}" 4135,2,90,2016-12-14 14:05:31,http://fahey.co/elise_harris,,131.253.170.169,"{""location"": ""PK"", ""is_mobile"": true}" 4136,2,90,2017-05-28 18:52:29,http://stokes.name/madisen_hodkiewicz,,219.249.121.217,"{""location"": ""PA"", ""is_mobile"": false}" 4137,2,90,2017-02-14 22:02:52,http://cormier.io/vallie,,115.168.136.143,"{""location"": ""BV"", ""is_mobile"": false}" 4138,2,90,2017-02-25 12:36:00,http://champlin.co/tamara_stark,,165.65.217.113,"{""location"": ""ZM"", ""is_mobile"": true}" 4139,2,90,2017-01-08 22:12:45,http://herzog.info/geraldine,,79.119.98.55,"{""location"": ""SV"", ""is_mobile"": true}" 4140,2,90,2016-12-25 23:07:28,http://hoppe.info/juliet,,164.152.196.138,"{""location"": ""MF"", ""is_mobile"": true}" 4141,2,90,2016-12-23 16:02:29,http://considine.co/georgette_wiegand,,46.69.133.154,"{""location"": ""BR"", ""is_mobile"": false}" 4142,2,90,2017-02-22 15:33:07,http://reynolds.info/jayde,,28.241.94.151,"{""location"": ""OM"", ""is_mobile"": true}" 4143,2,90,2016-12-13 17:12:59,http://dooley.net/herminio,,254.58.156.236,"{""location"": ""KN"", ""is_mobile"": false}" 4144,2,90,2017-04-12 02:22:38,http://schumm.com/daron,,119.181.94.218,"{""location"": ""TH"", ""is_mobile"": true}" 4145,2,90,2017-03-14 17:38:41,http://bogan.co/lorine,,140.218.192.37,"{""location"": ""UY"", ""is_mobile"": true}" 4146,2,90,2017-04-14 01:20:48,http://welch.name/melyna_beatty,,134.209.179.253,"{""location"": ""SN"", ""is_mobile"": false}" 4147,2,90,2017-04-20 10:06:38,http://rogahn.io/tyson,,109.44.141.196,"{""location"": ""BZ"", ""is_mobile"": true}" 4148,2,90,2017-05-05 18:12:10,http://heaneycrist.info/cristobal,,152.33.71.251,"{""location"": ""AW"", ""is_mobile"": true}" 4149,2,90,2017-03-25 19:12:55,http://schulist.com/aaliyah,,70.164.151.174,"{""location"": ""GH"", ""is_mobile"": true}" 4150,2,90,2017-03-15 16:00:54,http://hayes.net/marjorie,,100.92.251.78,"{""location"": ""BS"", ""is_mobile"": true}" 4151,2,90,2017-05-04 17:00:24,http://koch.com/william,,203.243.82.219,"{""location"": ""VN"", ""is_mobile"": false}" 4152,2,90,2017-02-07 08:44:00,http://rutherford.com/emely,,29.39.200.122,"{""location"": ""VN"", ""is_mobile"": true}" 4153,2,90,2017-04-09 12:48:51,http://goldner.com/arlo_bradtke,,39.80.72.83,"{""location"": ""UZ"", ""is_mobile"": true}" 4154,2,90,2017-06-06 16:23:20,http://larsonmcclure.co/estrella.kiehn,,249.9.82.36,"{""location"": ""ES"", ""is_mobile"": true}" 4155,2,90,2017-02-11 15:07:15,http://green.co/andreane.goldner,,160.253.5.38,"{""location"": ""MN"", ""is_mobile"": false}" 4156,2,91,2017-03-22 15:42:33,http://sawayn.name/eldora,,65.106.63.51,"{""location"": ""GE"", ""is_mobile"": false}" 4157,2,91,2017-05-16 02:22:52,http://beierwindler.biz/jamarcus,,37.106.50.164,"{""location"": ""BJ"", ""is_mobile"": false}" 4158,2,91,2017-03-07 14:04:35,http://jast.com/ron_mraz,,2.83.218.113,"{""location"": ""NU"", ""is_mobile"": false}" 4159,2,91,2016-12-27 08:33:08,http://sauer.co/sierra_kunze,,94.73.226.176,"{""location"": ""ST"", ""is_mobile"": false}" 4160,2,91,2017-01-06 22:28:52,http://eichmann.co/riley,,208.83.216.183,"{""location"": ""IM"", ""is_mobile"": false}" 4161,2,91,2017-04-13 18:28:01,http://bashirian.com/barry_little,,147.39.205.217,"{""location"": ""GR"", ""is_mobile"": true}" 4162,2,91,2017-01-27 21:31:14,http://dibbertdickens.org/nels,,159.62.233.59,"{""location"": ""PE"", ""is_mobile"": true}" 4163,2,91,2017-06-02 18:05:56,http://bartoletti.net/dorris.marvin,,167.53.90.27,"{""location"": ""VN"", ""is_mobile"": false}" 4164,2,91,2017-01-07 19:14:11,http://connelly.info/lavon.ryan,,213.132.168.128,"{""location"": ""GU"", ""is_mobile"": true}" 11052,4,247,2017-03-19 15:12:53,http://reichel.info/melba,,236.15.49.212,"{""location"": ""PA"", ""is_mobile"": false}" 11053,4,247,2017-02-05 23:46:38,http://fritschrohan.biz/frederic,,16.238.44.192,"{""location"": ""GI"", ""is_mobile"": true}" 11054,4,247,2017-02-02 11:59:25,http://stammcole.net/brianne_keeling,,52.153.63.167,"{""location"": ""AR"", ""is_mobile"": true}" 11055,4,247,2017-01-10 18:03:34,http://mayer.biz/marguerite,,164.134.143.217,"{""location"": ""NU"", ""is_mobile"": false}" 11056,4,247,2016-12-29 08:10:56,http://hilpertdach.co/olen_buckridge,,14.180.173.30,"{""location"": ""CH"", ""is_mobile"": true}" 11057,4,247,2017-05-16 15:32:40,http://thompson.org/carolina,,40.245.57.31,"{""location"": ""EC"", ""is_mobile"": false}" 11058,4,247,2017-03-27 21:16:39,http://millsrath.net/elisabeth.kuphal,,57.191.228.228,"{""location"": ""BA"", ""is_mobile"": false}" 11059,4,247,2017-04-11 05:37:29,http://nienowjenkins.name/gaetano.welch,,18.140.125.194,"{""location"": ""KW"", ""is_mobile"": false}" 11060,4,247,2017-01-03 13:32:48,http://cartwright.co/colt.bashirian,,92.134.110.179,"{""location"": ""NC"", ""is_mobile"": true}" 11061,4,247,2017-04-20 18:09:29,http://ankundinglarson.co/lorenza,,9.73.191.42,"{""location"": ""PM"", ""is_mobile"": false}" 11062,4,247,2017-01-23 05:44:39,http://cormier.net/norris_gaylord,,117.63.47.60,"{""location"": ""SJ"", ""is_mobile"": true}" 11063,4,247,2017-06-02 23:13:56,http://weber.io/lee,,32.84.172.220,"{""location"": ""AM"", ""is_mobile"": true}" 11064,4,247,2017-05-12 08:57:37,http://ortiz.com/breanne,,159.4.244.67,"{""location"": ""TT"", ""is_mobile"": true}" 11065,4,247,2017-04-18 11:20:29,http://bernharddavis.name/tyler,,184.217.219.216,"{""location"": ""BS"", ""is_mobile"": true}" 11066,4,247,2017-01-12 19:22:41,http://parisian.net/sierra_senger,,202.27.100.8,"{""location"": ""UG"", ""is_mobile"": true}" 11067,4,247,2017-02-18 14:20:11,http://oreilly.co/myron,,51.193.16.17,"{""location"": ""MC"", ""is_mobile"": false}" 11068,4,247,2017-02-06 15:10:44,http://lueilwitz.org/ozella_homenick,,164.191.177.214,"{""location"": ""BW"", ""is_mobile"": true}" 11069,4,247,2017-04-30 08:07:32,http://ankundingraynor.co/pearl.ruecker,,37.65.101.186,"{""location"": ""BN"", ""is_mobile"": false}" 11070,4,247,2016-12-30 18:46:58,http://wilderman.com/zackary.grant,,204.111.60.3,"{""location"": ""YE"", ""is_mobile"": false}" 11071,4,247,2017-05-13 21:10:15,http://morar.org/xzavier,,186.229.160.47,"{""location"": ""CH"", ""is_mobile"": true}" 11072,4,247,2017-04-01 17:10:14,http://altenwerth.com/raina_brown,,7.141.91.78,"{""location"": ""BJ"", ""is_mobile"": false}" 11073,4,247,2017-01-01 23:41:16,http://welchgoyette.org/elisha,,36.78.32.240,"{""location"": ""PM"", ""is_mobile"": true}" 11074,4,247,2017-01-19 15:22:37,http://schneiderbayer.com/joanne,,139.87.151.84,"{""location"": ""TN"", ""is_mobile"": true}" 11075,4,247,2017-06-06 03:57:17,http://donnelly.net/alta,,205.71.118.145,"{""location"": ""LR"", ""is_mobile"": true}" 11076,4,247,2017-04-25 05:30:59,http://herzog.info/ferne,,86.69.195.197,"{""location"": ""PS"", ""is_mobile"": false}" 11077,4,247,2017-01-22 04:07:32,http://walsh.org/nestor,,238.205.113.196,"{""location"": ""LT"", ""is_mobile"": true}" 11078,4,247,2017-04-28 20:30:02,http://cruickshank.io/valerie,,22.238.43.58,"{""location"": ""SC"", ""is_mobile"": true}" 11079,4,247,2017-06-07 01:46:29,http://fadelkulas.info/jamaal.rogahn,,3.12.64.182,"{""location"": ""GE"", ""is_mobile"": true}" 11080,4,247,2017-03-29 12:48:15,http://tromp.info/eudora,,213.115.219.208,"{""location"": ""CK"", ""is_mobile"": true}" 11081,4,247,2017-02-18 05:50:51,http://cummings.net/jerald,,178.101.124.88,"{""location"": ""MO"", ""is_mobile"": true}" 11082,4,248,2016-12-30 15:50:25,http://kreiger.io/earline_schmeler,,198.72.222.34,"{""location"": ""AR"", ""is_mobile"": true}" 11083,4,248,2017-04-04 12:45:23,http://lehner.co/lavinia_treutel,,165.39.213.218,"{""location"": ""CK"", ""is_mobile"": false}" 11084,4,248,2017-01-02 01:46:22,http://ortiz.net/trystan.marvin,,23.243.38.223,"{""location"": ""TJ"", ""is_mobile"": true}" 11085,4,248,2017-02-12 09:38:42,http://barrowscrona.biz/rachelle_reinger,,136.144.189.201,"{""location"": ""TK"", ""is_mobile"": false}" 11086,4,248,2017-04-28 01:20:22,http://bode.com/tia,,198.138.176.165,"{""location"": ""MV"", ""is_mobile"": true}" 11087,4,248,2017-06-07 06:33:54,http://schowalter.co/antonetta.watsica,,114.203.226.44,"{""location"": ""TG"", ""is_mobile"": true}" 11088,4,248,2017-04-06 02:06:24,http://rippinparisian.biz/kenya_wiza,,67.237.97.248,"{""location"": ""ZA"", ""is_mobile"": true}" 11089,4,248,2017-04-19 15:40:49,http://pollichnitzsche.info/rose.ebert,,156.129.141.227,"{""location"": ""LU"", ""is_mobile"": false}" 11090,4,248,2017-02-26 15:01:22,http://wolfwaelchi.co/brielle,,73.36.202.234,"{""location"": ""BL"", ""is_mobile"": true}" 11091,4,248,2017-04-22 00:39:03,http://wehnerharris.com/kelsi.batz,,103.178.76.24,"{""location"": ""CH"", ""is_mobile"": true}" 11092,4,248,2017-04-09 04:27:57,http://langosh.org/ettie,,22.149.177.7,"{""location"": ""FO"", ""is_mobile"": true}" 11093,4,248,2017-05-20 00:55:24,http://kris.io/ryder.huel,,97.216.131.116,"{""location"": ""HN"", ""is_mobile"": false}" 11094,4,248,2017-04-23 13:28:11,http://carroll.io/bettie,,127.162.218.173,"{""location"": ""GB"", ""is_mobile"": false}" 11095,4,248,2017-06-08 10:40:22,http://sipes.biz/bernadine.balistreri,,152.127.133.21,"{""location"": ""HN"", ""is_mobile"": true}" 11096,4,248,2017-05-06 10:20:44,http://damore.biz/joanie,,180.62.242.100,"{""location"": ""UY"", ""is_mobile"": true}" 11097,4,248,2017-04-08 05:25:45,http://monahan.com/dina.davis,,242.64.149.77,"{""location"": ""SJ"", ""is_mobile"": true}" 11098,4,248,2017-03-15 18:33:18,http://hoppe.co/sheila.renner,,65.148.34.19,"{""location"": ""GN"", ""is_mobile"": false}" 11099,4,248,2017-02-01 12:50:30,http://considine.name/britney_johnson,,40.224.254.212,"{""location"": ""PK"", ""is_mobile"": true}" 11100,4,248,2017-05-22 03:14:57,http://pouros.org/keanu.swift,,66.203.111.86,"{""location"": ""VE"", ""is_mobile"": false}" 11101,4,248,2017-02-02 14:58:21,http://schoen.co/keshawn.hahn,,233.46.160.139,"{""location"": ""MS"", ""is_mobile"": false}" 11102,4,248,2017-03-20 23:23:10,http://batz.org/deion,,133.20.168.193,"{""location"": ""CR"", ""is_mobile"": false}" 11103,4,248,2017-06-05 03:33:41,http://cormierhilll.info/cody_maggio,,115.63.207.143,"{""location"": ""GI"", ""is_mobile"": false}" 11104,4,248,2017-03-05 16:21:20,http://kelergoyette.biz/kelsie_berge,,24.178.86.97,"{""location"": ""FJ"", ""is_mobile"": false}" 11105,4,248,2017-05-09 21:12:14,http://cain.info/braulio,,206.169.244.62,"{""location"": ""PK"", ""is_mobile"": false}" 11106,4,248,2017-02-18 03:21:45,http://boyle.biz/meggie,,131.141.238.137,"{""location"": ""TK"", ""is_mobile"": true}" 8067,3,178,2017-04-26 00:27:02,http://gutmannconn.net/lon_waelchi,0.0769093221,57.19.69.50,"{""location"": ""AS"", ""is_mobile"": false}" 8068,3,178,2017-01-24 20:06:33,http://cain.net/vern,0.6347114855,195.90.35.63,"{""location"": ""VN"", ""is_mobile"": true}" 8069,3,178,2017-02-01 21:54:32,http://gerlach.io/emie,0.9543811588,75.195.26.87,"{""location"": ""UZ"", ""is_mobile"": false}" 8070,3,178,2017-02-14 23:00:33,http://stehrkovacek.io/wyatt,0.3270426849,94.3.182.235,"{""location"": ""PL"", ""is_mobile"": true}" 8071,3,178,2017-03-30 09:53:25,http://altenwerth.org/alfred_fritsch,0.4380045352,44.78.148.132,"{""location"": ""BW"", ""is_mobile"": false}" 8072,3,178,2016-12-25 23:17:50,http://crooks.biz/yesenia_fay,0.6964699730,206.10.120.6,"{""location"": ""NA"", ""is_mobile"": false}" 8073,3,178,2017-05-14 13:21:11,http://weimann.name/ofelia,0.7261745148,83.117.73.154,"{""location"": ""FK"", ""is_mobile"": false}" 8074,3,178,2017-03-28 03:04:43,http://borer.com/eleanore,0.0639538618,85.109.200.144,"{""location"": ""CH"", ""is_mobile"": true}" 8075,3,178,2017-04-27 14:20:03,http://ratke.io/jimmy,0.1333861617,15.180.29.162,"{""location"": ""BQ"", ""is_mobile"": false}" 8076,3,178,2017-04-10 14:59:01,http://reichel.co/tyra_koelpin,0.2490538686,113.63.119.110,"{""location"": ""HN"", ""is_mobile"": true}" 8077,3,178,2017-02-14 17:30:41,http://carroll.org/gretchen,0.8751089163,43.247.7.202,"{""location"": ""MG"", ""is_mobile"": true}" 8078,3,178,2017-03-03 14:20:07,http://dicki.io/melya,0.5179380654,129.205.154.240,"{""location"": ""BZ"", ""is_mobile"": false}" 8079,3,178,2017-04-12 07:49:35,http://goodwinsmitham.org/cameron.goldner,0.7173328295,244.62.96.220,"{""location"": ""GD"", ""is_mobile"": true}" 8080,3,178,2017-03-13 06:20:12,http://wuckert.org/javonte_leuschke,0.8576367199,94.199.191.199,"{""location"": ""MW"", ""is_mobile"": true}" 8081,3,178,2017-02-13 12:40:54,http://swift.co/alphonso_graham,0.7844783763,216.208.125.128,"{""location"": ""UY"", ""is_mobile"": true}" 8082,3,178,2016-12-19 06:24:01,http://walkerzulauf.com/destini,0.8173151043,223.3.80.14,"{""location"": ""LI"", ""is_mobile"": false}" 8083,3,178,2016-12-29 19:59:49,http://blanda.com/raymond,0.9603764927,208.35.237.81,"{""location"": ""NL"", ""is_mobile"": false}" 8084,3,178,2017-02-01 14:07:36,http://kaulke.org/alexander,0.0795164130,195.83.189.219,"{""location"": ""GP"", ""is_mobile"": false}" 8085,3,178,2017-02-03 18:44:19,http://hudson.info/elta,0.0963196862,51.132.130.232,"{""location"": ""PM"", ""is_mobile"": true}" 8086,3,178,2017-03-20 15:06:56,http://krajcik.co/michelle,0.6721164948,41.193.189.62,"{""location"": ""PL"", ""is_mobile"": true}" 8087,3,178,2017-01-26 01:04:03,http://stehrwalker.name/zetta.bins,0.9919396622,80.248.162.16,"{""location"": ""NE"", ""is_mobile"": true}" 8088,3,178,2016-12-18 12:11:15,http://schowalter.name/ladarius_wolff,0.5800206024,58.226.123.189,"{""location"": ""TO"", ""is_mobile"": true}" 8089,3,178,2017-06-10 12:48:39,http://wuckert.net/chanel.weimann,0.4769377616,106.106.10.17,"{""location"": ""KW"", ""is_mobile"": false}" 8090,3,178,2017-01-25 05:57:00,http://walker.com/logan.terry,0.6884506442,201.68.159.203,"{""location"": ""HM"", ""is_mobile"": true}" 8091,3,178,2017-05-09 01:13:06,http://wehneroberbrunner.org/misael,0.7741462301,136.39.153.127,"{""location"": ""MG"", ""is_mobile"": false}" 8092,3,178,2017-02-06 09:36:38,http://quitzonfarrell.org/ozzie_stark,0.2989137874,201.223.143.148,"{""location"": ""KG"", ""is_mobile"": false}" 8093,3,178,2017-01-12 03:13:16,http://streich.co/joelle,0.0064804259,216.237.27.134,"{""location"": ""MS"", ""is_mobile"": true}" 8094,3,178,2017-02-18 22:03:58,http://stokes.net/beau,0.2332959552,227.37.114.91,"{""location"": ""VE"", ""is_mobile"": false}" 8095,3,178,2017-05-17 19:11:55,http://boehmschulist.com/jackson,0.2631739185,32.161.140.123,"{""location"": ""GD"", ""is_mobile"": false}" 8096,3,178,2017-02-10 15:34:52,http://kautzer.net/hannah,0.6927469619,231.30.202.237,"{""location"": ""SI"", ""is_mobile"": false}" 8097,3,178,2017-03-13 17:06:20,http://weimannwillms.info/destany,0.8697490400,157.81.223.139,"{""location"": ""LY"", ""is_mobile"": false}" 8098,3,178,2017-03-01 14:04:34,http://braun.name/marcellus.bartoletti,0.8329557915,54.81.12.161,"{""location"": ""WS"", ""is_mobile"": false}" 8099,3,178,2017-02-27 17:27:41,http://quigley.com/lori_lowe,0.5676554910,75.98.53.148,"{""location"": ""JM"", ""is_mobile"": true}" 8100,3,178,2017-03-16 06:23:55,http://oconner.net/zola.konopelski,0.1195646064,110.86.122.155,"{""location"": ""SG"", ""is_mobile"": false}" 8101,3,178,2017-02-27 19:44:19,http://hermiston.org/horacio,0.4128763155,196.102.22.88,"{""location"": ""AW"", ""is_mobile"": true}" 8102,3,178,2016-12-24 06:00:36,http://mccullough.info/abbigail,0.9504996729,36.64.58.9,"{""location"": ""KN"", ""is_mobile"": false}" 8103,3,178,2017-03-03 21:43:57,http://grady.info/karlee,0.5790219514,22.80.26.237,"{""location"": ""KG"", ""is_mobile"": false}" 8104,3,178,2017-02-10 05:22:14,http://koch.io/micaela,0.6643691624,179.152.237.9,"{""location"": ""IN"", ""is_mobile"": false}" 8105,3,178,2017-05-14 12:28:00,http://hettinger.com/jaquelin,0.8580395068,90.250.219.187,"{""location"": ""VU"", ""is_mobile"": true}" 8106,3,178,2017-04-08 17:25:28,http://langlind.info/lillian,0.1030408438,19.206.151.112,"{""location"": ""KW"", ""is_mobile"": true}" 8107,3,178,2017-04-29 20:18:21,http://kemmer.name/lane_labadie,0.2583597282,143.101.54.250,"{""location"": ""TH"", ""is_mobile"": true}" 8108,3,178,2016-12-23 04:47:36,http://conroyflatley.com/rubie.gorczany,0.9906786065,210.126.213.76,"{""location"": ""IQ"", ""is_mobile"": true}" 8109,3,178,2017-05-02 06:14:34,http://huel.io/sincere,0.7386507419,50.186.193.99,"{""location"": ""AX"", ""is_mobile"": true}" 8110,3,178,2017-01-11 07:15:48,http://gulgowskiglover.co/charity,0.7098372113,106.172.225.187,"{""location"": ""SN"", ""is_mobile"": true}" 8111,3,178,2017-06-09 14:06:59,http://turcotteschiller.co/omari.hermiston,0.6701468437,104.252.49.160,"{""location"": ""IN"", ""is_mobile"": false}" 8112,3,178,2017-05-30 03:35:28,http://runtetorphy.co/stacey,0.5286765634,57.141.214.132,"{""location"": ""PT"", ""is_mobile"": false}" 8113,3,178,2017-02-19 20:48:53,http://hettinger.com/deanna.crooks,0.6388499393,253.89.115.251,"{""location"": ""CY"", ""is_mobile"": true}" 8114,3,178,2017-05-26 09:56:20,http://mills.org/dianna,0.1453451287,24.248.47.39,"{""location"": ""MD"", ""is_mobile"": true}" 8115,3,178,2017-05-14 08:08:53,http://ritchie.org/ola,0.2790327548,235.23.21.97,"{""location"": ""RS"", ""is_mobile"": false}" 8116,3,178,2017-05-24 19:42:04,http://legros.info/oren.schroeder,0.4277866950,66.118.188.247,"{""location"": ""AR"", ""is_mobile"": true}" 8117,3,179,2017-01-02 19:08:53,http://mills.com/lewis,0.1480287664,62.122.106.196,"{""location"": ""ST"", ""is_mobile"": true}" 8118,3,179,2016-12-24 08:18:16,http://zemlak.co/cornelius_schultz,0.3196769632,92.197.181.102,"{""location"": ""NL"", ""is_mobile"": true}" 14002,5,314,2017-05-28 23:01:57,http://rodriguez.info/mara.kihn,,190.47.176.97,"{""location"": ""MC"", ""is_mobile"": false}" 14003,5,314,2016-12-14 23:48:40,http://keeblerbrown.net/carter.will,,189.118.125.4,"{""location"": ""CM"", ""is_mobile"": true}" 14004,5,314,2017-05-28 06:54:45,http://ebert.name/monica.nader,,142.81.17.143,"{""location"": ""BG"", ""is_mobile"": false}" 14005,5,314,2017-03-23 00:12:32,http://mann.info/mathilde_wisoky,,9.63.218.2,"{""location"": ""RE"", ""is_mobile"": false}" 14006,5,314,2017-01-17 10:37:06,http://quitzon.net/maximilian.kshlerin,,37.146.69.89,"{""location"": ""LS"", ""is_mobile"": true}" 14007,5,314,2017-05-10 22:11:38,http://nicolas.net/travis_purdy,,54.64.27.174,"{""location"": ""IO"", ""is_mobile"": true}" 14008,5,314,2017-04-07 01:16:27,http://bergstromhoppe.co/wiley.oberbrunner,,241.40.154.90,"{""location"": ""TC"", ""is_mobile"": false}" 14009,5,314,2017-03-17 20:55:21,http://boylegrady.org/alexanne,,94.143.29.11,"{""location"": ""SV"", ""is_mobile"": false}" 14010,5,314,2016-12-26 01:53:58,http://bergstrom.biz/maxie,,245.225.240.96,"{""location"": ""UM"", ""is_mobile"": false}" 14011,5,314,2017-05-05 13:30:24,http://boyle.org/joshuah.hartmann,,13.108.251.64,"{""location"": ""HU"", ""is_mobile"": true}" 14012,5,314,2017-01-30 12:18:17,http://runolfonbarrows.com/turner.waelchi,,65.57.72.231,"{""location"": ""SO"", ""is_mobile"": true}" 14013,5,314,2017-04-06 22:32:52,http://trantow.info/angelita_mitchell,,136.213.27.58,"{""location"": ""GR"", ""is_mobile"": false}" 14014,5,314,2017-01-23 21:09:52,http://naderbeer.co/grayson,,165.157.153.78,"{""location"": ""IR"", ""is_mobile"": true}" 14015,5,314,2017-02-13 16:22:49,http://oconnerwalsh.biz/tyler_tromp,,178.180.103.108,"{""location"": ""KZ"", ""is_mobile"": false}" 14016,5,314,2017-01-30 04:07:50,http://rogahnrunte.info/scotty,,249.167.83.42,"{""location"": ""ID"", ""is_mobile"": true}" 14017,5,314,2017-02-17 09:36:34,http://grady.com/tate,,205.228.248.31,"{""location"": ""MK"", ""is_mobile"": false}" 14018,5,314,2017-05-24 22:06:31,http://goldner.com/horacio,,35.223.86.72,"{""location"": ""MD"", ""is_mobile"": true}" 14019,5,314,2016-12-18 00:56:06,http://schiller.biz/wilfrid,,7.35.9.84,"{""location"": ""NE"", ""is_mobile"": true}" 14020,5,314,2017-01-03 04:27:54,http://fay.com/spencer.grimes,,59.107.153.121,"{""location"": ""SN"", ""is_mobile"": true}" 14021,5,314,2017-03-08 10:04:00,http://pfeffer.info/eldridge_kovacek,,147.218.59.65,"{""location"": ""FJ"", ""is_mobile"": true}" 14022,5,314,2017-04-03 02:00:42,http://witting.com/stephany_batz,,124.52.237.76,"{""location"": ""BT"", ""is_mobile"": true}" 14023,5,314,2017-05-22 22:18:02,http://grimes.name/percival,,207.159.237.78,"{""location"": ""GP"", ""is_mobile"": false}" 14024,5,314,2017-05-08 11:15:23,http://feeneyarmstrong.name/shania.hermann,,233.150.152.208,"{""location"": ""FM"", ""is_mobile"": false}" 14025,5,314,2017-03-08 20:09:09,http://effertzbotsford.co/jamey.erdman,,61.193.217.84,"{""location"": ""MF"", ""is_mobile"": false}" 14026,5,314,2016-12-25 12:24:21,http://shanahan.biz/adolphus,,21.218.149.137,"{""location"": ""CF"", ""is_mobile"": true}" 14027,5,314,2016-12-28 12:49:40,http://christiansen.co/toby.ruecker,,189.159.139.190,"{""location"": ""ZM"", ""is_mobile"": true}" 14028,5,314,2017-03-03 20:02:20,http://anderson.co/shad,,139.141.244.242,"{""location"": ""KM"", ""is_mobile"": false}" 14029,5,314,2017-01-03 21:19:32,http://trantow.co/frederic,,163.16.207.122,"{""location"": ""TL"", ""is_mobile"": true}" 14030,5,314,2017-04-21 18:19:34,http://mann.com/elise.bednar,,139.60.201.217,"{""location"": ""AR"", ""is_mobile"": true}" 14031,5,314,2017-04-09 06:33:48,http://lebsack.net/eugenia,,201.62.101.102,"{""location"": ""MH"", ""is_mobile"": true}" 14032,5,314,2017-05-14 14:26:34,http://cronin.com/keanu_barrows,,8.20.6.148,"{""location"": ""ZM"", ""is_mobile"": true}" 14033,5,314,2017-01-12 02:43:49,http://jacobiadams.net/johnson_weinat,,121.115.171.224,"{""location"": ""NF"", ""is_mobile"": true}" 14034,5,314,2017-03-28 09:38:26,http://runte.io/edgar_rempel,,22.185.248.94,"{""location"": ""SA"", ""is_mobile"": false}" 14035,5,314,2017-03-01 06:03:46,http://borer.co/cecelia,,219.11.86.129,"{""location"": ""GM"", ""is_mobile"": false}" 14036,5,314,2016-12-25 07:57:27,http://halvorson.org/thea,,208.82.97.251,"{""location"": ""KG"", ""is_mobile"": true}" 14037,5,314,2017-05-22 05:47:24,http://larsonsporer.com/anibal_nader,,239.19.132.245,"{""location"": ""DK"", ""is_mobile"": true}" 14038,5,314,2017-05-16 05:33:50,http://nader.net/jarred,,115.155.181.254,"{""location"": ""TH"", ""is_mobile"": true}" 14039,5,314,2017-03-24 16:59:46,http://watsica.name/dylan.johnston,,50.167.103.185,"{""location"": ""AZ"", ""is_mobile"": true}" 14040,5,314,2017-02-02 23:16:39,http://greenfelder.org/kaylah,,72.150.130.150,"{""location"": ""RU"", ""is_mobile"": true}" 14041,5,314,2017-02-20 09:28:39,http://rice.net/haskell,,95.215.240.36,"{""location"": ""CU"", ""is_mobile"": true}" 14042,5,314,2017-01-03 05:07:03,http://jones.info/emilia,,110.171.216.242,"{""location"": ""SN"", ""is_mobile"": true}" 14043,5,315,2017-06-12 23:48:24,http://adams.info/morris_tremblay,,87.97.216.68,"{""location"": ""SD"", ""is_mobile"": false}" 14044,5,315,2017-04-25 21:52:26,http://connellyhills.org/lindsey.yundt,,54.142.12.188,"{""location"": ""AZ"", ""is_mobile"": true}" 14045,5,315,2016-12-29 12:51:06,http://feeney.com/jamarcus.bauch,,188.251.168.73,"{""location"": ""GQ"", ""is_mobile"": false}" 14046,5,315,2016-12-27 05:51:30,http://murray.com/ron,,247.165.130.70,"{""location"": ""HT"", ""is_mobile"": true}" 14047,5,315,2017-04-30 20:58:17,http://keler.co/elmira,,185.135.147.33,"{""location"": ""PL"", ""is_mobile"": false}" 14048,5,315,2017-01-06 11:35:29,http://schoen.com/jaron,,238.55.237.122,"{""location"": ""CO"", ""is_mobile"": true}" 14049,5,315,2017-03-12 04:57:14,http://sauer.net/albertha,,73.167.124.164,"{""location"": ""SV"", ""is_mobile"": true}" 14050,5,315,2017-01-31 04:10:54,http://turnerkoch.name/rene,,172.119.40.158,"{""location"": ""FO"", ""is_mobile"": false}" 14051,5,315,2017-05-19 22:08:54,http://green.io/anne,,63.9.169.102,"{""location"": ""UA"", ""is_mobile"": false}" 14052,5,315,2017-02-24 23:13:22,http://gottlieb.co/francesco.gleason,,197.45.140.235,"{""location"": ""VG"", ""is_mobile"": false}" 14053,5,315,2017-01-06 08:18:06,http://ebertschiller.com/alene.wiza,,242.20.250.229,"{""location"": ""SE"", ""is_mobile"": true}" 14054,5,315,2017-04-11 08:13:52,http://west.org/kristin,,230.254.10.203,"{""location"": ""TV"", ""is_mobile"": true}" 14055,5,315,2017-03-08 13:10:55,http://handtorp.name/buford.prohaska,,110.68.18.230,"{""location"": ""FI"", ""is_mobile"": false}" 14056,5,315,2017-01-11 10:25:33,http://hodkiewiczflatley.net/brooklyn_sawayn,,84.23.61.254,"{""location"": ""ZA"", ""is_mobile"": false}" 16983,6,384,2017-01-01 17:59:31,http://turner.org/kevin,,33.245.212.28,"{""location"": ""BZ"", ""is_mobile"": false}" 16984,6,384,2017-03-13 22:13:45,http://prohaska.net/rosie_kohler,,249.184.212.220,"{""location"": ""KP"", ""is_mobile"": true}" 16985,6,384,2017-02-13 20:48:19,http://rooborn.com/yasmin.mckenzie,,104.214.38.68,"{""location"": ""MR"", ""is_mobile"": true}" 16986,6,384,2017-04-06 07:44:42,http://hoppemitchell.org/reie_grant,,175.86.102.135,"{""location"": ""BH"", ""is_mobile"": false}" 16987,6,384,2017-06-06 14:11:02,http://collins.com/bailee,,209.94.18.120,"{""location"": ""GN"", ""is_mobile"": false}" 16988,6,384,2017-04-24 04:32:07,http://harriscarroll.biz/alana,,200.126.218.178,"{""location"": ""TR"", ""is_mobile"": false}" 16989,6,384,2017-01-31 09:34:06,http://muller.com/leilani,,82.28.108.175,"{""location"": ""CX"", ""is_mobile"": true}" 16990,6,384,2017-05-06 11:39:33,http://bednarbogisich.net/serena.breitenberg,,240.57.217.217,"{""location"": ""LT"", ""is_mobile"": false}" 16991,6,384,2017-01-23 18:57:07,http://simonis.biz/margarette,,163.21.123.25,"{""location"": ""VI"", ""is_mobile"": false}" 16992,6,384,2017-06-01 08:47:17,http://green.io/arvid.littel,,185.9.132.180,"{""location"": ""CN"", ""is_mobile"": true}" 16993,6,384,2017-03-22 12:51:23,http://mertz.name/adah,,113.6.29.235,"{""location"": ""IS"", ""is_mobile"": true}" 16994,6,384,2017-05-26 04:27:51,http://hartmannpowlowski.com/rodger,,56.162.139.196,"{""location"": ""DZ"", ""is_mobile"": true}" 16995,6,384,2017-01-01 14:20:18,http://borergrimes.net/cullen.ko,,198.151.108.141,"{""location"": ""WS"", ""is_mobile"": false}" 16996,6,385,2017-05-23 11:01:42,http://quitzon.com/karina,,157.160.245.108,"{""location"": ""KE"", ""is_mobile"": false}" 16997,6,385,2016-12-21 09:08:41,http://cruickshank.info/phoebe_grady,,56.26.203.118,"{""location"": ""PY"", ""is_mobile"": true}" 16998,6,385,2017-04-28 02:01:18,http://sawayn.biz/ruthe.harris,,209.128.27.92,"{""location"": ""HR"", ""is_mobile"": false}" 16999,6,385,2017-05-18 17:00:58,http://ortiz.com/romaine,,128.68.209.254,"{""location"": ""ZM"", ""is_mobile"": false}" 17000,6,385,2016-12-17 22:57:38,http://west.biz/eleazar.bayer,,157.211.71.20,"{""location"": ""ER"", ""is_mobile"": false}" 17001,6,385,2017-01-26 09:09:42,http://stoltenberg.com/daniella.frami,,123.41.129.83,"{""location"": ""BA"", ""is_mobile"": true}" 17002,6,385,2017-03-21 10:12:02,http://marquardt.biz/rollin,,251.30.154.237,"{""location"": ""GD"", ""is_mobile"": false}" 17003,6,385,2016-12-17 04:21:23,http://orn.io/raven.lubowitz,,177.245.239.215,"{""location"": ""VI"", ""is_mobile"": true}" 17004,6,385,2017-04-18 03:29:14,http://kunzehackett.biz/zoe,,3.183.224.136,"{""location"": ""SJ"", ""is_mobile"": false}" 17005,6,385,2017-06-07 20:16:36,http://thielmclaughlin.org/elva.runte,,169.147.131.98,"{""location"": ""AI"", ""is_mobile"": true}" 17006,6,385,2017-01-19 06:13:32,http://stanton.io/joannie_macejkovic,,153.45.143.73,"{""location"": ""MS"", ""is_mobile"": false}" 17007,6,385,2017-03-14 01:14:09,http://durgan.org/ruell.sauer,,217.130.168.48,"{""location"": ""BQ"", ""is_mobile"": false}" 17008,6,385,2016-12-19 00:51:28,http://jerde.io/billie,,162.128.200.158,"{""location"": ""BS"", ""is_mobile"": false}" 17009,6,385,2017-04-19 14:23:38,http://bernier.net/annie,,101.101.223.33,"{""location"": ""SJ"", ""is_mobile"": false}" 17010,6,385,2017-06-08 17:54:35,http://gradyswift.info/rudy,,87.212.183.221,"{""location"": ""SC"", ""is_mobile"": true}" 17011,6,385,2017-04-22 12:33:25,http://graham.co/eugenia,,234.143.230.221,"{""location"": ""ID"", ""is_mobile"": false}" 17012,6,385,2017-01-31 07:03:58,http://grady.co/gail,,181.248.220.18,"{""location"": ""SD"", ""is_mobile"": true}" 17013,6,385,2017-03-07 08:03:10,http://osinski.org/vivien,,105.24.175.117,"{""location"": ""JP"", ""is_mobile"": true}" 17014,6,385,2017-01-27 09:29:14,http://mayer.info/arianna.hand,,25.237.36.65,"{""location"": ""DK"", ""is_mobile"": true}" 17015,6,385,2017-02-10 16:21:27,http://hartmannluettgen.io/clemmie.adams,,69.42.192.172,"{""location"": ""AO"", ""is_mobile"": true}" 17016,6,385,2017-04-10 06:44:19,http://townemccullough.io/cloyd_huels,,174.252.25.58,"{""location"": ""ZM"", ""is_mobile"": true}" 17017,6,385,2017-05-30 16:34:31,http://beahan.org/aylin,,140.237.50.161,"{""location"": ""CX"", ""is_mobile"": true}" 17018,6,385,2017-04-29 05:53:28,http://satterfield.biz/mara.littel,,99.204.128.93,"{""location"": ""BS"", ""is_mobile"": false}" 17019,6,385,2017-03-25 02:38:10,http://thompson.biz/peter,,113.202.232.201,"{""location"": ""JP"", ""is_mobile"": true}" 17020,6,385,2017-05-10 21:17:07,http://reichert.net/sylvan,,178.38.226.168,"{""location"": ""ME"", ""is_mobile"": true}" 17021,6,385,2017-03-02 01:45:50,http://littel.com/dino_littel,,206.116.68.151,"{""location"": ""GG"", ""is_mobile"": true}" 17022,6,385,2017-01-09 15:44:47,http://mccullough.com/manuela,,228.176.30.26,"{""location"": ""MG"", ""is_mobile"": true}" 17023,6,385,2017-06-10 22:00:02,http://fadel.name/bradford_quitzon,,167.160.208.173,"{""location"": ""MZ"", ""is_mobile"": false}" 17024,6,385,2017-02-19 22:11:28,http://murphy.com/willa_heidenreich,,181.99.23.74,"{""location"": ""SJ"", ""is_mobile"": false}" 17025,6,385,2017-04-19 08:00:25,http://johnston.name/jerrell,,250.190.145.98,"{""location"": ""PW"", ""is_mobile"": false}" 17026,6,385,2017-03-16 19:22:24,http://moore.com/keagan_carter,,244.185.130.58,"{""location"": ""CU"", ""is_mobile"": false}" 17027,6,385,2017-04-11 02:22:40,http://jaskolski.io/agnes.damore,,45.136.101.22,"{""location"": ""YE"", ""is_mobile"": false}" 17028,6,385,2017-04-23 13:40:40,http://marquardt.com/dawson,,130.218.82.155,"{""location"": ""TR"", ""is_mobile"": true}" 17029,6,385,2017-04-08 04:17:52,http://hayeskutch.com/raven.mcclure,,201.164.231.248,"{""location"": ""CU"", ""is_mobile"": true}" 17030,6,385,2017-02-07 11:35:16,http://terrycarroll.io/judge_welch,,34.207.14.121,"{""location"": ""NO"", ""is_mobile"": false}" 17031,6,385,2017-02-11 13:31:27,http://gerlachhaley.name/hilda,,77.70.38.252,"{""location"": ""AE"", ""is_mobile"": false}" 17032,6,385,2017-05-17 13:04:45,http://hintzhauck.co/candida,,221.239.124.104,"{""location"": ""PW"", ""is_mobile"": true}" 17033,6,385,2016-12-23 08:41:16,http://bashirian.name/kaycee.stanton,,163.108.25.75,"{""location"": ""IT"", ""is_mobile"": true}" 17034,6,385,2017-04-07 08:15:51,http://kling.name/alexandria,,217.24.133.62,"{""location"": ""SG"", ""is_mobile"": true}" 17035,6,385,2017-04-29 01:40:18,http://weber.name/orval.torp,,225.87.208.110,"{""location"": ""IR"", ""is_mobile"": true}" 17036,6,385,2017-05-01 17:15:19,http://mann.biz/barbara,,156.199.72.7,"{""location"": ""DJ"", ""is_mobile"": true}" 17037,6,385,2017-03-24 21:11:38,http://larkin.co/pink.rath,,110.183.102.134,"{""location"": ""AD"", ""is_mobile"": false}" 17038,6,385,2017-02-05 05:35:36,http://cain.co/melisa,,12.120.77.14,"{""location"": ""PW"", ""is_mobile"": false}" 4165,2,91,2017-06-04 17:49:03,http://nienowweimann.org/brendon_price,,172.231.231.128,"{""location"": ""GQ"", ""is_mobile"": false}" 4166,2,91,2017-06-09 10:19:34,http://hermistonhamill.com/kiara,,44.233.246.114,"{""location"": ""MG"", ""is_mobile"": true}" 4167,2,91,2017-04-03 05:04:11,http://strackecain.name/blair.stracke,,19.163.88.229,"{""location"": ""TC"", ""is_mobile"": false}" 4168,2,91,2017-05-31 23:32:27,http://langosh.info/gerda.kuhn,,165.135.115.15,"{""location"": ""DJ"", ""is_mobile"": true}" 4169,2,91,2017-01-16 10:12:03,http://connellycartwright.co/oran.roob,,142.4.98.91,"{""location"": ""MD"", ""is_mobile"": true}" 4170,2,91,2016-12-18 20:49:38,http://fisher.info/clinton,,70.44.115.250,"{""location"": ""TF"", ""is_mobile"": false}" 4171,2,91,2017-05-18 21:57:55,http://hegmannfahey.biz/dasia_bartell,,145.164.161.156,"{""location"": ""AO"", ""is_mobile"": true}" 4172,2,91,2017-05-16 15:51:52,http://abshire.biz/keeley_cole,,120.170.210.34,"{""location"": ""GL"", ""is_mobile"": false}" 4173,2,91,2017-02-01 19:56:22,http://naderbergnaum.org/priscilla.towne,,193.19.158.18,"{""location"": ""LK"", ""is_mobile"": false}" 4174,2,91,2017-01-30 15:15:58,http://smithmills.org/hollis.bednar,,20.122.136.215,"{""location"": ""LV"", ""is_mobile"": false}" 4176,2,91,2017-03-13 15:30:28,http://lefflermcdermott.name/seth,,201.245.56.169,"{""location"": ""BD"", ""is_mobile"": true}" 4177,2,91,2017-04-28 11:19:16,http://wardgraham.com/kelley_buckridge,,200.171.41.169,"{""location"": ""NR"", ""is_mobile"": false}" 4178,2,91,2016-12-25 15:08:39,http://champlinpowlowski.com/laney,,250.8.59.95,"{""location"": ""BQ"", ""is_mobile"": true}" 4179,2,91,2017-05-13 18:33:43,http://wolf.name/alena_kuhlman,,107.12.44.61,"{""location"": ""ST"", ""is_mobile"": false}" 4180,2,91,2017-03-08 01:24:33,http://beer.info/elizabeth,,9.63.149.113,"{""location"": ""WS"", ""is_mobile"": true}" 4181,2,91,2017-05-15 11:36:57,http://collier.com/dorothea.kuhlman,,204.225.88.80,"{""location"": ""FK"", ""is_mobile"": true}" 4182,2,91,2017-04-10 22:17:06,http://quigleywest.io/trenton,,206.188.51.116,"{""location"": ""SO"", ""is_mobile"": true}" 4183,2,92,2017-04-27 11:22:06,http://damorekerluke.co/marianna,,97.22.10.241,"{""location"": ""NF"", ""is_mobile"": false}" 4184,2,92,2017-06-08 15:53:15,http://hoegermoriette.info/cesar.kuphal,,38.223.116.174,"{""location"": ""MH"", ""is_mobile"": false}" 4185,2,92,2017-02-12 19:14:16,http://renner.info/fanny.fay,,197.229.122.229,"{""location"": ""SH"", ""is_mobile"": false}" 4186,2,92,2016-12-15 12:07:25,http://weber.co/ruben,,131.181.235.170,"{""location"": ""DJ"", ""is_mobile"": false}" 4187,2,92,2017-03-23 11:12:47,http://considinehudson.com/luther_stehr,,27.188.158.121,"{""location"": ""CY"", ""is_mobile"": true}" 4188,2,92,2017-03-05 21:10:32,http://eichmann.info/kendall,,185.186.207.161,"{""location"": ""VU"", ""is_mobile"": false}" 4189,2,92,2017-05-22 22:46:49,http://kubschowalter.name/alf.baumbach,,205.62.178.126,"{""location"": ""NR"", ""is_mobile"": true}" 4190,2,92,2017-01-09 12:35:17,http://miller.info/wanda,,80.165.84.247,"{""location"": ""AD"", ""is_mobile"": true}" 4191,2,92,2017-05-23 19:56:45,http://loweberge.net/tania,,159.223.6.44,"{""location"": ""NU"", ""is_mobile"": true}" 4192,2,92,2017-02-22 21:40:16,http://ratke.co/paxton.larson,,82.216.199.5,"{""location"": ""TK"", ""is_mobile"": true}" 4193,2,92,2017-02-01 21:15:48,http://gottlieb.co/maxine,,85.119.156.149,"{""location"": ""HK"", ""is_mobile"": false}" 4194,2,92,2017-04-23 21:11:56,http://nader.com/kathlyn.walsh,,145.164.140.187,"{""location"": ""AX"", ""is_mobile"": false}" 4195,2,92,2017-06-08 02:08:36,http://macgyver.co/ole.king,,93.62.5.71,"{""location"": ""BM"", ""is_mobile"": true}" 4196,2,92,2017-03-06 22:36:17,http://hoppeabbott.co/cooper,,174.149.116.53,"{""location"": ""HN"", ""is_mobile"": false}" 4197,2,92,2017-01-05 02:50:04,http://hettingerstark.biz/claria,,217.83.62.162,"{""location"": ""KM"", ""is_mobile"": true}" 4198,2,92,2016-12-31 17:32:10,http://mante.info/jennifer.murphy,,222.89.10.148,"{""location"": ""GI"", ""is_mobile"": false}" 4199,2,92,2017-04-01 14:50:03,http://strosin.info/shana.feest,,246.17.41.5,"{""location"": ""VI"", ""is_mobile"": false}" 4200,2,92,2017-05-21 23:40:19,http://johnston.com/bruce,,72.203.24.209,"{""location"": ""MG"", ""is_mobile"": true}" 4201,2,92,2017-04-19 20:11:06,http://buckridgedaniel.co/camren,,40.55.114.94,"{""location"": ""LU"", ""is_mobile"": true}" 4202,2,92,2016-12-14 14:04:17,http://koch.org/hoyt,,67.58.89.163,"{""location"": ""SR"", ""is_mobile"": true}" 4203,2,92,2017-02-04 20:47:30,http://kerluke.co/easton,,79.137.214.68,"{""location"": ""SB"", ""is_mobile"": false}" 4204,2,92,2017-01-28 10:27:48,http://zboncak.com/tatum_fisher,,156.128.70.21,"{""location"": ""NO"", ""is_mobile"": true}" 4205,2,92,2017-05-12 11:13:59,http://willms.co/jamal,,4.192.43.235,"{""location"": ""CD"", ""is_mobile"": true}" 4206,2,92,2017-05-13 14:18:59,http://mills.com/dallas,,145.154.91.130,"{""location"": ""LA"", ""is_mobile"": true}" 4207,2,92,2017-05-31 14:13:52,http://deckow.info/mylene_kshlerin,,2.166.227.205,"{""location"": ""BQ"", ""is_mobile"": true}" 4208,2,92,2017-03-06 20:00:16,http://brekke.name/calista,,87.87.15.251,"{""location"": ""PM"", ""is_mobile"": false}" 4209,2,92,2017-03-24 23:03:08,http://gaylord.name/pascale,,248.58.23.41,"{""location"": ""KP"", ""is_mobile"": true}" 4210,2,92,2017-02-12 13:32:55,http://smitham.name/savanah,,176.171.67.123,"{""location"": ""KH"", ""is_mobile"": true}" 4211,2,92,2017-02-09 18:07:56,http://ankundingkoelpin.name/jillian_marquardt,,51.85.67.232,"{""location"": ""FR"", ""is_mobile"": false}" 4212,2,92,2017-01-19 11:20:56,http://runolfon.com/mozelle_boyle,,10.70.18.244,"{""location"": ""TN"", ""is_mobile"": true}" 4213,2,92,2017-05-06 01:43:23,http://bode.net/evan,,10.148.166.114,"{""location"": ""JE"", ""is_mobile"": true}" 4214,2,92,2017-02-24 18:21:14,http://effertz.biz/aiyana.stanton,,116.126.248.31,"{""location"": ""MA"", ""is_mobile"": true}" 4215,2,92,2017-05-23 13:48:32,http://vandervortspinka.name/caidy_wuckert,,252.128.192.133,"{""location"": ""SE"", ""is_mobile"": false}" 4216,2,92,2017-01-30 09:09:00,http://ortizbreitenberg.name/haskell.murray,,62.121.193.191,"{""location"": ""LS"", ""is_mobile"": true}" 4217,2,92,2017-02-14 22:07:28,http://sanfordschroeder.co/ward,,46.159.139.120,"{""location"": ""UZ"", ""is_mobile"": false}" 4218,2,92,2017-03-25 20:19:23,http://satterfield.net/alexane,,53.71.109.168,"{""location"": ""FR"", ""is_mobile"": true}" 4219,2,92,2017-05-19 07:23:57,http://aufderhar.biz/ozella.schowalter,,12.29.12.37,"{""location"": ""LS"", ""is_mobile"": false}" 4220,2,92,2017-04-05 23:13:21,http://ondrickakertzmann.net/nadia,,154.227.188.62,"{""location"": ""RU"", ""is_mobile"": false}" 4221,2,92,2017-02-25 20:17:24,http://hintzferry.com/andreane_mosciski,,223.8.194.230,"{""location"": ""AG"", ""is_mobile"": false}" 11107,4,248,2017-03-18 01:58:26,http://gaylord.info/santina,,243.154.126.57,"{""location"": ""UA"", ""is_mobile"": true}" 11108,4,248,2017-03-22 19:52:40,http://kuvalisterry.biz/maureen,,56.96.101.172,"{""location"": ""BH"", ""is_mobile"": true}" 11109,4,248,2017-01-11 23:11:04,http://rohan.name/donavon_stanton,,40.27.101.215,"{""location"": ""MO"", ""is_mobile"": false}" 11110,4,248,2017-01-20 08:10:00,http://bayer.biz/sonya,,152.68.74.99,"{""location"": ""WF"", ""is_mobile"": false}" 11111,4,248,2017-03-11 09:39:48,http://hammesrogahn.org/kyle_dicki,,96.227.172.253,"{""location"": ""GA"", ""is_mobile"": false}" 11112,4,248,2017-01-15 08:50:04,http://smithamemmerich.co/mariana.boyle,,160.143.99.230,"{""location"": ""MZ"", ""is_mobile"": true}" 11113,4,248,2016-12-23 05:21:52,http://armstrong.io/shirley,,200.247.77.191,"{""location"": ""JE"", ""is_mobile"": true}" 11114,4,248,2017-06-13 15:16:26,http://greenfelderbergnaum.biz/bernita_block,,128.189.13.80,"{""location"": ""MQ"", ""is_mobile"": true}" 11115,4,248,2017-06-02 02:32:25,http://bartoletti.net/willie.hayes,,152.202.200.40,"{""location"": ""TW"", ""is_mobile"": true}" 11116,4,248,2017-01-18 12:21:32,http://lehner.com/rosendo,,218.32.115.123,"{""location"": ""MW"", ""is_mobile"": true}" 11117,4,249,2017-01-04 08:51:25,http://smitham.biz/jarred,,229.167.248.29,"{""location"": ""IS"", ""is_mobile"": false}" 11118,4,249,2017-01-09 13:12:50,http://bergstrom.info/eloisa.mueller,,68.111.151.29,"{""location"": ""ID"", ""is_mobile"": false}" 11119,4,249,2017-03-17 08:06:30,http://williamson.com/chaya,,133.105.234.213,"{""location"": ""GL"", ""is_mobile"": true}" 11120,4,249,2016-12-18 06:21:48,http://bradtkeframi.name/kitty.cormier,,187.30.160.165,"{""location"": ""LI"", ""is_mobile"": true}" 11121,4,249,2017-05-05 22:22:48,http://greenholt.biz/savion_nikolaus,,60.91.43.15,"{""location"": ""OM"", ""is_mobile"": true}" 11122,4,249,2017-01-12 23:26:41,http://ratkeorn.biz/jailyn,,91.130.170.90,"{""location"": ""PG"", ""is_mobile"": true}" 11123,4,249,2017-01-19 16:02:47,http://bogan.io/joan.pfeffer,,164.45.201.36,"{""location"": ""NC"", ""is_mobile"": true}" 11124,4,249,2017-03-05 14:17:52,http://harvey.biz/darryl,,248.182.87.119,"{""location"": ""BI"", ""is_mobile"": true}" 11125,4,249,2017-01-08 07:05:07,http://grant.org/angeline.kozey,,208.244.64.179,"{""location"": ""PH"", ""is_mobile"": false}" 11126,4,249,2017-04-23 01:03:19,http://kovacek.co/declan.hoeger,,218.161.154.225,"{""location"": ""NE"", ""is_mobile"": false}" 11127,4,249,2017-05-31 06:18:08,http://vonruedenklein.com/sigmund_abbott,,109.238.74.14,"{""location"": ""SE"", ""is_mobile"": false}" 11128,4,249,2017-01-25 01:05:59,http://wilkinson.name/lisette,,157.78.84.31,"{""location"": ""KW"", ""is_mobile"": false}" 11129,4,249,2017-03-17 16:17:32,http://jerdeortiz.biz/imani,,114.150.58.202,"{""location"": ""MC"", ""is_mobile"": false}" 11130,4,249,2017-01-18 08:24:55,http://kaulke.io/ariane,,204.23.81.33,"{""location"": ""CD"", ""is_mobile"": true}" 11131,4,249,2017-01-16 21:49:37,http://bins.org/kamille,,163.62.225.135,"{""location"": ""LU"", ""is_mobile"": true}" 11132,4,249,2017-04-06 03:36:24,http://dooley.name/vergie,,218.45.168.111,"{""location"": ""AM"", ""is_mobile"": true}" 11133,4,249,2017-03-19 09:31:13,http://schimmelborer.com/aida.barton,,92.67.98.30,"{""location"": ""GQ"", ""is_mobile"": false}" 11134,4,249,2017-01-01 09:54:14,http://wehneraufderhar.name/michael.funk,,140.224.8.34,"{""location"": ""PH"", ""is_mobile"": false}" 11135,4,249,2017-06-08 20:55:01,http://gutkowskihowe.com/annette,,193.68.69.153,"{""location"": ""PG"", ""is_mobile"": true}" 11136,4,249,2017-02-24 01:22:50,http://carter.net/glennie_dubuque,,147.23.134.111,"{""location"": ""SC"", ""is_mobile"": true}" 11137,4,249,2017-04-19 04:04:04,http://pacocha.org/okey,,150.230.214.62,"{""location"": ""HM"", ""is_mobile"": false}" 11138,4,249,2016-12-16 14:02:41,http://turner.biz/aubree,,76.178.160.163,"{""location"": ""IO"", ""is_mobile"": true}" 11139,4,249,2017-01-09 15:57:17,http://zulaufkeebler.org/fatima,,193.58.48.218,"{""location"": ""RE"", ""is_mobile"": true}" 11140,4,249,2017-05-18 09:16:22,http://starkcrona.info/august_pfannerstill,,148.116.173.244,"{""location"": ""RO"", ""is_mobile"": true}" 11141,4,249,2017-02-08 19:08:53,http://rutherford.biz/cletus,,14.9.42.17,"{""location"": ""BG"", ""is_mobile"": true}" 11142,4,249,2017-01-25 21:44:21,http://rempel.net/fredy.bode,,127.189.223.171,"{""location"": ""GB"", ""is_mobile"": false}" 11143,4,249,2017-01-07 08:57:32,http://runte.io/gerda,,174.90.204.35,"{""location"": ""BB"", ""is_mobile"": false}" 11144,4,249,2017-04-08 00:59:09,http://hodkiewiczmarks.com/reggie.gusikowski,,39.3.17.107,"{""location"": ""JP"", ""is_mobile"": true}" 11145,4,249,2016-12-21 20:49:11,http://krajcikkeeling.info/lewis_stokes,,215.120.116.70,"{""location"": ""PS"", ""is_mobile"": false}" 11146,4,249,2017-02-03 05:04:03,http://blick.org/ricardo,,212.113.66.184,"{""location"": ""HM"", ""is_mobile"": true}" 11147,4,249,2017-04-18 08:50:25,http://feeneygutmann.io/sharon.stanton,,33.130.213.120,"{""location"": ""LA"", ""is_mobile"": true}" 11148,4,249,2017-05-10 11:06:18,http://torpschaefer.biz/jaeden,,239.125.74.5,"{""location"": ""YT"", ""is_mobile"": true}" 11149,4,249,2017-01-03 13:09:23,http://trantow.net/darrick_hilpert,,236.224.51.195,"{""location"": ""IO"", ""is_mobile"": true}" 11150,4,249,2017-03-27 05:00:09,http://schoenschmidt.com/benton.dickens,,67.105.103.47,"{""location"": ""MU"", ""is_mobile"": true}" 11151,4,249,2017-01-24 12:49:13,http://price.net/kris.hagenes,,188.192.110.51,"{""location"": ""MY"", ""is_mobile"": false}" 11152,4,249,2017-03-22 06:26:17,http://kreigerberge.org/dane_bernier,,202.137.105.7,"{""location"": ""JP"", ""is_mobile"": true}" 11153,4,249,2017-02-03 23:52:28,http://wisoky.info/kelli,,234.39.166.78,"{""location"": ""LV"", ""is_mobile"": false}" 11154,4,249,2017-01-07 18:38:46,http://webergibson.net/thelma,,60.188.193.49,"{""location"": ""VN"", ""is_mobile"": false}" 11155,4,249,2017-02-17 18:19:07,http://carroll.info/hyman,,178.104.42.84,"{""location"": ""MO"", ""is_mobile"": false}" 11156,4,249,2017-04-09 23:14:49,http://macejkovicmuller.io/antonina.muller,,164.165.169.129,"{""location"": ""IM"", ""is_mobile"": true}" 11157,4,249,2017-03-03 16:50:47,http://crist.biz/giles.oberbrunner,,159.194.231.193,"{""location"": ""CR"", ""is_mobile"": true}" 11158,4,249,2017-06-14 01:17:33,http://bednarsimonis.com/jon,,212.136.102.242,"{""location"": ""YE"", ""is_mobile"": false}" 11159,4,249,2017-05-16 02:19:39,http://becker.biz/reinhold,,121.54.128.251,"{""location"": ""ME"", ""is_mobile"": true}" 11160,4,249,2017-01-28 19:22:08,http://howell.biz/cordia.carter,,141.144.40.249,"{""location"": ""MG"", ""is_mobile"": false}" 11161,4,249,2017-01-15 15:52:45,http://sauer.name/jerome.jones,,203.197.84.84,"{""location"": ""JE"", ""is_mobile"": true}" 8119,3,179,2017-02-28 14:40:53,http://gaylorddietrich.name/ubaldo,0.9833766270,83.239.150.97,"{""location"": ""CY"", ""is_mobile"": false}" 8120,3,179,2016-12-16 23:14:58,http://swaniawski.net/felipa_ortiz,0.9752975927,245.167.249.145,"{""location"": ""BQ"", ""is_mobile"": true}" 8121,3,179,2017-03-31 21:33:28,http://lubowitz.biz/maurine_fisher,0.1800505568,57.124.226.173,"{""location"": ""RS"", ""is_mobile"": true}" 8122,3,179,2017-05-14 20:57:14,http://swiftebert.name/aurore,0.6416575664,70.252.178.31,"{""location"": ""GI"", ""is_mobile"": true}" 8123,3,179,2017-01-07 05:50:51,http://cormierko.co/roderick.labadie,0.6089417187,158.13.36.85,"{""location"": ""TL"", ""is_mobile"": false}" 8124,3,179,2016-12-24 12:03:36,http://schuster.com/bennett_funk,0.1694993514,66.99.81.56,"{""location"": ""MV"", ""is_mobile"": false}" 8125,3,179,2017-01-19 05:46:26,http://ziemann.info/mohammed,0.1703540219,48.80.3.149,"{""location"": ""CA"", ""is_mobile"": true}" 8126,3,179,2017-01-28 19:40:39,http://labadie.com/eusebio,0.1311062207,236.31.156.109,"{""location"": ""MG"", ""is_mobile"": false}" 8127,3,179,2017-03-10 04:59:10,http://mertz.io/esperanza.mclaughlin,0.7816323585,11.192.2.35,"{""location"": ""PG"", ""is_mobile"": false}" 8128,3,179,2017-05-27 10:21:31,http://frami.net/bartholome_wilderman,0.0808691505,105.198.88.32,"{""location"": ""YE"", ""is_mobile"": false}" 8129,3,179,2017-05-23 07:50:29,http://weimann.com/davonte_welch,0.6874861730,19.158.177.13,"{""location"": ""JE"", ""is_mobile"": true}" 8130,3,179,2017-04-16 15:17:48,http://bergstromcummerata.biz/camden,0.4292808910,102.69.153.25,"{""location"": ""EE"", ""is_mobile"": true}" 8131,3,179,2016-12-22 14:53:43,http://bradtkeborer.net/lester,0.3202388588,70.184.204.196,"{""location"": ""FK"", ""is_mobile"": true}" 8132,3,179,2017-03-06 00:37:57,http://stiedemannvolkman.info/dock,0.0787335352,234.8.232.216,"{""location"": ""KR"", ""is_mobile"": false}" 8133,3,179,2017-02-26 11:03:13,http://windlerhuel.biz/rafael.oberbrunner,0.9360750066,254.51.96.68,"{""location"": ""BQ"", ""is_mobile"": true}" 8134,3,179,2017-05-22 03:23:41,http://macejkovickuvalis.io/stanton,0.1736102132,126.86.137.87,"{""location"": ""NO"", ""is_mobile"": true}" 8135,3,179,2017-06-03 13:09:27,http://fritschbernier.io/kailyn.cartwright,0.3242093915,119.178.238.28,"{""location"": ""ST"", ""is_mobile"": true}" 8136,3,179,2017-02-02 23:40:33,http://medhurst.co/elmo,0.8692437666,67.134.114.51,"{""location"": ""GQ"", ""is_mobile"": true}" 8137,3,179,2017-02-02 03:54:58,http://rolfson.biz/alfred,0.4938066717,226.228.236.182,"{""location"": ""FO"", ""is_mobile"": false}" 8138,3,179,2017-06-02 23:28:26,http://gislason.co/derrick_ritchie,0.0582383972,99.19.145.141,"{""location"": ""NU"", ""is_mobile"": true}" 8139,3,179,2017-02-09 16:50:58,http://zboncak.info/thurman,0.9918884059,110.28.212.120,"{""location"": ""UZ"", ""is_mobile"": false}" 8140,3,179,2017-06-01 22:55:57,http://breitenberghuel.io/bradley,0.0951423070,71.29.82.113,"{""location"": ""PK"", ""is_mobile"": false}" 8141,3,179,2017-04-02 09:17:39,http://wintheiser.name/amina_franecki,0.1945663505,175.157.106.249,"{""location"": ""MN"", ""is_mobile"": true}" 8142,3,179,2017-02-23 21:54:10,http://toy.co/karina,0.0582550466,155.229.9.228,"{""location"": ""KE"", ""is_mobile"": true}" 8143,3,179,2017-01-08 22:56:07,http://kshlerinzemlak.biz/kody.kshlerin,0.4314491278,2.185.214.40,"{""location"": ""DK"", ""is_mobile"": false}" 8144,3,179,2017-02-15 08:43:18,http://schustergibson.org/manley.klein,0.1122279861,134.201.11.14,"{""location"": ""CU"", ""is_mobile"": true}" 8145,3,179,2017-05-10 16:13:14,http://labadiedoyle.net/grayce,0.4683074123,225.131.62.83,"{""location"": ""JO"", ""is_mobile"": false}" 8146,3,179,2017-04-20 08:14:14,http://dooley.info/adam.gleason,0.2314141091,94.199.77.191,"{""location"": ""BW"", ""is_mobile"": true}" 8147,3,179,2017-06-08 12:45:26,http://thiel.name/jensen.shanahan,0.7854670443,49.161.132.233,"{""location"": ""LS"", ""is_mobile"": true}" 8148,3,179,2017-02-24 19:37:48,http://mueller.net/friedrich_nolan,0.9584563608,113.3.86.142,"{""location"": ""MY"", ""is_mobile"": true}" 8149,3,180,2017-05-09 05:32:27,http://deckow.co/tillman,0.9855826547,76.159.60.49,"{""location"": ""FK"", ""is_mobile"": true}" 8150,3,180,2017-02-06 20:42:27,http://murraypouros.name/verna.rosenbaum,0.3361983200,145.63.147.146,"{""location"": ""SJ"", ""is_mobile"": false}" 8151,3,180,2017-03-08 08:15:11,http://huels.name/bertha,0.6639968164,50.243.24.191,"{""location"": ""RE"", ""is_mobile"": true}" 8152,3,180,2017-01-27 23:50:07,http://mueller.net/verdie.yundt,0.3773574593,96.92.33.43,"{""location"": ""BA"", ""is_mobile"": false}" 8153,3,180,2017-01-14 15:50:05,http://reichel.org/moie_shields,0.4219320101,56.194.32.134,"{""location"": ""MZ"", ""is_mobile"": true}" 8154,3,180,2016-12-29 21:32:13,http://jerde.biz/markus,0.0550689777,242.126.154.88,"{""location"": ""LK"", ""is_mobile"": false}" 8155,3,180,2017-03-13 06:18:55,http://watsica.biz/frederick,0.0468130118,193.136.184.11,"{""location"": ""TF"", ""is_mobile"": true}" 8156,3,180,2017-03-20 04:26:47,http://caspernader.io/shanon.leannon,0.6977839157,35.134.55.233,"{""location"": ""BO"", ""is_mobile"": true}" 8157,3,180,2017-02-15 07:11:22,http://bergnaumjohns.org/juliana,0.7221265357,129.40.24.125,"{""location"": ""MG"", ""is_mobile"": false}" 8158,3,180,2017-05-04 21:33:09,http://adams.co/simeon,0.6546617703,195.211.76.251,"{""location"": ""BE"", ""is_mobile"": true}" 8159,3,180,2016-12-21 21:53:09,http://lind.com/imani,0.0179045825,74.92.102.197,"{""location"": ""SH"", ""is_mobile"": false}" 8160,3,180,2017-04-06 22:30:56,http://lebsacklowe.biz/esteban,0.5952879980,148.21.182.180,"{""location"": ""UY"", ""is_mobile"": true}" 8161,3,180,2017-04-29 10:15:30,http://schadenyundt.com/giovanna,0.6107913709,183.2.154.153,"{""location"": ""SX"", ""is_mobile"": true}" 8162,3,180,2017-04-08 23:29:55,http://wiegand.co/hulda.weimann,0.6796279636,121.51.2.98,"{""location"": ""PF"", ""is_mobile"": false}" 8163,3,180,2017-05-31 21:23:28,http://tromp.info/sarah_ward,0.2936674032,249.230.186.249,"{""location"": ""TF"", ""is_mobile"": false}" 8164,3,180,2017-03-07 12:22:40,http://schowalterpaucek.org/jee,0.7866166519,140.227.116.105,"{""location"": ""YT"", ""is_mobile"": false}" 8165,3,180,2017-02-07 13:47:38,http://von.biz/wade,0.3692678191,137.46.86.105,"{""location"": ""CR"", ""is_mobile"": false}" 8166,3,180,2017-04-14 06:28:37,http://krajcik.co/adolph.eichmann,0.3531283773,158.250.29.41,"{""location"": ""MZ"", ""is_mobile"": false}" 8167,3,180,2017-01-20 03:20:25,http://lesch.net/travon_hilll,0.3158172017,222.219.90.41,"{""location"": ""LI"", ""is_mobile"": false}" 8168,3,180,2017-01-07 14:09:46,http://schneiderabbott.io/bria.emmerich,0.4104441183,40.31.40.159,"{""location"": ""GW"", ""is_mobile"": true}" 8169,3,180,2016-12-17 11:16:28,http://breitenberg.biz/alec_wiza,0.4124026464,194.91.54.140,"{""location"": ""DO"", ""is_mobile"": false}" 14057,5,315,2017-05-13 22:39:57,http://gaylord.io/adrian.will,,10.225.182.35,"{""location"": ""LY"", ""is_mobile"": true}" 14058,5,315,2017-03-20 01:57:33,http://kiehnbahringer.co/rosanna,,135.179.230.66,"{""location"": ""UZ"", ""is_mobile"": true}" 14059,5,315,2017-04-11 02:06:51,http://spinka.biz/brando,,74.222.148.229,"{""location"": ""NZ"", ""is_mobile"": true}" 14060,5,315,2017-02-28 09:37:36,http://kling.io/noah,,235.28.86.7,"{""location"": ""SN"", ""is_mobile"": true}" 14061,5,315,2017-01-16 05:01:29,http://ohara.com/eliezer,,248.42.134.190,"{""location"": ""HT"", ""is_mobile"": true}" 14062,5,315,2017-04-09 03:34:45,http://heaney.org/tina_bergstrom,,96.43.98.162,"{""location"": ""NC"", ""is_mobile"": false}" 14063,5,315,2017-03-01 21:36:02,http://dooley.biz/rasheed.walter,,226.163.156.168,"{""location"": ""FR"", ""is_mobile"": false}" 14064,5,315,2017-02-23 19:51:51,http://christiansen.biz/tyreek.oconnell,,234.42.152.124,"{""location"": ""SY"", ""is_mobile"": false}" 14065,5,316,2017-01-16 00:31:39,http://towne.org/maegan,,164.87.5.24,"{""location"": ""PW"", ""is_mobile"": false}" 14066,5,316,2017-05-13 15:50:26,http://dickinsonbashirian.com/brycen.keebler,,121.132.94.209,"{""location"": ""FM"", ""is_mobile"": false}" 14067,5,316,2017-05-18 12:12:33,http://emard.biz/rodrigo_mante,,99.100.124.21,"{""location"": ""KG"", ""is_mobile"": false}" 14068,5,316,2016-12-19 13:21:45,http://hayeslabadie.name/adrien_pacocha,,186.202.64.122,"{""location"": ""ES"", ""is_mobile"": false}" 14069,5,316,2017-05-28 19:01:49,http://wehner.io/eli.doyle,,119.118.160.201,"{""location"": ""AL"", ""is_mobile"": true}" 14070,5,316,2016-12-20 10:02:11,http://stokes.info/lourdes,,32.102.174.241,"{""location"": ""JP"", ""is_mobile"": true}" 14071,5,316,2017-03-17 06:34:42,http://goldnergutmann.net/richmond,,148.180.120.21,"{""location"": ""SS"", ""is_mobile"": true}" 14072,5,316,2016-12-30 14:24:04,http://armstrongweimann.net/lou_nitzsche,,36.122.153.102,"{""location"": ""AQ"", ""is_mobile"": true}" 14073,5,316,2017-03-19 02:56:16,http://luettgen.io/adolphus,,197.108.119.2,"{""location"": ""SM"", ""is_mobile"": false}" 14074,5,316,2017-05-06 02:12:24,http://hirthe.org/darlene.swift,,133.52.161.184,"{""location"": ""AO"", ""is_mobile"": false}" 14075,5,316,2017-01-11 17:49:28,http://champlinkohler.com/jazmyn,,148.200.32.233,"{""location"": ""GE"", ""is_mobile"": true}" 14076,5,316,2017-01-01 22:49:10,http://altenwerth.com/tracey_cummings,,49.203.144.82,"{""location"": ""TG"", ""is_mobile"": true}" 14077,5,316,2017-04-26 01:33:25,http://kiehn.co/armani.deckow,,183.200.252.122,"{""location"": ""BJ"", ""is_mobile"": true}" 14078,5,316,2017-06-12 08:21:12,http://eichmann.co/gudrun,,149.167.23.164,"{""location"": ""PA"", ""is_mobile"": true}" 14079,5,316,2017-05-03 10:05:15,http://glover.info/dayne_schmeler,,94.76.67.140,"{""location"": ""TD"", ""is_mobile"": true}" 14080,5,316,2017-04-16 14:08:58,http://mante.com/clay,,91.157.26.79,"{""location"": ""RS"", ""is_mobile"": false}" 14081,5,316,2017-02-22 05:53:53,http://jacobs.biz/delphine,,213.91.174.54,"{""location"": ""VG"", ""is_mobile"": true}" 14082,5,316,2017-04-29 02:00:05,http://senger.com/veronica.dach,,174.248.171.79,"{""location"": ""SV"", ""is_mobile"": true}" 14083,5,316,2017-03-27 22:17:31,http://maggio.net/hector_crooks,,165.143.56.139,"{""location"": ""VG"", ""is_mobile"": true}" 14084,5,316,2017-04-24 23:25:49,http://ortizwehner.io/laria_legros,,238.13.57.49,"{""location"": ""GG"", ""is_mobile"": true}" 14085,5,316,2017-01-13 16:05:42,http://metz.co/pamela,,197.45.159.235,"{""location"": ""US"", ""is_mobile"": true}" 14086,5,316,2017-06-02 12:24:49,http://hickle.biz/mireya.thompson,,19.122.71.192,"{""location"": ""DO"", ""is_mobile"": true}" 14087,5,316,2017-02-11 22:44:28,http://sanfordmayer.info/adriana,,36.234.192.19,"{""location"": ""MW"", ""is_mobile"": true}" 14088,5,316,2017-04-27 17:54:56,http://rippin.org/roxanne,,70.156.24.65,"{""location"": ""UA"", ""is_mobile"": true}" 14089,5,316,2017-03-22 13:37:17,http://rogahn.name/adonis,,215.37.187.6,"{""location"": ""UZ"", ""is_mobile"": true}" 14090,5,316,2017-02-15 09:04:28,http://marquardtkuvalis.name/hettie_robel,,81.229.23.161,"{""location"": ""UZ"", ""is_mobile"": true}" 14091,5,316,2017-05-30 22:34:58,http://greenholt.org/keshawn,,200.166.26.38,"{""location"": ""LI"", ""is_mobile"": false}" 14092,5,316,2017-01-20 19:58:24,http://hermiston.co/brionna_mante,,177.10.151.236,"{""location"": ""VC"", ""is_mobile"": true}" 14093,5,316,2017-01-22 05:42:41,http://murraymcglynn.org/sydney_lowe,,54.216.144.92,"{""location"": ""TG"", ""is_mobile"": false}" 14094,5,316,2017-03-31 05:53:06,http://cormier.io/shanel.reichel,,225.76.245.47,"{""location"": ""SA"", ""is_mobile"": true}" 14095,5,316,2017-03-07 17:12:44,http://leffler.net/krystal,,191.50.218.58,"{""location"": ""GM"", ""is_mobile"": false}" 14096,5,316,2017-01-01 13:05:58,http://ullrich.biz/keegan,,5.11.113.100,"{""location"": ""ET"", ""is_mobile"": true}" 14097,5,316,2017-01-24 14:29:28,http://raynorzulauf.net/kacie,,103.121.229.155,"{""location"": ""EE"", ""is_mobile"": false}" 14098,5,316,2017-02-24 12:04:57,http://oreilly.io/queen.lueilwitz,,39.160.85.114,"{""location"": ""VE"", ""is_mobile"": false}" 14099,5,316,2017-02-10 17:43:34,http://denesikprohaska.biz/osborne,,52.227.91.7,"{""location"": ""KI"", ""is_mobile"": true}" 14100,5,316,2017-02-14 12:49:12,http://hodkiewicz.org/bella.kuhlman,,112.110.244.209,"{""location"": ""AS"", ""is_mobile"": true}" 14101,5,316,2016-12-31 19:43:35,http://starkkovacek.info/kory.rath,,241.83.211.162,"{""location"": ""PT"", ""is_mobile"": false}" 14102,5,316,2017-02-17 12:37:18,http://pollichkuhic.net/tamara_denesik,,37.27.91.141,"{""location"": ""MN"", ""is_mobile"": true}" 14103,5,316,2016-12-20 22:28:29,http://kemmer.io/arlie,,122.14.75.43,"{""location"": ""EH"", ""is_mobile"": false}" 14104,5,316,2017-06-05 03:54:53,http://stoltenberg.com/roel,,102.130.226.191,"{""location"": ""JM"", ""is_mobile"": false}" 14105,5,316,2016-12-19 06:22:23,http://thielpadberg.net/rosalee_hansen,,236.101.184.63,"{""location"": ""SH"", ""is_mobile"": true}" 14106,5,316,2017-04-02 07:41:11,http://ullrich.biz/aditya,,233.59.107.95,"{""location"": ""SV"", ""is_mobile"": false}" 14107,5,316,2017-02-24 08:25:04,http://crist.biz/lucio,,62.94.180.175,"{""location"": ""VE"", ""is_mobile"": true}" 14108,5,316,2016-12-30 20:20:15,http://wilderman.name/andre.witting,,173.54.69.55,"{""location"": ""JE"", ""is_mobile"": false}" 14109,5,316,2016-12-18 21:29:48,http://durgan.io/oral,,250.135.179.156,"{""location"": ""GF"", ""is_mobile"": true}" 14110,5,316,2017-04-16 04:10:37,http://leuschke.com/haylie,,218.145.100.126,"{""location"": ""IL"", ""is_mobile"": false}" 14111,5,316,2017-03-07 05:54:26,http://orn.biz/bette,,237.232.31.229,"{""location"": ""SR"", ""is_mobile"": false}" 14112,5,316,2017-01-15 09:34:50,http://renner.co/tatyana,,142.30.243.30,"{""location"": ""SJ"", ""is_mobile"": false}" 17039,6,385,2017-03-20 11:51:41,http://croninjohnson.co/bennett,,120.127.64.40,"{""location"": ""RS"", ""is_mobile"": true}" 17040,6,385,2017-03-17 17:59:04,http://larsonrutherford.io/mathias.gottlieb,,190.12.63.171,"{""location"": ""MG"", ""is_mobile"": false}" 17041,6,385,2017-05-30 05:11:18,http://kuvalichaefer.name/retta,,224.185.219.216,"{""location"": ""GE"", ""is_mobile"": false}" 17042,6,385,2017-01-08 23:44:10,http://sauerkris.org/kennith,,56.67.226.115,"{""location"": ""LU"", ""is_mobile"": false}" 17043,6,385,2017-04-08 07:16:57,http://cain.name/aurore,,59.67.238.89,"{""location"": ""MZ"", ""is_mobile"": true}" 17044,6,385,2017-02-22 19:52:02,http://kulas.com/collin.towne,,74.5.134.115,"{""location"": ""LR"", ""is_mobile"": true}" 17045,6,385,2017-05-05 06:52:52,http://swaniawski.com/justyn_kertzmann,,196.190.50.135,"{""location"": ""MD"", ""is_mobile"": true}" 17046,6,385,2017-02-26 02:53:04,http://lesch.org/rylan.wiza,,182.106.131.46,"{""location"": ""NG"", ""is_mobile"": true}" 17047,6,385,2017-05-18 04:39:52,http://zboncak.info/georgianna,,153.226.117.22,"{""location"": ""FJ"", ""is_mobile"": false}" 17048,6,385,2017-06-09 07:53:43,http://grady.name/john,,114.187.113.223,"{""location"": ""FM"", ""is_mobile"": false}" 17049,6,385,2017-01-27 13:38:05,http://rath.com/amaya,,188.122.6.181,"{""location"": ""BO"", ""is_mobile"": false}" 17050,6,385,2017-01-13 22:23:41,http://batz.co/juanita_reinger,,92.204.134.235,"{""location"": ""MD"", ""is_mobile"": true}" 17051,6,385,2017-03-24 15:43:33,http://fishermarvin.com/edison,,88.127.53.188,"{""location"": ""LV"", ""is_mobile"": true}" 17052,6,385,2017-04-13 00:10:40,http://abernathyschroeder.com/myra,,45.210.135.144,"{""location"": ""GI"", ""is_mobile"": true}" 17053,6,385,2017-05-22 16:05:49,http://little.com/sasha,,30.33.82.195,"{""location"": ""ET"", ""is_mobile"": false}" 17054,6,385,2017-05-04 08:33:19,http://marks.io/maddison,,54.231.62.137,"{""location"": ""GI"", ""is_mobile"": true}" 17055,6,385,2017-05-03 18:31:18,http://prosaccohegmann.org/alejandrin,,244.139.245.7,"{""location"": ""WF"", ""is_mobile"": false}" 17056,6,385,2017-05-30 17:39:34,http://pollich.net/quentin.cummerata,,149.212.104.235,"{""location"": ""PE"", ""is_mobile"": true}" 17057,6,385,2017-06-05 21:31:11,http://cremin.biz/juvenal,,48.123.125.55,"{""location"": ""ST"", ""is_mobile"": true}" 17058,6,385,2017-06-10 22:18:26,http://kihnstiedemann.biz/leopold,,254.25.181.204,"{""location"": ""TT"", ""is_mobile"": true}" 17059,6,385,2017-03-17 23:25:04,http://kirlinjones.io/mariano,,121.53.247.10,"{""location"": ""FM"", ""is_mobile"": true}" 17060,6,386,2017-05-20 01:16:34,http://faheymarquardt.co/adan,,50.148.16.132,"{""location"": ""RO"", ""is_mobile"": false}" 17061,6,386,2017-02-19 17:39:41,http://konopelski.org/kattie,,73.209.180.220,"{""location"": ""TR"", ""is_mobile"": true}" 17062,6,386,2017-02-26 11:24:19,http://farrell.com/edmond.bailey,,208.7.175.140,"{""location"": ""NA"", ""is_mobile"": true}" 17063,6,386,2016-12-29 12:05:46,http://wisoky.info/bret,,186.41.141.41,"{""location"": ""QA"", ""is_mobile"": false}" 17064,6,386,2017-05-15 12:22:17,http://sengerkaulke.net/augustine,,41.124.204.49,"{""location"": ""TF"", ""is_mobile"": false}" 17065,6,386,2016-12-26 00:28:14,http://hackettlueilwitz.io/brody_baumbach,,114.80.209.249,"{""location"": ""FM"", ""is_mobile"": true}" 17066,6,386,2017-05-13 21:45:48,http://gottlieb.io/ellsworth_schulist,,158.161.36.165,"{""location"": ""NI"", ""is_mobile"": true}" 17067,6,386,2017-05-06 08:00:36,http://rippinmiller.co/marianne.harber,,132.149.70.105,"{""location"": ""MD"", ""is_mobile"": false}" 17068,6,386,2017-01-10 01:26:41,http://kub.info/lorine.hyatt,,171.126.211.81,"{""location"": ""BL"", ""is_mobile"": true}" 17069,6,386,2017-04-05 14:32:23,http://blickklocko.org/roma.hirthe,,248.212.236.179,"{""location"": ""KN"", ""is_mobile"": false}" 17070,6,386,2017-02-03 23:30:27,http://boyle.name/rogelio_bode,,108.70.180.228,"{""location"": ""SL"", ""is_mobile"": false}" 17074,6,386,2016-12-17 01:00:45,http://schroedersanford.name/kasey.nitzsche,,6.7.30.233,"{""location"": ""AI"", ""is_mobile"": false}" 17075,6,386,2017-04-05 08:41:26,http://tromp.info/deon_nienow,,139.173.3.230,"{""location"": ""NA"", ""is_mobile"": true}" 17076,6,386,2017-02-27 11:37:57,http://kemmer.net/izaiah,,94.141.96.228,"{""location"": ""CW"", ""is_mobile"": true}" 17077,6,386,2017-01-13 16:44:33,http://shanahannicolas.name/devon.grant,,218.12.196.128,"{""location"": ""GG"", ""is_mobile"": true}" 17078,6,386,2017-06-01 02:53:23,http://denesiknolan.org/gunner.swift,,148.141.209.84,"{""location"": ""TR"", ""is_mobile"": true}" 17079,6,386,2017-01-21 09:27:22,http://oreilly.org/ruthe,,236.248.15.206,"{""location"": ""TD"", ""is_mobile"": false}" 17080,6,386,2017-03-25 21:47:25,http://effertzherman.com/everardo,,122.112.201.98,"{""location"": ""RU"", ""is_mobile"": false}" 17081,6,386,2017-04-03 03:19:43,http://kerlukegislason.net/reilly,,102.140.215.52,"{""location"": ""LY"", ""is_mobile"": true}" 17082,6,386,2016-12-19 01:58:42,http://turcotte.com/troy_hintz,,230.75.42.18,"{""location"": ""MT"", ""is_mobile"": false}" 17083,6,386,2017-04-14 12:48:32,http://larson.info/jared,,177.214.16.189,"{""location"": ""NL"", ""is_mobile"": true}" 17084,6,386,2017-04-17 05:27:36,http://medhurst.name/gage.keebler,,151.12.81.207,"{""location"": ""NF"", ""is_mobile"": true}" 17085,6,386,2017-05-06 00:56:45,http://fahey.name/rodrick,,36.94.140.169,"{""location"": ""MP"", ""is_mobile"": false}" 17086,6,386,2017-04-26 05:35:33,http://connelly.name/wilhelmine,,80.138.102.19,"{""location"": ""KW"", ""is_mobile"": true}" 17087,6,386,2017-03-08 02:27:31,http://lesch.co/isabelle_nader,,72.212.227.151,"{""location"": ""SS"", ""is_mobile"": false}" 17088,6,386,2017-01-29 19:17:38,http://schmidt.name/david,,16.195.41.250,"{""location"": ""CA"", ""is_mobile"": false}" 17089,6,386,2017-05-01 00:30:21,http://rolfson.com/jerel.cruickshank,,165.93.8.12,"{""location"": ""MM"", ""is_mobile"": false}" 17090,6,386,2017-03-15 04:18:55,http://hodkiewiczgleason.com/cathy.eichmann,,200.106.190.35,"{""location"": ""IL"", ""is_mobile"": false}" 17091,6,386,2017-01-05 09:00:54,http://hane.org/bobby.rath,,218.26.147.62,"{""location"": ""BD"", ""is_mobile"": true}" 17092,6,386,2017-03-27 17:45:09,http://gottliebheathcote.net/ansley.ortiz,,93.218.30.183,"{""location"": ""ZW"", ""is_mobile"": false}" 17093,6,386,2017-05-10 17:47:36,http://jacobs.biz/blanca,,145.159.54.246,"{""location"": ""IR"", ""is_mobile"": false}" 17094,6,386,2017-03-26 10:29:00,http://considine.biz/cory,,38.203.186.191,"{""location"": ""MG"", ""is_mobile"": false}" 17095,6,386,2017-01-26 18:47:39,http://funk.co/shyann,,109.35.170.241,"{""location"": ""BR"", ""is_mobile"": false}" 17096,6,386,2017-04-08 14:00:18,http://vonruedengulgowski.info/kelly.conn,,144.17.247.113,"{""location"": ""TG"", ""is_mobile"": true}" 4222,2,92,2017-01-13 01:55:28,http://casperblick.org/hertha.altenwerth,,114.26.160.32,"{""location"": ""ZW"", ""is_mobile"": false}" 4223,2,92,2017-04-23 09:35:42,http://mcdermott.biz/newton.osinski,,217.9.34.197,"{""location"": ""DO"", ""is_mobile"": true}" 4224,2,92,2017-01-23 17:47:15,http://rosenbaumlangosh.co/rachel.smitham,,16.120.212.136,"{""location"": ""BE"", ""is_mobile"": false}" 4225,2,93,2017-05-01 21:26:51,http://bosco.name/mikel,,206.108.52.126,"{""location"": ""UA"", ""is_mobile"": true}" 4226,2,93,2017-02-03 08:23:56,http://koepp.io/marcelle,,98.128.220.245,"{""location"": ""TT"", ""is_mobile"": true}" 4227,2,93,2017-06-01 05:29:02,http://ankundingkerluke.com/danika.jerde,,211.94.131.215,"{""location"": ""KN"", ""is_mobile"": false}" 4228,2,93,2017-04-13 11:07:26,http://friesen.co/filomena,,231.147.40.129,"{""location"": ""MA"", ""is_mobile"": false}" 4229,2,93,2017-02-05 20:43:52,http://stehrhamill.org/favian.kulas,,70.79.35.203,"{""location"": ""BD"", ""is_mobile"": false}" 4230,2,93,2017-04-10 13:36:48,http://mrazhansen.io/percy,,170.84.107.163,"{""location"": ""PL"", ""is_mobile"": false}" 4231,2,93,2017-01-09 15:29:19,http://wiegand.com/kenna.barton,,229.82.149.72,"{""location"": ""IO"", ""is_mobile"": false}" 4232,2,93,2017-02-10 04:53:30,http://effertz.name/layne,,200.163.123.109,"{""location"": ""MV"", ""is_mobile"": false}" 4233,2,93,2017-03-09 20:41:09,http://wiegand.io/evans.ohara,,177.142.16.21,"{""location"": ""GU"", ""is_mobile"": true}" 4234,2,93,2017-03-23 23:34:58,http://blanda.com/elliott_rohan,,26.126.97.199,"{""location"": ""SN"", ""is_mobile"": false}" 4235,2,93,2017-02-15 11:31:03,http://monahan.biz/cayla.stiedemann,,173.156.201.243,"{""location"": ""MM"", ""is_mobile"": false}" 4236,2,93,2017-02-21 02:07:19,http://gutkowskikoelpin.io/raoul.oconner,,54.156.189.213,"{""location"": ""AF"", ""is_mobile"": true}" 4237,2,93,2017-03-17 10:04:38,http://gibsonstokes.net/salma.halvorson,,168.50.33.115,"{""location"": ""MN"", ""is_mobile"": true}" 4238,2,93,2016-12-26 20:31:09,http://connellykshlerin.co/ludie,,44.79.36.39,"{""location"": ""TV"", ""is_mobile"": false}" 4239,2,93,2017-03-03 09:26:05,http://heel.io/chauncey.kilback,,182.22.213.225,"{""location"": ""GL"", ""is_mobile"": false}" 4240,2,93,2017-05-15 01:58:18,http://herzogbotsford.name/camilla_goodwin,,149.68.166.60,"{""location"": ""SH"", ""is_mobile"": false}" 4241,2,93,2017-02-26 19:09:17,http://legros.com/kailee,,164.186.120.175,"{""location"": ""NA"", ""is_mobile"": false}" 4242,2,93,2017-03-17 10:38:10,http://krajcikkuvalis.info/cierra_cummerata,,140.191.237.220,"{""location"": ""ET"", ""is_mobile"": true}" 4243,2,93,2017-01-12 03:05:10,http://yundt.org/lonzo,,54.144.137.4,"{""location"": ""BA"", ""is_mobile"": false}" 4244,2,93,2017-04-04 17:20:39,http://mrazryan.biz/roger_schroeder,,235.65.183.253,"{""location"": ""LB"", ""is_mobile"": false}" 4245,2,93,2017-04-30 13:40:41,http://schneiderziemann.io/amelia_walsh,,137.109.170.81,"{""location"": ""GH"", ""is_mobile"": true}" 4246,2,93,2017-02-02 09:22:42,http://altenwerth.net/tina.frami,,121.50.127.164,"{""location"": ""BI"", ""is_mobile"": false}" 4247,2,93,2017-06-02 22:21:29,http://bechtelar.net/quinn,,229.18.209.156,"{""location"": ""BI"", ""is_mobile"": true}" 4248,2,93,2017-03-13 08:57:39,http://pouros.org/jaron,,11.180.196.246,"{""location"": ""NA"", ""is_mobile"": false}" 4249,2,93,2016-12-18 21:30:48,http://mosciski.info/lane.towne,,14.113.90.40,"{""location"": ""UZ"", ""is_mobile"": false}" 4250,2,93,2017-03-13 10:39:09,http://padberg.net/boyd,,190.54.209.210,"{""location"": ""SO"", ""is_mobile"": true}" 4251,2,93,2017-06-12 18:41:52,http://kunde.com/stacy,,20.68.196.75,"{""location"": ""FJ"", ""is_mobile"": false}" 4252,2,93,2017-04-17 12:59:44,http://lubowitzkutch.org/fay_bahringer,,83.251.124.232,"{""location"": ""PN"", ""is_mobile"": true}" 4253,2,93,2016-12-20 03:59:22,http://zulauf.biz/terrence.hartmann,,25.158.5.108,"{""location"": ""OM"", ""is_mobile"": true}" 4254,2,93,2017-01-05 16:37:37,http://johns.co/dakota_jaskolski,,62.150.91.156,"{""location"": ""GF"", ""is_mobile"": true}" 4255,2,93,2017-05-23 20:04:40,http://murphyharber.com/tyra,,151.226.82.169,"{""location"": ""LI"", ""is_mobile"": false}" 4256,2,93,2016-12-29 03:37:30,http://mckenzieprice.biz/quentin.ferry,,16.91.251.59,"{""location"": ""FR"", ""is_mobile"": true}" 4257,2,93,2017-06-04 16:21:48,http://skiles.com/mckayla_gutkowski,,192.34.16.118,"{""location"": ""SX"", ""is_mobile"": true}" 4258,2,93,2017-01-03 14:55:17,http://cummings.com/bud,,213.186.189.228,"{""location"": ""AD"", ""is_mobile"": true}" 4259,2,93,2017-03-06 07:20:36,http://kunde.name/vivianne,,125.186.220.233,"{""location"": ""MM"", ""is_mobile"": true}" 4260,2,93,2017-05-01 17:14:28,http://leuschke.name/hal.kuvalis,,130.211.140.14,"{""location"": ""BO"", ""is_mobile"": true}" 4261,2,93,2017-06-01 02:53:17,http://hammes.io/mohammad_mueller,,49.14.158.101,"{""location"": ""AS"", ""is_mobile"": true}" 4262,2,93,2017-03-03 00:12:15,http://pollich.biz/cruz.okon,,149.172.136.64,"{""location"": ""GT"", ""is_mobile"": true}" 4263,2,93,2017-01-26 14:29:03,http://cummings.com/ralph.lesch,,59.252.194.201,"{""location"": ""VC"", ""is_mobile"": false}" 4264,2,93,2017-04-07 10:39:52,http://steuber.biz/columbus,,62.221.197.101,"{""location"": ""CH"", ""is_mobile"": false}" 4265,2,93,2017-01-26 01:04:54,http://zemlak.co/fritz,,47.129.152.32,"{""location"": ""CX"", ""is_mobile"": false}" 4266,2,93,2017-03-21 02:16:07,http://nienowkub.info/lamont.bins,,129.139.57.128,"{""location"": ""IN"", ""is_mobile"": true}" 4267,2,93,2017-04-09 15:25:38,http://hand.info/vincent.monahan,,70.181.167.176,"{""location"": ""LC"", ""is_mobile"": false}" 4268,2,93,2017-01-08 23:52:58,http://heaney.co/tevin_wolff,,72.72.191.118,"{""location"": ""HR"", ""is_mobile"": true}" 4269,2,93,2016-12-26 15:40:48,http://fisher.org/clyde.lowe,,179.105.210.164,"{""location"": ""BB"", ""is_mobile"": true}" 4270,2,93,2017-01-25 22:40:00,http://langosh.info/cory,,252.82.50.176,"{""location"": ""TH"", ""is_mobile"": false}" 4271,2,93,2016-12-30 11:17:55,http://koch.biz/wendell,,170.172.239.109,"{""location"": ""CA"", ""is_mobile"": true}" 4272,2,93,2017-01-20 14:24:15,http://hartmann.co/felix,,175.45.231.121,"{""location"": ""FJ"", ""is_mobile"": true}" 4273,2,93,2017-04-20 22:43:05,http://turcotte.biz/sean.abernathy,,109.116.69.191,"{""location"": ""HR"", ""is_mobile"": false}" 4274,2,93,2017-04-29 11:38:05,http://hand.name/marley,,142.183.176.156,"{""location"": ""CK"", ""is_mobile"": false}" 4275,2,94,2017-03-30 21:46:41,http://witting.co/tevin,,23.102.124.170,"{""location"": ""YT"", ""is_mobile"": false}" 4276,2,94,2017-01-27 01:42:28,http://ernser.info/winifred,,108.162.124.98,"{""location"": ""EG"", ""is_mobile"": true}" 4277,2,94,2017-06-12 10:20:53,http://champlin.com/gabrielle,,57.134.198.183,"{""location"": ""KG"", ""is_mobile"": false}" 11162,4,249,2017-03-13 04:13:20,http://boehmziemann.co/ephraim.stark,,49.23.163.128,"{""location"": ""FI"", ""is_mobile"": false}" 11163,4,249,2017-01-14 05:38:40,http://considine.com/camren,,203.197.189.76,"{""location"": ""VN"", ""is_mobile"": false}" 11164,4,249,2016-12-26 09:07:48,http://schimmel.org/lonnie.ruecker,,225.136.209.13,"{""location"": ""KM"", ""is_mobile"": true}" 11165,4,249,2017-03-31 23:32:44,http://hackett.co/louvenia,,205.126.31.35,"{""location"": ""RS"", ""is_mobile"": true}" 11166,4,249,2017-02-14 16:18:20,http://cristshanahan.net/alberto.marquardt,,9.198.105.224,"{""location"": ""SH"", ""is_mobile"": true}" 11167,4,249,2017-02-15 05:15:39,http://roob.co/ernestina,,3.74.31.102,"{""location"": ""AE"", ""is_mobile"": false}" 11168,4,250,2017-02-28 15:47:29,http://reichert.com/kiarra.balistreri,,10.80.209.212,"{""location"": ""SN"", ""is_mobile"": false}" 11169,4,250,2017-05-26 08:46:24,http://lind.co/thea_stroman,,67.185.163.155,"{""location"": ""MV"", ""is_mobile"": false}" 11170,4,250,2017-04-13 21:43:12,http://lehner.biz/kyle.osinski,,109.11.177.128,"{""location"": ""TT"", ""is_mobile"": false}" 11171,4,250,2017-02-22 15:42:30,http://white.name/elwyn,,110.99.131.206,"{""location"": ""CY"", ""is_mobile"": false}" 11172,4,250,2017-05-20 22:39:37,http://sipes.org/jerad_metz,,189.49.116.194,"{""location"": ""TC"", ""is_mobile"": false}" 11173,4,250,2017-01-26 14:16:05,http://greenwuckert.net/lambert_sawayn,,197.218.21.145,"{""location"": ""IT"", ""is_mobile"": true}" 11174,4,250,2016-12-14 19:12:52,http://rolfsonhermann.biz/ima_wolf,,28.94.79.188,"{""location"": ""PH"", ""is_mobile"": true}" 11175,4,250,2017-01-03 14:35:37,http://hahn.name/garth,,129.249.214.122,"{""location"": ""RO"", ""is_mobile"": true}" 11176,4,250,2016-12-22 06:44:41,http://larkin.name/kelly,,238.74.160.49,"{""location"": ""GN"", ""is_mobile"": false}" 11177,4,250,2017-06-12 12:04:41,http://streich.name/hettie,,192.7.210.171,"{""location"": ""QA"", ""is_mobile"": true}" 11178,4,250,2017-04-15 11:48:58,http://kuvalis.com/emilia,,86.167.85.42,"{""location"": ""GG"", ""is_mobile"": true}" 11179,4,250,2017-05-09 00:21:48,http://littelstrosin.info/dallin,,14.90.12.160,"{""location"": ""GF"", ""is_mobile"": false}" 11180,4,250,2016-12-24 16:34:49,http://dibbert.biz/shayna,,13.157.127.132,"{""location"": ""US"", ""is_mobile"": true}" 11181,4,250,2017-04-29 05:03:11,http://weber.com/karen_eichmann,,108.183.70.5,"{""location"": ""SM"", ""is_mobile"": false}" 11182,4,250,2017-02-23 05:55:50,http://pacocha.net/brad.abernathy,,38.79.96.245,"{""location"": ""GE"", ""is_mobile"": true}" 11183,4,250,2017-04-27 15:10:39,http://padberg.name/willard.wiegand,,195.86.57.246,"{""location"": ""MM"", ""is_mobile"": false}" 11184,4,250,2017-03-11 16:56:09,http://thompson.biz/rahul,,22.189.143.26,"{""location"": ""CX"", ""is_mobile"": false}" 11185,4,250,2017-05-05 06:16:02,http://auercronin.net/jonathan_rowe,,127.25.168.148,"{""location"": ""VI"", ""is_mobile"": true}" 11186,4,250,2017-04-10 06:48:40,http://barton.name/helga.cronin,,81.45.185.144,"{""location"": ""TC"", ""is_mobile"": true}" 11187,4,250,2017-05-09 09:49:11,http://rogahn.org/dominic,,17.201.201.18,"{""location"": ""LI"", ""is_mobile"": false}" 11188,4,250,2017-04-05 11:36:11,http://ledner.io/mertie,,139.71.73.226,"{""location"": ""VU"", ""is_mobile"": true}" 11189,4,250,2017-02-01 13:29:09,http://walter.com/kristian,,50.193.218.238,"{""location"": ""DO"", ""is_mobile"": false}" 11190,4,250,2017-05-21 07:54:29,http://kiehn.io/kevon_kuvalis,,206.172.228.139,"{""location"": ""NU"", ""is_mobile"": true}" 11191,4,250,2017-04-06 06:26:12,http://buckridge.net/blanca.parisian,,146.175.180.8,"{""location"": ""AD"", ""is_mobile"": false}" 11192,4,250,2017-05-14 01:48:14,http://cronin.io/donavon,,79.169.46.43,"{""location"": ""SK"", ""is_mobile"": false}" 11193,4,250,2017-06-06 23:46:49,http://satterfield.co/ashlynn_crona,,50.170.121.182,"{""location"": ""ME"", ""is_mobile"": false}" 11194,4,251,2017-04-30 10:46:08,http://conndavis.com/lane.gulgowski,,85.19.53.151,"{""location"": ""TJ"", ""is_mobile"": false}" 11195,4,251,2017-03-02 02:31:53,http://oberbrunner.org/rogelio,,130.175.179.156,"{""location"": ""LY"", ""is_mobile"": true}" 11196,4,251,2017-03-02 19:14:16,http://heidenreich.biz/jaclyn.rogahn,,60.213.78.68,"{""location"": ""GL"", ""is_mobile"": true}" 11197,4,251,2017-02-21 18:59:29,http://carroll.net/kade,,16.115.121.242,"{""location"": ""SC"", ""is_mobile"": false}" 11198,4,251,2017-04-01 20:46:51,http://wiegandschaden.net/cole,,42.154.177.239,"{""location"": ""EE"", ""is_mobile"": false}" 11199,4,251,2017-01-08 05:49:16,http://daviscormier.io/mike.hackett,,150.158.192.20,"{""location"": ""EE"", ""is_mobile"": true}" 11200,4,251,2017-04-13 19:46:52,http://gorczanykrajcik.co/dolores,,223.32.252.11,"{""location"": ""BF"", ""is_mobile"": true}" 11201,4,251,2017-03-21 19:02:45,http://jerde.co/montana,,237.127.201.63,"{""location"": ""PT"", ""is_mobile"": false}" 11202,4,251,2017-04-12 07:02:02,http://heidenreich.net/madonna,,72.85.48.14,"{""location"": ""LT"", ""is_mobile"": false}" 11203,4,251,2017-02-22 01:02:47,http://heathcotebode.com/mckenzie_olson,,92.72.100.136,"{""location"": ""BO"", ""is_mobile"": false}" 11204,4,251,2016-12-22 14:05:22,http://emard.com/joesph,,76.148.44.190,"{""location"": ""RE"", ""is_mobile"": true}" 11205,4,251,2017-05-23 04:51:37,http://pfeffer.io/shannon_price,,95.39.72.21,"{""location"": ""BZ"", ""is_mobile"": false}" 11206,4,251,2017-01-26 00:25:42,http://hand.info/cara,,232.199.62.15,"{""location"": ""AQ"", ""is_mobile"": false}" 11207,4,251,2017-04-04 11:23:03,http://ritchieabernathy.com/terrell,,192.254.159.117,"{""location"": ""SL"", ""is_mobile"": true}" 11208,4,251,2017-01-11 06:11:34,http://gerlach.biz/dorothea_white,,91.212.163.18,"{""location"": ""VC"", ""is_mobile"": true}" 11209,4,251,2017-03-12 14:04:02,http://mayert.io/sandra,,31.107.120.98,"{""location"": ""IN"", ""is_mobile"": true}" 11210,4,251,2017-01-19 09:54:11,http://reichel.net/lenora_macejkovic,,206.173.231.215,"{""location"": ""SL"", ""is_mobile"": true}" 11211,4,251,2017-01-10 12:46:38,http://champlin.name/jameson,,146.32.51.21,"{""location"": ""SL"", ""is_mobile"": false}" 11212,4,251,2017-05-02 04:19:33,http://murphy.net/kyle,,167.145.229.233,"{""location"": ""GS"", ""is_mobile"": true}" 11213,4,251,2017-03-07 02:06:42,http://connellystamm.io/litzy.rau,,63.156.167.124,"{""location"": ""SJ"", ""is_mobile"": true}" 11214,4,251,2017-04-22 12:54:02,http://kreiger.net/isac,,68.238.79.20,"{""location"": ""MQ"", ""is_mobile"": false}" 11215,4,251,2017-02-08 19:23:28,http://andersonwilderman.io/fatima_rutherford,,245.221.163.109,"{""location"": ""BH"", ""is_mobile"": false}" 11216,4,251,2017-02-19 04:03:27,http://townestreich.io/brandi_veum,,197.160.168.47,"{""location"": ""BA"", ""is_mobile"": true}" 8170,3,180,2017-04-18 18:23:13,http://mcclure.biz/palma,0.6689673944,152.145.58.84,"{""location"": ""TD"", ""is_mobile"": false}" 8171,3,181,2017-06-01 19:20:18,http://corkeryzboncak.co/jarvis,0.6209441698,248.244.18.49,"{""location"": ""SM"", ""is_mobile"": false}" 8172,3,181,2017-02-10 04:09:00,http://adams.info/osborne_collins,0.0612289705,22.168.101.31,"{""location"": ""ML"", ""is_mobile"": false}" 8173,3,181,2017-02-26 06:09:42,http://kelereichmann.info/alexane,0.0366695163,67.127.128.240,"{""location"": ""CM"", ""is_mobile"": false}" 8174,3,181,2017-01-17 13:09:36,http://schaefer.name/keenan.wuckert,0.2610657100,111.142.223.84,"{""location"": ""CZ"", ""is_mobile"": true}" 8175,3,181,2016-12-21 11:01:41,http://runolfsdottirmorar.com/oral.rice,0.5323794326,237.17.53.77,"{""location"": ""SK"", ""is_mobile"": true}" 8176,3,181,2017-03-21 00:08:55,http://turner.com/marie,0.0279746137,244.116.9.235,"{""location"": ""TH"", ""is_mobile"": false}" 8177,3,181,2017-01-05 15:08:34,http://jenkinmith.com/bridgette_mccullough,0.1113406109,120.153.176.227,"{""location"": ""BY"", ""is_mobile"": false}" 8178,3,181,2017-05-16 20:51:23,http://skiles.info/trea,0.1585320071,147.25.79.243,"{""location"": ""TF"", ""is_mobile"": true}" 8179,3,181,2017-02-04 03:49:29,http://wymankreiger.io/leif,0.5343390720,5.37.5.83,"{""location"": ""MG"", ""is_mobile"": false}" 8180,3,181,2017-01-19 20:58:26,http://nolanemard.co/jefferey.emmerich,0.0123930880,96.155.150.154,"{""location"": ""GF"", ""is_mobile"": false}" 8181,3,181,2017-02-28 20:21:08,http://kunze.io/rodger_vonrueden,0.2659361171,109.35.148.192,"{""location"": ""MC"", ""is_mobile"": false}" 8182,3,181,2017-05-07 01:56:31,http://mayer.org/hunter,0.5244500982,230.245.7.38,"{""location"": ""SE"", ""is_mobile"": true}" 8183,3,181,2017-05-21 02:28:43,http://sipes.co/bruce,0.9347674939,129.244.52.211,"{""location"": ""MG"", ""is_mobile"": false}" 8184,3,181,2017-04-19 04:34:05,http://bins.io/kara_rippin,0.4649037542,97.90.32.78,"{""location"": ""BT"", ""is_mobile"": false}" 8185,3,181,2017-01-17 08:58:56,http://will.io/kurt_haag,0.5340803804,137.156.34.35,"{""location"": ""IE"", ""is_mobile"": false}" 8186,3,181,2017-04-23 13:18:45,http://maggio.info/marta,0.6596025501,133.192.118.112,"{""location"": ""MZ"", ""is_mobile"": true}" 8187,3,181,2017-06-12 11:21:18,http://robel.biz/lazaro,0.2216656035,118.48.81.179,"{""location"": ""GI"", ""is_mobile"": false}" 8188,3,181,2017-04-24 19:48:31,http://strosin.name/winfield,0.5630480784,101.151.30.144,"{""location"": ""KE"", ""is_mobile"": false}" 8189,3,181,2016-12-16 03:30:10,http://lebsack.net/camden_heathcote,0.9626469115,135.132.125.60,"{""location"": ""JM"", ""is_mobile"": false}" 8190,3,181,2017-02-24 10:22:46,http://harvey.biz/julius_kshlerin,0.6077187699,203.35.98.77,"{""location"": ""SI"", ""is_mobile"": true}" 8191,3,181,2017-01-08 13:19:08,http://lebsack.name/yesenia.weinat,0.1630130272,77.142.189.84,"{""location"": ""WF"", ""is_mobile"": true}" 8192,3,181,2017-05-21 14:51:51,http://gulgowskiweinat.io/quinten_leannon,0.7999305501,84.165.247.165,"{""location"": ""BA"", ""is_mobile"": false}" 8193,3,182,2017-01-22 05:39:43,http://goyette.name/rosalyn_hahn,0.8700501962,211.72.21.4,"{""location"": ""AL"", ""is_mobile"": true}" 8194,3,182,2017-01-16 07:46:57,http://hintz.name/oleta,0.0477622579,202.202.9.117,"{""location"": ""ER"", ""is_mobile"": false}" 8195,3,182,2017-05-12 12:13:30,http://rippinmcglynn.org/tyrell,0.7145243781,99.50.64.163,"{""location"": ""BW"", ""is_mobile"": true}" 8196,3,182,2017-02-22 16:32:22,http://hartmann.net/adolphus,0.6328294647,175.96.237.24,"{""location"": ""WF"", ""is_mobile"": true}" 8197,3,182,2017-03-18 07:36:15,http://keler.name/deron,0.3714287255,32.74.127.21,"{""location"": ""CD"", ""is_mobile"": false}" 8198,3,182,2017-01-04 04:24:29,http://cole.biz/freddie,0.4313237685,19.227.206.151,"{""location"": ""CD"", ""is_mobile"": true}" 8199,3,182,2017-06-10 15:20:35,http://murphy.biz/thalia,0.1159187925,53.21.199.216,"{""location"": ""NL"", ""is_mobile"": true}" 8200,3,182,2017-02-20 21:02:40,http://koeppturner.co/bobbie,0.3350078081,177.20.71.13,"{""location"": ""MQ"", ""is_mobile"": true}" 8201,3,182,2017-01-04 22:20:59,http://adamsokuneva.co/mara,0.9923592883,232.136.43.50,"{""location"": ""TR"", ""is_mobile"": false}" 8202,3,182,2017-02-04 06:32:17,http://gaylord.info/jordon.robel,0.7060708840,62.178.196.160,"{""location"": ""TN"", ""is_mobile"": true}" 8203,3,182,2017-02-10 04:35:22,http://muller.io/rosetta,0.7862678147,41.167.185.5,"{""location"": ""GN"", ""is_mobile"": false}" 8204,3,182,2017-03-14 09:05:04,http://rogahn.com/peggie,0.4203163104,114.46.231.125,"{""location"": ""PN"", ""is_mobile"": false}" 8205,3,182,2017-03-08 18:31:44,http://goodwin.biz/warren_bosco,0.8694166178,163.206.171.216,"{""location"": ""NC"", ""is_mobile"": false}" 8206,3,182,2017-02-21 09:52:51,http://murazik.name/mervin.damore,0.4686481876,168.246.63.18,"{""location"": ""FR"", ""is_mobile"": false}" 8207,3,182,2017-04-19 01:36:37,http://hegmann.net/romaine,0.8870173745,64.22.192.142,"{""location"": ""MG"", ""is_mobile"": false}" 8208,3,182,2017-05-17 21:40:24,http://marvin.name/domenic.okuneva,0.5710023227,178.3.44.132,"{""location"": ""TF"", ""is_mobile"": true}" 8209,3,182,2017-05-17 17:55:44,http://millsmorar.biz/archibald,0.2615607867,24.36.226.8,"{""location"": ""JO"", ""is_mobile"": true}" 8210,3,182,2017-05-02 09:36:34,http://creminhuel.io/douglas_volkman,0.3379744666,21.141.141.76,"{""location"": ""TD"", ""is_mobile"": false}" 8211,3,182,2017-06-02 08:09:21,http://kingtromp.co/lincoln,0.8997504122,238.189.27.82,"{""location"": ""VG"", ""is_mobile"": true}" 8212,3,182,2017-04-05 20:22:25,http://cremin.co/daren,0.9310972835,109.230.210.34,"{""location"": ""IT"", ""is_mobile"": true}" 8213,3,182,2016-12-15 12:54:20,http://schillerhintz.net/carlotta,0.8576071014,115.124.221.55,"{""location"": ""LV"", ""is_mobile"": true}" 8214,3,182,2016-12-23 22:55:15,http://schneiderrowe.com/derick.davis,0.9336066610,91.59.129.8,"{""location"": ""NL"", ""is_mobile"": false}" 8215,3,182,2016-12-27 06:32:59,http://wolffschuster.biz/augustine,0.6031123774,232.9.210.116,"{""location"": ""PM"", ""is_mobile"": true}" 8216,3,182,2017-06-07 20:36:52,http://jones.com/reanna,0.8723301799,53.58.137.48,"{""location"": ""NA"", ""is_mobile"": false}" 8217,3,182,2017-02-23 02:27:37,http://corwinwisozk.org/marlee_goldner,0.3093142382,6.137.225.6,"{""location"": ""BS"", ""is_mobile"": true}" 8218,3,182,2017-04-09 10:27:15,http://prosacco.org/leonel_zemlak,0.2723396481,48.140.241.172,"{""location"": ""SG"", ""is_mobile"": true}" 8219,3,182,2017-03-09 10:03:45,http://gibsonheaney.name/keanu,0.6187223410,53.55.186.123,"{""location"": ""NO"", ""is_mobile"": true}" 8220,3,182,2017-05-03 04:00:10,http://willziemann.biz/alyson.okeefe,0.9344558864,161.138.57.117,"{""location"": ""MW"", ""is_mobile"": false}" 8221,3,182,2017-01-12 18:49:18,http://predovic.co/neva,0.5816248388,208.49.8.5,"{""location"": ""QA"", ""is_mobile"": true}" 14113,5,316,2017-03-16 22:40:53,http://simonis.net/immanuel_spinka,,208.236.68.210,"{""location"": ""PG"", ""is_mobile"": false}" 14114,5,316,2017-02-09 09:59:30,http://bailey.info/alene,,134.207.17.104,"{""location"": ""VI"", ""is_mobile"": false}" 14115,5,316,2017-01-26 20:45:40,http://hodkiewicz.io/dolores,,9.214.74.71,"{""location"": ""NE"", ""is_mobile"": false}" 14116,5,316,2017-05-24 12:22:10,http://crooks.io/dylan,,250.51.119.63,"{""location"": ""LY"", ""is_mobile"": false}" 14117,5,316,2017-01-15 05:13:21,http://grimes.io/laron_leffler,,32.250.48.95,"{""location"": ""BY"", ""is_mobile"": true}" 14118,5,316,2017-02-06 07:21:39,http://farrellschuppe.io/kobe_konopelski,,133.58.189.92,"{""location"": ""TN"", ""is_mobile"": true}" 14119,5,316,2017-01-08 13:55:43,http://ko.info/walter,,212.166.19.135,"{""location"": ""MZ"", ""is_mobile"": true}" 14120,5,316,2017-03-29 14:57:51,http://hodkiewiczpfannerstill.io/rigoberto,,75.54.196.188,"{""location"": ""WS"", ""is_mobile"": false}" 14121,5,316,2017-01-24 09:47:29,http://hayeslind.org/felicia.pacocha,,117.105.28.138,"{""location"": ""SR"", ""is_mobile"": false}" 14122,5,316,2017-01-04 16:38:03,http://handgraham.info/lincoln,,81.215.76.125,"{""location"": ""GB"", ""is_mobile"": false}" 14123,5,316,2017-04-26 08:48:02,http://boyle.info/sophie.miller,,39.180.74.62,"{""location"": ""RO"", ""is_mobile"": false}" 14124,5,316,2017-05-03 16:17:14,http://daughertybeier.com/roxane,,10.16.46.122,"{""location"": ""MM"", ""is_mobile"": true}" 14125,5,317,2017-02-03 09:47:40,http://pfefferwilliamson.com/lloyd.howe,,44.12.123.191,"{""location"": ""SI"", ""is_mobile"": false}" 14126,5,317,2016-12-23 09:51:35,http://douglas.biz/meaghan.pagac,,73.40.107.124,"{""location"": ""MW"", ""is_mobile"": true}" 14127,5,317,2017-05-17 21:03:44,http://lehner.net/camden.lang,,167.135.111.144,"{""location"": ""SN"", ""is_mobile"": true}" 14128,5,317,2017-02-11 02:36:44,http://hintz.co/donavon,,5.29.64.238,"{""location"": ""TZ"", ""is_mobile"": true}" 14129,5,317,2016-12-13 13:13:08,http://haagfunk.org/aurelio.lang,,207.9.161.152,"{""location"": ""LR"", ""is_mobile"": true}" 14130,5,317,2017-04-22 22:40:24,http://kuvalis.io/gunnar.renner,,145.60.4.249,"{""location"": ""OM"", ""is_mobile"": false}" 14131,5,317,2017-02-24 10:10:53,http://kulasmante.io/carter_ruecker,,182.39.69.14,"{""location"": ""AG"", ""is_mobile"": false}" 14132,5,317,2017-01-14 11:34:22,http://schmidt.info/myrna,,83.97.127.45,"{""location"": ""KY"", ""is_mobile"": false}" 14133,5,317,2017-02-03 11:41:21,http://wintheiserhintz.com/cheyanne.gleason,,204.199.164.38,"{""location"": ""BV"", ""is_mobile"": false}" 14134,5,317,2017-02-04 19:04:19,http://beckeryost.org/louisa,,20.44.20.205,"{""location"": ""RW"", ""is_mobile"": false}" 14135,5,317,2017-02-23 17:25:02,http://greenholthansen.biz/greg_carter,,135.241.23.242,"{""location"": ""CO"", ""is_mobile"": true}" 14136,5,317,2017-03-05 06:00:31,http://raynorturcotte.co/damien,,164.16.176.73,"{""location"": ""AX"", ""is_mobile"": false}" 14137,5,317,2017-05-19 02:51:36,http://abshire.io/alivia,,235.119.240.171,"{""location"": ""LU"", ""is_mobile"": false}" 14138,5,317,2017-03-03 03:49:46,http://parisian.biz/floy,,116.173.103.71,"{""location"": ""AX"", ""is_mobile"": true}" 14139,5,317,2017-01-19 04:12:23,http://stoltenberg.com/ethelyn,,4.148.43.201,"{""location"": ""EH"", ""is_mobile"": true}" 14140,5,317,2016-12-27 10:27:53,http://kundeoberbrunner.org/katlyn,,158.211.84.60,"{""location"": ""IQ"", ""is_mobile"": true}" 14141,5,317,2017-02-07 19:40:28,http://carrollcremin.org/aliyah,,237.106.72.167,"{""location"": ""VI"", ""is_mobile"": true}" 14142,5,317,2017-03-27 00:49:43,http://hermann.info/malika,,97.226.188.198,"{""location"": ""TW"", ""is_mobile"": true}" 14143,5,317,2017-03-31 02:13:01,http://hicklekeebler.org/madelynn.tromp,,163.152.26.220,"{""location"": ""AM"", ""is_mobile"": true}" 14144,5,317,2017-05-29 15:52:36,http://handjacobson.name/arianna.walsh,,194.243.123.33,"{""location"": ""TZ"", ""is_mobile"": false}" 14145,5,317,2017-06-05 12:19:00,http://jakubowski.info/johnnie,,79.97.54.67,"{""location"": ""ER"", ""is_mobile"": false}" 14146,5,317,2017-01-18 23:11:25,http://mayert.info/morgan_johns,,185.221.21.31,"{""location"": ""CM"", ""is_mobile"": true}" 14147,5,317,2017-05-08 06:48:24,http://gerlach.com/dahlia,,157.197.251.207,"{""location"": ""VC"", ""is_mobile"": true}" 14148,5,317,2017-02-14 02:37:07,http://pacochasatterfield.com/raymond.mills,,26.20.96.162,"{""location"": ""GI"", ""is_mobile"": true}" 14149,5,318,2017-01-15 19:23:13,http://bailey.biz/corine.streich,,52.38.161.120,"{""location"": ""DE"", ""is_mobile"": true}" 14150,5,318,2017-03-18 03:31:03,http://wehner.co/rosalia,,192.245.211.198,"{""location"": ""BL"", ""is_mobile"": false}" 14151,5,318,2017-01-11 21:27:17,http://hayespowlowski.com/thora,,51.237.144.179,"{""location"": ""LT"", ""is_mobile"": false}" 14152,5,318,2017-05-14 23:03:03,http://breitenberg.com/carley.berge,,204.4.158.143,"{""location"": ""BI"", ""is_mobile"": true}" 14153,5,318,2017-03-24 22:03:31,http://kutch.biz/odie,,81.50.250.30,"{""location"": ""DK"", ""is_mobile"": false}" 14154,5,318,2017-05-13 19:22:33,http://manteokuneva.info/elroy.lemke,,246.208.125.125,"{""location"": ""DJ"", ""is_mobile"": true}" 14155,5,318,2017-01-02 14:08:06,http://damoredavis.co/mike.hammes,,200.69.143.86,"{""location"": ""MO"", ""is_mobile"": true}" 14156,5,318,2017-01-25 20:43:19,http://weber.com/jaeden,,136.151.76.102,"{""location"": ""IE"", ""is_mobile"": false}" 14157,5,318,2016-12-13 16:49:47,http://rice.net/max.blick,,219.65.20.76,"{""location"": ""BG"", ""is_mobile"": true}" 14158,5,318,2017-03-16 04:28:55,http://leuschkemiller.net/jasper,,177.118.91.248,"{""location"": ""GA"", ""is_mobile"": false}" 14159,5,318,2017-04-26 15:38:26,http://mantelueilwitz.name/nella.powlowski,,71.96.82.115,"{""location"": ""BO"", ""is_mobile"": false}" 14160,5,318,2017-03-08 23:28:09,http://wilderman.info/mara,,230.17.191.75,"{""location"": ""FM"", ""is_mobile"": true}" 14161,5,318,2017-04-29 20:28:31,http://feeney.biz/diana,,156.40.227.64,"{""location"": ""VA"", ""is_mobile"": true}" 14162,5,318,2017-02-03 09:10:56,http://dooley.info/claudine,,103.79.6.118,"{""location"": ""KG"", ""is_mobile"": true}" 14163,5,318,2017-05-15 00:53:15,http://stehr.com/maya,,44.107.242.125,"{""location"": ""VA"", ""is_mobile"": true}" 14164,5,318,2017-03-05 10:18:40,http://denesik.io/travis.runte,,30.133.76.110,"{""location"": ""IR"", ""is_mobile"": false}" 14165,5,318,2017-01-30 00:42:25,http://goldnerbotsford.info/trycia_runolfsdottir,,48.157.106.86,"{""location"": ""GW"", ""is_mobile"": true}" 14166,5,318,2017-02-14 14:27:18,http://zboncak.info/johnpaul,,94.114.80.23,"{""location"": ""CW"", ""is_mobile"": true}" 14167,5,318,2017-03-02 06:28:51,http://wiza.co/deron.bahringer,,10.66.124.168,"{""location"": ""CY"", ""is_mobile"": false}" 17097,6,386,2017-03-02 14:17:29,http://satterfield.biz/jacinthe,,131.117.116.70,"{""location"": ""BG"", ""is_mobile"": false}" 17098,6,386,2017-01-28 20:45:27,http://kohler.info/keegan,,133.179.155.233,"{""location"": ""VE"", ""is_mobile"": true}" 17099,6,386,2016-12-16 08:56:04,http://aufderharklein.net/judy_keler,,196.115.28.154,"{""location"": ""AT"", ""is_mobile"": false}" 17100,6,386,2017-01-16 09:39:16,http://oconnell.info/patricia,,213.188.17.58,"{""location"": ""KM"", ""is_mobile"": false}" 17101,6,386,2017-02-01 05:34:02,http://kozeygreenfelder.co/sidney,,65.48.88.14,"{""location"": ""FJ"", ""is_mobile"": false}" 17102,6,386,2017-01-14 05:01:32,http://labadiewindler.name/lillie,,73.23.89.215,"{""location"": ""CC"", ""is_mobile"": false}" 17103,6,386,2016-12-24 12:30:54,http://gaylordrutherford.info/ashleigh_swift,,100.172.174.175,"{""location"": ""AG"", ""is_mobile"": true}" 17104,6,386,2017-03-21 01:05:47,http://lueilwitz.org/kailyn_damore,,81.151.184.9,"{""location"": ""FJ"", ""is_mobile"": true}" 17105,6,386,2017-02-11 15:04:59,http://emmerich.io/otis.lesch,,93.14.40.175,"{""location"": ""BE"", ""is_mobile"": false}" 17106,6,386,2017-05-21 15:56:49,http://heel.net/roscoe.block,,110.186.93.220,"{""location"": ""ZM"", ""is_mobile"": true}" 17107,6,386,2017-02-04 16:19:05,http://satterfieldvolkman.info/friedrich_ferry,,202.29.230.164,"{""location"": ""NR"", ""is_mobile"": false}" 17108,6,386,2017-03-19 17:17:49,http://reilly.io/lincoln.conroy,,247.253.169.225,"{""location"": ""MW"", ""is_mobile"": true}" 17109,6,386,2017-03-23 11:41:12,http://mohrhuel.com/jake,,14.170.107.29,"{""location"": ""BY"", ""is_mobile"": false}" 17110,6,386,2017-03-28 10:22:34,http://manteherman.org/miguel.will,,24.188.182.90,"{""location"": ""BS"", ""is_mobile"": false}" 17111,6,386,2017-03-19 03:39:18,http://hackett.co/claudine.koepp,,16.166.179.119,"{""location"": ""HK"", ""is_mobile"": false}" 17112,6,386,2017-01-18 19:40:22,http://baumbachjacobs.biz/houston.senger,,191.144.220.221,"{""location"": ""FK"", ""is_mobile"": true}" 17113,6,386,2017-06-07 00:43:42,http://oconnerhettinger.io/graham_durgan,,119.102.125.25,"{""location"": ""MC"", ""is_mobile"": true}" 17114,6,386,2017-01-07 12:48:39,http://herzog.co/marianna,,87.32.19.91,"{""location"": ""FI"", ""is_mobile"": true}" 17115,6,386,2017-01-09 15:24:46,http://gleason.net/leora_shanahan,,91.229.193.117,"{""location"": ""VI"", ""is_mobile"": true}" 17116,6,386,2017-05-12 02:45:45,http://millslittel.net/arjun.mante,,38.209.42.157,"{""location"": ""CZ"", ""is_mobile"": false}" 17117,6,386,2017-05-07 23:10:12,http://strosin.io/hazel,,100.14.220.43,"{""location"": ""UZ"", ""is_mobile"": true}" 17118,6,386,2017-01-17 03:31:36,http://eichmann.org/martine,,217.114.68.134,"{""location"": ""FR"", ""is_mobile"": true}" 17119,6,386,2017-02-21 14:10:50,http://dibbert.net/albertha.thompson,,232.141.168.85,"{""location"": ""UZ"", ""is_mobile"": false}" 17120,6,386,2016-12-14 18:55:54,http://terryokeefe.org/mozell,,164.145.207.150,"{""location"": ""TG"", ""is_mobile"": false}" 17121,6,386,2017-02-02 04:41:15,http://sauer.net/leone,,180.186.2.188,"{""location"": ""AR"", ""is_mobile"": true}" 17122,6,387,2017-04-21 07:03:29,http://gaylord.info/franz,,70.253.112.203,"{""location"": ""PA"", ""is_mobile"": true}" 17123,6,387,2017-04-27 20:12:16,http://heel.name/darlene,,52.194.88.18,"{""location"": ""MD"", ""is_mobile"": true}" 17124,6,387,2017-05-02 17:05:13,http://cummeratagraham.com/modesta,,144.162.225.44,"{""location"": ""KY"", ""is_mobile"": true}" 17125,6,387,2017-01-27 06:58:41,http://rowekeebler.com/daisy.gutmann,,11.5.208.54,"{""location"": ""NR"", ""is_mobile"": true}" 17126,6,387,2017-01-31 05:59:50,http://parisian.co/rosina.corkery,,203.170.201.89,"{""location"": ""DK"", ""is_mobile"": false}" 17127,6,387,2017-05-01 01:55:42,http://leffler.biz/erica_wyman,,29.209.66.109,"{""location"": ""KY"", ""is_mobile"": true}" 17128,6,387,2017-03-06 06:49:13,http://klocko.biz/maximillian,,89.119.130.109,"{""location"": ""LU"", ""is_mobile"": true}" 17129,6,387,2017-06-08 07:40:23,http://beatty.net/darius,,147.89.87.12,"{""location"": ""MG"", ""is_mobile"": true}" 17130,6,387,2017-01-16 05:10:09,http://beermurazik.org/zachariah,,162.163.26.246,"{""location"": ""BV"", ""is_mobile"": true}" 17131,6,387,2017-02-15 03:39:51,http://yostlindgren.biz/evelyn.christiansen,,251.238.52.24,"{""location"": ""TL"", ""is_mobile"": true}" 17132,6,387,2016-12-17 05:02:29,http://kleinwilliamson.co/santino.grimes,,193.89.181.250,"{""location"": ""BJ"", ""is_mobile"": true}" 17133,6,387,2017-02-10 02:19:49,http://okunevafisher.biz/sherwood,,234.27.180.242,"{""location"": ""NZ"", ""is_mobile"": true}" 17134,6,387,2017-05-23 17:00:09,http://yostoconner.info/makenzie,,203.54.105.100,"{""location"": ""BQ"", ""is_mobile"": true}" 17135,6,387,2017-02-18 18:03:39,http://rice.co/beau_reilly,,133.112.99.67,"{""location"": ""MV"", ""is_mobile"": true}" 17136,6,387,2016-12-24 08:22:55,http://stamm.biz/frankie_lesch,,66.164.203.146,"{""location"": ""AU"", ""is_mobile"": true}" 17137,6,387,2017-05-21 12:15:04,http://hirthe.net/camylle.gulgowski,,164.186.58.211,"{""location"": ""IL"", ""is_mobile"": false}" 17138,6,387,2017-04-26 14:24:46,http://jenkins.name/maurine.brown,,234.64.90.81,"{""location"": ""VG"", ""is_mobile"": false}" 17139,6,387,2017-04-01 15:45:55,http://schoenrodriguez.biz/karson,,44.134.74.14,"{""location"": ""BO"", ""is_mobile"": false}" 17140,6,387,2017-04-01 23:54:00,http://kihn.net/dangelo,,193.68.39.18,"{""location"": ""MU"", ""is_mobile"": false}" 17141,6,387,2017-03-13 15:10:54,http://trompschroeder.info/vince.reichert,,62.69.11.21,"{""location"": ""FR"", ""is_mobile"": false}" 17142,6,387,2017-06-04 22:46:21,http://bodebeer.info/rupert,,40.105.102.98,"{""location"": ""UM"", ""is_mobile"": false}" 17143,6,387,2016-12-29 09:33:19,http://bogan.com/velva.ernser,,45.232.184.236,"{""location"": ""TF"", ""is_mobile"": true}" 17144,6,387,2016-12-27 18:56:00,http://murazik.name/ottis,,49.139.153.193,"{""location"": ""VC"", ""is_mobile"": true}" 17145,6,387,2017-01-29 12:29:34,http://price.org/elouise,,254.31.194.70,"{""location"": ""BO"", ""is_mobile"": false}" 17146,6,387,2016-12-14 02:09:27,http://hoeger.name/tierra,,200.113.80.239,"{""location"": ""SZ"", ""is_mobile"": true}" 17147,6,387,2017-02-07 21:20:26,http://parker.info/sallie_schowalter,,147.172.244.207,"{""location"": ""SC"", ""is_mobile"": false}" 17148,6,387,2017-06-11 17:03:13,http://doyle.name/sid,,77.101.110.199,"{""location"": ""FR"", ""is_mobile"": true}" 17149,6,387,2017-01-01 01:09:37,http://leannon.biz/emma.nikolaus,,48.124.170.246,"{""location"": ""US"", ""is_mobile"": true}" 17150,6,387,2017-03-18 21:03:01,http://hills.org/emilia_steuber,,200.100.202.35,"{""location"": ""TK"", ""is_mobile"": false}" 17151,6,387,2017-01-17 19:31:33,http://douglas.org/saul,,8.87.171.8,"{""location"": ""PG"", ""is_mobile"": true}" 4278,2,94,2017-05-21 15:42:58,http://bergstrom.org/monroe,,51.4.148.50,"{""location"": ""SV"", ""is_mobile"": true}" 4279,2,94,2016-12-30 06:26:37,http://grant.co/johnson.langworth,,52.155.164.169,"{""location"": ""ZW"", ""is_mobile"": true}" 4280,2,94,2017-04-28 22:23:57,http://smitham.com/chad,,173.19.189.231,"{""location"": ""IE"", ""is_mobile"": false}" 4281,2,94,2017-02-12 11:41:49,http://walter.com/john,,145.77.134.115,"{""location"": ""BM"", ""is_mobile"": true}" 4282,2,94,2016-12-19 04:58:38,http://hermistonwelch.net/madyson.marvin,,40.8.58.181,"{""location"": ""GP"", ""is_mobile"": true}" 4283,2,94,2017-01-31 08:22:35,http://harveykoepp.com/kade,,252.177.227.69,"{""location"": ""BH"", ""is_mobile"": false}" 4284,2,94,2017-03-21 17:13:35,http://crona.biz/marquise.konopelski,,68.134.168.157,"{""location"": ""MW"", ""is_mobile"": false}" 4285,2,94,2017-04-07 11:24:09,http://lang.biz/kathryn,,33.223.12.109,"{""location"": ""SI"", ""is_mobile"": false}" 4286,2,94,2017-02-18 10:27:57,http://adamsdietrich.net/rebeca_kemmer,,166.251.207.11,"{""location"": ""MM"", ""is_mobile"": true}" 4287,2,94,2016-12-16 23:14:01,http://mckenziehintz.com/gilberto,,202.247.88.24,"{""location"": ""SH"", ""is_mobile"": true}" 4288,2,94,2017-03-13 06:28:29,http://mcdermott.net/jaquelin,,189.205.30.110,"{""location"": ""MW"", ""is_mobile"": true}" 4289,2,94,2017-05-06 15:23:21,http://cummerata.info/katlynn,,153.244.89.194,"{""location"": ""CA"", ""is_mobile"": false}" 4290,2,94,2017-05-16 16:05:36,http://mclaughlin.co/winnifred,,92.159.18.120,"{""location"": ""CF"", ""is_mobile"": false}" 4291,2,94,2017-03-31 08:42:13,http://pfannerstill.io/susanna_schiller,,197.53.183.167,"{""location"": ""PN"", ""is_mobile"": true}" 4292,2,94,2016-12-18 03:34:04,http://bartonjerde.net/estella,,168.253.134.41,"{""location"": ""VC"", ""is_mobile"": true}" 4293,2,94,2017-04-10 13:41:49,http://dachkuphal.net/alyon_rodriguez,,11.77.76.152,"{""location"": ""BG"", ""is_mobile"": false}" 4294,2,94,2017-06-04 03:17:31,http://koch.info/jeffry.abbott,,211.154.216.172,"{""location"": ""MT"", ""is_mobile"": true}" 4295,2,94,2016-12-14 07:00:17,http://boehmhalvorson.net/opal,,103.38.67.242,"{""location"": ""CA"", ""is_mobile"": false}" 4296,2,94,2017-04-21 02:26:55,http://mills.com/jakob.jakubowski,,154.124.105.206,"{""location"": ""AF"", ""is_mobile"": false}" 4297,2,94,2017-01-07 22:20:22,http://auer.biz/candice,,151.43.220.118,"{""location"": ""NF"", ""is_mobile"": false}" 4298,2,94,2017-06-04 23:11:56,http://feest.co/carlos.pfannerstill,,164.190.61.46,"{""location"": ""CM"", ""is_mobile"": false}" 4299,2,94,2017-03-20 19:28:03,http://mitchell.co/rickie,,63.113.23.176,"{""location"": ""MR"", ""is_mobile"": true}" 4300,2,94,2017-04-12 11:10:32,http://schaden.org/damion.skiles,,101.41.109.86,"{""location"": ""BW"", ""is_mobile"": true}" 4301,2,94,2017-01-17 12:02:13,http://harrisbeatty.io/erich_romaguera,,254.190.88.174,"{""location"": ""NG"", ""is_mobile"": false}" 4302,2,94,2016-12-16 06:29:02,http://lesch.co/harvey,,76.239.247.88,"{""location"": ""ET"", ""is_mobile"": true}" 4303,2,94,2017-01-24 13:12:24,http://stoltenberg.biz/desmond,,145.139.231.57,"{""location"": ""NC"", ""is_mobile"": true}" 4304,2,94,2017-05-15 19:18:07,http://kaulke.org/reyes,,171.206.33.114,"{""location"": ""AF"", ""is_mobile"": true}" 4305,2,94,2017-01-26 19:24:11,http://reinger.co/scottie_klocko,,107.78.226.182,"{""location"": ""AD"", ""is_mobile"": false}" 4306,2,94,2016-12-22 07:28:15,http://white.io/nikko,,42.164.184.131,"{""location"": ""GA"", ""is_mobile"": false}" 4307,2,94,2017-03-30 19:43:33,http://stroman.com/antonietta.kris,,17.88.96.139,"{""location"": ""KW"", ""is_mobile"": true}" 4308,2,95,2017-04-12 22:56:26,http://bernierbaumbach.com/freddie,,174.103.11.223,"{""location"": ""LR"", ""is_mobile"": false}" 4309,2,95,2017-02-07 08:45:39,http://moen.biz/gunnar,,104.180.182.186,"{""location"": ""CO"", ""is_mobile"": false}" 4310,2,95,2017-01-03 19:50:53,http://kuphal.biz/diamond_goyette,,23.45.98.142,"{""location"": ""CN"", ""is_mobile"": false}" 4311,2,95,2017-02-05 12:15:16,http://beier.info/reta.roob,,225.82.221.165,"{""location"": ""VG"", ""is_mobile"": true}" 4312,2,95,2017-03-11 13:24:17,http://oconnell.info/keshaun_rohan,,118.160.26.143,"{""location"": ""SC"", ""is_mobile"": true}" 4313,2,95,2017-02-28 03:21:38,http://reichel.biz/chester.frami,,227.210.201.138,"{""location"": ""DM"", ""is_mobile"": true}" 4314,2,95,2017-05-07 12:44:16,http://jaskolski.io/okey_kutch,,212.97.149.95,"{""location"": ""PF"", ""is_mobile"": false}" 4315,2,95,2017-02-07 00:48:08,http://skiles.name/alvis,,228.186.92.118,"{""location"": ""ET"", ""is_mobile"": true}" 4316,2,95,2017-05-13 14:01:38,http://bechtelar.org/kira_pfeffer,,108.93.224.131,"{""location"": ""MH"", ""is_mobile"": true}" 4317,2,95,2017-01-14 15:27:59,http://windlerkris.info/clint.wintheiser,,86.33.91.198,"{""location"": ""GL"", ""is_mobile"": false}" 4318,2,95,2017-04-28 15:20:35,http://brekkedubuque.co/max,,76.31.176.208,"{""location"": ""CW"", ""is_mobile"": true}" 4319,2,95,2016-12-27 19:23:20,http://feil.co/shanny_mante,,20.36.6.110,"{""location"": ""AF"", ""is_mobile"": false}" 4320,2,95,2017-05-07 19:49:48,http://blick.net/megane,,202.230.43.242,"{""location"": ""IT"", ""is_mobile"": false}" 4321,2,95,2017-03-07 06:23:57,http://collierstehr.co/julianne_lesch,,45.194.245.186,"{""location"": ""TC"", ""is_mobile"": false}" 4322,2,95,2017-05-03 15:20:11,http://thiel.co/elian.osinski,,190.181.149.17,"{""location"": ""FI"", ""is_mobile"": false}" 4323,2,95,2017-03-12 12:18:45,http://nitzsche.biz/tyrese,,97.11.185.45,"{""location"": ""VC"", ""is_mobile"": true}" 4324,2,95,2017-05-28 23:47:27,http://balistreri.co/jasen,,200.203.154.148,"{""location"": ""KZ"", ""is_mobile"": true}" 4325,2,95,2017-03-12 18:38:20,http://stokes.io/norbert.gulgowski,,75.217.177.43,"{""location"": ""FO"", ""is_mobile"": false}" 4326,2,95,2017-05-15 22:23:41,http://predoviczieme.name/garret_christiansen,,186.163.133.224,"{""location"": ""NI"", ""is_mobile"": false}" 4327,2,95,2017-04-22 19:34:04,http://hamill.biz/trea_heathcote,,245.231.63.222,"{""location"": ""HR"", ""is_mobile"": false}" 4328,2,95,2017-06-04 03:04:33,http://jaskolski.biz/napoleon_franecki,,193.92.151.251,"{""location"": ""SK"", ""is_mobile"": false}" 4329,2,95,2017-01-07 08:45:51,http://collins.co/mellie.collier,,194.75.27.166,"{""location"": ""DM"", ""is_mobile"": true}" 4330,2,95,2017-04-19 16:05:18,http://stiedemannkihn.biz/beverly,,220.202.126.184,"{""location"": ""SJ"", ""is_mobile"": false}" 4331,2,95,2017-02-23 10:42:54,http://grimesrunolfon.co/shyanne,,171.202.244.252,"{""location"": ""ZM"", ""is_mobile"": true}" 4332,2,95,2017-01-30 13:31:18,http://ohara.biz/dorris_schroeder,,195.26.43.35,"{""location"": ""SI"", ""is_mobile"": true}" 4333,2,95,2017-01-02 07:25:12,http://mayert.name/camilla.lemke,,206.199.113.77,"{""location"": ""SD"", ""is_mobile"": false}" 11217,4,251,2017-04-26 12:13:17,http://bednar.name/lottie.bogisich,,32.30.192.6,"{""location"": ""MV"", ""is_mobile"": false}" 11218,4,251,2017-05-30 16:54:30,http://cain.com/alexandrine,,30.151.93.247,"{""location"": ""AF"", ""is_mobile"": true}" 11219,4,251,2017-04-04 02:55:11,http://hyatt.info/eliza,,5.16.64.68,"{""location"": ""MZ"", ""is_mobile"": false}" 11220,4,251,2017-01-12 04:25:32,http://kuhic.com/charley_schowalter,,61.130.239.89,"{""location"": ""LB"", ""is_mobile"": false}" 11221,4,251,2017-04-09 18:38:03,http://doyle.net/seth_vonrueden,,32.76.179.161,"{""location"": ""LS"", ""is_mobile"": false}" 11222,4,251,2017-03-27 00:48:22,http://gislason.info/clement_bogan,,89.81.102.189,"{""location"": ""PT"", ""is_mobile"": false}" 11223,4,251,2017-04-11 17:35:21,http://bergstrom.name/willow_davis,,59.133.174.61,"{""location"": ""DM"", ""is_mobile"": true}" 11224,4,251,2017-01-02 04:01:46,http://hammeteuber.name/ada_lakin,,253.150.194.120,"{""location"": ""SK"", ""is_mobile"": false}" 11225,4,251,2017-06-02 03:53:07,http://haag.info/kristoffer.kertzmann,,127.12.44.41,"{""location"": ""BQ"", ""is_mobile"": true}" 11226,4,251,2016-12-28 15:48:08,http://altenwertherdman.net/judson,,124.44.121.53,"{""location"": ""HK"", ""is_mobile"": true}" 11227,4,251,2017-02-07 16:15:37,http://bednar.info/santiago,,138.125.200.148,"{""location"": ""CF"", ""is_mobile"": true}" 11228,4,251,2017-05-09 17:29:25,http://cormierhammes.org/jeromy_hilll,,129.25.115.5,"{""location"": ""ES"", ""is_mobile"": false}" 11229,4,251,2017-01-27 11:38:57,http://olson.co/sven,,235.99.120.30,"{""location"": ""NP"", ""is_mobile"": false}" 11230,4,251,2017-01-29 12:25:07,http://satterfield.info/bonnie,,221.46.201.181,"{""location"": ""BR"", ""is_mobile"": false}" 11231,4,251,2017-01-30 01:54:39,http://jacobsonkiehn.biz/jerrod,,150.138.219.241,"{""location"": ""NR"", ""is_mobile"": true}" 11232,4,251,2017-02-12 03:07:15,http://hackett.com/javonte,,49.211.128.33,"{""location"": ""TM"", ""is_mobile"": true}" 11233,4,251,2016-12-24 00:27:03,http://kelerhahn.co/casper_lebsack,,194.154.231.88,"{""location"": ""CY"", ""is_mobile"": true}" 11234,4,251,2017-02-10 19:48:38,http://stark.name/elouise_reynolds,,214.205.109.173,"{""location"": ""CC"", ""is_mobile"": true}" 11235,4,251,2017-03-23 11:57:41,http://kiehn.co/herman,,64.226.128.247,"{""location"": ""BJ"", ""is_mobile"": true}" 11236,4,251,2016-12-26 13:40:54,http://lehnerwindler.biz/cecilia,,213.33.106.9,"{""location"": ""BG"", ""is_mobile"": false}" 11237,4,251,2017-02-09 18:04:38,http://witting.io/grayson.douglas,,67.44.81.95,"{""location"": ""FO"", ""is_mobile"": false}" 11238,4,251,2017-06-03 13:13:53,http://dietrich.io/travon,,168.133.63.148,"{""location"": ""SR"", ""is_mobile"": true}" 11239,4,251,2017-05-24 16:35:56,http://ziememueller.biz/irma,,203.252.127.25,"{""location"": ""TV"", ""is_mobile"": true}" 11240,4,251,2017-01-03 19:56:08,http://hoppegutmann.io/kim_volkman,,130.174.3.51,"{""location"": ""EH"", ""is_mobile"": false}" 11241,4,251,2016-12-18 17:42:53,http://swift.biz/alisa,,150.155.39.233,"{""location"": ""MS"", ""is_mobile"": true}" 11242,4,251,2017-05-02 08:27:44,http://hirtheskiles.io/aglae.vonrueden,,95.205.191.126,"{""location"": ""MP"", ""is_mobile"": false}" 11243,4,251,2017-02-18 20:25:21,http://ernserdavis.biz/kenyon,,186.170.97.111,"{""location"": ""TM"", ""is_mobile"": false}" 11244,4,251,2016-12-23 09:10:44,http://bauchconn.info/oren,,42.111.22.116,"{""location"": ""PW"", ""is_mobile"": true}" 11245,4,251,2017-05-18 01:54:50,http://okuneva.name/judd.rath,,116.9.37.217,"{""location"": ""GW"", ""is_mobile"": true}" 11246,4,251,2017-01-17 10:31:06,http://treutel.com/jayden.boyle,,96.99.112.122,"{""location"": ""OM"", ""is_mobile"": false}" 11247,4,251,2017-05-06 16:30:22,http://hansen.io/jalen,,225.224.11.52,"{""location"": ""UZ"", ""is_mobile"": false}" 11248,4,251,2017-03-01 17:48:38,http://hirthe.net/dorris_vonrueden,,170.131.137.6,"{""location"": ""TR"", ""is_mobile"": true}" 11249,4,251,2017-03-24 02:32:35,http://moriettecummerata.io/rosalee,,106.131.157.179,"{""location"": ""AM"", ""is_mobile"": false}" 11250,4,251,2017-05-14 09:07:47,http://reichert.co/quinton,,96.124.98.230,"{""location"": ""CH"", ""is_mobile"": true}" 11251,4,251,2017-02-05 06:23:06,http://balistreri.biz/felix,,34.211.68.144,"{""location"": ""BF"", ""is_mobile"": false}" 11252,4,251,2017-01-17 04:20:39,http://trantow.info/bernadine,,22.84.26.176,"{""location"": ""TM"", ""is_mobile"": false}" 11253,4,251,2017-04-28 04:42:54,http://shieldscarroll.com/rubye.thompson,,159.111.226.169,"{""location"": ""US"", ""is_mobile"": false}" 11254,4,251,2017-01-04 18:30:18,http://kiehn.org/novella,,68.53.249.211,"{""location"": ""GL"", ""is_mobile"": false}" 11255,4,251,2017-02-08 21:22:48,http://hamill.io/allison,,201.69.4.33,"{""location"": ""NU"", ""is_mobile"": true}" 11256,4,251,2017-04-06 05:59:45,http://mcculloughleuschke.co/cary,,11.90.9.233,"{""location"": ""AE"", ""is_mobile"": true}" 11257,4,252,2017-05-01 20:38:38,http://jones.name/anabel,,239.46.187.81,"{""location"": ""IT"", ""is_mobile"": false}" 11258,4,252,2017-05-17 05:37:20,http://hartmanncole.io/dane_bechtelar,,70.199.102.13,"{""location"": ""CC"", ""is_mobile"": false}" 11259,4,252,2017-01-06 14:08:01,http://simonis.info/tyra,,187.40.14.215,"{""location"": ""BT"", ""is_mobile"": false}" 11260,4,252,2017-04-22 00:17:53,http://franeckiwiza.co/jaquan,,35.130.82.4,"{""location"": ""LI"", ""is_mobile"": false}" 11261,4,252,2017-05-06 06:39:36,http://herzog.biz/bradley,,204.241.236.181,"{""location"": ""AO"", ""is_mobile"": true}" 11262,4,252,2017-03-20 16:42:06,http://bayer.net/avis,,178.72.196.121,"{""location"": ""JM"", ""is_mobile"": false}" 11263,4,252,2017-02-22 23:11:39,http://cummeratahoeger.name/abigail.homenick,,43.111.48.26,"{""location"": ""SO"", ""is_mobile"": false}" 11264,4,252,2017-01-06 13:48:45,http://corwin.org/bethany_feil,,117.188.109.100,"{""location"": ""EC"", ""is_mobile"": false}" 11265,4,252,2017-02-09 17:45:51,http://franeckiabshire.co/celestine.schuster,,11.124.188.78,"{""location"": ""SN"", ""is_mobile"": true}" 11266,4,252,2017-04-29 12:23:56,http://kovacekdaniel.co/britney,,28.174.140.215,"{""location"": ""GR"", ""is_mobile"": false}" 11267,4,252,2017-03-10 23:13:28,http://vonrueden.biz/angelo_bashirian,,157.203.194.77,"{""location"": ""GB"", ""is_mobile"": false}" 11268,4,252,2016-12-24 17:09:51,http://turnerweimann.name/khalil,,21.52.136.211,"{""location"": ""MH"", ""is_mobile"": false}" 11269,4,252,2017-05-28 20:38:23,http://zulauf.net/danyka.yundt,,192.158.173.202,"{""location"": ""SG"", ""is_mobile"": true}" 11270,4,252,2017-04-27 07:05:43,http://sauerharber.name/alexis_weber,,200.156.187.68,"{""location"": ""NE"", ""is_mobile"": true}" 11271,4,252,2017-01-22 21:35:36,http://wintheisermurray.biz/paul_gutkowski,,237.207.94.82,"{""location"": ""DO"", ""is_mobile"": false}" 11272,4,252,2017-06-07 21:27:15,http://kelerolson.io/ashleigh_mclaughlin,,174.85.143.155,"{""location"": ""SC"", ""is_mobile"": false}" 8222,3,182,2017-03-23 23:39:16,http://considine.com/robb.durgan,0.2962482277,205.181.84.97,"{""location"": ""UG"", ""is_mobile"": false}" 8223,3,183,2017-03-23 21:12:08,http://flatley.info/rick,0.0986641912,139.50.211.252,"{""location"": ""EC"", ""is_mobile"": false}" 8224,3,183,2017-02-11 00:05:14,http://spinka.co/eugenia,0.0079814200,228.213.82.221,"{""location"": ""FJ"", ""is_mobile"": true}" 8225,3,183,2017-05-23 17:52:47,http://casper.info/marianne,0.4677839533,58.222.11.130,"{""location"": ""TJ"", ""is_mobile"": false}" 8226,3,183,2017-02-08 10:37:33,http://moen.io/natasha,0.7039902549,83.125.244.166,"{""location"": ""NU"", ""is_mobile"": false}" 8227,3,183,2017-06-02 22:24:35,http://bosco.name/katheryn,0.6906414107,57.111.46.95,"{""location"": ""ZW"", ""is_mobile"": true}" 8228,3,183,2017-02-24 09:55:38,http://parisian.info/lorenzo_lueilwitz,0.8408279811,7.48.65.107,"{""location"": ""TF"", ""is_mobile"": false}" 8229,3,183,2017-05-18 03:08:23,http://reillyhayes.biz/haleigh,0.7657845760,95.93.47.221,"{""location"": ""SE"", ""is_mobile"": true}" 8230,3,183,2017-02-09 15:43:07,http://labadiefay.info/mose,0.4507568282,162.81.235.124,"{""location"": ""AT"", ""is_mobile"": false}" 8231,3,183,2017-05-21 00:31:19,http://kling.co/randall_upton,0.7945364698,252.148.86.232,"{""location"": ""BA"", ""is_mobile"": false}" 8232,3,183,2017-05-29 15:30:02,http://crist.name/yasmeen,0.5527805877,29.155.94.87,"{""location"": ""NI"", ""is_mobile"": false}" 8233,3,183,2017-03-22 01:53:32,http://smithamrenner.io/maria.quigley,0.0637617724,101.195.193.169,"{""location"": ""VA"", ""is_mobile"": false}" 8234,3,183,2016-12-17 10:10:11,http://farrellwisozk.co/berry,0.4617551486,84.222.149.111,"{""location"": ""SS"", ""is_mobile"": true}" 8235,3,183,2017-05-22 13:15:58,http://schneider.name/stefan.heidenreich,0.3708174513,85.173.227.221,"{""location"": ""MR"", ""is_mobile"": true}" 8236,3,183,2017-01-19 09:27:13,http://anderson.io/libby_streich,0.3609759830,96.38.219.210,"{""location"": ""CN"", ""is_mobile"": true}" 8237,3,183,2017-03-02 19:40:00,http://gulgowskikihn.net/herminio,0.5674923290,175.177.12.174,"{""location"": ""EH"", ""is_mobile"": false}" 8238,3,183,2017-05-12 07:27:37,http://olson.info/orval.volkman,0.4694310735,74.51.207.247,"{""location"": ""TG"", ""is_mobile"": true}" 8239,3,183,2017-03-13 05:08:46,http://stroman.com/marcia_schumm,0.5864230716,2.21.48.235,"{""location"": ""TN"", ""is_mobile"": false}" 8240,3,183,2017-03-03 02:12:20,http://bode.biz/norris_dooley,0.4041705089,142.253.16.83,"{""location"": ""BV"", ""is_mobile"": false}" 8241,3,183,2017-01-11 10:48:59,http://schusterolson.net/heber.kozey,0.7007590000,21.129.194.194,"{""location"": ""BZ"", ""is_mobile"": false}" 8242,3,183,2017-04-13 05:22:06,http://hermannjacobs.biz/stuart.bergstrom,0.2622483691,185.76.37.128,"{""location"": ""LU"", ""is_mobile"": false}" 8243,3,183,2017-02-07 11:02:09,http://leannon.name/ernest.hyatt,0.2423305636,10.180.207.138,"{""location"": ""LR"", ""is_mobile"": false}" 8244,3,183,2017-05-10 18:02:32,http://waters.io/leanna,0.7585655608,215.143.77.74,"{""location"": ""PW"", ""is_mobile"": true}" 8245,3,183,2017-01-21 11:51:11,http://trantow.co/terry.kuhn,0.8712726794,251.150.89.113,"{""location"": ""NI"", ""is_mobile"": true}" 8246,3,183,2017-02-26 00:13:54,http://gislason.org/alicia.weinat,0.6843434964,191.208.73.184,"{""location"": ""NZ"", ""is_mobile"": false}" 8247,3,183,2017-02-21 16:23:07,http://okunevablock.io/orlando_harber,0.1662792097,66.219.80.148,"{""location"": ""DZ"", ""is_mobile"": true}" 8248,3,183,2017-02-28 11:45:54,http://eichmann.biz/aunta,0.2468993075,104.120.103.248,"{""location"": ""MZ"", ""is_mobile"": false}" 8249,3,183,2017-02-04 16:29:44,http://pollich.info/maynard,0.1346319056,156.159.9.54,"{""location"": ""DM"", ""is_mobile"": false}" 8250,3,183,2017-03-02 08:27:44,http://halvorson.biz/emilio.keler,0.2954837738,61.167.51.18,"{""location"": ""BA"", ""is_mobile"": false}" 8251,3,183,2017-02-14 12:28:11,http://will.info/haie,0.8628490654,35.243.208.222,"{""location"": ""TH"", ""is_mobile"": false}" 8252,3,183,2017-03-25 15:25:58,http://oberbrunnerjast.org/eugenia_hane,0.5897202699,243.232.183.61,"{""location"": ""BL"", ""is_mobile"": false}" 8253,3,183,2016-12-21 03:28:40,http://klocko.co/ozella_klocko,0.3465612983,185.157.96.10,"{""location"": ""JO"", ""is_mobile"": false}" 8254,3,183,2016-12-27 20:28:31,http://conn.com/margarett,0.0159075601,150.129.142.93,"{""location"": ""HK"", ""is_mobile"": false}" 8255,3,183,2017-01-21 02:12:03,http://bechtelar.net/joe_hammes,0.3736383371,125.99.252.254,"{""location"": ""AM"", ""is_mobile"": true}" 8256,3,183,2017-06-07 01:58:29,http://raynor.biz/rigoberto.ferry,0.5635764953,254.188.137.215,"{""location"": ""RS"", ""is_mobile"": false}" 8257,3,183,2017-03-06 08:00:19,http://millerkeler.co/claudine.price,0.3383798651,86.127.19.217,"{""location"": ""KH"", ""is_mobile"": false}" 8258,3,183,2017-03-31 02:41:31,http://marquardt.io/tiffany_roberts,0.6536425368,108.176.174.73,"{""location"": ""KZ"", ""is_mobile"": false}" 8259,3,183,2017-01-10 21:09:08,http://kaulke.name/cordie,0.5869265810,4.102.146.57,"{""location"": ""CG"", ""is_mobile"": false}" 8260,3,183,2017-01-08 17:20:35,http://trantow.io/emmanuelle,0.9267603161,191.66.113.26,"{""location"": ""HN"", ""is_mobile"": true}" 8261,3,183,2017-03-05 20:11:57,http://lehnerlesch.co/remington,0.1520769775,226.39.125.124,"{""location"": ""VA"", ""is_mobile"": false}" 8262,3,183,2017-03-27 21:37:14,http://altenwerth.org/sheridan,0.4162691699,204.247.135.39,"{""location"": ""MH"", ""is_mobile"": false}" 8263,3,183,2017-02-13 11:48:50,http://mosciski.co/jeanne,0.2533960227,149.212.50.31,"{""location"": ""SY"", ""is_mobile"": true}" 8264,3,183,2017-03-12 00:59:12,http://andersonmarquardt.io/buddy_oberbrunner,0.1052939247,196.53.18.66,"{""location"": ""WF"", ""is_mobile"": false}" 8265,3,183,2017-04-27 13:27:58,http://mills.name/camden,0.4281029472,20.211.169.134,"{""location"": ""NC"", ""is_mobile"": false}" 8266,3,183,2017-05-04 16:56:49,http://mcdermott.name/whitney.keler,0.0728621901,126.214.141.239,"{""location"": ""FR"", ""is_mobile"": true}" 8267,3,183,2017-03-10 23:50:38,http://langschinner.biz/marcos_dibbert,0.2428124048,132.58.27.89,"{""location"": ""EC"", ""is_mobile"": false}" 8268,3,183,2017-01-07 14:28:08,http://reynolds.info/allison,0.5827405821,248.41.214.65,"{""location"": ""FJ"", ""is_mobile"": true}" 8269,3,184,2017-02-02 11:06:34,http://heathcote.com/casandra.kuhn,0.5287252876,38.239.217.192,"{""location"": ""BW"", ""is_mobile"": true}" 8270,3,184,2017-01-31 01:22:01,http://rippin.co/lisandro.goodwin,0.0966712014,79.224.6.114,"{""location"": ""SO"", ""is_mobile"": true}" 8271,3,184,2017-02-23 06:33:19,http://funk.biz/katelynn,0.7497825634,225.244.163.45,"{""location"": ""CN"", ""is_mobile"": true}" 8272,3,184,2017-03-19 12:32:20,http://stracke.io/agustin,0.9279269219,102.131.143.45,"{""location"": ""OM"", ""is_mobile"": false}" 8273,3,184,2017-05-05 11:24:55,http://wisozk.name/ulices,0.5639337120,119.12.150.23,"{""location"": ""BF"", ""is_mobile"": true}" 14168,5,318,2017-04-11 05:08:09,http://dooley.io/reed_price,,32.78.26.117,"{""location"": ""CY"", ""is_mobile"": true}" 14169,5,318,2016-12-20 17:55:56,http://rosenbaum.net/florine_robel,,110.141.213.49,"{""location"": ""PT"", ""is_mobile"": true}" 14170,5,318,2017-02-19 05:28:24,http://kris.net/laney,,154.81.253.105,"{""location"": ""EH"", ""is_mobile"": false}" 14171,5,318,2016-12-23 04:22:45,http://dibbert.info/leda.schroeder,,196.222.242.5,"{""location"": ""CM"", ""is_mobile"": true}" 14172,5,318,2017-03-28 19:12:13,http://greenholt.co/jovany_hermann,,66.41.46.97,"{""location"": ""AI"", ""is_mobile"": false}" 14173,5,318,2017-04-07 15:17:11,http://labadie.com/winston.bayer,,72.90.192.14,"{""location"": ""KY"", ""is_mobile"": true}" 14174,5,318,2017-05-11 15:02:42,http://kuphal.name/autumn,,140.85.174.178,"{""location"": ""AQ"", ""is_mobile"": false}" 14175,5,319,2017-01-26 08:10:15,http://toystanton.io/jeanette_lindgren,,133.17.194.212,"{""location"": ""VE"", ""is_mobile"": false}" 14176,5,319,2017-01-29 07:55:23,http://johnsonziemann.com/travon_anderson,,88.11.87.212,"{""location"": ""RS"", ""is_mobile"": false}" 14177,5,319,2017-02-14 08:47:57,http://cole.info/sophia,,95.148.123.98,"{""location"": ""MS"", ""is_mobile"": false}" 14178,5,319,2017-01-16 01:34:38,http://trantowebert.net/hayley,,192.159.12.61,"{""location"": ""VA"", ""is_mobile"": false}" 14179,5,319,2017-02-07 20:23:45,http://baumbachspencer.info/leann.kunde,,139.187.253.28,"{""location"": ""MQ"", ""is_mobile"": true}" 14180,5,319,2017-05-18 07:39:20,http://hodkiewicz.biz/anabel.kohler,,222.46.137.219,"{""location"": ""AO"", ""is_mobile"": true}" 14181,5,319,2017-04-12 07:46:27,http://brekke.info/marietta.ortiz,,5.223.222.129,"{""location"": ""VE"", ""is_mobile"": true}" 14182,5,319,2017-05-11 12:41:22,http://luettgenwolf.co/shanna.nikolaus,,97.247.97.40,"{""location"": ""VC"", ""is_mobile"": true}" 14183,5,319,2017-03-15 00:17:22,http://kuphal.org/bulah,,44.184.242.119,"{""location"": ""TO"", ""is_mobile"": true}" 14184,5,319,2017-04-16 11:53:08,http://harberjacobs.org/elvera.dickens,,131.206.228.87,"{""location"": ""ER"", ""is_mobile"": true}" 14185,5,319,2017-02-21 14:45:05,http://rempelhuels.com/rasheed.kunze,,241.91.215.176,"{""location"": ""KE"", ""is_mobile"": true}" 14186,5,319,2017-06-09 13:17:45,http://bartoletti.co/mikayla,,171.108.16.138,"{""location"": ""US"", ""is_mobile"": true}" 14187,5,319,2017-02-26 12:32:53,http://adams.biz/meggie.lubowitz,,61.6.218.183,"{""location"": ""PT"", ""is_mobile"": false}" 14188,5,319,2017-04-16 12:07:06,http://kunze.com/rick_littel,,195.218.209.62,"{""location"": ""FJ"", ""is_mobile"": true}" 14189,5,319,2017-04-13 09:28:23,http://reynolds.io/jacky.koch,,19.19.133.184,"{""location"": ""CY"", ""is_mobile"": true}" 14190,5,319,2017-03-17 16:47:44,http://gaylordrogahn.net/vladimir.goldner,,57.195.17.20,"{""location"": ""BZ"", ""is_mobile"": true}" 14191,5,319,2017-06-09 03:19:31,http://balistreri.co/hector,,205.231.253.17,"{""location"": ""TF"", ""is_mobile"": false}" 14192,5,319,2017-03-17 15:35:31,http://haag.name/era,,16.26.36.95,"{""location"": ""AU"", ""is_mobile"": true}" 14193,5,319,2017-05-09 23:37:17,http://vonrueden.io/liliane_hartmann,,3.30.32.204,"{""location"": ""MV"", ""is_mobile"": false}" 14194,5,319,2017-05-31 10:32:07,http://runte.co/lillie,,21.73.130.180,"{""location"": ""GH"", ""is_mobile"": true}" 14195,5,320,2017-02-01 03:54:15,http://pouros.net/kristopher,,52.7.51.230,"{""location"": ""NR"", ""is_mobile"": true}" 14196,5,320,2017-05-06 20:33:01,http://deckow.info/jeramy,,151.159.73.50,"{""location"": ""TJ"", ""is_mobile"": true}" 14197,5,320,2017-03-19 14:16:24,http://hintz.com/bailee,,45.126.231.7,"{""location"": ""GT"", ""is_mobile"": true}" 14198,5,320,2017-04-07 06:39:20,http://mohrschultz.com/janea_quitzon,,198.221.198.3,"{""location"": ""GN"", ""is_mobile"": true}" 14199,5,320,2017-02-01 18:02:55,http://barrows.io/mona.weinat,,237.214.252.43,"{""location"": ""TN"", ""is_mobile"": true}" 14200,5,320,2017-02-04 15:04:31,http://wunsch.co/corbin_trantow,,193.137.186.178,"{""location"": ""GW"", ""is_mobile"": true}" 14201,5,320,2017-01-01 04:30:31,http://schadenbarton.net/kaleigh,,44.204.132.223,"{""location"": ""EH"", ""is_mobile"": true}" 14202,5,320,2017-03-04 20:00:54,http://gusikowskibailey.com/esther.johnson,,47.31.18.138,"{""location"": ""TD"", ""is_mobile"": false}" 14203,5,320,2017-06-08 11:36:26,http://howell.io/rosemary_lockman,,81.135.218.119,"{""location"": ""AF"", ""is_mobile"": true}" 14204,5,320,2017-03-07 04:18:46,http://klockobernhard.name/dell_stark,,246.243.181.204,"{""location"": ""MV"", ""is_mobile"": false}" 14205,5,320,2017-01-06 01:53:21,http://effertz.biz/ofelia.dietrich,,79.82.116.190,"{""location"": ""SE"", ""is_mobile"": true}" 14206,5,320,2017-04-17 02:09:42,http://gleichner.co/lolita_graham,,199.90.138.49,"{""location"": ""ST"", ""is_mobile"": true}" 14207,5,320,2017-01-23 23:53:17,http://gutkowski.com/caleb_pagac,,30.43.66.95,"{""location"": ""BM"", ""is_mobile"": false}" 14208,5,320,2017-03-18 16:59:21,http://wilkinson.name/austyn,,149.20.64.8,"{""location"": ""UM"", ""is_mobile"": false}" 14209,5,320,2017-05-15 08:04:36,http://effertz.com/martin,,17.144.98.113,"{""location"": ""CA"", ""is_mobile"": false}" 14210,5,320,2017-05-03 15:19:28,http://turner.co/noelia.klocko,,116.59.242.226,"{""location"": ""WF"", ""is_mobile"": false}" 14211,5,320,2017-06-01 12:04:19,http://schultz.info/jairo.trantow,,128.220.3.37,"{""location"": ""CH"", ""is_mobile"": true}" 14212,5,320,2017-01-19 19:54:45,http://uptonabbott.com/stan,,244.104.2.169,"{""location"": ""DJ"", ""is_mobile"": false}" 14213,5,320,2017-04-15 20:34:55,http://ritchieernser.name/jey,,28.16.17.127,"{""location"": ""US"", ""is_mobile"": false}" 14214,5,320,2017-02-18 07:11:58,http://naderwyman.io/maximilian,,89.11.248.232,"{""location"": ""VA"", ""is_mobile"": true}" 14215,5,320,2017-03-07 20:42:01,http://reilly.biz/tierra.mcdermott,,144.196.129.47,"{""location"": ""ES"", ""is_mobile"": false}" 14216,5,320,2017-01-19 21:27:49,http://hagenesharris.com/nat,,89.111.2.18,"{""location"": ""VG"", ""is_mobile"": true}" 14217,5,320,2017-02-11 20:52:49,http://rohan.org/moriah.douglas,,75.194.196.139,"{""location"": ""NF"", ""is_mobile"": false}" 14218,5,320,2017-01-25 13:34:38,http://ortiz.name/renee.sauer,,121.153.172.240,"{""location"": ""VG"", ""is_mobile"": true}" 14219,5,320,2017-03-08 03:31:00,http://schadenhansen.name/edwina,,62.156.225.135,"{""location"": ""DK"", ""is_mobile"": true}" 14220,5,320,2017-04-13 00:29:00,http://swaniawski.biz/aiyana.denesik,,8.91.110.94,"{""location"": ""GU"", ""is_mobile"": true}" 14221,5,320,2017-05-05 01:50:16,http://windler.org/raegan_anderson,,9.238.137.50,"{""location"": ""SO"", ""is_mobile"": true}" 14222,5,320,2017-05-26 23:59:42,http://botsford.org/abel_schamberger,,25.52.234.72,"{""location"": ""RO"", ""is_mobile"": false}" 17152,6,387,2017-02-23 03:04:34,http://wehner.org/sandra,,127.204.151.115,"{""location"": ""BI"", ""is_mobile"": false}" 17153,6,387,2017-01-28 08:39:02,http://wolf.biz/ericka,,224.222.175.201,"{""location"": ""EC"", ""is_mobile"": false}" 17154,6,387,2017-04-19 17:33:43,http://maggio.com/jude,,162.31.35.76,"{""location"": ""LS"", ""is_mobile"": false}" 17155,6,387,2017-01-27 11:08:39,http://pfefferleuschke.net/nora.weinat,,247.239.155.77,"{""location"": ""GH"", ""is_mobile"": true}" 17156,6,387,2017-03-17 06:56:41,http://stokesvolkman.info/moises,,193.128.153.71,"{""location"": ""LC"", ""is_mobile"": false}" 17157,6,387,2017-06-06 16:48:23,http://beatty.org/hiram,,236.21.79.17,"{""location"": ""KN"", ""is_mobile"": false}" 17158,6,387,2017-04-21 22:59:45,http://cremin.org/dejon.hettinger,,77.160.80.35,"{""location"": ""NI"", ""is_mobile"": true}" 17159,6,387,2016-12-18 09:08:52,http://frami.org/victoria.crooks,,192.79.236.179,"{""location"": ""NE"", ""is_mobile"": true}" 17160,6,387,2017-05-12 19:14:14,http://fadel.com/tony,,19.119.60.62,"{""location"": ""ME"", ""is_mobile"": false}" 17161,6,387,2016-12-26 19:52:49,http://deckow.net/kirsten_lemke,,25.157.39.193,"{""location"": ""CK"", ""is_mobile"": false}" 17162,6,387,2017-06-03 23:49:14,http://lednerdietrich.co/ceasar,,232.139.222.150,"{""location"": ""KW"", ""is_mobile"": false}" 17163,6,387,2017-02-28 19:26:56,http://haag.biz/talon,,148.33.80.119,"{""location"": ""BY"", ""is_mobile"": true}" 17164,6,387,2017-05-02 13:26:50,http://goldner.io/lindsay_zboncak,,214.207.135.196,"{""location"": ""DZ"", ""is_mobile"": false}" 17165,6,387,2016-12-15 03:45:42,http://stanton.co/ethel,,128.91.172.190,"{""location"": ""FM"", ""is_mobile"": false}" 17166,6,387,2017-05-20 09:01:54,http://kemmeroconner.name/oceane,,124.34.53.128,"{""location"": ""LB"", ""is_mobile"": true}" 17167,6,387,2017-05-25 07:19:47,http://rolfson.name/leif,,108.5.121.131,"{""location"": ""RO"", ""is_mobile"": false}" 17168,6,387,2017-03-10 13:49:55,http://greenfelder.name/kathryn_bechtelar,,239.29.239.2,"{""location"": ""IM"", ""is_mobile"": true}" 17169,6,387,2017-06-04 20:30:37,http://turcotte.info/eve_schimmel,,47.161.238.14,"{""location"": ""KI"", ""is_mobile"": true}" 17170,6,387,2017-05-25 18:09:19,http://huelpowlowski.co/cameron_block,,201.74.191.41,"{""location"": ""KG"", ""is_mobile"": false}" 17171,6,387,2017-02-19 20:43:51,http://lynch.com/blair_herzog,,38.246.158.17,"{""location"": ""TV"", ""is_mobile"": true}" 17172,6,387,2017-04-14 11:46:02,http://gislason.com/chloe,,12.229.135.200,"{""location"": ""BD"", ""is_mobile"": false}" 17173,6,387,2017-02-17 20:39:23,http://steuber.name/iliana_daugherty,,166.137.239.228,"{""location"": ""LB"", ""is_mobile"": true}" 17174,6,388,2017-01-09 10:37:30,http://gibson.org/spencer,,122.232.247.124,"{""location"": ""SX"", ""is_mobile"": false}" 17175,6,388,2016-12-29 09:47:13,http://hauck.name/derek.heller,,152.75.33.173,"{""location"": ""MU"", ""is_mobile"": false}" 17176,6,388,2017-01-12 20:59:18,http://hilll.net/fae.conroy,,122.65.151.253,"{""location"": ""CL"", ""is_mobile"": false}" 17177,6,388,2017-01-14 05:36:30,http://barton.info/ismael,,5.230.208.176,"{""location"": ""YT"", ""is_mobile"": true}" 17178,6,388,2017-01-12 00:26:18,http://hammes.name/ima,,188.64.54.126,"{""location"": ""MD"", ""is_mobile"": false}" 17179,6,388,2017-05-04 10:06:38,http://lindgren.com/oma_schmeler,,163.64.5.4,"{""location"": ""IT"", ""is_mobile"": false}" 17180,6,388,2017-01-21 08:11:40,http://veumfarrell.co/keanu_stoltenberg,,7.157.215.229,"{""location"": ""FO"", ""is_mobile"": false}" 17181,6,388,2017-06-01 15:17:05,http://wilkinson.com/gwendolyn.kub,,116.97.23.180,"{""location"": ""SH"", ""is_mobile"": true}" 17182,6,388,2017-05-28 21:54:59,http://kohlergrant.co/eldora,,70.246.158.56,"{""location"": ""CZ"", ""is_mobile"": true}" 17183,6,388,2017-01-11 20:37:34,http://bartonbayer.io/norwood,,230.157.31.242,"{""location"": ""LR"", ""is_mobile"": true}" 17184,6,388,2016-12-25 21:20:38,http://grant.info/jillian,,4.100.122.112,"{""location"": ""MU"", ""is_mobile"": false}" 17185,6,388,2017-03-31 04:31:57,http://kozey.info/brendan,,195.143.4.171,"{""location"": ""YT"", ""is_mobile"": true}" 17186,6,388,2017-06-13 15:53:55,http://windlerwisozk.org/fannie,,20.167.148.238,"{""location"": ""SG"", ""is_mobile"": false}" 17187,6,388,2017-05-22 11:15:16,http://schneider.org/fleta.fay,,215.66.81.64,"{""location"": ""SR"", ""is_mobile"": false}" 17188,6,388,2017-01-19 03:09:33,http://bruen.com/giles_bogisich,,133.155.104.209,"{""location"": ""CA"", ""is_mobile"": true}" 17189,6,388,2017-01-08 06:06:09,http://adams.org/leila,,64.212.157.32,"{""location"": ""BQ"", ""is_mobile"": true}" 17190,6,388,2017-04-02 08:28:44,http://mann.name/damion,,48.41.93.112,"{""location"": ""GS"", ""is_mobile"": true}" 17191,6,388,2016-12-22 19:24:12,http://ziemepfannerstill.name/ernie,,40.213.147.154,"{""location"": ""WS"", ""is_mobile"": false}" 17192,6,388,2017-04-17 06:51:25,http://johnsonhuels.biz/natalie,,206.25.216.127,"{""location"": ""RO"", ""is_mobile"": false}" 17193,6,388,2017-04-14 21:10:31,http://ebertkuhic.net/olen.kub,,85.118.148.64,"{""location"": ""HU"", ""is_mobile"": false}" 17194,6,388,2017-02-12 23:42:26,http://koepp.net/dedric,,141.106.123.91,"{""location"": ""HN"", ""is_mobile"": false}" 17195,6,388,2017-01-12 12:20:53,http://ferry.name/stanton,,203.164.117.205,"{""location"": ""TJ"", ""is_mobile"": true}" 17196,6,388,2016-12-23 16:36:51,http://bogan.name/jaren.rutherford,,7.57.169.183,"{""location"": ""AR"", ""is_mobile"": true}" 17197,6,388,2017-05-07 10:47:07,http://doyle.co/dusty,,101.208.88.137,"{""location"": ""MO"", ""is_mobile"": true}" 17198,6,388,2017-01-17 20:08:18,http://frami.org/chad,,159.59.11.159,"{""location"": ""KR"", ""is_mobile"": false}" 17199,6,388,2017-05-31 10:15:13,http://wuckert.name/verda,,48.247.225.234,"{""location"": ""FI"", ""is_mobile"": false}" 17200,6,388,2017-02-06 19:44:54,http://ko.co/linnea_ondricka,,112.68.137.185,"{""location"": ""TF"", ""is_mobile"": false}" 17201,6,388,2017-02-13 05:18:49,http://gusikowski.co/newell_gutkowski,,96.249.197.125,"{""location"": ""LT"", ""is_mobile"": false}" 17202,6,388,2017-05-08 12:55:32,http://monahanveum.org/devante,,200.210.195.87,"{""location"": ""GL"", ""is_mobile"": false}" 17203,6,388,2017-05-07 09:17:14,http://beahan.biz/loyal.rath,,83.59.121.89,"{""location"": ""HN"", ""is_mobile"": false}" 17204,6,388,2017-05-08 03:55:55,http://ondricka.info/erick_schinner,,153.180.46.170,"{""location"": ""BR"", ""is_mobile"": false}" 17205,6,388,2016-12-25 23:35:04,http://boylebarrows.co/laila_breitenberg,,158.253.91.52,"{""location"": ""WS"", ""is_mobile"": false}" 17206,6,388,2017-05-17 08:41:31,http://schambergerrunte.biz/terrance,,248.60.119.157,"{""location"": ""LI"", ""is_mobile"": true}" 17207,6,388,2017-02-08 19:48:39,http://rice.name/carolina.volkman,,242.83.234.137,"{""location"": ""NR"", ""is_mobile"": true}" 4334,2,95,2017-03-13 12:24:57,http://mcclure.com/lavinia,,188.15.151.158,"{""location"": ""AI"", ""is_mobile"": false}" 4335,2,95,2017-05-22 06:54:00,http://skiles.net/caleigh.yundt,,10.231.45.227,"{""location"": ""BE"", ""is_mobile"": false}" 4336,2,95,2017-01-27 21:37:31,http://jakubowskinikolaus.io/elizabeth,,30.143.153.198,"{""location"": ""BJ"", ""is_mobile"": false}" 4337,2,95,2016-12-20 14:02:25,http://langosh.name/kayla.barton,,227.119.215.232,"{""location"": ""AI"", ""is_mobile"": true}" 4338,2,95,2017-02-12 17:47:44,http://langworth.net/jacky,,171.58.14.242,"{""location"": ""IT"", ""is_mobile"": false}" 4339,2,95,2017-02-28 18:00:19,http://leffler.net/myrtie_thiel,,75.20.194.155,"{""location"": ""BA"", ""is_mobile"": false}" 4340,2,95,2017-05-09 14:40:05,http://flatley.net/marjorie,,202.122.137.54,"{""location"": ""AI"", ""is_mobile"": true}" 4341,2,95,2017-01-27 15:00:30,http://jacobi.org/adah.bode,,202.85.116.27,"{""location"": ""MS"", ""is_mobile"": false}" 4342,2,95,2017-06-05 12:04:28,http://beahanrippin.io/erna.little,,135.67.9.219,"{""location"": ""KZ"", ""is_mobile"": true}" 4343,2,95,2017-05-03 14:07:16,http://emardbayer.io/luciano_ryan,,28.245.62.48,"{""location"": ""IQ"", ""is_mobile"": true}" 4344,2,95,2017-05-16 13:14:37,http://kulas.info/lyric,,72.157.211.66,"{""location"": ""CX"", ""is_mobile"": false}" 4345,2,95,2017-01-07 03:50:10,http://gaylord.info/hettie,,57.223.47.76,"{""location"": ""SY"", ""is_mobile"": false}" 4346,2,95,2016-12-30 08:08:01,http://farrell.info/hertha.dach,,156.208.219.153,"{""location"": ""JP"", ""is_mobile"": false}" 4347,2,95,2017-01-16 22:57:24,http://gusikowski.io/braeden_veum,,182.231.217.62,"{""location"": ""SM"", ""is_mobile"": true}" 4348,2,95,2017-05-30 15:59:19,http://schuster.com/georgette,,248.127.77.252,"{""location"": ""SK"", ""is_mobile"": true}" 4349,2,95,2017-05-29 03:09:52,http://osinski.name/bettye,,199.140.96.155,"{""location"": ""SO"", ""is_mobile"": true}" 4350,2,95,2017-03-31 11:01:47,http://bernhard.com/brando,,249.244.221.59,"{""location"": ""HU"", ""is_mobile"": false}" 4351,2,95,2017-01-07 02:11:31,http://thompson.org/kaitlin.nitzsche,,23.68.147.217,"{""location"": ""RU"", ""is_mobile"": true}" 4352,2,95,2016-12-21 05:38:27,http://hegmann.biz/edythe.wolff,,61.113.222.150,"{""location"": ""BJ"", ""is_mobile"": true}" 4353,2,95,2017-04-28 03:53:11,http://dare.com/clemens.schinner,,102.33.154.227,"{""location"": ""LT"", ""is_mobile"": true}" 4354,2,95,2016-12-30 04:28:06,http://roobnader.org/roselyn,,112.190.254.210,"{""location"": ""MG"", ""is_mobile"": true}" 4355,2,95,2016-12-27 07:43:31,http://mayert.org/melia.koepp,,80.154.22.158,"{""location"": ""NF"", ""is_mobile"": true}" 4356,2,95,2017-02-24 02:28:49,http://purdy.net/claria,,41.43.133.48,"{""location"": ""ES"", ""is_mobile"": true}" 4357,2,95,2017-02-13 11:13:37,http://hoppewolff.io/merlin_green,,239.39.29.102,"{""location"": ""GB"", ""is_mobile"": true}" 4358,2,96,2016-12-22 10:34:51,http://kertzmann.co/erin_brown,,67.80.41.172,"{""location"": ""IN"", ""is_mobile"": true}" 4359,2,96,2017-01-18 17:29:30,http://schambergerzulauf.io/joanne.olson,,62.234.207.134,"{""location"": ""CD"", ""is_mobile"": false}" 4360,2,96,2017-03-26 09:59:00,http://becker.net/augustine_schuster,,197.115.29.10,"{""location"": ""EE"", ""is_mobile"": false}" 4361,2,96,2016-12-14 10:29:09,http://hills.name/reagan.boyer,,176.40.248.248,"{""location"": ""NI"", ""is_mobile"": true}" 4362,2,96,2017-05-12 19:53:43,http://hudson.net/glenna.feeney,,62.127.190.181,"{""location"": ""MT"", ""is_mobile"": false}" 4363,2,96,2017-04-18 10:50:46,http://welch.co/kamryn.hagenes,,69.135.24.150,"{""location"": ""FJ"", ""is_mobile"": true}" 4364,2,96,2017-03-03 07:33:20,http://bergstrom.info/colin,,90.191.183.94,"{""location"": ""BN"", ""is_mobile"": true}" 4365,2,96,2017-02-02 19:44:10,http://prohaska.biz/frank,,204.82.130.40,"{""location"": ""YE"", ""is_mobile"": false}" 4366,2,96,2017-03-04 12:49:26,http://schimmel.biz/gregorio_schuster,,44.85.106.95,"{""location"": ""VE"", ""is_mobile"": true}" 4367,2,96,2017-06-10 10:08:45,http://quitzon.co/luella,,254.156.115.206,"{""location"": ""ME"", ""is_mobile"": false}" 4368,2,96,2016-12-25 08:24:04,http://hilll.info/hipolito.hirthe,,224.68.32.139,"{""location"": ""CC"", ""is_mobile"": true}" 4369,2,96,2017-05-28 18:14:10,http://lowe.org/della,,129.216.7.214,"{""location"": ""DO"", ""is_mobile"": false}" 4370,2,96,2017-04-23 21:29:41,http://hand.co/buster,,134.249.7.66,"{""location"": ""SD"", ""is_mobile"": true}" 4371,2,96,2017-03-19 21:17:23,http://kaulkelang.biz/rhianna,,68.17.250.224,"{""location"": ""SX"", ""is_mobile"": false}" 4372,2,96,2017-05-27 09:52:49,http://ohara.com/henriette,,168.10.160.37,"{""location"": ""BW"", ""is_mobile"": false}" 4373,2,96,2017-03-07 04:08:48,http://jacobi.name/vida_hegmann,,106.70.64.125,"{""location"": ""HR"", ""is_mobile"": false}" 4374,2,96,2017-05-03 22:07:23,http://hegmann.info/charlene.dicki,,40.228.115.244,"{""location"": ""HT"", ""is_mobile"": true}" 4375,2,96,2017-01-15 03:02:21,http://konopelski.biz/domenica.jacobi,,206.184.148.87,"{""location"": ""CU"", ""is_mobile"": false}" 4376,2,96,2017-03-12 08:23:49,http://shanahandenesik.info/isaac.torp,,153.117.199.153,"{""location"": ""BG"", ""is_mobile"": false}" 4377,2,96,2017-04-02 03:13:44,http://lockmanprosacco.info/hipolito,,5.33.127.247,"{""location"": ""NG"", ""is_mobile"": false}" 4378,2,96,2016-12-25 11:20:34,http://littel.biz/garrick,,97.73.246.245,"{""location"": ""CK"", ""is_mobile"": false}" 4379,2,96,2017-02-03 09:26:17,http://bernhardmarks.info/jayde_hayes,,49.138.113.173,"{""location"": ""SR"", ""is_mobile"": false}" 4380,2,96,2017-01-04 16:59:32,http://klein.org/lura,,185.136.245.177,"{""location"": ""PW"", ""is_mobile"": false}" 4381,2,96,2016-12-25 08:55:29,http://mcculloughward.name/ally,,160.230.47.140,"{""location"": ""VI"", ""is_mobile"": true}" 4382,2,96,2017-02-17 18:43:00,http://adams.biz/osbaldo,,242.72.205.242,"{""location"": ""PN"", ""is_mobile"": false}" 4383,2,96,2017-01-21 02:05:30,http://schillercrooks.co/jana,,153.87.186.129,"{""location"": ""AS"", ""is_mobile"": false}" 4384,2,96,2017-06-09 22:23:03,http://quigleyullrich.co/savannah,,236.200.59.163,"{""location"": ""GS"", ""is_mobile"": false}" 4385,2,96,2017-05-13 09:07:10,http://price.org/helena,,79.196.131.229,"{""location"": ""AO"", ""is_mobile"": true}" 4386,2,96,2017-05-01 08:36:17,http://cummings.co/uriel,,233.90.127.232,"{""location"": ""BF"", ""is_mobile"": false}" 4387,2,96,2017-05-29 23:46:46,http://senger.io/augustine_hermiston,,4.137.83.96,"{""location"": ""JO"", ""is_mobile"": true}" 4388,2,96,2017-01-01 04:45:00,http://ward.co/velva_mertz,,40.223.243.94,"{""location"": ""BT"", ""is_mobile"": true}" 4389,2,96,2017-05-26 04:27:52,http://jacobi.biz/lenny,,94.182.43.254,"{""location"": ""GW"", ""is_mobile"": false}" 11273,4,252,2017-05-02 16:24:54,http://wiza.co/rubie.trantow,,30.26.166.32,"{""location"": ""GB"", ""is_mobile"": false}" 11274,4,252,2017-01-13 00:56:14,http://bernhard.co/neal,,65.247.10.210,"{""location"": ""CV"", ""is_mobile"": true}" 11275,4,252,2017-04-07 04:51:30,http://bechtelarlang.info/spencer_armstrong,,4.53.227.121,"{""location"": ""AG"", ""is_mobile"": true}" 11276,4,252,2017-02-01 05:17:02,http://schamberger.net/bette_klein,,58.132.193.203,"{""location"": ""SY"", ""is_mobile"": true}" 11277,4,252,2017-02-10 08:30:55,http://lockman.co/vinnie,,100.223.201.231,"{""location"": ""PN"", ""is_mobile"": true}" 11278,4,252,2017-02-28 02:49:46,http://joneshuels.org/lydia_nitzsche,,4.2.174.154,"{""location"": ""KY"", ""is_mobile"": true}" 11279,4,252,2017-05-20 23:05:41,http://oconnelllockman.name/mallory,,182.41.51.214,"{""location"": ""MS"", ""is_mobile"": false}" 11280,4,252,2016-12-16 05:42:26,http://kunzewelch.io/priscilla.eichmann,,24.215.230.97,"{""location"": ""ZM"", ""is_mobile"": false}" 11281,4,252,2017-04-26 07:48:02,http://kutch.name/davin,,72.35.93.128,"{""location"": ""SS"", ""is_mobile"": true}" 11282,4,252,2017-06-09 05:21:19,http://little.name/eino,,30.34.68.137,"{""location"": ""BD"", ""is_mobile"": false}" 11283,4,252,2017-01-09 09:09:39,http://haley.com/alf,,91.38.14.250,"{""location"": ""KP"", ""is_mobile"": true}" 11284,4,252,2017-01-22 03:15:47,http://koepp.com/guie,,51.119.125.130,"{""location"": ""HT"", ""is_mobile"": true}" 11285,4,252,2017-01-11 05:44:12,http://hickle.net/amie,,181.114.209.145,"{""location"": ""SX"", ""is_mobile"": true}" 11286,4,252,2016-12-27 10:38:18,http://stracke.net/joe.mayert,,50.100.13.131,"{""location"": ""RU"", ""is_mobile"": false}" 11287,4,252,2017-01-14 17:11:18,http://hand.co/jaida_lowe,,134.144.126.115,"{""location"": ""GB"", ""is_mobile"": false}" 11288,4,252,2017-03-16 15:08:33,http://turner.org/nicholaus.rice,,194.83.251.27,"{""location"": ""RO"", ""is_mobile"": true}" 11289,4,252,2017-05-03 05:48:52,http://schinnerwyman.info/adelia.hilpert,,232.172.88.183,"{""location"": ""AW"", ""is_mobile"": true}" 11290,4,252,2017-02-07 23:28:14,http://reichert.net/dion.green,,104.75.41.44,"{""location"": ""NL"", ""is_mobile"": true}" 11291,4,252,2017-03-01 21:04:36,http://casper.co/zack,,133.207.81.168,"{""location"": ""AR"", ""is_mobile"": false}" 11292,4,252,2017-02-11 10:23:06,http://rogahncummings.co/jayme,,131.131.246.149,"{""location"": ""SX"", ""is_mobile"": false}" 11293,4,252,2017-04-03 14:52:00,http://whitewalter.com/idella_wiegand,,160.85.228.180,"{""location"": ""FJ"", ""is_mobile"": false}" 11294,4,253,2017-02-20 04:18:16,http://hansen.io/malachi.kohler,,99.221.63.125,"{""location"": ""CU"", ""is_mobile"": false}" 11295,4,253,2017-05-10 22:07:34,http://herman.biz/norma_nader,,236.64.28.182,"{""location"": ""GD"", ""is_mobile"": true}" 11296,4,253,2017-03-19 10:06:52,http://buckridge.name/spencer.stehr,,107.196.159.238,"{""location"": ""MQ"", ""is_mobile"": true}" 11297,4,253,2017-02-02 12:03:29,http://reinger.net/camren_jacobi,,159.189.189.224,"{""location"": ""AM"", ""is_mobile"": false}" 11298,4,253,2017-03-09 15:08:59,http://labadie.com/hortense,,168.214.108.227,"{""location"": ""MR"", ""is_mobile"": true}" 11299,4,253,2017-02-18 10:07:49,http://okon.co/pablo,,214.180.219.174,"{""location"": ""VA"", ""is_mobile"": true}" 11300,4,253,2017-01-09 06:46:20,http://harris.org/sibyl,,229.94.138.135,"{""location"": ""GA"", ""is_mobile"": false}" 11301,4,253,2017-01-10 18:02:41,http://bogangislason.biz/daniela.hoppe,,116.33.180.124,"{""location"": ""AS"", ""is_mobile"": true}" 11302,4,253,2017-04-19 13:01:18,http://wisokygrady.io/ashtyn.hills,,183.107.238.57,"{""location"": ""PH"", ""is_mobile"": true}" 11303,4,253,2017-04-02 08:26:41,http://dibbert.org/jewel.mcglynn,,103.2.253.102,"{""location"": ""MU"", ""is_mobile"": false}" 11304,4,253,2017-05-15 19:43:13,http://moriette.org/reilly,,168.198.22.227,"{""location"": ""CG"", ""is_mobile"": false}" 11305,4,253,2017-03-11 08:50:07,http://crooks.biz/carley.larson,,233.159.15.180,"{""location"": ""AE"", ""is_mobile"": true}" 11306,4,253,2017-01-12 04:12:44,http://hudson.co/clay.medhurst,,178.50.242.217,"{""location"": ""MS"", ""is_mobile"": false}" 11307,4,253,2016-12-23 17:40:11,http://gleichnerthiel.net/kenyon_white,,47.191.112.217,"{""location"": ""IO"", ""is_mobile"": true}" 11308,4,253,2017-01-15 22:24:55,http://jerde.net/grover.larkin,,26.220.132.238,"{""location"": ""TJ"", ""is_mobile"": true}" 11309,4,253,2017-01-31 02:57:23,http://kutch.name/kim,,168.105.97.211,"{""location"": ""BI"", ""is_mobile"": true}" 11310,4,253,2017-06-12 21:12:23,http://shanahan.biz/willow,,102.170.123.161,"{""location"": ""CF"", ""is_mobile"": true}" 11311,4,253,2017-05-11 04:09:08,http://dickinsonauer.biz/idella.schultz,,187.106.181.158,"{""location"": ""ZM"", ""is_mobile"": false}" 11312,4,253,2016-12-17 07:12:41,http://schimmel.name/lauretta_schaefer,,71.220.64.197,"{""location"": ""MP"", ""is_mobile"": true}" 11313,4,253,2017-05-24 12:30:17,http://adamswaters.org/rogelio.dibbert,,234.26.209.20,"{""location"": ""SB"", ""is_mobile"": true}" 11314,4,253,2016-12-24 17:36:35,http://baumbach.io/cesar,,168.63.194.245,"{""location"": ""MV"", ""is_mobile"": true}" 11315,4,253,2017-05-30 15:04:15,http://connellyklein.info/gia,,126.237.42.10,"{""location"": ""SJ"", ""is_mobile"": false}" 11316,4,253,2017-04-21 08:28:59,http://wyman.io/rhianna,,104.50.195.195,"{""location"": ""MY"", ""is_mobile"": false}" 11317,4,253,2017-05-04 20:00:34,http://glover.com/tom_thompson,,182.164.137.103,"{""location"": ""SZ"", ""is_mobile"": true}" 11318,4,253,2017-06-02 12:34:16,http://mrazmraz.co/mohammed,,80.252.84.29,"{""location"": ""UM"", ""is_mobile"": true}" 11319,4,253,2017-01-16 06:10:24,http://torphy.com/bethel,,120.158.91.146,"{""location"": ""CM"", ""is_mobile"": false}" 11320,4,253,2017-06-09 06:59:26,http://leannon.info/reuben.paucek,,52.28.170.44,"{""location"": ""MY"", ""is_mobile"": false}" 11321,4,253,2017-03-16 13:27:09,http://raynor.net/clifton,,137.146.143.247,"{""location"": ""TK"", ""is_mobile"": false}" 11322,4,253,2017-01-03 02:50:06,http://grant.info/dewayne.hettinger,,115.169.216.212,"{""location"": ""GU"", ""is_mobile"": false}" 11323,4,253,2017-04-10 19:15:44,http://ferryfeil.co/charity.bahringer,,110.230.67.8,"{""location"": ""LK"", ""is_mobile"": false}" 11324,4,253,2017-04-14 15:04:33,http://moenschimmel.name/jaylen,,52.97.135.152,"{""location"": ""SJ"", ""is_mobile"": true}" 11325,4,253,2017-04-26 15:15:06,http://medhurst.com/veronica,,153.111.63.73,"{""location"": ""NL"", ""is_mobile"": false}" 11326,4,253,2016-12-13 10:00:25,http://gleichner.biz/diamond,,210.129.210.37,"{""location"": ""JP"", ""is_mobile"": true}" 11327,4,253,2017-01-27 15:42:40,http://okuneva.com/jewell,,199.171.170.236,"{""location"": ""HR"", ""is_mobile"": false}" 8274,3,184,2017-03-28 04:16:16,http://strosinbosco.co/sigurd.schmidt,0.9783223706,173.74.172.140,"{""location"": ""HK"", ""is_mobile"": false}" 8275,3,184,2017-05-11 18:06:22,http://wisozk.org/christian,0.8513824783,129.78.116.56,"{""location"": ""BQ"", ""is_mobile"": false}" 8276,3,184,2017-03-06 14:20:07,http://mcclure.net/clinton,0.7003146206,18.66.14.131,"{""location"": ""CA"", ""is_mobile"": true}" 8277,3,184,2017-01-24 05:00:26,http://grantbins.name/calista,0.1162441637,143.253.122.24,"{""location"": ""HK"", ""is_mobile"": true}" 8278,3,184,2016-12-15 10:09:04,http://wunsch.net/carmela,0.8223773205,37.92.143.50,"{""location"": ""MV"", ""is_mobile"": true}" 8279,3,184,2017-01-27 15:30:44,http://olson.com/mckenzie,0.4572296631,98.11.135.113,"{""location"": ""PA"", ""is_mobile"": true}" 8280,3,184,2016-12-29 02:08:01,http://boehm.io/alana,0.5554985820,160.39.120.163,"{""location"": ""KR"", ""is_mobile"": false}" 8281,3,184,2017-01-03 20:35:56,http://howell.info/helen,0.8016626880,162.129.219.232,"{""location"": ""NZ"", ""is_mobile"": false}" 8282,3,184,2017-02-12 13:02:34,http://marks.net/marietta.boyer,0.8864731253,129.162.167.192,"{""location"": ""PR"", ""is_mobile"": false}" 8283,3,184,2017-05-17 20:43:42,http://hueltreutel.name/jordane,0.9022734266,186.26.104.176,"{""location"": ""KR"", ""is_mobile"": false}" 8284,3,184,2017-01-17 00:45:33,http://turcottewindler.name/kraig,0.8405767351,47.39.10.66,"{""location"": ""GN"", ""is_mobile"": false}" 8285,3,184,2017-02-01 20:00:41,http://schuster.com/tiffany,0.2172149412,37.117.147.51,"{""location"": ""NR"", ""is_mobile"": true}" 8286,3,184,2017-03-25 09:29:05,http://ondricka.biz/cleo,0.7610401102,210.89.86.100,"{""location"": ""DJ"", ""is_mobile"": true}" 8287,3,184,2017-03-12 03:22:46,http://franecki.biz/alicia,0.6309844983,181.52.93.82,"{""location"": ""MD"", ""is_mobile"": false}" 8288,3,184,2017-05-24 08:23:24,http://ullrich.biz/frederick_leannon,0.5364717343,110.65.54.36,"{""location"": ""NP"", ""is_mobile"": false}" 8289,3,184,2017-05-23 17:13:36,http://yostmuller.io/terrell,0.8929810937,149.227.167.125,"{""location"": ""AO"", ""is_mobile"": true}" 8290,3,184,2017-02-07 03:32:43,http://witting.name/wava,0.7995073475,233.211.234.171,"{""location"": ""FI"", ""is_mobile"": true}" 8291,3,184,2017-01-28 07:14:55,http://feil.co/letitia.glover,0.1504495792,78.251.191.132,"{""location"": ""SG"", ""is_mobile"": false}" 8292,3,184,2017-01-27 04:01:52,http://schultzyundt.net/roger,0.3945907817,34.107.96.121,"{""location"": ""AD"", ""is_mobile"": false}" 8293,3,184,2017-01-10 11:31:28,http://yostreichel.net/larry,0.5579391575,74.116.111.42,"{""location"": ""SN"", ""is_mobile"": true}" 8294,3,184,2017-01-13 10:44:55,http://boyerschowalter.info/fernando,0.2683045385,131.197.162.51,"{""location"": ""TK"", ""is_mobile"": false}" 8295,3,184,2016-12-14 03:11:10,http://reynolds.name/dylan,0.3459579897,184.220.57.161,"{""location"": ""VC"", ""is_mobile"": false}" 8296,3,184,2017-05-27 11:18:15,http://pollich.co/brandi_torphy,0.8818688379,115.173.127.133,"{""location"": ""TM"", ""is_mobile"": false}" 8297,3,185,2016-12-18 04:20:11,http://kundereynolds.info/nels,,22.175.25.12,"{""location"": ""PK"", ""is_mobile"": false}" 8298,3,185,2017-05-10 07:35:17,http://parker.io/marc,,97.77.181.83,"{""location"": ""ML"", ""is_mobile"": true}" 8299,3,185,2017-06-12 20:27:33,http://rau.biz/jamal_hamill,,210.206.60.164,"{""location"": ""CK"", ""is_mobile"": false}" 8300,3,185,2017-03-05 20:47:45,http://lang.org/robb,,250.211.62.46,"{""location"": ""GG"", ""is_mobile"": false}" 8301,3,185,2017-05-18 03:59:02,http://hackett.net/jada,,91.64.164.95,"{""location"": ""GS"", ""is_mobile"": false}" 8302,3,185,2017-04-30 21:37:45,http://roberts.com/kirk.murphy,,26.66.118.123,"{""location"": ""AE"", ""is_mobile"": true}" 8303,3,185,2017-03-02 20:56:10,http://leannonkoepp.net/telly,,190.17.100.184,"{""location"": ""DJ"", ""is_mobile"": true}" 8304,3,185,2016-12-15 16:24:38,http://hilll.biz/verna,,18.176.153.136,"{""location"": ""ZA"", ""is_mobile"": true}" 8305,3,185,2017-05-31 06:18:14,http://lynch.name/janet,,250.58.157.181,"{""location"": ""BS"", ""is_mobile"": true}" 8306,3,185,2017-06-03 23:05:28,http://fadel.biz/eulalia_donnelly,,131.130.13.125,"{""location"": ""SL"", ""is_mobile"": false}" 8307,3,185,2017-04-08 12:08:11,http://luettgen.info/agnes,,10.11.157.199,"{""location"": ""PL"", ""is_mobile"": false}" 8308,3,185,2017-04-16 01:26:08,http://mosciskiterry.info/enos.leuschke,,145.164.118.90,"{""location"": ""BG"", ""is_mobile"": false}" 8309,3,185,2017-01-07 07:41:09,http://fritsch.org/isabelle,,9.87.149.152,"{""location"": ""GG"", ""is_mobile"": true}" 8310,3,185,2017-02-05 12:54:25,http://weberdenesik.io/conner.glover,,222.103.45.70,"{""location"": ""BI"", ""is_mobile"": false}" 8311,3,185,2017-04-19 01:09:59,http://legros.com/ismael.heaney,,204.161.83.125,"{""location"": ""MY"", ""is_mobile"": false}" 8312,3,185,2017-04-13 23:56:31,http://vandervort.co/emmet_terry,,183.98.60.131,"{""location"": ""AF"", ""is_mobile"": true}" 8313,3,185,2017-06-09 20:50:15,http://schambergerweber.org/milo_heathcote,,111.218.142.174,"{""location"": ""CZ"", ""is_mobile"": true}" 8314,3,185,2017-05-25 04:03:52,http://parisiangulgowski.com/rollin,,74.152.115.192,"{""location"": ""KR"", ""is_mobile"": true}" 8315,3,185,2016-12-17 12:59:33,http://vongottlieb.co/vilma.stoltenberg,,191.165.179.231,"{""location"": ""CC"", ""is_mobile"": false}" 8316,3,185,2016-12-30 22:11:16,http://buckridge.io/marlon.renner,,157.102.20.222,"{""location"": ""HU"", ""is_mobile"": false}" 8317,3,185,2017-04-13 23:52:57,http://cronin.info/kaelyn,,244.253.239.111,"{""location"": ""TW"", ""is_mobile"": true}" 8318,3,185,2017-06-12 22:28:24,http://hamillgleichner.io/marlene,,194.199.204.225,"{""location"": ""TT"", ""is_mobile"": false}" 8319,3,185,2017-03-27 16:19:09,http://okeefe.co/emmie.ratke,,29.206.201.162,"{""location"": ""EG"", ""is_mobile"": true}" 8320,3,185,2017-02-06 04:01:10,http://monahanblanda.info/nedra,,219.22.205.115,"{""location"": ""FI"", ""is_mobile"": false}" 8321,3,185,2017-02-13 17:04:18,http://kiehn.name/chance,,155.43.123.238,"{""location"": ""KI"", ""is_mobile"": false}" 8322,3,185,2017-03-29 20:15:03,http://abshire.info/beulah.oreilly,,52.18.167.24,"{""location"": ""KZ"", ""is_mobile"": true}" 8323,3,185,2017-05-30 05:16:01,http://erdman.com/dolores,,215.89.210.165,"{""location"": ""UM"", ""is_mobile"": true}" 8324,3,185,2017-01-07 19:57:22,http://brown.org/manuela,,190.36.116.76,"{""location"": ""ER"", ""is_mobile"": false}" 8325,3,185,2017-01-10 22:02:39,http://schmidtschumm.co/veronica_purdy,,74.35.36.100,"{""location"": ""TL"", ""is_mobile"": false}" 8326,3,185,2017-05-09 19:18:51,http://romaguera.io/maria.barton,,166.41.212.144,"{""location"": ""IN"", ""is_mobile"": true}" 8327,3,185,2017-01-03 15:51:29,http://murphy.info/maximo,,163.86.189.17,"{""location"": ""TN"", ""is_mobile"": false}" 14223,5,320,2016-12-14 20:51:39,http://ziemann.info/erika.smitham,,125.37.235.183,"{""location"": ""PM"", ""is_mobile"": false}" 14224,5,320,2017-03-20 02:16:18,http://lueilwitz.name/trycia,,74.186.165.92,"{""location"": ""AU"", ""is_mobile"": false}" 14225,5,320,2017-03-05 13:00:49,http://fahey.com/danny_bernier,,76.109.178.202,"{""location"": ""CK"", ""is_mobile"": false}" 14226,5,320,2017-06-10 01:37:29,http://erdmandibbert.com/darrel,,222.212.251.56,"{""location"": ""TN"", ""is_mobile"": true}" 14227,5,320,2017-05-22 09:41:43,http://brownfritsch.net/chanelle,,200.156.216.27,"{""location"": ""GP"", ""is_mobile"": false}" 14228,5,320,2017-06-08 13:15:57,http://ullrich.name/katherine,,254.47.241.33,"{""location"": ""BH"", ""is_mobile"": false}" 14229,5,320,2017-06-06 19:16:26,http://adams.name/onie_zieme,,241.175.93.146,"{""location"": ""PN"", ""is_mobile"": true}" 14230,5,320,2017-05-11 16:00:25,http://cole.org/emma_mann,,188.193.204.194,"{""location"": ""BN"", ""is_mobile"": true}" 14231,5,320,2017-03-28 20:32:21,http://ritchieklein.co/yasmeen,,240.6.157.11,"{""location"": ""VU"", ""is_mobile"": true}" 14232,5,320,2017-01-26 07:40:53,http://goodwinwalsh.info/conor,,47.241.221.45,"{""location"": ""BQ"", ""is_mobile"": false}" 14233,5,320,2017-03-12 11:16:35,http://bogisich.biz/yvonne.bruen,,45.156.218.212,"{""location"": ""TG"", ""is_mobile"": false}" 14234,5,320,2017-05-08 19:35:03,http://shields.net/miller,,139.2.233.120,"{""location"": ""AD"", ""is_mobile"": false}" 14235,5,320,2017-02-19 21:37:08,http://wunsch.info/ruby,,35.56.243.154,"{""location"": ""BO"", ""is_mobile"": false}" 14236,5,321,2017-01-14 05:12:53,http://lakin.biz/verner,,232.52.185.51,"{""location"": ""BV"", ""is_mobile"": false}" 14237,5,321,2017-03-16 18:51:19,http://konopelski.com/alejandrin,,114.129.214.182,"{""location"": ""SJ"", ""is_mobile"": false}" 14238,5,321,2017-02-25 22:37:22,http://hackett.biz/dangelo,,198.153.77.70,"{""location"": ""AO"", ""is_mobile"": true}" 14239,5,321,2017-05-21 17:15:50,http://bernier.co/guie_schamberger,,62.28.19.166,"{""location"": ""SB"", ""is_mobile"": false}" 14240,5,321,2017-02-14 13:57:32,http://macgyver.name/kristina.zulauf,,122.54.23.159,"{""location"": ""PN"", ""is_mobile"": false}" 14241,5,321,2017-02-25 10:20:14,http://hyatt.org/adan,,46.235.208.116,"{""location"": ""BN"", ""is_mobile"": true}" 14242,5,321,2017-02-18 22:55:13,http://koepp.co/ike.strosin,,8.239.191.231,"{""location"": ""MN"", ""is_mobile"": true}" 14243,5,321,2017-03-15 17:08:48,http://mannwelch.info/remington,,19.24.224.219,"{""location"": ""CU"", ""is_mobile"": true}" 14244,5,321,2017-01-05 00:53:22,http://stehr.info/florence,,49.117.11.130,"{""location"": ""TZ"", ""is_mobile"": false}" 14245,5,321,2017-05-13 05:27:41,http://nader.name/diana_greenfelder,,105.156.91.206,"{""location"": ""IN"", ""is_mobile"": true}" 14246,5,321,2017-03-20 15:46:25,http://zemlak.com/euna,,252.252.194.26,"{""location"": ""CK"", ""is_mobile"": true}" 14247,5,321,2017-06-07 21:36:56,http://hartmann.info/aleen.okon,,35.26.140.33,"{""location"": ""SV"", ""is_mobile"": true}" 14248,5,321,2016-12-31 01:18:46,http://torphycartwright.org/roscoe.mueller,,165.173.145.241,"{""location"": ""AQ"", ""is_mobile"": false}" 14249,5,321,2017-04-25 16:40:54,http://pollich.io/ford,,168.36.64.58,"{""location"": ""PG"", ""is_mobile"": true}" 14250,5,321,2017-04-06 04:24:14,http://auer.biz/lukas,,64.92.15.147,"{""location"": ""ST"", ""is_mobile"": false}" 14251,5,321,2016-12-22 18:44:09,http://mrazoconnell.io/hermina.harber,,199.242.57.177,"{""location"": ""PR"", ""is_mobile"": true}" 14252,5,321,2017-02-19 05:16:27,http://legrosmoore.name/shea.prohaska,,146.144.207.183,"{""location"": ""PM"", ""is_mobile"": true}" 14253,5,321,2017-04-24 02:19:57,http://bechtelar.info/angelo,,67.201.32.144,"{""location"": ""EC"", ""is_mobile"": false}" 14254,5,321,2017-04-23 06:49:20,http://jakubowski.net/michaela,,7.18.184.55,"{""location"": ""KP"", ""is_mobile"": false}" 14255,5,321,2017-03-28 11:07:07,http://zboncak.info/reyes.gleason,,91.4.195.129,"{""location"": ""MZ"", ""is_mobile"": true}" 14256,5,321,2017-05-31 09:08:40,http://robel.org/grant,,233.227.82.105,"{""location"": ""NU"", ""is_mobile"": false}" 14257,5,321,2017-02-14 00:57:43,http://funkconnelly.biz/yvonne_wintheiser,,232.208.101.17,"{""location"": ""NF"", ""is_mobile"": true}" 14258,5,321,2016-12-22 17:04:54,http://schamberger.name/micaela,,40.48.209.46,"{""location"": ""PG"", ""is_mobile"": true}" 14259,5,321,2017-01-02 16:29:07,http://dach.net/braeden,,39.241.152.126,"{""location"": ""DE"", ""is_mobile"": true}" 14260,5,321,2017-01-09 18:22:19,http://schamberger.name/alisa.hoeger,,250.198.2.11,"{""location"": ""LV"", ""is_mobile"": false}" 14261,5,321,2017-05-09 08:37:52,http://baileygaylord.org/emely_kuhn,,186.138.191.42,"{""location"": ""MG"", ""is_mobile"": true}" 14262,5,321,2017-02-12 09:45:55,http://koelpincummings.co/ozzie,,222.97.86.124,"{""location"": ""MS"", ""is_mobile"": true}" 14263,5,321,2017-04-15 03:23:09,http://hodkiewiczryan.org/berneice,,9.232.181.129,"{""location"": ""BJ"", ""is_mobile"": true}" 14264,5,321,2017-03-23 10:32:14,http://hudson.biz/talon.haag,,251.164.204.26,"{""location"": ""LB"", ""is_mobile"": true}" 14265,5,321,2017-01-31 13:14:28,http://schmitt.name/madge.lehner,,231.114.143.237,"{""location"": ""CG"", ""is_mobile"": true}" 14266,5,321,2017-03-12 02:52:18,http://ohara.co/giovanny.leannon,,127.132.60.16,"{""location"": ""BH"", ""is_mobile"": true}" 14267,5,321,2017-05-06 05:07:00,http://bashirian.info/cielo,,169.150.235.18,"{""location"": ""AD"", ""is_mobile"": false}" 14268,5,321,2017-02-09 22:28:38,http://oreilly.name/fredy,,189.84.227.38,"{""location"": ""GI"", ""is_mobile"": false}" 14269,5,321,2017-01-25 12:01:47,http://goodwinconsidine.info/terry,,64.179.247.199,"{""location"": ""AF"", ""is_mobile"": true}" 14270,5,321,2017-03-20 08:21:48,http://orn.com/rhianna_lueilwitz,,207.136.251.196,"{""location"": ""CD"", ""is_mobile"": true}" 14271,5,321,2017-02-12 06:34:14,http://bechtelar.com/khalil,,191.24.184.80,"{""location"": ""ST"", ""is_mobile"": false}" 14272,5,321,2017-03-14 08:23:20,http://jenkinszieme.biz/araceli.stehr,,248.85.99.30,"{""location"": ""FI"", ""is_mobile"": false}" 14273,5,321,2017-01-20 14:45:05,http://windler.co/hillary,,145.161.229.146,"{""location"": ""MS"", ""is_mobile"": false}" 14274,5,321,2017-01-08 06:48:58,http://ko.com/bryce.zboncak,,186.248.116.139,"{""location"": ""RW"", ""is_mobile"": true}" 14275,5,321,2017-05-07 12:27:52,http://ko.name/kristin,,213.150.126.164,"{""location"": ""FM"", ""is_mobile"": true}" 14276,5,321,2017-06-08 12:31:59,http://cartwrightledner.com/karson,,86.198.83.117,"{""location"": ""TW"", ""is_mobile"": true}" 14277,5,321,2017-02-06 06:08:08,http://barrows.com/joanny.kohler,,184.7.93.138,"{""location"": ""GW"", ""is_mobile"": false}" 17208,6,388,2017-01-13 07:13:56,http://franeckischneider.com/deion_schmitt,,123.41.96.47,"{""location"": ""EG"", ""is_mobile"": true}" 17209,6,388,2017-02-07 20:28:34,http://mante.info/myron,,134.138.99.19,"{""location"": ""GI"", ""is_mobile"": false}" 17210,6,388,2016-12-16 22:12:45,http://vandervortdaugherty.com/gavin,,90.95.82.158,"{""location"": ""MH"", ""is_mobile"": false}" 17211,6,388,2017-02-19 06:24:45,http://rodriguezbraun.com/royce.torphy,,253.138.27.113,"{""location"": ""HN"", ""is_mobile"": true}" 17212,6,388,2017-06-08 10:29:38,http://windlerwelch.name/gianni,,80.52.129.26,"{""location"": ""GA"", ""is_mobile"": true}" 17213,6,388,2017-03-09 22:46:02,http://abshirelittle.info/winnifred,,84.13.211.172,"{""location"": ""TJ"", ""is_mobile"": true}" 17214,6,388,2017-04-12 20:55:13,http://hintzauer.io/clifford.beier,,201.54.170.67,"{""location"": ""PL"", ""is_mobile"": false}" 17215,6,388,2017-04-09 07:01:44,http://mohr.io/rylee,,3.61.13.200,"{""location"": ""HU"", ""is_mobile"": false}" 17216,6,388,2017-01-23 23:29:11,http://deckow.com/heaven,,61.80.72.20,"{""location"": ""BE"", ""is_mobile"": false}" 17217,6,389,2017-02-18 16:17:07,http://hirtheheel.io/lourdes.davis,,90.112.63.64,"{""location"": ""BO"", ""is_mobile"": true}" 17218,6,389,2017-02-27 09:50:40,http://yost.net/jonas,,103.5.57.146,"{""location"": ""GE"", ""is_mobile"": false}" 17219,6,389,2016-12-20 12:20:40,http://hauckklocko.biz/roger,,44.253.31.169,"{""location"": ""SD"", ""is_mobile"": true}" 17220,6,389,2017-04-29 23:46:51,http://altenwerth.name/tre_greenfelder,,100.77.84.115,"{""location"": ""CF"", ""is_mobile"": true}" 17221,6,389,2017-01-31 04:05:54,http://lowegutkowski.com/drew_kling,,85.61.23.167,"{""location"": ""PK"", ""is_mobile"": true}" 17222,6,389,2017-06-10 06:40:00,http://huel.co/malachi_greenholt,,237.176.40.15,"{""location"": ""PA"", ""is_mobile"": false}" 17223,6,389,2017-05-24 09:33:49,http://volkmanlangworth.info/franz,,81.66.66.55,"{""location"": ""DO"", ""is_mobile"": true}" 17224,6,389,2017-06-07 23:06:28,http://ernservon.info/jaiden_wehner,,72.47.34.139,"{""location"": ""UY"", ""is_mobile"": true}" 17225,6,389,2016-12-16 15:51:36,http://runolfsdottirtrantow.name/milo_bayer,,193.66.244.182,"{""location"": ""MU"", ""is_mobile"": false}" 17226,6,389,2017-04-05 21:09:58,http://moriette.io/hans,,135.167.165.9,"{""location"": ""AF"", ""is_mobile"": false}" 17227,6,389,2017-05-11 04:50:54,http://cainmarvin.io/randi,,209.115.222.136,"{""location"": ""BQ"", ""is_mobile"": true}" 17228,6,389,2017-04-21 10:19:59,http://dietrich.org/kyler,,180.3.137.244,"{""location"": ""BF"", ""is_mobile"": true}" 17229,6,389,2017-06-08 22:57:25,http://torphycormier.com/beryl,,78.76.239.136,"{""location"": ""CX"", ""is_mobile"": true}" 17230,6,389,2017-03-28 17:20:50,http://erdmanmcglynn.com/adolf,,61.229.121.156,"{""location"": ""LY"", ""is_mobile"": false}" 17231,6,389,2017-01-14 21:39:35,http://gleichnercummings.co/theresia_heel,,126.156.192.156,"{""location"": ""PT"", ""is_mobile"": false}" 17232,6,389,2017-02-13 23:01:49,http://hoppetorp.name/eloy_johnson,,124.144.206.168,"{""location"": ""BF"", ""is_mobile"": false}" 17233,6,389,2017-01-07 11:32:58,http://connellyoconner.net/mozelle,,78.44.136.52,"{""location"": ""KG"", ""is_mobile"": true}" 17234,6,389,2017-01-22 07:19:04,http://stamm.name/claude,,239.33.67.57,"{""location"": ""CR"", ""is_mobile"": true}" 17235,6,389,2017-04-23 09:56:05,http://champlinkuhic.io/burdette,,68.64.174.147,"{""location"": ""BL"", ""is_mobile"": true}" 17236,6,389,2017-02-28 20:54:09,http://blandaschiller.co/deanna,,146.26.6.246,"{""location"": ""ET"", ""is_mobile"": true}" 17237,6,389,2017-05-05 23:17:23,http://zboncakzboncak.biz/britney.beatty,,38.178.148.21,"{""location"": ""HK"", ""is_mobile"": true}" 17238,6,389,2017-02-04 22:46:45,http://dare.io/marisol.thiel,,32.19.201.5,"{""location"": ""NI"", ""is_mobile"": false}" 17239,6,389,2017-04-07 19:09:54,http://uptonmoen.biz/eli,,41.216.41.180,"{""location"": ""SC"", ""is_mobile"": false}" 17240,6,389,2017-04-26 16:15:17,http://corwin.com/rhianna,,204.80.172.175,"{""location"": ""WF"", ""is_mobile"": true}" 17241,6,389,2017-05-13 01:24:23,http://schmelerquitzon.org/kenton_hansen,,86.19.189.127,"{""location"": ""SD"", ""is_mobile"": true}" 17242,6,389,2017-01-17 06:36:58,http://champlin.info/cedrick_nicolas,,36.54.53.21,"{""location"": ""ES"", ""is_mobile"": true}" 17243,6,389,2017-01-10 19:54:48,http://smitham.io/kim.grant,,136.64.234.72,"{""location"": ""BQ"", ""is_mobile"": false}" 17244,6,389,2017-06-01 12:52:29,http://jacobi.net/heaven_brown,,115.86.65.6,"{""location"": ""ER"", ""is_mobile"": true}" 17245,6,389,2016-12-18 19:31:59,http://hudson.biz/jose.erdman,,63.156.140.204,"{""location"": ""KM"", ""is_mobile"": true}" 17246,6,389,2017-06-02 19:58:23,http://johns.co/hermina,,212.136.140.66,"{""location"": ""PH"", ""is_mobile"": false}" 17247,6,389,2017-04-25 08:29:06,http://marvinbahringer.info/mikel,,241.124.152.100,"{""location"": ""KM"", ""is_mobile"": false}" 17248,6,389,2017-05-26 13:20:29,http://rolfson.co/zola.moore,,216.195.175.40,"{""location"": ""CV"", ""is_mobile"": true}" 17249,6,389,2017-03-12 15:59:04,http://hintzkuhn.name/kyra.mosciski,,60.220.130.18,"{""location"": ""FR"", ""is_mobile"": false}" 17250,6,389,2017-03-29 15:28:30,http://oberbrunner.io/lafayette,,150.114.203.101,"{""location"": ""LB"", ""is_mobile"": true}" 17251,6,389,2017-02-18 19:56:23,http://hermann.name/josiane,,155.109.40.248,"{""location"": ""SY"", ""is_mobile"": false}" 17252,6,389,2016-12-18 21:06:40,http://okonhahn.io/aubrey,,17.14.132.246,"{""location"": ""ZW"", ""is_mobile"": false}" 17253,6,389,2017-05-09 11:17:21,http://green.info/seamus_legros,,221.65.251.55,"{""location"": ""GF"", ""is_mobile"": true}" 17254,6,389,2017-04-11 05:24:41,http://maggiocole.io/bryon,,13.119.89.87,"{""location"": ""BQ"", ""is_mobile"": true}" 17255,6,389,2017-01-23 20:53:51,http://rathtorphy.io/stephanie,,243.206.73.24,"{""location"": ""UM"", ""is_mobile"": true}" 17256,6,389,2016-12-27 08:30:15,http://blanda.org/stuart_oberbrunner,,204.230.190.124,"{""location"": ""LC"", ""is_mobile"": false}" 17257,6,389,2017-01-15 16:06:03,http://ohara.info/zelda_larson,,68.15.74.182,"{""location"": ""VI"", ""is_mobile"": false}" 17258,6,389,2017-04-24 08:46:07,http://bernhard.biz/bethany,,224.58.21.210,"{""location"": ""BY"", ""is_mobile"": false}" 17259,6,389,2017-05-19 13:11:56,http://pfannerstillkuphal.org/trycia.shanahan,,91.152.196.142,"{""location"": ""PA"", ""is_mobile"": false}" 17260,6,389,2017-05-22 07:33:45,http://reichert.io/laria,,149.43.151.172,"{""location"": ""CD"", ""is_mobile"": false}" 17261,6,389,2017-02-25 03:55:48,http://langokuneva.org/fernando_bruen,,163.128.243.103,"{""location"": ""VN"", ""is_mobile"": true}" 17262,6,389,2017-03-29 16:24:02,http://aufderhar.org/dylan,,248.148.153.124,"{""location"": ""NO"", ""is_mobile"": false}" 4390,2,96,2017-01-03 23:54:55,http://runolfsdottir.co/lamont.crooks,,154.73.250.103,"{""location"": ""SV"", ""is_mobile"": false}" 4391,2,96,2017-02-13 08:03:09,http://lebsack.org/magnus.wilderman,,168.62.221.49,"{""location"": ""CY"", ""is_mobile"": true}" 4392,2,96,2017-02-07 22:31:59,http://damore.net/camden.nikolaus,,96.247.237.123,"{""location"": ""SI"", ""is_mobile"": false}" 4393,2,97,2017-04-05 02:31:20,http://jones.org/garfield,,132.37.71.23,"{""location"": ""GS"", ""is_mobile"": true}" 4394,2,97,2017-05-14 20:55:19,http://senger.co/ava.doyle,,110.47.113.11,"{""location"": ""CI"", ""is_mobile"": true}" 4395,2,97,2017-04-03 01:44:03,http://thompson.org/destany_damore,,65.154.37.233,"{""location"": ""AS"", ""is_mobile"": false}" 4396,2,97,2017-02-22 17:07:49,http://littel.name/thea.schmeler,,27.75.199.186,"{""location"": ""VE"", ""is_mobile"": true}" 4397,2,97,2016-12-18 04:30:57,http://schmidt.co/madison.lueilwitz,,191.239.105.110,"{""location"": ""TZ"", ""is_mobile"": false}" 4398,2,97,2017-04-17 18:47:56,http://gerhold.biz/muriel,,236.14.16.221,"{""location"": ""PW"", ""is_mobile"": true}" 4399,2,97,2016-12-25 13:07:24,http://daviskautzer.io/eric,,54.172.240.47,"{""location"": ""SK"", ""is_mobile"": true}" 4400,2,97,2017-01-16 06:35:53,http://christiansen.io/torey,,20.92.83.154,"{""location"": ""PN"", ""is_mobile"": false}" 4401,2,97,2017-01-06 04:43:15,http://cremin.net/fidel,,245.135.169.130,"{""location"": ""KP"", ""is_mobile"": true}" 4402,2,97,2016-12-30 06:44:33,http://reinger.co/casimir_roberts,,115.41.254.130,"{""location"": ""LK"", ""is_mobile"": false}" 4403,2,97,2017-02-06 23:11:04,http://cruickshank.biz/aurelia,,143.251.189.254,"{""location"": ""MS"", ""is_mobile"": true}" 4404,2,97,2017-05-14 20:57:24,http://dibbert.net/reba_wolf,,240.37.130.74,"{""location"": ""JE"", ""is_mobile"": true}" 4405,2,97,2017-01-14 17:29:48,http://hilll.co/amari.stroman,,185.90.196.249,"{""location"": ""TJ"", ""is_mobile"": true}" 4406,2,97,2017-04-29 07:53:27,http://rolfson.net/amy_mosciski,,241.166.100.191,"{""location"": ""TG"", ""is_mobile"": false}" 4407,2,97,2017-05-17 01:02:50,http://wizamurazik.com/elta,,95.86.164.50,"{""location"": ""AQ"", ""is_mobile"": true}" 4408,2,97,2017-04-27 16:30:55,http://beahan.net/fanny,,142.196.182.27,"{""location"": ""AR"", ""is_mobile"": false}" 4409,2,97,2017-03-16 18:19:04,http://konopelskibernier.io/caroline,,95.157.203.178,"{""location"": ""ZA"", ""is_mobile"": true}" 4410,2,97,2017-02-03 20:30:48,http://predovic.co/magdalen,,241.224.45.177,"{""location"": ""PH"", ""is_mobile"": true}" 4411,2,97,2017-02-19 14:09:23,http://sipes.co/delbert_parker,,185.81.106.202,"{""location"": ""SS"", ""is_mobile"": true}" 4412,2,97,2017-05-26 03:47:37,http://krislesch.biz/keshawn.weber,,187.108.169.156,"{""location"": ""VG"", ""is_mobile"": false}" 4413,2,97,2017-03-15 19:22:55,http://eichmann.name/vito,,53.209.238.56,"{""location"": ""AL"", ""is_mobile"": false}" 4414,2,97,2016-12-23 22:21:44,http://pollichboyer.org/eudora,,65.188.48.174,"{""location"": ""BJ"", ""is_mobile"": false}" 4415,2,97,2017-02-01 00:41:15,http://mann.org/priscilla.jacobson,,201.11.217.75,"{""location"": ""BD"", ""is_mobile"": true}" 4416,2,97,2016-12-23 11:34:34,http://batz.com/erica,,188.53.17.73,"{""location"": ""GM"", ""is_mobile"": true}" 4417,2,97,2017-01-03 08:46:02,http://wymanschamberger.io/kaci,,184.196.117.146,"{""location"": ""KG"", ""is_mobile"": true}" 4418,2,97,2017-01-17 06:33:24,http://dicki.co/mabel.prosacco,,203.108.26.25,"{""location"": ""KR"", ""is_mobile"": true}" 4419,2,97,2016-12-19 01:35:09,http://walsh.info/thelma,,177.217.153.65,"{""location"": ""TL"", ""is_mobile"": true}" 4420,2,97,2017-02-19 22:43:49,http://block.biz/bailee,,125.243.47.12,"{""location"": ""CH"", ""is_mobile"": true}" 4421,2,97,2017-05-16 13:07:10,http://weimann.biz/bailey.christiansen,,54.249.140.233,"{""location"": ""IM"", ""is_mobile"": false}" 4422,2,97,2017-02-10 21:29:31,http://ankunding.co/mandy.wolf,,146.245.167.175,"{""location"": ""HR"", ""is_mobile"": false}" 4423,2,97,2017-01-16 00:51:41,http://heidenreich.com/pete,,171.22.49.206,"{""location"": ""ET"", ""is_mobile"": false}" 4424,2,97,2017-06-11 09:52:54,http://gerholdrunte.io/jordon,,75.110.246.58,"{""location"": ""MM"", ""is_mobile"": true}" 4425,2,97,2016-12-29 21:32:05,http://emard.co/ora_bednar,,221.240.159.254,"{""location"": ""CX"", ""is_mobile"": true}" 4426,2,97,2017-04-08 02:23:00,http://stammmann.com/elta_heathcote,,61.84.65.96,"{""location"": ""ID"", ""is_mobile"": false}" 4427,2,97,2017-01-07 12:53:42,http://lakinward.io/albina.halvorson,,17.173.24.225,"{""location"": ""CV"", ""is_mobile"": false}" 4428,2,97,2017-05-29 20:05:06,http://feil.org/hanna,,69.241.116.50,"{""location"": ""GP"", ""is_mobile"": false}" 4429,2,97,2017-06-13 22:35:30,http://kutch.co/libby,,53.74.47.162,"{""location"": ""BQ"", ""is_mobile"": false}" 4430,2,97,2017-05-24 13:12:29,http://runolfsdottirpaucek.info/augustine.hodkiewicz,,114.110.242.4,"{""location"": ""GA"", ""is_mobile"": true}" 4431,2,97,2016-12-15 18:09:28,http://boylehowell.net/perry.klein,,70.78.110.215,"{""location"": ""CM"", ""is_mobile"": false}" 4432,2,97,2017-03-20 12:10:14,http://stantonfisher.co/antoinette,,219.137.239.25,"{""location"": ""MT"", ""is_mobile"": false}" 4433,2,97,2017-04-27 21:49:16,http://mcglynn.name/ian.armstrong,,191.133.67.66,"{""location"": ""IN"", ""is_mobile"": true}" 4434,2,97,2017-03-13 22:59:54,http://botsfordpacocha.info/isabelle,,202.106.126.45,"{""location"": ""IL"", ""is_mobile"": false}" 4435,2,97,2017-03-30 00:02:34,http://west.org/jakob.feeney,,221.226.174.43,"{""location"": ""IT"", ""is_mobile"": true}" 4436,2,97,2017-04-29 19:19:22,http://smithnitzsche.co/watson_hayes,,38.143.142.168,"{""location"": ""EH"", ""is_mobile"": false}" 4437,2,97,2016-12-24 15:24:12,http://roobbreitenberg.name/aletha_monahan,,40.133.86.250,"{""location"": ""GY"", ""is_mobile"": true}" 4438,2,97,2017-02-19 08:57:30,http://welch.org/terrill,,6.155.122.152,"{""location"": ""UY"", ""is_mobile"": false}" 4439,2,97,2017-05-22 23:45:59,http://schinnermayer.info/florian_bechtelar,,192.96.220.157,"{""location"": ""LT"", ""is_mobile"": true}" 4440,2,97,2017-01-18 05:11:05,http://baumbachwatsica.io/boyd,,158.216.129.10,"{""location"": ""TW"", ""is_mobile"": false}" 4441,2,97,2017-05-15 06:42:21,http://boylelehner.io/roma_bahringer,,56.71.106.202,"{""location"": ""BF"", ""is_mobile"": true}" 4442,2,97,2016-12-28 13:33:51,http://barton.io/daniella,,84.5.239.162,"{""location"": ""HN"", ""is_mobile"": true}" 4443,2,97,2017-02-10 02:10:06,http://jenkins.biz/jon,,207.137.12.212,"{""location"": ""MQ"", ""is_mobile"": false}" 4444,2,97,2017-02-23 07:52:40,http://stiedemann.info/alek,,11.155.36.17,"{""location"": ""RS"", ""is_mobile"": false}" 4445,2,97,2017-04-02 19:40:58,http://murphy.net/reyes_daniel,,43.155.163.72,"{""location"": ""NR"", ""is_mobile"": false}" 11328,4,253,2017-04-08 13:43:01,http://stanton.net/monserrate,,42.10.16.57,"{""location"": ""KR"", ""is_mobile"": true}" 11329,4,253,2017-02-20 15:12:01,http://batz.net/kristina,,228.94.152.223,"{""location"": ""MA"", ""is_mobile"": true}" 11330,4,253,2017-01-16 18:27:51,http://lehner.info/peggie_ankunding,,251.211.2.231,"{""location"": ""CX"", ""is_mobile"": false}" 11331,4,253,2017-04-09 15:43:58,http://baumbachshanahan.co/stefan,,199.139.168.146,"{""location"": ""LI"", ""is_mobile"": false}" 11332,4,253,2017-01-28 10:40:50,http://flatleylockman.io/kiana_sauer,,162.194.149.160,"{""location"": ""GU"", ""is_mobile"": false}" 11333,4,253,2017-03-21 07:43:23,http://gottliebstiedemann.name/salvatore_bradtke,,56.89.16.105,"{""location"": ""PM"", ""is_mobile"": true}" 11334,4,253,2017-06-09 21:52:53,http://langoshheidenreich.name/santina.bartoletti,,116.4.90.22,"{""location"": ""KZ"", ""is_mobile"": false}" 11335,4,253,2017-02-11 04:19:36,http://douglas.com/steve_oconnell,,237.171.190.32,"{""location"": ""DK"", ""is_mobile"": false}" 11336,4,253,2017-01-06 09:39:16,http://bosco.com/jeie_gerhold,,82.70.123.38,"{""location"": ""NO"", ""is_mobile"": false}" 11337,4,253,2017-01-16 07:57:31,http://streich.info/jaylan.mcdermott,,122.61.6.222,"{""location"": ""IT"", ""is_mobile"": false}" 11338,4,254,2017-03-04 01:54:39,http://crist.net/leilani.abshire,,140.243.78.44,"{""location"": ""CF"", ""is_mobile"": false}" 11339,4,254,2017-01-21 03:37:06,http://wehner.biz/fredrick_kovacek,,211.156.197.91,"{""location"": ""IN"", ""is_mobile"": true}" 11340,4,254,2017-03-18 14:57:29,http://stroman.biz/antonio,,81.103.100.12,"{""location"": ""CK"", ""is_mobile"": true}" 11341,4,254,2017-05-14 14:43:47,http://auerkeebler.net/thalia_rodriguez,,61.151.22.168,"{""location"": ""MF"", ""is_mobile"": false}" 11342,4,254,2017-01-21 18:50:39,http://weinat.com/emie,,214.20.45.185,"{""location"": ""JO"", ""is_mobile"": true}" 11343,4,254,2017-06-10 09:02:08,http://klein.com/alvina_bahringer,,196.133.169.117,"{""location"": ""NE"", ""is_mobile"": false}" 11344,4,254,2017-04-22 14:58:36,http://lefflerhackett.org/shanon,,62.185.137.230,"{""location"": ""AO"", ""is_mobile"": true}" 11345,4,254,2017-03-19 14:31:16,http://gaylordpfeffer.info/jeffery_walker,,253.23.213.126,"{""location"": ""JE"", ""is_mobile"": true}" 11346,4,254,2017-02-28 01:53:12,http://keler.io/torrey,,34.70.46.133,"{""location"": ""US"", ""is_mobile"": true}" 11347,4,254,2016-12-26 05:16:57,http://stoltenberg.co/philip_kaulke,,67.178.60.163,"{""location"": ""MS"", ""is_mobile"": true}" 11348,4,254,2017-01-20 21:07:08,http://walsh.io/darren,,101.161.146.14,"{""location"": ""NU"", ""is_mobile"": true}" 11349,4,254,2017-02-28 04:38:44,http://kohler.info/frances_schumm,,53.164.174.19,"{""location"": ""ES"", ""is_mobile"": false}" 11350,4,254,2016-12-31 06:25:56,http://waters.info/gabrielle,,166.148.239.163,"{""location"": ""YE"", ""is_mobile"": false}" 11351,4,254,2017-02-27 17:21:56,http://gradyrice.io/eleanora_buckridge,,228.166.158.72,"{""location"": ""OM"", ""is_mobile"": false}" 11352,4,254,2017-04-19 09:22:49,http://jakubowski.org/brigitte.pfannerstill,,222.240.221.114,"{""location"": ""BJ"", ""is_mobile"": true}" 11353,4,254,2016-12-15 13:07:20,http://blick.info/dion,,61.120.165.221,"{""location"": ""TM"", ""is_mobile"": true}" 11354,4,254,2017-04-11 20:05:26,http://stoltenbergschulist.com/loyal_hintz,,88.53.213.133,"{""location"": ""ME"", ""is_mobile"": true}" 11355,4,254,2017-02-03 15:16:53,http://herman.org/mackenzie_gusikowski,,24.189.208.7,"{""location"": ""AS"", ""is_mobile"": true}" 11356,4,254,2017-05-30 18:08:36,http://rowe.name/jackeline.bins,,179.99.69.183,"{""location"": ""TG"", ""is_mobile"": false}" 11357,4,254,2017-02-22 05:04:33,http://bayer.com/mireille,,198.75.177.79,"{""location"": ""MO"", ""is_mobile"": false}" 11358,4,254,2017-05-23 13:09:21,http://morar.co/alyon,,10.202.121.60,"{""location"": ""FI"", ""is_mobile"": false}" 11359,4,254,2017-01-17 22:47:49,http://langworthyost.org/hilton_beahan,,228.111.52.225,"{""location"": ""RS"", ""is_mobile"": false}" 11360,4,254,2017-06-08 00:34:37,http://smithamskiles.com/janice,,104.181.127.13,"{""location"": ""PL"", ""is_mobile"": true}" 11361,4,254,2016-12-31 21:20:38,http://nienowhalvorson.biz/meredith.jones,,148.163.218.52,"{""location"": ""BG"", ""is_mobile"": false}" 11362,4,254,2017-05-14 21:59:13,http://schaefer.biz/marilie_bashirian,,189.236.20.116,"{""location"": ""CY"", ""is_mobile"": true}" 11363,4,254,2017-05-25 22:54:17,http://schuppe.name/arnulfo,,196.194.7.249,"{""location"": ""ER"", ""is_mobile"": true}" 11364,4,254,2017-03-14 06:45:55,http://swaniawski.info/rose,,42.78.104.226,"{""location"": ""IS"", ""is_mobile"": true}" 11365,4,254,2017-05-06 08:10:55,http://collins.net/josh.feeney,,143.7.103.65,"{""location"": ""VU"", ""is_mobile"": true}" 11366,4,254,2017-05-07 04:08:14,http://leffler.com/hillard.kuhic,,79.59.234.149,"{""location"": ""NE"", ""is_mobile"": false}" 11367,4,254,2017-02-13 01:38:27,http://legros.io/jeanie,,27.249.167.206,"{""location"": ""KG"", ""is_mobile"": true}" 11368,4,254,2017-01-08 23:07:45,http://torp.name/karson,,22.169.33.161,"{""location"": ""AU"", ""is_mobile"": false}" 11369,4,254,2017-05-17 19:44:49,http://waters.com/oma_dicki,,130.226.55.130,"{""location"": ""TV"", ""is_mobile"": false}" 11370,4,254,2017-01-23 17:59:31,http://beier.net/marques,,144.90.93.120,"{""location"": ""IE"", ""is_mobile"": false}" 11371,4,254,2017-05-21 06:28:39,http://erdman.io/bernard,,150.228.242.167,"{""location"": ""GR"", ""is_mobile"": true}" 11372,4,254,2017-02-17 00:36:39,http://dietrich.org/leatha,,65.218.221.65,"{""location"": ""PW"", ""is_mobile"": false}" 11373,4,254,2017-06-04 17:05:40,http://bruen.org/alberto,,11.128.248.198,"{""location"": ""LY"", ""is_mobile"": true}" 11374,4,254,2017-02-28 04:05:48,http://harvey.name/danial_pouros,,202.241.160.249,"{""location"": ""MX"", ""is_mobile"": false}" 11375,4,254,2017-02-01 14:12:27,http://franecki.biz/marisol_west,,160.46.65.33,"{""location"": ""CG"", ""is_mobile"": true}" 11376,4,254,2017-01-29 10:17:31,http://spinka.co/yasmine,,50.214.117.41,"{""location"": ""BT"", ""is_mobile"": true}" 11377,4,254,2017-05-24 09:01:36,http://wintheiser.org/randal,,57.94.145.253,"{""location"": ""BQ"", ""is_mobile"": false}" 11378,4,254,2017-05-20 07:17:53,http://windler.co/jaleel,,173.210.94.87,"{""location"": ""CM"", ""is_mobile"": true}" 11379,4,254,2017-03-20 09:23:13,http://batz.co/monserrate,,112.169.137.247,"{""location"": ""SV"", ""is_mobile"": false}" 11380,4,254,2017-02-02 19:26:39,http://champlinbeer.name/burnice.kuphal,,86.72.94.239,"{""location"": ""MO"", ""is_mobile"": false}" 11381,4,254,2017-02-12 12:31:43,http://greenfelderdonnelly.org/jeika.simonis,,73.45.52.36,"{""location"": ""CV"", ""is_mobile"": false}" 11382,4,254,2017-05-09 19:44:45,http://kochankunding.info/chelsea,,34.132.92.229,"{""location"": ""LS"", ""is_mobile"": true}" 8328,3,185,2017-03-11 10:28:49,http://stoltenberg.org/carole_nader,,13.84.179.104,"{""location"": ""GW"", ""is_mobile"": false}" 8329,3,185,2017-05-27 06:46:30,http://runolfon.com/jamil,,80.18.142.10,"{""location"": ""EH"", ""is_mobile"": true}" 8330,3,185,2016-12-27 07:03:25,http://walker.info/lacey.balistreri,,246.126.70.102,"{""location"": ""MY"", ""is_mobile"": false}" 8331,3,185,2017-05-30 08:34:06,http://halvorson.org/ahmed_reilly,,159.233.36.218,"{""location"": ""AX"", ""is_mobile"": true}" 8332,3,185,2017-01-07 21:59:36,http://vandervort.info/shane,,232.179.232.81,"{""location"": ""SA"", ""is_mobile"": true}" 8333,3,185,2017-02-24 08:51:17,http://hoppe.info/mitchel.harber,,223.112.89.6,"{""location"": ""BL"", ""is_mobile"": true}" 8334,3,185,2017-04-02 03:58:10,http://ratke.io/annabell,,232.241.32.73,"{""location"": ""GI"", ""is_mobile"": false}" 8335,3,185,2017-02-27 19:21:13,http://hirthe.net/erna.schmitt,,210.251.44.245,"{""location"": ""TG"", ""is_mobile"": false}" 8336,3,185,2017-05-08 03:07:16,http://altenwerthdooley.io/glen,,142.244.145.5,"{""location"": ""TT"", ""is_mobile"": true}" 8337,3,185,2017-05-02 05:41:37,http://mcglynn.info/zena_powlowski,,204.66.140.187,"{""location"": ""AM"", ""is_mobile"": true}" 8338,3,185,2017-06-08 07:34:34,http://gerlach.name/imogene,,176.218.219.222,"{""location"": ""ML"", ""is_mobile"": true}" 8339,3,186,2017-04-06 02:22:00,http://kundeyost.co/samson.roob,,194.230.130.110,"{""location"": ""TZ"", ""is_mobile"": false}" 8340,3,186,2017-05-25 03:44:57,http://mclaughlin.com/kenyon_witting,,59.13.216.153,"{""location"": ""SY"", ""is_mobile"": false}" 8341,3,186,2017-03-23 11:13:15,http://robelfeest.io/mauricio,,23.28.74.68,"{""location"": ""MF"", ""is_mobile"": true}" 8342,3,186,2017-04-16 21:21:06,http://bosco.info/maia_wisoky,,202.87.236.216,"{""location"": ""TT"", ""is_mobile"": true}" 8343,3,186,2017-03-05 23:34:01,http://trantow.com/erling.will,,65.127.230.4,"{""location"": ""LB"", ""is_mobile"": true}" 8344,3,186,2017-02-13 07:29:28,http://blanda.com/rhea.strosin,,27.145.207.77,"{""location"": ""VG"", ""is_mobile"": false}" 8345,3,186,2017-05-05 12:46:00,http://mueller.biz/gerald,,114.35.37.106,"{""location"": ""MG"", ""is_mobile"": true}" 8346,3,186,2017-02-05 23:46:59,http://berge.biz/nina.connelly,,166.7.47.139,"{""location"": ""MQ"", ""is_mobile"": true}" 8347,3,186,2017-06-12 03:16:36,http://dibbert.org/collin_corwin,,32.83.12.46,"{""location"": ""GB"", ""is_mobile"": true}" 8348,3,186,2017-01-21 14:29:48,http://veum.io/candida,,212.37.64.146,"{""location"": ""GE"", ""is_mobile"": false}" 8349,3,186,2017-04-09 07:34:46,http://mueller.co/lucienne.koepp,,25.40.112.109,"{""location"": ""LS"", ""is_mobile"": false}" 8350,3,186,2016-12-23 17:31:59,http://gislasonhilpert.net/allan.wisoky,,205.245.6.172,"{""location"": ""TL"", ""is_mobile"": false}" 8351,3,186,2017-04-14 20:10:01,http://schmidtlindgren.co/ian,,22.194.150.159,"{""location"": ""KI"", ""is_mobile"": true}" 8352,3,186,2017-05-09 04:09:14,http://boyle.co/delaney_damore,,36.71.184.54,"{""location"": ""BF"", ""is_mobile"": true}" 8353,3,186,2017-02-12 16:44:40,http://herzoghaley.io/rickie.strosin,,76.205.53.182,"{""location"": ""CL"", ""is_mobile"": false}" 8354,3,186,2017-04-30 21:31:23,http://lueilwitz.biz/bettie.grimes,,254.99.118.243,"{""location"": ""DJ"", ""is_mobile"": true}" 8355,3,186,2017-01-20 20:28:04,http://auer.name/gianni,,150.113.48.52,"{""location"": ""MM"", ""is_mobile"": true}" 8356,3,186,2017-04-04 03:15:05,http://ko.io/christopher,,41.138.177.6,"{""location"": ""TZ"", ""is_mobile"": true}" 8357,3,186,2017-04-29 21:28:09,http://lubowitz.co/ellen,,67.128.157.145,"{""location"": ""SL"", ""is_mobile"": true}" 8358,3,186,2017-02-27 23:31:36,http://lakin.co/reese,,166.130.180.198,"{""location"": ""EE"", ""is_mobile"": true}" 8359,3,186,2017-05-12 16:55:23,http://collins.co/thad.trantow,,141.171.126.232,"{""location"": ""SN"", ""is_mobile"": true}" 8360,3,186,2017-04-10 01:36:43,http://koch.io/ulises,,183.104.81.108,"{""location"": ""DJ"", ""is_mobile"": true}" 8361,3,186,2017-04-18 14:37:56,http://doyle.info/arnaldo.ledner,,46.7.117.248,"{""location"": ""ZM"", ""is_mobile"": true}" 8362,3,186,2017-02-18 03:31:34,http://brownkovacek.io/joanie_koelpin,,55.79.11.203,"{""location"": ""IM"", ""is_mobile"": false}" 8363,3,187,2017-04-24 06:21:25,http://mccullough.co/marco.herman,,51.17.147.119,"{""location"": ""HK"", ""is_mobile"": true}" 8364,3,187,2017-03-30 14:38:14,http://mayer.net/dudley_vonrueden,,162.183.229.39,"{""location"": ""SD"", ""is_mobile"": true}" 8365,3,187,2017-05-25 15:00:40,http://johnston.name/georgianna,,60.235.147.157,"{""location"": ""KI"", ""is_mobile"": true}" 8366,3,187,2017-04-20 00:35:12,http://kutchohara.co/hanna_hagenes,,160.24.3.21,"{""location"": ""TM"", ""is_mobile"": false}" 8367,3,187,2017-03-02 08:42:28,http://stoltenbergkoepp.biz/callie_hauck,,51.185.20.156,"{""location"": ""TT"", ""is_mobile"": true}" 8368,3,187,2017-05-07 13:25:30,http://leschdoyle.name/elza,,105.5.113.175,"{""location"": ""CK"", ""is_mobile"": true}" 8369,3,187,2017-05-22 15:28:37,http://lueilwitzschowalter.info/dejah.swaniawski,,45.62.136.200,"{""location"": ""VG"", ""is_mobile"": true}" 8370,3,187,2017-06-01 17:18:19,http://wintheiser.info/leonie_ruel,,54.187.136.187,"{""location"": ""CR"", ""is_mobile"": false}" 8371,3,187,2017-04-02 14:08:20,http://schmidt.biz/angelita,,185.60.55.133,"{""location"": ""NR"", ""is_mobile"": false}" 8372,3,187,2017-02-07 06:11:35,http://macgyver.org/casey,,41.86.35.201,"{""location"": ""DJ"", ""is_mobile"": true}" 8373,3,187,2017-02-08 08:45:21,http://collins.co/lizeth.beatty,,190.26.163.225,"{""location"": ""CF"", ""is_mobile"": true}" 8374,3,187,2017-01-21 10:01:14,http://gleichner.io/rahul,,54.252.109.9,"{""location"": ""MH"", ""is_mobile"": true}" 8375,3,187,2017-05-31 05:47:39,http://dibberthowe.net/janet,,191.23.213.108,"{""location"": ""TD"", ""is_mobile"": false}" 8376,3,187,2017-05-02 13:12:37,http://schneider.name/blair,,106.106.245.99,"{""location"": ""MO"", ""is_mobile"": false}" 8377,3,187,2017-04-25 04:45:05,http://fisher.com/betsy,,88.42.133.61,"{""location"": ""KZ"", ""is_mobile"": true}" 8378,3,187,2016-12-25 20:52:41,http://toy.io/fanny.wunsch,,61.157.4.81,"{""location"": ""IE"", ""is_mobile"": true}" 8379,3,187,2016-12-14 08:40:30,http://beereffertz.co/meggie_koelpin,,155.129.154.106,"{""location"": ""BY"", ""is_mobile"": true}" 8380,3,187,2017-05-03 23:37:35,http://nikolaus.co/brianne_cormier,,104.229.205.223,"{""location"": ""CR"", ""is_mobile"": false}" 8381,3,187,2017-02-23 15:53:38,http://leschfunk.net/concepcion.ortiz,,41.238.67.41,"{""location"": ""DM"", ""is_mobile"": true}" 8382,3,187,2017-03-27 16:30:26,http://fritsch.org/milan,,21.132.243.146,"{""location"": ""QA"", ""is_mobile"": false}" 8383,3,187,2017-01-21 21:30:53,http://legros.biz/darlene.frami,,71.185.67.72,"{""location"": ""CK"", ""is_mobile"": true}" 14278,5,321,2017-01-28 10:45:08,http://terryarmstrong.com/vesta,,224.117.140.36,"{""location"": ""CR"", ""is_mobile"": true}" 14279,5,321,2017-06-05 18:57:20,http://okuneva.info/leanne,,210.252.235.239,"{""location"": ""GB"", ""is_mobile"": true}" 14280,5,321,2017-03-21 09:47:19,http://ruel.io/eloisa,,13.149.117.156,"{""location"": ""CO"", ""is_mobile"": true}" 14281,5,321,2017-02-27 17:01:16,http://raynor.net/kathryn,,34.26.159.210,"{""location"": ""SB"", ""is_mobile"": false}" 14282,5,321,2016-12-22 20:53:40,http://krajcik.org/paolo,,163.165.123.133,"{""location"": ""PL"", ""is_mobile"": true}" 14283,5,321,2017-02-13 13:51:52,http://wisozk.net/etha,,17.229.176.163,"{""location"": ""CX"", ""is_mobile"": false}" 14284,5,321,2017-04-16 12:39:28,http://ondricka.co/gregg.gerlach,,3.39.22.165,"{""location"": ""EE"", ""is_mobile"": true}" 14285,5,321,2017-01-30 06:23:34,http://dibbertnikolaus.info/nick_murphy,,138.251.125.164,"{""location"": ""FR"", ""is_mobile"": true}" 14286,5,321,2017-01-24 20:23:49,http://bernhard.io/shirley,,172.111.201.94,"{""location"": ""AM"", ""is_mobile"": false}" 14287,5,321,2017-05-04 11:57:35,http://mills.com/patience,,15.180.208.75,"{""location"": ""SY"", ""is_mobile"": false}" 14288,5,321,2017-03-22 17:20:00,http://walshmraz.org/alicia,,251.51.84.46,"{""location"": ""HK"", ""is_mobile"": true}" 14289,5,321,2017-02-23 22:52:01,http://connborer.com/haskell_gerhold,,216.253.7.201,"{""location"": ""AW"", ""is_mobile"": true}" 14290,5,321,2017-06-03 01:35:25,http://reichert.biz/alyce.dickens,,77.196.21.240,"{""location"": ""FK"", ""is_mobile"": false}" 14291,5,321,2017-06-10 10:41:55,http://hauckcruickshank.com/johnathon.bode,,123.240.22.204,"{""location"": ""ER"", ""is_mobile"": true}" 14292,5,321,2017-02-20 22:08:22,http://gutmann.org/santos.wintheiser,,241.2.169.7,"{""location"": ""DK"", ""is_mobile"": true}" 14293,5,321,2017-04-12 22:06:12,http://legros.net/chanel_pacocha,,156.221.114.226,"{""location"": ""PG"", ""is_mobile"": false}" 14294,5,321,2017-02-08 06:48:31,http://adams.biz/christian,,243.245.230.251,"{""location"": ""MN"", ""is_mobile"": false}" 14295,5,321,2017-03-27 19:56:04,http://morietteschaefer.org/felix.walker,,131.223.80.205,"{""location"": ""NC"", ""is_mobile"": false}" 14296,5,321,2017-05-19 00:08:50,http://kuhn.co/monique_labadie,,200.101.87.228,"{""location"": ""BH"", ""is_mobile"": false}" 14297,5,321,2016-12-19 17:07:25,http://macejkovichegmann.info/emily_mills,,148.90.189.30,"{""location"": ""BD"", ""is_mobile"": true}" 14298,5,321,2017-02-21 18:46:45,http://schimmel.co/orion_bernier,,150.249.158.178,"{""location"": ""AW"", ""is_mobile"": false}" 14299,5,321,2017-02-11 03:12:44,http://auer.co/taylor.dietrich,,63.232.19.185,"{""location"": ""EH"", ""is_mobile"": true}" 14300,5,321,2017-01-26 09:35:40,http://krajcikchristiansen.co/dovie,,183.32.71.174,"{""location"": ""NC"", ""is_mobile"": false}" 14301,5,321,2016-12-30 19:08:18,http://wunsch.co/kathryn_hettinger,,89.210.13.6,"{""location"": ""PW"", ""is_mobile"": true}" 14302,5,322,2017-04-05 10:24:48,http://hammes.org/delia.yundt,,67.162.63.150,"{""location"": ""MW"", ""is_mobile"": true}" 14303,5,322,2017-01-12 01:55:32,http://bechtelargulgowski.info/ayana.smith,,13.158.241.193,"{""location"": ""IL"", ""is_mobile"": false}" 14304,5,322,2017-02-08 21:38:01,http://dickens.net/ryleigh.osinski,,44.253.171.79,"{""location"": ""UZ"", ""is_mobile"": true}" 14305,5,322,2017-06-11 04:00:07,http://gerhold.info/matilde.nolan,,15.164.73.144,"{""location"": ""PW"", ""is_mobile"": true}" 14306,5,322,2017-04-19 18:25:52,http://sipesabbott.name/ottilie,,197.171.26.224,"{""location"": ""IR"", ""is_mobile"": true}" 14307,5,322,2017-01-27 10:20:24,http://leuschke.com/melvina.hudson,,182.179.163.10,"{""location"": ""AT"", ""is_mobile"": true}" 14308,5,322,2017-06-05 09:17:17,http://heidenreich.org/furman_schowalter,,99.17.19.78,"{""location"": ""SJ"", ""is_mobile"": true}" 14309,5,322,2017-05-08 07:24:13,http://lehner.org/rosalyn.farrell,,222.16.64.88,"{""location"": ""HM"", ""is_mobile"": false}" 14310,5,322,2017-05-06 11:19:55,http://brekkebuckridge.net/elisa_keeling,,156.169.141.251,"{""location"": ""CG"", ""is_mobile"": false}" 14311,5,322,2017-03-21 09:11:23,http://mosciski.biz/americo_durgan,,140.106.125.156,"{""location"": ""GL"", ""is_mobile"": false}" 14312,5,322,2017-02-19 23:34:19,http://johnsonpowlowski.org/cristobal.halvorson,,175.237.62.234,"{""location"": ""IM"", ""is_mobile"": true}" 14313,5,322,2017-06-10 02:08:31,http://gerlach.net/sophia.reilly,,93.244.100.103,"{""location"": ""BI"", ""is_mobile"": false}" 14314,5,322,2017-02-09 07:34:30,http://ledner.name/carlie.schowalter,,198.45.13.75,"{""location"": ""JM"", ""is_mobile"": true}" 14315,5,322,2017-05-08 18:20:13,http://vonhegmann.biz/dorris,,147.84.230.154,"{""location"": ""HK"", ""is_mobile"": false}" 14316,5,322,2017-05-08 18:22:29,http://lynch.com/selina.shields,,12.46.50.231,"{""location"": ""KW"", ""is_mobile"": true}" 14317,5,322,2016-12-14 06:09:45,http://durgan.io/osbaldo.mills,,18.172.206.9,"{""location"": ""AQ"", ""is_mobile"": true}" 14318,5,322,2017-02-23 19:10:50,http://metz.io/oma_bayer,,13.169.62.178,"{""location"": ""MD"", ""is_mobile"": false}" 14319,5,322,2017-05-26 21:35:56,http://stantonolson.io/ignacio.nitzsche,,106.235.186.124,"{""location"": ""FI"", ""is_mobile"": true}" 14320,5,322,2017-02-27 15:32:58,http://cruickshank.net/lucius,,7.163.49.152,"{""location"": ""NG"", ""is_mobile"": false}" 14321,5,322,2017-02-14 08:35:51,http://rodriguez.name/neha.ebert,,231.134.111.72,"{""location"": ""SD"", ""is_mobile"": true}" 14322,5,322,2016-12-29 18:44:33,http://waelchi.com/tillman,,72.221.33.203,"{""location"": ""JM"", ""is_mobile"": true}" 14323,5,322,2017-03-05 18:15:00,http://jacobs.net/bo,,70.67.70.124,"{""location"": ""TW"", ""is_mobile"": false}" 14324,5,322,2017-04-23 04:52:20,http://gerholdritchie.name/mittie,,231.125.251.198,"{""location"": ""MW"", ""is_mobile"": true}" 14325,5,322,2017-05-12 03:14:53,http://ziemann.org/maverick,,14.176.16.101,"{""location"": ""PA"", ""is_mobile"": false}" 14326,5,322,2016-12-22 08:31:04,http://barton.com/aunta.parisian,,24.81.244.42,"{""location"": ""GH"", ""is_mobile"": true}" 14327,5,322,2016-12-16 20:34:15,http://lynchmarquardt.biz/erik,,192.187.28.137,"{""location"": ""SY"", ""is_mobile"": false}" 14328,5,322,2017-01-15 19:07:23,http://okeefe.org/isabell,,85.209.83.183,"{""location"": ""CR"", ""is_mobile"": false}" 14329,5,322,2017-02-22 02:53:27,http://wolffschmeler.net/jeremy,,73.180.99.222,"{""location"": ""PK"", ""is_mobile"": false}" 14330,5,322,2017-06-01 23:30:07,http://keeling.biz/izabella,,168.17.142.56,"{""location"": ""NC"", ""is_mobile"": true}" 14331,5,322,2017-01-05 05:40:55,http://roberts.info/trea.hane,,167.4.111.184,"{""location"": ""AS"", ""is_mobile"": false}" 14332,5,322,2017-02-02 10:34:04,http://dare.io/rosalinda.rogahn,,185.37.72.203,"{""location"": ""LB"", ""is_mobile"": true}" 17263,6,389,2017-04-05 23:00:29,http://kilbackhane.net/verla,,162.26.198.57,"{""location"": ""NO"", ""is_mobile"": true}" 17264,6,389,2017-04-26 03:41:25,http://greenfelder.co/mercedes,,221.56.188.9,"{""location"": ""SD"", ""is_mobile"": true}" 17265,6,389,2017-05-22 03:28:37,http://berge.name/kareem,,125.97.254.66,"{""location"": ""GP"", ""is_mobile"": true}" 17266,6,389,2017-03-17 13:16:04,http://abshire.io/gretchen_dare,,65.158.87.9,"{""location"": ""RO"", ""is_mobile"": false}" 17267,6,389,2017-05-11 21:44:19,http://wizazulauf.info/willis,,43.211.197.241,"{""location"": ""RW"", ""is_mobile"": true}" 17268,6,389,2017-05-07 20:43:58,http://altenwerth.net/darby_von,,100.159.147.109,"{""location"": ""WF"", ""is_mobile"": true}" 17269,6,389,2017-03-13 10:24:33,http://wolfruel.net/fermin,,46.70.81.40,"{""location"": ""AG"", ""is_mobile"": true}" 17270,6,389,2017-05-22 19:37:45,http://willms.co/berenice,,193.229.240.88,"{""location"": ""TR"", ""is_mobile"": true}" 17271,6,389,2017-04-03 12:06:16,http://hermiston.info/celestine.walker,,241.72.41.215,"{""location"": ""ET"", ""is_mobile"": false}" 17272,6,389,2016-12-30 12:11:19,http://quigleywaters.net/arely,,107.154.164.220,"{""location"": ""ER"", ""is_mobile"": true}" 17273,6,390,2017-04-30 01:39:41,http://kemmer.name/soledad.konopelski,,105.129.241.166,"{""location"": ""BN"", ""is_mobile"": false}" 17274,6,390,2017-01-20 13:37:16,http://conroy.io/garrick.bartoletti,,215.99.221.149,"{""location"": ""UZ"", ""is_mobile"": false}" 17275,6,390,2017-04-01 02:25:50,http://franeckibahringer.io/fletcher,,122.138.156.198,"{""location"": ""VU"", ""is_mobile"": true}" 17276,6,390,2017-01-17 14:14:00,http://jacobi.biz/nicola,,181.72.227.151,"{""location"": ""MG"", ""is_mobile"": false}" 17277,6,390,2017-05-08 19:07:16,http://emard.co/elouise,,104.77.146.219,"{""location"": ""GD"", ""is_mobile"": true}" 17278,6,390,2017-01-10 08:25:20,http://wisokyveum.io/cade,,86.75.82.71,"{""location"": ""CK"", ""is_mobile"": false}" 17279,6,390,2017-03-30 11:07:57,http://abernathykoepp.co/meagan,,57.12.109.63,"{""location"": ""CC"", ""is_mobile"": true}" 17280,6,390,2017-03-31 13:16:26,http://donnelly.net/glenna,,145.112.86.244,"{""location"": ""GG"", ""is_mobile"": true}" 17281,6,390,2017-04-26 17:15:56,http://white.name/ibrahim,,198.200.77.245,"{""location"": ""BA"", ""is_mobile"": false}" 17282,6,390,2017-02-04 16:20:38,http://goldner.com/willard_brown,,238.217.51.230,"{""location"": ""US"", ""is_mobile"": false}" 17283,6,390,2017-01-05 18:12:05,http://von.net/margaretta,,157.172.134.72,"{""location"": ""CL"", ""is_mobile"": true}" 17284,6,390,2016-12-13 18:11:19,http://wittingarmstrong.org/hudson_kerluke,,184.52.153.135,"{""location"": ""KE"", ""is_mobile"": false}" 17285,6,390,2017-04-05 16:57:11,http://schuppe.io/donny,,100.30.153.71,"{""location"": ""CK"", ""is_mobile"": false}" 17286,6,390,2017-04-08 18:46:19,http://satterfieldko.net/callie.white,,218.191.74.248,"{""location"": ""JE"", ""is_mobile"": false}" 17287,6,390,2017-03-17 08:22:54,http://schmeleroconner.biz/evangeline_conroy,,127.204.8.139,"{""location"": ""LV"", ""is_mobile"": false}" 17288,6,390,2017-05-03 07:39:21,http://mohrhoeger.name/alia.mcdermott,,51.143.219.104,"{""location"": ""KM"", ""is_mobile"": true}" 17289,6,390,2017-05-02 12:41:49,http://quitzonhowell.co/idell,,182.76.108.97,"{""location"": ""AL"", ""is_mobile"": false}" 17290,6,390,2017-01-16 10:42:58,http://kohler.org/haleigh,,111.53.254.214,"{""location"": ""SS"", ""is_mobile"": false}" 17291,6,390,2016-12-31 10:13:37,http://durgan.biz/wilhelm,,5.35.236.33,"{""location"": ""GN"", ""is_mobile"": false}" 17292,6,390,2017-04-12 12:10:21,http://casper.net/boris,,226.199.190.9,"{""location"": ""LR"", ""is_mobile"": true}" 17293,6,390,2017-03-22 08:08:08,http://watersromaguera.info/madelynn.lynch,,41.175.43.23,"{""location"": ""JP"", ""is_mobile"": false}" 17294,6,390,2016-12-22 00:55:45,http://dickinsonkuhic.net/titus.weimann,,206.215.21.55,"{""location"": ""IN"", ""is_mobile"": false}" 17295,6,390,2017-04-21 22:15:33,http://ziemann.net/nasir.jones,,110.215.119.105,"{""location"": ""RS"", ""is_mobile"": true}" 17296,6,390,2017-01-27 19:43:50,http://waelchi.com/elia.flatley,,106.14.130.200,"{""location"": ""MS"", ""is_mobile"": false}" 17297,6,390,2017-06-08 07:27:45,http://stehr.org/golden.stokes,,68.173.158.16,"{""location"": ""GT"", ""is_mobile"": true}" 17298,6,390,2017-01-28 03:03:12,http://osinski.org/orlando.ryan,,57.14.4.88,"{""location"": ""YE"", ""is_mobile"": true}" 17299,6,390,2017-05-07 15:15:16,http://macejkovichauck.info/manuel.white,,106.74.105.63,"{""location"": ""NC"", ""is_mobile"": true}" 17300,6,390,2017-01-11 10:31:45,http://rueckerkiehn.co/mona_kling,,133.176.180.107,"{""location"": ""LV"", ""is_mobile"": false}" 17301,6,390,2017-02-03 12:42:57,http://ratke.co/carey,,114.213.244.29,"{""location"": ""BY"", ""is_mobile"": true}" 17303,6,390,2017-05-24 10:26:12,http://schaefer.io/jacques,,32.163.182.10,"{""location"": ""RS"", ""is_mobile"": true}" 17304,6,390,2016-12-16 03:06:54,http://hermanhoppe.io/lavada,,112.139.212.193,"{""location"": ""GM"", ""is_mobile"": true}" 17305,6,390,2017-06-05 22:08:38,http://brakus.io/pansy,,153.149.98.27,"{""location"": ""CL"", ""is_mobile"": false}" 17306,6,390,2017-03-30 11:17:45,http://lang.info/reyna.upton,,21.147.74.226,"{""location"": ""JE"", ""is_mobile"": true}" 17307,6,390,2017-01-02 10:12:48,http://spencer.co/devan_franecki,,244.169.75.32,"{""location"": ""RW"", ""is_mobile"": true}" 17308,6,390,2017-04-19 08:20:44,http://cummingsaufderhar.name/marlin,,209.122.197.240,"{""location"": ""HN"", ""is_mobile"": true}" 17309,6,390,2017-01-05 06:08:22,http://vonruedenwhite.name/helga,,16.117.213.3,"{""location"": ""TL"", ""is_mobile"": true}" 17310,6,390,2017-01-01 10:42:34,http://hoppe.com/johnathan.fadel,,96.148.120.150,"{""location"": ""SE"", ""is_mobile"": false}" 17311,6,390,2017-03-06 18:49:40,http://fadelcorkery.com/trevion.bahringer,,146.253.13.76,"{""location"": ""BB"", ""is_mobile"": true}" 17312,6,390,2017-03-18 09:00:20,http://oconnercrist.net/patience,,15.194.237.229,"{""location"": ""MF"", ""is_mobile"": false}" 17313,6,390,2017-03-07 18:53:25,http://connhayes.info/kacie.streich,,245.147.126.247,"{""location"": ""NL"", ""is_mobile"": false}" 17314,6,390,2017-04-05 09:16:10,http://schamberger.com/crystal_paucek,,147.145.146.59,"{""location"": ""MQ"", ""is_mobile"": false}" 17315,6,391,2017-05-27 09:43:53,http://hickledavis.info/lilly_weber,,52.236.111.109,"{""location"": ""FO"", ""is_mobile"": false}" 17316,6,391,2016-12-25 05:22:35,http://breitenberg.name/matilde,,216.65.211.210,"{""location"": ""AI"", ""is_mobile"": false}" 17317,6,391,2017-02-02 10:12:53,http://baumbach.co/marcel_kshlerin,,119.210.172.121,"{""location"": ""BT"", ""is_mobile"": true}" 17318,6,391,2017-03-26 15:27:12,http://littel.io/loma,,27.100.221.83,"{""location"": ""BO"", ""is_mobile"": true}" 4446,2,97,2017-05-15 18:17:43,http://jaskolski.io/major,,236.127.180.143,"{""location"": ""LI"", ""is_mobile"": false}" 4447,2,97,2017-01-15 14:18:47,http://deckowbarrows.org/althea,,99.19.55.183,"{""location"": ""GT"", ""is_mobile"": true}" 4448,2,98,2017-02-20 19:03:54,http://beier.org/betty,,51.132.211.150,"{""location"": ""PW"", ""is_mobile"": true}" 4449,2,98,2017-03-31 00:53:37,http://gulgowski.co/bo,,27.251.82.198,"{""location"": ""NI"", ""is_mobile"": true}" 4450,2,98,2017-05-25 18:32:18,http://collier.co/rory.williamson,,63.155.185.207,"{""location"": ""MC"", ""is_mobile"": true}" 4451,2,98,2017-05-07 01:24:09,http://lueilwitz.com/aleandra,,70.49.24.53,"{""location"": ""UY"", ""is_mobile"": true}" 4452,2,98,2017-02-16 04:02:09,http://hintz.co/arden_altenwerth,,6.221.13.191,"{""location"": ""VE"", ""is_mobile"": false}" 4453,2,98,2017-04-01 11:01:34,http://mcdermottkihn.io/olin_hodkiewicz,,129.85.129.59,"{""location"": ""BR"", ""is_mobile"": true}" 4454,2,98,2017-02-07 03:52:57,http://bode.com/dolores,,14.119.205.252,"{""location"": ""AZ"", ""is_mobile"": false}" 4455,2,98,2017-04-21 14:08:15,http://bednar.co/pinkie,,56.71.107.209,"{""location"": ""CY"", ""is_mobile"": true}" 4456,2,98,2017-02-27 03:20:29,http://miller.biz/sean,,181.108.111.52,"{""location"": ""NI"", ""is_mobile"": false}" 4457,2,98,2017-01-31 14:01:34,http://gorczany.com/justyn,,96.78.20.182,"{""location"": ""VI"", ""is_mobile"": true}" 4458,2,98,2017-02-07 14:35:47,http://treutel.io/sim_gusikowski,,63.16.107.252,"{""location"": ""MY"", ""is_mobile"": true}" 4459,2,98,2017-01-01 04:11:30,http://fadel.io/corene.yost,,108.37.196.21,"{""location"": ""ES"", ""is_mobile"": false}" 4460,2,98,2017-01-02 09:23:53,http://brekkehackett.net/celine.koch,,19.91.246.126,"{""location"": ""AM"", ""is_mobile"": true}" 4461,2,98,2017-04-02 15:27:33,http://oreilly.co/carson_franecki,,184.191.109.118,"{""location"": ""KZ"", ""is_mobile"": false}" 4462,2,98,2017-05-19 13:23:30,http://lebsack.net/destany,,27.217.139.43,"{""location"": ""KP"", ""is_mobile"": false}" 4463,2,98,2017-03-29 07:43:13,http://dach.net/mabel,,131.190.99.97,"{""location"": ""LY"", ""is_mobile"": false}" 4464,2,98,2016-12-20 21:34:09,http://yost.net/rozella,,107.120.166.67,"{""location"": ""PG"", ""is_mobile"": false}" 4465,2,98,2017-05-08 07:28:10,http://bartell.info/alex.raynor,,214.157.149.253,"{""location"": ""CU"", ""is_mobile"": false}" 4466,2,98,2017-05-27 19:35:28,http://harveyeffertz.name/dorcas,,92.181.222.173,"{""location"": ""LR"", ""is_mobile"": true}" 4467,2,98,2016-12-26 11:31:14,http://kunde.name/andrew,,15.198.121.65,"{""location"": ""GB"", ""is_mobile"": true}" 4468,2,98,2017-04-10 00:39:34,http://marquardt.biz/else.ward,,28.123.59.27,"{""location"": ""TM"", ""is_mobile"": true}" 4469,2,98,2017-05-17 19:41:36,http://greenfelder.io/helga,,165.103.196.6,"{""location"": ""CD"", ""is_mobile"": true}" 4470,2,98,2017-02-13 06:29:47,http://hermann.org/cayla.tromp,,240.154.51.63,"{""location"": ""TK"", ""is_mobile"": true}" 4471,2,98,2017-01-13 16:09:47,http://yost.biz/janae.cummerata,,17.154.183.12,"{""location"": ""MS"", ""is_mobile"": false}" 4472,2,98,2017-01-16 18:21:31,http://colewill.co/bobby_satterfield,,30.140.43.59,"{""location"": ""AG"", ""is_mobile"": false}" 4473,2,98,2017-03-25 05:59:04,http://murphy.org/isabelle_marks,,16.62.88.155,"{""location"": ""BQ"", ""is_mobile"": false}" 4474,2,98,2017-04-23 09:57:18,http://schadenankunding.info/christina,,30.210.65.56,"{""location"": ""CU"", ""is_mobile"": false}" 4475,2,98,2017-03-11 07:40:13,http://gerhold.com/nia_daugherty,,27.24.149.140,"{""location"": ""RE"", ""is_mobile"": true}" 4476,2,98,2017-01-07 00:02:16,http://conroyhodkiewicz.co/brad.sauer,,112.17.108.115,"{""location"": ""HU"", ""is_mobile"": true}" 4477,2,98,2017-04-18 04:58:28,http://krisnikolaus.org/marianne.mohr,,100.109.142.40,"{""location"": ""CL"", ""is_mobile"": false}" 4478,2,98,2017-03-08 01:30:53,http://hand.org/joanie.dooley,,66.21.46.127,"{""location"": ""SI"", ""is_mobile"": false}" 4479,2,98,2017-03-29 15:21:22,http://reichelconsidine.co/destin,,79.26.5.16,"{""location"": ""KI"", ""is_mobile"": false}" 4480,2,98,2017-02-27 03:57:29,http://quigley.biz/stefanie_wilkinson,,127.155.54.177,"{""location"": ""FI"", ""is_mobile"": false}" 4481,2,99,2017-03-21 20:20:58,http://rennerrath.info/mattie.gusikowski,,240.145.6.250,"{""location"": ""YT"", ""is_mobile"": false}" 4482,2,99,2017-06-13 13:31:45,http://denesik.info/sibyl,,127.248.237.241,"{""location"": ""MY"", ""is_mobile"": false}" 4483,2,99,2017-02-26 21:34:09,http://mertz.io/milton,,42.203.254.176,"{""location"": ""AO"", ""is_mobile"": false}" 4484,2,99,2016-12-15 16:22:32,http://anderson.io/abelardo_mraz,,190.15.26.219,"{""location"": ""DZ"", ""is_mobile"": true}" 4485,2,99,2016-12-14 23:10:10,http://heller.name/cyril_kertzmann,,157.135.158.190,"{""location"": ""NA"", ""is_mobile"": false}" 4486,2,99,2017-01-09 22:55:09,http://effertz.net/wyatt_wiegand,,140.138.140.194,"{""location"": ""LB"", ""is_mobile"": false}" 4487,2,99,2017-03-02 13:45:10,http://torp.biz/rosalind,,129.212.251.111,"{""location"": ""MT"", ""is_mobile"": false}" 4488,2,99,2017-02-02 02:32:04,http://bode.info/mariano.corkery,,225.98.152.231,"{""location"": ""BI"", ""is_mobile"": false}" 4489,2,99,2017-04-18 11:45:38,http://swift.org/don_lemke,,219.245.130.191,"{""location"": ""MM"", ""is_mobile"": false}" 4490,2,99,2017-03-04 02:46:29,http://christiansen.net/sandy_muller,,88.194.65.245,"{""location"": ""CM"", ""is_mobile"": true}" 4491,2,99,2017-04-17 07:41:56,http://rice.biz/estevan_jerde,,99.34.34.235,"{""location"": ""AM"", ""is_mobile"": false}" 4492,2,99,2017-05-14 06:21:19,http://kuhlman.info/toney,,40.150.142.226,"{""location"": ""GI"", ""is_mobile"": true}" 4493,2,99,2017-06-03 21:02:40,http://lemke.name/karen.bode,,54.106.48.114,"{""location"": ""CY"", ""is_mobile"": false}" 4494,2,99,2017-06-11 11:45:40,http://kovacek.biz/layla,,56.93.157.16,"{""location"": ""BG"", ""is_mobile"": true}" 4495,2,99,2017-03-24 14:50:42,http://weimann.com/mortimer.koelpin,,74.43.60.150,"{""location"": ""CM"", ""is_mobile"": true}" 4496,2,99,2016-12-13 20:34:21,http://herman.io/ibrahim_friesen,,224.145.120.135,"{""location"": ""CU"", ""is_mobile"": true}" 4497,2,99,2017-06-08 16:42:59,http://macejkovicullrich.info/gloria_schultz,,213.125.69.59,"{""location"": ""IL"", ""is_mobile"": false}" 4498,2,99,2017-02-13 12:22:57,http://harvey.info/ernestina,,109.165.173.138,"{""location"": ""BM"", ""is_mobile"": false}" 4499,2,99,2017-05-23 08:31:52,http://conroyweber.org/toy.bauch,,103.160.232.163,"{""location"": ""TL"", ""is_mobile"": false}" 4500,2,99,2017-05-05 13:42:15,http://vonrueden.co/ali,,45.148.213.11,"{""location"": ""CO"", ""is_mobile"": true}" 4501,2,99,2017-02-24 21:03:35,http://ziemeprice.net/wallace_hermiston,,249.170.118.79,"{""location"": ""NU"", ""is_mobile"": true}" 11383,4,254,2017-02-16 19:17:49,http://schuster.info/dominic,,220.202.97.55,"{""location"": ""LC"", ""is_mobile"": true}" 11384,4,254,2017-06-11 23:43:38,http://morargaylord.org/toni.hilll,,250.114.6.117,"{""location"": ""TO"", ""is_mobile"": true}" 11385,4,254,2017-04-02 10:02:47,http://borerbreitenberg.net/eda,,139.224.228.240,"{""location"": ""MO"", ""is_mobile"": true}" 11386,4,254,2017-04-03 12:09:08,http://thiel.net/nestor_lubowitz,,204.59.34.181,"{""location"": ""LI"", ""is_mobile"": false}" 11387,4,254,2017-05-25 01:38:39,http://hoppeoconner.info/michaela,,150.254.145.5,"{""location"": ""OM"", ""is_mobile"": false}" 11388,4,254,2017-04-08 20:08:50,http://keeling.name/laria,,130.62.195.142,"{""location"": ""FI"", ""is_mobile"": false}" 11389,4,254,2017-02-07 21:41:15,http://rau.com/eliza,,234.46.78.96,"{""location"": ""BR"", ""is_mobile"": true}" 11390,4,254,2017-02-06 23:10:44,http://greenoconnell.org/freeda.turner,,163.156.30.117,"{""location"": ""TF"", ""is_mobile"": false}" 11391,4,254,2017-01-23 00:41:13,http://funk.com/dolores_powlowski,,202.110.17.134,"{""location"": ""BV"", ""is_mobile"": false}" 11392,4,254,2017-02-01 19:53:45,http://littleauer.org/sydnie,,252.177.208.101,"{""location"": ""MH"", ""is_mobile"": false}" 11393,4,254,2017-02-01 06:21:45,http://torphy.info/corene,,239.100.57.26,"{""location"": ""RE"", ""is_mobile"": false}" 11394,4,254,2017-01-14 04:17:52,http://brown.biz/kyla,,49.178.108.20,"{""location"": ""RU"", ""is_mobile"": false}" 11395,4,254,2016-12-28 18:42:42,http://homenick.name/armando,,134.174.188.63,"{""location"": ""TZ"", ""is_mobile"": false}" 11396,4,254,2017-03-30 06:09:14,http://goldnercrona.net/olaf_hilpert,,204.246.112.159,"{""location"": ""GI"", ""is_mobile"": false}" 11397,4,254,2017-02-07 10:34:24,http://tillman.net/charlene,,230.161.34.219,"{""location"": ""BS"", ""is_mobile"": false}" 11398,4,254,2017-04-19 19:33:57,http://terry.co/serena.dibbert,,123.203.71.96,"{""location"": ""BF"", ""is_mobile"": false}" 11399,4,254,2017-05-19 18:20:33,http://hammes.name/fannie,,19.91.220.237,"{""location"": ""PA"", ""is_mobile"": true}" 11400,4,254,2017-03-15 15:13:05,http://cronin.co/sven_schneider,,217.119.158.241,"{""location"": ""US"", ""is_mobile"": false}" 11401,4,254,2017-02-14 08:17:47,http://oconner.info/maximilian_casper,,113.182.207.133,"{""location"": ""JE"", ""is_mobile"": false}" 11402,4,254,2017-02-11 13:57:03,http://graham.io/keven.stiedemann,,124.58.242.34,"{""location"": ""NA"", ""is_mobile"": false}" 11403,4,254,2017-02-18 02:53:44,http://crooks.info/wilson,,80.87.82.47,"{""location"": ""ST"", ""is_mobile"": false}" 11404,4,254,2017-02-19 06:50:28,http://schusterjaskolski.info/amelie_carroll,,59.56.146.178,"{""location"": ""AX"", ""is_mobile"": true}" 11405,4,254,2017-03-15 23:31:34,http://toybeier.io/lee.white,,101.152.202.37,"{""location"": ""GF"", ""is_mobile"": false}" 11406,4,254,2017-04-27 00:15:13,http://gleasonlangworth.com/kylie.mosciski,,181.229.113.200,"{""location"": ""NE"", ""is_mobile"": false}" 11407,4,255,2016-12-14 06:22:25,http://keler.org/simeon,,100.221.70.117,"{""location"": ""RS"", ""is_mobile"": false}" 11408,4,255,2016-12-14 12:53:59,http://nolan.name/celestine,,160.193.96.122,"{""location"": ""AX"", ""is_mobile"": true}" 11409,4,255,2017-03-10 18:43:21,http://murphycummings.name/ozella.keler,,127.30.143.45,"{""location"": ""PF"", ""is_mobile"": true}" 11410,4,255,2017-04-08 13:05:44,http://douglas.name/berry,,34.180.4.211,"{""location"": ""TM"", ""is_mobile"": true}" 11411,4,255,2017-04-02 19:35:51,http://sauer.biz/christine_miller,,185.122.164.66,"{""location"": ""SE"", ""is_mobile"": false}" 11412,4,255,2017-06-04 18:02:17,http://ward.name/abelardo,,66.192.114.64,"{""location"": ""MO"", ""is_mobile"": true}" 11413,4,255,2017-03-28 01:42:19,http://stamm.info/oma_johns,,57.210.93.207,"{""location"": ""RE"", ""is_mobile"": true}" 11414,4,255,2017-01-16 11:33:51,http://runolfonmiller.net/gina,,19.82.204.159,"{""location"": ""TM"", ""is_mobile"": true}" 11415,4,255,2017-03-19 06:39:41,http://lowe.name/bartholome.kaulke,,142.158.38.53,"{""location"": ""TH"", ""is_mobile"": false}" 11416,4,255,2017-05-26 17:20:59,http://douglaswunsch.io/krystal_deckow,,59.19.136.241,"{""location"": ""EC"", ""is_mobile"": true}" 11417,4,255,2017-01-28 03:20:51,http://fritsch.io/diamond,,163.71.194.224,"{""location"": ""NF"", ""is_mobile"": false}" 11418,4,255,2017-01-25 08:51:38,http://goodwinheller.info/jazmyn,,252.139.8.205,"{""location"": ""LS"", ""is_mobile"": true}" 11419,4,255,2017-01-04 23:13:55,http://langworth.biz/neal,,95.67.119.51,"{""location"": ""MC"", ""is_mobile"": false}" 11420,4,255,2017-04-14 04:16:41,http://mayert.io/keegan_considine,,108.15.92.133,"{""location"": ""HR"", ""is_mobile"": false}" 11421,4,255,2017-01-13 09:54:02,http://wiza.info/domenico_blick,,172.245.228.177,"{""location"": ""TO"", ""is_mobile"": false}" 11422,4,255,2017-01-30 21:46:01,http://considineleannon.com/dwight,,163.8.55.19,"{""location"": ""MS"", ""is_mobile"": false}" 11423,4,255,2017-02-04 23:48:31,http://windler.net/schuyler_turner,,111.53.53.215,"{""location"": ""MC"", ""is_mobile"": true}" 11424,4,255,2017-06-12 10:06:07,http://daugherty.io/annabell,,10.248.225.143,"{""location"": ""MT"", ""is_mobile"": false}" 11425,4,255,2017-02-13 08:24:53,http://lueilwitz.io/sabrina.kuvalis,,176.244.110.168,"{""location"": ""DZ"", ""is_mobile"": false}" 11426,4,255,2017-05-31 15:51:07,http://jacobson.info/arno,,194.152.220.131,"{""location"": ""JP"", ""is_mobile"": false}" 11427,4,255,2017-05-20 08:32:48,http://morar.net/llewellyn.heaney,,81.109.128.115,"{""location"": ""PF"", ""is_mobile"": false}" 11428,4,255,2017-02-10 04:20:06,http://gislasonvon.biz/bailey,,94.214.157.243,"{""location"": ""AZ"", ""is_mobile"": true}" 11429,4,255,2017-03-06 04:12:35,http://kris.org/judy_parisian,,94.176.92.73,"{""location"": ""SG"", ""is_mobile"": false}" 11430,4,255,2017-03-27 14:08:35,http://denesikgusikowski.org/dustin_torp,,21.217.148.64,"{""location"": ""MX"", ""is_mobile"": true}" 11431,4,255,2017-01-26 11:24:35,http://barton.net/luther,,146.84.120.231,"{""location"": ""AD"", ""is_mobile"": false}" 11432,4,255,2017-01-08 23:49:02,http://grimespadberg.name/jaydon.hagenes,,225.118.148.96,"{""location"": ""SB"", ""is_mobile"": false}" 11433,4,255,2017-01-25 14:15:56,http://altenwerthsporer.com/aunta,,70.156.69.243,"{""location"": ""ST"", ""is_mobile"": false}" 11434,4,255,2017-02-28 04:15:00,http://schmidt.co/kaley_conn,,158.79.82.80,"{""location"": ""SK"", ""is_mobile"": true}" 11435,4,255,2017-06-03 18:08:17,http://schinnervolkman.info/wyman,,61.176.226.222,"{""location"": ""KI"", ""is_mobile"": true}" 11436,4,255,2017-06-07 19:02:12,http://colepadberg.biz/wendy.shanahan,,219.149.213.187,"{""location"": ""PK"", ""is_mobile"": true}" 11437,4,255,2016-12-25 22:38:32,http://deckow.io/nya.padberg,,92.140.247.192,"{""location"": ""SG"", ""is_mobile"": true}" 8384,3,187,2017-05-13 13:51:26,http://hirthe.co/isabel,,132.50.174.47,"{""location"": ""LS"", ""is_mobile"": false}" 8385,3,187,2017-02-24 21:50:21,http://keelingmiller.name/lempi,,182.175.196.37,"{""location"": ""TJ"", ""is_mobile"": true}" 8386,3,187,2017-05-18 10:29:44,http://schultz.io/alva,,107.121.165.74,"{""location"": ""HM"", ""is_mobile"": true}" 8387,3,187,2017-05-14 17:45:21,http://smith.org/erwin,,187.149.30.226,"{""location"": ""DE"", ""is_mobile"": false}" 8388,3,187,2016-12-23 04:06:59,http://dickens.name/wilma.emard,,144.70.77.9,"{""location"": ""GD"", ""is_mobile"": true}" 8389,3,187,2017-04-24 12:55:58,http://cronin.biz/alene,,213.38.158.91,"{""location"": ""IL"", ""is_mobile"": false}" 8390,3,187,2017-01-27 07:28:51,http://kris.co/kasey,,246.124.29.152,"{""location"": ""FI"", ""is_mobile"": true}" 8391,3,187,2016-12-24 05:46:09,http://osinski.org/anais_rodriguez,,179.242.39.249,"{""location"": ""PW"", ""is_mobile"": false}" 8392,3,187,2017-02-02 02:40:23,http://beahan.io/ellen.powlowski,,25.68.35.240,"{""location"": ""UG"", ""is_mobile"": false}" 8393,3,187,2017-02-18 15:17:52,http://rodriguez.name/orval,,238.46.37.111,"{""location"": ""BW"", ""is_mobile"": false}" 8394,3,187,2017-05-24 18:54:05,http://beckeryost.co/charlie.marvin,,105.203.241.147,"{""location"": ""MG"", ""is_mobile"": false}" 8395,3,187,2017-05-15 05:12:14,http://bode.org/seth,,87.48.234.69,"{""location"": ""LY"", ""is_mobile"": true}" 8396,3,187,2017-02-26 11:25:42,http://balistreri.info/geovanny,,198.92.230.78,"{""location"": ""TH"", ""is_mobile"": true}" 8397,3,187,2016-12-18 14:29:33,http://collins.io/daphnee_halvorson,,107.86.254.79,"{""location"": ""UA"", ""is_mobile"": true}" 8398,3,187,2017-02-20 04:35:24,http://gleason.info/dashawn_collier,,4.131.201.25,"{""location"": ""LU"", ""is_mobile"": false}" 8399,3,187,2017-03-05 16:03:35,http://howelllangosh.info/collin.fritsch,,127.199.212.123,"{""location"": ""LV"", ""is_mobile"": false}" 8400,3,187,2017-02-14 08:22:43,http://pagac.biz/andre,,155.16.87.48,"{""location"": ""GT"", ""is_mobile"": true}" 8401,3,187,2017-05-12 10:52:58,http://schulist.biz/linda.conroy,,242.5.30.36,"{""location"": ""CZ"", ""is_mobile"": true}" 8402,3,187,2017-03-11 15:18:26,http://schneiderpollich.biz/jada,,206.206.37.166,"{""location"": ""DK"", ""is_mobile"": false}" 8403,3,187,2017-06-13 22:32:36,http://hickle.com/mya.kihn,,239.154.43.143,"{""location"": ""TF"", ""is_mobile"": true}" 8404,3,187,2017-05-25 02:51:02,http://heidenreich.biz/jacey_kutch,,99.125.58.144,"{""location"": ""CI"", ""is_mobile"": true}" 8405,3,188,2017-04-21 22:24:23,http://uptonwilkinson.io/taurean.farrell,,59.189.242.207,"{""location"": ""PW"", ""is_mobile"": true}" 8406,3,188,2017-01-08 15:59:48,http://schummbartell.co/katheryn,,104.59.222.154,"{""location"": ""SH"", ""is_mobile"": true}" 8407,3,188,2017-05-27 15:52:47,http://hills.biz/wilfred,,183.41.50.200,"{""location"": ""MR"", ""is_mobile"": false}" 8408,3,188,2017-01-21 15:32:17,http://lemke.info/caterina.runolfsdottir,,153.173.209.211,"{""location"": ""TO"", ""is_mobile"": false}" 8409,3,188,2016-12-23 17:10:32,http://nienow.org/kamren,,148.106.75.22,"{""location"": ""MC"", ""is_mobile"": false}" 8410,3,188,2017-02-09 08:16:44,http://doyle.io/amanda,,65.27.5.82,"{""location"": ""UG"", ""is_mobile"": false}" 8411,3,188,2017-02-13 11:51:16,http://kozeykihn.name/alexander,,15.138.177.41,"{""location"": ""BJ"", ""is_mobile"": true}" 8412,3,188,2016-12-19 21:04:50,http://wisozkheaney.org/timmothy.dickens,,11.124.139.76,"{""location"": ""GH"", ""is_mobile"": true}" 8413,3,188,2017-06-12 19:22:18,http://lindgrensauer.net/gus.moen,,189.11.169.160,"{""location"": ""CU"", ""is_mobile"": true}" 8414,3,188,2017-05-24 19:15:23,http://marksveum.com/christa.okuneva,,252.99.16.92,"{""location"": ""SX"", ""is_mobile"": true}" 8415,3,188,2017-01-01 05:43:30,http://trantow.com/erich,,212.206.53.27,"{""location"": ""GR"", ""is_mobile"": true}" 8416,3,188,2017-06-05 22:35:35,http://oreilly.name/keaton_padberg,,36.101.11.224,"{""location"": ""MX"", ""is_mobile"": false}" 8417,3,188,2017-03-15 17:44:35,http://donnellyondricka.com/magnus,,219.71.136.230,"{""location"": ""NO"", ""is_mobile"": true}" 8418,3,188,2016-12-14 22:03:59,http://jakubowskiwolff.co/stephany.bode,,237.81.64.34,"{""location"": ""GI"", ""is_mobile"": false}" 8419,3,188,2017-03-02 16:48:53,http://gibson.co/aisha_murazik,,231.36.157.3,"{""location"": ""GL"", ""is_mobile"": false}" 8420,3,188,2017-04-19 14:07:45,http://schmeler.biz/alena,,177.65.9.196,"{""location"": ""NC"", ""is_mobile"": false}" 8421,3,188,2017-01-27 09:00:27,http://okunevajohnston.co/zane_labadie,,158.178.163.81,"{""location"": ""AT"", ""is_mobile"": false}" 8422,3,188,2017-02-26 22:59:59,http://nicolas.org/irwin,,16.200.119.229,"{""location"": ""BT"", ""is_mobile"": false}" 8423,3,188,2017-01-29 04:29:27,http://lemkekeeling.io/lamar.kling,,94.199.14.31,"{""location"": ""MG"", ""is_mobile"": true}" 8424,3,188,2017-02-09 04:28:04,http://rolfson.com/tillman_ledner,,67.62.190.176,"{""location"": ""CR"", ""is_mobile"": true}" 8425,3,188,2016-12-25 00:10:49,http://mosciski.co/nayeli,,29.84.72.206,"{""location"": ""HR"", ""is_mobile"": false}" 8426,3,188,2016-12-22 11:03:38,http://kunze.info/daniela,,185.32.247.100,"{""location"": ""BH"", ""is_mobile"": true}" 8427,3,188,2017-05-16 23:25:22,http://jacobsbauch.name/tatum,,218.97.56.75,"{""location"": ""TZ"", ""is_mobile"": true}" 8428,3,189,2017-05-23 17:08:35,http://stracke.com/jakob_keebler,,162.139.250.77,"{""location"": ""WF"", ""is_mobile"": true}" 8429,3,189,2017-03-17 07:08:54,http://hodkiewicz.org/johanna.toy,,7.153.154.186,"{""location"": ""LI"", ""is_mobile"": false}" 8430,3,189,2017-05-25 07:03:30,http://leuschkeking.com/wanda,,174.128.189.115,"{""location"": ""CO"", ""is_mobile"": false}" 8431,3,189,2017-05-11 16:02:46,http://waelchileannon.org/albina,,211.77.59.251,"{""location"": ""NU"", ""is_mobile"": true}" 8432,3,189,2017-01-17 01:10:46,http://goodwin.org/gina.cremin,,2.234.70.194,"{""location"": ""JP"", ""is_mobile"": true}" 8433,3,189,2017-02-09 13:01:39,http://kovacek.co/candido,,106.41.254.214,"{""location"": ""DO"", ""is_mobile"": false}" 8434,3,189,2017-05-28 18:57:37,http://shanahan.io/clara,,125.116.176.125,"{""location"": ""TZ"", ""is_mobile"": true}" 8435,3,189,2017-03-08 10:56:05,http://lubowitz.biz/caandra,,21.108.116.37,"{""location"": ""GT"", ""is_mobile"": false}" 8436,3,189,2017-06-05 00:29:43,http://fayreynolds.com/name.stracke,,240.222.179.6,"{""location"": ""HK"", ""is_mobile"": true}" 8437,3,189,2017-05-09 12:04:36,http://haneskiles.io/jadon,,179.94.12.56,"{""location"": ""HT"", ""is_mobile"": false}" 8438,3,189,2017-05-06 17:27:26,http://haag.com/ronny.schimmel,,188.191.203.189,"{""location"": ""QA"", ""is_mobile"": true}" 14333,5,322,2017-03-10 04:54:40,http://millerjenkins.biz/walton,,160.84.211.55,"{""location"": ""UY"", ""is_mobile"": false}" 14334,5,322,2017-01-20 18:54:56,http://hansen.org/lourdes.thompson,,251.187.43.190,"{""location"": ""IS"", ""is_mobile"": false}" 14335,5,322,2017-04-12 13:13:13,http://simonis.com/modesto_murazik,,105.139.122.171,"{""location"": ""TH"", ""is_mobile"": true}" 14336,5,322,2017-03-03 18:07:44,http://considine.org/jamaal.spinka,,98.171.201.188,"{""location"": ""GL"", ""is_mobile"": true}" 14337,5,322,2016-12-15 10:24:20,http://romaguera.net/ruthie.schamberger,,167.72.26.69,"{""location"": ""BG"", ""is_mobile"": true}" 14338,5,322,2017-05-25 03:37:07,http://hirthemacgyver.com/ethel,,47.76.107.96,"{""location"": ""JP"", ""is_mobile"": false}" 14339,5,322,2017-02-21 04:05:07,http://heaneyfeest.com/jordane_klocko,,115.26.114.19,"{""location"": ""PH"", ""is_mobile"": true}" 14340,5,322,2017-05-14 09:25:43,http://lueilwitz.com/rogers.torphy,,65.193.214.243,"{""location"": ""AU"", ""is_mobile"": false}" 14341,5,322,2017-01-24 22:47:59,http://west.co/dana,,94.45.232.127,"{""location"": ""MM"", ""is_mobile"": true}" 14342,5,322,2017-04-16 22:29:43,http://jaskolskidaniel.biz/lina_rolfson,,195.177.147.66,"{""location"": ""LU"", ""is_mobile"": true}" 14343,5,322,2017-04-22 19:24:45,http://bergnaum.net/fabian.heller,,16.243.94.62,"{""location"": ""JE"", ""is_mobile"": false}" 14344,5,322,2017-03-22 00:28:03,http://mcclure.net/clovis,,93.120.118.177,"{""location"": ""PT"", ""is_mobile"": false}" 14345,5,323,2017-02-12 06:36:14,http://maggio.co/leo.connelly,,240.121.15.110,"{""location"": ""NR"", ""is_mobile"": true}" 14346,5,323,2016-12-29 12:52:20,http://pouros.net/ottilie.weimann,,7.196.153.22,"{""location"": ""AW"", ""is_mobile"": false}" 14347,5,323,2017-02-01 09:34:13,http://metz.info/jonathon,,201.132.39.18,"{""location"": ""VE"", ""is_mobile"": true}" 14348,5,323,2017-06-13 04:54:57,http://lueilwitz.io/dariana,,174.24.67.73,"{""location"": ""NP"", ""is_mobile"": false}" 14349,5,323,2017-01-29 09:01:53,http://nolanbednar.net/trevion_kovacek,,140.98.128.185,"{""location"": ""IM"", ""is_mobile"": true}" 14350,5,323,2017-05-26 02:52:01,http://kemmer.info/clare,,40.74.43.215,"{""location"": ""CY"", ""is_mobile"": true}" 14351,5,323,2017-04-29 14:23:08,http://marvin.net/hildegard,,216.174.122.210,"{""location"": ""SA"", ""is_mobile"": false}" 14352,5,323,2017-06-10 20:50:28,http://crooks.com/julie,,66.123.53.233,"{""location"": ""SD"", ""is_mobile"": true}" 14353,5,323,2017-03-31 21:35:03,http://stantonlabadie.biz/aleia_orn,,234.25.218.15,"{""location"": ""PR"", ""is_mobile"": false}" 14354,5,323,2016-12-24 11:09:36,http://lesch.com/kelvin.corwin,,173.177.63.224,"{""location"": ""GG"", ""is_mobile"": false}" 14355,5,323,2017-02-06 11:25:03,http://grant.biz/robyn.legros,,143.185.13.169,"{""location"": ""AQ"", ""is_mobile"": true}" 14356,5,323,2017-03-26 20:54:24,http://vandervort.com/allie,,107.161.34.165,"{""location"": ""GI"", ""is_mobile"": false}" 14357,5,323,2017-03-18 09:48:39,http://ondricka.name/augustine,,224.186.81.187,"{""location"": ""NU"", ""is_mobile"": true}" 14358,5,323,2017-01-15 08:23:19,http://larson.co/odie,,247.63.237.254,"{""location"": ""MU"", ""is_mobile"": false}" 14359,5,323,2017-06-01 04:27:30,http://schmitt.name/ariane,,67.227.241.212,"{""location"": ""KP"", ""is_mobile"": false}" 14360,5,323,2017-05-23 00:51:10,http://pfeffer.biz/leonel,,198.159.55.166,"{""location"": ""CK"", ""is_mobile"": false}" 14361,5,323,2017-03-25 13:14:44,http://howellhammes.com/elroy.jones,,252.113.158.118,"{""location"": ""MO"", ""is_mobile"": true}" 14362,5,323,2017-03-24 18:49:18,http://hickle.net/ali,,218.23.60.69,"{""location"": ""MX"", ""is_mobile"": false}" 14363,5,323,2017-06-11 07:52:21,http://bernier.co/georgiana,,14.236.48.227,"{""location"": ""NI"", ""is_mobile"": true}" 14364,5,323,2017-05-21 18:50:50,http://harveyreichert.biz/ahmed,,153.55.167.96,"{""location"": ""SH"", ""is_mobile"": true}" 14365,5,323,2016-12-31 10:38:49,http://kilbackwilkinson.co/linnea,,188.172.226.79,"{""location"": ""AS"", ""is_mobile"": false}" 14366,5,323,2017-04-30 08:55:16,http://mayertklocko.biz/domenick,,23.181.232.153,"{""location"": ""AG"", ""is_mobile"": true}" 14367,5,323,2017-04-11 04:27:34,http://zemlak.name/jayson_witting,,17.213.14.109,"{""location"": ""MQ"", ""is_mobile"": true}" 14368,5,323,2017-05-01 23:27:31,http://ullrich.io/kaia.smitham,,20.81.161.110,"{""location"": ""VI"", ""is_mobile"": false}" 14369,5,323,2016-12-21 05:52:52,http://marks.io/trea_feil,,18.13.70.124,"{""location"": ""GY"", ""is_mobile"": true}" 14370,5,323,2017-03-28 17:47:35,http://lang.net/buford_rosenbaum,,94.92.154.15,"{""location"": ""LB"", ""is_mobile"": true}" 14371,5,323,2017-05-06 10:24:02,http://weber.net/elvera,,243.81.87.252,"{""location"": ""LV"", ""is_mobile"": false}" 14372,5,323,2017-01-03 07:41:50,http://lueilwitzernser.name/kacie_nicolas,,112.193.126.99,"{""location"": ""SH"", ""is_mobile"": false}" 14373,5,323,2017-02-14 10:08:40,http://braun.name/jeramy_gleason,,244.134.177.94,"{""location"": ""KG"", ""is_mobile"": false}" 14374,5,323,2017-01-13 09:35:54,http://brakus.org/dominique,,170.73.116.111,"{""location"": ""ER"", ""is_mobile"": false}" 14375,5,323,2017-03-24 02:55:14,http://runte.biz/monte,,152.217.51.80,"{""location"": ""MN"", ""is_mobile"": true}" 14376,5,323,2017-03-19 14:34:34,http://littlehickle.com/bernhard_hahn,,116.113.54.155,"{""location"": ""PF"", ""is_mobile"": false}" 14377,5,323,2016-12-27 06:46:19,http://legros.io/angelita,,164.118.72.156,"{""location"": ""SH"", ""is_mobile"": true}" 14378,5,323,2017-04-14 08:31:38,http://mohr.biz/rashawn,,192.8.66.253,"{""location"": ""JM"", ""is_mobile"": false}" 14379,5,323,2017-05-09 06:19:43,http://hauck.name/alize,,205.58.248.102,"{""location"": ""WS"", ""is_mobile"": false}" 14380,5,323,2017-03-25 22:46:34,http://jenkinsbartell.co/april,,168.170.215.3,"{""location"": ""MH"", ""is_mobile"": true}" 14381,5,323,2017-04-30 01:18:48,http://bechtelar.co/obie,,131.5.214.121,"{""location"": ""GM"", ""is_mobile"": true}" 14382,5,323,2017-03-09 05:13:42,http://mraz.net/moshe,,144.161.220.11,"{""location"": ""SA"", ""is_mobile"": false}" 14383,5,323,2017-05-01 05:54:07,http://ryanstroman.io/lura,,96.48.141.26,"{""location"": ""BR"", ""is_mobile"": false}" 14384,5,323,2017-02-28 00:35:02,http://king.co/malcolm_lowe,,58.51.8.69,"{""location"": ""BQ"", ""is_mobile"": false}" 14385,5,323,2017-04-28 05:41:45,http://tromp.biz/chelsey.davis,,241.244.5.20,"{""location"": ""AZ"", ""is_mobile"": false}" 14386,5,323,2017-01-03 22:32:22,http://weinat.com/billie_mann,,203.202.149.90,"{""location"": ""MT"", ""is_mobile"": false}" 14387,5,323,2017-04-24 14:22:59,http://reichel.com/dalton.reynolds,,134.104.240.86,"{""location"": ""SC"", ""is_mobile"": true}" 14388,5,323,2017-03-19 14:05:10,http://ornjenkins.com/isaac,,16.16.122.223,"{""location"": ""KI"", ""is_mobile"": false}" 17319,6,391,2017-03-15 08:27:09,http://lindgren.name/akeem_buckridge,,128.154.18.229,"{""location"": ""DJ"", ""is_mobile"": false}" 17320,6,391,2017-03-27 01:31:15,http://kaulke.com/karli,,87.83.247.248,"{""location"": ""HT"", ""is_mobile"": false}" 17321,6,391,2016-12-26 01:02:56,http://reillybayer.info/baby,,180.253.159.207,"{""location"": ""PH"", ""is_mobile"": false}" 17322,6,391,2017-05-12 12:41:28,http://leannonlindgren.co/abe,,183.86.222.89,"{""location"": ""LB"", ""is_mobile"": false}" 17323,6,391,2017-04-12 04:59:46,http://pfannerstillspencer.org/minerva,,159.36.129.49,"{""location"": ""MQ"", ""is_mobile"": false}" 17324,6,391,2017-04-09 09:55:59,http://tillmanweinat.info/boris,,24.79.130.9,"{""location"": ""GE"", ""is_mobile"": true}" 17325,6,391,2017-02-09 08:52:07,http://robel.net/conrad.lynch,,61.135.237.56,"{""location"": ""WF"", ""is_mobile"": true}" 17326,6,391,2017-05-04 17:46:59,http://schadenpowlowski.biz/elsie_gerhold,,193.228.33.242,"{""location"": ""AL"", ""is_mobile"": false}" 17327,6,391,2017-03-02 22:08:31,http://kunde.com/macie.carter,,100.162.99.88,"{""location"": ""DK"", ""is_mobile"": false}" 17328,6,391,2017-04-11 04:56:05,http://lindsmith.com/vivien,,238.6.150.58,"{""location"": ""IN"", ""is_mobile"": false}" 17329,6,391,2017-02-27 10:26:30,http://bartoletti.net/alayna_abbott,,112.42.153.24,"{""location"": ""MX"", ""is_mobile"": false}" 17330,6,391,2017-01-03 08:58:09,http://klingleffler.io/monroe.macejkovic,,191.225.219.110,"{""location"": ""NU"", ""is_mobile"": false}" 17331,6,391,2017-04-07 00:21:32,http://schulist.org/everardo.wolf,,207.163.49.122,"{""location"": ""AO"", ""is_mobile"": true}" 17332,6,391,2017-02-16 23:50:33,http://rueckerking.name/kamille_prosacco,,20.39.54.72,"{""location"": ""NL"", ""is_mobile"": true}" 17333,6,391,2017-04-12 05:09:11,http://wyman.biz/bradford,,45.40.7.70,"{""location"": ""SV"", ""is_mobile"": true}" 17334,6,391,2017-03-04 08:59:47,http://adams.co/kenna.parisian,,16.221.132.104,"{""location"": ""IN"", ""is_mobile"": true}" 17335,6,391,2017-06-08 11:24:11,http://tromp.com/wendell.west,,171.69.89.211,"{""location"": ""BJ"", ""is_mobile"": false}" 17336,6,391,2017-04-23 18:12:04,http://sengercremin.io/bobby_schaden,,132.86.52.92,"{""location"": ""SS"", ""is_mobile"": true}" 17337,6,391,2017-04-13 13:55:15,http://heidenreich.com/damian,,136.59.209.219,"{""location"": ""ML"", ""is_mobile"": false}" 17338,6,391,2017-04-15 08:51:51,http://spinka.info/elwyn,,145.73.222.135,"{""location"": ""JE"", ""is_mobile"": true}" 17339,6,391,2017-03-16 00:44:10,http://beier.info/nedra,,55.44.228.64,"{""location"": ""HM"", ""is_mobile"": true}" 17340,6,391,2016-12-24 04:41:09,http://jacobsonfeest.com/emie.kuvalis,,239.177.253.149,"{""location"": ""KN"", ""is_mobile"": false}" 17341,6,391,2016-12-31 01:14:57,http://vandervorttillman.name/americo,,244.187.124.125,"{""location"": ""AD"", ""is_mobile"": true}" 17342,6,391,2017-02-11 07:41:42,http://jacobijakubowski.name/jordan.rosenbaum,,82.82.149.242,"{""location"": ""VG"", ""is_mobile"": true}" 17343,6,391,2017-01-27 13:31:50,http://gutmann.net/georgianna_wuckert,,178.198.104.141,"{""location"": ""SD"", ""is_mobile"": true}" 17344,6,391,2017-06-03 06:02:32,http://andersonweimann.net/gabrielle,,252.240.171.198,"{""location"": ""MV"", ""is_mobile"": true}" 17345,6,391,2017-03-03 08:37:09,http://raynor.net/francesca,,10.225.225.143,"{""location"": ""UM"", ""is_mobile"": true}" 17346,6,391,2017-01-10 07:31:50,http://bashirian.org/darron_kuhn,,247.89.215.151,"{""location"": ""GL"", ""is_mobile"": false}" 17347,6,391,2016-12-16 00:38:27,http://wunschbaumbach.com/ethel,,197.187.133.106,"{""location"": ""SB"", ""is_mobile"": true}" 17348,6,391,2017-04-16 12:53:37,http://rowe.io/arch,,196.108.184.164,"{""location"": ""DK"", ""is_mobile"": true}" 17349,6,391,2017-06-05 07:06:30,http://cummerata.io/torrey.ruel,,103.250.56.171,"{""location"": ""VU"", ""is_mobile"": true}" 17350,6,391,2017-04-15 01:04:53,http://pfannerstillbreitenberg.net/sasha.ullrich,,186.130.175.236,"{""location"": ""AG"", ""is_mobile"": false}" 17351,6,391,2017-05-27 22:29:40,http://kunde.org/sibyl_wintheiser,,169.95.112.104,"{""location"": ""MQ"", ""is_mobile"": true}" 17352,6,391,2017-01-25 02:35:02,http://wiegandheel.io/brandyn_gutkowski,,66.58.125.197,"{""location"": ""CY"", ""is_mobile"": true}" 17353,6,391,2016-12-19 10:37:09,http://veum.co/kayleigh,,84.122.214.100,"{""location"": ""PK"", ""is_mobile"": false}" 17354,6,391,2017-05-12 23:36:31,http://spencer.io/nathen,,174.19.68.173,"{""location"": ""SM"", ""is_mobile"": false}" 17355,6,391,2017-05-19 03:05:23,http://smitham.biz/eldon,,48.71.181.65,"{""location"": ""UG"", ""is_mobile"": true}" 17356,6,391,2017-01-19 17:21:50,http://spencer.info/bradford,,14.40.203.82,"{""location"": ""TR"", ""is_mobile"": false}" 17357,6,391,2017-01-29 17:09:26,http://hayes.com/trea,,205.119.68.11,"{""location"": ""BN"", ""is_mobile"": false}" 17358,6,391,2017-05-27 16:39:09,http://kuvalis.org/giuseppe,,178.180.9.135,"{""location"": ""DK"", ""is_mobile"": false}" 17359,6,391,2017-01-15 10:01:46,http://parisianebert.io/freeman_daugherty,,54.29.204.41,"{""location"": ""SD"", ""is_mobile"": true}" 17360,6,392,2017-05-20 09:44:52,http://gerhold.co/rachelle.marquardt,,108.87.94.247,"{""location"": ""HN"", ""is_mobile"": false}" 17361,6,392,2017-04-25 07:44:25,http://kingdaugherty.org/sebastian,,220.12.47.111,"{""location"": ""VU"", ""is_mobile"": true}" 17362,6,392,2017-06-03 13:13:22,http://walter.org/taylor.pacocha,,59.106.228.33,"{""location"": ""IN"", ""is_mobile"": false}" 17363,6,392,2017-03-24 14:05:42,http://wilkinsonzemlak.name/emmie.heller,,130.15.2.193,"{""location"": ""BW"", ""is_mobile"": true}" 17364,6,392,2017-01-16 05:09:33,http://ziemannolson.co/frieda,,110.90.8.240,"{""location"": ""HT"", ""is_mobile"": true}" 17365,6,392,2017-02-07 10:15:23,http://jenkins.net/matilda,,41.12.22.61,"{""location"": ""PH"", ""is_mobile"": true}" 17366,6,392,2017-05-13 17:02:56,http://rueckeraufderhar.org/beulah_botsford,,21.103.44.171,"{""location"": ""AO"", ""is_mobile"": false}" 17367,6,392,2017-03-10 06:05:01,http://doyleoconnell.name/destiny_lesch,,110.169.9.74,"{""location"": ""LY"", ""is_mobile"": false}" 17368,6,392,2017-05-26 02:59:50,http://koch.co/gunner.luettgen,,189.153.130.133,"{""location"": ""NE"", ""is_mobile"": true}" 17369,6,392,2017-04-06 03:24:44,http://blanda.org/dwight,,63.191.176.4,"{""location"": ""DO"", ""is_mobile"": true}" 17370,6,392,2017-03-10 05:14:12,http://waelchi.com/keanu,,210.231.91.67,"{""location"": ""SL"", ""is_mobile"": false}" 17371,6,392,2017-01-19 22:09:45,http://daniel.co/carroll.rath,,88.252.175.59,"{""location"": ""IT"", ""is_mobile"": false}" 17372,6,392,2017-03-25 13:08:07,http://homenickvandervort.io/elvera_batz,,216.236.24.88,"{""location"": ""BZ"", ""is_mobile"": false}" 17373,6,392,2017-05-24 04:36:42,http://lakincruickshank.name/brant_kemmer,,76.29.130.25,"{""location"": ""FI"", ""is_mobile"": false}" 4502,2,99,2016-12-29 08:56:58,http://macejkovic.net/autumn,,175.88.125.50,"{""location"": ""LI"", ""is_mobile"": false}" 4503,2,99,2017-03-23 10:21:05,http://wizakoch.biz/eda_leannon,,40.250.193.67,"{""location"": ""SG"", ""is_mobile"": true}" 4504,2,99,2016-12-29 03:37:25,http://price.io/daren,,38.188.3.201,"{""location"": ""HT"", ""is_mobile"": false}" 4505,2,100,2017-01-13 13:07:36,http://casperstokes.io/lucius,,17.145.29.58,"{""location"": ""WS"", ""is_mobile"": true}" 4506,2,100,2017-02-23 19:40:49,http://stark.biz/elbert,,93.70.150.79,"{""location"": ""RE"", ""is_mobile"": false}" 4507,2,100,2017-03-21 10:08:49,http://herman.org/jamie,,49.35.135.225,"{""location"": ""YT"", ""is_mobile"": false}" 4508,2,100,2017-05-01 19:53:44,http://okeefe.io/vida_wintheiser,,85.155.209.80,"{""location"": ""SO"", ""is_mobile"": true}" 4509,2,100,2017-04-08 21:33:37,http://kihn.org/cynthia_metz,,213.171.169.171,"{""location"": ""IM"", ""is_mobile"": true}" 4510,2,100,2016-12-23 22:13:00,http://waelchi.io/christ.kshlerin,,83.202.15.17,"{""location"": ""CW"", ""is_mobile"": false}" 4511,2,100,2016-12-14 17:36:32,http://konopelskilind.biz/glennie.prosacco,,91.153.155.234,"{""location"": ""PR"", ""is_mobile"": false}" 4512,2,100,2017-05-10 08:16:59,http://colliersmith.biz/ayla_smitham,,44.204.55.197,"{""location"": ""IS"", ""is_mobile"": false}" 4513,2,100,2016-12-23 22:54:48,http://farrell.com/madie,,169.131.53.199,"{""location"": ""RW"", ""is_mobile"": false}" 4514,2,100,2017-02-15 13:36:07,http://goodwin.co/ila,,248.64.10.129,"{""location"": ""WS"", ""is_mobile"": true}" 4515,2,100,2017-01-31 14:05:18,http://heel.info/jany_brakus,,81.125.184.120,"{""location"": ""PR"", ""is_mobile"": true}" 4516,2,100,2017-02-11 04:14:18,http://fadel.org/kamron_herzog,,241.187.45.18,"{""location"": ""AS"", ""is_mobile"": true}" 4517,2,100,2017-02-22 02:14:06,http://keler.com/dell.berge,,119.80.218.10,"{""location"": ""KR"", ""is_mobile"": false}" 4518,2,100,2017-02-25 17:58:13,http://bogan.info/arvilla,,123.91.211.146,"{""location"": ""TV"", ""is_mobile"": true}" 4519,2,100,2017-06-13 12:09:28,http://connellyullrich.net/madyson,,214.85.174.194,"{""location"": ""TJ"", ""is_mobile"": false}" 4520,2,100,2017-03-27 13:33:51,http://zemlakkohler.biz/terrell,,70.236.185.228,"{""location"": ""BW"", ""is_mobile"": true}" 4521,2,100,2016-12-25 00:43:32,http://welchhaag.io/reyes.thompson,,125.64.185.157,"{""location"": ""VE"", ""is_mobile"": false}" 4522,2,100,2016-12-27 07:50:20,http://von.org/tyree,,236.222.115.118,"{""location"": ""CA"", ""is_mobile"": false}" 4523,2,100,2016-12-29 23:45:17,http://oconner.co/ewald.crist,,158.58.211.52,"{""location"": ""ZW"", ""is_mobile"": true}" 4524,2,100,2017-04-27 18:11:18,http://dooleypadberg.net/bobbie.strosin,,24.129.232.93,"{""location"": ""VC"", ""is_mobile"": false}" 4525,2,100,2017-05-17 04:15:46,http://kovacekkeeling.biz/kathlyn_sawayn,,183.30.223.50,"{""location"": ""DM"", ""is_mobile"": false}" 4526,2,100,2016-12-19 17:48:27,http://ondrickabreitenberg.net/preston,,133.15.22.28,"{""location"": ""HT"", ""is_mobile"": false}" 4527,2,100,2017-05-10 02:46:48,http://gislason.biz/bonnie.oconnell,,105.183.135.201,"{""location"": ""VC"", ""is_mobile"": true}" 4528,2,100,2017-05-18 14:18:01,http://beatty.com/charles,,172.248.142.213,"{""location"": ""PT"", ""is_mobile"": false}" 4529,2,100,2017-05-09 16:39:54,http://kozeyking.name/rollin,,139.169.209.93,"{""location"": ""RS"", ""is_mobile"": false}" 4530,2,100,2017-04-22 12:59:50,http://frami.org/trudie,,220.213.65.64,"{""location"": ""NA"", ""is_mobile"": false}" 4531,2,100,2016-12-15 02:19:31,http://barrows.org/gabriella.cronin,,72.139.147.193,"{""location"": ""LR"", ""is_mobile"": false}" 4532,2,100,2017-03-19 05:01:03,http://schmitt.io/aiyana_stiedemann,,170.215.30.87,"{""location"": ""LT"", ""is_mobile"": true}" 4533,2,100,2017-01-22 07:46:21,http://littelfunk.info/raoul.abbott,,222.187.21.159,"{""location"": ""DJ"", ""is_mobile"": true}" 4534,2,100,2016-12-14 06:53:42,http://walsh.org/eryn,,139.74.225.45,"{""location"": ""LT"", ""is_mobile"": false}" 4535,2,100,2016-12-13 20:25:27,http://sporer.biz/suzanne,,74.131.229.207,"{""location"": ""GR"", ""is_mobile"": true}" 4536,2,100,2017-05-14 09:16:54,http://connellylegros.biz/keely.frami,,84.156.245.33,"{""location"": ""HT"", ""is_mobile"": false}" 4537,2,100,2017-05-09 12:46:55,http://hoegerroberts.info/mabelle.runte,,130.36.241.222,"{""location"": ""HR"", ""is_mobile"": true}" 4538,2,101,2016-12-27 11:38:34,http://bins.info/ernestine,,120.156.225.18,"{""location"": ""MM"", ""is_mobile"": false}" 4539,2,101,2017-01-26 04:02:19,http://klocko.org/giovanni,,18.126.197.22,"{""location"": ""MO"", ""is_mobile"": true}" 4540,2,101,2017-05-11 04:55:23,http://hand.biz/adah,,220.138.111.102,"{""location"": ""UA"", ""is_mobile"": true}" 4541,2,101,2017-05-31 09:23:59,http://davitehr.com/noemy.torp,,225.214.138.66,"{""location"": ""IQ"", ""is_mobile"": false}" 4542,2,101,2017-05-13 22:40:28,http://keler.name/elza,,227.84.162.182,"{""location"": ""FR"", ""is_mobile"": true}" 4543,2,101,2017-01-18 05:27:07,http://langworth.com/branson.harris,,205.150.12.159,"{""location"": ""BO"", ""is_mobile"": true}" 4544,2,101,2017-04-04 13:20:40,http://hermistonboyle.io/una,,48.92.122.126,"{""location"": ""AF"", ""is_mobile"": true}" 4545,2,101,2017-02-04 03:08:33,http://waters.co/burnice_feil,,138.112.97.118,"{""location"": ""SK"", ""is_mobile"": true}" 4546,2,101,2017-02-28 22:31:17,http://trantowemmerich.name/kaela,,160.110.84.144,"{""location"": ""RE"", ""is_mobile"": true}" 4547,2,101,2017-06-08 10:13:16,http://bernhardsporer.co/kameron_marks,,60.238.71.48,"{""location"": ""SA"", ""is_mobile"": true}" 4548,2,101,2017-04-21 02:50:25,http://toy.co/samantha_lindgren,,56.238.123.146,"{""location"": ""MG"", ""is_mobile"": true}" 4549,2,101,2017-02-28 18:04:32,http://runolfon.info/loren.kuhic,,208.34.223.231,"{""location"": ""ET"", ""is_mobile"": false}" 4550,2,101,2017-04-27 17:52:22,http://mitchell.name/berneice,,144.96.224.114,"{""location"": ""LU"", ""is_mobile"": false}" 4551,2,101,2017-03-02 21:33:07,http://kutch.org/wallace,,71.222.196.61,"{""location"": ""SC"", ""is_mobile"": true}" 4552,2,101,2017-03-22 17:11:22,http://nolan.org/domenico,,151.169.89.242,"{""location"": ""NR"", ""is_mobile"": false}" 4553,2,101,2017-05-26 22:46:22,http://toy.info/colton_erdman,,186.33.184.195,"{""location"": ""MM"", ""is_mobile"": true}" 4554,2,101,2017-02-19 16:08:12,http://ullrich.biz/regan.bogan,,78.76.126.64,"{""location"": ""IQ"", ""is_mobile"": false}" 4555,2,101,2017-03-18 02:32:59,http://nikolauswiza.net/malinda,,37.163.208.78,"{""location"": ""AU"", ""is_mobile"": true}" 4556,2,101,2017-02-17 09:31:39,http://nicolas.com/eulalia,,100.243.5.194,"{""location"": ""TO"", ""is_mobile"": false}" 4557,2,101,2017-03-20 00:12:14,http://bayer.io/ariane_jast,,156.207.8.123,"{""location"": ""GS"", ""is_mobile"": false}" 11438,4,255,2017-05-12 10:07:05,http://muller.name/kariane.balistreri,,242.224.18.3,"{""location"": ""MR"", ""is_mobile"": false}" 11439,4,255,2017-04-19 12:09:22,http://schamberger.com/andreanne.becker,,86.14.154.18,"{""location"": ""HT"", ""is_mobile"": true}" 11440,4,255,2017-05-18 21:53:54,http://reingerboyle.info/okey_mcclure,,226.21.54.110,"{""location"": ""RW"", ""is_mobile"": false}" 11441,4,255,2017-02-17 21:22:39,http://schambergerwilderman.info/mona_gleichner,,76.54.147.167,"{""location"": ""TL"", ""is_mobile"": false}" 11442,4,255,2017-01-06 16:57:26,http://oconnell.net/richie,,69.177.94.254,"{""location"": ""HU"", ""is_mobile"": false}" 11443,4,255,2017-05-03 21:23:27,http://schambergerarmstrong.name/zora,,109.143.186.98,"{""location"": ""FO"", ""is_mobile"": false}" 11444,4,255,2017-06-03 17:41:15,http://jones.com/ophelia,,42.251.178.114,"{""location"": ""GQ"", ""is_mobile"": false}" 11445,4,255,2017-02-28 23:48:45,http://okuneva.net/uriel,,214.162.22.42,"{""location"": ""KG"", ""is_mobile"": false}" 11446,4,255,2016-12-18 20:30:44,http://wilkinson.info/elvera,,56.236.109.9,"{""location"": ""IR"", ""is_mobile"": true}" 11447,4,255,2017-03-14 07:36:02,http://langoshharvey.io/griffin,,202.116.24.188,"{""location"": ""PT"", ""is_mobile"": true}" 11448,4,255,2016-12-31 10:32:29,http://kemmer.co/ellsworth_cartwright,,221.194.77.45,"{""location"": ""AF"", ""is_mobile"": true}" 11449,4,255,2017-05-08 15:43:47,http://olson.com/lois,,215.73.117.101,"{""location"": ""CZ"", ""is_mobile"": false}" 11450,4,255,2017-02-03 17:40:49,http://emmerich.io/ivy,,113.43.6.92,"{""location"": ""PR"", ""is_mobile"": false}" 11451,4,255,2017-03-21 15:33:03,http://marvinjacobson.net/jairo_dietrich,,45.27.131.57,"{""location"": ""FK"", ""is_mobile"": false}" 11452,4,255,2017-05-30 12:52:10,http://schaden.io/hobart,,11.218.8.222,"{""location"": ""TL"", ""is_mobile"": true}" 11453,4,255,2017-02-25 04:03:31,http://rutherfordschiller.com/guido.borer,,238.3.138.73,"{""location"": ""CC"", ""is_mobile"": true}" 11454,4,255,2017-04-11 03:04:49,http://corwinrowe.co/cindy,,52.161.176.241,"{""location"": ""BR"", ""is_mobile"": true}" 11455,4,255,2017-06-07 08:50:17,http://howell.biz/sage,,10.113.162.18,"{""location"": ""CW"", ""is_mobile"": true}" 11456,4,255,2017-02-21 18:40:15,http://jerdegislason.net/stefanie_auer,,130.54.181.178,"{""location"": ""VN"", ""is_mobile"": false}" 11457,4,255,2017-03-23 22:58:51,http://satterfieldweimann.org/carlos,,29.59.132.86,"{""location"": ""CG"", ""is_mobile"": true}" 11458,4,255,2017-02-25 00:43:28,http://gusikowskikoch.com/melody.bode,,62.81.209.163,"{""location"": ""LY"", ""is_mobile"": false}" 11459,4,255,2017-01-20 15:17:06,http://wittingkuhic.name/virgie.jast,,20.114.128.114,"{""location"": ""NZ"", ""is_mobile"": true}" 11460,4,255,2017-02-11 07:30:47,http://nikolaus.name/kathleen,,179.148.102.246,"{""location"": ""TZ"", ""is_mobile"": false}" 11461,4,255,2017-01-26 17:30:49,http://homenick.co/virgie_romaguera,,9.221.92.154,"{""location"": ""PR"", ""is_mobile"": true}" 11462,4,255,2017-05-19 09:31:08,http://wiza.name/hilbert,,89.99.174.100,"{""location"": ""TN"", ""is_mobile"": true}" 11463,4,255,2017-01-09 11:18:44,http://bins.name/devon,,67.71.204.127,"{""location"": ""GE"", ""is_mobile"": false}" 11464,4,255,2017-01-22 03:24:27,http://wiegandmosciski.net/leonardo_barton,,175.169.192.14,"{""location"": ""TT"", ""is_mobile"": false}" 11465,4,255,2017-04-09 13:08:15,http://hilpert.info/hershel,,209.191.154.209,"{""location"": ""HK"", ""is_mobile"": true}" 11466,4,255,2017-01-06 05:51:47,http://franecki.org/danial,,220.242.210.192,"{""location"": ""KP"", ""is_mobile"": true}" 11467,4,255,2017-01-30 04:32:42,http://halvorson.org/eleonore,,42.244.168.89,"{""location"": ""KP"", ""is_mobile"": true}" 11468,4,255,2017-01-27 09:30:08,http://deckowgottlieb.io/stevie.hintz,,247.185.34.132,"{""location"": ""MU"", ""is_mobile"": true}" 11469,4,255,2016-12-15 18:47:01,http://blickbartell.net/armani_moriette,,177.224.133.55,"{""location"": ""SN"", ""is_mobile"": true}" 11470,4,255,2017-02-04 14:43:26,http://rutherford.io/alfreda,,236.51.196.152,"{""location"": ""SK"", ""is_mobile"": true}" 11471,4,255,2017-03-06 10:17:58,http://hicklebernhard.org/corrine.west,,148.244.97.118,"{""location"": ""BH"", ""is_mobile"": true}" 11472,4,255,2017-04-12 12:15:21,http://boyle.biz/reta,,111.202.254.66,"{""location"": ""AS"", ""is_mobile"": true}" 11473,4,255,2017-01-10 11:10:53,http://koeppmertz.info/danial_sporer,,152.202.246.128,"{""location"": ""PT"", ""is_mobile"": false}" 11474,4,255,2017-03-02 14:41:56,http://shields.net/jana,,249.229.16.199,"{""location"": ""IL"", ""is_mobile"": true}" 11475,4,255,2017-02-20 00:29:03,http://huel.co/nia,,123.228.115.83,"{""location"": ""TT"", ""is_mobile"": false}" 11476,4,256,2017-01-21 12:23:50,http://lefflerrippin.io/jackson,,172.168.189.222,"{""location"": ""PR"", ""is_mobile"": false}" 11477,4,256,2017-05-21 21:46:45,http://beer.name/cortney,,78.217.2.117,"{""location"": ""MR"", ""is_mobile"": false}" 11478,4,256,2017-02-10 08:32:02,http://jerde.name/ruel,,85.25.164.77,"{""location"": ""CX"", ""is_mobile"": false}" 11479,4,256,2016-12-26 04:35:11,http://nikolaupencer.name/garett,,150.160.172.117,"{""location"": ""SO"", ""is_mobile"": false}" 11480,4,256,2017-05-15 19:58:05,http://schinner.info/randall,,109.8.149.246,"{""location"": ""SI"", ""is_mobile"": false}" 11481,4,256,2017-03-20 04:57:46,http://welch.biz/evangeline.bashirian,,54.64.128.8,"{""location"": ""FM"", ""is_mobile"": true}" 11482,4,256,2017-02-09 16:36:21,http://haneturcotte.biz/cortney,,143.229.95.192,"{""location"": ""HR"", ""is_mobile"": true}" 11483,4,256,2017-01-03 22:58:14,http://wardaufderhar.info/aylin.rosenbaum,,151.130.161.150,"{""location"": ""SI"", ""is_mobile"": true}" 11484,4,256,2017-02-09 11:03:09,http://croninkoepp.biz/emily.hahn,,218.120.173.33,"{""location"": ""JO"", ""is_mobile"": true}" 11485,4,256,2017-04-17 23:01:28,http://bergstrom.biz/damian.bode,,24.21.213.38,"{""location"": ""CI"", ""is_mobile"": true}" 11486,4,256,2017-03-15 06:28:13,http://ankundingleffler.co/theresa,,99.150.241.122,"{""location"": ""TJ"", ""is_mobile"": true}" 11487,4,256,2017-01-01 17:40:34,http://gleason.net/ryann_lind,,92.67.196.235,"{""location"": ""ZA"", ""is_mobile"": false}" 11488,4,256,2017-03-17 03:36:05,http://baumbach.info/robbie.halvorson,,177.200.201.39,"{""location"": ""MC"", ""is_mobile"": false}" 11489,4,256,2017-04-26 12:56:24,http://torphy.io/kacie.runolfsdottir,,56.8.81.97,"{""location"": ""SO"", ""is_mobile"": false}" 11490,4,256,2017-01-24 03:47:01,http://abshire.org/ephraim,,161.42.166.35,"{""location"": ""LK"", ""is_mobile"": true}" 11491,4,256,2017-05-25 06:07:14,http://walsh.org/macie,,238.230.52.80,"{""location"": ""CL"", ""is_mobile"": false}" 11492,4,256,2017-05-27 07:17:52,http://jacobson.info/bobby_rolfson,,69.188.176.69,"{""location"": ""SN"", ""is_mobile"": false}" 8439,3,189,2017-03-10 03:39:41,http://joneswatsica.info/tara.lesch,,19.41.229.37,"{""location"": ""RO"", ""is_mobile"": false}" 8440,3,189,2017-05-06 18:20:53,http://ortizroob.info/jamey,,52.119.173.18,"{""location"": ""CA"", ""is_mobile"": true}" 8441,3,189,2017-03-29 03:08:51,http://bradtke.com/chelsey.hilll,,77.132.90.217,"{""location"": ""SV"", ""is_mobile"": false}" 8442,3,189,2017-03-10 16:01:37,http://stantondubuque.net/kian.macgyver,,40.47.172.180,"{""location"": ""VI"", ""is_mobile"": true}" 8443,3,189,2017-01-05 12:03:54,http://koelpin.net/brisa,,167.79.244.178,"{""location"": ""EH"", ""is_mobile"": false}" 8444,3,189,2017-01-26 12:26:30,http://runolfon.net/vena_bernhard,,250.77.42.66,"{""location"": ""IN"", ""is_mobile"": false}" 8445,3,189,2017-01-02 02:44:15,http://botsford.info/eula.abshire,,249.86.66.79,"{""location"": ""AG"", ""is_mobile"": true}" 8446,3,189,2017-05-16 11:39:23,http://fishernikolaus.co/tom.zulauf,,14.178.18.52,"{""location"": ""ET"", ""is_mobile"": true}" 8447,3,189,2017-03-19 20:54:48,http://lesch.info/kayla,,70.216.227.184,"{""location"": ""CR"", ""is_mobile"": false}" 8448,3,189,2017-02-04 22:17:27,http://luettgen.com/roselyn,,247.149.139.180,"{""location"": ""HU"", ""is_mobile"": true}" 8449,3,189,2017-03-21 22:05:58,http://reinger.org/carmella_hamill,,106.193.27.120,"{""location"": ""TC"", ""is_mobile"": true}" 8450,3,189,2017-04-08 19:05:21,http://rolfsondare.info/brandon,,3.13.100.34,"{""location"": ""ZA"", ""is_mobile"": true}" 8451,3,189,2017-04-12 00:12:51,http://ward.net/humberto,,28.108.35.220,"{""location"": ""CK"", ""is_mobile"": false}" 8452,3,189,2017-03-02 15:22:26,http://oconnell.info/hortense.herzog,,92.218.227.49,"{""location"": ""DJ"", ""is_mobile"": true}" 8453,3,189,2016-12-16 10:41:49,http://bogan.biz/isobel_mueller,,120.207.227.86,"{""location"": ""FI"", ""is_mobile"": false}" 8454,3,189,2017-05-02 21:23:43,http://cruickshank.org/frieda,,216.26.109.63,"{""location"": ""BQ"", ""is_mobile"": true}" 8455,3,189,2016-12-31 22:06:17,http://lakin.net/london.prohaska,,185.221.8.106,"{""location"": ""MM"", ""is_mobile"": false}" 8456,3,189,2017-01-18 20:00:35,http://weberkling.io/anna,,59.248.48.189,"{""location"": ""AG"", ""is_mobile"": false}" 8457,3,190,2017-01-21 08:06:07,http://wisokyspencer.com/graham.lindgren,,136.107.159.116,"{""location"": ""UG"", ""is_mobile"": true}" 8458,3,190,2017-04-06 15:16:46,http://wisoky.info/kirstin,,6.175.81.5,"{""location"": ""FJ"", ""is_mobile"": true}" 8459,3,190,2017-01-27 16:46:12,http://breitenberg.io/jeffery.doyle,,129.222.235.173,"{""location"": ""MY"", ""is_mobile"": true}" 8460,3,190,2017-05-25 22:54:08,http://jonesroob.io/darian,,147.54.169.218,"{""location"": ""MO"", ""is_mobile"": true}" 8461,3,190,2017-04-26 20:36:55,http://larson.io/noe,,232.248.163.153,"{""location"": ""TM"", ""is_mobile"": true}" 8462,3,190,2017-02-20 16:06:01,http://hyatthaag.co/julia,,93.238.6.177,"{""location"": ""SG"", ""is_mobile"": false}" 8463,3,190,2017-01-22 04:36:40,http://padberg.name/garfield.ebert,,64.189.155.78,"{""location"": ""VG"", ""is_mobile"": true}" 8464,3,190,2017-01-09 01:24:56,http://runolfsdottir.org/tiara_kuphal,,42.56.10.5,"{""location"": ""HU"", ""is_mobile"": true}" 8465,3,190,2017-02-23 20:47:33,http://rempelfunk.biz/elsie_lind,,106.100.128.234,"{""location"": ""TL"", ""is_mobile"": true}" 8466,3,190,2017-04-20 20:57:57,http://oreillyboyer.com/elvie,,235.49.218.139,"{""location"": ""SV"", ""is_mobile"": true}" 8467,3,190,2017-02-11 00:36:07,http://pfannerstillbeier.io/elmore,,141.241.25.245,"{""location"": ""MO"", ""is_mobile"": false}" 8468,3,190,2017-04-14 15:43:26,http://okeefe.net/shanel,,245.12.14.105,"{""location"": ""CK"", ""is_mobile"": true}" 8469,3,190,2016-12-21 04:14:59,http://nikolausgislason.net/fae,,44.84.171.73,"{""location"": ""BG"", ""is_mobile"": true}" 8470,3,190,2017-06-06 02:36:50,http://krajcik.org/jameson,,140.234.146.218,"{""location"": ""VA"", ""is_mobile"": false}" 8471,3,190,2017-01-22 12:26:00,http://nicolas.io/antwon.kunze,,146.2.244.137,"{""location"": ""AQ"", ""is_mobile"": false}" 8472,3,190,2017-02-06 06:38:50,http://wolf.co/melya_macejkovic,,37.101.21.89,"{""location"": ""GB"", ""is_mobile"": false}" 8473,3,190,2017-03-13 14:13:00,http://wiegand.biz/keyon,,18.153.92.138,"{""location"": ""IL"", ""is_mobile"": false}" 8474,3,190,2016-12-13 15:22:29,http://marks.co/liza.johnston,,205.101.209.203,"{""location"": ""PR"", ""is_mobile"": true}" 8475,3,190,2017-03-25 19:19:46,http://harberankunding.io/jeramy_corwin,,135.236.210.23,"{""location"": ""GQ"", ""is_mobile"": false}" 8476,3,190,2017-04-14 11:14:05,http://cronamann.co/laria,,226.89.140.188,"{""location"": ""MR"", ""is_mobile"": true}" 8477,3,191,2017-02-14 17:48:46,http://homenick.co/eladio.schuster,,94.224.160.91,"{""location"": ""CV"", ""is_mobile"": true}" 8478,3,191,2017-03-25 23:50:22,http://heller.com/giovanna_little,,169.139.198.68,"{""location"": ""LC"", ""is_mobile"": false}" 8479,3,191,2017-04-14 08:20:04,http://lehner.info/hailey_kunze,,147.66.132.147,"{""location"": ""YE"", ""is_mobile"": false}" 8480,3,191,2017-02-27 14:32:05,http://thieldonnelly.org/josie,,74.136.111.178,"{""location"": ""MA"", ""is_mobile"": true}" 8481,3,191,2017-01-21 08:54:19,http://ziemann.co/grayson,,125.119.10.181,"{""location"": ""PY"", ""is_mobile"": false}" 8482,3,191,2017-05-11 13:11:45,http://wuckert.biz/nelda.schmitt,,202.95.47.193,"{""location"": ""TN"", ""is_mobile"": true}" 8483,3,191,2017-03-30 07:54:23,http://franeckiherzog.biz/bret,,64.245.18.206,"{""location"": ""GQ"", ""is_mobile"": true}" 8484,3,191,2017-03-25 14:53:35,http://larson.net/tania,,234.27.162.95,"{""location"": ""CH"", ""is_mobile"": true}" 8485,3,191,2016-12-17 01:38:08,http://nitzsche.io/gabriel,,163.136.196.230,"{""location"": ""DK"", ""is_mobile"": true}" 8486,3,191,2017-03-16 07:06:31,http://uptonankunding.info/peyton,,164.26.233.135,"{""location"": ""MA"", ""is_mobile"": false}" 8487,3,191,2017-04-10 01:37:59,http://handweinat.info/gavin.blanda,,139.238.118.67,"{""location"": ""GP"", ""is_mobile"": true}" 8488,3,191,2017-01-09 07:38:45,http://sporerschoen.name/jesus,,159.22.137.152,"{""location"": ""SK"", ""is_mobile"": true}" 8489,3,191,2017-05-22 19:59:23,http://johnson.info/kaleigh_schmidt,,149.171.105.77,"{""location"": ""BA"", ""is_mobile"": false}" 8490,3,191,2017-01-19 18:36:24,http://mclaughlinhayes.net/lisette.bednar,,171.20.129.133,"{""location"": ""SM"", ""is_mobile"": true}" 8491,3,191,2017-05-29 18:35:55,http://mclaughlin.biz/christ_beatty,,3.40.108.226,"{""location"": ""AF"", ""is_mobile"": true}" 8492,3,191,2017-01-15 22:43:13,http://cummerata.name/rudy.conn,,86.76.164.81,"{""location"": ""GF"", ""is_mobile"": true}" 8493,3,191,2016-12-22 12:21:19,http://mohr.net/clark_schimmel,,114.17.149.204,"{""location"": ""GH"", ""is_mobile"": true}" 8494,3,191,2017-05-12 12:49:32,http://labadie.io/garth.kuhlman,,211.5.116.88,"{""location"": ""MC"", ""is_mobile"": false}" 14389,5,323,2017-04-29 11:43:03,http://ziemannleuschke.net/lavern_kuphal,,76.118.154.95,"{""location"": ""MR"", ""is_mobile"": true}" 14390,5,323,2017-04-09 18:27:15,http://schroeder.net/deontae,,90.239.80.224,"{""location"": ""SI"", ""is_mobile"": false}" 14391,5,323,2017-02-21 18:00:55,http://bayerbruen.org/albert_williamson,,17.115.163.90,"{""location"": ""GP"", ""is_mobile"": false}" 14392,5,323,2017-05-16 19:20:41,http://maggio.io/julianne_volkman,,82.17.217.232,"{""location"": ""ML"", ""is_mobile"": true}" 14393,5,324,2017-02-09 12:46:49,http://olson.org/godfrey,,7.98.170.254,"{""location"": ""TR"", ""is_mobile"": true}" 14394,5,324,2017-03-25 07:01:48,http://kshlerin.info/hailey,,52.216.218.251,"{""location"": ""SD"", ""is_mobile"": false}" 14395,5,324,2017-02-05 14:20:06,http://prohaskamueller.biz/emmie,,251.93.157.251,"{""location"": ""BE"", ""is_mobile"": true}" 14396,5,324,2017-03-08 04:42:43,http://koepprobel.com/dante_pagac,,54.185.188.131,"{""location"": ""AL"", ""is_mobile"": true}" 14397,5,324,2017-02-24 04:24:17,http://roberts.org/keyon.borer,,133.248.99.254,"{""location"": ""GY"", ""is_mobile"": false}" 14398,5,324,2017-02-08 23:20:11,http://hane.io/samantha,,46.177.14.15,"{""location"": ""TO"", ""is_mobile"": true}" 14399,5,324,2017-03-30 23:27:14,http://mcglynn.info/stephany_grady,,153.205.84.46,"{""location"": ""PL"", ""is_mobile"": false}" 14400,5,324,2017-03-25 17:56:15,http://wittinghamill.info/ezekiel.schiller,,94.23.114.132,"{""location"": ""AD"", ""is_mobile"": false}" 14401,5,324,2017-04-24 18:57:27,http://fisherjacobi.name/ena,,104.36.108.185,"{""location"": ""TT"", ""is_mobile"": true}" 14402,5,324,2016-12-20 04:30:39,http://denesik.org/bud_cain,,154.226.221.37,"{""location"": ""ES"", ""is_mobile"": true}" 14403,5,324,2017-03-23 11:55:55,http://gleason.name/paula.huel,,231.99.41.237,"{""location"": ""PK"", ""is_mobile"": true}" 14404,5,324,2017-02-04 17:01:43,http://gottlieb.co/riley,,187.118.205.144,"{""location"": ""LU"", ""is_mobile"": true}" 14405,5,324,2017-04-25 03:32:37,http://nader.com/jonatan.kulas,,129.16.252.65,"{""location"": ""TO"", ""is_mobile"": true}" 14406,5,324,2017-05-22 18:24:11,http://keeblerdicki.org/ernest,,163.209.95.151,"{""location"": ""BT"", ""is_mobile"": false}" 14407,5,324,2017-03-18 20:31:07,http://oconnell.biz/torey_schuppe,,66.121.142.111,"{""location"": ""TM"", ""is_mobile"": true}" 14408,5,324,2016-12-19 06:25:08,http://okeefecummerata.co/hertha,,147.44.37.159,"{""location"": ""GF"", ""is_mobile"": false}" 14409,5,324,2017-05-11 06:57:49,http://trantow.net/jermaine_feil,,253.50.220.167,"{""location"": ""CI"", ""is_mobile"": true}" 14410,5,324,2016-12-23 11:38:11,http://greenholtkuphal.io/sharon,,93.29.176.204,"{""location"": ""MV"", ""is_mobile"": false}" 14411,5,324,2017-04-23 22:38:40,http://nader.net/marion_zieme,,96.193.194.173,"{""location"": ""BJ"", ""is_mobile"": true}" 14412,5,324,2017-03-13 06:02:00,http://bahringer.io/mitchell.weimann,,224.194.238.139,"{""location"": ""MR"", ""is_mobile"": true}" 14413,5,324,2017-06-09 04:30:23,http://hahnpagac.org/clay,,99.204.23.203,"{""location"": ""GR"", ""is_mobile"": false}" 14414,5,324,2017-05-31 04:52:52,http://farrell.co/koby_kreiger,,129.177.253.138,"{""location"": ""LS"", ""is_mobile"": false}" 14415,5,324,2017-03-18 00:03:15,http://flatleycummerata.biz/zetta,,163.247.206.252,"{""location"": ""TJ"", ""is_mobile"": false}" 14416,5,324,2017-03-29 10:20:11,http://schoen.org/adella_renner,,25.220.119.115,"{""location"": ""PR"", ""is_mobile"": false}" 14417,5,324,2017-04-18 08:13:37,http://braun.biz/leonora.waters,,216.126.118.30,"{""location"": ""MM"", ""is_mobile"": true}" 14418,5,324,2017-04-22 22:06:14,http://bayer.biz/amina.pfeffer,,208.95.155.235,"{""location"": ""CX"", ""is_mobile"": false}" 14419,5,324,2017-04-15 20:28:32,http://hermistonmertz.co/martine,,225.156.59.20,"{""location"": ""VE"", ""is_mobile"": true}" 14420,5,324,2017-06-03 21:32:43,http://schmitt.org/jackie.bashirian,,241.188.216.173,"{""location"": ""VU"", ""is_mobile"": false}" 14421,5,324,2017-01-15 20:57:24,http://jast.info/jennyfer,,243.6.72.201,"{""location"": ""NU"", ""is_mobile"": true}" 14422,5,324,2017-02-26 17:59:14,http://collinshowell.io/melia,,84.168.120.83,"{""location"": ""HK"", ""is_mobile"": false}" 14423,5,324,2017-01-14 18:26:46,http://ruecker.com/earl,,29.139.85.81,"{""location"": ""JM"", ""is_mobile"": false}" 14424,5,324,2017-05-04 22:53:00,http://lesch.net/noemi,,110.51.42.217,"{""location"": ""TF"", ""is_mobile"": true}" 14425,5,324,2017-02-24 03:02:12,http://dooleygrimes.net/seth,,242.171.119.196,"{""location"": ""RU"", ""is_mobile"": true}" 14426,5,324,2017-02-26 21:30:02,http://armstrong.com/eveline_dickens,,51.210.220.113,"{""location"": ""PN"", ""is_mobile"": true}" 14427,5,324,2017-05-27 08:27:24,http://braun.org/nicole.kerluke,,45.123.17.82,"{""location"": ""BV"", ""is_mobile"": true}" 14428,5,324,2017-04-21 07:14:29,http://koelpin.co/elya,,73.212.88.20,"{""location"": ""KH"", ""is_mobile"": true}" 14429,5,324,2017-06-03 06:31:29,http://torphydaniel.net/jamel_wintheiser,,120.160.64.147,"{""location"": ""RW"", ""is_mobile"": false}" 14430,5,324,2017-03-03 06:11:57,http://sporeradams.info/anais_boyer,,36.189.240.208,"{""location"": ""LU"", ""is_mobile"": true}" 14431,5,324,2017-03-16 20:33:29,http://keeblerdamore.biz/sarah.frami,,40.197.166.140,"{""location"": ""CW"", ""is_mobile"": true}" 14432,5,324,2016-12-28 15:47:21,http://brakusmuller.name/jada_schimmel,,210.68.182.183,"{""location"": ""CO"", ""is_mobile"": true}" 14433,5,324,2017-02-17 09:42:41,http://corkerynikolaus.org/garry,,77.145.35.85,"{""location"": ""MA"", ""is_mobile"": false}" 14434,5,324,2017-02-11 08:45:26,http://gusikowski.com/moie_mckenzie,,144.46.142.128,"{""location"": ""KY"", ""is_mobile"": false}" 14435,5,325,2017-04-03 13:06:36,http://beahan.co/arturo,,49.76.141.147,"{""location"": ""PF"", ""is_mobile"": true}" 14436,5,325,2017-05-20 05:23:27,http://wuckert.net/nicklaus_bergstrom,,62.109.50.11,"{""location"": ""BI"", ""is_mobile"": false}" 14437,5,325,2017-04-20 06:07:48,http://hegmannnitzsche.com/birdie,,45.181.41.22,"{""location"": ""DK"", ""is_mobile"": true}" 14438,5,325,2017-03-24 21:27:36,http://conroy.com/michaela_mitchell,,102.127.58.156,"{""location"": ""BI"", ""is_mobile"": false}" 14439,5,325,2017-02-23 11:45:30,http://walker.io/hazle,,45.147.168.211,"{""location"": ""TW"", ""is_mobile"": false}" 14440,5,325,2017-03-18 02:04:40,http://wisozk.info/theo,,133.139.196.169,"{""location"": ""CM"", ""is_mobile"": true}" 14441,5,325,2017-03-26 10:36:30,http://klockokuhn.biz/skylar,,94.82.18.96,"{""location"": ""IS"", ""is_mobile"": false}" 14442,5,325,2017-04-01 23:18:50,http://lind.info/eldred,,21.145.244.79,"{""location"": ""KI"", ""is_mobile"": true}" 14443,5,325,2017-02-21 10:03:13,http://klocko.biz/cathrine,,187.169.33.62,"{""location"": ""GG"", ""is_mobile"": false}" 17374,6,392,2017-06-02 20:36:12,http://hackett.org/madison_mohr,,250.96.203.95,"{""location"": ""BL"", ""is_mobile"": false}" 17375,6,392,2017-02-27 22:55:02,http://goodwin.name/jody_medhurst,,39.198.233.12,"{""location"": ""UZ"", ""is_mobile"": true}" 17376,6,392,2017-01-21 04:10:15,http://danielhuels.com/pattie_shanahan,,117.179.68.213,"{""location"": ""FM"", ""is_mobile"": true}" 17377,6,392,2017-01-26 23:53:00,http://gerlach.com/lenny.green,,176.210.44.40,"{""location"": ""SE"", ""is_mobile"": false}" 17378,6,392,2017-02-05 18:10:20,http://jerdeledner.org/garrett_schamberger,,33.228.50.45,"{""location"": ""MT"", ""is_mobile"": true}" 17379,6,392,2017-05-17 11:41:59,http://douglastromp.io/darrick,,149.168.213.240,"{""location"": ""AT"", ""is_mobile"": true}" 17380,6,392,2017-02-12 09:48:57,http://swiftoberbrunner.co/hans,,107.186.119.250,"{""location"": ""AZ"", ""is_mobile"": false}" 17381,6,392,2017-02-22 18:53:14,http://runolfonmckenzie.io/murl.ankunding,,207.206.185.173,"{""location"": ""TZ"", ""is_mobile"": false}" 17382,6,392,2017-05-01 22:14:09,http://dickinsonraynor.org/crystal_moen,,218.243.12.33,"{""location"": ""YE"", ""is_mobile"": false}" 17383,6,392,2017-05-10 05:05:00,http://goldnersmith.info/trystan,,245.25.18.252,"{""location"": ""CF"", ""is_mobile"": false}" 17384,6,392,2017-05-10 17:33:08,http://hilpert.io/tianna_veum,,49.204.99.67,"{""location"": ""SD"", ""is_mobile"": true}" 17385,6,392,2017-04-20 12:44:21,http://reillychamplin.io/wilber_champlin,,120.181.151.136,"{""location"": ""NP"", ""is_mobile"": false}" 17386,6,392,2017-02-28 00:17:52,http://hudson.net/julien,,72.193.143.140,"{""location"": ""UA"", ""is_mobile"": true}" 17387,6,392,2017-03-30 10:43:53,http://howegutmann.co/vivianne_morar,,204.214.221.138,"{""location"": ""LR"", ""is_mobile"": false}" 17388,6,392,2017-06-12 20:31:16,http://oberbrunner.info/pearl,,141.232.198.66,"{""location"": ""MN"", ""is_mobile"": false}" 17389,6,392,2017-04-05 18:21:02,http://jenkinsziemann.name/al.gorczany,,119.216.216.194,"{""location"": ""GU"", ""is_mobile"": true}" 17390,6,392,2017-03-10 20:40:05,http://hicklestamm.com/lenny,,195.189.59.201,"{""location"": ""FO"", ""is_mobile"": true}" 17391,6,392,2017-02-12 13:58:31,http://bauchvon.co/bill_okeefe,,120.30.66.29,"{""location"": ""PS"", ""is_mobile"": false}" 17392,6,392,2017-01-08 18:46:50,http://huel.info/maud_hackett,,197.196.244.86,"{""location"": ""MD"", ""is_mobile"": true}" 17393,6,392,2017-01-20 11:36:47,http://adams.org/vicenta,,162.136.67.244,"{""location"": ""ZA"", ""is_mobile"": true}" 17394,6,392,2017-04-03 10:21:59,http://schultz.com/baron_waelchi,,175.221.148.164,"{""location"": ""BT"", ""is_mobile"": true}" 17395,6,392,2017-06-04 14:02:49,http://gerlachwolf.info/oleta,,182.117.131.49,"{""location"": ""CC"", ""is_mobile"": false}" 17396,6,392,2017-03-25 23:14:32,http://jacobi.biz/elsie,,15.140.185.122,"{""location"": ""US"", ""is_mobile"": false}" 17397,6,392,2017-06-12 19:41:07,http://kovacek.com/telly_dare,,83.246.144.65,"{""location"": ""VE"", ""is_mobile"": true}" 17398,6,392,2017-06-10 07:46:49,http://purdyosinski.io/brain_flatley,,249.61.180.188,"{""location"": ""MY"", ""is_mobile"": true}" 17399,6,392,2017-02-28 02:22:51,http://price.info/madison_jast,,33.174.115.77,"{""location"": ""NU"", ""is_mobile"": false}" 17400,6,392,2017-01-16 14:34:32,http://bernhardkeeling.co/rudolph_towne,,140.57.239.95,"{""location"": ""RO"", ""is_mobile"": false}" 17401,6,392,2017-03-16 09:08:58,http://hamill.co/caterina.crooks,,112.235.176.79,"{""location"": ""AR"", ""is_mobile"": false}" 17402,6,392,2017-05-13 16:39:56,http://auerbayer.name/carolyn_mitchell,,30.54.145.141,"{""location"": ""HT"", ""is_mobile"": true}" 17403,6,392,2017-01-17 19:09:37,http://vonrueden.name/belle_schmeler,,192.91.4.15,"{""location"": ""VC"", ""is_mobile"": true}" 17404,6,392,2017-06-13 13:12:32,http://bergstrom.org/ana,,82.211.213.81,"{""location"": ""TO"", ""is_mobile"": false}" 17405,6,392,2017-04-26 11:07:08,http://hane.info/charles_pagac,,63.56.217.240,"{""location"": ""CC"", ""is_mobile"": true}" 17406,6,392,2016-12-28 03:34:31,http://connelly.com/jude_lind,,204.112.175.107,"{""location"": ""NA"", ""is_mobile"": false}" 17407,6,392,2017-01-23 21:38:17,http://schmidt.info/frederique,,83.209.73.231,"{""location"": ""GI"", ""is_mobile"": false}" 17408,6,392,2017-01-11 12:03:50,http://hagenes.name/reese,,158.67.8.146,"{""location"": ""KP"", ""is_mobile"": true}" 17409,6,392,2017-02-18 02:30:50,http://konopelski.io/sheldon.zemlak,,5.73.104.42,"{""location"": ""LA"", ""is_mobile"": true}" 17410,6,392,2017-01-02 20:30:25,http://wisozkbraun.info/emma_huel,,248.68.217.104,"{""location"": ""NC"", ""is_mobile"": true}" 17411,6,392,2017-04-13 22:29:53,http://nolan.co/timothy.schmitt,,83.117.57.154,"{""location"": ""RE"", ""is_mobile"": false}" 17412,6,392,2017-01-01 08:53:09,http://grant.name/zelda.ward,,5.199.237.135,"{""location"": ""PK"", ""is_mobile"": false}" 17413,6,392,2017-01-05 07:35:20,http://turcotte.name/rocio.hodkiewicz,,98.96.14.115,"{""location"": ""SK"", ""is_mobile"": true}" 17414,6,392,2017-04-08 22:50:16,http://lesch.com/carolina_langosh,,137.14.168.110,"{""location"": ""TM"", ""is_mobile"": false}" 17415,6,392,2017-02-26 05:26:11,http://emmerichrolfson.info/colin,,243.173.90.35,"{""location"": ""AO"", ""is_mobile"": false}" 17416,6,392,2017-01-02 16:37:12,http://ledner.io/myrtie_schamberger,,68.146.75.61,"{""location"": ""JE"", ""is_mobile"": false}" 17418,6,392,2017-03-19 15:12:06,http://leuschke.net/dean.dickinson,,90.241.55.222,"{""location"": ""IM"", ""is_mobile"": false}" 17419,6,392,2017-04-10 04:00:31,http://keelingcole.com/ardith,,167.84.217.81,"{""location"": ""BD"", ""is_mobile"": false}" 17420,6,393,2017-03-02 15:38:05,http://mclaughlingaylord.com/wilhelm.brakus,,40.239.176.171,"{""location"": ""CR"", ""is_mobile"": true}" 17421,6,393,2017-03-11 18:22:22,http://auerwelch.biz/josue,,111.21.194.78,"{""location"": ""BZ"", ""is_mobile"": true}" 17422,6,393,2017-04-27 05:30:23,http://gutkowski.biz/emerson.huel,,23.118.90.66,"{""location"": ""NC"", ""is_mobile"": true}" 17423,6,393,2017-05-14 19:04:06,http://mayert.io/evangeline,,130.121.120.213,"{""location"": ""PS"", ""is_mobile"": false}" 17424,6,393,2017-02-06 07:24:16,http://marquardtcrooks.org/kari_nicolas,,14.127.254.131,"{""location"": ""NC"", ""is_mobile"": true}" 17425,6,393,2017-05-18 23:54:30,http://hodkiewicz.biz/izaiah,,54.58.23.193,"{""location"": ""NA"", ""is_mobile"": false}" 17426,6,393,2017-04-16 00:47:39,http://schultz.co/itzel.gottlieb,,228.210.201.159,"{""location"": ""AQ"", ""is_mobile"": false}" 17427,6,393,2017-01-21 01:51:15,http://witting.info/zander,,149.244.81.119,"{""location"": ""BQ"", ""is_mobile"": true}" 17428,6,393,2017-03-25 06:33:13,http://jast.biz/rosario_stanton,,65.156.109.16,"{""location"": ""HT"", ""is_mobile"": true}" 4558,2,101,2017-06-06 07:50:12,http://feilprosacco.io/abby,,96.233.23.219,"{""location"": ""GW"", ""is_mobile"": false}" 4559,2,101,2017-04-03 01:52:59,http://fahey.com/paolo,,114.88.130.82,"{""location"": ""SV"", ""is_mobile"": false}" 4560,2,101,2017-05-20 10:39:31,http://caingutkowski.co/stephania_shanahan,,25.88.109.30,"{""location"": ""TG"", ""is_mobile"": true}" 4561,2,101,2017-03-03 11:58:01,http://mclaughlinhilpert.info/olin.nitzsche,,111.223.155.188,"{""location"": ""LR"", ""is_mobile"": false}" 4562,2,101,2017-05-19 22:51:27,http://beier.co/porter,,37.23.89.112,"{""location"": ""GS"", ""is_mobile"": true}" 4563,2,101,2017-04-20 14:04:09,http://schowalterhilpert.info/layne,,206.66.105.66,"{""location"": ""QA"", ""is_mobile"": true}" 4564,2,101,2017-06-10 18:48:29,http://hicklewisoky.co/okey,,152.231.114.234,"{""location"": ""PS"", ""is_mobile"": false}" 4565,2,101,2017-01-16 20:47:22,http://hodkiewicz.biz/jarret,,231.96.230.41,"{""location"": ""NC"", ""is_mobile"": true}" 4566,2,101,2017-03-05 03:51:16,http://botsfordmayert.name/bonita,,53.2.162.241,"{""location"": ""LU"", ""is_mobile"": true}" 4567,2,101,2017-01-01 02:11:07,http://bednarboehm.net/margret,,100.4.49.192,"{""location"": ""UM"", ""is_mobile"": true}" 4568,2,101,2017-05-03 18:21:59,http://stoltenberg.io/aida.connelly,,235.115.179.92,"{""location"": ""KR"", ""is_mobile"": false}" 4569,2,101,2017-04-23 01:28:44,http://parker.net/emiliano,,103.34.227.221,"{""location"": ""GL"", ""is_mobile"": true}" 4570,2,101,2017-04-22 09:55:17,http://romaguera.name/tremaine_hyatt,,225.216.55.100,"{""location"": ""KY"", ""is_mobile"": false}" 4571,2,101,2017-03-15 12:12:45,http://wolff.co/jimmy.prohaska,,219.20.93.50,"{""location"": ""HM"", ""is_mobile"": true}" 4572,2,101,2017-01-17 16:13:53,http://okeefe.net/laverne,,63.158.79.67,"{""location"": ""BY"", ""is_mobile"": true}" 4573,2,101,2017-04-24 23:36:09,http://bartell.org/cary_doyle,,138.159.38.233,"{""location"": ""IS"", ""is_mobile"": false}" 4574,2,102,2017-04-22 04:37:08,http://mertzsenger.co/levi_schmidt,,180.83.232.161,"{""location"": ""MO"", ""is_mobile"": true}" 4575,2,102,2017-02-23 21:19:59,http://danielmccullough.info/roy_hackett,,146.136.88.65,"{""location"": ""GS"", ""is_mobile"": true}" 4576,2,102,2017-05-22 02:49:50,http://denesikjakubowski.biz/lexie,,177.200.236.59,"{""location"": ""KI"", ""is_mobile"": false}" 4577,2,102,2017-05-24 01:59:09,http://beahanoberbrunner.com/taylor.walsh,,3.90.231.189,"{""location"": ""GD"", ""is_mobile"": true}" 4578,2,102,2017-03-26 16:28:25,http://kling.org/vilma,,185.50.123.190,"{""location"": ""MP"", ""is_mobile"": true}" 4579,2,102,2017-04-24 00:48:12,http://baumbach.org/estrella.schamberger,,159.150.214.25,"{""location"": ""CY"", ""is_mobile"": false}" 4580,2,102,2016-12-18 13:27:02,http://mclaughlin.io/caesar.considine,,89.160.113.106,"{""location"": ""SV"", ""is_mobile"": true}" 4581,2,102,2017-06-10 15:44:31,http://tillman.io/nyah_bauch,,210.212.239.40,"{""location"": ""ZA"", ""is_mobile"": true}" 4582,2,102,2017-06-07 06:41:39,http://waters.io/hilton,,60.96.179.176,"{""location"": ""ES"", ""is_mobile"": true}" 4583,2,102,2016-12-29 20:42:13,http://friesen.co/otilia,,2.38.88.107,"{""location"": ""IL"", ""is_mobile"": false}" 4584,2,102,2017-02-02 00:06:34,http://hackett.biz/marcelino,,26.215.73.2,"{""location"": ""NI"", ""is_mobile"": false}" 4585,2,102,2017-04-20 15:19:03,http://heaneyromaguera.info/trystan.brakus,,248.62.120.57,"{""location"": ""CL"", ""is_mobile"": true}" 4586,2,102,2017-03-31 14:42:16,http://denesik.name/demario,,82.220.158.239,"{""location"": ""ZW"", ""is_mobile"": false}" 4587,2,102,2017-04-18 07:45:20,http://rathbauch.io/lynn,,218.227.141.176,"{""location"": ""OM"", ""is_mobile"": false}" 4588,2,102,2017-05-18 04:19:28,http://morar.net/milan.windler,,198.165.116.161,"{""location"": ""PT"", ""is_mobile"": true}" 4589,2,102,2017-03-30 16:35:01,http://weimannturner.co/floy_senger,,24.55.203.124,"{""location"": ""EE"", ""is_mobile"": false}" 4590,2,102,2017-02-25 08:37:16,http://robelrolfson.name/maeve.watsica,,3.245.128.99,"{""location"": ""GT"", ""is_mobile"": false}" 4591,2,102,2017-01-17 19:19:33,http://yost.co/gia,,104.201.28.103,"{""location"": ""CO"", ""is_mobile"": false}" 4592,2,102,2017-06-10 01:00:34,http://turnerkihn.io/bria_harris,,54.85.133.254,"{""location"": ""TR"", ""is_mobile"": true}" 4593,2,102,2017-03-13 03:14:17,http://stokes.org/adella_erdman,,164.62.95.70,"{""location"": ""VG"", ""is_mobile"": false}" 4594,2,102,2017-04-02 04:55:32,http://abernathy.com/jey_schoen,,145.22.227.68,"{""location"": ""MR"", ""is_mobile"": true}" 4595,2,102,2017-05-15 05:57:06,http://lubowitzgulgowski.name/consuelo,,81.214.204.175,"{""location"": ""LI"", ""is_mobile"": true}" 4596,2,102,2017-05-05 23:23:55,http://wisoky.org/dell.lueilwitz,,252.102.239.245,"{""location"": ""JE"", ""is_mobile"": true}" 4597,2,102,2017-04-25 09:28:09,http://mcglynn.com/bethany,,198.101.151.20,"{""location"": ""CW"", ""is_mobile"": true}" 4598,2,102,2017-05-18 13:00:19,http://langhickle.com/llewellyn.keler,,187.232.220.88,"{""location"": ""GL"", ""is_mobile"": true}" 4599,2,102,2017-04-09 02:41:21,http://dubuque.net/evans.zboncak,,50.81.125.194,"{""location"": ""TD"", ""is_mobile"": false}" 4600,2,102,2017-03-19 20:57:01,http://gerlach.name/neha.schuppe,,39.99.229.248,"{""location"": ""EG"", ""is_mobile"": false}" 4601,2,102,2017-04-14 07:45:32,http://kautzer.biz/annalise.dicki,,28.230.107.18,"{""location"": ""LV"", ""is_mobile"": false}" 4602,2,102,2017-04-03 08:32:10,http://wintheiser.info/fabian.conroy,,4.202.252.17,"{""location"": ""IO"", ""is_mobile"": false}" 4603,2,102,2017-01-16 07:58:07,http://jacobi.org/chad.barrows,,154.218.85.147,"{""location"": ""LV"", ""is_mobile"": false}" 4604,2,102,2017-05-11 09:40:51,http://boyle.co/judah,,64.118.19.171,"{""location"": ""PN"", ""is_mobile"": false}" 4605,2,102,2017-04-09 21:01:53,http://moen.name/devon,,135.40.207.120,"{""location"": ""TR"", ""is_mobile"": false}" 4606,2,102,2017-01-20 16:22:25,http://schoen.name/loyal,,100.227.45.71,"{""location"": ""JP"", ""is_mobile"": true}" 4607,2,102,2017-02-28 01:22:39,http://mccullough.name/alayna,,134.103.51.208,"{""location"": ""VE"", ""is_mobile"": false}" 4608,2,102,2017-01-23 02:43:43,http://greenfelderkoelpin.io/noah.wintheiser,,246.20.157.127,"{""location"": ""EG"", ""is_mobile"": true}" 4609,2,102,2017-03-25 13:52:36,http://keler.com/lisandro.jacobi,,195.220.105.135,"{""location"": ""ES"", ""is_mobile"": true}" 4610,2,102,2017-05-04 03:17:22,http://emmerich.io/verner,,226.96.22.13,"{""location"": ""ZA"", ""is_mobile"": true}" 4611,2,102,2017-01-08 12:08:12,http://stroman.io/amos,,234.38.32.5,"{""location"": ""TF"", ""is_mobile"": false}" 4612,2,102,2017-01-04 18:02:44,http://keebler.info/fabiola,,133.232.40.55,"{""location"": ""SJ"", ""is_mobile"": true}" 4613,2,102,2017-02-27 04:25:05,http://lefflerschowalter.info/aiyana,,43.177.73.69,"{""location"": ""LU"", ""is_mobile"": true}" 11493,4,256,2017-05-02 08:54:43,http://krajcikkoepp.co/cheyanne,,227.96.170.229,"{""location"": ""SL"", ""is_mobile"": true}" 11494,4,256,2017-04-30 11:41:06,http://adamsjast.io/elisa,,243.224.96.85,"{""location"": ""QA"", ""is_mobile"": true}" 11495,4,256,2017-05-28 20:57:40,http://kuvalis.co/mayra,,88.16.121.41,"{""location"": ""GA"", ""is_mobile"": true}" 11496,4,256,2016-12-17 12:55:56,http://wizalesch.co/ethel_white,,125.13.42.136,"{""location"": ""GM"", ""is_mobile"": false}" 11497,4,256,2017-06-03 04:12:38,http://kirlin.io/florida_daugherty,,106.122.143.63,"{""location"": ""DJ"", ""is_mobile"": false}" 11498,4,256,2017-04-15 22:02:28,http://ohara.com/parker.blanda,,213.65.205.132,"{""location"": ""PY"", ""is_mobile"": true}" 11499,4,256,2016-12-27 10:31:32,http://anderson.co/laisha.koepp,,63.191.71.245,"{""location"": ""TW"", ""is_mobile"": true}" 11501,4,256,2017-01-17 15:49:25,http://nadermohr.name/monique.durgan,,156.91.18.85,"{""location"": ""RW"", ""is_mobile"": false}" 11502,4,256,2017-02-20 23:46:00,http://hane.co/jaclyn,,117.169.151.70,"{""location"": ""HM"", ""is_mobile"": false}" 11503,4,257,2017-05-30 19:09:17,http://hansenhansen.info/cecilia.moore,0.4633392625,215.174.31.103,"{""location"": ""YT"", ""is_mobile"": true}" 11504,4,257,2017-03-21 13:10:08,http://lowepfeffer.biz/elda_jakubowski,0.7960573644,51.136.240.25,"{""location"": ""TK"", ""is_mobile"": true}" 11505,4,257,2017-05-07 07:44:34,http://lowe.io/imani,0.2517515881,143.56.213.54,"{""location"": ""BI"", ""is_mobile"": false}" 11506,4,257,2017-01-31 07:45:19,http://hintz.com/devin_medhurst,0.5024504847,190.79.61.27,"{""location"": ""HT"", ""is_mobile"": false}" 11507,4,257,2017-02-07 03:14:56,http://runolfsdottir.biz/ken,0.3150932455,88.234.16.24,"{""location"": ""GL"", ""is_mobile"": true}" 11508,4,257,2016-12-22 01:30:09,http://grady.net/leonor.rempel,0.3383840981,145.25.30.181,"{""location"": ""DE"", ""is_mobile"": true}" 11509,4,257,2017-03-11 22:02:38,http://moriettejacobi.io/ralph.reinger,0.9142528013,192.175.102.173,"{""location"": ""ST"", ""is_mobile"": false}" 11510,4,257,2017-03-04 02:43:39,http://rowe.org/carole_schmidt,0.9903054692,189.139.189.3,"{""location"": ""DM"", ""is_mobile"": true}" 11511,4,257,2017-02-20 22:48:06,http://blockreichel.org/bonita.barrows,0.9496117959,118.237.246.8,"{""location"": ""LS"", ""is_mobile"": false}" 11512,4,257,2017-06-02 06:22:15,http://fishersipes.co/caterina_stracke,0.3474244839,6.177.88.212,"{""location"": ""TL"", ""is_mobile"": false}" 11513,4,257,2016-12-20 07:35:29,http://sporereffertz.io/dagmar,0.3539464803,238.125.215.152,"{""location"": ""BI"", ""is_mobile"": true}" 11514,4,257,2017-02-24 06:57:40,http://wuckert.org/arnulfo,0.0016712258,233.145.81.211,"{""location"": ""KP"", ""is_mobile"": true}" 11515,4,257,2016-12-25 17:52:46,http://goyettekozey.name/chris,0.6702420562,252.198.110.23,"{""location"": ""SN"", ""is_mobile"": true}" 11516,4,257,2017-02-08 18:34:40,http://kling.co/declan.prohaska,0.4047119067,230.244.69.54,"{""location"": ""JP"", ""is_mobile"": false}" 11517,4,257,2017-01-27 02:49:41,http://jacobs.biz/kirk,0.8176344629,202.67.48.75,"{""location"": ""AS"", ""is_mobile"": true}" 11518,4,257,2017-06-12 21:25:18,http://parkerziemann.name/raoul.greenfelder,0.0627852717,129.39.10.90,"{""location"": ""PK"", ""is_mobile"": true}" 11519,4,257,2017-05-20 21:29:04,http://schoen.info/antonina_mitchell,0.1868133793,184.231.93.192,"{""location"": ""HN"", ""is_mobile"": false}" 11520,4,257,2017-04-18 15:25:59,http://bradtke.co/olin.hackett,0.1037683511,95.37.187.88,"{""location"": ""FR"", ""is_mobile"": true}" 11521,4,257,2017-03-16 14:16:45,http://rice.co/raphaelle_hilll,0.4357422821,63.207.167.224,"{""location"": ""BO"", ""is_mobile"": true}" 11522,4,257,2017-05-01 07:28:59,http://stanton.io/prince.beahan,0.1732560633,127.110.205.187,"{""location"": ""PY"", ""is_mobile"": false}" 11523,4,257,2016-12-23 07:17:30,http://hyatt.co/myrtice,0.7044671494,181.84.221.122,"{""location"": ""LT"", ""is_mobile"": true}" 11524,4,257,2017-02-21 21:43:45,http://marvingerhold.biz/tavares,0.8910622839,218.50.145.122,"{""location"": ""CO"", ""is_mobile"": true}" 11525,4,257,2017-03-15 00:14:19,http://heller.co/caitlyn_hahn,0.3675519977,128.124.201.244,"{""location"": ""KP"", ""is_mobile"": false}" 11526,4,257,2017-05-24 06:38:23,http://monahanjakubowski.net/olen.huels,0.4163701753,122.207.210.14,"{""location"": ""IE"", ""is_mobile"": true}" 11527,4,257,2017-02-15 10:12:41,http://hudson.net/jacynthe.anderson,0.9326168886,74.144.32.213,"{""location"": ""IS"", ""is_mobile"": false}" 11528,4,257,2016-12-16 02:41:13,http://bergebeahan.biz/shawna_bosco,0.3993878245,212.88.211.29,"{""location"": ""MH"", ""is_mobile"": false}" 11529,4,257,2017-05-29 02:11:40,http://robelluettgen.name/rosetta,0.8556819505,29.123.82.219,"{""location"": ""SX"", ""is_mobile"": true}" 11530,4,257,2017-01-17 05:14:59,http://willmsmclaughlin.net/jordane.stiedemann,0.1134318770,98.249.219.220,"{""location"": ""AE"", ""is_mobile"": false}" 11531,4,257,2016-12-21 23:27:28,http://sauer.name/gail_robel,0.9493475871,25.93.106.38,"{""location"": ""UZ"", ""is_mobile"": false}" 11532,4,257,2017-05-27 21:54:18,http://shieldsklocko.com/luis,0.8152181967,171.74.244.201,"{""location"": ""PF"", ""is_mobile"": true}" 11533,4,257,2017-05-10 18:18:35,http://cruickshankruel.io/mercedes_padberg,0.9522419536,148.5.9.146,"{""location"": ""LR"", ""is_mobile"": false}" 11534,4,257,2017-03-17 12:41:56,http://haagmckenzie.net/cathy.ondricka,0.2857463829,238.62.112.133,"{""location"": ""VI"", ""is_mobile"": false}" 11535,4,257,2017-01-10 14:13:56,http://heidenreichrodriguez.org/dayne.ferry,0.6086572738,140.62.229.111,"{""location"": ""FI"", ""is_mobile"": true}" 11536,4,257,2017-04-24 12:20:31,http://kutchlueilwitz.com/soledad,0.7859845903,153.108.20.211,"{""location"": ""CH"", ""is_mobile"": false}" 11537,4,257,2017-02-06 10:36:05,http://langoshwalter.com/whitney,0.4502194254,42.114.22.140,"{""location"": ""MZ"", ""is_mobile"": false}" 11538,4,257,2017-03-23 07:52:20,http://stiedemann.org/cierra_wisozk,0.0496087608,154.184.51.178,"{""location"": ""AG"", ""is_mobile"": true}" 11539,4,257,2016-12-23 09:08:32,http://kuphal.com/rodolfo,0.0639655674,224.119.248.45,"{""location"": ""JE"", ""is_mobile"": false}" 11540,4,257,2017-04-26 19:22:19,http://stammhermann.biz/emmitt.koch,0.2827400614,46.182.177.197,"{""location"": ""MK"", ""is_mobile"": true}" 11541,4,257,2017-02-14 23:10:24,http://nader.name/colin,0.3350360506,166.216.54.242,"{""location"": ""PH"", ""is_mobile"": true}" 11542,4,257,2017-06-12 21:05:09,http://padberg.io/dejuan,0.6840970109,187.221.17.85,"{""location"": ""SA"", ""is_mobile"": true}" 11543,4,257,2017-04-20 11:43:29,http://roob.org/yazmin.bode,0.3815136612,157.254.31.165,"{""location"": ""IO"", ""is_mobile"": true}" 11544,4,257,2017-01-05 18:43:40,http://oconnerhoeger.biz/treie,0.6944559609,116.223.17.54,"{""location"": ""MT"", ""is_mobile"": true}" 8495,3,191,2017-02-22 19:12:54,http://dach.biz/mauricio,,191.116.146.123,"{""location"": ""MC"", ""is_mobile"": true}" 8496,3,191,2016-12-24 10:49:15,http://reynolds.io/emmett,,173.110.197.238,"{""location"": ""FM"", ""is_mobile"": true}" 8497,3,191,2017-02-13 11:16:30,http://sipesmayer.io/roosevelt.heidenreich,,217.80.168.68,"{""location"": ""PA"", ""is_mobile"": true}" 8498,3,191,2017-01-23 13:13:22,http://muellerwisozk.biz/alek.shields,,181.129.94.127,"{""location"": ""SM"", ""is_mobile"": true}" 8499,3,191,2017-05-03 22:16:21,http://zieme.io/davon,,42.39.94.105,"{""location"": ""BS"", ""is_mobile"": false}" 8500,3,191,2017-04-12 22:25:32,http://volkman.co/fiona,,154.242.211.48,"{""location"": ""JE"", ""is_mobile"": true}" 8501,3,191,2017-03-05 07:38:32,http://dicki.com/idella,,41.78.151.232,"{""location"": ""SV"", ""is_mobile"": false}" 8502,3,191,2016-12-29 18:12:08,http://moore.info/harvey.wisozk,,194.160.51.129,"{""location"": ""PL"", ""is_mobile"": false}" 8503,3,191,2017-03-22 16:42:20,http://grimes.com/orpha,,124.116.183.121,"{""location"": ""MW"", ""is_mobile"": false}" 8504,3,191,2017-05-09 05:11:54,http://toy.com/terence,,42.51.148.75,"{""location"": ""LU"", ""is_mobile"": false}" 8505,3,191,2017-02-13 15:37:10,http://herzog.biz/lottie_buckridge,,133.77.243.59,"{""location"": ""PM"", ""is_mobile"": true}" 8506,3,191,2017-03-30 12:03:35,http://schmittwalter.io/annetta,,228.9.79.167,"{""location"": ""LA"", ""is_mobile"": true}" 8507,3,191,2017-01-25 22:25:09,http://hartmann.biz/sandra,,56.223.193.118,"{""location"": ""SN"", ""is_mobile"": false}" 8508,3,191,2016-12-25 18:42:35,http://rodriguez.org/christopher,,19.168.167.200,"{""location"": ""MY"", ""is_mobile"": false}" 8509,3,191,2017-05-09 02:53:05,http://schuppe.com/joanie,,156.212.179.134,"{""location"": ""PW"", ""is_mobile"": false}" 8510,3,191,2017-02-24 11:00:21,http://bartell.co/sallie.spinka,,4.4.182.34,"{""location"": ""NR"", ""is_mobile"": false}" 8511,3,191,2017-03-01 00:38:03,http://farrelllittle.com/alisha,,224.48.235.4,"{""location"": ""MN"", ""is_mobile"": true}" 8512,3,191,2017-01-31 17:11:36,http://deckow.org/shawna.lynch,,172.162.118.240,"{""location"": ""BY"", ""is_mobile"": true}" 8513,3,191,2017-02-15 06:14:29,http://tromp.org/elwyn,,157.34.137.51,"{""location"": ""TZ"", ""is_mobile"": false}" 8514,3,191,2017-06-01 05:55:40,http://jones.io/major_cummings,,24.184.124.29,"{""location"": ""OM"", ""is_mobile"": true}" 8515,3,191,2017-02-03 09:34:43,http://fay.net/deie,,101.19.25.199,"{""location"": ""IM"", ""is_mobile"": true}" 8516,3,191,2017-05-09 13:54:58,http://rathgrady.name/newell.hyatt,,144.107.12.137,"{""location"": ""JE"", ""is_mobile"": true}" 8517,3,191,2017-03-12 20:52:04,http://west.org/odell,,161.193.130.109,"{""location"": ""CI"", ""is_mobile"": false}" 8518,3,191,2017-01-11 19:30:18,http://tremblay.org/madelyn,,158.109.166.17,"{""location"": ""IM"", ""is_mobile"": false}" 8519,3,191,2017-04-06 13:07:32,http://heel.name/carlie,,175.45.185.34,"{""location"": ""PS"", ""is_mobile"": false}" 8520,3,191,2017-03-11 00:44:14,http://boyle.info/beverly,,118.221.173.105,"{""location"": ""LK"", ""is_mobile"": false}" 8521,3,191,2017-03-10 12:34:49,http://robel.biz/estell,,74.183.193.64,"{""location"": ""NR"", ""is_mobile"": false}" 8522,3,191,2017-04-06 21:25:42,http://herman.net/drake,,182.207.175.157,"{""location"": ""NC"", ""is_mobile"": false}" 8523,3,191,2017-04-01 23:59:39,http://yundt.biz/carmela,,32.137.9.40,"{""location"": ""BV"", ""is_mobile"": false}" 8524,3,191,2017-02-21 00:44:11,http://rutherfordkilback.name/blanche,,61.140.5.92,"{""location"": ""SS"", ""is_mobile"": false}" 8525,3,191,2017-01-19 01:33:07,http://murray.io/nia.stehr,,109.212.14.166,"{""location"": ""TT"", ""is_mobile"": false}" 8526,3,191,2017-01-05 23:16:45,http://casper.name/ania,,56.167.37.6,"{""location"": ""SB"", ""is_mobile"": false}" 8527,3,191,2017-03-18 16:52:10,http://markspaucek.org/anya,,160.226.102.96,"{""location"": ""CG"", ""is_mobile"": true}" 8528,3,191,2017-02-14 22:10:32,http://heel.biz/mario_schuppe,,115.152.118.138,"{""location"": ""BI"", ""is_mobile"": false}" 8529,3,192,2017-04-27 07:14:05,http://schumm.io/justice,,13.89.189.45,"{""location"": ""UG"", ""is_mobile"": false}" 8530,3,192,2017-03-11 14:45:50,http://hansencasper.com/roslyn,,127.40.105.6,"{""location"": ""DZ"", ""is_mobile"": false}" 8531,3,192,2017-04-09 14:36:36,http://schuster.info/cecilia.greenholt,,112.189.118.203,"{""location"": ""GM"", ""is_mobile"": true}" 8532,3,192,2017-01-27 06:54:36,http://paucek.biz/alicia.grimes,,95.10.107.54,"{""location"": ""LA"", ""is_mobile"": false}" 8533,3,192,2017-04-05 00:38:28,http://bartell.name/amelia,,195.124.209.144,"{""location"": ""ID"", ""is_mobile"": false}" 8534,3,192,2017-04-26 11:07:52,http://hermann.info/mara,,180.17.88.218,"{""location"": ""DZ"", ""is_mobile"": false}" 8535,3,192,2017-04-27 15:59:06,http://wiegandvolkman.org/vergie,,128.24.164.3,"{""location"": ""NZ"", ""is_mobile"": true}" 8536,3,192,2016-12-17 03:32:31,http://green.biz/israel,,223.55.248.134,"{""location"": ""ID"", ""is_mobile"": true}" 8537,3,192,2017-01-06 16:50:41,http://treutelpadberg.biz/makayla,,207.171.246.12,"{""location"": ""EC"", ""is_mobile"": false}" 8538,3,192,2017-03-27 01:08:22,http://sawayn.net/halie_mayert,,164.155.231.95,"{""location"": ""IL"", ""is_mobile"": true}" 8539,3,192,2017-04-20 01:39:02,http://mraz.biz/raina,,87.57.33.164,"{""location"": ""TW"", ""is_mobile"": false}" 8540,3,192,2016-12-27 02:26:16,http://wuckert.io/libby,,219.235.166.17,"{""location"": ""BA"", ""is_mobile"": true}" 8541,3,192,2017-04-09 10:09:29,http://greenholtdickens.co/garfield,,25.250.206.99,"{""location"": ""NO"", ""is_mobile"": true}" 8542,3,192,2017-01-12 18:52:52,http://becker.biz/lance.konopelski,,19.89.40.198,"{""location"": ""AO"", ""is_mobile"": false}" 8543,3,192,2017-01-03 06:54:25,http://kuvaliskuhic.co/alberto,,246.231.204.127,"{""location"": ""JP"", ""is_mobile"": true}" 8544,3,192,2017-02-15 11:49:11,http://marks.io/noe,,252.13.175.209,"{""location"": ""LK"", ""is_mobile"": false}" 8545,3,192,2017-01-27 11:47:19,http://schneider.net/darrel.ernser,,77.157.104.92,"{""location"": ""IL"", ""is_mobile"": true}" 8546,3,192,2017-03-23 04:11:52,http://auerheidenreich.info/elda.king,,12.82.98.152,"{""location"": ""BY"", ""is_mobile"": true}" 8547,3,192,2017-02-21 00:50:23,http://dubuque.org/emelia_langosh,,109.31.16.148,"{""location"": ""ZM"", ""is_mobile"": true}" 8548,3,192,2016-12-14 11:25:23,http://erdmandoyle.info/maverick,,95.199.29.165,"{""location"": ""IM"", ""is_mobile"": true}" 8549,3,192,2017-02-02 07:23:43,http://murphyheaney.co/dangelo.mclaughlin,,237.204.99.2,"{""location"": ""GQ"", ""is_mobile"": false}" 8550,3,192,2016-12-14 15:02:04,http://collins.com/rachael.jacobi,,69.78.85.34,"{""location"": ""LU"", ""is_mobile"": true}" 8551,3,192,2017-01-23 05:02:49,http://donnelly.org/eulalia,,60.227.88.190,"{""location"": ""MG"", ""is_mobile"": true}" 14444,5,325,2017-02-12 23:58:15,http://mcdermott.biz/arthur,,116.252.185.165,"{""location"": ""JO"", ""is_mobile"": false}" 14445,5,325,2017-04-21 14:05:30,http://bruen.net/isabelle.bergstrom,,13.73.17.132,"{""location"": ""AQ"", ""is_mobile"": false}" 14446,5,325,2017-06-07 08:29:36,http://marvin.com/anastasia,,97.165.85.120,"{""location"": ""CR"", ""is_mobile"": false}" 14447,5,325,2016-12-17 04:18:00,http://gleichner.com/onie_jakubowski,,230.120.91.128,"{""location"": ""AT"", ""is_mobile"": false}" 14448,5,325,2016-12-29 04:47:11,http://manteolson.io/gracie.spinka,,99.38.218.98,"{""location"": ""KH"", ""is_mobile"": false}" 14449,5,325,2017-04-11 19:24:20,http://rogahnspencer.io/aglae_prohaska,,238.80.190.211,"{""location"": ""PR"", ""is_mobile"": true}" 14450,5,325,2017-02-19 16:03:04,http://blockmann.name/maverick,,235.79.176.34,"{""location"": ""BO"", ""is_mobile"": false}" 14451,5,325,2017-04-22 17:19:58,http://pfefferprohaska.net/jacynthe,,64.18.88.44,"{""location"": ""ST"", ""is_mobile"": true}" 14452,5,325,2017-03-16 22:17:45,http://ohara.com/eryn.stroman,,252.83.32.80,"{""location"": ""GF"", ""is_mobile"": false}" 14453,5,325,2017-04-20 18:55:37,http://boyerleuschke.com/cleve,,112.57.217.112,"{""location"": ""YT"", ""is_mobile"": true}" 14454,5,325,2017-05-26 09:49:08,http://sawaynjast.co/stanley,,217.226.45.16,"{""location"": ""KW"", ""is_mobile"": false}" 14455,5,325,2017-02-01 16:23:36,http://gulgowskikub.org/demetris,,162.57.77.155,"{""location"": ""SX"", ""is_mobile"": true}" 14456,5,325,2017-02-18 11:01:38,http://greenholtjacobi.org/gianni,,159.153.58.132,"{""location"": ""VG"", ""is_mobile"": true}" 14457,5,325,2017-01-22 05:35:14,http://cruickshank.com/elnora_gerlach,,31.11.57.59,"{""location"": ""JE"", ""is_mobile"": false}" 14458,5,325,2017-05-17 19:05:14,http://rolfson.name/melvin_kozey,,81.116.60.218,"{""location"": ""ME"", ""is_mobile"": false}" 14459,5,325,2017-02-16 23:10:44,http://collins.info/brown,,168.82.113.210,"{""location"": ""BM"", ""is_mobile"": false}" 14460,5,325,2016-12-17 03:43:34,http://bradtke.io/braeden.brown,,244.60.24.252,"{""location"": ""AD"", ""is_mobile"": true}" 14461,5,325,2017-01-27 11:38:13,http://ferryokon.biz/nathen,,118.174.215.77,"{""location"": ""PN"", ""is_mobile"": false}" 14462,5,325,2017-01-03 02:22:17,http://macgyver.net/maeve.eichmann,,232.159.40.15,"{""location"": ""LK"", ""is_mobile"": true}" 14463,5,325,2017-01-24 20:27:21,http://hirthespinka.net/vaughn,,138.236.205.66,"{""location"": ""BA"", ""is_mobile"": true}" 14464,5,325,2017-05-31 09:28:43,http://hammesnienow.co/tomas_herman,,216.203.212.7,"{""location"": ""VN"", ""is_mobile"": true}" 14465,5,325,2016-12-23 05:04:13,http://kunde.name/jeanie_nienow,,43.120.76.21,"{""location"": ""AX"", ""is_mobile"": true}" 14466,5,325,2017-01-27 12:25:23,http://muller.name/bonita,,59.174.224.113,"{""location"": ""RO"", ""is_mobile"": true}" 14467,5,325,2017-02-09 11:12:05,http://leffler.com/lew,,56.159.114.15,"{""location"": ""DE"", ""is_mobile"": true}" 14468,5,325,2017-01-30 19:36:42,http://fisherswaniawski.info/dena,,89.32.205.143,"{""location"": ""VG"", ""is_mobile"": true}" 14469,5,325,2017-01-07 18:30:18,http://klocko.info/colton.runolfon,,194.137.88.108,"{""location"": ""NF"", ""is_mobile"": false}" 14470,5,325,2017-01-09 08:18:42,http://schimmelheathcote.name/maye.kuvalis,,186.37.69.187,"{""location"": ""SB"", ""is_mobile"": true}" 14471,5,325,2016-12-21 20:49:15,http://oconnereichmann.io/pauline_jacobs,,15.25.171.76,"{""location"": ""NO"", ""is_mobile"": false}" 14472,5,325,2017-03-22 00:22:21,http://kemmer.io/else,,149.156.5.206,"{""location"": ""SB"", ""is_mobile"": false}" 14473,5,325,2017-06-10 21:28:11,http://schimmel.info/alberta,,87.30.57.103,"{""location"": ""ID"", ""is_mobile"": true}" 14474,5,325,2017-04-27 00:14:58,http://breitenbergankunding.info/myah_hoeger,,33.16.191.125,"{""location"": ""MM"", ""is_mobile"": true}" 14475,5,325,2016-12-28 06:23:29,http://schulist.io/paul,,92.215.160.92,"{""location"": ""RE"", ""is_mobile"": false}" 14476,5,325,2017-01-06 16:13:45,http://hettinger.com/conner,,46.59.24.131,"{""location"": ""PF"", ""is_mobile"": true}" 14477,5,325,2017-05-24 22:28:29,http://monahanthompson.com/brett,,232.34.211.45,"{""location"": ""BV"", ""is_mobile"": false}" 14478,5,325,2017-05-12 22:37:52,http://sawaynschowalter.co/ottilie_hayes,,234.16.130.215,"{""location"": ""GE"", ""is_mobile"": true}" 14479,5,325,2017-04-03 18:31:59,http://mcdermotthickle.name/stevie,,155.56.247.113,"{""location"": ""MZ"", ""is_mobile"": true}" 14480,5,325,2017-01-07 07:26:14,http://mosciski.org/barton.lynch,,27.162.241.146,"{""location"": ""SB"", ""is_mobile"": true}" 14481,5,326,2017-04-11 06:10:52,http://kozeytremblay.biz/maymie,,179.205.230.121,"{""location"": ""GG"", ""is_mobile"": true}" 14482,5,326,2017-04-11 19:13:01,http://sanfordsmitham.org/kolby,,83.219.93.249,"{""location"": ""VG"", ""is_mobile"": true}" 14483,5,326,2017-01-26 06:09:19,http://turcotte.org/lauren.daniel,,181.181.104.72,"{""location"": ""KN"", ""is_mobile"": true}" 14484,5,326,2017-03-26 04:28:49,http://paucekmarks.biz/florida,,179.132.188.228,"{""location"": ""BD"", ""is_mobile"": true}" 14485,5,326,2017-02-06 19:59:49,http://hettinger.biz/danyka.lowe,,27.77.134.103,"{""location"": ""KG"", ""is_mobile"": true}" 14486,5,326,2017-05-19 03:19:12,http://gulgowskigusikowski.name/domenico_dickinson,,231.15.37.134,"{""location"": ""CI"", ""is_mobile"": true}" 14487,5,326,2017-03-20 22:50:24,http://kaulkecorwin.co/estell.wilkinson,,61.241.116.10,"{""location"": ""HK"", ""is_mobile"": false}" 14488,5,326,2017-05-14 18:25:40,http://greenfelder.io/veronica_dooley,,3.188.23.147,"{""location"": ""CN"", ""is_mobile"": true}" 14489,5,326,2017-04-19 19:17:51,http://rodriguez.com/shanny.morar,,34.121.216.254,"{""location"": ""CD"", ""is_mobile"": false}" 14490,5,326,2017-02-11 12:25:54,http://langworthberge.org/lorna.quitzon,,19.112.7.211,"{""location"": ""MT"", ""is_mobile"": false}" 14491,5,326,2017-04-05 00:06:08,http://satterfieldherman.org/scot_johns,,224.202.63.161,"{""location"": ""PN"", ""is_mobile"": false}" 14492,5,326,2017-04-11 21:10:51,http://roberts.net/candice,,58.159.119.232,"{""location"": ""PL"", ""is_mobile"": false}" 14493,5,326,2017-04-10 10:18:48,http://emmerichmosciski.io/giuseppe,,210.212.84.20,"{""location"": ""NF"", ""is_mobile"": true}" 14494,5,326,2017-06-02 01:10:48,http://stiedemann.co/erica_hudson,,30.121.94.125,"{""location"": ""EC"", ""is_mobile"": false}" 14495,5,326,2017-01-23 14:33:21,http://dibbert.info/mavis.ryan,,223.202.32.249,"{""location"": ""GW"", ""is_mobile"": true}" 14496,5,326,2017-05-16 02:17:58,http://mcdermott.co/angelina.gusikowski,,66.16.17.119,"{""location"": ""KY"", ""is_mobile"": false}" 14497,5,326,2017-05-13 13:56:33,http://goyette.biz/malvina,,228.147.115.203,"{""location"": ""CG"", ""is_mobile"": true}" 14498,5,326,2017-01-11 07:03:25,http://johnsonjohnson.org/kirk.bernier,,70.125.109.153,"{""location"": ""AX"", ""is_mobile"": true}" 17429,6,393,2017-01-19 05:25:42,http://murphy.org/selmer.bayer,,158.139.169.153,"{""location"": ""BI"", ""is_mobile"": false}" 17430,6,393,2017-05-29 20:22:08,http://prohaska.biz/mittie,,70.115.56.14,"{""location"": ""RS"", ""is_mobile"": true}" 17431,6,393,2017-03-24 14:54:40,http://pagac.co/jedediah,,160.204.33.240,"{""location"": ""LC"", ""is_mobile"": true}" 17432,6,393,2017-02-10 03:19:43,http://oharadaniel.co/maye,,91.218.14.56,"{""location"": ""CC"", ""is_mobile"": false}" 17433,6,393,2017-06-09 17:28:14,http://walker.net/elvis.osinski,,50.217.190.61,"{""location"": ""FR"", ""is_mobile"": false}" 17434,6,393,2017-04-12 15:40:17,http://kleinrice.io/candice_kiehn,,72.154.178.155,"{""location"": ""GG"", ""is_mobile"": false}" 17435,6,393,2016-12-31 13:59:38,http://blick.org/lia.schroeder,,99.126.128.133,"{""location"": ""TD"", ""is_mobile"": true}" 17436,6,393,2016-12-19 23:28:08,http://barton.name/lynn_rowe,,98.10.178.53,"{""location"": ""CZ"", ""is_mobile"": true}" 17437,6,393,2016-12-24 06:03:08,http://emmerichsawayn.com/rhianna.schaefer,,196.174.133.227,"{""location"": ""TG"", ""is_mobile"": true}" 17438,6,393,2016-12-25 10:48:07,http://ernser.info/kitty_haley,,45.211.128.165,"{""location"": ""LB"", ""is_mobile"": false}" 17439,6,393,2017-01-17 15:19:05,http://marksvon.net/denis,,81.34.106.52,"{""location"": ""BD"", ""is_mobile"": false}" 17440,6,393,2016-12-29 07:15:56,http://cummerata.net/clint,,24.184.91.54,"{""location"": ""CW"", ""is_mobile"": false}" 17441,6,393,2016-12-23 04:20:46,http://lebsack.name/romaine_williamson,,201.31.165.29,"{""location"": ""AQ"", ""is_mobile"": true}" 17442,6,393,2017-01-07 01:16:16,http://kohlergreenfelder.net/jaunita_prosacco,,70.190.163.164,"{""location"": ""TC"", ""is_mobile"": false}" 17443,6,393,2017-05-29 20:39:13,http://kuvaliskonopelski.net/winifred_grady,,238.24.133.102,"{""location"": ""AQ"", ""is_mobile"": true}" 17444,6,393,2017-04-16 07:32:27,http://lindgren.org/kyler.larkin,,157.92.26.205,"{""location"": ""NA"", ""is_mobile"": false}" 17445,6,393,2017-04-02 04:37:20,http://oharawalker.org/sandra.murray,,58.181.98.93,"{""location"": ""TD"", ""is_mobile"": true}" 17446,6,393,2017-02-09 22:13:14,http://pacocha.com/jevon,,63.204.102.114,"{""location"": ""EE"", ""is_mobile"": true}" 17447,6,393,2017-05-07 12:18:18,http://jaskolskidubuque.info/willy_heathcote,,18.250.91.210,"{""location"": ""MZ"", ""is_mobile"": false}" 17448,6,393,2017-01-03 22:32:43,http://toy.biz/isabella.rowe,,85.171.86.217,"{""location"": ""PY"", ""is_mobile"": true}" 17449,6,393,2017-04-20 04:18:38,http://kovacek.org/laurianne,,127.128.204.118,"{""location"": ""GI"", ""is_mobile"": false}" 17450,6,393,2017-03-10 00:08:55,http://fadel.info/zion,,180.39.24.245,"{""location"": ""BF"", ""is_mobile"": false}" 17451,6,393,2017-03-11 12:03:21,http://gusikowski.net/rosemarie,,57.240.50.167,"{""location"": ""BV"", ""is_mobile"": false}" 17452,6,393,2017-01-28 10:43:47,http://witting.info/jacinto,,3.215.173.223,"{""location"": ""ES"", ""is_mobile"": false}" 17453,6,393,2017-03-17 00:46:15,http://turnerkilback.io/garrett.marquardt,,254.234.41.62,"{""location"": ""BW"", ""is_mobile"": true}" 17454,6,393,2017-03-19 14:50:56,http://wilderman.co/penelope,,152.49.224.231,"{""location"": ""BG"", ""is_mobile"": false}" 17455,6,393,2017-05-25 21:51:28,http://klein.org/ollie_kirlin,,168.189.30.203,"{""location"": ""BY"", ""is_mobile"": false}" 17456,6,393,2016-12-19 12:45:43,http://turner.biz/alexandrea_marks,,147.7.193.84,"{""location"": ""GN"", ""is_mobile"": true}" 17457,6,393,2016-12-22 09:26:50,http://bruen.biz/jermey,,163.219.125.26,"{""location"": ""NR"", ""is_mobile"": false}" 17458,6,393,2016-12-19 04:57:37,http://heaney.net/eladio.prosacco,,131.189.116.196,"{""location"": ""BZ"", ""is_mobile"": true}" 17459,6,393,2017-02-25 07:43:02,http://heller.io/maribel_hagenes,,17.172.6.132,"{""location"": ""GM"", ""is_mobile"": true}" 17460,6,393,2017-02-22 10:06:46,http://jacobson.net/cordie,,135.19.59.195,"{""location"": ""BL"", ""is_mobile"": false}" 17461,6,393,2017-01-24 23:27:24,http://spencerlabadie.io/gracie,,116.221.157.46,"{""location"": ""EE"", ""is_mobile"": false}" 17462,6,393,2017-06-12 05:26:05,http://turnerruel.biz/nakia,,120.41.13.234,"{""location"": ""WS"", ""is_mobile"": false}" 17463,6,393,2017-03-02 15:15:39,http://pagacgleichner.biz/rudy,,42.23.33.55,"{""location"": ""KY"", ""is_mobile"": true}" 17464,6,393,2017-05-11 05:53:41,http://ondrickawhite.io/gene.collins,,239.176.252.40,"{""location"": ""MT"", ""is_mobile"": false}" 17465,6,393,2017-05-16 22:44:48,http://hirthe.com/bradly.reichel,,147.169.88.2,"{""location"": ""SS"", ""is_mobile"": false}" 17466,6,393,2017-02-07 20:58:18,http://bogisichgottlieb.name/ardella,,244.186.38.220,"{""location"": ""PH"", ""is_mobile"": true}" 17467,6,393,2017-04-28 08:26:38,http://priceparisian.name/lonny,,36.131.249.118,"{""location"": ""CZ"", ""is_mobile"": false}" 17468,6,393,2017-03-17 03:45:05,http://effertz.io/janie,,174.250.231.215,"{""location"": ""BS"", ""is_mobile"": true}" 17469,6,394,2017-03-21 07:57:26,http://graham.io/mariela,,118.77.144.197,"{""location"": ""BL"", ""is_mobile"": true}" 17470,6,394,2017-05-11 00:47:23,http://townemarquardt.biz/peggie_mills,,158.222.98.45,"{""location"": ""JO"", ""is_mobile"": true}" 17471,6,394,2017-05-03 21:38:50,http://stoltenberg.info/joyce,,27.237.142.65,"{""location"": ""CO"", ""is_mobile"": false}" 17472,6,394,2017-03-11 23:57:00,http://blandalangosh.org/matilde.wisozk,,215.90.135.30,"{""location"": ""MZ"", ""is_mobile"": false}" 17473,6,394,2017-01-17 07:24:03,http://bahringerrunolfsdottir.net/abbie,,246.53.162.87,"{""location"": ""MV"", ""is_mobile"": true}" 17474,6,394,2016-12-31 22:28:12,http://mantehaley.com/alysa,,8.95.16.48,"{""location"": ""IO"", ""is_mobile"": true}" 17475,6,394,2017-04-20 20:15:58,http://okon.com/rupert_rempel,,132.168.232.115,"{""location"": ""PN"", ""is_mobile"": true}" 17476,6,394,2017-04-26 06:49:17,http://okonritchie.org/maurine.lehner,,3.191.119.49,"{""location"": ""SS"", ""is_mobile"": true}" 17477,6,394,2017-03-31 22:21:20,http://nitzsche.co/libby,,138.28.184.230,"{""location"": ""SB"", ""is_mobile"": false}" 17478,6,394,2017-04-05 14:49:28,http://priceabernathy.com/dallas,,138.208.198.208,"{""location"": ""BB"", ""is_mobile"": false}" 17479,6,394,2017-03-09 00:08:05,http://towne.info/jevon,,71.226.65.110,"{""location"": ""UG"", ""is_mobile"": false}" 17480,6,394,2017-01-23 22:43:02,http://jenkinsvon.io/robyn_gottlieb,,108.125.18.84,"{""location"": ""TZ"", ""is_mobile"": true}" 17481,6,394,2017-05-10 00:24:57,http://zemlak.name/tara.rohan,,167.129.93.222,"{""location"": ""CF"", ""is_mobile"": true}" 17482,6,394,2017-04-11 21:16:57,http://yundt.co/joany,,185.158.157.244,"{""location"": ""JE"", ""is_mobile"": true}" 17483,6,394,2016-12-21 08:23:50,http://beatty.co/augusta_wilkinson,,132.147.102.173,"{""location"": ""FO"", ""is_mobile"": true}" 4614,2,102,2017-01-10 08:49:41,http://smitham.biz/tyrel,,208.138.71.208,"{""location"": ""MX"", ""is_mobile"": false}" 4615,2,102,2017-06-01 20:38:20,http://tremblay.org/stefan_little,,209.24.213.133,"{""location"": ""NL"", ""is_mobile"": false}" 4616,2,102,2017-04-11 16:51:09,http://koepp.info/jada_ruecker,,192.94.187.57,"{""location"": ""NO"", ""is_mobile"": false}" 4617,2,102,2017-02-24 02:48:04,http://connellylowe.org/jordon,,157.150.251.159,"{""location"": ""MY"", ""is_mobile"": true}" 4618,2,102,2017-04-01 03:33:19,http://dooleypowlowski.org/velva,,246.246.42.192,"{""location"": ""SH"", ""is_mobile"": true}" 4619,2,102,2017-04-29 06:05:50,http://heathcote.net/austen,,77.49.49.155,"{""location"": ""GG"", ""is_mobile"": true}" 4620,2,102,2017-01-06 13:21:38,http://denesik.biz/estelle.lakin,,118.241.128.247,"{""location"": ""BS"", ""is_mobile"": false}" 4621,2,102,2017-05-21 09:46:02,http://weimanncrooks.name/suzanne,,123.6.226.165,"{""location"": ""SZ"", ""is_mobile"": false}" 4622,2,102,2016-12-20 14:53:29,http://mcglynn.com/cyrus.boyer,,7.248.109.244,"{""location"": ""GP"", ""is_mobile"": true}" 4623,2,102,2017-04-11 10:16:05,http://block.org/rick.crist,,55.229.122.65,"{""location"": ""MT"", ""is_mobile"": false}" 4624,2,102,2017-01-23 14:03:45,http://dickinsongoodwin.biz/santiago,,100.249.120.169,"{""location"": ""MR"", ""is_mobile"": true}" 4625,2,102,2017-02-24 21:33:14,http://koelpin.net/eric,,76.243.136.97,"{""location"": ""FI"", ""is_mobile"": false}" 4626,2,102,2017-03-04 20:14:44,http://kovacek.info/jamie,,13.41.118.239,"{""location"": ""BO"", ""is_mobile"": true}" 4627,2,102,2017-04-07 05:47:46,http://rice.biz/brice.bahringer,,163.19.227.163,"{""location"": ""IN"", ""is_mobile"": false}" 4628,2,102,2017-04-22 17:21:13,http://williamson.info/frances,,233.109.156.193,"{""location"": ""JM"", ""is_mobile"": true}" 4629,2,102,2017-03-06 04:17:03,http://heidenreichkutch.net/amie.schneider,,125.220.98.35,"{""location"": ""ZM"", ""is_mobile"": false}" 4630,2,102,2017-06-12 22:21:50,http://effertzkeler.name/skyla.mckenzie,,233.161.35.210,"{""location"": ""WS"", ""is_mobile"": true}" 4631,2,102,2017-06-01 03:01:56,http://cartwright.org/ora,,143.212.133.154,"{""location"": ""GN"", ""is_mobile"": false}" 4632,2,102,2017-04-26 22:14:16,http://litteltillman.info/payton.rodriguez,,79.42.153.123,"{""location"": ""JO"", ""is_mobile"": true}" 4633,2,102,2017-04-17 23:10:52,http://oconnellcormier.biz/eloy.runolfsdottir,,238.109.192.25,"{""location"": ""BR"", ""is_mobile"": true}" 4634,2,102,2017-06-05 15:14:25,http://homenick.net/luciano,,12.155.20.156,"{""location"": ""MT"", ""is_mobile"": false}" 4635,2,102,2017-05-10 02:42:04,http://waelchi.co/rosina_schmeler,,87.185.193.42,"{""location"": ""AF"", ""is_mobile"": true}" 4636,2,102,2017-04-05 07:14:40,http://kiehn.co/arden,,252.186.69.58,"{""location"": ""SZ"", ""is_mobile"": true}" 4637,2,102,2016-12-29 20:36:23,http://wuckert.com/keara,,100.220.115.169,"{""location"": ""EH"", ""is_mobile"": false}" 4638,2,102,2017-02-16 05:25:23,http://maggio.io/sharon_kaulke,,27.195.239.204,"{""location"": ""SY"", ""is_mobile"": false}" 4639,2,102,2016-12-24 08:55:33,http://gerhold.org/micaela.haley,,92.105.107.52,"{""location"": ""GT"", ""is_mobile"": true}" 4640,2,103,2016-12-24 04:01:56,http://murphy.io/alfonzo,,246.148.55.15,"{""location"": ""KY"", ""is_mobile"": false}" 4641,2,103,2017-01-24 15:05:50,http://nolan.name/ernie_herzog,,66.151.162.227,"{""location"": ""BQ"", ""is_mobile"": false}" 4642,2,103,2016-12-16 08:45:21,http://mcglynnwiegand.io/ambrose,,185.135.158.214,"{""location"": ""GU"", ""is_mobile"": true}" 4643,2,103,2017-04-26 10:15:53,http://dietrichheller.info/lawson.frami,,214.45.157.235,"{""location"": ""VG"", ""is_mobile"": false}" 4644,2,103,2017-01-24 09:34:16,http://bednar.biz/joyce,,36.13.50.152,"{""location"": ""BG"", ""is_mobile"": false}" 4645,2,103,2017-05-27 03:29:21,http://adams.com/quinn.purdy,,211.161.21.56,"{""location"": ""IT"", ""is_mobile"": true}" 4646,2,103,2017-02-18 10:12:52,http://mccullough.org/johnnie_nader,,149.25.145.7,"{""location"": ""PE"", ""is_mobile"": true}" 4647,2,103,2017-02-02 15:15:23,http://larkinlarkin.co/harvey.jerde,,246.53.192.144,"{""location"": ""AM"", ""is_mobile"": false}" 4648,2,103,2017-04-11 12:09:32,http://daughertycarter.org/johnson,,81.171.198.25,"{""location"": ""SG"", ""is_mobile"": false}" 4649,2,103,2016-12-13 09:57:32,http://lang.info/joe.green,,148.225.157.152,"{""location"": ""ER"", ""is_mobile"": false}" 4650,2,103,2017-02-24 13:05:07,http://hammes.co/marisol.gottlieb,,101.6.123.127,"{""location"": ""VC"", ""is_mobile"": true}" 4651,2,103,2017-06-14 02:58:11,http://rodriguez.name/lukas,,24.144.191.140,"{""location"": ""LV"", ""is_mobile"": true}" 4652,2,103,2017-04-16 14:53:38,http://becker.io/lukas,,202.223.246.246,"{""location"": ""TO"", ""is_mobile"": false}" 4653,2,103,2017-03-04 21:21:53,http://hills.com/columbus.lockman,,175.74.236.203,"{""location"": ""GA"", ""is_mobile"": false}" 4654,2,103,2017-02-06 02:58:16,http://sanford.info/edd,,96.228.118.116,"{""location"": ""AG"", ""is_mobile"": true}" 4655,2,103,2017-03-08 22:13:34,http://collier.com/karl,,90.47.83.224,"{""location"": ""TD"", ""is_mobile"": false}" 4656,2,103,2017-04-05 05:55:02,http://ebertfahey.biz/khalid_stiedemann,,141.224.212.142,"{""location"": ""CN"", ""is_mobile"": true}" 4657,2,103,2017-06-06 03:17:56,http://daugherty.io/vidal_doyle,,231.199.31.88,"{""location"": ""RW"", ""is_mobile"": false}" 4658,2,103,2017-02-24 16:01:18,http://weber.biz/ocie,,23.209.77.150,"{""location"": ""WF"", ""is_mobile"": true}" 4659,2,103,2017-06-13 14:28:12,http://cummerata.co/jenifer,,109.184.67.235,"{""location"": ""NG"", ""is_mobile"": true}" 4660,2,103,2017-05-23 01:42:34,http://maggio.co/vinnie,,31.25.126.158,"{""location"": ""TH"", ""is_mobile"": true}" 4661,2,103,2017-03-03 17:17:42,http://monahanebert.biz/giovanny.considine,,35.81.86.95,"{""location"": ""TG"", ""is_mobile"": false}" 4662,2,103,2017-05-18 09:35:52,http://gusikowskigislason.co/derrick,,222.206.217.252,"{""location"": ""RO"", ""is_mobile"": false}" 4663,2,103,2017-03-19 20:07:18,http://wisozk.io/reece.oberbrunner,,5.119.110.42,"{""location"": ""PA"", ""is_mobile"": false}" 4664,2,103,2017-02-03 20:10:13,http://stiedemann.org/brant,,140.174.218.182,"{""location"": ""SJ"", ""is_mobile"": false}" 4665,2,103,2017-02-18 00:40:44,http://gutmann.net/rory.farrell,,226.174.39.229,"{""location"": ""CL"", ""is_mobile"": false}" 4666,2,103,2017-01-21 05:29:31,http://schummhansen.biz/jarod,,241.89.199.242,"{""location"": ""VA"", ""is_mobile"": false}" 4667,2,104,2017-06-12 11:22:44,http://aufderhar.org/tre,,48.210.58.72,"{""location"": ""ST"", ""is_mobile"": true}" 4668,2,104,2017-01-31 18:57:51,http://hackett.com/tillman,,248.156.251.90,"{""location"": ""CK"", ""is_mobile"": true}" 11545,4,257,2017-03-15 00:36:55,http://ferry.com/jillian_lind,0.9280659973,68.79.137.162,"{""location"": ""DE"", ""is_mobile"": true}" 11546,4,257,2017-03-21 17:36:33,http://hegmann.org/michaela,0.3542053402,182.118.60.70,"{""location"": ""CH"", ""is_mobile"": false}" 11547,4,257,2016-12-18 05:13:28,http://williamsonconsidine.org/addison_langosh,0.1065460354,116.245.7.183,"{""location"": ""NA"", ""is_mobile"": false}" 11548,4,257,2017-01-14 19:39:26,http://stoltenberg.io/dameon,0.4757582087,3.151.93.162,"{""location"": ""FM"", ""is_mobile"": false}" 11549,4,257,2017-03-03 09:10:24,http://schroederdamore.io/mireille,0.1696252014,40.180.8.238,"{""location"": ""GU"", ""is_mobile"": false}" 11550,4,257,2017-04-10 14:58:53,http://brekkeschimmel.com/leila,0.8446581441,140.162.171.52,"{""location"": ""LI"", ""is_mobile"": true}" 11551,4,257,2017-03-13 15:29:13,http://green.org/percival,0.9796250342,185.116.45.189,"{""location"": ""BR"", ""is_mobile"": false}" 11552,4,257,2016-12-25 16:04:36,http://ernser.net/ellen_welch,0.3579824059,203.169.36.228,"{""location"": ""CG"", ""is_mobile"": true}" 11553,4,257,2017-05-25 18:07:07,http://gloverkovacek.biz/keanu.gulgowski,0.8673735476,70.63.121.148,"{""location"": ""LR"", ""is_mobile"": false}" 11554,4,257,2017-03-23 11:08:36,http://luettgen.org/kariane.turcotte,0.5764153080,222.47.121.8,"{""location"": ""TV"", ""is_mobile"": false}" 11555,4,257,2017-05-13 19:55:15,http://murphy.io/ernest,0.2000862972,80.95.53.184,"{""location"": ""UM"", ""is_mobile"": false}" 11556,4,257,2017-05-30 00:17:48,http://kautzer.co/mariah,0.6410352675,169.183.169.3,"{""location"": ""CH"", ""is_mobile"": true}" 11557,4,257,2016-12-18 17:59:15,http://torp.info/taylor,0.0813071641,18.124.18.195,"{""location"": ""FI"", ""is_mobile"": false}" 11558,4,257,2017-06-13 11:28:17,http://durgan.info/thelma.walter,0.9707125083,43.222.80.183,"{""location"": ""KM"", ""is_mobile"": false}" 11559,4,257,2016-12-15 17:55:21,http://lockman.com/marlen_bradtke,0.8089259904,187.119.115.130,"{""location"": ""EE"", ""is_mobile"": true}" 11560,4,257,2017-06-12 17:16:37,http://blanda.name/everette_reynolds,0.5497470657,216.43.233.205,"{""location"": ""BW"", ""is_mobile"": true}" 11561,4,257,2017-05-05 18:29:08,http://gutmann.info/kaylie.kirlin,0.1887399049,232.65.80.232,"{""location"": ""PN"", ""is_mobile"": false}" 11562,4,257,2017-04-28 07:38:59,http://dietrich.io/leonardo_nikolaus,0.7456756390,32.226.182.166,"{""location"": ""GU"", ""is_mobile"": true}" 11563,4,258,2017-01-29 03:27:24,http://goodwin.co/madison_casper,0.3420433811,81.74.90.243,"{""location"": ""AU"", ""is_mobile"": true}" 11564,4,258,2016-12-16 16:26:02,http://hodkiewicz.io/wilma.steuber,0.3203987676,196.198.44.46,"{""location"": ""WF"", ""is_mobile"": false}" 11565,4,258,2017-01-16 23:19:22,http://maggio.biz/betty.heidenreich,0.7598164387,84.28.62.136,"{""location"": ""IS"", ""is_mobile"": false}" 11566,4,258,2017-01-30 23:35:09,http://dicki.io/nelson,0.1483920646,210.172.183.216,"{""location"": ""SM"", ""is_mobile"": true}" 11567,4,258,2016-12-23 17:08:27,http://west.org/jacey,0.2732236549,114.170.27.229,"{""location"": ""BG"", ""is_mobile"": false}" 11568,4,258,2016-12-18 09:01:20,http://franeckicollier.biz/parker.wolff,0.5352635125,18.8.155.147,"{""location"": ""AM"", ""is_mobile"": true}" 11569,4,258,2017-04-11 01:43:19,http://fay.net/amari,0.1869228067,184.71.8.4,"{""location"": ""GN"", ""is_mobile"": false}" 11570,4,258,2017-01-07 02:31:17,http://little.name/demond,0.1716153007,162.234.23.253,"{""location"": ""FM"", ""is_mobile"": true}" 11571,4,258,2017-02-01 00:56:38,http://orn.info/trace.conn,0.4809649094,201.206.222.213,"{""location"": ""BZ"", ""is_mobile"": false}" 11572,4,258,2017-03-05 21:19:59,http://harvey.info/mia,0.3567055792,82.157.187.25,"{""location"": ""AD"", ""is_mobile"": true}" 11573,4,258,2017-03-31 21:01:10,http://denesik.name/pascale,0.7504794898,240.63.216.57,"{""location"": ""ME"", ""is_mobile"": true}" 11574,4,258,2017-02-22 23:22:28,http://powlowski.com/maya_rolfson,0.1032670211,235.24.144.145,"{""location"": ""BO"", ""is_mobile"": false}" 11575,4,258,2017-02-26 08:01:39,http://zulauf.co/dennis,0.5043870284,178.172.122.32,"{""location"": ""CU"", ""is_mobile"": true}" 11576,4,258,2017-04-20 14:53:10,http://pfefferdenesik.biz/eleonore.borer,0.2204665496,222.104.2.197,"{""location"": ""SM"", ""is_mobile"": true}" 11577,4,258,2017-06-10 07:43:23,http://ziemann.net/holly,0.9506536424,229.160.20.161,"{""location"": ""NR"", ""is_mobile"": false}" 11578,4,258,2016-12-14 18:10:02,http://reichert.org/cynthia,0.3147175302,9.223.159.173,"{""location"": ""ZM"", ""is_mobile"": false}" 11579,4,258,2017-03-20 08:28:32,http://lednerjohns.name/constance,0.4778086708,81.33.233.209,"{""location"": ""SC"", ""is_mobile"": false}" 11580,4,258,2017-05-07 01:56:35,http://hermann.name/verner_kunze,0.6558076442,47.201.210.106,"{""location"": ""LU"", ""is_mobile"": true}" 11581,4,258,2017-01-30 17:22:56,http://treutel.name/mavis_carroll,0.1775367463,219.110.133.126,"{""location"": ""UG"", ""is_mobile"": true}" 11582,4,258,2017-02-03 06:45:49,http://daughertyparisian.biz/fae_lind,0.7272382041,118.93.156.189,"{""location"": ""CZ"", ""is_mobile"": true}" 11583,4,258,2017-05-31 16:21:14,http://pacochasmith.org/jaylan,0.8925104143,67.174.210.158,"{""location"": ""BW"", ""is_mobile"": false}" 11584,4,258,2017-01-14 17:10:31,http://wuckert.co/kamron,0.3332484618,250.138.249.196,"{""location"": ""PF"", ""is_mobile"": true}" 11585,4,258,2016-12-21 14:21:30,http://waters.net/polly,0.9591109098,96.179.170.65,"{""location"": ""KN"", ""is_mobile"": false}" 11586,4,258,2017-04-14 09:17:26,http://parker.info/reba,0.2514086660,92.200.184.218,"{""location"": ""KH"", ""is_mobile"": false}" 11587,4,258,2017-05-11 13:18:56,http://wisoky.io/abelardo,0.6340643031,45.39.26.64,"{""location"": ""DJ"", ""is_mobile"": true}" 11588,4,258,2017-03-30 14:49:38,http://padbergjast.io/clay_huels,0.7319772874,253.242.110.22,"{""location"": ""UA"", ""is_mobile"": true}" 11589,4,258,2017-04-12 20:50:27,http://ankunding.io/aida_kilback,0.2108026609,254.25.31.210,"{""location"": ""WF"", ""is_mobile"": true}" 11590,4,258,2017-04-04 06:01:00,http://donnelly.com/loy.pacocha,0.5160398141,253.8.7.155,"{""location"": ""GE"", ""is_mobile"": true}" 11591,4,258,2017-04-01 01:10:32,http://bauch.name/lyric_jacobson,0.3176902796,33.8.179.51,"{""location"": ""NZ"", ""is_mobile"": true}" 11592,4,258,2017-04-17 18:42:44,http://effertz.io/damion.medhurst,0.7779726516,187.50.165.100,"{""location"": ""BE"", ""is_mobile"": false}" 11593,4,258,2017-01-01 12:43:22,http://homenick.info/soledad_reinger,0.4763954647,83.15.156.218,"{""location"": ""BQ"", ""is_mobile"": true}" 11594,4,258,2017-05-12 06:34:39,http://bode.io/reggie.howell,0.9146283791,216.249.78.91,"{""location"": ""SG"", ""is_mobile"": true}" 11595,4,258,2017-01-19 13:34:33,http://schinner.biz/nicholas,0.1958545459,102.220.121.247,"{""location"": ""VG"", ""is_mobile"": false}" 11596,4,258,2017-05-04 17:04:48,http://nikolaus.io/brooklyn_dickens,0.4162357608,202.83.2.210,"{""location"": ""KE"", ""is_mobile"": false}" 8552,3,192,2016-12-22 20:41:54,http://schulist.info/dahlia,,164.240.220.166,"{""location"": ""GW"", ""is_mobile"": true}" 8553,3,192,2017-02-26 12:33:10,http://hackett.io/martine,,229.67.193.151,"{""location"": ""LC"", ""is_mobile"": true}" 8554,3,192,2017-01-12 03:45:46,http://abernathy.biz/ike,,251.85.5.178,"{""location"": ""JM"", ""is_mobile"": true}" 8555,3,192,2017-02-22 13:51:54,http://koepp.io/celestino_spencer,,207.60.105.82,"{""location"": ""BJ"", ""is_mobile"": true}" 8556,3,192,2017-02-18 02:47:17,http://runte.org/ethel_goldner,,218.74.21.164,"{""location"": ""BN"", ""is_mobile"": true}" 8557,3,192,2017-06-04 20:26:22,http://will.io/glennie,,201.20.90.184,"{""location"": ""EC"", ""is_mobile"": false}" 8558,3,192,2016-12-25 23:52:42,http://stokes.info/marcelino,,173.193.194.188,"{""location"": ""BA"", ""is_mobile"": false}" 8559,3,192,2017-03-08 22:00:06,http://schuster.com/orpha_zemlak,,123.119.221.113,"{""location"": ""DO"", ""is_mobile"": false}" 8560,3,192,2017-06-12 03:50:32,http://friesen.name/alfonso,,96.77.89.166,"{""location"": ""ML"", ""is_mobile"": false}" 8561,3,192,2017-01-21 04:09:12,http://cartwrightullrich.co/adonis,,49.128.189.81,"{""location"": ""AF"", ""is_mobile"": false}" 8562,3,192,2017-06-07 20:57:31,http://tremblay.net/esmeralda.lehner,,119.191.11.4,"{""location"": ""NF"", ""is_mobile"": false}" 8563,3,192,2017-03-26 09:03:36,http://barrows.biz/verna,,55.76.79.170,"{""location"": ""IT"", ""is_mobile"": false}" 8564,3,192,2017-03-20 06:04:32,http://auer.net/dalton.grimes,,159.171.33.75,"{""location"": ""UG"", ""is_mobile"": true}" 8565,3,192,2017-05-23 05:22:35,http://hills.com/maxine_pagac,,90.66.45.172,"{""location"": ""ZM"", ""is_mobile"": false}" 8566,3,192,2017-01-27 16:32:41,http://shieldsmiller.org/roderick,,192.96.151.221,"{""location"": ""WS"", ""is_mobile"": false}" 8567,3,192,2017-03-10 22:11:43,http://becker.name/greta_sipes,,76.18.214.248,"{""location"": ""LU"", ""is_mobile"": false}" 8568,3,192,2017-04-06 15:45:03,http://jast.name/loyal,,29.209.62.85,"{""location"": ""GP"", ""is_mobile"": true}" 8569,3,192,2017-05-05 23:11:13,http://moore.biz/orlando_feest,,177.141.32.73,"{""location"": ""HT"", ""is_mobile"": true}" 8570,3,192,2017-02-09 20:19:16,http://fisherdurgan.net/raheem.wilderman,,90.103.170.54,"{""location"": ""AO"", ""is_mobile"": false}" 8571,3,192,2017-02-14 05:56:18,http://heathcote.biz/aric.hackett,,9.155.174.13,"{""location"": ""SG"", ""is_mobile"": false}" 8572,3,192,2017-05-03 03:08:20,http://fay.org/ryley.bartell,,234.250.211.236,"{""location"": ""LA"", ""is_mobile"": true}" 8573,3,192,2017-03-08 13:55:39,http://barton.org/cory,,90.234.85.150,"{""location"": ""DO"", ""is_mobile"": false}" 8574,3,192,2017-03-29 06:57:18,http://ratke.info/myrna,,123.139.236.59,"{""location"": ""KW"", ""is_mobile"": false}" 8575,3,192,2017-06-09 17:51:47,http://schuppe.org/elmo.zboncak,,252.118.45.209,"{""location"": ""EH"", ""is_mobile"": true}" 8576,3,192,2017-04-22 10:17:21,http://powlowski.info/stephanie_ankunding,,53.191.180.186,"{""location"": ""HK"", ""is_mobile"": true}" 8577,3,192,2017-01-24 06:30:42,http://littel.info/aubree.orn,,61.134.186.44,"{""location"": ""BD"", ""is_mobile"": false}" 8578,3,192,2017-06-01 17:21:39,http://ziemannhowe.io/kathleen,,133.149.178.13,"{""location"": ""MX"", ""is_mobile"": false}" 8579,3,192,2017-06-06 03:37:44,http://grahamondricka.net/elvie,,137.228.218.216,"{""location"": ""FK"", ""is_mobile"": true}" 8580,3,192,2016-12-14 12:43:24,http://swift.biz/luisa.turner,,165.200.78.123,"{""location"": ""MY"", ""is_mobile"": false}" 8581,3,193,2017-01-07 07:54:17,http://nitzsche.net/berry_parker,,7.16.86.86,"{""location"": ""SC"", ""is_mobile"": false}" 8582,3,193,2017-01-14 19:32:10,http://little.name/duane,,212.108.127.164,"{""location"": ""AU"", ""is_mobile"": true}" 8583,3,193,2017-02-22 20:44:13,http://purdy.org/nola.miller,,74.51.234.175,"{""location"": ""ZA"", ""is_mobile"": true}" 8584,3,193,2017-02-25 06:57:27,http://mertz.co/anahi_hilll,,83.247.70.94,"{""location"": ""TT"", ""is_mobile"": false}" 8585,3,193,2017-02-25 15:37:36,http://schamberger.co/louie_brown,,250.9.146.132,"{""location"": ""LK"", ""is_mobile"": false}" 8586,3,193,2017-05-25 20:45:47,http://walter.name/kim,,38.129.235.92,"{""location"": ""SN"", ""is_mobile"": false}" 8587,3,193,2017-02-01 18:42:15,http://treutel.org/zaria.hills,,79.35.166.201,"{""location"": ""MH"", ""is_mobile"": false}" 8588,3,193,2017-04-28 12:38:45,http://windlereffertz.co/walton_white,,119.109.64.104,"{""location"": ""NA"", ""is_mobile"": false}" 8589,3,193,2017-04-29 21:28:52,http://nitzschekoepp.biz/concepcion_aufderhar,,190.233.249.14,"{""location"": ""SA"", ""is_mobile"": true}" 8590,3,193,2017-04-21 00:30:10,http://rice.io/ru,,251.216.17.45,"{""location"": ""IS"", ""is_mobile"": false}" 8591,3,193,2017-04-21 19:55:41,http://marvinhills.name/brennan.leuschke,,189.96.159.13,"{""location"": ""BQ"", ""is_mobile"": true}" 8592,3,193,2017-03-04 03:02:55,http://herman.net/hazle.christiansen,,248.239.183.22,"{""location"": ""AX"", ""is_mobile"": true}" 8593,3,193,2017-03-13 02:07:13,http://rutherford.biz/alan,,105.106.132.40,"{""location"": ""GW"", ""is_mobile"": false}" 8594,3,193,2017-03-22 19:41:18,http://wolf.co/jason,,239.162.7.103,"{""location"": ""PS"", ""is_mobile"": false}" 8595,3,193,2017-03-02 00:39:11,http://stoltenberg.io/edgardo,,134.207.238.48,"{""location"": ""FI"", ""is_mobile"": true}" 8596,3,193,2016-12-31 21:04:18,http://batz.org/jeanne.terry,,142.105.176.230,"{""location"": ""EG"", ""is_mobile"": true}" 8597,3,193,2017-03-13 01:55:11,http://pacocha.io/lynn_rohan,,165.176.73.62,"{""location"": ""AD"", ""is_mobile"": true}" 8598,3,193,2017-05-11 06:04:07,http://mckenzie.net/johnpaul,,29.131.226.152,"{""location"": ""GF"", ""is_mobile"": true}" 8599,3,193,2017-01-18 11:42:17,http://kohler.io/ulices.reynolds,,94.132.242.54,"{""location"": ""UM"", ""is_mobile"": true}" 8600,3,193,2017-04-22 23:47:23,http://feestmoriette.name/kenyatta_pacocha,,44.166.240.204,"{""location"": ""SN"", ""is_mobile"": false}" 8601,3,193,2017-06-06 00:26:58,http://batz.name/tate.maggio,,108.222.200.73,"{""location"": ""IN"", ""is_mobile"": true}" 8602,3,193,2017-03-09 10:53:34,http://osinski.org/aditya,,235.189.172.215,"{""location"": ""BI"", ""is_mobile"": false}" 8603,3,193,2017-01-21 17:14:48,http://gaylordglover.co/thea.quigley,,146.173.209.191,"{""location"": ""MO"", ""is_mobile"": true}" 8604,3,193,2017-01-30 07:53:54,http://johnson.org/terrance.king,,183.67.81.20,"{""location"": ""BB"", ""is_mobile"": false}" 8605,3,193,2017-06-09 13:09:48,http://spinka.io/derick,,238.79.70.28,"{""location"": ""NA"", ""is_mobile"": false}" 8606,3,193,2017-03-25 23:32:01,http://schamberger.org/zion,,49.241.75.154,"{""location"": ""SZ"", ""is_mobile"": false}" 8607,3,193,2017-05-20 09:20:15,http://hegmann.info/johann,,35.170.85.206,"{""location"": ""RW"", ""is_mobile"": true}" 14499,5,326,2017-05-22 16:53:57,http://eichmannwalter.io/daren,,239.178.218.171,"{""location"": ""CM"", ""is_mobile"": false}" 14500,5,326,2017-06-12 21:47:56,http://bauch.net/triston,,63.206.18.140,"{""location"": ""EE"", ""is_mobile"": false}" 14501,5,326,2017-05-28 11:21:59,http://tremblay.name/lora.bashirian,,95.203.126.55,"{""location"": ""PN"", ""is_mobile"": false}" 14502,5,326,2017-01-04 05:53:46,http://brakus.co/murl.ullrich,,217.237.169.29,"{""location"": ""SI"", ""is_mobile"": false}" 14503,5,327,2017-04-27 00:07:31,http://dickensmoore.org/floyd,0.1384120216,116.62.149.11,"{""location"": ""BS"", ""is_mobile"": false}" 14504,5,327,2017-03-13 01:44:02,http://bechtelar.com/yasmine.strosin,0.7039573489,175.83.220.183,"{""location"": ""CI"", ""is_mobile"": true}" 14505,5,327,2017-01-22 17:36:10,http://anderson.net/shemar_jacobs,0.9622484281,56.252.127.143,"{""location"": ""TO"", ""is_mobile"": true}" 14506,5,327,2016-12-26 19:27:28,http://auer.com/anne,0.4387544220,136.219.117.106,"{""location"": ""MC"", ""is_mobile"": false}" 14507,5,327,2017-03-29 17:48:22,http://sauerboehm.info/mckayla_koepp,0.0918569334,160.87.146.221,"{""location"": ""BZ"", ""is_mobile"": false}" 14508,5,327,2017-03-19 05:06:17,http://bergnaum.name/faye,0.2519265351,92.243.245.142,"{""location"": ""RU"", ""is_mobile"": false}" 14509,5,327,2017-04-15 20:19:35,http://kautzer.co/terrill.gottlieb,0.8497015316,27.74.53.13,"{""location"": ""UA"", ""is_mobile"": false}" 14510,5,327,2017-01-19 20:24:38,http://schmittbednar.org/claudine,0.3198189676,203.46.73.163,"{""location"": ""MN"", ""is_mobile"": false}" 14511,5,327,2017-06-03 02:28:39,http://thiel.io/edgardo_padberg,0.3771096653,55.16.108.223,"{""location"": ""BQ"", ""is_mobile"": true}" 14512,5,327,2017-03-21 23:55:21,http://shieldslockman.info/daija_herman,0.0949968615,133.180.9.101,"{""location"": ""BS"", ""is_mobile"": true}" 14513,5,327,2017-03-30 02:30:11,http://ruecker.name/luigi.mertz,0.0376904458,48.95.46.223,"{""location"": ""CZ"", ""is_mobile"": false}" 14514,5,327,2017-03-14 20:19:26,http://wiza.info/judge_strosin,0.2565915226,38.137.186.245,"{""location"": ""BG"", ""is_mobile"": true}" 14515,5,327,2017-04-09 09:29:16,http://dickensrosenbaum.biz/macie,0.6964404538,112.102.138.123,"{""location"": ""IM"", ""is_mobile"": true}" 14516,5,327,2017-04-22 03:29:24,http://stracke.name/nikolas_huels,0.2010071826,107.227.56.240,"{""location"": ""GW"", ""is_mobile"": true}" 14517,5,327,2017-03-20 18:12:20,http://boylekuhlman.co/evelyn.green,0.9566766332,105.39.143.185,"{""location"": ""GR"", ""is_mobile"": false}" 14518,5,327,2017-01-13 21:47:19,http://rogahn.com/kristoffer_pfannerstill,0.2422783351,6.37.223.76,"{""location"": ""MD"", ""is_mobile"": true}" 14519,5,327,2017-05-28 01:16:25,http://jacobs.net/anita_mccullough,0.8751413972,220.61.151.217,"{""location"": ""PY"", ""is_mobile"": true}" 14520,5,327,2017-04-29 08:00:05,http://casper.biz/quinn.rath,0.6942506101,19.199.36.244,"{""location"": ""JM"", ""is_mobile"": false}" 14521,5,327,2017-01-13 10:47:33,http://dooley.io/amy.rippin,0.5574630694,170.12.166.88,"{""location"": ""BQ"", ""is_mobile"": false}" 14522,5,327,2016-12-19 02:24:49,http://schulistboehm.info/bell.huels,0.1642967026,197.105.217.160,"{""location"": ""BE"", ""is_mobile"": true}" 14523,5,327,2017-02-05 22:42:07,http://anderson.biz/sam_mann,0.4945594178,7.189.26.127,"{""location"": ""GH"", ""is_mobile"": false}" 14524,5,327,2017-03-23 01:10:56,http://ullrich.net/jarred_hickle,0.4450312038,243.17.70.251,"{""location"": ""AT"", ""is_mobile"": false}" 14525,5,327,2017-04-11 00:17:46,http://zieme.io/gerardo,0.1358359501,188.95.184.241,"{""location"": ""MZ"", ""is_mobile"": false}" 14526,5,327,2017-03-06 19:51:53,http://johnstonabbott.info/ludwig,0.7068070115,78.235.67.89,"{""location"": ""DJ"", ""is_mobile"": false}" 14527,5,327,2017-04-03 23:59:55,http://deckow.biz/ronny,0.5763898697,240.245.189.176,"{""location"": ""DZ"", ""is_mobile"": true}" 14528,5,327,2017-05-08 00:40:31,http://okon.io/morton,0.6900680867,170.160.141.195,"{""location"": ""VI"", ""is_mobile"": false}" 14529,5,327,2017-01-20 01:00:45,http://lehner.name/coleman,0.3159058553,190.208.54.213,"{""location"": ""MU"", ""is_mobile"": true}" 14530,5,327,2016-12-23 08:03:36,http://eichmannberge.net/leda,0.6245302262,55.247.207.181,"{""location"": ""CV"", ""is_mobile"": true}" 14531,5,327,2017-05-20 19:37:01,http://lindgrenhodkiewicz.com/demarco,0.1529621480,51.195.154.32,"{""location"": ""YE"", ""is_mobile"": true}" 14532,5,327,2017-04-14 21:26:06,http://wisoky.org/schuyler_hettinger,0.7536770636,94.30.180.74,"{""location"": ""KN"", ""is_mobile"": true}" 14533,5,327,2017-03-22 04:37:13,http://schroeder.biz/eddie.satterfield,0.7754900537,162.211.39.143,"{""location"": ""KW"", ""is_mobile"": false}" 14534,5,327,2017-05-05 21:57:08,http://koelpin.name/anibal.bartell,0.8426241034,101.80.137.75,"{""location"": ""PM"", ""is_mobile"": true}" 14535,5,327,2016-12-30 00:26:41,http://beer.name/eden,0.6555693016,107.171.247.15,"{""location"": ""MK"", ""is_mobile"": true}" 14536,5,327,2017-01-31 19:46:07,http://hettinger.name/roxanne,0.2598105405,47.44.29.231,"{""location"": ""IN"", ""is_mobile"": true}" 14537,5,327,2017-04-30 00:00:40,http://roweruel.net/lurline.carroll,0.1126398162,87.158.98.61,"{""location"": ""BR"", ""is_mobile"": true}" 14538,5,327,2017-03-10 19:06:39,http://reilly.co/winifred.kuhn,0.2396389600,45.90.238.140,"{""location"": ""TL"", ""is_mobile"": false}" 14539,5,327,2017-03-25 19:06:36,http://walker.info/isom,0.9718005506,199.203.86.18,"{""location"": ""BQ"", ""is_mobile"": true}" 14540,5,327,2017-02-12 13:56:06,http://lueilwitzgraham.com/clemmie_weber,0.9304386349,124.42.163.205,"{""location"": ""NI"", ""is_mobile"": false}" 14541,5,327,2017-01-13 00:48:59,http://schroedertorphy.io/vicenta_emard,0.7540365704,251.103.137.194,"{""location"": ""GS"", ""is_mobile"": false}" 14542,5,327,2016-12-24 22:41:44,http://herzog.net/santino_stracke,0.6078702063,69.123.134.76,"{""location"": ""NP"", ""is_mobile"": true}" 14543,5,327,2016-12-22 16:46:49,http://schoen.net/yasmeen,0.8856699433,94.151.17.157,"{""location"": ""VN"", ""is_mobile"": true}" 14544,5,327,2017-04-11 08:01:44,http://goldnerdaniel.io/marcia,0.0431837654,45.130.72.109,"{""location"": ""CL"", ""is_mobile"": false}" 14545,5,327,2017-01-23 09:28:45,http://beier.org/antonietta,0.5603793324,158.92.40.5,"{""location"": ""GM"", ""is_mobile"": true}" 14546,5,328,2017-05-09 14:29:31,http://ebert.co/ole.prohaska,0.3729042165,46.182.220.141,"{""location"": ""SL"", ""is_mobile"": false}" 14547,5,328,2017-05-17 01:08:57,http://pagac.name/ahmed,0.4744185308,70.16.197.55,"{""location"": ""SA"", ""is_mobile"": false}" 14548,5,328,2017-06-05 05:35:37,http://grady.biz/lennie.mitchell,0.1943797502,88.190.156.39,"{""location"": ""AU"", ""is_mobile"": true}" 14549,5,328,2017-03-12 08:40:55,http://lockman.name/ariane.rice,0.9180027710,12.121.128.32,"{""location"": ""MX"", ""is_mobile"": false}" 17484,6,394,2017-05-03 11:46:49,http://lowe.name/quentin.schuster,,218.85.26.24,"{""location"": ""LB"", ""is_mobile"": false}" 17485,6,394,2017-03-03 01:01:08,http://cronahagenes.net/mireille,,67.89.124.107,"{""location"": ""KR"", ""is_mobile"": false}" 17486,6,394,2017-01-31 12:53:54,http://boscohahn.biz/martina_swift,,234.55.192.72,"{""location"": ""BT"", ""is_mobile"": true}" 17487,6,394,2017-03-08 17:14:18,http://kuvalis.biz/katlynn.oconnell,,45.120.38.207,"{""location"": ""RO"", ""is_mobile"": true}" 17488,6,394,2017-02-05 07:37:40,http://hills.net/gus,,136.19.42.197,"{""location"": ""DM"", ""is_mobile"": true}" 17489,6,394,2017-04-05 19:56:57,http://vandervort.co/henderson.bailey,,66.179.176.166,"{""location"": ""EG"", ""is_mobile"": true}" 17490,6,394,2017-04-22 21:35:04,http://treutelmarquardt.co/nikki,,250.242.226.243,"{""location"": ""TD"", ""is_mobile"": false}" 17491,6,394,2017-04-25 09:30:29,http://wolf.co/moses_vandervort,,249.249.17.212,"{""location"": ""BF"", ""is_mobile"": false}" 17492,6,394,2017-03-20 12:47:22,http://nikolaus.io/margot,,87.235.84.133,"{""location"": ""DZ"", ""is_mobile"": true}" 17493,6,394,2017-01-24 06:20:56,http://haley.name/wiley,,18.114.128.147,"{""location"": ""SL"", ""is_mobile"": false}" 17494,6,394,2017-03-22 18:04:34,http://bashirian.name/monica_sanford,,162.100.14.210,"{""location"": ""KI"", ""is_mobile"": true}" 17495,6,394,2017-02-21 11:51:13,http://block.name/vernice,,79.77.169.171,"{""location"": ""NG"", ""is_mobile"": true}" 17496,6,394,2017-04-05 21:49:59,http://schmidt.io/trycia,,18.200.250.91,"{""location"": ""BY"", ""is_mobile"": true}" 17497,6,394,2017-01-21 07:32:45,http://bins.com/jasen.swaniawski,,67.179.137.234,"{""location"": ""KN"", ""is_mobile"": true}" 17498,6,394,2017-01-03 03:42:20,http://hintz.io/ezekiel_fahey,,212.79.246.51,"{""location"": ""NC"", ""is_mobile"": true}" 17499,6,394,2016-12-17 15:39:09,http://jast.biz/maud_jerde,,18.32.160.96,"{""location"": ""AE"", ""is_mobile"": false}" 17500,6,394,2017-03-12 12:42:52,http://botsfordkuhic.co/alfonso.pouros,,226.68.136.32,"{""location"": ""KI"", ""is_mobile"": false}" 17501,6,394,2017-05-01 10:50:13,http://predovicwehner.net/bettie_wuckert,,28.72.63.84,"{""location"": ""LT"", ""is_mobile"": false}" 17502,6,394,2017-03-04 18:27:37,http://romaguera.co/avery,,211.172.139.32,"{""location"": ""TG"", ""is_mobile"": true}" 17503,6,394,2016-12-21 08:17:31,http://ryanmorar.net/dixie_padberg,,100.133.128.183,"{""location"": ""QA"", ""is_mobile"": false}" 17504,6,394,2017-05-30 17:15:21,http://corkeryreichel.co/ava.padberg,,41.165.243.185,"{""location"": ""EG"", ""is_mobile"": false}" 17505,6,394,2016-12-15 03:38:22,http://kohlerzulauf.net/martin_tremblay,,56.122.239.199,"{""location"": ""LB"", ""is_mobile"": true}" 17506,6,394,2017-04-10 10:44:52,http://zieme.io/jonathan,,47.74.166.96,"{""location"": ""VG"", ""is_mobile"": false}" 17507,6,394,2017-06-09 20:18:15,http://rosenbaum.com/gordon_gorczany,,22.132.216.58,"{""location"": ""SH"", ""is_mobile"": true}" 17508,6,395,2017-01-16 06:00:56,http://block.com/weldon,,195.23.119.81,"{""location"": ""BY"", ""is_mobile"": false}" 17509,6,395,2017-05-29 03:32:23,http://barton.co/brook.bruen,,16.165.28.71,"{""location"": ""CC"", ""is_mobile"": true}" 17510,6,395,2017-06-11 10:54:48,http://doyle.biz/etha_wehner,,67.115.89.33,"{""location"": ""SY"", ""is_mobile"": false}" 17511,6,395,2017-05-18 03:29:11,http://kiehnbailey.name/guadalupe.jerde,,112.146.203.125,"{""location"": ""MG"", ""is_mobile"": false}" 17512,6,395,2017-02-20 03:46:16,http://bashirian.com/van,,165.210.16.78,"{""location"": ""PG"", ""is_mobile"": false}" 17513,6,395,2017-03-02 03:47:04,http://bartoletti.biz/eudora.roberts,,28.2.146.208,"{""location"": ""HK"", ""is_mobile"": true}" 17514,6,395,2016-12-14 23:03:35,http://schmitt.co/marc.prosacco,,148.64.216.180,"{""location"": ""MW"", ""is_mobile"": true}" 17515,6,395,2016-12-29 02:18:18,http://kuhn.io/dan,,224.39.40.247,"{""location"": ""BH"", ""is_mobile"": true}" 17516,6,395,2017-03-30 02:28:59,http://shanahankirlin.co/mazie,,92.49.31.136,"{""location"": ""GD"", ""is_mobile"": true}" 17517,6,395,2017-02-03 03:51:42,http://christiansen.org/lenora,,94.229.78.47,"{""location"": ""FR"", ""is_mobile"": false}" 17518,6,395,2017-02-05 11:27:53,http://nicolasterry.co/nettie,,24.221.68.104,"{""location"": ""TR"", ""is_mobile"": true}" 17519,6,395,2016-12-31 11:01:30,http://douglasemard.biz/carolyne_nader,,211.6.34.34,"{""location"": ""BY"", ""is_mobile"": true}" 17520,6,395,2017-04-13 23:08:38,http://corkery.biz/mortimer,,17.98.210.35,"{""location"": ""IT"", ""is_mobile"": false}" 17521,6,395,2017-03-06 14:04:56,http://rosenbaum.io/reba.gusikowski,,181.184.36.101,"{""location"": ""SR"", ""is_mobile"": true}" 17522,6,395,2016-12-18 05:56:03,http://langworthohara.info/bertrand_harber,,232.129.76.167,"{""location"": ""ET"", ""is_mobile"": false}" 17523,6,395,2017-05-20 09:50:18,http://gusikowski.io/alexandria,,33.23.248.224,"{""location"": ""TR"", ""is_mobile"": true}" 17524,6,395,2017-01-25 16:51:14,http://kub.co/dimitri,,250.74.145.82,"{""location"": ""IR"", ""is_mobile"": true}" 17525,6,395,2016-12-21 22:10:38,http://baumbach.com/modesto,,253.145.223.170,"{""location"": ""LR"", ""is_mobile"": false}" 17526,6,395,2017-01-01 11:15:23,http://rowehaag.info/jonatan.braun,,38.52.84.15,"{""location"": ""CK"", ""is_mobile"": true}" 17527,6,395,2017-04-16 12:09:01,http://will.com/carmine,,6.204.185.219,"{""location"": ""RS"", ""is_mobile"": false}" 17528,6,395,2016-12-18 03:37:50,http://stanton.com/karelle,,40.11.50.238,"{""location"": ""CR"", ""is_mobile"": false}" 17529,6,395,2017-04-14 18:27:01,http://baumbachmraz.org/imelda,,13.169.212.31,"{""location"": ""ES"", ""is_mobile"": true}" 17530,6,395,2017-03-14 04:56:54,http://beiersporer.org/lorna,,79.253.79.88,"{""location"": ""BQ"", ""is_mobile"": true}" 17531,6,395,2017-04-23 07:48:25,http://kleinwest.org/bailee,,153.74.48.31,"{""location"": ""SS"", ""is_mobile"": false}" 17532,6,395,2017-04-12 06:20:35,http://cummerata.name/maximillian.dooley,,168.163.245.93,"{""location"": ""NC"", ""is_mobile"": false}" 17533,6,395,2017-04-08 11:24:07,http://howe.info/evans,,183.105.154.13,"{""location"": ""FO"", ""is_mobile"": true}" 17534,6,395,2017-03-26 05:58:21,http://ko.net/jeff,,82.31.178.240,"{""location"": ""MS"", ""is_mobile"": false}" 17535,6,395,2017-01-25 15:12:40,http://lindgren.co/trea,,165.187.134.224,"{""location"": ""GN"", ""is_mobile"": false}" 17536,6,395,2017-06-11 09:17:21,http://hoppevandervort.info/viola,,60.3.11.247,"{""location"": ""MG"", ""is_mobile"": true}" 17537,6,395,2017-04-08 20:11:38,http://strosinparisian.name/jennings_feeney,,96.178.215.81,"{""location"": ""PK"", ""is_mobile"": false}" 17538,6,395,2017-06-12 15:31:42,http://robel.co/emie,,195.148.203.246,"{""location"": ""SC"", ""is_mobile"": false}" 17539,6,395,2017-05-14 22:34:35,http://lindgrenyost.net/marlin,,55.97.45.245,"{""location"": ""GY"", ""is_mobile"": true}" 4669,2,104,2017-01-20 06:16:20,http://medhurst.biz/elena,,31.165.75.88,"{""location"": ""GM"", ""is_mobile"": true}" 4670,2,104,2017-05-09 11:57:10,http://lindgrenflatley.biz/hailey,,240.193.20.62,"{""location"": ""MM"", ""is_mobile"": true}" 4671,2,104,2017-04-27 20:05:03,http://wiza.biz/damien.mcdermott,,121.121.17.101,"{""location"": ""UG"", ""is_mobile"": true}" 4672,2,104,2017-02-19 12:27:52,http://rippin.co/alycia,,74.183.240.66,"{""location"": ""DM"", ""is_mobile"": false}" 4673,2,104,2017-01-04 10:09:51,http://hirthe.name/noble_rosenbaum,,110.19.52.139,"{""location"": ""GR"", ""is_mobile"": true}" 4674,2,104,2017-01-13 20:54:08,http://haagblock.name/addison,,221.132.138.45,"{""location"": ""NG"", ""is_mobile"": false}" 4675,2,104,2016-12-20 08:58:04,http://nicolasgreenholt.io/corine.schinner,,128.48.161.112,"{""location"": ""MO"", ""is_mobile"": false}" 4676,2,104,2016-12-18 21:07:13,http://kiehn.co/camila,,132.100.96.26,"{""location"": ""MV"", ""is_mobile"": false}" 4677,2,104,2017-06-12 23:36:52,http://douglascarter.info/else_schaefer,,113.95.178.66,"{""location"": ""NL"", ""is_mobile"": false}" 4678,2,104,2017-02-03 18:50:28,http://quigley.org/harrison,,238.137.87.195,"{""location"": ""FR"", ""is_mobile"": false}" 4679,2,104,2017-03-28 05:15:36,http://walterdach.io/flo.quigley,,117.127.197.240,"{""location"": ""IE"", ""is_mobile"": false}" 4680,2,104,2017-04-01 05:50:57,http://cristkohler.net/bartholome,,246.109.61.25,"{""location"": ""GH"", ""is_mobile"": false}" 4681,2,104,2017-02-03 09:27:07,http://hilllwaters.info/dexter,,122.134.75.62,"{""location"": ""CY"", ""is_mobile"": false}" 4682,2,104,2016-12-17 22:52:19,http://abbottschultz.io/mercedes_legros,,130.64.24.24,"{""location"": ""AS"", ""is_mobile"": false}" 4683,2,104,2017-02-25 21:18:14,http://boscogerlach.io/lura_swaniawski,,132.73.4.193,"{""location"": ""PE"", ""is_mobile"": false}" 4684,2,104,2016-12-28 20:37:34,http://cruickshank.co/ru,,46.70.221.39,"{""location"": ""SM"", ""is_mobile"": false}" 4685,2,104,2017-02-22 18:25:41,http://bode.org/celestine,,114.24.19.64,"{""location"": ""WS"", ""is_mobile"": true}" 4686,2,104,2017-02-02 14:49:03,http://pagacvolkman.co/madonna_mills,,140.248.31.17,"{""location"": ""WS"", ""is_mobile"": false}" 4687,2,104,2017-06-02 19:22:30,http://bechtelar.net/cole,,76.206.140.23,"{""location"": ""NZ"", ""is_mobile"": true}" 4688,2,104,2017-05-19 10:56:41,http://murphy.net/yesenia.kuhlman,,162.83.78.236,"{""location"": ""JE"", ""is_mobile"": false}" 4689,2,104,2016-12-18 05:07:31,http://cartwright.co/orie,,52.174.22.166,"{""location"": ""CW"", ""is_mobile"": false}" 4690,2,104,2017-04-18 03:42:58,http://orn.name/astrid,,158.210.135.3,"{""location"": ""HM"", ""is_mobile"": false}" 4691,2,104,2017-01-17 20:08:49,http://keeling.name/grayce,,105.203.131.247,"{""location"": ""SJ"", ""is_mobile"": false}" 4692,2,104,2016-12-17 06:54:57,http://lemkegrady.info/kameron,,22.15.63.248,"{""location"": ""AX"", ""is_mobile"": false}" 4693,2,104,2017-02-19 23:54:11,http://hermann.io/sincere_towne,,237.88.53.162,"{""location"": ""FO"", ""is_mobile"": true}" 4694,2,104,2017-04-26 01:21:11,http://hermistonkuhic.name/pink_corkery,,183.198.36.53,"{""location"": ""BF"", ""is_mobile"": false}" 4695,2,104,2017-02-05 19:25:10,http://kaulke.name/opal,,117.250.38.136,"{""location"": ""PH"", ""is_mobile"": true}" 4696,2,104,2016-12-18 14:48:58,http://little.info/frederique,,109.60.62.28,"{""location"": ""ID"", ""is_mobile"": false}" 4697,2,104,2017-04-05 05:28:17,http://bartell.name/ruthie,,67.117.148.213,"{""location"": ""AW"", ""is_mobile"": true}" 4698,2,104,2017-03-01 02:27:32,http://legrosbednar.net/randi_kerluke,,131.186.245.86,"{""location"": ""AQ"", ""is_mobile"": false}" 4699,2,104,2017-03-01 08:19:05,http://gerlachokeefe.org/barbara.reichert,,194.153.98.179,"{""location"": ""MZ"", ""is_mobile"": true}" 4700,2,104,2017-04-23 19:02:24,http://simonisheel.org/juliet,,128.50.91.51,"{""location"": ""NO"", ""is_mobile"": false}" 4701,2,104,2017-05-19 00:09:33,http://reynolds.co/kaelyn_corwin,,185.52.173.137,"{""location"": ""JE"", ""is_mobile"": true}" 4702,2,104,2017-01-30 06:08:21,http://waelchi.info/micheal_kling,,60.47.7.221,"{""location"": ""MR"", ""is_mobile"": true}" 4703,2,104,2017-01-16 04:29:02,http://jerderice.co/kaandra_corkery,,128.196.54.78,"{""location"": ""GA"", ""is_mobile"": true}" 4704,2,104,2017-06-03 17:36:26,http://kovacek.info/amanda_bergstrom,,226.30.157.20,"{""location"": ""BS"", ""is_mobile"": true}" 4705,2,104,2017-06-04 21:01:33,http://emard.name/astrid_mcglynn,,88.143.88.93,"{""location"": ""MM"", ""is_mobile"": true}" 4706,2,104,2017-06-06 12:53:32,http://wunsch.co/holden,,155.168.141.205,"{""location"": ""TD"", ""is_mobile"": true}" 4707,2,104,2017-06-09 00:41:43,http://damore.name/karley_schiller,,113.79.225.253,"{""location"": ""GQ"", ""is_mobile"": false}" 4708,2,104,2017-05-14 19:49:14,http://braun.io/ella.armstrong,,225.149.9.115,"{""location"": ""VC"", ""is_mobile"": true}" 4709,2,104,2017-06-06 13:39:50,http://schillervolkman.co/summer,,12.188.69.12,"{""location"": ""CD"", ""is_mobile"": true}" 4710,2,104,2017-05-29 23:52:04,http://dare.co/madonna_wunsch,,105.128.100.72,"{""location"": ""RW"", ""is_mobile"": true}" 4711,2,104,2017-02-20 13:00:57,http://monahanluettgen.io/conor,,242.61.123.84,"{""location"": ""BG"", ""is_mobile"": false}" 4712,2,104,2017-02-22 07:18:23,http://hickle.net/gage,,210.85.204.141,"{""location"": ""CG"", ""is_mobile"": true}" 4713,2,104,2017-03-09 01:20:06,http://dickens.com/morgan_moen,,178.210.92.74,"{""location"": ""PH"", ""is_mobile"": true}" 4714,2,104,2017-04-20 02:22:03,http://hirthe.io/felton.effertz,,167.95.147.135,"{""location"": ""BR"", ""is_mobile"": false}" 4715,2,104,2017-02-21 16:20:46,http://paucek.net/howell,,158.214.71.195,"{""location"": ""NE"", ""is_mobile"": false}" 4716,2,104,2016-12-25 13:21:56,http://powlowski.biz/aric,,232.124.9.57,"{""location"": ""AZ"", ""is_mobile"": false}" 4717,2,104,2017-04-04 13:44:48,http://sanford.co/solon,,244.70.22.195,"{""location"": ""TG"", ""is_mobile"": true}" 4718,2,104,2017-01-07 20:29:25,http://fadel.net/emerson,,178.40.109.232,"{""location"": ""TG"", ""is_mobile"": true}" 4719,2,104,2017-02-09 23:54:08,http://daugherty.io/demarco.rosenbaum,,12.188.118.106,"{""location"": ""PR"", ""is_mobile"": true}" 4720,2,104,2017-04-10 08:31:09,http://witting.com/kaylin,,122.72.201.49,"{""location"": ""BD"", ""is_mobile"": true}" 4721,2,104,2016-12-22 18:40:09,http://vonrueden.name/yvonne_gutmann,,218.219.97.31,"{""location"": ""UY"", ""is_mobile"": true}" 4722,2,104,2017-02-02 05:34:29,http://casper.net/petra.hills,,214.76.113.211,"{""location"": ""BM"", ""is_mobile"": false}" 4723,2,104,2017-01-12 10:05:41,http://rogahnwill.net/jermaine_rau,,120.110.204.204,"{""location"": ""CX"", ""is_mobile"": false}" 4724,2,104,2017-06-01 06:01:59,http://kshlerin.net/elisha.harris,,180.62.248.147,"{""location"": ""MU"", ""is_mobile"": false}" 11597,4,258,2017-01-20 11:08:49,http://browntoy.com/seth.nitzsche,0.3900888372,72.60.178.143,"{""location"": ""TG"", ""is_mobile"": false}" 11598,4,258,2017-01-24 11:29:38,http://cristbrekke.biz/germaine_mertz,0.9043735709,138.3.149.68,"{""location"": ""BH"", ""is_mobile"": false}" 11599,4,258,2017-05-07 13:04:12,http://erdman.info/dominique,0.5075391706,44.248.38.165,"{""location"": ""TK"", ""is_mobile"": false}" 11600,4,258,2017-02-10 22:42:42,http://schinnergusikowski.net/cordie.breitenberg,0.0143759593,244.20.220.226,"{""location"": ""IM"", ""is_mobile"": false}" 11601,4,258,2017-05-10 02:34:01,http://gaylord.co/mariano,0.8956517886,17.41.102.118,"{""location"": ""JO"", ""is_mobile"": false}" 11602,4,258,2017-01-30 08:08:14,http://swaniawski.biz/noah,0.1733337392,222.50.73.37,"{""location"": ""JM"", ""is_mobile"": true}" 11603,4,258,2017-02-21 17:37:20,http://cronin.net/angelica,0.0798292794,227.191.65.52,"{""location"": ""NZ"", ""is_mobile"": true}" 11604,4,258,2017-02-28 04:36:18,http://wiegand.biz/kara,0.3541409111,127.13.192.123,"{""location"": ""IM"", ""is_mobile"": false}" 11605,4,258,2017-05-24 13:07:58,http://kreiger.name/alva,0.8080927846,105.17.239.200,"{""location"": ""PF"", ""is_mobile"": false}" 11606,4,258,2017-05-15 21:13:09,http://wunsch.biz/lionel,0.9413530043,140.113.246.116,"{""location"": ""BJ"", ""is_mobile"": false}" 11607,4,258,2017-05-20 18:41:44,http://raynor.com/dan,0.4784039353,187.189.22.41,"{""location"": ""IM"", ""is_mobile"": false}" 11608,4,258,2017-03-03 17:41:33,http://walker.biz/delia,0.6215827555,160.190.212.42,"{""location"": ""PF"", ""is_mobile"": true}" 11609,4,258,2017-01-02 02:03:59,http://klocko.org/wyatt,0.1492230972,67.240.55.68,"{""location"": ""NL"", ""is_mobile"": true}" 11610,4,258,2016-12-31 09:20:59,http://flatley.org/gay.macgyver,0.9433086064,48.175.209.239,"{""location"": ""CG"", ""is_mobile"": false}" 11611,4,258,2017-04-01 16:53:45,http://osinski.io/jewell,0.6565067381,210.193.243.147,"{""location"": ""HT"", ""is_mobile"": true}" 11612,4,258,2017-06-09 16:20:40,http://botsford.com/jaylin_dietrich,0.4596350877,237.173.236.99,"{""location"": ""PW"", ""is_mobile"": false}" 11613,4,258,2017-02-10 09:25:43,http://nienow.com/aaron,0.5896261969,38.225.12.154,"{""location"": ""ML"", ""is_mobile"": true}" 11614,4,258,2017-01-22 01:47:21,http://rodriguez.net/demario,0.4315269652,211.242.142.25,"{""location"": ""SY"", ""is_mobile"": false}" 11615,4,258,2017-01-20 01:18:17,http://keebler.org/elna.bahringer,0.3768888458,179.39.206.171,"{""location"": ""IO"", ""is_mobile"": false}" 11616,4,258,2017-04-23 13:25:35,http://mante.com/adonis.koepp,0.8444580186,230.169.124.168,"{""location"": ""UA"", ""is_mobile"": false}" 11617,4,258,2017-06-11 19:23:56,http://satterfield.io/julianne.hettinger,0.0588100454,155.157.6.5,"{""location"": ""SI"", ""is_mobile"": true}" 11618,4,258,2017-02-20 05:27:41,http://funk.name/brandon_kovacek,0.0345742021,101.129.153.100,"{""location"": ""QA"", ""is_mobile"": true}" 11619,4,258,2016-12-20 03:17:07,http://senger.io/otis,0.6624472002,16.121.189.29,"{""location"": ""BQ"", ""is_mobile"": true}" 11620,4,258,2017-03-25 08:21:22,http://stanton.name/eryn.herzog,0.0355706625,107.140.51.117,"{""location"": ""MU"", ""is_mobile"": true}" 11621,4,258,2017-03-07 12:06:01,http://bartonbatz.org/bonita_wilderman,0.9233750255,145.53.238.166,"{""location"": ""SG"", ""is_mobile"": true}" 11622,4,258,2017-03-21 15:06:10,http://kovacekhowell.co/alfred.schowalter,0.2415424299,86.134.106.196,"{""location"": ""RW"", ""is_mobile"": true}" 11623,4,258,2017-06-05 06:34:36,http://frami.org/fannie,0.6530475413,52.145.157.112,"{""location"": ""KE"", ""is_mobile"": false}" 11624,4,259,2017-01-04 14:39:02,http://medhurst.biz/dahlia.reichel,0.0177962369,21.80.41.186,"{""location"": ""MG"", ""is_mobile"": true}" 11625,4,259,2017-04-29 09:43:45,http://harrisjohnston.co/izaiah_bosco,0.9534862279,98.179.241.82,"{""location"": ""IT"", ""is_mobile"": true}" 11626,4,259,2017-01-24 01:52:38,http://walterkirlin.io/trystan_paucek,0.6879869612,146.6.114.150,"{""location"": ""CF"", ""is_mobile"": false}" 11627,4,259,2017-06-06 14:24:47,http://kozeymaggio.io/sheila.batz,0.6485500657,83.118.52.254,"{""location"": ""FO"", ""is_mobile"": true}" 11628,4,259,2017-05-15 06:53:08,http://daviscollier.info/katherine,0.2495910457,108.90.57.215,"{""location"": ""BQ"", ""is_mobile"": true}" 11629,4,259,2017-05-21 23:56:53,http://bradtke.info/amiya_barton,0.7595586095,44.35.198.217,"{""location"": ""SG"", ""is_mobile"": false}" 11630,4,259,2017-05-28 01:40:03,http://haleyfritsch.biz/dannie.mcclure,0.3597333150,5.221.111.165,"{""location"": ""KI"", ""is_mobile"": false}" 11631,4,259,2017-05-09 13:07:36,http://windler.name/phoebe.orn,0.1897607168,179.27.120.102,"{""location"": ""NL"", ""is_mobile"": true}" 11632,4,259,2016-12-24 15:17:02,http://tremblayprice.co/gaylord,0.7199524437,98.175.232.164,"{""location"": ""TC"", ""is_mobile"": true}" 11633,4,259,2017-06-10 18:18:10,http://zieme.net/alexander.haag,0.9213563285,185.219.144.53,"{""location"": ""MY"", ""is_mobile"": false}" 11634,4,259,2017-03-29 13:14:31,http://herman.co/alverta,0.6432323453,95.87.137.196,"{""location"": ""SJ"", ""is_mobile"": false}" 11635,4,259,2017-01-15 08:44:32,http://hilpertschmeler.net/beulah,0.1990038265,56.50.11.43,"{""location"": ""KY"", ""is_mobile"": true}" 11636,4,259,2017-02-23 17:35:05,http://lueilwitz.net/ellen,0.4356598925,223.5.23.118,"{""location"": ""CL"", ""is_mobile"": true}" 11637,4,259,2017-02-11 06:35:41,http://hyattwiegand.name/nellie,0.1053231467,55.213.190.218,"{""location"": ""MK"", ""is_mobile"": true}" 11638,4,259,2017-02-17 14:33:53,http://boyer.info/adan.rice,0.4535596475,51.174.159.137,"{""location"": ""BI"", ""is_mobile"": true}" 11639,4,259,2017-02-20 11:35:39,http://schuppestark.io/armand.baumbach,0.5464048724,215.216.233.169,"{""location"": ""BQ"", ""is_mobile"": true}" 11640,4,259,2017-03-10 18:24:53,http://littel.co/barton,0.0000263975,195.95.43.124,"{""location"": ""KM"", ""is_mobile"": false}" 11641,4,259,2017-03-23 18:49:31,http://hermistonhaley.com/saul.wilkinson,0.3037041812,218.95.204.105,"{""location"": ""MC"", ""is_mobile"": false}" 11642,4,259,2017-04-25 07:38:32,http://thiel.io/donnell,0.6907806031,236.24.240.227,"{""location"": ""CN"", ""is_mobile"": true}" 11643,4,259,2017-04-29 23:18:34,http://okunevawaters.io/maximo,0.1599861134,196.144.96.234,"{""location"": ""FO"", ""is_mobile"": true}" 11644,4,259,2017-01-26 13:22:55,http://dicki.biz/rebeka,0.2692828993,251.76.220.192,"{""location"": ""PN"", ""is_mobile"": false}" 11645,4,260,2017-04-28 01:07:12,http://schneider.co/elena,0.7493779905,81.181.214.186,"{""location"": ""LC"", ""is_mobile"": true}" 11646,4,260,2017-04-30 07:58:36,http://gusikowski.name/lloyd,0.6218600175,218.159.90.144,"{""location"": ""AE"", ""is_mobile"": false}" 11647,4,260,2017-01-20 21:42:03,http://heelfadel.net/favian.denesik,0.0388832374,47.2.237.18,"{""location"": ""SD"", ""is_mobile"": true}" 8608,3,193,2017-04-06 11:49:43,http://hamillcorwin.info/daryl_ortiz,,244.114.96.230,"{""location"": ""SZ"", ""is_mobile"": true}" 8609,3,193,2017-03-01 05:38:18,http://wisoky.co/torrey_yost,,172.105.228.61,"{""location"": ""ZM"", ""is_mobile"": true}" 8610,3,193,2017-05-02 03:10:31,http://kreigerwalker.co/annabel_jacobi,,244.6.223.140,"{""location"": ""CM"", ""is_mobile"": false}" 8611,3,193,2016-12-24 04:48:15,http://keelinglesch.info/rozella_thompson,,80.130.125.87,"{""location"": ""DJ"", ""is_mobile"": false}" 8612,3,193,2017-02-01 07:12:27,http://moriette.net/shawna_mante,,129.24.92.166,"{""location"": ""SR"", ""is_mobile"": false}" 8613,3,193,2017-03-15 08:22:24,http://becker.co/elinore.bahringer,,12.66.139.194,"{""location"": ""KE"", ""is_mobile"": false}" 8614,3,193,2017-05-26 15:31:43,http://mertz.info/wilfredo_mosciski,,96.140.220.237,"{""location"": ""KW"", ""is_mobile"": false}" 8615,3,193,2017-02-19 20:14:27,http://marquardt.biz/rosalind,,200.232.107.86,"{""location"": ""SA"", ""is_mobile"": false}" 8616,3,193,2017-02-08 04:31:10,http://strackejaskolski.name/michael,,113.30.199.122,"{""location"": ""BQ"", ""is_mobile"": true}" 8617,3,193,2017-05-06 19:09:29,http://crist.org/jazmyne.harris,,11.57.52.185,"{""location"": ""SR"", ""is_mobile"": false}" 8618,3,193,2017-04-07 16:15:31,http://maggio.io/london,,197.195.42.163,"{""location"": ""MP"", ""is_mobile"": false}" 8619,3,193,2017-02-10 00:36:35,http://abbott.com/rosetta,,47.217.247.75,"{""location"": ""AM"", ""is_mobile"": false}" 8620,3,193,2017-06-10 08:57:16,http://rolfson.io/vaughn,,189.244.196.14,"{""location"": ""MX"", ""is_mobile"": false}" 8621,3,193,2017-05-05 23:32:31,http://gorczany.io/orlo,,173.234.224.203,"{""location"": ""AI"", ""is_mobile"": true}" 8622,3,193,2016-12-25 20:46:39,http://pacochahintz.biz/talia,,119.119.219.176,"{""location"": ""NL"", ""is_mobile"": true}" 8623,3,193,2017-04-26 14:43:09,http://oconnerschamberger.org/lina,,186.9.234.129,"{""location"": ""KP"", ""is_mobile"": false}" 8624,3,193,2017-05-21 21:08:19,http://gulgowskimetz.co/clifford,,12.128.231.203,"{""location"": ""NC"", ""is_mobile"": false}" 8625,3,193,2017-02-14 09:26:34,http://morar.info/adolph,,58.111.129.120,"{""location"": ""LI"", ""is_mobile"": false}" 8626,3,193,2017-06-07 14:57:32,http://toy.info/aryanna,,198.162.129.95,"{""location"": ""PA"", ""is_mobile"": false}" 8627,3,193,2017-02-06 23:34:14,http://thiel.co/selmer_predovic,,18.175.244.198,"{""location"": ""TR"", ""is_mobile"": true}" 8628,3,194,2017-04-09 16:22:22,http://hackett.org/lori,,153.9.24.101,"{""location"": ""NP"", ""is_mobile"": false}" 8629,3,194,2016-12-22 04:01:28,http://buckridgekilback.info/maritza,,185.27.118.51,"{""location"": ""TZ"", ""is_mobile"": false}" 8630,3,194,2017-02-09 16:49:30,http://ward.info/bernice_oconnell,,150.13.135.9,"{""location"": ""LU"", ""is_mobile"": false}" 8631,3,194,2017-05-25 10:48:25,http://ankundingbarton.com/webster,,8.75.170.32,"{""location"": ""SB"", ""is_mobile"": false}" 8632,3,194,2017-04-10 18:10:35,http://gibson.info/abbigail,,165.188.230.231,"{""location"": ""US"", ""is_mobile"": true}" 8633,3,194,2017-05-25 10:15:31,http://dicki.co/aidan.bechtelar,,216.163.154.135,"{""location"": ""GG"", ""is_mobile"": false}" 8634,3,194,2017-05-04 12:17:57,http://rempel.biz/annamarie_watsica,,217.5.112.248,"{""location"": ""VG"", ""is_mobile"": true}" 8635,3,194,2017-02-06 00:23:59,http://rempel.net/quinten_larson,,76.38.21.116,"{""location"": ""TW"", ""is_mobile"": true}" 8636,3,194,2017-03-16 06:25:39,http://spencerhammes.org/philip,,30.7.38.75,"{""location"": ""KR"", ""is_mobile"": true}" 8637,3,194,2017-05-31 20:33:16,http://medhurst.com/gregg.grant,,203.38.71.77,"{""location"": ""RW"", ""is_mobile"": true}" 8638,3,194,2017-03-20 14:58:31,http://dickinsonnienow.io/lauryn,,226.4.118.170,"{""location"": ""DJ"", ""is_mobile"": true}" 8639,3,194,2016-12-23 04:33:54,http://trompmante.net/marcel.roob,,195.31.173.217,"{""location"": ""VE"", ""is_mobile"": false}" 8640,3,194,2017-06-11 17:11:44,http://mayert.co/kendall_parisian,,76.4.63.13,"{""location"": ""GR"", ""is_mobile"": false}" 8641,3,194,2017-02-23 02:41:50,http://doylelebsack.name/jacquelyn,,159.177.61.111,"{""location"": ""BT"", ""is_mobile"": true}" 8642,3,194,2017-02-23 00:43:57,http://jacobs.org/jesus,,49.205.8.174,"{""location"": ""GW"", ""is_mobile"": true}" 8643,3,194,2017-04-11 22:07:32,http://haagheathcote.info/joshua,,76.238.119.12,"{""location"": ""AI"", ""is_mobile"": false}" 8644,3,194,2017-05-25 13:51:52,http://konopelski.info/brayan.breitenberg,,243.129.211.168,"{""location"": ""BB"", ""is_mobile"": true}" 8645,3,194,2017-04-04 10:45:36,http://roberts.net/crystal_aufderhar,,33.160.227.109,"{""location"": ""SB"", ""is_mobile"": false}" 8646,3,194,2017-04-23 22:22:31,http://franeckimaggio.org/dorris.dooley,,199.212.60.46,"{""location"": ""CD"", ""is_mobile"": true}" 8647,3,194,2017-02-26 06:23:59,http://grant.net/oral,,31.46.151.19,"{""location"": ""KG"", ""is_mobile"": true}" 8648,3,194,2017-04-26 04:52:00,http://hudson.com/jose,,117.253.95.202,"{""location"": ""GD"", ""is_mobile"": true}" 8649,3,194,2017-03-24 04:01:20,http://hahn.name/laurine,,111.28.142.92,"{""location"": ""NG"", ""is_mobile"": false}" 8650,3,194,2017-04-20 09:12:34,http://bergnaumpfannerstill.name/leann,,44.178.162.197,"{""location"": ""SI"", ""is_mobile"": false}" 8651,3,194,2017-06-04 07:17:42,http://ankunding.name/marc.reynolds,,58.153.106.61,"{""location"": ""IS"", ""is_mobile"": true}" 8652,3,194,2016-12-21 06:50:27,http://waelchi.biz/arnoldo.boehm,,229.123.156.235,"{""location"": ""MC"", ""is_mobile"": true}" 8653,3,194,2017-05-17 01:33:41,http://fisher.net/orval,,84.103.92.38,"{""location"": ""PG"", ""is_mobile"": false}" 8654,3,194,2017-05-15 19:32:10,http://altenwerthhowell.info/benton,,35.135.213.154,"{""location"": ""TN"", ""is_mobile"": false}" 8655,3,194,2017-05-19 08:12:47,http://heathcotelindgren.org/lisa,,250.119.53.18,"{""location"": ""DE"", ""is_mobile"": false}" 8656,3,194,2016-12-20 09:17:01,http://hills.io/eloise,,247.230.173.247,"{""location"": ""ZA"", ""is_mobile"": true}" 8657,3,194,2017-05-26 11:20:30,http://krisreinger.biz/korbin_hackett,,41.13.228.96,"{""location"": ""EH"", ""is_mobile"": true}" 8658,3,194,2017-03-10 16:57:01,http://wuckert.name/maxime,,113.205.171.204,"{""location"": ""GM"", ""is_mobile"": false}" 8659,3,194,2017-01-26 22:48:54,http://doyle.name/elton,,152.29.221.13,"{""location"": ""GQ"", ""is_mobile"": false}" 8660,3,194,2017-04-25 18:53:49,http://mueller.info/jeie.mueller,,172.250.191.101,"{""location"": ""SM"", ""is_mobile"": false}" 8661,3,194,2017-06-08 07:22:34,http://jones.net/olen_kaulke,,156.26.214.55,"{""location"": ""AD"", ""is_mobile"": false}" 8662,3,194,2017-05-24 13:53:22,http://zemlakmacejkovic.info/roosevelt_hirthe,,27.185.149.50,"{""location"": ""CR"", ""is_mobile"": true}" 14550,5,328,2017-02-10 02:51:42,http://champlinschowalter.biz/telly,0.4564437382,238.99.75.245,"{""location"": ""GD"", ""is_mobile"": false}" 14551,5,328,2016-12-16 01:16:19,http://faybahringer.name/karli.waters,0.8144665863,93.187.14.231,"{""location"": ""VC"", ""is_mobile"": false}" 14552,5,328,2017-03-29 17:31:55,http://harvey.info/alvena,0.9050428570,84.29.27.74,"{""location"": ""PL"", ""is_mobile"": true}" 14553,5,328,2017-04-25 18:38:53,http://durganratke.com/sincere,0.4635440620,244.41.31.51,"{""location"": ""LY"", ""is_mobile"": false}" 14554,5,328,2017-05-21 05:10:24,http://reingerabbott.co/ethyl,0.1638787548,188.248.163.143,"{""location"": ""UM"", ""is_mobile"": true}" 14555,5,328,2017-06-02 21:25:58,http://baileyschoen.org/name.braun,0.9053264796,243.4.176.34,"{""location"": ""LV"", ""is_mobile"": false}" 14556,5,328,2017-01-04 13:38:00,http://bartell.org/carlotta.boyle,0.7439635686,162.229.27.86,"{""location"": ""CD"", ""is_mobile"": true}" 14557,5,328,2017-05-18 00:06:34,http://wittingdickens.net/eloise,0.7697618414,157.63.209.18,"{""location"": ""HK"", ""is_mobile"": false}" 14558,5,328,2017-03-07 04:12:53,http://hansen.io/rolando,0.6233004508,220.32.50.133,"{""location"": ""BE"", ""is_mobile"": true}" 14559,5,328,2017-01-18 14:24:38,http://boyer.biz/carmela,0.9226546899,136.205.254.253,"{""location"": ""MD"", ""is_mobile"": true}" 14560,5,328,2017-02-01 06:37:22,http://stiedemannhamill.net/sabina,0.5646388168,59.180.154.246,"{""location"": ""MF"", ""is_mobile"": false}" 14561,5,328,2016-12-22 23:10:12,http://schmeler.co/myrtie,0.1284955313,166.125.82.133,"{""location"": ""MF"", ""is_mobile"": true}" 14562,5,328,2017-01-21 21:39:28,http://fahey.co/janelle_heidenreich,0.5910432435,105.15.5.228,"{""location"": ""CM"", ""is_mobile"": true}" 14563,5,328,2017-02-04 10:53:43,http://hudsonbartell.io/bennie_kuhic,0.7343763449,165.143.28.172,"{""location"": ""LR"", ""is_mobile"": false}" 14564,5,328,2017-02-28 18:15:05,http://kunzechamplin.info/urban,0.5479244681,213.139.140.231,"{""location"": ""FI"", ""is_mobile"": false}" 14565,5,328,2017-01-04 12:34:12,http://marvinlittle.biz/roberta,0.8203340665,154.245.207.46,"{""location"": ""BA"", ""is_mobile"": true}" 14566,5,328,2017-02-11 00:19:36,http://flatley.net/montana,0.3156031208,240.163.25.105,"{""location"": ""NE"", ""is_mobile"": false}" 14567,5,328,2017-04-17 03:45:33,http://schinner.org/alana.pfannerstill,0.1735591024,169.71.105.103,"{""location"": ""MD"", ""is_mobile"": false}" 14568,5,328,2017-01-18 05:45:50,http://mayervon.org/axel,0.4487964975,101.99.13.123,"{""location"": ""CH"", ""is_mobile"": false}" 14569,5,328,2017-04-05 22:50:13,http://mann.com/bart.ruecker,0.4113343047,88.135.48.85,"{""location"": ""PS"", ""is_mobile"": false}" 14570,5,328,2017-03-29 13:42:35,http://brakus.name/justus.armstrong,0.9561056189,87.153.163.195,"{""location"": ""SR"", ""is_mobile"": false}" 14571,5,328,2017-03-30 18:28:41,http://moore.org/pierce_nolan,0.2376822097,191.156.122.224,"{""location"": ""TJ"", ""is_mobile"": true}" 14572,5,328,2017-06-01 12:37:29,http://purdykeeling.name/antonio,0.9663228153,18.111.43.200,"{""location"": ""KW"", ""is_mobile"": false}" 14573,5,328,2017-05-31 10:54:51,http://langworthdickinson.co/erick_west,0.2667171005,112.87.153.95,"{""location"": ""TV"", ""is_mobile"": false}" 14574,5,328,2017-05-17 19:31:18,http://koelpin.info/marcos_schroeder,0.6920374386,76.173.23.135,"{""location"": ""GW"", ""is_mobile"": true}" 14575,5,328,2017-01-23 21:27:10,http://hammes.biz/rey_ernser,0.7693724317,97.209.190.228,"{""location"": ""FI"", ""is_mobile"": false}" 14576,5,328,2017-03-22 17:39:03,http://botsford.org/corene_anderson,0.2808732964,47.4.47.161,"{""location"": ""AS"", ""is_mobile"": false}" 14577,5,328,2016-12-19 10:06:26,http://bednarmills.com/brice_west,0.1004768496,139.228.61.168,"{""location"": ""DK"", ""is_mobile"": false}" 14578,5,328,2016-12-20 12:07:27,http://wolff.org/rosalee.kuphal,0.3223591949,5.244.61.52,"{""location"": ""BH"", ""is_mobile"": true}" 14579,5,328,2017-01-24 07:35:01,http://thompson.io/boris,0.3108516435,2.172.63.126,"{""location"": ""MT"", ""is_mobile"": true}" 14580,5,328,2017-06-02 08:51:34,http://hackett.org/kallie_rolfson,0.1337023238,240.100.104.125,"{""location"": ""LA"", ""is_mobile"": true}" 14581,5,328,2017-04-08 00:59:18,http://pollicheffertz.info/cecilia,0.0460379950,197.170.217.114,"{""location"": ""AX"", ""is_mobile"": true}" 14582,5,328,2017-01-05 19:58:41,http://davis.net/taylor.moen,0.0830697353,175.232.132.229,"{""location"": ""OM"", ""is_mobile"": true}" 14583,5,328,2016-12-24 06:39:32,http://haleyrodriguez.co/hellen,0.4383022794,115.127.25.240,"{""location"": ""BM"", ""is_mobile"": false}" 14584,5,328,2017-01-26 23:30:30,http://abshire.co/kira_denesik,0.6178580243,36.91.58.174,"{""location"": ""PA"", ""is_mobile"": false}" 14585,5,328,2017-04-14 04:30:05,http://jacobsonfay.org/justina.bailey,0.4496018982,145.195.143.146,"{""location"": ""CY"", ""is_mobile"": true}" 14586,5,328,2017-01-07 01:20:24,http://wintheiser.net/susanna,0.6628999980,190.148.50.243,"{""location"": ""DM"", ""is_mobile"": false}" 14587,5,328,2017-02-12 09:43:56,http://faheyhahn.org/payton_crona,0.2576172604,151.104.240.45,"{""location"": ""DK"", ""is_mobile"": false}" 14588,5,328,2017-01-18 06:22:50,http://gleichnerhermann.org/wyman,0.3478715874,143.232.222.142,"{""location"": ""FR"", ""is_mobile"": false}" 14589,5,328,2017-03-24 18:27:33,http://gutkowski.org/janelle,0.2064236482,93.176.157.123,"{""location"": ""BV"", ""is_mobile"": true}" 14590,5,328,2016-12-17 20:59:25,http://fritsch.biz/maymie_howe,0.5018278703,159.148.33.8,"{""location"": ""SX"", ""is_mobile"": true}" 14591,5,328,2017-03-30 20:58:14,http://heathcote.info/antonetta_becker,0.0986687737,17.27.86.250,"{""location"": ""GG"", ""is_mobile"": false}" 14592,5,328,2017-06-06 15:12:29,http://hodkiewicz.com/roie,0.5597413525,120.207.46.215,"{""location"": ""SY"", ""is_mobile"": true}" 14593,5,328,2017-05-05 15:06:16,http://weinatreichert.biz/jules.orn,0.7505744551,24.147.187.218,"{""location"": ""ZW"", ""is_mobile"": true}" 14594,5,328,2017-05-06 08:16:39,http://bogisich.co/aleen,0.5259781773,160.90.76.239,"{""location"": ""PW"", ""is_mobile"": true}" 14595,5,328,2017-06-05 20:18:38,http://medhurst.io/jayden,0.0105361749,103.80.196.225,"{""location"": ""EC"", ""is_mobile"": false}" 14596,5,328,2017-04-05 05:04:05,http://zulauf.info/weston.sauer,0.4113027299,96.124.104.116,"{""location"": ""KE"", ""is_mobile"": false}" 14597,5,328,2017-02-18 00:32:43,http://naderlittle.info/leonie,0.4342690384,162.108.89.24,"{""location"": ""AG"", ""is_mobile"": false}" 14598,5,328,2016-12-31 05:27:19,http://oconnell.biz/rashawn,0.7563748103,108.189.169.22,"{""location"": ""LA"", ""is_mobile"": true}" 14599,5,328,2016-12-29 06:36:27,http://bahringerprosacco.net/daphney.breitenberg,0.6584530050,71.104.136.102,"{""location"": ""ML"", ""is_mobile"": false}" 14600,5,328,2017-04-25 12:01:30,http://grady.com/conner.gutmann,0.3775506885,14.245.4.48,"{""location"": ""GA"", ""is_mobile"": false}" 17540,6,395,2017-01-11 12:40:46,http://sipesheidenreich.net/jermain,,233.59.47.144,"{""location"": ""RE"", ""is_mobile"": false}" 17541,6,396,2017-03-12 00:39:29,http://boehm.co/simone,0.8269868062,98.34.148.174,"{""location"": ""AG"", ""is_mobile"": false}" 17542,6,396,2017-06-02 11:46:14,http://hauckrowe.biz/dora,0.1287436370,39.72.93.43,"{""location"": ""AQ"", ""is_mobile"": false}" 17543,6,396,2017-02-05 13:50:21,http://koeppmraz.com/triston.witting,0.9989360957,221.143.2.8,"{""location"": ""IN"", ""is_mobile"": true}" 17544,6,396,2017-04-02 02:59:06,http://nienowfadel.net/jaylan,0.7274698604,103.109.207.52,"{""location"": ""LY"", ""is_mobile"": true}" 17545,6,396,2017-03-18 08:30:17,http://legrotark.com/gayle_kreiger,0.8832576629,21.177.201.237,"{""location"": ""AR"", ""is_mobile"": false}" 17546,6,396,2017-02-16 01:46:00,http://pagac.net/sherwood,0.6266033082,38.100.4.144,"{""location"": ""IQ"", ""is_mobile"": false}" 17547,6,396,2017-02-12 12:11:38,http://hegmann.net/erling_keeling,0.6054624820,39.193.230.183,"{""location"": ""GE"", ""is_mobile"": true}" 17548,6,396,2017-04-03 08:28:17,http://danielgleason.io/jimmie_adams,0.2960118179,4.221.112.116,"{""location"": ""DJ"", ""is_mobile"": true}" 17549,6,396,2017-04-27 10:17:08,http://oconner.name/friedrich,0.2195330101,40.211.177.154,"{""location"": ""BB"", ""is_mobile"": false}" 17550,6,396,2017-03-01 10:35:18,http://effertz.biz/alize,0.5476580971,244.37.66.215,"{""location"": ""BV"", ""is_mobile"": true}" 17551,6,396,2017-04-20 01:47:53,http://murphy.biz/myron,0.0998926812,231.196.211.121,"{""location"": ""EG"", ""is_mobile"": true}" 17552,6,396,2017-03-15 04:48:52,http://glover.net/jannie,0.8076755873,123.118.85.48,"{""location"": ""HK"", ""is_mobile"": false}" 17553,6,396,2017-04-02 08:26:32,http://nienow.info/audrey_lemke,0.0901234288,34.193.76.232,"{""location"": ""ZA"", ""is_mobile"": false}" 17554,6,396,2017-05-13 13:53:22,http://orn.org/raleigh,0.7275300223,218.17.201.124,"{""location"": ""DJ"", ""is_mobile"": true}" 17555,6,396,2016-12-18 15:31:16,http://stamm.org/marcelina,0.3826953955,242.137.42.245,"{""location"": ""NC"", ""is_mobile"": false}" 17556,6,396,2017-02-10 00:18:19,http://weinat.com/margarete_predovic,0.3508230748,9.209.224.199,"{""location"": ""IS"", ""is_mobile"": true}" 17557,6,396,2017-01-14 15:18:56,http://baumbachwaelchi.io/deborah.zieme,0.9444982064,44.15.202.195,"{""location"": ""KZ"", ""is_mobile"": false}" 17558,6,396,2017-04-27 06:02:24,http://nienow.org/meghan.prohaska,0.5736051446,122.25.214.236,"{""location"": ""PL"", ""is_mobile"": true}" 17559,6,396,2017-04-24 04:14:16,http://bauch.co/jovany,0.4397971580,190.111.165.135,"{""location"": ""SZ"", ""is_mobile"": false}" 17560,6,396,2017-01-25 01:19:31,http://paucekbergstrom.info/camron,0.2104795038,124.211.114.214,"{""location"": ""MD"", ""is_mobile"": false}" 17561,6,396,2017-03-27 08:27:12,http://emard.net/candido.bradtke,0.3269063860,218.125.44.116,"{""location"": ""KR"", ""is_mobile"": false}" 17562,6,396,2017-02-21 12:31:27,http://flatley.name/brooklyn_aufderhar,0.1889129187,120.248.170.112,"{""location"": ""VU"", ""is_mobile"": false}" 17563,6,396,2017-04-29 14:36:23,http://dickinsonmcclure.name/mayra.hettinger,0.1130581088,20.95.83.137,"{""location"": ""SD"", ""is_mobile"": false}" 17564,6,396,2017-01-17 07:03:45,http://romaguera.info/amos,0.1885866871,115.173.153.251,"{""location"": ""TZ"", ""is_mobile"": true}" 17565,6,396,2017-05-17 09:47:48,http://tillmangoyette.io/garett,0.2159858433,133.33.140.34,"{""location"": ""AO"", ""is_mobile"": true}" 17566,6,396,2017-06-07 14:30:19,http://haag.io/myrtis_gaylord,0.5163346656,86.253.127.97,"{""location"": ""GW"", ""is_mobile"": false}" 17567,6,396,2017-02-24 17:55:30,http://lednerjohnson.io/ada_treutel,0.4617176277,189.74.144.213,"{""location"": ""GR"", ""is_mobile"": false}" 17568,6,396,2017-05-13 22:23:55,http://lesch.org/violette_cruickshank,0.7161465196,214.241.238.168,"{""location"": ""IM"", ""is_mobile"": true}" 17569,6,396,2017-05-07 03:01:14,http://hagenesbecker.name/rafael,0.5768192243,234.142.118.248,"{""location"": ""CU"", ""is_mobile"": false}" 17570,6,396,2017-03-18 06:16:30,http://bednar.co/leo.mayer,0.5898919153,179.117.95.192,"{""location"": ""KP"", ""is_mobile"": true}" 17571,6,396,2017-02-19 18:08:35,http://andersongraham.com/bella.corkery,0.2893625288,227.61.142.252,"{""location"": ""IQ"", ""is_mobile"": true}" 17572,6,396,2017-05-10 10:08:33,http://kerlukeleuschke.info/kyler_morar,0.1993902074,57.186.74.80,"{""location"": ""FR"", ""is_mobile"": true}" 17573,6,396,2017-03-02 06:39:19,http://grady.com/noel_macejkovic,0.7700011373,97.250.97.45,"{""location"": ""PL"", ""is_mobile"": false}" 17574,6,396,2016-12-28 10:27:17,http://dietrichdickinson.co/jaeden,0.7302575102,103.126.58.216,"{""location"": ""BF"", ""is_mobile"": false}" 17575,6,396,2017-01-02 08:53:40,http://thiel.io/pansy,0.8108761572,229.97.54.190,"{""location"": ""CO"", ""is_mobile"": false}" 17576,6,396,2017-02-13 02:09:51,http://medhurstroob.biz/daryl,0.8360931837,10.214.253.104,"{""location"": ""ER"", ""is_mobile"": false}" 17577,6,396,2017-02-26 19:44:42,http://mccullough.biz/malcolm,0.4005772536,79.83.40.156,"{""location"": ""SJ"", ""is_mobile"": false}" 17578,6,396,2017-01-04 21:43:00,http://goyette.co/jaden_robel,0.0306393692,151.202.118.241,"{""location"": ""GY"", ""is_mobile"": false}" 17579,6,396,2017-06-14 00:29:14,http://sauermaggio.biz/anibal.cole,0.0219520194,223.253.123.98,"{""location"": ""RE"", ""is_mobile"": true}" 17580,6,396,2017-06-03 02:52:54,http://glover.io/miouri_rice,0.4586698112,169.68.71.57,"{""location"": ""AE"", ""is_mobile"": false}" 17581,6,396,2017-02-17 14:58:07,http://runolfsdottirwelch.info/holly.gibson,0.0422500020,204.166.123.67,"{""location"": ""RU"", ""is_mobile"": false}" 17582,6,396,2017-02-26 23:45:24,http://wilkinson.name/rogelio,0.3673508294,176.131.191.108,"{""location"": ""KZ"", ""is_mobile"": true}" 17583,6,396,2017-03-25 05:53:41,http://klocko.io/aileen.hoeger,0.4753547134,25.252.62.168,"{""location"": ""NE"", ""is_mobile"": false}" 17584,6,396,2017-05-12 11:34:05,http://zboncak.info/ryann,0.4612686158,30.167.97.100,"{""location"": ""CC"", ""is_mobile"": false}" 17585,6,396,2016-12-31 18:13:22,http://block.co/samson_keler,0.3283788142,20.230.66.252,"{""location"": ""TN"", ""is_mobile"": false}" 17586,6,396,2017-06-04 10:30:27,http://grady.org/natalie,0.6372258680,15.5.176.186,"{""location"": ""GT"", ""is_mobile"": true}" 17587,6,396,2016-12-20 01:23:42,http://koelpinwunsch.co/dillan,0.9163598667,197.154.4.249,"{""location"": ""NA"", ""is_mobile"": true}" 17588,6,396,2017-05-27 14:07:43,http://ferry.info/jazmyne,0.0018496331,27.79.38.73,"{""location"": ""MX"", ""is_mobile"": true}" 17589,6,396,2017-03-01 14:29:32,http://rennerdach.io/kiarra,0.9708681463,138.92.99.187,"{""location"": ""FI"", ""is_mobile"": true}" 17590,6,396,2017-01-22 16:02:39,http://gaylordfriesen.name/henriette,0.4278752143,151.37.200.35,"{""location"": ""SN"", ""is_mobile"": false}" 4725,2,104,2017-02-11 18:49:59,http://kuhic.co/bernita.kautzer,,18.226.164.119,"{""location"": ""ZM"", ""is_mobile"": true}" 4726,2,105,2017-02-13 09:45:13,http://mannwelch.net/lydia,,87.180.190.9,"{""location"": ""GU"", ""is_mobile"": true}" 4727,2,105,2017-06-04 09:25:40,http://wolff.name/ansley,,224.202.127.154,"{""location"": ""ZA"", ""is_mobile"": true}" 4728,2,105,2017-05-24 20:33:35,http://zboncak.biz/declan.auer,,246.206.86.102,"{""location"": ""SA"", ""is_mobile"": false}" 4729,2,105,2016-12-18 12:25:10,http://gleichner.biz/ari.rempel,,47.170.3.174,"{""location"": ""BD"", ""is_mobile"": true}" 4730,2,105,2017-06-07 10:41:44,http://bahringer.name/lonnie,,27.183.209.123,"{""location"": ""IE"", ""is_mobile"": true}" 4731,2,105,2017-04-15 08:42:59,http://dickiwuckert.com/andreane,,165.111.80.205,"{""location"": ""CR"", ""is_mobile"": false}" 4732,2,105,2017-01-09 01:11:57,http://hagenes.com/edwina.kuvalis,,16.55.132.222,"{""location"": ""LA"", ""is_mobile"": true}" 4733,2,105,2016-12-24 19:33:43,http://treutel.info/berneice.denesik,,201.172.113.31,"{""location"": ""IT"", ""is_mobile"": false}" 4734,2,105,2017-03-16 20:15:42,http://dibbert.co/eula,,178.65.108.16,"{""location"": ""CX"", ""is_mobile"": false}" 4735,2,105,2017-01-27 20:59:11,http://sanfordbeer.com/beth_will,,73.82.203.73,"{""location"": ""SY"", ""is_mobile"": true}" 4736,2,105,2016-12-31 23:52:30,http://lemkegerlach.biz/joanny,,38.22.118.117,"{""location"": ""IT"", ""is_mobile"": true}" 4737,2,105,2017-03-01 04:51:08,http://leuschke.biz/dewitt,,93.73.187.153,"{""location"": ""CY"", ""is_mobile"": true}" 4738,2,105,2017-01-05 19:46:34,http://reichertlubowitz.biz/daisy_bashirian,,179.17.30.157,"{""location"": ""BV"", ""is_mobile"": false}" 4739,2,105,2017-03-15 03:26:41,http://franecki.co/gardner.zboncak,,7.167.110.58,"{""location"": ""TW"", ""is_mobile"": true}" 4740,2,105,2017-03-22 05:36:29,http://hagenes.co/tiffany,,16.44.10.195,"{""location"": ""FM"", ""is_mobile"": true}" 4741,2,105,2017-03-25 09:54:44,http://kuhnreichert.net/monroe.barton,,22.212.166.136,"{""location"": ""LK"", ""is_mobile"": true}" 4742,2,105,2017-01-08 22:41:18,http://dietrichstark.co/terry,,30.171.61.107,"{""location"": ""GY"", ""is_mobile"": true}" 4743,2,105,2016-12-28 16:39:36,http://wisoky.com/jaylon.kuvalis,,170.50.24.250,"{""location"": ""NU"", ""is_mobile"": true}" 4744,2,105,2017-06-05 05:29:53,http://hartmann.co/andrew.kilback,,9.242.164.166,"{""location"": ""PS"", ""is_mobile"": true}" 4745,2,105,2017-05-16 03:06:24,http://goodwinfay.co/bonita_leffler,,240.104.131.73,"{""location"": ""LV"", ""is_mobile"": false}" 4746,2,105,2017-01-26 12:26:27,http://cummingsgutkowski.io/sadie.dibbert,,220.175.250.231,"{""location"": ""BA"", ""is_mobile"": true}" 4747,2,105,2017-02-20 08:00:03,http://ziemedeckow.net/teie.runolfsdottir,,63.100.11.92,"{""location"": ""IM"", ""is_mobile"": false}" 4748,2,105,2017-03-21 13:26:57,http://mullerwaelchi.net/georgette,,20.9.83.31,"{""location"": ""CU"", ""is_mobile"": false}" 4749,2,105,2017-02-13 03:56:21,http://ernsereffertz.biz/viva,,109.134.234.146,"{""location"": ""SM"", ""is_mobile"": false}" 4750,2,105,2017-02-16 21:00:30,http://moenmoriette.org/jamal.satterfield,,241.254.204.33,"{""location"": ""BW"", ""is_mobile"": false}" 4751,2,105,2017-06-03 21:00:07,http://greenholt.com/kaylee,,123.234.253.13,"{""location"": ""GF"", ""is_mobile"": true}" 4752,2,105,2017-03-27 21:27:22,http://lowe.co/jaunita.sauer,,67.94.88.217,"{""location"": ""PR"", ""is_mobile"": true}" 4753,2,105,2017-03-20 05:14:13,http://gerlach.io/vincenzo,,204.155.217.229,"{""location"": ""LB"", ""is_mobile"": false}" 4754,2,105,2017-02-28 04:45:46,http://johnson.name/timmy_zboncak,,62.215.247.198,"{""location"": ""CL"", ""is_mobile"": false}" 4755,2,105,2016-12-21 21:02:49,http://stamm.com/emmalee,,152.174.252.18,"{""location"": ""YE"", ""is_mobile"": false}" 4756,2,105,2017-03-03 17:23:04,http://runte.org/catharine.corwin,,168.17.138.161,"{""location"": ""AS"", ""is_mobile"": true}" 4757,2,105,2017-01-17 13:49:42,http://brakusheathcote.org/libby,,171.107.184.67,"{""location"": ""MX"", ""is_mobile"": true}" 4758,2,105,2017-04-28 03:19:46,http://langworthjacobi.net/vivian,,36.135.24.68,"{""location"": ""LS"", ""is_mobile"": true}" 4759,2,105,2017-02-08 19:06:44,http://schoen.name/marquise,,196.50.20.60,"{""location"": ""VU"", ""is_mobile"": true}" 4760,2,105,2017-02-24 08:48:33,http://cummerata.io/demetrius,,235.195.69.129,"{""location"": ""BQ"", ""is_mobile"": false}" 4761,2,105,2016-12-24 13:30:08,http://okon.info/jee_heathcote,,226.104.9.77,"{""location"": ""AF"", ""is_mobile"": true}" 4762,2,105,2017-04-06 13:36:09,http://moen.info/leann.keebler,,98.89.173.231,"{""location"": ""CX"", ""is_mobile"": false}" 4763,2,105,2016-12-25 04:51:10,http://keeling.info/rickey.denesik,,82.73.14.250,"{""location"": ""BH"", ""is_mobile"": false}" 4764,2,105,2017-05-19 16:47:59,http://brakus.io/vicente_bogan,,169.98.59.197,"{""location"": ""BQ"", ""is_mobile"": true}" 4765,2,105,2017-01-11 13:00:23,http://carrollokeefe.net/rowena.hammes,,71.11.53.138,"{""location"": ""KZ"", ""is_mobile"": false}" 4766,2,105,2017-05-12 19:24:05,http://denesik.net/jeanne,,117.221.86.171,"{""location"": ""MT"", ""is_mobile"": true}" 4767,2,105,2017-01-24 04:46:10,http://quigley.com/jerrod,,98.207.186.178,"{""location"": ""CA"", ""is_mobile"": true}" 4768,2,105,2017-04-21 19:00:35,http://dubuquekoch.biz/ottilie.bashirian,,243.6.237.215,"{""location"": ""GS"", ""is_mobile"": false}" 4769,2,105,2017-02-25 15:02:30,http://carroll.info/myrna_okon,,156.16.32.32,"{""location"": ""TG"", ""is_mobile"": false}" 4770,2,105,2017-03-07 01:17:33,http://ritchieheel.biz/jesus_bins,,78.237.132.95,"{""location"": ""VA"", ""is_mobile"": false}" 4771,2,105,2017-01-14 19:30:19,http://cruickshank.io/edison,,100.56.36.195,"{""location"": ""HK"", ""is_mobile"": true}" 4772,2,105,2017-04-27 05:38:51,http://stracke.name/christ_lindgren,,235.134.139.62,"{""location"": ""CU"", ""is_mobile"": false}" 4773,2,105,2017-05-21 20:52:50,http://quitzon.net/faustino,,74.203.79.40,"{""location"": ""LB"", ""is_mobile"": true}" 4774,2,105,2017-06-14 00:50:19,http://jakubowski.org/baron_romaguera,,48.52.163.85,"{""location"": ""ZW"", ""is_mobile"": false}" 4775,2,105,2017-01-24 01:08:47,http://schimmel.info/richmond.heaney,,115.252.203.87,"{""location"": ""TH"", ""is_mobile"": false}" 4776,2,105,2017-06-05 05:02:59,http://mrazbogisich.net/amiya_ankunding,,230.221.40.64,"{""location"": ""TH"", ""is_mobile"": true}" 4777,2,105,2017-03-03 00:54:33,http://doyle.co/hallie,,196.99.126.84,"{""location"": ""KH"", ""is_mobile"": true}" 4778,2,105,2017-06-02 22:51:29,http://jasthudson.co/kirstin,,179.23.5.162,"{""location"": ""SY"", ""is_mobile"": false}" 4779,2,105,2017-04-13 06:40:48,http://larkin.io/dayana.altenwerth,,241.94.140.245,"{""location"": ""CL"", ""is_mobile"": false}" 11648,4,260,2017-03-14 16:17:25,http://champlinblock.info/eulalia_rolfson,0.3272726382,40.38.189.208,"{""location"": ""CK"", ""is_mobile"": true}" 11649,4,260,2017-02-14 06:01:44,http://nienowjohnson.io/taryn.weimann,0.3817416203,216.93.33.134,"{""location"": ""CY"", ""is_mobile"": true}" 11650,4,260,2017-05-30 13:54:15,http://eichmann.name/davin.nolan,0.4123957080,73.243.131.254,"{""location"": ""GG"", ""is_mobile"": false}" 11651,4,260,2017-05-04 05:36:39,http://trantow.org/ivah.steuber,0.9488949032,240.188.88.249,"{""location"": ""ML"", ""is_mobile"": true}" 11652,4,260,2017-04-16 17:32:15,http://nolan.name/sibyl,0.5216739081,172.215.15.107,"{""location"": ""AZ"", ""is_mobile"": false}" 11653,4,260,2017-02-15 06:08:02,http://marquardt.com/mina,0.4641678635,137.190.251.195,"{""location"": ""CF"", ""is_mobile"": true}" 11654,4,260,2017-02-28 14:23:33,http://stokes.biz/trever_rempel,0.4971366393,156.215.229.211,"{""location"": ""AE"", ""is_mobile"": false}" 11655,4,260,2017-04-13 15:44:08,http://hamill.name/eduardo,0.4359254520,110.179.30.64,"{""location"": ""KH"", ""is_mobile"": false}" 11656,4,260,2017-01-02 18:55:04,http://dach.org/lelia.waters,0.9435299803,45.198.96.175,"{""location"": ""CK"", ""is_mobile"": false}" 11657,4,260,2017-05-30 19:52:14,http://krajcikdurgan.co/percival.hane,0.7594818592,234.227.180.203,"{""location"": ""BZ"", ""is_mobile"": true}" 11658,4,260,2017-04-03 16:46:04,http://donnellymertz.org/willis.wintheiser,0.5693078824,12.147.228.167,"{""location"": ""UY"", ""is_mobile"": false}" 11659,4,260,2017-05-09 11:12:38,http://bashirian.biz/raven_kulas,0.8178909588,56.85.227.80,"{""location"": ""PH"", ""is_mobile"": true}" 11660,4,260,2017-02-28 22:23:11,http://zboncak.biz/jamal,0.6524296656,183.230.171.25,"{""location"": ""DO"", ""is_mobile"": true}" 11661,4,260,2017-02-07 18:05:54,http://haley.io/mack_paucek,0.2935040997,233.49.232.58,"{""location"": ""ZW"", ""is_mobile"": false}" 11662,4,260,2016-12-23 10:36:24,http://bechtelar.org/edgardo,0.6496016616,215.211.175.137,"{""location"": ""CG"", ""is_mobile"": false}" 11663,4,260,2017-06-10 09:39:43,http://marvinbosco.co/mozell.will,0.7255222658,229.19.78.34,"{""location"": ""US"", ""is_mobile"": true}" 11664,4,260,2017-04-21 09:24:03,http://williamsonbashirian.name/okey,0.3221773580,217.138.83.36,"{""location"": ""BD"", ""is_mobile"": true}" 11665,4,260,2017-04-19 04:20:25,http://reilly.net/jackson.von,0.9454124798,248.184.69.34,"{""location"": ""VA"", ""is_mobile"": true}" 11666,4,260,2017-05-30 10:10:55,http://tromprunolfon.name/dereck.beatty,0.4761137122,97.243.128.135,"{""location"": ""TV"", ""is_mobile"": true}" 11667,4,260,2017-02-12 21:55:13,http://nader.net/marques.bradtke,0.2157249709,66.90.203.59,"{""location"": ""SB"", ""is_mobile"": true}" 11668,4,260,2017-01-23 11:47:07,http://turcottefritsch.info/gracie.casper,0.4265939720,253.4.106.114,"{""location"": ""LA"", ""is_mobile"": true}" 11669,4,260,2017-01-31 03:01:16,http://schmeler.biz/carrie_mayert,0.0068299564,116.90.127.62,"{""location"": ""KH"", ""is_mobile"": false}" 11670,4,260,2017-04-09 00:18:14,http://ward.org/joany,0.6263898908,15.81.61.163,"{""location"": ""MK"", ""is_mobile"": false}" 11671,4,260,2017-04-28 21:20:26,http://glover.org/axel_senger,0.3976075114,238.165.226.135,"{""location"": ""DJ"", ""is_mobile"": false}" 11672,4,260,2017-02-15 02:13:49,http://moenkshlerin.io/kathleen,0.8947703055,138.41.89.29,"{""location"": ""CV"", ""is_mobile"": false}" 11673,4,260,2017-02-23 16:55:40,http://morar.biz/janie_oberbrunner,0.4361175739,18.121.163.235,"{""location"": ""EC"", ""is_mobile"": false}" 11674,4,260,2017-02-12 13:49:27,http://kubschuster.com/julianne,0.8028296976,173.208.153.90,"{""location"": ""MT"", ""is_mobile"": true}" 11675,4,260,2016-12-26 01:42:59,http://wyman.com/cameron,0.6421118716,41.97.158.172,"{""location"": ""VC"", ""is_mobile"": false}" 11676,4,260,2016-12-14 06:45:39,http://hyattlang.com/robbie.tromp,0.6877885955,59.64.135.77,"{""location"": ""ZM"", ""is_mobile"": false}" 11677,4,260,2017-05-26 13:41:01,http://parker.io/otho_ortiz,0.7328938826,175.177.248.62,"{""location"": ""IN"", ""is_mobile"": true}" 11678,4,260,2016-12-15 03:31:53,http://terrylittel.org/denis_kovacek,0.1163061350,87.47.27.77,"{""location"": ""TL"", ""is_mobile"": true}" 11679,4,260,2017-01-07 15:11:59,http://little.info/adrian,0.9907061139,34.108.152.13,"{""location"": ""EG"", ""is_mobile"": true}" 11680,4,260,2017-03-25 20:12:57,http://kemmer.org/ed.lubowitz,0.9732950687,179.88.20.25,"{""location"": ""SY"", ""is_mobile"": false}" 11681,4,260,2017-04-01 15:13:27,http://flatley.biz/marcelino_hermiston,0.3283120356,243.184.44.103,"{""location"": ""KI"", ""is_mobile"": true}" 11682,4,260,2017-04-10 09:22:32,http://willmsprohaska.com/emmalee.baumbach,0.8493050746,35.87.245.143,"{""location"": ""CF"", ""is_mobile"": false}" 11683,4,260,2017-03-10 20:11:32,http://windler.io/delaney,0.1472668234,239.231.85.213,"{""location"": ""DO"", ""is_mobile"": true}" 11684,4,260,2017-03-04 17:53:50,http://hickleprice.org/raul,0.9237036914,206.21.162.52,"{""location"": ""BW"", ""is_mobile"": true}" 11685,4,260,2016-12-23 04:12:41,http://yundt.info/justice.quitzon,0.0531579867,146.62.75.48,"{""location"": ""DM"", ""is_mobile"": true}" 11686,4,260,2017-03-13 16:17:57,http://boyer.com/anna.hamill,0.0168632713,222.159.71.49,"{""location"": ""IO"", ""is_mobile"": true}" 11687,4,260,2017-03-26 01:59:08,http://baumbach.co/clint.veum,0.9262249759,64.132.242.69,"{""location"": ""PW"", ""is_mobile"": true}" 11688,4,260,2017-04-01 12:09:07,http://bogan.co/alfonso.king,0.9421169881,64.50.241.216,"{""location"": ""BN"", ""is_mobile"": true}" 11689,4,260,2017-03-15 05:38:23,http://frami.biz/emery_grant,0.5555718748,119.245.138.195,"{""location"": ""MF"", ""is_mobile"": true}" 11690,4,260,2016-12-26 06:07:06,http://sawayn.biz/krystina_schmidt,0.1754370518,137.19.132.206,"{""location"": ""LS"", ""is_mobile"": true}" 11691,4,261,2017-04-13 06:40:07,http://beerspinka.biz/allene_kling,0.1478201037,181.108.88.204,"{""location"": ""AF"", ""is_mobile"": true}" 11692,4,261,2016-12-24 17:57:18,http://considine.com/giovanna,0.1218645865,185.97.32.129,"{""location"": ""IO"", ""is_mobile"": false}" 11693,4,261,2017-01-20 23:51:52,http://glover.io/yvonne_bayer,0.4055639652,127.51.180.25,"{""location"": ""LU"", ""is_mobile"": false}" 11694,4,261,2017-01-04 16:40:59,http://pacocha.io/adelia,0.6598896915,162.83.151.157,"{""location"": ""PH"", ""is_mobile"": true}" 11695,4,261,2017-04-17 04:45:30,http://kozeyroob.io/oceane,0.9225986431,166.234.238.193,"{""location"": ""MN"", ""is_mobile"": true}" 11696,4,261,2017-04-22 06:49:42,http://mcclure.com/walter,0.4599166333,95.52.140.244,"{""location"": ""HT"", ""is_mobile"": false}" 11697,4,261,2017-03-24 17:38:47,http://jacobson.org/eleanora_connelly,0.2231418106,97.243.27.252,"{""location"": ""NU"", ""is_mobile"": false}" 11698,4,261,2017-01-06 09:25:25,http://kaulke.co/titus,0.2656916940,26.74.53.135,"{""location"": ""UY"", ""is_mobile"": true}" 8663,3,194,2017-03-25 13:36:11,http://stantondonnelly.co/mertie,,51.105.122.249,"{""location"": ""IQ"", ""is_mobile"": true}" 8664,3,194,2017-01-21 08:43:57,http://hamilltremblay.net/golda,,88.26.98.128,"{""location"": ""OM"", ""is_mobile"": false}" 8665,3,194,2017-04-19 17:01:27,http://treutel.net/verona.barton,,153.138.206.177,"{""location"": ""SZ"", ""is_mobile"": false}" 8666,3,194,2016-12-14 19:17:44,http://goldner.org/dariana_klein,,191.104.119.67,"{""location"": ""PN"", ""is_mobile"": false}" 8667,3,194,2016-12-24 16:04:54,http://wisokyabbott.io/noble_dickinson,,135.48.103.147,"{""location"": ""HR"", ""is_mobile"": true}" 8668,3,194,2017-05-14 05:03:41,http://strosin.name/rickey_walter,,27.232.100.161,"{""location"": ""PS"", ""is_mobile"": true}" 8669,3,194,2017-03-09 04:44:32,http://kovacek.org/olin,,216.99.176.253,"{""location"": ""BQ"", ""is_mobile"": true}" 8670,3,194,2017-04-20 02:04:53,http://hermann.info/terrill,,31.241.84.113,"{""location"": ""CX"", ""is_mobile"": false}" 8671,3,194,2017-03-16 05:18:13,http://wymanwiegand.info/maximus,,98.204.181.94,"{""location"": ""VU"", ""is_mobile"": false}" 8672,3,194,2017-06-05 17:34:08,http://rice.biz/chloe.lemke,,197.115.235.120,"{""location"": ""TV"", ""is_mobile"": false}" 8673,3,194,2016-12-20 19:22:16,http://padberg.org/laria,,49.107.73.14,"{""location"": ""CH"", ""is_mobile"": true}" 8674,3,194,2016-12-30 19:56:37,http://dickenskiehn.biz/anabelle,,79.84.189.149,"{""location"": ""TM"", ""is_mobile"": false}" 8675,3,194,2017-04-24 07:41:02,http://gutkowski.co/dallas_nikolaus,,87.226.133.161,"{""location"": ""CV"", ""is_mobile"": true}" 8676,3,194,2017-03-21 12:07:33,http://bednar.org/anderson.grant,,173.219.136.211,"{""location"": ""MK"", ""is_mobile"": true}" 8677,3,194,2016-12-15 23:54:34,http://renner.com/rashawn_mueller,,4.51.96.198,"{""location"": ""ER"", ""is_mobile"": true}" 8678,3,194,2017-04-03 09:42:28,http://jacobson.name/eduardo.shields,,215.214.37.159,"{""location"": ""CR"", ""is_mobile"": false}" 8679,3,194,2017-03-21 20:21:05,http://dickinsonfadel.info/daron,,127.195.208.44,"{""location"": ""AS"", ""is_mobile"": false}" 8680,3,194,2017-06-01 14:29:46,http://johns.co/cristobal,,124.108.77.57,"{""location"": ""LI"", ""is_mobile"": true}" 8681,3,194,2017-01-31 12:14:50,http://hoppelittel.biz/deja,,150.7.14.118,"{""location"": ""MF"", ""is_mobile"": true}" 8682,3,194,2017-03-26 23:06:37,http://gusikowskipurdy.biz/melba,,169.143.21.8,"{""location"": ""MA"", ""is_mobile"": true}" 14601,5,328,2016-12-14 13:57:17,http://bosco.com/aurore,0.4938302456,27.197.2.81,"{""location"": ""YT"", ""is_mobile"": true}" 14602,5,328,2017-04-24 11:58:56,http://shanahan.info/tod.keeling,0.2304215410,215.103.163.53,"{""location"": ""GT"", ""is_mobile"": true}" 14603,5,328,2017-03-09 09:37:32,http://stroman.net/dayna.okuneva,0.0467347543,254.217.112.3,"{""location"": ""ID"", ""is_mobile"": true}" 14604,5,329,2017-02-06 05:54:34,http://osinski.co/elmer,0.2979140030,148.230.111.224,"{""location"": ""PW"", ""is_mobile"": true}" 14605,5,329,2016-12-20 11:27:36,http://mraz.co/marge,0.3054259430,95.75.115.77,"{""location"": ""ID"", ""is_mobile"": false}" 14606,5,329,2017-03-02 11:10:03,http://lehner.biz/darius_donnelly,0.3647384217,189.98.231.233,"{""location"": ""ZA"", ""is_mobile"": true}" 14607,5,329,2017-03-26 14:17:59,http://erdman.biz/roosevelt.gaylord,0.1889105790,201.233.156.66,"{""location"": ""MF"", ""is_mobile"": true}" 14608,5,329,2016-12-21 09:01:03,http://mitchellhuel.net/deion,0.4946846992,67.92.143.3,"{""location"": ""PN"", ""is_mobile"": true}" 14609,5,329,2017-06-07 07:30:21,http://fay.info/aurelie_block,0.4799849593,89.219.184.217,"{""location"": ""AF"", ""is_mobile"": false}" 14610,5,329,2017-05-31 06:53:50,http://kuvalisolson.name/bud_bailey,0.8538804683,30.214.99.5,"{""location"": ""FO"", ""is_mobile"": false}" 14611,5,329,2017-01-12 23:08:50,http://dach.biz/kaylin.bogisich,0.8437003064,219.61.150.183,"{""location"": ""KI"", ""is_mobile"": true}" 14612,5,329,2017-06-12 15:48:47,http://rippin.co/hillary,0.4205839004,44.168.247.165,"{""location"": ""FI"", ""is_mobile"": true}" 14613,5,329,2017-01-06 23:39:08,http://roberts.net/martine,0.0462446851,148.169.154.186,"{""location"": ""SO"", ""is_mobile"": false}" 14614,5,329,2017-03-11 22:32:54,http://green.org/shanelle.emmerich,0.5892924791,150.58.168.74,"{""location"": ""IL"", ""is_mobile"": false}" 14615,5,329,2017-05-30 08:28:04,http://feest.biz/else.schamberger,0.9458818277,184.233.71.191,"{""location"": ""GH"", ""is_mobile"": false}" 14616,5,329,2017-02-06 11:51:20,http://hermiston.io/rosalyn.strosin,0.1289847067,160.3.119.168,"{""location"": ""AE"", ""is_mobile"": true}" 14617,5,329,2017-01-22 01:45:07,http://mclaughlin.com/rasheed,0.7935942403,237.225.97.199,"{""location"": ""RW"", ""is_mobile"": false}" 14618,5,329,2017-05-13 20:37:11,http://spinkacollier.net/marc,0.9908252653,67.144.131.133,"{""location"": ""WS"", ""is_mobile"": true}" 14619,5,329,2017-03-20 21:03:09,http://fisher.com/cristina_schumm,0.2818792953,65.204.119.105,"{""location"": ""BV"", ""is_mobile"": true}" 14620,5,329,2017-04-05 00:30:30,http://ernser.net/maida.schaefer,0.8133036569,121.33.164.155,"{""location"": ""TT"", ""is_mobile"": false}" 14621,5,329,2017-05-28 16:40:14,http://brownhoppe.com/joanne_smith,0.0530127776,210.214.206.202,"{""location"": ""MR"", ""is_mobile"": false}" 14622,5,329,2016-12-16 00:47:25,http://ziemann.io/adrienne.jacobson,0.6533413975,59.70.12.144,"{""location"": ""BA"", ""is_mobile"": true}" 14623,5,329,2017-05-25 03:41:36,http://mullerlittel.name/helmer,0.2907618692,21.253.111.169,"{""location"": ""KE"", ""is_mobile"": true}" 14624,5,329,2017-06-05 17:17:49,http://doyle.biz/tania_roob,0.6982486765,191.22.74.30,"{""location"": ""AX"", ""is_mobile"": false}" 14625,5,329,2017-04-06 16:36:26,http://eichmann.name/ramona,0.2042484255,36.226.23.80,"{""location"": ""NU"", ""is_mobile"": true}" 14626,5,329,2017-04-18 03:42:56,http://gislasongerhold.co/francisca,0.6238910704,121.199.233.125,"{""location"": ""DK"", ""is_mobile"": false}" 14627,5,329,2017-03-02 10:52:02,http://ohara.name/meaghan,0.1325450670,167.115.248.20,"{""location"": ""SE"", ""is_mobile"": true}" 14628,5,329,2017-04-17 03:00:38,http://rosenbaum.info/gennaro,0.3921923502,66.194.77.121,"{""location"": ""HT"", ""is_mobile"": true}" 14629,5,329,2017-01-01 20:54:00,http://klingkeler.name/ivory,0.5957394266,230.57.19.208,"{""location"": ""VC"", ""is_mobile"": false}" 14630,5,329,2016-12-26 15:27:18,http://yundt.co/gregoria_kuvalis,0.8670897939,202.189.70.149,"{""location"": ""AI"", ""is_mobile"": false}" 14631,5,329,2017-04-11 03:02:25,http://rath.net/domenick,0.5890477930,87.29.236.37,"{""location"": ""PN"", ""is_mobile"": false}" 14632,5,329,2017-01-03 01:02:32,http://schmeler.biz/sarina,0.0049859855,85.61.137.147,"{""location"": ""GA"", ""is_mobile"": false}" 14633,5,329,2017-02-06 08:07:26,http://herzoggleason.io/pearline,0.1050722482,175.83.182.5,"{""location"": ""IS"", ""is_mobile"": true}" 14634,5,329,2017-03-27 00:26:35,http://davisolson.info/milo,0.6648797344,172.216.249.131,"{""location"": ""FI"", ""is_mobile"": true}" 14635,5,329,2017-02-25 17:27:56,http://bartoletti.biz/elisha,0.9289754118,159.117.78.66,"{""location"": ""BA"", ""is_mobile"": true}" 14636,5,329,2017-05-27 04:06:29,http://smitham.org/alison,0.8280837570,119.239.192.113,"{""location"": ""TJ"", ""is_mobile"": true}" 14637,5,329,2017-02-25 21:28:01,http://johns.info/gerard.rempel,0.0160202248,199.118.30.123,"{""location"": ""SA"", ""is_mobile"": false}" 14638,5,329,2017-06-13 14:52:16,http://wolff.name/bradford.abshire,0.9736689961,89.91.186.114,"{""location"": ""SN"", ""is_mobile"": true}" 14639,5,329,2017-02-20 03:58:45,http://howedibbert.net/holden_pfannerstill,0.6514958504,117.22.187.66,"{""location"": ""EC"", ""is_mobile"": true}" 14640,5,329,2017-03-16 21:35:19,http://parisian.info/ubaldo.morar,0.5754576702,211.125.54.134,"{""location"": ""OM"", ""is_mobile"": true}" 14641,5,329,2017-04-03 11:06:03,http://heathcoteward.org/shayna_morar,0.6333359885,147.43.4.196,"{""location"": ""CY"", ""is_mobile"": false}" 14642,5,329,2016-12-29 13:21:17,http://wyman.io/tyrel_hand,0.6766201484,238.33.213.117,"{""location"": ""MP"", ""is_mobile"": false}" 14643,5,329,2017-03-18 07:38:00,http://roweluettgen.com/bettie.lowe,0.5771449745,223.235.52.253,"{""location"": ""LK"", ""is_mobile"": true}" 14644,5,329,2017-04-11 13:46:42,http://boyledeckow.biz/dedrick_botsford,0.6200701712,242.25.252.244,"{""location"": ""BM"", ""is_mobile"": false}" 14645,5,329,2017-04-06 16:45:13,http://whitethompson.org/ima,0.1139048706,113.120.216.11,"{""location"": ""PK"", ""is_mobile"": true}" 14646,5,330,2017-05-28 06:34:21,http://vonruedencronin.biz/edgardo,0.0272455763,91.6.196.94,"{""location"": ""MG"", ""is_mobile"": false}" 14647,5,330,2017-02-18 03:19:20,http://koch.com/josh.lesch,0.7627253588,148.133.28.180,"{""location"": ""CD"", ""is_mobile"": true}" 14648,5,330,2017-02-15 03:32:32,http://harvey.name/aurore,0.3293171884,107.55.38.234,"{""location"": ""KZ"", ""is_mobile"": false}" 14649,5,330,2016-12-14 02:56:35,http://flatley.info/savanna.blick,0.3873135315,252.59.157.84,"{""location"": ""FM"", ""is_mobile"": true}" 14650,5,330,2016-12-14 14:51:04,http://boscodouglas.org/joy_hills,0.3507887284,18.149.156.121,"{""location"": ""TK"", ""is_mobile"": false}" 14651,5,330,2017-03-30 06:25:26,http://satterfieldheel.net/darrell_nader,0.4826571415,214.192.242.91,"{""location"": ""KN"", ""is_mobile"": false}" 17591,6,396,2017-01-31 19:40:07,http://schustergreenfelder.info/louvenia.carter,0.1721131881,224.218.148.107,"{""location"": ""PK"", ""is_mobile"": false}" 17592,6,396,2017-04-29 10:17:19,http://kertzmann.name/idell,0.8515397491,54.75.151.11,"{""location"": ""GA"", ""is_mobile"": true}" 17593,6,396,2017-02-02 19:22:29,http://senger.biz/yasmin,0.2986219903,230.100.237.89,"{""location"": ""MQ"", ""is_mobile"": true}" 17594,6,396,2017-03-28 22:50:07,http://larson.org/delphia,0.3436174776,227.6.92.88,"{""location"": ""IN"", ""is_mobile"": true}" 17595,6,396,2017-01-07 16:35:49,http://jonesmurphy.co/reed_dietrich,0.8450530722,96.115.195.74,"{""location"": ""BQ"", ""is_mobile"": true}" 17596,6,396,2017-05-26 05:27:47,http://cummingsmoen.co/dallin,0.5656749579,40.251.200.158,"{""location"": ""CY"", ""is_mobile"": false}" 17597,6,396,2017-02-03 11:02:04,http://gusikowski.co/kameron.mante,0.6265438864,6.76.118.13,"{""location"": ""TJ"", ""is_mobile"": true}" 17598,6,396,2017-05-10 12:04:20,http://fadel.co/malachi.gerhold,0.6307204184,45.127.97.218,"{""location"": ""PW"", ""is_mobile"": true}" 17599,6,396,2017-04-16 20:42:03,http://armstrong.net/dorothy,0.7735923543,73.164.183.247,"{""location"": ""TF"", ""is_mobile"": true}" 17600,6,396,2017-05-04 18:14:26,http://borer.info/salvatore_morar,0.8089752299,74.178.75.171,"{""location"": ""LU"", ""is_mobile"": true}" 17601,6,396,2016-12-30 21:57:42,http://heel.org/neha_flatley,0.1390659300,147.114.234.122,"{""location"": ""CC"", ""is_mobile"": true}" 17602,6,396,2017-05-20 16:13:08,http://jakubowski.net/bobbie,0.8225662682,59.133.252.234,"{""location"": ""MU"", ""is_mobile"": false}" 17603,6,396,2017-03-22 02:29:41,http://cummingsbechtelar.io/jarrett_leuschke,0.4957458554,50.126.126.143,"{""location"": ""MF"", ""is_mobile"": false}" 17604,6,396,2017-01-02 07:35:03,http://pagac.co/vergie.grant,0.4954213593,224.36.29.212,"{""location"": ""HK"", ""is_mobile"": true}" 17605,6,396,2017-04-05 22:24:58,http://hoegerhansen.org/shaun_kuhic,0.4340931131,187.124.39.227,"{""location"": ""DK"", ""is_mobile"": false}" 17606,6,396,2017-02-06 07:58:06,http://rohan.info/grady,0.8681269933,126.69.8.81,"{""location"": ""UG"", ""is_mobile"": true}" 17607,6,396,2017-03-18 19:38:33,http://runolfsdottirpredovic.info/jermain_lindgren,0.9319994204,163.10.10.133,"{""location"": ""VI"", ""is_mobile"": false}" 17608,6,396,2017-04-10 00:54:49,http://wuckertbednar.io/norris,0.0945757170,185.18.187.128,"{""location"": ""GE"", ""is_mobile"": true}" 17609,6,396,2017-02-03 11:37:41,http://rau.name/mina,0.8454116393,233.34.219.73,"{""location"": ""DM"", ""is_mobile"": false}" 17610,6,396,2017-02-04 20:54:52,http://cruickshank.co/ashly,0.1057485600,253.152.156.67,"{""location"": ""BZ"", ""is_mobile"": true}" 17611,6,397,2017-04-13 10:38:13,http://marquardtruel.co/wilhelm,0.5247291643,241.21.218.190,"{""location"": ""GF"", ""is_mobile"": false}" 17612,6,397,2017-01-06 14:29:41,http://schroeder.co/muriel,0.0206358151,101.204.165.73,"{""location"": ""BN"", ""is_mobile"": false}" 17613,6,397,2017-05-26 03:55:11,http://treutel.org/madonna_murphy,0.9078711030,214.114.100.225,"{""location"": ""DZ"", ""is_mobile"": false}" 17614,6,397,2016-12-22 16:10:25,http://connwolff.net/lemuel.hauck,0.7509836236,39.36.78.122,"{""location"": ""SY"", ""is_mobile"": true}" 17615,6,397,2017-01-10 06:29:05,http://emardmoen.org/ewell,0.1948054540,235.202.31.69,"{""location"": ""CW"", ""is_mobile"": false}" 17616,6,397,2017-02-15 13:05:14,http://durganstehr.co/kirsten,0.2189394491,133.210.231.89,"{""location"": ""CX"", ""is_mobile"": true}" 17617,6,397,2017-02-01 16:07:12,http://cronahilpert.name/dejuan,0.6117390624,15.53.254.222,"{""location"": ""DM"", ""is_mobile"": false}" 17618,6,397,2017-03-29 10:28:43,http://hammeswest.info/forrest_vonrueden,0.0485405745,14.45.191.67,"{""location"": ""FO"", ""is_mobile"": true}" 17619,6,397,2017-03-14 23:58:45,http://predovic.net/christa,0.6890921445,172.106.71.189,"{""location"": ""SL"", ""is_mobile"": false}" 17620,6,397,2017-01-10 10:51:26,http://rempel.net/abigayle.bergstrom,0.6591556512,13.147.210.102,"{""location"": ""CR"", ""is_mobile"": true}" 17621,6,397,2017-05-06 23:15:21,http://abernathy.info/dayana_olson,0.8903187567,134.79.62.149,"{""location"": ""TT"", ""is_mobile"": true}" 17622,6,397,2017-05-17 07:39:26,http://strackeondricka.co/henri.stamm,0.1542310204,89.228.49.219,"{""location"": ""DO"", ""is_mobile"": true}" 17623,6,397,2017-05-16 12:27:04,http://millsfeil.com/durward.armstrong,0.8979026282,16.247.189.175,"{""location"": ""UY"", ""is_mobile"": true}" 17624,6,397,2017-05-24 06:43:23,http://beahan.biz/albina_bailey,0.0947845829,122.147.213.119,"{""location"": ""DO"", ""is_mobile"": true}" 17625,6,397,2017-04-02 17:04:19,http://altenwerth.com/andreane.schmitt,0.3091304031,177.160.97.16,"{""location"": ""MT"", ""is_mobile"": true}" 17626,6,397,2016-12-27 01:51:55,http://hegmann.io/reymundo.streich,0.1736976176,108.57.65.225,"{""location"": ""EC"", ""is_mobile"": true}" 17627,6,397,2017-01-07 17:29:47,http://wisozkstoltenberg.name/destiney.abshire,0.3058227262,223.106.151.90,"{""location"": ""IQ"", ""is_mobile"": false}" 17628,6,397,2017-03-31 04:28:34,http://gibson.net/jalyn,0.1318832456,151.103.58.139,"{""location"": ""MR"", ""is_mobile"": false}" 17629,6,397,2017-04-26 10:31:18,http://jones.net/joel.goldner,0.6448163686,229.210.238.225,"{""location"": ""LU"", ""is_mobile"": true}" 17630,6,397,2017-04-18 22:13:06,http://runolfsdottirwillms.co/shakira_jacobson,0.6826193408,160.221.172.10,"{""location"": ""BV"", ""is_mobile"": true}" 17631,6,397,2017-03-16 17:19:03,http://zboncakherman.info/samson,0.2337726849,51.140.232.31,"{""location"": ""TW"", ""is_mobile"": false}" 17632,6,397,2017-03-27 09:05:42,http://pfannerstill.net/caria,0.4235359338,78.181.190.92,"{""location"": ""SK"", ""is_mobile"": true}" 17633,6,397,2017-02-21 17:23:42,http://eichmann.name/zena,0.2557629251,229.171.117.113,"{""location"": ""MZ"", ""is_mobile"": true}" 17634,6,397,2017-04-02 20:05:23,http://binshayes.org/mckayla,0.6239015595,40.103.56.64,"{""location"": ""MX"", ""is_mobile"": true}" 17635,6,397,2017-01-07 10:01:44,http://parkerdenesik.info/dylan,0.9769936111,201.129.211.148,"{""location"": ""BI"", ""is_mobile"": false}" 17636,6,397,2017-05-11 10:03:14,http://hermann.net/marietta_mayer,0.3726025496,167.111.215.42,"{""location"": ""EE"", ""is_mobile"": true}" 17637,6,397,2017-04-24 19:02:53,http://simonis.com/dewayne_pouros,0.6610616968,66.120.153.194,"{""location"": ""RS"", ""is_mobile"": true}" 17638,6,397,2017-03-02 01:55:56,http://okuneva.com/kacey_ruel,0.6469739603,35.226.153.233,"{""location"": ""CD"", ""is_mobile"": true}" 17639,6,397,2017-01-01 05:22:17,http://homenicktromp.info/travis.leffler,0.4099863729,157.87.250.177,"{""location"": ""CH"", ""is_mobile"": false}" 17640,6,397,2017-05-29 23:02:17,http://collins.com/linnie,0.3334607812,184.128.188.217,"{""location"": ""GP"", ""is_mobile"": true}" 17641,6,397,2017-02-21 23:12:09,http://pfeffer.info/maya_hyatt,0.2482504869,201.239.214.173,"{""location"": ""VC"", ""is_mobile"": false}" 4780,2,105,2017-06-02 22:33:06,http://kiehn.net/jordan,,9.105.174.39,"{""location"": ""YE"", ""is_mobile"": false}" 4781,2,106,2017-01-02 20:39:11,http://fritsch.io/sandrine,,161.204.114.86,"{""location"": ""PM"", ""is_mobile"": true}" 4782,2,106,2017-01-09 13:49:34,http://ullrich.co/aron,,238.70.211.249,"{""location"": ""QA"", ""is_mobile"": true}" 4783,2,106,2017-05-11 21:11:26,http://schinner.com/keshaun,,117.10.219.34,"{""location"": ""FR"", ""is_mobile"": false}" 4784,2,106,2017-06-08 06:57:48,http://heidenreich.com/daisy_ziemann,,194.132.37.14,"{""location"": ""RE"", ""is_mobile"": false}" 4785,2,106,2017-03-01 11:15:01,http://hahnglover.net/adrian,,31.172.238.216,"{""location"": ""CK"", ""is_mobile"": false}" 4786,2,106,2017-03-12 01:57:45,http://glover.name/dorthy,,71.40.176.128,"{""location"": ""AG"", ""is_mobile"": false}" 4787,2,106,2017-02-05 14:23:21,http://wolff.name/matilda.nicolas,,35.5.189.43,"{""location"": ""MZ"", ""is_mobile"": true}" 4788,2,106,2017-02-19 03:37:38,http://shanahansauer.co/alvena.wolff,,78.200.13.86,"{""location"": ""SZ"", ""is_mobile"": false}" 4789,2,106,2017-04-28 15:24:43,http://kingkutch.info/else.kutch,,195.160.219.166,"{""location"": ""TR"", ""is_mobile"": false}" 4790,2,106,2017-01-22 18:51:13,http://moriette.com/ahmad,,111.122.138.42,"{""location"": ""RE"", ""is_mobile"": true}" 4791,2,106,2017-05-06 11:21:45,http://simoniskuhn.name/emmanuelle_deckow,,71.177.113.45,"{""location"": ""ZA"", ""is_mobile"": true}" 4792,2,106,2017-01-06 21:18:11,http://ebert.io/dixie.oconner,,65.226.242.124,"{""location"": ""MH"", ""is_mobile"": false}" 4793,2,106,2017-02-07 16:07:09,http://rice.biz/cayla_schiller,,200.239.22.203,"{""location"": ""CX"", ""is_mobile"": false}" 4794,2,106,2017-04-15 19:48:12,http://reilly.info/melya_douglas,,64.7.93.222,"{""location"": ""BF"", ""is_mobile"": true}" 4795,2,106,2017-03-14 17:05:01,http://hammes.biz/kim,,96.173.38.158,"{""location"": ""KW"", ""is_mobile"": false}" 4796,2,106,2016-12-28 22:11:04,http://sauerbauch.co/jacinto.boyle,,2.73.26.182,"{""location"": ""VG"", ""is_mobile"": false}" 4797,2,106,2017-03-22 03:22:04,http://hand.com/alek,,92.223.135.2,"{""location"": ""BN"", ""is_mobile"": false}" 4798,2,106,2017-06-01 00:47:47,http://shieldsharber.com/cecelia,,206.63.199.140,"{""location"": ""AF"", ""is_mobile"": true}" 4799,2,106,2017-03-01 15:03:32,http://grimes.co/akeem,,152.228.26.189,"{""location"": ""SA"", ""is_mobile"": true}" 4800,2,106,2017-01-04 03:04:30,http://torphy.info/gordon,,189.195.141.70,"{""location"": ""DZ"", ""is_mobile"": true}" 4801,2,106,2017-05-15 07:59:59,http://lakinmitchell.org/dayana,,136.14.158.144,"{""location"": ""AX"", ""is_mobile"": true}" 4802,2,106,2017-04-24 01:20:48,http://connelly.biz/wilmer,,220.215.131.19,"{""location"": ""AU"", ""is_mobile"": true}" 4803,2,106,2017-01-02 23:36:22,http://kuhn.org/naomie.schoen,,103.172.34.225,"{""location"": ""NZ"", ""is_mobile"": false}" 4804,2,106,2017-01-04 08:05:49,http://emmerich.net/marlin_keler,,100.228.10.75,"{""location"": ""AF"", ""is_mobile"": false}" 4805,2,106,2017-05-27 21:22:16,http://purdy.com/leonel.kemmer,,17.236.155.85,"{""location"": ""CL"", ""is_mobile"": true}" 4806,2,106,2017-01-07 19:03:28,http://wiegandfriesen.co/orrin.zulauf,,171.69.138.46,"{""location"": ""KH"", ""is_mobile"": false}" 4807,2,106,2017-04-29 00:26:38,http://wiegand.biz/joannie,,235.237.204.252,"{""location"": ""KN"", ""is_mobile"": true}" 4808,2,106,2017-03-16 12:58:40,http://krajcik.com/victoria,,39.179.244.44,"{""location"": ""DK"", ""is_mobile"": false}" 4809,2,106,2017-04-12 17:08:28,http://reingerfranecki.biz/jayme,,250.117.134.133,"{""location"": ""BY"", ""is_mobile"": false}" 4810,2,106,2017-05-11 13:25:50,http://kemmer.io/madaline_ziemann,,133.168.243.3,"{""location"": ""CL"", ""is_mobile"": true}" 4811,2,106,2017-01-13 12:40:39,http://macgyver.com/bertha,,189.6.60.151,"{""location"": ""PK"", ""is_mobile"": false}" 4812,2,106,2017-04-28 05:45:21,http://linddaugherty.net/katherine.mccullough,,242.247.168.90,"{""location"": ""VC"", ""is_mobile"": true}" 4813,2,106,2017-01-17 16:49:12,http://mosciski.biz/tiana_mckenzie,,51.201.109.73,"{""location"": ""CF"", ""is_mobile"": false}" 4814,2,106,2017-05-13 16:59:50,http://dietrichhauck.io/samara_weimann,,167.229.193.251,"{""location"": ""MX"", ""is_mobile"": true}" 4815,2,106,2017-01-26 20:31:38,http://kreiger.biz/meggie,,135.216.244.204,"{""location"": ""AF"", ""is_mobile"": false}" 4816,2,106,2017-02-21 16:47:16,http://boyle.io/emely,,230.212.71.15,"{""location"": ""TW"", ""is_mobile"": false}" 4817,2,106,2017-05-29 18:00:16,http://creminryan.io/emmanuelle_ortiz,,12.46.167.65,"{""location"": ""AR"", ""is_mobile"": true}" 4818,2,106,2017-03-05 01:07:20,http://hyatt.com/doug.gaylord,,167.61.233.54,"{""location"": ""VC"", ""is_mobile"": false}" 4819,2,106,2016-12-20 21:34:11,http://goyette.info/mauricio_hammes,,184.160.60.204,"{""location"": ""PF"", ""is_mobile"": true}" 4820,2,106,2017-05-11 08:46:18,http://dare.net/karley,,163.35.242.32,"{""location"": ""IE"", ""is_mobile"": true}" 4821,2,106,2017-01-01 01:40:18,http://altenwerthruel.name/buster,,241.36.2.132,"{""location"": ""LU"", ""is_mobile"": true}" 4822,2,106,2017-03-23 12:53:01,http://kuhn.io/henri_lubowitz,,29.190.125.73,"{""location"": ""EH"", ""is_mobile"": true}" 4823,2,107,2017-05-13 18:46:13,http://eichmann.org/emil.crist,,134.224.109.244,"{""location"": ""TC"", ""is_mobile"": true}" 4824,2,107,2017-04-01 23:22:53,http://senger.org/flo,,127.179.182.149,"{""location"": ""NZ"", ""is_mobile"": false}" 4825,2,107,2017-02-13 04:05:36,http://runte.name/coby_will,,226.117.2.227,"{""location"": ""TM"", ""is_mobile"": false}" 4826,2,107,2017-02-19 03:05:44,http://quigley.info/leif,,92.13.167.95,"{""location"": ""MP"", ""is_mobile"": true}" 4827,2,107,2017-05-02 06:37:23,http://douglas.biz/marcel_lakin,,92.139.22.15,"{""location"": ""KW"", ""is_mobile"": false}" 4828,2,107,2017-05-18 03:18:46,http://orn.com/lila.dibbert,,77.126.108.188,"{""location"": ""KE"", ""is_mobile"": true}" 4829,2,107,2017-02-07 18:47:34,http://stiedemann.com/jedediah,,171.56.16.100,"{""location"": ""BN"", ""is_mobile"": false}" 4830,2,107,2017-03-03 11:28:59,http://effertzwunsch.org/delores.feil,,166.183.30.129,"{""location"": ""IT"", ""is_mobile"": false}" 4831,2,107,2017-04-25 21:06:05,http://thielhintz.name/hailee.wisozk,,25.145.160.41,"{""location"": ""SS"", ""is_mobile"": false}" 4832,2,107,2017-06-07 12:39:03,http://ruecker.org/tobin,,151.205.87.87,"{""location"": ""SI"", ""is_mobile"": false}" 4833,2,107,2017-03-20 09:57:14,http://bernhard.biz/kaleigh,,94.254.117.110,"{""location"": ""KW"", ""is_mobile"": false}" 4834,2,107,2017-05-11 02:59:42,http://leannon.com/fae,,127.217.19.232,"{""location"": ""NE"", ""is_mobile"": false}" 4835,2,107,2017-02-18 22:46:47,http://strosin.biz/joshua.mccullough,,117.160.134.82,"{""location"": ""HU"", ""is_mobile"": true}" 11699,4,261,2017-05-19 01:43:26,http://walsh.name/kacey,0.8743415722,240.128.67.213,"{""location"": ""PF"", ""is_mobile"": true}" 11700,4,261,2017-04-22 21:06:12,http://kunzebreitenberg.net/madie_kilback,0.3560150484,93.171.2.239,"{""location"": ""SE"", ""is_mobile"": true}" 11701,4,261,2017-02-20 00:05:30,http://mcclure.io/tyrel_lockman,0.0855593485,179.97.251.37,"{""location"": ""SI"", ""is_mobile"": true}" 11702,4,261,2017-01-30 17:27:08,http://christiansenblanda.name/miles,0.5185737210,110.234.146.39,"{""location"": ""NL"", ""is_mobile"": true}" 11703,4,261,2017-04-30 06:39:58,http://okeefe.info/merritt_jakubowski,0.0664650429,140.74.97.247,"{""location"": ""AO"", ""is_mobile"": true}" 11704,4,261,2017-06-06 12:31:23,http://turner.biz/alexys.bayer,0.3068166761,102.206.73.142,"{""location"": ""TF"", ""is_mobile"": false}" 11705,4,261,2017-02-05 06:34:19,http://hyatt.net/krystina_bernier,0.2450087298,117.19.133.129,"{""location"": ""MO"", ""is_mobile"": true}" 11706,4,261,2017-06-03 07:16:33,http://bogan.org/ryleigh,0.6496984442,237.129.40.169,"{""location"": ""SK"", ""is_mobile"": true}" 11707,4,261,2016-12-22 19:28:38,http://walkerquigley.info/gennaro,0.3669014061,140.7.233.63,"{""location"": ""LR"", ""is_mobile"": true}" 11708,4,261,2017-05-02 10:02:33,http://boyle.io/kyra_steuber,0.8734812351,145.190.16.10,"{""location"": ""RE"", ""is_mobile"": true}" 11709,4,261,2017-05-13 15:35:41,http://simonis.net/kory_schmeler,0.2695039323,144.217.94.230,"{""location"": ""NE"", ""is_mobile"": true}" 11710,4,261,2017-04-14 23:14:50,http://nicolasglover.info/martina_schinner,0.3933549179,248.253.16.35,"{""location"": ""PY"", ""is_mobile"": false}" 11711,4,261,2017-05-03 19:01:27,http://hilllrogahn.info/steve,0.7664157669,80.77.92.160,"{""location"": ""HR"", ""is_mobile"": true}" 11712,4,261,2017-01-05 20:13:36,http://gulgowskiwiza.info/colton,0.2490651167,236.35.251.176,"{""location"": ""FJ"", ""is_mobile"": true}" 11713,4,261,2017-01-20 00:52:28,http://block.info/candace_hayes,0.6545086958,75.43.106.92,"{""location"": ""MR"", ""is_mobile"": false}" 11714,4,261,2017-03-20 16:25:04,http://shanahandurgan.info/cade_hills,0.1970440804,252.67.190.232,"{""location"": ""SO"", ""is_mobile"": true}" 11715,4,262,2017-01-29 22:24:22,http://barton.co/daryl.haley,0.3656582479,211.93.239.123,"{""location"": ""SI"", ""is_mobile"": false}" 11716,4,262,2017-04-23 06:06:45,http://doyle.net/estelle,0.6512117451,56.185.84.85,"{""location"": ""ET"", ""is_mobile"": false}" 11717,4,262,2017-01-22 02:02:12,http://johnston.co/lexie.stroman,0.2594070468,216.189.128.222,"{""location"": ""SR"", ""is_mobile"": false}" 11718,4,262,2017-03-25 16:14:37,http://ratharmstrong.com/nicholas,0.7474760921,43.247.55.19,"{""location"": ""BH"", ""is_mobile"": false}" 11719,4,262,2017-02-04 17:02:29,http://hermann.net/verona_maggio,0.0575981399,37.20.75.113,"{""location"": ""PL"", ""is_mobile"": true}" 11720,4,262,2017-02-08 16:09:02,http://stanton.net/moie,0.2123649615,205.173.209.200,"{""location"": ""ZM"", ""is_mobile"": false}" 11721,4,262,2016-12-18 17:33:05,http://altenwerthmonahan.biz/heber.hermiston,0.2925691142,71.113.75.230,"{""location"": ""BH"", ""is_mobile"": false}" 11722,4,262,2017-04-22 06:18:07,http://streichstokes.io/brianne,0.7710329015,116.103.174.99,"{""location"": ""TZ"", ""is_mobile"": false}" 11723,4,262,2017-01-29 07:06:43,http://goodwinvon.net/liana.bernier,0.3494733815,219.8.80.153,"{""location"": ""MF"", ""is_mobile"": true}" 11724,4,262,2017-01-02 08:21:08,http://hickle.co/estel,0.5950528313,128.96.194.124,"{""location"": ""ZW"", ""is_mobile"": false}" 11725,4,262,2017-05-27 21:29:15,http://harber.io/noe,0.4514563340,71.76.13.159,"{""location"": ""AT"", ""is_mobile"": false}" 11726,4,262,2016-12-29 15:25:08,http://cummingtehr.biz/orion.carroll,0.5935874076,203.64.183.60,"{""location"": ""MD"", ""is_mobile"": false}" 11727,4,262,2017-03-27 07:18:58,http://dibbert.biz/samanta,0.5419003321,238.15.209.199,"{""location"": ""SY"", ""is_mobile"": false}" 11728,4,262,2017-03-10 04:25:15,http://herzog.name/luella,0.6435586662,189.253.38.250,"{""location"": ""MT"", ""is_mobile"": true}" 11729,4,262,2017-01-03 15:37:16,http://ohara.com/andreanne,0.9572494647,120.15.162.38,"{""location"": ""BA"", ""is_mobile"": false}" 11730,4,262,2017-04-26 16:46:23,http://gleichnermarks.biz/liliana.mccullough,0.2402264278,4.231.48.90,"{""location"": ""SG"", ""is_mobile"": false}" 11731,4,262,2017-01-25 14:06:06,http://nitzsche.com/ardith,0.0853387935,121.73.125.213,"{""location"": ""CH"", ""is_mobile"": false}" 11732,4,262,2017-02-19 17:47:56,http://kihnrunolfsdottir.biz/wanda,0.6784124525,54.146.144.152,"{""location"": ""UZ"", ""is_mobile"": false}" 11733,4,262,2017-06-05 11:28:15,http://terry.name/javonte,0.9762075166,205.79.253.237,"{""location"": ""EH"", ""is_mobile"": true}" 11734,4,262,2017-05-02 06:22:27,http://krajcikkutch.net/anita,0.0232549296,80.6.29.135,"{""location"": ""SS"", ""is_mobile"": true}" 11735,4,262,2017-01-01 03:39:53,http://spencer.info/harvey,0.6660079839,7.59.148.80,"{""location"": ""GQ"", ""is_mobile"": true}" 11736,4,262,2017-04-09 12:11:40,http://ruecker.io/monserrat,0.9566614944,240.82.203.202,"{""location"": ""KZ"", ""is_mobile"": false}" 11737,4,262,2017-03-24 19:18:54,http://ornferry.info/clemens,0.6245346574,99.67.35.187,"{""location"": ""YE"", ""is_mobile"": false}" 11738,4,262,2017-04-21 21:06:54,http://turcotte.name/verla.cartwright,0.2821926132,225.46.139.145,"{""location"": ""JE"", ""is_mobile"": false}" 11739,4,262,2017-02-20 01:18:18,http://mrazboyer.info/sydnee_hermann,0.0526084622,108.164.89.254,"{""location"": ""SE"", ""is_mobile"": false}" 11740,4,262,2016-12-21 17:26:52,http://douglas.biz/emiliano,0.7235854972,254.56.250.117,"{""location"": ""IT"", ""is_mobile"": false}" 11741,4,262,2017-05-27 16:05:12,http://kingerdman.info/elyse,0.8940323175,41.108.62.230,"{""location"": ""HK"", ""is_mobile"": true}" 11742,4,262,2017-05-24 21:01:19,http://reichertbashirian.org/stephanie,0.8196002564,189.182.150.178,"{""location"": ""ST"", ""is_mobile"": true}" 11743,4,262,2017-01-23 12:19:36,http://nikolaus.io/neoma,0.1005516697,209.156.199.230,"{""location"": ""LV"", ""is_mobile"": true}" 11744,4,262,2017-04-03 04:32:38,http://aufderhar.co/zola_larkin,0.7303087541,149.211.69.118,"{""location"": ""CI"", ""is_mobile"": true}" 11745,4,262,2017-05-06 14:38:59,http://turcotte.net/jaunita_kihn,0.6029717536,163.12.134.32,"{""location"": ""EH"", ""is_mobile"": false}" 11746,4,262,2017-04-15 16:04:21,http://smitham.biz/cleveland_stanton,0.6588527583,250.156.173.22,"{""location"": ""DK"", ""is_mobile"": false}" 11747,4,262,2017-04-28 01:19:32,http://muller.info/je.robel,0.7659668083,219.53.108.185,"{""location"": ""SV"", ""is_mobile"": true}" 11748,4,262,2016-12-17 21:21:26,http://johnstonnienow.co/nikki.kautzer,0.9762927496,177.229.37.72,"{""location"": ""TH"", ""is_mobile"": true}" 11749,4,262,2017-02-23 03:05:47,http://keebler.org/louisa_aufderhar,0.7780791992,16.253.82.176,"{""location"": ""GA"", ""is_mobile"": false}" 14652,5,330,2017-05-04 19:59:47,http://jacobi.info/hazel,0.2325358978,15.43.158.10,"{""location"": ""SD"", ""is_mobile"": false}" 14653,5,330,2017-06-05 12:26:03,http://hermiston.info/consuelo,0.2646967671,110.218.118.139,"{""location"": ""GU"", ""is_mobile"": true}" 14654,5,330,2017-05-03 07:27:12,http://shields.io/amaya,0.8262764608,174.74.35.206,"{""location"": ""BH"", ""is_mobile"": false}" 14655,5,330,2017-02-13 12:56:37,http://daniel.co/lydia,0.6537342550,157.66.138.191,"{""location"": ""TM"", ""is_mobile"": false}" 14656,5,330,2017-05-25 04:38:22,http://fritsch.biz/jaren_pfannerstill,0.5201998057,165.253.55.228,"{""location"": ""EH"", ""is_mobile"": false}" 14657,5,330,2017-02-10 13:59:21,http://marquardtkris.info/casey,0.8893991117,145.124.83.200,"{""location"": ""DM"", ""is_mobile"": true}" 14658,5,330,2017-06-04 15:45:20,http://johnscruickshank.biz/stephania,0.1020652600,213.199.174.78,"{""location"": ""TN"", ""is_mobile"": false}" 14659,5,330,2017-01-21 21:46:18,http://borerhills.com/angela_rau,0.6659969416,97.190.105.241,"{""location"": ""ID"", ""is_mobile"": true}" 14660,5,330,2017-01-05 19:26:04,http://buckridge.org/alvis,0.8220261318,183.135.152.79,"{""location"": ""ME"", ""is_mobile"": true}" 14661,5,330,2016-12-20 01:33:14,http://dietrichthiel.com/glenda,0.7352875328,230.215.104.2,"{""location"": ""LC"", ""is_mobile"": true}" 14662,5,330,2017-05-06 11:21:38,http://lebsack.biz/nathan_yost,0.3064470622,21.163.185.251,"{""location"": ""ZM"", ""is_mobile"": true}" 14663,5,330,2017-06-01 15:10:18,http://kihnleuschke.com/caandra.bechtelar,0.7113466540,131.144.222.32,"{""location"": ""SH"", ""is_mobile"": true}" 14664,5,330,2017-03-06 15:15:12,http://schoen.info/donny_oberbrunner,0.2401347817,158.220.62.174,"{""location"": ""TZ"", ""is_mobile"": false}" 14665,5,330,2017-03-07 17:40:09,http://franeckispencer.info/cooper,0.9372319480,121.227.138.95,"{""location"": ""MK"", ""is_mobile"": false}" 14666,5,331,2016-12-29 11:27:50,http://schusterkris.net/patrick.dare,0.2443856292,146.254.154.111,"{""location"": ""JM"", ""is_mobile"": false}" 14667,5,331,2017-06-02 08:50:34,http://swift.net/chelsie_daniel,0.2898655278,237.72.30.62,"{""location"": ""US"", ""is_mobile"": true}" 14668,5,331,2017-04-27 06:59:25,http://block.com/zetta,0.7804417437,47.3.74.83,"{""location"": ""KP"", ""is_mobile"": true}" 14669,5,331,2016-12-19 19:27:57,http://okonwillms.name/rhea.lakin,0.7554555868,121.208.77.94,"{""location"": ""GH"", ""is_mobile"": true}" 14670,5,331,2017-01-19 02:09:18,http://daniel.co/liam,0.7642826636,94.183.187.38,"{""location"": ""CD"", ""is_mobile"": true}" 14671,5,331,2017-04-30 22:18:17,http://zemlak.co/autumn_hintz,0.4700178319,31.119.238.32,"{""location"": ""HN"", ""is_mobile"": false}" 14672,5,331,2016-12-18 09:36:15,http://erdman.name/veda_dietrich,0.8273486174,222.207.36.234,"{""location"": ""MT"", ""is_mobile"": true}" 14673,5,331,2017-05-15 11:08:08,http://colerowe.io/lysanne,0.4302946374,61.40.25.197,"{""location"": ""HK"", ""is_mobile"": false}" 14674,5,331,2017-03-05 12:57:30,http://framilemke.io/ebony,0.3203229262,219.99.161.207,"{""location"": ""GR"", ""is_mobile"": false}" 14675,5,331,2017-06-06 02:55:59,http://kirlin.net/maegan,0.3370020743,93.143.226.107,"{""location"": ""IS"", ""is_mobile"": true}" 14676,5,331,2017-05-12 00:50:51,http://von.net/verlie,0.6233119038,218.16.180.217,"{""location"": ""IO"", ""is_mobile"": true}" 14677,5,331,2017-05-14 17:15:10,http://nitzschekemmer.org/gus_cain,0.2365747328,170.189.103.78,"{""location"": ""US"", ""is_mobile"": true}" 14678,5,331,2017-05-10 01:46:08,http://morar.com/elaina.botsford,0.6967860857,98.192.61.117,"{""location"": ""NU"", ""is_mobile"": false}" 14679,5,331,2017-05-27 10:35:13,http://kerlukeboehm.com/giovanna_pagac,0.7050660887,207.73.171.55,"{""location"": ""IQ"", ""is_mobile"": true}" 14680,5,331,2017-01-17 00:41:29,http://schulist.biz/ottilie,0.0658972127,14.86.218.165,"{""location"": ""SJ"", ""is_mobile"": false}" 14681,5,331,2017-05-08 02:15:17,http://lehnerbashirian.org/maeve,0.4563063351,172.140.120.206,"{""location"": ""DM"", ""is_mobile"": true}" 14682,5,331,2017-02-09 16:08:47,http://tillman.org/nedra,0.6521436622,245.92.132.2,"{""location"": ""BE"", ""is_mobile"": false}" 14683,5,331,2017-06-10 16:44:03,http://koepp.name/jake.farrell,0.2338609575,205.49.160.216,"{""location"": ""BO"", ""is_mobile"": true}" 14684,5,331,2017-04-18 06:58:20,http://turcotte.net/leonie,0.2083530683,115.177.5.19,"{""location"": ""SV"", ""is_mobile"": false}" 14685,5,331,2017-04-01 12:28:42,http://thompsoncorwin.info/lorenza,0.8420526365,56.13.46.20,"{""location"": ""GS"", ""is_mobile"": false}" 14686,5,331,2016-12-17 10:00:15,http://kulaswolff.co/evert.wintheiser,0.3742161103,251.149.52.107,"{""location"": ""AU"", ""is_mobile"": false}" 14687,5,331,2017-05-23 18:02:44,http://heidenreich.info/lester,0.6043919789,42.238.164.93,"{""location"": ""MM"", ""is_mobile"": false}" 14688,5,331,2017-05-31 15:30:48,http://bradtke.com/rodrigo,0.5907889040,184.8.211.169,"{""location"": ""CH"", ""is_mobile"": false}" 14689,5,331,2017-02-11 05:17:44,http://hansen.com/chaim.auer,0.4008318024,26.75.233.185,"{""location"": ""CY"", ""is_mobile"": false}" 14690,5,331,2017-03-04 05:27:08,http://keebler.co/filiberto,0.8868175417,121.107.164.138,"{""location"": ""CK"", ""is_mobile"": false}" 14691,5,331,2017-03-12 07:32:41,http://ebertgleason.org/raven_stark,0.7425114570,144.116.165.142,"{""location"": ""PL"", ""is_mobile"": true}" 14692,5,331,2017-02-14 21:25:18,http://ruecker.biz/marianne,0.8225852173,182.23.155.232,"{""location"": ""HU"", ""is_mobile"": true}" 14693,5,331,2017-02-16 00:06:52,http://walsh.com/brendan,0.0983186296,199.108.152.242,"{""location"": ""WS"", ""is_mobile"": true}" 14694,5,331,2016-12-24 13:58:38,http://kuvalis.co/lloyd.schaden,0.0386907775,197.189.94.45,"{""location"": ""JM"", ""is_mobile"": true}" 14695,5,331,2017-03-07 11:53:01,http://littel.biz/leilani_thompson,0.9826430796,122.187.203.138,"{""location"": ""CX"", ""is_mobile"": false}" 14696,5,331,2017-03-26 03:48:01,http://schneider.io/emie.bayer,0.7819464022,157.253.189.113,"{""location"": ""GN"", ""is_mobile"": false}" 14697,5,331,2017-01-26 15:07:00,http://runolfon.io/heather.ratke,0.0860383083,111.64.49.191,"{""location"": ""UY"", ""is_mobile"": true}" 14698,5,331,2017-03-31 16:27:27,http://runolfsdottirschaden.org/stan.tremblay,0.2471061150,24.151.176.72,"{""location"": ""YT"", ""is_mobile"": true}" 14699,5,331,2017-03-18 04:01:10,http://bins.co/deron.price,0.6596700641,17.165.195.226,"{""location"": ""DJ"", ""is_mobile"": true}" 14700,5,331,2017-02-13 01:55:48,http://reichertmacgyver.net/vilma.dooley,0.9310456846,205.246.34.58,"{""location"": ""AE"", ""is_mobile"": true}" 14701,5,331,2017-05-22 04:47:40,http://wintheiser.co/rollin,0.6119566482,42.148.216.162,"{""location"": ""KR"", ""is_mobile"": false}" 14702,5,331,2017-06-03 17:48:52,http://beerreinger.co/mireille,0.7890395724,38.237.208.75,"{""location"": ""GS"", ""is_mobile"": true}" 17642,6,397,2017-02-09 09:18:35,http://lockmanwisozk.org/julian_schmitt,0.1207674398,176.123.189.55,"{""location"": ""DZ"", ""is_mobile"": true}" 17643,6,397,2017-04-04 10:59:04,http://pfannerstill.org/arnaldo.kihn,0.5884988578,239.132.70.79,"{""location"": ""LU"", ""is_mobile"": true}" 17644,6,397,2017-05-14 16:52:57,http://gleason.info/aditya_muller,0.2921562683,94.2.115.28,"{""location"": ""BG"", ""is_mobile"": true}" 17645,6,397,2017-03-25 07:30:25,http://bergnaumturner.name/blair.bogisich,0.4445086035,251.127.94.202,"{""location"": ""PT"", ""is_mobile"": true}" 17646,6,397,2017-05-21 09:51:47,http://streichboehm.io/donna.herman,0.4131711419,153.117.248.82,"{""location"": ""GP"", ""is_mobile"": false}" 17647,6,397,2017-03-01 18:11:00,http://wilkinson.info/enid_heidenreich,0.4166243511,230.135.47.177,"{""location"": ""SC"", ""is_mobile"": false}" 17648,6,397,2017-05-03 21:39:02,http://emard.io/herminia.jacobs,0.7270855991,75.164.53.45,"{""location"": ""KW"", ""is_mobile"": false}" 17649,6,397,2017-05-17 08:50:32,http://spinka.net/brant.kerluke,0.3880866900,64.225.23.210,"{""location"": ""TW"", ""is_mobile"": true}" 17650,6,397,2017-04-10 08:08:41,http://faheypredovic.com/letha_sauer,0.0090437921,109.99.83.254,"{""location"": ""CN"", ""is_mobile"": false}" 17651,6,397,2017-01-27 08:41:20,http://pfeffer.com/dashawn,0.4570704919,234.245.154.178,"{""location"": ""RO"", ""is_mobile"": false}" 17652,6,397,2017-01-17 04:54:47,http://heathcotebednar.name/bo_deckow,0.7755067289,195.202.91.155,"{""location"": ""NZ"", ""is_mobile"": false}" 17653,6,397,2017-05-15 01:31:38,http://haag.name/theodora_crona,0.5429641814,174.211.174.5,"{""location"": ""UY"", ""is_mobile"": true}" 17654,6,397,2017-03-29 13:32:31,http://cummings.com/melya,0.1459191072,236.73.251.159,"{""location"": ""LR"", ""is_mobile"": false}" 17655,6,397,2017-01-29 17:39:27,http://parisian.io/ozella_romaguera,0.4894780083,82.48.88.40,"{""location"": ""PA"", ""is_mobile"": true}" 17656,6,397,2017-02-03 22:38:41,http://kshlerin.co/eleanora,0.7802459643,15.242.81.209,"{""location"": ""TG"", ""is_mobile"": true}" 17657,6,397,2017-04-19 14:18:30,http://hansen.co/rene,0.9961141511,251.24.47.206,"{""location"": ""SC"", ""is_mobile"": false}" 17658,6,397,2017-02-17 14:56:31,http://ankunding.io/isac,0.6758877399,86.58.244.67,"{""location"": ""MY"", ""is_mobile"": true}" 17659,6,397,2017-04-15 21:13:47,http://lubowitzoreilly.org/frederick,0.9477762195,86.188.130.252,"{""location"": ""MG"", ""is_mobile"": true}" 17660,6,397,2017-03-09 10:35:03,http://macgyverrunolfsdottir.org/jefferey.volkman,0.5877242621,79.88.91.135,"{""location"": ""GG"", ""is_mobile"": false}" 17661,6,397,2017-01-24 07:03:32,http://jasttremblay.net/louvenia,0.0253228657,77.146.119.175,"{""location"": ""BH"", ""is_mobile"": false}" 17662,6,397,2016-12-15 10:44:53,http://rohan.info/oma_huel,0.7812498539,139.76.12.31,"{""location"": ""FO"", ""is_mobile"": true}" 17663,6,397,2017-04-14 02:48:07,http://muller.org/ignatius_pouros,0.4748161225,63.250.11.197,"{""location"": ""KW"", ""is_mobile"": false}" 17664,6,397,2017-01-03 16:41:40,http://koepp.net/nils,0.9921148356,36.27.236.179,"{""location"": ""RS"", ""is_mobile"": true}" 17665,6,397,2017-02-14 16:54:30,http://schowalterfritsch.net/angus,0.8715384341,245.42.147.98,"{""location"": ""BW"", ""is_mobile"": false}" 17666,6,397,2017-05-29 23:39:47,http://powlowski.net/ward_botsford,0.2908112356,252.122.62.19,"{""location"": ""RW"", ""is_mobile"": true}" 17667,6,397,2016-12-31 00:11:17,http://hettinger.biz/camryn_hermiston,0.1722919918,237.59.209.181,"{""location"": ""BV"", ""is_mobile"": false}" 17668,6,397,2017-05-16 11:37:39,http://blanda.net/caterina,0.9108275777,33.225.192.160,"{""location"": ""TH"", ""is_mobile"": false}" 17669,6,397,2016-12-23 12:56:32,http://ferry.io/owen_harber,0.1676347311,124.56.4.105,"{""location"": ""LT"", ""is_mobile"": true}" 17670,6,397,2017-03-18 11:12:52,http://hellerjerde.co/dora_weber,0.2921955598,100.190.211.167,"{""location"": ""IO"", ""is_mobile"": true}" 17671,6,397,2017-04-09 18:55:15,http://kuphal.org/ezra.weinat,0.8296761631,25.12.159.41,"{""location"": ""CK"", ""is_mobile"": false}" 17672,6,397,2016-12-18 16:20:46,http://baumbachkaulke.io/callie,0.9578062218,60.58.254.196,"{""location"": ""KP"", ""is_mobile"": false}" 17673,6,397,2017-05-30 10:24:21,http://harvey.biz/dillon.stanton,0.6945464826,111.235.173.150,"{""location"": ""LI"", ""is_mobile"": true}" 17674,6,397,2017-05-13 06:50:14,http://colelangworth.io/conrad_prosacco,0.2771849137,241.61.243.13,"{""location"": ""BZ"", ""is_mobile"": true}" 17675,6,397,2017-02-10 22:39:12,http://streich.name/una,0.9069206935,141.50.78.190,"{""location"": ""MW"", ""is_mobile"": true}" 17676,6,397,2017-05-12 13:10:08,http://heaneygaylord.biz/claudie.wiza,0.8462124866,176.105.100.179,"{""location"": ""GP"", ""is_mobile"": true}" 17677,6,398,2017-04-21 04:33:16,http://mante.info/alanis,0.8943627033,150.149.142.169,"{""location"": ""TO"", ""is_mobile"": true}" 17678,6,398,2017-03-16 10:07:33,http://bartoletti.name/eddie_fay,0.2164434329,121.133.166.147,"{""location"": ""ZW"", ""is_mobile"": false}" 17679,6,398,2017-05-13 18:09:21,http://mcglynn.info/coby.lindgren,0.1749993262,152.37.35.78,"{""location"": ""MG"", ""is_mobile"": true}" 17680,6,398,2017-03-17 12:55:26,http://bernier.co/romaine.sporer,0.1282572600,44.41.223.150,"{""location"": ""CM"", ""is_mobile"": true}" 17681,6,398,2016-12-30 14:47:49,http://reingerterry.com/brian,0.9111078919,145.168.3.191,"{""location"": ""PF"", ""is_mobile"": true}" 17682,6,398,2016-12-21 14:07:26,http://greenfeldercarroll.biz/raquel_huel,0.8084772702,120.50.9.139,"{""location"": ""HK"", ""is_mobile"": true}" 17683,6,398,2017-05-28 22:00:10,http://blick.org/joana.wuckert,0.0682506693,31.90.150.9,"{""location"": ""TC"", ""is_mobile"": true}" 17684,6,398,2017-06-02 12:22:44,http://dibbert.biz/vicky.romaguera,0.3405378293,83.166.55.203,"{""location"": ""AZ"", ""is_mobile"": false}" 17685,6,398,2017-03-27 03:28:34,http://feeneyboyer.co/kenton,0.0060133811,61.172.225.104,"{""location"": ""MS"", ""is_mobile"": true}" 17686,6,398,2017-02-07 06:30:25,http://murazik.net/adaline,0.6380084818,123.8.250.153,"{""location"": ""GY"", ""is_mobile"": false}" 17687,6,398,2017-02-19 14:46:30,http://baumbachhintz.name/yesenia_tillman,0.8570087596,142.65.11.42,"{""location"": ""JE"", ""is_mobile"": false}" 17688,6,398,2017-03-27 03:07:38,http://pacocha.net/kristopher.walker,0.6753941382,73.122.129.25,"{""location"": ""HT"", ""is_mobile"": true}" 17689,6,398,2017-02-14 09:14:19,http://dickenstrantow.info/vena,0.0477307774,171.73.157.96,"{""location"": ""MO"", ""is_mobile"": true}" 17690,6,398,2017-02-22 17:12:27,http://grahamgulgowski.info/elody,0.8327230784,11.231.231.34,"{""location"": ""SS"", ""is_mobile"": false}" 17691,6,398,2017-05-10 05:17:07,http://boyle.info/colten,0.6818436362,115.133.247.160,"{""location"": ""KR"", ""is_mobile"": false}" 4836,2,107,2017-01-12 22:17:20,http://conroy.org/alyon,,48.160.12.64,"{""location"": ""AS"", ""is_mobile"": true}" 4837,2,107,2017-02-06 10:12:10,http://bergstrom.org/kendrick_balistreri,,25.126.21.66,"{""location"": ""UM"", ""is_mobile"": false}" 4838,2,107,2017-03-13 12:05:12,http://mcclure.org/aniya_ziemann,,182.101.98.77,"{""location"": ""BQ"", ""is_mobile"": true}" 4839,2,107,2016-12-28 10:14:16,http://wiza.com/kiley,,63.247.248.209,"{""location"": ""SN"", ""is_mobile"": false}" 4840,2,107,2017-05-17 06:15:52,http://gorczany.biz/willard,,181.116.232.140,"{""location"": ""EE"", ""is_mobile"": true}" 4841,2,107,2017-03-06 13:39:26,http://schaden.co/shaniya_carter,,18.236.3.101,"{""location"": ""HN"", ""is_mobile"": true}" 4842,2,107,2017-01-16 08:38:08,http://veum.com/myah.barton,,61.160.103.54,"{""location"": ""NO"", ""is_mobile"": true}" 4843,2,107,2017-05-26 21:34:08,http://prosaccosmith.name/maryse,,92.132.38.53,"{""location"": ""KW"", ""is_mobile"": true}" 4844,2,107,2017-02-25 17:04:55,http://schulist.io/deangelo,,130.198.163.23,"{""location"": ""FM"", ""is_mobile"": true}" 4845,2,107,2017-04-25 11:53:04,http://cronin.info/cierra,,151.195.199.162,"{""location"": ""CG"", ""is_mobile"": true}" 4846,2,107,2016-12-20 01:49:46,http://stark.org/casandra,,46.134.167.241,"{""location"": ""UZ"", ""is_mobile"": false}" 4847,2,107,2017-04-04 11:35:30,http://lindgren.biz/bert_olson,,53.100.101.200,"{""location"": ""NG"", ""is_mobile"": false}" 4848,2,107,2016-12-31 05:04:38,http://schowaltermosciski.net/bradly,,65.239.153.7,"{""location"": ""BZ"", ""is_mobile"": false}" 4849,2,107,2017-05-21 21:02:06,http://roberts.info/kris,,189.205.251.150,"{""location"": ""VC"", ""is_mobile"": false}" 4850,2,107,2017-05-23 23:21:09,http://oreilly.name/sydni,,109.13.72.71,"{""location"": ""KP"", ""is_mobile"": true}" 4851,2,107,2016-12-29 05:24:50,http://wiegand.org/virgie,,220.236.81.103,"{""location"": ""GT"", ""is_mobile"": false}" 4852,2,107,2017-02-19 14:25:08,http://kaulkerunolfon.com/adalberto,,186.38.17.6,"{""location"": ""SA"", ""is_mobile"": false}" 4853,2,107,2017-05-02 00:57:53,http://kuhic.org/caie_botsford,,184.220.73.3,"{""location"": ""TH"", ""is_mobile"": true}" 4854,2,107,2017-05-07 08:41:01,http://hermanngusikowski.org/broderick,,204.104.182.198,"{""location"": ""UM"", ""is_mobile"": true}" 4855,2,107,2017-01-08 01:27:51,http://harris.name/clay,,31.13.11.49,"{""location"": ""AG"", ""is_mobile"": false}" 4856,2,107,2017-05-04 20:01:58,http://okon.net/mozell.harvey,,100.220.177.129,"{""location"": ""NE"", ""is_mobile"": false}" 4857,2,107,2016-12-27 14:53:19,http://koelpinhartmann.biz/melia,,86.250.31.238,"{""location"": ""AZ"", ""is_mobile"": false}" 4858,2,107,2017-04-05 15:42:21,http://sporer.info/luther,,171.96.181.177,"{""location"": ""CZ"", ""is_mobile"": false}" 4859,2,107,2017-03-29 14:05:14,http://stammhuels.co/drake_bednar,,104.92.183.208,"{""location"": ""HN"", ""is_mobile"": true}" 4860,2,107,2017-04-14 17:39:13,http://welch.info/jasper,,71.45.115.132,"{""location"": ""CM"", ""is_mobile"": false}" 4861,2,107,2017-01-12 18:01:24,http://roberts.org/elsie,,218.9.22.70,"{""location"": ""PW"", ""is_mobile"": true}" 4862,2,107,2017-06-05 09:28:02,http://bauchhayes.org/sid,,170.96.241.96,"{""location"": ""GF"", ""is_mobile"": true}" 4863,2,107,2017-05-07 23:06:43,http://robertsbeahan.name/annabell,,38.117.221.62,"{""location"": ""VI"", ""is_mobile"": true}" 4864,2,107,2017-05-22 02:40:36,http://schroedernolan.co/pearl,,151.85.162.32,"{""location"": ""KM"", ""is_mobile"": false}" 4865,2,107,2017-01-15 03:13:18,http://jast.net/royal.hansen,,62.153.191.47,"{""location"": ""AW"", ""is_mobile"": false}" 4866,2,107,2017-04-27 08:14:37,http://senger.biz/briana,,47.137.113.62,"{""location"": ""ES"", ""is_mobile"": false}" 4867,2,107,2017-02-04 21:09:30,http://mclaughlinstrosin.io/august,,220.250.185.170,"{""location"": ""DJ"", ""is_mobile"": true}" 4868,2,107,2017-03-16 02:27:49,http://moen.biz/gregg,,193.19.228.249,"{""location"": ""CK"", ""is_mobile"": true}" 4869,2,107,2017-04-04 17:36:49,http://gorczany.info/guadalupe,,66.173.145.197,"{""location"": ""BG"", ""is_mobile"": false}" 4870,2,107,2016-12-16 07:16:44,http://howepouros.com/dena,,237.33.232.41,"{""location"": ""MT"", ""is_mobile"": true}" 4871,2,107,2017-05-06 18:40:20,http://hoeger.biz/wilhelmine,,190.124.40.154,"{""location"": ""PL"", ""is_mobile"": true}" 4872,2,107,2017-02-01 20:59:20,http://dach.com/kariane,,185.54.178.11,"{""location"": ""AM"", ""is_mobile"": true}" 4873,2,107,2017-03-27 02:32:45,http://corkerysporer.net/anahi_erdman,,203.111.9.155,"{""location"": ""LB"", ""is_mobile"": true}" 4874,2,107,2017-05-10 05:36:48,http://white.name/judson_waelchi,,96.120.136.47,"{""location"": ""GM"", ""is_mobile"": true}" 4875,2,107,2017-03-07 12:20:51,http://hagenes.info/saige.walter,,39.254.179.227,"{""location"": ""AD"", ""is_mobile"": false}" 4876,2,107,2017-01-13 23:29:35,http://tillmanlesch.name/jade,,108.186.53.113,"{""location"": ""UY"", ""is_mobile"": false}" 4877,2,107,2016-12-31 07:02:17,http://bashiriankautzer.com/freddy,,221.198.247.242,"{""location"": ""ER"", ""is_mobile"": true}" 4878,2,107,2017-02-14 04:06:19,http://ondricka.name/lorenza,,38.189.133.64,"{""location"": ""TW"", ""is_mobile"": true}" 4879,2,107,2017-05-06 22:26:35,http://predovic.co/winnifred,,153.162.55.234,"{""location"": ""CM"", ""is_mobile"": false}" 4880,2,107,2017-03-10 06:52:37,http://vonruedenwalsh.co/weldon,,38.34.52.38,"{""location"": ""FR"", ""is_mobile"": true}" 4881,2,107,2017-03-16 19:42:56,http://abshirestoltenberg.name/keely,,9.238.220.73,"{""location"": ""MZ"", ""is_mobile"": true}" 4882,2,108,2017-02-17 04:38:32,http://torp.org/boyd.shanahan,,73.50.149.216,"{""location"": ""IO"", ""is_mobile"": true}" 4883,2,108,2017-01-27 11:13:13,http://mayert.com/yesenia_sanford,,83.146.145.79,"{""location"": ""ER"", ""is_mobile"": false}" 4884,2,108,2017-01-24 01:35:30,http://becker.net/roslyn.kunde,,186.173.248.103,"{""location"": ""JM"", ""is_mobile"": true}" 4885,2,108,2017-04-15 08:37:50,http://ziemebalistreri.net/maribel_shields,,155.64.210.249,"{""location"": ""FI"", ""is_mobile"": true}" 4886,2,108,2017-04-21 09:36:18,http://spinka.biz/winifred,,254.125.125.126,"{""location"": ""HT"", ""is_mobile"": true}" 4887,2,108,2017-05-24 20:52:44,http://ankunding.co/yazmin_heel,,21.78.121.190,"{""location"": ""SR"", ""is_mobile"": true}" 4888,2,108,2017-02-11 09:04:27,http://mcglynn.co/estevan.mayer,,224.99.5.60,"{""location"": ""IR"", ""is_mobile"": true}" 4889,2,108,2017-03-29 10:59:29,http://ricelubowitz.net/anibal,,208.242.6.25,"{""location"": ""SC"", ""is_mobile"": false}" 4890,2,108,2017-05-18 11:56:02,http://homenicksmitham.io/nellie.green,,78.229.26.123,"{""location"": ""ES"", ""is_mobile"": false}" 4891,2,108,2017-02-22 12:48:15,http://tremblay.name/casimer.leannon,,221.116.157.60,"{""location"": ""GF"", ""is_mobile"": true}" 11750,4,262,2017-02-22 08:28:36,http://littlejerde.org/bridget_hirthe,0.3378686770,38.20.132.148,"{""location"": ""AO"", ""is_mobile"": true}" 11751,4,262,2017-01-26 11:28:55,http://smith.net/felipa.batz,0.2174058446,107.97.228.95,"{""location"": ""MF"", ""is_mobile"": true}" 11752,4,262,2017-05-18 18:03:37,http://larkin.com/wellington_ward,0.8829933347,140.187.238.241,"{""location"": ""BV"", ""is_mobile"": true}" 17825,7,403,2017-03-12 08:46:21,http://tillman.org/clement_wiegand,,47.129.116.142,"{""location"": ""KN"", ""is_mobile"": false}" 17826,7,403,2017-01-08 19:21:51,http://gerholdhermiston.name/rory_crona,,183.3.121.210,"{""location"": ""CL"", ""is_mobile"": false}" 17827,7,403,2016-12-20 11:09:23,http://streichokuneva.io/mortimer,,63.196.194.98,"{""location"": ""SY"", ""is_mobile"": false}" 17828,7,403,2017-05-22 05:02:00,http://framibogisich.io/kellie,,190.17.14.165,"{""location"": ""TT"", ""is_mobile"": false}" 17829,7,403,2017-06-03 11:06:54,http://hirthereichel.info/sarai.dubuque,,241.212.11.45,"{""location"": ""TT"", ""is_mobile"": true}" 17830,7,403,2017-02-23 00:10:25,http://medhurst.org/annie,,186.5.89.84,"{""location"": ""NL"", ""is_mobile"": true}" 17831,7,403,2017-05-18 09:05:42,http://deckow.io/ezekiel.beier,,78.45.133.126,"{""location"": ""UG"", ""is_mobile"": true}" 17832,7,403,2017-04-17 22:52:04,http://zieme.io/jeramie,,159.132.175.52,"{""location"": ""PK"", ""is_mobile"": true}" 17833,7,403,2017-01-03 03:29:26,http://hansen.org/magali.botsford,,101.224.153.130,"{""location"": ""KH"", ""is_mobile"": false}" 17834,7,403,2017-02-11 19:16:18,http://bode.com/kenneth,,23.74.38.30,"{""location"": ""CO"", ""is_mobile"": true}" 17835,7,403,2017-05-14 12:28:37,http://funk.name/rhett_okon,,76.40.132.13,"{""location"": ""LU"", ""is_mobile"": true}" 17836,7,403,2017-05-28 15:10:15,http://schiller.biz/gladyce.moen,,168.116.52.24,"{""location"": ""BQ"", ""is_mobile"": true}" 17837,7,403,2017-04-08 22:53:56,http://beervolkman.net/jakayla,,176.219.208.171,"{""location"": ""MV"", ""is_mobile"": true}" 17838,7,403,2016-12-20 19:29:17,http://croninstroman.biz/marguerite.halvorson,,54.128.34.174,"{""location"": ""VC"", ""is_mobile"": false}" 17839,7,403,2017-06-07 09:07:21,http://feest.net/augusta,,238.79.126.118,"{""location"": ""CD"", ""is_mobile"": false}" 17840,7,403,2017-05-22 00:44:16,http://damore.net/kole_cartwright,,152.43.150.142,"{""location"": ""EC"", ""is_mobile"": true}" 17841,7,403,2016-12-22 05:12:13,http://lehner.org/earl,,195.219.189.70,"{""location"": ""MY"", ""is_mobile"": true}" 17842,7,403,2017-04-24 01:39:28,http://graham.io/shanny.wilderman,,70.216.196.161,"{""location"": ""UY"", ""is_mobile"": true}" 17843,7,403,2017-05-11 10:54:03,http://sporerschaden.org/mattie,,216.113.36.61,"{""location"": ""CN"", ""is_mobile"": true}" 17844,7,403,2017-01-17 04:28:27,http://upton.info/joany_hermann,,37.169.163.151,"{""location"": ""VN"", ""is_mobile"": true}" 17845,7,403,2017-03-05 16:39:42,http://macejkovic.biz/queen.johnson,,192.253.184.168,"{""location"": ""VE"", ""is_mobile"": true}" 17846,7,403,2017-04-18 16:35:44,http://brakusturcotte.com/elouise,,116.16.57.135,"{""location"": ""NI"", ""is_mobile"": true}" 17847,7,403,2016-12-14 19:31:24,http://schoen.info/conner,,202.225.235.126,"{""location"": ""VE"", ""is_mobile"": false}" 17848,7,403,2017-02-24 16:43:28,http://hackett.io/arielle.ankunding,,75.112.42.43,"{""location"": ""EC"", ""is_mobile"": false}" 17849,7,403,2017-03-07 00:30:46,http://baumbach.net/arianna_homenick,,216.136.93.193,"{""location"": ""GR"", ""is_mobile"": true}" 17850,7,403,2017-02-25 07:26:53,http://lockmandare.org/abigayle,,238.159.34.149,"{""location"": ""MF"", ""is_mobile"": false}" 17851,7,403,2016-12-27 00:29:22,http://pollich.io/joshuah,,115.25.242.19,"{""location"": ""CK"", ""is_mobile"": true}" 17852,7,403,2017-01-05 04:45:10,http://hills.name/gabriel,,18.106.132.227,"{""location"": ""SX"", ""is_mobile"": false}" 17853,7,403,2017-04-24 05:41:59,http://miller.name/dock_fritsch,,125.252.88.67,"{""location"": ""AE"", ""is_mobile"": true}" 17854,7,403,2017-05-03 13:03:19,http://wilkinson.net/alan_turner,,131.41.214.75,"{""location"": ""IR"", ""is_mobile"": false}" 17855,7,403,2017-03-23 10:46:43,http://doyle.net/omer_hammes,,198.167.72.12,"{""location"": ""TM"", ""is_mobile"": true}" 17856,7,403,2016-12-28 05:28:13,http://stammlehner.com/oma,,141.153.77.33,"{""location"": ""CR"", ""is_mobile"": true}" 17857,7,403,2017-06-01 11:49:47,http://schaeferbarton.biz/jayde,,40.148.209.206,"{""location"": ""PK"", ""is_mobile"": true}" 17858,7,403,2017-01-11 02:36:02,http://schuppe.biz/prince,,155.230.246.40,"{""location"": ""LU"", ""is_mobile"": true}" 17859,7,403,2017-04-26 15:52:16,http://considinehamill.com/geo.conn,,189.150.235.189,"{""location"": ""BM"", ""is_mobile"": true}" 17860,7,403,2017-01-01 05:19:08,http://schowalterjakubowski.biz/rick,,46.112.137.84,"{""location"": ""NU"", ""is_mobile"": true}" 17861,7,403,2017-05-01 12:43:58,http://bednarzemlak.io/breanne.stiedemann,,181.24.220.109,"{""location"": ""GU"", ""is_mobile"": true}" 17862,7,403,2017-02-04 19:50:34,http://ritchiehintz.co/emil,,109.221.116.96,"{""location"": ""CF"", ""is_mobile"": true}" 17863,7,403,2017-05-18 09:48:08,http://champlin.net/johann,,46.176.32.50,"{""location"": ""MS"", ""is_mobile"": true}" 17864,7,403,2017-03-15 19:07:57,http://mcclure.org/efrain,,42.95.230.126,"{""location"": ""VN"", ""is_mobile"": true}" 17865,7,404,2017-01-03 01:03:40,http://mckenzie.name/lou.funk,,52.186.141.52,"{""location"": ""ZA"", ""is_mobile"": false}" 17866,7,404,2017-05-25 02:41:59,http://kingrau.info/merlin,,13.79.40.84,"{""location"": ""MS"", ""is_mobile"": true}" 17867,7,404,2017-01-09 15:59:48,http://ohara.io/candice_herzog,,80.148.19.78,"{""location"": ""DE"", ""is_mobile"": true}" 17868,7,404,2017-03-11 17:49:10,http://hudsonkeebler.co/jaycee,,213.180.125.143,"{""location"": ""CW"", ""is_mobile"": true}" 17869,7,404,2017-01-31 20:54:15,http://hartmann.com/rebecca,,161.215.216.247,"{""location"": ""VA"", ""is_mobile"": true}" 17870,7,404,2017-03-18 18:45:01,http://schadendaniel.io/eugenia.lubowitz,,183.82.178.173,"{""location"": ""GW"", ""is_mobile"": false}" 17871,7,404,2016-12-31 05:30:51,http://kuvalitreich.net/darren,,65.25.171.229,"{""location"": ""MG"", ""is_mobile"": false}" 17872,7,404,2017-04-01 11:36:35,http://mcdermott.info/shaina,,198.171.81.225,"{""location"": ""BL"", ""is_mobile"": true}" 17873,7,404,2017-01-12 07:11:31,http://oconner.info/robyn_goyette,,242.102.78.254,"{""location"": ""BG"", ""is_mobile"": false}" 17874,7,404,2017-01-12 04:31:53,http://wintheiser.biz/don,,85.35.208.176,"{""location"": ""LC"", ""is_mobile"": false}" 17875,7,404,2017-03-26 16:19:29,http://mosciski.info/kirstin_jast,,131.7.56.137,"{""location"": ""AZ"", ""is_mobile"": true}" 17876,7,404,2016-12-17 00:12:42,http://donnelly.io/garfield_shanahan,,174.107.113.110,"{""location"": ""GN"", ""is_mobile"": true}" 14703,5,331,2017-05-14 17:18:17,http://lakincole.info/deshawn.kovacek,0.1367501830,9.2.253.172,"{""location"": ""KG"", ""is_mobile"": false}" 14704,5,331,2017-06-08 04:06:43,http://reinger.net/sean,0.9219266992,179.224.12.194,"{""location"": ""NA"", ""is_mobile"": false}" 14705,5,331,2017-01-13 17:31:10,http://durgan.com/yvonne_ernser,0.0072071357,246.59.227.108,"{""location"": ""BN"", ""is_mobile"": true}" 14706,5,331,2017-02-25 04:28:04,http://spencer.name/lincoln.lueilwitz,0.2018808724,56.69.238.83,"{""location"": ""MC"", ""is_mobile"": false}" 14707,5,331,2017-02-23 20:41:31,http://beer.name/rick_corwin,0.7461956022,92.6.119.248,"{""location"": ""GF"", ""is_mobile"": true}" 14708,5,331,2017-05-19 21:47:30,http://wymankeebler.name/kailyn_walter,0.9082239700,217.229.57.148,"{""location"": ""MF"", ""is_mobile"": true}" 14709,5,331,2017-05-16 02:56:02,http://bogisichwill.com/christopher,0.0689343116,2.251.5.14,"{""location"": ""KR"", ""is_mobile"": true}" 14710,5,331,2017-03-03 22:02:16,http://weimannskiles.net/erin_yost,0.8285622430,9.201.50.244,"{""location"": ""UM"", ""is_mobile"": true}" 14711,5,331,2017-05-14 11:47:52,http://metz.org/tracy,0.3752662051,103.159.80.5,"{""location"": ""KP"", ""is_mobile"": false}" 14712,5,331,2017-01-26 13:36:25,http://batz.biz/daniela,0.8770948383,27.237.147.239,"{""location"": ""IT"", ""is_mobile"": true}" 14713,5,331,2016-12-25 13:46:10,http://townekoepp.name/geo,0.6246946044,62.120.90.73,"{""location"": ""GY"", ""is_mobile"": true}" 14714,5,331,2017-04-08 13:37:56,http://marquardt.biz/willa_reilly,0.9455184382,252.230.108.142,"{""location"": ""CZ"", ""is_mobile"": false}" 14715,5,331,2017-05-22 04:28:59,http://wolff.com/sally.fahey,0.0999147003,189.166.182.178,"{""location"": ""RO"", ""is_mobile"": false}" 14716,5,331,2016-12-18 06:15:39,http://andersonrau.biz/evie,0.2203005410,86.55.21.89,"{""location"": ""TO"", ""is_mobile"": true}" 14717,5,331,2017-01-02 09:04:12,http://schneider.io/olaf,0.9907748150,49.156.34.143,"{""location"": ""FK"", ""is_mobile"": false}" 14718,5,331,2016-12-20 13:09:04,http://wilderman.net/jany,0.2161471299,194.105.15.96,"{""location"": ""AF"", ""is_mobile"": true}" 14719,5,331,2017-03-06 06:09:10,http://little.info/barbara.abshire,0.5462280100,166.200.70.32,"{""location"": ""CV"", ""is_mobile"": true}" 14720,5,331,2017-05-26 01:45:40,http://hamill.net/stephon_gulgowski,0.3833828098,233.4.169.15,"{""location"": ""AL"", ""is_mobile"": true}" 14721,5,331,2017-06-06 13:24:08,http://kulasfahey.info/estelle_botsford,0.2606890554,159.38.59.27,"{""location"": ""VG"", ""is_mobile"": false}" 14722,5,332,2017-03-12 22:29:16,http://corwin.info/jana,0.2386205386,234.88.166.242,"{""location"": ""SS"", ""is_mobile"": true}" 14723,5,332,2017-01-27 08:22:56,http://kovacek.info/casandra,0.4377326311,166.205.52.148,"{""location"": ""PE"", ""is_mobile"": true}" 14724,5,332,2017-05-02 15:57:44,http://macejkovic.info/sven,0.3436730370,18.194.249.28,"{""location"": ""FO"", ""is_mobile"": false}" 14725,5,332,2017-01-22 11:14:15,http://mraz.net/emelie,0.5858670199,239.96.115.189,"{""location"": ""GF"", ""is_mobile"": false}" 14726,5,332,2016-12-24 15:50:52,http://hartmann.info/herbert,0.8422704458,233.184.101.15,"{""location"": ""MN"", ""is_mobile"": true}" 14727,5,332,2017-06-12 13:05:57,http://will.com/landen_schumm,0.3007364159,180.125.242.6,"{""location"": ""BQ"", ""is_mobile"": true}" 14728,5,332,2017-03-08 08:26:55,http://pouros.co/deion.hauck,0.1501976817,62.170.168.97,"{""location"": ""FJ"", ""is_mobile"": false}" 14729,5,332,2017-03-06 09:26:09,http://johnston.co/mariela.metz,0.8120029292,54.246.191.78,"{""location"": ""JP"", ""is_mobile"": true}" 14730,5,332,2017-02-24 21:09:14,http://cremin.co/reie_kautzer,0.0716985997,2.127.65.248,"{""location"": ""MN"", ""is_mobile"": true}" 14731,5,332,2017-02-14 21:20:18,http://crona.com/ricardo_block,0.2341368880,69.145.88.168,"{""location"": ""GG"", ""is_mobile"": true}" 14732,5,332,2017-04-26 16:10:28,http://dubuquekrajcik.info/rosalia,0.3775255251,162.116.206.181,"{""location"": ""AZ"", ""is_mobile"": true}" 14733,5,332,2017-05-29 05:22:43,http://gibsonhartmann.co/paolo_hudson,0.2361965101,201.102.230.103,"{""location"": ""LK"", ""is_mobile"": false}" 14734,5,332,2017-02-15 10:05:14,http://jacobsbergstrom.io/johnathon_kemmer,0.5604634544,212.125.77.251,"{""location"": ""ER"", ""is_mobile"": true}" 14735,5,332,2017-03-19 05:07:27,http://nicolas.info/leone,0.6728938691,155.182.214.38,"{""location"": ""GT"", ""is_mobile"": true}" 14736,5,332,2016-12-17 10:51:15,http://lindgren.biz/enos.fritsch,0.3924670223,45.200.66.94,"{""location"": ""PA"", ""is_mobile"": false}" 14737,5,332,2017-05-09 11:44:23,http://koch.net/jonathon_medhurst,0.6550492306,51.23.169.170,"{""location"": ""GR"", ""is_mobile"": false}" 14738,5,332,2016-12-21 15:39:15,http://larkin.co/myrtice,0.6460994836,97.87.188.155,"{""location"": ""NZ"", ""is_mobile"": false}" 14739,5,332,2017-01-19 09:57:27,http://swaniawski.net/jean.carroll,0.9910162251,116.145.60.144,"{""location"": ""CR"", ""is_mobile"": false}" 14740,5,332,2017-02-18 14:36:56,http://armstrong.com/robbie,0.5205134163,35.144.79.195,"{""location"": ""BW"", ""is_mobile"": true}" 14741,5,332,2016-12-18 14:54:16,http://armstrongwaters.io/glenna,0.4850643600,39.37.32.98,"{""location"": ""TR"", ""is_mobile"": false}" 14742,5,332,2017-05-09 08:48:13,http://hackett.info/demond_bogan,0.9079141321,56.104.37.242,"{""location"": ""BI"", ""is_mobile"": false}" 14743,5,332,2016-12-16 07:59:15,http://herzog.net/jayden,0.9658499534,159.43.123.182,"{""location"": ""PL"", ""is_mobile"": true}" 14744,5,332,2017-04-13 08:12:00,http://cremin.info/hildegard_stokes,0.4144352455,142.235.96.31,"{""location"": ""SE"", ""is_mobile"": true}" 14745,5,332,2017-03-27 13:46:13,http://kreigerframi.biz/chandler.kuphal,0.4092450409,188.157.197.104,"{""location"": ""YT"", ""is_mobile"": false}" 14746,5,333,2016-12-23 23:48:20,http://krajcikfritsch.io/drew_schroeder,0.9822358828,209.133.109.169,"{""location"": ""MO"", ""is_mobile"": true}" 14747,5,333,2017-04-10 23:06:59,http://gutmannkuhic.net/danielle,0.3189304459,195.230.122.198,"{""location"": ""IQ"", ""is_mobile"": false}" 14748,5,333,2017-02-14 07:53:23,http://daremetz.co/jerald,0.9546816447,137.213.26.202,"{""location"": ""PF"", ""is_mobile"": true}" 14749,5,333,2017-04-09 13:37:21,http://macgyver.com/khalil,0.0405680813,65.63.142.145,"{""location"": ""VI"", ""is_mobile"": false}" 14750,5,333,2016-12-29 16:24:36,http://jacobs.biz/jerome,0.9077540361,195.157.18.62,"{""location"": ""NU"", ""is_mobile"": true}" 14751,5,333,2017-03-11 11:53:00,http://willmswintheiser.io/nannie,0.9251853611,214.94.108.81,"{""location"": ""FM"", ""is_mobile"": true}" 14752,5,333,2017-05-11 09:23:55,http://morar.net/eldridge,0.2394336846,35.179.234.82,"{""location"": ""BQ"", ""is_mobile"": true}" 14753,5,333,2017-04-12 14:02:45,http://kulasblanda.co/maribel_kunde,0.4046506687,137.16.222.5,"{""location"": ""AG"", ""is_mobile"": false}" 17692,6,398,2017-02-04 08:30:44,http://sawaynzulauf.biz/vernie,0.6287410715,26.198.29.182,"{""location"": ""GG"", ""is_mobile"": false}" 17693,6,398,2017-05-12 21:15:37,http://collinsnicolas.io/ferne_mertz,0.3987874898,189.152.140.26,"{""location"": ""PS"", ""is_mobile"": true}" 17694,6,398,2017-01-13 10:39:50,http://hahnoconner.biz/era.daugherty,0.9330704961,14.124.199.27,"{""location"": ""TH"", ""is_mobile"": false}" 17695,6,398,2017-02-23 04:26:56,http://nicolas.biz/jermey_nolan,0.0620368169,35.208.58.239,"{""location"": ""EH"", ""is_mobile"": true}" 17696,6,398,2017-02-23 21:07:27,http://batz.info/viola.ritchie,0.8362231651,79.118.183.112,"{""location"": ""DM"", ""is_mobile"": false}" 17697,6,398,2016-12-18 00:41:21,http://keebler.com/caitlyn.pfeffer,0.4008740068,102.73.112.33,"{""location"": ""IE"", ""is_mobile"": false}" 17698,6,398,2017-05-15 01:50:57,http://watsica.info/nash_smith,0.6721545576,6.64.202.8,"{""location"": ""CZ"", ""is_mobile"": false}" 17699,6,398,2017-04-01 11:51:20,http://stammbartell.biz/merlin,0.3025212671,251.76.17.27,"{""location"": ""AI"", ""is_mobile"": false}" 17700,6,398,2017-02-12 08:39:18,http://yost.biz/raymond,0.2061240069,59.181.232.227,"{""location"": ""GP"", ""is_mobile"": false}" 17701,6,398,2017-04-07 02:18:06,http://reichert.info/reid,0.5102469824,37.181.122.228,"{""location"": ""FK"", ""is_mobile"": false}" 17702,6,399,2017-03-23 10:39:20,http://batztowne.info/alexander_deckow,0.5487109987,173.170.65.105,"{""location"": ""CL"", ""is_mobile"": false}" 17703,6,399,2017-06-04 01:14:29,http://maggio.io/reagan,0.5449074943,57.142.76.99,"{""location"": ""ZW"", ""is_mobile"": true}" 17704,6,399,2017-02-20 08:13:56,http://stehr.info/lonzo,0.4933567305,129.206.116.39,"{""location"": ""GL"", ""is_mobile"": false}" 17705,6,399,2017-05-28 02:22:54,http://kirlincorkery.io/leon,0.4134862295,227.62.106.20,"{""location"": ""HU"", ""is_mobile"": false}" 17706,6,399,2017-04-23 23:50:48,http://muellerleannon.info/ara,0.9451264427,238.189.184.40,"{""location"": ""RS"", ""is_mobile"": false}" 17707,6,399,2017-01-12 15:23:31,http://schoencarter.name/floie,0.8115406439,94.83.140.146,"{""location"": ""JO"", ""is_mobile"": true}" 17708,6,399,2017-06-13 05:35:13,http://mckenzie.io/vernon,0.5545412368,129.46.176.200,"{""location"": ""SZ"", ""is_mobile"": true}" 17709,6,399,2017-01-08 09:14:18,http://jerdebraun.name/trevor,0.2180867498,188.177.196.76,"{""location"": ""BD"", ""is_mobile"": false}" 17710,6,399,2016-12-26 03:55:05,http://jakubowski.name/katelynn,0.1665825133,225.91.79.75,"{""location"": ""IE"", ""is_mobile"": false}" 17711,6,399,2017-05-24 03:04:32,http://bernhard.name/freda.emard,0.6636047331,130.45.139.97,"{""location"": ""CA"", ""is_mobile"": false}" 17712,6,399,2017-02-28 06:57:02,http://hammesbernhard.biz/arianna.gutkowski,0.4420130910,127.86.101.186,"{""location"": ""TM"", ""is_mobile"": false}" 17713,6,399,2017-04-04 22:33:29,http://kirlin.io/zoie.ruel,0.7248452224,172.227.154.21,"{""location"": ""UG"", ""is_mobile"": true}" 17714,6,399,2017-03-31 23:11:04,http://kutch.biz/humberto,0.7680616119,119.50.122.248,"{""location"": ""CZ"", ""is_mobile"": true}" 17715,6,399,2017-05-12 14:53:01,http://mclaughlin.io/gustave_jacobi,0.7907308132,185.243.6.43,"{""location"": ""CV"", ""is_mobile"": true}" 17716,6,399,2017-01-02 18:38:22,http://koelpinstoltenberg.io/hanna,0.7855979094,19.185.226.96,"{""location"": ""CN"", ""is_mobile"": false}" 17717,6,399,2017-04-17 01:40:00,http://hodkiewicz.org/zachery,0.5799441458,219.112.39.43,"{""location"": ""UG"", ""is_mobile"": false}" 17718,6,399,2017-02-05 11:45:48,http://lehner.co/karina.nitzsche,0.1477900563,196.81.83.147,"{""location"": ""KH"", ""is_mobile"": true}" 17719,6,399,2017-03-11 09:48:20,http://dubuquepadberg.biz/jeramie.beatty,0.7922005828,218.102.219.74,"{""location"": ""GP"", ""is_mobile"": true}" 17720,6,399,2017-03-23 03:27:44,http://padberg.com/jaeden.treutel,0.7721796568,85.18.83.58,"{""location"": ""FJ"", ""is_mobile"": true}" 17721,6,399,2017-03-31 22:27:25,http://barton.io/daphnee.crona,0.7749302953,206.61.203.101,"{""location"": ""LV"", ""is_mobile"": false}" 17722,6,399,2017-04-06 18:49:13,http://barrowsauer.name/taya,0.1781988620,145.175.242.151,"{""location"": ""IR"", ""is_mobile"": false}" 17723,6,399,2017-05-01 09:31:42,http://huelborer.biz/darion.toy,0.3495630395,47.48.103.106,"{""location"": ""IT"", ""is_mobile"": true}" 17724,6,399,2017-05-29 00:36:22,http://jerde.info/dedric_shields,0.4598322415,124.87.118.81,"{""location"": ""AI"", ""is_mobile"": false}" 17725,6,399,2017-01-20 23:12:46,http://sauermclaughlin.name/winnifred_jaskolski,0.6299211104,136.209.169.75,"{""location"": ""AS"", ""is_mobile"": false}" 17726,6,399,2016-12-29 13:16:29,http://reichertmertz.org/ava,0.0690251814,117.164.100.63,"{""location"": ""CA"", ""is_mobile"": true}" 17727,6,399,2017-05-07 09:06:30,http://ullrichkunde.org/marge,0.7051249415,194.173.93.155,"{""location"": ""GN"", ""is_mobile"": true}" 17728,6,399,2017-04-27 05:15:38,http://pollichnicolas.co/raphaelle,0.7314432492,193.162.237.156,"{""location"": ""CR"", ""is_mobile"": true}" 17729,6,399,2017-05-31 18:05:55,http://heller.co/skye_doyle,0.3643732336,216.102.95.56,"{""location"": ""GT"", ""is_mobile"": true}" 17730,6,399,2017-03-02 23:18:58,http://cruickshankhauck.co/tyreek,0.7826487759,91.146.127.104,"{""location"": ""CM"", ""is_mobile"": false}" 17731,6,399,2017-01-18 02:26:54,http://ritchie.biz/tyshawn,0.1766218510,185.76.50.73,"{""location"": ""PT"", ""is_mobile"": true}" 17732,6,399,2016-12-22 22:38:40,http://roberts.info/kelly_little,0.9709513749,73.234.86.210,"{""location"": ""ML"", ""is_mobile"": false}" 17733,6,400,2017-03-28 05:43:32,http://kris.info/imelda.gottlieb,0.8618895659,111.163.230.85,"{""location"": ""JM"", ""is_mobile"": true}" 17734,6,400,2017-06-12 12:21:55,http://medhurst.net/ricardo,0.4964682571,233.112.8.139,"{""location"": ""GS"", ""is_mobile"": false}" 17735,6,400,2017-02-23 23:34:21,http://kozey.name/betsy_hansen,0.9599432974,117.101.117.115,"{""location"": ""TG"", ""is_mobile"": true}" 17736,6,400,2017-06-11 12:43:14,http://stokes.biz/jared,0.0391132105,4.76.243.70,"{""location"": ""TK"", ""is_mobile"": false}" 17737,6,400,2017-04-07 22:38:21,http://dubuque.io/elian.mclaughlin,0.5294712981,121.177.39.254,"{""location"": ""VI"", ""is_mobile"": false}" 17738,6,400,2017-04-24 00:04:41,http://litteljacobi.info/leland,0.5434395398,217.215.210.20,"{""location"": ""CU"", ""is_mobile"": false}" 17739,6,400,2017-03-30 08:05:08,http://kuhn.net/megane,0.7045714148,42.242.205.123,"{""location"": ""WF"", ""is_mobile"": false}" 17740,6,400,2017-02-25 02:45:24,http://harvey.name/mavis.schaden,0.1850798196,155.243.76.40,"{""location"": ""MA"", ""is_mobile"": true}" 17741,6,400,2017-03-13 22:46:32,http://maggiowunsch.io/eldon,0.3320982580,183.160.168.239,"{""location"": ""MZ"", ""is_mobile"": true}" 17742,6,400,2016-12-26 01:09:47,http://damorepaucek.co/rodrigo,0.8168516603,75.183.97.190,"{""location"": ""OM"", ""is_mobile"": true}" 4892,2,108,2017-05-23 14:25:01,http://lebsackleffler.co/cleveland_kuhn,,230.236.47.31,"{""location"": ""EE"", ""is_mobile"": true}" 4893,2,108,2017-01-19 12:25:39,http://ruelmitchell.com/john_gottlieb,,248.36.94.33,"{""location"": ""UM"", ""is_mobile"": true}" 4894,2,108,2017-01-28 00:14:14,http://ruecker.name/stella,,70.159.107.118,"{""location"": ""ES"", ""is_mobile"": true}" 4895,2,108,2017-03-02 04:07:23,http://abshireborer.io/sophia,,250.211.109.213,"{""location"": ""NF"", ""is_mobile"": true}" 4896,2,108,2017-02-11 06:09:34,http://oberbrunner.info/mac.kub,,97.5.9.62,"{""location"": ""IO"", ""is_mobile"": true}" 4897,2,108,2017-02-12 01:46:08,http://lehnerborer.biz/donavon.maggio,,54.41.117.250,"{""location"": ""PW"", ""is_mobile"": true}" 4898,2,108,2017-03-14 20:17:42,http://gaylordnitzsche.info/sanford.reilly,,22.245.228.36,"{""location"": ""BL"", ""is_mobile"": true}" 4899,2,108,2017-02-03 18:24:44,http://volkman.com/aurelio,,168.230.39.120,"{""location"": ""NR"", ""is_mobile"": false}" 4900,2,108,2016-12-29 01:07:17,http://johnston.info/jocelyn,,109.97.250.156,"{""location"": ""SE"", ""is_mobile"": true}" 4901,2,108,2017-03-10 10:38:54,http://bergstromblick.net/reggie,,220.203.125.49,"{""location"": ""ET"", ""is_mobile"": true}" 4902,2,108,2016-12-24 16:03:32,http://hudsonkemmer.co/anne_reichert,,203.48.147.74,"{""location"": ""FJ"", ""is_mobile"": true}" 4903,2,108,2017-02-17 19:24:14,http://anderson.co/dusty.schimmel,,236.215.5.136,"{""location"": ""YE"", ""is_mobile"": true}" 4904,2,108,2016-12-19 09:43:24,http://haley.biz/ambrose,,233.13.239.67,"{""location"": ""MQ"", ""is_mobile"": true}" 4905,2,108,2017-01-05 19:51:01,http://brakus.co/richie.beahan,,20.48.41.204,"{""location"": ""GQ"", ""is_mobile"": true}" 4906,2,108,2017-05-27 17:31:38,http://schustergerhold.com/dane,,218.180.142.94,"{""location"": ""BH"", ""is_mobile"": true}" 4907,2,108,2017-05-27 09:41:53,http://labadie.biz/kayla,,51.218.56.101,"{""location"": ""GD"", ""is_mobile"": false}" 4908,2,108,2016-12-26 05:59:19,http://cremin.name/colton_dare,,51.155.23.18,"{""location"": ""MS"", ""is_mobile"": true}" 4909,2,108,2017-02-08 02:09:56,http://bergnaum.co/magnus,,187.140.236.231,"{""location"": ""UY"", ""is_mobile"": true}" 4910,2,108,2017-06-01 15:11:33,http://turcotterobel.co/amya,,225.228.220.171,"{""location"": ""PS"", ""is_mobile"": true}" 4911,2,108,2017-01-25 13:32:21,http://konopelskiokuneva.net/amanda,,226.76.92.39,"{""location"": ""MT"", ""is_mobile"": false}" 4912,2,108,2017-06-02 15:33:12,http://moriette.org/hilario.friesen,,238.129.149.173,"{""location"": ""MT"", ""is_mobile"": true}" 4913,2,108,2017-04-10 17:40:12,http://pollich.name/aleia,,28.199.74.156,"{""location"": ""MM"", ""is_mobile"": false}" 4914,2,108,2017-04-14 02:01:21,http://borerschneider.com/kamille_gibson,,144.244.34.6,"{""location"": ""NI"", ""is_mobile"": false}" 4915,2,108,2017-05-24 15:56:53,http://wuckert.io/kevin,,163.206.26.4,"{""location"": ""TF"", ""is_mobile"": true}" 4916,2,108,2017-05-28 19:12:21,http://kuhic.name/stephen_cronin,,173.44.126.2,"{""location"": ""TN"", ""is_mobile"": true}" 4917,2,108,2017-04-04 01:09:28,http://mosciskimurphy.com/titus,,85.83.232.71,"{""location"": ""SH"", ""is_mobile"": false}" 4918,2,108,2017-05-01 02:02:24,http://daniel.co/andy,,243.208.181.103,"{""location"": ""TF"", ""is_mobile"": true}" 4919,2,108,2017-04-23 21:41:21,http://kundeswift.info/rickey_thiel,,26.249.25.180,"{""location"": ""AQ"", ""is_mobile"": false}" 4920,2,108,2017-03-05 10:41:24,http://nader.io/harry_schulist,,37.161.91.92,"{""location"": ""LI"", ""is_mobile"": true}" 4921,2,108,2017-01-11 21:40:03,http://jastboehm.io/river,,202.118.129.242,"{""location"": ""WS"", ""is_mobile"": true}" 4922,2,108,2017-04-29 10:13:01,http://kulashalvorson.net/emelia_glover,,110.14.109.8,"{""location"": ""AZ"", ""is_mobile"": true}" 4923,2,108,2017-05-24 19:05:48,http://gorczanybergstrom.name/della.botsford,,128.228.72.6,"{""location"": ""TG"", ""is_mobile"": true}" 4924,2,108,2017-05-23 14:14:01,http://kozey.net/jackson,,118.212.64.182,"{""location"": ""MN"", ""is_mobile"": false}" 4925,2,108,2016-12-30 02:05:41,http://emard.com/alivia,,13.38.119.67,"{""location"": ""SH"", ""is_mobile"": true}" 4926,2,108,2017-02-03 13:08:33,http://feeney.net/olin,,193.151.121.99,"{""location"": ""AD"", ""is_mobile"": false}" 4927,2,108,2017-04-18 01:12:27,http://blickgraham.biz/max,,126.222.127.34,"{""location"": ""BJ"", ""is_mobile"": true}" 4928,2,108,2017-03-13 09:23:51,http://thiel.info/daniella_rodriguez,,166.169.229.73,"{""location"": ""RU"", ""is_mobile"": true}" 4929,2,108,2017-03-31 09:03:56,http://rodriguez.org/sydni,,81.251.115.173,"{""location"": ""RU"", ""is_mobile"": true}" 4930,2,108,2017-04-05 18:25:43,http://koelpin.org/izabella,,102.213.2.219,"{""location"": ""CO"", ""is_mobile"": false}" 4931,2,109,2017-06-03 16:06:22,http://bayer.biz/judy,,207.128.15.13,"{""location"": ""JO"", ""is_mobile"": false}" 4932,2,109,2016-12-26 15:58:08,http://klein.io/donny_roob,,28.231.93.74,"{""location"": ""VA"", ""is_mobile"": false}" 4933,2,109,2017-01-28 06:41:09,http://grimes.name/kylie,,65.172.251.141,"{""location"": ""WS"", ""is_mobile"": false}" 4934,2,109,2016-12-30 13:23:59,http://reilly.io/heath,,26.59.233.207,"{""location"": ""BR"", ""is_mobile"": false}" 4935,2,109,2017-03-23 23:42:35,http://rohan.net/kari_rosenbaum,,126.19.214.74,"{""location"": ""ET"", ""is_mobile"": true}" 4936,2,109,2017-05-25 19:23:19,http://keeling.info/jillian,,15.179.85.126,"{""location"": ""DO"", ""is_mobile"": true}" 4937,2,109,2016-12-16 04:56:11,http://ricecronin.name/davin.schultz,,217.21.56.46,"{""location"": ""AR"", ""is_mobile"": true}" 4938,2,109,2017-02-23 21:44:54,http://bosco.io/camden,,137.35.252.19,"{""location"": ""BF"", ""is_mobile"": true}" 4939,2,109,2017-02-10 07:12:25,http://willmsmarvin.com/laura,,30.6.99.142,"{""location"": ""MT"", ""is_mobile"": true}" 4940,2,109,2017-05-02 00:16:59,http://mann.net/korey,,15.44.134.76,"{""location"": ""MX"", ""is_mobile"": false}" 4941,2,109,2017-04-18 12:05:30,http://cruickshankrogahn.info/grace,,31.182.122.25,"{""location"": ""TM"", ""is_mobile"": false}" 4942,2,109,2017-01-17 00:08:08,http://moen.net/dahlia,,221.168.40.20,"{""location"": ""TN"", ""is_mobile"": false}" 4943,2,109,2017-05-30 23:51:04,http://stiedemann.org/dylan.douglas,,39.44.248.150,"{""location"": ""KR"", ""is_mobile"": true}" 4944,2,109,2017-06-10 18:57:00,http://hanereynolds.info/garth,,223.173.164.173,"{""location"": ""BR"", ""is_mobile"": false}" 4945,2,109,2017-01-10 08:46:52,http://wehnerbayer.name/serena.hayes,,108.3.77.60,"{""location"": ""PL"", ""is_mobile"": true}" 4946,2,109,2017-03-25 01:14:53,http://kuhnluettgen.io/dusty,,152.168.173.186,"{""location"": ""GA"", ""is_mobile"": false}" 4947,2,109,2017-02-21 20:37:30,http://dooleyoberbrunner.org/antonetta,,157.230.26.59,"{""location"": ""NA"", ""is_mobile"": false}" 17877,7,404,2017-03-02 07:53:08,http://hintzkaulke.com/patsy,,4.95.207.166,"{""location"": ""AR"", ""is_mobile"": true}" 17878,7,404,2017-05-11 04:49:40,http://schambergerkertzmann.io/foster_cronin,,224.151.54.224,"{""location"": ""VN"", ""is_mobile"": true}" 17879,7,404,2017-04-30 16:04:38,http://borer.org/nathen,,182.82.177.224,"{""location"": ""CY"", ""is_mobile"": true}" 17880,7,404,2017-04-05 16:27:53,http://kutch.com/hildegard_mayert,,199.225.82.114,"{""location"": ""AU"", ""is_mobile"": true}" 17881,7,404,2017-03-01 02:01:58,http://skiles.biz/ferne,,214.189.90.53,"{""location"": ""GY"", ""is_mobile"": false}" 17882,7,404,2016-12-20 18:05:39,http://wiegand.info/frederick_barton,,10.93.242.239,"{""location"": ""CD"", ""is_mobile"": true}" 17883,7,404,2017-04-27 22:33:00,http://stiedemann.biz/cleo.beahan,,139.34.179.61,"{""location"": ""CV"", ""is_mobile"": true}" 17884,7,404,2017-03-09 12:43:12,http://walker.name/francisca,,204.28.174.247,"{""location"": ""ID"", ""is_mobile"": true}" 17885,7,405,2017-04-06 03:56:19,http://crona.org/maude.sauer,,186.185.46.13,"{""location"": ""NA"", ""is_mobile"": false}" 17886,7,405,2017-06-05 19:15:21,http://grantspencer.info/javon,,176.5.49.187,"{""location"": ""GP"", ""is_mobile"": false}" 17887,7,405,2017-05-14 09:18:55,http://rutherford.biz/kevon.ferry,,180.40.15.86,"{""location"": ""BB"", ""is_mobile"": true}" 17888,7,405,2017-02-07 19:29:11,http://lemkeparisian.org/rita,,219.214.40.76,"{""location"": ""CR"", ""is_mobile"": true}" 17889,7,405,2016-12-17 00:46:30,http://wizagrady.io/wyatt,,187.47.117.16,"{""location"": ""BT"", ""is_mobile"": true}" 17890,7,405,2016-12-30 08:54:25,http://rowe.net/audie,,130.214.5.240,"{""location"": ""DM"", ""is_mobile"": true}" 17891,7,405,2017-05-05 19:50:20,http://carrollbergnaum.name/winifred,,253.242.19.164,"{""location"": ""CH"", ""is_mobile"": true}" 17892,7,405,2016-12-13 16:02:32,http://wilderman.com/omari,,235.142.192.82,"{""location"": ""JM"", ""is_mobile"": false}" 17893,7,405,2017-05-19 17:08:48,http://connellywhite.io/weldon.hane,,98.231.38.78,"{""location"": ""KG"", ""is_mobile"": false}" 17894,7,405,2017-04-05 23:56:24,http://hand.org/jeanette,,191.91.125.128,"{""location"": ""AG"", ""is_mobile"": true}" 17895,7,405,2017-02-28 19:46:31,http://klingconroy.info/melba,,82.139.163.136,"{""location"": ""FI"", ""is_mobile"": true}" 17896,7,405,2016-12-17 20:46:31,http://hickle.com/vergie.hammes,,86.177.92.12,"{""location"": ""AD"", ""is_mobile"": false}" 17897,7,405,2017-02-06 07:29:23,http://schmelerokuneva.net/bryce,,20.21.99.225,"{""location"": ""HT"", ""is_mobile"": false}" 17898,7,405,2017-01-13 22:03:27,http://labadie.biz/dane,,75.169.201.178,"{""location"": ""BJ"", ""is_mobile"": true}" 17899,7,405,2017-02-02 16:38:58,http://creminabshire.info/katelynn,,211.54.3.129,"{""location"": ""MQ"", ""is_mobile"": false}" 17900,7,405,2017-02-17 09:18:54,http://quitzonokon.name/dandre,,37.181.39.116,"{""location"": ""GT"", ""is_mobile"": false}" 17901,7,405,2017-02-07 20:05:59,http://gulgowski.biz/trace_stehr,,5.161.175.177,"{""location"": ""AF"", ""is_mobile"": true}" 17902,7,405,2017-04-20 21:54:58,http://rueckerrice.net/celine,,93.185.115.39,"{""location"": ""SC"", ""is_mobile"": true}" 17903,7,405,2017-06-08 13:54:45,http://collins.biz/aubree_romaguera,,130.33.65.231,"{""location"": ""JO"", ""is_mobile"": false}" 17904,7,405,2017-01-21 21:38:26,http://welchfisher.net/ivah,,16.80.18.86,"{""location"": ""NE"", ""is_mobile"": true}" 17905,7,405,2016-12-22 09:32:26,http://hicklebayer.io/pietro,,232.91.124.23,"{""location"": ""DO"", ""is_mobile"": false}" 17906,7,405,2017-03-27 23:58:49,http://orn.org/rubye.zboncak,,178.175.129.5,"{""location"": ""BM"", ""is_mobile"": false}" 17907,7,405,2017-05-03 12:48:34,http://rempel.info/winnifred,,8.80.40.25,"{""location"": ""MT"", ""is_mobile"": false}" 17908,7,405,2017-03-18 14:02:13,http://stamm.info/juliet,,145.4.154.143,"{""location"": ""PT"", ""is_mobile"": false}" 17909,7,405,2017-06-07 04:55:49,http://nicolatehr.info/kennith_stehr,,92.98.93.198,"{""location"": ""CY"", ""is_mobile"": false}" 17910,7,405,2017-05-12 13:17:41,http://christiansen.info/angeline,,250.21.142.60,"{""location"": ""SL"", ""is_mobile"": true}" 17911,7,405,2017-03-07 04:28:41,http://ullrich.io/colby.glover,,117.251.168.190,"{""location"": ""BR"", ""is_mobile"": false}" 17912,7,405,2017-05-19 18:46:38,http://jast.org/creola,,232.90.188.69,"{""location"": ""GS"", ""is_mobile"": true}" 17913,7,405,2017-03-16 18:47:38,http://johnston.com/cordie.konopelski,,70.204.153.156,"{""location"": ""NE"", ""is_mobile"": false}" 17914,7,405,2017-01-30 18:31:10,http://eichmannheathcote.co/lavonne_hilpert,,207.179.178.251,"{""location"": ""MO"", ""is_mobile"": true}" 17915,7,405,2017-01-16 03:37:55,http://bechtelar.net/kaleigh,,65.150.131.253,"{""location"": ""WF"", ""is_mobile"": true}" 17916,7,405,2017-02-25 04:26:10,http://konopelski.com/liam.bernhard,,204.137.68.63,"{""location"": ""FO"", ""is_mobile"": true}" 17917,7,405,2017-03-22 20:04:49,http://brakus.biz/lora,,175.37.228.231,"{""location"": ""SY"", ""is_mobile"": true}" 17918,7,405,2017-02-08 00:28:55,http://hermann.io/kim.larkin,,86.195.166.77,"{""location"": ""MA"", ""is_mobile"": false}" 17919,7,405,2017-04-26 19:09:51,http://kulas.name/lawson_davis,,137.137.118.13,"{""location"": ""MX"", ""is_mobile"": false}" 17920,7,405,2017-04-11 13:49:04,http://howell.info/daniella,,9.73.244.104,"{""location"": ""GU"", ""is_mobile"": true}" 17921,7,405,2017-03-10 09:52:45,http://stiedemann.net/tillman,,71.235.63.148,"{""location"": ""KG"", ""is_mobile"": true}" 17922,7,405,2017-05-16 02:41:58,http://williamsonhalvorson.biz/rachael,,32.202.148.141,"{""location"": ""JP"", ""is_mobile"": true}" 17923,7,405,2017-03-29 20:06:16,http://bechtelar.org/meggie_steuber,,196.159.31.227,"{""location"": ""TV"", ""is_mobile"": false}" 17924,7,405,2017-06-04 15:46:52,http://dooley.org/lacey,,78.49.214.77,"{""location"": ""WF"", ""is_mobile"": false}" 17925,7,405,2017-03-09 14:04:59,http://skiles.name/jazlyn.bogisich,,122.208.144.46,"{""location"": ""SS"", ""is_mobile"": true}" 17926,7,405,2017-05-03 16:07:36,http://miller.co/verla.leffler,,133.223.40.136,"{""location"": ""QA"", ""is_mobile"": false}" 17927,7,405,2017-05-16 13:51:24,http://kunze.com/oleta.dietrich,,253.5.140.214,"{""location"": ""KN"", ""is_mobile"": true}" 17928,7,405,2017-05-21 10:30:36,http://fisher.io/elva_kozey,,69.43.16.74,"{""location"": ""SB"", ""is_mobile"": true}" 17929,7,405,2017-04-25 23:54:49,http://stanton.name/rylan_sanford,,118.168.101.240,"{""location"": ""BT"", ""is_mobile"": true}" 17930,7,405,2017-04-20 08:44:34,http://swaniawskijacobson.name/vernon.crist,,53.53.16.84,"{""location"": ""MC"", ""is_mobile"": true}" 17931,7,405,2017-01-21 17:23:33,http://hammes.com/nelson,,43.107.213.230,"{""location"": ""ZW"", ""is_mobile"": true}" 14754,5,333,2017-01-14 05:52:16,http://klockowelch.name/hunter,0.0154178608,139.44.2.22,"{""location"": ""TN"", ""is_mobile"": false}" 14755,5,333,2017-03-30 21:39:50,http://streich.info/gaston,0.8708701607,164.176.157.156,"{""location"": ""SM"", ""is_mobile"": false}" 14756,5,333,2017-01-11 20:24:39,http://kling.io/ted,0.8075684005,149.63.211.236,"{""location"": ""TO"", ""is_mobile"": true}" 14757,5,333,2016-12-13 14:18:21,http://emard.net/cristian,0.0583404723,85.109.248.121,"{""location"": ""JP"", ""is_mobile"": false}" 14758,5,333,2017-03-09 06:58:12,http://smith.biz/houston,0.4978359865,143.5.31.19,"{""location"": ""GL"", ""is_mobile"": true}" 14759,5,333,2017-06-12 07:50:30,http://homenickoconnell.net/monte.wilderman,0.5342460037,53.83.144.11,"{""location"": ""TJ"", ""is_mobile"": true}" 14760,5,333,2016-12-28 17:20:05,http://klein.biz/janiya.ratke,0.0449911030,30.99.34.88,"{""location"": ""MD"", ""is_mobile"": false}" 14761,5,333,2017-03-10 10:26:29,http://weber.com/montana.conn,0.2030414064,9.205.247.63,"{""location"": ""NI"", ""is_mobile"": false}" 14762,5,333,2017-06-02 04:10:35,http://crooks.co/imelda,0.4099687776,68.208.151.69,"{""location"": ""TJ"", ""is_mobile"": false}" 14763,5,333,2017-01-20 14:26:17,http://blanda.org/flavie.bartell,0.2029236640,101.219.29.123,"{""location"": ""RU"", ""is_mobile"": false}" 14764,5,333,2017-01-25 04:35:48,http://hickle.co/kane_windler,0.9427166483,191.35.104.19,"{""location"": ""MO"", ""is_mobile"": false}" 14765,5,333,2017-02-27 20:03:35,http://olson.com/cole,0.5378219988,239.222.97.17,"{""location"": ""CY"", ""is_mobile"": false}" 14766,5,333,2017-02-28 01:48:05,http://feest.biz/palma,0.1382801516,76.158.153.165,"{""location"": ""AW"", ""is_mobile"": true}" 14767,5,333,2017-03-11 15:47:04,http://reynoldsgottlieb.biz/nikko_von,0.6161831292,156.39.130.201,"{""location"": ""RU"", ""is_mobile"": true}" 14768,5,333,2017-05-30 05:04:15,http://nader.net/corbin,0.3836002490,44.120.13.69,"{""location"": ""BT"", ""is_mobile"": true}" 14769,5,333,2017-04-11 06:19:10,http://lind.info/leanna,0.2151647470,37.212.25.100,"{""location"": ""DK"", ""is_mobile"": false}" 14770,5,333,2017-06-05 09:30:36,http://lemkekutch.info/lindsay.hane,0.4226778667,18.85.35.138,"{""location"": ""FJ"", ""is_mobile"": true}" 14771,5,333,2017-05-29 18:04:46,http://bogisich.com/noah_hagenes,0.0037660155,105.85.244.34,"{""location"": ""MH"", ""is_mobile"": false}" 14772,5,333,2017-05-25 03:41:34,http://windler.org/napoleon,0.5209924730,133.147.52.11,"{""location"": ""MK"", ""is_mobile"": false}" 14773,5,333,2017-03-25 14:08:07,http://kochmoore.net/santiago.harber,0.9788932245,16.171.192.72,"{""location"": ""GQ"", ""is_mobile"": true}" 14774,5,333,2017-03-11 18:27:08,http://danieltromp.co/koby.jakubowski,0.7570235082,207.120.129.221,"{""location"": ""KR"", ""is_mobile"": false}" 14775,5,333,2016-12-31 02:58:51,http://mckenzielang.net/kaylin,0.0518388555,113.93.189.103,"{""location"": ""SY"", ""is_mobile"": false}" 14776,5,333,2017-01-23 05:24:14,http://kovacek.net/abdul.hudson,0.9852601542,135.139.12.4,"{""location"": ""GG"", ""is_mobile"": true}" 14777,5,333,2017-03-30 22:02:44,http://conroymuller.io/jevon.moore,0.3566116725,250.18.198.254,"{""location"": ""IQ"", ""is_mobile"": true}" 14778,5,333,2017-06-04 23:50:21,http://smith.co/rozella,0.0460518910,96.173.131.44,"{""location"": ""TF"", ""is_mobile"": true}" 14779,5,333,2016-12-28 05:20:12,http://waelchi.io/aurelia.ratke,0.0207261167,68.43.186.221,"{""location"": ""JP"", ""is_mobile"": true}" 14780,5,333,2017-01-27 08:51:21,http://upton.com/danny,0.0967959936,29.189.35.213,"{""location"": ""FM"", ""is_mobile"": true}" 14781,5,333,2017-06-13 13:21:50,http://sawayn.biz/wilma,0.2386106715,50.229.229.20,"{""location"": ""US"", ""is_mobile"": true}" 14782,5,333,2016-12-20 13:02:45,http://zulauf.org/noemi_vandervort,0.7153790696,184.214.121.59,"{""location"": ""KP"", ""is_mobile"": true}" 14783,5,333,2017-04-27 01:07:46,http://kuvalisbahringer.biz/leanne,0.5575310801,14.119.162.42,"{""location"": ""EG"", ""is_mobile"": false}" 14784,5,333,2017-02-27 18:28:09,http://feestcummerata.biz/meredith,0.2924126158,147.100.77.151,"{""location"": ""ME"", ""is_mobile"": false}" 14785,5,333,2017-02-25 12:59:23,http://bauchstiedemann.name/mustafa,0.3821085634,245.141.137.32,"{""location"": ""CD"", ""is_mobile"": true}" 14786,5,333,2017-02-27 17:59:58,http://reichel.co/annetta.kuphal,0.7414083995,91.189.193.223,"{""location"": ""TG"", ""is_mobile"": true}" 14787,5,333,2017-05-19 12:18:12,http://price.com/katherine,0.8605903582,50.144.221.180,"{""location"": ""BW"", ""is_mobile"": false}" 14788,5,333,2017-04-02 04:28:25,http://aufderhar.com/isabella_kuhic,0.6096960457,240.196.199.100,"{""location"": ""PW"", ""is_mobile"": false}" 14789,5,333,2017-05-04 10:06:08,http://funk.io/jeromy,0.7788973814,24.184.140.205,"{""location"": ""MP"", ""is_mobile"": false}" 14790,5,333,2017-02-03 13:20:15,http://corwin.com/maddison_bayer,0.4826985890,101.168.132.200,"{""location"": ""CK"", ""is_mobile"": true}" 14791,5,333,2017-01-30 11:33:31,http://emard.info/devonte.weinat,0.7773630482,104.214.82.115,"{""location"": ""PT"", ""is_mobile"": false}" 14792,5,333,2017-05-28 12:26:44,http://orn.info/jacques,0.4559780094,137.157.18.105,"{""location"": ""EE"", ""is_mobile"": true}" 14793,5,333,2016-12-21 05:19:00,http://metzwill.biz/gwendolyn,0.4013447100,212.76.247.86,"{""location"": ""BV"", ""is_mobile"": true}" 14794,5,333,2017-04-25 22:55:20,http://boscocain.org/giovani,0.8530469523,31.114.203.209,"{""location"": ""EH"", ""is_mobile"": false}" 14795,5,333,2017-04-17 12:12:23,http://crooks.biz/tillman,0.0509182051,130.127.173.61,"{""location"": ""LC"", ""is_mobile"": true}" 14796,5,333,2017-01-25 21:39:58,http://bechtelar.biz/mina,0.9136161018,248.171.118.135,"{""location"": ""NP"", ""is_mobile"": false}" 14797,5,333,2017-04-29 12:44:06,http://jakubowski.co/jose.lueilwitz,0.9581178118,85.42.120.238,"{""location"": ""MY"", ""is_mobile"": true}" 14798,5,333,2017-02-17 20:19:43,http://schowaltercarter.org/katlynn_schaden,0.9526559837,184.36.216.134,"{""location"": ""GE"", ""is_mobile"": false}" 14799,5,333,2017-04-09 03:49:22,http://balistreribrakus.name/ashleigh.willms,0.5117037104,244.197.181.75,"{""location"": ""LV"", ""is_mobile"": false}" 14800,5,333,2017-06-08 14:01:18,http://emmerich.io/luna_heaney,0.7630292558,32.189.43.141,"{""location"": ""IS"", ""is_mobile"": false}" 14801,5,333,2017-03-26 08:09:37,http://osinski.biz/gustave_heathcote,0.6506484451,157.108.201.105,"{""location"": ""KZ"", ""is_mobile"": false}" 14802,5,333,2017-03-05 23:49:01,http://rennerjohns.info/kole_oconner,0.7114171727,87.205.253.245,"{""location"": ""YE"", ""is_mobile"": false}" 14803,5,333,2017-02-18 04:19:28,http://prohaska.name/ladarius_goldner,0.9320423016,45.193.206.106,"{""location"": ""SS"", ""is_mobile"": true}" 14804,5,333,2017-05-12 14:48:16,http://upton.com/arden_altenwerth,0.8576073391,207.216.91.177,"{""location"": ""VU"", ""is_mobile"": false}" 17743,6,400,2017-04-27 07:58:23,http://nader.name/bruce,0.3003676385,194.109.224.58,"{""location"": ""US"", ""is_mobile"": true}" 17744,6,400,2017-04-08 23:20:07,http://lubowitz.info/jarrett,0.8507836526,200.168.56.221,"{""location"": ""TL"", ""is_mobile"": true}" 17745,6,400,2017-06-07 02:07:40,http://leannon.name/doug_kirlin,0.9505358963,26.121.199.238,"{""location"": ""SI"", ""is_mobile"": false}" 17746,6,400,2017-03-21 04:16:16,http://oharakeler.info/ford,0.8279375320,244.200.17.135,"{""location"": ""NU"", ""is_mobile"": false}" 17747,6,400,2017-01-12 04:56:17,http://walker.com/bailey.sanford,0.4780100794,129.126.44.67,"{""location"": ""EC"", ""is_mobile"": true}" 17748,6,400,2016-12-29 18:13:26,http://treutel.co/zaria.terry,0.5467226349,89.56.95.111,"{""location"": ""SE"", ""is_mobile"": false}" 17749,6,400,2017-01-29 15:19:39,http://muellerdonnelly.net/treva_greenholt,0.8703019439,130.199.143.120,"{""location"": ""UY"", ""is_mobile"": false}" 17750,6,400,2017-02-18 01:13:35,http://altenwerthko.com/chester.cartwright,0.2813976903,39.212.242.12,"{""location"": ""MC"", ""is_mobile"": false}" 17751,6,400,2017-01-14 13:11:20,http://rolfsonchamplin.info/daron_bauch,0.9605289256,248.71.161.172,"{""location"": ""PG"", ""is_mobile"": true}" 17752,6,400,2017-03-25 10:21:48,http://cummingsrippin.net/mina_little,0.4951174532,86.96.198.53,"{""location"": ""FR"", ""is_mobile"": true}" 17753,6,400,2017-06-13 13:18:56,http://sengerrath.info/alena_pfannerstill,0.5866918858,244.59.12.15,"{""location"": ""KG"", ""is_mobile"": false}" 17754,6,400,2017-04-22 12:21:22,http://kuboreilly.name/weldon,0.0925881710,184.169.139.96,"{""location"": ""BM"", ""is_mobile"": false}" 17755,6,400,2017-02-11 01:29:57,http://ornrosenbaum.info/margarett,0.1819895889,51.181.53.184,"{""location"": ""AE"", ""is_mobile"": true}" 17756,6,400,2017-04-04 12:06:08,http://goodwin.info/francisca_jerde,0.2685873593,96.77.157.53,"{""location"": ""CY"", ""is_mobile"": false}" 17757,6,400,2017-05-30 06:31:25,http://raynorgreenholt.org/ahmad_damore,0.5249470828,98.177.96.242,"{""location"": ""EE"", ""is_mobile"": false}" 17758,6,400,2017-01-01 02:44:05,http://welch.net/leonora.runolfon,0.7585016104,52.41.149.15,"{""location"": ""LU"", ""is_mobile"": true}" 17759,6,400,2017-06-08 18:50:41,http://fisher.net/mayra,0.3527328745,47.205.40.106,"{""location"": ""NP"", ""is_mobile"": false}" 17760,6,400,2017-02-08 20:43:54,http://stracke.com/sigrid_johnson,0.4545747213,162.46.173.6,"{""location"": ""JP"", ""is_mobile"": true}" 17761,6,400,2017-05-29 19:04:19,http://hilpert.io/carter,0.4308062028,128.176.35.221,"{""location"": ""GT"", ""is_mobile"": false}" 17762,6,400,2017-02-07 21:31:01,http://frami.com/nikolas_brekke,0.2367161771,122.157.106.84,"{""location"": ""CX"", ""is_mobile"": true}" 17763,6,401,2017-04-24 18:57:04,http://wyman.info/emery_pollich,0.5546329172,108.219.66.203,"{""location"": ""BD"", ""is_mobile"": true}" 17764,6,401,2017-01-24 20:18:18,http://halvorson.net/emma_dubuque,0.1738263348,107.73.84.183,"{""location"": ""SS"", ""is_mobile"": false}" 17765,6,401,2017-01-01 10:47:31,http://von.com/ulises_thiel,0.1254377386,5.185.83.189,"{""location"": ""PL"", ""is_mobile"": true}" 17766,6,401,2017-03-28 20:02:28,http://herzogsporer.org/jeffery,0.0601743306,8.154.252.67,"{""location"": ""FK"", ""is_mobile"": true}" 17767,6,401,2017-01-17 17:40:56,http://wittingsmitham.com/mark.pfeffer,0.1831998726,78.170.75.215,"{""location"": ""CO"", ""is_mobile"": false}" 17768,6,401,2017-05-17 14:01:16,http://willkuhn.info/angelo,0.6452071309,55.127.226.44,"{""location"": ""PK"", ""is_mobile"": false}" 17769,6,401,2017-03-18 17:42:02,http://cruickshank.co/sophia,0.4894272524,212.22.121.168,"{""location"": ""TC"", ""is_mobile"": true}" 17770,6,401,2017-02-22 13:03:14,http://lockman.info/rickie.bergnaum,0.4483870541,154.75.57.21,"{""location"": ""NO"", ""is_mobile"": false}" 17771,6,401,2017-06-09 07:47:10,http://zulauf.co/isabella.fisher,0.9687829019,237.90.15.6,"{""location"": ""AD"", ""is_mobile"": false}" 17772,6,401,2017-04-18 01:49:30,http://emard.name/pierce_konopelski,0.8060151489,134.52.44.135,"{""location"": ""LB"", ""is_mobile"": false}" 17773,6,401,2017-06-07 11:48:32,http://hayesdickinson.co/antonia,0.6378363851,2.131.189.190,"{""location"": ""SE"", ""is_mobile"": false}" 17774,6,401,2017-03-31 05:03:24,http://millsbrown.net/stephen,0.9547896807,51.214.117.239,"{""location"": ""HT"", ""is_mobile"": true}" 17775,6,401,2017-03-05 12:14:44,http://adamsgutkowski.info/brandon,0.2213598236,115.22.128.216,"{""location"": ""DZ"", ""is_mobile"": false}" 17776,6,401,2017-06-07 20:50:57,http://conn.co/breana,0.1111170384,56.8.135.36,"{""location"": ""KY"", ""is_mobile"": true}" 17777,6,401,2016-12-30 02:42:13,http://schmitt.com/carmen_gleichner,0.1571452230,235.136.145.89,"{""location"": ""GW"", ""is_mobile"": false}" 17778,6,401,2017-02-08 06:18:43,http://kihn.biz/gordon,0.0813262319,121.221.29.48,"{""location"": ""ST"", ""is_mobile"": true}" 17779,6,401,2017-05-31 17:03:20,http://stroman.org/rowland,0.1802823323,212.29.20.228,"{""location"": ""CA"", ""is_mobile"": false}" 17780,6,401,2017-01-14 03:53:11,http://schimmel.io/joey_crist,0.6316644878,55.5.73.240,"{""location"": ""QA"", ""is_mobile"": true}" 17781,6,401,2017-02-08 08:54:52,http://nicolas.name/giuseppe_pfannerstill,0.4368183312,164.168.83.123,"{""location"": ""UG"", ""is_mobile"": true}" 17782,6,401,2017-02-22 17:37:00,http://rosenbaum.info/asia_konopelski,0.6610643288,139.233.234.201,"{""location"": ""PY"", ""is_mobile"": false}" 17783,6,401,2017-04-21 02:31:59,http://brownfritsch.net/celine,0.2342171793,59.28.36.74,"{""location"": ""GF"", ""is_mobile"": false}" 17784,6,401,2017-03-29 13:11:16,http://towneschaden.biz/nash,0.5031444857,11.208.161.244,"{""location"": ""UY"", ""is_mobile"": true}" 17785,6,401,2017-01-04 08:58:06,http://halvorson.name/janice_king,0.3999343641,57.89.175.68,"{""location"": ""PK"", ""is_mobile"": true}" 17786,6,401,2017-02-20 03:22:27,http://dach.co/fredrick_little,0.5738276300,135.20.249.64,"{""location"": ""YT"", ""is_mobile"": true}" 17787,6,401,2017-03-15 16:43:21,http://walter.info/estella_nikolaus,0.9257807631,242.40.96.161,"{""location"": ""MU"", ""is_mobile"": false}" 17788,6,401,2017-01-12 03:40:58,http://cruickshank.biz/nia,0.5841908217,123.110.149.230,"{""location"": ""KR"", ""is_mobile"": false}" 17789,6,401,2017-02-03 20:00:14,http://mitchellohara.biz/berniece.predovic,0.2684177130,113.133.96.131,"{""location"": ""IM"", ""is_mobile"": true}" 17790,6,401,2017-04-26 22:35:39,http://thompsoncollins.com/juston,0.7855468969,12.25.19.239,"{""location"": ""FO"", ""is_mobile"": true}" 17791,6,401,2017-06-09 19:29:29,http://ruel.biz/xavier_fritsch,0.5996875793,43.10.172.9,"{""location"": ""HU"", ""is_mobile"": true}" 17792,6,401,2016-12-28 19:43:54,http://tromp.co/christina,0.3220469112,64.200.233.96,"{""location"": ""NL"", ""is_mobile"": false}" 17793,6,401,2017-04-13 02:26:34,http://rogahn.name/kurt.zboncak,0.6836646779,66.4.236.10,"{""location"": ""SB"", ""is_mobile"": true}" 4948,2,109,2017-06-09 22:12:24,http://ziemann.org/carlotta,,143.19.233.5,"{""location"": ""PL"", ""is_mobile"": false}" 4949,2,109,2017-05-17 18:42:21,http://dickens.org/vivienne,,109.219.155.217,"{""location"": ""JO"", ""is_mobile"": true}" 4950,2,109,2017-01-12 06:34:28,http://mclaughlinhintz.name/rosie,,167.124.247.178,"{""location"": ""JM"", ""is_mobile"": false}" 4951,2,109,2017-03-23 18:34:40,http://quigleyschmidt.net/jordane_hermann,,102.171.12.159,"{""location"": ""AI"", ""is_mobile"": true}" 4952,2,109,2017-01-10 19:55:33,http://ruecker.biz/tanya,,140.239.36.98,"{""location"": ""WF"", ""is_mobile"": true}" 4953,2,109,2016-12-19 18:57:57,http://swift.co/roel,,221.33.219.79,"{""location"": ""IL"", ""is_mobile"": false}" 4954,2,109,2017-01-27 06:35:26,http://turner.net/mose.treutel,,80.30.191.128,"{""location"": ""KR"", ""is_mobile"": true}" 4955,2,109,2017-03-22 19:55:33,http://larson.info/kelsi,,155.158.55.212,"{""location"": ""KH"", ""is_mobile"": false}" 4956,2,109,2016-12-20 17:56:24,http://wintheisermckenzie.com/henri,,61.18.136.96,"{""location"": ""ER"", ""is_mobile"": false}" 4957,2,109,2017-05-24 15:55:15,http://turcotte.biz/emelie,,172.202.28.41,"{""location"": ""US"", ""is_mobile"": false}" 4958,2,109,2017-03-09 20:16:05,http://casper.net/meggie,,212.112.4.96,"{""location"": ""FJ"", ""is_mobile"": false}" 4959,2,109,2017-05-09 16:13:22,http://haag.co/enid,,205.188.110.25,"{""location"": ""NF"", ""is_mobile"": false}" 4960,2,109,2017-02-01 05:43:56,http://turner.co/ralph.wuckert,,56.79.109.238,"{""location"": ""TH"", ""is_mobile"": true}" 4961,2,109,2016-12-29 03:43:11,http://schoenemmerich.org/joshuah,,109.236.133.147,"{""location"": ""DE"", ""is_mobile"": false}" 4962,2,109,2017-03-20 04:45:06,http://lemke.com/guadalupe,,123.254.92.198,"{""location"": ""NF"", ""is_mobile"": true}" 4963,2,109,2017-01-31 13:49:13,http://goodwin.io/antonina,,233.147.78.99,"{""location"": ""AI"", ""is_mobile"": true}" 4964,2,109,2017-02-22 22:50:51,http://zboncakhuels.co/marshall,,207.92.80.91,"{""location"": ""SY"", ""is_mobile"": false}" 4965,2,109,2017-04-03 16:28:55,http://bartoletti.biz/sanford.zulauf,,230.188.210.212,"{""location"": ""SY"", ""is_mobile"": true}" 4966,2,109,2017-06-09 04:29:50,http://mclaughlinmoen.com/anabel.johnston,,21.42.164.20,"{""location"": ""HM"", ""is_mobile"": false}" 4967,2,109,2017-05-11 02:10:35,http://connelly.co/jasmin.predovic,,228.117.232.137,"{""location"": ""ST"", ""is_mobile"": false}" 4968,2,110,2017-05-20 05:14:39,http://hoppe.io/jordy,,168.154.20.136,"{""location"": ""SB"", ""is_mobile"": true}" 4969,2,110,2017-01-24 03:19:03,http://brekkeabernathy.io/victor,,166.108.231.88,"{""location"": ""RW"", ""is_mobile"": true}" 4970,2,110,2017-04-09 19:46:35,http://hagenesturner.net/adrianna_keler,,202.131.110.33,"{""location"": ""NI"", ""is_mobile"": false}" 4971,2,110,2017-01-31 03:42:22,http://wiegandcole.io/jake,,150.147.135.200,"{""location"": ""TK"", ""is_mobile"": true}" 4972,2,110,2017-02-28 13:25:13,http://coleeichmann.info/sally,,105.37.107.23,"{""location"": ""TZ"", ""is_mobile"": true}" 4973,2,110,2017-01-19 17:04:37,http://kozeyryan.info/dusty_beatty,,195.156.122.250,"{""location"": ""MF"", ""is_mobile"": false}" 4974,2,110,2017-05-23 04:44:47,http://gusikowskimedhurst.biz/ariel,,51.98.222.45,"{""location"": ""SK"", ""is_mobile"": false}" 4975,2,110,2017-06-01 00:41:29,http://batz.net/cooper,,172.227.89.211,"{""location"": ""KY"", ""is_mobile"": true}" 4976,2,110,2017-05-31 00:22:10,http://treutel.io/toni,,147.13.127.134,"{""location"": ""JM"", ""is_mobile"": false}" 4977,2,110,2017-03-06 23:44:54,http://gottliebmosciski.net/marcelle.bayer,,53.229.12.252,"{""location"": ""TT"", ""is_mobile"": true}" 4978,2,110,2017-04-05 03:23:57,http://heathcote.biz/neil,,176.3.131.10,"{""location"": ""EH"", ""is_mobile"": true}" 4979,2,110,2017-03-07 22:18:18,http://conroy.biz/ellen.goldner,,112.72.87.98,"{""location"": ""SR"", ""is_mobile"": true}" 4980,2,110,2017-06-07 09:46:42,http://reilly.net/alexys_tromp,,15.16.23.54,"{""location"": ""ES"", ""is_mobile"": false}" 4981,2,110,2017-01-15 09:16:56,http://rathprosacco.net/javon,,145.91.117.182,"{""location"": ""ZA"", ""is_mobile"": true}" 4982,2,110,2017-04-26 09:29:03,http://harber.com/coty_rogahn,,2.101.213.2,"{""location"": ""MY"", ""is_mobile"": true}" 4983,2,110,2017-01-24 14:19:37,http://lehner.io/travis.balistreri,,76.143.172.140,"{""location"": ""BJ"", ""is_mobile"": true}" 4984,2,110,2017-03-20 05:18:12,http://roobokeefe.org/casimir,,18.107.200.153,"{""location"": ""PL"", ""is_mobile"": false}" 4985,2,110,2017-06-07 03:26:47,http://ebert.io/greyson,,22.110.67.173,"{""location"": ""SB"", ""is_mobile"": false}" 4986,2,110,2017-06-04 20:50:03,http://walsh.info/gilbert,,174.83.95.241,"{""location"": ""SH"", ""is_mobile"": false}" 4987,2,110,2016-12-20 17:16:07,http://waters.co/bethel_leffler,,248.72.117.4,"{""location"": ""AO"", ""is_mobile"": false}" 4988,2,110,2017-05-25 11:23:02,http://wardhahn.co/marilie,,227.73.236.166,"{""location"": ""HN"", ""is_mobile"": true}" 4989,2,110,2016-12-25 16:28:48,http://durgan.org/clair,,226.80.81.120,"{""location"": ""GG"", ""is_mobile"": false}" 4990,2,110,2017-03-13 08:04:13,http://kerluke.info/valentin,,115.190.18.162,"{""location"": ""US"", ""is_mobile"": false}" 4991,2,110,2016-12-20 22:41:46,http://bechtelar.io/araceli,,25.74.221.10,"{""location"": ""LY"", ""is_mobile"": false}" 4992,2,110,2017-03-24 12:25:21,http://vonruedenborer.biz/samson,,196.113.113.59,"{""location"": ""SK"", ""is_mobile"": true}" 4993,2,110,2017-04-27 20:05:49,http://simonis.biz/gerard_cormier,,56.74.181.71,"{""location"": ""RW"", ""is_mobile"": false}" 4994,2,110,2016-12-24 12:32:57,http://ondricka.org/alvena,,97.113.184.149,"{""location"": ""NA"", ""is_mobile"": false}" 4995,2,110,2017-05-10 17:12:05,http://pfannerstill.name/demond,,142.100.208.189,"{""location"": ""CU"", ""is_mobile"": false}" 4996,2,110,2017-02-08 03:57:48,http://stehrjerde.name/cheyanne.purdy,,193.130.25.235,"{""location"": ""WS"", ""is_mobile"": false}" 4997,2,110,2017-05-31 06:41:07,http://oreilly.name/lionel,,119.91.95.155,"{""location"": ""BB"", ""is_mobile"": false}" 4998,2,110,2017-01-12 11:30:59,http://legros.co/trevor.jaskolski,,51.13.222.216,"{""location"": ""SS"", ""is_mobile"": true}" 4999,2,111,2017-06-06 16:04:26,http://harrisleannon.co/name_gleichner,,184.222.170.68,"{""location"": ""IS"", ""is_mobile"": true}" 5000,2,111,2017-05-29 23:11:52,http://stiedemannquitzon.biz/margie,,128.49.186.77,"{""location"": ""HU"", ""is_mobile"": false}" 5001,2,111,2017-05-05 20:39:35,http://ruel.biz/maye,,94.194.153.170,"{""location"": ""TV"", ""is_mobile"": false}" 5002,2,111,2016-12-29 03:21:59,http://beahanoberbrunner.co/evelyn.jenkins,,167.32.188.67,"{""location"": ""GB"", ""is_mobile"": false}" 5003,2,111,2017-05-02 09:34:13,http://hoppe.info/verdie_johnston,,204.142.56.177,"{""location"": ""ZW"", ""is_mobile"": false}" 17932,7,405,2017-02-13 09:30:12,http://boyer.io/loraine_hyatt,,245.142.18.150,"{""location"": ""GP"", ""is_mobile"": false}" 17933,7,405,2017-01-05 15:23:22,http://dibbertterry.org/elinore.hudson,,129.10.115.43,"{""location"": ""ML"", ""is_mobile"": false}" 17934,7,405,2017-01-11 01:54:14,http://bartoletti.com/dorcas,,27.123.158.220,"{""location"": ""KY"", ""is_mobile"": false}" 17935,7,405,2017-05-01 14:29:00,http://keeblerratke.com/titus,,32.83.224.145,"{""location"": ""TW"", ""is_mobile"": true}" 17936,7,405,2016-12-16 16:34:26,http://conn.co/josue.ledner,,158.6.46.191,"{""location"": ""TV"", ""is_mobile"": true}" 17937,7,405,2017-01-08 22:54:58,http://johns.info/chasity,,49.249.131.208,"{""location"": ""JE"", ""is_mobile"": false}" 17938,7,405,2017-04-22 23:08:10,http://rohan.biz/ora,,199.254.85.55,"{""location"": ""NZ"", ""is_mobile"": true}" 17939,7,405,2017-04-29 21:59:34,http://breitenberg.co/else,,115.50.28.174,"{""location"": ""CN"", ""is_mobile"": true}" 17940,7,405,2017-04-27 20:16:51,http://lemkelowe.info/charley_walter,,37.62.108.202,"{""location"": ""RS"", ""is_mobile"": true}" 17941,7,406,2017-06-04 17:33:29,http://carroll.co/jee,,244.103.155.237,"{""location"": ""RS"", ""is_mobile"": true}" 17942,7,406,2017-05-30 07:59:26,http://ornjenkins.info/mauricio,,82.229.199.119,"{""location"": ""MY"", ""is_mobile"": true}" 17943,7,406,2017-06-12 09:22:22,http://predovic.com/micaela,,120.77.177.12,"{""location"": ""TZ"", ""is_mobile"": false}" 17944,7,406,2017-02-22 13:24:54,http://keeblerborer.info/enrique,,178.40.32.158,"{""location"": ""BJ"", ""is_mobile"": false}" 17945,7,406,2017-04-23 11:29:16,http://strosinokuneva.biz/jefferey,,103.135.20.35,"{""location"": ""GS"", ""is_mobile"": false}" 17946,7,406,2017-04-16 17:43:41,http://bogankeeling.com/mabel.sawayn,,37.236.39.142,"{""location"": ""CR"", ""is_mobile"": true}" 17947,7,406,2017-06-01 08:01:07,http://mante.name/wilmer.spinka,,125.180.129.124,"{""location"": ""SN"", ""is_mobile"": true}" 17948,7,406,2017-02-07 18:57:49,http://williamson.co/petra.keler,,136.186.251.79,"{""location"": ""AM"", ""is_mobile"": true}" 17949,7,406,2017-06-12 17:53:33,http://bartoletti.co/roberta_kozey,,54.210.65.24,"{""location"": ""AT"", ""is_mobile"": false}" 17950,7,406,2017-05-22 16:54:21,http://ullrich.com/emilio.weber,,190.162.214.157,"{""location"": ""CC"", ""is_mobile"": false}" 17951,7,406,2017-02-12 23:36:24,http://bruen.com/chandler_feeney,,37.111.151.183,"{""location"": ""CV"", ""is_mobile"": true}" 17952,7,406,2017-02-03 10:04:41,http://leuschke.org/celine,,17.178.172.133,"{""location"": ""SN"", ""is_mobile"": true}" 17953,7,406,2017-01-21 09:28:12,http://lynchkoch.co/delaney.renner,,229.40.182.27,"{""location"": ""AU"", ""is_mobile"": true}" 17954,7,406,2017-01-22 14:50:52,http://hagenes.biz/reid,,243.99.30.230,"{""location"": ""SG"", ""is_mobile"": true}" 17955,7,406,2017-02-11 09:56:25,http://ward.io/caandra_deckow,,48.174.233.34,"{""location"": ""MH"", ""is_mobile"": true}" 17956,7,406,2017-03-01 01:09:51,http://ferrylind.name/ella,,57.88.60.122,"{""location"": ""TH"", ""is_mobile"": true}" 17957,7,406,2017-05-23 07:00:27,http://walshwilkinson.net/karine,,11.24.94.48,"{""location"": ""KI"", ""is_mobile"": true}" 17958,7,406,2017-02-18 20:17:06,http://osinskireichel.org/albin_schultz,,179.240.160.161,"{""location"": ""RE"", ""is_mobile"": true}" 17959,7,406,2016-12-21 15:24:03,http://prosaccowintheiser.io/javonte,,77.10.223.24,"{""location"": ""BO"", ""is_mobile"": false}" 17960,7,406,2017-03-05 10:51:57,http://grantheathcote.com/lisandro,,128.184.108.234,"{""location"": ""WS"", ""is_mobile"": true}" 17961,7,406,2017-05-27 05:06:45,http://glover.name/dereck.pfannerstill,,126.149.107.70,"{""location"": ""RE"", ""is_mobile"": true}" 17962,7,406,2016-12-20 15:27:58,http://kreiger.io/mack,,103.187.87.252,"{""location"": ""IQ"", ""is_mobile"": false}" 17963,7,406,2017-02-09 12:26:47,http://hicklebailey.name/lesly,,97.43.143.189,"{""location"": ""NA"", ""is_mobile"": false}" 17964,7,406,2017-05-17 12:15:00,http://donnelly.org/louie,,222.46.18.191,"{""location"": ""PY"", ""is_mobile"": true}" 17965,7,406,2017-03-22 16:25:02,http://senger.co/johnson_brakus,,152.11.101.17,"{""location"": ""KR"", ""is_mobile"": false}" 17966,7,406,2017-06-03 14:11:44,http://bayerbogisich.name/wade.mayer,,116.151.109.86,"{""location"": ""IL"", ""is_mobile"": true}" 17967,7,406,2017-05-15 14:19:18,http://barrows.info/clemens,,46.122.43.36,"{""location"": ""KP"", ""is_mobile"": false}" 17968,7,406,2017-06-13 23:52:59,http://haley.org/arch,,75.94.174.49,"{""location"": ""IR"", ""is_mobile"": false}" 17969,7,406,2017-06-12 03:00:09,http://schmitt.com/garett,,216.34.141.27,"{""location"": ""NP"", ""is_mobile"": false}" 17970,7,406,2017-02-07 23:29:03,http://romagueraabernathy.org/sadie.osinski,,103.120.221.150,"{""location"": ""SR"", ""is_mobile"": false}" 17971,7,406,2017-01-20 06:44:59,http://kuhlman.io/sigrid.haley,,71.196.198.3,"{""location"": ""GU"", ""is_mobile"": false}" 17972,7,406,2016-12-23 20:36:43,http://langoshkemmer.io/armando,,181.5.222.30,"{""location"": ""VG"", ""is_mobile"": true}" 17973,7,406,2017-04-26 17:04:43,http://rice.net/brittany_heller,,15.222.57.57,"{""location"": ""AE"", ""is_mobile"": false}" 17974,7,406,2017-03-24 06:52:54,http://weberlang.io/lou_keebler,,79.100.60.20,"{""location"": ""TM"", ""is_mobile"": false}" 17975,7,406,2017-04-05 00:45:28,http://bernier.io/nathen.bailey,,228.214.149.206,"{""location"": ""LS"", ""is_mobile"": true}" 17976,7,407,2017-01-19 10:39:13,http://okeefecollins.info/evelyn,,212.185.63.153,"{""location"": ""SN"", ""is_mobile"": true}" 17977,7,407,2016-12-27 09:02:03,http://nitzschekeeling.info/cruz_kiehn,,245.106.124.240,"{""location"": ""MC"", ""is_mobile"": true}" 17978,7,407,2017-02-16 10:29:17,http://kautzer.name/guillermo,,185.106.84.72,"{""location"": ""TH"", ""is_mobile"": false}" 17979,7,407,2017-04-06 09:11:57,http://zulauf.biz/darrel_bernhard,,70.153.42.119,"{""location"": ""TF"", ""is_mobile"": true}" 17980,7,407,2017-04-30 18:27:42,http://harvey.biz/micaela_hyatt,,242.3.58.204,"{""location"": ""AL"", ""is_mobile"": false}" 17981,7,407,2017-05-26 02:52:16,http://aufderhar.name/dax,,50.82.184.112,"{""location"": ""BG"", ""is_mobile"": true}" 17982,7,407,2017-01-04 02:10:38,http://jakubowski.info/giovanna_kovacek,,250.137.218.215,"{""location"": ""SN"", ""is_mobile"": false}" 17983,7,407,2017-05-23 21:30:31,http://schuppekonopelski.info/josefina,,132.111.203.177,"{""location"": ""OM"", ""is_mobile"": true}" 17984,7,407,2017-06-10 23:42:15,http://cain.name/garett,,248.67.23.174,"{""location"": ""SJ"", ""is_mobile"": true}" 17985,7,407,2017-05-05 08:39:03,http://deckowhirthe.biz/johann,,123.179.228.183,"{""location"": ""BM"", ""is_mobile"": false}" 17986,7,407,2017-03-18 09:08:29,http://kuhic.net/kenyatta,,92.77.147.24,"{""location"": ""AS"", ""is_mobile"": false}" 14805,5,333,2017-04-06 16:27:40,http://goyettelangosh.org/brenda,0.8949632230,163.84.95.34,"{""location"": ""RW"", ""is_mobile"": true}" 14806,5,333,2017-01-14 19:01:08,http://ortizschaefer.io/jordy,0.0110467469,50.244.139.197,"{""location"": ""LV"", ""is_mobile"": true}" 14807,5,333,2017-06-07 18:52:42,http://swift.biz/wendy,0.0789015432,69.140.179.241,"{""location"": ""AD"", ""is_mobile"": false}" 14808,5,333,2017-06-13 19:20:10,http://murazik.info/nelda,0.2228829319,213.189.157.111,"{""location"": ""ES"", ""is_mobile"": true}" 14809,5,333,2017-03-20 00:36:51,http://fay.info/yazmin.jacobi,0.2950804805,209.22.209.155,"{""location"": ""EG"", ""is_mobile"": false}" 14810,5,333,2017-01-04 21:29:03,http://mcdermott.name/parker_douglas,0.3266341490,217.5.91.244,"{""location"": ""TT"", ""is_mobile"": false}" 14811,5,333,2017-01-13 11:10:39,http://thiel.info/javier,0.5767643664,145.155.56.143,"{""location"": ""HK"", ""is_mobile"": false}" 14812,5,333,2017-05-03 07:08:59,http://toy.co/muhammad,0.7387387242,77.226.89.93,"{""location"": ""JO"", ""is_mobile"": true}" 14813,5,333,2017-05-23 04:36:56,http://ruecker.info/katlyn,0.0520191437,150.179.108.18,"{""location"": ""JM"", ""is_mobile"": false}" 14814,5,333,2017-03-06 09:20:01,http://wildermanleannon.info/sonya,0.3097641996,94.44.211.212,"{""location"": ""DJ"", ""is_mobile"": true}" 14815,5,334,2017-04-29 07:18:10,http://haagmckenzie.io/mollie.nicolas,0.9263457139,43.189.95.77,"{""location"": ""LY"", ""is_mobile"": true}" 14816,5,334,2017-03-04 07:16:41,http://haagko.io/myrna_roob,0.1164561007,181.16.193.236,"{""location"": ""RO"", ""is_mobile"": false}" 14817,5,334,2017-05-06 20:06:08,http://orn.info/marina.stracke,0.6463184422,231.230.75.97,"{""location"": ""KM"", ""is_mobile"": false}" 14818,5,334,2017-01-02 20:32:58,http://ernserharber.io/juston,0.0195307896,231.17.151.109,"{""location"": ""UZ"", ""is_mobile"": true}" 14819,5,334,2016-12-29 00:44:12,http://abernathywolff.info/charity_koch,0.5188536161,253.145.89.21,"{""location"": ""SD"", ""is_mobile"": true}" 14820,5,334,2017-04-19 08:09:22,http://parisianmann.io/ayla.spencer,0.7902000115,11.85.204.213,"{""location"": ""LV"", ""is_mobile"": true}" 14821,5,334,2017-06-06 21:15:34,http://howe.info/myrtie,0.6716637325,163.77.34.206,"{""location"": ""ML"", ""is_mobile"": false}" 14822,5,334,2017-03-12 16:05:44,http://deckow.net/richmond.moore,0.4266917538,248.216.44.124,"{""location"": ""TN"", ""is_mobile"": true}" 14823,5,334,2017-01-25 06:05:40,http://zieme.net/ardith_crist,0.6683800024,179.6.43.201,"{""location"": ""DM"", ""is_mobile"": false}" 14824,5,334,2016-12-25 07:00:11,http://hermiston.info/ike,0.1696066291,190.137.74.239,"{""location"": ""SL"", ""is_mobile"": true}" 14825,5,334,2017-03-31 22:21:15,http://bahringerparisian.co/jensen,0.2092787585,8.67.4.222,"{""location"": ""TJ"", ""is_mobile"": false}" 14826,5,334,2017-03-07 01:06:19,http://hirtheoconner.org/eric_vonrueden,0.8809697345,168.203.249.241,"{""location"": ""CD"", ""is_mobile"": false}" 14827,5,334,2017-05-03 19:02:06,http://padberg.co/hoyt_klocko,0.7817733038,250.115.190.203,"{""location"": ""KZ"", ""is_mobile"": false}" 14828,5,334,2017-03-05 14:36:16,http://dicki.name/cole_homenick,0.5319859108,14.127.182.163,"{""location"": ""DE"", ""is_mobile"": true}" 14829,5,334,2017-02-20 15:33:05,http://hoppe.com/braulio.mraz,0.0023731605,164.203.251.179,"{""location"": ""BD"", ""is_mobile"": true}" 14830,5,334,2017-04-22 17:57:06,http://padberg.info/delfina.keler,0.4303281742,45.173.2.85,"{""location"": ""GR"", ""is_mobile"": false}" 14831,5,334,2016-12-14 15:51:30,http://jacobi.io/joan,0.4517379684,46.222.21.224,"{""location"": ""CC"", ""is_mobile"": true}" 14832,5,334,2017-01-27 21:41:39,http://oberbrunner.name/ernesto,0.1946967689,180.75.132.57,"{""location"": ""ID"", ""is_mobile"": false}" 14833,5,334,2017-03-15 01:02:59,http://hettinger.name/bertrand,0.7095158243,31.216.236.211,"{""location"": ""IM"", ""is_mobile"": true}" 14834,5,334,2017-05-27 20:24:15,http://dooley.co/francisca,0.0348833838,99.207.66.134,"{""location"": ""TD"", ""is_mobile"": true}" 14835,5,334,2017-01-20 17:13:40,http://miller.net/caesar,0.9152311706,227.214.31.50,"{""location"": ""YE"", ""is_mobile"": false}" 14836,5,334,2017-06-06 07:08:22,http://altenwerth.com/toney.rodriguez,0.4847720637,232.105.120.157,"{""location"": ""KW"", ""is_mobile"": false}" 14837,5,334,2017-01-13 21:26:55,http://wiza.name/levi.lehner,0.8922165978,184.59.140.100,"{""location"": ""EC"", ""is_mobile"": false}" 14838,5,334,2017-04-06 18:33:05,http://gottlieb.co/wellington,0.3041502207,167.137.220.218,"{""location"": ""AL"", ""is_mobile"": false}" 14839,5,334,2017-01-15 19:52:41,http://schumm.org/meda,0.7439553285,42.106.31.25,"{""location"": ""DM"", ""is_mobile"": true}" 14840,5,334,2017-03-07 12:27:22,http://thompson.name/rhiannon_davis,0.3539875599,195.52.211.99,"{""location"": ""JE"", ""is_mobile"": false}" 14841,5,334,2017-04-05 04:20:39,http://lemke.com/kieran,0.7833709896,19.232.132.228,"{""location"": ""YE"", ""is_mobile"": false}" 14842,5,334,2016-12-31 18:38:13,http://murphy.net/christa,0.0422766943,10.205.129.213,"{""location"": ""KR"", ""is_mobile"": true}" 14843,5,334,2017-04-02 00:57:10,http://turner.biz/adolf_nikolaus,0.3596844914,195.30.231.237,"{""location"": ""KI"", ""is_mobile"": false}" 14844,5,334,2016-12-20 13:04:00,http://cronarogahn.info/willis.littel,0.8173389743,170.178.116.2,"{""location"": ""VG"", ""is_mobile"": false}" 14845,5,334,2016-12-24 14:46:09,http://ryan.net/nolan.osinski,0.0959474252,52.222.17.72,"{""location"": ""GP"", ""is_mobile"": false}" 14846,5,334,2017-02-25 15:55:41,http://homenickhintz.com/elwyn_frami,0.9084830013,4.69.91.189,"{""location"": ""LT"", ""is_mobile"": false}" 14847,5,334,2017-05-05 03:01:11,http://hirthemaggio.info/mariana_langworth,0.8874152858,183.178.143.101,"{""location"": ""TK"", ""is_mobile"": false}" 14848,5,334,2017-06-07 09:22:44,http://erdman.info/kailee_keler,0.0066674758,181.12.199.21,"{""location"": ""KH"", ""is_mobile"": false}" 14849,5,334,2017-03-13 00:45:23,http://cronastroman.org/alfonzo.jast,0.3195654580,173.190.22.97,"{""location"": ""AW"", ""is_mobile"": false}" 14850,5,334,2016-12-22 03:35:26,http://joneshagenes.org/ernestina,0.5646248772,216.195.127.96,"{""location"": ""BR"", ""is_mobile"": false}" 14851,5,334,2017-03-30 14:48:50,http://heathcote.biz/justen,0.7844926310,230.208.99.207,"{""location"": ""MV"", ""is_mobile"": true}" 14852,5,334,2017-03-16 04:32:14,http://harris.com/horacio,0.5438370984,61.123.204.241,"{""location"": ""NF"", ""is_mobile"": true}" 14853,5,334,2017-05-29 07:30:45,http://macgyver.biz/elliott.metz,0.0481887497,149.102.182.30,"{""location"": ""AF"", ""is_mobile"": true}" 14854,5,334,2017-01-11 05:43:32,http://schillerbuckridge.net/khalid_stokes,0.5003269028,187.152.184.104,"{""location"": ""BM"", ""is_mobile"": false}" 14855,5,334,2017-04-15 17:53:14,http://lesch.io/zaria_mckenzie,0.9220832370,46.215.77.190,"{""location"": ""US"", ""is_mobile"": false}" 14856,5,334,2017-05-24 23:44:05,http://paucek.info/stevie.ko,0.9285035236,163.184.161.23,"{""location"": ""LC"", ""is_mobile"": true}" 17794,6,401,2017-04-09 19:08:13,http://raynor.biz/yesenia,0.0885874910,170.121.205.221,"{""location"": ""LA"", ""is_mobile"": false}" 17795,6,401,2017-06-03 12:19:13,http://price.name/jaylon,0.6101637815,156.220.131.223,"{""location"": ""BI"", ""is_mobile"": true}" 17796,6,401,2017-06-03 04:00:03,http://schroeder.net/frank,0.2423566584,184.231.34.196,"{""location"": ""BW"", ""is_mobile"": false}" 17797,6,401,2017-01-01 08:48:36,http://barrows.info/aron_krajcik,0.5719899222,230.82.61.104,"{""location"": ""UM"", ""is_mobile"": true}" 17798,6,401,2017-05-20 12:08:43,http://wilderman.biz/jamey_witting,0.1792129596,6.141.169.32,"{""location"": ""CK"", ""is_mobile"": false}" 17799,6,402,2017-05-29 14:51:02,http://ricehodkiewicz.io/karley_casper,0.0437166773,24.3.221.201,"{""location"": ""ZA"", ""is_mobile"": false}" 17800,6,402,2017-02-11 15:22:38,http://kohler.org/margarete,0.2115146148,88.13.161.59,"{""location"": ""AR"", ""is_mobile"": false}" 17801,6,402,2017-04-07 00:42:17,http://goodwin.co/fatima,0.3695017664,95.61.151.30,"{""location"": ""PM"", ""is_mobile"": true}" 17802,6,402,2017-05-12 08:26:30,http://fisher.com/king,0.1160896013,81.61.192.212,"{""location"": ""CR"", ""is_mobile"": true}" 17803,6,402,2017-05-02 14:50:55,http://beatty.net/isabel,0.7745603134,172.55.36.191,"{""location"": ""NO"", ""is_mobile"": true}" 17804,6,402,2017-01-15 03:24:41,http://christiansen.info/nikko,0.6943300333,9.227.62.196,"{""location"": ""EC"", ""is_mobile"": true}" 17805,6,402,2016-12-30 17:01:51,http://altenwerth.org/landen,0.0211069043,185.89.175.138,"{""location"": ""MK"", ""is_mobile"": false}" 17806,6,402,2016-12-14 13:02:50,http://gorczanymarvin.co/eunice,0.4965794532,123.244.223.164,"{""location"": ""AG"", ""is_mobile"": false}" 17807,6,402,2017-05-11 23:36:44,http://keelingcronin.com/sophia_kreiger,0.4347450261,190.27.254.133,"{""location"": ""BT"", ""is_mobile"": true}" 17808,6,402,2017-06-07 05:45:56,http://jacobi.info/vivian.kohler,0.4386901309,13.5.15.158,"{""location"": ""BW"", ""is_mobile"": false}" 17809,6,402,2017-01-11 20:05:36,http://haagkeeling.name/haleigh,0.5849949724,242.215.194.153,"{""location"": ""UY"", ""is_mobile"": false}" 17810,6,402,2017-04-20 19:26:25,http://bartelldamore.biz/gerda,0.2807117956,180.11.122.157,"{""location"": ""EE"", ""is_mobile"": true}" 17811,6,402,2016-12-18 12:38:37,http://stiedemann.biz/hadley_fisher,0.2254377169,173.78.205.201,"{""location"": ""BO"", ""is_mobile"": true}" 17812,6,402,2017-06-09 22:16:51,http://konopelski.org/giles.vandervort,0.6483984874,41.103.36.60,"{""location"": ""BL"", ""is_mobile"": false}" 17813,6,402,2017-04-12 17:55:07,http://handskiles.co/clinton.towne,0.6213692123,69.161.169.144,"{""location"": ""AI"", ""is_mobile"": false}" 17814,6,402,2017-03-13 20:56:29,http://rodriguezthiel.info/dylan,0.3326371606,47.71.22.157,"{""location"": ""JP"", ""is_mobile"": true}" 17815,6,402,2017-03-06 02:50:56,http://hyatt.io/robin_schmeler,0.0338952333,27.11.66.24,"{""location"": ""AZ"", ""is_mobile"": false}" 17816,6,402,2016-12-24 14:12:49,http://hilllnitzsche.biz/louie,0.5988971452,217.64.181.136,"{""location"": ""ET"", ""is_mobile"": false}" 17817,6,402,2017-01-14 16:10:08,http://mitchell.com/adriel_hoeger,0.0295333358,117.47.16.6,"{""location"": ""IE"", ""is_mobile"": false}" 17818,6,402,2017-01-09 03:44:50,http://wiegandhammes.info/terry,0.9210997671,27.52.87.199,"{""location"": ""TZ"", ""is_mobile"": true}" 17819,6,402,2017-01-20 17:17:59,http://feil.name/lee,0.4260689976,136.77.194.227,"{""location"": ""NG"", ""is_mobile"": false}" 17820,6,402,2017-01-27 18:33:34,http://towneemmerich.org/tiana.hane,0.1149769482,213.221.12.151,"{""location"": ""TW"", ""is_mobile"": false}" 17821,6,402,2017-03-16 01:27:10,http://goodwinschmitt.biz/beie,0.0963811127,228.58.47.98,"{""location"": ""LA"", ""is_mobile"": false}" 17822,6,402,2017-05-11 11:02:53,http://luettgenhamill.org/saige_halvorson,0.7967879494,67.225.169.81,"{""location"": ""GY"", ""is_mobile"": true}" 17823,6,402,2017-02-07 17:58:57,http://lockman.name/mariam,0.5623703583,87.144.72.136,"{""location"": ""UZ"", ""is_mobile"": false}" 17824,6,402,2017-04-05 03:29:32,http://macejkovic.net/silas,0.6803988489,96.94.155.20,"{""location"": ""CM"", ""is_mobile"": true}" 5004,2,111,2017-06-06 07:02:37,http://batzratke.com/linnie,,187.78.192.163,"{""location"": ""AL"", ""is_mobile"": false}" 5005,2,111,2017-04-09 15:49:31,http://lueilwitz.co/giles.tromp,,209.84.64.60,"{""location"": ""CO"", ""is_mobile"": true}" 5006,2,111,2017-05-22 12:39:27,http://effertz.co/jay,,196.110.234.40,"{""location"": ""TC"", ""is_mobile"": false}" 5007,2,111,2016-12-22 20:40:52,http://grimesmuller.co/lucinda,,41.16.20.195,"{""location"": ""AX"", ""is_mobile"": true}" 5008,2,111,2017-02-19 10:57:03,http://casper.name/sigurd.mayer,,192.183.63.101,"{""location"": ""JE"", ""is_mobile"": true}" 5009,2,111,2017-02-03 17:38:51,http://bailey.biz/sherwood_schulist,,134.245.3.157,"{""location"": ""SK"", ""is_mobile"": false}" 5010,2,111,2016-12-14 13:47:20,http://swiftrunolfsdottir.com/anais_dach,,8.89.104.243,"{""location"": ""SM"", ""is_mobile"": false}" 5011,2,111,2017-03-24 07:50:41,http://ruel.biz/gregg_toy,,55.200.235.252,"{""location"": ""GE"", ""is_mobile"": true}" 5012,2,111,2017-02-28 18:21:21,http://weberkshlerin.org/florence_macgyver,,200.221.213.206,"{""location"": ""IQ"", ""is_mobile"": true}" 5013,2,111,2017-03-09 00:16:26,http://barton.biz/monserrat_powlowski,,182.60.183.104,"{""location"": ""TW"", ""is_mobile"": false}" 5014,2,111,2016-12-19 03:40:47,http://dareveum.co/bo,,8.105.129.12,"{""location"": ""PM"", ""is_mobile"": false}" 5015,2,111,2017-05-28 22:34:50,http://herzog.com/favian_kiehn,,53.32.104.78,"{""location"": ""HR"", ""is_mobile"": true}" 5016,2,111,2017-03-30 09:46:24,http://hansenhilpert.io/davon.waelchi,,79.88.163.31,"{""location"": ""TW"", ""is_mobile"": true}" 5017,2,111,2017-02-04 10:23:23,http://nikolausroob.co/judah,,135.170.125.240,"{""location"": ""EG"", ""is_mobile"": false}" 5018,2,111,2017-01-19 05:22:39,http://johnson.biz/wellington_witting,,213.190.236.16,"{""location"": ""WS"", ""is_mobile"": true}" 5019,2,111,2016-12-15 09:55:31,http://prosacco.com/osvaldo,,43.198.100.231,"{""location"": ""UY"", ""is_mobile"": false}" 5020,2,111,2017-01-29 21:18:59,http://homenick.io/reba,,64.114.162.110,"{""location"": ""MZ"", ""is_mobile"": false}" 5021,2,111,2017-04-16 05:15:11,http://stamm.name/tyshawn_sawayn,,193.172.245.243,"{""location"": ""CK"", ""is_mobile"": true}" 5022,2,111,2017-02-13 14:47:38,http://gottliebbosco.co/gayle_stanton,,89.64.197.152,"{""location"": ""CL"", ""is_mobile"": true}" 5023,2,111,2017-03-29 08:11:47,http://mraz.info/osvaldo_larkin,,163.98.241.226,"{""location"": ""NL"", ""is_mobile"": false}" 5024,2,111,2017-01-13 21:32:53,http://mayert.co/kian,,68.170.44.11,"{""location"": ""AO"", ""is_mobile"": false}" 5025,2,111,2017-03-21 06:38:03,http://sawayn.net/harold,,126.150.16.252,"{""location"": ""VE"", ""is_mobile"": true}" 5026,2,111,2017-05-26 10:54:05,http://olson.org/lempi,,48.214.173.19,"{""location"": ""CD"", ""is_mobile"": false}" 5027,2,111,2017-05-12 18:19:30,http://murraymacejkovic.net/demond_donnelly,,91.147.217.207,"{""location"": ""ML"", ""is_mobile"": true}" 5028,2,111,2017-02-08 23:03:35,http://jerde.com/parker_rogahn,,204.92.135.180,"{""location"": ""CM"", ""is_mobile"": false}" 5029,2,111,2017-03-12 08:31:27,http://treutel.name/itzel.goldner,,147.25.194.253,"{""location"": ""ES"", ""is_mobile"": true}" 5030,2,111,2017-02-03 09:42:40,http://cremin.net/coleman,,61.12.55.151,"{""location"": ""KZ"", ""is_mobile"": true}" 5031,2,111,2017-01-02 05:24:57,http://reillymoriette.io/davion,,198.225.229.101,"{""location"": ""UY"", ""is_mobile"": false}" 5032,2,111,2017-02-12 03:44:58,http://mohr.org/gerson.jacobson,,120.83.216.245,"{""location"": ""TL"", ""is_mobile"": false}" 5033,2,111,2017-01-19 04:20:52,http://mcdermottschaefer.name/shane_reilly,,226.43.235.254,"{""location"": ""MZ"", ""is_mobile"": false}" 5034,2,111,2016-12-18 01:56:11,http://mohr.biz/adeline.mclaughlin,,43.171.234.58,"{""location"": ""SX"", ""is_mobile"": true}" 5035,2,111,2017-06-07 23:59:28,http://kunze.name/delpha.rowe,,99.163.108.124,"{""location"": ""VC"", ""is_mobile"": false}" 5036,2,111,2016-12-15 10:56:32,http://grantbode.com/santiago.fadel,,31.221.161.194,"{""location"": ""BY"", ""is_mobile"": false}" 5037,2,111,2017-02-06 10:09:44,http://shanahangraham.com/evans_gleichner,,31.182.35.46,"{""location"": ""MX"", ""is_mobile"": false}" 5038,2,111,2017-01-05 11:20:59,http://mcglynn.name/annette,,147.157.117.137,"{""location"": ""KM"", ""is_mobile"": true}" 5039,2,111,2017-02-19 03:27:41,http://lefflerkutch.com/cleta,,109.163.162.57,"{""location"": ""BI"", ""is_mobile"": true}" 5040,2,111,2017-04-28 21:03:55,http://anderson.co/christop_hand,,72.64.248.63,"{""location"": ""CD"", ""is_mobile"": false}" 5041,2,111,2017-04-22 17:46:40,http://eichmannrodriguez.org/franz,,191.62.3.132,"{""location"": ""SC"", ""is_mobile"": true}" 5042,2,111,2017-02-08 12:39:11,http://kochhills.io/conrad.upton,,219.135.151.147,"{""location"": ""IL"", ""is_mobile"": true}" 5043,2,111,2017-05-09 07:33:25,http://witting.net/pink_bins,,68.201.91.94,"{""location"": ""CN"", ""is_mobile"": true}" 5044,2,111,2017-04-04 08:31:37,http://mraz.io/maida.ferry,,151.145.164.231,"{""location"": ""MA"", ""is_mobile"": true}" 5045,2,111,2017-01-21 17:10:17,http://schummcrist.net/retha_lakin,,59.246.244.127,"{""location"": ""BM"", ""is_mobile"": true}" 5046,2,111,2017-02-06 01:39:33,http://schmitt.info/harmony.bartoletti,,229.78.90.159,"{""location"": ""MM"", ""is_mobile"": false}" 5047,2,111,2017-03-09 13:49:18,http://grimetracke.co/valentin.paucek,,135.94.89.179,"{""location"": ""ET"", ""is_mobile"": false}" 5048,2,111,2017-01-05 03:43:37,http://rosenbaum.co/beryl_collier,,186.217.134.130,"{""location"": ""TJ"", ""is_mobile"": true}" 5049,2,111,2017-04-13 22:58:48,http://hanestoltenberg.com/griffin_bayer,,133.24.12.38,"{""location"": ""NL"", ""is_mobile"": false}" 5050,2,111,2017-03-31 09:00:10,http://olson.name/claudia_senger,,118.78.144.196,"{""location"": ""SC"", ""is_mobile"": true}" 5051,2,111,2017-06-10 00:26:02,http://rempelharber.org/stan_dare,,73.128.168.134,"{""location"": ""FR"", ""is_mobile"": false}" 5052,2,111,2017-06-03 12:10:04,http://mckenzie.co/timmothy,,168.228.52.7,"{""location"": ""CK"", ""is_mobile"": true}" 5053,2,111,2017-04-23 17:39:46,http://kohlerwitting.org/norma_torp,,87.199.76.194,"{""location"": ""PE"", ""is_mobile"": false}" 5054,2,111,2017-05-19 22:34:50,http://olson.info/margarett.ratke,,203.24.97.165,"{""location"": ""SR"", ""is_mobile"": true}" 5055,2,111,2017-06-10 12:02:10,http://eberthills.io/kacey,,27.220.84.74,"{""location"": ""UZ"", ""is_mobile"": true}" 5056,2,111,2017-04-12 23:59:10,http://deckow.info/dashawn,,29.119.43.252,"{""location"": ""CK"", ""is_mobile"": false}" 5057,2,111,2016-12-24 20:52:37,http://cronin.io/gay,,17.114.223.44,"{""location"": ""IO"", ""is_mobile"": false}" 5058,2,111,2017-01-07 12:14:55,http://franeckismith.info/verner_mohr,,158.148.229.223,"{""location"": ""SD"", ""is_mobile"": false}" 5059,2,111,2017-05-31 21:24:29,http://beatty.biz/sid_veum,,152.148.147.191,"{""location"": ""TL"", ""is_mobile"": false}" 17987,7,407,2017-03-19 23:10:33,http://haley.org/madelyn.moen,,187.138.48.43,"{""location"": ""YE"", ""is_mobile"": false}" 17988,7,407,2017-04-10 15:41:32,http://nitzsche.info/yasmine_wyman,,183.243.67.234,"{""location"": ""HM"", ""is_mobile"": true}" 17989,7,407,2017-02-26 03:19:59,http://daugherty.net/amelia,,139.101.229.179,"{""location"": ""MO"", ""is_mobile"": false}" 17990,7,407,2017-01-22 01:51:13,http://lowefahey.com/marge.anderson,,9.58.152.212,"{""location"": ""IN"", ""is_mobile"": false}" 17991,7,407,2017-02-04 20:30:59,http://faheyberge.net/vincenza_cummerata,,39.18.62.133,"{""location"": ""SR"", ""is_mobile"": true}" 17992,7,407,2017-01-25 14:33:20,http://sanford.biz/dina,,20.20.181.95,"{""location"": ""MU"", ""is_mobile"": false}" 17993,7,407,2016-12-19 03:37:43,http://beier.co/paxton.becker,,97.252.235.195,"{""location"": ""TO"", ""is_mobile"": true}" 17994,7,407,2017-02-20 07:38:22,http://padberg.info/arjun.farrell,,199.94.165.130,"{""location"": ""VN"", ""is_mobile"": false}" 17995,7,407,2017-05-05 14:59:02,http://kunzeswaniawski.co/eloy.schuppe,,104.177.214.57,"{""location"": ""NR"", ""is_mobile"": false}" 17996,7,407,2016-12-18 15:01:11,http://beahan.org/judah.harber,,84.221.66.146,"{""location"": ""LC"", ""is_mobile"": true}" 17997,7,407,2017-06-06 10:09:21,http://macgyver.com/isom,,114.250.169.212,"{""location"": ""DE"", ""is_mobile"": false}" 17998,7,407,2017-03-24 17:25:05,http://bergstromsenger.co/erik,,81.94.88.234,"{""location"": ""BD"", ""is_mobile"": false}" 17999,7,407,2017-06-08 17:25:45,http://swift.org/hipolito,,90.97.230.120,"{""location"": ""MQ"", ""is_mobile"": false}" 18000,7,407,2017-01-07 01:04:38,http://mosciskinienow.org/myrna.barrows,,217.188.125.248,"{""location"": ""KG"", ""is_mobile"": false}" 18001,7,407,2017-01-16 17:36:11,http://batzlesch.net/juvenal,,2.136.191.137,"{""location"": ""BQ"", ""is_mobile"": true}" 18002,7,407,2017-05-26 06:40:31,http://veum.co/kailey,,81.148.248.246,"{""location"": ""WS"", ""is_mobile"": true}" 18003,7,407,2017-01-07 21:41:51,http://connkrajcik.org/alexandrine,,186.116.15.83,"{""location"": ""ES"", ""is_mobile"": false}" 18004,7,407,2017-01-21 22:29:07,http://sipes.org/soledad,,82.10.134.192,"{""location"": ""BA"", ""is_mobile"": false}" 18005,7,407,2017-03-03 12:50:27,http://harriskautzer.com/delilah,,61.179.79.176,"{""location"": ""ET"", ""is_mobile"": false}" 18006,7,407,2016-12-13 17:02:09,http://gerlach.com/tierra,,185.106.46.46,"{""location"": ""VE"", ""is_mobile"": false}" 18007,7,407,2017-02-04 09:04:30,http://gleichnerrempel.org/presley,,77.144.237.221,"{""location"": ""BQ"", ""is_mobile"": true}" 18008,7,407,2017-06-07 11:06:40,http://blandastrosin.biz/neil.kohler,,197.227.71.94,"{""location"": ""MQ"", ""is_mobile"": true}" 18009,7,407,2017-06-06 03:17:30,http://kunze.com/abe,,138.49.145.235,"{""location"": ""VA"", ""is_mobile"": false}" 18010,7,407,2017-05-08 08:07:29,http://aufderhar.net/cleora,,148.176.220.119,"{""location"": ""FI"", ""is_mobile"": false}" 18011,7,408,2017-05-23 08:52:31,http://nader.org/lindsey.wilkinson,,172.243.51.172,"{""location"": ""SR"", ""is_mobile"": false}" 18012,7,408,2017-03-17 04:34:33,http://turcotte.name/prudence_haley,,104.131.43.95,"{""location"": ""CI"", ""is_mobile"": false}" 18013,7,408,2017-03-08 17:20:04,http://kilback.info/carmelo_mclaughlin,,101.248.5.170,"{""location"": ""MF"", ""is_mobile"": false}" 18014,7,408,2017-04-20 15:29:37,http://larson.name/lora,,70.249.236.73,"{""location"": ""LB"", ""is_mobile"": false}" 18015,7,408,2016-12-14 01:29:06,http://bahringer.biz/marietta,,228.234.152.126,"{""location"": ""IQ"", ""is_mobile"": false}" 18016,7,408,2017-01-15 03:50:03,http://dicki.co/tania,,252.49.103.206,"{""location"": ""BI"", ""is_mobile"": false}" 18017,7,408,2017-01-01 23:05:06,http://mohrbogisich.com/marie,,35.19.109.179,"{""location"": ""PS"", ""is_mobile"": false}" 18018,7,408,2017-02-20 01:45:07,http://smith.info/eda.hills,,245.128.243.140,"{""location"": ""LB"", ""is_mobile"": false}" 18019,7,408,2017-01-06 02:42:29,http://corkery.biz/marilou.anderson,,89.201.242.176,"{""location"": ""IS"", ""is_mobile"": false}" 18020,7,408,2017-04-11 22:41:59,http://millsreichel.io/tre,,184.249.98.14,"{""location"": ""GW"", ""is_mobile"": false}" 18021,7,408,2016-12-14 06:15:56,http://balistreri.biz/addie_bogisich,,198.10.93.42,"{""location"": ""MT"", ""is_mobile"": true}" 18022,7,408,2017-06-10 02:50:15,http://botsfordmcdermott.name/aubrey,,137.102.151.30,"{""location"": ""KI"", ""is_mobile"": false}" 18023,7,408,2017-02-14 13:48:40,http://wizabotsford.net/marjory.hagenes,,34.39.158.100,"{""location"": ""TV"", ""is_mobile"": false}" 18024,7,408,2017-05-11 10:30:59,http://lebsack.net/danial,,88.11.41.6,"{""location"": ""MP"", ""is_mobile"": true}" 18025,7,408,2017-03-08 00:51:09,http://reynoldsrenner.com/sterling,,252.67.163.62,"{""location"": ""MQ"", ""is_mobile"": false}" 18026,7,408,2017-03-22 16:33:57,http://tillmanrice.biz/lyda,,199.12.17.119,"{""location"": ""AS"", ""is_mobile"": false}" 18027,7,408,2017-01-16 03:54:14,http://schimmelboyle.com/amari_rutherford,,169.249.176.86,"{""location"": ""JO"", ""is_mobile"": false}" 18028,7,408,2017-05-19 02:12:48,http://keeling.com/waylon,,87.100.69.116,"{""location"": ""YT"", ""is_mobile"": true}" 18029,7,408,2017-04-12 16:05:41,http://weinat.name/israel_roberts,,41.241.20.106,"{""location"": ""QA"", ""is_mobile"": true}" 18030,7,408,2017-06-01 10:34:45,http://schroederullrich.org/elenor.hilll,,140.26.190.227,"{""location"": ""NR"", ""is_mobile"": true}" 18031,7,408,2017-05-04 21:46:04,http://riceschmeler.name/cecelia,,26.130.243.32,"{""location"": ""BQ"", ""is_mobile"": true}" 18032,7,408,2016-12-15 17:53:36,http://nikolaus.name/ruben.schiller,,73.230.4.135,"{""location"": ""HK"", ""is_mobile"": false}" 18033,7,409,2017-02-08 19:34:00,http://fahey.net/robin,,254.88.218.94,"{""location"": ""MN"", ""is_mobile"": false}" 18034,7,409,2017-02-27 05:25:27,http://kunde.org/connor_mueller,,25.250.78.176,"{""location"": ""GG"", ""is_mobile"": false}" 18035,7,409,2017-01-10 02:18:56,http://ondricka.co/kaelyn.ondricka,,67.108.171.20,"{""location"": ""JM"", ""is_mobile"": false}" 18036,7,409,2017-01-13 02:34:51,http://wisozkgrimes.com/alvah,,151.97.199.67,"{""location"": ""ID"", ""is_mobile"": false}" 18037,7,409,2017-05-08 16:06:55,http://abshire.co/ernie,,94.43.243.113,"{""location"": ""AR"", ""is_mobile"": true}" 18038,7,409,2017-01-26 08:12:52,http://hirtheking.biz/shirley_gorczany,,115.39.116.171,"{""location"": ""CO"", ""is_mobile"": false}" 18039,7,409,2017-03-12 17:48:24,http://hudson.co/gracie.gibson,,160.99.103.168,"{""location"": ""TM"", ""is_mobile"": true}" 18040,7,409,2017-04-02 20:11:56,http://boscogulgowski.co/hattie,,205.99.187.251,"{""location"": ""TL"", ""is_mobile"": false}" 18041,7,409,2017-02-15 02:36:20,http://conroy.com/trace.johnson,,188.41.169.89,"{""location"": ""GW"", ""is_mobile"": true}" 18042,7,409,2017-06-08 15:36:59,http://ritchie.net/nikko,,137.88.184.228,"{""location"": ""GA"", ""is_mobile"": false}" 14857,5,334,2017-04-29 05:05:12,http://braun.org/kelsie,0.7955821915,183.33.229.51,"{""location"": ""AE"", ""is_mobile"": true}" 14858,5,334,2017-03-09 21:29:19,http://wintheiserstanton.info/uriah,0.1090970533,51.109.71.192,"{""location"": ""TN"", ""is_mobile"": true}" 14859,5,334,2017-03-19 01:55:46,http://jaskolski.org/filiberto,0.3364525802,169.144.178.116,"{""location"": ""MZ"", ""is_mobile"": false}" 14860,5,334,2017-05-08 17:46:45,http://gottlieb.name/catalina_mcglynn,0.6390382937,224.184.6.17,"{""location"": ""GY"", ""is_mobile"": false}" 14861,5,334,2017-05-28 21:47:47,http://bahringer.org/nathen_bayer,0.9059479677,210.51.247.172,"{""location"": ""SM"", ""is_mobile"": false}" 14862,5,334,2017-01-04 15:52:13,http://pouroshudson.co/alexane,0.9282460861,160.60.102.218,"{""location"": ""SZ"", ""is_mobile"": true}" 14863,5,334,2017-03-17 06:17:17,http://funk.org/vada.mcclure,0.8159219933,215.191.90.86,"{""location"": ""FJ"", ""is_mobile"": false}" 14864,5,334,2017-05-13 14:07:56,http://ratke.info/maegan_baumbach,0.1222042069,19.184.230.27,"{""location"": ""SO"", ""is_mobile"": true}" 14865,5,335,2017-02-14 20:08:08,http://kirlin.io/jared,0.2238270035,225.15.52.13,"{""location"": ""EH"", ""is_mobile"": false}" 14866,5,335,2017-05-16 10:40:20,http://frami.com/hattie_murazik,0.4160646000,57.79.130.152,"{""location"": ""BH"", ""is_mobile"": true}" 14867,5,335,2017-04-22 16:09:55,http://boylemurphy.co/buck,0.6167972528,118.189.56.45,"{""location"": ""KE"", ""is_mobile"": false}" 14868,5,335,2016-12-15 23:28:16,http://kohlergibson.co/vladimir_corwin,0.8987802990,195.15.132.198,"{""location"": ""GB"", ""is_mobile"": false}" 14869,5,335,2017-01-24 22:06:35,http://metz.co/creola,0.8390200241,38.234.180.189,"{""location"": ""CU"", ""is_mobile"": false}" 14870,5,335,2017-05-22 06:32:08,http://turnerbailey.io/letha,0.6462490408,28.237.40.186,"{""location"": ""MA"", ""is_mobile"": true}" 14871,5,335,2017-05-29 05:54:14,http://yundt.com/rosalia,0.4591286197,134.147.49.114,"{""location"": ""BM"", ""is_mobile"": true}" 14872,5,335,2016-12-18 12:33:38,http://flatley.io/syble_rempel,0.3434840658,140.99.244.85,"{""location"": ""QA"", ""is_mobile"": false}" 14873,5,335,2017-06-07 01:05:17,http://braun.info/maximus_braun,0.8881730804,73.161.192.153,"{""location"": ""MR"", ""is_mobile"": true}" 14874,5,335,2017-02-11 18:46:02,http://batz.info/kenton_thiel,0.5845240570,70.71.2.114,"{""location"": ""NI"", ""is_mobile"": false}" 14875,5,335,2017-01-09 14:17:05,http://greenfelder.io/presley.hammes,0.4916420728,181.155.3.170,"{""location"": ""SV"", ""is_mobile"": true}" 14876,5,335,2017-05-01 23:50:13,http://kihn.org/mattie_bogisich,0.8379338199,204.115.246.174,"{""location"": ""PN"", ""is_mobile"": false}" 14877,5,335,2016-12-26 06:06:06,http://jones.co/rahsaan,0.8739959545,29.45.123.5,"{""location"": ""TK"", ""is_mobile"": true}" 14878,5,335,2017-01-06 18:04:32,http://willmsgibson.com/alda,0.5167544007,218.37.157.171,"{""location"": ""GD"", ""is_mobile"": false}" 14879,5,335,2017-02-24 01:04:52,http://schmidtgerhold.biz/nils_hahn,0.5503675261,230.176.239.17,"{""location"": ""KY"", ""is_mobile"": true}" 14880,5,335,2017-01-15 14:17:51,http://mcglynnsanford.com/asha_christiansen,0.9650068273,226.39.16.98,"{""location"": ""SH"", ""is_mobile"": true}" 14881,5,335,2016-12-22 19:54:06,http://kilbackwiza.co/marc,0.1974326324,49.171.90.166,"{""location"": ""ME"", ""is_mobile"": true}" 14882,5,335,2017-01-22 16:27:30,http://hilll.io/cedrick.trantow,0.1044500483,109.247.32.66,"{""location"": ""SY"", ""is_mobile"": true}" 14883,5,335,2017-04-05 14:55:37,http://dibbert.net/arlo_daniel,0.9915621866,107.63.34.78,"{""location"": ""BN"", ""is_mobile"": false}" 14884,5,335,2017-05-26 21:27:24,http://kiehndouglas.biz/elsie_franecki,0.3460806878,251.143.127.243,"{""location"": ""YE"", ""is_mobile"": true}" 14885,5,335,2017-03-28 14:23:15,http://gulgowskiullrich.co/cordia.gislason,0.5124294077,61.99.27.97,"{""location"": ""FJ"", ""is_mobile"": true}" 14886,5,335,2017-05-08 22:18:14,http://padbergsanford.io/je_mcdermott,0.6733214470,66.245.25.239,"{""location"": ""VE"", ""is_mobile"": true}" 14887,5,335,2017-06-03 18:31:38,http://ward.net/unique_ebert,0.1727547751,80.41.215.128,"{""location"": ""AD"", ""is_mobile"": false}" 14888,5,335,2017-04-22 04:21:22,http://hartmann.co/kennedi_witting,0.6445928313,213.124.78.111,"{""location"": ""MQ"", ""is_mobile"": false}" 5060,2,111,2017-01-29 16:28:50,http://schmidt.co/sydni,,187.166.239.167,"{""location"": ""CU"", ""is_mobile"": true}" 5061,2,111,2017-04-20 23:59:23,http://kuhlman.info/dale,,90.237.45.243,"{""location"": ""GP"", ""is_mobile"": false}" 5062,2,112,2017-02-12 06:53:05,http://larson.io/boyd.ferry,,254.230.166.221,"{""location"": ""WF"", ""is_mobile"": true}" 5063,2,112,2017-04-28 11:44:05,http://boyer.io/brandi,,213.129.235.39,"{""location"": ""ZM"", ""is_mobile"": false}" 5064,2,112,2017-01-09 13:13:43,http://lindhintz.biz/percival,,106.93.188.96,"{""location"": ""MY"", ""is_mobile"": true}" 5065,2,112,2017-04-07 14:44:46,http://gradybaumbach.com/erika.doyle,,122.34.165.187,"{""location"": ""AS"", ""is_mobile"": false}" 5066,2,112,2017-05-15 00:10:37,http://friesen.co/vito,,139.96.112.45,"{""location"": ""MR"", ""is_mobile"": false}" 5067,2,112,2017-05-08 15:12:09,http://thompsonnienow.io/darion.ward,,224.6.175.20,"{""location"": ""UZ"", ""is_mobile"": false}" 5068,2,112,2017-03-07 23:54:42,http://kochvon.io/sammy,,36.175.167.169,"{""location"": ""KW"", ""is_mobile"": false}" 5069,2,112,2017-02-16 21:03:06,http://zieme.io/lillie,,169.179.129.25,"{""location"": ""AR"", ""is_mobile"": true}" 5070,2,112,2017-03-29 22:21:53,http://dubuque.net/sterling.swift,,199.240.204.173,"{""location"": ""KN"", ""is_mobile"": true}" 5071,2,112,2017-04-26 08:08:27,http://hudson.org/eduardo_gleichner,,245.83.217.39,"{""location"": ""PS"", ""is_mobile"": false}" 5072,2,112,2017-01-01 04:42:23,http://weimann.co/declan,,195.93.27.118,"{""location"": ""SY"", ""is_mobile"": true}" 5073,2,112,2017-05-11 03:01:03,http://jacobimcglynn.io/meggie.hackett,,225.34.144.67,"{""location"": ""AQ"", ""is_mobile"": true}" 5074,2,112,2017-06-10 15:25:55,http://lakin.net/jarred.daugherty,,210.8.134.104,"{""location"": ""PH"", ""is_mobile"": true}" 5075,2,112,2017-02-08 00:00:13,http://koeppblanda.com/reece.block,,41.251.202.168,"{""location"": ""HU"", ""is_mobile"": false}" 5076,2,112,2017-01-01 15:38:50,http://reilly.com/isidro,,191.118.39.136,"{""location"": ""MX"", ""is_mobile"": false}" 5077,2,112,2017-05-01 13:24:44,http://waelchiwaelchi.info/green_heathcote,,222.218.217.92,"{""location"": ""PW"", ""is_mobile"": false}" 5078,2,112,2017-03-12 04:59:26,http://walsh.co/kolby,,81.214.228.105,"{""location"": ""VA"", ""is_mobile"": true}" 5079,2,112,2017-04-17 14:12:06,http://king.io/sedrick,,170.135.44.123,"{""location"": ""CL"", ""is_mobile"": false}" 5080,2,112,2016-12-20 01:58:31,http://king.com/jerrold.schuster,,10.152.101.246,"{""location"": ""LU"", ""is_mobile"": true}" 5081,2,112,2017-01-12 13:18:34,http://kutch.org/domenick.goldner,,51.206.31.253,"{""location"": ""SK"", ""is_mobile"": false}" 5082,2,112,2017-02-05 11:12:08,http://donnellyhayes.co/samir.kulas,,89.116.63.58,"{""location"": ""MQ"", ""is_mobile"": false}" 5083,2,112,2017-05-03 20:33:45,http://jacobson.net/enrique,,35.38.128.119,"{""location"": ""SN"", ""is_mobile"": false}" 5084,2,112,2017-01-10 08:44:25,http://champlin.co/garret_hills,,54.127.203.183,"{""location"": ""MY"", ""is_mobile"": false}" 5085,2,112,2017-02-11 01:11:38,http://ruel.name/cody_haag,,180.144.94.38,"{""location"": ""SK"", ""is_mobile"": true}" 5086,2,112,2017-05-12 07:14:51,http://durganmcdermott.name/aisha_heaney,,140.170.194.183,"{""location"": ""TV"", ""is_mobile"": false}" 5087,2,112,2017-03-09 10:21:43,http://greenholtschinner.biz/dewayne,,57.183.161.224,"{""location"": ""KY"", ""is_mobile"": false}" 5088,2,112,2017-02-04 04:46:19,http://collinshettinger.co/te.buckridge,,26.230.249.248,"{""location"": ""CG"", ""is_mobile"": false}" 5089,2,112,2017-05-04 14:17:28,http://bednarbrakus.io/jermaine,,8.191.107.65,"{""location"": ""LA"", ""is_mobile"": false}" 5090,2,112,2017-03-20 15:51:11,http://waelchi.com/damien,,48.140.112.49,"{""location"": ""AZ"", ""is_mobile"": false}" 5091,2,112,2017-04-07 20:49:36,http://nader.io/skyla_torp,,252.34.136.100,"{""location"": ""CL"", ""is_mobile"": false}" 5093,2,112,2017-01-29 00:19:17,http://uptonquigley.co/lori,,208.85.22.211,"{""location"": ""GS"", ""is_mobile"": false}" 5094,2,112,2017-01-19 06:43:54,http://lynch.info/filomena,,57.157.5.132,"{""location"": ""PA"", ""is_mobile"": true}" 5095,2,112,2017-03-04 23:27:33,http://bogan.io/gregoria,,198.150.76.135,"{""location"": ""VG"", ""is_mobile"": false}" 5096,2,112,2017-01-18 20:36:19,http://herzog.net/leola,,159.228.30.141,"{""location"": ""KG"", ""is_mobile"": true}" 5097,2,112,2017-01-08 05:04:48,http://langosh.net/reginald,,239.89.90.139,"{""location"": ""GF"", ""is_mobile"": false}" 5098,2,112,2017-02-20 15:03:51,http://durgan.net/yasmeen,,230.240.68.145,"{""location"": ""DJ"", ""is_mobile"": false}" 5099,2,112,2017-03-24 15:10:02,http://sauerrunte.name/dayne,,123.132.222.60,"{""location"": ""AZ"", ""is_mobile"": false}" 5100,2,112,2017-06-05 12:23:11,http://gleason.biz/toy.hansen,,135.44.224.92,"{""location"": ""BW"", ""is_mobile"": true}" 5101,2,112,2017-04-17 22:13:49,http://reilly.co/lon.daniel,,246.252.173.115,"{""location"": ""JP"", ""is_mobile"": true}" 5102,2,112,2017-02-21 06:20:13,http://upton.net/loraine,,137.90.241.214,"{""location"": ""ME"", ""is_mobile"": true}" 5103,2,112,2017-03-01 07:02:53,http://willms.info/autumn,,43.5.188.14,"{""location"": ""MZ"", ""is_mobile"": false}" 5104,2,112,2017-02-22 07:08:09,http://swiftcrona.biz/katelynn.hoeger,,25.170.246.238,"{""location"": ""TO"", ""is_mobile"": false}" 5105,2,112,2017-01-20 17:13:27,http://gusikowski.info/harold,,249.144.117.193,"{""location"": ""LC"", ""is_mobile"": false}" 5106,2,112,2017-05-01 05:58:37,http://weinat.net/timmothy,,205.30.155.16,"{""location"": ""SK"", ""is_mobile"": false}" 5107,2,112,2017-04-25 14:39:23,http://rodriguez.name/maddison,,59.36.133.199,"{""location"": ""CL"", ""is_mobile"": true}" 5108,2,112,2017-04-30 06:28:23,http://rogahnkuphal.name/johnpaul,,165.111.202.147,"{""location"": ""CI"", ""is_mobile"": false}" 5109,2,112,2017-01-05 08:59:48,http://streichbahringer.co/brayan,,105.246.104.218,"{""location"": ""IM"", ""is_mobile"": false}" 5110,2,112,2017-04-22 10:36:36,http://abbott.info/urban,,106.125.190.162,"{""location"": ""RS"", ""is_mobile"": false}" 5111,2,112,2017-05-02 16:45:42,http://schoen.info/thea_becker,,43.114.103.76,"{""location"": ""IS"", ""is_mobile"": false}" 5112,2,112,2017-05-18 01:25:38,http://walker.net/louisa,,5.76.3.209,"{""location"": ""WF"", ""is_mobile"": false}" 5113,2,113,2017-02-16 05:02:44,http://schuppe.io/jacynthe,,96.180.29.84,"{""location"": ""NC"", ""is_mobile"": false}" 5114,2,113,2017-02-27 20:32:22,http://farrellterry.name/arnaldo_cremin,,218.201.138.8,"{""location"": ""SR"", ""is_mobile"": true}" 5115,2,113,2017-03-11 11:22:08,http://yost.com/winona,,247.97.135.183,"{""location"": ""GT"", ""is_mobile"": false}" 5116,2,113,2017-01-31 15:56:40,http://feil.org/violet_stanton,,30.63.183.102,"{""location"": ""JE"", ""is_mobile"": true}" 18043,7,409,2017-02-03 22:39:11,http://lebsack.co/edgar.mckenzie,,206.67.67.190,"{""location"": ""GG"", ""is_mobile"": false}" 18044,7,409,2017-05-02 15:41:30,http://cremin.io/milton,,90.4.13.225,"{""location"": ""MO"", ""is_mobile"": true}" 18045,7,409,2016-12-28 03:19:09,http://handkeler.info/maiya_stark,,5.25.22.150,"{""location"": ""CX"", ""is_mobile"": true}" 18046,7,409,2017-03-05 09:05:30,http://bergeyost.net/jarod_treutel,,247.162.57.135,"{""location"": ""BA"", ""is_mobile"": true}" 18047,7,409,2017-06-12 14:59:17,http://windlerdaugherty.info/haven.armstrong,,239.230.187.45,"{""location"": ""KN"", ""is_mobile"": false}" 18048,7,409,2017-03-29 06:31:43,http://graham.biz/berenice,,178.203.44.110,"{""location"": ""BH"", ""is_mobile"": true}" 18049,7,409,2017-03-07 23:57:27,http://renner.biz/grayson_gerlach,,168.237.163.15,"{""location"": ""RS"", ""is_mobile"": true}" 18050,7,409,2017-03-29 14:19:04,http://wyman.name/emmitt_pfannerstill,,113.128.28.57,"{""location"": ""SC"", ""is_mobile"": true}" 18051,7,409,2017-03-14 19:25:01,http://skiletamm.info/vidal,,176.68.91.130,"{""location"": ""PT"", ""is_mobile"": true}" 18052,7,409,2017-01-11 06:20:13,http://emard.co/harley_prosacco,,161.150.73.86,"{""location"": ""GB"", ""is_mobile"": true}" 18053,7,409,2017-02-20 22:00:07,http://robelcronin.com/edwardo,,217.129.230.215,"{""location"": ""IO"", ""is_mobile"": true}" 18054,7,410,2017-03-29 18:10:39,http://zulauf.io/tyler_heathcote,,226.135.239.179,"{""location"": ""BY"", ""is_mobile"": true}" 18055,7,410,2017-05-22 08:19:04,http://rodriguez.info/marianne,,37.179.238.45,"{""location"": ""JO"", ""is_mobile"": false}" 18056,7,410,2017-05-10 09:43:30,http://schmidt.name/caden_price,,193.49.244.212,"{""location"": ""TO"", ""is_mobile"": true}" 18057,7,410,2017-01-09 06:06:44,http://zieme.co/cale_toy,,15.24.83.58,"{""location"": ""IE"", ""is_mobile"": true}" 18058,7,410,2017-05-29 21:05:40,http://waelchitreutel.com/donavon,,89.103.51.137,"{""location"": ""NZ"", ""is_mobile"": false}" 18059,7,410,2017-03-28 09:55:01,http://kihn.biz/shanon,,63.125.133.192,"{""location"": ""CD"", ""is_mobile"": true}" 18060,7,410,2017-05-26 04:17:04,http://padbergfeil.info/frederik_ruel,,254.54.112.24,"{""location"": ""BB"", ""is_mobile"": false}" 18061,7,410,2017-02-04 14:49:46,http://adams.net/casimir_weber,,124.127.35.70,"{""location"": ""HN"", ""is_mobile"": true}" 18062,7,410,2016-12-27 09:18:07,http://mooreschneider.org/ezekiel.zulauf,,70.37.81.31,"{""location"": ""CU"", ""is_mobile"": false}" 18063,7,410,2017-05-06 19:56:59,http://ernser.net/sarah_borer,,225.40.40.199,"{""location"": ""GL"", ""is_mobile"": true}" 18064,7,410,2017-04-29 07:04:17,http://kuphalhyatt.org/nelson,,47.252.189.81,"{""location"": ""YT"", ""is_mobile"": false}" 18065,7,410,2017-01-27 09:58:21,http://schumm.info/hope,,76.134.145.246,"{""location"": ""VA"", ""is_mobile"": true}" 18066,7,410,2017-03-24 12:06:00,http://hilpertlebsack.info/gerhard_grant,,202.235.226.96,"{""location"": ""AR"", ""is_mobile"": false}" 18067,7,410,2017-03-10 07:46:34,http://heathcote.name/arlo,,201.161.193.105,"{""location"": ""GP"", ""is_mobile"": true}" 18068,7,410,2017-05-20 20:43:52,http://bartell.org/brenna,,211.112.82.98,"{""location"": ""AX"", ""is_mobile"": false}" 18069,7,410,2017-01-04 19:53:56,http://breitenbergfarrell.net/herminio,,202.157.150.35,"{""location"": ""US"", ""is_mobile"": false}" 18070,7,410,2017-01-23 13:03:39,http://nikolaus.name/cyril,,59.226.245.57,"{""location"": ""NP"", ""is_mobile"": true}" 18071,7,410,2017-01-28 14:34:23,http://stehrritchie.info/elisa,,224.91.89.61,"{""location"": ""WS"", ""is_mobile"": false}" 18072,7,410,2017-03-15 02:32:31,http://mcglynn.net/rosemarie_greenholt,,41.60.148.27,"{""location"": ""BR"", ""is_mobile"": true}" 18073,7,410,2017-04-07 18:42:18,http://hagenes.io/estel,,239.43.204.24,"{""location"": ""CR"", ""is_mobile"": true}" 18074,7,410,2017-02-16 13:33:02,http://jacobi.com/sigrid,,31.158.173.163,"{""location"": ""PW"", ""is_mobile"": true}" 18075,7,410,2017-05-06 00:50:47,http://bednarheel.name/kelsie,,232.74.126.108,"{""location"": ""IQ"", ""is_mobile"": true}" 18076,7,410,2017-05-12 12:29:05,http://thielkreiger.co/wyatt.cremin,,221.199.147.216,"{""location"": ""RE"", ""is_mobile"": true}" 18077,7,410,2017-02-04 00:20:34,http://bergnaumquitzon.io/dereck_bergstrom,,52.199.174.120,"{""location"": ""GU"", ""is_mobile"": true}" 18078,7,410,2017-05-22 19:07:03,http://okunevareichert.info/elbert,,173.238.182.110,"{""location"": ""WS"", ""is_mobile"": false}" 18079,7,410,2016-12-21 16:08:23,http://kshlerinmacejkovic.org/favian,,130.205.52.97,"{""location"": ""ZA"", ""is_mobile"": true}" 18080,7,410,2017-03-07 16:28:38,http://feesthane.io/jacklyn,,165.92.88.198,"{""location"": ""CL"", ""is_mobile"": true}" 18081,7,410,2017-05-01 00:24:45,http://herzog.org/alfreda_reynolds,,158.221.74.53,"{""location"": ""HR"", ""is_mobile"": true}" 18082,7,410,2017-02-03 04:18:21,http://dooley.org/patsy,,130.27.223.198,"{""location"": ""BN"", ""is_mobile"": false}" 18083,7,410,2017-01-04 08:25:11,http://will.net/bernita,,16.172.245.102,"{""location"": ""CM"", ""is_mobile"": false}" 18084,7,410,2017-01-06 19:22:18,http://larkin.io/tyrique,,28.184.7.199,"{""location"": ""EG"", ""is_mobile"": false}" 18085,7,410,2017-05-12 08:47:29,http://ankunding.io/rex.marks,,166.72.186.232,"{""location"": ""CK"", ""is_mobile"": true}" 18086,7,410,2017-05-28 18:51:54,http://schaeferwisozk.io/elvera,,90.250.225.145,"{""location"": ""AO"", ""is_mobile"": false}" 18087,7,410,2016-12-28 07:07:24,http://harber.biz/antone,,206.180.233.112,"{""location"": ""GP"", ""is_mobile"": false}" 18088,7,410,2017-02-17 12:53:47,http://vandervort.io/glennie,,122.92.144.34,"{""location"": ""CI"", ""is_mobile"": false}" 18089,7,410,2017-01-02 09:30:23,http://cole.biz/richard.murazik,,206.27.219.30,"{""location"": ""BR"", ""is_mobile"": false}" 18090,7,410,2017-05-21 20:28:41,http://marks.com/ivory,,191.226.210.209,"{""location"": ""BY"", ""is_mobile"": true}" 18091,7,410,2017-02-24 15:21:22,http://mrazoreilly.biz/laverne,,79.206.209.173,"{""location"": ""CO"", ""is_mobile"": false}" 18092,7,410,2017-01-18 07:41:49,http://schowalter.co/deon.goodwin,,195.74.14.225,"{""location"": ""EC"", ""is_mobile"": true}" 18093,7,410,2017-04-14 11:26:04,http://hamill.co/hardy.johnston,,175.23.112.67,"{""location"": ""KH"", ""is_mobile"": false}" 18094,7,410,2017-06-12 08:11:02,http://jenkins.biz/vincenzo_hettinger,,194.152.243.199,"{""location"": ""SV"", ""is_mobile"": false}" 18095,7,410,2017-06-08 18:55:41,http://shanahan.net/janick,,147.177.161.209,"{""location"": ""TC"", ""is_mobile"": true}" 18096,7,410,2017-06-07 05:44:14,http://hilll.org/mac.gibson,,88.35.223.54,"{""location"": ""NC"", ""is_mobile"": false}" 18097,7,410,2017-05-10 18:06:45,http://frami.io/candido_lowe,,191.142.34.78,"{""location"": ""AX"", ""is_mobile"": true}" 5117,2,113,2017-04-14 22:25:51,http://prosacco.info/erik.kerluke,,123.81.218.94,"{""location"": ""IQ"", ""is_mobile"": true}" 5118,2,113,2017-04-07 18:33:21,http://kulas.io/deie,,233.72.231.170,"{""location"": ""SH"", ""is_mobile"": false}" 5119,2,113,2017-04-30 18:09:21,http://langworth.co/jade_kirlin,,135.140.128.73,"{""location"": ""MR"", ""is_mobile"": true}" 5120,2,113,2017-06-02 11:12:10,http://bartellohara.io/alyson.mante,,217.85.231.196,"{""location"": ""FK"", ""is_mobile"": true}" 5121,2,113,2017-03-08 03:44:22,http://weber.io/breana_mohr,,150.8.186.137,"{""location"": ""SA"", ""is_mobile"": false}" 5122,2,113,2017-04-13 11:23:27,http://roberts.co/rodolfo,,225.207.241.129,"{""location"": ""FR"", ""is_mobile"": false}" 5123,2,113,2017-06-08 18:44:10,http://daniel.org/diana_runte,,149.30.119.168,"{""location"": ""RU"", ""is_mobile"": true}" 5124,2,113,2017-04-22 19:01:19,http://volkman.net/olaf_ankunding,,230.42.112.201,"{""location"": ""BF"", ""is_mobile"": true}" 5125,2,113,2017-02-08 01:23:30,http://jacobson.io/isobel,,228.50.218.60,"{""location"": ""KP"", ""is_mobile"": true}" 5126,2,113,2017-01-09 19:02:08,http://koch.io/ethan.cronin,,233.59.40.122,"{""location"": ""KN"", ""is_mobile"": false}" 5127,2,113,2017-03-12 15:07:41,http://thiel.com/titus,,190.217.179.217,"{""location"": ""BM"", ""is_mobile"": true}" 5128,2,113,2017-04-29 18:37:05,http://corwin.biz/corbin_waelchi,,157.19.248.11,"{""location"": ""BR"", ""is_mobile"": false}" 5129,2,113,2017-03-10 15:03:43,http://romaguera.info/derek_beatty,,187.150.27.138,"{""location"": ""LB"", ""is_mobile"": false}" 5130,2,113,2016-12-27 19:35:06,http://strosin.info/elfrieda_feil,,143.103.112.252,"{""location"": ""SI"", ""is_mobile"": false}" 5131,2,113,2016-12-19 02:29:17,http://barton.org/delores.wiegand,,112.193.106.144,"{""location"": ""CY"", ""is_mobile"": false}" 5132,2,113,2017-06-05 06:16:30,http://stoltenbergmurphy.io/mckayla.zulauf,,212.143.137.212,"{""location"": ""ES"", ""is_mobile"": true}" 5133,2,113,2017-01-02 21:10:16,http://wuckert.name/katlyn,,189.222.40.230,"{""location"": ""PW"", ""is_mobile"": false}" 5134,2,113,2017-06-13 22:45:10,http://okondaugherty.com/easton_kulas,,216.65.188.184,"{""location"": ""MV"", ""is_mobile"": false}" 5135,2,113,2017-03-27 14:22:01,http://corkery.org/reinhold,,125.5.172.232,"{""location"": ""SG"", ""is_mobile"": false}" 5136,2,113,2016-12-24 18:58:41,http://becker.com/leopold.bauch,,105.180.122.117,"{""location"": ""SS"", ""is_mobile"": true}" 5137,2,113,2017-01-03 07:59:19,http://ward.io/sydni.terry,,224.205.184.229,"{""location"": ""YE"", ""is_mobile"": true}" 5138,2,113,2017-03-04 08:32:38,http://murazik.info/grant,,96.128.136.176,"{""location"": ""GI"", ""is_mobile"": true}" 5139,2,113,2017-06-13 20:34:29,http://murray.net/emmett_robel,,139.216.127.142,"{""location"": ""GD"", ""is_mobile"": false}" 5140,2,113,2017-06-02 01:21:34,http://kub.info/derek_maggio,,91.93.115.91,"{""location"": ""TZ"", ""is_mobile"": false}" 5141,2,113,2017-05-01 01:57:30,http://yundtbrown.com/guillermo_graham,,96.43.148.188,"{""location"": ""CC"", ""is_mobile"": false}" 5142,2,114,2017-05-13 01:48:49,http://swiftvandervort.net/clara_reinger,,47.103.112.106,"{""location"": ""AL"", ""is_mobile"": false}" 5143,2,114,2017-05-04 16:06:55,http://windler.name/haskell,,63.44.248.203,"{""location"": ""BF"", ""is_mobile"": false}" 5144,2,114,2017-06-11 01:22:33,http://hamill.biz/zakary,,128.43.233.144,"{""location"": ""PR"", ""is_mobile"": true}" 5145,2,114,2017-01-01 01:00:47,http://stoltenberg.co/steve,,168.2.169.238,"{""location"": ""MH"", ""is_mobile"": false}" 5146,2,114,2017-02-19 01:44:24,http://jerde.info/haven,,36.68.69.175,"{""location"": ""GR"", ""is_mobile"": false}" 5147,2,114,2017-05-02 13:18:51,http://rempel.org/janie,,122.22.250.105,"{""location"": ""MV"", ""is_mobile"": false}" 5148,2,114,2017-03-15 11:21:27,http://kiehnweimann.org/will,,148.121.4.251,"{""location"": ""BI"", ""is_mobile"": true}" 5149,2,114,2017-04-12 19:26:07,http://hauck.net/dortha_watsica,,209.54.211.93,"{""location"": ""MX"", ""is_mobile"": true}" 5150,2,114,2017-05-01 01:45:19,http://lebsack.info/delfina,,236.140.205.38,"{""location"": ""FI"", ""is_mobile"": true}" 5151,2,114,2017-04-13 01:59:40,http://sanford.net/priscilla.bernhard,,164.192.168.102,"{""location"": ""GH"", ""is_mobile"": false}" 5152,2,114,2017-04-06 18:10:41,http://gaylord.io/austen,,178.41.228.77,"{""location"": ""KE"", ""is_mobile"": false}" 5153,2,114,2017-04-06 10:20:35,http://wehner.io/ania,,127.23.177.81,"{""location"": ""MT"", ""is_mobile"": false}" 5154,2,114,2017-03-01 23:25:38,http://schuppe.info/garth,,42.120.4.95,"{""location"": ""AE"", ""is_mobile"": true}" 5155,2,114,2017-01-21 13:00:59,http://langoshwyman.com/wyman.aufderhar,,229.83.6.154,"{""location"": ""FM"", ""is_mobile"": true}" 5156,2,114,2017-04-01 08:47:30,http://christiansenruel.biz/ellie_moen,,246.64.144.97,"{""location"": ""LV"", ""is_mobile"": false}" 5157,2,114,2016-12-31 06:38:13,http://stracke.org/claudie_braun,,152.25.97.131,"{""location"": ""BM"", ""is_mobile"": true}" 5158,2,114,2017-01-23 04:25:24,http://botsford.com/ed.gottlieb,,186.50.246.47,"{""location"": ""BA"", ""is_mobile"": true}" 5159,2,114,2016-12-21 08:43:03,http://reichert.co/kenton_batz,,148.84.96.56,"{""location"": ""WS"", ""is_mobile"": false}" 5160,2,114,2017-04-11 15:19:08,http://kertzmannweber.name/chad.muller,,99.172.88.130,"{""location"": ""SK"", ""is_mobile"": true}" 5161,2,114,2017-01-21 03:54:24,http://hickle.info/matilda.hartmann,,190.192.223.4,"{""location"": ""BQ"", ""is_mobile"": false}" 5162,2,114,2017-03-11 12:41:57,http://denesik.co/mitchell_boyer,,113.104.43.162,"{""location"": ""GY"", ""is_mobile"": false}" 5163,2,114,2017-05-05 04:05:44,http://cole.org/wiley,,95.118.223.95,"{""location"": ""RO"", ""is_mobile"": false}" 5164,2,114,2017-05-12 08:50:57,http://carter.net/lukas.stehr,,18.88.187.77,"{""location"": ""PM"", ""is_mobile"": true}" 5165,2,114,2017-03-14 08:20:59,http://feeneyauer.co/keely,,143.7.205.140,"{""location"": ""HU"", ""is_mobile"": false}" 5166,2,114,2017-05-24 22:59:52,http://hodkiewicz.biz/gustave.lubowitz,,185.211.252.72,"{""location"": ""TC"", ""is_mobile"": false}" 5167,2,114,2017-05-01 20:49:12,http://sauerswaniawski.info/aliza.hane,,120.235.243.116,"{""location"": ""PS"", ""is_mobile"": false}" 5168,2,114,2017-01-01 12:27:14,http://stehrreichert.net/domenico,,32.229.6.78,"{""location"": ""EE"", ""is_mobile"": true}" 5169,2,114,2017-03-12 20:59:34,http://flatleymetz.io/florencio,,48.122.10.33,"{""location"": ""GD"", ""is_mobile"": false}" 5170,2,114,2017-05-19 18:49:34,http://kutch.net/effie,,141.8.179.194,"{""location"": ""SB"", ""is_mobile"": true}" 5171,2,114,2017-04-22 18:44:18,http://jast.io/eriberto,,199.60.156.171,"{""location"": ""NC"", ""is_mobile"": false}" 18098,7,410,2016-12-15 15:52:46,http://shanahan.biz/adolf,,208.130.237.185,"{""location"": ""MS"", ""is_mobile"": true}" 18099,7,410,2017-01-05 04:57:15,http://moore.com/kamryn.hintz,,45.162.248.248,"{""location"": ""BB"", ""is_mobile"": true}" 18100,7,410,2017-02-26 11:29:36,http://kilbackhilpert.co/jodie_shanahan,,134.89.107.214,"{""location"": ""MO"", ""is_mobile"": false}" 18101,7,410,2017-01-05 02:55:20,http://reichel.com/clinton_nicolas,,231.118.31.208,"{""location"": ""FI"", ""is_mobile"": false}" 18102,7,410,2017-01-07 23:07:17,http://fahey.io/sydney_kemmer,,128.22.149.238,"{""location"": ""LB"", ""is_mobile"": true}" 18103,7,410,2017-04-15 04:19:40,http://veum.org/anabelle,,46.14.145.35,"{""location"": ""LA"", ""is_mobile"": false}" 18104,7,410,2017-05-17 21:14:43,http://bernhard.org/madalyn_rath,,97.245.22.52,"{""location"": ""TF"", ""is_mobile"": true}" 18105,7,410,2017-03-09 03:58:38,http://heathcote.info/josh.mueller,,3.23.186.123,"{""location"": ""LY"", ""is_mobile"": false}" 18106,7,410,2017-05-29 22:58:14,http://mosciskiullrich.name/columbus_rolfson,,211.160.210.187,"{""location"": ""UA"", ""is_mobile"": false}" 18107,7,410,2017-03-04 23:32:52,http://strackebergstrom.org/letitia,,68.153.104.170,"{""location"": ""MM"", ""is_mobile"": true}" 18108,7,410,2017-02-16 18:05:07,http://raynor.net/juana,,218.193.122.245,"{""location"": ""JP"", ""is_mobile"": true}" 18109,7,410,2017-04-21 06:16:23,http://hamill.net/ulices,,152.176.245.193,"{""location"": ""GU"", ""is_mobile"": false}" 18110,7,410,2017-02-08 01:13:51,http://tromp.co/delmer,,251.250.158.77,"{""location"": ""GT"", ""is_mobile"": false}" 18111,7,410,2017-01-14 19:50:48,http://kuphalhilll.biz/jaclyn,,146.229.166.174,"{""location"": ""TM"", ""is_mobile"": true}" 18112,7,410,2017-01-01 10:12:06,http://dubuquesawayn.name/juston,,172.23.177.218,"{""location"": ""ER"", ""is_mobile"": false}" 18113,7,410,2017-02-27 03:25:15,http://wiegand.biz/reese,,184.81.174.203,"{""location"": ""IR"", ""is_mobile"": true}" 18114,7,410,2017-02-04 13:13:14,http://oconnell.org/edyth.nicolas,,161.15.20.8,"{""location"": ""MO"", ""is_mobile"": false}" 18115,7,411,2017-02-11 07:58:33,http://kshlerin.io/ian,,235.130.63.53,"{""location"": ""NG"", ""is_mobile"": true}" 18116,7,411,2017-01-08 05:04:30,http://runolfsdottir.info/casey.herzog,,237.117.128.155,"{""location"": ""BO"", ""is_mobile"": true}" 18117,7,411,2017-03-28 13:53:31,http://larson.info/antwan_schultz,,10.30.138.71,"{""location"": ""SS"", ""is_mobile"": true}" 18118,7,411,2017-02-06 19:31:57,http://terry.info/selmer_white,,34.135.61.182,"{""location"": ""CH"", ""is_mobile"": false}" 18119,7,411,2017-06-13 00:12:42,http://schowalterweinat.com/haan.cain,,138.164.15.212,"{""location"": ""SB"", ""is_mobile"": true}" 18120,7,411,2016-12-19 20:46:28,http://christiansenhaley.org/nikita,,224.100.213.172,"{""location"": ""AR"", ""is_mobile"": true}" 18121,7,411,2017-05-14 03:31:50,http://aufderhar.info/kaya.brakus,,243.62.120.24,"{""location"": ""CZ"", ""is_mobile"": true}" 18122,7,411,2017-02-24 23:12:59,http://ledner.io/nellie_parker,,120.22.133.104,"{""location"": ""HM"", ""is_mobile"": false}" 18123,7,411,2017-05-01 23:02:18,http://mohr.name/estell,,252.49.11.120,"{""location"": ""MN"", ""is_mobile"": false}" 18124,7,411,2017-03-31 10:17:06,http://flatley.name/lenore_hoeger,,168.83.153.211,"{""location"": ""GW"", ""is_mobile"": true}" 18125,7,411,2017-04-15 07:32:28,http://wisozkpaucek.co/florian.weimann,,251.182.95.122,"{""location"": ""BJ"", ""is_mobile"": true}" 18126,7,411,2017-03-01 06:24:24,http://dickinson.info/imogene_hermiston,,140.152.217.97,"{""location"": ""TW"", ""is_mobile"": true}" 18127,7,411,2017-01-11 08:44:11,http://treutelbahringer.co/ernestina,,129.119.140.146,"{""location"": ""BR"", ""is_mobile"": true}" 18128,7,411,2017-01-07 08:18:44,http://boyer.name/sherman,,139.200.101.114,"{""location"": ""CC"", ""is_mobile"": true}" 18129,7,411,2016-12-25 09:44:26,http://bartoletti.com/emmet.vonrueden,,145.232.146.227,"{""location"": ""YT"", ""is_mobile"": false}" 18130,7,411,2016-12-29 06:39:11,http://hoppe.org/colt.ferry,,231.48.13.121,"{""location"": ""SB"", ""is_mobile"": false}" 18131,7,411,2017-02-15 14:37:14,http://nitzsche.info/lester,,21.30.3.92,"{""location"": ""SM"", ""is_mobile"": false}" 18132,7,411,2017-05-18 08:57:32,http://macgyvereffertz.biz/grayce.senger,,54.73.124.141,"{""location"": ""SN"", ""is_mobile"": false}" 18133,7,411,2017-04-03 04:14:34,http://damorejohns.com/ursula,,184.44.40.236,"{""location"": ""NF"", ""is_mobile"": true}" 18134,7,411,2017-04-13 03:56:38,http://runte.io/alford,,242.86.118.34,"{""location"": ""NP"", ""is_mobile"": true}" 18135,7,411,2017-01-10 23:14:40,http://leschnitzsche.biz/milo,,36.63.109.37,"{""location"": ""TL"", ""is_mobile"": false}" 18136,7,411,2017-02-11 14:30:46,http://bogisich.biz/bud.olson,,49.153.170.235,"{""location"": ""MX"", ""is_mobile"": true}" 18137,7,411,2017-03-10 17:30:32,http://weimanncasper.name/marlen.ward,,38.43.81.52,"{""location"": ""SR"", ""is_mobile"": true}" 18138,7,411,2017-05-11 22:36:07,http://pollich.info/gage,,162.53.77.92,"{""location"": ""LS"", ""is_mobile"": false}" 18139,7,411,2017-04-27 23:35:00,http://crooks.com/sherman.sipes,,142.72.216.119,"{""location"": ""DK"", ""is_mobile"": false}" 18140,7,411,2017-02-02 02:48:27,http://wuckert.info/fay_torp,,31.250.28.112,"{""location"": ""AM"", ""is_mobile"": true}" 18141,7,411,2016-12-18 08:06:55,http://hartmannschaden.biz/annabell_luettgen,,92.141.59.27,"{""location"": ""TM"", ""is_mobile"": false}" 18142,7,411,2016-12-21 18:18:49,http://watsicamiller.org/loma_spencer,,82.123.67.18,"{""location"": ""BV"", ""is_mobile"": false}" 18143,7,411,2017-02-06 14:22:03,http://medhurstdoyle.org/jonas,,234.145.102.155,"{""location"": ""CD"", ""is_mobile"": true}" 18144,7,411,2017-05-06 03:51:10,http://bartell.info/annamae_lowe,,13.82.65.162,"{""location"": ""UZ"", ""is_mobile"": true}" 18145,7,411,2017-05-02 11:42:06,http://herzog.org/vivien,,15.22.36.142,"{""location"": ""OM"", ""is_mobile"": false}" 18146,7,411,2017-03-25 08:36:33,http://hodkiewicz.net/daren.kuhic,,21.54.218.218,"{""location"": ""IM"", ""is_mobile"": false}" 18147,7,411,2017-05-08 08:46:00,http://howell.info/willard,,208.146.103.169,"{""location"": ""BL"", ""is_mobile"": false}" 18148,7,411,2017-05-25 04:31:51,http://lemkehermann.com/jordy,,67.15.220.21,"{""location"": ""CR"", ""is_mobile"": false}" 18149,7,411,2016-12-28 11:16:44,http://ondricka.io/theodora_stokes,,137.228.188.223,"{""location"": ""BW"", ""is_mobile"": true}" 18150,7,411,2017-02-04 02:27:28,http://breitenberg.org/floie_thiel,,93.156.127.212,"{""location"": ""BG"", ""is_mobile"": true}" 18151,7,411,2017-04-02 19:29:50,http://steuber.com/dangelo_leannon,,57.20.245.123,"{""location"": ""MA"", ""is_mobile"": true}" 18152,7,411,2017-02-07 00:17:53,http://brown.org/esteban.stanton,,216.221.6.5,"{""location"": ""GN"", ""is_mobile"": true}" 5172,2,114,2017-04-14 03:30:25,http://kilback.io/zaria_harvey,,59.15.15.24,"{""location"": ""UA"", ""is_mobile"": true}" 5173,2,114,2017-04-20 09:25:37,http://bauch.name/barry.bosco,,4.71.149.103,"{""location"": ""TV"", ""is_mobile"": true}" 5174,2,114,2017-05-06 08:11:04,http://rohan.biz/giovanny,,146.58.106.84,"{""location"": ""UZ"", ""is_mobile"": true}" 5175,2,114,2017-02-01 05:02:02,http://hahn.org/anibal,,39.38.157.185,"{""location"": ""TT"", ""is_mobile"": true}" 5176,2,114,2017-05-10 21:35:22,http://sanfordlehner.io/easter,,30.85.111.15,"{""location"": ""KH"", ""is_mobile"": true}" 5177,2,114,2017-04-15 21:23:33,http://crooks.info/wilford,,209.85.194.131,"{""location"": ""TW"", ""is_mobile"": true}" 5178,2,114,2017-02-28 13:33:25,http://farrell.com/eduardo,,98.178.162.159,"{""location"": ""US"", ""is_mobile"": true}" 5179,2,114,2016-12-29 05:15:55,http://upton.io/murl,,32.230.78.121,"{""location"": ""GQ"", ""is_mobile"": true}" 5180,2,114,2017-03-16 12:58:43,http://lueilwitz.com/gunnar_mcdermott,,214.12.43.170,"{""location"": ""TD"", ""is_mobile"": false}" 5181,2,114,2017-04-08 14:13:57,http://braunhowell.name/aleandra,,15.213.114.65,"{""location"": ""LT"", ""is_mobile"": false}" 5182,2,114,2017-04-14 00:48:08,http://haley.io/maye_renner,,176.179.180.115,"{""location"": ""CU"", ""is_mobile"": false}" 5183,2,114,2017-02-21 22:31:37,http://volkmanbogisich.com/bridget,,145.195.175.116,"{""location"": ""EE"", ""is_mobile"": false}" 5184,2,114,2016-12-21 05:20:03,http://schroeder.org/audra,,40.122.238.149,"{""location"": ""NL"", ""is_mobile"": false}" 5185,2,114,2017-05-15 02:09:25,http://runolfsdottir.name/rhoda_hettinger,,136.189.84.210,"{""location"": ""SZ"", ""is_mobile"": true}" 5186,2,114,2017-02-13 01:37:24,http://morarkaulke.name/marcos,,22.42.198.241,"{""location"": ""SG"", ""is_mobile"": true}" 5187,2,114,2017-02-01 05:33:06,http://kling.org/mazie,,78.148.231.130,"{""location"": ""RU"", ""is_mobile"": true}" 5188,2,115,2017-01-08 06:07:17,http://bauch.biz/johnnie,,20.102.2.228,"{""location"": ""BD"", ""is_mobile"": true}" 5189,2,115,2017-02-15 22:34:08,http://wisozkmitchell.name/dell_keebler,,139.52.92.109,"{""location"": ""BV"", ""is_mobile"": true}" 5190,2,115,2017-03-04 13:52:30,http://kunze.org/delphia,,73.230.152.56,"{""location"": ""MP"", ""is_mobile"": true}" 5191,2,115,2017-05-23 15:34:08,http://ziemann.org/wendell.mcdermott,,178.185.227.35,"{""location"": ""TF"", ""is_mobile"": false}" 5192,2,115,2016-12-25 15:32:18,http://daughertydavis.co/ted_mcclure,,185.37.188.89,"{""location"": ""BY"", ""is_mobile"": false}" 5193,2,115,2017-05-27 15:21:53,http://hills.org/jamir_crooks,,24.46.30.181,"{""location"": ""TG"", ""is_mobile"": true}" 5194,2,115,2017-01-03 22:04:27,http://feil.net/arturo.koelpin,,106.65.84.158,"{""location"": ""CN"", ""is_mobile"": false}" 5195,2,115,2017-02-08 02:30:46,http://wintheiser.co/rosie,,19.175.226.103,"{""location"": ""BW"", ""is_mobile"": false}" 5196,2,115,2016-12-27 07:33:44,http://bruen.biz/jazmyne,,40.204.74.114,"{""location"": ""UM"", ""is_mobile"": true}" 5197,2,115,2017-02-01 20:41:36,http://harvey.co/scottie_schultz,,88.207.7.75,"{""location"": ""VA"", ""is_mobile"": false}" 5198,2,115,2017-04-08 11:40:05,http://hintz.com/name_brakus,,222.248.25.165,"{""location"": ""VN"", ""is_mobile"": true}" 5199,2,115,2017-05-18 07:09:58,http://wittingjast.io/rebekah,,104.226.244.141,"{""location"": ""FK"", ""is_mobile"": false}" 5200,2,115,2017-01-04 23:11:13,http://roobrowe.com/angela.mann,,231.245.227.177,"{""location"": ""TL"", ""is_mobile"": true}" 5201,2,115,2016-12-22 14:43:12,http://boyle.biz/maya_schmeler,,75.84.24.126,"{""location"": ""KG"", ""is_mobile"": false}" 5202,2,115,2017-01-16 17:02:11,http://gottlieb.info/malika.simonis,,28.239.53.30,"{""location"": ""SM"", ""is_mobile"": false}" 5203,2,115,2017-03-19 07:47:16,http://greenfelderullrich.co/cortez,,244.123.157.155,"{""location"": ""MP"", ""is_mobile"": true}" 5204,2,115,2017-03-27 13:48:36,http://ebert.biz/osbaldo_tillman,,53.95.80.197,"{""location"": ""AF"", ""is_mobile"": false}" 5205,2,115,2017-01-25 13:05:43,http://koeppgulgowski.info/destin.herman,,6.253.223.120,"{""location"": ""FR"", ""is_mobile"": true}" 5206,2,115,2017-02-11 20:50:44,http://lefflerkautzer.info/sister_gislason,,37.98.109.21,"{""location"": ""JM"", ""is_mobile"": false}" 5207,2,115,2017-06-10 18:03:13,http://thompson.com/cristobal.schmitt,,22.31.82.31,"{""location"": ""AM"", ""is_mobile"": false}" 5208,2,115,2017-01-15 10:33:49,http://ondricka.co/grace_krajcik,,134.208.69.114,"{""location"": ""PE"", ""is_mobile"": false}" 5209,2,115,2016-12-16 17:41:26,http://schmidt.net/dortha_baumbach,,168.232.28.14,"{""location"": ""KG"", ""is_mobile"": true}" 5210,2,115,2017-05-29 17:19:50,http://bahringerrunolfsdottir.co/madie,,183.123.249.140,"{""location"": ""TN"", ""is_mobile"": false}" 5211,2,115,2017-03-24 16:04:25,http://webermcdermott.name/mariela,,192.65.43.56,"{""location"": ""KZ"", ""is_mobile"": false}" 5212,2,115,2017-01-28 11:33:10,http://schulist.io/sydnie,,169.60.138.15,"{""location"": ""QA"", ""is_mobile"": false}" 5213,2,115,2017-02-15 22:28:38,http://okuneva.com/johnpaul_heaney,,57.61.100.116,"{""location"": ""ZA"", ""is_mobile"": false}" 5214,2,115,2017-05-09 15:27:04,http://little.io/ramon,,117.138.23.187,"{""location"": ""CU"", ""is_mobile"": true}" 5215,2,115,2017-01-12 15:07:28,http://oreillyrodriguez.com/arlie_king,,38.18.47.5,"{""location"": ""JO"", ""is_mobile"": true}" 5216,2,115,2017-01-29 13:25:30,http://crist.info/desiree_wilkinson,,237.150.99.156,"{""location"": ""AF"", ""is_mobile"": true}" 5217,2,115,2017-06-03 16:49:49,http://handcorkery.name/joanne,,235.142.203.141,"{""location"": ""SM"", ""is_mobile"": true}" 5218,2,115,2017-05-25 23:04:52,http://stehr.name/addie_bayer,,94.179.253.250,"{""location"": ""CA"", ""is_mobile"": false}" 5219,2,115,2017-01-02 12:46:56,http://cartwright.org/bridie_hills,,50.158.9.145,"{""location"": ""SZ"", ""is_mobile"": true}" 5220,2,115,2017-04-20 19:50:20,http://funk.net/angelica,,20.162.208.29,"{""location"": ""NG"", ""is_mobile"": true}" 5221,2,115,2017-03-30 16:31:19,http://goldner.co/christy,,18.137.246.152,"{""location"": ""FM"", ""is_mobile"": false}" 5222,2,115,2016-12-25 23:07:31,http://kemmer.biz/lorena.emmerich,,83.80.48.83,"{""location"": ""ST"", ""is_mobile"": true}" 5223,2,115,2016-12-19 14:40:06,http://shanahan.org/asa.thiel,,153.11.48.219,"{""location"": ""FJ"", ""is_mobile"": true}" 5224,2,115,2016-12-28 05:47:55,http://reingergerlach.com/hettie_kihn,,148.43.226.90,"{""location"": ""AX"", ""is_mobile"": true}" 5225,2,116,2017-01-25 05:40:34,http://raynor.biz/nicolette.ward,,127.26.212.39,"{""location"": ""KW"", ""is_mobile"": true}" 5226,2,116,2017-04-21 18:22:59,http://mckenzie.biz/noemie.crona,,86.94.88.134,"{""location"": ""MG"", ""is_mobile"": false}" 5227,2,116,2017-04-29 20:36:21,http://adams.com/rhoda,,169.25.145.238,"{""location"": ""SZ"", ""is_mobile"": false}" 18153,7,411,2017-06-13 21:26:26,http://auerprohaska.biz/ryley,,123.45.24.61,"{""location"": ""TT"", ""is_mobile"": false}" 18154,7,411,2017-03-11 15:51:44,http://adams.name/constance,,152.152.171.60,"{""location"": ""BO"", ""is_mobile"": false}" 18155,7,412,2017-05-17 14:29:33,http://danielfadel.net/seth.krajcik,0.2475055007,81.226.82.247,"{""location"": ""AD"", ""is_mobile"": false}" 18156,7,412,2016-12-19 08:06:19,http://borer.net/arno,0.4782058202,197.217.188.146,"{""location"": ""MS"", ""is_mobile"": false}" 18157,7,412,2017-01-24 16:14:08,http://toy.name/bruce,0.6663642073,157.51.40.203,"{""location"": ""KP"", ""is_mobile"": true}" 18158,7,412,2017-05-16 00:46:16,http://ritchie.com/earline_ebert,0.6653865978,216.33.210.246,"{""location"": ""BD"", ""is_mobile"": true}" 18159,7,412,2016-12-19 14:24:09,http://schimmel.co/warren,0.4970929015,52.14.182.192,"{""location"": ""AR"", ""is_mobile"": true}" 18160,7,412,2016-12-14 14:47:27,http://cole.biz/emmitt.kreiger,0.8944623803,86.235.152.48,"{""location"": ""PR"", ""is_mobile"": true}" 18161,7,412,2017-05-31 14:53:43,http://harris.biz/jadyn_haag,0.7564550772,227.72.96.33,"{""location"": ""PL"", ""is_mobile"": true}" 18162,7,412,2017-05-18 11:54:00,http://volkmanlangosh.name/oceane,0.8479121241,133.217.131.102,"{""location"": ""LI"", ""is_mobile"": true}" 18163,7,412,2017-06-04 14:21:11,http://abernathy.io/zachery.thompson,0.8076185935,191.223.63.241,"{""location"": ""GQ"", ""is_mobile"": false}" 18164,7,412,2017-01-17 09:47:30,http://bechtelar.org/alycia.raynor,0.1166746351,146.169.230.173,"{""location"": ""VC"", ""is_mobile"": false}" 18165,7,412,2017-02-18 08:16:56,http://graham.io/lily.upton,0.5038774821,175.109.254.160,"{""location"": ""CO"", ""is_mobile"": false}" 18166,7,412,2017-01-15 22:20:11,http://strosin.org/elouise,0.4584685611,21.101.213.99,"{""location"": ""BQ"", ""is_mobile"": false}" 18167,7,412,2017-05-11 12:17:31,http://vandervort.co/van_ratke,0.1032804727,30.71.237.191,"{""location"": ""AF"", ""is_mobile"": false}" 18168,7,412,2017-05-15 22:24:13,http://hammes.name/marilie.mckenzie,0.2116805410,104.236.198.245,"{""location"": ""BJ"", ""is_mobile"": true}" 18169,7,412,2017-02-09 13:58:18,http://kulasraynor.io/nicholas.corkery,0.2819116770,230.100.75.158,"{""location"": ""SO"", ""is_mobile"": false}" 18170,7,412,2017-04-16 06:21:49,http://brakuhields.name/ella,0.4330045137,49.90.115.128,"{""location"": ""TM"", ""is_mobile"": true}" 18171,7,412,2017-01-05 08:09:23,http://moore.info/monte_hammes,0.3060940720,163.55.26.143,"{""location"": ""NI"", ""is_mobile"": true}" 18172,7,412,2017-03-15 08:19:27,http://prohaskalakin.biz/cole.osinski,0.0841574449,35.134.110.5,"{""location"": ""NC"", ""is_mobile"": false}" 18173,7,412,2017-04-19 10:29:35,http://blick.info/jack,0.4917110323,59.82.71.221,"{""location"": ""GW"", ""is_mobile"": false}" 18174,7,412,2017-05-25 20:29:15,http://lueilwitz.com/maritza.goodwin,0.2017217135,120.42.23.141,"{""location"": ""US"", ""is_mobile"": true}" 18175,7,412,2016-12-26 00:33:13,http://schoen.name/alvina,0.3748777697,115.100.172.183,"{""location"": ""TH"", ""is_mobile"": false}" 18176,7,412,2016-12-20 19:45:27,http://jacobilubowitz.io/danny,0.2823873943,49.53.82.211,"{""location"": ""GN"", ""is_mobile"": true}" 18177,7,412,2017-03-19 14:47:10,http://hodkiewicz.net/dominic,0.8112028144,87.74.144.90,"{""location"": ""TN"", ""is_mobile"": true}" 18178,7,412,2017-02-19 04:08:03,http://fadel.io/benton,0.2670336509,231.208.152.79,"{""location"": ""BH"", ""is_mobile"": true}" 18179,7,412,2017-03-23 03:04:14,http://marquardtweber.net/aubrey_gaylord,0.8277131989,37.108.200.64,"{""location"": ""CH"", ""is_mobile"": true}" 18180,7,412,2017-04-09 07:24:39,http://metznicolas.info/lincoln,0.0585045907,13.58.229.173,"{""location"": ""PH"", ""is_mobile"": false}" 18181,7,412,2017-05-26 07:20:01,http://friesen.org/jimmie_pfannerstill,0.4331628860,150.193.56.2,"{""location"": ""CY"", ""is_mobile"": true}" 18182,7,412,2017-05-24 00:03:25,http://sanford.net/itzel,0.2567155166,244.118.53.164,"{""location"": ""BB"", ""is_mobile"": true}" 18183,7,412,2017-03-11 16:44:27,http://hanerenner.net/samara.kulas,0.2081050704,248.80.209.45,"{""location"": ""LB"", ""is_mobile"": false}" 18184,7,412,2017-04-07 19:16:17,http://wilderman.info/laron,0.8259666303,31.125.191.20,"{""location"": ""IS"", ""is_mobile"": true}" 18185,7,412,2017-02-17 05:18:43,http://jones.net/callie,0.8025016162,160.178.47.250,"{""location"": ""LS"", ""is_mobile"": false}" 18186,7,412,2017-01-15 08:41:25,http://jerde.biz/nash.hackett,0.3491432179,109.5.94.229,"{""location"": ""LR"", ""is_mobile"": false}" 18187,7,412,2017-02-27 16:53:35,http://kuhn.biz/keshawn,0.0881032026,213.5.41.169,"{""location"": ""IQ"", ""is_mobile"": false}" 18188,7,412,2017-05-02 07:08:12,http://jones.net/kayli_carroll,0.2271281960,137.157.13.107,"{""location"": ""NC"", ""is_mobile"": true}" 18189,7,412,2017-05-26 10:33:51,http://stantontillman.org/travis,0.6566856448,163.127.245.112,"{""location"": ""KP"", ""is_mobile"": true}" 18190,7,412,2016-12-25 18:07:44,http://predovickunze.net/ivy.sporer,0.0499665640,74.10.145.24,"{""location"": ""DJ"", ""is_mobile"": true}" 18191,7,412,2017-05-04 15:32:37,http://fadel.net/amari_bechtelar,0.0491835816,243.69.101.81,"{""location"": ""TH"", ""is_mobile"": false}" 18192,7,412,2017-01-29 23:12:50,http://weber.info/tre,0.4884859335,35.87.179.51,"{""location"": ""AX"", ""is_mobile"": true}" 18193,7,412,2017-02-27 09:20:09,http://frami.info/demarco,0.0709808532,105.157.85.250,"{""location"": ""PG"", ""is_mobile"": false}" 18194,7,412,2017-03-16 05:17:44,http://torphy.info/brendan,0.8288865903,190.115.132.191,"{""location"": ""GD"", ""is_mobile"": false}" 18195,7,412,2016-12-31 14:32:08,http://schamberger.net/raul,0.9377594776,144.104.54.96,"{""location"": ""CU"", ""is_mobile"": false}" 18196,7,412,2017-02-23 13:03:06,http://beer.name/tony_murray,0.6310415842,238.208.106.75,"{""location"": ""MQ"", ""is_mobile"": true}" 18197,7,412,2017-04-09 11:08:06,http://simonisokeefe.info/wilford,0.4366598755,182.72.171.64,"{""location"": ""TT"", ""is_mobile"": false}" 18198,7,412,2017-05-29 02:43:05,http://colliermante.name/luz.denesik,0.5572316311,190.34.65.112,"{""location"": ""MY"", ""is_mobile"": true}" 18199,7,412,2017-05-07 22:54:39,http://beierdeckow.org/wanda,0.5919323309,58.103.191.181,"{""location"": ""GI"", ""is_mobile"": false}" 18200,7,412,2017-02-06 09:25:35,http://mclaughlin.net/cathy.rau,0.8721308937,24.166.110.102,"{""location"": ""VA"", ""is_mobile"": true}" 18201,7,412,2017-03-22 11:06:17,http://hettinger.net/kaylin,0.4590516421,97.134.21.70,"{""location"": ""KR"", ""is_mobile"": false}" 18202,7,412,2017-03-21 21:43:52,http://grant.net/delphine_weimann,0.0676858863,184.220.93.59,"{""location"": ""MP"", ""is_mobile"": true}" 18203,7,412,2017-04-05 05:53:48,http://kertzmann.io/cicero_schroeder,0.0070825967,55.126.218.64,"{""location"": ""LB"", ""is_mobile"": true}" 5228,2,116,2017-04-18 10:40:49,http://raynor.info/giovanna,,67.42.118.8,"{""location"": ""GB"", ""is_mobile"": false}" 5229,2,116,2017-02-07 04:41:20,http://rippin.com/micaela_okon,,128.65.190.45,"{""location"": ""WS"", ""is_mobile"": false}" 5230,2,116,2017-04-30 04:29:02,http://hoppe.com/tyrese,,219.239.178.152,"{""location"": ""IS"", ""is_mobile"": false}" 5231,2,116,2017-01-26 21:49:45,http://lueilwitz.io/rachelle,,79.99.132.129,"{""location"": ""CF"", ""is_mobile"": true}" 5232,2,116,2017-04-11 16:50:26,http://lowe.co/asha,,64.103.2.236,"{""location"": ""GA"", ""is_mobile"": true}" 5233,2,116,2017-04-25 17:08:11,http://howellkuhlman.info/leie,,196.20.164.51,"{""location"": ""BB"", ""is_mobile"": true}" 5234,2,116,2017-02-22 15:40:56,http://rempelshanahan.net/ian,,135.42.56.198,"{""location"": ""JE"", ""is_mobile"": false}" 5235,2,116,2017-04-17 03:43:42,http://nicolaslakin.biz/dalton_connelly,,120.181.172.169,"{""location"": ""AU"", ""is_mobile"": false}" 5236,2,116,2017-04-26 20:22:16,http://gleasoncormier.net/aubrey_zieme,,162.152.198.171,"{""location"": ""LK"", ""is_mobile"": true}" 5237,2,116,2017-05-28 12:50:51,http://pfeffer.co/abelardo_dooley,,110.40.60.12,"{""location"": ""CO"", ""is_mobile"": true}" 5238,2,116,2017-05-14 08:33:40,http://durgan.org/carolina,,50.168.223.172,"{""location"": ""AD"", ""is_mobile"": true}" 5239,2,116,2016-12-28 03:56:26,http://dietrich.net/aleandra.white,,195.135.65.212,"{""location"": ""SY"", ""is_mobile"": true}" 5240,2,116,2017-05-21 12:09:06,http://schroeder.info/oral.oberbrunner,,25.86.140.55,"{""location"": ""AE"", ""is_mobile"": true}" 5241,2,116,2017-02-11 12:58:01,http://trantow.net/kellie_wolf,,31.2.102.38,"{""location"": ""IL"", ""is_mobile"": false}" 5242,2,116,2017-04-26 05:19:05,http://kelerharber.net/myrtice,,232.92.158.46,"{""location"": ""NL"", ""is_mobile"": true}" 5243,2,116,2017-05-14 10:15:48,http://quitzonmayert.net/luigi,,137.12.130.198,"{""location"": ""KY"", ""is_mobile"": false}" 5244,2,116,2017-03-30 01:48:35,http://jastschneider.info/alfonzo_daugherty,,130.26.79.142,"{""location"": ""PW"", ""is_mobile"": true}" 5245,2,116,2017-06-01 01:27:49,http://wolf.io/keaton_roberts,,117.171.10.122,"{""location"": ""GA"", ""is_mobile"": false}" 5246,2,116,2017-02-24 06:38:08,http://roweschuster.co/brayan_feeney,,207.203.105.127,"{""location"": ""KY"", ""is_mobile"": false}" 5247,2,116,2017-04-30 01:06:19,http://koelpin.com/selmer_bailey,,56.57.54.156,"{""location"": ""LU"", ""is_mobile"": true}" 5248,2,116,2016-12-13 08:08:13,http://keler.co/dandre_moriette,,43.135.52.150,"{""location"": ""SS"", ""is_mobile"": false}" 5249,2,116,2017-04-14 22:09:49,http://klocko.org/amparo,,91.88.38.70,"{""location"": ""TD"", ""is_mobile"": false}" 5250,2,116,2017-03-09 23:13:57,http://gerlach.org/leila.ziemann,,156.32.28.254,"{""location"": ""GW"", ""is_mobile"": true}" 5251,2,116,2016-12-19 01:25:40,http://hermiston.io/dominic,,21.66.240.210,"{""location"": ""SY"", ""is_mobile"": true}" 5252,2,116,2017-01-14 16:57:24,http://hauckgleason.org/amya,,148.121.25.193,"{""location"": ""CW"", ""is_mobile"": true}" 5253,2,116,2017-03-03 23:01:14,http://dubuque.info/ivy_pfannerstill,,170.73.206.207,"{""location"": ""MU"", ""is_mobile"": true}" 5254,2,116,2017-01-14 03:32:50,http://koeppeichmann.co/janet,,117.180.98.28,"{""location"": ""MR"", ""is_mobile"": false}" 5255,2,116,2017-02-26 09:39:24,http://heller.net/brycen.davis,,224.36.15.92,"{""location"": ""IO"", ""is_mobile"": true}" 5256,2,116,2017-05-09 14:22:45,http://gleichner.io/braden,,109.193.91.240,"{""location"": ""ME"", ""is_mobile"": true}" 5257,2,116,2017-05-26 10:39:46,http://barton.net/blake.romaguera,,161.111.208.219,"{""location"": ""WF"", ""is_mobile"": true}" 5258,2,116,2017-02-26 03:05:36,http://reilly.com/clement_pfeffer,,114.59.30.70,"{""location"": ""ER"", ""is_mobile"": true}" 5259,2,116,2017-05-21 18:49:11,http://nicolaskub.org/salvador_kuvalis,,145.76.244.229,"{""location"": ""RO"", ""is_mobile"": true}" 5260,2,116,2017-02-24 03:47:34,http://dickikiehn.org/jayce_spinka,,148.187.56.172,"{""location"": ""UY"", ""is_mobile"": true}" 5261,2,116,2016-12-29 23:30:11,http://botsford.co/mara,,50.170.34.213,"{""location"": ""JM"", ""is_mobile"": false}" 5262,2,116,2017-01-23 20:51:50,http://rippinharvey.name/adonis,,55.123.79.16,"{""location"": ""EC"", ""is_mobile"": false}" 5263,2,116,2017-01-28 13:12:59,http://strackepredovic.net/adelia_veum,,100.150.63.218,"{""location"": ""SE"", ""is_mobile"": true}" 5264,2,116,2017-01-31 21:05:08,http://gutkowski.io/gonzalo,,91.33.167.140,"{""location"": ""VN"", ""is_mobile"": true}" 5266,2,116,2017-05-26 18:32:11,http://jacobson.info/elliott,,196.237.243.245,"{""location"": ""CZ"", ""is_mobile"": true}" 5267,2,116,2016-12-26 19:20:05,http://green.biz/myriam_vandervort,,100.94.139.174,"{""location"": ""DK"", ""is_mobile"": true}" 5268,2,116,2017-04-14 12:12:28,http://orn.io/ransom.stehr,,175.21.140.241,"{""location"": ""AO"", ""is_mobile"": false}" 5269,2,116,2016-12-17 01:23:31,http://powlowski.name/mose,,105.75.157.161,"{""location"": ""GM"", ""is_mobile"": false}" 5270,2,116,2017-03-13 13:07:00,http://keebler.io/luigi.schinner,,42.47.211.226,"{""location"": ""AZ"", ""is_mobile"": true}" 5271,2,116,2017-04-07 18:07:18,http://breitenbergfeil.com/orlando,,43.125.204.162,"{""location"": ""BT"", ""is_mobile"": false}" 5272,2,116,2017-03-08 11:59:42,http://waelchi.net/mason,,134.186.119.188,"{""location"": ""CO"", ""is_mobile"": false}" 5273,2,116,2017-06-04 21:23:39,http://mann.io/haan_feeney,,143.81.50.43,"{""location"": ""ID"", ""is_mobile"": true}" 5274,2,116,2017-02-22 06:32:12,http://damoregorczany.name/raoul,,26.52.77.11,"{""location"": ""AQ"", ""is_mobile"": true}" 5275,2,116,2017-05-30 22:34:29,http://boyerbartoletti.io/ned.gerhold,,121.127.126.210,"{""location"": ""UY"", ""is_mobile"": false}" 5276,2,116,2017-02-22 06:04:08,http://gislasonokon.net/noelia,,184.46.178.84,"{""location"": ""NI"", ""is_mobile"": true}" 5277,2,116,2017-02-02 00:59:47,http://macejkovichaley.info/colby,,43.200.114.108,"{""location"": ""EH"", ""is_mobile"": true}" 5278,2,116,2017-02-10 12:31:33,http://koeppstamm.com/jeie,,8.194.117.100,"{""location"": ""RE"", ""is_mobile"": false}" 5279,2,116,2017-04-22 09:06:16,http://hanehammes.org/alisa,,58.161.187.217,"{""location"": ""ML"", ""is_mobile"": true}" 5280,2,116,2017-03-09 08:27:11,http://lefflermurray.net/mack_leffler,,159.159.78.176,"{""location"": ""MD"", ""is_mobile"": true}" 5281,2,116,2016-12-30 08:17:16,http://goodwinkautzer.name/kyra,,184.169.158.191,"{""location"": ""CH"", ""is_mobile"": true}" 5282,2,116,2017-03-03 06:02:39,http://funk.biz/agustina_hodkiewicz,,225.215.53.84,"{""location"": ""FI"", ""is_mobile"": false}" 5283,2,116,2017-02-26 13:47:21,http://rosenbaum.name/mary,,144.238.68.237,"{""location"": ""PW"", ""is_mobile"": true}" 5284,2,116,2017-01-27 20:39:55,http://vonprosacco.name/bertram,,155.198.105.54,"{""location"": ""VN"", ""is_mobile"": true}" 18204,7,412,2017-01-04 18:43:34,http://doylewiegand.co/lila,0.3290597401,79.99.155.166,"{""location"": ""GF"", ""is_mobile"": true}" 18205,7,412,2017-05-19 12:12:34,http://nicolashodkiewicz.biz/paula.koepp,0.2387989665,37.63.253.87,"{""location"": ""MG"", ""is_mobile"": true}" 18206,7,412,2017-05-04 05:16:55,http://emmerich.info/lesly.doyle,0.1129051210,88.70.145.249,"{""location"": ""EG"", ""is_mobile"": true}" 18207,7,412,2017-02-01 22:59:15,http://casperkeler.biz/romaine_kuhic,0.8367403998,64.96.207.14,"{""location"": ""HM"", ""is_mobile"": true}" 18208,7,412,2017-04-15 03:23:42,http://gusikowski.info/cathy.pagac,0.8062288962,251.207.194.66,"{""location"": ""FK"", ""is_mobile"": false}" 18209,7,412,2017-03-16 20:51:06,http://osinski.name/ignatius_lueilwitz,0.0202311535,66.212.203.212,"{""location"": ""VI"", ""is_mobile"": true}" 18210,7,412,2017-02-25 16:16:46,http://lefflersmitham.io/roselyn_zulauf,0.4649378848,26.170.60.196,"{""location"": ""CZ"", ""is_mobile"": true}" 18211,7,412,2017-05-01 14:23:27,http://creminpadberg.info/darien.becker,0.3505750577,144.135.76.165,"{""location"": ""KM"", ""is_mobile"": true}" 18212,7,412,2017-06-01 05:23:03,http://collieroconnell.biz/davion_grant,0.2656894126,238.243.90.61,"{""location"": ""JM"", ""is_mobile"": true}" 18213,7,412,2016-12-16 15:19:04,http://kulas.io/urban_prohaska,0.4349147808,94.160.230.189,"{""location"": ""BZ"", ""is_mobile"": false}" 18214,7,412,2017-06-12 11:09:42,http://auer.net/amari_jast,0.3545578736,231.133.201.55,"{""location"": ""TZ"", ""is_mobile"": true}" 18215,7,412,2017-02-20 16:07:14,http://gusikowski.net/darien,0.7843047747,56.184.211.14,"{""location"": ""MO"", ""is_mobile"": false}" 18216,7,412,2017-04-23 11:31:16,http://jacobi.biz/estelle_stoltenberg,0.3283227170,225.5.163.58,"{""location"": ""QA"", ""is_mobile"": true}" 18217,7,412,2017-03-03 03:39:15,http://lemke.name/keely_nolan,0.0504542907,247.139.204.38,"{""location"": ""JM"", ""is_mobile"": false}" 18218,7,412,2017-04-24 16:54:03,http://olson.biz/josh,0.9849161055,46.216.116.158,"{""location"": ""FR"", ""is_mobile"": true}" 18219,7,412,2017-05-19 01:20:18,http://schuppe.info/giovanny,0.3512212332,192.109.217.155,"{""location"": ""MS"", ""is_mobile"": true}" 18220,7,412,2016-12-19 08:02:25,http://framischultz.info/oran,0.8722886873,130.82.79.189,"{""location"": ""TK"", ""is_mobile"": true}" 18221,7,412,2017-01-18 08:36:47,http://kunde.info/margarete_cremin,0.0636933698,82.41.236.154,"{""location"": ""SL"", ""is_mobile"": true}" 18222,7,412,2017-02-05 22:08:03,http://christiansen.org/chelsie,0.2410600369,137.233.206.102,"{""location"": ""CX"", ""is_mobile"": false}" 18223,7,412,2017-04-10 00:01:12,http://kutch.com/shaylee_gusikowski,0.0465149048,110.65.213.195,"{""location"": ""BS"", ""is_mobile"": false}" 18224,7,413,2017-03-12 07:22:31,http://kshlerinbeer.co/sasha.brekke,0.2984459835,54.209.246.130,"{""location"": ""GG"", ""is_mobile"": false}" 18225,7,413,2017-01-09 19:14:48,http://reichelklein.biz/rebecca,0.9182663226,20.82.137.219,"{""location"": ""UY"", ""is_mobile"": true}" 18226,7,413,2017-01-28 17:23:53,http://bogisichkovacek.biz/solon_kertzmann,0.1394795326,149.130.221.90,"{""location"": ""GU"", ""is_mobile"": true}" 18227,7,413,2017-01-10 02:20:22,http://carterbernhard.net/thad,0.6196317298,145.94.102.64,"{""location"": ""MD"", ""is_mobile"": true}" 18228,7,413,2017-03-31 21:54:52,http://gerhold.info/francesco.koelpin,0.0873193642,161.87.8.151,"{""location"": ""IQ"", ""is_mobile"": true}" 18229,7,413,2017-01-02 06:33:05,http://bergekeeling.org/lauren,0.0129682477,250.102.124.161,"{""location"": ""KW"", ""is_mobile"": true}" 18230,7,413,2017-05-28 01:22:56,http://schumm.info/gwendolyn,0.4368173895,127.244.94.147,"{""location"": ""CW"", ""is_mobile"": false}" 18231,7,413,2017-02-12 07:06:14,http://lehnerfeest.biz/cletus_murazik,0.4053395304,209.241.221.67,"{""location"": ""SL"", ""is_mobile"": false}" 18232,7,413,2017-04-05 00:26:38,http://shanahan.name/arielle,0.7209472737,186.162.254.158,"{""location"": ""HK"", ""is_mobile"": true}" 18233,7,413,2017-05-09 06:17:27,http://fritsch.org/brenda.dickinson,0.5336984521,122.48.30.114,"{""location"": ""TR"", ""is_mobile"": false}" 18234,7,413,2017-04-10 06:15:46,http://zboncak.biz/amie_roberts,0.2549330612,97.224.48.23,"{""location"": ""LA"", ""is_mobile"": true}" 18235,7,413,2017-01-26 07:45:16,http://ondricka.biz/terrence.ziemann,0.8904361777,115.53.140.36,"{""location"": ""IS"", ""is_mobile"": true}" 18236,7,413,2017-01-07 05:02:02,http://pollichmacgyver.name/jonathan,0.4121121016,47.93.253.245,"{""location"": ""TH"", ""is_mobile"": true}" 18237,7,413,2017-04-29 13:17:25,http://champlinbradtke.io/owen_stokes,0.4893869482,203.45.211.26,"{""location"": ""MC"", ""is_mobile"": true}" 18238,7,413,2016-12-29 01:31:04,http://crooks.org/briana.simonis,0.2003453251,53.65.69.231,"{""location"": ""MD"", ""is_mobile"": false}" 18239,7,413,2017-02-11 02:30:46,http://weber.io/gaston,0.9705766445,101.164.123.98,"{""location"": ""EE"", ""is_mobile"": true}" 18240,7,413,2017-04-12 07:36:21,http://konopelski.co/emmie,0.7764639309,167.143.108.110,"{""location"": ""VI"", ""is_mobile"": false}" 18241,7,413,2017-03-14 14:15:40,http://nader.io/tristin,0.4295140544,52.79.228.206,"{""location"": ""RO"", ""is_mobile"": false}" 18242,7,413,2017-01-22 21:55:57,http://weinat.name/marvin,0.2219808952,202.109.152.10,"{""location"": ""IT"", ""is_mobile"": true}" 18243,7,413,2017-05-21 17:08:10,http://ledner.com/darrick,0.1617750355,61.67.142.237,"{""location"": ""VU"", ""is_mobile"": true}" 18244,7,413,2017-05-14 15:36:11,http://tillmanbeer.com/sigurd,0.6574607424,63.95.190.127,"{""location"": ""GE"", ""is_mobile"": true}" 18245,7,413,2017-02-20 17:16:23,http://wisokycole.name/alyson,0.8661699089,243.81.23.140,"{""location"": ""MF"", ""is_mobile"": true}" 18246,7,413,2017-02-13 13:28:22,http://cainwiegand.org/loyce_hilll,0.8388750611,243.89.129.182,"{""location"": ""CY"", ""is_mobile"": true}" 18247,7,413,2017-04-04 19:32:22,http://metz.com/dereck.will,0.8557535488,163.4.79.181,"{""location"": ""AI"", ""is_mobile"": true}" 18248,7,413,2017-05-06 14:45:39,http://crooksheel.io/adrienne_braun,0.2859725312,197.140.177.246,"{""location"": ""MU"", ""is_mobile"": false}" 18249,7,413,2017-03-01 07:54:44,http://blickauer.name/luciano.kovacek,0.6083793394,198.76.140.156,"{""location"": ""KM"", ""is_mobile"": true}" 18250,7,413,2017-01-04 13:48:25,http://franecki.name/mohamed.marks,0.1037669128,186.240.189.51,"{""location"": ""BT"", ""is_mobile"": false}" 18251,7,413,2017-01-27 10:19:39,http://abshire.biz/logan.johnston,0.2402437716,136.229.229.209,"{""location"": ""SK"", ""is_mobile"": true}" 18252,7,413,2017-04-11 15:18:43,http://ratkedare.net/adolfo_waelchi,0.6926983212,57.112.114.98,"{""location"": ""GG"", ""is_mobile"": false}" 18253,7,413,2017-04-22 11:56:50,http://sipes.info/annetta.sawayn,0.7383155412,241.53.63.247,"{""location"": ""ET"", ""is_mobile"": true}" 18254,7,413,2017-02-17 01:26:30,http://hillsthiel.com/moises,0.2941057518,71.73.110.98,"{""location"": ""PH"", ""is_mobile"": true}" 5285,2,116,2017-06-11 06:57:25,http://stiedemannhackett.info/diana,,216.229.231.31,"{""location"": ""FK"", ""is_mobile"": false}" 5286,2,116,2017-04-13 05:28:54,http://harrisbechtelar.co/ardella,,32.166.3.201,"{""location"": ""GS"", ""is_mobile"": false}" 5287,2,116,2017-04-28 21:29:50,http://wiegand.biz/gerald,,119.12.127.45,"{""location"": ""SE"", ""is_mobile"": false}" 5288,2,116,2017-05-12 23:15:34,http://keeblerreichel.name/sedrick,,100.118.188.225,"{""location"": ""SC"", ""is_mobile"": false}" 5289,2,116,2017-04-15 02:01:27,http://lindgren.info/clemens,,49.232.141.110,"{""location"": ""GD"", ""is_mobile"": false}" 5290,2,117,2017-02-10 13:13:54,http://hickle.net/griffin_gleason,,24.143.253.39,"{""location"": ""MW"", ""is_mobile"": false}" 5291,2,117,2017-03-14 01:31:15,http://weinatbaumbach.com/tillman_will,,44.145.34.104,"{""location"": ""IL"", ""is_mobile"": false}" 5292,2,117,2017-03-10 23:17:04,http://shields.name/pat_nienow,,84.80.59.219,"{""location"": ""CD"", ""is_mobile"": true}" 5293,2,117,2017-05-21 12:18:38,http://kriskoelpin.net/jarod,,7.100.195.193,"{""location"": ""PS"", ""is_mobile"": false}" 5294,2,117,2017-01-20 02:27:34,http://moen.name/unique,,39.248.113.228,"{""location"": ""MN"", ""is_mobile"": false}" 5295,2,117,2017-03-21 07:03:52,http://pollich.com/westley,,10.66.75.157,"{""location"": ""BW"", ""is_mobile"": true}" 5296,2,117,2017-03-12 00:51:38,http://howelllehner.co/camille,,239.81.98.251,"{""location"": ""PF"", ""is_mobile"": true}" 5297,2,117,2017-05-01 09:45:24,http://lesch.info/xzavier_torphy,,89.13.120.151,"{""location"": ""JP"", ""is_mobile"": true}" 5298,2,117,2017-05-16 22:59:30,http://monahangorczany.org/julianne,,127.4.65.251,"{""location"": ""PT"", ""is_mobile"": true}" 5299,2,117,2017-01-27 12:12:10,http://creminbosco.org/kamren_wolff,,177.242.93.50,"{""location"": ""AQ"", ""is_mobile"": false}" 5300,2,117,2017-01-08 00:08:14,http://bayer.info/vivianne,,152.120.207.61,"{""location"": ""TC"", ""is_mobile"": true}" 5301,2,117,2016-12-13 06:41:34,http://boyle.co/rosa,,103.91.54.147,"{""location"": ""AT"", ""is_mobile"": false}" 5302,2,117,2017-01-11 19:08:56,http://oconnell.net/stephon_schuster,,228.157.155.60,"{""location"": ""GS"", ""is_mobile"": false}" 5303,2,117,2017-02-02 02:42:06,http://hamill.biz/noelia_cummerata,,126.234.44.202,"{""location"": ""DE"", ""is_mobile"": false}" 5304,2,117,2017-02-23 07:02:11,http://schulist.org/weston,,218.134.198.95,"{""location"": ""FO"", ""is_mobile"": true}" 5305,2,117,2017-03-18 15:21:52,http://emmerich.name/antwon,,120.93.77.217,"{""location"": ""AR"", ""is_mobile"": false}" 5306,2,117,2017-02-15 23:40:57,http://halvorson.net/dan,,96.244.145.35,"{""location"": ""TF"", ""is_mobile"": true}" 5307,2,117,2017-05-06 11:34:47,http://grantgoldner.co/rafael.krajcik,,29.57.82.6,"{""location"": ""BH"", ""is_mobile"": false}" 5308,2,117,2017-03-11 22:07:22,http://macejkovic.net/buster,,109.41.97.234,"{""location"": ""KI"", ""is_mobile"": false}" 5309,2,117,2017-05-03 14:34:09,http://swaniawskitorp.net/caleigh,,172.229.23.208,"{""location"": ""EH"", ""is_mobile"": false}" 5310,2,117,2017-02-08 04:47:03,http://kreiger.biz/nicklaus.turner,,77.106.97.21,"{""location"": ""WS"", ""is_mobile"": false}" 5311,2,117,2017-02-25 22:02:39,http://barton.com/deanna,,180.225.191.231,"{""location"": ""SJ"", ""is_mobile"": false}" 5312,2,117,2017-02-12 09:36:44,http://kihnortiz.biz/bertha,,98.151.23.41,"{""location"": ""CZ"", ""is_mobile"": false}" 5313,2,117,2017-04-06 16:20:06,http://grahamwintheiser.org/jason_muller,,121.249.243.150,"{""location"": ""SZ"", ""is_mobile"": false}" 5314,2,117,2017-01-16 16:26:25,http://kiehnbuckridge.net/shayne_douglas,,184.220.105.228,"{""location"": ""BJ"", ""is_mobile"": false}" 5315,2,117,2017-02-08 03:25:20,http://friesen.name/rose,,188.57.92.141,"{""location"": ""SN"", ""is_mobile"": false}" 5316,2,117,2017-04-08 09:43:07,http://fay.co/arlene.reinger,,166.156.164.15,"{""location"": ""FM"", ""is_mobile"": true}" 5317,2,117,2017-02-15 21:20:20,http://jakubowski.biz/fernando.wisoky,,106.52.130.244,"{""location"": ""NA"", ""is_mobile"": true}" 5318,2,117,2017-06-07 12:22:14,http://friesenbeer.info/newton,,144.150.217.211,"{""location"": ""IN"", ""is_mobile"": false}" 5319,2,117,2017-01-15 13:57:21,http://waelchi.com/mortimer_weimann,,206.167.215.170,"{""location"": ""MP"", ""is_mobile"": true}" 5320,2,117,2017-02-25 21:41:52,http://jenkinwaniawski.info/rebecca,,83.119.202.201,"{""location"": ""EG"", ""is_mobile"": false}" 5321,2,117,2017-06-04 14:26:31,http://spinka.net/toby.cole,,146.243.38.198,"{""location"": ""MZ"", ""is_mobile"": false}" 5322,2,117,2017-06-11 21:49:12,http://bergstrom.info/eve_quigley,,41.176.120.16,"{""location"": ""UG"", ""is_mobile"": false}" 5323,2,117,2017-03-02 13:09:27,http://johnson.info/madalyn,,22.90.164.89,"{""location"": ""IM"", ""is_mobile"": true}" 5324,2,117,2017-02-25 23:37:30,http://cormier.biz/ora,,20.112.210.124,"{""location"": ""LV"", ""is_mobile"": true}" 5325,2,117,2017-06-13 10:14:22,http://schaefer.org/mikel,,30.201.27.242,"{""location"": ""SN"", ""is_mobile"": false}" 5326,2,117,2017-01-26 02:17:44,http://rempelbogan.info/melany,,122.122.164.227,"{""location"": ""NO"", ""is_mobile"": true}" 5327,2,117,2017-06-06 01:04:15,http://krajcik.biz/rodolfo,,114.231.51.73,"{""location"": ""WF"", ""is_mobile"": false}" 5328,2,117,2017-02-08 06:09:27,http://hansenmante.name/allison,,112.145.57.217,"{""location"": ""MC"", ""is_mobile"": false}" 5329,2,117,2017-01-05 04:42:32,http://schneider.com/raleigh_stark,,69.171.53.112,"{""location"": ""CR"", ""is_mobile"": true}" 5330,2,117,2017-01-13 21:23:47,http://emard.com/monte.purdy,,97.225.129.2,"{""location"": ""GL"", ""is_mobile"": true}" 5331,2,117,2017-06-05 13:01:13,http://frami.co/jaleel_friesen,,93.26.241.64,"{""location"": ""BN"", ""is_mobile"": false}" 5332,2,117,2017-05-09 02:31:52,http://hansenquitzon.org/sonny.kovacek,,136.160.209.19,"{""location"": ""PW"", ""is_mobile"": false}" 5333,2,117,2017-04-12 08:15:26,http://weinat.org/mac,,99.142.77.215,"{""location"": ""QA"", ""is_mobile"": false}" 5334,2,117,2017-02-18 23:58:22,http://considinebergnaum.name/blair.tillman,,64.83.62.63,"{""location"": ""GD"", ""is_mobile"": true}" 5335,2,117,2017-05-11 09:18:49,http://sauer.net/mckenna.cruickshank,,194.226.178.214,"{""location"": ""BQ"", ""is_mobile"": true}" 5336,2,117,2017-04-20 15:06:30,http://ondrickarutherford.net/mario.volkman,,74.31.14.223,"{""location"": ""BZ"", ""is_mobile"": true}" 5337,2,117,2017-03-13 11:56:18,http://fahey.biz/giovanni,,3.195.50.229,"{""location"": ""LR"", ""is_mobile"": true}" 5338,2,117,2017-04-23 07:13:41,http://goldner.co/beie_daugherty,,114.201.237.76,"{""location"": ""AQ"", ""is_mobile"": false}" 5339,2,117,2017-06-03 18:03:37,http://schmidtglover.io/lavina,,206.48.19.124,"{""location"": ""FJ"", ""is_mobile"": true}" 18255,7,413,2017-05-31 14:53:41,http://waters.name/dalton,0.9013478278,230.154.78.65,"{""location"": ""RU"", ""is_mobile"": true}" 18256,7,413,2016-12-20 16:17:20,http://schultz.com/ivory_gusikowski,0.5175505230,44.210.148.194,"{""location"": ""ST"", ""is_mobile"": true}" 18257,7,413,2016-12-14 01:34:12,http://treutelcasper.com/olaf,0.6439367974,247.237.12.35,"{""location"": ""EE"", ""is_mobile"": true}" 18258,7,413,2017-02-08 16:09:44,http://connelly.info/monserrate.satterfield,0.8959012724,143.240.239.135,"{""location"": ""CG"", ""is_mobile"": true}" 18259,7,413,2017-03-01 18:43:49,http://mraz.io/norbert_graham,0.0312851970,103.37.87.219,"{""location"": ""MV"", ""is_mobile"": true}" 18260,7,413,2017-05-16 03:26:38,http://rolfsonwaters.info/quentin_hagenes,0.7973845640,201.224.136.123,"{""location"": ""CA"", ""is_mobile"": true}" 18261,7,413,2017-01-11 13:55:38,http://schumm.info/chanelle,0.6937589904,50.72.186.216,"{""location"": ""AE"", ""is_mobile"": true}" 18262,7,413,2017-05-23 16:59:51,http://oconnerharvey.io/floie.kemmer,0.4887354456,33.233.50.200,"{""location"": ""CV"", ""is_mobile"": true}" 18263,7,413,2017-01-06 13:51:21,http://greenfelder.co/jermaine.schmeler,0.1359947353,119.27.75.47,"{""location"": ""BZ"", ""is_mobile"": false}" 18264,7,413,2017-04-26 00:35:31,http://keebler.com/easter,0.9430812407,18.74.118.5,"{""location"": ""NE"", ""is_mobile"": true}" 18265,7,413,2017-02-09 20:49:49,http://dooley.biz/mariah,0.1612580775,210.121.89.51,"{""location"": ""GM"", ""is_mobile"": true}" 18266,7,413,2017-05-11 16:35:03,http://pacocha.name/daphney.harvey,0.9105187406,165.131.162.39,"{""location"": ""AT"", ""is_mobile"": false}" 18267,7,413,2017-04-27 19:16:55,http://thompsoncrona.io/deshawn.cain,0.3240834556,167.180.56.196,"{""location"": ""TO"", ""is_mobile"": true}" 18268,7,413,2017-06-03 14:11:14,http://bosco.net/jazmyne.hammes,0.0247810521,156.115.21.142,"{""location"": ""AU"", ""is_mobile"": true}" 18269,7,413,2017-03-06 09:15:26,http://schowalterkoepp.com/hillard.stroman,0.3895253312,94.82.10.110,"{""location"": ""FM"", ""is_mobile"": false}" 18270,7,413,2017-04-07 17:28:36,http://wunsch.com/jolie,0.4561257136,251.11.163.71,"{""location"": ""TK"", ""is_mobile"": true}" 18271,7,413,2017-05-06 23:00:19,http://schiller.name/caroline.murray,0.7837829566,122.197.144.4,"{""location"": ""NA"", ""is_mobile"": false}" 18272,7,413,2017-02-24 12:50:06,http://kaulkekulas.com/kayli,0.7953028497,152.33.199.168,"{""location"": ""LI"", ""is_mobile"": false}" 18273,7,413,2017-04-23 18:50:32,http://boyer.org/tillman,0.2293116863,126.97.169.95,"{""location"": ""GB"", ""is_mobile"": true}" 18274,7,413,2017-03-25 07:22:36,http://fritsch.co/emilie_kulas,0.1777002933,14.47.189.3,"{""location"": ""PG"", ""is_mobile"": false}" 18275,7,413,2017-01-26 07:06:05,http://mcclure.info/ivah,0.1713454340,188.122.136.35,"{""location"": ""IQ"", ""is_mobile"": false}" 18276,7,413,2017-03-15 13:37:10,http://paucek.org/rahul,0.0103650447,193.79.225.203,"{""location"": ""BJ"", ""is_mobile"": true}" 18277,7,413,2017-04-13 07:00:11,http://heathcote.com/justyn_gottlieb,0.2144179597,66.25.204.248,"{""location"": ""PN"", ""is_mobile"": true}" 18278,7,413,2016-12-14 01:14:15,http://kilbacknicolas.info/arnaldo.funk,0.6845809790,197.226.115.148,"{""location"": ""SL"", ""is_mobile"": true}" 18279,7,413,2017-02-06 10:25:20,http://greenkonopelski.net/frankie,0.1590915518,91.8.18.169,"{""location"": ""BR"", ""is_mobile"": false}" 18280,7,413,2017-04-01 04:20:33,http://oconnell.com/georgianna.abshire,0.6365974172,49.220.24.253,"{""location"": ""CU"", ""is_mobile"": false}" 18281,7,413,2017-02-09 09:39:08,http://jacobson.name/talon,0.8755106178,38.2.56.22,"{""location"": ""AW"", ""is_mobile"": true}" 18282,7,413,2017-01-06 18:10:02,http://wilkinson.com/vallie_hartmann,0.7163069719,25.128.200.68,"{""location"": ""NR"", ""is_mobile"": false}" 18283,7,413,2017-05-14 09:31:56,http://ruecker.com/eulalia.blanda,0.1918535759,191.82.161.162,"{""location"": ""RO"", ""is_mobile"": false}" 18284,7,413,2017-03-18 06:26:49,http://mohrwintheiser.info/callie_armstrong,0.9563819129,192.82.190.72,"{""location"": ""BY"", ""is_mobile"": true}" 18285,7,413,2017-01-22 02:39:01,http://hayesprohaska.info/herbert,0.6205671890,208.49.79.193,"{""location"": ""LY"", ""is_mobile"": true}" 18286,7,414,2017-01-29 17:00:43,http://yostsmith.io/drake_lemke,0.3471171621,110.250.228.129,"{""location"": ""TM"", ""is_mobile"": true}" 18287,7,414,2017-04-20 09:01:03,http://aufderhar.co/belle_botsford,0.4261460224,53.50.107.85,"{""location"": ""IR"", ""is_mobile"": true}" 18288,7,414,2017-05-02 17:38:54,http://emard.name/antonia.sporer,0.3257829693,200.44.79.182,"{""location"": ""NC"", ""is_mobile"": false}" 18289,7,414,2017-04-12 19:11:40,http://marvindooley.net/marge,0.4265294089,156.16.98.9,"{""location"": ""LK"", ""is_mobile"": true}" 18290,7,414,2017-04-13 11:01:18,http://romaguera.biz/isobel,0.7755255087,205.223.55.80,"{""location"": ""SA"", ""is_mobile"": true}" 18291,7,414,2017-02-03 01:30:24,http://jakubowski.name/ivy_runte,0.6707866046,127.30.57.158,"{""location"": ""DK"", ""is_mobile"": true}" 18292,7,414,2017-05-10 06:57:02,http://wisoky.info/kurt_king,0.5296781054,71.140.16.246,"{""location"": ""IE"", ""is_mobile"": false}" 18293,7,414,2017-02-13 18:45:14,http://spinka.co/wilhelmine,0.9517529948,151.165.29.99,"{""location"": ""SH"", ""is_mobile"": true}" 18294,7,414,2017-06-02 00:33:40,http://kulas.io/aidan,0.5903294340,38.251.220.20,"{""location"": ""SZ"", ""is_mobile"": true}" 18295,7,414,2017-05-13 13:43:10,http://lehner.io/karley_kling,0.6276331548,8.22.139.133,"{""location"": ""KZ"", ""is_mobile"": true}" 18296,7,414,2016-12-29 03:40:28,http://wolff.name/baby_turcotte,0.6193567280,188.217.42.53,"{""location"": ""NU"", ""is_mobile"": false}" 18297,7,414,2017-01-14 17:30:56,http://gulgowski.io/isabelle.bode,0.8469381621,132.135.10.39,"{""location"": ""ST"", ""is_mobile"": false}" 18298,7,414,2017-04-25 00:03:43,http://howe.net/michel.hansen,0.4334255996,236.2.79.41,"{""location"": ""SY"", ""is_mobile"": true}" 18299,7,414,2017-05-06 15:17:08,http://toy.io/beulah,0.9754207723,70.175.48.223,"{""location"": ""BQ"", ""is_mobile"": false}" 18300,7,414,2016-12-30 21:44:46,http://ullrichhermiston.com/jett,0.3454432664,223.17.211.54,"{""location"": ""SM"", ""is_mobile"": false}" 18301,7,414,2017-05-18 00:18:44,http://goodwinbrakus.info/alvera,0.3355344511,145.6.174.237,"{""location"": ""TF"", ""is_mobile"": false}" 18302,7,414,2017-05-03 14:45:34,http://greenholt.net/derrick_herzog,0.2678260256,54.200.21.151,"{""location"": ""TM"", ""is_mobile"": false}" 18303,7,414,2017-02-08 16:01:39,http://hoeger.biz/manley.oconner,0.1364505300,241.210.188.35,"{""location"": ""AE"", ""is_mobile"": true}" 18304,7,414,2017-05-06 06:23:25,http://leuschke.io/jonas,0.4403285856,166.50.114.172,"{""location"": ""AZ"", ""is_mobile"": false}" 18305,7,414,2017-03-01 00:47:46,http://waters.info/tamara_brakus,0.2485162583,135.174.3.128,"{""location"": ""HT"", ""is_mobile"": false}" 5340,2,117,2017-01-23 16:50:29,http://boyer.com/leanna,,8.218.35.157,"{""location"": ""NG"", ""is_mobile"": false}" 5341,2,117,2017-06-10 09:21:08,http://kuhn.org/eden_glover,,30.231.98.210,"{""location"": ""KI"", ""is_mobile"": true}" 5342,2,117,2017-02-13 23:55:33,http://terryaufderhar.name/tremaine,,2.180.204.225,"{""location"": ""HR"", ""is_mobile"": false}" 5343,2,117,2017-01-11 10:35:15,http://zboncak.net/aracely,,241.70.105.193,"{""location"": ""GW"", ""is_mobile"": true}" 5344,2,117,2017-03-22 04:21:22,http://legrosdaniel.com/mikayla,,6.193.120.129,"{""location"": ""BB"", ""is_mobile"": false}" 5345,2,117,2017-01-02 15:58:31,http://rippinquigley.net/rocky,,204.96.176.202,"{""location"": ""SB"", ""is_mobile"": false}" 5346,2,117,2016-12-22 03:39:33,http://stamm.name/jacques,,47.132.145.213,"{""location"": ""TZ"", ""is_mobile"": true}" 5347,2,117,2017-01-13 09:25:37,http://lednerrau.io/tracey_harber,,182.137.240.156,"{""location"": ""IM"", ""is_mobile"": false}" 5348,2,117,2017-03-28 00:30:33,http://huel.biz/christop,,241.238.180.64,"{""location"": ""SB"", ""is_mobile"": false}" 5349,2,117,2017-01-08 16:30:18,http://renner.co/imani,,168.134.161.196,"{""location"": ""ZM"", ""is_mobile"": true}" 5350,2,117,2017-06-10 10:47:11,http://lindhuel.co/kareem,,97.204.248.101,"{""location"": ""FO"", ""is_mobile"": false}" 5351,2,117,2017-01-11 17:57:44,http://kulas.info/guadalupe.waters,,15.220.94.205,"{""location"": ""LC"", ""is_mobile"": false}" 5352,2,117,2017-03-29 01:08:20,http://welchkling.com/parker,,36.216.73.2,"{""location"": ""GQ"", ""is_mobile"": true}" 5353,2,117,2017-04-25 04:28:44,http://robel.biz/elaina,,102.164.109.218,"{""location"": ""HU"", ""is_mobile"": true}" 5354,2,117,2017-01-30 07:21:46,http://bodekuphal.info/vito_konopelski,,45.149.5.134,"{""location"": ""MS"", ""is_mobile"": true}" 5355,2,117,2017-01-03 16:25:56,http://bartell.name/demetrius_ankunding,,49.16.93.107,"{""location"": ""VA"", ""is_mobile"": false}" 5356,2,117,2017-01-28 11:51:13,http://schroeder.net/ursula,,132.131.247.208,"{""location"": ""DK"", ""is_mobile"": true}" 5357,2,118,2017-04-23 04:17:24,http://osinski.co/beaulah_stroman,,25.48.154.83,"{""location"": ""KM"", ""is_mobile"": false}" 5358,2,118,2016-12-21 07:31:11,http://marks.io/jeyca.ebert,,84.139.175.174,"{""location"": ""MT"", ""is_mobile"": true}" 5359,2,118,2017-03-13 10:55:23,http://durgan.net/donald.hagenes,,122.167.33.116,"{""location"": ""GE"", ""is_mobile"": true}" 5360,2,118,2017-03-04 13:35:18,http://mayeroconnell.biz/vernice,,77.206.143.97,"{""location"": ""GP"", ""is_mobile"": false}" 5361,2,118,2017-05-26 02:47:19,http://quigley.net/jason,,55.239.243.78,"{""location"": ""AT"", ""is_mobile"": true}" 5362,2,118,2017-02-27 23:38:18,http://beckerzulauf.biz/velva,,139.39.46.249,"{""location"": ""UY"", ""is_mobile"": true}" 5363,2,118,2016-12-24 18:03:23,http://gaylord.org/alba_lesch,,78.35.191.111,"{""location"": ""MV"", ""is_mobile"": false}" 5364,2,118,2017-05-25 22:27:52,http://ankunding.co/mitchell,,136.206.166.74,"{""location"": ""TF"", ""is_mobile"": true}" 5365,2,118,2017-05-14 02:56:58,http://rogahnbeer.co/leanne_wuckert,,181.92.207.143,"{""location"": ""MZ"", ""is_mobile"": true}" 5366,2,118,2016-12-31 17:15:50,http://gislason.net/otto_walsh,,152.254.228.73,"{""location"": ""DE"", ""is_mobile"": true}" 5367,2,118,2017-02-17 15:21:06,http://feest.biz/elda,,198.30.225.179,"{""location"": ""MG"", ""is_mobile"": true}" 5368,2,118,2017-05-13 20:01:36,http://baumbach.info/katheryn,,214.148.8.15,"{""location"": ""NZ"", ""is_mobile"": true}" 5369,2,118,2016-12-29 03:09:07,http://robelboyle.co/freddie_armstrong,,104.49.107.228,"{""location"": ""TK"", ""is_mobile"": false}" 5370,2,118,2017-04-25 08:15:54,http://labadie.com/nickolas,,75.216.205.204,"{""location"": ""GQ"", ""is_mobile"": false}" 5371,2,118,2017-05-11 15:26:41,http://klein.com/elmo.smith,,26.190.54.22,"{""location"": ""SG"", ""is_mobile"": true}" 5372,2,118,2017-02-04 21:57:14,http://nitzsche.io/carolina_nader,,145.128.134.20,"{""location"": ""PR"", ""is_mobile"": false}" 5373,2,118,2017-04-24 19:39:19,http://cainkiehn.biz/elwyn,,140.114.34.92,"{""location"": ""BT"", ""is_mobile"": true}" 5374,2,118,2016-12-31 12:31:27,http://keler.name/noel,,179.32.231.69,"{""location"": ""TK"", ""is_mobile"": true}" 5375,2,118,2017-05-09 04:47:54,http://corwinjast.co/jackeline,,223.46.34.60,"{""location"": ""UM"", ""is_mobile"": true}" 5376,2,118,2017-02-12 01:24:36,http://bode.name/melvina_schneider,,125.12.218.11,"{""location"": ""GH"", ""is_mobile"": true}" 5377,2,118,2017-05-13 22:38:01,http://goldner.name/graham,,76.214.87.189,"{""location"": ""GQ"", ""is_mobile"": true}" 5378,2,118,2017-03-20 15:17:44,http://waelchi.biz/rolando,,58.82.121.145,"{""location"": ""GP"", ""is_mobile"": false}" 5379,2,118,2017-04-13 23:48:04,http://gleichner.net/laurie_wolf,,113.9.119.224,"{""location"": ""SV"", ""is_mobile"": true}" 5380,2,118,2017-04-23 18:27:41,http://jenkins.io/myles_walter,,70.194.155.134,"{""location"": ""MX"", ""is_mobile"": false}" 5381,2,118,2017-04-14 21:20:41,http://feest.name/jennyfer_cain,,14.46.183.126,"{""location"": ""PF"", ""is_mobile"": false}" 5382,2,118,2017-01-26 23:23:27,http://nikolaus.co/adelbert,,136.207.184.21,"{""location"": ""MU"", ""is_mobile"": true}" 5383,2,118,2017-01-08 22:55:10,http://terry.co/pat.pouros,,44.10.138.63,"{""location"": ""SY"", ""is_mobile"": true}" 5384,2,118,2017-03-24 19:42:26,http://leuschkeherman.co/telly,,161.40.225.153,"{""location"": ""SZ"", ""is_mobile"": false}" 5385,2,118,2017-01-01 04:10:26,http://jerdewillms.com/allison_stracke,,113.242.207.237,"{""location"": ""AI"", ""is_mobile"": false}" 5386,2,118,2017-01-14 23:28:36,http://hilll.biz/vidal.pfeffer,,77.153.31.184,"{""location"": ""IE"", ""is_mobile"": true}" 5387,2,118,2017-01-18 11:24:49,http://lockman.com/marlee.leffler,,218.36.203.229,"{""location"": ""JM"", ""is_mobile"": true}" 5388,2,118,2017-03-24 03:09:24,http://lockman.biz/celine.armstrong,,143.205.232.65,"{""location"": ""PK"", ""is_mobile"": false}" 5389,2,118,2016-12-17 18:35:19,http://schowaltermoriette.io/violette,,84.164.154.21,"{""location"": ""BE"", ""is_mobile"": true}" 5390,2,118,2017-01-04 05:14:44,http://lowe.org/hailie.gaylord,,136.51.55.250,"{""location"": ""BB"", ""is_mobile"": false}" 5391,2,118,2017-06-04 02:14:19,http://ullrich.org/glennie.grady,,117.234.64.126,"{""location"": ""AU"", ""is_mobile"": false}" 5392,2,118,2017-04-07 21:03:16,http://mayert.biz/destin_wehner,,136.94.88.207,"{""location"": ""KM"", ""is_mobile"": true}" 5393,2,118,2017-02-25 05:46:41,http://smitham.io/winston.oberbrunner,,129.148.243.228,"{""location"": ""SE"", ""is_mobile"": true}" 5394,2,118,2017-04-23 17:54:47,http://hand.name/jordi.weinat,,36.57.232.125,"{""location"": ""MF"", ""is_mobile"": true}" 5395,2,118,2017-04-06 10:51:49,http://schummbins.biz/sedrick,,24.88.62.153,"{""location"": ""GL"", ""is_mobile"": true}" 18306,7,414,2017-04-02 20:41:31,http://mcdermott.net/elmo_hartmann,0.3384166287,216.208.222.229,"{""location"": ""BT"", ""is_mobile"": true}" 18307,7,414,2017-03-16 04:17:32,http://kelerdoyle.com/domenica_romaguera,0.8955626990,10.168.10.45,"{""location"": ""AT"", ""is_mobile"": true}" 18308,7,414,2017-04-02 11:20:13,http://sipes.net/louvenia_schinner,0.9150820092,241.138.42.70,"{""location"": ""NP"", ""is_mobile"": true}" 18309,7,414,2017-03-27 23:26:11,http://cruickshank.net/micah_sanford,0.7745826326,88.8.20.116,"{""location"": ""AX"", ""is_mobile"": true}" 18310,7,414,2017-06-12 10:08:51,http://schuppe.info/hipolito,0.4234439620,227.87.236.158,"{""location"": ""TT"", ""is_mobile"": false}" 18311,7,414,2016-12-19 07:00:36,http://pouros.com/oliver.torp,0.4424891707,87.86.212.103,"{""location"": ""DE"", ""is_mobile"": false}" 18312,7,414,2017-01-06 18:14:50,http://will.com/althea_murphy,0.9553044139,48.167.72.52,"{""location"": ""DO"", ""is_mobile"": false}" 18313,7,414,2017-05-29 04:28:30,http://torphyeffertz.co/wiley.schumm,0.2838863125,145.54.129.28,"{""location"": ""ES"", ""is_mobile"": true}" 18314,7,414,2017-03-13 22:31:20,http://grady.co/queenie.hartmann,0.5958307651,241.249.222.134,"{""location"": ""CV"", ""is_mobile"": true}" 18315,7,414,2017-04-02 00:44:12,http://bahringerhaag.net/lysanne_murray,0.6001990054,12.35.248.179,"{""location"": ""AF"", ""is_mobile"": true}" 18316,7,414,2017-02-01 06:30:56,http://nader.org/maryam,0.8590879427,161.61.66.172,"{""location"": ""HU"", ""is_mobile"": false}" 18317,7,414,2017-02-11 14:49:28,http://schoen.info/eden_veum,0.5982209259,199.217.239.234,"{""location"": ""ER"", ""is_mobile"": false}" 18318,7,414,2017-01-09 09:16:21,http://goodwinhintz.info/nola,0.6733536512,8.60.41.249,"{""location"": ""ZA"", ""is_mobile"": false}" 18319,7,414,2017-06-10 12:53:49,http://terry.biz/winnifred.cronin,0.4039770722,16.192.222.104,"{""location"": ""KR"", ""is_mobile"": false}" 18320,7,414,2017-03-04 12:42:44,http://ruelkuhic.name/archibald,0.4342060105,246.3.118.206,"{""location"": ""CH"", ""is_mobile"": true}" 18321,7,414,2017-04-02 20:03:20,http://dibbert.org/ocie,0.1498775225,167.91.46.135,"{""location"": ""AT"", ""is_mobile"": true}" 18322,7,414,2017-01-18 13:51:38,http://armstrong.biz/lazaro_considine,0.5973222551,224.193.140.91,"{""location"": ""KE"", ""is_mobile"": false}" 18323,7,414,2017-03-19 05:31:53,http://feest.io/maia.grimes,0.4433366735,3.177.236.214,"{""location"": ""AS"", ""is_mobile"": false}" 18324,7,414,2017-06-01 15:26:32,http://funk.co/malcolm,0.3202596868,56.169.66.224,"{""location"": ""JP"", ""is_mobile"": true}" 18325,7,414,2017-01-07 14:26:13,http://corwinmoore.co/jammie,0.3557559858,83.175.210.203,"{""location"": ""AX"", ""is_mobile"": true}" 18326,7,414,2017-02-02 02:19:28,http://haley.net/jonathon,0.5828188318,105.56.159.194,"{""location"": ""NI"", ""is_mobile"": true}" 18327,7,414,2017-05-31 19:22:24,http://howe.com/caden.mante,0.4879809430,225.94.136.242,"{""location"": ""AU"", ""is_mobile"": false}" 18328,7,414,2017-02-06 13:41:13,http://ruelmosciski.name/morgan.kerluke,0.3001206586,190.241.121.183,"{""location"": ""GW"", ""is_mobile"": false}" 18329,7,414,2017-05-12 20:01:09,http://larkin.biz/hulda.steuber,0.1903106680,17.169.156.199,"{""location"": ""DE"", ""is_mobile"": true}" 18330,7,414,2017-03-14 15:31:12,http://hammes.org/kenna,0.4353369766,198.231.223.99,"{""location"": ""CC"", ""is_mobile"": true}" 18331,7,414,2017-02-04 13:32:33,http://hageneshintz.co/rodolfo,0.4966014856,44.214.86.249,"{""location"": ""VC"", ""is_mobile"": false}" 18332,7,414,2017-03-28 05:59:04,http://friesenorn.net/jose_kunze,0.3520171442,235.248.252.213,"{""location"": ""BZ"", ""is_mobile"": true}" 18333,7,414,2017-04-17 17:18:32,http://harber.org/devante.langworth,0.8598844506,61.241.159.82,"{""location"": ""GF"", ""is_mobile"": true}" 18334,7,414,2017-04-26 15:42:50,http://smitham.info/maryjane.beer,0.8075898531,6.116.159.95,"{""location"": ""HN"", ""is_mobile"": false}" 18335,7,414,2017-02-25 14:22:52,http://rempelkihn.co/greta_robel,0.2548119771,170.66.130.204,"{""location"": ""MV"", ""is_mobile"": false}" 18336,7,414,2017-03-31 15:52:18,http://ratke.biz/buford_batz,0.9262118086,95.161.142.178,"{""location"": ""TT"", ""is_mobile"": false}" 18337,7,414,2017-03-15 22:24:38,http://gutmann.io/litzy,0.6617881228,21.139.197.106,"{""location"": ""MU"", ""is_mobile"": false}" 18338,7,414,2017-04-08 07:17:31,http://veumdach.org/kenyatta.hodkiewicz,0.0342793004,75.189.73.180,"{""location"": ""NI"", ""is_mobile"": true}" 18339,7,414,2017-04-03 14:38:56,http://rice.info/armani_jaskolski,0.7877287657,172.247.106.213,"{""location"": ""DJ"", ""is_mobile"": false}" 18340,7,414,2017-03-25 12:30:18,http://hammes.net/bernie,0.3839381026,188.25.233.169,"{""location"": ""LC"", ""is_mobile"": true}" 18341,7,415,2017-05-23 15:43:49,http://flatleyrosenbaum.info/darryl,0.0986228818,6.239.193.210,"{""location"": ""LB"", ""is_mobile"": true}" 18342,7,415,2017-05-08 21:38:19,http://douglas.io/quinton,0.3063229357,47.249.138.151,"{""location"": ""NC"", ""is_mobile"": false}" 18343,7,415,2017-06-12 22:48:02,http://ohara.io/frederick.kutch,0.1187569177,23.132.6.223,"{""location"": ""NI"", ""is_mobile"": true}" 18344,7,415,2017-02-12 23:57:58,http://cartwright.net/ambrose,0.0516480678,53.233.121.77,"{""location"": ""BG"", ""is_mobile"": false}" 18345,7,415,2016-12-17 12:19:27,http://littel.net/jenifer,0.4113069972,69.127.57.170,"{""location"": ""JE"", ""is_mobile"": false}" 18346,7,415,2017-05-30 01:38:17,http://kris.co/jacklyn.nikolaus,0.0010740846,127.106.22.144,"{""location"": ""AO"", ""is_mobile"": false}" 18347,7,415,2017-02-18 03:39:51,http://smithamtillman.org/ferne,0.1447721405,67.34.170.237,"{""location"": ""AS"", ""is_mobile"": false}" 18348,7,415,2017-04-27 15:57:18,http://collierkilback.biz/rodrigo.lowe,0.8249812764,3.139.157.96,"{""location"": ""PM"", ""is_mobile"": false}" 18349,7,415,2017-03-11 09:32:01,http://ritchiestrosin.net/alfonso.kuhic,0.4657332354,207.40.185.48,"{""location"": ""ZA"", ""is_mobile"": true}" 18350,7,415,2017-02-18 05:08:59,http://witting.biz/ashly.macgyver,0.9303896750,52.114.22.44,"{""location"": ""PA"", ""is_mobile"": false}" 18351,7,415,2017-02-26 18:57:35,http://blockschmeler.co/troy.fritsch,0.2272574142,156.176.214.88,"{""location"": ""KP"", ""is_mobile"": false}" 18352,7,415,2017-06-09 04:35:11,http://pouros.org/leonel_welch,0.1508420910,221.85.108.54,"{""location"": ""AZ"", ""is_mobile"": true}" 18353,7,415,2017-03-26 01:03:02,http://hayes.io/sienna_beier,0.6754990549,63.104.220.33,"{""location"": ""MQ"", ""is_mobile"": true}" 18354,7,415,2017-06-08 03:19:25,http://lebsack.com/cielo.reilly,0.7171427471,127.225.109.12,"{""location"": ""KH"", ""is_mobile"": true}" 18355,7,415,2017-03-04 00:19:32,http://cole.org/drake,0.3359478071,102.108.159.168,"{""location"": ""SZ"", ""is_mobile"": false}" 18356,7,415,2017-02-25 04:56:00,http://purdy.com/brycen.ohara,0.4385667854,162.105.206.148,"{""location"": ""SR"", ""is_mobile"": false}" 5396,2,118,2017-04-19 17:22:44,http://sanfordbogisich.co/donato,,182.209.243.205,"{""location"": ""BN"", ""is_mobile"": false}" 5397,2,118,2017-01-19 18:53:44,http://heathcote.io/wava_walker,,228.81.124.158,"{""location"": ""VN"", ""is_mobile"": true}" 5398,2,118,2017-05-26 08:22:12,http://bogisichmaggio.biz/dahlia_bogisich,,68.147.19.8,"{""location"": ""SZ"", ""is_mobile"": false}" 5399,2,118,2017-04-01 12:28:31,http://reynolds.io/emmalee.bogisich,,146.48.17.242,"{""location"": ""CU"", ""is_mobile"": false}" 5400,2,118,2016-12-30 15:29:32,http://nitzsche.co/tavares_hegmann,,177.94.116.101,"{""location"": ""IT"", ""is_mobile"": false}" 5401,2,118,2017-02-10 14:33:26,http://kiehnohara.org/rahul_braun,,246.5.207.242,"{""location"": ""TF"", ""is_mobile"": false}" 5402,2,118,2017-04-13 15:37:17,http://windleroconner.biz/violette,,117.130.235.27,"{""location"": ""BF"", ""is_mobile"": true}" 5403,2,118,2017-02-02 08:35:28,http://mertz.net/zena.aufderhar,,167.28.237.89,"{""location"": ""SV"", ""is_mobile"": false}" 5404,2,118,2017-04-15 04:44:47,http://wardfahey.name/marley,,143.192.69.72,"{""location"": ""GI"", ""is_mobile"": true}" 5405,2,118,2017-05-29 12:04:18,http://framiheathcote.com/niko.buckridge,,121.170.84.91,"{""location"": ""NR"", ""is_mobile"": false}" 5406,2,118,2017-04-29 15:26:54,http://kochpagac.com/luna_carter,,210.159.197.109,"{""location"": ""MC"", ""is_mobile"": true}" 5407,2,118,2017-02-04 17:21:04,http://volkman.name/delfina,,114.240.223.133,"{""location"": ""AI"", ""is_mobile"": false}" 5408,2,118,2017-05-20 21:28:40,http://farrell.net/cole.mills,,156.203.23.120,"{""location"": ""TL"", ""is_mobile"": false}" 5409,2,118,2017-03-27 07:23:22,http://sanfordraynor.org/frankie,,154.187.209.115,"{""location"": ""FM"", ""is_mobile"": true}" 5410,2,118,2017-06-03 06:40:23,http://spencer.co/cleta,,169.115.176.219,"{""location"": ""FM"", ""is_mobile"": false}" 5411,2,118,2017-05-07 16:15:42,http://hermann.co/lulu.mcclure,,65.136.117.186,"{""location"": ""IO"", ""is_mobile"": false}" 5412,2,118,2017-02-03 09:41:06,http://hirthe.io/henriette.collins,,168.69.39.85,"{""location"": ""AZ"", ""is_mobile"": true}" 5413,2,118,2017-05-22 10:39:14,http://eichmann.com/kiara,,141.228.91.81,"{""location"": ""AU"", ""is_mobile"": true}" 5414,2,118,2016-12-21 00:36:19,http://hirthe.co/nicklaus,,145.241.44.156,"{""location"": ""LR"", ""is_mobile"": true}" 5415,2,118,2017-05-18 09:02:27,http://hamillturner.biz/janelle_cronin,,241.181.47.182,"{""location"": ""FM"", ""is_mobile"": true}" 5416,2,118,2017-02-09 21:23:51,http://conroylynch.io/lucio_legros,,227.103.217.243,"{""location"": ""TW"", ""is_mobile"": false}" 5417,2,118,2017-01-07 18:56:08,http://franecki.biz/marcel_hettinger,,182.67.220.6,"{""location"": ""LK"", ""is_mobile"": false}" 5418,2,118,2017-04-21 18:01:28,http://padberg.biz/melyna.okuneva,,225.103.254.40,"{""location"": ""WS"", ""is_mobile"": true}" 5419,2,118,2017-01-13 15:01:35,http://jastcain.name/nils.boyle,,252.8.76.10,"{""location"": ""ES"", ""is_mobile"": false}" 5420,2,119,2017-05-27 08:34:53,http://browndooley.org/antone_witting,0.2492050845,185.131.102.72,"{""location"": ""GY"", ""is_mobile"": false}" 5421,2,119,2017-01-22 17:32:18,http://steuber.info/edmond,0.6889339223,22.196.50.224,"{""location"": ""PN"", ""is_mobile"": false}" 5422,2,119,2017-01-13 04:30:44,http://hettinger.net/alta,0.3449814628,56.67.73.140,"{""location"": ""NC"", ""is_mobile"": true}" 5423,2,119,2017-04-19 12:00:02,http://emard.io/reyes,0.6671358567,47.193.22.20,"{""location"": ""PA"", ""is_mobile"": false}" 5424,2,119,2017-01-04 13:24:03,http://roberts.info/pablo,0.5847843632,66.34.79.54,"{""location"": ""TZ"", ""is_mobile"": true}" 5425,2,119,2017-04-12 19:46:11,http://baumbach.com/solon,0.3586138928,90.68.251.94,"{""location"": ""GD"", ""is_mobile"": false}" 5426,2,119,2017-05-13 02:53:16,http://mclaughlin.net/kali,0.9559220816,99.248.234.49,"{""location"": ""GN"", ""is_mobile"": false}" 5427,2,119,2017-02-22 14:07:56,http://rippin.name/manuela,0.8009746544,138.162.87.206,"{""location"": ""ES"", ""is_mobile"": true}" 5428,2,119,2017-03-09 04:04:27,http://schulist.net/jeromy,0.4053409821,235.64.128.128,"{""location"": ""FI"", ""is_mobile"": true}" 5429,2,119,2017-03-05 16:48:13,http://cronatrantow.com/angelo,0.5994320575,75.80.254.114,"{""location"": ""CM"", ""is_mobile"": true}" 5430,2,119,2017-02-23 23:21:47,http://larson.net/joan_price,0.0779456961,20.3.131.129,"{""location"": ""CZ"", ""is_mobile"": true}" 5431,2,119,2017-02-14 07:05:26,http://stehr.co/neha,0.9459188907,237.60.180.144,"{""location"": ""PE"", ""is_mobile"": false}" 5432,2,119,2017-05-28 07:15:15,http://ohara.io/ethelyn.grady,0.2506039447,160.202.72.48,"{""location"": ""HT"", ""is_mobile"": true}" 5433,2,119,2017-04-03 17:14:54,http://franecki.io/carter_king,0.2689586009,206.31.217.71,"{""location"": ""CK"", ""is_mobile"": true}" 5434,2,119,2017-05-20 23:26:34,http://mante.net/triston,0.8348596354,228.148.54.180,"{""location"": ""SD"", ""is_mobile"": false}" 5435,2,119,2017-06-09 05:09:06,http://lefflerwalsh.com/oren.macgyver,0.5333056756,169.42.217.183,"{""location"": ""GR"", ""is_mobile"": true}" 5436,2,119,2017-02-03 20:44:12,http://schamberger.co/heath_metz,0.9337192633,160.74.18.61,"{""location"": ""GG"", ""is_mobile"": false}" 5437,2,119,2017-04-10 11:38:49,http://hilpert.net/domingo,0.7108595683,16.69.200.29,"{""location"": ""SK"", ""is_mobile"": false}" 5438,2,119,2017-01-07 04:53:43,http://beier.io/theresa.maggio,0.5161988492,194.126.49.26,"{""location"": ""YT"", ""is_mobile"": true}" 5439,2,119,2017-06-12 01:35:16,http://moen.biz/harley,0.0236461932,130.157.209.85,"{""location"": ""VN"", ""is_mobile"": false}" 5440,2,119,2017-01-21 00:06:07,http://boylemurray.biz/kamren,0.9578046921,2.200.4.30,"{""location"": ""GD"", ""is_mobile"": true}" 5441,2,119,2017-06-07 05:00:23,http://lehner.name/emilio,0.8817157658,90.115.90.172,"{""location"": ""FO"", ""is_mobile"": false}" 5442,2,119,2017-03-05 06:40:47,http://ziemannfahey.name/roberta.kunze,0.2809998475,248.79.104.173,"{""location"": ""TL"", ""is_mobile"": false}" 5443,2,119,2017-02-28 01:02:49,http://faylindgren.co/christian,0.0817535122,144.12.165.21,"{""location"": ""CA"", ""is_mobile"": true}" 5444,2,119,2017-06-12 09:04:37,http://parisian.biz/rosamond_mills,0.9875076698,174.37.241.150,"{""location"": ""NZ"", ""is_mobile"": false}" 5445,2,119,2017-01-23 11:42:09,http://deckow.io/freddy.schuster,0.8964179086,214.66.172.85,"{""location"": ""KG"", ""is_mobile"": false}" 5446,2,119,2017-01-26 11:50:40,http://ondrickamiller.name/isaac_walter,0.3578197129,117.49.70.203,"{""location"": ""LC"", ""is_mobile"": false}" 5447,2,119,2017-05-18 07:55:38,http://skileshammes.org/shana,0.6273497769,101.237.75.248,"{""location"": ""GA"", ""is_mobile"": false}" 5448,2,119,2017-01-24 01:49:37,http://bergnaum.io/brandi.von,0.3510352277,135.189.26.245,"{""location"": ""NA"", ""is_mobile"": false}" 18357,7,415,2017-05-20 13:34:28,http://lebsackvonrueden.net/felicia.farrell,0.9536816444,145.121.92.59,"{""location"": ""MR"", ""is_mobile"": false}" 18358,7,415,2017-01-16 10:13:43,http://howehahn.org/jeie.conn,0.2142225308,74.68.162.221,"{""location"": ""GI"", ""is_mobile"": true}" 18359,7,415,2017-02-13 07:52:07,http://greenholt.info/timmothy,0.5677431901,10.164.225.88,"{""location"": ""GT"", ""is_mobile"": true}" 18360,7,415,2017-05-17 08:38:51,http://welchbergstrom.co/mariano,0.4017832524,184.63.226.142,"{""location"": ""HK"", ""is_mobile"": true}" 18361,7,415,2017-01-22 12:18:37,http://millerhyatt.org/samir.kiehn,0.4597164152,30.238.226.234,"{""location"": ""EH"", ""is_mobile"": false}" 18362,7,415,2017-04-16 00:24:26,http://windler.info/naomi,0.7825656634,246.152.18.148,"{""location"": ""TZ"", ""is_mobile"": false}" 18363,7,415,2017-04-25 13:35:18,http://parisian.net/ken.emmerich,0.3402118103,254.63.142.191,"{""location"": ""BI"", ""is_mobile"": true}" 18364,7,415,2017-05-26 06:19:18,http://grimes.co/adelle_wisozk,0.5277950595,15.49.38.106,"{""location"": ""IM"", ""is_mobile"": false}" 18365,7,415,2017-05-19 00:08:00,http://greenfelder.com/melvina_schinner,0.7159736506,74.116.104.239,"{""location"": ""LI"", ""is_mobile"": false}" 18366,7,415,2017-03-11 07:26:46,http://pacochatorphy.co/roberto.howell,0.0965114058,77.245.40.57,"{""location"": ""LB"", ""is_mobile"": false}" 18367,7,415,2017-04-13 06:32:08,http://von.io/cruz.gottlieb,0.4895133213,89.61.22.70,"{""location"": ""VI"", ""is_mobile"": true}" 18368,7,415,2017-01-05 04:11:58,http://harberheller.com/joan.donnelly,0.1359900122,144.77.214.52,"{""location"": ""OM"", ""is_mobile"": true}" 18369,7,415,2017-03-29 10:47:59,http://riceko.biz/hillary_schmeler,0.7429887969,31.238.209.237,"{""location"": ""VE"", ""is_mobile"": false}" 18370,7,415,2017-01-29 17:07:20,http://kuhicprice.biz/roy.leuschke,0.5342587255,119.24.37.138,"{""location"": ""MZ"", ""is_mobile"": true}" 18371,7,415,2017-05-20 23:21:15,http://runolfsdottir.info/brianne.schuppe,0.8725656993,39.122.33.21,"{""location"": ""MK"", ""is_mobile"": true}" 18372,7,415,2017-01-09 09:27:29,http://weber.com/eddie.herzog,0.3876463258,163.181.90.31,"{""location"": ""CG"", ""is_mobile"": true}" 18373,7,415,2017-06-03 03:28:04,http://mcglynn.info/lafayette,0.4242364573,51.225.191.109,"{""location"": ""BT"", ""is_mobile"": false}" 18374,7,415,2017-05-31 10:57:29,http://cummings.info/carole.schinner,0.3258437098,67.169.228.76,"{""location"": ""BT"", ""is_mobile"": false}" 18375,7,415,2017-03-30 21:06:02,http://murphy.info/delbert,0.9964740587,88.105.143.234,"{""location"": ""EE"", ""is_mobile"": false}" 18376,7,415,2017-01-04 16:35:12,http://weimannbaumbach.name/devon,0.8040280540,168.36.17.18,"{""location"": ""TL"", ""is_mobile"": true}" 18377,7,415,2017-04-03 09:42:19,http://gaylord.net/nicklaus,0.6647349338,233.11.95.221,"{""location"": ""AM"", ""is_mobile"": true}" 18378,7,415,2017-03-29 10:49:47,http://rutherford.org/daron,0.8493735973,140.10.152.240,"{""location"": ""MU"", ""is_mobile"": false}" 18379,7,415,2017-04-19 19:00:20,http://mante.com/arvid,0.8997131259,53.222.133.215,"{""location"": ""PS"", ""is_mobile"": true}" 18380,7,415,2017-01-18 15:33:06,http://monahanryan.name/rodger_nienow,0.8026228335,166.8.65.135,"{""location"": ""SO"", ""is_mobile"": false}" 18381,7,415,2017-04-11 06:24:17,http://smitham.name/kaley,0.3342658064,30.174.40.47,"{""location"": ""SC"", ""is_mobile"": false}" 18382,7,415,2016-12-21 01:34:31,http://rolfson.info/angela,0.3189572262,75.19.35.153,"{""location"": ""BQ"", ""is_mobile"": true}" 18383,7,415,2017-03-12 23:39:41,http://trantowhilpert.info/tracy.bashirian,0.6971272872,149.121.27.77,"{""location"": ""BN"", ""is_mobile"": false}" 18384,7,415,2017-04-21 16:50:26,http://pfeffer.io/ralph,0.6162218182,158.238.24.129,"{""location"": ""SN"", ""is_mobile"": false}" 18385,7,415,2017-04-05 17:50:35,http://grant.co/marianna,0.8227554408,247.78.151.119,"{""location"": ""DO"", ""is_mobile"": false}" 18386,7,415,2016-12-27 21:30:19,http://gorczanydeckow.net/gabriella_jast,0.1941117073,202.251.153.217,"{""location"": ""ME"", ""is_mobile"": true}" 18387,7,415,2017-04-28 00:11:43,http://bernierwhite.biz/isobel_hilll,0.1677040099,184.39.59.185,"{""location"": ""AD"", ""is_mobile"": true}" 18388,7,415,2017-01-17 08:27:23,http://stiedemann.net/reie,0.7417731626,9.62.138.115,"{""location"": ""CR"", ""is_mobile"": true}" 18389,7,415,2017-01-30 04:18:28,http://murrayconn.biz/rafael,0.6332637990,201.95.16.244,"{""location"": ""GB"", ""is_mobile"": false}" 18390,7,415,2017-01-11 10:06:30,http://mueller.co/tobin,0.0375753172,121.206.184.39,"{""location"": ""MC"", ""is_mobile"": false}" 18391,7,415,2017-02-27 07:30:15,http://block.net/lupe_friesen,0.6157693426,179.251.227.11,"{""location"": ""BN"", ""is_mobile"": true}" 18392,7,415,2017-03-21 03:28:38,http://fahey.com/myles_kihn,0.8542414493,192.4.121.98,"{""location"": ""GS"", ""is_mobile"": false}" 18393,7,415,2017-02-25 19:39:23,http://altenwerth.co/dejah.medhurst,0.4959573022,73.152.117.126,"{""location"": ""LI"", ""is_mobile"": true}" 18394,7,415,2017-03-15 13:34:46,http://oreilly.info/elta.collier,0.9227965553,200.212.154.242,"{""location"": ""LV"", ""is_mobile"": false}" 18395,7,416,2017-04-25 10:17:48,http://rowe.info/harry,0.1000436170,235.205.51.232,"{""location"": ""MR"", ""is_mobile"": true}" 18396,7,416,2017-05-06 00:34:34,http://hermiston.net/linwood_ankunding,0.6730097068,66.105.84.204,"{""location"": ""LI"", ""is_mobile"": true}" 18397,7,416,2017-03-20 02:27:19,http://kerluke.co/olen_hauck,0.8659692913,88.170.65.154,"{""location"": ""ES"", ""is_mobile"": true}" 18398,7,416,2017-04-09 23:57:58,http://prohaskahodkiewicz.co/myah,0.6624466028,21.3.193.129,"{""location"": ""NE"", ""is_mobile"": false}" 18399,7,416,2017-02-15 09:38:19,http://leuschkehettinger.name/kasey_koepp,0.7602539492,63.214.72.212,"{""location"": ""GM"", ""is_mobile"": true}" 18400,7,416,2017-01-19 09:51:57,http://oconnellhaley.name/jammie,0.4470441458,219.232.26.194,"{""location"": ""ST"", ""is_mobile"": false}" 18401,7,416,2017-06-14 03:45:16,http://shields.name/kameron_corkery,0.1520191230,244.68.206.186,"{""location"": ""KN"", ""is_mobile"": true}" 18402,7,416,2017-05-10 07:46:50,http://hahn.info/aisha.jacobson,0.6517950225,85.247.117.190,"{""location"": ""TK"", ""is_mobile"": true}" 18403,7,416,2017-01-10 23:09:53,http://heaneywilkinson.biz/ethelyn.lockman,0.9555444550,146.92.206.64,"{""location"": ""JO"", ""is_mobile"": true}" 18404,7,416,2017-05-30 13:34:23,http://mccullough.org/bella_blick,0.0434454133,243.135.52.203,"{""location"": ""LC"", ""is_mobile"": false}" 18405,7,416,2017-03-28 08:14:03,http://davis.net/kennith,0.5509550306,83.226.129.208,"{""location"": ""GI"", ""is_mobile"": true}" 18406,7,416,2017-05-21 02:21:09,http://kerluke.com/lionel_hackett,0.7642745867,47.8.60.183,"{""location"": ""IO"", ""is_mobile"": false}" 18407,7,416,2017-02-06 04:37:22,http://treutel.net/clair,0.4122643459,6.141.207.185,"{""location"": ""YT"", ""is_mobile"": false}" 5449,2,119,2017-04-30 01:37:07,http://kiehn.com/sarah_schuster,0.6088684164,251.13.125.89,"{""location"": ""GG"", ""is_mobile"": false}" 5450,2,119,2017-01-06 08:17:24,http://glovercummings.io/reie.muller,0.5765107641,97.13.82.37,"{""location"": ""BH"", ""is_mobile"": true}" 5451,2,119,2017-04-02 09:34:39,http://herzog.name/audreanne,0.4674742074,195.135.97.208,"{""location"": ""MQ"", ""is_mobile"": true}" 5452,2,119,2017-01-28 07:05:15,http://jacobson.net/zora,0.5937096285,205.12.150.187,"{""location"": ""GP"", ""is_mobile"": true}" 5453,2,119,2017-06-12 11:41:26,http://deckow.biz/jeremy_donnelly,0.7162913971,159.122.98.58,"{""location"": ""PW"", ""is_mobile"": true}" 5454,2,119,2017-02-27 15:33:01,http://shanahanswaniawski.info/devin.treutel,0.9994977374,159.112.33.99,"{""location"": ""ME"", ""is_mobile"": true}" 5455,2,119,2017-01-25 20:19:28,http://jacobs.net/ashleigh,0.2400971339,39.71.93.121,"{""location"": ""LB"", ""is_mobile"": false}" 5456,2,119,2017-05-19 17:04:03,http://hilll.com/barry.gusikowski,0.3023623080,148.19.168.134,"{""location"": ""SL"", ""is_mobile"": true}" 5457,2,119,2017-03-17 04:50:45,http://reingerbradtke.net/nicola,0.6237094183,217.22.34.237,"{""location"": ""GH"", ""is_mobile"": true}" 5458,2,119,2017-06-02 11:41:44,http://yundtbogisich.name/rusty,0.2850381029,162.76.153.144,"{""location"": ""AX"", ""is_mobile"": false}" 5459,2,119,2017-01-31 20:45:34,http://tillman.com/kathryne_pouros,0.7941343620,133.210.186.244,"{""location"": ""CR"", ""is_mobile"": false}" 5460,2,119,2017-01-29 17:34:44,http://gulgowski.name/marcelina.lemke,0.6034034464,55.220.197.159,"{""location"": ""FK"", ""is_mobile"": false}" 5461,2,119,2017-04-22 19:02:41,http://quigley.net/hertha_skiles,0.3113281002,111.33.189.214,"{""location"": ""GQ"", ""is_mobile"": false}" 5462,2,119,2017-01-05 19:38:30,http://hirthemonahan.org/ryan.stiedemann,0.7415625376,3.117.235.227,"{""location"": ""ZM"", ""is_mobile"": true}" 5463,2,119,2017-04-19 18:40:48,http://fisherschaden.org/oswaldo.schroeder,0.9107412211,165.222.121.123,"{""location"": ""BQ"", ""is_mobile"": true}" 5464,2,119,2017-01-26 00:32:43,http://bosco.io/tavares_kshlerin,0.9868495826,5.219.187.130,"{""location"": ""CU"", ""is_mobile"": true}" 5465,2,119,2017-05-02 11:05:15,http://cartwright.org/meggie,0.3750326552,139.77.238.15,"{""location"": ""LS"", ""is_mobile"": true}" 5466,2,119,2017-05-24 18:28:04,http://huel.biz/ferne_trantow,0.4929106406,169.24.99.196,"{""location"": ""KH"", ""is_mobile"": false}" 5467,2,119,2017-05-17 12:15:41,http://mannolson.io/richard,0.2008329064,30.17.52.159,"{""location"": ""IS"", ""is_mobile"": false}" 5468,2,119,2017-05-16 19:25:41,http://dietrich.biz/marquis.rath,0.0525561023,44.63.124.241,"{""location"": ""SI"", ""is_mobile"": false}" 5469,2,119,2016-12-31 18:44:06,http://tromp.net/jimmy_adams,0.2472869188,84.16.106.64,"{""location"": ""SK"", ""is_mobile"": true}" 5470,2,119,2017-01-03 22:31:20,http://willmawayn.co/conrad,0.0410864125,124.93.155.150,"{""location"": ""TT"", ""is_mobile"": false}" 5471,2,119,2017-05-31 02:28:25,http://johnston.name/otto.abshire,0.7553744116,70.2.82.2,"{""location"": ""MR"", ""is_mobile"": true}" 5472,2,119,2017-03-04 02:23:46,http://ratke.biz/summer,0.4940357971,155.16.90.67,"{""location"": ""RW"", ""is_mobile"": false}" 5473,2,119,2017-04-27 03:18:55,http://gradygoyette.com/name,0.5868864724,28.235.81.9,"{""location"": ""IR"", ""is_mobile"": true}" 5474,2,119,2017-01-16 23:30:47,http://pouroswatsica.biz/janis_murphy,0.8236873187,151.239.52.134,"{""location"": ""EE"", ""is_mobile"": true}" 5475,2,119,2017-06-09 15:50:50,http://pagac.biz/ola,0.3533620915,219.188.110.21,"{""location"": ""ML"", ""is_mobile"": true}" 5476,2,119,2017-02-25 07:34:06,http://marks.net/myrl,0.3708815102,103.226.243.25,"{""location"": ""AO"", ""is_mobile"": true}" 5477,2,119,2017-05-17 04:10:16,http://larson.com/gilberto,0.7878404327,40.12.49.13,"{""location"": ""GD"", ""is_mobile"": false}" 5478,2,119,2017-02-25 12:47:56,http://leannon.info/hellen,0.9326705066,216.123.149.148,"{""location"": ""TT"", ""is_mobile"": false}" 5479,2,119,2017-03-13 23:19:03,http://leuschke.biz/alvis.bruen,0.2949698907,71.72.90.222,"{""location"": ""TT"", ""is_mobile"": true}" 5480,2,119,2017-06-06 04:10:06,http://jones.info/josue_emard,0.8785397550,100.70.173.61,"{""location"": ""US"", ""is_mobile"": false}" 5481,2,119,2017-03-07 21:23:26,http://lowe.info/ramiro,0.7410858999,44.79.116.75,"{""location"": ""NU"", ""is_mobile"": false}" 5482,2,119,2017-04-07 11:54:36,http://grimes.co/collin.kemmer,0.4221687637,98.60.113.75,"{""location"": ""AL"", ""is_mobile"": true}" 5483,2,119,2017-04-02 07:32:42,http://lemke.net/ashley,0.3493839012,72.212.126.181,"{""location"": ""HN"", ""is_mobile"": false}" 5484,2,119,2017-05-14 04:39:13,http://donnellystamm.info/gaston.doyle,0.1623175422,209.251.55.57,"{""location"": ""AL"", ""is_mobile"": false}" 5485,2,119,2017-02-25 14:37:27,http://erdman.biz/elenora,0.3852012353,234.39.208.101,"{""location"": ""ME"", ""is_mobile"": false}" 5486,2,119,2017-03-17 07:39:36,http://gorczanyyost.info/magnus,0.1895678989,118.86.81.90,"{""location"": ""BM"", ""is_mobile"": true}" 5487,2,119,2017-05-09 03:32:11,http://grimes.net/willy.johnson,0.5324589253,156.6.202.29,"{""location"": ""ME"", ""is_mobile"": true}" 5488,2,119,2016-12-20 21:28:00,http://botsford.name/nels,0.4280526859,94.182.14.2,"{""location"": ""AZ"", ""is_mobile"": true}" 5489,2,119,2017-01-15 09:32:44,http://olson.biz/erna,0.8510849072,57.57.94.72,"{""location"": ""IS"", ""is_mobile"": false}" 5490,2,120,2017-01-22 00:18:47,http://wiza.biz/coty_marquardt,0.9543919375,130.17.113.22,"{""location"": ""LT"", ""is_mobile"": true}" 5491,2,120,2017-03-17 13:53:30,http://king.name/jerrod_cummerata,0.0099144057,153.167.5.212,"{""location"": ""SV"", ""is_mobile"": false}" 5492,2,120,2017-05-22 23:28:48,http://pfannerstillcorwin.name/roel,0.9149966980,227.62.61.247,"{""location"": ""KG"", ""is_mobile"": true}" 5493,2,120,2017-06-01 17:49:23,http://mante.co/nia,0.2417519213,165.135.22.35,"{""location"": ""AR"", ""is_mobile"": false}" 5494,2,120,2017-03-23 16:43:40,http://douglas.biz/kaylin,0.7204583233,224.65.207.166,"{""location"": ""NR"", ""is_mobile"": true}" 5495,2,120,2017-01-09 21:20:46,http://raynor.org/deie.white,0.5578054638,253.61.151.218,"{""location"": ""KH"", ""is_mobile"": false}" 5496,2,120,2017-03-24 00:44:13,http://haley.biz/dominic.mraz,0.3266337348,94.8.62.176,"{""location"": ""ER"", ""is_mobile"": false}" 5497,2,120,2017-03-12 17:14:25,http://dubuquecremin.name/junius.gerhold,0.3456903413,41.24.106.141,"{""location"": ""CD"", ""is_mobile"": false}" 5498,2,120,2017-01-22 15:39:56,http://prohaskaemard.org/laria,0.5743522560,166.240.136.24,"{""location"": ""LT"", ""is_mobile"": false}" 5499,2,120,2017-06-08 22:33:46,http://anderson.biz/keara_sipes,0.3569993285,239.85.56.16,"{""location"": ""BE"", ""is_mobile"": false}" 5500,2,120,2016-12-17 07:33:56,http://quitzon.com/alan,0.5368718563,50.212.196.226,"{""location"": ""TN"", ""is_mobile"": true}" 18408,7,416,2017-05-28 02:17:59,http://spinka.io/charles_bartoletti,0.2675935009,97.177.237.69,"{""location"": ""TM"", ""is_mobile"": true}" 18409,7,416,2017-02-17 06:12:41,http://goldnertreutel.org/alison,0.9935325114,8.131.71.87,"{""location"": ""QA"", ""is_mobile"": true}" 18410,7,416,2017-01-14 08:19:22,http://rempel.name/oliver.swaniawski,0.1963997616,123.73.175.42,"{""location"": ""FO"", ""is_mobile"": false}" 18411,7,416,2016-12-24 11:36:38,http://heaney.name/tracey,0.6645956207,55.63.78.232,"{""location"": ""CY"", ""is_mobile"": false}" 18412,7,416,2017-02-02 23:49:21,http://balistreriosinski.biz/casper,0.4781632376,201.143.176.94,"{""location"": ""BV"", ""is_mobile"": true}" 18413,7,416,2017-04-09 16:22:18,http://baumbachkovacek.org/nicklaus_dibbert,0.4771254355,22.186.158.99,"{""location"": ""NC"", ""is_mobile"": true}" 18414,7,416,2017-04-09 12:01:33,http://gutkowski.info/verona.jacobson,0.1794544146,194.98.227.98,"{""location"": ""AE"", ""is_mobile"": true}" 18415,7,416,2017-03-22 05:45:51,http://stiedemann.com/rosella,0.8259341356,74.33.74.240,"{""location"": ""CW"", ""is_mobile"": false}" 18416,7,416,2017-06-04 18:34:50,http://blanda.co/mekhi_ondricka,0.4273942055,46.9.66.124,"{""location"": ""PF"", ""is_mobile"": false}" 18417,7,416,2017-03-12 22:22:35,http://nitzscherutherford.info/webster,0.2440273726,192.94.63.194,"{""location"": ""BS"", ""is_mobile"": false}" 18418,7,416,2017-05-28 04:43:46,http://ankundinghudson.co/evalyn,0.6610370025,225.145.247.109,"{""location"": ""ET"", ""is_mobile"": false}" 18419,7,416,2017-01-12 09:22:27,http://fahey.com/kory_wolff,0.3300961927,95.216.150.22,"{""location"": ""MC"", ""is_mobile"": true}" 18420,7,416,2017-05-07 23:59:05,http://mosciski.com/jonas.abshire,0.8750618151,39.17.57.235,"{""location"": ""NZ"", ""is_mobile"": false}" 18421,7,416,2017-04-22 15:30:47,http://marquardt.co/blanche,0.7325480628,161.119.54.146,"{""location"": ""ID"", ""is_mobile"": false}" 18422,7,416,2017-06-05 08:47:09,http://hillldouglas.name/candelario_batz,0.6504104899,200.207.175.151,"{""location"": ""GB"", ""is_mobile"": true}" 18423,7,416,2017-06-06 20:50:29,http://christiansengoodwin.io/sigurd,0.1091466296,93.216.210.110,"{""location"": ""PL"", ""is_mobile"": false}" 18424,7,416,2017-05-07 21:50:13,http://wisoky.info/austin.price,0.8093448163,9.80.254.180,"{""location"": ""NR"", ""is_mobile"": true}" 18425,7,416,2017-02-23 08:04:46,http://kirlin.org/shane,0.7029212359,65.31.23.21,"{""location"": ""IE"", ""is_mobile"": true}" 18426,7,416,2017-05-13 11:43:11,http://reichel.com/jayde,0.4266690125,22.66.166.120,"{""location"": ""AU"", ""is_mobile"": true}" 18427,7,416,2016-12-19 02:26:13,http://raynor.co/cristopher_wolff,0.9422052493,10.26.178.224,"{""location"": ""RS"", ""is_mobile"": false}" 18428,7,416,2017-03-07 03:22:37,http://rutherfordbecker.co/frieda,0.4307102854,123.144.150.191,"{""location"": ""KR"", ""is_mobile"": true}" 18429,7,416,2017-01-26 03:35:58,http://blickkaulke.info/caria_trantow,0.3313317357,140.69.68.180,"{""location"": ""BA"", ""is_mobile"": false}" 18430,7,416,2017-02-03 19:21:35,http://oconner.biz/lois,0.4111703541,79.139.203.30,"{""location"": ""AQ"", ""is_mobile"": false}" 18431,7,416,2017-03-25 08:46:28,http://hintz.org/vincenza,0.2862307847,152.73.236.145,"{""location"": ""CV"", ""is_mobile"": true}" 18432,7,416,2017-02-14 14:24:19,http://davis.biz/elmo_von,0.8156368163,206.198.248.190,"{""location"": ""GH"", ""is_mobile"": true}" 18433,7,416,2017-01-26 23:03:25,http://gutkowski.net/franco,0.7818963322,39.93.126.242,"{""location"": ""TJ"", ""is_mobile"": true}" 18434,7,416,2017-02-16 15:02:49,http://bahringer.net/jeramie,0.5548689796,199.94.228.112,"{""location"": ""BA"", ""is_mobile"": false}" 18435,7,416,2016-12-27 10:50:32,http://mosciski.name/itzel.bartell,0.5050450257,13.163.183.164,"{""location"": ""SE"", ""is_mobile"": false}" 18436,7,416,2017-01-30 11:46:45,http://connmayer.com/aurelio.gutkowski,0.0671781701,212.194.182.135,"{""location"": ""IM"", ""is_mobile"": true}" 18437,7,416,2017-04-16 23:13:36,http://fay.name/vella,0.1384333417,107.120.200.219,"{""location"": ""ES"", ""is_mobile"": false}" 18438,7,416,2017-05-26 11:04:40,http://sengerdavis.info/brent_boyer,0.9842614826,22.142.58.62,"{""location"": ""IM"", ""is_mobile"": false}" 18439,7,416,2017-02-10 02:29:18,http://kertzmann.biz/tremayne,0.0081475036,208.217.93.70,"{""location"": ""BR"", ""is_mobile"": true}" 18440,7,416,2017-04-30 01:41:20,http://padberg.biz/skyla.bailey,0.1730928909,90.153.123.181,"{""location"": ""SM"", ""is_mobile"": false}" 18441,7,416,2017-03-17 18:38:24,http://okeefe.co/gayle_harvey,0.1738790582,201.123.101.43,"{""location"": ""BG"", ""is_mobile"": true}" 18442,7,416,2017-03-09 16:37:40,http://roob.com/tyreek_pacocha,0.4926428450,178.206.200.29,"{""location"": ""LB"", ""is_mobile"": true}" 18443,7,416,2017-03-14 23:53:59,http://gleason.io/bart.wolff,0.9711146429,236.162.223.173,"{""location"": ""BF"", ""is_mobile"": true}" 18444,7,416,2017-03-03 09:42:22,http://greenfelderlegros.io/major_dietrich,0.5455859016,191.184.92.250,"{""location"": ""EG"", ""is_mobile"": false}" 18445,7,416,2017-04-22 12:52:26,http://kaulke.io/mafalda,0.9391166676,100.220.71.66,"{""location"": ""KM"", ""is_mobile"": false}" 18446,7,416,2017-06-12 01:23:43,http://emmerich.name/aileen_wolff,0.6660278943,248.27.139.191,"{""location"": ""TR"", ""is_mobile"": true}" 18447,7,416,2017-02-24 06:47:42,http://schaden.com/candice_gaylord,0.5078758378,230.241.248.254,"{""location"": ""AQ"", ""is_mobile"": true}" 18448,7,416,2017-04-08 00:14:54,http://rogahnquitzon.name/jonathan_marks,0.7688430350,130.165.152.211,"{""location"": ""IL"", ""is_mobile"": false}" 18449,7,416,2017-01-17 14:15:46,http://macgyver.co/nicholaus.schroeder,0.1805354450,13.236.36.192,"{""location"": ""BI"", ""is_mobile"": false}" 18450,7,416,2017-04-27 13:25:08,http://hand.org/domenic_ullrich,0.7644790687,153.118.109.157,"{""location"": ""MK"", ""is_mobile"": false}" 18451,7,416,2016-12-15 17:22:19,http://will.name/jeremie,0.0782472640,89.40.193.147,"{""location"": ""KG"", ""is_mobile"": true}" 18452,7,416,2017-05-23 20:47:38,http://dickens.com/whitney,0.2793112364,147.90.246.77,"{""location"": ""ET"", ""is_mobile"": true}" 18453,7,416,2017-01-31 19:55:00,http://mccullough.biz/marion,0.1604298044,25.197.139.110,"{""location"": ""AQ"", ""is_mobile"": true}" 18454,7,416,2017-02-12 22:27:06,http://steubercrooks.io/edwardo,0.7004791023,179.134.211.171,"{""location"": ""HM"", ""is_mobile"": true}" 18455,7,416,2017-03-10 09:03:33,http://gutkowski.co/jeromy,0.0404074546,133.49.192.181,"{""location"": ""BT"", ""is_mobile"": true}" 18456,7,416,2017-03-11 16:43:58,http://wiegand.com/joel_lesch,0.2865265216,231.8.181.133,"{""location"": ""WF"", ""is_mobile"": false}" 18457,7,416,2016-12-31 13:34:32,http://torphykuvalis.com/lucio,0.2121208940,159.220.183.149,"{""location"": ""SJ"", ""is_mobile"": true}" 18458,7,416,2017-01-03 11:15:27,http://bayer.org/malvina,0.4429551832,215.234.20.130,"{""location"": ""SZ"", ""is_mobile"": false}" 5501,2,120,2016-12-18 09:04:20,http://rathmueller.co/mylene.dubuque,0.8259377309,85.216.113.24,"{""location"": ""KN"", ""is_mobile"": true}" 5502,2,120,2017-03-24 19:41:10,http://bartoletti.com/angeline.lynch,0.1328882972,252.28.210.227,"{""location"": ""CA"", ""is_mobile"": true}" 5503,2,120,2017-06-05 01:22:14,http://schmidtwest.info/adolph_jacobs,0.1044139011,137.27.45.63,"{""location"": ""CH"", ""is_mobile"": true}" 5504,2,120,2017-02-10 07:29:09,http://pfannerstillemard.org/martin.bayer,0.9687006020,184.237.178.80,"{""location"": ""MM"", ""is_mobile"": true}" 5505,2,120,2017-02-25 07:51:45,http://schulisthudson.name/cielo_schneider,0.5173888883,176.242.68.251,"{""location"": ""MS"", ""is_mobile"": true}" 5506,2,120,2017-06-01 07:49:39,http://hackett.org/marjolaine,0.6093732472,150.248.34.30,"{""location"": ""FK"", ""is_mobile"": true}" 5507,2,120,2017-02-21 14:30:41,http://torp.co/carmel,0.8291072256,197.253.200.209,"{""location"": ""BO"", ""is_mobile"": true}" 5508,2,120,2017-03-05 03:20:52,http://bauch.org/zachariah.sipes,0.7318775478,212.49.185.54,"{""location"": ""NR"", ""is_mobile"": true}" 5509,2,120,2017-02-26 21:33:24,http://christiansen.info/veda.hackett,0.3498928026,22.244.100.167,"{""location"": ""UA"", ""is_mobile"": true}" 5510,2,120,2017-03-07 20:27:53,http://berge.com/eddie.botsford,0.7322815564,148.54.148.46,"{""location"": ""GS"", ""is_mobile"": true}" 5511,2,120,2017-03-30 15:50:13,http://willms.name/gilberto,0.9886240855,35.229.52.34,"{""location"": ""CN"", ""is_mobile"": false}" 5512,2,120,2017-03-21 19:11:22,http://fahey.com/nat,0.6434010238,60.170.187.167,"{""location"": ""MF"", ""is_mobile"": true}" 5513,2,120,2017-06-11 14:47:01,http://mills.info/josephine,0.6670096829,208.31.46.160,"{""location"": ""NP"", ""is_mobile"": false}" 5514,2,120,2017-03-30 03:44:55,http://gottlieboconnell.name/mathias.fritsch,0.5181405504,81.37.166.70,"{""location"": ""RE"", ""is_mobile"": false}" 5515,2,120,2017-01-12 22:13:53,http://westraynor.com/dawson.renner,0.2703050966,206.141.126.157,"{""location"": ""RO"", ""is_mobile"": false}" 5516,2,120,2016-12-24 16:53:20,http://morarjacobi.io/jenifer_lindgren,0.7823292917,232.127.229.242,"{""location"": ""PS"", ""is_mobile"": false}" 5517,2,120,2016-12-31 06:12:00,http://jaskolski.info/agustina.welch,0.5758787800,42.119.245.8,"{""location"": ""CR"", ""is_mobile"": false}" 5518,2,120,2017-03-01 08:06:30,http://torphy.co/keely_emmerich,0.9318438674,168.245.53.4,"{""location"": ""HU"", ""is_mobile"": true}" 5519,2,120,2017-04-22 06:30:51,http://anderson.name/selena,0.7408079713,21.163.6.34,"{""location"": ""BQ"", ""is_mobile"": false}" 5520,2,120,2016-12-16 08:23:02,http://sauerhane.com/walter,0.5390974594,225.42.59.203,"{""location"": ""ID"", ""is_mobile"": true}" 5521,2,120,2017-02-10 17:58:00,http://howe.io/johnathan,0.8119888924,194.11.31.20,"{""location"": ""FJ"", ""is_mobile"": false}" 5522,2,120,2017-04-23 01:39:10,http://bauch.biz/travon,0.2684072771,124.77.78.66,"{""location"": ""BW"", ""is_mobile"": false}" 5523,2,120,2017-03-30 19:01:59,http://leffler.net/ocie.stiedemann,0.8027683249,51.4.90.53,"{""location"": ""KH"", ""is_mobile"": true}" 5524,2,120,2017-04-26 19:56:21,http://lemkeherman.net/annette_kiehn,0.9662531647,11.81.145.43,"{""location"": ""MS"", ""is_mobile"": true}" 5525,2,120,2017-01-25 18:14:32,http://kuhlman.com/leonard,0.1965516624,173.155.207.174,"{""location"": ""RU"", ""is_mobile"": false}" 5526,2,120,2017-05-20 12:31:08,http://grimes.info/rhiannon.wilderman,0.1423853210,14.147.71.111,"{""location"": ""NR"", ""is_mobile"": false}" 5527,2,120,2016-12-26 06:43:05,http://mertz.com/joey_auer,0.5780280170,219.129.195.37,"{""location"": ""AU"", ""is_mobile"": false}" 5528,2,120,2017-03-28 23:58:11,http://rueckerharber.info/tracy_wisozk,0.0548812848,229.37.68.232,"{""location"": ""MQ"", ""is_mobile"": false}" 5529,2,120,2017-02-06 09:41:43,http://armstrongquigley.biz/marjolaine,0.0691797563,84.95.170.69,"{""location"": ""LU"", ""is_mobile"": true}" 5530,2,120,2017-04-17 19:51:21,http://wiza.io/alexys.gleason,0.0990591068,117.4.120.8,"{""location"": ""NZ"", ""is_mobile"": true}" 5531,2,120,2016-12-23 10:31:47,http://howell.info/natasha_moriette,0.9945563357,9.66.20.248,"{""location"": ""CO"", ""is_mobile"": true}" 5532,2,120,2016-12-29 08:05:32,http://heller.co/sabryna,0.4112693477,127.86.91.254,"{""location"": ""BW"", ""is_mobile"": true}" 5533,2,120,2017-04-20 22:07:31,http://damore.net/irma_hills,0.3377528199,73.250.194.246,"{""location"": ""CW"", ""is_mobile"": true}" 5534,2,120,2017-06-12 22:13:57,http://ortizheller.org/krystel,0.6680755529,128.92.114.191,"{""location"": ""GB"", ""is_mobile"": false}" 5535,2,120,2017-03-01 22:17:05,http://funk.org/ashley,0.3658564470,62.98.8.79,"{""location"": ""DK"", ""is_mobile"": true}" 5536,2,120,2017-02-06 21:41:33,http://corkery.net/cameron_adams,0.4346272081,202.15.36.223,"{""location"": ""BD"", ""is_mobile"": true}" 5537,2,120,2017-03-23 19:27:20,http://parker.biz/maximo_skiles,0.0585257071,148.52.90.148,"{""location"": ""BG"", ""is_mobile"": false}" 5538,2,120,2017-01-11 15:45:57,http://larsonlemke.info/lacy.howell,0.4716159134,230.225.94.46,"{""location"": ""HM"", ""is_mobile"": false}" 5539,2,120,2017-04-01 13:09:40,http://oreilly.co/adrian.crooks,0.8024919043,28.138.191.89,"{""location"": ""SC"", ""is_mobile"": true}" 5540,2,120,2017-02-04 16:25:27,http://mcclure.co/yasmine_gislason,0.8510236303,195.205.181.225,"{""location"": ""ML"", ""is_mobile"": true}" 5541,2,120,2016-12-15 07:06:45,http://jacobsondicki.info/favian_koelpin,0.9802510945,251.128.129.242,"{""location"": ""TF"", ""is_mobile"": false}" 5542,2,120,2016-12-31 15:10:31,http://simonis.io/hulda,0.0965267763,179.233.158.59,"{""location"": ""MS"", ""is_mobile"": true}" 5543,2,120,2017-05-13 07:18:01,http://bergnaum.info/sienna.skiles,0.6125258822,90.140.85.175,"{""location"": ""AM"", ""is_mobile"": true}" 5544,2,120,2017-05-06 22:36:30,http://weinat.io/manley.bins,0.6336454640,227.5.235.248,"{""location"": ""TZ"", ""is_mobile"": true}" 5545,2,120,2017-04-11 16:59:22,http://bradtke.com/ariane.labadie,0.7586789409,247.234.68.105,"{""location"": ""CM"", ""is_mobile"": true}" 5546,2,120,2017-06-01 23:19:10,http://nitzschepurdy.com/calista.kemmer,0.6687160733,84.47.57.135,"{""location"": ""UA"", ""is_mobile"": false}" 5547,2,120,2017-02-25 12:28:10,http://schimmel.io/maymie,0.4996392183,53.186.208.164,"{""location"": ""CH"", ""is_mobile"": true}" 5548,2,120,2017-06-07 11:02:42,http://ricekoelpin.co/gregg.ankunding,0.9425330303,141.91.142.204,"{""location"": ""PM"", ""is_mobile"": false}" 5549,2,120,2017-01-31 11:34:17,http://kubhagenes.com/quinn.hintz,0.4755195735,41.184.169.213,"{""location"": ""IN"", ""is_mobile"": true}" 5550,2,120,2017-03-29 01:03:23,http://satterfield.co/delbert_waelchi,0.4296005915,15.230.188.125,"{""location"": ""NE"", ""is_mobile"": true}" 5551,2,120,2017-02-19 20:00:28,http://legrosturner.net/chris.becker,0.7659125345,106.167.182.11,"{""location"": ""PW"", ""is_mobile"": false}" 18459,7,417,2017-02-15 16:53:43,http://block.biz/reagan,0.5184060773,98.24.26.207,"{""location"": ""AL"", ""is_mobile"": true}" 18460,7,417,2017-03-31 03:46:38,http://prosacco.co/olaf,0.6202498380,110.168.146.32,"{""location"": ""ID"", ""is_mobile"": false}" 18461,7,417,2017-01-16 02:26:49,http://grady.net/lauren,0.1622232166,232.219.181.95,"{""location"": ""QA"", ""is_mobile"": true}" 18462,7,417,2016-12-31 12:33:49,http://witting.org/emmitt,0.1961430411,125.133.76.39,"{""location"": ""SS"", ""is_mobile"": true}" 18463,7,417,2016-12-15 14:13:19,http://flatley.org/alphonso,0.0315891669,248.72.218.169,"{""location"": ""BJ"", ""is_mobile"": false}" 18464,7,417,2016-12-29 17:36:50,http://gerlach.io/alana.simonis,0.5022246738,241.83.208.248,"{""location"": ""MG"", ""is_mobile"": true}" 18465,7,417,2017-05-04 15:25:34,http://gusikowskicasper.co/corrine_thiel,0.2995710804,78.220.191.253,"{""location"": ""VG"", ""is_mobile"": true}" 18466,7,417,2017-04-08 01:24:49,http://schowalter.biz/easter_willms,0.1641752738,248.223.246.220,"{""location"": ""YT"", ""is_mobile"": true}" 18467,7,417,2017-03-20 10:38:32,http://moore.co/elsie,0.7621142493,129.163.19.110,"{""location"": ""EH"", ""is_mobile"": true}" 18468,7,417,2017-05-12 00:22:02,http://baileybrekke.co/evan,0.5937354992,126.68.217.31,"{""location"": ""VI"", ""is_mobile"": false}" 18469,7,417,2017-05-13 10:21:33,http://bradtke.io/benton,0.7121752938,55.251.87.81,"{""location"": ""BF"", ""is_mobile"": false}" 18470,7,417,2017-02-27 23:48:02,http://heidenreich.org/alia.kertzmann,0.4032712823,165.104.73.251,"{""location"": ""BQ"", ""is_mobile"": false}" 18471,7,417,2017-05-28 17:01:25,http://hauck.org/savion,0.4877386511,74.95.112.22,"{""location"": ""ES"", ""is_mobile"": true}" 18472,7,417,2017-05-13 22:55:10,http://von.name/bettye,0.7981097576,44.12.197.197,"{""location"": ""NI"", ""is_mobile"": false}" 18473,7,417,2016-12-22 05:38:09,http://ernser.org/gabriel.bruen,0.6743047493,154.14.144.144,"{""location"": ""HT"", ""is_mobile"": true}" 18474,7,417,2017-01-02 01:17:43,http://nolan.biz/verda_bruen,0.5583197644,7.8.202.104,"{""location"": ""AE"", ""is_mobile"": true}" 18475,7,417,2017-05-03 03:51:54,http://greenholtbechtelar.biz/ethan,0.2803176181,159.60.125.63,"{""location"": ""CY"", ""is_mobile"": true}" 18476,7,417,2017-04-28 17:04:03,http://bogan.com/reed.kiehn,0.9087704416,177.25.10.229,"{""location"": ""GW"", ""is_mobile"": true}" 18477,7,417,2017-03-04 00:40:39,http://cummings.com/keyshawn,0.3379930839,91.205.78.232,"{""location"": ""BT"", ""is_mobile"": true}" 18478,7,417,2017-01-31 00:03:33,http://kunze.io/heber,0.3029487179,189.227.14.156,"{""location"": ""MQ"", ""is_mobile"": true}" 18479,7,417,2017-06-05 03:07:23,http://gottlieb.io/alison,0.5000185614,69.15.7.52,"{""location"": ""JP"", ""is_mobile"": true}" 18480,7,417,2017-05-29 05:18:31,http://kohlerdavis.net/giles_roberts,0.1724714221,54.138.200.238,"{""location"": ""TM"", ""is_mobile"": false}" 18481,7,417,2017-03-25 20:30:19,http://emmerich.biz/camylle,0.0601073222,142.91.82.146,"{""location"": ""TL"", ""is_mobile"": false}" 18482,7,417,2017-01-19 01:37:57,http://langworth.info/earline,0.3569130300,77.242.33.236,"{""location"": ""PG"", ""is_mobile"": true}" 18483,7,417,2017-05-11 00:40:53,http://botsford.org/patience_medhurst,0.6383825136,127.101.115.75,"{""location"": ""NI"", ""is_mobile"": true}" 18484,7,417,2017-04-29 02:20:45,http://smith.info/thurman,0.9798645240,178.39.123.108,"{""location"": ""MG"", ""is_mobile"": false}" 18485,7,417,2017-05-08 10:50:41,http://cormier.name/kurtis,0.4052787813,154.17.121.232,"{""location"": ""SR"", ""is_mobile"": true}" 18486,7,417,2017-02-25 21:29:29,http://volkman.com/delilah_connelly,0.2877295627,6.34.18.2,"{""location"": ""CG"", ""is_mobile"": false}" 18487,7,417,2017-04-03 04:33:01,http://jerde.net/bradford,0.5417145290,21.209.171.32,"{""location"": ""IR"", ""is_mobile"": true}" 18488,7,417,2017-03-08 14:03:58,http://ernser.name/kenya,0.2937915294,226.120.217.167,"{""location"": ""MQ"", ""is_mobile"": false}" 18489,7,417,2016-12-20 22:31:12,http://wolff.io/casimir,0.7844972379,218.108.85.77,"{""location"": ""GQ"", ""is_mobile"": false}" 18490,7,417,2016-12-24 17:58:07,http://macejkovic.io/barbara,0.2897103268,2.32.148.250,"{""location"": ""AI"", ""is_mobile"": true}" 18491,7,417,2017-05-12 03:05:07,http://rau.org/jakayla,0.0147850738,161.40.219.230,"{""location"": ""CV"", ""is_mobile"": false}" 18492,7,417,2017-05-14 04:50:22,http://daughertyfadel.org/brook,0.7311137664,113.135.197.171,"{""location"": ""RO"", ""is_mobile"": true}" 18493,7,417,2017-03-12 02:29:32,http://jacobsonjacobs.io/orland.becker,0.3723712307,84.25.130.229,"{""location"": ""PK"", ""is_mobile"": false}" 18494,7,417,2017-06-05 15:55:39,http://west.co/jermain.vandervort,0.8167792060,117.153.222.17,"{""location"": ""BB"", ""is_mobile"": true}" 18495,7,417,2017-02-18 06:00:13,http://hermiston.co/lula.hahn,0.8191423484,246.252.184.107,"{""location"": ""VE"", ""is_mobile"": false}" 18496,7,417,2017-02-26 06:29:56,http://ohara.info/elia,0.6192668522,210.241.232.73,"{""location"": ""LY"", ""is_mobile"": false}" 18497,7,417,2017-02-16 10:12:55,http://kuhlman.net/mortimer,0.8930207122,60.64.167.17,"{""location"": ""PK"", ""is_mobile"": false}" 18498,7,417,2017-03-05 09:57:12,http://dickens.biz/reymundo,0.5814674391,210.109.116.113,"{""location"": ""TJ"", ""is_mobile"": true}" 18499,7,417,2017-02-12 14:24:48,http://nitzsche.co/colby.walter,0.8997184732,81.122.171.186,"{""location"": ""TR"", ""is_mobile"": true}" 18500,7,417,2017-06-05 17:09:43,http://hirtheshanahan.info/wellington_grady,0.8214712118,242.53.149.193,"{""location"": ""MD"", ""is_mobile"": false}" 18501,7,417,2017-03-12 07:22:39,http://lockman.com/gennaro,0.6654188502,7.184.212.237,"{""location"": ""BM"", ""is_mobile"": false}" 18502,7,417,2017-02-18 15:06:09,http://croninoberbrunner.name/brooklyn,0.2674198411,49.243.15.219,"{""location"": ""KN"", ""is_mobile"": true}" 18503,7,417,2017-01-08 21:39:06,http://goldner.biz/demarcus.hilpert,0.9454828466,68.102.75.155,"{""location"": ""PW"", ""is_mobile"": true}" 18504,7,417,2017-02-01 23:08:13,http://jaskolskihirthe.io/bennett.aufderhar,0.8105860600,145.20.59.51,"{""location"": ""MV"", ""is_mobile"": false}" 18505,7,417,2017-05-13 15:43:13,http://boyle.net/betty,0.5084118334,147.188.96.50,"{""location"": ""CR"", ""is_mobile"": false}" 18506,7,417,2017-02-05 04:22:12,http://simonis.net/donna_thiel,0.3626108852,210.174.129.9,"{""location"": ""EG"", ""is_mobile"": false}" 18507,7,417,2017-05-27 19:34:48,http://schuster.com/darron.watsica,0.4369563826,196.52.208.123,"{""location"": ""MG"", ""is_mobile"": false}" 18508,7,417,2017-02-07 15:09:39,http://jastzulauf.com/eudora,0.1179031297,73.100.107.86,"{""location"": ""GE"", ""is_mobile"": true}" 18509,7,417,2017-02-15 15:25:59,http://zboncak.info/augustus_bashirian,0.8364224235,126.81.231.98,"{""location"": ""SM"", ""is_mobile"": false}" 5552,2,120,2017-02-06 18:28:07,http://baumbachdamore.biz/camila_paucek,0.2769631784,25.77.128.93,"{""location"": ""VG"", ""is_mobile"": false}" 5553,2,120,2017-05-05 08:25:11,http://hackett.org/loy_schinner,0.2624665798,121.84.95.101,"{""location"": ""AF"", ""is_mobile"": false}" 5554,2,120,2017-02-23 09:30:15,http://cremin.com/ulises,0.3104429988,3.87.26.249,"{""location"": ""PT"", ""is_mobile"": true}" 5555,2,120,2017-02-25 07:55:16,http://lindgren.org/alfred_funk,0.5243660597,209.44.249.9,"{""location"": ""VC"", ""is_mobile"": true}" 5556,2,120,2017-04-02 19:40:39,http://jerdeanderson.biz/reanna_graham,0.5872046756,96.160.41.45,"{""location"": ""KN"", ""is_mobile"": true}" 5557,2,120,2017-04-14 22:03:11,http://krajcikbailey.org/letitia_mckenzie,0.8991767370,254.71.197.89,"{""location"": ""VI"", ""is_mobile"": true}" 5558,2,121,2017-05-06 23:48:53,http://raynor.io/napoleon_mann,0.7174410153,170.178.98.161,"{""location"": ""LV"", ""is_mobile"": true}" 5559,2,121,2016-12-29 18:31:00,http://koelpin.co/brenda,0.3291658923,200.62.131.209,"{""location"": ""HR"", ""is_mobile"": true}" 5560,2,121,2017-01-26 21:28:38,http://oconnellondricka.org/arlo,0.3260029647,30.49.92.178,"{""location"": ""SD"", ""is_mobile"": false}" 5561,2,121,2017-06-05 18:52:22,http://larson.info/taurean,0.8736773487,246.235.65.122,"{""location"": ""WS"", ""is_mobile"": true}" 5562,2,121,2017-03-25 09:19:18,http://macejkovicrice.biz/nayeli.lemke,0.2503879138,55.177.125.168,"{""location"": ""BZ"", ""is_mobile"": true}" 5563,2,121,2017-04-01 23:12:27,http://nicolaswehner.net/mitchell,0.9501968867,84.177.177.106,"{""location"": ""ZW"", ""is_mobile"": true}" 5564,2,121,2017-06-10 13:41:24,http://effertz.co/joanne,0.2393192311,152.248.17.94,"{""location"": ""SJ"", ""is_mobile"": true}" 5565,2,121,2017-03-14 04:13:27,http://hermannmurray.io/fritz.shanahan,0.6420546242,236.22.122.238,"{""location"": ""CF"", ""is_mobile"": false}" 5566,2,121,2017-02-22 20:48:11,http://schambergermurray.biz/hailey_jones,0.5501665785,78.217.238.80,"{""location"": ""IN"", ""is_mobile"": false}" 5567,2,121,2016-12-31 21:48:41,http://bogisich.co/kaylin,0.3629404283,250.132.193.31,"{""location"": ""MS"", ""is_mobile"": true}" 5568,2,121,2017-04-12 22:59:14,http://zieme.com/vicky.koepp,0.2712191309,11.88.191.69,"{""location"": ""PE"", ""is_mobile"": true}" 5569,2,121,2017-03-03 14:44:54,http://reilly.org/erica.lesch,0.7141491912,223.97.115.252,"{""location"": ""ID"", ""is_mobile"": false}" 5570,2,121,2017-02-08 01:31:52,http://brakus.name/naomi.waters,0.6222800761,194.212.193.197,"{""location"": ""MK"", ""is_mobile"": true}" 5571,2,121,2016-12-15 04:27:25,http://rogahn.com/brennan,0.0157249185,162.87.86.215,"{""location"": ""IR"", ""is_mobile"": false}" 5572,2,121,2017-03-10 17:35:58,http://little.co/myrl.johnston,0.7704758479,68.175.39.81,"{""location"": ""KI"", ""is_mobile"": false}" 5573,2,121,2016-12-16 06:42:40,http://kingklocko.org/carley_rodriguez,0.7177251067,231.194.55.144,"{""location"": ""PM"", ""is_mobile"": true}" 5574,2,121,2017-02-28 00:06:38,http://hauck.co/madyson,0.5809788234,141.162.66.99,"{""location"": ""LT"", ""is_mobile"": false}" 5575,2,121,2017-03-02 07:42:59,http://gleason.com/eliseo,0.4808586162,91.30.228.249,"{""location"": ""SN"", ""is_mobile"": false}" 5576,2,121,2017-02-17 03:13:41,http://gottlieb.org/don,0.5846346886,253.2.221.10,"{""location"": ""AE"", ""is_mobile"": false}" 5577,2,121,2017-03-02 02:45:48,http://stehr.name/sophie.schaefer,0.3063645881,89.166.235.38,"{""location"": ""MC"", ""is_mobile"": false}" 5578,2,121,2017-03-27 06:15:44,http://sauer.info/janice,0.5732882677,177.34.159.250,"{""location"": ""CZ"", ""is_mobile"": true}" 5579,2,121,2017-04-23 20:15:29,http://reilly.biz/zoila,0.8406832400,77.39.232.35,"{""location"": ""MV"", ""is_mobile"": false}" 5580,2,121,2017-05-05 22:22:18,http://shields.net/jaden,0.2359884292,194.126.3.58,"{""location"": ""GU"", ""is_mobile"": false}" 5581,2,121,2017-03-06 22:22:34,http://jakubowskistiedemann.co/orrin,0.4463581055,187.172.33.186,"{""location"": ""UA"", ""is_mobile"": false}" 5582,2,121,2017-04-29 01:32:20,http://hagenes.co/gerry,0.5756031944,221.83.220.177,"{""location"": ""NZ"", ""is_mobile"": true}" 5583,2,121,2017-04-22 16:16:14,http://kshlerin.org/lizzie_sauer,0.8930634425,214.112.233.202,"{""location"": ""CU"", ""is_mobile"": true}" 5584,2,121,2017-03-31 21:10:03,http://maggiodouglas.co/aleandra.johnston,0.3282436168,179.28.138.74,"{""location"": ""CU"", ""is_mobile"": true}" 5585,2,121,2017-01-28 18:55:02,http://bradtke.co/kian_heel,0.8189887782,193.208.165.153,"{""location"": ""MW"", ""is_mobile"": false}" 5586,2,121,2016-12-31 01:08:44,http://heller.co/tristian_schowalter,0.1290170533,78.16.243.21,"{""location"": ""MW"", ""is_mobile"": false}" 5587,2,121,2017-03-14 22:15:52,http://ward.biz/marie,0.2171582042,244.194.125.12,"{""location"": ""TF"", ""is_mobile"": true}" 5588,2,121,2016-12-26 07:55:17,http://schneiderbotsford.net/clement_labadie,0.5816226055,149.117.124.177,"{""location"": ""VU"", ""is_mobile"": true}" 5589,2,121,2017-03-15 01:38:22,http://bergstrom.name/madie_mcclure,0.8410342241,76.23.148.129,"{""location"": ""IM"", ""is_mobile"": true}" 5590,2,121,2017-03-05 09:18:00,http://zemlak.name/brycen_gusikowski,0.0750288517,20.226.94.68,"{""location"": ""LK"", ""is_mobile"": true}" 5591,2,121,2017-01-11 21:31:54,http://rathstroman.com/dallas.grant,0.4044119428,42.149.206.196,"{""location"": ""BG"", ""is_mobile"": false}" 5592,2,121,2017-02-19 23:12:47,http://weinat.io/marjorie,0.4101038468,61.102.161.123,"{""location"": ""UG"", ""is_mobile"": true}" 5593,2,121,2017-05-28 22:07:11,http://wisozkbraun.io/laura.corkery,0.8177847166,192.247.134.249,"{""location"": ""TL"", ""is_mobile"": true}" 5594,2,121,2017-02-21 03:14:17,http://huels.org/marjorie.rau,0.8572015392,120.43.144.164,"{""location"": ""IQ"", ""is_mobile"": true}" 5595,2,121,2017-03-27 15:46:41,http://spencer.biz/elton,0.5073776146,148.152.134.218,"{""location"": ""KP"", ""is_mobile"": false}" 5596,2,122,2017-06-11 05:23:21,http://upton.biz/elbert,0.0525197416,97.27.152.3,"{""location"": ""SX"", ""is_mobile"": true}" 5597,2,122,2017-01-25 03:52:14,http://schneider.name/timmothy_zemlak,0.6151952879,78.253.211.228,"{""location"": ""AO"", ""is_mobile"": true}" 5598,2,122,2016-12-21 16:37:56,http://kovacekfadel.co/mac,0.8012405835,110.208.38.185,"{""location"": ""MG"", ""is_mobile"": true}" 5599,2,122,2017-05-20 09:20:11,http://mayerspencer.org/ardith_pagac,0.0732647060,180.92.34.211,"{""location"": ""ME"", ""is_mobile"": false}" 5600,2,122,2017-02-03 00:54:47,http://schaeferlabadie.com/enos,0.5889282112,9.153.147.254,"{""location"": ""BS"", ""is_mobile"": true}" 5601,2,122,2017-02-16 04:13:36,http://brekke.net/alexandra_keeling,0.1781271260,114.117.86.72,"{""location"": ""TN"", ""is_mobile"": true}" 5602,2,122,2017-04-03 06:26:09,http://weinat.net/wilson.huels,0.2621352054,207.219.102.3,"{""location"": ""RO"", ""is_mobile"": true}" 18510,7,417,2017-04-14 18:14:33,http://parisianyundt.co/randall,0.8717181298,140.138.170.9,"{""location"": ""TR"", ""is_mobile"": false}" 18511,7,417,2017-03-20 01:28:08,http://durgan.org/willard.bruen,0.4289423634,233.78.183.227,"{""location"": ""IQ"", ""is_mobile"": false}" 18512,7,418,2017-03-29 23:31:39,http://west.name/lukas.turner,0.0301787969,33.47.119.76,"{""location"": ""IL"", ""is_mobile"": true}" 18513,7,418,2017-04-17 23:12:16,http://towne.io/sincere.beahan,0.6611728461,128.207.68.239,"{""location"": ""PY"", ""is_mobile"": true}" 18514,7,418,2017-03-03 00:44:17,http://ziemezieme.com/otilia,0.4612432409,140.208.175.203,"{""location"": ""MC"", ""is_mobile"": false}" 18515,7,418,2017-05-23 02:12:47,http://keelinggoyette.io/tyrel_dooley,0.2437858065,178.129.78.193,"{""location"": ""PL"", ""is_mobile"": false}" 18516,7,418,2017-02-24 11:52:49,http://schmelerlind.name/christine,0.9400889493,153.43.217.115,"{""location"": ""MH"", ""is_mobile"": true}" 18517,7,418,2017-03-27 23:05:35,http://abbottheaney.com/kiera,0.8430033825,147.206.93.5,"{""location"": ""ME"", ""is_mobile"": false}" 18518,7,418,2017-05-31 08:28:21,http://mertz.com/jennyfer,0.7463442173,43.178.125.43,"{""location"": ""CY"", ""is_mobile"": false}" 18519,7,418,2017-06-03 19:06:31,http://schimmel.com/augustus,0.7599732722,178.38.148.230,"{""location"": ""HN"", ""is_mobile"": false}" 18520,7,418,2017-04-09 03:59:06,http://lynchkoch.com/armani,0.6067520875,84.140.135.72,"{""location"": ""SO"", ""is_mobile"": false}" 18521,7,418,2017-02-14 10:50:39,http://langworthwilderman.info/london_ferry,0.3115539105,184.152.163.234,"{""location"": ""PG"", ""is_mobile"": false}" 18522,7,418,2017-01-02 12:58:51,http://langosh.co/isidro,0.5014771035,118.123.44.62,"{""location"": ""BQ"", ""is_mobile"": true}" 18523,7,418,2017-02-19 18:04:47,http://uptonmckenzie.io/dayton_jacobson,0.5799118817,243.148.4.25,"{""location"": ""AW"", ""is_mobile"": true}" 18524,7,418,2017-03-17 20:21:41,http://becker.biz/lauren_ward,0.0210379870,182.7.125.56,"{""location"": ""QA"", ""is_mobile"": false}" 18525,7,418,2017-03-08 23:21:05,http://schaefer.biz/domenic_bins,0.4673874946,31.99.78.181,"{""location"": ""CH"", ""is_mobile"": false}" 18526,7,418,2017-05-07 06:57:27,http://schustergutkowski.info/sam.bins,0.5499749960,56.120.68.183,"{""location"": ""MO"", ""is_mobile"": true}" 18527,7,418,2017-03-26 16:44:51,http://bednarhauck.co/catharine_mann,0.5069481611,8.131.124.130,"{""location"": ""AR"", ""is_mobile"": true}" 18528,7,418,2017-01-09 20:05:23,http://breitenbergsatterfield.info/kaley,0.7534040917,130.32.228.200,"{""location"": ""BQ"", ""is_mobile"": true}" 18529,7,418,2017-06-14 04:37:02,http://hamillfranecki.com/zula,0.2037844598,128.137.128.104,"{""location"": ""DM"", ""is_mobile"": true}" 18530,7,418,2017-05-07 02:12:06,http://schusterhudson.co/lura_lang,0.1159288876,140.84.130.106,"{""location"": ""LV"", ""is_mobile"": true}" 18531,7,418,2017-05-31 23:21:39,http://jerde.io/clovis,0.6758328080,127.72.78.170,"{""location"": ""BN"", ""is_mobile"": true}" 18532,7,418,2016-12-15 07:07:30,http://graham.org/rod_tromp,0.0890454975,151.250.155.198,"{""location"": ""HR"", ""is_mobile"": false}" 18533,7,418,2017-01-13 18:38:41,http://gislasondonnelly.com/mitchell,0.8325298254,159.208.49.113,"{""location"": ""KG"", ""is_mobile"": true}" 18534,7,418,2016-12-17 03:03:00,http://kuvalis.org/tre,0.3284567331,100.157.37.249,"{""location"": ""RU"", ""is_mobile"": false}" 18535,7,418,2017-01-17 06:10:01,http://reingerconn.co/jeffrey_hand,0.1293170239,166.181.58.139,"{""location"": ""MQ"", ""is_mobile"": false}" 18536,7,418,2016-12-16 09:28:56,http://predovicmarks.biz/maxwell,0.7122404881,105.157.245.238,"{""location"": ""PY"", ""is_mobile"": false}" 18537,7,418,2017-02-28 07:22:20,http://leschturcotte.biz/eugene.flatley,0.0357598565,31.200.36.4,"{""location"": ""AQ"", ""is_mobile"": false}" 18538,7,418,2017-06-10 14:22:08,http://fadelwalter.biz/jeremy.kunze,0.6757649290,159.189.172.109,"{""location"": ""SZ"", ""is_mobile"": false}" 18539,7,418,2017-05-23 09:01:33,http://brakus.info/kevin,0.8788702178,127.232.254.55,"{""location"": ""PT"", ""is_mobile"": false}" 18540,7,418,2017-05-01 17:51:05,http://larson.com/kaleigh,0.9617764927,133.12.174.71,"{""location"": ""KH"", ""is_mobile"": true}" 18541,7,418,2017-01-01 07:46:47,http://mertztowne.net/hermina_mohr,0.6041216142,134.128.213.144,"{""location"": ""GR"", ""is_mobile"": true}" 18542,7,418,2017-03-18 02:21:03,http://fay.net/martine_erdman,0.5020300660,162.143.9.173,"{""location"": ""PL"", ""is_mobile"": false}" 18543,7,418,2017-01-31 15:41:26,http://weimann.com/evans.schmitt,0.2189270209,172.108.94.58,"{""location"": ""CG"", ""is_mobile"": true}" 18544,7,418,2017-06-03 02:11:43,http://cronin.biz/ciara.hills,0.3581861300,172.65.59.218,"{""location"": ""FJ"", ""is_mobile"": true}" 18545,7,418,2017-04-11 17:56:56,http://mcdermottauer.com/beau.brekke,0.1489634535,197.221.9.214,"{""location"": ""MQ"", ""is_mobile"": true}" 18546,7,418,2017-06-13 13:39:28,http://stantonmarks.org/jeremy_bruen,0.0924847992,109.161.2.17,"{""location"": ""JP"", ""is_mobile"": true}" 18547,7,418,2017-01-07 12:54:07,http://larson.co/waldo.wuckert,0.5097729165,106.62.178.221,"{""location"": ""GS"", ""is_mobile"": false}" 18548,7,418,2017-02-21 12:04:38,http://brownlangosh.name/zane,0.7662681548,231.63.73.12,"{""location"": ""WS"", ""is_mobile"": true}" 18549,7,418,2017-01-19 03:10:27,http://durgan.biz/jackie_white,0.4981614923,35.108.47.201,"{""location"": ""UZ"", ""is_mobile"": false}" 18550,7,418,2017-01-08 09:13:36,http://kris.org/kirk.macgyver,0.1461896194,236.59.109.164,"{""location"": ""GH"", ""is_mobile"": false}" 18551,7,418,2017-04-01 14:13:06,http://denesikkoepp.net/brannon_beier,0.9989906343,72.20.52.59,"{""location"": ""ER"", ""is_mobile"": true}" 18552,7,418,2017-04-05 23:46:30,http://lang.org/dock.funk,0.7172034649,155.229.116.157,"{""location"": ""MM"", ""is_mobile"": true}" 18553,7,418,2017-04-23 19:41:13,http://hudsonraynor.biz/moshe,0.1221233251,202.111.100.32,"{""location"": ""SJ"", ""is_mobile"": true}" 18554,7,418,2017-05-24 15:49:42,http://nitzsche.io/trisha_gulgowski,0.2183577295,184.250.227.112,"{""location"": ""ER"", ""is_mobile"": true}" 18555,7,418,2017-03-11 22:00:18,http://bergnaum.com/stephan,0.9008008435,235.169.61.194,"{""location"": ""HK"", ""is_mobile"": false}" 18556,7,418,2017-01-14 20:49:19,http://oconnerkunde.name/alvah,0.5852602244,183.19.250.136,"{""location"": ""NG"", ""is_mobile"": true}" 18557,7,418,2017-05-08 04:36:01,http://mertz.info/benny,0.6766673304,147.228.56.201,"{""location"": ""MP"", ""is_mobile"": true}" 18558,7,418,2017-03-06 12:51:32,http://ohara.name/alexandrine_becker,0.1037649795,62.110.203.210,"{""location"": ""BJ"", ""is_mobile"": false}" 18559,7,418,2017-05-23 02:31:01,http://langworthwolf.io/caden,0.7307231736,228.61.37.179,"{""location"": ""VC"", ""is_mobile"": true}" 18560,7,418,2017-03-12 20:41:54,http://dickens.co/makenzie.macgyver,0.3741991620,26.191.6.171,"{""location"": ""AX"", ""is_mobile"": false}" 5603,2,122,2017-02-28 03:08:49,http://lehner.io/thalia_hamill,0.6552790678,169.106.242.209,"{""location"": ""US"", ""is_mobile"": true}" 5604,2,122,2017-03-03 05:33:57,http://stehrgerlach.co/mackenzie,0.8465299343,70.243.150.115,"{""location"": ""CG"", ""is_mobile"": false}" 5605,2,122,2017-03-16 11:46:06,http://kautzer.io/casimir,0.3418915507,150.28.54.183,"{""location"": ""BN"", ""is_mobile"": false}" 5606,2,122,2016-12-18 14:08:44,http://doylewiza.org/tobin,0.0689830195,4.196.238.40,"{""location"": ""SR"", ""is_mobile"": false}" 5607,2,122,2017-01-10 06:53:08,http://eichmannprosacco.net/adonis,0.8222809510,41.237.87.138,"{""location"": ""MG"", ""is_mobile"": false}" 5608,2,122,2017-05-14 07:23:30,http://boehm.info/roderick_vonrueden,0.1232292598,91.70.16.41,"{""location"": ""SK"", ""is_mobile"": false}" 5609,2,122,2017-03-23 20:14:15,http://hane.com/adalberto_stokes,0.0511921362,54.102.28.51,"{""location"": ""HT"", ""is_mobile"": false}" 5610,2,122,2017-06-12 04:28:04,http://will.name/lavern,0.2048638625,93.223.101.131,"{""location"": ""NP"", ""is_mobile"": false}" 5611,2,122,2017-02-14 06:04:59,http://langondricka.name/lauren.ledner,0.3938950614,14.11.236.184,"{""location"": ""LY"", ""is_mobile"": false}" 5612,2,122,2017-01-08 12:28:09,http://funk.org/jordy,0.5584174342,19.145.94.170,"{""location"": ""FO"", ""is_mobile"": true}" 5613,2,122,2017-02-09 18:38:06,http://doyle.biz/kayden,0.3975905337,156.213.243.45,"{""location"": ""MF"", ""is_mobile"": false}" 5614,2,122,2017-01-21 19:08:20,http://breitenberg.name/jaleel,0.2252362847,186.147.216.220,"{""location"": ""SS"", ""is_mobile"": false}" 5615,2,122,2017-04-16 05:01:44,http://beattylangosh.co/arlo,0.2373610426,146.31.222.51,"{""location"": ""PH"", ""is_mobile"": false}" 5616,2,122,2017-03-04 02:32:54,http://lockmanorn.net/monte,0.5203617269,113.11.182.230,"{""location"": ""DM"", ""is_mobile"": false}" 5617,2,122,2017-04-15 13:44:57,http://langarmstrong.com/jee,0.6489333067,136.40.142.102,"{""location"": ""EE"", ""is_mobile"": false}" 5618,2,123,2017-02-28 14:52:41,http://casper.net/isabell_koch,0.1977091970,178.23.82.102,"{""location"": ""UA"", ""is_mobile"": true}" 5619,2,123,2017-04-30 18:58:54,http://hills.org/anya_champlin,0.6130047890,153.136.87.218,"{""location"": ""VU"", ""is_mobile"": true}" 5620,2,123,2017-03-06 08:43:57,http://pouroscummings.name/harrison.streich,0.4953669465,105.130.216.217,"{""location"": ""MR"", ""is_mobile"": false}" 5621,2,123,2017-04-20 17:10:50,http://klockoreilly.co/reanna_feest,0.6596823521,225.182.184.149,"{""location"": ""VI"", ""is_mobile"": false}" 5622,2,123,2016-12-14 06:06:46,http://jaskolskilangworth.co/benjamin,0.7586395181,140.140.123.206,"{""location"": ""AM"", ""is_mobile"": false}" 5623,2,123,2017-02-27 01:33:21,http://pollich.org/shaina_torphy,0.1806073164,71.171.150.205,"{""location"": ""IN"", ""is_mobile"": true}" 5624,2,123,2017-01-29 16:41:03,http://daugherty.name/miguel,0.9640283984,105.75.128.238,"{""location"": ""PH"", ""is_mobile"": true}" 5625,2,123,2017-05-25 06:59:46,http://bernhard.info/terrence,0.1376543723,208.236.101.188,"{""location"": ""ZW"", ""is_mobile"": true}" 5626,2,123,2017-01-15 00:41:00,http://stiedemann.io/zoila,0.5488238597,116.153.115.35,"{""location"": ""MU"", ""is_mobile"": false}" 5627,2,123,2017-03-29 06:22:47,http://larkin.co/dale.kling,0.3315218729,96.17.82.203,"{""location"": ""VI"", ""is_mobile"": true}" 5628,2,123,2017-01-31 23:42:54,http://kovacekmoen.info/jackie,0.7619061454,32.225.68.30,"{""location"": ""BL"", ""is_mobile"": true}" 5629,2,123,2017-04-06 20:25:50,http://hettinger.io/lexus,0.2956006013,200.91.235.142,"{""location"": ""MY"", ""is_mobile"": false}" 5630,2,123,2017-04-19 06:50:06,http://ratkedamore.biz/marlen,0.3363867294,109.114.227.56,"{""location"": ""NC"", ""is_mobile"": true}" 5631,2,123,2017-05-19 11:16:43,http://willms.org/dusty_homenick,0.6639271765,100.45.205.56,"{""location"": ""TV"", ""is_mobile"": true}" 5632,2,123,2016-12-25 00:29:30,http://collins.name/jana,0.1172922812,201.8.4.233,"{""location"": ""BY"", ""is_mobile"": true}" 5633,2,123,2017-06-07 22:33:52,http://cremin.co/enoch.jones,0.4941839749,4.46.228.211,"{""location"": ""AD"", ""is_mobile"": true}" 5634,2,123,2017-06-10 00:08:37,http://gaylord.com/sierra.will,0.2271125552,70.58.146.198,"{""location"": ""AG"", ""is_mobile"": true}" 5635,2,123,2017-02-28 20:39:24,http://rodriguez.org/sheldon_lemke,0.4082438863,3.70.254.216,"{""location"": ""LC"", ""is_mobile"": true}" 5636,2,123,2017-01-10 02:18:26,http://rath.biz/nikita_predovic,0.8353409176,63.38.22.15,"{""location"": ""MM"", ""is_mobile"": false}" 5637,2,123,2017-03-13 18:19:02,http://lebsack.net/neil,0.0391403937,202.46.125.226,"{""location"": ""GR"", ""is_mobile"": false}" 5638,2,123,2017-02-25 13:57:48,http://greenholt.co/lesley,0.8703039451,38.93.174.201,"{""location"": ""UM"", ""is_mobile"": true}" 5639,2,123,2017-03-01 13:54:10,http://quigleyhackett.name/kenna,0.1733707779,34.12.11.33,"{""location"": ""QA"", ""is_mobile"": false}" 5640,2,123,2017-03-27 11:59:02,http://brekke.biz/al,0.6509157802,112.209.233.178,"{""location"": ""AS"", ""is_mobile"": false}" 5641,2,123,2017-01-14 15:29:28,http://bruen.co/dorcas.donnelly,0.0098901288,201.103.159.81,"{""location"": ""JO"", ""is_mobile"": false}" 5642,2,123,2016-12-19 17:00:53,http://nolanveum.co/stuart_baumbach,0.1958748831,56.252.213.118,"{""location"": ""FK"", ""is_mobile"": false}" 5643,2,123,2017-01-19 18:13:02,http://rosenbaumbraun.org/newell_nolan,0.6876655548,151.71.167.149,"{""location"": ""PE"", ""is_mobile"": true}" 5644,2,123,2017-06-11 12:41:43,http://botsfordweber.name/micheal.rogahn,0.0833434086,227.21.155.118,"{""location"": ""JO"", ""is_mobile"": false}" 5645,2,123,2017-03-15 04:45:27,http://weber.co/gladyce.lubowitz,0.6758070682,11.31.30.4,"{""location"": ""MT"", ""is_mobile"": true}" 5646,2,123,2017-03-23 00:37:43,http://watsica.org/breanna_boyer,0.9707997681,119.148.207.70,"{""location"": ""PE"", ""is_mobile"": true}" 5647,2,123,2017-05-12 20:27:07,http://paucek.info/alexis_johns,0.6935045532,158.81.50.197,"{""location"": ""VG"", ""is_mobile"": true}" 5648,2,123,2017-03-06 00:46:20,http://mayer.biz/alverta,0.9500753404,194.232.136.118,"{""location"": ""BW"", ""is_mobile"": true}" 5649,2,123,2017-03-03 19:59:20,http://barrows.com/amely,0.9095211672,80.99.171.141,"{""location"": ""TO"", ""is_mobile"": false}" 5650,2,123,2017-03-07 00:31:19,http://pourosjerde.net/amina,0.5417570059,209.165.149.48,"{""location"": ""TH"", ""is_mobile"": true}" 5651,2,123,2017-04-03 05:51:23,http://labadie.net/janis,0.8281787863,219.24.70.80,"{""location"": ""NA"", ""is_mobile"": false}" 5652,2,123,2017-06-07 03:27:46,http://cartwrightebert.biz/rosemary_moriette,0.0407220224,184.177.238.72,"{""location"": ""CZ"", ""is_mobile"": false}" 5653,2,123,2017-05-04 15:43:01,http://beatty.co/dayton_grant,0.2032023397,4.103.29.244,"{""location"": ""ID"", ""is_mobile"": true}" 5654,2,123,2017-03-25 20:50:52,http://brakus.io/jerod_huel,0.7082145468,8.190.11.40,"{""location"": ""BZ"", ""is_mobile"": true}" 18561,7,418,2017-02-20 06:03:48,http://skiles.info/yesenia_schamberger,0.6931588131,17.138.109.46,"{""location"": ""DM"", ""is_mobile"": true}" 18562,7,418,2017-04-14 17:45:21,http://dibbertreynolds.co/marcelo.hermann,0.9263711703,137.224.245.254,"{""location"": ""TV"", ""is_mobile"": false}" 18563,7,418,2016-12-23 09:36:40,http://robertsparisian.name/sofia,0.1651552198,167.208.52.34,"{""location"": ""BH"", ""is_mobile"": true}" 18564,7,418,2017-01-17 14:53:00,http://townebode.org/olaf,0.7328211955,14.100.90.168,"{""location"": ""KR"", ""is_mobile"": true}" 18565,7,418,2017-01-12 23:44:35,http://nienow.name/lincoln.konopelski,0.1526281651,203.211.122.222,"{""location"": ""UA"", ""is_mobile"": true}" 18566,7,418,2017-01-12 10:52:05,http://ritchieklein.name/johnpaul,0.8370281320,168.114.176.154,"{""location"": ""LB"", ""is_mobile"": false}" 18567,7,418,2017-03-28 05:09:38,http://jacobs.biz/brionna,0.2652550534,250.171.55.159,"{""location"": ""YT"", ""is_mobile"": true}" 18568,7,418,2017-01-03 03:44:01,http://carrollpadberg.io/zion,0.0821872285,69.17.245.40,"{""location"": ""NE"", ""is_mobile"": false}" 18569,7,418,2017-01-20 12:11:23,http://dietrich.name/eleanora_kutch,0.9704491274,251.75.228.72,"{""location"": ""RW"", ""is_mobile"": false}" 18570,7,418,2017-01-04 00:11:36,http://walsh.org/dee,0.6817774294,135.128.79.32,"{""location"": ""AM"", ""is_mobile"": true}" 18571,7,418,2017-05-24 02:57:18,http://abshire.name/elia,0.8786104802,184.39.204.100,"{""location"": ""LU"", ""is_mobile"": true}" 18572,7,418,2017-03-16 23:19:55,http://waelchi.io/olin,0.8168462119,111.88.156.131,"{""location"": ""IE"", ""is_mobile"": true}" 18573,7,418,2017-04-21 10:20:43,http://lesch.net/jada,0.8789025921,201.165.10.153,"{""location"": ""IO"", ""is_mobile"": false}" 18574,7,418,2017-05-01 20:20:02,http://zulauf.io/toy_green,0.2230369480,115.144.107.157,"{""location"": ""IM"", ""is_mobile"": true}" 18575,7,418,2017-01-14 09:32:27,http://moenfritsch.biz/rigoberto,0.0353390856,62.251.77.36,"{""location"": ""GW"", ""is_mobile"": false}" 18576,7,418,2017-04-11 23:16:11,http://mayert.co/camille_casper,0.6474404396,126.227.44.26,"{""location"": ""SV"", ""is_mobile"": true}" 18577,7,418,2017-02-19 20:00:21,http://raynor.org/gretchen.metz,0.6057090155,73.240.94.217,"{""location"": ""CO"", ""is_mobile"": true}" 18578,7,418,2016-12-28 10:27:49,http://ullrich.biz/tyson,0.3270929404,60.63.146.196,"{""location"": ""TM"", ""is_mobile"": true}" 18579,7,418,2017-03-19 00:41:17,http://marks.org/donna,0.2146488746,132.125.180.174,"{""location"": ""YT"", ""is_mobile"": false}" 18580,7,418,2017-06-02 12:50:26,http://goyettejacobs.info/colin,0.6860281696,25.171.236.195,"{""location"": ""MW"", ""is_mobile"": true}" 18581,7,419,2017-01-08 06:34:34,http://emard.info/brisa,0.6204600086,66.137.171.116,"{""location"": ""MT"", ""is_mobile"": true}" 18582,7,419,2017-05-18 15:12:03,http://barrows.io/helmer,0.3408333938,4.214.196.253,"{""location"": ""AO"", ""is_mobile"": true}" 18583,7,419,2017-05-12 01:20:35,http://braundamore.net/mustafa_streich,0.8533678981,206.207.170.177,"{""location"": ""CZ"", ""is_mobile"": true}" 18584,7,419,2017-04-14 20:14:40,http://ebert.name/michel.daniel,0.4590760856,231.33.202.103,"{""location"": ""VN"", ""is_mobile"": false}" 18585,7,419,2017-01-05 07:37:11,http://coleschuppe.co/karli,0.1008267552,61.109.161.160,"{""location"": ""BO"", ""is_mobile"": false}" 18586,7,419,2017-04-03 22:10:12,http://dubuque.net/irving,0.5712131003,66.145.105.206,"{""location"": ""MY"", ""is_mobile"": true}" 18587,7,419,2017-05-30 13:32:09,http://wisoky.co/shanon,0.2317456837,142.170.94.73,"{""location"": ""EG"", ""is_mobile"": false}" 18588,7,419,2017-04-17 03:02:39,http://beatty.org/penelope.dickinson,0.5194353192,22.121.79.82,"{""location"": ""AM"", ""is_mobile"": true}" 18589,7,419,2017-03-22 06:03:51,http://osinski.io/alene,0.0843663061,200.141.173.171,"{""location"": ""EE"", ""is_mobile"": false}" 18590,7,419,2017-03-20 06:37:42,http://carter.info/shaniya,0.7558983671,123.215.36.191,"{""location"": ""CV"", ""is_mobile"": false}" 18591,7,419,2017-05-01 19:57:45,http://friesenbuckridge.biz/josephine_trantow,0.8530990932,47.135.148.8,"{""location"": ""PS"", ""is_mobile"": true}" 18592,7,419,2017-05-19 06:58:35,http://brekke.name/ettie,0.2249988232,115.94.164.240,"{""location"": ""BO"", ""is_mobile"": true}" 18593,7,419,2017-03-06 11:50:44,http://feestabbott.com/khalid,0.3810928544,164.16.55.55,"{""location"": ""AS"", ""is_mobile"": false}" 18594,7,419,2017-05-02 10:03:24,http://reinger.com/minnie,0.8782992899,174.244.205.177,"{""location"": ""GI"", ""is_mobile"": true}" 18595,7,419,2017-01-19 17:46:19,http://keler.org/eloisa.denesik,0.7055818251,139.118.236.124,"{""location"": ""HN"", ""is_mobile"": true}" 18596,7,419,2017-01-22 11:11:18,http://cummeratamarquardt.name/everardo,0.2110117857,142.125.3.48,"{""location"": ""PA"", ""is_mobile"": true}" 18597,7,419,2017-05-07 16:17:03,http://schinnerkertzmann.info/mara,0.1080676228,184.28.240.198,"{""location"": ""ZW"", ""is_mobile"": true}" 18598,7,419,2017-03-17 20:40:53,http://dare.co/dillan_beier,0.1686890602,147.244.68.214,"{""location"": ""MD"", ""is_mobile"": false}" 18599,7,419,2017-05-19 11:38:02,http://wyman.com/trenton.kris,0.0757121430,76.220.184.231,"{""location"": ""KG"", ""is_mobile"": true}" 18600,7,419,2017-06-08 11:37:32,http://lind.net/jamarcus,0.1369721214,4.174.179.220,"{""location"": ""LI"", ""is_mobile"": false}" 18601,7,419,2017-02-27 14:11:05,http://wildermanhand.co/guillermo,0.0218152477,182.178.13.88,"{""location"": ""IR"", ""is_mobile"": false}" 18602,7,419,2017-04-08 05:42:45,http://quigley.name/prudence,0.3216867933,170.77.245.240,"{""location"": ""MD"", ""is_mobile"": true}" 18603,7,419,2017-05-03 14:21:18,http://parisianmclaughlin.io/terrell_lynch,0.4612640062,13.29.27.109,"{""location"": ""VG"", ""is_mobile"": false}" 18604,7,419,2017-01-31 13:55:53,http://nikolaus.org/kenyatta,0.3960438848,234.245.190.191,"{""location"": ""NO"", ""is_mobile"": true}" 18605,7,419,2017-01-13 08:29:12,http://spencerbahringer.net/nikita,0.7358828051,55.135.221.203,"{""location"": ""JP"", ""is_mobile"": true}" 18606,7,419,2017-04-29 12:53:00,http://schowalter.net/jennie,0.7373861824,194.249.107.89,"{""location"": ""JP"", ""is_mobile"": false}" 18607,7,419,2016-12-31 08:37:34,http://reichel.com/otha_cruickshank,0.8614764957,142.148.245.131,"{""location"": ""SC"", ""is_mobile"": false}" 18608,7,419,2017-04-10 03:32:58,http://connweber.biz/leanne.douglas,0.7029770384,236.221.203.75,"{""location"": ""TM"", ""is_mobile"": true}" 18609,7,419,2017-03-02 14:38:31,http://dickiromaguera.com/carmine,0.4631965600,116.22.118.15,"{""location"": ""NR"", ""is_mobile"": true}" 18610,7,419,2017-04-14 19:11:33,http://walker.net/robyn,0.9720840630,243.84.141.151,"{""location"": ""GR"", ""is_mobile"": false}" 18611,7,419,2017-06-07 01:46:14,http://trantow.biz/maxine,0.3445042321,32.106.67.204,"{""location"": ""TK"", ""is_mobile"": true}" 5655,2,123,2017-04-06 13:18:03,http://koelpinkoepp.biz/florine,0.9335310549,241.55.65.251,"{""location"": ""TD"", ""is_mobile"": false}" 5656,2,123,2017-04-11 11:01:19,http://wisozk.info/dimitri,0.3134116083,147.112.227.170,"{""location"": ""TO"", ""is_mobile"": false}" 5657,2,123,2017-01-03 15:03:14,http://stracke.com/hettie.homenick,0.0857529851,115.200.63.80,"{""location"": ""CZ"", ""is_mobile"": false}" 5658,2,123,2017-04-17 23:12:02,http://nienow.co/garett_kuhic,0.8255140213,142.61.224.210,"{""location"": ""KW"", ""is_mobile"": true}" 5659,2,123,2017-04-02 15:01:57,http://stanton.org/maria_kutch,0.7386932645,195.245.131.144,"{""location"": ""BB"", ""is_mobile"": true}" 5660,2,123,2017-04-13 02:28:30,http://wardwisozk.info/jordy,0.0477991954,46.166.125.240,"{""location"": ""PT"", ""is_mobile"": false}" 5661,2,123,2017-04-24 05:04:27,http://ankundingdach.biz/dayne_price,0.3981853780,193.132.233.14,"{""location"": ""LU"", ""is_mobile"": false}" 5662,2,123,2017-02-13 07:47:47,http://mitchell.io/susana.turner,0.0533075858,52.226.14.230,"{""location"": ""ML"", ""is_mobile"": false}" 5663,2,123,2017-02-11 03:12:50,http://becker.org/mittie,0.2915007985,206.186.210.231,"{""location"": ""SD"", ""is_mobile"": true}" 5664,2,124,2017-05-24 11:37:31,http://thiel.info/keely.mohr,0.8405438866,166.165.59.167,"{""location"": ""TN"", ""is_mobile"": true}" 5665,2,124,2017-02-20 07:36:08,http://hodkiewicz.org/josiah,0.2806561176,70.164.141.201,"{""location"": ""KN"", ""is_mobile"": true}" 5666,2,124,2016-12-31 08:32:48,http://batzharris.net/emma,0.7316693190,239.33.243.139,"{""location"": ""GN"", ""is_mobile"": false}" 5667,2,124,2017-01-06 13:10:02,http://watsicabaumbach.biz/yasmin,0.9629460603,36.70.50.154,"{""location"": ""GB"", ""is_mobile"": false}" 5668,2,124,2017-06-04 22:43:36,http://mcdermottschmeler.biz/briana,0.4395656165,227.74.211.72,"{""location"": ""SI"", ""is_mobile"": false}" 5669,2,124,2017-04-03 15:15:33,http://beahan.info/ryder,0.7247823506,188.16.231.232,"{""location"": ""DK"", ""is_mobile"": false}" 5670,2,124,2017-05-17 03:17:02,http://cristkuphal.co/cloyd,0.1586950389,25.98.49.80,"{""location"": ""DE"", ""is_mobile"": true}" 5671,2,124,2017-04-11 21:38:16,http://walterlindgren.co/jorge_lowe,0.9868320089,53.118.162.128,"{""location"": ""NF"", ""is_mobile"": false}" 5672,2,124,2017-03-03 02:06:02,http://kihn.name/tobin,0.5741109510,36.171.118.111,"{""location"": ""MF"", ""is_mobile"": false}" 5673,2,124,2017-05-20 17:23:45,http://will.net/stacey_christiansen,0.9258507477,72.39.87.99,"{""location"": ""EH"", ""is_mobile"": true}" 5674,2,124,2017-05-19 09:17:32,http://kaulke.co/aleandra.schoen,0.8014101535,192.185.69.234,"{""location"": ""KI"", ""is_mobile"": true}" 5675,2,124,2017-02-13 09:41:05,http://wiegand.org/libbie.borer,0.4388111177,197.40.226.228,"{""location"": ""ES"", ""is_mobile"": true}" 5676,2,124,2017-06-08 19:05:22,http://veum.biz/heber,0.2923166925,124.135.92.96,"{""location"": ""SR"", ""is_mobile"": false}" 5677,2,124,2017-02-17 00:39:27,http://bartell.io/hortense.powlowski,0.1484269955,167.179.221.208,"{""location"": ""LK"", ""is_mobile"": true}" 5678,2,124,2017-01-02 04:03:21,http://kihn.co/alberto_schamberger,0.8967549967,102.69.128.127,"{""location"": ""IO"", ""is_mobile"": false}" 5679,2,124,2017-04-20 18:12:45,http://heller.biz/aditya.kaulke,0.2824785719,198.172.101.224,"{""location"": ""CW"", ""is_mobile"": false}" 5680,2,124,2017-02-27 22:03:06,http://ebert.org/charlie,0.8589684329,100.210.219.233,"{""location"": ""MN"", ""is_mobile"": false}" 5681,2,124,2017-05-30 09:02:47,http://kuhlman.net/sandrine,0.3366069807,184.3.202.217,"{""location"": ""QA"", ""is_mobile"": true}" 5682,2,124,2017-03-11 21:13:50,http://beahan.name/providenci.beatty,0.2907061814,88.239.142.245,"{""location"": ""PW"", ""is_mobile"": false}" 5683,2,124,2016-12-25 20:50:08,http://collins.name/quinn,0.0107070467,203.98.228.115,"{""location"": ""CX"", ""is_mobile"": false}" 5684,2,124,2016-12-19 00:28:54,http://haag.org/gust,0.2249859458,21.163.116.132,"{""location"": ""KE"", ""is_mobile"": false}" 5685,2,124,2017-01-24 11:44:47,http://heller.org/niko,0.4411218813,194.247.150.3,"{""location"": ""PN"", ""is_mobile"": true}" 5686,2,124,2017-05-20 23:23:54,http://jerde.info/boyd.lowe,0.7871691249,242.50.45.72,"{""location"": ""VA"", ""is_mobile"": true}" 5687,2,124,2017-01-11 15:12:00,http://robelmarks.io/jermain,0.7969071278,30.144.62.248,"{""location"": ""GT"", ""is_mobile"": false}" 5688,2,124,2017-05-19 11:35:11,http://smitham.io/lee,0.7661389335,2.177.6.132,"{""location"": ""SG"", ""is_mobile"": false}" 5689,2,124,2017-05-18 13:40:35,http://dietrich.info/lela,0.3077776691,168.47.47.6,"{""location"": ""TK"", ""is_mobile"": false}" 5690,2,124,2017-01-04 00:41:48,http://spinka.name/jettie,0.5417384372,234.125.39.174,"{""location"": ""MM"", ""is_mobile"": true}" 5691,2,124,2017-05-10 10:28:46,http://hahn.net/dominic_hickle,0.4454823206,162.163.247.122,"{""location"": ""VA"", ""is_mobile"": false}" 5692,2,124,2017-02-11 02:26:09,http://runolfsdottirgrant.net/rosina_jerde,0.7561602164,166.9.119.207,"{""location"": ""ZW"", ""is_mobile"": true}" 5693,2,124,2017-05-27 13:08:18,http://labadie.net/carley.balistreri,0.0160168194,252.98.229.152,"{""location"": ""LY"", ""is_mobile"": false}" 5694,2,124,2017-06-07 21:31:31,http://bernier.org/lavern,0.1340360039,66.49.38.129,"{""location"": ""UY"", ""is_mobile"": true}" 5695,2,124,2017-03-03 09:25:40,http://kelerleuschke.info/jaylan,0.6176727127,30.161.118.251,"{""location"": ""TH"", ""is_mobile"": false}" 5696,2,125,2017-02-08 16:10:11,http://walshpadberg.com/abbie,0.3464973249,203.14.161.239,"{""location"": ""KG"", ""is_mobile"": false}" 5697,2,125,2016-12-20 08:42:53,http://williamson.biz/abe,0.0347840189,40.169.250.129,"{""location"": ""NI"", ""is_mobile"": false}" 5698,2,125,2017-03-02 09:58:12,http://wisozkfeeney.io/elena,0.4882208024,34.101.113.246,"{""location"": ""GS"", ""is_mobile"": true}" 5699,2,125,2017-05-08 12:34:55,http://kovacek.info/susie_veum,0.4791404052,142.187.227.9,"{""location"": ""WF"", ""is_mobile"": true}" 5700,2,125,2017-05-27 22:18:26,http://schowalterhuels.biz/tremaine.green,0.8983678558,183.125.36.71,"{""location"": ""VG"", ""is_mobile"": false}" 5701,2,125,2017-02-19 09:10:40,http://jacobs.biz/lurline_stark,0.9132341113,10.11.165.225,"{""location"": ""MO"", ""is_mobile"": true}" 5702,2,125,2017-04-09 08:50:09,http://hagenesjaskolski.name/rose_armstrong,0.1122665714,91.245.190.194,"{""location"": ""CV"", ""is_mobile"": true}" 5703,2,125,2017-03-13 21:01:55,http://barrowskrajcik.com/giovanny,0.3671157882,164.133.194.226,"{""location"": ""MT"", ""is_mobile"": false}" 5704,2,125,2017-03-05 22:48:31,http://schiller.io/abelardo_gaylord,0.4720550590,238.147.105.175,"{""location"": ""GA"", ""is_mobile"": true}" 5705,2,125,2017-02-02 02:20:29,http://howelloconner.net/kristoffer,0.4810909228,12.115.239.150,"{""location"": ""VI"", ""is_mobile"": true}" 5706,2,125,2017-01-27 18:21:49,http://fisher.info/sigrid,0.4759769567,12.203.245.145,"{""location"": ""JP"", ""is_mobile"": false}" 18612,7,419,2017-04-24 07:59:54,http://marks.com/reynold_wisozk,0.1937712935,194.115.219.73,"{""location"": ""MU"", ""is_mobile"": true}" 18613,7,419,2017-06-05 01:52:34,http://robel.net/rhoda,0.6999549475,140.88.114.93,"{""location"": ""SN"", ""is_mobile"": true}" 18614,7,419,2016-12-28 12:47:48,http://kochbergnaum.com/nikita,0.5266354258,131.197.118.227,"{""location"": ""KN"", ""is_mobile"": true}" 18615,7,419,2017-01-29 07:25:43,http://labadiewelch.com/onie,0.5033666487,194.40.74.91,"{""location"": ""SR"", ""is_mobile"": true}" 18616,7,419,2017-01-15 09:37:37,http://mckenzie.io/marcelino,0.1925604771,139.165.242.169,"{""location"": ""CW"", ""is_mobile"": false}" 18617,7,419,2017-04-24 12:14:22,http://erdmantrantow.io/kenya,0.7701233203,40.133.250.239,"{""location"": ""KH"", ""is_mobile"": false}" 18618,7,419,2017-05-26 19:21:54,http://kundecarroll.co/michel_hand,0.2709972438,139.114.44.157,"{""location"": ""HT"", ""is_mobile"": false}" 18619,7,419,2017-01-08 07:41:23,http://bahringer.org/cale_koch,0.9519592116,19.36.232.96,"{""location"": ""ER"", ""is_mobile"": true}" 18620,7,419,2017-01-04 18:00:00,http://olson.co/mervin.spinka,0.2472054413,22.62.201.64,"{""location"": ""LK"", ""is_mobile"": false}" 18621,7,419,2016-12-16 07:18:45,http://feeney.biz/lorenzo,0.5655499789,143.110.22.213,"{""location"": ""VA"", ""is_mobile"": false}" 18622,7,419,2017-03-04 17:09:40,http://schmittondricka.info/harmon.pfeffer,0.9022058989,146.39.167.37,"{""location"": ""LV"", ""is_mobile"": true}" 18623,7,419,2017-01-22 11:29:04,http://lynch.co/dylan,0.4912149774,178.107.247.90,"{""location"": ""AU"", ""is_mobile"": true}" 18624,7,419,2017-05-21 17:50:22,http://zieme.com/ashleigh.shanahan,0.2076619599,29.5.197.115,"{""location"": ""RW"", ""is_mobile"": false}" 18625,7,419,2017-05-19 16:52:00,http://hansen.info/chance,0.0194977910,183.108.66.224,"{""location"": ""MG"", ""is_mobile"": true}" 18626,7,419,2016-12-29 05:35:10,http://reichelhamill.net/lauren.bauch,0.8698815196,234.100.10.177,"{""location"": ""GD"", ""is_mobile"": false}" 18627,7,419,2017-01-29 13:16:50,http://barrows.io/alex,0.8551501617,109.57.135.62,"{""location"": ""SD"", ""is_mobile"": true}" 18628,7,419,2017-01-02 01:19:37,http://simonis.biz/octavia_oreilly,0.7091323866,45.47.91.79,"{""location"": ""TD"", ""is_mobile"": false}" 18629,7,419,2016-12-31 10:17:38,http://gleichner.org/faustino_schuster,0.5540109884,108.140.79.6,"{""location"": ""VI"", ""is_mobile"": false}" 18630,7,419,2017-05-10 03:53:43,http://homenick.net/roie.raynor,0.5402966620,218.33.181.135,"{""location"": ""BW"", ""is_mobile"": true}" 18631,7,419,2017-03-03 05:18:03,http://deckow.info/solon,0.4230222890,33.48.86.19,"{""location"": ""UZ"", ""is_mobile"": false}" 18632,7,419,2016-12-27 08:55:59,http://harberbeahan.org/garrett,0.1775928646,131.61.194.241,"{""location"": ""BI"", ""is_mobile"": false}" 18633,7,419,2017-05-09 01:43:29,http://rice.org/wilmer.gislason,0.7779141485,62.217.226.81,"{""location"": ""AM"", ""is_mobile"": false}" 18634,7,419,2016-12-17 02:50:58,http://herzog.name/benedict_williamson,0.8140199007,121.24.117.119,"{""location"": ""AI"", ""is_mobile"": false}" 18635,7,419,2017-03-14 15:18:29,http://lebsack.org/candido,0.8799269078,11.2.198.252,"{""location"": ""YE"", ""is_mobile"": false}" 18636,7,419,2017-05-31 10:27:43,http://hodkiewiczgottlieb.org/sabina,0.0679025015,147.8.122.87,"{""location"": ""CW"", ""is_mobile"": true}" 18637,7,419,2017-05-10 16:15:19,http://brakus.name/shania.hills,0.6189204345,122.225.77.116,"{""location"": ""PE"", ""is_mobile"": false}" 18638,7,419,2017-05-05 23:09:48,http://larsonhaag.name/hermina.turcotte,0.8758381364,27.32.254.131,"{""location"": ""BB"", ""is_mobile"": false}" 18639,7,419,2017-01-11 05:03:37,http://beierglover.name/mohammed,0.3327202858,228.20.180.98,"{""location"": ""UM"", ""is_mobile"": true}" 18640,7,419,2016-12-30 18:24:51,http://mohr.co/kasey,0.5436893746,16.24.208.2,"{""location"": ""KY"", ""is_mobile"": true}" 18641,7,420,2017-06-08 20:08:43,http://orn.co/eddie,0.1839117382,146.162.93.66,"{""location"": ""GW"", ""is_mobile"": true}" 18642,7,420,2017-04-04 17:40:39,http://jacobson.name/neha.wyman,0.3665096985,173.77.116.140,"{""location"": ""TF"", ""is_mobile"": true}" 18643,7,420,2017-03-09 22:44:47,http://cruickshank.co/louie_streich,0.0182442208,59.23.88.31,"{""location"": ""GU"", ""is_mobile"": false}" 18644,7,420,2017-03-08 02:52:12,http://tillman.org/chris,0.0748292146,181.156.34.107,"{""location"": ""VA"", ""is_mobile"": false}" 18645,7,420,2017-05-09 05:57:03,http://ratke.net/luna.hoppe,0.8119468535,62.133.168.180,"{""location"": ""TZ"", ""is_mobile"": false}" 18646,7,420,2017-05-13 21:41:53,http://littlecrona.net/magali.flatley,0.2580786977,98.250.244.101,"{""location"": ""PK"", ""is_mobile"": true}" 18647,7,420,2017-04-17 15:48:00,http://goodwin.net/abbie_green,0.9732770747,243.92.171.104,"{""location"": ""KR"", ""is_mobile"": false}" 18648,7,420,2017-01-23 05:12:04,http://osinskihaag.name/arne.reichert,0.0850321967,47.42.172.243,"{""location"": ""MD"", ""is_mobile"": false}" 18649,7,420,2017-01-19 19:42:02,http://dickinson.io/jerel,0.0258945725,19.77.54.156,"{""location"": ""TG"", ""is_mobile"": false}" 18650,7,420,2017-04-27 14:21:01,http://beer.io/arlo,0.6497238361,245.20.76.169,"{""location"": ""HT"", ""is_mobile"": false}" 18651,7,420,2017-01-05 06:33:36,http://zieme.co/cooper_hirthe,0.9145242025,145.106.154.177,"{""location"": ""YT"", ""is_mobile"": true}" 18652,7,420,2017-03-29 02:25:10,http://leffler.name/giovanna.walsh,0.9348739745,75.127.167.100,"{""location"": ""GN"", ""is_mobile"": true}" 18653,7,420,2017-04-29 21:54:18,http://kubhauck.biz/jerrold.konopelski,0.5682223392,21.163.34.61,"{""location"": ""MP"", ""is_mobile"": true}" 18654,7,420,2017-01-14 07:03:25,http://mclaughlin.io/aleen_hammes,0.0955544667,72.76.224.80,"{""location"": ""KR"", ""is_mobile"": true}" 18655,7,420,2017-06-06 07:53:36,http://price.io/gwendolyn,0.5485752464,209.225.228.21,"{""location"": ""CM"", ""is_mobile"": true}" 18656,7,420,2017-01-05 20:11:13,http://swaniawski.biz/abbigail.franecki,0.6502061795,197.5.227.251,"{""location"": ""BQ"", ""is_mobile"": true}" 18657,7,420,2016-12-15 15:31:40,http://beer.io/enrique.kohler,0.6166477783,26.55.148.60,"{""location"": ""LU"", ""is_mobile"": true}" 18658,7,420,2017-02-06 17:00:25,http://schumm.net/karlee.moore,0.0209405660,93.108.155.63,"{""location"": ""AX"", ""is_mobile"": false}" 18659,7,420,2017-01-06 10:23:24,http://oconnell.info/dustin,0.3276830708,202.52.228.105,"{""location"": ""MM"", ""is_mobile"": false}" 18660,7,420,2017-02-04 12:16:23,http://king.com/cade_harris,0.7454817077,110.142.7.85,"{""location"": ""VA"", ""is_mobile"": true}" 18661,7,420,2017-01-22 21:40:23,http://mcclure.net/darron.brekke,0.9804861014,156.213.11.10,"{""location"": ""AF"", ""is_mobile"": true}" 18662,7,420,2017-03-18 18:40:24,http://gutkowski.biz/braulio.gerlach,0.6462051146,15.7.236.55,"{""location"": ""SN"", ""is_mobile"": false}" 18663,7,420,2016-12-22 13:03:28,http://gerhold.co/darlene.davis,0.1018772520,48.86.113.7,"{""location"": ""NU"", ""is_mobile"": true}" 5707,2,125,2017-04-25 06:39:56,http://graham.info/mattie.grady,0.6879303366,198.153.18.58,"{""location"": ""GD"", ""is_mobile"": false}" 5708,2,125,2017-03-03 13:50:12,http://koepp.biz/trenton,0.4858386925,43.185.44.75,"{""location"": ""RE"", ""is_mobile"": true}" 5709,2,125,2017-06-03 19:53:05,http://harveyvonrueden.org/spencer,0.9245473294,204.254.207.2,"{""location"": ""AZ"", ""is_mobile"": true}" 5710,2,125,2016-12-26 09:25:46,http://altenwerth.co/millie,0.9872360031,107.253.227.230,"{""location"": ""NZ"", ""is_mobile"": false}" 5711,2,125,2017-03-15 16:55:19,http://price.info/abbigail_bogan,0.6538980573,66.167.90.50,"{""location"": ""PM"", ""is_mobile"": false}" 5712,2,125,2016-12-21 20:06:34,http://roob.info/arnaldo,0.5865488798,31.14.152.202,"{""location"": ""WS"", ""is_mobile"": true}" 5713,2,125,2017-04-02 00:41:36,http://spinka.com/jacinto,0.8309431320,193.203.249.115,"{""location"": ""BQ"", ""is_mobile"": false}" 5714,2,125,2017-04-24 06:19:57,http://hahn.com/hellen_barrows,0.7483152285,49.95.132.75,"{""location"": ""GL"", ""is_mobile"": false}" 5715,2,125,2017-02-17 22:14:50,http://reichertkulas.co/vincenza,0.5028912570,167.174.23.231,"{""location"": ""UA"", ""is_mobile"": true}" 5716,2,125,2017-04-08 08:28:12,http://abshire.biz/reyes.eichmann,0.5763745445,29.228.229.96,"{""location"": ""MX"", ""is_mobile"": true}" 5717,2,125,2017-04-11 08:45:05,http://ohara.io/annabel.hoeger,0.8079369172,12.52.177.103,"{""location"": ""YT"", ""is_mobile"": true}" 5718,2,125,2017-05-13 02:29:05,http://bashirian.biz/sylvia_carroll,0.0889986954,18.115.126.112,"{""location"": ""MM"", ""is_mobile"": false}" 5719,2,125,2017-04-27 11:39:49,http://stehr.name/lura.bailey,0.2049648940,34.95.173.197,"{""location"": ""AR"", ""is_mobile"": false}" 5720,2,125,2017-03-12 20:06:57,http://toy.io/robert_west,0.1019208932,237.8.83.39,"{""location"": ""UY"", ""is_mobile"": false}" 5721,2,125,2017-05-21 05:50:28,http://pacocha.net/ryann,0.9893897625,176.184.145.142,"{""location"": ""MR"", ""is_mobile"": false}" 5722,2,125,2016-12-15 21:56:42,http://schulistledner.io/adolph,0.6652935204,153.129.63.219,"{""location"": ""AF"", ""is_mobile"": false}" 5723,2,125,2017-05-04 13:00:57,http://davisgutmann.net/bryce,0.0467733859,63.176.55.237,"{""location"": ""SX"", ""is_mobile"": true}" 5724,2,125,2017-01-06 22:22:47,http://emmerich.com/alec.gleichner,0.5097088406,139.20.184.101,"{""location"": ""FJ"", ""is_mobile"": true}" 5725,2,125,2017-03-21 14:49:36,http://skiles.info/kiarra_hyatt,0.2623198137,52.5.7.243,"{""location"": ""EE"", ""is_mobile"": true}" 5726,2,125,2017-01-03 11:07:36,http://yundt.net/yazmin,0.4110089486,95.36.164.28,"{""location"": ""IN"", ""is_mobile"": true}" 5727,2,125,2017-01-20 11:08:33,http://watsica.net/cindy.kuhlman,0.1563485564,160.253.61.116,"{""location"": ""LT"", ""is_mobile"": false}" 5728,2,125,2017-03-08 04:10:59,http://langosh.com/dominique.pfeffer,0.3951057650,78.230.84.198,"{""location"": ""KZ"", ""is_mobile"": true}" 5729,2,125,2017-05-30 18:54:51,http://rodriguez.biz/nasir_rolfson,0.3308764539,167.20.99.55,"{""location"": ""VC"", ""is_mobile"": true}" 5730,2,125,2016-12-25 01:58:47,http://miller.io/jacinthe,0.2993533731,188.90.157.241,"{""location"": ""PA"", ""is_mobile"": false}" 5731,2,125,2017-01-14 16:03:08,http://pagac.org/edyth.block,0.5347196123,9.50.188.35,"{""location"": ""KI"", ""is_mobile"": true}" 5732,2,125,2017-05-04 22:33:14,http://kerluke.info/nelda,0.2076213924,137.89.167.71,"{""location"": ""LC"", ""is_mobile"": true}" 5733,2,125,2017-02-17 13:55:55,http://kuhn.co/kevon_johnston,0.3148997644,20.3.206.234,"{""location"": ""TV"", ""is_mobile"": false}" 5734,2,125,2017-04-27 13:36:33,http://grimes.info/lenna_moore,0.3917430880,151.206.16.57,"{""location"": ""PY"", ""is_mobile"": true}" 5735,2,125,2017-04-06 08:29:58,http://denesik.co/general,0.5451744971,159.67.71.59,"{""location"": ""MC"", ""is_mobile"": true}" 5736,2,125,2016-12-16 08:35:25,http://christiansen.name/aidan.mckenzie,0.0080310611,30.40.182.13,"{""location"": ""PM"", ""is_mobile"": true}" 5737,2,125,2017-06-07 13:47:08,http://nolan.net/laney,0.3537142880,64.187.190.68,"{""location"": ""GF"", ""is_mobile"": false}" 5738,2,125,2017-03-01 02:31:29,http://ankunding.co/noemi,0.3315876561,58.162.43.14,"{""location"": ""TR"", ""is_mobile"": false}" 5739,2,125,2016-12-31 21:43:00,http://wolfgislason.net/alvis.hickle,0.4075203074,195.57.25.65,"{""location"": ""KP"", ""is_mobile"": false}" 5740,2,125,2017-05-15 06:25:15,http://rohan.com/leonard_dickens,0.8373116051,202.6.145.102,"{""location"": ""IT"", ""is_mobile"": false}" 5741,2,125,2017-01-28 11:11:20,http://mckenzie.org/marcella.sporer,0.9079151795,156.83.53.242,"{""location"": ""NP"", ""is_mobile"": false}" 5742,2,125,2017-05-26 13:28:54,http://jacobi.net/haylie,0.7748926883,52.24.70.42,"{""location"": ""DK"", ""is_mobile"": true}" 5743,2,125,2017-04-06 02:34:54,http://cain.com/vicente_bosco,0.1402625990,15.33.113.49,"{""location"": ""LV"", ""is_mobile"": false}" 5744,2,125,2017-04-23 14:07:34,http://hackett.co/zachery,0.6858901046,212.9.109.59,"{""location"": ""MA"", ""is_mobile"": false}" 5745,2,125,2017-03-29 10:00:10,http://frami.biz/luther_feeney,0.7946323803,189.197.201.142,"{""location"": ""RO"", ""is_mobile"": false}" 5746,2,125,2016-12-26 04:27:12,http://whiterau.net/willie,0.9639645596,205.204.128.22,"{""location"": ""AL"", ""is_mobile"": false}" 5747,2,125,2017-03-22 06:23:45,http://pollich.io/gielle.larkin,0.8030997661,15.134.19.59,"{""location"": ""ZW"", ""is_mobile"": true}" 5748,2,125,2017-03-04 18:02:30,http://cormier.co/daphnee,0.0128740019,223.202.10.75,"{""location"": ""BD"", ""is_mobile"": true}" 5749,2,125,2017-05-10 08:02:46,http://spencergrady.co/adelle,0.4460829373,75.33.41.253,"{""location"": ""US"", ""is_mobile"": false}" 5750,2,125,2017-04-18 17:48:20,http://hettingerprice.com/desiree_kilback,0.1001380591,243.36.191.72,"{""location"": ""AO"", ""is_mobile"": false}" 5751,2,125,2017-03-03 17:19:07,http://breitenberg.net/neil.ko,0.0408304577,48.28.69.142,"{""location"": ""AF"", ""is_mobile"": false}" 5752,2,125,2017-05-22 10:03:27,http://brekke.info/edwardo,0.1523261269,251.31.125.33,"{""location"": ""LC"", ""is_mobile"": true}" 5753,2,125,2017-02-11 03:39:27,http://jerde.info/fabiola.olson,0.3222941682,13.204.94.211,"{""location"": ""HN"", ""is_mobile"": false}" 5754,2,125,2017-02-23 10:08:38,http://bruen.biz/stone.breitenberg,0.8404515410,213.128.94.28,"{""location"": ""US"", ""is_mobile"": false}" 5755,2,125,2016-12-15 16:14:34,http://borerrogahn.name/titus,0.7284782756,102.9.170.186,"{""location"": ""AF"", ""is_mobile"": true}" 5756,2,125,2017-01-11 20:46:08,http://schuster.info/jasmin.heel,0.3039356598,119.183.116.228,"{""location"": ""MT"", ""is_mobile"": false}" 5757,2,125,2017-01-19 02:51:07,http://ward.biz/alana,0.1391968998,141.162.10.126,"{""location"": ""KZ"", ""is_mobile"": true}" 5758,2,125,2017-05-01 04:57:41,http://blocknolan.net/sincere_grant,0.2472235401,243.67.151.103,"{""location"": ""IQ"", ""is_mobile"": true}" 18664,7,420,2016-12-26 09:29:17,http://johnsondickinson.name/alexander,0.8331171230,11.180.7.52,"{""location"": ""PF"", ""is_mobile"": false}" 18665,7,420,2017-05-15 11:59:53,http://okeefe.biz/laverne.schumm,0.9207832080,2.197.146.168,"{""location"": ""RE"", ""is_mobile"": false}" 18666,7,420,2017-04-02 23:29:25,http://brekke.com/hal_schroeder,0.4419138618,193.42.193.71,"{""location"": ""AT"", ""is_mobile"": false}" 18667,7,420,2017-02-04 05:54:43,http://leuschke.name/hollie,0.5264180643,151.56.100.231,"{""location"": ""SV"", ""is_mobile"": false}" 18668,7,420,2017-01-22 00:52:38,http://barton.name/brook.anderson,0.5024227634,140.100.151.25,"{""location"": ""CK"", ""is_mobile"": false}" 18669,7,420,2017-05-02 09:11:52,http://lebsack.biz/zoila,0.0760032177,195.214.142.35,"{""location"": ""AX"", ""is_mobile"": true}" 18670,7,420,2017-03-09 04:55:02,http://gusikowski.co/mikayla.hoeger,0.3965342115,45.136.38.157,"{""location"": ""LS"", ""is_mobile"": false}" 18671,7,420,2016-12-14 09:25:59,http://dubuquezulauf.name/candida_kuhlman,0.2377473345,56.238.192.70,"{""location"": ""GP"", ""is_mobile"": true}" 18672,7,420,2017-01-22 08:46:42,http://kihn.biz/bethel_batz,0.2914282227,9.87.12.137,"{""location"": ""BY"", ""is_mobile"": false}" 18673,7,421,2017-01-29 09:24:27,http://champlin.co/brandy_lowe,0.6408389231,68.103.244.231,"{""location"": ""CF"", ""is_mobile"": true}" 18674,7,421,2017-04-07 16:22:52,http://berge.biz/angie,0.2550252876,88.178.227.128,"{""location"": ""DE"", ""is_mobile"": false}" 18675,7,421,2017-01-13 07:22:20,http://oberbrunner.info/johnpaul,0.3654884191,236.6.102.62,"{""location"": ""SK"", ""is_mobile"": true}" 18676,7,421,2017-02-10 06:22:51,http://okeefe.com/noemie_ritchie,0.9894539430,232.2.228.102,"{""location"": ""CG"", ""is_mobile"": false}" 18677,7,421,2017-05-25 20:15:56,http://greenholtwest.io/daryl,0.7433163908,144.51.135.213,"{""location"": ""TH"", ""is_mobile"": true}" 18678,7,421,2017-01-11 06:47:49,http://jacobi.name/gordon,0.2747754596,228.122.111.103,"{""location"": ""LI"", ""is_mobile"": false}" 18679,7,421,2017-03-07 20:43:49,http://gutmann.name/jakob.shanahan,0.1250963092,35.170.132.203,"{""location"": ""ES"", ""is_mobile"": true}" 18680,7,421,2017-04-07 12:29:54,http://schmidtmarquardt.info/gerson_cronin,0.2110727370,172.96.140.110,"{""location"": ""CO"", ""is_mobile"": true}" 18681,7,421,2017-04-29 05:22:50,http://kovacek.io/janea_gorczany,0.7240982690,215.168.43.156,"{""location"": ""AL"", ""is_mobile"": false}" 18682,7,421,2016-12-21 10:48:30,http://schiller.io/kayley,0.9419713241,21.229.136.219,"{""location"": ""TL"", ""is_mobile"": true}" 18683,7,421,2017-03-21 17:29:29,http://powlowski.com/teagan_dicki,0.3827397944,174.34.77.46,"{""location"": ""TN"", ""is_mobile"": false}" 18684,7,421,2017-06-09 14:18:21,http://beahangrant.info/carlotta,0.9531253724,227.47.61.232,"{""location"": ""CL"", ""is_mobile"": false}" 18685,7,421,2017-06-02 17:39:14,http://sauerhoeger.com/patsy_marvin,0.3277269441,58.81.168.216,"{""location"": ""PW"", ""is_mobile"": false}" 18686,7,421,2017-01-16 15:39:32,http://green.net/catharine,0.3152517760,47.59.253.22,"{""location"": ""VU"", ""is_mobile"": false}" 18687,7,421,2017-04-07 06:51:30,http://mclaughlinkuhlman.com/milford_smith,0.9359290859,195.170.230.174,"{""location"": ""KM"", ""is_mobile"": false}" 18688,7,421,2017-05-13 06:45:44,http://okon.biz/christelle,0.2306708552,23.251.145.60,"{""location"": ""VE"", ""is_mobile"": true}" 18689,7,421,2017-01-26 17:25:42,http://schimmelmckenzie.org/noemie.grimes,0.7676126698,164.40.106.98,"{""location"": ""LB"", ""is_mobile"": true}" 18690,7,421,2016-12-15 01:43:50,http://osinski.biz/berry,0.4528190301,176.79.172.169,"{""location"": ""PY"", ""is_mobile"": true}" 18691,7,421,2017-04-25 19:43:56,http://pfannerstill.name/newton.jast,0.3896514015,55.219.224.232,"{""location"": ""BO"", ""is_mobile"": true}" 18692,7,421,2017-01-28 09:08:31,http://balistreri.co/tierra.kris,0.9875079054,6.156.212.210,"{""location"": ""PS"", ""is_mobile"": false}" 18693,7,421,2017-03-26 18:17:24,http://croninbahringer.com/ladarius,0.2036684588,65.29.151.154,"{""location"": ""ET"", ""is_mobile"": false}" 18694,7,421,2017-04-07 09:23:28,http://dach.net/oral.nikolaus,0.8034129725,240.71.193.226,"{""location"": ""SI"", ""is_mobile"": true}" 18695,7,421,2017-02-09 18:37:11,http://reichert.co/carroll.ferry,0.9772909142,233.113.146.61,"{""location"": ""LK"", ""is_mobile"": false}" 18696,7,421,2017-04-07 09:11:52,http://nicolas.name/charity,0.3280880635,162.144.180.36,"{""location"": ""MV"", ""is_mobile"": true}" 18697,7,421,2017-01-23 17:42:51,http://ernser.org/efren.cain,0.5422840943,11.4.5.142,"{""location"": ""HR"", ""is_mobile"": true}" 18698,7,421,2017-05-07 00:53:25,http://runolfon.name/vito,0.6668328485,53.71.139.212,"{""location"": ""BH"", ""is_mobile"": false}" 18699,7,421,2017-05-25 11:20:01,http://brekke.biz/kim.bartoletti,0.9669548086,80.35.129.84,"{""location"": ""AO"", ""is_mobile"": true}" 18700,7,421,2017-04-16 06:49:48,http://emmerich.biz/greyson,0.9390479754,123.125.218.229,"{""location"": ""MU"", ""is_mobile"": false}" 18701,7,421,2017-02-12 15:10:36,http://friesenprohaska.com/sydnee_gerhold,0.8091859141,190.66.209.16,"{""location"": ""MX"", ""is_mobile"": true}" 18702,7,421,2017-01-03 15:15:16,http://mayer.biz/mariano.boyer,0.0313623118,183.153.195.153,"{""location"": ""DJ"", ""is_mobile"": true}" 18703,7,421,2017-03-02 23:18:17,http://bode.com/armando,0.7107091005,39.83.151.245,"{""location"": ""KZ"", ""is_mobile"": true}" 18704,7,421,2017-03-12 08:27:35,http://pagaclindgren.name/rylan,0.2219702868,151.58.15.177,"{""location"": ""HM"", ""is_mobile"": false}" 18705,7,421,2017-02-20 18:33:09,http://larkinklein.org/simone,0.9827321966,117.170.165.193,"{""location"": ""PW"", ""is_mobile"": false}" 18706,7,421,2017-03-07 12:34:51,http://kuhlman.io/foster_lebsack,0.4975826006,50.138.15.215,"{""location"": ""AQ"", ""is_mobile"": false}" 18707,7,421,2017-02-09 19:48:18,http://fritschhomenick.co/jackson,0.5224766200,90.164.210.112,"{""location"": ""GH"", ""is_mobile"": false}" 18708,7,421,2016-12-31 14:43:49,http://roobcruickshank.co/willis,0.0144242891,111.215.62.248,"{""location"": ""RW"", ""is_mobile"": false}" 18709,7,421,2017-05-13 13:00:42,http://crist.io/emerson,0.7403719257,222.219.217.86,"{""location"": ""MM"", ""is_mobile"": true}" 18710,7,421,2017-01-01 04:07:16,http://hammes.org/raven,0.3636948243,42.47.195.124,"{""location"": ""KR"", ""is_mobile"": true}" 18711,7,421,2017-01-20 20:07:31,http://zemlakkreiger.co/rodrigo_kirlin,0.5886571700,129.100.206.72,"{""location"": ""SI"", ""is_mobile"": true}" 18712,7,421,2017-01-31 22:54:17,http://greenfelderskiles.co/dannie,0.5840463277,248.158.85.77,"{""location"": ""PN"", ""is_mobile"": true}" 18713,7,421,2017-03-24 07:35:32,http://casper.org/marquis_pfeffer,0.2572122852,47.127.59.219,"{""location"": ""GF"", ""is_mobile"": false}" 18714,7,421,2016-12-30 10:31:37,http://dickens.co/veronica,0.4382266900,18.128.148.68,"{""location"": ""CK"", ""is_mobile"": true}" 5759,2,125,2017-06-07 16:14:45,http://osinskirutherford.org/mertie,0.7268262465,114.171.122.212,"{""location"": ""GA"", ""is_mobile"": true}" 5760,2,125,2017-04-06 04:24:46,http://funkkeeling.biz/travis.lockman,0.5203302906,34.104.222.240,"{""location"": ""BH"", ""is_mobile"": true}" 18715,7,421,2017-04-03 17:22:05,http://erdman.net/eileen,0.6104548598,88.74.171.226,"{""location"": ""CF"", ""is_mobile"": true}" 18716,7,421,2017-02-23 10:30:42,http://johnson.co/austyn.hartmann,0.1691366230,176.74.8.235,"{""location"": ""BM"", ""is_mobile"": true}" 18717,7,421,2016-12-17 01:46:57,http://mcglynn.co/santina_gutmann,0.2092302928,92.224.136.125,"{""location"": ""YE"", ""is_mobile"": false}" 18718,7,421,2017-01-14 12:38:44,http://nikolaus.io/ewell_ruecker,0.5333841450,177.144.39.193,"{""location"": ""LS"", ""is_mobile"": false}" 18719,7,421,2017-03-02 04:48:27,http://danielbeier.net/kellie,0.1207128197,80.115.161.184,"{""location"": ""VG"", ""is_mobile"": false}" 18720,7,421,2017-02-04 20:53:31,http://gaylord.name/santos_altenwerth,0.4112577677,93.210.121.30,"{""location"": ""LB"", ""is_mobile"": false}" 18721,7,421,2016-12-31 15:14:05,http://gorczany.net/esmeralda,0.4427082397,230.56.197.43,"{""location"": ""HT"", ""is_mobile"": true}" 18722,7,421,2017-03-19 16:29:14,http://koepphyatt.info/torrance,0.1707628121,77.81.163.17,"{""location"": ""PK"", ""is_mobile"": true}" 18723,7,421,2016-12-26 02:59:35,http://mccullough.name/francesco,0.2646414391,233.23.234.17,"{""location"": ""CF"", ""is_mobile"": true}" 18724,7,421,2016-12-16 19:33:15,http://hauck.name/justice_connelly,0.9098955065,196.197.93.167,"{""location"": ""AZ"", ""is_mobile"": true}" 18725,7,421,2017-01-31 17:54:32,http://abbottoconner.io/frankie_schiller,0.7195487788,224.230.49.206,"{""location"": ""JP"", ""is_mobile"": false}" 18726,7,421,2016-12-24 07:07:20,http://westschimmel.co/quinton,0.0428506336,210.132.72.4,"{""location"": ""GH"", ""is_mobile"": false}" 18727,7,421,2017-05-02 01:50:16,http://toyyundt.org/treie.herman,0.3942108595,45.210.173.229,"{""location"": ""DZ"", ""is_mobile"": true}" 18728,7,421,2017-06-12 10:42:47,http://brown.io/ru.heidenreich,0.2424074613,253.214.66.135,"{""location"": ""AT"", ""is_mobile"": true}" 18729,7,421,2017-02-19 19:37:40,http://fay.info/sebastian_hickle,0.1373344996,175.39.163.127,"{""location"": ""SR"", ""is_mobile"": true}" 18730,7,421,2017-05-12 08:51:32,http://kohler.name/liam,0.4251598376,214.7.68.7,"{""location"": ""BN"", ""is_mobile"": true}" 18731,7,422,2017-01-12 14:49:12,http://jacobson.biz/torrance,0.1397712337,89.157.29.62,"{""location"": ""CW"", ""is_mobile"": true}" 18732,7,422,2017-03-18 17:52:54,http://kris.com/king,0.8178504621,41.108.4.174,"{""location"": ""CR"", ""is_mobile"": true}" 18733,7,422,2017-03-29 04:07:54,http://borer.name/kianna_hegmann,0.6865084450,231.145.230.19,"{""location"": ""BY"", ""is_mobile"": false}" 18734,7,422,2017-02-14 18:02:16,http://stanton.net/filomena.gislason,0.0349397868,167.217.161.41,"{""location"": ""NP"", ""is_mobile"": true}" 18735,7,422,2017-03-15 08:01:17,http://mcculloughhamill.co/janae.hodkiewicz,0.9646312950,138.72.112.33,"{""location"": ""YT"", ""is_mobile"": false}" 18736,7,422,2017-05-21 17:20:29,http://kiehn.org/krystina,0.1685891472,124.228.244.139,"{""location"": ""CR"", ""is_mobile"": false}" 18737,7,422,2017-02-16 21:12:44,http://bauchkaulke.org/bartholome,0.5844397715,135.230.83.156,"{""location"": ""RE"", ""is_mobile"": false}" 18738,7,422,2017-06-08 08:19:48,http://homenick.co/alexandro,0.6654047646,243.192.24.232,"{""location"": ""MP"", ""is_mobile"": false}" 18739,7,422,2017-04-20 09:47:20,http://mckenzie.net/ebony,0.2690423219,22.13.29.149,"{""location"": ""PT"", ""is_mobile"": true}" 18740,7,422,2017-02-22 05:45:11,http://towne.info/kristian_nolan,0.9261665420,48.12.25.71,"{""location"": ""CY"", ""is_mobile"": true}" 18741,7,422,2016-12-21 12:09:47,http://kuvalis.org/jimmy.braun,0.8604778607,96.63.222.155,"{""location"": ""MG"", ""is_mobile"": true}" 18742,7,422,2017-01-16 17:17:44,http://runolfonbeier.org/charles.crist,0.7634561829,224.220.122.194,"{""location"": ""SM"", ""is_mobile"": false}" 18743,7,422,2017-03-16 07:24:56,http://cormier.net/jey.west,0.7159501845,205.240.167.158,"{""location"": ""UY"", ""is_mobile"": false}" 18744,7,422,2017-04-12 09:32:58,http://deckowcartwright.info/lea.murphy,0.6872713548,245.189.137.6,"{""location"": ""QA"", ""is_mobile"": true}" 18745,7,422,2016-12-26 21:01:48,http://jast.biz/carolyn.heller,0.5557902267,107.227.225.196,"{""location"": ""AR"", ""is_mobile"": false}" 18746,7,422,2016-12-21 13:01:39,http://halvorsonbrakus.com/loraine_klocko,0.0850726953,61.110.200.130,"{""location"": ""LR"", ""is_mobile"": true}" 18747,7,422,2016-12-31 00:23:16,http://pourochulist.org/chadd_hodkiewicz,0.9410907621,156.31.154.227,"{""location"": ""PY"", ""is_mobile"": true}" 18748,7,422,2017-02-19 02:25:41,http://ohara.co/sid_ledner,0.6151563942,249.92.31.210,"{""location"": ""KH"", ""is_mobile"": true}" 18749,7,422,2017-04-14 05:52:00,http://hellerlubowitz.io/elliot,0.6807388582,111.241.193.238,"{""location"": ""ME"", ""is_mobile"": false}" 18750,7,422,2017-05-13 01:47:05,http://mcglynn.net/esmeralda_dubuque,0.1672716102,175.131.173.136,"{""location"": ""IE"", ""is_mobile"": true}" 18751,7,422,2017-05-06 03:46:15,http://wilkinson.io/keshawn,0.1396268983,168.85.166.204,"{""location"": ""IS"", ""is_mobile"": false}" 18752,7,422,2017-06-11 23:35:41,http://kutchhickle.io/brady_bergnaum,0.6924088078,29.108.163.134,"{""location"": ""CZ"", ""is_mobile"": false}" 18753,7,422,2017-01-11 14:36:50,http://bayerlueilwitz.info/mikel.mayert,0.5613044650,229.232.249.149,"{""location"": ""JP"", ""is_mobile"": false}" 18754,7,422,2017-02-12 04:45:14,http://douglas.org/barney,0.9599832764,184.55.65.73,"{""location"": ""DE"", ""is_mobile"": false}" 18755,7,422,2017-05-10 14:34:45,http://rempelwunsch.info/etha_witting,0.4466870980,138.213.101.31,"{""location"": ""BL"", ""is_mobile"": true}" 18756,7,422,2017-01-27 16:24:20,http://lednerherman.com/haven,0.3723039754,81.8.144.54,"{""location"": ""CL"", ""is_mobile"": true}" 18757,7,423,2016-12-20 06:57:33,http://hyattbuckridge.com/dwight_oberbrunner,0.1084255265,214.119.36.130,"{""location"": ""IT"", ""is_mobile"": false}" 18758,7,423,2017-02-20 23:58:49,http://doyledenesik.io/alfredo,0.4880590557,23.65.181.76,"{""location"": ""NL"", ""is_mobile"": true}" 18759,7,423,2017-06-01 20:20:23,http://haag.net/zakary,0.6442926918,27.40.120.159,"{""location"": ""PS"", ""is_mobile"": true}" 18760,7,423,2017-04-18 03:50:03,http://littel.biz/isabelle,0.6041362333,29.99.49.79,"{""location"": ""TL"", ""is_mobile"": true}" 18761,7,423,2017-01-20 01:23:37,http://franecki.org/monte_towne,0.2228415107,127.158.145.226,"{""location"": ""MY"", ""is_mobile"": true}" 18762,7,423,2017-03-12 14:51:55,http://tillman.net/delphia,0.6243527426,185.249.42.155,"{""location"": ""SC"", ""is_mobile"": true}" 18763,7,423,2017-02-28 16:50:22,http://lindhackett.org/leonardo,0.9064792447,128.153.4.67,"{""location"": ""MG"", ""is_mobile"": true}" 18764,7,423,2017-05-03 22:11:00,http://okeefekaulke.org/haie_rodriguez,0.3444117020,238.94.20.200,"{""location"": ""RE"", ""is_mobile"": false}" 18765,7,423,2017-03-26 09:25:40,http://waters.com/amya_runte,0.6959851710,118.152.249.73,"{""location"": ""TF"", ""is_mobile"": true}" 18766,7,423,2017-01-09 19:57:53,http://millerkihn.name/casimir,0.4847022732,245.34.184.104,"{""location"": ""LY"", ""is_mobile"": true}" 18767,7,423,2017-03-21 11:45:32,http://luettgenpfannerstill.name/johnny_hansen,0.7055212822,24.244.220.104,"{""location"": ""DO"", ""is_mobile"": false}" 18768,7,423,2017-04-21 04:31:08,http://marksrempel.info/kaci.abbott,0.7746874300,203.60.39.251,"{""location"": ""AI"", ""is_mobile"": false}" 18769,7,423,2017-04-22 14:45:18,http://ernser.net/austen_larkin,0.6089508439,26.155.133.236,"{""location"": ""CU"", ""is_mobile"": false}" 18770,7,423,2017-01-21 03:23:58,http://pacochagutkowski.net/cathrine,0.9071000577,34.229.144.194,"{""location"": ""HT"", ""is_mobile"": true}" 18771,7,423,2017-04-19 08:28:57,http://lebsackcole.info/kari_beier,0.6850290182,123.71.7.22,"{""location"": ""GH"", ""is_mobile"": false}" 18772,7,423,2017-01-13 17:42:03,http://koepp.io/elta_bode,0.4023853512,111.41.205.81,"{""location"": ""DJ"", ""is_mobile"": true}" 18773,7,423,2017-05-13 05:22:39,http://beatty.io/claire_gleason,0.4778280485,253.18.184.100,"{""location"": ""TN"", ""is_mobile"": true}" 18774,7,423,2017-06-02 23:53:31,http://framigrimes.name/jimmy.romaguera,0.5941875477,69.70.244.149,"{""location"": ""KP"", ""is_mobile"": true}" 18775,7,423,2017-03-10 21:06:47,http://millsernser.info/adolphus.crooks,0.1528826185,221.5.14.198,"{""location"": ""SK"", ""is_mobile"": true}" 18776,7,423,2017-01-25 07:27:22,http://jacobioconnell.info/cale,0.0557757833,122.130.112.180,"{""location"": ""LT"", ""is_mobile"": true}" 18777,7,423,2017-02-06 03:17:55,http://mills.org/ebony,0.2657109497,162.253.185.188,"{""location"": ""PN"", ""is_mobile"": true}" 18778,7,423,2017-01-08 17:33:07,http://wolffmaggio.co/gretchen,0.7480388670,13.249.179.5,"{""location"": ""JP"", ""is_mobile"": false}" 18779,7,423,2017-03-09 07:25:19,http://funkboyle.net/rita.abernathy,0.6246213783,208.243.24.122,"{""location"": ""ZW"", ""is_mobile"": true}" 18780,7,423,2017-04-21 04:57:39,http://mullerdietrich.name/pauline,0.1801731008,94.66.125.48,"{""location"": ""ML"", ""is_mobile"": false}" 18781,7,423,2017-03-31 20:08:06,http://tremblay.biz/carlos_miller,0.9758454383,152.108.147.111,"{""location"": ""HM"", ""is_mobile"": true}" 18782,7,423,2017-05-11 10:19:02,http://wisozk.biz/preston_dare,0.8654015927,149.89.214.146,"{""location"": ""GW"", ""is_mobile"": false}" 18783,7,423,2017-01-10 16:16:49,http://nikolaus.co/ines_fay,0.5689677720,254.248.91.13,"{""location"": ""MK"", ""is_mobile"": true}" 18784,7,423,2017-03-24 17:33:28,http://murazik.name/alayna,0.5775581111,159.157.15.182,"{""location"": ""TD"", ""is_mobile"": true}" 18785,7,423,2017-02-11 20:46:01,http://bartell.name/minerva_balistreri,0.8486969180,150.147.94.131,"{""location"": ""FK"", ""is_mobile"": false}" 18786,7,423,2017-06-05 15:22:23,http://brekkeherzog.name/laria,0.1171609518,38.185.154.103,"{""location"": ""GW"", ""is_mobile"": true}" 18787,7,423,2017-03-28 22:55:04,http://torpkoepp.net/terence_reilly,0.5542935401,103.237.240.226,"{""location"": ""DM"", ""is_mobile"": false}" 18788,7,423,2017-05-06 08:56:53,http://mclaughlin.name/camryn,0.2621027162,111.196.228.103,"{""location"": ""LK"", ""is_mobile"": false}" 18789,7,423,2017-06-11 03:29:13,http://kuvalis.co/ambrose.reinger,0.7118096172,104.250.74.84,"{""location"": ""GR"", ""is_mobile"": true}" 18790,7,423,2017-02-20 19:38:59,http://blanda.name/anastasia.littel,0.9271415326,149.208.185.31,"{""location"": ""GT"", ""is_mobile"": false}" 18791,7,423,2017-04-26 21:42:49,http://weinathintz.org/jared,0.1654651059,167.143.207.175,"{""location"": ""PL"", ""is_mobile"": true}" 18792,7,423,2017-06-05 05:15:51,http://millswuckert.biz/carmelo,0.7048228243,210.16.184.201,"{""location"": ""MW"", ""is_mobile"": false}" 18793,7,423,2017-04-22 20:54:30,http://hahn.com/amanda.ward,0.2379880511,11.3.78.81,"{""location"": ""PS"", ""is_mobile"": true}" 18794,7,423,2017-02-28 03:26:00,http://wisoky.io/cameron,0.6798202822,69.202.165.27,"{""location"": ""SV"", ""is_mobile"": false}" 18795,7,423,2017-05-30 18:09:24,http://wuckert.co/name.dickens,0.9509489666,164.201.58.23,"{""location"": ""UY"", ""is_mobile"": false}" 18796,7,423,2017-04-23 22:53:48,http://powlowski.co/susana_mckenzie,0.1540995475,94.164.60.44,"{""location"": ""DM"", ""is_mobile"": true}" 18797,7,423,2017-03-07 01:27:31,http://cristpowlowski.org/bernadine,0.8407123444,120.26.31.59,"{""location"": ""LU"", ""is_mobile"": true}" 18798,7,423,2017-04-23 23:05:57,http://runte.info/amy,0.6710479579,32.235.181.43,"{""location"": ""MP"", ""is_mobile"": true}" 18799,7,424,2017-02-15 18:58:09,http://legrosblick.co/giovani,0.7057047089,18.240.8.247,"{""location"": ""UY"", ""is_mobile"": false}" 18800,7,424,2017-02-28 17:23:10,http://hermiston.biz/aurelia,0.2752460620,197.25.138.98,"{""location"": ""GD"", ""is_mobile"": false}" 18801,7,424,2017-03-27 02:26:21,http://haag.org/sarah_turner,0.3039557715,70.223.142.149,"{""location"": ""SV"", ""is_mobile"": true}" 18802,7,424,2016-12-13 18:23:41,http://hoppe.org/josephine,0.2840345531,72.155.217.25,"{""location"": ""CC"", ""is_mobile"": true}" 18803,7,424,2017-05-26 06:40:51,http://kerluke.info/jerome,0.5459567946,34.204.183.131,"{""location"": ""TF"", ""is_mobile"": false}" 18804,7,424,2017-01-11 20:38:59,http://thiel.co/kaylee,0.4396290312,90.103.248.120,"{""location"": ""CK"", ""is_mobile"": true}" 18805,7,424,2017-04-26 11:01:04,http://collins.net/claudia,0.6827068769,101.103.166.9,"{""location"": ""AZ"", ""is_mobile"": false}" 18806,7,424,2017-06-04 10:29:28,http://stoltenbergfarrell.io/ryann.carter,0.6345564765,131.234.29.227,"{""location"": ""SE"", ""is_mobile"": true}" 18807,7,424,2017-05-27 12:25:30,http://bartonvolkman.com/loren,0.9603213807,228.146.127.25,"{""location"": ""NF"", ""is_mobile"": true}" 18808,7,424,2017-03-26 11:39:09,http://terry.org/alfredo,0.5841196994,151.211.97.149,"{""location"": ""PF"", ""is_mobile"": false}" 18809,7,424,2016-12-28 14:10:58,http://hayeshahn.co/clare,0.0020311440,195.189.187.86,"{""location"": ""MQ"", ""is_mobile"": true}" 18810,7,424,2017-03-13 08:38:16,http://wiegand.co/nikko,0.8371923710,234.51.108.185,"{""location"": ""SI"", ""is_mobile"": false}" 18811,7,424,2017-03-22 08:33:09,http://lang.biz/bridgette_oconnell,0.9564314602,228.55.148.31,"{""location"": ""NU"", ""is_mobile"": true}" 18812,7,424,2017-02-23 20:39:03,http://mclaughlindickens.biz/kade.kirlin,0.0760666776,160.137.192.108,"{""location"": ""ZW"", ""is_mobile"": true}" 18813,7,424,2017-05-19 15:04:46,http://heel.net/alison,0.4863040777,128.44.220.204,"{""location"": ""BA"", ""is_mobile"": false}" 18814,7,424,2017-03-01 08:32:18,http://erdmansipes.info/eloisa.rohan,0.1313289922,121.94.163.214,"{""location"": ""YT"", ""is_mobile"": false}" 18815,7,424,2017-03-12 03:51:22,http://koepp.io/emily.conroy,0.7710888795,39.179.137.11,"{""location"": ""JP"", ""is_mobile"": true}" 18816,7,424,2017-03-15 09:42:20,http://rodriguez.com/dale,0.3152990493,25.16.182.126,"{""location"": ""VC"", ""is_mobile"": false}" 18817,7,424,2017-02-22 09:13:35,http://konopelskizboncak.com/gage.borer,0.7386919344,95.28.34.188,"{""location"": ""NF"", ""is_mobile"": false}" 18818,7,424,2017-05-11 22:45:52,http://hagenes.biz/kirk,0.9451548207,161.76.198.176,"{""location"": ""GU"", ""is_mobile"": true}" 18819,7,424,2017-06-13 23:08:09,http://bednar.co/kacey.yost,0.7239267627,187.205.8.90,"{""location"": ""ID"", ""is_mobile"": true}" 18820,7,424,2017-02-12 05:03:55,http://mcglynn.name/carmelo,0.4430942827,126.26.89.109,"{""location"": ""IE"", ""is_mobile"": false}" 18821,7,424,2017-02-23 09:43:59,http://robelpurdy.io/zoe_volkman,0.9543472506,124.60.32.235,"{""location"": ""AQ"", ""is_mobile"": false}" 18822,7,424,2017-01-27 15:55:23,http://kuhnmcclure.info/amiya_hand,0.5957609892,53.149.6.149,"{""location"": ""LY"", ""is_mobile"": false}" 18823,7,424,2017-05-21 07:16:26,http://okeefebuckridge.com/annabelle.marks,0.3898827988,37.143.16.192,"{""location"": ""SV"", ""is_mobile"": true}" 18824,7,424,2017-01-23 17:29:36,http://joneshudson.info/steve_kirlin,0.9967681595,201.86.234.202,"{""location"": ""NP"", ""is_mobile"": false}" 18825,7,424,2017-03-07 08:39:37,http://thielmaggio.com/wendell_hilll,0.0898476878,109.110.206.104,"{""location"": ""BH"", ""is_mobile"": false}" 18826,7,424,2017-03-18 02:56:31,http://kubglover.info/juwan,0.7279446237,64.104.70.131,"{""location"": ""BW"", ""is_mobile"": false}" 18827,7,424,2016-12-24 11:27:02,http://pfannerstill.io/german,0.6421894274,75.104.100.170,"{""location"": ""CM"", ""is_mobile"": true}" 18828,7,424,2017-01-25 05:47:05,http://schmitt.co/savanah_ohara,0.2668193754,170.62.13.118,"{""location"": ""MS"", ""is_mobile"": false}" 18829,7,424,2017-03-05 03:31:38,http://mosciskinolan.com/libbie.bechtelar,0.6902012320,19.19.215.235,"{""location"": ""PT"", ""is_mobile"": false}" 18830,7,424,2017-06-09 06:02:34,http://altenwerthgoldner.biz/kameron_hammes,0.3188945287,231.184.210.226,"{""location"": ""TV"", ""is_mobile"": true}" 18831,7,424,2017-01-06 00:58:32,http://huel.net/forest_johns,0.9979699951,121.217.38.246,"{""location"": ""PG"", ""is_mobile"": true}" 18832,7,424,2017-02-18 15:11:44,http://beierschaden.co/vladimir_blick,0.1736462305,130.127.38.18,"{""location"": ""PT"", ""is_mobile"": false}" 18833,7,424,2017-05-20 15:49:20,http://west.org/sanford,0.2980051739,38.212.163.10,"{""location"": ""SJ"", ""is_mobile"": false}" 18834,7,424,2017-04-04 12:11:15,http://kuvalisconsidine.org/chad_rolfson,0.9250327917,13.88.171.200,"{""location"": ""IR"", ""is_mobile"": false}" 18835,7,424,2017-05-20 03:26:17,http://reichelwelch.io/ivory.sawayn,0.6415398637,51.93.47.247,"{""location"": ""BO"", ""is_mobile"": false}" 18836,7,424,2016-12-25 01:09:36,http://huel.name/kraig_vonrueden,0.1466624533,11.36.205.6,"{""location"": ""AO"", ""is_mobile"": true}" 18837,7,424,2017-03-19 11:59:55,http://keeblergleason.com/zelda,0.4967049605,172.206.208.81,"{""location"": ""IO"", ""is_mobile"": false}" 18838,7,424,2017-02-14 16:17:25,http://kihnrowe.biz/caroline.schoen,0.3520093230,43.191.197.80,"{""location"": ""ET"", ""is_mobile"": true}" 18839,7,424,2017-03-01 08:35:51,http://uptonhermann.com/buster_leuschke,0.8355263812,210.37.191.178,"{""location"": ""VI"", ""is_mobile"": true}" 18840,7,424,2017-01-05 09:33:19,http://kuhicdeckow.io/aditya_hills,0.6436877200,84.47.234.201,"{""location"": ""LS"", ""is_mobile"": false}" 18841,7,424,2017-05-16 05:19:19,http://ritchie.name/margarett,0.9279654882,108.219.19.183,"{""location"": ""TM"", ""is_mobile"": false}" 18842,7,424,2017-05-04 00:44:37,http://hansenokuneva.net/retha_morar,0.8517722666,81.204.101.46,"{""location"": ""GN"", ""is_mobile"": true}" 18843,7,424,2017-06-08 04:36:32,http://kelerkrajcik.net/jo_runolfon,0.8950329092,85.5.69.129,"{""location"": ""PH"", ""is_mobile"": false}" 18844,7,424,2017-04-20 07:04:51,http://hackettschinner.co/caleigh.mueller,0.0734160858,113.74.115.98,"{""location"": ""BZ"", ""is_mobile"": false}" 18845,7,424,2017-04-09 09:06:41,http://walkerwill.info/ryleigh.senger,0.0762922065,65.229.176.18,"{""location"": ""UA"", ""is_mobile"": true}" 18846,7,424,2017-03-26 12:40:13,http://bins.io/izabella,0.7279309891,223.216.227.5,"{""location"": ""YE"", ""is_mobile"": false}" 18847,7,424,2017-06-10 18:54:05,http://yost.info/jayce_harvey,0.8596169908,71.8.6.207,"{""location"": ""MU"", ""is_mobile"": false}" 18848,7,424,2017-05-08 05:42:52,http://mcculloughsporer.co/thalia,0.7922783448,173.63.211.162,"{""location"": ""ZM"", ""is_mobile"": false}" 18849,7,424,2017-06-14 01:52:23,http://schoennikolaus.org/dorthy,0.4022323617,74.210.94.9,"{""location"": ""GP"", ""is_mobile"": true}" 18850,7,424,2017-04-18 07:36:32,http://ondrickamcclure.io/makayla,0.2872190698,48.244.116.212,"{""location"": ""MH"", ""is_mobile"": false}" 18851,7,424,2017-02-22 13:32:34,http://naderschinner.info/dudley,0.4299741657,18.180.96.78,"{""location"": ""BY"", ""is_mobile"": true}" 18852,7,424,2017-04-27 11:24:53,http://bayerschiller.org/anais.abshire,0.9668415481,88.136.223.180,"{""location"": ""PK"", ""is_mobile"": false}" 18853,7,424,2017-02-12 21:56:25,http://shieldshayes.io/mikel,0.6813928048,12.103.147.110,"{""location"": ""BV"", ""is_mobile"": true}" 18854,7,424,2017-02-10 00:44:20,http://heaney.name/tony,0.7343459390,231.122.8.28,"{""location"": ""ML"", ""is_mobile"": true}" 18855,7,424,2017-04-05 19:42:45,http://heathcote.co/jesus,0.8548761029,193.14.210.124,"{""location"": ""CX"", ""is_mobile"": false}" 18856,7,424,2017-01-16 19:32:54,http://heller.com/janet.feeney,0.4505842499,172.142.174.33,"{""location"": ""BQ"", ""is_mobile"": true}" 18857,7,424,2017-05-03 11:15:14,http://hirthelarson.info/vena.strosin,0.1610218110,23.242.88.8,"{""location"": ""CD"", ""is_mobile"": true}" 18858,7,424,2017-01-06 12:48:45,http://larkin.io/geo,0.3053404252,23.200.147.186,"{""location"": ""EC"", ""is_mobile"": true}" 18859,7,424,2017-04-15 15:35:49,http://veum.biz/freda,0.1001481383,148.94.250.182,"{""location"": ""BZ"", ""is_mobile"": true}" 18860,7,424,2017-04-25 07:51:47,http://satterfieldbrown.name/raina.doyle,0.9285968439,134.35.106.45,"{""location"": ""GL"", ""is_mobile"": false}" 18861,7,425,2017-02-05 16:15:23,http://nitzsche.biz/roxanne,0.5821259369,63.110.157.17,"{""location"": ""BQ"", ""is_mobile"": false}" 18862,7,425,2017-05-16 05:14:02,http://gislason.net/angie.bins,0.2758675486,208.136.92.155,"{""location"": ""NO"", ""is_mobile"": true}" 18863,7,425,2017-05-22 12:00:28,http://schmidttillman.io/jeyca_wilkinson,0.5411108295,161.190.65.146,"{""location"": ""IN"", ""is_mobile"": false}" 18864,7,425,2017-01-15 14:24:42,http://koeppwalsh.info/gaetano,0.1207049644,47.186.220.7,"{""location"": ""GG"", ""is_mobile"": true}" 18865,7,425,2016-12-27 20:50:00,http://hills.info/cameron,0.5479459627,97.163.56.35,"{""location"": ""MU"", ""is_mobile"": true}" 18866,7,425,2017-03-30 19:31:31,http://rohan.co/madaline,0.8509683040,97.63.238.67,"{""location"": ""MP"", ""is_mobile"": true}" 18867,7,425,2017-04-30 17:35:19,http://farrell.name/barney_zboncak,0.7929119054,202.41.42.154,"{""location"": ""CR"", ""is_mobile"": true}" 18868,7,425,2016-12-15 22:12:55,http://johnston.net/hettie,0.6108610126,163.94.48.80,"{""location"": ""TO"", ""is_mobile"": false}" 18869,7,425,2017-04-10 23:15:45,http://bodeskiles.org/odea_mohr,0.8512465565,210.183.202.31,"{""location"": ""GR"", ""is_mobile"": true}" 18870,7,425,2017-02-19 08:25:05,http://king.biz/lorenz,0.9161939051,139.139.127.196,"{""location"": ""NE"", ""is_mobile"": false}" 18871,7,425,2017-06-13 04:13:52,http://hermannwaelchi.info/gerson.feil,0.6703363834,172.87.213.102,"{""location"": ""PW"", ""is_mobile"": true}" 18872,7,425,2017-01-21 04:45:10,http://harris.net/jeyca,0.4943332863,197.147.208.149,"{""location"": ""SI"", ""is_mobile"": true}" 18873,7,425,2017-03-21 14:19:45,http://murphy.net/elsa_mueller,0.0386805183,68.229.147.205,"{""location"": ""KN"", ""is_mobile"": true}" 18874,7,425,2017-01-07 18:24:27,http://cole.info/kristoffer.schuster,0.0375527458,40.151.58.71,"{""location"": ""MR"", ""is_mobile"": false}" 18875,7,425,2017-06-09 01:26:50,http://ko.io/kareem,0.0733072613,41.70.47.37,"{""location"": ""TK"", ""is_mobile"": false}" 18876,7,425,2017-01-20 03:29:55,http://schneider.info/marc,0.4388430720,98.13.26.147,"{""location"": ""LU"", ""is_mobile"": false}" 18877,7,425,2017-02-22 11:49:48,http://mcglynn.name/yesenia.hickle,0.3395245308,54.100.21.247,"{""location"": ""IS"", ""is_mobile"": true}" 18878,7,425,2017-05-28 07:25:22,http://murazik.co/everett_kaulke,0.0328551975,142.166.143.159,"{""location"": ""TG"", ""is_mobile"": true}" 18879,7,425,2017-02-27 11:32:19,http://weimannschuster.com/lora,0.3083424019,185.203.49.132,"{""location"": ""BQ"", ""is_mobile"": true}" 18880,7,425,2017-06-04 23:32:34,http://hoegermcclure.org/corene,0.6579782693,119.233.100.59,"{""location"": ""VE"", ""is_mobile"": true}" 18881,7,425,2017-05-01 13:21:38,http://bechtelar.org/bertrand,0.7128746576,226.227.212.57,"{""location"": ""ER"", ""is_mobile"": true}" 18882,7,425,2017-04-12 10:02:49,http://mitchell.name/laurie,0.7291928600,167.9.147.101,"{""location"": ""BS"", ""is_mobile"": true}" 18883,7,425,2017-02-03 15:25:01,http://kuphal.org/noel,0.1172615827,135.150.24.147,"{""location"": ""JO"", ""is_mobile"": false}" 18884,7,425,2017-06-08 09:10:49,http://veum.co/jodie,0.7374242241,64.118.207.78,"{""location"": ""AS"", ""is_mobile"": true}" 18885,7,425,2017-02-20 01:55:07,http://mertz.org/marco,0.1362357160,120.85.29.159,"{""location"": ""MM"", ""is_mobile"": true}" 18886,7,425,2017-02-25 09:06:33,http://ritchie.biz/austin,0.2250461512,81.17.101.125,"{""location"": ""RW"", ""is_mobile"": false}" 18887,7,425,2017-04-05 19:08:15,http://denesikjacobson.name/dolly.hilll,0.8656749959,166.19.229.38,"{""location"": ""MM"", ""is_mobile"": true}" 18888,7,425,2017-03-28 06:23:06,http://rosenbaum.io/jocelyn_kaulke,0.7007745679,151.104.193.216,"{""location"": ""IE"", ""is_mobile"": false}" 18889,7,425,2017-05-29 00:24:04,http://marksankunding.com/davin_keler,0.6042596553,245.85.163.118,"{""location"": ""PN"", ""is_mobile"": true}" 18890,7,425,2017-01-21 01:56:23,http://collinsheaney.io/johan_spencer,0.3688629410,159.52.60.140,"{""location"": ""WS"", ""is_mobile"": false}" 18891,7,425,2017-06-11 16:30:12,http://effertz.net/jacinthe,0.7570197560,64.71.24.51,"{""location"": ""CX"", ""is_mobile"": false}" 18892,7,425,2017-05-03 06:56:47,http://cartwright.org/nova.rutherford,0.0606327131,103.243.121.129,"{""location"": ""LA"", ""is_mobile"": true}" 18893,7,425,2017-06-10 13:29:10,http://jerde.co/johnny.kaulke,0.1943964822,159.90.202.60,"{""location"": ""BW"", ""is_mobile"": true}" 18894,7,425,2017-02-02 17:22:04,http://schoenhaag.co/josephine,0.5210939574,89.106.9.149,"{""location"": ""IS"", ""is_mobile"": false}" 18895,7,425,2017-03-19 15:26:51,http://paucek.co/mariano,0.7969896052,24.12.74.226,"{""location"": ""HM"", ""is_mobile"": false}" 18896,7,425,2017-05-31 15:19:44,http://bogisich.io/wava,0.4893807962,122.67.19.122,"{""location"": ""CV"", ""is_mobile"": false}" 18897,7,425,2017-02-06 12:36:05,http://pfeffer.org/peggie,0.4037772951,222.26.244.87,"{""location"": ""NI"", ""is_mobile"": true}" 18898,7,425,2017-03-13 17:47:57,http://moen.info/aliyah,0.3849622502,38.109.185.45,"{""location"": ""KM"", ""is_mobile"": true}" 18899,7,425,2017-01-21 07:21:53,http://hillwaniawski.biz/brycen.bergnaum,0.0439720279,60.204.163.71,"{""location"": ""KH"", ""is_mobile"": true}" 18900,7,425,2017-05-10 08:56:44,http://roob.org/darron,0.0386805868,121.11.228.228,"{""location"": ""GS"", ""is_mobile"": true}" 18901,7,425,2017-02-10 14:58:40,http://kuhn.io/randy,0.9831097267,249.90.10.240,"{""location"": ""CV"", ""is_mobile"": true}" 18902,7,425,2017-01-22 14:19:46,http://kling.info/samson,0.9046765181,158.147.120.171,"{""location"": ""TJ"", ""is_mobile"": true}" 18903,7,425,2016-12-19 01:22:28,http://hoppe.com/alejandra_kemmer,0.0522724592,87.225.189.73,"{""location"": ""KI"", ""is_mobile"": true}" 18904,7,425,2017-05-18 15:27:09,http://haag.net/alda,0.9733864068,195.26.6.45,"{""location"": ""BM"", ""is_mobile"": false}" 18905,7,425,2017-05-01 05:25:31,http://ledner.net/helen_abshire,0.4162259035,98.48.195.109,"{""location"": ""UY"", ""is_mobile"": true}" 18906,7,426,2017-03-11 06:45:56,http://hegmann.net/eino.towne,0.4294517699,205.131.160.38,"{""location"": ""GY"", ""is_mobile"": false}" 18907,7,426,2017-02-10 03:24:22,http://lang.io/arnold,0.0086292703,91.38.62.119,"{""location"": ""CL"", ""is_mobile"": false}" 18908,7,426,2017-02-18 22:09:03,http://rueckerspinka.io/christelle,0.2504043590,77.40.93.231,"{""location"": ""OM"", ""is_mobile"": true}" 18909,7,426,2017-04-23 11:26:13,http://hills.io/zackery,0.6226209609,245.76.98.177,"{""location"": ""AF"", ""is_mobile"": false}" 18910,7,426,2017-02-19 19:53:30,http://abshire.biz/judah,0.4061295610,100.75.209.72,"{""location"": ""WF"", ""is_mobile"": false}" 18911,7,426,2017-04-22 23:13:37,http://kirlin.com/antwan,0.6763836925,205.232.204.181,"{""location"": ""SE"", ""is_mobile"": false}" 18912,7,426,2017-04-19 09:50:41,http://walterstark.name/zane.reilly,0.9051106655,146.111.94.87,"{""location"": ""SA"", ""is_mobile"": false}" 18913,7,426,2017-06-12 20:55:15,http://fritschortiz.info/carolyne_monahan,0.4622283874,155.184.113.46,"{""location"": ""DE"", ""is_mobile"": true}" 18914,7,426,2016-12-20 01:32:28,http://hoegerdickinson.io/annabelle,0.7865352208,140.102.77.75,"{""location"": ""LK"", ""is_mobile"": false}" 18915,7,426,2017-06-01 05:42:54,http://sauer.com/hattie_ullrich,0.9701937192,60.236.159.215,"{""location"": ""BZ"", ""is_mobile"": false}" 18916,7,426,2017-04-17 06:27:12,http://collins.info/durward,0.7647278915,187.65.195.177,"{""location"": ""SB"", ""is_mobile"": true}" 18917,7,426,2017-02-04 07:23:39,http://hirthejenkins.name/ansley_heel,0.3740679580,2.38.234.25,"{""location"": ""CK"", ""is_mobile"": false}" 18918,7,426,2017-04-24 14:00:07,http://dooley.org/taya_effertz,0.3520311896,152.21.131.251,"{""location"": ""NG"", ""is_mobile"": true}" 18919,7,426,2017-02-23 23:32:42,http://brakus.com/eloisa.mcclure,0.0073553153,185.128.32.70,"{""location"": ""GP"", ""is_mobile"": true}" 18920,7,426,2017-04-21 03:19:36,http://klingdooley.io/abdiel,0.9190063822,70.34.218.156,"{""location"": ""EE"", ""is_mobile"": true}" 18921,7,426,2017-04-09 06:52:04,http://little.info/domenico.hartmann,0.8593080909,169.57.203.95,"{""location"": ""TM"", ""is_mobile"": false}" 18922,7,426,2016-12-23 07:39:34,http://kshlerin.com/morgan.schroeder,0.1533359026,153.138.170.139,"{""location"": ""KZ"", ""is_mobile"": false}" 18923,7,426,2017-03-07 03:30:02,http://murazik.info/helmer,0.6358662741,229.75.92.154,"{""location"": ""YT"", ""is_mobile"": true}" 18924,7,426,2017-05-04 05:00:51,http://mosciskiankunding.info/felix,0.4802037942,116.243.191.73,"{""location"": ""SH"", ""is_mobile"": false}" 18925,7,426,2017-05-16 20:59:07,http://halvorson.net/alyon,0.0744733044,251.185.130.181,"{""location"": ""KZ"", ""is_mobile"": true}" 18926,7,426,2017-05-13 09:11:25,http://legros.org/lesley,0.0927159294,174.123.149.56,"{""location"": ""MD"", ""is_mobile"": true}" 18927,7,426,2017-02-17 14:47:41,http://mannkirlin.net/jacques,0.5749792152,105.161.217.253,"{""location"": ""MV"", ""is_mobile"": true}" 18928,7,426,2017-01-12 10:02:54,http://orn.info/bernice_gusikowski,0.4063275015,188.168.11.252,"{""location"": ""HR"", ""is_mobile"": false}" 18929,7,426,2017-03-14 14:43:02,http://gaylordpaucek.name/theodore,0.6645056537,45.58.52.46,"{""location"": ""BE"", ""is_mobile"": false}" 18930,7,426,2017-05-12 15:36:03,http://langworth.name/norene,0.3852759727,103.16.230.49,"{""location"": ""LB"", ""is_mobile"": true}" 18931,7,426,2017-01-28 15:42:07,http://farrell.name/adelia.altenwerth,0.0337799691,55.186.8.165,"{""location"": ""GH"", ""is_mobile"": true}" 18932,7,426,2017-06-09 09:59:35,http://balistreri.co/wellington.reynolds,0.8622185270,167.102.213.64,"{""location"": ""OM"", ""is_mobile"": true}" 18933,7,426,2017-05-13 22:50:16,http://hickle.name/demarcus,0.2581259735,225.84.139.104,"{""location"": ""SI"", ""is_mobile"": false}" 18934,7,426,2017-01-21 02:38:46,http://haley.org/wanda.marquardt,0.2418921009,7.224.98.209,"{""location"": ""SE"", ""is_mobile"": false}" 18935,7,426,2017-05-05 06:52:46,http://goyette.net/collin_greenfelder,0.4476510048,223.184.226.85,"{""location"": ""SZ"", ""is_mobile"": false}" 18936,7,426,2017-02-13 03:01:28,http://kling.com/catherine_skiles,0.6747213880,173.218.6.11,"{""location"": ""FR"", ""is_mobile"": false}" 18937,7,426,2017-01-06 05:28:59,http://streichheller.io/graciela.homenick,0.2949914716,11.59.247.20,"{""location"": ""SV"", ""is_mobile"": false}" 18938,7,426,2017-04-23 22:17:30,http://rowe.net/madisen.hansen,0.3089262603,158.50.16.246,"{""location"": ""BM"", ""is_mobile"": false}" 18939,7,426,2017-06-01 11:58:22,http://tremblay.net/dock_leannon,0.3058609278,148.142.215.219,"{""location"": ""MC"", ""is_mobile"": true}" 18940,7,426,2017-06-12 02:58:13,http://kub.name/charley,0.7156406523,144.54.142.57,"{""location"": ""CO"", ""is_mobile"": true}" 18941,7,426,2017-06-03 19:55:25,http://rempel.net/amos,0.0140923330,2.33.153.88,"{""location"": ""SM"", ""is_mobile"": true}" 18942,7,426,2017-06-12 12:24:51,http://johnston.info/mazie.bashirian,0.9158931845,226.53.201.49,"{""location"": ""CM"", ""is_mobile"": true}" 18943,7,426,2017-02-09 02:58:03,http://grahamschamberger.info/jaclyn.stracke,0.4130724746,69.210.113.114,"{""location"": ""AT"", ""is_mobile"": true}" 18944,7,426,2016-12-31 23:40:18,http://farrellboyle.info/merl_pagac,0.5267899264,119.133.57.144,"{""location"": ""TM"", ""is_mobile"": true}" 18945,7,426,2017-05-04 16:54:48,http://reilly.biz/fritz,0.3081975003,205.225.169.131,"{""location"": ""OM"", ""is_mobile"": false}" 18946,7,426,2017-03-25 14:49:44,http://gibson.org/alda,0.8697548115,157.54.177.168,"{""location"": ""GM"", ""is_mobile"": true}" 18947,7,426,2017-01-27 07:20:00,http://hills.info/vergie,0.9896333763,243.78.22.228,"{""location"": ""NF"", ""is_mobile"": true}" 18948,7,426,2017-02-15 14:45:13,http://mayer.io/toby.wunsch,0.0435741173,171.39.40.93,"{""location"": ""MF"", ""is_mobile"": true}" 18949,7,426,2017-05-02 03:18:40,http://wehnerrolfson.net/camila,0.5392535323,150.52.14.21,"{""location"": ""AZ"", ""is_mobile"": true}" 18950,7,427,2017-04-05 06:24:55,http://klein.info/casey.mccullough,0.7576334508,242.32.72.103,"{""location"": ""YE"", ""is_mobile"": true}" 18951,7,427,2017-01-30 07:25:07,http://schuster.net/andrew_predovic,0.8529680713,152.56.147.161,"{""location"": ""AL"", ""is_mobile"": false}" 18952,7,427,2017-02-12 16:32:49,http://abernathy.name/zander_howe,0.3551304883,6.160.28.217,"{""location"": ""MQ"", ""is_mobile"": true}" 18953,7,427,2017-04-04 17:34:49,http://weber.info/cecilia_fritsch,0.8331105991,24.105.173.186,"{""location"": ""BT"", ""is_mobile"": true}" 18954,7,427,2017-05-19 21:41:46,http://langworth.info/carolina,0.7335877499,187.67.68.182,"{""location"": ""PF"", ""is_mobile"": false}" 18955,7,427,2017-05-14 07:02:07,http://lubowitz.info/ida,0.9687174809,46.142.122.149,"{""location"": ""SB"", ""is_mobile"": true}" 18956,7,427,2016-12-16 12:06:08,http://brekke.net/taryn,0.4829981026,210.200.113.146,"{""location"": ""NC"", ""is_mobile"": true}" 18957,7,427,2017-02-16 15:01:39,http://hegmann.net/vernice.wintheiser,0.4169756099,134.244.131.121,"{""location"": ""GH"", ""is_mobile"": true}" 18958,7,427,2017-04-26 18:46:16,http://morar.biz/nikita.rice,0.2400067598,150.241.53.183,"{""location"": ""ML"", ""is_mobile"": false}" 18959,7,427,2017-03-26 06:21:05,http://mrazullrich.name/berenice,0.3020694857,4.32.118.25,"{""location"": ""HU"", ""is_mobile"": false}" 18960,7,427,2016-12-25 03:05:01,http://kuhn.org/morris,0.6333359796,240.186.28.139,"{""location"": ""SJ"", ""is_mobile"": true}" 18961,7,427,2017-01-15 06:49:43,http://greenfelder.biz/mabel_schmidt,0.2139445384,213.160.199.21,"{""location"": ""IN"", ""is_mobile"": false}" 18962,7,427,2017-06-13 21:58:57,http://powlowski.name/kelli_kovacek,0.1551217532,21.147.83.73,"{""location"": ""OM"", ""is_mobile"": false}" 18963,7,427,2017-04-15 12:29:18,http://mitchell.net/liam,0.2165492341,65.115.106.206,"{""location"": ""KY"", ""is_mobile"": true}" 18964,7,427,2017-04-29 05:40:16,http://rau.org/raymond_williamson,0.8577014316,30.5.30.212,"{""location"": ""MQ"", ""is_mobile"": false}" 18965,7,427,2017-05-05 06:55:26,http://mayer.com/kevon,0.8756326505,250.115.244.50,"{""location"": ""GF"", ""is_mobile"": false}" 18966,7,427,2016-12-31 18:09:47,http://yundttorphy.org/mack.willms,0.0994225229,47.118.165.143,"{""location"": ""BJ"", ""is_mobile"": false}" 18967,7,427,2017-04-18 05:36:22,http://friesenwillms.org/esteban.gerlach,0.5906951441,118.174.117.230,"{""location"": ""SI"", ""is_mobile"": true}" 18968,7,427,2017-02-12 08:24:41,http://stroman.io/nigel,0.6276292914,181.216.177.180,"{""location"": ""FK"", ""is_mobile"": true}" 18969,7,427,2017-06-12 20:44:37,http://shanahanhahn.name/jovan.runolfon,0.9684960820,104.139.206.144,"{""location"": ""VC"", ""is_mobile"": false}" 18970,7,427,2017-01-20 11:34:08,http://heel.info/alize.bins,0.4057892702,28.198.203.212,"{""location"": ""QA"", ""is_mobile"": true}" 18971,7,427,2017-06-06 08:04:15,http://walker.io/colleen,0.4284207031,211.135.73.243,"{""location"": ""MR"", ""is_mobile"": true}" 18972,7,427,2017-04-21 11:02:38,http://stehr.co/felicity,0.3734471476,174.245.84.208,"{""location"": ""KP"", ""is_mobile"": true}" 18973,7,427,2016-12-23 15:32:04,http://bechtelar.name/aric_paucek,0.0333618060,181.25.87.71,"{""location"": ""ID"", ""is_mobile"": true}" 18974,7,427,2017-01-13 15:40:36,http://labadie.org/judy,0.2385336360,99.195.34.31,"{""location"": ""GI"", ""is_mobile"": true}" 18975,7,427,2017-04-05 14:13:48,http://treutel.name/helmer_keeling,0.6030522055,98.231.179.96,"{""location"": ""FI"", ""is_mobile"": false}" 18976,7,427,2017-03-01 20:26:21,http://greenfelderkilback.org/gudrun,0.0157478924,153.66.136.50,"{""location"": ""TV"", ""is_mobile"": true}" 18977,7,427,2017-05-14 18:07:47,http://glover.biz/orie.hermann,0.5090140007,66.170.181.27,"{""location"": ""AZ"", ""is_mobile"": true}" 18978,7,427,2016-12-14 12:00:30,http://kochdickinson.biz/anya_hagenes,0.4040356011,112.134.44.34,"{""location"": ""NO"", ""is_mobile"": false}" 18979,7,427,2017-02-14 00:17:22,http://vonrueden.name/anastacio,0.1068795941,242.176.119.226,"{""location"": ""AX"", ""is_mobile"": false}" 18980,7,427,2017-04-20 00:45:46,http://quitzonjohnson.biz/marcellus_weinat,0.3766739390,136.162.221.239,"{""location"": ""GQ"", ""is_mobile"": true}" 18981,7,427,2017-03-26 07:11:00,http://hyatt.com/ashly_tillman,0.4671923210,5.9.16.127,"{""location"": ""BM"", ""is_mobile"": true}" 18982,7,427,2017-06-07 09:03:16,http://strosinblanda.info/otha_goodwin,0.0151850135,193.216.196.51,"{""location"": ""EC"", ""is_mobile"": false}" 18983,7,427,2016-12-25 14:46:57,http://hodkiewicz.com/wyatt,0.5331922586,134.118.194.67,"{""location"": ""NG"", ""is_mobile"": false}" 18984,7,427,2017-01-05 22:54:56,http://trantow.org/trace_buckridge,0.7630491209,208.108.116.118,"{""location"": ""DZ"", ""is_mobile"": false}" 18985,7,427,2017-01-18 01:29:49,http://sipes.biz/mallory.gutmann,0.0601174091,5.97.205.3,"{""location"": ""SD"", ""is_mobile"": true}" 18986,7,427,2017-03-08 11:14:47,http://zemlak.info/nichole.cummings,0.0450422821,89.68.33.29,"{""location"": ""BN"", ""is_mobile"": true}" 18987,7,427,2017-04-25 15:10:00,http://grant.name/sherman,0.5366644655,116.125.224.95,"{""location"": ""BR"", ""is_mobile"": false}" 18988,7,427,2017-05-09 01:09:39,http://johnson.com/opal.gutmann,0.9434287992,116.203.31.95,"{""location"": ""TC"", ""is_mobile"": true}" 18989,7,427,2017-01-07 11:42:35,http://jaskolski.com/sister,0.2239283060,132.152.62.120,"{""location"": ""YE"", ""is_mobile"": true}" 18990,7,427,2016-12-19 16:28:10,http://wisoky.co/brooklyn.erdman,0.2682414450,43.173.254.69,"{""location"": ""HR"", ""is_mobile"": true}" 18991,7,427,2017-04-26 16:17:41,http://schroederernser.name/brian_schroeder,0.4822956081,9.11.6.40,"{""location"": ""CR"", ""is_mobile"": true}" 18992,7,427,2017-04-22 22:14:33,http://halvorson.com/adolph.kihn,0.5324550275,181.215.171.32,"{""location"": ""TD"", ""is_mobile"": false}" 18993,7,427,2017-04-08 20:26:23,http://mraz.name/dixie,0.8691700342,159.240.190.15,"{""location"": ""BW"", ""is_mobile"": false}" 18994,7,427,2017-06-02 04:24:44,http://hettinger.biz/manuel.mcglynn,0.7642585601,198.161.114.121,"{""location"": ""TG"", ""is_mobile"": false}" 18995,7,427,2017-01-18 02:21:48,http://ratke.io/ansel,0.8398247009,10.140.222.194,"{""location"": ""GP"", ""is_mobile"": false}" 18996,7,427,2017-03-15 17:52:05,http://mantehoeger.io/warren_spinka,0.1123914691,235.226.122.69,"{""location"": ""BW"", ""is_mobile"": false}" 18997,7,427,2017-06-01 15:40:00,http://fahey.com/malcolm.jacobs,0.1527839764,171.185.31.136,"{""location"": ""DO"", ""is_mobile"": false}" 18998,7,427,2016-12-19 17:38:44,http://sauer.io/marques_barton,0.7466241166,6.204.183.136,"{""location"": ""GH"", ""is_mobile"": true}" 18999,7,427,2017-03-21 06:59:50,http://champlin.com/stephon.oberbrunner,0.9542421775,17.196.32.236,"{""location"": ""GQ"", ""is_mobile"": true}" 19000,7,427,2017-04-14 06:35:43,http://grant.net/myra,0.7291043324,136.41.211.254,"{""location"": ""MA"", ""is_mobile"": false}" 19001,7,427,2017-01-27 17:31:28,http://wolff.info/augusta_runolfsdottir,0.7512145947,97.41.204.77,"{""location"": ""EE"", ""is_mobile"": true}" 19002,7,427,2017-01-10 21:13:28,http://huel.info/porter_smitham,0.3920148133,107.42.194.138,"{""location"": ""BD"", ""is_mobile"": false}" 19003,7,427,2017-02-12 21:41:38,http://lesch.info/florian,0.0208996369,12.126.69.37,"{""location"": ""DJ"", ""is_mobile"": true}" 19004,7,427,2017-06-07 02:37:41,http://faheyhammes.org/terrance.paucek,0.2594419403,33.107.42.242,"{""location"": ""PT"", ""is_mobile"": false}" 19005,7,427,2017-05-21 04:36:21,http://ruel.io/lyda,0.5286473770,87.109.151.71,"{""location"": ""MQ"", ""is_mobile"": false}" 19006,7,427,2017-01-09 23:40:44,http://runolfsdottir.name/mortimer.herzog,0.9150390317,124.52.144.48,"{""location"": ""BV"", ""is_mobile"": false}" 19007,7,427,2017-04-11 23:24:40,http://kemmer.name/nathen,0.4734909360,160.223.101.248,"{""location"": ""TR"", ""is_mobile"": false}" 19008,7,427,2017-03-23 10:55:16,http://luettgen.co/dallas_mcglynn,0.2074056003,233.248.32.202,"{""location"": ""FO"", ""is_mobile"": true}" 19009,7,427,2017-04-03 05:32:03,http://collins.net/lenna,0.7066170221,10.165.218.95,"{""location"": ""SA"", ""is_mobile"": false}" 19010,7,427,2017-04-05 12:18:41,http://conroy.org/luna_nader,0.9544507585,2.87.134.57,"{""location"": ""MH"", ""is_mobile"": false}" 19011,7,427,2017-06-04 11:56:39,http://dubuque.name/chaz,0.6605090923,196.24.102.78,"{""location"": ""BN"", ""is_mobile"": true}" 19012,7,427,2017-02-11 17:19:58,http://reillyruel.org/favian_bashirian,0.4678278904,233.214.154.146,"{""location"": ""SD"", ""is_mobile"": true}" 19013,7,427,2017-03-31 23:42:43,http://shields.net/kathryn,0.2661984637,185.86.87.115,"{""location"": ""CR"", ""is_mobile"": false}" 19014,7,427,2017-03-25 14:53:49,http://okon.com/wanda,0.1714935409,11.207.20.122,"{""location"": ""WS"", ""is_mobile"": false}" 19015,7,428,2017-03-15 10:07:22,http://mcglynnmclaughlin.biz/israel,0.2745220596,90.52.186.168,"{""location"": ""HU"", ""is_mobile"": true}" 19016,7,428,2017-01-29 08:47:49,http://vonrueden.info/collin.mayer,0.2652139441,144.78.254.6,"{""location"": ""KG"", ""is_mobile"": false}" 19017,7,428,2017-02-23 00:56:08,http://schuppe.net/rick,0.0862767219,181.180.167.192,"{""location"": ""CA"", ""is_mobile"": false}" 19018,7,428,2017-03-10 10:41:05,http://wolffschulist.co/mckenna_hickle,0.3455486523,69.97.68.251,"{""location"": ""FK"", ""is_mobile"": true}" 19019,7,428,2017-03-28 07:26:33,http://conn.io/nelle_jacobs,0.6284229977,100.156.106.2,"{""location"": ""CR"", ""is_mobile"": false}" 19020,7,428,2017-01-05 19:29:19,http://pfefferpacocha.com/uriah_lemke,0.0113270956,147.176.214.249,"{""location"": ""IN"", ""is_mobile"": true}" 19021,7,428,2017-04-12 10:38:46,http://dickens.net/cooper,0.4334752925,127.157.157.75,"{""location"": ""IE"", ""is_mobile"": true}" 19022,7,428,2017-01-04 07:49:48,http://cummingchuppe.com/alanna,0.8506323934,206.84.71.209,"{""location"": ""AS"", ""is_mobile"": true}" 19023,7,428,2017-04-08 17:11:51,http://lubowitz.com/elsa,0.2819020560,56.194.231.230,"{""location"": ""TV"", ""is_mobile"": false}" 19024,7,428,2017-03-13 04:53:43,http://homenick.com/bernadette_upton,0.0150343498,198.101.161.197,"{""location"": ""FO"", ""is_mobile"": true}" 19025,7,428,2017-05-31 10:39:49,http://wilderman.io/daron,0.4978751305,218.200.22.113,"{""location"": ""VG"", ""is_mobile"": false}" 19026,7,428,2016-12-26 23:02:13,http://brown.biz/vilma,0.2382526621,171.113.133.109,"{""location"": ""IM"", ""is_mobile"": false}" 19027,7,428,2017-05-09 10:13:03,http://pfeffer.name/jonathon_connelly,0.5919482289,69.108.86.223,"{""location"": ""KY"", ""is_mobile"": false}" 19028,7,428,2017-04-05 19:54:53,http://boyle.info/devante,0.7832313697,108.170.140.11,"{""location"": ""GF"", ""is_mobile"": false}" 19029,7,428,2017-04-17 23:04:41,http://hegmann.net/daren_shanahan,0.1935665060,47.227.141.231,"{""location"": ""TT"", ""is_mobile"": true}" 19030,7,428,2017-05-09 17:59:54,http://hicklecremin.net/naomie,0.0770459220,198.95.208.254,"{""location"": ""CA"", ""is_mobile"": true}" 19031,7,428,2017-02-03 06:36:42,http://rolfson.biz/marcellus,0.5801586438,206.250.107.94,"{""location"": ""LC"", ""is_mobile"": false}" 19032,7,428,2017-01-28 15:50:10,http://feeneytorphy.info/alyson_beatty,0.2933218082,70.168.226.114,"{""location"": ""KR"", ""is_mobile"": true}" 19033,7,428,2017-01-13 19:47:13,http://moore.info/sam_hilpert,0.4536278542,74.14.14.157,"{""location"": ""AF"", ""is_mobile"": true}" 19034,7,428,2017-02-18 08:56:20,http://tillmanrohan.info/libby,0.4979785301,30.21.158.137,"{""location"": ""LC"", ""is_mobile"": false}" 19035,7,428,2017-01-15 16:48:12,http://collins.com/werner,0.9635852337,113.117.110.237,"{""location"": ""MC"", ""is_mobile"": true}" 19036,7,428,2017-03-26 06:54:32,http://denesik.net/evert.reinger,0.1438154631,233.32.190.202,"{""location"": ""TM"", ""is_mobile"": true}" 19037,7,429,2017-04-15 23:04:53,http://lowe.org/margret,0.5182091536,27.75.14.233,"{""location"": ""GT"", ""is_mobile"": true}" 19038,7,429,2017-03-16 22:15:41,http://haley.co/pearline.leffler,0.1237532065,86.78.203.234,"{""location"": ""MM"", ""is_mobile"": true}" 19039,7,429,2017-05-21 06:40:40,http://hodkiewicz.com/cornelius_jakubowski,0.9179219213,170.212.190.166,"{""location"": ""ST"", ""is_mobile"": true}" 19040,7,429,2017-03-13 02:03:23,http://ritchiefranecki.biz/donna_watsica,0.1952112932,178.53.181.65,"{""location"": ""MV"", ""is_mobile"": false}" 19041,7,429,2017-03-28 05:42:03,http://hayesbrown.net/janae_fadel,0.7442527821,46.173.28.234,"{""location"": ""SS"", ""is_mobile"": false}" 19042,7,429,2017-05-16 20:24:05,http://ankundinghuels.name/khalil_brakus,0.2107031712,209.116.48.79,"{""location"": ""MU"", ""is_mobile"": true}" 19043,7,429,2017-02-04 03:14:50,http://berge.io/dayton.harvey,0.1221169705,19.126.21.16,"{""location"": ""HR"", ""is_mobile"": true}" 19044,7,429,2017-01-04 12:55:43,http://cummerata.org/magdalena,0.9197538012,177.87.125.23,"{""location"": ""KH"", ""is_mobile"": true}" 19045,7,429,2017-05-05 03:13:05,http://satterfield.name/rowland.terry,0.1232725479,57.55.57.242,"{""location"": ""PG"", ""is_mobile"": true}" 19046,7,429,2017-01-06 20:21:11,http://kuhic.biz/tillman,0.5986801042,163.77.81.241,"{""location"": ""TF"", ""is_mobile"": true}" 19047,7,429,2016-12-24 08:43:48,http://medhurst.co/ryleigh,0.0376113270,60.251.141.172,"{""location"": ""HR"", ""is_mobile"": false}" 19048,7,429,2017-04-09 11:49:44,http://carrolllittel.com/deie.brown,0.4272487617,9.244.161.36,"{""location"": ""CK"", ""is_mobile"": true}" 19049,7,429,2017-03-17 07:24:11,http://grant.org/ted_breitenberg,0.9449467938,161.4.234.48,"{""location"": ""SY"", ""is_mobile"": false}" 19050,7,429,2017-01-30 01:16:32,http://emardflatley.io/yasmine_treutel,0.3092434651,135.45.94.211,"{""location"": ""BH"", ""is_mobile"": false}" 19051,7,429,2017-02-14 18:32:35,http://watsicaheidenreich.info/geovanny_erdman,0.4314858016,71.126.85.104,"{""location"": ""RO"", ""is_mobile"": true}" 19052,7,429,2017-05-08 05:24:45,http://wintheiser.net/kaley_stark,0.0825669889,143.73.208.113,"{""location"": ""VE"", ""is_mobile"": false}" 19053,7,429,2017-02-13 02:35:52,http://moriette.net/roberta,0.9576020241,12.244.117.154,"{""location"": ""HR"", ""is_mobile"": false}" 19054,7,429,2017-02-04 12:09:02,http://von.io/susanna_volkman,0.7315659049,67.164.221.109,"{""location"": ""NE"", ""is_mobile"": true}" 19055,7,429,2017-03-05 17:33:03,http://wolf.name/lenore,0.2785521589,120.219.181.106,"{""location"": ""FJ"", ""is_mobile"": false}" 19056,7,429,2017-02-22 06:31:05,http://purdy.net/augusta,0.9597705035,172.85.78.132,"{""location"": ""SH"", ""is_mobile"": true}" 19057,7,429,2017-02-28 02:40:59,http://trantowbeahan.biz/angelina_daugherty,0.9755004689,242.87.34.226,"{""location"": ""ZA"", ""is_mobile"": true}" 19058,7,429,2017-05-10 22:54:15,http://medhurst.name/marisol_haag,0.8330861295,174.79.192.70,"{""location"": ""PS"", ""is_mobile"": true}" 19059,7,429,2017-04-14 22:00:17,http://weber.info/chet,0.8715131232,65.74.163.104,"{""location"": ""QA"", ""is_mobile"": true}" 19060,7,429,2017-01-20 22:39:44,http://runolfon.co/denis,0.2282702882,42.189.78.226,"{""location"": ""GM"", ""is_mobile"": true}" 19061,7,429,2016-12-18 13:25:19,http://emardzieme.io/chasity,0.5840311422,142.33.201.238,"{""location"": ""MO"", ""is_mobile"": true}" 19062,7,429,2017-02-13 08:08:58,http://tromp.co/katrine_okon,0.2053526318,61.77.65.188,"{""location"": ""AF"", ""is_mobile"": true}" 19063,7,429,2016-12-24 14:11:29,http://goodwinruecker.name/ollie,0.5298457402,163.36.226.81,"{""location"": ""SI"", ""is_mobile"": true}" 19064,7,429,2017-03-16 23:37:14,http://sauer.io/bud,0.2470211136,65.62.185.173,"{""location"": ""GM"", ""is_mobile"": true}" 19065,7,429,2017-04-01 02:30:20,http://dickens.com/braeden.satterfield,0.3759014284,124.13.208.241,"{""location"": ""FK"", ""is_mobile"": true}" 19066,7,429,2017-04-06 23:18:50,http://blick.org/wanda.howe,0.5104041106,30.127.233.17,"{""location"": ""LK"", ""is_mobile"": false}" 19067,7,429,2017-05-18 02:25:05,http://marvin.io/royce,0.9796879005,142.142.98.150,"{""location"": ""LR"", ""is_mobile"": true}" 19068,7,429,2017-05-11 05:01:10,http://johnston.com/christiana,0.9488145589,89.159.147.227,"{""location"": ""FK"", ""is_mobile"": false}" 19069,7,429,2017-01-24 01:15:13,http://hilll.biz/glen.emmerich,0.0605781343,113.44.209.168,"{""location"": ""TZ"", ""is_mobile"": true}" 19070,7,429,2017-02-02 23:59:53,http://ledner.com/hector,0.6049578857,81.151.130.201,"{""location"": ""GD"", ""is_mobile"": false}" 19071,7,429,2017-06-11 12:40:49,http://howe.info/clotilde_koepp,0.8174673048,245.159.185.148,"{""location"": ""KE"", ""is_mobile"": true}" 19072,7,429,2017-03-31 17:16:34,http://hane.io/darryl.lueilwitz,0.2105300744,7.4.88.171,"{""location"": ""GD"", ""is_mobile"": true}" 19073,7,429,2016-12-21 03:42:39,http://mertz.net/ford,0.4663740358,225.252.145.101,"{""location"": ""KH"", ""is_mobile"": false}" 19074,7,429,2017-04-05 18:03:51,http://balistreri.net/bennie,0.7155146463,174.16.179.209,"{""location"": ""HN"", ""is_mobile"": true}" 19075,7,429,2017-02-06 10:20:22,http://frami.biz/kory_hilpert,0.9163563514,216.192.121.49,"{""location"": ""CG"", ""is_mobile"": true}" 19076,7,429,2017-02-08 02:50:24,http://hauckhoppe.net/fern.davis,0.4115034549,121.83.79.147,"{""location"": ""SS"", ""is_mobile"": true}" 19077,7,429,2017-05-03 23:46:08,http://willms.info/hailey,0.9896121995,110.37.171.82,"{""location"": ""SE"", ""is_mobile"": true}" 19078,7,429,2017-05-11 17:04:03,http://sporermills.biz/jalen,0.6589864726,193.76.213.122,"{""location"": ""DZ"", ""is_mobile"": false}" 19079,7,429,2017-04-16 15:56:26,http://davis.biz/caandra_smith,0.2756109521,66.197.2.196,"{""location"": ""HK"", ""is_mobile"": true}" 19080,7,429,2017-04-09 02:55:21,http://kulas.info/marielle_cain,0.3696429386,192.93.186.54,"{""location"": ""GU"", ""is_mobile"": false}" 19081,7,429,2017-03-04 21:06:41,http://treutel.info/ayla,0.4639756348,78.166.114.95,"{""location"": ""AM"", ""is_mobile"": false}" 19082,7,429,2017-04-15 10:36:01,http://kihn.co/dahlia.sauer,0.5898669890,81.97.143.248,"{""location"": ""LK"", ""is_mobile"": false}" 19083,7,429,2017-05-22 23:32:45,http://ferry.info/troy,0.8404307810,90.70.130.192,"{""location"": ""FR"", ""is_mobile"": false}" 19084,7,430,2017-01-23 10:38:38,http://rath.io/jeanette,0.9442018844,244.72.216.129,"{""location"": ""MK"", ""is_mobile"": true}" 19085,7,430,2017-03-21 01:17:44,http://nitzsche.biz/eduardo_jacobson,0.7041721895,21.187.3.137,"{""location"": ""BB"", ""is_mobile"": false}" 19086,7,430,2017-01-15 15:13:07,http://koepp.org/chelsie,0.7017735169,157.191.216.164,"{""location"": ""AL"", ""is_mobile"": false}" 19087,7,430,2017-01-31 19:53:55,http://dickens.info/sydnee,0.4386028390,145.38.250.31,"{""location"": ""IM"", ""is_mobile"": false}" 19088,7,430,2017-05-24 02:43:59,http://larkin.org/addison,0.8163125569,219.2.29.2,"{""location"": ""BO"", ""is_mobile"": false}" 19089,7,430,2017-05-23 07:04:00,http://rohan.biz/pierce,0.8063943861,135.47.32.71,"{""location"": ""AI"", ""is_mobile"": true}" 19090,7,430,2017-04-01 21:05:51,http://hermiston.org/bailee,0.9015150320,240.239.61.175,"{""location"": ""CZ"", ""is_mobile"": true}" 19091,7,430,2017-02-10 11:01:01,http://padbergveum.org/zelda,0.2434684029,71.137.149.184,"{""location"": ""SV"", ""is_mobile"": true}" 19092,7,430,2017-02-11 10:06:48,http://grant.io/aliyah_mitchell,0.1932067052,233.14.70.139,"{""location"": ""BR"", ""is_mobile"": true}" 19093,7,430,2017-04-15 22:33:43,http://turcottegrimes.info/frederic,0.9371822376,201.142.201.96,"{""location"": ""ME"", ""is_mobile"": true}" 19094,7,430,2017-03-12 23:47:07,http://gottlieb.name/miles_damore,0.5645486698,93.136.75.60,"{""location"": ""PN"", ""is_mobile"": false}" 19095,7,430,2017-02-02 16:41:50,http://gutkowskiorn.io/lulu,0.5427576897,72.135.245.183,"{""location"": ""GT"", ""is_mobile"": false}" 19096,7,430,2017-04-20 12:10:44,http://jakubowskibayer.info/timmothy,0.3072360987,230.168.142.153,"{""location"": ""LB"", ""is_mobile"": false}" 19097,7,430,2017-02-07 20:56:44,http://larsonkuhic.com/roberto_herzog,0.2621290885,62.98.61.252,"{""location"": ""PA"", ""is_mobile"": false}" 19098,7,430,2017-06-10 14:30:54,http://romaguera.name/lorenzo.walker,0.3054911970,34.248.221.219,"{""location"": ""SV"", ""is_mobile"": true}" 19099,7,430,2016-12-24 09:49:32,http://torphydubuque.name/brittany.jacobi,0.0233194095,137.221.9.60,"{""location"": ""MO"", ""is_mobile"": false}" 19100,7,430,2017-06-04 19:13:29,http://kiehn.net/eleanore,0.8489264120,159.84.185.87,"{""location"": ""NO"", ""is_mobile"": true}" 19101,7,430,2017-02-09 00:11:21,http://erdmanroob.co/name_reichel,0.0798206062,156.121.29.145,"{""location"": ""CH"", ""is_mobile"": true}" 19102,7,430,2017-03-24 22:48:13,http://moorekuphal.com/salvador,0.8148968281,80.22.76.105,"{""location"": ""MP"", ""is_mobile"": false}" 19103,7,430,2017-05-20 05:38:43,http://bruen.biz/shyanne,0.3942129750,246.127.18.205,"{""location"": ""CN"", ""is_mobile"": false}" 19104,7,430,2017-03-20 16:19:27,http://mertzlangosh.name/curtis,0.4760729725,195.119.136.117,"{""location"": ""AZ"", ""is_mobile"": true}" 19105,7,430,2017-04-22 10:21:47,http://bogan.info/donny,0.5416580512,43.245.221.102,"{""location"": ""GP"", ""is_mobile"": true}" 19106,7,430,2017-02-26 23:20:05,http://erdman.net/lisette_williamson,0.8234078454,166.53.218.222,"{""location"": ""CO"", ""is_mobile"": true}" 19107,7,430,2017-06-02 07:00:27,http://kirlin.org/estrella,0.1727153718,90.26.234.72,"{""location"": ""TM"", ""is_mobile"": true}" 19108,7,430,2017-01-07 04:42:38,http://schaefer.net/effie,0.5371502123,35.10.232.7,"{""location"": ""BO"", ""is_mobile"": true}" 19109,7,430,2017-03-22 15:38:18,http://rosenbaum.info/lynn,0.3909754397,16.130.176.189,"{""location"": ""MS"", ""is_mobile"": false}" 19110,7,430,2017-04-23 18:56:41,http://kling.name/martine,0.7903912734,7.152.82.183,"{""location"": ""SZ"", ""is_mobile"": true}" 19111,7,431,2017-02-14 19:39:29,http://schroeder.co/leilani.kerluke,0.8551824863,21.170.196.238,"{""location"": ""MW"", ""is_mobile"": true}" 19112,7,431,2017-02-16 16:08:25,http://wisozk.info/erwin.cain,0.7891024901,179.161.141.200,"{""location"": ""TW"", ""is_mobile"": true}" 19113,7,431,2017-02-07 02:00:06,http://eichmann.biz/larry_koch,0.0454653556,244.85.8.181,"{""location"": ""BY"", ""is_mobile"": false}" 19114,7,431,2017-01-08 10:12:16,http://stantondonnelly.info/francesco,0.4072434660,187.202.62.225,"{""location"": ""MD"", ""is_mobile"": true}" 19115,7,431,2017-04-19 12:43:21,http://marvin.org/rodolfo_auer,0.3742028492,173.117.243.132,"{""location"": ""KR"", ""is_mobile"": true}" 19116,7,431,2017-01-06 02:45:02,http://kohler.io/delbert,0.1397286866,78.33.207.215,"{""location"": ""BW"", ""is_mobile"": false}" 19117,7,431,2017-01-26 21:21:10,http://cole.org/justine.cronin,0.6660359042,195.47.44.38,"{""location"": ""CC"", ""is_mobile"": true}" 19118,7,431,2017-04-12 05:23:51,http://faheypadberg.info/makenna_bosco,0.2206883711,216.147.109.103,"{""location"": ""HT"", ""is_mobile"": false}" 19119,7,431,2017-05-12 18:40:29,http://bayer.info/carmel.corwin,0.1586184195,238.37.127.172,"{""location"": ""BH"", ""is_mobile"": true}" 19120,7,431,2017-04-18 01:25:25,http://casperreinger.name/marjory.rice,0.1803574113,236.18.182.166,"{""location"": ""KM"", ""is_mobile"": false}" 19121,7,431,2017-05-14 08:01:54,http://carrollpouros.name/arnold_heaney,0.8323397234,150.254.220.44,"{""location"": ""KZ"", ""is_mobile"": false}" 19122,7,431,2017-03-17 22:40:22,http://kochmertz.org/eryn.roberts,0.1509183988,13.143.170.217,"{""location"": ""PS"", ""is_mobile"": false}" 19123,7,431,2017-01-25 05:12:25,http://wehnercorwin.com/nathan.reichert,0.6921706314,170.252.195.140,"{""location"": ""PL"", ""is_mobile"": true}" 19124,7,431,2017-03-04 13:35:17,http://carter.info/gaylord,0.5553901716,249.15.162.216,"{""location"": ""MD"", ""is_mobile"": true}" 19125,7,431,2017-02-21 08:26:08,http://zulaufgerhold.co/arturo,0.6884513535,104.208.102.131,"{""location"": ""PY"", ""is_mobile"": false}" 19126,7,431,2017-05-02 08:54:25,http://fritsch.co/talon.weinat,0.8485629707,28.88.38.188,"{""location"": ""LB"", ""is_mobile"": true}" 19127,7,431,2017-03-23 14:21:56,http://hirthe.org/geovanni,0.5921038372,77.111.54.185,"{""location"": ""GR"", ""is_mobile"": false}" 19128,7,431,2017-03-17 07:50:36,http://stoltenberg.com/leta,0.1019472945,183.211.12.150,"{""location"": ""CY"", ""is_mobile"": false}" 19129,7,431,2017-04-01 06:27:58,http://wittingmacgyver.co/van,0.1563322790,21.86.212.245,"{""location"": ""HM"", ""is_mobile"": false}" 19130,7,431,2017-03-12 10:16:26,http://auergutkowski.com/aliza.connelly,0.6832730403,181.172.13.246,"{""location"": ""GD"", ""is_mobile"": false}" 19131,7,431,2017-03-04 18:41:21,http://schuster.org/neal.doyle,0.3587616591,234.30.239.20,"{""location"": ""BH"", ""is_mobile"": true}" 19132,7,431,2017-01-18 14:15:58,http://huels.org/camden.mann,0.3409766861,236.110.202.22,"{""location"": ""CZ"", ""is_mobile"": true}" 19133,7,431,2017-04-28 16:42:38,http://spinka.biz/yasmin,0.4193828584,183.61.79.211,"{""location"": ""NI"", ""is_mobile"": true}" 19134,7,431,2017-05-12 00:56:45,http://konopelskigorczany.info/jan.oconner,0.2039640904,122.100.171.41,"{""location"": ""CU"", ""is_mobile"": true}" 19135,7,431,2017-02-06 05:56:26,http://simonis.org/devonte,0.4494304194,148.105.206.150,"{""location"": ""TO"", ""is_mobile"": false}" 19136,7,431,2017-01-09 04:41:39,http://barrows.biz/glenda,0.8780483943,203.49.148.39,"{""location"": ""MW"", ""is_mobile"": false}" 19137,7,431,2017-05-14 01:09:26,http://stehrbarrows.co/pinkie.greenholt,0.8539221794,138.12.234.163,"{""location"": ""GQ"", ""is_mobile"": false}" 19138,7,431,2017-03-27 23:39:05,http://fisherheller.name/randy,0.3733252827,111.152.218.206,"{""location"": ""CR"", ""is_mobile"": true}" 19139,7,431,2017-05-16 06:27:16,http://denesik.biz/josefa_hayes,0.0607076716,120.195.12.141,"{""location"": ""CA"", ""is_mobile"": true}" 19140,7,431,2017-05-15 18:38:32,http://daugherty.name/tia,0.8719554768,239.82.225.47,"{""location"": ""YT"", ""is_mobile"": true}" 19141,7,431,2017-04-15 18:17:34,http://heel.com/niko_mitchell,0.1210792052,48.225.165.58,"{""location"": ""CR"", ""is_mobile"": false}" 19142,7,431,2017-04-16 21:53:44,http://botsford.io/suzanne,0.2172621439,95.132.143.219,"{""location"": ""LS"", ""is_mobile"": true}" 19143,7,431,2017-05-27 07:00:55,http://kleinwiza.info/dagmar,0.8043929651,44.99.57.15,"{""location"": ""SM"", ""is_mobile"": false}" 19144,7,431,2017-03-27 03:13:50,http://abernathyabshire.info/rupert.bins,0.2154781606,149.117.86.3,"{""location"": ""ZW"", ""is_mobile"": false}" 19145,7,431,2017-04-05 10:03:14,http://dubuque.com/chanelle.wuckert,0.2815475409,90.252.233.46,"{""location"": ""DE"", ""is_mobile"": true}" 19146,7,431,2017-01-27 12:45:35,http://cristlangworth.co/jordon.lemke,0.4841303465,254.217.68.238,"{""location"": ""LV"", ""is_mobile"": true}" 19147,7,431,2017-06-02 20:45:51,http://weimann.io/tyson,0.6895748453,62.111.49.36,"{""location"": ""AE"", ""is_mobile"": false}" 19148,7,431,2016-12-14 18:08:14,http://wunschziemann.org/ethan,0.5464468088,18.136.59.74,"{""location"": ""KI"", ""is_mobile"": true}" 19149,7,431,2017-03-05 07:52:18,http://batz.co/luna_runte,0.7139212198,243.174.85.147,"{""location"": ""TJ"", ""is_mobile"": false}" 19150,7,431,2017-05-04 18:11:35,http://weimann.net/alba,0.4198401519,216.110.113.224,"{""location"": ""BH"", ""is_mobile"": false}" 19151,7,431,2017-02-09 01:40:18,http://mosciskimante.name/jaeden,0.8418701244,134.201.148.244,"{""location"": ""AZ"", ""is_mobile"": true}" 19152,7,431,2017-02-24 06:02:34,http://weinatfranecki.org/reggie_feil,0.4882981539,243.178.113.38,"{""location"": ""VA"", ""is_mobile"": true}" 19153,7,431,2017-03-24 05:07:15,http://schmittbeatty.info/rodrigo.schamberger,0.1819384088,14.95.77.225,"{""location"": ""NA"", ""is_mobile"": false}" 19154,7,431,2016-12-26 02:31:49,http://brakusmorar.net/leonard.yundt,0.8622472947,14.192.103.252,"{""location"": ""LV"", ""is_mobile"": true}" 19155,7,431,2017-01-03 18:32:32,http://kirlin.com/zola_greenfelder,0.6444492467,105.14.181.56,"{""location"": ""VI"", ""is_mobile"": true}" 19156,7,431,2017-03-02 19:49:55,http://swift.name/johnathan,0.4055303989,234.177.175.30,"{""location"": ""MU"", ""is_mobile"": false}" 19157,7,431,2017-02-06 00:13:53,http://moencartwright.co/roger,0.5322759850,191.75.33.200,"{""location"": ""CO"", ""is_mobile"": false}" 19158,7,431,2017-02-16 01:43:49,http://zieme.org/jamal,0.8898468021,18.54.230.187,"{""location"": ""CU"", ""is_mobile"": true}" 19159,7,431,2017-05-04 07:42:42,http://cremin.com/octavia,0.2456547642,56.205.61.131,"{""location"": ""BY"", ""is_mobile"": true}" 19160,7,431,2017-05-23 03:45:36,http://reichertwolf.com/alan,0.7158791819,209.95.221.246,"{""location"": ""SE"", ""is_mobile"": false}" 19161,7,431,2016-12-13 06:02:37,http://strackesipes.io/enoch,0.2268984988,79.3.41.169,"{""location"": ""AQ"", ""is_mobile"": true}" 19162,7,431,2017-02-24 03:15:43,http://fishermacejkovic.org/laila.rice,0.1532992624,126.174.121.242,"{""location"": ""PL"", ""is_mobile"": true}" 19163,7,431,2016-12-20 23:44:52,http://wardeffertz.org/antonietta.ko,0.9754175816,24.184.195.38,"{""location"": ""TJ"", ""is_mobile"": true}" 19164,7,432,2017-01-30 20:47:55,http://casper.io/jeffrey_ritchie,0.4045322801,151.175.98.81,"{""location"": ""JE"", ""is_mobile"": true}" 19165,7,432,2017-01-23 07:29:25,http://moriette.net/andy_bayer,0.8234106268,54.243.28.217,"{""location"": ""PW"", ""is_mobile"": true}" 19166,7,432,2017-05-22 17:38:12,http://west.com/vaughn,0.9238606911,186.252.178.55,"{""location"": ""IL"", ""is_mobile"": false}" 19167,7,432,2017-01-12 05:13:20,http://schusterkshlerin.org/jermaine.stamm,0.9727139968,12.249.14.170,"{""location"": ""SM"", ""is_mobile"": false}" 19168,7,432,2017-03-17 09:56:54,http://friesenlittel.co/eveline,0.1955769125,40.185.203.158,"{""location"": ""MP"", ""is_mobile"": true}" 19169,7,432,2016-12-20 07:09:07,http://jastwaelchi.io/natalia.smitham,0.9059808610,144.177.58.70,"{""location"": ""LR"", ""is_mobile"": false}" 19170,7,432,2017-01-25 05:53:43,http://bartoletti.org/amy,0.2873068587,227.35.43.206,"{""location"": ""KZ"", ""is_mobile"": false}" 19171,7,432,2017-01-24 10:28:04,http://rolfson.co/lynn_leannon,0.9190223272,87.56.76.116,"{""location"": ""BH"", ""is_mobile"": true}" 19172,7,432,2016-12-17 12:16:30,http://schaden.com/bart_goldner,0.1024777250,165.11.14.35,"{""location"": ""CW"", ""is_mobile"": false}" 19173,7,432,2017-04-26 10:42:36,http://grant.co/chad,0.0725835013,232.11.61.122,"{""location"": ""EG"", ""is_mobile"": false}" 19174,7,432,2017-02-10 18:52:50,http://swaniawskimitchell.com/lea,0.8392268363,161.5.210.53,"{""location"": ""SJ"", ""is_mobile"": true}" 19175,7,432,2017-01-12 01:11:18,http://vonrueden.biz/ruel,0.1289284546,202.150.49.65,"{""location"": ""BW"", ""is_mobile"": false}" 19176,7,432,2016-12-13 14:14:41,http://cartermayer.info/leila_schulist,0.1383196191,36.33.107.148,"{""location"": ""GM"", ""is_mobile"": false}" 19177,7,432,2017-01-05 09:05:06,http://adams.net/emmanuel,0.5296292770,86.33.15.225,"{""location"": ""CR"", ""is_mobile"": true}" 19178,7,432,2017-04-06 17:04:41,http://hagenehanahan.org/frederique,0.3764823938,46.154.183.78,"{""location"": ""SR"", ""is_mobile"": false}" 19179,7,432,2017-01-10 22:32:17,http://hyattrau.io/tamara_turcotte,0.4187300099,30.20.2.25,"{""location"": ""AW"", ""is_mobile"": true}" 19180,7,432,2017-01-24 20:31:40,http://larkin.co/oliver.stamm,0.3061400619,200.238.157.227,"{""location"": ""AW"", ""is_mobile"": false}" 19181,7,432,2017-02-20 16:49:01,http://nienow.net/lysanne_hills,0.0892539447,240.113.157.159,"{""location"": ""KG"", ""is_mobile"": false}" 19182,7,432,2017-02-05 03:49:01,http://whitestrosin.net/kirsten,0.0027793514,25.229.24.82,"{""location"": ""OM"", ""is_mobile"": false}" 19183,7,432,2017-03-31 15:39:24,http://hilllwest.name/saul,0.2284265638,22.104.105.87,"{""location"": ""LR"", ""is_mobile"": false}" 19184,7,432,2017-06-06 09:04:03,http://ruelabbott.org/zakary,0.4771687864,117.29.195.213,"{""location"": ""AX"", ""is_mobile"": true}" 19185,7,432,2017-03-25 23:32:50,http://schillerjaskolski.biz/reese_quitzon,0.7745158636,215.221.122.250,"{""location"": ""NE"", ""is_mobile"": false}" 19186,7,432,2017-02-17 05:42:24,http://wunsch.co/maximillia,0.0332066575,56.53.221.163,"{""location"": ""WF"", ""is_mobile"": false}" 19187,7,432,2017-03-24 00:01:14,http://dickiosinski.biz/mike.mccullough,0.0822002826,52.215.129.4,"{""location"": ""AQ"", ""is_mobile"": false}" 19188,7,432,2017-03-22 01:12:39,http://padberg.com/ophelia,0.9833664308,142.66.58.68,"{""location"": ""CD"", ""is_mobile"": false}" 19189,7,433,2017-05-16 06:06:50,http://hansenkuphal.co/kaylie.gibson,0.9079437401,139.186.243.5,"{""location"": ""OM"", ""is_mobile"": true}" 19190,7,433,2017-05-30 07:58:53,http://leannon.net/alanis.zemlak,0.3020117204,10.188.10.36,"{""location"": ""VI"", ""is_mobile"": false}" 19191,7,433,2017-01-09 17:14:52,http://muller.org/diego.emard,0.3939639216,211.246.35.84,"{""location"": ""LU"", ""is_mobile"": true}" 19192,7,433,2017-01-25 11:50:59,http://hoppe.biz/gretchen,0.6494100546,88.90.39.181,"{""location"": ""FI"", ""is_mobile"": false}" 19193,7,433,2017-05-22 09:55:54,http://pagac.name/carlos,0.8090123365,142.78.210.46,"{""location"": ""GG"", ""is_mobile"": true}" 19194,7,433,2017-05-09 23:19:43,http://weimann.net/clifton,0.5599798674,3.29.213.120,"{""location"": ""SN"", ""is_mobile"": false}" 19195,7,433,2017-02-10 17:56:39,http://franecki.name/karolann.littel,0.8653383353,156.171.48.218,"{""location"": ""GI"", ""is_mobile"": false}" 19196,7,433,2017-02-22 11:26:05,http://markshegmann.net/guiseppe,0.6138858656,102.196.130.26,"{""location"": ""SY"", ""is_mobile"": true}" 19197,7,433,2016-12-29 23:45:00,http://eichmannhahn.io/nina_langosh,0.1479398096,115.221.197.95,"{""location"": ""CY"", ""is_mobile"": true}" 19198,7,433,2017-04-09 19:57:30,http://trantowcrist.biz/elwyn_senger,0.2539247377,108.251.70.102,"{""location"": ""TR"", ""is_mobile"": false}" 19199,7,433,2017-03-07 09:56:17,http://johnsongaylord.org/natasha.keebler,0.0107671904,202.209.52.131,"{""location"": ""PY"", ""is_mobile"": false}" 19200,7,433,2017-02-26 17:59:48,http://ebertbrown.co/raegan_boyle,0.0942366675,251.92.161.64,"{""location"": ""YE"", ""is_mobile"": true}" 19201,7,433,2016-12-13 22:03:51,http://simonis.net/kattie,0.4044438643,67.149.20.47,"{""location"": ""AX"", ""is_mobile"": true}" 19202,7,433,2017-06-08 11:13:43,http://vandervorthilpert.name/granville,0.0606465062,126.13.191.168,"{""location"": ""PK"", ""is_mobile"": false}" 19203,7,433,2017-01-12 15:31:29,http://crona.name/heidi_oconnell,0.9268730679,49.27.9.247,"{""location"": ""AD"", ""is_mobile"": false}" 19204,7,433,2017-05-15 20:52:29,http://shields.com/katherine_gaylord,0.9944317649,67.37.16.120,"{""location"": ""PH"", ""is_mobile"": false}" 19205,7,433,2017-02-26 02:56:50,http://volkman.net/dasia.kautzer,0.4696588505,123.135.4.43,"{""location"": ""AE"", ""is_mobile"": true}" 19206,7,433,2017-05-20 07:27:41,http://darekeebler.info/shawna,0.7481839055,241.236.108.213,"{""location"": ""FM"", ""is_mobile"": false}" 19207,7,433,2017-05-02 16:05:52,http://wintheiserbarton.org/zackery,0.6142528252,182.230.141.79,"{""location"": ""MT"", ""is_mobile"": true}" 19208,7,433,2017-02-21 07:56:11,http://krajcikgerlach.com/shemar.jerde,0.1497658268,252.206.195.127,"{""location"": ""BT"", ""is_mobile"": true}" 19209,7,433,2017-02-19 20:01:18,http://berge.net/robb.terry,0.8908552694,154.237.210.111,"{""location"": ""PW"", ""is_mobile"": true}" 19210,7,433,2017-03-13 05:49:14,http://heidenreich.info/angelo.bahringer,0.4476078453,249.104.225.210,"{""location"": ""IR"", ""is_mobile"": false}" 19211,7,433,2017-05-28 06:41:59,http://stokes.com/madisen,0.5929885231,137.100.239.10,"{""location"": ""BS"", ""is_mobile"": false}" 19212,7,433,2017-04-22 20:35:22,http://becker.io/alaina,0.7865318921,175.155.239.177,"{""location"": ""IL"", ""is_mobile"": false}" 19213,7,433,2016-12-24 23:25:57,http://bashirian.info/savanna,0.2357477522,244.168.87.81,"{""location"": ""ES"", ""is_mobile"": false}" 19214,7,433,2017-05-19 04:55:36,http://schneider.org/vena_purdy,0.7972181944,205.36.198.39,"{""location"": ""NF"", ""is_mobile"": true}" 19215,7,433,2017-04-13 00:32:47,http://king.info/josefina,0.9505073281,76.167.246.28,"{""location"": ""TJ"", ""is_mobile"": false}" 19216,7,433,2016-12-14 07:39:30,http://schaefer.net/alisha.casper,0.3392938867,197.132.77.129,"{""location"": ""KM"", ""is_mobile"": false}" 19217,7,433,2017-05-14 21:26:56,http://rau.io/blaise.balistreri,0.6936777518,57.194.111.78,"{""location"": ""TD"", ""is_mobile"": true}" 19218,7,433,2017-04-19 02:42:16,http://bogisichdare.net/amira_crooks,0.8582723616,206.15.163.222,"{""location"": ""LC"", ""is_mobile"": false}" 19219,7,433,2017-05-31 15:16:14,http://kuvalis.net/lucas,0.1994338655,46.250.105.94,"{""location"": ""PR"", ""is_mobile"": true}" 19220,7,433,2017-04-07 23:08:10,http://shanahan.biz/eve,0.6824431245,227.49.55.115,"{""location"": ""MN"", ""is_mobile"": true}" 19221,7,433,2017-05-12 18:07:56,http://considine.io/ashly,0.0243650170,117.69.16.251,"{""location"": ""SC"", ""is_mobile"": false}" 19222,7,433,2017-01-26 01:39:49,http://torphy.info/pierce_muller,0.4709275418,119.141.207.54,"{""location"": ""HT"", ""is_mobile"": true}" 19223,7,433,2016-12-13 20:07:20,http://halvorson.org/wilfredo,0.9572022714,185.153.2.51,"{""location"": ""ST"", ""is_mobile"": true}" 19224,7,434,2017-01-06 15:19:39,http://okonkutch.com/josiane,0.8565884682,104.218.145.59,"{""location"": ""GH"", ""is_mobile"": false}" 19225,7,434,2017-01-28 08:55:34,http://heathcotelangosh.net/vance_bogisich,0.5571656300,160.8.35.114,"{""location"": ""VG"", ""is_mobile"": true}" 19226,7,434,2017-05-01 10:56:36,http://walker.biz/autumn,0.3868769005,108.123.110.175,"{""location"": ""LR"", ""is_mobile"": false}" 19227,7,434,2017-06-11 19:03:37,http://watsicahermiston.name/faustino,0.5379977392,249.14.146.91,"{""location"": ""PF"", ""is_mobile"": false}" 19228,7,434,2017-03-28 14:35:11,http://carrollwatsica.org/newell_wilkinson,0.4410697945,154.221.11.142,"{""location"": ""BG"", ""is_mobile"": false}" 19229,7,434,2017-01-20 00:00:05,http://koch.io/vincenzo,0.8276739262,174.124.177.98,"{""location"": ""TZ"", ""is_mobile"": false}" 19230,7,434,2017-05-14 13:55:11,http://schowalterjacobson.com/orlo.mitchell,0.6641366790,57.44.73.50,"{""location"": ""AW"", ""is_mobile"": false}" 19231,7,434,2017-04-05 20:47:27,http://ruel.biz/sanford_mcclure,0.9545897655,195.51.198.33,"{""location"": ""KH"", ""is_mobile"": false}" 19232,7,434,2017-02-10 07:15:32,http://reingergoldner.info/rodger,0.3765338583,29.160.222.107,"{""location"": ""IT"", ""is_mobile"": true}" 19233,7,434,2016-12-27 07:21:06,http://sipes.io/shanie,0.4827338070,102.3.171.29,"{""location"": ""SI"", ""is_mobile"": false}" 19234,7,434,2017-04-12 10:22:47,http://kuhlman.co/lavon_balistreri,0.8556530955,214.26.230.77,"{""location"": ""EH"", ""is_mobile"": true}" 19235,7,434,2017-03-11 07:28:45,http://schuppe.net/elody,0.9663019254,108.195.195.142,"{""location"": ""CG"", ""is_mobile"": false}" 19236,7,434,2017-06-05 14:24:32,http://gutkowski.co/terrance,0.6239644716,245.236.163.22,"{""location"": ""KN"", ""is_mobile"": false}" 19237,7,434,2017-06-07 13:53:44,http://collins.io/arjun_reichel,0.1510900045,68.220.209.166,"{""location"": ""KN"", ""is_mobile"": false}" 19238,7,434,2017-04-30 07:24:03,http://grant.io/morgan.harber,0.1592700078,103.181.5.42,"{""location"": ""KP"", ""is_mobile"": false}" 19239,7,434,2017-04-28 05:50:05,http://homenick.biz/edmond_wiegand,0.8797599023,121.187.218.88,"{""location"": ""GS"", ""is_mobile"": false}" 19240,7,434,2017-05-24 21:40:06,http://farrell.io/dorian,0.3008895014,187.89.30.174,"{""location"": ""GT"", ""is_mobile"": false}" 19241,7,434,2017-04-18 09:01:58,http://carroll.io/lina.hansen,0.9279847140,4.128.43.14,"{""location"": ""CH"", ""is_mobile"": false}" 19242,7,434,2017-06-06 01:08:03,http://green.com/emmalee,0.6681584703,58.106.163.58,"{""location"": ""BG"", ""is_mobile"": false}" 19243,7,434,2017-03-11 02:28:06,http://stroman.biz/consuelo.douglas,0.1660971930,57.195.151.187,"{""location"": ""GN"", ""is_mobile"": true}" 19244,7,434,2017-05-20 05:38:23,http://sauermaggio.info/florian_hirthe,0.2260282809,176.250.163.25,"{""location"": ""CU"", ""is_mobile"": true}" 19245,7,434,2017-03-11 01:49:13,http://corkery.name/lloyd_grimes,0.5174716550,4.210.194.174,"{""location"": ""SS"", ""is_mobile"": true}" 19246,7,434,2017-04-23 06:11:07,http://langosh.net/cathrine.streich,0.8375219316,149.16.198.190,"{""location"": ""PE"", ""is_mobile"": true}" 19247,7,434,2017-03-12 18:42:10,http://dubuqueshields.net/sadie_bosco,0.6820788837,39.236.5.194,"{""location"": ""NO"", ""is_mobile"": true}" 19248,7,434,2017-06-09 03:09:23,http://cruickshank.org/joannie.nitzsche,0.6903846519,155.131.239.220,"{""location"": ""GG"", ""is_mobile"": true}" 19249,7,434,2017-04-08 06:53:33,http://prosacco.com/pedro,0.5645434301,122.32.123.143,"{""location"": ""CX"", ""is_mobile"": true}" 19250,7,434,2017-02-10 05:40:34,http://okonhodkiewicz.org/quinton,0.1830904530,122.123.22.224,"{""location"": ""VC"", ""is_mobile"": false}" 19251,7,434,2017-05-06 11:58:17,http://heaney.io/ceasar,0.0704778176,71.93.41.213,"{""location"": ""SK"", ""is_mobile"": true}" 19252,7,434,2017-04-23 07:59:14,http://hodkiewicz.com/abner.deckow,0.7905017055,172.66.222.39,"{""location"": ""HU"", ""is_mobile"": false}" 19253,7,434,2017-01-22 12:27:21,http://sipes.info/joey.carroll,0.5478566736,168.28.127.154,"{""location"": ""PK"", ""is_mobile"": false}" 19254,7,434,2017-03-23 08:18:18,http://lockmankovacek.name/kara_rosenbaum,0.7226226383,155.224.15.45,"{""location"": ""TK"", ""is_mobile"": false}" 19255,7,434,2017-05-10 22:46:27,http://schaefer.name/adela,0.1380021494,196.75.245.58,"{""location"": ""CM"", ""is_mobile"": false}" 19256,7,434,2017-03-07 03:02:48,http://kilbackreichert.biz/bonnie.roberts,0.8039962552,175.35.155.227,"{""location"": ""GI"", ""is_mobile"": false}" 19257,7,434,2017-04-27 13:00:25,http://walter.net/nick,0.6122206709,21.163.189.20,"{""location"": ""CA"", ""is_mobile"": false}" 19258,7,434,2017-02-21 14:26:44,http://dare.com/abner.stokes,0.1158551086,55.102.4.64,"{""location"": ""UZ"", ""is_mobile"": true}" 19259,7,434,2017-05-03 10:39:41,http://hamillspinka.net/obie,0.5449599243,53.146.134.200,"{""location"": ""RO"", ""is_mobile"": true}" 19260,7,434,2016-12-30 17:14:46,http://krajcik.io/adella_kuhn,0.9495532738,221.109.94.192,"{""location"": ""PF"", ""is_mobile"": false}" 19261,7,434,2017-03-30 04:37:53,http://hagenes.net/keanu_kirlin,0.7554506255,249.47.242.191,"{""location"": ""TD"", ""is_mobile"": true}" 19262,7,434,2017-04-07 09:28:57,http://mccullough.net/antonia.macgyver,0.1738503457,253.150.207.39,"{""location"": ""BN"", ""is_mobile"": true}" 19263,7,434,2017-01-31 01:17:47,http://sauerklocko.com/queen_schaden,0.5719006891,120.171.34.47,"{""location"": ""FR"", ""is_mobile"": false}" 19264,7,435,2017-04-09 14:29:21,http://walter.info/reese_thompson,0.4965963551,49.151.33.126,"{""location"": ""CG"", ""is_mobile"": false}" 19265,7,435,2017-01-25 08:06:25,http://macejkovicwilliamson.co/dallas,0.4803641462,112.165.219.131,"{""location"": ""LA"", ""is_mobile"": true}" 19266,7,435,2017-03-27 19:37:43,http://moen.io/mustafa.cummerata,0.9354750196,165.77.205.33,"{""location"": ""MT"", ""is_mobile"": false}" 19267,7,435,2017-05-04 14:16:05,http://murazik.io/hellen_abernathy,0.5692468821,191.185.170.242,"{""location"": ""VN"", ""is_mobile"": false}" 19268,7,435,2017-05-17 15:07:05,http://cremin.co/jade,0.7448666428,180.36.64.165,"{""location"": ""TT"", ""is_mobile"": true}" 19269,7,435,2017-01-09 16:26:14,http://cartwright.io/helga_kunde,0.6508672991,186.112.89.94,"{""location"": ""CO"", ""is_mobile"": true}" 19270,7,435,2016-12-27 04:04:23,http://heidenreich.info/kris,0.6726196979,227.38.186.16,"{""location"": ""BQ"", ""is_mobile"": true}" 19271,7,435,2017-04-14 01:57:13,http://gusikowski.name/lizzie,0.5830112063,214.197.175.198,"{""location"": ""PY"", ""is_mobile"": false}" 19272,7,435,2017-03-01 16:14:55,http://crist.net/angeline.wintheiser,0.1759480298,3.254.189.172,"{""location"": ""PT"", ""is_mobile"": false}" 19273,7,435,2016-12-30 17:54:25,http://harveytremblay.info/brandyn,0.9082858454,165.252.23.222,"{""location"": ""MK"", ""is_mobile"": false}" 19274,7,435,2017-06-04 18:35:43,http://legrosaltenwerth.net/cedrick,0.0909172437,22.67.209.65,"{""location"": ""KM"", ""is_mobile"": false}" 19275,7,435,2017-04-11 17:59:58,http://heller.biz/maribel_mayer,0.1961691658,95.29.206.232,"{""location"": ""VA"", ""is_mobile"": true}" 19276,7,435,2017-03-04 06:42:53,http://franecki.com/columbus,0.4550987455,50.102.213.67,"{""location"": ""FK"", ""is_mobile"": true}" 19277,7,435,2017-05-07 18:20:25,http://ratke.com/jared,0.3480881229,86.94.241.184,"{""location"": ""NG"", ""is_mobile"": false}" 19278,7,435,2017-05-13 18:18:09,http://beahanglover.net/natalia.baumbach,0.3675836265,202.29.190.21,"{""location"": ""GF"", ""is_mobile"": true}" 19279,7,435,2017-04-21 08:51:48,http://barrows.info/tad,0.4877750758,253.46.166.250,"{""location"": ""DJ"", ""is_mobile"": true}" 19280,7,435,2017-03-02 06:10:50,http://conroy.io/cedrick_turcotte,0.3670381089,159.221.163.231,"{""location"": ""MT"", ""is_mobile"": true}" 19281,7,435,2017-02-14 11:22:57,http://bergstrom.name/jordyn,0.2270984078,116.109.192.29,"{""location"": ""CK"", ""is_mobile"": false}" 19282,7,435,2017-05-17 03:39:27,http://ratkelesch.co/leatha,0.2907802434,222.15.9.50,"{""location"": ""BJ"", ""is_mobile"": true}" 19283,7,435,2017-02-19 18:50:36,http://kulas.biz/colleen_graham,0.6693754259,240.76.96.85,"{""location"": ""MX"", ""is_mobile"": true}" 19284,7,435,2016-12-29 08:23:29,http://denesik.net/filiberto.glover,0.4163556521,197.245.144.138,"{""location"": ""SY"", ""is_mobile"": true}" 19285,7,435,2017-05-30 12:11:07,http://rau.co/laila,0.1930928099,54.153.171.110,"{""location"": ""WS"", ""is_mobile"": false}" 19286,7,435,2016-12-25 11:25:02,http://mosciski.io/jensen.schuster,0.5098755882,157.141.85.23,"{""location"": ""KN"", ""is_mobile"": true}" 19287,7,435,2017-01-17 02:11:52,http://lynchschulist.info/enoch,0.4748075190,219.35.148.101,"{""location"": ""CW"", ""is_mobile"": true}" 19288,7,435,2017-02-22 09:27:23,http://hammes.com/adrien_white,0.6891645214,2.229.177.214,"{""location"": ""US"", ""is_mobile"": true}" 19289,7,435,2017-01-12 21:09:50,http://swaniawskimarquardt.net/stanley_beier,0.5144745311,7.42.60.208,"{""location"": ""TZ"", ""is_mobile"": false}" 19290,7,435,2017-04-12 09:50:19,http://jerde.net/sabrina.schroeder,0.9677031988,166.131.210.98,"{""location"": ""LU"", ""is_mobile"": false}" 19291,7,435,2017-02-20 01:56:27,http://fritschzemlak.net/stone,0.9140191148,210.45.92.156,"{""location"": ""EH"", ""is_mobile"": false}" 19292,7,435,2017-04-21 06:59:22,http://marquardt.org/nora.pfannerstill,0.6703550596,163.161.44.146,"{""location"": ""JO"", ""is_mobile"": true}" 19293,7,435,2017-01-23 18:47:17,http://ko.org/quinten,0.0727213803,37.116.231.221,"{""location"": ""GN"", ""is_mobile"": false}" 19294,7,435,2017-03-01 19:16:25,http://predovic.com/prince_marquardt,0.0773468436,227.230.42.124,"{""location"": ""LB"", ""is_mobile"": false}" 19295,7,435,2017-02-02 06:16:05,http://mcculloughmurray.name/paige_collins,0.0874399151,156.26.183.241,"{""location"": ""NR"", ""is_mobile"": true}" 19296,7,435,2017-01-20 16:47:48,http://ryan.com/rosalyn_bartoletti,0.9231210116,198.230.213.115,"{""location"": ""PE"", ""is_mobile"": false}" 19297,7,435,2017-04-14 13:19:03,http://aufderhar.io/madalyn,0.9360343407,241.212.190.57,"{""location"": ""PW"", ""is_mobile"": true}" 19298,7,435,2017-04-18 01:33:08,http://block.co/marilyne.wisoky,0.3282297005,239.36.105.170,"{""location"": ""AI"", ""is_mobile"": false}" 19299,7,435,2017-05-06 04:52:24,http://little.info/pearline.bradtke,0.1072166889,112.96.95.115,"{""location"": ""KE"", ""is_mobile"": true}" 19300,7,435,2017-03-29 06:31:37,http://aufderhar.net/emmanuelle,0.9191894886,58.58.220.24,"{""location"": ""BJ"", ""is_mobile"": false}" 19301,7,435,2017-05-27 08:48:10,http://cartwrightratke.name/sadie_haag,0.9960533779,7.145.18.113,"{""location"": ""KN"", ""is_mobile"": false}" 19302,7,435,2016-12-14 03:05:56,http://mcdermott.name/clinton,0.5608125955,235.195.250.23,"{""location"": ""CO"", ""is_mobile"": false}" 19303,7,436,2017-01-07 13:37:00,http://ruecker.name/orpha,0.6990218684,106.246.64.20,"{""location"": ""BR"", ""is_mobile"": true}" 19304,7,436,2017-02-02 02:40:36,http://purdy.com/jimmie.herzog,0.6765526842,45.186.90.149,"{""location"": ""CZ"", ""is_mobile"": false}" 19305,7,436,2017-05-23 18:03:55,http://erdman.net/willie,0.0955192204,113.207.28.27,"{""location"": ""SX"", ""is_mobile"": false}" 19306,7,436,2017-06-04 12:58:56,http://bogan.co/leonora.stehr,0.6013070568,105.177.143.110,"{""location"": ""SE"", ""is_mobile"": true}" 19307,7,436,2017-01-22 09:23:19,http://brown.info/elmira.reilly,0.2431304430,174.3.42.2,"{""location"": ""CR"", ""is_mobile"": false}" 19308,7,436,2016-12-14 15:53:43,http://walter.net/bernard,0.2286594614,24.224.171.177,"{""location"": ""HM"", ""is_mobile"": true}" 19309,7,436,2016-12-19 17:28:42,http://wolfdibbert.name/maribel_daniel,0.7459146876,91.179.101.155,"{""location"": ""NI"", ""is_mobile"": false}" 19310,7,436,2017-03-08 23:44:31,http://rennerfarrell.net/brain,0.0350114650,110.223.82.214,"{""location"": ""TL"", ""is_mobile"": true}" 19311,7,436,2017-01-07 23:38:59,http://koelpin.co/marshall,0.3575798078,80.39.165.155,"{""location"": ""NE"", ""is_mobile"": false}" 19312,7,436,2017-05-07 11:56:00,http://stanton.info/dashawn,0.0502439885,237.229.86.79,"{""location"": ""BN"", ""is_mobile"": false}" 19313,7,436,2017-02-27 14:25:59,http://hills.org/graciela_schulist,0.4561248728,218.229.64.61,"{""location"": ""HU"", ""is_mobile"": true}" 19314,7,436,2017-06-11 06:20:57,http://wunsch.org/hertha_keeling,0.5347957004,130.177.21.137,"{""location"": ""UA"", ""is_mobile"": false}" 19315,7,436,2017-01-02 14:03:02,http://homenick.biz/crystel.schiller,0.8087231529,128.224.117.20,"{""location"": ""HT"", ""is_mobile"": true}" 19316,7,436,2017-02-28 20:22:33,http://yost.name/golden_marvin,0.5151054377,123.100.87.22,"{""location"": ""CN"", ""is_mobile"": false}" 19317,7,436,2017-04-02 15:23:31,http://bradtke.name/valentina,0.8871129816,203.146.109.99,"{""location"": ""AU"", ""is_mobile"": false}" 19318,7,436,2017-01-25 14:58:24,http://keebler.info/floie_daugherty,0.7808908171,10.220.76.22,"{""location"": ""KY"", ""is_mobile"": false}" 19319,7,436,2017-06-05 19:47:42,http://hansen.biz/libby.reinger,0.6786396099,59.19.248.161,"{""location"": ""WS"", ""is_mobile"": true}" 19320,7,436,2017-04-29 16:43:23,http://mann.com/bernita.bahringer,0.1913744451,209.51.250.156,"{""location"": ""KE"", ""is_mobile"": true}" 19321,7,436,2017-02-26 16:23:38,http://carroll.com/alanna.glover,0.1337063317,127.111.194.47,"{""location"": ""EE"", ""is_mobile"": true}" 19322,7,436,2017-03-15 10:26:52,http://goyette.info/collin,0.5250296423,246.82.34.151,"{""location"": ""BV"", ""is_mobile"": true}" 19323,7,436,2017-04-09 13:54:20,http://jenkinsmraz.com/jovany_collier,0.6674125454,53.179.110.42,"{""location"": ""CN"", ""is_mobile"": false}" 19324,7,436,2017-05-01 12:19:44,http://wisokyhackett.org/emilio,0.3259684569,58.251.224.145,"{""location"": ""EE"", ""is_mobile"": true}" 19325,7,436,2017-05-15 04:03:03,http://keeblerkuphal.io/lane_bogan,0.8175580611,211.33.161.109,"{""location"": ""PL"", ""is_mobile"": false}" 19326,7,436,2017-04-06 06:00:32,http://grimes.info/loyal_larson,0.7758494330,105.230.241.55,"{""location"": ""MA"", ""is_mobile"": false}" 19327,7,436,2017-05-10 17:12:00,http://lueilwitz.info/brenda_jakubowski,0.6435306764,165.207.32.203,"{""location"": ""LK"", ""is_mobile"": true}" 19328,7,436,2016-12-14 20:20:09,http://metz.org/celestino,0.6914839289,235.12.17.58,"{""location"": ""SC"", ""is_mobile"": true}" 19329,7,436,2017-05-25 23:00:46,http://mann.biz/david,0.6570238409,174.19.11.24,"{""location"": ""SK"", ""is_mobile"": false}" 19330,7,436,2017-03-13 17:02:53,http://hodkiewicz.info/ruthe_beer,0.8292433085,145.216.108.206,"{""location"": ""IT"", ""is_mobile"": true}" 19331,7,436,2017-05-24 02:21:59,http://carroll.com/kenny.beer,0.4352508641,210.246.8.120,"{""location"": ""CX"", ""is_mobile"": false}" 19332,7,436,2017-01-05 20:39:10,http://jones.biz/johann.champlin,0.5995534289,195.229.246.38,"{""location"": ""GR"", ""is_mobile"": true}" 19333,7,436,2017-02-13 19:55:21,http://darehermann.io/columbus,0.2966273569,245.232.128.179,"{""location"": ""RE"", ""is_mobile"": true}" 19334,7,436,2017-02-22 10:42:05,http://shields.info/isadore,0.3385859833,230.88.120.176,"{""location"": ""QA"", ""is_mobile"": false}" 19335,7,436,2017-01-04 08:11:36,http://gerlach.com/meagan.lowe,0.2321268989,168.184.142.149,"{""location"": ""LK"", ""is_mobile"": true}" 19336,7,436,2017-01-03 09:07:17,http://kiehngleichner.co/dane,0.8480267611,220.130.14.227,"{""location"": ""MG"", ""is_mobile"": true}" 19337,7,436,2017-03-09 18:42:18,http://yostmacgyver.co/elfrieda_kertzmann,0.5499228091,166.5.53.87,"{""location"": ""HM"", ""is_mobile"": false}" 19338,7,436,2016-12-13 23:32:15,http://bartoletti.org/ashlee_moriette,0.3675226286,113.132.46.237,"{""location"": ""VU"", ""is_mobile"": true}" 19339,7,436,2017-06-02 01:57:51,http://becker.co/clifford,0.8985787049,105.236.159.2,"{""location"": ""NC"", ""is_mobile"": true}" 19340,7,436,2016-12-17 09:06:24,http://vonrueden.name/urban,0.3226290555,21.183.127.105,"{""location"": ""DE"", ""is_mobile"": true}" 19341,7,436,2017-06-03 16:46:47,http://wymanmccullough.org/gilbert,0.0879206027,95.175.124.218,"{""location"": ""JE"", ""is_mobile"": false}" 19342,7,436,2017-02-23 22:40:18,http://uptonhalvorson.co/edwin.beer,0.7628928850,35.231.24.226,"{""location"": ""TC"", ""is_mobile"": false}" 19343,7,436,2017-01-09 06:54:51,http://schmidtwisozk.co/toney,0.5852921166,187.165.210.46,"{""location"": ""PW"", ""is_mobile"": false}" 19344,7,436,2017-02-11 22:30:18,http://rath.io/nora.koch,0.0858814639,105.183.154.154,"{""location"": ""BH"", ""is_mobile"": true}" 19345,7,436,2017-04-06 18:10:19,http://trantow.co/darren.steuber,0.8329950757,19.160.116.42,"{""location"": ""ES"", ""is_mobile"": true}" 19346,7,436,2017-05-29 22:09:27,http://murrayjerde.co/jasmin_stracke,0.3303443960,105.84.187.101,"{""location"": ""MN"", ""is_mobile"": true}" 19347,7,436,2016-12-21 16:45:41,http://pricestokes.net/gerald_rice,0.5678984913,213.134.254.166,"{""location"": ""MN"", ""is_mobile"": false}" 19348,7,436,2017-01-13 01:03:52,http://rempelthompson.org/wiley,0.6860213068,60.106.41.233,"{""location"": ""BY"", ""is_mobile"": true}" 19349,7,436,2017-05-10 12:58:30,http://gaylord.net/margarette,0.0951653473,69.85.43.183,"{""location"": ""LT"", ""is_mobile"": true}" 19350,7,436,2016-12-30 07:11:27,http://howellrolfson.com/willis_rippin,0.5737650881,27.225.196.109,"{""location"": ""SS"", ""is_mobile"": true}" 19351,7,436,2017-06-13 11:04:19,http://veum.co/karine_mohr,0.0831646532,192.189.199.144,"{""location"": ""LB"", ""is_mobile"": false}" 19352,7,436,2017-05-05 11:22:30,http://rathwilkinson.co/miracle,0.5073178608,217.75.20.235,"{""location"": ""GQ"", ""is_mobile"": true}" 19353,7,436,2017-04-01 00:26:36,http://schroederpouros.info/novella,0.6466877717,42.49.31.165,"{""location"": ""BJ"", ""is_mobile"": true}" 19354,7,436,2017-01-01 14:54:46,http://nitzsche.net/madelyn,0.2177615274,141.16.106.243,"{""location"": ""ST"", ""is_mobile"": true}" 19355,7,436,2017-01-04 01:29:07,http://emmerich.biz/melvin,0.5584486911,203.164.26.233,"{""location"": ""RW"", ""is_mobile"": true}" 19356,7,436,2017-05-01 13:27:03,http://connellytreutel.co/royce.orn,0.7556217996,59.10.183.83,"{""location"": ""DE"", ""is_mobile"": false}" 19357,7,436,2017-05-02 07:24:15,http://quigley.io/olin.gislason,0.5077153490,48.59.88.60,"{""location"": ""TN"", ""is_mobile"": false}" 19358,7,436,2017-02-04 14:53:16,http://larsonortiz.com/camren,0.1847943662,236.74.52.230,"{""location"": ""MN"", ""is_mobile"": false}" 19359,7,436,2017-03-20 21:42:52,http://dibbertmitchell.io/roderick,0.2587050212,194.231.3.133,"{""location"": ""KW"", ""is_mobile"": true}" 19360,7,436,2017-03-22 23:43:18,http://pacochaharris.net/moriah,0.9420208811,33.77.212.197,"{""location"": ""EG"", ""is_mobile"": true}" 19361,7,437,2017-04-28 03:38:49,http://marquardt.info/otto,0.2936159126,3.137.121.179,"{""location"": ""TM"", ""is_mobile"": false}" 19362,7,437,2017-05-25 16:27:17,http://kuhlman.com/jose_goyette,0.0048084671,88.173.33.78,"{""location"": ""TD"", ""is_mobile"": true}" 19363,7,437,2017-05-25 23:02:26,http://rau.biz/arlene,0.5158672359,115.22.191.238,"{""location"": ""KP"", ""is_mobile"": false}" 19364,7,437,2017-03-12 09:52:22,http://strosinwatsica.co/modesto,0.8330214512,103.119.38.227,"{""location"": ""VA"", ""is_mobile"": false}" 19365,7,437,2017-04-27 20:57:21,http://osinski.name/kyra_morar,0.2583135208,235.33.238.110,"{""location"": ""GR"", ""is_mobile"": true}" 19366,7,437,2017-02-25 15:34:28,http://botsfordborer.com/mattie,0.1965058013,211.221.196.131,"{""location"": ""NO"", ""is_mobile"": true}" 19367,7,437,2016-12-20 19:21:12,http://morar.com/kevon,0.0740035610,104.80.156.203,"{""location"": ""FK"", ""is_mobile"": true}" 19368,7,437,2017-02-17 00:13:55,http://sanfordmonahan.biz/nyah,0.8815004721,94.116.14.178,"{""location"": ""CC"", ""is_mobile"": false}" 19369,7,437,2017-04-15 20:32:59,http://denesikmoen.biz/kathleen.doyle,0.9519569775,19.161.10.119,"{""location"": ""VC"", ""is_mobile"": false}" 19370,7,437,2017-01-19 10:33:41,http://fisherspencer.biz/meggie,0.9657555277,221.116.235.82,"{""location"": ""SJ"", ""is_mobile"": true}" 19371,7,437,2017-02-19 23:44:56,http://simonisroob.net/adolfo_connelly,0.1359376379,231.90.33.239,"{""location"": ""CV"", ""is_mobile"": false}" 19372,7,437,2017-06-11 15:17:29,http://satterfieldmueller.info/coty,0.9154371281,240.158.206.127,"{""location"": ""VN"", ""is_mobile"": true}" 19373,7,437,2017-05-04 23:54:46,http://schowalter.biz/monroe,0.5181530278,217.248.57.88,"{""location"": ""SS"", ""is_mobile"": false}" 19374,7,437,2017-04-05 14:33:17,http://mohr.info/eula,0.2477125017,151.237.239.182,"{""location"": ""MK"", ""is_mobile"": true}" 19375,7,437,2017-06-02 22:26:40,http://cummerata.info/reva.simonis,0.9880516776,10.120.80.101,"{""location"": ""TZ"", ""is_mobile"": true}" 19376,7,437,2017-03-30 15:47:14,http://keeling.name/daryl,0.8694148970,174.223.128.77,"{""location"": ""AX"", ""is_mobile"": false}" 19377,7,437,2017-03-12 05:16:34,http://oharawyman.io/oscar,0.5159772360,145.206.124.176,"{""location"": ""US"", ""is_mobile"": false}" 19378,7,437,2017-05-10 08:48:45,http://beahan.info/doyle_jones,0.3607761390,5.153.111.189,"{""location"": ""TG"", ""is_mobile"": true}" 19379,7,437,2017-02-01 08:30:26,http://hane.co/ruby_spinka,0.3293592382,236.27.22.16,"{""location"": ""WS"", ""is_mobile"": true}" 19380,7,437,2017-05-27 06:53:02,http://lubowitznader.com/gielle_hirthe,0.0313268506,229.149.138.194,"{""location"": ""MF"", ""is_mobile"": false}" 19381,7,437,2017-04-17 07:42:22,http://robertsfisher.name/ottilie,0.9291701201,199.162.28.63,"{""location"": ""GD"", ""is_mobile"": false}" 19382,7,437,2017-01-27 06:43:02,http://koelpinbosco.biz/nova_grimes,0.5238402803,70.56.135.157,"{""location"": ""HT"", ""is_mobile"": true}" 19383,7,437,2017-04-22 09:14:30,http://boscoklocko.com/roxane_mayert,0.4152496551,9.26.149.222,"{""location"": ""MC"", ""is_mobile"": true}" 19384,7,437,2017-03-26 17:49:45,http://langworthrunolfsdottir.net/ben.leffler,0.3558803294,16.170.218.210,"{""location"": ""BT"", ""is_mobile"": false}" 19385,7,437,2017-05-03 02:12:53,http://jacobi.info/rhea_predovic,0.8209920123,236.94.2.220,"{""location"": ""AO"", ""is_mobile"": false}" 19386,7,437,2017-01-14 20:24:50,http://lesch.biz/jaylen,0.2552621670,179.215.126.81,"{""location"": ""PY"", ""is_mobile"": false}" 19387,7,437,2017-06-09 10:22:34,http://okon.co/rudolph,0.2993179172,74.16.207.43,"{""location"": ""JO"", ""is_mobile"": false}" 19388,7,437,2017-01-07 19:16:24,http://berge.info/reginald_hagenes,0.3487480234,18.19.143.129,"{""location"": ""AI"", ""is_mobile"": false}" 19389,7,437,2017-02-07 02:08:07,http://beckerkreiger.net/karley,0.3098333871,29.6.112.247,"{""location"": ""CM"", ""is_mobile"": false}" 19390,7,437,2017-01-19 17:32:40,http://blanda.io/daren_baumbach,0.0820974895,20.149.196.234,"{""location"": ""LI"", ""is_mobile"": true}" 19391,7,437,2017-05-20 03:11:45,http://hamill.biz/alda.kreiger,0.7492107205,239.116.124.176,"{""location"": ""BJ"", ""is_mobile"": false}" 19392,7,437,2016-12-23 07:02:24,http://batzhettinger.net/grady.wiza,0.8943490896,59.244.27.48,"{""location"": ""NI"", ""is_mobile"": false}" 19393,7,437,2017-04-19 22:34:57,http://heller.org/ali_schumm,0.2030994990,194.160.23.87,"{""location"": ""VC"", ""is_mobile"": false}" 19394,7,437,2017-03-23 20:13:54,http://moen.info/consuelo.mcclure,0.4590498233,170.151.7.136,"{""location"": ""AZ"", ""is_mobile"": false}" 19395,7,437,2017-04-15 18:17:17,http://grady.com/randy,0.4593683755,250.136.22.216,"{""location"": ""PL"", ""is_mobile"": true}" 19396,7,437,2017-03-07 21:40:01,http://bergstrom.org/zaria,0.1565026894,196.232.118.81,"{""location"": ""SJ"", ""is_mobile"": true}" 19397,7,437,2017-01-20 15:42:52,http://lynch.org/sheldon_king,0.3833346603,102.75.217.148,"{""location"": ""GL"", ""is_mobile"": false}" 19398,7,437,2017-03-11 21:59:19,http://dietrichdare.info/georgette,0.1162860017,246.210.228.108,"{""location"": ""NG"", ""is_mobile"": false}" 19399,7,437,2017-01-19 23:14:18,http://muellerraynor.net/stone_rowe,0.2167310549,236.68.239.139,"{""location"": ""LA"", ""is_mobile"": true}" 19400,7,437,2017-05-19 02:39:18,http://aufderharschultz.biz/scot_hilpert,0.5149431669,248.79.204.144,"{""location"": ""RS"", ""is_mobile"": true}" 19401,7,437,2017-02-07 10:00:17,http://kiehn.org/kamren.kutch,0.9408925295,229.197.193.10,"{""location"": ""GW"", ""is_mobile"": false}" 19402,7,437,2017-02-27 10:21:34,http://bernhardorn.org/freeman.lesch,0.3392729540,37.175.171.88,"{""location"": ""US"", ""is_mobile"": true}" 19403,7,437,2017-03-19 10:59:04,http://huels.io/buford_macejkovic,0.9544016454,68.213.234.215,"{""location"": ""AF"", ""is_mobile"": false}" 19404,7,437,2017-04-17 11:34:06,http://lindgrenstrosin.io/lavina.osinski,0.8976437013,236.25.188.14,"{""location"": ""BS"", ""is_mobile"": true}" 19405,7,437,2017-05-04 00:44:27,http://tremblay.name/blaze_lehner,0.2987974260,58.99.218.15,"{""location"": ""ES"", ""is_mobile"": false}" 19406,7,437,2017-05-27 19:05:13,http://schmeler.biz/maryse,0.6587963510,123.123.196.223,"{""location"": ""YT"", ""is_mobile"": false}" 19407,7,437,2017-04-15 07:45:12,http://gerhold.com/antwan,0.1041400434,152.149.128.241,"{""location"": ""TV"", ""is_mobile"": true}" 19408,7,437,2017-04-23 18:57:22,http://gutkowski.info/estella,0.8929698010,79.181.213.216,"{""location"": ""ZA"", ""is_mobile"": true}" 19409,7,437,2017-03-13 20:37:07,http://hand.org/lizeth,0.6058382266,178.163.71.95,"{""location"": ""GU"", ""is_mobile"": false}" 19410,7,437,2017-04-25 18:32:01,http://lindyost.co/lelah,0.2661820340,153.8.39.88,"{""location"": ""GQ"", ""is_mobile"": true}" 19411,7,437,2017-03-18 23:42:26,http://trantow.net/ruthe_jast,0.0563644436,186.53.199.19,"{""location"": ""GA"", ""is_mobile"": true}" 19412,7,437,2017-03-09 06:46:50,http://gleasonzulauf.info/devan.daniel,0.7163560516,32.227.27.66,"{""location"": ""TC"", ""is_mobile"": true}" 19413,7,437,2017-01-25 01:12:05,http://crooksherman.com/chyna_oreilly,0.0627577167,72.204.192.79,"{""location"": ""BD"", ""is_mobile"": true}" 19414,7,437,2017-01-31 06:30:57,http://ortizhyatt.com/kiarra.blanda,0.3858808407,208.145.13.87,"{""location"": ""BQ"", ""is_mobile"": true}" 19415,7,437,2017-04-28 00:35:52,http://prosaccocartwright.org/lewis,0.3880068274,2.170.99.179,"{""location"": ""SM"", ""is_mobile"": false}" 19416,7,437,2017-05-06 18:29:25,http://quitzon.net/linda.collins,0.9945611200,198.121.127.176,"{""location"": ""LI"", ""is_mobile"": true}" 19417,7,437,2017-01-21 05:11:39,http://bashirianschuppe.info/august,0.5354954041,90.7.94.184,"{""location"": ""CW"", ""is_mobile"": true}" 19418,7,437,2017-05-17 06:26:05,http://blick.info/salma_windler,0.1236777061,154.145.53.191,"{""location"": ""ME"", ""is_mobile"": false}" 19419,7,437,2017-03-16 05:48:46,http://denesik.com/samantha.ohara,0.3660451833,111.196.216.154,"{""location"": ""AF"", ""is_mobile"": true}" 19420,7,437,2017-02-26 21:13:03,http://bernhard.co/chauncey_jenkins,0.0891780007,116.36.95.19,"{""location"": ""KH"", ""is_mobile"": false}" 19421,7,437,2017-03-11 10:34:49,http://dickinsonrobel.biz/maximillia.harvey,0.4873245084,43.117.12.97,"{""location"": ""IQ"", ""is_mobile"": true}" 19422,7,437,2017-03-18 15:48:30,http://ondrickastracke.biz/sven.marvin,0.7314159649,245.77.116.195,"{""location"": ""UA"", ""is_mobile"": false}" 19423,7,437,2017-05-13 13:22:15,http://connmoen.biz/edna.konopelski,0.0478780947,93.143.158.41,"{""location"": ""TH"", ""is_mobile"": false}" 19424,7,437,2017-05-28 01:07:00,http://johns.org/marquise.paucek,0.8259093264,208.52.50.79,"{""location"": ""MT"", ""is_mobile"": false}" 19425,7,437,2017-05-02 17:55:17,http://heelblanda.io/hanna_schimmel,0.9426320087,183.54.203.219,"{""location"": ""ID"", ""is_mobile"": false}" 19426,7,437,2017-03-27 18:16:01,http://trompmacejkovic.com/verla,0.3163437541,13.92.48.158,"{""location"": ""BL"", ""is_mobile"": false}" 19427,7,437,2016-12-23 02:58:12,http://wiegand.name/justyn,0.0878461472,153.40.11.14,"{""location"": ""GH"", ""is_mobile"": false}" 19428,7,438,2017-04-21 02:13:15,http://beer.org/elvis_dach,0.3264541887,195.169.57.38,"{""location"": ""NU"", ""is_mobile"": true}" 19429,7,438,2017-04-23 21:54:42,http://quitzon.info/mariana_kuhlman,0.3543945987,67.144.230.247,"{""location"": ""GE"", ""is_mobile"": true}" 19430,7,438,2017-02-18 01:01:16,http://wehner.name/otha.witting,0.8178275395,210.187.92.59,"{""location"": ""CU"", ""is_mobile"": true}" 19431,7,438,2017-03-01 23:14:35,http://jones.biz/guie.kulas,0.7979304972,158.230.239.108,"{""location"": ""CM"", ""is_mobile"": false}" 19432,7,438,2017-04-11 12:30:54,http://treutel.io/nikolas,0.1631620813,201.121.119.140,"{""location"": ""GN"", ""is_mobile"": false}" 19433,7,438,2017-04-16 10:34:15,http://johns.org/mohammed_nienow,0.3031753247,122.16.230.190,"{""location"": ""SK"", ""is_mobile"": true}" 19434,7,438,2017-03-19 10:09:42,http://yost.biz/allie.williamson,0.1287658352,85.99.120.205,"{""location"": ""VI"", ""is_mobile"": true}" 19435,7,438,2017-06-05 23:41:13,http://mertz.co/karina,0.9093141757,144.167.76.177,"{""location"": ""MR"", ""is_mobile"": false}" 19436,7,438,2017-04-10 11:22:42,http://renner.net/tamia,0.8508010910,87.45.34.144,"{""location"": ""SZ"", ""is_mobile"": false}" 19437,7,438,2017-02-10 05:47:20,http://rempelhackett.biz/calista,0.7166554685,115.103.169.180,"{""location"": ""YE"", ""is_mobile"": true}" 19438,7,438,2017-06-08 22:11:30,http://turner.name/kristina.ledner,0.8299925150,143.42.238.212,"{""location"": ""AL"", ""is_mobile"": true}" 19439,7,438,2016-12-24 19:34:16,http://heidenreichhermann.com/kendrick,0.3334206266,164.110.19.197,"{""location"": ""RW"", ""is_mobile"": true}" 19440,7,438,2017-01-19 04:58:20,http://jenkintehr.io/ulices,0.0355354851,61.167.7.92,"{""location"": ""VU"", ""is_mobile"": true}" 19441,7,438,2016-12-19 01:06:37,http://durgan.org/verona,0.0684624037,213.137.84.21,"{""location"": ""LY"", ""is_mobile"": false}" 19442,7,438,2017-05-10 03:44:31,http://bailey.biz/susie,0.6365195473,210.182.184.248,"{""location"": ""PA"", ""is_mobile"": true}" 19443,7,438,2017-05-22 19:34:05,http://labadie.net/louie_ferry,0.5849884927,33.172.185.153,"{""location"": ""CO"", ""is_mobile"": false}" 19444,7,438,2017-03-26 12:32:36,http://reichert.com/royal.terry,0.6990075739,34.137.24.160,"{""location"": ""NZ"", ""is_mobile"": true}" 19445,7,438,2016-12-23 08:35:52,http://moriettehalvorson.co/vincent_hammes,0.2804372099,3.88.14.95,"{""location"": ""KY"", ""is_mobile"": false}" 19446,7,438,2017-04-05 11:34:23,http://towne.biz/kayden_becker,0.6520673948,56.193.141.120,"{""location"": ""BT"", ""is_mobile"": true}" 19447,7,438,2017-01-02 07:18:09,http://halvorsonlarson.net/alexandrine_jenkins,0.2086088486,113.210.153.54,"{""location"": ""LB"", ""is_mobile"": false}" 19448,7,438,2017-04-14 17:00:35,http://mills.com/newell,0.2671485761,129.233.173.157,"{""location"": ""MK"", ""is_mobile"": false}" 19449,7,438,2017-01-29 13:12:58,http://beierjakubowski.io/wilhelm,0.1917829265,190.134.125.222,"{""location"": ""ES"", ""is_mobile"": false}" 19450,7,438,2017-04-13 04:52:12,http://gislasonward.info/maya_dietrich,0.5289212538,248.133.251.170,"{""location"": ""GG"", ""is_mobile"": false}" 19451,7,438,2017-03-20 21:27:56,http://tremblay.io/kenton_little,0.7951791715,79.183.24.179,"{""location"": ""GG"", ""is_mobile"": false}" 19452,7,438,2017-01-05 12:26:39,http://hilpert.com/john.abernathy,0.3387173144,152.188.231.238,"{""location"": ""YE"", ""is_mobile"": false}" 19453,7,438,2017-04-10 19:32:52,http://boscoritchie.io/heaven_schumm,0.5495527092,133.123.63.30,"{""location"": ""SX"", ""is_mobile"": false}" 19454,7,438,2017-03-28 02:16:18,http://vonrueden.co/hulda_paucek,0.9848759368,147.60.6.190,"{""location"": ""IL"", ""is_mobile"": false}" 19455,7,438,2016-12-26 15:31:45,http://zboncak.biz/bryce,0.7726250731,232.36.196.185,"{""location"": ""JM"", ""is_mobile"": true}" 19456,7,438,2017-02-07 03:38:09,http://little.com/adrien,0.9466538827,227.30.237.16,"{""location"": ""CV"", ""is_mobile"": false}" 19457,7,438,2017-06-05 21:42:45,http://hermistonmitchell.biz/marcia_rutherford,0.0477819322,111.6.158.81,"{""location"": ""AX"", ""is_mobile"": true}" 19458,7,438,2017-04-10 12:49:53,http://kihn.info/arvilla,0.6508691408,228.167.21.64,"{""location"": ""CI"", ""is_mobile"": true}" 19459,7,438,2017-05-22 00:14:21,http://kling.org/gloria,0.2595620876,23.208.86.35,"{""location"": ""AI"", ""is_mobile"": true}" 19460,7,438,2017-04-16 00:40:29,http://hagenes.info/stacey,0.8504740166,238.114.227.232,"{""location"": ""SK"", ""is_mobile"": false}" 19461,7,438,2016-12-17 22:24:34,http://jacobson.org/harvey,0.7013045956,140.141.117.141,"{""location"": ""ZW"", ""is_mobile"": true}" 19462,7,438,2017-05-29 22:13:51,http://reichel.co/marley_schaefer,0.2234932896,20.109.169.176,"{""location"": ""CV"", ""is_mobile"": true}" 19463,7,438,2017-02-08 22:42:50,http://schaden.biz/lewis_toy,0.1529790218,67.170.158.246,"{""location"": ""PT"", ""is_mobile"": true}" 19464,7,438,2017-04-28 10:41:54,http://borerkovacek.com/kaley,0.7028910688,205.2.60.27,"{""location"": ""VG"", ""is_mobile"": false}" 19465,7,438,2017-05-12 22:38:14,http://lowe.info/agustina,0.6411421695,94.106.206.82,"{""location"": ""NF"", ""is_mobile"": false}" 19466,7,438,2017-04-10 09:51:02,http://gottlieb.com/margarette.koelpin,0.1392908571,111.58.68.156,"{""location"": ""HU"", ""is_mobile"": true}" 19467,7,438,2017-01-27 08:44:34,http://mitchell.name/mariano,0.2423815571,159.22.44.248,"{""location"": ""DO"", ""is_mobile"": false}" 19468,7,438,2017-04-10 00:28:13,http://hilll.io/myrna,0.1543903365,44.105.205.181,"{""location"": ""ML"", ""is_mobile"": true}" 19469,7,438,2016-12-26 11:17:29,http://franecki.name/jude_boyle,0.3081476820,9.59.130.221,"{""location"": ""SH"", ""is_mobile"": true}" 19470,7,438,2017-03-25 20:44:12,http://grady.co/freddie,0.3476775189,177.246.218.15,"{""location"": ""YT"", ""is_mobile"": false}" 19471,7,438,2017-01-17 20:59:58,http://baumbach.io/daphne,0.0605619923,184.82.42.9,"{""location"": ""CD"", ""is_mobile"": false}" 19472,7,438,2017-03-28 21:05:20,http://altenwerth.biz/collin,0.8071972993,146.176.107.202,"{""location"": ""CY"", ""is_mobile"": true}" 19473,7,438,2017-03-27 07:05:47,http://corwin.name/oran,0.6636772607,129.88.65.68,"{""location"": ""RW"", ""is_mobile"": true}" 19474,7,439,2017-05-24 13:45:42,http://kuphalmarvin.name/alexa_okuneva,0.5679580252,23.81.111.180,"{""location"": ""MN"", ""is_mobile"": false}" 19475,7,439,2017-02-26 11:45:14,http://willms.biz/kay_marquardt,0.3998110771,146.196.103.236,"{""location"": ""LS"", ""is_mobile"": false}" 19476,7,439,2017-06-05 18:31:20,http://pfeffer.co/darion.dickinson,0.8961477257,25.109.199.231,"{""location"": ""GY"", ""is_mobile"": false}" 19477,7,439,2017-01-08 13:41:02,http://ullrichreynolds.co/dawn,0.4272436273,130.41.90.101,"{""location"": ""BO"", ""is_mobile"": false}" 19478,7,439,2017-03-09 14:34:24,http://jerde.biz/lolita.labadie,0.8701679631,53.27.102.91,"{""location"": ""MX"", ""is_mobile"": true}" 19479,7,439,2017-02-24 05:49:23,http://beatty.info/cade,0.0997826696,217.166.61.125,"{""location"": ""TN"", ""is_mobile"": true}" 19480,7,439,2017-06-04 07:16:02,http://nitzsche.net/gianni_kuhn,0.7755155486,15.93.149.226,"{""location"": ""MY"", ""is_mobile"": false}" 19481,7,439,2016-12-22 07:34:41,http://heel.com/amiya_waters,0.7583893304,12.148.20.129,"{""location"": ""PG"", ""is_mobile"": true}" 19482,7,439,2017-05-12 07:51:44,http://hirthe.org/donnie.rohan,0.5691699678,22.116.17.178,"{""location"": ""NF"", ""is_mobile"": false}" 19483,7,439,2017-03-05 08:24:11,http://jenkins.co/cole,0.1509034502,36.82.59.226,"{""location"": ""SG"", ""is_mobile"": false}" 19484,7,439,2017-02-28 03:23:15,http://schowalter.io/queenie,0.2115958270,141.253.205.98,"{""location"": ""LU"", ""is_mobile"": false}" 19485,7,439,2017-03-02 20:57:31,http://ebertoberbrunner.info/jarrod,0.5551900192,90.119.137.188,"{""location"": ""CI"", ""is_mobile"": false}" 19486,7,439,2017-04-22 01:13:41,http://hoeger.name/asha_shanahan,0.5242020484,8.7.8.68,"{""location"": ""UM"", ""is_mobile"": false}" 19487,7,439,2017-04-05 12:17:17,http://rowe.info/brandi.oberbrunner,0.5976215681,162.133.162.110,"{""location"": ""BJ"", ""is_mobile"": false}" 19488,7,439,2017-04-15 00:41:48,http://jakubowskiturcotte.org/ethan.dubuque,0.2123093733,150.182.220.126,"{""location"": ""US"", ""is_mobile"": true}" 19489,7,439,2017-01-06 05:57:21,http://thompsonfeest.org/jaylon,0.1917592019,144.61.128.186,"{""location"": ""FM"", ""is_mobile"": false}" 19490,7,439,2017-03-04 11:52:06,http://smitham.com/elvie,0.8640143270,170.225.215.54,"{""location"": ""IM"", ""is_mobile"": false}" 19491,7,439,2017-04-02 21:35:22,http://herzoglockman.name/cordell,0.5418249420,34.183.102.241,"{""location"": ""VI"", ""is_mobile"": false}" 19492,7,439,2016-12-26 22:11:36,http://cummerata.name/madelynn.lind,0.5553095274,238.141.246.78,"{""location"": ""DK"", ""is_mobile"": false}" 19493,7,439,2017-03-03 14:46:57,http://cummeratalemke.name/marquis_kerluke,0.9961390799,40.111.208.104,"{""location"": ""VA"", ""is_mobile"": true}" 19494,7,439,2016-12-20 16:04:08,http://durgan.co/alfredo.sauer,0.2017059918,205.150.235.68,"{""location"": ""TK"", ""is_mobile"": false}" 19495,7,439,2017-05-03 16:58:41,http://prohaska.net/jerrod,0.0982831331,147.157.179.226,"{""location"": ""LS"", ""is_mobile"": true}" 19496,7,439,2017-04-21 07:25:26,http://oberbrunnernikolaus.info/reba_romaguera,0.9132871737,119.52.63.38,"{""location"": ""GE"", ""is_mobile"": true}" 19497,7,439,2017-06-05 19:19:45,http://cainweinat.org/khalid.koepp,0.0617269845,10.60.19.232,"{""location"": ""MR"", ""is_mobile"": true}" 19498,7,439,2017-04-08 14:24:06,http://von.biz/johann_mohr,0.8836142084,119.224.82.80,"{""location"": ""ZW"", ""is_mobile"": true}" 19499,7,439,2017-01-12 12:24:14,http://macgyver.biz/elaina_fadel,0.6792642160,249.196.15.64,"{""location"": ""PL"", ""is_mobile"": true}" 19500,7,439,2017-05-24 01:21:31,http://gibson.org/janae,0.4939656473,94.14.118.80,"{""location"": ""KW"", ""is_mobile"": true}" 19501,7,439,2017-04-07 05:50:27,http://stracke.name/reymundo.doyle,0.4581285890,15.62.132.6,"{""location"": ""UG"", ""is_mobile"": true}" 19502,7,439,2017-02-17 15:33:42,http://cruickshank.org/jeica,0.8268612634,24.232.113.112,"{""location"": ""HM"", ""is_mobile"": true}" 19503,7,439,2017-02-15 17:34:55,http://effertz.io/marilie,0.5792882335,165.41.21.215,"{""location"": ""AW"", ""is_mobile"": false}" 19504,7,439,2017-06-04 03:15:50,http://stokes.org/tyrel_leuschke,0.0385576580,113.143.52.29,"{""location"": ""KY"", ""is_mobile"": true}" 19505,7,439,2017-02-24 18:57:00,http://gutmann.com/edd,0.2775151457,205.172.218.115,"{""location"": ""BI"", ""is_mobile"": true}" 19506,7,439,2017-05-28 12:10:17,http://ritchie.org/murphy,0.6528699920,100.101.113.238,"{""location"": ""MN"", ""is_mobile"": true}" 19507,7,439,2017-04-17 11:10:28,http://runolfsdottir.co/wayne_ullrich,0.4767541624,75.227.206.170,"{""location"": ""BZ"", ""is_mobile"": true}" 19508,7,439,2017-01-05 15:34:25,http://watsica.info/raven.hegmann,0.9329569955,44.186.48.115,"{""location"": ""UM"", ""is_mobile"": false}" 19509,7,439,2017-01-16 02:16:53,http://harrisabernathy.name/justice.hoeger,0.5102907514,161.127.247.160,"{""location"": ""PH"", ""is_mobile"": false}" 19510,7,439,2017-05-06 12:01:11,http://hillljacobi.name/janie,0.0866433721,71.85.212.98,"{""location"": ""ZM"", ""is_mobile"": true}" 19511,7,439,2017-02-04 04:33:13,http://kris.org/albert,0.0651579417,2.226.169.138,"{""location"": ""NE"", ""is_mobile"": false}" 19512,7,439,2017-04-14 16:09:34,http://graham.info/jadyn_larson,0.6675473608,210.38.93.252,"{""location"": ""BW"", ""is_mobile"": true}" 19513,7,439,2017-03-24 02:04:48,http://effertzmills.net/orville_sawayn,0.6856207105,136.222.97.107,"{""location"": ""FR"", ""is_mobile"": false}" 19514,7,439,2017-03-22 16:49:49,http://bayer.net/kirk,0.6804911453,215.94.44.91,"{""location"": ""CW"", ""is_mobile"": false}" 19515,7,439,2016-12-18 14:32:46,http://ohara.biz/geraldine,0.7790209694,71.166.252.251,"{""location"": ""TJ"", ""is_mobile"": false}" 19516,7,439,2017-01-19 13:18:37,http://goyette.biz/audra_beahan,0.3156226280,10.195.23.55,"{""location"": ""SJ"", ""is_mobile"": false}" 19517,7,439,2017-04-15 17:06:35,http://thiel.net/rodolfo,0.4323517662,200.38.169.156,"{""location"": ""GU"", ""is_mobile"": false}" 19518,7,439,2017-01-09 03:42:54,http://kunde.name/gerda,0.0003925644,249.197.184.81,"{""location"": ""NA"", ""is_mobile"": true}" 19519,7,439,2017-02-14 00:25:27,http://stanton.biz/ethel_stark,0.7637305116,66.103.211.200,"{""location"": ""MC"", ""is_mobile"": false}" 19520,7,439,2017-06-09 12:55:23,http://jacobi.biz/ezekiel,0.4475354180,79.38.133.76,"{""location"": ""NR"", ""is_mobile"": true}" 19521,7,439,2017-01-03 20:58:32,http://padbergkozey.co/elsa_rau,0.8912980484,17.239.154.238,"{""location"": ""MC"", ""is_mobile"": false}" 19522,7,439,2017-04-06 06:16:39,http://gusikowski.net/kane_howe,0.3331294009,178.113.44.27,"{""location"": ""PH"", ""is_mobile"": true}" 19523,7,439,2017-01-18 00:03:30,http://bernhardkris.co/gladyce,0.5095950434,94.160.201.19,"{""location"": ""ZA"", ""is_mobile"": true}" 19524,7,439,2017-05-06 03:06:32,http://hellerkreiger.com/meghan,0.6433001896,154.239.70.86,"{""location"": ""PF"", ""is_mobile"": false}" 19525,7,439,2017-04-09 20:16:56,http://sporerhuel.org/bridget.hane,0.8060124951,127.92.18.108,"{""location"": ""IQ"", ""is_mobile"": false}" 19526,7,439,2017-01-11 02:11:31,http://champlin.info/barney,0.0840226491,203.5.25.97,"{""location"": ""ML"", ""is_mobile"": true}" 19527,7,439,2017-01-19 20:30:43,http://maggio.co/daniella.hammes,0.9978383325,156.197.241.154,"{""location"": ""MZ"", ""is_mobile"": false}" 19528,7,439,2017-05-16 16:43:51,http://kreiger.com/easter_hickle,0.8466497521,238.130.205.122,"{""location"": ""VN"", ""is_mobile"": true}" 19529,7,439,2017-01-30 03:50:06,http://kilback.org/trycia.medhurst,0.3373854282,13.127.120.195,"{""location"": ""MP"", ""is_mobile"": false}" 19530,7,440,2017-02-27 15:44:27,http://predovicmedhurst.biz/charity.ankunding,0.0942658899,133.12.241.69,"{""location"": ""GQ"", ""is_mobile"": true}" 19531,7,440,2017-05-01 08:56:31,http://howeschulist.net/stefan,0.2313030603,235.18.96.218,"{""location"": ""BM"", ""is_mobile"": true}" 19532,7,440,2017-03-17 04:14:13,http://durgankeeling.io/sophie,0.0921386268,183.15.78.12,"{""location"": ""GB"", ""is_mobile"": false}" 19533,7,440,2016-12-29 02:16:52,http://andersonlebsack.biz/erik,0.5578169324,16.172.83.8,"{""location"": ""GI"", ""is_mobile"": false}" 19534,7,440,2017-01-09 16:29:29,http://fisher.com/christine_fisher,0.6393479405,174.124.144.108,"{""location"": ""ZM"", ""is_mobile"": true}" 19535,7,440,2017-03-02 08:50:48,http://maggio.name/everett.breitenberg,0.7559755573,173.56.26.35,"{""location"": ""ZA"", ""is_mobile"": false}" 19536,7,440,2017-04-18 10:49:25,http://tromp.net/cathryn.kozey,0.1204576187,176.186.176.127,"{""location"": ""CW"", ""is_mobile"": true}" 19537,7,440,2017-01-14 23:09:03,http://crona.com/julia,0.5580320682,141.21.180.228,"{""location"": ""UY"", ""is_mobile"": true}" 19538,7,440,2017-01-22 22:00:56,http://cainbruen.io/desiree.little,0.2345894994,192.43.231.233,"{""location"": ""ER"", ""is_mobile"": true}" 19539,7,440,2016-12-23 05:03:20,http://oconnell.biz/mozelle,0.7225699710,105.26.162.71,"{""location"": ""QA"", ""is_mobile"": true}" 19540,7,440,2017-06-06 05:22:33,http://connelly.info/alec_wolff,0.7531374383,110.154.146.193,"{""location"": ""CI"", ""is_mobile"": true}" 19541,7,440,2017-01-16 13:33:04,http://lubowitz.info/jayne.konopelski,0.5418522510,241.207.213.215,"{""location"": ""LU"", ""is_mobile"": false}" 19542,7,440,2017-01-31 04:00:49,http://feil.net/daniela_prohaska,0.0423697244,4.28.51.91,"{""location"": ""RE"", ""is_mobile"": true}" 19543,7,440,2016-12-24 12:19:24,http://kuhn.net/brent_price,0.0814590303,122.51.13.2,"{""location"": ""IS"", ""is_mobile"": true}" 19544,7,440,2017-05-03 20:41:53,http://walker.io/jeyca_heathcote,0.4011520598,135.165.130.108,"{""location"": ""SM"", ""is_mobile"": false}" 19545,7,440,2017-01-08 12:14:40,http://lynchspencer.net/otilia_casper,0.7802577215,81.167.55.140,"{""location"": ""US"", ""is_mobile"": false}" 19546,7,440,2017-02-11 02:35:37,http://adams.co/maia,0.7582663615,60.86.239.148,"{""location"": ""IN"", ""is_mobile"": true}" 19547,7,440,2017-04-15 08:36:56,http://maggio.name/kristoffer,0.7292674118,63.84.139.87,"{""location"": ""MW"", ""is_mobile"": true}" 19548,7,440,2017-05-31 09:19:26,http://langworth.co/earline,0.2316779311,246.210.46.100,"{""location"": ""RE"", ""is_mobile"": true}" 19549,7,440,2017-02-24 22:45:42,http://dickinsonmohr.co/marc_bergstrom,0.6956366257,254.70.60.222,"{""location"": ""CZ"", ""is_mobile"": false}" 19550,7,440,2016-12-13 13:54:44,http://christiansen.net/wilhelmine_wintheiser,0.6101495672,19.210.52.177,"{""location"": ""FI"", ""is_mobile"": false}" 19551,7,440,2017-01-23 07:10:53,http://berge.co/sydni,0.5269091893,186.10.242.234,"{""location"": ""VA"", ""is_mobile"": true}" 19552,7,440,2017-03-12 13:49:39,http://macgyver.info/myrna,0.5107295899,90.142.142.54,"{""location"": ""ES"", ""is_mobile"": true}" 19553,7,440,2017-04-25 19:15:57,http://olson.io/rhianna,0.8220042374,46.16.67.82,"{""location"": ""BW"", ""is_mobile"": true}" 19554,7,440,2017-03-22 09:15:04,http://walshsporer.co/marlen,0.0971093316,126.115.176.56,"{""location"": ""AR"", ""is_mobile"": false}" 19555,7,440,2016-12-18 19:52:18,http://schimmelupton.com/lillian,0.6308171506,44.26.2.158,"{""location"": ""MN"", ""is_mobile"": false}" 19556,7,440,2017-01-10 06:42:54,http://altenwerthprice.co/william,0.9692550788,184.95.43.97,"{""location"": ""CV"", ""is_mobile"": false}" 19557,7,440,2017-06-02 12:39:29,http://larson.net/declan,0.9843033986,21.237.251.17,"{""location"": ""IQ"", ""is_mobile"": true}" 19558,7,440,2017-01-25 08:57:07,http://gaylord.info/jermaine,0.7987270672,206.51.133.225,"{""location"": ""QA"", ""is_mobile"": false}" 19559,7,440,2017-03-12 19:08:09,http://mcclure.org/cordia,0.4148879006,192.66.126.228,"{""location"": ""CC"", ""is_mobile"": false}" 19560,7,440,2017-05-25 02:22:46,http://kuhn.co/rosemarie,0.7664360307,224.182.38.19,"{""location"": ""BN"", ""is_mobile"": false}" 19561,7,440,2017-04-04 18:00:57,http://rodriguezeichmann.net/wendell,0.7823928044,28.159.193.222,"{""location"": ""KI"", ""is_mobile"": true}" 19562,7,440,2017-01-04 13:35:42,http://hettinger.co/chaz_mraz,0.2759403213,231.242.163.144,"{""location"": ""TT"", ""is_mobile"": false}" 19563,7,441,2017-04-13 14:09:11,http://wisozk.io/jey.glover,0.8670222872,188.55.99.5,"{""location"": ""BO"", ""is_mobile"": true}" 19564,7,441,2017-05-21 10:43:33,http://bradtkecormier.io/joanny_kirlin,0.7591794068,184.105.195.122,"{""location"": ""MR"", ""is_mobile"": true}" 19565,7,441,2017-01-29 00:26:33,http://hayes.name/rafael_orn,0.3113514515,87.139.155.193,"{""location"": ""LT"", ""is_mobile"": true}" 19566,7,441,2017-02-17 17:55:05,http://herzoglind.co/americo,0.8476196896,245.16.220.185,"{""location"": ""HK"", ""is_mobile"": true}" 19567,7,441,2017-04-20 16:24:12,http://jast.net/golden,0.8869959113,164.202.223.181,"{""location"": ""TZ"", ""is_mobile"": true}" 19568,7,441,2017-01-13 22:34:58,http://grimes.co/lindsay,0.0340345326,207.30.11.139,"{""location"": ""MK"", ""is_mobile"": false}" 19569,7,441,2017-03-22 02:50:30,http://towne.name/camille,0.3547136298,113.209.172.182,"{""location"": ""KR"", ""is_mobile"": false}" 19570,7,441,2017-02-10 17:27:35,http://wiza.name/leonor,0.5573843559,132.215.234.54,"{""location"": ""NZ"", ""is_mobile"": true}" 19571,7,441,2017-01-09 01:55:23,http://durgan.net/kaycee,0.4829766337,253.148.115.227,"{""location"": ""AO"", ""is_mobile"": true}" 19572,7,441,2017-01-13 15:21:19,http://gottliebauer.org/edmund,0.1020699768,199.83.76.161,"{""location"": ""SY"", ""is_mobile"": true}" 19573,7,441,2017-02-10 13:27:37,http://konopelski.io/fae,0.8692917036,96.249.246.6,"{""location"": ""GQ"", ""is_mobile"": false}" 19574,7,441,2017-03-30 10:53:35,http://pouros.info/jermaine,0.7649468557,213.154.233.165,"{""location"": ""CY"", ""is_mobile"": true}" 19575,7,441,2017-03-13 17:32:34,http://emmerich.biz/ima.kling,0.0491787595,130.248.150.205,"{""location"": ""FI"", ""is_mobile"": true}" 19576,7,441,2017-01-04 08:56:12,http://schaefercummings.info/monica_zboncak,0.8338738951,122.157.253.72,"{""location"": ""LS"", ""is_mobile"": false}" 19577,7,441,2017-02-17 12:42:58,http://von.co/sheldon,0.3554692635,21.196.175.192,"{""location"": ""CH"", ""is_mobile"": true}" 19578,7,441,2017-05-11 23:11:16,http://mcclure.io/aisha.jerde,0.6847278867,243.30.137.154,"{""location"": ""MD"", ""is_mobile"": false}" 19579,7,441,2017-06-11 19:15:23,http://reichert.io/laurel_boyle,0.1743031502,234.27.107.193,"{""location"": ""DO"", ""is_mobile"": false}" 19580,7,441,2017-06-07 22:59:11,http://fadel.co/scotty,0.2056355056,229.239.253.159,"{""location"": ""RE"", ""is_mobile"": false}" 19581,7,441,2017-01-24 05:46:54,http://ferry.co/ofelia.barton,0.5665589639,244.170.184.29,"{""location"": ""GF"", ""is_mobile"": true}" 19582,7,441,2017-05-01 12:39:38,http://cruickshank.biz/helmer,0.3581386158,69.7.224.251,"{""location"": ""TN"", ""is_mobile"": true}" 19583,7,441,2017-03-29 20:01:06,http://dickibahringer.biz/branson,0.2323289140,124.7.179.159,"{""location"": ""VE"", ""is_mobile"": false}" 19584,7,441,2016-12-14 01:13:06,http://frami.co/eliza_hermiston,0.1361940706,190.160.204.251,"{""location"": ""MD"", ""is_mobile"": true}" 19585,7,441,2017-05-09 03:37:15,http://balistrerijones.name/kattie_leuschke,0.6296184210,225.153.47.21,"{""location"": ""MH"", ""is_mobile"": true}" 19586,7,441,2017-05-19 15:19:11,http://walsh.io/cody,0.5349650564,88.232.50.173,"{""location"": ""CX"", ""is_mobile"": true}" 19587,7,441,2017-04-11 08:38:47,http://ankunding.org/samantha_glover,0.4761389649,188.57.190.77,"{""location"": ""CZ"", ""is_mobile"": true}" 19588,7,441,2017-02-10 12:53:49,http://pfannerstillrunolfsdottir.biz/gayle.effertz,0.2506628200,69.225.109.47,"{""location"": ""MC"", ""is_mobile"": false}" 19589,7,441,2017-05-25 03:10:40,http://frami.info/webster_kub,0.5072315957,213.176.247.79,"{""location"": ""BZ"", ""is_mobile"": true}" 19590,7,441,2016-12-18 06:25:36,http://grady.co/yazmin,0.7256938526,71.90.25.78,"{""location"": ""HU"", ""is_mobile"": false}" 19591,7,441,2017-01-22 02:48:21,http://schmittmccullough.biz/emile_damore,0.6716308475,65.48.149.140,"{""location"": ""IS"", ""is_mobile"": false}" 19592,7,441,2017-05-30 16:37:38,http://hagenes.info/aleandra,0.3841781933,251.111.51.237,"{""location"": ""RE"", ""is_mobile"": true}" 19593,7,441,2016-12-14 06:16:48,http://gorczany.com/samantha_stiedemann,0.9236519733,167.30.26.112,"{""location"": ""MH"", ""is_mobile"": true}" 19594,7,441,2017-03-28 14:18:37,http://beatty.biz/annie,0.9870678170,131.61.197.238,"{""location"": ""MT"", ""is_mobile"": true}" 19595,7,441,2017-06-08 23:30:23,http://greenholt.biz/janae,0.2739576855,20.209.38.65,"{""location"": ""GI"", ""is_mobile"": true}" 19596,7,441,2017-04-27 04:31:32,http://grady.info/houston,0.7799695472,84.179.33.36,"{""location"": ""SD"", ""is_mobile"": false}" 19597,7,441,2017-05-25 01:38:41,http://boehmschimmel.biz/ashlynn_dare,0.8547236757,66.40.229.119,"{""location"": ""BN"", ""is_mobile"": false}" 19598,7,441,2017-04-30 01:46:33,http://yundt.biz/norris.grady,0.1158134121,129.128.252.185,"{""location"": ""KR"", ""is_mobile"": true}" 19599,7,441,2017-02-10 15:26:35,http://lemke.io/orlando_rutherford,0.3672857886,168.40.206.107,"{""location"": ""RS"", ""is_mobile"": false}" 19600,7,441,2017-01-05 02:14:38,http://crooks.info/preston,0.2544682763,135.129.122.175,"{""location"": ""TJ"", ""is_mobile"": true}" 19601,7,441,2017-05-22 15:52:51,http://abernathyrobel.info/margaretta,0.0096675325,94.200.252.112,"{""location"": ""MG"", ""is_mobile"": false}" 19602,7,441,2017-05-30 23:15:16,http://willnitzsche.info/rod,0.0690548226,150.149.89.26,"{""location"": ""MV"", ""is_mobile"": false}" 19603,7,441,2017-03-06 11:37:47,http://collier.com/donavon.kling,0.8400101289,85.150.156.252,"{""location"": ""JE"", ""is_mobile"": true}" 19604,7,441,2016-12-24 23:12:22,http://jaskolskiruel.net/hailee,0.4488187901,111.47.187.97,"{""location"": ""SZ"", ""is_mobile"": true}" 19605,7,441,2017-01-12 04:57:48,http://lueilwitzmacejkovic.com/louie_sauer,0.8401310587,99.227.82.136,"{""location"": ""BG"", ""is_mobile"": false}" 19606,7,442,2017-04-11 11:43:51,http://tromp.io/juliana_wisoky,0.6209368795,109.111.25.214,"{""location"": ""DK"", ""is_mobile"": false}" 19607,7,442,2017-06-10 04:59:13,http://zboncakabbott.biz/phoebe,0.5219821737,186.246.191.169,"{""location"": ""KR"", ""is_mobile"": false}" 19608,7,442,2017-04-21 05:28:08,http://prosacco.com/jaquan_reinger,0.7193336751,95.235.90.20,"{""location"": ""CR"", ""is_mobile"": false}" 19609,7,442,2017-01-05 13:46:52,http://osinskihammes.co/edmond.white,0.5658363012,19.203.12.3,"{""location"": ""TF"", ""is_mobile"": false}" 19610,7,442,2017-06-05 07:37:35,http://ankundingjohnston.org/nikolas_larson,0.8973665665,21.125.84.3,"{""location"": ""CI"", ""is_mobile"": false}" 19611,7,442,2017-05-04 20:15:04,http://lowe.com/virgie,0.4072345117,234.182.178.71,"{""location"": ""DM"", ""is_mobile"": false}" 19612,7,442,2017-03-31 21:55:36,http://parisian.io/fay,0.9112006059,219.171.162.46,"{""location"": ""BI"", ""is_mobile"": true}" 19613,7,442,2017-05-09 03:44:49,http://oconnell.org/gideon_kuhn,0.0717987719,18.120.188.247,"{""location"": ""DJ"", ""is_mobile"": false}" 19614,7,442,2017-05-04 03:11:48,http://framibrekke.co/bobbie_dickinson,0.6610394706,203.61.155.82,"{""location"": ""PE"", ""is_mobile"": false}" 19615,7,442,2017-01-02 03:58:53,http://rice.info/hunter_hermiston,0.0946420074,248.56.5.2,"{""location"": ""YE"", ""is_mobile"": true}" 19616,7,442,2017-01-24 22:34:40,http://kulas.info/mozell.nikolaus,0.2989197529,78.6.201.20,"{""location"": ""JE"", ""is_mobile"": true}" 19617,7,442,2017-01-09 23:23:33,http://moriette.com/hillard,0.5591397724,75.236.127.86,"{""location"": ""TW"", ""is_mobile"": false}" 19618,7,442,2017-01-15 18:19:09,http://bosco.net/janea.kiehn,0.9532549832,158.235.169.214,"{""location"": ""KM"", ""is_mobile"": true}" 19619,7,442,2017-02-12 21:22:44,http://hagenes.biz/maximillian.okon,0.6404196590,47.40.193.134,"{""location"": ""LY"", ""is_mobile"": false}" 19620,7,442,2017-02-24 05:09:53,http://blandahowell.io/gordon.lynch,0.0207757113,5.144.236.129,"{""location"": ""NI"", ""is_mobile"": false}" 19621,7,442,2016-12-19 23:37:08,http://boylekertzmann.org/justyn_kautzer,0.5123411535,91.158.144.207,"{""location"": ""SG"", ""is_mobile"": false}" 19622,7,442,2017-03-30 01:37:09,http://gleason.info/johnny,0.6722638684,215.34.115.73,"{""location"": ""BZ"", ""is_mobile"": false}" 19623,7,442,2017-04-24 07:28:49,http://mcglynn.co/else,0.2157759203,171.213.34.181,"{""location"": ""MQ"", ""is_mobile"": false}" 19624,7,442,2017-04-16 16:05:22,http://hyatt.info/aiden,0.6083632024,207.85.133.18,"{""location"": ""BY"", ""is_mobile"": true}" 19625,7,442,2017-05-14 01:47:01,http://beatty.net/rashad,0.3434029141,55.217.232.138,"{""location"": ""BA"", ""is_mobile"": false}" 19626,7,442,2017-03-26 03:34:44,http://gottlieb.info/mazie,0.6618835598,111.100.57.218,"{""location"": ""CX"", ""is_mobile"": false}" 19627,7,442,2017-01-19 09:15:00,http://padbergtowne.io/mia.powlowski,0.3729598671,98.70.220.98,"{""location"": ""MV"", ""is_mobile"": false}" 19628,7,442,2017-01-25 10:43:52,http://harrisabernathy.biz/steve_larkin,0.2701172856,178.174.47.96,"{""location"": ""NC"", ""is_mobile"": false}" 19629,7,442,2017-03-26 19:39:10,http://zulaufbartell.com/herman_stiedemann,0.5487510924,30.156.200.127,"{""location"": ""KM"", ""is_mobile"": true}" 19630,7,442,2017-03-16 09:03:38,http://zulaufcormier.biz/aniya.berge,0.1011773246,111.94.15.223,"{""location"": ""GF"", ""is_mobile"": false}" 19631,7,442,2017-05-06 14:23:36,http://torp.net/aryanna_hoppe,0.3668525786,191.75.82.37,"{""location"": ""RE"", ""is_mobile"": false}" 19632,7,442,2017-06-03 03:55:46,http://moorerempel.net/jonathan,0.1187774024,159.108.17.208,"{""location"": ""SR"", ""is_mobile"": true}" 19633,7,442,2017-03-07 21:22:06,http://adamsreynolds.com/elena_prohaska,0.0629636804,227.27.66.153,"{""location"": ""MC"", ""is_mobile"": true}" 19634,7,442,2017-04-15 14:07:39,http://hartmann.biz/alverta_roob,0.8915126652,61.247.116.45,"{""location"": ""JO"", ""is_mobile"": false}" 19635,7,442,2016-12-31 21:15:48,http://leannonstoltenberg.biz/lawson,0.3982673825,125.234.35.161,"{""location"": ""SH"", ""is_mobile"": false}" 19636,7,442,2017-04-14 02:12:20,http://wisozk.org/federico.ruel,0.3094504299,154.33.25.198,"{""location"": ""NE"", ""is_mobile"": true}" 19637,7,442,2016-12-24 04:18:34,http://streichkuphal.com/jakayla,0.7692826393,153.92.213.106,"{""location"": ""MF"", ""is_mobile"": true}" 19638,7,442,2017-05-21 06:57:14,http://block.net/malvina,0.1582159588,45.183.92.218,"{""location"": ""RS"", ""is_mobile"": false}" 19639,7,442,2017-02-15 00:32:22,http://adams.name/nelda.gorczany,0.4285022720,56.54.157.221,"{""location"": ""GL"", ""is_mobile"": true}" 19640,7,442,2017-02-27 20:40:17,http://ritchie.name/katlynn_rippin,0.0552916742,196.37.254.24,"{""location"": ""KM"", ""is_mobile"": true}" 19641,7,442,2017-01-08 05:00:21,http://stehr.info/virginia,0.5073778200,75.17.18.205,"{""location"": ""KY"", ""is_mobile"": true}" 19642,7,442,2017-03-15 09:54:43,http://mcclure.net/amiya,0.8137124440,171.78.237.237,"{""location"": ""GU"", ""is_mobile"": false}" 19643,7,442,2017-02-03 03:18:59,http://hudsonlittel.name/scarlett.boyle,0.5544233846,139.237.104.153,"{""location"": ""MM"", ""is_mobile"": true}" 19644,7,442,2017-01-22 10:57:31,http://moriette.info/abbie.kuhlman,0.2720240777,27.167.39.79,"{""location"": ""CM"", ""is_mobile"": false}" 19645,7,442,2017-03-05 04:22:17,http://nikolaushilpert.co/tod,0.7473047306,215.195.118.92,"{""location"": ""BV"", ""is_mobile"": false}" 19646,7,442,2017-03-07 20:13:59,http://walker.net/augusta.pfeffer,0.8170471663,116.197.240.3,"{""location"": ""IM"", ""is_mobile"": true}" 19647,7,442,2017-01-02 21:11:58,http://baumbach.io/alexander,0.2866458636,32.222.6.127,"{""location"": ""KZ"", ""is_mobile"": false}" 19648,7,443,2017-05-04 00:52:21,http://price.co/melia.daugherty,0.1225193900,26.121.127.101,"{""location"": ""SC"", ""is_mobile"": true}" 19649,7,443,2017-05-06 11:55:48,http://barrows.co/dwight,0.2383812366,59.110.138.184,"{""location"": ""AW"", ""is_mobile"": true}" 19650,7,443,2017-02-12 00:13:23,http://abshirekonopelski.net/erling,0.7713834734,190.46.193.20,"{""location"": ""TJ"", ""is_mobile"": false}" 19651,7,443,2017-03-06 06:53:48,http://torp.name/jayda,0.4381348958,235.5.87.94,"{""location"": ""IQ"", ""is_mobile"": true}" 19652,7,443,2017-04-18 23:38:57,http://lubowitz.io/eleanore,0.6605250044,244.107.141.107,"{""location"": ""DK"", ""is_mobile"": true}" 19653,7,443,2016-12-21 05:50:40,http://bogan.net/tianna.macejkovic,0.6574824163,118.31.186.111,"{""location"": ""MG"", ""is_mobile"": false}" 19654,7,443,2017-03-15 22:49:28,http://watsica.co/lorenza_senger,0.8623125695,18.195.61.214,"{""location"": ""EC"", ""is_mobile"": false}" 19655,7,443,2016-12-30 10:27:04,http://murphy.co/kellie,0.6436932304,96.9.165.190,"{""location"": ""NI"", ""is_mobile"": false}" 19656,7,443,2017-06-10 21:02:38,http://kovacekcormier.name/chandler_stiedemann,0.5129516031,101.140.155.117,"{""location"": ""SM"", ""is_mobile"": false}" 19657,7,443,2017-03-29 14:08:06,http://ondricka.biz/gaston.deckow,0.3949355260,214.177.181.66,"{""location"": ""RU"", ""is_mobile"": false}" 19658,7,443,2017-05-18 21:02:34,http://nicolas.info/aleen,0.3677090038,188.251.5.44,"{""location"": ""DK"", ""is_mobile"": false}" 19659,7,443,2017-06-06 10:20:16,http://thompsonruel.org/rogers,0.2246546217,226.13.42.213,"{""location"": ""NZ"", ""is_mobile"": false}" 19660,7,443,2017-03-24 20:01:22,http://hicklebuckridge.com/julian.hansen,0.1934123219,233.232.180.152,"{""location"": ""CC"", ""is_mobile"": false}" 19661,7,443,2017-01-27 08:00:06,http://keler.info/nathanial.pfannerstill,0.4443891634,186.187.56.192,"{""location"": ""KE"", ""is_mobile"": false}" 19662,7,443,2017-03-11 05:36:51,http://bechtelar.io/greyson_lebsack,0.5225393026,43.101.33.148,"{""location"": ""MD"", ""is_mobile"": true}" 19663,7,443,2017-06-07 23:51:03,http://schmitt.net/jackson.welch,0.8581394119,122.135.226.187,"{""location"": ""BM"", ""is_mobile"": false}" 19664,7,443,2017-04-18 05:47:38,http://hamill.info/hilbert,0.8715917939,224.52.230.42,"{""location"": ""ES"", ""is_mobile"": false}" 19665,7,443,2017-01-01 17:28:05,http://rath.info/spencer,0.1987452746,126.134.220.238,"{""location"": ""MO"", ""is_mobile"": true}" 19666,7,443,2017-05-23 04:51:24,http://kovacek.name/hershel,0.1468033461,95.65.254.209,"{""location"": ""TD"", ""is_mobile"": true}" 19667,7,443,2017-01-04 01:22:38,http://wolff.org/steve,0.3811262540,235.194.75.183,"{""location"": ""MQ"", ""is_mobile"": false}" 19668,7,443,2016-12-24 22:46:52,http://harris.co/kelton,0.0577696705,27.65.232.163,"{""location"": ""UM"", ""is_mobile"": true}" 19669,7,443,2016-12-21 06:00:29,http://gerholdmaggio.io/flo.barton,0.6736343070,164.218.67.23,"{""location"": ""CC"", ""is_mobile"": false}" 19670,7,443,2017-02-04 17:48:35,http://stehrstoltenberg.biz/tom.schamberger,0.7896468151,253.85.68.79,"{""location"": ""VG"", ""is_mobile"": false}" 19671,7,443,2017-05-21 18:23:21,http://schowalter.biz/imani,0.4565596166,190.130.84.250,"{""location"": ""MP"", ""is_mobile"": true}" 19672,7,443,2017-02-19 22:25:47,http://price.info/tremaine,0.5218716878,220.11.67.8,"{""location"": ""CZ"", ""is_mobile"": true}" 19673,7,443,2017-04-20 08:06:59,http://halvorson.info/johnpaul_swift,0.9153586881,172.236.167.121,"{""location"": ""SB"", ""is_mobile"": true}" 19674,7,443,2017-01-09 00:02:23,http://williamson.info/shanna,0.0734843063,161.171.191.246,"{""location"": ""MU"", ""is_mobile"": false}" 19675,7,443,2017-04-07 02:58:05,http://stoltenberg.net/beverly_simonis,0.3089618503,176.73.133.186,"{""location"": ""EC"", ""is_mobile"": false}" 19676,7,443,2017-01-10 06:23:48,http://kundemoore.name/hanna.turcotte,0.1921771904,195.64.187.5,"{""location"": ""NU"", ""is_mobile"": false}" 19677,7,443,2017-04-06 20:34:04,http://mohrhilll.com/arvel_stehr,0.7563170970,35.146.4.25,"{""location"": ""RS"", ""is_mobile"": true}" 19678,7,443,2017-05-09 01:42:59,http://hauck.com/andre,0.2366914113,182.14.9.76,"{""location"": ""EG"", ""is_mobile"": true}" 19679,7,443,2016-12-13 20:38:54,http://conn.io/kayli,0.9736347154,189.184.25.146,"{""location"": ""BW"", ""is_mobile"": true}" 19680,7,443,2017-01-28 03:18:16,http://ryan.name/johnny,0.3918529799,117.224.111.20,"{""location"": ""GA"", ""is_mobile"": false}" 19681,7,443,2017-03-08 18:23:24,http://ferry.net/fredrick.weimann,0.7667330416,239.230.101.193,"{""location"": ""EH"", ""is_mobile"": false}" 19682,7,443,2017-04-14 19:17:23,http://gulgowski.co/kathleen,0.9162763341,234.224.69.32,"{""location"": ""TK"", ""is_mobile"": true}" 19683,7,443,2017-01-26 06:26:43,http://pfefferschroeder.org/ansley_douglas,0.5336769960,42.94.101.223,"{""location"": ""VG"", ""is_mobile"": false}" 19684,7,443,2017-04-22 00:40:30,http://langworth.net/sebastian,0.7098052344,175.5.226.239,"{""location"": ""CN"", ""is_mobile"": false}" 19685,7,443,2017-04-14 07:25:55,http://cummings.com/margarete,0.9645013953,195.111.155.142,"{""location"": ""DE"", ""is_mobile"": false}" 19686,7,443,2017-04-15 01:41:05,http://ruecker.co/braxton,0.7801000351,252.99.126.159,"{""location"": ""TL"", ""is_mobile"": false}" 19687,7,443,2017-06-12 22:03:10,http://haagschroeder.com/georgette_ruel,0.9668981768,8.26.245.94,"{""location"": ""MQ"", ""is_mobile"": false}" 19688,7,443,2017-01-18 01:04:10,http://rogahn.io/dasia,0.8276276148,191.252.20.189,"{""location"": ""ZW"", ""is_mobile"": false}" 19689,7,443,2017-04-11 05:55:04,http://cruickshank.io/melvin.powlowski,0.2380127806,20.161.139.84,"{""location"": ""LR"", ""is_mobile"": true}" 19690,7,443,2017-03-21 14:43:07,http://hills.co/emmet.kuvalis,0.5006976499,82.38.120.204,"{""location"": ""CC"", ""is_mobile"": true}" 19691,7,443,2017-04-30 15:07:08,http://wisozk.io/hannah_robel,0.2999779655,225.57.151.194,"{""location"": ""NU"", ""is_mobile"": true}" 19692,7,443,2017-01-09 18:20:46,http://zieme.biz/jonathan_durgan,0.3110469202,82.90.164.104,"{""location"": ""BQ"", ""is_mobile"": true}" 19693,7,443,2017-05-14 03:26:42,http://wehneraufderhar.org/deion,0.9759897800,36.161.246.49,"{""location"": ""SZ"", ""is_mobile"": true}" 19694,7,443,2017-05-15 05:43:11,http://zieme.name/philip.gusikowski,0.8931834298,30.109.128.95,"{""location"": ""CK"", ""is_mobile"": false}" 19695,7,443,2017-03-22 15:19:26,http://zemlak.name/jarrod,0.6781219411,20.137.231.140,"{""location"": ""ID"", ""is_mobile"": true}" 19696,7,443,2017-05-20 09:04:36,http://schumm.io/cleo,0.0408481942,61.29.231.37,"{""location"": ""GW"", ""is_mobile"": true}" 19697,7,443,2017-01-05 23:47:08,http://orn.info/annetta,0.0449994133,100.203.217.154,"{""location"": ""AU"", ""is_mobile"": false}" 19698,7,443,2017-05-26 09:42:59,http://monahan.net/jewel,0.4609891247,168.168.24.164,"{""location"": ""LY"", ""is_mobile"": true}" 19699,7,443,2017-05-15 17:47:47,http://gleichnerbogisich.io/bert_watsica,0.2493793057,109.66.220.172,"{""location"": ""TN"", ""is_mobile"": false}" 19700,7,443,2017-01-07 08:58:29,http://lebsack.io/kamille,0.6432757372,21.205.194.100,"{""location"": ""CX"", ""is_mobile"": true}" 19701,7,443,2017-03-30 17:34:21,http://lehner.info/brannon,0.4893731155,98.59.3.84,"{""location"": ""GH"", ""is_mobile"": false}" 19702,7,443,2016-12-30 16:09:23,http://crooks.info/daniella,0.9491194207,106.47.27.16,"{""location"": ""KE"", ""is_mobile"": true}" 19703,7,443,2017-05-02 10:31:39,http://fahey.com/kitty,0.5449586144,171.211.100.112,"{""location"": ""GH"", ""is_mobile"": true}" 19704,7,443,2017-01-09 19:52:26,http://reichert.co/justyn.kirlin,0.3932263762,162.242.233.211,"{""location"": ""IQ"", ""is_mobile"": false}" 19705,7,443,2017-05-19 02:42:50,http://bernhard.name/cheyenne.kuhic,0.8592078219,169.187.109.217,"{""location"": ""TF"", ""is_mobile"": true}" 19706,7,443,2017-05-17 18:11:28,http://kunzehilpert.net/jarrett_pfannerstill,0.6793745747,58.76.232.214,"{""location"": ""VC"", ""is_mobile"": true}" 19707,7,443,2017-04-09 19:12:08,http://smith.org/herta,0.6976586336,30.85.226.156,"{""location"": ""SB"", ""is_mobile"": false}" 19708,7,443,2017-05-14 23:53:38,http://dicki.net/green.howell,0.4446681502,104.115.146.138,"{""location"": ""KP"", ""is_mobile"": true}" 19709,7,443,2017-03-01 03:23:23,http://wehner.com/julien,0.3780134519,116.240.250.92,"{""location"": ""LT"", ""is_mobile"": true}" 19710,7,443,2017-05-09 15:53:45,http://aufderhar.biz/raina,0.1902623164,194.197.84.84,"{""location"": ""TD"", ""is_mobile"": false}" 19711,7,443,2017-01-18 16:17:36,http://hermannmacgyver.biz/carmel,0.2814613551,239.229.215.15,"{""location"": ""VE"", ""is_mobile"": true}" 19712,7,443,2017-03-18 11:37:08,http://flatley.io/elia,0.7001805562,253.115.138.43,"{""location"": ""JM"", ""is_mobile"": true}" 19713,7,444,2017-04-20 03:14:44,http://dooleylehner.info/lora,0.0619632067,167.32.218.158,"{""location"": ""IS"", ""is_mobile"": false}" 19714,7,444,2017-01-05 10:27:06,http://oreillygutkowski.org/aric.macejkovic,0.8545478158,163.161.163.201,"{""location"": ""TG"", ""is_mobile"": false}" 19715,7,444,2016-12-28 09:06:41,http://marquardtko.io/shaylee_price,0.5876390920,80.153.222.25,"{""location"": ""TH"", ""is_mobile"": false}" 19716,7,444,2017-02-17 05:24:34,http://schuppe.co/jamel_daniel,0.9532822756,192.243.202.243,"{""location"": ""AO"", ""is_mobile"": false}" 19717,7,444,2016-12-26 22:24:02,http://koch.co/isabella_erdman,0.8238588049,13.187.164.52,"{""location"": ""CG"", ""is_mobile"": false}" 19718,7,444,2017-01-13 04:24:10,http://bartoletti.co/chet_brakus,0.1841494407,33.230.249.121,"{""location"": ""CH"", ""is_mobile"": false}" 19719,7,444,2017-06-03 19:29:24,http://lindgren.biz/lenore,0.6988375907,131.139.37.67,"{""location"": ""LR"", ""is_mobile"": true}" 19720,7,444,2016-12-16 12:16:20,http://bode.info/sonny,0.5020229342,47.152.201.229,"{""location"": ""SA"", ""is_mobile"": false}" 19721,7,444,2017-05-18 12:44:09,http://corwin.info/tina.dooley,0.6983668267,203.165.169.33,"{""location"": ""MW"", ""is_mobile"": true}" 19722,7,444,2017-02-03 10:38:22,http://huels.biz/german,0.5456514479,173.116.237.157,"{""location"": ""RE"", ""is_mobile"": false}" 19723,7,444,2017-01-10 06:23:46,http://oreilly.name/lyda,0.7578362455,101.239.196.3,"{""location"": ""NI"", ""is_mobile"": false}" 19724,7,444,2017-02-18 01:30:47,http://wintheiser.name/jena_bechtelar,0.0624061007,167.83.28.102,"{""location"": ""HT"", ""is_mobile"": false}" 19725,7,444,2017-01-22 02:38:28,http://rath.org/fanny.mclaughlin,0.3212635498,250.147.245.41,"{""location"": ""NZ"", ""is_mobile"": false}" 19726,7,444,2017-06-09 17:58:51,http://yundtruel.com/donny_ankunding,0.2428915674,204.75.151.227,"{""location"": ""NZ"", ""is_mobile"": false}" 19727,7,444,2017-06-03 16:54:18,http://lubowitz.name/adela,0.6118696148,169.113.52.99,"{""location"": ""AM"", ""is_mobile"": false}" 19728,7,444,2017-01-03 04:03:50,http://keler.org/braxton,0.6262531462,98.116.135.124,"{""location"": ""CF"", ""is_mobile"": true}" 19729,7,444,2017-03-25 05:38:59,http://konopelski.net/dariana,0.7380521719,126.57.9.135,"{""location"": ""EC"", ""is_mobile"": true}" 19730,7,444,2017-03-31 07:19:38,http://corkerybartoletti.io/candida,0.8230901735,98.59.190.145,"{""location"": ""TK"", ""is_mobile"": false}" 19731,7,444,2017-02-23 02:26:44,http://lynch.info/tre.schmidt,0.3491701771,224.109.191.240,"{""location"": ""SH"", ""is_mobile"": true}" 19732,7,444,2017-03-09 15:11:25,http://reichertfeeney.biz/peter.windler,0.9176353022,70.84.243.204,"{""location"": ""JE"", ""is_mobile"": true}" 19733,7,444,2017-01-28 15:23:23,http://schuppe.org/antonia,0.6949826966,233.154.231.76,"{""location"": ""CM"", ""is_mobile"": true}" 19734,7,444,2017-05-19 01:22:48,http://labadie.net/myriam,0.7788296295,47.242.186.230,"{""location"": ""TO"", ""is_mobile"": false}" 19735,7,444,2017-02-25 09:25:52,http://heaney.co/loma,0.0388489415,244.75.35.254,"{""location"": ""TF"", ""is_mobile"": true}" 19736,7,444,2017-03-01 19:24:22,http://kozey.org/naomi,0.4645092967,5.41.138.215,"{""location"": ""BS"", ""is_mobile"": true}" 19737,7,444,2017-03-02 22:02:41,http://langosh.net/raleigh.sauer,0.4908219665,118.162.201.5,"{""location"": ""BQ"", ""is_mobile"": true}" 19738,7,444,2017-03-22 18:45:04,http://brekke.com/hailee,0.8217332022,181.83.121.208,"{""location"": ""DZ"", ""is_mobile"": true}" 19739,7,444,2017-03-31 22:23:51,http://kuhn.net/marquise,0.4126286919,207.33.242.58,"{""location"": ""VI"", ""is_mobile"": true}" 19740,7,444,2017-02-12 15:30:56,http://haneauer.com/javonte,0.4903425143,240.59.53.189,"{""location"": ""QA"", ""is_mobile"": true}" 19741,7,444,2016-12-27 03:43:43,http://hettinger.biz/josefina_dibbert,0.1725732533,247.98.125.142,"{""location"": ""WF"", ""is_mobile"": true}" 19742,7,444,2017-02-09 03:26:42,http://marvingerhold.io/erick,0.5989431066,154.209.200.100,"{""location"": ""UG"", ""is_mobile"": false}" 19743,7,444,2017-01-04 19:57:09,http://yundt.info/cyrus_carroll,0.7249368651,246.112.130.187,"{""location"": ""NR"", ""is_mobile"": true}" 19744,7,444,2017-03-12 01:05:13,http://hackett.org/abbie_ullrich,0.5402926016,186.140.28.166,"{""location"": ""PW"", ""is_mobile"": true}" 19745,7,444,2017-04-13 22:28:04,http://gerholdschamberger.co/cullen_boyle,0.1686637401,88.27.19.84,"{""location"": ""MC"", ""is_mobile"": true}" 19746,7,444,2017-06-06 21:02:33,http://welch.co/adolf,0.4627308344,250.229.106.89,"{""location"": ""RS"", ""is_mobile"": true}" 19747,7,444,2017-03-01 01:43:14,http://quigley.net/sunny_gutmann,0.0310120354,142.119.250.51,"{""location"": ""KM"", ""is_mobile"": true}" 19748,7,444,2017-06-13 18:25:25,http://nolanherman.biz/adriel,0.9296986130,106.28.52.207,"{""location"": ""PW"", ""is_mobile"": true}" 19749,7,444,2017-04-10 22:14:29,http://quitzon.com/maynard,0.4531194766,188.61.178.42,"{""location"": ""SZ"", ""is_mobile"": false}" 19750,7,444,2017-02-10 17:11:43,http://braun.info/estel,0.0104729257,97.42.240.107,"{""location"": ""BR"", ""is_mobile"": true}" 19751,7,444,2017-04-15 22:48:22,http://jones.org/pete.crooks,0.3815869957,103.162.181.45,"{""location"": ""ML"", ""is_mobile"": true}" 19752,7,444,2017-04-13 19:24:08,http://gutmann.co/keeley_armstrong,0.0027265837,204.19.242.138,"{""location"": ""BZ"", ""is_mobile"": false}" 19753,7,444,2017-02-11 02:15:20,http://block.name/della,0.9065344500,89.24.63.246,"{""location"": ""VU"", ""is_mobile"": false}" 19754,7,444,2017-04-04 09:47:51,http://osinskiwalter.com/penelope.casper,0.4616732658,107.207.217.27,"{""location"": ""AE"", ""is_mobile"": true}" 19755,7,444,2017-06-04 17:36:03,http://kulachneider.info/lacy.watsica,0.9063828740,126.112.253.18,"{""location"": ""SZ"", ""is_mobile"": true}" 19756,7,445,2017-04-03 09:31:03,http://nikolausdamore.name/yvonne_hickle,0.3012898436,48.90.251.169,"{""location"": ""CC"", ""is_mobile"": false}" 19757,7,445,2017-01-04 08:48:02,http://abshire.co/abbie,0.4042448225,236.253.223.4,"{""location"": ""BY"", ""is_mobile"": false}" 19758,7,445,2017-01-13 11:12:46,http://skiles.com/breanne_orn,0.9138047927,231.149.122.128,"{""location"": ""WF"", ""is_mobile"": true}" 19759,7,445,2017-04-27 04:45:24,http://wehnerkuhn.org/dominic,0.1648245670,28.76.26.253,"{""location"": ""GG"", ""is_mobile"": false}" 19760,7,445,2017-05-13 10:29:31,http://ratke.net/tyler,0.0907613806,211.201.232.40,"{""location"": ""LY"", ""is_mobile"": true}" 19761,7,445,2017-03-01 08:43:55,http://hackettjast.org/thalia,0.9833397691,10.221.111.74,"{""location"": ""ID"", ""is_mobile"": false}" 19762,7,445,2017-05-20 23:45:27,http://turcottegleason.co/blaze,0.0408388082,177.58.30.195,"{""location"": ""NG"", ""is_mobile"": true}" 19763,7,445,2017-04-12 00:08:59,http://wisokylindgren.net/jayson.goldner,0.1029923915,15.139.114.18,"{""location"": ""PM"", ""is_mobile"": true}" 19764,7,445,2017-03-09 10:01:47,http://gorczany.name/buster,0.2966321910,68.139.224.49,"{""location"": ""TD"", ""is_mobile"": true}" 19765,7,445,2017-04-25 02:10:53,http://greenkulas.co/nathanial,0.4193785597,196.74.183.203,"{""location"": ""CU"", ""is_mobile"": true}" 19766,7,445,2017-02-02 19:57:39,http://reilly.biz/georgiana,0.4857991853,53.241.122.90,"{""location"": ""PM"", ""is_mobile"": true}" 19767,7,445,2017-03-20 04:57:16,http://pagac.biz/kelli,0.8460524923,100.2.186.95,"{""location"": ""ML"", ""is_mobile"": true}" 19768,7,445,2017-02-15 17:00:07,http://robelzemlak.name/buster.turcotte,0.3857207812,247.107.142.162,"{""location"": ""DJ"", ""is_mobile"": false}" 19769,7,445,2017-06-06 16:49:04,http://hartmannreilly.co/mozelle_jones,0.0114440873,251.167.5.220,"{""location"": ""DJ"", ""is_mobile"": false}" 19770,7,445,2016-12-22 10:35:27,http://metzleannon.biz/tatum_koelpin,0.6290479793,80.218.51.41,"{""location"": ""PH"", ""is_mobile"": true}" 19771,7,445,2017-02-26 21:16:04,http://herzog.co/beverly_ruel,0.4360360659,121.11.29.167,"{""location"": ""KZ"", ""is_mobile"": true}" 19772,7,445,2017-05-06 13:41:48,http://bogisich.name/tony,0.4785370274,83.228.146.89,"{""location"": ""PR"", ""is_mobile"": true}" 19773,7,445,2017-03-24 15:57:04,http://bernhard.org/nathanial,0.9948555592,200.155.231.139,"{""location"": ""CU"", ""is_mobile"": false}" 19774,7,445,2017-02-18 02:27:50,http://lind.info/josiah,0.5315466039,155.18.212.209,"{""location"": ""JM"", ""is_mobile"": true}" 19775,7,445,2017-05-18 03:13:30,http://tromp.co/adelia,0.2745135216,4.79.6.217,"{""location"": ""MN"", ""is_mobile"": false}" 19776,7,445,2017-05-22 07:46:04,http://hauck.com/jaden,0.6399019982,46.21.109.11,"{""location"": ""AT"", ""is_mobile"": true}" 19777,7,445,2017-06-11 15:06:20,http://wisozk.biz/alanis,0.1577075965,48.33.109.39,"{""location"": ""GD"", ""is_mobile"": false}" 19778,7,445,2017-05-31 19:55:16,http://rogahnfay.net/emely,0.6452665864,19.37.183.148,"{""location"": ""TL"", ""is_mobile"": true}" 19779,7,445,2017-03-15 00:05:23,http://krajcik.com/adaline,0.9295453876,23.227.253.144,"{""location"": ""IT"", ""is_mobile"": true}" 19780,7,445,2017-02-16 13:31:43,http://mann.name/marcos.witting,0.7604996812,72.177.31.252,"{""location"": ""ET"", ""is_mobile"": true}" 19781,7,445,2017-04-22 17:02:20,http://langworthboyer.com/jewell_stracke,0.1668950079,108.253.218.104,"{""location"": ""BL"", ""is_mobile"": true}" 19782,7,445,2017-05-11 11:35:13,http://kohler.biz/abelardo,0.1214966297,132.96.142.92,"{""location"": ""VE"", ""is_mobile"": false}" 19783,7,445,2017-01-17 12:34:15,http://schaeferkunde.com/junior.windler,0.0684231034,228.108.219.175,"{""location"": ""PF"", ""is_mobile"": false}" 19784,7,445,2017-05-31 11:47:50,http://shields.org/efrain_heathcote,0.4714970128,23.217.152.92,"{""location"": ""CR"", ""is_mobile"": false}" 19785,7,445,2017-01-06 07:21:02,http://reinger.io/theo_lowe,0.4325708336,26.177.73.49,"{""location"": ""SI"", ""is_mobile"": true}" 19786,7,445,2017-04-27 06:03:16,http://wunsch.org/damian,0.9679457345,220.219.126.198,"{""location"": ""MT"", ""is_mobile"": true}" 19787,7,445,2017-01-02 02:03:04,http://vonrueden.biz/mozelle,0.7560734882,251.72.80.54,"{""location"": ""MU"", ""is_mobile"": true}" 19788,7,445,2016-12-29 19:13:49,http://hauck.info/magnolia,0.5320740480,229.164.176.155,"{""location"": ""VC"", ""is_mobile"": false}" 19789,7,445,2016-12-14 14:44:14,http://sawayn.biz/ollie_hermiston,0.2104631325,9.249.46.68,"{""location"": ""MR"", ""is_mobile"": true}" 19790,7,445,2017-05-24 01:55:03,http://goldner.org/adell,0.0723474088,100.67.196.126,"{""location"": ""LA"", ""is_mobile"": true}" 19791,7,445,2017-04-27 04:58:29,http://wiza.org/ariel.gutkowski,0.0368458952,135.177.93.130,"{""location"": ""GS"", ""is_mobile"": false}" 19792,7,445,2017-04-03 14:11:53,http://halvorson.biz/katheryn.bruen,0.3138953040,225.95.70.199,"{""location"": ""BW"", ""is_mobile"": true}" 19793,7,445,2017-01-26 18:14:23,http://hilpert.org/wilhelmine.breitenberg,0.3350452787,211.238.206.179,"{""location"": ""ML"", ""is_mobile"": false}" 19794,7,445,2017-03-19 10:57:13,http://legros.co/heidi,0.4382614246,199.232.52.126,"{""location"": ""SG"", ""is_mobile"": true}" 19795,7,445,2017-05-08 01:59:19,http://reynolds.net/desmond.nitzsche,0.7264977776,38.102.66.118,"{""location"": ""CA"", ""is_mobile"": true}" 19796,7,445,2017-01-17 13:09:05,http://mcdermottmayert.biz/darryl_denesik,0.6466706685,114.91.144.100,"{""location"": ""LV"", ""is_mobile"": true}" 19797,7,445,2017-04-23 05:12:47,http://hyattmacejkovic.info/reese.hand,0.6050099217,244.93.227.79,"{""location"": ""US"", ""is_mobile"": false}" 19798,7,445,2017-02-13 08:39:07,http://christiansen.info/kristopher,0.4809983489,61.135.71.118,"{""location"": ""FI"", ""is_mobile"": true}" 19799,7,445,2017-05-26 09:06:14,http://powlowskihaag.biz/vladimir_kuphal,0.3404575072,98.95.63.178,"{""location"": ""DJ"", ""is_mobile"": false}" 19800,7,445,2017-01-23 16:27:25,http://dachreichel.info/jakayla,0.8293980364,98.18.79.213,"{""location"": ""MU"", ""is_mobile"": true}" 19801,7,445,2017-01-30 17:50:28,http://dubuquemccullough.biz/dulce,0.7809267441,101.99.68.247,"{""location"": ""SK"", ""is_mobile"": true}" 19802,7,445,2017-01-22 10:44:42,http://hauck.org/ariel,0.9565000817,202.174.11.30,"{""location"": ""ID"", ""is_mobile"": false}" 19803,7,445,2017-05-31 00:20:48,http://eichmann.io/rachael_reinger,0.0511797215,121.71.97.214,"{""location"": ""RS"", ""is_mobile"": true}" 19804,7,445,2017-03-24 11:43:39,http://damorenitzsche.io/roscoe.vonrueden,0.9479168968,223.129.51.243,"{""location"": ""CA"", ""is_mobile"": true}" 19805,7,445,2017-02-28 22:27:56,http://schroeder.co/demetrius,0.7499512721,120.190.119.203,"{""location"": ""ZM"", ""is_mobile"": false}" 19806,7,445,2017-01-26 21:37:51,http://spinka.name/imani_emmerich,0.6996261298,202.22.210.156,"{""location"": ""DZ"", ""is_mobile"": false}" 19807,7,445,2017-06-01 17:01:33,http://boscoryan.org/soledad,0.7182514237,177.77.100.32,"{""location"": ""KW"", ""is_mobile"": false}" 19808,7,445,2017-02-17 20:04:24,http://feeney.name/kaley.koch,0.5108435989,231.41.150.162,"{""location"": ""VI"", ""is_mobile"": false}" 19809,7,445,2017-01-02 23:39:38,http://buckridge.biz/tyreek_reichel,0.4864316964,144.91.91.56,"{""location"": ""ID"", ""is_mobile"": true}" 19810,7,445,2017-06-06 16:44:59,http://kihn.biz/trystan,0.0950261270,71.229.22.234,"{""location"": ""UA"", ""is_mobile"": true}" 19811,7,445,2017-01-13 22:13:02,http://collins.io/herminio,0.3765793834,174.59.249.210,"{""location"": ""KG"", ""is_mobile"": false}" 19812,7,445,2017-04-06 04:23:18,http://schmittpfannerstill.io/sylvan,0.4660054804,135.237.143.106,"{""location"": ""PF"", ""is_mobile"": false}" 19813,7,445,2017-05-01 20:50:50,http://wiza.org/dorcas,0.6969862193,254.87.121.87,"{""location"": ""GH"", ""is_mobile"": true}" 19814,7,445,2017-04-28 23:25:44,http://luettgen.com/mike,0.0581838053,152.47.158.48,"{""location"": ""BR"", ""is_mobile"": true}" 19815,7,445,2017-02-07 11:27:40,http://gerholdhammes.net/luz_weinat,0.0389785187,191.79.133.40,"{""location"": ""VI"", ""is_mobile"": false}" 19816,7,445,2017-05-09 20:24:54,http://sanfordokon.io/bell,0.5917260737,44.137.199.100,"{""location"": ""VI"", ""is_mobile"": false}" 19817,7,445,2017-05-28 10:00:01,http://wilderman.co/norene.tremblay,0.2974840065,248.194.38.227,"{""location"": ""CU"", ""is_mobile"": true}" 19818,7,445,2017-01-19 04:23:07,http://wisoky.info/gardner,0.9560449658,198.133.88.32,"{""location"": ""NG"", ""is_mobile"": false}" 19819,7,445,2016-12-24 09:35:57,http://kautzer.co/kris_hodkiewicz,0.3172784783,147.210.117.132,"{""location"": ""LS"", ""is_mobile"": false}" 19820,7,445,2017-05-01 04:41:55,http://hudsonmedhurst.net/jeramie,0.0523580566,137.59.91.243,"{""location"": ""CO"", ""is_mobile"": true}" 19821,7,445,2016-12-22 12:05:42,http://legroshuel.co/jonathon,0.1204294698,145.80.132.118,"{""location"": ""RW"", ""is_mobile"": false}" 19822,7,445,2017-06-05 11:19:42,http://stammbechtelar.com/darion.rolfson,0.6069907292,166.35.231.216,"{""location"": ""GG"", ""is_mobile"": true}" 19823,7,446,2017-04-05 05:28:22,http://cormierstehr.io/retha.hartmann,0.3667065477,71.19.106.211,"{""location"": ""WF"", ""is_mobile"": false}" 19824,7,446,2017-03-13 23:35:21,http://haneparker.co/linnie_kris,0.9547454232,203.145.82.186,"{""location"": ""AT"", ""is_mobile"": true}" 19825,7,446,2017-01-27 00:20:04,http://will.io/rollin,0.5814945055,156.129.62.207,"{""location"": ""BN"", ""is_mobile"": true}" 19826,7,446,2017-02-08 10:36:41,http://schimmelhaag.org/greta.schultz,0.1718125307,228.197.201.81,"{""location"": ""SR"", ""is_mobile"": true}" 19827,7,446,2017-04-22 01:53:13,http://daugherty.org/josephine_trantow,0.5788097915,53.49.38.232,"{""location"": ""MP"", ""is_mobile"": true}" 19828,7,446,2017-01-18 12:57:55,http://okeefemcclure.com/romaine,0.8886257430,155.44.77.91,"{""location"": ""MY"", ""is_mobile"": false}" 19829,7,446,2017-05-08 22:31:45,http://greenokon.biz/abagail.thiel,0.1704710643,140.182.237.9,"{""location"": ""IO"", ""is_mobile"": false}" 19830,7,446,2017-03-23 13:29:38,http://lang.com/hobart,0.6497802762,89.72.169.40,"{""location"": ""LB"", ""is_mobile"": true}" 19831,7,446,2017-04-13 20:15:28,http://schroederklein.biz/herminio_swaniawski,0.3538336421,228.19.222.60,"{""location"": ""GF"", ""is_mobile"": false}" 19832,7,446,2017-02-21 03:16:43,http://green.org/wilfred.pfannerstill,0.1265019424,81.233.148.73,"{""location"": ""GW"", ""is_mobile"": false}" 19833,7,446,2017-02-14 20:46:29,http://christiansen.io/akeem_aufderhar,0.6896098761,72.10.19.34,"{""location"": ""ZW"", ""is_mobile"": true}" 19834,7,446,2017-02-21 11:06:50,http://kelerstreich.info/malachi.murray,0.6059924951,113.228.93.73,"{""location"": ""TK"", ""is_mobile"": false}" 19835,7,446,2016-12-19 01:42:39,http://bergstrom.info/lucius,0.0021521802,165.59.18.79,"{""location"": ""BT"", ""is_mobile"": true}" 19836,7,446,2017-05-08 01:33:47,http://cummerataspencer.info/elian,0.5141668623,41.145.226.193,"{""location"": ""LR"", ""is_mobile"": true}" 19837,7,446,2017-01-23 04:58:11,http://cronin.io/georgiana,0.3329181166,103.124.57.96,"{""location"": ""IE"", ""is_mobile"": false}" 19838,7,446,2017-05-18 23:25:44,http://douglas.co/deven.keebler,0.5703182478,164.97.15.102,"{""location"": ""JP"", ""is_mobile"": false}" 19839,7,446,2017-06-07 02:02:39,http://kuvaliskeler.biz/eino.nader,0.6702293997,182.66.123.13,"{""location"": ""KG"", ""is_mobile"": false}" 19840,7,446,2017-05-09 12:37:34,http://ratke.io/jenifer.paucek,0.8629230367,9.235.185.95,"{""location"": ""VC"", ""is_mobile"": false}" 19841,7,446,2016-12-24 03:15:15,http://anderson.org/nat.yundt,0.1465649514,210.37.90.20,"{""location"": ""BQ"", ""is_mobile"": true}" 19842,7,446,2017-04-15 20:29:17,http://schumm.io/hilario.frami,0.9571323686,203.179.70.201,"{""location"": ""IO"", ""is_mobile"": false}" 19843,7,446,2016-12-29 20:05:49,http://bartell.biz/ora,0.2613275531,12.138.58.135,"{""location"": ""BG"", ""is_mobile"": false}" 19844,7,446,2017-04-22 11:14:00,http://collins.name/domingo.lowe,0.5981523593,180.92.32.56,"{""location"": ""ID"", ""is_mobile"": false}" 19845,7,446,2017-02-23 12:40:13,http://prosacco.co/corene.runte,0.2817227877,240.40.42.182,"{""location"": ""ID"", ""is_mobile"": true}" 19846,7,446,2017-04-19 19:02:58,http://gottliebstokes.biz/murphy.bogisich,0.8451707631,43.209.231.132,"{""location"": ""KN"", ""is_mobile"": true}" 19847,7,446,2017-02-11 10:51:20,http://borer.co/reuben,0.6474235584,223.61.102.158,"{""location"": ""SY"", ""is_mobile"": true}" 19848,7,446,2017-06-07 04:23:49,http://barrowskrajcik.com/marion_langworth,0.9598765492,35.244.112.11,"{""location"": ""ME"", ""is_mobile"": false}" 19849,7,446,2017-04-19 15:28:47,http://morarmueller.name/lavinia.block,0.2334105942,101.191.250.189,"{""location"": ""ET"", ""is_mobile"": false}" 19850,7,446,2017-01-12 05:52:18,http://rathdavis.net/eva.white,0.4719264193,132.112.20.68,"{""location"": ""BJ"", ""is_mobile"": true}" 19851,7,446,2017-03-27 22:10:48,http://herzoggrant.info/ethelyn.breitenberg,0.6298324373,90.102.241.179,"{""location"": ""KP"", ""is_mobile"": false}" 19852,7,446,2017-01-21 21:06:15,http://uptonbernier.co/tito.lynch,0.7123488604,118.113.55.145,"{""location"": ""UZ"", ""is_mobile"": false}" 19853,7,446,2017-05-12 00:44:10,http://rohanpadberg.biz/hobart,0.7575000473,147.228.41.201,"{""location"": ""MG"", ""is_mobile"": true}" 19854,7,446,2017-01-26 06:34:26,http://dickisawayn.info/jimmy,0.8031834849,254.3.234.220,"{""location"": ""CV"", ""is_mobile"": false}" 19855,7,446,2017-02-25 11:56:49,http://mueller.org/aylin.watsica,0.9631571700,129.91.244.245,"{""location"": ""BL"", ""is_mobile"": false}" 19856,7,446,2017-06-04 04:28:43,http://kuhn.org/geo_schuppe,0.9151701771,23.15.84.167,"{""location"": ""MR"", ""is_mobile"": true}" 19857,7,446,2017-01-18 05:08:27,http://pouros.io/wilbert,0.9358739806,174.235.233.156,"{""location"": ""GI"", ""is_mobile"": false}" 19858,7,446,2017-05-11 13:26:12,http://zulaufshanahan.io/jameson.blanda,0.1341643463,120.32.140.144,"{""location"": ""GT"", ""is_mobile"": true}" 19859,7,446,2017-04-11 13:51:56,http://nikolaus.co/lexi,0.3480416213,133.218.163.19,"{""location"": ""MK"", ""is_mobile"": false}" 19860,7,446,2017-02-17 06:50:16,http://lubowitz.com/claire,0.2768121295,116.244.104.28,"{""location"": ""NE"", ""is_mobile"": true}" 19861,7,446,2017-04-10 20:07:06,http://pfeffer.io/reina,0.1360233540,63.231.69.2,"{""location"": ""LR"", ""is_mobile"": true}" 19862,7,446,2017-02-16 04:37:32,http://jakubowskiyundt.info/keegan.padberg,0.1973569759,132.89.67.127,"{""location"": ""TL"", ""is_mobile"": false}" 19863,7,446,2017-04-11 19:55:02,http://nadertillman.org/jolie.johnson,0.5252482968,170.165.211.96,"{""location"": ""CX"", ""is_mobile"": false}" 19864,7,446,2017-05-12 12:17:36,http://moen.org/salvatore,0.2847419581,128.241.231.28,"{""location"": ""SC"", ""is_mobile"": true}" 19865,7,446,2017-03-16 22:49:46,http://trompheller.info/michale.gusikowski,0.0313025729,81.217.105.146,"{""location"": ""NA"", ""is_mobile"": false}" 19866,7,446,2017-02-06 21:53:05,http://erdmanspinka.net/aron_cartwright,0.9304760863,103.172.139.245,"{""location"": ""BH"", ""is_mobile"": true}" 19867,7,446,2017-01-15 14:26:40,http://gutkowski.name/rusty,0.0070562558,222.85.190.117,"{""location"": ""TR"", ""is_mobile"": false}" 19868,7,446,2016-12-24 06:43:30,http://davis.biz/eleanore_borer,0.1668731722,10.55.30.112,"{""location"": ""HK"", ""is_mobile"": false}" 19869,7,446,2017-04-05 23:36:43,http://ziemannmccullough.net/bryana,0.8867023634,121.163.121.208,"{""location"": ""KY"", ""is_mobile"": true}" 19870,7,446,2017-01-12 04:20:50,http://bartell.org/harrison.nicolas,0.1254613271,8.137.67.123,"{""location"": ""RS"", ""is_mobile"": false}" 19871,7,446,2017-06-08 06:01:02,http://balistrerigreenfelder.com/name.medhurst,0.2946881249,208.3.130.253,"{""location"": ""AL"", ""is_mobile"": true}" 19872,7,446,2016-12-14 18:11:50,http://hodkiewiczankunding.com/magnolia.bogan,0.4638797704,94.175.85.159,"{""location"": ""GN"", ""is_mobile"": true}" 19873,7,446,2017-04-30 16:28:17,http://waelchi.net/peggie_towne,0.8599837656,62.102.197.234,"{""location"": ""IO"", ""is_mobile"": true}" 19874,7,447,2017-05-29 16:23:53,http://torphy.info/dallin,0.4853187048,243.101.197.226,"{""location"": ""FK"", ""is_mobile"": false}" 19875,7,447,2017-01-11 05:30:32,http://dickens.com/benedict.mertz,0.6066742178,182.4.119.59,"{""location"": ""GY"", ""is_mobile"": false}" 19876,7,447,2017-02-28 08:51:46,http://nolan.net/linwood,0.9610919964,68.26.142.225,"{""location"": ""NZ"", ""is_mobile"": false}" 19877,7,447,2017-02-10 19:01:43,http://mante.com/filiberto,0.5485873090,193.169.73.200,"{""location"": ""BJ"", ""is_mobile"": true}" 19878,7,447,2016-12-31 08:36:22,http://hoppe.com/devante.quigley,0.2536187848,224.116.115.72,"{""location"": ""AU"", ""is_mobile"": true}" 19879,7,447,2017-02-12 21:30:04,http://okon.name/olin.huels,0.7775783715,132.89.14.81,"{""location"": ""GU"", ""is_mobile"": true}" 19880,7,447,2017-03-31 07:37:10,http://stamm.biz/laisha,0.3784012303,214.76.141.164,"{""location"": ""AT"", ""is_mobile"": false}" 19881,7,447,2016-12-26 22:15:55,http://bergnaumbarrows.com/edward,0.5780201590,144.92.206.251,"{""location"": ""LR"", ""is_mobile"": true}" 19882,7,447,2017-02-03 00:26:15,http://gleasonwalker.org/waino.maggio,0.4221100525,83.81.14.168,"{""location"": ""CL"", ""is_mobile"": true}" 19883,7,447,2017-01-29 17:10:38,http://ondrickaaufderhar.info/jadon_morar,0.2601126746,204.66.243.246,"{""location"": ""GP"", ""is_mobile"": true}" 19884,7,447,2017-01-03 18:18:11,http://corkery.net/cleora_moen,0.6422126380,29.20.211.58,"{""location"": ""AZ"", ""is_mobile"": true}" 19885,7,447,2017-05-10 04:12:02,http://halvorson.name/rigoberto,0.7058206102,77.43.57.161,"{""location"": ""IT"", ""is_mobile"": true}" 19886,7,447,2017-05-26 11:27:08,http://kihn.co/casimir,0.0104750345,28.205.61.149,"{""location"": ""CW"", ""is_mobile"": false}" 19887,7,447,2017-01-09 14:14:48,http://gorczany.com/erin,0.5684985402,85.29.35.23,"{""location"": ""SB"", ""is_mobile"": true}" 19888,7,447,2017-03-19 22:04:49,http://gorczany.org/melya.marquardt,0.2141204178,177.125.97.25,"{""location"": ""GN"", ""is_mobile"": true}" 19889,7,447,2017-05-28 14:22:23,http://labadie.co/ozella_willms,0.6073482084,32.239.189.120,"{""location"": ""AO"", ""is_mobile"": true}" 19890,7,447,2017-04-30 16:52:57,http://reichel.name/lincoln_damore,0.4872071079,110.214.183.142,"{""location"": ""LR"", ""is_mobile"": true}" 19891,7,447,2017-01-21 03:57:58,http://trantowohara.net/natasha.smitham,0.1395539437,160.237.11.141,"{""location"": ""MV"", ""is_mobile"": false}" 19892,7,447,2016-12-19 13:45:23,http://millerhagenes.info/kobe.langworth,0.7279191394,216.185.40.241,"{""location"": ""NR"", ""is_mobile"": false}" 19893,7,447,2017-05-03 04:29:51,http://beatty.net/eulalia.lockman,0.9614330260,14.222.252.109,"{""location"": ""UG"", ""is_mobile"": true}" 19894,7,447,2017-01-23 01:14:47,http://hodkiewicz.biz/christina_towne,0.2252828875,196.103.47.92,"{""location"": ""ZW"", ""is_mobile"": false}" 19895,7,447,2017-05-01 16:22:47,http://kreigerlesch.net/rodger.barrows,0.4883790397,202.80.91.107,"{""location"": ""IS"", ""is_mobile"": false}" 19896,7,447,2017-03-05 18:35:39,http://huelmurazik.net/vida,0.5043971977,82.51.72.145,"{""location"": ""BM"", ""is_mobile"": false}" 19897,7,447,2017-05-26 14:51:56,http://durganprohaska.net/jerod_tremblay,0.5449111217,22.33.161.241,"{""location"": ""TM"", ""is_mobile"": true}" 19898,7,448,2017-02-11 23:28:14,http://handturcotte.info/zachery.swift,0.7221189231,204.154.35.48,"{""location"": ""KR"", ""is_mobile"": true}" 19899,7,448,2017-03-26 09:50:56,http://mcculloughfritsch.com/maurice,0.3870437842,173.230.149.71,"{""location"": ""KW"", ""is_mobile"": false}" 19900,7,448,2017-02-20 19:36:02,http://hettingerpfeffer.biz/darryl,0.1737565404,217.69.124.81,"{""location"": ""TN"", ""is_mobile"": true}" 19901,7,448,2016-12-22 05:54:22,http://halvorson.co/beie,0.4867643153,57.11.171.90,"{""location"": ""LA"", ""is_mobile"": true}" 19902,7,448,2017-03-03 17:49:24,http://uptonzemlak.net/jody,0.9110570063,193.38.14.127,"{""location"": ""VC"", ""is_mobile"": true}" 19903,7,448,2017-04-24 08:46:09,http://lebsack.co/lexi.fahey,0.4971459958,124.105.237.43,"{""location"": ""GA"", ""is_mobile"": false}" 19904,7,448,2017-03-25 22:01:26,http://okon.org/laurence,0.9195223876,4.71.68.76,"{""location"": ""LS"", ""is_mobile"": true}" 19905,7,448,2017-02-21 23:01:17,http://schmelerhyatt.net/aileen.kutch,0.8250507837,200.11.135.57,"{""location"": ""DK"", ""is_mobile"": false}" 19906,7,448,2017-03-09 04:43:02,http://wymanchristiansen.net/santina,0.6306117401,10.200.193.16,"{""location"": ""CK"", ""is_mobile"": true}" 19907,7,448,2017-01-21 20:45:42,http://rowewisoky.org/mertie.simonis,0.7770441570,202.143.91.175,"{""location"": ""KZ"", ""is_mobile"": true}" 19908,7,448,2017-01-17 13:17:28,http://bechtelarsteuber.info/joaquin,0.7185593055,43.202.192.104,"{""location"": ""SH"", ""is_mobile"": true}" 19909,7,448,2017-02-18 07:57:22,http://bernhardnicolas.biz/etha,0.2642945121,214.81.58.14,"{""location"": ""KE"", ""is_mobile"": true}" 19910,7,448,2017-01-21 17:46:37,http://stoltenberg.net/camden.nitzsche,0.8832544668,243.21.48.15,"{""location"": ""AQ"", ""is_mobile"": false}" 19911,7,448,2017-06-09 01:43:55,http://simonis.com/gennaro.christiansen,0.5144353132,17.32.209.216,"{""location"": ""TH"", ""is_mobile"": true}" 19912,7,448,2017-04-15 03:11:26,http://mayert.biz/elena.leffler,0.6475385900,110.103.93.109,"{""location"": ""BG"", ""is_mobile"": false}" 19913,7,448,2017-01-19 14:09:50,http://mccluremraz.io/mable.lockman,0.0778479612,140.29.249.132,"{""location"": ""BS"", ""is_mobile"": false}" 19914,7,448,2017-01-18 21:46:42,http://greenfelderparker.co/craig_boyle,0.4222004224,54.135.216.156,"{""location"": ""EG"", ""is_mobile"": true}" 19915,7,448,2017-03-19 13:16:53,http://dicki.net/daphney.herzog,0.0518001983,85.108.236.181,"{""location"": ""DE"", ""is_mobile"": true}" 19916,7,448,2017-01-14 00:05:02,http://gleason.org/leopold,0.1833823797,230.132.95.192,"{""location"": ""GM"", ""is_mobile"": false}" 19917,7,448,2017-05-05 22:25:24,http://bayer.info/eric_renner,0.9466158122,200.113.76.50,"{""location"": ""VN"", ""is_mobile"": false}" 19918,7,448,2017-06-12 11:03:19,http://mcdermott.info/jettie.blick,0.2235157309,232.74.162.13,"{""location"": ""MC"", ""is_mobile"": true}" 19919,7,448,2017-05-28 01:27:38,http://armstrongdamore.name/sunny_keeling,0.7818557595,40.253.34.188,"{""location"": ""MX"", ""is_mobile"": true}" 19920,7,448,2017-01-25 02:11:10,http://tillman.info/marian_ortiz,0.0203748274,130.231.197.171,"{""location"": ""SX"", ""is_mobile"": false}" 19921,7,448,2017-03-18 02:26:15,http://brakus.com/tyrese,0.5434196556,232.45.93.109,"{""location"": ""MG"", ""is_mobile"": true}" 19922,7,448,2017-05-06 06:39:11,http://kuphal.io/jett_leffler,0.3592694092,32.194.102.189,"{""location"": ""BA"", ""is_mobile"": false}" 19923,7,448,2017-04-30 23:36:02,http://bartell.net/bryana,0.0341597810,111.107.176.230,"{""location"": ""CX"", ""is_mobile"": true}" 19924,7,448,2017-05-11 01:08:24,http://goldnerwest.io/greg,0.6683598520,196.213.66.158,"{""location"": ""EH"", ""is_mobile"": true}" 19925,7,448,2017-06-07 17:22:16,http://goodwin.biz/kyle.olson,0.4873611921,120.126.206.123,"{""location"": ""MW"", ""is_mobile"": true}" 19926,7,448,2017-02-13 22:02:50,http://hammes.io/susie.wunsch,0.7796090299,105.155.132.177,"{""location"": ""BE"", ""is_mobile"": false}" 19927,7,448,2017-03-29 01:13:00,http://dach.name/matt.thompson,0.9522301570,44.223.226.149,"{""location"": ""ML"", ""is_mobile"": false}" 19928,7,449,2017-03-16 12:59:39,http://oberbrunner.biz/marina.okeefe,0.2129032165,20.238.97.96,"{""location"": ""KH"", ""is_mobile"": true}" 19929,7,449,2017-03-04 07:10:57,http://roobstamm.biz/sherwood,0.7309105783,252.174.137.198,"{""location"": ""KI"", ""is_mobile"": false}" 19930,7,449,2017-04-21 14:07:57,http://harvey.org/lavonne,0.7884092824,190.36.54.4,"{""location"": ""AL"", ""is_mobile"": true}" 19931,7,449,2017-04-07 12:41:18,http://volkman.net/chris,0.9489798984,31.176.81.100,"{""location"": ""FM"", ""is_mobile"": true}" 19932,7,449,2016-12-19 16:42:13,http://hudson.info/elise_kautzer,0.2537434235,241.212.204.73,"{""location"": ""MK"", ""is_mobile"": true}" 19933,7,449,2016-12-17 11:01:37,http://greenholt.co/millie,0.9520291122,60.235.216.33,"{""location"": ""BB"", ""is_mobile"": false}" 19934,7,449,2017-04-18 20:33:44,http://leannon.biz/marty.braun,0.7310939616,51.100.173.64,"{""location"": ""RU"", ""is_mobile"": false}" 19935,7,449,2017-05-01 00:45:31,http://balistreridaugherty.io/kirsten,0.6265857798,233.187.216.125,"{""location"": ""VU"", ""is_mobile"": false}" 19936,7,449,2017-03-27 21:23:27,http://kulasjacobs.name/electa,0.4705129910,55.58.192.179,"{""location"": ""DE"", ""is_mobile"": true}" 19937,7,449,2017-01-18 22:04:24,http://watsicabogisich.com/kaleb.ebert,0.4965878322,176.221.168.183,"{""location"": ""GW"", ""is_mobile"": true}" 19938,7,449,2017-05-09 15:20:36,http://gerlach.org/cheyanne,0.2632492768,247.14.78.42,"{""location"": ""FK"", ""is_mobile"": false}" 19939,7,449,2017-01-11 13:08:28,http://jaskolskijones.biz/florine,0.1617192583,27.14.235.66,"{""location"": ""MU"", ""is_mobile"": true}" 19940,7,449,2017-02-28 01:24:56,http://abernathy.net/sherman_marvin,0.3822796959,80.169.20.96,"{""location"": ""PL"", ""is_mobile"": true}" 19941,7,449,2017-03-22 22:59:14,http://torpmills.org/karson,0.3629335657,86.165.124.3,"{""location"": ""MM"", ""is_mobile"": true}" 19942,7,449,2017-01-02 21:50:43,http://mcculloughwolff.co/domingo,0.4059612271,90.189.94.165,"{""location"": ""SD"", ""is_mobile"": true}" 19943,7,449,2017-05-08 22:05:35,http://mertz.io/verda,0.5092581650,225.219.87.80,"{""location"": ""BS"", ""is_mobile"": true}" 19944,7,449,2017-01-09 03:40:48,http://veum.org/lucio,0.0745150563,186.119.251.82,"{""location"": ""KN"", ""is_mobile"": false}" 19945,7,449,2017-03-13 02:20:24,http://dickibahringer.io/madalyn_hyatt,0.8778706778,168.240.180.106,"{""location"": ""GP"", ""is_mobile"": true}" 19946,7,449,2017-04-25 00:49:26,http://kuhn.org/ella.buckridge,0.5962503400,253.227.77.176,"{""location"": ""IL"", ""is_mobile"": false}" 19947,7,449,2017-05-22 01:41:58,http://nader.io/dimitri_vonrueden,0.5228558376,3.115.126.165,"{""location"": ""LC"", ""is_mobile"": true}" 19948,7,449,2017-03-22 02:50:29,http://dickenimonis.info/isabell,0.2877072266,151.169.214.9,"{""location"": ""PA"", ""is_mobile"": true}" 19949,7,449,2017-04-15 04:30:22,http://greenholt.co/wava_paucek,0.6775401341,81.74.204.197,"{""location"": ""GY"", ""is_mobile"": true}" 19950,7,449,2017-06-07 12:03:35,http://walkerjacobi.co/lenna.moore,0.8265959959,107.114.116.70,"{""location"": ""NU"", ""is_mobile"": true}" 19951,7,449,2016-12-21 21:14:33,http://feil.io/felix_dubuque,0.3083414280,134.110.122.222,"{""location"": ""US"", ""is_mobile"": false}" 19952,7,449,2017-01-31 07:39:17,http://gorczanykiehn.org/esteban_farrell,0.9328444768,13.3.224.113,"{""location"": ""BN"", ""is_mobile"": false}" 19953,7,449,2017-02-02 05:07:31,http://barton.biz/delbert,0.5715674792,99.203.214.251,"{""location"": ""CN"", ""is_mobile"": false}" 19954,7,450,2017-01-07 16:22:54,http://schultzkeebler.net/justus,0.2628296107,173.93.14.109,"{""location"": ""VC"", ""is_mobile"": true}" 19955,7,450,2017-01-09 08:59:28,http://harber.name/cordie_larson,0.7584927367,12.137.50.81,"{""location"": ""PY"", ""is_mobile"": true}" 19956,7,450,2017-06-03 09:53:42,http://nicolasko.info/moises,0.1882253383,118.250.160.91,"{""location"": ""CD"", ""is_mobile"": true}" 19957,7,450,2017-01-03 07:43:38,http://cristdurgan.info/cali.bergnaum,0.8794934766,201.27.253.235,"{""location"": ""SM"", ""is_mobile"": true}" 19958,7,450,2017-02-12 09:24:31,http://skilescrist.biz/stephania.ryan,0.9068629577,88.79.248.228,"{""location"": ""JE"", ""is_mobile"": true}" 19959,7,450,2017-03-01 10:58:36,http://ernserheaney.io/lavinia.reinger,0.4672795362,145.161.105.174,"{""location"": ""MA"", ""is_mobile"": false}" 19960,7,450,2017-05-01 06:59:44,http://ryanbradtke.com/evelyn,0.6674609555,145.233.41.254,"{""location"": ""AT"", ""is_mobile"": false}" 19961,7,450,2017-01-04 05:48:37,http://schiller.name/jed.quitzon,0.0650430190,188.108.51.139,"{""location"": ""FK"", ""is_mobile"": false}" 19962,7,450,2017-01-22 10:55:30,http://williamson.info/candelario_mccullough,0.0221284619,235.5.136.133,"{""location"": ""FO"", ""is_mobile"": false}" 19963,7,450,2017-01-18 05:44:28,http://sporer.info/katrina,0.1343091990,147.189.162.43,"{""location"": ""TT"", ""is_mobile"": false}" 19964,7,450,2017-05-20 06:39:37,http://vandervortledner.co/mallie,0.2871714297,60.99.248.53,"{""location"": ""GY"", ""is_mobile"": true}" 19965,7,450,2017-02-14 13:37:43,http://bernier.net/mia.conn,0.6312411967,7.133.154.137,"{""location"": ""NC"", ""is_mobile"": true}" 19966,7,450,2017-03-13 09:57:37,http://denesik.co/rowan.rau,0.8957951575,48.17.94.59,"{""location"": ""BT"", ""is_mobile"": true}" 19967,7,450,2017-02-06 17:13:48,http://purdy.com/maryam,0.3169947304,160.90.21.78,"{""location"": ""SY"", ""is_mobile"": false}" 19968,7,450,2017-04-15 16:48:56,http://steuber.name/ramiro,0.7774363054,193.110.105.32,"{""location"": ""CN"", ""is_mobile"": true}" 19969,7,450,2017-01-17 06:45:03,http://waelchihauck.io/rhianna.kertzmann,0.7321297659,107.10.125.9,"{""location"": ""MD"", ""is_mobile"": false}" 19970,7,450,2017-03-26 15:07:31,http://gutmannmarvin.com/cary,0.1171158047,40.32.249.191,"{""location"": ""CR"", ""is_mobile"": true}" 19971,7,450,2017-03-08 14:18:33,http://predovicfeeney.com/genesis_harber,0.1669495641,26.139.100.47,"{""location"": ""GN"", ""is_mobile"": false}" 19972,7,450,2017-01-29 19:35:24,http://heel.org/rebeca.brakus,0.7818870765,158.98.240.92,"{""location"": ""GQ"", ""is_mobile"": true}" 19973,7,450,2017-04-26 05:13:48,http://simonis.biz/geo_fisher,0.6449607116,103.169.94.22,"{""location"": ""SB"", ""is_mobile"": false}" 19974,7,450,2017-03-20 18:35:12,http://king.net/elinore_flatley,0.0874728442,144.130.208.204,"{""location"": ""MX"", ""is_mobile"": true}" 19975,7,450,2017-03-01 10:50:31,http://kovacek.biz/gunner.treutel,0.5981250087,162.76.20.177,"{""location"": ""VE"", ""is_mobile"": false}" 19976,7,450,2017-05-06 21:31:54,http://klockomckenzie.com/marian_langosh,0.8219454019,7.64.220.214,"{""location"": ""KR"", ""is_mobile"": true}" 19977,7,450,2017-03-01 00:05:12,http://johnston.name/tamara.sanford,0.5540924125,238.110.174.75,"{""location"": ""YT"", ""is_mobile"": false}" 19978,7,450,2016-12-27 20:29:04,http://jones.biz/domenic.steuber,0.1349992136,143.241.125.149,"{""location"": ""IT"", ""is_mobile"": false}" 19979,7,450,2017-06-08 23:55:24,http://kertzmannkunze.io/wallace_rohan,0.1519256474,65.183.55.3,"{""location"": ""GP"", ""is_mobile"": true}" 19980,7,450,2017-01-13 18:40:11,http://turnerfriesen.co/hillard_wilderman,0.9933214674,11.145.70.23,"{""location"": ""NR"", ""is_mobile"": true}" 19981,7,450,2017-02-10 11:55:07,http://beatty.biz/roxane.quitzon,0.9463700267,40.169.43.67,"{""location"": ""MX"", ""is_mobile"": true}" 19982,7,450,2017-02-06 10:41:31,http://corwin.name/santiago,0.4146215154,26.23.136.44,"{""location"": ""TR"", ""is_mobile"": true}" 19983,7,450,2017-04-10 05:54:43,http://willmscrona.name/emelia,0.9025451064,218.250.32.17,"{""location"": ""GQ"", ""is_mobile"": false}" 19984,7,450,2017-05-26 04:42:23,http://kautzerkovacek.org/marianne,0.7684219259,214.243.51.175,"{""location"": ""LT"", ""is_mobile"": false}" 19985,7,450,2017-04-20 04:22:57,http://wiza.io/baby,0.8067559627,13.30.40.80,"{""location"": ""KM"", ""is_mobile"": false}" 19986,7,450,2017-01-25 07:25:10,http://leffler.com/isidro_kutch,0.0566794168,206.193.41.44,"{""location"": ""LR"", ""is_mobile"": false}" 19987,7,450,2017-05-15 08:16:17,http://baumbachcrist.net/jacklyn_rolfson,0.4459026923,230.114.110.73,"{""location"": ""KI"", ""is_mobile"": true}" 19988,7,450,2017-03-26 15:26:08,http://lesch.name/zakary.mann,0.3461426800,143.163.140.179,"{""location"": ""NL"", ""is_mobile"": false}" 19989,7,450,2016-12-14 21:52:15,http://bernier.co/rey,0.3787848710,102.31.23.72,"{""location"": ""RS"", ""is_mobile"": true}" 19990,7,450,2016-12-17 13:28:20,http://barton.biz/lysanne.fay,0.1161963131,141.169.41.67,"{""location"": ""CR"", ""is_mobile"": false}" 19991,7,450,2017-02-04 05:23:05,http://carterhuel.org/ernesto_connelly,0.1065653865,230.207.4.35,"{""location"": ""VI"", ""is_mobile"": true}" 19992,7,450,2017-01-13 20:19:31,http://padberg.io/raphael,0.2628363620,246.248.243.123,"{""location"": ""BO"", ""is_mobile"": false}" 19993,7,450,2017-05-07 23:45:09,http://boyer.org/blaise_tremblay,0.5134728018,223.195.76.191,"{""location"": ""IE"", ""is_mobile"": false}" 19994,7,451,2017-04-12 12:39:30,http://pagac.biz/harold_gibson,0.5440416345,212.2.92.164,"{""location"": ""VE"", ""is_mobile"": false}" 19995,7,451,2017-01-20 13:22:01,http://connellyschoen.biz/kenneth,0.8806951850,251.200.138.3,"{""location"": ""TC"", ""is_mobile"": true}" 19996,7,451,2016-12-13 16:54:38,http://gerlachmitchell.io/shakira_weinat,0.8639770608,228.88.194.6,"{""location"": ""BO"", ""is_mobile"": false}" 19997,7,451,2017-04-19 16:23:13,http://harber.biz/roie.waelchi,0.6698739063,173.165.74.235,"{""location"": ""KZ"", ""is_mobile"": true}" 19998,7,451,2017-05-28 13:04:53,http://sawaynhintz.io/vita.gibson,0.6503405346,34.89.154.240,"{""location"": ""DM"", ""is_mobile"": true}" 19999,7,451,2017-02-16 13:57:37,http://bednar.info/bert_kaulke,0.7984595099,171.96.124.203,"{""location"": ""TM"", ""is_mobile"": false}" 20000,7,451,2017-04-02 01:40:15,http://littel.co/emmanuelle_feeney,0.6234458740,152.136.251.193,"{""location"": ""BH"", ""is_mobile"": true}" 20001,7,451,2017-03-25 21:58:07,http://schultzhirthe.io/rollin,0.6281701462,177.169.138.211,"{""location"": ""BW"", ""is_mobile"": false}" 20002,7,451,2017-06-01 12:15:55,http://steuber.name/reta.waelchi,0.2598265238,100.7.135.194,"{""location"": ""SN"", ""is_mobile"": true}" 20003,7,451,2017-03-31 19:28:09,http://pouros.net/laury_ryan,0.9459334348,183.43.66.197,"{""location"": ""BQ"", ""is_mobile"": true}" 20004,7,451,2017-04-18 13:40:15,http://kautzer.biz/bert_upton,0.1428861470,78.216.209.201,"{""location"": ""BM"", ""is_mobile"": false}" 20005,7,451,2017-03-11 00:01:56,http://hamill.biz/valentine.feil,0.0279833966,23.226.40.221,"{""location"": ""TK"", ""is_mobile"": false}" 20006,7,451,2017-05-11 10:15:37,http://wolfbernhard.name/freddy_krajcik,0.7809194400,89.238.228.207,"{""location"": ""GS"", ""is_mobile"": true}" 20007,7,451,2017-03-02 16:05:40,http://simonis.biz/layne.mcclure,0.5716131155,152.81.134.198,"{""location"": ""BI"", ""is_mobile"": true}" 20008,7,451,2017-06-01 23:56:06,http://parker.name/hillard_bayer,0.4223482891,110.40.212.149,"{""location"": ""AZ"", ""is_mobile"": false}" 20009,7,451,2017-05-09 07:07:53,http://ullrichernser.org/kailee,0.3798040057,108.138.168.33,"{""location"": ""TF"", ""is_mobile"": false}" 20010,7,451,2017-03-25 02:42:47,http://bins.io/guy.schoen,0.4846568059,160.188.202.14,"{""location"": ""AU"", ""is_mobile"": true}" 20011,7,451,2017-05-14 08:25:19,http://stehr.co/demarco,0.0175834674,141.111.245.171,"{""location"": ""KY"", ""is_mobile"": false}" 20012,7,451,2017-03-26 18:53:44,http://conn.name/reagan_grant,0.3773185506,186.128.101.171,"{""location"": ""GH"", ""is_mobile"": true}" 20013,7,451,2017-06-14 00:25:27,http://barrowkiles.org/francesco.huel,0.7857109341,37.223.53.117,"{""location"": ""MQ"", ""is_mobile"": false}" 20014,7,451,2017-06-09 11:17:33,http://stiedemann.name/abagail,0.8073841135,181.57.103.212,"{""location"": ""EG"", ""is_mobile"": false}" 20015,7,451,2017-02-20 11:48:27,http://bogan.info/arvid.schuster,0.5508920074,207.210.219.54,"{""location"": ""MD"", ""is_mobile"": true}" 20016,7,451,2017-05-06 11:52:25,http://rice.com/ryann_koepp,0.4455253465,34.66.152.14,"{""location"": ""ZW"", ""is_mobile"": true}" 20017,7,451,2017-03-18 10:09:02,http://koepp.biz/eileen,0.8225260899,10.150.71.225,"{""location"": ""MS"", ""is_mobile"": true}" 20018,7,451,2017-03-24 02:09:53,http://bosco.com/jevon_metz,0.9802093226,144.154.110.216,"{""location"": ""MH"", ""is_mobile"": false}" 20019,7,451,2017-02-20 01:52:39,http://murazikdooley.name/werner,0.0532413387,160.236.223.141,"{""location"": ""ST"", ""is_mobile"": true}" 20020,7,451,2017-02-01 00:42:31,http://koelpin.co/omer,0.4277268489,221.234.82.113,"{""location"": ""PG"", ""is_mobile"": false}" 20021,7,451,2017-03-11 12:20:47,http://bashirian.org/georgette.denesik,0.8328098194,232.41.37.85,"{""location"": ""RW"", ""is_mobile"": true}" 20022,7,451,2017-05-19 07:51:07,http://streich.co/rusty,0.1133982093,154.19.195.89,"{""location"": ""IS"", ""is_mobile"": true}" 20023,7,451,2017-03-08 02:55:52,http://reichel.com/hayden_sauer,0.8795992360,130.54.83.202,"{""location"": ""JP"", ""is_mobile"": true}" 20024,7,451,2017-01-11 20:53:26,http://zieme.io/carmel_abshire,0.5132044268,231.142.49.207,"{""location"": ""MY"", ""is_mobile"": false}" 20025,7,452,2017-03-28 08:14:27,http://homenick.io/chris,0.1424929230,104.56.149.69,"{""location"": ""KW"", ""is_mobile"": true}" 20026,7,452,2017-02-26 06:03:21,http://toy.org/fredy.wiegand,0.5638119411,209.84.251.163,"{""location"": ""DM"", ""is_mobile"": true}" 20027,7,452,2017-01-22 19:18:58,http://daugherty.org/clay,0.1887266200,17.107.66.137,"{""location"": ""MM"", ""is_mobile"": false}" 20028,7,452,2017-02-25 01:13:09,http://towneschinner.info/brenna.hegmann,0.2827146766,25.70.8.162,"{""location"": ""KR"", ""is_mobile"": true}" 20029,7,452,2017-03-26 22:42:53,http://heller.com/scot,0.1901443920,246.130.252.188,"{""location"": ""RE"", ""is_mobile"": false}" 20030,7,452,2016-12-30 22:50:52,http://considine.biz/hilda.kunze,0.3743318281,212.155.250.138,"{""location"": ""TZ"", ""is_mobile"": true}" 20031,7,452,2017-04-10 09:58:38,http://maggio.name/amina_casper,0.9970411664,144.8.152.117,"{""location"": ""JM"", ""is_mobile"": true}" 20032,7,452,2016-12-22 14:40:38,http://ryan.biz/judah.oreilly,0.1769774798,147.56.20.90,"{""location"": ""MM"", ""is_mobile"": true}" 20033,7,452,2017-04-26 09:30:32,http://morar.com/judge,0.7956837041,3.98.35.194,"{""location"": ""NC"", ""is_mobile"": true}" 20034,7,452,2017-04-25 02:01:17,http://mitchellgreen.io/eduardo,0.6209708346,70.101.181.9,"{""location"": ""PL"", ""is_mobile"": false}" 20035,7,452,2017-03-20 02:43:43,http://kilback.com/emmanuelle,0.8876753127,244.109.240.142,"{""location"": ""NC"", ""is_mobile"": false}" 20036,7,452,2017-01-10 10:18:39,http://ward.org/loyal,0.6300397214,122.105.51.182,"{""location"": ""BA"", ""is_mobile"": false}" 20037,7,452,2017-04-21 21:41:28,http://cain.com/eloisa,0.9798189819,251.144.218.67,"{""location"": ""IQ"", ""is_mobile"": true}" 20038,7,452,2017-06-14 03:45:08,http://mcdermott.io/dewayne,0.3147852888,59.218.36.164,"{""location"": ""ES"", ""is_mobile"": false}" 20039,7,452,2017-03-07 05:44:11,http://bruen.co/hipolito.schmitt,0.9447708429,2.222.213.242,"{""location"": ""BO"", ""is_mobile"": false}" 20040,7,452,2017-01-29 15:05:47,http://kovacekdibbert.net/geovanny.mayer,0.9911476526,44.105.208.242,"{""location"": ""TN"", ""is_mobile"": false}" 20041,7,452,2017-06-12 16:56:51,http://goodwin.name/madeline_shanahan,0.1999867589,186.83.175.198,"{""location"": ""TG"", ""is_mobile"": false}" 20042,7,452,2017-03-29 17:36:25,http://mohr.net/sofia.mills,0.2129477794,79.215.107.167,"{""location"": ""GI"", ""is_mobile"": false}" 20043,7,452,2017-03-25 21:08:10,http://durgan.io/lucas.bernhard,0.3192161724,52.136.243.56,"{""location"": ""CK"", ""is_mobile"": true}" 20044,7,452,2017-05-26 23:13:11,http://wuckertstehr.io/yvette_gerlach,0.1454363882,231.134.58.105,"{""location"": ""KW"", ""is_mobile"": false}" 20045,7,452,2017-02-20 01:14:35,http://binsgrady.co/jackeline,0.4683177115,212.18.175.155,"{""location"": ""HT"", ""is_mobile"": true}" 20046,7,452,2017-02-26 13:30:12,http://rowe.com/otha.rodriguez,0.8783672071,168.218.130.2,"{""location"": ""NA"", ""is_mobile"": false}" 20047,7,452,2017-01-20 14:02:30,http://krajcik.org/fay,0.8833221651,199.82.112.94,"{""location"": ""GY"", ""is_mobile"": false}" 20048,7,452,2017-01-06 12:43:05,http://kreiger.org/joana.senger,0.1355797065,184.231.155.41,"{""location"": ""ZW"", ""is_mobile"": false}" 20049,7,452,2017-01-31 13:48:26,http://lynch.org/barton,0.8994114064,90.122.20.184,"{""location"": ""SD"", ""is_mobile"": false}" 20050,7,452,2016-12-26 12:34:00,http://konopelski.name/holden_lesch,0.9383141546,229.19.65.146,"{""location"": ""SZ"", ""is_mobile"": true}" 20051,7,452,2017-04-09 20:17:58,http://muraziklubowitz.com/cary.jaskolski,0.6933992315,234.137.62.200,"{""location"": ""LS"", ""is_mobile"": true}" 20052,7,452,2017-03-11 01:08:29,http://walker.co/maggie,0.9656165120,179.222.108.36,"{""location"": ""GU"", ""is_mobile"": true}" 20053,7,452,2017-02-27 09:11:56,http://franecki.info/micah,0.3667443720,101.204.47.141,"{""location"": ""NC"", ""is_mobile"": false}" 20054,7,452,2017-04-30 12:28:25,http://anderson.name/dudley.jacobson,0.4348877830,59.217.171.212,"{""location"": ""KM"", ""is_mobile"": true}" 20055,7,452,2017-04-02 03:52:09,http://treutelfritsch.name/lonzo.crist,0.8379667496,242.38.184.102,"{""location"": ""AG"", ""is_mobile"": false}" 20056,7,452,2017-03-31 11:43:02,http://abbott.name/mikayla,0.5995996595,206.99.46.68,"{""location"": ""MD"", ""is_mobile"": false}" 20057,7,452,2016-12-19 17:22:41,http://zboncak.net/bart,0.9064631899,21.65.206.111,"{""location"": ""SC"", ""is_mobile"": true}" 20058,7,452,2017-02-25 19:37:58,http://greencronin.net/omari,0.8301432453,228.241.195.148,"{""location"": ""AL"", ""is_mobile"": false}" 20059,7,452,2017-03-07 10:07:32,http://blickhintz.net/lavern,0.7210194921,38.199.163.250,"{""location"": ""FJ"", ""is_mobile"": true}" 20060,7,452,2017-02-07 07:16:52,http://stromansatterfield.info/alexys_ritchie,0.3101273641,63.215.204.217,"{""location"": ""QA"", ""is_mobile"": false}" 20061,7,452,2017-05-09 00:23:23,http://waelchileannon.info/kendrick.hammes,0.3605867161,65.254.32.110,"{""location"": ""VU"", ""is_mobile"": false}" 20062,7,452,2017-06-06 02:05:51,http://schultzbatz.co/moshe.stroman,0.2869869633,227.210.116.17,"{""location"": ""AQ"", ""is_mobile"": false}" 20063,7,452,2017-01-11 04:38:55,http://mraz.org/alva_lebsack,0.1529891853,10.7.245.171,"{""location"": ""MS"", ""is_mobile"": true}" 20064,7,452,2017-01-21 12:57:28,http://olson.name/charlene.cartwright,0.1340270222,90.145.6.105,"{""location"": ""ID"", ""is_mobile"": true}" 20065,7,453,2017-02-11 23:33:24,http://cruickshank.org/griffin_nikolaus,0.4607441393,222.211.56.151,"{""location"": ""AQ"", ""is_mobile"": true}" 20066,7,453,2017-01-23 06:23:06,http://dibbert.com/brendon,0.8738310337,42.186.190.189,"{""location"": ""NZ"", ""is_mobile"": true}" 20067,7,453,2017-04-10 23:56:32,http://hermistonlynch.net/cayla,0.8746677256,119.4.153.47,"{""location"": ""KH"", ""is_mobile"": true}" 20068,7,453,2017-02-18 23:00:36,http://hillsziemann.io/scotty,0.9494868625,214.206.238.65,"{""location"": ""YT"", ""is_mobile"": true}" 20069,7,453,2017-06-08 22:24:32,http://damore.io/ezekiel,0.5863183098,235.196.136.201,"{""location"": ""CA"", ""is_mobile"": true}" 20070,7,453,2016-12-28 09:54:55,http://treutelgleason.co/izaiah_senger,0.3387897445,19.184.47.173,"{""location"": ""MZ"", ""is_mobile"": true}" 20071,7,453,2017-06-04 04:49:37,http://feil.net/ernesto.gleichner,0.2555174358,53.232.159.38,"{""location"": ""SG"", ""is_mobile"": false}" 20072,7,453,2017-03-29 18:46:23,http://nienow.org/demarco,0.4842839511,199.87.56.55,"{""location"": ""HK"", ""is_mobile"": false}" 20073,7,453,2017-04-07 14:19:14,http://medhurst.net/gilda.kovacek,0.0250914802,205.129.42.176,"{""location"": ""GW"", ""is_mobile"": true}" 20074,7,453,2017-03-20 20:26:32,http://lueilwitz.com/cierra_pfannerstill,0.0363293071,172.234.22.161,"{""location"": ""MC"", ""is_mobile"": true}" 20075,7,453,2017-05-23 23:06:00,http://gloverstroman.co/alvena,0.4313477803,3.4.144.111,"{""location"": ""PR"", ""is_mobile"": false}" 20076,7,453,2017-06-06 04:49:14,http://lynch.biz/roy,0.7594560095,37.16.219.77,"{""location"": ""AG"", ""is_mobile"": false}" 20077,7,453,2016-12-28 20:04:11,http://haleyrodriguez.info/shemar,0.8220964991,12.108.127.173,"{""location"": ""NG"", ""is_mobile"": false}" 20078,7,453,2017-05-23 23:59:08,http://mosciskiprohaska.co/freddie,0.2646724330,219.159.51.216,"{""location"": ""KP"", ""is_mobile"": false}" 20079,7,453,2017-03-09 09:57:59,http://weinat.net/gino,0.4402651395,81.5.154.45,"{""location"": ""BG"", ""is_mobile"": true}" 20080,7,453,2016-12-26 16:46:47,http://beatty.co/jacey,0.8939643885,175.172.25.172,"{""location"": ""YT"", ""is_mobile"": true}" 20081,7,453,2017-01-13 03:09:49,http://weber.info/keith.mitchell,0.6172424077,124.212.45.22,"{""location"": ""YE"", ""is_mobile"": false}" 20082,7,453,2017-03-10 10:52:57,http://goldner.biz/daniela,0.2947625392,31.229.62.9,"{""location"": ""BB"", ""is_mobile"": true}" 20083,7,453,2017-01-26 19:40:15,http://wiza.info/ray.zemlak,0.5645605074,209.123.102.34,"{""location"": ""EC"", ""is_mobile"": false}" 20084,7,453,2017-02-11 21:43:58,http://stokes.biz/margie,0.0361391675,83.235.222.176,"{""location"": ""GM"", ""is_mobile"": true}" 20085,7,453,2017-01-18 02:55:20,http://welchlangosh.co/terrence_lemke,0.0937365061,179.124.118.212,"{""location"": ""SE"", ""is_mobile"": true}" 20086,7,453,2017-03-13 09:24:24,http://boyer.com/cydney,0.0133174499,87.60.182.218,"{""location"": ""CG"", ""is_mobile"": false}" 20087,7,453,2017-02-17 16:27:21,http://simonis.info/teresa,0.7107941983,121.51.57.192,"{""location"": ""PK"", ""is_mobile"": false}" 20088,7,453,2017-03-29 07:57:45,http://berge.name/roman.dooley,0.6078462708,206.133.225.179,"{""location"": ""BT"", ""is_mobile"": false}" 20089,7,453,2017-04-27 13:15:54,http://hodkiewicz.name/kristoffer,0.8740169193,226.217.161.16,"{""location"": ""FO"", ""is_mobile"": true}" 20090,7,453,2017-05-16 01:03:23,http://lueilwitz.name/desiree,0.5253260393,139.205.203.122,"{""location"": ""SN"", ""is_mobile"": false}" 20091,7,453,2017-04-05 11:09:19,http://bernhardvolkman.io/elian,0.7442859638,16.251.182.238,"{""location"": ""NP"", ""is_mobile"": true}" 20092,7,453,2017-01-28 02:54:23,http://vonruedenrowe.biz/agustin_durgan,0.0309683423,180.92.144.6,"{""location"": ""MF"", ""is_mobile"": true}" 20093,7,453,2017-04-20 14:07:51,http://rippin.net/macey,0.7281940178,190.108.37.89,"{""location"": ""AO"", ""is_mobile"": true}" 20094,7,453,2017-04-17 15:03:23,http://connelly.io/deontae,0.2874486315,154.224.43.165,"{""location"": ""UM"", ""is_mobile"": false}" 20095,7,453,2017-04-18 03:16:55,http://andersonhickle.org/meghan_howe,0.3756739375,119.24.135.176,"{""location"": ""CZ"", ""is_mobile"": false}" 20096,7,453,2017-04-16 12:42:04,http://hammes.net/joanne_mertz,0.7094501702,92.109.153.225,"{""location"": ""SR"", ""is_mobile"": false}" 20097,7,453,2016-12-14 14:20:02,http://walshsawayn.com/arvid,0.7286981762,6.52.174.161,"{""location"": ""GU"", ""is_mobile"": false}" 20098,7,453,2017-05-02 12:27:18,http://block.org/yazmin,0.5549433935,192.176.87.127,"{""location"": ""VU"", ""is_mobile"": true}" 20099,7,453,2017-02-16 18:32:24,http://yost.io/kariane,0.0174151520,42.184.198.30,"{""location"": ""ML"", ""is_mobile"": true}" 20100,7,453,2017-06-01 11:42:32,http://weinatsawayn.com/justice,0.9500469916,142.152.194.26,"{""location"": ""CH"", ""is_mobile"": false}" 20101,7,453,2017-02-23 02:12:33,http://zemlakschowalter.io/verona,0.9096817670,110.46.78.178,"{""location"": ""ST"", ""is_mobile"": true}" 20102,7,453,2017-05-17 21:46:51,http://schummmacejkovic.info/lily,0.0416177013,61.166.104.84,"{""location"": ""RO"", ""is_mobile"": false}" 20103,7,453,2017-03-01 13:07:40,http://abbott.co/lauryn.schaefer,0.9427335595,217.98.78.23,"{""location"": ""VC"", ""is_mobile"": false}" 20104,7,453,2016-12-22 22:12:40,http://kunze.com/elmore_hauck,0.4648129703,149.185.194.230,"{""location"": ""BQ"", ""is_mobile"": true}" 20105,7,453,2017-04-26 08:41:08,http://wardschmitt.io/orlo,0.3691511606,112.62.34.132,"{""location"": ""NC"", ""is_mobile"": false}" 20106,7,453,2017-04-16 13:18:24,http://gleichner.info/carmen_oberbrunner,0.8216546483,193.40.172.107,"{""location"": ""CM"", ""is_mobile"": true}" 20107,7,453,2017-01-16 23:20:21,http://hagenesoreilly.net/fiona,0.5072559274,161.113.123.39,"{""location"": ""LB"", ""is_mobile"": false}" 20108,7,453,2017-01-01 10:53:10,http://padberg.net/evie_moriette,0.0733334962,9.139.211.196,"{""location"": ""QA"", ""is_mobile"": true}" 20109,7,453,2017-02-14 08:18:41,http://kuhlman.io/herminio_marquardt,0.9697277641,74.89.75.158,"{""location"": ""TK"", ""is_mobile"": true}" 20110,7,453,2017-01-25 19:44:07,http://robel.net/dagmar_schmidt,0.4481791212,152.136.208.111,"{""location"": ""IQ"", ""is_mobile"": true}" 20111,7,453,2017-03-17 05:40:33,http://bernier.biz/dorthy,0.0141854131,110.149.108.123,"{""location"": ""RU"", ""is_mobile"": true}" 20112,7,453,2016-12-14 14:34:35,http://glover.name/isac,0.9657659499,169.90.109.201,"{""location"": ""NA"", ""is_mobile"": true}" 20113,7,453,2016-12-20 09:09:38,http://blanda.biz/hobart_legros,0.9666211204,222.245.175.181,"{""location"": ""IE"", ""is_mobile"": true}" 20114,7,453,2017-06-01 10:42:40,http://daugherty.name/hulda,0.9142201514,119.237.23.215,"{""location"": ""GQ"", ""is_mobile"": true}" 20115,7,453,2017-06-10 08:19:20,http://boehm.net/clement.nicolas,0.3721970375,113.68.164.233,"{""location"": ""BL"", ""is_mobile"": true}" 20116,7,453,2017-01-22 13:32:37,http://greenfelder.info/will,0.8233371800,67.150.165.144,"{""location"": ""MC"", ""is_mobile"": false}" 20117,7,453,2017-01-15 16:23:31,http://stamm.com/carson.schuster,0.6302664365,103.237.246.95,"{""location"": ""PY"", ""is_mobile"": true}" 20118,7,453,2017-05-07 04:39:53,http://jacobskunze.co/allene,0.5180995441,237.188.52.214,"{""location"": ""CL"", ""is_mobile"": false}" 20119,7,453,2017-04-08 13:19:04,http://wuckert.com/fidel,0.1314310999,33.154.85.220,"{""location"": ""SN"", ""is_mobile"": false}" 20120,7,453,2017-02-15 22:13:34,http://konopelski.com/davon.weber,0.4867926662,233.61.33.12,"{""location"": ""CN"", ""is_mobile"": true}" 20121,7,453,2017-04-12 17:51:05,http://hermiston.info/alvina,0.7969459883,202.96.111.168,"{""location"": ""TJ"", ""is_mobile"": false}" 20122,7,453,2016-12-25 10:04:32,http://spinka.net/valentina.schaden,0.3701627648,157.107.232.208,"{""location"": ""BL"", ""is_mobile"": false}" 20123,7,453,2017-05-03 20:02:18,http://macejkovic.com/federico,0.9012986161,205.33.129.22,"{""location"": ""VG"", ""is_mobile"": false}" 20124,7,453,2017-04-30 15:14:45,http://miller.io/devyn,0.4057740475,190.97.40.116,"{""location"": ""TG"", ""is_mobile"": false}" 20125,7,453,2017-03-27 10:41:29,http://beer.co/charlene.rogahn,0.5842719900,52.141.53.60,"{""location"": ""MX"", ""is_mobile"": true}" 20126,7,453,2017-02-17 18:34:35,http://starkernser.net/mollie,0.8425656158,224.225.41.60,"{""location"": ""SN"", ""is_mobile"": true}" 20127,7,454,2017-04-13 07:05:21,http://dibberthansen.name/marjory,0.1565310800,175.23.149.200,"{""location"": ""DZ"", ""is_mobile"": true}" 20128,7,454,2017-05-16 06:57:51,http://bergstrom.co/darron,0.1641211412,249.146.58.46,"{""location"": ""EE"", ""is_mobile"": true}" 20129,7,454,2017-04-18 10:52:30,http://schusterbode.net/henriette.hodkiewicz,0.4074724308,74.204.226.195,"{""location"": ""SA"", ""is_mobile"": true}" 20130,7,454,2016-12-15 23:46:04,http://bogan.org/abdul,0.7283562527,136.121.132.56,"{""location"": ""TG"", ""is_mobile"": false}" 20131,7,454,2017-04-22 08:22:52,http://heathcote.org/alvena,0.1507634904,60.3.10.140,"{""location"": ""LA"", ""is_mobile"": false}" 20132,7,454,2017-04-14 11:17:02,http://leannon.info/donato,0.9600211824,171.17.164.11,"{""location"": ""BJ"", ""is_mobile"": true}" 20133,7,454,2017-01-15 19:54:52,http://lakin.name/isidro.hoppe,0.4312138574,148.70.128.2,"{""location"": ""IM"", ""is_mobile"": true}" 20134,7,454,2017-04-04 08:34:10,http://olsonbogisich.org/makayla,0.9244004021,34.80.49.22,"{""location"": ""ZM"", ""is_mobile"": false}" 20135,7,454,2017-01-11 15:35:16,http://nienowgottlieb.io/eileen,0.4714690129,196.119.31.94,"{""location"": ""KH"", ""is_mobile"": true}" 20136,7,454,2017-03-10 21:20:29,http://abernathy.info/lia.lockman,0.8269034430,175.183.36.33,"{""location"": ""DZ"", ""is_mobile"": false}" 20137,7,454,2017-01-22 11:20:21,http://keelingbradtke.co/lysanne,0.6742722468,139.25.70.53,"{""location"": ""CC"", ""is_mobile"": true}" 20138,7,454,2017-04-10 17:58:17,http://hodkiewicz.net/brisa,0.8697056549,126.160.151.222,"{""location"": ""SE"", ""is_mobile"": true}" 20139,7,454,2017-06-12 02:53:39,http://stanton.io/chris_homenick,0.8561129111,18.167.225.173,"{""location"": ""KY"", ""is_mobile"": true}" 20140,7,454,2017-02-11 02:17:28,http://kreigerlindgren.org/verla_hyatt,0.2556469804,15.43.108.172,"{""location"": ""RS"", ""is_mobile"": false}" 20141,7,454,2016-12-28 02:10:35,http://prohaskatowne.org/rosalinda,0.9100843647,200.164.17.3,"{""location"": ""MS"", ""is_mobile"": false}" 20142,7,454,2017-05-07 11:12:04,http://lindgrenmcglynn.info/alysa_harris,0.4608370729,2.248.130.83,"{""location"": ""KH"", ""is_mobile"": true}" 20143,7,454,2017-03-23 06:44:14,http://leffler.name/teie.koepp,0.6670016296,60.173.70.59,"{""location"": ""SH"", ""is_mobile"": false}" 20144,7,454,2017-02-21 00:10:01,http://weberwiza.name/clinton,0.8077196244,227.202.111.219,"{""location"": ""TJ"", ""is_mobile"": true}" 20145,7,454,2017-05-15 14:54:00,http://price.com/jannie_kuhic,0.9154760413,68.39.163.252,"{""location"": ""SO"", ""is_mobile"": false}" 20146,7,454,2017-03-01 16:36:49,http://dicki.name/coralie_hilpert,0.9028130353,77.229.5.127,"{""location"": ""SM"", ""is_mobile"": false}" 20147,7,454,2017-01-04 19:31:14,http://reichel.biz/yadira.lockman,0.1412285795,16.182.12.5,"{""location"": ""TG"", ""is_mobile"": true}" 20148,7,454,2017-02-17 02:07:33,http://connelly.name/sofia.johnson,0.1906207780,227.105.18.170,"{""location"": ""FR"", ""is_mobile"": false}" 20149,7,454,2017-01-06 23:47:12,http://leffler.co/tiana,0.4133583026,154.222.232.66,"{""location"": ""PE"", ""is_mobile"": false}" 20150,7,454,2017-02-06 01:01:47,http://parisian.io/hilda.nicolas,0.8138395467,174.244.203.152,"{""location"": ""CC"", ""is_mobile"": false}" 20151,7,454,2017-04-12 09:09:13,http://okunevadietrich.org/rahul.bosco,0.8798152699,205.237.88.214,"{""location"": ""GS"", ""is_mobile"": true}" 20152,7,454,2017-04-17 01:07:13,http://mertzbode.co/carlee,0.7923017025,228.100.172.5,"{""location"": ""HU"", ""is_mobile"": true}" 20153,7,454,2017-01-21 00:26:29,http://heathcote.org/autumn_wunsch,0.2164194298,171.182.153.223,"{""location"": ""SN"", ""is_mobile"": true}" 20154,7,454,2017-05-25 21:21:26,http://howell.biz/domenica.oberbrunner,0.6043591818,239.175.111.235,"{""location"": ""TK"", ""is_mobile"": true}" 20155,7,454,2016-12-22 21:39:39,http://upton.biz/tamara,0.9098246152,246.227.44.17,"{""location"": ""GB"", ""is_mobile"": false}" 20156,7,454,2017-02-17 14:05:13,http://jast.org/ottilie,0.3054036756,246.168.184.224,"{""location"": ""SO"", ""is_mobile"": false}" 20157,7,454,2017-05-22 07:28:36,http://stanton.com/laisha.bayer,0.5141584358,62.214.126.58,"{""location"": ""TO"", ""is_mobile"": true}" 20158,7,454,2017-04-22 08:57:23,http://boehm.name/shawna,0.9513482271,25.205.31.135,"{""location"": ""RW"", ""is_mobile"": false}" 20159,7,454,2017-02-06 11:43:11,http://wyman.info/eunice,0.7745252937,105.96.87.233,"{""location"": ""DO"", ""is_mobile"": true}" 20160,7,454,2017-02-28 05:59:06,http://kaulkedouglas.info/emmy,0.9069653640,32.253.237.101,"{""location"": ""MU"", ""is_mobile"": false}" 20161,7,454,2016-12-31 06:44:24,http://nicolas.biz/mellie.beier,0.8867022296,4.98.229.212,"{""location"": ""AM"", ""is_mobile"": false}" 20162,7,454,2016-12-31 17:33:00,http://goyette.io/benton.tillman,0.6652965874,62.192.181.112,"{""location"": ""MR"", ""is_mobile"": true}" 20163,7,454,2017-02-27 21:39:50,http://yundt.com/marlin,0.6366144501,76.186.52.242,"{""location"": ""SJ"", ""is_mobile"": false}" 20164,7,454,2017-02-19 17:39:56,http://moore.biz/claud,0.2178176715,167.201.47.209,"{""location"": ""PL"", ""is_mobile"": false}" 20165,7,454,2017-01-19 00:29:47,http://nader.info/brown.maggio,0.1294600693,203.107.35.59,"{""location"": ""EG"", ""is_mobile"": false}" 20166,7,454,2017-02-01 16:35:19,http://conn.biz/leanne.eichmann,0.4520506909,20.236.24.239,"{""location"": ""KZ"", ""is_mobile"": false}" 20167,7,454,2016-12-22 21:13:53,http://hayesbayer.com/wilmer.deckow,0.7940482083,222.108.45.126,"{""location"": ""KG"", ""is_mobile"": false}" 20168,7,454,2017-01-15 16:14:48,http://ritchie.co/alva.kshlerin,0.5832637203,87.161.181.70,"{""location"": ""BB"", ""is_mobile"": false}" 20169,7,454,2017-02-03 16:35:40,http://shields.co/arely,0.7162053792,224.6.159.152,"{""location"": ""KE"", ""is_mobile"": false}" 20170,7,454,2017-03-13 05:45:24,http://kozey.net/frederik,0.4262684987,184.77.234.63,"{""location"": ""FM"", ""is_mobile"": false}" 20171,7,454,2017-05-12 02:57:23,http://pfeffer.info/fritz_reichert,0.0864068490,222.97.116.85,"{""location"": ""KE"", ""is_mobile"": false}" 20172,7,455,2017-02-11 14:43:34,http://rippin.io/meaghan_morar,0.5586492005,186.117.211.248,"{""location"": ""MX"", ""is_mobile"": false}" 20173,7,455,2017-04-02 19:54:15,http://schneider.org/liana.lockman,0.0545489894,10.35.61.214,"{""location"": ""MK"", ""is_mobile"": false}" 20174,7,455,2017-05-19 10:58:14,http://goyette.org/fabian,0.1162703469,250.218.44.192,"{""location"": ""GE"", ""is_mobile"": false}" 20175,7,455,2017-03-31 02:27:59,http://walter.name/fritz_brown,0.1336207366,70.163.75.129,"{""location"": ""BM"", ""is_mobile"": false}" 20176,7,455,2017-02-25 14:35:56,http://hintz.co/nickolas,0.0219743970,126.141.127.72,"{""location"": ""BD"", ""is_mobile"": true}" 20177,7,455,2017-01-11 22:20:25,http://breitenberghowell.org/israel,0.5248609106,189.198.97.19,"{""location"": ""SD"", ""is_mobile"": true}" 20178,7,455,2017-01-06 05:25:27,http://schuster.name/salma,0.5663467380,196.37.73.113,"{""location"": ""RO"", ""is_mobile"": false}" 20179,7,455,2017-01-21 21:14:40,http://jacobi.com/fannie.harber,0.0674846197,17.145.118.221,"{""location"": ""BJ"", ""is_mobile"": false}" 20180,7,455,2017-06-05 16:39:32,http://crona.org/rex,0.3245762393,111.254.114.248,"{""location"": ""ME"", ""is_mobile"": true}" 20181,7,455,2017-04-28 18:11:09,http://streich.io/geraldine_parisian,0.9188970977,178.161.41.37,"{""location"": ""JM"", ""is_mobile"": false}" 20182,7,455,2017-01-30 08:32:42,http://watsica.info/jey,0.2582170519,53.199.45.244,"{""location"": ""ES"", ""is_mobile"": true}" 20183,7,455,2016-12-24 23:07:52,http://bartonwisoky.info/emmy,0.5895308892,253.135.243.212,"{""location"": ""HR"", ""is_mobile"": true}" 20184,7,455,2017-05-22 11:38:46,http://treutel.biz/lela,0.9098635947,175.2.199.25,"{""location"": ""ZM"", ""is_mobile"": false}" 20185,7,455,2017-06-05 06:27:19,http://blanda.name/guillermo_lind,0.4952850720,216.236.51.133,"{""location"": ""NE"", ""is_mobile"": true}" 20186,7,455,2017-02-13 04:09:43,http://murazikwalker.org/dale_murazik,0.0531864378,121.6.191.84,"{""location"": ""DM"", ""is_mobile"": false}" 20187,7,455,2017-04-05 02:40:21,http://bergstrom.name/keara_ruecker,0.4068115930,72.193.58.151,"{""location"": ""KR"", ""is_mobile"": true}" 20188,7,455,2017-02-20 08:20:23,http://hintz.biz/chandler,0.2119566411,22.2.75.220,"{""location"": ""BZ"", ""is_mobile"": false}" 20189,7,455,2017-03-30 00:14:56,http://bosco.io/amy.gerhold,0.6234410175,235.18.119.119,"{""location"": ""ZW"", ""is_mobile"": true}" 20190,7,455,2017-03-21 20:30:27,http://crist.org/lorna.lindgren,0.6854845192,54.32.112.199,"{""location"": ""EC"", ""is_mobile"": false}" 20191,7,455,2017-03-19 10:04:39,http://dubuquefritsch.net/chesley,0.3424936111,222.76.46.163,"{""location"": ""NF"", ""is_mobile"": false}" 20192,7,455,2017-02-08 21:27:24,http://littel.com/melisa.sipes,0.0500693963,221.75.173.160,"{""location"": ""FO"", ""is_mobile"": false}" 20193,7,455,2016-12-26 12:45:41,http://kohler.org/markus,0.9061969684,92.37.137.230,"{""location"": ""PY"", ""is_mobile"": false}" 20194,7,455,2017-03-20 10:00:42,http://swift.io/katharina,0.0735210372,29.252.55.236,"{""location"": ""GT"", ""is_mobile"": true}" 20195,7,455,2017-03-27 04:15:38,http://rau.info/hiram.nader,0.0560321725,237.207.56.23,"{""location"": ""SC"", ""is_mobile"": true}" 20196,7,455,2017-05-27 10:23:33,http://swift.name/jazmin.feest,0.2244240667,173.133.196.83,"{""location"": ""TD"", ""is_mobile"": false}" 20197,7,455,2017-02-24 07:40:27,http://dare.net/guy.walter,0.8386266935,122.186.83.161,"{""location"": ""JO"", ""is_mobile"": true}" 20198,7,455,2017-05-17 10:44:07,http://bailey.biz/ebba.corkery,0.3326412491,71.153.117.200,"{""location"": ""PW"", ""is_mobile"": true}" 20199,7,455,2017-01-01 22:33:49,http://rath.com/earl,0.5517931384,167.215.246.9,"{""location"": ""TR"", ""is_mobile"": false}" 20200,7,455,2017-02-13 07:49:39,http://lockman.name/kyle.marquardt,0.4932212990,17.104.175.12,"{""location"": ""VC"", ""is_mobile"": false}" 20201,7,455,2017-05-22 20:53:36,http://kelergutmann.net/nya.weinat,0.3892757578,228.14.192.215,"{""location"": ""BM"", ""is_mobile"": true}" 20202,7,456,2017-05-22 15:05:02,http://ferry.name/henry_muller,0.1743677088,194.238.34.247,"{""location"": ""AE"", ""is_mobile"": true}" 20203,7,456,2017-02-14 12:57:06,http://ortizrenner.org/carley_armstrong,0.0921361259,248.75.109.241,"{""location"": ""KW"", ""is_mobile"": false}" 20204,7,456,2017-05-17 22:43:31,http://schinner.com/liza,0.4699390676,100.238.35.87,"{""location"": ""KN"", ""is_mobile"": true}" 20205,7,456,2017-03-02 22:35:01,http://schiller.org/helmer,0.2887658823,241.179.75.48,"{""location"": ""KI"", ""is_mobile"": true}" 20206,7,456,2017-04-19 20:18:21,http://stiedemann.io/genesis,0.6559949842,76.206.74.82,"{""location"": ""JP"", ""is_mobile"": false}" 20207,7,456,2017-05-11 16:45:17,http://maggio.co/gwen,0.1717243132,92.156.4.117,"{""location"": ""CU"", ""is_mobile"": true}" 20208,7,456,2017-03-29 18:43:43,http://kingrutherford.org/nikolas,0.4812353150,245.207.169.176,"{""location"": ""FO"", ""is_mobile"": true}" 20209,7,456,2017-03-13 12:05:55,http://feil.co/adah,0.2548993199,50.72.184.115,"{""location"": ""TG"", ""is_mobile"": true}" 20210,7,456,2017-02-12 14:09:30,http://quigley.name/jordy,0.3070709074,11.27.240.148,"{""location"": ""CV"", ""is_mobile"": true}" 20211,7,456,2017-04-07 09:15:26,http://monahan.net/jazlyn_heaney,0.9937216016,86.21.175.7,"{""location"": ""SC"", ""is_mobile"": true}" 20212,7,456,2017-03-06 12:34:37,http://rosenbaumcrooks.biz/callie,0.7145944057,110.202.31.243,"{""location"": ""GU"", ""is_mobile"": false}" 20213,7,456,2017-03-14 07:19:38,http://stracke.com/lilly_deckow,0.4693122541,47.36.70.241,"{""location"": ""RU"", ""is_mobile"": true}" 20214,7,456,2017-03-10 10:03:08,http://reynolds.name/keira,0.0073902591,141.186.254.129,"{""location"": ""PL"", ""is_mobile"": false}" 20215,7,456,2017-04-07 08:57:44,http://streichbayer.org/sibyl_fritsch,0.3464954103,81.26.111.198,"{""location"": ""NC"", ""is_mobile"": false}" 20216,7,456,2017-04-15 20:39:39,http://reingermarks.biz/marshall.herman,0.4342721112,212.249.103.178,"{""location"": ""AI"", ""is_mobile"": true}" 20217,7,456,2017-05-02 23:42:43,http://white.org/charlie.roob,0.1703346430,81.106.83.95,"{""location"": ""PG"", ""is_mobile"": false}" 20218,7,456,2017-01-09 10:55:26,http://dietrich.name/toy,0.7323955486,118.130.183.2,"{""location"": ""BY"", ""is_mobile"": false}" 20219,7,456,2017-04-06 20:30:14,http://marvinbeatty.io/telly.yundt,0.2966351966,198.51.92.150,"{""location"": ""TG"", ""is_mobile"": false}" 20220,7,456,2017-01-21 18:47:11,http://wilkinson.name/kelsie,0.7618833124,173.159.190.125,"{""location"": ""MV"", ""is_mobile"": true}" 20221,7,456,2017-05-09 19:48:00,http://rohan.com/breana,0.2000101679,68.129.65.249,"{""location"": ""LU"", ""is_mobile"": false}" 20222,7,456,2017-02-23 13:41:55,http://gaylord.co/buford.bayer,0.7077819160,172.139.87.181,"{""location"": ""ZA"", ""is_mobile"": false}" 20223,7,456,2017-03-28 16:35:49,http://wisokysipes.name/jeffry,0.2865335906,39.18.118.88,"{""location"": ""MC"", ""is_mobile"": false}" 20224,7,456,2017-05-05 17:04:40,http://davis.biz/peter,0.2799199698,185.171.98.224,"{""location"": ""IE"", ""is_mobile"": true}" 20225,7,456,2017-02-20 17:15:16,http://blick.name/damaris.pouros,0.6257435742,137.144.94.187,"{""location"": ""CO"", ""is_mobile"": false}" 20226,7,456,2017-03-11 07:50:39,http://fadelkshlerin.org/lysanne.white,0.0526322371,76.207.175.144,"{""location"": ""MY"", ""is_mobile"": false}" 20227,7,456,2017-02-01 22:15:17,http://gerlach.net/lesly.reinger,0.8827255840,141.156.97.23,"{""location"": ""BT"", ""is_mobile"": false}" 20228,7,456,2016-12-29 11:23:40,http://haley.name/judy.halvorson,0.4398981705,202.180.48.99,"{""location"": ""HK"", ""is_mobile"": true}" 20229,7,456,2017-01-27 10:34:39,http://heaney.biz/aida.dicki,0.7029109805,219.97.98.27,"{""location"": ""MD"", ""is_mobile"": false}" 20230,7,456,2017-05-26 11:54:13,http://leannonlarson.com/nya.lynch,0.3203653383,161.220.47.224,"{""location"": ""CI"", ""is_mobile"": true}" 20231,7,456,2017-01-18 04:17:58,http://bins.org/owen.labadie,0.8962039380,68.138.32.142,"{""location"": ""CH"", ""is_mobile"": false}" 20232,7,456,2017-03-22 13:55:55,http://davichmidt.com/alia.king,0.6431296056,110.247.125.159,"{""location"": ""UM"", ""is_mobile"": false}" 20233,7,456,2017-04-26 16:12:15,http://hickle.name/shanna,0.3981366605,224.98.204.244,"{""location"": ""TM"", ""is_mobile"": false}" 20234,7,456,2017-04-20 23:28:22,http://dickinsoncollins.info/andreane.torp,0.8995006630,180.120.41.20,"{""location"": ""AW"", ""is_mobile"": true}" 20235,7,456,2017-03-25 15:48:36,http://daniel.info/sammie,0.6685174152,81.170.227.31,"{""location"": ""CZ"", ""is_mobile"": false}" 20236,7,456,2017-05-22 13:47:15,http://ondricka.org/mae,0.5953337714,127.96.50.206,"{""location"": ""IO"", ""is_mobile"": false}" 20237,7,456,2017-05-29 20:46:28,http://koelpin.org/caitlyn,0.6383575951,66.159.63.4,"{""location"": ""MF"", ""is_mobile"": true}" 20238,7,456,2017-02-10 19:53:05,http://conroyking.com/taurean,0.3126727791,183.77.167.42,"{""location"": ""AQ"", ""is_mobile"": false}" 20239,7,456,2017-05-09 14:44:36,http://gerhold.io/aron.bruen,0.8005662347,177.90.247.182,"{""location"": ""GI"", ""is_mobile"": false}" 20240,7,456,2017-05-18 06:20:36,http://strackeherman.io/catharine,0.8874757705,168.82.92.216,"{""location"": ""PT"", ""is_mobile"": false}" 20241,7,456,2017-05-13 23:45:11,http://gerhold.net/kavon_bernier,0.4526218795,205.149.94.115,"{""location"": ""BM"", ""is_mobile"": true}" 20242,7,456,2017-01-22 20:52:12,http://gibsoncremin.co/rosie,0.2062231182,183.24.93.153,"{""location"": ""NU"", ""is_mobile"": false}" 20243,7,456,2017-04-23 00:01:02,http://welchschinner.net/monserrat_quitzon,0.3895392350,102.171.252.12,"{""location"": ""MU"", ""is_mobile"": true}" 20244,7,456,2016-12-14 07:31:55,http://oconner.com/rod_pacocha,0.6147645490,12.34.42.229,"{""location"": ""DE"", ""is_mobile"": true}" 20245,7,456,2017-03-01 07:45:04,http://brekkegrimes.com/sincere,0.9589009777,40.47.142.33,"{""location"": ""LS"", ""is_mobile"": false}" 20246,7,456,2017-01-30 12:13:11,http://marks.com/margaret,0.0277538902,211.203.106.251,"{""location"": ""OM"", ""is_mobile"": false}" 20247,7,456,2017-04-30 12:46:26,http://kulas.name/lizzie,0.7284399782,77.114.195.212,"{""location"": ""MG"", ""is_mobile"": false}" 20248,7,456,2017-03-15 00:04:20,http://conroy.com/aracely,0.3977206178,160.203.51.39,"{""location"": ""SM"", ""is_mobile"": true}" 20249,7,456,2017-02-15 12:51:08,http://schultz.biz/kasandra,0.6958528259,103.228.27.179,"{""location"": ""MG"", ""is_mobile"": true}" 20250,7,456,2017-03-09 07:39:12,http://wymandeckow.net/lisandro_smith,0.5488899606,101.171.193.128,"{""location"": ""TT"", ""is_mobile"": false}" 20251,7,456,2017-01-18 21:25:17,http://haag.name/dusty.heaney,0.6282172027,239.251.83.34,"{""location"": ""BZ"", ""is_mobile"": false}" 20252,7,456,2017-03-04 22:14:14,http://schultz.com/destinee.farrell,0.5688629798,66.59.146.239,"{""location"": ""QA"", ""is_mobile"": true}" 20253,7,456,2016-12-20 03:30:54,http://watsicadaugherty.org/herminio,0.6364837842,76.231.241.216,"{""location"": ""OM"", ""is_mobile"": false}" 20254,7,456,2017-04-10 07:47:13,http://nader.io/elian,0.3942892598,175.240.39.68,"{""location"": ""BI"", ""is_mobile"": false}" 20255,7,456,2017-05-18 23:09:11,http://schillerdaniel.com/marcella,0.7220398627,19.189.143.192,"{""location"": ""SY"", ""is_mobile"": false}" 20256,7,456,2017-02-21 12:52:48,http://sauer.org/lisa.boehm,0.8721457538,116.175.231.204,"{""location"": ""SO"", ""is_mobile"": false}" 20257,7,457,2017-03-07 00:43:54,http://schaden.biz/luciano.hermann,0.6167741499,106.152.135.147,"{""location"": ""IQ"", ""is_mobile"": false}" 20258,7,457,2017-03-31 09:25:06,http://damoresanford.biz/bailey,0.8873429687,44.95.126.10,"{""location"": ""MK"", ""is_mobile"": false}" 20259,7,457,2016-12-16 05:29:35,http://ruel.io/scot,0.8724430277,36.158.239.110,"{""location"": ""AZ"", ""is_mobile"": false}" 20260,7,457,2017-02-05 20:18:07,http://metzgleason.biz/branson,0.9420530928,143.154.95.239,"{""location"": ""TR"", ""is_mobile"": true}" 20261,7,457,2016-12-24 09:57:37,http://jerde.io/maxime_veum,0.6225101476,216.33.154.253,"{""location"": ""MT"", ""is_mobile"": true}" 20262,7,457,2017-05-05 02:31:41,http://kiehn.name/giovanni.champlin,0.8721173080,111.182.192.50,"{""location"": ""HU"", ""is_mobile"": true}" 20263,7,457,2017-04-14 10:54:47,http://welch.org/mekhi_gislason,0.7819502495,17.123.100.173,"{""location"": ""EE"", ""is_mobile"": false}" 20264,7,457,2017-05-05 11:26:02,http://olson.net/hattie,0.4551393156,225.243.57.232,"{""location"": ""BA"", ""is_mobile"": true}" 20265,7,457,2017-06-13 02:03:58,http://welch.name/maggie,0.6292586638,183.120.224.220,"{""location"": ""KZ"", ""is_mobile"": false}" 20266,7,457,2017-03-27 16:09:31,http://armstrong.biz/devan,0.3990196149,208.82.75.160,"{""location"": ""AT"", ""is_mobile"": false}" 20267,7,457,2017-02-02 03:38:11,http://goldner.biz/kolby_weinat,0.9789102236,239.152.236.228,"{""location"": ""QA"", ""is_mobile"": false}" 20268,7,457,2017-05-02 23:47:28,http://wolftremblay.info/mafalda_ohara,0.3745720540,140.193.83.47,"{""location"": ""BO"", ""is_mobile"": false}" 20269,7,457,2017-05-31 22:38:46,http://gerhold.org/clyde.schmidt,0.1667452667,32.9.55.254,"{""location"": ""BZ"", ""is_mobile"": false}" 20270,7,457,2017-05-04 07:17:09,http://streich.io/shakira,0.0227991730,211.58.77.243,"{""location"": ""BQ"", ""is_mobile"": true}" 20271,7,457,2017-02-18 05:23:42,http://gaylord.info/isai.satterfield,0.8544580987,91.98.247.48,"{""location"": ""NO"", ""is_mobile"": true}" 20272,7,457,2017-04-26 17:21:07,http://ko.co/timmy,0.7454016816,87.218.111.101,"{""location"": ""BR"", ""is_mobile"": true}" 20273,7,457,2017-04-14 18:55:35,http://wintheiser.co/araceli.jacobson,0.4213046456,119.75.219.19,"{""location"": ""AM"", ""is_mobile"": false}" 20274,7,457,2017-01-24 06:44:21,http://mccullough.org/mathias,0.9141970249,41.95.183.185,"{""location"": ""SJ"", ""is_mobile"": true}" 20275,7,457,2017-05-30 03:43:50,http://jenkins.biz/pearl,0.9057703429,216.248.216.61,"{""location"": ""KH"", ""is_mobile"": true}" 20276,7,457,2017-05-11 20:14:55,http://nitzschepfannerstill.info/yasmin,0.4880628804,40.119.248.137,"{""location"": ""NP"", ""is_mobile"": false}" 20277,7,457,2017-01-04 18:59:46,http://west.name/patricia.beier,0.8904859017,135.218.230.44,"{""location"": ""HM"", ""is_mobile"": false}" 20278,7,457,2017-01-07 09:11:12,http://crookscrooks.com/kaylie,0.0620780146,163.146.208.25,"{""location"": ""HU"", ""is_mobile"": true}" 20279,7,457,2017-04-24 06:09:23,http://cain.biz/lucious,0.9276977672,249.72.132.230,"{""location"": ""EE"", ""is_mobile"": true}" 20280,7,457,2017-01-04 06:42:55,http://schneider.io/malinda_kozey,0.4593138255,214.217.62.17,"{""location"": ""GY"", ""is_mobile"": true}" 20281,7,457,2017-04-14 17:33:30,http://ruecker.org/leopoldo,0.8059134669,162.65.240.229,"{""location"": ""KE"", ""is_mobile"": true}" 20282,7,457,2017-01-19 18:06:36,http://cremin.biz/dewitt.mckenzie,0.8035038445,253.153.102.143,"{""location"": ""ZM"", ""is_mobile"": true}" 20283,7,457,2017-03-20 12:29:35,http://dickens.org/evert,0.0600358872,125.205.164.6,"{""location"": ""EG"", ""is_mobile"": true}" 20284,7,457,2017-05-12 05:40:02,http://stehr.co/carrie,0.2968682201,229.54.234.30,"{""location"": ""AZ"", ""is_mobile"": true}" 20285,7,457,2017-02-07 10:55:20,http://leffler.co/ethan,0.8722179280,16.112.41.70,"{""location"": ""TR"", ""is_mobile"": false}" 20286,7,457,2017-04-19 18:19:55,http://reilly.io/ewald.dooley,0.3619664180,224.7.103.189,"{""location"": ""ZM"", ""is_mobile"": false}" 20287,7,457,2017-05-02 09:19:37,http://kuphalconn.biz/dedrick.ziemann,0.4901530372,111.177.166.254,"{""location"": ""MX"", ""is_mobile"": true}" 20288,7,457,2017-05-10 08:27:10,http://boyle.com/meaghan_lueilwitz,0.8785970000,245.198.25.203,"{""location"": ""BS"", ""is_mobile"": false}" 20289,7,457,2017-03-21 13:45:42,http://nicolas.co/enoch,0.2733629075,30.102.6.168,"{""location"": ""PR"", ""is_mobile"": false}" 20290,7,457,2017-06-07 07:05:42,http://grady.name/jayden.stokes,0.9570592343,22.199.68.223,"{""location"": ""EH"", ""is_mobile"": false}" 20291,7,457,2017-01-19 22:06:57,http://schimmelwiegand.com/frances_lowe,0.3532811451,42.23.117.175,"{""location"": ""RS"", ""is_mobile"": true}" 20292,7,457,2017-04-19 03:29:14,http://olsonschinner.name/nettie,0.1331167964,57.82.127.253,"{""location"": ""AX"", ""is_mobile"": true}" 20293,7,457,2017-02-17 21:03:40,http://price.name/trycia.crist,0.1876498991,225.107.213.133,"{""location"": ""PG"", ""is_mobile"": false}" 20294,7,457,2017-03-12 10:19:30,http://schaefer.name/annamarie.jast,0.0112510155,240.161.4.238,"{""location"": ""GD"", ""is_mobile"": true}" 20295,7,457,2017-05-20 02:24:20,http://padbergromaguera.name/lambert.morar,0.6107560702,217.96.134.225,"{""location"": ""CH"", ""is_mobile"": true}" 20296,7,457,2017-02-09 18:34:50,http://homenickgreen.com/glenna,0.3836935342,88.47.178.143,"{""location"": ""UG"", ""is_mobile"": false}" 20297,7,457,2017-01-06 18:37:02,http://jaskolski.io/amaya,0.2778524067,56.108.176.44,"{""location"": ""MC"", ""is_mobile"": true}" 20298,7,457,2016-12-24 07:44:53,http://haag.co/mabel,0.5197233285,3.27.187.104,"{""location"": ""DE"", ""is_mobile"": true}" 20299,7,457,2017-02-28 01:12:36,http://beier.co/delphia,0.0173243047,241.132.9.241,"{""location"": ""BL"", ""is_mobile"": true}" 20300,7,457,2017-03-19 08:50:59,http://okeefe.com/helmer.bechtelar,0.5460240228,168.62.10.175,"{""location"": ""LB"", ""is_mobile"": false}" 20301,7,457,2017-01-28 18:00:07,http://considinepurdy.co/myrtis.mcdermott,0.0128894437,117.218.17.2,"{""location"": ""NP"", ""is_mobile"": false}" 20302,7,457,2017-04-29 20:03:42,http://kozey.name/urban,0.6191559116,42.2.222.69,"{""location"": ""PY"", ""is_mobile"": false}" 20303,7,457,2017-06-10 01:43:34,http://fahey.co/shaylee_welch,0.4660586600,106.241.135.21,"{""location"": ""SJ"", ""is_mobile"": false}" 20304,7,457,2016-12-26 14:02:37,http://johnsonjohns.com/alfonzo_goldner,0.3881081057,171.114.83.75,"{""location"": ""TO"", ""is_mobile"": false}" 20305,7,457,2017-04-23 06:04:05,http://denesik.com/kaylah_monahan,0.6611958271,139.148.228.84,"{""location"": ""GY"", ""is_mobile"": true}" 20306,7,457,2017-05-07 01:00:13,http://macejkovicgoldner.org/malcolm,0.2656384495,136.119.15.31,"{""location"": ""KP"", ""is_mobile"": false}" 20307,7,457,2017-03-29 18:08:26,http://crist.com/francesca_mertz,0.5860774145,20.212.156.55,"{""location"": ""VE"", ""is_mobile"": true}" 20308,7,457,2017-02-25 19:02:55,http://gleichner.biz/camryn,0.1203447290,225.28.112.53,"{""location"": ""VC"", ""is_mobile"": true}" 20309,7,457,2017-01-23 11:41:24,http://orn.net/modesto_mayert,0.5999962802,65.72.5.141,"{""location"": ""VC"", ""is_mobile"": true}" 20310,7,458,2017-01-28 13:22:17,http://prosacco.biz/rogers,0.0657405008,24.241.139.171,"{""location"": ""ZM"", ""is_mobile"": false}" 20311,7,458,2016-12-19 13:45:30,http://lowe.name/francis_moore,0.4254576537,130.137.225.68,"{""location"": ""NO"", ""is_mobile"": false}" 20312,7,458,2017-02-07 15:01:35,http://kunzemcclure.net/cody,0.3207187130,253.58.81.202,"{""location"": ""CM"", ""is_mobile"": false}" 20313,7,458,2017-06-10 01:36:45,http://bauchsimonis.io/wilbert,0.8111343529,76.227.69.56,"{""location"": ""KM"", ""is_mobile"": true}" 20314,7,458,2017-03-13 01:12:11,http://grimes.name/rebeka,0.8409745914,97.33.158.221,"{""location"": ""CI"", ""is_mobile"": false}" 20315,7,458,2017-03-20 02:03:46,http://kreigermiller.info/marvin,0.3896437299,253.35.136.14,"{""location"": ""JM"", ""is_mobile"": false}" 20316,7,458,2017-02-25 06:40:49,http://senger.com/candace.tillman,0.2352081350,97.25.26.72,"{""location"": ""RS"", ""is_mobile"": true}" 20317,7,458,2017-02-02 17:49:01,http://schaden.info/mathew_crooks,0.8440698574,54.186.189.60,"{""location"": ""ID"", ""is_mobile"": true}" 20318,7,458,2017-04-07 08:55:27,http://stantonwillms.biz/spencer_bernier,0.7410133146,61.214.217.204,"{""location"": ""QA"", ""is_mobile"": false}" 20319,7,458,2017-05-27 06:31:30,http://gleason.name/karine,0.9333776410,72.231.242.13,"{""location"": ""RW"", ""is_mobile"": false}" 20320,7,458,2017-06-06 06:18:27,http://oconner.info/general.sawayn,0.5574738281,156.254.208.187,"{""location"": ""TV"", ""is_mobile"": true}" 20321,7,458,2017-02-18 23:54:52,http://welch.biz/franz.bayer,0.7232154333,229.166.229.39,"{""location"": ""IN"", ""is_mobile"": true}" 20322,7,458,2017-02-24 06:25:46,http://williamson.io/cristopher,0.5693670831,179.156.11.32,"{""location"": ""BA"", ""is_mobile"": false}" 20323,7,458,2017-06-09 01:30:16,http://hirthe.biz/jazmyn,0.9590525178,213.11.42.132,"{""location"": ""TD"", ""is_mobile"": false}" 20324,7,458,2016-12-19 13:03:04,http://hermanbosco.com/pierre,0.0402623595,60.137.89.108,"{""location"": ""CH"", ""is_mobile"": false}" 20325,7,458,2017-01-19 19:41:45,http://mayer.biz/uriah_stamm,0.8645220265,248.96.15.204,"{""location"": ""SY"", ""is_mobile"": true}" 20326,7,458,2017-01-17 09:06:03,http://kuvalis.info/rosemary,0.3730481781,116.40.117.151,"{""location"": ""DK"", ""is_mobile"": false}" 20327,7,458,2017-06-01 06:08:13,http://schulist.co/gilberto_botsford,0.9925948147,131.5.32.174,"{""location"": ""UZ"", ""is_mobile"": false}" 20328,7,458,2017-02-23 02:18:38,http://walker.name/winifred_wunsch,0.2924607865,128.162.220.116,"{""location"": ""GD"", ""is_mobile"": false}" 20329,7,458,2017-03-30 05:41:15,http://eichmann.co/ali.witting,0.0678543480,248.128.86.219,"{""location"": ""TT"", ""is_mobile"": false}" 20330,7,458,2017-05-18 18:33:12,http://raucorwin.name/lea,0.6077144856,115.123.125.32,"{""location"": ""BM"", ""is_mobile"": true}" 20331,7,458,2017-04-11 14:24:03,http://ebertferry.co/alphonso,0.3853444098,12.26.105.154,"{""location"": ""BI"", ""is_mobile"": false}" 20332,7,458,2017-02-17 15:07:50,http://halvorsonweinat.org/jairo.lebsack,0.1208609510,145.95.225.252,"{""location"": ""TH"", ""is_mobile"": true}" 20333,7,458,2017-03-31 14:27:00,http://gibson.biz/junior_west,0.9877590382,36.155.253.241,"{""location"": ""SV"", ""is_mobile"": false}" 20334,7,458,2017-06-11 19:53:39,http://bins.net/theo.gorczany,0.0930208852,162.13.253.77,"{""location"": ""YE"", ""is_mobile"": true}" 20335,7,458,2016-12-27 04:18:01,http://langworth.io/clare.mohr,0.0895288133,190.244.72.69,"{""location"": ""TO"", ""is_mobile"": false}" 20336,7,458,2017-02-15 16:59:26,http://grant.org/dewayne.mohr,0.0014907330,9.46.132.23,"{""location"": ""BW"", ""is_mobile"": true}" 20337,7,458,2017-01-20 18:55:55,http://sawayn.com/moises.kuhn,0.6669614148,169.188.114.79,"{""location"": ""BI"", ""is_mobile"": false}" 20338,7,458,2017-05-22 04:26:07,http://grimes.biz/catherine.mitchell,0.4073406215,155.180.40.211,"{""location"": ""ZM"", ""is_mobile"": true}" 20339,7,458,2017-04-07 14:41:00,http://thompson.net/selena_oconnell,0.9969953348,77.224.138.212,"{""location"": ""NI"", ""is_mobile"": true}" 20340,7,458,2017-06-01 10:44:33,http://schultzgoldner.name/sim,0.2997667927,4.178.38.186,"{""location"": ""BA"", ""is_mobile"": false}" 20341,7,458,2017-05-29 05:47:06,http://heller.info/clemmie,0.5421952694,147.231.249.112,"{""location"": ""AI"", ""is_mobile"": true}" 20342,7,458,2017-01-01 21:36:31,http://deckow.info/esta,0.1213497887,198.172.173.17,"{""location"": ""SE"", ""is_mobile"": true}" 20343,7,458,2017-06-09 14:46:18,http://bernier.com/cleo_torphy,0.0563993442,154.85.115.66,"{""location"": ""IR"", ""is_mobile"": false}" 20344,7,458,2016-12-29 00:31:09,http://spinka.info/reece_ratke,0.6853576131,254.171.141.192,"{""location"": ""LB"", ""is_mobile"": false}" 20345,7,458,2017-04-23 15:34:11,http://lindgren.org/milan,0.0127271542,233.61.165.197,"{""location"": ""BI"", ""is_mobile"": false}" 20346,7,458,2016-12-25 04:58:05,http://wuckertgreen.net/perry.hudson,0.3514873832,145.8.18.96,"{""location"": ""FR"", ""is_mobile"": false}" 20347,7,458,2017-02-21 15:32:43,http://wunsch.org/jolie,0.5658921886,28.98.88.66,"{""location"": ""PS"", ""is_mobile"": false}" 20348,7,458,2016-12-22 15:25:43,http://friesen.co/elenor,0.3841652827,64.57.56.21,"{""location"": ""DJ"", ""is_mobile"": true}" 20349,7,458,2017-03-22 01:51:51,http://nitzsche.name/kristoffer.kshlerin,0.1338506263,212.148.26.43,"{""location"": ""TO"", ""is_mobile"": false}" 20350,7,458,2017-04-19 12:05:26,http://medhurstebert.co/veronica.littel,0.5460280565,117.87.19.239,"{""location"": ""OM"", ""is_mobile"": false}" 20351,7,458,2016-12-14 19:07:06,http://keeling.info/ruthie,0.6273254582,166.139.204.110,"{""location"": ""AR"", ""is_mobile"": false}" 20352,7,458,2017-05-14 02:09:49,http://beerthiel.info/ignacio_hodkiewicz,0.0864295128,185.103.248.77,"{""location"": ""CC"", ""is_mobile"": false}" 20353,7,458,2017-02-16 15:21:48,http://windlermante.info/jerrod.schneider,0.1226761942,78.8.212.105,"{""location"": ""RW"", ""is_mobile"": false}" 20354,7,458,2017-06-11 16:42:27,http://grant.io/norwood,0.5174488139,40.34.200.226,"{""location"": ""CA"", ""is_mobile"": false}" 20355,7,458,2017-01-29 00:27:50,http://reillyblanda.co/percival,0.6916314315,33.37.70.92,"{""location"": ""FM"", ""is_mobile"": true}" 20356,7,458,2017-05-02 01:47:01,http://collins.net/je,0.5151054317,242.185.33.67,"{""location"": ""KI"", ""is_mobile"": true}" 20357,7,458,2017-02-13 04:00:50,http://howe.net/timmy.collins,0.9621941719,103.52.99.97,"{""location"": ""MH"", ""is_mobile"": false}" 20358,7,458,2017-06-06 06:19:13,http://waters.name/jovanny,0.7453629153,173.84.244.162,"{""location"": ""HT"", ""is_mobile"": true}" 20359,7,458,2017-02-18 16:38:17,http://heaneymorar.io/sophia.emmerich,0.9347496640,23.47.244.99,"{""location"": ""SV"", ""is_mobile"": false}" 20360,7,458,2017-05-19 04:24:11,http://boehm.biz/deja.murphy,0.7469309241,162.2.166.234,"{""location"": ""SB"", ""is_mobile"": true}" 20361,7,458,2017-06-07 10:00:11,http://zulauf.info/lorine,0.9367457762,143.53.18.66,"{""location"": ""CY"", ""is_mobile"": false}" 20362,7,458,2017-02-01 19:58:10,http://hermanlegros.com/alek,0.9666201468,216.200.15.97,"{""location"": ""EC"", ""is_mobile"": true}" 20363,7,458,2017-04-20 17:11:48,http://hickle.info/alexandre.parker,0.7718268294,171.80.97.78,"{""location"": ""ZM"", ""is_mobile"": false}" 20364,7,458,2017-01-22 23:13:20,http://turnerterry.info/maddison,0.7918600683,109.86.104.75,"{""location"": ""IL"", ""is_mobile"": true}" 20365,7,458,2016-12-24 13:08:23,http://bartellbecker.info/brenna,0.9863855541,65.210.82.194,"{""location"": ""TG"", ""is_mobile"": false}" 20366,7,458,2017-05-12 16:02:54,http://schroederjacobs.com/kellen,0.9644703944,205.42.8.213,"{""location"": ""CG"", ""is_mobile"": false}" 20367,7,458,2017-01-22 10:59:33,http://considine.info/jermey.bergstrom,0.1151994320,92.114.227.163,"{""location"": ""RO"", ""is_mobile"": false}" 20368,7,458,2017-01-10 02:45:46,http://smith.net/myrtle.watsica,0.6692737881,72.58.117.12,"{""location"": ""KI"", ""is_mobile"": true}" 20369,7,458,2017-05-02 15:38:24,http://kertzmann.co/isadore,0.1219855216,9.51.253.218,"{""location"": ""BE"", ""is_mobile"": false}" 20370,7,458,2016-12-31 08:30:58,http://shanahanankunding.io/bertram_beahan,0.4694222791,49.69.192.133,"{""location"": ""ST"", ""is_mobile"": true}" 20371,7,458,2017-06-08 19:14:39,http://weberhoppe.net/nola_kunze,0.9295167619,16.250.65.85,"{""location"": ""IL"", ""is_mobile"": true}" 20372,7,458,2017-01-07 06:47:08,http://jones.net/raoul,0.3112580176,173.180.101.157,"{""location"": ""YE"", ""is_mobile"": false}" 20373,7,458,2017-01-26 04:53:37,http://toy.org/pearlie,0.9240641680,85.110.30.94,"{""location"": ""GL"", ""is_mobile"": true}" 20374,7,458,2017-03-09 02:30:05,http://vandervort.name/matteo,0.4318091509,195.43.181.39,"{""location"": ""BI"", ""is_mobile"": true}" 20375,7,458,2017-02-26 22:44:55,http://stokes.biz/hope_bartoletti,0.6784227513,83.3.193.134,"{""location"": ""AX"", ""is_mobile"": false}" 20376,7,458,2017-04-28 20:12:32,http://white.io/mariah,0.4668340881,102.88.248.81,"{""location"": ""GY"", ""is_mobile"": false}" 20377,7,458,2017-02-18 08:18:28,http://cremin.name/haven,0.2192987253,62.246.214.148,"{""location"": ""US"", ""is_mobile"": true}" 20378,7,459,2016-12-27 11:09:44,http://macejkovic.org/alejandrin,0.1774726879,184.244.184.227,"{""location"": ""GD"", ""is_mobile"": true}" 20379,7,459,2017-06-13 08:28:10,http://monahan.org/felicia.crist,0.6820175006,176.136.69.69,"{""location"": ""BT"", ""is_mobile"": false}" 20380,7,459,2017-01-01 09:44:05,http://leschprohaska.net/stewart,0.0449240903,142.79.67.18,"{""location"": ""AS"", ""is_mobile"": true}" 20381,7,459,2017-06-02 10:38:40,http://runte.co/dave_haley,0.5900471383,12.90.139.19,"{""location"": ""LV"", ""is_mobile"": true}" 20382,7,459,2017-05-31 18:00:29,http://boyer.name/lance,0.6727171741,224.153.205.231,"{""location"": ""CC"", ""is_mobile"": true}" 20383,7,459,2017-04-09 18:52:47,http://watsicaemard.org/sallie.raynor,0.9779469704,76.11.164.233,"{""location"": ""UY"", ""is_mobile"": true}" 20384,7,459,2017-04-07 07:10:05,http://homenick.co/clair.volkman,0.6765359479,165.63.160.33,"{""location"": ""IR"", ""is_mobile"": true}" 20385,7,459,2017-05-03 08:24:55,http://skiles.io/gordon,0.7105146827,236.30.44.216,"{""location"": ""SA"", ""is_mobile"": true}" 20386,7,459,2017-01-23 07:43:41,http://hartmannreichel.co/avis_von,0.0903264332,153.52.105.65,"{""location"": ""SL"", ""is_mobile"": true}" 20387,7,459,2017-04-04 09:23:50,http://davis.io/trea,0.0921529047,94.159.157.113,"{""location"": ""MV"", ""is_mobile"": false}" 20388,7,459,2017-02-22 13:26:41,http://romagueramoore.info/oscar.beahan,0.2345166377,92.100.97.126,"{""location"": ""TL"", ""is_mobile"": false}" 20389,7,459,2017-02-13 00:09:11,http://west.com/malachi,0.4894206282,107.125.48.86,"{""location"": ""NL"", ""is_mobile"": false}" 20390,7,459,2017-05-20 19:16:46,http://gutkowski.org/wellington,0.8908165760,65.209.63.246,"{""location"": ""MZ"", ""is_mobile"": false}" 20391,7,459,2017-02-07 07:18:46,http://price.info/ralph.bartell,0.4980153918,73.176.67.156,"{""location"": ""QA"", ""is_mobile"": true}" 20392,7,459,2017-06-04 09:56:03,http://nicolatreich.org/elenor,0.6610886802,240.54.170.163,"{""location"": ""LA"", ""is_mobile"": true}" 20393,7,459,2017-01-24 19:49:31,http://thiel.name/graham,0.8864319514,104.215.240.8,"{""location"": ""SR"", ""is_mobile"": false}" 20394,7,459,2017-05-25 22:14:17,http://ryanturcotte.info/leslie,0.4107727447,229.73.214.227,"{""location"": ""CK"", ""is_mobile"": false}" 20395,7,459,2017-06-03 12:21:01,http://gleichner.net/ro,0.0162763569,188.113.176.224,"{""location"": ""TD"", ""is_mobile"": false}" 20396,7,459,2017-03-13 22:12:03,http://shanahanwelch.info/dexter,0.4370542766,134.99.28.246,"{""location"": ""BJ"", ""is_mobile"": true}" 20397,7,459,2017-02-28 10:22:15,http://klingconsidine.io/arjun,0.4670077191,142.88.250.50,"{""location"": ""CR"", ""is_mobile"": true}" 20398,7,459,2017-02-19 21:37:50,http://konopelski.info/saige,0.1085647875,160.154.13.144,"{""location"": ""TH"", ""is_mobile"": false}" 20399,7,459,2016-12-28 14:09:47,http://rempel.co/tevin_flatley,0.6153830220,212.101.9.103,"{""location"": ""IE"", ""is_mobile"": true}" 20400,7,459,2017-04-14 20:28:54,http://hyatt.info/gilda,0.3952125807,251.187.201.160,"{""location"": ""UY"", ""is_mobile"": false}" 20401,7,459,2017-02-19 04:21:26,http://lemke.io/ervin,0.3972757163,190.94.140.176,"{""location"": ""LA"", ""is_mobile"": true}" 20402,7,459,2017-02-13 13:58:27,http://connelly.info/terrell,0.1489538814,224.21.91.200,"{""location"": ""JO"", ""is_mobile"": true}" 20403,7,459,2017-02-03 05:08:46,http://andersonbergnaum.io/beth.bednar,0.8990156920,231.190.104.37,"{""location"": ""PK"", ""is_mobile"": true}" 20404,7,459,2016-12-22 02:56:42,http://zulaufmoore.net/hertha,0.6900647882,27.251.8.202,"{""location"": ""BG"", ""is_mobile"": false}" 20405,7,459,2017-06-02 06:02:56,http://adamsbarton.org/cole,0.3779920427,196.208.170.184,"{""location"": ""SZ"", ""is_mobile"": true}" 20406,7,459,2017-05-17 05:21:02,http://donnelly.org/zoe,0.1524159436,155.102.216.50,"{""location"": ""MY"", ""is_mobile"": false}" 20407,7,459,2017-05-21 18:16:28,http://metz.io/jimmy,0.0086102309,46.39.143.144,"{""location"": ""UY"", ""is_mobile"": true}" 20408,7,459,2017-02-13 21:14:06,http://kaulke.com/laisha,0.6655099225,97.250.22.164,"{""location"": ""NE"", ""is_mobile"": true}" 20409,7,459,2017-04-18 12:13:08,http://hamill.net/darlene,0.0911324188,32.3.190.19,"{""location"": ""DO"", ""is_mobile"": true}" 20410,7,459,2016-12-18 22:25:22,http://torphy.name/helga.fisher,0.1500730628,124.129.107.231,"{""location"": ""UY"", ""is_mobile"": false}" 20411,7,459,2017-01-31 04:07:38,http://gleason.com/sedrick,0.8972606161,212.226.166.175,"{""location"": ""LK"", ""is_mobile"": true}" 20412,7,459,2017-06-05 15:36:12,http://abbotthansen.name/beth,0.5555663342,78.212.245.56,"{""location"": ""GW"", ""is_mobile"": true}" 20413,7,459,2017-05-21 13:07:10,http://boyerhermann.co/lennie.littel,0.7737225199,40.20.30.144,"{""location"": ""BG"", ""is_mobile"": false}" 20414,7,459,2016-12-29 09:59:06,http://rau.net/glen.mitchell,0.6071966051,128.216.142.213,"{""location"": ""RO"", ""is_mobile"": false}" 20415,7,459,2017-04-24 09:27:57,http://williamsondibbert.org/magnus,0.9889825327,69.75.178.44,"{""location"": ""BI"", ""is_mobile"": false}" 20416,7,459,2017-02-05 19:57:40,http://wisoky.com/sonny.moriette,0.2477858461,4.115.253.121,"{""location"": ""PE"", ""is_mobile"": false}" 20417,7,459,2017-05-07 00:16:26,http://feeney.io/adrain,0.0856218038,194.167.144.25,"{""location"": ""LR"", ""is_mobile"": false}" 20418,7,459,2017-01-25 10:46:14,http://pouros.co/lazaro.abbott,0.5513458384,49.20.178.123,"{""location"": ""TK"", ""is_mobile"": true}" 20419,7,459,2017-04-12 01:58:29,http://west.org/rey.waters,0.9756855583,131.14.45.103,"{""location"": ""BQ"", ""is_mobile"": true}" 20420,7,459,2016-12-17 04:10:53,http://hicklehane.io/vernie,0.1004779556,225.157.238.250,"{""location"": ""DE"", ""is_mobile"": false}" 20421,7,459,2017-02-01 13:04:03,http://orn.biz/gene,0.6231290446,168.216.64.153,"{""location"": ""BM"", ""is_mobile"": false}" 20422,7,459,2017-05-16 00:13:45,http://stark.biz/josh,0.5422409734,174.208.227.35,"{""location"": ""MH"", ""is_mobile"": false}" 20423,7,459,2017-02-26 04:55:38,http://eichmannkeler.info/aleandro,0.8350294634,47.250.211.82,"{""location"": ""RW"", ""is_mobile"": false}" 20424,7,459,2017-02-16 03:51:42,http://murphy.com/okey,0.0640895028,2.195.92.42,"{""location"": ""MD"", ""is_mobile"": false}" 20425,7,459,2017-06-11 16:19:33,http://senger.net/bell,0.8303497725,247.119.174.49,"{""location"": ""MO"", ""is_mobile"": true}" 20426,7,459,2016-12-27 17:00:50,http://howell.io/graciela,0.1713042747,13.93.129.224,"{""location"": ""CK"", ""is_mobile"": true}" 20427,7,459,2017-06-13 02:48:17,http://dubuque.info/muriel_bruen,0.6962711451,246.206.116.96,"{""location"": ""BR"", ""is_mobile"": true}" 20428,7,459,2017-03-23 16:35:47,http://hirthewolff.info/laury,0.6494221374,147.29.147.188,"{""location"": ""NG"", ""is_mobile"": false}" 20429,7,459,2016-12-26 02:47:51,http://aufderhar.net/dario,0.9293687293,71.7.241.70,"{""location"": ""MD"", ""is_mobile"": false}" 20430,7,459,2017-05-27 11:06:24,http://franeckiheel.info/jaida_bogan,0.5134819747,226.32.206.214,"{""location"": ""PW"", ""is_mobile"": false}" 20431,7,459,2017-04-06 16:12:13,http://parkerweber.io/ivory,0.7374426191,135.51.149.33,"{""location"": ""BY"", ""is_mobile"": true}" 20432,7,459,2017-06-09 11:44:45,http://treutel.com/alexandro,0.7058545964,94.159.144.183,"{""location"": ""MV"", ""is_mobile"": false}" 20433,7,459,2017-04-16 16:18:39,http://yostmohr.com/chet.mante,0.1301738510,35.102.91.59,"{""location"": ""EE"", ""is_mobile"": false}" 20434,7,459,2017-04-26 19:18:17,http://gottliebwilliamson.info/junius,0.3781206306,210.250.104.248,"{""location"": ""ET"", ""is_mobile"": false}" 20435,7,459,2017-01-01 22:14:07,http://murrayrutherford.net/emilia.parisian,0.7449360640,83.111.164.34,"{""location"": ""AE"", ""is_mobile"": false}" 20436,7,459,2017-03-24 06:34:21,http://mosciski.io/isaias,0.2311649785,6.104.73.53,"{""location"": ""TC"", ""is_mobile"": false}" 20437,7,459,2017-04-02 13:36:39,http://ferry.com/leta.hammes,0.4579389880,69.20.76.124,"{""location"": ""NR"", ""is_mobile"": true}" 20438,7,459,2017-05-01 18:09:46,http://langosh.info/dovie,0.1712189808,69.155.160.71,"{""location"": ""JO"", ""is_mobile"": true}" 20439,7,459,2016-12-29 22:42:05,http://fritschprohaska.info/bettie,0.2945224601,144.82.168.31,"{""location"": ""UG"", ""is_mobile"": false}" 20440,7,459,2017-02-22 05:41:00,http://gutmannhuels.name/jonathan.emard,0.1711115879,173.204.97.43,"{""location"": ""SC"", ""is_mobile"": false}" 20441,7,459,2017-03-04 04:20:29,http://buckridge.biz/wilford,0.9182525289,118.172.153.181,"{""location"": ""SZ"", ""is_mobile"": false}" 20442,7,459,2017-05-17 10:50:15,http://daugherty.org/axel,0.1158320210,224.39.114.31,"{""location"": ""NU"", ""is_mobile"": true}" 20443,7,459,2017-05-19 08:20:09,http://gulgowski.co/fernando,0.4488181446,116.115.251.13,"{""location"": ""MQ"", ""is_mobile"": false}" 20444,7,459,2017-02-26 09:08:06,http://conroy.io/catalina,0.2548739728,117.43.66.25,"{""location"": ""BH"", ""is_mobile"": true}" 20445,7,460,2017-01-19 08:18:41,http://turcottewisozk.co/larue.bogisich,0.2698391726,147.112.218.54,"{""location"": ""RS"", ""is_mobile"": false}" 20446,7,460,2017-01-29 00:04:06,http://powlowskikautzer.co/julien,0.4623388210,153.88.155.182,"{""location"": ""MM"", ""is_mobile"": false}" 20447,7,460,2017-03-10 06:32:12,http://wuckertgulgowski.io/jammie,0.0545398921,236.52.166.112,"{""location"": ""MX"", ""is_mobile"": false}" 20448,7,460,2017-03-21 16:13:47,http://jacobsbradtke.info/earl,0.7910042580,186.27.167.157,"{""location"": ""GD"", ""is_mobile"": true}" 20449,7,460,2017-04-04 19:41:15,http://witting.net/laurine,0.8964527907,85.239.118.81,"{""location"": ""RS"", ""is_mobile"": true}" 20450,7,460,2016-12-17 08:55:03,http://wuckerthowell.net/reed.walsh,0.6354823793,85.201.32.167,"{""location"": ""DO"", ""is_mobile"": false}" 20451,7,460,2016-12-21 15:44:33,http://keebler.com/gielle,0.2660865892,105.186.170.213,"{""location"": ""BQ"", ""is_mobile"": true}" 20452,7,460,2016-12-16 07:14:05,http://parker.io/noah.johnston,0.7535968160,218.214.136.144,"{""location"": ""JP"", ""is_mobile"": true}" 20453,7,460,2017-05-29 14:58:17,http://konopelski.info/valentin,0.8896195644,70.32.170.12,"{""location"": ""TO"", ""is_mobile"": true}" 20454,7,460,2016-12-13 06:43:42,http://mayert.name/jaren,0.8892370604,236.152.65.249,"{""location"": ""NG"", ""is_mobile"": true}" 20455,7,460,2017-02-01 04:42:59,http://purdywisozk.info/terry,0.7025448649,43.2.176.98,"{""location"": ""MX"", ""is_mobile"": false}" 20456,7,460,2017-02-20 09:46:02,http://durgan.co/geovanny,0.7074232174,217.9.164.216,"{""location"": ""WF"", ""is_mobile"": true}" 20457,7,460,2017-05-04 13:38:54,http://hodkiewicz.io/althea,0.4064072966,59.143.120.46,"{""location"": ""CY"", ""is_mobile"": false}" 20458,7,460,2017-04-11 09:32:28,http://considinebeer.biz/electa.emmerich,0.5294936549,150.41.31.87,"{""location"": ""RO"", ""is_mobile"": true}" 20459,7,460,2017-06-06 14:01:58,http://jacobson.biz/arnaldo,0.6940564876,207.66.16.10,"{""location"": ""BZ"", ""is_mobile"": false}" 20460,7,460,2017-02-21 22:52:36,http://hintz.info/cheyanne.kihn,0.6150241341,34.198.185.247,"{""location"": ""DE"", ""is_mobile"": false}" 20461,7,460,2017-03-09 12:40:44,http://shields.com/kellen_heathcote,0.2668711472,105.41.92.229,"{""location"": ""ES"", ""is_mobile"": false}" 20462,7,460,2016-12-23 09:05:59,http://kirlinsipes.net/mylene,0.8629622747,211.72.69.164,"{""location"": ""BT"", ""is_mobile"": false}" 20463,7,460,2017-03-27 15:41:02,http://kertzmann.name/avery_gerhold,0.3141044919,246.232.103.147,"{""location"": ""LK"", ""is_mobile"": true}" 20464,7,460,2017-06-08 17:09:41,http://simonis.co/ignatius,0.0272649455,208.91.197.169,"{""location"": ""IM"", ""is_mobile"": true}" 20465,7,460,2017-02-17 20:39:35,http://vonrueden.name/dustin_herzog,0.8886427298,172.57.78.139,"{""location"": ""LI"", ""is_mobile"": true}" 20466,7,460,2017-01-15 01:45:30,http://wilderman.info/schuyler.shields,0.1869628904,114.18.166.142,"{""location"": ""MZ"", ""is_mobile"": true}" 20467,7,460,2017-04-23 12:56:01,http://kertzmannschneider.info/newton,0.1300107203,163.66.236.94,"{""location"": ""OM"", ""is_mobile"": true}" 20468,7,460,2017-05-16 03:38:03,http://ruel.co/osborne_fritsch,0.5606247742,223.40.6.109,"{""location"": ""ZW"", ""is_mobile"": false}" 20469,7,460,2017-01-30 04:49:14,http://braun.name/enrico_gibson,0.8447333931,113.172.142.55,"{""location"": ""KN"", ""is_mobile"": false}" 20470,7,460,2017-01-27 14:42:09,http://mitchell.co/dallas,0.5514177976,170.22.199.211,"{""location"": ""SY"", ""is_mobile"": true}" 20471,7,460,2017-04-13 09:13:54,http://dietrich.io/howard_rolfson,0.4391790270,178.66.49.182,"{""location"": ""CM"", ""is_mobile"": false}" 20472,7,460,2017-01-30 02:23:28,http://ritchie.info/buddy,0.4591446939,65.144.98.254,"{""location"": ""PM"", ""is_mobile"": false}" 20473,7,460,2017-03-06 11:11:22,http://king.name/eulah_yundt,0.0958843669,52.86.134.19,"{""location"": ""ES"", ""is_mobile"": false}" 20474,7,460,2017-01-24 11:39:39,http://cartwright.net/daija.zemlak,0.5299589132,165.217.219.143,"{""location"": ""FO"", ""is_mobile"": false}" 20475,7,460,2017-05-23 14:33:25,http://romaguera.org/shemar_klein,0.5070393872,192.140.79.221,"{""location"": ""UA"", ""is_mobile"": true}" 20476,7,461,2017-02-06 21:40:37,http://ward.io/asia,0.6765391541,221.196.166.58,"{""location"": ""GG"", ""is_mobile"": false}" 20477,7,461,2017-02-19 15:29:18,http://stark.info/janelle,0.5408847258,61.98.158.165,"{""location"": ""RU"", ""is_mobile"": true}" 20478,7,461,2017-01-15 20:36:43,http://blockkulas.biz/eryn,0.6261763557,244.219.231.195,"{""location"": ""LC"", ""is_mobile"": false}" 20479,7,461,2016-12-29 02:45:40,http://kochschinner.io/destany,0.3574212152,93.204.172.226,"{""location"": ""RE"", ""is_mobile"": false}" 20480,7,461,2017-05-27 20:54:11,http://huel.info/hosea.rau,0.8361857777,5.46.69.52,"{""location"": ""BQ"", ""is_mobile"": false}" 20481,7,461,2017-04-01 22:47:47,http://wuckert.co/camryn.ruecker,0.0531465418,5.190.24.194,"{""location"": ""AQ"", ""is_mobile"": true}" 20482,7,461,2017-02-02 06:06:27,http://bayertremblay.com/brennon,0.7085416386,247.157.7.247,"{""location"": ""PA"", ""is_mobile"": true}" 20483,7,461,2017-05-13 23:50:01,http://weber.com/oran.langworth,0.0003302118,57.90.35.140,"{""location"": ""TG"", ""is_mobile"": false}" 20484,7,461,2017-01-15 03:58:31,http://johnston.net/jennie,0.7260026923,186.27.168.5,"{""location"": ""SK"", ""is_mobile"": false}" 20485,7,461,2017-04-19 02:35:50,http://mcclure.info/benny_marquardt,0.3567910721,102.28.243.14,"{""location"": ""ZW"", ""is_mobile"": true}" 20486,7,461,2017-04-13 22:52:26,http://huels.com/myriam.langosh,0.5015630554,99.28.190.112,"{""location"": ""NP"", ""is_mobile"": false}" 20487,7,461,2017-02-07 17:46:33,http://mayert.io/domenick_heller,0.2499664897,56.186.232.81,"{""location"": ""ML"", ""is_mobile"": true}" 20488,7,461,2017-02-20 08:19:56,http://mraz.co/willie,0.7327423723,17.217.23.17,"{""location"": ""GY"", ""is_mobile"": true}" 20489,7,461,2017-02-05 06:57:07,http://muellerhaag.io/leonard_bradtke,0.0534500874,76.228.148.246,"{""location"": ""AE"", ""is_mobile"": false}" 20490,7,461,2017-03-21 19:36:14,http://schoen.biz/florida_bosco,0.1613674622,229.240.176.11,"{""location"": ""VA"", ""is_mobile"": true}" 20491,7,461,2017-02-21 09:09:47,http://wuckerthauck.co/markus.murphy,0.8583326545,59.79.76.237,"{""location"": ""BM"", ""is_mobile"": true}" 20492,7,461,2017-02-05 00:29:15,http://lefflerjacobs.org/dane,0.4065496298,117.248.254.128,"{""location"": ""ET"", ""is_mobile"": true}" 20493,7,461,2017-05-01 09:28:15,http://rowekihn.name/aleandro,0.2963735846,68.111.87.233,"{""location"": ""IR"", ""is_mobile"": false}" 20494,7,461,2017-02-22 07:23:07,http://keeblerquitzon.info/clark,0.9818972480,77.56.4.161,"{""location"": ""TN"", ""is_mobile"": true}" 20495,7,461,2017-05-01 04:51:37,http://yundthettinger.co/blanche.jaskolski,0.3305986004,48.78.155.241,"{""location"": ""UG"", ""is_mobile"": true}" 20496,7,461,2017-03-25 16:40:58,http://bartellswift.co/ashlynn.stanton,0.5924407664,74.169.13.192,"{""location"": ""MH"", ""is_mobile"": false}" 20497,7,461,2017-04-20 14:22:21,http://breitenbergkuhn.net/riley.langosh,0.9478298058,163.24.110.11,"{""location"": ""NC"", ""is_mobile"": true}" 20498,7,461,2017-03-27 15:12:18,http://zieme.info/garfield,0.1295235871,5.98.143.76,"{""location"": ""FK"", ""is_mobile"": false}" 20499,7,461,2017-05-18 17:35:20,http://dietrich.net/fernando,0.6635082509,82.162.147.38,"{""location"": ""TV"", ""is_mobile"": false}" 20500,7,461,2017-02-10 08:56:29,http://mcclure.co/lorine_cruickshank,0.9011319465,70.87.96.77,"{""location"": ""HT"", ""is_mobile"": false}" 20501,7,461,2017-04-20 15:50:51,http://torpbins.io/lonnie_steuber,0.5839723854,101.145.112.146,"{""location"": ""IE"", ""is_mobile"": false}" 20502,7,461,2017-03-17 19:15:15,http://marvin.co/landen.douglas,0.4869572251,153.126.193.51,"{""location"": ""DO"", ""is_mobile"": true}" 20503,7,461,2017-02-06 05:17:08,http://blandareichel.co/elda_stroman,0.9242760249,130.133.84.107,"{""location"": ""JO"", ""is_mobile"": true}" 20504,7,461,2017-04-25 15:25:58,http://smitham.io/layla.schmitt,0.4120661861,207.48.129.38,"{""location"": ""PL"", ""is_mobile"": false}" 20505,7,461,2017-04-26 12:59:17,http://wyman.co/ian.metz,0.0861156669,223.66.84.155,"{""location"": ""AF"", ""is_mobile"": true}" 20506,7,461,2017-01-22 06:18:19,http://turcotte.co/ivah.dickinson,0.8393131624,70.58.199.16,"{""location"": ""SY"", ""is_mobile"": false}" 20507,7,461,2017-03-09 19:28:14,http://barrows.biz/kaley.medhurst,0.5347517250,219.117.166.241,"{""location"": ""MG"", ""is_mobile"": false}" 20508,7,461,2017-06-13 07:02:37,http://abshirewillms.name/maida,0.0182511951,114.214.45.202,"{""location"": ""AW"", ""is_mobile"": false}" 20509,7,461,2017-04-29 17:19:02,http://barton.io/carey,0.3497663871,148.191.58.228,"{""location"": ""AD"", ""is_mobile"": true}" 20510,7,461,2017-05-02 10:57:01,http://walter.info/taryn,0.6839685481,233.243.223.25,"{""location"": ""MD"", ""is_mobile"": true}" 20511,7,461,2017-01-07 07:37:18,http://luettgenlabadie.com/sydnie,0.4022297931,148.227.29.202,"{""location"": ""ET"", ""is_mobile"": false}" 20512,7,461,2017-04-05 14:30:26,http://adamswuckert.name/hope,0.3551192524,223.67.157.243,"{""location"": ""HU"", ""is_mobile"": false}" 20513,7,461,2017-06-11 21:06:01,http://kirlin.info/lucas,0.4079809696,113.231.99.143,"{""location"": ""FM"", ""is_mobile"": false}" 20514,7,461,2017-03-16 02:00:17,http://denesik.org/ramona_beier,0.3337569335,197.125.48.161,"{""location"": ""TF"", ""is_mobile"": false}" 20515,7,461,2017-03-15 17:27:09,http://hartmann.com/ansel.kub,0.0357296385,78.77.13.13,"{""location"": ""KH"", ""is_mobile"": false}" 20516,7,461,2017-03-22 00:31:53,http://baileywyman.io/devan_blanda,0.9705709396,76.190.123.91,"{""location"": ""LV"", ""is_mobile"": false}" 20517,7,461,2017-06-13 17:54:40,http://mills.biz/isaiah_kaulke,0.5371540714,84.195.222.146,"{""location"": ""IT"", ""is_mobile"": false}" 20518,7,461,2017-05-27 23:07:46,http://buckridgecain.net/stephen,0.2852246021,231.204.2.21,"{""location"": ""MZ"", ""is_mobile"": false}" 20519,7,461,2017-01-06 15:14:40,http://rippin.name/jaida.kulas,0.5903413466,132.248.155.110,"{""location"": ""LS"", ""is_mobile"": true}" 20520,7,461,2017-01-04 02:07:21,http://treutel.name/maiya_nader,0.7262063338,122.165.61.193,"{""location"": ""MF"", ""is_mobile"": true}" 20521,7,461,2017-03-03 04:41:51,http://mitchelldach.name/vesta.dibbert,0.2456912671,205.73.69.146,"{""location"": ""PE"", ""is_mobile"": false}" 20522,7,461,2017-03-27 17:26:57,http://balistreriruecker.io/vita,0.5021253177,86.219.63.118,"{""location"": ""AR"", ""is_mobile"": false}" 20523,7,461,2017-05-01 12:36:38,http://connelly.info/anibal,0.1770442761,31.168.204.4,"{""location"": ""KH"", ""is_mobile"": false}" 20524,7,461,2016-12-16 04:39:10,http://dickioconnell.org/alexis_homenick,0.3526142148,223.2.37.52,"{""location"": ""IS"", ""is_mobile"": false}" 20525,7,461,2017-01-19 21:11:31,http://botsford.net/marian.hirthe,0.0091201011,131.155.122.121,"{""location"": ""TN"", ""is_mobile"": true}" 20526,7,461,2017-04-26 23:10:49,http://ko.co/parker_rodriguez,0.9452017858,246.228.50.2,"{""location"": ""GQ"", ""is_mobile"": true}" 20527,7,461,2017-04-13 07:21:32,http://abernathy.net/carolyne.johnson,0.1120538305,206.57.162.9,"{""location"": ""BQ"", ""is_mobile"": true}" 20528,7,461,2017-03-04 07:04:15,http://anderson.net/annie_schneider,0.0086994231,214.196.181.238,"{""location"": ""RW"", ""is_mobile"": false}" 20529,7,461,2017-02-24 23:37:27,http://waterskutch.net/john_hyatt,0.5690397003,214.143.167.215,"{""location"": ""MO"", ""is_mobile"": false}" 20530,7,461,2017-04-27 06:43:58,http://wintheiserhahn.co/dominique_bins,0.0094055585,144.63.78.30,"{""location"": ""DJ"", ""is_mobile"": true}" 20531,7,461,2017-03-02 06:25:22,http://kutchbernhard.org/berry_brakus,0.6843621181,105.210.148.25,"{""location"": ""ER"", ""is_mobile"": true}" 20532,7,461,2017-05-09 02:58:00,http://murphyupton.biz/laron_paucek,0.8117936878,150.253.8.21,"{""location"": ""SH"", ""is_mobile"": false}" 20533,7,461,2017-04-23 01:43:53,http://wizaeffertz.biz/burnice,0.4603382976,168.242.85.214,"{""location"": ""AE"", ""is_mobile"": true}" 20534,7,461,2017-04-23 07:00:59,http://murazik.org/mitchel,0.1117089435,136.220.178.41,"{""location"": ""NC"", ""is_mobile"": false}" 20535,7,461,2017-05-19 03:25:09,http://swiftkulas.io/quinton.rau,0.7432072300,16.247.247.11,"{""location"": ""CX"", ""is_mobile"": false}" 20536,7,461,2017-01-19 00:13:47,http://lehner.co/nicholas_thompson,0.7662858503,137.212.220.109,"{""location"": ""TH"", ""is_mobile"": false}" 20537,7,462,2017-03-14 19:18:30,http://hauckkuhn.name/clara,0.0945343614,31.20.39.154,"{""location"": ""AU"", ""is_mobile"": true}" 20538,7,462,2017-01-20 15:04:29,http://turner.co/jayda_boyle,0.0189378381,149.249.222.134,"{""location"": ""DE"", ""is_mobile"": true}" 20539,7,462,2017-04-25 23:27:43,http://huel.org/carlo_dare,0.2271429562,252.137.155.177,"{""location"": ""MH"", ""is_mobile"": false}" 20540,7,462,2017-02-26 17:07:55,http://wiegand.net/carleton,0.1513109462,120.143.127.161,"{""location"": ""TW"", ""is_mobile"": true}" 20541,7,462,2017-05-19 16:26:12,http://vonruedencartwright.info/carolanne.daniel,0.4472891010,207.204.218.142,"{""location"": ""NP"", ""is_mobile"": true}" 20542,7,462,2017-01-01 09:31:58,http://schadenwhite.com/ayana,0.7965623605,59.7.102.162,"{""location"": ""TC"", ""is_mobile"": true}" 20543,7,462,2016-12-29 08:47:39,http://beatty.org/pat.stroman,0.3275208803,138.5.111.246,"{""location"": ""CF"", ""is_mobile"": false}" 20544,7,462,2017-02-16 00:26:38,http://bashirian.biz/winona_damore,0.2162774051,69.27.214.245,"{""location"": ""CX"", ""is_mobile"": false}" 20545,7,462,2017-05-30 18:16:46,http://dibbertsipes.co/selina,0.6960933800,20.145.72.183,"{""location"": ""BW"", ""is_mobile"": true}" 20546,7,462,2017-05-21 20:09:33,http://hand.info/oliver_jaskolski,0.8898932803,8.68.171.18,"{""location"": ""MH"", ""is_mobile"": true}" 20547,7,462,2017-04-16 04:16:23,http://harberschulist.info/minerva,0.5074218120,4.18.20.243,"{""location"": ""CR"", ""is_mobile"": false}" 20548,7,462,2017-04-07 01:50:10,http://feestkunze.net/lucile.schneider,0.2704984208,183.117.241.147,"{""location"": ""IT"", ""is_mobile"": true}" 20549,7,462,2017-05-26 05:34:00,http://hills.info/jermain,0.4381525080,233.111.169.138,"{""location"": ""HU"", ""is_mobile"": true}" 20550,7,462,2017-04-07 03:21:34,http://jastgaylord.net/bernice.becker,0.6873713769,189.253.241.120,"{""location"": ""AR"", ""is_mobile"": false}" 20551,7,462,2017-01-05 14:47:45,http://hettinger.org/christian,0.0030581784,193.248.141.89,"{""location"": ""GT"", ""is_mobile"": true}" 20552,7,462,2017-05-05 03:33:33,http://hudson.biz/amos_adams,0.4273246033,240.20.123.253,"{""location"": ""BV"", ""is_mobile"": true}" 20553,7,462,2017-05-09 07:38:57,http://kihn.co/general,0.7233097561,86.8.18.242,"{""location"": ""MZ"", ""is_mobile"": true}" 20554,7,462,2016-12-24 12:24:12,http://west.co/kris,0.1274979865,164.182.166.247,"{""location"": ""ST"", ""is_mobile"": true}" 20555,7,462,2016-12-26 01:13:49,http://feestkonopelski.info/alexandro,0.3459427642,132.219.58.211,"{""location"": ""MH"", ""is_mobile"": true}" 20556,7,462,2016-12-28 23:01:22,http://gaylord.net/laurine_treutel,0.7382684776,43.67.92.123,"{""location"": ""JO"", ""is_mobile"": true}" 20557,7,462,2017-05-20 23:19:15,http://hilll.name/enola.moen,0.5800277757,162.47.242.174,"{""location"": ""FR"", ""is_mobile"": true}" 20558,7,462,2017-01-14 00:49:55,http://swaniawski.name/evan.kunde,0.5380685195,55.118.229.119,"{""location"": ""SN"", ""is_mobile"": true}" 20559,7,462,2017-02-17 11:10:08,http://ankunding.io/vincenza,0.4520818613,158.228.89.204,"{""location"": ""HU"", ""is_mobile"": false}" 20560,7,462,2017-01-26 09:53:19,http://franeckiwalsh.io/norris,0.4905584257,184.7.219.240,"{""location"": ""HK"", ""is_mobile"": true}" 20561,7,462,2017-03-15 16:16:07,http://torphy.org/maiya_bauch,0.7583886916,5.236.214.31,"{""location"": ""SN"", ""is_mobile"": true}" 20562,7,462,2017-06-12 08:05:39,http://price.name/ora_morar,0.3966139565,123.63.148.123,"{""location"": ""MM"", ""is_mobile"": false}" 20563,7,462,2017-01-29 07:49:08,http://schamberger.biz/travon.mosciski,0.2538696618,164.18.4.4,"{""location"": ""BI"", ""is_mobile"": false}" 20564,7,462,2017-02-05 18:49:40,http://auer.net/general,0.1411818528,248.47.197.131,"{""location"": ""LU"", ""is_mobile"": true}" 20565,7,462,2017-06-10 13:14:07,http://wiegand.io/angelita.simonis,0.1805672116,171.57.30.142,"{""location"": ""AU"", ""is_mobile"": true}" 20566,7,462,2016-12-16 09:01:10,http://hackett.biz/casper_littel,0.8442826486,13.132.107.193,"{""location"": ""CI"", ""is_mobile"": true}" 20567,7,462,2017-03-11 10:52:27,http://bahringer.biz/mortimer,0.9796338545,54.94.109.247,"{""location"": ""GT"", ""is_mobile"": true}" 20568,7,462,2017-03-09 06:06:49,http://wiegand.biz/lula_bergstrom,0.6601253639,30.95.68.13,"{""location"": ""KY"", ""is_mobile"": true}" 20569,7,462,2016-12-13 10:41:51,http://kovacek.info/ken_christiansen,0.4809446945,32.128.47.78,"{""location"": ""BL"", ""is_mobile"": false}" 20570,7,462,2016-12-31 15:15:01,http://sauerohara.org/jordan.feeney,0.5583940497,169.187.155.7,"{""location"": ""TF"", ""is_mobile"": true}" 20571,7,462,2017-04-05 13:25:51,http://mrazpfannerstill.org/nikki,0.1492373932,188.79.248.104,"{""location"": ""GG"", ""is_mobile"": false}" 20572,7,462,2017-04-08 20:43:56,http://daniel.io/krista_blick,0.9297490862,20.192.158.179,"{""location"": ""RE"", ""is_mobile"": false}" 20573,7,462,2017-02-20 15:24:17,http://hickle.biz/frank.bartoletti,0.9637539200,90.22.9.136,"{""location"": ""AE"", ""is_mobile"": true}" 20574,7,462,2017-02-18 20:26:48,http://williamsonohara.name/orion_glover,0.1713837552,19.191.38.163,"{""location"": ""NF"", ""is_mobile"": false}" 20575,7,462,2017-01-18 17:15:47,http://okeefemraz.com/georgiana.buckridge,0.2002533224,203.99.187.8,"{""location"": ""NC"", ""is_mobile"": true}" 20576,7,462,2017-03-10 23:38:41,http://farrell.co/hardy,0.6157458264,73.219.165.3,"{""location"": ""CK"", ""is_mobile"": false}" 20577,7,462,2017-03-18 21:26:47,http://kunze.name/deshawn.kerluke,0.9332043449,139.218.167.67,"{""location"": ""GB"", ""is_mobile"": true}" 20578,7,462,2017-02-19 14:47:36,http://bins.name/jacquelyn,0.8136325883,119.250.37.225,"{""location"": ""VE"", ""is_mobile"": false}" 20579,7,462,2016-12-30 17:17:48,http://mccullough.info/jeffery_upton,0.8774578287,252.238.59.138,"{""location"": ""CY"", ""is_mobile"": false}" 20580,7,462,2016-12-15 18:26:37,http://quitzon.com/clark.becker,0.5978482989,172.144.120.212,"{""location"": ""EE"", ""is_mobile"": false}" 20581,7,462,2017-04-18 21:39:36,http://lubowitz.net/colin,0.4433748658,111.244.53.123,"{""location"": ""NP"", ""is_mobile"": false}" 20582,7,462,2017-04-11 21:25:38,http://ankunding.net/coralie,0.7686556876,206.221.42.116,"{""location"": ""MC"", ""is_mobile"": false}" 20583,7,462,2017-06-02 03:43:18,http://sauer.co/sylvester,0.6975777221,237.167.84.14,"{""location"": ""BF"", ""is_mobile"": false}" 20584,7,462,2017-01-16 12:04:00,http://bauchledner.io/noel_feest,0.2816123062,214.177.193.166,"{""location"": ""EG"", ""is_mobile"": false}" 20585,7,462,2016-12-28 01:42:23,http://runolfsdottir.io/gregorio,0.8456592720,20.71.245.24,"{""location"": ""RO"", ""is_mobile"": true}" 20586,7,462,2017-06-07 15:26:38,http://weinat.biz/lamont,0.4464005877,67.219.221.24,"{""location"": ""NZ"", ""is_mobile"": false}" 20587,7,462,2017-02-03 22:52:18,http://klein.io/maya,0.1058497737,35.103.186.133,"{""location"": ""VN"", ""is_mobile"": false}" 20588,7,462,2017-02-09 16:54:03,http://mertztowne.biz/brandon,0.4183055618,238.106.209.72,"{""location"": ""KW"", ""is_mobile"": true}" 20589,7,462,2017-03-02 09:13:27,http://runtewolff.com/marian,0.2426991058,93.191.87.111,"{""location"": ""SZ"", ""is_mobile"": true}" 20590,7,462,2017-03-01 09:34:38,http://gerhold.net/ansley,0.7570914303,223.52.213.172,"{""location"": ""BO"", ""is_mobile"": true}" 20591,7,462,2016-12-19 13:23:58,http://hyatt.co/annette,0.1724399421,33.154.234.164,"{""location"": ""TF"", ""is_mobile"": false}" 20592,7,462,2017-02-12 02:35:19,http://kshlerin.net/jenifer,0.2654034205,109.2.97.179,"{""location"": ""DO"", ""is_mobile"": false}" 20593,7,462,2017-02-16 08:33:04,http://brekke.org/allene,0.0022226707,111.139.196.233,"{""location"": ""NO"", ""is_mobile"": false}" 20594,7,462,2017-06-04 22:50:49,http://lowe.io/louie,0.7293006218,243.172.118.220,"{""location"": ""LS"", ""is_mobile"": false}" 20595,7,462,2017-04-02 20:25:24,http://toy.biz/lew,0.3639540202,53.248.180.172,"{""location"": ""SI"", ""is_mobile"": true}" 20596,7,462,2017-01-11 03:06:30,http://volkman.info/meaghan_witting,0.1048041616,230.177.241.83,"{""location"": ""CM"", ""is_mobile"": false}" 20597,7,462,2017-02-18 04:43:38,http://klein.info/kurtis,0.8442176372,137.154.244.126,"{""location"": ""JP"", ""is_mobile"": false}" 20598,7,462,2017-03-07 19:15:07,http://halvorsonschaefer.net/antwan,0.9091517907,27.220.193.58,"{""location"": ""UA"", ""is_mobile"": true}" 20599,7,462,2017-06-05 00:40:30,http://hansen.org/clemmie,0.9348182343,118.74.106.106,"{""location"": ""IL"", ""is_mobile"": false}" 20600,7,462,2017-01-11 06:37:14,http://cain.io/lance,0.9056704278,107.207.243.80,"{""location"": ""NF"", ""is_mobile"": false}" 20601,7,462,2016-12-23 05:55:01,http://altenwerth.name/dell,0.0125544969,24.73.78.98,"{""location"": ""MF"", ""is_mobile"": true}" 20602,7,462,2016-12-16 02:36:07,http://dooleywilliamson.co/jayme_sauer,0.2843900681,217.26.177.118,"{""location"": ""ZA"", ""is_mobile"": true}" 20603,7,462,2016-12-29 08:06:38,http://wuckert.co/jey_braun,0.5935775258,5.107.237.128,"{""location"": ""AQ"", ""is_mobile"": true}" 20604,7,462,2017-01-07 22:16:56,http://ebertokuneva.biz/jamil.schroeder,0.3650511850,112.219.237.186,"{""location"": ""CV"", ""is_mobile"": true}" 20605,7,462,2017-03-17 19:30:15,http://collins.org/rosanna,0.2961223934,150.86.210.149,"{""location"": ""IO"", ""is_mobile"": false}" 20606,7,462,2017-02-20 17:55:47,http://tremblay.org/rhiannon.schmitt,0.4414758421,209.174.129.90,"{""location"": ""DJ"", ""is_mobile"": false}" 20607,7,463,2017-02-14 10:23:09,http://quigley.biz/stephon_white,0.1735824547,186.4.132.6,"{""location"": ""JM"", ""is_mobile"": false}" 20608,7,463,2017-02-15 20:37:11,http://klingjacobi.name/michaela.bahringer,0.7096724095,171.244.203.91,"{""location"": ""KE"", ""is_mobile"": true}" 20609,7,463,2017-06-04 03:32:04,http://simonismarks.org/adelia.steuber,0.7940213578,75.185.73.111,"{""location"": ""DJ"", ""is_mobile"": true}" 20610,7,463,2017-05-20 03:46:33,http://hintz.co/austen,0.9490886976,179.19.116.47,"{""location"": ""GD"", ""is_mobile"": true}" 20611,7,463,2017-01-24 04:17:43,http://rutherford.org/jordane,0.3172445363,164.185.161.38,"{""location"": ""HK"", ""is_mobile"": true}" 20612,7,463,2017-01-28 10:02:42,http://lesch.net/molly.baumbach,0.2875007738,220.237.220.219,"{""location"": ""KH"", ""is_mobile"": false}" 20613,7,463,2017-03-13 23:21:38,http://dare.net/cielo.goodwin,0.1771021143,4.87.27.183,"{""location"": ""UG"", ""is_mobile"": false}" 20614,7,463,2017-04-03 02:10:46,http://braungreenfelder.info/taryn,0.5377825218,104.8.119.230,"{""location"": ""HT"", ""is_mobile"": true}" 20615,7,463,2017-05-18 13:04:24,http://bogantromp.co/noe.kub,0.7755756202,16.208.76.230,"{""location"": ""IR"", ""is_mobile"": true}" 20616,7,463,2017-03-07 18:31:28,http://boyle.net/eve,0.7181861309,254.185.237.29,"{""location"": ""YT"", ""is_mobile"": true}" 20617,7,463,2017-04-15 09:09:14,http://oreillymccullough.net/alden_luettgen,0.6257164736,184.148.142.63,"{""location"": ""UY"", ""is_mobile"": true}" 20618,7,463,2016-12-31 00:54:30,http://stokes.co/johann.okon,0.2142418431,140.153.15.11,"{""location"": ""FO"", ""is_mobile"": false}" 20619,7,463,2017-02-06 16:22:16,http://rogahncasper.com/richmond_oberbrunner,0.7722902539,18.160.118.126,"{""location"": ""SO"", ""is_mobile"": false}" 20620,7,463,2017-04-07 09:54:15,http://lind.com/aleandra,0.7527854663,55.46.46.200,"{""location"": ""GR"", ""is_mobile"": false}" 20621,7,463,2017-03-30 18:20:44,http://oconnellstanton.org/vernie_heel,0.0180662454,151.206.166.117,"{""location"": ""HK"", ""is_mobile"": true}" 20622,7,463,2017-04-03 14:27:43,http://kautzer.org/maxime,0.6472409280,30.89.82.33,"{""location"": ""QA"", ""is_mobile"": false}" 20623,7,463,2017-01-09 01:04:28,http://littel.com/nestor.leuschke,0.1875516591,119.80.188.112,"{""location"": ""PK"", ""is_mobile"": true}" 20624,7,463,2017-01-12 13:20:59,http://schmeler.biz/antonietta.thompson,0.9366288977,87.229.135.221,"{""location"": ""YT"", ""is_mobile"": true}" 20625,7,463,2017-05-21 17:16:53,http://weimann.org/monte,0.7200350545,61.189.149.19,"{""location"": ""AQ"", ""is_mobile"": true}" 20626,7,463,2017-03-17 23:20:25,http://kleinohara.name/korbin_ko,0.4157655493,98.189.175.63,"{""location"": ""LV"", ""is_mobile"": true}" 20627,7,463,2017-06-03 13:20:08,http://yost.net/greta,0.0308760670,76.43.198.91,"{""location"": ""BW"", ""is_mobile"": false}" 20628,7,463,2017-06-11 08:09:01,http://rogahnmcclure.io/ova,0.5492472117,134.93.93.175,"{""location"": ""TK"", ""is_mobile"": false}" 20629,7,463,2017-03-23 19:04:16,http://bashirian.name/adonis,0.5797671717,208.66.83.212,"{""location"": ""PH"", ""is_mobile"": false}" 20630,7,463,2017-03-24 08:28:53,http://pacocharomaguera.name/kurtis,0.5737046537,81.249.103.172,"{""location"": ""LU"", ""is_mobile"": true}" 20631,7,463,2017-06-05 05:12:31,http://mohr.info/neal.jaskolski,0.8590862698,242.115.187.207,"{""location"": ""CU"", ""is_mobile"": false}" 20632,7,463,2017-02-14 08:23:02,http://howe.net/caroline,0.4039133147,220.20.167.87,"{""location"": ""SA"", ""is_mobile"": false}" 20633,7,463,2017-05-22 13:33:21,http://yost.net/elenor_kihn,0.5459873248,245.54.159.66,"{""location"": ""GA"", ""is_mobile"": true}" 20634,7,463,2017-06-03 14:03:17,http://rodriguez.com/bertha.olson,0.7555875356,253.39.172.167,"{""location"": ""IM"", ""is_mobile"": false}" 20635,7,463,2017-03-30 21:15:56,http://kerluke.name/travon,0.4969776711,75.241.106.191,"{""location"": ""MV"", ""is_mobile"": false}" 20636,7,463,2017-02-24 13:28:24,http://beer.com/kayley.witting,0.3799572545,191.58.47.25,"{""location"": ""TG"", ""is_mobile"": false}" 20637,7,463,2017-02-05 22:47:41,http://dach.co/alan.ortiz,0.0718670616,124.93.30.42,"{""location"": ""BA"", ""is_mobile"": true}" 20638,7,463,2017-01-17 07:20:33,http://lueilwitzkeler.net/birdie.bahringer,0.6914998676,199.71.170.182,"{""location"": ""EE"", ""is_mobile"": false}" 20639,7,463,2017-05-03 03:10:44,http://purdy.co/camilla_raynor,0.0177320347,93.128.96.41,"{""location"": ""LV"", ""is_mobile"": false}" 20640,7,463,2017-06-02 23:04:58,http://swiftroob.name/gerhard,0.9323192100,18.30.204.130,"{""location"": ""CC"", ""is_mobile"": false}" 20641,7,463,2017-01-10 00:00:32,http://sawaynschaden.org/royal.gutkowski,0.0679063670,124.135.181.218,"{""location"": ""DZ"", ""is_mobile"": true}" 20642,7,463,2017-05-22 13:04:15,http://weimann.com/kiley.kemmer,0.9475287651,126.188.215.106,"{""location"": ""ML"", ""is_mobile"": false}" 20643,7,463,2017-01-27 11:09:58,http://marvin.biz/della,0.8993254470,228.194.69.150,"{""location"": ""MY"", ""is_mobile"": true}" 20644,7,463,2017-02-27 05:37:05,http://olsonheidenreich.info/mya,0.0348009912,45.228.105.223,"{""location"": ""GL"", ""is_mobile"": true}" 20645,7,463,2017-04-01 05:23:18,http://herzog.biz/montana.rodriguez,0.6499917124,30.70.146.224,"{""location"": ""FO"", ""is_mobile"": true}" 20646,7,463,2017-05-04 23:34:46,http://schmelerbuckridge.biz/sammie_abshire,0.0900730187,37.138.37.157,"{""location"": ""BL"", ""is_mobile"": true}" 20647,7,463,2017-04-04 13:12:40,http://gutmann.co/ron,0.1088638050,193.225.49.15,"{""location"": ""HN"", ""is_mobile"": false}" 20648,7,463,2017-01-30 16:24:26,http://larsonschinner.net/derrick,0.1253004166,223.235.238.197,"{""location"": ""ZA"", ""is_mobile"": false}" 20649,7,463,2017-01-01 10:49:47,http://kozey.co/eleazar,0.2521910634,222.125.6.3,"{""location"": ""MD"", ""is_mobile"": true}" 20650,7,463,2017-01-08 01:47:22,http://vonmetz.info/albina,0.8138741353,222.197.175.197,"{""location"": ""FR"", ""is_mobile"": true}" 20651,7,463,2017-01-17 22:44:31,http://nienow.io/orion,0.8481916212,125.31.73.9,"{""location"": ""GI"", ""is_mobile"": false}" 20652,7,463,2017-04-05 17:43:53,http://wehner.co/garth.jacobson,0.5647061458,188.46.162.14,"{""location"": ""BM"", ""is_mobile"": false}" 20653,7,463,2016-12-14 03:45:27,http://romaguera.co/eliane.jakubowski,0.4067478331,48.76.3.89,"{""location"": ""BA"", ""is_mobile"": true}" 20654,7,463,2017-01-08 13:05:34,http://lebsackheaney.info/patience,0.2398581857,64.254.23.115,"{""location"": ""EE"", ""is_mobile"": false}" 20655,7,463,2017-05-02 13:50:17,http://mante.org/dolores,0.3321204214,41.132.15.179,"{""location"": ""YE"", ""is_mobile"": false}" 20656,7,463,2017-03-07 14:46:39,http://heathcoteschmeler.net/gonzalo_fay,0.4141631719,144.173.204.83,"{""location"": ""ES"", ""is_mobile"": false}" 20657,7,463,2016-12-20 11:49:08,http://kojohnston.name/green.oreilly,0.5889941318,86.142.225.39,"{""location"": ""IM"", ""is_mobile"": true}" 20658,7,463,2017-03-26 19:13:42,http://shanahan.net/aimee,0.0273242832,210.198.235.95,"{""location"": ""SJ"", ""is_mobile"": false}" 20659,7,463,2017-02-07 22:06:57,http://kuvalis.io/rhiannon.conn,0.3894184426,244.42.68.146,"{""location"": ""RO"", ""is_mobile"": true}" 20660,7,463,2017-03-30 00:22:32,http://gleichner.net/rodolfo,0.2631553983,12.41.243.217,"{""location"": ""CR"", ""is_mobile"": false}" 20661,7,463,2017-05-03 09:47:40,http://harber.name/brody_baumbach,0.6544719070,140.163.173.211,"{""location"": ""SD"", ""is_mobile"": false}" 20662,7,463,2017-04-01 07:59:54,http://pacocha.co/constance,0.7011394629,135.164.191.148,"{""location"": ""DM"", ""is_mobile"": true}" 20663,7,463,2016-12-20 04:49:14,http://price.net/magnus.barrows,0.2574064906,190.111.112.148,"{""location"": ""GP"", ""is_mobile"": false}" 20664,7,463,2017-04-13 15:02:11,http://murphy.net/jeffrey,0.4464747018,249.172.10.95,"{""location"": ""DK"", ""is_mobile"": false}" 20665,7,463,2017-06-06 02:11:50,http://kautzer.co/benny,0.4597787984,104.230.50.7,"{""location"": ""KZ"", ""is_mobile"": false}" 20666,7,463,2017-05-09 17:43:03,http://dietrichrolfson.net/ibrahim_olson,0.6298198025,211.208.4.111,"{""location"": ""AG"", ""is_mobile"": true}" 20667,7,463,2017-05-30 15:58:11,http://upton.com/crystel_thiel,0.0942367782,194.126.222.134,"{""location"": ""JM"", ""is_mobile"": false}" 20668,7,464,2017-02-15 08:42:33,http://welch.net/micheal,0.7771281469,176.200.162.190,"{""location"": ""IL"", ""is_mobile"": false}" 20669,7,464,2017-01-03 03:21:05,http://quigley.biz/savion.ward,0.9591071755,245.151.171.207,"{""location"": ""CO"", ""is_mobile"": true}" 20670,7,464,2017-02-12 03:25:57,http://berge.org/arely_pollich,0.1123515732,154.198.47.226,"{""location"": ""TR"", ""is_mobile"": true}" 20671,7,464,2016-12-28 17:53:03,http://vonrueden.info/scottie,0.9035246850,108.230.64.159,"{""location"": ""PT"", ""is_mobile"": true}" 20672,7,464,2016-12-28 02:26:49,http://kuhnschultz.name/felicia.marks,0.7880815778,251.182.146.194,"{""location"": ""SA"", ""is_mobile"": true}" 20673,7,464,2017-06-05 08:38:31,http://veum.co/ariane,0.1594036039,136.227.201.46,"{""location"": ""MP"", ""is_mobile"": false}" 20674,7,464,2016-12-22 01:48:16,http://rueckerharris.name/elda,0.2958373810,234.54.188.249,"{""location"": ""BD"", ""is_mobile"": false}" 20675,7,464,2017-05-20 21:13:23,http://mohr.biz/mandy,0.0772311855,182.141.153.61,"{""location"": ""ID"", ""is_mobile"": true}" 20676,7,464,2017-05-09 04:22:13,http://kuhic.io/johnny.mitchell,0.5028327726,218.170.31.201,"{""location"": ""IO"", ""is_mobile"": false}" 20677,7,464,2017-04-06 07:08:50,http://greenholtwilkinson.com/oran.ohara,0.0526857484,144.142.125.249,"{""location"": ""ZA"", ""is_mobile"": false}" 20678,7,464,2016-12-26 16:17:17,http://stiedemannward.net/doyle,0.2296227008,205.25.97.213,"{""location"": ""BD"", ""is_mobile"": true}" 20679,7,464,2017-01-13 15:11:05,http://prosacco.org/claudia_feil,0.5399573719,98.88.12.203,"{""location"": ""ID"", ""is_mobile"": false}" 20680,7,464,2017-01-09 15:38:58,http://little.name/eulah.schultz,0.3012797400,142.61.146.4,"{""location"": ""NL"", ""is_mobile"": true}" 20681,7,464,2016-12-18 04:23:42,http://mrazkunze.io/jerad,0.9235316698,15.237.173.212,"{""location"": ""SZ"", ""is_mobile"": true}" 20682,7,464,2017-01-28 22:15:24,http://reynolds.com/joany_upton,0.6174983838,99.14.172.56,"{""location"": ""TN"", ""is_mobile"": false}" 20683,7,464,2017-03-15 14:01:50,http://doyleokon.net/wilfred,0.8533102235,126.122.116.99,"{""location"": ""YT"", ""is_mobile"": false}" 20684,7,464,2017-04-22 23:51:08,http://lakin.co/lizeth,0.4828241363,49.199.18.139,"{""location"": ""GP"", ""is_mobile"": false}" 20685,7,464,2017-04-27 11:52:23,http://gerholdcruickshank.name/eleanore,0.4533873498,197.180.183.41,"{""location"": ""GD"", ""is_mobile"": false}" 20686,7,464,2017-04-23 06:49:03,http://gerholdquigley.org/afton,0.4959406843,106.213.161.18,"{""location"": ""LB"", ""is_mobile"": false}" 20687,7,464,2017-03-30 16:27:41,http://donnelly.com/marina.gleason,0.1547354422,7.13.76.144,"{""location"": ""AM"", ""is_mobile"": true}" 20688,7,464,2017-04-17 20:22:37,http://hirthedickinson.name/allie.wilkinson,0.7152977059,177.38.12.91,"{""location"": ""NF"", ""is_mobile"": true}" 20689,7,464,2016-12-20 05:46:39,http://cremin.info/delmer,0.4385304038,80.34.49.160,"{""location"": ""DE"", ""is_mobile"": true}" 20690,7,464,2016-12-21 18:37:52,http://tremblayerdman.info/sienna,0.0649381109,25.233.231.209,"{""location"": ""TW"", ""is_mobile"": false}" 20691,7,464,2017-04-13 08:06:06,http://yundtwaelchi.org/wilfred_simonis,0.0447212653,156.205.96.79,"{""location"": ""KY"", ""is_mobile"": true}" 20692,7,464,2017-05-16 04:55:26,http://mohr.io/reyes,0.5253013240,128.245.184.29,"{""location"": ""SY"", ""is_mobile"": true}" 20693,7,464,2017-01-22 04:55:29,http://conroymacejkovic.org/misael_bartoletti,0.5741443610,67.207.159.106,"{""location"": ""GF"", ""is_mobile"": true}" 20694,7,464,2017-02-26 20:30:13,http://wuckert.name/alexane,0.0317358356,32.193.90.10,"{""location"": ""NG"", ""is_mobile"": false}" 20695,7,464,2017-02-14 19:25:54,http://runte.biz/wilmer_rodriguez,0.9082299032,150.246.141.19,"{""location"": ""PN"", ""is_mobile"": false}" 20696,7,464,2017-05-26 05:02:25,http://roob.io/hollie.mueller,0.0618998819,40.183.249.9,"{""location"": ""BV"", ""is_mobile"": true}" 20697,7,464,2017-02-13 11:53:35,http://schamberger.com/soledad.thiel,0.7355551218,122.50.111.171,"{""location"": ""BL"", ""is_mobile"": false}" 20698,7,464,2017-01-23 09:27:15,http://wolff.io/alexzander,0.5201348082,126.136.233.27,"{""location"": ""BV"", ""is_mobile"": true}" 20699,7,464,2017-01-29 14:56:09,http://rau.biz/taya,0.2621586861,112.163.158.236,"{""location"": ""GP"", ""is_mobile"": false}" 20700,7,464,2016-12-19 05:58:54,http://botsford.org/mohammed.mclaughlin,0.9708658472,40.168.35.243,"{""location"": ""EH"", ""is_mobile"": false}" 20701,7,464,2016-12-26 01:32:21,http://hermann.info/salvador_gutkowski,0.7178655437,102.84.178.65,"{""location"": ""AL"", ""is_mobile"": true}" 20702,7,464,2017-03-31 04:13:41,http://brekkedickens.com/karley,0.3246634590,213.45.25.142,"{""location"": ""GN"", ""is_mobile"": true}" 20703,7,464,2017-05-30 05:55:54,http://ruel.org/jody.kaulke,0.6204214454,150.193.70.213,"{""location"": ""KY"", ""is_mobile"": false}" 20704,7,464,2017-04-06 15:55:59,http://ondricka.com/violette.jakubowski,0.0856653617,226.161.17.210,"{""location"": ""CW"", ""is_mobile"": false}" 20705,7,464,2017-03-12 15:29:34,http://stanton.co/drake,0.7658475616,187.195.142.70,"{""location"": ""TH"", ""is_mobile"": true}" 20706,7,464,2017-05-18 09:43:39,http://lockman.biz/ward,0.6234949636,200.83.180.249,"{""location"": ""FM"", ""is_mobile"": false}" 20707,7,464,2017-06-11 01:17:46,http://marvinbraun.info/keshawn,0.4227261939,144.222.99.243,"{""location"": ""CA"", ""is_mobile"": true}" 20708,7,464,2017-05-27 00:45:33,http://upton.biz/theodore.beer,0.6159790710,73.104.69.23,"{""location"": ""MU"", ""is_mobile"": true}" 20709,7,464,2017-05-10 16:36:11,http://purdywintheiser.biz/thomas,0.4599664841,14.253.131.89,"{""location"": ""BQ"", ""is_mobile"": false}" 20710,7,464,2017-05-22 00:35:59,http://goodwin.biz/elwyn,0.2934692156,180.189.118.154,"{""location"": ""SC"", ""is_mobile"": true}" 20711,7,464,2017-01-07 01:53:33,http://davis.net/stacy_skiles,0.3973370525,154.112.228.206,"{""location"": ""HR"", ""is_mobile"": false}" 20712,7,464,2017-01-04 00:25:19,http://moore.org/helmer,0.9526056056,118.87.21.235,"{""location"": ""GG"", ""is_mobile"": true}" 20713,7,464,2017-05-24 04:13:09,http://kuphaljones.net/kendra,0.1183698339,249.12.217.37,"{""location"": ""MO"", ""is_mobile"": true}" 20714,7,464,2017-01-31 09:28:05,http://lynch.com/libby.feeney,0.5188299417,237.112.174.64,"{""location"": ""AZ"", ""is_mobile"": true}" 20715,7,464,2016-12-21 15:20:14,http://pollich.io/elroy,0.0680576374,183.134.211.128,"{""location"": ""FI"", ""is_mobile"": true}" 20716,7,464,2017-01-28 16:34:16,http://schamberger.info/kiel,0.1322103981,244.181.184.249,"{""location"": ""DZ"", ""is_mobile"": false}" 20717,7,464,2017-05-30 16:29:35,http://legros.io/hugh_block,0.3853684895,210.128.28.196,"{""location"": ""MU"", ""is_mobile"": true}" 20718,7,464,2017-04-16 22:22:11,http://dachauer.info/carmine,0.4245075758,248.37.176.108,"{""location"": ""BW"", ""is_mobile"": true}" 20719,7,464,2017-06-08 20:20:07,http://baumbachmosciski.biz/orland_homenick,0.7124332814,225.227.206.238,"{""location"": ""RE"", ""is_mobile"": true}" 20720,7,464,2017-05-21 16:09:14,http://fisherjakubowski.org/reggie.satterfield,0.4259934313,133.193.173.210,"{""location"": ""SS"", ""is_mobile"": false}" 20721,7,464,2017-01-01 23:59:39,http://stark.com/eugene_mertz,0.0336354613,169.93.126.25,"{""location"": ""DK"", ""is_mobile"": false}" 20722,7,464,2017-06-06 05:45:10,http://conn.co/godfrey.davis,0.2674962451,157.54.193.70,"{""location"": ""UG"", ""is_mobile"": false}" 20723,7,464,2017-04-11 11:43:56,http://boyer.info/stanton,0.5098841318,128.221.222.121,"{""location"": ""IT"", ""is_mobile"": true}" 20724,7,464,2017-06-13 20:20:52,http://wehnermann.org/werner_powlowski,0.5678745076,7.60.250.53,"{""location"": ""GS"", ""is_mobile"": false}" 20725,7,464,2017-01-21 12:59:30,http://dicki.biz/hattie,0.4277350558,198.127.197.182,"{""location"": ""YT"", ""is_mobile"": true}" 20726,7,464,2017-02-05 00:40:24,http://stehrrenner.org/janae_trantow,0.1961715293,130.72.19.225,"{""location"": ""HN"", ""is_mobile"": false}" 20727,7,464,2017-02-28 22:00:42,http://volkman.com/yazmin_reinger,0.9128622028,194.254.11.188,"{""location"": ""GS"", ""is_mobile"": false}" 20728,7,465,2017-03-20 12:57:56,http://erdmankertzmann.io/alexandre.pouros,0.8668532803,25.108.196.116,"{""location"": ""AX"", ""is_mobile"": true}" 20729,7,465,2017-02-26 16:52:39,http://mayertstiedemann.name/fred_bahringer,0.2201874422,102.126.52.197,"{""location"": ""MG"", ""is_mobile"": false}" 20730,7,465,2017-03-15 09:57:10,http://harvey.co/august_dooley,0.1024390323,102.65.199.132,"{""location"": ""MU"", ""is_mobile"": true}" 20731,7,465,2017-04-23 14:09:33,http://schmeler.org/anna_roob,0.2405552519,195.8.38.242,"{""location"": ""VG"", ""is_mobile"": true}" 20732,7,465,2017-05-29 04:21:53,http://moorebotsford.io/khalil_schamberger,0.7413494407,157.37.46.11,"{""location"": ""MK"", ""is_mobile"": true}" 20733,7,465,2017-01-26 12:59:19,http://effertzmuller.net/rowan_reichert,0.7478708921,194.51.59.247,"{""location"": ""PL"", ""is_mobile"": true}" 20734,7,465,2017-01-07 03:15:54,http://keeling.info/lenny.carroll,0.6506244590,157.89.200.132,"{""location"": ""FM"", ""is_mobile"": false}" 20735,7,465,2017-04-23 12:20:46,http://kohler.co/juliana,0.7163178441,87.52.56.53,"{""location"": ""ST"", ""is_mobile"": true}" 20736,7,465,2017-05-21 05:43:32,http://lehner.io/ettie_kohler,0.2861530705,49.50.184.57,"{""location"": ""DJ"", ""is_mobile"": false}" 20737,7,465,2017-04-01 06:55:57,http://schambergermoen.net/eleanora,0.6832681426,156.214.230.197,"{""location"": ""AR"", ""is_mobile"": false}" 20738,7,465,2017-04-23 03:16:01,http://boyleschaden.org/kira.frami,0.7757851649,199.225.126.210,"{""location"": ""CV"", ""is_mobile"": false}" 20739,7,465,2017-04-15 01:31:53,http://satterfieldtrantow.biz/zackary,0.1451707386,117.251.108.222,"{""location"": ""JE"", ""is_mobile"": false}" 20740,7,465,2017-03-10 10:00:41,http://price.org/lamar,0.3663464751,31.118.18.12,"{""location"": ""LV"", ""is_mobile"": false}" 20741,7,465,2017-05-29 01:03:51,http://steuber.co/maryam,0.3634874806,119.147.174.69,"{""location"": ""JM"", ""is_mobile"": false}" 20742,7,465,2017-02-17 22:02:58,http://conroy.org/eulalia,0.7318829594,55.117.225.44,"{""location"": ""EH"", ""is_mobile"": true}" 20743,7,465,2017-01-11 13:30:23,http://boyer.name/charlie,0.2709582728,196.207.97.204,"{""location"": ""NF"", ""is_mobile"": false}" 20744,7,465,2017-04-21 15:45:09,http://pouroskohler.net/earnestine,0.7666231048,53.157.53.9,"{""location"": ""CX"", ""is_mobile"": false}" 20745,7,465,2017-06-13 00:33:27,http://hansen.info/liza,0.2322296365,237.246.21.245,"{""location"": ""CD"", ""is_mobile"": true}" 20746,7,465,2017-02-05 18:24:16,http://rutherfordfadel.name/heloise.schoen,0.1777425800,227.110.173.208,"{""location"": ""DZ"", ""is_mobile"": true}" 20747,7,465,2017-04-15 23:59:59,http://lynch.io/johnny_luettgen,0.5583506265,208.173.21.118,"{""location"": ""DO"", ""is_mobile"": true}" 20748,7,465,2016-12-24 19:27:31,http://dach.co/benton,0.8467192068,250.51.235.236,"{""location"": ""JP"", ""is_mobile"": true}" 20749,7,465,2017-03-22 09:34:31,http://mrazhayes.org/jazlyn,0.4312089758,155.113.53.186,"{""location"": ""CV"", ""is_mobile"": false}" 20750,7,465,2017-01-29 12:00:40,http://andersonveum.com/daren.kunde,0.0700273867,124.181.166.154,"{""location"": ""ES"", ""is_mobile"": true}" 20751,7,465,2017-01-09 14:44:55,http://herzog.name/vince,0.8235911405,151.201.122.247,"{""location"": ""AR"", ""is_mobile"": false}" 20752,7,465,2016-12-27 09:35:09,http://legros.com/vern,0.3227479276,48.171.94.124,"{""location"": ""BQ"", ""is_mobile"": false}" 20753,7,465,2017-04-28 17:29:20,http://medhurst.name/nash.mckenzie,0.4346939075,168.109.44.71,"{""location"": ""EC"", ""is_mobile"": true}" 20754,7,465,2017-01-02 13:18:45,http://grady.info/ian_bergnaum,0.6576590651,190.228.251.237,"{""location"": ""BQ"", ""is_mobile"": false}" 20755,7,465,2017-04-18 00:45:28,http://kiehn.biz/roxanne,0.6556239678,218.55.94.78,"{""location"": ""KW"", ""is_mobile"": false}" 20756,7,465,2017-03-23 09:50:50,http://wehner.com/mayra_nikolaus,0.1900667676,250.58.129.134,"{""location"": ""LA"", ""is_mobile"": false}" 20757,7,465,2017-02-14 01:03:44,http://champlin.info/deontae,0.2413020067,104.181.5.89,"{""location"": ""UZ"", ""is_mobile"": true}" 20758,7,465,2016-12-31 03:17:38,http://mosciskibernier.biz/madisyn.kertzmann,0.2057230900,104.48.155.111,"{""location"": ""BA"", ""is_mobile"": false}" 20759,7,465,2016-12-21 14:55:20,http://kling.com/dock_dietrich,0.9768993600,238.4.224.166,"{""location"": ""FM"", ""is_mobile"": true}" 20760,7,465,2016-12-24 15:41:30,http://stark.co/francisca,0.0064629913,12.164.136.123,"{""location"": ""IN"", ""is_mobile"": true}" 20761,7,465,2017-06-06 05:18:02,http://howell.com/audreanne,0.9165147073,117.243.156.52,"{""location"": ""UY"", ""is_mobile"": true}" 20762,7,466,2017-04-06 22:39:49,http://zulauf.net/ismael,,139.175.103.102,"{""location"": ""NR"", ""is_mobile"": false}" 20763,7,466,2017-01-24 03:47:27,http://leannonjaskolski.com/gerardo.hyatt,,197.127.187.165,"{""location"": ""KG"", ""is_mobile"": false}" 20764,7,466,2017-05-10 07:07:24,http://schmitt.biz/greyson_ferry,,114.51.129.87,"{""location"": ""TR"", ""is_mobile"": false}" 20765,7,466,2017-02-16 01:45:32,http://abshire.co/jarrett_hoppe,,30.56.78.111,"{""location"": ""FI"", ""is_mobile"": true}" 20766,7,466,2017-04-20 15:15:02,http://boscomclaughlin.name/reyna.stokes,,254.130.242.146,"{""location"": ""NG"", ""is_mobile"": false}" 20767,7,466,2017-03-15 15:10:34,http://lesch.name/ania.stark,,125.147.232.126,"{""location"": ""UA"", ""is_mobile"": true}" 20768,7,466,2017-03-01 19:30:55,http://west.co/carolyne.stamm,,154.118.65.110,"{""location"": ""LA"", ""is_mobile"": true}" 20769,7,466,2017-02-20 02:38:33,http://corwin.co/kianna.skiles,,71.98.159.193,"{""location"": ""RW"", ""is_mobile"": false}" 20770,7,466,2017-05-23 00:08:34,http://mann.com/bernardo,,195.177.142.222,"{""location"": ""AD"", ""is_mobile"": false}" 20771,7,466,2017-02-18 00:05:02,http://hintzschmitt.info/reva,,182.163.33.68,"{""location"": ""NG"", ""is_mobile"": true}" 20772,7,466,2017-05-30 15:47:13,http://heathcotezieme.net/jillian,,195.146.9.17,"{""location"": ""FR"", ""is_mobile"": true}" 20773,7,466,2017-05-09 22:46:06,http://kuhic.org/marcelle_wehner,,213.71.183.230,"{""location"": ""PK"", ""is_mobile"": true}" 20774,7,466,2017-05-06 08:34:45,http://fahey.io/lexi.heller,,251.63.51.151,"{""location"": ""SB"", ""is_mobile"": true}" 20775,7,466,2017-02-12 00:42:05,http://carroll.org/esperanza,,164.15.37.50,"{""location"": ""AI"", ""is_mobile"": false}" 20776,7,466,2017-01-16 06:05:31,http://leuschke.co/maude_rosenbaum,,143.193.156.174,"{""location"": ""IT"", ""is_mobile"": true}" 20777,7,466,2017-03-07 11:30:15,http://millerbreitenberg.com/immanuel,,121.98.102.210,"{""location"": ""JP"", ""is_mobile"": true}" 20778,7,466,2016-12-31 13:41:04,http://rempelhalvorson.name/berenice.erdman,,15.77.33.81,"{""location"": ""AZ"", ""is_mobile"": false}" 20779,7,466,2017-02-05 20:58:48,http://hackett.info/emilio_gutmann,,222.159.250.154,"{""location"": ""SS"", ""is_mobile"": false}" 20780,7,466,2017-02-18 21:40:18,http://mosciskiheathcote.name/selina_schimmel,,237.236.244.219,"{""location"": ""LB"", ""is_mobile"": false}" 20781,7,466,2017-05-06 23:49:00,http://bosco.com/ryann,,162.78.34.33,"{""location"": ""LR"", ""is_mobile"": false}" 20782,7,466,2017-04-21 16:00:03,http://hettinger.net/myron_runolfsdottir,,51.174.142.118,"{""location"": ""BW"", ""is_mobile"": true}" 20783,7,466,2017-02-25 15:36:56,http://feest.info/aletha.larkin,,166.190.83.80,"{""location"": ""AO"", ""is_mobile"": false}" 20784,7,466,2017-04-25 10:00:48,http://creminmayert.net/wilmer,,42.253.174.10,"{""location"": ""FO"", ""is_mobile"": true}" 20785,7,466,2017-05-26 18:25:34,http://hagenes.info/damaris,,234.254.144.34,"{""location"": ""BR"", ""is_mobile"": true}" 20786,7,466,2017-06-06 22:16:42,http://bins.name/andre,,211.39.235.248,"{""location"": ""DE"", ""is_mobile"": true}" 20787,7,466,2017-05-09 15:36:36,http://gibsonruecker.com/keegan,,249.251.157.154,"{""location"": ""SM"", ""is_mobile"": false}" 20788,7,466,2017-04-16 15:50:37,http://ankunding.org/ervin,,55.137.166.181,"{""location"": ""DO"", ""is_mobile"": true}" 20789,7,466,2017-01-05 07:45:39,http://fadel.io/henri,,218.171.144.7,"{""location"": ""TG"", ""is_mobile"": true}" 20790,7,466,2017-05-21 09:00:24,http://steuber.com/lenny_kreiger,,3.201.213.83,"{""location"": ""CH"", ""is_mobile"": true}" 20791,7,466,2016-12-21 10:08:14,http://berniergleichner.co/adolphus,,187.64.90.181,"{""location"": ""MV"", ""is_mobile"": false}" 20792,7,466,2017-04-24 19:36:05,http://abshire.name/cathy,,125.137.179.190,"{""location"": ""CN"", ""is_mobile"": false}" 20793,7,466,2017-04-29 13:57:14,http://senger.name/gielle,,204.243.125.230,"{""location"": ""TC"", ""is_mobile"": true}" 20794,7,467,2017-01-17 15:35:03,http://gorczany.net/garrett,,134.101.38.36,"{""location"": ""ES"", ""is_mobile"": true}" 20795,7,467,2017-03-01 14:16:07,http://hermannschneider.info/angelo.jaskolski,,223.139.134.231,"{""location"": ""BQ"", ""is_mobile"": false}" 20796,7,467,2017-03-20 19:53:09,http://ziemannhyatt.info/nikita.jerde,,117.61.134.112,"{""location"": ""JO"", ""is_mobile"": false}" 20797,7,467,2017-06-06 11:38:40,http://spencer.name/alexander,,96.5.238.214,"{""location"": ""SO"", ""is_mobile"": false}" 20798,7,467,2017-05-17 17:56:01,http://kirlin.org/brittany_mills,,199.150.181.21,"{""location"": ""ID"", ""is_mobile"": true}" 20799,7,467,2016-12-24 00:29:15,http://boscospinka.io/bethany.bruen,,120.49.196.3,"{""location"": ""LV"", ""is_mobile"": true}" 20800,7,467,2017-03-04 23:06:28,http://legros.io/leta_roberts,,156.43.186.15,"{""location"": ""KY"", ""is_mobile"": true}" 20801,7,467,2017-05-06 00:17:20,http://haagkertzmann.info/filomena_kertzmann,,197.76.238.249,"{""location"": ""FR"", ""is_mobile"": true}" 20802,7,467,2017-04-07 17:40:06,http://feest.net/landen,,105.225.97.91,"{""location"": ""GY"", ""is_mobile"": true}" 20803,7,467,2017-05-09 20:08:33,http://bayer.name/beverly_hills,,99.63.188.137,"{""location"": ""KZ"", ""is_mobile"": true}" 20804,7,467,2016-12-23 03:52:37,http://lynchzulauf.net/naomie,,20.131.162.212,"{""location"": ""CK"", ""is_mobile"": true}" 20805,7,467,2017-05-28 05:00:25,http://ondrickaterry.com/al,,3.104.197.32,"{""location"": ""RO"", ""is_mobile"": true}" 20806,7,467,2017-04-17 14:07:03,http://romaguera.biz/ezekiel,,146.109.235.5,"{""location"": ""KZ"", ""is_mobile"": false}" 20807,7,467,2017-04-05 05:17:36,http://lebsack.io/dina,,141.182.228.158,"{""location"": ""CK"", ""is_mobile"": false}" 20808,7,467,2017-04-24 23:59:24,http://kertzmann.co/dee,,127.105.7.151,"{""location"": ""DE"", ""is_mobile"": false}" 20809,7,467,2017-01-07 09:08:22,http://vonbode.org/flavio.vonrueden,,117.253.51.74,"{""location"": ""NC"", ""is_mobile"": false}" 20810,7,467,2017-01-13 22:12:53,http://haagkulas.name/johnson,,124.148.69.25,"{""location"": ""RE"", ""is_mobile"": false}" 20811,7,467,2017-05-16 15:33:52,http://gusikowskimueller.org/king,,126.240.39.96,"{""location"": ""GQ"", ""is_mobile"": false}" 20812,7,467,2017-06-12 07:09:25,http://okeefefeest.com/mike.thompson,,65.107.105.210,"{""location"": ""BW"", ""is_mobile"": true}" 20813,7,467,2017-02-05 00:18:09,http://kirlin.io/ayana.hintz,,26.165.44.241,"{""location"": ""MP"", ""is_mobile"": false}" 20814,7,467,2017-04-19 21:29:19,http://walsh.net/casandra_bergstrom,,43.90.242.172,"{""location"": ""TD"", ""is_mobile"": false}" 20815,7,468,2017-01-13 04:06:21,http://wymanhansen.net/ward.kuhn,,86.2.123.56,"{""location"": ""GW"", ""is_mobile"": false}" 20816,7,468,2017-06-02 06:07:44,http://wuckertbrown.biz/audrey,,37.207.254.43,"{""location"": ""AZ"", ""is_mobile"": false}" 20817,7,468,2017-05-13 07:50:58,http://yost.co/sienna,,164.22.29.56,"{""location"": ""DZ"", ""is_mobile"": true}" 20818,7,468,2017-01-25 04:19:52,http://schinnerdubuque.biz/tyson_hermiston,,235.242.139.228,"{""location"": ""ZW"", ""is_mobile"": true}" 20819,7,468,2017-01-19 06:42:53,http://hintzjacobi.co/marlen,,140.74.189.47,"{""location"": ""ZW"", ""is_mobile"": false}" 20820,7,468,2017-03-12 10:29:50,http://sawaynmarquardt.biz/constantin,,230.205.232.79,"{""location"": ""ZA"", ""is_mobile"": false}" 20821,7,468,2017-03-01 21:20:50,http://schoen.io/tyrique,,24.83.212.234,"{""location"": ""SH"", ""is_mobile"": true}" 20822,7,468,2017-04-06 04:59:15,http://paucek.co/mathias_oconner,,26.218.115.48,"{""location"": ""MZ"", ""is_mobile"": false}" 20823,7,468,2017-03-12 00:00:14,http://reynolds.biz/jailyn,,221.84.32.32,"{""location"": ""TT"", ""is_mobile"": false}" 20824,7,468,2017-05-31 18:30:45,http://friesen.org/leta,,196.27.152.62,"{""location"": ""MA"", ""is_mobile"": false}" 20825,7,468,2017-01-01 11:35:58,http://kunze.info/jackson,,113.191.15.208,"{""location"": ""CM"", ""is_mobile"": true}" 20826,7,468,2017-04-29 16:09:22,http://kovacekrutherford.net/chanel,,153.72.117.71,"{""location"": ""VI"", ""is_mobile"": false}" 20827,7,468,2017-02-08 04:00:33,http://luettgen.biz/graham,,218.228.91.9,"{""location"": ""VN"", ""is_mobile"": true}" 20828,7,468,2017-02-06 14:14:44,http://bayermuller.info/olen,,235.180.61.216,"{""location"": ""UY"", ""is_mobile"": true}" 20829,7,468,2017-06-06 19:16:39,http://kautzer.name/alvina,,185.78.20.224,"{""location"": ""PF"", ""is_mobile"": true}" 20830,7,468,2017-05-07 05:09:39,http://johns.name/joany,,165.133.179.24,"{""location"": ""SV"", ""is_mobile"": true}" 20831,7,468,2016-12-18 09:56:44,http://heathcote.com/mathew,,26.2.114.68,"{""location"": ""PW"", ""is_mobile"": false}" 20832,7,468,2017-01-26 10:56:09,http://baumbach.org/nicholaus,,61.84.164.11,"{""location"": ""GR"", ""is_mobile"": false}" 20833,7,468,2017-03-30 15:41:25,http://fay.name/reagan.shanahan,,159.113.205.244,"{""location"": ""CL"", ""is_mobile"": true}" 20834,7,468,2017-02-21 12:21:06,http://dickens.io/candelario,,35.201.14.134,"{""location"": ""SH"", ""is_mobile"": false}" 20835,7,468,2017-05-14 14:55:20,http://hoppe.net/ernesto.will,,234.130.228.83,"{""location"": ""AF"", ""is_mobile"": true}" 20836,7,468,2017-05-09 16:08:10,http://bogisich.com/curtis,,175.55.148.101,"{""location"": ""MD"", ""is_mobile"": true}" 20837,7,468,2017-01-31 09:26:19,http://nicolas.com/alexzander_mcdermott,,27.116.170.43,"{""location"": ""MS"", ""is_mobile"": false}" 20838,7,468,2017-06-13 01:20:16,http://feeney.org/itzel_west,,210.55.219.187,"{""location"": ""CN"", ""is_mobile"": true}" 20839,7,468,2017-01-19 14:31:35,http://anderson.net/dejuan.padberg,,108.210.57.182,"{""location"": ""SZ"", ""is_mobile"": true}" 20840,7,468,2017-02-12 17:57:13,http://brown.org/orlo.spencer,,194.74.241.53,"{""location"": ""BN"", ""is_mobile"": true}" 20841,7,468,2016-12-19 10:57:32,http://willledner.co/ruth.denesik,,239.252.181.158,"{""location"": ""BQ"", ""is_mobile"": true}" 20842,7,468,2017-01-18 11:40:43,http://ullrichfriesen.io/americo.hilpert,,181.189.220.16,"{""location"": ""BW"", ""is_mobile"": true}" 20843,7,468,2017-05-17 10:24:58,http://olson.info/gregory.kub,,84.245.45.34,"{""location"": ""EG"", ""is_mobile"": false}" 20844,7,468,2017-04-16 14:17:47,http://kunze.co/santino.pfannerstill,,118.204.155.41,"{""location"": ""BO"", ""is_mobile"": false}" 20845,7,468,2017-03-07 18:31:38,http://bailey.io/jerad.runolfon,,7.210.64.184,"{""location"": ""LI"", ""is_mobile"": false}" 20846,7,468,2017-01-04 23:26:19,http://cole.org/keven,,254.100.193.99,"{""location"": ""IL"", ""is_mobile"": false}" 20847,7,468,2017-01-05 18:15:27,http://wilkinson.name/annie,,186.53.187.223,"{""location"": ""MC"", ""is_mobile"": false}" 20848,7,468,2017-06-12 18:43:38,http://reichel.org/milo,,188.168.242.221,"{""location"": ""BZ"", ""is_mobile"": false}" 20849,7,468,2017-01-31 16:18:12,http://colliersanford.com/rodrigo_senger,,75.217.122.129,"{""location"": ""PN"", ""is_mobile"": true}" 20850,7,468,2017-05-14 10:07:18,http://leschko.biz/reece.frami,,83.92.73.26,"{""location"": ""WS"", ""is_mobile"": false}" 20851,7,468,2016-12-29 14:20:24,http://boyer.co/burnice,,74.127.111.197,"{""location"": ""IE"", ""is_mobile"": true}" 20852,7,468,2017-01-07 09:39:02,http://lynchhuel.io/adelbert_moore,,131.83.161.107,"{""location"": ""PR"", ""is_mobile"": false}" 20853,7,468,2017-01-03 05:09:44,http://walker.io/kennith.goldner,,222.152.6.148,"{""location"": ""UY"", ""is_mobile"": true}" 20854,7,469,2017-04-08 06:23:38,http://frami.io/bradly,,120.206.47.115,"{""location"": ""CW"", ""is_mobile"": false}" 20855,7,469,2016-12-18 12:52:38,http://beer.com/valentina_langworth,,208.183.68.105,"{""location"": ""FR"", ""is_mobile"": false}" 20856,7,469,2017-02-19 10:09:37,http://mohr.com/trea.langosh,,195.33.69.90,"{""location"": ""LC"", ""is_mobile"": false}" 20857,7,469,2017-02-17 11:37:56,http://mosciski.info/jameson,,78.101.14.146,"{""location"": ""GW"", ""is_mobile"": true}" 20858,7,469,2017-05-06 08:56:00,http://ortiz.io/omari,,118.230.153.140,"{""location"": ""MD"", ""is_mobile"": true}" 20859,7,469,2017-02-27 07:36:02,http://johns.info/destini,,178.191.225.52,"{""location"": ""SH"", ""is_mobile"": true}" 20860,7,469,2017-04-11 11:50:47,http://gorczany.net/corrine,,193.143.183.124,"{""location"": ""IM"", ""is_mobile"": true}" 20861,7,469,2016-12-17 02:46:51,http://hintzritchie.biz/laura,,39.135.41.76,"{""location"": ""US"", ""is_mobile"": false}" 20862,7,469,2017-02-04 02:58:05,http://gerholdhayes.co/beverly.kuhic,,161.67.78.175,"{""location"": ""DK"", ""is_mobile"": false}" 20863,7,469,2017-02-12 20:17:58,http://keeling.com/rigoberto,,57.211.117.152,"{""location"": ""VC"", ""is_mobile"": false}" 20864,7,469,2017-03-06 11:39:54,http://zboncakweimann.net/maye.connelly,,231.191.65.85,"{""location"": ""SN"", ""is_mobile"": false}" 20865,7,469,2017-05-01 14:33:45,http://hoppe.biz/destiney_toy,,146.179.178.124,"{""location"": ""SN"", ""is_mobile"": false}" 20866,7,469,2017-06-11 03:10:37,http://heel.net/juliet,,18.94.183.125,"{""location"": ""AI"", ""is_mobile"": true}" 20867,7,469,2017-01-13 20:30:20,http://pfefferupton.info/virginia,,76.83.178.176,"{""location"": ""AO"", ""is_mobile"": true}" 20868,7,469,2017-04-05 11:23:09,http://schaefersporer.name/haley.reichert,,107.19.64.175,"{""location"": ""ID"", ""is_mobile"": true}" 20869,7,469,2017-02-15 23:34:47,http://barrows.net/billie,,16.19.20.39,"{""location"": ""SM"", ""is_mobile"": false}" 20870,7,469,2017-03-06 16:51:11,http://okon.co/evie,,253.136.66.183,"{""location"": ""IN"", ""is_mobile"": false}" 20871,7,469,2017-05-28 16:13:49,http://lakin.org/hoyt,,18.214.11.74,"{""location"": ""TG"", ""is_mobile"": true}" 20874,7,469,2017-01-18 08:55:26,http://jakubowskihamill.com/heath,,159.169.224.10,"{""location"": ""RS"", ""is_mobile"": false}" 20875,7,469,2017-04-19 00:17:31,http://doylekoch.com/orrin.bins,,4.220.132.38,"{""location"": ""TR"", ""is_mobile"": true}" 20876,7,469,2017-03-03 01:01:51,http://kihn.co/arnold_jacobi,,121.51.61.210,"{""location"": ""SE"", ""is_mobile"": true}" 20877,7,469,2017-01-16 16:23:52,http://wildermanharber.biz/claudie.schaefer,,22.82.185.203,"{""location"": ""IL"", ""is_mobile"": true}" 20878,7,470,2017-04-07 00:26:21,http://konopelskistehr.com/jerad.schoen,,211.30.51.48,"{""location"": ""NP"", ""is_mobile"": true}" 20879,7,470,2017-03-23 15:17:37,http://kub.biz/davion.grant,,239.90.63.19,"{""location"": ""GN"", ""is_mobile"": false}" 20880,7,470,2017-01-20 06:01:21,http://shanahan.com/pietro_farrell,,3.78.72.78,"{""location"": ""SN"", ""is_mobile"": true}" 20881,7,470,2017-04-19 04:25:21,http://breitenberg.name/lila.reinger,,110.29.229.95,"{""location"": ""AL"", ""is_mobile"": false}" 20882,7,470,2017-05-06 21:09:53,http://hagenes.io/cory,,190.109.89.48,"{""location"": ""OM"", ""is_mobile"": true}" 20883,7,470,2017-04-27 18:20:20,http://dibbert.org/federico,,182.129.205.70,"{""location"": ""SX"", ""is_mobile"": true}" 20884,7,470,2017-03-27 21:47:56,http://pacochakirlin.net/dante,,30.196.7.228,"{""location"": ""MZ"", ""is_mobile"": true}" 20885,7,470,2017-01-27 23:04:22,http://hyattcrist.info/cali_mraz,,183.40.86.203,"{""location"": ""HK"", ""is_mobile"": false}" 20886,7,470,2017-05-12 21:49:47,http://fritschlubowitz.info/marcus,,97.72.10.41,"{""location"": ""LI"", ""is_mobile"": true}" 20887,7,470,2016-12-17 05:36:00,http://wilkinsonmuller.info/brendan_tromp,,16.86.147.181,"{""location"": ""TW"", ""is_mobile"": true}" 20888,7,470,2017-01-08 06:29:26,http://mertzreichert.info/morton.effertz,,48.135.103.114,"{""location"": ""SY"", ""is_mobile"": true}" 20889,7,470,2017-02-26 12:29:36,http://nitzsche.com/vernie.shields,,94.115.223.78,"{""location"": ""AQ"", ""is_mobile"": false}" 20890,7,470,2017-02-25 06:01:13,http://lindernser.org/bernadette,,234.102.35.87,"{""location"": ""MD"", ""is_mobile"": false}" 20891,7,470,2017-02-26 12:34:07,http://leuschkeschroeder.io/nathaniel_kris,,252.194.14.54,"{""location"": ""BN"", ""is_mobile"": true}" 20892,7,470,2017-02-11 08:20:12,http://gerhold.org/monica.greenholt,,150.56.155.125,"{""location"": ""VN"", ""is_mobile"": false}" 20893,7,470,2017-01-15 02:07:48,http://gerhold.co/ramon_renner,,21.130.22.115,"{""location"": ""AZ"", ""is_mobile"": false}" 20894,7,470,2017-03-19 20:00:13,http://lockman.org/aliza.roob,,126.208.48.220,"{""location"": ""ET"", ""is_mobile"": false}" 20895,7,470,2017-02-25 14:25:12,http://brekke.com/theo_anderson,,213.73.226.207,"{""location"": ""SH"", ""is_mobile"": true}" 20896,7,470,2017-05-04 11:52:13,http://leschmohr.name/gina,,86.74.112.248,"{""location"": ""BE"", ""is_mobile"": false}" 20897,7,470,2017-03-08 17:55:11,http://marvin.io/aiyana_harber,,45.238.248.179,"{""location"": ""TM"", ""is_mobile"": true}" 20898,7,470,2017-06-10 08:46:55,http://rogahnglover.org/rosalee.nader,,203.9.207.79,"{""location"": ""PM"", ""is_mobile"": true}" 20899,7,470,2017-06-10 01:31:34,http://vonrueden.org/louie,,63.123.174.144,"{""location"": ""BY"", ""is_mobile"": false}" 20900,7,470,2017-03-14 01:05:18,http://kertzmann.net/ena.pacocha,,211.17.130.70,"{""location"": ""RE"", ""is_mobile"": true}" 20901,7,470,2017-04-29 12:01:58,http://howe.io/orion_nicolas,,98.6.76.140,"{""location"": ""AZ"", ""is_mobile"": true}" 20902,7,470,2017-04-02 18:23:19,http://ratke.org/sheila,,126.241.199.82,"{""location"": ""MG"", ""is_mobile"": false}" 20903,7,470,2017-04-21 13:29:41,http://kshlerin.info/ashton,,226.51.149.100,"{""location"": ""AD"", ""is_mobile"": false}" 20904,7,470,2017-05-24 20:15:25,http://mayer.com/laurel,,138.15.46.204,"{""location"": ""GM"", ""is_mobile"": true}" 20905,7,470,2017-03-27 21:01:00,http://romaguera.net/naomi,,174.190.54.254,"{""location"": ""NO"", ""is_mobile"": true}" 20906,7,470,2017-01-04 22:44:02,http://marquardt.net/esther,,153.206.208.233,"{""location"": ""IM"", ""is_mobile"": false}" 20907,7,470,2017-04-14 13:15:28,http://greenholt.co/april,,42.147.46.167,"{""location"": ""ES"", ""is_mobile"": true}" 20908,7,470,2017-06-12 18:37:54,http://padbergharris.com/molly.bergnaum,,106.32.236.177,"{""location"": ""NI"", ""is_mobile"": false}" 20909,7,470,2017-01-13 18:57:55,http://orngrant.biz/ashley.sporer,,225.141.8.61,"{""location"": ""GU"", ""is_mobile"": true}" 20910,7,470,2017-03-24 18:07:25,http://olson.co/hubert,,230.81.165.195,"{""location"": ""BL"", ""is_mobile"": false}" 20911,7,470,2017-05-23 18:53:09,http://wittingcormier.net/pamela,,53.230.34.105,"{""location"": ""DK"", ""is_mobile"": true}" 20912,7,470,2017-03-27 14:37:29,http://mccullough.info/jeremy,,204.145.25.205,"{""location"": ""MQ"", ""is_mobile"": false}" 20913,7,470,2017-01-12 05:49:55,http://braun.biz/nico,,225.130.193.102,"{""location"": ""TW"", ""is_mobile"": true}" 20914,7,470,2016-12-27 14:57:49,http://lindgren.io/derrick,,241.161.167.245,"{""location"": ""NP"", ""is_mobile"": false}" 20915,7,470,2017-06-13 00:13:55,http://goyette.com/bertrand_moore,,111.70.133.137,"{""location"": ""BI"", ""is_mobile"": false}" 20916,7,470,2017-04-18 00:15:50,http://nicolas.com/myrtis,,202.72.50.170,"{""location"": ""FK"", ""is_mobile"": false}" 20917,7,470,2017-06-06 18:32:06,http://oberbrunnerleffler.info/flavio,,199.106.89.147,"{""location"": ""KZ"", ""is_mobile"": true}" 20918,7,470,2017-05-19 22:33:39,http://dickinson.org/lela,,104.168.50.13,"{""location"": ""MO"", ""is_mobile"": false}" 20919,7,471,2017-03-28 19:01:54,http://oberbrunnerframi.name/willis.heathcote,,172.179.233.18,"{""location"": ""CU"", ""is_mobile"": true}" 20920,7,471,2017-04-14 03:34:40,http://simonis.com/mabelle_considine,,103.235.57.16,"{""location"": ""MT"", ""is_mobile"": false}" 20921,7,471,2016-12-19 03:35:11,http://treutelzieme.co/ettie.conn,,24.69.233.104,"{""location"": ""KI"", ""is_mobile"": true}" 20922,7,471,2017-05-03 17:13:42,http://herzog.biz/maximillian,,135.176.50.96,"{""location"": ""VC"", ""is_mobile"": true}" 20923,7,471,2017-02-10 18:03:11,http://mcglynn.biz/eugene,,236.116.160.221,"{""location"": ""CO"", ""is_mobile"": false}" 20924,7,471,2017-05-16 08:59:42,http://gerlachbeahan.com/alf,,162.213.235.212,"{""location"": ""ZW"", ""is_mobile"": true}" 20925,7,471,2017-02-04 17:27:52,http://douglas.biz/newell,,53.49.185.125,"{""location"": ""PM"", ""is_mobile"": false}" 20926,7,471,2017-02-20 04:48:43,http://dickinson.org/brook,,41.77.28.220,"{""location"": ""KN"", ""is_mobile"": false}" 20927,7,471,2017-01-12 23:05:15,http://collins.name/uriah,,150.181.135.232,"{""location"": ""MY"", ""is_mobile"": false}" 20928,7,471,2017-03-19 07:31:06,http://rippin.org/nya_marvin,,128.14.20.206,"{""location"": ""MX"", ""is_mobile"": true}" 20929,7,471,2016-12-21 20:28:02,http://reinger.org/margarett,,211.62.65.254,"{""location"": ""TT"", ""is_mobile"": true}" 20930,7,471,2017-03-25 14:06:40,http://hansenmurphy.name/daryl.kshlerin,,48.188.181.63,"{""location"": ""NI"", ""is_mobile"": true}" 20931,7,471,2017-02-10 21:22:12,http://considinekuvalis.name/murphy.witting,,77.95.58.220,"{""location"": ""NC"", ""is_mobile"": true}" 20932,7,471,2017-05-20 04:51:30,http://mosciski.com/dusty,,140.94.250.87,"{""location"": ""BD"", ""is_mobile"": true}" 20933,7,471,2017-03-27 01:55:37,http://erdmanjones.name/kyle.bins,,99.133.66.172,"{""location"": ""BF"", ""is_mobile"": false}" 20934,7,471,2017-01-23 19:53:33,http://bruenfeeney.com/salvatore_lynch,,71.69.169.54,"{""location"": ""AF"", ""is_mobile"": true}" 20935,7,471,2017-04-22 11:49:12,http://lynch.biz/michale,,158.11.249.172,"{""location"": ""GS"", ""is_mobile"": false}" 20936,7,471,2017-05-26 09:06:35,http://stehrconn.info/eulah,,254.33.194.192,"{""location"": ""KP"", ""is_mobile"": true}" 20937,7,471,2017-06-09 09:19:10,http://glovermueller.co/demarco_vandervort,,147.119.70.50,"{""location"": ""FO"", ""is_mobile"": true}" 20938,7,471,2017-02-08 11:20:09,http://mayer.co/dorthy.daugherty,,205.197.206.145,"{""location"": ""DE"", ""is_mobile"": false}" 20939,7,471,2017-05-05 03:53:23,http://wiza.info/darryl_gusikowski,,169.36.18.122,"{""location"": ""MH"", ""is_mobile"": false}" 20940,7,471,2017-02-20 17:40:58,http://williamson.org/jamel,,170.49.130.156,"{""location"": ""GQ"", ""is_mobile"": false}" 20941,7,471,2017-02-03 10:52:48,http://kuhn.io/laury,,94.221.83.143,"{""location"": ""TF"", ""is_mobile"": false}" 20942,7,471,2017-03-20 01:22:07,http://ko.info/carolyne_deckow,,33.137.171.114,"{""location"": ""TK"", ""is_mobile"": true}" 20943,7,471,2017-04-07 18:02:16,http://mueller.com/clare.collins,,211.125.5.131,"{""location"": ""SJ"", ""is_mobile"": true}" 20944,7,471,2017-02-14 17:37:53,http://runolfsdottirhettinger.com/rubie,,187.81.106.146,"{""location"": ""UZ"", ""is_mobile"": false}" 20945,7,471,2017-04-24 12:19:16,http://schaefer.biz/enrique,,224.24.234.227,"{""location"": ""JE"", ""is_mobile"": false}" 20946,7,471,2017-01-05 03:04:40,http://batzgibson.com/caitlyn.mitchell,,92.40.204.170,"{""location"": ""PE"", ""is_mobile"": true}" 20947,7,471,2017-04-28 02:09:38,http://gorczany.co/phyllis_volkman,,133.159.46.24,"{""location"": ""MM"", ""is_mobile"": true}" 20948,7,471,2017-05-01 22:47:31,http://simonismayer.net/stan,,24.223.19.152,"{""location"": ""ES"", ""is_mobile"": false}" 20949,7,471,2017-01-25 22:39:41,http://rempel.com/audra.jaskolski,,4.34.144.221,"{""location"": ""FR"", ""is_mobile"": true}" 20950,7,471,2017-04-25 03:41:33,http://blanda.biz/ashley.moore,,223.50.27.162,"{""location"": ""SV"", ""is_mobile"": false}" 20951,7,471,2017-05-12 17:08:20,http://paucek.io/wilson,,62.127.231.45,"{""location"": ""TF"", ""is_mobile"": false}" 20952,7,472,2017-02-14 17:30:03,http://daniel.net/domenica,,152.60.42.238,"{""location"": ""VE"", ""is_mobile"": false}" 20953,7,472,2016-12-29 18:50:12,http://ebertdavis.org/lyla,,244.230.14.174,"{""location"": ""MR"", ""is_mobile"": false}" 20954,7,472,2017-04-13 14:37:54,http://hickle.info/leone,,28.118.221.96,"{""location"": ""TV"", ""is_mobile"": true}" 20955,7,472,2017-04-18 19:49:38,http://jerde.io/tanner,,225.5.43.22,"{""location"": ""KH"", ""is_mobile"": true}" 20956,7,472,2017-04-02 19:38:03,http://hammes.org/kiara,,171.9.25.253,"{""location"": ""NP"", ""is_mobile"": false}" 20957,7,472,2016-12-15 18:58:46,http://gerholdkris.org/taylor_wintheiser,,151.144.13.217,"{""location"": ""AM"", ""is_mobile"": true}" 20958,7,472,2017-01-17 23:52:25,http://larkinbailey.net/anabelle,,39.177.207.7,"{""location"": ""BN"", ""is_mobile"": true}" 20959,7,472,2017-06-02 01:41:54,http://schmidt.biz/joan.upton,,139.198.163.90,"{""location"": ""EG"", ""is_mobile"": true}" 20960,7,472,2016-12-26 15:07:19,http://batzbergstrom.org/maye_simonis,,15.135.213.165,"{""location"": ""NA"", ""is_mobile"": false}" 20961,7,472,2017-03-16 07:05:57,http://schuster.io/darrin.murray,,245.190.8.81,"{""location"": ""SI"", ""is_mobile"": true}" 20962,7,472,2017-03-04 16:21:16,http://gislason.org/kelley,,10.215.211.203,"{""location"": ""GG"", ""is_mobile"": false}" 20963,7,472,2017-01-28 15:10:17,http://corwin.name/josie.stracke,,210.96.214.30,"{""location"": ""SA"", ""is_mobile"": false}" 20964,7,472,2017-02-11 15:01:24,http://sawayn.name/ransom_medhurst,,179.208.122.55,"{""location"": ""ZA"", ""is_mobile"": false}" 20965,7,472,2017-02-26 15:37:14,http://johns.org/genoveva.ward,,214.156.192.223,"{""location"": ""MR"", ""is_mobile"": true}" 20966,7,472,2017-04-15 07:16:21,http://lockmanhermann.net/blanca_ko,,143.231.137.201,"{""location"": ""DZ"", ""is_mobile"": true}" 20967,7,472,2017-01-03 13:38:12,http://jerde.org/gail_trantow,,181.59.190.173,"{""location"": ""TD"", ""is_mobile"": false}" 20968,7,472,2017-02-17 20:19:43,http://gorczany.info/anastacio.bins,,13.114.172.103,"{""location"": ""PW"", ""is_mobile"": false}" 20969,7,472,2017-03-26 12:56:24,http://bartoletti.io/marcel,,54.24.61.120,"{""location"": ""HR"", ""is_mobile"": true}" 20970,7,472,2017-05-04 19:24:58,http://howell.co/viola.macejkovic,,212.6.171.161,"{""location"": ""GL"", ""is_mobile"": true}" 20971,7,472,2017-01-07 09:12:52,http://miller.name/mervin.cartwright,,81.37.251.142,"{""location"": ""GF"", ""is_mobile"": false}" 20972,7,472,2017-04-03 07:36:10,http://dietrich.name/lance,,207.222.9.114,"{""location"": ""GB"", ""is_mobile"": false}" 20973,7,472,2017-03-09 22:01:03,http://howewalter.com/fae.hintz,,159.99.103.146,"{""location"": ""BL"", ""is_mobile"": true}" 20974,7,472,2017-01-02 03:54:42,http://wolff.co/jacquelyn.jast,,70.8.75.124,"{""location"": ""PH"", ""is_mobile"": true}" 20975,7,472,2017-02-11 21:34:30,http://gibson.io/judson,,63.118.43.105,"{""location"": ""TJ"", ""is_mobile"": false}" 20976,7,472,2017-04-17 12:20:07,http://will.org/walton.tillman,,202.202.186.155,"{""location"": ""IM"", ""is_mobile"": true}" 20977,7,472,2017-03-25 00:13:14,http://murphylindgren.net/antwon.gottlieb,,85.153.148.228,"{""location"": ""BD"", ""is_mobile"": false}" 20978,7,472,2017-04-12 08:53:44,http://hahn.com/jo.grady,,138.168.147.59,"{""location"": ""GM"", ""is_mobile"": true}" 20979,7,472,2017-03-29 01:17:36,http://balistreri.co/vivienne,,110.213.146.44,"{""location"": ""LA"", ""is_mobile"": true}" 20980,7,472,2017-04-01 12:06:05,http://padbergblanda.co/laverna,,108.171.198.184,"{""location"": ""VN"", ""is_mobile"": false}" 20981,7,472,2017-05-29 02:50:50,http://doylebechtelar.com/melany.kihn,,46.186.187.169,"{""location"": ""MR"", ""is_mobile"": false}" 20982,7,472,2017-03-26 05:06:10,http://pollich.name/eulah_wilderman,,6.197.56.140,"{""location"": ""BT"", ""is_mobile"": true}" 20983,7,472,2016-12-25 08:15:31,http://gleichner.org/saige,,184.77.172.50,"{""location"": ""GR"", ""is_mobile"": true}" 20984,7,472,2017-05-10 13:40:20,http://blickfarrell.name/stewart,,235.196.94.164,"{""location"": ""CW"", ""is_mobile"": true}" 20985,7,472,2017-03-24 06:21:43,http://gleason.net/adam,,118.105.212.28,"{""location"": ""PM"", ""is_mobile"": false}" 20986,7,472,2017-04-11 02:46:53,http://thompson.name/shaylee,,229.73.190.119,"{""location"": ""EC"", ""is_mobile"": false}" 20987,7,472,2017-03-14 07:56:03,http://nikolaus.io/isabelle_senger,,122.73.122.156,"{""location"": ""NF"", ""is_mobile"": false}" 20988,7,472,2017-04-12 05:57:15,http://schulistbode.name/nora_gaylord,,207.83.215.249,"{""location"": ""ZA"", ""is_mobile"": false}" 20989,7,472,2017-01-03 07:34:53,http://schaefernicolas.net/loyce,,185.116.21.100,"{""location"": ""NU"", ""is_mobile"": false}" 20990,7,472,2017-05-21 14:28:58,http://leannon.org/elvie.schoen,,89.34.204.152,"{""location"": ""KN"", ""is_mobile"": true}" 20991,7,472,2017-05-30 16:02:12,http://dibbert.com/trey,,46.199.90.123,"{""location"": ""MK"", ""is_mobile"": false}" 20992,7,472,2017-01-28 02:44:04,http://block.co/neha,,59.184.247.164,"{""location"": ""CY"", ""is_mobile"": true}" 20993,7,472,2016-12-18 20:21:15,http://schmeler.net/abner.carroll,,227.117.98.166,"{""location"": ""AL"", ""is_mobile"": true}" 20994,7,472,2017-05-22 19:52:52,http://macgyver.com/darby_oconnell,,149.55.233.193,"{""location"": ""TD"", ""is_mobile"": false}" 20995,7,472,2017-03-12 19:53:01,http://veumprohaska.biz/sabryna.abernathy,,210.229.110.26,"{""location"": ""GQ"", ""is_mobile"": false}" 20996,7,473,2017-06-07 02:49:34,http://gusikowski.info/maud.auer,,43.40.201.248,"{""location"": ""CZ"", ""is_mobile"": true}" 20997,7,473,2016-12-24 00:12:11,http://macejkovicnienow.org/stephen.fadel,,154.153.219.22,"{""location"": ""RW"", ""is_mobile"": false}" 20998,7,473,2017-01-12 20:13:18,http://hermann.biz/rachel,,156.105.194.187,"{""location"": ""CM"", ""is_mobile"": true}" 20999,7,473,2017-04-10 20:54:20,http://lueilwitzharvey.co/jamel.sanford,,251.211.14.151,"{""location"": ""MG"", ""is_mobile"": true}" 21000,7,473,2016-12-14 17:21:39,http://crooks.org/lisa,,187.76.82.93,"{""location"": ""NA"", ""is_mobile"": true}" 21001,7,473,2017-01-13 06:09:22,http://nicolas.biz/imelda.rice,,219.241.227.57,"{""location"": ""EG"", ""is_mobile"": true}" 21002,7,473,2017-04-11 12:05:09,http://oconnell.net/bailee_howe,,89.25.126.98,"{""location"": ""TN"", ""is_mobile"": false}" 21003,7,473,2017-05-27 04:58:46,http://lockman.biz/allen,,92.187.134.47,"{""location"": ""LC"", ""is_mobile"": false}" 21004,7,473,2016-12-18 06:48:52,http://grimes.com/kade,,116.156.104.42,"{""location"": ""PF"", ""is_mobile"": true}" 21005,7,473,2017-03-01 10:05:46,http://ledner.info/camilla,,226.142.13.248,"{""location"": ""HM"", ""is_mobile"": true}" 21006,7,473,2017-01-02 14:36:04,http://criststroman.net/enrique.funk,,148.164.200.48,"{""location"": ""AQ"", ""is_mobile"": true}" 21007,7,473,2017-01-26 21:42:47,http://reichel.io/myra,,252.91.140.216,"{""location"": ""VG"", ""is_mobile"": true}" 21008,7,473,2016-12-30 08:26:04,http://jacobsgislason.com/graciela_rempel,,254.148.9.42,"{""location"": ""JM"", ""is_mobile"": true}" 21009,7,473,2017-01-08 06:38:28,http://stehr.name/favian,,137.151.68.82,"{""location"": ""SN"", ""is_mobile"": false}" 21010,7,473,2017-04-20 19:33:48,http://denesikrunolfon.net/leilani_balistreri,,213.246.144.134,"{""location"": ""LR"", ""is_mobile"": false}" 21011,7,473,2016-12-18 23:45:16,http://dibbert.com/gilbert,,38.136.185.67,"{""location"": ""AD"", ""is_mobile"": false}" 21012,7,473,2017-02-27 02:14:24,http://dietrichberge.net/fiona,,55.34.238.71,"{""location"": ""AD"", ""is_mobile"": false}" 21013,7,473,2017-05-17 11:39:23,http://adams.co/bernard_feest,,34.92.172.92,"{""location"": ""FR"", ""is_mobile"": false}" 21014,7,473,2017-02-19 10:10:22,http://schambergerdurgan.net/zakary_franecki,,99.131.147.82,"{""location"": ""ME"", ""is_mobile"": false}" 21015,7,473,2017-04-28 01:18:28,http://ortiz.info/thora,,15.100.55.227,"{""location"": ""GW"", ""is_mobile"": false}" 21016,7,473,2017-03-26 02:16:36,http://robel.info/kadin.terry,,33.224.186.220,"{""location"": ""BM"", ""is_mobile"": false}" 21017,7,473,2017-06-07 18:16:09,http://cole.name/merl,,57.244.147.4,"{""location"": ""KM"", ""is_mobile"": false}" 21018,7,473,2017-02-22 21:24:16,http://gislason.io/quinton,,241.170.116.158,"{""location"": ""MD"", ""is_mobile"": true}" 21019,7,473,2017-04-25 17:34:14,http://ratke.info/misael,,11.173.234.86,"{""location"": ""CZ"", ""is_mobile"": false}" 21020,7,473,2016-12-23 23:38:15,http://ko.net/lexie_brekke,,30.25.227.180,"{""location"": ""BA"", ""is_mobile"": false}" 21021,7,473,2017-03-06 21:59:16,http://howe.name/selena.towne,,214.162.219.210,"{""location"": ""MM"", ""is_mobile"": false}" 21022,7,473,2017-01-28 03:18:20,http://gulgowski.biz/mikel,,248.219.195.218,"{""location"": ""IO"", ""is_mobile"": true}" 21023,7,473,2017-04-15 08:10:18,http://lynch.biz/leif,,29.126.254.21,"{""location"": ""SG"", ""is_mobile"": true}" 21024,7,473,2017-01-08 06:40:11,http://wuckert.biz/karley.veum,,65.68.104.161,"{""location"": ""CF"", ""is_mobile"": true}" 21025,7,473,2017-06-04 13:16:27,http://wolftoy.net/lydia,,88.240.104.13,"{""location"": ""GM"", ""is_mobile"": false}" 21026,7,473,2017-04-21 21:36:54,http://wolf.org/joseph_keeling,,43.201.109.124,"{""location"": ""SZ"", ""is_mobile"": false}" 21027,7,473,2016-12-30 21:33:29,http://bayer.com/santa,,216.5.108.239,"{""location"": ""VA"", ""is_mobile"": true}" 21028,7,473,2017-04-24 11:56:11,http://toy.co/marisa_terry,,187.46.24.83,"{""location"": ""EG"", ""is_mobile"": false}" 21029,7,473,2017-01-28 17:15:51,http://greenholt.net/arne,,24.134.234.121,"{""location"": ""AS"", ""is_mobile"": false}" 21030,7,473,2017-02-16 04:41:51,http://brakus.org/loy,,184.73.50.157,"{""location"": ""TM"", ""is_mobile"": true}" 21031,7,473,2017-01-17 05:09:44,http://schneider.net/tamara,,163.43.135.208,"{""location"": ""AX"", ""is_mobile"": true}" 21032,7,473,2017-03-14 20:05:29,http://smithwelch.net/jennifer.bins,,164.115.40.56,"{""location"": ""UG"", ""is_mobile"": false}" 21033,7,473,2017-01-16 09:18:01,http://emardsmitham.info/ladarius,,173.46.214.118,"{""location"": ""PG"", ""is_mobile"": true}" 21034,7,473,2017-06-07 11:21:23,http://frami.biz/solon,,213.150.5.254,"{""location"": ""TV"", ""is_mobile"": false}" 21035,7,473,2017-03-01 13:39:16,http://goldner.co/susie.donnelly,,23.249.237.224,"{""location"": ""IS"", ""is_mobile"": false}" 21036,7,473,2017-02-24 20:57:07,http://cronin.org/aric,,129.252.140.176,"{""location"": ""ME"", ""is_mobile"": true}" 21037,7,473,2016-12-31 00:30:57,http://lehner.info/reba_padberg,,246.18.54.175,"{""location"": ""CH"", ""is_mobile"": false}" 21038,7,473,2017-04-06 20:21:38,http://lang.org/mylene,,29.214.147.93,"{""location"": ""SZ"", ""is_mobile"": false}" 21039,7,473,2017-02-09 03:44:06,http://predovicgerhold.com/gerry,,207.15.30.212,"{""location"": ""NF"", ""is_mobile"": true}" 21040,7,473,2017-01-03 19:58:51,http://crona.biz/dorothea_schimmel,,46.164.229.23,"{""location"": ""AO"", ""is_mobile"": false}" 21041,7,473,2017-01-25 21:04:30,http://pfeffer.name/evie,,108.17.78.119,"{""location"": ""CM"", ""is_mobile"": false}" 21042,7,473,2017-05-20 21:32:03,http://klockoabbott.io/lilliana,,164.22.252.99,"{""location"": ""GM"", ""is_mobile"": true}" 21043,7,473,2017-02-17 23:19:43,http://denesik.biz/devan,,71.160.251.120,"{""location"": ""CX"", ""is_mobile"": true}" 21044,7,473,2017-01-30 23:28:55,http://abbott.biz/ian,,179.134.245.188,"{""location"": ""NE"", ""is_mobile"": true}" 21045,7,473,2017-02-28 00:22:43,http://grady.co/jayden.mayer,,153.105.253.144,"{""location"": ""SX"", ""is_mobile"": true}" 21046,7,473,2017-02-19 00:25:08,http://vonrueden.com/dorris,,100.132.154.151,"{""location"": ""PF"", ""is_mobile"": false}" 21047,7,473,2017-05-12 10:19:58,http://bins.co/ardella.strosin,,77.226.215.67,"{""location"": ""CC"", ""is_mobile"": false}" ================================================ FILE: src/test/regress/data/large_records.data ================================================ 1|aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 2|a ================================================ FILE: src/test/regress/data/lineitem.1.data ================================================ 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular courts above the 1|67310|7311|2|36|45983.16|0.09|0.06|N|O|1996-04-12|1996-02-28|1996-04-20|TAKE BACK RETURN|MAIL|ly final dependencies: slyly bold 1|63700|3701|3|8|13309.60|0.10|0.02|N|O|1996-01-29|1996-03-05|1996-01-31|TAKE BACK RETURN|REG AIR|riously. regular, express dep 1|2132|4633|4|28|28955.64|0.09|0.06|N|O|1996-04-21|1996-03-30|1996-05-16|NONE|AIR|lites. fluffily even de 1|24027|1534|5|24|22824.48|0.10|0.04|N|O|1996-03-30|1996-03-14|1996-04-01|NONE|FOB| pending foxes. slyly re 1|15635|638|6|32|49620.16|0.07|0.02|N|O|1996-01-30|1996-02-07|1996-02-03|DELIVER IN PERSON|MAIL|arefully slyly ex 2|106170|1191|1|38|44694.46|0.00|0.05|N|O|1997-01-28|1997-01-14|1997-02-02|TAKE BACK RETURN|RAIL|ven requests. deposits breach a 3|4297|1798|1|45|54058.05|0.06|0.00|R|F|1994-02-02|1994-01-04|1994-02-23|NONE|AIR|ongside of the furiously brave acco 3|19036|6540|2|49|46796.47|0.10|0.00|R|F|1993-11-09|1993-12-20|1993-11-24|TAKE BACK RETURN|RAIL| unusual accounts. eve 3|128449|3474|3|27|39890.88|0.06|0.07|A|F|1994-01-16|1993-11-22|1994-01-23|DELIVER IN PERSON|SHIP|nal foxes wake. 3|29380|1883|4|2|2618.76|0.01|0.06|A|F|1993-12-04|1994-01-07|1994-01-01|NONE|TRUCK|y. fluffily pending d 3|183095|650|5|28|32986.52|0.04|0.00|R|F|1993-12-14|1994-01-10|1994-01-01|TAKE BACK RETURN|FOB|ages nag slyly pending 3|62143|9662|6|26|28733.64|0.10|0.02|A|F|1993-10-29|1993-12-18|1993-11-04|TAKE BACK RETURN|RAIL|ges sleep after the caref 4|88035|5560|1|30|30690.90|0.03|0.08|N|O|1996-01-10|1995-12-14|1996-01-18|DELIVER IN PERSON|REG AIR|- quickly regular packages sleep. idly 5|108570|8571|1|15|23678.55|0.02|0.04|R|F|1994-10-31|1994-08-31|1994-11-20|NONE|AIR|ts wake furiously 5|123927|3928|2|26|50723.92|0.07|0.08|R|F|1994-10-16|1994-09-25|1994-10-19|NONE|FOB|sts use slyly quickly special instruc 5|37531|35|3|50|73426.50|0.08|0.03|A|F|1994-08-08|1994-10-13|1994-08-26|DELIVER IN PERSON|AIR|eodolites. fluffily unusual 6|139636|2150|1|37|61998.31|0.08|0.03|A|F|1992-04-27|1992-05-15|1992-05-02|TAKE BACK RETURN|TRUCK|p furiously special foxes 7|182052|9607|1|12|13608.60|0.07|0.03|N|O|1996-05-07|1996-03-13|1996-06-03|TAKE BACK RETURN|FOB|ss pinto beans wake against th 7|145243|7758|2|9|11594.16|0.08|0.08|N|O|1996-02-01|1996-03-02|1996-02-19|TAKE BACK RETURN|SHIP|es. instructions 7|94780|9799|3|46|81639.88|0.10|0.07|N|O|1996-01-15|1996-03-27|1996-02-03|COLLECT COD|MAIL| unusual reques 7|163073|3074|4|28|31809.96|0.03|0.04|N|O|1996-03-21|1996-04-08|1996-04-20|NONE|FOB|. slyly special requests haggl 7|151894|9440|5|38|73943.82|0.08|0.01|N|O|1996-02-11|1996-02-24|1996-02-18|DELIVER IN PERSON|TRUCK|ns haggle carefully ironic deposits. bl 7|79251|1759|6|35|43058.75|0.06|0.03|N|O|1996-01-16|1996-02-23|1996-01-22|TAKE BACK RETURN|FOB|jole. excuses wake carefully alongside of 7|157238|2269|7|5|6476.15|0.04|0.02|N|O|1996-02-10|1996-03-26|1996-02-13|NONE|FOB|ithely regula 32|82704|7721|1|28|47227.60|0.05|0.08|N|O|1995-10-23|1995-08-27|1995-10-26|TAKE BACK RETURN|TRUCK|sleep quickly. req 32|197921|441|2|32|64605.44|0.02|0.00|N|O|1995-08-14|1995-10-07|1995-08-27|COLLECT COD|AIR|lithely regular deposits. fluffily 32|44161|6666|3|2|2210.32|0.09|0.02|N|O|1995-08-07|1995-10-07|1995-08-23|DELIVER IN PERSON|AIR| express accounts wake according to the 32|2743|7744|4|4|6582.96|0.09|0.03|N|O|1995-08-04|1995-10-01|1995-09-03|NONE|REG AIR|e slyly final pac 32|85811|8320|5|44|79059.64|0.05|0.06|N|O|1995-08-28|1995-08-20|1995-09-14|DELIVER IN PERSON|AIR|symptotes nag according to the ironic depo 32|11615|4117|6|6|9159.66|0.04|0.03|N|O|1995-07-21|1995-09-23|1995-07-25|COLLECT COD|RAIL| gifts cajole carefully. 33|61336|8855|1|31|40217.23|0.09|0.04|A|F|1993-10-29|1993-12-19|1993-11-08|COLLECT COD|TRUCK|ng to the furiously ironic package 33|60519|5532|2|32|47344.32|0.02|0.05|A|F|1993-12-09|1994-01-04|1993-12-28|COLLECT COD|MAIL|gular theodolites 33|137469|9983|3|5|7532.30|0.05|0.03|A|F|1993-12-09|1993-12-25|1993-12-23|TAKE BACK RETURN|AIR|. stealthily bold exc 33|33918|3919|4|41|75928.31|0.09|0.00|R|F|1993-11-09|1994-01-24|1993-11-11|TAKE BACK RETURN|MAIL|unusual packages doubt caref 34|88362|871|1|13|17554.68|0.00|0.07|N|O|1998-10-23|1998-09-14|1998-11-06|NONE|REG AIR|nic accounts. deposits are alon 34|89414|1923|2|22|30875.02|0.08|0.06|N|O|1998-10-09|1998-10-16|1998-10-12|NONE|FOB|thely slyly p 34|169544|4577|3|6|9681.24|0.02|0.06|N|O|1998-10-30|1998-09-20|1998-11-05|NONE|FOB|ar foxes sleep 35|450|2951|1|24|32410.80|0.02|0.00|N|O|1996-02-21|1996-01-03|1996-03-18|TAKE BACK RETURN|FOB|, regular tithe 35|161940|4457|2|34|68065.96|0.06|0.08|N|O|1996-01-22|1996-01-06|1996-01-27|DELIVER IN PERSON|RAIL|s are carefully against the f 35|120896|8433|3|7|13418.23|0.06|0.04|N|O|1996-01-19|1995-12-22|1996-01-29|NONE|MAIL| the carefully regular 35|85175|7684|4|25|29004.25|0.06|0.05|N|O|1995-11-26|1995-12-25|1995-12-21|DELIVER IN PERSON|SHIP| quickly unti 35|119917|4940|5|34|65854.94|0.08|0.06|N|O|1995-11-08|1996-01-15|1995-11-26|COLLECT COD|MAIL|. silent, unusual deposits boost 35|30762|3266|6|28|47397.28|0.03|0.02|N|O|1996-02-01|1995-12-24|1996-02-28|COLLECT COD|RAIL|ly alongside of 36|119767|9768|1|42|75043.92|0.09|0.00|N|O|1996-02-03|1996-01-21|1996-02-23|COLLECT COD|SHIP| careful courts. special 37|22630|5133|1|40|62105.20|0.09|0.03|A|F|1992-07-21|1992-08-01|1992-08-15|NONE|REG AIR|luffily regular requests. slyly final acco 37|126782|1807|2|39|70542.42|0.05|0.02|A|F|1992-07-02|1992-08-18|1992-07-28|TAKE BACK RETURN|RAIL|the final requests. ca 37|12903|5405|3|43|78083.70|0.05|0.08|A|F|1992-07-10|1992-07-06|1992-08-02|DELIVER IN PERSON|TRUCK|iously ste 38|175839|874|1|44|84252.52|0.04|0.02|N|O|1996-09-29|1996-11-17|1996-09-30|COLLECT COD|MAIL|s. blithely unusual theodolites am 39|2320|9821|1|44|53782.08|0.09|0.06|N|O|1996-11-14|1996-12-15|1996-12-12|COLLECT COD|RAIL|eodolites. careful 39|186582|4137|2|26|43383.08|0.08|0.04|N|O|1996-11-04|1996-10-20|1996-11-20|NONE|FOB|ckages across the slyly silent 39|67831|5350|3|46|82746.18|0.06|0.08|N|O|1996-09-26|1996-12-19|1996-10-26|DELIVER IN PERSON|AIR|he carefully e 39|20590|3093|4|32|48338.88|0.07|0.05|N|O|1996-10-02|1996-12-19|1996-10-14|COLLECT COD|MAIL|heodolites sleep silently pending foxes. ac 39|54519|9530|5|43|63360.93|0.01|0.01|N|O|1996-10-17|1996-11-14|1996-10-26|COLLECT COD|MAIL|yly regular i 39|94368|6878|6|40|54494.40|0.06|0.05|N|O|1996-12-08|1996-10-22|1997-01-01|COLLECT COD|AIR|quickly ironic fox 64|85951|5952|1|21|40675.95|0.05|0.02|R|F|1994-09-30|1994-09-18|1994-10-26|DELIVER IN PERSON|REG AIR|ch slyly final, thin platelets. 65|59694|4705|1|26|42995.94|0.03|0.03|A|F|1995-04-20|1995-04-25|1995-05-13|NONE|TRUCK|pending deposits nag even packages. ca 65|73815|8830|2|22|39353.82|0.00|0.05|N|O|1995-07-17|1995-06-04|1995-07-19|COLLECT COD|FOB| ideas. special, r 65|1388|3889|3|21|27076.98|0.09|0.07|N|O|1995-07-06|1995-05-14|1995-07-31|DELIVER IN PERSON|RAIL|bove the even packages. accounts nag carefu 66|115118|7630|1|31|35126.41|0.00|0.08|R|F|1994-02-19|1994-03-11|1994-02-20|TAKE BACK RETURN|RAIL|ut the unusual accounts sleep at the bo 66|173489|3490|2|41|64061.68|0.04|0.07|A|F|1994-02-21|1994-03-01|1994-03-18|COLLECT COD|AIR| regular de 67|21636|9143|1|4|6230.52|0.09|0.04|N|O|1997-04-17|1997-01-31|1997-04-20|NONE|SHIP| cajole thinly expres 67|20193|5198|2|12|13358.28|0.09|0.05|N|O|1997-01-27|1997-02-21|1997-02-22|NONE|REG AIR| even packages cajole 67|173600|6118|3|5|8368.00|0.03|0.07|N|O|1997-02-20|1997-02-12|1997-02-21|DELIVER IN PERSON|TRUCK|y unusual packages thrash pinto 67|87514|7515|4|44|66066.44|0.08|0.06|N|O|1997-03-18|1997-01-29|1997-04-13|DELIVER IN PERSON|RAIL|se quickly above the even, express reques 67|40613|8126|5|23|35733.03|0.05|0.07|N|O|1997-04-19|1997-02-14|1997-05-06|DELIVER IN PERSON|REG AIR|ly regular deposit 67|178306|824|6|29|40144.70|0.02|0.05|N|O|1997-01-25|1997-01-27|1997-01-27|DELIVER IN PERSON|FOB|ultipliers 68|7068|9569|1|3|2925.18|0.05|0.02|N|O|1998-07-04|1998-06-05|1998-07-21|NONE|RAIL|fully special instructions cajole. furious 68|175180|2732|2|46|57738.28|0.02|0.05|N|O|1998-06-26|1998-06-07|1998-07-05|NONE|MAIL| requests are unusual, regular pinto 68|34980|7484|3|46|88089.08|0.04|0.05|N|O|1998-08-13|1998-07-08|1998-08-29|NONE|RAIL|egular dependencies affix ironically along 68|94728|2256|4|20|34454.40|0.07|0.01|N|O|1998-06-27|1998-05-23|1998-07-02|NONE|REG AIR| excuses integrate fluffily 68|82758|5267|5|27|47000.25|0.03|0.06|N|O|1998-06-19|1998-06-25|1998-06-29|DELIVER IN PERSON|SHIP|ccounts. deposits use. furiously 68|102561|5072|6|30|46906.80|0.05|0.06|N|O|1998-08-11|1998-07-11|1998-08-14|NONE|RAIL|oxes are slyly blithely fin 68|139247|1761|7|41|52735.84|0.09|0.08|N|O|1998-06-24|1998-06-27|1998-07-06|NONE|SHIP|eposits nag special ideas. furiousl 69|115209|7721|1|48|58761.60|0.01|0.07|A|F|1994-08-17|1994-08-11|1994-09-08|NONE|TRUCK|regular epitaphs. carefully even ideas hag 69|104180|9201|2|32|37893.76|0.08|0.06|A|F|1994-08-24|1994-08-17|1994-08-31|NONE|REG AIR|s sleep carefully bold, 69|137267|4807|3|17|22172.42|0.09|0.00|A|F|1994-07-02|1994-07-07|1994-07-03|TAKE BACK RETURN|AIR|final, pending instr 69|37502|2509|4|3|4318.50|0.09|0.04|R|F|1994-06-06|1994-07-27|1994-06-15|NONE|MAIL| blithely final d 69|92070|7089|5|42|44606.94|0.07|0.04|R|F|1994-07-31|1994-07-26|1994-08-28|DELIVER IN PERSON|REG AIR|tect regular, speci 69|18504|1006|6|23|32717.50|0.05|0.00|A|F|1994-10-03|1994-08-06|1994-10-24|NONE|SHIP|nding accounts ca 70|64128|9141|1|8|8736.96|0.03|0.08|R|F|1994-01-12|1994-02-27|1994-01-14|TAKE BACK RETURN|FOB|ggle. carefully pending dependenc 70|196156|1195|2|13|16277.95|0.06|0.06|A|F|1994-03-03|1994-02-13|1994-03-26|COLLECT COD|AIR|lyly special packag 70|179809|7361|3|1|1888.80|0.03|0.05|R|F|1994-01-26|1994-03-05|1994-01-28|TAKE BACK RETURN|RAIL|quickly. fluffily unusual theodolites c 70|45734|743|4|11|18477.03|0.01|0.05|A|F|1994-03-17|1994-03-17|1994-03-27|NONE|MAIL|alongside of the deposits. fur 70|37131|2138|5|37|39520.81|0.09|0.04|R|F|1994-02-13|1994-03-16|1994-02-21|COLLECT COD|MAIL|n accounts are. q 70|55655|3171|6|19|30602.35|0.06|0.03|A|F|1994-01-26|1994-02-17|1994-02-06|TAKE BACK RETURN|SHIP| packages wake pending accounts. 71|61931|1932|1|25|47323.25|0.09|0.07|N|O|1998-04-10|1998-04-22|1998-04-11|COLLECT COD|FOB|ckly. slyly 71|65916|3435|2|3|5645.73|0.09|0.07|N|O|1998-05-23|1998-04-03|1998-06-02|COLLECT COD|SHIP|y. pinto beans haggle after the 71|34432|1942|3|45|61489.35|0.00|0.07|N|O|1998-02-23|1998-03-20|1998-03-24|DELIVER IN PERSON|SHIP| ironic packages believe blithely a 71|96645|9155|4|33|54174.12|0.00|0.01|N|O|1998-04-12|1998-03-20|1998-04-15|NONE|FOB| serve quickly fluffily bold deposi 71|103255|5766|5|39|49071.75|0.08|0.06|N|O|1998-01-29|1998-04-07|1998-02-18|DELIVER IN PERSON|RAIL|l accounts sleep across the pack 71|195635|674|6|34|58841.42|0.04|0.01|N|O|1998-03-05|1998-04-22|1998-03-30|DELIVER IN PERSON|TRUCK|s cajole. 96|123076|613|1|23|25278.61|0.10|0.06|A|F|1994-07-19|1994-06-29|1994-07-25|DELIVER IN PERSON|TRUCK|ep-- carefully reg 96|135390|5391|2|30|42761.70|0.01|0.06|R|F|1994-06-03|1994-05-29|1994-06-22|DELIVER IN PERSON|TRUCK|e quickly even ideas. furiou 97|119477|1989|1|13|19454.11|0.00|0.02|R|F|1993-04-01|1993-04-04|1993-04-08|NONE|TRUCK|ayers cajole against the furiously 97|49568|2073|2|37|56149.72|0.02|0.06|A|F|1993-04-13|1993-03-30|1993-04-14|DELIVER IN PERSON|SHIP|ic requests boost carefully quic 97|77699|5221|3|19|31857.11|0.06|0.08|R|F|1993-05-14|1993-03-05|1993-05-25|TAKE BACK RETURN|RAIL|gifts. furiously ironic packages cajole. 98|40216|217|1|28|32373.88|0.06|0.07|A|F|1994-12-24|1994-10-25|1995-01-16|COLLECT COD|REG AIR| pending, regular accounts s 98|109743|7274|2|1|1752.74|0.00|0.00|A|F|1994-12-01|1994-12-12|1994-12-15|DELIVER IN PERSON|TRUCK|. unusual instructions against 98|44706|4707|3|14|23109.80|0.05|0.02|A|F|1994-12-30|1994-11-22|1995-01-27|COLLECT COD|AIR| cajole furiously. blithely ironic ideas 98|167180|7181|4|10|12471.80|0.03|0.03|A|F|1994-10-23|1994-11-08|1994-11-09|COLLECT COD|RAIL| carefully. quickly ironic ideas 99|87114|4639|1|10|11011.10|0.02|0.01|A|F|1994-05-18|1994-06-03|1994-05-23|COLLECT COD|RAIL|kages. requ 99|123766|3767|2|5|8948.80|0.02|0.07|R|F|1994-05-06|1994-05-28|1994-05-20|TAKE BACK RETURN|RAIL|ests cajole fluffily waters. blithe 99|134082|1622|3|42|46875.36|0.02|0.02|A|F|1994-04-19|1994-05-18|1994-04-20|NONE|RAIL|kages are fluffily furiously ir 99|108338|849|4|36|48467.88|0.09|0.02|A|F|1994-07-04|1994-04-17|1994-07-30|DELIVER IN PERSON|AIR|slyly. slyly e 100|62029|2030|1|28|27748.56|0.04|0.05|N|O|1998-05-08|1998-05-13|1998-06-07|COLLECT COD|TRUCK|sts haggle. slowl 100|115979|8491|2|22|43889.34|0.00|0.07|N|O|1998-06-24|1998-04-12|1998-06-29|DELIVER IN PERSON|SHIP|nto beans alongside of the fi 100|46150|8655|3|46|50422.90|0.03|0.04|N|O|1998-05-02|1998-04-10|1998-05-22|TAKE BACK RETURN|SHIP|ular accounts. even 100|38024|3031|4|14|13468.28|0.06|0.03|N|O|1998-05-22|1998-05-01|1998-06-03|COLLECT COD|MAIL|y. furiously ironic ideas gr 100|53439|955|5|37|51519.91|0.05|0.00|N|O|1998-03-06|1998-04-16|1998-03-31|TAKE BACK RETURN|TRUCK|nd the quickly s 101|118282|5816|1|49|63713.72|0.10|0.00|N|O|1996-06-21|1996-05-27|1996-06-29|DELIVER IN PERSON|REG AIR|ts-- final packages sleep furiousl 101|163334|883|2|36|50303.88|0.00|0.01|N|O|1996-05-19|1996-05-01|1996-06-04|DELIVER IN PERSON|AIR|tes. blithely pending dolphins x-ray f 101|138418|5958|3|12|17476.92|0.06|0.02|N|O|1996-03-29|1996-04-20|1996-04-12|COLLECT COD|MAIL|. quickly regular 102|88914|3931|1|37|70407.67|0.06|0.00|N|O|1997-07-24|1997-08-02|1997-08-07|TAKE BACK RETURN|SHIP|ully across the ideas. final deposit 102|169238|6787|2|34|44445.82|0.03|0.08|N|O|1997-08-09|1997-07-28|1997-08-26|TAKE BACK RETURN|SHIP|eposits cajole across 102|182321|4840|3|25|35083.00|0.01|0.01|N|O|1997-07-31|1997-07-24|1997-08-17|NONE|RAIL|bits. ironic accoun 102|61158|8677|4|15|16787.25|0.07|0.07|N|O|1997-06-02|1997-07-13|1997-06-04|DELIVER IN PERSON|SHIP|final packages. carefully even excu 103|194658|2216|1|6|10515.90|0.03|0.05|N|O|1996-10-11|1996-07-25|1996-10-28|NONE|FOB|cajole. carefully ex 103|10426|2928|2|37|49447.54|0.02|0.07|N|O|1996-09-17|1996-07-27|1996-09-20|TAKE BACK RETURN|MAIL|ies. quickly ironic requests use blithely 103|28431|8432|3|23|31266.89|0.01|0.04|N|O|1996-09-11|1996-09-18|1996-09-26|NONE|FOB|ironic accou 103|29022|4027|4|32|30432.64|0.01|0.07|N|O|1996-07-30|1996-08-06|1996-08-04|NONE|RAIL|kages doze. special, regular deposit 128|106828|9339|1|38|69723.16|0.06|0.01|A|F|1992-09-01|1992-08-27|1992-10-01|TAKE BACK RETURN|FOB| cajole careful 129|2867|5368|1|46|81413.56|0.08|0.02|R|F|1993-02-15|1993-01-24|1993-03-05|COLLECT COD|TRUCK|uietly bold theodolites. fluffil 129|185164|5165|2|36|44969.76|0.01|0.02|A|F|1992-11-25|1992-12-25|1992-12-09|TAKE BACK RETURN|REG AIR|packages are care 129|39444|1948|3|33|45653.52|0.04|0.06|A|F|1993-01-08|1993-02-14|1993-01-29|COLLECT COD|SHIP|sts nag bravely. fluffily 129|135137|164|4|34|39852.42|0.00|0.01|R|F|1993-01-29|1993-02-14|1993-02-10|COLLECT COD|MAIL|quests. express ideas 129|31373|8883|5|24|31304.88|0.06|0.00|A|F|1992-12-07|1993-01-02|1992-12-11|TAKE BACK RETURN|FOB|uests. foxes cajole slyly after the ca 129|77050|4572|6|22|22595.10|0.06|0.01|R|F|1993-02-15|1993-01-31|1993-02-24|COLLECT COD|SHIP|e. fluffily regular 129|168569|3602|7|1|1637.56|0.05|0.04|R|F|1993-01-26|1993-01-08|1993-02-24|DELIVER IN PERSON|FOB|e carefully blithely bold dolp 130|128816|8817|1|14|25827.34|0.08|0.05|A|F|1992-08-15|1992-07-25|1992-09-13|COLLECT COD|RAIL| requests. final instruction 130|1739|4240|2|48|78755.04|0.03|0.02|R|F|1992-07-01|1992-07-12|1992-07-24|NONE|AIR|lithely alongside of the regu 130|11860|1861|3|18|31893.48|0.04|0.08|A|F|1992-07-04|1992-06-14|1992-07-29|DELIVER IN PERSON|MAIL| slyly ironic decoys abou 130|115635|3169|4|13|21458.19|0.09|0.02|R|F|1992-06-26|1992-07-29|1992-07-05|NONE|FOB| pending dolphins sleep furious 130|69130|4143|5|31|34073.03|0.06|0.05|R|F|1992-09-01|1992-07-18|1992-09-02|TAKE BACK RETURN|RAIL|thily about the ruth 131|167505|22|1|45|70762.50|0.10|0.02|R|F|1994-09-14|1994-09-02|1994-10-04|NONE|FOB|ironic, bold accounts. careful 131|44255|9264|2|50|59962.50|0.02|0.04|A|F|1994-09-17|1994-08-10|1994-09-21|NONE|SHIP|ending requests. final, ironic pearls slee 131|189021|1540|3|4|4440.08|0.04|0.03|A|F|1994-09-20|1994-08-30|1994-09-23|COLLECT COD|REG AIR| are carefully slyly i 132|140449|2964|1|18|26809.92|0.00|0.08|R|F|1993-07-10|1993-08-05|1993-07-13|NONE|TRUCK|ges. platelets wake furio 132|119053|9054|2|43|46098.15|0.01|0.08|R|F|1993-09-01|1993-08-16|1993-09-22|NONE|TRUCK|y pending theodolites 132|114419|4420|3|32|45869.12|0.04|0.04|A|F|1993-07-12|1993-08-05|1993-08-05|COLLECT COD|TRUCK|d instructions hagg 132|28082|5589|4|23|23231.84|0.10|0.00|A|F|1993-06-16|1993-08-27|1993-06-23|DELIVER IN PERSON|AIR|refully blithely bold acco 133|103432|5943|1|27|38756.61|0.00|0.02|N|O|1997-12-21|1998-02-23|1997-12-27|TAKE BACK RETURN|MAIL|yly even gifts after the sl 133|176279|3831|2|12|16263.24|0.02|0.06|N|O|1997-12-02|1998-01-15|1997-12-29|DELIVER IN PERSON|REG AIR|ts cajole fluffily quickly i 133|117350|4884|3|29|39653.15|0.09|0.08|N|O|1998-02-28|1998-01-30|1998-03-09|DELIVER IN PERSON|RAIL| the carefully regular theodoli 133|89855|7380|4|11|20293.35|0.06|0.01|N|O|1998-03-21|1998-01-15|1998-04-04|DELIVER IN PERSON|REG AIR|e quickly across the dolphins 134|641|642|1|21|32374.44|0.00|0.03|A|F|1992-07-17|1992-07-08|1992-07-26|COLLECT COD|SHIP|s. quickly regular 134|164645|9678|2|35|59837.40|0.06|0.07|A|F|1992-08-23|1992-06-01|1992-08-24|NONE|MAIL|ajole furiously. instructio 134|188252|3289|3|26|34846.50|0.09|0.06|A|F|1992-06-20|1992-07-12|1992-07-16|NONE|RAIL| among the pending depos 134|144002|4003|4|47|49162.00|0.05|0.00|A|F|1992-08-16|1992-07-06|1992-08-28|NONE|REG AIR|s! carefully unusual requests boost careful 134|35172|5173|5|12|13286.04|0.05|0.02|A|F|1992-07-03|1992-06-01|1992-07-11|COLLECT COD|TRUCK|nts are quic 134|133103|5617|6|12|13633.20|0.00|0.00|A|F|1992-08-08|1992-07-07|1992-08-20|TAKE BACK RETURN|FOB|lyly regular pac 135|108205|8206|1|47|57020.40|0.06|0.08|N|O|1996-02-18|1996-01-01|1996-02-25|COLLECT COD|RAIL|ctions wake slyly abo 135|198344|5902|2|21|30289.14|0.00|0.07|N|O|1996-02-11|1996-01-12|1996-02-13|DELIVER IN PERSON|SHIP| deposits believe. furiously regular p 135|157510|5056|3|33|51727.83|0.02|0.00|N|O|1996-01-03|1995-11-21|1996-02-01|TAKE BACK RETURN|MAIL|ptotes boost slowly care 135|67005|9512|4|34|33048.00|0.02|0.03|N|O|1996-01-12|1996-01-19|1996-02-05|NONE|TRUCK|counts doze against the blithely ironi 135|136248|1275|5|20|25684.80|0.01|0.04|N|O|1996-01-25|1995-11-20|1996-02-09|NONE|MAIL|theodolites. quickly p 135|115000|2534|6|13|13195.00|0.04|0.02|N|O|1995-11-12|1995-12-22|1995-11-17|NONE|FOB|nal ideas. final instr 160|14785|9788|1|36|61192.08|0.07|0.01|N|O|1997-03-11|1997-03-11|1997-03-20|COLLECT COD|MAIL|old, ironic deposits are quickly abov 160|86382|8891|2|22|30104.36|0.00|0.04|N|O|1997-02-18|1997-03-05|1997-03-05|COLLECT COD|RAIL|ncies about the request 160|20080|5085|3|34|34002.72|0.01|0.05|N|O|1997-01-31|1997-03-13|1997-02-14|NONE|FOB|st sleep even gifts. dependencies along 161|102810|341|1|19|34443.39|0.01|0.01|A|F|1994-12-13|1994-11-19|1994-12-26|DELIVER IN PERSON|TRUCK|, regular sheaves sleep along 162|189288|4325|1|2|2754.56|0.02|0.01|N|O|1995-09-02|1995-06-17|1995-09-08|COLLECT COD|FOB|es! final somas integrate 163|167545|5094|1|43|69339.22|0.01|0.00|N|O|1997-09-19|1997-11-19|1997-10-03|COLLECT COD|REG AIR|al, bold dependencies wake. iron 163|120702|703|2|13|22395.10|0.01|0.04|N|O|1997-11-11|1997-10-18|1997-12-07|DELIVER IN PERSON|TRUCK|inal requests. even pinto beans hag 163|36818|9322|3|27|47379.87|0.04|0.08|N|O|1997-12-26|1997-11-28|1998-01-05|COLLECT COD|REG AIR|ously express dependen 163|192642|5162|4|5|8673.20|0.02|0.00|N|O|1997-11-17|1997-10-09|1997-12-05|DELIVER IN PERSON|TRUCK| must belie 163|126090|8603|5|12|13393.08|0.10|0.00|N|O|1997-12-18|1997-10-26|1997-12-22|COLLECT COD|TRUCK|ly blithe accounts cajole 163|190825|5864|6|20|38316.40|0.00|0.07|N|O|1997-09-27|1997-11-15|1997-10-07|TAKE BACK RETURN|FOB|tructions integrate b 164|91309|3819|1|26|33807.80|0.09|0.04|A|F|1993-01-04|1992-11-21|1993-01-07|NONE|RAIL|s. blithely special courts are blithel 164|18488|3491|2|24|33755.52|0.05|0.05|R|F|1992-12-22|1992-11-27|1993-01-06|NONE|AIR|side of the slyly unusual theodolites. f 164|125509|3046|3|38|58311.00|0.03|0.06|R|F|1992-12-04|1992-11-23|1993-01-02|TAKE BACK RETURN|AIR|counts cajole fluffily regular packages. b 164|17526|28|4|32|46192.64|0.05|0.01|R|F|1992-12-21|1992-12-23|1992-12-28|COLLECT COD|RAIL|ts wake again 164|147505|2534|5|43|66757.50|0.06|0.01|R|F|1992-11-26|1993-01-03|1992-12-08|COLLECT COD|RAIL|y carefully regular dep 164|108896|8897|6|27|51432.03|0.10|0.04|R|F|1992-12-23|1993-01-16|1993-01-10|DELIVER IN PERSON|AIR|ayers wake carefully a 164|3037|5538|7|23|21620.69|0.09|0.04|A|F|1992-11-03|1992-12-02|1992-11-12|NONE|REG AIR|ress packages haggle ideas. blithely spec 165|33175|8182|1|3|3324.51|0.01|0.08|R|F|1993-03-29|1993-03-06|1993-04-12|DELIVER IN PERSON|REG AIR|riously requests. depos 165|161627|9176|2|43|72610.66|0.08|0.05|R|F|1993-02-27|1993-04-19|1993-03-03|DELIVER IN PERSON|TRUCK|jole slyly according 165|58520|6036|3|15|22177.80|0.00|0.05|R|F|1993-04-10|1993-03-29|1993-05-01|COLLECT COD|SHIP| bold packages mainta 165|139190|4217|4|49|60230.31|0.07|0.06|A|F|1993-02-20|1993-04-02|1993-03-10|COLLECT COD|REG AIR|uses sleep slyly ruthlessly regular a 165|155084|7600|5|27|30755.16|0.01|0.04|R|F|1993-04-27|1993-03-04|1993-05-13|NONE|MAIL|around the ironic, even orb 166|64888|9901|1|37|68556.56|0.09|0.03|N|O|1995-11-16|1995-10-17|1995-12-13|NONE|MAIL|lar frays wake blithely a 166|166366|6367|2|13|18620.68|0.09|0.05|N|O|1995-11-09|1995-11-18|1995-11-14|COLLECT COD|SHIP|fully above the blithely fina 166|99652|2162|3|41|67717.65|0.07|0.03|N|O|1995-11-13|1995-11-07|1995-12-08|COLLECT COD|FOB|hily along the blithely pending fo 166|45027|7532|4|8|7776.16|0.05|0.02|N|O|1995-12-30|1995-11-29|1996-01-29|DELIVER IN PERSON|RAIL|e carefully bold 167|101171|1172|1|28|32820.76|0.06|0.01|R|F|1993-02-19|1993-02-16|1993-03-03|DELIVER IN PERSON|TRUCK|sly during the u 167|171555|4073|2|27|43916.85|0.09|0.00|R|F|1993-05-01|1993-03-31|1993-05-31|TAKE BACK RETURN|FOB|eans affix furiously-- packages 192|97017|2036|1|23|23322.23|0.00|0.00|N|O|1998-02-05|1998-02-06|1998-03-07|TAKE BACK RETURN|AIR|ly pending theodolites haggle quickly fluf 192|161368|8917|2|20|28587.20|0.07|0.01|N|O|1998-03-13|1998-02-02|1998-03-31|TAKE BACK RETURN|REG AIR|tes. carefu 192|110252|5275|3|15|18933.75|0.09|0.01|N|O|1998-01-30|1998-02-10|1998-02-23|TAKE BACK RETURN|TRUCK|he ironic requests haggle about 192|196400|3958|4|2|2992.80|0.06|0.02|N|O|1998-03-06|1998-02-03|1998-03-24|COLLECT COD|SHIP|s. dependencies nag furiously alongside 192|82915|7932|5|25|47447.75|0.02|0.03|N|O|1998-02-15|1998-01-11|1998-03-17|COLLECT COD|TRUCK|. carefully regular 192|141003|3518|6|45|46980.00|0.00|0.05|N|O|1998-03-11|1998-01-09|1998-04-03|NONE|MAIL|equests. ideas sleep idea 193|92638|5148|1|9|14675.67|0.06|0.06|A|F|1993-09-17|1993-10-08|1993-09-30|COLLECT COD|TRUCK|against the fluffily regular d 193|153954|1500|2|15|30119.25|0.02|0.07|R|F|1993-11-22|1993-10-09|1993-12-05|TAKE BACK RETURN|SHIP|ffily. regular packages d 193|93878|6388|3|23|43053.01|0.06|0.05|A|F|1993-08-21|1993-10-11|1993-09-02|DELIVER IN PERSON|TRUCK|ly even accounts wake blithely bold 194|2594|5095|1|17|25442.03|0.05|0.04|R|F|1992-05-24|1992-05-22|1992-05-30|COLLECT COD|AIR| regular deposi 194|183523|6042|2|1|1606.52|0.04|0.06|R|F|1992-04-30|1992-05-18|1992-05-23|NONE|REG AIR| regular theodolites. regular, iron 194|65994|3513|3|13|25479.87|0.08|0.08|A|F|1992-05-07|1992-06-18|1992-05-10|NONE|AIR|about the blit 194|145146|5147|4|36|42881.04|0.00|0.05|R|F|1992-05-21|1992-05-18|1992-05-27|TAKE BACK RETURN|RAIL|pecial packages wake after the slyly r 194|56176|1187|5|8|9057.36|0.04|0.00|R|F|1992-07-06|1992-06-25|1992-07-11|COLLECT COD|FOB|uriously unusual excuses 194|148984|1499|6|16|32527.68|0.06|0.03|A|F|1992-05-14|1992-06-14|1992-05-21|TAKE BACK RETURN|TRUCK|y regular requests. furious 194|167828|345|7|21|39812.22|0.02|0.01|R|F|1992-05-06|1992-05-20|1992-05-07|COLLECT COD|REG AIR|accounts detect quickly dogged 195|84590|9607|1|6|9447.54|0.04|0.02|A|F|1994-01-09|1994-03-27|1994-01-28|COLLECT COD|REG AIR|y, even deposits haggle carefully. bli 195|93847|1375|2|41|75474.44|0.05|0.07|A|F|1994-02-24|1994-02-11|1994-03-20|NONE|TRUCK|rts detect in place of t 195|85446|7955|3|34|48668.96|0.08|0.08|R|F|1994-01-31|1994-02-11|1994-02-12|NONE|TRUCK| cajole furiously bold i 195|85442|7951|4|41|58525.04|0.06|0.04|R|F|1994-03-14|1994-03-13|1994-04-09|COLLECT COD|RAIL|ggle fluffily foxes. fluffily ironic ex 196|135052|79|1|19|20653.95|0.03|0.02|R|F|1993-04-17|1993-05-27|1993-04-30|NONE|SHIP|sts maintain foxes. furiously regular p 196|9852|2353|2|15|26427.75|0.03|0.04|A|F|1993-07-05|1993-05-08|1993-07-06|TAKE BACK RETURN|SHIP|s accounts. furio 197|98494|1004|1|39|58207.11|0.02|0.04|N|O|1995-07-21|1995-07-01|1995-08-14|TAKE BACK RETURN|AIR|press accounts. daringly sp 197|177103|9621|2|8|9440.80|0.09|0.02|A|F|1995-04-17|1995-07-01|1995-04-27|DELIVER IN PERSON|SHIP|y blithely even deposits. blithely fina 197|155829|8345|3|17|32041.94|0.06|0.02|N|O|1995-08-02|1995-06-23|1995-08-03|COLLECT COD|REG AIR|ts. careful 197|17936|2939|4|25|46348.25|0.04|0.01|N|F|1995-06-13|1995-05-23|1995-06-24|TAKE BACK RETURN|FOB|s-- quickly final accounts 197|41466|3971|5|14|19704.44|0.09|0.01|R|F|1995-05-08|1995-05-24|1995-05-12|TAKE BACK RETURN|RAIL|use slyly slyly silent depo 197|105880|901|6|1|1885.88|0.07|0.05|N|O|1995-07-15|1995-06-21|1995-08-11|COLLECT COD|RAIL| even, thin dependencies sno 198|56061|6062|1|33|33562.98|0.07|0.02|N|O|1998-01-05|1998-03-20|1998-01-10|TAKE BACK RETURN|TRUCK|carefully caref 198|15229|7731|2|20|22884.40|0.03|0.00|N|O|1998-01-15|1998-03-31|1998-01-25|DELIVER IN PERSON|FOB|carefully final escapades a 198|148058|3087|3|15|16590.75|0.04|0.02|N|O|1998-04-12|1998-02-26|1998-04-15|COLLECT COD|MAIL|es. quickly pending deposits s 198|10371|2873|4|35|44847.95|0.08|0.02|N|O|1998-02-27|1998-03-23|1998-03-14|TAKE BACK RETURN|RAIL|ests nod quickly furiously sly pinto be 198|101952|1953|5|33|64480.35|0.02|0.01|N|O|1998-03-22|1998-03-12|1998-04-14|DELIVER IN PERSON|SHIP|ending foxes acr 199|132072|9612|1|50|55203.50|0.02|0.00|N|O|1996-06-12|1996-06-03|1996-07-04|DELIVER IN PERSON|MAIL|essly regular ideas boost sly 199|133998|3999|2|30|60959.70|0.08|0.05|N|O|1996-03-27|1996-05-29|1996-04-14|NONE|TRUCK|ilent packages doze quickly. thinly 224|150111|112|1|16|18577.76|0.04|0.00|A|F|1994-08-01|1994-07-30|1994-08-27|DELIVER IN PERSON|MAIL|y unusual foxes 224|108609|1120|2|34|54998.40|0.04|0.08|R|F|1994-07-13|1994-08-25|1994-07-31|COLLECT COD|TRUCK| carefully. final platelets 224|189967|7522|3|41|84335.36|0.07|0.04|A|F|1994-09-01|1994-09-15|1994-09-02|TAKE BACK RETURN|SHIP|after the furiou 224|166377|1410|4|12|17320.44|0.08|0.06|R|F|1994-10-12|1994-08-29|1994-10-20|DELIVER IN PERSON|MAIL|uriously regular packages. slyly fina 224|93857|8876|5|45|83288.25|0.07|0.07|R|F|1994-08-14|1994-09-02|1994-08-27|COLLECT COD|AIR|leep furiously regular requests. furiousl 224|50010|7526|6|4|3840.04|0.02|0.00|R|F|1994-09-08|1994-08-24|1994-10-04|DELIVER IN PERSON|FOB|tructions 225|171925|1926|1|4|7987.68|0.09|0.07|N|O|1995-08-05|1995-08-19|1995-09-03|TAKE BACK RETURN|SHIP|ng the ironic packages. asymptotes among 225|130565|8105|2|3|4786.68|0.00|0.08|N|O|1995-07-25|1995-07-08|1995-08-17|DELIVER IN PERSON|REG AIR| fluffily about the carefully bold a 225|198212|3251|3|45|58959.45|0.06|0.01|N|O|1995-08-17|1995-08-20|1995-08-30|TAKE BACK RETURN|FOB|the slyly even platelets use aro 225|146071|8586|4|24|26809.68|0.00|0.06|N|O|1995-09-23|1995-08-05|1995-10-16|COLLECT COD|MAIL|ironic accounts are final account 225|7589|5090|5|31|46393.98|0.04|0.06|N|O|1995-06-21|1995-07-24|1995-07-04|TAKE BACK RETURN|FOB|special platelets. quickly r 225|131835|9375|6|12|22401.96|0.00|0.00|A|F|1995-06-04|1995-07-15|1995-06-08|COLLECT COD|MAIL| unusual requests. bus 225|141233|8776|7|44|56066.12|0.10|0.06|N|O|1995-09-22|1995-08-16|1995-10-22|NONE|REG AIR|leep slyly 226|96909|9419|1|4|7623.60|0.00|0.00|R|F|1993-03-31|1993-04-30|1993-04-10|NONE|TRUCK|c foxes integrate carefully against th 226|137802|5342|2|46|84630.80|0.06|0.01|A|F|1993-07-06|1993-04-24|1993-07-13|COLLECT COD|FOB|s. carefully bold accounts cajol 226|37309|4819|3|35|43620.50|0.09|0.03|A|F|1993-03-31|1993-05-18|1993-04-01|NONE|RAIL|osits cajole. final, even foxes a 226|40633|8146|4|45|70813.35|0.10|0.02|R|F|1993-04-17|1993-05-27|1993-05-11|DELIVER IN PERSON|AIR| carefully pending pi 226|117956|5490|5|2|3947.90|0.07|0.02|R|F|1993-03-26|1993-04-13|1993-04-20|TAKE BACK RETURN|SHIP|al platelets. express somas 226|82937|7954|6|48|92156.64|0.02|0.00|A|F|1993-06-11|1993-05-15|1993-06-19|NONE|REG AIR|efully silent packages. final deposit 226|117961|5495|7|14|27705.44|0.09|0.00|R|F|1993-05-20|1993-06-05|1993-05-27|COLLECT COD|MAIL|ep carefully regular accounts. ironic 227|165335|2884|1|19|26606.27|0.05|0.06|N|O|1995-12-10|1996-01-30|1995-12-26|NONE|RAIL|s cajole furiously a 227|174102|1654|2|24|28226.40|0.07|0.07|N|O|1996-02-03|1995-12-24|1996-02-12|COLLECT COD|SHIP|uses across the blithe dependencies cajol 228|4039|6540|1|3|2829.09|0.10|0.08|A|F|1993-05-20|1993-04-08|1993-05-26|DELIVER IN PERSON|SHIP|ckages. sly 229|83580|8597|1|20|31271.60|0.02|0.03|R|F|1994-01-11|1994-01-31|1994-01-26|DELIVER IN PERSON|REG AIR|le. instructions use across the quickly fin 229|128904|8905|2|29|56054.10|0.07|0.00|A|F|1994-03-15|1994-03-02|1994-03-26|COLLECT COD|SHIP|s, final request 229|78526|8527|3|28|42126.56|0.02|0.02|R|F|1994-02-10|1994-02-02|1994-03-10|DELIVER IN PERSON|FOB| final, regular requests. platel 229|176948|1983|4|3|6074.82|0.02|0.08|R|F|1994-03-22|1994-03-24|1994-04-04|DELIVER IN PERSON|REG AIR|posits. furiously regular theodol 229|155180|211|5|33|40760.94|0.03|0.06|R|F|1994-03-25|1994-02-11|1994-04-13|NONE|FOB| deposits; bold, ruthless theodolites 229|105393|7904|6|29|40553.31|0.04|0.00|R|F|1994-01-14|1994-02-16|1994-01-22|NONE|FOB|uriously pending 230|185863|900|1|46|89647.56|0.09|0.00|R|F|1994-02-03|1994-01-15|1994-02-23|TAKE BACK RETURN|SHIP|old packages ha 230|194908|7428|2|6|12017.40|0.03|0.08|A|F|1994-01-26|1994-01-25|1994-02-13|NONE|REG AIR| sleep furiously about the p 230|7367|4868|3|1|1274.36|0.07|0.06|R|F|1994-01-22|1994-01-03|1994-02-05|TAKE BACK RETURN|RAIL|blithely unusual dolphins. bold, ex 230|9164|1665|4|44|47219.04|0.08|0.06|R|F|1994-02-09|1994-01-18|1994-03-11|NONE|MAIL|deposits integrate slyly sile 230|18923|6427|5|8|14735.36|0.09|0.06|R|F|1993-11-03|1994-01-20|1993-11-11|TAKE BACK RETURN|TRUCK|g the instructions. fluffil 230|33927|1437|6|8|14887.36|0.00|0.05|R|F|1993-11-21|1994-01-05|1993-12-19|TAKE BACK RETURN|FOB|nal ideas. silent, reg 231|158356|8357|1|16|22629.60|0.04|0.08|R|F|1994-11-20|1994-10-29|1994-12-17|TAKE BACK RETURN|AIR|e furiously ironic pinto beans. 231|83359|884|2|46|61748.10|0.04|0.05|R|F|1994-12-13|1994-12-02|1994-12-14|DELIVER IN PERSON|SHIP|affix blithely. bold requests among the f 231|198124|644|3|50|61106.00|0.09|0.01|A|F|1994-12-11|1994-12-14|1994-12-13|NONE|RAIL|onic packages haggle fluffily a 231|56760|6761|4|31|53219.56|0.08|0.02|A|F|1994-11-05|1994-12-27|1994-11-30|TAKE BACK RETURN|SHIP|iously special decoys wake q 256|88233|742|1|22|26867.06|0.09|0.02|R|F|1994-01-12|1993-12-28|1994-01-26|COLLECT COD|FOB|ke quickly ironic, ironic deposits. reg 256|118399|3422|2|40|56695.60|0.10|0.01|A|F|1993-11-30|1993-12-13|1993-12-02|NONE|FOB|nal theodolites. deposits cajole s 256|129111|4136|3|45|51304.95|0.02|0.08|R|F|1994-01-14|1994-01-17|1994-02-10|COLLECT COD|SHIP| grouches. ideas wake quickly ar 257|146229|6230|1|7|8926.54|0.05|0.02|N|O|1998-06-18|1998-05-15|1998-06-27|COLLECT COD|FOB|ackages sleep bold realms. f 258|106194|3725|1|8|9601.52|0.00|0.07|R|F|1994-01-20|1994-03-21|1994-02-09|NONE|REG AIR|ully about the fluffily silent dependencies 258|196119|3677|2|40|48604.40|0.10|0.01|A|F|1994-03-13|1994-02-23|1994-04-05|DELIVER IN PERSON|FOB|silent frets nod daringly busy, bold 258|161762|1763|3|45|82069.20|0.07|0.07|R|F|1994-03-04|1994-02-13|1994-03-30|DELIVER IN PERSON|TRUCK|regular excuses-- fluffily ruthl 258|132912|5426|4|31|60292.21|0.02|0.05|A|F|1994-04-20|1994-03-20|1994-04-28|COLLECT COD|REG AIR| slyly blithely special mul 258|35959|8463|5|25|47373.75|0.08|0.02|A|F|1994-04-13|1994-02-26|1994-04-29|TAKE BACK RETURN|TRUCK|leep pending packages. 258|146467|8982|6|36|54484.56|0.09|0.04|A|F|1994-01-11|1994-03-04|1994-01-18|DELIVER IN PERSON|AIR|nic asymptotes. slyly silent r 259|98779|8780|1|14|24888.78|0.00|0.08|A|F|1993-12-17|1993-12-09|1993-12-31|COLLECT COD|SHIP|ons against the express acco 259|161982|4499|2|14|28615.72|0.03|0.05|R|F|1993-11-10|1993-11-20|1993-11-17|DELIVER IN PERSON|FOB|ully even, regul 259|23514|3515|3|42|60375.42|0.09|0.00|R|F|1993-10-20|1993-11-18|1993-11-12|NONE|TRUCK|the slyly ironic pinto beans. fi 259|195335|2893|4|3|4290.99|0.08|0.06|R|F|1993-10-04|1993-11-07|1993-10-14|TAKE BACK RETURN|SHIP|ng slyly at the accounts. 259|192201|7240|5|6|7759.20|0.00|0.05|R|F|1993-12-05|1993-12-22|1993-12-21|COLLECT COD|TRUCK| requests sleep 260|155887|5888|1|50|97144.00|0.07|0.08|N|O|1997-03-24|1997-02-09|1997-04-20|TAKE BACK RETURN|REG AIR|c deposits 260|182736|2737|2|26|47286.98|0.02|0.07|N|O|1996-12-12|1997-02-06|1996-12-15|NONE|TRUCK|ld theodolites boost fl 260|41222|8735|3|27|31406.94|0.05|0.08|N|O|1997-03-23|1997-02-15|1997-04-22|TAKE BACK RETURN|RAIL|ions according to the 260|5337|338|4|29|36027.57|0.10|0.06|N|O|1997-03-15|1997-01-14|1997-04-13|NONE|MAIL|fluffily even asymptotes. express wa 260|95286|305|5|44|56376.32|0.01|0.05|N|O|1997-03-26|1997-02-03|1997-04-19|DELIVER IN PERSON|MAIL|above the blithely ironic instr 261|1349|6350|1|34|42511.56|0.05|0.08|R|F|1993-08-18|1993-09-24|1993-08-20|COLLECT COD|REG AIR|c packages. asymptotes da 261|65662|5663|2|20|32553.20|0.00|0.06|R|F|1993-10-21|1993-08-02|1993-11-04|DELIVER IN PERSON|RAIL|ites hinder 261|173959|8994|3|28|56922.60|0.08|0.03|R|F|1993-07-24|1993-08-20|1993-08-05|COLLECT COD|AIR|ironic packages nag slyly. carefully fin 261|118455|967|4|49|72199.05|0.04|0.05|R|F|1993-09-12|1993-08-31|1993-10-07|COLLECT COD|SHIP|ions. bold accounts 261|60469|7988|5|49|70043.54|0.01|0.08|A|F|1993-09-29|1993-09-08|1993-10-01|COLLECT COD|SHIP| pinto beans haggle slyly furiously pending 261|96989|9499|6|20|39719.60|0.06|0.06|A|F|1993-10-15|1993-09-05|1993-11-07|NONE|AIR|ing to the special, ironic deposi 262|191186|1187|1|39|49810.02|0.01|0.05|N|O|1996-01-15|1996-02-18|1996-01-28|COLLECT COD|RAIL|usual, regular requests 262|60074|7593|2|33|34124.31|0.09|0.03|N|O|1996-03-10|1996-01-31|1996-03-27|TAKE BACK RETURN|AIR|atelets sleep furiously. requests cajole. b 262|58695|6211|3|35|57879.15|0.05|0.08|N|O|1996-03-12|1996-02-14|1996-04-11|COLLECT COD|MAIL|lites cajole along the pending packag 263|23960|6463|1|22|41447.12|0.06|0.08|R|F|1994-08-24|1994-06-20|1994-09-09|NONE|FOB|efully express fo 263|84557|9574|2|9|13873.95|0.08|0.00|A|F|1994-07-21|1994-07-16|1994-08-08|TAKE BACK RETURN|TRUCK|lms wake bl 263|142891|434|3|50|96694.50|0.06|0.04|R|F|1994-08-18|1994-07-31|1994-08-22|NONE|TRUCK|re the packages. special 288|50641|8157|1|31|49340.84|0.00|0.03|N|O|1997-03-17|1997-04-28|1997-04-06|TAKE BACK RETURN|AIR|instructions wa 288|116386|8898|2|49|68716.62|0.08|0.05|N|O|1997-04-19|1997-05-19|1997-05-18|TAKE BACK RETURN|TRUCK|ic excuses sleep always spe 288|98833|8834|3|36|65945.88|0.02|0.02|N|O|1997-02-22|1997-05-07|1997-03-07|TAKE BACK RETURN|TRUCK|yly pending excu 288|78406|8407|4|19|26303.60|0.07|0.07|N|O|1997-03-14|1997-04-04|1997-03-26|NONE|MAIL|deposits. blithely quick courts ar 288|161894|6927|5|31|60632.59|0.10|0.04|N|O|1997-05-29|1997-04-24|1997-06-20|TAKE BACK RETURN|RAIL|ns. fluffily 289|173280|832|1|25|33832.00|0.07|0.05|N|O|1997-03-18|1997-05-05|1997-04-15|DELIVER IN PERSON|FOB|out the quickly bold theodol 289|111800|9334|2|6|10870.80|0.06|0.05|N|O|1997-02-18|1997-05-08|1997-03-19|DELIVER IN PERSON|SHIP|d packages use fluffily furiously 289|16996|1999|3|44|84171.56|0.10|0.08|N|O|1997-06-05|1997-04-20|1997-07-02|COLLECT COD|MAIL|ly ironic foxes. asymptotes 289|39439|1943|4|48|66164.64|0.01|0.08|N|O|1997-03-14|1997-03-30|1997-03-24|DELIVER IN PERSON|RAIL|sits cajole. bold pinto beans x-ray fl 289|46285|8790|5|13|16006.64|0.10|0.03|N|O|1997-06-08|1997-04-06|1997-06-18|TAKE BACK RETURN|REG AIR|ts. quickly bold deposits alongside 290|5351|352|1|35|43972.25|0.01|0.02|R|F|1994-04-01|1994-02-05|1994-04-27|NONE|MAIL|ove the final foxes detect slyly fluffily 290|128923|1436|2|2|3903.84|0.05|0.04|A|F|1994-01-30|1994-02-13|1994-02-21|TAKE BACK RETURN|TRUCK|. permanently furious reques 290|1888|4389|3|5|8949.40|0.03|0.05|A|F|1994-01-19|1994-02-24|1994-01-27|NONE|MAIL|ans integrate. requests sleep. fur 290|123741|6254|4|23|40589.02|0.05|0.08|R|F|1994-03-14|1994-02-21|1994-04-09|NONE|AIR|refully unusual packages. 291|122565|102|1|21|33338.76|0.05|0.07|A|F|1994-05-26|1994-05-10|1994-06-23|COLLECT COD|TRUCK|y quickly regular theodolites. final t 291|137316|7317|2|19|25712.89|0.08|0.02|R|F|1994-06-14|1994-04-25|1994-06-19|NONE|REG AIR|e. ruthlessly final accounts after the 291|60874|5887|3|30|55046.10|0.10|0.02|R|F|1994-03-22|1994-04-30|1994-03-24|DELIVER IN PERSON|FOB| fluffily regular deposits. quickl 292|153561|3562|1|8|12916.48|0.10|0.03|R|F|1992-02-18|1992-03-30|1992-03-18|DELIVER IN PERSON|RAIL|sily bold deposits alongside of the ex 292|99249|9250|2|24|29957.76|0.08|0.04|R|F|1992-03-24|1992-03-06|1992-04-20|COLLECT COD|TRUCK| bold, pending theodolites u 293|8960|6461|1|14|26165.44|0.02|0.05|R|F|1992-10-19|1992-12-23|1992-11-10|DELIVER IN PERSON|SHIP|es. packages above the 293|186406|6407|2|11|16416.40|0.10|0.04|R|F|1992-12-24|1992-12-01|1993-01-12|COLLECT COD|MAIL| affix carefully quickly special idea 293|117267|4801|3|13|16695.38|0.04|0.02|A|F|1992-12-17|1992-12-26|1992-12-22|COLLECT COD|RAIL| wake after the quickly even deposits. bli 294|59620|7136|1|31|48968.22|0.00|0.01|R|F|1993-08-06|1993-08-19|1993-08-13|TAKE BACK RETURN|AIR|le fluffily along the quick 295|197507|27|1|29|46530.50|0.02|0.07|A|F|1994-11-09|1994-12-08|1994-12-07|COLLECT COD|MAIL|inst the carefully ironic pinto beans. blit 295|91344|8872|2|26|34718.84|0.04|0.03|R|F|1994-12-13|1994-11-30|1995-01-06|DELIVER IN PERSON|AIR|ts above the slyly regular requests x-ray q 295|15283|7785|3|8|9586.24|0.10|0.07|R|F|1995-01-13|1994-11-17|1995-01-25|NONE|TRUCK| final instructions h 295|60621|3128|4|26|41122.12|0.10|0.04|A|F|1995-01-12|1994-11-22|1995-01-22|DELIVER IN PERSON|MAIL| carefully iron 320|4415|1916|1|30|39582.30|0.05|0.01|N|O|1997-12-04|1998-01-21|1997-12-13|NONE|RAIL| ironic, final accounts wake quick de 320|192158|4678|2|13|16251.95|0.03|0.00|N|O|1997-12-16|1997-12-26|1997-12-17|TAKE BACK RETURN|AIR|he furiously regular pinto beans. car 321|318|7819|1|21|25584.51|0.01|0.08|A|F|1993-07-18|1993-04-24|1993-08-13|TAKE BACK RETURN|REG AIR|hockey players sleep slyly sl 321|140433|5462|2|41|60410.63|0.08|0.07|R|F|1993-06-21|1993-06-07|1993-07-09|NONE|REG AIR|special packages shall have to doze blit 322|152499|7530|1|12|18617.88|0.08|0.07|A|F|1992-06-29|1992-05-30|1992-07-11|NONE|AIR|ular theodolites promise qu 322|43662|3663|2|48|77071.68|0.02|0.07|A|F|1992-06-11|1992-06-16|1992-06-26|COLLECT COD|RAIL|dolites detect qu 322|12673|177|3|20|31713.40|0.04|0.01|R|F|1992-04-26|1992-05-04|1992-05-22|DELIVER IN PERSON|MAIL|ckly toward 322|183246|5765|4|10|13292.40|0.06|0.03|R|F|1992-04-12|1992-05-13|1992-04-14|DELIVER IN PERSON|AIR| deposits grow slyly according to th 322|11605|9109|5|35|53081.00|0.07|0.06|A|F|1992-07-17|1992-05-03|1992-08-14|TAKE BACK RETURN|RAIL|egular accounts cajole carefully. even d 322|33310|8317|6|3|3729.93|0.08|0.05|A|F|1992-07-03|1992-05-10|1992-07-28|NONE|AIR|ending, ironic deposits along the blith 322|37435|4945|7|5|6862.15|0.01|0.02|A|F|1992-04-15|1992-05-12|1992-04-26|COLLECT COD|REG AIR| special grouches sleep quickly instructio 323|163628|1177|1|50|84581.00|0.05|0.04|A|F|1994-04-20|1994-04-25|1994-05-12|DELIVER IN PERSON|REG AIR|cial requests 323|95136|7646|2|18|20360.34|0.06|0.07|R|F|1994-04-13|1994-06-02|1994-05-10|DELIVER IN PERSON|TRUCK|posits cajole furiously pinto beans. 323|142725|2726|3|9|15909.48|0.07|0.04|A|F|1994-06-26|1994-06-10|1994-07-13|COLLECT COD|TRUCK|nic accounts. regular, regular pack 324|199475|4514|1|26|40936.22|0.07|0.01|R|F|1992-04-19|1992-05-28|1992-05-12|DELIVER IN PERSON|RAIL|ross the slyly regular s 325|158791|6337|1|34|62892.86|0.09|0.04|A|F|1993-10-28|1993-12-13|1993-11-17|TAKE BACK RETURN|MAIL|ly bold deposits. always iron 325|185139|7658|2|5|6120.65|0.07|0.08|A|F|1994-01-02|1994-01-05|1994-01-04|TAKE BACK RETURN|MAIL| theodolites. 325|18788|1290|3|35|59737.30|0.07|0.07|A|F|1993-12-06|1994-01-03|1993-12-26|DELIVER IN PERSON|REG AIR|packages wa 326|179094|4129|1|41|48096.69|0.06|0.03|N|O|1995-08-30|1995-07-09|1995-09-12|DELIVER IN PERSON|TRUCK|ily quickly bold ideas. 326|19480|1982|2|38|53180.24|0.02|0.08|N|O|1995-09-12|1995-08-23|1995-09-14|COLLECT COD|RAIL|es sleep slyly. carefully regular inst 326|183739|8776|3|25|45568.25|0.03|0.04|N|O|1995-08-03|1995-07-27|1995-08-16|NONE|AIR|ily furiously unusual accounts. 326|84836|9853|4|5|9104.15|0.03|0.08|N|O|1995-07-29|1995-07-13|1995-08-12|NONE|REG AIR|deas sleep according to the sometimes spe 326|34543|9550|5|31|45803.74|0.04|0.08|N|O|1995-09-27|1995-07-06|1995-10-22|NONE|TRUCK|cies sleep quick 326|156712|4258|6|41|72517.11|0.02|0.00|N|O|1995-07-05|1995-07-23|1995-07-20|TAKE BACK RETURN|REG AIR|to beans wake before the furiously re 326|42134|4639|7|47|50578.11|0.04|0.04|N|O|1995-09-16|1995-07-04|1995-10-04|NONE|REG AIR| special accounts sleep 327|143503|1046|1|16|24744.00|0.03|0.01|N|O|1995-07-05|1995-06-07|1995-07-09|TAKE BACK RETURN|TRUCK|cial ideas sleep af 327|41715|4220|2|9|14910.39|0.09|0.05|A|F|1995-05-24|1995-07-11|1995-06-05|NONE|AIR| asymptotes are fu 352|63762|3763|1|17|29337.92|0.07|0.05|R|F|1994-06-02|1994-05-31|1994-06-29|NONE|FOB|pending deposits sleep furiously 353|119305|4328|1|41|54296.30|0.00|0.06|A|F|1994-03-25|1994-03-31|1994-03-30|DELIVER IN PERSON|AIR|refully final theodoli 353|147542|7543|2|29|46096.66|0.09|0.00|A|F|1994-01-11|1994-03-19|1994-02-09|COLLECT COD|FOB|ctions impr 353|134318|1858|3|12|16227.72|0.06|0.01|R|F|1994-01-02|1994-03-26|1994-01-19|DELIVER IN PERSON|RAIL|g deposits cajole 353|77071|2086|4|46|48211.22|0.00|0.04|A|F|1994-04-14|1994-01-31|1994-05-05|DELIVER IN PERSON|FOB| ironic dolphins 353|116803|1826|5|9|16378.20|0.02|0.02|A|F|1994-03-15|1994-03-20|1994-03-18|TAKE BACK RETURN|RAIL|ual accounts! carefu 353|102699|2700|6|39|66365.91|0.02|0.05|A|F|1994-01-15|1994-03-30|1994-02-01|NONE|MAIL|losely quickly even accounts. c 354|49480|1985|1|14|20012.72|0.08|0.04|N|O|1996-04-12|1996-06-03|1996-05-08|NONE|SHIP|quickly regular grouches will eat. careful 354|193864|1422|2|24|46988.64|0.01|0.01|N|O|1996-05-08|1996-05-17|1996-06-07|DELIVER IN PERSON|AIR|y silent requests. regular, even accounts 354|58125|8126|3|50|54156.00|0.08|0.05|N|O|1996-03-21|1996-05-20|1996-04-04|COLLECT COD|TRUCK|to beans s 354|106672|4203|4|7|11750.69|0.06|0.01|N|O|1996-05-07|1996-04-18|1996-05-24|NONE|MAIL|ously idly ironic accounts-- quickl 354|30527|528|5|18|26235.36|0.04|0.08|N|O|1996-03-31|1996-05-13|1996-04-27|DELIVER IN PERSON|RAIL| about the carefully unusual 354|61082|3589|6|36|37550.88|0.03|0.02|N|O|1996-03-19|1996-05-29|1996-03-30|NONE|AIR|onic requests thrash bold g 354|4660|9661|7|14|21905.24|0.01|0.07|N|O|1996-07-06|1996-06-08|1996-07-10|TAKE BACK RETURN|MAIL|t thinly above the ironic, 355|113959|8982|1|31|61161.45|0.09|0.07|A|F|1994-07-13|1994-08-18|1994-07-18|DELIVER IN PERSON|FOB|y unusual, ironic 355|96030|3558|2|41|42067.23|0.05|0.00|A|F|1994-08-15|1994-07-19|1994-09-06|DELIVER IN PERSON|TRUCK| deposits. carefully r 356|45214|5215|1|4|4636.84|0.10|0.01|A|F|1994-07-28|1994-08-01|1994-08-04|DELIVER IN PERSON|REG AIR| the dependencies nod unusual, final ac 356|107463|2484|2|48|70582.08|0.02|0.03|R|F|1994-08-12|1994-07-31|1994-08-26|NONE|FOB|unusual packages. furiously 356|118002|514|3|35|35700.00|0.08|0.07|R|F|1994-10-14|1994-07-31|1994-10-23|COLLECT COD|TRUCK|s. unusual, final 356|55342|353|4|41|53190.94|0.07|0.05|A|F|1994-09-28|1994-09-20|1994-10-07|COLLECT COD|SHIP| according to the express foxes will 356|124271|1808|5|37|47924.99|0.05|0.03|A|F|1994-07-15|1994-08-24|1994-08-09|DELIVER IN PERSON|FOB|ndencies are since the packag 357|113143|3144|1|26|30059.64|0.06|0.03|N|O|1996-12-28|1996-11-26|1997-01-13|NONE|FOB| carefully pending accounts use a 357|185814|8333|2|36|68393.16|0.07|0.06|N|O|1996-12-28|1996-11-13|1997-01-24|DELIVER IN PERSON|AIR|d the carefully even requests. 357|164807|9840|3|32|59897.60|0.05|0.07|N|O|1997-01-28|1996-12-29|1997-02-14|NONE|MAIL|y above the carefully final accounts 358|190028|2548|1|41|45838.82|0.06|0.01|A|F|1993-11-18|1993-11-14|1993-11-28|NONE|TRUCK|ely frets. furious deposits sleep 358|189955|7510|2|32|65438.40|0.05|0.08|A|F|1993-10-18|1993-12-12|1993-10-31|NONE|TRUCK|y final foxes sleep blithely sl 358|168710|3743|3|40|71148.40|0.09|0.01|A|F|1993-12-05|1993-11-04|1994-01-01|COLLECT COD|MAIL|ng the ironic theo 358|96557|1576|4|15|23303.25|0.08|0.08|A|F|1993-10-04|1993-12-17|1993-10-23|TAKE BACK RETURN|MAIL|out the blithely ironic deposits slee 358|28629|6136|5|18|28037.16|0.01|0.02|R|F|1993-10-07|1993-11-01|1993-10-26|COLLECT COD|SHIP|olphins haggle ironic accounts. f 358|161283|1284|6|32|43016.96|0.03|0.05|R|F|1993-12-21|1993-11-06|1994-01-17|DELIVER IN PERSON|RAIL|lyly express deposits 358|82916|7933|7|45|85450.95|0.05|0.02|A|F|1993-12-08|1993-10-29|1993-12-30|NONE|REG AIR|to beans. regular, unusual deposits sl 359|165980|5981|1|30|61379.40|0.00|0.08|A|F|1995-01-06|1995-02-20|1995-01-20|TAKE BACK RETURN|AIR|uses detect spec 359|11158|6161|2|18|19244.70|0.00|0.03|A|F|1995-01-27|1995-03-18|1995-01-31|DELIVER IN PERSON|RAIL|unusual warthogs. ironically sp 359|131463|3977|3|17|25405.82|0.07|0.06|A|F|1995-01-31|1995-03-18|1995-02-10|COLLECT COD|SHIP|sts according to the blithely 359|89985|2494|4|38|75049.24|0.10|0.08|R|F|1995-03-30|1995-01-20|1995-04-25|DELIVER IN PERSON|RAIL|g furiously. regular, sile 359|167239|2272|5|11|14368.53|0.01|0.03|A|F|1995-02-15|1995-01-27|1995-02-18|NONE|FOB|rets wake blithely. slyly final dep 359|182663|218|6|23|40150.18|0.04|0.07|R|F|1995-01-31|1995-03-11|1995-02-16|DELIVER IN PERSON|REG AIR|ic courts snooze quickly furiously final fo 384|178442|3477|1|38|57776.72|0.07|0.01|R|F|1992-06-02|1992-04-18|1992-06-10|DELIVER IN PERSON|TRUCK|totes cajole blithely against the even 384|63342|5849|2|49|63961.66|0.09|0.07|A|F|1992-04-01|1992-04-25|1992-04-18|COLLECT COD|AIR|refully carefully ironic instructions. bl 384|181502|6539|3|11|17418.50|0.02|0.08|A|F|1992-04-02|1992-04-21|1992-04-15|COLLECT COD|MAIL|ash carefully 384|92053|7072|4|11|11495.55|0.00|0.06|R|F|1992-06-24|1992-05-29|1992-07-22|COLLECT COD|TRUCK|nic excuses are furiously above the blith 384|131403|8943|5|14|20081.60|0.08|0.06|R|F|1992-06-14|1992-05-29|1992-07-05|DELIVER IN PERSON|TRUCK|ckages are slyly after the slyly specia 385|166446|8963|1|7|10587.08|0.05|0.06|N|O|1996-05-23|1996-05-09|1996-06-06|DELIVER IN PERSON|REG AIR| special asymptote 385|53025|8036|2|46|44988.92|0.08|0.07|N|O|1996-03-29|1996-05-17|1996-04-18|NONE|REG AIR|lthily ironic f 386|152405|9951|1|39|56838.60|0.10|0.07|A|F|1995-05-10|1995-02-28|1995-05-25|NONE|SHIP|hely. carefully regular accounts hag 386|68123|5642|2|16|17457.92|0.06|0.01|A|F|1995-04-12|1995-04-18|1995-05-11|DELIVER IN PERSON|MAIL|lithely fluffi 386|130081|82|3|37|41109.96|0.09|0.04|A|F|1995-05-23|1995-03-01|1995-05-25|TAKE BACK RETURN|MAIL|ending pearls breach fluffily. slyly pen 387|136667|1694|1|1|1703.66|0.08|0.03|N|O|1997-05-06|1997-04-23|1997-05-10|NONE|SHIP| pinto beans wake furiously carefu 387|152800|2801|2|42|77817.60|0.07|0.05|N|O|1997-05-25|1997-02-25|1997-05-29|DELIVER IN PERSON|RAIL|lithely final theodolites. 387|96392|1411|3|40|55535.60|0.09|0.02|N|O|1997-03-08|1997-04-18|1997-03-31|COLLECT COD|TRUCK| quickly ironic platelets are slyly. fluff 387|55927|5928|4|19|35775.48|0.08|0.00|N|O|1997-03-14|1997-04-21|1997-04-04|NONE|REG AIR|gular dependencies 387|148313|828|5|32|43561.92|0.08|0.06|N|O|1997-05-02|1997-04-11|1997-05-11|DELIVER IN PERSON|TRUCK|gle. silent, fur 388|32590|100|1|42|63948.78|0.05|0.06|R|F|1993-02-21|1993-02-26|1993-03-15|COLLECT COD|FOB|accounts sleep furiously 388|127808|7809|2|46|84446.80|0.07|0.01|A|F|1993-03-22|1993-01-26|1993-03-24|COLLECT COD|FOB|to beans nag about the careful reque 388|64486|9499|3|40|58019.20|0.06|0.01|A|F|1992-12-24|1993-01-28|1993-01-19|TAKE BACK RETURN|REG AIR|quests against the carefully unusual epi 389|189295|1814|1|2|2768.58|0.09|0.00|R|F|1994-04-13|1994-04-10|1994-04-25|TAKE BACK RETURN|RAIL|fts. courts eat blithely even dependenc 390|106523|9034|1|10|15295.20|0.02|0.05|N|O|1998-05-26|1998-07-06|1998-06-23|TAKE BACK RETURN|SHIP| requests. final accounts x-ray beside the 390|123353|890|2|17|23397.95|0.09|0.06|N|O|1998-06-07|1998-06-14|1998-07-07|COLLECT COD|SHIP|ending, pending pinto beans wake slyl 390|183266|8303|3|46|62065.96|0.07|0.04|N|O|1998-06-06|1998-05-20|1998-06-14|DELIVER IN PERSON|SHIP|cial excuses. bold, pending packages 390|141937|1938|4|42|83115.06|0.01|0.05|N|O|1998-06-06|1998-06-22|1998-07-05|COLLECT COD|SHIP|counts nag across the sly, sil 390|127657|170|5|13|21900.45|0.02|0.06|N|O|1998-07-08|1998-05-10|1998-07-18|DELIVER IN PERSON|SHIP|sleep carefully idle packages. blithely 390|124632|9657|6|11|18222.93|0.09|0.06|N|O|1998-05-05|1998-05-15|1998-06-01|DELIVER IN PERSON|SHIP|according to the foxes are furiously 390|84937|2462|7|24|46126.32|0.05|0.02|N|O|1998-04-18|1998-05-19|1998-04-28|TAKE BACK RETURN|AIR|y. enticingly final depos 391|121586|6611|1|14|22506.12|0.09|0.02|R|F|1995-02-11|1995-02-03|1995-02-13|TAKE BACK RETURN|TRUCK| escapades sleep furiously about 416|93563|6073|1|25|38914.00|0.00|0.05|A|F|1993-10-11|1993-11-26|1993-10-21|DELIVER IN PERSON|TRUCK|y final theodolites about 416|110869|8403|2|22|41356.92|0.10|0.00|R|F|1993-12-27|1993-12-17|1994-01-09|COLLECT COD|RAIL|rint blithely above the pending sentim 416|174101|6619|3|25|29377.50|0.07|0.01|R|F|1993-10-16|1993-12-03|1993-10-29|NONE|AIR|ses boost after the bold requests. 417|39560|9561|1|39|58482.84|0.01|0.02|A|F|1994-05-31|1994-05-02|1994-06-06|NONE|SHIP|y regular requests wake along 417|69212|4225|2|18|21261.78|0.00|0.01|R|F|1994-03-29|1994-04-10|1994-04-26|TAKE BACK RETURN|FOB|- final requests sle 417|44192|6697|3|41|46583.79|0.10|0.01|R|F|1994-04-11|1994-03-08|1994-05-06|COLLECT COD|RAIL|tes. regular requests across the 417|131087|1088|4|2|2236.16|0.01|0.03|R|F|1994-02-13|1994-04-19|1994-03-15|DELIVER IN PERSON|SHIP|uriously bol 418|18552|1054|1|31|45587.05|0.00|0.03|N|F|1995-06-05|1995-06-18|1995-06-26|COLLECT COD|FOB|final theodolites. fluffil 418|1062|3563|2|1|963.06|0.04|0.07|N|O|1995-06-23|1995-06-16|1995-07-23|DELIVER IN PERSON|AIR|regular, silent pinto 418|34829|7333|3|3|5291.46|0.04|0.06|N|O|1995-06-29|1995-07-12|1995-07-01|COLLECT COD|AIR|ly furiously regular w 419|152691|7722|1|33|57541.77|0.05|0.02|N|O|1996-11-06|1996-12-25|1996-11-20|TAKE BACK RETURN|TRUCK|y above the bli 419|64192|9205|2|32|36998.08|0.01|0.06|N|O|1996-12-04|1996-12-04|1996-12-24|COLLECT COD|SHIP|blithely regular requests. special pinto 419|70495|3003|3|15|21982.35|0.07|0.04|N|O|1996-12-17|1996-11-28|1996-12-19|TAKE BACK RETURN|REG AIR| sleep final, regular theodolites. fluffi 419|8756|6257|4|15|24971.25|0.01|0.02|N|O|1997-01-09|1996-12-22|1997-01-25|COLLECT COD|FOB|of the careful, thin theodolites. quickly s 419|148401|3430|5|17|24639.80|0.01|0.00|N|O|1997-01-13|1996-12-20|1997-02-01|COLLECT COD|REG AIR|lar dependencies: carefully regu 420|100885|5906|1|5|9429.40|0.04|0.03|N|O|1995-11-04|1996-01-02|1995-11-30|NONE|REG AIR|cajole blit 420|161079|8628|2|22|25081.54|0.05|0.04|N|O|1996-01-25|1995-12-16|1996-02-03|TAKE BACK RETURN|AIR|ly against the blithely re 420|47557|2566|3|45|67704.75|0.09|0.08|N|O|1996-01-14|1996-01-01|1996-01-26|COLLECT COD|FOB| final accounts. furiously express forges 420|74795|4796|4|12|21237.48|0.08|0.08|N|O|1996-02-05|1996-01-03|1996-02-12|TAKE BACK RETURN|REG AIR|c instructions are 420|72918|7933|5|37|69963.67|0.02|0.00|N|O|1995-11-16|1995-12-13|1995-11-19|DELIVER IN PERSON|SHIP|rbits. bold requests along the quickl 420|123736|1273|6|40|70389.20|0.01|0.05|N|O|1995-11-26|1995-12-26|1995-12-20|TAKE BACK RETURN|FOB| after the special 420|15978|5979|7|39|73864.83|0.00|0.08|N|O|1995-12-09|1995-12-16|1995-12-31|DELIVER IN PERSON|REG AIR|s. ironic waters about the car 421|133070|3071|1|1|1103.07|0.02|0.07|R|F|1992-05-29|1992-04-27|1992-06-09|NONE|TRUCK|oldly busy deposit 422|151816|4332|1|25|46695.25|0.10|0.07|N|O|1997-07-01|1997-08-17|1997-07-09|DELIVER IN PERSON|SHIP|carefully bold theodolit 422|170666|3184|2|10|17366.60|0.02|0.03|N|O|1997-06-15|1997-08-04|1997-07-08|TAKE BACK RETURN|AIR|he furiously ironic theodolite 422|175984|3536|3|46|94759.08|0.09|0.00|N|O|1997-06-21|1997-07-14|1997-06-27|DELIVER IN PERSON|RAIL| ideas. qu 422|161622|9171|4|25|42090.50|0.10|0.04|N|O|1997-08-24|1997-07-09|1997-09-22|NONE|FOB|ep along the furiousl 423|131890|6917|1|27|51891.03|0.06|0.03|N|O|1996-08-20|1996-08-01|1996-08-23|TAKE BACK RETURN|SHIP|ccounts. blithely regular pack 448|125197|5198|1|4|4888.76|0.00|0.04|N|O|1995-11-25|1995-10-20|1995-11-26|TAKE BACK RETURN|MAIL|nts thrash quickly among the b 448|172359|9911|2|46|65842.10|0.05|0.00|N|O|1995-08-31|1995-09-30|1995-09-09|COLLECT COD|SHIP| to the fluffily ironic packages. 448|26809|1814|3|35|60753.00|0.10|0.08|N|O|1995-09-27|1995-11-19|1995-10-20|COLLECT COD|REG AIR|ses nag quickly quickly ir 448|169045|9046|4|8|8912.32|0.10|0.00|N|O|1995-11-02|1995-10-16|1995-11-15|COLLECT COD|TRUCK|ounts wake blithely. furiously pending 448|137283|7284|5|23|30366.44|0.02|0.05|N|O|1995-09-26|1995-11-02|1995-10-17|NONE|SHIP|ious, final gifts 449|151908|6939|1|12|23518.80|0.02|0.08|N|O|1995-11-06|1995-08-25|1995-11-18|TAKE BACK RETURN|SHIP|ly. blithely ironic 449|108408|5939|2|4|5665.60|0.10|0.06|N|O|1995-10-27|1995-09-14|1995-11-21|DELIVER IN PERSON|FOB|are fluffily. requests are furiously 449|9982|9983|3|3|5675.94|0.07|0.08|N|O|1995-07-28|1995-09-11|1995-08-01|NONE|RAIL| bold deposits. express theodolites haggle 449|157659|2690|4|22|37766.30|0.07|0.00|N|O|1995-08-17|1995-09-04|1995-09-10|COLLECT COD|FOB|furiously final theodolites eat careful 450|161582|9131|1|42|69030.36|0.03|0.00|N|F|1995-06-07|1995-05-29|1995-06-23|TAKE BACK RETURN|SHIP|y asymptotes. regular depen 450|106298|6299|2|5|6521.45|0.03|0.02|A|F|1995-04-02|1995-05-06|1995-04-13|TAKE BACK RETURN|TRUCK|the pinto bea 450|142528|7557|3|32|50256.64|0.06|0.03|N|O|1995-07-02|1995-04-25|1995-07-30|TAKE BACK RETURN|SHIP| accounts nod fluffily even, pending 450|56267|3783|4|40|48930.40|0.05|0.03|R|F|1995-03-20|1995-05-25|1995-04-14|NONE|RAIL|ve. asymptote 450|78048|8049|5|2|2052.08|0.09|0.00|A|F|1995-03-11|1995-05-21|1995-03-16|COLLECT COD|AIR|y even pinto beans; qui 450|152726|5242|6|33|58697.76|0.08|0.05|R|F|1995-05-18|1995-05-22|1995-05-23|TAKE BACK RETURN|REG AIR|ily carefully final depo 451|129532|4557|1|36|56215.08|0.02|0.06|N|O|1998-06-18|1998-08-14|1998-06-20|TAKE BACK RETURN|AIR|rges can haggle carefully ironic, dogged 451|32028|7035|2|42|40320.84|0.05|0.01|N|O|1998-08-01|1998-08-05|1998-08-30|DELIVER IN PERSON|TRUCK|express excuses. blithely ironic pin 451|86136|6137|3|1|1122.13|0.07|0.05|N|O|1998-07-13|1998-07-03|1998-08-04|DELIVER IN PERSON|AIR| carefully ironic packages solve furiously 451|76558|4080|4|28|42967.40|0.04|0.05|N|O|1998-06-16|1998-07-09|1998-06-17|DELIVER IN PERSON|SHIP| theodolites. even cou 452|114639|4640|1|2|3307.26|0.04|0.03|N|O|1997-12-26|1998-01-03|1998-01-12|COLLECT COD|FOB|y express instru 453|197917|2956|1|45|90670.95|0.01|0.00|N|O|1997-06-30|1997-08-20|1997-07-19|COLLECT COD|REG AIR|ifts wake carefully. 453|175131|2683|2|38|45832.94|0.08|0.04|N|O|1997-06-30|1997-07-08|1997-07-16|DELIVER IN PERSON|REG AIR| furiously f 453|13144|8147|3|38|40171.32|0.10|0.01|N|O|1997-08-10|1997-07-24|1997-09-07|NONE|SHIP|sts cajole. furiously un 453|95748|5749|4|45|78468.30|0.10|0.01|N|O|1997-09-18|1997-06-29|1997-10-14|TAKE BACK RETURN|AIR|ironic foxes. slyly pending depos 453|25722|8225|5|32|52727.04|0.04|0.01|N|O|1997-07-15|1997-06-27|1997-07-18|NONE|REG AIR|s. fluffily bold packages cajole. unu 453|94318|6828|6|28|36744.68|0.07|0.07|N|O|1997-08-16|1997-08-12|1997-08-27|NONE|MAIL|final dependencies. slyly special pl 454|117595|5129|1|24|38702.16|0.06|0.01|N|O|1996-04-26|1996-03-23|1996-05-20|NONE|TRUCK|le. deposits after the ideas nag unusual pa 455|156485|4031|1|42|64742.16|0.10|0.02|N|O|1997-01-26|1997-01-10|1997-02-22|DELIVER IN PERSON|REG AIR|around the quickly blit 455|27230|7231|2|44|50918.12|0.05|0.08|N|O|1997-01-17|1997-02-22|1997-02-12|TAKE BACK RETURN|TRUCK| accounts sleep slyly ironic asymptote 455|48360|3369|3|45|58876.20|0.04|0.06|N|O|1996-12-20|1997-01-31|1997-01-07|TAKE BACK RETURN|SHIP|thrash ironically regular packages. qui 455|170012|7564|4|11|11902.11|0.01|0.02|N|O|1997-03-15|1997-02-14|1997-03-26|DELIVER IN PERSON|MAIL|g deposits against the slyly idle foxes u 480|52148|2149|1|22|24203.08|0.04|0.02|A|F|1993-06-16|1993-07-28|1993-07-09|NONE|MAIL|into beans cajole furiously. accounts s 481|18649|6153|1|17|26649.88|0.07|0.05|A|F|1992-10-21|1992-12-09|1992-11-19|DELIVER IN PERSON|MAIL|. quickly final accounts among the 481|20646|647|2|19|29766.16|0.08|0.01|R|F|1993-01-09|1992-11-27|1993-01-14|TAKE BACK RETURN|AIR|p blithely after t 481|185785|5786|3|42|78572.76|0.08|0.08|A|F|1992-11-27|1992-11-11|1992-12-08|COLLECT COD|RAIL|mptotes are furiously among the iron 481|81009|6026|4|11|10890.00|0.05|0.06|A|F|1993-01-12|1992-11-17|1993-02-05|NONE|FOB|eful attai 481|111956|6979|5|31|61006.45|0.05|0.01|A|F|1993-01-15|1992-12-31|1993-01-21|DELIVER IN PERSON|AIR|usly final packages believe. quick 482|137343|7344|1|32|44170.88|0.00|0.02|N|O|1996-05-22|1996-05-14|1996-05-29|NONE|SHIP|usual deposits affix against 482|121382|8919|2|1|1403.38|0.05|0.08|N|O|1996-05-29|1996-05-20|1996-05-31|COLLECT COD|AIR|es. quickly ironic escapades sleep furious 482|61141|6154|3|31|34166.34|0.04|0.03|N|O|1996-06-01|1996-05-06|1996-06-17|NONE|MAIL| blithe pin 482|195826|5827|4|8|15374.56|0.02|0.05|N|O|1996-04-19|1996-05-05|1996-04-21|NONE|TRUCK|tructions near the final, regular ideas de 482|38215|3222|5|46|53047.66|0.01|0.06|N|O|1996-07-19|1996-06-05|1996-08-10|NONE|MAIL|furiously thin realms. final, fina 482|78696|8697|6|19|31819.11|0.04|0.00|N|O|1996-03-27|1996-04-25|1996-04-15|NONE|FOB|ts hinder carefully silent requests 483|32694|5198|1|8|13013.52|0.00|0.08|N|O|1995-08-22|1995-08-23|1995-09-18|COLLECT COD|RAIL|osits. carefully fin 483|79758|9759|2|23|39968.25|0.04|0.06|N|O|1995-07-20|1995-08-11|1995-08-04|DELIVER IN PERSON|MAIL|requests was quickly against th 483|87745|254|3|9|15594.66|0.04|0.03|N|O|1995-09-10|1995-09-02|1995-09-13|NONE|AIR| carefully express ins 484|30133|5140|1|49|52093.37|0.10|0.02|N|O|1997-03-06|1997-02-28|1997-03-23|COLLECT COD|TRUCK|ven accounts 484|31950|9460|2|45|84687.75|0.06|0.07|N|O|1997-04-09|1997-03-20|1997-04-19|DELIVER IN PERSON|TRUCK|usly final excuses boost slyly blithe 484|183351|5870|3|50|71717.50|0.06|0.05|N|O|1997-01-24|1997-03-27|1997-02-22|DELIVER IN PERSON|MAIL|uctions wake. final, silent requests haggle 484|164805|4806|4|22|41135.60|0.07|0.03|N|O|1997-04-29|1997-03-26|1997-05-17|TAKE BACK RETURN|SHIP|es are pending instructions. furiously unu 484|76308|1323|5|48|61646.40|0.00|0.05|N|O|1997-03-05|1997-02-08|1997-03-22|TAKE BACK RETURN|MAIL|l, bold packages? even mult 484|96871|9381|6|10|18678.70|0.01|0.08|N|O|1997-04-06|1997-02-14|1997-04-16|COLLECT COD|FOB|x fluffily carefully regular 485|149523|9524|1|50|78626.00|0.01|0.00|N|O|1997-03-28|1997-05-26|1997-04-18|TAKE BACK RETURN|MAIL|iously quick excuses. carefully final f 485|27973|2978|2|40|76038.80|0.08|0.01|N|O|1997-04-29|1997-05-08|1997-04-30|TAKE BACK RETURN|TRUCK|al escapades 485|136884|4424|3|22|42259.36|0.00|0.05|N|O|1997-04-06|1997-04-27|1997-05-01|DELIVER IN PERSON|TRUCK|refully final notornis haggle according 486|75437|5438|1|36|50847.48|0.00|0.01|N|O|1996-06-25|1996-05-06|1996-07-07|COLLECT COD|AIR|deposits around the quickly regular packa 486|67040|7041|2|40|40281.60|0.03|0.08|N|O|1996-05-21|1996-06-06|1996-06-07|COLLECT COD|SHIP|ts nag quickly among the slyl 486|135912|8426|3|26|50645.66|0.04|0.03|N|O|1996-03-16|1996-05-25|1996-03-31|NONE|RAIL|forges along the 486|71865|6880|4|38|69800.68|0.08|0.05|N|O|1996-05-07|1996-04-26|1996-05-26|TAKE BACK RETURN|TRUCK| blithely final pinto 486|28099|5606|5|3|3081.27|0.07|0.05|N|O|1996-07-07|1996-04-20|1996-07-23|DELIVER IN PERSON|RAIL|ccounts ha 486|46543|9048|6|46|68518.84|0.00|0.03|N|O|1996-04-18|1996-05-02|1996-04-20|COLLECT COD|AIR|theodolites eat carefully furious 487|91896|1897|1|47|88730.83|0.06|0.06|R|F|1992-09-30|1992-10-08|1992-10-24|NONE|TRUCK|tions. blithely reg 487|82099|2100|2|2|2162.18|0.02|0.06|R|F|1992-10-19|1992-11-04|1992-11-11|COLLECT COD|TRUCK|oss the unusual pinto beans. reg 512|188804|1323|1|19|35963.20|0.08|0.05|N|O|1995-07-12|1995-07-11|1995-08-04|COLLECT COD|MAIL| sleep. requests alongside of the fluff 512|22847|7852|2|37|65484.08|0.01|0.04|N|O|1995-06-20|1995-07-05|1995-07-16|NONE|RAIL|nic depths cajole? blithely b 512|179419|9420|3|40|59936.40|0.05|0.02|N|O|1995-07-06|1995-07-08|1995-07-08|COLLECT COD|TRUCK|quests are da 512|82470|7487|4|10|14524.70|0.09|0.02|N|O|1995-09-16|1995-07-29|1995-10-07|NONE|AIR|xes. pinto beans cajole carefully; 512|64154|4155|5|6|6708.90|0.03|0.05|R|F|1995-06-10|1995-06-21|1995-06-16|DELIVER IN PERSON|FOB|en ideas haggle 512|32014|4518|6|12|11352.12|0.04|0.00|R|F|1995-05-21|1995-08-03|1995-06-09|COLLECT COD|FOB|old furiously express deposits. specia 512|50769|3275|7|2|3439.52|0.09|0.08|N|O|1995-06-19|1995-08-13|1995-06-24|NONE|TRUCK|e slyly silent accounts serve with 513|61732|9251|1|20|33874.60|0.09|0.07|N|O|1995-07-12|1995-05-31|1995-07-31|NONE|AIR|efully ironic ideas doze slyl 513|121628|9165|2|44|72583.28|0.01|0.01|N|O|1995-07-14|1995-07-14|1995-08-12|NONE|MAIL|kages sleep boldly ironic theodolites. acco 514|78713|1221|1|21|35525.91|0.06|0.02|N|O|1996-06-09|1996-05-15|1996-07-07|DELIVER IN PERSON|RAIL|s sleep quickly blithely 514|117452|9964|2|34|49961.30|0.08|0.02|N|O|1996-04-14|1996-06-03|1996-04-23|COLLECT COD|REG AIR|ily even patterns. bold, silent instruc 514|12812|5314|3|6|10348.86|0.06|0.01|N|O|1996-05-30|1996-06-04|1996-06-28|COLLECT COD|SHIP|as haggle blithely; quickly s 514|115362|5363|4|43|59226.48|0.00|0.08|N|O|1996-06-07|1996-05-14|1996-07-01|TAKE BACK RETURN|FOB|thely regular 515|104014|6525|1|10|10180.10|0.03|0.02|A|F|1993-10-04|1993-11-03|1993-10-08|NONE|FOB|ar deposits th 515|147605|2634|2|38|62798.80|0.10|0.07|A|F|1993-09-19|1993-11-12|1993-10-03|DELIVER IN PERSON|SHIP|ays. furiously express requests haggle furi 515|182145|9700|3|11|13498.54|0.00|0.02|R|F|1993-09-04|1993-10-02|1993-09-05|DELIVER IN PERSON|FOB|ly pending accounts haggle blithel 515|108606|8607|4|34|54896.40|0.09|0.03|R|F|1993-10-03|1993-10-26|1993-10-15|DELIVER IN PERSON|REG AIR|ic dependencie 515|130881|3395|5|32|61180.16|0.01|0.07|R|F|1993-10-10|1993-10-08|1993-11-02|TAKE BACK RETURN|FOB|r sauternes boost. final theodolites wake a 515|108692|3713|6|25|42517.25|0.04|0.08|R|F|1993-11-14|1993-11-07|1993-12-03|DELIVER IN PERSON|MAIL|e packages engag 516|24974|9979|1|11|20888.67|0.01|0.06|N|O|1998-05-02|1998-05-23|1998-05-12|DELIVER IN PERSON|FOB|ongside of the blithely final reque 517|44551|4552|1|28|41875.40|0.03|0.02|N|O|1997-04-30|1997-05-18|1997-05-17|COLLECT COD|MAIL| requests. special, fi 517|155391|7907|2|15|21695.85|0.02|0.00|N|O|1997-04-09|1997-06-26|1997-05-01|NONE|TRUCK| slyly. express requests ar 517|40932|3437|3|9|16856.37|0.04|0.00|N|O|1997-05-03|1997-06-16|1997-05-24|COLLECT COD|SHIP| slyly stealthily express instructions. 517|132197|2198|4|11|13521.09|0.06|0.02|N|O|1997-06-20|1997-06-01|1997-06-27|NONE|REG AIR|ly throughout the fu 517|23349|8354|5|23|29263.82|0.00|0.01|N|O|1997-04-19|1997-05-07|1997-05-12|COLLECT COD|RAIL| kindle. furiously bold requests mus 518|164711|4712|1|30|53271.30|0.07|0.05|N|O|1998-02-18|1998-03-27|1998-03-16|COLLECT COD|TRUCK|slyly by the packages. carefull 518|83164|689|2|23|26384.68|0.05|0.07|N|O|1998-02-20|1998-05-05|1998-03-11|COLLECT COD|TRUCK| special requests. fluffily ironic re 518|133178|8205|3|12|14534.04|0.01|0.06|N|O|1998-03-08|1998-03-31|1998-04-06|NONE|AIR| packages thrash slyly 518|121990|1991|4|46|92551.54|0.07|0.02|N|O|1998-04-07|1998-04-17|1998-04-29|NONE|MAIL|. blithely even ideas cajole furiously. b 518|70019|20|5|16|15824.16|0.01|0.01|N|O|1998-03-15|1998-03-24|1998-04-08|NONE|MAIL|use quickly expre 518|196358|1397|6|39|56719.65|0.09|0.08|N|O|1998-02-26|1998-03-17|1998-03-21|DELIVER IN PERSON|FOB| the bold, special deposits are carefully 518|185956|8475|7|48|98013.60|0.03|0.07|N|O|1998-03-06|1998-04-22|1998-03-14|NONE|FOB| slyly final platelets; quickly even deposi 519|158970|4001|1|1|2028.97|0.07|0.07|N|O|1997-12-01|1998-01-26|1997-12-23|COLLECT COD|REG AIR|bold requests believe furiou 519|2946|2947|2|38|70259.72|0.05|0.08|N|O|1998-02-19|1997-12-15|1998-03-19|DELIVER IN PERSON|FOB|gular excuses detect quickly furiously 519|105900|921|3|19|36212.10|0.00|0.02|N|O|1998-01-09|1998-01-03|1998-02-06|COLLECT COD|AIR|asymptotes. p 519|46267|3780|4|27|32758.02|0.08|0.06|N|O|1997-11-20|1997-12-06|1997-12-16|DELIVER IN PERSON|REG AIR|le. even, final dependencies 519|9041|4042|5|13|12350.52|0.06|0.08|N|O|1998-02-06|1997-12-02|1998-03-03|TAKE BACK RETURN|TRUCK|c accounts wake along the ironic so 519|150926|5957|6|3|5930.76|0.04|0.00|N|O|1998-02-01|1998-01-25|1998-02-27|TAKE BACK RETURN|FOB|erve blithely blithely ironic asymp 544|138474|8475|1|47|71086.09|0.08|0.06|R|F|1993-03-14|1993-03-27|1993-03-27|COLLECT COD|SHIP|ecial pains. deposits grow foxes. 545|169547|9548|1|4|6466.16|0.02|0.00|N|O|1996-02-23|1995-12-16|1996-03-21|DELIVER IN PERSON|FOB|, ironic grouches cajole over 545|170188|5223|2|18|22647.24|0.00|0.00|N|O|1996-02-21|1996-01-17|1996-02-26|NONE|RAIL|al, final packages affix. even a 546|84585|2110|1|16|25113.28|0.08|0.02|N|O|1997-02-04|1996-12-30|1997-02-25|DELIVER IN PERSON|TRUCK|de of the orbits. sometimes regula 547|70789|5804|1|44|77430.32|0.08|0.08|N|O|1996-10-18|1996-08-17|1996-10-27|TAKE BACK RETURN|FOB|thely express dependencies. qu 547|136347|1374|2|48|66400.32|0.01|0.04|N|O|1996-10-21|1996-08-04|1996-11-20|COLLECT COD|SHIP|thely specia 547|181345|6382|3|3|4279.02|0.05|0.02|N|O|1996-09-04|1996-08-01|1996-09-21|COLLECT COD|SHIP|pinto beans. ironi 548|196550|6551|1|2|3293.10|0.06|0.05|A|F|1994-11-26|1994-11-06|1994-12-06|COLLECT COD|MAIL|ests haggle quickly eve 548|4641|4642|2|6|9273.84|0.00|0.08|A|F|1995-01-18|1994-12-08|1995-02-10|NONE|TRUCK|sits wake furiously regular 548|182|7683|3|21|22725.78|0.03|0.08|A|F|1995-01-13|1994-12-18|1995-01-25|NONE|AIR|ideas. special accounts above the furiou 548|56720|4236|4|21|35211.12|0.08|0.03|A|F|1994-10-27|1994-12-04|1994-11-21|DELIVER IN PERSON|AIR| engage quickly. regular theo 548|92995|523|5|19|37771.81|0.00|0.02|A|F|1994-09-24|1994-11-24|1994-10-01|DELIVER IN PERSON|MAIL|courts boost care 548|152753|7784|6|32|57784.00|0.06|0.04|A|F|1994-12-16|1994-11-20|1994-12-29|NONE|REG AIR|c instruction 549|195061|100|1|18|20809.08|0.07|0.04|R|F|1992-10-19|1992-08-12|1992-11-13|COLLECT COD|REG AIR|furiously according to the ironic, regular 549|188735|8736|2|38|69301.74|0.07|0.05|A|F|1992-08-17|1992-08-28|1992-09-05|COLLECT COD|RAIL|the regular, furious excuses. carefu 549|65213|5214|3|36|42415.56|0.08|0.04|R|F|1992-09-11|1992-10-11|1992-09-12|DELIVER IN PERSON|AIR|ts against the ironic, even theodolites eng 549|20101|7608|4|18|18379.80|0.09|0.01|A|F|1992-07-31|1992-09-11|1992-08-08|NONE|RAIL|ely regular accounts above the 549|23480|987|5|38|53332.24|0.06|0.02|R|F|1992-08-23|1992-08-12|1992-08-25|COLLECT COD|REG AIR|eposits. carefully regular depos 550|190307|2827|1|31|43316.30|0.04|0.02|N|O|1995-10-24|1995-09-27|1995-11-04|COLLECT COD|AIR|thely silent packages. unusual 551|23786|6289|1|8|13678.24|0.08|0.02|N|O|1995-07-29|1995-07-18|1995-08-02|NONE|REG AIR| wake quickly slyly pending platel 551|158813|3844|2|20|37436.20|0.00|0.07|N|O|1995-09-18|1995-08-25|1995-10-11|COLLECT COD|TRUCK|r ideas. final, even ideas hinder alongside 551|161089|6122|3|16|18401.28|0.07|0.06|N|O|1995-07-29|1995-08-19|1995-08-10|COLLECT COD|MAIL|y along the carefully ex 576|86490|1507|1|2|2952.98|0.07|0.01|N|O|1997-05-15|1997-06-30|1997-05-28|NONE|RAIL|ccounts along the ac 576|33096|8103|2|6|6174.54|0.06|0.05|N|O|1997-05-15|1997-07-26|1997-06-03|DELIVER IN PERSON|TRUCK|al deposits. slyly even sauternes a 576|36565|9069|3|6|9009.36|0.08|0.07|N|O|1997-08-28|1997-06-16|1997-09-25|DELIVER IN PERSON|FOB|ts. ironic multipliers 576|137608|2635|4|5|8228.00|0.03|0.07|N|O|1997-06-11|1997-06-17|1997-07-05|NONE|REG AIR|l foxes boost slyly. accounts af 577|25886|891|1|25|45297.00|0.06|0.01|A|F|1995-04-09|1995-02-20|1995-05-09|TAKE BACK RETURN|AIR|ve slyly of the frets. careful 577|63233|8246|2|14|16747.22|0.08|0.03|R|F|1995-03-19|1995-02-25|1995-04-09|DELIVER IN PERSON|RAIL|l accounts wake deposits. ironic packa 578|155542|5543|1|40|63901.60|0.02|0.08|N|O|1997-02-10|1997-03-18|1997-02-11|NONE|SHIP|usly even platel 578|187025|2062|2|23|25576.46|0.05|0.08|N|O|1997-03-06|1997-03-03|1997-03-20|TAKE BACK RETURN|FOB|nstructions. ironic deposits 579|150618|5649|1|9|15017.49|0.00|0.05|N|O|1998-06-20|1998-04-28|1998-07-19|DELIVER IN PERSON|RAIL|e ironic, express deposits are furiously 579|32145|7152|2|39|42008.46|0.02|0.01|N|O|1998-06-21|1998-06-03|1998-06-26|COLLECT COD|REG AIR|ncies. furiously final r 579|59048|4059|3|6|6042.24|0.03|0.00|N|O|1998-04-24|1998-05-03|1998-05-08|TAKE BACK RETURN|TRUCK|ickly final requests-- bold accou 579|6189|8690|4|41|44902.38|0.04|0.05|N|O|1998-05-28|1998-05-01|1998-06-04|COLLECT COD|REG AIR|bold, express requests sublate slyly. blith 579|12612|5114|5|28|42689.08|0.00|0.03|N|O|1998-07-10|1998-05-24|1998-07-19|NONE|RAIL|ic ideas until th 579|166717|9234|6|5|8918.55|0.05|0.08|N|O|1998-05-02|1998-04-25|1998-05-05|COLLECT COD|REG AIR|refully silent ideas cajole furious 580|84916|2441|1|33|62730.03|0.03|0.05|N|O|1997-10-11|1997-09-19|1997-10-16|TAKE BACK RETURN|FOB|y express theodolites cajole carefully 580|173320|3321|2|31|43192.92|0.04|0.08|N|O|1997-10-04|1997-09-08|1997-10-15|COLLECT COD|FOB|ose alongside of the sl 580|184444|6963|3|19|29040.36|0.04|0.04|N|O|1997-07-23|1997-09-21|1997-08-15|NONE|FOB|mong the special packag 581|63384|8397|1|41|55242.58|0.09|0.07|N|O|1997-05-26|1997-04-06|1997-06-10|TAKE BACK RETURN|MAIL|nts. quickly 581|92527|5037|2|14|21273.28|0.06|0.08|N|O|1997-05-17|1997-04-14|1997-06-08|NONE|MAIL|. deposits s 581|100106|5127|3|49|54198.90|0.10|0.02|N|O|1997-02-27|1997-04-24|1997-03-10|TAKE BACK RETURN|MAIL|. slyly regular pinto beans acr 581|74925|9940|4|30|56997.60|0.10|0.08|N|O|1997-06-19|1997-05-21|1997-06-22|TAKE BACK RETURN|TRUCK| regular ideas grow furio 582|56409|3925|1|7|9557.80|0.07|0.00|N|O|1997-11-16|1997-11-29|1997-12-10|TAKE BACK RETURN|FOB|ithely unusual t 582|50262|263|2|49|59400.74|0.05|0.03|N|O|1997-12-17|1998-01-12|1997-12-31|COLLECT COD|REG AIR|nts according to the furiously regular pin 582|140309|5338|3|42|56670.60|0.07|0.00|N|O|1997-11-15|1997-12-21|1997-12-03|COLLECT COD|SHIP|iously beside the silent de 582|167750|7751|4|36|65439.00|0.06|0.01|N|O|1997-12-09|1997-11-27|1997-12-26|TAKE BACK RETURN|SHIP|lar requests. quickly 583|144364|4365|1|1|1408.36|0.07|0.07|N|O|1997-06-17|1997-04-29|1997-06-28|NONE|TRUCK| regular, regular ideas. even, bra 583|119625|2137|2|47|77297.14|0.10|0.06|N|O|1997-07-14|1997-05-12|1997-08-11|DELIVER IN PERSON|AIR|nts are fluffily. furiously even re 583|129431|1944|3|34|49654.62|0.01|0.02|N|O|1997-05-11|1997-04-24|1997-06-03|DELIVER IN PERSON|MAIL|express req 583|141250|8793|4|33|42611.25|0.10|0.01|N|O|1997-05-28|1997-04-25|1997-06-24|NONE|AIR|kages cajole slyly across the 583|188537|6092|5|13|21131.89|0.04|0.06|N|O|1997-06-23|1997-05-29|1997-07-08|COLLECT COD|TRUCK|y sly theodolites. ironi 608|153579|1125|1|19|31018.83|0.08|0.06|N|O|1996-04-19|1996-05-02|1996-05-03|DELIVER IN PERSON|RAIL|ideas. the 608|197310|2349|2|40|56292.40|0.03|0.01|N|O|1996-05-21|1996-04-11|1996-06-02|NONE|AIR| alongside of the regular tithes. sly 609|65533|8040|1|21|31469.13|0.01|0.05|R|F|1994-08-24|1994-08-23|1994-08-27|DELIVER IN PERSON|FOB|de of the special warthogs. excu 610|110792|5815|1|49|88336.71|0.10|0.07|N|O|1995-08-29|1995-10-26|1995-09-12|TAKE BACK RETURN|SHIP|ular instruc 610|67896|5415|2|11|20502.79|0.07|0.08|N|O|1995-10-31|1995-10-25|1995-11-18|NONE|MAIL|blithely final 610|117617|7618|3|26|42499.86|0.09|0.04|N|O|1995-11-22|1995-09-09|1995-12-04|TAKE BACK RETURN|AIR|cross the furiously even theodolites sl 610|185206|7725|4|17|21950.40|0.03|0.03|N|O|1995-11-01|1995-10-30|1995-11-04|COLLECT COD|FOB|p quickly instead of the slyly pending foxe 610|145743|5744|5|39|69760.86|0.08|0.05|N|O|1995-10-30|1995-10-21|1995-11-11|TAKE BACK RETURN|REG AIR|counts. ironic warhorses are 610|94365|6875|6|5|6796.80|0.00|0.07|N|O|1995-08-11|1995-10-22|1995-08-26|TAKE BACK RETURN|FOB|n pinto beans. iro 610|189280|4317|7|27|36970.56|0.06|0.03|N|O|1995-09-02|1995-09-19|1995-09-15|NONE|REG AIR| ironic pinto beans haggle. blithe 611|16855|4359|1|39|69102.15|0.05|0.06|R|F|1993-05-06|1993-04-09|1993-05-22|TAKE BACK RETURN|SHIP|nto beans 611|80676|677|2|1|1656.67|0.08|0.07|R|F|1993-05-17|1993-02-26|1993-06-15|DELIVER IN PERSON|MAIL|ts. pending platelets aff 611|119545|2057|3|39|61017.06|0.09|0.02|A|F|1993-03-10|1993-03-10|1993-03-17|TAKE BACK RETURN|TRUCK|the evenly bold requests. furious 612|184959|9996|1|5|10219.75|0.07|0.00|R|F|1992-11-08|1992-11-20|1992-12-03|TAKE BACK RETURN|RAIL|structions. q 612|194864|7384|2|28|54848.08|0.07|0.06|R|F|1993-01-02|1992-12-11|1993-01-30|DELIVER IN PERSON|TRUCK|regular instructions affix bl 612|66130|1143|3|49|53710.37|0.00|0.08|A|F|1993-01-08|1992-11-25|1993-01-17|TAKE BACK RETURN|REG AIR|theodolite 612|38942|1446|4|28|52666.32|0.05|0.00|A|F|1992-11-12|1992-12-05|1992-12-02|TAKE BACK RETURN|REG AIR|lyly regular asym 612|87737|246|5|1|1724.73|0.08|0.04|R|F|1992-12-18|1992-12-13|1992-12-20|TAKE BACK RETURN|FOB| requests. 612|188203|5758|6|33|42609.60|0.10|0.03|R|F|1992-11-30|1992-12-01|1992-12-12|COLLECT COD|MAIL|bove the blithely even ideas. careful 613|90027|7555|1|17|17289.34|0.06|0.06|N|O|1995-09-23|1995-08-04|1995-10-15|NONE|SHIP|ar dependencie 613|78348|5870|2|6|7958.04|0.05|0.05|N|O|1995-08-05|1995-08-09|1995-08-08|TAKE BACK RETURN|MAIL|y ironic deposits eat 613|185016|7535|3|3|3303.03|0.03|0.01|N|O|1995-09-27|1995-09-11|1995-10-05|NONE|TRUCK|ccounts cajole. 613|158304|8305|4|7|9536.10|0.02|0.04|N|O|1995-09-07|1995-08-02|1995-09-16|DELIVER IN PERSON|MAIL|ously blithely final pinto beans. regula 614|194109|9148|1|21|25265.10|0.00|0.03|R|F|1993-03-29|1993-01-06|1993-04-16|TAKE BACK RETURN|TRUCK|arefully. slyly express packag 614|186897|9416|2|48|95226.72|0.07|0.07|A|F|1993-03-09|1993-01-19|1993-03-19|DELIVER IN PERSON|SHIP|riously special excuses haggle along the 614|166963|4512|3|43|87288.28|0.05|0.00|A|F|1993-03-07|1993-02-22|1993-03-18|DELIVER IN PERSON|SHIP| express accounts wake. slyly ironic ins 614|146951|4494|4|14|27971.30|0.04|0.06|A|F|1992-12-03|1993-02-14|1992-12-27|DELIVER IN PERSON|SHIP|ular packages haggle about the pack 614|195308|7828|5|30|42099.00|0.08|0.07|R|F|1993-01-16|1993-02-08|1993-02-12|TAKE BACK RETURN|FOB|tructions are f 614|136241|1268|6|48|61307.52|0.04|0.08|A|F|1992-12-14|1993-01-22|1993-01-11|NONE|TRUCK| regular platelets cajole quickly eve 615|104545|4546|1|36|55783.44|0.10|0.01|A|F|1992-06-01|1992-07-14|1992-06-27|NONE|FOB| packages. carefully final pinto bea 640|92997|525|1|49|97509.51|0.09|0.02|R|F|1993-03-27|1993-04-17|1993-04-15|NONE|RAIL|s haggle slyly 640|416|2917|2|40|52656.40|0.09|0.05|A|F|1993-05-11|1993-04-11|1993-05-15|COLLECT COD|TRUCK|oach according to the bol 640|179475|7027|3|22|34198.34|0.05|0.07|A|F|1993-05-07|1993-04-14|1993-05-21|TAKE BACK RETURN|TRUCK|osits across the slyly regular theodo 640|31474|1475|4|45|63246.15|0.07|0.07|R|F|1993-04-15|1993-04-23|1993-04-21|DELIVER IN PERSON|REG AIR|ong the qui 641|125192|2729|1|18|21909.42|0.01|0.08|R|F|1993-10-17|1993-10-11|1993-10-29|DELIVER IN PERSON|AIR|p blithely bold packages. quick 641|99477|1987|2|1|1476.47|0.09|0.02|R|F|1993-12-03|1993-10-28|1993-12-26|TAKE BACK RETURN|RAIL| nag across the regular foxes. 641|94311|6821|3|40|52212.40|0.05|0.06|R|F|1993-11-22|1993-10-20|1993-12-11|DELIVER IN PERSON|REG AIR|lets. furiously regular requests cajo 641|70043|5058|4|25|25326.00|0.03|0.02|A|F|1993-12-04|1993-11-18|1993-12-18|TAKE BACK RETURN|FOB|d, regular d 641|3794|8795|5|41|69609.39|0.07|0.04|R|F|1993-11-29|1993-10-27|1993-12-04|TAKE BACK RETURN|FOB| asymptotes are quickly. bol 642|53624|3625|1|26|41018.12|0.10|0.03|A|F|1994-04-16|1994-02-01|1994-04-27|COLLECT COD|REG AIR|quests according to the unu 643|12260|9764|1|28|32823.28|0.00|0.08|A|F|1995-04-13|1995-05-12|1995-04-14|TAKE BACK RETURN|TRUCK|ly regular requests nag sly 643|50168|169|2|48|53671.68|0.01|0.02|N|O|1995-07-10|1995-06-07|1995-08-01|NONE|FOB|ly ironic accounts 643|162447|4964|3|23|34717.12|0.05|0.03|N|O|1995-07-09|1995-05-18|1995-07-31|COLLECT COD|RAIL|sits are carefully according to the e 643|44743|2256|4|39|65821.86|0.08|0.04|A|F|1995-06-08|1995-06-16|1995-06-13|COLLECT COD|RAIL| the pains. carefully s 643|189459|4496|5|47|72777.15|0.10|0.03|R|F|1995-04-05|1995-06-14|1995-04-26|DELIVER IN PERSON|RAIL|y against 644|133143|5657|1|46|54102.44|0.02|0.01|A|F|1992-05-20|1992-06-14|1992-06-14|DELIVER IN PERSON|RAIL| special requests was sometimes expre 644|129821|7358|2|11|20359.02|0.05|0.02|A|F|1992-08-20|1992-07-21|1992-09-11|TAKE BACK RETURN|TRUCK|ealthy pinto beans use carefu 644|100047|5068|3|44|46069.76|0.04|0.04|R|F|1992-08-17|1992-07-26|1992-08-20|COLLECT COD|REG AIR|iously ironic pinto beans. bold packa 644|79744|7266|4|7|12066.18|0.01|0.02|A|F|1992-05-18|1992-07-01|1992-06-07|COLLECT COD|RAIL| regular requests are blithely. slyly 644|49295|9296|5|23|28618.67|0.02|0.04|R|F|1992-07-31|1992-07-28|1992-08-13|DELIVER IN PERSON|TRUCK|uctions nag quickly alongside of t 644|84932|2457|6|33|63258.69|0.00|0.07|R|F|1992-08-26|1992-07-27|1992-08-28|NONE|AIR|ages sleep. bold, bo 644|50239|2745|7|38|45190.74|0.08|0.06|R|F|1992-05-17|1992-07-10|1992-06-06|TAKE BACK RETURN|MAIL| packages. blithely slow accounts nag quic 645|159694|2210|1|33|57871.77|0.01|0.02|A|F|1994-12-09|1995-02-21|1995-01-03|NONE|TRUCK|heodolites b 645|169422|9423|2|47|70096.74|0.07|0.05|R|F|1995-02-16|1995-02-15|1995-02-25|COLLECT COD|TRUCK|hely regular instructions alon 645|69227|4240|3|46|55026.12|0.10|0.01|A|F|1995-01-04|1995-02-21|1995-01-21|COLLECT COD|REG AIR| regular dependencies across the speci 645|95402|421|4|49|68472.60|0.05|0.03|R|F|1995-01-24|1995-01-06|1995-02-17|NONE|TRUCK|y. slyly iron 645|4703|7204|5|43|69131.10|0.06|0.02|A|F|1995-02-12|1995-02-27|1995-03-06|TAKE BACK RETURN|REG AIR| furiously accounts. slyly 645|33631|8638|6|18|28163.34|0.10|0.08|A|F|1995-03-02|1995-02-08|1995-03-03|COLLECT COD|RAIL|ep. slyly even 645|27031|7032|7|9|8622.27|0.03|0.03|A|F|1994-12-25|1995-01-04|1995-01-15|COLLECT COD|REG AIR|special deposits. regular, final th 646|108975|6506|1|31|61503.07|0.00|0.05|R|F|1994-12-17|1995-02-16|1995-01-04|COLLECT COD|MAIL|ag furiousl 646|126723|6724|2|1|1749.72|0.07|0.01|A|F|1994-12-05|1995-01-07|1994-12-31|TAKE BACK RETURN|MAIL|t blithely regular deposits. quic 646|29744|4749|3|24|40169.76|0.06|0.02|A|F|1995-02-20|1994-12-30|1995-03-16|TAKE BACK RETURN|TRUCK|regular accounts haggle dog 646|98738|3757|4|34|59048.82|0.01|0.00|R|F|1994-12-28|1994-12-27|1994-12-31|COLLECT COD|SHIP|slow accounts. fluffily idle instructions 646|89173|4190|5|17|19756.89|0.04|0.01|A|F|1994-12-31|1994-12-26|1995-01-01|DELIVER IN PERSON|REG AIR|inal packages haggle carefully 646|114481|9504|6|40|59819.20|0.10|0.01|R|F|1995-01-01|1995-01-13|1995-01-11|COLLECT COD|TRUCK|ronic packages sleep across th 647|16310|8812|1|41|50278.71|0.08|0.08|N|O|1997-11-19|1997-09-24|1997-12-15|COLLECT COD|REG AIR|r instructions. quickly unusu 647|112177|7200|2|5|5945.85|0.10|0.00|N|O|1997-09-25|1997-09-22|1997-10-25|TAKE BACK RETURN|AIR|ly express packages haggle caref 647|152882|7913|3|15|29023.20|0.08|0.00|N|O|1997-09-23|1997-10-09|1997-10-21|NONE|MAIL|ve the even, bold foxes sleep 672|172190|2191|1|41|51749.79|0.06|0.06|R|F|1994-06-20|1994-07-03|1994-06-22|COLLECT COD|REG AIR| dependencies in 672|189656|9657|2|9|15710.85|0.03|0.04|R|F|1994-06-25|1994-06-06|1994-07-19|TAKE BACK RETURN|TRUCK|haggle carefully carefully reg 672|142390|9933|3|35|50133.65|0.02|0.01|R|F|1994-07-13|1994-06-04|1994-07-14|COLLECT COD|RAIL| dependencies haggle quickly. theo 673|70495|5510|1|22|32240.78|0.03|0.02|R|F|1994-03-15|1994-04-27|1994-03-29|TAKE BACK RETURN|TRUCK| the regular, even requests. carefully fin 674|101366|3877|1|23|31449.28|0.06|0.07|A|F|1992-10-25|1992-10-15|1992-11-03|COLLECT COD|SHIP|ve the quickly even deposits. blithe 674|58285|3296|2|4|4973.12|0.02|0.07|R|F|1992-10-05|1992-11-22|1992-10-22|NONE|RAIL|ly express pinto beans sleep car 675|156455|4001|1|1|1511.45|0.04|0.08|N|O|1997-11-27|1997-09-30|1997-12-12|DELIVER IN PERSON|REG AIR|ide of the slyly regular packages. unus 675|136633|4173|2|35|58437.05|0.08|0.07|N|O|1997-08-19|1997-10-16|1997-09-17|DELIVER IN PERSON|REG AIR|s. furiously expre 675|175802|8320|3|34|63845.20|0.10|0.04|N|O|1997-11-17|1997-10-07|1997-11-27|NONE|FOB|y final accounts unwind around the 675|99269|6797|4|15|19023.90|0.09|0.05|N|O|1997-10-18|1997-09-28|1997-11-13|COLLECT COD|TRUCK|posits after the furio 675|4669|7170|5|46|72388.36|0.09|0.05|N|O|1997-09-18|1997-10-14|1997-10-01|DELIVER IN PERSON|AIR| deposits along the express foxes 676|50972|8488|1|9|17306.73|0.09|0.02|N|O|1997-04-03|1997-02-02|1997-04-08|COLLECT COD|REG AIR|aintain sl 676|77668|5190|2|20|32913.20|0.07|0.07|N|O|1997-02-02|1997-02-01|1997-02-11|NONE|REG AIR|riously around the blithely 676|162330|2331|3|35|48731.55|0.05|0.01|N|O|1996-12-30|1997-01-13|1997-01-19|DELIVER IN PERSON|RAIL|into beans. blithe 676|72825|347|4|24|43147.68|0.01|0.06|N|O|1997-02-05|1997-01-16|1997-03-07|TAKE BACK RETURN|TRUCK|ress, regular dep 676|165127|2676|5|31|36955.72|0.01|0.06|N|O|1997-02-06|1997-02-28|1997-03-08|COLLECT COD|TRUCK|ial deposits cajo 676|75930|5931|6|33|62895.69|0.09|0.05|N|O|1997-03-02|1997-02-22|1997-03-19|TAKE BACK RETURN|TRUCK|as wake slyly furiously close pinto b 676|142123|7152|7|11|12816.32|0.07|0.02|N|O|1997-03-09|1997-03-06|1997-03-31|TAKE BACK RETURN|MAIL|he final acco 677|58986|1492|1|32|62239.36|0.04|0.08|R|F|1994-01-06|1994-01-31|1994-02-02|NONE|RAIL|slyly final 677|167361|7362|2|39|55706.04|0.00|0.07|R|F|1993-12-19|1994-02-11|1994-01-05|TAKE BACK RETURN|SHIP|ges. furiously regular packages use 677|23226|3227|3|46|52864.12|0.01|0.02|R|F|1993-12-02|1994-02-12|1993-12-06|COLLECT COD|RAIL|ng theodolites. furiously unusual theodo 677|147638|5181|4|1|1685.63|0.06|0.05|R|F|1993-12-01|1994-01-14|1993-12-26|DELIVER IN PERSON|MAIL|ly. regular 677|149613|7156|5|25|41565.25|0.00|0.05|A|F|1994-03-12|1994-02-02|1994-03-28|DELIVER IN PERSON|AIR| packages integrate blithely 678|145537|5538|1|20|31650.60|0.05|0.08|R|F|1993-06-21|1993-04-07|1993-07-10|TAKE BACK RETURN|MAIL|furiously express excuses. foxes eat fu 678|36553|4063|2|22|32770.10|0.01|0.02|A|F|1993-05-10|1993-04-29|1993-06-08|NONE|REG AIR|de of the carefully even requests. bl 678|142489|5004|3|16|24503.68|0.06|0.02|R|F|1993-03-20|1993-04-13|1993-04-16|DELIVER IN PERSON|REG AIR|equests cajole around the carefully regular 678|198067|8068|4|48|55922.88|0.08|0.08|R|F|1993-02-28|1993-04-04|1993-03-24|NONE|REG AIR|ithely. slyly express foxes 678|97451|7452|5|16|23175.20|0.06|0.04|R|F|1993-03-09|1993-04-18|1993-04-07|NONE|AIR| about the 678|42888|2889|6|11|20139.68|0.09|0.00|R|F|1993-04-28|1993-05-16|1993-05-11|COLLECT COD|TRUCK|ess deposits dazzle f 679|191759|1760|1|9|16656.75|0.09|0.00|N|O|1995-12-20|1996-01-27|1996-01-07|COLLECT COD|REG AIR|leep slyly. entici 704|189981|9982|1|40|82839.20|0.05|0.05|N|O|1997-01-30|1997-01-10|1997-02-20|COLLECT COD|AIR|ggle quickly. r 704|3839|3840|2|14|24399.62|0.07|0.08|N|O|1997-02-02|1996-12-26|1997-02-19|DELIVER IN PERSON|REG AIR|ve the quickly final forges. furiously p 705|188322|841|1|46|64874.72|0.05|0.06|N|O|1997-04-18|1997-05-06|1997-05-05|DELIVER IN PERSON|SHIP|ss deposits. ironic packa 705|116218|3752|2|35|43197.35|0.10|0.04|N|O|1997-03-25|1997-03-20|1997-04-23|TAKE BACK RETURN|FOB|carefully ironic accounts 706|196629|9149|1|23|39689.26|0.05|0.00|N|O|1995-12-06|1995-12-02|1995-12-16|COLLECT COD|SHIP|ckey players. requests above the 707|154736|4737|1|34|60884.82|0.01|0.02|R|F|1994-12-08|1995-01-15|1995-01-02|NONE|RAIL| dependencies 707|42642|5147|2|22|34862.08|0.00|0.06|A|F|1995-01-12|1994-12-28|1995-01-16|DELIVER IN PERSON|REG AIR| kindle ironically 708|123805|1342|1|3|5486.40|0.05|0.02|N|O|1998-10-09|1998-09-22|1998-11-07|COLLECT COD|FOB|e slyly pending foxes. 708|179124|9125|2|19|22859.28|0.06|0.00|N|O|1998-10-28|1998-09-23|1998-11-25|COLLECT COD|SHIP| requests. even, thin ideas 708|121298|8835|3|33|43536.57|0.09|0.06|N|O|1998-09-10|1998-09-20|1998-09-22|COLLECT COD|RAIL|s boost carefully ruthless theodolites. f 708|55176|5177|4|5|5655.85|0.07|0.07|N|O|1998-07-22|1998-08-15|1998-07-28|TAKE BACK RETURN|REG AIR|c pinto beans nag after the account 708|142490|33|5|36|55169.64|0.08|0.01|N|O|1998-07-16|1998-09-04|1998-08-11|NONE|SHIP|ests. even, regular hockey p 708|22352|9859|6|7|8920.45|0.10|0.03|N|O|1998-08-16|1998-08-15|1998-09-10|COLLECT COD|REG AIR|lly express ac 709|86203|1220|1|7|8324.40|0.00|0.00|N|O|1998-06-14|1998-06-08|1998-06-18|TAKE BACK RETURN|RAIL| special orbits cajole 709|197250|9770|2|15|20208.75|0.08|0.00|N|O|1998-07-10|1998-06-26|1998-08-09|NONE|RAIL|ily regular deposits. sauternes was accor 709|168496|1013|3|10|15644.90|0.01|0.02|N|O|1998-06-04|1998-06-30|1998-06-11|NONE|REG AIR|ts cajole boldly 709|107229|7230|4|40|49448.80|0.10|0.08|N|O|1998-08-12|1998-06-20|1998-08-20|DELIVER IN PERSON|RAIL|ggle fluffily carefully ironic 710|162111|9660|1|47|55136.17|0.06|0.08|A|F|1993-01-18|1993-03-24|1993-01-24|TAKE BACK RETURN|MAIL|usual ideas into th 710|192916|2917|2|38|76338.58|0.07|0.02|R|F|1993-04-18|1993-03-12|1993-05-15|COLLECT COD|FOB|sts boost fluffily aft 710|138984|6524|3|7|14160.86|0.04|0.06|R|F|1993-01-20|1993-03-28|1993-02-15|TAKE BACK RETURN|REG AIR|xpress, special ideas. bl 710|89308|9309|4|25|32432.50|0.00|0.05|R|F|1993-03-31|1993-02-05|1993-04-22|COLLECT COD|FOB|eas detect do 710|185454|491|5|12|18473.40|0.01|0.02|A|F|1993-02-18|1993-02-27|1993-03-07|DELIVER IN PERSON|MAIL|ions. slyly express theodolites al 710|113665|1199|6|21|35251.86|0.04|0.06|R|F|1993-03-22|1993-03-05|1993-03-27|DELIVER IN PERSON|SHIP|es. furiously p 710|159288|6834|7|46|61974.88|0.03|0.07|R|F|1993-04-16|1993-03-27|1993-05-05|COLLECT COD|MAIL|ges use; blithely pending excuses inte 711|145767|8282|1|2|3625.52|0.10|0.04|R|F|1993-12-01|1993-12-09|1993-12-16|DELIVER IN PERSON|REG AIR|ely across t 711|102501|7522|2|27|40594.50|0.00|0.08|A|F|1993-10-02|1993-10-26|1993-10-08|DELIVER IN PERSON|MAIL|slyly. ironic asy 711|127086|2111|3|46|51201.68|0.10|0.00|R|F|1993-12-26|1993-11-19|1994-01-21|TAKE BACK RETURN|MAIL|deposits. permanen 711|127682|7683|4|20|34193.60|0.09|0.00|A|F|1994-01-17|1993-11-10|1994-01-31|DELIVER IN PERSON|TRUCK|kly regular acco 736|157380|7381|1|46|66119.48|0.05|0.01|N|O|1998-07-16|1998-09-01|1998-08-09|NONE|AIR|uctions cajole 736|79423|9424|2|23|32255.66|0.02|0.05|N|O|1998-10-08|1998-08-27|1998-10-19|TAKE BACK RETURN|AIR|k accounts are carefully 736|56626|4142|3|13|20574.06|0.00|0.03|N|O|1998-08-16|1998-07-26|1998-08-19|DELIVER IN PERSON|FOB|st furiously among the 736|97851|5379|4|14|25883.90|0.06|0.04|N|O|1998-10-04|1998-08-14|1998-10-16|COLLECT COD|REG AIR|nstructions. 736|168002|3035|5|32|34240.00|0.04|0.03|N|O|1998-07-30|1998-08-22|1998-08-12|DELIVER IN PERSON|RAIL|iously final accoun 737|181815|9370|1|12|22761.72|0.01|0.01|R|F|1992-04-28|1992-06-30|1992-05-08|COLLECT COD|RAIL|posits after the slyly bold du 738|197203|2242|1|34|44206.80|0.00|0.06|R|F|1993-06-09|1993-04-15|1993-07-09|TAKE BACK RETURN|TRUCK|s against the ironic exc 738|187016|7017|2|4|4412.04|0.00|0.03|A|F|1993-06-20|1993-04-08|1993-07-09|NONE|AIR|ar packages. fluffily bo 738|169700|9701|3|23|40703.10|0.04|0.08|A|F|1993-03-17|1993-04-02|1993-04-05|TAKE BACK RETURN|SHIP|nic, final excuses promise quickly regula 738|140786|8329|4|12|21921.36|0.04|0.08|A|F|1993-06-16|1993-05-05|1993-06-22|NONE|SHIP|ove the slyly regular p 738|174837|9872|5|30|57354.90|0.02|0.00|A|F|1993-06-12|1993-05-29|1993-06-25|NONE|AIR|ecial instructions haggle blithely regula 739|84489|6998|1|28|41257.44|0.00|0.03|N|O|1998-06-03|1998-08-04|1998-06-29|TAKE BACK RETURN|RAIL|elets about the pe 739|3502|6003|2|50|70275.00|0.07|0.06|N|O|1998-08-26|1998-07-16|1998-09-02|COLLECT COD|MAIL|ndencies. blith 739|48733|3742|3|12|20180.76|0.05|0.00|N|O|1998-08-20|1998-07-24|1998-08-22|NONE|MAIL|le slyly along the close i 739|43470|983|4|47|66433.09|0.09|0.07|N|O|1998-08-12|1998-07-09|1998-08-28|NONE|REG AIR|deas according to the theodolites sn 739|187523|5078|5|30|48315.60|0.07|0.06|N|O|1998-06-19|1998-08-26|1998-07-02|DELIVER IN PERSON|REG AIR|above the even deposits. ironic requests 740|1206|8707|1|22|24358.40|0.10|0.02|N|O|1995-07-24|1995-09-11|1995-08-11|TAKE BACK RETURN|FOB|odolites cajole ironic, pending instruc 740|65211|2730|2|35|41167.35|0.00|0.00|N|O|1995-09-06|1995-08-22|1995-10-02|NONE|TRUCK|p quickly. fu 740|198037|8038|3|29|32915.87|0.06|0.05|N|O|1995-10-26|1995-09-17|1995-10-29|DELIVER IN PERSON|FOB|ntly bold pinto beans sleep quickl 741|186404|1441|1|25|37260.00|0.03|0.06|N|O|1998-07-15|1998-08-27|1998-08-12|DELIVER IN PERSON|MAIL|accounts. blithely bold pa 741|90396|5415|2|22|30500.58|0.09|0.01|N|O|1998-09-07|1998-09-28|1998-09-12|COLLECT COD|AIR|ven deposits about the regular, ironi 742|101309|1310|1|46|60273.80|0.04|0.08|A|F|1995-03-12|1995-03-20|1995-03-16|TAKE BACK RETURN|SHIP|e slyly bold deposits cajole according to 742|95395|7905|2|15|20855.85|0.08|0.05|A|F|1995-02-26|1995-03-20|1995-03-03|NONE|SHIP|blithely unusual pinto 742|101553|4064|3|24|37309.20|0.08|0.08|A|F|1995-02-12|1995-03-12|1995-02-14|DELIVER IN PERSON|SHIP|affix slyly. furiously i 742|191891|4411|4|16|31726.24|0.01|0.05|A|F|1995-01-15|1995-02-25|1995-01-24|COLLECT COD|AIR|eodolites haggle carefully regul 742|100006|2517|5|48|48288.00|0.09|0.08|R|F|1995-03-24|1995-01-23|1995-04-08|TAKE BACK RETURN|TRUCK| platelets 742|191966|9524|6|49|100840.04|0.02|0.07|A|F|1995-01-13|1995-02-13|1995-01-26|TAKE BACK RETURN|RAIL| carefully bold foxes sle 743|191460|6499|1|21|32580.66|0.01|0.04|N|O|1996-10-26|1996-11-05|1996-11-11|COLLECT COD|MAIL|d requests. packages afte 768|195813|5814|1|39|74443.59|0.06|0.08|N|O|1996-09-25|1996-10-27|1996-10-20|NONE|SHIP|out the ironic 768|17862|7863|2|2|3559.72|0.00|0.04|N|O|1996-11-13|1996-10-03|1996-11-25|DELIVER IN PERSON|SHIP|ular courts. slyly dogged accou 768|5964|965|3|30|56098.80|0.06|0.05|N|O|1996-09-22|1996-11-03|1996-10-13|NONE|MAIL| furiously fluffy pinto beans haggle along 768|24493|2000|4|37|52447.13|0.10|0.00|N|O|1996-10-02|1996-09-23|1996-10-14|TAKE BACK RETURN|REG AIR|ending requests across the quickly 768|46500|1509|5|47|67985.50|0.06|0.05|N|O|1996-11-28|1996-10-30|1996-12-12|NONE|TRUCK|foxes. slyly ironic deposits a 768|111635|6658|6|43|70805.09|0.10|0.06|N|O|1996-09-22|1996-11-03|1996-10-22|TAKE BACK RETURN|AIR|sual ideas wake quickly 768|48062|8063|7|33|33331.98|0.01|0.04|N|O|1996-09-06|1996-09-29|1996-10-01|COLLECT COD|RAIL|sly ironic instructions. excuses can hagg 769|175068|7586|1|36|41150.16|0.02|0.02|A|F|1993-10-01|1993-08-07|1993-10-15|NONE|AIR|es. furiously iro 769|159077|1593|2|4|4544.28|0.01|0.04|R|F|1993-06-25|1993-08-12|1993-07-15|DELIVER IN PERSON|FOB| ideas. even 770|180119|2638|1|39|46765.29|0.09|0.06|N|O|1998-07-19|1998-08-09|1998-08-04|NONE|REG AIR|osits. foxes cajole 770|53281|5787|2|25|30857.00|0.03|0.02|N|O|1998-05-26|1998-07-23|1998-06-04|TAKE BACK RETURN|AIR| deposits dazzle fluffily alongside of 771|6983|4484|1|12|22679.76|0.10|0.08|N|O|1995-07-18|1995-08-02|1995-08-07|COLLECT COD|TRUCK|carefully. pending in 771|160505|3022|2|38|59489.00|0.03|0.08|N|O|1995-07-22|1995-09-10|1995-07-29|TAKE BACK RETURN|REG AIR| quickly final requests are final packages. 771|6393|6394|3|14|18191.46|0.02|0.05|N|O|1995-07-31|1995-08-13|1995-08-07|DELIVER IN PERSON|AIR|r, final packages are slyly iro 771|41262|1263|4|7|8422.82|0.06|0.02|N|O|1995-06-18|1995-08-31|1995-06-20|NONE|REG AIR|theodolites after the fluffily express 771|77402|4924|5|13|17932.20|0.09|0.01|N|O|1995-08-10|1995-08-21|1995-08-30|NONE|FOB|packages affix slyly about the quickly 771|81921|9446|6|23|43767.16|0.08|0.03|N|O|1995-06-19|1995-09-07|1995-07-09|COLLECT COD|FOB|cajole besides the quickly ironic pin 772|52698|214|1|35|57774.15|0.10|0.06|R|F|1993-07-05|1993-06-05|1993-08-02|NONE|SHIP|kly thin packages wake slowly 772|83007|8024|2|10|9900.00|0.05|0.01|R|F|1993-05-20|1993-05-19|1993-06-15|DELIVER IN PERSON|MAIL| deposits cajole carefully instructions. t 772|85119|5120|3|35|38643.85|0.03|0.04|R|F|1993-04-18|1993-06-13|1993-05-01|COLLECT COD|MAIL|ng ideas. special packages haggle alon 772|179834|7386|4|10|19138.30|0.08|0.02|A|F|1993-05-17|1993-06-09|1993-05-29|COLLECT COD|AIR|o the furiously final deposits. furi 772|53649|3650|5|42|67310.88|0.02|0.07|A|F|1993-06-09|1993-07-16|1993-06-12|DELIVER IN PERSON|MAIL| express foxes abo 773|99317|9318|1|5|6581.55|0.06|0.04|A|F|1993-11-21|1993-12-19|1993-12-21|COLLECT COD|MAIL|ar requests. regular, thin packages u 773|10005|2507|2|31|28365.00|0.02|0.06|A|F|1993-12-30|1993-11-02|1994-01-01|TAKE BACK RETURN|MAIL|e slyly unusual deposit 773|150082|7628|3|39|44151.12|0.06|0.05|A|F|1994-01-04|1993-12-23|1994-01-26|DELIVER IN PERSON|FOB|quickly eve 773|28056|3061|4|28|27553.40|0.10|0.06|R|F|1994-01-19|1993-11-05|1994-01-23|NONE|TRUCK|he furiously slow deposits. 773|133007|3008|5|9|9360.00|0.09|0.02|R|F|1993-10-09|1993-12-25|1993-11-04|TAKE BACK RETURN|FOB|ent orbits haggle fluffily after the 773|39505|9506|6|43|62113.50|0.07|0.03|A|F|1993-11-06|1993-11-20|1993-11-08|TAKE BACK RETURN|SHIP|furiously bold dependencies. blithel 774|182652|2653|1|49|84997.85|0.08|0.03|N|O|1995-12-06|1996-01-07|1995-12-14|DELIVER IN PERSON|SHIP|ess accounts are carefully 774|16038|1041|2|3|2862.09|0.10|0.06|N|O|1996-02-13|1996-01-14|1996-03-04|COLLECT COD|FOB| slyly even courts nag blith 774|147809|5352|3|34|63131.20|0.02|0.07|N|O|1996-03-16|1996-01-03|1996-03-22|COLLECT COD|FOB|lar excuses are furiously final instr 774|14993|4994|4|8|15263.92|0.00|0.02|N|O|1996-01-24|1996-01-15|1996-02-13|COLLECT COD|RAIL|ully ironic requests c 774|176160|3712|5|44|54391.04|0.09|0.07|N|O|1996-02-29|1996-01-16|1996-03-06|NONE|REG AIR|s according to the deposits unwind ca 774|119813|9814|6|2|3665.62|0.07|0.03|N|O|1995-12-11|1996-02-10|1995-12-14|TAKE BACK RETURN|SHIP|accounts; slyly regular 775|31640|6647|1|16|25146.24|0.10|0.06|N|F|1995-05-23|1995-05-07|1995-06-19|NONE|TRUCK|un quickly slyly 775|173244|796|2|21|27662.04|0.01|0.06|R|F|1995-05-01|1995-06-02|1995-05-13|DELIVER IN PERSON|FOB| quickly sile 775|107093|4624|3|20|22001.80|0.01|0.08|N|F|1995-06-17|1995-05-22|1995-07-13|COLLECT COD|AIR|en dependencies nag slowly 800|71833|6848|1|38|68583.54|0.00|0.05|N|O|1998-07-21|1998-09-25|1998-08-07|TAKE BACK RETURN|TRUCK|according to the bold, final dependencies 800|84262|6771|2|21|26171.46|0.04|0.05|N|O|1998-07-23|1998-10-01|1998-08-20|TAKE BACK RETURN|RAIL|ckly even requests after the carefully r 800|175028|63|3|26|28678.52|0.01|0.02|N|O|1998-07-23|1998-10-08|1998-07-25|DELIVER IN PERSON|FOB|bove the pending requests. 801|5111|2612|1|13|13209.43|0.10|0.02|R|F|1992-04-25|1992-04-24|1992-05-16|COLLECT COD|RAIL|s are fluffily stealthily expres 801|94616|9635|2|21|33822.81|0.05|0.02|A|F|1992-03-14|1992-04-01|1992-04-05|COLLECT COD|AIR|wake silently furiously idle deposits. 801|2700|2701|3|21|33656.70|0.05|0.03|A|F|1992-04-25|1992-03-20|1992-05-04|COLLECT COD|REG AIR|cial, special packages. 801|163556|1105|4|12|19434.60|0.08|0.04|A|F|1992-06-06|1992-04-14|1992-06-12|TAKE BACK RETURN|RAIL|s. ironic pinto b 801|73144|666|5|45|50271.30|0.01|0.06|R|F|1992-03-22|1992-03-22|1992-03-25|COLLECT COD|REG AIR| even asymptotes 801|121868|4381|6|10|18898.60|0.08|0.01|A|F|1992-06-05|1992-05-15|1992-06-21|DELIVER IN PERSON|MAIL|al accounts. carefully regular foxes wake 801|25641|646|7|11|17233.04|0.01|0.03|A|F|1992-05-09|1992-04-19|1992-05-15|DELIVER IN PERSON|REG AIR|y special pinto beans cajole 802|142568|7597|1|40|64422.40|0.08|0.08|A|F|1995-01-07|1995-04-03|1995-01-23|DELIVER IN PERSON|RAIL|y bold accou 802|132751|2752|2|34|60647.50|0.08|0.06|A|F|1995-03-01|1995-03-15|1995-03-12|COLLECT COD|AIR|instructions cajole carefully. quietl 802|130920|921|3|44|85840.48|0.07|0.04|R|F|1995-01-09|1995-02-04|1995-01-18|TAKE BACK RETURN|SHIP|rmanently idly special requ 802|156598|1629|4|18|29782.62|0.09|0.02|R|F|1995-03-06|1995-02-07|1995-03-19|TAKE BACK RETURN|RAIL|y regular requests engage furiously final d 802|131708|1709|5|19|33054.30|0.08|0.06|A|F|1995-04-01|1995-02-20|1995-04-23|DELIVER IN PERSON|REG AIR|old, furious 803|53401|8412|1|8|10835.20|0.07|0.01|N|O|1997-08-04|1997-06-19|1997-08-12|NONE|SHIP|ronic theodo 803|98771|8772|2|21|37165.17|0.08|0.06|N|O|1997-08-25|1997-06-30|1997-09-10|TAKE BACK RETURN|AIR|ironic packages cajole slyly. un 804|125293|5294|1|30|39548.70|0.08|0.04|A|F|1993-03-29|1993-05-07|1993-04-14|COLLECT COD|REG AIR|ehind the quietly regular pac 804|198375|5933|2|2|2946.74|0.02|0.00|A|F|1993-06-23|1993-04-30|1993-06-25|NONE|TRUCK|slyly silent 804|75519|534|3|44|65758.44|0.04|0.05|R|F|1993-07-06|1993-04-13|1993-07-28|DELIVER IN PERSON|TRUCK|ly final deposits? special 804|37511|2518|4|21|30418.71|0.01|0.00|A|F|1993-04-12|1993-06-06|1993-04-20|DELIVER IN PERSON|RAIL|ular, ironic foxes. quickly even accounts 805|197826|346|1|25|48095.50|0.07|0.06|N|O|1995-08-05|1995-09-30|1995-08-06|NONE|AIR|ide of the pending, sly requests. quickly f 805|56623|9129|2|29|45808.98|0.07|0.01|N|O|1995-08-24|1995-08-15|1995-09-16|TAKE BACK RETURN|AIR|dolites according to the slyly f 805|46414|6415|3|12|16324.92|0.01|0.06|N|O|1995-07-13|1995-09-27|1995-08-02|TAKE BACK RETURN|REG AIR| regular foxes. furio 805|75351|7859|4|26|34485.10|0.08|0.07|N|O|1995-08-28|1995-09-24|1995-09-11|TAKE BACK RETURN|RAIL|. ironic deposits sleep across 806|104752|2283|1|1|1756.75|0.04|0.07|N|O|1996-07-14|1996-09-12|1996-07-25|COLLECT COD|RAIL|ar accounts? pending, pending foxes a 806|159749|4780|2|22|39792.28|0.08|0.02|N|O|1996-10-03|1996-08-11|1996-10-20|DELIVER IN PERSON|REG AIR|fily pending 806|90335|2845|3|4|5301.32|0.04|0.03|N|O|1996-08-09|1996-09-18|1996-08-13|COLLECT COD|TRUCK|eans. quickly ironic ideas 807|116568|4102|1|49|77643.44|0.00|0.00|R|F|1993-12-05|1994-01-13|1993-12-25|COLLECT COD|REG AIR| furiously according to the un 807|154875|9906|2|49|94563.63|0.01|0.06|A|F|1994-01-17|1994-01-24|1994-01-22|COLLECT COD|TRUCK|y regular requests haggle. 807|180228|7783|3|48|62794.56|0.07|0.07|A|F|1994-01-08|1994-02-02|1994-01-15|DELIVER IN PERSON|SHIP|kly across the f 807|79226|9227|4|10|12052.20|0.09|0.00|R|F|1994-01-19|1994-02-12|1994-01-28|NONE|TRUCK|furiously final depths sleep a 807|142418|7447|5|30|43812.30|0.02|0.01|R|F|1994-01-19|1994-01-09|1994-01-27|NONE|RAIL|cial accoun 807|11627|9131|6|11|16924.82|0.02|0.04|R|F|1994-03-25|1994-01-26|1994-04-14|NONE|FOB|unts above the slyly final ex 807|149|5150|7|19|19933.66|0.08|0.05|A|F|1994-02-10|1994-02-20|1994-03-06|NONE|SHIP|ns haggle quickly across the furi 832|102949|5460|1|45|87837.30|0.01|0.02|A|F|1992-05-08|1992-06-06|1992-06-04|COLLECT COD|MAIL|foxes engage slyly alon 832|47411|2420|2|24|32601.84|0.05|0.06|A|F|1992-06-15|1992-07-14|1992-06-17|NONE|TRUCK|ully. carefully speci 833|53556|3557|1|1|1509.55|0.04|0.04|R|F|1994-04-26|1994-04-05|1994-04-29|COLLECT COD|MAIL|ffily ironic theodolites 833|111833|4345|2|38|70103.54|0.05|0.05|A|F|1994-04-05|1994-04-21|1994-05-01|COLLECT COD|TRUCK| platelets promise furiously. 833|161174|8723|3|9|11116.53|0.05|0.07|A|F|1994-02-28|1994-04-26|1994-03-20|TAKE BACK RETURN|FOB|ecial, even requests. even, bold instructi 834|144119|6634|1|36|41871.96|0.06|0.04|R|F|1994-06-28|1994-07-25|1994-07-07|TAKE BACK RETURN|SHIP|ccounts haggle after the furiously 834|6951|1952|2|11|20437.45|0.03|0.00|A|F|1994-09-18|1994-08-03|1994-10-02|DELIVER IN PERSON|TRUCK|inst the regular packa 835|106431|1452|1|33|47435.19|0.09|0.06|N|O|1995-11-01|1995-12-02|1995-11-24|DELIVER IN PERSON|RAIL|onic instructions among the carefully iro 835|184562|9599|2|28|46103.68|0.02|0.02|N|O|1995-12-27|1995-12-11|1996-01-21|NONE|SHIP| fluffily furious pinto beans 836|187915|5470|1|6|12017.46|0.09|0.03|N|O|1996-12-09|1997-01-31|1996-12-29|COLLECT COD|TRUCK|fully bold theodolites are daringly across 836|83534|1059|2|18|27315.54|0.03|0.05|N|O|1997-02-27|1997-02-11|1997-03-22|NONE|REG AIR|y pending packages use alon 836|140671|3186|3|46|78736.82|0.05|0.07|N|O|1997-03-21|1997-02-06|1997-04-05|NONE|REG AIR|boldly final pinto beans haggle furiously 837|56565|9071|1|39|59340.84|0.03|0.08|A|F|1994-07-22|1994-08-10|1994-08-11|NONE|RAIL|ecial pinto bea 837|87341|4866|2|24|31880.16|0.08|0.00|R|F|1994-06-27|1994-09-02|1994-07-27|DELIVER IN PERSON|FOB|p carefully. theodolites use. bold courts a 838|133708|6222|1|20|34834.00|0.10|0.07|N|O|1998-04-11|1998-03-25|1998-04-19|COLLECT COD|TRUCK| furiously final ideas. slow, bold 838|28244|8245|2|27|31650.48|0.05|0.07|N|O|1998-02-15|1998-04-03|1998-02-20|DELIVER IN PERSON|SHIP| pending pinto beans haggle about t 838|94785|7295|3|23|40934.94|0.10|0.07|N|O|1998-03-26|1998-04-17|1998-04-02|COLLECT COD|AIR|ets haggle furiously furiously regular r 838|43152|3153|4|18|19712.70|0.09|0.00|N|O|1998-03-28|1998-04-06|1998-03-31|TAKE BACK RETURN|AIR|hely unusual foxes. furio 839|157077|4623|1|23|26083.61|0.07|0.02|N|O|1995-10-17|1995-11-03|1995-11-04|COLLECT COD|AIR|ng ideas haggle accord 839|188315|8316|2|47|65955.57|0.08|0.00|N|O|1995-10-17|1995-11-06|1995-11-10|NONE|AIR|refully final excuses about 864|129655|2168|1|34|57278.10|0.03|0.04|N|O|1997-12-16|1997-10-23|1998-01-12|TAKE BACK RETURN|SHIP|gside of the furiously special 864|97941|2960|2|7|13572.58|0.01|0.02|N|O|1997-11-13|1997-10-07|1997-12-13|TAKE BACK RETURN|MAIL|ven requests should sleep along 864|79304|1812|3|34|43632.20|0.03|0.00|N|O|1997-09-14|1997-11-04|1997-09-21|TAKE BACK RETURN|REG AIR|to the furiously ironic platelets! 865|197300|9820|1|16|22356.80|0.07|0.03|R|F|1993-08-24|1993-06-26|1993-08-28|TAKE BACK RETURN|TRUCK|y even accounts. quickly bold decoys 865|19188|4191|2|3|3321.54|0.02|0.05|A|F|1993-07-17|1993-07-14|1993-08-01|NONE|MAIL|fully regular the 865|86034|1051|3|15|15300.45|0.00|0.06|R|F|1993-07-05|1993-06-25|1993-07-26|NONE|SHIP| deposits sleep quickl 865|168398|5947|4|34|49857.26|0.09|0.06|A|F|1993-05-09|1993-07-28|1993-05-18|DELIVER IN PERSON|REG AIR|furiously fluffily unusual account 866|135703|5704|1|5|8693.50|0.08|0.00|R|F|1993-01-22|1993-01-14|1993-02-07|TAKE BACK RETURN|AIR|tegrate fluffily. carefully f 867|138798|8799|1|7|12857.53|0.04|0.07|A|F|1994-02-19|1993-12-25|1994-02-25|DELIVER IN PERSON|TRUCK|pendencies-- slyly unusual packages hagg 868|167908|7909|1|8|15807.20|0.06|0.03|R|F|1992-10-07|1992-08-01|1992-10-16|NONE|MAIL|l deposits. blithely regular pint 868|28659|3664|2|13|20639.45|0.05|0.07|R|F|1992-07-25|1992-08-26|1992-08-04|NONE|AIR|gged instructi 868|67432|2445|3|19|26589.17|0.09|0.06|R|F|1992-06-22|1992-08-27|1992-07-04|COLLECT COD|SHIP|lyly ironic platelets wake. rut 868|121059|6084|4|43|46442.15|0.02|0.04|A|F|1992-07-02|1992-07-22|1992-07-21|COLLECT COD|SHIP|kly silent deposits wake dar 868|24498|2005|5|27|38407.23|0.04|0.01|R|F|1992-08-01|1992-08-25|1992-08-12|TAKE BACK RETURN|RAIL|oss the fluffily unusual pinto 868|124220|4221|6|19|23640.18|0.02|0.05|R|F|1992-09-20|1992-07-18|1992-10-04|NONE|FOB|ely even deposits lose blithe 869|62275|4782|1|27|33406.29|0.07|0.07|N|O|1997-01-30|1997-02-17|1997-02-26|TAKE BACK RETURN|TRUCK|uffily even excuses? slyly even deposits 869|46068|8573|2|36|36506.16|0.04|0.01|N|O|1997-05-03|1997-03-17|1997-05-24|NONE|RAIL|ong the furiously bold instructi 870|49110|6623|1|36|38127.96|0.04|0.07|A|F|1993-10-18|1993-09-16|1993-11-15|COLLECT COD|MAIL|fily. furiously final accounts are 870|185850|5851|2|5|9679.25|0.06|0.05|A|F|1993-08-13|1993-09-11|1993-08-24|COLLECT COD|FOB|e slyly excuses. ironi 871|96392|6393|1|48|66642.72|0.10|0.03|N|O|1996-02-25|1996-02-09|1996-03-18|NONE|AIR|coys dazzle slyly slow notornis. f 871|54591|9602|2|47|72642.73|0.07|0.03|N|O|1995-12-25|1996-02-01|1996-01-24|TAKE BACK RETURN|RAIL|ss, final dep 871|107967|5498|3|13|25674.48|0.09|0.01|N|O|1996-01-25|1996-01-24|1996-02-03|NONE|REG AIR| haggle furiou 871|189917|9918|4|29|58200.39|0.06|0.07|N|O|1995-11-16|1996-01-27|1995-12-16|DELIVER IN PERSON|RAIL|ests are carefu 871|127480|2505|5|8|12059.84|0.00|0.01|N|O|1995-11-25|1996-01-12|1995-12-12|DELIVER IN PERSON|AIR|lar ideas-- slyly even accou 871|142499|42|6|26|40078.74|0.00|0.06|N|O|1996-02-07|1996-01-05|1996-02-25|COLLECT COD|AIR|symptotes use quickly near the 871|173371|8406|7|4|5777.48|0.00|0.07|N|O|1996-03-09|1996-01-20|1996-03-26|COLLECT COD|FOB|l, regular dependencies w 896|38675|8676|1|47|75842.49|0.07|0.08|R|F|1993-05-28|1993-05-15|1993-06-15|DELIVER IN PERSON|TRUCK|ly even pinto beans integrate. b 896|197621|5179|2|10|17186.20|0.03|0.07|A|F|1993-07-07|1993-06-03|1993-07-24|COLLECT COD|SHIP| quickly even theodolites. carefully regu 896|1925|9426|3|7|12788.44|0.09|0.02|A|F|1993-05-02|1993-05-24|1993-05-31|DELIVER IN PERSON|MAIL| requests 896|151663|1664|4|11|18861.26|0.08|0.04|A|F|1993-05-19|1993-05-22|1993-06-08|COLLECT COD|MAIL|the multipliers sleep 896|187481|5036|5|34|53328.32|0.00|0.05|R|F|1993-05-21|1993-06-01|1993-05-23|NONE|TRUCK|ular, close requests cajo 896|176862|1897|6|44|85309.84|0.09|0.08|R|F|1993-05-19|1993-04-14|1993-06-02|DELIVER IN PERSON|FOB|lar, pending packages. deposits are q 896|108889|1400|7|11|20876.68|0.01|0.07|A|F|1993-05-01|1993-04-09|1993-05-06|TAKE BACK RETURN|FOB|rding to the pinto beans wa 897|90450|5469|1|15|21606.75|0.07|0.04|R|F|1995-05-25|1995-05-09|1995-06-07|COLLECT COD|REG AIR|r ideas. slyly spec 897|183170|725|2|26|32582.42|0.05|0.08|N|O|1995-07-01|1995-06-10|1995-07-14|COLLECT COD|MAIL|tions sleep according to the special 897|125704|8217|3|13|22486.10|0.07|0.00|A|F|1995-03-30|1995-05-17|1995-04-21|TAKE BACK RETURN|MAIL|bold accounts mold carefully! braids 897|101182|6203|4|2|2366.36|0.08|0.08|R|F|1995-05-22|1995-05-07|1995-06-16|COLLECT COD|RAIL|into beans. slyly special fox 898|160159|160|1|9|10972.35|0.07|0.08|A|F|1993-07-04|1993-07-09|1993-07-25|NONE|AIR|e slyly across the blithe 898|178512|6064|2|37|58848.87|0.03|0.05|A|F|1993-08-17|1993-08-04|1993-09-01|DELIVER IN PERSON|REG AIR|packages sleep furiously 898|48242|5755|3|11|13092.64|0.01|0.00|A|F|1993-09-13|1993-08-31|1993-09-25|TAKE BACK RETURN|MAIL|etly bold accounts 898|192166|7205|4|36|45293.76|0.04|0.07|R|F|1993-08-04|1993-07-25|1993-08-23|DELIVER IN PERSON|REG AIR| after the carefully 899|60085|2592|1|18|18811.44|0.04|0.05|N|O|1998-08-06|1998-05-09|1998-09-05|DELIVER IN PERSON|AIR|re daring, pending deposits. blit 899|46070|8575|2|25|25401.75|0.00|0.07|N|O|1998-07-21|1998-05-12|1998-08-16|NONE|REG AIR|rly final sentiments. bold pinto beans 899|84976|2501|3|4|7843.88|0.09|0.05|N|O|1998-06-02|1998-06-28|1998-06-14|TAKE BACK RETURN|REG AIR|ter the carefully regular deposits are agai 899|179704|4739|4|14|24971.80|0.05|0.03|N|O|1998-05-21|1998-05-28|1998-06-03|TAKE BACK RETURN|FOB|ades impress carefully 899|70776|5791|5|4|6987.08|0.06|0.02|N|O|1998-04-11|1998-05-14|1998-04-27|NONE|TRUCK|ges. blithe, ironic waters cajole care 899|119083|1595|6|47|51797.76|0.00|0.04|N|O|1998-04-14|1998-05-30|1998-05-13|DELIVER IN PERSON|TRUCK|furiously final foxes after the s 899|13423|8426|7|11|14700.62|0.02|0.08|N|O|1998-06-03|1998-06-15|1998-06-20|COLLECT COD|REG AIR|t the ironic 900|198532|1052|1|44|71743.32|0.01|0.06|R|F|1994-12-15|1994-12-03|1994-12-27|COLLECT COD|MAIL| detect quick 900|114624|4625|2|48|78653.76|0.08|0.04|A|F|1994-12-22|1994-11-08|1995-01-19|COLLECT COD|TRUCK|cial pinto beans nag 900|74130|4131|3|24|26499.12|0.03|0.00|R|F|1994-10-21|1994-12-25|1994-10-22|TAKE BACK RETURN|SHIP|-ray furiously un 901|21113|3616|1|36|37227.96|0.01|0.01|N|O|1998-08-11|1998-10-09|1998-08-27|DELIVER IN PERSON|REG AIR|. accounts are care 901|45222|5223|2|2|2334.44|0.09|0.02|N|O|1998-10-25|1998-09-27|1998-11-01|DELIVER IN PERSON|AIR|d foxes use slyly 901|42615|5120|3|37|57631.57|0.04|0.08|N|O|1998-11-01|1998-09-13|1998-11-05|NONE|AIR|ickly final deposits 901|17758|7759|4|11|18433.25|0.00|0.06|N|O|1998-11-13|1998-10-19|1998-11-14|TAKE BACK RETURN|TRUCK|ourts among the quickly expre 902|110509|510|1|3|4558.50|0.06|0.00|R|F|1994-10-01|1994-10-25|1994-10-28|COLLECT COD|MAIL|into beans thrash blithely about the flu 902|117211|9723|2|8|9825.68|0.06|0.07|R|F|1994-10-25|1994-09-20|1994-11-07|COLLECT COD|RAIL| orbits al 902|164836|9869|3|24|45619.92|0.02|0.05|R|F|1994-11-08|1994-10-12|1994-11-26|NONE|FOB|. blithely even accounts poach furiously i 903|64367|1886|1|27|35946.72|0.04|0.03|N|O|1995-09-18|1995-09-20|1995-10-02|TAKE BACK RETURN|SHIP|lly pending foxes. furiously 903|8890|1391|2|35|62961.15|0.06|0.05|N|O|1995-09-18|1995-08-21|1995-10-12|TAKE BACK RETURN|TRUCK|rets wake fin 903|8238|739|3|33|37825.59|0.02|0.03|N|O|1995-09-24|1995-09-01|1995-10-12|COLLECT COD|MAIL|ely ironic packages wake blithely 903|55408|419|4|9|12270.60|0.09|0.00|N|O|1995-10-06|1995-09-14|1995-10-24|NONE|TRUCK|he slyly ev 903|41041|1042|5|1|982.04|0.04|0.00|N|O|1995-10-22|1995-09-13|1995-11-03|NONE|AIR|y final platelets sublate among the 903|167067|7068|6|13|14742.78|0.07|0.02|N|O|1995-09-11|1995-10-04|1995-10-03|COLLECT COD|SHIP|sleep along the final 928|168148|8149|1|29|35268.06|0.07|0.02|R|F|1995-05-17|1995-05-12|1995-05-21|NONE|REG AIR|ly alongside of the s 928|47775|5288|2|24|41346.48|0.05|0.08|A|F|1995-04-06|1995-05-08|1995-04-24|DELIVER IN PERSON|AIR|s the furiously regular warthogs im 928|151774|4290|3|46|83985.42|0.08|0.00|A|F|1995-05-09|1995-04-09|1995-06-01|DELIVER IN PERSON|REG AIR| beans sleep against the carefully ir 928|51313|8829|4|43|54365.33|0.10|0.05|A|F|1995-04-14|1995-04-21|1995-05-09|NONE|REG AIR|blithely. express, silent requests doze at 928|11725|1726|5|38|62195.36|0.02|0.08|N|F|1995-06-08|1995-04-15|1995-06-30|TAKE BACK RETURN|SHIP|xpress grouc 928|54038|4039|6|50|49601.50|0.05|0.00|N|F|1995-06-07|1995-04-15|1995-07-01|DELIVER IN PERSON|TRUCK| slyly slyly special request 928|10337|2839|7|11|13720.63|0.00|0.01|A|F|1995-04-29|1995-04-16|1995-04-30|NONE|AIR|longside of 929|128672|3697|1|45|76530.15|0.09|0.01|R|F|1993-01-24|1992-12-06|1993-02-16|DELIVER IN PERSON|REG AIR|ges haggle careful 929|174218|6736|2|44|56857.24|0.02|0.00|A|F|1992-10-09|1992-11-20|1992-10-22|DELIVER IN PERSON|SHIP|s. excuses cajole. carefully regu 929|73036|3037|3|14|14126.42|0.06|0.07|A|F|1992-10-21|1992-11-17|1992-11-15|NONE|FOB|gainst the 929|101639|4150|4|7|11484.41|0.06|0.01|A|F|1992-12-24|1992-12-19|1993-01-08|TAKE BACK RETURN|TRUCK|ithely. slyly c 930|44804|2317|1|36|62956.80|0.10|0.04|R|F|1994-12-21|1995-02-20|1994-12-24|COLLECT COD|RAIL|quickly regular pinto beans sle 930|17295|4799|2|47|56977.63|0.08|0.00|A|F|1995-03-20|1995-02-04|1995-04-04|DELIVER IN PERSON|AIR|ackages. fluffily e 930|64230|1749|3|10|11942.30|0.07|0.08|A|F|1994-12-18|1995-01-27|1995-01-16|COLLECT COD|AIR|ckly regular requests: regular instructions 930|99635|2145|4|21|34327.23|0.06|0.02|A|F|1995-02-16|1995-03-03|1995-03-13|DELIVER IN PERSON|SHIP|foxes. regular deposits integrate carefu 930|163239|788|5|50|65111.50|0.03|0.06|A|F|1995-04-03|1995-01-29|1995-04-22|COLLECT COD|MAIL| excuses among the furiously express ideas 930|144557|2100|6|10|16015.50|0.00|0.04|A|F|1995-02-09|1995-02-17|1995-02-16|NONE|SHIP|blithely bold i 930|166196|1229|7|30|37865.70|0.07|0.08|R|F|1995-01-20|1995-02-28|1995-02-04|TAKE BACK RETURN|RAIL|g accounts sleep along the platelets. 931|39783|9784|1|18|31010.04|0.00|0.05|A|F|1993-04-04|1993-01-11|1993-04-13|NONE|RAIL|slyly ironic re 931|16788|4292|2|10|17047.80|0.05|0.07|A|F|1993-03-01|1993-01-09|1993-03-07|TAKE BACK RETURN|SHIP|ajole quickly. slyly sil 931|146052|3595|3|48|52706.40|0.01|0.08|A|F|1993-02-03|1993-03-02|1993-02-09|TAKE BACK RETURN|FOB|ep alongside of the fluffy 931|81324|6341|4|38|49602.16|0.08|0.08|A|F|1993-03-06|1993-02-24|1993-03-27|DELIVER IN PERSON|RAIL|usly final packages integrate carefully 932|43763|6268|1|41|69977.16|0.01|0.05|N|O|1997-06-05|1997-07-22|1997-06-26|COLLECT COD|RAIL|foxes. ironic pl 933|48267|5780|1|23|27950.98|0.02|0.04|R|F|1992-08-13|1992-09-18|1992-08-25|DELIVER IN PERSON|MAIL| the furiously bold dinos. sly 933|12132|2133|2|27|28191.51|0.02|0.01|R|F|1992-10-03|1992-10-02|1992-10-26|DELIVER IN PERSON|RAIL|ests. express 933|99192|1702|3|26|30970.94|0.05|0.00|A|F|1992-11-09|1992-11-03|1992-11-16|DELIVER IN PERSON|AIR| the deposits affix slyly after t 934|117900|2923|1|18|34522.20|0.07|0.01|N|O|1996-09-10|1996-09-20|1996-09-25|COLLECT COD|RAIL|y unusual requests dazzle above t 935|27410|9913|1|23|30760.43|0.05|0.00|N|O|1997-11-11|1997-11-22|1997-11-29|COLLECT COD|REG AIR|ular accounts about 935|64553|2072|2|23|34903.65|0.02|0.01|N|O|1998-01-11|1997-11-25|1998-02-05|COLLECT COD|TRUCK|hes haggle furiously dolphins. qu 935|134031|6545|3|36|38341.08|0.06|0.00|N|O|1997-11-05|1997-12-05|1997-11-25|TAKE BACK RETURN|AIR|leep about the exp 935|57192|2203|4|13|14939.47|0.08|0.04|N|O|1998-01-13|1997-11-30|1998-02-08|DELIVER IN PERSON|TRUCK|ld platelet 935|12130|4632|5|8|8337.04|0.02|0.05|N|O|1998-01-12|1997-11-02|1998-02-05|NONE|TRUCK|cept the quickly regular p 935|58299|5815|6|1|1257.29|0.01|0.08|N|O|1997-12-14|1997-11-22|1998-01-08|DELIVER IN PERSON|TRUCK| instructions. ironic acc 960|106999|9510|1|1|2005.99|0.07|0.00|A|F|1994-12-24|1994-10-26|1995-01-20|DELIVER IN PERSON|AIR|y ironic packages. quickly even 960|116212|3746|2|25|30705.25|0.06|0.08|R|F|1994-12-01|1994-10-29|1994-12-27|DELIVER IN PERSON|RAIL|ts. fluffily regular requests 960|174776|2328|3|32|59224.64|0.01|0.08|R|F|1995-01-19|1994-12-17|1995-02-04|DELIVER IN PERSON|FOB|around the blithe, even pl 961|117921|2944|1|7|13572.44|0.10|0.00|N|O|1995-07-23|1995-07-20|1995-08-11|TAKE BACK RETURN|RAIL|usual dolphins. ironic pearls sleep blit 961|90469|470|2|18|26270.28|0.09|0.05|N|O|1995-07-01|1995-08-14|1995-07-04|DELIVER IN PERSON|AIR|rmanent foxes haggle speci 961|96938|6939|3|42|81267.06|0.06|0.01|N|O|1995-08-24|1995-08-21|1995-09-10|TAKE BACK RETURN|SHIP|ests do cajole blithely. furiously bo 961|33476|5980|4|29|40874.63|0.00|0.07|N|F|1995-06-10|1995-08-20|1995-06-26|TAKE BACK RETURN|TRUCK|l accounts use blithely against the 961|25276|5277|5|38|45648.26|0.03|0.05|N|O|1995-08-21|1995-07-19|1995-08-27|NONE|RAIL|he blithely special requests. furiousl 961|196988|6989|6|30|62549.40|0.09|0.03|N|O|1995-07-06|1995-07-20|1995-07-26|DELIVER IN PERSON|MAIL|warhorses slee 962|56598|6599|1|36|55965.24|0.01|0.03|R|F|1994-08-09|1994-07-10|1994-09-02|COLLECT COD|TRUCK|al foxes. iron 962|35276|7780|2|27|32704.29|0.09|0.02|A|F|1994-05-11|1994-07-10|1994-06-03|TAKE BACK RETURN|SHIP|y slyly express deposits. final i 962|79649|9650|3|3|4885.92|0.07|0.08|A|F|1994-05-08|1994-07-06|1994-06-02|DELIVER IN PERSON|FOB|ag furiously. even pa 962|56308|8814|4|20|25286.00|0.04|0.02|R|F|1994-08-26|1994-06-27|1994-09-11|DELIVER IN PERSON|SHIP| deposits use fluffily according to 962|151996|7027|5|12|24575.88|0.02|0.00|A|F|1994-06-09|1994-06-07|1994-06-11|COLLECT COD|TRUCK|across the furiously regular escapades daz 962|187354|2391|6|5|7206.75|0.02|0.05|A|F|1994-08-29|1994-07-15|1994-09-19|COLLECT COD|TRUCK|efully bold packages run slyly caref 963|193152|710|1|7|8716.05|0.01|0.00|R|F|1994-09-12|1994-07-18|1994-09-17|DELIVER IN PERSON|REG AIR|s. slyly regular depe 963|97483|9993|2|48|71063.04|0.10|0.06|R|F|1994-08-25|1994-08-12|1994-09-21|DELIVER IN PERSON|RAIL|ages. quickly express deposits cajole pe 964|198936|8937|1|39|79362.27|0.04|0.01|N|O|1995-06-21|1995-07-24|1995-06-24|NONE|AIR|se furiously regular instructions. blith 964|112231|2232|2|1|1243.23|0.02|0.05|N|O|1995-08-20|1995-07-29|1995-09-10|DELIVER IN PERSON|REG AIR|unts. quickly even platelets s 964|56790|9296|3|49|85592.71|0.01|0.03|N|O|1995-09-06|1995-08-10|1995-10-05|NONE|MAIL|ounts. blithely regular packag 964|54514|7020|4|44|64614.44|0.05|0.02|N|O|1995-09-18|1995-08-02|1995-10-17|TAKE BACK RETURN|TRUCK|ronic deposit 965|107207|9718|1|20|24284.00|0.04|0.05|N|F|1995-06-16|1995-07-20|1995-07-06|COLLECT COD|MAIL|kly. carefully pending requ 965|17015|2018|2|23|21436.23|0.06|0.08|N|O|1995-07-12|1995-07-08|1995-08-11|COLLECT COD|MAIL|ld kindle carefully across th 966|179337|6889|1|19|26910.27|0.07|0.01|N|O|1998-05-26|1998-07-15|1998-05-29|COLLECT COD|FOB|efully final pinto beans. quickly 966|116560|1583|2|42|66215.52|0.02|0.06|N|O|1998-06-28|1998-06-20|1998-07-05|NONE|TRUCK|tions boost furiously car 966|21218|6223|3|42|47846.82|0.06|0.08|N|O|1998-06-15|1998-06-08|1998-07-05|NONE|RAIL|sly ironic asymptotes hagg 966|4189|1690|4|20|21863.60|0.04|0.01|N|O|1998-07-19|1998-07-15|1998-07-27|NONE|TRUCK|pecial ins 967|58957|3968|1|41|78553.95|0.05|0.05|R|F|1992-09-21|1992-08-15|1992-10-21|NONE|MAIL|ld foxes wake closely special 967|84402|1927|2|4|5545.60|0.01|0.02|A|F|1992-07-15|1992-07-27|1992-07-18|DELIVER IN PERSON|REG AIR|platelets hang carefully along 967|131550|4064|3|10|15815.50|0.00|0.02|A|F|1992-09-18|1992-08-06|1992-09-19|DELIVER IN PERSON|MAIL|old pinto beans alongside of the exp 967|147690|5233|4|49|85146.81|0.01|0.04|A|F|1992-09-28|1992-09-15|1992-10-14|NONE|SHIP|the slyly even ideas. carefully even 967|16893|9395|5|41|74205.49|0.08|0.04|A|F|1992-07-23|1992-08-07|1992-08-13|TAKE BACK RETURN|FOB|efully special ide 967|105311|7822|6|17|22377.27|0.05|0.06|A|F|1992-10-02|1992-08-19|1992-10-25|NONE|MAIL|y ironic foxes caj 967|160878|5911|7|18|34899.66|0.00|0.02|A|F|1992-10-06|1992-08-05|1992-10-15|DELIVER IN PERSON|RAIL|ngage blith 992|59906|7422|1|14|26122.60|0.10|0.03|N|O|1998-01-29|1997-12-29|1998-02-18|TAKE BACK RETURN|MAIL|the unusual, even dependencies affix fluff 992|37967|7968|2|34|64768.64|0.02|0.00|N|O|1997-11-29|1998-01-21|1997-11-30|NONE|RAIL|s use silently. blithely regular ideas b 992|104722|4723|3|30|51801.60|0.10|0.00|N|O|1997-12-15|1998-02-02|1998-01-12|NONE|SHIP|nic instructions n 992|47212|9717|4|21|24343.41|0.06|0.06|N|O|1997-11-13|1997-12-28|1997-12-10|NONE|TRUCK|fily. quickly special deposit 992|91050|3560|5|7|7287.35|0.09|0.05|N|O|1997-11-30|1997-12-24|1997-12-16|DELIVER IN PERSON|TRUCK|ideas haggle. special theodolit 992|74101|1623|6|41|44079.10|0.10|0.01|N|O|1997-11-14|1998-02-04|1997-11-23|TAKE BACK RETURN|AIR|eodolites cajole across the accounts. 993|174476|6994|1|33|51165.51|0.01|0.05|N|O|1996-01-03|1995-11-28|1996-01-23|DELIVER IN PERSON|AIR| the deposits affix agains 993|2145|4646|2|28|29319.92|0.06|0.08|N|O|1995-10-24|1995-11-20|1995-11-06|DELIVER IN PERSON|RAIL|lites. even theodolite 993|39924|4931|3|10|18639.20|0.03|0.08|N|O|1995-12-17|1995-11-13|1995-12-20|NONE|RAIL|encies wake fur 993|190751|5790|4|40|73670.00|0.01|0.01|N|O|1995-11-16|1995-11-01|1995-12-05|TAKE BACK RETURN|RAIL|gle above the furiously 993|145484|5485|5|33|50472.84|0.09|0.08|N|O|1995-09-28|1995-10-24|1995-10-03|COLLECT COD|RAIL|fluffily. quiet excuses sleep furiously sly 993|136979|4519|6|35|70558.95|0.04|0.02|N|O|1995-10-26|1995-10-20|1995-11-05|DELIVER IN PERSON|FOB|es. ironic, ironic requests 993|4967|2468|7|15|28079.40|0.09|0.03|N|O|1995-09-27|1995-10-21|1995-10-17|DELIVER IN PERSON|MAIL|sits. pending pinto beans haggle? ca 994|64486|4487|1|4|5801.92|0.07|0.03|R|F|1994-07-05|1994-05-21|1994-07-20|COLLECT COD|SHIP|aggle carefully acc 994|9177|1678|2|11|11947.87|0.01|0.00|R|F|1994-05-03|1994-06-10|1994-05-22|NONE|AIR|ular accounts sleep 994|30745|8255|3|5|8378.70|0.08|0.08|A|F|1994-06-24|1994-06-14|1994-06-26|NONE|MAIL|ainst the pending requests. packages sl 994|130471|8011|4|25|37536.75|0.10|0.00|A|F|1994-06-03|1994-06-02|1994-06-06|COLLECT COD|RAIL|usual pinto beans. 995|172294|9846|1|15|20494.35|0.08|0.05|N|O|1995-06-30|1995-08-04|1995-07-27|NONE|REG AIR|uses. fluffily fina 995|128707|1220|2|28|48599.60|0.08|0.03|N|F|1995-06-12|1995-07-20|1995-06-19|DELIVER IN PERSON|SHIP|pades. quick, final frays use flu 995|165067|100|3|45|50942.70|0.00|0.05|N|O|1995-08-02|1995-07-21|1995-08-03|DELIVER IN PERSON|SHIP|lar packages detect blithely above t 995|65736|749|4|25|42543.25|0.01|0.08|N|O|1995-09-08|1995-08-05|1995-09-28|NONE|TRUCK|lyly even 995|23196|3197|5|18|20145.42|0.06|0.03|N|O|1995-07-03|1995-07-29|1995-07-22|TAKE BACK RETURN|AIR| even accounts unwind c 996|172396|7431|1|43|63140.77|0.03|0.06|N|O|1998-03-27|1998-03-25|1998-04-06|COLLECT COD|SHIP| the blithely ironic foxes. slyly silent d 997|162614|2615|1|11|18442.71|0.00|0.02|N|O|1997-06-16|1997-07-21|1997-07-14|DELIVER IN PERSON|TRUCK|p furiously according to t 997|47438|7439|2|17|23552.31|0.03|0.00|N|O|1997-07-28|1997-07-26|1997-08-20|DELIVER IN PERSON|SHIP|aggle quickly furiously 998|9924|7425|1|22|40346.24|0.04|0.05|A|F|1994-12-03|1995-02-17|1994-12-19|TAKE BACK RETURN|RAIL|lites. qui 998|180410|5447|2|7|10432.87|0.10|0.05|R|F|1995-03-24|1995-01-18|1995-04-03|NONE|MAIL|nic deposits. even asym 998|141752|4267|3|30|53812.50|0.05|0.07|A|F|1994-12-02|1995-01-23|1994-12-23|NONE|SHIP|lyly idle Tir 998|10143|5146|4|6|6318.84|0.09|0.05|R|F|1995-03-20|1994-12-27|1995-04-13|DELIVER IN PERSON|MAIL|refully accounts. carefully express ac 998|72407|7422|5|1|1379.40|0.04|0.00|R|F|1995-01-05|1995-01-06|1995-01-13|NONE|SHIP|es sleep. regular dependencies use bl 999|60071|7590|1|34|35056.38|0.00|0.08|R|F|1993-10-30|1993-10-17|1993-10-31|TAKE BACK RETURN|SHIP|its. daringly final instruc 999|198758|1278|2|41|76126.75|0.08|0.01|A|F|1993-10-16|1993-12-04|1993-11-03|DELIVER IN PERSON|REG AIR|us depths. carefully ironic instruc 999|117178|2201|3|15|17927.55|0.07|0.06|A|F|1993-12-12|1993-10-18|1994-01-08|COLLECT COD|REG AIR|y ironic requests. carefully regu 999|2643|2644|4|10|15456.40|0.05|0.02|A|F|1993-11-23|1993-12-02|1993-11-29|NONE|MAIL|efully pending 999|18337|8338|5|3|3765.99|0.03|0.00|R|F|1993-09-17|1993-10-22|1993-10-13|NONE|FOB|nic, pending ideas. bl 999|180934|8489|6|37|74552.41|0.00|0.04|R|F|1994-01-03|1993-10-28|1994-01-12|DELIVER IN PERSON|TRUCK|ckly slyly unusual packages: packages hagg 1024|198068|3107|1|49|57136.94|0.03|0.05|N|O|1998-03-06|1998-01-26|1998-03-29|TAKE BACK RETURN|FOB|ts. asymptotes nag fur 1024|125344|369|2|34|46557.56|0.00|0.01|N|O|1998-01-06|1998-02-05|1998-01-26|COLLECT COD|SHIP|des the slyly even 1024|43565|1078|3|28|42239.68|0.04|0.01|N|O|1998-03-04|1998-03-12|1998-03-15|TAKE BACK RETURN|TRUCK|e blithely regular pi 1024|183259|5778|4|13|17449.25|0.02|0.04|N|O|1998-04-11|1998-02-26|1998-04-18|NONE|FOB|e slyly around the slyly special instructi 1024|20829|8336|5|49|85741.18|0.02|0.04|N|O|1998-02-27|1998-03-10|1998-03-27|COLLECT COD|FOB| carefully bold 1025|149335|9336|1|36|49835.88|0.03|0.04|A|F|1995-05-15|1995-07-05|1995-06-10|COLLECT COD|FOB|e unusual, regular instr 1025|68247|8248|2|23|27950.52|0.08|0.03|N|F|1995-06-02|1995-07-29|1995-06-23|COLLECT COD|RAIL| regular platelets nag carefu 1025|22445|7450|3|25|34186.00|0.06|0.05|R|F|1995-05-29|1995-06-21|1995-06-13|DELIVER IN PERSON|REG AIR|xpress foxes. furiousl 1026|37213|4723|1|36|41407.56|0.10|0.02|N|O|1997-06-14|1997-07-20|1997-06-23|NONE|SHIP|st the ide 1026|36855|1862|2|6|10751.10|0.10|0.08|N|O|1997-07-07|1997-08-16|1997-07-14|TAKE BACK RETURN|TRUCK|to beans. special, regular packages hagg 1027|155942|973|1|43|85911.42|0.07|0.08|R|F|1992-06-17|1992-08-28|1992-07-10|DELIVER IN PERSON|MAIL|oxes. carefully regular deposits 1027|112414|7437|2|20|28528.20|0.01|0.02|A|F|1992-06-08|1992-08-29|1992-06-14|NONE|TRUCK|ar excuses eat f 1027|125692|3229|3|2|3435.38|0.01|0.02|R|F|1992-08-28|1992-07-09|1992-09-10|NONE|FOB|s. quickly unusual waters inside 1027|99229|6757|4|13|15966.86|0.08|0.01|R|F|1992-08-22|1992-07-10|1992-09-12|DELIVER IN PERSON|RAIL|ily ironic ideas use 1027|135427|7941|5|22|32173.24|0.02|0.00|A|F|1992-09-03|1992-08-14|1992-10-01|DELIVER IN PERSON|FOB|the furiously express ex 1027|104958|7469|6|10|19629.50|0.06|0.08|R|F|1992-08-28|1992-08-06|1992-09-03|COLLECT COD|REG AIR|ilent, express foxes near the blithely sp 1028|127811|324|1|2|3677.62|0.09|0.03|A|F|1994-01-10|1994-03-22|1994-01-26|COLLECT COD|FOB|s alongside of the regular asymptotes sleep 1028|111876|6899|2|39|73626.93|0.06|0.05|R|F|1994-02-18|1994-03-22|1994-03-06|TAKE BACK RETURN|MAIL| final dependencies affix a 1028|99556|4575|3|8|12444.40|0.03|0.07|A|F|1994-02-14|1994-03-28|1994-02-22|NONE|AIR|e carefully final packages. furiously fi 1028|31239|3743|4|26|30425.98|0.07|0.02|A|F|1994-03-18|1994-02-08|1994-03-19|TAKE BACK RETURN|RAIL|ronic platelets. carefully f 1028|28550|6057|5|27|39920.85|0.00|0.04|A|F|1994-04-03|1994-02-07|1994-04-26|NONE|REG AIR|ial accounts nag. slyly 1028|25111|7614|6|39|40408.29|0.03|0.02|A|F|1994-02-27|1994-02-16|1994-03-02|DELIVER IN PERSON|AIR|c theodoli 1028|30960|5967|7|22|41601.12|0.03|0.00|R|F|1994-04-24|1994-02-27|1994-05-08|NONE|REG AIR| Tiresias alongside of the carefully spec 1029|136137|8651|1|45|52790.85|0.05|0.07|R|F|1994-07-21|1994-08-30|1994-07-29|TAKE BACK RETURN|FOB|sits boost blithely 1030|64397|1916|1|17|23143.63|0.06|0.06|R|F|1994-10-13|1994-08-01|1994-11-10|DELIVER IN PERSON|RAIL|ly. carefully even packages dazz 1031|45551|5552|1|15|22448.25|0.10|0.08|A|F|1994-11-07|1994-10-29|1994-11-09|TAKE BACK RETURN|FOB|about the carefully bold a 1031|164788|7305|2|28|51877.84|0.05|0.01|A|F|1994-12-10|1994-10-29|1994-12-18|COLLECT COD|FOB|ly ironic accounts across the q 1031|186928|4483|3|27|54402.84|0.07|0.02|R|F|1994-09-20|1994-10-18|1994-10-10|DELIVER IN PERSON|SHIP|gular deposits cajole. blithely unus 1031|87154|2171|4|7|7988.05|0.03|0.03|R|F|1994-12-07|1994-11-11|1994-12-30|COLLECT COD|FOB|r instructions. car 1031|190193|7751|5|44|56460.36|0.01|0.07|R|F|1994-11-20|1994-11-24|1994-12-11|NONE|AIR|re slyly above the furio 1056|120970|3483|1|37|73665.89|0.04|0.06|R|F|1995-02-18|1995-04-01|1995-03-20|NONE|TRUCK| special packages. qui 1057|192240|4760|1|29|38634.96|0.10|0.01|A|F|1992-05-05|1992-05-05|1992-06-03|TAKE BACK RETURN|SHIP|es wake according to the q 1057|168078|595|2|11|12606.77|0.00|0.02|R|F|1992-03-31|1992-04-18|1992-04-18|COLLECT COD|AIR|yly final theodolites. furi 1057|84143|9160|3|21|23669.94|0.03|0.04|A|F|1992-02-28|1992-05-01|1992-03-10|NONE|REG AIR|ar orbits boost bli 1057|181895|6932|4|20|39537.80|0.06|0.03|R|F|1992-03-02|1992-05-19|1992-03-13|DELIVER IN PERSON|TRUCK|s wake bol 1057|96404|3932|5|7|9802.80|0.06|0.05|R|F|1992-06-05|1992-04-30|1992-06-20|NONE|TRUCK|y slyly express theodolites. slyly bo 1057|51461|6472|6|19|26836.74|0.04|0.07|A|F|1992-05-31|1992-05-09|1992-06-02|DELIVER IN PERSON|FOB|r-- packages haggle alon 1058|139555|7095|1|24|38269.20|0.08|0.04|A|F|1993-07-09|1993-05-28|1993-07-22|DELIVER IN PERSON|TRUCK|fully ironic accounts. express accou 1058|88088|5613|2|5|5380.40|0.04|0.07|R|F|1993-05-11|1993-05-29|1993-05-27|COLLECT COD|TRUCK|refully even requests boost along 1058|89249|9250|3|44|54482.56|0.10|0.01|R|F|1993-06-26|1993-06-21|1993-07-20|COLLECT COD|TRUCK|uriously f 1058|4119|1620|4|25|25577.75|0.09|0.01|A|F|1993-05-27|1993-06-10|1993-06-20|TAKE BACK RETURN|MAIL| the final requests believe carefully 1059|177583|7584|1|16|26569.28|0.07|0.02|A|F|1994-04-24|1994-03-31|1994-04-28|DELIVER IN PERSON|SHIP|y ironic pinto 1059|28839|6346|2|7|12374.81|0.07|0.06|R|F|1994-03-30|1994-04-01|1994-04-24|DELIVER IN PERSON|MAIL|the furiously silent excuses are e 1059|87935|444|3|45|86531.85|0.00|0.02|R|F|1994-06-10|1994-05-08|1994-06-21|COLLECT COD|RAIL|riously even theodolites. slyly regula 1059|109784|7315|4|26|46638.28|0.09|0.01|A|F|1994-03-17|1994-04-18|1994-03-26|DELIVER IN PERSON|TRUCK|ar pinto beans at the furiously 1059|138061|575|5|37|40665.22|0.09|0.04|R|F|1994-03-31|1994-05-08|1994-04-06|COLLECT COD|RAIL| packages lose in place of the slyly unusu 1059|189215|1734|6|50|65210.50|0.00|0.03|A|F|1994-06-15|1994-05-11|1994-06-29|NONE|MAIL|s impress furiously about 1059|122187|2188|7|13|15719.34|0.01|0.03|R|F|1994-06-12|1994-05-11|1994-07-02|COLLECT COD|TRUCK|usly regular theodo 1060|195479|3037|1|8|12595.76|0.07|0.04|R|F|1993-05-21|1993-05-06|1993-06-10|DELIVER IN PERSON|FOB|iously. furiously regular in 1060|7614|5115|2|26|39561.86|0.06|0.08|R|F|1993-04-12|1993-04-01|1993-04-20|DELIVER IN PERSON|TRUCK|counts; even deposits are carefull 1060|163141|5658|3|11|13245.54|0.01|0.07|A|F|1993-05-13|1993-05-08|1993-05-17|TAKE BACK RETURN|MAIL|e regular deposits: re 1060|109815|7346|4|16|29196.96|0.03|0.06|A|F|1993-06-15|1993-04-18|1993-07-05|COLLECT COD|SHIP|ccounts. foxes maintain care 1060|52852|7863|5|1|1804.85|0.04|0.06|A|F|1993-06-19|1993-05-10|1993-06-21|COLLECT COD|RAIL|posits detect carefully abo 1060|71383|3891|6|26|35213.88|0.01|0.03|A|F|1993-02-28|1993-04-01|1993-03-09|TAKE BACK RETURN|FOB|quickly abo 1060|120102|5127|7|36|40395.60|0.09|0.01|R|F|1993-03-14|1993-03-24|1993-04-02|TAKE BACK RETURN|FOB|r the quickly 1061|150016|5047|1|7|7462.07|0.04|0.04|N|O|1998-08-09|1998-08-12|1998-08-16|COLLECT COD|FOB|es are slyly expr 1061|118529|1041|2|2|3095.04|0.06|0.02|N|O|1998-08-15|1998-08-05|1998-08-24|COLLECT COD|MAIL|. regular accounts impre 1061|110165|5188|3|26|30554.16|0.08|0.02|N|O|1998-06-18|1998-07-25|1998-06-24|TAKE BACK RETURN|AIR|ave to slee 1061|135309|5310|4|41|55116.30|0.00|0.05|N|O|1998-06-29|1998-07-02|1998-07-27|NONE|MAIL|s are. ironic theodolites cajole. dep 1061|130014|15|5|50|52200.50|0.04|0.08|N|O|1998-05-25|1998-07-22|1998-06-22|COLLECT COD|AIR|nding excuses are around the e 1061|143492|6007|6|35|53742.15|0.09|0.05|N|O|1998-07-05|1998-07-07|1998-07-30|TAKE BACK RETURN|MAIL|ending requests nag careful 1062|136066|6067|1|38|41878.28|0.00|0.01|N|O|1997-01-27|1997-03-07|1997-02-16|DELIVER IN PERSON|TRUCK|deas. pending acc 1063|95042|61|1|42|43555.68|0.03|0.02|A|F|1994-07-10|1994-05-25|1994-07-26|NONE|RAIL|tructions about the blithely ex 1088|106899|6900|1|30|57176.70|0.07|0.03|A|F|1992-05-22|1992-06-25|1992-06-11|TAKE BACK RETURN|SHIP|long the packages snooze careful 1088|36841|4351|2|11|19556.24|0.06|0.00|A|F|1992-08-30|1992-07-25|1992-09-10|TAKE BACK RETURN|AIR|inal requests. fluffily express theod 1088|180349|2868|3|5|7146.70|0.03|0.07|A|F|1992-07-01|1992-07-25|1992-07-02|NONE|AIR|refully ironic packages. r 1088|123962|3963|4|3|5957.88|0.09|0.03|A|F|1992-06-15|1992-08-02|1992-06-18|DELIVER IN PERSON|MAIL|pecial theodolites 1089|150036|7582|1|47|51043.41|0.05|0.06|N|O|1996-06-26|1996-06-25|1996-07-11|NONE|TRUCK|aggle furiously among the bravely eve 1089|49934|2439|2|35|65937.55|0.03|0.00|N|O|1996-08-14|1996-07-10|1996-08-26|NONE|TRUCK|ly express deposits haggle 1089|25189|5190|3|23|25626.14|0.10|0.05|N|O|1996-06-24|1996-07-25|1996-07-20|DELIVER IN PERSON|AIR|g dolphins. deposits integrate. s 1089|140435|7978|4|1|1475.43|0.01|0.03|N|O|1996-07-08|1996-07-07|1996-07-17|COLLECT COD|RAIL|n courts among the caref 1090|21522|1523|1|5|7217.60|0.02|0.05|N|O|1998-02-19|1997-12-25|1998-02-24|DELIVER IN PERSON|AIR|s above the 1090|112475|7498|2|28|41649.16|0.08|0.08|N|O|1998-02-20|1998-01-03|1998-03-19|NONE|FOB|s cajole above the regular 1091|37233|7234|1|40|46809.20|0.10|0.06|N|O|1996-12-17|1996-10-14|1996-12-24|TAKE BACK RETURN|REG AIR|platelets. regular packag 1092|183493|3494|1|48|75671.52|0.04|0.04|N|O|1995-06-25|1995-04-06|1995-07-18|DELIVER IN PERSON|AIR|unusual accounts. fluffi 1092|152637|183|2|1|1689.63|0.01|0.06|A|F|1995-03-10|1995-04-21|1995-04-06|COLLECT COD|RAIL|lent, pending requests-- requests nag accor 1092|160701|5734|3|28|49327.60|0.05|0.08|R|F|1995-04-08|1995-05-01|1995-05-02|DELIVER IN PERSON|FOB|affix carefully. u 1092|85157|5158|4|2|2284.30|0.05|0.07|R|F|1995-04-09|1995-05-12|1995-05-03|TAKE BACK RETURN|TRUCK|ans. slyly eve 1093|86927|1944|1|7|13397.44|0.04|0.02|N|O|1997-11-24|1997-09-23|1997-11-25|TAKE BACK RETURN|SHIP|bold deposits. blithely ironic depos 1093|176300|3852|2|37|50923.10|0.08|0.04|N|O|1997-11-06|1997-10-08|1997-11-22|COLLECT COD|FOB|le furiously across the carefully sp 1093|60434|435|3|34|47410.62|0.01|0.06|N|O|1997-11-07|1997-09-06|1997-11-28|TAKE BACK RETURN|REG AIR|sits. express accounts play carefully. bol 1094|114492|4493|1|9|13558.41|0.07|0.06|N|O|1997-12-28|1998-03-16|1998-01-18|DELIVER IN PERSON|AIR|as. slyly pe 1095|136083|8597|1|33|36929.64|0.01|0.02|N|O|1995-10-03|1995-09-22|1995-10-13|NONE|MAIL|slyly around the iron 1095|135953|3493|2|24|47734.80|0.04|0.06|N|O|1995-08-24|1995-10-20|1995-09-09|COLLECT COD|TRUCK|packages nod furiously above the carefully 1095|155206|7722|3|13|16395.60|0.06|0.01|N|O|1995-08-24|1995-10-19|1995-09-02|TAKE BACK RETURN|REG AIR|ously even accounts. slyly bold a 1095|134380|1920|4|28|39602.64|0.08|0.03|N|O|1995-09-20|1995-11-18|1995-10-02|DELIVER IN PERSON|SHIP| regular pac 1095|111798|9332|5|40|72391.60|0.09|0.03|N|O|1995-10-18|1995-11-14|1995-11-09|NONE|MAIL| bold accounts haggle slyly furiously even 1095|180389|2908|6|37|54367.06|0.07|0.08|N|O|1995-10-04|1995-11-13|1995-10-12|NONE|SHIP|. quickly even dolphins sle 1120|177951|469|1|10|20289.50|0.08|0.05|N|O|1997-12-17|1998-01-21|1997-12-23|DELIVER IN PERSON|MAIL|dependencies. blithel 1120|19045|9046|2|49|47237.96|0.01|0.07|N|O|1998-01-03|1998-02-02|1998-01-09|TAKE BACK RETURN|RAIL|heodolites. quick re 1120|75472|7980|3|21|30396.87|0.06|0.01|N|O|1998-01-11|1998-02-04|1998-01-19|COLLECT COD|REG AIR|s: fluffily even packages c 1120|45653|662|4|22|35170.30|0.09|0.08|N|O|1997-11-15|1998-01-25|1997-12-07|TAKE BACK RETURN|REG AIR|ons. slyly silent requests sleep silent 1120|82435|2436|5|10|14174.30|0.07|0.08|N|O|1997-11-10|1998-02-01|1997-11-28|TAKE BACK RETURN|AIR|ages haggle furiously 1121|167726|5275|1|42|75336.24|0.04|0.05|N|O|1997-03-05|1997-03-18|1997-03-14|DELIVER IN PERSON|SHIP|nts are slyly special packages. f 1121|160276|2793|2|27|36079.29|0.08|0.00|N|O|1997-05-08|1997-03-28|1997-05-14|NONE|MAIL|ly ironic accounts cajole slyly abou 1121|156826|9342|3|10|18828.20|0.00|0.04|N|O|1997-04-17|1997-03-18|1997-05-02|TAKE BACK RETURN|RAIL|dencies. quickly regular theodolites n 1121|165255|2804|4|29|38287.25|0.02|0.01|N|O|1997-03-07|1997-04-02|1997-04-01|DELIVER IN PERSON|REG AIR| use furiously. quickly silent package 1121|29726|4731|5|47|77818.84|0.09|0.03|N|O|1997-04-27|1997-03-28|1997-05-14|COLLECT COD|FOB|ly idle, i 1121|199422|9423|6|50|76071.00|0.06|0.03|N|O|1997-04-21|1997-02-16|1997-04-25|NONE|TRUCK|odolites. slyly even accounts 1121|79080|6602|7|37|39185.96|0.06|0.01|N|O|1997-02-27|1997-03-04|1997-03-02|COLLECT COD|RAIL|special packages. fluffily final requests s 1122|91176|8704|1|8|9337.36|0.10|0.06|N|O|1997-02-02|1997-04-03|1997-02-22|TAKE BACK RETURN|RAIL|c foxes are along the slyly r 1122|181767|6804|2|29|53614.04|0.05|0.04|N|O|1997-05-07|1997-04-07|1997-05-15|COLLECT COD|SHIP|ptotes. quickl 1122|146422|3965|3|25|36710.50|0.09|0.01|N|O|1997-03-21|1997-03-03|1997-04-07|TAKE BACK RETURN|RAIL|d furiously. pinto 1122|105680|8191|4|40|67427.20|0.08|0.08|N|O|1997-02-07|1997-03-25|1997-02-25|NONE|REG AIR|packages sleep after the asym 1122|150352|353|5|15|21035.25|0.05|0.03|N|O|1997-04-15|1997-03-15|1997-05-07|COLLECT COD|SHIP|olve blithely regular, 1122|161814|9363|6|24|45019.44|0.04|0.01|N|O|1997-03-08|1997-02-20|1997-04-05|NONE|RAIL|blithely requests. slyly pending r 1122|299|5300|7|38|45573.02|0.00|0.08|N|O|1997-01-23|1997-04-02|1997-02-16|NONE|TRUCK|t theodolites sleep. even, ironic 1123|11173|8677|1|10|10841.70|0.05|0.08|N|O|1996-11-12|1996-10-04|1996-11-30|NONE|MAIL|ckages are above the depths. slyly ir 1123|177599|117|2|39|65387.01|0.03|0.08|N|O|1996-08-25|1996-10-21|1996-09-04|DELIVER IN PERSON|REG AIR|rding to the furiously ironic requests: r 1123|100807|3318|3|38|68696.40|0.03|0.08|N|O|1996-09-23|1996-10-04|1996-09-27|DELIVER IN PERSON|FOB| blithely carefully unusual reques 1124|197704|5262|1|1|1801.70|0.09|0.08|N|O|1998-10-06|1998-10-02|1998-10-30|NONE|REG AIR| instructions cajole qu 1124|5301|302|2|13|15681.90|0.05|0.04|N|O|1998-09-05|1998-10-03|1998-09-30|DELIVER IN PERSON|SHIP|t the slyly 1124|92298|4808|3|35|45160.15|0.10|0.05|N|O|1998-11-25|1998-10-08|1998-12-25|TAKE BACK RETURN|AIR|ut the slyly bold pinto beans; fi 1124|49104|9105|4|25|26327.50|0.08|0.05|N|O|1998-08-05|1998-10-14|1998-08-11|NONE|MAIL|ggle slyly according 1124|74888|7396|5|33|61475.04|0.05|0.04|N|O|1998-10-19|1998-09-17|1998-10-26|TAKE BACK RETURN|SHIP|eposits sleep slyly. stealthily f 1124|26745|1750|6|43|71884.82|0.01|0.03|N|O|1998-09-19|1998-10-28|1998-10-10|COLLECT COD|MAIL|across the 1124|94195|4196|7|1|1189.19|0.09|0.01|N|O|1998-10-07|1998-08-31|1998-10-12|NONE|TRUCK|ly bold accou 1125|132073|7100|1|4|4420.28|0.08|0.02|A|F|1994-12-10|1994-12-28|1994-12-30|NONE|MAIL| quickly express packages a 1125|137772|7773|2|24|43434.48|0.10|0.03|R|F|1995-01-31|1994-12-02|1995-02-20|COLLECT COD|AIR|es about the slyly s 1125|121039|3552|3|26|27560.78|0.05|0.04|A|F|1995-02-24|1995-01-18|1995-03-05|COLLECT COD|TRUCK|l instruction 1125|97823|2842|4|29|52803.78|0.06|0.00|A|F|1994-11-29|1994-12-20|1994-12-10|DELIVER IN PERSON|RAIL| platelets wake against the carefully i 1126|35025|2535|1|44|42240.88|0.08|0.03|N|O|1998-05-07|1998-04-02|1998-05-29|NONE|TRUCK|es. carefully special 1126|57176|2187|2|7|7932.19|0.06|0.01|N|O|1998-05-02|1998-03-22|1998-05-21|COLLECT COD|MAIL|ons. final, unusual 1126|146694|1723|3|14|24369.66|0.07|0.07|N|O|1998-04-17|1998-04-15|1998-05-12|DELIVER IN PERSON|TRUCK|nstructions. blithe 1127|42596|5101|1|35|53850.65|0.02|0.03|N|O|1995-11-25|1995-11-03|1995-12-17|NONE|TRUCK|l instructions boost blithely according 1127|109765|4786|2|38|67440.88|0.09|0.05|N|O|1995-11-07|1995-11-11|1995-11-26|DELIVER IN PERSON|RAIL|. never final packages boost acro 1127|19646|9647|3|29|45403.56|0.09|0.07|N|O|1995-09-20|1995-11-21|1995-10-11|DELIVER IN PERSON|REG AIR|y. blithely r 1127|174304|4305|4|7|9648.10|0.07|0.05|N|O|1995-11-05|1995-11-02|1995-11-11|DELIVER IN PERSON|FOB| idly pending pains 1152|8646|8647|1|23|35756.72|0.06|0.04|A|F|1994-10-14|1994-10-22|1994-10-21|DELIVER IN PERSON|MAIL|equests alongside of the unusual 1152|99604|2114|2|25|40090.00|0.04|0.08|R|F|1994-10-20|1994-09-18|1994-10-28|DELIVER IN PERSON|REG AIR|efully ironic accounts. sly instructions wa 1152|41846|4351|3|6|10727.04|0.07|0.03|A|F|1994-12-07|1994-11-05|1994-12-25|DELIVER IN PERSON|FOB|p furiously; packages above th 1153|85058|2583|1|15|15645.75|0.00|0.08|N|O|1996-04-24|1996-07-17|1996-04-29|TAKE BACK RETURN|SHIP|uctions boost fluffily according to 1153|168992|1509|2|50|103049.50|0.00|0.07|N|O|1996-06-27|1996-07-13|1996-07-05|COLLECT COD|REG AIR|ronic asymptotes nag slyly. 1153|43393|3394|3|25|33409.75|0.00|0.05|N|O|1996-06-18|1996-06-28|1996-07-09|NONE|TRUCK| theodolites 1153|91126|1127|4|43|48036.16|0.01|0.00|N|O|1996-06-09|1996-06-01|1996-07-04|DELIVER IN PERSON|MAIL|special instructions are. unusual, final du 1153|141212|6241|5|45|56394.45|0.00|0.02|N|O|1996-06-18|1996-06-20|1996-07-03|TAKE BACK RETURN|AIR|oss the ex 1153|135360|5361|6|26|36279.36|0.02|0.03|N|O|1996-08-16|1996-07-12|1996-09-08|NONE|MAIL|kages haggle carefully. f 1153|191834|4354|7|5|9629.15|0.02|0.03|N|O|1996-05-03|1996-06-12|1996-05-28|TAKE BACK RETURN|FOB|special excuses promi 1154|142724|5239|1|31|54768.32|0.06|0.06|A|F|1992-04-17|1992-04-26|1992-05-17|COLLECT COD|AIR|ithely. final, blithe 1154|147409|4952|2|50|72820.00|0.07|0.06|A|F|1992-04-22|1992-04-21|1992-05-01|NONE|TRUCK|ove the furiously bold Tires 1154|96103|3631|3|5|5495.50|0.09|0.04|A|F|1992-06-07|1992-05-07|1992-07-05|DELIVER IN PERSON|MAIL|the furiously 1154|574|575|4|35|51609.95|0.00|0.07|A|F|1992-03-30|1992-04-02|1992-04-21|DELIVER IN PERSON|TRUCK|the carefully regular pinto beans boost 1154|35462|7966|5|18|25154.28|0.02|0.03|A|F|1992-02-26|1992-03-24|1992-03-20|TAKE BACK RETURN|REG AIR|y regular excuses cajole blithely. fi 1154|195101|7621|6|50|59805.00|0.06|0.03|A|F|1992-03-04|1992-04-01|1992-04-01|TAKE BACK RETURN|TRUCK| even, special 1155|69329|9330|1|4|5193.28|0.09|0.05|N|O|1997-10-19|1997-12-09|1997-11-02|DELIVER IN PERSON|SHIP|ic foxes according to the carefully final 1155|195749|788|2|39|71944.86|0.08|0.05|N|O|1998-01-29|1998-01-03|1998-02-01|COLLECT COD|TRUCK|ckly final pinto beans was. 1155|146710|9225|3|23|40404.33|0.08|0.03|N|O|1997-11-24|1997-11-28|1997-12-06|DELIVER IN PERSON|FOB|ly unusual packages. iro 1155|139205|9206|4|12|14930.40|0.01|0.06|N|O|1997-11-01|1998-01-03|1997-11-19|DELIVER IN PERSON|RAIL|packages do 1155|4313|1814|5|49|59648.19|0.04|0.08|N|O|1997-12-07|1997-12-30|1997-12-08|NONE|AIR|ccounts are alongside of t 1156|86609|1626|1|15|23934.00|0.07|0.06|N|O|1996-12-21|1997-01-03|1997-01-10|TAKE BACK RETURN|AIR|the furiously pen 1156|32488|2489|2|21|29830.08|0.02|0.08|N|O|1996-11-07|1997-01-14|1996-12-03|NONE|AIR|dolphins. fluffily ironic packages sleep re 1156|11149|8653|3|29|30744.06|0.09|0.06|N|O|1997-01-24|1996-12-26|1997-02-04|DELIVER IN PERSON|TRUCK|ts sleep sly 1156|171422|1423|4|42|62723.64|0.02|0.00|N|O|1997-01-18|1997-01-12|1997-02-13|NONE|REG AIR|s. quickly bold pains are 1156|73755|6263|5|49|84708.75|0.04|0.01|N|O|1996-11-16|1996-12-02|1996-12-05|COLLECT COD|AIR|ithely unusual in 1156|194006|1564|6|42|46200.00|0.02|0.06|N|O|1997-01-27|1997-01-09|1997-01-28|DELIVER IN PERSON|MAIL|even requests boost ironic deposits. pe 1156|46115|3628|7|20|21222.20|0.08|0.07|N|O|1997-01-01|1997-01-06|1997-01-16|COLLECT COD|MAIL|deposits sleep bravel 1157|48128|3137|1|16|17217.92|0.06|0.00|N|O|1998-04-12|1998-03-09|1998-04-23|DELIVER IN PERSON|MAIL|tions hang 1157|82041|7058|2|4|4092.16|0.10|0.05|N|O|1998-02-24|1998-03-30|1998-03-24|DELIVER IN PERSON|SHIP|ounts. ironic deposits 1157|47985|5498|3|8|15463.84|0.02|0.00|N|O|1998-03-25|1998-03-16|1998-03-29|NONE|REG AIR|blithely even pa 1157|76907|6908|4|46|86659.40|0.07|0.08|N|O|1998-04-19|1998-03-13|1998-04-23|NONE|FOB|slyly regular excuses. accounts 1157|159850|4881|5|14|26737.90|0.03|0.03|N|O|1998-04-17|1998-03-03|1998-05-01|NONE|FOB|theodolites. fluffily re 1158|44218|6723|1|5|5811.05|0.02|0.04|N|O|1996-10-20|1996-07-30|1996-11-14|COLLECT COD|AIR|symptotes along the care 1158|156084|3630|2|23|26221.84|0.00|0.08|N|O|1996-10-21|1996-08-19|1996-10-31|COLLECT COD|MAIL|ularly ironic requests use care 1159|108033|8034|1|39|40600.17|0.01|0.00|A|F|1992-11-20|1992-10-28|1992-12-18|TAKE BACK RETURN|FOB| blithely express reques 1159|95490|509|2|7|10398.43|0.08|0.00|A|F|1992-11-25|1992-10-27|1992-12-20|NONE|AIR|olve somet 1159|97431|9941|3|11|15712.73|0.10|0.03|R|F|1992-12-09|1992-12-07|1992-12-18|DELIVER IN PERSON|MAIL|h furiousl 1184|46728|9233|1|27|45217.44|0.01|0.00|N|O|1998-01-10|1997-12-02|1998-02-06|TAKE BACK RETURN|REG AIR|s wake fluffily. fl 1184|146616|1645|2|4|6650.44|0.04|0.03|N|O|1997-12-25|1998-01-24|1998-01-18|DELIVER IN PERSON|RAIL| express packages. slyly expres 1184|163940|3941|3|7|14027.58|0.05|0.00|N|O|1998-02-14|1998-01-06|1998-03-11|COLLECT COD|TRUCK|ckly warthogs. blithely bold foxes hag 1184|125544|3081|4|3|4708.62|0.02|0.05|N|O|1998-01-15|1997-12-19|1998-02-02|NONE|REG AIR|ar packages. final packages cajol 1185|71925|6940|1|8|15175.36|0.01|0.06|A|F|1992-12-05|1992-10-05|1992-12-28|DELIVER IN PERSON|MAIL|ely according to the furiously regular r 1185|30749|750|2|28|47032.72|0.07|0.06|A|F|1992-09-24|1992-10-07|1992-10-10|DELIVER IN PERSON|REG AIR|ke. slyly regular t 1185|189350|1869|3|12|17272.20|0.05|0.06|R|F|1992-10-12|1992-09-26|1992-11-11|NONE|REG AIR|instructions. daringly pend 1186|2457|2458|1|28|38064.60|0.08|0.07|N|O|1996-12-08|1996-10-17|1996-12-15|TAKE BACK RETURN|TRUCK|ffily spec 1186|91123|6142|2|11|12255.32|0.07|0.05|N|O|1996-10-03|1996-10-21|1996-10-17|DELIVER IN PERSON|AIR|s haggle furiously; slyl 1186|100774|775|3|20|35495.40|0.07|0.07|N|O|1996-08-20|1996-10-23|1996-09-05|COLLECT COD|FOB|ely alongside of the blithel 1186|105917|5918|4|27|51918.57|0.06|0.04|N|O|1996-10-08|1996-11-06|1996-10-09|TAKE BACK RETURN|SHIP|accounts. express, e 1187|177779|5331|1|29|53846.33|0.01|0.04|R|F|1992-12-10|1993-02-09|1992-12-29|TAKE BACK RETURN|RAIL|riously express ac 1187|130451|2965|2|15|22221.75|0.03|0.04|A|F|1992-12-22|1993-01-13|1993-01-01|NONE|TRUCK|ests. foxes wake. carefu 1187|77243|9751|3|40|48809.60|0.08|0.06|R|F|1993-03-05|1992-12-31|1993-03-12|NONE|TRUCK|ar, brave deposits nag blithe 1188|114959|7471|1|2|3947.90|0.00|0.04|N|O|1996-05-22|1996-05-23|1996-06-06|COLLECT COD|RAIL|its breach blit 1188|112024|2025|2|9|9324.18|0.01|0.08|N|O|1996-08-04|1996-06-04|1996-08-19|NONE|REG AIR|ow carefully ironic d 1188|178737|8738|3|41|74444.93|0.07|0.04|N|O|1996-06-29|1996-05-21|1996-07-21|TAKE BACK RETURN|TRUCK|althy packages. fluffily unusual ideas h 1189|50676|677|1|23|37413.41|0.06|0.00|R|F|1994-07-25|1994-06-07|1994-08-02|COLLECT COD|FOB|s. fluffy Tiresias run quickly. bra 1189|104889|2420|2|32|60604.16|0.09|0.02|R|F|1994-05-06|1994-07-03|1994-05-15|TAKE BACK RETURN|FOB|e regular deposits. quickly quiet deposi 1189|56505|9011|3|22|32153.00|0.05|0.03|R|F|1994-06-09|1994-06-29|1994-06-23|DELIVER IN PERSON|TRUCK|quickly unusual platelets lose forges. ca 1190|83788|1313|1|32|56696.96|0.07|0.06|N|O|1997-05-08|1997-04-17|1997-06-01|COLLECT COD|FOB|y final packages? slyly even 1191|48288|793|1|29|35852.12|0.00|0.04|N|O|1996-01-24|1996-01-28|1996-02-17|COLLECT COD|AIR| regular pin 1216|96917|4445|1|8|15311.28|0.03|0.04|R|F|1993-02-01|1993-03-06|1993-02-08|TAKE BACK RETURN|TRUCK| of the carefully express 1216|74067|1589|2|48|49970.88|0.10|0.01|R|F|1993-01-17|1993-02-01|1993-02-13|COLLECT COD|SHIP|symptotes use against th 1216|41738|1739|3|18|30235.14|0.00|0.03|A|F|1993-01-20|1993-01-28|1993-02-02|COLLECT COD|MAIL|y final packages nod 1217|59148|4159|1|45|49821.30|0.07|0.02|A|F|1992-07-01|1992-06-23|1992-07-06|COLLECT COD|AIR|riously close ideas 1218|139055|1569|1|16|17504.80|0.04|0.07|A|F|1994-06-26|1994-08-07|1994-06-30|TAKE BACK RETURN|FOB|ven realms be 1218|93393|5903|2|41|56841.99|0.06|0.06|R|F|1994-08-04|1994-08-05|1994-08-11|TAKE BACK RETURN|SHIP|dolphins. theodolites beyond th 1218|47852|5365|3|44|79193.40|0.07|0.06|A|F|1994-10-05|1994-09-03|1994-10-30|COLLECT COD|TRUCK|thely ironic accounts wake slyly 1218|41364|3869|4|1|1305.36|0.01|0.08|R|F|1994-09-15|1994-09-07|1994-10-03|COLLECT COD|TRUCK|press furio 1219|131882|6909|1|6|11483.28|0.08|0.04|N|O|1995-11-13|1995-12-24|1995-11-18|NONE|MAIL|pecial, ironic requ 1219|128599|1112|2|4|6510.36|0.01|0.04|N|O|1995-11-24|1995-11-22|1995-12-07|TAKE BACK RETURN|SHIP|lly quick requests. blithely even h 1220|168916|6465|1|25|49622.75|0.10|0.03|N|O|1996-10-15|1996-11-07|1996-11-06|COLLECT COD|REG AIR| regular orbi 1220|159710|4741|2|36|63709.56|0.01|0.02|N|O|1996-12-10|1996-11-14|1997-01-07|COLLECT COD|SHIP|ar packages. blithely final acc 1220|36807|6808|3|3|5231.40|0.08|0.06|N|O|1996-09-06|1996-11-03|1996-09-10|COLLECT COD|REG AIR| final theodolites. blithely silent 1220|5144|145|4|36|37769.04|0.07|0.03|N|O|1996-12-12|1996-10-03|1996-12-15|TAKE BACK RETURN|TRUCK|unusual, silent pinto beans aga 1220|48992|4001|5|25|48524.75|0.03|0.08|N|O|1996-09-11|1996-10-09|1996-09-25|DELIVER IN PERSON|RAIL|packages affi 1221|80049|2558|1|43|44248.72|0.05|0.05|R|F|1992-06-22|1992-07-15|1992-07-20|DELIVER IN PERSON|FOB|y slyly above the slyly unusual ideas 1221|169470|9471|2|12|18473.64|0.00|0.08|R|F|1992-08-07|1992-06-24|1992-08-13|COLLECT COD|AIR|yly ironic 1221|68229|3242|3|3|3591.66|0.10|0.08|R|F|1992-07-01|1992-06-04|1992-07-27|COLLECT COD|TRUCK|ing to the fluffily 1221|119731|7265|4|41|71779.93|0.06|0.02|A|F|1992-04-28|1992-07-02|1992-05-19|NONE|RAIL|ns. bold deposit 1221|107651|162|5|13|21562.45|0.10|0.00|R|F|1992-08-01|1992-06-29|1992-08-27|TAKE BACK RETURN|AIR|ajole furiously. blithely expres 1221|84249|4250|6|7|8632.68|0.08|0.06|A|F|1992-06-27|1992-06-16|1992-07-23|TAKE BACK RETURN|RAIL|xpress accounts 1222|71266|8788|1|12|14847.12|0.09|0.02|A|F|1993-02-12|1993-03-14|1993-03-12|TAKE BACK RETURN|RAIL|s print permanently unusual packages. 1222|158637|1153|2|12|20347.56|0.08|0.01|A|F|1993-05-05|1993-03-27|1993-05-18|TAKE BACK RETURN|REG AIR| furiously bold instructions 1222|7554|55|3|26|38000.30|0.02|0.08|R|F|1993-02-13|1993-03-20|1993-02-22|TAKE BACK RETURN|MAIL|, even accounts are ironic 1223|99460|9461|1|28|40864.88|0.10|0.06|N|O|1996-08-07|1996-07-24|1996-08-13|TAKE BACK RETURN|MAIL| quickly ironic requests. furious 1248|163061|3062|1|45|50582.70|0.00|0.08|A|F|1992-04-17|1992-03-31|1992-05-13|NONE|RAIL|ter the pending pl 1248|150275|2791|2|37|49034.99|0.06|0.06|R|F|1992-01-26|1992-02-05|1992-02-13|COLLECT COD|TRUCK|. final requests integrate quickly. blit 1248|55368|2884|3|26|34407.36|0.09|0.06|A|F|1992-01-16|1992-03-01|1992-02-06|TAKE BACK RETURN|AIR| ironic dependen 1248|155169|5170|4|49|59983.84|0.02|0.01|A|F|1992-04-24|1992-02-18|1992-05-03|TAKE BACK RETURN|AIR|beans run quickly according to the carefu 1248|121465|3978|5|20|29729.20|0.08|0.00|A|F|1992-03-12|1992-03-23|1992-04-07|TAKE BACK RETURN|AIR|nal foxes cajole carefully slyl 1248|61457|6470|6|30|42553.50|0.10|0.01|R|F|1992-02-01|1992-03-24|1992-02-08|TAKE BACK RETURN|MAIL|fily special foxes kindle am 1249|58157|3168|1|49|54642.35|0.07|0.05|A|F|1994-03-03|1994-02-28|1994-03-08|NONE|RAIL|ffily express theodo 1250|1035|1036|1|15|14040.45|0.10|0.06|A|F|1992-11-05|1992-12-17|1992-12-03|TAKE BACK RETURN|SHIP| regular, i 1251|3325|3326|1|37|45447.84|0.08|0.08|N|O|1997-12-21|1998-01-12|1997-12-26|COLLECT COD|AIR|. furiously 1251|77902|7903|2|36|67676.40|0.07|0.04|N|O|1997-11-29|1998-01-07|1997-12-03|TAKE BACK RETURN|RAIL|y ironic Tiresias are slyly furio 1251|98778|6306|3|37|65740.49|0.09|0.02|N|O|1998-01-11|1997-12-01|1998-01-23|DELIVER IN PERSON|RAIL|finally bold requests 1251|149818|7361|4|7|13074.67|0.07|0.00|N|O|1998-01-08|1997-12-27|1998-01-18|COLLECT COD|MAIL|riously pe 1251|187484|2521|5|1|1571.48|0.02|0.03|N|O|1997-12-08|1998-01-06|1998-01-01|DELIVER IN PERSON|REG AIR| use quickly final packages. iron 1252|86435|8944|1|13|18478.59|0.10|0.01|N|O|1997-09-07|1997-09-12|1997-10-01|COLLECT COD|REG AIR|sts dazzle 1252|110885|5908|2|27|51188.76|0.00|0.08|N|O|1997-10-22|1997-10-10|1997-11-10|TAKE BACK RETURN|REG AIR|packages hag 1252|39144|9145|3|19|20579.66|0.07|0.02|N|O|1997-10-13|1997-10-23|1997-10-18|NONE|AIR|ts wake carefully-- packages sleep. quick 1252|91193|3703|4|11|13026.09|0.10|0.01|N|O|1997-10-16|1997-09-22|1997-10-28|COLLECT COD|AIR|s are. slyly final requests among the 1252|78748|8749|5|26|44895.24|0.05|0.05|N|O|1997-08-05|1997-10-24|1997-08-07|DELIVER IN PERSON|SHIP|onic pinto beans haggle furiously 1253|179988|7540|1|14|28951.72|0.00|0.06|R|F|1993-04-03|1993-04-16|1993-04-27|TAKE BACK RETURN|MAIL|lar foxes sleep furiously final, final pack 1253|53958|8969|2|13|24855.35|0.01|0.06|A|F|1993-03-05|1993-04-26|1993-03-08|DELIVER IN PERSON|FOB|al packages 1253|69375|9376|3|22|29576.14|0.05|0.06|A|F|1993-02-23|1993-04-06|1993-03-07|TAKE BACK RETURN|SHIP|telets cajole alongside of the final reques 1253|175589|624|4|23|38285.34|0.09|0.02|R|F|1993-04-18|1993-04-18|1993-05-07|COLLECT COD|FOB| the slyly silent re 1253|113918|6430|5|19|36706.29|0.05|0.05|A|F|1993-04-01|1993-04-22|1993-04-14|TAKE BACK RETURN|AIR|al pinto bea 1254|192385|4905|1|6|8864.28|0.08|0.01|N|O|1996-02-02|1996-03-21|1996-02-29|NONE|REG AIR|lithely even deposits eat! 1254|199414|4453|2|47|71130.27|0.05|0.06|N|O|1996-03-07|1996-02-20|1996-04-05|COLLECT COD|MAIL| platelets cajol 1254|134485|4486|3|35|53181.80|0.05|0.06|N|O|1996-04-08|1996-02-29|1996-04-18|DELIVER IN PERSON|FOB|ckages boost. furious warhorses cajole 1255|191801|4321|1|12|22713.60|0.00|0.02|A|F|1994-08-17|1994-06-29|1994-09-04|TAKE BACK RETURN|REG AIR| regular, express accounts are 1255|193712|1270|2|46|83062.66|0.07|0.05|R|F|1994-07-06|1994-07-14|1994-08-05|NONE|MAIL|ons nag qui 1280|128696|3721|1|17|29319.73|0.01|0.01|A|F|1993-02-04|1993-04-10|1993-02-07|NONE|FOB|ructions integrate across the th 1280|188305|5860|2|6|8359.80|0.05|0.06|R|F|1993-03-30|1993-02-16|1993-04-18|DELIVER IN PERSON|AIR|gular deposits 1280|32176|7183|3|13|14406.21|0.03|0.02|R|F|1993-03-06|1993-03-11|1993-03-18|DELIVER IN PERSON|TRUCK|blithely final accounts use evenly 1280|174752|2304|4|5|9133.75|0.06|0.03|R|F|1993-02-03|1993-02-11|1993-02-23|DELIVER IN PERSON|AIR|beans haggle. quickly bold instructions h 1280|51971|4477|5|24|46151.28|0.07|0.02|R|F|1993-03-20|1993-03-01|1993-04-09|COLLECT COD|RAIL|y pending orbits boost after the slyly 1280|65229|242|6|9|10747.98|0.00|0.05|R|F|1993-04-18|1993-03-28|1993-05-04|DELIVER IN PERSON|FOB|usual accou 1280|91590|9118|7|19|30050.21|0.02|0.06|A|F|1993-02-07|1993-02-28|1993-02-12|NONE|TRUCK|lyly along the furiously regular 1281|137813|327|1|33|61076.73|0.07|0.08|R|F|1995-02-01|1995-01-18|1995-03-03|NONE|REG AIR|dencies. thinly final pinto beans wake 1281|6372|1373|2|37|47299.69|0.08|0.03|A|F|1995-03-19|1995-02-02|1995-03-27|NONE|AIR|ounts detect 1281|93625|8644|3|2|3237.24|0.05|0.06|A|F|1994-12-27|1995-01-26|1995-01-21|TAKE BACK RETURN|FOB|ly unusual requests. final reques 1281|153005|5521|4|38|40204.00|0.04|0.06|R|F|1995-03-28|1995-01-11|1995-04-14|TAKE BACK RETURN|MAIL| ideas-- blithely regular 1281|151357|3873|5|13|18308.55|0.03|0.07|A|F|1995-02-06|1995-02-13|1995-02-18|DELIVER IN PERSON|TRUCK|fully final platelets wa 1281|49270|6783|6|4|4877.08|0.07|0.04|R|F|1995-03-15|1995-02-21|1995-03-20|NONE|SHIP|ggle against the even requests. requests 1281|77886|5408|7|43|80146.84|0.10|0.02|R|F|1995-01-28|1995-02-08|1995-02-10|DELIVER IN PERSON|AIR|final accounts. final packages slee 1282|22493|2494|1|14|19816.86|0.04|0.02|R|F|1992-06-29|1992-04-05|1992-07-21|TAKE BACK RETURN|REG AIR|ecial deposit 1282|29129|4134|2|10|10581.20|0.09|0.06|R|F|1992-04-10|1992-04-16|1992-05-01|DELIVER IN PERSON|SHIP|r theodolite 1282|159521|9522|3|19|30029.88|0.01|0.03|R|F|1992-05-07|1992-04-07|1992-05-13|NONE|RAIL|ts x-ray across the furi 1282|58316|8317|4|19|24211.89|0.00|0.05|A|F|1992-06-20|1992-04-17|1992-07-05|DELIVER IN PERSON|REG AIR|nto beans. carefully close theodo 1283|92682|210|1|47|78709.96|0.05|0.03|N|O|1996-10-21|1996-10-29|1996-11-12|DELIVER IN PERSON|TRUCK|even instructions boost slyly blithely 1283|105334|355|2|1|1339.33|0.00|0.08|N|O|1996-10-07|1996-10-12|1996-10-08|NONE|RAIL|d the sauternes. slyly ev 1283|137074|4614|3|18|19999.26|0.02|0.01|N|O|1996-10-14|1996-11-07|1996-10-22|DELIVER IN PERSON|AIR|equests use along the fluff 1283|191999|4519|4|40|83639.60|0.07|0.03|N|O|1996-11-09|1996-11-23|1996-11-28|NONE|MAIL|riously. even, ironic instructions after 1283|123473|5986|5|43|64348.21|0.01|0.04|N|O|1996-09-29|1996-11-19|1996-10-26|TAKE BACK RETURN|RAIL|requests sleep slyly about the 1283|7533|5034|6|30|43215.90|0.06|0.07|N|O|1996-11-22|1996-11-22|1996-12-15|COLLECT COD|TRUCK|t the fluffily 1283|196227|6228|7|21|27787.62|0.04|0.03|N|O|1996-09-12|1996-10-02|1996-10-12|NONE|REG AIR|fully regular 1284|177427|2462|1|49|73716.58|0.00|0.06|N|O|1996-04-11|1996-03-04|1996-04-16|NONE|MAIL|lar packages. special packages ac 1284|5304|5305|2|4|4837.20|0.07|0.06|N|O|1996-02-29|1996-02-11|1996-03-01|TAKE BACK RETURN|TRUCK| regular asymptotes. 1284|132592|2593|3|39|63359.01|0.08|0.00|N|O|1996-01-11|1996-02-07|1996-02-05|COLLECT COD|MAIL|even accoun 1284|58252|8253|4|1|1210.25|0.01|0.07|N|O|1996-04-28|1996-04-02|1996-05-08|DELIVER IN PERSON|SHIP|al packages use carefully express de 1284|33328|3329|5|9|11351.88|0.05|0.06|N|O|1996-03-03|1996-03-19|1996-04-01|DELIVER IN PERSON|REG AIR|after the pending 1285|21548|1549|1|12|17634.48|0.00|0.06|A|F|1992-06-21|1992-08-16|1992-07-12|COLLECT COD|MAIL|ss foxes. blithe theodolites cajole slyly 1285|142274|4789|2|45|59232.15|0.01|0.02|R|F|1992-09-05|1992-08-08|1992-10-02|COLLECT COD|REG AIR| special requests haggle blithely. 1285|188148|5703|3|4|4944.56|0.09|0.06|A|F|1992-07-20|1992-08-17|1992-07-26|DELIVER IN PERSON|FOB|l packages sleep slyly quiet i 1285|187439|9958|4|39|59530.77|0.05|0.01|A|F|1992-09-15|1992-08-05|1992-10-05|DELIVER IN PERSON|TRUCK|uctions. car 1285|83541|8558|5|33|50309.82|0.00|0.08|R|F|1992-09-08|1992-08-25|1992-09-16|NONE|SHIP|ites affix 1286|177013|7014|1|49|53410.49|0.08|0.01|R|F|1993-06-24|1993-08-12|1993-06-26|DELIVER IN PERSON|SHIP|gged accoun 1286|48230|735|2|48|56555.04|0.01|0.04|A|F|1993-07-11|1993-07-11|1993-08-01|COLLECT COD|TRUCK|unts alongs 1286|188576|3613|3|11|18310.27|0.03|0.04|R|F|1993-08-08|1993-07-30|1993-09-05|DELIVER IN PERSON|FOB| slyly even packages. requ 1286|183767|3768|4|37|68478.12|0.00|0.02|R|F|1993-05-27|1993-07-11|1993-06-01|COLLECT COD|SHIP|lyly ironic pinto beans cajole furiously s 1286|164048|1597|5|14|15568.56|0.00|0.01|R|F|1993-05-23|1993-08-09|1993-06-01|NONE|REG AIR|blithely bo 1286|145926|3469|6|41|80848.72|0.04|0.05|R|F|1993-08-02|1993-08-06|1993-08-07|TAKE BACK RETURN|FOB| the furiously expre 1287|173172|8207|1|35|43580.95|0.09|0.06|A|F|1994-09-07|1994-09-12|1994-09-30|TAKE BACK RETURN|FOB|s wake unusual grou 1287|94556|9575|2|10|15505.50|0.08|0.03|R|F|1994-07-08|1994-08-28|1994-07-10|TAKE BACK RETURN|RAIL|thely alongside of the unusual, ironic pa 1287|278|279|3|30|35348.10|0.00|0.07|R|F|1994-07-12|1994-09-23|1994-08-07|NONE|RAIL|ar packages. even, even 1287|61652|9171|4|10|16136.50|0.01|0.05|A|F|1994-09-03|1994-08-12|1994-09-16|TAKE BACK RETURN|REG AIR|ding, regular accounts 1287|178903|3938|5|21|41619.90|0.06|0.02|A|F|1994-10-06|1994-09-25|1994-10-16|TAKE BACK RETURN|TRUCK|y quickly bold theodoli 1287|20481|5486|6|26|36438.48|0.03|0.08|R|F|1994-10-03|1994-09-27|1994-10-30|DELIVER IN PERSON|RAIL|egular foxes. theodolites nag along t 1312|80689|5706|1|9|15027.12|0.04|0.08|R|F|1994-07-19|1994-06-29|1994-07-24|TAKE BACK RETURN|MAIL|. furiously 1312|135990|5991|2|28|56727.72|0.06|0.06|A|F|1994-09-09|1994-08-01|1994-10-02|TAKE BACK RETURN|FOB|uriously final frays should use quick 1312|172935|487|3|18|36142.74|0.03|0.07|A|F|1994-09-13|1994-07-08|1994-09-22|TAKE BACK RETURN|MAIL|. slyly ironic 1313|51361|8877|1|48|62993.28|0.01|0.03|A|F|1994-12-20|1994-10-29|1995-01-07|COLLECT COD|MAIL|s are quick 1314|197722|242|1|5|9098.60|0.03|0.01|A|F|1994-05-26|1994-08-06|1994-05-31|TAKE BACK RETURN|AIR|equests nag across the furious 1314|109303|4324|2|39|51179.70|0.01|0.03|R|F|1994-08-09|1994-06-14|1994-08-31|TAKE BACK RETURN|TRUCK| unusual accounts slee 1314|40911|912|3|11|20371.01|0.01|0.04|A|F|1994-05-16|1994-07-30|1994-05-31|COLLECT COD|REG AIR|tegrate furious 1315|95509|8019|1|27|40621.50|0.01|0.03|N|O|1998-07-04|1998-06-13|1998-07-28|NONE|SHIP|latelets. fluffily ironic account 1315|15742|3246|2|15|24866.10|0.05|0.01|N|O|1998-07-12|1998-06-10|1998-08-07|COLLECT COD|AIR|. foxes integrate carefully special 1315|167190|4739|3|25|31429.75|0.01|0.08|N|O|1998-06-26|1998-06-10|1998-07-06|TAKE BACK RETURN|FOB|lites. unusual foxes affi 1315|160919|8468|4|19|37618.29|0.02|0.05|N|O|1998-07-05|1998-05-23|1998-08-04|TAKE BACK RETURN|SHIP|nal, regular warhorses about the fu 1315|158532|1048|5|32|50896.96|0.10|0.05|N|O|1998-03-30|1998-06-12|1998-04-25|NONE|SHIP|neath the final p 1316|126429|1454|1|46|66949.32|0.05|0.04|A|F|1994-01-13|1994-01-24|1994-02-03|COLLECT COD|TRUCK|ges haggle of the 1316|78158|666|2|15|17042.25|0.02|0.01|R|F|1994-03-12|1994-03-02|1994-03-14|COLLECT COD|FOB|se. furiously final depo 1316|197244|7245|3|33|44260.92|0.10|0.06|R|F|1994-03-31|1994-01-23|1994-04-20|TAKE BACK RETURN|AIR|manently; blithely special deposits 1316|65047|60|4|15|15180.60|0.00|0.06|R|F|1993-12-17|1994-02-04|1993-12-20|NONE|RAIL|fully express dugouts. furiously silent ide 1316|40746|747|5|40|67469.60|0.01|0.03|R|F|1994-02-04|1994-02-09|1994-02-27|NONE|REG AIR|l dugouts. co 1316|3390|5891|6|7|9053.73|0.05|0.04|A|F|1993-12-09|1994-01-12|1993-12-30|TAKE BACK RETURN|MAIL|. furiously even accounts a 1316|162096|9645|7|8|9264.72|0.10|0.04|A|F|1994-03-26|1994-02-08|1994-04-19|NONE|SHIP|packages against the express requests wa 1317|133799|3800|1|34|62314.86|0.08|0.04|N|O|1995-08-13|1995-08-08|1995-09-10|COLLECT COD|RAIL|deposits boost thinly blithely final id 1317|159586|7132|2|7|11519.06|0.05|0.01|A|F|1995-06-08|1995-08-03|1995-06-16|TAKE BACK RETURN|SHIP| pinto beans according to the final, pend 1317|157179|7180|3|26|32140.42|0.01|0.02|N|O|1995-07-13|1995-06-26|1995-08-06|COLLECT COD|RAIL|leep along th 1317|105156|2687|4|35|40640.25|0.05|0.02|N|O|1995-07-16|1995-07-07|1995-07-22|TAKE BACK RETURN|FOB|r packages impress blithely car 1317|149415|6958|5|36|52718.76|0.02|0.00|N|O|1995-09-03|1995-07-06|1995-09-04|DELIVER IN PERSON|AIR| deposits. quic 1318|113682|1216|1|24|40696.32|0.08|0.06|N|O|1998-09-27|1998-09-15|1998-10-12|TAKE BACK RETURN|AIR|ual, unusual packages. fluffy, iro 1318|45822|8327|2|26|45963.32|0.01|0.03|N|O|1998-09-26|1998-08-09|1998-10-07|DELIVER IN PERSON|FOB|ly. regular, u 1318|128344|857|3|31|42542.54|0.01|0.04|N|O|1998-08-25|1998-07-31|1998-08-31|COLLECT COD|AIR|ve the carefully expr 1319|60918|5931|1|21|39457.11|0.03|0.04|N|O|1996-10-05|1996-12-02|1996-10-28|COLLECT COD|FOB|s: carefully express 1319|36685|1692|2|12|19460.16|0.09|0.05|N|O|1996-11-05|1996-12-12|1996-11-29|DELIVER IN PERSON|TRUCK|packages integrate furiously. expres 1344|140001|5030|1|15|15615.00|0.10|0.07|A|F|1992-06-22|1992-06-24|1992-06-23|TAKE BACK RETURN|MAIL|rding to the blithely ironic theodolite 1344|189888|4925|2|29|57358.52|0.09|0.00|A|F|1992-07-17|1992-06-07|1992-07-21|NONE|REG AIR|ffily quiet foxes wake blithely. slyly 1345|197110|7111|1|49|59148.39|0.08|0.00|A|F|1992-12-27|1993-01-23|1993-01-06|NONE|FOB|sly. furiously final accounts are blithely 1345|11261|6264|2|37|43373.62|0.10|0.07|A|F|1992-11-27|1992-12-11|1992-12-07|COLLECT COD|FOB|e slyly express requests. ironic accounts c 1345|56666|6667|3|31|50302.46|0.08|0.07|R|F|1992-12-02|1992-12-29|1992-12-14|COLLECT COD|REG AIR|. slyly silent accounts sublat 1346|159096|1612|1|29|33497.61|0.07|0.05|A|F|1992-08-18|1992-09-15|1992-09-17|TAKE BACK RETURN|REG AIR|the pinto 1346|124499|4500|2|48|73127.52|0.06|0.03|A|F|1992-09-28|1992-07-22|1992-10-13|TAKE BACK RETURN|REG AIR| along the carefully spec 1346|53819|3820|3|13|23046.53|0.10|0.04|A|F|1992-07-22|1992-08-10|1992-08-06|NONE|SHIP|arefully brave deposits into the slyly iro 1346|123555|3556|4|6|9471.30|0.02|0.02|R|F|1992-09-13|1992-07-21|1992-09-27|TAKE BACK RETURN|AIR|inst the furiously final theodolites. caref 1346|186043|6044|5|30|33871.20|0.01|0.07|R|F|1992-10-01|1992-07-22|1992-10-24|NONE|SHIP| nag blithely. unusual, ru 1346|15002|2506|6|45|41265.00|0.02|0.04|A|F|1992-09-11|1992-08-06|1992-09-12|COLLECT COD|FOB|press deposits. 1347|80130|7655|1|45|49955.85|0.02|0.05|N|O|1997-08-24|1997-09-03|1997-09-08|COLLECT COD|AIR|ages wake around t 1347|142381|7410|2|34|48394.92|0.07|0.04|N|O|1997-06-25|1997-09-08|1997-07-24|COLLECT COD|FOB|r packages. f 1347|184871|4872|3|23|44985.01|0.03|0.04|N|O|1997-07-31|1997-08-25|1997-08-21|COLLECT COD|SHIP|ronic pinto beans. express reques 1347|112077|4589|4|28|30493.96|0.01|0.00|N|O|1997-07-30|1997-07-22|1997-08-18|TAKE BACK RETURN|FOB|foxes after the blithely special i 1347|64173|4174|5|9|10234.53|0.01|0.03|N|O|1997-08-28|1997-09-16|1997-09-26|DELIVER IN PERSON|AIR| detect blithely above the fina 1347|152357|7388|6|21|29596.35|0.06|0.04|N|O|1997-10-10|1997-08-16|1997-11-02|NONE|FOB|g pinto beans affix car 1347|50283|7799|7|10|12332.80|0.02|0.07|N|O|1997-07-04|1997-07-23|1997-07-05|DELIVER IN PERSON|SHIP|y ironic pin 1348|94296|6806|1|13|16773.77|0.01|0.01|N|O|1998-04-28|1998-06-05|1998-05-12|TAKE BACK RETURN|SHIP| blithely r 1348|21908|9415|2|41|75025.90|0.07|0.03|N|O|1998-05-02|1998-05-26|1998-05-09|COLLECT COD|RAIL|kages. platelets about the ca 1348|198221|8222|3|40|52768.80|0.07|0.05|N|O|1998-08-14|1998-07-10|1998-08-27|COLLECT COD|AIR|fter the regu 1348|97544|2563|4|2|3083.08|0.01|0.04|N|O|1998-05-30|1998-06-20|1998-06-05|COLLECT COD|MAIL|lly final packages use fluffily express ac 1349|180575|3094|1|1|1655.57|0.06|0.03|N|O|1998-01-07|1998-01-14|1998-02-03|COLLECT COD|REG AIR| express inst 1349|117686|198|2|45|76665.60|0.03|0.02|N|O|1997-12-24|1998-01-17|1997-12-28|NONE|AIR| ironic, unusual deposits wake carefu 1350|53683|8694|1|21|34370.28|0.04|0.04|A|F|1993-12-17|1993-10-17|1993-12-25|COLLECT COD|REG AIR|lyly above the evenly 1350|43604|3605|2|32|49523.20|0.03|0.00|R|F|1993-11-18|1993-09-30|1993-12-16|COLLECT COD|MAIL|ic, final 1351|107227|7228|1|25|30855.50|0.06|0.04|N|O|1998-06-02|1998-05-25|1998-06-22|COLLECT COD|SHIP|iously regul 1376|168528|1045|1|22|35123.44|0.01|0.03|N|O|1997-08-05|1997-07-08|1997-09-03|NONE|REG AIR|inst the final, pending 1377|153593|1139|1|5|8232.95|0.06|0.05|N|O|1998-05-06|1998-07-08|1998-06-01|TAKE BACK RETURN|FOB| final, final grouches. accoun 1377|32599|5103|2|3|4594.77|0.10|0.04|N|O|1998-04-30|1998-07-02|1998-05-14|DELIVER IN PERSON|REG AIR|yly enticing requ 1377|83378|8395|3|26|35395.62|0.07|0.07|N|O|1998-05-28|1998-06-11|1998-06-25|COLLECT COD|SHIP|egular deposits. quickly regular acco 1377|120024|7561|4|39|40716.78|0.00|0.03|N|O|1998-07-27|1998-07-18|1998-08-13|DELIVER IN PERSON|SHIP|e ironic, regular requests. carefully 1377|32294|4798|5|19|23299.51|0.10|0.00|N|O|1998-06-20|1998-06-27|1998-07-20|NONE|AIR|ught to are bold foxes 1377|153755|1301|6|17|30748.75|0.03|0.04|N|O|1998-06-19|1998-07-20|1998-07-14|NONE|REG AIR|s must have to mold b 1378|196057|1096|1|34|39203.70|0.09|0.07|N|O|1996-07-08|1996-04-23|1996-07-09|COLLECT COD|RAIL|le furiously slyly final accounts. careful 1378|123358|5871|2|18|24864.30|0.05|0.02|N|O|1996-06-19|1996-05-16|1996-06-21|DELIVER IN PERSON|RAIL| theodolites. i 1378|72873|2874|3|11|20304.57|0.10|0.03|N|O|1996-06-07|1996-05-09|1996-07-05|COLLECT COD|TRUCK| blithely express hoc 1378|170689|690|4|12|21116.16|0.02|0.06|N|O|1996-06-16|1996-05-23|1996-07-09|COLLECT COD|SHIP|notornis. b 1378|155233|5234|5|9|11594.07|0.06|0.05|N|O|1996-04-20|1996-04-13|1996-05-09|COLLECT COD|REG AIR|e carefully. carefully iron 1378|193453|5973|6|29|44847.05|0.05|0.05|N|O|1996-04-15|1996-04-23|1996-05-14|NONE|REG AIR|ual packages are furiously blith 1379|72894|5402|1|13|24269.57|0.04|0.01|N|O|1998-06-08|1998-07-13|1998-06-16|NONE|AIR|ully across the furiously iron 1379|117107|9619|2|50|56205.00|0.07|0.08|N|O|1998-08-31|1998-07-13|1998-09-02|TAKE BACK RETURN|FOB|olphins. ca 1379|12350|4852|3|24|30296.40|0.05|0.02|N|O|1998-07-06|1998-07-09|1998-07-29|DELIVER IN PERSON|MAIL|ages cajole carefully idly express re 1380|148197|3226|1|6|7471.14|0.00|0.04|N|O|1996-08-06|1996-10-01|1996-08-14|NONE|RAIL|e foxes. slyly specia 1380|140792|5821|2|40|73311.60|0.02|0.02|N|O|1996-10-01|1996-08-14|1996-10-20|COLLECT COD|RAIL|ly final frets. ironic, 1380|77746|7747|3|15|25856.10|0.05|0.02|N|O|1996-07-14|1996-08-12|1996-08-03|NONE|FOB|riously ironic foxes aff 1380|60776|3283|4|33|57313.41|0.04|0.07|N|O|1996-08-23|1996-10-01|1996-09-18|TAKE BACK RETURN|SHIP|e ironic, even excuses haggle 1381|143074|5589|1|47|52502.29|0.08|0.04|N|O|1998-09-22|1998-08-12|1998-10-12|DELIVER IN PERSON|AIR|ly ironic deposits 1381|33128|5632|2|12|12733.44|0.07|0.08|N|O|1998-08-13|1998-08-12|1998-08-28|TAKE BACK RETURN|AIR| furiously regular package 1382|161373|1374|1|18|25818.66|0.08|0.03|R|F|1993-08-30|1993-10-19|1993-09-03|DELIVER IN PERSON|AIR|hely regular deposits. fluffy s 1382|180829|830|2|29|55384.78|0.08|0.04|A|F|1993-10-08|1993-11-11|1993-10-10|COLLECT COD|FOB| haggle: closely even asymptot 1382|177866|2901|3|43|83585.98|0.10|0.04|A|F|1993-09-02|1993-10-06|1993-09-15|DELIVER IN PERSON|AIR|ress deposits. slyly ironic foxes are blit 1382|180589|8144|4|11|18365.38|0.04|0.04|R|F|1993-09-17|1993-09-29|1993-09-21|NONE|SHIP|furiously unusual packages play quickly 1382|156162|6163|5|31|37762.96|0.07|0.03|R|F|1993-10-26|1993-10-15|1993-11-09|TAKE BACK RETURN|FOB|hely regular dependencies. f 1382|9855|4856|6|38|67064.30|0.07|0.07|R|F|1993-11-17|1993-09-28|1993-11-20|COLLECT COD|SHIP|ake pending pinto beans. s 1382|22788|2789|7|5|8553.90|0.07|0.01|R|F|1993-10-02|1993-09-29|1993-10-12|DELIVER IN PERSON|REG AIR|ter the carefully final excuses. blit 1383|192894|452|1|14|27816.46|0.07|0.06|A|F|1993-08-25|1993-07-09|1993-09-12|DELIVER IN PERSON|RAIL|ole carefully silent requests. car 1383|160122|2639|2|19|22460.28|0.06|0.04|R|F|1993-05-24|1993-07-07|1993-06-14|NONE|AIR|lyly unusual accounts sle 1408|147902|5445|1|29|56547.10|0.03|0.04|N|O|1998-03-12|1998-02-14|1998-03-17|COLLECT COD|MAIL|en accounts grow furiousl 1408|172326|7361|2|7|9788.24|0.05|0.06|N|O|1998-01-14|1998-03-21|1998-01-29|COLLECT COD|AIR|fully final instructions. theodolites ca 1408|75549|8057|3|11|16769.94|0.00|0.03|N|O|1998-04-04|1998-01-29|1998-04-18|NONE|REG AIR|y even accounts thrash care 1408|147092|9607|4|20|22781.80|0.06|0.00|N|O|1998-04-21|1998-01-25|1998-05-12|DELIVER IN PERSON|TRUCK| blithely fluffi 1408|169688|9689|5|41|72064.88|0.02|0.06|N|O|1998-02-25|1998-02-03|1998-03-13|COLLECT COD|REG AIR|ep along the fina 1408|133484|1024|6|42|63734.16|0.05|0.08|N|O|1998-01-30|1998-02-07|1998-02-18|TAKE BACK RETURN|REG AIR|even packages. even accounts cajole 1408|54519|4520|7|26|38311.26|0.00|0.00|N|O|1998-03-19|1998-03-14|1998-04-01|COLLECT COD|RAIL|ic foxes ca 1409|98868|1378|1|23|42937.78|0.01|0.03|A|F|1993-04-18|1993-02-25|1993-05-06|DELIVER IN PERSON|FOB|ions. slyly ironic packages wake quick 1409|64412|9425|2|36|49550.76|0.09|0.02|A|F|1993-01-27|1993-01-31|1993-02-07|COLLECT COD|FOB|ncies sleep carefully r 1409|159012|9013|3|17|18207.17|0.07|0.00|R|F|1993-04-15|1993-03-01|1993-04-29|NONE|REG AIR|pending accounts poach. care 1410|120271|5296|1|15|19369.05|0.06|0.05|N|O|1997-05-25|1997-07-08|1997-06-15|NONE|SHIP| bold packages are fluf 1410|178206|724|2|18|23115.60|0.03|0.00|N|O|1997-06-03|1997-05-17|1997-06-07|TAKE BACK RETURN|RAIL|gle furiously fluffily regular requests 1410|108109|3130|3|37|41332.70|0.02|0.01|N|O|1997-04-17|1997-06-18|1997-04-19|COLLECT COD|TRUCK|to beans b 1410|187510|5065|4|22|35145.22|0.10|0.00|N|O|1997-07-31|1997-05-17|1997-08-19|TAKE BACK RETURN|RAIL|gular account 1410|65802|3321|5|25|44195.00|0.09|0.02|N|O|1997-05-07|1997-07-10|1997-05-16|NONE|REG AIR|unts haggle against the furiously fina 1411|16321|3825|1|9|11135.88|0.06|0.04|A|F|1995-03-08|1995-03-04|1995-03-11|DELIVER IN PERSON|AIR|accounts. furiou 1411|106434|6435|2|26|37451.18|0.02|0.02|A|F|1995-04-12|1995-01-24|1995-05-03|TAKE BACK RETURN|TRUCK|c packages. 1411|26013|1018|3|37|34743.37|0.00|0.06|A|F|1995-02-27|1995-03-02|1995-03-24|NONE|MAIL|d excuses. furiously final pear 1411|199946|4985|4|20|40918.80|0.01|0.03|R|F|1995-04-06|1995-03-16|1995-04-17|COLLECT COD|FOB|s against the 1411|82816|7833|5|46|82745.26|0.08|0.05|A|F|1995-04-03|1995-01-20|1995-04-05|DELIVER IN PERSON|REG AIR|ly daring instructions 1411|76148|1163|6|30|33724.20|0.09|0.04|A|F|1995-01-12|1995-02-01|1995-01-23|DELIVER IN PERSON|MAIL|ious foxes wake courts. caref 1412|57200|2211|1|37|42816.40|0.06|0.01|A|F|1993-04-10|1993-04-19|1993-04-12|DELIVER IN PERSON|RAIL|hely express excuses are 1412|155021|52|2|20|21520.40|0.10|0.05|A|F|1993-07-04|1993-05-18|1993-07-22|DELIVER IN PERSON|REG AIR|odolites sleep ironically 1412|22620|7625|3|2|3085.24|0.10|0.07|R|F|1993-04-01|1993-05-03|1993-04-12|DELIVER IN PERSON|REG AIR|s among the requests are a 1412|166155|6156|4|11|13432.65|0.05|0.07|R|F|1993-05-27|1993-05-30|1993-06-07|DELIVER IN PERSON|MAIL|en packages. regular packages dete 1412|157647|163|5|11|18751.04|0.08|0.06|A|F|1993-03-30|1993-05-25|1993-04-21|NONE|FOB|se slyly. special, unusual accounts nag bl 1413|177936|7937|1|18|36250.74|0.08|0.05|N|O|1997-10-11|1997-08-17|1997-10-25|NONE|FOB|yly bold packages haggle quickly acr 1413|164292|1841|2|49|66458.21|0.07|0.06|N|O|1997-08-28|1997-08-23|1997-09-12|DELIVER IN PERSON|MAIL|nstructions br 1413|41545|4050|3|6|8919.24|0.04|0.02|N|O|1997-09-07|1997-07-30|1997-09-21|TAKE BACK RETURN|MAIL|lithely excuses. f 1414|37241|4751|1|39|45951.36|0.10|0.03|N|O|1995-09-22|1995-09-30|1995-10-07|NONE|MAIL|quickly aro 1414|106423|6424|2|4|5717.68|0.02|0.05|N|O|1995-09-16|1995-11-01|1995-10-02|COLLECT COD|AIR| haggle quickly 1415|148235|8236|1|25|32080.75|0.06|0.00|A|F|1994-09-03|1994-07-12|1994-09-13|DELIVER IN PERSON|RAIL|ect never fluff 1440|192213|7252|1|3|3915.63|0.06|0.01|N|O|1995-10-30|1995-10-17|1995-11-08|COLLECT COD|SHIP|instructions boost. fluffily regul 1440|113748|1282|2|46|81040.04|0.02|0.03|N|O|1995-09-21|1995-10-19|1995-10-19|NONE|RAIL|blithely even instructions. 1441|143130|8159|1|5|5865.65|0.04|0.01|N|O|1997-05-17|1997-05-11|1997-05-30|COLLECT COD|MAIL|egular courts. fluffily even grouches 1441|176636|9154|2|5|8563.15|0.02|0.05|N|O|1997-04-25|1997-04-16|1997-05-23|COLLECT COD|FOB|he quickly enticing pac 1441|117735|2758|3|14|24538.22|0.01|0.03|N|O|1997-06-30|1997-04-29|1997-07-24|DELIVER IN PERSON|REG AIR|special requests ha 1441|159879|2395|4|37|71738.19|0.01|0.00|N|O|1997-04-26|1997-04-27|1997-04-29|NONE|REG AIR|accounts. slyly special dolphins b 1441|71377|8899|5|34|45844.58|0.09|0.00|N|O|1997-06-12|1997-05-11|1997-06-29|TAKE BACK RETURN|RAIL|e carefully. blithely ironic dep 1441|24329|9334|6|15|18799.80|0.09|0.08|N|O|1997-05-21|1997-05-06|1997-06-04|NONE|REG AIR| dependencies-- cour 1441|95795|3323|7|50|89539.50|0.03|0.01|N|O|1997-06-07|1997-05-12|1997-06-08|NONE|SHIP| requests. blithely e 1442|25080|85|1|8|8040.64|0.05|0.01|A|F|1994-10-31|1994-09-04|1994-11-25|COLLECT COD|AIR|c deposits haggle after the even 1443|33679|6183|1|47|75795.49|0.04|0.06|N|O|1997-02-05|1997-02-02|1997-03-03|NONE|RAIL|carefully ironic requests sl 1444|169552|7101|1|42|68105.10|0.01|0.02|R|F|1994-12-22|1995-03-03|1994-12-31|NONE|SHIP|ly bold packages boost regular ideas. spe 1444|56665|1676|2|34|55136.44|0.04|0.08|A|F|1995-02-22|1995-02-15|1995-03-19|TAKE BACK RETURN|AIR|y. doggedly pend 1444|154729|7245|3|34|60646.48|0.02|0.07|R|F|1994-12-17|1995-01-12|1995-01-03|COLLECT COD|AIR|ular accounts 1444|118714|3737|4|6|10396.26|0.06|0.03|A|F|1995-01-07|1995-03-05|1995-01-17|COLLECT COD|RAIL|al accounts. br 1444|19173|9174|5|35|38225.95|0.02|0.05|A|F|1995-02-25|1995-03-05|1995-03-24|DELIVER IN PERSON|SHIP|aggle furiou 1444|32266|7273|6|42|50326.92|0.00|0.02|A|F|1994-12-16|1995-02-18|1994-12-22|DELIVER IN PERSON|RAIL|ss requests. ironic ideas wake above 1444|81925|1926|7|12|22883.04|0.00|0.03|R|F|1994-12-23|1995-01-15|1995-01-13|COLLECT COD|TRUCK|ly among the bol 1445|99368|9369|1|24|32816.64|0.01|0.00|A|F|1995-02-21|1995-02-22|1995-03-18|DELIVER IN PERSON|SHIP|al accounts use furiously a 1445|66877|6878|2|48|88505.76|0.10|0.02|A|F|1995-02-28|1995-03-16|1995-03-12|COLLECT COD|MAIL|. final ideas are carefully dar 1445|191125|3645|3|7|8512.84|0.10|0.04|A|F|1995-04-25|1995-02-25|1995-05-10|NONE|SHIP|structions: slyly regular re 1445|27878|5385|4|17|30699.79|0.04|0.07|A|F|1995-04-02|1995-04-04|1995-05-01|COLLECT COD|FOB|ges. furiously regular pint 1445|134431|6945|5|24|35170.32|0.10|0.06|R|F|1995-04-23|1995-02-16|1995-05-18|NONE|REG AIR|rate after the carefully reg 1445|167046|7047|6|39|43408.56|0.03|0.02|A|F|1995-02-05|1995-02-20|1995-02-06|NONE|MAIL|ully unusual reques 1446|71867|1868|1|31|57004.66|0.10|0.02|N|O|1998-05-01|1998-05-17|1998-05-30|NONE|REG AIR|. slyly reg 1447|166076|1109|1|19|21699.33|0.06|0.04|A|F|1993-01-31|1992-12-07|1993-02-04|COLLECT COD|MAIL|. quickly ironic 1447|31049|6056|2|6|5880.24|0.01|0.05|A|F|1992-10-24|1992-12-10|1992-11-05|DELIVER IN PERSON|AIR|as! regular packages poach above the 1447|38700|6210|3|9|14748.30|0.04|0.00|R|F|1992-11-15|1993-01-07|1992-11-29|DELIVER IN PERSON|MAIL|counts wake s 1447|21725|9232|4|8|13173.76|0.09|0.08|R|F|1992-11-20|1993-01-12|1992-12-14|COLLECT COD|FOB|ost carefully 1447|129435|9436|5|23|33681.89|0.02|0.07|A|F|1992-12-07|1992-12-25|1993-01-06|TAKE BACK RETURN|AIR| dazzle quickly deposits. f 1447|199588|4627|6|41|69190.78|0.08|0.02|R|F|1993-01-06|1993-01-05|1993-01-13|TAKE BACK RETURN|MAIL|rts boost s 1472|7243|4744|1|36|41408.64|0.04|0.05|N|O|1996-11-06|1996-11-13|1996-11-12|COLLECT COD|SHIP|riously silent deposits to the pending d 1472|132029|7056|2|26|27586.52|0.03|0.05|N|O|1996-11-08|1996-11-13|1996-12-02|DELIVER IN PERSON|FOB|ic packages w 1472|567|8068|3|6|8805.36|0.08|0.01|N|O|1996-10-24|1996-11-19|1996-11-23|COLLECT COD|FOB|onic theodolites hinder slyly slyly r 1473|53482|8493|1|50|71774.00|0.04|0.03|N|O|1997-05-05|1997-05-20|1997-05-09|NONE|TRUCK|requests wake express deposits. special, ir 1473|67401|4920|2|32|43788.80|0.00|0.08|N|O|1997-04-18|1997-05-12|1997-05-10|DELIVER IN PERSON|REG AIR|out the packages lose furiously ab 1474|14918|2422|1|5|9164.55|0.05|0.04|A|F|1995-04-22|1995-02-20|1995-05-06|COLLECT COD|SHIP|ully final a 1474|122645|5158|2|30|50029.20|0.04|0.02|A|F|1995-03-23|1995-02-11|1995-04-17|DELIVER IN PERSON|TRUCK|usly. evenly express 1474|91052|6071|3|18|18774.90|0.06|0.02|A|F|1995-01-23|1995-03-28|1995-02-03|NONE|RAIL|after the special 1475|167997|5546|1|15|30974.85|0.08|0.06|N|O|1998-02-12|1997-12-17|1998-03-02|TAKE BACK RETURN|SHIP|xpress requests haggle after the final, fi 1475|117282|7283|2|18|23387.04|0.07|0.00|N|O|1998-03-08|1998-01-18|1998-03-10|TAKE BACK RETURN|AIR|al deposits use. ironic packages along the 1475|143964|6479|3|30|60238.80|0.03|0.02|N|O|1998-03-11|1997-12-30|1998-03-15|COLLECT COD|REG AIR| regular theodolites mold across th 1475|186972|6973|4|50|102948.50|0.03|0.05|N|O|1997-12-14|1997-12-13|1997-12-21|COLLECT COD|AIR|. slyly bold re 1475|31490|6497|5|33|46909.17|0.01|0.06|N|O|1998-01-02|1998-01-27|1998-01-11|NONE|FOB|quickly fluffy 1475|49379|1884|6|12|15940.44|0.04|0.04|N|O|1998-01-09|1997-12-30|1998-01-23|NONE|TRUCK|arefully-- excuses sublate 1475|111789|1790|7|23|41417.94|0.02|0.00|N|O|1998-02-13|1998-02-05|1998-03-08|NONE|TRUCK|hely regular hocke 1476|30547|3051|1|20|29550.80|0.02|0.03|N|O|1996-08-11|1996-09-18|1996-08-26|TAKE BACK RETURN|AIR|. bold deposits are carefully amo 1477|71965|6980|1|31|60045.76|0.00|0.06|N|O|1997-12-16|1997-09-30|1997-12-17|COLLECT COD|RAIL| requests. fluffily final 1477|109605|7136|2|8|12916.80|0.09|0.05|N|O|1997-10-25|1997-10-18|1997-11-16|COLLECT COD|MAIL|ironic realms wake unusual, even ac 1477|124479|4480|3|42|63145.74|0.06|0.00|N|O|1997-11-02|1997-11-02|1997-11-20|DELIVER IN PERSON|SHIP|lithely after the ir 1477|106201|6202|4|32|38630.40|0.05|0.08|N|O|1997-09-12|1997-10-26|1997-10-12|TAKE BACK RETURN|AIR|; quickly regula 1477|114850|4851|5|41|76458.85|0.04|0.06|N|O|1997-12-16|1997-10-31|1998-01-12|DELIVER IN PERSON|REG AIR|y. final pearls kindle. accounts 1477|68931|3944|6|49|93096.57|0.06|0.00|N|O|1997-11-18|1997-11-06|1997-11-27|COLLECT COD|FOB|ise according to the sly, bold p 1477|119012|1524|7|33|34023.33|0.06|0.00|N|O|1997-11-12|1997-11-06|1997-11-24|DELIVER IN PERSON|TRUCK|yly regular p 1478|33650|3651|1|21|33256.65|0.00|0.06|N|O|1997-09-20|1997-10-25|1997-10-06|TAKE BACK RETURN|MAIL| fluffily pending acc 1479|148818|1333|1|33|61604.73|0.10|0.01|N|O|1996-03-12|1996-02-28|1996-03-31|DELIVER IN PERSON|FOB| carefully special courts affix. fluff 1504|81389|3898|1|42|57555.96|0.02|0.03|R|F|1992-10-18|1992-10-14|1992-11-10|TAKE BACK RETURN|FOB|ep. carefully ironic excuses haggle quickl 1504|102385|9916|2|22|30522.36|0.04|0.03|A|F|1992-09-09|1992-10-29|1992-09-10|NONE|REG AIR| accounts sleep. furiou 1504|177148|9666|3|9|11026.26|0.07|0.02|R|F|1992-11-02|1992-10-12|1992-11-15|TAKE BACK RETURN|RAIL|y slyly regular courts. 1504|114073|9096|4|10|10870.70|0.04|0.07|A|F|1992-09-22|1992-10-22|1992-10-13|TAKE BACK RETURN|TRUCK|final theodolites. furiously e 1504|19772|7276|5|7|11842.39|0.02|0.00|R|F|1992-11-20|1992-11-23|1992-12-13|COLLECT COD|MAIL|y final packa 1505|119898|4921|1|4|7671.56|0.09|0.00|A|F|1992-12-14|1992-11-11|1993-01-02|COLLECT COD|SHIP|side of the s 1505|122702|5215|2|50|86235.00|0.00|0.02|R|F|1992-11-22|1992-09-24|1992-11-26|TAKE BACK RETURN|FOB|lyly special platelets. requests ar 1506|132024|2025|1|46|48576.92|0.04|0.05|R|F|1993-01-18|1992-11-11|1993-02-09|COLLECT COD|REG AIR|sits whithout the blithely ironic packages 1506|113893|1427|2|30|57206.70|0.07|0.02|A|F|1992-11-22|1992-10-25|1992-12-04|DELIVER IN PERSON|FOB|deposits cajole 1506|190697|3217|3|28|50055.32|0.10|0.06|A|F|1992-09-22|1992-11-19|1992-10-09|TAKE BACK RETURN|AIR| unwind carefully: theodolit 1506|27798|2803|4|37|63854.23|0.00|0.03|R|F|1992-11-04|1992-12-01|1992-11-23|TAKE BACK RETURN|TRUCK|carefully bold dolphins. accounts su 1506|194708|9747|5|15|27040.50|0.05|0.00|R|F|1992-09-24|1992-11-11|1992-10-05|NONE|REG AIR| carefully fluffy packages-- caref 1506|49549|4558|6|38|56944.52|0.05|0.02|R|F|1992-12-02|1992-12-19|1992-12-29|NONE|REG AIR|xpress, regular excuse 1506|168077|3110|7|4|4580.28|0.07|0.00|R|F|1993-01-03|1992-12-06|1993-01-05|COLLECT COD|REG AIR|posits. furiou 1507|67094|2107|1|25|26527.25|0.01|0.08|R|F|1994-01-07|1994-01-06|1994-01-11|NONE|RAIL|xes. slyly busy de 1507|39977|2481|2|33|63260.01|0.04|0.02|A|F|1993-10-29|1993-12-23|1993-11-14|DELIVER IN PERSON|REG AIR| asymptotes nag furiously above t 1507|85770|8279|3|39|68475.03|0.03|0.07|R|F|1993-11-04|1993-12-16|1993-12-03|TAKE BACK RETURN|REG AIR|ly even instructions. 1508|50350|7866|1|16|20805.60|0.02|0.06|N|O|1998-06-21|1998-05-30|1998-07-11|COLLECT COD|MAIL|riously across the ironic, unusua 1508|24829|9834|2|20|35076.40|0.06|0.01|N|O|1998-04-17|1998-06-11|1998-05-17|DELIVER IN PERSON|MAIL|nic platelets. carefully final fra 1508|92173|9701|3|43|50102.31|0.01|0.02|N|O|1998-06-01|1998-06-24|1998-06-03|TAKE BACK RETURN|TRUCK|ndencies h 1508|147102|4645|4|1|1149.10|0.02|0.02|N|O|1998-07-13|1998-06-03|1998-07-17|TAKE BACK RETURN|AIR|s the blithely bold instruction 1508|134671|9698|5|29|49464.43|0.02|0.00|N|O|1998-08-03|1998-07-08|1998-08-22|COLLECT COD|RAIL|r instructions. carefully 1508|2673|174|6|5|7878.35|0.06|0.08|N|O|1998-05-22|1998-07-06|1998-06-04|COLLECT COD|REG AIR|cording to the furiously ironic depe 1508|116953|6954|7|38|74858.10|0.03|0.06|N|O|1998-04-30|1998-06-23|1998-05-18|DELIVER IN PERSON|RAIL|tes wake furiously regular w 1509|27109|2114|1|14|14505.40|0.04|0.01|A|F|1993-10-04|1993-09-25|1993-10-21|NONE|TRUCK|nal realms 1509|10883|884|2|46|82518.48|0.08|0.02|A|F|1993-10-15|1993-10-04|1993-11-01|TAKE BACK RETURN|FOB|uriously regula 1509|106243|6244|3|17|21237.08|0.06|0.05|A|F|1993-07-25|1993-08-28|1993-08-19|DELIVER IN PERSON|AIR| furiously. blithely regular ideas haggle c 1509|19486|1988|4|11|15460.28|0.03|0.08|R|F|1993-11-04|1993-10-03|1993-11-14|TAKE BACK RETURN|FOB|ily ironic packages nod carefully. 1509|89534|7059|5|37|56370.61|0.01|0.08|A|F|1993-08-31|1993-09-10|1993-09-24|NONE|FOB|he slyly even deposits wake a 1509|186349|3904|6|31|44495.54|0.04|0.03|A|F|1993-07-14|1993-08-21|1993-08-06|COLLECT COD|SHIP|ic deposits cajole carefully. quickly bold 1509|156167|1198|7|27|33025.32|0.01|0.01|A|F|1993-09-29|1993-09-08|1993-10-04|TAKE BACK RETURN|FOB|lithely after the 1510|97465|4993|1|11|16087.06|0.09|0.04|N|O|1996-09-23|1996-12-03|1996-10-01|DELIVER IN PERSON|RAIL|e of the unusual accounts. stealthy deposit 1510|83096|621|2|24|25898.16|0.05|0.04|N|O|1996-10-07|1996-10-22|1996-11-03|DELIVER IN PERSON|REG AIR|yly brave theod 1510|189992|2511|3|36|74951.64|0.07|0.02|N|O|1996-10-02|1996-11-23|1996-10-05|NONE|SHIP|old deposits along the carefully 1510|181428|8983|4|8|12075.36|0.01|0.08|N|O|1996-10-26|1996-11-07|1996-10-30|TAKE BACK RETURN|RAIL|blithely express 1510|58453|8454|5|27|38109.15|0.08|0.06|N|O|1996-10-20|1996-12-05|1996-11-02|NONE|MAIL|he blithely regular req 1510|13698|3699|6|3|4835.07|0.05|0.02|N|O|1996-10-31|1996-12-03|1996-11-13|COLLECT COD|RAIL|along the slyly regular pin 1510|21462|6467|7|50|69173.00|0.04|0.05|N|O|1996-11-01|1996-10-17|1996-11-28|NONE|MAIL|even packages. carefully regular fo 1511|97166|4694|1|29|33731.64|0.01|0.04|N|O|1997-03-17|1997-02-11|1997-03-27|DELIVER IN PERSON|AIR|s cajole furiously against 1511|61718|6731|2|32|53750.72|0.04|0.01|N|O|1997-01-06|1997-03-21|1997-01-26|TAKE BACK RETURN|REG AIR| deposits. carefully ironi 1536|193719|3720|1|5|9063.55|0.08|0.03|N|O|1997-02-08|1997-03-11|1997-03-02|COLLECT COD|MAIL|requests sleep pe 1537|17492|9994|1|17|23961.33|0.01|0.03|A|F|1992-04-12|1992-04-19|1992-04-13|NONE|TRUCK|he regular pack 1537|178379|3414|2|50|72868.50|0.08|0.00|R|F|1992-05-30|1992-05-14|1992-06-23|TAKE BACK RETURN|MAIL|special packages haggle slyly at the silent 1537|12480|2481|3|44|61269.12|0.05|0.04|R|F|1992-04-01|1992-03-31|1992-04-21|NONE|TRUCK|lar courts. 1537|139612|2126|4|3|4954.83|0.08|0.07|R|F|1992-03-20|1992-04-14|1992-03-21|TAKE BACK RETURN|SHIP|s, final ideas detect sl 1538|101743|4254|1|32|55831.68|0.05|0.05|N|O|1995-07-08|1995-07-29|1995-08-01|TAKE BACK RETURN|RAIL|uses maintain blithely. fluffily 1538|191738|1739|2|27|49402.71|0.05|0.01|N|O|1995-09-19|1995-08-03|1995-09-24|DELIVER IN PERSON|TRUCK|ngly even packag 1538|129762|7299|3|36|64503.36|0.08|0.04|N|O|1995-07-11|1995-09-10|1995-07-26|DELIVER IN PERSON|MAIL|al deposits mo 1538|103730|1261|4|28|48544.44|0.10|0.04|N|O|1995-09-19|1995-08-27|1995-10-10|COLLECT COD|RAIL|bout the fluffily unusual 1538|177315|2350|5|13|18100.03|0.01|0.05|N|O|1995-06-26|1995-07-30|1995-07-25|NONE|SHIP|ly. packages sleep f 1538|127205|9718|6|42|51752.40|0.08|0.08|N|O|1995-10-10|1995-09-12|1995-11-08|DELIVER IN PERSON|TRUCK|equests cajole blithely 1539|195722|761|1|21|38172.12|0.08|0.02|R|F|1995-04-19|1995-05-10|1995-04-27|COLLECT COD|TRUCK|ounts haggle. busy 1539|85981|998|2|11|21636.78|0.01|0.08|A|F|1995-05-27|1995-04-13|1995-06-10|TAKE BACK RETURN|TRUCK|ly express requests. furiously 1539|67462|2475|3|7|10006.22|0.09|0.04|R|F|1995-05-14|1995-04-16|1995-05-30|DELIVER IN PERSON|AIR|. fluffily reg 1540|172385|9937|1|38|55380.44|0.03|0.01|R|F|1992-09-30|1992-10-27|1992-10-12|TAKE BACK RETURN|SHIP| final grouches bo 1540|59905|7421|2|35|65271.50|0.02|0.07|R|F|1992-10-31|1992-09-04|1992-11-05|TAKE BACK RETURN|SHIP|e blithely a 1540|7166|2167|3|25|26829.00|0.08|0.04|R|F|1992-11-15|1992-10-24|1992-12-14|DELIVER IN PERSON|SHIP|ironic deposits amo 1540|24603|2110|4|6|9165.60|0.09|0.03|R|F|1992-08-28|1992-09-17|1992-09-14|COLLECT COD|MAIL|ing to the slyly express asymptote 1540|86906|9415|5|27|51108.30|0.10|0.08|R|F|1992-12-02|1992-10-18|1992-12-31|NONE|SHIP|carefully final packages; b 1541|63380|5887|1|44|59108.72|0.10|0.05|N|O|1995-08-24|1995-07-13|1995-08-26|TAKE BACK RETURN|MAIL|o beans boost fluffily abou 1541|25684|5685|2|8|12877.44|0.10|0.08|N|F|1995-06-05|1995-08-07|1995-06-21|TAKE BACK RETURN|TRUCK|y pending packages. blithely fi 1542|57750|7751|1|37|63186.75|0.07|0.06|A|F|1993-12-15|1993-10-17|1994-01-07|TAKE BACK RETURN|REG AIR|e blithely unusual accounts. quic 1542|2488|4989|2|12|16685.76|0.09|0.06|R|F|1993-10-29|1993-11-02|1993-11-09|TAKE BACK RETURN|RAIL|carefully 1542|5566|5567|3|18|26488.08|0.05|0.05|R|F|1993-10-17|1993-11-15|1993-10-26|TAKE BACK RETURN|FOB|pending instr 1542|142665|5180|4|21|35860.86|0.01|0.05|R|F|1993-10-13|1993-12-13|1993-11-12|NONE|RAIL|y pending foxes nag blithely 1542|154955|2501|5|46|92457.70|0.00|0.00|R|F|1993-09-28|1993-11-03|1993-10-15|COLLECT COD|FOB|ial instructions. ironically 1543|70883|5898|1|34|63031.92|0.02|0.08|N|O|1997-05-25|1997-03-30|1997-06-04|NONE|AIR|ic requests are ac 1543|114492|7004|2|6|9038.94|0.09|0.01|N|O|1997-04-16|1997-05-20|1997-05-16|DELIVER IN PERSON|MAIL| among the carefully bold or 1543|66020|6021|3|42|41412.84|0.06|0.01|N|O|1997-05-26|1997-03-30|1997-06-12|DELIVER IN PERSON|FOB|its sleep until the fur 1543|188591|3628|4|42|70542.78|0.05|0.06|N|O|1997-04-11|1997-04-11|1997-04-23|TAKE BACK RETURN|MAIL|xpress instructions. regular acc 1543|39868|9869|5|9|16270.74|0.08|0.06|N|O|1997-03-14|1997-05-19|1997-03-26|DELIVER IN PERSON|FOB|ravely special requests 1543|48782|6295|6|3|5192.34|0.10|0.04|N|O|1997-03-29|1997-05-10|1997-04-22|COLLECT COD|MAIL|sleep along the furiou 1543|67763|270|7|3|5192.28|0.00|0.02|N|O|1997-03-22|1997-04-06|1997-03-30|NONE|AIR|quickly. final accounts haggle slyl 1568|89369|1878|1|36|48900.96|0.02|0.03|N|O|1997-05-31|1997-04-22|1997-06-21|TAKE BACK RETURN|RAIL|platelets-- furiously sly excu 1568|8928|1429|2|46|84498.32|0.04|0.00|N|O|1997-04-06|1997-04-08|1997-04-23|TAKE BACK RETURN|MAIL|g the blithely even acco 1569|74460|1982|1|5|7172.30|0.07|0.00|N|O|1998-04-16|1998-06-21|1998-04-18|COLLECT COD|REG AIR| packages. ironic, even excuses a 1569|38749|8750|2|16|27003.84|0.01|0.08|N|O|1998-04-26|1998-06-16|1998-05-26|COLLECT COD|MAIL|deposits. blithely final asymptotes ac 1569|48818|8819|3|43|75972.83|0.10|0.03|N|O|1998-06-05|1998-05-31|1998-06-28|DELIVER IN PERSON|FOB| instructions. 1569|69193|9194|4|30|34865.70|0.02|0.03|N|O|1998-07-19|1998-06-04|1998-08-10|NONE|SHIP|packages. excuses lose evenly carefully reg 1570|182632|5151|1|25|42865.75|0.00|0.06|N|O|1998-05-03|1998-06-02|1998-06-02|DELIVER IN PERSON|REG AIR|its. slyly regular sentiments 1570|85017|34|2|7|7014.07|0.05|0.05|N|O|1998-07-10|1998-06-01|1998-07-23|TAKE BACK RETURN|MAIL|requests boost quickly re 1571|51371|1372|1|47|62151.39|0.00|0.05|R|F|1992-12-07|1993-02-24|1993-01-01|TAKE BACK RETURN|REG AIR|ng to the fluffily unusual 1571|182725|280|2|6|10846.32|0.03|0.00|A|F|1993-01-08|1993-02-13|1993-02-07|COLLECT COD|SHIP| special, ironic depo 1571|58030|536|3|18|17784.54|0.05|0.08|A|F|1993-01-09|1993-01-12|1993-01-31|COLLECT COD|AIR| pending grouches 1571|100330|2841|4|48|63855.84|0.05|0.05|A|F|1992-12-28|1993-01-04|1993-01-04|DELIVER IN PERSON|RAIL|slyly pending p 1571|41509|6518|5|10|14505.00|0.03|0.06|R|F|1992-12-12|1993-02-13|1992-12-29|DELIVER IN PERSON|AIR|lets. carefully regular ideas wake 1571|33276|786|6|24|29022.48|0.05|0.07|A|F|1993-03-22|1993-01-31|1993-04-09|NONE|TRUCK|warthogs wake carefully acro 1572|23675|3676|1|41|65545.47|0.02|0.00|N|O|1996-05-16|1996-04-09|1996-05-28|TAKE BACK RETURN|REG AIR|. pinto beans alongside 1572|92330|9858|2|10|13223.30|0.04|0.06|N|O|1996-05-17|1996-03-26|1996-05-19|NONE|AIR| accounts affix slyly. 1573|185330|5331|1|5|7076.65|0.05|0.01|A|F|1993-04-24|1993-03-13|1993-05-17|TAKE BACK RETURN|MAIL|ymptotes could u 1573|30683|684|2|17|27432.56|0.00|0.06|R|F|1993-02-24|1993-02-16|1993-03-08|TAKE BACK RETURN|TRUCK|carefully regular deposits. 1573|82204|2205|3|16|18979.20|0.04|0.03|A|F|1993-03-15|1993-03-16|1993-03-31|COLLECT COD|AIR|ely. furiously final requests wake slyl 1573|193749|8788|4|11|20270.14|0.09|0.01|R|F|1993-03-23|1993-03-24|1993-04-12|TAKE BACK RETURN|RAIL|nently pending 1573|136810|1837|5|7|12927.67|0.00|0.01|R|F|1993-01-30|1993-03-14|1993-02-27|DELIVER IN PERSON|SHIP|eodolites sleep slyly. slyly f 1573|153063|609|6|30|33481.80|0.03|0.01|A|F|1992-12-29|1993-03-06|1993-01-02|DELIVER IN PERSON|TRUCK|. blithely even theodolites boos 1574|47706|5219|1|41|67801.70|0.06|0.02|N|O|1997-03-08|1997-02-09|1997-04-01|COLLECT COD|AIR|s. slyly regular depen 1574|190353|7911|2|50|72167.50|0.00|0.05|N|O|1996-12-14|1997-02-14|1996-12-16|TAKE BACK RETURN|FOB|le regular, regular foxes. blithely e 1574|54013|6519|3|25|24175.25|0.06|0.02|N|O|1997-01-16|1997-02-14|1997-02-12|DELIVER IN PERSON|TRUCK|ly silent accounts. 1574|190880|5919|4|6|11825.28|0.03|0.05|N|O|1997-02-24|1997-02-03|1997-03-01|NONE|AIR|e silent, final packages. speci 1574|108301|3322|5|6|7855.80|0.05|0.05|N|O|1997-02-09|1997-03-02|1997-02-14|COLLECT COD|MAIL|nic, final ideas snooze. 1574|4733|2234|6|42|68784.66|0.07|0.01|N|O|1996-12-19|1997-01-13|1996-12-28|NONE|FOB|o beans according t 1574|135755|782|7|14|25070.50|0.04|0.01|N|O|1996-12-30|1997-01-19|1997-01-20|NONE|AIR|ily bold a 1575|28443|8444|1|42|57600.48|0.05|0.08|N|O|1995-10-21|1995-11-25|1995-10-24|DELIVER IN PERSON|RAIL|ly pending pinto beans. 1575|35236|243|2|39|45677.97|0.00|0.06|N|O|1995-10-30|1995-10-15|1995-11-10|COLLECT COD|TRUCK| ironic requests snooze ironic, regular acc 1575|1887|4388|3|12|21466.56|0.01|0.05|N|O|1995-12-27|1995-11-11|1996-01-23|TAKE BACK RETURN|AIR| bold accounts. furi 1575|110162|7696|4|39|45714.24|0.07|0.00|N|O|1995-09-23|1995-11-05|1995-09-25|TAKE BACK RETURN|TRUCK| after the unusual asym 1575|82215|4724|5|10|11972.10|0.09|0.00|N|O|1996-01-10|1995-11-20|1996-01-13|DELIVER IN PERSON|RAIL|k excuses. pinto beans wake a 1575|177660|5212|6|14|24327.24|0.08|0.06|N|O|1995-10-31|1995-12-06|1995-11-30|NONE|AIR|beans breach among the furiously specia 1575|116986|9498|7|48|96143.04|0.08|0.04|N|O|1995-11-19|1995-10-25|1995-12-07|DELIVER IN PERSON|TRUCK|cies. regu 1600|171408|8960|1|20|29588.00|0.02|0.01|R|F|1993-06-16|1993-04-23|1993-07-02|COLLECT COD|FOB|pths sleep blithely about the 1600|43128|641|2|48|51413.76|0.07|0.02|R|F|1993-04-17|1993-04-14|1993-05-03|DELIVER IN PERSON|FOB|furiously silent foxes could wake. car 1600|38465|3472|3|8|11227.68|0.04|0.07|R|F|1993-03-07|1993-04-22|1993-03-26|TAKE BACK RETURN|FOB|cajole furiously fluf 1600|68481|988|4|25|36237.00|0.00|0.06|A|F|1993-05-25|1993-04-07|1993-06-05|TAKE BACK RETURN|REG AIR|press packages. ironic excuses bo 1600|146041|6042|5|30|32611.20|0.03|0.08|R|F|1993-06-03|1993-05-03|1993-06-07|DELIVER IN PERSON|RAIL|al escapades alongside of the depo 1601|166156|6157|1|6|7332.90|0.00|0.00|A|F|1994-10-19|1994-09-28|1994-10-23|COLLECT COD|SHIP| bold sheaves. furiously per 1601|174374|1926|2|50|72418.50|0.03|0.02|R|F|1994-12-24|1994-10-23|1995-01-11|COLLECT COD|FOB|ideas doubt 1601|89887|2396|3|14|26276.32|0.04|0.08|R|F|1994-09-17|1994-11-22|1994-10-03|DELIVER IN PERSON|RAIL|he special, fin 1602|182806|2807|1|4|7555.20|0.08|0.06|R|F|1993-10-31|1993-09-05|1993-11-21|NONE|RAIL|y. even excuses 1603|38191|5701|1|1|1129.19|0.08|0.00|R|F|1993-08-17|1993-09-04|1993-08-22|TAKE BACK RETURN|REG AIR|d accounts. special warthogs use fur 1603|65209|7716|2|29|34051.80|0.06|0.08|A|F|1993-09-28|1993-09-20|1993-10-28|NONE|SHIP|ses wake furiously. theodolite 1604|41563|1564|1|15|22568.40|0.09|0.08|R|F|1993-09-22|1993-09-03|1993-09-29|TAKE BACK RETURN|MAIL| instructions haggle 1604|140413|5442|2|37|53776.17|0.06|0.06|A|F|1993-08-22|1993-09-21|1993-09-10|COLLECT COD|SHIP|requests. blithely ironic somas s 1604|113214|5726|3|19|23316.99|0.09|0.07|A|F|1993-10-15|1993-10-04|1993-11-09|COLLECT COD|RAIL| ideas. bol 1604|174239|9274|4|15|19698.45|0.03|0.00|R|F|1993-09-10|1993-08-31|1993-09-30|TAKE BACK RETURN|RAIL|ending realms along the special, p 1604|20091|7598|5|23|23255.07|0.08|0.05|A|F|1993-10-11|1993-08-30|1993-10-18|DELIVER IN PERSON|RAIL|en requests. blithely fin 1605|141821|9364|1|47|87552.54|0.00|0.01|N|O|1998-04-29|1998-06-12|1998-05-20|DELIVER IN PERSON|AIR|. carefully r 1605|179691|7243|2|18|31872.42|0.10|0.00|N|O|1998-05-13|1998-06-17|1998-06-03|COLLECT COD|REG AIR|ly regular foxes wake carefully. bol 1605|58546|8547|3|39|58677.06|0.02|0.03|N|O|1998-07-12|1998-06-05|1998-08-09|DELIVER IN PERSON|MAIL|nal dependencies-- quickly final frets acc 1605|182574|129|4|25|41414.25|0.06|0.02|N|O|1998-05-26|1998-06-14|1998-06-05|COLLECT COD|AIR|ole carefully car 1606|114082|4083|1|21|23017.68|0.04|0.00|N|O|1997-06-02|1997-07-02|1997-06-27|DELIVER IN PERSON|RAIL| pending theodolites prom 1606|173565|8600|2|35|57349.60|0.00|0.02|N|O|1997-06-20|1997-06-19|1997-06-22|COLLECT COD|TRUCK|carefully sil 1606|99477|7005|3|23|33958.81|0.00|0.06|N|O|1997-04-19|1997-06-26|1997-04-30|NONE|MAIL|ously final requests. slowly ironic ex 1606|96286|8796|4|20|25645.60|0.02|0.04|N|O|1997-05-01|1997-05-26|1997-05-28|TAKE BACK RETURN|TRUCK|fily carefu 1606|70181|5196|5|14|16116.52|0.10|0.01|N|O|1997-05-19|1997-07-05|1997-06-10|COLLECT COD|FOB|structions haggle f 1607|189131|6686|1|2|2440.26|0.02|0.00|N|O|1996-01-11|1996-02-15|1996-01-19|DELIVER IN PERSON|MAIL|packages haggle. regular requests boost s 1607|118923|1435|2|37|71851.04|0.05|0.02|N|O|1996-02-27|1996-02-18|1996-03-16|NONE|AIR|alongside 1607|122680|2681|3|39|66404.52|0.00|0.00|N|O|1996-02-01|1996-02-12|1996-02-16|NONE|FOB|uches cajole. accounts ar 1607|75587|8095|4|34|53127.72|0.05|0.06|N|O|1996-01-06|1996-02-24|1996-01-10|DELIVER IN PERSON|SHIP| quickly above the 1607|177948|466|5|48|97245.12|0.00|0.05|N|O|1996-02-22|1996-02-13|1996-03-09|TAKE BACK RETURN|MAIL|ular forges. deposits a 1632|190513|8071|1|47|75364.97|0.08|0.00|N|O|1997-01-25|1997-02-09|1997-02-19|TAKE BACK RETURN|RAIL|g to the closely special no 1632|147914|5457|2|14|27466.74|0.08|0.05|N|O|1997-01-15|1997-02-25|1997-01-28|NONE|RAIL|oxes. deposits nag slyly along the slyly 1632|176169|1204|3|47|58522.52|0.03|0.04|N|O|1997-01-29|1997-03-03|1997-02-21|NONE|MAIL|sts. blithely regular 1632|56164|3680|4|33|36965.28|0.09|0.02|N|O|1997-04-01|1997-02-24|1997-04-29|TAKE BACK RETURN|REG AIR|ructions! slyly 1632|141072|8615|5|43|47862.01|0.10|0.03|N|O|1997-02-24|1997-02-19|1997-03-25|DELIVER IN PERSON|FOB|ts. blithe, bold ideas cajo 1633|177903|2938|1|35|69331.50|0.01|0.02|N|O|1996-01-09|1995-12-02|1996-01-21|COLLECT COD|REG AIR|ly against the dolph 1633|4124|4125|2|15|15421.80|0.00|0.05|N|O|1995-12-13|1995-11-13|1996-01-04|TAKE BACK RETURN|FOB|ges wake fluffil 1634|47063|7064|1|21|21211.26|0.00|0.00|N|O|1996-10-04|1996-10-22|1996-11-01|NONE|MAIL|counts alo 1634|171632|1633|2|44|74959.72|0.05|0.01|N|O|1996-09-17|1996-11-09|1996-10-03|COLLECT COD|SHIP|requests affix slyly. quickly even pack 1634|18568|8569|3|21|31217.76|0.06|0.07|N|O|1996-11-16|1996-10-21|1996-11-27|NONE|TRUCK|y along the excuses. 1634|67465|4984|4|17|24351.82|0.08|0.07|N|O|1996-10-29|1996-10-15|1996-11-02|TAKE BACK RETURN|SHIP|cial, bold platelets alongside of the f 1634|75908|5909|5|2|3767.80|0.07|0.04|N|O|1996-11-22|1996-10-28|1996-12-17|NONE|SHIP|ly. carefully regular asymptotes wake 1634|169681|2198|6|11|19257.48|0.01|0.08|N|O|1996-10-04|1996-12-06|1996-10-14|DELIVER IN PERSON|SHIP|final requests 1634|12416|4918|7|35|46494.35|0.06|0.02|N|O|1996-11-25|1996-11-25|1996-12-12|TAKE BACK RETURN|RAIL|cies. regular, special de 1635|70167|2675|1|3|3411.48|0.06|0.08|N|O|1997-03-13|1997-03-25|1997-03-27|COLLECT COD|FOB| quickly ironic r 1635|89018|9019|2|8|8056.08|0.04|0.05|N|O|1997-04-30|1997-04-21|1997-05-09|DELIVER IN PERSON|AIR|ravely carefully express 1635|113791|3792|3|20|36095.80|0.07|0.01|N|O|1997-05-19|1997-04-01|1997-06-17|TAKE BACK RETURN|FOB|oost according to the carefully even accou 1635|76063|3585|4|40|41562.40|0.01|0.04|N|O|1997-02-25|1997-03-20|1997-03-12|TAKE BACK RETURN|RAIL|uriously up the ironic deposits. slyly i 1636|84083|1608|1|2|2134.16|0.09|0.03|N|O|1997-09-26|1997-08-22|1997-10-05|NONE|TRUCK|nal foxes cajole above the blithely reg 1636|168925|8926|2|45|89726.40|0.03|0.01|N|O|1997-07-14|1997-08-08|1997-07-27|COLLECT COD|RAIL|ely express reque 1636|107869|380|3|24|45044.64|0.07|0.08|N|O|1997-10-07|1997-08-12|1997-11-04|TAKE BACK RETURN|MAIL|e carefully unusual ideas are f 1636|152367|4883|4|43|61032.48|0.06|0.00|N|O|1997-08-23|1997-08-10|1997-09-17|NONE|REG AIR|blithely special r 1636|18378|3381|5|22|28520.14|0.05|0.02|N|O|1997-07-22|1997-08-18|1997-08-03|COLLECT COD|AIR|ular, regu 1636|62910|5417|6|34|63678.94|0.10|0.01|N|O|1997-08-11|1997-09-09|1997-08-23|NONE|TRUCK|ular depos 1636|113018|8041|7|7|7217.07|0.04|0.00|N|O|1997-07-28|1997-09-10|1997-07-31|NONE|MAIL|ronic instructions. final 1637|85755|5756|1|49|85296.75|0.02|0.03|N|F|1995-06-08|1995-04-19|1995-07-01|COLLECT COD|REG AIR|. blithely i 1637|72269|7284|2|1|1241.26|0.10|0.02|A|F|1995-02-14|1995-03-26|1995-03-09|TAKE BACK RETURN|AIR|ly final pinto beans. furiously 1637|21652|6657|3|10|15736.50|0.02|0.05|R|F|1995-02-21|1995-03-17|1995-03-11|NONE|AIR|uriously? blithely even sauternes wake. 1637|92629|5139|4|42|68108.04|0.06|0.01|A|F|1995-03-18|1995-04-24|1995-03-31|COLLECT COD|SHIP|blithely a 1637|4005|6506|5|25|22725.00|0.05|0.00|R|F|1995-06-07|1995-03-26|1995-06-08|COLLECT COD|RAIL| haggle carefully silent accou 1637|108780|3801|6|38|67973.64|0.02|0.08|R|F|1995-03-20|1995-05-05|1995-04-14|DELIVER IN PERSON|SHIP|even, pending foxes nod regular 1637|51705|4211|7|21|34790.70|0.07|0.08|A|F|1995-04-30|1995-04-30|1995-05-05|COLLECT COD|SHIP|ly ironic theodolites use b 1638|5840|5841|1|46|80308.64|0.03|0.02|N|O|1997-10-16|1997-10-28|1997-11-09|COLLECT COD|MAIL|otes haggle before the slyly bold instructi 1638|148976|8977|2|30|60749.10|0.00|0.04|N|O|1997-12-05|1997-09-17|1997-12-06|NONE|REG AIR|s cajole boldly bold requests. closely 1638|30158|2662|3|5|5440.75|0.08|0.07|N|O|1997-10-15|1997-11-01|1997-11-08|DELIVER IN PERSON|FOB|xcuses sleep furiou 1638|55974|3490|4|19|36669.43|0.00|0.08|N|O|1997-10-15|1997-10-27|1997-11-03|DELIVER IN PERSON|MAIL| quickly expres 1638|142867|7896|5|25|47746.50|0.05|0.03|N|O|1997-10-06|1997-09-30|1997-11-02|DELIVER IN PERSON|REG AIR|gle final, ironic pinto beans. 1638|154436|9467|6|46|68559.78|0.07|0.08|N|O|1997-08-20|1997-10-10|1997-09-09|COLLECT COD|AIR|ckages are carefully even instru 1639|186473|6474|1|24|37427.28|0.07|0.00|N|O|1995-08-24|1995-10-06|1995-08-31|COLLECT COD|REG AIR| the regular packages. courts dou 1639|42945|7954|2|38|71741.72|0.01|0.04|N|O|1995-08-23|1995-11-09|1995-08-29|TAKE BACK RETURN|FOB|y regular packages. b 1639|170376|5411|3|41|59301.17|0.04|0.02|N|O|1995-12-19|1995-11-11|1996-01-12|DELIVER IN PERSON|FOB|structions w 1664|117307|2330|1|48|63566.40|0.04|0.02|N|O|1996-06-21|1996-05-01|1996-07-19|TAKE BACK RETURN|RAIL| use. ironic deposits integrate. slyly unu 1664|172152|7187|2|30|36724.50|0.06|0.05|N|O|1996-04-04|1996-05-04|1996-05-03|COLLECT COD|FOB|ess multip 1664|150899|900|3|10|19498.90|0.00|0.06|N|O|1996-04-10|1996-05-13|1996-05-07|TAKE BACK RETURN|RAIL|instructions up the acc 1664|154273|6789|4|35|46454.45|0.00|0.04|N|O|1996-03-06|1996-05-16|1996-03-09|DELIVER IN PERSON|REG AIR|y regular ide 1664|56747|6748|5|9|15333.66|0.07|0.04|N|O|1996-04-15|1996-05-14|1996-05-11|DELIVER IN PERSON|TRUCK|ges. fluffil 1664|140115|2630|6|40|46204.40|0.09|0.07|N|O|1996-04-02|1996-04-22|1996-04-17|COLLECT COD|REG AIR|se blithely unusual pains. carefully 1665|46402|3915|1|4|5393.60|0.02|0.03|A|F|1994-09-01|1994-06-07|1994-09-12|DELIVER IN PERSON|TRUCK|ely final requests. requests 1665|77098|4620|2|1|1075.09|0.03|0.05|R|F|1994-05-22|1994-07-06|1994-05-24|TAKE BACK RETURN|TRUCK|sly final p 1666|184344|1899|1|30|42850.20|0.04|0.03|N|O|1995-10-28|1995-11-30|1995-11-18|TAKE BACK RETURN|AIR| breach evenly final accounts. r 1666|63994|9007|2|20|39159.80|0.01|0.00|N|O|1996-01-27|1995-12-12|1996-01-31|NONE|REG AIR|uietly regular foxes wake quick 1666|133479|1019|3|31|46886.57|0.05|0.07|N|O|1996-02-11|1996-01-11|1996-02-28|COLLECT COD|RAIL|ding to the express, bold accounts. fu 1666|168802|1319|4|41|76702.80|0.06|0.08|N|O|1995-11-29|1996-01-04|1995-12-24|NONE|TRUCK|ly regular excuses; regular ac 1667|20267|7774|1|6|7123.56|0.04|0.02|N|O|1997-12-07|1997-11-16|1998-01-02|COLLECT COD|FOB|riously busy requests. blithely final a 1667|21164|6169|2|29|31469.64|0.06|0.07|N|O|1997-10-15|1997-11-09|1997-11-11|TAKE BACK RETURN|MAIL|l accounts. furiously final courts h 1667|94941|9960|3|48|92925.12|0.05|0.01|N|O|1998-01-27|1998-01-06|1998-02-09|TAKE BACK RETURN|SHIP|tes sleep furiously. carefully eve 1667|58119|5635|4|24|25850.64|0.04|0.01|N|O|1997-10-14|1997-12-01|1997-11-09|TAKE BACK RETURN|MAIL|hrash final requests. care 1667|194315|1873|5|2|2818.62|0.07|0.00|N|O|1997-12-17|1997-11-22|1998-01-16|NONE|SHIP|pecial requests hag 1667|47733|5246|6|6|10084.38|0.01|0.03|N|O|1998-01-21|1997-12-19|1998-01-28|NONE|TRUCK| nag quickly above th 1667|39956|2460|7|19|36023.05|0.09|0.03|N|O|1998-01-23|1997-11-24|1998-01-26|DELIVER IN PERSON|SHIP|around the pinto beans. express, special 1668|131636|4150|1|8|13341.04|0.06|0.01|N|O|1997-07-23|1997-10-09|1997-08-06|DELIVER IN PERSON|FOB|arefully regular tithes! slyl 1668|945|8446|2|25|46148.50|0.01|0.06|N|O|1997-08-08|1997-09-28|1997-09-01|NONE|TRUCK|y ironic requests. bold, final ideas a 1668|74160|6668|3|42|47634.72|0.08|0.01|N|O|1997-08-09|1997-09-08|1997-08-31|NONE|FOB|ole carefully excuses. final 1668|190774|8332|4|9|16782.93|0.05|0.03|N|O|1997-10-17|1997-09-05|1997-11-01|COLLECT COD|RAIL|wake furiously even instructions. sil 1668|127924|7925|5|25|48798.00|0.01|0.02|N|O|1997-10-08|1997-09-20|1997-10-11|DELIVER IN PERSON|REG AIR|even platelets across the silent 1668|9920|2421|6|38|69536.96|0.07|0.01|N|O|1997-08-26|1997-09-17|1997-09-05|DELIVER IN PERSON|TRUCK|ep slyly across the furi 1669|78373|8374|1|24|32432.88|0.04|0.08|N|O|1997-09-04|1997-07-30|1997-09-20|DELIVER IN PERSON|RAIL| regular, final deposits use quick 1670|31411|6418|1|41|55038.81|0.07|0.01|N|O|1997-07-19|1997-08-20|1997-07-23|DELIVER IN PERSON|TRUCK|thely according to the sly 1670|121378|1379|2|10|13993.70|0.07|0.03|N|O|1997-09-14|1997-08-16|1997-09-23|NONE|SHIP|fily special ideas 1670|185284|321|3|41|56140.48|0.07|0.07|N|O|1997-07-19|1997-08-05|1997-07-26|COLLECT COD|SHIP|al gifts. speci 1671|148885|3914|1|21|40611.48|0.02|0.07|N|O|1996-07-28|1996-09-28|1996-08-08|TAKE BACK RETURN|AIR|s accounts slee 1671|95111|2639|2|4|4424.44|0.05|0.00|N|O|1996-08-30|1996-09-19|1996-09-23|DELIVER IN PERSON|TRUCK|lyly regular ac 1671|123591|8616|3|11|17760.49|0.06|0.08|N|O|1996-09-16|1996-10-21|1996-09-18|NONE|SHIP|tes sleep blithely 1671|177303|2338|4|5|6901.50|0.00|0.00|N|O|1996-11-14|1996-10-20|1996-11-25|TAKE BACK RETURN|FOB|luffily regular deposits 1671|126590|6591|5|12|19399.08|0.07|0.04|N|O|1996-11-17|1996-09-02|1996-12-17|COLLECT COD|RAIL|special, ironic 1671|196217|8737|6|46|60407.66|0.08|0.05|N|O|1996-09-13|1996-10-14|1996-09-28|TAKE BACK RETURN|REG AIR|. slyly bold instructions boost. furiousl 1696|15419|422|1|8|10675.28|0.04|0.02|N|O|1998-04-28|1998-02-07|1998-05-10|NONE|TRUCK|the blithely 1696|138312|826|2|13|17554.03|0.08|0.06|N|O|1998-03-01|1998-03-25|1998-03-24|TAKE BACK RETURN|TRUCK|tructions play slyly q 1696|1107|3608|3|19|19153.90|0.08|0.05|N|O|1998-05-03|1998-03-13|1998-05-28|TAKE BACK RETURN|REG AIR|its maintain alongside of the f 1696|192712|2713|4|21|37898.91|0.05|0.00|N|O|1998-05-04|1998-02-18|1998-05-07|NONE|MAIL|y players sleep along the final, pending 1696|93788|8807|5|43|76616.54|0.03|0.06|N|O|1998-02-14|1998-03-29|1998-02-20|COLLECT COD|FOB|arefully regular dep 1697|74452|6960|1|6|8558.70|0.05|0.00|N|O|1997-01-28|1996-11-27|1997-01-31|NONE|FOB|accounts breach slyly even de 1697|103417|5928|2|24|34089.84|0.00|0.08|N|O|1996-12-29|1996-12-19|1997-01-10|NONE|SHIP|ts cajole carefully above the carefully 1697|123693|6206|3|27|46350.63|0.06|0.00|N|O|1997-01-20|1996-12-02|1997-02-05|COLLECT COD|MAIL|ly regular packages across the silent, b 1697|93083|3084|4|49|52727.92|0.08|0.04|N|O|1996-12-07|1997-01-02|1996-12-31|COLLECT COD|TRUCK|lar foxes. fluffily furious ideas doubt qu 1697|34808|7312|5|19|33113.20|0.03|0.07|N|O|1997-01-08|1996-11-12|1997-01-11|DELIVER IN PERSON|FOB|ons? special, special accounts after 1698|96537|6538|1|44|67475.32|0.05|0.05|N|O|1997-05-16|1997-07-05|1997-05-27|NONE|RAIL|ts wake slyly after t 1698|92106|4616|2|6|6588.60|0.08|0.00|N|O|1997-08-21|1997-06-08|1997-09-03|DELIVER IN PERSON|RAIL| pending packages affix ne 1698|20546|3049|3|22|32263.88|0.03|0.04|N|O|1997-08-07|1997-05-28|1997-08-24|DELIVER IN PERSON|TRUCK|oward the furiously iro 1698|111909|4421|4|19|36497.10|0.00|0.07|N|O|1997-07-04|1997-06-21|1997-08-01|NONE|RAIL| fluffily e 1698|52417|2418|5|37|50668.17|0.00|0.03|N|O|1997-05-16|1997-05-29|1997-05-27|NONE|AIR|ly regular ideas. deposit 1698|165824|5825|6|15|28347.30|0.10|0.01|N|O|1997-07-20|1997-06-07|1997-07-21|TAKE BACK RETURN|RAIL|final ideas. even, ironic 1699|37350|2357|1|50|64367.50|0.00|0.06|A|F|1994-03-26|1994-03-23|1994-04-20|NONE|FOB|to the final requests are carefully silent 1699|134825|4826|2|17|31616.94|0.07|0.02|R|F|1994-01-12|1994-03-12|1994-02-08|NONE|AIR|haggle blithely slyly 1700|139702|4729|1|38|66184.60|0.04|0.04|N|O|1996-10-03|1996-07-27|1996-10-22|NONE|RAIL|ular dependencies engage slyly 1700|155943|5944|2|49|97948.06|0.04|0.00|N|O|1996-09-26|1996-07-28|1996-10-16|NONE|TRUCK|kly even dependencies haggle fluffi 1701|149418|6961|1|47|68968.27|0.08|0.05|R|F|1992-05-25|1992-06-29|1992-06-15|NONE|RAIL|slyly final requests cajole requests. f 1701|53004|3005|2|2|1914.00|0.01|0.04|R|F|1992-06-24|1992-07-12|1992-06-29|COLLECT COD|SHIP|ween the pending, final accounts. 1701|34683|7187|3|26|42059.68|0.10|0.06|R|F|1992-06-04|1992-07-11|1992-07-04|DELIVER IN PERSON|FOB| accounts. blithely pending pinto be 1702|66974|4493|1|19|36878.43|0.02|0.01|N|F|1995-06-02|1995-06-30|1995-06-29|NONE|REG AIR|ies haggle blith 1702|29762|2265|2|38|64286.88|0.00|0.00|N|O|1995-09-01|1995-06-10|1995-09-10|DELIVER IN PERSON|REG AIR|as believe blithely. bo 1702|194425|4426|3|46|69893.32|0.00|0.08|N|O|1995-07-14|1995-06-30|1995-07-20|NONE|FOB|y even foxes. carefully final dependencies 1702|92039|2040|4|28|28868.84|0.07|0.05|R|F|1995-06-10|1995-07-26|1995-06-16|TAKE BACK RETURN|AIR|nts haggle along the packa 1702|88038|8039|5|34|34885.02|0.01|0.06|N|O|1995-07-04|1995-06-08|1995-07-28|DELIVER IN PERSON|AIR|y careful packages; dogged acco 1702|41506|4011|6|28|40530.00|0.10|0.00|N|O|1995-08-14|1995-07-31|1995-09-08|COLLECT COD|RAIL|ackages sleep. furiously even excuses snooz 1703|165404|7921|1|36|52898.40|0.09|0.01|R|F|1993-04-22|1993-03-05|1993-04-24|DELIVER IN PERSON|SHIP|riously express 1703|136118|1145|2|35|40393.85|0.01|0.08|R|F|1993-04-14|1993-03-31|1993-04-27|NONE|RAIL|he carefully 1703|123460|3461|3|48|71206.08|0.06|0.02|R|F|1993-02-07|1993-04-20|1993-02-24|TAKE BACK RETURN|AIR|ggle slyly furiously regular theodol 1728|125205|230|1|1|1230.20|0.07|0.04|N|O|1996-09-16|1996-08-19|1996-09-18|COLLECT COD|FOB|lly. carefully ex 1728|104383|6894|2|23|31909.74|0.05|0.02|N|O|1996-09-08|1996-07-24|1996-09-20|NONE|FOB|ns. pending, final ac 1728|164065|1614|3|44|49678.64|0.08|0.07|N|O|1996-07-31|1996-06-22|1996-08-06|COLLECT COD|FOB|ide of the slyly blithe 1728|26347|6348|4|34|43293.56|0.08|0.05|N|O|1996-08-28|1996-07-20|1996-09-12|DELIVER IN PERSON|MAIL|special req 1728|198088|3127|5|31|36768.48|0.09|0.02|N|O|1996-07-26|1996-06-28|1996-08-14|NONE|REG AIR|kly sly theodolites. 1729|156948|6949|1|12|24059.28|0.08|0.04|A|F|1992-08-11|1992-07-24|1992-08-16|COLLECT COD|RAIL|y pending packages detect. carefully re 1730|165429|7946|1|41|61271.22|0.01|0.03|N|O|1998-08-11|1998-08-29|1998-09-02|TAKE BACK RETURN|TRUCK| instructions. unusual, even Tiresi 1730|161312|1313|2|15|20599.65|0.07|0.04|N|O|1998-09-07|1998-09-12|1998-09-30|TAKE BACK RETURN|AIR|pinto beans cajole. bravely bold 1730|161029|3546|3|9|9810.18|0.10|0.00|N|O|1998-09-18|1998-09-15|1998-09-21|DELIVER IN PERSON|FOB|gular dependencies wake. blithely final e 1730|9457|6958|4|40|54658.00|0.02|0.03|N|O|1998-10-02|1998-10-06|1998-10-03|NONE|SHIP|ven dinos slee 1730|140291|5320|5|43|57245.47|0.04|0.06|N|O|1998-10-26|1998-10-22|1998-11-02|DELIVER IN PERSON|TRUCK|ng deposits cajo 1731|183932|3933|1|36|72573.48|0.10|0.00|N|O|1996-04-18|1996-04-03|1996-04-29|TAKE BACK RETURN|MAIL|ngside of the even instruct 1731|138267|3294|2|7|9136.82|0.04|0.07|N|O|1996-04-11|1996-02-13|1996-04-30|DELIVER IN PERSON|REG AIR|fily quick asymptotes 1731|50920|3426|3|50|93546.00|0.05|0.04|N|O|1996-01-14|1996-03-13|1996-01-29|COLLECT COD|RAIL|ly slyly speci 1731|195874|3432|4|23|45307.01|0.10|0.04|N|O|1996-04-22|1996-02-25|1996-05-16|TAKE BACK RETURN|RAIL|rays? bold, express pac 1731|52355|2356|5|37|48371.95|0.10|0.05|N|O|1996-04-30|1996-03-17|1996-05-27|TAKE BACK RETURN|RAIL| beans use furiously slyly b 1731|123416|953|6|41|59015.81|0.03|0.08|N|O|1996-04-05|1996-02-28|1996-05-01|TAKE BACK RETURN|RAIL|haggle across the blithely ironi 1732|4397|4398|1|50|65069.50|0.02|0.01|R|F|1993-12-05|1994-01-23|1993-12-20|TAKE BACK RETURN|FOB|fily final asymptotes according 1732|98183|8184|2|36|42522.48|0.01|0.03|A|F|1994-03-15|1994-02-09|1994-04-02|DELIVER IN PERSON|TRUCK|ve the accounts. slowly ironic multip 1732|160895|5928|3|41|80191.49|0.00|0.04|R|F|1994-02-20|1994-01-07|1994-02-27|TAKE BACK RETURN|AIR|quests sublate against the silent 1732|151257|1258|4|9|11774.25|0.04|0.04|A|F|1994-02-25|1994-01-29|1994-03-16|TAKE BACK RETURN|FOB|ular platelets. deposits wak 1732|168899|1416|5|25|49197.25|0.02|0.05|A|F|1994-02-15|1994-01-07|1994-02-21|COLLECT COD|REG AIR|nag slyly. even, special de 1732|72930|452|6|16|30446.88|0.01|0.05|R|F|1994-01-07|1994-01-02|1994-01-25|COLLECT COD|SHIP|ix carefully at the furiously regular pac 1733|110444|2956|1|41|59632.04|0.08|0.01|N|O|1996-06-13|1996-07-08|1996-07-07|TAKE BACK RETURN|AIR|ess notornis. fur 1733|23938|1445|2|16|29790.88|0.00|0.04|N|O|1996-08-28|1996-07-25|1996-09-27|COLLECT COD|MAIL|slyly express deposits sleep abo 1733|119673|7207|3|29|49087.43|0.10|0.06|N|O|1996-07-16|1996-08-08|1996-07-28|NONE|TRUCK|ns detect among the special accounts. qu 1733|135517|544|4|38|58995.38|0.01|0.03|N|O|1996-08-26|1996-07-23|1996-08-28|NONE|FOB| deposits 1733|33586|8593|5|22|33430.76|0.06|0.07|N|O|1996-07-16|1996-07-24|1996-07-30|COLLECT COD|AIR|gainst the final deposits. carefully final 1733|65422|5423|6|9|12486.78|0.06|0.08|N|O|1996-05-25|1996-07-23|1996-06-10|COLLECT COD|TRUCK|ven foxes was according to t 1733|145889|918|7|13|25153.44|0.02|0.03|N|O|1996-08-03|1996-08-02|1996-08-18|NONE|MAIL|olites sleep furious 1734|154883|7399|1|38|73639.44|0.03|0.03|R|F|1994-08-09|1994-09-07|1994-08-12|COLLECT COD|FOB|ts doubt b 1734|117726|238|2|4|6974.88|0.06|0.03|A|F|1994-08-20|1994-07-17|1994-08-25|DELIVER IN PERSON|AIR|final warhorses. 1735|155456|5457|1|43|64992.35|0.02|0.06|A|F|1993-01-14|1993-03-25|1993-02-02|DELIVER IN PERSON|FOB|iously after the 1735|138514|1028|2|49|76072.99|0.03|0.04|A|F|1992-12-31|1993-02-03|1993-01-25|TAKE BACK RETURN|TRUCK|y express accounts above the exp 1760|95414|433|1|38|53557.58|0.09|0.03|N|O|1996-06-15|1996-06-29|1996-07-11|NONE|MAIL|tions. blithely regular orbits against the 1760|7256|7257|2|3|3489.75|0.00|0.06|N|O|1996-07-18|1996-07-01|1996-08-01|NONE|RAIL|lyly bold dolphins haggle carefully. sl 1760|136818|1845|3|44|81611.64|0.05|0.01|N|O|1996-06-11|1996-06-16|1996-07-02|COLLECT COD|REG AIR|instructions poach slyly ironic theodolites 1761|51042|8558|1|33|32770.32|0.09|0.03|R|F|1994-01-03|1994-01-23|1994-01-31|NONE|FOB|s. excuses a 1761|51137|1138|2|37|40260.81|0.02|0.07|R|F|1994-02-17|1994-03-08|1994-03-16|NONE|RAIL| integrate. quickly unusual 1761|48901|1406|3|37|68446.30|0.06|0.04|R|F|1994-01-02|1994-03-12|1994-01-25|DELIVER IN PERSON|TRUCK|regular packages wake after 1761|72503|25|4|49|72299.50|0.06|0.07|R|F|1994-01-08|1994-03-03|1994-02-05|TAKE BACK RETURN|FOB|y even packages promise 1761|156877|9393|5|37|71553.19|0.03|0.04|R|F|1994-04-24|1994-03-14|1994-04-29|TAKE BACK RETURN|MAIL|express requests print blithely around the 1761|23419|926|6|12|16108.92|0.01|0.05|A|F|1994-04-16|1994-03-08|1994-04-21|DELIVER IN PERSON|AIR| sleep furiously. deposits are acco 1761|811|5812|7|13|22253.53|0.03|0.08|R|F|1994-03-06|1994-03-18|1994-03-22|DELIVER IN PERSON|TRUCK|ons boost fu 1762|25243|248|1|15|17523.60|0.04|0.08|A|F|1994-12-18|1994-10-29|1995-01-17|TAKE BACK RETURN|REG AIR|old packages thrash. care 1762|49506|4515|2|39|56764.50|0.10|0.02|A|F|1994-09-12|1994-11-09|1994-10-08|DELIVER IN PERSON|MAIL| ironic platelets sleep along t 1762|31865|4369|3|7|12578.02|0.05|0.01|R|F|1994-09-03|1994-10-02|1994-09-10|NONE|REG AIR|uickly express packages wake slyly-- regul 1762|144131|6646|4|24|28203.12|0.03|0.03|A|F|1994-11-30|1994-11-02|1994-12-20|NONE|REG AIR|accounts solve alongside of the fluffily 1762|7408|7409|5|49|64454.60|0.08|0.05|A|F|1994-10-20|1994-11-02|1994-11-10|TAKE BACK RETURN|SHIP| packages sleep fluffily pen 1762|93430|8449|6|35|49820.05|0.05|0.05|A|F|1994-11-25|1994-10-21|1994-11-28|COLLECT COD|AIR|ind quickly. accounts ca 1762|72971|5479|7|47|91366.59|0.03|0.01|A|F|1994-11-02|1994-10-07|1994-11-08|NONE|SHIP| blithely brave 1763|11731|6734|1|22|36140.06|0.09|0.06|N|O|1997-01-17|1997-01-15|1997-02-03|TAKE BACK RETURN|SHIP|ld. fluffily final ideas boos 1763|156496|9012|2|43|66757.07|0.04|0.04|N|O|1996-11-04|1996-12-09|1996-11-28|DELIVER IN PERSON|FOB|r deposits integrate blithely pending, quic 1763|24137|6640|3|16|16978.08|0.06|0.02|N|O|1996-12-12|1996-12-04|1996-12-25|DELIVER IN PERSON|RAIL|ously pending asymptotes a 1763|60187|7706|4|44|50475.92|0.04|0.05|N|O|1996-12-04|1997-01-06|1996-12-25|DELIVER IN PERSON|REG AIR| instructions need to integrate deposits. 1763|146555|9070|5|13|20820.15|0.03|0.05|N|O|1996-11-23|1997-01-24|1996-12-05|TAKE BACK RETURN|SHIP|s sleep carefully. fluffily unusua 1763|142048|2049|6|3|3270.12|0.05|0.03|N|O|1996-12-10|1996-12-06|1997-01-04|TAKE BACK RETURN|FOB|ut the slyly pending deposi 1763|183931|1486|7|2|4029.86|0.05|0.07|N|O|1997-02-27|1996-12-04|1997-03-27|COLLECT COD|FOB|even pinto beans snooze fluffi 1764|120300|301|1|20|26406.00|0.09|0.02|A|F|1992-06-09|1992-05-22|1992-07-06|COLLECT COD|MAIL|y quickly regular packages. car 1764|66465|1478|2|3|4294.38|0.07|0.07|R|F|1992-05-13|1992-06-07|1992-05-26|COLLECT COD|RAIL|es wake slowly. 1764|77176|4698|3|27|31135.59|0.07|0.04|A|F|1992-05-06|1992-05-11|1992-05-23|COLLECT COD|TRUCK|ly final foxes wake blithely even requests 1765|160566|567|1|36|58556.16|0.08|0.04|N|O|1996-03-02|1996-02-17|1996-03-14|DELIVER IN PERSON|SHIP|he blithely pending accou 1766|86716|6717|1|32|54486.72|0.08|0.01|N|O|1997-01-08|1996-11-11|1997-01-31|TAKE BACK RETURN|AIR|ess accounts. stealthily ironic accou 1766|33969|6473|2|12|22835.52|0.05|0.01|N|O|1996-10-28|1996-12-18|1996-11-15|DELIVER IN PERSON|AIR|heodolites above the final, regular acc 1766|110761|8295|3|1|1771.76|0.10|0.02|N|O|1997-01-21|1997-01-07|1997-02-19|NONE|TRUCK|ly blithely pending accounts. reg 1767|24933|9938|1|32|59453.76|0.08|0.04|A|F|1995-05-22|1995-05-14|1995-05-23|COLLECT COD|SHIP|to the bravely ironic requests i 1767|41434|8947|2|1|1375.43|0.09|0.05|N|O|1995-06-23|1995-05-25|1995-07-03|TAKE BACK RETURN|RAIL|ing to the slyly fin 1767|173977|3978|3|24|49223.28|0.06|0.03|R|F|1995-03-16|1995-04-29|1995-04-11|DELIVER IN PERSON|RAIL|luffy theodolites need to detect furi 1767|22387|4890|4|50|65469.00|0.01|0.02|R|F|1995-05-29|1995-04-14|1995-06-15|NONE|REG AIR|y unusual foxe 1767|51549|4055|5|40|60021.60|0.06|0.00|R|F|1995-04-16|1995-05-06|1995-04-21|TAKE BACK RETURN|AIR|ep. accounts nag blithely fu 1792|87191|2208|1|9|10603.71|0.09|0.04|R|F|1994-02-28|1993-12-11|1994-03-12|TAKE BACK RETURN|AIR|final packages s 1792|8700|6201|2|5|8043.50|0.04|0.02|R|F|1994-02-13|1994-01-03|1994-02-28|DELIVER IN PERSON|TRUCK|ely regular accounts are slyly. pending, bo 1792|8139|640|3|8|8377.04|0.01|0.04|A|F|1994-02-21|1994-01-26|1994-02-27|DELIVER IN PERSON|RAIL|nts. fluffily special instructions integr 1792|190135|2655|4|45|55130.85|0.00|0.01|A|F|1994-02-27|1993-12-24|1994-03-07|DELIVER IN PERSON|MAIL|ests are. ironic, regular asy 1792|198786|3825|5|35|65967.30|0.06|0.05|R|F|1994-01-31|1994-01-20|1994-02-17|NONE|FOB|e against the quic 1793|47119|9624|1|29|30917.19|0.01|0.06|R|F|1992-10-24|1992-09-20|1992-11-23|NONE|MAIL|ar excuses. 1793|125194|2731|2|4|4876.76|0.07|0.05|A|F|1992-07-28|1992-08-26|1992-08-21|COLLECT COD|RAIL|nic foxes along the even 1793|130060|2574|3|6|6540.36|0.01|0.05|R|F|1992-09-21|1992-09-05|1992-10-01|DELIVER IN PERSON|REG AIR|uctions; depo 1793|117679|5213|4|4|6786.68|0.00|0.08|R|F|1992-09-27|1992-09-21|1992-10-07|DELIVER IN PERSON|AIR|equests nod ac 1793|24521|4522|5|42|60711.84|0.03|0.03|A|F|1992-10-13|1992-10-02|1992-11-06|NONE|RAIL|uctions sleep carefully special, fl 1794|167389|7390|1|36|52429.68|0.09|0.08|N|O|1997-11-07|1997-11-01|1997-11-18|TAKE BACK RETURN|FOB|ely fluffily ironi 1794|94575|9594|2|3|4708.71|0.02|0.03|N|O|1997-11-15|1997-12-16|1997-11-20|DELIVER IN PERSON|FOB| sentiments according to the q 1794|116160|6161|3|23|27051.68|0.08|0.04|N|O|1997-10-13|1997-11-30|1997-10-28|TAKE BACK RETURN|AIR|usly unusual theodolites doze about 1794|84627|9644|4|34|54795.08|0.06|0.08|N|O|1997-09-29|1997-11-13|1997-10-07|TAKE BACK RETURN|SHIP|rs above the accoun 1794|116434|1457|5|47|68170.21|0.10|0.06|N|O|1998-01-15|1997-11-30|1998-02-14|DELIVER IN PERSON|TRUCK| haggle slyly. furiously express orbit 1794|90185|2695|6|37|43481.66|0.01|0.01|N|O|1998-01-12|1997-12-21|1998-01-17|DELIVER IN PERSON|MAIL|ackages. pinto 1795|136200|6201|1|44|54392.80|0.08|0.08|A|F|1994-04-28|1994-05-24|1994-05-27|NONE|AIR|ites sleep carefully slyly p 1795|113154|3155|2|34|39683.10|0.08|0.00|A|F|1994-04-24|1994-06-01|1994-05-08|DELIVER IN PERSON|SHIP|closely regular instructions wake. 1795|167545|5094|3|25|40313.50|0.07|0.01|A|F|1994-05-18|1994-05-22|1994-05-20|TAKE BACK RETURN|RAIL|he always express accounts ca 1795|124863|2400|4|32|60411.52|0.03|0.06|R|F|1994-05-10|1994-04-21|1994-05-17|DELIVER IN PERSON|SHIP| asymptotes across the bold, 1795|162306|9855|5|11|15051.30|0.08|0.02|R|F|1994-06-19|1994-04-24|1994-07-02|TAKE BACK RETURN|TRUCK|slyly. special pa 1796|9888|9889|1|28|50340.64|0.08|0.04|A|F|1992-12-01|1993-01-01|1992-12-24|DELIVER IN PERSON|FOB|y quickly ironic accounts. 1796|184834|9871|2|8|15350.64|0.00|0.08|R|F|1993-01-07|1993-01-04|1993-01-10|NONE|SHIP|slyly bold accounts are furiously agains 1797|30639|3143|1|17|26683.71|0.01|0.02|N|O|1996-08-06|1996-07-11|1996-08-29|NONE|TRUCK| cajole carefully. unusual Tiresias e 1797|144215|6730|2|16|20147.36|0.01|0.00|N|O|1996-06-03|1996-07-21|1996-06-07|NONE|FOB|o beans wake regular accounts. blit 1797|11709|6712|3|21|34034.70|0.02|0.01|N|O|1996-08-05|1996-08-05|1996-08-06|DELIVER IN PERSON|AIR|ns. regular, regular deposit 1798|108858|8859|1|43|80274.55|0.01|0.08|N|O|1997-08-27|1997-10-23|1997-09-09|DELIVER IN PERSON|MAIL|ld packages sleep furiously. depend 1799|51159|3665|1|8|8881.20|0.04|0.08|R|F|1994-06-14|1994-05-27|1994-06-27|TAKE BACK RETURN|MAIL|ealms upon the special, ironic waters 1799|26318|3825|2|42|52261.02|0.02|0.02|R|F|1994-04-05|1994-04-28|1994-04-09|DELIVER IN PERSON|FOB|es pending 1824|119669|7203|1|45|75989.70|0.03|0.02|R|F|1994-08-21|1994-06-21|1994-09-19|NONE|RAIL|ent Tiresias. quickly express 1824|68198|5717|2|40|46647.60|0.10|0.03|A|F|1994-05-08|1994-07-24|1994-06-06|NONE|FOB|es mold furiously final instructions. s 1825|155040|71|1|43|47086.72|0.05|0.05|A|F|1994-02-18|1994-02-19|1994-03-02|TAKE BACK RETURN|RAIL| accounts breach fluffily spe 1825|147333|9848|2|39|53832.87|0.00|0.00|R|F|1994-04-01|1994-01-12|1994-04-21|DELIVER IN PERSON|REG AIR|ual, bold ideas haggle above the quickly ir 1825|16370|1373|3|7|9004.59|0.04|0.03|A|F|1994-01-02|1994-01-30|1994-01-30|TAKE BACK RETURN|REG AIR|fully ironic requests. requests cajole ex 1825|120223|5248|4|23|28594.06|0.05|0.01|R|F|1994-01-08|1994-02-08|1994-01-19|NONE|MAIL| wake express, even r 1825|177840|7841|5|33|63288.72|0.04|0.04|A|F|1993-12-07|1994-03-01|1993-12-16|TAKE BACK RETURN|RAIL|about the ne 1826|26601|4108|1|4|6110.40|0.06|0.00|R|F|1992-07-05|1992-06-12|1992-08-04|DELIVER IN PERSON|MAIL|alongside of the quickly unusual re 1826|67102|4621|2|9|9621.90|0.07|0.07|R|F|1992-07-12|1992-07-11|1992-07-15|DELIVER IN PERSON|TRUCK| blithely special 1826|175212|2764|3|14|18020.94|0.05|0.01|A|F|1992-04-28|1992-05-31|1992-05-25|COLLECT COD|TRUCK|uriously bold pinto beans are carefully ag 1826|179826|4861|4|6|11434.92|0.05|0.04|R|F|1992-06-30|1992-05-17|1992-07-30|DELIVER IN PERSON|RAIL|kages. blithely silent 1826|134617|2157|5|46|75974.06|0.05|0.06|R|F|1992-05-02|1992-06-25|1992-05-26|TAKE BACK RETURN|FOB|ously? quickly pe 1826|107352|2373|6|43|58452.05|0.02|0.03|A|F|1992-07-28|1992-06-14|1992-08-03|NONE|MAIL|ss tithes use even ideas. fluffily final t 1827|89460|6985|1|47|68124.62|0.00|0.01|N|O|1996-08-01|1996-08-07|1996-08-23|TAKE BACK RETURN|RAIL|. pending courts about the even e 1827|153091|8122|2|48|54916.32|0.03|0.05|N|O|1996-08-28|1996-09-15|1996-09-01|COLLECT COD|RAIL|oxes. special, final asymptote 1827|199818|9819|3|37|70958.97|0.01|0.07|N|O|1996-07-20|1996-08-18|1996-08-08|DELIVER IN PERSON|REG AIR|ously ironic theodolites serve quickly af 1827|126488|4025|4|4|6057.92|0.04|0.04|N|O|1996-07-22|1996-09-10|1996-08-11|DELIVER IN PERSON|RAIL|special requests. blithely 1827|79528|2036|5|24|36180.48|0.00|0.08|N|O|1996-08-07|1996-09-01|1996-09-04|DELIVER IN PERSON|SHIP|al gifts! re 1827|20469|470|6|7|9726.22|0.10|0.02|N|O|1996-08-28|1996-08-07|1996-08-31|DELIVER IN PERSON|AIR|egular foxes 1827|5933|5934|7|38|69879.34|0.05|0.01|N|O|1996-10-17|1996-08-29|1996-11-07|TAKE BACK RETURN|SHIP| blithely. express, bo 1828|99555|7083|1|33|51300.15|0.05|0.04|R|F|1994-06-27|1994-06-10|1994-07-24|COLLECT COD|FOB|s boost carefully. pending d 1828|12436|9940|2|40|53937.20|0.08|0.07|R|F|1994-05-05|1994-07-02|1994-05-19|COLLECT COD|REG AIR|s use above the quietly fin 1828|195860|5861|3|11|21514.46|0.07|0.08|R|F|1994-07-21|1994-05-28|1994-08-13|DELIVER IN PERSON|FOB| wake blithely 1828|7764|2765|4|45|75229.20|0.02|0.05|R|F|1994-05-15|1994-05-29|1994-05-28|COLLECT COD|RAIL| accounts run slyly 1828|78153|5675|5|14|15836.10|0.01|0.08|A|F|1994-05-20|1994-06-02|1994-05-25|TAKE BACK RETURN|SHIP|. final packages along the carefully bold 1829|149926|2441|1|12|23711.04|0.05|0.06|A|F|1994-08-23|1994-07-13|1994-09-04|DELIVER IN PERSON|FOB|ges wake furiously express pinto 1829|4612|4613|2|11|16682.71|0.04|0.05|A|F|1994-05-18|1994-06-13|1994-06-07|COLLECT COD|MAIL|ding orbits 1829|103502|8523|3|49|73769.50|0.09|0.08|A|F|1994-08-26|1994-08-01|1994-09-16|NONE|TRUCK|ound the quickly 1829|152994|2995|4|14|28657.86|0.03|0.06|A|F|1994-08-15|1994-06-08|1994-08-30|TAKE BACK RETURN|AIR|regular deposits alongside of the flu 1829|165795|8312|5|6|11164.74|0.02|0.07|A|F|1994-08-09|1994-08-05|1994-09-05|DELIVER IN PERSON|MAIL|s haggle! slyl 1829|114621|2155|6|36|58882.32|0.09|0.04|R|F|1994-06-10|1994-06-23|1994-06-22|NONE|FOB|ackages-- express requests sleep; pen 1830|119595|2107|1|38|61354.42|0.00|0.07|R|F|1995-04-20|1995-05-22|1995-04-24|TAKE BACK RETURN|TRUCK|ely even a 1830|24483|6986|2|9|12667.32|0.05|0.07|R|F|1995-03-09|1995-05-24|1995-03-14|NONE|SHIP|st furiously among 1830|81005|3514|3|36|35496.00|0.07|0.07|R|F|1995-04-21|1995-04-14|1995-05-10|DELIVER IN PERSON|SHIP| slowly unusual orbits. carefull 1831|135126|7640|1|9|10450.08|0.02|0.03|A|F|1993-12-17|1994-01-27|1993-12-26|NONE|TRUCK|mptotes. furiously regular dolphins al 1831|47035|7036|2|9|8838.27|0.07|0.06|R|F|1994-03-22|1994-01-07|1994-04-06|COLLECT COD|MAIL|ent deposits. regular saute 1831|114911|2445|3|17|32740.47|0.02|0.08|R|F|1994-01-18|1994-02-12|1994-01-30|TAKE BACK RETURN|MAIL|s boost ironic foxe 1831|94173|9192|4|23|26844.91|0.06|0.02|R|F|1993-12-21|1994-02-08|1994-01-04|NONE|SHIP|ests. express pinto beans abou 1856|54971|9982|1|10|19259.70|0.05|0.07|R|F|1992-05-11|1992-05-20|1992-06-02|TAKE BACK RETURN|FOB|he furiously even theodolites. account 1856|96908|1927|2|47|89530.30|0.07|0.07|R|F|1992-03-22|1992-06-09|1992-04-17|DELIVER IN PERSON|FOB|ingly blithe theodolites. slyly pending 1856|116508|4042|3|20|30490.00|0.04|0.06|R|F|1992-05-04|1992-05-06|1992-05-11|DELIVER IN PERSON|MAIL|ost carefully. slyly bold accounts 1856|149295|9296|4|22|29574.38|0.08|0.02|A|F|1992-05-02|1992-05-26|1992-05-20|TAKE BACK RETURN|REG AIR|platelets detect slyly regular packages. ca 1856|189673|9674|5|14|24677.38|0.01|0.01|A|F|1992-04-14|1992-05-02|1992-05-11|COLLECT COD|SHIP|ans are even requests. deposits caj 1856|22677|184|6|36|57588.12|0.03|0.05|A|F|1992-06-19|1992-05-12|1992-06-28|TAKE BACK RETURN|TRUCK|ly even foxes kindle blithely even realm 1856|129838|7375|7|42|78448.86|0.04|0.00|R|F|1992-05-23|1992-06-06|1992-06-19|COLLECT COD|RAIL|usly final deposits 1857|173807|3808|1|15|28212.00|0.10|0.03|R|F|1993-04-05|1993-02-28|1993-04-13|COLLECT COD|RAIL|egular, regular inst 1857|166343|8860|2|40|56373.60|0.10|0.00|R|F|1993-02-15|1993-03-08|1993-02-21|NONE|AIR|slyly close d 1857|118981|1493|3|8|15999.84|0.01|0.07|R|F|1993-01-27|1993-04-04|1993-02-20|TAKE BACK RETURN|AIR|slyly about the fluffily silent req 1857|99627|4646|4|41|66691.42|0.07|0.07|A|F|1993-04-16|1993-02-16|1993-04-18|NONE|REG AIR| the slyly 1858|13932|6434|1|33|60915.69|0.01|0.02|N|O|1997-12-28|1998-02-03|1998-01-13|NONE|RAIL|tect along the slyly final 1859|74969|4970|1|18|34991.28|0.10|0.00|N|O|1997-08-08|1997-06-30|1997-08-26|TAKE BACK RETURN|SHIP|e carefully a 1859|187741|5296|2|36|65834.64|0.02|0.01|N|O|1997-05-05|1997-07-08|1997-05-25|TAKE BACK RETURN|REG AIR|regular requests. carefully unusual theo 1859|157708|5254|3|5|8828.50|0.06|0.03|N|O|1997-06-20|1997-05-20|1997-07-19|TAKE BACK RETURN|AIR|across the p 1859|190876|877|4|21|41304.27|0.00|0.03|N|O|1997-08-06|1997-05-29|1997-08-26|TAKE BACK RETURN|REG AIR|lar packages wake quickly exp 1859|45333|7838|5|11|14061.63|0.06|0.06|N|O|1997-07-15|1997-06-05|1997-07-29|TAKE BACK RETURN|SHIP|ffily ironic pac 1859|104305|6816|6|12|15711.60|0.08|0.03|N|O|1997-05-22|1997-06-08|1997-06-07|COLLECT COD|TRUCK|es. unusual, silent request 1860|112942|2943|1|9|17594.46|0.04|0.04|N|O|1996-08-03|1996-05-31|1996-08-04|DELIVER IN PERSON|TRUCK|c realms print carefully car 1861|67093|2106|1|7|7420.63|0.08|0.05|A|F|1994-01-14|1994-04-03|1994-01-16|COLLECT COD|RAIL|s foxes. slyly 1861|26765|6766|2|31|52444.56|0.10|0.05|R|F|1994-01-29|1994-03-07|1994-02-15|TAKE BACK RETURN|RAIL|arefully unusual 1861|23676|6179|3|23|36792.41|0.00|0.08|A|F|1994-04-09|1994-03-04|1994-04-11|DELIVER IN PERSON|MAIL|in packages sleep silent dolphins; sly 1861|115108|2642|4|38|42677.80|0.10|0.05|R|F|1994-02-26|1994-02-05|1994-03-01|NONE|RAIL|pending deposits cajole quic 1861|15110|113|5|2|2050.22|0.03|0.08|R|F|1994-04-26|1994-03-15|1994-05-15|TAKE BACK RETURN|MAIL|e final, regular requests. carefully 1862|29629|2132|1|41|63903.42|0.10|0.00|N|O|1998-06-05|1998-05-17|1998-07-04|COLLECT COD|FOB| carefully along 1862|165414|5415|2|37|54738.17|0.06|0.02|N|O|1998-04-15|1998-05-15|1998-05-14|TAKE BACK RETURN|MAIL|l deposits. carefully even dep 1862|103981|1512|3|26|51609.48|0.02|0.01|N|O|1998-03-25|1998-05-17|1998-04-17|TAKE BACK RETURN|TRUCK|g carefully: thinly ironic deposits af 1863|62472|4979|1|48|68854.56|0.09|0.04|A|F|1993-10-10|1993-12-09|1993-10-19|NONE|FOB|ans hinder furiou 1863|156951|1982|2|48|96381.60|0.04|0.08|A|F|1993-11-08|1993-11-05|1993-12-08|COLLECT COD|AIR|onic theodolites alongside of the pending a 1888|97643|153|1|27|44297.28|0.03|0.06|R|F|1994-02-13|1994-01-16|1994-02-25|NONE|REG AIR|. carefully special dolphins sle 1888|73403|3404|2|38|52303.20|0.03|0.03|R|F|1993-11-29|1994-01-16|1993-12-08|TAKE BACK RETURN|TRUCK|dazzle carefull 1888|79623|9624|3|49|78528.38|0.07|0.05|A|F|1994-02-27|1994-01-14|1994-03-28|DELIVER IN PERSON|FOB|lar accounts haggle carefu 1888|18538|8539|4|9|13108.77|0.01|0.04|A|F|1994-02-09|1994-01-22|1994-02-19|NONE|AIR| packages are blithely. carefu 1888|159509|9510|5|4|6274.00|0.03|0.06|R|F|1993-12-28|1993-12-19|1994-01-11|COLLECT COD|FOB|lphins. ironically special theodolit 1888|52536|7547|6|48|71449.44|0.08|0.08|R|F|1994-02-28|1993-12-16|1994-03-15|COLLECT COD|TRUCK|ar ideas cajole. regular p 1888|166333|8850|7|50|69966.50|0.04|0.07|R|F|1993-12-22|1994-01-10|1994-01-06|DELIVER IN PERSON|FOB|ependencies affix blithely regular warhors 1889|151174|8720|1|41|50231.97|0.10|0.02|N|O|1997-06-15|1997-05-10|1997-07-08|NONE|AIR|s! furiously pending r 1889|171604|1605|2|13|21782.80|0.05|0.00|N|O|1997-06-12|1997-04-28|1997-06-23|NONE|REG AIR|to the regular accounts. carefully express 1889|137156|2183|3|36|42953.40|0.05|0.07|N|O|1997-05-19|1997-06-14|1997-05-23|NONE|SHIP|l pinto beans kindle 1889|167656|2689|4|5|8618.25|0.02|0.07|N|O|1997-06-26|1997-06-09|1997-07-21|COLLECT COD|AIR|ording to the blithely silent r 1890|140281|2796|1|26|34353.28|0.03|0.07|N|O|1997-04-02|1997-03-13|1997-04-22|DELIVER IN PERSON|FOB|ngage. slyly ironic 1890|99403|9404|2|43|60303.20|0.07|0.03|N|O|1996-12-30|1997-01-31|1997-01-19|DELIVER IN PERSON|FOB|p ironic, express accounts. fu 1890|58873|6389|3|24|43964.88|0.06|0.04|N|O|1997-02-09|1997-02-10|1997-02-12|COLLECT COD|MAIL|is wake carefully above the even id 1890|67265|7266|4|43|52987.18|0.09|0.04|N|O|1997-04-08|1997-02-19|1997-04-30|TAKE BACK RETURN|FOB|lyly. instructions across the furiously 1890|121175|1176|5|45|53827.65|0.08|0.05|N|O|1997-04-15|1997-03-16|1997-04-19|COLLECT COD|FOB|he carefully regular sauternes. ironic fret 1890|180789|3308|6|16|29916.48|0.08|0.02|N|O|1997-02-13|1997-02-18|1997-03-12|TAKE BACK RETURN|TRUCK|ged pinto beans. regular, regular id 1890|120205|7742|7|10|12252.00|0.01|0.04|N|O|1996-12-24|1997-02-19|1997-01-01|DELIVER IN PERSON|AIR|. even, unusual inst 1891|76099|6100|1|45|48379.05|0.07|0.04|A|F|1994-12-20|1995-01-16|1995-01-05|NONE|RAIL|ests along 1891|183091|5610|2|18|21133.62|0.06|0.00|A|F|1995-01-24|1995-01-29|1995-02-14|NONE|RAIL| foxes above the carefu 1891|197159|7160|3|15|18842.25|0.03|0.00|R|F|1995-03-11|1995-03-05|1995-03-18|TAKE BACK RETURN|MAIL| accounts are furiou 1892|112777|5289|1|48|85908.96|0.02|0.01|A|F|1994-06-16|1994-06-16|1994-06-28|NONE|RAIL|tornis detect regul 1892|42003|9516|2|35|33075.00|0.04|0.08|R|F|1994-04-05|1994-05-09|1994-05-03|NONE|MAIL|hes nod furiously around the instruc 1892|133278|8305|3|37|48516.99|0.10|0.03|R|F|1994-04-11|1994-06-04|1994-04-24|TAKE BACK RETURN|SHIP|nts. slyly regular asymptot 1892|196964|9484|4|14|28853.44|0.06|0.07|R|F|1994-04-08|1994-06-12|1994-04-27|DELIVER IN PERSON|FOB|furiously about the furiously 1893|98266|776|1|43|54363.18|0.10|0.00|N|O|1998-01-25|1998-01-06|1998-02-14|COLLECT COD|SHIP|he carefully regular 1893|147335|7336|2|49|67734.17|0.03|0.05|N|O|1998-01-19|1998-01-28|1998-02-02|TAKE BACK RETURN|FOB|y final foxes bo 1893|44119|4120|3|3|3189.33|0.03|0.02|N|O|1998-02-10|1998-01-18|1998-02-25|DELIVER IN PERSON|MAIL|gular, even ideas. fluffily bol 1893|100605|5626|4|18|28900.80|0.07|0.06|N|O|1998-01-24|1998-01-12|1998-02-13|TAKE BACK RETURN|RAIL|g packages. fluffily final reques 1893|52070|2071|5|6|6132.42|0.10|0.02|N|O|1998-01-23|1997-12-22|1998-02-09|DELIVER IN PERSON|TRUCK|ar accounts use. daringly ironic packag 1894|168292|8293|1|40|54411.60|0.03|0.07|R|F|1992-06-07|1992-05-11|1992-07-01|DELIVER IN PERSON|FOB|ily furiously bold packages. flu 1895|160192|7741|1|43|53844.17|0.09|0.07|R|F|1994-07-26|1994-07-19|1994-08-11|NONE|AIR| carefully eve 1920|95392|5393|1|24|33297.36|0.04|0.05|N|O|1998-09-27|1998-08-23|1998-10-15|DELIVER IN PERSON|AIR|thely. bold, pend 1920|50707|5718|2|31|51388.70|0.05|0.06|N|O|1998-08-01|1998-08-30|1998-08-17|COLLECT COD|SHIP|lly. ideas wa 1920|17189|9691|3|6|6637.08|0.01|0.05|N|O|1998-10-01|1998-08-20|1998-10-24|COLLECT COD|SHIP|l ideas boost slyly pl 1920|83349|3350|4|50|66617.00|0.09|0.06|N|O|1998-10-03|1998-08-04|1998-10-29|DELIVER IN PERSON|MAIL|e blithely unusual foxes. brave packages 1920|33789|1299|5|14|24118.92|0.08|0.05|N|O|1998-10-22|1998-08-10|1998-10-27|DELIVER IN PERSON|AIR|ickly ironic d 1921|20643|5648|1|9|14072.76|0.08|0.00|R|F|1994-02-01|1994-03-20|1994-03-01|DELIVER IN PERSON|FOB|to beans. even excuses integrate specia 1921|139685|2199|2|21|36218.28|0.02|0.06|R|F|1994-02-08|1994-03-28|1994-02-15|COLLECT COD|FOB|ckly regula 1921|70822|823|3|27|48406.14|0.00|0.04|A|F|1994-04-26|1994-04-07|1994-04-30|TAKE BACK RETURN|FOB|ing pinto beans above the pend 1922|9775|4776|1|13|21902.01|0.05|0.03|N|O|1996-10-24|1996-09-21|1996-11-15|NONE|SHIP|quests. furiously 1923|36198|6199|1|9|10207.71|0.01|0.08|N|O|1997-08-29|1997-09-13|1997-09-07|NONE|FOB|lites. ironic instructions integrate bravel 1923|177330|9848|2|23|32368.59|0.07|0.05|N|O|1997-09-08|1997-08-11|1997-09-14|TAKE BACK RETURN|MAIL|aggle carefully. furiously permanent 1923|179628|9629|3|11|18783.82|0.03|0.03|N|O|1997-07-12|1997-09-04|1997-08-01|TAKE BACK RETURN|REG AIR|ages wake slyly about the furiously regular 1923|192622|5142|4|49|84016.38|0.06|0.05|N|O|1997-07-21|1997-08-08|1997-07-26|NONE|AIR|de of the carefully expre 1923|183057|8094|5|25|28501.25|0.10|0.08|N|O|1997-08-18|1997-08-20|1997-09-12|DELIVER IN PERSON|TRUCK|the ideas: slyly pendin 1923|36955|9459|6|50|94597.50|0.03|0.03|N|O|1997-11-04|1997-08-08|1997-11-25|NONE|TRUCK|uickly along the bold courts. bold the 1924|72567|89|1|7|10776.92|0.06|0.07|N|O|1997-01-01|1996-12-02|1997-01-08|COLLECT COD|SHIP|osits. even accounts nag furious 1924|17366|4870|2|47|60317.92|0.02|0.06|N|O|1996-11-24|1996-10-18|1996-12-13|COLLECT COD|REG AIR|silent requests cajole blithely final pack 1924|56788|6789|3|40|69791.20|0.04|0.08|N|O|1996-10-31|1996-11-30|1996-11-21|NONE|REG AIR|ains sleep carefully 1924|33179|8186|4|31|34477.27|0.03|0.03|N|O|1996-09-20|1996-10-19|1996-10-19|DELIVER IN PERSON|SHIP| the slyly regular foxes. ruthle 1924|35419|426|5|17|23024.97|0.04|0.05|N|O|1996-12-31|1996-11-12|1997-01-25|COLLECT COD|TRUCK|e carefully theodolites. ironically ironic 1924|75709|3231|6|15|25270.50|0.02|0.04|N|O|1997-01-04|1996-11-13|1997-01-27|NONE|SHIP|he package 1924|39531|9532|7|21|30881.13|0.09|0.03|N|O|1996-09-21|1996-11-12|1996-10-02|TAKE BACK RETURN|AIR| blithely reg 1925|183718|8755|1|50|90085.50|0.01|0.02|R|F|1992-04-12|1992-04-23|1992-05-08|TAKE BACK RETURN|TRUCK|usual pinto 1925|134273|1813|2|35|45754.45|0.06|0.06|R|F|1992-05-11|1992-04-10|1992-05-14|TAKE BACK RETURN|AIR|counts. carefully ironic packages boost ab 1925|115014|7526|3|40|41160.40|0.08|0.08|A|F|1992-05-17|1992-05-20|1992-06-08|TAKE BACK RETURN|AIR|e carefully regul 1925|29757|2260|4|17|28674.75|0.06|0.02|R|F|1992-05-18|1992-04-06|1992-06-16|TAKE BACK RETURN|MAIL|instructions sleep. pinto bea 1926|50307|2813|1|24|30175.20|0.06|0.05|N|O|1996-05-04|1996-03-14|1996-06-01|DELIVER IN PERSON|RAIL|e theodolites. 1926|105027|7538|2|29|29928.58|0.09|0.08|N|O|1996-02-26|1996-03-14|1996-03-14|TAKE BACK RETURN|TRUCK|es. dependencies according to the fl 1926|177645|5197|3|10|17226.40|0.02|0.03|N|O|1996-05-23|1996-03-02|1996-06-04|NONE|AIR|usly bold accounts. express accounts 1926|67772|7773|4|13|22617.01|0.04|0.02|N|O|1996-04-26|1996-04-13|1996-05-08|DELIVER IN PERSON|MAIL|eans wake bli 1926|39928|4935|5|29|54169.68|0.06|0.00|N|O|1996-02-29|1996-03-13|1996-03-24|DELIVER IN PERSON|MAIL|hily unusual packages are fluffily am 1927|67893|2906|1|3|5582.67|0.00|0.05|N|O|1995-10-06|1995-12-08|1995-11-05|COLLECT COD|FOB|ccounts affi 1927|72462|7477|2|15|21516.90|0.08|0.08|N|O|1995-12-25|1995-12-26|1995-12-31|COLLECT COD|RAIL| carefully regular requests sleep car 1927|64783|2302|3|6|10486.68|0.05|0.05|N|O|1995-11-29|1995-11-20|1995-12-08|TAKE BACK RETURN|TRUCK|furiously even wat 1952|52059|7070|1|7|7077.35|0.04|0.05|A|F|1994-05-06|1994-06-11|1994-05-12|NONE|RAIL|about the express, even requ 1952|141675|6704|2|6|10300.02|0.06|0.05|A|F|1994-05-09|1994-05-21|1994-05-26|DELIVER IN PERSON|AIR|packages haggle. 1953|127615|5152|1|25|41065.25|0.07|0.06|A|F|1994-01-07|1994-01-28|1994-01-29|TAKE BACK RETURN|RAIL|ular, regular i 1953|13880|3881|2|35|62785.80|0.06|0.06|R|F|1994-02-03|1994-02-25|1994-02-14|DELIVER IN PERSON|FOB|among the fur 1954|151270|6301|1|31|40959.37|0.06|0.06|N|O|1997-08-18|1997-07-07|1997-09-03|DELIVER IN PERSON|RAIL|against the packages. bold, ironic e 1954|181175|6212|2|1|1256.17|0.03|0.01|N|O|1997-09-16|1997-07-08|1997-10-07|COLLECT COD|MAIL|te. furiously final deposits hag 1954|198665|3704|3|11|19400.26|0.07|0.07|N|O|1997-08-07|1997-07-23|1997-08-25|DELIVER IN PERSON|TRUCK|y carefully ironi 1954|158218|3249|4|12|15314.52|0.02|0.08|N|O|1997-07-19|1997-07-04|1997-08-06|COLLECT COD|AIR|ongside of the slyly unusual requests. reg 1954|169702|4735|5|29|51379.30|0.08|0.08|N|O|1997-08-25|1997-07-15|1997-09-02|DELIVER IN PERSON|RAIL|use thinly furiously regular asy 1954|176272|6273|6|13|17527.51|0.00|0.07|N|O|1997-06-15|1997-08-22|1997-06-20|TAKE BACK RETURN|MAIL|y ironic instructions cajole 1954|193476|3477|7|49|76904.03|0.05|0.06|N|O|1997-06-04|1997-08-29|1997-06-14|COLLECT COD|TRUCK|eans. final pinto beans sleep furiousl 1955|136052|8566|1|32|34817.60|0.02|0.02|A|F|1992-07-05|1992-06-29|1992-08-03|TAKE BACK RETURN|TRUCK|g to the carefully sile 1955|17074|4578|2|2|1982.14|0.03|0.01|R|F|1992-07-06|1992-07-06|1992-08-01|COLLECT COD|TRUCK|ickly aroun 1955|157697|213|3|41|71942.29|0.08|0.06|A|F|1992-08-01|1992-06-04|1992-08-07|COLLECT COD|AIR| carefully against the furiously reg 1955|8245|3246|4|16|18451.84|0.03|0.07|A|F|1992-04-30|1992-06-23|1992-05-23|TAKE BACK RETURN|FOB|odolites eat s 1955|158432|8433|5|11|16394.73|0.09|0.01|A|F|1992-06-03|1992-07-04|1992-06-07|NONE|REG AIR|ously quickly pendi 1956|176350|6351|1|8|11410.80|0.02|0.04|A|F|1992-12-25|1992-11-24|1993-01-12|TAKE BACK RETURN|AIR|efully about the ironic, ironic de 1956|102954|5465|2|16|31311.20|0.00|0.05|R|F|1992-11-11|1992-11-11|1992-11-30|NONE|FOB|es cajole blithely. pen 1956|138920|1434|3|39|76397.88|0.08|0.02|A|F|1992-09-24|1992-11-26|1992-10-15|DELIVER IN PERSON|REG AIR|r theodolites sleep above the b 1956|28284|8285|4|11|13335.08|0.10|0.00|A|F|1992-12-19|1992-10-29|1993-01-07|TAKE BACK RETURN|AIR| the braids slee 1956|154900|9931|5|16|31278.40|0.08|0.02|R|F|1992-09-28|1992-10-21|1992-09-30|TAKE BACK RETURN|FOB| wake after the 1957|78284|792|1|50|63114.00|0.09|0.05|N|O|1998-08-08|1998-09-28|1998-08-27|COLLECT COD|FOB|gainst the re 1957|118027|539|2|31|32395.62|0.10|0.08|N|O|1998-08-13|1998-08-31|1998-08-16|NONE|REG AIR|express packages maintain fluffi 1958|72052|7067|1|9|9216.45|0.01|0.05|N|O|1995-12-08|1995-12-17|1995-12-18|DELIVER IN PERSON|REG AIR|ickly. slyly bold 1958|175065|5066|2|29|33061.74|0.05|0.06|N|O|1996-01-19|1995-12-05|1996-02-14|COLLECT COD|SHIP|d pinto beans 1958|101773|1774|3|4|7099.08|0.04|0.02|N|O|1995-10-24|1995-12-09|1995-10-28|DELIVER IN PERSON|AIR|he slyly even dependencies 1958|82603|7620|4|38|60252.80|0.09|0.07|N|O|1995-10-09|1995-11-26|1995-11-05|COLLECT COD|TRUCK|yly. slyly regular courts use silentl 1958|100912|8443|5|31|59300.21|0.08|0.01|N|O|1995-10-31|1995-11-12|1995-11-07|TAKE BACK RETURN|TRUCK|r deposits c 1958|16175|1178|6|44|48011.48|0.08|0.04|N|O|1995-12-17|1995-11-30|1996-01-15|TAKE BACK RETURN|RAIL|c theodolites after the unusual deposit 1958|38399|5909|7|29|38784.31|0.02|0.05|N|O|1995-10-14|1995-11-06|1995-11-01|NONE|REG AIR|final requests nag according to the 1959|168675|8676|1|46|80208.82|0.04|0.00|N|O|1997-05-05|1997-03-03|1997-05-24|TAKE BACK RETURN|AIR| furiously ex 1959|119439|4462|2|15|21876.45|0.08|0.07|N|O|1997-01-20|1997-02-18|1997-02-08|DELIVER IN PERSON|MAIL| quickly sp 1984|52054|9570|1|45|45272.25|0.03|0.04|N|O|1998-04-09|1998-06-11|1998-05-01|COLLECT COD|AIR|p. quickly final ideas sle 1984|69990|5003|2|35|68599.65|0.01|0.07|N|O|1998-05-18|1998-05-04|1998-06-01|COLLECT COD|RAIL|tes. quickly pending packages haggle boldl 1985|27157|4664|1|33|35776.95|0.10|0.03|R|F|1994-12-04|1994-11-01|1994-12-05|DELIVER IN PERSON|FOB|s are express packages. pendin 1985|20393|2896|2|50|65669.50|0.04|0.02|R|F|1994-09-30|1994-10-18|1994-10-12|COLLECT COD|AIR|ate carefully. carefully 1985|133207|5721|3|20|24804.00|0.07|0.03|R|F|1994-10-29|1994-11-12|1994-11-27|NONE|TRUCK|regular requests. furiously express 1985|198140|8141|4|30|37144.20|0.05|0.07|R|F|1994-09-06|1994-10-10|1994-09-26|NONE|RAIL|uickly. instr 1985|123819|6332|5|42|77398.02|0.05|0.05|R|F|1994-10-25|1994-11-03|1994-11-19|DELIVER IN PERSON|SHIP| patterns? final requests after the sp 1985|19676|4679|6|2|3191.34|0.02|0.00|A|F|1994-11-25|1994-10-09|1994-12-25|TAKE BACK RETURN|FOB| silent inst 1986|91481|1482|1|12|17669.76|0.06|0.05|A|F|1994-08-17|1994-06-28|1994-09-02|COLLECT COD|RAIL|sleep furiously fluffily final 1986|104675|7186|2|10|16796.70|0.10|0.03|R|F|1994-05-14|1994-06-21|1994-06-02|COLLECT COD|REG AIR|yly into the carefully even 1986|62714|5221|3|14|23473.94|0.04|0.02|R|F|1994-07-14|1994-06-19|1994-08-08|NONE|SHIP|the packages. pending, unusual 1987|15018|2522|1|7|6531.07|0.03|0.03|A|F|1994-07-30|1994-07-06|1994-08-29|NONE|REG AIR| regular a 1988|71007|6022|1|36|35208.00|0.09|0.04|N|O|1996-01-21|1995-11-24|1996-01-27|NONE|RAIL|gular theodolites. 1988|198445|6003|2|19|29325.36|0.08|0.08|N|O|1996-02-03|1995-12-10|1996-02-14|COLLECT COD|FOB|lly about the slyly thin instructions. f 1988|53790|1306|3|8|13950.32|0.06|0.01|N|O|1995-10-20|1995-11-11|1995-11-18|DELIVER IN PERSON|AIR|le quickly ac 1988|35669|8173|4|27|43325.82|0.08|0.00|N|O|1996-01-27|1995-12-24|1996-02-24|TAKE BACK RETURN|TRUCK|uests. regular requests are according to t 1988|78175|3190|5|26|29982.42|0.08|0.04|N|O|1996-01-25|1995-12-15|1996-01-26|COLLECT COD|SHIP| ironic dolphins haggl 1988|85943|3468|6|9|17360.46|0.08|0.03|N|O|1995-12-26|1996-01-02|1996-01-25|DELIVER IN PERSON|MAIL|lar platelets. slyly ironic packa 1989|9871|7372|1|47|83700.89|0.10|0.02|R|F|1994-06-21|1994-05-27|1994-06-22|TAKE BACK RETURN|REG AIR|final deposits s 1990|100961|962|1|46|90250.16|0.01|0.07|R|F|1994-12-29|1995-03-14|1995-01-13|NONE|TRUCK|ar sentiments. 1991|109006|1517|1|39|39585.00|0.06|0.02|A|F|1993-01-01|1992-11-29|1993-01-10|TAKE BACK RETURN|TRUCK|ckages? carefully bold depos 1991|52810|5316|2|49|86377.69|0.08|0.06|R|F|1992-10-19|1992-11-29|1992-10-25|NONE|SHIP|nd the ideas affi 1991|173555|3556|3|6|9771.30|0.02|0.01|A|F|1992-11-02|1992-10-08|1992-11-14|TAKE BACK RETURN|REG AIR|hes nag slyly 1991|137375|2402|4|6|8474.22|0.10|0.06|A|F|1992-11-21|1992-11-03|1992-11-27|NONE|RAIL|uickly blithely final de 1991|59310|1816|5|49|62196.19|0.06|0.00|R|F|1992-09-10|1992-11-30|1992-10-07|NONE|AIR|quests cajole blithely 2016|146003|8518|1|2|2098.00|0.02|0.07|N|O|1996-10-12|1996-11-09|1996-10-31|DELIVER IN PERSON|TRUCK|carefully according to the 2016|62672|191|2|15|24520.05|0.04|0.05|N|O|1996-09-24|1996-10-05|1996-10-21|TAKE BACK RETURN|MAIL|uests haggle carefully furiously regul 2016|121423|3936|3|8|11555.36|0.09|0.02|N|O|1996-09-19|1996-10-21|1996-10-13|TAKE BACK RETURN|SHIP|mptotes haggle ideas. packages wake flu 2017|102545|2546|1|49|75829.46|0.10|0.06|N|O|1998-05-26|1998-07-01|1998-06-06|COLLECT COD|TRUCK| after the unusual instructions. sly 2017|70072|73|2|14|14588.98|0.07|0.04|N|O|1998-06-28|1998-06-15|1998-07-11|NONE|TRUCK|ily final w 2017|83446|8463|3|11|15723.84|0.05|0.02|N|O|1998-05-22|1998-07-13|1998-05-26|TAKE BACK RETURN|TRUCK|gside of the slyly dogged dolp 2018|194201|4202|1|2|2590.40|0.02|0.07|N|O|1995-06-25|1995-06-20|1995-07-04|NONE|TRUCK|ly ironic accounts against the slyly sly 2018|128111|8112|2|23|26199.53|0.05|0.01|R|F|1995-05-05|1995-05-12|1995-05-22|TAKE BACK RETURN|RAIL|ingly even theodolites s 2019|3228|8229|1|31|35067.82|0.07|0.03|R|F|1992-11-18|1992-12-26|1992-11-24|DELIVER IN PERSON|FOB|l ideas across the slowl 2019|51852|6863|2|18|32469.30|0.04|0.03|R|F|1993-01-24|1992-12-22|1993-02-02|NONE|MAIL|are carefully furiously regular requ 2020|33521|1031|1|50|72726.00|0.06|0.01|R|F|1993-07-12|1993-08-28|1993-08-02|COLLECT COD|TRUCK|ts against the pending ideas serve along 2020|175666|3218|2|40|69666.40|0.09|0.00|A|F|1993-10-17|1993-09-14|1993-10-29|TAKE BACK RETURN|RAIL|ently across the 2020|13808|1312|3|30|51654.00|0.07|0.04|A|F|1993-09-08|1993-08-11|1993-09-29|TAKE BACK RETURN|AIR|ly about the blithely ironic foxes. bold 2020|60852|5865|4|27|48946.95|0.05|0.06|A|F|1993-07-14|1993-09-02|1993-08-03|NONE|FOB|e of the bold foxes haggle 2021|84701|2226|1|7|11799.90|0.08|0.04|N|O|1995-10-17|1995-09-29|1995-10-20|NONE|MAIL| accounts boost blithely. blithely reg 2021|165276|309|2|19|25484.13|0.04|0.05|N|O|1995-08-14|1995-09-05|1995-08-23|NONE|RAIL| above the slyly fl 2022|168410|927|1|38|56179.58|0.00|0.08|R|F|1992-07-05|1992-04-20|1992-07-13|TAKE BACK RETURN|REG AIR| against the express accounts wake ca 2022|54766|7272|2|38|65388.88|0.05|0.04|R|F|1992-06-17|1992-05-15|1992-06-28|COLLECT COD|SHIP|instructions dazzle carefull 2022|48896|8897|3|48|88554.72|0.10|0.02|A|F|1992-06-14|1992-06-04|1992-07-12|DELIVER IN PERSON|SHIP|counts. slyly enticing accounts are during 2022|181336|1337|4|16|22677.28|0.05|0.03|R|F|1992-06-23|1992-05-22|1992-07-07|NONE|TRUCK|ages wake slyly care 2022|99490|9491|5|36|53621.64|0.05|0.02|R|F|1992-03-24|1992-05-07|1992-04-13|NONE|MAIL|ly after the foxes. regular, final inst 2022|128142|5679|6|20|23402.80|0.08|0.08|A|F|1992-03-31|1992-04-17|1992-04-02|NONE|SHIP|r deposits kindle 2022|77154|7155|7|13|14704.95|0.06|0.08|R|F|1992-04-04|1992-05-30|1992-04-21|NONE|FOB| orbits haggle fluffily fl 2023|126248|3785|1|9|11468.16|0.05|0.04|R|F|1992-06-04|1992-06-30|1992-06-10|NONE|AIR|ly regular pinto beans poa 2023|37736|5246|2|2|3347.46|0.01|0.00|R|F|1992-08-27|1992-07-16|1992-08-29|DELIVER IN PERSON|RAIL|ing packages. fluffily silen 2023|18539|3542|3|25|36438.25|0.10|0.03|A|F|1992-07-19|1992-07-07|1992-08-15|NONE|REG AIR| wake furiously among the slyly final 2023|184818|9855|4|9|17125.29|0.02|0.00|A|F|1992-07-23|1992-07-04|1992-08-20|TAKE BACK RETURN|AIR|nts maintain blithely alongside of the 2023|19435|6939|5|22|29797.46|0.04|0.06|A|F|1992-06-15|1992-07-13|1992-06-21|TAKE BACK RETURN|SHIP|ronic attainments. 2023|42824|337|6|29|51237.78|0.02|0.06|A|F|1992-08-29|1992-07-28|1992-09-18|COLLECT COD|RAIL|usual instructions. bli 2023|133971|6485|7|50|100248.50|0.00|0.03|R|F|1992-06-20|1992-07-04|1992-06-23|DELIVER IN PERSON|FOB|its! carefully ex 2048|34296|6800|1|7|8612.03|0.06|0.01|R|F|1993-12-07|1994-01-31|1994-01-05|TAKE BACK RETURN|REG AIR|lent platelets boost deposits. carefully sp 2048|7372|4873|2|5|6396.85|0.04|0.04|A|F|1994-01-18|1994-02-01|1994-01-29|TAKE BACK RETURN|TRUCK|affix carefully against 2048|100536|537|3|12|18438.36|0.01|0.05|R|F|1994-01-28|1994-01-19|1994-02-08|NONE|AIR| even theodoli 2048|96292|3820|4|11|14171.19|0.10|0.03|R|F|1993-12-20|1994-01-19|1994-01-04|TAKE BACK RETURN|MAIL|totes. idly ironic packages nag 2049|188067|8068|1|25|28876.50|0.08|0.00|N|O|1996-03-31|1996-02-29|1996-04-15|DELIVER IN PERSON|MAIL| excuses above the 2049|34609|7113|2|31|47851.60|0.10|0.05|N|O|1995-12-25|1996-02-25|1995-12-29|TAKE BACK RETURN|MAIL| packages are slyly alongside 2049|66556|9063|3|18|27405.90|0.05|0.05|N|O|1996-01-09|1996-01-22|1996-01-25|TAKE BACK RETURN|AIR| sleep fluffily. dependencies use never 2049|5410|5411|4|39|51300.99|0.02|0.05|N|O|1996-01-17|1996-01-21|1996-02-03|TAKE BACK RETURN|MAIL|the even pinto beans 2049|125275|7788|5|30|39008.10|0.04|0.06|N|O|1995-12-16|1996-02-04|1995-12-22|NONE|TRUCK|ial accounts are among the furiously perma 2049|83083|8100|6|17|18123.36|0.07|0.00|N|O|1996-02-04|1996-03-01|1996-02-24|NONE|FOB|al, regular foxes. pending, 2050|72622|7637|1|47|74947.14|0.05|0.03|A|F|1994-08-25|1994-07-18|1994-09-15|DELIVER IN PERSON|TRUCK|tside the blithely pending packages eat f 2050|151587|1588|2|48|78651.84|0.05|0.01|A|F|1994-09-30|1994-08-23|1994-10-29|COLLECT COD|AIR| final packages. pinto 2050|112099|2100|3|41|45554.69|0.10|0.04|A|F|1994-06-08|1994-08-27|1994-06-23|NONE|AIR| final theodolites. depende 2050|31521|9031|4|11|15977.72|0.02|0.01|A|F|1994-07-27|1994-08-18|1994-08-02|DELIVER IN PERSON|REG AIR|ns. bold, final ideas cajole among the fi 2050|167826|7827|5|16|30301.12|0.07|0.01|R|F|1994-08-17|1994-07-28|1994-09-05|DELIVER IN PERSON|REG AIR|al accounts. closely even 2050|48412|3421|6|29|39451.89|0.00|0.05|A|F|1994-09-23|1994-08-01|1994-10-23|TAKE BACK RETURN|MAIL|oxes alongsid 2050|47268|9773|7|25|30381.50|0.10|0.00|R|F|1994-08-18|1994-07-04|1994-09-04|TAKE BACK RETURN|RAIL|y according to 2051|24284|4285|1|43|51956.04|0.08|0.04|N|O|1996-04-22|1996-06-16|1996-04-28|COLLECT COD|RAIL|ounts sleep fluffily even requ 2051|129342|9343|2|48|65824.32|0.01|0.02|N|O|1996-05-04|1996-06-14|1996-05-19|NONE|TRUCK|unts. pending platelets believe about 2052|67826|333|1|50|89691.00|0.09|0.08|R|F|1992-06-22|1992-06-03|1992-07-19|DELIVER IN PERSON|AIR|wake after the decoy 2052|134863|7377|2|35|66425.10|0.09|0.05|A|F|1992-05-29|1992-05-24|1992-06-11|NONE|TRUCK|ts according t 2052|42203|9716|3|16|18323.20|0.01|0.08|A|F|1992-06-30|1992-07-09|1992-07-12|NONE|SHIP|y final deposits cajole according 2052|95911|5912|4|47|89624.77|0.08|0.01|A|F|1992-06-18|1992-05-16|1992-07-02|TAKE BACK RETURN|REG AIR|final requests. stealt 2053|100687|3198|1|20|33753.60|0.09|0.00|A|F|1995-04-25|1995-04-12|1995-05-13|NONE|TRUCK|ly ironic foxes haggle slyly speci 2053|32300|2301|2|34|41898.20|0.07|0.00|A|F|1995-03-15|1995-03-20|1995-04-09|TAKE BACK RETURN|TRUCK|ions. unusual dependencies 2053|64279|9292|3|46|57190.42|0.01|0.03|R|F|1995-04-01|1995-04-02|1995-04-18|NONE|RAIL|tions. furiously even requests hagg 2053|120953|3466|4|31|61192.45|0.06|0.08|R|F|1995-03-23|1995-03-13|1995-04-16|DELIVER IN PERSON|SHIP|ts. fluffily final mul 2054|112318|2319|1|11|14633.41|0.03|0.05|R|F|1992-08-13|1992-08-26|1992-08-22|NONE|AIR|ular accou 2054|119303|4326|2|31|40991.30|0.05|0.08|A|F|1992-08-18|1992-09-04|1992-08-24|NONE|FOB|se bold, regular accounts. unusual depos 2054|120984|985|3|32|64159.36|0.06|0.00|A|F|1992-06-23|1992-07-08|1992-07-22|NONE|FOB| packages thrash. carefully final 2054|173988|9023|4|14|28867.72|0.10|0.05|R|F|1992-06-25|1992-09-05|1992-07-14|DELIVER IN PERSON|SHIP|uickly final 2054|5970|971|5|40|75038.80|0.08|0.06|R|F|1992-06-23|1992-08-09|1992-07-04|TAKE BACK RETURN|RAIL|n pinto beans. ironic courts are iro 2054|133446|5960|6|17|25150.48|0.08|0.01|A|F|1992-06-09|1992-08-28|1992-06-16|NONE|AIR|ges nag acc 2054|10356|7860|7|4|5065.40|0.00|0.08|R|F|1992-08-12|1992-08-31|1992-08-15|DELIVER IN PERSON|AIR|lyly careful requests wake fl 2055|44447|4448|1|15|20871.60|0.04|0.06|A|F|1993-09-15|1993-10-06|1993-10-07|NONE|REG AIR|furiously bold 2055|8279|8280|2|15|17809.05|0.06|0.05|R|F|1993-10-30|1993-11-21|1993-11-22|COLLECT COD|RAIL|gular foxes. b 2055|134333|6847|3|12|16407.96|0.00|0.02|A|F|1993-10-26|1993-11-23|1993-11-22|COLLECT COD|TRUCK|al pains. acco 2055|133702|6216|4|16|27771.20|0.02|0.02|A|F|1993-11-16|1993-11-12|1993-11-28|NONE|TRUCK|arefully daringly regular accounts. 2080|6855|4356|1|5|8809.25|0.08|0.05|R|F|1993-08-26|1993-08-07|1993-09-02|DELIVER IN PERSON|TRUCK|refully unusual theo 2080|196959|9479|2|39|80182.05|0.07|0.04|A|F|1993-08-22|1993-09-09|1993-08-23|COLLECT COD|FOB|ic deposits haggle slyly carefully eve 2081|88449|5974|1|26|37373.44|0.03|0.08|N|O|1997-10-21|1997-10-03|1997-11-10|NONE|FOB|among the slyly express accounts. silen 2081|148309|3338|2|13|17644.90|0.07|0.05|N|O|1997-08-23|1997-08-22|1997-09-09|TAKE BACK RETURN|MAIL|fter the even deposi 2081|12941|7944|3|32|59326.08|0.09|0.07|N|O|1997-09-05|1997-09-26|1997-10-03|TAKE BACK RETURN|SHIP|e. final, regular dependencies sleep slyly! 2081|84662|4663|4|23|37873.18|0.03|0.08|N|O|1997-07-06|1997-09-11|1997-07-21|TAKE BACK RETURN|MAIL|ual requests wake blithely above the 2081|112141|4653|5|19|21909.66|0.02|0.06|N|O|1997-10-01|1997-08-12|1997-10-18|COLLECT COD|SHIP|s affix sometimes express requests. quickly 2081|141545|4060|6|31|49182.74|0.03|0.06|N|O|1997-09-19|1997-09-13|1997-09-27|NONE|AIR| silent, spe 2082|74452|1974|1|36|51352.20|0.00|0.00|R|F|1995-01-20|1995-03-18|1995-01-31|COLLECT COD|MAIL|haggle furiously silent pinto beans 2082|104536|9557|2|12|18486.36|0.08|0.05|A|F|1995-01-27|1995-02-11|1995-02-07|NONE|FOB| ironic instructions. carefull 2083|23917|8922|1|37|68113.67|0.07|0.00|R|F|1993-09-07|1993-09-30|1993-09-18|TAKE BACK RETURN|MAIL|ng the special foxes wake packages. f 2084|181162|8717|1|42|52212.72|0.03|0.05|A|F|1993-03-29|1993-05-05|1993-04-22|COLLECT COD|REG AIR|y fluffily even foxes. 2084|179838|2356|2|23|44110.09|0.09|0.08|A|F|1993-06-05|1993-05-26|1993-06-06|DELIVER IN PERSON|AIR|es against 2084|135354|2894|3|37|51405.95|0.07|0.05|A|F|1993-07-16|1993-04-20|1993-08-06|NONE|AIR|y careful courts. 2084|93773|1301|4|9|15900.93|0.02|0.02|A|F|1993-03-18|1993-06-08|1993-03-30|NONE|TRUCK|heaves boost slyly after the pla 2084|26853|4360|5|28|49835.80|0.07|0.02|R|F|1993-05-04|1993-05-14|1993-05-31|COLLECT COD|TRUCK|cajole quickly carefu 2084|114964|7476|6|15|29684.40|0.09|0.04|A|F|1993-06-23|1993-04-25|1993-07-23|COLLECT COD|SHIP|tithes. bravely pendi 2084|193410|5930|7|34|51115.94|0.09|0.02|R|F|1993-06-20|1993-05-28|1993-06-25|DELIVER IN PERSON|RAIL| carefully ironic requests. fluffil 2085|40196|2701|1|45|51128.55|0.00|0.07|R|F|1994-02-27|1994-01-11|1994-03-29|TAKE BACK RETURN|MAIL|. carefully e 2086|59829|9830|1|22|39354.04|0.03|0.07|R|F|1994-12-04|1994-12-16|1994-12-20|DELIVER IN PERSON|RAIL|idly busy acc 2086|140912|8455|2|32|62493.12|0.04|0.06|A|F|1994-11-15|1995-01-05|1994-12-09|TAKE BACK RETURN|TRUCK|e carefully along th 2086|104938|4939|3|44|85488.92|0.02|0.01|A|F|1994-12-04|1994-11-30|1994-12-21|DELIVER IN PERSON|FOB|latelets s 2086|83891|8908|4|27|50622.03|0.02|0.00|A|F|1994-11-04|1995-01-14|1994-11-25|COLLECT COD|REG AIR|theodolites haggle blithely blithe p 2086|155067|98|5|33|37027.98|0.04|0.00|A|F|1995-02-06|1994-11-25|1995-02-15|NONE|SHIP| slyly regular foxes. un 2086|199165|4204|6|20|25283.20|0.01|0.03|R|F|1994-11-30|1994-12-28|1994-12-07|COLLECT COD|FOB|lithely ironic acc 2086|155955|3501|7|7|14076.65|0.04|0.05|R|F|1994-12-27|1994-12-10|1995-01-05|COLLECT COD|RAIL| beans haggle car 2087|126684|6685|1|1|1710.68|0.05|0.04|N|O|1998-03-27|1998-03-24|1998-04-18|DELIVER IN PERSON|REG AIR|the quickly idle acco 2087|167256|4805|2|46|60869.50|0.10|0.03|N|O|1998-02-24|1998-04-02|1998-03-04|DELIVER IN PERSON|AIR|ter the dolphins. 2087|61740|1741|3|1|1701.74|0.02|0.05|N|O|1998-05-27|1998-04-11|1998-06-12|COLLECT COD|REG AIR|hely final acc 2087|58738|6254|4|6|10180.38|0.03|0.08|N|O|1998-04-23|1998-03-27|1998-05-18|DELIVER IN PERSON|REG AIR|dazzle after the slyly si 2112|70095|96|1|18|19171.62|0.02|0.05|N|O|1997-05-02|1997-03-16|1997-05-25|TAKE BACK RETURN|TRUCK|lphins solve ideas. even, special reque 2113|122730|5243|1|40|70109.20|0.04|0.06|N|O|1998-01-16|1997-12-11|1998-02-06|TAKE BACK RETURN|TRUCK|bout the quickly ironic t 2113|111214|8748|2|24|29405.04|0.03|0.02|N|O|1998-02-19|1998-01-08|1998-03-16|COLLECT COD|MAIL|kly regular accounts hinder about the 2114|167477|7478|1|50|77223.50|0.05|0.05|A|F|1995-02-05|1995-03-18|1995-02-13|COLLECT COD|RAIL|pecial pinto bean 2114|185076|113|2|26|30187.82|0.02|0.02|A|F|1995-04-30|1995-04-16|1995-05-28|NONE|SHIP|ar asymptotes sleep 2114|161760|4277|3|25|45544.00|0.07|0.01|A|F|1995-02-15|1995-03-13|1995-02-22|COLLECT COD|AIR|unts. regular, express accounts wake. b 2115|195180|7700|1|27|34429.86|0.06|0.03|N|O|1998-09-01|1998-07-29|1998-09-04|NONE|AIR|de of the carefully bold accounts 2115|183023|578|2|43|47558.86|0.06|0.02|N|O|1998-07-14|1998-07-25|1998-07-24|COLLECT COD|FOB| carefully pending requests alongs 2115|50851|8367|3|3|5405.55|0.03|0.04|N|O|1998-07-23|1998-07-30|1998-08-14|DELIVER IN PERSON|FOB|quickly ironic dolphin 2115|48123|8124|4|47|50342.64|0.06|0.07|N|O|1998-08-29|1998-07-30|1998-09-05|TAKE BACK RETURN|REG AIR|regular accounts integrate brav 2115|198956|6514|5|13|26714.35|0.04|0.00|N|O|1998-08-07|1998-08-06|1998-08-13|DELIVER IN PERSON|REG AIR|into beans. even accounts abou 2116|130050|51|1|2|2160.10|0.00|0.02|R|F|1994-10-16|1994-11-24|1994-11-09|DELIVER IN PERSON|TRUCK|r theodolites use blithely about the ir 2116|139382|9383|2|47|66804.86|0.10|0.06|R|F|1994-09-01|1994-11-18|1994-09-25|COLLECT COD|MAIL|iously ironic dependencies around the iro 2116|183247|5766|3|11|14632.64|0.03|0.05|R|F|1994-09-15|1994-10-21|1994-09-21|NONE|FOB| pinto beans. final, final sauternes play 2117|164531|9564|1|36|57439.08|0.10|0.01|N|O|1997-08-06|1997-07-15|1997-08-07|DELIVER IN PERSON|SHIP|ronic accounts wake 2117|60193|7712|2|19|21910.61|0.04|0.00|N|O|1997-07-30|1997-06-18|1997-08-13|DELIVER IN PERSON|REG AIR|s between the slyly regula 2117|57434|2445|3|43|59831.49|0.04|0.03|N|O|1997-06-27|1997-06-12|1997-07-22|DELIVER IN PERSON|SHIP| foxes sleep furiously 2117|90507|5526|4|24|35940.00|0.00|0.07|N|O|1997-06-15|1997-05-27|1997-06-18|COLLECT COD|SHIP|thely slyly pending platelets. ironic, 2117|146773|6774|5|3|5459.31|0.02|0.05|N|O|1997-05-05|1997-07-20|1997-05-26|TAKE BACK RETURN|TRUCK|tes cajole 2117|179|2680|6|27|29137.59|0.09|0.08|N|O|1997-06-30|1997-06-27|1997-07-11|TAKE BACK RETURN|REG AIR| the carefully ironic ideas 2118|159552|9553|1|24|38677.20|0.10|0.03|N|O|1997-01-06|1996-12-14|1997-01-14|TAKE BACK RETURN|RAIL|about the slyly bold depende 2118|183278|3279|2|4|5445.08|0.08|0.01|N|O|1996-10-25|1996-11-10|1996-11-22|COLLECT COD|AIR|theodolites affix according 2118|144382|1925|3|11|15690.18|0.05|0.04|N|O|1996-12-23|1996-12-20|1997-01-01|COLLECT COD|RAIL|y ironic accounts sleep upon the packages. 2119|101751|6772|1|36|63099.00|0.04|0.00|N|O|1996-11-10|1996-10-25|1996-12-03|TAKE BACK RETURN|RAIL|ly bold foxes. ironic accoun 2144|91477|9005|1|33|48459.51|0.00|0.07|R|F|1994-04-04|1994-06-20|1994-04-23|NONE|AIR| ironic excuses haggle final dependencies. 2144|50884|3390|2|46|84404.48|0.03|0.08|R|F|1994-04-08|1994-04-29|1994-05-07|COLLECT COD|SHIP| foxes haggle blithel 2144|3496|8497|3|29|40585.21|0.00|0.07|R|F|1994-05-03|1994-05-16|1994-06-01|DELIVER IN PERSON|FOB|ns wake carefully carefully ironic 2144|157171|7172|4|10|12281.70|0.00|0.04|R|F|1994-06-16|1994-05-03|1994-07-05|COLLECT COD|AIR| furiously unusual ideas. carefull 2145|77370|9878|1|13|17515.81|0.04|0.05|A|F|1992-11-12|1992-12-13|1992-12-07|TAKE BACK RETURN|MAIL|alongside of the slyly final 2145|153894|1440|2|6|11687.34|0.05|0.01|A|F|1992-10-10|1992-11-29|1992-10-14|NONE|AIR|s. fluffily express accounts sleep. slyl 2146|56891|9397|1|42|77611.38|0.10|0.01|A|F|1992-09-21|1992-11-02|1992-09-23|NONE|AIR|ns according to the doggedly 2146|156627|9143|2|6|10101.72|0.07|0.05|A|F|1993-01-03|1992-10-24|1993-01-24|DELIVER IN PERSON|RAIL|ing to the requests. dependencies boost 2146|24361|1868|3|14|17995.04|0.03|0.01|R|F|1992-09-16|1992-10-16|1992-09-20|COLLECT COD|SHIP|ecial, express a 2146|25952|3459|4|31|58216.45|0.02|0.00|A|F|1993-01-04|1992-10-24|1993-01-15|DELIVER IN PERSON|TRUCK|lly even deposit 2146|168542|6091|5|28|45095.12|0.02|0.05|R|F|1993-01-03|1992-10-17|1993-01-08|COLLECT COD|MAIL|r accounts sleep furio 2146|70175|7697|6|32|36645.44|0.07|0.03|R|F|1993-01-10|1992-10-19|1993-02-05|COLLECT COD|TRUCK|y regular foxes wake among the final 2146|24278|4279|7|39|46888.53|0.07|0.06|R|F|1993-01-05|1992-11-06|1993-01-14|DELIVER IN PERSON|TRUCK|uickly regular excuses detect. regular c 2147|28768|3773|1|50|84838.00|0.04|0.06|R|F|1992-11-18|1992-11-30|1992-11-30|NONE|RAIL|al accounts. even, even foxes wake 2147|100604|605|2|4|6418.40|0.01|0.04|A|F|1992-09-27|1992-11-15|1992-10-22|NONE|AIR|mong the blithely special 2147|43483|8492|3|34|48500.32|0.10|0.04|R|F|1992-11-29|1992-11-08|1992-12-22|TAKE BACK RETURN|REG AIR|egular deposits hang car 2147|10236|5239|4|11|12608.53|0.06|0.07|A|F|1992-09-27|1992-11-16|1992-10-16|NONE|AIR| the fluffily 2148|115933|3467|1|21|40927.53|0.09|0.01|R|F|1995-05-28|1995-05-26|1995-06-15|NONE|FOB|deposits ag 2149|18213|5717|1|12|13574.52|0.05|0.07|R|F|1993-06-01|1993-05-06|1993-06-11|TAKE BACK RETURN|TRUCK|riously bl 2149|98974|8975|2|10|19729.70|0.06|0.01|R|F|1993-06-09|1993-04-17|1993-06-16|DELIVER IN PERSON|TRUCK|eposits sleep above 2149|48678|3687|3|47|76453.49|0.00|0.04|R|F|1993-06-27|1993-05-12|1993-07-11|COLLECT COD|AIR|hely final depo 2149|128105|3130|4|18|20395.80|0.06|0.00|A|F|1993-04-05|1993-05-11|1993-04-23|DELIVER IN PERSON|REG AIR|uriously final pac 2149|59360|4371|5|22|29025.92|0.06|0.04|R|F|1993-05-24|1993-04-23|1993-06-20|TAKE BACK RETURN|SHIP|ptotes sleep along the blithely ir 2150|77886|2901|1|26|48460.88|0.00|0.03|A|F|1994-06-21|1994-08-05|1994-06-23|NONE|TRUCK|. always unusual packages 2150|17738|5242|2|29|48016.17|0.04|0.03|A|F|1994-09-02|1994-08-04|1994-10-02|TAKE BACK RETURN|RAIL|y ironic theodolites. foxes ca 2150|106769|1790|3|29|51497.04|0.04|0.08|R|F|1994-06-10|1994-07-31|1994-06-26|COLLECT COD|RAIL|arefully final att 2150|53727|1243|4|39|65548.08|0.05|0.02|R|F|1994-07-31|1994-08-17|1994-08-11|TAKE BACK RETURN|TRUCK|ess accounts nag. unusual asymptotes haggl 2150|182496|2497|5|35|55247.15|0.01|0.01|A|F|1994-09-27|1994-08-17|1994-10-13|COLLECT COD|RAIL|refully pending dependen 2150|6331|8832|6|12|14847.96|0.09|0.03|A|F|1994-08-27|1994-08-22|1994-09-18|COLLECT COD|AIR|press platelets haggle until the slyly fi 2151|166944|4493|1|23|46251.62|0.06|0.02|N|O|1996-11-20|1996-12-17|1996-11-30|DELIVER IN PERSON|AIR| silent dependencies about the slyl 2151|14480|6982|2|29|40439.92|0.00|0.02|N|O|1997-03-04|1996-12-27|1997-03-21|TAKE BACK RETURN|SHIP| bold packages acro 2151|164838|9871|3|49|93238.67|0.07|0.01|N|O|1997-01-20|1997-02-09|1997-02-18|NONE|FOB| packages. f 2151|17670|2673|4|28|44454.76|0.10|0.08|N|O|1996-12-11|1996-12-26|1996-12-12|DELIVER IN PERSON|AIR|y special packages. carefully ironic instru 2176|190213|5252|1|38|49521.98|0.02|0.08|R|F|1992-11-29|1993-01-14|1992-12-22|DELIVER IN PERSON|REG AIR|lithely ironic pinto beans. furious 2176|94137|9156|2|14|15835.82|0.00|0.06|A|F|1992-11-17|1993-01-07|1992-12-03|DELIVER IN PERSON|SHIP|ely ironic platelets 2176|159466|9467|3|25|38136.50|0.02|0.02|R|F|1993-02-23|1993-01-05|1993-03-07|COLLECT COD|RAIL| ruthless deposits according to the ent 2176|142226|7255|4|2|2536.44|0.05|0.06|A|F|1993-02-26|1993-01-08|1993-03-23|DELIVER IN PERSON|AIR|s pinto beans 2177|128562|8563|1|45|71575.20|0.02|0.01|N|O|1997-02-11|1997-02-27|1997-02-17|NONE|SHIP|. theodolites haggle carefu 2177|138448|962|2|27|40133.88|0.04|0.08|N|O|1997-01-29|1997-03-20|1997-02-04|DELIVER IN PERSON|SHIP|even, regula 2177|80066|2575|3|23|24059.38|0.07|0.05|N|O|1997-01-28|1997-03-02|1997-02-13|DELIVER IN PERSON|AIR|he silent foxes. iro 2177|54933|7439|4|34|64189.62|0.05|0.07|N|O|1997-02-03|1997-04-10|1997-02-21|COLLECT COD|REG AIR|tes are doggedly quickly 2177|56437|3953|5|46|64097.78|0.09|0.05|N|O|1997-05-10|1997-02-23|1997-05-28|COLLECT COD|RAIL|ending asymptotes. 2177|121616|4129|6|11|18013.71|0.02|0.04|N|O|1997-03-20|1997-03-07|1997-04-09|DELIVER IN PERSON|MAIL|gainst the ca 2178|156127|1158|1|15|17746.80|0.10|0.01|N|O|1997-03-27|1997-03-10|1997-04-18|NONE|REG AIR|l accounts. quickly expr 2178|15804|8306|2|27|46434.60|0.01|0.02|N|O|1997-02-26|1997-02-19|1997-03-25|NONE|MAIL| across the ironic reques 2178|4825|2326|3|40|69192.80|0.00|0.03|N|O|1997-03-17|1997-02-09|1997-04-15|COLLECT COD|RAIL|foxes are slowly regularly specia 2178|77639|5161|4|3|4849.89|0.07|0.07|N|O|1997-04-07|1997-01-23|1997-04-18|COLLECT COD|MAIL| permanentl 2179|129479|4504|1|22|33186.34|0.05|0.08|N|O|1996-11-16|1996-11-03|1996-11-25|DELIVER IN PERSON|FOB|lphins cajole acr 2179|138056|570|2|20|21881.00|0.03|0.01|N|O|1996-09-30|1996-11-10|1996-10-30|NONE|REG AIR|ncies. fin 2179|103363|8384|3|5|6831.80|0.03|0.02|N|O|1996-11-09|1996-10-08|1996-11-11|DELIVER IN PERSON|REG AIR|ts haggle blithely. ironic, careful theodol 2179|5267|2768|4|24|28134.24|0.04|0.04|N|O|1996-10-26|1996-11-05|1996-11-16|COLLECT COD|RAIL| cajole carefully. 2179|107337|4868|5|7|9410.31|0.00|0.02|N|O|1996-10-24|1996-11-14|1996-11-21|TAKE BACK RETURN|RAIL|gular dependencies. ironic packages haggle 2180|15125|128|1|31|32243.72|0.06|0.04|N|O|1996-10-20|1996-11-21|1996-11-06|COLLECT COD|REG AIR|n requests are furiously at the quickly 2180|192230|9788|2|39|51566.97|0.01|0.00|N|O|1997-01-03|1996-10-29|1997-01-25|NONE|RAIL|ep furiously furiously final request 2180|196985|9505|3|24|49967.52|0.03|0.00|N|O|1997-01-03|1996-10-24|1997-01-19|NONE|SHIP|uriously f 2180|110249|2761|4|47|59184.28|0.07|0.02|N|O|1996-09-23|1996-12-08|1996-10-12|NONE|FOB|pending, regular ideas. iron 2180|142825|368|5|23|42959.86|0.02|0.06|N|O|1996-11-08|1996-10-25|1996-11-28|NONE|TRUCK|ggle alongside of the fluffily speci 2180|54140|4141|6|48|52518.72|0.09|0.03|N|O|1996-12-30|1996-11-22|1997-01-16|DELIVER IN PERSON|RAIL|nic instructions haggle careful 2181|177848|7849|1|4|7703.36|0.05|0.04|N|O|1995-09-25|1995-11-12|1995-09-28|COLLECT COD|FOB|tes. slyly silent packages use along th 2181|87791|2808|2|46|81824.34|0.00|0.02|N|O|1995-11-28|1995-10-17|1995-12-26|COLLECT COD|AIR|osits. final packages sleep 2181|90740|741|3|15|25961.10|0.08|0.05|N|O|1995-10-05|1995-10-27|1995-11-03|DELIVER IN PERSON|FOB|e above the fluffily regul 2181|54662|9673|4|28|45266.48|0.04|0.05|N|O|1995-12-21|1995-10-23|1996-01-04|TAKE BACK RETURN|AIR|s excuses sleep car 2181|95783|5784|5|9|16009.02|0.06|0.07|N|O|1996-01-05|1995-12-05|1996-01-08|COLLECT COD|TRUCK|ward the quietly even requests. ir 2182|131671|9211|1|27|45972.09|0.02|0.07|R|F|1994-05-10|1994-07-04|1994-06-04|DELIVER IN PERSON|SHIP|en platele 2182|189760|7315|2|3|5549.28|0.05|0.03|R|F|1994-04-20|1994-07-04|1994-04-24|TAKE BACK RETURN|SHIP|y bold theodolites wi 2182|93343|5853|3|34|45435.56|0.02|0.03|R|F|1994-05-28|1994-06-02|1994-06-10|COLLECT COD|MAIL| slow tithes. ironi 2182|6068|3569|4|12|11688.72|0.04|0.07|A|F|1994-05-08|1994-06-02|1994-05-09|COLLECT COD|REG AIR|ments are fu 2182|178869|1387|5|37|72070.82|0.06|0.02|A|F|1994-04-08|1994-06-29|1994-04-18|TAKE BACK RETURN|TRUCK|ges. blithely ironic 2183|70665|3173|1|29|47434.14|0.05|0.01|N|O|1996-07-21|1996-08-24|1996-08-15|TAKE BACK RETURN|RAIL|ly unusual deposits sleep carefully 2183|51094|1095|2|25|26127.25|0.06|0.02|N|O|1996-07-06|1996-08-21|1996-08-05|NONE|RAIL|he quickly f 2208|57232|2243|1|48|57083.04|0.08|0.07|A|F|1995-05-13|1995-06-30|1995-05-20|COLLECT COD|MAIL|sits. idly permanent request 2208|96798|4326|2|11|19742.69|0.08|0.01|A|F|1995-05-06|1995-07-19|1995-05-22|COLLECT COD|TRUCK|ding waters lose. furiously regu 2208|73389|5897|3|41|55857.58|0.08|0.02|N|O|1995-08-18|1995-06-19|1995-09-05|COLLECT COD|RAIL|nd the furious, express dependencies. 2208|42342|9855|4|50|64217.00|0.07|0.07|N|F|1995-06-11|1995-05-31|1995-06-29|TAKE BACK RETURN|FOB|al foxes will hav 2208|29570|2073|5|43|64481.51|0.03|0.06|A|F|1995-05-10|1995-06-02|1995-06-09|TAKE BACK RETURN|MAIL|es. accounts cajole. fi 2208|166229|3778|6|18|23313.96|0.02|0.08|R|F|1995-06-06|1995-06-10|1995-06-11|TAKE BACK RETURN|TRUCK|packages are quickly bold de 2208|6106|1107|7|45|45544.50|0.00|0.08|A|F|1995-05-05|1995-06-10|1995-05-11|NONE|SHIP|e fluffily regular theodolites caj 2209|22676|7681|1|40|63946.80|0.05|0.01|R|F|1992-11-01|1992-09-25|1992-11-08|DELIVER IN PERSON|SHIP|ully special sheaves serve 2209|102771|2772|2|10|17737.70|0.00|0.02|R|F|1992-09-02|1992-09-24|1992-09-21|DELIVER IN PERSON|AIR|players. carefully reg 2209|63521|8534|3|11|16329.72|0.01|0.01|A|F|1992-07-12|1992-08-24|1992-08-10|DELIVER IN PERSON|REG AIR|express, regular pinto be 2209|180897|3416|4|39|77137.71|0.08|0.07|R|F|1992-11-04|1992-09-02|1992-11-11|TAKE BACK RETURN|MAIL|ly around the final packages. deposits ca 2209|123364|901|5|24|33296.64|0.08|0.06|R|F|1992-08-09|1992-08-18|1992-08-25|COLLECT COD|AIR| along the bol 2209|177413|2448|6|7|10432.87|0.09|0.07|A|F|1992-08-18|1992-09-09|1992-09-12|DELIVER IN PERSON|AIR| quickly regular pack 2210|77764|2779|1|36|62703.36|0.10|0.00|A|F|1992-03-04|1992-03-24|1992-03-21|DELIVER IN PERSON|AIR| requests wake enticingly final 2211|47822|2831|1|25|44245.50|0.04|0.01|A|F|1994-10-09|1994-08-04|1994-11-03|TAKE BACK RETURN|RAIL|deas. carefully special theodolites along 2211|139773|2287|2|40|72510.80|0.09|0.06|A|F|1994-09-30|1994-09-10|1994-10-26|NONE|MAIL|posits among the express dolphins 2211|159820|7366|3|25|46995.50|0.00|0.07|A|F|1994-08-13|1994-08-17|1994-08-16|NONE|AIR|ly regular, express 2211|84711|9728|4|23|39001.33|0.03|0.02|R|F|1994-10-05|1994-09-13|1994-10-17|DELIVER IN PERSON|AIR|ependencies 2211|134301|6815|5|3|4005.90|0.02|0.04|A|F|1994-08-28|1994-09-10|1994-09-06|TAKE BACK RETURN|SHIP|pendencies after the regular f 2211|186189|6190|6|18|22953.24|0.05|0.08|A|F|1994-08-31|1994-09-07|1994-09-22|NONE|TRUCK|c grouches. slyly express pinto 2211|78139|647|7|3|3351.39|0.06|0.05|R|F|1994-09-21|1994-08-10|1994-10-19|TAKE BACK RETURN|RAIL|y slyly final 2212|70810|5825|1|18|32054.58|0.07|0.06|R|F|1994-06-22|1994-06-18|1994-06-25|TAKE BACK RETURN|FOB| cajole. final, pending ideas should are bl 2213|117158|4692|1|20|23503.00|0.01|0.00|A|F|1993-01-21|1993-04-14|1993-01-29|COLLECT COD|REG AIR|iously express accounts; 2213|59646|9647|2|4|6422.56|0.09|0.05|R|F|1993-04-15|1993-04-15|1993-05-05|COLLECT COD|SHIP| affix carefully furiously 2213|69541|7060|3|1|1510.54|0.05|0.05|A|F|1993-04-25|1993-04-06|1993-04-28|TAKE BACK RETURN|AIR|s along the ironic reques 2213|173204|8239|4|39|49810.80|0.09|0.05|A|F|1993-05-12|1993-04-07|1993-05-23|TAKE BACK RETURN|SHIP|the blithely 2213|37976|2983|5|43|82300.71|0.04|0.03|A|F|1993-04-18|1993-03-11|1993-05-11|TAKE BACK RETURN|RAIL|r packages are along the carefully bol 2213|47675|180|6|41|66529.47|0.01|0.00|R|F|1993-01-31|1993-03-31|1993-02-28|COLLECT COD|FOB| carefully pend 2213|63836|1355|7|3|5399.49|0.02|0.04|A|F|1993-03-09|1993-03-17|1993-04-07|TAKE BACK RETURN|AIR|o wake. ironic platel 2214|75623|638|1|27|43162.74|0.04|0.04|N|O|1998-05-31|1998-06-07|1998-06-19|DELIVER IN PERSON|REG AIR|x fluffily along the even packages-- 2214|193950|3951|2|50|102197.50|0.00|0.02|N|O|1998-07-06|1998-06-16|1998-07-16|TAKE BACK RETURN|MAIL|accounts. blith 2214|112406|4918|3|42|59572.80|0.04|0.08|N|O|1998-05-26|1998-07-13|1998-06-22|COLLECT COD|FOB|ons. deposi 2214|195389|428|4|22|32656.36|0.01|0.01|N|O|1998-05-30|1998-07-02|1998-06-09|DELIVER IN PERSON|RAIL|t the blithely 2215|72044|9566|1|33|33529.32|0.00|0.00|N|O|1996-07-19|1996-08-10|1996-07-30|COLLECT COD|RAIL|dolites cajole b 2215|32526|5030|2|30|43755.60|0.01|0.00|N|O|1996-08-15|1996-09-10|1996-08-25|NONE|FOB|ckages caj 2215|56652|9158|3|30|48259.50|0.07|0.03|N|O|1996-09-09|1996-07-20|1996-09-28|COLLECT COD|TRUCK|against the carefu 2215|145112|7627|4|20|23142.20|0.02|0.02|N|O|1996-09-09|1996-08-10|1996-09-19|NONE|MAIL| unusual deposits haggle carefully. ide 2240|163898|6415|1|6|11771.34|0.01|0.00|A|F|1992-06-23|1992-05-17|1992-07-20|COLLECT COD|AIR|ymptotes boost. furiously bold p 2240|27897|5404|2|37|67520.93|0.03|0.07|R|F|1992-03-16|1992-05-31|1992-04-05|COLLECT COD|FOB| quickly after the packages? blithely si 2240|52103|9619|3|39|41148.90|0.08|0.06|A|F|1992-05-22|1992-05-10|1992-06-08|NONE|FOB|y orbits. final depos 2240|85867|884|4|10|18528.60|0.09|0.00|A|F|1992-05-25|1992-04-14|1992-06-23|DELIVER IN PERSON|REG AIR|are across the ironic packages. 2240|160522|3039|5|29|45893.08|0.02|0.06|A|F|1992-03-29|1992-05-08|1992-04-09|COLLECT COD|MAIL|lyly even ideas w 2240|80721|722|6|32|54455.04|0.06|0.06|R|F|1992-04-11|1992-04-18|1992-04-22|NONE|MAIL|ss thinly deposits. blithely bold package 2240|77504|2519|7|24|35556.00|0.04|0.05|R|F|1992-05-13|1992-04-09|1992-05-14|DELIVER IN PERSON|FOB|ng the silent accounts. slyly ironic t 2241|4034|4035|1|25|23450.75|0.00|0.08|R|F|1993-08-11|1993-07-23|1993-09-01|DELIVER IN PERSON|MAIL| final deposits use fluffily. even f 2241|194616|9655|2|38|65003.18|0.04|0.06|A|F|1993-08-04|1993-07-31|1993-08-06|TAKE BACK RETURN|TRUCK| silent, unusual d 2241|96673|1692|3|48|80144.16|0.08|0.04|A|F|1993-05-14|1993-07-30|1993-05-26|TAKE BACK RETURN|RAIL|ss accounts engage furiously. slyly even re 2241|166305|1338|4|19|26054.70|0.10|0.00|A|F|1993-06-01|1993-08-05|1993-06-07|TAKE BACK RETURN|TRUCK| are furiously quickl 2241|81485|1486|5|2|2932.96|0.04|0.03|A|F|1993-08-16|1993-08-02|1993-08-24|NONE|REG AIR|, express deposits. pear 2241|115401|424|6|22|31160.80|0.02|0.08|R|F|1993-08-13|1993-06-15|1993-08-16|DELIVER IN PERSON|TRUCK|, ironic depen 2241|141592|1593|7|9|14702.31|0.09|0.03|A|F|1993-05-14|1993-07-12|1993-05-29|NONE|AIR|lyly final 2242|122099|2100|1|15|16816.35|0.09|0.08|N|O|1997-08-04|1997-09-21|1997-08-11|COLLECT COD|FOB|its. carefully express packages cajole. bli 2243|126988|6989|1|10|20149.80|0.04|0.06|N|O|1995-07-26|1995-07-18|1995-08-03|NONE|RAIL|express, daring foxes affix fur 2244|50086|5097|1|3|3108.24|0.02|0.02|A|F|1993-04-30|1993-03-15|1993-05-19|TAKE BACK RETURN|FOB| beans for the regular platel 2244|192426|7465|2|16|24294.72|0.01|0.06|R|F|1993-02-12|1993-03-09|1993-02-28|COLLECT COD|FOB|rate around the reques 2245|75664|5665|1|44|72145.04|0.03|0.03|A|F|1993-06-12|1993-06-10|1993-06-16|NONE|TRUCK|refully even sheaves 2245|73630|8645|2|28|44901.64|0.05|0.03|R|F|1993-08-19|1993-07-27|1993-09-04|COLLECT COD|TRUCK|e requests sleep furiou 2245|85310|7819|3|33|42745.23|0.03|0.01|R|F|1993-06-26|1993-06-11|1993-07-17|TAKE BACK RETURN|AIR|ing to the carefully ruthless accounts 2245|188474|8475|4|14|21874.58|0.02|0.04|R|F|1993-05-06|1993-07-21|1993-05-19|DELIVER IN PERSON|RAIL|nts. always unusual dep 2245|79264|6786|5|33|41027.58|0.03|0.07|R|F|1993-06-16|1993-06-05|1993-07-07|NONE|MAIL| across the express reques 2246|52330|2331|1|22|28211.26|0.02|0.01|N|O|1996-07-25|1996-08-03|1996-08-24|DELIVER IN PERSON|SHIP|ructions wake carefully fina 2246|103090|3091|2|43|47002.87|0.07|0.06|N|O|1996-08-25|1996-08-23|1996-09-19|DELIVER IN PERSON|AIR|ainst the ironic theodolites haggle fi 2246|17158|4662|3|11|11826.65|0.10|0.00|N|O|1996-06-21|1996-07-24|1996-07-18|TAKE BACK RETURN|TRUCK|quests alongside o 2246|162721|270|4|13|23188.36|0.08|0.05|N|O|1996-09-15|1996-07-21|1996-10-08|DELIVER IN PERSON|AIR|equests. fluffily special epitaphs use 2247|171922|4440|1|12|23927.04|0.02|0.07|A|F|1992-09-06|1992-09-18|1992-09-26|NONE|MAIL|final accounts. requests across the furiou 2272|89989|9990|1|18|35621.64|0.04|0.00|R|F|1993-08-01|1993-07-06|1993-08-25|NONE|MAIL|ons along the blithely e 2272|33466|976|2|40|55978.40|0.07|0.00|A|F|1993-04-25|1993-07-12|1993-05-15|DELIVER IN PERSON|FOB|lithely ir 2272|55426|7932|3|36|49731.12|0.03|0.02|A|F|1993-05-25|1993-05-23|1993-06-09|TAKE BACK RETURN|RAIL|about the ironic packages; quickly iron 2272|137602|116|4|30|49188.00|0.09|0.07|A|F|1993-07-27|1993-05-15|1993-08-13|NONE|RAIL|quests at the foxes haggle evenly pack 2272|75858|3380|5|12|22006.20|0.03|0.03|A|F|1993-04-19|1993-05-14|1993-04-23|NONE|RAIL| accounts cajole. quickly b 2273|183596|8633|1|34|57106.06|0.02|0.03|N|O|1997-01-08|1997-02-02|1997-01-23|COLLECT COD|MAIL| furiously carefully bold de 2273|84638|9655|2|35|56792.05|0.00|0.05|N|O|1997-01-02|1997-01-19|1997-01-14|NONE|REG AIR|arefully f 2273|94138|9157|3|8|9057.04|0.00|0.04|N|O|1996-12-15|1997-02-27|1997-01-10|NONE|FOB|dependencies. slyly ir 2273|160895|8444|4|20|39117.80|0.06|0.04|N|O|1997-03-05|1997-02-25|1997-04-01|NONE|RAIL|cuses. quickly enticing requests wake 2273|161026|8575|5|18|19566.36|0.07|0.00|N|O|1996-12-16|1997-01-21|1997-01-03|COLLECT COD|TRUCK| beans. doggedly final packages wake 2273|154366|1912|6|16|22725.76|0.10|0.03|N|O|1997-01-10|1997-02-03|1997-02-01|TAKE BACK RETURN|RAIL|furiously above the ironic requests. 2273|19493|9494|7|7|9887.43|0.05|0.05|N|O|1997-02-19|1997-01-22|1997-02-21|TAKE BACK RETURN|TRUCK|ts. furiou 2274|11038|3540|1|18|17082.54|0.04|0.03|R|F|1993-09-06|1993-12-03|1993-09-22|COLLECT COD|SHIP|usly final re 2274|110510|5533|2|23|34971.73|0.04|0.03|R|F|1993-10-28|1993-11-03|1993-11-05|NONE|MAIL|kly special warhorse 2274|128795|8796|3|18|32828.22|0.03|0.06|R|F|1993-09-28|1993-11-22|1993-10-12|DELIVER IN PERSON|SHIP| express packages. even accounts hagg 2275|33183|8190|1|30|33485.40|0.08|0.05|R|F|1993-01-10|1992-11-21|1993-01-22|NONE|REG AIR|re slyly slyly special idea 2275|90399|5418|2|11|15283.29|0.08|0.03|A|F|1993-01-16|1992-12-10|1993-01-25|COLLECT COD|REG AIR|ost across the never express instruction 2276|118185|5719|1|5|6015.90|0.07|0.08|N|O|1996-05-09|1996-06-18|1996-05-13|DELIVER IN PERSON|FOB|ias instea 2276|134871|2411|2|13|24776.31|0.08|0.04|N|O|1996-07-24|1996-06-18|1996-08-16|COLLECT COD|RAIL|arefully ironic foxes cajole q 2276|170995|996|3|27|55781.73|0.07|0.08|N|O|1996-07-30|1996-06-10|1996-07-31|DELIVER IN PERSON|RAIL|the carefully unusual accoun 2276|108060|5591|4|38|40586.28|0.06|0.03|N|O|1996-07-07|1996-06-28|1996-07-17|COLLECT COD|RAIL|ans. pinto beans boost c 2276|152402|9948|5|50|72720.00|0.03|0.05|N|O|1996-07-13|1996-06-25|1996-07-22|DELIVER IN PERSON|REG AIR| accounts dete 2276|5041|7542|6|4|3784.16|0.10|0.03|N|O|1996-07-05|1996-06-30|1996-08-04|COLLECT COD|FOB|s. deposits 2277|136849|6850|1|38|71661.92|0.03|0.07|R|F|1995-04-23|1995-03-25|1995-05-20|TAKE BACK RETURN|TRUCK|fully bold 2277|7914|415|2|2|3643.82|0.10|0.08|A|F|1995-02-01|1995-02-04|1995-03-02|TAKE BACK RETURN|AIR|endencies sleep idly pending p 2277|197832|352|3|4|7719.32|0.05|0.06|R|F|1995-04-27|1995-03-16|1995-04-29|TAKE BACK RETURN|SHIP|. quickly unusual deposi 2277|158467|3498|4|31|47289.26|0.02|0.00|R|F|1995-03-07|1995-03-19|1995-03-26|TAKE BACK RETURN|MAIL|ic instructions detect ru 2278|44247|6752|1|36|42884.64|0.04|0.05|N|O|1998-06-04|1998-06-06|1998-06-30|NONE|TRUCK|y ironic pinto beans br 2278|44868|7373|2|50|90643.00|0.02|0.00|N|O|1998-08-09|1998-07-08|1998-09-05|DELIVER IN PERSON|RAIL|into beans. blit 2278|96343|8853|3|22|29465.48|0.03|0.00|N|O|1998-05-15|1998-07-14|1998-06-04|TAKE BACK RETURN|REG AIR|ep regular accounts. blithely even 2279|13676|3677|1|12|19076.04|0.07|0.08|A|F|1993-05-10|1993-03-25|1993-06-02|COLLECT COD|REG AIR|lets across the excuses nag quickl 2279|40972|973|2|38|72692.86|0.08|0.07|R|F|1993-06-09|1993-04-06|1993-06-26|COLLECT COD|TRUCK|s above the furiously express dep 2279|3443|5944|3|3|4039.32|0.09|0.04|A|F|1993-05-31|1993-05-07|1993-06-05|COLLECT COD|REG AIR|ing foxes above the even accounts use slyly 2279|51132|8648|4|42|45491.46|0.02|0.00|R|F|1993-02-28|1993-04-25|1993-03-02|TAKE BACK RETURN|REG AIR| above the furiously ironic deposits. 2279|168844|1361|5|9|17215.56|0.05|0.04|R|F|1993-05-21|1993-03-29|1993-06-17|DELIVER IN PERSON|MAIL|ns cajole after the final platelets. s 2279|146161|1190|6|12|14485.92|0.02|0.00|R|F|1993-05-04|1993-04-26|1993-05-28|DELIVER IN PERSON|FOB|ccounts. slyl 2279|118915|6449|7|32|61885.12|0.05|0.05|A|F|1993-04-20|1993-05-22|1993-05-18|DELIVER IN PERSON|RAIL|re quickly. furiously ironic ide 2304|199490|2010|1|42|66758.58|0.00|0.01|A|F|1994-01-20|1994-03-04|1994-02-05|COLLECT COD|RAIL|quests are blithely alongside of 2304|18186|5690|2|48|53000.64|0.00|0.00|R|F|1994-02-12|1994-02-16|1994-03-10|COLLECT COD|REG AIR| deposits cajole blithely e 2304|47934|7935|3|3|5645.79|0.00|0.05|R|F|1994-03-19|1994-03-04|1994-03-20|DELIVER IN PERSON|AIR|l excuses after the ev 2305|173979|6497|1|3|6158.91|0.00|0.01|A|F|1993-03-24|1993-04-05|1993-03-29|NONE|AIR|kages haggle quickly across the blithely 2305|59018|1524|2|39|38103.39|0.07|0.00|R|F|1993-04-16|1993-04-17|1993-04-22|COLLECT COD|MAIL|ms after the foxes 2305|101013|1014|3|32|32448.32|0.03|0.06|A|F|1993-04-02|1993-03-18|1993-04-03|NONE|AIR| haggle caref 2305|111463|1464|4|17|25065.82|0.00|0.05|A|F|1993-02-21|1993-03-30|1993-03-19|TAKE BACK RETURN|MAIL| carefully alongside of 2305|154380|1926|5|26|37293.88|0.06|0.07|A|F|1993-05-14|1993-02-28|1993-06-04|NONE|SHIP|arefully final theodo 2305|50969|8485|6|7|13439.72|0.06|0.00|R|F|1993-05-15|1993-04-25|1993-06-09|DELIVER IN PERSON|RAIL|gular deposits boost about the foxe 2306|195761|800|1|50|92838.00|0.09|0.01|N|O|1995-07-27|1995-09-26|1995-08-06|DELIVER IN PERSON|FOB|y quickly 2306|148556|3585|2|39|62577.45|0.04|0.00|N|O|1995-09-07|1995-09-13|1995-10-03|COLLECT COD|SHIP|f the slyly unusual accounts. furiousl 2306|177684|5236|3|35|61658.80|0.01|0.07|N|O|1995-08-18|1995-08-30|1995-08-20|TAKE BACK RETURN|RAIL|raids along the furiously unusual asympto 2306|118936|1448|4|21|41053.53|0.06|0.01|N|O|1995-10-07|1995-09-18|1995-10-17|COLLECT COD|MAIL| ironic pinto 2306|141746|4261|5|42|75085.08|0.04|0.07|N|O|1995-09-05|1995-08-25|1995-09-28|COLLECT COD|MAIL|furiously final acco 2306|123507|3508|6|29|44384.50|0.00|0.03|N|O|1995-11-01|1995-09-01|1995-11-22|DELIVER IN PERSON|REG AIR|uld have to mold. s 2306|175400|2952|7|19|28032.60|0.07|0.01|N|O|1995-11-17|1995-09-06|1995-11-30|DELIVER IN PERSON|AIR|tainments nag furiously carefull 2307|141367|3882|1|24|33800.64|0.10|0.05|R|F|1993-10-07|1993-08-05|1993-10-20|COLLECT COD|AIR|stealthily special packages nag a 2307|139516|2030|2|2|3111.02|0.01|0.00|A|F|1993-09-21|1993-08-22|1993-10-03|COLLECT COD|SHIP|ously. furiously furious requ 2307|33306|5810|3|7|8675.10|0.07|0.04|R|F|1993-08-03|1993-09-04|1993-08-28|DELIVER IN PERSON|AIR|ven instructions wake fluffily 2307|164357|4358|4|19|27005.65|0.08|0.06|R|F|1993-10-23|1993-09-09|1993-11-09|TAKE BACK RETURN|TRUCK|olites haggle furiously around the 2307|142685|2686|5|7|12093.76|0.01|0.06|R|F|1993-09-01|1993-08-08|1993-09-29|NONE|AIR| packages cajo 2308|117514|7515|1|24|36756.24|0.06|0.04|R|F|1993-02-23|1992-12-24|1993-03-10|NONE|MAIL|ts sleep. busy excuses along the s 2308|55630|641|2|36|57082.68|0.05|0.06|A|F|1992-11-11|1992-11-27|1992-11-23|NONE|MAIL|ong the pending hockey players. blithe 2309|169856|4889|1|14|26961.90|0.10|0.03|N|O|1996-01-01|1995-10-22|1996-01-23|NONE|AIR|asymptotes. furiously pending acco 2309|168253|770|2|1|1321.25|0.01|0.05|N|O|1995-12-08|1995-11-03|1995-12-31|COLLECT COD|RAIL|eposits alongside of the final re 2309|14764|9767|3|5|8393.80|0.01|0.00|N|O|1995-12-10|1995-10-29|1996-01-06|TAKE BACK RETURN|SHIP|s. requests wake blithely specia 2309|138046|3073|4|46|49865.84|0.08|0.04|N|O|1995-10-02|1995-10-30|1995-10-30|NONE|REG AIR|sly according to the carefully 2309|136205|3745|5|9|11170.80|0.00|0.07|N|O|1995-12-21|1995-10-10|1996-01-20|COLLECT COD|AIR|ding, unusual instructions. dep 2309|194880|9919|6|21|41472.48|0.09|0.00|N|O|1995-11-05|1995-11-07|1995-11-22|NONE|AIR|unts around the dolphins ar 2309|137744|258|7|48|85523.52|0.03|0.05|N|O|1995-10-21|1995-11-21|1995-11-09|NONE|MAIL|ccounts. id 2310|57879|385|1|36|66127.32|0.03|0.03|N|O|1996-10-09|1996-10-28|1996-10-29|TAKE BACK RETURN|RAIL|iously against the slyly special accounts 2310|170444|445|2|6|9086.64|0.07|0.01|N|O|1996-11-08|1996-12-09|1996-12-07|COLLECT COD|REG AIR|e slyly about the quickly ironic theodo 2310|41031|8544|3|48|46657.44|0.08|0.02|N|O|1996-10-04|1996-11-20|1996-10-25|TAKE BACK RETURN|FOB|ep slyly alongside of the 2311|140068|2583|1|18|19945.08|0.01|0.01|N|F|1995-06-11|1995-06-18|1995-07-02|NONE|FOB| fluffily even patterns haggle blithely. re 2311|121572|6597|2|49|78084.93|0.09|0.02|R|F|1995-05-14|1995-07-11|1995-05-20|COLLECT COD|FOB|ideas sleep 2311|53793|3794|3|15|26201.85|0.08|0.04|N|O|1995-06-23|1995-06-06|1995-07-09|COLLECT COD|AIR|ve the blithely pending accounts. furio 2311|89740|4757|4|42|72649.08|0.01|0.06|R|F|1995-06-03|1995-06-27|1995-06-11|DELIVER IN PERSON|MAIL|gle furiously. bold 2311|46446|1455|5|1|1392.44|0.05|0.02|A|F|1995-06-07|1995-06-20|1995-06-10|NONE|AIR|ptotes. furiously regular theodolite 2311|11993|6996|6|32|60959.68|0.01|0.03|N|O|1995-07-19|1995-06-26|1995-07-26|NONE|RAIL|sts along the slyly 2336|192161|4681|1|20|25063.20|0.01|0.03|N|O|1996-03-12|1996-02-25|1996-03-18|NONE|REG AIR|across the fi 2337|44792|7297|1|49|85102.71|0.06|0.05|N|O|1997-08-08|1997-08-15|1997-08-31|TAKE BACK RETURN|FOB| along the packages. furiously p 2338|51054|6065|1|30|30151.50|0.07|0.06|N|O|1997-12-10|1997-10-15|1997-12-11|TAKE BACK RETURN|REG AIR|ould have to nag quickly 2339|191542|1543|1|22|35937.88|0.03|0.03|A|F|1994-01-06|1994-03-06|1994-01-10|NONE|FOB| furiously above 2339|29709|2212|2|28|45883.60|0.00|0.00|R|F|1994-01-25|1994-01-22|1994-01-28|DELIVER IN PERSON|RAIL|e bold, even packag 2339|116749|1772|3|13|22954.62|0.06|0.08|R|F|1994-03-10|1994-02-18|1994-03-20|TAKE BACK RETURN|REG AIR|ges. blithely special depend 2340|137788|5328|1|9|16432.02|0.08|0.02|N|O|1996-05-01|1996-02-24|1996-05-16|COLLECT COD|RAIL|. carefully ironic 2340|192904|5424|2|21|41934.90|0.06|0.02|N|O|1996-01-17|1996-03-04|1996-01-29|DELIVER IN PERSON|SHIP| asymptotes. unusual theo 2341|46009|1018|1|12|11460.00|0.08|0.03|R|F|1993-06-06|1993-07-08|1993-06-17|DELIVER IN PERSON|FOB|. quickly final deposits sl 2341|70364|5379|2|37|49371.32|0.07|0.08|A|F|1993-09-23|1993-07-25|1993-10-14|DELIVER IN PERSON|RAIL|was blithel 2341|194541|9580|3|8|13084.32|0.03|0.07|R|F|1993-06-08|1993-07-09|1993-06-10|COLLECT COD|FOB|ns affix above the iron 2342|41169|8682|1|12|13321.92|0.00|0.08|N|O|1996-07-31|1996-07-26|1996-08-14|NONE|TRUCK|print blithely even deposits. carefull 2342|116462|8974|2|24|35483.04|0.10|0.06|N|O|1996-09-30|1996-07-22|1996-10-28|TAKE BACK RETURN|AIR|nstructions c 2342|169920|9921|3|50|99496.00|0.10|0.01|N|O|1996-08-28|1996-07-18|1996-09-22|COLLECT COD|RAIL|cial asymptotes pr 2342|35809|5810|4|1|1744.80|0.04|0.06|N|O|1996-08-31|1996-08-09|1996-09-07|DELIVER IN PERSON|REG AIR|ffily. unusual pinto beans wake c 2342|26756|9259|5|22|37020.50|0.08|0.01|N|O|1996-08-10|1996-08-02|1996-08-31|DELIVER IN PERSON|AIR|s. ironic 2343|109899|9900|1|27|51540.03|0.00|0.00|N|O|1995-11-10|1995-11-17|1995-12-10|TAKE BACK RETURN|SHIP|old theodolites. 2343|65555|3074|2|35|53219.25|0.03|0.06|N|O|1995-10-24|1995-11-09|1995-10-26|COLLECT COD|TRUCK|ges haggle furiously carefully regular req 2343|178754|6306|3|21|38487.75|0.00|0.03|N|O|1995-09-07|1995-10-26|1995-10-07|TAKE BACK RETURN|RAIL|osits. unusual theodolites boost furio 2368|151986|1987|1|16|32607.68|0.04|0.03|R|F|1993-10-31|1993-10-22|1993-11-06|NONE|REG AIR|telets wake carefully iro 2368|13052|3053|2|32|30881.60|0.03|0.00|R|F|1993-09-23|1993-10-07|1993-09-27|COLLECT COD|TRUCK|gular courts use blithely around the 2368|148805|1320|3|39|72298.20|0.08|0.03|R|F|1993-09-03|1993-09-20|1993-09-28|COLLECT COD|RAIL|ng the doggedly ironic requests are blithe 2368|155900|3446|4|17|33250.30|0.10|0.08|A|F|1993-10-03|1993-09-27|1993-10-05|NONE|FOB|fily. slyly final ideas alongside o 2369|23834|1341|1|30|52734.90|0.05|0.04|N|O|1997-04-23|1997-02-12|1997-05-21|COLLECT COD|REG AIR|pecial deposits sleep. blithely unusual w 2369|168416|8417|2|47|69767.27|0.10|0.02|N|O|1997-01-02|1997-02-18|1997-01-13|COLLECT COD|RAIL| to the regular dep 2370|45041|7546|1|3|2958.12|0.03|0.07|R|F|1994-03-24|1994-03-26|1994-04-15|COLLECT COD|SHIP|ly regular Tiresia 2370|1051|3552|2|24|22849.20|0.00|0.05|A|F|1994-05-15|1994-04-09|1994-06-12|NONE|REG AIR|final depen 2370|60844|845|3|32|57754.88|0.05|0.02|A|F|1994-04-24|1994-03-03|1994-05-15|DELIVER IN PERSON|MAIL|ies since the final deposits 2370|5044|2545|4|21|19929.84|0.04|0.01|R|F|1994-02-01|1994-02-19|1994-02-09|TAKE BACK RETURN|MAIL|ecial dependencies must have to 2371|158603|3634|1|37|61479.20|0.05|0.05|N|O|1998-02-11|1998-03-24|1998-02-27|DELIVER IN PERSON|TRUCK|s boost fluffil 2371|34147|6651|2|21|22703.94|0.00|0.05|N|O|1998-04-14|1998-02-14|1998-04-18|COLLECT COD|AIR|gle furiously regu 2371|100521|3032|3|11|16736.72|0.05|0.02|N|O|1998-02-25|1998-04-06|1998-03-23|TAKE BACK RETURN|TRUCK|requests. regular pinto beans wake. car 2371|42076|7085|4|33|33596.31|0.05|0.08|N|O|1998-03-30|1998-02-06|1998-04-05|DELIVER IN PERSON|AIR|deas are. express r 2371|164508|9541|5|22|34595.00|0.02|0.05|N|O|1998-03-26|1998-03-19|1998-04-16|DELIVER IN PERSON|REG AIR|y daring accounts. regular ins 2371|85027|5028|6|39|39468.78|0.05|0.03|N|O|1998-04-01|1998-03-13|1998-04-27|NONE|REG AIR|tructions. regular, stealthy packages wak 2371|35714|8218|7|32|52790.72|0.07|0.07|N|O|1998-02-15|1998-04-03|1998-02-23|NONE|REG AIR|the ruthless accounts. 2372|42661|2662|1|42|67353.72|0.08|0.02|N|O|1998-01-04|1998-01-02|1998-02-02|COLLECT COD|REG AIR|lar packages. regular 2372|2319|9820|2|17|20762.27|0.07|0.01|N|O|1997-12-17|1998-01-17|1997-12-25|NONE|RAIL|xcuses. slyly ironic theod 2372|163333|8366|3|12|16755.96|0.04|0.04|N|O|1998-03-21|1997-12-21|1998-04-12|DELIVER IN PERSON|SHIP|lyly according to 2372|121677|6702|4|4|6794.68|0.00|0.07|N|O|1997-12-14|1997-12-28|1997-12-16|TAKE BACK RETURN|REG AIR|e carefully blithely even epitaphs. r 2372|19420|4423|5|5|6697.10|0.02|0.04|N|O|1998-02-08|1998-01-18|1998-03-02|TAKE BACK RETURN|RAIL|ets against the 2372|188437|5992|6|11|16779.73|0.02|0.06|N|O|1998-02-14|1998-01-18|1998-03-10|TAKE BACK RETURN|FOB| silent, pending de 2372|56012|6013|7|19|18392.19|0.01|0.06|N|O|1997-12-26|1998-02-19|1998-01-02|COLLECT COD|SHIP| beans haggle sometimes 2373|190025|7583|1|17|18955.34|0.02|0.01|R|F|1994-03-29|1994-05-19|1994-04-20|COLLECT COD|AIR|auternes. blithely even pinto bea 2373|135214|2754|2|3|3747.63|0.08|0.08|R|F|1994-05-15|1994-06-10|1994-06-04|COLLECT COD|TRUCK|dependencies wake ironical 2373|140804|3319|3|29|53499.20|0.05|0.02|A|F|1994-06-01|1994-05-14|1994-06-17|NONE|TRUCK|yly silent ideas affix furiousl 2373|90692|8220|4|5|8413.45|0.10|0.01|R|F|1994-06-02|1994-05-03|1994-06-21|NONE|REG AIR|uffily blithely ironic requests 2374|117116|9628|1|41|46457.51|0.07|0.00|A|F|1994-01-27|1993-12-11|1994-02-12|TAKE BACK RETURN|RAIL|heodolites. requests 2374|159783|7329|2|24|44226.72|0.07|0.08|A|F|1994-02-02|1994-01-12|1994-02-04|DELIVER IN PERSON|TRUCK|. requests are above t 2374|60682|5695|3|2|3285.36|0.06|0.02|R|F|1993-12-30|1994-01-24|1994-01-02|COLLECT COD|FOB|, unusual ideas. deposits cajole quietl 2374|73993|3994|4|28|55075.72|0.04|0.08|R|F|1994-02-19|1993-12-16|1994-03-15|COLLECT COD|MAIL|ets cajole fu 2374|309|310|5|25|30232.50|0.08|0.00|A|F|1993-11-26|1993-12-15|1993-12-10|COLLECT COD|RAIL|refully pending d 2375|167272|7273|1|3|4017.81|0.02|0.08|N|O|1997-02-14|1996-12-25|1997-02-22|COLLECT COD|RAIL|slyly across the furiously e 2375|131282|3796|2|9|11819.52|0.09|0.02|N|O|1997-02-17|1996-12-27|1997-02-27|DELIVER IN PERSON|MAIL|ly against the packages. bold pinto bean 2375|46470|8975|3|26|36828.22|0.02|0.06|N|O|1997-03-18|1997-02-02|1997-03-29|TAKE BACK RETURN|TRUCK|rate across the 2375|4801|7302|4|5|8529.00|0.01|0.00|N|O|1997-01-31|1997-01-25|1997-02-22|COLLECT COD|REG AIR|final packages cajole according to the furi 2375|87210|4735|5|42|50282.82|0.01|0.08|N|O|1997-01-24|1997-02-15|1997-02-07|DELIVER IN PERSON|FOB|apades. idea 2375|125524|5525|6|20|30990.40|0.09|0.08|N|O|1996-12-01|1996-12-26|1996-12-19|TAKE BACK RETURN|SHIP|ckages! blithely enticing deposi 2400|102546|5057|1|48|74329.92|0.01|0.02|N|O|1998-10-07|1998-08-30|1998-11-03|DELIVER IN PERSON|MAIL|fore the car 2400|89966|2475|2|1|1955.96|0.04|0.07|N|O|1998-08-18|1998-09-12|1998-09-11|NONE|MAIL|silent deposits serve furious 2400|52503|19|3|23|33476.50|0.02|0.08|N|O|1998-08-05|1998-08-28|1998-08-30|NONE|SHIP|tions. fluffily ironic platelets cajole c 2400|16820|4324|4|23|39946.86|0.09|0.04|N|O|1998-10-04|1998-10-04|1998-10-31|NONE|RAIL|ages lose carefully around the regula 2401|181615|4134|1|39|66167.79|0.00|0.03|N|O|1997-09-29|1997-10-21|1997-10-17|DELIVER IN PERSON|FOB|ould affix 2401|2221|7222|2|49|55037.78|0.05|0.07|N|O|1997-09-02|1997-09-11|1997-09-13|TAKE BACK RETURN|AIR|lites cajole carefully 2402|85533|8042|1|43|65296.79|0.03|0.08|N|O|1996-09-17|1996-11-20|1996-09-22|DELIVER IN PERSON|RAIL|slyly slyly blithe sheaves 2402|151328|8874|2|24|33103.68|0.02|0.05|N|O|1996-11-21|1996-10-19|1996-11-29|DELIVER IN PERSON|SHIP|as; blithely ironic requ 2403|82868|393|1|34|62929.24|0.04|0.07|N|O|1998-05-30|1998-06-19|1998-06-05|NONE|REG AIR| slyly bold re 2403|151901|9447|2|19|37105.10|0.08|0.07|N|O|1998-04-20|1998-07-02|1998-05-13|DELIVER IN PERSON|FOB|sits. ironic in 2403|192434|2435|3|27|41213.61|0.05|0.03|N|O|1998-07-27|1998-07-08|1998-08-03|NONE|SHIP|deposits sleep slyly special theodolit 2403|30422|423|4|30|40572.60|0.05|0.06|N|O|1998-08-08|1998-06-17|1998-08-20|NONE|TRUCK|ackages sleep furiously pendin 2404|146703|1732|1|36|62989.20|0.07|0.00|N|O|1997-03-27|1997-05-16|1997-04-06|COLLECT COD|REG AIR|s nag furi 2404|35641|8145|2|1|1576.64|0.02|0.04|N|O|1997-05-22|1997-06-06|1997-05-28|DELIVER IN PERSON|MAIL|from the final orbits? even pinto beans hag 2404|17378|2381|3|41|53110.17|0.02|0.06|N|O|1997-06-12|1997-05-03|1997-07-12|NONE|AIR| dolphins are 2404|56668|6669|4|19|30868.54|0.09|0.03|N|O|1997-05-07|1997-05-24|1997-05-24|TAKE BACK RETURN|SHIP|cuses. quickly even in 2404|3058|8059|5|18|17298.90|0.00|0.04|N|O|1997-06-25|1997-05-06|1997-07-02|NONE|RAIL|packages. even requests according to 2405|88560|3577|1|18|27874.08|0.09|0.07|N|O|1997-01-23|1997-03-10|1997-02-03|COLLECT COD|REG AIR|carefully ironic accounts. slyly 2405|26169|3676|2|30|32854.80|0.10|0.08|N|O|1997-03-24|1997-03-10|1997-04-14|TAKE BACK RETURN|AIR|y final deposits are slyly caref 2405|16135|6136|3|49|51505.37|0.00|0.06|N|O|1996-12-24|1997-03-23|1997-01-01|TAKE BACK RETURN|FOB|cial requests. ironic, regu 2405|176791|9309|4|23|42959.17|0.08|0.05|N|O|1996-12-28|1997-01-29|1997-01-07|NONE|AIR|t wake blithely blithely regular idea 2406|169075|6624|1|18|20593.26|0.07|0.05|N|O|1997-02-17|1996-12-25|1997-02-19|COLLECT COD|MAIL|azzle furiously careful 2406|40475|2980|2|40|56618.80|0.02|0.07|N|O|1997-01-09|1996-12-02|1997-01-16|NONE|SHIP|gular accounts caj 2406|49072|9073|3|16|16337.12|0.07|0.03|N|O|1996-10-31|1996-11-28|1996-11-08|TAKE BACK RETURN|SHIP| special accou 2406|145035|64|4|34|36721.02|0.07|0.06|N|O|1996-12-01|1996-12-07|1996-12-16|NONE|AIR|hinly even accounts are slyly q 2406|186950|9469|5|25|50923.75|0.08|0.02|N|O|1996-12-03|1996-12-14|1996-12-26|COLLECT COD|MAIL|al, regular in 2406|58277|3288|6|22|27175.94|0.05|0.02|N|O|1996-11-22|1997-01-17|1996-12-15|NONE|TRUCK|hely even foxes unwind furiously aga 2406|59491|7007|7|30|43514.70|0.07|0.07|N|O|1997-01-17|1997-01-12|1997-01-22|TAKE BACK RETURN|TRUCK| final pinto beans han 2407|63032|5539|1|14|13930.42|0.04|0.02|N|O|1998-10-10|1998-08-25|1998-10-27|NONE|FOB|l dependencies s 2407|165166|5167|2|9|11080.44|0.07|0.05|N|O|1998-08-06|1998-08-11|1998-08-20|TAKE BACK RETURN|TRUCK|ts. special deposits are closely. 2407|130261|262|3|39|50359.14|0.02|0.02|N|O|1998-08-20|1998-09-12|1998-08-22|DELIVER IN PERSON|MAIL|iously final deposits solv 2407|90439|5458|4|10|14294.30|0.01|0.07|N|O|1998-08-14|1998-09-10|1998-08-29|COLLECT COD|FOB| pending instructions. theodolites x- 2407|197553|2592|5|14|23107.70|0.04|0.05|N|O|1998-09-24|1998-08-18|1998-10-06|DELIVER IN PERSON|FOB|tructions wake stealt 2407|70290|7812|6|18|22685.22|0.04|0.01|N|O|1998-10-03|1998-08-30|1998-10-19|TAKE BACK RETURN|MAIL| wake carefully. fluffily 2407|160877|5910|7|7|13565.09|0.07|0.03|N|O|1998-09-11|1998-08-15|1998-09-30|TAKE BACK RETURN|MAIL|totes are carefully accordin 2432|49371|4380|1|30|39611.10|0.03|0.02|N|O|1996-09-05|1996-10-10|1996-10-05|TAKE BACK RETURN|TRUCK| requests wake alongside of 2432|161797|1798|2|8|14870.32|0.07|0.01|N|O|1996-10-16|1996-10-01|1996-11-13|COLLECT COD|RAIL|s about the bold, close deposit 2432|108837|1348|3|13|23995.79|0.07|0.06|N|O|1996-09-03|1996-10-10|1996-10-03|NONE|RAIL|arefully about the caref 2432|12405|2406|4|14|18443.60|0.00|0.06|N|O|1996-08-18|1996-09-04|1996-08-27|TAKE BACK RETURN|RAIL|riously regular packages. p 2433|86794|6795|1|39|69450.81|0.01|0.04|R|F|1994-11-20|1994-09-23|1994-12-10|DELIVER IN PERSON|SHIP|ly final asy 2433|133323|8350|2|20|27126.40|0.05|0.06|A|F|1994-12-09|1994-10-20|1994-12-15|COLLECT COD|REG AIR|lithely blithely final ide 2433|156347|1378|3|38|53326.92|0.08|0.03|A|F|1994-10-15|1994-10-23|1994-11-06|DELIVER IN PERSON|SHIP|. slyly regular requests sle 2433|120254|2767|4|43|54792.75|0.01|0.05|A|F|1994-10-16|1994-10-23|1994-11-08|DELIVER IN PERSON|RAIL|ular requests. slyly even pa 2433|107164|9675|5|3|3513.48|0.06|0.02|A|F|1994-11-08|1994-09-24|1994-11-17|COLLECT COD|AIR|usly pending depos 2434|94433|4434|1|1|1427.43|0.01|0.06|N|O|1997-08-02|1997-05-28|1997-08-19|TAKE BACK RETURN|MAIL| furiously express packages. ironic, pend 2434|126777|4314|2|39|70347.03|0.09|0.05|N|O|1997-06-10|1997-06-08|1997-07-03|COLLECT COD|RAIL|r deposits sleep furiou 2434|129445|6982|3|28|41284.32|0.02|0.05|N|O|1997-06-28|1997-06-26|1997-07-15|COLLECT COD|RAIL|ven theodolites around the slyly 2434|167173|7174|4|49|60768.33|0.00|0.05|N|O|1997-08-08|1997-07-23|1997-08-27|DELIVER IN PERSON|FOB| after the requests haggle bold, fina 2435|38868|8869|1|8|14454.88|0.08|0.03|A|F|1993-06-08|1993-04-04|1993-06-29|COLLECT COD|SHIP|e fluffily quickly final accounts. care 2435|48350|3359|2|43|55829.05|0.03|0.08|A|F|1993-03-27|1993-05-20|1993-04-18|DELIVER IN PERSON|TRUCK|alongside of the s 2435|11457|6460|3|24|32842.80|0.07|0.08|R|F|1993-03-14|1993-05-20|1993-03-26|DELIVER IN PERSON|SHIP|s. carefully regular d 2435|155868|8384|4|22|42324.92|0.02|0.05|R|F|1993-05-23|1993-04-14|1993-06-04|NONE|SHIP|e final, final deposits. carefully regular 2435|71676|4184|5|3|4943.01|0.07|0.07|R|F|1993-06-01|1993-03-25|1993-06-27|DELIVER IN PERSON|FOB| final accounts ar 2435|45524|533|6|17|24981.84|0.02|0.02|A|F|1993-06-05|1993-05-05|1993-06-14|NONE|TRUCK|cajole aft 2435|120686|5711|7|8|13653.44|0.07|0.02|R|F|1993-05-03|1993-04-02|1993-05-17|COLLECT COD|SHIP|ng the fluffily special foxes nag 2436|154062|4063|1|48|53570.88|0.04|0.02|N|O|1995-10-22|1995-10-22|1995-11-16|DELIVER IN PERSON|FOB|he furiously 2436|116509|4043|2|18|27459.00|0.05|0.03|N|O|1995-10-14|1995-11-21|1995-11-12|TAKE BACK RETURN|TRUCK|y ironic accounts. furiously even packa 2436|163647|6164|3|6|10263.84|0.06|0.08|N|O|1995-10-25|1995-11-30|1995-11-24|DELIVER IN PERSON|RAIL|odolites. ep 2437|93699|6209|1|46|77863.74|0.07|0.04|A|F|1993-08-12|1993-06-16|1993-08-29|NONE|RAIL|e of the bold, dogged requests 2437|189398|1917|2|26|38672.14|0.00|0.04|A|F|1993-06-25|1993-05-22|1993-07-07|DELIVER IN PERSON|REG AIR|lyly regular accounts. 2437|1745|6746|3|23|37875.02|0.01|0.00|A|F|1993-08-15|1993-06-28|1993-08-23|TAKE BACK RETURN|SHIP|s deposits. pendi 2437|115739|8251|4|12|21056.76|0.03|0.08|A|F|1993-04-27|1993-07-01|1993-05-18|TAKE BACK RETURN|FOB|thely regular deposits. ironic fray 2437|16697|4201|5|29|46797.01|0.02|0.06|A|F|1993-05-12|1993-06-10|1993-05-25|NONE|FOB|ress dolphins. furiously fin 2437|18571|1073|6|10|14895.70|0.10|0.06|A|F|1993-05-20|1993-06-23|1993-05-22|TAKE BACK RETURN|MAIL|unts. even, ironic pl 2438|164492|9525|1|45|70042.05|0.01|0.00|A|F|1993-10-27|1993-09-24|1993-11-02|COLLECT COD|REG AIR|en theodolites w 2438|12785|2786|2|31|52631.18|0.08|0.01|R|F|1993-10-16|1993-08-31|1993-11-10|COLLECT COD|REG AIR|t. slyly ironic sh 2438|67446|9953|3|10|14134.40|0.10|0.00|R|F|1993-08-18|1993-08-28|1993-09-08|NONE|SHIP|engage car 2438|160408|5441|4|27|39646.80|0.01|0.02|R|F|1993-07-27|1993-10-01|1993-08-06|TAKE BACK RETURN|FOB|inal accounts. slyly final reques 2438|165750|783|5|28|50841.00|0.07|0.06|R|F|1993-11-05|1993-08-22|1993-11-22|TAKE BACK RETURN|TRUCK|ctions. bli 2438|148426|941|6|23|33911.66|0.09|0.02|R|F|1993-10-06|1993-08-17|1993-10-16|DELIVER IN PERSON|MAIL|ely; blithely special pinto beans breach 2438|182589|7626|7|46|76892.68|0.02|0.05|R|F|1993-10-27|1993-08-30|1993-11-14|COLLECT COD|SHIP| ironic requests cajole f 2439|163036|8069|1|2|2198.06|0.09|0.03|N|O|1997-04-14|1997-06-11|1997-05-09|COLLECT COD|MAIL|courts boos 2439|143488|3489|2|5|7657.40|0.07|0.01|N|O|1997-04-23|1997-04-26|1997-04-28|DELIVER IN PERSON|FOB|ites. furiously 2439|194676|7196|3|33|58432.11|0.08|0.05|N|O|1997-06-01|1997-05-15|1997-06-07|TAKE BACK RETURN|FOB|asymptotes wake packages-- furiously 2464|48367|5880|1|10|13153.60|0.05|0.03|N|O|1998-02-04|1997-12-29|1998-02-16|TAKE BACK RETURN|RAIL|slyly final pinto bean 2464|100451|5472|2|20|29029.00|0.01|0.07|N|O|1997-12-26|1998-01-02|1998-01-24|DELIVER IN PERSON|FOB|sts. slyly close ideas shall h 2465|67190|2203|1|27|31244.13|0.05|0.02|N|O|1995-09-05|1995-09-07|1995-09-17|DELIVER IN PERSON|FOB|posits boost carefully unusual instructio 2465|50636|8152|2|34|53945.42|0.02|0.05|N|O|1995-10-02|1995-08-04|1995-10-09|COLLECT COD|RAIL|posits wake. regular package 2465|31032|6039|3|8|7704.24|0.10|0.00|N|O|1995-10-16|1995-08-26|1995-11-07|TAKE BACK RETURN|FOB|s across the express deposits wak 2465|147716|5259|4|45|79366.95|0.03|0.01|N|O|1995-09-27|1995-08-25|1995-10-06|NONE|TRUCK|y silent foxes. final pinto beans above 2465|46762|9267|5|50|85438.00|0.01|0.04|N|O|1995-09-01|1995-09-06|1995-09-18|TAKE BACK RETURN|TRUCK|the pending th 2465|123555|3556|6|20|31571.00|0.03|0.03|N|O|1995-08-16|1995-08-13|1995-09-02|COLLECT COD|FOB|uriously? furiously ironic excu 2466|185322|2877|1|16|22517.12|0.00|0.02|R|F|1994-04-20|1994-04-20|1994-05-09|COLLECT COD|FOB|to beans sl 2466|104696|7207|2|10|17006.90|0.00|0.00|A|F|1994-05-08|1994-04-06|1994-06-05|DELIVER IN PERSON|AIR|sly regular deposits. regular, regula 2466|13198|8201|3|29|32224.51|0.10|0.07|A|F|1994-06-11|1994-04-27|1994-07-10|DELIVER IN PERSON|FOB|ckages. bold requests nag carefully. 2466|10394|5397|4|29|37827.31|0.04|0.04|A|F|1994-04-01|1994-04-20|1994-04-23|DELIVER IN PERSON|MAIL|es boost fluffily ab 2466|78716|8717|5|30|50841.30|0.02|0.01|A|F|1994-04-11|1994-05-02|1994-05-02|DELIVER IN PERSON|REG AIR|. fluffily even pinto beans are idly. f 2466|172628|7663|6|19|32311.78|0.10|0.07|R|F|1994-06-12|1994-04-18|1994-07-12|NONE|MAIL|ccounts cajole a 2466|154218|1764|7|35|44527.35|0.10|0.00|A|F|1994-06-01|1994-05-27|1994-06-21|COLLECT COD|AIR| packages detect carefully: ironically sl 2467|132883|423|1|7|13411.16|0.00|0.00|N|O|1995-07-28|1995-10-04|1995-08-27|NONE|REG AIR|gular packages cajole 2468|93522|8541|1|46|69713.92|0.00|0.04|N|O|1997-07-16|1997-08-09|1997-08-07|COLLECT COD|SHIP|unusual theodolites su 2468|20527|5532|2|43|62243.36|0.00|0.04|N|O|1997-08-17|1997-08-21|1997-08-30|DELIVER IN PERSON|FOB|uriously eve 2468|194115|4116|3|44|53200.84|0.00|0.03|N|O|1997-10-01|1997-08-02|1997-10-09|TAKE BACK RETURN|RAIL|egular, silent sheave 2468|81607|1608|4|5|7943.00|0.08|0.00|N|O|1997-06-28|1997-08-02|1997-07-22|NONE|MAIL| sleep fluffily acc 2468|158353|869|5|18|25404.30|0.07|0.00|N|O|1997-07-25|1997-08-26|1997-08-14|DELIVER IN PERSON|REG AIR|cies. fluffily r 2469|165688|3237|1|11|19290.48|0.00|0.04|N|O|1997-02-09|1997-01-26|1997-02-16|NONE|TRUCK|ies wake carefully b 2469|113359|8382|2|16|21957.60|0.07|0.06|N|O|1997-02-19|1997-02-04|1997-03-18|NONE|MAIL|ing asymptotes 2469|10263|2765|3|48|56316.48|0.05|0.06|N|O|1997-01-11|1997-01-03|1997-01-15|TAKE BACK RETURN|AIR|riously even theodolites u 2469|87876|5401|4|35|65235.45|0.06|0.06|N|O|1997-02-04|1997-02-02|1997-02-17|DELIVER IN PERSON|RAIL|ld packages haggle regular frets. fluffily 2469|120671|8208|5|30|50750.10|0.09|0.01|N|O|1996-12-21|1997-01-29|1997-01-02|COLLECT COD|SHIP| accounts. regular theodolites affix fu 2469|103013|3014|6|49|49784.49|0.02|0.02|N|O|1997-03-03|1996-12-26|1997-03-13|NONE|AIR| requests are car 2469|126264|3801|7|8|10322.08|0.02|0.00|N|O|1997-03-15|1997-01-20|1997-04-13|NONE|TRUCK|s. regular 2470|109485|4506|1|12|17933.76|0.06|0.06|N|O|1997-07-12|1997-05-24|1997-07-17|TAKE BACK RETURN|FOB|l accounts. deposits nag daringly. express, 2470|99927|7455|2|50|96346.00|0.03|0.03|N|O|1997-06-02|1997-06-01|1997-06-09|COLLECT COD|AIR| packages 2470|63936|6443|3|10|18999.30|0.05|0.08|N|O|1997-06-20|1997-06-19|1997-06-24|TAKE BACK RETURN|FOB| ironic requests a 2470|161046|1047|4|30|33211.20|0.04|0.08|N|O|1997-08-04|1997-07-13|1997-08-14|DELIVER IN PERSON|AIR|s across the furiously fina 2471|83845|3846|1|37|67667.08|0.05|0.01|N|O|1998-05-28|1998-04-17|1998-06-08|COLLECT COD|TRUCK|ounts mold blithely carefully express depo 2496|140193|2708|1|38|46861.22|0.02|0.07|R|F|1994-03-26|1994-04-06|1994-04-23|COLLECT COD|RAIL| bold accounts. furi 2496|22609|2610|2|39|59732.40|0.03|0.00|R|F|1994-03-23|1994-02-18|1994-04-10|TAKE BACK RETURN|FOB|arefully special dependencies abo 2496|188554|3591|3|36|59131.80|0.09|0.04|R|F|1994-03-27|1994-03-15|1994-04-17|TAKE BACK RETURN|SHIP|ully ironic f 2496|23236|5739|4|30|34776.90|0.04|0.01|A|F|1994-01-27|1994-03-11|1994-01-31|DELIVER IN PERSON|RAIL|ake. ironic foxes cajole quickly. fu 2497|11060|8564|1|34|33016.04|0.02|0.03|R|F|1992-09-02|1992-10-19|1992-09-12|COLLECT COD|AIR|ronic accounts. p 2497|76887|9395|2|15|27958.20|0.09|0.02|A|F|1992-12-23|1992-11-20|1993-01-18|DELIVER IN PERSON|SHIP|sly against the 2497|33578|8585|3|28|42323.96|0.02|0.08|A|F|1992-12-02|1992-11-21|1992-12-04|DELIVER IN PERSON|REG AIR|ouches. special, regular requests 2497|143832|3833|4|48|90039.84|0.06|0.05|A|F|1992-09-29|1992-11-13|1992-10-19|TAKE BACK RETURN|AIR| even, regular requests across 2497|174813|7331|5|28|52858.68|0.04|0.05|A|F|1992-11-10|1992-09-30|1992-11-18|DELIVER IN PERSON|MAIL|hely bold ideas. unusual instructions ac 2497|70598|599|6|19|29803.21|0.05|0.08|A|F|1992-11-10|1992-11-20|1992-12-05|TAKE BACK RETURN|TRUCK| instructions? carefully daring accounts 2498|142210|9753|1|48|60106.08|0.10|0.01|R|F|1993-11-25|1994-01-09|1993-12-24|DELIVER IN PERSON|RAIL|onic requests wake 2499|149310|4339|1|15|20389.65|0.04|0.06|N|O|1995-12-21|1995-12-06|1996-01-19|DELIVER IN PERSON|FOB| slyly across the slyly 2499|45085|7590|2|48|49443.84|0.09|0.03|N|O|1995-10-14|1995-12-12|1995-11-11|DELIVER IN PERSON|AIR|ronic ideas cajole quickly requests. caref 2499|132086|4600|3|31|34660.48|0.09|0.05|N|O|1995-12-09|1995-10-28|1996-01-05|COLLECT COD|AIR|to beans across the carefully ironic theodo 2499|158406|922|4|39|57111.60|0.06|0.02|N|O|1995-10-26|1995-10-27|1995-11-07|TAKE BACK RETURN|SHIP|otes sublat 2499|129132|4157|5|6|6966.78|0.02|0.01|N|O|1995-11-19|1995-12-14|1995-12-08|NONE|SHIP|cording to the 2499|118583|1095|6|12|19218.96|0.04|0.05|N|O|1995-11-18|1995-12-13|1995-11-23|COLLECT COD|REG AIR|le furiously along the r 2500|191268|1269|1|40|54370.40|0.00|0.02|A|F|1992-09-02|1992-09-30|1992-09-06|DELIVER IN PERSON|SHIP|efully unusual dolphins s 2500|36252|6253|2|34|40400.50|0.06|0.02|R|F|1992-10-03|1992-11-11|1992-10-29|DELIVER IN PERSON|TRUCK| stealthy a 2500|79210|1718|3|41|48757.61|0.02|0.00|R|F|1992-09-02|1992-11-11|1992-09-06|DELIVER IN PERSON|RAIL|s could have to integrate after the 2500|68182|689|4|17|19553.06|0.01|0.02|A|F|1992-09-30|1992-10-16|1992-10-05|DELIVER IN PERSON|REG AIR|encies-- ironic, even packages 2501|83413|938|1|4|5585.64|0.10|0.06|N|O|1997-07-17|1997-07-27|1997-07-22|COLLECT COD|RAIL|quests. furiously final 2501|105961|982|2|33|64909.68|0.01|0.04|N|O|1997-07-14|1997-08-09|1997-07-26|NONE|MAIL|leep furiously packages. even sauternes 2501|71879|4387|3|20|37017.40|0.10|0.06|N|O|1997-09-23|1997-07-01|1997-10-03|DELIVER IN PERSON|RAIL|equests. furiou 2501|57411|4927|4|26|35578.66|0.09|0.01|N|O|1997-07-15|1997-08-15|1997-07-28|DELIVER IN PERSON|SHIP|c accounts. express, iron 2502|162155|2156|1|33|40165.95|0.10|0.06|R|F|1993-08-12|1993-07-22|1993-09-04|COLLECT COD|REG AIR|have to print 2503|122548|7573|1|33|51827.82|0.06|0.01|R|F|1993-07-06|1993-08-14|1993-08-02|NONE|SHIP|nal courts integrate according to the 2503|64060|1579|2|28|28673.68|0.06|0.01|R|F|1993-08-08|1993-08-31|1993-08-10|NONE|SHIP|s wake quickly slyly 2503|45308|5309|3|50|62665.00|0.09|0.01|A|F|1993-09-22|1993-08-17|1993-09-29|DELIVER IN PERSON|TRUCK|s around the slyly 2503|90061|7589|4|27|28378.62|0.09|0.00|A|F|1993-07-12|1993-07-24|1993-07-22|DELIVER IN PERSON|TRUCK|lly even p 2503|47679|184|5|3|4880.01|0.04|0.02|A|F|1993-07-10|1993-09-17|1993-07-19|TAKE BACK RETURN|TRUCK|s cajole. slyly close courts nod f 2503|127722|2747|6|39|68239.08|0.05|0.05|R|F|1993-10-11|1993-09-09|1993-10-16|NONE|MAIL|d carefully fluffily 2503|18588|3591|7|17|25611.86|0.09|0.08|R|F|1993-09-04|1993-07-31|1993-09-23|DELIVER IN PERSON|SHIP|c accounts haggle blithel 2528|195|196|1|10|10951.90|0.02|0.03|R|F|1994-12-12|1994-12-29|1994-12-28|COLLECT COD|REG AIR|ely. fluffily even re 2528|73433|8448|2|13|18283.59|0.00|0.03|A|F|1994-11-27|1995-01-20|1994-12-03|TAKE BACK RETURN|REG AIR|ggle furiously. slyly final asympt 2528|174348|4349|3|35|49781.90|0.10|0.00|R|F|1994-12-19|1995-02-04|1995-01-15|NONE|MAIL|, even excuses. even, 2528|64125|6632|4|37|40297.44|0.00|0.01|A|F|1994-12-25|1995-02-02|1994-12-31|COLLECT COD|AIR|ng the pending excuses haggle after the bl 2529|130636|8176|1|4|6666.52|0.07|0.07|N|O|1996-10-19|1996-11-18|1996-10-24|DELIVER IN PERSON|SHIP|al dependencies haggle slyly alongsi 2530|20319|320|1|9|11153.79|0.09|0.03|R|F|1994-05-10|1994-04-30|1994-05-24|TAKE BACK RETURN|REG AIR|lyly ironic 2530|92397|9925|2|42|58354.38|0.04|0.08|R|F|1994-03-27|1994-05-20|1994-03-29|NONE|RAIL|ng platelets wake s 2530|107824|335|3|8|14654.56|0.10|0.08|A|F|1994-05-02|1994-05-08|1994-05-24|DELIVER IN PERSON|MAIL|ial asymptotes snooze slyly regular 2531|147722|5265|1|9|15927.48|0.03|0.07|N|O|1996-07-27|1996-07-03|1996-08-01|DELIVER IN PERSON|AIR|t the dogged, un 2531|157000|2031|2|3|3171.00|0.07|0.06|N|O|1996-07-20|1996-06-20|1996-08-10|NONE|MAIL|he quickly ev 2531|85917|934|3|20|38058.20|0.06|0.04|N|O|1996-07-18|1996-06-25|1996-07-29|TAKE BACK RETURN|TRUCK|into beans. furious 2531|190889|8447|4|36|71275.68|0.08|0.01|N|O|1996-06-11|1996-07-26|1996-06-27|NONE|MAIL|y ironic, bold packages. blithely e 2531|55891|8397|5|28|51712.92|0.03|0.07|N|O|1996-07-06|1996-07-31|1996-07-19|TAKE BACK RETURN|REG AIR|its. busily 2531|144342|1885|6|46|63771.64|0.10|0.08|N|O|1996-07-03|1996-06-27|1996-07-12|TAKE BACK RETURN|REG AIR|e final, bold pains. ir 2532|52631|2632|1|3|4750.89|0.06|0.07|N|O|1995-12-14|1995-11-28|1995-12-15|COLLECT COD|FOB|unusual sentiments. even pinto 2532|159799|7345|2|33|61340.07|0.06|0.05|N|O|1995-11-23|1996-01-04|1995-12-16|DELIVER IN PERSON|TRUCK|rve carefully slyly ironic accounts! fluf 2532|134223|1763|3|1|1257.22|0.00|0.06|N|O|1996-01-27|1995-11-23|1996-01-29|DELIVER IN PERSON|REG AIR|ely final ideas cajole despite the ca 2532|77534|42|4|50|75576.50|0.02|0.02|N|O|1995-11-13|1996-01-01|1995-11-26|NONE|TRUCK|yly after the fluffily regul 2532|113046|8069|5|9|9531.36|0.09|0.04|N|O|1995-11-30|1995-11-23|1995-12-12|DELIVER IN PERSON|TRUCK|cial ideas haggle slyly pending request 2532|149281|9282|6|20|26605.60|0.09|0.05|N|O|1995-12-02|1995-11-26|1995-12-08|TAKE BACK RETURN|AIR|er the slyly pending 2533|53585|8596|1|36|55388.88|0.06|0.04|N|O|1997-06-10|1997-04-28|1997-07-01|NONE|REG AIR|ss requests sleep neve 2533|197072|9592|2|5|5845.35|0.10|0.04|N|O|1997-05-26|1997-06-02|1997-06-24|NONE|FOB|ccounts. ironic, special accounts boo 2533|182962|517|3|37|75663.52|0.00|0.08|N|O|1997-05-10|1997-04-26|1997-05-28|COLLECT COD|SHIP| haggle carefully 2533|29341|1844|4|17|21595.78|0.06|0.02|N|O|1997-05-23|1997-05-10|1997-06-18|NONE|FOB|ackages. blith 2533|125554|8067|5|38|60022.90|0.09|0.00|N|O|1997-05-10|1997-06-02|1997-05-28|TAKE BACK RETURN|REG AIR|of the regular accounts. even packages caj 2533|183334|3335|6|20|28346.60|0.05|0.08|N|O|1997-07-04|1997-04-30|1997-07-05|COLLECT COD|FOB|thless excuses are b 2533|93767|8786|7|14|24650.64|0.06|0.04|N|O|1997-07-06|1997-05-08|1997-08-03|COLLECT COD|FOB|ut the pending, special depos 2534|138552|6092|1|29|46125.95|0.07|0.07|N|O|1996-08-09|1996-09-29|1996-08-11|COLLECT COD|TRUCK|ugouts haggle slyly. final 2534|26716|1721|2|49|80492.79|0.08|0.08|N|O|1996-09-01|1996-08-20|1996-09-06|NONE|SHIP|sometimes regular requests. blithely unus 2534|802|3303|3|50|85140.00|0.10|0.06|N|O|1996-09-25|1996-10-07|1996-10-09|TAKE BACK RETURN|AIR|ideas. deposits use. slyly regular pa 2534|74217|1739|4|43|51222.03|0.09|0.02|N|O|1996-10-25|1996-09-30|1996-11-05|TAKE BACK RETURN|REG AIR|ngly final depos 2534|164808|9841|5|14|26219.20|0.05|0.02|N|O|1996-08-12|1996-09-26|1996-08-28|COLLECT COD|MAIL|eposits doze quickly final 2534|115962|8474|6|12|23735.52|0.02|0.02|N|O|1996-07-29|1996-10-12|1996-08-14|TAKE BACK RETURN|AIR|sual depos 2534|172536|5054|7|17|27345.01|0.02|0.07|N|O|1996-07-22|1996-09-15|1996-08-03|NONE|SHIP|riously regular 2535|198027|3066|1|5|5625.10|0.06|0.01|A|F|1993-09-07|1993-07-25|1993-09-29|DELIVER IN PERSON|REG AIR|, unusual reque 2535|38012|516|2|12|11400.12|0.08|0.05|A|F|1993-07-17|1993-08-17|1993-07-31|TAKE BACK RETURN|FOB|uses sleep among the packages. excuses 2535|53012|3013|3|5|4825.05|0.09|0.06|R|F|1993-07-28|1993-08-14|1993-08-11|DELIVER IN PERSON|SHIP| across the express requests. silent, eve 2535|159290|4321|4|19|25636.51|0.01|0.02|A|F|1993-06-01|1993-08-01|1993-06-19|DELIVER IN PERSON|FOB|ructions. final requests 2535|173753|8788|5|25|45668.75|0.07|0.04|A|F|1993-07-19|1993-08-07|1993-07-27|NONE|REG AIR|ions believe ab 2560|168823|8824|1|41|77564.62|0.07|0.01|R|F|1992-10-23|1992-11-11|1992-11-22|NONE|SHIP| after the accounts. regular foxes are be 2560|3764|8765|2|27|45029.52|0.00|0.01|R|F|1992-12-03|1992-11-16|1992-12-30|NONE|MAIL| against the carefully 2560|45914|3427|3|31|57657.21|0.01|0.05|A|F|1992-11-14|1992-10-14|1992-12-11|DELIVER IN PERSON|AIR|to beans. blithely regular Tiresias int 2560|71128|6143|4|36|39568.32|0.01|0.02|A|F|1992-10-18|1992-10-30|1992-11-05|TAKE BACK RETURN|MAIL|accounts alongside of the excuses are 2560|41835|9348|5|9|15991.47|0.04|0.02|A|F|1992-10-23|1992-10-29|1992-11-02|COLLECT COD|REG AIR| deposits affix quickly. unusual, eve 2560|107511|7512|6|13|19740.63|0.03|0.06|A|F|1992-09-07|1992-10-21|1992-09-24|COLLECT COD|FOB|slyly final accoun 2561|24579|9584|1|32|48114.24|0.02|0.01|N|O|1998-01-05|1997-12-28|1998-01-26|DELIVER IN PERSON|REG AIR|bold packages wake slyly. slyly 2561|97066|2085|2|5|5315.30|0.07|0.04|N|O|1997-12-27|1998-01-23|1998-01-13|TAKE BACK RETURN|AIR|p ironic, regular pinto beans. 2561|172518|2519|3|47|74753.97|0.04|0.02|N|O|1997-11-19|1998-01-21|1997-12-03|DELIVER IN PERSON|REG AIR|larly pending t 2561|107536|7537|4|39|60197.67|0.08|0.06|N|O|1998-01-20|1997-12-16|1998-02-05|TAKE BACK RETURN|MAIL|equests are furiously against the 2561|149929|4958|5|2|3957.84|0.04|0.08|N|O|1998-03-14|1998-01-21|1998-03-27|DELIVER IN PERSON|TRUCK|s are. silently silent foxes sleep about 2561|50268|5279|6|14|17055.64|0.02|0.03|N|O|1998-03-07|1998-02-04|1998-03-21|COLLECT COD|RAIL|ep unusual, ironic accounts 2562|52202|9718|1|28|32317.60|0.04|0.03|R|F|1992-10-04|1992-09-24|1992-10-09|COLLECT COD|MAIL|ans haggle special, special packages. 2562|147205|7206|2|1|1252.20|0.01|0.06|R|F|1992-10-16|1992-09-18|1992-10-17|NONE|TRUCK| slyly final ideas haggle car 2562|65577|5578|3|25|38564.25|0.05|0.03|A|F|1992-11-23|1992-10-08|1992-12-19|DELIVER IN PERSON|REG AIR| accounts-- silent, unusual ideas a 2562|147127|2156|4|37|43442.44|0.08|0.03|R|F|1992-10-29|1992-10-06|1992-11-09|COLLECT COD|FOB|. slyly regular ideas according to the fl 2562|159068|1584|5|29|32684.74|0.05|0.08|A|F|1992-11-01|1992-09-29|1992-11-13|TAKE BACK RETURN|MAIL|eep against the furiously r 2562|49696|2201|6|17|27976.73|0.01|0.06|A|F|1992-10-15|1992-10-08|1992-10-26|DELIVER IN PERSON|TRUCK|lar pinto beans. blithely ev 2563|64353|6860|1|10|13173.50|0.07|0.04|A|F|1994-01-26|1993-12-19|1994-01-28|DELIVER IN PERSON|AIR|tealthily abo 2563|166759|1792|2|28|51121.00|0.04|0.03|R|F|1994-03-17|1994-02-04|1994-04-13|TAKE BACK RETURN|RAIL|hely regular depe 2563|118456|5990|3|39|57503.55|0.07|0.00|R|F|1994-02-10|1993-12-31|1994-02-19|COLLECT COD|FOB|lent requests should integrate; carefully e 2563|89071|6596|4|50|53003.50|0.01|0.01|A|F|1994-01-26|1994-01-03|1994-02-09|DELIVER IN PERSON|SHIP|ly regular, regular excuses. bold plate 2563|14990|4991|5|42|80009.58|0.06|0.08|R|F|1994-02-21|1994-02-14|1994-03-04|DELIVER IN PERSON|AIR|ymptotes nag furiously slyly even inst 2563|120830|831|6|5|9254.15|0.10|0.00|R|F|1993-12-27|1993-12-19|1994-01-02|DELIVER IN PERSON|REG AIR| the quickly final theodolite 2564|111220|1221|1|4|4924.88|0.02|0.00|R|F|1994-11-12|1994-10-29|1994-12-04|NONE|MAIL|y express requests sleep furi 2565|143630|3631|1|42|70292.46|0.04|0.08|N|O|1998-04-07|1998-04-02|1998-05-04|NONE|AIR|ngly silent 2565|188845|8846|2|26|50279.84|0.05|0.08|N|O|1998-05-07|1998-04-09|1998-05-15|DELIVER IN PERSON|TRUCK| pinto beans about the slyly regula 2565|114394|1928|3|34|47885.26|0.06|0.06|N|O|1998-03-19|1998-04-12|1998-04-17|DELIVER IN PERSON|SHIP|nstructions was carefu 2565|16224|3728|4|25|28505.50|0.10|0.08|N|O|1998-06-27|1998-05-20|1998-07-13|DELIVER IN PERSON|RAIL|, express accounts. final id 2565|75465|5466|5|26|37451.96|0.08|0.03|N|O|1998-03-05|1998-04-11|1998-03-11|TAKE BACK RETURN|AIR|ites wake. ironic acco 2565|140191|5220|6|48|59097.12|0.08|0.07|N|O|1998-06-18|1998-05-06|1998-07-13|DELIVER IN PERSON|TRUCK|r instructions sleep qui 2566|147224|9739|1|19|24153.18|0.06|0.07|R|F|1992-12-21|1992-11-24|1992-12-22|DELIVER IN PERSON|MAIL|ests. silent 2566|180474|8029|2|42|65287.74|0.08|0.02|R|F|1992-12-20|1992-12-22|1992-12-29|COLLECT COD|MAIL|ously ironic accounts 2566|22648|5151|3|18|28271.52|0.09|0.02|A|F|1992-11-16|1992-12-24|1992-12-16|COLLECT COD|FOB| braids according t 2566|41449|3954|4|3|4171.32|0.05|0.02|A|F|1992-11-04|1992-12-30|1992-12-04|TAKE BACK RETURN|FOB|ckages are ironic Tiresias. furious 2566|21214|1215|5|9|10216.89|0.04|0.03|R|F|1992-12-14|1992-12-28|1992-12-16|NONE|FOB|blithely bold accounts? quickl 2566|127422|9935|6|1|1449.42|0.07|0.03|A|F|1992-10-28|1992-11-20|1992-11-22|TAKE BACK RETURN|AIR|theodolites wake pending 2567|25711|3218|1|39|63831.69|0.03|0.04|N|O|1998-05-10|1998-05-10|1998-05-21|NONE|SHIP|ns. furiously final dependencies cajo 2567|111886|1887|2|50|94894.00|0.06|0.05|N|O|1998-05-05|1998-04-18|1998-05-09|DELIVER IN PERSON|TRUCK|. carefully pending foxes are furi 2567|51679|4185|3|6|9784.02|0.03|0.06|N|O|1998-04-21|1998-04-14|1998-05-11|NONE|RAIL|s cajole regular, final acco 2567|157535|51|4|50|79626.50|0.05|0.03|N|O|1998-03-27|1998-05-25|1998-04-23|DELIVER IN PERSON|FOB|pinto beans? r 2567|80318|7843|5|46|59722.26|0.07|0.02|N|O|1998-06-02|1998-04-30|1998-06-13|COLLECT COD|AIR|efully pending epitaphs. carefully reg 2567|99455|4474|6|32|46542.40|0.01|0.07|N|O|1998-05-24|1998-04-30|1998-06-14|NONE|RAIL| the even, iro 2567|134307|9334|7|43|57675.90|0.06|0.02|N|O|1998-05-11|1998-04-15|1998-05-29|NONE|RAIL|requests. final courts cajole 2592|89634|9635|1|7|11365.41|0.10|0.04|R|F|1993-03-13|1993-04-25|1993-04-01|NONE|REG AIR| carefully special theodolites integrate 2592|65700|3219|2|2|3331.40|0.10|0.00|A|F|1993-03-24|1993-04-05|1993-04-16|DELIVER IN PERSON|RAIL|side of the b 2593|104022|1553|1|37|37962.74|0.08|0.06|R|F|1993-12-14|1993-10-08|1994-01-04|NONE|SHIP|s wake bravel 2593|89185|6710|2|28|32877.04|0.08|0.03|A|F|1993-10-30|1993-10-18|1993-11-06|DELIVER IN PERSON|SHIP|y even escapades shall 2593|127040|9553|3|6|6402.24|0.04|0.05|A|F|1993-11-28|1993-10-04|1993-12-28|TAKE BACK RETURN|REG AIR|ular packages. re 2593|160005|2522|4|44|46860.00|0.02|0.08|A|F|1993-09-05|1993-10-23|1993-09-29|NONE|RAIL|ents impress furiously; unusual theodoli 2593|3498|3499|5|3|4204.47|0.03|0.00|A|F|1993-12-16|1993-11-01|1993-12-29|COLLECT COD|SHIP|the furiously 2593|174570|4571|6|1|1644.57|0.08|0.08|A|F|1993-11-23|1993-10-25|1993-12-04|DELIVER IN PERSON|RAIL| accounts wake slyly 2593|191389|6428|7|11|16284.18|0.00|0.07|R|F|1993-11-01|1993-11-19|1993-11-28|TAKE BACK RETURN|RAIL|express packages sleep bold re 2594|71490|1491|1|7|10230.43|0.06|0.02|R|F|1993-03-26|1993-03-05|1993-04-24|DELIVER IN PERSON|FOB|arls cajole 2594|123171|5684|2|13|15524.21|0.10|0.05|R|F|1993-02-06|1993-03-01|1993-02-23|TAKE BACK RETURN|TRUCK|fully special accounts use courts 2594|125492|8005|3|24|36419.76|0.03|0.00|A|F|1993-01-31|1993-03-10|1993-02-04|COLLECT COD|REG AIR|lar accounts sleep fur 2594|143957|8986|4|46|92043.70|0.00|0.08|R|F|1993-04-17|1993-03-06|1993-04-21|TAKE BACK RETURN|SHIP|beans. instructions across t 2595|60945|946|1|42|80049.48|0.08|0.02|N|O|1996-03-24|1996-01-28|1996-04-10|DELIVER IN PERSON|MAIL|ggle furiou 2595|87061|9570|2|30|31441.80|0.05|0.01|N|O|1996-03-05|1996-02-23|1996-03-19|NONE|AIR|ctions. regula 2595|23369|8374|3|19|24554.84|0.01|0.05|N|O|1995-12-23|1996-03-02|1996-01-17|COLLECT COD|MAIL|ns are neve 2595|158785|6331|4|29|53469.62|0.07|0.05|N|O|1996-01-01|1996-02-13|1996-01-18|TAKE BACK RETURN|RAIL|ronic accounts haggle carefully fin 2595|85077|94|5|30|31862.10|0.09|0.07|N|O|1996-03-16|1996-01-31|1996-04-05|TAKE BACK RETURN|FOB|. final orbits cajole 2595|81859|9384|6|31|57066.35|0.06|0.04|N|O|1996-02-07|1996-02-10|1996-03-05|DELIVER IN PERSON|AIR|tipliers w 2596|169486|7035|1|6|9332.88|0.05|0.01|N|O|1996-12-15|1996-11-02|1996-12-29|TAKE BACK RETURN|TRUCK|ily special re 2596|138885|3912|2|43|82726.84|0.07|0.03|N|O|1996-09-03|1996-10-26|1996-09-15|NONE|FOB|ial packages haggl 2596|38443|947|3|19|26247.36|0.10|0.00|N|O|1996-09-02|1996-11-03|1996-09-06|COLLECT COD|AIR|ias mold! sp 2596|104266|4267|4|10|12702.60|0.06|0.05|N|O|1996-08-25|1996-11-05|1996-09-13|DELIVER IN PERSON|REG AIR| instructions shall have 2597|83782|8799|1|24|42378.72|0.07|0.00|A|F|1993-05-15|1993-03-06|1993-05-25|TAKE BACK RETURN|FOB|pending packages. enticingly fi 2598|6983|4484|1|12|22679.76|0.00|0.01|N|O|1996-06-17|1996-04-12|1996-06-24|COLLECT COD|TRUCK|express packages nag sly 2598|147005|4548|2|40|42080.00|0.07|0.02|N|O|1996-05-11|1996-05-19|1996-06-08|TAKE BACK RETURN|AIR|the enticing 2598|103709|8730|3|4|6850.80|0.03|0.03|N|O|1996-05-23|1996-05-13|1996-05-25|COLLECT COD|AIR| across the furiously fi 2598|22081|7086|4|19|19058.52|0.02|0.00|N|O|1996-04-09|1996-05-30|1996-04-17|TAKE BACK RETURN|RAIL|nic packages. even accounts 2598|105924|3455|5|12|23159.04|0.01|0.08|N|O|1996-04-14|1996-04-24|1996-04-21|TAKE BACK RETURN|REG AIR|eposits cajol 2599|100735|3246|1|11|19093.03|0.08|0.08|N|O|1997-02-01|1996-12-14|1997-02-27|TAKE BACK RETURN|FOB| express accoun 2599|41370|6379|2|26|34095.62|0.03|0.04|N|O|1996-11-08|1996-12-21|1996-11-24|TAKE BACK RETURN|AIR|nag carefully 2599|98819|8820|3|29|52716.49|0.09|0.03|N|O|1997-01-10|1996-12-10|1997-02-02|COLLECT COD|RAIL|ly express dolphins. special, 2624|62426|7439|1|15|20826.30|0.03|0.07|N|O|1997-02-28|1997-02-19|1997-03-21|DELIVER IN PERSON|AIR|le. quickly pending requests 2624|188685|8686|2|12|21284.16|0.07|0.00|N|O|1997-02-24|1997-02-22|1997-02-27|DELIVER IN PERSON|SHIP|er the quickly unu 2625|19346|9347|1|42|53144.28|0.02|0.04|R|F|1992-10-18|1992-11-17|1992-10-23|DELIVER IN PERSON|AIR| even accounts haggle furiously 2626|21431|8938|1|45|60859.35|0.09|0.04|N|O|1995-11-22|1995-11-01|1995-11-23|NONE|AIR|deposits wake blithely according to 2626|174354|1906|2|2|2856.70|0.05|0.07|N|O|1995-10-19|1995-11-09|1995-10-24|TAKE BACK RETURN|FOB|uffy accounts haggle furiously above 2626|153317|5833|3|40|54812.40|0.05|0.07|N|O|1995-09-28|1995-12-03|1995-10-10|NONE|REG AIR|eans. ironic deposits haggle. depo 2627|130582|8122|1|28|45152.24|0.09|0.02|R|F|1992-05-14|1992-05-09|1992-05-31|COLLECT COD|SHIP|ggedly final excuses nag packages. f 2628|105867|8378|1|44|82405.84|0.07|0.03|R|F|1994-01-11|1994-01-14|1994-01-13|DELIVER IN PERSON|SHIP|lyly final, pending ide 2628|105425|7936|2|14|20025.88|0.01|0.03|A|F|1994-01-28|1993-11-30|1994-02-20|TAKE BACK RETURN|SHIP|g the furiously unusual pi 2628|63884|1403|3|42|77610.96|0.00|0.00|A|F|1993-11-20|1994-01-04|1993-12-19|DELIVER IN PERSON|TRUCK|ld notornis alongside 2628|94014|6524|4|23|23184.23|0.08|0.04|A|F|1993-10-27|1994-01-08|1993-11-12|DELIVER IN PERSON|TRUCK|usual packages sleep about the fina 2628|89996|2505|5|50|99299.50|0.07|0.01|A|F|1994-01-13|1993-12-11|1994-01-14|NONE|AIR|posits serve carefully toward 2629|117875|7876|1|6|11357.22|0.06|0.05|N|O|1998-06-10|1998-05-29|1998-06-13|DELIVER IN PERSON|SHIP|dolites hinder bli 2629|123076|613|2|31|34071.17|0.08|0.03|N|O|1998-05-24|1998-05-26|1998-06-10|COLLECT COD|AIR|ate blithely bold, regular deposits. bold 2629|127428|7429|3|29|42207.18|0.08|0.07|N|O|1998-07-09|1998-06-17|1998-07-12|TAKE BACK RETURN|AIR|eposits serve unusual, express i 2629|69303|6822|4|33|41985.90|0.06|0.03|N|O|1998-05-29|1998-05-14|1998-05-30|NONE|TRUCK|es. slowly express accounts are along the 2630|28480|3485|1|46|64790.08|0.05|0.03|R|F|1992-11-05|1992-12-17|1992-12-05|TAKE BACK RETURN|MAIL|uests cajole. e 2630|56285|1296|2|8|9930.24|0.09|0.07|A|F|1992-11-16|1993-01-01|1992-12-07|DELIVER IN PERSON|TRUCK|indle fluffily silent, ironic pi 2630|172605|7640|3|45|75492.00|0.08|0.07|A|F|1993-01-04|1993-01-11|1993-01-09|NONE|FOB|edly express ideas. carefully final 2630|161536|6569|4|29|46328.37|0.08|0.07|A|F|1992-12-03|1993-01-04|1992-12-12|DELIVER IN PERSON|SHIP|efully unusual dependencies. even i 2631|121656|4169|1|42|70461.30|0.00|0.03|A|F|1994-01-04|1993-12-01|1994-01-16|TAKE BACK RETURN|SHIP|ect carefully at the furiously final the 2631|66551|1564|2|4|6070.20|0.07|0.06|R|F|1993-11-03|1993-12-17|1993-11-05|COLLECT COD|AIR|special theodolites. a 2631|117876|5410|3|15|28408.05|0.06|0.05|A|F|1993-09-30|1993-11-06|1993-10-13|DELIVER IN PERSON|SHIP|y. furiously even pinto be 2656|180671|3190|1|10|17516.70|0.02|0.06|R|F|1993-06-28|1993-07-04|1993-07-12|TAKE BACK RETURN|TRUCK|s nag regularly about the deposits. slyly 2656|136884|6885|2|38|72993.44|0.07|0.02|A|F|1993-06-25|1993-06-04|1993-07-24|NONE|RAIL|structions wake along the furio 2656|1209|3710|3|19|21093.80|0.03|0.02|R|F|1993-08-03|1993-07-25|1993-08-20|TAKE BACK RETURN|MAIL|ts serve deposi 2656|109246|1757|4|40|50209.60|0.05|0.04|R|F|1993-06-09|1993-07-24|1993-06-21|DELIVER IN PERSON|RAIL|refully final pearls. final ideas wake. qu 2657|114332|6844|1|22|29619.26|0.02|0.03|N|O|1995-12-08|1995-12-28|1995-12-21|TAKE BACK RETURN|MAIL|r ideas. furiously special dolphins 2657|164877|9910|2|15|29128.05|0.08|0.05|N|O|1995-12-09|1995-12-16|1995-12-18|NONE|RAIL|ole carefully above the ironic ideas. b 2657|78233|741|3|25|30280.75|0.02|0.04|N|O|1995-10-21|1995-12-12|1995-11-09|COLLECT COD|FOB|lly pinto beans. final 2657|54839|2355|4|11|19732.13|0.04|0.08|N|O|1995-11-19|1995-12-11|1995-11-24|COLLECT COD|TRUCK|ckly enticing requests. fur 2657|77611|7612|5|42|66721.62|0.06|0.03|N|O|1996-01-23|1995-11-22|1996-01-25|COLLECT COD|RAIL|ckly slyly even accounts. platelets x-ray 2657|193153|8192|6|31|38630.65|0.01|0.03|N|O|1995-11-10|1995-11-27|1995-12-06|COLLECT COD|RAIL|re blithely 2658|131174|1175|1|41|49411.97|0.05|0.04|N|O|1995-11-07|1995-11-04|1995-12-04|NONE|MAIL|eposits. furiously final theodolite 2658|28573|1076|2|22|33034.54|0.08|0.05|N|O|1995-11-12|1995-11-18|1995-11-14|DELIVER IN PERSON|TRUCK|ts cajole. pending packages affix 2658|17832|2835|3|13|22747.79|0.07|0.06|N|O|1995-10-24|1995-12-12|1995-11-14|COLLECT COD|FOB|s kindle blithely regular accounts. 2658|91939|6958|4|22|42480.46|0.04|0.04|N|O|1995-12-02|1995-11-03|1995-12-26|DELIVER IN PERSON|SHIP| dependencies. blithely pending foxes abou 2658|6721|6722|5|45|73247.40|0.03|0.01|N|O|1995-11-02|1995-11-08|1995-11-29|DELIVER IN PERSON|MAIL|e special requests. quickly ex 2658|146785|9300|6|27|49458.06|0.05|0.07|N|O|1995-09-26|1995-12-08|1995-09-30|NONE|AIR|ecial packages use abov 2659|41783|9296|1|28|48293.84|0.08|0.05|A|F|1994-03-17|1994-01-24|1994-03-19|NONE|FOB|idle tithes 2659|42253|9766|2|21|25100.25|0.00|0.00|A|F|1993-12-23|1994-02-10|1994-01-17|DELIVER IN PERSON|RAIL|y beyond the furiously even co 2659|134547|2087|3|24|37956.96|0.04|0.03|R|F|1994-03-28|1994-02-20|1994-04-05|DELIVER IN PERSON|REG AIR| haggle carefully 2659|118095|3118|4|2|2226.18|0.00|0.08|R|F|1994-02-19|1994-03-12|1994-02-21|NONE|MAIL|sts above the fluffily express fo 2659|6161|3662|5|9|9604.44|0.08|0.03|A|F|1994-02-07|1994-03-17|1994-03-04|DELIVER IN PERSON|AIR|ly final packages sleep ac 2660|47703|5216|1|17|28061.90|0.00|0.05|N|O|1995-08-18|1995-09-13|1995-09-17|NONE|SHIP|al pinto beans wake after the furious 2661|177159|7160|1|31|38320.65|0.03|0.02|N|O|1997-04-07|1997-03-10|1997-04-23|TAKE BACK RETURN|AIR|e ironicall 2661|102402|7423|2|22|30896.80|0.08|0.02|N|O|1997-03-14|1997-03-17|1997-04-08|COLLECT COD|REG AIR| foxes affix quickly ironic request 2661|66183|8690|3|11|12640.98|0.00|0.08|N|O|1997-04-14|1997-02-11|1997-05-05|TAKE BACK RETURN|FOB|equests are a 2661|136127|1154|4|41|47687.92|0.06|0.02|N|O|1997-03-06|1997-03-27|1997-03-15|DELIVER IN PERSON|AIR|iously ironically ironic requests. 2662|101609|4120|1|43|69255.80|0.09|0.07|N|O|1996-11-24|1996-11-04|1996-12-08|NONE|RAIL|. slyly specia 2662|127424|7425|2|8|11611.36|0.02|0.07|N|O|1996-09-10|1996-10-09|1996-09-21|TAKE BACK RETURN|REG AIR|ajole carefully. sp 2662|1049|3550|3|6|5700.24|0.02|0.00|N|O|1996-11-30|1996-09-20|1996-12-03|DELIVER IN PERSON|REG AIR|olites cajole quickly along the b 2662|29760|9761|4|34|57451.84|0.06|0.07|N|O|1996-10-04|1996-11-05|1996-10-19|NONE|SHIP|ding theodolites use carefully. p 2663|113221|755|1|35|43197.70|0.02|0.01|N|O|1995-12-11|1995-10-16|1996-01-07|TAKE BACK RETURN|REG AIR|tect. slyly fina 2688|17768|2771|1|45|75859.20|0.08|0.08|R|F|1992-05-21|1992-04-14|1992-05-28|NONE|FOB|sits run carefully 2688|14500|4501|2|46|65067.00|0.01|0.01|R|F|1992-05-24|1992-04-01|1992-05-26|COLLECT COD|TRUCK|elets. regular reque 2688|88810|1319|3|30|53964.30|0.05|0.04|A|F|1992-04-18|1992-03-18|1992-05-18|TAKE BACK RETURN|RAIL|ithely final 2688|24410|6913|4|3|4003.23|0.00|0.03|R|F|1992-02-04|1992-03-18|1992-02-24|DELIVER IN PERSON|RAIL|e fluffily 2688|58449|8450|5|22|30963.68|0.02|0.05|R|F|1992-02-09|1992-04-09|1992-02-11|DELIVER IN PERSON|RAIL|press, ironic excuses wake carefully id 2688|148297|8298|6|42|56502.18|0.01|0.01|R|F|1992-04-29|1992-04-04|1992-05-17|TAKE BACK RETURN|FOB|lly even account 2689|5731|732|1|45|73652.85|0.02|0.04|R|F|1992-04-29|1992-06-22|1992-04-30|COLLECT COD|SHIP|e quickly. carefully silent 2690|139065|4092|1|44|48578.64|0.05|0.06|N|O|1996-05-30|1996-05-19|1996-06-26|NONE|REG AIR|ly alongside of th 2690|50963|964|2|50|95698.00|0.03|0.03|N|O|1996-06-13|1996-05-22|1996-06-14|DELIVER IN PERSON|MAIL| doubt careful 2690|124448|4449|3|45|66259.80|0.02|0.07|N|O|1996-05-23|1996-06-02|1996-05-29|DELIVER IN PERSON|MAIL|ounts. slyly regular dependencies wa 2690|194436|4437|4|12|18365.16|0.04|0.07|N|O|1996-07-18|1996-06-03|1996-07-25|NONE|AIR|nal, regular atta 2690|85190|2715|5|30|35255.70|0.01|0.08|N|O|1996-05-20|1996-06-01|1996-06-04|TAKE BACK RETURN|SHIP|d accounts above the express req 2690|188034|5589|6|3|3366.09|0.07|0.01|N|O|1996-07-04|1996-05-28|1996-07-06|TAKE BACK RETURN|RAIL|. final reques 2690|78616|6138|7|35|55811.35|0.05|0.06|N|O|1996-07-25|1996-05-14|1996-08-03|COLLECT COD|FOB|y silent pinto be 2691|90549|3059|1|11|16934.94|0.04|0.07|R|F|1992-06-21|1992-06-08|1992-07-09|COLLECT COD|FOB|leep alongside of the accounts. slyly ironi 2691|47716|5229|2|2|3327.42|0.00|0.07|R|F|1992-05-10|1992-06-04|1992-05-11|TAKE BACK RETURN|TRUCK|s cajole at the blithely ironic warthog 2691|161253|1254|3|16|21028.00|0.09|0.03|R|F|1992-06-11|1992-07-29|1992-06-29|NONE|RAIL|bove the even foxes. unusual theodoli 2691|165770|803|4|1|1835.77|0.08|0.00|A|F|1992-08-11|1992-06-07|1992-08-16|NONE|SHIP|egular instructions b 2692|16131|8633|1|3|3141.39|0.10|0.04|N|O|1998-02-25|1998-01-29|1998-03-27|TAKE BACK RETURN|MAIL|equests. bold, even foxes haggle slyl 2692|113501|8524|2|21|31804.50|0.03|0.05|N|O|1998-03-11|1998-02-11|1998-03-19|NONE|SHIP|posits. final, express requests nag furi 2693|8670|8671|1|26|41045.42|0.04|0.00|N|O|1996-09-14|1996-10-07|1996-10-03|COLLECT COD|MAIL|cajole alo 2693|101842|1843|2|43|79285.12|0.03|0.04|N|O|1996-10-24|1996-10-24|1996-11-03|TAKE BACK RETURN|TRUCK|as are according to th 2694|152518|5034|1|30|47115.30|0.02|0.06|N|O|1996-06-20|1996-06-01|1996-07-15|NONE|TRUCK|oxes. never iro 2694|156889|1920|2|35|68105.80|0.07|0.03|N|O|1996-05-24|1996-06-01|1996-05-25|NONE|RAIL|atelets past the furiously final deposits 2694|18030|532|3|15|14220.45|0.08|0.02|N|O|1996-06-30|1996-05-01|1996-07-25|TAKE BACK RETURN|REG AIR|e blithely even platelets. special wa 2694|19451|6955|4|12|16445.40|0.00|0.05|N|O|1996-04-24|1996-04-22|1996-05-14|DELIVER IN PERSON|RAIL|foxes atop the hockey pla 2694|107185|7186|5|10|11921.80|0.08|0.08|N|O|1996-06-23|1996-05-28|1996-06-27|COLLECT COD|REG AIR|fluffily fluffy accounts. even packages hi 2695|183259|8296|1|21|28187.25|0.07|0.00|N|O|1996-10-04|1996-11-02|1996-10-21|NONE|MAIL|y regular pinto beans. evenly regular packa 2695|18835|6339|2|44|77168.52|0.09|0.07|N|O|1996-10-05|1996-10-10|1996-11-01|NONE|MAIL|ts. busy platelets boost 2695|143480|8509|3|21|31993.08|0.02|0.07|N|O|1996-09-13|1996-09-25|1996-10-13|NONE|TRUCK|s. furiously ironic platelets ar 2695|57398|9904|4|16|21686.24|0.08|0.08|N|O|1996-11-16|1996-10-05|1996-11-22|NONE|TRUCK|its. theodolites sleep slyly 2695|85282|2807|5|40|50691.20|0.02|0.03|N|O|1996-11-02|1996-10-26|1996-11-14|NONE|FOB|ructions. pending 2720|44483|4484|1|5|7137.40|0.10|0.06|A|F|1993-06-24|1993-08-08|1993-07-08|NONE|FOB|ously ironic foxes thrash 2720|16655|6656|2|42|66009.30|0.09|0.03|R|F|1993-07-25|1993-07-23|1993-08-23|COLLECT COD|REG AIR|fter the inst 2720|119065|9066|3|50|54203.00|0.10|0.02|A|F|1993-08-10|1993-07-29|1993-09-06|NONE|SHIP|l requests. deposits nag furiously 2720|108938|1449|4|49|95399.57|0.06|0.02|A|F|1993-07-09|1993-07-14|1993-07-13|NONE|REG AIR| accounts. fluffily bold pack 2720|120884|3397|5|27|51431.76|0.04|0.00|R|F|1993-06-29|1993-08-06|1993-07-28|NONE|TRUCK|eas. carefully regular 2721|182460|4979|1|49|75580.54|0.00|0.08|N|O|1996-02-14|1996-04-26|1996-03-02|DELIVER IN PERSON|AIR|ounts poach carefu 2721|2633|2634|2|2|3071.26|0.02|0.05|N|O|1996-02-13|1996-03-14|1996-02-28|TAKE BACK RETURN|TRUCK| slyly final requests against 2722|123803|1340|1|21|38362.80|0.09|0.01|A|F|1994-07-29|1994-06-26|1994-08-09|NONE|RAIL|e carefully around the furiously ironic pac 2722|145739|5740|2|15|26770.95|0.05|0.03|R|F|1994-07-02|1994-06-01|1994-07-13|COLLECT COD|AIR|refully final asympt 2722|33709|6213|3|16|26283.20|0.04|0.06|R|F|1994-05-25|1994-06-09|1994-05-26|NONE|MAIL|ts besides the fluffy, 2723|12688|5190|1|47|75231.96|0.09|0.07|N|O|1995-12-05|1995-11-19|1995-12-11|TAKE BACK RETURN|AIR|furiously r 2723|31913|6920|2|10|18449.10|0.06|0.08|N|O|1995-11-27|1995-11-29|1995-12-12|DELIVER IN PERSON|MAIL|al, special r 2723|161747|4264|3|2|3617.48|0.10|0.01|N|O|1995-11-09|1995-11-10|1995-11-14|TAKE BACK RETURN|FOB| courts boost quickly about th 2723|81212|6229|4|12|14318.52|0.01|0.05|N|O|1995-12-24|1995-11-15|1996-01-17|DELIVER IN PERSON|RAIL|bold foxes are bold packages. regular, fin 2723|128077|8078|5|40|44202.80|0.09|0.05|N|O|1995-11-17|1995-11-22|1995-11-18|TAKE BACK RETURN|MAIL|unwind fluffily carefully regular realms. 2724|91316|3826|1|47|61443.57|0.09|0.01|A|F|1994-11-23|1994-11-13|1994-12-03|COLLECT COD|TRUCK|unusual patterns nag. special p 2724|146966|6967|2|21|42272.16|0.09|0.02|A|F|1994-11-25|1994-10-15|1994-12-07|COLLECT COD|RAIL|as. carefully regular dependencies wak 2724|49839|4848|3|22|39354.26|0.04|0.06|A|F|1994-09-19|1994-11-18|1994-10-17|TAKE BACK RETURN|TRUCK|express fo 2724|34495|9502|4|1|1429.49|0.07|0.03|A|F|1994-12-26|1994-11-27|1995-01-07|NONE|MAIL|lyly carefully blithe theodolites-- pl 2724|148071|3100|5|29|32453.03|0.05|0.06|A|F|1995-01-10|1994-11-17|1995-02-04|COLLECT COD|MAIL|l requests hagg 2725|117595|107|1|23|37089.57|0.10|0.08|R|F|1994-08-25|1994-06-22|1994-08-28|TAKE BACK RETURN|REG AIR|y regular deposits. brave foxes 2725|4819|7320|2|41|70676.21|0.01|0.00|R|F|1994-07-05|1994-06-29|1994-08-02|DELIVER IN PERSON|TRUCK|ns sleep furiously c 2725|188318|5873|3|15|21094.65|0.07|0.03|R|F|1994-08-06|1994-08-09|1994-08-15|TAKE BACK RETURN|AIR|? furiously regular a 2726|655|5656|1|50|77782.50|0.00|0.06|R|F|1993-03-04|1993-01-29|1993-03-28|COLLECT COD|TRUCK| furiously bold theodolites 2727|150928|5959|1|3|5936.76|0.03|0.01|N|O|1998-06-18|1998-06-06|1998-06-23|NONE|RAIL| the carefully regular foxes u 2752|30702|5709|1|41|66940.70|0.02|0.05|A|F|1994-03-02|1994-01-31|1994-03-06|DELIVER IN PERSON|AIR|tructions hag 2752|6810|1811|2|29|49787.49|0.02|0.04|R|F|1994-01-22|1994-01-08|1994-01-28|COLLECT COD|TRUCK|gly blithely re 2752|55050|5051|3|4|4020.20|0.08|0.00|A|F|1993-12-14|1994-02-13|1994-01-05|DELIVER IN PERSON|TRUCK|telets haggle. regular, final 2752|23153|660|4|40|43046.00|0.09|0.06|A|F|1994-01-24|1994-01-18|1994-02-22|DELIVER IN PERSON|MAIL|into beans are after the sly 2752|125934|959|5|22|43118.46|0.03|0.04|A|F|1994-03-20|1994-02-08|1994-04-01|TAKE BACK RETURN|TRUCK|equests nag. regular dependencies are furio 2752|169347|6896|6|21|29743.14|0.09|0.05|R|F|1994-01-01|1994-01-24|1994-01-24|COLLECT COD|SHIP| along the quickly 2752|198918|8919|7|38|76642.58|0.08|0.00|R|F|1994-02-23|1993-12-23|1994-03-24|DELIVER IN PERSON|SHIP|es boost. slyly silent ideas 2753|12855|359|1|6|10607.10|0.10|0.04|A|F|1993-12-30|1994-01-28|1994-01-29|COLLECT COD|TRUCK|s accounts 2753|47010|4523|2|40|38280.40|0.03|0.05|A|F|1994-01-06|1994-02-13|1994-02-03|DELIVER IN PERSON|SHIP|latelets kindle slyly final depos 2753|88608|3625|3|30|47898.00|0.00|0.07|A|F|1994-01-26|1994-01-29|1994-02-02|NONE|RAIL|ans wake fluffily blithely iro 2753|30853|3357|4|7|12486.95|0.07|0.03|R|F|1994-02-11|1994-01-22|1994-03-10|DELIVER IN PERSON|AIR|xpress ideas detect b 2753|136899|6900|5|36|69692.04|0.04|0.08|R|F|1994-03-15|1994-01-03|1994-04-03|DELIVER IN PERSON|SHIP|gle slyly final c 2753|49203|9204|6|17|19587.40|0.01|0.08|A|F|1994-03-08|1994-01-17|1994-03-11|TAKE BACK RETURN|REG AIR| carefully bold deposits sublate s 2753|147138|7139|7|20|23702.60|0.01|0.06|R|F|1994-02-24|1994-02-04|1994-03-23|DELIVER IN PERSON|FOB| express pack 2754|148092|607|1|4|4560.36|0.05|0.08|A|F|1994-07-13|1994-05-15|1994-08-02|NONE|REG AIR|blithely silent requests. regular depo 2754|176662|4214|2|19|33034.54|0.01|0.07|A|F|1994-06-27|1994-05-06|1994-06-28|NONE|FOB|latelets hag 2755|91404|3914|1|19|26512.60|0.10|0.00|R|F|1992-02-11|1992-03-15|1992-02-14|TAKE BACK RETURN|MAIL|furiously special deposits 2755|23483|8488|2|11|15471.28|0.03|0.08|A|F|1992-04-12|1992-05-07|1992-04-21|COLLECT COD|RAIL|egular excuses sleep carefully. 2755|63720|6227|3|21|35358.12|0.08|0.04|R|F|1992-02-13|1992-04-20|1992-03-02|NONE|AIR|furious re 2755|130096|7636|4|5|5630.45|0.01|0.00|A|F|1992-02-27|1992-04-07|1992-03-09|TAKE BACK RETURN|AIR|e the furi 2755|115039|5040|5|48|50593.44|0.05|0.06|R|F|1992-03-22|1992-03-10|1992-04-14|DELIVER IN PERSON|MAIL|yly even epitaphs for the 2756|117254|7255|1|35|44493.75|0.03|0.02|R|F|1994-06-08|1994-06-01|1994-06-21|TAKE BACK RETURN|AIR| deposits grow bold sheaves; iro 2756|79212|4227|2|47|55986.87|0.06|0.01|R|F|1994-05-10|1994-05-25|1994-05-13|NONE|AIR|e final, f 2756|104806|7317|3|31|56134.80|0.01|0.07|A|F|1994-07-27|1994-07-06|1994-08-22|TAKE BACK RETURN|TRUCK|en instructions use quickly. 2756|71763|4271|4|30|52042.80|0.00|0.04|A|F|1994-06-05|1994-06-30|1994-06-14|DELIVER IN PERSON|TRUCK|ular packages. regular deposi 2757|147369|9884|1|26|36825.36|0.07|0.00|N|O|1995-08-19|1995-10-02|1995-09-06|DELIVER IN PERSON|MAIL|around the blithely 2757|21865|4368|2|12|21442.32|0.07|0.08|N|O|1995-08-01|1995-09-04|1995-08-08|TAKE BACK RETURN|SHIP| regular, eve 2757|72740|5248|3|17|29116.58|0.10|0.04|N|O|1995-09-06|1995-09-27|1995-09-22|DELIVER IN PERSON|AIR|er the furiously silent 2757|139197|4224|4|25|30904.75|0.08|0.01|N|O|1995-11-09|1995-09-12|1995-11-23|NONE|AIR|uickly regular 2757|69696|4709|5|14|23319.66|0.04|0.05|N|O|1995-09-01|1995-08-24|1995-09-03|TAKE BACK RETURN|SHIP|special deposits u 2758|120880|5905|1|20|38017.60|0.02|0.04|N|O|1998-07-27|1998-09-10|1998-08-21|TAKE BACK RETURN|AIR|ptotes sleep furiously 2758|22897|5400|2|17|30938.13|0.10|0.06|N|O|1998-09-25|1998-10-03|1998-10-25|NONE|MAIL| accounts! qui 2758|25560|565|3|1|1485.56|0.06|0.02|N|O|1998-10-09|1998-09-15|1998-10-16|NONE|TRUCK|ake furious 2759|58995|6511|1|10|19539.90|0.10|0.03|R|F|1993-12-14|1994-01-08|1994-01-01|COLLECT COD|FOB|s. busily ironic theodo 2759|112674|7697|2|37|62406.79|0.00|0.06|R|F|1994-03-05|1994-02-22|1994-03-18|DELIVER IN PERSON|REG AIR|lar Tiresias affix ironically carefully sp 2759|111646|6669|3|11|18234.04|0.03|0.08|A|F|1994-01-24|1994-01-16|1994-02-21|DELIVER IN PERSON|TRUCK|hely regular 2759|22777|7782|4|31|52692.87|0.02|0.05|A|F|1994-01-11|1994-01-15|1994-01-23|NONE|SHIP|ithely aft 2784|32335|7342|1|45|57029.85|0.03|0.01|N|O|1998-02-15|1998-04-07|1998-02-26|COLLECT COD|AIR|yly along the asymptotes. reque 2784|53524|3525|2|23|33982.96|0.03|0.05|N|O|1998-03-28|1998-02-07|1998-04-17|DELIVER IN PERSON|AIR|uests lose after 2784|174715|9750|3|40|71588.40|0.07|0.01|N|O|1998-04-28|1998-03-19|1998-05-03|DELIVER IN PERSON|TRUCK|deas nag furiously never unusual 2784|28004|8005|4|3|2796.00|0.04|0.03|N|O|1998-01-19|1998-04-05|1998-02-05|TAKE BACK RETURN|AIR|n packages. foxes haggle quickly sile 2785|99049|4068|1|34|35633.36|0.08|0.06|N|O|1995-08-07|1995-09-09|1995-09-05|NONE|RAIL|ly final packages haggl 2785|109744|7275|2|37|64888.38|0.08|0.04|N|O|1995-07-25|1995-09-12|1995-08-06|DELIVER IN PERSON|TRUCK|tructions. furiously 2785|64524|2043|3|33|49121.16|0.08|0.06|N|O|1995-10-16|1995-08-24|1995-11-02|DELIVER IN PERSON|MAIL|fter the furiously final p 2785|47167|2176|4|34|37881.44|0.00|0.02|N|O|1995-09-16|1995-09-09|1995-10-11|COLLECT COD|SHIP|kages wake carefully silent 2786|135566|3106|1|15|24023.40|0.03|0.04|A|F|1992-05-19|1992-05-08|1992-05-28|COLLECT COD|TRUCK|low deposits are ironic 2786|50144|7660|2|42|45953.88|0.10|0.04|R|F|1992-05-15|1992-04-22|1992-05-30|DELIVER IN PERSON|AIR|unts are against the furious 2786|155318|349|3|41|56305.71|0.04|0.05|R|F|1992-07-01|1992-06-04|1992-07-13|COLLECT COD|RAIL|ix requests. bold requests a 2786|22696|2697|4|24|38848.56|0.05|0.02|A|F|1992-04-04|1992-06-09|1992-05-02|DELIVER IN PERSON|MAIL|ans. slyly unusual platelets detect. unus 2786|49478|4487|5|43|61381.21|0.06|0.03|R|F|1992-04-22|1992-05-13|1992-04-29|NONE|RAIL|ons. theodolites after 2786|161228|3745|6|21|27073.62|0.08|0.00|A|F|1992-05-03|1992-05-01|1992-05-14|COLLECT COD|AIR|slow instructi 2787|32479|9989|1|4|5645.88|0.04|0.04|N|O|1996-01-26|1995-11-26|1996-02-20|TAKE BACK RETURN|SHIP|ts. instructions nag furiously according 2788|176196|6197|1|16|20355.04|0.06|0.06|A|F|1994-10-04|1994-11-25|1994-10-18|DELIVER IN PERSON|AIR| requests wake carefully. carefully si 2789|162410|9959|1|16|23558.56|0.03|0.02|N|O|1998-04-18|1998-05-25|1998-05-12|DELIVER IN PERSON|REG AIR|o beans use carefully 2789|22835|2836|2|41|72071.03|0.02|0.05|N|O|1998-03-20|1998-05-15|1998-03-21|COLLECT COD|MAIL|d packages-- fluffily specia 2789|175892|927|3|33|64940.37|0.06|0.02|N|O|1998-04-21|1998-05-02|1998-04-30|COLLECT COD|TRUCK|deposits. ironic 2789|15982|985|4|47|89205.06|0.02|0.04|N|O|1998-03-29|1998-05-05|1998-04-07|NONE|RAIL|usly busy packages wake against the unusual 2789|196629|4187|5|23|39689.26|0.02|0.07|N|O|1998-03-25|1998-05-10|1998-04-24|COLLECT COD|RAIL|cording to the careful de 2789|143603|3604|6|16|26345.60|0.07|0.03|N|O|1998-05-11|1998-05-08|1998-05-24|TAKE BACK RETURN|RAIL|d the carefully iron 2789|132206|7233|7|42|52004.40|0.01|0.00|N|O|1998-04-28|1998-05-17|1998-05-24|TAKE BACK RETURN|AIR|ending packages shoul 2790|184300|4301|1|27|37376.10|0.06|0.08|R|F|1994-09-04|1994-09-27|1994-09-16|TAKE BACK RETURN|MAIL|ilent packages cajole. quickly ironic requ 2790|116975|9487|2|50|99598.50|0.00|0.06|A|F|1994-12-08|1994-11-17|1994-12-19|NONE|RAIL|fter the regular ideas. f 2790|183138|8175|3|19|23201.47|0.06|0.00|R|F|1994-10-23|1994-10-03|1994-10-26|TAKE BACK RETURN|RAIL|uffily even excuses. furiously thin 2790|196404|6405|4|24|36009.60|0.07|0.01|A|F|1994-12-04|1994-10-10|1994-12-25|NONE|MAIL|ments. slyly f 2790|147955|7956|5|11|22032.45|0.08|0.03|A|F|1994-09-28|1994-11-14|1994-10-04|TAKE BACK RETURN|AIR|lar requests poach slyly foxes 2790|72637|5145|6|13|20925.19|0.08|0.00|R|F|1994-09-20|1994-10-10|1994-10-20|COLLECT COD|SHIP|n deposits according to the regul 2790|3931|1432|7|32|58717.76|0.08|0.02|A|F|1994-09-25|1994-10-26|1994-10-01|NONE|SHIP|ully pending 2791|58694|8695|1|49|80981.81|0.10|0.04|A|F|1995-01-11|1994-11-10|1995-02-08|COLLECT COD|MAIL| accounts sleep at the bold, regular pinto 2791|62804|2805|2|4|7067.20|0.10|0.08|A|F|1995-01-02|1994-12-28|1995-01-29|NONE|SHIP|slyly bold packages boost. slyly 2791|132300|4814|3|44|58621.20|0.08|0.06|R|F|1994-11-17|1994-11-12|1994-12-14|NONE|FOB|heodolites use furio 2791|155894|3440|4|24|46797.36|0.04|0.02|R|F|1995-01-30|1994-11-20|1995-02-08|DELIVER IN PERSON|TRUCK|ilent forges. quickly special pinto beans 2791|104521|2052|5|8|12204.16|0.02|0.04|R|F|1995-01-30|1994-11-24|1995-02-13|NONE|FOB|se. close ideas alongs 2791|74435|1957|6|9|12684.87|0.08|0.02|R|F|1994-11-19|1994-12-14|1994-12-10|TAKE BACK RETURN|AIR|pendencies. blithely bold patterns acr 2791|28635|6142|7|26|40654.38|0.06|0.03|R|F|1995-02-06|1994-12-07|1995-02-23|DELIVER IN PERSON|AIR|uriously special instructio 2816|58258|8259|1|33|40136.25|0.00|0.07|R|F|1994-10-19|1994-11-10|1994-11-09|NONE|REG AIR|s; slyly even theodo 2816|141342|1343|2|4|5533.36|0.05|0.04|R|F|1994-12-11|1994-12-07|1995-01-03|NONE|FOB|. blithely pending id 2816|120406|2919|3|4|5705.60|0.02|0.06|R|F|1994-12-12|1994-12-05|1994-12-30|NONE|RAIL| requests print above the final deposits 2817|59597|2103|1|25|38914.75|0.07|0.01|R|F|1994-04-21|1994-06-20|1994-05-07|DELIVER IN PERSON|FOB|doze blithely. 2817|31271|3775|2|5|6011.35|0.03|0.04|A|F|1994-05-07|1994-05-31|1994-05-12|TAKE BACK RETURN|AIR|furiously unusual theodolites use furiou 2817|171177|8729|3|35|43685.95|0.01|0.07|A|F|1994-05-20|1994-06-03|1994-05-22|COLLECT COD|FOB|gular foxes 2817|160884|885|4|4|7779.52|0.00|0.05|R|F|1994-06-04|1994-06-11|1994-06-10|NONE|TRUCK|n accounts wake across the fluf 2818|120379|7916|1|12|16792.44|0.10|0.03|A|F|1995-02-01|1995-03-10|1995-02-16|NONE|AIR|lms. quickly bold asymp 2818|198766|3805|2|22|41024.72|0.06|0.07|R|F|1995-02-28|1995-03-10|1995-03-06|TAKE BACK RETURN|RAIL|egrate toward the carefully iron 2818|44234|4235|3|11|12960.53|0.01|0.06|R|F|1995-02-18|1995-02-11|1995-03-19|TAKE BACK RETURN|TRUCK|ggle across the carefully blithe 2818|39711|7221|4|32|52822.72|0.08|0.08|R|F|1995-02-04|1995-03-05|1995-02-18|COLLECT COD|REG AIR|arefully! ac 2818|17380|4884|5|42|54489.96|0.08|0.04|A|F|1995-02-12|1995-02-19|1995-03-13|COLLECT COD|MAIL|ar accounts wake carefully a 2818|90636|8164|6|7|11386.41|0.06|0.03|R|F|1995-03-24|1995-03-09|1995-04-06|TAKE BACK RETURN|TRUCK|ly according to the r 2819|69083|9084|1|17|17885.36|0.08|0.08|A|F|1994-07-16|1994-07-15|1994-07-17|TAKE BACK RETURN|RAIL|en deposits above the f 2819|66461|3980|2|12|17129.52|0.03|0.08|R|F|1994-07-18|1994-06-24|1994-07-28|NONE|MAIL| regular, regular a 2819|4924|2425|3|28|51209.76|0.03|0.08|R|F|1994-05-09|1994-07-02|1994-05-15|NONE|RAIL|ckages sublate carefully closely regular 2819|152830|2831|4|5|9414.15|0.00|0.02|R|F|1994-05-29|1994-06-12|1994-06-28|NONE|TRUCK| fluffily unusual foxes sleep caref 2819|199179|4218|5|6|7669.02|0.03|0.01|A|F|1994-07-22|1994-08-02|1994-07-29|NONE|REG AIR|eas after the carefully express pack 2820|173150|702|1|23|28132.45|0.04|0.08|R|F|1994-07-10|1994-08-08|1994-07-21|NONE|MAIL| was furiously. deposits among the ironic 2820|125718|3255|2|33|57542.43|0.08|0.06|A|F|1994-07-07|1994-08-17|1994-08-02|DELIVER IN PERSON|AIR|carefully even pinto beans. 2820|140013|7556|3|38|40014.38|0.03|0.08|A|F|1994-09-10|1994-08-07|1994-10-07|TAKE BACK RETURN|MAIL|ests despite the carefully unusual a 2820|196270|8790|4|40|54650.80|0.06|0.06|A|F|1994-08-08|1994-07-30|1994-08-21|TAKE BACK RETURN|REG AIR|g multipliers. final c 2821|180834|8389|1|4|7659.32|0.00|0.00|A|F|1993-09-15|1993-10-02|1993-09-17|TAKE BACK RETURN|TRUCK|nding foxes. 2821|71159|6174|2|4|4520.60|0.09|0.00|A|F|1993-11-19|1993-09-20|1993-11-27|TAKE BACK RETURN|TRUCK|ual multipliers. final deposits cajol 2821|163639|8672|3|27|45971.01|0.01|0.01|A|F|1993-11-27|1993-10-11|1993-12-08|COLLECT COD|TRUCK|requests. blit 2822|150765|3281|1|39|70814.64|0.04|0.02|R|F|1993-09-11|1993-08-29|1993-09-18|NONE|MAIL|kly about the sly 2823|85712|8221|1|45|76396.95|0.03|0.04|N|O|1995-12-28|1995-11-27|1996-01-02|DELIVER IN PERSON|SHIP|furiously special idea 2823|159624|4655|2|18|30305.16|0.00|0.03|N|O|1995-11-11|1995-10-30|1995-12-08|TAKE BACK RETURN|TRUCK| final deposits. furiously regular foxes u 2823|185277|7796|3|11|14984.97|0.07|0.02|N|O|1995-12-10|1995-11-24|1995-12-21|DELIVER IN PERSON|SHIP|bold requests nag blithely s 2823|138806|3833|4|48|88550.40|0.09|0.03|N|O|1995-11-21|1995-10-30|1995-11-27|NONE|SHIP|ously busily slow excus 2823|98943|3962|5|18|34954.92|0.04|0.06|N|O|1995-11-09|1995-10-30|1995-11-19|NONE|AIR|eas. decoys cajole deposi 2823|122443|7468|6|20|29308.80|0.07|0.00|N|O|1995-11-13|1995-12-06|1995-12-07|NONE|MAIL|its sleep between the unusual, ironic pac 2823|85076|2601|7|12|12732.84|0.02|0.04|N|O|1995-12-22|1995-11-20|1996-01-13|NONE|REG AIR|the slyly ironic dolphins; fin 2848|64222|6729|1|44|52193.68|0.01|0.05|R|F|1992-04-14|1992-05-09|1992-04-19|DELIVER IN PERSON|MAIL|ions. slyly express instructions n 2848|164452|4453|2|8|12131.60|0.07|0.01|A|F|1992-03-21|1992-05-18|1992-04-07|DELIVER IN PERSON|TRUCK|. silent, final ideas sublate packages. ir 2848|137332|9846|3|8|10954.64|0.07|0.08|A|F|1992-06-20|1992-04-12|1992-07-09|NONE|SHIP|sly regular foxes. 2848|124948|4949|4|34|67079.96|0.02|0.08|A|F|1992-03-15|1992-04-24|1992-04-12|TAKE BACK RETURN|RAIL|ts along the blithely regu 2848|194197|6717|5|18|23241.42|0.07|0.03|R|F|1992-04-10|1992-06-01|1992-05-05|DELIVER IN PERSON|TRUCK|osits haggle. stealthily ironic packa 2849|153805|6321|1|16|29740.80|0.09|0.08|N|O|1996-05-20|1996-07-23|1996-06-18|NONE|TRUCK|. furiously regular requ 2849|186843|1880|2|39|75263.76|0.10|0.03|N|O|1996-05-22|1996-07-18|1996-06-05|TAKE BACK RETURN|SHIP|s sleep furiously silently regul 2849|59872|9873|3|24|43964.88|0.01|0.05|N|O|1996-06-12|1996-07-10|1996-06-27|TAKE BACK RETURN|AIR|e slyly even asymptotes. slo 2849|54230|1746|4|48|56843.04|0.05|0.02|N|O|1996-05-03|1996-06-05|1996-05-28|NONE|AIR|mong the carefully regular theodol 2849|27255|2260|5|30|35467.50|0.10|0.06|N|O|1996-08-24|1996-07-08|1996-09-03|TAKE BACK RETURN|SHIP|ly. carefully silent 2849|68651|6170|6|30|48589.50|0.06|0.07|N|O|1996-06-20|1996-07-23|1996-07-06|NONE|FOB|yly furiously even id 2850|96931|4459|1|43|82900.99|0.02|0.05|N|O|1997-01-11|1996-11-03|1997-02-01|COLLECT COD|REG AIR|unusual accounts 2850|109283|6814|2|30|38768.40|0.09|0.01|N|O|1996-12-14|1996-11-29|1997-01-03|COLLECT COD|AIR|even ideas. busy pinto beans sleep above t 2850|104731|4732|3|49|85050.77|0.09|0.04|N|O|1996-10-07|1996-12-12|1996-10-12|TAKE BACK RETURN|MAIL| slyly unusual req 2850|198372|5930|4|4|5881.48|0.04|0.04|N|O|1996-10-28|1996-12-26|1996-11-07|COLLECT COD|RAIL|al deposits cajole carefully quickly 2851|147452|9967|1|8|11995.60|0.09|0.03|N|O|1997-11-12|1997-11-22|1997-12-11|NONE|REG AIR|y special theodolites. carefully 2852|176179|1214|1|6|7531.02|0.01|0.01|R|F|1993-03-02|1993-04-11|1993-03-11|TAKE BACK RETURN|RAIL| accounts above the furiously un 2852|40065|7578|2|24|24121.44|0.05|0.07|R|F|1993-01-18|1993-03-13|1993-02-14|DELIVER IN PERSON|MAIL| the blithe 2852|163403|952|3|29|42525.60|0.09|0.05|R|F|1993-04-21|1993-03-22|1993-05-02|COLLECT COD|SHIP|lyly ironi 2852|99183|4202|4|12|14186.16|0.08|0.02|A|F|1993-02-25|1993-03-24|1993-03-07|TAKE BACK RETURN|TRUCK|le. request 2852|153182|5698|5|28|34585.04|0.05|0.03|R|F|1993-02-08|1993-03-30|1993-02-11|NONE|MAIL|e accounts. caref 2853|138855|1369|1|14|26513.90|0.07|0.05|R|F|1994-05-16|1994-07-01|1994-05-27|NONE|TRUCK|oach slyly along t 2853|133051|591|2|26|28185.30|0.06|0.01|R|F|1994-06-26|1994-06-05|1994-07-02|TAKE BACK RETURN|MAIL|dolphins wake slyly. blith 2853|172674|5192|3|40|69866.80|0.06|0.04|A|F|1994-08-06|1994-06-24|1994-08-29|NONE|RAIL|lyly. pearls cajole. final accounts ca 2853|131257|8797|4|20|25765.00|0.02|0.04|A|F|1994-08-30|1994-06-16|1994-09-06|TAKE BACK RETURN|TRUCK|e slyly silent foxes. express deposits sno 2853|35872|879|5|1|1807.87|0.08|0.05|R|F|1994-09-01|1994-06-27|1994-09-12|TAKE BACK RETURN|FOB|refully slyly quick packages. final c 2854|180193|7748|1|46|58566.74|0.00|0.04|A|F|1994-09-22|1994-08-02|1994-09-30|COLLECT COD|AIR|. furiously regular deposits across th 2854|87581|7582|2|29|45488.82|0.09|0.07|R|F|1994-07-06|1994-08-26|1994-07-09|COLLECT COD|SHIP|y slyly ironic accounts. foxes haggle slyl 2854|159170|1686|3|20|24583.40|0.08|0.01|R|F|1994-09-18|1994-08-03|1994-10-12|COLLECT COD|AIR|rs impress after the deposits. 2854|169448|9449|4|34|51592.96|0.06|0.03|A|F|1994-09-06|1994-08-07|1994-09-22|NONE|REG AIR|age carefully 2854|101316|1317|5|7|9221.17|0.03|0.06|A|F|1994-09-23|1994-08-14|1994-10-10|DELIVER IN PERSON|REG AIR| the pending 2854|17762|264|6|13|21836.88|0.04|0.03|R|F|1994-09-15|1994-08-18|1994-09-19|DELIVER IN PERSON|SHIP| excuses wak 2855|32224|7231|1|50|57811.00|0.03|0.07|A|F|1993-05-20|1993-06-28|1993-06-16|TAKE BACK RETURN|TRUCK|beans. deposits 2880|34078|9085|1|40|40482.80|0.09|0.00|A|F|1992-05-26|1992-06-01|1992-05-31|COLLECT COD|TRUCK|even requests. quick 2880|138854|6394|2|26|49214.10|0.07|0.07|R|F|1992-04-12|1992-04-15|1992-04-28|NONE|RAIL|ully among the regular warthogs 2880|114490|7002|3|42|63188.58|0.01|0.01|R|F|1992-06-17|1992-05-29|1992-07-11|NONE|REG AIR|ions. carefully final accounts are unusual, 2880|17656|158|4|46|72387.90|0.02|0.02|A|F|1992-04-21|1992-06-05|1992-05-16|COLLECT COD|RAIL|eep quickly according to t 2881|179343|1861|1|16|22757.44|0.02|0.06|A|F|1992-06-21|1992-06-27|1992-07-03|TAKE BACK RETURN|TRUCK|usly bold 2881|9181|9182|2|1|1090.18|0.09|0.03|A|F|1992-05-13|1992-07-21|1992-05-18|COLLECT COD|MAIL|final theodolites. quickly 2881|92018|7037|3|21|21210.21|0.07|0.03|A|F|1992-05-28|1992-07-03|1992-06-02|TAKE BACK RETURN|SHIP|hely express Tiresias. final dependencies 2881|139814|7354|4|7|12976.67|0.06|0.01|R|F|1992-08-03|1992-07-10|1992-08-27|NONE|REG AIR|ironic packages are carefully final ac 2882|3763|6264|1|14|23334.64|0.09|0.02|N|O|1995-09-28|1995-11-11|1995-10-18|TAKE BACK RETURN|MAIL|kly. even requests w 2882|41007|8520|2|30|28440.00|0.00|0.00|N|O|1995-10-15|1995-10-13|1995-10-25|NONE|REG AIR|among the furiously even theodolites. regu 2882|196935|9455|3|29|58925.97|0.10|0.08|N|O|1995-09-10|1995-11-01|1995-10-02|NONE|TRUCK|kages. furiously ironic 2882|77298|4820|4|27|34432.83|0.06|0.02|N|O|1995-09-04|1995-11-11|1995-09-12|DELIVER IN PERSON|MAIL|rding to the regu 2882|133878|3879|5|32|61179.84|0.07|0.03|N|O|1995-10-21|1995-11-10|1995-11-01|COLLECT COD|RAIL|sts. quickly regular e 2882|86425|6426|6|47|66336.74|0.06|0.03|N|O|1995-09-13|1995-09-21|1995-09-14|NONE|REG AIR|l, special 2883|91|2592|1|33|32705.97|0.08|0.07|R|F|1995-02-26|1995-03-04|1995-03-01|NONE|RAIL|s. final i 2883|124341|4342|2|27|36864.18|0.00|0.02|A|F|1995-03-12|1995-03-10|1995-04-04|TAKE BACK RETURN|REG AIR|s. brave pinto beans nag furiously 2883|188690|6245|3|47|83598.43|0.05|0.04|R|F|1995-01-29|1995-04-19|1995-02-05|DELIVER IN PERSON|SHIP|ep carefully ironic 2883|97483|5011|4|23|34051.04|0.00|0.02|R|F|1995-02-03|1995-03-17|1995-02-19|TAKE BACK RETURN|AIR| even requests cajole. special, regular 2883|194465|9504|5|36|56140.56|0.07|0.06|A|F|1995-05-02|1995-03-14|1995-05-30|COLLECT COD|MAIL|ests detect slyly special packages 2884|70492|493|1|41|59962.09|0.03|0.00|N|O|1998-01-02|1997-12-17|1998-01-20|DELIVER IN PERSON|TRUCK|ep. slyly even accounts a 2884|145132|2675|2|25|29428.25|0.09|0.08|N|O|1998-01-18|1997-12-06|1998-02-16|TAKE BACK RETURN|MAIL|onic theodolites with the instructi 2884|25106|5107|3|8|8248.80|0.08|0.08|N|O|1997-11-30|1997-11-28|1997-12-14|COLLECT COD|TRUCK|pending accounts about 2885|3054|8055|1|6|5742.30|0.10|0.01|A|F|1993-01-05|1992-12-12|1993-01-19|COLLECT COD|FOB|ctions solve. slyly regular requests n 2885|111966|1967|2|4|7911.84|0.07|0.00|A|F|1992-10-09|1992-12-17|1992-11-04|TAKE BACK RETURN|SHIP| pending packages wake. 2885|716|5717|3|45|72751.95|0.10|0.04|A|F|1992-12-24|1992-10-30|1993-01-04|NONE|SHIP|ess ideas. regular, silen 2885|31307|6314|4|15|18574.50|0.03|0.04|R|F|1992-10-31|1992-11-24|1992-11-21|DELIVER IN PERSON|MAIL|odolites. boldly pending packages han 2885|174189|6707|5|43|54316.74|0.06|0.00|R|F|1992-11-17|1992-10-30|1992-12-04|DELIVER IN PERSON|SHIP|cial deposits use bold 2885|189615|7170|6|5|8523.05|0.01|0.02|R|F|1993-01-06|1992-11-13|1993-02-05|TAKE BACK RETURN|TRUCK|s. slyly express th 2885|49234|6747|7|40|47329.20|0.05|0.03|A|F|1992-09-23|1992-11-15|1992-10-07|TAKE BACK RETURN|AIR| express depos 2886|59748|9749|1|1|1707.74|0.09|0.05|A|F|1995-02-01|1994-12-18|1995-02-28|COLLECT COD|REG AIR|eposits fr 2886|183832|3833|2|38|72801.54|0.02|0.04|A|F|1995-01-21|1995-01-08|1995-01-30|NONE|SHIP|old requests along the fur 2886|62934|453|3|2|3793.86|0.04|0.07|A|F|1994-11-18|1995-01-31|1994-12-05|COLLECT COD|REG AIR|ar theodolites. e 2886|129991|7528|4|46|92965.54|0.03|0.08|A|F|1995-02-02|1995-01-26|1995-02-15|TAKE BACK RETURN|SHIP|ously final packages sleep blithely regular 2887|65504|517|1|11|16164.50|0.06|0.00|N|O|1997-07-08|1997-07-17|1997-07-15|COLLECT COD|SHIP|ackages. unusual, speci 2887|111534|4046|2|17|26274.01|0.00|0.08|N|O|1997-08-31|1997-07-04|1997-09-17|DELIVER IN PERSON|SHIP|fily final packages. regula 2912|121445|6470|1|8|11731.52|0.06|0.04|A|F|1992-04-09|1992-04-19|1992-04-26|NONE|RAIL|hs cajole over the slyl 2912|114577|7089|2|18|28648.26|0.00|0.08|R|F|1992-03-13|1992-04-19|1992-03-30|TAKE BACK RETURN|RAIL|unts cajole reg 2913|122736|273|1|39|68590.47|0.06|0.04|N|O|1997-08-28|1997-09-27|1997-09-02|TAKE BACK RETURN|AIR|. final packages a 2913|21727|9234|2|22|36271.84|0.10|0.07|N|O|1997-09-18|1997-08-11|1997-10-02|COLLECT COD|MAIL|riously pending realms. blithely even pac 2913|165309|2858|3|17|23363.10|0.07|0.04|N|O|1997-10-21|1997-09-25|1997-11-20|NONE|FOB|requests doze quickly. furious 2913|142502|2503|4|5|7722.50|0.10|0.07|N|O|1997-10-07|1997-08-25|1997-10-09|TAKE BACK RETURN|RAIL|haggle. even, bold instructi 2913|14885|7387|5|13|23398.44|0.03|0.01|N|O|1997-10-02|1997-08-20|1997-10-26|COLLECT COD|MAIL|inos are carefully alongside of the bol 2913|167718|2751|6|35|62499.85|0.06|0.08|N|O|1997-08-30|1997-08-21|1997-09-03|COLLECT COD|MAIL|es. quickly even braids against 2914|65685|5686|1|22|36314.96|0.05|0.06|R|F|1993-05-11|1993-04-09|1993-05-22|DELIVER IN PERSON|FOB| carefully about the fluffily ironic gifts 2914|162197|7230|2|25|31479.75|0.03|0.04|A|F|1993-05-14|1993-04-04|1993-05-22|NONE|SHIP|cross the carefully even accounts. 2914|34588|7092|3|4|6090.32|0.00|0.05|R|F|1993-06-11|1993-04-09|1993-06-14|TAKE BACK RETURN|SHIP|s integrate. bold deposits sleep req 2914|120324|325|4|9|12098.88|0.06|0.01|R|F|1993-06-17|1993-05-26|1993-06-19|NONE|REG AIR|s. carefully final foxes ar 2915|174124|6642|1|28|33547.36|0.10|0.02|R|F|1994-04-17|1994-06-09|1994-05-10|NONE|MAIL|yly special 2915|93386|8405|2|12|16552.56|0.00|0.03|A|F|1994-07-18|1994-06-11|1994-07-27|TAKE BACK RETURN|RAIL|accounts. slyly final 2915|135401|7915|3|15|21546.00|0.07|0.00|A|F|1994-05-01|1994-06-12|1994-05-15|DELIVER IN PERSON|TRUCK|al requests haggle furiousl 2915|80116|2625|4|43|47132.73|0.06|0.05|R|F|1994-06-02|1994-05-24|1994-06-06|DELIVER IN PERSON|SHIP|into beans dazzle alongside of 2916|82501|5010|1|21|31153.50|0.06|0.04|N|O|1996-03-11|1996-02-21|1996-03-30|NONE|REG AIR|uickly express ideas over the slyly even 2917|92630|2631|1|36|58414.68|0.10|0.01|N|O|1998-04-07|1998-02-23|1998-05-01|DELIVER IN PERSON|RAIL|usly ironic d 2917|20711|712|2|20|32634.20|0.06|0.03|N|O|1997-12-31|1998-01-22|1998-01-12|NONE|MAIL|slyly even ideas wa 2917|89342|6867|3|4|5325.36|0.02|0.07|N|O|1998-01-10|1998-01-18|1998-02-08|TAKE BACK RETURN|REG AIR|s. unusual instruct 2917|166826|4375|4|5|9464.10|0.05|0.01|N|O|1997-12-16|1998-01-26|1998-01-07|NONE|RAIL|bove the furiously silent packages. pend 2917|40412|7925|5|37|50039.17|0.04|0.01|N|O|1997-12-12|1998-02-03|1997-12-23|COLLECT COD|RAIL|dependencies. express 2917|193921|1479|6|7|14104.44|0.05|0.01|N|O|1998-03-21|1998-03-03|1998-03-25|NONE|REG AIR|ly about the regular accounts. carefully pe 2918|77412|2427|1|24|33345.84|0.10|0.03|N|O|1996-12-20|1996-10-28|1996-12-26|DELIVER IN PERSON|FOB| quickly. express requests haggle careful 2919|101323|3834|1|2|2648.64|0.03|0.05|R|F|1993-12-28|1994-02-23|1994-01-18|COLLECT COD|TRUCK|re slyly. regular ideas detect furiousl 2919|120203|7740|2|49|59936.80|0.07|0.02|R|F|1993-12-16|1994-02-28|1993-12-19|COLLECT COD|FOB|hely final inst 2919|45595|3108|3|44|67785.96|0.07|0.07|A|F|1994-04-01|1994-01-12|1994-04-07|TAKE BACK RETURN|TRUCK|final ideas haggle carefully fluff 2919|101408|3919|4|44|62013.60|0.00|0.05|R|F|1994-02-04|1994-02-03|1994-03-02|TAKE BACK RETURN|AIR|es doze around the furiously 2944|119834|9835|1|44|81568.52|0.08|0.05|N|O|1997-12-25|1997-10-28|1998-01-21|COLLECT COD|AIR|ickly special theodolit 2944|41779|4284|2|44|75713.88|0.06|0.02|N|O|1997-10-28|1997-11-22|1997-11-10|NONE|SHIP|ickly. regular requests haggle. idea 2944|169657|7206|3|2|3453.30|0.06|0.07|N|O|1997-12-13|1997-12-01|1998-01-08|DELIVER IN PERSON|REG AIR|luffily expr 2944|16331|3835|4|23|28688.59|0.02|0.03|N|O|1998-01-12|1997-12-03|1998-01-17|TAKE BACK RETURN|MAIL| excuses? regular platelets e 2944|74247|9262|5|18|21982.32|0.10|0.01|N|O|1998-01-07|1997-10-26|1998-01-27|TAKE BACK RETURN|FOB| furiously slyl 2944|59267|6783|6|17|20846.42|0.00|0.03|N|O|1997-10-18|1997-11-27|1997-10-29|TAKE BACK RETURN|SHIP|slyly final dolphins sleep silent the 2944|89264|1773|7|7|8772.82|0.01|0.06|N|O|1997-10-30|1997-11-03|1997-11-03|DELIVER IN PERSON|FOB|fluffily blithely express pea 2945|58944|8945|1|37|70408.78|0.00|0.02|N|O|1996-02-10|1996-03-20|1996-02-12|COLLECT COD|SHIP|l instructions. regular, regular 2945|71252|3760|2|30|36697.50|0.05|0.01|N|O|1996-01-19|1996-02-11|1996-01-26|NONE|TRUCK|ular instructions 2945|126197|6198|3|28|34249.32|0.06|0.02|N|O|1996-03-17|1996-03-13|1996-04-15|COLLECT COD|FOB|le slyly along the eve 2945|187861|7862|4|34|66261.24|0.08|0.06|N|O|1996-02-03|1996-03-17|1996-02-29|COLLECT COD|REG AIR|at the unusual theodolite 2945|172280|9832|5|10|13522.80|0.09|0.05|N|O|1996-03-13|1996-03-10|1996-04-06|COLLECT COD|FOB|thely. final courts could hang qu 2945|96154|8664|6|45|51756.75|0.07|0.00|N|O|1996-03-01|1996-03-25|1996-03-08|TAKE BACK RETURN|MAIL|ainst the final packages 2945|51230|3736|7|47|55517.81|0.07|0.05|N|O|1996-01-05|1996-02-11|1996-01-12|DELIVER IN PERSON|MAIL|quests use 2946|9193|4194|1|25|27554.75|0.05|0.02|N|O|1996-05-06|1996-04-23|1996-05-16|DELIVER IN PERSON|SHIP|ic deposits. furiously 2946|93176|3177|2|48|56120.16|0.03|0.07|N|O|1996-06-02|1996-03-31|1996-06-16|COLLECT COD|TRUCK|oss the platelets. furi 2946|2968|5469|3|35|65483.60|0.03|0.00|N|O|1996-03-15|1996-04-02|1996-03-26|NONE|REG AIR| sublate along the fluffily iron 2947|9960|9961|1|37|69188.52|0.09|0.07|N|O|1995-08-09|1995-07-05|1995-08-20|DELIVER IN PERSON|RAIL|e accounts: expres 2947|185207|7726|2|10|12922.00|0.09|0.07|A|F|1995-06-07|1995-06-26|1995-06-08|NONE|MAIL|lly special 2948|117469|7470|1|48|71350.08|0.00|0.04|R|F|1994-08-29|1994-10-23|1994-09-23|NONE|TRUCK|unusual excuses use about the 2948|91710|1711|2|49|83383.79|0.04|0.07|R|F|1994-12-16|1994-11-08|1995-01-07|DELIVER IN PERSON|MAIL|ress requests. furiously blithe foxes 2949|20139|2642|1|4|4236.52|0.06|0.06|A|F|1994-06-07|1994-06-17|1994-07-04|TAKE BACK RETURN|REG AIR|gular pinto beans wake alongside of the reg 2949|69423|6942|2|50|69621.00|0.05|0.04|A|F|1994-08-04|1994-06-23|1994-08-17|TAKE BACK RETURN|FOB|gular courts cajole across t 2949|179465|4500|3|38|58689.48|0.02|0.06|R|F|1994-05-22|1994-05-25|1994-05-27|COLLECT COD|REG AIR|se slyly requests. carefull 2950|129486|9487|1|32|48495.36|0.01|0.05|N|O|1997-09-21|1997-08-25|1997-10-08|DELIVER IN PERSON|REG AIR|its wake carefully slyly final ideas. 2950|65786|5787|2|18|31532.04|0.10|0.01|N|O|1997-07-19|1997-08-29|1997-08-17|COLLECT COD|TRUCK|uests cajole furio 2950|52680|2681|3|14|22857.52|0.01|0.02|N|O|1997-07-29|1997-08-05|1997-07-31|TAKE BACK RETURN|MAIL|ccounts haggle carefully according 2950|186367|6368|4|45|65401.20|0.08|0.00|N|O|1997-09-05|1997-09-23|1997-09-11|NONE|FOB|ides the b 2950|60950|951|5|46|87903.70|0.02|0.05|N|O|1997-07-15|1997-09-30|1997-07-25|COLLECT COD|RAIL|to the regular accounts are slyly carefu 2950|173548|3549|6|27|43781.58|0.01|0.03|N|O|1997-10-01|1997-09-13|1997-10-08|NONE|TRUCK|are alongside of the carefully silent 2951|2079|7080|1|5|4905.35|0.03|0.03|N|O|1996-03-27|1996-04-16|1996-03-30|NONE|REG AIR|to beans wake ac 2951|135955|8469|2|24|47782.80|0.07|0.03|N|O|1996-03-24|1996-04-16|1996-04-08|NONE|SHIP| ironic multipliers. express, regular 2951|186046|3601|3|40|45281.60|0.02|0.07|N|O|1996-05-03|1996-04-20|1996-05-22|COLLECT COD|REG AIR|ial deposits wake fluffily about th 2951|72188|4696|4|21|24363.78|0.06|0.08|N|O|1996-04-12|1996-04-27|1996-04-14|DELIVER IN PERSON|REG AIR|nt instructions toward the f 2951|50131|5142|5|15|16216.95|0.07|0.00|N|O|1996-03-25|1996-04-23|1996-03-27|COLLECT COD|REG AIR|inal account 2951|137099|4639|6|18|20449.62|0.06|0.00|N|O|1996-04-04|1996-04-27|1996-04-06|COLLECT COD|FOB|ep about the final, even package 2976|8521|3522|1|32|45744.64|0.06|0.00|A|F|1994-01-26|1994-02-13|1994-02-10|NONE|MAIL|nding, ironic deposits sleep f 2976|3059|3060|2|24|23089.20|0.00|0.03|A|F|1994-03-19|1994-01-26|1994-04-18|COLLECT COD|TRUCK|ronic pinto beans. slyly bol 2976|9775|4776|3|35|58966.95|0.10|0.07|R|F|1993-12-19|1994-02-14|1994-01-11|NONE|RAIL|boost slyly about the regular, regular re 2976|81070|6087|4|22|23123.54|0.00|0.04|A|F|1994-02-08|1994-03-03|1994-02-12|TAKE BACK RETURN|FOB|ncies kindle furiously. carefull 2976|133261|8288|5|13|16825.38|0.00|0.06|A|F|1994-02-06|1994-02-02|1994-02-19|NONE|FOB| furiously final courts boost 2976|108348|859|6|30|40690.20|0.08|0.03|R|F|1994-03-27|1994-02-01|1994-04-26|TAKE BACK RETURN|RAIL|c ideas! unusual 2977|69769|7288|1|25|43469.00|0.03|0.07|N|O|1996-09-21|1996-10-06|1996-10-13|TAKE BACK RETURN|RAIL|furiously pe 2978|89697|9698|1|29|48914.01|0.00|0.08|A|F|1995-06-03|1995-07-25|1995-06-06|NONE|SHIP|ecial ideas promise slyly 2978|126998|9511|2|42|85049.58|0.01|0.06|N|O|1995-08-19|1995-07-18|1995-09-07|DELIVER IN PERSON|MAIL|ial requests nag blithely alongside of th 2978|42972|485|3|26|49789.22|0.07|0.05|N|O|1995-07-29|1995-07-22|1995-08-20|COLLECT COD|REG AIR|as haggle against the carefully express dep 2978|27039|4546|4|7|6762.21|0.00|0.00|N|O|1995-07-18|1995-07-03|1995-07-23|NONE|FOB|. final ideas are blithe 2978|28403|5910|5|33|43936.20|0.09|0.03|R|F|1995-05-06|1995-07-23|1995-05-16|COLLECT COD|FOB|s. blithely unusual pack 2978|167063|9580|6|4|4520.24|0.08|0.04|N|O|1995-07-06|1995-07-31|1995-07-19|COLLECT COD|AIR|ffily unusual 2979|8022|5523|1|8|7440.16|0.00|0.08|N|O|1996-06-18|1996-05-21|1996-07-06|COLLECT COD|REG AIR|st blithely; blithely regular gifts dazz 2979|10662|663|2|47|73915.02|0.05|0.00|N|O|1996-03-25|1996-05-13|1996-04-04|TAKE BACK RETURN|SHIP|iously unusual dependencies wake across 2979|187872|5427|3|35|68595.45|0.04|0.03|N|O|1996-05-25|1996-06-11|1996-06-24|DELIVER IN PERSON|MAIL|old ideas beneath the blit 2979|164037|6554|4|28|30828.84|0.05|0.08|N|O|1996-06-04|1996-04-23|1996-06-24|DELIVER IN PERSON|FOB|ing, regular pinto beans. blithel 2980|36367|8871|1|2|2606.72|0.09|0.03|N|O|1996-11-18|1996-10-22|1996-11-27|TAKE BACK RETURN|SHIP|enly across the special, pending packag 2980|9576|7077|2|48|71307.36|0.04|0.05|N|O|1996-09-25|1996-12-09|1996-10-12|NONE|REG AIR|totes. regular pinto 2980|132090|4604|3|27|30296.43|0.08|0.08|N|O|1996-12-08|1996-12-03|1996-12-14|NONE|REG AIR| theodolites cajole blithely sl 2980|24644|7147|4|49|76863.36|0.03|0.02|N|O|1996-10-04|1996-12-04|1996-10-06|NONE|RAIL|hy packages sleep quic 2980|186048|6049|5|24|27216.96|0.05|0.04|N|O|1997-01-12|1996-10-27|1997-01-14|NONE|MAIL|elets. fluffily regular in 2980|108666|3687|6|43|72010.38|0.01|0.01|N|O|1996-12-07|1996-11-10|1997-01-02|COLLECT COD|AIR|sts. slyly regu 2981|13505|1009|1|17|24114.50|0.03|0.05|N|O|1998-10-17|1998-10-02|1998-10-21|DELIVER IN PERSON|RAIL|, unusual packages x-ray. furious 2981|175430|2982|2|8|12043.44|0.06|0.03|N|O|1998-08-21|1998-09-28|1998-09-05|DELIVER IN PERSON|MAIL|ng to the f 2981|36615|4125|3|14|21722.54|0.03|0.07|N|O|1998-08-30|1998-10-04|1998-09-04|DELIVER IN PERSON|MAIL|kages detect furiously express requests. 2982|111936|4448|1|21|40906.53|0.00|0.01|A|F|1995-04-03|1995-06-08|1995-04-18|DELIVER IN PERSON|AIR|ironic deposits. furiously ex 2982|98071|3090|2|13|13897.91|0.02|0.08|R|F|1995-03-31|1995-05-07|1995-04-18|TAKE BACK RETURN|RAIL|regular deposits unwind alongside 2982|69479|6998|3|21|30417.87|0.01|0.01|R|F|1995-04-19|1995-06-03|1995-04-28|COLLECT COD|SHIP|egular ideas use furiously? bl 2983|162174|2175|1|44|54391.48|0.03|0.06|R|F|1992-02-09|1992-03-07|1992-03-09|TAKE BACK RETURN|AIR|ly regular instruct 2983|48039|5552|2|11|10857.33|0.09|0.06|A|F|1992-04-29|1992-02-27|1992-05-26|NONE|MAIL|aids integrate s 3008|131565|6592|1|8|12772.48|0.10|0.04|N|O|1995-12-06|1996-01-12|1995-12-22|TAKE BACK RETURN|FOB|yly ironic foxes. regular requests h 3008|199716|4755|2|31|56287.01|0.05|0.06|N|O|1995-12-14|1995-12-11|1995-12-31|TAKE BACK RETURN|AIR| bold packages. quic 3008|23054|3055|3|40|39082.00|0.01|0.03|N|O|1995-12-18|1996-01-06|1996-01-11|COLLECT COD|AIR|esias. theodolites detect blithely 3008|59018|9019|4|48|46896.48|0.07|0.06|N|O|1996-01-23|1996-01-07|1996-02-09|COLLECT COD|SHIP|ld theodolites. fluffily bold theodolit 3008|104156|9177|5|31|35964.65|0.03|0.02|N|O|1995-12-01|1996-01-20|1995-12-28|COLLECT COD|RAIL|nts use thinly around the carefully iro 3009|44862|9871|1|48|86729.28|0.10|0.02|N|O|1997-03-19|1997-05-13|1997-04-11|TAKE BACK RETURN|TRUCK| dependencies sleep quickly a 3009|184598|7117|2|38|63938.42|0.00|0.01|N|O|1997-05-01|1997-04-10|1997-05-17|TAKE BACK RETURN|AIR|nal packages should haggle slyly. quickl 3009|129770|7307|3|26|46794.02|0.08|0.02|N|O|1997-05-15|1997-05-10|1997-06-13|TAKE BACK RETURN|SHIP|uriously specia 3010|137941|455|1|23|45515.62|0.04|0.00|N|O|1996-03-08|1996-02-29|1996-03-27|NONE|TRUCK|ounts. pendin 3010|173015|5533|2|22|23936.22|0.09|0.06|N|O|1996-03-06|1996-04-06|1996-03-18|COLLECT COD|REG AIR| final deposit 3010|57050|9556|3|24|24169.20|0.04|0.07|N|O|1996-05-09|1996-03-14|1996-05-15|DELIVER IN PERSON|RAIL|ar, even reques 3010|23675|1182|4|28|44762.76|0.09|0.06|N|O|1996-03-05|1996-03-28|1996-04-03|DELIVER IN PERSON|FOB|ake carefully carefully even request 3010|103499|3500|5|9|13522.41|0.02|0.02|N|O|1996-04-28|1996-03-17|1996-05-18|NONE|SHIP|inal packages. quickly even pinto 3010|91955|1956|6|38|73984.10|0.05|0.07|N|O|1996-04-15|1996-03-16|1996-04-21|DELIVER IN PERSON|RAIL|accounts ar 3011|197592|112|1|5|8447.95|0.02|0.04|R|F|1992-04-21|1992-02-23|1992-05-15|NONE|TRUCK|nusual sentiments. carefully bold idea 3011|122709|2710|2|42|72731.40|0.05|0.00|A|F|1992-02-01|1992-03-18|1992-02-29|NONE|TRUCK|osits haggle quickly pending, 3012|194274|6794|1|49|67045.23|0.00|0.00|A|F|1993-08-07|1993-07-01|1993-08-08|NONE|MAIL| quickly furious packages. silently unusua 3012|160226|227|2|37|47590.14|0.06|0.03|A|F|1993-08-16|1993-06-07|1993-08-24|TAKE BACK RETURN|REG AIR|uickly permanent packages sleep caref 3013|93613|6123|1|31|49804.91|0.08|0.08|N|O|1997-05-03|1997-04-05|1997-05-25|NONE|AIR|y furious depen 3013|138866|6406|2|30|57145.80|0.05|0.06|N|O|1997-05-02|1997-03-09|1997-05-12|TAKE BACK RETURN|MAIL|ronic packages. slyly even 3013|119628|7162|3|35|57666.70|0.00|0.03|N|O|1997-04-02|1997-05-04|1997-04-16|COLLECT COD|MAIL|ely accord 3013|180929|3448|4|17|34168.64|0.01|0.07|N|O|1997-02-26|1997-05-02|1997-03-27|DELIVER IN PERSON|SHIP|fully unusual account 3013|59896|4907|5|20|37117.80|0.00|0.04|N|O|1997-05-06|1997-03-18|1997-05-12|COLLECT COD|RAIL|unts boost regular ideas. slyly pe 3013|71557|4065|6|19|29042.45|0.08|0.07|N|O|1997-05-11|1997-04-18|1997-05-15|COLLECT COD|REG AIR|fluffily pending packages nag furiously al 3014|162368|2369|1|36|51492.96|0.05|0.03|A|F|1992-11-16|1993-01-20|1992-11-28|TAKE BACK RETURN|FOB|ding accounts boost fu 3014|105025|46|2|36|37080.72|0.00|0.08|R|F|1992-12-28|1992-12-29|1993-01-24|COLLECT COD|MAIL|iously ironic r 3014|150885|3401|3|48|92922.24|0.06|0.02|A|F|1992-12-19|1993-01-08|1992-12-25|DELIVER IN PERSON|REG AIR|y pending theodolites wake. reg 3014|113767|8790|4|14|24930.64|0.10|0.02|R|F|1992-11-19|1993-01-01|1992-12-17|DELIVER IN PERSON|SHIP|. slyly brave platelets nag. careful, 3014|74210|6718|5|28|33157.88|0.02|0.08|R|F|1993-01-09|1992-12-18|1993-01-10|TAKE BACK RETURN|FOB|es are. final braids nag slyly. fluff 3014|37443|9947|6|30|41413.20|0.04|0.01|R|F|1993-02-28|1993-01-02|1993-03-20|TAKE BACK RETURN|AIR| final foxes. 3015|2643|7644|1|5|7728.20|0.09|0.00|A|F|1993-01-10|1992-12-02|1993-01-19|TAKE BACK RETURN|RAIL| the furiously pendi 3015|17510|12|2|17|24267.67|0.03|0.01|R|F|1992-10-16|1992-11-20|1992-10-28|COLLECT COD|AIR|s above the fluffily final t 3015|90543|5562|3|23|35271.42|0.03|0.05|A|F|1992-12-03|1992-11-19|1992-12-23|DELIVER IN PERSON|FOB|s are slyly carefully special pinto bea 3015|155005|5006|4|7|7420.00|0.10|0.03|A|F|1992-12-07|1992-12-17|1992-12-30|DELIVER IN PERSON|REG AIR| after the evenly special packages ca 3015|164614|7131|5|42|70501.62|0.04|0.02|R|F|1993-01-21|1992-11-07|1993-02-11|DELIVER IN PERSON|AIR|encies haggle furious 3015|65123|5124|6|18|19586.16|0.02|0.03|R|F|1992-10-10|1992-11-19|1992-10-18|TAKE BACK RETURN|MAIL|equests wake fluffil 3040|15347|2851|1|18|22722.12|0.08|0.04|R|F|1993-06-25|1993-07-06|1993-07-19|TAKE BACK RETURN|SHIP|ly thin accou 3040|132715|255|2|9|15729.39|0.00|0.01|A|F|1993-06-12|1993-05-16|1993-06-14|NONE|RAIL|ges. pending packages wake. requests 3040|125608|633|3|30|49008.00|0.01|0.01|A|F|1993-08-06|1993-05-18|1993-08-19|NONE|MAIL|x furiously bold packages. expres 3040|82065|7082|4|14|14658.84|0.05|0.04|A|F|1993-05-13|1993-05-18|1993-05-19|TAKE BACK RETURN|REG AIR| haggle carefully. express hocke 3040|51516|1517|5|43|63102.93|0.04|0.04|R|F|1993-05-21|1993-05-25|1993-05-26|NONE|MAIL|sts nag slyly alongside of the depos 3040|17878|2881|6|10|17958.70|0.08|0.04|R|F|1993-05-16|1993-06-24|1993-06-11|DELIVER IN PERSON|MAIL|ely regular foxes haggle dari 3041|180636|8191|1|5|8583.15|0.07|0.04|N|O|1997-07-20|1997-07-15|1997-08-17|COLLECT COD|FOB|posits dazzle special p 3041|145837|866|2|9|16945.47|0.03|0.03|N|O|1997-06-29|1997-08-14|1997-07-19|COLLECT COD|AIR|iously across the silent pinto beans. furi 3041|67448|2461|3|9|12738.96|0.09|0.06|N|O|1997-08-28|1997-07-23|1997-09-16|TAKE BACK RETURN|FOB|scapades after the special 3042|104915|2446|1|30|57597.30|0.08|0.06|A|F|1995-01-12|1995-02-15|1995-01-24|DELIVER IN PERSON|SHIP|the requests detect fu 3042|101459|1460|2|28|40892.60|0.05|0.03|A|F|1994-11-24|1995-01-02|1994-12-06|TAKE BACK RETURN|MAIL|ng the furiously r 3042|13664|6166|3|34|53640.44|0.04|0.00|R|F|1994-12-11|1995-02-03|1994-12-21|TAKE BACK RETURN|TRUCK|can wake after the enticingly stealthy i 3042|47338|2347|4|19|24421.27|0.02|0.01|A|F|1995-03-05|1995-01-24|1995-03-17|COLLECT COD|TRUCK|e carefully. regul 3043|45387|396|1|23|30644.74|0.07|0.04|R|F|1992-05-08|1992-07-22|1992-05-18|COLLECT COD|TRUCK|uickly above the pending, 3043|5894|3395|2|15|26998.35|0.03|0.05|A|F|1992-05-27|1992-06-03|1992-06-09|COLLECT COD|FOB|usly furiously 3043|59231|9232|3|42|49989.66|0.10|0.07|R|F|1992-07-15|1992-06-19|1992-07-23|NONE|MAIL|ide of the un 3043|90453|454|4|5|7217.25|0.10|0.01|A|F|1992-05-22|1992-07-02|1992-06-20|TAKE BACK RETURN|TRUCK|ake blithely re 3044|100992|993|1|10|19929.90|0.07|0.08|N|O|1996-07-13|1996-05-06|1996-07-21|TAKE BACK RETURN|REG AIR| slyly ironic requests. s 3044|167015|9532|2|3|3246.03|0.06|0.02|N|O|1996-07-27|1996-05-26|1996-08-15|TAKE BACK RETURN|AIR|ecoys haggle furiously pending requests. 3044|18478|980|3|47|65634.09|0.09|0.00|N|O|1996-05-24|1996-06-22|1996-05-30|NONE|REG AIR|ly around the car 3045|87670|7671|1|41|67964.47|0.05|0.01|N|O|1995-09-30|1995-11-24|1995-10-03|TAKE BACK RETURN|MAIL|ely final foxes. carefully ironic pinto b 3045|68109|3122|2|48|51700.80|0.02|0.03|N|O|1995-10-01|1995-12-16|1995-10-10|TAKE BACK RETURN|MAIL|ole quickly outside th 3046|73235|3236|1|44|53162.12|0.03|0.03|N|O|1996-03-03|1996-02-25|1996-04-01|NONE|AIR| are quickly. blithe 3046|53645|3646|2|46|73537.44|0.03|0.08|N|O|1996-03-22|1996-02-28|1996-04-07|TAKE BACK RETURN|AIR|sits sleep furious 3046|1537|9038|3|31|44594.43|0.03|0.07|N|O|1996-03-24|1996-01-30|1996-03-26|NONE|RAIL|y pending somas alongside of the slyly iro 3047|103994|3995|1|17|33965.83|0.08|0.02|N|O|1997-06-14|1997-04-20|1997-06-23|COLLECT COD|FOB|onic instruction 3047|13957|8960|2|23|43031.85|0.00|0.04|N|O|1997-05-20|1997-06-14|1997-05-28|TAKE BACK RETURN|REG AIR| slyly ironi 3072|56531|4047|1|6|8925.18|0.09|0.05|R|F|1994-02-09|1994-03-24|1994-02-28|DELIVER IN PERSON|REG AIR|gular requests abov 3072|107152|2173|2|36|41729.40|0.07|0.02|R|F|1994-04-14|1994-04-22|1994-05-06|COLLECT COD|AIR| theodolites. blithely e 3072|96715|6716|3|7|11981.97|0.04|0.07|R|F|1994-05-09|1994-03-31|1994-05-19|COLLECT COD|TRUCK|uests. ironic, ironic depos 3072|82745|5254|4|39|67381.86|0.05|0.08|A|F|1994-05-27|1994-04-20|1994-06-14|COLLECT COD|MAIL|es; slyly spe 3072|87380|7381|5|1|1367.38|0.01|0.08|R|F|1994-02-26|1994-03-14|1994-03-19|NONE|AIR| slyly ironic attainments. car 3073|193992|9031|1|16|33375.84|0.07|0.01|R|F|1994-03-02|1994-03-23|1994-03-31|DELIVER IN PERSON|AIR|n requests. ironi 3073|21735|9242|2|47|77866.31|0.09|0.00|R|F|1994-03-26|1994-02-12|1994-04-21|NONE|REG AIR|eposits. fluffily 3073|86050|8559|3|10|10360.50|0.03|0.00|R|F|1994-02-11|1994-03-24|1994-02-26|COLLECT COD|FOB| furiously caref 3073|28574|1077|4|14|21035.98|0.09|0.07|R|F|1994-03-24|1994-04-01|1994-04-07|NONE|RAIL|ilently quiet epitaphs. 3073|40433|7946|5|25|34335.75|0.00|0.07|R|F|1994-04-14|1994-03-07|1994-04-22|NONE|TRUCK|nag asymptotes. pinto beans sleep 3073|146763|6764|6|39|70580.64|0.09|0.02|R|F|1994-05-01|1994-02-16|1994-05-12|DELIVER IN PERSON|AIR|lar excuses across the furiously even 3073|43770|3771|7|11|18851.47|0.08|0.07|A|F|1994-05-01|1994-03-06|1994-05-08|COLLECT COD|SHIP|instructions sleep according to the 3074|36636|1643|1|50|78631.50|0.08|0.08|A|F|1993-01-31|1992-12-15|1993-02-20|NONE|AIR|furiously pending requests haggle s 3074|138309|5849|2|39|52544.70|0.03|0.00|R|F|1992-12-08|1993-01-28|1992-12-09|DELIVER IN PERSON|TRUCK|iously throu 3075|8905|6406|1|39|70742.10|0.02|0.03|A|F|1994-06-10|1994-06-21|1994-06-20|NONE|FOB|ing deposits nag 3075|51164|3670|2|2|2230.32|0.07|0.08|R|F|1994-06-14|1994-06-10|1994-06-25|TAKE BACK RETURN|AIR|. unusual, unusual accounts haggle furious 3076|84436|4437|1|44|62498.92|0.00|0.05|A|F|1993-09-14|1993-10-04|1993-09-17|TAKE BACK RETURN|FOB| instructions h 3076|105266|287|2|22|27967.72|0.08|0.00|A|F|1993-09-05|1993-09-10|1993-09-27|NONE|REG AIR|packages wake furiou 3076|4172|6673|3|31|33361.27|0.06|0.06|A|F|1993-08-10|1993-09-17|1993-08-17|TAKE BACK RETURN|SHIP|regular depos 3077|71785|4293|1|25|43919.50|0.06|0.01|N|O|1997-09-14|1997-10-16|1997-10-06|NONE|TRUCK|lent account 3077|90110|2620|2|40|44004.40|0.05|0.06|N|O|1997-10-22|1997-09-19|1997-11-19|DELIVER IN PERSON|AIR|to the enticing packag 3077|77820|2835|3|13|23371.66|0.03|0.07|N|O|1997-09-09|1997-10-15|1997-09-19|NONE|TRUCK|luffily close depende 3077|114603|2137|4|23|37204.80|0.03|0.02|N|O|1997-11-05|1997-09-16|1997-11-20|NONE|MAIL|lly. fluffily pending dinos across 3078|131450|1451|1|25|37036.25|0.01|0.03|A|F|1993-04-22|1993-05-01|1993-04-28|TAKE BACK RETURN|AIR|express dinos. carefully ironic 3078|77158|9666|2|21|23838.15|0.09|0.07|A|F|1993-03-20|1993-03-21|1993-04-01|COLLECT COD|AIR|e fluffily. 3079|69026|6545|1|20|19900.40|0.05|0.00|N|O|1997-10-18|1997-10-26|1997-11-14|NONE|RAIL|ets are according to the quickly dari 3079|116489|9001|2|38|57208.24|0.08|0.07|N|O|1997-11-07|1997-11-25|1997-12-06|NONE|RAIL|e carefully regular realms 3079|16662|6663|3|40|63146.40|0.02|0.08|N|O|1997-09-26|1997-12-11|1997-10-09|NONE|RAIL|ide of the pending, special deposi 3079|23216|3217|4|2|2278.42|0.00|0.08|N|O|1998-01-05|1997-11-17|1998-01-28|NONE|FOB|ly busy requests believ 3079|187516|35|5|2|3207.02|0.10|0.00|N|O|1997-12-27|1997-10-25|1998-01-08|COLLECT COD|SHIP|y regular asymptotes doz 3079|165391|2940|6|46|66993.94|0.00|0.00|N|O|1997-11-19|1997-11-04|1997-11-25|DELIVER IN PERSON|REG AIR|es. final, regula 3104|50589|5600|1|20|30791.60|0.01|0.08|A|F|1993-12-31|1993-11-24|1994-01-12|DELIVER IN PERSON|REG AIR|s are. furiously s 3104|47520|2529|2|47|68973.44|0.02|0.05|A|F|1993-12-25|1993-11-02|1994-01-12|COLLECT COD|RAIL|ily daring acc 3104|62977|2978|3|11|21339.67|0.02|0.03|A|F|1993-10-05|1993-11-30|1993-10-27|NONE|TRUCK| special deposits u 3104|37868|7869|4|26|46952.36|0.02|0.08|R|F|1994-01-02|1993-12-05|1994-01-31|TAKE BACK RETURN|TRUCK|es boost carefully. slyly 3105|183486|8523|1|11|17264.28|0.01|0.06|N|O|1997-02-07|1997-02-09|1997-03-01|NONE|FOB|kly bold depths caj 3105|44436|4437|2|9|12423.87|0.08|0.08|N|O|1996-12-25|1997-02-04|1997-01-09|COLLECT COD|SHIP|es wake among t 3105|24472|9477|3|48|67030.56|0.02|0.05|N|O|1997-02-28|1997-01-31|1997-03-18|DELIVER IN PERSON|REG AIR|ending platelets wake carefully ironic inst 3105|90456|7984|4|23|33268.35|0.04|0.07|N|O|1997-03-08|1996-12-14|1997-03-18|COLLECT COD|REG AIR| detect slyly. blithely unusual requests ar 3105|89347|4364|5|8|10690.72|0.07|0.07|N|O|1996-12-28|1996-12-28|1997-01-25|NONE|FOB|s. blithely unusual ideas was after 3105|46261|3774|6|30|36217.80|0.08|0.05|N|O|1997-03-03|1997-02-03|1997-03-05|NONE|FOB|ess accounts boost among t 3106|85699|8208|1|22|37063.18|0.03|0.02|N|O|1997-02-28|1997-02-12|1997-03-03|DELIVER IN PERSON|FOB|structions atop the blithely 3106|135816|3356|2|49|90738.69|0.06|0.06|N|O|1997-02-27|1997-03-11|1997-03-12|NONE|TRUCK|lets. quietly regular courts 3106|51764|6775|3|42|72061.92|0.09|0.07|N|O|1997-04-05|1997-03-17|1997-04-22|COLLECT COD|REG AIR|nstructions wake. furiously 3106|195649|3207|4|6|10467.84|0.10|0.07|N|O|1997-02-02|1997-04-11|1997-02-27|COLLECT COD|REG AIR|symptotes. slyly bold platelets cajol 3106|64580|9593|5|16|24713.28|0.09|0.08|N|O|1997-02-25|1997-04-10|1997-03-16|NONE|AIR|sits wake slyl 3107|148150|665|1|16|19170.40|0.05|0.04|N|O|1997-08-30|1997-10-20|1997-09-20|TAKE BACK RETURN|REG AIR|regular pinto beans. ironic ideas haggle 3107|141690|1691|2|35|60609.15|0.05|0.06|N|O|1997-08-27|1997-11-19|1997-09-14|COLLECT COD|TRUCK|ets doubt furiously final ideas. final 3107|169733|2250|3|23|41462.79|0.03|0.06|N|O|1997-12-10|1997-11-11|1997-12-14|TAKE BACK RETURN|SHIP|atelets must ha 3107|86080|8589|4|27|28784.16|0.00|0.08|N|O|1997-11-15|1997-10-31|1997-11-28|DELIVER IN PERSON|FOB|furiously final 3108|108989|1500|1|37|73925.26|0.06|0.04|A|F|1993-10-16|1993-10-01|1993-11-09|DELIVER IN PERSON|RAIL| final requests. 3108|165850|3399|2|26|49812.10|0.08|0.05|A|F|1993-11-12|1993-10-05|1993-12-09|COLLECT COD|TRUCK| slyly slow foxes wake furious 3109|17514|16|1|32|45808.32|0.08|0.03|A|F|1993-09-05|1993-10-06|1993-09-18|DELIVER IN PERSON|FOB|ecial orbits are furiou 3109|144736|2279|2|49|87255.77|0.08|0.06|R|F|1993-10-24|1993-09-30|1993-11-21|TAKE BACK RETURN|AIR| even pearls. furiously pending 3109|175265|2817|3|43|57631.18|0.04|0.07|R|F|1993-09-29|1993-09-06|1993-10-13|COLLECT COD|MAIL|ding to the foxes. 3109|78107|8108|4|26|28212.60|0.01|0.05|R|F|1993-11-16|1993-10-18|1993-12-06|TAKE BACK RETURN|TRUCK| sleep slyly according to t 3109|142097|9640|5|50|56954.50|0.01|0.08|A|F|1993-09-17|1993-10-16|1993-10-11|NONE|FOB| regular packages boost blithely even, re 3109|14933|7435|6|10|18479.30|0.10|0.04|A|F|1993-10-26|1993-10-03|1993-11-09|NONE|TRUCK|sits haggle carefully. regular, unusual ac 3110|88455|3472|1|1|1443.45|0.02|0.07|A|F|1995-01-15|1995-01-20|1995-01-30|DELIVER IN PERSON|REG AIR|c theodolites a 3110|56453|1464|2|31|43692.95|0.01|0.06|R|F|1995-03-31|1995-03-07|1995-04-21|TAKE BACK RETURN|REG AIR|en deposits. ironic 3110|2097|9598|3|34|33969.06|0.02|0.02|A|F|1995-02-23|1995-01-27|1995-03-09|TAKE BACK RETURN|FOB|ly pending requests ha 3110|39004|9005|4|16|15088.00|0.04|0.04|A|F|1995-01-10|1995-02-06|1995-01-26|NONE|MAIL|across the regular acco 3110|139664|7204|5|39|66442.74|0.09|0.01|A|F|1995-02-09|1995-01-21|1995-02-21|NONE|MAIL|side of the blithely unusual courts. slyly 3111|136959|1986|1|22|43910.90|0.06|0.05|N|O|1995-09-21|1995-11-09|1995-10-17|COLLECT COD|REG AIR|quests. regular dolphins against the 3111|57194|4710|2|30|34535.70|0.06|0.05|N|O|1995-10-05|1995-11-15|1995-11-01|TAKE BACK RETURN|TRUCK|eas are furiously slyly special deposits. 3111|51332|1333|3|10|12833.30|0.02|0.02|N|O|1995-11-10|1995-11-02|1995-12-04|NONE|FOB|ng the slyly ironic inst 3111|131080|6107|4|31|34443.48|0.00|0.08|N|O|1995-10-26|1995-09-26|1995-11-02|TAKE BACK RETURN|MAIL|kages detect express attainments 3111|53483|999|5|14|20110.72|0.05|0.04|N|O|1995-10-17|1995-10-19|1995-10-19|TAKE BACK RETURN|SHIP|re. pinto 3111|85509|8018|6|5|7472.50|0.03|0.08|N|O|1995-08-30|1995-10-16|1995-09-04|DELIVER IN PERSON|TRUCK|. carefully even ideas 3111|147305|7306|7|41|55444.30|0.09|0.05|N|O|1995-11-22|1995-11-01|1995-12-01|TAKE BACK RETURN|FOB|fily slow ideas. 3136|141341|6370|1|30|41470.20|0.02|0.08|R|F|1994-08-13|1994-10-02|1994-09-02|TAKE BACK RETURN|RAIL|leep blithel 3136|102634|2635|2|7|11456.41|0.05|0.07|A|F|1994-10-08|1994-09-14|1994-10-11|TAKE BACK RETURN|SHIP|ic pinto beans are slyly. f 3136|157204|2235|3|43|54231.60|0.00|0.07|A|F|1994-09-05|1994-09-25|1994-09-11|NONE|RAIL|. special theodolites ha 3136|115183|2717|4|26|31152.68|0.04|0.05|A|F|1994-10-13|1994-11-07|1994-11-05|TAKE BACK RETURN|AIR|eep fluffily. daringly silent attainments d 3136|66751|6752|5|2|3435.50|0.08|0.07|R|F|1994-11-21|1994-11-03|1994-11-26|DELIVER IN PERSON|TRUCK|? special, silent 3136|79827|9828|6|29|52397.78|0.08|0.07|A|F|1994-11-16|1994-10-03|1994-12-14|NONE|FOB|latelets. final 3137|2357|2358|1|6|7556.10|0.02|0.02|N|O|1995-09-19|1995-10-23|1995-10-16|NONE|SHIP|ly express as 3137|5178|2679|2|4|4332.68|0.06|0.04|N|O|1995-10-01|1995-09-11|1995-10-30|COLLECT COD|RAIL|posits wake. silent excuses boost about 3138|92417|4927|1|7|9865.87|0.05|0.05|R|F|1994-03-04|1994-03-14|1994-03-20|NONE|AIR|lithely quickly even packages. packages 3138|43554|3555|2|27|40433.85|0.09|0.01|R|F|1994-03-24|1994-03-23|1994-04-18|DELIVER IN PERSON|FOB|counts cajole fluffily carefully special i 3138|196512|6513|3|32|51472.32|0.00|0.01|R|F|1994-02-24|1994-05-07|1994-02-28|TAKE BACK RETURN|MAIL|inal foxes affix slyly. fluffily regul 3138|171358|1359|4|38|54315.30|0.07|0.04|R|F|1994-02-21|1994-03-21|1994-03-13|COLLECT COD|FOB|lithely fluffily un 3138|9594|9595|5|12|18043.08|0.09|0.02|A|F|1994-03-04|1994-04-11|1994-03-21|COLLECT COD|FOB|. bold pinto beans haggl 3138|43107|8116|6|25|26252.50|0.05|0.08|A|F|1994-05-19|1994-04-07|1994-06-17|TAKE BACK RETURN|AIR|dolites around the carefully busy the 3139|39310|1814|1|46|57468.26|0.08|0.03|R|F|1992-04-28|1992-03-04|1992-05-19|TAKE BACK RETURN|FOB|of the unusual, unusual re 3140|6539|4040|1|21|30356.13|0.08|0.02|R|F|1992-04-12|1992-05-31|1992-04-21|NONE|REG AIR| furiously sly excuses according to the 3140|88674|1183|2|10|16626.70|0.07|0.01|A|F|1992-05-30|1992-05-09|1992-06-09|COLLECT COD|RAIL|accounts. expres 3140|132668|2669|3|28|47618.48|0.06|0.00|R|F|1992-06-08|1992-07-07|1992-07-08|TAKE BACK RETURN|SHIP|lar ideas. slyly ironic d 3141|176416|1451|1|32|47757.12|0.06|0.00|N|O|1995-11-21|1995-12-18|1995-11-26|DELIVER IN PERSON|FOB|oxes are quickly about t 3141|9358|6859|2|37|46891.95|0.10|0.05|N|O|1996-01-24|1995-12-16|1996-01-27|DELIVER IN PERSON|AIR|press pinto beans. bold accounts boost b 3141|78185|5707|3|9|10468.62|0.09|0.02|N|O|1995-11-11|1995-12-10|1995-12-02|DELIVER IN PERSON|MAIL|uickly ironic, pendi 3141|45956|965|4|47|89391.65|0.03|0.01|N|O|1995-11-29|1996-01-13|1995-12-10|TAKE BACK RETURN|TRUCK| are slyly pi 3142|119225|4248|1|15|18663.30|0.03|0.08|R|F|1992-08-15|1992-08-18|1992-08-22|DELIVER IN PERSON|AIR|instructions are. ironic packages doz 3143|89831|7356|1|22|40058.26|0.02|0.00|A|F|1993-05-11|1993-03-26|1993-05-20|TAKE BACK RETURN|MAIL|l, special instructions nag 3143|182257|9812|2|40|53570.00|0.03|0.08|A|F|1993-05-07|1993-03-29|1993-05-17|COLLECT COD|FOB|sly unusual theodolites. slyly ev 3143|182464|4983|3|22|34022.12|0.05|0.03|A|F|1993-03-18|1993-05-09|1993-04-14|DELIVER IN PERSON|MAIL|beans. fluf 3143|65651|5652|4|46|74365.90|0.05|0.08|R|F|1993-04-19|1993-03-21|1993-05-05|COLLECT COD|REG AIR|low forges haggle. even packages use bli 3168|59901|2407|1|46|85601.40|0.08|0.08|R|F|1992-02-14|1992-03-02|1992-03-02|TAKE BACK RETURN|SHIP|y across the express accounts. fluff 3168|153658|3659|2|1|1711.65|0.06|0.08|A|F|1992-05-27|1992-03-12|1992-06-09|TAKE BACK RETURN|SHIP|pinto beans. slyly regular courts haggle 3168|127053|9566|3|13|14040.65|0.09|0.02|A|F|1992-03-05|1992-04-29|1992-03-15|NONE|SHIP|ironic somas haggle quick 3168|164440|1989|4|11|16548.84|0.02|0.05|R|F|1992-04-12|1992-03-17|1992-05-12|COLLECT COD|SHIP|ously furious dependenc 3169|191052|3572|1|12|13716.60|0.01|0.04|R|F|1994-01-05|1994-03-18|1994-01-21|COLLECT COD|REG AIR| regular d 3169|199869|4908|2|17|33470.62|0.05|0.04|R|F|1994-03-02|1994-01-21|1994-03-03|DELIVER IN PERSON|TRUCK|usly regular packages. ironi 3169|187322|7323|3|12|16911.84|0.08|0.07|A|F|1994-04-18|1994-03-12|1994-05-08|TAKE BACK RETURN|FOB|atelets. pac 3169|104999|5000|4|26|52103.74|0.10|0.04|R|F|1994-04-08|1994-03-21|1994-04-29|NONE|TRUCK|ter the regular ideas. slyly iro 3169|107704|7705|5|6|10270.20|0.09|0.01|A|F|1994-03-24|1994-02-22|1994-04-04|TAKE BACK RETURN|AIR|ular instructions. ca 3169|176053|6054|6|46|51936.30|0.02|0.07|A|F|1994-02-01|1994-01-22|1994-02-24|DELIVER IN PERSON|RAIL|thely bold theodolites are fl 3170|39883|7393|1|12|21874.56|0.03|0.03|N|O|1998-02-12|1998-01-17|1998-02-24|NONE|TRUCK|ing accounts along the speci 3170|99060|1570|2|21|22240.26|0.01|0.00|N|O|1997-12-09|1998-01-31|1997-12-21|DELIVER IN PERSON|MAIL|o beans. carefully final requests dou 3170|88383|3400|3|27|37027.26|0.00|0.05|N|O|1998-02-25|1998-01-29|1998-02-27|COLLECT COD|AIR|efully bold foxes. regular, ev 3170|40846|847|4|34|60752.56|0.05|0.04|N|O|1998-02-01|1998-01-11|1998-02-20|TAKE BACK RETURN|TRUCK|s about the fluffily final de 3170|89767|9768|5|32|56216.32|0.02|0.04|N|O|1997-11-24|1997-12-12|1997-12-15|COLLECT COD|SHIP|ggle about the furiously r 3170|109150|4171|6|43|49843.45|0.08|0.05|N|O|1998-01-05|1998-01-04|1998-01-14|NONE|REG AIR|. express dolphins use sly 3170|83873|3874|7|26|48278.62|0.10|0.05|N|O|1998-02-12|1997-12-22|1998-02-28|COLLECT COD|TRUCK|s engage furiously. 3171|46382|8887|1|34|45164.92|0.04|0.00|A|F|1993-05-30|1993-05-27|1993-06-06|DELIVER IN PERSON|REG AIR|r the final, even packages. quickly 3171|138340|8341|2|50|68917.00|0.01|0.04|A|F|1993-07-19|1993-05-15|1993-07-31|TAKE BACK RETURN|REG AIR|riously final foxes about the ca 3172|95767|786|1|4|7051.04|0.06|0.07|A|F|1992-09-26|1992-08-15|1992-10-20|DELIVER IN PERSON|TRUCK|s are slyly thin package 3172|147816|5359|2|43|80143.83|0.05|0.07|R|F|1992-08-22|1992-07-07|1992-08-26|COLLECT COD|MAIL| final packages. 3172|131989|7016|3|13|26272.74|0.03|0.01|R|F|1992-07-06|1992-08-06|1992-08-05|DELIVER IN PERSON|MAIL|inal deposits haggle along the 3172|134555|9582|4|28|44507.40|0.08|0.04|R|F|1992-07-09|1992-07-14|1992-07-16|NONE|MAIL|regular ideas. packages are furi 3172|63775|3776|5|31|53901.87|0.05|0.08|A|F|1992-09-01|1992-08-27|1992-09-23|NONE|SHIP|. slyly regular dependencies haggle quiet 3173|194121|4122|1|35|42529.20|0.01|0.08|N|O|1996-09-09|1996-10-15|1996-10-04|TAKE BACK RETURN|RAIL| across the slyly even requests. 3173|177060|2095|2|5|5685.30|0.09|0.07|N|O|1996-12-06|1996-09-17|1996-12-07|DELIVER IN PERSON|REG AIR|express depo 3173|45832|841|3|16|28445.28|0.06|0.01|N|O|1996-08-12|1996-09-21|1996-08-22|NONE|SHIP|e special, 3173|93949|3950|4|2|3885.88|0.00|0.00|N|O|1996-10-15|1996-11-06|1996-10-18|COLLECT COD|MAIL|ular pearls 3173|184484|4485|5|2|3136.96|0.00|0.06|N|O|1996-08-18|1996-09-21|1996-09-07|DELIVER IN PERSON|MAIL|fluffily above t 3174|185221|7740|1|6|7837.32|0.04|0.08|N|O|1996-03-13|1996-02-09|1996-03-22|DELIVER IN PERSON|AIR| furiously ironic 3174|193509|8548|2|4|6410.00|0.01|0.05|N|O|1995-11-17|1996-01-08|1995-11-27|DELIVER IN PERSON|RAIL|deas sleep thi 3174|91355|3865|3|21|28273.35|0.08|0.05|N|O|1996-02-20|1995-12-28|1996-03-17|NONE|MAIL|iously. idly bold theodolites a 3174|191449|9007|4|13|20025.72|0.08|0.06|N|O|1996-01-11|1996-01-26|1996-02-01|DELIVER IN PERSON|SHIP|leep quickly? slyly special platelets 3174|71002|3510|5|39|37947.00|0.02|0.06|N|O|1995-12-02|1996-02-08|1995-12-12|TAKE BACK RETURN|TRUCK| wake slyly foxes. bold requests p 3174|119325|4348|6|8|10754.56|0.07|0.08|N|O|1995-12-07|1996-01-08|1995-12-29|DELIVER IN PERSON|TRUCK|nic deposits among t 3175|119225|6759|1|28|34838.16|0.10|0.01|R|F|1994-09-27|1994-10-05|1994-10-04|NONE|FOB|ore the even, silent foxes. b 3175|532|3033|2|38|54436.14|0.01|0.07|R|F|1994-10-10|1994-08-25|1994-10-28|NONE|MAIL|the quickly even dolph 3175|128833|1346|3|12|22341.96|0.09|0.07|R|F|1994-10-16|1994-09-15|1994-10-18|NONE|AIR|ter the pending deposits. slyly e 3175|84978|2503|4|14|27481.58|0.02|0.05|R|F|1994-10-21|1994-09-05|1994-11-15|NONE|MAIL|nt dependencies are quietly even 3175|17028|4532|5|47|44415.94|0.08|0.03|R|F|1994-08-08|1994-09-10|1994-08-21|COLLECT COD|REG AIR| final requests x-r 3175|174487|4488|6|44|68705.12|0.01|0.00|R|F|1994-09-26|1994-08-30|1994-10-24|TAKE BACK RETURN|MAIL|are carefully furiously ironic accounts. e 3175|864|3365|7|32|56475.52|0.01|0.02|R|F|1994-09-29|1994-09-20|1994-10-10|TAKE BACK RETURN|SHIP|lites sleep 3200|115903|3437|1|17|32621.30|0.10|0.00|N|O|1996-06-06|1996-04-21|1996-06-14|DELIVER IN PERSON|AIR|side of the furiously pendin 3200|165487|3036|2|27|41916.96|0.03|0.00|N|O|1996-05-07|1996-05-01|1996-05-09|TAKE BACK RETURN|REG AIR|as haggle furiously against the fluff 3200|130983|6010|3|36|72503.28|0.01|0.01|N|O|1996-03-22|1996-03-19|1996-03-30|DELIVER IN PERSON|FOB|f the carefu 3200|29905|4910|4|11|20183.90|0.10|0.02|N|O|1996-03-18|1996-03-21|1996-04-14|COLLECT COD|RAIL|osits sleep fur 3200|197667|7668|5|16|28234.56|0.05|0.00|N|O|1996-02-28|1996-03-13|1996-03-11|NONE|RAIL|ly against the quiet packages. blith 3200|174336|1888|6|25|35258.25|0.10|0.01|N|O|1996-02-08|1996-04-11|1996-03-06|COLLECT COD|FOB| slyly regular hockey players! pinto beans 3201|45786|5787|1|11|19049.58|0.10|0.06|A|F|1993-09-27|1993-08-29|1993-10-18|NONE|TRUCK|ing to the furiously expr 3201|117792|2815|2|27|48864.33|0.08|0.02|R|F|1993-08-31|1993-08-24|1993-09-08|TAKE BACK RETURN|FOB|deposits are slyly along 3201|118958|3981|3|50|98847.50|0.00|0.08|R|F|1993-10-27|1993-09-30|1993-11-16|COLLECT COD|TRUCK| deposits. express, ir 3202|182676|7713|1|30|52760.10|0.09|0.02|A|F|1993-03-18|1993-03-10|1993-03-23|COLLECT COD|SHIP|ven platelets. furiously final 3202|19748|2250|2|22|36690.28|0.01|0.02|R|F|1993-02-16|1993-02-16|1993-03-16|TAKE BACK RETURN|MAIL|the express packages. fu 3203|143488|3489|1|23|35224.04|0.01|0.07|N|O|1998-01-04|1998-01-12|1998-01-24|COLLECT COD|SHIP|uses. fluffily ironic pinto bea 3203|187707|2744|2|22|39483.40|0.03|0.03|N|O|1998-02-12|1998-01-01|1998-02-18|TAKE BACK RETURN|REG AIR|e the blithely regular accounts boost f 3204|11924|9428|1|10|18359.20|0.10|0.07|R|F|1993-01-27|1993-03-08|1993-01-29|COLLECT COD|SHIP|counts. bold 3204|6474|8975|2|39|53838.33|0.10|0.03|R|F|1993-02-11|1993-03-19|1993-02-28|TAKE BACK RETURN|MAIL|sits sleep theodolites. slyly bo 3205|67629|2642|1|7|11176.34|0.09|0.00|R|F|1992-07-05|1992-06-17|1992-07-07|NONE|SHIP|ly alongsi 3205|28792|8793|2|32|55065.28|0.08|0.03|A|F|1992-06-01|1992-07-10|1992-06-06|TAKE BACK RETURN|RAIL|lar accoun 3205|102663|5174|3|38|63295.08|0.10|0.08|A|F|1992-07-31|1992-06-03|1992-08-20|DELIVER IN PERSON|AIR|usly quiet accounts. slyly pending pinto 3205|55351|5352|4|10|13063.50|0.01|0.07|A|F|1992-06-18|1992-07-04|1992-07-16|COLLECT COD|RAIL| deposits cajole careful 3205|69090|1597|5|18|19063.62|0.03|0.03|A|F|1992-07-04|1992-06-14|1992-08-03|TAKE BACK RETURN|RAIL|symptotes. slyly even deposits ar 3205|194388|9427|6|19|28165.22|0.07|0.08|R|F|1992-05-28|1992-05-30|1992-06-05|COLLECT COD|AIR|yly pending packages snooz 3205|68994|1501|7|36|70667.64|0.06|0.03|A|F|1992-05-31|1992-06-19|1992-06-03|TAKE BACK RETURN|SHIP|s. ironic platelets above the s 3206|175184|2736|1|1|1259.18|0.07|0.05|N|O|1996-11-22|1996-10-16|1996-12-07|TAKE BACK RETURN|FOB|y unusual foxes cajole ab 3206|110775|3287|2|37|66073.49|0.07|0.01|N|O|1996-09-06|1996-10-31|1996-09-25|COLLECT COD|SHIP| quick theodolites hagg 3206|185591|5592|3|24|40238.16|0.00|0.08|N|O|1996-08-25|1996-10-01|1996-09-04|COLLECT COD|TRUCK|encies sleep deposits-- 3207|112470|4|1|2|2964.94|0.10|0.03|N|O|1998-06-15|1998-04-20|1998-06-21|COLLECT COD|MAIL|among the ironic, even packages 3207|70504|8026|2|42|61929.00|0.00|0.00|N|O|1998-05-02|1998-05-10|1998-06-01|NONE|SHIP|to the quickly special accounts? ironically 3207|151474|6505|3|17|25932.99|0.03|0.04|N|O|1998-03-27|1998-04-06|1998-03-28|COLLECT COD|RAIL|eep against the instructions. gifts hag 3207|18890|3893|4|32|57884.48|0.00|0.03|N|O|1998-06-17|1998-04-26|1998-07-07|TAKE BACK RETURN|SHIP|y across the slyly express foxes. bl 3207|82724|5233|5|8|13653.76|0.00|0.06|N|O|1998-06-13|1998-04-26|1998-07-11|COLLECT COD|SHIP|y. final pint 3207|133661|8688|6|32|54229.12|0.03|0.05|N|O|1998-04-19|1998-05-01|1998-05-08|COLLECT COD|FOB|l deposits wake beyond the carefully 3232|13217|3218|1|22|24864.62|0.10|0.01|A|F|1992-11-30|1992-12-09|1992-12-04|NONE|RAIL|thely. furio 3232|134360|6874|2|34|47408.24|0.07|0.04|R|F|1993-01-09|1992-11-14|1993-02-03|NONE|SHIP|old packages integrate quickly 3232|180226|7781|3|3|3918.66|0.04|0.06|R|F|1992-12-14|1992-12-11|1992-12-29|DELIVER IN PERSON|FOB|ily blithely ironic acco 3233|50584|585|1|23|35295.34|0.04|0.05|A|F|1994-12-07|1995-01-11|1994-12-26|NONE|AIR|pending instructions use after the carefu 3233|153443|989|2|6|8978.64|0.02|0.08|A|F|1994-12-06|1994-12-05|1994-12-07|TAKE BACK RETURN|REG AIR|requests are quickly above the slyly p 3233|99840|7368|3|2|3679.68|0.04|0.06|R|F|1995-01-03|1995-01-02|1995-01-21|TAKE BACK RETURN|AIR| across the bold packages 3233|8532|1033|4|25|36013.25|0.04|0.07|A|F|1994-11-24|1995-01-07|1994-12-11|NONE|RAIL|oss the pl 3234|78973|8974|1|45|87838.65|0.01|0.04|N|O|1996-05-15|1996-05-09|1996-06-02|DELIVER IN PERSON|TRUCK| express packages are carefully. f 3234|83220|745|2|23|27674.06|0.03|0.00|N|O|1996-05-29|1996-05-15|1996-06-17|DELIVER IN PERSON|AIR|d-- fluffily special packag 3234|74096|9111|3|16|17121.44|0.06|0.05|N|O|1996-06-10|1996-05-30|1996-06-18|COLLECT COD|RAIL|ithely ironic accounts wake along t 3234|121031|6056|4|50|52601.50|0.09|0.05|N|O|1996-06-11|1996-05-19|1996-06-18|NONE|MAIL|ly regular ideas according to the regula 3234|164274|9307|5|14|18735.78|0.01|0.07|N|O|1996-04-06|1996-05-30|1996-04-13|NONE|REG AIR|lithely regular f 3235|108187|698|1|9|10756.62|0.07|0.00|N|O|1995-11-17|1995-12-24|1995-11-30|COLLECT COD|AIR|l courts sleep quickly slyly 3235|94886|4887|2|43|80877.84|0.10|0.07|N|O|1995-12-25|1996-01-23|1996-01-09|COLLECT COD|MAIL|ckly final instru 3235|137620|7621|3|29|48070.98|0.06|0.06|N|O|1996-01-28|1995-12-26|1996-02-12|DELIVER IN PERSON|RAIL|e fluffy pinto bea 3235|177363|7364|4|23|33128.28|0.00|0.01|N|O|1996-02-16|1996-01-05|1996-03-07|DELIVER IN PERSON|SHIP|ldly ironic pinto beans 3236|116245|1268|1|10|12612.40|0.06|0.05|N|O|1996-11-15|1996-12-14|1996-11-29|TAKE BACK RETURN|AIR|arefully. fluffily reg 3236|121623|4136|2|21|34537.02|0.01|0.07|N|O|1996-12-23|1996-12-12|1997-01-21|NONE|AIR| final pinto 3236|117099|9611|3|7|7812.63|0.07|0.01|N|O|1996-12-27|1996-12-18|1997-01-24|DELIVER IN PERSON|SHIP|dolites. slyly unus 3237|10711|3213|1|11|17838.81|0.02|0.07|A|F|1992-08-03|1992-07-31|1992-08-13|TAKE BACK RETURN|AIR|es. permanently express platelets besid 3238|71612|1613|1|12|19003.32|0.06|0.01|R|F|1993-03-06|1993-05-08|1993-04-01|DELIVER IN PERSON|AIR|ackages affix furiously. furiously bol 3238|172398|7433|2|26|38230.14|0.01|0.06|A|F|1993-02-25|1993-04-04|1993-03-20|TAKE BACK RETURN|REG AIR|g accounts sleep furiously ironic attai 3238|80096|2605|3|1|1076.09|0.00|0.04|R|F|1993-05-17|1993-04-18|1993-05-27|NONE|SHIP|wake alongs 3239|44485|9494|1|50|71474.00|0.05|0.01|N|O|1998-02-09|1998-04-02|1998-02-22|NONE|FOB|d blithely stea 3239|44840|9849|2|43|76748.12|0.01|0.06|N|O|1998-01-15|1998-03-12|1998-01-29|COLLECT COD|REG AIR|y. bold pinto beans use 3239|12947|5449|3|13|24179.22|0.01|0.05|N|O|1998-02-10|1998-02-19|1998-02-25|DELIVER IN PERSON|MAIL|r deposits solve fluf 3239|194825|4826|4|26|49915.32|0.03|0.05|N|O|1998-01-21|1998-03-21|1998-02-08|DELIVER IN PERSON|SHIP|ngly pending platelets are fluff 3239|11739|6742|5|31|51172.63|0.10|0.08|N|O|1998-04-14|1998-03-24|1998-04-17|DELIVER IN PERSON|FOB|foxes. pendin 3264|199532|9533|1|39|63629.67|0.06|0.06|N|O|1996-11-07|1996-12-12|1996-11-20|TAKE BACK RETURN|REG AIR|sleep carefully after the slyly final 3264|130251|252|2|34|43562.50|0.00|0.01|N|O|1997-01-03|1997-01-06|1997-01-29|TAKE BACK RETURN|REG AIR|rns haggle carefully. blit 3264|124205|1742|3|11|13521.20|0.09|0.03|N|O|1996-12-11|1996-12-19|1996-12-15|DELIVER IN PERSON|SHIP|regular packages 3264|108626|8627|4|24|39230.88|0.09|0.07|N|O|1997-01-07|1996-12-13|1997-01-11|TAKE BACK RETURN|RAIL|ctions. quick 3264|62126|2127|5|6|6528.72|0.04|0.03|N|O|1996-11-10|1996-12-05|1996-11-22|TAKE BACK RETURN|SHIP|press packages. ironical 3264|140345|346|6|43|59569.62|0.06|0.06|N|O|1997-01-17|1997-01-24|1997-02-01|TAKE BACK RETURN|TRUCK|leep at the blithely bold 3265|24887|9892|1|8|14495.04|0.06|0.02|A|F|1992-09-01|1992-09-12|1992-09-27|DELIVER IN PERSON|TRUCK|thely ironic requests sleep slyly-- i 3265|71780|4288|2|7|12262.46|0.09|0.00|R|F|1992-09-16|1992-09-04|1992-10-14|DELIVER IN PERSON|MAIL|he forges. fluffily regular asym 3265|190046|5085|3|28|31809.12|0.09|0.08|A|F|1992-10-22|1992-08-23|1992-10-25|NONE|RAIL|n requests. quickly final dinos 3266|63179|8192|1|31|35407.27|0.09|0.02|N|O|1995-06-19|1995-05-04|1995-07-06|COLLECT COD|MAIL|grate among the quickly express deposits 3266|37805|309|2|43|74940.40|0.06|0.07|R|F|1995-05-04|1995-05-30|1995-05-11|COLLECT COD|AIR|ular asymptotes use careful 3267|184283|1838|1|33|45120.24|0.06|0.01|N|O|1997-03-30|1997-03-25|1997-04-23|TAKE BACK RETURN|AIR|es boost. 3268|95677|5678|1|1|1672.67|0.06|0.08|A|F|1994-09-12|1994-08-31|1994-09-16|NONE|TRUCK|. ironic, bold requests use carefull 3268|41914|4419|2|40|74236.40|0.08|0.01|R|F|1994-06-30|1994-08-22|1994-07-25|COLLECT COD|FOB|ly. bold, eve 3269|160745|3262|1|40|72229.60|0.02|0.07|N|O|1996-06-11|1996-05-06|1996-06-15|DELIVER IN PERSON|TRUCK|es. pending d 3269|37202|4712|2|46|52403.20|0.00|0.02|N|O|1996-04-21|1996-04-12|1996-05-10|DELIVER IN PERSON|MAIL|final asymptotes nag 3269|43059|572|3|39|39079.95|0.02|0.03|N|O|1996-03-13|1996-05-26|1996-03-19|COLLECT COD|MAIL|he express packages? 3269|82008|7025|4|37|36630.00|0.07|0.05|N|O|1996-06-14|1996-04-27|1996-07-07|NONE|MAIL|egular requests. carefully un 3269|92446|9974|5|42|60414.48|0.09|0.05|N|O|1996-03-19|1996-04-24|1996-04-18|COLLECT COD|TRUCK| the special packages. 3269|130091|7631|6|16|17937.44|0.01|0.08|N|O|1996-03-03|1996-04-06|1996-03-06|NONE|RAIL|s cajole. silent deposits are f 3270|34060|6564|1|11|10934.66|0.07|0.06|N|O|1997-07-29|1997-08-11|1997-08-05|TAKE BACK RETURN|AIR| solve at the regular deposits. 3270|37156|4666|2|44|48098.60|0.10|0.05|N|O|1997-07-20|1997-08-15|1997-08-04|DELIVER IN PERSON|SHIP| accounts. carefully even 3270|64294|6801|3|20|25165.80|0.01|0.02|N|O|1997-08-26|1997-07-31|1997-08-30|DELIVER IN PERSON|FOB|en accounts among the c 3270|188091|5646|4|29|34193.61|0.06|0.05|N|O|1997-07-01|1997-07-23|1997-07-10|TAKE BACK RETURN|MAIL|sly regular asymptotes. slyly dog 3270|33740|1250|5|32|53559.68|0.03|0.00|N|O|1997-09-23|1997-08-17|1997-09-27|NONE|REG AIR|promise carefully. 3270|56984|9490|6|29|56288.42|0.01|0.04|N|O|1997-08-22|1997-08-17|1997-09-06|COLLECT COD|RAIL|ptotes nag above the quickly bold deposits 3270|116753|9265|7|9|15927.75|0.06|0.08|N|O|1997-08-14|1997-08-11|1997-09-09|DELIVER IN PERSON|SHIP|ual packages 3271|56433|3949|1|30|41682.90|0.01|0.04|A|F|1992-01-16|1992-03-20|1992-01-17|DELIVER IN PERSON|AIR|r the unusual Tiresia 3271|53210|3211|2|18|20937.78|0.09|0.06|R|F|1992-05-01|1992-03-28|1992-05-29|DELIVER IN PERSON|FOB| packages eat around the furiously regul 3271|94870|4871|3|14|26108.18|0.05|0.01|A|F|1992-02-24|1992-02-14|1992-03-23|NONE|AIR|ending, even packa 3271|63369|8382|4|29|38638.44|0.07|0.04|A|F|1992-03-10|1992-02-05|1992-03-14|COLLECT COD|MAIL|lar instructions. carefully regular 3296|83475|1000|1|12|17501.64|0.06|0.07|R|F|1994-12-08|1994-12-14|1994-12-24|COLLECT COD|AIR|y about the slyly bold pinto bea 3296|148438|5981|2|31|46079.33|0.08|0.00|R|F|1995-01-26|1994-12-25|1995-02-16|NONE|REG AIR|ainst the furi 3296|184182|4183|3|29|36719.22|0.02|0.04|A|F|1995-01-12|1994-11-26|1995-02-06|DELIVER IN PERSON|SHIP|ss ideas are reg 3296|139691|4718|4|47|81342.43|0.06|0.00|A|F|1994-11-08|1994-12-20|1994-11-30|NONE|FOB|egular deposits. quic 3296|176498|1533|5|16|25191.84|0.06|0.02|R|F|1995-01-11|1994-12-27|1995-01-12|DELIVER IN PERSON|SHIP|kages cajole carefully 3296|196504|4062|6|40|64020.00|0.00|0.04|A|F|1994-12-28|1994-12-08|1995-01-13|COLLECT COD|REG AIR|ronic ideas across the 3296|35755|3265|7|6|10144.50|0.02|0.01|R|F|1995-01-03|1994-12-23|1995-01-27|TAKE BACK RETURN|AIR|carefully fur 3297|133262|5776|1|10|12952.60|0.10|0.04|A|F|1992-12-14|1993-01-21|1992-12-26|NONE|SHIP|ironic idea 3298|148414|929|1|9|13161.69|0.01|0.06|N|O|1996-08-15|1996-05-24|1996-09-12|COLLECT COD|REG AIR|ly final accou 3298|185925|5926|2|27|54294.84|0.06|0.06|N|O|1996-07-10|1996-05-21|1996-07-15|DELIVER IN PERSON|FOB|lar packages. regular deposit 3298|28881|6388|3|25|45247.00|0.10|0.08|N|O|1996-06-30|1996-05-31|1996-07-23|COLLECT COD|SHIP|ly express f 3298|190710|8268|4|1|1800.71|0.10|0.03|N|O|1996-07-31|1996-05-23|1996-08-24|TAKE BACK RETURN|FOB|refully regular requ 3299|182485|40|1|40|62699.20|0.03|0.02|A|F|1994-03-21|1994-03-23|1994-04-12|COLLECT COD|AIR|lyly even request 3300|128765|1278|1|3|5381.28|0.07|0.02|N|O|1995-11-01|1995-10-02|1995-11-20|NONE|REG AIR|g according to the dugouts. caref 3300|148043|8044|2|23|25093.92|0.02|0.02|N|O|1995-08-17|1995-09-03|1995-09-04|COLLECT COD|TRUCK|he fluffily final a 3301|168995|1512|1|45|92879.55|0.04|0.05|A|F|1994-11-19|1994-10-27|1994-11-24|TAKE BACK RETURN|FOB|nusual, final excuses after the entici 3302|35376|7880|1|45|59011.65|0.09|0.00|N|O|1996-01-24|1995-12-16|1996-02-13|COLLECT COD|FOB|counts use quickl 3303|183830|3831|1|25|47845.75|0.06|0.01|N|O|1998-03-25|1998-01-31|1998-04-12|NONE|SHIP|lly regular pi 3303|20879|880|2|15|26998.05|0.04|0.06|N|O|1998-01-29|1998-01-22|1998-02-21|COLLECT COD|SHIP| detect sly 3303|98533|8534|3|37|56666.61|0.05|0.02|N|O|1998-02-16|1998-03-07|1998-02-18|TAKE BACK RETURN|TRUCK| carefully ironic asympt 3303|35272|2782|4|26|31389.02|0.09|0.00|N|O|1998-01-18|1998-03-11|1998-02-11|DELIVER IN PERSON|REG AIR|ickly permanent requests w 3328|112585|5097|1|6|9585.48|0.03|0.08|A|F|1993-03-07|1993-01-25|1993-03-29|COLLECT COD|TRUCK|ffily even instructions detect b 3328|4400|1901|2|23|30001.20|0.01|0.06|R|F|1993-01-12|1993-02-07|1993-01-30|TAKE BACK RETURN|MAIL|y. careful 3328|138924|8925|3|44|86368.48|0.05|0.00|R|F|1992-12-03|1992-12-19|1992-12-09|TAKE BACK RETURN|FOB|dly quickly final foxes? re 3328|94005|1533|4|42|41958.00|0.01|0.05|R|F|1992-11-24|1992-12-20|1992-12-06|DELIVER IN PERSON|AIR|ronic requests 3328|130322|7862|5|25|33808.00|0.05|0.00|R|F|1993-01-28|1993-01-04|1993-01-31|NONE|RAIL|e unusual, r 3329|137884|398|1|36|69187.68|0.09|0.08|N|O|1995-08-06|1995-08-03|1995-08-14|DELIVER IN PERSON|TRUCK|ts at the re 3329|5898|3399|2|9|16235.01|0.00|0.02|N|O|1995-07-24|1995-08-02|1995-08-01|COLLECT COD|MAIL|lly final depo 3329|122108|2109|3|1|1130.10|0.04|0.08|N|O|1995-08-22|1995-09-28|1995-09-09|COLLECT COD|REG AIR|regular packages are carefull 3330|19095|4098|1|49|49690.41|0.05|0.01|R|F|1995-03-02|1995-03-03|1995-03-16|DELIVER IN PERSON|TRUCK|haggle carefully alongside of the bold r 3331|63261|780|1|9|11018.34|0.08|0.07|A|F|1993-07-18|1993-07-03|1993-08-16|TAKE BACK RETURN|AIR|odolites. bold accounts 3331|20445|446|2|38|51886.72|0.06|0.04|R|F|1993-07-24|1993-06-22|1993-08-23|NONE|AIR|ymptotes haggle across the ca 3331|2546|47|3|26|37662.04|0.09|0.05|A|F|1993-08-05|1993-07-17|1993-08-29|DELIVER IN PERSON|MAIL|p asymptotes. carefully unusual in 3332|83771|3772|1|28|49133.56|0.10|0.02|R|F|1994-12-30|1995-01-16|1995-01-16|COLLECT COD|FOB|s against the carefully special multipl 3332|135964|3504|2|21|41999.16|0.08|0.04|R|F|1995-02-04|1995-01-08|1995-02-06|COLLECT COD|MAIL| quick packages sle 3332|133508|8535|3|27|41620.50|0.03|0.02|A|F|1994-12-10|1995-01-14|1994-12-11|TAKE BACK RETURN|FOB|ording to the slyly regula 3333|149926|7469|1|27|53349.84|0.06|0.08|A|F|1992-12-06|1992-10-26|1992-12-07|COLLECT COD|SHIP|s dazzle fluffil 3333|198900|6458|2|36|71960.40|0.08|0.07|R|F|1992-11-20|1992-11-06|1992-12-16|TAKE BACK RETURN|FOB|foxes sleep neve 3333|107224|9735|3|38|46786.36|0.05|0.05|A|F|1992-10-30|1992-11-03|1992-11-04|NONE|MAIL|ccounts promise bl 3333|112230|2231|4|49|60869.27|0.07|0.07|R|F|1992-10-02|1992-11-30|1992-10-12|DELIVER IN PERSON|MAIL|riously ironic r 3333|42929|442|5|45|84236.40|0.07|0.08|A|F|1992-10-04|1992-11-08|1992-10-27|COLLECT COD|SHIP|dolites. quickly r 3334|186434|1471|1|20|30408.60|0.04|0.03|N|O|1996-05-21|1996-04-08|1996-05-26|TAKE BACK RETURN|AIR|uses nag furiously. instructions are ca 3334|189013|4050|2|7|7714.07|0.09|0.07|N|O|1996-04-28|1996-04-08|1996-05-25|NONE|SHIP|nts sublate slyly express pack 3335|104399|9420|1|13|18244.07|0.06|0.07|N|O|1996-01-20|1995-12-20|1996-02-09|COLLECT COD|REG AIR|out the special asymptotes 3335|30734|735|2|44|73248.12|0.07|0.02|N|O|1996-01-05|1995-12-25|1996-01-18|DELIVER IN PERSON|SHIP|r packages cajole ac 3335|139710|2224|3|16|27995.36|0.01|0.06|N|O|1995-10-18|1995-12-08|1995-11-03|DELIVER IN PERSON|SHIP|g packages. carefully regular reque 3335|89353|6878|4|47|63090.45|0.10|0.03|N|O|1995-12-02|1995-11-19|1995-12-27|NONE|MAIL| quickly special ideas. 3360|173707|6225|1|31|55201.70|0.08|0.04|N|O|1998-04-24|1998-04-12|1998-05-23|COLLECT COD|REG AIR|quests. carefully even deposits wake acros 3360|90182|2692|2|29|33993.22|0.00|0.06|N|O|1998-04-15|1998-02-25|1998-05-13|TAKE BACK RETURN|FOB|press asymptotes. furiously final 3360|81355|1356|3|39|52117.65|0.08|0.03|N|O|1998-04-09|1998-04-20|1998-05-05|DELIVER IN PERSON|REG AIR|s. blithely express pinto bean 3360|116741|4275|4|29|50974.46|0.10|0.01|N|O|1998-05-19|1998-03-03|1998-06-09|TAKE BACK RETURN|FOB|hely gifts. spe 3360|57076|9582|5|4|4132.28|0.08|0.07|N|O|1998-02-27|1998-03-23|1998-03-28|COLLECT COD|SHIP|ly busy inst 3360|70055|2563|6|42|43052.10|0.04|0.01|N|O|1998-05-07|1998-04-18|1998-06-04|DELIVER IN PERSON|FOB|ages cajole. pending, 3361|143748|3749|1|6|10750.44|0.02|0.02|R|F|1992-10-02|1992-10-25|1992-10-05|DELIVER IN PERSON|FOB| packages sleep. furiously unus 3361|170012|5047|2|33|35706.33|0.01|0.02|R|F|1992-11-09|1992-10-15|1992-11-11|TAKE BACK RETURN|MAIL|uriously ironic accounts. ironic, ir 3361|190085|7643|3|31|36427.48|0.06|0.04|R|F|1992-08-29|1992-10-13|1992-09-08|NONE|FOB|ts. pending, regular accounts sleep fur 3362|21372|8879|1|14|18107.18|0.06|0.05|N|O|1995-08-01|1995-09-06|1995-08-22|NONE|FOB|even Tires 3362|194902|4903|2|41|81872.90|0.05|0.03|N|O|1995-10-31|1995-09-04|1995-11-17|COLLECT COD|REG AIR|ake alongside of the 3362|114822|7334|3|40|73472.80|0.05|0.06|N|O|1995-08-19|1995-10-17|1995-09-05|TAKE BACK RETURN|FOB|packages haggle furi 3362|1127|6128|4|3|3084.36|0.03|0.01|N|O|1995-08-26|1995-09-02|1995-09-17|NONE|SHIP|its cajole blithely excuses. de 3362|137109|7110|5|36|41259.60|0.06|0.00|N|O|1995-10-05|1995-08-28|1995-11-03|TAKE BACK RETURN|RAIL|es against the quickly permanent pint 3362|187847|7848|6|46|89002.64|0.09|0.05|N|O|1995-08-02|1995-10-12|1995-08-28|COLLECT COD|REG AIR|ly bold packages. regular deposits cajol 3363|9770|2271|1|42|70550.34|0.00|0.08|N|O|1995-11-09|1995-11-25|1995-11-15|TAKE BACK RETURN|RAIL| blithely final ideas nag after 3363|190181|5220|2|21|26694.78|0.08|0.08|N|O|1995-12-10|1995-10-28|1995-12-28|COLLECT COD|RAIL|he regular, brave deposits. f 3363|158620|1136|3|2|3357.24|0.01|0.07|N|O|1996-01-22|1995-12-01|1996-02-18|TAKE BACK RETURN|SHIP|uickly bold ide 3363|112620|154|4|20|32652.40|0.07|0.06|N|O|1995-12-11|1995-11-15|1995-12-21|COLLECT COD|MAIL|carefully quiet excuses wake. sl 3363|199176|6734|5|4|5100.68|0.00|0.08|N|O|1995-10-30|1995-11-17|1995-11-22|COLLECT COD|FOB| ironic dependencie 3364|89208|6733|1|49|58662.80|0.03|0.05|N|O|1997-09-17|1997-08-23|1997-10-06|NONE|SHIP|d accounts? caref 3364|110107|108|2|38|42449.80|0.02|0.02|N|O|1997-08-30|1997-09-12|1997-09-27|COLLECT COD|REG AIR| slyly express 3364|155969|8485|3|10|20249.60|0.00|0.01|N|O|1997-08-10|1997-08-24|1997-08-15|TAKE BACK RETURN|SHIP|g the accounts. final, busy accounts wi 3364|159356|4387|4|7|9907.45|0.10|0.05|N|O|1997-07-09|1997-08-01|1997-07-16|NONE|TRUCK|furiously regular ideas haggle furiously b 3364|80467|5484|5|3|4342.38|0.01|0.00|N|O|1997-10-19|1997-08-15|1997-10-28|TAKE BACK RETURN|TRUCK|c theodolites. blithely ir 3365|150138|5169|1|37|43960.81|0.02|0.08|R|F|1994-12-22|1995-02-07|1995-01-20|TAKE BACK RETURN|SHIP|requests. quickly pending instructions a 3365|166378|3927|2|37|53441.69|0.07|0.08|A|F|1994-11-24|1995-01-09|1994-11-27|NONE|REG AIR|oze blithely. furiously ironic theodolit 3365|114118|4119|3|13|14717.43|0.09|0.02|R|F|1995-02-25|1995-01-31|1995-03-16|NONE|RAIL|pths wake r 3365|175036|2588|4|49|54440.47|0.02|0.07|R|F|1995-01-03|1995-01-01|1995-01-18|COLLECT COD|MAIL|lyly unusual asymptotes. final 3365|15264|267|5|2|2358.52|0.00|0.03|R|F|1995-02-04|1994-12-30|1995-03-06|TAKE BACK RETURN|FOB|es cajole fluffily pe 3365|125386|411|6|24|33873.12|0.01|0.00|R|F|1995-02-27|1995-01-09|1995-03-27|DELIVER IN PERSON|REG AIR|into beans? carefully regula 3366|39754|9755|1|4|6775.00|0.07|0.01|N|O|1997-05-20|1997-06-25|1997-06-03|DELIVER IN PERSON|AIR| carefully about 3366|135173|7687|2|9|10873.53|0.00|0.08|N|O|1997-06-02|1997-07-05|1997-06-26|COLLECT COD|REG AIR|ackages sleep carefully across the bli 3367|40397|7910|1|27|36109.53|0.01|0.03|A|F|1993-04-13|1993-03-16|1993-04-26|NONE|RAIL|kly even instructions caj 3367|140913|8456|2|34|66432.94|0.04|0.08|A|F|1993-03-30|1993-02-23|1993-04-11|COLLECT COD|MAIL| accounts wake slyly 3367|119437|4460|3|38|55344.34|0.03|0.03|R|F|1993-03-13|1993-02-12|1993-03-31|NONE|RAIL|even packages sleep blithely slyly expr 3392|170472|5507|1|40|61698.80|0.01|0.01|N|O|1996-02-18|1995-12-16|1996-02-26|COLLECT COD|MAIL|ress instructions affix carefully. fur 3392|122112|7137|2|13|14743.43|0.09|0.02|N|O|1995-11-26|1996-01-17|1995-12-01|NONE|MAIL|across the fluffily bold deposits. 3392|126515|4052|3|34|52411.34|0.10|0.08|N|O|1996-01-20|1996-01-21|1996-01-24|DELIVER IN PERSON|MAIL|e carefully even braids. 3392|123260|8285|4|7|8982.82|0.08|0.05|N|O|1995-12-07|1996-01-09|1995-12-29|TAKE BACK RETURN|RAIL|as. express, final accounts dou 3393|116495|4029|1|16|24183.84|0.01|0.00|N|O|1995-07-17|1995-08-19|1995-08-04|COLLECT COD|TRUCK|uses. instructions after the blithely 3393|124847|9872|2|44|82360.96|0.08|0.04|N|O|1995-10-16|1995-08-05|1995-11-01|NONE|AIR|ld requests hag 3393|96369|3897|3|25|34134.00|0.07|0.02|N|O|1995-10-17|1995-08-12|1995-11-11|DELIVER IN PERSON|MAIL|ng excuses 3393|71005|3513|4|48|46848.00|0.06|0.06|N|O|1995-07-12|1995-09-15|1995-08-02|NONE|FOB| blithely final reques 3393|177894|2929|5|37|72959.93|0.07|0.02|N|O|1995-10-16|1995-08-19|1995-10-19|COLLECT COD|AIR|ss the slyly ironic pinto beans. ironic, 3393|61866|9385|6|17|31073.62|0.04|0.01|N|O|1995-08-15|1995-09-07|1995-09-10|COLLECT COD|MAIL|kly ironic deposits could 3394|154659|4660|1|33|56550.45|0.07|0.08|N|O|1996-08-07|1996-07-17|1996-09-02|TAKE BACK RETURN|SHIP|ideas alongside of th 3394|145731|8246|2|43|76399.39|0.08|0.03|N|O|1996-08-23|1996-07-20|1996-08-25|COLLECT COD|RAIL|hockey players. slyly regular requests afte 3394|87945|454|3|26|50256.44|0.01|0.00|N|O|1996-08-08|1996-06-12|1996-09-05|TAKE BACK RETURN|RAIL|its use furiously. even, even account 3394|80118|119|4|14|15373.54|0.08|0.00|N|O|1996-06-02|1996-07-02|1996-06-19|COLLECT COD|MAIL|e furiously final theodolites. furio 3394|126483|6484|5|30|45284.40|0.04|0.06|N|O|1996-05-12|1996-07-24|1996-05-19|COLLECT COD|REG AIR|t ideas according to the fluffily iro 3394|183447|8484|6|14|21426.16|0.05|0.05|N|O|1996-06-18|1996-06-24|1996-07-17|NONE|REG AIR|arefully regular do 3395|141634|1635|1|21|35188.23|0.03|0.06|R|F|1994-12-19|1995-01-13|1994-12-25|TAKE BACK RETURN|SHIP| careful dep 3395|35345|2855|2|38|48652.92|0.01|0.07|R|F|1995-01-13|1995-01-13|1995-01-25|COLLECT COD|SHIP| silent accounts are blithely 3395|42169|2170|3|43|47779.88|0.06|0.07|A|F|1994-12-13|1995-01-07|1994-12-14|COLLECT COD|AIR|ckages above the furiously regu 3395|121513|6538|4|39|59845.89|0.05|0.07|R|F|1994-12-03|1995-01-17|1994-12-10|NONE|AIR|riously unusual theodolites. fur 3396|127938|2963|1|34|66841.62|0.00|0.06|A|F|1994-05-30|1994-08-16|1994-06-11|NONE|AIR|. slyly unusual packages wak 3396|48322|827|2|43|54623.76|0.03|0.08|A|F|1994-07-03|1994-08-09|1994-07-14|TAKE BACK RETURN|MAIL|cial packages cajole blithely around the 3396|137579|93|3|9|14549.13|0.01|0.06|R|F|1994-07-01|1994-08-18|1994-07-21|DELIVER IN PERSON|AIR|usly special foxes. accounts wake careful 3396|74334|1856|4|32|41866.56|0.06|0.02|R|F|1994-08-07|1994-08-10|1994-09-05|COLLECT COD|TRUCK|osits are slyly. final, bold foxes s 3396|125022|47|5|27|28269.54|0.02|0.01|A|F|1994-09-14|1994-07-26|1994-09-28|DELIVER IN PERSON|FOB| theodolites 3396|38687|8688|6|18|29262.24|0.10|0.00|A|F|1994-07-27|1994-06-26|1994-08-25|TAKE BACK RETURN|REG AIR|l requests haggle furiously along the fur 3396|197228|4786|7|31|41081.82|0.05|0.06|A|F|1994-06-07|1994-06-23|1994-06-19|TAKE BACK RETURN|REG AIR|l, express pinto beans. quic 3397|194289|9328|1|8|11066.24|0.07|0.01|A|F|1994-08-05|1994-08-11|1994-08-08|DELIVER IN PERSON|RAIL|y final foxes 3397|12042|9546|2|11|10494.44|0.00|0.07|A|F|1994-07-29|1994-09-18|1994-08-12|DELIVER IN PERSON|REG AIR|iously careful packages. s 3397|183905|6424|3|1|1988.90|0.07|0.05|R|F|1994-08-03|1994-07-30|1994-08-28|NONE|RAIL| regular packag 3397|85592|5593|4|33|52060.47|0.05|0.01|R|F|1994-09-04|1994-08-06|1994-09-22|COLLECT COD|RAIL|gular accounts. blithely re 3397|131561|1562|5|28|44591.68|0.05|0.05|R|F|1994-07-13|1994-08-26|1994-07-17|NONE|TRUCK|counts around the final reques 3398|172957|2958|1|1|2029.95|0.01|0.08|N|O|1996-11-22|1996-11-16|1996-12-09|COLLECT COD|MAIL| blithely final deposits. 3399|133853|8880|1|28|52831.80|0.09|0.05|N|O|1995-06-29|1995-05-19|1995-07-12|COLLECT COD|AIR|oggedly final theodolites grow. fi 3399|54517|4518|2|8|11772.08|0.01|0.05|A|F|1995-05-15|1995-04-19|1995-06-05|COLLECT COD|TRUCK|s use carefully carefully ir 3399|66486|1499|3|3|4357.44|0.03|0.00|N|F|1995-06-16|1995-04-04|1995-06-23|NONE|SHIP|hely pending dugouts 3399|13659|3660|4|21|33025.65|0.09|0.06|A|F|1995-03-12|1995-05-18|1995-03-28|TAKE BACK RETURN|MAIL|se final courts. exc 3424|180866|8421|1|39|75927.54|0.06|0.07|N|O|1996-11-03|1996-11-08|1996-11-23|DELIVER IN PERSON|MAIL|bits boost closely slyly p 3425|119527|9528|1|11|17011.72|0.03|0.08|N|O|1996-04-24|1996-05-29|1996-05-23|DELIVER IN PERSON|FOB|ckly final deposits use quickly? 3425|78421|5943|2|37|51778.54|0.06|0.03|N|O|1996-06-04|1996-05-09|1996-06-12|NONE|SHIP|as sleep carefully into the caref 3425|13460|964|3|8|10987.68|0.06|0.08|N|O|1996-07-22|1996-06-07|1996-07-26|TAKE BACK RETURN|AIR|iously regular theodolites wake. s 3425|18914|8915|4|37|67817.67|0.04|0.01|N|O|1996-07-10|1996-05-10|1996-08-02|NONE|SHIP|ngside of the furiously thin dol 3425|78322|830|5|48|62415.36|0.08|0.04|N|O|1996-04-14|1996-05-25|1996-04-23|TAKE BACK RETURN|AIR|uctions wake fluffily. care 3425|147931|7932|6|24|47494.32|0.05|0.04|N|O|1996-04-22|1996-06-24|1996-04-25|TAKE BACK RETURN|AIR|ajole blithely sl 3426|109706|4727|1|20|34314.00|0.05|0.04|N|O|1996-11-10|1996-12-24|1996-12-01|COLLECT COD|FOB|sits cajole blit 3426|13356|860|2|19|24117.65|0.10|0.08|N|O|1996-11-02|1997-01-13|1996-11-15|DELIVER IN PERSON|RAIL|slyly special packages oug 3426|66426|8933|3|19|26455.98|0.08|0.05|N|O|1996-12-07|1996-12-15|1996-12-14|DELIVER IN PERSON|FOB|c accounts cajole carefu 3426|5263|5264|4|9|10514.34|0.09|0.05|N|O|1996-12-24|1997-01-14|1997-01-13|NONE|FOB|pecial theodolites haggle fluf 3426|48579|1084|5|31|47354.67|0.07|0.08|N|O|1996-11-11|1996-12-10|1996-12-10|DELIVER IN PERSON|SHIP| even sentiment 3427|53394|3395|1|41|55242.99|0.10|0.01|N|O|1997-09-11|1997-07-03|1997-10-04|COLLECT COD|RAIL|s the carefully 3427|188918|3955|2|24|48165.84|0.02|0.04|N|O|1997-07-01|1997-07-28|1997-07-30|NONE|SHIP|y bold, sly deposits. pendi 3427|138997|1511|3|40|81439.60|0.06|0.05|N|O|1997-06-12|1997-08-19|1997-06-23|COLLECT COD|MAIL|patterns cajole ca 3427|118088|3111|4|31|34288.48|0.08|0.04|N|O|1997-08-12|1997-07-26|1997-08-25|COLLECT COD|RAIL|s are carefull 3428|197651|7652|1|4|6994.60|0.00|0.03|N|O|1996-05-09|1996-06-13|1996-06-02|NONE|REG AIR|sly pending requests int 3428|117436|7437|2|35|50870.05|0.02|0.03|N|O|1996-05-01|1996-06-07|1996-05-20|COLLECT COD|TRUCK|ly regular pinto beans sleep 3428|135307|5308|3|47|63088.10|0.07|0.05|N|O|1996-04-16|1996-06-08|1996-05-05|NONE|REG AIR|y final pinto 3429|136768|1795|1|48|86628.48|0.06|0.02|N|O|1997-04-08|1997-03-09|1997-04-25|TAKE BACK RETURN|SHIP| haggle furiously ir 3429|58421|927|2|15|20691.30|0.03|0.04|N|O|1997-02-04|1997-03-09|1997-03-01|TAKE BACK RETURN|TRUCK|beans are fu 3429|68561|6080|3|10|15295.60|0.05|0.07|N|O|1997-01-19|1997-02-22|1997-01-25|TAKE BACK RETURN|REG AIR|ackages. quickly e 3429|88366|875|4|28|37922.08|0.10|0.07|N|O|1997-01-30|1997-03-18|1997-02-17|TAKE BACK RETURN|AIR|nstructions boost. thin 3429|164054|4055|5|45|50312.25|0.10|0.00|N|O|1997-04-21|1997-03-08|1997-05-05|COLLECT COD|REG AIR|ites poach a 3430|188117|3154|1|2|2410.22|0.07|0.06|R|F|1995-03-07|1995-01-28|1995-03-30|TAKE BACK RETURN|MAIL|sh furiously according to the evenly e 3430|80604|5621|2|32|50707.20|0.08|0.00|R|F|1995-01-17|1995-01-28|1995-02-06|NONE|TRUCK|egular instruction 3430|96381|6382|3|41|56472.58|0.06|0.04|R|F|1995-02-18|1995-02-21|1995-03-11|TAKE BACK RETURN|AIR|cuses. silent excuses h 3430|64955|9968|4|50|95997.50|0.01|0.00|R|F|1994-12-15|1995-03-03|1994-12-24|COLLECT COD|REG AIR|ironic theodolites. carefully regular pac 3430|94745|2273|5|5|8698.70|0.05|0.05|A|F|1995-04-02|1995-02-12|1995-04-08|DELIVER IN PERSON|FOB|even accounts haggle slyly bol 3430|170718|5753|6|15|26830.65|0.08|0.07|A|F|1995-02-01|1995-03-12|1995-02-04|COLLECT COD|SHIP|cajole around the accounts. qui 3430|51323|6334|7|23|29309.36|0.09|0.08|A|F|1995-03-06|1995-03-01|1995-03-10|COLLECT COD|MAIL|eas according to the 3431|179230|6782|1|41|53678.43|0.03|0.06|A|F|1993-09-26|1993-10-13|1993-10-22|NONE|AIR| sleep carefully ironically special 3456|110400|5423|1|34|47953.60|0.10|0.06|A|F|1993-08-29|1993-08-26|1993-09-07|TAKE BACK RETURN|SHIP|usy pinto beans b 3457|181229|6266|1|29|37996.38|0.03|0.02|R|F|1995-05-12|1995-07-13|1995-06-05|NONE|TRUCK|refully final excuses wake 3457|105915|5916|2|22|42260.02|0.06|0.01|N|O|1995-06-23|1995-06-16|1995-06-29|NONE|SHIP|packages nag furiously against 3457|108003|514|3|7|7077.00|0.07|0.08|N|O|1995-08-14|1995-07-06|1995-08-18|COLLECT COD|SHIP| pending accounts along the 3457|983|984|4|24|45215.52|0.07|0.07|N|O|1995-08-03|1995-05-30|1995-08-14|TAKE BACK RETURN|REG AIR|tructions haggle alongsid 3457|108595|3616|5|42|67350.78|0.05|0.01|A|F|1995-06-12|1995-06-14|1995-06-14|COLLECT COD|MAIL|riously final instruc 3457|143724|6239|6|45|79547.40|0.08|0.01|N|O|1995-08-12|1995-07-18|1995-08-23|TAKE BACK RETURN|SHIP| packages. care 3457|166641|1674|7|9|15368.76|0.04|0.00|R|F|1995-05-29|1995-06-30|1995-06-12|DELIVER IN PERSON|FOB|quests. foxes sleep quickly 3458|132484|7511|1|48|72791.04|0.06|0.04|R|F|1995-03-17|1995-01-25|1995-03-28|TAKE BACK RETURN|AIR|iously pending dep 3458|49179|4188|2|46|51895.82|0.06|0.06|R|F|1995-03-08|1995-01-21|1995-03-10|TAKE BACK RETURN|SHIP|nod across the boldly even instruct 3458|142061|2062|3|36|39710.16|0.01|0.06|R|F|1995-04-20|1995-02-14|1995-05-09|TAKE BACK RETURN|REG AIR|s lose. blithely ironic requests boost 3458|15253|7755|4|16|18692.00|0.09|0.03|R|F|1995-03-01|1995-02-25|1995-03-06|TAKE BACK RETURN|AIR|s grow carefully. express, final grouc 3458|156396|8912|5|2|2904.78|0.09|0.03|A|F|1995-02-05|1995-02-01|1995-03-07|COLLECT COD|FOB|ironic packages haggle past the furiously 3458|141853|9396|6|6|11369.10|0.09|0.04|A|F|1995-03-10|1995-02-02|1995-03-23|TAKE BACK RETURN|AIR|dolites; regular theodolites cajole 3459|178065|5617|1|31|35434.86|0.06|0.01|A|F|1994-09-05|1994-10-20|1994-10-03|NONE|REG AIR|y regular pain 3459|129270|4295|2|30|38978.10|0.04|0.08|R|F|1994-11-22|1994-09-12|1994-12-11|NONE|REG AIR|nic theodolites; evenly i 3459|40035|2540|3|45|43876.35|0.04|0.05|A|F|1994-07-31|1994-09-09|1994-08-02|TAKE BACK RETURN|REG AIR|ntly speci 3459|68031|8032|4|10|9990.30|0.05|0.06|A|F|1994-10-06|1994-09-16|1994-11-03|TAKE BACK RETURN|REG AIR| furiously silent dolphi 3459|188574|6129|5|10|16625.70|0.02|0.02|R|F|1994-08-01|1994-10-17|1994-08-11|TAKE BACK RETURN|FOB|. blithely ironic pinto beans above 3460|10655|8159|1|40|62626.00|0.10|0.06|N|O|1995-12-28|1995-12-14|1996-01-02|NONE|REG AIR|odolites are slyly bold deposits 3460|73365|5873|2|3|4015.08|0.06|0.00|N|O|1996-01-19|1995-12-28|1996-01-31|COLLECT COD|AIR|er quickly 3460|34658|7162|3|40|63706.00|0.08|0.07|N|O|1995-10-29|1995-11-10|1995-11-24|TAKE BACK RETURN|REG AIR|o the even deposits 3460|94647|9666|4|50|82082.00|0.02|0.07|N|O|1996-01-30|1995-12-10|1996-02-06|DELIVER IN PERSON|SHIP|e slyly about the sly 3460|129391|9392|5|47|66758.33|0.08|0.05|N|O|1995-12-09|1995-11-12|1995-12-22|TAKE BACK RETURN|SHIP|es haggle slyly regular accounts. fi 3460|62084|7097|6|46|48119.68|0.03|0.07|N|O|1996-01-27|1996-01-01|1996-02-01|NONE|TRUCK|uses run among the carefully even deposits 3460|44512|7017|7|28|40782.28|0.00|0.01|N|O|1995-10-28|1995-11-13|1995-11-17|COLLECT COD|SHIP|inal, ironic instructions. carefully 3461|99796|7324|1|49|87993.71|0.06|0.06|A|F|1993-03-09|1993-04-16|1993-03-13|DELIVER IN PERSON|RAIL|ual request 3461|62417|2418|2|27|37244.07|0.06|0.06|A|F|1993-02-10|1993-03-02|1993-03-04|COLLECT COD|SHIP|ely unusual deposits. quickly ir 3461|38930|1434|3|44|82232.92|0.09|0.06|A|F|1993-05-20|1993-04-03|1993-05-27|COLLECT COD|RAIL| haggle quickly even ideas. fin 3461|94288|6798|4|41|52573.48|0.09|0.02|R|F|1993-02-19|1993-04-20|1993-02-21|NONE|TRUCK|heodolites. blithely ironi 3461|89771|4788|5|16|28172.32|0.08|0.06|A|F|1993-05-09|1993-04-29|1993-05-26|TAKE BACK RETURN|TRUCK| pending deposi 3461|166821|4370|6|24|45307.68|0.10|0.00|A|F|1993-06-01|1993-03-12|1993-06-20|TAKE BACK RETURN|MAIL|thely. carefully re 3462|150880|8426|1|4|7723.52|0.09|0.04|N|O|1997-06-12|1997-07-31|1997-06-16|COLLECT COD|RAIL|ackages. fu 3462|39432|4439|2|43|58971.49|0.08|0.03|N|O|1997-08-01|1997-07-18|1997-08-29|NONE|RAIL| carefully. final, final ideas sleep slyly 3462|128298|811|3|6|7957.74|0.05|0.04|N|O|1997-06-02|1997-08-09|1997-06-30|NONE|RAIL|iously regular fo 3462|98093|5621|4|2|2182.18|0.09|0.07|N|O|1997-09-10|1997-08-08|1997-09-19|NONE|AIR|nic packages. even accounts alongside 3462|37353|4863|5|14|18064.90|0.01|0.02|N|O|1997-05-31|1997-07-05|1997-06-24|COLLECT COD|MAIL|yly. blithely bold theodolites wa 3463|60779|3286|1|45|78289.65|0.02|0.02|A|F|1993-10-30|1993-11-04|1993-11-08|DELIVER IN PERSON|FOB|nts are slyly 3463|97924|2943|2|43|82642.56|0.04|0.02|A|F|1993-10-28|1993-09-24|1993-11-03|DELIVER IN PERSON|FOB| across the 3488|159702|4733|1|1|1761.70|0.04|0.01|A|F|1995-03-06|1995-02-16|1995-03-23|DELIVER IN PERSON|FOB| final excuses. carefully even waters hagg 3488|103455|8476|2|48|70005.60|0.00|0.03|A|F|1995-03-29|1995-03-26|1995-04-28|COLLECT COD|SHIP|sly? final requests 3488|159080|9081|3|11|12529.88|0.03|0.08|R|F|1995-03-25|1995-02-08|1995-04-16|COLLECT COD|TRUCK|unusual re 3488|41028|3533|4|12|11628.24|0.05|0.07|R|F|1995-04-27|1995-02-16|1995-05-09|DELIVER IN PERSON|RAIL|e slyly; furiously final packages wak 3488|155132|163|5|18|21368.34|0.09|0.06|A|F|1995-03-18|1995-03-19|1995-03-29|DELIVER IN PERSON|FOB|s the carefully r 3489|185663|8182|1|19|33224.54|0.09|0.05|A|F|1993-07-31|1993-10-26|1993-08-15|NONE|SHIP|c deposits alongside of the pending, fu 3489|28091|594|2|46|46878.14|0.00|0.00|A|F|1993-08-02|1993-10-09|1993-08-10|TAKE BACK RETURN|TRUCK|xcuses? quickly stealthy dependenci 3490|91360|8888|1|43|58108.48|0.05|0.05|N|O|1997-08-04|1997-08-06|1997-08-14|TAKE BACK RETURN|SHIP|. even requests cajol 3490|85447|2972|2|50|71622.00|0.05|0.07|N|O|1997-06-27|1997-08-15|1997-06-28|NONE|RAIL| haggle carefu 3490|92453|9981|3|8|11563.60|0.10|0.04|N|O|1997-08-11|1997-07-25|1997-08-28|COLLECT COD|MAIL|inal deposits use furiousl 3491|153491|6007|1|28|43245.72|0.04|0.03|N|O|1998-09-29|1998-09-08|1998-10-23|COLLECT COD|FOB|ccounts. sly 3491|121773|1774|2|22|39484.94|0.08|0.02|N|O|1998-08-19|1998-08-22|1998-09-03|TAKE BACK RETURN|REG AIR| grow against the boldly pending pinto bea 3492|155860|5861|1|3|5747.58|0.02|0.08|R|F|1994-11-26|1994-12-28|1994-12-19|COLLECT COD|REG AIR|the deposits. carefully 3492|125526|3063|2|7|10860.64|0.04|0.00|R|F|1995-03-10|1995-01-03|1995-03-16|COLLECT COD|FOB|thely regular dolphi 3492|108869|8870|3|34|63847.24|0.05|0.06|A|F|1994-12-07|1994-12-29|1994-12-24|COLLECT COD|AIR| unusual requests. ir 3492|146899|4442|4|30|58376.70|0.02|0.06|A|F|1995-01-29|1995-01-02|1995-02-13|DELIVER IN PERSON|MAIL| detect furiously permanent, unusual accou 3492|121499|6524|5|47|71463.03|0.09|0.07|R|F|1995-03-24|1994-12-28|1995-03-29|NONE|REG AIR|deposits. quickly express 3492|21118|3621|6|47|48838.17|0.04|0.07|R|F|1994-12-12|1995-01-18|1994-12-26|COLLECT COD|RAIL|ronic instructions u 3493|92506|7525|1|31|46453.50|0.06|0.07|R|F|1993-10-22|1993-10-12|1993-11-07|DELIVER IN PERSON|REG AIR|ructions. slyly regular accounts across the 3493|131994|1995|2|10|20259.90|0.02|0.06|R|F|1993-08-27|1993-10-07|1993-09-23|COLLECT COD|TRUCK|hall have to integ 3494|116147|8659|1|40|46525.60|0.05|0.04|R|F|1993-07-10|1993-06-01|1993-07-25|TAKE BACK RETURN|TRUCK|lites haggle furiously about the fin 3494|74124|4125|2|23|25256.76|0.10|0.01|A|F|1993-06-19|1993-06-04|1993-07-14|NONE|FOB|osits nag 3494|197161|4719|3|40|50326.40|0.02|0.08|A|F|1993-05-30|1993-07-02|1993-06-20|TAKE BACK RETURN|MAIL|uests cajole blithely 3494|76076|6077|4|30|31562.10|0.04|0.03|R|F|1993-07-01|1993-06-08|1993-07-15|TAKE BACK RETURN|TRUCK|ns are quickly regular, 3495|27321|9824|1|20|24966.40|0.10|0.03|N|O|1996-04-24|1996-05-18|1996-05-01|TAKE BACK RETURN|RAIL|posits are carefully; forges cajole qui 3495|172119|9671|2|24|28586.64|0.05|0.02|N|O|1996-03-22|1996-04-10|1996-04-07|DELIVER IN PERSON|RAIL|ic, final pains along the even request 3495|198597|8598|3|16|27129.44|0.08|0.02|N|O|1996-03-30|1996-04-02|1996-04-12|TAKE BACK RETURN|AIR|y bold dependencies; blithely idle sautern 3520|27999|5506|1|30|57809.70|0.04|0.02|N|O|1997-11-11|1997-10-02|1997-12-06|COLLECT COD|SHIP|deas should solve blithely among the ironi 3520|166759|1792|2|38|69378.50|0.00|0.04|N|O|1997-08-14|1997-10-26|1997-09-09|NONE|RAIL|yly final packages according to the quickl 3520|105258|7769|3|5|6316.25|0.01|0.02|N|O|1997-11-13|1997-09-22|1997-12-09|NONE|MAIL|ly even ideas haggle 3520|63745|3746|4|41|70058.34|0.01|0.01|N|O|1997-08-06|1997-09-20|1997-08-20|TAKE BACK RETURN|AIR| carefully pendi 3520|162174|7207|5|35|43265.95|0.02|0.02|N|O|1997-09-16|1997-09-03|1997-09-24|DELIVER IN PERSON|FOB|s nag carefully. sometimes unusual account 3521|58448|3459|1|48|67509.12|0.09|0.03|A|F|1993-01-03|1992-12-31|1993-01-22|NONE|AIR|ses use. furiously express ideas wake f 3521|130897|898|2|2|3855.78|0.05|0.06|R|F|1993-01-29|1992-12-20|1993-02-23|NONE|MAIL|refully duri 3521|177971|489|3|38|77860.86|0.00|0.08|A|F|1993-02-15|1992-12-10|1993-03-10|COLLECT COD|FOB|ges hang q 3521|143978|9007|4|26|52571.22|0.02|0.08|R|F|1993-01-04|1993-01-20|1993-01-17|DELIVER IN PERSON|AIR|onic dependencies haggle. fur 3521|35052|59|5|28|27637.40|0.10|0.01|A|F|1993-01-06|1993-01-22|1993-02-02|TAKE BACK RETURN|FOB|e slyly above the slyly final 3522|3671|8672|1|6|9448.02|0.08|0.03|A|F|1995-01-21|1994-12-09|1995-01-23|NONE|SHIP|tes snooze 3522|86438|8947|2|48|68372.64|0.00|0.03|R|F|1994-12-05|1994-10-30|1994-12-26|TAKE BACK RETURN|SHIP|ve the quickly special packages 3522|156693|1724|3|46|80485.74|0.09|0.02|A|F|1994-11-12|1994-11-30|1994-11-20|NONE|AIR|d the express, silent foxes. blit 3522|129818|4843|4|7|12934.67|0.10|0.02|A|F|1994-10-31|1994-11-19|1994-11-28|NONE|TRUCK|e stealthil 3522|49144|6657|5|27|29514.78|0.02|0.05|R|F|1994-11-29|1994-12-15|1994-12-08|COLLECT COD|REG AIR|ic tithes. car 3522|157970|5516|6|18|36503.46|0.01|0.03|A|F|1994-11-16|1994-10-29|1994-11-29|COLLECT COD|RAIL|sits wake carefully pen 3523|24474|4475|1|15|20977.05|0.06|0.02|N|O|1998-06-26|1998-05-22|1998-07-24|COLLECT COD|REG AIR|se slyly pending, sp 3523|132127|9667|2|4|4636.48|0.03|0.06|N|O|1998-05-08|1998-05-18|1998-05-25|TAKE BACK RETURN|MAIL|ts. final accounts detect furiously along 3523|49146|1651|3|24|26283.36|0.07|0.04|N|O|1998-08-02|1998-06-22|1998-08-27|COLLECT COD|FOB|ke according to the doggedly re 3523|191238|3758|4|36|47852.28|0.06|0.08|N|O|1998-05-26|1998-06-04|1998-06-25|DELIVER IN PERSON|SHIP|accounts. fluffily regu 3523|133109|3110|5|48|54820.80|0.00|0.01|N|O|1998-07-22|1998-06-25|1998-08-19|DELIVER IN PERSON|AIR| regular requests 3524|136119|6120|1|5|5775.55|0.01|0.04|R|F|1992-05-23|1992-07-25|1992-06-19|DELIVER IN PERSON|RAIL|ts whithout the bold depende 3524|142948|7977|2|17|33845.98|0.09|0.08|A|F|1992-09-01|1992-07-17|1992-09-05|DELIVER IN PERSON|FOB|g, final epitaphs about the pinto 3525|45478|5479|1|12|17081.64|0.01|0.03|N|O|1996-03-08|1996-03-18|1996-03-16|NONE|TRUCK|lar excuses wake carefull 3525|137210|2237|2|27|33674.67|0.03|0.03|N|O|1995-12-30|1996-01-23|1996-01-02|DELIVER IN PERSON|SHIP|y slyly special asymptotes 3525|74397|6905|3|31|42513.09|0.00|0.03|N|O|1996-03-08|1996-02-27|1996-03-13|COLLECT COD|TRUCK|he careful 3525|183399|954|4|28|41506.92|0.03|0.02|N|O|1996-01-22|1996-02-08|1996-01-27|COLLECT COD|FOB| nag according 3526|97645|7646|1|11|18069.04|0.02|0.03|R|F|1995-05-23|1995-05-28|1995-05-24|NONE|TRUCK|ges. furiously regular d 3526|116383|3917|2|23|32185.74|0.03|0.04|A|F|1995-05-01|1995-05-31|1995-05-25|DELIVER IN PERSON|FOB|special, regular packages cajole. 3526|32541|51|3|20|29470.80|0.05|0.08|N|F|1995-06-16|1995-04-26|1995-06-22|DELIVER IN PERSON|REG AIR|kages. bold, special requests detect sl 3527|101388|6409|1|47|65300.86|0.07|0.02|N|O|1997-07-14|1997-07-29|1997-07-21|DELIVER IN PERSON|RAIL|unts. express re 3527|25083|2590|2|33|33266.64|0.01|0.02|N|O|1997-09-25|1997-09-17|1997-10-12|NONE|FOB|kly alongside of 3527|161298|8847|3|50|67964.50|0.09|0.07|N|O|1997-07-17|1997-08-03|1997-07-29|DELIVER IN PERSON|SHIP|e even accounts was about th 3527|127163|9676|4|17|20232.72|0.02|0.05|N|O|1997-07-30|1997-09-01|1997-08-17|COLLECT COD|MAIL|ular instruction 3552|196483|6484|1|18|28430.64|0.01|0.07|N|O|1997-08-11|1997-07-14|1997-08-15|DELIVER IN PERSON|TRUCK|s deposits against the blithely unusual pin 3552|89318|1827|2|44|57521.64|0.01|0.00|N|O|1997-08-08|1997-06-15|1997-08-29|COLLECT COD|FOB|ns after the blithely reg 3552|160455|8004|3|36|54556.20|0.04|0.08|N|O|1997-06-29|1997-06-24|1997-07-21|COLLECT COD|TRUCK|ly regular theodolites. fin 3553|142076|4591|1|4|4472.28|0.05|0.01|R|F|1994-06-13|1994-07-10|1994-07-03|COLLECT COD|RAIL|olites boost bli 3553|64118|6625|2|26|28134.86|0.05|0.08|A|F|1994-08-06|1994-07-30|1994-08-23|DELIVER IN PERSON|MAIL|fily special p 3553|21291|8798|3|18|21821.22|0.04|0.03|A|F|1994-07-03|1994-06-30|1994-07-07|COLLECT COD|RAIL|. quickly ironic 3553|31009|3513|4|40|37600.00|0.06|0.00|A|F|1994-09-14|1994-06-26|1994-09-25|NONE|RAIL| slyly pending asymptotes against the furi 3553|156020|1051|5|36|38736.72|0.06|0.08|R|F|1994-08-12|1994-06-25|1994-09-06|DELIVER IN PERSON|TRUCK| realms. pending, bold theodolites 3554|174151|6669|1|32|39204.80|0.01|0.05|N|O|1995-09-28|1995-09-01|1995-10-07|NONE|RAIL|. blithely ironic t 3554|144071|4072|2|18|20071.26|0.03|0.00|N|O|1995-09-11|1995-08-12|1995-10-04|DELIVER IN PERSON|REG AIR| haggle. furiously fluffy requests ac 3554|191547|1548|3|41|67180.14|0.02|0.01|N|O|1995-07-13|1995-08-28|1995-07-27|DELIVER IN PERSON|MAIL|ent dependencies. sly 3555|165543|576|1|11|17693.94|0.05|0.02|N|O|1996-09-25|1996-10-01|1996-10-03|NONE|FOB|oost caref 3555|78411|8412|2|15|20841.15|0.03|0.08|N|O|1996-07-13|1996-09-01|1996-08-02|TAKE BACK RETURN|RAIL|y across the pending a 3555|42078|9591|3|25|25501.75|0.09|0.07|N|O|1996-10-01|1996-08-23|1996-10-24|TAKE BACK RETURN|MAIL|sual packages. quickly 3555|4429|4430|4|19|25334.98|0.00|0.05|N|O|1996-09-08|1996-09-14|1996-10-01|COLLECT COD|REG AIR|leep special theodolit 3555|32976|7983|5|29|55360.13|0.07|0.04|N|O|1996-08-02|1996-09-04|1996-08-08|DELIVER IN PERSON|TRUCK|deas. carefully s 3555|27342|9845|6|33|41888.22|0.04|0.08|N|O|1996-09-20|1996-09-23|1996-10-05|TAKE BACK RETURN|AIR|fluffily regular a 3555|125254|279|7|9|11513.25|0.07|0.02|N|O|1996-10-13|1996-10-02|1996-10-22|NONE|SHIP|are. slyly final foxes acro 3556|141443|3958|1|45|66799.80|0.05|0.06|A|F|1992-10-14|1992-12-21|1992-10-16|NONE|TRUCK|ckages boost quickl 3556|30350|5357|2|43|55055.05|0.02|0.06|R|F|1993-01-18|1992-11-09|1993-02-04|NONE|FOB|wake carefull 3556|86856|6857|3|28|51599.80|0.10|0.04|A|F|1993-01-06|1992-11-27|1993-01-16|NONE|MAIL|refully final instructions? ironic packa 3557|174245|1797|1|41|54088.84|0.01|0.07|R|F|1993-01-30|1992-12-31|1993-02-18|COLLECT COD|FOB|ideas breach c 3557|128842|8843|2|37|69221.08|0.03|0.05|R|F|1993-02-16|1993-01-05|1993-03-15|DELIVER IN PERSON|RAIL|gside of the ca 3558|86334|6335|1|8|10562.64|0.01|0.03|N|O|1996-05-31|1996-05-26|1996-06-25|COLLECT COD|AIR|? even requests sle 3558|9567|7068|2|28|41343.68|0.02|0.08|N|O|1996-06-02|1996-04-18|1996-06-24|COLLECT COD|TRUCK|l deposits 3558|186452|1489|3|3|4615.35|0.03|0.06|N|O|1996-05-19|1996-04-28|1996-05-26|DELIVER IN PERSON|RAIL|l, final deposits haggle. fina 3558|90560|8088|4|22|34112.32|0.06|0.03|N|O|1996-04-27|1996-04-19|1996-04-30|DELIVER IN PERSON|SHIP|refully ironic theodolites are fu 3558|28850|3855|5|38|67596.30|0.03|0.08|N|O|1996-05-29|1996-05-02|1996-06-09|COLLECT COD|RAIL|refully permanently iron 3558|71821|6836|6|17|30477.94|0.07|0.07|N|O|1996-03-14|1996-05-04|1996-04-05|NONE|RAIL|ithely unusual packa 3559|89591|9592|1|29|45837.11|0.00|0.07|R|F|1992-12-10|1992-12-03|1992-12-20|COLLECT COD|REG AIR|l, regular accounts wake flu 3584|10196|5199|1|4|4424.76|0.04|0.08|N|O|1997-08-16|1997-10-31|1997-08-28|DELIVER IN PERSON|TRUCK|nal packag 3584|159495|2011|2|23|35753.27|0.00|0.03|N|O|1997-09-10|1997-10-15|1997-09-30|COLLECT COD|TRUCK|l platelets until the asymptotes 3584|23297|3298|3|6|7321.74|0.03|0.06|N|O|1997-10-28|1997-11-09|1997-11-24|TAKE BACK RETURN|MAIL|deposits across the 3584|145596|3139|4|11|18057.49|0.06|0.02|N|O|1997-11-27|1997-10-15|1997-12-08|NONE|REG AIR|lithely slyly 3584|17924|2927|5|39|71834.88|0.09|0.07|N|O|1997-09-20|1997-10-31|1997-10-06|COLLECT COD|AIR|eposits. carefu 3585|121713|6738|1|21|36428.91|0.05|0.04|A|F|1994-12-04|1994-12-25|1995-01-01|TAKE BACK RETURN|TRUCK|ounts use. express, final platelets us 3585|18535|8536|2|40|58141.20|0.03|0.00|R|F|1995-01-22|1995-01-17|1995-02-07|TAKE BACK RETURN|RAIL|elets affix. even asymptotes play care 3585|111350|8884|3|11|14974.85|0.01|0.04|R|F|1995-01-04|1995-02-14|1995-01-15|NONE|MAIL|even packages 3585|47911|2920|4|33|61344.03|0.08|0.08|A|F|1994-12-14|1995-01-19|1994-12-22|NONE|RAIL|ironic dependencies serve furi 3585|24543|2050|5|13|19078.02|0.06|0.07|R|F|1995-03-15|1995-01-22|1995-03-17|DELIVER IN PERSON|AIR|ccording to the foxes. slyly iro 3585|93452|8471|6|7|10118.15|0.10|0.02|A|F|1994-12-13|1995-01-20|1995-01-05|TAKE BACK RETURN|TRUCK|dependencies sleep un 3585|41685|9198|7|45|73200.60|0.03|0.00|A|F|1995-01-20|1995-02-19|1995-02-11|DELIVER IN PERSON|MAIL|are blithely c 3586|193699|8738|1|2|3585.38|0.03|0.08|R|F|1994-02-10|1994-01-07|1994-03-03|DELIVER IN PERSON|RAIL|he even, unusual decoy 3586|83314|3315|2|29|37621.99|0.04|0.07|R|F|1994-03-06|1994-03-02|1994-03-13|DELIVER IN PERSON|RAIL| slyly unusual i 3586|57422|2433|3|2|2758.84|0.03|0.06|R|F|1994-03-22|1994-02-20|1994-04-08|NONE|REG AIR|unts. slyly final ideas agai 3586|83722|3723|4|33|56288.76|0.06|0.01|R|F|1994-01-24|1994-02-09|1994-02-07|NONE|TRUCK|refully across the fur 3586|107949|460|5|8|15655.52|0.06|0.02|A|F|1994-03-29|1994-02-26|1994-04-02|NONE|FOB|theodolites hagg 3586|98598|1108|6|8|12772.72|0.09|0.01|A|F|1994-03-18|1994-01-17|1994-04-06|DELIVER IN PERSON|RAIL| ironic pinto beans cajole carefully theo 3586|122699|2700|7|33|56815.77|0.05|0.04|A|F|1994-02-11|1994-01-15|1994-03-03|NONE|REG AIR|iously regular pinto beans integrate 3587|196712|1751|1|5|9043.55|0.09|0.07|N|O|1996-09-03|1996-07-05|1996-09-11|DELIVER IN PERSON|SHIP|ithely regular decoys above the 3587|131240|8780|2|48|61019.52|0.00|0.03|N|O|1996-08-02|1996-07-02|1996-08-05|TAKE BACK RETURN|MAIL|beans. blithely final depe 3587|150350|7896|3|36|50412.60|0.05|0.05|N|O|1996-07-26|1996-06-16|1996-08-23|TAKE BACK RETURN|MAIL|ully regular excuse 3587|123620|6133|4|31|50952.22|0.03|0.01|N|O|1996-07-21|1996-07-01|1996-07-23|COLLECT COD|SHIP|press fluffily regul 3587|69585|4598|5|12|18654.96|0.06|0.03|N|O|1996-08-30|1996-07-04|1996-09-22|DELIVER IN PERSON|RAIL|g the even pinto beans. special, 3587|106080|1101|6|16|17377.28|0.01|0.03|N|O|1996-05-11|1996-06-19|1996-06-04|COLLECT COD|FOB|y ruthless dolphins to 3587|73225|747|7|23|27559.06|0.07|0.05|N|O|1996-08-30|1996-07-01|1996-09-10|COLLECT COD|FOB|l multipliers sleep theodolites-- slyly 3588|90135|7663|1|28|31503.64|0.04|0.08|R|F|1995-05-03|1995-05-03|1995-05-14|DELIVER IN PERSON|TRUCK|special pinto beans cajole slyly. slyly 3588|87759|268|2|6|10480.50|0.06|0.08|A|F|1995-04-09|1995-05-30|1995-04-10|TAKE BACK RETURN|MAIL|s. fluffily fluf 3588|158716|8717|3|45|79861.95|0.04|0.02|R|F|1995-05-07|1995-05-04|1995-05-28|TAKE BACK RETURN|TRUCK|ecial pains integrate blithely. reques 3588|126284|3821|4|22|28826.16|0.05|0.00|A|F|1995-04-08|1995-05-06|1995-04-27|NONE|RAIL|inal accounts. pending, bo 3588|54948|7454|5|28|53282.32|0.03|0.03|A|F|1995-04-23|1995-05-25|1995-04-28|DELIVER IN PERSON|TRUCK| express sheaves. unusual theodo 3588|109433|1944|6|37|53369.91|0.08|0.04|N|F|1995-06-17|1995-05-25|1995-06-24|TAKE BACK RETURN|RAIL|xcuses sleep quickly along th 3588|38463|967|7|46|64467.16|0.08|0.07|A|F|1995-06-06|1995-05-08|1995-06-08|NONE|AIR| slyly ironic deposits sublate ab 3589|36519|4029|1|42|61131.42|0.08|0.08|R|F|1994-08-11|1994-07-17|1994-08-23|DELIVER IN PERSON|AIR|he blithely unusual pac 3590|175980|8498|1|10|20559.80|0.08|0.00|N|O|1995-07-17|1995-06-26|1995-08-12|TAKE BACK RETURN|SHIP|t the quickly ironic 3590|94336|4337|2|19|25276.27|0.03|0.03|N|O|1995-08-02|1995-06-20|1995-08-08|NONE|SHIP|special pinto beans. blithely reg 3590|95583|602|3|43|67878.94|0.07|0.06|N|O|1995-07-12|1995-07-25|1995-07-16|DELIVER IN PERSON|SHIP|s could have to use 3590|55334|2850|4|26|33522.58|0.01|0.03|N|O|1995-07-08|1995-06-17|1995-08-02|DELIVER IN PERSON|SHIP|arefully along th 3590|190226|227|5|37|48700.14|0.00|0.08|N|O|1995-09-01|1995-06-29|1995-09-10|NONE|SHIP|ccounts above the silent waters thrash f 3590|118843|8844|6|31|57717.04|0.03|0.01|N|O|1995-06-24|1995-07-12|1995-06-25|DELIVER IN PERSON|REG AIR|ve furiously final instructions. slyly regu 3590|193517|8556|7|44|70862.44|0.05|0.04|N|F|1995-06-07|1995-06-15|1995-06-27|NONE|MAIL|s sleep after the regular platelets. blit 3591|28873|3878|1|21|37839.27|0.03|0.03|A|F|1994-02-25|1994-02-02|1994-03-05|DELIVER IN PERSON|TRUCK|structions against 3591|68232|3245|2|24|28805.52|0.04|0.04|R|F|1993-12-26|1994-01-07|1994-01-25|COLLECT COD|FOB|ages. slyly regular dependencies cajo 3591|163239|788|3|4|5208.92|0.01|0.03|A|F|1994-04-04|1994-02-19|1994-05-02|DELIVER IN PERSON|RAIL|he final packages. deposits serve quick 3591|152523|2524|4|49|77200.48|0.01|0.00|A|F|1994-03-21|1994-01-26|1994-03-28|COLLECT COD|AIR| mold slyly. bl 3616|196546|9066|1|30|49276.20|0.01|0.00|A|F|1994-05-05|1994-04-24|1994-05-12|TAKE BACK RETURN|FOB|ly ironic accounts unwind b 3616|137984|7985|2|28|56615.44|0.08|0.06|R|F|1994-02-20|1994-04-18|1994-03-05|DELIVER IN PERSON|REG AIR|ironic packages. furiously ev 3617|116065|6066|1|46|49728.76|0.03|0.02|N|O|1996-05-19|1996-05-14|1996-06-11|NONE|RAIL|ar theodolites. regu 3617|97942|7943|2|16|31039.04|0.05|0.02|N|O|1996-05-08|1996-06-03|1996-05-19|COLLECT COD|RAIL| slyly on th 3617|97724|5252|3|32|55095.04|0.00|0.06|N|O|1996-04-20|1996-06-07|1996-05-19|DELIVER IN PERSON|MAIL|uriously against the express accounts. ex 3617|40706|8219|4|22|36227.40|0.10|0.05|N|O|1996-07-11|1996-05-02|1996-07-25|NONE|REG AIR|uffily even accounts. packages sleep blithe 3617|136286|1313|5|11|14545.08|0.08|0.05|N|O|1996-07-16|1996-04-23|1996-07-28|COLLECT COD|MAIL|ly quickly even requests. final 3618|139641|4668|1|38|63864.32|0.08|0.00|N|O|1997-12-22|1998-02-23|1998-01-03|TAKE BACK RETURN|TRUCK|nts haggle fluffily above the regular 3618|143397|3398|2|48|69138.72|0.04|0.00|N|O|1998-03-12|1998-02-13|1998-03-29|DELIVER IN PERSON|TRUCK|tructions atop the ironi 3618|62032|4539|3|24|23856.72|0.01|0.04|N|O|1998-01-26|1998-01-15|1998-02-17|TAKE BACK RETURN|AIR|xpress acc 3618|160418|419|4|26|38438.66|0.01|0.05|N|O|1998-03-23|1998-01-24|1998-04-15|DELIVER IN PERSON|AIR|iously regular deposits cajole ruthless 3619|95741|5742|1|49|85100.26|0.01|0.08|N|O|1997-01-22|1996-12-21|1997-02-17|TAKE BACK RETURN|MAIL| waters. furiously even deposits 3619|115622|8134|2|27|44215.74|0.08|0.04|N|O|1996-12-12|1997-01-18|1996-12-18|TAKE BACK RETURN|SHIP|pecial accounts haggle care 3619|47022|4535|3|46|44574.92|0.08|0.03|N|O|1997-01-31|1997-01-27|1997-02-11|NONE|SHIP|press, expres 3619|92969|7988|4|18|35315.28|0.04|0.02|N|O|1997-03-18|1996-12-24|1997-03-21|COLLECT COD|AIR|eodolites 3619|119269|6803|5|38|48953.88|0.05|0.08|N|O|1996-12-08|1997-02-03|1997-01-07|NONE|RAIL|theodolites detect abo 3619|151951|1952|6|43|86126.85|0.01|0.01|N|O|1997-01-25|1997-01-06|1997-02-07|COLLECT COD|RAIL| bold, even 3620|58688|1194|1|41|67513.88|0.03|0.08|N|O|1997-03-21|1997-04-20|1997-03-30|COLLECT COD|FOB|t attainments cajole qui 3620|166574|1607|2|16|26249.12|0.00|0.06|N|O|1997-05-17|1997-05-08|1997-06-03|COLLECT COD|SHIP|s. even, pending in 3621|16475|6476|1|29|40352.63|0.02|0.06|A|F|1993-08-03|1993-07-08|1993-08-10|DELIVER IN PERSON|FOB|al requests. fl 3621|92329|4839|2|13|17177.16|0.09|0.04|R|F|1993-08-30|1993-06-30|1993-09-01|NONE|REG AIR|r the unusual packages. brave theodoli 3621|163518|1067|3|45|71167.95|0.07|0.07|R|F|1993-08-09|1993-06-18|1993-09-05|DELIVER IN PERSON|AIR| doubt about the bold deposits. carefully 3621|43242|755|4|20|23704.80|0.05|0.04|R|F|1993-05-27|1993-07-04|1993-06-22|TAKE BACK RETURN|SHIP|gular accounts use carefully with 3622|174754|4755|1|47|85951.25|0.09|0.00|N|O|1996-02-24|1996-02-22|1996-03-12|TAKE BACK RETURN|TRUCK|are careful 3622|88392|901|2|4|5521.56|0.04|0.04|N|O|1996-02-03|1996-02-19|1996-02-16|TAKE BACK RETURN|TRUCK|lithely brave foxes. furi 3622|189480|9481|3|46|72196.08|0.07|0.07|N|O|1995-12-18|1996-01-23|1996-01-12|TAKE BACK RETURN|AIR|sits wake. blithe 3622|176517|6518|4|9|14341.59|0.08|0.05|N|O|1995-12-12|1996-02-09|1995-12-13|TAKE BACK RETURN|SHIP|arefully. furiously regular ideas n 3623|79831|2339|1|32|57946.56|0.05|0.00|N|O|1997-04-18|1997-03-15|1997-05-09|COLLECT COD|SHIP| courts. furiously regular ideas b 3623|116203|1226|2|33|40233.60|0.08|0.01|N|O|1997-03-17|1997-02-13|1997-04-02|TAKE BACK RETURN|TRUCK|odolites. blithely spe 3623|23025|532|3|21|19908.42|0.02|0.02|N|O|1997-01-19|1997-03-18|1997-01-24|NONE|FOB|ress ideas are furio 3623|164747|9780|4|42|76093.08|0.05|0.06|N|O|1997-01-11|1997-03-24|1997-01-21|COLLECT COD|RAIL|g to the slyly regular packa 3623|87761|5286|5|30|52462.80|0.10|0.04|N|O|1997-04-04|1997-03-03|1997-05-01|NONE|RAIL| ironic somas sleep fluffily 3623|185158|195|6|7|8702.05|0.01|0.02|N|O|1997-01-05|1997-03-26|1997-01-26|NONE|TRUCK|aves. slyly special packages cajole. fu 3623|139856|2370|7|13|24646.05|0.03|0.08|N|O|1997-01-02|1997-02-26|1997-01-26|DELIVER IN PERSON|SHIP|deas. furiously expres 3648|143090|3091|1|16|18129.44|0.02|0.06|A|F|1993-08-14|1993-08-14|1993-08-15|COLLECT COD|FOB|s nag packages. 3648|104769|2300|2|30|53212.80|0.00|0.01|R|F|1993-08-31|1993-09-06|1993-09-06|DELIVER IN PERSON|FOB| above the somas boost furious 3648|45851|5852|3|34|61092.90|0.10|0.00|A|F|1993-08-21|1993-07-25|1993-09-15|DELIVER IN PERSON|FOB| deposits are furiously. careful, 3648|12026|7029|4|16|15008.32|0.06|0.03|R|F|1993-07-27|1993-08-26|1993-08-24|DELIVER IN PERSON|FOB|uriously stealthy deposits haggle furi 3648|116962|4496|5|25|49474.00|0.06|0.03|R|F|1993-08-15|1993-08-25|1993-09-09|TAKE BACK RETURN|TRUCK|s requests. silent asymp 3648|168406|8407|6|14|20641.60|0.08|0.06|R|F|1993-10-02|1993-08-26|1993-10-09|COLLECT COD|AIR|sly pending excuses. carefully i 3648|194110|4111|7|49|59001.39|0.09|0.03|R|F|1993-06-27|1993-07-27|1993-07-24|TAKE BACK RETURN|FOB|egular instructions. slyly regular pinto 3649|4730|4731|1|25|40868.25|0.10|0.04|A|F|1994-10-27|1994-08-23|1994-11-05|TAKE BACK RETURN|TRUCK|special re 3649|88968|3985|2|23|45010.08|0.08|0.00|R|F|1994-09-26|1994-10-01|1994-09-28|NONE|REG AIR|rs promise blithe 3649|69238|4251|3|14|16901.22|0.02|0.04|A|F|1994-09-19|1994-08-17|1994-10-12|DELIVER IN PERSON|TRUCK|ithely bold accounts wake 3649|75499|3021|4|40|58979.60|0.00|0.08|R|F|1994-07-20|1994-08-30|1994-08-14|TAKE BACK RETURN|RAIL|luffy somas sleep quickly-- ironic de 3649|99669|9670|5|24|40047.84|0.05|0.03|A|F|1994-07-07|1994-08-20|1994-07-27|TAKE BACK RETURN|FOB|c accounts. quickly final theodo 3649|121063|1064|6|3|3252.18|0.10|0.04|A|F|1994-07-17|1994-08-10|1994-08-03|NONE|FOB|lly bold requests nag; 3650|135535|8049|1|30|47115.90|0.10|0.00|A|F|1992-08-26|1992-07-05|1992-09-01|DELIVER IN PERSON|SHIP|ckly special platelets. furiously sil 3650|127935|7936|2|43|84405.99|0.05|0.05|A|F|1992-09-07|1992-08-12|1992-09-10|COLLECT COD|TRUCK|gside of the quick 3650|1634|9135|3|1|1535.63|0.04|0.06|A|F|1992-06-23|1992-07-18|1992-07-08|NONE|REG AIR|re about the pinto 3650|62057|4564|4|31|31590.55|0.10|0.08|R|F|1992-06-15|1992-07-01|1992-07-15|DELIVER IN PERSON|RAIL| against the ironic accounts cajol 3650|186345|6346|5|19|27195.46|0.05|0.04|R|F|1992-08-29|1992-08-09|1992-09-21|DELIVER IN PERSON|AIR|y even forges. fluffily furious accounts 3650|93525|1053|6|27|41000.04|0.07|0.08|A|F|1992-07-03|1992-07-23|1992-07-13|COLLECT COD|MAIL|ular requests snooze fluffily regular pi 3650|69009|4022|7|43|42054.00|0.10|0.07|A|F|1992-06-25|1992-07-09|1992-07-22|DELIVER IN PERSON|RAIL|structions use caref 3651|18097|5601|1|20|20301.80|0.01|0.04|N|O|1998-06-10|1998-06-06|1998-06-23|NONE|SHIP|tect quickly among the r 3651|154336|1882|2|24|33367.92|0.09|0.04|N|O|1998-06-22|1998-07-17|1998-07-10|DELIVER IN PERSON|RAIL|excuses haggle according to th 3651|112460|7483|3|41|60370.86|0.00|0.05|N|O|1998-05-10|1998-07-09|1998-05-13|NONE|RAIL|blithely. furiously 3651|109126|4147|4|27|30648.24|0.05|0.03|N|O|1998-05-03|1998-06-30|1998-05-05|DELIVER IN PERSON|RAIL| sleep blithely furiously do 3652|179779|7331|1|24|44610.48|0.05|0.03|N|O|1997-06-07|1997-04-07|1997-06-12|COLLECT COD|MAIL|the final p 3652|136489|6490|2|37|56442.76|0.02|0.05|N|O|1997-05-11|1997-04-06|1997-06-05|COLLECT COD|MAIL|osits haggle carefu 3652|162640|189|3|39|66402.96|0.01|0.02|N|O|1997-03-10|1997-04-03|1997-03-21|NONE|REG AIR|y express instructions. un 3652|79671|4686|4|1|1650.67|0.01|0.04|N|O|1997-04-20|1997-05-03|1997-05-18|DELIVER IN PERSON|SHIP| bold dependencies sublate. r 3653|144753|2296|1|38|68314.50|0.08|0.05|A|F|1994-06-26|1994-05-13|1994-07-13|NONE|REG AIR|ainst the 3653|63021|8034|2|29|28536.58|0.07|0.01|A|F|1994-04-11|1994-06-11|1994-04-29|COLLECT COD|RAIL|ording to the special, final 3653|180579|3098|3|17|28212.69|0.09|0.03|R|F|1994-06-24|1994-06-02|1994-07-17|DELIVER IN PERSON|RAIL|gle slyly regular 3653|185220|257|4|9|11746.98|0.10|0.07|R|F|1994-04-03|1994-05-19|1994-04-10|COLLECT COD|FOB|slyly silent account 3653|187794|313|5|41|77153.39|0.08|0.01|A|F|1994-06-18|1994-05-18|1994-06-20|COLLECT COD|RAIL|onic packages affix sly 3653|42462|2463|6|9|12640.14|0.05|0.03|A|F|1994-07-21|1994-05-31|1994-08-17|NONE|MAIL|tes: blithely bo 3653|48442|947|7|2|2780.88|0.06|0.03|R|F|1994-06-02|1994-05-31|1994-06-29|NONE|FOB|n accounts. fina 3654|164675|9708|1|46|80024.82|0.08|0.05|A|F|1992-06-05|1992-08-19|1992-06-06|DELIVER IN PERSON|FOB|usly regular foxes. furio 3654|92688|2689|2|29|48739.72|0.07|0.06|A|F|1992-09-11|1992-07-20|1992-10-04|DELIVER IN PERSON|FOB|odolites detect. quickly r 3654|1676|6677|3|37|58373.79|0.07|0.05|A|F|1992-09-22|1992-07-20|1992-10-19|TAKE BACK RETURN|RAIL|unts doze bravely ab 3654|167246|7247|4|11|14445.64|0.08|0.00|A|F|1992-07-20|1992-07-30|1992-07-23|TAKE BACK RETURN|SHIP|quickly along the express, ironic req 3654|93515|3516|5|34|51289.34|0.04|0.00|R|F|1992-07-26|1992-08-26|1992-08-12|TAKE BACK RETURN|REG AIR| the quick 3654|106597|4128|6|20|32071.80|0.03|0.02|A|F|1992-07-30|1992-07-05|1992-08-05|COLLECT COD|SHIP|s sleep about the slyly 3654|172225|9777|7|45|58374.90|0.01|0.07|A|F|1992-09-15|1992-07-04|1992-09-20|DELIVER IN PERSON|FOB|sly ironic notornis nag slyly 3655|183811|1366|1|5|9474.05|0.03|0.04|R|F|1993-01-17|1992-12-31|1993-01-23|DELIVER IN PERSON|TRUCK|riously bold pinto be 3655|96652|1671|2|1|1648.65|0.10|0.06|R|F|1992-10-24|1992-12-18|1992-11-07|DELIVER IN PERSON|AIR|arefully slow pinto beans are 3655|29968|2471|3|35|66428.60|0.01|0.04|R|F|1992-12-20|1992-11-16|1993-01-15|TAKE BACK RETURN|MAIL|blithely even accounts! furiously regular 3655|71347|1348|4|35|46141.90|0.04|0.07|R|F|1992-10-17|1992-12-23|1992-10-28|COLLECT COD|MAIL|ng foxes cajole fluffily slyly final fo 3680|176451|1486|1|48|73317.60|0.00|0.06|R|F|1993-01-16|1993-01-23|1993-01-19|COLLECT COD|FOB|packages. quickly fluff 3680|4228|6729|2|41|46421.02|0.00|0.04|A|F|1993-01-06|1993-03-02|1993-01-08|NONE|FOB|iously ironic platelets in 3680|55001|7507|3|33|31548.00|0.09|0.08|R|F|1993-03-16|1993-02-19|1993-04-05|NONE|FOB|ts. ironic, fina 3681|105232|7743|1|35|43303.05|0.03|0.08|R|F|1992-07-31|1992-05-18|1992-08-07|COLLECT COD|FOB|lyly special pinto 3682|60637|3144|1|6|9585.78|0.07|0.02|N|O|1997-05-06|1997-04-04|1997-05-11|NONE|AIR|ronic deposits wake slyly. ca 3682|115598|5599|2|18|29044.62|0.06|0.06|N|O|1997-04-30|1997-03-21|1997-05-10|NONE|FOB|regular dependencies 3682|46639|1648|3|17|26955.71|0.03|0.05|N|O|1997-02-12|1997-04-04|1997-02-22|COLLECT COD|FOB|, ironic packages wake a 3682|56495|9001|4|30|43544.70|0.09|0.05|N|O|1997-04-16|1997-04-16|1997-04-29|NONE|MAIL|he requests cajole quickly pending package 3683|100842|3353|1|35|64499.40|0.05|0.03|A|F|1993-05-31|1993-04-17|1993-06-14|NONE|SHIP| the furiously expr 3683|48689|6202|2|41|67144.88|0.01|0.06|A|F|1993-03-26|1993-05-06|1993-04-09|NONE|TRUCK|ress instructions. slyly express a 3683|99314|4333|3|23|30206.13|0.00|0.08|R|F|1993-07-02|1993-05-16|1993-07-30|NONE|TRUCK|xpress accounts sleep slyly re 3684|125106|5107|1|48|54292.80|0.04|0.06|A|F|1993-08-20|1993-09-02|1993-09-10|DELIVER IN PERSON|REG AIR|its boost alongside 3684|45384|5385|2|6|7976.28|0.06|0.08|R|F|1993-08-09|1993-10-05|1993-09-06|DELIVER IN PERSON|FOB|he silent requests. packages sleep fu 3684|162500|49|3|19|29687.50|0.04|0.02|A|F|1993-10-19|1993-08-25|1993-11-02|COLLECT COD|FOB|e slyly carefully pending foxes. d 3684|134811|2351|4|13|23995.53|0.02|0.05|A|F|1993-07-23|1993-09-16|1993-08-06|NONE|TRUCK|ing, unusual pinto beans! thinly p 3685|46199|8704|1|37|42372.03|0.02|0.03|R|F|1992-03-11|1992-04-09|1992-04-05|DELIVER IN PERSON|TRUCK|ress attai 3685|57413|9919|2|7|9592.87|0.05|0.00|R|F|1992-05-16|1992-02-23|1992-05-17|DELIVER IN PERSON|FOB|sits. special asymptotes about the r 3685|133706|3707|3|38|66108.60|0.08|0.03|A|F|1992-05-17|1992-03-16|1992-06-06|TAKE BACK RETURN|TRUCK|thely unusual pack 3685|191626|6665|4|39|66987.18|0.10|0.05|R|F|1992-02-19|1992-04-06|1992-03-02|COLLECT COD|FOB|ic courts nag carefully after the 3685|55782|5783|5|37|64297.86|0.00|0.01|A|F|1992-03-02|1992-04-10|1992-03-04|NONE|FOB|. carefully sly requests are regular, regu 3686|121557|9094|1|7|11049.85|0.02|0.04|N|O|1998-07-15|1998-08-22|1998-07-30|DELIVER IN PERSON|TRUCK| furiously unusual accou 3686|199146|1666|2|38|47315.32|0.06|0.03|N|O|1998-09-04|1998-08-11|1998-09-19|DELIVER IN PERSON|AIR|y silent foxes! carefully ruthless cour 3686|44727|4728|3|31|51823.32|0.10|0.06|N|O|1998-09-09|1998-08-28|1998-10-09|COLLECT COD|MAIL|gle across the courts. furiously regu 3686|116078|8590|4|7|7658.49|0.10|0.01|N|O|1998-07-16|1998-09-02|1998-07-22|NONE|FOB|ake carefully carefully q 3687|144005|1548|1|32|33568.00|0.03|0.06|R|F|1993-05-07|1993-04-05|1993-05-25|DELIVER IN PERSON|AIR|deas cajole fo 3687|80501|3010|2|2|2963.00|0.00|0.08|R|F|1993-02-23|1993-03-25|1993-03-11|NONE|TRUCK| express requests. slyly regular depend 3687|173664|6182|3|10|17376.60|0.01|0.02|A|F|1993-02-11|1993-03-22|1993-03-09|NONE|FOB|ing pinto beans 3687|161643|6676|4|19|32388.16|0.02|0.05|A|F|1993-05-14|1993-04-24|1993-06-01|DELIVER IN PERSON|MAIL|ly final asymptotes according to t 3687|118039|5573|5|31|32767.93|0.07|0.08|A|F|1993-05-28|1993-03-20|1993-06-05|DELIVER IN PERSON|FOB|foxes cajole quickly about the furiously f 3712|140013|5042|1|27|28431.27|0.01|0.05|R|F|1992-02-01|1992-02-26|1992-03-02|TAKE BACK RETURN|SHIP|ctions. even accounts haggle alongside 3712|184915|2470|2|13|25998.83|0.03|0.03|R|F|1992-04-30|1992-02-11|1992-05-30|DELIVER IN PERSON|FOB|s around the furiously ironic account 3712|63923|8936|3|44|83024.48|0.01|0.01|A|F|1992-03-26|1992-02-19|1992-04-18|TAKE BACK RETURN|FOB|ously permanently regular req 3712|147022|4565|4|38|40622.76|0.01|0.06|A|F|1992-01-15|1992-03-24|1992-01-27|COLLECT COD|RAIL|s nag carefully-- even, reg 3713|111310|3822|1|41|54173.71|0.07|0.08|N|O|1998-05-11|1998-07-17|1998-05-22|COLLECT COD|RAIL|eposits wake blithely fina 3713|176914|9432|2|19|37827.29|0.04|0.04|N|O|1998-06-25|1998-07-24|1998-07-08|DELIVER IN PERSON|AIR|tructions serve blithely around the furi 3713|179662|9663|3|19|33091.54|0.03|0.02|N|O|1998-05-19|1998-07-06|1998-06-09|DELIVER IN PERSON|REG AIR|quests cajole careful 3713|168912|8913|4|45|89140.95|0.06|0.04|N|O|1998-06-15|1998-07-30|1998-07-14|DELIVER IN PERSON|MAIL|al pinto beans affix after the slyly 3713|89165|4182|5|46|53091.36|0.10|0.04|N|O|1998-08-22|1998-06-27|1998-08-31|NONE|MAIL|totes. carefully special theodolites s 3713|181492|4011|6|29|45631.21|0.09|0.03|N|O|1998-08-04|1998-06-13|1998-08-21|NONE|RAIL|the regular dugouts wake furiously sil 3713|129891|9892|7|14|26892.46|0.04|0.00|N|O|1998-07-19|1998-07-02|1998-07-28|DELIVER IN PERSON|SHIP|eposits impress according 3714|68433|3446|1|13|18218.59|0.07|0.03|N|O|1998-06-26|1998-06-17|1998-07-07|TAKE BACK RETURN|REG AIR| the furiously final 3714|145675|8190|2|14|24089.38|0.02|0.05|N|O|1998-05-30|1998-06-30|1998-05-31|DELIVER IN PERSON|RAIL|ending ideas. thinly unusual theodo 3714|158877|8878|3|16|30973.92|0.00|0.02|N|O|1998-05-25|1998-07-07|1998-06-17|TAKE BACK RETURN|AIR|ccounts cajole fu 3714|29287|4292|4|44|53516.32|0.04|0.02|N|O|1998-07-18|1998-07-10|1998-07-22|DELIVER IN PERSON|AIR|s. quickly ironic dugouts sublat 3715|96625|4153|1|13|21081.06|0.00|0.03|N|O|1996-05-11|1996-04-25|1996-06-09|TAKE BACK RETURN|SHIP|e quickly ironic 3715|168955|3988|2|16|32383.20|0.01|0.06|N|O|1996-06-28|1996-04-22|1996-06-30|TAKE BACK RETURN|AIR|usly regular pearls haggle final packages 3715|11629|1630|3|37|57002.94|0.05|0.02|N|O|1996-05-03|1996-04-30|1996-05-17|NONE|SHIP|ut the carefully expr 3716|31701|4205|1|10|16327.00|0.09|0.04|N|O|1997-12-02|1997-11-09|1997-12-14|TAKE BACK RETURN|SHIP|ts. quickly sly ideas slee 3716|193081|3082|2|39|45789.12|0.02|0.08|N|O|1997-11-27|1997-10-23|1997-12-24|COLLECT COD|REG AIR|even deposits. 3716|106882|6883|3|42|79332.96|0.02|0.08|N|O|1997-12-03|1997-10-12|1997-12-15|NONE|TRUCK| of the pend 3716|164383|1932|4|19|27500.22|0.05|0.08|N|O|1997-09-25|1997-10-18|1997-10-12|NONE|TRUCK|arefully unusual accounts. flu 3716|181638|4157|5|25|42990.75|0.06|0.05|N|O|1997-11-23|1997-10-24|1997-11-24|COLLECT COD|REG AIR|fully unusual accounts. carefu 3717|152273|7304|1|45|59637.15|0.07|0.04|N|O|1998-08-09|1998-08-18|1998-08-14|TAKE BACK RETURN|TRUCK|ests wake whithout the blithely final pl 3717|52014|9530|2|3|2898.03|0.01|0.07|N|O|1998-06-09|1998-07-31|1998-06-14|NONE|REG AIR|nside the regular packages sleep 3717|195221|5222|3|45|59229.90|0.05|0.08|N|O|1998-09-19|1998-07-22|1998-09-28|DELIVER IN PERSON|MAIL|s the blithely unu 3717|68318|3331|4|5|6431.55|0.06|0.03|N|O|1998-09-02|1998-08-20|1998-09-26|TAKE BACK RETURN|AIR|quickly among 3717|15272|5273|5|7|8310.89|0.09|0.02|N|O|1998-09-08|1998-07-18|1998-09-10|DELIVER IN PERSON|RAIL| after the packa 3717|63587|8600|6|38|58922.04|0.01|0.07|N|O|1998-07-10|1998-07-08|1998-07-29|COLLECT COD|RAIL|ly about the car 3717|105267|5268|7|28|35623.28|0.03|0.01|N|O|1998-07-25|1998-08-12|1998-08-16|COLLECT COD|RAIL|ts sleep q 3718|20775|5780|1|40|67830.80|0.01|0.04|N|O|1996-11-20|1996-12-17|1996-12-03|DELIVER IN PERSON|MAIL|out the express deposits 3718|162663|212|2|16|27610.56|0.02|0.06|N|O|1996-11-11|1996-12-25|1996-11-12|COLLECT COD|TRUCK|slyly even accounts. blithely special acco 3718|69553|7072|3|8|12180.40|0.05|0.03|N|O|1996-12-06|1996-12-06|1996-12-15|TAKE BACK RETURN|AIR| the even deposits sleep carefully b 3719|21430|8937|1|35|47300.05|0.06|0.08|N|O|1997-06-11|1997-04-03|1997-06-15|TAKE BACK RETURN|TRUCK|ly foxes. pending braids haggle furio 3719|173998|6516|2|2|4143.98|0.02|0.08|N|O|1997-02-17|1997-04-25|1997-03-03|NONE|REG AIR|ccounts boost carefu 3719|181502|6539|3|12|19002.00|0.05|0.06|N|O|1997-06-10|1997-05-04|1997-07-09|TAKE BACK RETURN|REG AIR|grate according to the 3719|89266|6791|4|13|16318.38|0.02|0.00|N|O|1997-05-03|1997-04-16|1997-05-27|TAKE BACK RETURN|SHIP|iously. regular dep 3719|77643|151|5|19|30792.16|0.06|0.08|N|O|1997-05-22|1997-03-20|1997-06-12|COLLECT COD|TRUCK|he regular ideas integrate acros 3719|141336|6365|6|43|59225.19|0.03|0.08|N|O|1997-05-08|1997-04-15|1997-06-06|COLLECT COD|RAIL|the furiously special pinto bean 3719|18433|8434|7|16|21622.88|0.10|0.01|N|O|1997-03-02|1997-03-18|1997-03-28|TAKE BACK RETURN|RAIL| express asymptotes. ir 3744|194756|9795|1|30|55522.50|0.05|0.06|A|F|1992-05-07|1992-02-12|1992-05-17|TAKE BACK RETURN|FOB|nts among 3745|136294|1321|1|18|23945.22|0.01|0.05|A|F|1993-10-17|1993-11-16|1993-11-13|DELIVER IN PERSON|SHIP| slyly bold pinto beans according to 3746|164012|4013|1|37|39812.37|0.07|0.00|A|F|1994-12-29|1994-10-25|1995-01-03|COLLECT COD|FOB|e of the careful 3746|143595|8624|2|28|45880.52|0.06|0.08|R|F|1994-09-20|1994-10-21|1994-09-27|DELIVER IN PERSON|FOB|s after the even, special requests 3746|187888|7889|3|3|5927.64|0.10|0.01|R|F|1994-11-03|1994-12-10|1994-11-12|NONE|MAIL| the silent ideas cajole carefully 3746|27486|2491|4|11|15548.28|0.00|0.05|R|F|1994-10-02|1994-11-19|1994-10-10|COLLECT COD|SHIP| ironic theodolites are among th 3747|140559|8102|1|42|67181.10|0.05|0.05|N|O|1996-11-10|1996-10-19|1996-11-19|TAKE BACK RETURN|REG AIR|y. blithely fina 3747|169914|9915|2|33|65469.03|0.01|0.03|N|O|1996-10-14|1996-11-12|1996-11-11|NONE|REG AIR| regular p 3747|138552|8553|3|30|47716.50|0.00|0.07|N|O|1996-12-16|1996-11-15|1996-12-17|NONE|RAIL|! furiously f 3747|32970|480|4|21|39962.37|0.00|0.06|N|O|1996-11-18|1996-09-23|1996-11-26|TAKE BACK RETURN|AIR|ithely bold orbits mold furiously blit 3747|125802|827|5|32|58489.60|0.08|0.05|N|O|1996-09-10|1996-11-04|1996-10-10|DELIVER IN PERSON|MAIL|quests shall h 3747|153991|3992|6|14|28629.86|0.08|0.07|N|O|1996-11-03|1996-10-29|1996-11-06|TAKE BACK RETURN|AIR|packages cajole carefu 3747|117993|505|7|23|46252.77|0.00|0.04|N|O|1996-11-08|1996-11-10|1996-12-03|NONE|REG AIR|kages are ironic 3748|103539|6050|1|12|18510.36|0.06|0.01|N|O|1998-04-17|1998-04-15|1998-05-12|NONE|AIR|old reques 3748|164717|7234|2|24|42761.04|0.08|0.04|N|O|1998-06-07|1998-05-02|1998-06-21|DELIVER IN PERSON|TRUCK|al deposits. blithely 3748|196085|3643|3|19|22440.52|0.05|0.01|N|O|1998-04-23|1998-05-17|1998-05-23|COLLECT COD|RAIL|pinto beans run carefully quic 3748|186693|1730|4|5|8898.45|0.00|0.07|N|O|1998-06-29|1998-05-06|1998-07-12|DELIVER IN PERSON|MAIL| regular accounts sleep quickly-- furious 3748|146699|9214|5|21|36659.49|0.07|0.08|N|O|1998-03-30|1998-04-07|1998-04-05|TAKE BACK RETURN|MAIL|fix carefully furiously express ideas. furi 3749|172888|5406|1|11|21569.68|0.07|0.05|N|O|1995-06-25|1995-05-23|1995-07-10|TAKE BACK RETURN|RAIL|egular requests along the 3749|128102|3127|2|9|10170.90|0.08|0.05|A|F|1995-04-23|1995-04-18|1995-04-26|NONE|REG AIR|uses cajole blithely pla 3749|198854|3893|3|31|60538.35|0.00|0.05|N|F|1995-06-11|1995-05-20|1995-06-27|COLLECT COD|REG AIR|s. foxes sleep slyly unusual grouc 3749|130302|5329|4|7|9326.10|0.07|0.06|A|F|1995-03-31|1995-04-05|1995-04-11|NONE|TRUCK|he slyly ironic packages 3749|182309|4828|5|14|19478.20|0.02|0.00|N|F|1995-06-11|1995-05-19|1995-07-11|DELIVER IN PERSON|SHIP|press instruc 3749|53716|1232|6|10|16697.10|0.10|0.03|N|O|1995-06-24|1995-05-24|1995-07-18|COLLECT COD|SHIP|essly. regular pi 3750|133299|5813|1|37|49294.73|0.04|0.03|N|O|1995-07-08|1995-07-28|1995-07-28|DELIVER IN PERSON|REG AIR|usly busy account 3750|151799|1800|2|33|61076.07|0.05|0.03|N|O|1995-06-27|1995-06-20|1995-07-03|TAKE BACK RETURN|REG AIR|theodolites haggle. slyly pendin 3750|79370|1878|3|20|26987.40|0.09|0.05|N|F|1995-06-17|1995-06-06|1995-06-28|TAKE BACK RETURN|REG AIR|ss, ironic requests! fur 3750|165817|3366|4|33|62132.73|0.04|0.03|N|F|1995-06-15|1995-06-04|1995-06-29|COLLECT COD|RAIL|ep blithely according to the flu 3750|82651|5160|5|1|1633.65|0.05|0.01|N|O|1995-07-24|1995-06-25|1995-08-21|DELIVER IN PERSON|REG AIR|l dolphins against the slyly 3750|112383|4895|6|47|65582.86|0.01|0.08|R|F|1995-05-11|1995-06-13|1995-06-02|TAKE BACK RETURN|FOB|slowly regular accounts. blithely ev 3751|171747|4265|1|37|67293.38|0.00|0.04|R|F|1994-04-30|1994-05-30|1994-05-30|NONE|REG AIR|ly express courts 3751|140684|3199|2|32|55189.76|0.03|0.05|R|F|1994-05-05|1994-07-02|1994-06-02|COLLECT COD|MAIL|rthogs could have to slee 3751|64236|9249|3|45|54010.35|0.08|0.06|R|F|1994-05-27|1994-06-19|1994-06-14|NONE|RAIL|according to 3751|13891|1395|4|39|70390.71|0.07|0.01|A|F|1994-08-16|1994-07-11|1994-09-12|COLLECT COD|TRUCK|refully according to the iro 3751|57100|2111|5|12|12685.20|0.02|0.03|A|F|1994-08-09|1994-06-30|1994-08-12|TAKE BACK RETURN|TRUCK|accounts wake furious 3751|75535|550|6|39|58910.67|0.02|0.08|R|F|1994-08-01|1994-06-01|1994-08-26|COLLECT COD|SHIP|to beans. pending, express packages c 3776|2128|9629|1|39|40174.68|0.05|0.01|R|F|1993-01-03|1993-02-05|1993-01-08|COLLECT COD|FOB|yly blithely pending packages 3776|158157|3188|2|14|17012.10|0.06|0.08|R|F|1992-12-30|1993-02-12|1993-01-27|DELIVER IN PERSON|RAIL|y special ideas. express packages pr 3776|140995|3510|3|49|99763.51|0.01|0.08|R|F|1992-12-03|1993-02-16|1992-12-28|TAKE BACK RETURN|RAIL|equests. final, thin grouches 3776|91553|9081|4|49|75682.95|0.08|0.05|A|F|1993-02-11|1993-01-06|1993-02-27|COLLECT COD|MAIL|es: careful warthogs haggle fluffi 3777|99495|7023|1|11|16439.39|0.02|0.03|A|F|1994-04-09|1994-06-05|1994-04-14|NONE|FOB|ld ideas. even theodolites 3777|7436|4937|2|10|13434.30|0.03|0.01|R|F|1994-05-22|1994-05-29|1994-06-13|COLLECT COD|RAIL|le. ironic depths a 3777|165409|5410|3|18|26539.20|0.10|0.06|R|F|1994-05-04|1994-05-23|1994-05-22|COLLECT COD|REG AIR|eful packages use slyly: even deposits 3777|17130|7131|4|35|36649.55|0.10|0.04|A|F|1994-05-25|1994-05-26|1994-06-13|COLLECT COD|AIR|s. carefully express asymptotes accordi 3777|97412|9922|5|14|19731.74|0.04|0.05|R|F|1994-05-06|1994-06-24|1994-05-31|NONE|TRUCK|ording to the iro 3778|56669|1680|1|21|34138.86|0.01|0.06|R|F|1993-05-27|1993-07-10|1993-06-03|COLLECT COD|REG AIR|ts. blithely special theodoli 3778|28589|8590|2|32|48562.56|0.09|0.00|A|F|1993-06-22|1993-08-18|1993-07-03|TAKE BACK RETURN|MAIL|tes affix carefully above the 3778|93436|5946|3|41|58606.63|0.05|0.00|R|F|1993-06-21|1993-07-27|1993-07-15|COLLECT COD|FOB|e the furiously ironi 3778|168841|6390|4|28|53475.52|0.03|0.05|R|F|1993-08-18|1993-07-10|1993-09-06|TAKE BACK RETURN|REG AIR|y silent orbits print carefully against 3778|97021|4549|5|28|28504.56|0.01|0.06|R|F|1993-09-02|1993-08-08|1993-10-02|DELIVER IN PERSON|FOB|r deposits. theodol 3778|19451|4454|6|26|35631.70|0.00|0.01|A|F|1993-09-24|1993-07-06|1993-10-22|NONE|TRUCK| against the fluffily 3778|104793|4794|7|49|88091.71|0.02|0.04|A|F|1993-06-13|1993-08-08|1993-07-04|DELIVER IN PERSON|MAIL|ans. furiously 3779|45738|3251|1|28|47144.44|0.04|0.05|N|O|1997-05-06|1997-04-01|1997-05-18|TAKE BACK RETURN|AIR|s. close requests sleep 3779|109004|1515|2|5|5065.00|0.07|0.03|N|O|1997-01-07|1997-03-26|1997-02-05|DELIVER IN PERSON|AIR|heodolites. slyly regular a 3780|126173|6174|1|25|29979.25|0.08|0.04|N|O|1996-06-27|1996-07-02|1996-07-22|NONE|AIR|l, unusual 3780|189496|7051|2|40|63419.60|0.10|0.04|N|O|1996-06-06|1996-05-29|1996-07-01|COLLECT COD|SHIP|gular deposits-- furiously regular 3781|13185|3186|1|48|52712.64|0.02|0.06|N|O|1996-08-22|1996-08-13|1996-09-15|NONE|REG AIR|equests may cajole careful 3781|187496|7497|2|39|61756.11|0.10|0.00|N|O|1996-08-20|1996-08-16|1996-09-01|DELIVER IN PERSON|REG AIR|unts are carefully. ir 3781|29895|9896|3|17|31023.13|0.01|0.03|N|O|1996-06-23|1996-09-04|1996-07-19|TAKE BACK RETURN|REG AIR|. theodolite 3781|30810|5817|4|15|26112.15|0.05|0.00|N|O|1996-08-23|1996-08-08|1996-09-06|TAKE BACK RETURN|AIR| carefully blithe 3781|15001|2505|5|23|21068.00|0.09|0.08|N|O|1996-09-05|1996-08-18|1996-09-27|DELIVER IN PERSON|SHIP|pendencies are b 3782|26844|4351|1|29|51354.36|0.01|0.07|N|O|1996-09-17|1996-10-03|1996-10-07|DELIVER IN PERSON|REG AIR|quickly unusual pinto beans. carefully fina 3782|152897|5413|2|10|19498.90|0.03|0.05|N|O|1996-09-07|1996-11-19|1996-10-04|COLLECT COD|FOB|ven pinto b 3782|135201|5202|3|30|37086.00|0.06|0.06|N|O|1996-12-19|1996-10-31|1997-01-14|TAKE BACK RETURN|MAIL|slyly even pinto beans hag 3782|116305|3839|4|34|44924.20|0.02|0.06|N|O|1996-11-07|1996-10-22|1996-11-19|DELIVER IN PERSON|MAIL|gage after the even 3782|129239|6776|5|40|50729.20|0.09|0.04|N|O|1996-12-16|1996-11-22|1997-01-01|COLLECT COD|AIR|s instructions. regular accou 3783|166578|1611|1|36|59204.52|0.04|0.08|R|F|1993-12-17|1994-02-26|1994-01-03|DELIVER IN PERSON|SHIP|ites haggle among the carefully unusu 3783|72856|5364|2|36|65838.60|0.02|0.02|R|F|1994-03-02|1994-02-09|1994-03-15|COLLECT COD|TRUCK|egular accounts 3783|84967|7476|3|50|97598.00|0.04|0.01|R|F|1994-03-14|1994-01-09|1994-04-10|DELIVER IN PERSON|FOB|he furiously regular deposits. 3783|26279|1284|4|37|44594.99|0.10|0.05|R|F|1993-12-09|1994-02-17|1993-12-30|COLLECT COD|REG AIR|ing to the ideas. regular accounts de 3808|42716|5221|1|28|46443.88|0.02|0.01|R|F|1994-05-27|1994-06-18|1994-06-22|TAKE BACK RETURN|FOB|lly final accounts alo 3808|126319|1344|2|47|63229.57|0.04|0.08|R|F|1994-06-12|1994-06-03|1994-07-02|COLLECT COD|TRUCK|fully for the quickly final deposits: flu 3808|30291|292|3|45|54958.05|0.00|0.03|R|F|1994-07-03|1994-05-29|1994-07-14|TAKE BACK RETURN|REG AIR| carefully special 3808|99468|9469|4|34|49893.64|0.07|0.04|R|F|1994-08-13|1994-07-22|1994-08-31|DELIVER IN PERSON|FOB| pearls will have to 3808|154755|2301|5|29|52482.75|0.08|0.03|A|F|1994-06-22|1994-05-26|1994-07-06|TAKE BACK RETURN|TRUCK| deposits across the pac 3808|167127|2160|6|44|52541.28|0.06|0.06|A|F|1994-06-07|1994-06-04|1994-06-25|NONE|REG AIR|the blithely regular foxes. even, final 3809|190322|2842|1|17|24009.44|0.10|0.04|N|O|1996-08-14|1996-07-05|1996-09-04|DELIVER IN PERSON|FOB|es detect furiously sil 3809|132963|2964|2|32|63870.72|0.01|0.02|N|O|1996-07-03|1996-06-01|1996-07-25|COLLECT COD|SHIP|xcuses would boost against the fluffily eve 3809|104789|4790|3|46|82513.88|0.10|0.06|N|O|1996-08-20|1996-06-01|1996-08-24|TAKE BACK RETURN|TRUCK|l asymptotes. special 3809|177249|7250|4|43|57028.32|0.00|0.04|N|O|1996-05-06|1996-06-22|1996-06-05|TAKE BACK RETURN|TRUCK|yly ironic decoys; regular, iron 3810|183458|1013|1|49|75531.05|0.05|0.01|R|F|1992-11-27|1992-10-30|1992-12-16|COLLECT COD|AIR|cajole. fur 3810|168724|1241|2|18|32268.96|0.01|0.04|A|F|1992-11-28|1992-11-15|1992-12-27|DELIVER IN PERSON|SHIP|s. furiously careful deposi 3810|136788|9302|3|41|74815.98|0.08|0.08|A|F|1992-10-26|1992-10-27|1992-11-05|COLLECT COD|SHIP|l requests boost slyly along the slyl 3810|181036|3555|4|11|12287.33|0.06|0.04|A|F|1992-12-18|1992-12-11|1993-01-15|DELIVER IN PERSON|MAIL| the pending pinto beans. expr 3811|163383|5900|1|24|34713.12|0.04|0.02|N|O|1998-07-13|1998-05-16|1998-08-12|TAKE BACK RETURN|TRUCK|deposits. slyly regular accounts cajo 3811|165958|8475|2|2|4047.90|0.01|0.08|N|O|1998-06-16|1998-06-16|1998-06-23|NONE|MAIL|slyly fluff 3811|42469|7478|3|19|26817.74|0.02|0.06|N|O|1998-07-20|1998-06-14|1998-07-29|NONE|MAIL|s boost blithely furiou 3811|170416|2934|4|50|74320.50|0.08|0.03|N|O|1998-07-28|1998-07-06|1998-08-16|COLLECT COD|FOB|ts are slyly fluffy ideas. furiou 3811|181104|3623|5|23|27257.30|0.00|0.04|N|O|1998-08-13|1998-07-09|1998-08-29|COLLECT COD|AIR|nstructions sleep quickly. slyly final 3811|1635|6636|6|35|53782.05|0.04|0.07|N|O|1998-04-17|1998-06-30|1998-04-25|NONE|REG AIR|yly final dolphins? quickly ironic frets 3812|144641|2184|1|33|55626.12|0.00|0.05|N|O|1996-10-10|1996-10-05|1996-10-15|TAKE BACK RETURN|MAIL|posits engage. ironic, regular p 3812|172434|7469|2|33|49712.19|0.06|0.03|N|O|1996-10-05|1996-10-13|1996-10-22|TAKE BACK RETURN|MAIL|inal excuses d 3813|175506|5507|1|37|58515.50|0.05|0.04|N|O|1998-10-13|1998-09-19|1998-10-28|NONE|REG AIR|ravely special packages haggle p 3813|122720|7745|2|39|67966.08|0.05|0.00|N|O|1998-08-30|1998-08-12|1998-09-29|COLLECT COD|FOB|y ideas. final ideas about the sp 3814|130445|7985|1|7|10328.08|0.02|0.02|R|F|1995-05-01|1995-05-09|1995-05-28|DELIVER IN PERSON|REG AIR|es sleep furiou 3814|172629|5147|2|14|23822.68|0.01|0.00|R|F|1995-03-17|1995-05-10|1995-04-16|DELIVER IN PERSON|AIR|sits along the final, ironic deposit 3814|167125|9642|3|36|42916.32|0.06|0.02|N|O|1995-06-19|1995-04-18|1995-06-28|COLLECT COD|SHIP|beans cajole quickly sl 3814|65004|5005|4|20|19380.00|0.04|0.07|R|F|1995-02-23|1995-03-26|1995-03-04|DELIVER IN PERSON|SHIP|. doggedly ironic deposits will have to wa 3814|106211|1232|5|15|18258.15|0.03|0.04|N|O|1995-06-23|1995-03-25|1995-07-09|COLLECT COD|SHIP| carefully final deposits haggle slyly 3814|82129|7146|6|47|52222.64|0.09|0.05|A|F|1995-04-16|1995-04-03|1995-05-14|DELIVER IN PERSON|AIR|nusual requests. bli 3814|131307|3821|7|12|16059.60|0.10|0.01|R|F|1995-03-18|1995-04-16|1995-03-20|TAKE BACK RETURN|REG AIR|ages cajole. packages haggle. final 3815|76048|8556|1|3|3072.12|0.07|0.00|N|O|1997-11-16|1997-11-15|1997-11-30|NONE|FOB|egular, express ideas. ironic, final dep 3815|129840|2353|2|11|20568.24|0.02|0.04|N|O|1997-11-01|1997-11-05|1997-11-27|COLLECT COD|TRUCK|sleep blithe 3840|186402|8921|1|45|66978.00|0.02|0.08|N|O|1998-10-31|1998-09-19|1998-11-30|DELIVER IN PERSON|TRUCK|o beans are. carefully final courts x 3840|45522|531|2|12|17610.24|0.04|0.07|N|O|1998-10-02|1998-08-19|1998-10-20|TAKE BACK RETURN|RAIL|xpress pinto beans. accounts a 3840|72943|2944|3|45|86217.30|0.02|0.05|N|O|1998-10-12|1998-10-12|1998-10-28|TAKE BACK RETURN|FOB|onic, even packages are. pe 3840|147247|7248|4|41|53063.84|0.07|0.02|N|O|1998-07-21|1998-10-08|1998-08-01|TAKE BACK RETURN|MAIL| nag slyly? slyly pending accounts 3840|172923|5441|5|7|13971.44|0.09|0.08|N|O|1998-09-17|1998-09-20|1998-10-14|DELIVER IN PERSON|MAIL|. furiously final gifts sleep carefully pin 3840|106112|6113|6|33|36897.63|0.10|0.02|N|O|1998-07-29|1998-10-06|1998-08-04|DELIVER IN PERSON|SHIP|hely silent deposits w 3841|156889|9405|1|1|1945.88|0.06|0.03|A|F|1994-10-10|1994-11-12|1994-10-21|DELIVER IN PERSON|AIR| boost even re 3841|20882|5887|2|31|55889.28|0.09|0.03|A|F|1995-01-24|1994-11-25|1995-02-20|TAKE BACK RETURN|SHIP|n theodolites shall promise carefully. qui 3841|151821|4337|3|40|74912.80|0.06|0.02|A|F|1995-02-02|1994-11-30|1995-02-14|TAKE BACK RETURN|MAIL|its. quickly regular ideas nag carefully 3841|49764|9765|4|9|15423.84|0.10|0.07|A|F|1994-11-21|1994-12-26|1994-11-26|NONE|FOB|s according to the courts shall nag s 3841|175794|5795|5|3|5609.37|0.04|0.02|R|F|1994-10-24|1994-12-07|1994-11-09|COLLECT COD|FOB|foxes integrate 3841|162372|9921|6|48|68849.76|0.03|0.00|R|F|1994-11-23|1994-11-22|1994-12-01|DELIVER IN PERSON|FOB| according to the regular, 3842|161893|9442|1|28|54736.92|0.05|0.07|A|F|1992-06-17|1992-06-03|1992-06-24|DELIVER IN PERSON|TRUCK|s excuses thrash carefully. 3842|121783|6808|2|21|37900.38|0.07|0.05|R|F|1992-07-15|1992-06-02|1992-07-21|NONE|RAIL|r pinto be 3842|193009|8048|3|28|30856.00|0.00|0.00|A|F|1992-06-20|1992-05-22|1992-07-13|DELIVER IN PERSON|MAIL|lly alongside of the 3842|87373|4898|4|15|20405.55|0.07|0.01|A|F|1992-06-26|1992-06-23|1992-07-09|COLLECT COD|MAIL|ave packages are slyl 3842|67312|4831|5|13|16631.03|0.09|0.02|R|F|1992-04-13|1992-06-22|1992-05-11|COLLECT COD|RAIL|t blithely. busily regular accounts alon 3842|106971|4502|6|24|47471.28|0.08|0.08|R|F|1992-08-05|1992-06-29|1992-08-16|TAKE BACK RETURN|MAIL|phins are quickly 3843|14732|4733|1|7|11527.11|0.10|0.03|N|O|1997-02-13|1997-02-21|1997-02-20|TAKE BACK RETURN|SHIP|slyly even instructions. furiously eve 3843|484|2985|2|30|41534.40|0.01|0.05|N|O|1997-02-14|1997-03-25|1997-03-13|DELIVER IN PERSON|AIR| wake. slyly even packages boost 3844|134522|7036|1|2|3113.04|0.03|0.07|R|F|1995-02-24|1995-02-03|1995-03-18|TAKE BACK RETURN|AIR|es haggle final acco 3844|101965|6986|2|5|9834.80|0.10|0.03|R|F|1995-04-29|1995-02-24|1995-05-05|TAKE BACK RETURN|RAIL| unwind quickly about the pending, i 3845|33809|3810|1|44|76683.20|0.01|0.08|A|F|1992-07-20|1992-07-15|1992-07-24|DELIVER IN PERSON|REG AIR|s haggle among the fluffily regula 3845|23875|1382|2|16|28781.92|0.09|0.05|A|F|1992-08-08|1992-06-08|1992-08-26|DELIVER IN PERSON|SHIP|ely bold ideas use. ex 3845|58967|6483|3|17|32741.32|0.08|0.01|A|F|1992-06-12|1992-07-05|1992-06-26|TAKE BACK RETURN|RAIL|counts haggle. reg 3845|45398|407|4|1|1343.39|0.04|0.05|R|F|1992-05-21|1992-06-07|1992-06-17|COLLECT COD|REG AIR| blithely ironic t 3845|195390|5391|5|27|40105.53|0.00|0.05|R|F|1992-08-20|1992-07-17|1992-09-02|COLLECT COD|REG AIR|kages. care 3845|104773|7284|6|30|53333.10|0.09|0.06|R|F|1992-08-21|1992-07-07|1992-08-25|COLLECT COD|FOB|counts do wake blithely. ironic requests 3846|60152|2659|1|15|16682.25|0.06|0.03|N|O|1998-02-17|1998-04-27|1998-02-21|NONE|REG AIR|uternes. carefully even 3846|170299|300|2|30|41078.70|0.08|0.07|N|O|1998-05-01|1998-03-12|1998-05-20|TAKE BACK RETURN|FOB|deposits according to the fur 3846|14707|2211|3|49|79463.30|0.08|0.07|N|O|1998-02-14|1998-03-22|1998-02-17|DELIVER IN PERSON|RAIL|efully even packages against the blithe 3846|164058|1607|4|33|37027.65|0.05|0.00|N|O|1998-05-12|1998-03-14|1998-05-14|DELIVER IN PERSON|TRUCK|s instructions are. fu 3847|188290|8291|1|7|9648.03|0.08|0.00|A|F|1993-05-06|1993-06-06|1993-05-22|COLLECT COD|MAIL| about the blithely daring Tiresias. fl 3872|180331|5368|1|28|39517.24|0.10|0.04|N|O|1996-11-05|1996-11-10|1996-11-24|DELIVER IN PERSON|REG AIR|t after the carefully ironic excuses. f 3872|16635|1638|2|38|58961.94|0.04|0.05|N|O|1996-10-18|1996-12-03|1996-11-15|TAKE BACK RETURN|AIR|iously against the ironic, unusual a 3872|168467|6016|3|18|27638.28|0.07|0.07|N|O|1996-12-25|1996-10-24|1997-01-08|TAKE BACK RETURN|SHIP|s. regular, brave accounts sleep blith 3872|10643|644|4|41|63699.24|0.07|0.03|N|O|1996-11-23|1996-11-12|1996-12-03|COLLECT COD|REG AIR|ly regular epitaphs boost 3872|69266|4279|5|42|51880.92|0.03|0.00|N|O|1997-01-03|1996-10-12|1997-01-16|COLLECT COD|MAIL|s the furio 3872|139604|7144|6|40|65744.00|0.07|0.05|N|O|1997-01-02|1996-10-29|1997-01-14|NONE|REG AIR|nts? regularly ironic ex 3873|67790|5309|1|19|33398.01|0.04|0.04|N|O|1998-05-15|1998-05-10|1998-05-17|NONE|FOB|y final ac 3873|144960|9989|2|44|88218.24|0.05|0.05|N|O|1998-07-23|1998-05-22|1998-08-14|COLLECT COD|AIR|yly even platelets wake. 3873|139267|1781|3|29|37881.54|0.01|0.04|N|O|1998-06-22|1998-05-20|1998-07-05|COLLECT COD|REG AIR|olphins af 3874|169193|4226|1|21|26505.99|0.09|0.08|R|F|1993-06-19|1993-07-20|1993-07-08|DELIVER IN PERSON|SHIP| requests cajole fluff 3874|18904|3907|2|48|87499.20|0.06|0.07|R|F|1993-06-13|1993-07-20|1993-06-20|NONE|RAIL| ideas throughout 3875|80973|8498|1|24|46895.28|0.02|0.08|N|O|1997-10-15|1997-11-27|1997-11-09|COLLECT COD|AIR|ecial packages. 3875|112716|5228|2|49|84706.79|0.04|0.04|N|O|1997-10-18|1997-10-13|1997-10-19|NONE|MAIL|sleep furiously about the deposits. quickl 3876|140105|2620|1|12|13741.20|0.06|0.07|N|O|1996-09-16|1996-10-23|1996-10-05|TAKE BACK RETURN|REG AIR|y above the pending tithes. blithely ironi 3876|139212|1726|2|37|46294.77|0.00|0.03|N|O|1996-11-30|1996-10-18|1996-12-18|DELIVER IN PERSON|AIR|t dependencies. blithely final packages u 3876|126650|6651|3|41|68742.65|0.02|0.04|N|O|1996-10-15|1996-10-17|1996-10-19|NONE|AIR| quickly blit 3877|49774|2279|1|12|20685.24|0.06|0.01|R|F|1993-05-30|1993-08-09|1993-06-24|TAKE BACK RETURN|FOB|nal requests. even requests are. pac 3877|144635|2178|2|47|78942.61|0.05|0.00|A|F|1993-08-01|1993-08-16|1993-08-04|NONE|FOB|furiously quick requests nag along the theo 3877|79690|7212|3|44|73466.36|0.09|0.00|A|F|1993-06-07|1993-07-15|1993-07-06|DELIVER IN PERSON|REG AIR|elets. quickly regular accounts caj 3877|147844|7845|4|36|68106.24|0.06|0.01|A|F|1993-07-27|1993-07-13|1993-08-11|DELIVER IN PERSON|AIR|lithely about the dogged ideas. ac 3877|4649|4650|5|41|63699.24|0.03|0.07|A|F|1993-06-30|1993-07-20|1993-07-01|DELIVER IN PERSON|FOB|integrate against the expres 3877|122502|2503|6|7|10671.50|0.04|0.08|R|F|1993-06-14|1993-07-09|1993-06-28|NONE|TRUCK|lar dolphins cajole silently 3878|199696|9697|1|6|10774.14|0.07|0.04|N|O|1997-06-21|1997-05-22|1997-07-01|COLLECT COD|FOB|s. regular instru 3878|87395|7396|2|13|17971.07|0.01|0.06|N|O|1997-06-08|1997-06-03|1997-06-25|TAKE BACK RETURN|TRUCK|leep ruthlessly about the carefu 3878|40052|2557|3|20|19841.00|0.08|0.03|N|O|1997-06-20|1997-05-24|1997-07-20|TAKE BACK RETURN|MAIL|the furiously careful ideas cajole slyly sl 3878|151976|1977|4|20|40559.40|0.01|0.07|N|O|1997-07-13|1997-05-22|1997-07-20|NONE|FOB|about the carefully ironic pa 3879|125689|714|1|45|77160.60|0.10|0.08|N|O|1996-03-18|1996-01-03|1996-04-03|COLLECT COD|RAIL|ly according to the expr 3879|44020|1533|2|35|33740.70|0.00|0.07|N|O|1995-12-08|1996-01-23|1995-12-28|TAKE BACK RETURN|MAIL|o beans. accounts cajole furiously. re 3904|37424|4934|1|22|29951.24|0.04|0.03|N|O|1998-02-02|1998-02-09|1998-02-10|TAKE BACK RETURN|REG AIR|structions cajole carefully. carefully f 3904|183753|1308|2|19|34898.25|0.09|0.01|N|O|1998-02-10|1998-02-13|1998-02-20|TAKE BACK RETURN|AIR| excuses sleep slyly according to th 3905|100739|8270|1|43|74808.39|0.07|0.08|A|F|1994-03-30|1994-02-18|1994-04-09|DELIVER IN PERSON|REG AIR|uses are care 3905|115915|8427|2|7|13516.37|0.03|0.00|R|F|1994-03-01|1994-02-19|1994-03-11|DELIVER IN PERSON|AIR|ully furiously furious packag 3905|169943|4976|3|6|12077.64|0.07|0.02|R|F|1994-04-07|1994-03-07|1994-04-21|DELIVER IN PERSON|RAIL|ow furiously. deposits wake ironic 3906|152987|5503|1|42|85679.16|0.00|0.04|R|F|1992-09-03|1992-07-22|1992-09-04|COLLECT COD|RAIL|jole blithely after the furiously regular 3906|39340|4347|2|50|63967.00|0.01|0.07|R|F|1992-09-24|1992-08-24|1992-09-29|NONE|MAIL|ke slyly. stealt 3906|179655|4690|3|15|26019.75|0.06|0.02|R|F|1992-07-30|1992-08-26|1992-08-02|TAKE BACK RETURN|FOB|dependencies at the 3906|58264|8265|4|36|44001.36|0.08|0.08|A|F|1992-08-07|1992-08-08|1992-08-24|NONE|SHIP|y. ironic deposits haggle sl 3907|111784|4296|1|41|73626.98|0.06|0.02|A|F|1992-09-13|1992-10-23|1992-09-29|COLLECT COD|MAIL|ackages wake along the carefully regul 3907|144491|2034|2|41|62955.09|0.03|0.00|A|F|1992-10-25|1992-10-17|1992-11-01|TAKE BACK RETURN|RAIL|s above the unusual ideas sleep furiousl 3907|51501|9017|3|45|65362.50|0.02|0.07|R|F|1992-09-21|1992-09-19|1992-10-18|COLLECT COD|REG AIR| about the regular pac 3907|175693|728|4|48|84897.12|0.05|0.07|A|F|1992-09-24|1992-10-16|1992-10-06|DELIVER IN PERSON|TRUCK|nt asymptotes lose across th 3907|61958|1959|5|22|42238.90|0.09|0.01|R|F|1992-09-20|1992-10-30|1992-09-29|TAKE BACK RETURN|TRUCK|ly. furiously unusual deposits use afte 3907|125062|2599|6|34|36960.04|0.02|0.02|R|F|1992-09-06|1992-10-08|1992-09-12|COLLECT COD|FOB| requests according to the slyly pending 3907|109234|4255|7|8|9945.84|0.10|0.01|A|F|1992-09-18|1992-10-29|1992-09-27|NONE|REG AIR|furiously final packages. 3908|91840|4350|1|50|91592.00|0.05|0.04|R|F|1993-06-19|1993-04-27|1993-07-05|DELIVER IN PERSON|MAIL| even accounts wake 3908|147084|7085|2|8|9048.64|0.06|0.03|A|F|1993-03-12|1993-04-13|1993-03-22|DELIVER IN PERSON|SHIP|r instructions was requests. ironically 3909|177914|5466|1|30|59757.30|0.03|0.07|N|O|1998-10-17|1998-10-14|1998-10-28|COLLECT COD|TRUCK|ly even deposits across the ironic notorni 3909|190896|5935|2|46|91396.94|0.03|0.01|N|O|1998-10-08|1998-10-15|1998-10-24|NONE|FOB|the blithely unusual ideas 3910|138602|3629|1|10|16406.00|0.00|0.08|N|O|1996-10-18|1996-10-31|1996-11-14|DELIVER IN PERSON|FOB|tions boost furiously unusual e 3910|70782|5797|2|31|54336.18|0.05|0.03|N|O|1996-12-22|1996-11-14|1997-01-01|TAKE BACK RETURN|SHIP|ess instructions. 3910|19716|4719|3|6|9814.26|0.04|0.04|N|O|1996-12-08|1996-10-30|1996-12-31|DELIVER IN PERSON|MAIL|ly sly platelets are fluffily slyly si 3910|152611|5127|4|1|1663.61|0.03|0.06|N|O|1996-09-12|1996-10-21|1996-09-19|DELIVER IN PERSON|FOB|s sleep neve 3911|112842|5354|1|10|18548.40|0.07|0.06|N|O|1995-06-22|1995-05-30|1995-06-28|COLLECT COD|FOB|ss theodolites are blithely along t 3911|118861|6395|2|14|26318.04|0.08|0.05|R|F|1995-04-28|1995-05-03|1995-05-22|NONE|RAIL|e blithely brave depo 3911|91085|6104|3|12|12912.96|0.10|0.05|R|F|1995-04-04|1995-04-16|1995-04-10|COLLECT COD|FOB|uctions. blithely regula 3936|136372|6373|1|25|35209.25|0.06|0.03|N|O|1996-12-03|1996-12-27|1997-01-01|DELIVER IN PERSON|RAIL|gular requests nag quic 3936|187641|160|2|24|41487.36|0.10|0.07|N|O|1996-11-22|1997-01-01|1996-12-08|NONE|AIR|ns. accounts mold fl 3936|82065|7082|3|42|43976.52|0.00|0.07|N|O|1997-01-03|1997-01-29|1997-01-14|COLLECT COD|AIR|elets wake amo 3936|61051|8570|4|12|12144.60|0.06|0.05|N|O|1996-11-25|1997-01-09|1996-12-06|DELIVER IN PERSON|SHIP|ithely across the carefully brave req 3936|83519|3520|5|35|52587.85|0.02|0.08|N|O|1996-12-04|1997-01-06|1996-12-22|NONE|SHIP|lly ironic requ 3936|102029|4540|6|26|26806.52|0.01|0.02|N|O|1997-02-27|1997-01-16|1997-03-22|NONE|RAIL|quickly pen 3937|69439|4452|1|48|67604.64|0.10|0.02|N|O|1998-03-15|1998-02-22|1998-03-30|DELIVER IN PERSON|FOB|gainst the thinl 3937|47321|2330|2|30|38049.60|0.01|0.07|N|O|1998-01-17|1998-01-03|1998-02-08|COLLECT COD|TRUCK|al packages slee 3937|114474|2008|3|27|40188.69|0.03|0.00|N|O|1998-02-06|1998-01-12|1998-02-20|NONE|MAIL|ven ideas. slyly expr 3937|153056|5572|4|50|55452.50|0.01|0.02|N|O|1998-01-15|1998-01-09|1998-02-04|DELIVER IN PERSON|AIR|ong the carefully exp 3937|2904|405|5|29|52400.10|0.03|0.07|N|O|1998-03-06|1998-02-22|1998-03-14|NONE|TRUCK|nt pinto beans above the pending instr 3937|192002|7041|6|6|6564.00|0.00|0.00|N|O|1998-01-24|1998-02-13|1998-01-27|DELIVER IN PERSON|FOB|into beans. slyly silent orbits alongside o 3937|163846|1395|7|1|1909.84|0.02|0.05|N|O|1998-03-29|1998-01-08|1998-04-27|TAKE BACK RETURN|TRUCK|refully agains 3938|158367|3398|1|46|65566.56|0.10|0.07|R|F|1993-05-20|1993-05-04|1993-06-12|DELIVER IN PERSON|FOB|ly even foxes are slyly fu 3939|159523|2039|1|8|12660.16|0.03|0.06|N|O|1996-01-29|1996-04-05|1996-02-26|COLLECT COD|REG AIR|e packages. express, pen 3940|177038|2073|1|33|36795.99|0.10|0.07|N|O|1996-05-19|1996-04-19|1996-05-23|TAKE BACK RETURN|RAIL|ly ironic packages about the pending accou 3940|68501|6020|2|40|58780.00|0.08|0.02|N|O|1996-02-29|1996-03-22|1996-03-04|NONE|MAIL|ts. regular fox 3940|88876|8877|3|8|14918.96|0.07|0.08|N|O|1996-04-04|1996-04-12|1996-04-18|DELIVER IN PERSON|RAIL|ions cajole furiously regular pinto beans. 3940|136206|8720|4|11|13664.20|0.09|0.05|N|O|1996-03-09|1996-05-13|1996-03-17|COLLECT COD|REG AIR|e of the special packages. furiously 3940|942|5943|5|41|75560.54|0.00|0.07|N|O|1996-05-08|1996-05-03|1996-06-03|COLLECT COD|MAIL|thily. deposits cajole. 3941|40836|837|1|47|83511.01|0.05|0.07|N|O|1996-11-24|1996-10-09|1996-12-22|DELIVER IN PERSON|RAIL| carefully pending 3941|122533|70|2|19|29555.07|0.05|0.00|N|O|1996-11-10|1996-10-26|1996-12-05|COLLECT COD|RAIL|eposits haggle furiously even 3941|9693|2194|3|2|3205.38|0.01|0.03|N|O|1996-12-04|1996-10-01|1996-12-25|NONE|REG AIR|es wake after the 3941|109348|6879|4|29|39362.86|0.00|0.03|N|O|1996-09-14|1996-10-04|1996-09-19|NONE|MAIL|g the blithely 3942|182281|7318|1|6|8179.68|0.05|0.05|A|F|1993-07-01|1993-09-14|1993-07-23|DELIVER IN PERSON|SHIP|ep ruthlessly carefully final accounts: s 3942|193715|8754|2|5|9043.55|0.06|0.02|R|F|1993-09-27|1993-09-24|1993-10-07|DELIVER IN PERSON|MAIL|. fluffily pending deposits above the flu 3942|155497|8013|3|25|38812.25|0.04|0.06|R|F|1993-09-13|1993-08-01|1993-09-29|COLLECT COD|RAIL|d the quick packages 3943|197698|5256|1|15|26935.35|0.03|0.01|N|O|1997-01-13|1996-12-17|1997-02-02|COLLECT COD|REG AIR| grow fluffily according to the 3943|95557|5558|2|9|13972.95|0.00|0.06|N|O|1996-11-27|1997-01-03|1996-12-17|COLLECT COD|RAIL|refully ironic 3943|16818|1821|3|32|55513.92|0.00|0.02|N|O|1996-10-22|1996-12-17|1996-11-04|TAKE BACK RETURN|TRUCK| unusual ideas into the furiously even pack 3943|49034|9035|4|5|4915.15|0.04|0.04|N|O|1997-01-09|1996-11-10|1997-02-06|COLLECT COD|RAIL|arefully regular deposits accord 3968|53646|6152|1|27|43190.28|0.04|0.05|N|O|1997-04-25|1997-04-17|1997-05-11|TAKE BACK RETURN|MAIL|t silently. 3968|25166|2673|2|45|49102.20|0.00|0.07|N|O|1997-06-18|1997-04-24|1997-06-25|DELIVER IN PERSON|FOB|ully slyly fi 3968|155118|5119|3|43|50443.73|0.07|0.06|N|O|1997-04-30|1997-05-14|1997-05-18|TAKE BACK RETURN|SHIP|ly regular accounts 3968|60159|5172|4|7|7834.05|0.07|0.02|N|O|1997-03-30|1997-05-01|1997-04-12|DELIVER IN PERSON|SHIP|efully bold instructions. express 3969|51062|8578|1|39|39509.34|0.04|0.04|N|O|1997-06-12|1997-06-13|1997-07-05|NONE|MAIL|ly bold ideas s 3969|196745|4303|2|26|47885.24|0.05|0.03|N|O|1997-07-08|1997-07-30|1997-07-10|TAKE BACK RETURN|AIR|fluffily; braids detect. 3969|78703|3718|3|46|77358.20|0.04|0.02|N|O|1997-05-29|1997-06-15|1997-06-10|TAKE BACK RETURN|SHIP|fully final requests sleep stealthily. care 3969|150168|2684|4|21|25581.36|0.07|0.04|N|O|1997-08-31|1997-07-16|1997-09-02|TAKE BACK RETURN|MAIL|unts doze quickly final reque 3969|71513|1514|5|40|59380.40|0.09|0.00|N|O|1997-05-19|1997-08-02|1997-06-05|COLLECT COD|TRUCK|lar requests cajole furiously blithely regu 3969|104963|7474|6|4|7871.84|0.02|0.01|N|O|1997-06-04|1997-07-31|1997-06-13|COLLECT COD|REG AIR|dencies wake blithely? quickly even theodo 3970|87268|7269|1|2|2510.52|0.01|0.07|R|F|1992-04-24|1992-06-03|1992-05-16|TAKE BACK RETURN|RAIL|carefully pending foxes wake blithely 3970|108084|5615|2|18|19657.44|0.03|0.08|A|F|1992-06-06|1992-06-18|1992-07-05|DELIVER IN PERSON|TRUCK| maintain slyly. ir 3970|153569|1115|3|10|16225.60|0.10|0.04|A|F|1992-07-01|1992-05-31|1992-07-02|NONE|AIR| special packages wake after the final br 3970|21600|9107|4|34|51734.40|0.05|0.00|A|F|1992-06-25|1992-05-23|1992-07-12|COLLECT COD|SHIP|y final gifts are. carefully pe 3970|29048|6555|5|23|22471.92|0.05|0.04|A|F|1992-06-04|1992-06-14|1992-06-13|COLLECT COD|TRUCK| above the final braids. regular 3970|8771|6272|6|46|77269.42|0.07|0.04|R|F|1992-04-29|1992-05-14|1992-05-24|NONE|FOB|yly ironic 3970|4076|6577|7|46|45083.22|0.08|0.08|R|F|1992-05-02|1992-05-12|1992-05-10|COLLECT COD|MAIL|ix slyly. quickly silen 3971|95389|7899|1|47|65065.86|0.06|0.04|N|O|1996-07-07|1996-08-08|1996-08-01|TAKE BACK RETURN|RAIL|e slyly final dependencies x-ray 3971|190223|7781|2|2|2626.44|0.04|0.03|N|O|1996-07-15|1996-08-12|1996-07-26|NONE|SHIP|haggle abou 3972|50440|7956|1|2|2780.88|0.05|0.03|A|F|1994-07-24|1994-06-30|1994-08-13|TAKE BACK RETURN|SHIP|y final theodolite 3973|29590|4595|1|21|31911.39|0.02|0.06|R|F|1992-06-18|1992-06-03|1992-07-02|COLLECT COD|REG AIR|equests. furiously 3973|114205|9228|2|37|45110.40|0.07|0.00|A|F|1992-05-29|1992-05-04|1992-06-23|TAKE BACK RETURN|SHIP|inos wake fluffily. pending requests nag 3973|39243|1747|3|40|47289.60|0.08|0.05|R|F|1992-05-03|1992-06-09|1992-05-21|COLLECT COD|RAIL|g the carefully blithe f 3974|21091|6096|1|47|47568.23|0.10|0.03|N|O|1996-06-03|1996-05-08|1996-06-28|NONE|TRUCK|dencies above the re 3974|60454|5467|2|17|24045.65|0.05|0.07|N|O|1996-04-05|1996-05-21|1996-04-28|COLLECT COD|TRUCK|ions eat slyly after the blithely 3975|56042|3558|1|38|37925.52|0.01|0.05|N|O|1995-08-02|1995-06-18|1995-08-19|COLLECT COD|TRUCK|es are furiously: furi 4000|195697|5698|1|41|73500.29|0.06|0.01|A|F|1992-03-02|1992-03-14|1992-03-27|COLLECT COD|FOB|ve the even, fi 4000|74526|7034|2|44|66022.88|0.09|0.06|A|F|1992-03-27|1992-02-18|1992-03-31|COLLECT COD|AIR|equests use blithely blithely bold d 4001|105185|206|1|26|30944.68|0.00|0.01|N|O|1997-07-26|1997-06-18|1997-08-08|DELIVER IN PERSON|RAIL|tegrate blithely 4001|40918|8431|2|19|35319.29|0.03|0.02|N|O|1997-08-23|1997-06-15|1997-09-18|COLLECT COD|SHIP|ackages. carefully ironi 4001|93081|3082|3|18|19333.44|0.07|0.00|N|O|1997-06-04|1997-06-22|1997-06-13|DELIVER IN PERSON|MAIL|lithely ironic d 4001|1923|9424|4|39|71171.88|0.00|0.00|N|O|1997-06-13|1997-06-17|1997-06-25|NONE|SHIP| dogged excuses. blithe 4002|110728|3240|1|35|60855.20|0.01|0.08|N|O|1997-05-16|1997-06-15|1997-06-02|DELIVER IN PERSON|TRUCK|eep. quickly 4002|197837|7838|2|20|38696.60|0.00|0.03|N|O|1997-06-15|1997-05-20|1997-07-11|NONE|MAIL|lly even ins 4002|39841|4848|3|6|10685.04|0.08|0.07|N|O|1997-05-02|1997-07-07|1997-05-16|TAKE BACK RETURN|RAIL| furiously furiously special theodoli 4002|198650|6208|4|6|10491.90|0.06|0.06|N|O|1997-07-01|1997-05-15|1997-07-31|NONE|MAIL|he slyly iro 4002|98186|696|5|4|4736.72|0.08|0.07|N|O|1997-05-06|1997-06-15|1997-05-24|NONE|REG AIR|ccording to the careful 4003|51112|8628|1|18|19135.98|0.04|0.07|R|F|1993-02-02|1993-04-15|1993-02-28|TAKE BACK RETURN|AIR|ar grouches s 4004|120061|62|1|23|24864.38|0.07|0.02|A|F|1993-08-12|1993-07-13|1993-08-16|TAKE BACK RETURN|TRUCK| bold theodolites? special packages accordi 4004|63505|3506|2|47|69019.50|0.07|0.04|R|F|1993-06-25|1993-08-03|1993-07-12|NONE|SHIP|thely instead of the even, unu 4004|113136|3137|3|39|44816.07|0.10|0.05|R|F|1993-07-12|1993-07-27|1993-07-18|NONE|MAIL|ccounts sleep furious 4004|73634|6142|4|46|73950.98|0.10|0.04|R|F|1993-09-04|1993-07-13|1993-09-28|COLLECT COD|FOB|ncies. slyly pending dolphins sleep furio 4004|154767|7283|5|9|16395.84|0.04|0.06|A|F|1993-08-25|1993-06-10|1993-09-24|COLLECT COD|MAIL|ly ironic requests. quickly pending ide 4004|160037|2554|6|44|48269.32|0.07|0.05|R|F|1993-07-25|1993-07-23|1993-08-16|TAKE BACK RETURN|REG AIR|ut the sauternes. bold, ironi 4004|125377|2914|7|20|28047.40|0.07|0.05|A|F|1993-06-19|1993-06-14|1993-07-04|COLLECT COD|REG AIR|. ironic deposits cajole blithely? 4005|3667|1168|1|26|40837.16|0.09|0.05|N|O|1996-12-01|1997-02-03|1996-12-15|NONE|REG AIR| to the quic 4005|16605|6606|2|28|42604.80|0.02|0.06|N|O|1996-12-11|1997-01-24|1996-12-17|DELIVER IN PERSON|REG AIR|ly carefully ironic deposits. slyly 4005|71317|8839|3|28|36072.68|0.03|0.01|N|O|1996-12-08|1997-01-14|1996-12-30|TAKE BACK RETURN|MAIL|y pending dependenc 4005|14650|7152|4|49|76667.85|0.09|0.00|N|O|1997-01-31|1996-12-24|1997-03-02|NONE|RAIL|tions sleep across the silent d 4005|5747|5748|5|14|23138.36|0.09|0.08|N|O|1996-11-27|1997-01-09|1996-12-25|NONE|TRUCK|ld requests. slyly final instructi 4006|54773|2289|1|11|19005.47|0.05|0.08|A|F|1995-04-29|1995-02-21|1995-05-20|TAKE BACK RETURN|RAIL|ress foxes cajole quick 4006|158031|3062|2|18|19602.54|0.05|0.03|A|F|1995-01-29|1995-03-08|1995-02-02|TAKE BACK RETURN|MAIL|gouts! slyly iron 4006|23176|3177|3|15|16487.55|0.01|0.02|R|F|1995-02-23|1995-04-02|1995-02-25|TAKE BACK RETURN|RAIL|n deposits cajole slyl 4006|113466|3467|4|25|36986.50|0.00|0.07|A|F|1995-02-23|1995-02-09|1995-02-24|DELIVER IN PERSON|SHIP| requests use depos 4007|56091|1102|1|32|33506.88|0.00|0.03|R|F|1993-09-30|1993-08-16|1993-10-03|DELIVER IN PERSON|RAIL|nal accounts across t 4007|115470|7982|2|41|60904.27|0.04|0.06|A|F|1993-10-11|1993-08-30|1993-11-04|DELIVER IN PERSON|TRUCK|eposits. regular epitaphs boost blithely. 4007|101478|9009|3|5|7397.35|0.09|0.06|A|F|1993-09-17|1993-08-29|1993-10-12|TAKE BACK RETURN|FOB|y unusual packa 4007|137992|5532|4|15|30449.85|0.05|0.02|A|F|1993-09-01|1993-07-19|1993-09-03|DELIVER IN PERSON|FOB|le furiously quickly 4007|25894|5895|5|23|41857.47|0.02|0.07|A|F|1993-10-08|1993-09-09|1993-10-23|COLLECT COD|MAIL|ter the accounts. expr 4032|101625|1626|1|8|13012.96|0.06|0.00|N|O|1998-06-04|1998-05-17|1998-07-03|TAKE BACK RETURN|RAIL|ometimes even cou 4032|1652|9153|2|27|41948.55|0.09|0.00|N|O|1998-05-31|1998-04-19|1998-06-24|COLLECT COD|REG AIR|le furiously according to 4032|153608|6124|3|23|38216.80|0.09|0.06|N|O|1998-06-12|1998-05-11|1998-06-24|COLLECT COD|MAIL|ording to the 4032|84644|7153|4|10|16286.40|0.09|0.05|N|O|1998-03-31|1998-04-22|1998-04-07|NONE|REG AIR| carefully bol 4033|109529|9530|1|27|41540.04|0.01|0.04|R|F|1993-08-08|1993-08-14|1993-08-09|NONE|AIR|pinto beans 4033|37321|9825|2|34|42782.88|0.07|0.00|R|F|1993-07-19|1993-08-05|1993-07-26|NONE|RAIL|t the blithely dogg 4034|189949|4986|1|48|97869.12|0.03|0.03|A|F|1994-03-01|1994-01-16|1994-03-16|NONE|RAIL| blithely regular requests play carefull 4034|56730|9236|2|47|79276.31|0.07|0.05|A|F|1994-01-27|1993-12-26|1994-02-04|NONE|TRUCK|eodolites was slyly ironic ideas. de 4034|53884|3885|3|43|79028.84|0.10|0.03|A|F|1993-11-29|1994-01-08|1993-12-10|DELIVER IN PERSON|FOB|posits wake carefully af 4034|27679|7680|4|46|73906.82|0.06|0.00|A|F|1994-02-22|1994-01-09|1994-03-04|DELIVER IN PERSON|AIR|uests. furiously unusual instructions wake 4034|195196|2754|5|7|9038.33|0.07|0.06|R|F|1994-03-04|1994-01-22|1994-04-01|NONE|AIR|y even theodolites. slyly regular instru 4034|49283|6796|6|5|6161.40|0.01|0.06|A|F|1994-02-12|1994-01-24|1994-02-13|COLLECT COD|AIR|fully around the furiously ironic re 4035|96790|6791|1|4|7147.16|0.08|0.03|R|F|1992-04-21|1992-04-23|1992-04-25|COLLECT COD|AIR|ilent, even pear 4035|135094|5095|2|4|4516.36|0.07|0.00|A|F|1992-05-21|1992-04-24|1992-05-24|DELIVER IN PERSON|FOB|en instructions sleep blith 4035|117010|4544|3|1|1027.01|0.03|0.01|R|F|1992-06-18|1992-05-19|1992-07-02|COLLECT COD|FOB| requests. quickly 4035|181284|8839|4|13|17748.64|0.00|0.01|R|F|1992-06-10|1992-05-16|1992-07-10|NONE|SHIP|s. furiously even courts wake slyly 4036|5585|586|1|46|68566.68|0.09|0.00|N|O|1997-06-21|1997-05-29|1997-07-18|NONE|REG AIR|usly across the even th 4036|52428|4934|2|21|28988.82|0.09|0.07|N|O|1997-08-08|1997-06-28|1997-08-09|COLLECT COD|MAIL|e carefully. qui 4036|141687|1688|3|6|10372.08|0.07|0.01|N|O|1997-06-19|1997-06-16|1997-07-01|DELIVER IN PERSON|SHIP|equests wake about the bold id 4036|126016|3553|4|20|20840.20|0.08|0.02|N|O|1997-08-11|1997-07-11|1997-09-03|NONE|TRUCK|slyly bold deposits cajole pending, blithe 4037|63703|1222|1|32|53334.40|0.00|0.06|A|F|1993-05-06|1993-06-08|1993-05-31|DELIVER IN PERSON|AIR|e of the pending, iron 4037|46766|6767|2|4|6851.04|0.09|0.07|A|F|1993-07-05|1993-06-12|1993-08-03|DELIVER IN PERSON|RAIL|s around the blithely ironic ac 4038|195610|3168|1|40|68224.40|0.05|0.01|N|O|1996-01-15|1996-03-13|1996-01-25|COLLECT COD|TRUCK|t. slyly silent pinto beans amo 4038|11283|6286|2|37|44188.36|0.04|0.03|N|O|1996-03-17|1996-03-19|1996-04-07|DELIVER IN PERSON|REG AIR| packages 4038|31841|1842|3|24|42548.16|0.10|0.04|N|O|1996-04-06|1996-02-15|1996-04-18|TAKE BACK RETURN|RAIL|the furiously regu 4038|149264|9265|4|29|38084.54|0.07|0.06|N|O|1996-01-07|1996-03-08|1996-01-13|NONE|FOB|ffix. quietly ironic packages a 4038|78767|6289|5|24|41898.24|0.07|0.06|N|O|1996-04-01|1996-04-05|1996-04-28|DELIVER IN PERSON|TRUCK|ake quickly after the final, ironic ac 4038|35867|8371|6|6|10817.16|0.07|0.05|N|O|1996-02-09|1996-03-05|1996-03-10|COLLECT COD|SHIP| special instructions. packa 4039|93195|3196|1|38|45151.22|0.03|0.06|N|O|1998-03-09|1997-12-31|1998-03-21|DELIVER IN PERSON|REG AIR|sual asymptotes. ironic deposits nag aft 4039|121085|8622|2|17|18803.36|0.10|0.04|N|O|1998-01-15|1998-01-20|1998-01-28|TAKE BACK RETURN|MAIL| regular foxes haggle carefully bo 4039|63297|8310|3|9|11342.61|0.10|0.01|N|O|1998-03-08|1998-02-05|1998-04-05|TAKE BACK RETURN|FOB|t? pinto beans cajole across the thinly r 4039|27080|9583|4|43|43304.44|0.01|0.02|N|O|1998-01-02|1997-12-22|1998-01-15|NONE|FOB|beans believe bene 4039|133273|3274|5|43|56169.61|0.09|0.00|N|O|1998-01-20|1998-01-11|1998-02-05|COLLECT COD|SHIP|sts along the regular in 4064|198313|833|1|3|4233.93|0.10|0.04|N|O|1997-01-04|1997-01-01|1997-01-23|NONE|SHIP|its! quickly sp 4064|39472|1976|2|15|21172.05|0.02|0.02|N|O|1996-11-09|1996-12-04|1996-11-18|DELIVER IN PERSON|MAIL|braids affix across the regular sheave 4064|196654|1693|3|32|56020.80|0.04|0.07|N|O|1997-01-14|1997-01-01|1997-01-21|COLLECT COD|REG AIR|es boost. careful 4064|162125|9674|4|24|28490.88|0.02|0.02|N|O|1997-01-01|1996-12-31|1997-01-23|DELIVER IN PERSON|SHIP|ly regular ideas. 4064|20658|659|5|12|18943.80|0.08|0.08|N|O|1997-02-08|1996-12-18|1997-03-06|TAKE BACK RETURN|RAIL|ding to the requests 4064|183388|943|6|46|67683.48|0.03|0.00|N|O|1996-10-13|1997-01-05|1996-11-06|DELIVER IN PERSON|REG AIR|alongside of the f 4064|199699|2219|7|9|16188.21|0.01|0.06|N|O|1996-12-17|1996-12-13|1997-01-12|NONE|AIR|furiously f 4065|137373|7374|1|14|19745.18|0.04|0.02|A|F|1994-08-22|1994-07-29|1994-09-19|DELIVER IN PERSON|TRUCK|e furiously outside 4065|14060|4061|2|46|44806.76|0.03|0.05|A|F|1994-06-29|1994-08-01|1994-07-03|TAKE BACK RETURN|SHIP|, regular requests may mold above the 4065|96443|1462|3|33|47501.52|0.00|0.03|A|F|1994-09-03|1994-08-16|1994-09-13|DELIVER IN PERSON|AIR|ain blithely 4065|106463|1484|4|8|11755.68|0.00|0.01|R|F|1994-10-04|1994-08-05|1994-10-25|TAKE BACK RETURN|SHIP|ages haggle carefully 4065|122670|2671|5|29|49087.43|0.02|0.07|A|F|1994-06-29|1994-08-19|1994-07-17|NONE|RAIL|equests. packages sleep slyl 4065|109059|4080|6|16|17088.80|0.05|0.00|R|F|1994-08-25|1994-08-06|1994-09-09|COLLECT COD|TRUCK|ncies use furiously. quickly un 4065|143199|8228|7|11|13664.09|0.10|0.04|A|F|1994-07-25|1994-08-02|1994-07-30|NONE|RAIL|hang silently about 4066|138329|5869|1|9|12305.88|0.01|0.05|N|O|1997-05-06|1997-03-25|1997-05-27|COLLECT COD|FOB|nal, ironic accounts. blithel 4066|92550|5060|2|19|29308.45|0.05|0.00|N|O|1997-05-13|1997-04-17|1997-06-08|NONE|TRUCK|quests. slyly regu 4066|75789|804|3|8|14118.24|0.03|0.03|N|O|1997-04-24|1997-03-11|1997-05-20|NONE|REG AIR|accounts. special pinto beans 4066|178139|657|4|49|59639.37|0.01|0.01|N|O|1997-02-17|1997-03-24|1997-02-19|NONE|TRUCK|ial braids. furiously final deposits sl 4066|170234|235|5|43|56081.89|0.05|0.02|N|O|1997-02-16|1997-04-14|1997-02-18|DELIVER IN PERSON|MAIL|r instructions. slyly special 4066|108491|1002|6|44|65977.56|0.01|0.00|N|O|1997-03-01|1997-04-27|1997-03-29|DELIVER IN PERSON|MAIL|express accounts nag bli 4067|179063|9064|1|18|20557.08|0.03|0.08|A|F|1993-01-24|1992-12-23|1993-02-20|TAKE BACK RETURN|FOB|e the slyly final packages d 4067|95338|2866|2|14|18666.62|0.00|0.00|R|F|1993-02-03|1992-12-02|1993-02-07|TAKE BACK RETURN|TRUCK|ructions. quickly ironic accounts detect 4067|140221|7764|3|17|21440.74|0.03|0.05|A|F|1993-01-26|1992-11-23|1993-01-27|NONE|REG AIR|ts haggle slyly unusual, final 4067|89682|2191|4|40|66867.20|0.07|0.08|R|F|1993-01-09|1992-11-21|1993-01-16|DELIVER IN PERSON|TRUCK|lar theodolites nag blithely above the 4067|84828|9845|5|17|30817.94|0.08|0.03|A|F|1993-01-20|1992-12-29|1993-02-03|DELIVER IN PERSON|REG AIR|r accounts. slyly special pa 4067|95501|8011|6|12|17958.00|0.04|0.03|A|F|1992-12-12|1992-11-28|1992-12-15|DELIVER IN PERSON|AIR|lly slyly even theodol 4067|82675|5184|7|17|28180.39|0.01|0.01|R|F|1992-12-12|1992-12-23|1992-12-30|NONE|AIR|ts affix. regular, regular requests s 4068|109101|9102|1|43|47734.30|0.05|0.06|N|O|1996-11-28|1996-11-16|1996-12-22|NONE|AIR|ructions. regular, special packag 4068|56989|9495|2|31|60325.38|0.08|0.03|N|O|1996-12-11|1996-12-07|1996-12-30|NONE|SHIP|ds wake carefully amon 4069|128033|5570|1|39|41380.17|0.09|0.02|R|F|1992-09-06|1992-07-22|1992-09-25|COLLECT COD|SHIP|ven theodolites nag quickly. fluffi 4069|42553|2554|2|32|47857.60|0.10|0.08|A|F|1992-06-18|1992-07-20|1992-07-07|TAKE BACK RETURN|TRUCK|unts. deposit 4069|185023|7542|3|3|3324.06|0.06|0.01|R|F|1992-07-26|1992-07-07|1992-08-04|COLLECT COD|FOB|l packages. even, 4069|78465|3480|4|22|31756.12|0.10|0.05|A|F|1992-08-05|1992-08-04|1992-08-25|COLLECT COD|SHIP|ts. slyly special instruction 4069|156895|9411|5|50|97594.50|0.09|0.06|A|F|1992-07-26|1992-06-30|1992-08-01|TAKE BACK RETURN|REG AIR|even foxes among the express wate 4069|124947|2484|6|3|5915.82|0.02|0.01|A|F|1992-05-24|1992-06-18|1992-06-12|COLLECT COD|MAIL|y final deposits wake furiously! slyl 4069|183483|6002|7|50|78324.00|0.00|0.01|R|F|1992-09-03|1992-06-14|1992-10-01|NONE|REG AIR|ages. carefully regular 4070|182612|7649|1|2|3389.22|0.09|0.08|N|O|1995-08-03|1995-09-10|1995-08-17|TAKE BACK RETURN|REG AIR|ptotes affix 4070|154868|7384|2|40|76914.40|0.07|0.07|N|O|1995-07-13|1995-07-23|1995-08-06|COLLECT COD|MAIL|about the sentiments. quick 4070|61774|1775|3|11|19093.47|0.00|0.08|N|O|1995-08-23|1995-08-15|1995-08-31|TAKE BACK RETURN|MAIL| carefully final pack 4070|28495|998|4|46|65480.54|0.02|0.02|N|O|1995-06-22|1995-07-14|1995-07-11|DELIVER IN PERSON|REG AIR|nticing ideas. boldly 4071|111872|9406|1|22|41445.14|0.02|0.07|N|O|1996-10-31|1996-12-14|1996-11-05|NONE|REG AIR|sits cajole carefully final instructio 4071|17736|5240|2|47|77725.31|0.00|0.03|N|O|1996-11-04|1996-12-09|1996-11-16|NONE|TRUCK|ts cajole furiously along the 4096|26850|4357|1|31|55082.35|0.10|0.02|A|F|1992-07-14|1992-09-03|1992-07-31|COLLECT COD|TRUCK|y final, even platelets. boldly 4096|56025|3541|2|17|16677.34|0.07|0.03|R|F|1992-09-30|1992-08-11|1992-10-11|TAKE BACK RETURN|REG AIR|platelets alongside of the 4096|8635|8636|3|21|32416.23|0.08|0.00|A|F|1992-08-24|1992-09-04|1992-09-11|DELIVER IN PERSON|MAIL|tes mold flu 4096|127636|149|4|20|33272.60|0.02|0.07|R|F|1992-08-24|1992-09-13|1992-08-28|DELIVER IN PERSON|TRUCK|sual requests. furiously bold packages wake 4097|73486|3487|1|50|72974.00|0.04|0.04|N|O|1996-08-31|1996-08-14|1996-09-27|DELIVER IN PERSON|MAIL|egular deposits. blithely pending 4097|73230|5738|2|46|55348.58|0.10|0.01|N|O|1996-07-29|1996-08-19|1996-08-25|COLLECT COD|AIR| even depend 4097|173706|1258|3|42|74747.40|0.06|0.06|N|O|1996-08-11|1996-07-30|1996-08-15|NONE|FOB|carefully silent foxes are against the 4098|199417|9418|1|46|69754.86|0.07|0.03|N|O|1997-01-26|1997-01-27|1997-02-13|TAKE BACK RETURN|SHIP|e slyly blithely silent deposits. fluff 4099|3007|5508|1|29|26390.00|0.09|0.07|R|F|1992-11-21|1992-11-04|1992-11-30|NONE|FOB| slowly final warthogs sleep blithely. q 4099|136994|4534|2|3|6092.97|0.04|0.06|A|F|1992-09-12|1992-10-18|1992-10-01|NONE|RAIL|. special packages sleep 4099|50012|7528|3|36|34632.36|0.06|0.06|R|F|1992-11-06|1992-09-28|1992-12-02|NONE|FOB|beans cajole slyly quickly ironic 4099|138940|6480|4|7|13852.58|0.05|0.02|A|F|1992-09-12|1992-11-13|1992-09-14|TAKE BACK RETURN|AIR|onic foxes. quickly final fox 4099|162936|7969|5|48|95948.64|0.00|0.02|R|F|1992-10-18|1992-10-14|1992-11-01|NONE|REG AIR|ts haggle according to the slyly f 4099|58106|8107|6|39|41499.90|0.07|0.02|R|F|1992-12-13|1992-11-13|1992-12-26|DELIVER IN PERSON|REG AIR|fluffy accounts impress pending, iro 4099|179228|6780|7|46|60132.12|0.06|0.07|R|F|1992-10-29|1992-11-03|1992-11-10|DELIVER IN PERSON|REG AIR|ages nag requests. 4100|73624|3625|1|4|6390.48|0.03|0.03|N|O|1996-06-20|1996-04-29|1996-06-21|TAKE BACK RETURN|FOB|lyly regular, bold requ 4101|114376|9399|1|22|30588.14|0.05|0.02|R|F|1994-02-02|1994-02-19|1994-02-12|COLLECT COD|AIR|ly express instructions. careful 4102|9762|2263|1|17|28419.92|0.02|0.02|N|O|1996-06-03|1996-05-06|1996-07-02|COLLECT COD|AIR|ly silent theodolites sleep unusual exc 4102|68535|1042|2|5|7517.65|0.08|0.02|N|O|1996-05-11|1996-05-11|1996-05-16|COLLECT COD|AIR| the furiously even 4102|66779|1792|3|39|68085.03|0.08|0.01|N|O|1996-04-14|1996-05-18|1996-04-20|DELIVER IN PERSON|AIR|ffix blithely slyly special 4102|139097|1611|4|39|44307.51|0.02|0.00|N|O|1996-06-15|1996-06-06|1996-06-30|DELIVER IN PERSON|SHIP|y among the furiously special 4102|175|5176|5|32|34405.44|0.08|0.01|N|O|1996-05-14|1996-04-29|1996-05-29|NONE|RAIL| the even requests; regular pinto 4102|136116|1143|6|7|8064.77|0.02|0.01|N|O|1996-06-19|1996-05-21|1996-07-15|NONE|REG AIR|bove the carefully pending the 4103|74738|9753|1|40|68509.20|0.05|0.03|R|F|1992-09-19|1992-08-14|1992-09-21|COLLECT COD|RAIL|usly across the slyly busy accounts! fin 4128|195824|8344|1|5|9599.10|0.04|0.04|N|O|1995-10-18|1995-11-28|1995-10-28|TAKE BACK RETURN|FOB|ake permanently 4129|55253|2769|1|32|38664.00|0.03|0.04|A|F|1993-09-16|1993-08-25|1993-09-25|TAKE BACK RETURN|MAIL|ckages haggl 4129|26905|1910|2|39|71444.10|0.06|0.07|R|F|1993-10-21|1993-08-04|1993-10-29|COLLECT COD|MAIL|y regular foxes. slyly ironic deposits 4130|177966|5518|1|44|89934.24|0.07|0.04|N|O|1996-05-14|1996-04-15|1996-05-15|COLLECT COD|TRUCK|eaves haggle qui 4130|62910|7923|2|2|3745.82|0.05|0.06|N|O|1996-05-19|1996-04-24|1996-06-17|TAKE BACK RETURN|RAIL|uriously regular instructions around th 4131|49740|2245|1|6|10138.44|0.05|0.01|N|O|1998-04-27|1998-04-18|1998-04-29|TAKE BACK RETURN|MAIL|ns cajole slyly. even, iro 4131|177535|53|2|32|51600.96|0.08|0.01|N|O|1998-03-02|1998-03-21|1998-03-07|TAKE BACK RETURN|TRUCK| furiously regular asymptotes nod sly 4131|25254|2761|3|25|29481.25|0.02|0.07|N|O|1998-02-24|1998-03-01|1998-02-27|TAKE BACK RETURN|FOB|uickly exp 4131|35077|84|4|8|8096.56|0.04|0.01|N|O|1998-03-03|1998-03-15|1998-03-26|COLLECT COD|FOB| after the furiously ironic d 4131|124360|4361|5|30|41530.80|0.01|0.01|N|O|1998-04-01|1998-04-13|1998-04-08|TAKE BACK RETURN|FOB|he fluffily express depen 4131|101926|6947|6|47|90612.24|0.02|0.00|N|O|1998-03-09|1998-04-05|1998-03-13|TAKE BACK RETURN|RAIL|ges. ironic pinto be 4132|137396|9910|1|28|40134.92|0.07|0.03|N|O|1995-08-16|1995-08-01|1995-08-29|TAKE BACK RETURN|SHIP|pths wake against the stealthily special pi 4132|14238|1742|2|23|26501.29|0.07|0.07|N|O|1995-06-27|1995-07-27|1995-07-13|TAKE BACK RETURN|FOB|d deposits. fluffily even requests haggle b 4132|86626|1643|3|18|29027.16|0.09|0.04|A|F|1995-06-01|1995-08-01|1995-06-02|TAKE BACK RETURN|RAIL|y final de 4133|23646|3647|1|35|54937.40|0.02|0.00|A|F|1992-11-25|1992-09-15|1992-12-25|NONE|AIR|g above the quickly bold packages. ev 4134|120218|7755|1|34|42099.14|0.02|0.05|R|F|1995-04-29|1995-03-13|1995-05-11|DELIVER IN PERSON|FOB|e furiously regular sheaves sleep 4134|95661|3189|2|34|56326.44|0.01|0.03|A|F|1995-05-06|1995-03-28|1995-05-13|DELIVER IN PERSON|SHIP|ual asymptotes wake carefully alo 4134|170507|8059|3|12|18930.00|0.05|0.04|A|F|1995-03-19|1995-03-27|1995-04-14|COLLECT COD|TRUCK|kly above the quickly regular 4134|99989|7517|4|45|89504.10|0.08|0.02|A|F|1995-04-11|1995-03-27|1995-04-17|TAKE BACK RETURN|MAIL|ironic pin 4135|1947|1948|1|23|42525.62|0.06|0.01|N|O|1997-04-09|1997-05-12|1997-04-16|TAKE BACK RETURN|FOB|posits cajole furiously carefully 4135|119317|9318|2|32|42761.92|0.07|0.00|N|O|1997-03-14|1997-04-23|1997-04-12|TAKE BACK RETURN|TRUCK| ideas. requests use. furiously 4135|159231|4262|3|33|42577.59|0.05|0.05|N|O|1997-05-01|1997-05-23|1997-05-23|DELIVER IN PERSON|AIR|he fluffil 4135|194719|4720|4|13|23578.23|0.04|0.07|N|O|1997-03-16|1997-05-19|1997-04-03|COLLECT COD|RAIL|efully special account 4160|112413|7436|1|25|35635.25|0.10|0.04|N|O|1996-09-22|1996-10-17|1996-09-24|NONE|SHIP|ar accounts sleep blithe 4160|121668|4181|2|12|20275.92|0.00|0.03|N|O|1996-11-22|1996-09-25|1996-12-10|DELIVER IN PERSON|REG AIR|y bold package 4160|62498|2499|3|48|70103.52|0.04|0.04|N|O|1996-09-19|1996-11-02|1996-09-24|COLLECT COD|FOB| unusual dolphins 4161|121535|4048|1|12|18678.36|0.08|0.02|R|F|1993-08-25|1993-10-04|1993-09-22|COLLECT COD|RAIL|onic dolphins. in 4161|27319|9822|2|47|58576.57|0.05|0.00|A|F|1993-12-20|1993-10-29|1994-01-19|TAKE BACK RETURN|RAIL|r requests about the final, even foxes hag 4161|137027|9541|3|42|44688.84|0.03|0.04|R|F|1993-11-12|1993-10-04|1993-11-27|COLLECT COD|MAIL|thely across the even attainments. express 4161|9463|4464|4|45|61760.70|0.02|0.06|A|F|1993-10-22|1993-10-17|1993-10-30|COLLECT COD|REG AIR|about the ironic packages cajole blithe 4161|28343|8344|5|46|58481.64|0.05|0.01|A|F|1993-11-09|1993-11-17|1993-11-17|TAKE BACK RETURN|TRUCK|he stealthily ironic foxes. ideas haggl 4161|147061|7062|6|19|21053.14|0.07|0.00|R|F|1993-08-22|1993-11-11|1993-09-01|TAKE BACK RETURN|REG AIR|beans breach s 4162|73106|8121|1|45|48559.50|0.10|0.07|A|F|1992-03-21|1992-05-02|1992-03-29|DELIVER IN PERSON|AIR|elets. slyly regular i 4162|89349|9350|2|29|38811.86|0.00|0.05|R|F|1992-02-25|1992-04-25|1992-03-17|NONE|REG AIR|nding pinto beans haggle blithe 4163|32870|2871|1|13|23437.31|0.08|0.03|A|F|1993-02-17|1993-03-13|1993-03-15|DELIVER IN PERSON|REG AIR|phins wake. pending requests inte 4164|119658|4681|1|9|15098.85|0.07|0.02|N|O|1998-08-25|1998-08-13|1998-09-19|DELIVER IN PERSON|SHIP|re fluffily slyly bold requests. 4165|40102|103|1|12|12505.20|0.00|0.01|N|O|1997-09-20|1997-10-20|1997-10-12|TAKE BACK RETURN|REG AIR|nwind slow theodolites. carefully pending 4166|140721|8264|1|8|14093.76|0.00|0.08|A|F|1993-06-05|1993-04-10|1993-07-05|COLLECT COD|MAIL|uickly. blithely pending de 4166|92165|4675|2|8|9257.28|0.06|0.04|A|F|1993-06-07|1993-04-17|1993-06-16|DELIVER IN PERSON|REG AIR|es along the furiously regular acc 4166|6324|8825|3|17|20915.44|0.02|0.06|R|F|1993-06-29|1993-05-15|1993-07-24|DELIVER IN PERSON|SHIP|ackages. re 4166|85553|3078|4|36|55387.80|0.06|0.05|R|F|1993-03-01|1993-05-25|1993-03-05|COLLECT COD|MAIL|unts. furiously express accounts w 4166|76379|1394|5|5|6776.85|0.08|0.01|A|F|1993-06-19|1993-04-24|1993-06-27|NONE|REG AIR|hely unusual packages are above the f 4166|101840|4351|6|6|11051.04|0.04|0.08|R|F|1993-04-30|1993-04-17|1993-05-08|DELIVER IN PERSON|MAIL|ily ironic deposits print furiously. iron 4166|23149|3150|7|26|27875.64|0.09|0.01|R|F|1993-03-17|1993-05-09|1993-03-25|NONE|MAIL|lar dependencies. s 4167|60623|5636|1|47|74430.14|0.04|0.02|N|O|1998-08-02|1998-08-24|1998-08-28|DELIVER IN PERSON|REG AIR| carefully final asymptotes. slyly bo 4167|86504|6505|2|17|25338.50|0.06|0.07|N|O|1998-09-18|1998-09-06|1998-10-07|COLLECT COD|REG AIR|ly around the even instr 4167|72505|5013|3|1|1477.50|0.03|0.06|N|O|1998-10-11|1998-08-14|1998-10-13|COLLECT COD|TRUCK|xpress platelets. blithely 4192|10837|8341|1|36|62921.88|0.06|0.08|N|O|1998-04-25|1998-05-26|1998-05-03|COLLECT COD|TRUCK|eodolites sleep 4192|120696|3209|2|15|25750.35|0.04|0.08|N|O|1998-06-26|1998-05-26|1998-07-16|COLLECT COD|AIR|e slyly special grouches. express pinto b 4192|134922|9949|3|7|13698.44|0.06|0.03|N|O|1998-05-19|1998-07-08|1998-05-31|COLLECT COD|FOB|y; excuses use. ironic, close instru 4192|23256|8261|4|32|37736.00|0.09|0.04|N|O|1998-06-23|1998-06-25|1998-07-17|NONE|FOB|ounts are fluffily slyly bold req 4192|47037|4550|5|48|47233.44|0.08|0.01|N|O|1998-08-17|1998-07-11|1998-09-03|NONE|AIR|ests. quickly bol 4192|149567|2082|6|44|71128.64|0.10|0.02|N|O|1998-08-06|1998-07-09|1998-08-20|NONE|FOB|structions mai 4192|169807|7356|7|27|50673.60|0.02|0.00|N|O|1998-07-03|1998-06-26|1998-07-13|TAKE BACK RETURN|AIR| carefully even escapades. care 4193|130286|7826|1|37|48702.36|0.09|0.06|A|F|1994-04-25|1994-02-24|1994-05-08|NONE|AIR|er the quickly regular dependencies wake 4193|116403|3937|2|3|4258.20|0.09|0.05|R|F|1994-04-29|1994-03-20|1994-05-29|TAKE BACK RETURN|REG AIR|osits above the depo 4193|178441|8442|3|10|15194.40|0.06|0.03|A|F|1994-02-10|1994-03-22|1994-03-09|COLLECT COD|RAIL|uffily spe 4193|50040|2546|4|29|28711.16|0.09|0.05|A|F|1994-02-11|1994-03-11|1994-03-13|TAKE BACK RETURN|RAIL|ly. final packages use blit 4193|19877|4880|5|50|89843.50|0.01|0.01|R|F|1994-04-28|1994-03-23|1994-05-09|NONE|FOB| beans. regular accounts cajole. de 4193|65248|2767|6|21|25478.04|0.02|0.04|R|F|1994-04-26|1994-03-22|1994-05-23|DELIVER IN PERSON|TRUCK|accounts cajole b 4194|196755|4313|1|43|79625.25|0.08|0.06|A|F|1994-11-06|1994-12-09|1994-11-16|NONE|TRUCK|olites are after the exp 4194|46577|1586|2|18|27424.26|0.07|0.07|A|F|1995-02-14|1994-12-04|1995-03-11|TAKE BACK RETURN|TRUCK|ld packages. quickly eve 4195|5693|8194|1|14|22381.66|0.09|0.04|R|F|1993-09-06|1993-07-21|1993-09-18|DELIVER IN PERSON|REG AIR|ironic packages. carefully express 4195|65973|3492|2|22|42657.34|0.10|0.08|R|F|1993-07-01|1993-07-23|1993-07-28|COLLECT COD|RAIL|lly express pinto bea 4195|193646|1204|3|19|33053.16|0.01|0.06|R|F|1993-09-06|1993-08-13|1993-09-15|TAKE BACK RETURN|REG AIR|telets sleep even requests. final, even i 4196|155079|7595|1|30|34022.10|0.02|0.06|N|O|1998-08-09|1998-06-30|1998-09-05|COLLECT COD|SHIP|egular foxes us 4196|8234|5735|2|31|35409.13|0.09|0.08|N|O|1998-06-12|1998-07-28|1998-07-11|NONE|MAIL|ut the blithely ironic inst 4196|177505|7506|3|46|72795.00|0.05|0.00|N|O|1998-09-05|1998-06-28|1998-09-10|TAKE BACK RETURN|MAIL|according to t 4196|113898|6410|4|42|80299.38|0.04|0.06|N|O|1998-08-13|1998-07-18|1998-09-07|TAKE BACK RETURN|AIR| instructions. courts cajole slyly ev 4196|71690|4198|5|3|4985.07|0.01|0.03|N|O|1998-05-17|1998-07-21|1998-05-18|DELIVER IN PERSON|TRUCK| accounts. fu 4196|86060|8569|6|43|44980.58|0.01|0.06|N|O|1998-08-12|1998-07-12|1998-08-22|DELIVER IN PERSON|FOB|es. slyly even 4196|3066|567|7|3|2907.18|0.00|0.06|N|O|1998-08-05|1998-07-28|1998-08-15|DELIVER IN PERSON|REG AIR|y regular packages haggle furiously alongs 4197|128035|3060|1|50|53151.50|0.06|0.03|N|O|1996-11-15|1996-11-01|1996-11-20|NONE|FOB|. carefully bold asymptotes nag blithe 4197|69664|2171|2|39|63712.74|0.02|0.08|N|O|1996-10-07|1996-10-11|1996-10-18|DELIVER IN PERSON|RAIL|ronic requests. quickly bold packages in 4197|31921|4425|3|28|51881.76|0.06|0.02|N|O|1996-10-05|1996-10-24|1996-10-22|TAKE BACK RETURN|AIR|regular pin 4197|95301|5302|4|23|29814.90|0.00|0.03|N|O|1996-09-10|1996-10-10|1996-09-25|NONE|AIR|l instructions print slyly past the reg 4197|120952|3465|5|37|72999.15|0.03|0.04|N|O|1996-10-20|1996-10-10|1996-11-10|COLLECT COD|TRUCK|carefully enticing decoys boo 4197|30658|3162|6|48|76255.20|0.08|0.00|N|O|1996-10-07|1996-10-25|1996-10-23|COLLECT COD|REG AIR| final instructions. blithe, spe 4198|145163|192|1|48|57991.68|0.09|0.05|N|O|1997-09-03|1997-07-18|1997-09-11|NONE|REG AIR|cajole carefully final, ironic ide 4198|142247|7276|2|46|59305.04|0.09|0.01|N|O|1997-08-17|1997-09-08|1997-09-11|COLLECT COD|TRUCK|posits among th 4198|144524|2067|3|13|20390.76|0.03|0.04|N|O|1997-07-18|1997-07-24|1997-08-10|NONE|REG AIR| furious excuses. bli 4199|69662|7181|1|16|26106.56|0.10|0.00|A|F|1992-06-11|1992-04-10|1992-07-10|COLLECT COD|TRUCK|ncies. furiously special accounts 4199|8840|8841|2|18|31479.12|0.00|0.01|A|F|1992-06-01|1992-03-30|1992-06-28|DELIVER IN PERSON|RAIL|pending, regular accounts. carefully 4224|198265|8266|1|27|36808.02|0.05|0.03|N|O|1997-09-05|1997-08-19|1997-09-30|NONE|SHIP|ly special deposits sleep qui 4224|36701|4211|2|20|32754.00|0.07|0.05|N|O|1997-11-09|1997-08-23|1997-11-14|NONE|FOB|unts promise across the requests. blith 4224|23726|1233|3|4|6598.88|0.08|0.05|N|O|1997-09-07|1997-09-05|1997-09-25|TAKE BACK RETURN|FOB| even dinos. carefull 4224|159653|7199|4|50|85632.50|0.10|0.06|N|O|1997-07-30|1997-09-10|1997-08-19|COLLECT COD|RAIL|side of the carefully silent dep 4224|84136|6645|5|48|53766.24|0.00|0.04|N|O|1997-10-03|1997-08-31|1997-10-10|NONE|RAIL| final, regular asymptotes use alway 4225|48772|6285|1|25|43019.25|0.08|0.04|N|O|1997-07-10|1997-08-08|1997-07-31|TAKE BACK RETURN|TRUCK|se fluffily. busily ironic requests are; 4225|95002|7512|2|23|22931.00|0.02|0.04|N|O|1997-09-18|1997-08-31|1997-10-11|TAKE BACK RETURN|RAIL|. quickly b 4225|97926|436|3|28|53869.76|0.08|0.03|N|O|1997-07-11|1997-09-01|1997-08-03|DELIVER IN PERSON|FOB|ts are requests. even, bold depos 4226|187065|9584|1|27|31105.62|0.06|0.08|A|F|1993-05-03|1993-04-12|1993-05-16|COLLECT COD|AIR|sly alongside of the slyly ironic pac 4227|157199|9715|1|19|23867.61|0.01|0.08|A|F|1995-05-05|1995-05-03|1995-05-22|COLLECT COD|REG AIR|ns sleep along the blithely even theodolit 4227|32622|2623|2|8|12436.96|0.09|0.00|N|F|1995-06-11|1995-04-30|1995-06-28|COLLECT COD|REG AIR| packages since the bold, u 4227|74624|4625|3|11|17584.82|0.10|0.04|A|F|1995-03-30|1995-05-02|1995-04-26|DELIVER IN PERSON|SHIP|l requests-- bold requests cajole dogg 4227|199283|6841|4|2|2764.56|0.02|0.05|R|F|1995-04-24|1995-05-09|1995-05-21|DELIVER IN PERSON|AIR|ep. specia 4227|146835|4378|5|49|92209.67|0.05|0.06|R|F|1995-05-19|1995-04-12|1995-06-12|TAKE BACK RETURN|REG AIR|ts sleep blithely carefully unusual ideas. 4228|140949|8492|1|20|39798.80|0.00|0.06|N|O|1997-04-24|1997-05-29|1997-05-17|NONE|RAIL|f the slyly fluffy pinto beans are 4229|95318|337|1|44|57785.64|0.02|0.05|N|O|1998-05-29|1998-05-12|1998-06-16|DELIVER IN PERSON|AIR|s. carefully e 4229|4570|7071|2|34|50135.38|0.07|0.05|N|O|1998-05-26|1998-04-13|1998-06-08|DELIVER IN PERSON|MAIL|thely final accounts use even packa 4230|45124|2637|1|38|40626.56|0.10|0.03|A|F|1992-04-28|1992-04-21|1992-05-28|TAKE BACK RETURN|FOB|ly regular packages. regular ideas boost 4230|198644|6202|2|43|74933.52|0.02|0.08|R|F|1992-03-14|1992-05-13|1992-03-28|NONE|FOB|ses lose blithely slyly final e 4230|195055|94|3|10|11500.50|0.06|0.02|A|F|1992-06-11|1992-04-11|1992-07-02|TAKE BACK RETURN|MAIL|ar packages are 4230|74334|4335|4|28|36633.24|0.01|0.03|R|F|1992-05-12|1992-05-10|1992-06-01|TAKE BACK RETURN|MAIL|nt instruct 4230|124748|7261|5|50|88637.00|0.00|0.01|A|F|1992-03-29|1992-05-19|1992-04-20|TAKE BACK RETURN|SHIP|ts. final instructions in 4230|34864|9871|6|30|53965.80|0.05|0.07|A|F|1992-03-11|1992-04-29|1992-03-30|NONE|AIR|s. final excuses across the 4230|151041|1042|7|18|19656.72|0.10|0.04|R|F|1992-06-23|1992-05-10|1992-07-04|COLLECT COD|SHIP| the final acco 4231|141535|1536|1|47|74096.91|0.09|0.03|N|O|1997-11-27|1998-01-26|1997-12-17|NONE|REG AIR|hely along the silent at 4231|165189|222|2|4|5016.72|0.06|0.02|N|O|1997-11-28|1998-01-26|1997-12-12|TAKE BACK RETURN|MAIL|lithely even packages. 4231|120334|335|3|31|41984.23|0.07|0.08|N|O|1998-02-14|1997-12-27|1998-03-01|DELIVER IN PERSON|FOB|ublate. theodoli 4231|39030|4037|4|35|33916.05|0.10|0.00|N|O|1998-02-21|1998-01-24|1998-03-18|DELIVER IN PERSON|FOB|le quickly regular, unus 4256|150384|2900|1|22|31556.36|0.05|0.05|R|F|1992-07-30|1992-05-14|1992-08-14|NONE|TRUCK|, final platelets are slyly final pint 4257|64416|1935|1|3|4141.23|0.10|0.03|N|O|1995-06-18|1995-05-01|1995-07-12|DELIVER IN PERSON|MAIL|thin the theodolites use after the bl 4257|34970|9977|2|5|9524.85|0.01|0.04|R|F|1995-04-29|1995-06-05|1995-05-13|TAKE BACK RETURN|TRUCK|n deposits. furiously e 4257|127007|7008|3|33|34122.00|0.03|0.04|A|F|1995-05-23|1995-05-03|1995-05-31|COLLECT COD|AIR|uffily regular accounts ar 4258|165025|5026|1|36|39240.72|0.02|0.06|N|O|1997-02-23|1997-01-25|1997-02-27|TAKE BACK RETURN|SHIP|ns use alongs 4258|161733|4250|2|19|34099.87|0.03|0.02|N|O|1997-01-14|1996-12-12|1997-01-20|TAKE BACK RETURN|AIR|ly busily ironic foxes. f 4258|30847|8357|3|46|81780.64|0.04|0.07|N|O|1997-01-02|1996-12-26|1997-01-12|DELIVER IN PERSON|AIR| furiously pend 4258|34234|4235|4|22|25701.06|0.04|0.04|N|O|1996-12-12|1996-12-06|1996-12-20|TAKE BACK RETURN|AIR|e regular, even asym 4258|162806|7839|5|9|16819.20|0.04|0.03|N|O|1996-12-04|1996-12-08|1996-12-20|DELIVER IN PERSON|TRUCK|counts wake permanently after the bravely 4259|42595|7604|1|14|21526.26|0.05|0.03|N|O|1998-01-09|1997-11-21|1998-01-29|TAKE BACK RETURN|RAIL| furiously pending excuses. ideas hagg 4260|23397|904|1|21|27728.19|0.08|0.04|R|F|1992-08-06|1992-06-18|1992-08-22|NONE|AIR|al, pending accounts must 4261|109388|9389|1|12|16768.56|0.05|0.01|A|F|1992-11-01|1993-01-01|1992-11-12|NONE|FOB|into beans 4261|81703|9228|2|4|6738.80|0.02|0.07|R|F|1992-12-11|1992-12-18|1992-12-24|DELIVER IN PERSON|FOB|ackages unwind furiously fluff 4261|174248|6766|3|3|3966.72|0.07|0.02|R|F|1992-11-10|1992-12-14|1992-11-17|COLLECT COD|RAIL|ly even deposits eat blithely alo 4261|173239|8274|4|36|47240.28|0.04|0.06|R|F|1992-12-02|1992-12-18|1992-12-25|NONE|REG AIR| slyly pendi 4261|23678|1185|5|28|44846.76|0.07|0.06|A|F|1992-10-08|1992-12-23|1992-10-11|TAKE BACK RETURN|MAIL|packages. fluffily i 4262|75546|5547|1|30|45646.20|0.01|0.03|N|O|1996-08-11|1996-10-11|1996-09-09|TAKE BACK RETURN|RAIL|tes after the carefully 4262|95696|5697|2|5|8458.45|0.02|0.05|N|O|1996-09-27|1996-09-05|1996-10-25|COLLECT COD|SHIP|blithely final asymptotes integrate 4262|161033|3550|3|5|5470.15|0.08|0.00|N|O|1996-10-02|1996-10-16|1996-10-05|NONE|REG AIR|ironic accounts are unusu 4262|73333|855|4|45|58784.85|0.02|0.01|N|O|1996-11-09|1996-09-09|1996-11-12|DELIVER IN PERSON|SHIP|ackages boost. pending, even instruction 4262|99103|4122|5|28|30858.80|0.06|0.02|N|O|1996-10-22|1996-09-06|1996-11-13|DELIVER IN PERSON|FOB|ironic, regular depend 4262|16420|3924|6|26|34746.92|0.03|0.02|N|O|1996-08-29|1996-09-25|1996-08-31|NONE|RAIL|s boost slyly along the bold, iro 4262|159435|4466|7|41|61271.63|0.03|0.01|N|O|1996-08-28|1996-09-14|1996-09-20|COLLECT COD|RAIL|cuses unwind ac 4263|17017|7018|1|9|8406.09|0.08|0.07|N|O|1998-04-04|1998-04-29|1998-05-04|COLLECT COD|AIR|structions cajole quic 4263|195562|3120|2|28|46411.68|0.05|0.03|N|O|1998-06-24|1998-06-08|1998-07-14|NONE|MAIL|ideas for the carefully re 4263|10259|7763|3|38|44431.50|0.01|0.01|N|O|1998-07-10|1998-05-08|1998-07-17|NONE|TRUCK|rding to the dep 4263|18148|650|4|20|21322.80|0.02|0.07|N|O|1998-04-09|1998-04-30|1998-05-04|NONE|RAIL|uietly regular deposits. sly deposits w 4263|197901|5459|5|14|27984.60|0.09|0.06|N|O|1998-05-06|1998-04-17|1998-05-11|DELIVER IN PERSON|TRUCK|d accounts. daringly regular accounts hagg 4263|112955|7978|6|47|92493.65|0.08|0.06|N|O|1998-06-28|1998-05-09|1998-07-02|DELIVER IN PERSON|TRUCK|y. theodolites wake idly ironic do 4263|28779|1282|7|6|10246.62|0.04|0.04|N|O|1998-05-01|1998-06-02|1998-05-14|TAKE BACK RETURN|REG AIR|g the final, regular instructions: 4288|73060|3061|1|32|33057.92|0.10|0.07|R|F|1993-03-19|1993-01-26|1993-04-18|TAKE BACK RETURN|AIR|e blithely even instructions. speci 4288|104952|4953|2|39|76321.05|0.05|0.02|R|F|1993-03-25|1993-02-06|1993-03-28|DELIVER IN PERSON|AIR|uffy theodolites run 4288|124632|2169|3|7|11596.41|0.03|0.01|A|F|1993-01-15|1993-02-05|1993-01-26|NONE|TRUCK|ngside of the special platelet 4289|195619|5620|1|19|32577.59|0.06|0.06|R|F|1993-12-31|1993-11-06|1994-01-23|DELIVER IN PERSON|TRUCK|e carefully regular ideas. sl 4290|136795|9309|1|23|42131.17|0.06|0.04|R|F|1995-04-04|1995-02-16|1995-04-07|TAKE BACK RETURN|REG AIR|uests cajole carefully. 4290|98378|3397|2|3|4129.11|0.09|0.03|A|F|1995-03-25|1995-03-07|1995-04-11|NONE|RAIL|lar platelets cajole 4291|191809|9367|1|3|5702.40|0.08|0.08|A|F|1994-03-17|1994-02-21|1994-03-27|COLLECT COD|SHIP|tes sleep slyly above the quickly sl 4291|124889|2426|2|43|82296.84|0.01|0.06|A|F|1994-02-01|1994-02-27|1994-02-06|DELIVER IN PERSON|REG AIR|s. quietly regular 4291|7976|477|3|25|47099.25|0.09|0.08|R|F|1994-02-14|1994-02-08|1994-03-15|COLLECT COD|AIR|uctions. furiously regular ins 4292|43509|1022|1|22|31955.00|0.08|0.03|R|F|1992-02-14|1992-02-16|1992-03-01|NONE|FOB|refully expres 4292|39824|7334|2|1|1763.82|0.03|0.01|A|F|1992-02-07|1992-03-16|1992-02-10|DELIVER IN PERSON|FOB| the furiously ev 4292|119234|6768|3|35|43863.05|0.03|0.06|A|F|1992-03-23|1992-04-04|1992-04-02|COLLECT COD|TRUCK|dugouts use. furiously bold packag 4292|162381|7414|4|40|57735.20|0.05|0.04|A|F|1992-04-27|1992-03-07|1992-05-04|COLLECT COD|REG AIR|ounts according to the furiously 4292|130292|7832|5|6|7933.74|0.07|0.08|R|F|1992-03-03|1992-02-24|1992-03-25|COLLECT COD|FOB|bove the silently regula 4292|3041|542|6|47|44369.88|0.05|0.00|R|F|1992-05-02|1992-03-21|1992-05-27|TAKE BACK RETURN|FOB|y packages; even ideas boost 4293|957|5958|1|34|63170.30|0.03|0.08|N|O|1996-11-05|1996-10-12|1996-12-04|NONE|FOB|ions sleep blithely on 4293|76837|4359|2|50|90691.50|0.01|0.05|N|O|1996-11-27|1996-10-30|1996-12-22|COLLECT COD|MAIL| special deposits. furiousl 4293|198566|1086|3|47|78234.32|0.08|0.02|N|O|1996-09-07|1996-10-24|1996-09-15|NONE|RAIL|ithely pending deposits af 4293|87832|5357|4|25|45495.75|0.04|0.04|N|O|1996-09-11|1996-11-14|1996-09-22|DELIVER IN PERSON|FOB|inal asympt 4293|180656|3175|5|1|1736.65|0.06|0.05|N|O|1996-11-15|1996-10-09|1996-11-26|COLLECT COD|AIR|eposits should boost along the 4293|78993|6515|6|45|88739.55|0.10|0.04|N|O|1996-11-04|1996-11-06|1996-11-23|NONE|MAIL|lar ideas use carefully 4294|104911|7422|1|19|36402.29|0.03|0.04|A|F|1992-10-16|1992-11-13|1992-10-26|DELIVER IN PERSON|AIR|nt dependencies. furiously regular ideas d 4294|26561|9064|2|16|23800.96|0.01|0.02|R|F|1992-08-17|1992-09-24|1992-09-04|TAKE BACK RETURN|REG AIR|lithely pint 4294|197032|2071|3|30|33870.90|0.01|0.00|A|F|1992-09-12|1992-11-06|1992-09-25|NONE|MAIL|olites. bold foxes affix ironic theodolite 4294|104405|1936|4|34|47919.60|0.02|0.01|R|F|1992-09-09|1992-11-06|1992-10-04|TAKE BACK RETURN|REG AIR|pendencies! 4294|118956|1468|5|37|73073.15|0.05|0.01|R|F|1992-09-07|1992-10-13|1992-09-08|NONE|REG AIR|cial packages nag f 4294|86273|8782|6|42|52889.34|0.02|0.03|A|F|1992-09-30|1992-11-13|1992-10-15|DELIVER IN PERSON|FOB| carefully; furiously ex 4294|174787|2339|7|47|87503.66|0.02|0.08|R|F|1992-11-09|1992-11-03|1992-12-05|TAKE BACK RETURN|SHIP|es. blithely r 4295|28410|5917|1|49|65582.09|0.09|0.01|N|O|1996-05-25|1996-03-17|1996-06-19|TAKE BACK RETURN|REG AIR|refully silent requests. f 4295|70554|8076|2|4|6098.20|0.09|0.07|N|O|1996-06-05|1996-04-26|1996-06-13|DELIVER IN PERSON|TRUCK|arefully according to the pending ac 4295|192627|2628|3|3|5158.86|0.04|0.00|N|O|1996-06-04|1996-04-24|1996-06-24|DELIVER IN PERSON|AIR|telets cajole bravely 4295|79439|4454|4|30|42552.90|0.07|0.06|N|O|1996-03-22|1996-04-23|1996-04-20|NONE|SHIP|yly ironic frets. pending foxes after 4320|45894|3407|1|28|51516.92|0.02|0.06|N|O|1997-01-28|1997-02-07|1997-02-07|COLLECT COD|FOB|nts. even, ironic excuses hagg 4320|139168|1682|2|6|7242.96|0.08|0.08|N|O|1997-01-11|1997-01-26|1997-01-22|DELIVER IN PERSON|SHIP|against the carefully careful asym 4320|187628|2665|3|33|56615.46|0.09|0.02|N|O|1996-12-11|1997-02-27|1997-01-08|TAKE BACK RETURN|SHIP|ess asymptotes so 4321|146895|4438|1|33|64082.37|0.09|0.02|A|F|1994-09-01|1994-08-17|1994-09-05|DELIVER IN PERSON|TRUCK|yly special excuses. fluffily 4321|53251|5757|2|45|54191.25|0.00|0.08|R|F|1994-11-13|1994-09-15|1994-11-18|DELIVER IN PERSON|SHIP| haggle ironically bold theodolites. quick 4321|185298|5299|3|23|31815.67|0.01|0.05|A|F|1994-11-03|1994-10-08|1994-11-06|DELIVER IN PERSON|SHIP|ly even orbits slee 4321|90271|272|4|4|5045.08|0.02|0.00|R|F|1994-09-10|1994-10-06|1994-09-11|NONE|FOB|ironic deposi 4321|171625|4143|5|10|16966.20|0.04|0.03|A|F|1994-09-07|1994-08-23|1994-09-17|TAKE BACK RETURN|SHIP|wake carefully alongside of 4322|68766|6285|1|39|67655.64|0.04|0.02|N|O|1998-04-27|1998-06-03|1998-05-04|TAKE BACK RETURN|MAIL|its integrate fluffily 4322|139800|9801|2|9|16558.20|0.05|0.08|N|O|1998-05-18|1998-04-27|1998-05-28|COLLECT COD|AIR|ual instructio 4322|7548|7549|3|12|17466.48|0.09|0.05|N|O|1998-03-29|1998-06-05|1998-04-16|DELIVER IN PERSON|TRUCK|e blithely against the slyly unusu 4322|45880|5881|4|17|31039.96|0.09|0.08|N|O|1998-05-31|1998-05-31|1998-06-10|TAKE BACK RETURN|FOB|ructions boost 4322|101331|6352|5|10|13323.30|0.00|0.05|N|O|1998-05-31|1998-04-27|1998-06-25|TAKE BACK RETURN|REG AIR| regular ideas engage carefully quick 4322|59477|1983|6|39|56022.33|0.09|0.08|N|O|1998-03-16|1998-05-21|1998-04-11|COLLECT COD|AIR|ccounts. dogged pin 4322|13150|654|7|34|36147.10|0.05|0.00|N|O|1998-05-27|1998-04-12|1998-06-16|NONE|REG AIR|ounts haggle fluffily ideas. pend 4323|358|359|1|33|41525.55|0.09|0.02|A|F|1994-05-04|1994-03-06|1994-05-23|COLLECT COD|TRUCK|the slyly bold deposits slee 4324|50292|293|1|44|54660.76|0.05|0.04|N|O|1995-10-15|1995-09-07|1995-11-07|DELIVER IN PERSON|AIR|ainst the u 4324|47117|4630|2|12|12769.32|0.04|0.02|N|O|1995-10-05|1995-09-07|1995-10-18|NONE|REG AIR|c packages. furiously express sauternes 4324|81374|8899|3|14|18975.18|0.07|0.06|N|O|1995-11-12|1995-08-26|1995-11-21|COLLECT COD|AIR| packages nag express excuses. qui 4324|49985|2490|4|14|27089.72|0.02|0.04|N|O|1995-09-20|1995-10-08|1995-10-06|COLLECT COD|RAIL| express ideas. blithely blit 4324|83480|5989|5|22|32196.56|0.07|0.03|N|O|1995-09-13|1995-10-04|1995-09-23|DELIVER IN PERSON|SHIP|ke express, special ideas. 4324|42432|9945|6|31|42607.33|0.08|0.04|N|O|1995-10-23|1995-09-14|1995-11-09|COLLECT COD|RAIL|efully flu 4324|153018|564|7|46|49266.46|0.00|0.03|N|O|1995-11-03|1995-09-28|1995-11-22|NONE|SHIP|ular, final theodo 4325|159610|7156|1|18|30052.98|0.01|0.07|N|O|1996-10-07|1996-09-28|1996-10-31|DELIVER IN PERSON|RAIL|. blithely 4326|162728|2729|1|11|19697.92|0.01|0.01|N|O|1997-02-02|1996-12-10|1997-02-20|DELIVER IN PERSON|TRUCK|press reque 4326|166954|9471|2|27|54565.65|0.06|0.01|N|O|1996-11-29|1997-01-20|1996-12-23|COLLECT COD|AIR|inal packages. final asymptotes about t 4327|94813|9832|1|18|32540.58|0.08|0.00|N|F|1995-06-16|1995-04-20|1995-07-12|COLLECT COD|RAIL|y final excuses. ironic, special requests a 4327|105917|8428|2|40|76916.40|0.07|0.01|N|F|1995-05-26|1995-04-17|1995-06-18|NONE|AIR|quests. packages are after th 4327|144418|6933|3|11|16086.51|0.10|0.07|R|F|1995-04-24|1995-05-27|1995-05-24|TAKE BACK RETURN|FOB| ironic dolphins 4327|20003|5008|4|8|7384.00|0.04|0.08|N|F|1995-05-26|1995-05-28|1995-06-19|DELIVER IN PERSON|AIR|eodolites cajole; unusual Tiresias 4327|189038|9039|5|39|43954.17|0.01|0.00|N|O|1995-06-23|1995-04-18|1995-07-13|TAKE BACK RETURN|FOB|kages against the blit 4327|151058|8604|6|10|11090.50|0.00|0.06|A|F|1995-04-28|1995-06-11|1995-05-07|TAKE BACK RETURN|TRUCK|arefully sile 4352|105786|8297|1|18|32252.04|0.00|0.03|N|O|1998-02-27|1998-02-02|1998-03-01|DELIVER IN PERSON|RAIL|ding to th 4353|93627|1155|1|22|35653.64|0.05|0.05|N|O|1998-01-19|1998-01-23|1998-02-10|COLLECT COD|FOB|ent packages. accounts are slyly. 4354|14218|6720|1|30|33966.30|0.08|0.07|R|F|1995-01-27|1994-11-24|1995-02-25|TAKE BACK RETURN|REG AIR|around the ir 4354|152107|7138|2|23|26659.30|0.01|0.08|R|F|1994-11-20|1994-12-23|1994-11-27|TAKE BACK RETURN|AIR|kly along the ironic, ent 4354|50734|5745|3|2|3369.46|0.10|0.04|A|F|1995-01-09|1994-12-15|1995-01-24|TAKE BACK RETURN|REG AIR|s nag quickly 4354|85171|2696|4|36|41622.12|0.05|0.05|A|F|1994-11-20|1994-12-06|1994-12-06|DELIVER IN PERSON|AIR| wake slyly eve 4354|64313|1832|5|37|47260.47|0.06|0.02|R|F|1995-01-13|1994-12-29|1995-01-31|DELIVER IN PERSON|FOB|deas use blithely! special foxes print af 4354|107393|2414|6|36|50414.04|0.03|0.04|R|F|1994-12-03|1994-12-05|1995-01-02|TAKE BACK RETURN|TRUCK|efully special packages use fluffily 4354|138728|1242|7|18|31800.96|0.03|0.04|A|F|1994-12-07|1994-12-11|1994-12-11|TAKE BACK RETURN|SHIP|ross the furiously 4355|194040|6560|1|32|36289.28|0.10|0.02|N|O|1996-12-29|1997-02-08|1997-01-24|DELIVER IN PERSON|REG AIR|y silent deposits. b 4355|16565|9067|2|4|5926.24|0.05|0.02|N|O|1997-02-25|1997-01-29|1997-03-17|TAKE BACK RETURN|TRUCK|slyly blithely regular packag 4355|738|739|3|13|21303.49|0.07|0.05|N|O|1997-01-21|1996-12-22|1997-02-14|COLLECT COD|TRUCK| ought to mold. blithely pending ideas 4355|193886|6406|4|14|27718.32|0.04|0.02|N|O|1997-03-08|1997-01-22|1997-03-26|NONE|RAIL|he furiously ironic accounts. quickly iro 4355|30531|8041|5|50|73076.50|0.10|0.00|N|O|1996-11-25|1997-01-01|1996-12-06|DELIVER IN PERSON|REG AIR| regular accounts boost along the 4355|121860|4373|6|35|65865.10|0.00|0.08|N|O|1997-01-28|1997-01-28|1997-02-20|NONE|FOB|ess accounts affix ironic 4355|100815|3326|7|47|85343.07|0.09|0.02|N|O|1996-12-28|1996-12-29|1997-01-09|NONE|RAIL|e. realms integrate 4356|193770|3771|1|35|65231.95|0.00|0.04|R|F|1994-05-30|1994-06-14|1994-06-08|COLLECT COD|MAIL|arefully ironic 4357|83306|831|1|50|64465.00|0.04|0.07|N|O|1997-11-25|1997-12-03|1997-12-17|DELIVER IN PERSON|RAIL|s. final, e 4357|107896|7897|2|17|32366.13|0.10|0.07|N|O|1998-02-01|1997-12-08|1998-02-09|DELIVER IN PERSON|MAIL|e carefully furiou 4358|125030|55|1|47|49586.41|0.04|0.00|N|O|1997-10-15|1997-10-14|1997-11-04|DELIVER IN PERSON|SHIP|refully busy dep 4359|173759|8794|1|41|75142.75|0.03|0.07|A|F|1993-04-06|1993-05-06|1993-04-14|COLLECT COD|RAIL|s affix sly 4359|152794|7825|2|8|14774.32|0.03|0.08|R|F|1993-06-27|1993-05-16|1993-07-04|DELIVER IN PERSON|MAIL|packages affix. fluffily regular f 4359|192211|7250|3|32|41702.72|0.10|0.03|R|F|1993-06-18|1993-04-04|1993-07-18|COLLECT COD|MAIL|olites nag quietly caref 4359|77506|14|4|1|1483.50|0.05|0.03|R|F|1993-04-27|1993-05-09|1993-05-08|NONE|MAIL| fluffily ironic, bold pac 4359|32507|7514|5|22|31669.00|0.04|0.01|A|F|1993-03-28|1993-06-01|1993-04-13|NONE|REG AIR|accounts wake ironic deposits. ironic 4384|135485|5486|1|5|7602.40|0.09|0.01|A|F|1992-08-22|1992-08-24|1992-09-20|DELIVER IN PERSON|MAIL|instructions sleep. blithely express pa 4384|88695|6220|2|38|63980.22|0.07|0.06|A|F|1992-10-18|1992-09-24|1992-11-04|NONE|FOB|ly final requests. regu 4384|88921|6446|3|11|21009.12|0.05|0.04|R|F|1992-08-31|1992-10-04|1992-09-28|TAKE BACK RETURN|FOB|deposits promise carefully even, regular e 4385|110994|6017|1|38|76189.62|0.00|0.02|N|O|1996-11-22|1996-10-30|1996-12-21|DELIVER IN PERSON|TRUCK|inal frays. final, bold exc 4386|129394|6931|1|10|14233.90|0.05|0.07|N|O|1998-06-03|1998-04-16|1998-06-28|TAKE BACK RETURN|MAIL|gainst the quickly expre 4386|117055|9567|2|28|30017.40|0.03|0.06|N|O|1998-03-19|1998-05-01|1998-03-27|NONE|FOB|. quick packages play slyly 4386|139628|4655|3|4|6670.48|0.07|0.05|N|O|1998-04-07|1998-03-25|1998-04-19|COLLECT COD|FOB|ns wake carefully carefully iron 4386|120458|459|4|21|31047.45|0.09|0.00|N|O|1998-05-05|1998-03-19|1998-05-13|NONE|RAIL|e pending, sp 4386|129214|6751|5|39|48485.19|0.09|0.06|N|O|1998-03-05|1998-03-15|1998-03-16|NONE|RAIL|structions cajole quickly express 4386|89453|9454|6|18|25964.10|0.02|0.05|N|O|1998-04-12|1998-04-09|1998-05-12|TAKE BACK RETURN|SHIP| deposits use according to the pending, 4386|19821|2323|7|16|27853.12|0.07|0.02|N|O|1998-05-05|1998-03-17|1998-06-03|COLLECT COD|AIR|e furiously final pint 4387|121746|9283|1|3|5303.22|0.02|0.01|N|O|1996-01-17|1996-01-14|1996-01-28|COLLECT COD|AIR| boost slyly ironic instructions. furiou 4387|176820|4372|2|48|91047.36|0.06|0.05|N|O|1995-10-29|1995-12-11|1995-11-01|NONE|REG AIR|sleep slyly. blithely sl 4387|1622|4123|3|15|22854.30|0.00|0.03|N|O|1996-01-11|1996-01-14|1996-01-30|TAKE BACK RETURN|REG AIR|s hinder quietly across the pla 4387|46843|6844|4|9|16108.56|0.00|0.03|N|O|1996-01-04|1995-12-26|1996-01-12|DELIVER IN PERSON|REG AIR|c ideas. slyly regular packages sol 4387|81324|6341|5|3|3915.96|0.05|0.08|N|O|1995-11-17|1995-12-28|1995-11-25|COLLECT COD|SHIP| pinto beans 4387|5012|2513|6|40|36680.40|0.02|0.04|N|O|1995-11-29|1995-12-10|1995-12-20|NONE|REG AIR|deas according to the blithely regular fox 4388|64943|2462|1|30|57238.20|0.02|0.07|N|O|1996-06-07|1996-05-07|1996-06-22|DELIVER IN PERSON|FOB|s cajole fluffil 4388|83255|8272|2|28|34671.00|0.05|0.04|N|O|1996-05-08|1996-06-20|1996-05-12|TAKE BACK RETURN|RAIL|ove the ide 4388|51377|8893|3|13|17268.81|0.07|0.05|N|O|1996-06-28|1996-05-23|1996-07-04|DELIVER IN PERSON|REG AIR|ly even, expre 4389|156730|1761|1|20|35734.60|0.08|0.00|A|F|1994-06-06|1994-06-17|1994-06-17|DELIVER IN PERSON|SHIP|ng the carefully express d 4389|152822|368|2|13|24372.66|0.00|0.00|A|F|1994-08-18|1994-06-06|1994-08-20|NONE|RAIL|nal, regula 4389|78001|509|3|39|38181.00|0.04|0.07|A|F|1994-06-08|1994-06-04|1994-06-10|TAKE BACK RETURN|TRUCK| unusual, final excuses cajole carefully 4389|159554|7100|4|5|8067.75|0.09|0.00|A|F|1994-09-03|1994-06-23|1994-09-16|NONE|FOB| ironic request 4389|10461|2963|5|22|30172.12|0.08|0.00|R|F|1994-07-05|1994-06-12|1994-07-12|NONE|TRUCK|lly silent de 4389|1428|1429|6|22|29247.24|0.01|0.04|R|F|1994-06-07|1994-06-29|1994-06-19|COLLECT COD|TRUCK|at the final excuses hinder carefully a 4389|184028|4029|7|4|4448.08|0.09|0.08|R|F|1994-06-14|1994-06-30|1994-07-06|NONE|REG AIR| blithely even d 4390|151140|3656|1|35|41689.90|0.07|0.04|R|F|1995-05-30|1995-07-02|1995-06-15|DELIVER IN PERSON|TRUCK|ongside of the slyly regular ideas 4390|195068|7588|2|28|32565.68|0.03|0.00|N|O|1995-09-07|1995-06-22|1995-10-05|COLLECT COD|SHIP|ld braids haggle atop the for 4390|100762|8293|3|42|74035.92|0.05|0.08|A|F|1995-06-12|1995-07-16|1995-06-17|NONE|AIR|arefully even accoun 4390|97112|4640|4|32|35491.52|0.07|0.08|N|O|1995-09-15|1995-08-12|1995-10-05|TAKE BACK RETURN|TRUCK|ctions across 4391|160992|3509|1|1|2052.99|0.09|0.00|R|F|1992-06-18|1992-04-27|1992-06-20|COLLECT COD|TRUCK|ong the silent deposits 4391|186943|4498|2|45|91347.30|0.07|0.04|R|F|1992-04-01|1992-05-01|1992-04-13|TAKE BACK RETURN|TRUCK|ep quickly after 4416|93919|8938|1|37|70777.67|0.08|0.03|A|F|1992-10-23|1992-08-23|1992-11-16|COLLECT COD|RAIL|fluffily ironic 4416|88976|8977|2|3|5894.91|0.06|0.03|R|F|1992-10-22|1992-08-06|1992-11-13|DELIVER IN PERSON|SHIP| requests sleep along the 4416|8657|6158|3|45|70454.25|0.09|0.03|A|F|1992-10-16|1992-09-09|1992-10-28|COLLECT COD|AIR|the final pinto beans. special frets 4417|74480|6988|1|28|40725.44|0.08|0.02|N|O|1998-09-04|1998-10-04|1998-09-19|TAKE BACK RETURN|REG AIR|ies across the furious 4417|180729|5766|2|1|1809.72|0.06|0.08|N|O|1998-10-23|1998-08-22|1998-10-24|NONE|REG AIR|press deposits promise stealthily amo 4417|97701|5229|3|35|59454.50|0.06|0.04|N|O|1998-08-08|1998-09-23|1998-09-02|DELIVER IN PERSON|FOB|slyly regular, silent courts. even packag 4418|34773|7277|1|32|54648.64|0.02|0.06|A|F|1993-05-28|1993-06-02|1993-05-30|TAKE BACK RETURN|RAIL|ly. bold pinto b 4418|21913|9420|2|14|25688.74|0.03|0.04|A|F|1993-05-20|1993-06-18|1993-06-05|TAKE BACK RETURN|SHIP| blithely regular requests. blith 4418|78027|5549|3|3|3015.06|0.00|0.02|R|F|1993-04-08|1993-06-04|1993-05-02|NONE|SHIP|luffily across the unusual ideas. reque 4419|107398|7399|1|45|63242.55|0.01|0.05|N|O|1996-07-20|1996-09-07|1996-08-18|DELIVER IN PERSON|TRUCK|s doze sometimes fluffily regular a 4419|31638|4142|2|42|65924.46|0.00|0.03|N|O|1996-09-18|1996-07-25|1996-09-21|COLLECT COD|RAIL|sts. furious 4419|131738|6765|3|6|10618.38|0.02|0.08|N|O|1996-06-25|1996-09-04|1996-07-20|DELIVER IN PERSON|AIR|ts wake slyly final dugou 4420|7317|4818|1|7|8570.17|0.07|0.03|R|F|1994-08-30|1994-09-03|1994-09-25|NONE|FOB| regular instructions sleep around 4421|97523|5051|1|37|56259.24|0.09|0.08|N|O|1997-07-22|1997-06-27|1997-07-25|DELIVER IN PERSON|SHIP|l accounts. ironic request 4421|55380|391|2|46|61427.48|0.04|0.04|N|O|1997-04-21|1997-05-13|1997-05-15|DELIVER IN PERSON|FOB|reful packages. bold, 4421|166181|8698|3|46|57370.28|0.00|0.06|N|O|1997-05-25|1997-05-21|1997-06-23|COLLECT COD|TRUCK|g dependenci 4421|190694|5733|4|32|57110.08|0.06|0.04|N|O|1997-07-09|1997-06-03|1997-07-25|NONE|SHIP|ar ideas eat among the furiousl 4421|189912|9913|5|32|64061.12|0.06|0.04|N|O|1997-07-28|1997-06-14|1997-08-13|NONE|REG AIR|uickly final pinto beans impress. bold 4421|46583|4096|6|44|67301.52|0.09|0.06|N|O|1997-06-17|1997-06-20|1997-06-29|NONE|TRUCK|le carefully. bl 4421|115306|329|7|18|23783.40|0.01|0.00|N|O|1997-06-07|1997-05-13|1997-06-10|DELIVER IN PERSON|FOB|. regular, s 4422|134519|2059|1|5|7767.55|0.09|0.07|N|O|1995-07-17|1995-08-13|1995-07-25|NONE|SHIP|e furiously about t 4422|47451|9956|2|41|57336.45|0.08|0.05|N|F|1995-06-12|1995-07-09|1995-06-20|COLLECT COD|TRUCK| theodolites shal 4422|102348|9879|3|39|52663.26|0.00|0.05|N|O|1995-09-02|1995-06-24|1995-09-14|NONE|TRUCK|en hockey players engage 4422|152192|2193|4|4|4976.76|0.02|0.05|N|O|1995-09-18|1995-08-12|1995-10-18|COLLECT COD|FOB|cies along the bo 4422|79692|4707|5|20|33433.80|0.07|0.05|N|O|1995-08-17|1995-07-16|1995-09-13|DELIVER IN PERSON|RAIL|ructions wake slyly al 4423|149239|6782|1|3|3864.69|0.03|0.00|A|F|1995-03-22|1995-04-06|1995-04-19|NONE|TRUCK| final theodolites nag after the bli 4423|59576|4587|2|2|3071.14|0.07|0.04|A|F|1995-03-04|1995-04-04|1995-03-08|TAKE BACK RETURN|REG AIR|old sheaves sleep 4448|51574|6585|1|24|36613.68|0.10|0.07|N|O|1998-09-09|1998-07-06|1998-09-27|DELIVER IN PERSON|SHIP|nal packages along the ironic instructi 4448|188627|1146|2|13|22303.06|0.00|0.01|N|O|1998-07-26|1998-07-03|1998-08-14|COLLECT COD|MAIL|fluffily express accounts integrate furiou 4448|40980|5989|3|35|67234.30|0.10|0.06|N|O|1998-09-18|1998-07-27|1998-10-08|NONE|REG AIR|aggle carefully alongside of the q 4448|140138|139|4|3|3534.39|0.01|0.01|N|O|1998-07-20|1998-07-10|1998-08-07|DELIVER IN PERSON|TRUCK|ronic theod 4448|90140|141|5|41|46335.74|0.00|0.08|N|O|1998-07-30|1998-08-09|1998-08-03|NONE|AIR|pon the permanently even excuses nag 4448|171401|1402|6|12|17668.80|0.06|0.03|N|O|1998-08-21|1998-06-30|1998-09-09|COLLECT COD|RAIL|sits about the ironic, bu 4449|31484|6491|1|42|59450.16|0.10|0.07|N|O|1998-03-22|1998-05-09|1998-04-03|NONE|FOB| packages. blithely final 4449|140638|3153|2|10|16786.30|0.02|0.03|N|O|1998-05-09|1998-05-04|1998-05-15|NONE|SHIP|ccounts alongside of the platelets integr 4450|173191|3192|1|44|55624.36|0.10|0.00|N|O|1997-10-12|1997-10-13|1997-10-29|DELIVER IN PERSON|RAIL| the slyly eve 4450|14714|4715|2|9|14658.39|0.03|0.03|N|O|1997-08-13|1997-08-16|1997-08-15|NONE|FOB|gular requests cajole carefully. regular c 4450|95500|8010|3|45|67297.50|0.08|0.01|N|O|1997-09-01|1997-10-06|1997-09-19|NONE|TRUCK|express ideas are furiously regular 4450|61930|6943|4|13|24595.09|0.00|0.00|N|O|1997-08-26|1997-09-18|1997-09-20|COLLECT COD|MAIL| brave foxes. slyly unusual 4450|55040|5041|5|6|5970.24|0.09|0.01|N|O|1997-09-02|1997-09-30|1997-09-09|NONE|FOB|eposits. foxes cajole unusual fox 4451|163032|3033|1|40|43801.20|0.03|0.03|A|F|1994-11-18|1994-12-25|1994-11-26|DELIVER IN PERSON|RAIL|y. slyly special deposits are sly 4451|62282|2283|2|34|42305.52|0.10|0.02|A|F|1994-11-30|1994-12-04|1994-12-13|COLLECT COD|SHIP| regular ideas. 4451|158586|8587|3|19|31247.02|0.05|0.06|R|F|1994-10-09|1994-11-26|1994-10-23|COLLECT COD|FOB|ly after the fluffi 4452|113429|5941|1|21|30290.82|0.07|0.03|R|F|1994-10-06|1994-08-23|1994-10-15|COLLECT COD|TRUCK|multipliers x-ray carefully in place of 4452|149|7650|2|47|49309.58|0.01|0.06|A|F|1994-10-08|1994-08-09|1994-10-09|TAKE BACK RETURN|TRUCK|ts. slyly regular cour 4453|146971|2000|1|41|82736.77|0.00|0.08|N|O|1997-07-17|1997-05-15|1997-07-31|NONE|REG AIR|anent theodolites are slyly except t 4453|132711|2712|2|16|27899.36|0.03|0.00|N|O|1997-07-22|1997-05-05|1997-08-03|COLLECT COD|FOB|ar excuses nag quickly even accounts. b 4453|61826|9345|3|48|85815.36|0.02|0.07|N|O|1997-05-29|1997-06-24|1997-06-03|NONE|SHIP|eep. fluffily express accounts at the furi 4453|101706|4217|4|26|44400.20|0.06|0.07|N|O|1997-05-07|1997-06-07|1997-05-22|NONE|TRUCK|express packages are 4454|150851|3367|1|20|38037.00|0.10|0.08|R|F|1994-05-06|1994-03-17|1994-05-20|COLLECT COD|SHIP|lar theodolites. even instructio 4454|151032|3548|2|22|23826.66|0.06|0.02|A|F|1994-02-06|1994-04-11|1994-03-06|DELIVER IN PERSON|RAIL|ully. carefully final accounts accordi 4454|191552|1553|3|45|73959.75|0.07|0.04|A|F|1994-03-29|1994-03-26|1994-04-04|TAKE BACK RETURN|RAIL|ests promise. packages print fur 4454|1324|1325|4|1|1225.32|0.09|0.05|A|F|1994-02-05|1994-04-19|1994-02-12|COLLECT COD|RAIL|equests run. 4454|51060|8576|5|48|48530.88|0.00|0.07|R|F|1994-04-23|1994-04-03|1994-04-26|COLLECT COD|FOB|to beans wake across th 4454|159632|2148|6|20|33832.60|0.10|0.03|A|F|1994-04-08|1994-03-06|1994-04-26|DELIVER IN PERSON|TRUCK|quickly regular requests. furiously 4455|69731|7250|1|20|34014.60|0.01|0.05|A|F|1994-01-31|1993-11-21|1994-03-02|DELIVER IN PERSON|MAIL| express packages. packages boost quickly 4455|152587|2588|2|47|77060.26|0.09|0.01|R|F|1994-01-01|1993-12-25|1994-01-05|COLLECT COD|FOB| requests. even, even accou 4455|122723|7748|3|34|59354.48|0.00|0.06|A|F|1993-10-24|1993-11-27|1993-11-04|TAKE BACK RETURN|AIR| slyly ironic requests. quickly even d 4480|107888|5419|1|30|56876.40|0.08|0.03|R|F|1994-07-29|1994-06-22|1994-08-01|NONE|FOB|ven braids us 4481|23918|6421|1|50|92095.50|0.02|0.06|N|O|1996-07-22|1996-05-13|1996-08-14|DELIVER IN PERSON|RAIL|ar packages. regula 4481|189040|1559|2|27|30484.08|0.02|0.03|N|O|1996-04-06|1996-05-17|1996-04-12|TAKE BACK RETURN|AIR|ackages haggle even, 4482|70869|870|1|32|58875.52|0.06|0.03|A|F|1995-05-16|1995-07-22|1995-06-07|NONE|RAIL| quickly pendin 4482|95182|201|2|32|37669.76|0.01|0.06|N|O|1995-08-16|1995-06-26|1995-09-10|DELIVER IN PERSON|AIR|eans wake according 4483|5133|5134|1|32|33220.16|0.07|0.07|R|F|1992-04-05|1992-05-25|1992-04-08|DELIVER IN PERSON|MAIL|ests haggle. slyl 4483|61832|4339|2|50|89691.50|0.01|0.06|A|F|1992-06-19|1992-05-12|1992-07-08|DELIVER IN PERSON|TRUCK|ag blithely even 4483|8334|3335|3|50|62116.50|0.00|0.04|R|F|1992-06-10|1992-04-18|1992-06-17|DELIVER IN PERSON|MAIL|ackages. furiously ironi 4484|94779|2307|1|4|7095.08|0.06|0.03|N|O|1997-04-09|1997-02-11|1997-04-12|TAKE BACK RETURN|TRUCK|packages de 4484|136812|6813|2|39|72103.59|0.05|0.02|N|O|1997-04-01|1997-01-26|1997-04-21|NONE|RAIL|onic accounts wake blithel 4484|190000|1|3|38|41420.00|0.06|0.07|N|O|1997-03-07|1997-01-31|1997-04-01|COLLECT COD|REG AIR|. even requests un 4484|121153|8690|4|41|48140.15|0.06|0.03|N|O|1997-01-25|1997-02-15|1997-01-29|TAKE BACK RETURN|REG AIR|ress accounts. ironic deposits unwind fur 4484|2585|2586|5|42|62478.36|0.03|0.07|N|O|1997-03-25|1997-02-21|1997-04-05|DELIVER IN PERSON|REG AIR|ding, pending requests wake. fluffily 4484|35516|5517|6|29|42093.79|0.09|0.06|N|O|1996-12-27|1997-03-10|1997-01-13|NONE|FOB| wake blithely ironic 4484|102267|7288|7|50|63463.00|0.07|0.01|N|O|1997-03-17|1997-03-16|1997-03-21|COLLECT COD|FOB|the ironic, final theodo 4485|190656|8214|1|1|1746.65|0.03|0.05|R|F|1994-12-04|1995-02-07|1994-12-09|NONE|AIR|play according to the ironic, ironic 4485|140072|7615|2|46|51155.22|0.04|0.06|R|F|1995-03-09|1994-12-14|1995-03-23|DELIVER IN PERSON|AIR|. ironic foxes haggle. regular war 4485|174651|4652|3|43|74202.95|0.01|0.05|R|F|1995-01-17|1995-02-11|1995-02-07|DELIVER IN PERSON|TRUCK|al accounts according to the slyly r 4485|143072|3073|4|43|47948.01|0.08|0.06|R|F|1995-01-28|1995-01-26|1995-02-07|DELIVER IN PERSON|AIR|. blithely 4485|5035|5036|5|47|44181.41|0.08|0.04|R|F|1995-03-11|1995-01-11|1995-03-21|TAKE BACK RETURN|RAIL|luffily pending acc 4486|134209|6723|1|46|57187.20|0.08|0.00|N|O|1998-05-02|1998-04-05|1998-05-08|COLLECT COD|MAIL|ackages. specia 4486|48748|3757|2|19|32238.06|0.10|0.01|N|O|1998-06-07|1998-05-28|1998-07-02|NONE|MAIL|pending foxes after 4486|95192|5193|3|47|55797.93|0.02|0.07|N|O|1998-04-09|1998-05-24|1998-05-07|DELIVER IN PERSON|MAIL|ts around the quiet packages ar 4486|90018|5037|4|28|28224.28|0.07|0.02|N|O|1998-04-21|1998-04-19|1998-04-26|TAKE BACK RETURN|AIR|to the furious, regular foxes play abov 4487|137105|9619|1|37|42257.70|0.03|0.07|R|F|1993-02-28|1993-04-18|1993-03-17|TAKE BACK RETURN|MAIL|bove the fu 4487|112846|7869|2|49|91083.16|0.10|0.00|R|F|1993-06-13|1993-05-08|1993-07-10|COLLECT COD|FOB|sual packages should ha 4487|189582|2101|3|1|1671.58|0.02|0.07|A|F|1993-05-11|1993-05-23|1993-05-17|TAKE BACK RETURN|FOB|ithely final asym 4487|92846|2847|4|25|45971.00|0.07|0.03|A|F|1993-03-09|1993-04-27|1993-03-30|COLLECT COD|RAIL|g the final instructions. slyly c 4512|161210|3727|1|30|38136.30|0.07|0.07|N|O|1996-01-28|1995-12-22|1996-02-22|TAKE BACK RETURN|TRUCK|ly unusual package 4512|40933|5942|2|24|44974.32|0.04|0.06|N|O|1995-12-16|1996-01-16|1995-12-25|NONE|SHIP|ly regular pinto beans. carefully bold depo 4512|144996|25|3|21|42860.79|0.00|0.00|N|O|1995-10-31|1995-12-30|1995-11-15|NONE|REG AIR|lly unusual pinto b 4512|140248|249|4|32|41223.68|0.10|0.01|N|O|1995-11-25|1995-12-28|1995-12-06|NONE|FOB|counts are against the quickly regular 4512|132998|8025|5|43|87332.57|0.06|0.00|N|O|1995-12-20|1995-11-28|1996-01-14|NONE|AIR|are carefully. theodolites wake 4513|169018|9019|1|29|31523.29|0.03|0.01|N|O|1996-05-18|1996-05-23|1996-06-08|NONE|REG AIR|cajole. regular packages boost. s 4513|69944|2451|2|39|74643.66|0.01|0.04|N|O|1996-06-25|1996-05-14|1996-07-24|NONE|MAIL|slyly furiously unusual deposits. blit 4513|137348|4888|3|34|47101.56|0.00|0.03|N|O|1996-03-27|1996-06-12|1996-04-06|DELIVER IN PERSON|SHIP|sits. quickly even instructions 4513|191361|8919|4|13|18880.68|0.08|0.08|N|O|1996-04-12|1996-05-19|1996-04-25|DELIVER IN PERSON|AIR|l, final excuses detect furi 4514|163673|1222|1|27|46890.09|0.06|0.06|R|F|1994-07-01|1994-07-13|1994-07-26|COLLECT COD|AIR| even, silent foxes be 4514|45129|7634|2|15|16111.80|0.10|0.04|R|F|1994-08-24|1994-07-11|1994-09-14|DELIVER IN PERSON|RAIL|! unusual, special deposits afte 4514|77425|9933|3|10|14024.20|0.09|0.05|A|F|1994-06-19|1994-06-25|1994-07-01|COLLECT COD|SHIP|ake furiously. carefully regular requests 4514|80738|3247|4|9|15468.57|0.10|0.03|A|F|1994-08-04|1994-07-01|1994-09-01|DELIVER IN PERSON|REG AIR|wly. quick 4514|148628|6171|5|12|20119.44|0.02|0.03|R|F|1994-08-20|1994-06-09|1994-09-15|TAKE BACK RETURN|FOB| carefully ironic foxes nag caref 4514|188706|8707|6|38|68198.60|0.03|0.05|A|F|1994-07-28|1994-07-06|1994-08-25|NONE|AIR|ending excuses. sl 4514|176429|6430|7|27|40646.34|0.04|0.06|A|F|1994-06-24|1994-07-14|1994-06-30|TAKE BACK RETURN|TRUCK|. slyly sile 4515|38003|8004|1|15|14115.00|0.06|0.01|R|F|1992-05-26|1992-05-25|1992-06-03|NONE|SHIP|posits wake 4515|102050|9581|2|50|52602.50|0.06|0.03|A|F|1992-03-28|1992-05-16|1992-04-20|NONE|AIR|ding instructions again 4515|153202|748|3|27|33890.40|0.09|0.01|A|F|1992-06-06|1992-06-08|1992-06-07|DELIVER IN PERSON|REG AIR| against the even re 4515|53814|3815|4|32|56569.92|0.06|0.03|R|F|1992-04-07|1992-05-11|1992-04-09|COLLECT COD|MAIL|carefully express depo 4515|44659|9668|5|22|35280.30|0.09|0.07|A|F|1992-07-16|1992-05-07|1992-07-23|NONE|SHIP|le quickly above the even, bold ideas. 4515|179908|7460|6|23|45721.70|0.04|0.00|R|F|1992-05-23|1992-06-15|1992-06-20|TAKE BACK RETURN|FOB|ns. bold r 4516|169570|2087|1|34|55745.38|0.05|0.04|A|F|1994-05-16|1994-06-23|1994-06-12|NONE|SHIP|even pinto beans wake qui 4517|42540|2541|1|50|74127.00|0.01|0.02|N|O|1998-06-08|1998-04-18|1998-06-20|DELIVER IN PERSON|MAIL|refully pending acco 4518|143741|8770|1|9|16062.66|0.09|0.04|N|O|1997-06-26|1997-07-07|1997-07-10|NONE|RAIL| pending deposits. slyly re 4518|44083|4084|2|19|19514.52|0.10|0.05|N|O|1997-08-09|1997-06-06|1997-08-27|COLLECT COD|RAIL|ter the slyly bo 4519|54680|7186|1|30|49040.40|0.09|0.07|R|F|1993-04-11|1993-06-05|1993-04-22|DELIVER IN PERSON|REG AIR|totes. slyly bold somas after the 4519|190062|2582|2|37|42626.22|0.06|0.08|R|F|1993-07-22|1993-06-16|1993-08-19|COLLECT COD|AIR|ly slyly furious depth 4544|130682|3196|1|40|68507.20|0.07|0.01|N|O|1997-08-15|1997-10-16|1997-08-20|DELIVER IN PERSON|RAIL| detect slyly. evenly pending instru 4544|171088|3606|2|19|22022.52|0.08|0.01|N|O|1997-08-14|1997-09-08|1997-08-25|NONE|SHIP|regular ideas are furiously about 4544|70629|8151|3|20|31992.40|0.02|0.07|N|O|1997-10-12|1997-10-11|1997-10-13|COLLECT COD|REG AIR| waters about the 4544|50571|5582|4|39|59341.23|0.07|0.05|N|O|1997-08-20|1997-09-07|1997-08-27|COLLECT COD|REG AIR|ular packages. s 4544|132280|7307|5|31|40680.68|0.09|0.03|N|O|1997-08-09|1997-09-29|1997-08-17|COLLECT COD|TRUCK|dolites detect quickly reg 4544|26100|6101|6|8|8208.80|0.10|0.03|N|O|1997-10-13|1997-10-06|1997-10-25|COLLECT COD|AIR|olites. fi 4545|172885|437|1|38|74399.44|0.06|0.06|R|F|1993-01-27|1993-03-01|1993-02-04|NONE|TRUCK|nts serve according to th 4545|62456|2457|2|27|38298.15|0.01|0.06|R|F|1993-02-07|1993-02-18|1993-02-18|NONE|FOB|ously bold asymptotes! blithely pen 4545|86195|3720|3|9|10630.71|0.10|0.06|R|F|1993-03-20|1993-02-23|1993-04-11|TAKE BACK RETURN|AIR|xpress accounts 4545|63981|1500|4|2|3889.96|0.10|0.00|R|F|1993-04-16|1993-04-17|1993-05-03|NONE|REG AIR|ages use. slyly even i 4545|116732|9244|5|27|47215.71|0.08|0.05|A|F|1993-03-18|1993-02-22|1993-03-23|NONE|RAIL|ccounts haggle carefully. deposits 4545|108682|1193|6|8|13525.44|0.03|0.02|A|F|1993-05-01|1993-03-12|1993-05-15|NONE|FOB| boost slyly. slyly 4545|8362|863|7|36|45732.96|0.10|0.04|R|F|1993-01-28|1993-03-30|1993-02-04|DELIVER IN PERSON|SHIP|sublate slyly. furiously ironic accounts b 4546|132467|7494|1|10|14994.60|0.09|0.02|N|O|1995-09-23|1995-10-10|1995-10-23|COLLECT COD|TRUCK|osits alongside of the 4546|170327|5362|2|15|20959.80|0.04|0.07|N|O|1995-07-31|1995-10-17|1995-08-06|NONE|REG AIR|ught to cajole furiously. qu 4546|76977|6978|3|4|7815.88|0.06|0.08|N|O|1995-08-14|1995-10-07|1995-08-16|COLLECT COD|MAIL|kly pending dependencies along the furio 4546|148745|1260|4|10|17937.40|0.08|0.02|N|O|1995-09-02|1995-09-16|1995-09-10|DELIVER IN PERSON|FOB|above the enticingly ironic dependencies 4547|187798|317|1|15|28286.85|0.10|0.04|A|F|1993-12-08|1993-11-15|1993-12-22|NONE|REG AIR|ets haggle. regular dinos affix fu 4547|115181|7693|2|7|8373.26|0.10|0.02|A|F|1993-09-04|1993-09-29|1993-09-20|COLLECT COD|RAIL|slyly express a 4547|44955|7460|3|15|28499.25|0.00|0.00|R|F|1993-11-18|1993-10-06|1993-12-13|NONE|TRUCK|e carefully across the unus 4547|147087|4630|4|15|17011.20|0.05|0.08|R|F|1993-11-29|1993-10-12|1993-12-29|COLLECT COD|REG AIR|ironic gifts integrate 4548|13272|5774|1|21|24890.67|0.10|0.05|N|O|1996-07-11|1996-09-04|1996-07-30|COLLECT COD|REG AIR|pecial theodoli 4548|46001|1010|2|17|16099.00|0.00|0.08|N|O|1996-07-23|1996-09-21|1996-07-26|DELIVER IN PERSON|REG AIR|y ironic requests above the fluffily d 4548|122710|7735|3|47|81437.37|0.05|0.04|N|O|1996-07-24|1996-09-12|1996-08-08|NONE|MAIL|ts. excuses use slyly spec 4548|176323|1358|4|22|30785.04|0.07|0.01|N|O|1996-07-06|1996-08-23|1996-07-15|DELIVER IN PERSON|RAIL|s. furiously ironic theodolites c 4548|44556|2069|5|36|54019.80|0.04|0.06|N|O|1996-08-19|1996-09-12|1996-09-08|COLLECT COD|FOB|tions integrat 4549|158023|5569|1|44|47564.88|0.08|0.00|N|O|1998-03-13|1998-04-15|1998-03-27|TAKE BACK RETURN|TRUCK|ding to the regular, silent requests 4549|88030|539|2|1|1018.03|0.05|0.08|N|O|1998-05-04|1998-04-11|1998-05-14|TAKE BACK RETURN|AIR| requests wake. furiously even 4550|149038|1553|1|9|9783.27|0.05|0.06|R|F|1995-04-19|1995-02-07|1995-04-24|COLLECT COD|SHIP|l dependencies boost slyly after th 4550|65547|8054|2|19|28738.26|0.06|0.04|A|F|1995-01-01|1995-02-13|1995-01-20|NONE|AIR|quests. express 4551|10447|7951|1|6|8144.64|0.08|0.08|N|O|1996-05-18|1996-04-23|1996-06-13|DELIVER IN PERSON|TRUCK|fily silent fo 4551|178094|3129|2|26|30474.34|0.02|0.04|N|O|1996-04-14|1996-04-26|1996-04-17|TAKE BACK RETURN|RAIL|le. carefully dogged accounts use furiousl 4551|21706|6711|3|22|35809.40|0.08|0.01|N|O|1996-05-12|1996-03-17|1996-05-29|TAKE BACK RETURN|REG AIR|ly ironic reques 4551|197665|185|4|27|47591.82|0.00|0.01|N|O|1996-04-28|1996-03-22|1996-05-22|TAKE BACK RETURN|RAIL|y along the slyly even 4576|89514|9515|1|5|7517.55|0.09|0.03|N|O|1996-08-23|1996-11-08|1996-09-20|TAKE BACK RETURN|AIR|ly express, special asymptote 4576|57907|7908|2|43|80190.70|0.08|0.06|N|O|1996-10-24|1996-09-23|1996-11-10|NONE|SHIP|ly final deposits. never 4576|41350|8863|3|14|18078.90|0.09|0.01|N|O|1996-09-12|1996-09-30|1996-09-24|COLLECT COD|MAIL|detect slyly. 4577|184089|6608|1|43|50442.44|0.01|0.03|N|O|1998-06-16|1998-07-09|1998-06-17|TAKE BACK RETURN|AIR|packages. 4577|176990|2025|2|43|88880.57|0.05|0.03|N|O|1998-08-24|1998-06-02|1998-09-14|TAKE BACK RETURN|RAIL|ly accounts. carefully 4577|68494|3507|3|12|17549.88|0.07|0.05|N|O|1998-07-29|1998-06-17|1998-08-04|DELIVER IN PERSON|TRUCK|equests alongsi 4578|73111|633|1|10|10841.10|0.09|0.06|R|F|1993-01-01|1992-11-19|1993-01-28|TAKE BACK RETURN|REG AIR|uests. blithely unus 4578|168393|8394|2|42|61378.38|0.06|0.00|R|F|1993-01-05|1992-11-06|1993-01-13|DELIVER IN PERSON|FOB|s are caref 4578|178494|3529|3|15|23587.35|0.01|0.01|R|F|1992-10-23|1992-11-22|1992-11-09|DELIVER IN PERSON|REG AIR|gular theodo 4578|138667|3694|4|7|11939.62|0.09|0.08|A|F|1992-12-07|1992-11-27|1993-01-05|TAKE BACK RETURN|SHIP|odolites. carefully unusual ideas accor 4578|162341|4858|5|20|28066.80|0.04|0.02|A|F|1993-01-11|1992-11-09|1993-01-23|TAKE BACK RETURN|RAIL|iously pending theodolites-- 4579|174085|9120|1|14|16227.12|0.02|0.02|N|O|1996-02-01|1996-01-08|1996-02-08|TAKE BACK RETURN|MAIL|nding theodolites. fluffil 4579|41886|1887|2|28|51180.64|0.02|0.05|N|O|1996-01-22|1996-02-13|1996-02-03|DELIVER IN PERSON|RAIL|slyly across the 4579|177457|7458|3|34|52171.30|0.05|0.02|N|O|1996-02-26|1996-02-22|1996-03-16|COLLECT COD|MAIL|hely. carefully blithe dependen 4579|119554|9555|4|8|12588.40|0.05|0.06|N|O|1995-12-16|1996-01-15|1995-12-18|TAKE BACK RETURN|AIR|posits. carefully perman 4580|91060|6079|1|22|23123.32|0.01|0.05|A|F|1994-01-16|1994-01-26|1994-02-05|COLLECT COD|AIR|nticingly final packag 4580|31880|6887|2|10|18118.80|0.05|0.04|R|F|1993-12-20|1993-12-30|1994-01-17|COLLECT COD|RAIL|gular, pending deposits. fina 4580|963|8464|3|41|76422.36|0.00|0.07|R|F|1993-12-13|1994-01-31|1994-01-06|NONE|SHIP|requests. quickly silent asymptotes sle 4580|177771|289|4|5|9243.85|0.07|0.00|A|F|1994-01-28|1993-12-17|1994-02-22|NONE|TRUCK|o beans. f 4580|188804|1323|5|39|73819.20|0.03|0.02|R|F|1993-12-28|1993-12-26|1994-01-23|NONE|RAIL|. fluffily final dolphins use furiously al 4581|164947|7464|1|37|74441.78|0.01|0.04|A|F|1992-10-17|1992-11-05|1992-11-04|DELIVER IN PERSON|MAIL|e the blithely bold pearls ha 4581|49672|4681|2|7|11351.69|0.01|0.02|A|F|1992-10-09|1992-10-20|1992-10-21|TAKE BACK RETURN|MAIL|express accounts d 4581|20508|5513|3|46|65711.00|0.04|0.04|A|F|1992-09-09|1992-11-27|1992-09-26|NONE|REG AIR|nag toward the carefully final accounts. 4582|191457|6496|1|17|26323.65|0.09|0.08|N|O|1996-08-17|1996-08-26|1996-08-20|COLLECT COD|REG AIR|ng packages. depo 4583|140441|442|1|17|25184.48|0.01|0.05|A|F|1994-11-08|1994-11-03|1994-11-29|COLLECT COD|MAIL|romise. reques 4583|186542|6543|2|43|70027.22|0.04|0.04|A|F|1994-10-30|1994-12-17|1994-11-16|COLLECT COD|RAIL|fully after the speci 4583|195513|3071|3|28|45038.28|0.00|0.07|A|F|1994-10-29|1994-11-21|1994-11-28|NONE|SHIP|to beans haggle sly 4583|172130|2131|4|27|32457.51|0.08|0.03|R|F|1995-01-11|1994-12-24|1995-02-10|DELIVER IN PERSON|TRUCK| detect silent requests. furiously speci 4583|183800|8837|5|36|67816.80|0.09|0.06|A|F|1995-01-06|1994-11-25|1995-01-29|DELIVER IN PERSON|RAIL|ar requests haggle after the furiously 4583|121705|4218|6|14|24173.80|0.09|0.01|R|F|1994-11-17|1994-11-08|1994-11-21|DELIVER IN PERSON|AIR|detect. doggedly regular pi 4583|86867|1884|7|32|59323.52|0.04|0.00|A|F|1995-01-13|1994-10-29|1995-02-08|TAKE BACK RETURN|RAIL|across the pinto beans-- quickly 4608|172217|9769|1|30|38676.30|0.08|0.05|R|F|1994-10-08|1994-07-18|1994-10-25|DELIVER IN PERSON|SHIP|s cajole. slyly 4608|46212|6213|2|50|57910.50|0.06|0.01|A|F|1994-07-25|1994-09-01|1994-08-10|NONE|FOB| theodolites 4608|78184|692|3|50|58109.00|0.03|0.01|A|F|1994-08-04|1994-09-10|1994-08-13|COLLECT COD|TRUCK| wake closely. even decoys haggle above 4608|30557|558|4|36|53551.80|0.05|0.06|R|F|1994-10-04|1994-08-02|1994-10-21|COLLECT COD|FOB|ages wake quickly slyly iron 4609|46409|3922|1|28|37951.20|0.10|0.05|N|O|1997-02-02|1997-02-17|1997-03-02|DELIVER IN PERSON|REG AIR|ously. quickly final requests cajole fl 4609|184318|6837|2|3|4206.93|0.09|0.03|N|O|1996-12-28|1997-02-06|1997-01-20|NONE|FOB|nstructions. furious instructions 4609|22399|2400|3|46|60783.94|0.05|0.05|N|O|1997-02-11|1997-01-16|1997-03-07|NONE|FOB|r foxes. fluffily ironic ideas ha 4610|86177|1194|1|21|24426.57|0.07|0.07|R|F|1993-08-10|1993-08-05|1993-08-27|NONE|REG AIR|ly special theodolites. even, 4610|174842|7360|2|14|26835.76|0.00|0.07|R|F|1993-07-28|1993-07-25|1993-07-31|TAKE BACK RETURN|SHIP| ironic frays. dependencies detect blithel 4610|158379|5925|3|44|63244.28|0.05|0.03|A|F|1993-08-05|1993-07-20|1993-08-19|COLLECT COD|TRUCK| final theodolites 4610|74538|2060|4|26|39325.78|0.06|0.03|R|F|1993-07-01|1993-07-19|1993-07-19|NONE|MAIL| to the fluffily ironic requests h 4610|146158|6159|5|29|34920.35|0.08|0.04|R|F|1993-08-09|1993-07-27|1993-08-16|DELIVER IN PERSON|AIR| foxes. special, express package 4611|51267|6278|1|47|57258.22|0.09|0.06|A|F|1993-03-05|1993-03-01|1993-03-17|COLLECT COD|TRUCK|iously. furiously regular 4611|34872|4873|2|31|56012.97|0.04|0.02|A|F|1993-01-28|1993-02-14|1993-01-29|TAKE BACK RETURN|AIR| final pinto beans. permanent, sp 4611|81649|1650|3|50|81532.00|0.08|0.01|R|F|1993-01-22|1993-03-30|1993-02-16|TAKE BACK RETURN|AIR|l platelets. 4611|70451|7973|4|48|68229.60|0.02|0.08|R|F|1993-02-28|1993-02-12|1993-03-01|COLLECT COD|AIR|ular accounts 4612|5989|8490|1|20|37899.60|0.02|0.03|R|F|1993-09-24|1993-12-18|1993-10-22|NONE|AIR|beans sleep blithely iro 4612|49750|2255|2|17|28895.75|0.10|0.06|A|F|1994-01-09|1993-11-08|1994-02-06|TAKE BACK RETURN|REG AIR|equests haggle carefully silent excus 4612|136371|1398|3|40|56294.80|0.08|0.01|R|F|1993-10-08|1993-11-23|1993-10-24|DELIVER IN PERSON|RAIL|special platelets. 4612|184664|7183|4|10|17486.60|0.10|0.06|A|F|1993-11-11|1993-11-19|1993-11-13|TAKE BACK RETURN|SHIP|unusual theodol 4613|37372|7373|1|17|22259.29|0.09|0.07|N|O|1998-06-07|1998-05-11|1998-06-29|DELIVER IN PERSON|SHIP|liers cajole a 4613|107883|394|2|25|47272.00|0.05|0.04|N|O|1998-05-22|1998-04-11|1998-05-27|TAKE BACK RETURN|SHIP|y pending platelets x-ray ironically! pend 4613|173667|8702|3|15|26109.90|0.10|0.02|N|O|1998-05-31|1998-04-16|1998-06-25|DELIVER IN PERSON|MAIL|against the quickly r 4613|7183|9684|4|36|39246.48|0.04|0.01|N|O|1998-04-22|1998-05-05|1998-05-04|DELIVER IN PERSON|AIR|gainst the furiously ironic 4613|110121|5144|5|35|39589.20|0.04|0.06|N|O|1998-06-04|1998-04-17|1998-06-20|COLLECT COD|MAIL|e blithely against the even, bold pi 4613|195878|8398|6|47|92771.89|0.04|0.04|N|O|1998-07-03|1998-05-26|1998-07-09|NONE|FOB|uriously special requests wak 4613|118713|1225|7|39|67536.69|0.09|0.05|N|O|1998-06-12|1998-06-01|1998-07-06|DELIVER IN PERSON|REG AIR|ously express 4614|6014|1015|1|19|17480.19|0.09|0.08|N|O|1996-05-17|1996-06-21|1996-06-08|TAKE BACK RETURN|AIR|ix. carefully regular 4614|64629|4630|2|3|4780.86|0.08|0.01|N|O|1996-07-22|1996-07-21|1996-08-07|NONE|MAIL|ions engage final, ironic 4614|7136|9637|3|36|37552.68|0.10|0.04|N|O|1996-07-05|1996-06-26|1996-07-07|NONE|REG AIR|onic foxes affix furi 4614|125036|2573|4|6|6366.18|0.09|0.01|N|O|1996-06-11|1996-05-30|1996-07-03|COLLECT COD|REG AIR|ake quickly quickly regular epitap 4614|72044|4552|5|24|24384.96|0.07|0.06|N|O|1996-07-01|1996-06-24|1996-07-08|COLLECT COD|REG AIR|regular, even 4614|33149|3150|6|32|34628.48|0.10|0.05|N|O|1996-08-21|1996-05-28|1996-09-16|NONE|REG AIR|ickly furio 4614|127060|4597|7|41|44569.46|0.01|0.07|N|O|1996-07-31|1996-07-12|1996-08-16|COLLECT COD|REG AIR|ackages haggle carefully about the even, b 4615|91296|3806|1|10|12872.90|0.02|0.08|A|F|1993-11-20|1993-10-05|1993-12-08|DELIVER IN PERSON|AIR|sits. slyly express deposits are 4640|87422|4947|1|5|7047.10|0.03|0.08|N|O|1996-02-05|1996-02-14|1996-02-15|TAKE BACK RETURN|RAIL| warthogs against the regular 4640|87287|4812|2|9|11468.52|0.03|0.05|N|O|1996-02-12|1996-02-14|1996-02-29|DELIVER IN PERSON|AIR| accounts. unu 4640|26265|3772|3|18|21442.68|0.02|0.07|N|O|1996-02-28|1996-03-06|1996-03-28|DELIVER IN PERSON|RAIL|boost furiously accord 4640|22346|7351|4|36|45660.24|0.06|0.08|N|O|1996-01-03|1996-03-09|1996-01-11|DELIVER IN PERSON|RAIL|iously furious accounts boost. carefully 4640|155974|1005|5|15|30449.55|0.03|0.02|N|O|1996-03-19|1996-02-09|1996-04-11|TAKE BACK RETURN|FOB|y regular instructions doze furiously. reg 4641|189796|7351|1|45|84860.55|0.07|0.03|R|F|1993-05-11|1993-04-19|1993-05-21|DELIVER IN PERSON|MAIL| about the close 4641|94261|6771|2|39|48955.14|0.06|0.00|R|F|1993-02-10|1993-03-06|1993-02-15|TAKE BACK RETURN|REG AIR| the bold reque 4641|35287|5288|3|15|18334.20|0.01|0.08|R|F|1993-01-25|1993-04-09|1993-02-05|TAKE BACK RETURN|AIR|s. carefully even exc 4642|193181|8220|1|11|14015.98|0.04|0.07|A|F|1995-05-23|1995-04-26|1995-06-04|COLLECT COD|TRUCK|lithely express asympt 4642|179594|2112|2|34|56902.06|0.04|0.07|R|F|1995-04-01|1995-05-11|1995-04-23|COLLECT COD|SHIP|theodolites detect among the ironically sp 4642|20152|153|3|10|10721.50|0.04|0.02|R|F|1995-04-16|1995-04-28|1995-04-24|COLLECT COD|RAIL|urts. even deposits nag beneath 4642|93698|8717|4|18|30450.42|0.00|0.04|N|F|1995-06-16|1995-04-16|1995-06-21|NONE|TRUCK|ily pending accounts hag 4642|178937|8938|5|41|82653.13|0.10|0.00|R|F|1995-04-08|1995-04-13|1995-05-01|DELIVER IN PERSON|MAIL|s are blithely. requests wake above the fur 4643|184931|2486|1|50|100796.50|0.08|0.05|N|O|1995-09-11|1995-08-13|1995-09-30|DELIVER IN PERSON|SHIP|. ironic deposits cajo 4644|176873|9391|1|4|7799.48|0.06|0.03|N|O|1998-05-06|1998-03-19|1998-05-28|NONE|MAIL|gular requests? pendi 4644|96503|6504|2|16|23992.00|0.03|0.04|N|O|1998-03-13|1998-02-21|1998-04-03|COLLECT COD|SHIP|lar excuses across the 4644|114307|6819|3|10|13213.00|0.02|0.02|N|O|1998-02-21|1998-02-28|1998-03-19|COLLECT COD|REG AIR|osits according to the 4644|153953|6469|4|45|90312.75|0.10|0.07|N|O|1998-02-02|1998-04-08|1998-02-15|COLLECT COD|SHIP| carefully a 4644|86667|4192|5|10|16536.60|0.08|0.08|N|O|1998-03-12|1998-03-11|1998-03-19|TAKE BACK RETURN|REG AIR| the slow, final fo 4645|49353|1858|1|45|58605.75|0.09|0.05|A|F|1994-12-27|1994-11-02|1994-12-31|DELIVER IN PERSON|AIR|ular ideas. slyly 4645|65060|5061|2|32|32801.92|0.10|0.08|A|F|1994-11-17|1994-10-30|1994-11-18|COLLECT COD|REG AIR| final accounts alongside 4645|53599|3600|3|25|38814.75|0.03|0.00|R|F|1994-10-25|1994-12-11|1994-11-14|NONE|REG AIR|braids. ironic dependencies main 4645|36614|1621|4|42|65125.62|0.10|0.02|R|F|1994-12-02|1994-12-18|1994-12-16|COLLECT COD|TRUCK|regular pinto beans amon 4645|160043|2560|5|35|38606.40|0.03|0.07|A|F|1994-12-08|1994-11-25|1994-12-09|TAKE BACK RETURN|FOB|sias believe bl 4645|41437|3942|6|27|37217.61|0.09|0.08|R|F|1994-11-26|1994-10-25|1994-12-04|NONE|SHIP|ously express pinto beans. ironic depos 4645|30142|143|7|42|45029.88|0.10|0.06|A|F|1994-12-31|1994-10-22|1995-01-28|DELIVER IN PERSON|AIR|e slyly regular pinto beans. thin 4646|190170|2690|1|24|30244.08|0.02|0.05|N|O|1996-09-18|1996-08-09|1996-09-21|TAKE BACK RETURN|RAIL|ic platelets lose carefully. blithely unu 4646|177583|5135|2|26|43175.08|0.07|0.00|N|O|1996-10-02|1996-08-25|1996-10-27|DELIVER IN PERSON|MAIL|ix according to the slyly spe 4646|33120|630|3|18|18956.16|0.01|0.00|N|O|1996-06-30|1996-08-10|1996-07-12|TAKE BACK RETURN|TRUCK|beans sleep car 4646|39105|9106|4|38|39675.80|0.08|0.01|N|O|1996-09-01|1996-08-23|1996-09-27|COLLECT COD|SHIP|al platelets cajole. slyly final dol 4646|25896|8399|5|22|40081.58|0.01|0.08|N|O|1996-07-14|1996-08-06|1996-07-29|DELIVER IN PERSON|MAIL|cies are blithely after the slyly reg 4647|92862|7881|1|16|29677.76|0.09|0.07|R|F|1994-09-07|1994-07-15|1994-10-06|COLLECT COD|RAIL|o beans about the fluffily special the 4647|128884|6421|2|34|65037.92|0.01|0.02|R|F|1994-05-20|1994-06-20|1994-05-29|COLLECT COD|TRUCK|ly sly accounts 4647|146614|6615|3|27|44836.47|0.03|0.08|R|F|1994-05-20|1994-06-26|1994-05-30|NONE|FOB|ully even ti 4647|138275|8276|4|2|2626.54|0.04|0.07|R|F|1994-07-03|1994-07-22|1994-07-22|TAKE BACK RETURN|RAIL|dolites wake furiously special pinto be 4647|186829|6830|5|2|3831.64|0.07|0.06|A|F|1994-05-27|1994-08-05|1994-06-10|TAKE BACK RETURN|FOB| pinto beans believe furiously slyly silent 4647|28488|991|6|28|39661.44|0.02|0.03|A|F|1994-08-25|1994-08-06|1994-09-18|DELIVER IN PERSON|FOB| are above the fluffily fin 4672|58147|653|1|22|24313.08|0.01|0.07|N|O|1995-12-03|1995-12-08|1995-12-17|COLLECT COD|AIR|l instructions. blithely ironic packages 4672|60961|3468|2|41|78800.36|0.00|0.00|N|O|1995-12-01|1995-12-15|1995-12-12|COLLECT COD|RAIL| slyly quie 4672|162043|7076|3|24|26520.96|0.04|0.03|N|O|1995-11-11|1995-12-28|1995-12-04|NONE|REG AIR|y fluffily stealt 4672|56171|1182|4|13|14653.21|0.10|0.03|N|O|1996-02-02|1995-12-13|1996-03-02|DELIVER IN PERSON|RAIL|ar requests? pending accounts against 4672|54022|9033|5|45|43920.90|0.08|0.07|N|O|1996-02-07|1996-01-16|1996-02-14|DELIVER IN PERSON|MAIL| platelets use amon 4672|140922|3437|6|20|39258.40|0.02|0.07|N|O|1995-12-08|1996-01-25|1995-12-19|COLLECT COD|REG AIR|s boost at the ca 4672|71853|9375|7|38|69344.30|0.01|0.01|N|O|1995-11-28|1995-12-08|1995-12-13|COLLECT COD|SHIP|ests. idle, regular ex 4673|16607|6608|1|8|12188.80|0.08|0.01|N|O|1996-10-12|1996-10-05|1996-11-04|TAKE BACK RETURN|FOB|lithely final re 4673|100894|895|2|44|83375.16|0.06|0.01|N|O|1996-12-11|1996-10-31|1997-01-08|DELIVER IN PERSON|RAIL| gifts cajole dari 4673|122258|7283|3|9|11522.25|0.04|0.07|N|O|1996-10-15|1996-09-30|1996-10-30|DELIVER IN PERSON|MAIL|ages nag across 4674|149045|1560|1|50|54702.00|0.07|0.08|A|F|1994-05-13|1994-06-15|1994-06-05|COLLECT COD|RAIL|haggle about the blithel 4674|188714|1233|2|35|63094.85|0.02|0.05|A|F|1994-08-02|1994-06-04|1994-08-21|COLLECT COD|FOB|le quickly after the express sent 4674|110443|2955|3|3|4360.32|0.01|0.05|A|F|1994-07-19|1994-05-28|1994-07-23|TAKE BACK RETURN|RAIL| regular requests na 4674|12164|4666|4|21|22599.36|0.02|0.08|R|F|1994-05-08|1994-07-02|1994-06-04|COLLECT COD|RAIL|ent accounts sublate deposits. instruc 4675|170390|391|1|6|8762.34|0.00|0.05|R|F|1994-01-22|1994-01-06|1994-02-12|TAKE BACK RETURN|TRUCK| unusual ideas thrash bl 4675|143689|8718|2|12|20792.16|0.00|0.04|A|F|1993-12-22|1994-01-12|1993-12-23|TAKE BACK RETURN|AIR|posits affix carefully 4675|180963|964|3|5|10219.80|0.05|0.05|A|F|1994-01-16|1994-01-05|1994-01-18|DELIVER IN PERSON|RAIL|lent pinto beans 4675|33646|1156|4|26|41070.64|0.03|0.01|A|F|1993-12-16|1993-12-29|1993-12-23|DELIVER IN PERSON|SHIP|nts. express requests are quickly 4675|80498|499|5|18|26612.82|0.01|0.08|R|F|1994-02-23|1994-01-18|1994-03-05|TAKE BACK RETURN|FOB|cajole unusual dep 4675|118089|8090|6|1|1107.08|0.10|0.06|R|F|1994-03-18|1994-02-14|1994-04-17|NONE|SHIP|unts. caref 4676|164646|9679|1|47|80400.08|0.03|0.06|N|O|1995-12-20|1995-10-04|1996-01-09|NONE|AIR|lithely about the carefully special requ 4676|5043|44|2|33|31285.32|0.08|0.05|N|O|1995-12-29|1995-10-01|1996-01-18|TAKE BACK RETURN|FOB|yly express 4676|145173|7688|3|4|4872.68|0.10|0.06|N|O|1995-12-12|1995-10-22|1995-12-13|TAKE BACK RETURN|TRUCK|detect above the ironic platelets. fluffily 4676|110831|832|4|50|92091.50|0.07|0.01|N|O|1995-09-20|1995-11-20|1995-10-18|TAKE BACK RETURN|AIR|r deposits boost boldly quickly quick asymp 4676|121096|3609|5|29|32395.61|0.01|0.02|N|O|1995-12-29|1995-11-12|1996-01-06|TAKE BACK RETURN|RAIL|ly regular theodolites sleep. 4676|45569|5570|6|8|12116.48|0.08|0.08|N|O|1995-12-05|1995-10-18|1996-01-02|COLLECT COD|AIR|cuses boost above 4676|63244|8257|7|13|15694.12|0.05|0.07|N|O|1995-11-18|1995-11-07|1995-12-10|TAKE BACK RETURN|TRUCK| at the slyly bold attainments. silently e 4677|127864|377|1|25|47296.50|0.04|0.04|N|O|1998-04-11|1998-05-11|1998-04-18|TAKE BACK RETURN|REG AIR|unts doubt furiousl 4678|57388|9894|1|35|47088.30|0.04|0.08|N|O|1998-11-27|1998-10-02|1998-12-17|TAKE BACK RETURN|AIR|he accounts. fluffily bold sheaves b 4678|116672|9184|2|18|30396.06|0.03|0.06|N|O|1998-10-30|1998-09-22|1998-11-25|TAKE BACK RETURN|SHIP|usly ironic 4678|95355|374|3|13|17554.55|0.10|0.07|N|O|1998-11-03|1998-10-17|1998-11-06|TAKE BACK RETURN|SHIP|its. carefully final fr 4678|21704|6709|4|23|37391.10|0.06|0.05|N|O|1998-09-03|1998-09-20|1998-09-04|DELIVER IN PERSON|SHIP|ily sly deposi 4678|177771|7772|5|40|73950.80|0.03|0.07|N|O|1998-11-11|1998-10-27|1998-11-24|TAKE BACK RETURN|AIR|. final, unusual requests sleep thinl 4679|189136|4173|1|7|8575.91|0.10|0.05|R|F|1993-05-11|1993-04-11|1993-05-16|NONE|TRUCK|kages. bold, regular packa 4704|77934|5456|1|14|26767.02|0.04|0.04|N|O|1996-10-27|1996-11-02|1996-11-07|DELIVER IN PERSON|TRUCK| above the slyly final requests. quickly 4704|27671|174|2|7|11190.69|0.03|0.04|N|O|1996-12-04|1996-10-30|1996-12-23|DELIVER IN PERSON|SHIP|ers wake car 4704|63081|3082|3|44|45939.52|0.02|0.05|N|O|1996-09-02|1996-10-07|1996-09-17|DELIVER IN PERSON|REG AIR|out the care 4705|110778|5801|1|22|39352.94|0.04|0.04|R|F|1992-07-05|1992-05-11|1992-07-29|DELIVER IN PERSON|SHIP| fluffily pending accounts ca 4705|30527|3031|2|14|20405.28|0.00|0.08|R|F|1992-07-14|1992-05-23|1992-07-25|DELIVER IN PERSON|TRUCK|ain carefully amon 4705|55426|437|3|16|22102.72|0.07|0.08|R|F|1992-07-02|1992-06-06|1992-07-06|DELIVER IN PERSON|RAIL|special ideas nag sl 4705|129147|6684|4|31|36460.34|0.03|0.03|R|F|1992-04-03|1992-05-30|1992-04-05|COLLECT COD|TRUCK|furiously final accou 4705|162432|7465|5|28|41844.04|0.10|0.01|A|F|1992-06-03|1992-06-07|1992-06-22|DELIVER IN PERSON|MAIL|tes wake according to the unusual plate 4705|183715|6234|6|23|41370.33|0.06|0.03|R|F|1992-06-22|1992-06-11|1992-07-18|DELIVER IN PERSON|MAIL| above the furiously ev 4705|88534|3551|7|40|60901.20|0.08|0.06|A|F|1992-04-19|1992-04-28|1992-05-07|COLLECT COD|TRUCK|blithely. sly 4706|181872|4391|1|37|72293.19|0.02|0.06|A|F|1993-02-20|1993-03-05|1993-03-03|DELIVER IN PERSON|TRUCK|kly final deposits c 4706|121133|1134|2|23|26544.99|0.03|0.01|A|F|1993-04-01|1993-03-13|1993-05-01|COLLECT COD|FOB|deas across t 4706|67134|2147|3|6|6606.78|0.01|0.04|R|F|1993-01-20|1993-03-18|1993-01-26|NONE|MAIL|efully eve 4706|115455|7967|4|5|7352.25|0.06|0.06|R|F|1993-02-14|1993-01-31|1993-02-26|NONE|REG AIR|ptotes haggle ca 4706|49189|1694|5|27|30730.86|0.06|0.08|A|F|1993-04-04|1993-03-11|1993-04-09|COLLECT COD|REG AIR|into beans. finally special instruct 4707|33590|8597|1|7|10665.13|0.02|0.05|R|F|1995-05-14|1995-04-06|1995-06-06|COLLECT COD|SHIP|ecial sheaves boost blithely accor 4707|135550|577|2|49|77691.95|0.00|0.07|N|F|1995-06-17|1995-05-16|1995-06-25|COLLECT COD|FOB| alongside of the slyly ironic instructio 4708|190913|5952|1|18|36070.38|0.02|0.04|A|F|1994-11-11|1994-11-15|1994-11-26|NONE|REG AIR|special, eve 4708|74046|1568|2|5|5100.20|0.05|0.05|A|F|1994-10-15|1994-12-02|1994-11-12|COLLECT COD|MAIL|ely. carefully sp 4708|76824|9332|3|32|57626.24|0.04|0.07|A|F|1994-11-12|1994-11-14|1994-11-23|TAKE BACK RETURN|MAIL|the accounts. e 4709|24334|4335|1|25|31458.25|0.03|0.05|N|O|1996-02-21|1996-02-11|1996-03-17|DELIVER IN PERSON|AIR|deposits grow. fluffily unusual accounts 4709|176003|3555|2|25|26975.00|0.05|0.03|N|O|1996-01-22|1996-03-03|1996-02-21|DELIVER IN PERSON|REG AIR|inst the ironic, regul 4710|182634|189|1|40|68665.20|0.10|0.08|A|F|1995-03-09|1995-02-25|1995-03-29|TAKE BACK RETURN|AIR|cross the blithely bold packages. silen 4710|127253|9766|2|47|60171.75|0.04|0.01|R|F|1995-02-22|1995-01-12|1995-02-28|NONE|RAIL|blithely express packages. even, ironic re 4711|132347|2348|1|7|9655.38|0.03|0.01|N|O|1998-05-12|1998-06-24|1998-05-24|COLLECT COD|MAIL|ly. bold accounts use fluff 4711|144056|4057|2|15|16500.75|0.08|0.07|N|O|1998-06-09|1998-07-30|1998-06-18|COLLECT COD|SHIP| beans wake. deposits could bo 4711|149014|9015|3|22|23386.22|0.02|0.03|N|O|1998-06-21|1998-06-18|1998-07-19|TAKE BACK RETURN|REG AIR|along the quickly careful packages. bli 4711|64012|1531|4|8|7808.08|0.07|0.00|N|O|1998-06-17|1998-06-13|1998-06-27|TAKE BACK RETURN|SHIP|g to the carefully ironic deposits. specia 4711|48820|3829|5|15|26532.30|0.05|0.01|N|O|1998-09-03|1998-07-15|1998-09-13|TAKE BACK RETURN|SHIP|ld requests: furiously final inst 4711|115669|5670|6|45|75809.70|0.05|0.06|N|O|1998-05-19|1998-07-14|1998-05-21|COLLECT COD|SHIP| ironic theodolites 4711|45160|2673|7|18|19892.88|0.05|0.04|N|O|1998-07-03|1998-07-31|1998-07-23|DELIVER IN PERSON|RAIL| blithely. bold asymptote 4736|195526|3084|1|26|42159.52|0.03|0.03|N|O|1996-02-02|1996-01-18|1996-02-09|DELIVER IN PERSON|AIR|efully speci 4736|3390|891|2|43|55615.77|0.06|0.07|N|O|1996-02-05|1995-12-21|1996-02-06|COLLECT COD|MAIL|quests. carefully 4737|190140|7698|1|37|45515.18|0.03|0.04|R|F|1993-05-17|1993-04-10|1993-05-30|DELIVER IN PERSON|TRUCK|s. fluffily regular 4737|68303|810|2|22|27968.60|0.04|0.04|A|F|1993-03-29|1993-05-22|1993-04-16|TAKE BACK RETURN|RAIL| hang fluffily around t 4738|186778|4333|1|9|16782.93|0.04|0.04|A|F|1992-06-01|1992-06-26|1992-06-02|COLLECT COD|TRUCK|posits serve slyly. unusual pint 4738|172482|5000|2|16|24871.68|0.07|0.08|A|F|1992-06-17|1992-06-20|1992-06-21|NONE|MAIL|nic deposits are slyly! carefu 4738|99175|1685|3|50|58708.50|0.04|0.02|A|F|1992-06-18|1992-07-04|1992-07-07|TAKE BACK RETURN|TRUCK|the blithely ironic braids sleep slyly 4738|28274|777|4|22|26449.94|0.02|0.08|A|F|1992-05-25|1992-05-19|1992-06-12|COLLECT COD|SHIP|ld, even packages. furio 4738|186242|6243|5|13|17267.12|0.04|0.05|R|F|1992-05-30|1992-06-11|1992-06-26|COLLECT COD|AIR| wake. unusual platelets for the 4738|158558|6104|6|10|16165.50|0.10|0.01|R|F|1992-07-10|1992-06-16|1992-07-25|TAKE BACK RETURN|SHIP|hins above the 4738|82750|275|7|28|48517.00|0.05|0.07|A|F|1992-06-09|1992-07-05|1992-06-25|NONE|AIR|e furiously ironic excuses. care 4739|167941|7942|1|8|16071.52|0.07|0.07|R|F|1993-06-22|1993-05-10|1993-07-11|TAKE BACK RETURN|SHIP|cording to the 4739|184061|6580|2|31|35496.86|0.09|0.06|R|F|1993-06-20|1993-05-18|1993-06-26|COLLECT COD|SHIP|blithely special pin 4739|99183|6711|3|30|35465.40|0.09|0.00|A|F|1993-05-29|1993-04-12|1993-06-18|NONE|TRUCK|ly even packages use across th 4740|2282|2283|1|22|26054.16|0.06|0.01|N|O|1996-10-04|1996-08-17|1996-10-05|TAKE BACK RETURN|RAIL|final dependencies nag 4740|152247|9793|2|24|31181.76|0.08|0.02|N|O|1996-09-10|1996-09-27|1996-10-07|TAKE BACK RETURN|TRUCK|hely regular deposits 4741|72763|7778|1|24|41658.24|0.00|0.01|A|F|1992-09-16|1992-09-19|1992-09-20|DELIVER IN PERSON|RAIL|deas boost furiously slyly regular id 4741|112165|2166|2|16|18834.56|0.01|0.07|R|F|1992-08-25|1992-08-10|1992-08-29|TAKE BACK RETURN|FOB|final foxes haggle r 4741|155605|3151|3|24|39854.40|0.05|0.08|A|F|1992-11-04|1992-08-14|1992-11-06|TAKE BACK RETURN|MAIL|even requests. 4741|50954|8470|4|39|74293.05|0.09|0.06|R|F|1992-10-28|1992-10-03|1992-11-11|COLLECT COD|SHIP|t, regular requests 4741|178507|8508|5|40|63420.00|0.09|0.03|R|F|1992-09-20|1992-09-23|1992-10-09|TAKE BACK RETURN|REG AIR| fluffily slow deposits. fluffily regu 4741|156686|9202|6|34|59251.12|0.02|0.07|R|F|1992-08-25|1992-08-18|1992-09-20|DELIVER IN PERSON|RAIL|sly special packages after the furiously 4742|155564|8080|1|32|51825.92|0.10|0.08|R|F|1995-04-04|1995-06-12|1995-04-19|COLLECT COD|RAIL|eposits boost blithely. carefully regular a 4742|154833|2379|2|29|54747.07|0.02|0.03|N|F|1995-06-15|1995-05-05|1995-06-24|COLLECT COD|REG AIR|integrate closely among t 4742|71018|8540|3|15|14835.15|0.06|0.04|N|O|1995-07-20|1995-05-26|1995-08-11|NONE|SHIP|terns are sl 4742|187940|5495|4|31|62866.14|0.05|0.08|N|F|1995-06-13|1995-05-08|1995-06-24|COLLECT COD|REG AIR|ke slyly among the furiousl 4742|99331|9332|5|45|59864.85|0.05|0.00|R|F|1995-05-12|1995-05-14|1995-06-07|TAKE BACK RETURN|RAIL|ke carefully. do 4743|59402|4413|1|19|25866.60|0.04|0.07|A|F|1993-06-23|1993-05-03|1993-07-20|COLLECT COD|AIR|hely even accounts 4743|158379|3410|2|3|4312.11|0.01|0.03|R|F|1993-04-14|1993-06-08|1993-05-09|NONE|TRUCK|al requests. express idea 4743|72936|7951|3|21|40087.53|0.08|0.03|A|F|1993-07-02|1993-06-15|1993-07-26|DELIVER IN PERSON|RAIL|ake blithely against the packages. reg 4743|33527|3528|4|27|39434.04|0.08|0.05|R|F|1993-07-26|1993-05-27|1993-08-24|DELIVER IN PERSON|AIR|aids use. express deposits 4768|35822|829|1|5|8789.10|0.00|0.03|R|F|1993-12-27|1994-02-09|1994-01-11|NONE|MAIL|egular accounts. bravely final fra 4769|34240|1750|1|16|18787.84|0.08|0.05|N|O|1995-07-16|1995-07-05|1995-07-22|TAKE BACK RETURN|FOB| deposits. slyly even asymptote 4769|62918|437|2|34|63950.94|0.06|0.07|N|O|1995-07-26|1995-05-18|1995-08-03|COLLECT COD|REG AIR|ven instructions. ca 4769|46414|1423|3|36|48974.76|0.10|0.03|N|O|1995-07-22|1995-06-16|1995-08-11|NONE|RAIL|. slyly even deposit 4769|68431|8432|4|45|62974.35|0.08|0.06|R|F|1995-06-01|1995-07-13|1995-06-04|TAKE BACK RETURN|RAIL|accounts are. even accounts sleep 4769|111623|4135|5|15|24519.30|0.07|0.08|N|F|1995-06-12|1995-07-07|1995-07-04|NONE|SHIP|egular platelets can cajole across the 4770|31961|9471|1|41|77611.36|0.00|0.08|N|O|1995-09-04|1995-08-08|1995-09-10|COLLECT COD|FOB|ithely even packages sleep caref 4770|156036|8552|2|30|32760.90|0.09|0.07|N|O|1995-08-25|1995-08-27|1995-09-07|COLLECT COD|SHIP|ffily carefully ironic ideas. ironic d 4771|48612|8613|1|9|14045.49|0.01|0.00|R|F|1993-02-28|1993-02-19|1993-03-25|NONE|FOB|riously after the packages. fina 4771|15316|5317|2|21|25857.51|0.09|0.01|R|F|1993-01-19|1993-02-10|1993-02-01|NONE|FOB|fluffily pendi 4771|11568|1569|3|5|7397.80|0.06|0.08|R|F|1993-01-07|1993-01-19|1993-01-26|NONE|RAIL|ar, quiet accounts nag furiously express id 4771|8448|3449|4|21|28485.24|0.05|0.04|A|F|1992-12-20|1993-01-22|1992-12-26|TAKE BACK RETURN|SHIP| carefully re 4772|86830|6831|1|1|1816.83|0.10|0.00|R|F|1994-11-13|1994-10-25|1994-11-15|DELIVER IN PERSON|AIR|ans. slyly even acc 4772|145505|534|2|16|24808.00|0.07|0.06|R|F|1994-10-27|1994-12-07|1994-10-29|TAKE BACK RETURN|MAIL|egular accounts wake s 4772|94751|4752|3|31|54118.25|0.02|0.04|A|F|1994-10-02|1994-10-21|1994-10-13|TAKE BACK RETURN|FOB|ests are thinly. furiously unusua 4772|70799|5814|4|15|26546.85|0.02|0.07|R|F|1994-09-19|1994-10-22|1994-09-26|COLLECT COD|TRUCK| requests. express, regular th 4773|143131|3132|1|23|27004.99|0.00|0.08|N|O|1996-01-01|1996-03-19|1996-01-04|NONE|FOB|ly express grouches wak 4773|196476|8996|2|36|56608.92|0.09|0.04|N|O|1996-04-08|1996-03-03|1996-05-01|COLLECT COD|REG AIR| dependencies. quickly 4773|166028|6029|3|49|53606.98|0.05|0.02|N|O|1996-01-26|1996-02-29|1996-01-27|TAKE BACK RETURN|FOB|y final reque 4773|19370|6874|4|49|63179.13|0.09|0.04|N|O|1996-01-12|1996-02-17|1996-02-05|TAKE BACK RETURN|TRUCK|ly pending theodolites cajole caref 4773|149461|4490|5|20|30209.20|0.02|0.07|N|O|1995-12-28|1996-02-17|1996-01-15|COLLECT COD|TRUCK| blithely final deposits nag after t 4773|189276|1795|6|11|15017.97|0.10|0.06|N|O|1996-01-02|1996-01-29|1996-01-24|DELIVER IN PERSON|REG AIR|en accounts. slyly b 4773|157333|2364|7|6|8341.98|0.07|0.01|N|O|1996-03-09|1996-03-18|1996-03-27|NONE|AIR|latelets haggle s 4774|83182|5691|1|45|52433.10|0.10|0.00|R|F|1993-07-07|1993-06-08|1993-07-31|COLLECT COD|TRUCK| haggle busily afte 4774|38828|1332|2|4|7067.28|0.02|0.03|A|F|1993-08-03|1993-05-30|1993-08-19|COLLECT COD|FOB|xes according to the foxes wake above the f 4774|172460|2461|3|47|72025.62|0.10|0.08|R|F|1993-06-13|1993-07-04|1993-07-09|TAKE BACK RETURN|FOB|regular dolphins above the furi 4774|129151|6688|4|30|35404.50|0.05|0.08|A|F|1993-08-18|1993-06-08|1993-08-21|DELIVER IN PERSON|REG AIR|tions against the blithely final theodolit 4775|73979|6487|1|1|1952.97|0.10|0.02|N|O|1995-09-06|1995-09-28|1995-09-29|DELIVER IN PERSON|MAIL|furiously ironic theodolite 4775|152779|5295|2|37|67775.49|0.02|0.01|N|O|1995-09-06|1995-09-28|1995-09-28|COLLECT COD|TRUCK|ts. pinto beans use according to th 4775|152242|9788|3|34|44004.16|0.09|0.06|N|O|1995-09-14|1995-10-15|1995-09-21|DELIVER IN PERSON|MAIL|onic epitaphs. f 4775|118461|5995|4|39|57698.94|0.07|0.04|N|O|1995-08-30|1995-10-12|1995-09-20|NONE|AIR|eep never with the slyly regular acc 4800|96498|1517|1|11|16439.39|0.03|0.03|R|F|1992-01-27|1992-03-16|1992-02-19|TAKE BACK RETURN|RAIL|ic dependenc 4800|25491|496|2|1|1416.49|0.06|0.06|A|F|1992-02-23|1992-03-16|1992-03-20|TAKE BACK RETURN|MAIL|nal accounts are blithely deposits. bol 4800|10518|5521|3|21|29998.71|0.09|0.05|A|F|1992-02-14|1992-03-15|1992-02-26|NONE|SHIP|ithely according to 4800|175228|5229|4|38|49522.36|0.10|0.08|R|F|1992-02-01|1992-02-28|1992-02-21|NONE|TRUCK|s sleep fluffily. furiou 4800|52240|2241|5|24|28613.76|0.08|0.04|R|F|1992-01-14|1992-02-23|1992-01-25|NONE|TRUCK|ully carefully r 4801|183937|1492|1|37|74774.41|0.10|0.02|N|O|1996-03-09|1996-02-29|1996-03-25|TAKE BACK RETURN|FOB|uests hinder blithely against the instr 4801|25246|7749|2|34|39822.16|0.03|0.02|N|O|1996-02-05|1996-04-16|1996-02-23|NONE|SHIP|y final requests 4801|109282|9283|3|4|5165.12|0.04|0.04|N|O|1996-03-23|1996-04-04|1996-03-25|COLLECT COD|RAIL|pitaphs. regular, reg 4801|91183|1184|4|39|45793.02|0.07|0.01|N|O|1996-03-19|1996-03-21|1996-04-17|TAKE BACK RETURN|REG AIR|warhorses wake never for the care 4802|39444|9445|1|6|8300.64|0.00|0.06|N|O|1997-04-16|1997-03-25|1997-04-21|TAKE BACK RETURN|SHIP|unusual accounts wake blithely. b 4803|131052|6079|1|2|2166.10|0.08|0.03|N|O|1996-04-16|1996-03-20|1996-05-15|NONE|REG AIR|gular reque 4803|175165|2717|2|47|58287.52|0.10|0.00|N|O|1996-03-14|1996-03-30|1996-03-15|DELIVER IN PERSON|FOB|ly final excuses. slyly express requ 4803|195742|8262|3|42|77185.08|0.04|0.08|N|O|1996-04-27|1996-05-05|1996-05-17|NONE|TRUCK| accounts affix quickly ar 4803|21312|6317|4|24|29599.44|0.10|0.04|N|O|1996-02-24|1996-04-02|1996-02-28|NONE|MAIL|t blithely slyly special decoys. 4803|188148|5703|5|21|25958.94|0.03|0.06|N|O|1996-05-25|1996-03-15|1996-06-09|COLLECT COD|FOB| silent packages use. b 4803|193599|3600|6|19|32159.21|0.07|0.00|N|O|1996-04-20|1996-03-25|1996-04-27|TAKE BACK RETURN|RAIL|sts. enticing, even 4804|127009|4546|1|44|45584.00|0.06|0.08|A|F|1992-05-02|1992-03-24|1992-05-28|TAKE BACK RETURN|AIR|aggle quickly among the slyly fi 4804|34942|9949|2|41|76954.54|0.10|0.02|R|F|1992-04-06|1992-04-12|1992-05-03|COLLECT COD|MAIL|. deposits haggle express tithes? 4804|64829|9842|3|33|59196.06|0.09|0.05|A|F|1992-03-02|1992-04-14|1992-03-13|DELIVER IN PERSON|AIR|, thin excuses. 4805|149235|9236|1|7|8989.61|0.09|0.03|A|F|1992-05-01|1992-07-09|1992-05-09|NONE|FOB| requests. regular deposit 4805|188967|1486|2|45|92518.20|0.02|0.03|R|F|1992-06-16|1992-06-08|1992-07-03|NONE|TRUCK|the furiously sly t 4805|153857|1403|3|44|84077.40|0.01|0.02|R|F|1992-05-14|1992-06-23|1992-05-25|DELIVER IN PERSON|SHIP|eposits sleep furiously qui 4805|64284|9297|4|13|16227.64|0.04|0.04|R|F|1992-07-16|1992-06-07|1992-08-10|COLLECT COD|AIR|its serve about the accounts. slyly regu 4805|8529|8530|5|42|60375.84|0.03|0.03|R|F|1992-08-17|1992-07-03|1992-09-14|NONE|REG AIR|the regular, fina 4805|135959|986|6|18|35909.10|0.06|0.04|A|F|1992-06-07|1992-07-10|1992-06-12|COLLECT COD|TRUCK|o use pending, unusu 4806|15046|5047|1|26|24987.04|0.10|0.05|R|F|1993-05-28|1993-06-07|1993-05-29|DELIVER IN PERSON|SHIP| bold pearls sublate blithely. quickly pe 4806|71151|8673|2|6|6732.90|0.01|0.06|A|F|1993-05-17|1993-07-19|1993-05-29|TAKE BACK RETURN|SHIP|even theodolites. packages sl 4806|28806|1309|3|8|13878.40|0.09|0.00|A|F|1993-05-08|1993-07-16|1993-05-28|NONE|TRUCK|requests boost blithely. qui 4807|121903|6928|1|9|17324.10|0.04|0.08|N|O|1997-04-23|1997-03-01|1997-05-15|TAKE BACK RETURN|TRUCK|may are blithely. carefully even pinto b 4807|9767|9768|2|41|68747.16|0.07|0.08|N|O|1997-05-02|1997-03-31|1997-05-15|TAKE BACK RETURN|AIR| fluffily re 4807|144309|4310|3|34|46012.20|0.06|0.02|N|O|1997-01-31|1997-03-13|1997-02-01|NONE|SHIP|ecial ideas. deposits according to the fin 4807|189336|4373|4|32|45610.56|0.05|0.00|N|O|1997-04-04|1997-03-21|1997-04-16|NONE|RAIL|efully even dolphins slee 4807|158706|6252|5|2|3529.40|0.02|0.05|N|O|1997-05-09|1997-04-03|1997-06-05|TAKE BACK RETURN|RAIL|deas wake bli 4807|159022|9023|6|22|23782.44|0.09|0.06|N|O|1997-03-13|1997-02-23|1997-04-01|NONE|FOB|es use final excuses. furiously final 4832|14730|4731|1|23|37828.79|0.03|0.01|N|O|1997-12-05|1998-01-05|1997-12-10|NONE|RAIL|y express depo 4832|151303|8849|2|10|13543.00|0.00|0.06|N|O|1998-01-08|1998-02-01|1998-01-11|DELIVER IN PERSON|MAIL|ly. blithely bold pinto beans should have 4832|148228|743|3|4|5104.88|0.04|0.01|N|O|1998-01-16|1998-02-12|1998-02-08|TAKE BACK RETURN|AIR|ages. slyly express deposits cajole car 4832|63875|3876|4|6|11033.22|0.02|0.01|N|O|1997-12-08|1998-02-03|1997-12-10|COLLECT COD|TRUCK|ages cajole after the bold requests. furi 4832|137354|4894|5|43|59828.05|0.10|0.08|N|O|1997-12-31|1998-02-20|1998-01-26|COLLECT COD|RAIL|oze according to the accou 4833|106154|8665|1|31|35964.65|0.08|0.04|N|O|1996-06-24|1996-07-15|1996-07-02|NONE|SHIP|ven instructions cajole against the caref 4833|116084|3618|2|11|12100.88|0.03|0.01|N|O|1996-08-24|1996-07-26|1996-09-19|NONE|REG AIR|s nag above the busily sile 4833|17043|7044|3|26|24961.04|0.08|0.04|N|O|1996-05-13|1996-07-12|1996-05-31|NONE|SHIP|s packages. even gif 4833|35345|352|4|19|24326.46|0.07|0.07|N|O|1996-08-21|1996-07-09|1996-09-10|TAKE BACK RETURN|AIR|y quick theodolit 4833|34266|1776|5|4|4801.04|0.10|0.02|N|O|1996-08-16|1996-06-29|1996-08-22|NONE|AIR|y pending packages sleep blithely regular r 4834|182667|222|1|27|47240.82|0.06|0.02|N|O|1997-01-09|1996-10-27|1997-01-27|DELIVER IN PERSON|RAIL|es nag blithe 4834|70960|3468|2|26|50204.96|0.01|0.00|N|O|1996-10-04|1996-10-21|1996-10-10|DELIVER IN PERSON|TRUCK|ages dazzle carefully. slyly daring foxes 4834|22952|7957|3|34|63748.30|0.03|0.01|N|O|1996-12-09|1996-11-26|1996-12-10|NONE|MAIL|ounts haggle bo 4834|142668|5183|4|38|65005.08|0.03|0.06|N|O|1997-01-10|1996-12-06|1997-01-22|COLLECT COD|FOB|alongside of the carefully even plate 4835|178289|8290|1|18|24611.04|0.00|0.03|R|F|1995-02-17|1994-12-14|1995-03-17|DELIVER IN PERSON|MAIL|eat furiously against the slyly 4835|90575|3085|2|3|4696.71|0.09|0.06|R|F|1995-01-24|1995-01-12|1995-02-16|COLLECT COD|AIR|etimes final pac 4835|85483|5484|3|27|39648.96|0.05|0.00|A|F|1994-12-10|1994-12-13|1995-01-02|DELIVER IN PERSON|REG AIR| accounts after the car 4835|101125|6146|4|23|25900.76|0.08|0.07|A|F|1995-02-05|1995-01-04|1995-02-28|NONE|SHIP|e carefully regular foxes. deposits are sly 4836|161531|4048|1|22|35035.66|0.01|0.03|N|O|1997-03-03|1997-02-23|1997-03-04|NONE|SHIP|al pinto beans. care 4836|47331|9836|2|16|20453.28|0.07|0.08|N|O|1997-01-14|1997-03-05|1997-01-30|COLLECT COD|MAIL|gular packages against the express reque 4836|75666|3188|3|14|22983.24|0.03|0.08|N|O|1997-02-21|1997-02-06|1997-03-08|COLLECT COD|MAIL|lites. unusual, bold dolphins ar 4836|105341|362|4|15|20195.10|0.10|0.00|N|O|1997-03-08|1997-03-14|1997-03-30|TAKE BACK RETURN|TRUCK|eep slyly. even requests cajole 4836|50120|5131|5|12|12841.44|0.01|0.04|N|O|1997-02-02|1997-02-10|1997-02-03|COLLECT COD|TRUCK|sly ironic accoun 4837|41768|9281|1|16|27356.16|0.09|0.04|N|O|1998-08-12|1998-06-06|1998-08-26|COLLECT COD|FOB|ing requests are blithely regular instructi 4837|192758|5278|2|16|29612.00|0.01|0.02|N|O|1998-08-19|1998-06-18|1998-08-26|NONE|RAIL|counts cajole slyly furiou 4837|67706|2719|3|42|70295.40|0.10|0.00|N|O|1998-06-19|1998-07-06|1998-06-23|COLLECT COD|MAIL|o the furiously final theodolites boost 4838|121792|1793|1|35|63482.65|0.01|0.00|R|F|1992-10-30|1992-10-23|1992-11-21|TAKE BACK RETURN|RAIL|ly blithely unusual foxes. even package 4838|147002|9517|2|2|2098.00|0.03|0.08|R|F|1992-08-11|1992-09-16|1992-08-26|COLLECT COD|MAIL|hely final notornis are furiously blithe 4838|51200|1201|3|26|29931.20|0.06|0.04|R|F|1992-09-03|1992-10-25|1992-09-11|TAKE BACK RETURN|FOB|ular requests boost about the packages. r 4839|59762|7278|1|5|8608.80|0.10|0.07|A|F|1994-09-07|1994-07-15|1994-10-05|DELIVER IN PERSON|FOB|ses integrate. regular deposits are about 4839|9631|9632|2|25|38515.75|0.02|0.02|R|F|1994-05-20|1994-07-08|1994-05-30|NONE|REG AIR|regular packages ab 4839|59660|9661|3|18|29153.88|0.06|0.01|R|F|1994-05-18|1994-06-13|1994-06-09|TAKE BACK RETURN|FOB|blithely ironic theodolites use along 4839|99142|9143|4|19|21681.66|0.07|0.08|R|F|1994-05-20|1994-07-14|1994-05-30|NONE|REG AIR| deposits sublate furiously ir 4839|70988|6003|5|9|17630.82|0.05|0.01|R|F|1994-06-17|1994-06-18|1994-07-10|NONE|SHIP|ounts haggle carefully above 4864|149726|7269|1|28|49720.16|0.06|0.08|A|F|1993-02-06|1992-12-15|1993-02-10|COLLECT COD|REG AIR|thely around the bli 4864|37481|9985|2|38|53902.24|0.10|0.02|R|F|1992-12-20|1993-01-07|1993-01-06|TAKE BACK RETURN|SHIP|ording to the ironic, ir 4864|132447|7474|3|45|66574.80|0.02|0.01|A|F|1992-11-17|1993-01-02|1992-11-26|COLLECT COD|SHIP|round the furiously careful pa 4864|30832|5839|4|46|81090.18|0.07|0.03|A|F|1993-02-24|1993-01-02|1993-03-17|TAKE BACK RETURN|RAIL|sts use carefully across the carefull 4865|161406|8955|1|16|23478.40|0.07|0.05|N|O|1997-10-02|1997-08-20|1997-10-04|COLLECT COD|TRUCK|osits haggle. fur 4865|136210|6211|2|4|4984.84|0.07|0.01|N|O|1997-07-24|1997-07-25|1997-08-07|TAKE BACK RETURN|FOB|sts. blithely special instruction 4865|67618|5137|3|44|69766.84|0.10|0.08|N|O|1997-07-25|1997-08-20|1997-08-22|COLLECT COD|FOB|even deposits sleep against the quickly r 4865|49947|4956|4|21|39835.74|0.04|0.02|N|O|1997-07-17|1997-08-10|1997-07-21|NONE|RAIL|eposits detect sly 4865|53744|8755|5|33|56025.42|0.00|0.05|N|O|1997-07-17|1997-08-16|1997-07-30|TAKE BACK RETURN|FOB|y pending notornis ab 4865|64770|9783|6|47|81534.19|0.00|0.05|N|O|1997-08-26|1997-08-07|1997-08-31|NONE|RAIL|y unusual packages. packages 4866|10585|5588|1|9|13460.22|0.01|0.05|N|O|1997-08-30|1997-09-18|1997-09-24|TAKE BACK RETURN|MAIL|ven dependencies x-ray. quic 4866|101252|1253|2|1|1253.25|0.06|0.00|N|O|1997-10-15|1997-10-01|1997-11-14|TAKE BACK RETURN|AIR|latelets nag. q 4866|130432|2946|3|17|24861.31|0.07|0.00|N|O|1997-11-26|1997-10-11|1997-12-12|COLLECT COD|TRUCK|ess packages doubt. even somas wake f 4867|81666|1667|1|7|11533.62|0.09|0.03|A|F|1992-07-17|1992-08-17|1992-07-22|COLLECT COD|FOB|e carefully even packages. slyly ironic i 4867|159641|2157|2|3|5101.92|0.04|0.08|R|F|1992-07-04|1992-07-15|1992-07-21|NONE|AIR|yly silent deposits 4868|72576|5084|1|47|72782.79|0.03|0.03|N|O|1997-04-29|1997-04-27|1997-05-11|DELIVER IN PERSON|SHIP|gle unusual, fluffy packages. foxes cajol 4868|179044|9045|2|8|8984.32|0.10|0.08|N|O|1997-03-26|1997-05-09|1997-04-16|NONE|RAIL|ly special th 4868|190723|724|3|49|88872.28|0.09|0.03|N|O|1997-04-23|1997-05-07|1997-04-26|NONE|SHIP|ys engage. th 4868|79046|9047|4|34|34851.36|0.04|0.02|N|O|1997-05-19|1997-04-27|1997-06-15|NONE|RAIL|en instructions about th 4868|121574|1575|5|22|35102.54|0.07|0.06|N|O|1997-04-26|1997-05-16|1997-05-01|DELIVER IN PERSON|FOB|osits. final foxes boost regular, 4869|40808|3313|1|31|54212.80|0.10|0.01|A|F|1995-01-17|1994-11-30|1995-02-02|NONE|SHIP|ins. always unusual ideas across the ir 4869|57367|2378|2|24|31784.64|0.09|0.06|A|F|1994-11-17|1994-11-07|1994-11-27|COLLECT COD|MAIL|olites cajole after the ideas. special t 4869|156025|6026|3|25|27025.50|0.00|0.05|R|F|1994-11-25|1994-11-14|1994-12-19|DELIVER IN PERSON|AIR|e according t 4869|102539|7560|4|24|36996.72|0.10|0.07|R|F|1994-11-23|1994-11-18|1994-12-11|DELIVER IN PERSON|MAIL|se deposits above the sly, q 4869|172852|7887|5|42|80843.70|0.07|0.04|R|F|1994-10-16|1994-12-10|1994-11-07|TAKE BACK RETURN|REG AIR| slyly even instructions. 4869|121042|8579|6|30|31891.20|0.00|0.05|A|F|1995-01-09|1994-11-20|1995-02-02|COLLECT COD|RAIL|gedly even requests. s 4870|47881|386|1|49|89615.12|0.05|0.05|R|F|1994-11-14|1994-10-24|1994-12-12|TAKE BACK RETURN|SHIP| regular packages 4870|126176|3713|2|6|7213.02|0.06|0.08|A|F|1994-09-09|1994-10-16|1994-09-21|DELIVER IN PERSON|TRUCK|ress requests. bold, silent pinto bea 4870|30179|5186|3|5|5545.85|0.05|0.00|R|F|1994-10-11|1994-10-07|1994-10-24|NONE|AIR|s haggle furiously. slyly ironic dinos 4870|5978|8479|4|4|7535.88|0.03|0.08|A|F|1994-10-23|1994-09-16|1994-11-04|COLLECT COD|RAIL|its wake quickly. slyly quick 4870|70598|3106|5|36|56469.24|0.09|0.06|A|F|1994-09-06|1994-09-17|1994-10-01|COLLECT COD|REG AIR| instructions. carefully pending pac 4871|176651|4203|1|14|24187.10|0.07|0.03|N|O|1995-09-30|1995-07-29|1995-10-18|TAKE BACK RETURN|REG AIR|inst the never ironic 4871|160509|8058|2|17|26681.50|0.07|0.03|N|O|1995-09-09|1995-09-01|1995-10-02|DELIVER IN PERSON|AIR|es. carefully ev 4871|62067|2068|3|3|3087.18|0.03|0.06|N|O|1995-10-03|1995-08-10|1995-10-06|DELIVER IN PERSON|TRUCK|y special packages wak 4871|148471|6014|4|35|53181.45|0.08|0.07|N|O|1995-08-11|1995-07-18|1995-08-29|DELIVER IN PERSON|TRUCK|ackages sle 4871|151422|1423|5|10|14734.20|0.09|0.02|N|O|1995-09-12|1995-09-02|1995-10-05|TAKE BACK RETURN|AIR|s integrate after the a 4871|135500|3040|6|36|55278.00|0.02|0.08|N|O|1995-09-18|1995-08-29|1995-10-05|TAKE BACK RETURN|AIR|ely according 4871|139040|6580|7|10|10790.40|0.10|0.02|N|O|1995-07-13|1995-08-19|1995-07-29|NONE|REG AIR|p ironic theodolites. slyly even platel 4896|40766|767|1|19|32428.44|0.09|0.05|A|F|1992-12-13|1992-11-13|1993-01-09|NONE|AIR|nusual requ 4896|139589|4616|2|44|71657.52|0.04|0.03|A|F|1992-11-24|1992-11-15|1992-12-18|COLLECT COD|MAIL|e after the slowly f 4896|57169|4685|3|6|6756.96|0.04|0.04|A|F|1992-10-30|1992-11-12|1992-11-28|DELIVER IN PERSON|TRUCK|usly regular deposits 4896|22746|2747|4|5|8343.70|0.08|0.02|R|F|1992-12-02|1992-11-11|1992-12-19|COLLECT COD|SHIP|eposits hang carefully. sly 4896|85033|2558|5|21|21378.63|0.07|0.08|R|F|1992-11-18|1992-11-18|1992-11-29|DELIVER IN PERSON|TRUCK|ly express deposits. carefully pending depo 4897|54403|4404|1|26|35292.40|0.01|0.01|R|F|1992-12-22|1992-10-25|1992-12-27|DELIVER IN PERSON|TRUCK|. carefully ironic dep 4897|142171|7200|2|34|41247.78|0.02|0.00|R|F|1992-12-31|1992-11-11|1993-01-30|COLLECT COD|AIR|ts. special dependencies use fluffily 4897|54770|2286|3|42|72440.34|0.09|0.03|A|F|1992-09-23|1992-10-28|1992-10-02|DELIVER IN PERSON|FOB|sts. blithely regular deposits will have 4897|103352|3353|4|19|25751.65|0.03|0.00|A|F|1992-11-08|1992-12-14|1992-12-03|DELIVER IN PERSON|FOB|! ironic, pending dependencies doze furiou 4898|71592|6607|1|44|68797.96|0.07|0.02|A|F|1994-09-13|1994-08-18|1994-09-16|NONE|FOB|y regular grouches about 4899|33087|597|1|14|14281.12|0.06|0.00|R|F|1993-11-10|1994-01-10|1993-11-20|NONE|REG AIR| foxes eat 4900|115397|420|1|40|56495.60|0.10|0.03|A|F|1992-09-02|1992-09-25|1992-09-21|COLLECT COD|TRUCK|heodolites. request 4900|76356|6357|2|33|43967.55|0.06|0.06|R|F|1992-08-18|1992-09-20|1992-08-19|COLLECT COD|MAIL|nto beans nag slyly reg 4900|102019|7040|3|48|49008.48|0.02|0.00|R|F|1992-09-18|1992-08-14|1992-09-28|TAKE BACK RETURN|MAIL|uickly ironic ideas kindle s 4900|31089|1090|4|20|20401.60|0.05|0.00|R|F|1992-09-22|1992-09-23|1992-09-27|TAKE BACK RETURN|MAIL|yers. accounts affix somet 4900|104891|7402|5|40|75835.60|0.03|0.02|R|F|1992-07-14|1992-09-05|1992-07-20|NONE|REG AIR|luffily final dol 4900|102405|4916|6|46|64740.40|0.06|0.08|R|F|1992-07-11|1992-09-19|1992-07-16|TAKE BACK RETURN|SHIP|ly final acco 4901|140928|8471|1|37|72850.04|0.00|0.04|N|O|1998-01-26|1998-02-20|1998-01-31|DELIVER IN PERSON|TRUCK| furiously ev 4901|164332|6849|2|12|16755.96|0.00|0.04|N|O|1998-01-12|1998-02-06|1998-02-03|COLLECT COD|REG AIR|y unusual deposits prom 4901|119727|2239|3|16|27947.52|0.05|0.08|N|O|1998-04-19|1998-03-18|1998-04-21|NONE|AIR|deposits. blithely fin 4901|35623|5624|4|41|63903.42|0.03|0.00|N|O|1998-03-18|1998-02-18|1998-04-14|TAKE BACK RETURN|AIR|efully bold packages affix carefully eve 4901|115677|5678|5|40|67706.80|0.06|0.02|N|O|1998-01-08|1998-01-30|1998-01-15|DELIVER IN PERSON|MAIL|ect across the furiou 4902|195198|2756|1|22|28450.18|0.00|0.04|N|O|1998-10-17|1998-08-10|1998-10-21|COLLECT COD|RAIL|r the furiously final fox 4902|82370|7387|2|1|1352.37|0.09|0.04|N|O|1998-10-12|1998-08-20|1998-11-08|NONE|RAIL|daring foxes? even, bold requests wake f 4903|120758|759|1|1|1778.75|0.06|0.03|R|F|1992-04-23|1992-06-13|1992-05-03|NONE|SHIP|nusual requests 4903|164392|4393|2|6|8738.34|0.09|0.07|R|F|1992-04-01|1992-05-16|1992-04-11|DELIVER IN PERSON|SHIP|azzle quickly along the blithely final pla 4903|119755|7289|3|27|47918.25|0.07|0.06|A|F|1992-06-29|1992-06-09|1992-07-08|COLLECT COD|RAIL|pinto beans are; 4928|99357|9358|1|4|5425.40|0.04|0.02|R|F|1993-10-25|1993-12-24|1993-11-16|TAKE BACK RETURN|REG AIR|bout the slyly final accounts. carefull 4928|92887|2888|2|20|37597.60|0.03|0.08|A|F|1994-01-19|1993-11-29|1994-02-13|DELIVER IN PERSON|SHIP|quiet theodolites ca 4928|148249|5792|3|34|44106.16|0.06|0.05|A|F|1993-10-12|1993-12-31|1993-10-14|DELIVER IN PERSON|AIR|, regular depos 4929|13595|8598|1|20|30171.80|0.00|0.04|N|O|1996-03-12|1996-05-23|1996-03-20|COLLECT COD|REG AIR| final pinto beans detect. final, 4929|78044|5566|2|40|40881.60|0.08|0.03|N|O|1996-05-30|1996-04-13|1996-06-22|TAKE BACK RETURN|AIR|unts against 4929|76404|8912|3|32|44172.80|0.08|0.02|N|O|1996-04-28|1996-05-23|1996-04-30|COLLECT COD|TRUCK|usly at the blithely pending pl 4929|108935|3956|4|26|50542.18|0.00|0.05|N|O|1996-06-10|1996-05-29|1996-06-26|DELIVER IN PERSON|RAIL| slyly. fl 4929|66430|6431|5|24|33514.32|0.09|0.05|N|O|1996-04-15|1996-04-30|1996-05-09|NONE|MAIL| accounts boost 4930|186572|6573|1|35|58049.95|0.03|0.01|A|F|1994-07-09|1994-07-30|1994-07-15|NONE|RAIL|lose slyly regular dependencies. fur 4930|114748|2282|2|20|35254.80|0.02|0.04|A|F|1994-08-21|1994-06-17|1994-08-24|COLLECT COD|FOB|he carefully 4930|167762|279|3|28|51233.28|0.00|0.08|R|F|1994-08-27|1994-06-27|1994-09-18|COLLECT COD|TRUCK|e ironic, unusual courts. regula 4930|165538|5539|4|42|67348.26|0.00|0.00|A|F|1994-06-18|1994-06-22|1994-07-10|COLLECT COD|AIR|ions haggle. furiously regular ideas use 4930|189517|7072|5|38|61047.38|0.02|0.03|A|F|1994-06-06|1994-06-18|1994-07-03|TAKE BACK RETURN|AIR|bold requests sleep never 4931|193048|8087|1|1|1141.04|0.08|0.06|A|F|1995-01-24|1994-12-19|1995-02-07|DELIVER IN PERSON|SHIP| furiously 4931|150506|8052|2|8|12452.00|0.06|0.02|R|F|1994-12-15|1995-01-14|1995-01-06|NONE|SHIP|ts boost. packages wake sly 4931|143024|3025|3|20|21340.40|0.09|0.00|A|F|1995-01-25|1994-12-21|1995-02-06|DELIVER IN PERSON|MAIL|the furious 4931|199283|6841|4|50|69114.00|0.04|0.01|A|F|1994-12-15|1994-12-18|1994-12-23|COLLECT COD|REG AIR|s haggle al 4931|149123|1638|5|25|29303.00|0.05|0.05|R|F|1994-12-19|1995-01-05|1994-12-21|COLLECT COD|FOB|aggle bravely according to the quic 4931|102419|4930|6|8|11371.28|0.02|0.03|A|F|1995-02-16|1994-12-30|1995-03-15|DELIVER IN PERSON|SHIP|dependencies are slyly 4932|50248|7764|1|13|15577.12|0.04|0.03|A|F|1993-09-13|1993-10-16|1993-09-20|DELIVER IN PERSON|SHIP|slyly according to the furiously fin 4932|102499|30|2|15|22522.35|0.01|0.02|R|F|1993-11-15|1993-10-25|1993-11-29|NONE|REG AIR|yly. unusu 4932|86827|9336|3|5|9069.10|0.06|0.06|A|F|1993-10-01|1993-09-13|1993-10-04|NONE|MAIL| haggle furiously. slyly ironic packages sl 4932|97299|2318|4|11|14259.19|0.09|0.06|A|F|1993-09-21|1993-09-30|1993-09-23|COLLECT COD|SHIP|as. special depende 4933|31326|8836|1|48|60351.36|0.08|0.00|N|O|1995-10-10|1995-10-03|1995-11-04|COLLECT COD|SHIP|ideas. sly 4933|81917|6934|2|2|3797.82|0.09|0.00|N|O|1995-10-01|1995-09-29|1995-10-19|DELIVER IN PERSON|MAIL|ctions nag final instructions. accou 4934|96074|1093|1|48|51363.36|0.00|0.01|N|O|1997-05-20|1997-04-22|1997-06-02|TAKE BACK RETURN|SHIP| ideas cajol 4934|109380|9381|2|41|56964.58|0.06|0.06|N|O|1997-06-04|1997-04-11|1997-06-25|TAKE BACK RETURN|FOB|wake final, ironic f 4934|139725|9726|3|8|14117.76|0.03|0.06|N|O|1997-05-20|1997-04-30|1997-05-27|TAKE BACK RETURN|MAIL|arefully express pains cajo 4934|147233|9748|4|9|11522.07|0.06|0.08|N|O|1997-06-10|1997-04-09|1997-06-12|TAKE BACK RETURN|REG AIR| haggle alongside of the 4934|137089|7090|5|29|32656.32|0.09|0.03|N|O|1997-04-10|1997-05-05|1997-05-04|DELIVER IN PERSON|AIR|aggle furiously among the busily final re 4934|51634|1635|6|42|66596.46|0.00|0.07|N|O|1997-03-19|1997-05-05|1997-03-25|NONE|MAIL|ven, ironic ideas 4934|10672|3174|7|2|3165.34|0.10|0.06|N|O|1997-06-05|1997-03-26|1997-06-09|COLLECT COD|MAIL|ongside of the brave, regula 4935|160733|734|1|13|23318.49|0.09|0.01|A|F|1993-06-20|1993-08-13|1993-06-27|COLLECT COD|REG AIR|ly requests. final deposits might 4935|39163|6673|2|37|40779.92|0.01|0.05|R|F|1993-08-30|1993-07-23|1993-09-07|TAKE BACK RETURN|RAIL|y even dependencies nag a 4935|10868|5871|3|24|42692.64|0.06|0.04|A|F|1993-05-29|1993-08-17|1993-06-22|NONE|RAIL|ly quickly s 4935|44935|4936|4|49|92116.57|0.06|0.01|A|F|1993-09-16|1993-08-21|1993-10-12|COLLECT COD|TRUCK|ffily after the furiou 4935|9508|9509|5|14|19845.00|0.08|0.08|A|F|1993-05-30|1993-07-25|1993-05-31|COLLECT COD|FOB|slowly. blith 4935|187915|2952|6|36|72104.76|0.10|0.00|R|F|1993-07-11|1993-07-04|1993-08-01|DELIVER IN PERSON|RAIL|requests across the quick 4960|17091|2094|1|36|36291.24|0.01|0.05|R|F|1995-03-06|1995-05-04|1995-04-05|TAKE BACK RETURN|RAIL|c, unusual accou 4960|44360|9369|2|6|7826.16|0.03|0.08|R|F|1995-03-21|1995-05-13|1995-04-14|TAKE BACK RETURN|SHIP|ual package 4960|148041|5584|3|9|9801.36|0.01|0.03|A|F|1995-03-20|1995-05-05|1995-04-17|COLLECT COD|RAIL|e blithely carefully fina 4960|119052|4075|4|14|14994.70|0.00|0.06|A|F|1995-04-03|1995-04-17|1995-04-07|NONE|RAIL|accounts. warhorses are. grouches 4960|97171|2190|5|8|9345.36|0.07|0.04|R|F|1995-03-14|1995-04-18|1995-04-09|NONE|FOB|as. busily regular packages nag. 4960|145105|5106|6|37|42553.70|0.10|0.04|R|F|1995-05-23|1995-04-12|1995-06-01|DELIVER IN PERSON|MAIL|ending theodolites w 4960|169635|9636|7|42|71594.46|0.08|0.07|A|F|1995-04-19|1995-04-11|1995-05-08|NONE|SHIP|s requests cajole. 4961|43748|8757|1|38|64286.12|0.10|0.07|N|O|1998-07-09|1998-06-03|1998-07-11|TAKE BACK RETURN|FOB|e on the blithely bold accounts. unu 4961|59997|5008|2|1|1956.99|0.08|0.08|N|O|1998-07-08|1998-05-25|1998-07-12|DELIVER IN PERSON|MAIL|s affix carefully silent dependen 4961|161177|1178|3|41|50764.97|0.02|0.02|N|O|1998-07-15|1998-06-15|1998-08-05|TAKE BACK RETURN|REG AIR|ily against the n 4961|99354|4373|4|10|13533.50|0.02|0.04|N|O|1998-04-15|1998-07-03|1998-04-18|DELIVER IN PERSON|MAIL|quests. regular, ironic ideas at the ironi 4962|18377|3380|1|46|59587.02|0.01|0.07|R|F|1993-08-23|1993-09-04|1993-08-27|COLLECT COD|REG AIR| pinto beans grow about the sl 4963|167840|2873|1|38|72497.92|0.08|0.02|N|O|1996-12-25|1996-12-12|1997-01-02|COLLECT COD|AIR|tegrate daringly accou 4963|75326|2848|2|16|20821.12|0.00|0.03|N|O|1996-11-20|1997-01-13|1996-12-06|COLLECT COD|MAIL| carefully slyly u 4964|132406|4920|1|29|41713.60|0.04|0.01|N|O|1997-10-18|1997-08-30|1997-11-01|NONE|AIR|k accounts nag carefully-- ironic, fin 4964|147067|9582|2|46|51246.76|0.06|0.06|N|O|1997-10-05|1997-09-12|1997-10-11|NONE|TRUCK|althy deposits 4964|142559|2560|3|18|28827.90|0.00|0.06|N|O|1997-10-13|1997-09-01|1997-11-10|DELIVER IN PERSON|AIR| platelets. furio 4964|179773|2291|4|12|22233.24|0.08|0.01|N|O|1997-09-03|1997-10-25|1997-09-15|NONE|TRUCK|ully silent instructions ca 4964|40697|8210|5|42|68782.98|0.06|0.04|N|O|1997-09-04|1997-08-28|1997-10-02|TAKE BACK RETURN|AIR| hinder. idly even 4964|192728|286|6|22|40055.84|0.04|0.08|N|O|1997-09-11|1997-10-06|1997-09-29|NONE|AIR|equests doubt quickly. caref 4964|172770|2771|7|28|51597.56|0.00|0.05|N|O|1997-08-30|1997-09-15|1997-09-18|COLLECT COD|RAIL|among the carefully regula 4965|130042|5069|1|28|30017.12|0.05|0.03|A|F|1994-01-02|1993-11-20|1994-01-04|TAKE BACK RETURN|REG AIR| deposits. requests sublate quickly 4965|12001|7004|2|25|22825.00|0.10|0.02|R|F|1994-02-05|1993-12-15|1994-02-24|TAKE BACK RETURN|MAIL|wake at the carefully speci 4965|100514|8045|3|27|40891.77|0.05|0.06|R|F|1993-11-06|1993-12-24|1993-11-30|TAKE BACK RETURN|SHIP|efully final foxes 4965|137469|2496|4|33|49713.18|0.04|0.04|A|F|1993-12-31|1993-11-29|1994-01-27|DELIVER IN PERSON|REG AIR|iously slyly 4966|75766|8274|1|10|17417.60|0.06|0.03|N|O|1996-09-23|1996-11-02|1996-10-07|TAKE BACK RETURN|SHIP| requests. carefully pending requests 4966|193834|6354|2|6|11566.98|0.02|0.01|N|O|1996-12-09|1996-11-29|1996-12-30|NONE|AIR|d deposits are sly excuses. slyly iro 4966|164992|4993|3|7|14398.93|0.00|0.01|N|O|1996-12-08|1996-10-09|1997-01-06|COLLECT COD|MAIL|ckly ironic tithe 4966|15871|3375|4|26|46458.62|0.08|0.03|N|O|1996-11-14|1996-11-29|1996-12-05|COLLECT COD|REG AIR|nt pearls haggle carefully slyly even 4966|143660|6175|5|12|20443.92|0.02|0.07|N|O|1996-12-07|1996-11-23|1996-12-20|DELIVER IN PERSON|RAIL|eodolites. ironic requests across the exp 4967|70796|3304|1|50|88339.50|0.07|0.01|N|O|1997-05-27|1997-05-13|1997-06-12|NONE|REG AIR|kages. final, unusual accounts c 4967|52398|9914|2|43|58066.77|0.00|0.07|N|O|1997-05-28|1997-04-10|1997-06-09|NONE|TRUCK|ons. slyly ironic requests 4967|49975|9976|3|15|28874.55|0.08|0.02|N|O|1997-04-16|1997-04-12|1997-05-08|TAKE BACK RETURN|MAIL|y. blithel 4967|122478|7503|4|1|1500.47|0.10|0.07|N|O|1997-06-04|1997-03-29|1997-06-23|NONE|FOB|osits. unusual frets thrash furiously 4992|183528|3529|1|42|67683.84|0.07|0.01|R|F|1992-07-19|1992-06-16|1992-08-17|TAKE BACK RETURN|RAIL|foxes about the quickly final platele 4992|146436|8951|2|47|69674.21|0.10|0.08|A|F|1992-09-04|1992-08-05|1992-09-21|COLLECT COD|MAIL|atterns use fluffily. 4992|143147|8176|3|17|20232.38|0.03|0.03|A|F|1992-07-05|1992-07-19|1992-07-30|TAKE BACK RETURN|FOB|s along the perma 4992|69920|4933|4|25|47248.00|0.04|0.06|R|F|1992-08-06|1992-07-11|1992-08-20|NONE|SHIP|ly about the never ironic requests. pe 4992|138500|6040|5|23|35385.50|0.01|0.08|R|F|1992-06-28|1992-07-15|1992-07-12|DELIVER IN PERSON|MAIL|uickly regul 4992|162601|150|6|44|73198.40|0.05|0.02|A|F|1992-06-01|1992-07-22|1992-06-03|NONE|RAIL|rmanent, sly packages print slyly. regula 4993|37231|4741|1|34|39719.82|0.05|0.00|R|F|1994-09-21|1994-10-31|1994-09-24|TAKE BACK RETURN|REG AIR|ular, pending packages at the even packa 4993|128552|1065|2|39|61641.45|0.03|0.08|R|F|1994-09-10|1994-09-04|1994-09-26|COLLECT COD|SHIP|pending, regular requests solve caref 4993|165619|3168|3|42|70753.62|0.06|0.00|A|F|1994-08-27|1994-09-24|1994-09-05|NONE|MAIL| final packages at the q 4993|157398|9914|4|31|45117.09|0.10|0.06|A|F|1994-10-02|1994-10-29|1994-10-15|NONE|AIR|nwind thinly platelets. a 4994|155833|3379|1|36|67997.88|0.00|0.06|N|O|1996-09-29|1996-07-30|1996-10-03|TAKE BACK RETURN|TRUCK|ess ideas. blithely silent brai 4994|79276|4291|2|47|58997.69|0.04|0.05|N|O|1996-09-20|1996-08-04|1996-10-15|COLLECT COD|TRUCK|sts. blithely close ideas sleep quic 4994|182034|4553|3|29|32364.87|0.08|0.01|N|O|1996-08-26|1996-09-27|1996-09-25|DELIVER IN PERSON|RAIL|ptotes boost carefully 4994|38154|8155|4|40|43686.00|0.01|0.06|N|O|1996-08-25|1996-08-16|1996-09-07|TAKE BACK RETURN|REG AIR|eposits. regula 4994|41357|3862|5|24|31160.40|0.01|0.07|N|O|1996-08-19|1996-09-24|1996-08-25|TAKE BACK RETURN|FOB|s. slyly ironic deposits cajole f 4994|72561|2562|6|6|9201.36|0.01|0.02|N|O|1996-09-05|1996-08-04|1996-09-30|TAKE BACK RETURN|FOB|grate carefully around th 4994|129748|9749|7|31|55109.94|0.07|0.04|N|O|1996-10-14|1996-09-23|1996-11-08|TAKE BACK RETURN|RAIL|lar decoys cajole fluffil 4995|64943|7450|1|16|30527.04|0.02|0.05|N|O|1996-02-27|1996-04-03|1996-02-29|DELIVER IN PERSON|MAIL|egular, bold packages. accou 4995|80476|8001|2|43|62628.21|0.00|0.06|N|O|1996-02-24|1996-02-20|1996-03-07|NONE|AIR|ts. blithely silent ideas after t 4995|155462|5463|3|22|33384.12|0.03|0.06|N|O|1996-03-17|1996-03-12|1996-04-01|DELIVER IN PERSON|MAIL|s wake furious, express dependencies. 4995|39933|4940|4|9|16856.37|0.07|0.07|N|O|1996-03-07|1996-03-17|1996-03-11|DELIVER IN PERSON|FOB| ironic packages cajole across t 4995|147648|5191|5|48|81390.72|0.08|0.07|N|O|1996-03-22|1996-04-01|1996-04-07|NONE|SHIP|t blithely. requests affix blithely. 4995|109830|4851|6|48|88311.84|0.09|0.07|N|O|1996-04-14|1996-04-04|1996-05-07|DELIVER IN PERSON|RAIL|nstructions. carefully final depos 4996|55891|902|1|35|64641.15|0.07|0.01|A|F|1992-10-30|1992-10-27|1992-11-05|TAKE BACK RETURN|SHIP|s. unusual, regular dolphins integrate care 4996|155848|5849|2|39|74249.76|0.02|0.07|A|F|1992-09-19|1992-10-19|1992-10-06|COLLECT COD|FOB|equests are carefully final 4996|127777|2802|3|12|21657.24|0.04|0.06|R|F|1993-01-09|1992-11-22|1993-02-04|DELIVER IN PERSON|SHIP|usly bold requests sleep dogge 4996|143576|1119|4|13|21054.41|0.00|0.00|A|F|1992-09-17|1992-12-02|1992-10-07|DELIVER IN PERSON|TRUCK|o beans use about the furious 4997|78040|5562|1|44|44793.76|0.02|0.05|N|O|1998-06-09|1998-06-12|1998-07-07|NONE|RAIL|r escapades ca 4997|16188|3692|2|5|5520.90|0.02|0.04|N|O|1998-05-16|1998-06-05|1998-06-07|COLLECT COD|REG AIR|cuses are furiously unusual asymptotes 4997|57193|7194|3|24|27604.56|0.04|0.06|N|O|1998-04-20|1998-04-23|1998-05-16|NONE|AIR|xpress, bo 4997|39081|1585|4|5|5100.40|0.10|0.03|N|O|1998-06-12|1998-04-24|1998-06-13|DELIVER IN PERSON|TRUCK|aggle slyly alongside of the slyly i 4997|21826|4329|5|46|80399.72|0.00|0.04|N|O|1998-04-28|1998-06-04|1998-05-08|TAKE BACK RETURN|SHIP|ecial courts are carefully 4997|28912|6419|6|2|3681.82|0.07|0.01|N|O|1998-07-09|1998-06-10|1998-07-21|TAKE BACK RETURN|REG AIR|counts. slyl 4998|153471|5987|1|12|18293.64|0.04|0.03|A|F|1992-02-20|1992-03-06|1992-03-01|TAKE BACK RETURN|RAIL| sleep slyly furiously final accounts. ins 4998|182630|7667|2|15|25689.45|0.06|0.00|R|F|1992-04-24|1992-03-21|1992-05-02|NONE|REG AIR|heodolites sleep quickly. 4998|58992|8993|3|27|52676.73|0.06|0.02|R|F|1992-03-17|1992-02-26|1992-04-05|DELIVER IN PERSON|MAIL|the blithely ironic 4998|62376|7389|4|47|62903.39|0.10|0.04|A|F|1992-02-07|1992-03-07|1992-02-19|DELIVER IN PERSON|TRUCK|mong the careful 4998|144589|2132|5|24|39205.92|0.01|0.04|R|F|1992-01-25|1992-03-16|1992-01-27|COLLECT COD|REG AIR| unwind about 4998|98492|1002|6|8|11923.92|0.03|0.07|A|F|1992-05-01|1992-03-03|1992-05-24|TAKE BACK RETURN|AIR|ions nag quickly according to the theodolit 4999|152687|7718|1|30|52190.40|0.00|0.02|A|F|1993-08-20|1993-08-15|1993-08-30|NONE|AIR|ades cajole carefully unusual ide 4999|9204|9205|2|44|48980.80|0.03|0.01|A|F|1993-08-01|1993-08-04|1993-08-17|COLLECT COD|REG AIR|ependencies. slowly regu 4999|85996|5997|3|30|59459.70|0.09|0.01|R|F|1993-07-21|1993-08-11|1993-08-20|DELIVER IN PERSON|RAIL|s cajole among the blithel 5024|165411|444|1|17|25098.97|0.10|0.02|N|O|1996-11-24|1997-01-10|1996-12-04|NONE|AIR| to the expre 5024|57578|84|2|41|62958.37|0.06|0.01|N|O|1996-11-09|1996-12-03|1996-12-01|COLLECT COD|REG AIR|osits hinder carefully 5024|111009|3521|3|18|18360.00|0.04|0.03|N|O|1996-12-02|1997-01-16|1996-12-05|NONE|MAIL|zle carefully sauternes. quickly 5024|122508|5021|4|42|64281.00|0.03|0.06|N|O|1996-12-02|1996-12-08|1996-12-04|DELIVER IN PERSON|RAIL|tegrate. busily spec 5025|29421|4426|1|11|14854.62|0.00|0.04|N|O|1997-02-21|1997-04-16|1997-03-14|COLLECT COD|SHIP|the carefully final esc 5025|77470|2485|2|10|14474.70|0.07|0.04|N|O|1997-06-04|1997-04-29|1997-06-28|COLLECT COD|RAIL|lly silent deposits boost busily again 5026|95360|7870|1|13|17619.68|0.02|0.04|N|O|1997-12-23|1997-11-02|1998-01-03|TAKE BACK RETURN|SHIP|endencies sleep carefully alongs 5027|97261|4789|1|6|7549.56|0.04|0.05|N|O|1997-09-28|1997-11-24|1997-10-25|NONE|FOB|ar, ironic deposi 5027|61744|1745|2|39|66523.86|0.06|0.01|N|O|1997-09-09|1997-11-13|1997-09-21|TAKE BACK RETURN|FOB|ess requests! quickly regular pac 5027|125140|165|3|32|37284.48|0.00|0.01|N|O|1997-11-13|1997-10-29|1997-11-18|TAKE BACK RETURN|RAIL|cording to 5027|25594|5595|4|37|56224.83|0.02|0.00|N|O|1997-10-05|1997-10-30|1997-10-26|NONE|REG AIR|ost slyly fluffily 5027|142732|2733|5|3|5324.19|0.03|0.06|N|O|1997-09-30|1997-11-26|1997-10-05|DELIVER IN PERSON|AIR|t the even mu 5027|86493|9002|6|25|36987.25|0.06|0.00|N|O|1997-09-16|1997-11-25|1997-10-08|TAKE BACK RETURN|RAIL|ic ideas. requests sleep fluffily am 5027|80444|2953|7|50|71222.00|0.07|0.02|N|O|1997-09-18|1997-11-07|1997-10-05|DELIVER IN PERSON|MAIL| beans dazzle according to the fluffi 5028|13876|8879|1|15|26848.05|0.07|0.07|R|F|1992-07-17|1992-07-16|1992-08-05|COLLECT COD|REG AIR|es are quickly final pains. furiously pend 5028|198686|8687|2|15|26770.20|0.03|0.07|R|F|1992-08-02|1992-07-09|1992-08-30|NONE|REG AIR|gular, bold pinto bea 5029|153597|3598|1|17|28060.03|0.02|0.01|A|F|1993-03-12|1992-12-18|1993-04-02|DELIVER IN PERSON|FOB|! packages boost blithely. furious 5029|96432|8942|2|2|2856.86|0.00|0.04|A|F|1992-11-25|1993-01-04|1992-12-20|DELIVER IN PERSON|MAIL|packages. furiously ironi 5030|101356|1357|1|22|29861.70|0.04|0.06|N|O|1998-09-01|1998-08-15|1998-09-30|TAKE BACK RETURN|TRUCK|. quickly regular foxes believe 5030|79762|4777|2|50|87088.00|0.05|0.06|N|O|1998-08-22|1998-07-25|1998-09-18|TAKE BACK RETURN|FOB|ss excuses serve bli 5031|49942|9943|1|15|28379.10|0.02|0.05|R|F|1995-04-01|1995-02-24|1995-04-12|DELIVER IN PERSON|AIR|yly pending theodolites. 5031|160895|8444|2|40|78235.60|0.10|0.04|A|F|1994-12-04|1995-01-27|1995-01-01|NONE|TRUCK|ns hang blithely across th 5031|153067|613|3|4|4480.24|0.01|0.07|R|F|1994-12-26|1995-02-24|1995-01-11|NONE|RAIL|after the even frays: ironic, unusual th 5031|180435|2954|4|31|46978.33|0.10|0.08|R|F|1995-01-15|1995-01-08|1995-02-09|COLLECT COD|MAIL|ts across the even requests doze furiously 5056|47302|4815|1|7|8745.10|0.09|0.01|N|O|1997-04-28|1997-04-07|1997-05-15|DELIVER IN PERSON|TRUCK|rouches after the pending instruc 5056|196388|3946|2|19|28203.22|0.04|0.00|N|O|1997-03-24|1997-05-05|1997-04-23|DELIVER IN PERSON|AIR|c theodolites. ironic a 5056|89040|4057|3|23|23667.92|0.02|0.05|N|O|1997-05-12|1997-04-28|1997-05-25|NONE|SHIP|ickly regular requests cajole. depos 5056|86915|1932|4|14|26626.74|0.08|0.00|N|O|1997-06-09|1997-04-13|1997-07-06|COLLECT COD|SHIP|sts haggle carefully along the slyl 5057|36914|4424|1|38|70334.58|0.02|0.03|N|O|1997-10-24|1997-09-07|1997-10-30|TAKE BACK RETURN|MAIL|packages. stealthily bold wa 5057|7083|9584|2|45|44553.60|0.08|0.07|N|O|1997-09-20|1997-10-02|1997-10-20|NONE|FOB| asymptotes wake slyl 5058|192440|4960|1|16|24519.04|0.09|0.07|N|O|1998-07-12|1998-06-09|1998-07-15|DELIVER IN PERSON|SHIP| the special foxes 5059|69538|7057|1|5|7537.65|0.03|0.08|R|F|1993-12-23|1994-01-12|1993-12-24|TAKE BACK RETURN|FOB|ts affix slyly accordi 5059|122882|7907|2|19|36192.72|0.06|0.04|R|F|1994-03-02|1993-12-26|1994-03-14|TAKE BACK RETURN|MAIL| special ideas poach blithely qu 5059|76305|8813|3|45|57658.50|0.02|0.00|A|F|1994-01-28|1994-01-08|1994-02-18|DELIVER IN PERSON|MAIL|enly. requests doze. express, close pa 5060|24932|2439|1|27|50137.11|0.10|0.07|R|F|1992-07-23|1992-09-05|1992-08-07|COLLECT COD|SHIP|s. ironic 5060|31033|3537|2|28|26992.84|0.04|0.04|R|F|1992-09-25|1992-08-11|1992-10-09|NONE|REG AIR|c requests 5060|160257|258|3|15|19758.75|0.06|0.01|A|F|1992-08-28|1992-08-20|1992-09-01|DELIVER IN PERSON|AIR|ular deposits sl 5061|164896|9929|1|18|35296.02|0.03|0.00|A|F|1993-10-20|1993-10-05|1993-10-28|TAKE BACK RETURN|SHIP|atelets among the ca 5061|197491|2530|2|8|12707.92|0.01|0.02|R|F|1993-09-07|1993-10-31|1993-10-04|DELIVER IN PERSON|REG AIR|regular foxes. ir 5061|23216|3217|3|26|29619.46|0.02|0.05|A|F|1993-11-07|1993-09-13|1993-11-13|NONE|REG AIR| cajole slyly. carefully spe 5062|100784|3295|1|9|16063.02|0.08|0.00|R|F|1993-01-02|1992-12-01|1993-01-20|TAKE BACK RETURN|MAIL| silent theodolites wake. c 5062|74681|4682|2|4|6622.72|0.02|0.02|R|F|1993-02-06|1992-12-14|1993-03-03|DELIVER IN PERSON|AIR|ke furiously express theodolites. 5062|158504|8505|3|50|78125.00|0.09|0.07|A|F|1992-12-25|1992-12-13|1992-12-29|TAKE BACK RETURN|MAIL| the regular, unusual pains. specia 5062|160148|2665|4|18|21746.52|0.03|0.07|R|F|1992-11-04|1992-12-25|1992-11-05|NONE|SHIP|furiously pending requests are ruthles 5062|193632|1190|5|25|43140.75|0.08|0.02|R|F|1992-12-15|1992-11-17|1993-01-01|NONE|TRUCK|uthless excuses ag 5063|128529|8530|1|31|48283.12|0.08|0.01|N|O|1997-06-02|1997-06-20|1997-06-27|NONE|RAIL|kages. ironic, ironic courts wake. carefu 5063|173640|1192|2|43|73686.52|0.04|0.08|N|O|1997-09-14|1997-07-05|1997-10-05|TAKE BACK RETURN|TRUCK|latelets might nod blithely regular requ 5063|166201|1234|3|2|2534.40|0.02|0.03|N|O|1997-06-17|1997-07-27|1997-06-24|COLLECT COD|SHIP|kly regular i 5063|134525|9552|4|18|28071.36|0.08|0.05|N|O|1997-06-02|1997-06-18|1997-06-06|TAKE BACK RETURN|RAIL|refully quiet reques 5063|160636|5669|5|1|1696.63|0.06|0.07|N|O|1997-09-03|1997-06-26|1997-10-03|NONE|FOB|ously special 5088|77900|5422|1|23|43191.70|0.06|0.06|R|F|1993-03-03|1993-03-07|1993-03-08|NONE|FOB|cording to the fluffily expr 5088|50009|7525|2|41|39319.00|0.09|0.00|R|F|1993-01-22|1993-03-07|1993-02-09|TAKE BACK RETURN|TRUCK|ing requests. 5088|85245|2770|3|36|44288.64|0.10|0.05|A|F|1993-04-16|1993-04-03|1993-05-14|NONE|TRUCK|the furiously final deposits. furiously re 5088|108479|6010|4|10|14874.70|0.04|0.05|R|F|1993-04-07|1993-02-06|1993-04-26|NONE|FOB|beans. special requests af 5089|157064|9580|1|4|4484.24|0.05|0.06|R|F|1992-09-18|1992-09-28|1992-10-13|DELIVER IN PERSON|TRUCK|nts sleep blithely 5089|161548|1549|2|20|32190.80|0.00|0.07|R|F|1992-10-10|1992-10-07|1992-11-06|COLLECT COD|RAIL| ironic accounts 5089|123570|1107|3|46|73304.22|0.03|0.04|A|F|1992-11-09|1992-10-13|1992-11-10|TAKE BACK RETURN|RAIL|above the express accounts. exc 5089|33436|5940|4|38|52038.34|0.05|0.03|R|F|1992-11-23|1992-09-11|1992-12-22|TAKE BACK RETURN|SHIP|regular instructions are 5090|21309|1310|1|22|27066.60|0.07|0.00|N|O|1997-05-10|1997-05-25|1997-05-24|TAKE BACK RETURN|TRUCK|ets integrate ironic, regul 5090|128542|8543|2|46|72244.84|0.05|0.00|N|O|1997-04-05|1997-04-14|1997-05-01|COLLECT COD|REG AIR|lose theodolites sleep blit 5090|1804|9305|3|22|37527.60|0.09|0.05|N|O|1997-07-03|1997-04-12|1997-07-26|NONE|REG AIR|ular requests su 5090|113646|6158|4|2|3319.28|0.03|0.06|N|O|1997-04-07|1997-04-23|1997-05-01|TAKE BACK RETURN|AIR|tes. slowly iro 5090|47604|7605|5|21|32583.60|0.10|0.02|N|O|1997-03-29|1997-04-24|1997-04-25|TAKE BACK RETURN|FOB|ly express accounts. slyly even r 5090|79662|4677|6|30|49249.80|0.02|0.03|N|O|1997-05-04|1997-04-14|1997-05-30|COLLECT COD|MAIL|osits nag slyly. fluffily ex 5091|77312|4834|1|50|64465.50|0.05|0.03|N|O|1998-07-21|1998-06-22|1998-07-26|COLLECT COD|REG AIR|al dependencies. r 5092|163090|8123|1|30|34592.70|0.06|0.00|N|O|1995-12-27|1995-12-08|1996-01-09|DELIVER IN PERSON|MAIL|ss, ironic deposits. furiously stea 5092|44716|2229|2|34|56464.14|0.04|0.02|N|O|1995-12-09|1995-12-26|1995-12-21|TAKE BACK RETURN|AIR|ckages nag 5092|139286|6826|3|13|17228.64|0.06|0.01|N|O|1995-11-21|1996-01-05|1995-12-19|TAKE BACK RETURN|SHIP|es detect sly 5092|179246|9247|4|14|18553.36|0.04|0.00|N|O|1996-02-20|1995-11-30|1996-03-20|DELIVER IN PERSON|REG AIR| deposits cajole furiously against the sly 5092|185660|8179|5|42|73317.72|0.01|0.02|N|O|1995-11-06|1996-01-01|1995-12-06|DELIVER IN PERSON|AIR|s use along t 5092|177555|5107|6|11|17958.05|0.03|0.03|N|O|1995-12-02|1995-12-27|1995-12-11|COLLECT COD|MAIL|ly against the slyly silen 5092|158325|8326|7|50|69166.00|0.10|0.03|N|O|1995-11-30|1996-01-14|1995-12-19|NONE|REG AIR|r platelets maintain car 5093|167649|7650|1|40|68665.60|0.05|0.01|R|F|1993-09-16|1993-11-04|1993-10-05|TAKE BACK RETURN|REG AIR|ing pinto beans. quickly bold dependenci 5093|73419|941|2|15|20886.15|0.01|0.04|A|F|1993-12-02|1993-11-18|1994-01-01|DELIVER IN PERSON|FOB|ly among the unusual foxe 5093|150780|3296|3|31|56754.18|0.00|0.02|R|F|1993-09-22|1993-11-14|1993-09-26|TAKE BACK RETURN|REG AIR| against the 5093|155669|700|4|37|63812.42|0.04|0.00|A|F|1993-10-26|1993-12-02|1993-10-27|NONE|TRUCK|courts. qui 5093|114865|9888|5|30|56395.80|0.06|0.05|A|F|1993-11-22|1993-11-27|1993-12-14|DELIVER IN PERSON|TRUCK|ithely ironic sheaves use fluff 5093|120197|2710|6|31|37732.89|0.01|0.08|A|F|1993-12-17|1993-11-14|1994-01-02|NONE|SHIP|he final foxes. fluffily ironic 5094|142519|5034|1|19|29668.69|0.03|0.03|R|F|1993-03-31|1993-06-12|1993-04-04|NONE|AIR|ronic foxes. furi 5094|107604|5135|2|23|37066.80|0.05|0.07|R|F|1993-06-13|1993-05-19|1993-07-06|NONE|MAIL|st furiously above the fluffily care 5094|91455|8983|3|11|15910.95|0.04|0.08|A|F|1993-06-25|1993-06-24|1993-07-18|TAKE BACK RETURN|MAIL|s cajole quickly against the furiously ex 5094|78510|8511|4|21|31258.71|0.09|0.08|R|F|1993-07-26|1993-05-03|1993-08-16|NONE|MAIL| blithely furiously final re 5095|64242|1761|1|46|55487.04|0.07|0.01|A|F|1992-06-26|1992-06-25|1992-07-05|COLLECT COD|RAIL|egular instruction 5095|105488|3019|2|2|2986.96|0.07|0.08|A|F|1992-07-09|1992-05-25|1992-07-21|DELIVER IN PERSON|REG AIR|detect car 5095|122758|5271|3|28|49861.00|0.01|0.04|A|F|1992-06-20|1992-06-27|1992-06-22|DELIVER IN PERSON|AIR| into the final courts. ca 5095|177984|3019|4|42|86603.16|0.08|0.08|R|F|1992-05-23|1992-06-01|1992-06-18|COLLECT COD|TRUCK|ccounts. packages could have t 5095|165425|5426|5|9|13413.78|0.10|0.07|R|F|1992-08-14|1992-06-23|1992-08-16|TAKE BACK RETURN|REG AIR|bold theodolites wake about the expr 5095|96768|6769|6|15|26471.40|0.01|0.06|A|F|1992-07-11|1992-07-12|1992-08-09|COLLECT COD|AIR| to the packages wake sly 5095|168890|8891|7|40|78355.60|0.05|0.02|A|F|1992-07-11|1992-06-07|1992-07-26|DELIVER IN PERSON|MAIL|carefully unusual plat 5120|132365|7392|1|28|39126.08|0.06|0.03|N|O|1996-07-20|1996-08-31|1996-08-06|NONE|RAIL| across the silent requests. caref 5121|183562|1117|1|23|37847.88|0.06|0.01|A|F|1992-05-18|1992-06-20|1992-06-02|TAKE BACK RETURN|REG AIR|even courts are blithely ironically 5121|110722|8256|2|45|77972.40|0.08|0.04|A|F|1992-08-13|1992-07-27|1992-09-12|NONE|TRUCK|pecial accounts cajole ca 5121|96595|1614|3|27|42972.93|0.08|0.07|R|F|1992-06-17|1992-06-11|1992-06-19|NONE|MAIL|ly silent theodolit 5121|67014|9521|4|10|9810.10|0.04|0.05|R|F|1992-06-08|1992-07-10|1992-07-02|TAKE BACK RETURN|FOB|e quickly according 5121|88202|8203|5|46|54749.20|0.03|0.02|R|F|1992-05-27|1992-07-19|1992-05-28|TAKE BACK RETURN|FOB|use express foxes. slyly 5121|79|7580|6|2|1958.14|0.04|0.07|R|F|1992-08-10|1992-06-28|1992-08-11|NONE|FOB| final, regular account 5122|182287|9842|1|28|38339.84|0.03|0.00|N|O|1996-04-20|1996-03-29|1996-04-29|DELIVER IN PERSON|FOB|g the busily ironic accounts boos 5122|81957|9482|2|43|83374.85|0.09|0.03|N|O|1996-05-31|1996-04-12|1996-06-13|NONE|MAIL|ut the carefully special foxes. idle, 5122|44058|4059|3|12|12024.60|0.07|0.03|N|O|1996-04-02|1996-04-27|1996-04-10|DELIVER IN PERSON|AIR|lar instructions 5123|25745|5746|1|13|21719.62|0.08|0.07|N|O|1998-05-17|1998-03-23|1998-06-02|COLLECT COD|MAIL|regular pearls 5124|54193|1709|1|43|49329.17|0.00|0.02|N|O|1997-07-10|1997-05-13|1997-07-31|COLLECT COD|AIR|onic package 5124|5101|2602|2|41|41250.10|0.02|0.06|N|O|1997-07-05|1997-06-29|1997-07-23|DELIVER IN PERSON|RAIL|wake across the 5124|124798|4799|3|44|80202.76|0.03|0.03|N|O|1997-07-13|1997-06-26|1997-08-01|DELIVER IN PERSON|RAIL|equests. carefully unusual d 5124|69465|1972|4|36|51640.56|0.10|0.07|N|O|1997-04-20|1997-07-03|1997-05-04|TAKE BACK RETURN|AIR|r deposits ab 5125|5424|7925|1|38|50517.96|0.09|0.05|N|O|1998-03-20|1998-04-14|1998-03-22|COLLECT COD|MAIL|ily even deposits w 5125|159077|9078|2|5|5680.35|0.08|0.06|N|O|1998-04-07|1998-04-14|1998-04-29|COLLECT COD|RAIL| thinly even pack 5126|23568|8573|1|33|49221.48|0.02|0.02|R|F|1993-02-04|1992-12-23|1993-02-14|NONE|RAIL|ipliers promise furiously whithout the 5126|100832|5853|2|43|78811.69|0.09|0.04|R|F|1993-01-07|1992-12-19|1993-01-16|COLLECT COD|MAIL|e silently. ironic, unusual accounts 5126|77521|29|3|23|34465.96|0.08|0.01|R|F|1993-01-02|1993-01-02|1993-01-05|COLLECT COD|TRUCK|egular, blithe packages. 5127|18637|1139|1|33|51335.79|0.08|0.04|N|O|1997-03-25|1997-03-02|1997-04-04|NONE|SHIP| bold deposits use carefully a 5127|31122|3626|2|20|21062.40|0.01|0.03|N|O|1997-05-11|1997-02-26|1997-05-12|TAKE BACK RETURN|SHIP|dolites about the final platelets w 5152|104917|2448|1|9|17297.19|0.04|0.03|N|O|1997-04-11|1997-02-11|1997-04-18|COLLECT COD|AIR| cajole furiously alongside of the bo 5152|133398|5912|2|50|71569.50|0.04|0.04|N|O|1997-03-10|1997-02-04|1997-03-15|COLLECT COD|FOB| the final deposits. slyly ironic warth 5153|34465|6969|1|42|58777.32|0.03|0.01|N|O|1995-10-03|1995-11-09|1995-10-11|COLLECT COD|RAIL|re thinly. ironic 5153|52804|320|2|14|24595.20|0.05|0.05|N|O|1995-11-29|1995-10-21|1995-12-08|TAKE BACK RETURN|TRUCK| slyly daring pinto beans lose blithely fi 5153|67863|370|3|30|54925.80|0.09|0.01|N|O|1995-11-10|1995-11-14|1995-11-16|DELIVER IN PERSON|AIR|beans sleep bl 5153|172162|7197|4|32|39493.12|0.10|0.08|N|O|1995-12-05|1995-09-25|1996-01-03|DELIVER IN PERSON|MAIL|egular deposits. ironi 5153|111676|9210|5|36|60756.12|0.01|0.03|N|O|1995-12-15|1995-11-08|1995-12-30|COLLECT COD|TRUCK| ironic instru 5153|135055|2595|6|42|45782.10|0.00|0.03|N|O|1995-10-19|1995-11-23|1995-11-06|TAKE BACK RETURN|RAIL|ickly even deposi 5154|189775|2294|1|11|20512.47|0.02|0.05|N|O|1997-08-06|1997-06-30|1997-09-04|NONE|RAIL|luffily bold foxes. final 5154|143948|3949|2|15|29879.10|0.07|0.08|N|O|1997-06-23|1997-07-11|1997-07-11|NONE|AIR|even packages. packages use 5155|47932|7933|1|1|1879.93|0.00|0.00|A|F|1994-07-03|1994-08-11|1994-07-29|COLLECT COD|FOB|oze slyly after the silent, regular idea 5155|187548|2585|2|5|8177.70|0.08|0.02|A|F|1994-06-30|1994-08-13|1994-07-15|TAKE BACK RETURN|AIR|ole blithely slyly ironic 5155|105449|2980|3|28|40724.32|0.05|0.02|R|F|1994-07-01|1994-07-19|1994-07-18|COLLECT COD|REG AIR|s cajole. accounts wake. thinly quiet pla 5155|78860|6382|4|39|71715.54|0.09|0.06|A|F|1994-08-25|1994-09-01|1994-09-18|COLLECT COD|TRUCK|l dolphins nag caref 5156|116520|1543|1|21|32266.92|0.06|0.03|N|O|1997-01-01|1997-01-30|1997-01-11|TAKE BACK RETURN|TRUCK|ts detect against the furiously reg 5156|147020|2049|2|36|38412.72|0.04|0.07|N|O|1997-02-12|1996-12-10|1997-03-13|TAKE BACK RETURN|REG AIR| slyly even orbi 5157|54165|1681|1|35|39170.60|0.06|0.08|N|O|1997-07-28|1997-09-30|1997-08-15|TAKE BACK RETURN|REG AIR|to the furiously sil 5157|137834|7835|2|18|33692.94|0.10|0.04|N|O|1997-09-06|1997-10-03|1997-09-19|COLLECT COD|MAIL|y bold deposits nag blithely. final reque 5157|166387|6388|3|15|21800.70|0.09|0.00|N|O|1997-07-27|1997-08-30|1997-08-08|DELIVER IN PERSON|RAIL|cajole. spec 5157|58188|694|4|25|28654.50|0.00|0.03|N|O|1997-08-24|1997-09-23|1997-08-28|COLLECT COD|REG AIR| packages detect. even requests against th 5157|148947|6490|5|40|79837.60|0.09|0.06|N|O|1997-08-11|1997-08-28|1997-09-01|TAKE BACK RETURN|FOB|ial packages according to 5157|149156|6699|6|26|31333.90|0.10|0.01|N|O|1997-07-28|1997-08-22|1997-08-22|NONE|FOB|nto beans cajole car 5157|48745|6258|7|12|20324.88|0.10|0.08|N|O|1997-10-19|1997-08-07|1997-10-26|NONE|FOB|es. busily 5158|44249|1762|1|43|51309.32|0.10|0.04|N|O|1997-04-10|1997-03-06|1997-04-15|DELIVER IN PERSON|AIR|nusual platelets. slyly even foxes cajole 5158|84248|4249|2|18|22180.32|0.04|0.04|N|O|1997-04-30|1997-03-28|1997-05-12|COLLECT COD|REG AIR|hely regular pa 5158|141878|4393|3|41|78714.67|0.05|0.05|N|O|1997-02-25|1997-03-19|1997-03-03|COLLECT COD|AIR|deposits. quickly special 5158|130049|7589|4|49|52872.96|0.05|0.01|N|O|1997-04-10|1997-03-21|1997-04-30|NONE|REG AIR|r requests sleep q 5158|118468|6002|5|20|29729.20|0.01|0.04|N|O|1997-02-03|1997-02-20|1997-02-08|TAKE BACK RETURN|AIR|latelets use accordin 5158|87969|478|6|39|76321.44|0.08|0.04|N|O|1997-05-15|1997-04-04|1997-06-02|DELIVER IN PERSON|FOB|lithely fina 5158|90030|7558|7|38|38761.14|0.10|0.05|N|O|1997-05-09|1997-03-03|1997-06-04|NONE|SHIP|uffily regular ac 5159|123367|904|1|39|54224.04|0.06|0.07|N|O|1996-12-17|1996-12-08|1997-01-10|COLLECT COD|MAIL|re furiously after the pending dolphin 5159|16681|9183|2|46|73493.28|0.01|0.01|N|O|1996-12-15|1996-12-07|1996-12-30|DELIVER IN PERSON|SHIP|s kindle slyly carefully regular 5159|151238|8784|3|22|28363.06|0.01|0.02|N|O|1996-11-06|1996-11-04|1996-11-15|TAKE BACK RETURN|SHIP|he furiously sile 5159|51907|1908|4|5|9294.50|0.10|0.00|N|O|1996-11-25|1996-12-19|1996-12-25|TAKE BACK RETURN|FOB|nal deposits. pending, ironic ideas grow 5159|197897|417|5|36|71816.04|0.06|0.01|N|O|1997-01-24|1996-11-07|1997-02-08|NONE|REG AIR|packages wake. 5184|152521|7552|1|33|51926.16|0.07|0.04|N|O|1998-08-17|1998-10-16|1998-08-24|TAKE BACK RETURN|AIR|posits. carefully express asympto 5184|15749|3253|2|47|78242.78|0.05|0.01|N|O|1998-11-02|1998-08-19|1998-11-07|COLLECT COD|TRUCK|se. carefully express pinto beans x 5184|87427|9936|3|39|55162.38|0.03|0.06|N|O|1998-10-27|1998-10-17|1998-11-19|DELIVER IN PERSON|FOB|es above the care 5184|175958|5959|4|26|52882.70|0.05|0.08|N|O|1998-11-11|1998-08-26|1998-12-01|TAKE BACK RETURN|TRUCK| packages are 5184|123097|5610|5|19|21281.71|0.06|0.03|N|O|1998-11-15|1998-10-12|1998-11-21|COLLECT COD|REG AIR|refully express platelets sleep carefull 5184|79525|4540|6|49|73721.48|0.02|0.00|N|O|1998-09-18|1998-08-28|1998-10-14|COLLECT COD|FOB|thlessly closely even reque 5185|196408|3966|1|37|55662.80|0.00|0.04|N|O|1997-08-08|1997-09-08|1997-08-14|COLLECT COD|SHIP|gainst the courts dazzle care 5185|24597|2104|2|32|48690.88|0.06|0.00|N|O|1997-08-17|1997-09-30|1997-08-24|TAKE BACK RETURN|REG AIR|ackages. slyly even requests 5185|195889|928|3|41|81380.08|0.00|0.05|N|O|1997-10-15|1997-10-11|1997-11-02|COLLECT COD|REG AIR|ly blithe deposits. furi 5185|95730|5731|4|30|51771.90|0.09|0.04|N|O|1997-10-17|1997-09-16|1997-10-23|TAKE BACK RETURN|SHIP|ress packages are furiously 5185|127706|7707|5|8|13869.60|0.04|0.00|N|O|1997-08-30|1997-09-02|1997-09-22|COLLECT COD|REG AIR|sts around the slyly perma 5185|145806|835|6|50|92590.00|0.03|0.04|N|O|1997-10-15|1997-10-19|1997-11-06|TAKE BACK RETURN|FOB|final platelets. ideas sleep careful 5186|54412|9423|1|38|51923.58|0.06|0.02|N|O|1996-11-23|1996-09-21|1996-12-11|DELIVER IN PERSON|MAIL|y ruthless foxes. fluffily 5186|90920|921|2|31|59238.52|0.09|0.03|N|O|1996-10-19|1996-09-26|1996-10-25|TAKE BACK RETURN|REG AIR| accounts use furiously slyly spe 5186|88279|3296|3|26|32949.02|0.03|0.02|N|O|1996-08-08|1996-10-05|1996-08-21|DELIVER IN PERSON|FOB|capades. accounts sublate. pinto 5186|89226|9227|4|8|9721.76|0.10|0.05|N|O|1996-09-23|1996-09-29|1996-09-30|COLLECT COD|RAIL|y regular notornis k 5186|17403|9905|5|28|36971.20|0.09|0.03|N|O|1996-10-05|1996-10-27|1996-10-19|TAKE BACK RETURN|RAIL|al decoys. blit 5186|81990|4499|6|35|69019.65|0.00|0.05|N|O|1996-10-20|1996-10-12|1996-11-12|TAKE BACK RETURN|RAIL|sly silent pack 5186|197847|367|7|44|85572.96|0.00|0.08|N|O|1996-09-23|1996-10-14|1996-10-01|NONE|TRUCK|old, final accounts cajole sl 5187|10977|8481|1|49|92510.53|0.04|0.06|N|O|1997-10-20|1997-10-12|1997-10-26|DELIVER IN PERSON|AIR|l, regular platelets instead of the foxes w 5187|82848|2849|2|1|1830.84|0.10|0.08|N|O|1997-08-08|1997-08-24|1997-08-22|DELIVER IN PERSON|REG AIR|aggle never bold 5188|117534|46|1|18|27927.54|0.04|0.03|N|O|1995-06-19|1995-05-19|1995-06-25|DELIVER IN PERSON|AIR|p according to the sometimes regu 5188|193702|1260|2|36|64645.20|0.04|0.02|A|F|1995-03-09|1995-05-16|1995-03-19|NONE|TRUCK|packages? blithely s 5188|147932|2961|3|9|17819.37|0.06|0.08|A|F|1995-05-09|1995-05-22|1995-05-19|TAKE BACK RETURN|REG AIR|r attainments are across the 5189|137325|2352|1|44|59942.08|0.02|0.06|A|F|1994-01-13|1994-02-07|1994-01-21|DELIVER IN PERSON|MAIL|y finally pendin 5189|15881|884|2|38|68281.44|0.06|0.00|A|F|1994-03-26|1994-01-28|1994-04-20|DELIVER IN PERSON|REG AIR|ideas. idle, final deposits de 5189|109889|4910|3|4|7595.52|0.09|0.02|A|F|1993-12-21|1994-02-23|1994-01-09|DELIVER IN PERSON|REG AIR|. blithely exp 5189|93195|8214|4|49|58221.31|0.05|0.01|R|F|1994-01-22|1994-01-19|1994-02-04|TAKE BACK RETURN|SHIP| requests 5189|122046|7071|5|14|14952.56|0.02|0.03|A|F|1994-01-23|1994-01-05|1994-02-12|DELIVER IN PERSON|REG AIR|unusual packag 5189|16138|6139|6|41|43219.33|0.02|0.06|R|F|1993-12-12|1994-02-05|1994-01-09|DELIVER IN PERSON|RAIL|ial theodolites cajole slyly. slyly unus 5190|55509|520|1|43|62973.50|0.09|0.06|A|F|1992-08-19|1992-06-10|1992-09-01|DELIVER IN PERSON|FOB|encies use fluffily unusual requests? hoc 5190|131308|6335|2|6|8035.80|0.10|0.08|A|F|1992-08-08|1992-07-14|1992-08-22|COLLECT COD|RAIL|furiously regular pinto beans. furiously i 5190|88403|3420|3|45|62613.00|0.04|0.03|A|F|1992-07-23|1992-06-16|1992-08-04|NONE|FOB|y carefully final ideas. f 5191|114713|4714|1|41|70836.11|0.00|0.08|A|F|1995-02-05|1995-02-27|1995-02-15|DELIVER IN PERSON|AIR|uests! ironic theodolites cajole care 5191|167719|236|2|40|71468.40|0.02|0.01|A|F|1995-03-31|1995-02-21|1995-04-02|NONE|AIR|nes haggle sometimes. requests eng 5191|42244|2245|3|27|32028.48|0.07|0.05|A|F|1994-12-26|1995-01-24|1995-01-14|DELIVER IN PERSON|RAIL|tructions nag bravely within the re 5191|182828|383|4|7|13375.74|0.01|0.04|A|F|1995-03-24|1995-01-30|1995-03-30|NONE|RAIL|eposits. express 5216|68007|8008|1|17|16575.00|0.04|0.06|N|O|1997-08-20|1997-11-07|1997-09-14|COLLECT COD|FOB|s according to the accounts bo 5217|79528|9529|1|50|75376.00|0.05|0.02|N|O|1995-12-26|1995-11-21|1996-01-24|DELIVER IN PERSON|MAIL|s. express, express accounts c 5217|15345|5346|2|23|28987.82|0.06|0.07|N|O|1996-01-18|1995-12-24|1996-02-10|COLLECT COD|RAIL|ven ideas. requests amo 5217|101296|6317|3|23|29837.67|0.03|0.02|N|O|1995-11-15|1995-12-17|1995-11-27|DELIVER IN PERSON|FOB|pending packages cajole ne 5217|80311|2820|4|47|60691.57|0.04|0.00|N|O|1995-11-24|1995-12-25|1995-11-25|COLLECT COD|AIR|ronic packages i 5218|82747|272|1|43|74378.82|0.05|0.04|A|F|1992-08-04|1992-09-12|1992-08-17|DELIVER IN PERSON|SHIP|k theodolites. express, even id 5218|124729|9754|2|33|57872.76|0.06|0.01|R|F|1992-09-16|1992-09-30|1992-09-27|NONE|TRUCK|ronic instructi 5219|134892|4893|1|2|3853.78|0.08|0.00|N|O|1997-06-26|1997-04-29|1997-07-08|TAKE BACK RETURN|FOB| blithely according to the stea 5219|118635|6169|2|20|33072.60|0.05|0.00|N|O|1997-04-20|1997-05-26|1997-05-13|COLLECT COD|FOB|e along the ironic, 5220|82721|7738|1|27|46000.44|0.10|0.04|R|F|1992-09-21|1992-08-29|1992-10-16|DELIVER IN PERSON|RAIL|s cajole blithely furiously iron 5221|103446|8467|1|24|34786.56|0.07|0.03|N|O|1995-10-04|1995-08-11|1995-10-30|COLLECT COD|REG AIR|s pinto beans sleep. sly 5221|8710|8711|2|34|55036.14|0.01|0.05|N|O|1995-09-11|1995-07-17|1995-10-10|COLLECT COD|SHIP|eans. furio 5221|179197|1715|3|16|20419.04|0.04|0.01|N|O|1995-08-29|1995-09-06|1995-09-12|TAKE BACK RETURN|TRUCK|ending request 5222|150471|8017|1|1|1521.47|0.00|0.00|A|F|1994-08-19|1994-07-16|1994-09-08|TAKE BACK RETURN|FOB|idle requests. carefully pending pinto bean 5223|44573|2086|1|24|36421.68|0.00|0.00|A|F|1994-10-03|1994-09-20|1994-10-11|TAKE BACK RETURN|TRUCK|refully bold courts besides the regular, 5223|123366|5879|2|25|34734.00|0.09|0.02|R|F|1994-07-12|1994-08-13|1994-08-01|NONE|FOB|y express ideas impress 5223|5732|3233|3|19|31116.87|0.04|0.01|R|F|1994-10-28|1994-08-26|1994-10-31|COLLECT COD|REG AIR|ntly. furiously even excuses a 5223|129873|4898|4|40|76114.80|0.01|0.04|R|F|1994-10-01|1994-09-18|1994-10-28|COLLECT COD|SHIP|kly pending 5248|80249|7774|1|39|47940.36|0.05|0.03|N|O|1995-08-10|1995-07-04|1995-09-09|TAKE BACK RETURN|MAIL|yly even accounts. spe 5248|137103|2130|2|45|51304.50|0.00|0.06|A|F|1995-05-09|1995-07-12|1995-05-27|DELIVER IN PERSON|FOB|. bold, pending foxes h 5249|49720|7233|1|31|51761.32|0.07|0.03|A|F|1994-11-21|1994-11-19|1994-12-08|NONE|REG AIR|f the excuses. furiously fin 5249|30026|2530|2|44|42064.88|0.05|0.00|A|F|1994-12-28|1994-11-29|1994-12-29|TAKE BACK RETURN|MAIL|ole furiousl 5249|31770|4274|3|13|22123.01|0.09|0.00|R|F|1994-09-27|1994-10-20|1994-10-05|DELIVER IN PERSON|SHIP|ites. finally exp 5249|145318|7833|4|29|39535.99|0.00|0.05|A|F|1994-09-16|1994-11-03|1994-10-06|NONE|TRUCK| players. f 5249|157742|258|5|12|21596.88|0.01|0.08|R|F|1994-12-28|1994-11-07|1995-01-15|COLLECT COD|MAIL|press depths could have to sleep carefu 5250|43529|1042|1|2|2945.04|0.08|0.04|N|O|1995-08-09|1995-10-10|1995-08-13|COLLECT COD|AIR|its. final pinto 5250|191534|9092|2|27|43889.31|0.10|0.05|N|O|1995-10-24|1995-09-03|1995-11-18|COLLECT COD|TRUCK|l forges are. furiously unusual pin 5251|138965|3992|1|36|72142.56|0.10|0.01|N|O|1995-07-16|1995-07-05|1995-07-28|DELIVER IN PERSON|FOB|slowly! bli 5252|140986|8529|1|13|26350.74|0.02|0.01|N|O|1996-03-02|1996-05-10|1996-03-11|NONE|FOB|boost fluffily across 5252|138978|1492|2|39|78661.83|0.06|0.05|N|O|1996-05-17|1996-04-23|1996-05-23|COLLECT COD|AIR|gular requests. 5252|194020|1578|3|9|10026.18|0.09|0.03|N|O|1996-05-30|1996-05-03|1996-06-26|TAKE BACK RETURN|RAIL|x. slyly special depos 5252|86576|1593|4|48|75003.36|0.01|0.06|N|O|1996-04-17|1996-03-19|1996-05-03|COLLECT COD|AIR|bold requests. furious 5252|67746|2759|5|24|41129.76|0.04|0.05|N|O|1996-05-11|1996-04-17|1996-05-12|COLLECT COD|REG AIR|posits after the fluffi 5252|2202|9703|6|41|45272.20|0.02|0.03|N|O|1996-03-16|1996-04-18|1996-03-17|NONE|TRUCK|ording to the blithely express somas sho 5253|30616|5623|1|35|54131.35|0.02|0.00|N|O|1995-07-23|1995-06-12|1995-08-03|DELIVER IN PERSON|AIR|ven deposits. careful 5253|149475|1990|2|38|57929.86|0.02|0.06|N|O|1995-08-03|1995-06-14|1995-08-27|DELIVER IN PERSON|REG AIR|onic dependencies are furiou 5253|13981|3982|3|9|17054.82|0.03|0.08|N|F|1995-06-08|1995-05-12|1995-06-23|DELIVER IN PERSON|REG AIR|lyly express deposits use furiou 5253|165076|2625|4|25|28526.75|0.04|0.03|A|F|1995-05-21|1995-06-13|1995-06-09|COLLECT COD|TRUCK|urts. even theodoli 5254|110141|142|1|35|40289.90|0.01|0.07|A|F|1992-07-28|1992-09-05|1992-08-07|COLLECT COD|REG AIR|ntegrate carefully among the pending 5254|134265|9292|2|10|12992.60|0.05|0.04|A|F|1992-11-19|1992-10-20|1992-12-15|COLLECT COD|SHIP| accounts. silent deposit 5254|191752|6791|3|32|59000.00|0.00|0.08|A|F|1992-08-10|1992-09-21|1992-08-16|NONE|RAIL|ts impress closely furi 5254|162208|4725|4|45|57159.00|0.05|0.06|A|F|1992-11-11|1992-09-01|1992-12-07|COLLECT COD|REG AIR| wake. blithely silent excuse 5254|28653|3658|5|23|36377.95|0.02|0.06|A|F|1992-08-16|1992-09-05|1992-09-15|COLLECT COD|RAIL|lyly regular accounts. furiously pendin 5254|157626|2657|6|34|57243.08|0.09|0.02|R|F|1992-08-29|1992-10-16|1992-09-20|TAKE BACK RETURN|RAIL| furiously above the furiously 5254|19583|4586|7|9|13523.22|0.09|0.03|R|F|1992-07-29|1992-10-15|1992-08-20|TAKE BACK RETURN|REG AIR| wake blithely fluff 5255|130706|8246|1|2|3473.40|0.04|0.08|N|O|1996-09-27|1996-10-04|1996-10-04|DELIVER IN PERSON|RAIL|ajole blithely fluf 5255|171139|8691|2|30|36303.90|0.04|0.08|N|O|1996-09-20|1996-08-18|1996-10-09|TAKE BACK RETURN|AIR| to the silent requests cajole b 5255|129900|7437|3|41|79125.90|0.09|0.03|N|O|1996-08-21|1996-09-24|1996-09-05|COLLECT COD|FOB|tect blithely against t 5280|96078|8588|1|16|17185.12|0.02|0.03|N|O|1998-03-29|1998-01-28|1998-04-03|TAKE BACK RETURN|SHIP| foxes are furiously. theodoli 5280|175550|585|2|46|74775.30|0.01|0.06|N|O|1998-01-04|1998-01-21|1998-02-03|TAKE BACK RETURN|FOB|efully carefully pen 5281|113555|8578|1|37|58036.35|0.05|0.02|N|O|1995-11-10|1996-01-31|1995-11-22|DELIVER IN PERSON|MAIL|ronic dependencies. fluffily final p 5281|104572|2103|2|38|59909.66|0.00|0.05|N|O|1996-02-17|1995-12-19|1996-02-29|NONE|RAIL|n asymptotes could wake about th 5281|126120|8633|3|23|26360.76|0.08|0.00|N|O|1995-12-30|1996-01-26|1996-01-23|COLLECT COD|REG AIR|. final theodolites cajole. ironic p 5281|86431|1448|4|48|68036.64|0.03|0.05|N|O|1996-01-31|1995-12-23|1996-02-08|TAKE BACK RETURN|REG AIR|ss the furiously 5281|42872|5377|5|33|59890.71|0.01|0.07|N|O|1996-03-01|1995-12-28|1996-03-05|COLLECT COD|RAIL|ly brave foxes. bold deposits above the 5282|117371|9883|1|36|49981.32|0.05|0.02|N|O|1998-05-20|1998-04-10|1998-06-14|DELIVER IN PERSON|TRUCK|re slyly accor 5282|51792|4298|2|32|55801.28|0.02|0.05|N|O|1998-03-01|1998-03-31|1998-03-03|NONE|FOB|onic deposits; furiou 5282|57236|4752|3|28|33410.44|0.03|0.06|N|O|1998-05-06|1998-04-24|1998-05-30|COLLECT COD|SHIP|fily final instruc 5283|4165|1666|1|20|21383.20|0.05|0.02|A|F|1994-09-16|1994-08-03|1994-10-15|TAKE BACK RETURN|TRUCK|al deposits? blithely even pinto beans 5283|185925|8444|2|1|2010.92|0.10|0.08|R|F|1994-06-20|1994-08-03|1994-07-01|COLLECT COD|FOB|deposits within the furio 5284|172419|9971|1|16|23862.56|0.04|0.02|N|O|1995-08-17|1995-08-23|1995-08-26|DELIVER IN PERSON|TRUCK|unts detect furiously even d 5284|43086|8095|2|24|24697.92|0.03|0.08|N|O|1995-10-21|1995-08-23|1995-10-27|COLLECT COD|AIR| haggle according 5285|192958|5478|1|31|63579.45|0.08|0.00|A|F|1994-04-17|1994-04-05|1994-05-09|NONE|RAIL|ubt. quickly blithe 5285|30061|5068|2|37|36669.22|0.09|0.02|R|F|1994-02-26|1994-02-18|1994-03-27|NONE|SHIP|uffily regu 5285|33846|1356|3|24|42716.16|0.02|0.04|A|F|1994-04-19|1994-04-03|1994-04-25|DELIVER IN PERSON|FOB|ess packages. quick, even deposits snooze b 5285|42130|9643|4|12|12865.56|0.05|0.06|A|F|1994-04-22|1994-04-07|1994-05-19|NONE|AIR| deposits-- quickly bold requests hag 5285|70694|695|5|1|1664.69|0.03|0.05|R|F|1994-03-14|1994-02-20|1994-04-10|COLLECT COD|TRUCK|e fluffily about the slyly special pa 5285|145672|5673|6|1|1717.67|0.06|0.01|R|F|1994-02-08|1994-04-02|1994-02-17|COLLECT COD|SHIP|ing deposits integra 5286|198566|1086|1|1|1664.56|0.01|0.07|N|O|1997-11-25|1997-11-07|1997-12-17|COLLECT COD|REG AIR|ly! furiously final pack 5286|96223|3751|2|7|8534.54|0.06|0.05|N|O|1997-10-23|1997-12-10|1997-11-20|TAKE BACK RETURN|RAIL|y express instructions sleep carefull 5286|15473|7975|3|3|4165.41|0.06|0.08|N|O|1997-12-04|1997-11-06|1997-12-09|COLLECT COD|MAIL|re fluffily 5286|39019|1523|4|6|5748.06|0.04|0.03|N|O|1997-10-15|1997-12-05|1997-11-12|COLLECT COD|RAIL|y special a 5286|185301|5302|5|38|52679.40|0.07|0.05|N|O|1997-11-29|1997-11-26|1997-12-15|TAKE BACK RETURN|SHIP|fluffily. special, ironic deposit 5286|137943|7944|6|24|47542.56|0.08|0.00|N|O|1997-09-27|1997-12-21|1997-09-30|COLLECT COD|TRUCK|s. express foxes of the 5287|38269|3276|1|32|38632.32|0.01|0.01|A|F|1994-01-29|1994-01-27|1994-02-08|NONE|RAIL|heodolites haggle caref 5312|60844|8363|1|27|48730.68|0.04|0.08|A|F|1995-04-20|1995-04-09|1995-04-25|COLLECT COD|SHIP|tructions cajol 5312|1004|3505|2|43|38915.00|0.05|0.08|A|F|1995-03-24|1995-05-07|1995-03-28|NONE|TRUCK|ly unusual 5313|16090|8592|1|34|34207.06|0.10|0.02|N|O|1997-08-07|1997-08-12|1997-08-24|DELIVER IN PERSON|FOB|ccording to the blithely final account 5313|12109|7112|2|17|17358.70|0.00|0.02|N|O|1997-09-02|1997-08-20|1997-09-07|NONE|SHIP|uests wake 5313|111704|6727|3|47|80637.90|0.06|0.08|N|O|1997-08-12|1997-08-18|1997-08-13|TAKE BACK RETURN|RAIL|pinto beans across the 5313|196554|4112|4|16|26408.80|0.08|0.00|N|O|1997-10-04|1997-08-02|1997-10-25|COLLECT COD|REG AIR|ckages wake carefully aga 5313|71942|6957|5|30|57418.20|0.06|0.08|N|O|1997-06-27|1997-07-18|1997-06-30|NONE|SHIP|nding packages use 5313|119930|4953|6|21|40948.53|0.05|0.05|N|O|1997-09-26|1997-09-02|1997-10-18|COLLECT COD|FOB|he blithely regular packages. quickly 5314|117209|7210|1|10|12262.00|0.07|0.05|N|O|1995-09-26|1995-07-24|1995-10-19|DELIVER IN PERSON|RAIL|latelets haggle final 5314|124781|4782|2|16|28892.48|0.00|0.04|N|O|1995-09-25|1995-07-08|1995-10-17|COLLECT COD|SHIP|hely unusual packages acc 5315|34544|2054|1|12|17742.48|0.08|0.06|R|F|1992-12-18|1993-01-16|1993-01-10|NONE|AIR|ccounts. furiously ironi 5315|178618|8619|2|39|66167.79|0.00|0.06|R|F|1992-11-09|1992-12-29|1992-12-07|NONE|SHIP|ly alongside of the ca 5316|107714|225|1|29|49929.59|0.10|0.05|R|F|1994-03-28|1994-04-29|1994-04-09|DELIVER IN PERSON|REG AIR|ckly unusual foxes bo 5316|135125|5126|2|31|35963.72|0.00|0.08|A|F|1994-04-01|1994-04-21|1994-04-12|DELIVER IN PERSON|MAIL|s. deposits cajole around t 5317|81001|8526|1|29|28478.00|0.02|0.06|A|F|1994-11-28|1994-11-27|1994-12-16|COLLECT COD|FOB|oss the carefull 5317|170294|295|2|18|24557.22|0.06|0.06|A|F|1995-01-02|1994-10-29|1995-01-16|NONE|RAIL|g to the blithely p 5317|119181|1693|3|37|44406.66|0.09|0.00|R|F|1994-09-15|1994-10-24|1994-09-23|NONE|TRUCK|totes nag theodolites. pend 5317|66431|8938|4|50|69871.50|0.09|0.01|A|F|1994-10-17|1994-10-25|1994-11-03|NONE|REG AIR|cajole furiously. accounts use quick 5317|94765|9784|5|19|33435.44|0.07|0.07|R|F|1994-12-15|1994-10-18|1994-12-27|NONE|MAIL|onic requests boost bli 5317|114725|7237|6|48|83506.56|0.01|0.03|A|F|1994-09-19|1994-11-25|1994-10-03|COLLECT COD|MAIL|ts about the packages cajole furio 5317|168493|6042|7|30|46844.70|0.07|0.07|A|F|1994-10-13|1994-10-31|1994-10-28|NONE|AIR|cross the attainments. slyly 5318|60046|7565|1|13|13078.52|0.10|0.04|R|F|1993-07-15|1993-06-25|1993-08-13|COLLECT COD|REG AIR|ly silent ideas. ideas haggle among the 5318|179521|9522|2|26|41613.52|0.00|0.04|R|F|1993-07-07|1993-05-23|1993-07-28|COLLECT COD|SHIP|al, express foxes. bold requests sleep alwa 5318|6173|8674|3|37|39929.29|0.07|0.05|A|F|1993-07-09|1993-06-22|1993-07-21|COLLECT COD|SHIP|ickly final deposi 5318|141228|6257|4|31|39345.82|0.01|0.04|R|F|1993-07-28|1993-05-06|1993-08-06|DELIVER IN PERSON|REG AIR|requests must sleep slyly quickly 5319|149703|7246|1|31|54333.70|0.04|0.07|N|O|1996-03-26|1996-03-07|1996-04-24|COLLECT COD|TRUCK|d carefully about the courts. fluffily spe 5319|43282|795|2|39|47785.92|0.09|0.05|N|O|1996-05-17|1996-03-14|1996-06-11|NONE|TRUCK|unts. furiously silent 5344|18176|678|1|6|6565.02|0.07|0.01|N|O|1998-08-04|1998-09-03|1998-08-11|TAKE BACK RETURN|REG AIR|ithely about the pending plate 5344|78150|658|2|37|41741.55|0.03|0.07|N|O|1998-10-09|1998-07-26|1998-11-08|NONE|TRUCK|thely express packages 5344|66249|6250|3|26|31596.24|0.02|0.06|N|O|1998-08-27|1998-08-22|1998-09-24|NONE|AIR|furiously pending, silent multipliers. 5344|38164|3171|4|21|23145.36|0.03|0.01|N|O|1998-08-31|1998-09-06|1998-09-02|NONE|MAIL|xes. furiously even pinto beans sleep f 5345|82522|5031|1|3|4513.56|0.05|0.01|N|O|1997-12-10|1997-10-03|1998-01-05|COLLECT COD|SHIP|ites wake carefully unusual 5345|145618|3161|2|2|3327.22|0.10|0.02|N|O|1997-11-18|1997-10-12|1997-12-08|NONE|MAIL|ut the slyly specia 5345|191094|6133|3|46|54514.14|0.06|0.04|N|O|1997-10-06|1997-09-27|1997-10-18|COLLECT COD|REG AIR|slyly special deposits. fin 5345|113634|1168|4|37|60962.31|0.01|0.01|N|O|1997-11-01|1997-10-09|1997-11-26|DELIVER IN PERSON|AIR| along the ironically fina 5345|33600|6104|5|22|33739.20|0.02|0.02|N|O|1997-08-27|1997-11-22|1997-09-10|TAKE BACK RETURN|MAIL|leep slyly regular fox 5346|148105|5648|1|21|24215.10|0.07|0.08|R|F|1994-03-11|1994-03-07|1994-04-04|DELIVER IN PERSON|RAIL|integrate blithely a 5346|191665|6704|2|13|22836.58|0.04|0.04|A|F|1994-02-03|1994-02-05|1994-02-09|COLLECT COD|TRUCK|y. fluffily bold accounts grow. furio 5346|108881|1392|3|7|13229.16|0.08|0.05|A|F|1994-01-30|1994-03-26|1994-01-31|DELIVER IN PERSON|SHIP|equests use carefully care 5346|161238|1239|4|35|45473.05|0.06|0.02|A|F|1994-02-09|1994-03-01|1994-02-14|TAKE BACK RETURN|FOB|nic excuses cajole entic 5346|120595|596|5|25|40389.75|0.05|0.06|R|F|1993-12-28|1994-03-19|1994-01-09|TAKE BACK RETURN|REG AIR|he ironic ideas are boldly slyly ironi 5346|32956|466|6|6|11333.70|0.08|0.04|R|F|1994-03-01|1994-02-04|1994-03-09|NONE|REG AIR|escapades sleep furiously beside the 5346|79234|4249|7|41|49742.43|0.05|0.04|R|F|1994-01-10|1994-02-15|1994-01-26|TAKE BACK RETURN|REG AIR|fully close instructi 5347|82283|7300|1|48|60733.44|0.04|0.08|A|F|1995-02-25|1995-04-26|1995-03-26|NONE|SHIP|equests are slyly. blithely regu 5347|123439|8464|2|47|68734.21|0.02|0.01|N|F|1995-06-05|1995-03-29|1995-06-28|COLLECT COD|AIR|across the slyly bol 5347|22862|7867|3|34|60685.24|0.06|0.00|A|F|1995-05-18|1995-04-04|1995-06-02|DELIVER IN PERSON|SHIP| pending deposits. fluffily regular senti 5347|39456|9457|4|4|5581.80|0.06|0.03|A|F|1995-03-24|1995-04-03|1995-04-01|NONE|SHIP|ldly pending asymptotes ki 5347|130711|5738|5|21|36575.91|0.08|0.04|R|F|1995-04-01|1995-04-16|1995-04-23|NONE|SHIP|sly slyly final requests. careful 5347|55468|479|6|6|8540.76|0.06|0.02|A|F|1995-04-11|1995-04-14|1995-05-02|NONE|TRUCK|lly unusual ideas. sl 5347|49567|2072|7|18|27298.08|0.01|0.01|N|F|1995-05-24|1995-05-07|1995-06-19|NONE|FOB|he ideas among the requests 5348|68446|5965|1|21|29703.24|0.10|0.04|N|O|1997-12-11|1997-12-24|1997-12-28|NONE|REG AIR| regular theodolites haggle car 5348|155865|896|2|31|59546.66|0.07|0.02|N|O|1998-01-04|1997-12-09|1998-01-17|COLLECT COD|RAIL|are finally 5348|16638|6639|3|16|24874.08|0.06|0.08|N|O|1998-02-28|1997-12-25|1998-03-12|DELIVER IN PERSON|AIR|uriously thin pinto beans 5348|19183|1685|4|7|7715.26|0.04|0.00|N|O|1998-01-29|1997-12-20|1998-02-10|DELIVER IN PERSON|RAIL|even foxes. epitap 5348|1818|4319|5|37|63632.97|0.06|0.07|N|O|1997-12-01|1998-02-02|1997-12-07|NONE|FOB|y according to the carefully pending acco 5348|142875|5390|6|14|26850.18|0.06|0.05|N|O|1997-12-16|1998-01-12|1997-12-24|COLLECT COD|FOB|en pinto beans. somas cajo 5349|155065|5066|1|19|21281.14|0.06|0.01|N|O|1996-09-11|1996-11-18|1996-09-22|TAKE BACK RETURN|FOB|endencies use whithout the special 5349|167110|4659|2|14|16479.54|0.06|0.00|N|O|1996-11-07|1996-11-17|1996-11-20|TAKE BACK RETURN|TRUCK|fully regular 5349|3482|3483|3|6|8312.88|0.10|0.01|N|O|1996-12-30|1996-10-08|1997-01-01|DELIVER IN PERSON|MAIL|inal deposits affix carefully 5350|121045|1046|1|19|20254.76|0.02|0.06|R|F|1993-10-20|1993-11-15|1993-11-17|DELIVER IN PERSON|RAIL|romise slyly alongsi 5350|190829|5868|2|44|84472.08|0.04|0.06|R|F|1993-10-30|1993-11-23|1993-11-25|DELIVER IN PERSON|AIR|p above the ironic, pending dep 5350|53095|8106|3|12|12577.08|0.10|0.04|A|F|1994-01-30|1993-11-21|1994-02-15|COLLECT COD|REG AIR| cajole. even instructions haggle. blithe 5350|154932|9963|4|7|13908.51|0.08|0.00|R|F|1993-10-19|1993-12-28|1993-11-04|NONE|SHIP|alongside of th 5350|128746|8747|5|27|47917.98|0.07|0.04|A|F|1993-11-25|1993-12-27|1993-12-08|COLLECT COD|TRUCK|es. blithe theodolites haggl 5351|6095|1096|1|36|36039.24|0.06|0.05|N|O|1998-07-27|1998-07-06|1998-08-25|NONE|MAIL|ss the ironic, regular asymptotes cajole 5351|32041|4545|2|47|45732.88|0.04|0.01|N|O|1998-05-30|1998-08-08|1998-06-23|DELIVER IN PERSON|REG AIR|s. grouches cajole. sile 5351|105327|2858|3|2|2664.64|0.00|0.02|N|O|1998-05-12|1998-07-15|1998-05-24|NONE|TRUCK|g accounts wake furiously slyly even dolph 5376|60778|8297|1|42|73028.34|0.10|0.04|A|F|1994-09-20|1994-08-30|1994-09-29|TAKE BACK RETURN|REG AIR|y even asymptotes. courts are unusual pa 5376|90990|6009|2|44|87163.56|0.05|0.02|R|F|1994-08-30|1994-08-05|1994-09-07|COLLECT COD|AIR|ithe packages detect final theodolites. f 5376|64871|4872|3|18|33045.66|0.02|0.08|A|F|1994-10-29|1994-09-13|1994-11-01|COLLECT COD|MAIL| accounts boo 5377|78172|3187|1|40|46006.80|0.00|0.04|N|O|1997-05-21|1997-06-15|1997-05-26|DELIVER IN PERSON|AIR|lithely ironic theodolites are care 5377|29316|6823|2|17|21170.27|0.09|0.00|N|O|1997-07-05|1997-05-25|1997-07-22|COLLECT COD|RAIL|dencies. carefully regular re 5377|102464|7485|3|23|33728.58|0.07|0.08|N|O|1997-06-26|1997-07-13|1997-07-08|COLLECT COD|RAIL| silent wa 5377|103123|5634|4|12|13513.44|0.05|0.07|N|O|1997-05-08|1997-06-15|1997-05-15|DELIVER IN PERSON|MAIL| ironic, final 5377|172292|4810|5|27|36835.83|0.08|0.02|N|O|1997-07-11|1997-06-12|1997-08-08|TAKE BACK RETURN|MAIL|press theodolites. e 5378|154773|7289|1|39|71283.03|0.07|0.04|R|F|1992-11-25|1992-12-22|1992-12-02|COLLECT COD|AIR|ts are quickly around the 5378|61083|6096|2|46|48027.68|0.01|0.04|A|F|1993-02-17|1993-01-20|1993-02-26|COLLECT COD|REG AIR|into beans sleep. fu 5378|9760|7261|3|18|30055.68|0.02|0.03|R|F|1992-11-25|1992-12-21|1992-12-10|COLLECT COD|FOB|onic accounts was bold, 5379|198787|1307|1|40|75431.20|0.01|0.08|N|O|1995-10-01|1995-10-19|1995-10-30|COLLECT COD|MAIL|carefully final accounts haggle blithely. 5380|181291|1292|1|14|19212.06|0.10|0.01|N|O|1997-12-18|1997-12-03|1998-01-06|NONE|RAIL|final platelets. 5380|146343|3886|2|10|13893.40|0.09|0.05|N|O|1997-11-24|1998-01-10|1997-12-21|COLLECT COD|AIR|refully pending deposits. special, even t 5380|183714|6233|3|40|71908.40|0.02|0.08|N|O|1997-12-30|1997-11-27|1998-01-09|DELIVER IN PERSON|SHIP|ar asymptotes. blithely r 5380|65789|802|4|6|10528.68|0.09|0.05|N|O|1997-11-15|1998-01-08|1997-12-11|COLLECT COD|MAIL|es. fluffily brave accounts across t 5380|106753|6754|5|48|84468.00|0.04|0.03|N|O|1997-12-01|1997-12-28|1997-12-05|DELIVER IN PERSON|FOB|encies haggle car 5381|187310|9829|1|37|51700.47|0.04|0.01|A|F|1993-04-08|1993-04-07|1993-04-12|DELIVER IN PERSON|SHIP|ly final deposits print carefully. unusua 5381|110002|5025|2|48|48576.00|0.04|0.03|R|F|1993-04-22|1993-04-17|1993-05-14|TAKE BACK RETURN|FOB|luffily spec 5381|191097|1098|3|13|15445.17|0.08|0.03|R|F|1993-05-09|1993-04-26|1993-05-25|NONE|FOB|s after the f 5381|167208|4757|4|17|21678.40|0.05|0.05|R|F|1993-05-25|1993-04-14|1993-06-17|NONE|MAIL|ckly final requests haggle qui 5381|62375|9894|5|49|65531.13|0.06|0.02|R|F|1993-05-08|1993-04-07|1993-06-03|NONE|FOB| accounts. regular, regula 5381|131056|6083|6|33|35872.65|0.10|0.00|A|F|1993-04-09|1993-04-03|1993-04-22|DELIVER IN PERSON|SHIP|ly special deposits 5381|43349|862|7|31|40062.54|0.04|0.05|A|F|1993-04-10|1993-03-22|1993-04-13|TAKE BACK RETURN|MAIL|the carefully expre 5382|152952|7983|1|34|68168.30|0.03|0.03|R|F|1992-02-22|1992-02-18|1992-03-02|DELIVER IN PERSON|FOB|gular accounts. even accounts integrate 5382|54879|7385|2|13|23840.31|0.09|0.06|A|F|1992-01-16|1992-03-12|1992-02-06|NONE|MAIL|eodolites. final foxes 5382|148097|8098|3|3|3435.27|0.10|0.06|A|F|1992-03-22|1992-03-06|1992-04-19|TAKE BACK RETURN|AIR|efully unusua 5382|61771|6784|4|20|34655.40|0.08|0.02|A|F|1992-03-26|1992-02-17|1992-04-15|DELIVER IN PERSON|FOB|carefully regular accounts. slyly ev 5382|176212|6213|5|14|18034.94|0.02|0.02|A|F|1992-04-05|1992-04-05|1992-05-04|TAKE BACK RETURN|FOB| brave platelets. ev 5382|179706|4741|6|6|10714.20|0.02|0.01|A|F|1992-03-07|1992-04-02|1992-03-18|TAKE BACK RETURN|FOB|y final foxes by the sl 5382|104356|1887|7|48|65296.80|0.05|0.05|A|F|1992-02-14|1992-03-19|1992-02-25|DELIVER IN PERSON|REG AIR|nts integrate quickly ca 5383|95116|5117|1|12|13333.32|0.04|0.00|N|O|1995-07-02|1995-08-16|1995-08-01|TAKE BACK RETURN|AIR|y regular instructi 5408|101941|6962|1|2|3885.88|0.07|0.04|R|F|1992-08-21|1992-10-03|1992-08-28|DELIVER IN PERSON|MAIL|cross the dolphins h 5408|117718|230|2|35|60749.85|0.04|0.05|R|F|1992-10-02|1992-10-17|1992-10-13|TAKE BACK RETURN|AIR|thely ironic requests alongside of the sl 5408|75127|7635|3|34|37472.08|0.10|0.02|A|F|1992-10-22|1992-08-25|1992-11-16|DELIVER IN PERSON|TRUCK|requests detect blithely a 5408|53108|5614|4|48|50932.80|0.04|0.05|R|F|1992-09-30|1992-08-27|1992-10-27|NONE|TRUCK|. furiously regular 5408|182529|7566|5|8|12892.16|0.03|0.07|A|F|1992-10-24|1992-09-06|1992-11-03|NONE|AIR|thely regular hocke 5409|193319|877|1|27|38132.37|0.01|0.02|A|F|1992-02-14|1992-03-18|1992-02-23|DELIVER IN PERSON|AIR|eodolites 5409|103573|3574|2|38|59909.66|0.01|0.02|A|F|1992-03-17|1992-03-29|1992-04-13|NONE|REG AIR|onic, regular accounts! blithely even 5409|140347|7890|3|17|23584.78|0.07|0.00|A|F|1992-01-13|1992-04-05|1992-01-20|DELIVER IN PERSON|AIR|cross the sil 5409|403|7904|4|9|11730.60|0.07|0.03|A|F|1992-02-15|1992-04-02|1992-02-28|DELIVER IN PERSON|AIR| unusual, unusual reques 5409|158141|8142|5|37|44368.18|0.06|0.04|R|F|1992-05-07|1992-02-10|1992-05-20|DELIVER IN PERSON|FOB|ously regular packages. packages 5409|63902|6409|6|14|26122.60|0.03|0.08|R|F|1992-02-14|1992-03-26|1992-02-29|DELIVER IN PERSON|AIR|osits cajole furiously 5410|116768|6769|1|48|85668.48|0.04|0.08|N|O|1998-09-27|1998-09-11|1998-10-01|TAKE BACK RETURN|AIR| about the slyly even courts. quickly regul 5410|104411|6922|2|41|58031.81|0.01|0.07|N|O|1998-08-25|1998-10-20|1998-09-01|DELIVER IN PERSON|REG AIR|sly. slyly ironic theodolites 5410|28027|530|3|40|38200.80|0.07|0.08|N|O|1998-11-17|1998-10-02|1998-11-27|COLLECT COD|TRUCK|iously special accounts are along th 5410|49503|2008|4|8|11620.00|0.05|0.04|N|O|1998-09-12|1998-10-22|1998-09-22|DELIVER IN PERSON|TRUCK|ly. fluffily ironic platelets alon 5411|95432|451|1|17|24266.31|0.05|0.01|N|O|1997-07-22|1997-07-14|1997-07-30|TAKE BACK RETURN|REG AIR| slyly slyly even deposits. carefully b 5411|112029|4541|2|10|10410.20|0.08|0.01|N|O|1997-07-19|1997-08-04|1997-07-26|TAKE BACK RETURN|MAIL|nding, special foxes unw 5411|55057|5058|3|5|5060.25|0.10|0.01|N|O|1997-09-12|1997-08-03|1997-09-23|DELIVER IN PERSON|FOB| bold, ironic theodo 5411|128590|3615|4|15|24278.85|0.08|0.05|N|O|1997-07-01|1997-07-15|1997-07-07|COLLECT COD|RAIL|attainments sleep slyly ironic 5411|3016|3017|5|19|17461.19|0.05|0.08|N|O|1997-05-25|1997-07-30|1997-06-19|COLLECT COD|RAIL|ial accounts according to the f 5412|53078|8089|1|2|2062.14|0.03|0.07|N|O|1998-04-14|1998-04-02|1998-04-19|TAKE BACK RETURN|REG AIR| sleep above the furiou 5412|65168|2687|2|48|54391.68|0.01|0.08|N|O|1998-02-22|1998-03-28|1998-03-18|TAKE BACK RETURN|TRUCK|s. slyly final packages cajole blithe 5412|73678|1200|3|31|51201.77|0.05|0.08|N|O|1998-03-23|1998-04-17|1998-04-10|NONE|SHIP|t the accounts detect slyly about the c 5412|96319|1338|4|26|34198.06|0.02|0.08|N|O|1998-01-22|1998-04-19|1998-02-17|NONE|AIR| the blithel 5413|125457|5458|1|48|71157.60|0.02|0.08|N|O|1998-01-25|1997-11-20|1998-02-22|COLLECT COD|SHIP| theodolites. furiously ironic instr 5413|141805|4320|2|37|68331.60|0.02|0.07|N|O|1997-12-08|1998-01-01|1997-12-13|COLLECT COD|TRUCK|usly bold instructions affix idly unusual, 5413|110854|5877|3|36|67134.60|0.02|0.07|N|O|1997-12-12|1997-11-28|1997-12-25|NONE|TRUCK|ular, regular ideas mold! final requests 5413|109216|1727|4|22|26954.62|0.02|0.08|N|O|1997-11-10|1997-11-24|1997-11-22|DELIVER IN PERSON|FOB|posits. quick 5413|188636|6191|5|5|8623.15|0.10|0.01|N|O|1997-11-28|1997-11-24|1997-12-05|NONE|RAIL|tes are al 5413|189019|9020|6|32|35456.32|0.02|0.03|N|O|1997-10-28|1998-01-03|1997-11-10|NONE|TRUCK|refully special package 5413|30016|2520|7|32|30272.32|0.06|0.07|N|O|1997-10-23|1997-12-09|1997-11-17|NONE|TRUCK|he quickly ironic ideas. slyly ironic ide 5414|67051|7052|1|40|40722.00|0.07|0.06|R|F|1993-04-07|1993-05-18|1993-04-23|COLLECT COD|AIR|ts are evenly across 5414|122106|4619|2|48|54148.80|0.06|0.07|R|F|1993-06-08|1993-05-14|1993-07-06|DELIVER IN PERSON|FOB| silent dolphins; fluffily regular tithe 5414|34768|2278|3|23|39163.48|0.10|0.00|A|F|1993-07-22|1993-05-26|1993-08-08|COLLECT COD|MAIL|e bold, express dolphins. spec 5414|132077|2078|4|15|16636.05|0.06|0.08|R|F|1993-05-18|1993-06-09|1993-05-27|DELIVER IN PERSON|REG AIR|e slyly about the carefully regula 5414|8372|873|5|19|24327.03|0.01|0.05|R|F|1993-04-06|1993-05-12|1993-05-02|DELIVER IN PERSON|RAIL|ffily silent theodolites na 5414|97119|2138|6|28|31251.08|0.10|0.05|A|F|1993-03-27|1993-06-04|1993-04-07|TAKE BACK RETURN|SHIP|ts sleep sl 5415|101746|4257|1|44|76900.56|0.00|0.06|A|F|1992-08-19|1992-10-26|1992-09-17|TAKE BACK RETURN|TRUCK| requests. unusual theodolites sleep agains 5415|30220|7730|2|16|18403.52|0.08|0.00|A|F|1992-09-29|1992-09-12|1992-10-10|NONE|AIR|pinto beans haggle furiously 5415|101066|6087|3|6|6402.36|0.10|0.03|A|F|1992-10-28|1992-09-09|1992-11-20|COLLECT COD|RAIL|ges around the fur 5415|15099|5100|4|43|43605.87|0.01|0.02|R|F|1992-11-17|1992-09-14|1992-12-14|DELIVER IN PERSON|SHIP|yly blithely stealthy deposits. carefu 5415|160014|7563|5|11|11814.11|0.00|0.01|R|F|1992-11-22|1992-10-19|1992-12-10|DELIVER IN PERSON|SHIP|gle among t 5415|143712|6227|6|46|80762.66|0.03|0.03|R|F|1992-08-25|1992-09-10|1992-09-22|DELIVER IN PERSON|REG AIR|ve the fluffily 5415|152967|2968|7|11|22219.56|0.08|0.06|A|F|1992-08-21|1992-09-04|1992-08-23|NONE|TRUCK|unts maintain carefully unusual 5440|114100|9123|1|3|3342.30|0.02|0.08|N|O|1997-02-18|1997-02-28|1997-03-15|NONE|SHIP|y. accounts haggle along the blit 5441|163452|8485|1|3|4546.35|0.00|0.02|R|F|1994-08-12|1994-10-14|1994-09-01|TAKE BACK RETURN|REG AIR|are. unusual, 5441|130764|765|2|49|87943.24|0.02|0.03|A|F|1994-09-23|1994-09-22|1994-10-22|NONE|FOB|ording to the furio 5441|143016|559|3|33|34947.33|0.09|0.02|R|F|1994-10-09|1994-10-06|1994-10-30|DELIVER IN PERSON|TRUCK|ges. final instruction 5441|66124|1137|4|47|51235.64|0.07|0.08|R|F|1994-11-19|1994-10-16|1994-12-16|TAKE BACK RETURN|FOB|ounts wake slyly about the express instr 5442|41917|6926|1|16|29742.56|0.00|0.00|N|O|1998-04-12|1998-03-03|1998-05-04|TAKE BACK RETURN|RAIL|r packages. accounts haggle dependencies. f 5442|87111|4636|2|45|49414.95|0.08|0.01|N|O|1998-03-30|1998-02-24|1998-04-18|TAKE BACK RETURN|AIR|old slyly after 5442|60318|5331|3|12|15339.72|0.01|0.08|N|O|1998-04-15|1998-03-18|1998-05-05|DELIVER IN PERSON|TRUCK|fully final 5442|157234|7235|4|21|27115.83|0.07|0.06|N|O|1998-03-13|1998-02-19|1998-04-06|COLLECT COD|MAIL|ffily furiously ironic theodolites. furio 5442|15990|5991|5|25|47649.75|0.04|0.00|N|O|1998-03-29|1998-02-13|1998-04-13|TAKE BACK RETURN|REG AIR|ake furiously. slyly express th 5442|143507|1050|6|26|40313.00|0.08|0.07|N|O|1998-03-21|1998-03-21|1998-03-25|TAKE BACK RETURN|AIR|have to sleep furiously bold ideas. blith 5443|177288|7289|1|14|19113.92|0.02|0.00|N|O|1996-10-27|1996-11-11|1996-11-21|DELIVER IN PERSON|RAIL|s after the regular, regular deposits hag 5443|71326|1327|2|39|50595.48|0.03|0.07|N|O|1996-11-01|1996-11-30|1996-11-19|NONE|RAIL|gage carefully across the furiously 5443|159719|4750|3|25|44467.75|0.05|0.00|N|O|1996-12-07|1997-01-08|1997-01-05|NONE|FOB|use carefully above the pinto bea 5443|190188|5227|4|6|7669.08|0.05|0.02|N|O|1996-11-17|1996-12-03|1996-11-30|TAKE BACK RETURN|AIR|p fluffily foxe 5443|82716|5225|5|40|67948.40|0.03|0.03|N|O|1997-01-28|1996-12-10|1997-02-13|NONE|FOB|n courts. special re 5444|185509|3064|1|21|33484.50|0.01|0.07|A|F|1995-04-11|1995-04-25|1995-04-21|DELIVER IN PERSON|RAIL|ar packages haggle above th 5444|42182|7191|2|40|44967.20|0.05|0.08|N|O|1995-07-09|1995-04-25|1995-07-19|COLLECT COD|TRUCK|ously bold ideas. instructions wake slyl 5444|149858|7401|3|40|76314.00|0.08|0.01|A|F|1995-04-06|1995-05-08|1995-05-06|DELIVER IN PERSON|AIR| even packages. 5444|58950|3961|4|33|62995.35|0.05|0.04|N|O|1995-06-24|1995-04-24|1995-07-13|DELIVER IN PERSON|SHIP|ut the courts cajole blithely excuses 5444|170692|8244|5|21|37016.49|0.04|0.00|R|F|1995-05-05|1995-05-25|1995-05-29|TAKE BACK RETURN|REG AIR|aves serve sly 5444|19026|4029|6|21|19845.42|0.07|0.01|A|F|1995-03-30|1995-05-01|1995-03-31|COLLECT COD|AIR|furiously even theodolites. 5445|89126|4143|1|33|36798.96|0.08|0.07|A|F|1993-10-21|1993-10-14|1993-10-28|DELIVER IN PERSON|REG AIR|ests. final instructions 5445|130119|120|2|12|13789.32|0.09|0.08|R|F|1993-11-02|1993-09-05|1993-11-26|COLLECT COD|FOB| slyly pending pinto beans was slyly al 5445|102859|7880|3|46|85645.10|0.04|0.07|A|F|1993-10-06|1993-09-15|1993-10-28|DELIVER IN PERSON|RAIL|old depend 5445|148677|8678|4|10|17256.70|0.08|0.06|A|F|1993-09-16|1993-10-05|1993-10-01|NONE|TRUCK|ncies abou 5445|12496|7499|5|14|19718.86|0.00|0.02|R|F|1993-11-19|1993-10-18|1993-12-07|NONE|RAIL| requests. bravely i 5446|189366|4403|1|27|39294.72|0.05|0.07|R|F|1994-07-21|1994-08-25|1994-08-17|TAKE BACK RETURN|RAIL|ously across the quic 5447|98536|1046|1|31|47570.43|0.09|0.03|N|O|1996-07-14|1996-05-07|1996-07-17|COLLECT COD|SHIP| foxes sleep. blithely unusual accounts det 5472|58956|8957|1|27|51703.65|0.09|0.06|A|F|1993-08-04|1993-07-07|1993-09-03|COLLECT COD|TRUCK|fily pending attainments. unus 5472|67697|5216|2|28|46611.32|0.00|0.03|A|F|1993-07-28|1993-05-28|1993-08-11|TAKE BACK RETURN|FOB|ffily pendin 5472|177295|2330|3|45|61753.05|0.06|0.02|R|F|1993-06-05|1993-05-14|1993-06-10|NONE|TRUCK| idle packages. furi 5472|183690|8727|4|37|65626.53|0.07|0.05|R|F|1993-06-15|1993-07-03|1993-07-09|DELIVER IN PERSON|RAIL|egrate carefully dependencies. 5472|74051|4052|5|40|41002.00|0.02|0.05|A|F|1993-04-13|1993-07-04|1993-05-04|NONE|REG AIR|e requests detect furiously. ruthlessly un 5472|166920|4469|6|39|77489.88|0.02|0.03|R|F|1993-04-18|1993-07-10|1993-05-12|TAKE BACK RETURN|MAIL|uriously carefully 5472|14390|1894|7|1|1304.39|0.03|0.02|A|F|1993-04-14|1993-06-28|1993-04-16|NONE|RAIL|s use furiou 5473|47878|383|1|9|16432.83|0.03|0.07|R|F|1992-06-03|1992-05-30|1992-06-09|TAKE BACK RETURN|AIR| excuses sleep blithely! regular dep 5473|69115|9116|2|27|29270.97|0.01|0.03|A|F|1992-04-06|1992-04-26|1992-04-29|TAKE BACK RETURN|MAIL|the deposits. warthogs wake fur 5473|14473|1977|3|33|45786.51|0.09|0.00|R|F|1992-05-18|1992-06-10|1992-06-13|TAKE BACK RETURN|MAIL|efully above the even, 5474|183763|3764|1|38|70176.88|0.01|0.08|A|F|1992-07-15|1992-07-16|1992-07-20|NONE|REG AIR| slyly beneath 5474|93631|1159|2|10|16246.30|0.06|0.00|R|F|1992-08-08|1992-08-10|1992-08-24|TAKE BACK RETURN|TRUCK|pinto bean 5474|47831|2840|3|31|55143.73|0.00|0.08|R|F|1992-08-02|1992-07-12|1992-08-04|NONE|TRUCK|the furiously express ideas. speci 5474|89506|4523|4|46|68793.00|0.03|0.04|A|F|1992-06-07|1992-07-11|1992-06-22|NONE|TRUCK|nstructions. furio 5475|182422|9977|1|10|15044.20|0.09|0.08|N|O|1996-07-19|1996-08-22|1996-07-23|COLLECT COD|AIR|ding to the deposits wake fina 5476|47660|2669|1|13|20899.58|0.01|0.04|N|O|1997-12-27|1997-12-08|1997-12-29|COLLECT COD|TRUCK|iously special ac 5476|19679|2181|2|17|27177.39|0.10|0.01|N|O|1998-02-02|1998-01-28|1998-02-14|COLLECT COD|FOB|ng dependencies until the f 5477|79222|6744|1|20|24024.40|0.03|0.01|N|O|1998-03-21|1998-02-09|1998-04-07|TAKE BACK RETURN|SHIP|platelets about the ironic 5477|76155|8663|2|21|23754.15|0.03|0.00|N|O|1998-01-28|1998-02-15|1998-02-24|TAKE BACK RETURN|SHIP|blate slyly. silent 5477|133790|3791|3|31|56537.49|0.04|0.01|N|O|1998-01-11|1998-01-30|1998-02-04|DELIVER IN PERSON|MAIL| special Tiresias cajole furiously. pending 5477|192129|7168|4|16|19537.92|0.00|0.01|N|O|1998-03-07|1998-03-12|1998-04-06|COLLECT COD|RAIL|regular, s 5477|95742|761|5|23|39968.02|0.00|0.06|N|O|1998-01-04|1998-02-23|1998-01-24|NONE|REG AIR|telets wake blithely ab 5477|120319|2832|6|19|25446.89|0.10|0.03|N|O|1998-02-03|1998-01-30|1998-03-04|TAKE BACK RETURN|MAIL|ost carefully packages. 5478|7283|7284|1|39|46420.92|0.09|0.06|N|O|1996-08-19|1996-06-25|1996-09-08|DELIVER IN PERSON|SHIP|s. furiously 5478|1079|3580|2|47|46063.29|0.10|0.01|N|O|1996-08-15|1996-07-12|1996-08-31|NONE|RAIL| instructions; slyly even accounts hagg 5478|118847|1359|3|25|46646.00|0.09|0.07|N|O|1996-06-08|1996-07-12|1996-07-07|NONE|TRUCK|unusual, pending requests haggle accoun 5479|137771|285|1|50|90438.50|0.02|0.02|A|F|1993-12-24|1994-02-14|1994-01-18|DELIVER IN PERSON|MAIL|ironic gifts. even dependencies sno 5479|103906|3907|2|19|36288.10|0.05|0.03|A|F|1994-01-22|1994-03-07|1994-02-11|DELIVER IN PERSON|SHIP|arefully bo 5504|67102|2115|1|4|4276.40|0.10|0.07|A|F|1993-04-30|1993-03-01|1993-05-22|DELIVER IN PERSON|AIR|into beans boost. 5504|176056|6057|2|7|7924.35|0.03|0.05|R|F|1993-04-25|1993-03-15|1993-05-06|NONE|TRUCK|packages detect furiously express reques 5504|159600|7146|3|29|48128.40|0.05|0.03|A|F|1993-01-28|1993-02-13|1993-02-27|NONE|SHIP|ajole carefully. care 5505|24501|2008|1|43|61296.50|0.07|0.01|N|O|1997-12-30|1997-11-28|1998-01-09|TAKE BACK RETURN|TRUCK|y alongside of the special requests. 5505|181434|8989|2|33|50009.19|0.05|0.08|N|O|1998-01-11|1997-11-11|1998-01-30|TAKE BACK RETURN|AIR|ithely unusual excuses integrat 5505|154641|9672|3|10|16956.40|0.06|0.01|N|O|1997-10-28|1997-11-27|1997-10-29|DELIVER IN PERSON|AIR| furiously special asym 5505|39422|4429|4|18|24505.56|0.04|0.04|N|O|1997-10-25|1997-12-12|1997-10-30|TAKE BACK RETURN|RAIL| to the quickly express pac 5505|161992|7025|5|46|94483.54|0.05|0.00|N|O|1998-01-06|1997-11-04|1998-02-04|TAKE BACK RETURN|SHIP|usly ironic dependencies haggle across 5506|139309|9310|1|2|2696.60|0.00|0.03|R|F|1994-02-04|1994-01-13|1994-02-17|COLLECT COD|MAIL|onic theodolites are fluffil 5506|159024|9025|2|6|6498.12|0.07|0.06|R|F|1994-02-21|1994-01-30|1994-02-27|DELIVER IN PERSON|MAIL|hely according to the furiously unusua 5507|9600|4601|1|23|34720.80|0.05|0.04|N|O|1998-09-04|1998-07-04|1998-09-18|TAKE BACK RETURN|AIR|ously slow packages poach whithout the 5507|137601|7602|2|48|78652.80|0.03|0.01|N|O|1998-08-03|1998-08-10|1998-08-24|DELIVER IN PERSON|AIR|yly idle deposits. final, final fox 5507|44676|7181|3|4|6482.68|0.04|0.06|N|O|1998-06-06|1998-07-02|1998-06-27|TAKE BACK RETURN|RAIL|into beans are 5507|66850|9357|4|22|39970.70|0.07|0.01|N|O|1998-07-08|1998-08-10|1998-07-22|DELIVER IN PERSON|TRUCK|gular ideas. carefully unu 5507|131412|1413|5|48|69283.68|0.06|0.01|N|O|1998-07-21|1998-07-15|1998-07-31|DELIVER IN PERSON|SHIP|uriously regular acc 5508|116947|4481|1|4|7855.76|0.10|0.04|N|O|1996-09-01|1996-08-02|1996-09-17|COLLECT COD|AIR|fluffily about the even 5509|196663|1702|1|3|5278.98|0.03|0.02|A|F|1994-06-14|1994-05-11|1994-06-17|NONE|SHIP| quickly fin 5509|98560|6088|2|17|26495.52|0.03|0.07|R|F|1994-07-01|1994-06-30|1994-07-31|COLLECT COD|AIR|ccounts wake ar 5509|92611|139|3|30|48108.30|0.04|0.04|A|F|1994-07-23|1994-06-01|1994-08-08|NONE|AIR|counts haggle pinto beans. furiously 5509|99849|4868|4|45|83197.80|0.00|0.07|A|F|1994-07-24|1994-05-28|1994-08-20|COLLECT COD|AIR|counts sleep. f 5509|155719|3265|5|35|62114.85|0.04|0.03|A|F|1994-04-17|1994-06-29|1994-04-24|COLLECT COD|RAIL|c accounts. ca 5510|15828|3332|1|8|13950.56|0.01|0.01|A|F|1993-03-16|1993-03-29|1993-03-24|DELIVER IN PERSON|FOB|n packages boost sly 5510|19300|6804|2|46|56087.80|0.02|0.07|A|F|1993-03-12|1993-02-09|1993-03-19|NONE|TRUCK|silent packages cajole doggedly regular 5510|161054|1055|3|47|52407.35|0.03|0.01|A|F|1993-01-20|1993-03-25|1993-02-15|DELIVER IN PERSON|SHIP|riously even requests. slyly bold accou 5510|23375|882|4|29|37652.73|0.09|0.08|A|F|1993-02-28|1993-03-28|1993-03-12|COLLECT COD|AIR|lithely fluffily ironic req 5511|164477|6994|1|16|24663.52|0.10|0.05|A|F|1995-02-02|1995-01-06|1995-02-19|TAKE BACK RETURN|RAIL|thely bold theodolites 5511|164130|1679|2|31|37018.03|0.09|0.01|A|F|1995-02-23|1995-01-21|1995-03-02|COLLECT COD|REG AIR|gular excuses. fluffily even pinto beans c 5511|127764|277|3|49|87796.24|0.05|0.05|R|F|1994-12-21|1995-01-27|1994-12-26|NONE|REG AIR|bout the requests. theodolites 5511|121474|3987|4|4|5981.88|0.08|0.02|R|F|1994-12-28|1995-01-16|1995-01-24|TAKE BACK RETURN|RAIL|lphins. carefully blithe de 5511|8948|1449|5|23|42709.62|0.10|0.07|A|F|1995-03-11|1995-01-21|1995-03-27|TAKE BACK RETURN|TRUCK|ing dugouts 5511|187419|2456|6|5|7532.05|0.08|0.05|R|F|1994-12-29|1995-01-16|1995-01-24|DELIVER IN PERSON|MAIL|al theodolites. blithely final de 5511|142696|239|7|23|39989.87|0.02|0.07|R|F|1995-02-03|1995-01-05|1995-02-18|COLLECT COD|REG AIR|ully deposits. warthogs hagg 5536|89717|7242|1|14|23893.94|0.08|0.06|N|O|1998-05-18|1998-05-08|1998-06-05|COLLECT COD|MAIL|instructions sleep 5536|61023|3530|2|20|19680.40|0.08|0.04|N|O|1998-05-08|1998-05-10|1998-05-31|DELIVER IN PERSON|REG AIR|equests mo 5536|196848|9368|3|35|68069.40|0.07|0.02|N|O|1998-05-19|1998-06-08|1998-06-05|NONE|MAIL|c, final theo 5536|8827|8828|4|30|52074.60|0.05|0.07|N|O|1998-04-15|1998-05-23|1998-05-03|NONE|FOB|arefully regular theodolites according 5536|140844|845|5|11|20733.24|0.02|0.08|N|O|1998-03-18|1998-05-12|1998-03-28|TAKE BACK RETURN|FOB| snooze furio 5537|44015|9024|1|10|9590.10|0.05|0.08|N|O|1997-01-13|1996-12-25|1997-01-28|TAKE BACK RETURN|AIR| sleep carefully slyly bold depos 5537|149459|7002|2|15|22626.75|0.07|0.04|N|O|1997-01-13|1996-12-25|1997-01-27|COLLECT COD|AIR|eposits. permanently pending packag 5537|150896|5927|3|39|75928.71|0.03|0.00|N|O|1996-12-17|1996-11-08|1997-01-15|COLLECT COD|REG AIR| slyly bold packages are. qu 5537|96020|3548|4|38|38608.76|0.01|0.00|N|O|1996-11-06|1996-11-23|1996-11-12|TAKE BACK RETURN|MAIL|s above the carefully ironic deposits 5538|153301|8332|1|42|56880.60|0.05|0.00|A|F|1994-04-08|1994-03-17|1994-05-05|DELIVER IN PERSON|REG AIR|vely ironic accounts. furiously unusual acc 5538|120083|84|2|4|4412.32|0.02|0.03|R|F|1994-03-21|1994-02-17|1994-04-11|TAKE BACK RETURN|REG AIR|ithely along the c 5538|18521|1023|3|38|54701.76|0.03|0.06|R|F|1994-03-17|1994-02-11|1994-04-10|TAKE BACK RETURN|FOB|ular pinto beans. silent ideas above 5538|77667|5189|4|9|14801.94|0.00|0.01|R|F|1993-12-26|1994-01-31|1994-01-03|TAKE BACK RETURN|REG AIR|encies across the blithely fina 5539|64404|1923|1|42|57472.80|0.10|0.08|A|F|1994-09-29|1994-09-17|1994-10-20|DELIVER IN PERSON|RAIL|ons across the carefully si 5540|180895|896|1|42|82987.38|0.02|0.08|N|O|1996-11-12|1996-12-18|1996-12-05|TAKE BACK RETURN|RAIL|ss dolphins haggle 5540|101022|1023|2|2|2046.04|0.06|0.02|N|O|1996-12-12|1997-01-09|1996-12-25|DELIVER IN PERSON|MAIL|nic asymptotes could hav 5540|63336|5843|3|19|24687.27|0.01|0.03|N|O|1997-02-06|1996-11-18|1997-02-20|DELIVER IN PERSON|SHIP| slyly slyl 5540|71466|8988|4|24|34499.04|0.10|0.05|N|O|1997-01-09|1996-12-02|1997-01-23|COLLECT COD|FOB|deposits! ironic depths may engage-- b 5541|95359|7869|1|39|52819.65|0.08|0.05|N|O|1997-11-17|1997-12-27|1997-12-11|TAKE BACK RETURN|RAIL|ding theodolites haggle against the slyly 5542|188250|8251|1|6|8029.50|0.03|0.01|N|O|1996-06-14|1996-05-28|1996-07-11|DELIVER IN PERSON|TRUCK| foxes doubt. theodolites ca 5543|142617|5132|1|14|23234.54|0.02|0.03|R|F|1993-10-09|1993-12-09|1993-10-21|NONE|SHIP|ecial reque 5543|161544|9093|2|22|35321.88|0.04|0.00|A|F|1993-11-06|1993-11-02|1993-12-02|DELIVER IN PERSON|SHIP|instructions. deposits use quickly. ir 5543|66993|9500|3|3|5879.97|0.08|0.05|R|F|1993-11-18|1993-11-05|1993-12-17|NONE|FOB|ress, even 5543|146766|1795|4|8|14502.08|0.05|0.01|R|F|1993-10-28|1993-11-18|1993-11-07|NONE|SHIP|totes? iron 5543|79919|9920|5|32|60765.12|0.03|0.03|R|F|1993-10-04|1993-11-14|1993-11-03|DELIVER IN PERSON|AIR|ully around the 5543|183372|927|6|1|1455.37|0.03|0.07|A|F|1993-10-29|1993-11-11|1993-11-23|TAKE BACK RETURN|FOB|uriously. slyly 5543|128171|3196|7|39|46767.63|0.06|0.00|R|F|1993-10-07|1993-11-15|1993-10-28|TAKE BACK RETURN|MAIL|l excuses are furiously. slyly unusual requ 5568|165351|7868|1|50|70817.50|0.05|0.05|N|O|1995-07-14|1995-09-04|1995-08-03|COLLECT COD|TRUCK|furious ide 5568|43012|3013|2|18|17190.18|0.01|0.08|N|O|1995-08-19|1995-08-18|1995-08-24|DELIVER IN PERSON|SHIP|structions haggle. carefully regular 5568|88766|8767|3|35|61416.60|0.08|0.07|N|O|1995-09-17|1995-09-04|1995-10-14|NONE|SHIP|lyly. blit 5569|28825|1328|1|25|43845.50|0.10|0.03|R|F|1993-06-29|1993-07-18|1993-07-05|TAKE BACK RETURN|TRUCK| deposits cajole above 5569|57218|4734|2|26|30555.46|0.09|0.06|A|F|1993-08-21|1993-07-22|1993-09-09|DELIVER IN PERSON|MAIL|pitaphs. ironic req 5569|54713|7219|3|48|80050.08|0.02|0.03|R|F|1993-06-16|1993-06-15|1993-07-09|COLLECT COD|SHIP|the fluffily 5569|146526|1555|4|19|29877.88|0.10|0.08|R|F|1993-07-30|1993-06-21|1993-08-13|TAKE BACK RETURN|FOB| detect ca 5569|58516|6032|5|15|22117.65|0.02|0.06|A|F|1993-06-29|1993-07-06|1993-07-05|DELIVER IN PERSON|MAIL|lithely bold requests boost fur 5570|160555|8104|1|37|59775.35|0.08|0.02|N|O|1996-08-29|1996-10-23|1996-09-11|NONE|RAIL|y ironic pin 5570|38607|3614|2|15|23184.00|0.09|0.02|N|O|1996-10-04|1996-10-05|1996-10-28|TAKE BACK RETURN|REG AIR|beans nag slyly special, regular pack 5570|59349|9350|3|29|37941.86|0.02|0.05|N|O|1996-10-12|1996-10-20|1996-11-08|TAKE BACK RETURN|SHIP|he silent, enticing requests. 5571|153454|5970|1|32|48238.40|0.05|0.01|R|F|1992-12-25|1993-03-01|1993-01-23|NONE|FOB| the blithely even packages nag q 5571|93241|769|2|31|38261.44|0.09|0.07|R|F|1993-01-05|1993-01-18|1993-02-04|DELIVER IN PERSON|SHIP|uffily even accounts. quickly re 5571|91430|8958|3|18|25585.74|0.10|0.05|R|F|1993-03-11|1993-02-28|1993-04-03|COLLECT COD|REG AIR|uests haggle furiously pending d 5572|21549|6554|1|24|35292.96|0.08|0.08|R|F|1994-10-30|1994-10-02|1994-11-27|TAKE BACK RETURN|MAIL|ests cajole. evenly ironic exc 5572|171660|9212|2|27|46754.82|0.03|0.04|A|F|1994-08-29|1994-09-10|1994-08-30|TAKE BACK RETURN|SHIP| accounts. carefully final accoun 5572|86234|6235|3|19|23184.37|0.10|0.00|A|F|1994-08-12|1994-10-07|1994-09-01|DELIVER IN PERSON|RAIL|es. final, final requests wake blithely ag 5572|134303|1843|4|46|61515.80|0.02|0.01|R|F|1994-09-08|1994-10-14|1994-10-01|NONE|REG AIR|ully regular platelet 5572|23056|8061|5|34|33287.70|0.10|0.08|R|F|1994-10-22|1994-08-16|1994-11-08|NONE|TRUCK|asymptotes integrate. s 5572|100044|45|6|14|14616.56|0.04|0.05|A|F|1994-11-02|1994-09-20|1994-11-03|COLLECT COD|RAIL|he fluffily express packages. fluffily fina 5572|25241|7744|7|24|27989.76|0.01|0.05|R|F|1994-09-26|1994-09-04|1994-10-22|DELIVER IN PERSON|FOB| beans. foxes sleep fluffily across th 5573|20138|2641|1|32|33860.16|0.05|0.07|N|O|1996-09-30|1996-10-25|1996-10-15|DELIVER IN PERSON|RAIL|egular depths haggl 5573|49469|4478|2|2|2836.92|0.01|0.07|N|O|1996-08-26|1996-09-29|1996-09-04|COLLECT COD|TRUCK| even foxes. specia 5573|10720|5723|3|46|75013.12|0.06|0.01|N|O|1996-11-04|1996-10-02|1996-11-15|DELIVER IN PERSON|MAIL|s haggle qu 5573|168653|6202|4|43|74030.95|0.10|0.03|N|O|1996-10-22|1996-11-03|1996-11-02|TAKE BACK RETURN|FOB| furiously pending packages against 5573|137649|7650|5|43|72525.52|0.05|0.04|N|O|1996-09-09|1996-09-24|1996-09-28|COLLECT COD|AIR| bold package 5574|184533|7052|1|46|74406.38|0.02|0.07|A|F|1992-06-20|1992-04-19|1992-07-11|NONE|FOB|arefully express requests wake furiousl 5574|32304|7311|2|21|25962.30|0.05|0.08|A|F|1992-03-22|1992-04-26|1992-04-16|TAKE BACK RETURN|TRUCK|fully final dugouts. express foxes nag 5574|118564|3587|3|27|42729.12|0.10|0.06|R|F|1992-05-08|1992-05-19|1992-06-05|TAKE BACK RETURN|REG AIR|ecial realms. furiously entici 5574|93359|5869|4|14|18932.90|0.09|0.01|R|F|1992-05-20|1992-04-09|1992-05-23|COLLECT COD|REG AIR| use slyly carefully special requests? slyl 5574|84053|9070|5|19|19703.95|0.05|0.03|A|F|1992-05-28|1992-04-24|1992-06-11|TAKE BACK RETURN|REG AIR|old deposits int 5575|57026|4542|1|7|6881.14|0.01|0.07|N|O|1995-10-01|1995-09-30|1995-10-06|NONE|FOB|s. slyly pending theodolites prin 5575|30170|7680|2|23|25303.91|0.04|0.02|N|O|1995-10-26|1995-10-09|1995-11-13|TAKE BACK RETURN|AIR|enticingly final requests. ironically 5575|62433|9952|3|16|22326.88|0.00|0.07|N|O|1995-08-17|1995-10-14|1995-08-30|NONE|RAIL|jole boldly beyond the final as 5575|109718|9719|4|7|12093.97|0.01|0.04|N|O|1995-10-15|1995-09-14|1995-10-18|DELIVER IN PERSON|RAIL|special requests. final, final 5600|186280|8799|1|34|46453.52|0.02|0.00|N|O|1997-03-22|1997-04-05|1997-04-09|TAKE BACK RETURN|MAIL|ly above the stealthy ideas. permane 5600|7352|4853|2|19|23927.65|0.00|0.01|N|O|1997-04-10|1997-03-24|1997-04-16|TAKE BACK RETURN|TRUCK|dencies. carefully p 5601|37982|5492|1|29|55679.42|0.09|0.04|A|F|1992-04-06|1992-02-24|1992-04-29|DELIVER IN PERSON|TRUCK| ironic ideas. final 5601|163175|8208|2|45|55717.65|0.10|0.07|A|F|1992-03-25|1992-04-03|1992-04-04|TAKE BACK RETURN|MAIL|ts-- blithely final accounts cajole. carefu 5601|72205|2206|3|38|44733.60|0.07|0.00|A|F|1992-01-08|1992-03-01|1992-01-09|TAKE BACK RETURN|REG AIR|ter the evenly final deposit 5601|147847|362|4|12|22738.08|0.03|0.01|A|F|1992-02-27|1992-03-16|1992-03-27|COLLECT COD|TRUCK|ep carefully a 5602|175324|2876|1|9|12593.88|0.08|0.03|N|O|1997-10-14|1997-09-14|1997-11-11|COLLECT COD|FOB|lar foxes; quickly ironic ac 5602|61015|8534|2|31|30256.31|0.04|0.08|N|O|1997-09-04|1997-10-24|1997-09-07|NONE|TRUCK|rate fluffily regular platelets. blithel 5602|67680|2693|3|30|49430.40|0.04|0.00|N|O|1997-09-20|1997-10-25|1997-10-12|DELIVER IN PERSON|FOB|e slyly even packages. careful 5603|97812|5340|1|50|90490.50|0.03|0.02|A|F|1992-10-06|1992-08-20|1992-10-08|COLLECT COD|SHIP|final theodolites accor 5603|115704|3238|2|49|84265.30|0.06|0.05|A|F|1992-06-24|1992-07-28|1992-07-01|DELIVER IN PERSON|FOB|fully silent requests. carefully fin 5603|31836|9346|3|49|86623.67|0.00|0.02|R|F|1992-10-07|1992-07-21|1992-10-10|DELIVER IN PERSON|TRUCK|nic, pending dependencies print 5604|135036|5037|1|44|47125.32|0.05|0.01|N|O|1998-08-06|1998-07-08|1998-09-04|NONE|RAIL|efully ironi 5604|135056|7570|2|49|53461.45|0.10|0.00|N|O|1998-05-02|1998-07-07|1998-05-20|NONE|FOB|ove the regula 5604|77004|9512|3|10|9810.00|0.07|0.05|N|O|1998-08-03|1998-06-23|1998-08-04|COLLECT COD|SHIP|ly final realms wake blit 5605|86747|1764|1|50|86687.00|0.08|0.05|N|O|1996-08-26|1996-10-15|1996-09-04|TAKE BACK RETURN|RAIL|instructions sleep carefully ironic req 5605|150710|711|2|7|12324.97|0.06|0.01|N|O|1996-12-13|1996-10-13|1996-12-15|TAKE BACK RETURN|FOB|lowly special courts nag among the furi 5605|172146|7181|3|3|3654.42|0.01|0.02|N|O|1996-09-01|1996-10-02|1996-09-20|TAKE BACK RETURN|AIR|posits. accounts boost. t 5605|54851|7357|4|45|81263.25|0.00|0.01|N|O|1996-09-05|1996-10-04|1996-09-13|COLLECT COD|FOB|ly unusual instructions. carefully ironic p 5605|69922|4935|5|39|73784.88|0.00|0.08|N|O|1996-12-13|1996-11-03|1996-12-24|DELIVER IN PERSON|REG AIR|cial deposits. theodolites w 5605|165991|5992|6|29|59652.71|0.08|0.08|N|O|1996-09-19|1996-10-22|1996-10-06|DELIVER IN PERSON|SHIP| quickly. quickly pending sen 5606|173945|3946|1|47|94890.18|0.10|0.04|N|O|1996-12-23|1997-01-31|1997-01-20|DELIVER IN PERSON|REG AIR|carefully final foxes. pending, final 5606|91684|1685|2|34|56973.12|0.09|0.06|N|O|1997-02-23|1997-02-08|1997-03-09|TAKE BACK RETURN|REG AIR|uses. slyly final 5606|126750|6751|3|46|81730.50|0.04|0.00|N|O|1997-03-11|1997-01-13|1997-03-23|DELIVER IN PERSON|REG AIR|ter the ironic accounts. even, ironic depos 5606|81657|1658|4|30|49159.50|0.08|0.04|N|O|1997-02-06|1997-01-26|1997-02-16|DELIVER IN PERSON|REG AIR| nag always. blithely express packages 5606|6317|1318|5|25|30582.75|0.06|0.00|N|O|1996-12-25|1997-01-12|1997-01-11|TAKE BACK RETURN|AIR|breach about the furiously bold 5606|153929|3930|6|3|5948.76|0.04|0.06|N|O|1997-01-11|1997-01-04|1997-02-08|COLLECT COD|AIR| sauternes. asympto 5606|73859|3860|7|46|84311.10|0.07|0.01|N|O|1997-02-01|1997-01-31|1997-02-15|DELIVER IN PERSON|TRUCK|ow requests wake around the regular accoun 5607|131532|4046|1|23|35961.19|0.02|0.06|R|F|1992-04-17|1992-02-12|1992-04-30|DELIVER IN PERSON|MAIL|the special, final patterns 5632|9074|1575|1|48|47187.36|0.06|0.06|N|O|1996-05-08|1996-03-24|1996-06-04|TAKE BACK RETURN|FOB|unts. decoys u 5632|105591|5592|2|21|33528.39|0.02|0.08|N|O|1996-03-22|1996-03-10|1996-04-10|NONE|AIR|refully regular pinto beans. ironic reques 5632|66291|3810|3|24|30174.96|0.04|0.06|N|O|1996-03-23|1996-04-02|1996-03-30|TAKE BACK RETURN|MAIL|beans detect. quickly final i 5633|159226|6772|1|28|35986.16|0.02|0.00|N|O|1998-08-14|1998-07-24|1998-08-26|TAKE BACK RETURN|SHIP|as boost quickly. unusual pinto 5633|101711|1712|2|10|17127.10|0.09|0.04|N|O|1998-07-15|1998-08-03|1998-08-03|COLLECT COD|AIR|its cajole fluffily fluffily special pinto 5633|45872|5873|3|27|49082.49|0.03|0.02|N|O|1998-09-28|1998-07-28|1998-10-12|DELIVER IN PERSON|AIR|ructions. even ideas haggle carefully r 5633|163653|3654|4|50|85832.50|0.02|0.05|N|O|1998-07-23|1998-07-09|1998-08-21|DELIVER IN PERSON|TRUCK|ts. slyly regular 5633|99323|1833|5|48|63471.36|0.01|0.05|N|O|1998-06-24|1998-07-22|1998-07-18|DELIVER IN PERSON|TRUCK|even courts haggle slyly at the requ 5633|106564|1585|6|1|1570.56|0.02|0.03|N|O|1998-09-29|1998-08-28|1998-10-19|NONE|RAIL|thely notornis: 5633|10697|3199|7|39|62699.91|0.02|0.08|N|O|1998-07-12|1998-07-03|1998-07-13|COLLECT COD|TRUCK|ding ideas cajole furiously after 5634|184279|1834|1|26|35445.02|0.10|0.08|N|O|1996-10-29|1996-09-15|1996-11-24|COLLECT COD|REG AIR|ptotes mold qu 5634|174875|2427|2|22|42897.14|0.02|0.05|N|O|1996-09-01|1996-08-31|1996-09-05|DELIVER IN PERSON|MAIL|silently unusual foxes above the blithely 5634|108573|6104|3|16|25305.12|0.08|0.02|N|O|1996-11-15|1996-09-14|1996-12-04|NONE|AIR|ess ideas are carefully pending, even re 5634|181044|6081|4|29|32626.16|0.00|0.01|N|O|1996-08-10|1996-10-29|1996-08-11|TAKE BACK RETURN|MAIL|ely final ideas. deposits sleep. reg 5634|309|310|5|1|1209.30|0.04|0.02|N|O|1996-10-02|1996-10-21|1996-10-27|COLLECT COD|MAIL|ctions haggle carefully. carefully clo 5635|82066|7083|1|43|45066.58|0.03|0.00|R|F|1992-10-12|1992-09-29|1992-11-01|TAKE BACK RETURN|TRUCK|cross the d 5635|71392|1393|2|5|6816.95|0.05|0.08|R|F|1992-10-02|1992-11-05|1992-10-26|TAKE BACK RETURN|REG AIR|yly along the ironic, fi 5635|71245|6260|3|12|14594.88|0.09|0.02|A|F|1992-10-18|1992-09-24|1992-11-17|NONE|REG AIR|ke slyly against the carefully final req 5635|7781|5282|4|40|67551.20|0.03|0.01|A|F|1992-09-25|1992-11-05|1992-10-11|NONE|FOB|pending foxes. regular packages 5635|168511|8512|5|38|60021.38|0.05|0.06|A|F|1992-10-09|1992-09-25|1992-10-18|NONE|MAIL|ckly pendin 5635|161513|6546|6|23|36213.73|0.05|0.04|A|F|1992-08-24|1992-11-10|1992-09-21|NONE|AIR|ily pending packages. bold, 5635|136198|8712|7|32|39494.08|0.03|0.08|R|F|1992-11-24|1992-09-20|1992-12-17|TAKE BACK RETURN|TRUCK|slyly even 5636|69821|2328|1|18|32234.76|0.05|0.03|R|F|1995-05-14|1995-05-17|1995-06-12|DELIVER IN PERSON|REG AIR|slyly express requests. furiously pen 5636|69120|6639|2|26|28317.12|0.03|0.06|A|F|1995-03-05|1995-05-16|1995-03-23|TAKE BACK RETURN|AIR| furiously final pinto beans o 5636|89315|9316|3|21|27390.51|0.03|0.03|A|F|1995-03-13|1995-05-11|1995-03-24|COLLECT COD|AIR| are furiously unusual 5636|108256|5787|4|15|18963.75|0.03|0.04|R|F|1995-04-21|1995-04-30|1995-05-05|DELIVER IN PERSON|REG AIR|efully special 5636|46791|9296|5|13|22591.27|0.10|0.03|A|F|1995-05-11|1995-04-27|1995-05-26|COLLECT COD|AIR|en, fluffy accounts amon 5636|11278|1279|6|33|39245.91|0.06|0.04|A|F|1995-03-09|1995-04-05|1995-03-23|DELIVER IN PERSON|MAIL|ding to the 5636|133257|5771|7|24|30966.00|0.10|0.05|R|F|1995-04-12|1995-03-27|1995-04-16|DELIVER IN PERSON|RAIL|counts sleep furiously b 5637|46670|9175|1|14|22633.38|0.03|0.05|N|O|1996-07-20|1996-07-26|1996-08-14|COLLECT COD|MAIL|y bold deposits wak 5637|171887|1888|2|35|68560.80|0.09|0.08|N|O|1996-08-01|1996-08-04|1996-08-20|NONE|AIR|s sleep blithely alongside of the ironic 5637|95346|2874|3|22|29509.48|0.01|0.07|N|O|1996-08-28|1996-07-30|1996-09-17|COLLECT COD|REG AIR|nding requests are ca 5637|65175|2694|4|16|18242.72|0.03|0.03|N|O|1996-09-08|1996-08-31|1996-09-29|TAKE BACK RETURN|TRUCK|d packages. express requests 5637|195700|5701|5|10|17957.00|0.01|0.00|N|O|1996-08-25|1996-08-11|1996-09-23|TAKE BACK RETURN|MAIL|ickly ironic gifts. blithely even cour 5637|128281|794|6|27|35350.56|0.01|0.05|N|O|1996-06-27|1996-08-09|1996-07-27|DELIVER IN PERSON|REG AIR|oss the carefully express warhorses 5638|137834|7835|1|45|84232.35|0.09|0.07|A|F|1994-05-17|1994-03-09|1994-06-15|NONE|TRUCK|ar foxes. fluffily pending accounts 5638|167678|5227|2|12|20948.04|0.02|0.05|A|F|1994-02-05|1994-04-01|1994-02-25|COLLECT COD|TRUCK|n, even requests. furiously ironic not 5638|161923|6956|3|21|41683.32|0.08|0.00|A|F|1994-03-13|1994-03-27|1994-03-17|DELIVER IN PERSON|TRUCK|press courts use f 5639|46609|1618|1|11|17111.60|0.09|0.02|R|F|1994-09-18|1994-07-10|1994-10-12|TAKE BACK RETURN|SHIP|g the unusual pinto beans caj 5664|121079|6104|1|25|27501.75|0.00|0.06|N|O|1998-10-29|1998-09-23|1998-11-25|COLLECT COD|FOB|eposits: furiously ironic grouch 5664|172408|7443|2|9|13323.60|0.07|0.05|N|O|1998-07-31|1998-08-26|1998-08-12|COLLECT COD|RAIL| ironic deposits haggle furiously. re 5664|52439|2440|3|31|43134.33|0.01|0.03|N|O|1998-11-10|1998-09-12|1998-12-07|TAKE BACK RETURN|FOB|ainst the never silent request 5664|137545|2572|4|33|52223.82|0.08|0.03|N|O|1998-08-29|1998-09-17|1998-09-25|DELIVER IN PERSON|RAIL|d the final 5664|111779|9313|5|44|78793.88|0.01|0.06|N|O|1998-09-24|1998-09-26|1998-10-23|NONE|TRUCK|ang thinly bold pa 5664|67733|2746|6|34|57824.82|0.09|0.01|N|O|1998-09-10|1998-10-05|1998-09-15|COLLECT COD|RAIL|st. fluffily pending foxes na 5664|181490|9045|7|9|14143.41|0.01|0.05|N|O|1998-11-04|1998-10-15|1998-11-20|TAKE BACK RETURN|REG AIR|yly. express ideas agai 5665|100935|936|1|32|61949.76|0.00|0.02|A|F|1993-08-11|1993-08-01|1993-09-07|NONE|AIR|f the slyly even requests! regular request 5665|4924|7425|2|14|25604.88|0.02|0.00|R|F|1993-06-29|1993-09-16|1993-07-16|DELIVER IN PERSON|AIR|- special pinto beans sleep quickly blithel 5665|157104|7105|3|41|47605.10|0.09|0.02|A|F|1993-08-23|1993-09-22|1993-09-11|COLLECT COD|REG AIR| idle ideas across 5665|45118|127|4|47|49966.17|0.01|0.01|A|F|1993-10-06|1993-09-19|1993-11-01|NONE|RAIL|s mold fluffily. final deposits along the 5666|121226|8763|1|7|8730.54|0.09|0.08|R|F|1994-05-10|1994-04-06|1994-05-21|NONE|FOB| ideas. regular packag 5666|35783|790|2|14|24062.92|0.08|0.01|A|F|1994-02-27|1994-04-11|1994-03-06|DELIVER IN PERSON|TRUCK|lar deposits nag against the slyly final d 5666|192178|7217|3|39|49536.63|0.00|0.01|A|F|1994-05-13|1994-04-02|1994-06-12|DELIVER IN PERSON|TRUCK|the even, final foxes. quickly iron 5666|130643|5670|4|24|40167.36|0.07|0.01|R|F|1994-02-14|1994-03-09|1994-03-06|DELIVER IN PERSON|FOB|on the carefully pending asympto 5666|108483|8484|5|36|53693.28|0.07|0.07|R|F|1994-03-15|1994-03-16|1994-03-18|COLLECT COD|TRUCK|accounts. furiousl 5667|144431|1974|1|37|54590.91|0.09|0.06|N|O|1995-09-24|1995-09-17|1995-10-03|NONE|REG AIR|s cajole blit 5668|3645|8646|1|15|23229.60|0.03|0.04|A|F|1995-04-06|1995-05-12|1995-04-17|COLLECT COD|FOB| the express, pending requests. bo 5669|190847|848|1|7|13564.88|0.06|0.06|N|O|1996-06-19|1996-07-07|1996-07-11|COLLECT COD|SHIP|yly regular requests lose blithely. careful 5669|155814|3360|2|2|3739.62|0.06|0.07|N|O|1996-08-04|1996-06-15|1996-08-20|NONE|SHIP| blithely excuses. slyly 5669|157654|7655|3|40|68466.00|0.00|0.02|N|O|1996-08-30|1996-06-15|1996-09-07|TAKE BACK RETURN|FOB|ar accounts alongside of the final, p 5669|89245|9246|4|31|38261.44|0.04|0.05|N|O|1996-08-05|1996-06-10|1996-08-29|COLLECT COD|AIR|to beans against the regular depo 5669|139680|7220|5|30|51590.40|0.07|0.01|N|O|1996-07-14|1996-07-28|1996-08-10|TAKE BACK RETURN|TRUCK|l accounts. care 5670|89569|9570|1|27|42081.12|0.10|0.06|R|F|1993-05-09|1993-05-30|1993-06-06|TAKE BACK RETURN|REG AIR| ideas promise bli 5670|185257|7776|2|43|57716.75|0.06|0.00|A|F|1993-07-09|1993-06-03|1993-07-14|DELIVER IN PERSON|FOB|ests in place of the carefully sly depos 5670|6482|6483|3|24|33323.52|0.09|0.04|A|F|1993-07-17|1993-07-01|1993-08-03|NONE|AIR|press, express requests haggle 5670|141383|3898|4|11|15668.18|0.06|0.06|R|F|1993-07-11|1993-06-26|1993-07-24|DELIVER IN PERSON|MAIL|etect furiously among the even pin 5671|119482|4505|1|25|37537.00|0.00|0.08|N|O|1998-04-17|1998-03-28|1998-05-06|DELIVER IN PERSON|AIR|cording to the quickly final requests-- 5671|128838|3863|2|46|85874.18|0.05|0.08|N|O|1998-03-28|1998-04-22|1998-04-19|TAKE BACK RETURN|MAIL|lar pinto beans detect care 5671|171340|8892|3|13|18347.42|0.10|0.06|N|O|1998-03-02|1998-04-03|1998-03-08|TAKE BACK RETURN|TRUCK|bold theodolites about 5671|110212|7746|4|42|51332.82|0.00|0.07|N|O|1998-02-17|1998-04-24|1998-03-17|TAKE BACK RETURN|SHIP|carefully slyly special deposit 5671|128097|610|5|13|14626.17|0.09|0.00|N|O|1998-04-24|1998-03-26|1998-04-27|NONE|REG AIR|ers according to the ironic, unusual excu 5671|113340|8363|6|30|40600.20|0.09|0.07|N|O|1998-06-06|1998-04-15|1998-07-01|DELIVER IN PERSON|TRUCK|fily ironi 5696|136316|3856|1|28|37864.68|0.03|0.06|N|O|1995-07-03|1995-06-14|1995-07-27|COLLECT COD|REG AIR| the fluffily brave pearls 5696|58914|6430|2|46|86153.86|0.01|0.00|N|O|1995-08-10|1995-07-08|1995-08-25|COLLECT COD|AIR|ter the instruct 5696|166282|3831|3|42|56627.76|0.04|0.01|N|F|1995-06-06|1995-06-11|1995-06-19|TAKE BACK RETURN|SHIP|te furious 5696|97646|156|4|20|32872.80|0.08|0.00|N|O|1995-06-25|1995-07-18|1995-07-16|NONE|TRUCK|silent, pending ideas sleep fluffil 5696|123505|6018|5|19|29041.50|0.07|0.05|N|O|1995-08-31|1995-06-13|1995-09-10|COLLECT COD|SHIP|unusual requests sleep furiously ru 5696|131928|9468|6|37|72517.04|0.04|0.05|N|O|1995-07-21|1995-06-23|1995-08-19|NONE|RAIL| carefully expres 5696|101569|9100|7|6|9423.36|0.07|0.05|N|O|1995-08-03|1995-07-15|1995-09-01|DELIVER IN PERSON|REG AIR|n patterns lose slyly fina 5697|54193|1709|1|24|27532.56|0.10|0.07|R|F|1992-10-27|1992-11-28|1992-11-20|NONE|RAIL|uffily iro 5697|15283|7785|2|43|51526.04|0.06|0.02|R|F|1992-12-08|1992-12-03|1992-12-17|TAKE BACK RETURN|FOB|blithely reg 5697|55460|2976|3|42|59449.32|0.03|0.01|A|F|1992-12-19|1992-12-08|1993-01-03|COLLECT COD|TRUCK|inal theodolites cajole after the bli 5698|10470|5473|1|30|41414.10|0.01|0.05|A|F|1994-05-26|1994-08-16|1994-06-19|COLLECT COD|AIR|its. quickly regular foxes aro 5698|162073|2074|2|25|28376.75|0.08|0.07|R|F|1994-08-06|1994-06-21|1994-08-25|NONE|SHIP| asymptotes sleep slyly above the 5698|154975|7491|3|45|91348.65|0.03|0.01|A|F|1994-06-23|1994-08-13|1994-07-02|NONE|FOB|ng excuses. slyly express asymptotes 5698|57104|9610|4|15|15916.50|0.07|0.08|R|F|1994-06-29|1994-07-03|1994-07-02|COLLECT COD|REG AIR|ly ironic frets haggle carefully 5698|139406|9407|5|37|53479.80|0.06|0.06|A|F|1994-06-30|1994-06-23|1994-07-22|TAKE BACK RETURN|SHIP|ts. even, ironic 5698|187630|149|6|1|1717.63|0.06|0.04|R|F|1994-05-31|1994-07-10|1994-06-03|DELIVER IN PERSON|MAIL|nts. slyly quiet pinto beans nag carefu 5699|1884|6885|1|24|42861.12|0.01|0.07|A|F|1992-10-21|1992-09-04|1992-11-04|COLLECT COD|AIR|kages. fin 5699|54419|9430|2|26|35708.66|0.06|0.06|R|F|1992-08-11|1992-09-21|1992-08-14|COLLECT COD|MAIL|y final deposits wake fluffily u 5699|17473|9975|3|48|66742.56|0.10|0.05|R|F|1992-11-23|1992-10-20|1992-11-29|DELIVER IN PERSON|TRUCK|s. carefully regul 5699|54313|6819|4|46|58296.26|0.08|0.02|A|F|1992-11-28|1992-09-23|1992-12-27|TAKE BACK RETURN|FOB|o the slyly 5699|27286|2291|5|21|25478.88|0.02|0.02|A|F|1992-10-13|1992-09-30|1992-10-19|NONE|MAIL|lyly final pla 5699|190036|7594|6|30|33780.90|0.08|0.05|R|F|1992-11-13|1992-10-01|1992-12-11|DELIVER IN PERSON|AIR| the carefully final 5699|128219|3244|7|45|56124.45|0.09|0.06|A|F|1992-09-23|1992-10-22|1992-10-04|DELIVER IN PERSON|SHIP|rmanent packages sleep across the f 5700|167890|2923|1|24|46989.36|0.09|0.00|N|O|1997-12-26|1998-01-28|1998-01-18|DELIVER IN PERSON|REG AIR|ix carefully 5700|122421|4934|2|30|43302.60|0.00|0.06|N|O|1998-04-19|1998-03-13|1998-04-27|COLLECT COD|MAIL|ly blithely final instructions. fl 5700|125320|345|3|23|30942.36|0.03|0.05|N|O|1998-01-30|1998-01-31|1998-01-31|NONE|REG AIR| wake quickly carefully fluffy hockey 5701|53487|5993|1|17|24488.16|0.02|0.05|N|O|1997-03-27|1997-04-08|1997-04-21|DELIVER IN PERSON|RAIL|tes. quickly final a 5702|76997|9505|1|44|86855.56|0.06|0.02|R|F|1994-01-04|1993-11-25|1994-01-22|NONE|RAIL|lites. carefully final requests doze b 5702|85532|5533|2|37|56148.61|0.10|0.05|R|F|1993-12-14|1993-10-21|1994-01-08|NONE|FOB|ix slyly. regular instructions slee 5702|130640|8180|3|44|73508.16|0.00|0.02|R|F|1993-11-28|1993-12-02|1993-12-22|NONE|TRUCK|ake according to th 5702|62204|9723|4|31|36152.20|0.00|0.04|A|F|1994-01-04|1993-10-22|1994-01-26|DELIVER IN PERSON|TRUCK|pinto beans. blithely 5703|87698|207|1|2|3371.38|0.09|0.01|R|F|1993-05-29|1993-07-26|1993-06-05|TAKE BACK RETURN|REG AIR|nts against the blithely sile 5728|43599|6104|1|47|72501.73|0.10|0.05|A|F|1994-12-13|1995-01-25|1994-12-25|TAKE BACK RETURN|MAIL|nd the bravely final deposits. final ideas 5728|158786|6332|2|40|73791.20|0.05|0.08|A|F|1995-03-28|1995-01-17|1995-04-14|TAKE BACK RETURN|SHIP|final deposits. theodolite 5729|142388|2389|1|5|7151.90|0.07|0.00|R|F|1994-11-27|1994-11-11|1994-12-23|TAKE BACK RETURN|MAIL|s. even sheaves nag courts. 5729|106136|8647|2|39|44543.07|0.10|0.00|A|F|1995-01-22|1994-11-21|1995-02-13|TAKE BACK RETURN|MAIL|. special pl 5729|11181|1182|3|50|54609.00|0.00|0.05|R|F|1994-12-09|1994-12-31|1994-12-24|TAKE BACK RETURN|AIR|ly special sentiments. car 5730|150009|10|1|2|2118.00|0.08|0.00|N|O|1998-02-24|1998-03-15|1998-03-11|COLLECT COD|SHIP|ely ironic foxes. carefu 5730|199690|9691|2|9|16107.21|0.10|0.01|N|O|1998-03-05|1998-02-02|1998-03-28|DELIVER IN PERSON|MAIL|s lose blithely. specia 5731|191796|9354|1|13|24541.27|0.02|0.04|N|O|1997-07-30|1997-06-23|1997-08-13|COLLECT COD|RAIL|ngside of the quickly regular depos 5731|104178|4179|2|11|13003.87|0.00|0.08|N|O|1997-06-06|1997-07-08|1997-06-25|NONE|MAIL| furiously final accounts wake. d 5731|110812|813|3|6|10936.86|0.01|0.04|N|O|1997-07-02|1997-07-01|1997-07-08|COLLECT COD|SHIP|sits integrate slyly close platelets. quick 5731|13471|8474|4|6|8306.82|0.03|0.06|N|O|1997-09-07|1997-06-20|1997-09-20|TAKE BACK RETURN|RAIL|rs. quickly regular theo 5731|194960|4961|5|19|39044.24|0.08|0.02|N|O|1997-06-29|1997-06-27|1997-07-15|NONE|REG AIR|ly unusual ideas above the 5732|138287|5827|1|26|34457.28|0.02|0.07|N|O|1997-08-18|1997-10-25|1997-09-12|TAKE BACK RETURN|TRUCK|totes cajole according to the theodolites. 5733|32868|2869|1|39|70233.54|0.01|0.07|A|F|1993-03-22|1993-05-24|1993-04-04|DELIVER IN PERSON|FOB|side of the 5734|182945|5464|1|29|58810.26|0.05|0.01|N|O|1997-12-01|1997-12-08|1997-12-23|NONE|RAIL|structions cajole final, express 5734|149299|4328|2|6|8089.74|0.07|0.00|N|O|1997-10-27|1997-12-19|1997-11-02|COLLECT COD|RAIL|s. regular platelets cajole furiously. regu 5734|66727|6728|3|10|16937.20|0.01|0.03|N|O|1997-12-28|1997-12-24|1998-01-24|DELIVER IN PERSON|TRUCK|equests; accounts above 5735|59754|9755|1|41|70263.75|0.01|0.01|R|F|1994-12-23|1995-02-10|1995-01-22|COLLECT COD|MAIL|lthily ruthless i 5760|667|8168|1|6|9405.96|0.09|0.03|R|F|1994-07-30|1994-07-31|1994-08-16|COLLECT COD|REG AIR|ng the acco 5760|5757|758|2|24|39906.00|0.04|0.05|A|F|1994-07-15|1994-07-04|1994-08-08|NONE|MAIL|s. bravely ironic accounts among 5760|147375|9890|3|8|11378.96|0.07|0.04|A|F|1994-09-06|1994-08-03|1994-10-06|NONE|AIR|l accounts among the carefully even de 5760|122293|2294|4|19|24990.51|0.10|0.01|R|F|1994-08-02|1994-08-02|1994-08-15|COLLECT COD|SHIP|sits nag. even, regular ideas cajole b 5760|165638|3187|5|6|10221.78|0.03|0.07|R|F|1994-06-09|1994-07-06|1994-06-16|DELIVER IN PERSON|MAIL| shall have to cajole along the 5761|46273|3786|1|41|49990.07|0.08|0.00|N|O|1998-07-31|1998-08-09|1998-08-08|TAKE BACK RETURN|TRUCK|pecial deposits. qu 5761|107289|7290|2|36|46666.08|0.00|0.07|N|O|1998-09-07|1998-09-21|1998-09-11|TAKE BACK RETURN|TRUCK| pinto beans thrash alongside of the pendi 5761|197395|4953|3|49|73127.11|0.04|0.08|N|O|1998-07-14|1998-08-20|1998-07-25|NONE|SHIP|ly bold accounts wake above the 5762|174993|4994|1|6|12407.94|0.05|0.02|N|O|1997-04-07|1997-03-25|1997-05-02|NONE|AIR|ironic dependencies doze carefu 5762|101749|9280|2|27|47269.98|0.02|0.08|N|O|1997-02-21|1997-05-08|1997-03-23|NONE|REG AIR|across the bold ideas. carefully sp 5762|88317|8318|3|40|52212.40|0.00|0.08|N|O|1997-04-30|1997-05-09|1997-05-08|COLLECT COD|SHIP|al instructions. furiousl 5762|132375|2376|4|47|66146.39|0.05|0.06|N|O|1997-03-02|1997-03-23|1997-03-19|NONE|RAIL|equests sleep after the furiously ironic pa 5762|24527|4528|5|28|40642.56|0.02|0.06|N|O|1997-02-22|1997-03-25|1997-02-24|TAKE BACK RETURN|SHIP|ic foxes among the blithely qui 5762|11673|4175|6|12|19016.04|0.00|0.06|N|O|1997-04-18|1997-04-27|1997-05-11|DELIVER IN PERSON|REG AIR|ages are abo 5763|130799|5826|1|32|58553.28|0.02|0.06|N|O|1998-07-16|1998-09-13|1998-08-02|DELIVER IN PERSON|FOB|ding instruct 5763|135695|8209|2|23|39805.87|0.09|0.04|N|O|1998-07-25|1998-09-21|1998-08-15|DELIVER IN PERSON|SHIP|re after the blithel 5763|12360|9864|3|25|31809.00|0.01|0.02|N|O|1998-10-04|1998-08-16|1998-10-09|DELIVER IN PERSON|REG AIR|inal theodolites. even re 5763|120969|3482|4|47|93528.12|0.09|0.00|N|O|1998-08-22|1998-09-22|1998-09-04|NONE|REG AIR|gle slyly. slyly final re 5763|122996|2997|5|8|16151.92|0.06|0.05|N|O|1998-09-23|1998-09-15|1998-09-27|DELIVER IN PERSON|TRUCK|foxes wake slyly. car 5763|189104|9105|6|9|10737.90|0.08|0.02|N|O|1998-09-24|1998-09-01|1998-10-02|NONE|AIR| deposits. instru 5764|100625|626|1|28|45517.36|0.04|0.04|A|F|1993-12-07|1993-12-20|1993-12-26|TAKE BACK RETURN|RAIL|sleep furi 5764|199143|4182|2|20|24842.80|0.10|0.05|A|F|1993-10-17|1993-12-24|1993-10-18|TAKE BACK RETURN|FOB|ng to the fluffily qu 5764|187173|4728|3|4|5040.68|0.03|0.05|A|F|1993-10-25|1993-12-23|1993-11-06|DELIVER IN PERSON|AIR|ily regular courts haggle 5765|161940|9489|1|31|62060.14|0.00|0.06|A|F|1995-01-11|1995-02-13|1995-01-23|TAKE BACK RETURN|AIR|r foxes. ev 5765|123802|6315|2|29|52948.20|0.07|0.08|A|F|1994-12-29|1995-02-01|1995-01-26|NONE|RAIL|nic requests. deposits wake quickly among 5765|138309|3336|3|31|41766.30|0.05|0.01|R|F|1995-03-01|1995-01-23|1995-03-31|TAKE BACK RETURN|REG AIR|the furiou 5765|151733|9279|4|46|82097.58|0.07|0.07|R|F|1995-03-13|1995-02-12|1995-03-20|DELIVER IN PERSON|MAIL|ccounts sleep about th 5765|173421|8456|5|48|71732.16|0.09|0.02|A|F|1995-03-30|1995-01-14|1995-04-09|DELIVER IN PERSON|SHIP|theodolites integrate furiously 5765|82052|7069|6|41|42396.05|0.04|0.00|A|F|1994-12-31|1995-02-11|1995-01-17|TAKE BACK RETURN|SHIP| furiously. slyly sile 5765|41209|6218|7|21|24154.20|0.05|0.04|R|F|1995-04-05|1995-02-12|1995-05-05|COLLECT COD|TRUCK|ole furiously. quick, special dependencies 5766|187050|9569|1|1|1137.05|0.10|0.01|R|F|1994-01-16|1993-11-16|1994-01-23|NONE|MAIL|blithely regular the 5766|148529|6072|2|39|61523.28|0.02|0.07|A|F|1993-10-24|1993-12-07|1993-11-08|DELIVER IN PERSON|SHIP| furiously unusual courts. slyly final pear 5766|117751|5285|3|4|7075.00|0.08|0.08|R|F|1993-11-10|1993-10-30|1993-12-01|COLLECT COD|TRUCK|ly even requests. furiou 5767|166864|6865|1|11|21239.46|0.08|0.01|A|F|1992-06-02|1992-05-30|1992-06-08|NONE|TRUCK|instructions. carefully final accou 5767|68555|1062|2|15|22853.25|0.07|0.05|R|F|1992-06-05|1992-07-28|1992-06-08|DELIVER IN PERSON|MAIL|warthogs. carefully unusual g 5767|190490|3010|3|42|66380.58|0.06|0.01|R|F|1992-07-31|1992-06-09|1992-08-09|COLLECT COD|TRUCK| blithe deposi 5767|152475|2476|4|34|51933.98|0.06|0.01|R|F|1992-06-02|1992-06-23|1992-06-17|NONE|FOB|sits among the 5767|45270|5271|5|36|43749.72|0.03|0.00|A|F|1992-07-17|1992-06-10|1992-07-19|COLLECT COD|AIR|ake carefully. packages 5792|177019|9537|1|34|37264.34|0.08|0.07|R|F|1993-05-23|1993-06-25|1993-06-12|NONE|RAIL|requests are against t 5792|156410|8926|2|47|68921.27|0.10|0.00|A|F|1993-06-08|1993-05-10|1993-06-26|COLLECT COD|AIR|regular, ironic excuses n 5792|182806|2807|3|32|60441.60|0.05|0.08|R|F|1993-06-26|1993-05-23|1993-07-07|COLLECT COD|RAIL|s are slyly against the ev 5792|13894|6396|4|14|25310.46|0.09|0.02|A|F|1993-07-28|1993-06-17|1993-08-27|DELIVER IN PERSON|RAIL|olites print carefully 5792|101362|8893|5|31|42264.16|0.02|0.01|A|F|1993-06-17|1993-05-05|1993-07-01|COLLECT COD|TRUCK|s? furiously even instructions 5793|52084|9600|1|20|20721.60|0.05|0.03|N|O|1997-10-05|1997-09-04|1997-10-30|COLLECT COD|AIR|e carefully ex 5793|169992|7541|2|41|84541.59|0.06|0.06|N|O|1997-08-04|1997-10-10|1997-08-12|DELIVER IN PERSON|TRUCK|snooze quick 5793|42777|2778|3|8|13758.16|0.07|0.03|N|O|1997-08-16|1997-09-08|1997-08-28|COLLECT COD|AIR|al foxes l 5793|147676|5219|4|48|82736.16|0.02|0.02|N|O|1997-09-27|1997-08-23|1997-10-27|DELIVER IN PERSON|REG AIR|quickly enticing excuses use slyly abov 5794|157144|7145|1|42|50447.88|0.06|0.05|R|F|1993-06-29|1993-05-30|1993-07-28|COLLECT COD|REG AIR|he careful 5794|114786|9809|2|14|25210.92|0.09|0.02|R|F|1993-04-19|1993-07-02|1993-05-18|COLLECT COD|SHIP|uriously carefully ironic reque 5794|6827|6828|3|15|26007.30|0.09|0.06|R|F|1993-06-25|1993-06-27|1993-07-09|NONE|MAIL|blithely regular ideas. final foxes haggle 5794|136244|8758|4|47|60171.28|0.00|0.08|A|F|1993-07-16|1993-06-21|1993-08-05|TAKE BACK RETURN|REG AIR|quests. blithely final excu 5795|192991|8030|1|34|70855.66|0.09|0.05|A|F|1992-08-21|1992-07-30|1992-08-27|COLLECT COD|REG AIR|al instructions must affix along the ironic 5796|57971|2982|1|27|52082.19|0.10|0.00|N|O|1996-04-06|1996-02-29|1996-04-20|DELIVER IN PERSON|RAIL|s wake quickly aro 5797|60967|8486|1|17|32775.32|0.09|0.03|N|O|1997-12-13|1998-01-12|1997-12-23|NONE|REG AIR|the ironic, even theodoli 5798|126015|6016|1|2|2082.02|0.09|0.00|N|O|1998-05-25|1998-06-22|1998-06-09|COLLECT COD|FOB|e furiously across 5798|123814|6327|2|14|25729.34|0.06|0.05|N|O|1998-04-01|1998-06-14|1998-04-27|NONE|RAIL|he special, bold packages. carefully iron 5798|133608|8635|3|22|36115.20|0.02|0.01|N|O|1998-06-24|1998-06-06|1998-07-20|COLLECT COD|TRUCK|sits poach carefully 5798|145699|8214|4|40|69787.60|0.08|0.06|N|O|1998-07-09|1998-06-24|1998-07-16|NONE|TRUCK| integrate carefu 5798|148631|6174|5|7|11757.41|0.06|0.07|N|O|1998-06-06|1998-05-10|1998-06-07|NONE|SHIP|ts against the blithely final p 5798|37444|4954|6|9|12432.96|0.06|0.02|N|O|1998-05-05|1998-05-25|1998-05-09|DELIVER IN PERSON|REG AIR|e blithely 5798|114851|7363|7|32|59707.20|0.08|0.01|N|O|1998-04-27|1998-05-03|1998-05-08|TAKE BACK RETURN|REG AIR|ubt blithely above the 5799|94586|4587|1|41|64803.78|0.04|0.02|N|O|1995-11-13|1995-10-31|1995-11-16|COLLECT COD|TRUCK|al accounts sleep ruthlessl 5799|99882|4901|2|30|56456.40|0.03|0.08|N|O|1995-09-12|1995-09-13|1995-09-19|NONE|RAIL| furiously s 5824|76111|8619|1|40|43484.40|0.06|0.06|N|O|1997-01-14|1997-01-17|1997-02-02|NONE|REG AIR|he final packag 5824|181001|6038|2|42|45444.00|0.09|0.00|N|O|1997-02-01|1997-02-20|1997-02-07|COLLECT COD|SHIP|ts sleep. carefully regular accounts h 5824|72131|9653|3|16|17650.08|0.03|0.02|N|O|1997-02-13|1997-01-07|1997-02-17|TAKE BACK RETURN|TRUCK|sly express Ti 5824|91100|6119|4|32|34915.20|0.03|0.02|N|O|1997-02-16|1997-01-24|1997-02-20|DELIVER IN PERSON|RAIL|ven requests. 5824|107000|7001|5|44|44308.00|0.08|0.03|N|O|1997-01-24|1997-01-31|1997-02-11|COLLECT COD|TRUCK|fily fluffily bold 5825|158214|730|1|23|29260.83|0.10|0.05|R|F|1995-05-10|1995-04-28|1995-05-13|DELIVER IN PERSON|TRUCK| special pinto beans. dependencies haggl 5826|143968|6483|1|4|8047.84|0.03|0.06|N|O|1998-07-31|1998-09-10|1998-08-27|NONE|AIR| packages across the fluffily spec 5826|63777|3778|2|18|31333.86|0.04|0.01|N|O|1998-07-17|1998-09-03|1998-07-22|NONE|SHIP|atelets use above t 5827|186619|6620|1|30|51168.30|0.03|0.05|N|O|1998-11-11|1998-09-27|1998-11-30|DELIVER IN PERSON|RAIL|ounts may c 5827|102134|4645|2|23|26130.99|0.09|0.05|N|O|1998-11-16|1998-09-14|1998-11-17|COLLECT COD|RAIL|ans. furiously special instruct 5827|163706|8739|3|3|5309.10|0.03|0.06|N|O|1998-10-17|1998-09-29|1998-10-28|DELIVER IN PERSON|MAIL|uses eat along the furiously 5827|199067|9068|4|26|30317.56|0.06|0.00|N|O|1998-07-29|1998-09-24|1998-07-30|DELIVER IN PERSON|SHIP|arefully special packages wake thin 5827|111610|6633|5|38|61621.18|0.03|0.06|N|O|1998-10-18|1998-08-27|1998-10-23|TAKE BACK RETURN|TRUCK|ly ruthless accounts 5827|16474|1477|6|14|19466.58|0.05|0.01|N|O|1998-08-31|1998-09-06|1998-09-13|TAKE BACK RETURN|RAIL|rges. fluffily pending 5828|1343|8844|1|28|34841.52|0.10|0.03|A|F|1994-05-15|1994-05-20|1994-06-08|DELIVER IN PERSON|MAIL| special ideas haggle slyly ac 5828|157303|2334|2|37|50331.10|0.01|0.00|R|F|1994-06-07|1994-05-30|1994-06-17|NONE|RAIL|e carefully spec 5829|39657|4664|1|4|6386.60|0.01|0.02|N|O|1997-03-01|1997-02-17|1997-03-22|NONE|TRUCK|ithely; accounts cajole ideas. regular foxe 5829|106523|9034|2|40|61180.80|0.04|0.01|N|O|1997-04-21|1997-02-12|1997-05-04|COLLECT COD|TRUCK| the carefully ironic accounts. a 5829|128238|3263|3|6|7597.38|0.05|0.06|N|O|1997-01-22|1997-03-12|1997-02-02|TAKE BACK RETURN|AIR|sts. slyly special fo 5829|89153|4170|4|42|47970.30|0.02|0.07|N|O|1997-03-26|1997-04-01|1997-03-30|COLLECT COD|REG AIR|pearls. slyly bold deposits solve final 5829|190779|8337|5|49|91618.73|0.05|0.01|N|O|1997-01-31|1997-03-13|1997-02-18|NONE|MAIL| ironic excuses use fluf 5829|17069|2072|6|17|16763.02|0.09|0.02|N|O|1997-04-10|1997-03-29|1997-04-22|COLLECT COD|AIR|after the furiously ironic ideas no 5829|77942|7943|7|27|51838.38|0.08|0.04|N|O|1997-02-25|1997-03-31|1997-03-03|DELIVER IN PERSON|AIR|ns about the excuses are c 5830|159261|6807|1|29|38287.54|0.10|0.02|R|F|1993-06-19|1993-05-10|1993-07-13|DELIVER IN PERSON|REG AIR|y bold excuses 5831|190330|331|1|2|2840.66|0.10|0.01|N|O|1997-02-09|1997-01-20|1997-03-07|TAKE BACK RETURN|TRUCK|quickly silent req 5831|73475|8490|2|33|47799.51|0.04|0.03|N|O|1996-11-20|1997-01-18|1996-12-18|TAKE BACK RETURN|MAIL| instructions wake. slyly sil 5831|81622|4131|3|6|9621.72|0.05|0.07|N|O|1997-01-29|1997-01-14|1997-02-09|NONE|MAIL|ly ironic accounts nag pendin 5831|12962|7965|4|46|86248.16|0.06|0.02|N|O|1997-02-24|1997-01-18|1997-03-02|COLLECT COD|MAIL|ly final pa 5831|42828|2829|5|37|65520.34|0.05|0.01|N|O|1997-01-17|1997-02-08|1997-02-01|NONE|FOB|uriously even requests 5856|3680|1181|1|1|1583.68|0.03|0.02|A|F|1994-12-29|1995-01-07|1995-01-10|TAKE BACK RETURN|MAIL|tly. special deposits wake blithely even 5856|34305|9312|2|35|43375.50|0.09|0.02|R|F|1994-11-24|1994-12-23|1994-11-30|COLLECT COD|AIR|excuses. finally ir 5856|152739|2740|3|39|69877.47|0.05|0.03|A|F|1995-01-18|1995-01-11|1995-01-19|DELIVER IN PERSON|TRUCK|uickly quickly fluffy in 5857|57226|7227|1|25|29580.50|0.03|0.02|N|O|1997-12-02|1997-12-17|1997-12-08|DELIVER IN PERSON|REG AIR|ding platelets. pending excu 5857|194769|2327|2|50|93188.00|0.06|0.07|N|O|1997-12-04|1997-12-16|1997-12-20|NONE|TRUCK|y regular d 5857|67860|5379|3|1|1827.86|0.03|0.01|N|O|1998-02-01|1997-12-09|1998-02-20|TAKE BACK RETURN|SHIP|instructions detect final reques 5857|117998|510|4|12|24191.88|0.03|0.08|N|O|1998-01-24|1997-12-27|1998-02-10|TAKE BACK RETURN|AIR|counts. express, final 5857|191260|3780|5|14|18917.64|0.07|0.07|N|O|1997-12-10|1998-01-06|1998-01-04|TAKE BACK RETURN|TRUCK|ffily pendin 5857|92862|5372|6|49|90888.14|0.00|0.04|N|O|1998-01-23|1997-12-12|1998-01-28|DELIVER IN PERSON|REG AIR|egular pinto beans 5858|120832|8369|1|20|37056.60|0.02|0.06|A|F|1992-07-23|1992-08-26|1992-07-24|COLLECT COD|SHIP|uffily unusual pinto beans sleep 5858|15003|5004|2|36|33048.00|0.00|0.05|A|F|1992-09-25|1992-08-16|1992-10-11|NONE|SHIP|osits wake quickly quickly sile 5858|147996|511|3|7|14307.93|0.08|0.02|A|F|1992-10-07|1992-08-16|1992-10-15|TAKE BACK RETURN|REG AIR|. doggedly regular packages use pendin 5858|163490|1039|4|46|71460.54|0.07|0.06|R|F|1992-09-07|1992-10-06|1992-10-06|DELIVER IN PERSON|MAIL|posits withi 5858|160181|5214|5|18|22341.24|0.00|0.07|A|F|1992-11-05|1992-10-08|1992-12-03|NONE|TRUCK|al excuses. bold 5858|153937|8968|6|7|13936.51|0.04|0.00|A|F|1992-09-14|1992-10-01|1992-10-01|TAKE BACK RETURN|RAIL|dly pending ac 5858|10709|3211|7|50|80985.00|0.06|0.00|R|F|1992-07-20|1992-10-07|1992-07-25|NONE|TRUCK|r the ironic ex 5859|174988|23|1|50|103149.00|0.07|0.01|N|O|1997-07-08|1997-06-20|1997-07-27|COLLECT COD|MAIL|ly regular deposits use. ironic 5859|8773|6274|2|17|28590.09|0.03|0.03|N|O|1997-05-15|1997-06-30|1997-05-26|DELIVER IN PERSON|AIR|ly ironic requests. quickly unusual pin 5859|45701|8206|3|33|54341.10|0.10|0.04|N|O|1997-07-08|1997-06-22|1997-07-18|TAKE BACK RETURN|TRUCK|eposits unwind furiously final pinto bea 5859|92269|2270|4|40|50450.40|0.09|0.02|N|O|1997-08-05|1997-06-17|1997-08-20|NONE|REG AIR|l dependenci 5859|152689|7720|5|35|60958.80|0.00|0.08|N|O|1997-05-28|1997-07-14|1997-06-15|COLLECT COD|TRUCK|egular acco 5859|43446|3447|6|9|12504.96|0.01|0.02|N|O|1997-06-15|1997-06-06|1997-06-20|NONE|RAIL|ges boost quickly. blithely r 5859|190127|7685|7|27|32862.24|0.05|0.08|N|O|1997-07-30|1997-07-08|1997-08-08|NONE|MAIL| across th 5860|50168|7684|1|10|11181.60|0.04|0.04|A|F|1992-03-11|1992-03-30|1992-03-31|NONE|MAIL|ual patterns try to eat carefully above 5861|190867|8425|1|32|62651.52|0.00|0.03|N|O|1997-05-27|1997-05-29|1997-05-28|TAKE BACK RETURN|MAIL|nt asymptotes. carefully express request 5861|85611|628|2|6|9579.66|0.10|0.03|N|O|1997-07-28|1997-05-18|1997-08-24|TAKE BACK RETURN|TRUCK|olites. slyly 5862|112061|4573|1|4|4292.24|0.09|0.06|N|O|1997-06-04|1997-04-26|1997-06-19|NONE|TRUCK|yly silent deposit 5862|1334|6335|2|29|35824.57|0.03|0.05|N|O|1997-04-02|1997-04-16|1997-04-04|NONE|FOB|e fluffily. furiously 5863|160562|3079|1|45|73015.20|0.07|0.06|A|F|1993-12-19|1994-01-25|1994-01-05|NONE|REG AIR| deposits are ab 5863|159369|1885|2|21|29995.56|0.09|0.03|R|F|1994-01-13|1994-01-09|1994-01-28|DELIVER IN PERSON|FOB|atelets nag blithely furi 5888|61291|8810|1|46|57605.34|0.02|0.00|N|O|1996-11-18|1996-11-05|1996-12-08|TAKE BACK RETURN|FOB|yly final accounts hag 5888|111595|1596|2|24|38558.16|0.03|0.01|N|O|1996-11-07|1996-11-30|1996-11-20|COLLECT COD|SHIP|ing to the spe 5889|76129|8637|1|17|18787.04|0.09|0.02|N|O|1995-07-01|1995-08-12|1995-07-25|NONE|AIR|blithely pending packages. flu 5890|112221|2222|1|38|46862.36|0.01|0.08|A|F|1993-02-14|1992-12-09|1993-02-27|COLLECT COD|FOB| accounts. carefully final asymptotes 5891|84905|4906|1|22|41577.80|0.00|0.06|R|F|1993-01-01|1993-02-18|1993-01-14|DELIVER IN PERSON|TRUCK|iresias cajole deposits. special, ir 5891|185580|8099|2|9|14990.22|0.03|0.07|R|F|1993-01-20|1993-02-27|1993-02-10|COLLECT COD|REG AIR|cajole carefully 5891|29674|4679|3|10|16036.70|0.08|0.01|A|F|1993-04-14|1993-02-07|1993-04-15|DELIVER IN PERSON|RAIL|nding requests. b 5892|147722|7723|1|7|12388.04|0.02|0.03|N|O|1995-06-26|1995-07-18|1995-07-25|COLLECT COD|AIR|e furiously. quickly even deposits da 5892|149926|7469|2|37|73109.04|0.09|0.06|N|O|1995-08-12|1995-06-11|1995-09-05|NONE|REG AIR|maintain. bold, expre 5892|2064|2065|3|28|27049.68|0.03|0.06|N|O|1995-08-16|1995-07-06|1995-08-22|DELIVER IN PERSON|MAIL|ithely unusual accounts will have to integ 5892|74798|4799|4|23|40774.17|0.08|0.04|R|F|1995-05-18|1995-07-06|1995-05-29|COLLECT COD|MAIL| foxes nag slyly about the qui 5893|133707|1247|1|43|74850.10|0.05|0.02|R|F|1992-11-02|1992-09-27|1992-11-21|TAKE BACK RETURN|RAIL|s. regular courts above the carefully silen 5893|1868|9369|2|2|3539.72|0.10|0.04|R|F|1992-07-18|1992-09-10|1992-08-12|NONE|RAIL|ckages wake sly 5894|7312|4813|1|23|28044.13|0.04|0.08|A|F|1994-09-05|1994-10-27|1994-09-13|NONE|TRUCK| furiously even deposits haggle alw 5894|78446|3461|2|48|68373.12|0.04|0.08|A|F|1994-09-04|1994-11-03|1994-09-17|NONE|TRUCK| asymptotes among the blithely silent 5895|14728|7230|1|38|62423.36|0.05|0.08|N|O|1997-04-05|1997-03-06|1997-05-03|DELIVER IN PERSON|RAIL|ts are furiously. regular, final excuses 5895|121538|1539|2|47|73297.91|0.04|0.06|N|O|1997-04-27|1997-03-17|1997-05-07|DELIVER IN PERSON|AIR|r packages wake carefull 5895|83712|1237|3|49|83089.79|0.03|0.07|N|O|1997-03-15|1997-02-17|1997-04-04|NONE|TRUCK|permanent foxes. packages 5895|145714|5715|4|31|54551.01|0.03|0.01|N|O|1997-03-03|1997-03-30|1997-03-08|TAKE BACK RETURN|TRUCK| final deposits nod slyly careful 5895|199172|9173|5|20|25423.40|0.07|0.00|N|O|1997-04-30|1997-02-07|1997-05-08|DELIVER IN PERSON|AIR|gular deposits wake blithely carefully fin 5895|77733|2748|6|15|25660.95|0.08|0.08|N|O|1997-04-19|1997-03-09|1997-05-13|TAKE BACK RETURN|RAIL|silent package 5920|186069|6070|1|50|57753.00|0.06|0.00|A|F|1995-03-13|1995-01-03|1995-03-31|TAKE BACK RETURN|RAIL|across the carefully pending platelets 5920|57793|7794|2|24|42018.96|0.01|0.05|A|F|1994-12-28|1995-01-21|1994-12-31|DELIVER IN PERSON|FOB|fully regular dolphins. furiousl 5920|116952|9464|3|2|3937.90|0.08|0.07|A|F|1995-02-18|1995-01-13|1995-03-04|NONE|SHIP| evenly spe 5920|11323|8827|4|28|34560.96|0.06|0.02|R|F|1994-12-17|1995-02-13|1994-12-31|NONE|SHIP|le slyly slyly even deposits. f 5920|99165|6693|5|42|48894.72|0.09|0.08|A|F|1994-12-18|1995-01-07|1995-01-14|COLLECT COD|AIR|lar, ironic dependencies sno 5921|98016|5544|1|44|44616.44|0.07|0.01|R|F|1994-07-14|1994-06-30|1994-07-15|NONE|TRUCK|ain about the special 5921|145670|699|2|25|42891.75|0.06|0.01|A|F|1994-05-19|1994-06-15|1994-06-17|COLLECT COD|TRUCK|nd the slyly regular deposits. quick 5921|67177|2190|3|17|19450.89|0.06|0.01|R|F|1994-05-20|1994-05-26|1994-05-23|NONE|FOB|final asymptotes. even packages boost 5921|27331|2336|4|26|32716.58|0.03|0.04|A|F|1994-05-03|1994-07-06|1994-05-06|NONE|AIR|hy dependenc 5921|142246|4761|5|41|52817.84|0.04|0.02|R|F|1994-04-13|1994-05-31|1994-04-26|DELIVER IN PERSON|AIR|nusual, regular theodol 5921|114421|4422|6|5|7177.10|0.02|0.00|R|F|1994-06-01|1994-05-07|1994-06-10|COLLECT COD|TRUCK|eas cajole across the final, fi 5922|195631|3189|1|9|15539.67|0.07|0.00|N|O|1996-12-04|1997-01-20|1996-12-08|DELIVER IN PERSON|RAIL|haggle slyly even packages. packages 5922|156595|1626|2|37|61108.83|0.01|0.04|N|O|1996-12-19|1996-12-16|1997-01-15|COLLECT COD|RAIL|s wake slyly. requests cajole furiously asy 5922|89515|4532|3|35|52657.85|0.08|0.00|N|O|1996-12-12|1997-01-21|1997-01-01|DELIVER IN PERSON|SHIP|accounts. regu 5922|65048|5049|4|13|13169.52|0.08|0.07|N|O|1997-03-08|1996-12-26|1997-04-03|DELIVER IN PERSON|FOB|sly special accounts wake ironically. 5922|56428|8934|5|39|53992.38|0.04|0.07|N|O|1997-03-04|1997-01-17|1997-03-25|TAKE BACK RETURN|SHIP|e of the instructions. quick 5922|178810|1328|6|10|18888.10|0.04|0.01|N|O|1997-02-23|1996-12-26|1997-03-04|NONE|REG AIR|sly regular deposits haggle quickly ins 5923|176934|6935|1|27|54295.11|0.08|0.03|N|O|1997-08-16|1997-06-27|1997-08-29|DELIVER IN PERSON|RAIL|arefully i 5923|118577|1089|2|42|67013.94|0.01|0.08|N|O|1997-09-16|1997-07-23|1997-09-27|COLLECT COD|REG AIR|y regular theodolites w 5923|107597|5128|3|2|3209.18|0.06|0.05|N|O|1997-06-19|1997-07-31|1997-06-28|TAKE BACK RETURN|TRUCK|express patterns. even deposits 5923|173158|5676|4|46|56632.90|0.05|0.04|N|O|1997-07-29|1997-07-23|1997-08-23|COLLECT COD|SHIP|nto beans cajole blithe 5923|58535|3546|5|35|52273.55|0.04|0.05|N|O|1997-07-21|1997-07-11|1997-08-01|DELIVER IN PERSON|AIR|sts affix unusual, final requests. request 5924|175145|180|1|38|46365.32|0.06|0.05|N|O|1995-12-17|1995-12-11|1996-01-06|TAKE BACK RETURN|AIR|ions cajole carefully along the 5924|52349|4855|2|49|63765.66|0.04|0.00|N|O|1995-10-25|1995-12-11|1995-11-08|NONE|MAIL|inly final excuses. blithely regular requ 5924|16705|6706|3|24|38920.80|0.09|0.08|N|O|1996-01-12|1995-12-13|1996-01-25|COLLECT COD|REG AIR| use carefully. special, e 5925|86543|9052|1|42|64240.68|0.05|0.02|N|O|1996-03-05|1996-01-13|1996-03-10|COLLECT COD|SHIP|to the furiously 5925|124966|9991|2|31|61719.76|0.03|0.03|N|O|1996-01-02|1995-12-14|1996-01-07|TAKE BACK RETURN|FOB|e slyly. furiously regular deposi 5925|88288|3305|3|50|63814.00|0.03|0.04|N|O|1996-02-14|1996-01-10|1996-02-15|NONE|TRUCK|es. stealthily express pains print bli 5925|53309|8320|4|30|37869.00|0.02|0.07|N|O|1996-02-21|1996-02-11|1996-03-10|NONE|TRUCK| the packa 5925|159746|9747|5|41|74035.34|0.00|0.06|N|O|1996-02-03|1995-12-24|1996-02-20|NONE|SHIP| across the pending deposits nag caref 5925|49812|7325|6|48|84566.88|0.02|0.00|N|O|1996-02-03|1996-01-19|1996-03-04|DELIVER IN PERSON|REG AIR| haggle after the fo 5926|89216|4233|1|8|9641.68|0.02|0.00|R|F|1994-07-17|1994-07-20|1994-08-11|COLLECT COD|MAIL|gle furiously express foxes. bo 5926|49773|7286|2|27|46514.79|0.09|0.05|A|F|1994-07-05|1994-08-11|1994-08-02|DELIVER IN PERSON|MAIL|ironic requests 5926|126721|6722|3|46|80395.12|0.01|0.03|R|F|1994-09-05|1994-08-12|1994-09-11|COLLECT COD|RAIL|ts integrate. courts haggl 5926|189693|7248|4|23|41001.87|0.01|0.02|A|F|1994-07-23|1994-08-10|1994-07-27|DELIVER IN PERSON|FOB|ickly special packages among 5927|89541|2050|1|44|67343.76|0.04|0.05|N|O|1997-11-29|1997-11-21|1997-12-13|DELIVER IN PERSON|TRUCK|rding to the special, final decoy 5927|114570|9593|2|8|12676.56|0.04|0.05|N|O|1997-09-24|1997-11-15|1997-10-22|TAKE BACK RETURN|SHIP|ilent dependencies nod c 5927|166058|8575|3|32|35969.60|0.10|0.07|N|O|1997-12-26|1997-10-27|1997-12-31|COLLECT COD|AIR|telets. carefully bold accounts was 5952|199810|2330|1|49|93580.69|0.10|0.02|N|O|1997-06-30|1997-07-10|1997-07-02|COLLECT COD|AIR|e furiously regular 5952|190128|7686|2|11|13399.32|0.10|0.05|N|O|1997-05-13|1997-06-04|1997-05-27|DELIVER IN PERSON|FOB|y nag blithely aga 5952|70338|339|3|43|56258.19|0.01|0.01|N|O|1997-06-29|1997-06-06|1997-07-15|COLLECT COD|MAIL|posits sleep furiously quickly final p 5952|157819|2850|4|23|43166.63|0.00|0.07|N|O|1997-05-13|1997-06-27|1997-05-20|NONE|TRUCK|e blithely packages. eve 5953|128103|8104|1|36|40719.60|0.03|0.00|R|F|1992-05-28|1992-06-24|1992-05-29|DELIVER IN PERSON|FOB| cajole furio 5953|12029|4531|2|34|31994.68|0.03|0.04|A|F|1992-05-04|1992-06-12|1992-06-02|NONE|RAIL|hockey players use furiously against th 5953|161015|6048|3|5|5380.05|0.07|0.06|A|F|1992-04-10|1992-04-27|1992-04-14|NONE|SHIP|s. blithely 5953|168308|825|4|23|31654.90|0.09|0.02|R|F|1992-06-05|1992-06-03|1992-06-29|TAKE BACK RETURN|FOB|he silent ideas. silent foxes po 5954|146706|4249|1|8|14021.60|0.03|0.00|A|F|1993-03-27|1993-01-22|1993-04-04|TAKE BACK RETURN|AIR|unusual th 5954|80235|2744|2|40|48609.20|0.02|0.01|A|F|1992-12-30|1993-01-16|1993-01-09|COLLECT COD|RAIL|iously ironic deposits after 5954|93387|915|3|20|27607.60|0.09|0.07|A|F|1992-12-25|1993-02-05|1992-12-31|COLLECT COD|REG AIR| accounts wake carefu 5954|144672|2215|4|20|34333.40|0.00|0.01|R|F|1993-02-27|1993-01-04|1993-03-08|NONE|TRUCK|ke furiously blithely special packa 5954|99780|7308|5|35|62292.30|0.04|0.06|A|F|1993-03-17|1993-02-06|1993-04-10|NONE|SHIP|tions maintain slyly. furious 5954|192029|4549|6|39|43719.78|0.04|0.08|A|F|1993-02-27|1993-02-25|1993-03-29|DELIVER IN PERSON|REG AIR| always regular dolphins. furiously p 5955|139375|4402|1|14|19801.18|0.08|0.08|N|O|1995-06-22|1995-05-23|1995-06-24|DELIVER IN PERSON|TRUCK| unusual, bold theodolit 5955|61553|9072|2|15|22718.25|0.08|0.07|R|F|1995-04-22|1995-05-28|1995-04-27|NONE|FOB|y final accounts above the regu 5955|111196|6219|3|40|48287.60|0.03|0.00|R|F|1995-04-01|1995-06-11|1995-04-27|NONE|FOB|oss the fluffily regular 5956|154047|6563|1|10|11010.40|0.04|0.05|N|O|1998-07-27|1998-07-04|1998-08-21|NONE|MAIL|ic packages am 5956|54179|1695|2|23|26062.91|0.08|0.03|N|O|1998-06-06|1998-07-10|1998-06-15|DELIVER IN PERSON|RAIL|ly slyly special 5956|174834|7352|3|47|89715.01|0.04|0.06|N|O|1998-09-06|1998-06-29|1998-09-18|TAKE BACK RETURN|MAIL|lyly express theodol 5956|19995|7499|4|40|76599.60|0.09|0.05|N|O|1998-06-11|1998-07-19|1998-06-21|NONE|MAIL|final theodolites sleep carefully ironic c 5957|14617|7119|1|37|56669.57|0.07|0.00|A|F|1994-04-18|1994-02-19|1994-05-11|NONE|AIR| ideas use ruthlessly. 5957|58726|3737|2|46|77497.12|0.04|0.08|A|F|1994-01-23|1994-01-30|1994-02-07|NONE|SHIP|platelets. furiously unusual requests 5957|1377|6378|3|17|21732.29|0.01|0.01|A|F|1994-01-24|1994-02-16|1994-02-08|TAKE BACK RETURN|SHIP|. final, pending packages 5957|131499|6526|4|29|44384.21|0.01|0.03|R|F|1994-02-24|1994-03-04|1994-03-08|COLLECT COD|REG AIR|sits. final, even asymptotes cajole quickly 5957|87262|4787|5|40|49970.40|0.04|0.04|R|F|1994-01-07|1994-02-05|1994-01-26|DELIVER IN PERSON|SHIP|ironic asymptotes sleep blithely again 5957|5079|80|6|41|40346.87|0.10|0.07|R|F|1994-03-25|1994-02-20|1994-03-31|DELIVER IN PERSON|MAIL|es across the regular requests maint 5957|158431|5977|7|32|47661.76|0.10|0.07|A|F|1994-03-05|1994-02-20|1994-03-09|NONE|TRUCK| boost carefully across the 5958|148834|6377|1|33|62133.39|0.02|0.04|N|O|1995-09-24|1995-12-12|1995-10-05|COLLECT COD|MAIL|lar, regular accounts wake furi 5958|42932|7941|2|23|43123.39|0.03|0.04|N|O|1995-09-26|1995-10-19|1995-09-27|COLLECT COD|SHIP|regular requests. bold, bold deposits unwin 5958|152606|7637|3|42|69661.20|0.10|0.00|N|O|1995-12-12|1995-10-19|1996-01-09|NONE|AIR|n accounts. final, ironic packages 5958|38433|8434|4|18|24685.74|0.04|0.05|N|O|1995-12-02|1995-10-17|1995-12-22|COLLECT COD|FOB|regular requests haggle 5958|131786|4300|5|32|58168.96|0.06|0.00|N|O|1995-09-20|1995-12-10|1995-10-14|COLLECT COD|REG AIR|e carefully special theodolites. carefully 5959|134156|6670|1|49|58317.35|0.07|0.03|R|F|1992-07-16|1992-08-09|1992-08-14|DELIVER IN PERSON|SHIP|usual packages haggle slyly pi 5959|146297|6298|2|17|22835.93|0.09|0.07|R|F|1992-06-10|1992-07-06|1992-06-23|COLLECT COD|MAIL|ackages. blithely ex 5959|4379|4380|3|4|5133.48|0.04|0.03|R|F|1992-06-14|1992-07-05|1992-07-01|NONE|MAIL|gular requests ar 5959|195921|5922|4|13|26219.96|0.03|0.00|A|F|1992-07-29|1992-07-13|1992-08-20|COLLECT COD|SHIP|ar forges. deposits det 5959|39979|7489|5|37|71001.89|0.04|0.01|R|F|1992-06-05|1992-07-18|1992-06-29|NONE|TRUCK|endencies. brai 5959|118109|621|6|35|39448.50|0.03|0.00|A|F|1992-05-27|1992-06-19|1992-06-23|NONE|TRUCK|ely silent deposits. 5959|42154|4659|7|47|51519.05|0.02|0.01|R|F|1992-08-28|1992-07-24|1992-09-09|TAKE BACK RETURN|RAIL|deposits. slyly special cou 5984|69454|6973|1|13|18504.85|0.06|0.07|R|F|1994-10-16|1994-09-06|1994-11-11|NONE|MAIL|lar platelets. f 5984|101208|1209|2|25|30230.00|0.05|0.08|R|F|1994-10-06|1994-07-21|1994-10-28|COLLECT COD|RAIL|gular accounts. even packages nag slyly 5984|321|2822|3|8|9770.56|0.10|0.00|R|F|1994-09-17|1994-08-28|1994-09-25|COLLECT COD|RAIL|its. express, 5984|189708|9709|4|35|62919.50|0.00|0.01|A|F|1994-08-25|1994-08-05|1994-08-31|DELIVER IN PERSON|SHIP|le fluffily regula 5985|85717|8226|1|4|6810.84|0.02|0.02|A|F|1995-05-04|1995-04-01|1995-05-17|DELIVER IN PERSON|MAIL|ole along the quickly slow d 5986|78789|6311|1|26|45962.28|0.00|0.00|R|F|1992-08-10|1992-05-23|1992-08-24|TAKE BACK RETURN|SHIP|e fluffily ironic ideas. silent 5986|195254|7774|2|25|33731.25|0.03|0.06|A|F|1992-06-16|1992-07-17|1992-06-29|TAKE BACK RETURN|MAIL| instructions. slyly regular de 5986|29636|2139|3|1|1565.63|0.07|0.06|A|F|1992-05-21|1992-06-21|1992-05-24|DELIVER IN PERSON|REG AIR|fix quickly quickly final deposits. fluffil 5986|89590|7115|4|31|48967.29|0.00|0.03|A|F|1992-08-21|1992-06-29|1992-09-14|NONE|AIR|structions! furiously pending instructi 5986|135143|5144|5|6|7068.84|0.05|0.02|A|F|1992-07-16|1992-06-10|1992-07-29|DELIVER IN PERSON|RAIL|al foxes within the slyly speci ================================================ FILE: src/test/regress/data/lineitem.2.data ================================================ [File too large to display: 724.4 KB] ================================================ FILE: src/test/regress/data/nation.data ================================================ 0|ALGERIA|0| haggle. carefully final deposits detect slyly agai 1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon 2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special 3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold 4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d 5|ETHIOPIA|0|ven packages wake quickly. regu 6|FRANCE|3|refully final requests. regular, ironi 7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco 8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun 9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 10|IRAN|4|efully alongside of the slyly final dependencies. 11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula 12|JAPAN|2|ously. final, express gifts cajole a 13|JORDAN|4|ic deposits are blithely about the carefully regular pa 14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t 15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets? 16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r 17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun 18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos 19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account 20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely 21|VIETNAM|2|hely enticingly express accounts. even, final 22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint 23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull 24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be ================================================ FILE: src/test/regress/data/null_values.csv ================================================ ,{NULL},"(,)" ,, ================================================ FILE: src/test/regress/data/orders.1.data ================================================ [File too large to display: 158.5 KB] ================================================ FILE: src/test/regress/data/orders.2.data ================================================ 8997|607|F|147614.36|1994-07-02|5-LOW|Clerk#000000404|0|lithely. express, ironic pearls nag permanently. 8998|80|F|147264.16|1993-01-04|5-LOW|Clerk#000000733|0| fluffily pending sauternes cajo 8999|1175|F|113671.53|1994-06-13|2-HIGH|Clerk#000000097|0|ly even foxes. slyly express a 9024|1469|F|298241.36|1992-06-03|3-MEDIUM|Clerk#000000901|0|ar the theodolites. fluffily stealthy requests among the quickly regular asy 9025|739|F|97454.69|1994-05-20|5-LOW|Clerk#000000379|0|ording to the quickly regular ideas integrate above the sly platelets. sly 9026|677|O|63256.87|1996-07-24|5-LOW|Clerk#000000320|0|ironic escapades would wake carefully 9027|514|O|155500.43|1995-09-03|5-LOW|Clerk#000000918|0| sleep carefully with th 9028|1475|F|63063.84|1993-12-22|5-LOW|Clerk#000000364|0| the regular packages. daringly even f 9029|1213|F|78703.86|1992-11-20|3-MEDIUM|Clerk#000000965|0| excuses nag quickly carefully unusual excuse 9030|535|O|225136.87|1998-07-14|3-MEDIUM|Clerk#000000872|0|foxes according to the furiously silent excuses could haggle unusual d 9031|1379|F|270589.18|1993-12-27|5-LOW|Clerk#000000890|0|r packages. slyly express ideas 9056|121|O|170074.18|1996-08-08|3-MEDIUM|Clerk#000000117|0|was carefully after the furiously bold dug 9057|71|F|320227.49|1994-11-05|1-URGENT|Clerk#000000539|0| fluffily quickly ironic packages. furiously ironic accounts a 9058|403|F|63464.13|1993-06-29|2-HIGH|Clerk#000000376|0|ealthily special deposits. quickly regular requests wake silently. fur 9059|926|O|187590.77|1996-08-01|2-HIGH|Clerk#000000725|0|ar pinto beans sleep special 9060|463|O|45295.71|1996-06-09|1-URGENT|Clerk#000000438|0|iously. slyly regular dol 9061|325|O|66191.69|1996-01-15|2-HIGH|Clerk#000000428|0|silent excuses! slyl 9062|1102|P|212710.32|1995-03-21|2-HIGH|Clerk#000000201|0|sts cajole slyly according to the carefully slow foxes. furio 9063|1399|O|51019.90|1997-02-12|2-HIGH|Clerk#000000323|0|ffily regular grouche 9088|610|F|224148.01|1994-06-23|1-URGENT|Clerk#000000975|0|ter the blithely final deposits. furiously 9089|370|F|39118.01|1993-05-24|1-URGENT|Clerk#000000538|0|d platelets are deposits. pinto beans cajole boldly. 9090|775|O|9848.22|1996-11-21|4-NOT SPECIFIED|Clerk#000000699|0|cial theodolites at the evenl 9091|679|F|242198.88|1993-11-12|2-HIGH|Clerk#000000426|0|final packages wake carefully. 9092|664|F|202836.36|1994-07-25|4-NOT SPECIFIED|Clerk#000000144|0| slyly final waters. special packages solve 9093|619|O|19431.49|1995-12-10|3-MEDIUM|Clerk#000000333|0|even accounts. special, 9094|1430|O|46784.32|1998-05-27|3-MEDIUM|Clerk#000000805|0|es boost slyly after the platelets. shea 9095|220|O|49946.10|1995-06-11|2-HIGH|Clerk#000000621|0|riously regular accounts wake slyly special theodolites. asymptotes use f 9120|1010|F|128719.48|1992-06-22|3-MEDIUM|Clerk#000000252|0|ly ironic realms. furio 9121|434|O|94637.46|1996-07-05|5-LOW|Clerk#000000930|0| final dependencies. carefully even excuses after the 9122|79|O|234630.52|1996-12-14|4-NOT SPECIFIED|Clerk#000000991|0|across the carefull 9123|367|F|64575.87|1993-06-27|1-URGENT|Clerk#000000988|0|wake carefully pendin 9124|559|O|173690.73|1995-10-17|5-LOW|Clerk#000000381|0|nstructions after the c 9125|151|O|195514.25|1998-05-30|2-HIGH|Clerk#000000591|0|kages around the fluffily bold f 9126|371|F|106144.91|1994-11-18|3-MEDIUM|Clerk#000000319|0|deposits boost carefully against the fluffily final instructions. i 9127|611|O|79003.80|1995-10-11|4-NOT SPECIFIED|Clerk#000000628|0|ress packages across the furio 9152|1210|F|129410.55|1993-08-21|4-NOT SPECIFIED|Clerk#000000612|0|nto beans! final instructions nag slyly. slyly ironic foxes nag blith 9153|1147|O|210245.80|1997-07-19|4-NOT SPECIFIED|Clerk#000000488|0|uffily express instructions haggle carefully. quickly fluffy th 9154|1|O|357345.46|1997-06-23|4-NOT SPECIFIED|Clerk#000000328|0|y ironic packages cajole. blithely final depende 9155|355|F|83263.25|1992-08-31|1-URGENT|Clerk#000000253|0|ickly regular requests sleep alongside of the car 9156|154|F|135183.22|1994-02-07|1-URGENT|Clerk#000000387|0|deposits. dependencies wake ca 9157|595|F|52660.33|1992-06-20|5-LOW|Clerk#000000736|0|requests detect furiously special, silent packages. carefu 9158|271|F|96814.40|1995-04-25|4-NOT SPECIFIED|Clerk#000000195|0|ecial, even packages. even foxes print. regu 9159|1135|O|99594.61|1995-07-26|1-URGENT|Clerk#000000892|0|xcuses. quickly ironic deposits wake alongside of the quickly pending p 9184|1156|O|56334.07|1997-07-19|1-URGENT|Clerk#000000086|0|blithely final packages haggle according to the slyly quick pack 9185|145|F|64122.78|1994-06-16|2-HIGH|Clerk#000000951|0|leep carefully blithely regular orbits. blithely regular foxes cajol 9186|457|F|167929.61|1992-03-30|5-LOW|Clerk#000000044|0|lar, silent asymptotes haggl 9187|769|F|212469.45|1994-08-22|2-HIGH|Clerk#000000985|0|lithely ironic theodolites. furiously ironic instructions print along 9188|619|O|224857.18|1998-01-25|1-URGENT|Clerk#000000765|0|aggle quickly above the accounts. excuses 9189|1069|O|109530.95|1996-12-02|3-MEDIUM|Clerk#000000501|0|r waters haggle furiously fluffily ironic id 9190|460|O|272754.44|1998-03-03|2-HIGH|Clerk#000000196|0|side of the permanently ironic courts unwind 9191|535|O|21182.03|1996-03-24|3-MEDIUM|Clerk#000000095|0|final packages wake carefully. 9216|643|O|147294.33|1995-04-15|3-MEDIUM|Clerk#000000320|0|its nag regular deposits. slyly even reques 9217|1489|O|117161.58|1997-05-26|1-URGENT|Clerk#000000051|0| above the blithely regular d 9218|241|O|31407.74|1997-12-10|4-NOT SPECIFIED|Clerk#000000795|0|hely regular foxes are alongside of the furiously regular deposit 9219|412|F|205446.61|1994-09-29|5-LOW|Clerk#000000382|0|nst the furiously regular dinos. regular asymptotes against the slyly fina 9220|482|O|992.46|1998-04-07|1-URGENT|Clerk#000000880|0|y bravely ironic deposits. furiously unusual sentiments about the fluffily ir 9221|85|F|273797.07|1995-01-04|5-LOW|Clerk#000000240|0|ess accounts sleep carefully. carefully final pinto beans sleep f 9222|211|F|258460.61|1994-02-11|2-HIGH|Clerk#000000148|0|ding to the instructions. regular requests nag f 9223|304|O|215658.19|1997-08-05|4-NOT SPECIFIED|Clerk#000000785|0|ully according to the blithely 9248|1163|F|110964.49|1994-12-24|4-NOT SPECIFIED|Clerk#000000259|0|ids cajole regular, regular dependencies. deposits along the dolphi 9249|1348|F|161815.08|1993-05-20|2-HIGH|Clerk#000000756|0|slyly pending accounts. packages wake c 9250|1345|F|85572.38|1992-08-14|1-URGENT|Clerk#000000535|0|y final packages; carefully pendi 9251|323|F|29910.23|1993-08-30|3-MEDIUM|Clerk#000000374|0|ld requests. deposits use blithely ruthlessly unusual packages. fluffil 9252|412|O|89891.37|1997-12-11|2-HIGH|Clerk#000000454|0|encies affix slyly perma 9253|818|F|255856.03|1992-01-31|1-URGENT|Clerk#000000307|0|es. sometimes regular grouches wa 9254|1276|F|56365.27|1993-05-15|3-MEDIUM|Clerk#000000761|0|r deposits. quickly bold requests use- 9255|472|O|153124.60|1995-12-08|4-NOT SPECIFIED|Clerk#000000476|0|y bold asymptotes-- carefully 9280|1223|O|275205.91|1998-06-12|5-LOW|Clerk#000000022|0|beans alongside of the fluffily express asymptotes integrate 9281|904|F|173278.28|1992-02-24|1-URGENT|Clerk#000000530|0|eep furiously according to the requests; ideas integrate 9282|79|O|34765.35|1995-11-26|4-NOT SPECIFIED|Clerk#000000824|0|ses? carefully express platelets sleep blithely against the blithely specia 9283|1255|F|7798.12|1994-06-03|3-MEDIUM|Clerk#000000859|0|ully even platelets. silent packages 9284|661|F|56207.47|1994-06-14|4-NOT SPECIFIED|Clerk#000000544|0| furiously across the final deposits. quickly ironic requests accordin 9285|1012|F|118683.77|1994-01-03|1-URGENT|Clerk#000000553|0|sts. express accounts snooze; furiously final 9286|505|P|155839.92|1995-04-04|1-URGENT|Clerk#000000557|0|ake. carefully bold packages promise with the 9287|790|O|13525.92|1998-01-02|4-NOT SPECIFIED|Clerk#000000636|0|ously unusual packages. regular foxes detect. blithely slow ideas sl 9312|920|F|226584.81|1992-05-19|4-NOT SPECIFIED|Clerk#000000897|0|ckly slyly regular packages. unusual packages are. regular 9313|1348|O|222818.15|1996-04-06|3-MEDIUM|Clerk#000000926|0|s wake slyly against the slyly express packages. ironic packages 9314|1313|F|14188.37|1995-02-15|4-NOT SPECIFIED|Clerk#000000907|0|eas. blithely regular asymptotes sleep carefully across the slyly even 9315|587|O|16012.39|1998-03-17|3-MEDIUM|Clerk#000000559|0|ackages are furiously alongside of the slyly regular theodolite 9316|1393|O|225240.88|1995-09-18|4-NOT SPECIFIED|Clerk#000000363|0|inal pinto beans haggle c 9317|100|F|156751.98|1994-03-15|4-NOT SPECIFIED|Clerk#000000347|0|aves across the unusual, regular sauternes wake 9318|1463|F|109251.02|1992-07-28|1-URGENT|Clerk#000000738|0|ss courts across the slyly silent foxes are regular, final pac 9319|880|F|104039.47|1992-09-11|1-URGENT|Clerk#000000155|0|. fluffily ironic instructions detect 9344|664|O|348335.72|1995-12-07|5-LOW|Clerk#000000961|0|ial deposits nag blithely alongside of the caref 9345|1376|O|163627.01|1996-01-22|4-NOT SPECIFIED|Clerk#000000126|0|out the daringly ironic packages affix furiously fluffi 9346|917|O|173943.50|1996-10-09|4-NOT SPECIFIED|Clerk#000000088|0|furiously final packages integrate stealthily expres 9347|673|F|162913.90|1992-08-10|5-LOW|Clerk#000000268|0|he gifts. slyly permanent requests 9348|658|O|120413.73|1996-06-07|5-LOW|Clerk#000000416|0|ackages. carefully ironic pinto beans about the close accoun 9349|1414|O|217101.05|1997-08-29|4-NOT SPECIFIED|Clerk#000000804|0|. fluffily final accounts haggle; furiously reg 9350|145|F|255516.01|1992-11-01|1-URGENT|Clerk#000000253|0|ackages doze evenly across the foxes. quickly ironic reques 9351|898|O|81166.06|1996-05-17|5-LOW|Clerk#000000430|0|ven dependencies. furiously ironic dependencies promise along the slyly 9376|835|O|145196.26|1997-07-24|2-HIGH|Clerk#000000348|0|ng the carefully pending ac 9377|340|F|9021.26|1993-04-22|2-HIGH|Clerk#000000466|0|ers above the carefull 9378|457|O|115914.73|1997-10-16|1-URGENT|Clerk#000000014|0|ultipliers wake furiously never special deposits! 9379|350|F|152004.78|1992-01-04|1-URGENT|Clerk#000000764|0|nusual deposits. furiously final packages wake evenly even tithes. qu 9380|887|F|167911.91|1994-08-26|1-URGENT|Clerk#000000962|0| deposits boost along the carefully ironic deposits. express requests use. 9381|1184|O|94270.85|1995-12-18|4-NOT SPECIFIED|Clerk#000000215|0|y final Tiresias are. requests 9382|337|O|150840.29|1996-02-19|1-URGENT|Clerk#000000308|0|l accounts. furiously ironic deposits use. 9383|140|F|180583.64|1994-03-17|4-NOT SPECIFIED|Clerk#000000237|0|uffily final pinto beans cajole ca 9408|184|F|11891.10|1992-10-21|3-MEDIUM|Clerk#000000664|0|xes cajole carefully furiously final pains. carefully express accou 9409|970|F|155483.25|1992-05-06|1-URGENT|Clerk#000000300|0|ly express platelets above the sl 9410|787|O|88246.41|1997-07-18|5-LOW|Clerk#000000985|0|theodolites solve fluffily. blithely iron 9411|661|F|55576.82|1995-02-07|2-HIGH|Clerk#000000594|0|rses. furiously regular requests haggl 9412|1216|O|245785.65|1998-07-14|2-HIGH|Clerk#000000016|0|requests. carefully regular packages play carefully pending dependencies. ca 9413|148|O|150237.99|1995-07-13|5-LOW|Clerk#000000202|0|latelets are blithely requests. ironic packages boost slyly across t 9414|1333|F|98169.43|1993-09-10|3-MEDIUM|Clerk#000000661|0|above the slyly ironic 9415|1049|F|241463.63|1992-08-24|1-URGENT|Clerk#000000311|0|ccounts boost blithely. final 9440|643|F|4331.68|1994-06-08|5-LOW|Clerk#000000711|0| requests. regular, final pinto beans are. furiously regular accounts det 9441|334|O|3216.31|1997-06-18|4-NOT SPECIFIED|Clerk#000000381|0|iously final deposits. express, regular waters haggle sl 9442|1267|F|103119.65|1993-03-11|4-NOT SPECIFIED|Clerk#000000948|0|hely special courts. furiously final foxes sleep quickly. quick, 9443|304|O|166940.25|1997-05-07|4-NOT SPECIFIED|Clerk#000000317|0|y along the carefully reg 9444|106|O|36255.65|1996-07-18|2-HIGH|Clerk#000000387|0| quickly. always silent platelets must sleep f 9445|928|O|76754.58|1997-11-14|4-NOT SPECIFIED|Clerk#000000394|0|as. slyly ironic deposits along 9446|1358|O|216362.20|1997-12-29|1-URGENT|Clerk#000000173|0|ake pinto beans. slyly f 9447|1162|O|79483.65|1995-06-19|2-HIGH|Clerk#000000115|0|theodolites haggle carefully instructio 9472|481|O|44176.66|1995-06-17|1-URGENT|Clerk#000000979|0|bove the quickly final deposits are carefully 9473|23|F|212682.92|1992-10-12|4-NOT SPECIFIED|Clerk#000000541|0|ts sleep above the accounts. slyly final deposits are regularly. r 9474|133|P|246662.69|1995-05-12|3-MEDIUM|Clerk#000000710|0|ely final requests: dependencies cajole slyly. slyly 9475|1442|O|206441.11|1996-11-22|2-HIGH|Clerk#000000233|0| notornis. ironically even instructions en 9476|1283|F|177524.44|1994-05-26|4-NOT SPECIFIED|Clerk#000000088|0|e slyly final requests might integrate carefully against the slyly express p 9477|170|O|269394.57|1998-03-19|3-MEDIUM|Clerk#000000812|0|lites along the even deposits use bold pinto beans. regular requests 9478|940|F|173044.15|1994-03-29|2-HIGH|Clerk#000000033|0|ound the blithely express instructions. furiously 9479|1373|F|76316.92|1994-09-22|3-MEDIUM|Clerk#000000047|0|ter the carefully ironic 9504|736|F|171165.10|1992-11-26|3-MEDIUM|Clerk#000000902|0|among the quickly bold deposits haggle exp 9505|316|F|283831.70|1992-06-08|2-HIGH|Clerk#000000873|0|hely ironic asymptot 9506|1232|F|47720.50|1994-02-11|4-NOT SPECIFIED|Clerk#000000441|0|carefully ironic accou 9507|517|F|34161.51|1994-11-30|5-LOW|Clerk#000000438|0|press pinto beans are slyly. final packages cajole. quickly regular excus 9508|448|O|235237.15|1996-04-22|5-LOW|Clerk#000000312|0|ss attainments slee 9509|37|F|224330.32|1992-10-08|4-NOT SPECIFIED|Clerk#000000350|0|iously final instructions sleep fu 9510|235|O|93930.42|1996-11-20|5-LOW|Clerk#000000302|0|packages sleep furiously. bold theodolites haggle slyly. ironic, 9511|1141|O|96086.28|1996-08-01|5-LOW|Clerk#000000555|0|out the packages may 9536|1138|O|105639.69|1997-08-20|4-NOT SPECIFIED|Clerk#000000224|0|. slyly even ideas doubt slyly. slowly regular orbits cajole. fur 9537|806|F|249384.96|1995-02-14|4-NOT SPECIFIED|Clerk#000000203|0| packages. final, regular foxes haggle bli 9538|1399|F|32388.43|1994-03-11|4-NOT SPECIFIED|Clerk#000000040|0|ly. slyly final accounts nag carefully regular ideas. blithely ironic theodo 9539|262|O|26683.19|1995-11-15|2-HIGH|Clerk#000000917|0|iously ironic deposits affix furiously alongside of the slyly even foxes. even 9540|784|F|96163.55|1994-06-11|1-URGENT|Clerk#000000048|0|t above the stealthily bold theodolites 9541|598|F|105885.33|1992-03-26|4-NOT SPECIFIED|Clerk#000000840|0|xcuses dazzle furiously regular the 9542|1147|O|121584.41|1995-12-16|2-HIGH|Clerk#000000349|0|theodolites. blithely pendin 9543|850|F|145441.29|1992-05-10|1-URGENT|Clerk#000000187|0| unusual warhorses cajole 9568|619|F|80658.16|1993-03-08|4-NOT SPECIFIED|Clerk#000000270|0|ng the quickly unusual pinto beans sublate blithely ca 9569|1264|O|210401.62|1997-11-01|2-HIGH|Clerk#000000856|0|pecial deposits after the final, pending packages wake carefully 9570|1435|O|79124.61|1996-06-04|2-HIGH|Clerk#000000397|0|ccounts. foxes nag special, special theodolites 9571|1366|O|58131.32|1996-04-30|4-NOT SPECIFIED|Clerk#000000263|0| furiously express requests. even requests are. carefully final a 9572|49|O|85669.13|1998-03-28|4-NOT SPECIFIED|Clerk#000000452|0|atelets. slyly regular requests nag quickly ev 9573|1361|F|140154.64|1992-06-14|3-MEDIUM|Clerk#000000250|0|ix carefully busily unusual deposits. thin excuses haggle quickly. quickly 9574|790|O|145838.53|1995-07-31|3-MEDIUM|Clerk#000000814|0|efully regular platelets. pending packages unwind carefully among the fur 9575|916|F|27475.15|1992-03-31|5-LOW|Clerk#000000133|0|nts haggle busily unusual, even packages. regular packages u 9600|536|F|99615.09|1993-11-08|4-NOT SPECIFIED|Clerk#000000467|0|he unusual, ironic requests nag furiously ironic accounts 9601|1435|O|143520.49|1996-03-20|1-URGENT|Clerk#000000677|0|ugh the slyly regular requests. furiously even pinto beans are blithely slyl 9602|1481|F|193843.49|1992-11-11|1-URGENT|Clerk#000000086|0| across the slyly ironic ideas. carefully unusual requests 9603|1204|O|315928.32|1998-01-09|5-LOW|Clerk#000000191|0|ng the quickly special requests caj 9604|620|O|48743.62|1996-06-25|2-HIGH|Clerk#000000414|0|usly ironic theodolites! blithely final ideas 9605|1220|F|113112.29|1992-12-26|4-NOT SPECIFIED|Clerk#000000777|0|yly bold pinto beans are furiously packages. slyly express courts play 9606|152|F|149838.50|1994-07-28|4-NOT SPECIFIED|Clerk#000000474|0| above the always regular pinto beans. special, unusual accounts grow. pending 9607|643|O|44978.35|1995-12-18|1-URGENT|Clerk#000000752|0| ironic requests are carefully. silent 9632|1429|F|146044.97|1992-01-17|5-LOW|Clerk#000000857|0|y ironic theodolites are silent foxes. blithely final plate 9633|1336|O|191021.09|1995-09-12|4-NOT SPECIFIED|Clerk#000000918|0|ns around the slyly ironic pinto beans cajole carefully ironic depths. careful 9634|958|O|314926.50|1998-01-13|2-HIGH|Clerk#000000166|0| carefully. slyly ironic courts mold. ironic 9635|847|F|168349.65|1994-04-14|1-URGENT|Clerk#000000127|0|the quickly ironic ideas sleep at the quickly 9636|304|O|109568.95|1997-01-30|1-URGENT|Clerk#000000263|0|lar foxes are requests. quickly even pinto be 9637|1411|F|112613.87|1994-01-04|5-LOW|Clerk#000000953|0|s. furiously final requests believe furiously against the brave 9638|1126|O|112055.66|1996-06-08|2-HIGH|Clerk#000000953|0|requests after the furiously unusual accounts integrate slyly u 9639|722|F|244899.69|1993-09-06|4-NOT SPECIFIED|Clerk#000000666|0|press, express pinto 9664|310|F|105001.15|1993-04-08|2-HIGH|Clerk#000000715|0|ccounts across the slyly ironic pinto be 9665|1067|F|81734.81|1994-04-24|5-LOW|Clerk#000000277|0|sleep furiously carefully ironic foxes. pinto beans wake. final deposits caj 9666|221|O|240666.92|1996-05-09|1-URGENT|Clerk#000000511|0|ss the fluffily express tithes. 9667|1102|O|208860.66|1996-02-24|3-MEDIUM|Clerk#000000142|0|nal instructions. special accounts along the regular 9668|847|F|77447.46|1994-09-07|5-LOW|Clerk#000000469|0|refully ironic accounts. furiously bold packages a 9669|619|F|79738.94|1994-04-23|3-MEDIUM|Clerk#000000297|0|deposits promise blithely ironic theodolites. fluffily bold requests 9670|1387|O|200921.16|1997-11-08|3-MEDIUM|Clerk#000000035|0|unts? furiously express deposits are blithely. regular, special forges sle 9671|136|O|157241.36|1995-09-13|2-HIGH|Clerk#000000417|0|ages boost slyly against the furiously 9696|575|F|211382.08|1995-02-20|4-NOT SPECIFIED|Clerk#000000971|0|xpress requests would are pinto beans. 9697|164|F|174800.10|1995-01-27|3-MEDIUM|Clerk#000000432|0|courts are. even platelets was alongside of 9698|1487|O|7676.88|1995-08-07|2-HIGH|Clerk#000000040|0| around the quickly bold packages sle 9699|875|F|10782.39|1995-03-12|3-MEDIUM|Clerk#000000853|0|kly regular platelets. slyly silent accounts alongside of the blithely r 9700|544|O|266611.98|1995-08-09|1-URGENT|Clerk#000000137|0|althy pinto beans wake blithely. qui 9701|112|F|277385.28|1993-02-28|5-LOW|Clerk#000000399|0|bold deposits cajole furiously bold requests. even excuses 9702|1201|O|182347.71|1996-08-22|3-MEDIUM|Clerk#000000852|0|tions haggle slyly among the ironic theodolites. blithel 9703|1126|F|28954.25|1994-10-07|5-LOW|Clerk#000000101|0|realms wake. express, ironic accou 9728|826|F|39398.24|1993-07-03|5-LOW|Clerk#000000647|0|ng requests. fluffily enticing re 9729|970|O|162532.59|1996-07-06|3-MEDIUM|Clerk#000000148|0|ndencies about the care 9730|143|O|278148.33|1997-03-04|5-LOW|Clerk#000000743|0| final dependencies doze furiously slyly silent ideas? furiousl 9731|283|F|21613.76|1993-10-21|5-LOW|Clerk#000000586|0| dolphins nag slyly even deposits-- blithely 9732|820|P|69634.87|1995-03-14|4-NOT SPECIFIED|Clerk#000000599|0|f the special, even instructions. carefully final requests use along the 9733|1000|F|72374.84|1994-03-18|4-NOT SPECIFIED|Clerk#000000997|0|n dolphins haggle. slyly express accounts wake carefully final asymptotes. qui 9734|835|O|124297.04|1997-02-03|1-URGENT|Clerk#000000086|0|equests affix quietly bold deposits. dependencies affix among the quietly 9735|337|O|115276.76|1997-06-28|4-NOT SPECIFIED|Clerk#000000454|0|nic foxes. accounts detect fur 9760|1181|O|158777.10|1998-04-14|5-LOW|Clerk#000000207|0|anent accounts. accounts sleep carefully slyl 9761|433|F|70880.34|1992-01-23|4-NOT SPECIFIED|Clerk#000000468|0| permanent requests. furious requests nag blithely about the q 9762|1492|F|238951.03|1994-01-04|2-HIGH|Clerk#000000861|0|ic requests cajole quickly unusual accounts. regular foxes are blithely. in 9763|1492|O|132201.56|1995-08-14|3-MEDIUM|Clerk#000000374|0| requests detect carefull 9764|1256|F|56685.16|1992-12-30|1-URGENT|Clerk#000000289|0|nts. platelets wake blithe 9765|1153|F|45183.21|1992-10-13|5-LOW|Clerk#000000364|0|ts. even pinto beans haggle alongside of the theodolites. 9766|235|F|165508.82|1992-08-15|1-URGENT|Clerk#000000265|0|s excuses cajole quickly. regul 9767|973|F|46852.11|1994-06-28|5-LOW|Clerk#000000546|0|asymptotes hinder blithely after the busily final fo 9792|1114|O|171667.54|1995-07-24|5-LOW|Clerk#000000929|0| the special accounts. carefully special ideas mainta 9793|1198|F|80712.82|1992-04-04|3-MEDIUM|Clerk#000000770|0| carefully ironic courts haggle evenly. carefully final requests ca 9794|505|F|216443.67|1992-03-25|4-NOT SPECIFIED|Clerk#000000852|0|dolites grow furiously fluffily final requests. blithely special 9795|370|O|215242.29|1998-02-04|3-MEDIUM|Clerk#000000481|0|ng the furiously iron 9796|1052|F|246896.17|1993-09-11|1-URGENT|Clerk#000000150|0|lyly regular packages. furio 9797|43|F|209264.66|1993-09-13|1-URGENT|Clerk#000000296|0|packages. blithe requests affix quickly carefully unusual ideas. blithe 9798|1135|O|23604.81|1996-03-09|1-URGENT|Clerk#000000485|0|ial deposits. accounts detect dependencies. silent accounts cajole flu 9799|461|O|152265.02|1996-04-07|4-NOT SPECIFIED|Clerk#000000526|0|ully final requests cajole carefully according to t 9824|472|F|240691.30|1995-01-23|1-URGENT|Clerk#000000653|0|furiously even theodolites impress even, special asymptotes. idle ideas dazzl 9825|28|O|93645.30|1996-04-09|1-URGENT|Clerk#000000990|0|rding to the pending 9826|698|F|105223.73|1992-03-23|3-MEDIUM|Clerk#000000777|0|deas haggle slyly final reques 9827|1289|O|36448.45|1998-07-04|4-NOT SPECIFIED|Clerk#000000085|0|e unusual dependencies. regular accounts alongside of the quickly regular f 9828|1306|F|269649.39|1992-04-25|3-MEDIUM|Clerk#000000355|0| deposits are carefully bold notornis. blith 9829|1379|O|6613.63|1995-12-09|1-URGENT|Clerk#000000004|0|ke across the requests. silently final foxes wake never amo 9830|493|O|202703.99|1997-09-08|2-HIGH|Clerk#000000191|0|final packages are furiously beside the quickly final e 9831|790|O|86410.08|1996-06-20|4-NOT SPECIFIED|Clerk#000000010|0|mas above the even instructions haggle ironical 9856|802|F|236403.75|1993-05-30|3-MEDIUM|Clerk#000000278|0|usual tithes against the express requests sleep across t 9857|172|F|158381.98|1993-05-12|2-HIGH|Clerk#000000627|0|ons haggle quickly above the blithely regular sauternes. slyly express instru 9858|1399|O|103920.84|1997-12-08|3-MEDIUM|Clerk#000000432|0|e of the fluffily final packa 9859|475|O|247788.34|1996-07-28|3-MEDIUM|Clerk#000000989|0|xes boost slyly. furiously regul 9860|623|F|151734.45|1993-10-11|1-URGENT|Clerk#000000345|0|carefully. ironic ideas haggle theodolites. quickly speci 9861|1477|F|66189.04|1992-10-29|5-LOW|Clerk#000000113|0|le after the unusual deposits. even acc 9862|10|O|193015.83|1997-10-10|2-HIGH|Clerk#000000168|0|among the instructions was after the slyly special packages. final, final hoc 9863|247|O|362398.22|1995-09-05|1-URGENT|Clerk#000000473|0|urts nag finally even, stealthy instructions. quickly final ideas 9888|1319|F|74228.41|1994-09-13|5-LOW|Clerk#000000434|0|g to the carefully final foxes. fur 9889|82|F|35362.96|1994-09-09|3-MEDIUM|Clerk#000000786|0|lly atop the foxes. doggedly final packages integrate thin 9890|731|P|87423.78|1995-02-22|2-HIGH|Clerk#000000168|0|uickly silent theodolites. pe 9891|1093|P|61268.10|1995-06-09|1-URGENT|Clerk#000000544|0|ut the theodolites. bold, even grouches across t 9892|691|F|16104.32|1992-07-17|3-MEDIUM|Clerk#000000387|0|ts. blithe deposits wake carefully carefully 9893|412|F|105380.78|1994-08-17|3-MEDIUM|Clerk#000000931|0|y special dependencies run 9894|851|F|100473.64|1994-03-19|2-HIGH|Clerk#000000424|0| asymptotes along the regular, bold instructions eat quickly again 9895|142|F|245503.17|1993-04-21|5-LOW|Clerk#000000965|0|lithely. furiously even pinto beans h 9920|1073|O|75014.21|1996-06-06|1-URGENT|Clerk#000000007|0|ites. special requests believe against the furiously regular requests. furiou 9921|1379|P|143610.80|1995-05-15|5-LOW|Clerk#000000936|0|r, blithe deposits. furiously bold 9922|802|P|186264.65|1995-04-04|4-NOT SPECIFIED|Clerk#000000131|0|ajole blithely across the unusual, unusual packages. car 9923|1217|F|223804.09|1994-10-06|2-HIGH|Clerk#000000818|0|instructions use slyl 9924|799|O|192545.87|1997-05-09|4-NOT SPECIFIED|Clerk#000000639|0|onic excuses wake slyly 9925|52|F|296109.93|1992-01-31|1-URGENT|Clerk#000000751|0|ve the always ironic pinto bean 9926|254|F|225023.94|1994-10-12|2-HIGH|Clerk#000000090|0|sits believe carefully ironic pinto beans. sl 9927|13|O|238640.42|1995-08-17|2-HIGH|Clerk#000000388|0|sts sleep regular, dogged packages. s 9952|521|F|87414.24|1992-05-02|2-HIGH|Clerk#000000549|0|ully about the daringly pending idea 9953|946|F|18034.69|1995-02-28|2-HIGH|Clerk#000000364|0|iresias. furiously bold deposits wake. furiously 9954|19|F|207935.24|1994-06-20|4-NOT SPECIFIED|Clerk#000000651|0| furiously express packages. slyly ironic packages nag sly 9955|1010|O|188555.97|1995-09-05|1-URGENT|Clerk#000000588|0|gainst the quickly regular deposits. carefully final accounts are furiously 9956|530|F|38746.89|1993-03-17|2-HIGH|Clerk#000000001|0|s? ironic foxes wake even, ironic de 9957|1465|F|64682.58|1993-12-01|4-NOT SPECIFIED|Clerk#000000874|0| unusual sauternes. fluffily e 9958|764|F|192897.71|1993-11-26|4-NOT SPECIFIED|Clerk#000000199|0| idle depths. quickly even pearls are express theodolites. final pint 9959|689|O|19023.25|1995-09-25|3-MEDIUM|Clerk#000000755|0|y special deposits. slyly final deposits against 9984|376|O|232843.69|1998-07-10|1-URGENT|Clerk#000000161|0|ronic packages-- final packages boost! even, express frets boo 9985|1207|F|82406.34|1993-08-08|5-LOW|Clerk#000000636|0|ing depths boost about the even depo 9986|289|O|150409.70|1997-09-22|3-MEDIUM|Clerk#000000913|0|the ruthlessly regular courts. ironic pack 9987|997|O|241701.18|1995-11-28|5-LOW|Clerk#000000308|0|ding to the regular foxes dazzle slyly f 9988|1372|O|139134.68|1997-12-05|4-NOT SPECIFIED|Clerk#000000015|0|d, final foxes. fluffily even sheaves w 9989|1210|F|113256.41|1993-06-22|3-MEDIUM|Clerk#000000298|0| bold orbits. pinto beans haggle carefully unusual accounts. furiously final 9990|391|F|80673.96|1992-03-28|2-HIGH|Clerk#000000466|0|y express foxes. blithely 9991|823|O|88850.20|1996-06-10|2-HIGH|Clerk#000000542|0|foxes nag fluffily packages. b 10016|1295|F|29180.26|1993-02-02|1-URGENT|Clerk#000000546|0|ptotes. platelets across the blithely regular accounts us 10017|811|O|100168.42|1998-04-09|4-NOT SPECIFIED|Clerk#000000617|0|ly accounts. carefully regular pinto beans wake according to th 10018|307|F|85330.56|1993-06-10|3-MEDIUM|Clerk#000000855|0|ld accounts above the bold, regular requests ha 10019|919|F|169596.33|1994-07-19|4-NOT SPECIFIED|Clerk#000000887|0|ests are carefully blithely ironic instructi 10020|469|O|99184.61|1998-04-08|3-MEDIUM|Clerk#000000162|0|yly against the final warth 10021|1282|F|50708.88|1992-08-01|4-NOT SPECIFIED|Clerk#000000321|0|ide of the slyly even Tiresias. carefully express fox 10022|929|F|71058.00|1994-03-05|5-LOW|Clerk#000000445|0|nto beans. furiously silent deposits nag. bold requests are ironic 10023|1204|O|107965.02|1996-12-02|1-URGENT|Clerk#000000596|0|onically even packages haggle among the slyly brave pin 10048|803|F|63327.47|1994-05-16|4-NOT SPECIFIED|Clerk#000000893|0|nt pinto beans are. ironi 10049|683|O|169774.94|1997-07-23|1-URGENT|Clerk#000000648|0|ly special forges. blithely bo 10050|1046|O|151895.89|1996-09-03|2-HIGH|Clerk#000000530|0|bove the permanently regular multipliers. special req 10051|661|O|168753.53|1996-05-25|2-HIGH|Clerk#000000804|0|ly. quickly unusual de 10052|763|F|195461.73|1994-09-08|4-NOT SPECIFIED|Clerk#000000488|0|carefully final packages. ironic foxes cajo 10053|1441|F|152632.75|1992-01-11|2-HIGH|Clerk#000000508|0|ual deposits haggle against the furiously final realms. b 10054|665|O|111854.62|1995-04-29|2-HIGH|Clerk#000000093|0|furiously regular platele 10055|1276|O|108949.78|1996-02-29|5-LOW|Clerk#000000508|0|e bravely bold notornis. carefully reg 10080|109|F|219815.04|1993-02-13|5-LOW|Clerk#000000362|0|carefully blithely express epitaph 10081|493|F|267898.00|1993-08-08|4-NOT SPECIFIED|Clerk#000000097|0|eans would sleep. carefully iro 10082|1264|F|216021.80|1994-08-31|4-NOT SPECIFIED|Clerk#000000049|0|ayers use special excuses. special req 10083|565|O|55168.12|1995-09-26|5-LOW|Clerk#000000949|0|t the brave decoys sleep regularly among the iron 10084|1424|O|5317.21|1997-07-08|1-URGENT|Clerk#000000088|0|grow blithely packages. slyly final requests cajole; accounts 10085|518|O|224630.63|1996-03-31|4-NOT SPECIFIED|Clerk#000000074|0|o beans wake against the blithely final accounts. fluffily silent f 10086|397|O|51011.55|1995-05-16|5-LOW|Clerk#000000728|0|its wake alongside of the f 10087|404|O|78013.94|1997-03-31|2-HIGH|Clerk#000000001|0|ounts integrate despite the regular ideas. furiously quiet idea 10112|223|F|35243.42|1993-12-21|5-LOW|Clerk#000000586|0| deposits. furiously even asymptotes alongside of the slyly regular 10113|56|F|193866.35|1994-06-13|1-URGENT|Clerk#000000094|0|e quickly. regular accounts boost bravely regular instructions. quickly fina 10114|1064|F|46899.44|1993-02-09|1-URGENT|Clerk#000000757|0|al requests. quickly express de 10115|799|O|144986.20|1998-07-07|2-HIGH|Clerk#000000530|0|even accounts. express ideas 10116|118|O|222761.53|1997-03-26|5-LOW|Clerk#000000991|0|mptotes. carefully express depo 10117|734|F|130230.17|1993-11-17|3-MEDIUM|Clerk#000000343|0|ilent theodolites sleep furiously 10118|742|F|24563.35|1994-04-11|1-URGENT|Clerk#000000431|0|ithely pending foxes integrate across the fin 10119|509|O|51762.72|1996-07-11|3-MEDIUM|Clerk#000000868|0|ar depths. slyly sly theodolites sleep slyly. furiously bold foxes integrate 10144|311|F|235400.84|1995-01-03|1-URGENT|Clerk#000000751|0|ironic accounts cajole blithely around the blithe deposits. ir 10145|646|P|302026.89|1995-05-25|5-LOW|Clerk#000000550|0|y unusual deposits sleep furiously i 10146|994|F|237459.34|1994-05-02|5-LOW|Clerk#000000653|0|hin gifts nag. furiously pending deposits after the sly frets sl 10147|1166|F|203640.84|1993-10-10|2-HIGH|Clerk#000000212|0|ilently ironic pinto beans. 10148|1462|F|140605.75|1994-03-26|5-LOW|Clerk#000000334|0|the bold ideas are carefully ruthlessly special pac 10149|26|F|169118.62|1994-09-24|2-HIGH|Clerk#000000681|0|final sentiments. slyly special bra 10150|1325|F|291469.33|1992-04-30|1-URGENT|Clerk#000000802|0|r the requests sleep s 10151|589|F|10295.06|1992-01-15|1-URGENT|Clerk#000000645|0|ideas. deposits haggle quickly accounts; f 10176|985|O|102621.47|1996-08-26|3-MEDIUM|Clerk#000000456|0|unts. bold accounts are carefully according to the quickly p 10177|568|O|209213.44|1996-06-29|3-MEDIUM|Clerk#000000221|0|nstructions; bold, regular attainments among th 10178|1148|F|46311.36|1993-12-15|4-NOT SPECIFIED|Clerk#000000756|0|ray across the ironic, final dependencies. 10179|1157|O|153533.06|1996-04-30|4-NOT SPECIFIED|Clerk#000000803|0|sly bold packages haggle about the the 10180|712|O|76230.32|1997-03-10|1-URGENT|Clerk#000000743|0|ke blithely carefully even packag 10181|874|O|46249.63|1996-02-13|3-MEDIUM|Clerk#000000866|0|, pending deposits. express 10182|268|F|199446.10|1994-05-25|5-LOW|Clerk#000000304|0|s. slyly final pinto beans wake a 10183|250|F|110520.40|1994-11-25|1-URGENT|Clerk#000000534|0| dependencies. blithely pending requests about the furiously regular pac 10208|1028|O|286549.53|1996-08-26|3-MEDIUM|Clerk#000000259|0| ideas. slow, ironic instructions doze furiou 10209|1210|F|400191.77|1993-11-30|3-MEDIUM|Clerk#000000153|0|ts wake. slyly blithe 10210|937|O|148557.25|1995-06-24|4-NOT SPECIFIED|Clerk#000000573|0|ts. furiously bold ideas about the furiously bold pac 10211|797|O|192242.04|1997-07-29|4-NOT SPECIFIED|Clerk#000000363|0| careful theodolites. special accounts caj 10212|1471|F|126475.34|1993-05-24|5-LOW|Clerk#000000011|0|uickly according to the pending pinto beans. instructions brea 10213|1288|F|42898.55|1995-02-18|2-HIGH|Clerk#000000845|0|ly ironic dependencies. final excuses acros 10214|913|O|145287.98|1996-01-30|2-HIGH|Clerk#000000991|0|encies after the deposits sleep furiously bold packages. even deposits 10215|1031|O|156012.73|1996-08-17|1-URGENT|Clerk#000000770|0|pecial instructions. fluffily regular packa 10240|691|F|41343.99|1994-05-16|4-NOT SPECIFIED|Clerk#000000109|0|al accounts are along the unusual packages. unusual, even platelets use slyl 10241|271|O|54454.72|1997-09-30|5-LOW|Clerk#000000366|0|y unusual asymptotes. regular dependencies wake fluffily about the 10242|101|F|193761.97|1993-12-13|3-MEDIUM|Clerk#000000234|0|s was slyly alongside of the deposits. 10243|385|O|72189.37|1995-12-25|2-HIGH|Clerk#000000844|0|nic requests. boldly regular deposits haggle blithely. orbits integrate 10244|298|O|129419.40|1996-04-04|3-MEDIUM|Clerk#000000459|0|lent pinto beans. furiously iro 10245|587|F|268471.58|1995-02-14|3-MEDIUM|Clerk#000000785|0| beans. foxes haggle around the car 10246|1231|O|188201.88|1997-06-19|1-URGENT|Clerk#000000571|0|s. blithely unusual packages affix according to the hockey players. regular 10247|866|F|187970.24|1993-01-11|3-MEDIUM|Clerk#000000646|0|fully after the carefully pending requests. even, pending ideas alongside 10272|730|F|303855.22|1994-01-18|3-MEDIUM|Clerk#000000680|0| blithely express accounts. slyly regular pinto beans use slyly. 10273|112|O|272748.60|1998-04-24|1-URGENT|Clerk#000000303|0|n, ironic packages. slyly close accounts sleep. blithe 10274|1369|F|50775.14|1993-12-10|1-URGENT|Clerk#000000939|0|sits. fluffily bold requests cajole accounts 10275|83|O|78422.44|1997-05-16|5-LOW|Clerk#000000202|0|fts at the furiously express accounts are whithout the slyly slow deposits. f 10276|1463|O|264683.22|1996-02-24|3-MEDIUM|Clerk#000000706|0|y even deposits. even accounts nag among the ironic asymptotes. req 10277|538|F|139188.55|1994-04-08|3-MEDIUM|Clerk#000000655|0|counts sleep around the special 10278|359|O|162008.27|1995-09-13|5-LOW|Clerk#000000697|0|le quickly. pending, bold theo 10279|1223|O|227770.72|1996-09-22|5-LOW|Clerk#000000051|0|nal deposits. fluffily silent ideas are across the 10304|814|O|302630.21|1995-11-11|3-MEDIUM|Clerk#000000230|0|l deposits nag pending, regular attainments. ironic ideas detect. unusu 10305|25|F|153676.38|1994-07-08|5-LOW|Clerk#000000429|0|phins use furiously about the quickly bold id 10306|152|F|91506.19|1993-07-22|5-LOW|Clerk#000000345|0| slyly never final requests. perm 10307|298|O|230682.65|1998-01-28|4-NOT SPECIFIED|Clerk#000000488|0|yly ironic foxes. quickly regular 10308|413|P|278555.38|1995-04-21|2-HIGH|Clerk#000001000|0|are above the furiously final deposits. special packages nag s 10309|1312|F|215890.35|1994-11-22|5-LOW|Clerk#000000622|0| above the carefully bold packages. carefully even pinto beans mai 10310|1027|F|213874.91|1994-02-06|5-LOW|Clerk#000000645|0| accounts kindle qu 10311|52|F|5195.95|1992-08-18|2-HIGH|Clerk#000000147|0|ntegrate carefully above the regular pinto beans. quick 10336|1358|O|57610.67|1997-06-15|4-NOT SPECIFIED|Clerk#000000519|0|. carefully special pinto beans are ironic accounts. foxes 10337|1360|O|63068.37|1996-03-19|3-MEDIUM|Clerk#000000499|0|ly silent instructions wake qui 10338|1213|O|100334.47|1995-07-15|4-NOT SPECIFIED|Clerk#000000964|0|gular packages. slyly even packages haggle furious 10339|154|O|116081.18|1995-10-12|5-LOW|Clerk#000000585|0|ts. always ironic deposits haggle thinly. 10340|1486|O|243706.09|1995-06-23|2-HIGH|Clerk#000000064|0|hely final warthogs detect blithely regular pa 10341|1276|F|250921.23|1992-12-22|2-HIGH|Clerk#000000964|0|efully final packages cajole f 10342|638|O|273325.12|1995-11-23|3-MEDIUM|Clerk#000000277|0|theodolites. carefully regul 10343|91|F|44177.42|1992-03-27|4-NOT SPECIFIED|Clerk#000000600|0|ackages. carefully pending idea 10368|508|O|64465.48|1996-06-25|3-MEDIUM|Clerk#000000722|0|carefully special theodolites doze. fluffily ironic pinto be 10369|1397|O|120738.63|1995-11-10|5-LOW|Clerk#000000636|0| deposits are quickly; regular requests nag carefully regular i 10370|839|F|112585.48|1994-04-28|4-NOT SPECIFIED|Clerk#000000084|0|thrash. accounts boost never quickly final ac 10371|1022|F|204611.98|1994-09-17|2-HIGH|Clerk#000000231|0|nding requests. ideas af 10372|1241|O|101993.71|1995-06-28|4-NOT SPECIFIED|Clerk#000000276|0|sleep after the carefully ironic platelets. regular, unusual deposits wake qui 10373|671|F|254861.41|1993-06-26|1-URGENT|Clerk#000000681|0|cajole blithely final foxes. carefully 10374|337|F|28405.59|1993-07-23|5-LOW|Clerk#000000885|0|ing courts? carefully unusual ideas 10375|814|O|55990.35|1997-01-08|2-HIGH|Clerk#000000384|0|ending deposits. special, regular tith 10400|230|O|214820.02|1996-10-24|2-HIGH|Clerk#000000375|0|cing accounts sleep slyly regu 10401|1405|O|15159.38|1998-07-20|4-NOT SPECIFIED|Clerk#000000351|0|ts x-ray? slyly unusual a 10402|7|O|69976.43|1997-12-20|1-URGENT|Clerk#000000487|0|ing pinto beans for the carefully regular 10403|820|O|273217.58|1996-05-21|5-LOW|Clerk#000000523|0|ounts. carefully special accounts grow 10404|1255|O|148128.43|1998-03-29|1-URGENT|Clerk#000000142|0|uffy foxes about the qui 10405|929|O|156274.53|1998-01-04|4-NOT SPECIFIED|Clerk#000000343|0|uriously final dolphins. sly 10406|46|O|143470.97|1996-02-29|4-NOT SPECIFIED|Clerk#000000923|0| the slyly express deposits. deposits 10407|442|O|72476.48|1996-03-26|4-NOT SPECIFIED|Clerk#000000011|0|onic foxes. asymptotes across the ironi 10432|1231|F|167753.06|1992-01-06|5-LOW|Clerk#000000429|0|thely bold pinto bea 10433|1295|F|19194.57|1993-02-20|2-HIGH|Clerk#000000566|0|ajole blithely. furiously re 10434|781|F|63237.89|1994-12-24|2-HIGH|Clerk#000000939|0| blithely regular ideas are deposits. requests nod slyly final r 10435|1195|F|185835.34|1993-08-11|3-MEDIUM|Clerk#000000660|0|te. slyly final theodolites are after th 10436|916|F|32528.92|1995-01-03|5-LOW|Clerk#000001000|0|lthily special asymptotes. blithely pending deposits after the unusual, e 10437|305|F|64469.14|1994-08-19|1-URGENT|Clerk#000000098|0|inal requests about the regular, 10438|715|F|189906.48|1992-12-03|5-LOW|Clerk#000000148|0|tect among the blithely 10439|940|F|300103.82|1992-03-29|4-NOT SPECIFIED|Clerk#000000626|0|nal, even requests h 10464|233|F|41683.39|1994-08-11|1-URGENT|Clerk#000000752|0|e slyly after the platelets; furiously even ac 10465|79|O|289512.24|1997-04-10|4-NOT SPECIFIED|Clerk#000000791|0|ic tithes sleep carefully across the furiously ironic ideas. sl 10466|616|F|147826.40|1992-12-26|2-HIGH|Clerk#000000899|0|thely even ideas cajole quickly across the carefu 10467|1088|O|200254.31|1997-03-12|4-NOT SPECIFIED|Clerk#000000125|0|ffily unusual deposits. carefully regular asymptotes use fur 10468|574|F|50946.66|1994-12-12|2-HIGH|Clerk#000000429|0|osits wake-- quickly pending foxes alongside of the slyly even 10469|622|O|89980.03|1996-09-03|2-HIGH|Clerk#000000913|0|hely unusual, unusual accounts. furiously fi 10470|184|F|321900.81|1992-05-05|5-LOW|Clerk#000000632|0|even, ironic accounts among the bold theodolites use across the ste 10471|262|F|309852.68|1992-10-10|2-HIGH|Clerk#000000590|0|lites wake doggedly accounts. blithely final deposits sleep evenly. 10496|400|O|66825.60|1997-02-20|2-HIGH|Clerk#000000907|0|iously regular requests. furiously slow hockey players are quickly even ac 10497|119|O|103079.38|1996-06-22|3-MEDIUM|Clerk#000000423|0|ar hockey players sleep furiously until the furious 10498|1081|F|146950.27|1993-10-24|5-LOW|Clerk#000000424|0|lithely bold packages. re 10499|872|F|186269.45|1993-11-17|4-NOT SPECIFIED|Clerk#000000065|0|es haggle slyly even instructions. ironica 10500|998|F|26808.34|1994-01-24|2-HIGH|Clerk#000000179|0| packages wake quickly pearls. slyly even deposits haggle blithe 10501|911|O|227512.69|1998-07-03|3-MEDIUM|Clerk#000000977|0|equests. packages nag slyly. quickly even accounts toward the ironic 10502|1462|F|177583.01|1993-05-15|4-NOT SPECIFIED|Clerk#000000530|0|tructions alongside of the blithely bold foxes could have to sleep slyly a 10503|853|O|150884.39|1996-06-09|5-LOW|Clerk#000000028|0|eodolites play against the pending accounts. special packages are slyly ab 10528|166|F|127054.41|1994-09-14|3-MEDIUM|Clerk#000000724|0|press pinto beans. express theodolites do cajole furiously. slyly f 10529|692|O|141709.31|1997-10-18|1-URGENT|Clerk#000000573|0|yly. doggedly final epitaphs haggle among the flu 10530|1276|F|120484.95|1994-01-28|3-MEDIUM|Clerk#000000879|0|ounts. blithely unus 10531|55|F|176261.61|1992-03-19|2-HIGH|Clerk#000000021|0|regular somas. fluffily special instructions doze blithely. quick 10532|1337|O|318050.97|1997-11-28|5-LOW|Clerk#000000805|0|ly slyly bold accounts. carefully express theodolites nag 10533|961|O|186070.73|1998-05-01|3-MEDIUM|Clerk#000000778|0| nag busily bold dependencies. special, ironic p 10534|1070|F|4438.69|1993-04-19|4-NOT SPECIFIED|Clerk#000000973|0|ly according to the slyly r 10535|1225|O|249751.99|1995-05-28|5-LOW|Clerk#000000967|0|lyly unusual frets nag alongsi 10560|91|O|185333.40|1997-07-29|1-URGENT|Clerk#000000157|0|lar requests among the blithely regular packages doze unusual requests; 10561|718|O|219244.25|1997-04-27|5-LOW|Clerk#000000435|0|r requests. even, even deposits b 10562|1246|F|297373.68|1994-11-12|4-NOT SPECIFIED|Clerk#000000034|0|ests across the courts print r 10563|2|F|143707.70|1993-09-30|1-URGENT|Clerk#000000889|0|etect after the pending requests. packa 10564|1019|O|77547.24|1996-03-14|4-NOT SPECIFIED|Clerk#000000362|0|ests. final, express foxes are slyly regular ideas. darin 10565|613|O|138255.60|1997-08-17|1-URGENT|Clerk#000000126|0| slyly unusual deposits after the carefully 10566|578|O|220319.71|1995-08-20|4-NOT SPECIFIED|Clerk#000000541|0|s sleep blithely permanently even ideas. bl 10567|328|F|104838.06|1992-03-30|2-HIGH|Clerk#000000046|0|y even pinto beans cajole bold instructions. carefully sly wate 10592|778|O|84645.88|1995-11-04|2-HIGH|Clerk#000000435|0|as-- ironic packages 10593|1123|F|160662.68|1992-02-13|2-HIGH|Clerk#000000573|0|excuses after the fina 10594|214|O|6650.94|1996-03-21|3-MEDIUM|Clerk#000000126|0|ly express asymptotes against th 10595|818|O|23233.65|1997-11-22|3-MEDIUM|Clerk#000000425|0|gular excuses. slowly ca 10596|308|O|106243.04|1997-03-31|5-LOW|Clerk#000000662|0|oss the furiously regular theodolit 10597|619|O|67904.94|1996-09-13|2-HIGH|Clerk#000000040|0|ch. bold ideas about the deposits wake fluffily 10598|461|F|19331.61|1992-03-08|4-NOT SPECIFIED|Clerk#000000682|0|e among the fluffily regular deposits. slyly final requests haggle furious 10599|194|O|142776.50|1997-07-21|4-NOT SPECIFIED|Clerk#000000559|0|ly regular requests wake regular instructions. pending, pending dolphins 10624|1288|F|192178.14|1994-12-21|1-URGENT|Clerk#000000666|0|re furiously along the slyly final ideas. carefu 10625|259|O|233381.66|1996-01-30|1-URGENT|Clerk#000000688|0|. pending deposits above the car 10626|997|O|19103.82|1995-12-14|3-MEDIUM|Clerk#000000656|0| regular deposits cajo 10627|127|F|210497.85|1993-02-25|5-LOW|Clerk#000000236|0|s. unusual instructions are. furiously express requests a 10628|1429|O|5714.19|1998-06-28|4-NOT SPECIFIED|Clerk#000000437|0|etimes quickly final packa 10629|650|F|55072.00|1994-09-12|1-URGENT|Clerk#000000229|0|endencies wake. accounts are carefully. a 10630|313|F|143006.19|1993-02-05|3-MEDIUM|Clerk#000000607|0|iously special instruct 10631|1243|F|258430.09|1993-05-27|1-URGENT|Clerk#000000844|0|affix blithely blithely regular dinos. furiously blithe instructions ar 10656|371|O|96681.23|1995-10-25|1-URGENT|Clerk#000000233|0| accounts use never blithely bold deposits. i 10657|734|F|91096.35|1993-09-26|3-MEDIUM|Clerk#000000502|0|ts wake. bold packages cajole fluffily according to the furi 10658|1199|F|117872.35|1994-08-28|5-LOW|Clerk#000000204|0|xes cajole thinly ironic requests. theodolites 10659|397|F|290769.79|1994-05-15|1-URGENT|Clerk#000000587|0| slyly across the bold request 10660|469|O|135068.75|1998-05-01|2-HIGH|Clerk#000000352|0|onic depths? furiously 10661|179|F|90250.70|1994-01-24|4-NOT SPECIFIED|Clerk#000000986|0| about the final foxes. eve 10662|1420|P|157252.72|1995-05-13|5-LOW|Clerk#000000241|0|tain blithely quickly regular pinto beans. furiously bold deposits 10663|317|F|153682.58|1994-05-24|2-HIGH|Clerk#000000137|0|s above the furiously final foxes nag idly requests. caref 10688|4|F|43453.24|1992-05-13|3-MEDIUM|Clerk#000000226|0|ly ironic instructions lose quickly alongside of the carefully 10689|203|O|99242.62|1996-07-23|5-LOW|Clerk#000000436|0|usly. furiously ironic instructions wak 10690|103|O|54775.41|1997-12-15|5-LOW|Clerk#000000041|0|ltipliers cajole slyly busy accounts. express theodolites af 10691|664|F|87254.45|1995-03-14|3-MEDIUM|Clerk#000000536|0|al decoys among the carefull 10692|187|F|158201.74|1994-07-27|5-LOW|Clerk#000000832|0|orbits. asymptotes cajole. furiously eve 10693|1030|F|43518.47|1995-04-17|4-NOT SPECIFIED|Clerk#000000797|0|s. slyly regular accounts are among the slyly regular sentiments. 10694|1423|O|183151.32|1997-12-05|2-HIGH|Clerk#000000812|0|olites. quickly bold package 10695|1456|P|93566.64|1995-05-03|2-HIGH|Clerk#000000039|0| asymptotes. quickly unus 10720|736|O|89368.05|1998-03-31|2-HIGH|Clerk#000000597|0|ccounts sleep bold, even theodol 10721|781|O|107381.35|1996-02-27|5-LOW|Clerk#000000599|0|haggle slyly blithely even instructions. quickly even dolphins wa 10722|79|F|218680.31|1995-01-20|4-NOT SPECIFIED|Clerk#000000743|0|ding instructions print. f 10723|1040|O|69126.13|1998-05-21|3-MEDIUM|Clerk#000000144|0| bold accounts. carefully busy platelets sleep fluffily. spe 10724|1492|F|215958.98|1994-09-14|2-HIGH|Clerk#000000165|0|furiously regular dolphins. regular foxes along the daringly express 10725|1444|O|77193.41|1998-04-09|5-LOW|Clerk#000000505|0| carefully unusual sauternes after the fluffily regular foxes use pending, 10726|164|F|215356.45|1993-04-25|4-NOT SPECIFIED|Clerk#000000210|0| regular dependencies. quickly spe 10727|496|F|311532.26|1992-08-05|5-LOW|Clerk#000000396|0|nusual pinto beans. special, pending requests breach slyly fina 10752|199|O|223930.05|1995-09-11|1-URGENT|Clerk#000000610|0|ic accounts. carefully ironic courts against the caref 10753|1430|F|20768.66|1994-05-02|1-URGENT|Clerk#000000218|0|ously ironic packag 10754|1061|F|229997.58|1993-03-18|5-LOW|Clerk#000000827|0|r pinto beans sleep even 10755|1435|F|132981.98|1993-07-15|1-URGENT|Clerk#000000257|0|se furiously excuses. always unusual packages above the furiously unusual de 10756|1087|F|40769.85|1992-04-15|2-HIGH|Clerk#000000994|0| even, express platelets nag above 10757|128|O|92703.05|1998-06-06|4-NOT SPECIFIED|Clerk#000001000|0|patterns haggle quickly across the bravely 10758|368|O|136484.05|1996-11-15|2-HIGH|Clerk#000000209|0|es. pinto beans sleep carefully final 10759|34|O|54534.36|1996-09-22|4-NOT SPECIFIED|Clerk#000000258|0| boost evenly final pinto beans. 10784|1396|F|184632.93|1993-07-06|4-NOT SPECIFIED|Clerk#000000982|0|en packages sleep. final foxes detec 10785|148|F|1508.63|1992-06-26|3-MEDIUM|Clerk#000000948|0|iously unusual packages use quickly furiously 10786|154|O|70962.05|1995-08-06|3-MEDIUM|Clerk#000000705|0| ironic, special theodolites caj 10787|1117|O|372954.40|1997-02-12|2-HIGH|Clerk#000000133|0|ckages could have to wake blithely above the final requests. furiously 10788|4|O|147767.08|1997-01-15|3-MEDIUM|Clerk#000000905|0|esides the slyly ironic dependencies. ironic foxes cajole fur 10789|1378|F|93918.24|1993-08-24|4-NOT SPECIFIED|Clerk#000000463|0|s unwind blithely agains 10790|113|F|77512.26|1994-12-15|5-LOW|Clerk#000000381|0|ly alongside of the express, regular foxes. req 10791|319|O|173850.27|1995-08-20|4-NOT SPECIFIED|Clerk#000000131|0|efully special pinto beans dazzle fluffily according to the express, silen 10816|1303|O|45628.88|1996-01-11|2-HIGH|Clerk#000000003|0|after the final, spe 10817|712|O|213714.56|1996-12-09|1-URGENT|Clerk#000000852|0|final ideas across the 10818|157|O|245383.91|1998-05-07|2-HIGH|Clerk#000000735|0|ids nag carefully against the final deposit 10819|571|F|79971.29|1993-08-25|1-URGENT|Clerk#000000144|0|ncies use furiously alongside of the fluffily final instructions. quickly re 10820|661|O|220224.56|1995-12-28|4-NOT SPECIFIED|Clerk#000000630|0|ecial ideas. silently unusual somas are blith 10821|590|O|179780.96|1996-05-13|2-HIGH|Clerk#000000856|0| the furiously final excuses. blithely final pack 10822|1186|F|19208.70|1993-12-01|5-LOW|Clerk#000000046|0|hely regular requests. carefully silent instructions use slyly f 10823|391|F|109958.79|1992-06-13|5-LOW|Clerk#000000869|0|blithely boldly pending deposits. silent hockey players acros 10848|841|O|67233.17|1996-10-26|3-MEDIUM|Clerk#000000131|0| according to the boldly final accounts. slyly bold asymptotes detect qui 10849|56|O|309269.38|1997-01-31|1-URGENT|Clerk#000000279|0|y even requests. blithely silent theodolites cajole carefully ev 10850|1052|O|78474.67|1996-09-23|2-HIGH|Clerk#000000055|0|its cajole quickly. permanently even pinto beans dazz 10851|1201|F|256653.25|1994-04-07|2-HIGH|Clerk#000000841|0|ffix blithely blithely even dugouts? quickly re 10852|559|F|290873.68|1993-10-27|2-HIGH|Clerk#000000623|0|oxes. carefully final pinto be 10853|275|F|68787.35|1994-01-06|4-NOT SPECIFIED|Clerk#000000276|0|ronic dugouts wake s 10854|917|O|4885.38|1998-06-20|2-HIGH|Clerk#000000219|0|old, ironic dependencies. fluffily bold excuses shall have to integrate amon 10855|608|O|227086.67|1997-07-16|1-URGENT|Clerk#000000149|0|fully unusual pinto beans sleep sl 10880|278|F|40046.10|1992-05-23|4-NOT SPECIFIED|Clerk#000000553|0|e furiously bold deposits. slyly spe 10881|1001|F|235385.68|1994-02-16|1-URGENT|Clerk#000000872|0|quickly final pinto beans engage quickly pending, final 10882|1462|O|103982.52|1998-06-02|5-LOW|Clerk#000000949|0|gouts sleep fluffily according to the dep 10883|226|O|118687.62|1998-02-03|5-LOW|Clerk#000000759|0|sly silent accounts. carefully express pinto bean 10884|749|F|193592.93|1994-11-26|5-LOW|Clerk#000000802|0| express, regular accounts nag furiously! platele 10885|991|O|253167.35|1998-04-18|5-LOW|Clerk#000000893|0|thely ironic accounts sleep carefully depo 10886|1312|F|275313.31|1994-08-24|1-URGENT|Clerk#000000422|0|along the final epitaphs dazzle carefull 10887|436|O|50293.15|1995-05-26|4-NOT SPECIFIED|Clerk#000000575|0|ag against the dependencies. furiously regular foxes sleep fu 10912|293|O|243339.25|1996-12-25|1-URGENT|Clerk#000000579|0|odolites wake sometimes on the unusual deposits. epitaphs cajole b 10913|514|F|125458.51|1993-02-07|5-LOW|Clerk#000000114|0|ches are blithely around t 10914|689|F|213268.60|1995-01-23|4-NOT SPECIFIED|Clerk#000000582|0| quickly regular packages above the furiously bold pl 10915|1357|F|178357.32|1993-11-11|5-LOW|Clerk#000000649|0|bold packages nag. furiously regular excu 10916|328|P|253026.49|1995-03-11|3-MEDIUM|Clerk#000000047|0|pinto beans cajole furiously according to the dependencies. fi 10917|251|F|127026.23|1992-02-11|1-URGENT|Clerk#000000078|0|beans across the regular theodolites affix silent instructions. even deposits 10918|1030|O|80094.04|1995-10-16|3-MEDIUM|Clerk#000000874|0|structions. ironic, express dep 10919|1339|F|56302.36|1993-03-28|2-HIGH|Clerk#000000180|0|press requests haggle blithely. blithely 10944|319|O|129953.86|1995-11-13|2-HIGH|Clerk#000000077|0|ily unusual requests. special instruct 10945|248|F|168422.71|1992-01-04|3-MEDIUM|Clerk#000000296|0|ages are. final, silent deposits haggle quickly! ironic 10946|1147|O|332313.82|1997-03-22|3-MEDIUM|Clerk#000000975|0|nal deposits detect among the quickly 10947|1070|O|135370.80|1996-01-07|2-HIGH|Clerk#000000347|0|ar excuses x-ray fluffily a 10948|670|F|223943.13|1992-08-14|3-MEDIUM|Clerk#000000489|0|nic grouches detect stealthily fluffily even de 10949|331|F|112812.10|1994-03-31|3-MEDIUM|Clerk#000000874|0|luffily even grouches. regular notornis against the sl 10950|767|O|37856.60|1997-06-11|3-MEDIUM|Clerk#000000015|0|y carefully pending packages. silent accounts nod careful 10951|1159|F|175198.95|1992-11-07|4-NOT SPECIFIED|Clerk#000000619|0|dependencies. sometimes pending foxes cajole. quickly 10976|685|F|186488.10|1992-10-02|4-NOT SPECIFIED|Clerk#000000282|0| deposits: brave, ironic packages maintain fluffily about the regular 10977|757|O|99211.94|1998-06-04|2-HIGH|Clerk#000000089|0|rious accounts about the regular excuses wake ironically special r 10978|1346|F|106794.18|1994-04-02|1-URGENT|Clerk#000000546|0|cial multipliers wake pending, pending theodolites: blithe 10979|1267|O|166156.36|1995-10-05|3-MEDIUM|Clerk#000000527|0|ans cajole carefully. regular pinto be 10980|754|O|75950.23|1996-07-29|4-NOT SPECIFIED|Clerk#000000818|0| ironic, stealthy patterns detect 10981|919|F|235684.93|1993-05-15|2-HIGH|Clerk#000000341|0|ly bold attainments nag! furiously special acco 10982|739|F|199304.65|1992-12-11|4-NOT SPECIFIED|Clerk#000000481|0|tithes use slyly silent, unusual a 10983|599|O|106734.94|1997-02-06|1-URGENT|Clerk#000000299|0|sts sleep of the regular foxes. permanent re 11008|523|F|222231.62|1994-02-25|1-URGENT|Clerk#000000927|0|ickly regular deposits detect fluffily: caref 11009|1303|O|65554.60|1997-03-05|3-MEDIUM|Clerk#000000515|0|deposits are blithely furio 11010|440|O|163532.15|1997-03-22|4-NOT SPECIFIED|Clerk#000000659|0|ithely-- express, final ideas sleep slyly to the blithely express reque 11011|14|F|248409.17|1992-05-14|2-HIGH|Clerk#000000948|0|luffily even warhors 11012|1420|O|114085.80|1998-04-07|1-URGENT|Clerk#000000771|0|ze fluffily quickly sly excuses. quickly pending frays haggle accordin 11013|395|F|2591.65|1994-04-18|2-HIGH|Clerk#000000759|0|ng deposits nag. special deposits sublate furio 11014|319|F|150733.48|1993-09-08|2-HIGH|Clerk#000000934|0|yly regular depths. re 11015|514|O|132711.70|1997-09-16|5-LOW|Clerk#000000216|0|its cajole against the unusual ideas? sly deposits boost fluffily. furiously 11040|82|O|211495.01|1995-10-24|4-NOT SPECIFIED|Clerk#000000003|0|arefully pending dolphins print brave 11041|1306|O|128785.82|1997-03-30|4-NOT SPECIFIED|Clerk#000000892|0|instructions. blithely pending packages solve of the furiously ironic requests 11042|857|O|223492.72|1998-05-07|4-NOT SPECIFIED|Clerk#000000677|0|ccounts. quickly unusual pinto beans nag enticingly a 11043|820|F|102166.94|1995-02-20|1-URGENT|Clerk#000000196|0|dolphins. accounts are f 11044|164|F|68884.04|1992-06-25|3-MEDIUM|Clerk#000000861|0|he deposits hinder above the furiously express excuses. silent depo 11045|556|F|226703.02|1992-03-26|3-MEDIUM|Clerk#000000216|0|uriously ironic asy 11046|1411|O|337558.78|1997-06-17|5-LOW|Clerk#000000454|0|lly special theodolites use furiously. carefully 11047|175|F|70365.24|1993-12-11|5-LOW|Clerk#000000223|0|. pending accounts alongside o 11072|265|O|207339.71|1996-07-14|2-HIGH|Clerk#000000804|0|osits: pending requests sleep quietly silently unusual deposits. special fo 11073|548|F|177610.35|1994-12-02|4-NOT SPECIFIED|Clerk#000000009|0|ing deposits-- slyly express dolphins detect fluffily ironic dep 11074|709|O|302642.18|1995-06-10|2-HIGH|Clerk#000000449|0|t the special asymptotes. furiously even dolphins sleep quickly ir 11075|1244|O|215012.92|1996-03-05|5-LOW|Clerk#000000379|0|efully furiously special pinto beans. fluffily unusual acc 11076|1012|F|54498.05|1994-08-18|1-URGENT|Clerk#000000361|0|lyly special notornis. bold requests haggle careful 11077|610|F|76484.04|1994-03-21|5-LOW|Clerk#000000024|0|gularly pending deposits. quick 11078|1301|F|103896.23|1992-05-28|2-HIGH|Clerk#000000227|0|ag. ironic, unusual accounts nag fluffily across the ironic, final ideas. car 11079|218|O|98046.49|1997-08-11|3-MEDIUM|Clerk#000000507|0|tainments affix doggedly slyly special asymptotes. 11104|1315|O|51132.08|1998-07-07|4-NOT SPECIFIED|Clerk#000000062|0|s cajole slyly alon 11105|1124|O|20773.48|1995-11-30|4-NOT SPECIFIED|Clerk#000000769|0|blithely about the closely silent foxes. blithely r 11106|1175|P|59390.31|1995-04-01|2-HIGH|Clerk#000000828|0|fter the quickly bold theodolites are blithely across 11107|445|O|64188.60|1995-10-24|5-LOW|Clerk#000000896|0|ymptotes. escapades use slyly? slyly b 11108|1186|F|238113.15|1992-08-17|4-NOT SPECIFIED|Clerk#000000137|0|ond the carefully unusual courts. regul 11109|664|F|5341.59|1995-03-16|3-MEDIUM|Clerk#000000578|0|. silent platelets haggle slyly. quickly ironic asymptotes 11110|757|O|153722.23|1996-06-23|4-NOT SPECIFIED|Clerk#000000243|0|es are quickly alongside of the spe 11111|631|F|121957.72|1995-01-18|3-MEDIUM|Clerk#000000301|0|nic deposits affix quickly regula 11136|1313|F|60530.16|1993-11-14|4-NOT SPECIFIED|Clerk#000000987|0|ular foxes haggle against the unusual frays. i 11137|668|F|204264.90|1992-11-30|2-HIGH|Clerk#000000515|0|bout the final accounts. regular ideas cajole slyly. blithely sly d 11138|181|F|162678.89|1995-02-27|4-NOT SPECIFIED|Clerk#000000983|0|d platelets. carefully ironic packages alongside of the carefully 11139|883|O|115264.28|1998-05-17|5-LOW|Clerk#000000891|0|the furiously silent i 11140|988|O|194048.42|1997-07-26|1-URGENT|Clerk#000000117|0|. fluffily busy sauternes cajole slyly above the quickly express excuses. exp 11141|1030|O|72800.37|1998-01-17|1-URGENT|Clerk#000000084|0|ckages. carefully regul 11142|1375|O|395039.05|1997-10-03|4-NOT SPECIFIED|Clerk#000000457|0| even accounts sublate carefully 11143|97|F|172537.11|1992-11-27|5-LOW|Clerk#000000613|0|unusual instructions. q 11168|1441|F|74602.03|1992-10-22|1-URGENT|Clerk#000000585|0|es. even patterns use fluff 11169|1222|F|121781.45|1993-05-22|3-MEDIUM|Clerk#000000536|0|eodolites wake. fluffily express packages 11170|736|O|152992.55|1997-07-30|1-URGENT|Clerk#000000832|0|eyond the furiously regular pinto beans. car 11171|730|O|104672.68|1995-07-17|1-URGENT|Clerk#000000792|0|l deposits promise according to the express deposits. fluffily final r 11172|1076|O|166824.37|1997-12-30|1-URGENT|Clerk#000000187|0|ructions. furiously pending dependencies wake. deposits 11173|319|F|179045.62|1992-03-31|1-URGENT|Clerk#000000815|0|e ironically silent deposits wake blithely across the regular instructions 11174|1315|F|149433.95|1994-04-10|3-MEDIUM|Clerk#000000321|0|se about the pending, special decoys-- fluffily express 11175|1421|O|44114.12|1998-02-26|2-HIGH|Clerk#000000812|0| slyly even deposits are blithely; packages integrate furiously. bravely ironi 11200|956|O|113819.40|1997-02-07|1-URGENT|Clerk#000000417|0|ajole. silent requests boost carefully blithely thin multiplier 11201|632|O|3297.46|1996-12-08|3-MEDIUM|Clerk#000000746|0|ns cajole after the pending pinto beans. carefully ironic requests wake r 11202|1168|F|157836.48|1992-03-18|3-MEDIUM|Clerk#000000614|0|wake daringly. carefully regular account 11203|1498|O|84795.43|1996-06-15|2-HIGH|Clerk#000000010|0|al instructions haggle slyly. fu 11204|1249|O|29604.53|1998-07-25|2-HIGH|Clerk#000000752|0| express requests. slyly special packages cajole fluffily; quickly ironi 11205|1294|O|93334.58|1996-09-15|1-URGENT|Clerk#000000800|0|ckages. furiously final packages ac 11206|379|F|209373.08|1994-12-29|5-LOW|Clerk#000000139|0|ending packages cajole busily. slyly 11207|1213|O|84129.92|1996-04-03|2-HIGH|Clerk#000000354|0|ress accounts haggle permanently about the furiously unusua 11232|661|O|77549.43|1996-02-01|1-URGENT|Clerk#000000676|0|fully regular accounts. final, unusual deposits sleep eve 11233|680|O|96728.84|1995-08-31|4-NOT SPECIFIED|Clerk#000000820|0|lphins. carefully ironic ideas use after the quickly unusual a 11234|1153|F|117092.84|1993-03-01|4-NOT SPECIFIED|Clerk#000000291|0|e slyly across the carefully even requests. slyly 11235|590|O|278256.83|1996-04-03|4-NOT SPECIFIED|Clerk#000000495|0|ackages solve carefully furiously final accounts. fluf 11236|223|F|210654.72|1993-10-31|2-HIGH|Clerk#000000792|0|ess pinto beans. foxes b 11237|1423|O|155158.38|1996-06-16|4-NOT SPECIFIED|Clerk#000000627|0|e the carefully special ideas inte 11238|535|O|126764.00|1996-07-23|1-URGENT|Clerk#000000809|0|gular foxes run after the even instructions. regular instructions wake. unu 11239|475|F|310241.51|1992-02-10|1-URGENT|Clerk#000000240|0|e regular, even excuses. 11264|676|O|152483.91|1996-09-02|4-NOT SPECIFIED|Clerk#000000227|0|usly silent deposits. carefully unusual asymptotes ha 11265|905|O|88998.81|1997-05-20|3-MEDIUM|Clerk#000000427|0|ias. carefully pending platel 11266|292|O|13609.27|1997-08-10|4-NOT SPECIFIED|Clerk#000000315|0|lets nag about the carefully special packages. ironic sheaves 11267|548|F|45572.31|1992-02-15|5-LOW|Clerk#000000096|0| wake slyly even deposits. c 11268|43|O|189596.87|1998-06-30|1-URGENT|Clerk#000000130|0|ptotes are blithely slyly silen 11269|199|F|256118.53|1992-05-27|5-LOW|Clerk#000000752|0|ath the decoys. final excuses must have to wake. ironic asymptotes wake q 11270|160|O|197781.66|1995-07-01|2-HIGH|Clerk#000000757|0|ily pending theodolites a 11271|745|O|255335.88|1995-10-19|2-HIGH|Clerk#000000144|0|sts. furiously ironic foxes wake ca 11296|832|F|378166.33|1992-01-10|1-URGENT|Clerk#000000966|0|s-- ironic, unusual requests haggle furiously. carefully special depend 11297|694|F|109722.85|1993-01-09|4-NOT SPECIFIED|Clerk#000000247|0|hely. express escapades kindle blithely. even requests wake 11298|805|O|228663.94|1998-01-03|2-HIGH|Clerk#000000160|0| blithely express ideas sleep slyly ruthl 11299|50|F|253114.92|1993-11-13|4-NOT SPECIFIED|Clerk#000000885|0|ns haggle bravely alongside of the fluffily express requests. 11300|346|O|77247.48|1996-07-10|5-LOW|Clerk#000000697|0|he final platelets. blithely even theodolites along the car 11301|1358|F|191271.29|1992-03-23|3-MEDIUM|Clerk#000000321|0|hes detect fluffily ironic requests. deposits br 11302|451|F|243997.09|1994-03-02|5-LOW|Clerk#000000492|0|. never ironic requests sleep furiousl 11303|388|F|92968.88|1992-03-30|5-LOW|Clerk#000000788|0|he slyly pending requests. final deposits 11328|466|F|18702.67|1992-03-21|1-URGENT|Clerk#000000101|0|theodolites. blithely pending accounts above the carefully 11329|1285|O|272307.07|1995-08-05|1-URGENT|Clerk#000000903|0|on the daringly express instructions. fluf 11330|281|F|3817.37|1992-03-28|5-LOW|Clerk#000000429|0|he slyly ironic ideas are q 11331|179|O|18118.60|1996-01-21|3-MEDIUM|Clerk#000000616|0|platelets wake carefully ruthless requests. silent deposits believe 11332|422|F|213792.99|1994-11-17|5-LOW|Clerk#000000572|0|o the unusual foxes-- ironic requests wake slyly enticingly re 11333|103|F|15730.53|1994-06-10|2-HIGH|Clerk#000000914|0|excuses cajole blithely against the slyly express r 11334|245|O|343561.63|1997-07-29|3-MEDIUM|Clerk#000000843|0|ronic deposits haggle fu 11335|871|F|133549.00|1994-10-22|2-HIGH|Clerk#000000669|0|ealms. theodolites maintain. regular, even instructions against t 11360|886|O|67314.59|1997-09-07|1-URGENT|Clerk#000000561|0|. slyly regular deposits lose 11361|442|F|251600.98|1994-09-28|1-URGENT|Clerk#000000345|0| final requests boost never pinto beans. special accounts are slyly unusual t 11362|1042|F|250063.55|1992-09-11|5-LOW|Clerk#000000922|0|ly blithe deposits cajole blithely slyly silent deposits. regular 11363|1358|O|163511.72|1998-02-09|5-LOW|Clerk#000000844|0| silent pinto beans haggle al 11364|163|O|107551.91|1997-03-11|1-URGENT|Clerk#000000582|0|sts wake fluffily about the slyly special accounts. qu 11365|1214|O|211938.69|1997-08-29|3-MEDIUM|Clerk#000000520|0|de of the blithely final requests. fluffily regular dolphin 11366|863|F|12711.31|1992-06-08|3-MEDIUM|Clerk#000000993|0|lites nag blithely un 11367|418|F|18148.48|1994-12-17|1-URGENT|Clerk#000000566|0| carefully regular deposits. bold theodolites haggle. enticingly final fo 11392|1340|F|135223.44|1994-09-06|5-LOW|Clerk#000000607|0|tect blithely across the express dependencies. instructions after the slyly re 11393|1150|F|138539.25|1992-03-19|5-LOW|Clerk#000000022|0|al packages. carefully speci 11394|1255|O|181795.18|1998-03-06|2-HIGH|Clerk#000000559|0| deposits! fluffily sp 11395|940|O|46826.01|1997-08-26|1-URGENT|Clerk#000000306|0|beans nag carefully even sentiments-- express platelets 11396|1273|F|107605.76|1992-07-22|1-URGENT|Clerk#000000265|0|ously regular accounts haggle 11397|757|O|55531.65|1996-10-29|5-LOW|Clerk#000000603|0|oss the regular pac 11398|587|F|258130.52|1992-08-07|2-HIGH|Clerk#000000407|0|f the theodolites are fu 11399|346|F|39653.25|1994-04-19|5-LOW|Clerk#000000460|0|ffily furiously even pa 11424|343|F|58887.47|1993-04-03|4-NOT SPECIFIED|Clerk#000000547|0|s. quickly final requests around the slyly unusual dependencies cajol 11425|1123|O|156058.42|1995-06-23|1-URGENT|Clerk#000000336|0|arefully special requests cajole bravely fluffy pinto beans. ironic 11426|1357|F|94572.71|1994-05-08|3-MEDIUM|Clerk#000000066|0|ilent notornis boost blithely quickly even pinto beans. carefully speci 11427|427|F|306906.43|1993-11-25|3-MEDIUM|Clerk#000000401|0|y slyly brave excuses. slow packages sleep quickl 11428|628|F|7046.94|1992-01-22|3-MEDIUM|Clerk#000000926|0|ep slyly even, unusual packages. spe 11429|1192|F|145545.35|1993-03-04|5-LOW|Clerk#000000765|0| bold courts sleep blithely. regular, even r 11430|1342|O|3641.35|1998-01-05|5-LOW|Clerk#000000817|0|tions breach. regular, express pinto beans wake across the carefully 11431|10|F|230289.60|1992-08-02|2-HIGH|Clerk#000000055|0|press ideas use slyly regular pinto beans. furiously 11456|271|F|53466.58|1993-04-13|5-LOW|Clerk#000000209|0|ar foxes against the quickly special th 11457|1430|O|232660.01|1995-12-14|1-URGENT|Clerk#000000294|0|the slyly quiet ideas. idly final deposits nag. carefully regu 11458|1468|O|201085.02|1998-04-02|5-LOW|Clerk#000000402|0|l pinto beans cajole carefully 11459|325|O|230925.13|1996-07-15|4-NOT SPECIFIED|Clerk#000000852|0|ular requests use blithely. quickly special packages boost furiously. deposits 11460|763|F|51498.95|1993-08-25|2-HIGH|Clerk#000000014|0| brave dependencies nag. blithely final foxe 11461|1049|O|80204.90|1996-05-01|1-URGENT|Clerk#000000630|0|ymptotes. packages haggle whithout the evenly final requests. req 11462|226|O|164290.80|1996-08-25|2-HIGH|Clerk#000000210|0|grouches. closely dogged deposits sleep. special, pending packages slee 11463|140|O|258622.38|1997-12-28|3-MEDIUM|Clerk#000000886|0|ular deposits. slyly final attainments detect careful 11488|751|F|92903.96|1993-08-23|4-NOT SPECIFIED|Clerk#000000119|0|ickly final requests. furiously even attainments sleep fluffily slyly regu 11489|929|O|100935.67|1996-09-19|4-NOT SPECIFIED|Clerk#000000155|0|ions. ironic foxes wake. regular platelets cajole slyly close 11490|745|O|66256.76|1996-09-27|1-URGENT|Clerk#000000489|0|quests affix according to the special p 11491|158|F|242695.19|1993-08-18|5-LOW|Clerk#000000329|0|cial foxes across the final packag 11492|478|O|141195.82|1997-03-06|2-HIGH|Clerk#000000708|0|ely regular instructions was fluffily among the fluffil 11493|1486|O|63852.92|1995-10-21|1-URGENT|Clerk#000000543|0|e blithely regular accounts 11494|1484|O|90512.73|1997-07-20|3-MEDIUM|Clerk#000000486|0|s. blithely unusual pi 11495|1048|O|43956.73|1995-07-05|2-HIGH|Clerk#000000684|0|y bold accounts cajo 11520|1217|F|111920.94|1994-09-28|3-MEDIUM|Clerk#000000106|0|courts. carefully final requests along the carefully final 11521|55|O|19730.00|1996-09-09|1-URGENT|Clerk#000000010|0| express platelets according to the iro 11522|47|F|21454.81|1993-09-16|5-LOW|Clerk#000000121|0|ual foxes x-ray carefully alo 11523|355|O|210039.66|1998-07-27|4-NOT SPECIFIED|Clerk#000000008|0|y: fluffily regular instructions sleep quickly enticing orbits. carefully un 11524|766|O|7445.25|1995-12-14|5-LOW|Clerk#000000943|0|encies. foxes detect around the final, even theodolites. special depos 11525|1495|O|169021.86|1997-03-20|1-URGENT|Clerk#000000159|0|ong the ironic accounts are slyly furiously pending dependencies. even exc 11526|466|O|124012.92|1998-02-23|5-LOW|Clerk#000000798|0|ickly regular packages use slyly ironic, 11527|1108|F|46961.88|1994-07-03|3-MEDIUM|Clerk#000000187|0|ously final packages was slyly above the bli 11552|1421|F|34742.82|1992-12-15|1-URGENT|Clerk#000000960|0|g, dogged notornis. carefully even instructions across the furiou 11553|196|F|207629.22|1994-04-19|3-MEDIUM|Clerk#000000467|0|s thrash blithely carefully ironic requests. quickl 11554|344|O|269777.97|1998-06-07|2-HIGH|Clerk#000000746|0| platelets are fluffily above the ironic theodolites. slyly pending 11555|1318|O|242411.22|1996-05-26|3-MEDIUM|Clerk#000000308|0|hinly bold deposits detect along the courts. express theodo 11556|707|O|237841.53|1996-03-03|5-LOW|Clerk#000000790|0|around the furiously ironic courts. blithely unusual deposits cajo 11557|1412|O|39415.83|1997-04-12|5-LOW|Clerk#000000522|0|ily. fluffily dogged asymptotes wake slyly regular grouches. a 11558|61|F|47438.62|1994-03-28|2-HIGH|Clerk#000000181|0| the slyly final excuses. 11559|931|O|54171.71|1995-09-26|1-URGENT|Clerk#000000582|0| pending excuses; carefully final dependencies 11584|643|F|350276.97|1994-11-23|3-MEDIUM|Clerk#000000150|0|l pinto beans haggle bold packages. fina 11585|895|F|59671.89|1994-03-16|3-MEDIUM|Clerk#000000105|0|g to the slyly unusual deposits. bold, final requests boost f 11586|1013|F|24377.52|1993-11-17|4-NOT SPECIFIED|Clerk#000000180|0|ously special packages solve carefully above th 11587|220|O|149730.78|1998-03-05|4-NOT SPECIFIED|Clerk#000000103|0| the slyly regular platelets. silently final packages may nag. foxes accordi 11588|1396|O|187668.87|1997-10-30|5-LOW|Clerk#000000492|0|ep. final, even excuses dazzle about the carefully special req 11589|1315|F|106954.20|1992-05-19|4-NOT SPECIFIED|Clerk#000000420|0|y express foxes shall have to cajole about the quickly unusu 11590|883|O|251411.55|1996-04-15|2-HIGH|Clerk#000000744|0|dolites sleep across the carefully regular depo 11591|269|F|108716.86|1993-01-21|2-HIGH|Clerk#000000691|0|pinto beans. bold deposits a 11616|403|O|261053.80|1996-08-02|2-HIGH|Clerk#000000003|0|cording to the express 11617|1459|F|186543.96|1994-12-11|4-NOT SPECIFIED|Clerk#000000519|0|etect blithely pending 11618|356|O|126867.40|1998-02-04|1-URGENT|Clerk#000000594|0|s kindle fluffily after 11619|55|O|42866.58|1998-07-12|2-HIGH|Clerk#000000245|0|ress instructions. ideas solve carefully special requests. patterns p 11620|353|O|115080.33|1997-07-22|4-NOT SPECIFIED|Clerk#000000961|0|ies nag slyly along 11621|964|O|233849.43|1995-11-11|5-LOW|Clerk#000000316|0|pinto beans. ruthless pinto beans haggle. slyly regular 11622|349|O|169440.18|1996-04-27|4-NOT SPECIFIED|Clerk#000000006|0| never instructions. express Tiresias boost. unusual, even pinto bean 11623|715|O|335923.49|1995-12-16|2-HIGH|Clerk#000000827|0|atelets cajole quietly final requests. slyly final courts are ironic, final 11648|949|P|223995.17|1995-04-10|5-LOW|Clerk#000000916|0| cajole carefully alon 11649|718|O|336620.13|1996-06-30|5-LOW|Clerk#000000666|0|kly even pinto beans cajole fu 11650|667|F|153010.80|1992-09-25|4-NOT SPECIFIED|Clerk#000000444|0|ccording to the ironic, regular excuses are 11651|689|F|66834.76|1995-02-23|3-MEDIUM|Clerk#000000982|0|sly furious sheaves. regular requests c 11652|1442|O|228052.10|1995-11-11|4-NOT SPECIFIED|Clerk#000000098|0|uriously bold foxes promise among the blithely ironic packages. fluffily regu 11653|314|O|191078.29|1997-07-05|2-HIGH|Clerk#000000533|0|kages doubt according to the final theodolites. furiously regular 11654|791|F|115033.02|1993-09-28|4-NOT SPECIFIED|Clerk#000000756|0|onic waters. carefully even packages haggle f 11655|910|O|184768.86|1998-06-27|1-URGENT|Clerk#000000614|0|slyly special asymptotes. final, unusual requests engage carefully pe 11680|1187|F|288910.60|1994-09-05|3-MEDIUM|Clerk#000000731|0|pecial asymptotes. final platelets cajole among the ironic pinto beans. pla 11681|857|F|246493.05|1992-10-17|1-URGENT|Clerk#000000485|0|final packages. carefully final accounts affi 11682|5|F|153588.74|1993-07-05|5-LOW|Clerk#000000852|0|s grow slyly. express, fin 11683|209|F|183381.18|1992-02-08|4-NOT SPECIFIED|Clerk#000000583|0|to beans cajole. furiously even frets detect. unusual pinto beans are 11684|323|F|282136.80|1992-07-28|2-HIGH|Clerk#000000986|0|instructions. final, even packages throughout the epi 11685|445|O|254981.92|1997-03-19|3-MEDIUM|Clerk#000000641|0|leep slyly unusual foxes. blithely even ideas through the t 11686|1039|F|169100.57|1994-01-28|4-NOT SPECIFIED|Clerk#000000169|0|ts wake furiously. accounts about the regular ideas are furiously 11687|1025|O|182611.71|1995-12-26|2-HIGH|Clerk#000000390|0|fluffily ideas. blithely pending packages lose blithely. slyly 11712|173|F|117391.91|1994-04-27|5-LOW|Clerk#000000376|0|special excuses. final accounts use. dependencies detect by the carefu 11713|1397|F|124407.44|1993-12-25|4-NOT SPECIFIED|Clerk#000000481|0|nic packages. unusual, unusual Ti 11714|1250|F|170165.48|1994-08-05|1-URGENT|Clerk#000000877|0|sts could have to wake ironic dependencies. furiously regul 11715|863|F|67076.38|1994-09-07|2-HIGH|Clerk#000000106|0|xpress foxes. pinto beans above the express deposits are final foxes. furio 11716|481|F|178602.59|1993-10-12|3-MEDIUM|Clerk#000000021|0|sly ironic packages haggle blithely around the unusual accounts. slyly re 11717|413|O|153316.94|1998-03-19|5-LOW|Clerk#000000185|0|furiously regular waters 11718|688|F|35335.31|1994-12-19|3-MEDIUM|Clerk#000000131|0|ly regular asymptotes. decoys integrate slyly among the bo 11719|1147|F|240503.39|1995-02-06|4-NOT SPECIFIED|Clerk#000000307|0|hely slyly special instr 11744|127|O|183922.62|1996-07-05|1-URGENT|Clerk#000000197|0|pecial packages above the furio 11745|25|F|147334.29|1992-07-20|2-HIGH|Clerk#000000549|0|lar ideas. slyly expre 11746|5|O|177360.54|1998-04-29|3-MEDIUM|Clerk#000000038|0|heodolites. final asymptotes was above the furiously final pin 11747|460|F|46484.18|1993-02-26|3-MEDIUM|Clerk#000000537|0|. requests use across 11748|733|F|43216.31|1994-06-04|4-NOT SPECIFIED|Clerk#000000147|0|uffily regular accounts cajole blithely alongside of the furiously pending p 11749|818|O|75180.45|1998-02-17|5-LOW|Clerk#000000819|0| excuses boost against the final pai 11750|823|O|117385.37|1997-03-05|2-HIGH|Clerk#000000801|0| furiously packages. carefully silent instruc 11751|1282|O|150450.18|1996-05-23|1-URGENT|Clerk#000000605|0|uickly express pinto beans. blithely bold foxes use above the dep 11776|386|F|90162.21|1992-02-25|5-LOW|Clerk#000000548|0|ck theodolites integrate furiously along the bold deposits. even, pending 11777|271|F|56387.90|1994-12-26|5-LOW|Clerk#000000292|0|g dependencies; quickly dogged courts wake quick 11778|217|O|102865.25|1997-03-28|5-LOW|Clerk#000000253|0|ss the final, unusual packages. quickly quick theodolites haggle. slyl 11779|1045|F|183456.75|1993-06-30|4-NOT SPECIFIED|Clerk#000000740|0|according to the care 11780|28|F|129283.92|1993-09-22|1-URGENT|Clerk#000000793|0|ove the regular theodolites are furiously regular notornis. 11781|1484|O|147766.37|1996-04-16|5-LOW|Clerk#000000619|0|s. final, final deposi 11782|427|F|163855.06|1992-07-29|5-LOW|Clerk#000000609|0|bold, unusual requests are slyly against the quickly bold dependencies. s 11783|1498|O|22953.13|1998-04-03|1-URGENT|Clerk#000000028|0|. regular theodolites snooze furiou 11808|874|F|60831.20|1992-08-01|4-NOT SPECIFIED|Clerk#000000710|0|could detect quickly unusu 11809|556|O|161697.76|1996-04-25|2-HIGH|Clerk#000000617|0|old realms alongside of the special dugouts use along the fluffily iro 11810|647|F|77198.94|1992-04-23|1-URGENT|Clerk#000000804|0| final requests boost among the furiously bold accounts. final 11811|527|F|7385.35|1992-10-28|2-HIGH|Clerk#000000940|0| packages haggle furiously pend 11812|940|F|17214.00|1993-06-03|3-MEDIUM|Clerk#000000844|0|y regular accounts. regular, pending requests print careful 11813|376|F|211762.16|1994-02-10|1-URGENT|Clerk#000000813|0|ly regular instructions. quickly ironi 11814|121|F|198422.64|1993-01-26|2-HIGH|Clerk#000000404|0|final ideas sleep sometimes deposits. final, final foxes boost fu 11815|1396|O|199250.12|1995-09-13|5-LOW|Clerk#000000342|0|counts use according to the bli 11840|596|F|283439.67|1994-08-15|1-URGENT|Clerk#000000466|0|ggle slowly across the sly 11841|727|F|158024.06|1993-04-21|3-MEDIUM|Clerk#000000521|0|s sleep carefully unusual ac 11842|1414|O|90262.04|1996-04-08|4-NOT SPECIFIED|Clerk#000000566|0|furiously unusual platelets. express requests 11843|730|F|211984.66|1994-07-18|3-MEDIUM|Clerk#000000747|0|g requests snooze care 11844|121|O|86777.18|1997-01-18|2-HIGH|Clerk#000000609|0|ular deposits haggle 11845|887|O|125342.31|1997-03-29|1-URGENT|Clerk#000000902|0|ts. final accounts detect furiously bold foxes. carefully regular packages h 11846|1046|F|51318.21|1993-04-21|5-LOW|Clerk#000000688|0|uriously ironic packages cajole furiously express, even ideas. ironic ideas 11847|1007|O|94551.77|1997-10-02|2-HIGH|Clerk#000000860|0|ely pending deposits haggle furiously across the pending 11872|145|F|75770.24|1994-07-25|5-LOW|Clerk#000000620|0| courts across the furiously 11873|863|O|21009.93|1996-04-06|4-NOT SPECIFIED|Clerk#000000322|0|press, final deposits da 11874|481|F|221467.60|1992-09-10|3-MEDIUM|Clerk#000000163|0|r the slyly bold deposits wake about the c 11875|698|F|98592.08|1992-08-21|3-MEDIUM|Clerk#000000300|0|. quickly special asymptotes after the slowly final pa 11876|611|O|130732.20|1995-07-04|2-HIGH|Clerk#000000325|0| to the quickly even pains: always 11877|268|F|122384.64|1993-05-24|4-NOT SPECIFIED|Clerk#000000312|0|ly above the unusua 11878|1349|F|92047.44|1993-02-25|4-NOT SPECIFIED|Clerk#000000041|0|ole blithely fluffy, unusua 11879|1057|F|185103.30|1993-06-20|2-HIGH|Clerk#000000371|0|to the special excuses detect carefully carefully express accounts. carefully 11904|415|O|279468.40|1997-12-02|4-NOT SPECIFIED|Clerk#000000924|0|accounts. platelets k 11905|343|O|101799.49|1997-07-04|5-LOW|Clerk#000000543|0|thely regular foxes. fluffily regular pinto beans i 11906|524|O|325004.58|1996-12-15|5-LOW|Clerk#000000929|0|deas cajole quickly? blithely final dolphins boost. blithely final deposits wa 11907|1327|F|127812.95|1993-08-24|5-LOW|Clerk#000000062|0|osits above the quickly ironic instructions are carefully after t 11908|1063|F|180189.92|1993-05-26|4-NOT SPECIFIED|Clerk#000000456|0|atelets across the express deposits kindle evenly unu 11909|868|F|51603.50|1994-11-25|4-NOT SPECIFIED|Clerk#000000012|0|lithely unusual accounts aga 11910|644|O|102331.93|1995-06-16|2-HIGH|Clerk#000000664|0|uriously final ideas cajole furi 11911|355|F|219307.51|1993-11-11|1-URGENT|Clerk#000000563|0| requests try to wake according to the carefully regular deposit 11936|749|O|167898.60|1995-09-16|1-URGENT|Clerk#000000869|0|ns wake blithely even, express theodolites; final 11937|1424|O|159471.41|1998-06-27|1-URGENT|Clerk#000000941|0|ic deposits sleep carefully special, regular foxes. special theo 11938|67|O|161199.79|1998-07-25|1-URGENT|Clerk#000000911|0|iously above the reg 11939|1333|O|84765.05|1996-10-01|2-HIGH|Clerk#000000610|0|iously regular dinos cajole furiously foxes. blithely regular foxes acr 11940|683|O|149688.34|1998-05-30|2-HIGH|Clerk#000000152|0|ular ideas cajole quickly 11941|659|F|157391.90|1993-06-25|1-URGENT|Clerk#000000392|0| instructions. carefully final theodolites wake quickly. carefully iro 11942|1324|F|43794.61|1994-05-09|1-URGENT|Clerk#000000317|0|ely bold accounts are carefully alongside of the unusual packages. slyly eve 11943|244|F|235683.21|1993-04-08|5-LOW|Clerk#000000724|0|requests haggle quickly about the carefully 11968|311|F|43428.32|1995-03-17|5-LOW|Clerk#000000526|0|g frets was above the fur 11969|449|F|270998.74|1992-01-20|4-NOT SPECIFIED|Clerk#000000641|0|efully. blithely pending deposits haggle regular, regular 11970|937|O|255762.04|1998-05-24|1-URGENT|Clerk#000000885|0|cial pinto beans. blithely expres 11971|1351|O|119482.38|1997-03-21|2-HIGH|Clerk#000000026|0|slyly regular requests lose quickly: depen 11972|1396|F|147645.78|1994-01-04|4-NOT SPECIFIED|Clerk#000000751|0|riously silent gifts affix slyl 11973|631|F|158758.99|1994-03-04|4-NOT SPECIFIED|Clerk#000000545|0| regular theodolites use carefully pen 11974|1366|F|88216.78|1992-10-09|4-NOT SPECIFIED|Clerk#000000595|0|slyly unusual accounts according to the blithely ironic ideas boost furio 11975|322|O|230472.15|1995-04-14|1-URGENT|Clerk#000000551|0|on the bold ideas cajole across the f 12000|1150|F|89148.77|1994-05-13|4-NOT SPECIFIED|Clerk#000000683|0|s against the furiously ironic grouches sleep according to the e 12001|739|F|138635.75|1994-07-07|2-HIGH|Clerk#000000863|0|old, even theodolites. regular, special theodolites use furio 12002|826|F|79579.51|1993-11-30|3-MEDIUM|Clerk#000000431|0| regular packages wake qui 12003|1205|O|16311.25|1998-05-24|1-URGENT|Clerk#000000708|0|s along the quickly regular instructions haggle carefully furiously u 12004|656|F|74814.13|1994-12-05|3-MEDIUM|Clerk#000000633|0|nis against the slyly specia 12005|1102|F|250917.29|1992-06-25|4-NOT SPECIFIED|Clerk#000000904|0| after the ironic, unusua 12006|944|O|129075.40|1997-06-02|4-NOT SPECIFIED|Clerk#000000501|0|n packages. carefully ironic accounts are after the pending platele 12007|613|F|239431.45|1994-09-28|4-NOT SPECIFIED|Clerk#000000363|0|even requests wake carefully unusual packages. quickly 12032|902|O|102348.27|1997-08-10|4-NOT SPECIFIED|Clerk#000000349|0| after the even, regular instructions. blithe tithes use furiously. b 12033|334|F|214360.53|1992-09-12|2-HIGH|Clerk#000000274|0| bold pearls haggle. carefully ironic pinto beans cajole. blit 12034|121|O|88674.23|1996-12-08|2-HIGH|Clerk#000000363|0|ges. deposits sleep slyly. ideas sleep 12035|1010|O|245999.17|1996-11-09|5-LOW|Clerk#000000963|0|furiously special instructions. pending deposits nod. blithely unusual pint 12036|838|O|241236.07|1998-01-16|5-LOW|Clerk#000000041|0|ic requests. unusual pinto beans sleep fluffily about the furiously regular 12037|1462|O|139739.68|1995-07-01|5-LOW|Clerk#000000375|0|pending foxes shall cajol 12038|164|O|59423.94|1995-11-10|2-HIGH|Clerk#000000587|0|ironic asymptotes mainta 12039|1252|F|303373.40|1993-06-17|2-HIGH|Clerk#000000765|0|packages integrate c 12064|38|F|133067.27|1992-09-16|5-LOW|Clerk#000000994|0| blithely quickly pending t 12065|557|O|60068.73|1997-01-30|4-NOT SPECIFIED|Clerk#000000064|0|the final deposits boost pending deposits. pending 12066|1066|F|187325.65|1995-02-07|3-MEDIUM|Clerk#000000084|0|heodolites cajole a 12067|257|F|227467.91|1993-02-05|1-URGENT|Clerk#000000019|0|requests. quickly regular packages run f 12068|1231|O|20347.58|1996-05-08|2-HIGH|Clerk#000000402|0|ctions. furiously even accounts 12069|1381|O|235666.55|1995-11-08|2-HIGH|Clerk#000000460|0|ong the pinto beans. deposits among the excuses cajole 12070|746|O|256785.35|1998-05-23|5-LOW|Clerk#000000287|0|bold pinto beans hagg 12071|956|O|100260.23|1998-06-26|1-URGENT|Clerk#000000205|0|ly bold multipliers cajole quickly re 12096|1004|F|257237.41|1992-09-20|3-MEDIUM|Clerk#000000156|0|hely final requests kindle among the regular foxes. orb 12097|118|F|156644.02|1993-11-10|1-URGENT|Clerk#000000239|0|out the slyly regular theodolites. regularly reg 12098|1325|F|34956.25|1993-05-01|2-HIGH|Clerk#000000214|0| carefully ironic, express deposits. ideas are slyly a 12099|112|F|230209.47|1994-09-12|1-URGENT|Clerk#000000042|0|furiously regular accounts haggle quic 12100|1298|O|62857.66|1996-03-15|2-HIGH|Clerk#000000518|0|ress frays use blithely pending requests-- quickly regular somas acros 12101|754|F|103072.24|1995-01-07|1-URGENT|Clerk#000000853|0|fily regular packages w 12102|1264|O|208734.50|1995-11-18|3-MEDIUM|Clerk#000000981|0|eposits are blithely along 12103|229|O|169239.68|1996-11-25|3-MEDIUM|Clerk#000000605|0|ironic foxes. quickly brave pinto beans 12128|1192|O|163301.92|1997-06-27|4-NOT SPECIFIED|Clerk#000000102|0|yly across the furiously ironic accounts. carefully special real 12129|418|F|117561.31|1992-12-26|3-MEDIUM|Clerk#000000119|0|dencies cajole furiously about the 12130|1261|O|157170.80|1995-12-10|4-NOT SPECIFIED|Clerk#000000894|0|according to the even, regular packages. furiously unusual pinto beans wake 12131|1172|O|285746.20|1998-06-29|3-MEDIUM|Clerk#000000074|0|ironic ideas. blithely unus 12132|1277|O|271963.86|1996-09-06|1-URGENT|Clerk#000000831|0|ptotes boost permanently. carefully unusual instr 12133|361|F|52225.60|1992-04-15|1-URGENT|Clerk#000000634|0|st after the furiously special 12134|355|O|56547.11|1996-05-31|3-MEDIUM|Clerk#000000868|0|blithely blithely regular theodolites. slyly even packages nag slyly slyly e 12135|1420|O|268120.03|1995-10-10|5-LOW|Clerk#000000246|0|ual excuses alongside of th 12160|1423|F|67966.80|1993-12-13|5-LOW|Clerk#000000707|0|ully about the furiously ironic braids-- carefully en 12161|1307|O|215072.11|1997-03-15|4-NOT SPECIFIED|Clerk#000000382|0|le blithely across the blithely slow fox 12162|1222|O|47989.76|1997-03-18|3-MEDIUM|Clerk#000000066|0| players according to the fluffily ironic dolphins cajole slyly spe 12163|88|O|205826.29|1997-07-20|5-LOW|Clerk#000000951|0|se furiously carefully special pinto beans. blithely special 12164|1313|F|180895.16|1993-10-17|1-URGENT|Clerk#000000148|0|ests. instructions haggle blithely express dependencies. furiously bold war 12165|1489|F|153853.42|1994-04-11|5-LOW|Clerk#000000091|0|nic pinto beans boost carefully 12166|62|F|18006.96|1995-03-15|4-NOT SPECIFIED|Clerk#000000469|0|of the slyly even pinto bean 12167|91|O|60173.82|1998-05-31|1-URGENT|Clerk#000000618|0|counts. bravely special pac 12192|277|F|27209.81|1994-10-19|1-URGENT|Clerk#000000387|0| bold ideas boost slyly slyly final frays. carefully unusual pinto b 12193|802|O|159382.08|1996-07-28|3-MEDIUM|Clerk#000000559|0|y. blithely final pinto beans according to the theodolites haggle care 12194|271|F|59806.63|1995-03-02|1-URGENT|Clerk#000000922|0|atelets haggle among the regular accounts. furiously even plate 12195|73|O|108946.27|1997-04-27|4-NOT SPECIFIED|Clerk#000000947|0|iously silent foxes eat carefull 12196|1261|F|83958.83|1993-06-03|4-NOT SPECIFIED|Clerk#000000284|0|e furiously ironic ideas affix fluffily above the quickly 12197|577|O|206318.73|1997-06-27|2-HIGH|Clerk#000000976|0|ular packages affix upon the slyly 12198|1300|F|59977.08|1993-02-28|3-MEDIUM|Clerk#000000917|0| ironic deposits. furiously express escapades detect 12199|520|O|103350.42|1995-12-15|5-LOW|Clerk#000000478|0|he ironic accounts. ideas detect. slyly final packa 12224|646|F|212674.36|1994-12-13|2-HIGH|Clerk#000000672|0|. slyly final accounts boost. blithely express deposits haggle sl 12225|656|O|188067.32|1997-01-29|2-HIGH|Clerk#000000903|0|ons. blithely bold requests are carefu 12226|1348|O|120809.59|1998-04-01|5-LOW|Clerk#000000415|0| across the final instructi 12227|1282|O|210133.42|1998-06-27|3-MEDIUM|Clerk#000000980|0| packages. quickly ironic accounts affix quickly. ironic, even packages 12228|226|F|276130.19|1994-06-26|4-NOT SPECIFIED|Clerk#000000474|0|fully regular excuses? ironic foxes are across the dogge 12229|1250|O|106159.32|1996-07-24|1-URGENT|Clerk#000000293|0|ptotes boost slyly even accounts. 12230|712|O|161632.17|1998-04-05|2-HIGH|Clerk#000000383|0|s. regular foxes breach regularly pending theodo 12231|1021|O|243683.82|1997-07-30|4-NOT SPECIFIED|Clerk#000000012|0|sual sauternes use across the bold theodolites. req 12256|532|F|146646.67|1992-12-19|3-MEDIUM|Clerk#000000238|0|iously alongside of the 12257|149|O|128270.26|1996-04-17|2-HIGH|Clerk#000000278|0|y above the unusual foxes. slyly even accounts haggl 12258|1408|F|300041.05|1994-12-01|5-LOW|Clerk#000000878|0|ing foxes boost. stealthy 12259|250|F|173853.14|1993-02-15|2-HIGH|Clerk#000000282|0| boost about the carefully ironic ideas. fluffily iron 12260|496|F|172336.18|1992-11-26|3-MEDIUM|Clerk#000000484|0|s along the slyly ruthless pinto beans haggle about the 12261|1126|F|362237.85|1993-10-24|5-LOW|Clerk#000000279|0|ccounts use quickly about the furiously bold foxes. furiously final 12262|1441|O|139469.07|1995-12-11|5-LOW|Clerk#000000886|0|efully. slyly final theodolit 12263|550|O|161275.31|1995-07-07|2-HIGH|Clerk#000000566|0|ts sleep across the carefully bold instruc 12288|1153|O|125448.87|1996-11-06|1-URGENT|Clerk#000000183|0|slyly regular deposits above the foxes are at the packages- 12289|421|O|128835.39|1995-12-25|5-LOW|Clerk#000000269|0| even deposits. ironic, regular deposits haggle blith 12290|1304|O|103022.30|1995-07-10|3-MEDIUM|Clerk#000000947|0|ully pending instructions boost slyly furiously 12291|740|O|19328.06|1998-03-23|1-URGENT|Clerk#000000074|0|al deposits. warhorses inte 12292|739|F|176370.50|1992-06-03|4-NOT SPECIFIED|Clerk#000000375|0|sometimes final foxes after the qui 12293|329|O|102326.14|1995-06-19|2-HIGH|Clerk#000000103|0| quietly ironic instructions sleep carefully furiously iro 12294|784|O|182534.50|1995-07-06|5-LOW|Clerk#000000583|0|ns run about the qui 12295|883|F|84218.70|1993-11-10|5-LOW|Clerk#000000955|0|ost around the slowly iron 12320|298|O|118435.28|1995-12-21|1-URGENT|Clerk#000000356|0|o beans wake carefully theodolites. final gifts haggle quickly 12321|79|F|12511.06|1994-04-04|3-MEDIUM|Clerk#000000705|0|uriously ironic deposits cajole furiously doggedly ironic depend 12322|856|P|42466.03|1995-05-12|1-URGENT|Clerk#000000760|0| the special instructions detect fluffily ac 12323|926|P|139824.58|1995-04-03|4-NOT SPECIFIED|Clerk#000000019|0|ts detect above the even dep 12324|1033|O|202248.40|1998-08-02|3-MEDIUM|Clerk#000000746|0|ending theodolites try to thrash. regular deposits about the fluffily pending 12325|973|F|70150.43|1995-01-05|1-URGENT|Clerk#000000440|0|packages are final p 12326|506|F|181528.08|1995-02-03|1-URGENT|Clerk#000000765|0|ccounts are fluffily carefu 12327|166|O|39113.28|1998-04-23|4-NOT SPECIFIED|Clerk#000000969|0|tions haggle carefully slyly 12352|1466|F|331745.34|1992-03-24|3-MEDIUM|Clerk#000000997|0|eas. regular instructions need to boost slyly re 12353|595|O|219706.88|1996-04-13|4-NOT SPECIFIED|Clerk#000000312|0|ependencies. special dependencies nag quickly even packages. slyly final de 12354|1243|O|54975.46|1997-01-21|5-LOW|Clerk#000000859|0|g to the carefully pending excuses cajole 12355|178|P|286003.34|1995-05-29|4-NOT SPECIFIED|Clerk#000000081|0|arhorses. close, final foxes are slyly 12356|656|F|121696.19|1992-10-19|1-URGENT|Clerk#000000540|0|ges haggle quickly! blithely careful courts impress alongside of the bold fox 12357|154|O|90986.72|1995-08-23|2-HIGH|Clerk#000000169|0|instructions poach furiously final requests. quickly express depo 12358|880|O|231011.45|1996-10-10|2-HIGH|Clerk#000000335|0| blithely pending ideas. quickly ironic pinto b 12359|1400|O|215672.79|1997-05-29|4-NOT SPECIFIED|Clerk#000000382|0| express packages grow above the blithely even foxes: 12384|1168|O|213609.26|1998-08-02|3-MEDIUM|Clerk#000000311|0|gle. furiously unus 12385|821|F|92862.99|1992-11-24|5-LOW|Clerk#000000324|0|sts use furiously. ironi 12386|424|F|314497.24|1992-03-21|3-MEDIUM|Clerk#000000709|0|e fluffily ironic dolphins. quick 12387|638|O|52638.42|1997-07-03|2-HIGH|Clerk#000000051|0|o the quickly regular requests use slyly unusual theodolites. 12388|952|O|331123.13|1997-10-16|4-NOT SPECIFIED|Clerk#000000611|0|boost furiously furiously final accounts. slyly regular deposits are am 12389|653|F|242568.31|1994-08-05|3-MEDIUM|Clerk#000000139|0|s boost according to the slyly even dolphins. final depend 12390|1285|O|49792.77|1996-06-24|3-MEDIUM|Clerk#000000421|0|posits doze blithely. fluffily ironic d 12391|1000|O|94747.78|1998-07-01|5-LOW|Clerk#000000433|0|cies are blithely. furious 12416|1150|F|9933.08|1993-06-26|1-URGENT|Clerk#000000400|0|ow ideas cajole furiou 12417|1081|F|230795.43|1993-10-25|1-URGENT|Clerk#000000152|0|requests. furiously express instructions haggle. e 12418|1300|F|124402.47|1992-02-19|4-NOT SPECIFIED|Clerk#000000547|0|tructions. foxes nag furiously abo 12419|982|O|268861.58|1995-10-16|1-URGENT|Clerk#000000104|0|y. furiously final sauternes sleep slyly above the pending 12420|1489|F|179274.43|1993-12-27|3-MEDIUM|Clerk#000000798|0|ackages cajole permanently. blithely quick packages alongs 12421|673|O|212213.47|1995-11-02|1-URGENT|Clerk#000000203|0|pending theodolites wake: ironic, express platelets cajole furiously. 12422|575|P|217235.25|1995-05-22|3-MEDIUM|Clerk#000000349|0|ssly pending requests cajole quietly final i 12423|986|F|90571.46|1992-07-18|1-URGENT|Clerk#000000742|0|nic courts haggle carefully finally regular requests. depo 12448|511|O|158476.25|1995-11-09|1-URGENT|Clerk#000000711|0|he quickly pending accounts. ironic 12449|461|F|133597.11|1993-02-01|3-MEDIUM|Clerk#000000312|0|s hinder carefully across the iro 12450|1321|F|41643.96|1994-03-13|4-NOT SPECIFIED|Clerk#000000550|0|kages use quickly final accounts. carefully regular packages integrate blithe 12451|1150|F|298853.62|1993-04-20|5-LOW|Clerk#000000319|0|uriously even ideas hagg 12452|364|F|173339.75|1993-12-08|3-MEDIUM|Clerk#000000867|0|indle furiously near the quickly regular accounts: silent, 12453|1213|F|180966.45|1994-01-14|5-LOW|Clerk#000000085|0|ic ideas promise. slyly ironic pinto beans above the dinos wake quickly bold 12454|217|O|42464.91|1997-03-21|1-URGENT|Clerk#000000910|0|ic, regular deposits are against the carefully unusual accounts. e 12455|292|F|216570.72|1992-09-04|4-NOT SPECIFIED|Clerk#000000594|0|nusual asymptotes. regular ideas haggle blithely across the ironic ideas. 12480|320|F|122198.33|1994-07-08|3-MEDIUM|Clerk#000000055|0|ructions wake fluffily fluffily final gifts! furiou 12481|271|O|85849.06|1995-07-11|1-URGENT|Clerk#000000785|0|uests sleep furiously bold deposits. blithely express acc 12482|998|O|153413.61|1996-01-08|5-LOW|Clerk#000000311|0|y express dependencies along the f 12483|124|F|153915.13|1993-08-05|1-URGENT|Clerk#000000652|0|cial ideas sleep furiously against the final, regular re 12484|1027|O|184195.36|1995-08-22|3-MEDIUM|Clerk#000000268|0|olphins. blithely ironic platelets s 12485|188|F|209369.25|1993-08-08|1-URGENT|Clerk#000000994|0|tions along the ideas 12486|1286|P|184001.37|1995-02-21|2-HIGH|Clerk#000000206|0| deposits; quickly ironic packages use stealthily about the qui 12487|401|F|41222.94|1994-11-30|2-HIGH|Clerk#000000575|0|xes. requests sleep carefully 12512|1402|O|106021.12|1996-04-02|2-HIGH|Clerk#000000813|0|nal foxes are fluffily. foxes snooze about the ironi 12513|419|O|62011.18|1997-09-03|5-LOW|Clerk#000000422|0| wake final, special requests. express instructions cajole after the furi 12514|1495|F|38477.49|1994-04-25|2-HIGH|Clerk#000000120|0|c courts doubt express, bo 12515|1459|F|128077.61|1993-05-14|3-MEDIUM|Clerk#000000624|0|kly special accounts. blithely regular packages boost slyly packages. pl 12516|1373|F|112837.28|1994-05-25|5-LOW|Clerk#000000087|0|y after the quickly unusual gifts. fluffily exp 12517|1303|O|84279.32|1997-02-01|5-LOW|Clerk#000000162|0|packages. express packages impress furiously even, bold theodolite 12518|1082|F|153987.45|1993-07-23|5-LOW|Clerk#000000194|0|ultipliers serve furiously a 12519|913|F|187489.35|1994-01-01|2-HIGH|Clerk#000000387|0|ependencies. carefully unusual deposits use finally ironic deposits. e 12544|440|F|62982.76|1994-03-04|5-LOW|Clerk#000000600|0|thely above the even 12545|175|O|74242.81|1996-07-01|1-URGENT|Clerk#000000472|0|ly. carefully silent excuses use carefully around the reg 12546|217|O|89344.66|1996-02-08|4-NOT SPECIFIED|Clerk#000000547|0|ounts mold. blithely stealthy depths haggle blithely blithe 12547|1141|O|205195.87|1998-04-14|2-HIGH|Clerk#000000729|0|odolites? blithely ir 12548|145|O|209786.08|1997-11-20|4-NOT SPECIFIED|Clerk#000000517|0|s excuses x-ray against 12549|28|O|100471.72|1997-06-03|1-URGENT|Clerk#000000845|0|he furiously regular pint 12550|878|F|39446.33|1993-03-09|4-NOT SPECIFIED|Clerk#000000282|0|as. blithely ironic pinto beans wake. slyly bold deposits detect blithely alon 12551|790|F|176659.70|1992-06-12|5-LOW|Clerk#000000090|0| slyly alongside of the special waters. asymptotes outside the pend 12576|1057|F|33914.11|1993-01-26|4-NOT SPECIFIED|Clerk#000000762|0|n decoys behind the carefully final requests 12577|616|O|134783.72|1998-05-18|4-NOT SPECIFIED|Clerk#000000586|0|special instruction 12578|1207|O|255196.69|1996-11-15|2-HIGH|Clerk#000000150|0|p slyly furious foxes. slyly pending a 12579|193|O|47191.62|1995-06-17|4-NOT SPECIFIED|Clerk#000000536|0|y silent pinto beans-- ironic deposits affix. furiously fina 12580|589|F|41858.68|1993-01-14|5-LOW|Clerk#000000373|0|y bold theodolites above the carefully regular 12581|823|F|106379.64|1994-10-21|2-HIGH|Clerk#000000582|0| final requests after the final patterns wake blithel 12582|538|F|22603.80|1993-01-22|4-NOT SPECIFIED|Clerk#000000671|0| ironic packages against the requests slee 12583|943|O|42860.56|1996-01-18|5-LOW|Clerk#000000676|0|yly fluffily even accounts. quickly careful asymptotes boost. ironic, bo 12608|301|F|61932.81|1993-03-08|1-URGENT|Clerk#000000087|0| have to nag quietly among the carefully bold pinto beans. expres 12609|805|O|128069.23|1996-01-17|3-MEDIUM|Clerk#000000207|0|l requests haggle furiously 12610|85|F|10061.76|1993-07-17|5-LOW|Clerk#000000952|0| bold theodolites engage blithely against th 12611|41|O|185736.53|1995-10-16|3-MEDIUM|Clerk#000000579|0|ckly carefully regular deposits 12612|1444|F|185433.93|1993-12-22|5-LOW|Clerk#000000381|0| accounts. furiously ironic requests along the final, ruthless accounts cajol 12613|328|F|116318.24|1993-04-11|3-MEDIUM|Clerk#000000550|0|lithely express ins 12614|670|O|273057.78|1996-02-28|3-MEDIUM|Clerk#000000387|0|wake. quickly stealthy foxes affix slyly ironic requests. 12615|118|F|20866.10|1995-03-06|4-NOT SPECIFIED|Clerk#000000993|0|old packages. accounts use slyly after the foxes. 12640|413|O|76597.62|1998-07-20|2-HIGH|Clerk#000000081|0|ites. slyly express deposits integrate fluffily. ironic courts abo 12641|1097|F|239626.14|1995-02-23|5-LOW|Clerk#000000617|0|its wake according to the slyly unusual excuses. even patterns are carefully 12642|1243|F|247694.31|1994-04-28|1-URGENT|Clerk#000000015|0|symptotes thrash blithely above the furiously regular accounts. q 12643|322|F|124302.25|1993-03-20|5-LOW|Clerk#000000486|0| slyly silent requests. qui 12644|1159|F|150689.23|1992-02-01|2-HIGH|Clerk#000000394|0| the final instructions. blithely ironic asymptotes boost carefully regular 12645|250|F|231538.56|1994-07-07|5-LOW|Clerk#000000705|0|de of the blithely regular accounts use slyly final accounts. fluff 12646|1334|F|116946.28|1993-02-18|1-URGENT|Clerk#000000155|0|regular ideas. slyly ironic re 12647|1198|O|38285.07|1997-02-27|4-NOT SPECIFIED|Clerk#000000190|0|. regular theodolites sleep after the care 12672|257|F|37187.79|1994-07-07|4-NOT SPECIFIED|Clerk#000000963|0|es are carefully blithely unusual dugouts. regular, i 12673|62|F|21911.85|1994-06-12|1-URGENT|Clerk#000000435|0|ffily. express accounts sleep slyly regular deposit 12674|586|F|96952.15|1992-07-05|5-LOW|Clerk#000000811|0|xcuses. express deposits wake quickl 12675|679|F|155533.85|1994-01-28|3-MEDIUM|Clerk#000000921|0|r theodolites. furiously pending gifts alongside 12676|839|F|5884.55|1994-01-16|2-HIGH|Clerk#000000960|0|ake slyly special deposits. slyly un 12677|190|O|187187.63|1996-05-25|5-LOW|Clerk#000000500|0|ely above the furiously silent accounts; carefully regular depend 12678|844|O|77831.02|1998-01-26|5-LOW|Clerk#000000395|0|around the blithely slow packages nag quickly express packages. carefully 12679|1117|O|75859.23|1997-06-22|2-HIGH|Clerk#000000326|0|s. furiously pending deposits sleep furiously a 12704|439|F|143027.03|1993-02-10|5-LOW|Clerk#000000553|0|ackages sleep. pinto beans haggle furiously. quickly final dolphins use blit 12705|295|F|74933.03|1994-09-16|5-LOW|Clerk#000000983|0|grow above the fluffily final packages. slyly pending ideas nag furiou 12706|458|F|157344.88|1994-11-21|1-URGENT|Clerk#000000997|0|ing instructions. deposits cajole. slyly sp 12707|1219|F|253413.30|1993-05-30|2-HIGH|Clerk#000000820|0|ffix furiously according to the final, ironic dolphin 12708|172|F|262921.58|1993-01-08|5-LOW|Clerk#000000882|0|ly ironic deposits. quickly final sentiments cajole car 12709|268|O|74462.32|1996-05-11|5-LOW|Clerk#000000991|0|hely alongside of the a 12710|1169|F|305191.71|1993-08-18|1-URGENT|Clerk#000000329|0|es wake furiously ironic accounts. fluffily ironic pinto bea 12711|85|F|80711.39|1992-02-20|1-URGENT|Clerk#000000326|0|usual instructions. pending, final deposits use. 12736|751|F|346186.75|1993-03-09|5-LOW|Clerk#000000385|0|ithely express deposits. pinto beans n 12737|1187|F|320360.43|1994-05-04|1-URGENT|Clerk#000000047|0|ly regular dinos affix slyly. ironic, quick packages boost carefully. specia 12738|1334|O|182506.16|1998-06-12|2-HIGH|Clerk#000000613|0|sts boost: carefully even decoys integrate against the furiously reg 12739|772|F|56336.51|1992-03-02|4-NOT SPECIFIED|Clerk#000000078|0|side the bold, express depend 12740|439|O|61728.22|1997-06-05|2-HIGH|Clerk#000000643|0| the special deposits ca 12741|1348|F|162616.34|1992-07-20|4-NOT SPECIFIED|Clerk#000000800|0| of the instructions. furiou 12742|748|O|36355.83|1997-06-07|5-LOW|Clerk#000000591|0|always final sauternes. carefully regula 12743|695|F|101594.83|1993-06-24|2-HIGH|Clerk#000000015|0|inal foxes cajole quickly. bli 12768|1202|F|121309.88|1994-04-09|1-URGENT|Clerk#000000642|0|sly even accounts haggle slyly. blithely pending epitaphs boost. carefully f 12769|1057|F|29166.32|1992-01-08|4-NOT SPECIFIED|Clerk#000000590|0|blithely across the furiously express foxes; slyly silent pa 12770|1415|F|129333.56|1993-04-18|4-NOT SPECIFIED|Clerk#000000243|0|bove the special, regular ideas. final pinto beans grow? iro 12771|451|F|203114.43|1994-01-18|1-URGENT|Clerk#000000548|0|counts. furiously final dependencies sleep. fluffi 12772|1451|P|221390.02|1995-05-12|4-NOT SPECIFIED|Clerk#000000248|0|ual requests wake quickly even pinto 12773|817|F|304639.93|1995-01-12|5-LOW|Clerk#000000670|0|s, pending ideas. fluffily ironic deposits cajole blithely furiously unusual 12774|1108|F|60657.84|1992-10-10|1-URGENT|Clerk#000000067|0|ounts. quickly final requests 12775|979|O|127678.72|1996-10-15|5-LOW|Clerk#000000553|0|s the quickly unusual 12800|11|F|302431.56|1993-05-23|5-LOW|Clerk#000000898|0|. slyly final accounts cajole i 12801|1412|O|226079.43|1997-08-28|3-MEDIUM|Clerk#000000582|0|bove the fluffily express asymptot 12802|1481|O|233609.74|1998-03-02|1-URGENT|Clerk#000000774|0|ounts. special, pending ideas cajole pe 12803|392|F|27391.77|1993-02-26|4-NOT SPECIFIED|Clerk#000000876|0|sly final accounts use slyly final accounts. blithe 12804|734|F|358058.27|1992-05-26|3-MEDIUM|Clerk#000000544|0|l packages boost quickly ironic, 12805|478|O|76234.32|1996-12-11|3-MEDIUM|Clerk#000000035|0| to the quickly special platelets? 12806|526|O|208746.01|1995-12-25|4-NOT SPECIFIED|Clerk#000000181|0|s about the slyly regular pinto beans nag 12807|391|O|34178.61|1998-01-16|5-LOW|Clerk#000000393|0|uffily final accounts wake fur 12832|362|F|66392.28|1994-12-11|3-MEDIUM|Clerk#000000566|0|unts are after the regular packages. furiously pending 12833|1285|O|102301.35|1998-06-14|5-LOW|Clerk#000000773|0|nts. even, pending foxes are 12834|757|O|263553.51|1996-11-03|4-NOT SPECIFIED|Clerk#000000370|0|ns. unusual ideas are. somas b 12835|370|O|142811.34|1997-01-20|4-NOT SPECIFIED|Clerk#000000894|0|ges. slyly final id 12836|1057|O|222420.89|1996-04-18|3-MEDIUM|Clerk#000000728|0|es sleep furiously against the quickly even inst 12837|590|F|112787.85|1992-12-03|3-MEDIUM|Clerk#000000431|0|ter the slowly unusual foxes. blithely express pinto beans haggle. furiou 12838|112|O|104930.86|1998-07-01|1-URGENT|Clerk#000000424|0|fully ironic foxes. quickly pending accounts about the fin 12839|1424|O|178226.93|1998-01-08|3-MEDIUM|Clerk#000001000|0|ng pinto beans. unusual theodolites engag 12864|647|F|142470.94|1994-05-17|2-HIGH|Clerk#000000620|0| the ironic, silent foxes. special 12865|1271|O|244542.24|1997-05-23|5-LOW|Clerk#000000362|0|requests boost slyly carefully even instructions. carefully regular packages c 12866|1433|O|165429.43|1997-01-17|3-MEDIUM|Clerk#000000996|0|y regular requests play carefully against the always final 12867|59|F|197098.68|1995-03-12|4-NOT SPECIFIED|Clerk#000000638|0|lar platelets. blithely unusual deposits cajole carefully s 12868|1042|P|229577.12|1995-03-02|1-URGENT|Clerk#000000993|0|ns sleep above the carefully ex 12869|1396|F|131379.62|1994-01-26|4-NOT SPECIFIED|Clerk#000000871|0|s the pending foxes: fluffily regular requests wake slyly against the slyl 12870|1366|F|51595.17|1993-07-28|1-URGENT|Clerk#000000157|0| furiously dogged theodolites. regular requests among the blithely 12871|1174|O|194923.17|1995-10-05|4-NOT SPECIFIED|Clerk#000000646|0|ally ironic packages. blithely regular asymptotes are a 12896|370|O|321570.67|1996-12-22|1-URGENT|Clerk#000000337|0|nic theodolites. slyly even pinto beans use quickly. care 12897|1169|O|153229.93|1995-07-19|4-NOT SPECIFIED|Clerk#000000854|0|ost carefully regular packages. blithely 12898|1045|F|44854.49|1994-09-17|3-MEDIUM|Clerk#000000873|0|packages maintain carefully after 12899|868|O|40541.89|1996-04-07|4-NOT SPECIFIED|Clerk#000000006|0|efully pending ideas cajole fluf 12900|220|F|41268.96|1993-05-04|2-HIGH|Clerk#000000285|0|s. even requests sleep carefully blithely unusual ac 12901|1283|F|302296.36|1994-05-06|2-HIGH|Clerk#000000994|0|ely final requests. fu 12902|640|F|321334.39|1993-10-30|4-NOT SPECIFIED|Clerk#000000254|0|y regular deposits. regular instructions sleep 12903|301|F|89081.29|1994-09-26|1-URGENT|Clerk#000000914|0|wake furiously fluffily bold dolphins. blithely regular pinto beans sle 12928|634|F|258054.14|1994-03-28|2-HIGH|Clerk#000000465|0|ng slyly after the final requests. furiously special 12929|917|F|138543.10|1993-08-31|1-URGENT|Clerk#000000397|0|g the blithely bold asymptotes. pending dependencies 12930|1072|O|10854.39|1995-11-25|2-HIGH|Clerk#000000162|0|es cajole alongside of the fluffily pending dependencies. regular 12931|775|F|72091.62|1992-04-25|3-MEDIUM|Clerk#000000196|0|final foxes boost across the slyly pending dependencies. even, 12932|238|O|151284.17|1997-08-27|3-MEDIUM|Clerk#000000787|0|ackages. foxes cajole blithely regular 12933|173|O|253181.34|1998-02-17|3-MEDIUM|Clerk#000000224|0|haggle according to 12934|623|F|273516.60|1994-03-06|4-NOT SPECIFIED|Clerk#000000441|0|sual, pending dependencies among 12935|509|F|69433.26|1994-05-10|3-MEDIUM|Clerk#000000549|0|gouts at the even packages cajole slyly final packages. slyly enticin 12960|1243|F|132107.73|1993-10-10|2-HIGH|Clerk#000000340|0|ng theodolites haggle after the final id 12961|1144|F|133684.91|1994-12-13|2-HIGH|Clerk#000000913|0|rding to the enticing, final foxes use blithely furiou 12962|1111|O|130541.81|1995-10-29|1-URGENT|Clerk#000000488|0|. accounts breach carefully blithely ironic deposits. 12963|1232|O|72621.23|1998-07-08|3-MEDIUM|Clerk#000000546|0|uriously regular deposits wake slowly along t 12964|556|O|228128.18|1998-01-13|2-HIGH|Clerk#000000980|0|ronic instructions. quickly regular a 12965|964|F|107134.13|1993-06-26|3-MEDIUM|Clerk#000000650|0|ate slyly against the carefully final packages. furiously final requests alo 12966|224|O|51608.06|1997-10-11|5-LOW|Clerk#000000252|0|ly furiously final packages. carefully regular packages integrate quic 12967|1147|F|9777.25|1993-09-22|2-HIGH|Clerk#000000650|0|gly final deposits are slyly carefully ironic cour 12992|139|O|199084.84|1997-02-25|5-LOW|Clerk#000000099|0|xpress theodolites use across the un 12993|1348|O|46875.17|1998-05-14|5-LOW|Clerk#000000435|0| across the depths. idle, final accounts lose furiously regular, 12994|1451|F|54188.56|1994-11-18|5-LOW|Clerk#000000557|0|ily among the slyly unusual deposits. quickly pending acco 12995|1162|O|200689.05|1996-07-29|3-MEDIUM|Clerk#000000027|0|ts except the quickly express request 12996|967|O|107497.47|1995-12-05|4-NOT SPECIFIED|Clerk#000000531|0|ccounts. blithely ironic pinto beans cajol 12997|520|O|23304.56|1996-05-27|1-URGENT|Clerk#000000085|0|accounts boost. final asymptotes affix blithely fluff 12998|799|O|25952.72|1996-10-03|2-HIGH|Clerk#000000213|0|instructions sleep. express, final patterns detect furiously. furi 12999|940|F|216567.07|1993-04-04|2-HIGH|Clerk#000000745|0|nusual foxes. express accounts sublate q 13024|664|F|58015.05|1992-03-14|3-MEDIUM|Clerk#000000700|0| bold requests nag quickly. d 13025|158|O|138165.03|1998-01-28|2-HIGH|Clerk#000000785|0|lly regular instructions serve. special deposits against the careful 13026|1126|O|194927.69|1997-04-01|2-HIGH|Clerk#000000050|0| bold ideas! express ideas across the furiously dogged theodolites use bold, 13027|955|F|160057.71|1992-01-16|1-URGENT|Clerk#000000236|0|ongside of the ideas wak 13028|685|F|231028.06|1994-04-09|1-URGENT|Clerk#000000687|0|regular platelets a 13029|340|O|177062.13|1997-11-11|3-MEDIUM|Clerk#000000699|0|riously above the blithely f 13030|1459|F|146978.50|1994-07-23|3-MEDIUM|Clerk#000000878|0|hely ironic packages are slyly regular ideas. quickly final pack 13031|7|F|237595.83|1992-12-05|5-LOW|Clerk#000000325|0|s are furiously. busy requests haggle furiously pinto beans. asymptotes are. 13056|437|F|193618.95|1994-04-23|4-NOT SPECIFIED|Clerk#000000174|0|ounts. quickly regular packages above the furiously unusual sauternes ha 13057|67|O|102973.85|1997-12-16|1-URGENT|Clerk#000000125|0|y. final requests breach furiously. regular deposits engage. 13058|1156|O|12461.38|1997-03-26|1-URGENT|Clerk#000000685|0|ly! special instructions sleep furiously throughout the deposits. 13059|136|O|197435.35|1997-05-31|5-LOW|Clerk#000000463|0| requests. blithely special 13060|1345|O|272394.13|1997-02-06|4-NOT SPECIFIED|Clerk#000000724|0|ously above the pending, special theodolites. dolphins ar 13061|821|O|76034.36|1997-09-17|1-URGENT|Clerk#000000937|0|regular deposits. s 13062|1291|O|63228.29|1996-05-03|2-HIGH|Clerk#000000216|0|s asymptotes cajole carefully regular depen 13063|1412|O|318204.60|1997-01-15|5-LOW|Clerk#000000506|0|packages are slyly fluffil 13088|1460|F|261581.56|1993-04-23|4-NOT SPECIFIED|Clerk#000000089|0|s. even, final packages on the carefully regular platelets cajole quick 13089|1403|O|277440.38|1995-12-15|4-NOT SPECIFIED|Clerk#000000502|0|e slyly ironic pinto beans. blithely express accounts boost. pen 13090|130|F|146458.92|1993-07-27|4-NOT SPECIFIED|Clerk#000000717|0|packages impress quickly across the fluffily regul 13091|856|F|200741.15|1994-11-16|3-MEDIUM|Clerk#000000786|0|ironic requests nag furiously stealthy pinto beans. slyly even pains 13092|1430|O|142733.13|1998-06-24|1-URGENT|Clerk#000000241|0|eans haggle furiously about the slyly daring deposi 13093|1132|O|144898.05|1996-05-05|2-HIGH|Clerk#000000244|0| slyly even sheaves haggl 13094|1129|F|64336.85|1993-08-19|4-NOT SPECIFIED|Clerk#000000376|0|theodolites serve around the slyly special accounts. instruc 13095|1454|O|282335.80|1996-07-22|1-URGENT|Clerk#000000837|0|ve theodolites detect according to the slyly special grouches. un 13120|1039|F|90929.21|1994-12-15|5-LOW|Clerk#000000149|0| to the quickly express requests nag carefully stealthily regular requ 13121|878|F|190958.73|1993-03-08|2-HIGH|Clerk#000000148|0|cross the blithely ironic the 13122|242|F|164977.59|1994-09-24|2-HIGH|Clerk#000000229|0|t blithely furiously special accounts. carefully regular 13123|841|O|208768.85|1998-07-03|2-HIGH|Clerk#000000231|0|fter the slyly speci 13124|10|F|57676.34|1993-12-18|3-MEDIUM|Clerk#000000862|0|ggle quickly according to the carefully even pac 13125|355|F|190034.02|1993-12-14|4-NOT SPECIFIED|Clerk#000000302|0|ly quickly final ideas. carefully bold ideas slee 13126|956|O|131328.54|1995-10-20|4-NOT SPECIFIED|Clerk#000000919|0|ithely bold foxes wake. accounts according to the quickly ev 13127|1165|F|38692.83|1993-08-01|3-MEDIUM|Clerk#000000178|0|cording to the ironic asymptotes sleep care 13152|850|F|283938.70|1994-02-15|2-HIGH|Clerk#000000367|0|ts. blithely final instructions sleep quickly final requ 13153|1144|O|165659.63|1996-11-06|1-URGENT|Clerk#000000666|0|refully across the carefully careful notornis. qu 13154|520|O|286298.87|1997-10-22|5-LOW|Clerk#000000557|0|foxes? quickly blithe ideas nag slyly. blithely regular packages cajole. fluff 13155|1079|O|190720.20|1996-06-18|1-URGENT|Clerk#000000727|0|ns play. express requests cajole quickly theodolites. carefully furious dino 13156|1283|O|180693.69|1995-10-06|5-LOW|Clerk#000000482|0| about the carefully regular a 13157|656|F|277209.62|1992-02-12|3-MEDIUM|Clerk#000000655|0|regular requests wake. never special ac 13158|134|F|198301.99|1992-11-21|1-URGENT|Clerk#000000638|0|ns promise slyly even requests. carefully ironic packages haggle 13159|784|O|175684.88|1996-12-02|1-URGENT|Clerk#000000262|0|efully among the blithely regular instructions. carefu 13184|472|O|21843.01|1998-03-28|4-NOT SPECIFIED|Clerk#000000845|0|ilent deposits haggle furiously about the silent deposits. 13185|805|O|300236.26|1997-07-15|5-LOW|Clerk#000000862|0|ly even tithes sleep daringly. unusual, ironic accou 13186|556|O|206062.94|1996-09-18|2-HIGH|Clerk#000000427|0|ckly pending accounts. quick 13187|923|F|78566.07|1993-12-09|2-HIGH|Clerk#000000421|0|ns haggle slyly. slyly pending instructions at the carefully final 13188|1436|O|95591.25|1998-07-20|3-MEDIUM|Clerk#000000061|0|ckey players. accounts haggle fluffily against the foxes. unusual, expre 13189|94|F|2361.93|1995-04-05|1-URGENT|Clerk#000000002|0|l deposits above the pending pinto beans are 13190|136|O|24775.00|1998-05-04|4-NOT SPECIFIED|Clerk#000000173|0|furiously express accounts. carefully i 13191|214|O|331578.75|1997-12-24|5-LOW|Clerk#000000080|0|regular accounts at the blithely even sh 13216|1075|F|97503.77|1992-04-14|3-MEDIUM|Clerk#000000901|0|e furiously express pinto beans. unusual deposits are. even instructio 13217|946|O|67106.36|1997-09-25|5-LOW|Clerk#000000659|0|se dependencies nag blithely after the blithely regular ideas. 13218|1438|F|210525.16|1994-05-17|2-HIGH|Clerk#000000715|0|ccounts. slyly pending instru 13219|535|O|190074.19|1997-10-15|2-HIGH|Clerk#000000251|0|s. regular, special platelets wake. unusual, even 13220|1480|O|134628.97|1998-07-31|1-URGENT|Clerk#000000586|0|refully final platelets: accounts of the busily special 13221|946|O|106536.33|1995-10-15|4-NOT SPECIFIED|Clerk#000000459|0|lthy accounts around the 13222|1363|O|93525.50|1997-05-04|4-NOT SPECIFIED|Clerk#000000435|0|quests. furiously silent packages 13223|352|O|45917.67|1996-08-08|4-NOT SPECIFIED|Clerk#000000713|0| slyly blithely regular requests. furiously regular multipliers wake 13248|892|O|39800.14|1997-02-02|3-MEDIUM|Clerk#000000920|0|eans use. even packages eat after the ca 13249|569|O|222028.79|1997-07-18|5-LOW|Clerk#000000660|0|nstructions nag blithely quick 13250|1325|O|141857.68|1997-08-18|3-MEDIUM|Clerk#000000150|0|sts about the final requests a 13251|982|O|123168.81|1997-03-28|1-URGENT|Clerk#000000928|0|cording to the blithely silent deposit 13252|1309|F|163473.91|1994-03-23|4-NOT SPECIFIED|Clerk#000000026|0|n ideas must boost blithely. expre 13253|790|O|277363.52|1996-10-28|2-HIGH|Clerk#000000767|0|refully even packages print. blithely bold deposits boost fluffily 13254|1066|O|204264.00|1998-02-23|3-MEDIUM|Clerk#000000890|0|blithely silent packages. fin 13255|1115|F|113899.92|1993-07-18|3-MEDIUM|Clerk#000000993|0|metimes across the even deposits. pending accounts affix blithely furi 13280|314|F|236922.45|1994-09-12|2-HIGH|Clerk#000000073|0|pinto beans promise carefully final requests. r 13281|1480|F|164425.24|1992-04-23|1-URGENT|Clerk#000000327|0|slyly. regularly ironic deposits haggle bli 13282|817|F|153262.10|1992-05-10|5-LOW|Clerk#000000436|0|luffily express foxes 13283|946|O|129981.62|1997-12-29|4-NOT SPECIFIED|Clerk#000000842|0|dependencies? quickly quick deposits cajol 13284|851|F|136289.59|1992-08-13|2-HIGH|Clerk#000000186|0|equests. slyly final de 13285|406|O|8165.39|1995-05-17|4-NOT SPECIFIED|Clerk#000000312|0|r packages. silent, even 13286|1114|O|81226.73|1995-10-12|2-HIGH|Clerk#000000186|0|osits. slyly express requests promise quickly. bold, final asymptote 13287|895|O|168246.03|1997-03-10|3-MEDIUM|Clerk#000000335|0|e theodolites. blithely even deposits dazzle care 13312|1324|O|80963.75|1996-11-04|1-URGENT|Clerk#000000904|0|final accounts wake slyl 13313|1438|F|254146.32|1994-07-17|4-NOT SPECIFIED|Clerk#000000036|0|d platelets. slyly express requests print carefully across the quickl 13314|412|F|226838.60|1994-05-14|2-HIGH|Clerk#000000842|0|ajole stealthily even deposits. furiously express foxes was evenly r 13315|449|O|192954.51|1996-10-16|2-HIGH|Clerk#000000759|0|thinly ironic accoun 13316|1316|F|42308.46|1992-10-23|5-LOW|Clerk#000000555|0|tain even, pending instructions. quickly pending pinto beans sleep against the 13317|865|O|43720.88|1996-04-12|3-MEDIUM|Clerk#000000096|0|ggedly unusual packages along the carefully final 13318|325|O|247805.76|1997-06-11|4-NOT SPECIFIED|Clerk#000000381|0|ounts. sometimes ironic packages sleep stealthily. slyly regular accounts are 13319|425|O|138467.85|1996-01-19|1-URGENT|Clerk#000000158|0|ecial excuses boost. carefully regular asymptotes was acco 13344|367|O|90946.06|1997-03-27|4-NOT SPECIFIED|Clerk#000000924|0|de the quickly regular excuses. blithely express packages nod fur 13345|1339|F|136543.60|1992-09-03|4-NOT SPECIFIED|Clerk#000000876|0|thely special dolphins. slyly regular foxes sleep carefull 13346|1012|F|244898.42|1992-11-16|5-LOW|Clerk#000000594|0|thely bold foxes sleep final reque 13347|560|F|250026.98|1993-04-24|4-NOT SPECIFIED|Clerk#000000933|0|uriously against the carefully final foxes. carefully 13348|1267|F|94270.40|1993-11-08|3-MEDIUM|Clerk#000000878|0|ns doubt fluffily across the requests. even theodolites around the fluffi 13349|1454|F|91069.45|1994-09-20|2-HIGH|Clerk#000000916|0|usual courts haggle ruthlessly. final deposi 13350|1307|O|246713.06|1998-05-06|2-HIGH|Clerk#000000152|0|gular requests cajole quickl 13351|709|F|203403.50|1993-05-07|3-MEDIUM|Clerk#000000892|0|eposits haggle slowly alongside of 13376|1456|F|245552.88|1992-10-10|3-MEDIUM|Clerk#000000807|0| deposits. furiously regular packages unwind slyly bl 13377|394|O|60077.37|1995-07-19|2-HIGH|Clerk#000000094|0|ily along the bold deposits. carefully unusual accounts use quickl 13378|1151|F|207411.39|1994-09-22|5-LOW|Clerk#000000729|0|nis integrate; deposits wake bravely special frets. furiously expre 13379|286|F|107484.04|1993-11-08|5-LOW|Clerk#000000398|0|hely special dependencies. carefully unusual deposits are regularly. b 13380|934|F|129938.34|1994-07-16|1-URGENT|Clerk#000000683|0|ffy deposits haggle blithely. bold deposits amon 13381|1109|O|102850.64|1995-08-18|3-MEDIUM|Clerk#000000835|0|s platelets nod carefully. final, unusual deposits nag blith 13382|1207|O|64313.96|1995-11-02|4-NOT SPECIFIED|Clerk#000000427|0|kages. regular packages boost slyly alongside 13383|1346|F|165841.68|1992-06-16|4-NOT SPECIFIED|Clerk#000000258|0|pecial ideas boost stealthily about the regular, express pinto bean 13408|1370|F|211990.67|1994-02-15|5-LOW|Clerk#000000772|0| since the pinto beans. furiously pending accounts ru 13409|1279|F|46643.47|1993-07-14|4-NOT SPECIFIED|Clerk#000000002|0|n dependencies detect furi 13410|592|F|71187.66|1993-06-07|4-NOT SPECIFIED|Clerk#000000922|0|lly ironic braids are. quickly final gifts haggle si 13411|1376|F|130230.72|1994-04-13|2-HIGH|Clerk#000000179|0|l foxes. silent theodolit 13412|1240|O|115408.80|1997-07-12|1-URGENT|Clerk#000000124|0|lithely slow packages. packages past the ru 13413|473|F|231772.27|1993-01-30|1-URGENT|Clerk#000000565|0|ironic instructions. special instructions boost quick 13414|1333|O|143244.82|1995-07-07|3-MEDIUM|Clerk#000000693|0|across the blithely regular 13415|1222|O|271662.08|1997-05-31|2-HIGH|Clerk#000000851|0|ully. packages haggle slyly around the 13440|865|F|24449.73|1993-10-14|4-NOT SPECIFIED|Clerk#000000189|0|fily ironic accounts. quietly regular deposits among th 13441|1282|O|60048.20|1995-08-31|1-URGENT|Clerk#000000983|0|tes. blithely regular packages wake enticingly. quickly da 13442|238|O|138794.82|1997-08-06|4-NOT SPECIFIED|Clerk#000000830|0|p quickly fluffily 13443|545|F|216308.57|1993-10-07|3-MEDIUM|Clerk#000000379|0|uriously silent packages nag. ideas cajole quickly carefully regula 13444|176|F|304179.27|1992-01-06|2-HIGH|Clerk#000000199|0|. forges are carefully. furiously final packages are blithely ironic the 13445|682|O|21693.67|1997-12-24|4-NOT SPECIFIED|Clerk#000000599|0|quickly. excuses cajole. idly express packages cajole blithely pending pinto b 13446|484|O|55646.82|1996-02-29|1-URGENT|Clerk#000000050|0|lly regular requests. blithely express excuses integra 13447|244|F|72392.32|1993-01-09|2-HIGH|Clerk#000000925|0|ts. blithely ironic ideas accord 13472|814|O|69332.61|1995-10-27|1-URGENT|Clerk#000000454|0|uests. regular asymptotes haggle quickly aro 13473|655|O|192774.21|1996-03-22|5-LOW|Clerk#000000655|0|usual pinto beans haggle blithely at the carefully regular p 13474|1069|O|101285.14|1997-06-09|3-MEDIUM|Clerk#000000912|0|out the bold instructions. requests a 13475|1403|F|295575.64|1994-06-06|1-URGENT|Clerk#000000426|0|ake among the quickly express depo 13476|1459|F|264702.77|1993-10-17|4-NOT SPECIFIED|Clerk#000000203|0|r, permanent accounts. ironic, ironic packages dazzle at the 13477|1276|O|143935.11|1997-11-23|3-MEDIUM|Clerk#000000242|0|express instructions. regular ideas 13478|325|O|167211.22|1998-01-04|2-HIGH|Clerk#000000849|0|ajole ironic, stealthy theodolites. d 13479|1217|F|170168.00|1994-12-05|2-HIGH|Clerk#000000469|0|s the furiously regular accounts. blithely 13504|1495|O|126515.95|1997-06-18|2-HIGH|Clerk#000000486|0|ngly after the pending accounts. final instr 13505|286|O|219315.83|1997-06-03|3-MEDIUM|Clerk#000000880|0|sly against the furiously regular packages. care 13506|1099|O|111048.07|1995-07-22|5-LOW|Clerk#000000974|0|. slyly ironic requests among the 13507|923|F|117398.65|1992-12-30|3-MEDIUM|Clerk#000000127|0|. bold deposits affix furiously. furiously ironi 13508|802|O|2962.11|1997-04-17|4-NOT SPECIFIED|Clerk#000000951|0|accounts. blithely eve 13509|946|O|68465.06|1996-10-07|2-HIGH|Clerk#000000562|0|p. dolphins are. regular reques 13510|710|O|80383.62|1997-10-15|3-MEDIUM|Clerk#000000723|0|tions. slyly ironic in 13511|467|O|134751.21|1998-01-03|4-NOT SPECIFIED|Clerk#000000611|0|ly regular foxes are slyly furiously bold deposits. furiously fin 13536|316|F|192964.26|1994-08-19|3-MEDIUM|Clerk#000000262|0|posits. blithely final i 13537|430|F|26435.99|1993-05-31|5-LOW|Clerk#000000623|0|telets haggle carefully. quickly regular frays are! slyly regular pla 13538|782|O|65147.01|1997-02-01|2-HIGH|Clerk#000000598|0|lly alongside of the pending, unusual reque 13539|1231|O|190664.21|1997-02-13|3-MEDIUM|Clerk#000000022|0|ding dolphins use. final pinto beans 13540|460|F|277929.48|1993-05-24|2-HIGH|Clerk#000000994|0|. blithely even asymptotes poach blithely alongside of the doggedly un 13541|634|O|53890.86|1996-07-29|4-NOT SPECIFIED|Clerk#000000094|0| the furiously quick accounts. final packages nag 13542|1016|F|302527.26|1994-03-09|5-LOW|Clerk#000000170|0|ts are until the furiously 13543|790|F|24097.46|1992-08-09|3-MEDIUM|Clerk#000000636|0|l dependencies nag against the packages. expr 13568|31|F|81474.17|1993-05-07|5-LOW|Clerk#000000024|0|sts. carefully bold dolphins cajole across the even requ 13569|839|P|187147.11|1995-04-11|1-URGENT|Clerk#000000973|0|s? regular platelets cajole above the furiousl 13570|1468|F|226767.02|1992-12-03|4-NOT SPECIFIED|Clerk#000000602|0|e of the theodolites. slyly pending deposits between the regula 13571|1231|O|142142.08|1998-06-04|5-LOW|Clerk#000000452|0|ites? carefully regular 13572|1498|F|115177.91|1992-05-18|1-URGENT|Clerk#000000144|0|inal requests are regularly against th 13573|898|O|78389.93|1997-04-23|5-LOW|Clerk#000000144|0| slyly final packages. carefully e 13574|151|O|72886.11|1997-11-14|1-URGENT|Clerk#000000472|0|y above the fluffily final instructions. express 13575|349|F|92113.36|1992-08-30|1-URGENT|Clerk#000000008|0|! express, express instruct 13600|1129|F|124567.73|1993-07-11|4-NOT SPECIFIED|Clerk#000000910|0|lites boost furiously after the quickly ironic packages. carefully fina 13601|8|F|256717.52|1992-08-17|5-LOW|Clerk#000000608|0| ironic, special foxes- 13602|1348|F|284393.60|1994-01-14|5-LOW|Clerk#000000312|0|nusual packages cajole stealthily regular accounts. regular, 13603|496|O|198335.91|1997-06-09|4-NOT SPECIFIED|Clerk#000000777|0|nic deposits? blithely ironic packages do nag 13604|1141|F|280511.75|1994-02-03|4-NOT SPECIFIED|Clerk#000000798|0|p stealthily. requests thrash carefully. accounts wake furiously afte 13605|1388|O|201717.90|1996-06-14|3-MEDIUM|Clerk#000000099|0| regular depths use furiou 13606|160|F|209589.77|1994-04-27|4-NOT SPECIFIED|Clerk#000000310|0|eas. unusual courts haggle slyly ruthless realms. even, unusua 13607|358|F|210043.39|1993-11-19|1-URGENT|Clerk#000000366|0|es mold. blithely express packages integrate after the bold pinto beans. b 13632|1084|F|264841.12|1994-03-11|3-MEDIUM|Clerk#000000635|0|quickly special ideas use bravely bold, bold packag 13633|1373|F|173408.51|1992-04-04|4-NOT SPECIFIED|Clerk#000000419|0|ifts cajole quickly 13634|869|P|43165.51|1995-04-19|2-HIGH|Clerk#000000538|0|ain carefully according to t 13635|269|F|121071.76|1994-05-02|1-URGENT|Clerk#000000172|0|uctions thrash quickly. furiously regular requests about the n 13636|107|F|171401.11|1994-01-23|5-LOW|Clerk#000000855|0|gular ideas wake among th 13637|188|O|66820.97|1995-12-11|1-URGENT|Clerk#000000074|0| quickly ironic requests. care 13638|1361|O|327915.84|1995-10-01|2-HIGH|Clerk#000000261|0| ironic requests haggle quickly ironic packages. final 13639|1009|O|126608.97|1997-06-13|2-HIGH|Clerk#000000848|0|s cajole slyly doggedly ironic asymptotes. ca 13664|814|O|292106.85|1995-12-16|3-MEDIUM|Clerk#000000542|0|ely bold theodolites nag. pinto beans above the p 13665|1039|O|181151.79|1996-04-08|1-URGENT|Clerk#000000591|0|as wake even, pending requests 13666|445|F|59180.35|1992-03-03|4-NOT SPECIFIED|Clerk#000000031|0| the packages. ironi 13667|1216|O|208665.97|1996-12-04|2-HIGH|Clerk#000000893|0| ruthless ideas poach ironically-- unusual 13668|878|F|218731.00|1992-02-25|1-URGENT|Clerk#000000198|0|press ideas. blithely unusual instructions are blithely. carefully final pac 13669|454|O|178957.15|1996-08-15|3-MEDIUM|Clerk#000000955|0| instructions could have to cajol 13670|226|F|153065.25|1992-05-24|1-URGENT|Clerk#000000988|0|s haggle at the even platelets. even 13671|1291|O|218819.92|1996-07-14|1-URGENT|Clerk#000000321|0| blithely regular requests. furiously special idea 13696|722|O|118614.92|1997-08-28|5-LOW|Clerk#000000882|0|ag fluffily carefully even theodolite 13697|1457|O|103807.36|1998-04-06|4-NOT SPECIFIED|Clerk#000000473|0|st the blithely unusual foxes cajole fluf 13698|1483|F|155040.76|1994-08-05|4-NOT SPECIFIED|Clerk#000000640|0|iously bold deposits are carefully blithely unusual 13699|577|O|22846.00|1997-04-08|4-NOT SPECIFIED|Clerk#000000746|0|. ironically ironic foxes among the f 13700|1063|F|20210.10|1992-02-14|1-URGENT|Clerk#000000801|0|heodolites cajole blithely. blithely unusual de 13701|1270|O|222424.50|1998-05-18|2-HIGH|Clerk#000000270|0|final accounts wake furiously among the regular foxes. slyly bold t 13702|1132|O|51969.64|1995-09-11|5-LOW|Clerk#000000834|0|round the fluffily silen 13703|236|O|219200.99|1996-12-29|5-LOW|Clerk#000000778|0| quickly furiously regular accou 13728|4|O|123722.52|1995-12-11|2-HIGH|Clerk#000000094|0|theodolites. ironic deposits boost among the slyly regular instru 13729|79|F|88426.57|1994-01-02|4-NOT SPECIFIED|Clerk#000000866|0|y regular platelets 13730|952|F|36769.57|1995-05-10|2-HIGH|Clerk#000000826|0|- quickly even pinto beans boost fluffily ironic, even requests. ironic, u 13731|1475|F|209159.47|1993-05-10|4-NOT SPECIFIED|Clerk#000000549|0|beans detect boldly. sheaves after the eve 13732|605|F|166563.09|1992-11-28|1-URGENT|Clerk#000000599|0|uriously regular courts 13733|100|O|328307.40|1998-01-03|5-LOW|Clerk#000000431|0|e quickly express foxes. blithely ironic instructio 13734|124|O|156502.58|1996-08-14|2-HIGH|Clerk#000000344|0|n deposits should have to thrash fluffily quickly regu 13735|58|F|130804.16|1994-05-22|3-MEDIUM|Clerk#000000589|0|ccounts are slyly blithely final accounts. furiously final accounts w 13760|964|O|130352.76|1996-06-03|5-LOW|Clerk#000000053|0|s wake. blithely bold dependencies 13761|169|F|199387.32|1994-10-18|5-LOW|Clerk#000000226|0|ainst the quickly regular accounts. quickly en 13762|1337|F|210739.37|1993-06-22|3-MEDIUM|Clerk#000000053|0|re blithely ideas. blithely regular ideas haggle above the foxe 13763|1142|O|164241.80|1997-06-27|3-MEDIUM|Clerk#000000027|0|as affix carefully express ideas. carefully ironic asymptotes unwin 13764|598|O|122061.97|1995-07-14|3-MEDIUM|Clerk#000000236|0|ronic deposits? furiously re 13765|715|F|156724.02|1994-04-06|4-NOT SPECIFIED|Clerk#000000969|0|ronic theodolites will have to haggle except the slyl 13766|1387|F|45293.38|1992-10-19|1-URGENT|Clerk#000000274|0|boost unusual accounts. regular reques 13767|923|F|211468.27|1993-08-27|2-HIGH|Clerk#000000679|0|. thinly express dolphins sleep; slyly final instr 13792|1366|O|48602.86|1996-02-04|4-NOT SPECIFIED|Clerk#000000980|0|y slyly final foxes. slyly even ideas affix furiously dependencies. caref 13793|904|O|18039.82|1995-09-18|2-HIGH|Clerk#000000222|0|e blithely above the slyly pending ideas. furiously furious pl 13794|587|F|234351.58|1994-01-05|1-URGENT|Clerk#000000119|0|ages cajole furiously alongside of the slyly express ideas- 13795|1406|P|233445.40|1995-04-11|4-NOT SPECIFIED|Clerk#000000673|0|ld dolphins cajole quickly unusual instructions. 13796|1004|O|245467.47|1997-06-22|4-NOT SPECIFIED|Clerk#000000344|0|special deposits wake 13797|272|O|227061.71|1996-08-26|1-URGENT|Clerk#000000379|0|usly silent theodolites wake slyly across 13798|1298|O|123932.29|1996-02-12|1-URGENT|Clerk#000000778|0|fluffily special, bol 13799|994|F|142646.96|1993-12-02|1-URGENT|Clerk#000000945|0|g according to the regularly even requests. carefully bold accoun 13824|1096|F|115028.49|1994-12-28|5-LOW|Clerk#000000604|0|equests. slow, sly ideas cajole fu 13825|56|F|174217.02|1994-03-22|2-HIGH|Clerk#000000038|0|ests. slyly express pin 13826|232|O|160230.62|1997-02-22|3-MEDIUM|Clerk#000000053|0|ost packages. slyly pending requests are c 13827|479|F|44465.43|1993-06-21|5-LOW|Clerk#000000455|0|busily close requests. ironic, even pinto beans use furiously requests. c 13828|985|F|85589.73|1994-07-13|1-URGENT|Clerk#000000682|0| unusual instructions sleep quickly. packages 13829|1075|O|188654.94|1996-09-22|3-MEDIUM|Clerk#000000010|0|raids alongside of the quick 13830|598|F|169613.65|1994-12-22|2-HIGH|Clerk#000000693|0|impress. even, special requests alongside of the instructions cajole fu 13831|740|O|234924.78|1996-04-17|1-URGENT|Clerk#000000204|0|efully unusual theodolites s 13856|1156|O|154144.16|1997-11-28|4-NOT SPECIFIED|Clerk#000000601|0| special theodolites cajole fluffi 13857|275|O|9265.55|1995-08-15|1-URGENT|Clerk#000000012|0|uses boost furiously even 13858|1105|F|44168.23|1994-07-13|2-HIGH|Clerk#000000003|0| fluffily even, regular accounts. furiously permanent dependencies are. slyl 13859|221|O|228467.72|1997-01-09|4-NOT SPECIFIED|Clerk#000000630|0|; enticingly ironic deposits detect along the blithely bold 13860|454|F|61875.64|1994-09-25|3-MEDIUM|Clerk#000000878|0|ccounts nod carefully. express requests whithout the slyly regular in 13861|1249|F|223838.87|1993-08-12|2-HIGH|Clerk#000000153|0|ng requests. quickly ironic packages around th 13862|1240|O|233858.28|1997-09-03|3-MEDIUM|Clerk#000000838|0|odolites integrate carefully unusual requests. sly ideas are 13863|1399|F|132450.97|1992-05-31|2-HIGH|Clerk#000000471|0|y special deposits boost ca 13888|733|F|6632.13|1993-05-04|3-MEDIUM|Clerk#000000635|0|ly regular pinto beans boost about the sheav 13889|848|O|73735.52|1995-09-05|3-MEDIUM|Clerk#000000379|0|ng the silent foxes. even, quiet patterns about th 13890|34|O|50473.48|1996-05-25|5-LOW|Clerk#000000263|0|eposits along the fluffily final accounts wake final, pending foxe 13891|508|O|62430.54|1996-07-05|4-NOT SPECIFIED|Clerk#000000147|0|es are slyly. special packages use slyly against the theodolites. pa 13892|1013|F|216407.28|1992-04-28|2-HIGH|Clerk#000000365|0|ithely ironic packages sleep after the ruthless instructions. quickly dogged t 13893|289|O|132790.59|1998-04-12|2-HIGH|Clerk#000000931|0| blithely regular, unusual dep 13894|1246|F|125452.99|1992-04-11|3-MEDIUM|Clerk#000000524|0|y pending hockey players cajole fluffily special sentiments. closely ironic 13895|671|F|225938.31|1993-11-21|2-HIGH|Clerk#000000381|0|across the fluffily even accounts. carefully silent ideas wake. blithe 13920|37|F|155193.67|1994-10-10|1-URGENT|Clerk#000000060|0| unusual deposits sleep blithely 13921|49|O|152288.97|1997-08-19|4-NOT SPECIFIED|Clerk#000000358|0|elets doubt against the slyly final 13922|241|O|252983.77|1995-08-03|2-HIGH|Clerk#000000301|0|requests use ironic de 13923|1000|F|155909.13|1992-07-09|2-HIGH|Clerk#000000189|0|s. regular packages 13924|328|F|3658.90|1994-12-20|2-HIGH|Clerk#000000251|0|haggle quickly ironic platelets. even tithes wake blit 13925|73|O|162158.06|1997-01-06|1-URGENT|Clerk#000000303|0|uriously ironic accounts around the never regular packages 13926|49|O|52124.55|1997-07-13|4-NOT SPECIFIED|Clerk#000000881|0|ages haggle furiously bold, final accounts. special foxes 13927|562|F|121815.82|1994-08-27|5-LOW|Clerk#000000534|0|lar, silent packages wake quickly 13952|571|O|115051.12|1997-04-25|4-NOT SPECIFIED|Clerk#000000896|0|ly ironic pinto beans. furiously even accounts haggle carefully accordi 13953|388|O|233895.85|1995-06-01|1-URGENT|Clerk#000000337|0|riously deposits. blithely 13954|775|O|132924.82|1996-05-02|5-LOW|Clerk#000000809|0|t furiously above the final requ 13955|1409|O|161957.19|1998-05-12|2-HIGH|Clerk#000000359|0|ar foxes. packages nag blithely fina 13956|716|P|107050.72|1995-03-13|1-URGENT|Clerk#000000062|0|e final deposits are according to the regula 13957|475|O|176779.20|1995-07-22|4-NOT SPECIFIED|Clerk#000000019|0|mpress carefully slyly even requests. blithely pending t 13958|616|F|52000.53|1993-01-24|5-LOW|Clerk#000000863|0|ly. carefully special accounts among the slyly express requests sleep afte 13959|451|F|99030.36|1994-04-05|3-MEDIUM|Clerk#000000952|0|efully bold requests above the busy, express deposits use express p 13984|401|F|197913.83|1992-09-25|5-LOW|Clerk#000000750|0|final, special foxes. carefully express theodolites boost ne 13985|397|O|174830.89|1998-06-14|2-HIGH|Clerk#000000876|0|old accounts wake instructions. furiously ironic requests use quickly. quickl 13986|1333|O|103675.53|1995-10-12|2-HIGH|Clerk#000000793|0|riously final foxes. fur 13987|643|O|64387.39|1998-04-14|4-NOT SPECIFIED|Clerk#000000268|0|es. furiously even 13988|53|F|114321.03|1992-08-02|4-NOT SPECIFIED|Clerk#000000556|0|le daring packages. final, final escapades cajole expre 13989|1480|O|189523.80|1997-07-06|5-LOW|Clerk#000000996|0|ate about the bold, express packages. closely brave accou 13990|751|O|143454.90|1998-02-07|1-URGENT|Clerk#000000683|0| slyly according to the furiously spe 13991|1204|F|261061.37|1992-03-07|2-HIGH|Clerk#000000245|0|nic, special accounts. quickly pending accounts wake 14016|1145|F|201366.56|1992-07-12|5-LOW|Clerk#000000150|0|ymptotes: packages around the 14017|1033|F|142714.54|1994-03-08|2-HIGH|Clerk#000000238|0|unusual deposits sleep furiously 14018|367|F|101182.03|1993-06-07|2-HIGH|Clerk#000000376|0|uctions. pending instructions cajole about th 14019|184|O|216427.67|1997-09-08|1-URGENT|Clerk#000000961|0|tect fluffily final pac 14020|841|O|75424.30|1998-01-29|2-HIGH|Clerk#000000275|0|aggle quickly along the blithely bold ins 14021|62|P|206447.05|1995-03-08|2-HIGH|Clerk#000000467|0|sly. carefully regular 14022|685|F|71271.43|1995-02-14|2-HIGH|Clerk#000000732|0|es. quickly express deposits nag across the iron 14023|787|F|127450.06|1993-02-21|4-NOT SPECIFIED|Clerk#000000349|0|ironic, regular dugouts 14048|1016|F|221205.20|1993-05-18|4-NOT SPECIFIED|Clerk#000000548|0|s. blithely even asy 14049|952|O|70228.24|1996-01-29|5-LOW|Clerk#000000189|0|yly express epitaphs are slyly express depo 14050|88|O|161689.12|1997-08-22|5-LOW|Clerk#000000283|0|en courts. carefully express packages 14051|599|F|22238.93|1993-09-17|4-NOT SPECIFIED|Clerk#000000098|0|ss blithely among the closely r 14052|1190|F|23078.95|1995-02-07|4-NOT SPECIFIED|Clerk#000000647|0|mong the regular foxes boost blithely special pinto beans. furious 14053|1453|F|25549.20|1993-02-09|2-HIGH|Clerk#000000597|0| carefully ironic courts sleep quickly pending, pending packag 14054|517|O|285667.97|1996-01-11|4-NOT SPECIFIED|Clerk#000000890|0|unusual waters above the dependencies cajole r 14055|904|F|90239.70|1994-09-02|2-HIGH|Clerk#000000825|0|ounts. carefully final somas unwind fluffily. 14080|791|O|62517.87|1998-05-29|4-NOT SPECIFIED|Clerk#000000106|0|nic packages use final, bold 14081|895|F|114279.17|1993-10-07|2-HIGH|Clerk#000000857|0|uests. fluffily final packages wake slow, ironic dep 14082|875|O|48224.95|1998-05-08|4-NOT SPECIFIED|Clerk#000000700|0|bove the sometimes even deposits. ruthlessly unusual deposit 14083|464|F|11732.48|1994-08-16|4-NOT SPECIFIED|Clerk#000000420|0| quickly ruthless a 14084|647|O|303760.75|1995-08-03|4-NOT SPECIFIED|Clerk#000000160|0|luffily above the final packages. requests 14085|895|F|136353.74|1994-02-05|2-HIGH|Clerk#000000503|0|eas boost carefully evenly pending requests-- furiously thin accounts 14086|346|F|54201.92|1993-10-21|5-LOW|Clerk#000000289|0|ial deposits sleep. express requests nag carefull 14087|275|O|208589.10|1997-02-12|5-LOW|Clerk#000000927|0|dependencies. quickly ironic frets above the fluffily ironic accounts use 14112|112|F|20742.84|1994-07-25|5-LOW|Clerk#000000463|0|ickly dogged accounts breach doggedly 14113|835|O|27091.83|1995-08-05|2-HIGH|Clerk#000000885|0|; carefully pending accounts use among the pending packag 14114|895|F|171306.58|1994-04-07|1-URGENT|Clerk#000000397|0|beans play regular, ironic deposits. instructio 14115|46|F|68488.60|1994-08-15|4-NOT SPECIFIED|Clerk#000000493|0|ckages sleep across the regular, silent pinto beans. bold, un 14116|1291|O|192210.86|1995-10-10|2-HIGH|Clerk#000000065|0|. carefully bold ideas sleep. carefully bold idea 14117|1297|F|281939.12|1993-08-26|4-NOT SPECIFIED|Clerk#000000480|0|uickly ironic ideas boost among the furiously pending deposits. 14118|385|F|41143.91|1993-05-23|4-NOT SPECIFIED|Clerk#000000730|0| toward the express, unusual instructions sleep accordin 14119|482|F|203405.63|1993-06-01|5-LOW|Clerk#000000989|0|osits. furiously pending accounts across the slyly 14144|301|O|308016.67|1997-04-26|1-URGENT|Clerk#000000089|0|nusual instructions sleep. blithely silent requests thr 14145|7|O|270751.41|1997-04-17|1-URGENT|Clerk#000000920|0|x carefully ideas. evenly silent 14146|592|O|108225.88|1996-01-03|3-MEDIUM|Clerk#000000968|0|xpress accounts. even packages about the pen 14147|728|F|262030.95|1993-03-08|4-NOT SPECIFIED|Clerk#000000152|0|, special epitaphs haggle a 14148|965|O|130798.36|1998-04-30|3-MEDIUM|Clerk#000000188|0|ag carefully special foxes. carefully unus 14149|73|O|157174.97|1998-01-28|1-URGENT|Clerk#000000303|0|ests. ironic, special pinto beans try to print slyly along the pending 14150|89|F|196189.74|1994-02-09|2-HIGH|Clerk#000000695|0|y ironic accounts haggle across the furi 14151|1418|O|286984.16|1997-06-06|4-NOT SPECIFIED|Clerk#000000278|0|special accounts. blithely express deposits cajole slyly slyly ex 14176|991|F|141050.11|1994-03-13|5-LOW|Clerk#000000407|0|eodolites. slyly final platelets ca 14177|565|F|112612.65|1992-04-03|5-LOW|Clerk#000000442|0|nooze carefully slyly final packages. platelets about the q 14178|391|F|11236.45|1995-01-01|4-NOT SPECIFIED|Clerk#000000246|0| quick accounts nag carefully. regular accounts could 14179|458|O|384265.43|1997-09-07|1-URGENT|Clerk#000000053|0|about the bold, final pinto bean 14180|292|O|187330.32|1997-05-25|3-MEDIUM|Clerk#000000478|0|out the carefully regular accoun 14181|94|O|226844.98|1998-02-13|3-MEDIUM|Clerk#000000106|0|efully even depende 14182|934|F|68283.44|1994-06-23|5-LOW|Clerk#000000791|0|arefully. sentiments integrate 14183|875|O|85725.94|1997-03-17|5-LOW|Clerk#000000052|0| final requests detect slyly at the 14208|385|O|121214.10|1995-05-01|4-NOT SPECIFIED|Clerk#000000758|0|sly slyly silent requests. carefully special accounts sleep 14209|1025|O|137491.26|1996-11-04|2-HIGH|Clerk#000000211|0|leep slyly against the carefully bold dolphins. 14210|25|F|23549.76|1992-06-27|4-NOT SPECIFIED|Clerk#000000356|0|atelets nag carefully final foxes. ironic, silent 14211|1202|F|189400.90|1994-02-13|4-NOT SPECIFIED|Clerk#000000937|0|the blithely bold deposits are according to the accounts. f 14212|806|F|196439.01|1992-02-17|1-URGENT|Clerk#000000384|0| packages affix special deposits? carefully special pinto bean 14213|478|F|50949.10|1993-12-28|1-URGENT|Clerk#000000959|0|, careful platelets abo 14214|79|O|128802.82|1997-08-15|1-URGENT|Clerk#000000891|0|thely unusual pinto beans against the blithely special warhorses hagg 14215|1408|F|50568.60|1992-07-14|3-MEDIUM|Clerk#000000823|0|ounts haggle furiously. special gifts am 14240|689|F|210064.90|1992-02-18|2-HIGH|Clerk#000000728|0|sly. express, unusual asymptotes across the never fin 14241|928|F|153112.39|1992-06-15|1-URGENT|Clerk#000000866|0|c packages. instructions a 14242|913|P|115246.93|1995-05-09|2-HIGH|Clerk#000000927|0|d have to haggle according 14243|1073|O|63765.25|1997-02-15|3-MEDIUM|Clerk#000000943|0|sly ironic foxes nag carefully along t 14244|205|O|53072.12|1998-03-05|5-LOW|Clerk#000000139|0|press excuses doubt permanently. 14245|1309|F|73717.72|1993-02-12|1-URGENT|Clerk#000000975|0|furiously final foxes boost silent, final requests. slyly unusual deposi 14246|289|O|133280.40|1997-10-10|4-NOT SPECIFIED|Clerk#000000806|0| bold pinto beans. regular accounts 14247|671|O|219431.33|1995-08-17|1-URGENT|Clerk#000000531|0|ests haggle furiously about the pa 14272|1348|F|97840.61|1992-02-13|3-MEDIUM|Clerk#000000075|0|ses. blithely final deposits are. stealthy accounts engage. slyly 14273|1033|O|90555.37|1996-03-13|5-LOW|Clerk#000000938|0| special foxes lose quickly; fina 14274|565|F|244782.98|1994-01-07|4-NOT SPECIFIED|Clerk#000000564|0|nt requests are furiously furiously daring accounts. even 14275|1412|F|168905.36|1993-06-14|5-LOW|Clerk#000000789|0|furiously. slyly pending packages sublate furious 14276|499|F|241009.94|1994-06-19|3-MEDIUM|Clerk#000000149|0|ly bold requests haggle slyly according to the closely 14277|946|O|116805.58|1998-02-14|4-NOT SPECIFIED|Clerk#000000951|0|eep. thin theodolites are bl 14278|559|O|266421.62|1998-06-03|2-HIGH|Clerk#000000768|0|jole slyly ironic theodolites. carefully even deposi 14279|1117|O|157844.27|1998-02-09|2-HIGH|Clerk#000000441|0| after the furiously unusual realms. 14304|788|O|215723.64|1997-11-13|3-MEDIUM|Clerk#000000488|0|egular requests affix. even requ 14305|1427|O|80794.79|1996-07-23|1-URGENT|Clerk#000000396|0|mas cajole above the 14306|587|F|27239.18|1993-10-30|4-NOT SPECIFIED|Clerk#000000981|0|furiously regular e 14307|484|O|178901.14|1997-06-24|1-URGENT|Clerk#000000255|0| deposits wake. regular, special excuses us 14308|5|O|132817.36|1997-08-11|3-MEDIUM|Clerk#000000354|0|iously. packages wake according to the furio 14309|1382|F|103342.31|1994-01-31|1-URGENT|Clerk#000000753|0|s haggle furiously against the carefully final asymptotes. blith 14310|1393|O|59609.50|1997-09-13|5-LOW|Clerk#000000627|0|p furiously evenly final requests. even, ironic excuses are 14311|1412|F|46918.86|1994-08-02|1-URGENT|Clerk#000000916|0|nic pinto beans along the enticingly final reque 14336|760|F|172716.15|1994-11-19|2-HIGH|Clerk#000000699|0|eodolites cajole carefully. quickly ironic foxes about the fluffily ironic 14337|299|O|280177.62|1997-10-09|4-NOT SPECIFIED|Clerk#000000949|0| special ideas into the silently quick dep 14338|752|F|97316.04|1993-02-01|5-LOW|Clerk#000000658|0|l accounts. quickly special packages h 14339|1199|O|209018.11|1997-07-23|4-NOT SPECIFIED|Clerk#000000303|0|ar theodolites after the express, final pinto beans engage car 14340|391|O|222337.16|1995-10-13|5-LOW|Clerk#000000386|0|theodolites. furiously ironic pinto beans are busily 14341|667|F|197341.53|1993-09-03|4-NOT SPECIFIED|Clerk#000000669|0|lphins cajole furiously after the c 14342|160|F|137933.49|1994-04-28|5-LOW|Clerk#000000040|0| haggle quickly. bold excuses against the regular re 14343|1408|O|58414.89|1996-02-25|3-MEDIUM|Clerk#000000792|0| unusual deposits wake slyly even, even packages. slyly final depe 14368|676|O|121598.53|1995-09-11|2-HIGH|Clerk#000000228|0|ly express requests affix furiously. ironically final asymptotes according 14369|1357|O|272333.98|1996-11-26|1-URGENT|Clerk#000000063|0|sublate quickly furiously bold asymptot 14370|958|O|163080.52|1997-05-03|2-HIGH|Clerk#000000475|0|ly regular accounts. fluffil 14371|1192|F|116441.72|1993-06-23|2-HIGH|Clerk#000000831|0|. carefully express packages at th 14372|523|F|166914.23|1992-10-24|1-URGENT|Clerk#000000449|0|y. ironic deposits sleep? finally special pinto bean 14373|511|F|53131.83|1993-09-18|3-MEDIUM|Clerk#000000099|0|nto beans. fluffy requests affix bravely fluffily iron 14374|1180|O|121655.04|1995-05-17|1-URGENT|Clerk#000000237|0|ross the blithely final deposits. asymptotes are. slyl 14375|733|F|140929.00|1993-02-01|3-MEDIUM|Clerk#000000046|0|ans sleep. blithely regular foxes cajole before the ironic packa 14400|632|F|262307.57|1994-02-05|3-MEDIUM|Clerk#000000479|0|uests. furiously unusual platelets hinder final packages. bold 14401|1489|O|156911.31|1995-08-05|4-NOT SPECIFIED|Clerk#000000059|0|ly carefully even instructions. epitaphs solve instructions! bli 14402|838|F|109228.63|1993-10-15|3-MEDIUM|Clerk#000000672|0|azzle slyly. carefully regular instructions affix carefully deposits. careful 14403|241|O|120881.79|1998-03-02|4-NOT SPECIFIED|Clerk#000000202|0|hely packages. blithely pending dependencies wake furiously 14404|7|O|354885.81|1996-11-03|5-LOW|Clerk#000000657|0| the furiously unus 14405|269|O|94417.57|1996-04-24|5-LOW|Clerk#000000460|0| beans until the final, regular theodolites 14406|409|F|194997.99|1993-03-09|3-MEDIUM|Clerk#000000623|0|s. excuses boost bl 14407|952|F|21880.64|1993-11-05|2-HIGH|Clerk#000000124|0|cajole ruthless theodolites. carefully ironic req 14432|1226|O|44225.65|1996-11-12|4-NOT SPECIFIED|Clerk#000000784|0|re carefully against the fluffily final theodolites. furiously unusual d 14433|1082|O|203663.86|1996-05-13|2-HIGH|Clerk#000000011|0|es. furiously final deposits wake b 14434|1015|F|171433.55|1995-02-16|4-NOT SPECIFIED|Clerk#000000769|0|he unusual pinto beans. special realms cajole. quietly blith 14435|269|F|190900.78|1992-12-02|4-NOT SPECIFIED|Clerk#000000308|0|quests nag. final platelets haggle among the st 14436|658|F|231251.51|1994-03-13|5-LOW|Clerk#000000906|0|regular requests run furiously. unusual, 14437|721|F|285484.64|1994-05-15|1-URGENT|Clerk#000000348|0|refully even excuses alongside of the packages are busily final 14438|1117|O|107298.91|1995-08-23|3-MEDIUM|Clerk#000000059|0| the silent ideas wake after the express requests. unusual, final inst 14439|523|F|4033.74|1992-08-31|2-HIGH|Clerk#000000944|0|riously even courts. ev 14464|1429|O|43902.48|1998-01-23|5-LOW|Clerk#000000197|0| blithely. special deposits 14465|617|F|108913.02|1994-06-12|2-HIGH|Clerk#000000816|0|ndencies nag against the furiously special pin 14466|353|O|19731.95|1996-05-22|3-MEDIUM|Clerk#000000795|0|lithely after the slyly special deposits-- quickly regular gifts acros 14467|1126|O|160773.46|1996-05-31|4-NOT SPECIFIED|Clerk#000000927|0|regular, brave foxes 14468|716|O|60718.43|1998-07-13|3-MEDIUM|Clerk#000000164|0|g pinto beans after the special accounts sleep carefully after the regula 14469|1076|O|82731.79|1997-02-01|2-HIGH|Clerk#000000837|0|ep regular, final accounts? fluffily pending the 14470|949|F|140145.86|1995-01-25|3-MEDIUM|Clerk#000000018|0| asymptotes cajole carefully final a 14471|94|P|324194.82|1995-05-27|4-NOT SPECIFIED|Clerk#000001000|0|l pinto beans ought to nag carefully carefully ironic foxes. 14496|1153|O|206156.59|1996-04-11|3-MEDIUM|Clerk#000000350|0|g theodolites eat c 14497|1492|F|130424.80|1992-05-21|3-MEDIUM|Clerk#000000053|0|symptotes. slyly fluffy excuses ought to wake according to the slyly even de 14498|115|O|172949.99|1996-10-05|3-MEDIUM|Clerk#000000709|0| deposits! quickly pend 14499|361|F|306903.43|1993-11-15|5-LOW|Clerk#000000358|0|packages. carefully final ide 14500|1222|O|100773.58|1995-06-25|4-NOT SPECIFIED|Clerk#000000451|0| slyly regular ideas haggle slyly unusual packages. quickly unusual pinto 14501|607|O|83186.27|1997-08-31|2-HIGH|Clerk#000000826|0|e ideas. ironic deposits sleep according to the blithely ironi 14502|799|O|87836.22|1996-07-21|2-HIGH|Clerk#000000875|0| furiously ironic deposits cajole around the carefully silent accounts. expre 14503|671|F|198098.70|1995-02-06|4-NOT SPECIFIED|Clerk#000000519|0|ely regular depths haggle carefull 14528|334|F|155412.29|1994-06-16|1-URGENT|Clerk#000000568|0|gle instead of the carefully pending pinto beans. express, ex 14529|1372|F|193341.12|1993-09-21|5-LOW|Clerk#000000521|0|arefully unusual packages haggle carefully slyly final pinto beans. exp 14530|133|F|240313.77|1994-04-07|2-HIGH|Clerk#000000444|0|eodolites may wake final requests. furious 14531|1331|O|222583.58|1997-06-11|2-HIGH|Clerk#000000003|0|across the blithely even instructions. carefull 14532|1252|O|15069.98|1996-04-17|5-LOW|Clerk#000000906|0|ep furiously according to the quickly final deposits. reg 14533|43|O|161212.58|1996-05-06|4-NOT SPECIFIED|Clerk#000000394|0|ans alongside of the carefully ironic re 14534|1414|F|248752.30|1993-07-19|3-MEDIUM|Clerk#000000324|0|furiously. silent foxes boost finally. dependencies among the furiously fi 14535|1306|F|95671.80|1993-10-02|4-NOT SPECIFIED|Clerk#000000983|0|thely unusual accounts are fluffi 14560|1258|O|33048.76|1997-12-15|3-MEDIUM|Clerk#000000587|0|ckages grow furiously-- carefully final foxes according 14561|697|O|274551.87|1998-06-29|3-MEDIUM|Clerk#000000840|0|onic accounts. even, express excuses cajole carefully. bl 14562|973|O|150256.16|1996-07-20|5-LOW|Clerk#000000189|0|ve the blithely even dependencies-- quickly busy realms wake quickly. permane 14563|568|O|236613.87|1995-12-03|1-URGENT|Clerk#000000904|0|quests. quickly even theodolites nag blit 14564|520|F|1358.25|1993-05-22|4-NOT SPECIFIED|Clerk#000000799|0|carefully even deposits. furiously express acco 14565|1301|O|50022.91|1997-11-23|1-URGENT|Clerk#000000409|0| special foxes. blithely ironic ideas sleep carefully quietly final instruct 14566|1480|O|221337.42|1996-10-25|3-MEDIUM|Clerk#000000118|0|ns? slyly regular theodolites haggle quickly. fluffily bol 14567|1060|O|176372.95|1996-10-18|1-URGENT|Clerk#000000527|0|efully blithe excuses. slyly regular ideas promise. escapad 14592|926|O|93588.02|1995-07-06|3-MEDIUM|Clerk#000000402|0|ts affix alongside of the blithely ironic pinto beans. 14593|184|O|61820.82|1998-01-23|1-URGENT|Clerk#000000880|0| dependencies. slyly 14594|143|F|16529.84|1993-09-07|3-MEDIUM|Clerk#000000022|0| across the carefull 14595|202|O|32654.01|1997-12-03|3-MEDIUM|Clerk#000000854|0|kages. asymptotes dazzle. final, bold pinto be 14596|1198|F|167802.21|1994-01-11|1-URGENT|Clerk#000000754|0| even accounts above the furiously brave dep 14597|1192|F|293673.06|1993-08-31|1-URGENT|Clerk#000000768|0|haggle ironic, quick accounts. quickly special foxes nag slyly above the furi 14598|898|O|171449.52|1998-02-10|4-NOT SPECIFIED|Clerk#000000229|0|slyly even deposits are carefully. final, pe 14599|814|F|13810.80|1992-05-09|2-HIGH|Clerk#000000704|0|ke final theodolites 14624|1177|O|210247.47|1998-04-17|4-NOT SPECIFIED|Clerk#000000014|0|heodolites. quickly regular theodolites haggle ag 14625|1075|F|2059.98|1992-02-12|1-URGENT|Clerk#000000491|0|instructions play blithely? silent excuses snooze quick 14626|1453|O|118335.57|1997-08-12|3-MEDIUM|Clerk#000000450|0|egular platelets are. even requ 14627|679|F|212366.75|1994-02-05|3-MEDIUM|Clerk#000000520|0|quickly according to the b 14628|668|F|75317.77|1993-06-07|3-MEDIUM|Clerk#000000076|0|accounts hang furiously. furiously regular theodolites 14629|1469|F|78503.51|1993-01-08|2-HIGH|Clerk#000000344|0|uests sleep quickly. furiously ironic accounts against the quickly ironic id 14630|277|F|189160.54|1992-10-17|5-LOW|Clerk#000000575|0| theodolites. carefully 14631|1291|O|34502.66|1997-04-14|2-HIGH|Clerk#000000022|0|ly above the pendin 14656|1|O|28599.83|1997-11-18|2-HIGH|Clerk#000000270|0|uests. blithely even platelet 14657|370|F|116160.53|1994-02-28|1-URGENT|Clerk#000000756|0|ly across the ironic, ironic instructions. bold ideas 14658|1381|F|48274.02|1994-04-07|1-URGENT|Clerk#000000175|0|kly regular requests? regular theod 14659|25|O|145504.68|1998-02-10|3-MEDIUM|Clerk#000000260|0|l, ironic attainment 14660|899|O|139267.14|1997-05-24|4-NOT SPECIFIED|Clerk#000000605|0|c pinto beans. fluff 14661|1468|F|38295.53|1993-01-25|4-NOT SPECIFIED|Clerk#000000429|0| believe. silent packages haggle express instructio 14662|1330|O|200128.12|1995-11-17|2-HIGH|Clerk#000000988|0|efully pending accounts about the bold 14663|592|F|42406.05|1992-11-17|1-URGENT|Clerk#000000596|0|ietly above the packages. regular frets haggle slyly blithely regular pinto b 14688|427|O|127535.78|1997-02-18|5-LOW|Clerk#000000822|0|unusual, bold deposits. furiously bold ideas cajole fluffily ironic theodo 14689|226|O|68912.05|1998-07-31|2-HIGH|Clerk#000000100|0|quickly regular realms are along the carefully spe 14690|935|O|41205.72|1997-05-24|1-URGENT|Clerk#000000342|0|ic packages affix sly 14691|1381|O|81185.28|1998-04-11|5-LOW|Clerk#000000211|0|ffily even instructions use blithely. careful 14692|478|O|53591.54|1996-02-27|5-LOW|Clerk#000000309|0|xpress deposits wake slyly after the deposits. slyly re 14693|67|F|246072.95|1995-01-10|1-URGENT|Clerk#000000039|0|to beans. Tiresias above the special, bold packages sleep 14694|379|O|318967.92|1998-07-28|1-URGENT|Clerk#000000742|0|nic pinto beans sleep blithely pending, unusual somas. blithely ironic id 14695|1406|F|205288.40|1992-09-06|2-HIGH|Clerk#000000475|0|. ideas boost carefully around the even, final instruc 14720|401|F|171435.70|1993-09-05|3-MEDIUM|Clerk#000000354|0|ongside of the quickly final excuses sleep quickly dolphins. dinos 14721|1087|O|166081.83|1997-05-05|4-NOT SPECIFIED|Clerk#000000701|0|instructions haggle slyly. 14722|742|O|264702.15|1997-05-28|5-LOW|Clerk#000000048|0|blithely bold requests ar 14723|1282|O|40317.37|1997-01-04|5-LOW|Clerk#000000822|0|ic deposits affix carefully above th 14724|1396|O|41941.26|1995-11-18|1-URGENT|Clerk#000000886|0|bold dependencies about the busy instructions haggle regular ins 14725|569|O|261801.45|1995-06-17|2-HIGH|Clerk#000000177|0|ng asymptotes. final, ironic accounts cajole after 14726|1279|F|93802.35|1992-01-09|5-LOW|Clerk#000000590|0| foxes. deposits cajole blithely even grouches. b 14727|316|F|102382.66|1992-07-20|3-MEDIUM|Clerk#000000383|0|structions. daringly even packages wake slyly final requests. c 14752|1051|F|31543.83|1994-01-30|5-LOW|Clerk#000000802|0|. carefully regular pinto beans grow idly abou 14753|1118|O|45387.36|1997-01-05|1-URGENT|Clerk#000000855|0|n deposits across the 14754|86|O|112289.25|1996-06-23|3-MEDIUM|Clerk#000000283|0|ns. quickly ironic packages sleep furiously fluffily unusual excuses. de 14755|592|F|358175.60|1993-01-06|2-HIGH|Clerk#000000867|0|egular requests sleep careful packages. quickly r 14756|328|F|132718.02|1994-09-01|5-LOW|Clerk#000000491|0|olites. ironic, final instructions pro 14757|1420|O|55954.21|1997-12-24|4-NOT SPECIFIED|Clerk#000000011|0|ggle furiously. carefully special packages are c 14758|1225|F|37812.49|1993-10-27|2-HIGH|Clerk#000000687|0|ages nag about the furio 14759|70|O|40915.46|1997-01-05|3-MEDIUM|Clerk#000000034|0|he dolphins. ruthlessly regular packages play carefully. f 14784|1036|F|188067.67|1992-03-15|3-MEDIUM|Clerk#000000479|0|lyly final theodoli 14785|1249|F|107683.78|1994-10-10|2-HIGH|Clerk#000000446|0| slyly about the quickly sp 14786|1255|O|34058.87|1997-04-08|2-HIGH|Clerk#000000656|0|e carefully special deposits can nag blithely express, express accounts 14787|578|O|135287.72|1998-07-19|5-LOW|Clerk#000000522|0|deas against the blithely r 14788|1192|O|51229.59|1997-10-23|1-URGENT|Clerk#000000647|0|e the slyly pending deposits. c 14789|451|F|214256.97|1993-11-30|4-NOT SPECIFIED|Clerk#000000616|0|st furiously about the ca 14790|613|O|270163.54|1996-08-21|2-HIGH|Clerk#000000347|0|p. regular deposits wake. final n 14791|289|F|86492.66|1993-02-15|3-MEDIUM|Clerk#000000770|0|ideas wake blithely regularly regular requests. s 14816|508|F|89977.38|1993-07-15|4-NOT SPECIFIED|Clerk#000000721|0| requests. slyly even requests haggle? unusual, regula 14817|235|F|138541.57|1992-07-28|3-MEDIUM|Clerk#000000963|0|arefully unusual dolphins. furiously final accounts above the slyly 14818|643|O|182026.46|1996-11-18|4-NOT SPECIFIED|Clerk#000000588|0|totes. bold, final requests are according to the deposits. quickly regular 14819|473|F|141776.24|1993-01-26|4-NOT SPECIFIED|Clerk#000000641|0| permanent deposits. blithely final warthogs x-ray blithely slyly ir 14820|1135|F|160021.51|1992-01-10|2-HIGH|Clerk#000000090|0|posits. regular pinto beans detect carefully at the final pinto beans. unu 14821|1435|O|322002.95|1998-06-12|2-HIGH|Clerk#000000630|0|n packages are furiously ironic ideas. d 14822|473|O|182443.15|1996-03-26|2-HIGH|Clerk#000000675|0|es. even, special request 14823|832|F|190065.85|1994-02-13|5-LOW|Clerk#000000844|0| slyly final accounts: packages integrate quickly along the packages 14848|256|O|115009.62|1996-10-17|5-LOW|Clerk#000000567|0|warthogs use furiously across the 14849|739|O|82597.02|1997-02-07|3-MEDIUM|Clerk#000000123|0| cajole fluffily against the final, reg 14850|1348|O|227401.85|1997-12-04|1-URGENT|Clerk#000000960|0| quickly regular braids. ironic, regular accounts are carefully 14851|175|F|229811.50|1992-05-12|4-NOT SPECIFIED|Clerk#000000075|0|. blithely regular requests wake blithely after the furiously final accou 14852|1060|O|34634.76|1995-12-25|3-MEDIUM|Clerk#000000907|0|ages haggle accounts. careful, ironic excuses sleep quickly dogged asymptotes 14853|458|O|152920.35|1996-09-10|5-LOW|Clerk#000000036|0|ding instructions about the quickly pending requests sleep quickly iro 14854|472|F|75626.15|1993-04-29|5-LOW|Clerk#000000872|0|ronic pinto beans? even deposits nag c 14855|836|F|141609.96|1993-05-01|4-NOT SPECIFIED|Clerk#000000655|0|nent accounts sleep dependencies. furiously regular ideas ca 14880|1210|O|209395.33|1997-11-23|4-NOT SPECIFIED|Clerk#000000545|0|equests above the foxes wake blithely about the packages: ironic ex 14881|23|O|126051.68|1996-10-20|2-HIGH|Clerk#000000826|0|ades past the fluffily ironic foxes m 14882|833|O|89650.26|1997-02-13|1-URGENT|Clerk#000000353|0|y against the slyly ironic accounts. blithely silent acc 14883|865|P|148947.38|1995-05-07|5-LOW|Clerk#000000543|0|detect blithely: carefully regular asymptotes caj 14884|1450|O|278225.35|1995-09-28|2-HIGH|Clerk#000000481|0| carefully ironic multipliers. furiously final deposits 14885|19|O|261263.74|1997-09-21|5-LOW|Clerk#000000462|0|p slyly regular excu 14886|337|F|84441.18|1994-06-23|3-MEDIUM|Clerk#000000757|0|slyly above the deposi 14887|245|O|124720.14|1996-03-25|2-HIGH|Clerk#000000176|0|he pinto beans boost slyly regular deposits. fi 14912|752|F|28125.10|1993-11-07|1-URGENT|Clerk#000000924|0| ideas cajole slyly around the ironic, bold asymptotes. 14913|1411|F|178192.17|1994-02-15|2-HIGH|Clerk#000000823|0|gainst the carefully final orbits hang furiously above t 14914|1199|O|191598.59|1998-02-08|1-URGENT|Clerk#000000536|0|e carefully carefully ironic requests: never final packag 14915|811|O|59911.87|1996-09-10|4-NOT SPECIFIED|Clerk#000000708|0| accounts are above the enticingly express pin 14916|380|F|72290.87|1994-02-28|5-LOW|Clerk#000000656|0|al requests solve slyly above the never express requests. furious 14917|1148|O|280961.18|1998-01-13|1-URGENT|Clerk#000000974|0|as would use fluffily after the 14918|884|O|68255.07|1997-06-01|5-LOW|Clerk#000000674|0|ets. bold, pending deposits sleep. foxes wake 14919|838|O|36633.03|1996-09-10|2-HIGH|Clerk#000000455|0|as. carefully final ideas cajole finally blithely express foxes. slow 14944|535|O|119586.69|1997-10-14|2-HIGH|Clerk#000000962|0|lly. even instructions against 14945|68|O|210519.05|1996-03-30|1-URGENT|Clerk#000000467|0|nts? fluffily bold grouches after 14946|580|O|100402.47|1996-11-12|1-URGENT|Clerk#000000116|0|ffily bold dependencies wake. furiously regular instructions aro 14947|580|O|100402.47|1996-11-12|1-URGENT|Clerk#000000116|0|ffily bold dependencies wake. furiously regular instructions aro ================================================ FILE: src/test/regress/data/other_types.csv ================================================ f,\xdeadbeef,$1.00,192.168.1.2,10101,a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,"{""key"": ""value""}" t,\xcdb0,$1.50,127.0.0.1,"",a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,[] ================================================ FILE: src/test/regress/data/part.data ================================================ 1|goldenrod lavender spring chocolate lace|Manufacturer#1|Brand#13|PROMO BURNISHED COPPER|7|JUMBO PKG|901.00|ly. slyly ironi 2|blush thistle blue yellow saddle|Manufacturer#1|Brand#13|LARGE BRUSHED BRASS|1|LG CASE|902.00|lar accounts amo 3|spring green yellow purple cornsilk|Manufacturer#4|Brand#42|STANDARD POLISHED BRASS|21|WRAP CASE|903.00|egular deposits hag 4|cornflower chocolate smoke green pink|Manufacturer#3|Brand#34|SMALL PLATED BRASS|14|MED DRUM|904.00|p furiously r 5|forest brown coral puff cream|Manufacturer#3|Brand#32|STANDARD POLISHED TIN|15|SM PKG|905.00| wake carefully 6|bisque cornflower lawn forest magenta|Manufacturer#2|Brand#24|PROMO PLATED STEEL|4|MED BAG|906.00|sual a 7|moccasin green thistle khaki floral|Manufacturer#1|Brand#11|SMALL PLATED COPPER|45|SM BAG|907.00|lyly. ex 8|misty lace thistle snow royal|Manufacturer#4|Brand#44|PROMO BURNISHED TIN|41|LG DRUM|908.00|eposi 9|thistle dim navajo dark gainsboro|Manufacturer#4|Brand#43|SMALL BURNISHED STEEL|12|WRAP CASE|909.00|ironic foxe 10|linen pink saddle puff powder|Manufacturer#5|Brand#54|LARGE BURNISHED STEEL|44|LG CAN|910.01|ithely final deposit 11|spring maroon seashell almond orchid|Manufacturer#2|Brand#25|STANDARD BURNISHED NICKEL|43|WRAP BOX|911.01|ng gr 12|cornflower wheat orange maroon ghost|Manufacturer#3|Brand#33|MEDIUM ANODIZED STEEL|25|JUMBO CASE|912.01| quickly 13|ghost olive orange rosy thistle|Manufacturer#5|Brand#55|MEDIUM BURNISHED NICKEL|1|JUMBO PACK|913.01|osits. 14|khaki seashell rose cornsilk navajo|Manufacturer#1|Brand#13|SMALL POLISHED STEEL|28|JUMBO BOX|914.01|kages c 15|blanched honeydew sky turquoise medium|Manufacturer#1|Brand#15|LARGE ANODIZED BRASS|45|LG CASE|915.01|usual ac 16|deep sky turquoise drab peach|Manufacturer#3|Brand#32|PROMO PLATED TIN|2|MED PACK|916.01|unts a 17|indian navy coral pink deep|Manufacturer#4|Brand#43|ECONOMY BRUSHED STEEL|16|LG BOX|917.01| regular accounts 18|turquoise indian lemon lavender misty|Manufacturer#1|Brand#11|SMALL BURNISHED STEEL|42|JUMBO PACK|918.01|s cajole slyly a 19|chocolate navy tan deep brown|Manufacturer#2|Brand#23|SMALL ANODIZED NICKEL|33|WRAP BOX|919.01| pending acc 20|ivory navy honeydew sandy midnight|Manufacturer#1|Brand#12|LARGE POLISHED NICKEL|48|MED BAG|920.02|are across the asympt 21|lemon floral azure frosted lime|Manufacturer#3|Brand#33|SMALL BURNISHED TIN|31|MED BAG|921.02|ss packages. pendin 22|medium forest blue ghost black|Manufacturer#4|Brand#43|PROMO POLISHED BRASS|19|LG DRUM|922.02| even p 23|coral lavender seashell rosy burlywood|Manufacturer#3|Brand#35|MEDIUM BURNISHED TIN|42|JUMBO JAR|923.02|nic, fina 24|seashell coral metallic midnight floral|Manufacturer#5|Brand#52|MEDIUM PLATED STEEL|20|MED CASE|924.02| final the 25|aquamarine steel firebrick light turquoise|Manufacturer#5|Brand#55|STANDARD BRUSHED COPPER|3|JUMBO BAG|925.02|requests wake 26|beige frosted moccasin chocolate snow|Manufacturer#3|Brand#32|SMALL BRUSHED STEEL|32|SM CASE|926.02| instructions i 27|saddle puff beige linen yellow|Manufacturer#1|Brand#14|LARGE ANODIZED TIN|20|MED PKG|927.02|s wake. ir 28|navajo yellow drab white misty|Manufacturer#4|Brand#44|SMALL PLATED COPPER|19|JUMBO PKG|928.02|x-ray pending, iron 29|lemon sky grey salmon orchid|Manufacturer#3|Brand#33|PROMO PLATED COPPER|7|LG DRUM|929.02| carefully fluffi 30|cream misty steel spring medium|Manufacturer#4|Brand#42|PROMO ANODIZED TIN|17|LG BOX|930.03|carefully bus 31|slate seashell steel medium moccasin|Manufacturer#5|Brand#53|STANDARD BRUSHED TIN|10|LG BAG|931.03|uriously s 32|sandy wheat coral spring burnished|Manufacturer#4|Brand#42|ECONOMY PLATED BRASS|31|LG CASE|932.03|urts. carefully fin 33|spring bisque salmon slate pink|Manufacturer#2|Brand#22|ECONOMY PLATED NICKEL|16|LG PKG|933.03|ly eve 34|khaki steel rose ghost salmon|Manufacturer#1|Brand#13|LARGE BRUSHED STEEL|8|JUMBO BOX|934.03|riously ironic 35|green blush tomato burlywood seashell|Manufacturer#4|Brand#43|MEDIUM ANODIZED BRASS|14|JUMBO PACK|935.03|e carefully furi 36|chiffon tan forest moccasin dark|Manufacturer#2|Brand#25|SMALL BURNISHED COPPER|3|JUMBO CAN|936.03|olites o 37|royal coral orange burnished navajo|Manufacturer#4|Brand#45|LARGE POLISHED TIN|48|JUMBO BOX|937.03|silent 38|seashell papaya white mint brown|Manufacturer#4|Brand#43|ECONOMY ANODIZED BRASS|11|SM JAR|938.03|structions inte 39|rose medium floral salmon powder|Manufacturer#5|Brand#53|SMALL POLISHED TIN|43|JUMBO JAR|939.03|se slowly above the fl 40|lemon midnight metallic sienna steel|Manufacturer#2|Brand#25|ECONOMY BURNISHED COPPER|27|SM CASE|940.04|! blithely specia 41|burlywood goldenrod pink peru sienna|Manufacturer#2|Brand#23|ECONOMY ANODIZED TIN|7|WRAP JAR|941.04|uriously. furiously cl 42|midnight turquoise lawn beige thistle|Manufacturer#5|Brand#52|MEDIUM BURNISHED TIN|45|LG BOX|942.04|the slow 43|medium lace midnight royal chartreuse|Manufacturer#4|Brand#44|PROMO POLISHED STEEL|5|WRAP CASE|943.04|e slyly along the ir 44|saddle cream wheat lemon burnished|Manufacturer#4|Brand#45|MEDIUM PLATED TIN|48|SM PACK|944.04|pinto beans. carefully 45|lawn peru ghost khaki maroon|Manufacturer#4|Brand#43|SMALL BRUSHED NICKEL|9|WRAP BAG|945.04|nts bo 46|honeydew turquoise aquamarine spring tan|Manufacturer#1|Brand#11|STANDARD POLISHED TIN|45|WRAP CASE|946.04|the blithely unusual 47|honeydew red azure magenta brown|Manufacturer#4|Brand#45|LARGE BURNISHED BRASS|14|JUMBO PACK|947.04| even plate 48|slate thistle cornsilk pale forest|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|27|JUMBO CASE|948.04|ng to the depo 49|light firebrick cyan puff blue|Manufacturer#2|Brand#24|SMALL BURNISHED TIN|31|MED DRUM|949.04|ar pack 50|linen blanched tomato slate medium|Manufacturer#3|Brand#33|LARGE ANODIZED TIN|25|WRAP PKG|950.05|kages m 51|lime frosted indian dodger linen|Manufacturer#4|Brand#45|ECONOMY BURNISHED NICKEL|34|JUMBO PACK|951.05|n foxes 52|lemon midnight lace sky deep|Manufacturer#3|Brand#35|STANDARD BURNISHED TIN|25|WRAP CASE|952.05| final deposits. fu 53|bisque rose cornsilk seashell purple|Manufacturer#2|Brand#23|ECONOMY BURNISHED NICKEL|32|MED BAG|953.05|mptot 54|blanched mint yellow papaya cyan|Manufacturer#2|Brand#21|LARGE BURNISHED COPPER|19|WRAP CASE|954.05|e blithely 55|sky cream deep tomato rosy|Manufacturer#2|Brand#23|ECONOMY BRUSHED COPPER|9|MED BAG|955.05|ly final pac 56|antique beige brown deep dodger|Manufacturer#1|Brand#12|MEDIUM PLATED STEEL|20|WRAP DRUM|956.05|ts. blithel 57|purple blue light sienna deep|Manufacturer#3|Brand#32|MEDIUM BURNISHED BRASS|49|MED PKG|957.05|lly abov 58|linen hot cornsilk drab bisque|Manufacturer#5|Brand#53|STANDARD POLISHED TIN|44|LG PACK|958.05| fluffily blithely reg 59|misty brown medium mint salmon|Manufacturer#5|Brand#53|MEDIUM POLISHED TIN|2|LG BAG|959.05|regular exc 60|snow spring sandy olive tomato|Manufacturer#1|Brand#11|LARGE POLISHED COPPER|27|JUMBO CASE|960.06| integ 61|light tan linen tomato peach|Manufacturer#5|Brand#54|SMALL BURNISHED NICKEL|18|WRAP DRUM|961.06|es. blithely en 62|tan cornsilk spring grey chocolate|Manufacturer#3|Brand#35|STANDARD BRUSHED BRASS|39|JUMBO BOX|962.06|ckly across the carefu 63|burnished puff coral light papaya|Manufacturer#3|Brand#32|STANDARD BURNISHED NICKEL|10|JUMBO CAN|963.06| quickly 64|aquamarine coral lemon ivory gainsboro|Manufacturer#2|Brand#21|MEDIUM ANODIZED BRASS|1|JUMBO CAN|964.06|efully regular pi 65|slate drab medium puff gainsboro|Manufacturer#5|Brand#53|MEDIUM BRUSHED COPPER|3|MED CAN|965.06|posits after the quic 66|cornflower pale almond lemon linen|Manufacturer#3|Brand#35|PROMO ANODIZED NICKEL|46|SM CASE|966.06|haggle blithely iro 67|slate salmon rose spring seashell|Manufacturer#2|Brand#21|SMALL BRUSHED TIN|31|WRAP DRUM|967.06| regular, p 68|bisque ivory mint purple almond|Manufacturer#1|Brand#11|PROMO ANODIZED STEEL|10|WRAP BOX|968.06|eposits shall h 69|lace burnished rosy antique metallic|Manufacturer#5|Brand#52|MEDIUM POLISHED BRASS|2|SM BOX|969.06|ely final depo 70|violet seashell firebrick dark navajo|Manufacturer#1|Brand#11|STANDARD BRUSHED STEEL|42|LG PACK|970.07|inal gifts. sl 71|violet firebrick cream peru white|Manufacturer#3|Brand#33|STANDARD PLATED BRASS|26|WRAP DRUM|971.07| packages alongside 72|hot spring yellow azure dodger|Manufacturer#2|Brand#23|STANDARD ANODIZED TIN|25|JUMBO PACK|972.07|efully final the 73|cream moccasin royal dim chiffon|Manufacturer#2|Brand#21|SMALL BRUSHED COPPER|35|WRAP DRUM|973.07|ts haggl 74|frosted grey aquamarine thistle papaya|Manufacturer#5|Brand#55|ECONOMY ANODIZED BRASS|25|JUMBO CASE|974.07|ent foxes 75|aquamarine maroon wheat salmon metallic|Manufacturer#3|Brand#35|SMALL BURNISHED NICKEL|39|SM JAR|975.07|s sleep furiou 76|rosy light lime puff sandy|Manufacturer#3|Brand#34|MEDIUM BRUSHED COPPER|9|SM PKG|976.07|n accounts sleep qu 77|mint bisque chiffon snow firebrick|Manufacturer#5|Brand#52|STANDARD BRUSHED COPPER|13|MED PKG|977.07|uests. 78|blush forest slate seashell puff|Manufacturer#1|Brand#14|ECONOMY POLISHED STEEL|24|LG JAR|978.07|icing deposits wake 79|gainsboro pink grey tan almond|Manufacturer#4|Brand#45|PROMO ANODIZED BRASS|22|JUMBO BAG|979.07| foxes are slyly regu 80|tomato chartreuse coral turquoise linen|Manufacturer#4|Brand#44|PROMO PLATED BRASS|28|MED CAN|980.08|unusual dependencies i 81|misty sandy cornsilk dodger blush|Manufacturer#5|Brand#53|ECONOMY BRUSHED TIN|21|MED BAG|981.08|ove the furiou 82|khaki tomato purple almond tan|Manufacturer#1|Brand#15|ECONOMY POLISHED TIN|12|WRAP BOX|982.08|ial requests haggle 83|blush green dim lawn peru|Manufacturer#1|Brand#12|PROMO BURNISHED NICKEL|47|SM CAN|983.08|ly regul 84|salmon floral cream rose dark|Manufacturer#4|Brand#45|SMALL ANODIZED NICKEL|26|JUMBO PACK|984.08|ideas nag 85|dim deep aquamarine smoke pale|Manufacturer#5|Brand#55|PROMO ANODIZED NICKEL|16|LG BAG|985.08| silent 86|green blanched firebrick dim cream|Manufacturer#4|Brand#44|STANDARD PLATED TIN|37|LG CASE|986.08| daring sheaves 87|purple lace seashell antique orange|Manufacturer#4|Brand#41|LARGE PLATED STEEL|41|WRAP PACK|987.08|yly final 88|lime orange bisque chartreuse lemon|Manufacturer#4|Brand#44|PROMO PLATED COPPER|16|SM CASE|988.08|e regular packages. 89|ghost lace lemon sienna saddle|Manufacturer#5|Brand#53|STANDARD BURNISHED STEEL|7|MED JAR|989.08|y final pinto 90|hot rosy violet plum pale|Manufacturer#5|Brand#51|ECONOMY POLISHED STEEL|49|JUMBO CAN|990.09|caref 91|misty bisque lavender spring turquoise|Manufacturer#2|Brand#21|STANDARD BRUSHED TIN|32|JUMBO PKG|991.09|counts dete 92|blush magenta ghost tomato rose|Manufacturer#2|Brand#22|STANDARD ANODIZED TIN|35|JUMBO PKG|992.09|he ironic accounts. sp 93|pale yellow cornsilk dodger moccasin|Manufacturer#2|Brand#24|LARGE ANODIZED TIN|2|WRAP DRUM|993.09| platel 94|blanched pink frosted mint snow|Manufacturer#3|Brand#35|STANDARD POLISHED BRASS|32|SM BOX|994.09|s accounts cajo 95|dodger beige wheat orchid navy|Manufacturer#3|Brand#33|LARGE BRUSHED TIN|36|WRAP DRUM|995.09| final pinto beans 96|chocolate light firebrick rose indian|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|32|SM CASE|996.09|ng to the bli 97|coral dodger beige black chartreuse|Manufacturer#3|Brand#33|MEDIUM POLISHED BRASS|49|WRAP CAN|997.09|ss excuses sleep am 98|frosted peru chiffon yellow aquamarine|Manufacturer#5|Brand#54|STANDARD ANODIZED BRASS|22|MED JAR|998.09|e the q 99|mint grey purple sienna metallic|Manufacturer#2|Brand#21|SMALL BURNISHED STEEL|11|JUMBO PKG|999.09|press 100|cyan orchid indian cornflower saddle|Manufacturer#3|Brand#33|ECONOMY ANODIZED TIN|4|LG BAG|1000.10|of the steal 101|powder deep lavender violet gainsboro|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|26|JUMBO JAR|1001.10|ly even, 102|papaya maroon blush powder sky|Manufacturer#3|Brand#31|MEDIUM BURNISHED BRASS|17|SM DRUM|1002.10|ular packa 103|navy sky spring orchid forest|Manufacturer#2|Brand#25|MEDIUM PLATED BRASS|45|WRAP DRUM|1003.10|e blithely blith 104|plum cyan cornflower midnight royal|Manufacturer#1|Brand#13|MEDIUM ANODIZED STEEL|36|JUMBO BAG|1004.10|ites sleep quickly 105|dodger slate pale mint navajo|Manufacturer#1|Brand#15|SMALL POLISHED COPPER|27|LG DRUM|1005.10|odolites was 106|cornsilk bisque seashell lemon frosted|Manufacturer#3|Brand#31|MEDIUM PLATED BRASS|28|WRAP DRUM|1006.10|unts maintain 107|violet honeydew bisque sienna orchid|Manufacturer#5|Brand#53|SMALL BURNISHED TIN|12|MED BOX|1007.10|slyly special depos 108|bisque peach magenta tomato yellow|Manufacturer#1|Brand#12|PROMO PLATED NICKEL|41|MED PKG|1008.10|after the carefully 109|lemon black indian cornflower pale|Manufacturer#3|Brand#33|ECONOMY POLISHED TIN|11|LG PACK|1009.10|instruction 110|firebrick navy rose beige black|Manufacturer#3|Brand#33|STANDARD BURNISHED COPPER|46|LG DRUM|1010.11|t quickly a 111|orange cornflower mint snow peach|Manufacturer#5|Brand#54|LARGE BRUSHED COPPER|28|JUMBO JAR|1011.11|kly bold epitaphs 112|hot aquamarine tomato lace indian|Manufacturer#4|Brand#43|PROMO BRUSHED STEEL|42|JUMBO CAN|1012.11|the express, 113|almond seashell azure blanched light|Manufacturer#3|Brand#31|PROMO POLISHED TIN|23|LG CAN|1013.11|finally even 114|pink black blanched lace chartreuse|Manufacturer#5|Brand#51|MEDIUM POLISHED NICKEL|41|MED PACK|1014.11|ully final foxes. pint 115|spring chiffon cream orchid dodger|Manufacturer#4|Brand#45|STANDARD POLISHED STEEL|24|MED CAN|1015.11|counts nag! caref 116|goldenrod black slate forest red|Manufacturer#5|Brand#53|PROMO POLISHED NICKEL|33|SM PACK|1016.11|usly final courts 117|tomato honeydew pale red yellow|Manufacturer#1|Brand#14|SMALL BRUSHED TIN|25|LG BAG|1017.11|ages acc 118|ghost plum brown coral cornsilk|Manufacturer#2|Brand#25|PROMO ANODIZED TIN|31|MED PACK|1018.11|ly ironic pinto 119|olive metallic slate peach green|Manufacturer#4|Brand#43|LARGE POLISHED STEEL|30|WRAP CASE|1019.11|out the quickly r 120|pink powder mint moccasin navajo|Manufacturer#1|Brand#14|SMALL ANODIZED NICKEL|45|WRAP JAR|1020.12|lly a 121|bisque royal goldenrod medium thistle|Manufacturer#1|Brand#14|ECONOMY BRUSHED COPPER|13|SM PKG|1021.12|deposi 122|gainsboro royal forest dark lace|Manufacturer#2|Brand#21|MEDIUM ANODIZED TIN|8|LG DRUM|1022.12|sts c 123|deep dim peach light beige|Manufacturer#1|Brand#12|SMALL BURNISHED TIN|31|JUMBO PKG|1023.12|ray regula 124|wheat blush forest metallic navajo|Manufacturer#3|Brand#32|PROMO ANODIZED STEEL|1|LG BOX|1024.12|g the expr 125|mint ivory saddle peach midnight|Manufacturer#1|Brand#12|STANDARD BRUSHED BRASS|17|WRAP BAG|1025.12|kages against 126|burnished black blue metallic orchid|Manufacturer#4|Brand#45|MEDIUM BRUSHED NICKEL|4|LG BAG|1026.12|es sleep al 127|royal coral orchid spring sky|Manufacturer#5|Brand#52|SMALL BURNISHED NICKEL|14|LG JAR|1027.12|lithely expr 128|dark burlywood burnished snow sky|Manufacturer#2|Brand#22|PROMO PLATED TIN|5|SM BAG|1028.12|e of the furiously ex 129|grey spring chiffon thistle lime|Manufacturer#1|Brand#15|LARGE POLISHED TIN|20|SM JAR|1029.12| careful 130|gainsboro powder cyan pale rosy|Manufacturer#2|Brand#23|SMALL PLATED NICKEL|26|LG BOX|1030.13|ake slyly 131|tomato moccasin cyan brown goldenrod|Manufacturer#5|Brand#52|STANDARD ANODIZED BRASS|43|MED DRUM|1031.13|nts wake dar 132|seashell papaya tomato lime hot|Manufacturer#4|Brand#45|STANDARD BURNISHED BRASS|2|WRAP DRUM|1032.13|ckly expre 133|firebrick black dodger pink salmon|Manufacturer#1|Brand#13|SMALL BRUSHED NICKEL|19|LG PKG|1033.13| final pinto beans 134|steel beige mint maroon indian|Manufacturer#4|Brand#42|SMALL POLISHED STEEL|35|SM PKG|1034.13|es. bold pa 135|thistle chocolate ghost gainsboro peru|Manufacturer#2|Brand#21|MEDIUM BURNISHED STEEL|24|JUMBO CASE|1035.13|l frets 136|cornsilk maroon blanched thistle rosy|Manufacturer#2|Brand#22|SMALL PLATED STEEL|2|WRAP BAG|1036.13|kages print carefully 137|cornsilk drab ghost sandy royal|Manufacturer#3|Brand#31|ECONOMY PLATED STEEL|25|MED PACK|1037.13|the t 138|dark aquamarine tomato medium puff|Manufacturer#1|Brand#13|ECONOMY BURNISHED COPPER|42|JUMBO DRUM|1038.13|ts solve acro 139|floral steel burlywood navy cream|Manufacturer#3|Brand#32|MEDIUM BRUSHED STEEL|7|SM BOX|1039.13|ter t 140|aquamarine lavender maroon slate hot|Manufacturer#5|Brand#53|STANDARD PLATED STEEL|45|SM BOX|1040.14|oss the carefu 141|honeydew magenta tomato spring medium|Manufacturer#3|Brand#35|STANDARD ANODIZED STEEL|23|SM PKG|1041.14|ans nag furiously pen 142|chartreuse linen grey slate saddle|Manufacturer#5|Brand#55|STANDARD ANODIZED BRASS|36|MED JAR|1042.14|he accounts. pac 143|bisque dodger blanched steel maroon|Manufacturer#3|Brand#34|ECONOMY PLATED TIN|44|MED BAG|1043.14|nts across the 144|hot midnight orchid dim steel|Manufacturer#1|Brand#14|SMALL ANODIZED TIN|26|SM BOX|1044.14|owly 145|navajo lavender chocolate deep hot|Manufacturer#5|Brand#53|PROMO BRUSHED COPPER|24|SM BAG|1045.14|es wake furiously blit 146|azure smoke mint cream burlywood|Manufacturer#3|Brand#34|STANDARD BRUSHED COPPER|11|WRAP PACK|1046.14|unts cajole 147|honeydew orange dodger linen lace|Manufacturer#1|Brand#11|MEDIUM PLATED COPPER|29|JUMBO PKG|1047.14|wake never bold 148|yellow white ghost lavender salmon|Manufacturer#3|Brand#31|STANDARD PLATED STEEL|20|SM BOX|1048.14|platelets wake fu 149|tan thistle frosted indian lawn|Manufacturer#2|Brand#24|MEDIUM BURNISHED NICKEL|6|MED PKG|1049.14|leep requests. dog 150|pale rose navajo firebrick aquamarine|Manufacturer#3|Brand#35|LARGE BRUSHED TIN|21|SM BAG|1050.15|ironic foxes 151|chartreuse linen violet ghost thistle|Manufacturer#3|Brand#34|LARGE PLATED BRASS|45|MED CAN|1051.15|ccounts nag i 152|white sky antique tomato chartreuse|Manufacturer#5|Brand#53|MEDIUM POLISHED STEEL|48|MED CASE|1052.15|thely regular t 153|linen frosted slate coral peru|Manufacturer#1|Brand#11|STANDARD PLATED TIN|20|MED BAG|1053.15|thlessly. silen 154|peru moccasin peach pale spring|Manufacturer#1|Brand#11|ECONOMY ANODIZED TIN|1|JUMBO BAG|1054.15|posits 155|puff yellow cyan tomato purple|Manufacturer#2|Brand#21|SMALL BRUSHED NICKEL|28|WRAP CASE|1055.15|lly ironic, r 156|almond ghost powder blush forest|Manufacturer#4|Brand#43|SMALL POLISHED NICKEL|2|LG PKG|1056.15| pinto beans. eve 157|navajo linen coral brown forest|Manufacturer#1|Brand#11|ECONOMY ANODIZED STEEL|26|JUMBO PACK|1057.15|ial courts. ru 158|magenta light misty navy honeydew|Manufacturer#4|Brand#45|MEDIUM BURNISHED COPPER|47|LG JAR|1058.15| ideas detect slyl 159|white orange antique beige aquamarine|Manufacturer#4|Brand#43|SMALL ANODIZED BRASS|46|SM BAG|1059.15| ironic requests-- pe 160|frosted cornflower khaki salmon metallic|Manufacturer#5|Brand#55|STANDARD POLISHED COPPER|47|JUMBO CAN|1060.16|nts are carefully 161|metallic khaki navy forest cyan|Manufacturer#2|Brand#22|STANDARD PLATED TIN|17|SM PACK|1061.16|r the bl 162|burlywood cornflower aquamarine misty snow|Manufacturer#3|Brand#33|MEDIUM ANODIZED COPPER|35|JUMBO PACK|1062.16|e slyly around th 163|blush metallic maroon lawn forest|Manufacturer#2|Brand#21|ECONOMY PLATED TIN|34|WRAP DRUM|1063.16|nly s 164|orange cyan magenta navajo indian|Manufacturer#2|Brand#23|LARGE PLATED BRASS|35|JUMBO BAG|1064.16|mong th 165|white dim cornflower sky seashell|Manufacturer#1|Brand#15|STANDARD PLATED STEEL|24|SM CAN|1065.16| carefully fin 166|linen bisque tomato gainsboro goldenrod|Manufacturer#5|Brand#52|LARGE POLISHED COPPER|4|MED BAG|1066.16|ss the 167|almond floral grey dim sky|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|46|WRAP BOX|1067.16|ic ac 168|lace gainsboro burlywood smoke tomato|Manufacturer#1|Brand#13|SMALL BRUSHED COPPER|20|JUMBO DRUM|1068.16|ss package 169|bisque misty sky cornflower peach|Manufacturer#5|Brand#55|STANDARD POLISHED BRASS|10|JUMBO CASE|1069.16|lets alongside of 170|peru grey blanched goldenrod yellow|Manufacturer#3|Brand#33|LARGE POLISHED COPPER|28|LG DRUM|1070.17|yly s 171|beige violet black magenta chartreuse|Manufacturer#1|Brand#11|STANDARD BURNISHED COPPER|40|LG JAR|1071.17| the r 172|medium goldenrod linen sky coral|Manufacturer#5|Brand#53|PROMO PLATED NICKEL|28|MED CASE|1072.17|quick as 173|chartreuse seashell powder navy grey|Manufacturer#1|Brand#12|ECONOMY BURNISHED TIN|17|LG CASE|1073.17|sly bold excuses haggl 174|hot cornflower slate saddle pale|Manufacturer#1|Brand#15|ECONOMY BRUSHED COPPER|25|LG CASE|1074.17| accounts nag ab 175|magenta blue chartreuse tan green|Manufacturer#1|Brand#11|PROMO ANODIZED TIN|45|JUMBO JAR|1075.17|ole against the 176|pink drab ivory papaya grey|Manufacturer#2|Brand#24|SMALL ANODIZED STEEL|40|MED CAN|1076.17|blithely. ironic 177|indian turquoise purple green spring|Manufacturer#2|Brand#21|MEDIUM BRUSHED STEEL|42|LG BAG|1077.17|ermanently eve 178|lace blanched magenta yellow almond|Manufacturer#1|Brand#13|STANDARD POLISHED TIN|10|LG JAR|1078.17|regular instructions. 179|deep puff brown blue burlywood|Manufacturer#4|Brand#43|ECONOMY BRUSHED STEEL|20|LG JAR|1079.17|ely regul 180|seashell maroon lace burnished lavender|Manufacturer#3|Brand#33|STANDARD BURNISHED NICKEL|7|WRAP BAG|1080.18|oss the 181|antique plum smoke pink dodger|Manufacturer#2|Brand#24|MEDIUM PLATED STEEL|19|WRAP CAN|1081.18|al deposits 182|beige cyan burlywood chiffon light|Manufacturer#3|Brand#31|MEDIUM ANODIZED COPPER|11|JUMBO CAN|1082.18|bits are 183|ivory white burnished papaya cornflower|Manufacturer#5|Brand#52|PROMO POLISHED STEEL|35|LG PKG|1083.18|ly regular excus 184|ghost honeydew cyan lawn powder|Manufacturer#5|Brand#53|SMALL POLISHED TIN|42|LG BOX|1084.18|ding courts. idly iro 185|firebrick black ivory spring medium|Manufacturer#4|Brand#44|ECONOMY POLISHED TIN|4|WRAP BAG|1085.18|even foxe 186|grey purple chocolate turquoise plum|Manufacturer#2|Brand#23|ECONOMY BRUSHED TIN|15|JUMBO PKG|1086.18|ly reg 187|white red lace deep pale|Manufacturer#4|Brand#45|PROMO ANODIZED BRASS|45|MED CAN|1087.18|leep slyly s 188|moccasin steel rosy drab white|Manufacturer#5|Brand#54|ECONOMY ANODIZED BRASS|9|MED CAN|1088.18| above the silent p 189|dodger moccasin lemon purple thistle|Manufacturer#2|Brand#22|MEDIUM BRUSHED BRASS|13|WRAP DRUM|1089.18|en requests. sauternes 190|chartreuse goldenrod midnight cornflower blush|Manufacturer#5|Brand#53|LARGE BURNISHED NICKEL|23|WRAP BAG|1090.19| furiously even d 191|mint midnight puff forest peach|Manufacturer#3|Brand#31|MEDIUM POLISHED BRASS|36|WRAP BOX|1091.19| asymptote 192|thistle puff pink cream orange|Manufacturer#3|Brand#34|STANDARD BRUSHED COPPER|17|MED BAG|1092.19|uickly regular, expr 193|turquoise lime royal metallic azure|Manufacturer#4|Brand#45|ECONOMY BURNISHED BRASS|31|SM PKG|1093.19|final ideas wake furi 194|brown black cream navy plum|Manufacturer#5|Brand#51|ECONOMY POLISHED STEEL|7|SM CAN|1094.19|y special accoun 195|bisque sienna hot goldenrod khaki|Manufacturer#4|Brand#41|STANDARD BRUSHED NICKEL|40|MED CASE|1095.19|oxes sleep care 196|pale peru linen hot maroon|Manufacturer#3|Brand#33|SMALL BURNISHED NICKEL|3|JUMBO JAR|1096.19|uickly special 197|lawn lemon khaki rosy blue|Manufacturer#5|Brand#52|SMALL ANODIZED COPPER|18|SM JAR|1097.19|lithely after the eve 198|orange cornflower indian aquamarine white|Manufacturer#4|Brand#41|PROMO BRUSHED NICKEL|43|SM PACK|1098.19|ackages? carefully re 199|ivory slate lavender tan royal|Manufacturer#3|Brand#31|ECONOMY PLATED STEEL|23|JUMBO DRUM|1099.19|ickly regul 200|peach cornsilk navy rosy red|Manufacturer#5|Brand#54|MEDIUM POLISHED BRASS|22|LG PKG|1100.20|furiously even depo 201|dodger white chiffon moccasin green|Manufacturer#4|Brand#42|SMALL POLISHED STEEL|18|JUMBO BAG|1101.20| courts sl 202|brown violet turquoise frosted navajo|Manufacturer#4|Brand#45|MEDIUM BRUSHED COPPER|49|MED DRUM|1102.20| use slyly c 203|beige cornflower gainsboro chiffon wheat|Manufacturer#5|Brand#55|STANDARD ANODIZED STEEL|42|MED BAG|1103.20|usual instructio 204|floral powder deep linen hot|Manufacturer#5|Brand#54|LARGE BURNISHED COPPER|22|SM JAR|1104.20|egular pl 205|navajo bisque gainsboro forest coral|Manufacturer#5|Brand#54|ECONOMY PLATED NICKEL|2|JUMBO DRUM|1105.20| blithely regular 206|chiffon coral wheat peru lavender|Manufacturer#2|Brand#25|LARGE ANODIZED NICKEL|23|JUMBO CAN|1106.20|ng theodolites 207|blush chocolate light coral midnight|Manufacturer#3|Brand#35|STANDARD PLATED COPPER|48|MED CAN|1107.20|lar, 208|medium pink metallic honeydew ghost|Manufacturer#3|Brand#35|LARGE PLATED COPPER|5|MED PACK|1108.20|l deposits wak 209|sky plum pink lavender firebrick|Manufacturer#1|Brand#11|ECONOMY PLATED STEEL|6|MED CAN|1109.20| haggle a 210|white sky mint chiffon blanched|Manufacturer#3|Brand#32|ECONOMY BURNISHED TIN|5|LG JAR|1110.21|ly until the flu 211|turquoise forest orchid cream blue|Manufacturer#3|Brand#32|MEDIUM ANODIZED NICKEL|43|LG PKG|1111.21|s cajole 212|light sandy white bisque burlywood|Manufacturer#5|Brand#53|PROMO BRUSHED BRASS|42|JUMBO BOX|1112.21|uffily fin 213|pale midnight light ghost saddle|Manufacturer#3|Brand#35|SMALL POLISHED COPPER|14|WRAP CAN|1113.21|der slyly accor 214|cornsilk tomato firebrick tan puff|Manufacturer#4|Brand#41|PROMO PLATED BRASS|30|LG BAG|1114.21|quests. regular, fi 215|lace pink black orange cornflower|Manufacturer#5|Brand#52|ECONOMY BURNISHED NICKEL|19|SM PACK|1115.21| the carefully i 216|tan azure slate white mint|Manufacturer#4|Brand#45|MEDIUM PLATED NICKEL|35|MED JAR|1116.21|gside of the unus 217|lemon thistle sandy royal peru|Manufacturer#5|Brand#52|PROMO PLATED COPPER|44|MED DRUM|1117.21|heodolites integrat 218|sandy khaki ghost cornflower metallic|Manufacturer#4|Brand#41|PROMO BURNISHED TIN|24|WRAP PACK|1118.21| furiously 219|cyan aquamarine red plum frosted|Manufacturer#4|Brand#45|PROMO ANODIZED BRASS|31|MED JAR|1119.21|riously ironic reque 220|lemon firebrick lime white wheat|Manufacturer#1|Brand#14|PROMO BRUSHED BRASS|25|MED CASE|1120.22|heodolites sleep. c 221|chartreuse azure lemon saddle dark|Manufacturer#5|Brand#51|PROMO BRUSHED COPPER|7|LG DRUM|1121.22|ld asymptotes sleep ca 222|aquamarine puff antique drab beige|Manufacturer#3|Brand#33|LARGE ANODIZED BRASS|35|LG CASE|1122.22|ross the ironic, un 223|rose salmon yellow turquoise grey|Manufacturer#3|Brand#33|LARGE ANODIZED STEEL|29|MED BAG|1123.22|ons. even depos 224|drab lavender moccasin almond purple|Manufacturer#1|Brand#13|STANDARD POLISHED NICKEL|46|WRAP BOX|1124.22|y special depos 225|cornsilk powder bisque chartreuse spring|Manufacturer#4|Brand#45|ECONOMY BRUSHED TIN|23|MED CASE|1125.22|ts ca 226|blush navy indian peru lemon|Manufacturer#3|Brand#33|PROMO ANODIZED STEEL|7|SM PKG|1126.22|ular packages. some 227|dim yellow aquamarine lavender peach|Manufacturer#4|Brand#44|SMALL BRUSHED TIN|21|LG PACK|1127.22| silent r 228|saddle lawn blue burlywood white|Manufacturer#4|Brand#44|ECONOMY PLATED STEEL|12|SM DRUM|1128.22|f the fluffily 229|orchid misty cornsilk chartreuse medium|Manufacturer#1|Brand#15|SMALL POLISHED STEEL|19|MED CAN|1129.22|ng to the quick 230|thistle navy lawn sky slate|Manufacturer#5|Brand#51|STANDARD PLATED STEEL|20|SM PKG|1130.23|fter the ironic pin 231|bisque blush beige honeydew slate|Manufacturer#5|Brand#51|MEDIUM BURNISHED COPPER|17|MED PACK|1131.23|ffily. fur 232|ivory peru lavender orange dark|Manufacturer#5|Brand#53|LARGE BURNISHED NICKEL|50|SM PKG|1132.23|r, unusual requests 233|seashell tomato red lemon saddle|Manufacturer#3|Brand#34|MEDIUM ANODIZED BRASS|25|SM PACK|1133.23|ully ironic 234|chocolate slate maroon azure goldenrod|Manufacturer#1|Brand#13|MEDIUM ANODIZED NICKEL|26|WRAP BOX|1134.23| furiously special 235|sky mint aquamarine dark cornsilk|Manufacturer#3|Brand#32|ECONOMY BRUSHED COPPER|4|MED JAR|1135.23|s. carefully regular d 236|salmon antique burlywood linen peach|Manufacturer#5|Brand#55|ECONOMY ANODIZED STEEL|31|LG JAR|1136.23|ss packages hag 237|yellow orchid dark light smoke|Manufacturer#2|Brand#25|LARGE BURNISHED TIN|49|SM DRUM|1137.23|g the furiously 238|purple dark lawn navajo indian|Manufacturer#3|Brand#34|SMALL POLISHED TIN|35|LG CAN|1138.23|d, expres 239|medium olive pink dim firebrick|Manufacturer#2|Brand#24|LARGE PLATED NICKEL|36|WRAP DRUM|1139.23|lites are perman 240|rose beige magenta coral sandy|Manufacturer#3|Brand#32|MEDIUM BURNISHED NICKEL|1|LG CAN|1140.24|ructions. 241|purple drab puff peach tomato|Manufacturer#5|Brand#51|STANDARD BRUSHED COPPER|3|WRAP CASE|1141.24| quickly regular fo 242|spring green lemon medium olive|Manufacturer#3|Brand#35|SMALL POLISHED STEEL|42|LG BAG|1142.24|ously final theodo 243|bisque chiffon orange misty tan|Manufacturer#3|Brand#32|MEDIUM ANODIZED STEEL|49|SM BOX|1143.24|ress sentim 244|seashell ghost cyan burlywood thistle|Manufacturer#5|Brand#51|ECONOMY BURNISHED BRASS|48|LG BOX|1144.24|ns use above the ir 245|beige orchid chartreuse powder slate|Manufacturer#2|Brand#23|PROMO BURNISHED BRASS|39|JUMBO PKG|1145.24| instructions. ca 246|burlywood orchid dark drab dodger|Manufacturer#2|Brand#21|PROMO BRUSHED BRASS|18|SM CAN|1146.24|e caref 247|medium gainsboro lawn coral peach|Manufacturer#5|Brand#53|LARGE BURNISHED BRASS|4|JUMBO PACK|1147.24|r accounts. carefully 248|drab aquamarine red papaya pale|Manufacturer#1|Brand#15|SMALL PLATED BRASS|8|MED PACK|1148.24|furiously fluffily 249|hot sandy lavender saddle rosy|Manufacturer#4|Brand#44|ECONOMY BURNISHED BRASS|15|LG JAR|1149.24| excuses kindle f 250|antique goldenrod floral forest slate|Manufacturer#4|Brand#44|PROMO POLISHED NICKEL|35|WRAP CASE|1150.25|nstructions affix furi 251|dodger gainsboro violet steel papaya|Manufacturer#1|Brand#15|STANDARD BRUSHED NICKEL|41|WRAP CAN|1151.25|inal ideas aff 252|royal sienna magenta lace brown|Manufacturer#1|Brand#11|PROMO PLATED NICKEL|22|WRAP DRUM|1152.25|e slyly after the de 253|blanched ghost turquoise burlywood spring|Manufacturer#4|Brand#45|PROMO ANODIZED TIN|16|LG PKG|1153.25|he even, re 254|navy light red royal olive|Manufacturer#4|Brand#44|MEDIUM PLATED TIN|10|MED JAR|1154.25|ly at t 255|aquamarine seashell grey navy white|Manufacturer#2|Brand#21|LARGE ANODIZED NICKEL|24|JUMBO CAN|1155.25|fter the c 256|royal cream lawn purple powder|Manufacturer#3|Brand#32|LARGE BURNISHED TIN|2|SM DRUM|1156.25|bove the fur 257|blue gainsboro maroon green burnished|Manufacturer#4|Brand#41|ECONOMY POLISHED COPPER|11|SM JAR|1157.25|riously final foxes 258|royal frosted blue pale dim|Manufacturer#4|Brand#43|STANDARD ANODIZED COPPER|18|WRAP DRUM|1158.25|leep blith 259|ghost puff sky linen burnished|Manufacturer#2|Brand#23|SMALL BURNISHED BRASS|41|WRAP PACK|1159.25|l platelets. evenly 260|firebrick thistle lime frosted khaki|Manufacturer#5|Brand#55|STANDARD ANODIZED STEEL|10|LG BAG|1160.26| haggle f 261|floral moccasin puff rosy ghost|Manufacturer#4|Brand#41|STANDARD POLISHED TIN|44|JUMBO PACK|1161.26|ges was i 262|drab lavender lawn purple puff|Manufacturer#5|Brand#52|MEDIUM PLATED NICKEL|16|JUMBO BAG|1162.26|lar theodolites. f 263|linen red plum purple steel|Manufacturer#1|Brand#14|ECONOMY BURNISHED COPPER|46|MED CASE|1163.26|ly ironic the 264|plum floral sienna magenta ivory|Manufacturer#1|Brand#14|SMALL BURNISHED TIN|23|SM CAN|1164.26|ake. noto 265|aquamarine magenta seashell orange royal|Manufacturer#4|Brand#43|SMALL POLISHED STEEL|19|WRAP CASE|1165.26| slyly ac 266|medium brown tomato cornflower moccasin|Manufacturer#4|Brand#43|ECONOMY BRUSHED STEEL|25|LG DRUM|1166.26|odoli 267|pink salmon moccasin orange wheat|Manufacturer#2|Brand#24|LARGE POLISHED BRASS|41|JUMBO PACK|1167.26|ckly speci 268|cornsilk seashell lime aquamarine violet|Manufacturer#2|Brand#22|MEDIUM PLATED NICKEL|8|MED JAR|1168.26| carefully sil 269|chartreuse navajo moccasin orchid spring|Manufacturer#1|Brand#13|LARGE PLATED BRASS|5|WRAP BOX|1169.26|ully about the ca 270|mint deep white navajo floral|Manufacturer#2|Brand#25|PROMO BURNISHED COPPER|45|MED BAG|1170.27|ly even pinto beans. c 271|lace floral peru dim magenta|Manufacturer#5|Brand#52|PROMO PLATED COPPER|32|MED PKG|1171.27| fluffy acco 272|red purple tomato medium papaya|Manufacturer#5|Brand#52|LARGE POLISHED BRASS|39|LG PKG|1172.27| furiousl 273|pink white sky burnished coral|Manufacturer#2|Brand#25|STANDARD BRUSHED BRASS|50|LG BOX|1173.27|ackages along the 274|cyan lawn snow sky peru|Manufacturer#2|Brand#23|MEDIUM POLISHED STEEL|46|LG JAR|1174.27|hless accounts prin 275|steel chartreuse red plum sienna|Manufacturer#4|Brand#45|LARGE BURNISHED BRASS|37|SM BOX|1175.27|equests 276|orange yellow drab grey blanched|Manufacturer#2|Brand#25|SMALL BRUSHED COPPER|5|LG JAR|1176.27|iously. 277|grey navy rosy red burnished|Manufacturer#4|Brand#43|ECONOMY ANODIZED NICKEL|49|JUMBO JAR|1177.27|arefully special dep 278|blanched lime almond moccasin floral|Manufacturer#4|Brand#42|STANDARD BURNISHED NICKEL|49|SM JAR|1178.27| bold p 279|linen olive lawn blanched gainsboro|Manufacturer#3|Brand#35|SMALL ANODIZED BRASS|27|MED CASE|1179.27|eans boos 280|blue azure slate lace burlywood|Manufacturer#2|Brand#21|STANDARD BURNISHED STEEL|33|LG CAN|1180.28|egular, s 281|red mint brown powder rose|Manufacturer#1|Brand#12|SMALL PLATED STEEL|7|MED CAN|1181.28|regula 282|steel burnished cornsilk beige lace|Manufacturer#1|Brand#11|SMALL BRUSHED COPPER|44|WRAP CAN|1182.28|packages. final ex 283|metallic burlywood lavender midnight cornsilk|Manufacturer#1|Brand#12|SMALL PLATED TIN|8|SM DRUM|1183.28| above the i 284|burlywood chocolate almond ivory coral|Manufacturer#5|Brand#51|ECONOMY POLISHED COPPER|9|MED DRUM|1184.28|its haggle fu 285|cream seashell orange frosted brown|Manufacturer#2|Brand#25|LARGE ANODIZED TIN|42|SM PACK|1185.28| the furiously fi 286|chocolate cornsilk goldenrod violet puff|Manufacturer#5|Brand#53|MEDIUM POLISHED TIN|1|WRAP CASE|1186.28| foxes. regular, fi 287|misty snow thistle burlywood wheat|Manufacturer#1|Brand#15|SMALL PLATED NICKEL|27|SM DRUM|1187.28|y even foxes. final, 288|tan purple yellow ivory olive|Manufacturer#2|Brand#23|STANDARD BURNISHED BRASS|31|SM CASE|1188.28|dencies mold among the 289|peach antique deep peru saddle|Manufacturer#4|Brand#43|PROMO POLISHED TIN|40|LG CASE|1189.28|odolites sleep care 290|deep wheat forest powder firebrick|Manufacturer#1|Brand#12|LARGE BURNISHED BRASS|19|JUMBO JAR|1190.29|according to t 291|goldenrod aquamarine olive white pink|Manufacturer#5|Brand#54|ECONOMY PLATED BRASS|26|SM JAR|1191.29|kages alo 292|navajo tan coral slate thistle|Manufacturer#1|Brand#12|SMALL PLATED NICKEL|2|MED CAN|1192.29|zzle among t 293|lawn cyan plum bisque cream|Manufacturer#5|Brand#52|STANDARD POLISHED STEEL|24|SM PACK|1193.29|onic requests 294|midnight misty royal floral brown|Manufacturer#2|Brand#24|ECONOMY POLISHED TIN|35|SM JAR|1194.29|truct 295|honeydew lace powder almond red|Manufacturer#2|Brand#25|MEDIUM BURNISHED NICKEL|49|JUMBO JAR|1195.29|ully ironic 296|tan seashell chocolate lemon orange|Manufacturer#1|Brand#13|LARGE BURNISHED BRASS|28|SM PKG|1196.29|, regul 297|antique powder almond navajo lace|Manufacturer#2|Brand#22|LARGE BURNISHED STEEL|23|LG PACK|1197.29|e of the slyl 298|azure sky tomato midnight lace|Manufacturer#4|Brand#45|PROMO BRUSHED COPPER|13|JUMBO CAN|1198.29|al deposits wake 299|deep coral dodger green peach|Manufacturer#4|Brand#41|STANDARD ANODIZED COPPER|26|WRAP PKG|1199.29|regular dol 300|light goldenrod green lime papaya|Manufacturer#4|Brand#44|PROMO ANODIZED BRASS|1|WRAP PACK|1200.30|tructions int 301|misty snow lace burnished linen|Manufacturer#4|Brand#42|ECONOMY BRUSHED STEEL|6|WRAP CAN|1201.30|orges 302|black seashell sky grey snow|Manufacturer#4|Brand#41|PROMO POLISHED BRASS|34|SM BAG|1202.30| ideas. special asy 303|almond chocolate firebrick black bisque|Manufacturer#4|Brand#45|ECONOMY PLATED BRASS|7|JUMBO CASE|1203.30|yly according 304|forest mint sienna navy ghost|Manufacturer#4|Brand#43|SMALL POLISHED NICKEL|12|LG DRUM|1204.30|ly regu 305|moccasin tan chartreuse cornsilk drab|Manufacturer#1|Brand#15|SMALL ANODIZED TIN|6|SM BAG|1205.30|ake above 306|lemon peach spring rose ghost|Manufacturer#3|Brand#33|SMALL BURNISHED NICKEL|25|LG PKG|1206.30| regular req 307|peru beige firebrick royal navy|Manufacturer#2|Brand#21|LARGE BURNISHED BRASS|30|WRAP DRUM|1207.30|eas. e 308|olive ivory turquoise sienna wheat|Manufacturer#2|Brand#21|STANDARD BURNISHED BRASS|40|WRAP JAR|1208.30|ngside of the s 309|red smoke blue pale lace|Manufacturer#4|Brand#42|LARGE ANODIZED NICKEL|49|JUMBO PKG|1209.30|lly even c 310|sandy sky dark navy blanched|Manufacturer#1|Brand#14|ECONOMY BRUSHED NICKEL|32|MED JAR|1210.31|arefully. requ 311|indian orchid navajo dark forest|Manufacturer#4|Brand#44|MEDIUM PLATED TIN|18|LG DRUM|1211.31|final, 312|powder firebrick white peach goldenrod|Manufacturer#3|Brand#32|PROMO BURNISHED STEEL|29|MED CASE|1212.31|nts doze slyly. d 313|moccasin rosy lemon linen royal|Manufacturer#3|Brand#34|PROMO BRUSHED TIN|13|WRAP PACK|1213.31|nly regular 314|grey royal seashell violet misty|Manufacturer#3|Brand#35|STANDARD PLATED STEEL|14|LG DRUM|1214.31|ular deposits. 315|navy cyan orchid seashell moccasin|Manufacturer#2|Brand#23|LARGE ANODIZED BRASS|20|LG JAR|1215.31|s was slyly pac 316|almond rosy green hot midnight|Manufacturer#4|Brand#41|MEDIUM BRUSHED TIN|9|MED CASE|1216.31|deposits haggle 317|linen peru thistle turquoise medium|Manufacturer#1|Brand#15|LARGE POLISHED STEEL|34|JUMBO BOX|1217.31|e furiously regular pa 318|rosy rose indian puff pale|Manufacturer#1|Brand#12|PROMO BURNISHED NICKEL|20|JUMBO CASE|1218.31|ar re 319|royal metallic lawn spring firebrick|Manufacturer#3|Brand#31|MEDIUM BURNISHED TIN|44|LG BOX|1219.31|kages boost along 320|chiffon linen rose white rosy|Manufacturer#5|Brand#53|LARGE ANODIZED STEEL|26|WRAP DRUM|1220.32|structions a 321|tomato drab cream tan navajo|Manufacturer#5|Brand#51|ECONOMY ANODIZED STEEL|9|SM JAR|1221.32|tructions boost car 322|dark black rosy yellow ivory|Manufacturer#3|Brand#35|STANDARD PLATED STEEL|32|JUMBO CASE|1222.32|uses. blithely pendin 323|moccasin goldenrod tan maroon bisque|Manufacturer#4|Brand#41|MEDIUM BRUSHED BRASS|15|MED CASE|1223.32|ular pi 324|aquamarine ivory magenta linen black|Manufacturer#2|Brand#25|SMALL PLATED TIN|36|SM BAG|1224.32|ages. requests wake. 325|plum cyan almond dark orchid|Manufacturer#1|Brand#15|LARGE PLATED COPPER|22|MED PACK|1225.32|ackages. slyly pendin 326|sky papaya green azure chiffon|Manufacturer#3|Brand#34|LARGE ANODIZED STEEL|48|JUMBO CAN|1226.32|aggle slyly special f 327|metallic lavender green firebrick ghost|Manufacturer#2|Brand#23|PROMO POLISHED BRASS|14|SM PACK|1227.32|regular 328|turquoise bisque wheat cornflower forest|Manufacturer#3|Brand#34|PROMO POLISHED COPPER|24|JUMBO CASE|1228.32|ronic dependencies 329|mint dodger chocolate papaya azure|Manufacturer#3|Brand#31|MEDIUM ANODIZED NICKEL|21|LG PKG|1229.32|. quickly pending pa 330|royal lime misty frosted smoke|Manufacturer#3|Brand#35|LARGE PLATED COPPER|18|LG JAR|1230.33|efully pending depend 331|metallic ivory cornsilk mint hot|Manufacturer#2|Brand#25|SMALL BURNISHED COPPER|15|WRAP JAR|1231.33| the slyly iron 332|frosted blush firebrick linen salmon|Manufacturer#2|Brand#24|STANDARD BRUSHED BRASS|14|SM CAN|1232.33|instru 333|chocolate tomato light chartreuse orange|Manufacturer#2|Brand#21|ECONOMY POLISHED COPPER|1|LG DRUM|1233.33|accordi 334|lawn wheat dodger forest firebrick|Manufacturer#3|Brand#31|ECONOMY PLATED BRASS|32|LG JAR|1234.33|deas. blithely express 335|dark cream plum goldenrod smoke|Manufacturer#3|Brand#32|SMALL BRUSHED COPPER|3|SM JAR|1235.33|ly ironic deposits h 336|maroon medium green tomato rosy|Manufacturer#2|Brand#25|PROMO ANODIZED TIN|3|WRAP JAR|1236.33|nto beans. 337|magenta lavender blush peru aquamarine|Manufacturer#2|Brand#23|MEDIUM BRUSHED COPPER|29|MED DRUM|1237.33|uests. carefully fin 338|turquoise honeydew dim saddle grey|Manufacturer#1|Brand#14|LARGE ANODIZED NICKEL|26|JUMBO JAR|1238.33|of th 339|saddle white red royal midnight|Manufacturer#2|Brand#23|PROMO BURNISHED COPPER|14|LG PACK|1239.33|across the slyly expre 340|sienna purple lawn coral navy|Manufacturer#2|Brand#23|STANDARD BRUSHED TIN|3|JUMBO BOX|1240.34|l ideas wake. quic 341|pale smoke drab spring burnished|Manufacturer#4|Brand#45|PROMO POLISHED COPPER|34|MED PKG|1241.34|ully regular a 342|purple indian burnished khaki puff|Manufacturer#4|Brand#43|LARGE BURNISHED NICKEL|37|JUMBO CAN|1242.34|riously final pac 343|blush thistle chartreuse tomato navy|Manufacturer#3|Brand#35|PROMO ANODIZED NICKEL|7|SM JAR|1243.34|s-- ironic foxes boost 344|lime chiffon sky magenta midnight|Manufacturer#4|Brand#45|LARGE ANODIZED TIN|39|SM BOX|1244.34| according to the pend 345|cyan frosted spring orange puff|Manufacturer#4|Brand#45|ECONOMY BURNISHED NICKEL|22|LG CAN|1245.34|un carefull 346|medium honeydew blush lace lemon|Manufacturer#5|Brand#51|ECONOMY BRUSHED COPPER|8|LG PKG|1246.34|benea 347|deep antique royal bisque rosy|Manufacturer#3|Brand#34|SMALL ANODIZED TIN|28|WRAP BAG|1247.34|r, even instruct 348|blush navajo peru chartreuse dim|Manufacturer#5|Brand#53|MEDIUM POLISHED STEEL|16|WRAP BOX|1248.34| pending 349|ghost black hot salmon lace|Manufacturer#2|Brand#24|ECONOMY BURNISHED NICKEL|12|MED JAR|1249.34|tes. unusual acco 350|peach thistle royal papaya cornsilk|Manufacturer#3|Brand#32|STANDARD ANODIZED NICKEL|25|WRAP CAN|1250.35|luffily p 351|slate pale misty sienna red|Manufacturer#3|Brand#32|STANDARD PLATED TIN|48|WRAP CAN|1251.35|sts integrate. bl 352|sandy almond peach ivory violet|Manufacturer#4|Brand#42|MEDIUM BURNISHED BRASS|44|JUMBO CAN|1252.35|lites ne 353|azure linen grey blanched dim|Manufacturer#3|Brand#34|PROMO BURNISHED TIN|43|WRAP JAR|1253.35|final packages use 354|frosted deep powder beige lemon|Manufacturer#4|Brand#42|PROMO BURNISHED STEEL|38|LG CAN|1254.35|d requests wake 355|mint goldenrod sandy burnished olive|Manufacturer#5|Brand#52|ECONOMY POLISHED NICKEL|16|JUMBO JAR|1255.35| slyly bold theodoli 356|hot pink peach lemon chartreuse|Manufacturer#1|Brand#14|STANDARD BRUSHED NICKEL|46|WRAP BAG|1256.35|according to the furio 357|frosted rosy powder floral mint|Manufacturer#4|Brand#42|STANDARD PLATED COPPER|19|SM DRUM|1257.35|s use at 358|purple maroon lavender cornsilk salmon|Manufacturer#2|Brand#24|STANDARD POLISHED BRASS|39|JUMBO PACK|1258.35|g grouches use slyly 359|pink medium olive rosy linen|Manufacturer#3|Brand#35|PROMO PLATED BRASS|19|LG DRUM|1259.35|ithely 360|midnight papaya violet peach cream|Manufacturer#3|Brand#35|SMALL POLISHED COPPER|14|WRAP BOX|1260.36|. bold 361|indian dim tan antique puff|Manufacturer#5|Brand#52|SMALL ANODIZED COPPER|42|WRAP DRUM|1261.36| dolphins. carefully r 362|sienna green ghost rose lawn|Manufacturer#1|Brand#13|ECONOMY POLISHED COPPER|9|JUMBO BAG|1262.36|accounts. furi 363|rosy grey blush dark lime|Manufacturer#2|Brand#22|MEDIUM ANODIZED BRASS|16|WRAP JAR|1263.36|ideas. regular ac 364|puff almond blanched drab misty|Manufacturer#4|Brand#43|LARGE BRUSHED NICKEL|21|MED PKG|1264.36|ach caref 365|olive blue navy frosted violet|Manufacturer#3|Brand#31|SMALL POLISHED STEEL|8|LG CAN|1265.36|efully regula 366|seashell antique beige thistle cream|Manufacturer#5|Brand#53|ECONOMY BURNISHED STEEL|46|WRAP DRUM|1266.36|unts cajole 367|black lavender dark cornflower grey|Manufacturer#2|Brand#23|STANDARD PLATED BRASS|2|JUMBO PACK|1267.36|ly about the tithe 368|cornsilk blush bisque hot cyan|Manufacturer#5|Brand#52|ECONOMY BRUSHED TIN|8|SM PACK|1268.36|t the even, 369|drab plum olive firebrick beige|Manufacturer#2|Brand#21|SMALL POLISHED BRASS|11|SM BOX|1269.36|ilent request 370|almond linen pale aquamarine ghost|Manufacturer#5|Brand#52|LARGE BRUSHED COPPER|46|JUMBO BOX|1270.37|ost quickly against 371|maroon dark beige forest drab|Manufacturer#4|Brand#45|ECONOMY ANODIZED COPPER|27|MED BAG|1271.37|o beans. even inst 372|black frosted aquamarine maroon orange|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|35|LG CAN|1272.37| regular accounts 373|beige blanched coral cornsilk sky|Manufacturer#5|Brand#53|STANDARD PLATED TIN|37|JUMBO PACK|1273.37| after the fl 374|orange lemon cream dim midnight|Manufacturer#1|Brand#11|LARGE PLATED STEEL|18|MED BOX|1274.37| packages. furiou 375|floral blush lawn puff brown|Manufacturer#1|Brand#15|SMALL POLISHED BRASS|16|JUMBO JAR|1275.37| ironic courts aga 376|lace drab wheat red tan|Manufacturer#4|Brand#44|MEDIUM BRUSHED STEEL|44|SM PKG|1276.37|blithe 377|green forest rose slate cornflower|Manufacturer#4|Brand#44|STANDARD BURNISHED COPPER|16|WRAP BOX|1277.37|ly regula 378|burnished lime green turquoise red|Manufacturer#3|Brand#32|SMALL PLATED COPPER|45|JUMBO PACK|1278.37| slyly careful, ironic 379|cream sienna sandy thistle medium|Manufacturer#3|Brand#33|ECONOMY POLISHED STEEL|5|SM CASE|1279.37|l have to sl 380|khaki powder royal maroon snow|Manufacturer#4|Brand#41|LARGE BRUSHED TIN|6|JUMBO DRUM|1280.38|tions. blithely ev 381|sienna peach wheat linen tomato|Manufacturer#5|Brand#53|ECONOMY ANODIZED COPPER|37|LG CASE|1281.38|ons. furiousl 382|honeydew ghost lace plum sky|Manufacturer#4|Brand#41|LARGE POLISHED BRASS|4|SM CAN|1282.38|blithely 383|seashell tan purple almond coral|Manufacturer#2|Brand#23|SMALL BRUSHED COPPER|45|SM PACK|1283.38|telet 384|papaya thistle rosy rose puff|Manufacturer#4|Brand#43|STANDARD BURNISHED TIN|2|WRAP CAN|1284.38|blithel 385|snow chocolate ivory lemon magenta|Manufacturer#1|Brand#12|STANDARD PLATED TIN|41|WRAP JAR|1285.38|nts wake 386|red blanched thistle bisque wheat|Manufacturer#1|Brand#13|STANDARD PLATED STEEL|4|SM BOX|1286.38|ests. final a 387|smoke white metallic sky firebrick|Manufacturer#3|Brand#34|LARGE BRUSHED COPPER|23|WRAP CASE|1287.38|gular w 388|lawn puff grey forest indian|Manufacturer#1|Brand#11|STANDARD POLISHED STEEL|20|LG PKG|1288.38|nt pinto beans after t 389|magenta ivory lawn antique turquoise|Manufacturer#1|Brand#11|PROMO BRUSHED TIN|16|MED PKG|1289.38|cial pinto beans amo 390|maroon magenta saddle linen lime|Manufacturer#5|Brand#54|SMALL ANODIZED COPPER|24|MED DRUM|1290.39|unts integrate 391|seashell cornsilk metallic royal sky|Manufacturer#4|Brand#41|SMALL PLATED STEEL|18|LG PACK|1291.39|ickly ironic f 392|medium plum powder blush yellow|Manufacturer#4|Brand#42|MEDIUM BRUSHED COPPER|21|SM BOX|1292.39|even request 393|blue coral gainsboro slate lawn|Manufacturer#3|Brand#34|STANDARD PLATED NICKEL|36|JUMBO CAN|1293.39|gular pi 394|chocolate royal cornflower papaya yellow|Manufacturer#1|Brand#11|MEDIUM POLISHED STEEL|33|MED JAR|1294.39|ely even excuses. 395|drab steel royal medium chartreuse|Manufacturer#3|Brand#32|MEDIUM BURNISHED BRASS|9|LG BOX|1295.39|sits haggle fluffily 396|medium seashell lavender almond tan|Manufacturer#3|Brand#35|SMALL BURNISHED COPPER|37|SM CASE|1296.39|ctions. silent packag 397|purple spring turquoise goldenrod grey|Manufacturer#1|Brand#15|LARGE ANODIZED COPPER|3|JUMBO JAR|1297.39|ly ironic pains. f 398|red salmon sandy white hot|Manufacturer#3|Brand#32|MEDIUM BRUSHED NICKEL|32|SM DRUM|1298.39|ickly sp 399|orange peru yellow blanched seashell|Manufacturer#2|Brand#21|LARGE BRUSHED BRASS|37|WRAP BAG|1299.39|nts sleep dog 400|rose blue smoke olive sandy|Manufacturer#2|Brand#21|ECONOMY POLISHED BRASS|32|SM BOX|1300.40| nag. quick 401|almond chiffon indian green dim|Manufacturer#2|Brand#21|ECONOMY POLISHED STEEL|10|SM PKG|1301.40|ideas cajole 402|black rosy ghost yellow ivory|Manufacturer#4|Brand#45|LARGE ANODIZED BRASS|16|SM PACK|1302.40|s above t 403|indian lemon linen almond tomato|Manufacturer#4|Brand#45|MEDIUM BURNISHED STEEL|19|JUMBO DRUM|1303.40|ic requ 404|maroon olive pink rose cornflower|Manufacturer#4|Brand#41|STANDARD ANODIZED NICKEL|9|SM CASE|1304.40|ckages integrate blit 405|ivory yellow peru almond cornsilk|Manufacturer#3|Brand#34|STANDARD BRUSHED BRASS|17|WRAP JAR|1305.40|e to maintain careful 406|thistle saddle white puff lemon|Manufacturer#2|Brand#21|STANDARD PLATED STEEL|43|JUMBO JAR|1306.40|ake foxes- 407|tomato turquoise dim powder ivory|Manufacturer#5|Brand#53|LARGE BRUSHED NICKEL|39|SM JAR|1307.40|quickl 408|indian metallic peach gainsboro pale|Manufacturer#2|Brand#24|PROMO POLISHED STEEL|4|MED PKG|1308.40|refully b 409|smoke forest metallic saddle hot|Manufacturer#4|Brand#44|SMALL PLATED NICKEL|40|MED CASE|1309.40|usly above 410|tomato sienna hot chartreuse dodger|Manufacturer#2|Brand#25|STANDARD PLATED NICKEL|36|JUMBO BOX|1310.41| ironic pa 411|lemon dark khaki antique slate|Manufacturer#1|Brand#12|ECONOMY BURNISHED STEEL|17|JUMBO DRUM|1311.41|o beans a 412|slate forest spring pink peru|Manufacturer#1|Brand#14|SMALL ANODIZED STEEL|33|WRAP PKG|1312.41|ccording to the fur 413|chiffon ivory lawn metallic beige|Manufacturer#3|Brand#33|PROMO PLATED STEEL|49|MED PKG|1313.41|r deposits wake. alw 414|pink brown purple puff snow|Manufacturer#4|Brand#41|SMALL BURNISHED STEEL|50|WRAP CASE|1314.41|efully. dolph 415|dark chocolate wheat ivory orchid|Manufacturer#2|Brand#22|MEDIUM ANODIZED COPPER|42|SM DRUM|1315.41|blithely blit 416|ghost misty chocolate peach green|Manufacturer#1|Brand#13|MEDIUM POLISHED STEEL|11|WRAP JAR|1316.41|yly pending deposit 417|gainsboro sky turquoise ghost mint|Manufacturer#1|Brand#14|MEDIUM PLATED TIN|31|LG BAG|1317.41| nag blithely: ex 418|lavender forest puff beige tan|Manufacturer#1|Brand#11|MEDIUM ANODIZED NICKEL|32|JUMBO JAR|1318.41| ironic deposits. requ 419|moccasin navy seashell turquoise sienna|Manufacturer#4|Brand#43|ECONOMY BRUSHED TIN|42|WRAP BAG|1319.41|boost slyly among t 420|chocolate cornsilk midnight beige floral|Manufacturer#2|Brand#23|PROMO BRUSHED TIN|15|LG JAR|1320.42|y. fluffi 421|white black burnished brown medium|Manufacturer#1|Brand#12|LARGE PLATED STEEL|36|WRAP PACK|1321.42|regular 422|honeydew smoke violet pink bisque|Manufacturer#3|Brand#33|STANDARD POLISHED TIN|16|LG CAN|1322.42| bold 423|bisque misty dark hot blush|Manufacturer#3|Brand#31|PROMO BRUSHED BRASS|2|SM JAR|1323.42|accou 424|dim almond turquoise beige royal|Manufacturer#3|Brand#34|SMALL BRUSHED TIN|9|LG CASE|1324.42|usly slyl 425|ivory wheat chiffon navy burlywood|Manufacturer#3|Brand#32|PROMO BRUSHED STEEL|23|JUMBO BOX|1325.42|ndenci 426|maroon linen sienna firebrick burnished|Manufacturer#1|Brand#14|STANDARD ANODIZED NICKEL|42|WRAP PACK|1326.42|furiously even pa 427|honeydew azure brown royal sky|Manufacturer#2|Brand#22|PROMO BURNISHED NICKEL|47|JUMBO CAN|1327.42|ts. express, unusual 428|beige spring floral wheat drab|Manufacturer#5|Brand#54|LARGE BURNISHED TIN|14|JUMBO CASE|1328.42|furiously ir 429|red firebrick smoke magenta moccasin|Manufacturer#4|Brand#45|ECONOMY BRUSHED COPPER|21|MED DRUM|1329.42| carefully even 430|snow mint deep brown goldenrod|Manufacturer#3|Brand#34|PROMO BURNISHED NICKEL|34|MED CAN|1330.43| sleep. slyly pend 431|grey smoke navy peach white|Manufacturer#4|Brand#45|MEDIUM POLISHED TIN|30|SM PACK|1331.43|iousl 432|white cornflower honeydew red ivory|Manufacturer#2|Brand#21|MEDIUM PLATED BRASS|6|MED JAR|1332.43|s nag furiously fluff 433|olive moccasin almond tomato plum|Manufacturer#1|Brand#12|STANDARD BURNISHED TIN|2|SM PACK|1333.43|ts haggle furiously 434|turquoise coral violet dim midnight|Manufacturer#5|Brand#51|ECONOMY BURNISHED COPPER|20|JUMBO BAG|1334.43|y against the c 435|coral misty brown grey hot|Manufacturer#3|Brand#35|ECONOMY ANODIZED BRASS|17|WRAP BAG|1335.43|carefully ironic pack 436|turquoise yellow dim purple antique|Manufacturer#1|Brand#14|LARGE POLISHED BRASS|50|WRAP CASE|1336.43| the regul 437|purple slate gainsboro powder magenta|Manufacturer#5|Brand#54|ECONOMY PLATED TIN|17|WRAP PACK|1337.43|e furiously 438|antique orchid cornflower puff almond|Manufacturer#1|Brand#15|LARGE BURNISHED TIN|31|LG DRUM|1338.43|epitaphs wake 439|bisque cyan rosy ivory blanched|Manufacturer#2|Brand#21|MEDIUM BURNISHED TIN|32|SM PKG|1339.43|uests wake slyly over 440|khaki turquoise violet maroon misty|Manufacturer#3|Brand#34|ECONOMY BURNISHED BRASS|22|MED CASE|1340.44|furiously above the 441|goldenrod royal slate grey burlywood|Manufacturer#3|Brand#35|PROMO BURNISHED STEEL|24|MED BOX|1341.44|s wake atop the ru 442|honeydew cornsilk powder salmon purple|Manufacturer#4|Brand#41|LARGE BURNISHED TIN|22|MED PACK|1342.44|refully al 443|chocolate burnished orchid mint royal|Manufacturer#4|Brand#44|STANDARD BURNISHED BRASS|10|SM BOX|1343.44|arefully. sl 444|ghost blanched forest khaki rose|Manufacturer#3|Brand#32|MEDIUM ANODIZED COPPER|42|LG JAR|1344.44|ickly special excuse 445|metallic ivory floral cornsilk burnished|Manufacturer#1|Brand#12|LARGE BURNISHED NICKEL|47|SM DRUM|1345.44|s about the re 446|sandy brown midnight tan bisque|Manufacturer#1|Brand#12|STANDARD BURNISHED NICKEL|11|WRAP CASE|1346.44|s sentiments affix 447|forest smoke aquamarine cyan dim|Manufacturer#5|Brand#53|STANDARD BURNISHED COPPER|22|LG JAR|1347.44|s theodolites above th 448|wheat tomato cyan lemon maroon|Manufacturer#1|Brand#13|PROMO POLISHED BRASS|31|LG CAN|1348.44| excuses affix silen 449|almond thistle cornsilk bisque blush|Manufacturer#2|Brand#25|MEDIUM BRUSHED BRASS|47|JUMBO BAG|1349.44|ular, unusual p 450|sky lace green pale lime|Manufacturer#4|Brand#43|MEDIUM BURNISHED TIN|23|SM CASE|1350.45|slyly 451|rosy ghost sky mint smoke|Manufacturer#4|Brand#42|PROMO PLATED TIN|28|WRAP DRUM|1351.45| detect blithely bl 452|saddle sandy cream sienna blue|Manufacturer#1|Brand#11|STANDARD ANODIZED STEEL|48|JUMBO BOX|1352.45|ic requests wake 453|lemon navy seashell khaki sienna|Manufacturer#3|Brand#34|ECONOMY ANODIZED TIN|20|LG CASE|1353.45|y above the pen 454|royal lime saddle magenta violet|Manufacturer#5|Brand#53|MEDIUM BURNISHED STEEL|32|LG BOX|1354.45|ual requests. 455|aquamarine blanched misty moccasin sky|Manufacturer#2|Brand#22|LARGE BRUSHED BRASS|18|MED JAR|1355.45|ongside of the careful 456|antique maroon olive turquoise lace|Manufacturer#5|Brand#53|PROMO POLISHED BRASS|40|SM PKG|1356.45|usual, 457|navy linen forest moccasin slate|Manufacturer#5|Brand#52|SMALL POLISHED TIN|46|SM CAN|1357.45|express accounts. sp 458|beige papaya grey ivory white|Manufacturer#2|Brand#22|PROMO BRUSHED COPPER|38|MED BAG|1358.45|structions sleep 459|beige slate magenta dim salmon|Manufacturer#2|Brand#23|MEDIUM ANODIZED NICKEL|27|SM PACK|1359.45| blithely. fur 460|maroon papaya orange spring light|Manufacturer#3|Brand#35|STANDARD ANODIZED BRASS|47|JUMBO DRUM|1360.46|he carefully 461|goldenrod dim wheat tan violet|Manufacturer#5|Brand#54|SMALL BRUSHED TIN|44|JUMBO CASE|1361.46|: slyly express pa 462|lime yellow sandy floral khaki|Manufacturer#5|Brand#54|MEDIUM BURNISHED BRASS|11|WRAP CAN|1362.46|s sleep quickly care 463|steel cream grey mint dark|Manufacturer#2|Brand#21|MEDIUM ANODIZED COPPER|48|MED PACK|1363.46|unts use fluffily 464|azure magenta lace smoke ghost|Manufacturer#4|Brand#42|LARGE BURNISHED NICKEL|34|LG BOX|1364.46|l decoys. close exc 465|smoke sky bisque ghost black|Manufacturer#4|Brand#42|SMALL POLISHED BRASS|12|MED JAR|1365.46|nts. slyly special ac 466|deep ghost cornflower ivory burnished|Manufacturer#3|Brand#33|PROMO POLISHED BRASS|12|LG PKG|1366.46|ly special pains. iron 467|cornflower lime midnight plum forest|Manufacturer#2|Brand#24|STANDARD BURNISHED STEEL|48|JUMBO PACK|1367.46|ependenc 468|spring grey azure khaki lace|Manufacturer#4|Brand#41|MEDIUM BURNISHED STEEL|24|SM PACK|1368.46|o the 469|burlywood dark steel sky blue|Manufacturer#1|Brand#13|MEDIUM POLISHED STEEL|43|WRAP PKG|1369.46|osits. slyly unusua 470|honeydew lime khaki saddle indian|Manufacturer#2|Brand#24|STANDARD BRUSHED TIN|38|JUMBO CASE|1370.47|ess deposits 471|goldenrod honeydew frosted almond chiffon|Manufacturer#1|Brand#11|LARGE BRUSHED STEEL|21|JUMBO BAG|1371.47|sits nag caref 472|dim misty lime purple cornsilk|Manufacturer#3|Brand#31|PROMO BRUSHED COPPER|24|SM JAR|1372.47| according 473|moccasin antique lace peach chiffon|Manufacturer#1|Brand#12|ECONOMY PLATED STEEL|2|MED DRUM|1373.47|e furious 474|papaya green royal burlywood saddle|Manufacturer#1|Brand#14|ECONOMY PLATED STEEL|45|LG PACK|1374.47|ously even reques 475|coral peru forest thistle khaki|Manufacturer#3|Brand#34|STANDARD ANODIZED NICKEL|30|MED PACK|1375.47|thely unusual th 476|cornflower deep turquoise slate sandy|Manufacturer#3|Brand#33|STANDARD POLISHED COPPER|33|WRAP PACK|1376.47| according to the blit 477|plum peru spring firebrick lavender|Manufacturer#1|Brand#11|ECONOMY ANODIZED COPPER|35|JUMBO BAG|1377.47|lent s 478|lemon snow coral lime seashell|Manufacturer#4|Brand#45|STANDARD POLISHED COPPER|11|MED CASE|1378.47|ly bold foxes ca 479|snow frosted slate magenta sky|Manufacturer#2|Brand#25|MEDIUM PLATED STEEL|35|SM BAG|1379.47| pending ac 480|saddle light black drab navajo|Manufacturer#5|Brand#54|MEDIUM ANODIZED STEEL|31|WRAP BOX|1380.48|eans. quickly 481|slate red firebrick beige aquamarine|Manufacturer#1|Brand#14|MEDIUM BURNISHED BRASS|17|JUMBO PACK|1381.48|s cajole som 482|burnished spring azure green chocolate|Manufacturer#5|Brand#53|MEDIUM POLISHED STEEL|11|SM DRUM|1382.48|deposits. fluffily b 483|deep white khaki floral cornflower|Manufacturer#1|Brand#15|MEDIUM ANODIZED COPPER|9|SM PKG|1383.48|usly final instru 484|bisque rosy olive peach chiffon|Manufacturer#1|Brand#15|MEDIUM POLISHED BRASS|34|SM CASE|1384.48|uffily final deposits. 485|orchid dark antique deep ivory|Manufacturer#3|Brand#31|PROMO PLATED STEEL|44|LG JAR|1385.48| requests. regularly r 486|tomato lime moccasin turquoise grey|Manufacturer#4|Brand#45|SMALL PLATED COPPER|15|MED PKG|1386.48|ent accounts among t 487|firebrick saddle chocolate peru ghost|Manufacturer#5|Brand#52|SMALL BRUSHED COPPER|43|WRAP PACK|1387.48|iously ruthles 488|forest puff drab beige blue|Manufacturer#2|Brand#21|PROMO PLATED BRASS|47|JUMBO JAR|1388.48| the slyly pending r 489|smoke green blush deep royal|Manufacturer#2|Brand#22|LARGE BURNISHED TIN|36|LG CAN|1389.48|unts. quickly bold id 490|yellow mint saddle dodger orchid|Manufacturer#4|Brand#41|SMALL PLATED NICKEL|11|MED BOX|1390.49|fluffily express packa 491|snow mint rose almond frosted|Manufacturer#3|Brand#31|SMALL POLISHED BRASS|42|MED JAR|1391.49| deposit 492|turquoise rose navajo deep chiffon|Manufacturer#4|Brand#42|MEDIUM BRUSHED COPPER|13|JUMBO DRUM|1392.49|yly slyly express th 493|blue gainsboro sky burnished puff|Manufacturer#4|Brand#45|MEDIUM BURNISHED BRASS|8|MED PKG|1393.49|uses. bold 494|navy pale frosted lawn spring|Manufacturer#4|Brand#42|STANDARD BRUSHED NICKEL|40|MED PKG|1394.49|y pending theodo 495|lemon burlywood chartreuse forest honeydew|Manufacturer#4|Brand#42|SMALL BRUSHED BRASS|28|MED PKG|1395.49|, ironic pac 496|orchid bisque antique ivory lavender|Manufacturer#1|Brand#12|STANDARD BRUSHED BRASS|11|SM BAG|1396.49|ealthily 497|hot powder dim cream metallic|Manufacturer#2|Brand#25|PROMO PLATED TIN|1|SM CAN|1397.49|l accounts sl 498|sandy sky gainsboro peach cornflower|Manufacturer#2|Brand#25|MEDIUM POLISHED STEEL|21|WRAP CASE|1398.49|lthily even co 499|lemon pale blue burnished white|Manufacturer#3|Brand#35|SMALL PLATED COPPER|38|JUMBO PKG|1399.49|furiously across 500|hot medium spring orange violet|Manufacturer#4|Brand#41|MEDIUM POLISHED STEEL|13|SM JAR|1400.50|regul 501|tomato navy midnight cyan chiffon|Manufacturer#4|Brand#45|SMALL BURNISHED COPPER|37|SM BAG|1401.50|press pinto b 502|maroon dark plum brown red|Manufacturer#2|Brand#21|SMALL PLATED TIN|6|JUMBO DRUM|1402.50|quests integrate 503|lavender cyan red lace light|Manufacturer#2|Brand#24|PROMO BRUSHED NICKEL|30|SM PKG|1403.50| the dep 504|pink tan papaya ivory lace|Manufacturer#2|Brand#24|PROMO BRUSHED STEEL|10|SM JAR|1404.50|refully after the fu 505|pink cornsilk antique mint forest|Manufacturer#4|Brand#42|STANDARD PLATED BRASS|32|WRAP JAR|1405.50|ly even 506|beige floral goldenrod lace almond|Manufacturer#1|Brand#14|PROMO BRUSHED NICKEL|30|WRAP PACK|1406.50|y ruthless, fi 507|red misty salmon ivory khaki|Manufacturer#4|Brand#43|SMALL ANODIZED BRASS|20|SM CASE|1407.50|ial foxes. furiously 508|pale peach maroon burlywood tan|Manufacturer#1|Brand#14|LARGE ANODIZED TIN|40|WRAP PKG|1408.50|ect according 509|tomato cornsilk chartreuse blue magenta|Manufacturer#1|Brand#13|MEDIUM BRUSHED NICKEL|30|MED CAN|1409.50|eposits. slyly 510|blush thistle orchid red lace|Manufacturer#4|Brand#43|SMALL PLATED NICKEL|25|JUMBO PACK|1410.51|al platelets. iro 511|red pale plum orchid moccasin|Manufacturer#1|Brand#15|STANDARD BRUSHED COPPER|2|LG JAR|1411.51|ss package 512|firebrick azure mint chocolate wheat|Manufacturer#5|Brand#51|PROMO ANODIZED COPPER|18|LG PACK|1412.51|dolphins cajo 513|mint thistle chiffon magenta salmon|Manufacturer#3|Brand#32|MEDIUM BRUSHED STEEL|10|WRAP CAN|1413.51| after th 514|rosy sienna purple light rose|Manufacturer#4|Brand#42|LARGE POLISHED BRASS|10|JUMBO BOX|1414.51|omise 515|cream navajo drab dark peru|Manufacturer#5|Brand#53|MEDIUM PLATED TIN|37|WRAP BOX|1415.51|ular asymptotes im 516|purple linen lace blanched snow|Manufacturer#1|Brand#12|SMALL BRUSHED NICKEL|28|LG CAN|1416.51|ly express 517|aquamarine rosy violet moccasin snow|Manufacturer#3|Brand#34|SMALL BURNISHED COPPER|30|WRAP CAN|1417.51|uses. 518|yellow tomato lawn rosy lemon|Manufacturer#4|Brand#43|PROMO BRUSHED BRASS|33|JUMBO DRUM|1418.51|n requests hag 519|violet medium dark red smoke|Manufacturer#3|Brand#32|LARGE BRUSHED BRASS|24|WRAP PACK|1419.51|xes are fluffily fluf 520|spring honeydew sky sandy almond|Manufacturer#1|Brand#12|PROMO BURNISHED BRASS|49|MED CASE|1420.52| foxes. 521|grey drab honeydew coral pale|Manufacturer#3|Brand#35|PROMO BRUSHED STEEL|21|JUMBO DRUM|1421.52|the blithe 522|frosted lawn violet turquoise coral|Manufacturer#3|Brand#32|STANDARD BRUSHED NICKEL|15|MED CASE|1422.52|deas. brave ac 523|light goldenrod honeydew indian lemon|Manufacturer#4|Brand#42|ECONOMY BRUSHED NICKEL|7|JUMBO JAR|1423.52|e blithe 524|burnished hot drab midnight tan|Manufacturer#1|Brand#14|SMALL ANODIZED STEEL|17|JUMBO PACK|1424.52|ly reg 525|plum midnight coral snow lemon|Manufacturer#3|Brand#33|STANDARD BURNISHED STEEL|47|JUMBO CAN|1425.52|ly except 526|dodger moccasin wheat puff lace|Manufacturer#5|Brand#53|PROMO ANODIZED BRASS|47|JUMBO PKG|1426.52|ronic, iro 527|moccasin light purple sky magenta|Manufacturer#3|Brand#32|MEDIUM POLISHED NICKEL|39|LG JAR|1427.52|counts wake abov 528|dim deep puff pink floral|Manufacturer#2|Brand#24|STANDARD PLATED STEEL|40|LG CAN|1428.52|ly ironic pl 529|powder tan brown royal blush|Manufacturer#3|Brand#31|MEDIUM PLATED NICKEL|33|SM PKG|1429.52|y along the 530|mint grey chiffon magenta saddle|Manufacturer#4|Brand#42|PROMO PLATED STEEL|45|JUMBO CASE|1430.53|kages about t 531|navajo hot forest puff olive|Manufacturer#2|Brand#22|ECONOMY POLISHED NICKEL|16|WRAP DRUM|1431.53| atta 532|spring wheat purple chiffon puff|Manufacturer#1|Brand#14|LARGE BRUSHED NICKEL|45|WRAP BAG|1432.53|s. blithely 533|cream lemon goldenrod rosy aquamarine|Manufacturer#2|Brand#24|PROMO PLATED STEEL|38|JUMBO PACK|1433.53|ng the blithely final 534|bisque saddle hot steel frosted|Manufacturer#5|Brand#53|STANDARD PLATED NICKEL|27|LG CASE|1434.53|rts poach over th 535|mint chocolate tomato peach sky|Manufacturer#5|Brand#53|STANDARD PLATED TIN|17|SM CASE|1435.53|he unusual idea 536|orchid snow thistle peru moccasin|Manufacturer#2|Brand#23|STANDARD PLATED STEEL|36|JUMBO BAG|1436.53|le slyl 537|hot orange red cornsilk goldenrod|Manufacturer#1|Brand#15|ECONOMY BRUSHED TIN|26|WRAP BOX|1437.53|fully regular platelet 538|rose black peach orchid cyan|Manufacturer#1|Brand#15|ECONOMY POLISHED STEEL|24|SM PACK|1438.53|ic epitap 539|dodger midnight salmon drab saddle|Manufacturer#3|Brand#34|MEDIUM BURNISHED TIN|4|SM DRUM|1439.53| deposits. fi 540|violet white steel chocolate burlywood|Manufacturer#1|Brand#14|MEDIUM PLATED NICKEL|7|LG PACK|1440.54|cial, pending account 541|magenta tan antique sky pale|Manufacturer#3|Brand#35|STANDARD ANODIZED NICKEL|34|LG JAR|1441.54|s are fluffily above t 542|light lace gainsboro coral lavender|Manufacturer#1|Brand#11|SMALL BRUSHED STEEL|40|WRAP CASE|1442.54|sits bo 543|sky powder saddle metallic moccasin|Manufacturer#5|Brand#54|ECONOMY POLISHED BRASS|4|JUMBO BOX|1443.54| cajole 544|peach red azure indian lavender|Manufacturer#2|Brand#22|STANDARD BURNISHED NICKEL|42|SM CASE|1444.54|leep slyl 545|purple peru ivory gainsboro peach|Manufacturer#3|Brand#31|LARGE PLATED NICKEL|14|MED JAR|1445.54|arefully regular packa 546|metallic grey black rosy papaya|Manufacturer#2|Brand#25|ECONOMY ANODIZED STEEL|8|LG CASE|1446.54|ly unusu 547|white turquoise olive cornflower pale|Manufacturer#4|Brand#43|MEDIUM BURNISHED COPPER|48|WRAP DRUM|1447.54|ial theodolites among 548|cyan lime lawn misty mint|Manufacturer#3|Brand#33|ECONOMY ANODIZED TIN|28|SM CAN|1448.54|gular 549|thistle green indian sandy purple|Manufacturer#5|Brand#53|SMALL BRUSHED BRASS|36|SM CAN|1449.54|ly unus 550|brown blush dark white mint|Manufacturer#4|Brand#44|STANDARD BRUSHED STEEL|27|SM JAR|1450.55|arefully iron 551|tan misty floral firebrick puff|Manufacturer#5|Brand#54|LARGE BURNISHED TIN|10|MED JAR|1451.55|dolites cajole furious 552|lime violet puff snow chiffon|Manufacturer#4|Brand#44|LARGE POLISHED BRASS|1|JUMBO CAN|1452.55|longside of the sl 553|gainsboro tomato lawn peru bisque|Manufacturer#2|Brand#23|SMALL POLISHED NICKEL|1|WRAP DRUM|1453.55|fily regular depen 554|blue medium royal blush ghost|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|31|LG JAR|1454.55|ts? ironically 555|midnight honeydew brown beige slate|Manufacturer#4|Brand#42|PROMO BURNISHED TIN|49|SM JAR|1455.55|es hag 556|frosted antique aquamarine seashell powder|Manufacturer#2|Brand#23|SMALL BRUSHED BRASS|49|WRAP JAR|1456.55|ar theodolites ma 557|firebrick ghost steel green chartreuse|Manufacturer#3|Brand#35|LARGE BURNISHED TIN|8|JUMBO PKG|1457.55|ccounts im 558|blanched seashell navy black dim|Manufacturer#3|Brand#31|STANDARD PLATED BRASS|25|MED BOX|1458.55|eposits. slyly 559|wheat coral navajo cornflower dark|Manufacturer#3|Brand#31|LARGE PLATED STEEL|27|SM PKG|1459.55|stealthily express 560|tan indian lime olive dodger|Manufacturer#2|Brand#25|STANDARD BRUSHED COPPER|37|WRAP PKG|1460.56|e special pac 561|firebrick slate grey coral goldenrod|Manufacturer#1|Brand#14|SMALL POLISHED STEEL|44|JUMBO CASE|1461.56|-- ironic, iron 562|tan blue thistle forest cornsilk|Manufacturer#3|Brand#32|SMALL BURNISHED STEEL|16|MED PACK|1462.56| fluffi 563|blanched light indian goldenrod dim|Manufacturer#3|Brand#31|MEDIUM POLISHED STEEL|39|MED DRUM|1463.56|furiously bold, 564|drab bisque puff ghost floral|Manufacturer#2|Brand#22|LARGE ANODIZED BRASS|41|SM PKG|1464.56|blithely after the ev 565|coral mint sky rosy red|Manufacturer#1|Brand#14|PROMO BRUSHED STEEL|9|JUMBO JAR|1465.56| blithely unusual r 566|seashell dark yellow turquoise white|Manufacturer#2|Brand#25|ECONOMY BURNISHED TIN|19|LG PKG|1466.56|ounts wake across t 567|cornsilk lace white medium honeydew|Manufacturer#5|Brand#55|STANDARD BRUSHED STEEL|19|JUMBO BAG|1467.56|pending pinto 568|purple burlywood pale cornsilk chartreuse|Manufacturer#3|Brand#31|PROMO POLISHED STEEL|6|SM BOX|1468.56|ts. ideas along t 569|cornflower burlywood blue spring purple|Manufacturer#4|Brand#41|SMALL BRUSHED TIN|46|MED BOX|1469.56|y according to 570|cyan navajo turquoise forest firebrick|Manufacturer#5|Brand#51|ECONOMY BRUSHED COPPER|44|SM PKG|1470.57|le. s 571|medium orchid purple metallic ghost|Manufacturer#2|Brand#22|LARGE PLATED TIN|1|WRAP CAN|1471.57|ly ironic 572|navy plum lime goldenrod saddle|Manufacturer#4|Brand#43|LARGE BURNISHED COPPER|3|SM BOX|1472.57|requests use 573|dark midnight forest lace sandy|Manufacturer#2|Brand#23|SMALL PLATED TIN|42|WRAP PACK|1473.57|expres 574|bisque snow turquoise lace blush|Manufacturer#2|Brand#23|SMALL PLATED BRASS|36|SM PACK|1474.57|oss th 575|brown seashell hot lavender frosted|Manufacturer#1|Brand#14|SMALL BRUSHED NICKEL|10|MED CAN|1475.57|ns integrate a 576|puff lavender black yellow lawn|Manufacturer#3|Brand#32|PROMO BURNISHED TIN|27|SM CAN|1476.57| foxes. accounts bo 577|lime green dark misty chiffon|Manufacturer#2|Brand#22|LARGE BRUSHED STEEL|34|JUMBO BAG|1477.57|ding, fur 578|black lawn light plum blush|Manufacturer#4|Brand#41|MEDIUM ANODIZED NICKEL|9|WRAP DRUM|1478.57| furiously iron 579|spring brown peach bisque sienna|Manufacturer#3|Brand#31|STANDARD POLISHED STEEL|8|WRAP JAR|1479.57|ades. carefully 580|navajo chocolate lemon hot green|Manufacturer#1|Brand#12|LARGE BURNISHED COPPER|30|JUMBO PACK|1480.58|usly expre 581|frosted gainsboro chiffon lawn green|Manufacturer#2|Brand#25|ECONOMY ANODIZED BRASS|44|SM BAG|1481.58| furious foxes. silen 582|tan cyan peach rose salmon|Manufacturer#4|Brand#45|LARGE BRUSHED TIN|23|MED JAR|1482.58|y bold 583|turquoise burnished orchid firebrick blush|Manufacturer#3|Brand#33|LARGE BURNISHED NICKEL|35|JUMBO CAN|1483.58| sleep 584|goldenrod magenta papaya medium chiffon|Manufacturer#1|Brand#12|SMALL POLISHED COPPER|42|MED JAR|1484.58|lar, ironic ac 585|slate hot honeydew midnight burnished|Manufacturer#5|Brand#53|SMALL PLATED COPPER|7|JUMBO PKG|1485.58|sts haggle q 586|ivory rosy royal powder firebrick|Manufacturer#1|Brand#11|PROMO ANODIZED BRASS|11|LG DRUM|1486.58|ts haggle fluffily 587|blanched maroon blush burlywood salmon|Manufacturer#4|Brand#44|SMALL POLISHED STEEL|35|MED PKG|1487.58|c accounts. qui 588|cyan powder brown sandy burlywood|Manufacturer#5|Brand#55|SMALL PLATED COPPER|45|WRAP BAG|1488.58|ss foxes. even, ex 589|red orange salmon rosy thistle|Manufacturer#1|Brand#13|LARGE POLISHED BRASS|3|SM PKG|1489.58|cies. regul 590|azure burnished honeydew navy cyan|Manufacturer#4|Brand#42|LARGE POLISHED COPPER|38|WRAP PKG|1490.59| dependencies sho 591|sienna burlywood salmon navy green|Manufacturer#1|Brand#14|ECONOMY PLATED NICKEL|40|SM PKG|1491.59| the b 592|dodger burnished yellow wheat antique|Manufacturer#2|Brand#23|LARGE POLISHED STEEL|7|MED DRUM|1492.59|ide of the notorni 593|metallic sienna sandy spring frosted|Manufacturer#2|Brand#25|PROMO POLISHED STEEL|20|JUMBO CASE|1493.59|ake after 594|seashell magenta rosy cornsilk smoke|Manufacturer#3|Brand#31|PROMO BRUSHED BRASS|28|SM JAR|1494.59|s detect fu 595|chiffon lace linen steel bisque|Manufacturer#5|Brand#51|PROMO BRUSHED BRASS|23|JUMBO JAR|1495.59|yly fin 596|drab snow chartreuse dim ivory|Manufacturer#5|Brand#52|STANDARD POLISHED TIN|35|JUMBO JAR|1496.59|epend 597|cyan orchid grey tan bisque|Manufacturer#4|Brand#44|STANDARD ANODIZED NICKEL|36|SM DRUM|1497.59|ronic foxes. speci 598|deep royal drab violet chartreuse|Manufacturer#2|Brand#24|MEDIUM POLISHED BRASS|9|MED JAR|1498.59|he fluffily regular 599|cornsilk peru steel beige blue|Manufacturer#5|Brand#51|MEDIUM BRUSHED NICKEL|30|JUMBO PACK|1499.59|ckages against t 600|puff bisque midnight seashell burlywood|Manufacturer#4|Brand#42|STANDARD POLISHED NICKEL|4|SM JAR|1500.60|ow furiously agai 601|coral thistle blanched linen tan|Manufacturer#1|Brand#13|STANDARD POLISHED BRASS|13|SM PKG|1501.60|lithely thi 602|metallic smoke purple rosy magenta|Manufacturer#1|Brand#12|PROMO BRUSHED COPPER|8|SM BAG|1502.60|telets a 603|dim blush puff chiffon salmon|Manufacturer#1|Brand#12|PROMO ANODIZED COPPER|41|WRAP PKG|1503.60|uests. fluffily regul 604|bisque light frosted white olive|Manufacturer#4|Brand#41|MEDIUM PLATED STEEL|19|MED PACK|1504.60|are after the regula 605|grey burlywood linen tan tomato|Manufacturer#1|Brand#11|LARGE POLISHED STEEL|30|LG JAR|1505.60| bold, final accou 606|navajo lace sandy aquamarine light|Manufacturer#3|Brand#32|STANDARD PLATED STEEL|36|LG DRUM|1506.60|hins p 607|dark sienna thistle sandy maroon|Manufacturer#5|Brand#54|MEDIUM POLISHED TIN|13|MED JAR|1507.60|ctions wake qu 608|mint medium blush black metallic|Manufacturer#2|Brand#25|PROMO PLATED BRASS|35|JUMBO PKG|1508.60|xes. express r 609|almond lemon olive smoke moccasin|Manufacturer#2|Brand#21|ECONOMY BRUSHED TIN|49|JUMBO DRUM|1509.60|cial deposits haggle 610|plum purple seashell white lemon|Manufacturer#1|Brand#11|ECONOMY BURNISHED TIN|38|JUMBO BOX|1510.61|slyly pending re 611|light burnished linen sienna dim|Manufacturer#4|Brand#42|MEDIUM PLATED TIN|45|SM DRUM|1511.61|iously even packages. 612|midnight azure moccasin rosy lawn|Manufacturer#4|Brand#42|PROMO PLATED STEEL|19|LG BOX|1512.61|se quickly; 613|cream coral sandy thistle spring|Manufacturer#3|Brand#34|LARGE BURNISHED COPPER|22|LG DRUM|1513.61| cajo 614|dodger beige maroon puff medium|Manufacturer#2|Brand#23|STANDARD BURNISHED NICKEL|6|WRAP PKG|1514.61|ending accounts ca 615|rosy seashell brown blush tan|Manufacturer#4|Brand#41|PROMO ANODIZED COPPER|4|WRAP PKG|1515.61| dogged 616|tan powder lavender dim goldenrod|Manufacturer#2|Brand#25|STANDARD BRUSHED STEEL|1|WRAP JAR|1516.61|w packages cajole b 617|peru grey snow papaya firebrick|Manufacturer#2|Brand#22|MEDIUM ANODIZED NICKEL|37|WRAP DRUM|1517.61|furious 618|ivory wheat smoke deep lemon|Manufacturer#2|Brand#22|PROMO BRUSHED BRASS|9|LG BOX|1518.61|ogs. furiously 619|powder medium drab lemon tomato|Manufacturer#3|Brand#34|LARGE ANODIZED STEEL|46|MED JAR|1519.61|cuses are 620|frosted tomato almond salmon powder|Manufacturer#3|Brand#35|MEDIUM BRUSHED COPPER|4|LG PKG|1520.62|efully carefully speci 621|almond azure drab ghost mint|Manufacturer#5|Brand#53|PROMO PLATED TIN|47|JUMBO DRUM|1521.62|ffily furiously fin 622|burlywood dark blue thistle navajo|Manufacturer#5|Brand#51|LARGE POLISHED STEEL|19|WRAP BAG|1522.62|sly deposits. f 623|chiffon bisque steel orange tomato|Manufacturer#3|Brand#32|ECONOMY POLISHED NICKEL|8|MED BOX|1523.62|ly ironic 624|burlywood ghost orchid misty coral|Manufacturer#1|Brand#12|LARGE BURNISHED NICKEL|29|SM BAG|1524.62|dencies. even d 625|beige indian lime sienna plum|Manufacturer#3|Brand#35|SMALL PLATED COPPER|38|LG JAR|1525.62|blithely slow multipl 626|cream ivory brown peach honeydew|Manufacturer#1|Brand#12|SMALL ANODIZED COPPER|23|SM JAR|1526.62|furiously-- 627|drab yellow spring turquoise rose|Manufacturer#4|Brand#44|PROMO ANODIZED NICKEL|13|SM BAG|1527.62|y ironic 628|black mint olive drab medium|Manufacturer#3|Brand#33|ECONOMY BRUSHED BRASS|40|LG BOX|1528.62|sits. 629|magenta lace red hot pale|Manufacturer#5|Brand#51|MEDIUM POLISHED COPPER|25|WRAP BOX|1529.62|fully even co 630|lawn lace burnished hot frosted|Manufacturer#2|Brand#25|MEDIUM PLATED NICKEL|20|LG DRUM|1530.63|g to th 631|moccasin lawn cornsilk chartreuse midnight|Manufacturer#2|Brand#22|STANDARD ANODIZED NICKEL|34|WRAP BAG|1531.63|ely silen 632|blush floral tan thistle chiffon|Manufacturer#3|Brand#34|PROMO PLATED NICKEL|2|MED PACK|1532.63|y ironic instru 633|navy burnished wheat slate deep|Manufacturer#3|Brand#33|ECONOMY POLISHED NICKEL|24|MED PKG|1533.63|bold brai 634|grey aquamarine deep violet blush|Manufacturer#3|Brand#35|SMALL ANODIZED NICKEL|34|MED DRUM|1534.63|ions. 635|ghost aquamarine gainsboro lemon peach|Manufacturer#5|Brand#51|PROMO POLISHED STEEL|49|WRAP JAR|1535.63|y ironic excuses. 636|burlywood deep antique snow wheat|Manufacturer#5|Brand#55|PROMO BRUSHED STEEL|36|SM PACK|1536.63|ithely 637|brown smoke sandy honeydew antique|Manufacturer#5|Brand#51|LARGE BURNISHED NICKEL|48|JUMBO CASE|1537.63|oxes after the blithel 638|lace royal pink cornsilk antique|Manufacturer#5|Brand#54|SMALL PLATED TIN|37|JUMBO PACK|1538.63| carefully. blithely 639|black plum tomato cornflower medium|Manufacturer#2|Brand#25|STANDARD POLISHED BRASS|33|SM BAG|1539.63|s cajole quickly. care 640|grey seashell bisque indian deep|Manufacturer#1|Brand#12|MEDIUM BRUSHED TIN|20|WRAP CASE|1540.64|ly regular acco 641|mint saddle yellow sienna frosted|Manufacturer#5|Brand#55|MEDIUM PLATED COPPER|18|SM JAR|1541.64| bold instr 642|moccasin ghost sandy goldenrod cornsilk|Manufacturer#4|Brand#43|ECONOMY BRUSHED BRASS|12|MED PACK|1542.64| bold packages bey 643|frosted chocolate dodger honeydew ghost|Manufacturer#3|Brand#32|MEDIUM POLISHED STEEL|8|MED DRUM|1543.64|refully fina 644|rosy bisque hot burnished dark|Manufacturer#5|Brand#52|LARGE PLATED STEEL|34|SM PACK|1544.64|lphins. blithely 645|thistle sky antique khaki chartreuse|Manufacturer#4|Brand#45|MEDIUM BURNISHED NICKEL|45|LG CASE|1545.64| the theodol 646|tan honeydew lime light white|Manufacturer#5|Brand#52|SMALL POLISHED COPPER|21|LG PACK|1546.64|ajole according to t 647|rose khaki drab smoke peach|Manufacturer#3|Brand#35|LARGE BURNISHED STEEL|38|MED PKG|1547.64|ters. ironic pinto 648|bisque blue drab cyan almond|Manufacturer#3|Brand#34|SMALL BRUSHED STEEL|2|WRAP JAR|1548.64|eas. regular 649|lavender blush rosy beige sky|Manufacturer#1|Brand#12|STANDARD BURNISHED TIN|33|JUMBO DRUM|1549.64|fully! 650|bisque navajo mint medium seashell|Manufacturer#5|Brand#53|PROMO ANODIZED NICKEL|40|MED JAR|1550.65|ar dependen 651|bisque floral dim burlywood moccasin|Manufacturer#5|Brand#52|MEDIUM POLISHED BRASS|4|WRAP BAG|1551.65|efully final instruct 652|navajo white indian yellow grey|Manufacturer#1|Brand#12|LARGE BRUSHED TIN|39|WRAP PKG|1552.65|le furiously unusua 653|lime cornflower sky beige antique|Manufacturer#1|Brand#15|ECONOMY BURNISHED BRASS|28|JUMBO DRUM|1553.65|ias. i 654|cyan burnished tomato chiffon navy|Manufacturer#2|Brand#23|PROMO PLATED COPPER|2|WRAP BAG|1554.65|posits haggle 655|tomato firebrick yellow rosy orchid|Manufacturer#1|Brand#12|STANDARD BURNISHED TIN|23|JUMBO CAN|1555.65|ic instructions. 656|maroon dodger ghost gainsboro pink|Manufacturer#1|Brand#15|STANDARD BURNISHED COPPER|17|MED CASE|1556.65|tions wake ca 657|saddle lawn blue cyan cornsilk|Manufacturer#3|Brand#31|ECONOMY BURNISHED COPPER|36|LG PACK|1557.65| packages accord 658|ghost mint hot bisque salmon|Manufacturer#1|Brand#15|MEDIUM PLATED COPPER|27|SM JAR|1558.65| somas haggle quickly 659|black bisque plum pale cyan|Manufacturer#3|Brand#34|MEDIUM BRUSHED BRASS|20|LG JAR|1559.65|across the 660|sky dim thistle saddle pale|Manufacturer#5|Brand#51|STANDARD POLISHED TIN|8|SM CAN|1560.66|ages cajole fluffily 661|snow beige moccasin red lime|Manufacturer#3|Brand#32|PROMO PLATED STEEL|35|SM PKG|1561.66|ng asympt 662|ivory chocolate slate midnight ghost|Manufacturer#4|Brand#45|STANDARD PLATED STEEL|35|LG PACK|1562.66|e eve 663|ivory orchid azure frosted light|Manufacturer#4|Brand#41|PROMO POLISHED COPPER|6|WRAP PACK|1563.66| beans 664|frosted burnished tomato chiffon seashell|Manufacturer#4|Brand#43|MEDIUM BURNISHED TIN|26|LG BAG|1564.66|about the slyly s 665|rose antique cyan cornflower drab|Manufacturer#1|Brand#13|MEDIUM POLISHED STEEL|14|MED DRUM|1565.66|usly final excuse 666|ivory dim orchid antique spring|Manufacturer#1|Brand#13|ECONOMY BURNISHED NICKEL|20|MED PACK|1566.66|ronic theodolites ca 667|wheat ghost honeydew plum grey|Manufacturer#4|Brand#43|MEDIUM PLATED STEEL|42|MED JAR|1567.66|ly pending packages. 668|honeydew pink dodger cream dim|Manufacturer#2|Brand#22|SMALL ANODIZED NICKEL|10|LG PACK|1568.66|odolites. furio 669|khaki metallic plum smoke hot|Manufacturer#1|Brand#11|STANDARD BRUSHED TIN|37|MED BAG|1569.66|bold deposi 670|slate black smoke pale moccasin|Manufacturer#1|Brand#12|ECONOMY PLATED COPPER|24|LG DRUM|1570.67|es use fluffily unusu 671|powder metallic salmon slate chocolate|Manufacturer#5|Brand#53|SMALL PLATED STEEL|31|JUMBO BOX|1571.67|es. s 672|lawn rosy cornsilk floral misty|Manufacturer#2|Brand#23|PROMO BURNISHED STEEL|13|LG PACK|1572.67|, pen 673|navy azure sandy yellow sky|Manufacturer#2|Brand#23|MEDIUM ANODIZED COPPER|26|SM DRUM|1573.67|y final 674|tomato chartreuse cornflower green pale|Manufacturer#3|Brand#35|SMALL BRUSHED TIN|5|LG PKG|1574.67|unts use slyly bold 675|misty hot pale ghost yellow|Manufacturer#1|Brand#11|LARGE BRUSHED COPPER|2|SM CAN|1575.67|ely re 676|moccasin cornflower burlywood tomato light|Manufacturer#1|Brand#11|MEDIUM ANODIZED BRASS|45|SM CAN|1576.67|ges. enticingly ironi 677|pale steel blush mint cream|Manufacturer#4|Brand#44|PROMO ANODIZED BRASS|6|WRAP BAG|1577.67|accounts 678|salmon ivory red dodger spring|Manufacturer#4|Brand#44|PROMO BRUSHED TIN|15|SM BAG|1578.67|ronic pin 679|purple blanched linen metallic indian|Manufacturer#4|Brand#41|SMALL BURNISHED TIN|50|MED BOX|1579.67|iously ironic in 680|burnished ghost coral maroon yellow|Manufacturer#1|Brand#14|SMALL PLATED COPPER|44|LG PACK|1580.68|l depo 681|dark slate almond ghost chartreuse|Manufacturer#3|Brand#32|ECONOMY PLATED COPPER|9|WRAP CAN|1581.68|ic requests wake accor 682|lawn khaki green cornflower sienna|Manufacturer#5|Brand#53|LARGE POLISHED NICKEL|48|SM BOX|1582.68|ackages. 683|sienna cornsilk chiffon olive blush|Manufacturer#5|Brand#53|LARGE BURNISHED NICKEL|1|MED PACK|1583.68|its hag 684|metallic azure hot orange spring|Manufacturer#3|Brand#33|PROMO BURNISHED TIN|6|WRAP BOX|1584.68|unts are fluff 685|turquoise orchid plum green tomato|Manufacturer#3|Brand#31|SMALL PLATED COPPER|36|LG JAR|1585.68|s. furiously ruthless 686|goldenrod deep cornflower dodger rose|Manufacturer#5|Brand#54|STANDARD BRUSHED COPPER|15|WRAP BAG|1586.68|lly ironic accounts 687|frosted cornsilk tomato burnished smoke|Manufacturer#5|Brand#51|LARGE POLISHED NICKEL|47|JUMBO BOX|1587.68|packages. even, 688|navajo khaki almond royal chartreuse|Manufacturer#4|Brand#45|PROMO BRUSHED COPPER|38|JUMBO CAN|1588.68|rious ideas 689|bisque red peru almond grey|Manufacturer#4|Brand#43|LARGE PLATED COPPER|22|SM PACK|1589.68|ual package 690|cornflower dodger saddle ivory tan|Manufacturer#3|Brand#32|SMALL PLATED BRASS|31|WRAP PACK|1590.69|al, bold fra 691|burnished dim gainsboro thistle blue|Manufacturer#3|Brand#35|PROMO BURNISHED BRASS|3|SM JAR|1591.69|ully slyly unusu 692|puff beige smoke seashell sienna|Manufacturer#1|Brand#14|MEDIUM BRUSHED BRASS|20|SM DRUM|1592.69|tructions. ev 693|honeydew ghost azure yellow magenta|Manufacturer#5|Brand#54|SMALL PLATED TIN|14|WRAP CASE|1593.69|lets sle 694|pink maroon blanched beige cyan|Manufacturer#3|Brand#35|ECONOMY PLATED STEEL|35|JUMBO CASE|1594.69|es. final 695|deep peru lavender antique royal|Manufacturer#2|Brand#23|PROMO ANODIZED COPPER|19|MED DRUM|1595.69|eep blithely 696|forest lemon cream black pink|Manufacturer#3|Brand#33|MEDIUM ANODIZED BRASS|3|LG CAN|1596.69|al acco 697|dark white slate honeydew maroon|Manufacturer#4|Brand#41|LARGE PLATED TIN|8|MED CAN|1597.69| special pa 698|blush sienna honeydew yellow goldenrod|Manufacturer#4|Brand#42|ECONOMY ANODIZED COPPER|44|LG PACK|1598.69|ronic, ironic 699|plum orchid red linen misty|Manufacturer#1|Brand#11|MEDIUM PLATED STEEL|5|WRAP CAN|1599.69| regular notornis b 700|lace hot khaki steel orchid|Manufacturer#3|Brand#34|PROMO BRUSHED STEEL|31|SM JAR|1600.70|ly even foxes. fi 701|frosted lavender black deep ghost|Manufacturer#4|Brand#43|MEDIUM ANODIZED COPPER|36|WRAP DRUM|1601.70| use blithe 702|navajo lavender dim puff bisque|Manufacturer#4|Brand#41|STANDARD BURNISHED BRASS|26|SM CAN|1602.70|yly f 703|puff floral red linen dark|Manufacturer#3|Brand#32|ECONOMY POLISHED TIN|20|WRAP CASE|1603.70|nstructions wake alon 704|salmon lawn chocolate lace honeydew|Manufacturer#3|Brand#34|LARGE ANODIZED BRASS|23|MED CAN|1604.70|l, ironic 705|snow drab lawn dark tan|Manufacturer#3|Brand#33|ECONOMY PLATED TIN|47|LG PKG|1605.70|ironic 706|cream white navajo frosted puff|Manufacturer#3|Brand#33|LARGE PLATED BRASS|6|SM PKG|1606.70|quests. i 707|salmon khaki misty deep peru|Manufacturer#1|Brand#11|SMALL POLISHED BRASS|17|SM BAG|1607.70|osits about the fluf 708|brown midnight plum violet tomato|Manufacturer#3|Brand#32|SMALL BRUSHED NICKEL|29|SM DRUM|1608.70|ully final instru 709|tomato sienna cornflower black misty|Manufacturer#4|Brand#43|SMALL BRUSHED TIN|16|SM BAG|1609.70|ns. blithely i 710|chartreuse thistle midnight magenta violet|Manufacturer#1|Brand#15|SMALL POLISHED TIN|27|JUMBO PACK|1610.71|fluffily unu 711|goldenrod sienna linen steel floral|Manufacturer#1|Brand#11|PROMO BRUSHED NICKEL|24|LG DRUM|1611.71|s hang slyly regula 712|blanched rose royal tomato hot|Manufacturer#3|Brand#33|ECONOMY BRUSHED COPPER|25|SM CASE|1612.71|even, fluffy 713|gainsboro plum powder seashell lawn|Manufacturer#2|Brand#25|SMALL ANODIZED COPPER|20|SM CAN|1613.71|onic de 714|chartreuse medium gainsboro honeydew saddle|Manufacturer#1|Brand#11|MEDIUM POLISHED NICKEL|35|SM JAR|1614.71|ions. furiously final 715|deep yellow coral sienna white|Manufacturer#2|Brand#24|LARGE BRUSHED TIN|28|WRAP BAG|1615.71|ons boost. furiou 716|grey floral sienna cyan gainsboro|Manufacturer#1|Brand#12|SMALL POLISHED TIN|37|JUMBO CAN|1616.71|osits! packages hag 717|papaya turquoise spring midnight medium|Manufacturer#2|Brand#22|STANDARD PLATED TIN|28|JUMBO JAR|1617.71| final notornis na 718|khaki navy saddle ghost orchid|Manufacturer#3|Brand#35|PROMO BURNISHED COPPER|24|SM CAN|1618.71|ests. final, exp 719|chocolate honeydew khaki aquamarine white|Manufacturer#3|Brand#35|STANDARD BRUSHED NICKEL|49|JUMBO BAG|1619.71|ect. furiously 720|light khaki dodger dark lawn|Manufacturer#4|Brand#45|SMALL BURNISHED NICKEL|8|JUMBO BAG|1620.72|as. furiously 721|gainsboro chocolate maroon hot sienna|Manufacturer#5|Brand#52|STANDARD ANODIZED STEEL|33|MED PKG|1621.72|ly special instruct 722|forest chartreuse plum royal papaya|Manufacturer#4|Brand#43|ECONOMY BURNISHED NICKEL|43|JUMBO BAG|1622.72|ic packages. fin 723|slate magenta lemon peru pink|Manufacturer#3|Brand#31|LARGE BRUSHED BRASS|34|WRAP CASE|1623.72|unts. excu 724|rosy coral thistle ghost papaya|Manufacturer#3|Brand#35|PROMO POLISHED COPPER|28|JUMBO BOX|1624.72| carefully pend 725|hot blanched salmon slate burlywood|Manufacturer#4|Brand#43|LARGE PLATED NICKEL|11|JUMBO BOX|1625.72|r theo 726|cream aquamarine violet medium midnight|Manufacturer#4|Brand#43|PROMO POLISHED NICKEL|41|JUMBO BAG|1626.72|arefully? carefully i 727|royal pink smoke bisque lime|Manufacturer#3|Brand#35|PROMO PLATED BRASS|35|SM PKG|1627.72| about the furio 728|cornsilk grey sienna blue sky|Manufacturer#3|Brand#33|ECONOMY PLATED TIN|27|LG JAR|1628.72|fily regular package 729|papaya cornsilk gainsboro rose pink|Manufacturer#3|Brand#35|ECONOMY BRUSHED TIN|6|LG JAR|1629.72|egular accou 730|rose blush saddle cyan hot|Manufacturer#4|Brand#45|MEDIUM PLATED TIN|8|SM PKG|1630.73|ironic 731|firebrick indian forest rose khaki|Manufacturer#3|Brand#33|PROMO BURNISHED NICKEL|34|MED CASE|1631.73|snooze above the 732|violet frosted red papaya drab|Manufacturer#2|Brand#21|MEDIUM BRUSHED BRASS|1|SM BAG|1632.73|. sauternes haggl 733|burnished goldenrod royal plum midnight|Manufacturer#4|Brand#43|PROMO ANODIZED STEEL|1|MED CASE|1633.73| special instruct 734|lace light bisque dodger aquamarine|Manufacturer#2|Brand#21|MEDIUM BRUSHED STEEL|34|WRAP DRUM|1634.73|quests 735|wheat white honeydew almond coral|Manufacturer#4|Brand#42|SMALL BRUSHED STEEL|44|SM CAN|1635.73|kages are according 736|slate honeydew pink magenta lace|Manufacturer#3|Brand#32|PROMO ANODIZED BRASS|44|JUMBO DRUM|1636.73|telets wake carefu 737|orange navajo blanched cyan snow|Manufacturer#5|Brand#53|PROMO POLISHED NICKEL|25|SM BAG|1637.73|re slyly. bold, regul 738|peru floral sky steel maroon|Manufacturer#2|Brand#21|ECONOMY BURNISHED COPPER|29|JUMBO CAN|1638.73|ckages wake. slyly fin 739|powder lavender wheat pale puff|Manufacturer#3|Brand#31|ECONOMY BRUSHED COPPER|24|JUMBO BOX|1639.73|the regular 740|almond aquamarine mint misty red|Manufacturer#3|Brand#35|STANDARD POLISHED COPPER|7|WRAP PACK|1640.74| even dep 741|thistle seashell cyan cornsilk indian|Manufacturer#4|Brand#44|STANDARD PLATED TIN|33|WRAP PACK|1641.74|oxes slee 742|dodger cream snow pink floral|Manufacturer#5|Brand#51|STANDARD BRUSHED NICKEL|30|SM PACK|1642.74|out the furiously 743|navajo midnight peru orange salmon|Manufacturer#4|Brand#42|LARGE ANODIZED COPPER|4|MED JAR|1643.74|al packages ar 744|burlywood deep hot turquoise chartreuse|Manufacturer#5|Brand#53|MEDIUM BURNISHED COPPER|47|MED PKG|1644.74|uriously brave ideas 745|seashell dark lime midnight puff|Manufacturer#3|Brand#31|PROMO POLISHED COPPER|25|JUMBO BOX|1645.74| silent 746|honeydew brown sienna red chocolate|Manufacturer#2|Brand#23|ECONOMY ANODIZED STEEL|31|MED CAN|1646.74|osits affix furiously 747|lace red hot mint goldenrod|Manufacturer#1|Brand#11|LARGE BURNISHED NICKEL|20|WRAP PKG|1647.74|deposits b 748|forest indian snow grey peru|Manufacturer#5|Brand#54|STANDARD BURNISHED BRASS|32|SM CAN|1648.74|arefully bo 749|snow goldenrod puff violet mint|Manufacturer#2|Brand#24|SMALL PLATED COPPER|16|MED JAR|1649.74| bold re 750|puff slate magenta powder lawn|Manufacturer#4|Brand#43|LARGE POLISHED NICKEL|47|MED PACK|1650.75|tipliers about the q 751|steel blush indian rosy pink|Manufacturer#1|Brand#11|SMALL POLISHED TIN|17|SM DRUM|1651.75|uests haggle bl 752|medium blue midnight misty frosted|Manufacturer#5|Brand#53|MEDIUM ANODIZED COPPER|1|MED PACK|1652.75|lites mold b 753|blanched plum navajo beige indian|Manufacturer#2|Brand#25|STANDARD ANODIZED TIN|13|LG DRUM|1653.75|nal foxes. quickly f 754|thistle steel sky light red|Manufacturer#5|Brand#51|SMALL ANODIZED STEEL|42|WRAP JAR|1654.75|es. blithely dogge 755|spring dodger floral metallic moccasin|Manufacturer#2|Brand#22|STANDARD PLATED TIN|12|WRAP PACK|1655.75| wake. even packages a 756|ghost antique snow cream cyan|Manufacturer#5|Brand#52|STANDARD PLATED BRASS|44|MED BAG|1656.75|rious 757|papaya dim slate saddle navajo|Manufacturer#3|Brand#35|ECONOMY POLISHED NICKEL|12|JUMBO CASE|1657.75|the quickly 758|yellow orchid dim cyan burlywood|Manufacturer#1|Brand#11|ECONOMY ANODIZED TIN|25|MED BOX|1658.75|haggle 759|yellow powder navajo maroon chartreuse|Manufacturer#3|Brand#34|LARGE BURNISHED TIN|20|JUMBO CAN|1659.75|struction 760|orange navy hot aquamarine sienna|Manufacturer#4|Brand#44|MEDIUM POLISHED COPPER|6|LG JAR|1660.76|jole according 761|seashell green dodger beige linen|Manufacturer#3|Brand#34|ECONOMY ANODIZED TIN|7|SM JAR|1661.76|ages according 762|peach grey firebrick dim smoke|Manufacturer#1|Brand#15|LARGE POLISHED TIN|35|MED PKG|1662.76|xpress ideas. fluff 763|wheat seashell azure chartreuse dodger|Manufacturer#4|Brand#44|LARGE BRUSHED TIN|50|SM PKG|1663.76|counts. regu 764|cyan moccasin blanched light purple|Manufacturer#1|Brand#13|SMALL ANODIZED NICKEL|11|LG JAR|1664.76|es. final, bold 765|thistle red smoke chartreuse orange|Manufacturer#3|Brand#35|STANDARD BRUSHED BRASS|3|WRAP BAG|1665.76|ly regular pint 766|midnight sienna orange gainsboro black|Manufacturer#2|Brand#24|MEDIUM BURNISHED NICKEL|20|MED PACK|1666.76|inal ideas. asy 767|blush firebrick misty blanched purple|Manufacturer#2|Brand#24|LARGE POLISHED TIN|50|MED DRUM|1667.76|ts. carefully unu 768|maroon gainsboro seashell hot sandy|Manufacturer#4|Brand#43|LARGE BRUSHED COPPER|41|SM DRUM|1668.76|olites haggle: car 769|tomato royal firebrick turquoise cream|Manufacturer#2|Brand#24|MEDIUM BRUSHED TIN|8|LG CASE|1669.76|ake carefull 770|lemon yellow coral deep lime|Manufacturer#2|Brand#25|LARGE BRUSHED TIN|7|SM PACK|1670.77|ave ideas. 771|plum maroon lavender tan firebrick|Manufacturer#1|Brand#15|SMALL BURNISHED TIN|17|LG DRUM|1671.77| cajole slyly fina 772|dodger firebrick peach ivory seashell|Manufacturer#3|Brand#35|SMALL PLATED BRASS|33|MED PKG|1672.77|dolites haggle sp 773|saddle medium beige purple plum|Manufacturer#3|Brand#32|PROMO BURNISHED STEEL|16|LG CASE|1673.77| final, sp 774|wheat chiffon cyan misty moccasin|Manufacturer#2|Brand#24|STANDARD ANODIZED NICKEL|26|WRAP CASE|1674.77|ly express dependen 775|papaya misty orchid snow metallic|Manufacturer#1|Brand#14|LARGE PLATED TIN|2|MED PACK|1675.77|azzle carefully 776|powder indian dodger hot lemon|Manufacturer#5|Brand#51|STANDARD ANODIZED BRASS|21|WRAP DRUM|1676.77|egular orbits haggl 777|blanched indian pink frosted grey|Manufacturer#5|Brand#53|SMALL ANODIZED STEEL|50|JUMBO JAR|1677.77| theodolites 778|saddle mint navy cyan cornflower|Manufacturer#3|Brand#32|PROMO BURNISHED STEEL|20|WRAP BOX|1678.77|nic re 779|salmon burnished orange rose cornsilk|Manufacturer#2|Brand#22|ECONOMY BRUSHED STEEL|26|WRAP DRUM|1679.77| regular epitaphs are 780|rosy chocolate tan moccasin salmon|Manufacturer#5|Brand#55|MEDIUM ANODIZED STEEL|31|MED CASE|1680.78|t quickly sly 781|light orchid purple black navy|Manufacturer#1|Brand#13|ECONOMY PLATED STEEL|22|LG DRUM|1681.78|gular, regu 782|peru firebrick coral chartreuse rosy|Manufacturer#1|Brand#13|LARGE POLISHED BRASS|29|SM BAG|1682.78|hely u 783|cyan turquoise azure pink dark|Manufacturer#1|Brand#13|SMALL PLATED NICKEL|4|MED DRUM|1683.78|ss acc 784|light smoke seashell snow chartreuse|Manufacturer#1|Brand#12|MEDIUM BRUSHED COPPER|47|LG DRUM|1684.78|unts. furiously 785|coral saddle indian lime frosted|Manufacturer#3|Brand#33|PROMO ANODIZED TIN|41|SM PACK|1685.78|use carefully 786|royal azure ivory moccasin salmon|Manufacturer#2|Brand#23|STANDARD PLATED NICKEL|20|SM PACK|1686.78|eodoli 787|pale metallic ivory peach slate|Manufacturer#5|Brand#55|SMALL ANODIZED NICKEL|35|WRAP PKG|1687.78|es alongside of 788|floral dark dodger chartreuse lavender|Manufacturer#3|Brand#34|LARGE PLATED STEEL|10|LG JAR|1688.78|s solv 789|dodger spring antique lace papaya|Manufacturer#3|Brand#35|PROMO BURNISHED STEEL|14|LG PACK|1689.78|ts hagg 790|lavender yellow pink puff olive|Manufacturer#4|Brand#42|PROMO POLISHED COPPER|35|JUMBO BOX|1690.79|s are 791|indian blush medium thistle lime|Manufacturer#2|Brand#24|STANDARD ANODIZED COPPER|24|MED PACK|1691.79| special pinto bean 792|plum indian cornflower frosted purple|Manufacturer#1|Brand#13|SMALL BURNISHED BRASS|8|MED PKG|1692.79|efully unusual deposi 793|white chiffon blue green violet|Manufacturer#5|Brand#51|LARGE POLISHED NICKEL|28|JUMBO CAN|1693.79|tions. furiou 794|ivory peach light thistle antique|Manufacturer#5|Brand#51|PROMO PLATED COPPER|47|MED PKG|1694.79|le bl 795|cream royal light yellow hot|Manufacturer#5|Brand#54|MEDIUM POLISHED TIN|46|MED JAR|1695.79|ully regula 796|beige frosted cyan hot puff|Manufacturer#5|Brand#51|ECONOMY BRUSHED STEEL|50|WRAP CAN|1696.79|yly fina 797|violet peach puff orange white|Manufacturer#4|Brand#41|STANDARD BURNISHED STEEL|10|WRAP BAG|1697.79|ments 798|turquoise indian white bisque chartreuse|Manufacturer#1|Brand#12|ECONOMY PLATED NICKEL|18|LG JAR|1698.79|ffily even excuses 799|green azure rose aquamarine floral|Manufacturer#3|Brand#35|PROMO POLISHED TIN|7|MED JAR|1699.79|he slyly brave excuses 800|maroon mint medium lace plum|Manufacturer#1|Brand#11|PROMO BRUSHED BRASS|29|WRAP BAG|1700.80|s sleep about the car 801|linen steel salmon beige lemon|Manufacturer#5|Brand#54|LARGE BRUSHED NICKEL|18|JUMBO DRUM|1701.80|latelets. slyly fi 802|rosy ghost cyan puff dark|Manufacturer#3|Brand#34|ECONOMY PLATED BRASS|41|WRAP JAR|1702.80|equest 803|brown navy tan salmon honeydew|Manufacturer#5|Brand#52|SMALL ANODIZED TIN|50|MED PKG|1703.80|ly at the accou 804|bisque grey honeydew goldenrod ghost|Manufacturer#2|Brand#23|SMALL POLISHED BRASS|9|SM BAG|1704.80|itaphs sle 805|tan ghost cyan salmon goldenrod|Manufacturer#5|Brand#52|STANDARD PLATED BRASS|8|JUMBO PKG|1705.80|t blithe 806|blue violet yellow khaki azure|Manufacturer#3|Brand#32|PROMO BURNISHED TIN|45|LG BAG|1706.80| ironic theodolites a 807|turquoise snow navy brown lime|Manufacturer#2|Brand#24|LARGE ANODIZED NICKEL|25|WRAP JAR|1707.80| quietly express pi 808|rosy indian sky frosted blush|Manufacturer#5|Brand#51|SMALL BRUSHED TIN|36|MED PKG|1708.80| pending asymptotes a 809|almond steel maroon chiffon frosted|Manufacturer#3|Brand#32|STANDARD BRUSHED COPPER|17|SM BAG|1709.80|n pla 810|blanched lemon magenta medium dark|Manufacturer#5|Brand#54|LARGE ANODIZED TIN|13|WRAP BOX|1710.81|instructions. 811|ivory bisque black chiffon gainsboro|Manufacturer#3|Brand#33|PROMO BRUSHED NICKEL|32|SM BOX|1711.81|pecial, ironic pac 812|firebrick olive chartreuse frosted ivory|Manufacturer#3|Brand#33|LARGE PLATED TIN|11|WRAP BAG|1712.81|asymptot 813|sandy ghost khaki frosted goldenrod|Manufacturer#4|Brand#43|MEDIUM POLISHED STEEL|2|WRAP DRUM|1713.81|riously ironic deposi 814|burnished seashell floral moccasin antique|Manufacturer#5|Brand#54|ECONOMY PLATED NICKEL|43|MED PKG|1714.81|xpress requests. 815|light peach saddle medium firebrick|Manufacturer#2|Brand#22|ECONOMY BRUSHED NICKEL|5|LG PACK|1715.81|nt requests a 816|thistle ivory pink olive puff|Manufacturer#1|Brand#13|STANDARD POLISHED NICKEL|35|LG CAN|1716.81| thin 817|azure chocolate lavender blanched burnished|Manufacturer#4|Brand#41|STANDARD ANODIZED TIN|36|JUMBO PKG|1717.81|carefu 818|navajo saddle indian cornflower steel|Manufacturer#4|Brand#43|PROMO BRUSHED BRASS|36|MED JAR|1718.81|ckages 819|light magenta indian khaki lavender|Manufacturer#5|Brand#54|ECONOMY BRUSHED STEEL|27|SM CAN|1719.81|as are slyly against 820|wheat salmon forest lavender papaya|Manufacturer#1|Brand#11|STANDARD BURNISHED STEEL|20|WRAP JAR|1720.82|ely regular theodo 821|smoke drab rose hot spring|Manufacturer#4|Brand#42|PROMO POLISHED TIN|15|MED BOX|1721.82|gle across t 822|purple lawn ivory black slate|Manufacturer#5|Brand#52|PROMO ANODIZED BRASS|21|LG CASE|1722.82|ding to t 823|maroon salmon puff medium brown|Manufacturer#2|Brand#25|SMALL BURNISHED COPPER|21|WRAP CASE|1723.82|inal the 824|dodger snow orange almond magenta|Manufacturer#1|Brand#15|LARGE BURNISHED TIN|34|MED BAG|1724.82|s. ironic 825|puff deep hot tomato powder|Manufacturer#1|Brand#13|PROMO PLATED TIN|47|LG CAN|1725.82|lar ide 826|ghost rosy beige salmon lemon|Manufacturer#3|Brand#34|ECONOMY PLATED TIN|21|SM CASE|1726.82|ifts could 827|linen ghost smoke blanched cream|Manufacturer#1|Brand#13|PROMO POLISHED COPPER|15|JUMBO PACK|1727.82|sits; depe 828|lemon light grey burlywood drab|Manufacturer#4|Brand#44|LARGE BRUSHED NICKEL|11|LG PKG|1728.82|symptotes. furiously 829|cream indian deep linen antique|Manufacturer#3|Brand#35|MEDIUM BURNISHED BRASS|33|SM JAR|1729.82|tegrate. 830|cornflower medium salmon coral magenta|Manufacturer#5|Brand#54|SMALL POLISHED TIN|37|JUMBO CAN|1730.83|ep final instructi 831|violet dark midnight lawn cream|Manufacturer#5|Brand#51|STANDARD POLISHED COPPER|8|LG CAN|1731.83|se blithely. special, 832|orange peach midnight burnished lavender|Manufacturer#4|Brand#44|STANDARD ANODIZED COPPER|3|WRAP BAG|1732.83|fily bo 833|thistle puff cream papaya deep|Manufacturer#4|Brand#41|SMALL BRUSHED STEEL|34|JUMBO CAN|1733.83|s thrash furiously fin 834|aquamarine frosted orange dodger khaki|Manufacturer#5|Brand#52|MEDIUM POLISHED TIN|43|JUMBO CAN|1734.83|mptotes integrate 835|turquoise peru light aquamarine dark|Manufacturer#5|Brand#53|MEDIUM PLATED STEEL|21|SM BOX|1735.83|uses. furi 836|almond sky red lime smoke|Manufacturer#3|Brand#31|LARGE BURNISHED TIN|32|SM JAR|1736.83|lyly bold accounts. t 837|papaya goldenrod burlywood purple brown|Manufacturer#1|Brand#13|MEDIUM BURNISHED BRASS|27|WRAP PKG|1737.83|hely furious packages 838|lime peach puff papaya olive|Manufacturer#2|Brand#22|MEDIUM ANODIZED TIN|9|WRAP BAG|1738.83| final pac 839|cornflower white papaya violet navajo|Manufacturer#2|Brand#25|LARGE BRUSHED NICKEL|19|JUMBO CAN|1739.83|ggle a 840|khaki mint chartreuse dark frosted|Manufacturer#4|Brand#44|ECONOMY ANODIZED BRASS|42|WRAP PKG|1740.84|ending ideas w 841|floral khaki grey metallic firebrick|Manufacturer#1|Brand#15|MEDIUM BURNISHED BRASS|48|WRAP BOX|1741.84|y regul 842|gainsboro dim navajo chartreuse olive|Manufacturer#1|Brand#11|SMALL ANODIZED BRASS|4|MED PACK|1742.84|en accounts! carefu 843|frosted metallic mint lawn blanched|Manufacturer#1|Brand#15|MEDIUM POLISHED STEEL|50|LG PACK|1743.84| the 844|honeydew smoke lemon ghost navajo|Manufacturer#3|Brand#35|LARGE PLATED BRASS|22|SM CASE|1744.84|ix furiously beyond 845|papaya burlywood bisque linen navy|Manufacturer#4|Brand#41|LARGE PLATED STEEL|25|LG JAR|1745.84|ully regula 846|purple papaya saddle cream medium|Manufacturer#2|Brand#24|ECONOMY POLISHED NICKEL|6|SM BAG|1746.84|sometimes bold foxes 847|chartreuse pink azure tan slate|Manufacturer#2|Brand#25|SMALL PLATED STEEL|21|SM PACK|1747.84|ely blith 848|orange olive puff midnight almond|Manufacturer#5|Brand#53|LARGE BURNISHED TIN|26|LG JAR|1748.84|gular requests u 849|lawn cornflower puff rosy saddle|Manufacturer#4|Brand#44|STANDARD POLISHED TIN|41|MED DRUM|1749.84|nts among the pending 850|peach goldenrod honeydew moccasin sienna|Manufacturer#4|Brand#41|STANDARD BRUSHED NICKEL|31|JUMBO DRUM|1750.85|kages are carefu 851|maroon beige navy forest honeydew|Manufacturer#5|Brand#54|ECONOMY BURNISHED COPPER|30|LG CAN|1751.85| asympto 852|lemon slate khaki misty hot|Manufacturer#2|Brand#24|PROMO PLATED NICKEL|47|JUMBO BOX|1752.85|side of the 853|peach honeydew cyan peru light|Manufacturer#5|Brand#51|SMALL ANODIZED NICKEL|42|JUMBO CASE|1753.85|ts sleep! 854|pale wheat lace midnight papaya|Manufacturer#4|Brand#41|SMALL POLISHED COPPER|29|JUMBO CASE|1754.85|ests cajole furious 855|dim blue slate dodger yellow|Manufacturer#3|Brand#31|STANDARD BURNISHED TIN|46|MED BOX|1755.85|ly express fox 856|blush frosted powder antique blanched|Manufacturer#1|Brand#13|MEDIUM BRUSHED STEEL|12|MED BAG|1756.85|posits integrate sly 857|goldenrod beige lemon midnight cornflower|Manufacturer#3|Brand#35|PROMO PLATED TIN|8|MED PACK|1757.85|final de 858|papaya maroon hot blue pink|Manufacturer#3|Brand#33|ECONOMY BURNISHED COPPER|14|SM CAN|1758.85|elets breach f 859|ghost blue indian cornsilk drab|Manufacturer#3|Brand#35|ECONOMY ANODIZED COPPER|10|WRAP BAG|1759.85| even Tiresias wake fu 860|burlywood peach papaya violet midnight|Manufacturer#3|Brand#32|SMALL POLISHED STEEL|36|LG PKG|1760.86|oss the f 861|blue sienna navy coral violet|Manufacturer#4|Brand#43|ECONOMY BRUSHED COPPER|48|JUMBO BOX|1761.86| furiousl 862|tan dim chiffon steel purple|Manufacturer#5|Brand#54|LARGE POLISHED NICKEL|12|LG PKG|1762.86| pending depo 863|light khaki lime lemon burnished|Manufacturer#2|Brand#25|STANDARD ANODIZED COPPER|41|LG CAN|1763.86| regular pinto beans 864|linen drab blush maroon sienna|Manufacturer#1|Brand#11|STANDARD POLISHED TIN|39|JUMBO PACK|1764.86|boost. 865|brown seashell red bisque sandy|Manufacturer#1|Brand#11|SMALL ANODIZED BRASS|12|JUMBO PACK|1765.86|s along t 866|dodger antique tan coral honeydew|Manufacturer#1|Brand#11|STANDARD PLATED COPPER|13|WRAP BOX|1766.86|eas sleep quickly 867|snow spring black white burnished|Manufacturer#5|Brand#54|PROMO POLISHED NICKEL|4|LG BOX|1767.86|nts wake care 868|navajo azure beige aquamarine blush|Manufacturer#3|Brand#33|PROMO BRUSHED BRASS|48|JUMBO PKG|1768.86|packages. e 869|snow blush violet lace ghost|Manufacturer#2|Brand#24|STANDARD ANODIZED COPPER|28|LG BAG|1769.86|y fina 870|antique hot snow burnished ghost|Manufacturer#5|Brand#54|SMALL PLATED NICKEL|11|JUMBO CAN|1770.87|ldly unusual depth 871|midnight dark coral ivory burlywood|Manufacturer#5|Brand#55|SMALL ANODIZED BRASS|27|SM PKG|1771.87|olites snooze quickl 872|papaya frosted cornflower green almond|Manufacturer#1|Brand#15|MEDIUM PLATED STEEL|41|LG CASE|1772.87|gular inst 873|goldenrod maroon snow cream indian|Manufacturer#1|Brand#11|MEDIUM BRUSHED BRASS|45|MED CAN|1773.87|ecial ideas. slyly 874|drab slate hot black khaki|Manufacturer#1|Brand#15|MEDIUM PLATED TIN|9|MED BOX|1774.87|e regular req 875|pale peru red orchid almond|Manufacturer#2|Brand#24|ECONOMY BRUSHED COPPER|1|MED DRUM|1775.87|e furiously. b 876|frosted lace turquoise sky sandy|Manufacturer#3|Brand#31|MEDIUM PLATED NICKEL|25|SM CAN|1776.87|s after the fur 877|lime violet tomato drab blush|Manufacturer#3|Brand#31|STANDARD ANODIZED NICKEL|20|MED PACK|1777.87|thely rut 878|steel lace wheat orchid linen|Manufacturer#3|Brand#32|STANDARD POLISHED NICKEL|40|JUMBO CASE|1778.87| slyly ironic g 879|spring red indian floral sky|Manufacturer#3|Brand#34|STANDARD BURNISHED TIN|23|MED PKG|1779.87|ar, even request 880|sandy turquoise cream firebrick rose|Manufacturer#2|Brand#25|STANDARD ANODIZED COPPER|10|LG CASE|1780.88|ly stealthy deposits 881|bisque frosted khaki linen royal|Manufacturer#2|Brand#21|LARGE BRUSHED BRASS|12|MED PACK|1781.88|ully for 882|chiffon royal lime almond midnight|Manufacturer#4|Brand#44|ECONOMY BRUSHED STEEL|12|MED BAG|1782.88| silent, pendi 883|smoke grey dark yellow brown|Manufacturer#3|Brand#31|STANDARD PLATED NICKEL|18|LG PACK|1783.88|fluffily unusual pint 884|peach medium goldenrod cyan ghost|Manufacturer#1|Brand#11|SMALL POLISHED NICKEL|38|WRAP BAG|1784.88|luffy accounts. 885|dim peach moccasin snow floral|Manufacturer#4|Brand#42|ECONOMY BRUSHED STEEL|31|WRAP DRUM|1785.88|lar, even ideas caj 886|turquoise dodger lemon antique green|Manufacturer#5|Brand#55|LARGE BRUSHED COPPER|46|MED BAG|1786.88|tornis haggle! 887|navajo cream salmon orange smoke|Manufacturer#4|Brand#41|SMALL BRUSHED STEEL|48|JUMBO CAN|1787.88|x slyly aroun 888|pale turquoise rose olive chiffon|Manufacturer#3|Brand#31|STANDARD BRUSHED NICKEL|44|SM DRUM|1788.88|y according t 889|saddle midnight almond drab blanched|Manufacturer#5|Brand#51|LARGE BURNISHED BRASS|19|JUMBO DRUM|1789.88|efully regular 890|sandy lawn chartreuse peru blue|Manufacturer#5|Brand#52|SMALL BRUSHED STEEL|23|MED PKG|1790.89|ests d 891|plum floral dodger lemon almond|Manufacturer#4|Brand#42|ECONOMY POLISHED STEEL|5|MED CASE|1791.89|yly regular, expr 892|thistle chocolate sandy powder drab|Manufacturer#2|Brand#22|SMALL BRUSHED NICKEL|3|LG DRUM|1792.89|quickly regular r 893|dark moccasin bisque cyan azure|Manufacturer#1|Brand#14|PROMO BRUSHED STEEL|9|LG JAR|1793.89|ly regular 894|chocolate blue smoke azure sandy|Manufacturer#3|Brand#35|SMALL BURNISHED TIN|43|MED BAG|1794.89|s. carefully expr 895|thistle azure magenta mint burnished|Manufacturer#4|Brand#41|MEDIUM BRUSHED STEEL|17|SM BOX|1795.89|eaves might 896|peach ivory rose honeydew lace|Manufacturer#4|Brand#43|PROMO ANODIZED TIN|7|LG CASE|1796.89|e across 897|misty olive purple cream red|Manufacturer#4|Brand#41|SMALL BRUSHED COPPER|26|SM JAR|1797.89|cajole bli 898|olive burlywood royal cream drab|Manufacturer#4|Brand#45|PROMO POLISHED NICKEL|8|WRAP BAG|1798.89|e against the pack 899|gainsboro dim misty violet coral|Manufacturer#2|Brand#25|SMALL ANODIZED NICKEL|50|WRAP CASE|1799.89|lithely pending excuse 900|medium antique khaki cream slate|Manufacturer#2|Brand#21|STANDARD PLATED NICKEL|40|LG CAN|1800.90|foxes. carefully s 901|white honeydew orange saddle black|Manufacturer#2|Brand#23|ECONOMY PLATED TIN|38|LG CAN|1801.90|ffily final theodoli 902|red chartreuse firebrick burlywood papaya|Manufacturer#2|Brand#23|LARGE ANODIZED COPPER|14|MED CAN|1802.90|inal pint 903|navajo chiffon mint ghost burlywood|Manufacturer#4|Brand#44|ECONOMY BURNISHED BRASS|4|SM BAG|1803.90|the i 904|black olive light sandy linen|Manufacturer#1|Brand#12|STANDARD ANODIZED TIN|20|WRAP DRUM|1804.90|ar theodolites sleep b 905|burlywood honeydew cornsilk indian green|Manufacturer#2|Brand#25|PROMO BRUSHED BRASS|27|MED CASE|1805.90|es. furiously ir 906|rosy forest navajo light turquoise|Manufacturer#4|Brand#44|MEDIUM BURNISHED COPPER|45|MED DRUM|1806.90|unusua 907|plum lawn black chartreuse sandy|Manufacturer#5|Brand#52|ECONOMY ANODIZED STEEL|8|LG CASE|1807.90|es. carefu 908|pink navy lace papaya brown|Manufacturer#2|Brand#25|MEDIUM BURNISHED TIN|13|JUMBO JAR|1808.90|tions ca 909|cornsilk tomato chiffon bisque navy|Manufacturer#1|Brand#12|STANDARD PLATED TIN|40|WRAP PKG|1809.90|ht boost quic 910|coral antique peach navy violet|Manufacturer#4|Brand#42|STANDARD ANODIZED TIN|15|JUMBO JAR|1810.91|ly final 911|blush coral medium ivory snow|Manufacturer#1|Brand#14|ECONOMY BURNISHED TIN|23|MED CAN|1811.91|foxes cajole after the 912|indian tomato rose powder turquoise|Manufacturer#5|Brand#55|STANDARD POLISHED NICKEL|17|SM BAG|1812.91|ar theodo 913|seashell medium burnished moccasin blue|Manufacturer#2|Brand#23|LARGE PLATED BRASS|49|SM DRUM|1813.91|yly even dol 914|snow rose moccasin tomato linen|Manufacturer#3|Brand#35|ECONOMY POLISHED STEEL|8|LG PKG|1814.91|ckly busil 915|aquamarine purple spring gainsboro salmon|Manufacturer#2|Brand#21|LARGE ANODIZED BRASS|25|WRAP BAG|1815.91|depos 916|chartreuse tomato rose chocolate magenta|Manufacturer#2|Brand#21|MEDIUM BURNISHED BRASS|28|MED CAN|1816.91|ons. bravely ironic 917|midnight khaki medium linen peru|Manufacturer#2|Brand#24|SMALL ANODIZED COPPER|3|SM CASE|1817.91|riously past th 918|ghost steel beige turquoise ivory|Manufacturer#5|Brand#55|ECONOMY PLATED TIN|15|LG PKG|1818.91|losely according to 919|sandy cream royal cyan orchid|Manufacturer#1|Brand#13|PROMO BRUSHED COPPER|49|MED BOX|1819.91|along 920|sky lemon misty slate lawn|Manufacturer#4|Brand#44|MEDIUM BRUSHED STEEL|19|WRAP CAN|1820.92|y regu 921|khaki yellow plum cyan forest|Manufacturer#1|Brand#11|LARGE BURNISHED STEEL|35|JUMBO DRUM|1821.92|pecial in 922|frosted dodger cyan medium indian|Manufacturer#3|Brand#34|LARGE BURNISHED BRASS|30|SM CASE|1822.92|uctions. sl 923|moccasin lace rose navajo forest|Manufacturer#2|Brand#24|STANDARD PLATED TIN|26|WRAP CASE|1823.92|he packag 924|indian smoke orange ghost papaya|Manufacturer#1|Brand#11|ECONOMY POLISHED STEEL|41|JUMBO PKG|1824.92|the ac 925|tomato antique plum snow burlywood|Manufacturer#4|Brand#44|STANDARD POLISHED NICKEL|44|MED PACK|1825.92|aggle slyly. blithe 926|tan deep medium smoke slate|Manufacturer#4|Brand#45|STANDARD BRUSHED TIN|39|MED JAR|1826.92|gle quickly ironic 927|dodger purple green peach navajo|Manufacturer#1|Brand#14|LARGE ANODIZED TIN|13|WRAP BAG|1827.92|otes snooz 928|violet dodger cyan orange moccasin|Manufacturer#4|Brand#43|SMALL ANODIZED TIN|43|SM BOX|1828.92|ully ev 929|burnished pale medium turquoise lace|Manufacturer#3|Brand#33|LARGE PLATED COPPER|40|MED DRUM|1829.92|hely iron 930|violet ivory orchid rose frosted|Manufacturer#2|Brand#23|SMALL BRUSHED BRASS|26|LG DRUM|1830.93|uickly ironic 931|red puff chiffon sienna dark|Manufacturer#3|Brand#31|STANDARD ANODIZED NICKEL|37|LG CAN|1831.93|are fluffily. 932|cornflower sky powder cyan linen|Manufacturer#2|Brand#24|SMALL ANODIZED COPPER|12|LG BOX|1832.93|ly regular depos 933|pink magenta frosted blanched lime|Manufacturer#3|Brand#33|STANDARD BRUSHED STEEL|25|LG CASE|1833.93|he blit 934|beige saddle magenta cyan dim|Manufacturer#4|Brand#45|STANDARD ANODIZED BRASS|40|MED JAR|1834.93|ual packages 935|royal cornflower maroon khaki cornsilk|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|8|JUMBO DRUM|1835.93|arefully ironic 936|white beige firebrick dodger olive|Manufacturer#2|Brand#24|MEDIUM PLATED BRASS|6|WRAP DRUM|1836.93|longside of the reg 937|drab peru royal pink peach|Manufacturer#3|Brand#33|PROMO ANODIZED TIN|2|JUMBO CAN|1837.93|ithely 938|chartreuse peach spring cornflower tan|Manufacturer#2|Brand#22|LARGE BRUSHED NICKEL|1|JUMBO CASE|1838.93|ajole slyly r 939|salmon magenta orange coral aquamarine|Manufacturer#1|Brand#14|PROMO BURNISHED BRASS|41|WRAP PACK|1839.93|ages wake doggedly of 940|violet coral blanched orange light|Manufacturer#1|Brand#15|PROMO POLISHED TIN|49|JUMBO JAR|1840.94|ate blithely bold, si 941|lace cornsilk pink medium slate|Manufacturer#2|Brand#25|MEDIUM PLATED NICKEL|8|LG PKG|1841.94|fully 942|coral peach bisque green seashell|Manufacturer#3|Brand#35|PROMO POLISHED TIN|44|MED BAG|1842.94|nic pinto b 943|smoke sky chiffon green steel|Manufacturer#5|Brand#55|ECONOMY BRUSHED BRASS|46|SM PKG|1843.94|s wake 944|medium sandy olive wheat turquoise|Manufacturer#3|Brand#33|SMALL PLATED STEEL|1|LG CAN|1844.94| requests nag furiou 945|chiffon steel peach black moccasin|Manufacturer#5|Brand#52|MEDIUM PLATED NICKEL|22|MED JAR|1845.94|uffily abo 946|medium drab rosy spring salmon|Manufacturer#2|Brand#25|MEDIUM PLATED TIN|30|LG BOX|1846.94|y along 947|black tan salmon maroon chartreuse|Manufacturer#1|Brand#13|MEDIUM BURNISHED STEEL|16|JUMBO JAR|1847.94|usual accounts. b 948|gainsboro chocolate bisque sandy steel|Manufacturer#2|Brand#22|MEDIUM ANODIZED BRASS|31|WRAP PKG|1848.94|furious 949|thistle burnished pink yellow wheat|Manufacturer#5|Brand#51|LARGE BURNISHED TIN|47|LG PKG|1849.94|l theodolit 950|steel purple midnight beige coral|Manufacturer#3|Brand#31|SMALL ANODIZED STEEL|26|WRAP CASE|1850.95|kages cajole quickly 951|gainsboro indian aquamarine lemon blush|Manufacturer#2|Brand#21|STANDARD BURNISHED STEEL|40|JUMBO JAR|1851.95| final deposits nag a 952|frosted purple sky steel papaya|Manufacturer#3|Brand#31|MEDIUM BRUSHED TIN|4|MED PACK|1852.95|l deposits sleep bli 953|tomato cornflower honeydew sandy grey|Manufacturer#4|Brand#41|PROMO ANODIZED TIN|2|WRAP JAR|1853.95|ly quickly final acco 954|turquoise bisque khaki coral antique|Manufacturer#5|Brand#55|ECONOMY ANODIZED STEEL|35|JUMBO CASE|1854.95|furious 955|blue olive tan rosy navajo|Manufacturer#3|Brand#32|PROMO BURNISHED STEEL|43|SM CASE|1855.95|tions 956|orchid yellow dark lawn thistle|Manufacturer#1|Brand#13|STANDARD BRUSHED STEEL|7|LG CASE|1856.95|furiously pending d 957|snow turquoise coral lime goldenrod|Manufacturer#3|Brand#33|SMALL ANODIZED COPPER|23|SM CAN|1857.95|ndencies wake s 958|turquoise sandy drab dodger cyan|Manufacturer#3|Brand#35|LARGE PLATED NICKEL|13|LG BOX|1858.95|quickly careful i 959|sky snow peach olive spring|Manufacturer#4|Brand#41|SMALL BURNISHED STEEL|5|WRAP JAR|1859.95|iously bold depos 960|sienna royal light orchid moccasin|Manufacturer#5|Brand#55|LARGE PLATED COPPER|26|LG CASE|1860.96|lly b 961|cornflower chocolate floral burnished green|Manufacturer#1|Brand#15|PROMO BRUSHED BRASS|1|SM PACK|1861.96|fully. f 962|magenta linen ghost almond floral|Manufacturer#1|Brand#13|MEDIUM PLATED BRASS|38|JUMBO BOX|1862.96|lithely. 963|violet beige lime dark metallic|Manufacturer#4|Brand#44|MEDIUM PLATED BRASS|44|LG CASE|1863.96|y final deposits wake. 964|sky orange indian pale burlywood|Manufacturer#5|Brand#54|SMALL POLISHED BRASS|48|MED BOX|1864.96|nal asymptotes. regul 965|dim cyan bisque black smoke|Manufacturer#3|Brand#34|MEDIUM PLATED STEEL|39|LG PKG|1865.96|dolite 966|turquoise antique brown violet plum|Manufacturer#4|Brand#42|PROMO BURNISHED TIN|24|MED CAN|1866.96| the carefully 967|cyan snow azure slate papaya|Manufacturer#5|Brand#55|SMALL BURNISHED COPPER|16|JUMBO CAN|1867.96|leep fur 968|rose chiffon chocolate lace papaya|Manufacturer#1|Brand#11|LARGE PLATED TIN|23|LG PKG|1868.96|press accounts acro 969|hot azure royal red dark|Manufacturer#5|Brand#55|PROMO POLISHED STEEL|34|JUMBO CASE|1869.96|le aga 970|sienna azure light medium green|Manufacturer#1|Brand#11|STANDARD BURNISHED STEEL|44|SM CAN|1870.97|ironic 971|orange misty green aquamarine forest|Manufacturer#5|Brand#54|LARGE POLISHED BRASS|19|SM DRUM|1871.97|ugouts. bl 972|tomato burlywood pale moccasin plum|Manufacturer#3|Brand#32|MEDIUM PLATED BRASS|21|JUMBO JAR|1872.97|yly ironic deposits 973|medium lavender blue tomato slate|Manufacturer#2|Brand#22|STANDARD ANODIZED STEEL|42|LG PACK|1873.97|along 974|almond steel slate sky rose|Manufacturer#4|Brand#44|MEDIUM ANODIZED BRASS|35|WRAP BAG|1874.97|s. caref 975|moccasin tomato peach yellow drab|Manufacturer#3|Brand#34|MEDIUM ANODIZED COPPER|22|LG JAR|1875.97|torni 976|white maroon firebrick snow light|Manufacturer#5|Brand#51|STANDARD BRUSHED NICKEL|19|JUMBO PACK|1876.97| special reque 977|frosted tomato deep burnished indian|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|2|LG CASE|1877.97|the quick 978|cream tomato rose floral steel|Manufacturer#4|Brand#45|MEDIUM POLISHED TIN|46|LG DRUM|1878.97|quests sleep abo 979|blush cyan azure snow sandy|Manufacturer#4|Brand#45|STANDARD ANODIZED TIN|4|MED CAN|1879.97| accounts. blithe 980|saddle burlywood drab deep red|Manufacturer#4|Brand#41|SMALL ANODIZED STEEL|45|LG CASE|1880.98|ronic deposi 981|floral blanched papaya khaki light|Manufacturer#2|Brand#25|MEDIUM POLISHED COPPER|31|WRAP CAN|1881.98| ironic as 982|drab steel linen puff rosy|Manufacturer#3|Brand#31|SMALL POLISHED BRASS|28|SM BAG|1882.98|boost blithely 983|violet mint moccasin black olive|Manufacturer#4|Brand#43|ECONOMY POLISHED TIN|39|MED BOX|1883.98| pack 984|tan frosted antique midnight thistle|Manufacturer#2|Brand#22|LARGE PLATED TIN|40|SM BAG|1884.98|ly. furiously pe 985|drab rosy orange peach medium|Manufacturer#5|Brand#53|MEDIUM BURNISHED STEEL|10|JUMBO PKG|1885.98|s beside 986|forest purple lawn yellow azure|Manufacturer#3|Brand#33|PROMO BRUSHED BRASS|50|JUMBO DRUM|1886.98|ronic foxes 987|seashell pink linen salmon khaki|Manufacturer#5|Brand#53|PROMO POLISHED BRASS|39|LG BOX|1887.98|furiou 988|rose saddle linen peach cream|Manufacturer#2|Brand#24|STANDARD BURNISHED COPPER|27|WRAP PKG|1888.98|special requests sleep 989|misty ivory plum tan steel|Manufacturer#5|Brand#55|LARGE PLATED TIN|9|JUMBO JAR|1889.98|lets. e 990|azure cornsilk indian floral aquamarine|Manufacturer#1|Brand#14|SMALL BURNISHED COPPER|14|SM BAG|1890.99|quick 991|indian yellow red lime seashell|Manufacturer#3|Brand#32|MEDIUM POLISHED BRASS|9|SM CASE|1891.99|ickly slowly spe 992|hot peru beige magenta metallic|Manufacturer#5|Brand#51|PROMO BURNISHED COPPER|39|JUMBO BOX|1892.99| haggle 993|chiffon papaya lavender tomato spring|Manufacturer#5|Brand#54|MEDIUM BRUSHED NICKEL|2|MED JAR|1893.99|bold pear 994|almond saddle papaya seashell burlywood|Manufacturer#2|Brand#25|PROMO ANODIZED STEEL|38|JUMBO BOX|1894.99|ost slyly. boldly bo 995|papaya violet navy khaki sky|Manufacturer#3|Brand#34|ECONOMY BRUSHED TIN|47|MED DRUM|1895.99|ke furiously fluffily 996|slate pink navajo orange firebrick|Manufacturer#3|Brand#31|STANDARD POLISHED COPPER|13|LG CAN|1896.99|ously 997|khaki lawn rose drab cornsilk|Manufacturer#5|Brand#51|STANDARD ANODIZED BRASS|22|JUMBO BAG|1897.99|riously final 998|ivory maroon cream red peru|Manufacturer#5|Brand#53|STANDARD BRUSHED NICKEL|2|MED BOX|1898.99|s. bold requests wake 999|indian honeydew chartreuse navy cyan|Manufacturer#4|Brand#44|STANDARD PLATED STEEL|16|WRAP CAN|1899.99|ans. slyly expre 1000|wheat frosted chiffon aquamarine saddle|Manufacturer#2|Brand#24|ECONOMY BRUSHED NICKEL|10|SM DRUM|901.00|g fluf ================================================ FILE: src/test/regress/data/part.more.data ================================================ 6001|black lavender tomato brown violet|Manufacturer#3|Brand#32|STANDARD BURNISHED STEEL|41|LG BAG|907.00|eposits. 6002|cyan saddle plum navajo tan|Manufacturer#5|Brand#53|MEDIUM POLISHED BRASS|19|MED BOX|908.00|final pinto beans 6003|chocolate wheat cornflower spring linen|Manufacturer#4|Brand#42|LARGE PLATED BRASS|50|LG JAR|909.00|structions. fluffily 6004|white indian burlywood smoke medium|Manufacturer#1|Brand#13|PROMO BURNISHED TIN|19|JUMBO CAN|910.00|about the care 6005|powder green red peru lavender|Manufacturer#2|Brand#25|PROMO BURNISHED TIN|40|SM CAN|911.00|lar notorni 6006|maroon green blanched indian ghost|Manufacturer#5|Brand#51|SMALL BRUSHED COPPER|46|LG CASE|912.00|es. slyly re 6007|steel cornsilk puff navajo drab|Manufacturer#5|Brand#54|PROMO BURNISHED STEEL|37|SM BOX|913.00|t excus 6008|medium midnight orange papaya brown|Manufacturer#5|Brand#53|PROMO POLISHED STEEL|15|LG BAG|914.00|s unwind unusual 6009|navy floral puff misty salmon|Manufacturer#5|Brand#54|STANDARD ANODIZED STEEL|27|SM BAG|915.00|arefull 6010|saddle green salmon red olive|Manufacturer#3|Brand#32|STANDARD ANODIZED COPPER|42|LG JAR|916.01|bold, spec 6011|turquoise royal papaya azure grey|Manufacturer#1|Brand#15|PROMO BRUSHED NICKEL|50|SM BAG|917.01|ts maintai 6012|burlywood chocolate peru deep rose|Manufacturer#4|Brand#42|STANDARD BRUSHED COPPER|18|SM CASE|918.01|final deposits doubt 6013|gainsboro lime rose hot sky|Manufacturer#3|Brand#32|PROMO ANODIZED COPPER|42|JUMBO JAR|919.01|ate slyly silent, iro 6014|red frosted powder medium sienna|Manufacturer#5|Brand#55|STANDARD ANODIZED NICKEL|46|MED DRUM|920.01|egular packages 6015|chartreuse chocolate peru smoke lace|Manufacturer#1|Brand#11|MEDIUM BRUSHED NICKEL|50|LG DRUM|921.01|nts. evenly unusual 6016|red linen orange floral tomato|Manufacturer#1|Brand#14|ECONOMY POLISHED BRASS|41|MED BAG|922.01| pinto be 6017|firebrick light smoke mint midnight|Manufacturer#3|Brand#32|LARGE BRUSHED NICKEL|44|LG BOX|923.01|ly against the 6018|lavender plum gainsboro mint salmon|Manufacturer#3|Brand#33|ECONOMY POLISHED BRASS|46|WRAP BAG|924.01|ckly r 6019|plum rose midnight lawn papaya|Manufacturer#5|Brand#51|PROMO BRUSHED TIN|38|MED CASE|925.01|press dolp 6020|olive forest purple magenta smoke|Manufacturer#5|Brand#55|STANDARD POLISHED TIN|17|WRAP BOX|926.02|ar packages hagg 6021|lavender slate sienna blush tomato|Manufacturer#5|Brand#52|LARGE PLATED STEEL|17|LG PACK|927.02|ve furiou 6022|cream royal white lace rose|Manufacturer#1|Brand#14|PROMO POLISHED COPPER|18|SM DRUM|928.02|ickly 6023|blush chartreuse gainsboro beige maroon|Manufacturer#1|Brand#15|ECONOMY BURNISHED BRASS|43|SM PKG|929.02|ave packa 6024|almond blush snow salmon midnight|Manufacturer#5|Brand#55|LARGE BURNISHED COPPER|5|LG CAN|930.02|l accoun 6025|purple medium light aquamarine dark|Manufacturer#5|Brand#55|MEDIUM PLATED STEEL|34|SM BAG|931.02|ake after t 6026|coral blush honeydew rose dark|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|10|JUMBO BAG|932.02|final 6027|snow cream salmon misty tan|Manufacturer#3|Brand#32|LARGE BURNISHED NICKEL|6|JUMBO JAR|933.02|al dep 6028|orchid almond honeydew grey lace|Manufacturer#1|Brand#12|SMALL PLATED NICKEL|38|SM BOX|934.02|ss the pe 6029|seashell moccasin bisque goldenrod violet|Manufacturer#1|Brand#13|MEDIUM ANODIZED TIN|20|MED CASE|935.02| the special pac 6030|linen yellow orchid drab bisque|Manufacturer#5|Brand#51|SMALL PLATED STEEL|28|JUMBO DRUM|936.03|ly across 6031|green drab coral metallic orchid|Manufacturer#1|Brand#14|LARGE POLISHED BRASS|36|MED CAN|937.03|e. regularly ironic 6032|papaya cream cornsilk khaki navajo|Manufacturer#3|Brand#33|ECONOMY BRUSHED TIN|29|SM PACK|938.03|dolites. evenly ir 6033|drab green smoke salmon navy|Manufacturer#3|Brand#32|STANDARD BRUSHED STEEL|31|JUMBO DRUM|939.03|he fur 6034|forest lawn steel smoke dodger|Manufacturer#4|Brand#42|ECONOMY ANODIZED BRASS|44|MED DRUM|940.03|usual packages. depe 6035|royal orange pink burnished salmon|Manufacturer#1|Brand#11|MEDIUM POLISHED BRASS|39|WRAP CAN|941.03|y thin 6036|chartreuse antique linen puff medium|Manufacturer#3|Brand#34|PROMO ANODIZED NICKEL|1|WRAP BAG|942.03|slyly afte 6037|chocolate rose black steel cyan|Manufacturer#1|Brand#13|ECONOMY PLATED STEEL|34|WRAP BAG|943.03|s are speci 6038|dim almond red burnished lace|Manufacturer#2|Brand#24|MEDIUM ANODIZED TIN|1|WRAP PKG|944.03| deposits. 6039|lavender grey dark drab purple|Manufacturer#4|Brand#45|STANDARD BURNISHED TIN|14|LG CAN|945.03|ress packages cajole q 6040|honeydew dark midnight khaki bisque|Manufacturer#4|Brand#42|MEDIUM PLATED NICKEL|29|LG CASE|946.04|l, even asympt 6041|seashell forest linen brown goldenrod|Manufacturer#2|Brand#21|STANDARD ANODIZED NICKEL|19|MED DRUM|947.04|ecial 6042|peach chocolate firebrick frosted papaya|Manufacturer#5|Brand#54|LARGE BURNISHED STEEL|48|WRAP PKG|948.04|y quiet theodol 6043|khaki misty deep almond medium|Manufacturer#4|Brand#41|SMALL ANODIZED TIN|38|SM CASE|949.04|ideas affix 6044|dodger turquoise midnight papaya seashell|Manufacturer#2|Brand#22|SMALL BURNISHED TIN|4|JUMBO CASE|950.04|ccounts. blith 6045|turquoise blanched chocolate blush almond|Manufacturer#4|Brand#42|SMALL BRUSHED NICKEL|37|WRAP DRUM|951.04| final theodolites 6046|dim green wheat light honeydew|Manufacturer#5|Brand#51|ECONOMY BURNISHED BRASS|14|JUMBO PKG|952.04|l packages 6047|orchid dodger magenta beige snow|Manufacturer#2|Brand#22|MEDIUM POLISHED NICKEL|8|JUMBO BOX|953.04|ut the warthog 6048|ivory saddle drab steel pink|Manufacturer#1|Brand#12|MEDIUM BURNISHED STEEL|34|SM PACK|954.04|oss the carefully exp 6049|khaki floral yellow lemon metallic|Manufacturer#5|Brand#55|MEDIUM POLISHED COPPER|17|LG PKG|955.04|uests use 6050|dark thistle orchid slate brown|Manufacturer#4|Brand#42|ECONOMY BRUSHED BRASS|7|JUMBO BAG|956.05|nding packages 6051|linen steel brown medium floral|Manufacturer#3|Brand#33|ECONOMY POLISHED STEEL|20|LG PKG|957.05|usual requests. regul 6052|almond purple burnished dim coral|Manufacturer#2|Brand#21|STANDARD PLATED BRASS|20|SM BOX|958.05| the quickly special 6053|puff chocolate snow sky pale|Manufacturer#2|Brand#24|PROMO BRUSHED TIN|10|MED CAN|959.05|tes. even, 6054|aquamarine antique firebrick saddle cornsilk|Manufacturer#2|Brand#24|MEDIUM ANODIZED COPPER|45|WRAP PACK|960.05|ecial 6055|gainsboro firebrick coral lace hot|Manufacturer#3|Brand#33|SMALL PLATED BRASS|25|MED BOX|961.05|ses are dolphins. qu 6056|green dark tomato blanched cornflower|Manufacturer#4|Brand#43|PROMO PLATED COPPER|42|WRAP CAN|962.05|s. furiously re 6057|cornflower lawn royal linen cyan|Manufacturer#4|Brand#44|STANDARD BURNISHED NICKEL|11|SM CASE|963.05|ly pend 6058|light tomato ivory white medium|Manufacturer#5|Brand#51|LARGE BRUSHED COPPER|21|WRAP CASE|964.05|o the slyly regula 6059|azure peach peru pale ghost|Manufacturer#3|Brand#33|STANDARD POLISHED NICKEL|15|SM BAG|965.05|sits affix around 6060|bisque firebrick antique sandy white|Manufacturer#3|Brand#35|ECONOMY POLISHED COPPER|6|JUMBO BOX|966.06|lyly re 6061|snow coral green midnight floral|Manufacturer#1|Brand#12|SMALL BRUSHED NICKEL|23|MED BOX|967.06|fully iron 6062|lace pink lavender frosted snow|Manufacturer#5|Brand#54|PROMO BURNISHED BRASS|11|JUMBO DRUM|968.06|al ideas use slyly 6063|pale plum aquamarine violet cornflower|Manufacturer#2|Brand#23|PROMO ANODIZED TIN|28|SM BAG|969.06|y unus 6064|dark navajo floral navy chocolate|Manufacturer#5|Brand#55|STANDARD PLATED NICKEL|2|SM BAG|970.06|es-- always regu 6065|royal sienna khaki frosted dim|Manufacturer#1|Brand#11|STANDARD BURNISHED BRASS|3|WRAP BAG|971.06|ronic Tiresias. e 6066|brown papaya seashell magenta burnished|Manufacturer#3|Brand#34|SMALL ANODIZED COPPER|24|JUMBO BOX|972.06| haggle caref 6067|spring sandy blue orange cyan|Manufacturer#4|Brand#43|MEDIUM PLATED BRASS|12|WRAP DRUM|973.06|ly carefully silent 6068|cyan sienna moccasin grey tan|Manufacturer#4|Brand#44|SMALL BURNISHED TIN|2|LG PKG|974.06|gedly regular platel 6069|gainsboro lime maroon cornflower olive|Manufacturer#2|Brand#22|STANDARD ANODIZED TIN|32|WRAP PKG|975.06|ding to t 6070|orange cornsilk khaki salmon snow|Manufacturer#3|Brand#33|MEDIUM POLISHED TIN|35|MED CAN|976.07|s boo 6071|burlywood pink lime turquoise orchid|Manufacturer#2|Brand#24|PROMO BRUSHED NICKEL|35|JUMBO PACK|977.07| maintain ag 6072|pink orange cornflower floral light|Manufacturer#4|Brand#44|SMALL BRUSHED NICKEL|46|MED BAG|978.07|sauternes 6073|cornsilk puff metallic cyan peru|Manufacturer#3|Brand#35|LARGE PLATED STEEL|18|WRAP BOX|979.07|nic instructions d 6074|puff cornflower bisque black khaki|Manufacturer#3|Brand#35|MEDIUM POLISHED STEEL|44|JUMBO CAN|980.07|equests after t 6075|wheat mint chartreuse papaya spring|Manufacturer#3|Brand#35|MEDIUM BRUSHED COPPER|17|WRAP BAG|981.07|unusual, regular 6076|thistle magenta green floral olive|Manufacturer#5|Brand#51|PROMO PLATED NICKEL|41|JUMBO BOX|982.07|ously iron 6077|peru ivory dim pink blush|Manufacturer#1|Brand#15|LARGE BRUSHED NICKEL|32|LG BAG|983.07|arefully. 6078|pale navy coral lace peach|Manufacturer#2|Brand#21|STANDARD BURNISHED NICKEL|23|LG PACK|984.07|y bold d 6079|olive hot magenta maroon thistle|Manufacturer#2|Brand#23|PROMO ANODIZED COPPER|13|MED BOX|985.07|as. express accounts 6080|coral red dim maroon green|Manufacturer#5|Brand#51|PROMO POLISHED NICKEL|44|WRAP CAN|986.08|al instructions wake 6081|blush puff midnight wheat navajo|Manufacturer#5|Brand#53|SMALL PLATED TIN|27|WRAP DRUM|987.08|sts. fluffily fi 6082|bisque blue burnished turquoise steel|Manufacturer#1|Brand#15|LARGE BURNISHED BRASS|34|MED PKG|988.08|lly even courts wake 6083|cyan slate dim purple sky|Manufacturer#2|Brand#25|LARGE ANODIZED COPPER|25|WRAP BAG|989.08|nal requests along 6084|gainsboro floral goldenrod rose papaya|Manufacturer#4|Brand#42|SMALL BURNISHED STEEL|42|SM PKG|990.08| dolphi 6085|blush burnished lime purple mint|Manufacturer#2|Brand#24|ECONOMY POLISHED NICKEL|29|WRAP PKG|991.08|foxes 6086|misty hot tomato gainsboro azure|Manufacturer#1|Brand#11|SMALL BURNISHED COPPER|30|JUMBO CAN|992.08|ly dogged accounts 6087|indian cornsilk khaki burlywood pale|Manufacturer#1|Brand#12|MEDIUM POLISHED BRASS|7|SM BAG|993.08|ly. ex 6088|slate wheat orange orchid chartreuse|Manufacturer#4|Brand#45|SMALL BURNISHED BRASS|8|WRAP PACK|994.08|ffix 6089|light royal green yellow dark|Manufacturer#5|Brand#52|PROMO BURNISHED STEEL|31|JUMBO BAG|995.08|ing excu 6090|turquoise thistle cornsilk forest linen|Manufacturer#4|Brand#42|MEDIUM ANODIZED NICKEL|26|LG BAG|996.09| accounts 6091|wheat violet saddle royal metallic|Manufacturer#5|Brand#51|ECONOMY PLATED COPPER|41|LG PACK|997.09|es. carefully pendin 6092|forest goldenrod deep maroon misty|Manufacturer#3|Brand#34|PROMO ANODIZED STEEL|22|JUMBO JAR|998.09|ns run f 6093|bisque antique cream gainsboro navajo|Manufacturer#4|Brand#43|LARGE ANODIZED BRASS|40|JUMBO CASE|999.09|g depende 6094|bisque frosted drab sky snow|Manufacturer#5|Brand#55|ECONOMY BRUSHED TIN|28|JUMBO BAG|1000.09|ess, pending requests 6095|green orchid dark misty moccasin|Manufacturer#1|Brand#14|ECONOMY BRUSHED STEEL|30|JUMBO JAR|1001.09|. furiously 6096|khaki gainsboro black firebrick royal|Manufacturer#3|Brand#31|SMALL BURNISHED BRASS|30|MED CAN|1002.09|nd the blithely 6097|chocolate violet gainsboro smoke puff|Manufacturer#3|Brand#31|PROMO ANODIZED NICKEL|17|SM CASE|1003.09|cajole blithely 6098|ghost chocolate steel moccasin goldenrod|Manufacturer#1|Brand#13|MEDIUM BRUSHED NICKEL|26|MED PKG|1004.09|. daringly 6099|royal green plum honeydew medium|Manufacturer#2|Brand#25|LARGE BURNISHED STEEL|11|LG PACK|1005.09|ng packages hag 6100|papaya yellow chocolate sky indian|Manufacturer#5|Brand#55|MEDIUM PLATED NICKEL|20|LG BOX|1006.10|ns integrate. regu 6101|goldenrod rosy metallic plum seashell|Manufacturer#2|Brand#25|PROMO ANODIZED COPPER|34|MED BAG|1007.10|. furiously 6102|rosy yellow violet sienna snow|Manufacturer#5|Brand#51|ECONOMY ANODIZED BRASS|50|SM BOX|1008.10|ts are. c 6103|olive mint deep dodger bisque|Manufacturer#4|Brand#44|STANDARD ANODIZED BRASS|8|WRAP PKG|1009.10|leep blith 6104|aquamarine bisque cornflower cyan pale|Manufacturer#5|Brand#52|SMALL BRUSHED STEEL|37|WRAP JAR|1010.10|gedly a 6105|goldenrod cornflower light sandy rosy|Manufacturer#4|Brand#42|MEDIUM BRUSHED TIN|14|MED BAG|1011.10|ns. regular ideas 6106|lace pale dark blush frosted|Manufacturer#3|Brand#35|LARGE ANODIZED COPPER|26|SM BOX|1012.10|ccounts doze careful 6107|dim chartreuse brown salmon antique|Manufacturer#4|Brand#42|ECONOMY ANODIZED TIN|40|MED BOX|1013.10|are carefully among th 6108|cream medium rosy navy brown|Manufacturer#5|Brand#54|SMALL PLATED COPPER|42|SM CAN|1014.10| ironic requests inte 6109|saddle lawn goldenrod yellow dark|Manufacturer#1|Brand#13|STANDARD PLATED COPPER|19|SM PKG|1015.10|y among th 6110|chiffon snow ivory blanched cream|Manufacturer#1|Brand#12|SMALL BURNISHED NICKEL|42|LG JAR|1016.11|ual requests-- fluffi 6111|deep yellow dark lawn antique|Manufacturer#5|Brand#52|PROMO BRUSHED TIN|18|SM JAR|1017.11|tain above the 6112|metallic dark indian brown chartreuse|Manufacturer#4|Brand#44|ECONOMY ANODIZED NICKEL|30|MED PACK|1018.11|equests amo 6113|peach purple puff beige spring|Manufacturer#4|Brand#45|PROMO PLATED COPPER|10|JUMBO PKG|1019.11|he slyly ironic 6114|pale powder chartreuse lime moccasin|Manufacturer#4|Brand#41|STANDARD BRUSHED BRASS|41|LG CASE|1020.11|thely s 6115|chiffon rose plum puff peach|Manufacturer#1|Brand#11|ECONOMY BURNISHED STEEL|33|SM JAR|1021.11|n, unus 6116|burnished khaki wheat sienna violet|Manufacturer#5|Brand#54|PROMO BRUSHED TIN|43|LG BOX|1022.11|ing acc 6117|brown dim lace honeydew burlywood|Manufacturer#5|Brand#51|STANDARD PLATED COPPER|13|SM JAR|1023.11|sual instructions. 6118|rose violet linen burnished frosted|Manufacturer#3|Brand#32|ECONOMY ANODIZED STEEL|37|LG BOX|1024.11|ly acr 6119|royal almond magenta puff navy|Manufacturer#3|Brand#32|MEDIUM BURNISHED BRASS|36|LG BAG|1025.11|atelets. quickly regul 6120|peach white puff dim yellow|Manufacturer#2|Brand#23|ECONOMY ANODIZED COPPER|28|MED BAG|1026.12|bold instructions. 6121|sky papaya bisque gainsboro aquamarine|Manufacturer#1|Brand#13|MEDIUM BRUSHED TIN|29|LG DRUM|1027.12|ly pending pinto b 6122|tomato drab misty grey medium|Manufacturer#5|Brand#55|ECONOMY PLATED NICKEL|41|JUMBO BAG|1028.12|sublate furi 6123|cornsilk sandy khaki indian lawn|Manufacturer#3|Brand#33|PROMO POLISHED STEEL|23|JUMBO CAN|1029.12|bove 6124|red papaya lime cornsilk maroon|Manufacturer#1|Brand#13|SMALL ANODIZED TIN|15|MED DRUM|1030.12|efully fi 6125|ghost cornflower forest indian antique|Manufacturer#4|Brand#44|ECONOMY BURNISHED BRASS|34|MED BAG|1031.12|ve the express pint 6126|yellow smoke cyan khaki steel|Manufacturer#5|Brand#55|MEDIUM PLATED NICKEL|20|WRAP BOX|1032.12|of the blithel 6127|dim lavender thistle magenta sienna|Manufacturer#3|Brand#35|STANDARD BURNISHED TIN|15|WRAP PKG|1033.12| bold deposits ca 6128|sandy tomato light cream midnight|Manufacturer#2|Brand#24|LARGE POLISHED COPPER|8|SM JAR|1034.12|ans integrate quickl 6129|grey cyan sky seashell light|Manufacturer#4|Brand#43|LARGE POLISHED TIN|44|MED JAR|1035.12| should boos 6130|coral azure chartreuse tomato turquoise|Manufacturer#3|Brand#31|MEDIUM BURNISHED STEEL|9|WRAP PACK|1036.13|ggle furiousl 6131|coral brown forest puff lemon|Manufacturer#1|Brand#11|ECONOMY POLISHED NICKEL|1|WRAP BAG|1037.13|es must nag across the 6132|spring lime beige chiffon ghost|Manufacturer#1|Brand#13|MEDIUM PLATED COPPER|41|SM DRUM|1038.13|eas among the slyl 6133|turquoise goldenrod coral hot chartreuse|Manufacturer#3|Brand#32|STANDARD ANODIZED NICKEL|32|LG BOX|1039.13|ully special 6134|brown steel magenta blush sky|Manufacturer#4|Brand#42|LARGE BURNISHED NICKEL|18|MED CAN|1040.13|uests dazzle 6135|cream yellow puff lavender almond|Manufacturer#4|Brand#44|LARGE BURNISHED BRASS|48|JUMBO JAR|1041.13|riously regular 6136|slate peach moccasin mint forest|Manufacturer#4|Brand#41|ECONOMY POLISHED COPPER|16|LG BAG|1042.13|ar pinto 6137|snow misty honeydew thistle ivory|Manufacturer#1|Brand#11|ECONOMY PLATED COPPER|19|JUMBO DRUM|1043.13|riously 6138|forest black deep mint navy|Manufacturer#1|Brand#11|MEDIUM BRUSHED COPPER|8|SM JAR|1044.13|ges play after t 6139|rose lavender snow forest cream|Manufacturer#4|Brand#45|ECONOMY BRUSHED STEEL|4|JUMBO CASE|1045.13|es. asympt 6140|almond medium midnight cornflower beige|Manufacturer#2|Brand#24|SMALL PLATED NICKEL|6|MED CASE|1046.14|equests 6141|purple honeydew midnight lavender red|Manufacturer#3|Brand#35|STANDARD POLISHED TIN|32|SM BOX|1047.14|ole carefully speci 6142|drab deep ivory ghost lace|Manufacturer#2|Brand#23|SMALL BURNISHED NICKEL|30|MED JAR|1048.14|endencies sleep furio 6143|lawn peru medium maroon blue|Manufacturer#1|Brand#12|ECONOMY ANODIZED BRASS|8|LG BAG|1049.14|equests. fluffily 6144|burlywood grey sky sienna gainsboro|Manufacturer#1|Brand#12|STANDARD BURNISHED STEEL|33|WRAP DRUM|1050.14|erns are. sl 6145|dodger peru metallic ghost cornsilk|Manufacturer#1|Brand#15|STANDARD PLATED BRASS|30|WRAP CASE|1051.14|etly ironic accounts. 6146|pink powder white firebrick smoke|Manufacturer#3|Brand#34|SMALL BURNISHED NICKEL|37|JUMBO CASE|1052.14|rhorses wake. caref 6147|yellow bisque cream light medium|Manufacturer#5|Brand#54|ECONOMY POLISHED BRASS|34|MED BOX|1053.14|ously r 6148|sienna spring turquoise almond orange|Manufacturer#3|Brand#34|PROMO BRUSHED STEEL|50|LG PACK|1054.14|y silent 6149|deep papaya blanched honeydew snow|Manufacturer#3|Brand#35|ECONOMY BRUSHED TIN|50|MED CAN|1055.14|coys. 6150|orange powder tomato lemon brown|Manufacturer#1|Brand#11|STANDARD ANODIZED TIN|26|SM PKG|1056.15| blithe do 6151|floral ivory cyan sienna pale|Manufacturer#4|Brand#42|LARGE BRUSHED NICKEL|7|JUMBO CASE|1057.15|hins should cajole. 6152|burnished chiffon cornsilk linen orange|Manufacturer#3|Brand#32|PROMO BURNISHED TIN|42|WRAP PKG|1058.15|ly. unusua 6153|almond orange indian ghost chocolate|Manufacturer#2|Brand#23|MEDIUM BURNISHED COPPER|25|MED BAG|1059.15|instr 6154|lavender metallic black chiffon orchid|Manufacturer#5|Brand#54|PROMO PLATED BRASS|1|LG DRUM|1060.15|ss, express accoun 6155|cyan orange yellow papaya moccasin|Manufacturer#5|Brand#52|PROMO ANODIZED COPPER|2|JUMBO CAN|1061.15|es affi 6156|red maroon lavender burnished brown|Manufacturer#3|Brand#34|PROMO BRUSHED STEEL|36|SM PKG|1062.15|nal instructions 6157|blanched ghost deep sky red|Manufacturer#2|Brand#21|MEDIUM ANODIZED NICKEL|38|JUMBO CASE|1063.15|s wake 6158|navajo peach pink orchid midnight|Manufacturer#3|Brand#34|STANDARD POLISHED STEEL|13|WRAP PACK|1064.15|lly b 6159|orchid salmon tomato honeydew indian|Manufacturer#3|Brand#34|LARGE POLISHED STEEL|15|JUMBO DRUM|1065.15|cording to the final, 6160|navy navajo gainsboro chocolate pale|Manufacturer#2|Brand#22|SMALL POLISHED TIN|40|LG BAG|1066.16|ithely ironic 6161|smoke medium green gainsboro violet|Manufacturer#2|Brand#22|STANDARD PLATED STEEL|12|MED JAR|1067.16| fluffily. slyly 6162|seashell dim snow saddle grey|Manufacturer#3|Brand#32|STANDARD ANODIZED COPPER|16|JUMBO BAG|1068.16|hely slyly regu 6163|linen ghost dodger drab burlywood|Manufacturer#1|Brand#11|MEDIUM BURNISHED TIN|1|WRAP BOX|1069.16|t, final foxes cajo 6164|white drab saddle snow purple|Manufacturer#1|Brand#14|PROMO BRUSHED TIN|8|JUMBO BAG|1070.16|sly slow pearls wake 6165|frosted moccasin red orchid khaki|Manufacturer#1|Brand#12|SMALL ANODIZED BRASS|44|JUMBO JAR|1071.16|carefully b 6166|indian orange orchid turquoise blush|Manufacturer#4|Brand#45|MEDIUM PLATED NICKEL|25|WRAP PACK|1072.16|ven pinto b 6167|lawn seashell cornflower coral indian|Manufacturer#5|Brand#54|SMALL PLATED COPPER|16|LG BOX|1073.16|packages mold slyly r 6168|midnight plum blue spring floral|Manufacturer#1|Brand#14|STANDARD BURNISHED BRASS|12|JUMBO BOX|1074.16|ideas affix 6169|dim violet sienna bisque saddle|Manufacturer#4|Brand#43|STANDARD PLATED BRASS|26|LG PKG|1075.16|even accounts boost. f 6170|honeydew yellow deep blanched smoke|Manufacturer#3|Brand#31|LARGE BRUSHED BRASS|5|LG DRUM|1076.17| fluffily ab 6171|peru light sandy sky antique|Manufacturer#4|Brand#42|ECONOMY ANODIZED STEEL|48|JUMBO JAR|1077.17| along the c 6172|khaki rose dodger metallic orange|Manufacturer#5|Brand#54|PROMO BRUSHED COPPER|34|JUMBO CAN|1078.17|ests integrate 6173|rosy mint cream pale lavender|Manufacturer#4|Brand#43|SMALL PLATED STEEL|10|WRAP BOX|1079.17|ang slyly of 6174|dark seashell spring lavender sienna|Manufacturer#2|Brand#25|SMALL BRUSHED STEEL|8|JUMBO DRUM|1080.17|onic, ironic requests 6175|plum olive papaya purple orange|Manufacturer#1|Brand#15|STANDARD POLISHED TIN|43|SM JAR|1081.17|y against the slyly p 6176|tomato cyan chartreuse sky navy|Manufacturer#5|Brand#55|SMALL BRUSHED STEEL|47|SM CAN|1082.17|ully care 6177|ghost peach turquoise goldenrod spring|Manufacturer#5|Brand#51|ECONOMY BURNISHED COPPER|45|SM CASE|1083.17|ids integrate sil 6178|blush aquamarine black moccasin slate|Manufacturer#3|Brand#33|STANDARD POLISHED COPPER|46|JUMBO PKG|1084.17|ial instructions 6179|beige dark peru moccasin brown|Manufacturer#3|Brand#33|LARGE ANODIZED STEEL|3|LG PKG|1085.17|nts of the unu 6180|mint gainsboro powder maroon royal|Manufacturer#2|Brand#21|MEDIUM POLISHED BRASS|14|LG BOX|1086.18|e carefully unusual th 6181|brown mint drab lemon orange|Manufacturer#2|Brand#23|ECONOMY POLISHED COPPER|8|WRAP BAG|1087.18|ans cajole. 6182|burlywood light dodger honeydew cyan|Manufacturer#3|Brand#34|MEDIUM BRUSHED NICKEL|5|LG BOX|1088.18|ly inside the pla 6183|sienna sandy aquamarine cornsilk deep|Manufacturer#5|Brand#52|SMALL BRUSHED STEEL|39|WRAP BAG|1089.18|ly ironic accounts int 6184|dark orchid blush goldenrod khaki|Manufacturer#5|Brand#55|LARGE PLATED BRASS|15|MED JAR|1090.18|fully final 6185|light snow tan cornsilk misty|Manufacturer#4|Brand#41|STANDARD POLISHED COPPER|39|LG BOX|1091.18|ts bo 6186|antique burnished moccasin cornflower light|Manufacturer#1|Brand#13|LARGE BRUSHED NICKEL|29|LG JAR|1092.18| careful 6187|black burnished cornflower light saddle|Manufacturer#1|Brand#11|LARGE PLATED STEEL|5|JUMBO BAG|1093.18|sits. 6188|indian steel magenta misty olive|Manufacturer#2|Brand#23|SMALL BRUSHED STEEL|13|MED CASE|1094.18| beans boost. r 6189|chocolate midnight deep papaya salmon|Manufacturer#5|Brand#55|SMALL BURNISHED COPPER|29|JUMBO CASE|1095.18|bold foxes wake 6190|chiffon slate navy orchid almond|Manufacturer#4|Brand#43|PROMO BURNISHED COPPER|4|WRAP BOX|1096.19|e fluffily a 6191|papaya cyan purple aquamarine ghost|Manufacturer#4|Brand#43|STANDARD BURNISHED STEEL|4|SM BAG|1097.19|ts. idl 6192|frosted medium mint light drab|Manufacturer#5|Brand#52|PROMO POLISHED BRASS|1|MED PACK|1098.19|s. express pl 6193|aquamarine thistle green almond spring|Manufacturer#4|Brand#45|MEDIUM POLISHED COPPER|38|MED BOX|1099.19|gside of the ruthless, 6194|gainsboro violet pink blanched royal|Manufacturer#1|Brand#11|LARGE BURNISHED COPPER|24|WRAP CAN|1100.19| foxes cajole. s 6195|violet tan salmon moccasin linen|Manufacturer#4|Brand#45|LARGE ANODIZED NICKEL|10|SM JAR|1101.19|ular theodolites. r 6196|cream maroon metallic orange peru|Manufacturer#4|Brand#43|ECONOMY POLISHED TIN|48|MED JAR|1102.19|d, unu 6197|violet saddle cyan orange ivory|Manufacturer#4|Brand#44|SMALL POLISHED STEEL|27|MED JAR|1103.19|gular pinto beans nag 6198|saddle cyan mint linen turquoise|Manufacturer#3|Brand#32|LARGE POLISHED NICKEL|33|JUMBO CASE|1104.19|accounts across t 6199|chartreuse snow saddle ghost medium|Manufacturer#1|Brand#12|LARGE BURNISHED STEEL|18|MED JAR|1105.19|r courts. sl 6200|sky indian lavender coral midnight|Manufacturer#4|Brand#41|ECONOMY BURNISHED COPPER|25|JUMBO BAG|1106.20|ccounts are deposits. 6201|orchid sky frosted yellow lemon|Manufacturer#2|Brand#25|LARGE ANODIZED STEEL|46|SM CAN|1107.20|ole carefully sile 6202|cornsilk metallic sky puff papaya|Manufacturer#5|Brand#53|PROMO BRUSHED BRASS|45|WRAP PACK|1108.20|courts 6203|navy plum linen gainsboro midnight|Manufacturer#5|Brand#54|MEDIUM BRUSHED COPPER|50|WRAP DRUM|1109.20| tithes. furiously ev 6204|hot gainsboro beige rosy chiffon|Manufacturer#5|Brand#51|SMALL PLATED COPPER|40|LG CASE|1110.20|ecial depos 6205|antique gainsboro floral puff blush|Manufacturer#1|Brand#13|LARGE ANODIZED BRASS|16|SM PKG|1111.20|ar requests sleep abov 6206|salmon sandy floral khaki mint|Manufacturer#3|Brand#31|SMALL ANODIZED BRASS|35|LG JAR|1112.20|refully final 6207|antique goldenrod rosy metallic cornsilk|Manufacturer#4|Brand#42|STANDARD POLISHED NICKEL|45|MED PKG|1113.20|nal packa 6208|bisque light saddle black sandy|Manufacturer#1|Brand#11|PROMO BRUSHED TIN|17|SM CAN|1114.20|y unusual deposits a 6209|maroon sienna peach tomato deep|Manufacturer#2|Brand#25|MEDIUM PLATED BRASS|47|JUMBO DRUM|1115.20|ial theodoli 6210|lime lavender grey chiffon burlywood|Manufacturer#5|Brand#51|MEDIUM ANODIZED TIN|9|MED BAG|1116.21|ons lose 6211|orange cyan light beige chiffon|Manufacturer#4|Brand#42|STANDARD POLISHED NICKEL|2|SM JAR|1117.21|ss asymptot 6212|lace medium burnished peach tan|Manufacturer#2|Brand#25|LARGE POLISHED BRASS|9|SM BAG|1118.21|lithely sly 6213|steel snow slate papaya dark|Manufacturer#3|Brand#35|MEDIUM ANODIZED BRASS|15|LG BAG|1119.21|across the furiously 6214|floral spring light firebrick blush|Manufacturer#5|Brand#55|MEDIUM PLATED NICKEL|42|MED CASE|1120.21|ideas. accou 6215|midnight cornflower cornsilk brown antique|Manufacturer#3|Brand#35|STANDARD POLISHED TIN|28|LG JAR|1121.21|e fluffi 6216|forest beige red blush deep|Manufacturer#5|Brand#51|MEDIUM BRUSHED BRASS|47|MED JAR|1122.21|ly about the furiou 6217|aquamarine peru chartreuse green pink|Manufacturer#4|Brand#43|PROMO POLISHED STEEL|36|LG JAR|1123.21|egular pinto beans caj 6218|powder coral olive maroon lace|Manufacturer#3|Brand#33|STANDARD ANODIZED NICKEL|8|WRAP BOX|1124.21|ounts must 6219|honeydew wheat chartreuse drab sky|Manufacturer#4|Brand#44|ECONOMY ANODIZED NICKEL|37|LG PKG|1125.21|nic de 6220|purple light misty frosted puff|Manufacturer#5|Brand#54|LARGE PLATED NICKEL|18|SM BAG|1126.22|ly iron 6221|chartreuse rosy peach tan goldenrod|Manufacturer#4|Brand#41|STANDARD ANODIZED NICKEL|42|MED PKG|1127.22|lly final idea 6222|yellow lemon blush beige light|Manufacturer#2|Brand#25|SMALL POLISHED TIN|16|LG JAR|1128.22| quickly expre 6223|bisque sienna drab yellow almond|Manufacturer#3|Brand#32|PROMO POLISHED COPPER|40|SM BAG|1129.22|lly ironic th 6224|sky dim orchid snow pink|Manufacturer#5|Brand#53|LARGE BRUSHED NICKEL|23|LG PKG|1130.22|realms nag carefu 6225|red cyan green dodger burnished|Manufacturer#5|Brand#54|LARGE BRUSHED STEEL|9|JUMBO BAG|1131.22|usly unusual 6226|drab black tomato pink almond|Manufacturer#3|Brand#34|LARGE ANODIZED TIN|18|JUMBO CAN|1132.22|ls. foxes i 6227|sienna firebrick misty linen sandy|Manufacturer#4|Brand#42|ECONOMY POLISHED TIN|40|LG CASE|1133.22|lly regular theodolit 6228|frosted burlywood dodger red aquamarine|Manufacturer#1|Brand#13|PROMO PLATED STEEL|16|LG BOX|1134.22|d excuses c 6229|bisque white khaki honeydew azure|Manufacturer#4|Brand#43|PROMO BRUSHED BRASS|29|WRAP PKG|1135.22|lar fo 6230|powder navajo linen salmon turquoise|Manufacturer#2|Brand#22|SMALL POLISHED COPPER|35|SM PACK|1136.23|ickly express i 6231|green dodger almond gainsboro indian|Manufacturer#2|Brand#21|PROMO ANODIZED STEEL|20|MED PACK|1137.23| even accounts m 6232|blanched linen cream almond orange|Manufacturer#1|Brand#11|MEDIUM ANODIZED BRASS|30|LG JAR|1138.23|unusual, regular acco 6233|blush floral bisque dodger drab|Manufacturer#3|Brand#33|ECONOMY BURNISHED BRASS|49|JUMBO BOX|1139.23| the final foxes s 6234|green steel grey olive thistle|Manufacturer#4|Brand#45|STANDARD BURNISHED BRASS|50|LG PACK|1140.23|phins wake caref 6235|white tomato goldenrod wheat grey|Manufacturer#2|Brand#25|SMALL PLATED TIN|50|WRAP PACK|1141.23|packages boost 6236|misty lace metallic ivory pink|Manufacturer#4|Brand#45|ECONOMY BRUSHED NICKEL|41|MED PKG|1142.23|onic grouches detect 6237|firebrick white papaya sandy magenta|Manufacturer#1|Brand#14|ECONOMY PLATED TIN|15|SM PKG|1143.23|y after the 6238|powder lace deep lime steel|Manufacturer#1|Brand#15|LARGE ANODIZED TIN|30|MED PACK|1144.23|under the final cour 6239|chocolate spring wheat lawn tan|Manufacturer#5|Brand#51|LARGE BURNISHED STEEL|8|SM JAR|1145.23|y regular multipliers 6240|chartreuse grey thistle azure moccasin|Manufacturer#2|Brand#22|MEDIUM ANODIZED STEEL|34|JUMBO CAN|1146.24|y eve 6241|misty cornflower plum slate medium|Manufacturer#1|Brand#14|SMALL ANODIZED TIN|23|WRAP JAR|1147.24|ptotes across the eve 6242|linen powder brown lace red|Manufacturer#2|Brand#22|PROMO POLISHED STEEL|14|MED BAG|1148.24|ly regular the 6243|royal blush forest papaya navajo|Manufacturer#5|Brand#53|SMALL POLISHED TIN|1|SM CASE|1149.24|he express, reg 6244|azure plum magenta rosy sandy|Manufacturer#5|Brand#55|SMALL ANODIZED TIN|35|LG CAN|1150.24|lly express packages 6245|almond burnished beige peru brown|Manufacturer#1|Brand#12|STANDARD PLATED BRASS|38|WRAP PKG|1151.24|al platele 6246|aquamarine azure smoke lime rose|Manufacturer#4|Brand#44|PROMO PLATED BRASS|47|MED BOX|1152.24|y iro 6247|rose yellow chiffon lemon thistle|Manufacturer#3|Brand#33|LARGE PLATED BRASS|18|MED BOX|1153.24|ong the unusual acco 6248|orange goldenrod chocolate papaya smoke|Manufacturer#3|Brand#35|SMALL POLISHED NICKEL|29|MED PKG|1154.24|efully stealthy req 6249|metallic chiffon ghost goldenrod salmon|Manufacturer#4|Brand#42|MEDIUM BRUSHED BRASS|25|JUMBO BOX|1155.24|e unusua 6250|navajo lace slate steel purple|Manufacturer#3|Brand#34|PROMO BURNISHED NICKEL|2|MED DRUM|1156.25| requests. blithel 6251|lemon gainsboro khaki lavender ivory|Manufacturer#3|Brand#34|PROMO POLISHED BRASS|40|SM BOX|1157.25|egular, unusual re 6252|salmon peach chocolate sky blush|Manufacturer#2|Brand#23|MEDIUM ANODIZED STEEL|45|SM CAN|1158.25|ong the deposits. 6253|seashell navajo snow sky grey|Manufacturer#2|Brand#21|STANDARD BRUSHED STEEL|18|LG CASE|1159.25|uffily ironic 6254|dark tan beige black moccasin|Manufacturer#3|Brand#32|SMALL BRUSHED BRASS|17|WRAP DRUM|1160.25|ular acc 6255|misty olive saddle peach navajo|Manufacturer#5|Brand#55|LARGE BURNISHED BRASS|38|LG PKG|1161.25|ymptotes about the fr 6256|steel lace lavender hot tomato|Manufacturer#3|Brand#33|ECONOMY ANODIZED STEEL|21|LG PACK|1162.25|sly pending gifts ha 6257|tan smoke azure rosy firebrick|Manufacturer#2|Brand#25|ECONOMY ANODIZED NICKEL|22|SM BOX|1163.25|wake. ironic pearls i 6258|red plum tan snow navajo|Manufacturer#5|Brand#55|ECONOMY PLATED STEEL|48|JUMBO BAG|1164.25|eposits. ironic, 6259|chartreuse cream saddle hot spring|Manufacturer#3|Brand#35|STANDARD ANODIZED TIN|33|WRAP PKG|1165.25|ly ironic ideas will 6260|dark cornsilk thistle cornflower saddle|Manufacturer#1|Brand#11|LARGE PLATED STEEL|23|SM PACK|1166.26|ss requests sleep fl 6261|khaki pale blush turquoise mint|Manufacturer#2|Brand#25|ECONOMY BURNISHED COPPER|11|MED PACK|1167.26|ns wake regu 6262|goldenrod aquamarine forest beige purple|Manufacturer#1|Brand#11|MEDIUM POLISHED STEEL|28|WRAP PKG|1168.26|le. pinto beans d 6263|saddle cornsilk aquamarine coral magenta|Manufacturer#5|Brand#52|STANDARD BRUSHED COPPER|14|MED PACK|1169.26|s haggle final pinto 6264|khaki ghost azure antique red|Manufacturer#2|Brand#21|ECONOMY POLISHED TIN|1|WRAP PKG|1170.26|heodolite 6265|lawn lemon indian thistle firebrick|Manufacturer#1|Brand#12|PROMO PLATED COPPER|27|SM PACK|1171.26|ons are careful 6266|powder lace indian rose brown|Manufacturer#1|Brand#13|ECONOMY BURNISHED BRASS|38|SM CAN|1172.26|iously. final instr 6267|azure sky peach slate frosted|Manufacturer#3|Brand#35|ECONOMY POLISHED STEEL|13|WRAP DRUM|1173.26|. carefully b 6268|spring puff coral sandy drab|Manufacturer#1|Brand#11|MEDIUM ANODIZED TIN|17|LG CASE|1174.26|es. fi 6269|navajo saddle thistle papaya black|Manufacturer#3|Brand#31|LARGE ANODIZED NICKEL|41|LG PKG|1175.26|g the 6270|wheat coral chiffon puff blanched|Manufacturer#3|Brand#34|STANDARD BURNISHED STEEL|28|LG BOX|1176.27|l deposits cajole ca 6271|azure sandy blue dodger honeydew|Manufacturer#5|Brand#52|LARGE BURNISHED NICKEL|42|JUMBO JAR|1177.27|onic platelets above 6272|blue floral forest steel salmon|Manufacturer#3|Brand#32|SMALL BRUSHED COPPER|20|JUMBO DRUM|1178.27|among the carefully re 6273|ghost saddle black seashell moccasin|Manufacturer#5|Brand#52|ECONOMY PLATED TIN|26|WRAP BAG|1179.27|inal 6274|lace azure blush wheat frosted|Manufacturer#2|Brand#24|MEDIUM BURNISHED STEEL|18|WRAP DRUM|1180.27|o the even grouches 6275|gainsboro ghost tan rosy ivory|Manufacturer#1|Brand#13|MEDIUM PLATED STEEL|10|SM DRUM|1181.27|foxes. f 6276|khaki slate red frosted snow|Manufacturer#5|Brand#53|SMALL ANODIZED TIN|47|WRAP JAR|1182.27|sts dazzle blit 6277|black forest orange pink medium|Manufacturer#1|Brand#11|SMALL POLISHED TIN|18|LG BAG|1183.27|ily fi 6278|saddle navy orchid forest olive|Manufacturer#5|Brand#54|STANDARD PLATED BRASS|23|WRAP PKG|1184.27|nts. sl 6279|steel ivory blush chiffon cornsilk|Manufacturer#4|Brand#41|SMALL BURNISHED STEEL|15|JUMBO PACK|1185.27|elets 6280|azure antique thistle dark pink|Manufacturer#1|Brand#11|LARGE ANODIZED COPPER|1|MED CASE|1186.28|he regular, regu 6281|tomato honeydew lawn black orange|Manufacturer#3|Brand#33|LARGE BRUSHED BRASS|4|JUMBO PKG|1187.28|deposits haggle 6282|navajo linen dodger honeydew misty|Manufacturer#5|Brand#51|MEDIUM BRUSHED STEEL|3|LG PKG|1188.28|y. even req 6283|puff dim sandy chocolate snow|Manufacturer#5|Brand#52|PROMO ANODIZED NICKEL|50|WRAP CASE|1189.28|ecial orbits bo 6284|maroon spring dodger black blush|Manufacturer#3|Brand#33|MEDIUM PLATED BRASS|38|LG CAN|1190.28|deposits 6285|steel plum mint purple green|Manufacturer#2|Brand#23|STANDARD PLATED STEEL|1|MED PKG|1191.28|nder 6286|wheat pale thistle chartreuse brown|Manufacturer#3|Brand#32|PROMO BRUSHED BRASS|21|JUMBO BOX|1192.28|s. sl 6287|frosted peru bisque powder hot|Manufacturer#5|Brand#55|SMALL PLATED COPPER|49|WRAP DRUM|1193.28|anent deposits haggle 6288|yellow lawn pink peach mint|Manufacturer#2|Brand#21|PROMO BRUSHED TIN|27|MED DRUM|1194.28|the slyly pending the 6289|beige aquamarine dodger chiffon black|Manufacturer#4|Brand#45|MEDIUM BURNISHED BRASS|38|SM BAG|1195.28|s. final, even requ 6290|orange black tomato powder purple|Manufacturer#2|Brand#24|LARGE ANODIZED STEEL|44|WRAP CASE|1196.29|ts boost car 6291|burnished powder chiffon orange wheat|Manufacturer#3|Brand#34|MEDIUM BURNISHED NICKEL|39|WRAP CAN|1197.29|eaves sle 6292|bisque brown moccasin snow maroon|Manufacturer#4|Brand#44|PROMO PLATED BRASS|28|MED DRUM|1198.29|ugouts. final accounts 6293|floral azure brown blue tomato|Manufacturer#5|Brand#54|PROMO BRUSHED COPPER|28|MED CASE|1199.29|dependencies. even 6294|metallic magenta aquamarine midnight linen|Manufacturer#4|Brand#44|MEDIUM POLISHED TIN|44|SM BOX|1200.29|excuses above the 6295|blanched cyan turquoise saddle white|Manufacturer#4|Brand#43|LARGE BRUSHED BRASS|40|SM BAG|1201.29|kly fluffily unusual 6296|coral metallic yellow plum goldenrod|Manufacturer#1|Brand#13|PROMO BRUSHED NICKEL|47|MED BOX|1202.29|cial deposits d 6297|steel burnished dodger cornflower coral|Manufacturer#4|Brand#45|ECONOMY POLISHED TIN|2|JUMBO DRUM|1203.29|ly unusual theodol 6298|cornsilk sky violet green burnished|Manufacturer#3|Brand#35|MEDIUM POLISHED NICKEL|47|SM BAG|1204.29|tes. furiously pend 6299|antique wheat blue seashell hot|Manufacturer#3|Brand#34|PROMO BRUSHED STEEL|43|JUMBO JAR|1205.29|totes. regularly 6300|navy black cornflower white aquamarine|Manufacturer#4|Brand#41|LARGE POLISHED BRASS|10|JUMBO BAG|1206.30|quests haggle 6301|blush ghost azure peach lemon|Manufacturer#4|Brand#45|LARGE ANODIZED BRASS|8|LG CAN|1207.30| pinto b 6302|lace frosted mint misty cream|Manufacturer#4|Brand#42|LARGE PLATED TIN|28|LG JAR|1208.30|blithely even instru 6303|dark powder almond honeydew burnished|Manufacturer#1|Brand#11|SMALL ANODIZED STEEL|46|JUMBO BOX|1209.30|ckages; 6304|chocolate dodger coral maroon beige|Manufacturer#2|Brand#21|MEDIUM BURNISHED BRASS|8|JUMBO DRUM|1210.30|ackages cajole at th 6305|white metallic hot antique olive|Manufacturer#3|Brand#33|ECONOMY BURNISHED COPPER|11|MED PKG|1211.30|ing packages 6306|navajo papaya metallic antique tan|Manufacturer#5|Brand#54|SMALL BRUSHED COPPER|5|WRAP PKG|1212.30|ound the care 6307|orchid magenta papaya cornflower wheat|Manufacturer#5|Brand#52|PROMO ANODIZED STEEL|25|JUMBO CAN|1213.30|kages. caref 6308|seashell red medium violet misty|Manufacturer#2|Brand#22|ECONOMY BRUSHED BRASS|14|LG CAN|1214.30|lose requests. s 6309|hot gainsboro cornflower chartreuse chiffon|Manufacturer#4|Brand#44|PROMO BRUSHED NICKEL|22|SM BOX|1215.30|realms cajole fu 6310|brown cream olive black goldenrod|Manufacturer#4|Brand#44|STANDARD ANODIZED TIN|12|WRAP PKG|1216.31|bout the 6311|rose misty ghost burlywood blush|Manufacturer#2|Brand#25|PROMO PLATED NICKEL|6|LG BOX|1217.31|ully even accounts 6312|burnished smoke metallic mint purple|Manufacturer#2|Brand#24|LARGE BURNISHED COPPER|48|JUMBO CAN|1218.31|ccounts-- sly 6313|yellow salmon gainsboro beige lace|Manufacturer#1|Brand#14|SMALL BURNISHED STEEL|27|LG CAN|1219.31|gular inst 6314|white turquoise plum goldenrod smoke|Manufacturer#5|Brand#51|MEDIUM PLATED TIN|3|LG CAN|1220.31|ake after the sly 6315|purple olive slate pale rosy|Manufacturer#5|Brand#51|LARGE PLATED COPPER|20|JUMBO PKG|1221.31|ecial a 6316|light peach wheat medium orange|Manufacturer#2|Brand#22|LARGE ANODIZED TIN|29|MED DRUM|1222.31|phs. iro 6317|orchid snow maroon ghost smoke|Manufacturer#2|Brand#22|PROMO BRUSHED NICKEL|34|WRAP JAR|1223.31|en accounts. 6318|dim indian coral seashell tomato|Manufacturer#5|Brand#54|ECONOMY BRUSHED COPPER|43|JUMBO BOX|1224.31|dolites. slyly iro 6319|tan pink salmon lavender spring|Manufacturer#2|Brand#23|PROMO BURNISHED COPPER|29|JUMBO CAN|1225.31|sits after the caref 6320|smoke orange purple white dark|Manufacturer#2|Brand#24|ECONOMY BRUSHED COPPER|21|MED BAG|1226.32|nly. furiously unu 6321|green medium goldenrod yellow honeydew|Manufacturer#2|Brand#23|LARGE PLATED COPPER|3|JUMBO PACK|1227.32|ts sleep along 6322|yellow violet medium firebrick turquoise|Manufacturer#5|Brand#52|ECONOMY POLISHED TIN|16|WRAP CAN|1228.32|y final foxes. bli 6323|aquamarine linen moccasin coral medium|Manufacturer#3|Brand#35|MEDIUM POLISHED BRASS|13|JUMBO PKG|1229.32|ng to the 6324|beige dodger burnished misty pink|Manufacturer#3|Brand#35|SMALL BURNISHED TIN|22|MED PKG|1230.32|gularly bo 6325|puff honeydew blush coral magenta|Manufacturer#1|Brand#11|LARGE ANODIZED STEEL|20|WRAP DRUM|1231.32|even deposits. 6326|blue olive lemon goldenrod slate|Manufacturer#3|Brand#35|PROMO POLISHED TIN|45|JUMBO CAN|1232.32|ounts dazzle 6327|forest navy powder black papaya|Manufacturer#1|Brand#11|MEDIUM BRUSHED BRASS|37|MED CAN|1233.32|ncies. even, even p 6328|chocolate red papaya lavender honeydew|Manufacturer#2|Brand#25|MEDIUM BURNISHED TIN|9|WRAP BOX|1234.32|kages. a 6329|lawn hot brown sienna tomato|Manufacturer#4|Brand#41|LARGE PLATED COPPER|47|JUMBO BOX|1235.32| special the 6330|firebrick lime ghost honeydew coral|Manufacturer#1|Brand#14|SMALL ANODIZED STEEL|15|SM PACK|1236.33|ts haggle furiousl 6331|maroon floral blanched burlywood red|Manufacturer#5|Brand#53|LARGE ANODIZED NICKEL|8|WRAP CAN|1237.33|ackages. bo 6332|honeydew sienna rose aquamarine moccasin|Manufacturer#4|Brand#44|LARGE PLATED STEEL|27|MED BOX|1238.33|among the regular 6333|blanched pink black drab wheat|Manufacturer#2|Brand#23|LARGE BURNISHED TIN|28|JUMBO BAG|1239.33|ages. careful 6334|indian wheat turquoise blanched linen|Manufacturer#4|Brand#42|ECONOMY PLATED STEEL|46|JUMBO DRUM|1240.33|e furiously unus 6335|lime burnished purple lemon linen|Manufacturer#3|Brand#31|ECONOMY BURNISHED TIN|12|JUMBO JAR|1241.33|ep. permanent, fina 6336|lemon dodger peach cyan blue|Manufacturer#2|Brand#24|SMALL BRUSHED STEEL|18|SM BAG|1242.33|. express 6337|steel orchid ivory antique navajo|Manufacturer#5|Brand#55|PROMO BURNISHED STEEL|5|SM DRUM|1243.33|xpress packa 6338|plum lace almond cyan seashell|Manufacturer#1|Brand#11|ECONOMY PLATED NICKEL|45|MED PKG|1244.33|unts again 6339|orange navajo coral mint black|Manufacturer#3|Brand#33|ECONOMY ANODIZED BRASS|10|WRAP DRUM|1245.33|eans cajo 6340|lavender midnight cornflower misty red|Manufacturer#2|Brand#21|LARGE POLISHED BRASS|27|MED DRUM|1246.34|nusual pac 6341|sky slate ghost smoke bisque|Manufacturer#5|Brand#51|STANDARD BRUSHED TIN|34|WRAP CASE|1247.34|ely thin accoun 6342|navy puff olive hot violet|Manufacturer#5|Brand#54|ECONOMY BRUSHED BRASS|30|MED CAN|1248.34|slyly unusual asymp 6343|saddle yellow moccasin tomato plum|Manufacturer#2|Brand#22|SMALL BURNISHED BRASS|6|SM CAN|1249.34|sts. fu 6344|blue frosted black linen chocolate|Manufacturer#3|Brand#33|SMALL BURNISHED BRASS|19|WRAP PKG|1250.34|s cajole fluffily. 6345|thistle sienna almond grey lemon|Manufacturer#5|Brand#53|SMALL BURNISHED COPPER|46|LG BAG|1251.34|kly f 6346|saddle ghost orchid brown blush|Manufacturer#5|Brand#55|STANDARD BRUSHED TIN|45|MED JAR|1252.34|nts haggle. ev 6347|honeydew cornsilk rosy linen blue|Manufacturer#4|Brand#44|LARGE BURNISHED BRASS|49|LG BOX|1253.34| alongside of 6348|purple rosy tomato maroon black|Manufacturer#5|Brand#54|ECONOMY POLISHED NICKEL|18|JUMBO PACK|1254.34| blithel 6349|forest frosted bisque orchid sky|Manufacturer#2|Brand#21|SMALL BURNISHED TIN|19|WRAP BAG|1255.34| the f 6350|blush papaya salmon sky gainsboro|Manufacturer#5|Brand#54|ECONOMY BRUSHED BRASS|28|JUMBO BAG|1256.35| carefully 6351|tomato orange honeydew blanched metallic|Manufacturer#1|Brand#11|SMALL BRUSHED BRASS|32|JUMBO BAG|1257.35|refully regular, unu 6352|rosy plum light tomato wheat|Manufacturer#5|Brand#53|MEDIUM BURNISHED TIN|14|LG DRUM|1258.35|t ideas affix along 6353|drab linen blue steel floral|Manufacturer#5|Brand#54|LARGE PLATED COPPER|46|MED CAN|1259.35|ain qui 6354|lawn midnight olive plum blush|Manufacturer#4|Brand#43|SMALL PLATED NICKEL|22|MED PACK|1260.35| final theodolites caj 6355|burlywood moccasin beige dodger lace|Manufacturer#5|Brand#55|STANDARD BURNISHED COPPER|41|LG PACK|1261.35|reful 6356|moccasin magenta dim slate orchid|Manufacturer#3|Brand#35|STANDARD POLISHED STEEL|8|MED CASE|1262.35|according to the 6357|chartreuse snow sandy saddle blush|Manufacturer#1|Brand#12|LARGE BURNISHED NICKEL|14|LG BAG|1263.35|ng foxe 6358|slate royal ghost cyan dark|Manufacturer#5|Brand#51|ECONOMY BURNISHED TIN|35|JUMBO PACK|1264.35|slyly fluffily 6359|cream mint yellow grey sandy|Manufacturer#3|Brand#33|PROMO BRUSHED BRASS|10|WRAP CAN|1265.35|es are blithely. 6360|floral drab salmon purple navy|Manufacturer#3|Brand#32|PROMO BRUSHED COPPER|26|JUMBO PACK|1266.36|al instructions a 6361|orchid magenta indian cream ivory|Manufacturer#5|Brand#55|STANDARD PLATED TIN|35|MED PACK|1267.36|ts use. slyly exp 6362|maroon peru navy drab lawn|Manufacturer#2|Brand#21|SMALL PLATED COPPER|7|JUMBO PACK|1268.36|ecial accounts at 6363|midnight orange maroon forest tomato|Manufacturer#1|Brand#12|ECONOMY BRUSHED TIN|16|LG JAR|1269.36|press the 6364|yellow almond tan chartreuse cream|Manufacturer#5|Brand#54|ECONOMY ANODIZED COPPER|11|LG BAG|1270.36|longside of th 6365|maroon coral magenta salmon mint|Manufacturer#5|Brand#55|SMALL BRUSHED TIN|37|SM CASE|1271.36|le fluffily reque 6366|lawn blue rose gainsboro navajo|Manufacturer#5|Brand#54|STANDARD BURNISHED BRASS|23|SM BAG|1272.36|he furiously d 6367|dark papaya misty forest violet|Manufacturer#4|Brand#44|LARGE PLATED STEEL|14|MED DRUM|1273.36|sleep after th 6368|moccasin linen cornflower light chartreuse|Manufacturer#3|Brand#31|MEDIUM POLISHED BRASS|2|SM CASE|1274.36|r the depo 6369|chocolate lawn navajo violet antique|Manufacturer#4|Brand#44|MEDIUM POLISHED TIN|4|WRAP PKG|1275.36|riously specia 6370|dodger sienna pink aquamarine sky|Manufacturer#5|Brand#53|LARGE ANODIZED TIN|28|LG PKG|1276.37|ctions 6371|peach chocolate dark midnight frosted|Manufacturer#2|Brand#24|STANDARD PLATED BRASS|7|LG JAR|1277.37|ely even reques 6372|plum wheat tomato blue dim|Manufacturer#2|Brand#22|STANDARD PLATED BRASS|5|SM CAN|1278.37|e fluf 6373|royal gainsboro pale lemon misty|Manufacturer#5|Brand#52|STANDARD ANODIZED BRASS|40|MED DRUM|1279.37| across the slyly ex 6374|navajo white plum lemon midnight|Manufacturer#5|Brand#53|ECONOMY BRUSHED STEEL|40|MED CAN|1280.37|xpress 6375|dim peach cornsilk magenta gainsboro|Manufacturer#3|Brand#33|MEDIUM ANODIZED NICKEL|3|LG CAN|1281.37|eodolites. ironic r 6376|white coral burlywood ivory ghost|Manufacturer#3|Brand#32|LARGE BURNISHED STEEL|22|JUMBO JAR|1282.37|to the busily fi 6377|magenta black burnished lawn blush|Manufacturer#1|Brand#12|MEDIUM BURNISHED COPPER|38|LG BAG|1283.37|fily eve 6378|turquoise dim pale red tomato|Manufacturer#5|Brand#52|STANDARD ANODIZED COPPER|21|JUMBO BAG|1284.37|furiously final ide 6379|almond magenta light violet chiffon|Manufacturer#3|Brand#31|LARGE ANODIZED COPPER|27|JUMBO CAN|1285.37|hout the 6380|sky saddle azure yellow cornflower|Manufacturer#3|Brand#33|LARGE BRUSHED NICKEL|29|WRAP JAR|1286.38|the quic 6381|seashell chocolate midnight cyan misty|Manufacturer#2|Brand#25|LARGE BURNISHED BRASS|49|WRAP BOX|1287.38|nag quickly q 6382|orange peru chartreuse linen honeydew|Manufacturer#4|Brand#42|MEDIUM BRUSHED BRASS|8|SM JAR|1288.38|s lose blithely 6383|slate light spring magenta snow|Manufacturer#2|Brand#22|MEDIUM POLISHED STEEL|4|WRAP BAG|1289.38|gular asymptotes. 6384|dark snow maroon spring cornsilk|Manufacturer#5|Brand#51|SMALL ANODIZED BRASS|18|LG PKG|1290.38|unts. blithely dar 6385|sienna white gainsboro peach aquamarine|Manufacturer#1|Brand#11|LARGE BURNISHED NICKEL|24|WRAP BOX|1291.38|s slee 6386|navy azure blue chiffon papaya|Manufacturer#2|Brand#24|STANDARD POLISHED NICKEL|18|MED BAG|1292.38|furiously. 6387|slate firebrick orchid light azure|Manufacturer#4|Brand#45|STANDARD BURNISHED COPPER|25|SM PKG|1293.38|ar instructions. caref 6388|seashell pale maroon floral coral|Manufacturer#2|Brand#23|PROMO POLISHED TIN|47|SM DRUM|1294.38|onic package 6389|antique blush green khaki purple|Manufacturer#5|Brand#51|STANDARD PLATED BRASS|12|JUMBO PACK|1295.38|r foxes. blithely b 6390|bisque pink gainsboro rose lavender|Manufacturer#3|Brand#32|PROMO BRUSHED BRASS|22|MED BOX|1296.39|ctions cajole about 6391|rosy powder yellow linen cyan|Manufacturer#2|Brand#25|LARGE BURNISHED COPPER|1|LG DRUM|1297.39|unusua 6392|linen orchid navajo white ivory|Manufacturer#5|Brand#53|ECONOMY POLISHED COPPER|28|MED BAG|1298.39|furiousl 6393|cornflower dodger khaki rose blush|Manufacturer#4|Brand#42|PROMO ANODIZED TIN|32|JUMBO PACK|1299.39|oxes. regular accou 6394|salmon pink dim azure tomato|Manufacturer#1|Brand#12|PROMO BRUSHED TIN|26|MED PACK|1300.39|cording to the pe 6395|hot turquoise rose ghost burnished|Manufacturer#3|Brand#33|LARGE BRUSHED TIN|22|MED BOX|1301.39|quickly i 6396|dim cornflower frosted pale purple|Manufacturer#3|Brand#34|MEDIUM PLATED BRASS|8|SM BAG|1302.39|t the enticingly fina 6397|turquoise ivory hot rosy violet|Manufacturer#3|Brand#34|STANDARD PLATED COPPER|9|SM JAR|1303.39|accounts sle 6398|papaya sandy black forest lace|Manufacturer#1|Brand#13|ECONOMY BURNISHED BRASS|31|SM BOX|1304.39|ve the iro 6399|papaya saddle midnight sky aquamarine|Manufacturer#4|Brand#42|LARGE BURNISHED COPPER|30|SM JAR|1305.39|nstruction 6400|royal papaya navajo rose thistle|Manufacturer#4|Brand#42|SMALL ANODIZED COPPER|14|SM DRUM|1306.40|d the ironic instruc 6401|floral smoke saddle antique mint|Manufacturer#3|Brand#33|SMALL BRUSHED STEEL|11|SM BAG|1307.40|ial dolphins abov 6402|lawn light puff dark deep|Manufacturer#1|Brand#12|STANDARD POLISHED NICKEL|35|MED JAR|1308.40|he carefully ironi 6403|chiffon tan navajo grey plum|Manufacturer#2|Brand#21|LARGE ANODIZED BRASS|28|WRAP BOX|1309.40|e blithely. sly 6404|rosy turquoise saddle ghost lace|Manufacturer#2|Brand#22|PROMO BURNISHED COPPER|29|SM DRUM|1310.40| requ 6405|wheat metallic sandy ivory spring|Manufacturer#3|Brand#33|LARGE ANODIZED STEEL|24|LG BAG|1311.40| the ironic frays 6406|dim metallic burnished plum salmon|Manufacturer#2|Brand#25|PROMO ANODIZED BRASS|41|LG CAN|1312.40|furio 6407|blue red forest azure violet|Manufacturer#2|Brand#25|STANDARD POLISHED TIN|36|LG CASE|1313.40|y regular requests sl 6408|turquoise frosted tomato navy moccasin|Manufacturer#5|Brand#54|PROMO PLATED TIN|23|JUMBO PKG|1314.40| accounts 6409|frosted plum dark purple aquamarine|Manufacturer#1|Brand#12|PROMO POLISHED TIN|7|SM CAN|1315.40|osits. regu 6410|violet misty rosy antique olive|Manufacturer#4|Brand#44|PROMO BURNISHED TIN|12|MED CAN|1316.41|pecial instruc 6411|lemon deep almond red yellow|Manufacturer#1|Brand#14|STANDARD PLATED TIN|11|WRAP PKG|1317.41| the furiously sp 6412|misty plum moccasin slate orchid|Manufacturer#1|Brand#11|MEDIUM PLATED COPPER|37|JUMBO CAN|1318.41|its. ev 6413|peach azure cornsilk chocolate brown|Manufacturer#5|Brand#51|SMALL PLATED TIN|11|SM CASE|1319.41|inal a 6414|ghost floral khaki aquamarine hot|Manufacturer#4|Brand#43|PROMO BURNISHED TIN|20|WRAP DRUM|1320.41|tructions haggle 6415|forest saddle royal maroon slate|Manufacturer#5|Brand#55|STANDARD BRUSHED STEEL|28|WRAP BOX|1321.41|pending packag 6416|firebrick ivory tomato chiffon floral|Manufacturer#3|Brand#31|PROMO PLATED BRASS|22|JUMBO CASE|1322.41|lyly around t 6417|lace dark thistle medium navajo|Manufacturer#1|Brand#13|PROMO BURNISHED STEEL|23|SM PKG|1323.41|its nod 6418|rose puff orange moccasin royal|Manufacturer#2|Brand#22|PROMO PLATED NICKEL|6|JUMBO CASE|1324.41|furio 6419|chartreuse seashell lemon floral pale|Manufacturer#1|Brand#11|STANDARD POLISHED TIN|47|MED CAN|1325.41|s. perman 6420|azure sky honeydew puff papaya|Manufacturer#2|Brand#24|ECONOMY POLISHED BRASS|36|WRAP CAN|1326.42|reful 6421|sienna peru lace dodger mint|Manufacturer#2|Brand#23|LARGE POLISHED STEEL|10|MED PKG|1327.42|tructions. dogged 6422|olive dark black wheat navy|Manufacturer#2|Brand#24|MEDIUM ANODIZED STEEL|39|MED PACK|1328.42| the s 6423|magenta ghost pale blue honeydew|Manufacturer#3|Brand#33|SMALL PLATED TIN|36|LG BOX|1329.42| even packages hagg 6424|rosy khaki black burlywood floral|Manufacturer#1|Brand#13|LARGE POLISHED BRASS|17|JUMBO BOX|1330.42| pinto beans h 6425|dodger lawn honeydew ghost blush|Manufacturer#1|Brand#12|STANDARD POLISHED BRASS|50|LG PKG|1331.42|ackages. blithel 6426|green steel frosted burlywood puff|Manufacturer#2|Brand#23|LARGE PLATED TIN|44|LG CASE|1332.42|long the carefull 6427|thistle powder floral bisque sienna|Manufacturer#1|Brand#14|MEDIUM POLISHED BRASS|28|SM JAR|1333.42| the 6428|chartreuse lime burlywood magenta lawn|Manufacturer#4|Brand#43|SMALL POLISHED NICKEL|38|JUMBO CAN|1334.42|ironic dependenci 6429|olive tomato rose blue maroon|Manufacturer#2|Brand#22|SMALL ANODIZED TIN|36|JUMBO DRUM|1335.42|ess blithely around t 6430|turquoise tan indian olive ghost|Manufacturer#2|Brand#22|SMALL PLATED STEEL|3|LG JAR|1336.43| regular asymptotes 6431|yellow beige pale gainsboro orchid|Manufacturer#4|Brand#45|ECONOMY BRUSHED COPPER|50|WRAP BOX|1337.43|ironic epitaphs. pint 6432|floral spring snow papaya black|Manufacturer#2|Brand#23|LARGE PLATED TIN|33|SM BOX|1338.43|s. quick, spec 6433|khaki peru turquoise blanched lemon|Manufacturer#1|Brand#14|ECONOMY BRUSHED BRASS|19|LG DRUM|1339.43|dencies doubt. unusu 6434|midnight aquamarine chartreuse powder khaki|Manufacturer#3|Brand#35|MEDIUM ANODIZED NICKEL|46|SM BOX|1340.43|e slyly 6435|frosted wheat royal grey ghost|Manufacturer#5|Brand#51|ECONOMY ANODIZED TIN|12|MED JAR|1341.43|press pack 6436|navy saddle misty blanched gainsboro|Manufacturer#3|Brand#34|PROMO BURNISHED BRASS|40|MED CAN|1342.43|across 6437|brown drab mint sky chocolate|Manufacturer#3|Brand#31|PROMO POLISHED BRASS|16|LG CAN|1343.43| special re 6438|ivory coral misty steel salmon|Manufacturer#5|Brand#52|ECONOMY POLISHED BRASS|23|LG CASE|1344.43|ly express instr 6439|thistle steel lace chocolate chiffon|Manufacturer#5|Brand#55|SMALL ANODIZED COPPER|41|LG JAR|1345.43|arefully bold 6440|floral dim almond ghost thistle|Manufacturer#1|Brand#12|SMALL BURNISHED COPPER|32|LG PKG|1346.44|ously pending r 6441|peach honeydew red blush puff|Manufacturer#1|Brand#13|MEDIUM POLISHED COPPER|16|SM JAR|1347.44|deposi 6442|magenta ivory orange coral grey|Manufacturer#4|Brand#41|ECONOMY BRUSHED TIN|22|WRAP PACK|1348.44|ckages. ca 6443|rose misty brown lavender orange|Manufacturer#5|Brand#55|LARGE PLATED BRASS|26|JUMBO JAR|1349.44|ep quietly across 6444|blanched frosted blush forest burnished|Manufacturer#4|Brand#41|ECONOMY ANODIZED NICKEL|15|SM JAR|1350.44| ironic accounts si 6445|navy lemon grey sky rosy|Manufacturer#1|Brand#11|LARGE ANODIZED STEEL|9|SM CAN|1351.44|pinto beans haggle c 6446|almond floral metallic blush azure|Manufacturer#5|Brand#52|PROMO ANODIZED STEEL|29|SM BOX|1352.44|y regula 6447|olive purple chiffon linen magenta|Manufacturer#2|Brand#21|SMALL PLATED COPPER|48|WRAP BOX|1353.44| sleep. blithely 6448|tomato chartreuse chocolate seashell moccasin|Manufacturer#1|Brand#12|SMALL ANODIZED NICKEL|17|LG BOX|1354.44| fluffily. regula 6449|yellow orchid cream puff dark|Manufacturer#3|Brand#34|SMALL BURNISHED STEEL|12|LG PACK|1355.44|special d 6450|puff light red goldenrod salmon|Manufacturer#5|Brand#55|PROMO ANODIZED BRASS|14|LG PKG|1356.45|equests w 6451|chartreuse cream deep moccasin hot|Manufacturer#1|Brand#13|LARGE POLISHED COPPER|27|WRAP CASE|1357.45| haggle slyly qui 6452|red steel magenta frosted firebrick|Manufacturer#2|Brand#22|PROMO ANODIZED STEEL|36|MED DRUM|1358.45|regular reque 6453|peach wheat sienna dark black|Manufacturer#5|Brand#51|MEDIUM BURNISHED TIN|37|MED JAR|1359.45|y. blithely silen 6454|wheat olive papaya pale goldenrod|Manufacturer#4|Brand#41|LARGE BRUSHED BRASS|20|MED DRUM|1360.45|ackages. slyl 6455|rose snow olive grey navajo|Manufacturer#5|Brand#51|LARGE BURNISHED COPPER|14|WRAP JAR|1361.45|nic dep 6456|lemon almond spring ghost forest|Manufacturer#4|Brand#43|SMALL POLISHED STEEL|22|JUMBO JAR|1362.45|e furiously bold depo 6457|antique pink yellow salmon coral|Manufacturer#3|Brand#33|LARGE BURNISHED NICKEL|50|MED BAG|1363.45| unusual, 6458|turquoise wheat honeydew maroon midnight|Manufacturer#2|Brand#25|MEDIUM POLISHED STEEL|11|SM CAN|1364.45|s sleep blithely abov 6459|navajo almond ghost chartreuse plum|Manufacturer#1|Brand#14|MEDIUM BRUSHED STEEL|14|JUMBO BOX|1365.45|d the ideas nag 6460|peach rose spring snow moccasin|Manufacturer#4|Brand#44|PROMO ANODIZED STEEL|36|MED DRUM|1366.46| reque 6461|black navy khaki sandy ivory|Manufacturer#2|Brand#21|MEDIUM BURNISHED BRASS|33|JUMBO PKG|1367.46|ular accounts hag 6462|papaya lemon lime misty light|Manufacturer#4|Brand#45|MEDIUM BRUSHED NICKEL|16|JUMBO DRUM|1368.46|ns sleep 6463|linen rose blue white honeydew|Manufacturer#3|Brand#31|PROMO BRUSHED BRASS|11|LG BOX|1369.46| sometime 6464|blanched spring chiffon lace indian|Manufacturer#3|Brand#31|PROMO PLATED BRASS|20|JUMBO PKG|1370.46|thely ideas. 6465|metallic cornsilk purple light firebrick|Manufacturer#3|Brand#35|SMALL ANODIZED NICKEL|41|LG JAR|1371.46|he busily regu 6466|thistle azure orange honeydew ivory|Manufacturer#5|Brand#54|LARGE ANODIZED NICKEL|33|MED DRUM|1372.46|lithely 6467|forest sandy azure thistle pale|Manufacturer#4|Brand#41|MEDIUM BURNISHED TIN|3|SM BOX|1373.46|lites. caref 6468|grey honeydew cornsilk light firebrick|Manufacturer#1|Brand#15|SMALL ANODIZED BRASS|1|JUMBO CASE|1374.46| regular a 6469|turquoise navajo indian olive forest|Manufacturer#1|Brand#14|MEDIUM ANODIZED STEEL|8|WRAP JAR|1375.46|ged req 6470|cornflower khaki powder blue black|Manufacturer#4|Brand#43|SMALL PLATED COPPER|45|WRAP CAN|1376.47|ts int 6471|cyan drab almond red burnished|Manufacturer#4|Brand#41|ECONOMY ANODIZED COPPER|40|WRAP PACK|1377.47|g pinto beans dete 6472|cornflower metallic cream bisque plum|Manufacturer#2|Brand#23|PROMO POLISHED STEEL|36|SM PACK|1378.47|al courts wake fur 6473|royal linen antique cornsilk cyan|Manufacturer#3|Brand#31|ECONOMY ANODIZED COPPER|35|LG CASE|1379.47| packages wake s 6474|navy sandy khaki gainsboro tan|Manufacturer#5|Brand#51|MEDIUM ANODIZED BRASS|25|SM BAG|1380.47|nding attainments. 6475|peru peach navajo white lemon|Manufacturer#4|Brand#41|LARGE BURNISHED COPPER|39|LG BAG|1381.47|special reque 6476|peru thistle moccasin olive smoke|Manufacturer#4|Brand#42|MEDIUM POLISHED NICKEL|42|JUMBO JAR|1382.47|uriously 6477|antique black seashell deep burnished|Manufacturer#3|Brand#34|SMALL BURNISHED NICKEL|6|MED BAG|1383.47| careful, ironic 6478|bisque royal seashell floral coral|Manufacturer#1|Brand#12|ECONOMY ANODIZED TIN|29|MED DRUM|1384.47| deposits. pending, do 6479|rose maroon powder orange snow|Manufacturer#3|Brand#33|MEDIUM BURNISHED NICKEL|22|WRAP JAR|1385.47| the final, regul 6480|floral spring beige black cornsilk|Manufacturer#5|Brand#54|STANDARD ANODIZED TIN|3|WRAP BAG|1386.48|re. quickly ir 6481|green khaki cornsilk misty lavender|Manufacturer#4|Brand#42|SMALL BRUSHED BRASS|26|LG JAR|1387.48|ages sleep flu 6482|indian sky orchid orange chocolate|Manufacturer#4|Brand#44|MEDIUM POLISHED NICKEL|24|LG CASE|1388.48| ironic asymptotes wak 6483|almond blush navy mint ghost|Manufacturer#3|Brand#35|PROMO ANODIZED STEEL|5|JUMBO CAN|1389.48|ccounts nag 6484|smoke drab peach pale misty|Manufacturer#2|Brand#25|PROMO PLATED COPPER|1|WRAP BAG|1390.48|w slyly pendi 6485|cream saddle slate floral frosted|Manufacturer#2|Brand#24|MEDIUM POLISHED TIN|26|MED BAG|1391.48|. quickl 6486|frosted indian drab seashell ghost|Manufacturer#2|Brand#23|PROMO BRUSHED TIN|22|WRAP PACK|1392.48|ithely regular dep 6487|slate dodger linen drab green|Manufacturer#1|Brand#11|ECONOMY ANODIZED TIN|46|SM BOX|1393.48|e even 6488|puff royal spring firebrick honeydew|Manufacturer#4|Brand#44|ECONOMY BURNISHED BRASS|4|LG BOX|1394.48|atelet 6489|ivory dodger floral antique plum|Manufacturer#5|Brand#53|STANDARD ANODIZED BRASS|28|LG CASE|1395.48|hes. ironically unusua 6490|cream gainsboro slate puff spring|Manufacturer#3|Brand#34|SMALL PLATED COPPER|36|MED CASE|1396.49|packages. furiously fi 6491|blue goldenrod rose lawn lavender|Manufacturer#2|Brand#21|SMALL ANODIZED STEEL|13|SM DRUM|1397.49|ests boost sly 6492|navy burnished deep cornflower sandy|Manufacturer#5|Brand#55|SMALL POLISHED STEEL|47|WRAP JAR|1398.49|s play carefully 6493|antique turquoise snow gainsboro slate|Manufacturer#5|Brand#52|SMALL PLATED COPPER|36|MED PACK|1399.49|s. silent, even d 6494|brown cyan light peach ivory|Manufacturer#1|Brand#12|LARGE BRUSHED BRASS|11|LG BOX|1400.49|packages. bl 6495|bisque dark orange lawn frosted|Manufacturer#4|Brand#42|LARGE BURNISHED NICKEL|27|SM PACK|1401.49|posits around the 6496|sienna blue goldenrod bisque magenta|Manufacturer#3|Brand#35|MEDIUM BRUSHED BRASS|2|JUMBO CAN|1402.49|e quick 6497|lace plum black deep almond|Manufacturer#4|Brand#45|PROMO BURNISHED TIN|46|WRAP CAN|1403.49|ly above the 6498|grey tomato rosy blanched brown|Manufacturer#1|Brand#14|SMALL POLISHED TIN|26|WRAP PACK|1404.49| packages 6499|cornflower magenta dodger metallic spring|Manufacturer#2|Brand#23|STANDARD POLISHED BRASS|46|WRAP PKG|1405.49|sts are s 6500|antique deep almond black aquamarine|Manufacturer#2|Brand#23|ECONOMY ANODIZED STEEL|1|WRAP DRUM|1406.50|detect slyly 6501|medium snow floral yellow sky|Manufacturer#2|Brand#22|SMALL ANODIZED TIN|46|JUMBO BOX|1407.50|sleep quickly e 6502|cornflower saddle lemon spring peru|Manufacturer#2|Brand#23|LARGE POLISHED COPPER|6|LG BOX|1408.50|ending, brave 6503|royal almond papaya gainsboro orchid|Manufacturer#4|Brand#42|SMALL BRUSHED NICKEL|1|MED CASE|1409.50| deposits cajole furio 6504|orange salmon dark wheat purple|Manufacturer#3|Brand#34|PROMO BRUSHED TIN|48|SM PACK|1410.50|gle slyly ironic, sile 6505|lavender metallic burlywood almond khaki|Manufacturer#4|Brand#42|ECONOMY ANODIZED STEEL|4|LG JAR|1411.50| special i 6506|slate green navy burlywood almond|Manufacturer#5|Brand#52|MEDIUM PLATED COPPER|44|LG CAN|1412.50|de of the express 6507|thistle lemon orchid lace beige|Manufacturer#1|Brand#14|STANDARD PLATED COPPER|47|WRAP CAN|1413.50|uctions haggl 6508|beige maroon almond misty cornflower|Manufacturer#4|Brand#44|SMALL BURNISHED STEEL|10|SM JAR|1414.50|even accou 6509|sienna misty thistle medium powder|Manufacturer#3|Brand#32|MEDIUM ANODIZED STEEL|27|LG PKG|1415.50|ackages for t 6510|indian aquamarine hot yellow drab|Manufacturer#5|Brand#52|MEDIUM BURNISHED BRASS|29|LG JAR|1416.51|sts across the 6511|burnished hot navy steel peru|Manufacturer#3|Brand#33|STANDARD BURNISHED TIN|39|MED JAR|1417.51|gular excuses. fluf 6512|light coral wheat powder rosy|Manufacturer#2|Brand#25|STANDARD PLATED BRASS|3|JUMBO CASE|1418.51|efully expres 6513|moccasin cornsilk misty frosted honeydew|Manufacturer#5|Brand#53|ECONOMY BURNISHED STEEL|21|LG BAG|1419.51| ironic dolphins i 6514|violet spring powder saddle burnished|Manufacturer#5|Brand#51|SMALL PLATED BRASS|28|JUMBO PACK|1420.51|al excuses ca 6515|sienna khaki brown linen purple|Manufacturer#2|Brand#21|SMALL BURNISHED BRASS|2|MED CAN|1421.51|accoun 6516|azure sandy ghost orange antique|Manufacturer#3|Brand#35|ECONOMY BURNISHED TIN|9|LG CASE|1422.51|ar pa 6517|metallic grey plum olive cyan|Manufacturer#4|Brand#41|SMALL BURNISHED COPPER|21|MED PACK|1423.51|nstructi 6518|honeydew goldenrod floral white yellow|Manufacturer#3|Brand#31|SMALL BRUSHED NICKEL|16|WRAP CASE|1424.51|according t 6519|indian seashell cream azure coral|Manufacturer#4|Brand#45|STANDARD ANODIZED NICKEL|17|LG PACK|1425.51|ckly regular 6520|green honeydew cyan yellow peru|Manufacturer#2|Brand#22|PROMO PLATED STEEL|4|LG DRUM|1426.52|y steal 6521|misty powder black linen forest|Manufacturer#2|Brand#24|PROMO BURNISHED NICKEL|42|SM CASE|1427.52|osits after the 6522|lavender almond spring navy blanched|Manufacturer#2|Brand#24|ECONOMY BURNISHED TIN|16|SM PACK|1428.52|s dazzle: careful 6523|hot maroon lawn moccasin ivory|Manufacturer#1|Brand#11|MEDIUM ANODIZED STEEL|43|MED BOX|1429.52|out the quickly 6524|powder blue linen brown peru|Manufacturer#1|Brand#15|PROMO BURNISHED TIN|34|MED CASE|1430.52|unts detect quick 6525|ghost dim antique honeydew dodger|Manufacturer#1|Brand#11|PROMO POLISHED COPPER|11|JUMBO BOX|1431.52|rding t 6526|forest black azure ghost beige|Manufacturer#4|Brand#41|ECONOMY PLATED COPPER|26|MED PKG|1432.52|accounts wake careful 6527|lace coral royal deep sienna|Manufacturer#2|Brand#21|ECONOMY PLATED STEEL|33|SM BOX|1433.52|theodolite 6528|midnight orange floral dim cyan|Manufacturer#5|Brand#51|PROMO ANODIZED COPPER|34|SM DRUM|1434.52|decoys 6529|sienna turquoise olive dark tomato|Manufacturer#1|Brand#15|SMALL BRUSHED STEEL|7|SM BOX|1435.52|ld accou 6530|lawn maroon azure rosy blue|Manufacturer#3|Brand#33|ECONOMY ANODIZED COPPER|33|JUMBO CASE|1436.53|ual packages 6531|floral metallic medium honeydew firebrick|Manufacturer#2|Brand#22|LARGE BURNISHED COPPER|28|WRAP CASE|1437.53|ths after the 6532|navajo burnished misty blue sandy|Manufacturer#5|Brand#54|SMALL BRUSHED COPPER|20|SM DRUM|1438.53| theodolites detect 6533|misty slate cream forest olive|Manufacturer#3|Brand#35|SMALL ANODIZED NICKEL|41|WRAP PKG|1439.53|symptotes 6534|blue metallic midnight forest powder|Manufacturer#3|Brand#32|STANDARD BRUSHED STEEL|11|WRAP JAR|1440.53|reful 6535|linen thistle sandy brown chocolate|Manufacturer#5|Brand#53|SMALL POLISHED COPPER|8|JUMBO JAR|1441.53|ular packa 6536|rosy plum violet peru lemon|Manufacturer#2|Brand#22|ECONOMY BRUSHED NICKEL|31|WRAP BAG|1442.53|aggle. reg 6537|olive cornsilk blanched seashell magenta|Manufacturer#5|Brand#53|PROMO POLISHED NICKEL|20|MED BOX|1443.53|furiously unu 6538|powder turquoise bisque grey steel|Manufacturer#5|Brand#51|PROMO ANODIZED NICKEL|10|JUMBO BAG|1444.53|hins wake ac 6539|black olive metallic azure spring|Manufacturer#1|Brand#13|SMALL POLISHED COPPER|29|MED JAR|1445.53|symptotes? quickly 6540|papaya salmon sienna ivory lime|Manufacturer#3|Brand#32|MEDIUM ANODIZED COPPER|5|JUMBO PACK|1446.54|beans wake re 6541|metallic thistle plum frosted light|Manufacturer#2|Brand#23|SMALL POLISHED BRASS|49|JUMBO DRUM|1447.54|its haggle along 6542|violet peach sienna blue rosy|Manufacturer#3|Brand#34|MEDIUM BRUSHED BRASS|38|SM BAG|1448.54|asymptote 6543|sky red goldenrod bisque tomato|Manufacturer#1|Brand#14|STANDARD POLISHED COPPER|28|JUMBO PACK|1449.54| packages haggle car 6544|firebrick aquamarine black midnight red|Manufacturer#2|Brand#25|STANDARD BRUSHED TIN|13|SM PKG|1450.54|e slyl 6545|powder blue orchid slate lemon|Manufacturer#4|Brand#45|SMALL PLATED COPPER|36|LG PACK|1451.54|ording to the slyly 6546|magenta sandy pale lavender steel|Manufacturer#2|Brand#25|LARGE BRUSHED BRASS|41|MED PKG|1452.54|ic pinto beans use 6547|forest khaki firebrick brown light|Manufacturer#2|Brand#23|LARGE PLATED COPPER|33|WRAP CASE|1453.54|ructions 6548|green slate brown orange sandy|Manufacturer#5|Brand#54|PROMO BURNISHED STEEL|7|WRAP BAG|1454.54|atele 6549|gainsboro sandy slate beige burlywood|Manufacturer#3|Brand#33|SMALL BRUSHED NICKEL|45|LG CAN|1455.54|ckages. 6550|cyan cornflower magenta cornsilk lawn|Manufacturer#2|Brand#24|LARGE POLISHED STEEL|23|WRAP PACK|1456.55|requests 6551|firebrick deep smoke mint peach|Manufacturer#5|Brand#51|PROMO ANODIZED BRASS|35|JUMBO BOX|1457.55|s. carefully 6552|cream lemon chocolate sienna black|Manufacturer#5|Brand#51|LARGE ANODIZED NICKEL|25|LG BOX|1458.55|ronic 6553|white green misty wheat indian|Manufacturer#2|Brand#24|SMALL POLISHED COPPER|33|JUMBO CASE|1459.55|ending excu 6554|drab maroon deep cyan floral|Manufacturer#1|Brand#13|MEDIUM POLISHED NICKEL|42|JUMBO PKG|1460.55|pending accounts 6555|brown thistle frosted sienna moccasin|Manufacturer#2|Brand#25|LARGE ANODIZED STEEL|23|WRAP BAG|1461.55|are furiou 6556|forest misty seashell navy ghost|Manufacturer#3|Brand#32|MEDIUM BRUSHED NICKEL|42|MED CAN|1462.55|nusual ideas. sp 6557|lawn blue navy rose navajo|Manufacturer#1|Brand#15|STANDARD BURNISHED NICKEL|28|SM BOX|1463.55|pendin 6558|floral cornflower burlywood rose firebrick|Manufacturer#5|Brand#52|SMALL ANODIZED BRASS|3|LG DRUM|1464.55|d ideas are 6559|honeydew steel seashell peach cornflower|Manufacturer#3|Brand#31|PROMO POLISHED TIN|8|JUMBO DRUM|1465.55|pades boost blithel 6560|salmon dim moccasin frosted indian|Manufacturer#3|Brand#34|STANDARD BRUSHED BRASS|7|SM BAG|1466.56|d the daringly regula 6561|dodger bisque blush peach lawn|Manufacturer#3|Brand#35|MEDIUM PLATED BRASS|42|LG BOX|1467.56|r the blithely spec 6562|spring hot blanched chocolate orange|Manufacturer#5|Brand#52|ECONOMY ANODIZED STEEL|3|LG PACK|1468.56|g accounts slee 6563|peach royal lemon frosted tomato|Manufacturer#5|Brand#55|ECONOMY POLISHED COPPER|43|LG PACK|1469.56|yly unusual patt 6564|slate brown floral firebrick grey|Manufacturer#4|Brand#44|PROMO BURNISHED TIN|44|WRAP JAR|1470.56|pendencies boost blith 6565|cyan lace chiffon papaya coral|Manufacturer#5|Brand#53|MEDIUM POLISHED STEEL|32|MED DRUM|1471.56|ing to the s 6566|misty salmon frosted antique honeydew|Manufacturer#4|Brand#42|PROMO ANODIZED BRASS|38|LG CASE|1472.56| furiously fur 6567|snow magenta dark saddle sky|Manufacturer#1|Brand#13|STANDARD POLISHED NICKEL|47|WRAP BAG|1473.56|express deposits 6568|dark lime sandy burnished blanched|Manufacturer#2|Brand#23|MEDIUM POLISHED BRASS|50|MED JAR|1474.56|t, pendi 6569|ivory snow red olive lawn|Manufacturer#4|Brand#44|SMALL BURNISHED COPPER|14|JUMBO JAR|1475.56|g the brave foxes dazz 6570|medium lawn puff magenta saddle|Manufacturer#2|Brand#25|SMALL BURNISHED NICKEL|43|WRAP PACK|1476.57|kages. r 6571|lace saddle lavender khaki pink|Manufacturer#5|Brand#51|SMALL PLATED BRASS|34|JUMBO CASE|1477.57|en dolphi 6572|green salmon lace powder lemon|Manufacturer#3|Brand#33|PROMO ANODIZED NICKEL|7|JUMBO PKG|1478.57|quickly final foxe 6573|cyan smoke firebrick papaya brown|Manufacturer#1|Brand#13|LARGE ANODIZED BRASS|32|JUMBO PACK|1479.57|ges affix. 6574|lavender mint olive honeydew lemon|Manufacturer#3|Brand#33|LARGE BURNISHED COPPER|45|MED PACK|1480.57|c pinto beans haggle c 6575|aquamarine red lime pale sandy|Manufacturer#3|Brand#34|STANDARD ANODIZED NICKEL|41|SM BAG|1481.57|into beans 6576|lawn black cyan tomato cornsilk|Manufacturer#5|Brand#51|LARGE BRUSHED NICKEL|9|LG CAN|1482.57|. carefully silent pi 6577|firebrick royal purple khaki burnished|Manufacturer#2|Brand#22|ECONOMY BURNISHED COPPER|41|WRAP CASE|1483.57|oost quickly acro 6578|yellow cyan moccasin aquamarine cornflower|Manufacturer#3|Brand#35|PROMO ANODIZED BRASS|46|SM CASE|1484.57| requests wake blithel 6579|rosy chartreuse green red navy|Manufacturer#4|Brand#45|STANDARD BRUSHED STEEL|17|SM CAN|1485.57|special accounts wil 6580|dark thistle coral cornsilk red|Manufacturer#5|Brand#52|PROMO BRUSHED STEEL|43|JUMBO BOX|1486.58|s caj 6581|khaki purple lawn royal slate|Manufacturer#3|Brand#33|LARGE BRUSHED COPPER|20|MED PKG|1487.58|ily qui 6582|rosy frosted puff drab dim|Manufacturer#1|Brand#15|LARGE BURNISHED BRASS|19|JUMBO CASE|1488.58|y bold ideas. bold 6583|magenta blue burnished azure midnight|Manufacturer#5|Brand#52|SMALL BRUSHED BRASS|10|SM CAN|1489.58|y unusual deposit 6584|peach hot bisque white snow|Manufacturer#4|Brand#42|STANDARD BRUSHED BRASS|44|LG BAG|1490.58|ent th 6585|tomato snow cornflower black mint|Manufacturer#4|Brand#43|SMALL BRUSHED NICKEL|39|JUMBO BOX|1491.58|ly regular deposit 6586|navy yellow frosted tomato wheat|Manufacturer#3|Brand#32|MEDIUM POLISHED COPPER|46|WRAP DRUM|1492.58| nod stealthily 6587|tan cornflower plum dark peru|Manufacturer#1|Brand#14|SMALL PLATED TIN|30|WRAP PKG|1493.58|e slyly 6588|floral tomato white medium burlywood|Manufacturer#3|Brand#34|LARGE BURNISHED BRASS|31|JUMBO BAG|1494.58|refully pending pin 6589|olive white spring misty cream|Manufacturer#1|Brand#11|ECONOMY PLATED NICKEL|30|LG PACK|1495.58| even pin 6590|firebrick chartreuse goldenrod honeydew steel|Manufacturer#1|Brand#13|SMALL BRUSHED COPPER|28|MED DRUM|1496.59|es sleep about th 6591|gainsboro khaki chocolate rosy navajo|Manufacturer#1|Brand#11|ECONOMY PLATED NICKEL|29|SM BAG|1497.59|x daringly bol 6592|puff olive drab grey red|Manufacturer#4|Brand#44|LARGE POLISHED NICKEL|4|MED BAG|1498.59| acco 6593|blanched lime lace magenta rosy|Manufacturer#1|Brand#14|ECONOMY POLISHED COPPER|41|MED PKG|1499.59| slyly regular the 6594|black khaki navajo purple turquoise|Manufacturer#2|Brand#21|MEDIUM BURNISHED TIN|20|WRAP PACK|1500.59|pendenci 6595|bisque lemon firebrick gainsboro pale|Manufacturer#3|Brand#31|SMALL PLATED NICKEL|36|MED CASE|1501.59|ar theod 6596|mint misty coral turquoise cornsilk|Manufacturer#2|Brand#24|LARGE POLISHED TIN|15|SM DRUM|1502.59|y unusual deposits ar 6597|metallic tan khaki beige salmon|Manufacturer#4|Brand#44|LARGE POLISHED TIN|11|WRAP CAN|1503.59|xes against the care 6598|cream brown goldenrod lawn burnished|Manufacturer#3|Brand#32|LARGE PLATED COPPER|33|SM PACK|1504.59|odolites 6599|plum midnight turquoise tan violet|Manufacturer#5|Brand#54|LARGE ANODIZED NICKEL|49|MED BOX|1505.59|fluffily against the f 6600|aquamarine blanched drab floral burnished|Manufacturer#3|Brand#31|PROMO PLATED STEEL|17|JUMBO CASE|1506.60| are around 6601|cyan orange metallic blush navajo|Manufacturer#2|Brand#21|LARGE PLATED NICKEL|8|MED CASE|1507.60|ingly brave dol 6602|blush violet azure black tan|Manufacturer#5|Brand#53|SMALL BURNISHED COPPER|45|JUMBO CASE|1508.60|ly silent deposits. a 6603|wheat blanched grey slate brown|Manufacturer#4|Brand#42|PROMO BRUSHED NICKEL|33|MED JAR|1509.60|posits are furiously 6604|powder smoke rose maroon cornsilk|Manufacturer#4|Brand#43|STANDARD BURNISHED NICKEL|48|SM CASE|1510.60|ckly bold ideas. 6605|peru pale spring orange papaya|Manufacturer#3|Brand#32|MEDIUM ANODIZED STEEL|10|SM PKG|1511.60|lyly. furiously 6606|rosy grey midnight wheat lavender|Manufacturer#5|Brand#51|SMALL ANODIZED STEEL|7|SM DRUM|1512.60|stealthily even ideas 6607|khaki goldenrod floral misty navy|Manufacturer#1|Brand#14|LARGE BRUSHED BRASS|39|SM CAN|1513.60|d dependencies. 6608|lemon ivory cornsilk navy seashell|Manufacturer#2|Brand#21|SMALL PLATED TIN|26|WRAP DRUM|1514.60|lyly among the fu 6609|cornflower medium grey royal slate|Manufacturer#3|Brand#33|SMALL ANODIZED BRASS|41|WRAP BOX|1515.60|cross the fluff 6610|moccasin lavender khaki dim white|Manufacturer#3|Brand#31|ECONOMY BRUSHED COPPER|46|SM JAR|1516.61|are furio 6611|chocolate puff rose indian purple|Manufacturer#4|Brand#42|PROMO ANODIZED BRASS|32|MED BOX|1517.61|fix slyly 6612|blue hot wheat misty gainsboro|Manufacturer#1|Brand#14|MEDIUM PLATED TIN|44|JUMBO JAR|1518.61|dinos. blithely 6613|rose slate antique maroon beige|Manufacturer#3|Brand#31|SMALL BURNISHED COPPER|7|WRAP PACK|1519.61|slyly 6614|misty peach blue cornflower indian|Manufacturer#5|Brand#53|MEDIUM BURNISHED TIN|46|JUMBO DRUM|1520.61|ly bold packages cajo 6615|rosy drab sienna misty royal|Manufacturer#5|Brand#54|SMALL ANODIZED BRASS|16|MED PACK|1521.61|ar requests. blithe 6616|azure coral sienna midnight lavender|Manufacturer#5|Brand#51|ECONOMY ANODIZED STEEL|9|LG PKG|1522.61| the q 6617|dodger black tomato powder peru|Manufacturer#3|Brand#33|ECONOMY BURNISHED COPPER|3|WRAP PACK|1523.61|ithely a 6618|floral spring antique moccasin peru|Manufacturer#4|Brand#43|PROMO BRUSHED BRASS|25|SM CASE|1524.61|r the unusual, 6619|violet frosted medium ivory cornsilk|Manufacturer#2|Brand#22|LARGE ANODIZED COPPER|21|JUMBO BOX|1525.61|s hang bli 6620|puff khaki peru midnight chocolate|Manufacturer#5|Brand#53|STANDARD ANODIZED BRASS|34|MED PACK|1526.62|ounts. pen 6621|steel deep brown cornflower burnished|Manufacturer#3|Brand#31|SMALL BRUSHED COPPER|7|JUMBO CAN|1527.62|tructions bre 6622|rose burlywood blue deep salmon|Manufacturer#2|Brand#22|ECONOMY POLISHED COPPER|16|LG PACK|1528.62| foxes are b 6623|salmon lavender blue medium papaya|Manufacturer#4|Brand#41|ECONOMY POLISHED NICKEL|5|WRAP DRUM|1529.62|lyly above the 6624|tan blanched cream dodger grey|Manufacturer#2|Brand#23|MEDIUM BRUSHED TIN|26|SM CASE|1530.62|s above the slyly cl 6625|chartreuse ghost dim frosted sandy|Manufacturer#2|Brand#23|MEDIUM BRUSHED BRASS|18|MED CASE|1531.62| at the slyly iro 6626|chartreuse lime hot saddle rose|Manufacturer#3|Brand#31|MEDIUM BURNISHED COPPER|17|LG DRUM|1532.62|ctions 6627|medium peru chartreuse orange royal|Manufacturer#4|Brand#42|STANDARD POLISHED NICKEL|32|JUMBO PACK|1533.62|ronic, even dep 6628|lemon sienna medium pink saddle|Manufacturer#5|Brand#55|PROMO PLATED TIN|3|SM CASE|1534.62| across the silent, 6629|deep bisque lavender snow ivory|Manufacturer#4|Brand#43|STANDARD PLATED TIN|29|JUMBO JAR|1535.62|ggle blithely expres 6630|spring metallic chocolate deep blush|Manufacturer#1|Brand#14|STANDARD ANODIZED BRASS|47|MED BAG|1536.63|t slyly al 6631|drab moccasin lace white ghost|Manufacturer#3|Brand#33|SMALL PLATED BRASS|28|WRAP CASE|1537.63|es nag. multi 6632|sandy orchid misty navajo antique|Manufacturer#1|Brand#15|ECONOMY PLATED COPPER|35|JUMBO CAN|1538.63|eas cajole bl 6633|orchid thistle sky linen khaki|Manufacturer#2|Brand#21|ECONOMY ANODIZED STEEL|27|WRAP JAR|1539.63|ounts thrash 6634|lace olive indian orange dim|Manufacturer#4|Brand#41|STANDARD BURNISHED STEEL|34|SM CAN|1540.63| above the 6635|frosted midnight khaki tomato sandy|Manufacturer#4|Brand#43|PROMO PLATED NICKEL|38|SM CAN|1541.63|side 6636|red linen dodger floral hot|Manufacturer#5|Brand#52|STANDARD POLISHED NICKEL|45|MED BOX|1542.63|nstruct 6637|chiffon bisque frosted magenta red|Manufacturer#1|Brand#13|LARGE PLATED BRASS|41|MED CAN|1543.63|. blithely 6638|sandy antique cornflower dim honeydew|Manufacturer#1|Brand#13|PROMO BRUSHED NICKEL|50|LG BAG|1544.63|uickly final 6639|turquoise midnight smoke maroon chiffon|Manufacturer#1|Brand#11|MEDIUM POLISHED STEEL|34|LG CASE|1545.63|y deposits 6640|pink sandy red mint black|Manufacturer#5|Brand#54|ECONOMY POLISHED TIN|3|JUMBO PKG|1546.64|ld depos 6641|cyan indian grey drab chartreuse|Manufacturer#2|Brand#23|ECONOMY ANODIZED COPPER|11|SM BOX|1547.64|ts. quickly 6642|gainsboro navajo cream medium maroon|Manufacturer#3|Brand#33|ECONOMY BURNISHED NICKEL|34|JUMBO BOX|1548.64|al accounts. carefull 6643|white lemon blanched deep yellow|Manufacturer#3|Brand#33|PROMO ANODIZED TIN|44|SM BOX|1549.64|the ironic ins 6644|khaki sandy moccasin goldenrod almond|Manufacturer#2|Brand#24|SMALL BRUSHED COPPER|25|MED CAN|1550.64|ts nag blithely above 6645|blush antique lemon plum dim|Manufacturer#4|Brand#41|STANDARD POLISHED STEEL|43|LG JAR|1551.64|pendencies affix. regu 6646|cornflower green slate spring medium|Manufacturer#5|Brand#55|STANDARD BRUSHED STEEL|2|WRAP BAG|1552.64|equests. de 6647|turquoise forest misty dodger aquamarine|Manufacturer#1|Brand#13|ECONOMY PLATED TIN|3|MED PKG|1553.64|ptotes. slyly ir 6648|forest beige chiffon hot peach|Manufacturer#1|Brand#14|STANDARD PLATED NICKEL|2|SM DRUM|1554.64|gside of the 6649|rosy saddle beige almond light|Manufacturer#2|Brand#22|ECONOMY ANODIZED NICKEL|18|WRAP BOX|1555.64|t. regular ideas are: 6650|almond white misty goldenrod blanched|Manufacturer#3|Brand#32|MEDIUM ANODIZED COPPER|27|SM PACK|1556.65| after the warhors 6651|burlywood sandy olive orange smoke|Manufacturer#2|Brand#23|ECONOMY ANODIZED NICKEL|20|WRAP PKG|1557.65| frays. 6652|red puff green sandy smoke|Manufacturer#1|Brand#12|ECONOMY BRUSHED TIN|33|SM JAR|1558.65|. blithely bold ac 6653|sienna papaya cornflower coral peru|Manufacturer#3|Brand#32|ECONOMY POLISHED NICKEL|10|LG PACK|1559.65|bout the unusual, i 6654|gainsboro dim goldenrod maroon light|Manufacturer#5|Brand#52|STANDARD BURNISHED COPPER|40|MED BAG|1560.65|en decoy 6655|burlywood snow frosted dark drab|Manufacturer#4|Brand#42|SMALL BURNISHED BRASS|36|WRAP PACK|1561.65|? blithel 6656|chiffon indian lace purple cream|Manufacturer#2|Brand#22|LARGE PLATED BRASS|31|MED PACK|1562.65|al excuses 6657|tomato lavender blush drab lawn|Manufacturer#2|Brand#24|SMALL POLISHED NICKEL|19|LG CASE|1563.65|uests. bravely unusua 6658|thistle goldenrod yellow beige black|Manufacturer#4|Brand#43|ECONOMY PLATED BRASS|20|MED BAG|1564.65|nding pl 6659|yellow pink pale rose honeydew|Manufacturer#2|Brand#21|LARGE BRUSHED NICKEL|22|MED JAR|1565.65|xpress asympto 6660|lime blanched rose salmon forest|Manufacturer#2|Brand#23|STANDARD PLATED BRASS|39|SM PKG|1566.66|p. qu 6661|beige coral ghost papaya antique|Manufacturer#2|Brand#23|PROMO BURNISHED STEEL|2|MED CASE|1567.66|ly special instr 6662|almond salmon hot sandy forest|Manufacturer#4|Brand#43|SMALL PLATED TIN|7|SM PKG|1568.66|r requests. 6663|spring seashell violet aquamarine dodger|Manufacturer#2|Brand#25|LARGE ANODIZED COPPER|32|LG JAR|1569.66|c ideas. 6664|lace peach snow pale spring|Manufacturer#3|Brand#35|LARGE POLISHED NICKEL|21|LG CAN|1570.66| quickly pending dol 6665|goldenrod chocolate peru midnight lemon|Manufacturer#5|Brand#55|LARGE BURNISHED BRASS|22|WRAP JAR|1571.66| furiously to 6666|azure blue smoke metallic powder|Manufacturer#1|Brand#15|LARGE PLATED STEEL|42|SM CAN|1572.66|y deposits. even, regu 6667|coral beige turquoise chiffon burnished|Manufacturer#1|Brand#14|MEDIUM POLISHED STEEL|13|MED PKG|1573.66|ents along the quic 6668|turquoise burnished honeydew aquamarine ghost|Manufacturer#5|Brand#55|STANDARD PLATED BRASS|23|WRAP BOX|1574.66|ong the furiously 6669|rose forest lace chocolate peru|Manufacturer#1|Brand#12|STANDARD PLATED BRASS|40|LG CAN|1575.66|ckly regular do 6670|snow wheat steel mint pink|Manufacturer#4|Brand#43|PROMO PLATED TIN|3|WRAP JAR|1576.67|iously regul 6671|lemon linen wheat frosted honeydew|Manufacturer#5|Brand#52|SMALL BURNISHED TIN|35|JUMBO PACK|1577.67|ncies? care 6672|slate dodger pink blanched thistle|Manufacturer#3|Brand#31|MEDIUM BURNISHED STEEL|32|LG CASE|1578.67| pending accou 6673|lavender forest grey slate chiffon|Manufacturer#3|Brand#31|LARGE PLATED BRASS|20|LG PACK|1579.67|ic dependenc 6674|lace frosted rosy chiffon blanched|Manufacturer#2|Brand#21|SMALL ANODIZED NICKEL|47|JUMBO JAR|1580.67|refully even deco 6675|burlywood purple deep goldenrod pink|Manufacturer#5|Brand#53|SMALL POLISHED COPPER|39|WRAP JAR|1581.67| furious foxe 6676|saddle tan sienna burlywood green|Manufacturer#5|Brand#53|SMALL BURNISHED NICKEL|41|WRAP CAN|1582.67|e final asymptotes. s 6677|chiffon royal lace yellow drab|Manufacturer#3|Brand#31|MEDIUM ANODIZED BRASS|31|WRAP CASE|1583.67|riously. unusual depen 6678|spring peru burnished moccasin khaki|Manufacturer#3|Brand#35|PROMO PLATED TIN|11|SM JAR|1584.67|ages ha 6679|lace moccasin puff violet plum|Manufacturer#2|Brand#21|SMALL PLATED TIN|37|LG BOX|1585.67| cour 6680|dim lace cream honeydew wheat|Manufacturer#1|Brand#14|STANDARD POLISHED BRASS|24|SM BAG|1586.68|n dolphins. slyly reg 6681|steel gainsboro burlywood cream hot|Manufacturer#2|Brand#23|STANDARD ANODIZED STEEL|7|LG PACK|1587.68|en excuses. sl 6682|slate sandy mint chiffon peru|Manufacturer#3|Brand#33|MEDIUM BRUSHED COPPER|30|MED CAN|1588.68|e. even packa 6683|cream sandy dodger dim cornflower|Manufacturer#1|Brand#13|SMALL POLISHED STEEL|49|LG BAG|1589.68| pending, silent pac 6684|green firebrick blanched goldenrod grey|Manufacturer#1|Brand#12|MEDIUM BURNISHED STEEL|2|MED CAN|1590.68|y about the c 6685|royal sienna mint green khaki|Manufacturer#4|Brand#45|ECONOMY ANODIZED BRASS|49|MED PKG|1591.68|detect 6686|frosted cornsilk misty light lime|Manufacturer#5|Brand#53|MEDIUM BURNISHED NICKEL|6|WRAP PACK|1592.68| the requests wake sl 6687|bisque pink navy ivory orange|Manufacturer#4|Brand#42|ECONOMY BURNISHED COPPER|8|SM CASE|1593.68|ully silent pa 6688|sienna olive lemon dark blue|Manufacturer#5|Brand#53|STANDARD BRUSHED NICKEL|34|JUMBO PACK|1594.68|ully regular theo 6689|maroon gainsboro cream turquoise cyan|Manufacturer#5|Brand#53|MEDIUM PLATED BRASS|23|WRAP PKG|1595.68| blithely special r 6690|azure wheat lawn midnight spring|Manufacturer#4|Brand#44|SMALL BRUSHED TIN|20|JUMBO CASE|1596.69|deposits. closely ev 6691|seashell chiffon peru chocolate antique|Manufacturer#2|Brand#24|MEDIUM PLATED COPPER|7|JUMBO BAG|1597.69| depo 6692|white deep cornflower purple ivory|Manufacturer#3|Brand#32|ECONOMY PLATED COPPER|24|MED CAN|1598.69|iously ironic court 6693|moccasin cornflower lemon goldenrod light|Manufacturer#4|Brand#44|MEDIUM ANODIZED COPPER|26|JUMBO DRUM|1599.69|regular d 6694|misty linen white spring metallic|Manufacturer#4|Brand#45|LARGE BRUSHED BRASS|26|MED PACK|1600.69|gular requests. 6695|brown yellow mint burlywood medium|Manufacturer#4|Brand#45|STANDARD POLISHED STEEL|13|WRAP BOX|1601.69|riously un 6696|moccasin mint saddle gainsboro seashell|Manufacturer#2|Brand#24|SMALL PLATED STEEL|1|MED DRUM|1602.69|ithely stealt 6697|cream smoke orange brown lawn|Manufacturer#4|Brand#42|PROMO POLISHED STEEL|8|SM PKG|1603.69|ic deposits. slyly spe 6698|orange lace lawn pink papaya|Manufacturer#4|Brand#45|PROMO BURNISHED BRASS|2|SM PACK|1604.69|s. carefu 6699|almond antique metallic honeydew green|Manufacturer#4|Brand#43|MEDIUM BRUSHED TIN|30|LG BAG|1605.69|wake around 6700|frosted metallic khaki coral steel|Manufacturer#4|Brand#44|MEDIUM BURNISHED BRASS|16|SM JAR|1606.70| even theodolit 6701|antique pale honeydew spring maroon|Manufacturer#2|Brand#23|PROMO BRUSHED NICKEL|10|WRAP PKG|1607.70|nto beans doubt qu 6702|violet sienna gainsboro brown cornsilk|Manufacturer#4|Brand#44|SMALL POLISHED NICKEL|44|MED CAN|1608.70|hy reques 6703|light coral sky rose almond|Manufacturer#1|Brand#15|LARGE BRUSHED BRASS|37|JUMBO BOX|1609.70|ic pinto be 6704|lemon burnished firebrick hot royal|Manufacturer#5|Brand#51|SMALL POLISHED NICKEL|23|SM DRUM|1610.70|lyly ironic p 6705|drab sienna cream yellow navajo|Manufacturer#1|Brand#15|STANDARD BURNISHED NICKEL|11|LG BAG|1611.70|ing to the quickly fi 6706|black drab peru almond moccasin|Manufacturer#4|Brand#44|STANDARD ANODIZED STEEL|8|SM PKG|1612.70|fluffil 6707|burlywood snow dark peru saddle|Manufacturer#2|Brand#22|MEDIUM BRUSHED NICKEL|14|SM BOX|1613.70|nusual 6708|goldenrod linen orchid antique azure|Manufacturer#2|Brand#24|SMALL POLISHED BRASS|3|SM DRUM|1614.70|ic deposits 6709|lawn pale red steel rose|Manufacturer#4|Brand#45|LARGE BURNISHED STEEL|16|WRAP CAN|1615.70|iously 6710|plum brown peru cornflower red|Manufacturer#3|Brand#33|LARGE ANODIZED TIN|45|WRAP DRUM|1616.71| are blithely slyly i 6711|honeydew brown powder blue plum|Manufacturer#1|Brand#15|PROMO PLATED NICKEL|35|MED BOX|1617.71|ctions are furiously 6712|moccasin burlywood peru dodger smoke|Manufacturer#3|Brand#35|ECONOMY ANODIZED COPPER|41|WRAP JAR|1618.71|ests hag 6713|sandy midnight honeydew dodger orchid|Manufacturer#5|Brand#55|MEDIUM PLATED NICKEL|31|JUMBO BOX|1619.71|deposits was 6714|midnight lemon blue frosted misty|Manufacturer#3|Brand#31|LARGE BURNISHED COPPER|50|SM BAG|1620.71|detect 6715|ivory azure lavender cyan dim|Manufacturer#3|Brand#33|MEDIUM ANODIZED COPPER|13|WRAP BOX|1621.71|y final deposits a 6716|frosted almond azure hot dim|Manufacturer#3|Brand#31|LARGE ANODIZED STEEL|11|MED CAN|1622.71|ly. platelets are ca 6717|lace sienna plum gainsboro blanched|Manufacturer#1|Brand#15|LARGE BRUSHED STEEL|14|JUMBO DRUM|1623.71|usly ironic pinto 6718|lawn coral papaya steel chartreuse|Manufacturer#4|Brand#41|SMALL BRUSHED COPPER|5|LG CAN|1624.71|eep ironic war 6719|ghost black papaya ivory coral|Manufacturer#2|Brand#21|ECONOMY ANODIZED NICKEL|21|LG CAN|1625.71|kages. 6720|beige burlywood spring olive orange|Manufacturer#5|Brand#53|SMALL BRUSHED NICKEL|8|MED JAR|1626.72|refully regular p 6721|burlywood chocolate midnight puff pink|Manufacturer#2|Brand#23|PROMO POLISHED BRASS|20|LG PKG|1627.72|ely ironic courts. i 6722|ghost moccasin lemon bisque chartreuse|Manufacturer#2|Brand#23|MEDIUM PLATED TIN|10|LG CAN|1628.72|ly even deposits 6723|mint firebrick sienna yellow chocolate|Manufacturer#5|Brand#53|SMALL ANODIZED STEEL|27|SM CAN|1629.72|quickly 6724|seashell sandy pink lemon magenta|Manufacturer#3|Brand#33|PROMO ANODIZED COPPER|27|MED CAN|1630.72|y regular dep 6725|bisque khaki pink chartreuse goldenrod|Manufacturer#4|Brand#45|ECONOMY POLISHED BRASS|50|LG PACK|1631.72|ronic, re 6726|dodger metallic honeydew cream lavender|Manufacturer#3|Brand#33|ECONOMY POLISHED NICKEL|19|SM DRUM|1632.72|fluff 6727|tomato antique brown smoke dark|Manufacturer#2|Brand#24|ECONOMY PLATED TIN|41|WRAP BOX|1633.72|ly. packages 6728|cream antique slate honeydew plum|Manufacturer#5|Brand#53|SMALL BRUSHED COPPER|14|LG PACK|1634.72|riously regular 6729|forest mint cyan goldenrod azure|Manufacturer#1|Brand#12|LARGE BURNISHED TIN|36|LG BOX|1635.72|ong the warthogs i 6730|slate rose cyan blue khaki|Manufacturer#2|Brand#21|STANDARD BURNISHED STEEL|38|SM PKG|1636.73|lites. 6731|steel hot blue dodger ivory|Manufacturer#2|Brand#21|MEDIUM ANODIZED STEEL|10|SM BAG|1637.73|es caj 6732|dark smoke navy dodger papaya|Manufacturer#3|Brand#32|STANDARD PLATED COPPER|16|LG CAN|1638.73|nts-- fluffily i 6733|sienna coral lemon gainsboro grey|Manufacturer#1|Brand#12|SMALL BRUSHED STEEL|11|LG BAG|1639.73|y. slyl 6734|yellow plum powder navajo lavender|Manufacturer#5|Brand#52|SMALL BRUSHED TIN|32|JUMBO DRUM|1640.73|above 6735|sky wheat goldenrod spring plum|Manufacturer#4|Brand#41|MEDIUM BURNISHED STEEL|44|LG PACK|1641.73|ly regular asympto 6736|olive burlywood azure cyan plum|Manufacturer#2|Brand#24|MEDIUM POLISHED TIN|16|SM BOX|1642.73|ts. ex 6737|frosted plum saddle black maroon|Manufacturer#5|Brand#52|PROMO BURNISHED NICKEL|9|WRAP BOX|1643.73|c forges. quickly s 6738|drab slate steel sienna papaya|Manufacturer#3|Brand#32|STANDARD PLATED TIN|50|SM PKG|1644.73|ajole regul 6739|bisque tan coral blue salmon|Manufacturer#1|Brand#11|STANDARD POLISHED COPPER|43|JUMBO CAN|1645.73|blithely unusual 6740|gainsboro mint slate blue coral|Manufacturer#2|Brand#23|LARGE POLISHED STEEL|43|LG JAR|1646.74|xcuses. fluff 6741|brown salmon medium aquamarine magenta|Manufacturer#3|Brand#32|MEDIUM BURNISHED TIN|20|SM BOX|1647.74|osits. furio 6742|wheat yellow dark maroon black|Manufacturer#5|Brand#54|LARGE PLATED BRASS|34|JUMBO PACK|1648.74|ithely special deposi 6743|misty gainsboro sky turquoise forest|Manufacturer#5|Brand#54|STANDARD ANODIZED COPPER|38|MED JAR|1649.74|hely final 6744|navy lavender magenta white dodger|Manufacturer#1|Brand#15|SMALL POLISHED STEEL|6|WRAP DRUM|1650.74| carefully final a 6745|cream steel magenta pale bisque|Manufacturer#3|Brand#33|ECONOMY POLISHED COPPER|12|WRAP BOX|1651.74| theodol 6746|thistle rosy deep mint plum|Manufacturer#5|Brand#51|ECONOMY BURNISHED COPPER|24|MED DRUM|1652.74|refully? slyly 6747|brown drab puff dim thistle|Manufacturer#1|Brand#12|MEDIUM ANODIZED TIN|40|WRAP CAN|1653.74|ymptotes mai 6748|sienna plum papaya green drab|Manufacturer#3|Brand#35|SMALL BURNISHED TIN|26|LG CAN|1654.74|ing to th 6749|yellow royal thistle beige lawn|Manufacturer#4|Brand#41|MEDIUM POLISHED COPPER|44|JUMBO BAG|1655.74|e fluffily after the b 6750|indian dark rosy navy dodger|Manufacturer#5|Brand#51|LARGE POLISHED BRASS|50|SM DRUM|1656.75|arefully ironic packag 6751|sandy steel tomato pink royal|Manufacturer#1|Brand#13|ECONOMY BURNISHED STEEL|15|WRAP PACK|1657.75| foxes. slyl 6752|medium rose sandy puff navy|Manufacturer#4|Brand#42|STANDARD BRUSHED NICKEL|4|SM BOX|1658.75| idea 6753|firebrick lemon burnished ghost maroon|Manufacturer#5|Brand#54|STANDARD BURNISHED STEEL|14|SM PACK|1659.75|ar ideas 6754|medium pink magenta rose burlywood|Manufacturer#1|Brand#15|MEDIUM ANODIZED STEEL|13|WRAP PACK|1660.75|lithely 6755|wheat hot medium honeydew grey|Manufacturer#4|Brand#41|LARGE PLATED COPPER|43|SM BAG|1661.75| to the even p 6756|rosy drab bisque purple ghost|Manufacturer#1|Brand#15|LARGE BURNISHED BRASS|47|JUMBO PACK|1662.75|ickly e 6757|steel brown navajo firebrick medium|Manufacturer#5|Brand#52|LARGE ANODIZED STEEL|5|JUMBO BOX|1663.75|ites serve quickly i 6758|lavender rose honeydew frosted metallic|Manufacturer#5|Brand#55|PROMO PLATED BRASS|38|JUMBO BAG|1664.75|s sleep 6759|burlywood antique bisque brown orchid|Manufacturer#2|Brand#24|ECONOMY BRUSHED COPPER|23|SM DRUM|1665.75|ts nag blithel 6760|rosy azure linen gainsboro chiffon|Manufacturer#5|Brand#52|STANDARD BRUSHED NICKEL|34|MED BAG|1666.76|furiously regula 6761|misty pale aquamarine orange azure|Manufacturer#5|Brand#54|SMALL BURNISHED TIN|26|LG DRUM|1667.76|ular platelets. 6762|rose royal chiffon magenta metallic|Manufacturer#1|Brand#15|SMALL ANODIZED BRASS|34|JUMBO BOX|1668.76|ckage 6763|lace papaya azure aquamarine powder|Manufacturer#4|Brand#43|MEDIUM PLATED STEEL|41|JUMBO PKG|1669.76|wake ca 6764|puff bisque forest almond salmon|Manufacturer#2|Brand#24|PROMO BURNISHED STEEL|50|JUMBO BOX|1670.76|uses cajole furio 6765|maroon tomato royal lavender papaya|Manufacturer#4|Brand#45|MEDIUM ANODIZED TIN|20|JUMBO BAG|1671.76|cing deposits. furi 6766|lace moccasin smoke honeydew burlywood|Manufacturer#2|Brand#22|SMALL POLISHED STEEL|30|MED JAR|1672.76|osits. reg 6767|maroon ghost rosy violet lemon|Manufacturer#2|Brand#25|STANDARD ANODIZED BRASS|12|SM CASE|1673.76|e bli 6768|lace moccasin orchid red beige|Manufacturer#1|Brand#13|SMALL ANODIZED COPPER|40|SM CAN|1674.76|r package 6769|turquoise yellow sky pale olive|Manufacturer#1|Brand#13|PROMO PLATED BRASS|26|JUMBO PKG|1675.76| pack 6770|red pale navajo slate black|Manufacturer#1|Brand#13|LARGE POLISHED COPPER|16|SM DRUM|1676.77|ly ironic asymptotes 6771|misty papaya almond royal chocolate|Manufacturer#3|Brand#31|MEDIUM ANODIZED STEEL|7|JUMBO CASE|1677.77|ully s 6772|lime steel black puff orange|Manufacturer#4|Brand#41|ECONOMY BURNISHED COPPER|8|JUMBO DRUM|1678.77|accounts cajole blithe 6773|orange lime slate powder plum|Manufacturer#2|Brand#23|MEDIUM ANODIZED TIN|24|LG DRUM|1679.77|slyly silent account 6774|cornsilk royal gainsboro powder cream|Manufacturer#5|Brand#51|SMALL ANODIZED NICKEL|40|MED PKG|1680.77|e evenly unusu 6775|rosy navy cream cornflower burnished|Manufacturer#5|Brand#51|LARGE ANODIZED BRASS|22|LG BAG|1681.77|efully expr 6776|puff sienna burlywood midnight drab|Manufacturer#2|Brand#25|SMALL BRUSHED COPPER|26|SM PKG|1682.77|equests boos 6777|honeydew cornflower cyan navajo tan|Manufacturer#1|Brand#15|STANDARD ANODIZED BRASS|35|JUMBO CASE|1683.77| carefully specia 6778|medium goldenrod antique green sandy|Manufacturer#2|Brand#25|MEDIUM POLISHED COPPER|18|MED CASE|1684.77|lways final requests. 6779|floral violet navy tomato maroon|Manufacturer#4|Brand#42|ECONOMY PLATED COPPER|23|LG PACK|1685.77|thely u 6780|rose deep cyan puff navajo|Manufacturer#4|Brand#41|PROMO POLISHED NICKEL|17|SM PKG|1686.78|es. carefully stealthy 6781|saddle pale tan lawn sienna|Manufacturer#2|Brand#25|LARGE BRUSHED STEEL|2|MED BAG|1687.78|ecial 6782|spring khaki bisque honeydew pale|Manufacturer#1|Brand#11|STANDARD BURNISHED STEEL|17|SM CASE|1688.78|counts sleep blithely 6783|hot ghost navy peru blush|Manufacturer#4|Brand#45|LARGE ANODIZED BRASS|7|LG BAG|1689.78|c pinto 6784|thistle burlywood papaya gainsboro lemon|Manufacturer#4|Brand#42|LARGE ANODIZED STEEL|32|JUMBO BOX|1690.78|unts nag blithely wit 6785|chartreuse violet pale puff orchid|Manufacturer#5|Brand#54|STANDARD BURNISHED BRASS|32|MED PKG|1691.78|uickly theodoli 6786|chiffon khaki lawn gainsboro turquoise|Manufacturer#1|Brand#12|LARGE ANODIZED NICKEL|11|JUMBO CAN|1692.78|ently. even, 6787|magenta coral red blush green|Manufacturer#4|Brand#43|STANDARD PLATED NICKEL|14|JUMBO BAG|1693.78|le carefully af 6788|blanched tomato hot chiffon steel|Manufacturer#2|Brand#25|LARGE ANODIZED TIN|50|JUMBO DRUM|1694.78|y bold 6789|cream dodger moccasin violet green|Manufacturer#4|Brand#45|STANDARD ANODIZED TIN|11|LG BAG|1695.78| among the final de 6790|bisque firebrick mint plum burnished|Manufacturer#4|Brand#42|PROMO BURNISHED COPPER|39|SM BOX|1696.79| among the b 6791|moccasin magenta chocolate snow beige|Manufacturer#4|Brand#43|LARGE BURNISHED STEEL|48|WRAP CAN|1697.79|s. pinto beans mainta 6792|aquamarine powder green spring honeydew|Manufacturer#1|Brand#13|MEDIUM PLATED NICKEL|48|MED BAG|1698.79|y above the stealthil 6793|misty grey violet plum lace|Manufacturer#3|Brand#35|PROMO BRUSHED NICKEL|14|JUMBO BAG|1699.79|refully r 6794|khaki seashell pale orange smoke|Manufacturer#3|Brand#34|LARGE BRUSHED NICKEL|23|MED CAN|1700.79|es among 6795|plum maroon azure hot slate|Manufacturer#5|Brand#53|ECONOMY BRUSHED NICKEL|48|WRAP CASE|1701.79|xes w 6796|cyan blue lavender puff tan|Manufacturer#2|Brand#21|PROMO BRUSHED STEEL|14|JUMBO PKG|1702.79|accounts detect furio 6797|almond indian cyan antique snow|Manufacturer#1|Brand#14|STANDARD POLISHED STEEL|8|LG JAR|1703.79|ake furiou 6798|sandy cyan navy hot black|Manufacturer#5|Brand#52|MEDIUM POLISHED TIN|36|SM CAN|1704.79|ely ironic accoun 6799|seashell wheat hot almond rose|Manufacturer#3|Brand#32|MEDIUM BURNISHED NICKEL|1|SM DRUM|1705.79|requests wake f 6800|ghost lawn chartreuse drab plum|Manufacturer#1|Brand#13|STANDARD POLISHED TIN|15|MED CASE|1706.80| express instructi 6801|seashell ghost azure firebrick cornsilk|Manufacturer#3|Brand#34|SMALL ANODIZED BRASS|36|WRAP PKG|1707.80|oxes. foxes among t 6802|floral blush grey frosted wheat|Manufacturer#1|Brand#14|STANDARD BRUSHED NICKEL|41|WRAP BAG|1708.80|ing platele 6803|sienna saddle dodger azure smoke|Manufacturer#3|Brand#33|SMALL BRUSHED BRASS|46|MED BAG|1709.80|ular excus 6804|turquoise medium thistle puff peach|Manufacturer#2|Brand#25|MEDIUM BRUSHED COPPER|40|SM PKG|1710.80|lyly pending ideas. 6805|misty azure green orchid orange|Manufacturer#1|Brand#14|PROMO BURNISHED COPPER|30|WRAP PKG|1711.80| furiously 6806|blush bisque antique lemon gainsboro|Manufacturer#5|Brand#51|PROMO ANODIZED BRASS|28|SM BOX|1712.80|ages. bold foxe 6807|drab wheat snow moccasin saddle|Manufacturer#1|Brand#12|LARGE PLATED NICKEL|48|JUMBO PACK|1713.80| instructions de 6808|steel turquoise drab sky misty|Manufacturer#3|Brand#34|PROMO BURNISHED STEEL|3|MED PKG|1714.80|ously unusual 6809|puff slate misty powder spring|Manufacturer#5|Brand#54|MEDIUM ANODIZED STEEL|22|SM CASE|1715.80|ar requests cajol 6810|tan tomato seashell navajo lawn|Manufacturer#4|Brand#43|STANDARD BRUSHED NICKEL|21|JUMBO JAR|1716.81|uriously regular r 6811|navy blush cornflower lace dim|Manufacturer#5|Brand#54|STANDARD ANODIZED COPPER|42|WRAP BOX|1717.81|kly blithely even a 6812|lime chartreuse papaya blue blush|Manufacturer#4|Brand#41|MEDIUM BURNISHED STEEL|45|SM CAN|1718.81| accounts haggl 6813|light azure royal brown sienna|Manufacturer#2|Brand#25|STANDARD ANODIZED COPPER|17|MED JAR|1719.81|uickly 6814|dim cornflower yellow lemon almond|Manufacturer#4|Brand#44|STANDARD POLISHED BRASS|4|WRAP BOX|1720.81|kages. blithely 6815|deep burlywood aquamarine puff dark|Manufacturer#1|Brand#11|SMALL POLISHED COPPER|14|JUMBO DRUM|1721.81|ss accounts eat dar 6816|peru brown chartreuse dark frosted|Manufacturer#2|Brand#24|STANDARD BURNISHED NICKEL|26|MED BAG|1722.81|s are! slyly 6817|cornsilk peru lavender goldenrod cream|Manufacturer#5|Brand#52|PROMO ANODIZED TIN|8|SM JAR|1723.81|al pinto be 6818|floral smoke light deep pale|Manufacturer#5|Brand#51|PROMO POLISHED BRASS|40|JUMBO PACK|1724.81|nusua 6819|snow violet royal dark khaki|Manufacturer#2|Brand#24|STANDARD PLATED STEEL|5|JUMBO BOX|1725.81| prom 6820|light smoke orchid plum wheat|Manufacturer#1|Brand#11|STANDARD ANODIZED STEEL|1|SM PACK|1726.82|nal deposi 6821|red chocolate tomato orange black|Manufacturer#2|Brand#22|LARGE PLATED BRASS|48|SM DRUM|1727.82|g accounts cajole car 6822|sky deep medium goldenrod lime|Manufacturer#5|Brand#55|STANDARD BRUSHED NICKEL|4|MED CASE|1728.82|ependencies are thinl 6823|tomato white magenta floral ivory|Manufacturer#4|Brand#43|ECONOMY PLATED STEEL|9|JUMBO BOX|1729.82| ironi 6824|tomato firebrick antique mint lemon|Manufacturer#3|Brand#33|LARGE ANODIZED COPPER|45|WRAP JAR|1730.82|ctions alongs 6825|gainsboro lace steel floral orchid|Manufacturer#4|Brand#45|LARGE POLISHED COPPER|24|LG BAG|1731.82|ns. ex 6826|sienna cornflower green lime tomato|Manufacturer#5|Brand#51|MEDIUM POLISHED STEEL|19|JUMBO BOX|1732.82|nts. careful 6827|bisque indian tan red midnight|Manufacturer#3|Brand#32|SMALL BURNISHED BRASS|8|SM BOX|1733.82|accounts-- furiously 6828|bisque sandy powder cornflower blanched|Manufacturer#1|Brand#15|MEDIUM POLISHED TIN|11|JUMBO JAR|1734.82|ic req 6829|midnight lawn steel chocolate salmon|Manufacturer#1|Brand#13|LARGE BRUSHED BRASS|6|MED BAG|1735.82|e regu 6830|violet grey navajo dim blush|Manufacturer#5|Brand#54|PROMO BRUSHED TIN|44|MED CAN|1736.83|s haggle quickly 6831|medium lime papaya cream sky|Manufacturer#1|Brand#15|SMALL BURNISHED BRASS|21|LG JAR|1737.83| final asymptotes. fi 6832|medium blush aquamarine bisque deep|Manufacturer#2|Brand#22|SMALL POLISHED TIN|39|SM BOX|1738.83|xpress 6833|metallic snow black khaki coral|Manufacturer#4|Brand#41|MEDIUM BURNISHED COPPER|25|LG BOX|1739.83|usly final accounts 6834|grey yellow green honeydew navajo|Manufacturer#3|Brand#34|STANDARD BURNISHED COPPER|17|WRAP PACK|1740.83|s inte 6835|blush floral midnight magenta frosted|Manufacturer#4|Brand#44|SMALL ANODIZED NICKEL|38|SM CASE|1741.83|blithely 6836|pale lavender bisque yellow indian|Manufacturer#3|Brand#31|SMALL PLATED TIN|7|LG JAR|1742.83|ges. 6837|drab salmon floral mint burlywood|Manufacturer#3|Brand#31|ECONOMY ANODIZED TIN|29|SM CASE|1743.83|sits na 6838|antique gainsboro aquamarine green spring|Manufacturer#1|Brand#15|PROMO ANODIZED STEEL|35|SM PACK|1744.83|ing packages. bold acc 6839|purple metallic saddle midnight magenta|Manufacturer#3|Brand#32|STANDARD PLATED BRASS|4|MED JAR|1745.83|s wake slyly. 6840|light drab snow coral deep|Manufacturer#1|Brand#15|SMALL ANODIZED NICKEL|26|WRAP BAG|1746.84|riously final ac 6841|lime cornflower ivory dim slate|Manufacturer#1|Brand#14|LARGE POLISHED NICKEL|3|SM DRUM|1747.84|y ruthles 6842|lime medium coral bisque pale|Manufacturer#3|Brand#31|ECONOMY PLATED NICKEL|1|WRAP JAR|1748.84|kages alongside 6843|honeydew violet cornsilk sienna antique|Manufacturer#3|Brand#35|STANDARD BRUSHED STEEL|46|LG PKG|1749.84|s use 6844|cyan chiffon orchid plum rose|Manufacturer#2|Brand#22|LARGE POLISHED TIN|49|SM CAN|1750.84|ess deposits sleep 6845|ghost lime violet cornsilk peach|Manufacturer#4|Brand#45|MEDIUM PLATED STEEL|40|WRAP JAR|1751.84|less fo 6846|white maroon chocolate smoke pink|Manufacturer#5|Brand#53|MEDIUM BURNISHED BRASS|35|WRAP CASE|1752.84|egular pinto 6847|pink moccasin peru puff saddle|Manufacturer#1|Brand#14|SMALL ANODIZED NICKEL|45|SM PACK|1753.84| accounts accord 6848|rosy honeydew plum tomato deep|Manufacturer#1|Brand#14|PROMO POLISHED BRASS|9|LG BOX|1754.84|romis 6849|powder pale dodger firebrick wheat|Manufacturer#3|Brand#34|STANDARD BURNISHED STEEL|32|JUMBO CASE|1755.84|ost sly 6850|brown blue tan hot blush|Manufacturer#5|Brand#53|ECONOMY ANODIZED STEEL|26|MED CAN|1756.85|ts integrate accordin 6851|red peach seashell honeydew burlywood|Manufacturer#5|Brand#51|MEDIUM BRUSHED TIN|36|JUMBO BAG|1757.85|uests kindle f 6852|pale cornsilk metallic brown beige|Manufacturer#2|Brand#23|PROMO POLISHED COPPER|25|SM PKG|1758.85| fluffily even ac 6853|purple sandy dark powder indian|Manufacturer#1|Brand#14|ECONOMY POLISHED COPPER|44|MED CASE|1759.85|bout the p 6854|azure navy gainsboro cream ghost|Manufacturer#4|Brand#41|ECONOMY BRUSHED BRASS|15|JUMBO JAR|1760.85|bout the carefully reg 6855|magenta spring orchid azure frosted|Manufacturer#5|Brand#51|PROMO ANODIZED NICKEL|6|WRAP DRUM|1761.85|l orbi 6856|blanched midnight cream papaya drab|Manufacturer#4|Brand#41|SMALL BRUSHED TIN|5|JUMBO CAN|1762.85|s haggle above the flu 6857|red spring blush cornsilk peach|Manufacturer#4|Brand#42|STANDARD POLISHED TIN|46|LG JAR|1763.85| quickly unu 6858|blush puff firebrick beige deep|Manufacturer#1|Brand#11|SMALL BRUSHED BRASS|30|WRAP BOX|1764.85|t evenly express ac 6859|snow lavender navy linen chartreuse|Manufacturer#3|Brand#35|ECONOMY ANODIZED NICKEL|35|MED PKG|1765.85|sual dolphi 6860|snow green linen dim orange|Manufacturer#1|Brand#14|LARGE POLISHED STEEL|9|LG JAR|1766.86|ly caref 6861|white violet coral olive metallic|Manufacturer#2|Brand#23|STANDARD POLISHED BRASS|32|MED BAG|1767.86| the slyly 6862|violet cornflower pink rose burnished|Manufacturer#1|Brand#13|PROMO POLISHED TIN|36|JUMBO CAN|1768.86|ckages cajole sl 6863|brown bisque linen cyan drab|Manufacturer#3|Brand#35|ECONOMY BURNISHED NICKEL|25|JUMBO CAN|1769.86|aggle slyly ste 6864|burnished misty firebrick cream papaya|Manufacturer#4|Brand#45|SMALL PLATED NICKEL|20|WRAP CAN|1770.86|y even accounts acros 6865|seashell salmon lace mint aquamarine|Manufacturer#3|Brand#35|MEDIUM BRUSHED NICKEL|17|SM BOX|1771.86|l depend 6866|goldenrod midnight metallic cornsilk wheat|Manufacturer#5|Brand#54|ECONOMY POLISHED COPPER|23|SM DRUM|1772.86|quickly along the slyl 6867|green puff blanched light lawn|Manufacturer#2|Brand#21|MEDIUM BRUSHED TIN|21|WRAP DRUM|1773.86|ns. final foxes al 6868|gainsboro goldenrod white forest navy|Manufacturer#5|Brand#52|STANDARD ANODIZED BRASS|17|MED BAG|1774.86|es. furiousl 6869|grey salmon seashell bisque linen|Manufacturer#1|Brand#14|MEDIUM PLATED BRASS|39|WRAP BAG|1775.86|nic ac 6870|chartreuse firebrick indian royal sky|Manufacturer#5|Brand#53|LARGE POLISHED TIN|33|WRAP BOX|1776.87|ar accounts affix bli 6871|peach steel turquoise sky forest|Manufacturer#1|Brand#11|ECONOMY PLATED STEEL|46|WRAP PKG|1777.87|r theodolites. ca 6872|hot almond azure lace forest|Manufacturer#2|Brand#21|PROMO BRUSHED TIN|41|LG PACK|1778.87|l packages boost aft 6873|medium red dark saddle white|Manufacturer#4|Brand#43|PROMO PLATED STEEL|47|MED DRUM|1779.87|unts snooze by the 6874|violet burnished misty medium frosted|Manufacturer#5|Brand#53|SMALL BRUSHED TIN|3|MED PKG|1780.87|above th 6875|blue papaya steel lemon cyan|Manufacturer#2|Brand#25|ECONOMY PLATED NICKEL|22|JUMBO JAR|1781.87|ts are deposits. slyl 6876|lavender sandy mint light deep|Manufacturer#1|Brand#14|LARGE BRUSHED TIN|5|MED PKG|1782.87|ickly regular, silent 6877|lemon ghost green thistle peru|Manufacturer#3|Brand#33|LARGE POLISHED COPPER|16|JUMBO PKG|1783.87|kly b 6878|cream wheat aquamarine yellow burlywood|Manufacturer#1|Brand#12|ECONOMY ANODIZED COPPER|44|LG BOX|1784.87|ular dep 6879|sienna slate chartreuse lavender rose|Manufacturer#5|Brand#51|STANDARD PLATED COPPER|25|JUMBO DRUM|1785.87|r the d 6880|lemon tan orchid white grey|Manufacturer#4|Brand#42|LARGE PLATED NICKEL|11|LG JAR|1786.88|ss deposits. carefull 6881|chiffon indian green salmon antique|Manufacturer#3|Brand#35|STANDARD POLISHED TIN|45|LG CAN|1787.88|he even pinto 6882|burlywood antique light cornflower beige|Manufacturer#2|Brand#21|LARGE ANODIZED BRASS|5|MED CASE|1788.88|e furiously bold 6883|royal aquamarine frosted moccasin beige|Manufacturer#1|Brand#14|ECONOMY BRUSHED BRASS|11|LG BAG|1789.88|venly express ideas. q 6884|smoke misty blanched burnished antique|Manufacturer#4|Brand#41|STANDARD BURNISHED COPPER|35|SM BOX|1790.88|tes use ac 6885|tomato pale dodger ghost orchid|Manufacturer#3|Brand#35|MEDIUM PLATED COPPER|46|JUMBO CASE|1791.88|ly regular d 6886|lace olive navy lime snow|Manufacturer#2|Brand#24|SMALL POLISHED COPPER|16|WRAP PKG|1792.88|e the even d 6887|linen ivory dim smoke seashell|Manufacturer#4|Brand#41|STANDARD BRUSHED BRASS|24|MED BAG|1793.88|s sleep 6888|azure purple moccasin magenta peru|Manufacturer#1|Brand#11|MEDIUM POLISHED TIN|4|SM PKG|1794.88|r request 6889|chiffon royal lemon sandy ivory|Manufacturer#2|Brand#22|MEDIUM PLATED BRASS|4|MED CASE|1795.88|ing to the even pl 6890|magenta misty peach medium peru|Manufacturer#2|Brand#24|ECONOMY POLISHED TIN|22|JUMBO CAN|1796.89|accounts. 6891|salmon brown red lavender sky|Manufacturer#1|Brand#11|SMALL ANODIZED COPPER|31|LG BAG|1797.89|pecia 6892|dim almond deep drab pale|Manufacturer#2|Brand#25|LARGE POLISHED NICKEL|37|WRAP DRUM|1798.89| beans; q 6893|metallic beige azure salmon brown|Manufacturer#2|Brand#21|PROMO BRUSHED BRASS|5|SM PACK|1799.89|ly ironic acc 6894|cyan grey violet deep almond|Manufacturer#1|Brand#13|SMALL PLATED TIN|30|LG JAR|1800.89|. expres 6895|thistle turquoise olive mint antique|Manufacturer#1|Brand#14|MEDIUM BRUSHED BRASS|20|JUMBO CASE|1801.89|sly ironic reque 6896|olive smoke lemon goldenrod moccasin|Manufacturer#3|Brand#35|SMALL PLATED STEEL|43|JUMBO BOX|1802.89|ts. enticingly expre 6897|coral magenta mint firebrick white|Manufacturer#4|Brand#41|ECONOMY BRUSHED BRASS|4|LG BAG|1803.89| blithely 6898|green yellow blue peach thistle|Manufacturer#1|Brand#13|LARGE PLATED STEEL|29|MED DRUM|1804.89|. pinto 6899|lemon maroon drab lace peru|Manufacturer#3|Brand#34|MEDIUM BRUSHED BRASS|49|JUMBO CASE|1805.89|sily express theodolit 6900|bisque blue navy turquoise saddle|Manufacturer#2|Brand#25|MEDIUM BRUSHED NICKEL|42|SM BOX|1806.90|le. bold 6901|indian drab lavender purple bisque|Manufacturer#4|Brand#42|PROMO ANODIZED STEEL|25|WRAP CASE|1807.90|e carefully 6902|ghost green violet mint bisque|Manufacturer#2|Brand#23|STANDARD ANODIZED BRASS|19|LG CAN|1808.90| deposits boost ab 6903|mint grey turquoise burnished spring|Manufacturer#3|Brand#32|LARGE POLISHED NICKEL|3|JUMBO BAG|1809.90|its acros 6904|deep salmon olive drab thistle|Manufacturer#3|Brand#32|SMALL PLATED BRASS|6|LG CAN|1810.90|deas promise ab 6905|sandy powder bisque orchid hot|Manufacturer#1|Brand#12|LARGE BURNISHED BRASS|32|LG BAG|1811.90| ironic packages al 6906|black moccasin deep lace blush|Manufacturer#5|Brand#51|STANDARD BURNISHED STEEL|23|SM PACK|1812.90|unts will sleep sp 6907|grey dodger lawn thistle tomato|Manufacturer#1|Brand#12|ECONOMY BRUSHED NICKEL|22|SM CAN|1813.90|ts mold furiously. 6908|royal rose peach rosy azure|Manufacturer#4|Brand#44|ECONOMY BRUSHED BRASS|7|LG CAN|1814.90|. furiously fi 6909|plum cornflower gainsboro tomato lavender|Manufacturer#5|Brand#53|SMALL ANODIZED NICKEL|12|SM CASE|1815.90|y ironic request 6910|coral steel deep sienna orange|Manufacturer#4|Brand#42|MEDIUM BRUSHED STEEL|6|JUMBO BAG|1816.91| silent accounts aft 6911|sky papaya magenta tomato peach|Manufacturer#2|Brand#22|PROMO POLISHED BRASS|31|JUMBO BOX|1817.91|ously through the 6912|olive brown navy chartreuse rose|Manufacturer#5|Brand#54|MEDIUM BRUSHED NICKEL|6|JUMBO PKG|1818.91|eat b 6913|grey rosy tan puff sienna|Manufacturer#1|Brand#13|SMALL PLATED COPPER|15|MED CAN|1819.91| the slyly 6914|peach indian maroon chiffon saddle|Manufacturer#2|Brand#22|STANDARD PLATED NICKEL|18|WRAP PACK|1820.91|ly final depos 6915|linen khaki saddle chocolate medium|Manufacturer#5|Brand#51|LARGE ANODIZED COPPER|37|SM BAG|1821.91|pendenc 6916|azure snow rose sandy chartreuse|Manufacturer#3|Brand#33|MEDIUM PLATED STEEL|1|SM BAG|1822.91|usly unusua 6917|steel black moccasin lace ghost|Manufacturer#3|Brand#34|SMALL ANODIZED NICKEL|42|LG PACK|1823.91| theo 6918|bisque mint drab gainsboro orange|Manufacturer#1|Brand#14|SMALL BRUSHED TIN|36|SM CASE|1824.91|ges? furiously e 6919|linen smoke light powder papaya|Manufacturer#2|Brand#22|ECONOMY BRUSHED TIN|18|WRAP BAG|1825.91|ully blithely bold id 6920|rose pink chartreuse dim deep|Manufacturer#3|Brand#34|STANDARD PLATED COPPER|50|WRAP CAN|1826.92|carefully. 6921|green puff forest pale lawn|Manufacturer#3|Brand#34|MEDIUM PLATED COPPER|10|SM BOX|1827.92|usly. stealthy courts 6922|firebrick lime cream magenta maroon|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|25|WRAP JAR|1828.92|the blithely bold th 6923|ivory firebrick seashell peach navajo|Manufacturer#1|Brand#12|STANDARD POLISHED BRASS|49|JUMBO DRUM|1829.92|s. quickly unusual 6924|powder blanched maroon violet magenta|Manufacturer#2|Brand#21|MEDIUM BURNISHED COPPER|43|WRAP PACK|1830.92| fluffy depende 6925|dodger forest cream khaki aquamarine|Manufacturer#5|Brand#51|LARGE POLISHED NICKEL|23|WRAP DRUM|1831.92|areful 6926|azure drab floral khaki spring|Manufacturer#5|Brand#52|LARGE PLATED STEEL|10|WRAP PACK|1832.92|ly pendin 6927|plum honeydew azure lime pale|Manufacturer#5|Brand#52|PROMO POLISHED BRASS|23|JUMBO BAG|1833.92| furi 6928|violet magenta peach lemon burnished|Manufacturer#2|Brand#22|MEDIUM ANODIZED STEEL|39|WRAP CAN|1834.92|s. blith 6929|spring goldenrod violet lemon slate|Manufacturer#3|Brand#35|PROMO PLATED STEEL|4|SM PKG|1835.92|ole fluffily. s 6930|chartreuse plum orchid violet lavender|Manufacturer#4|Brand#43|MEDIUM PLATED BRASS|15|WRAP CASE|1836.93| express, sile 6931|ghost black coral magenta mint|Manufacturer#5|Brand#52|SMALL PLATED NICKEL|43|SM CAN|1837.93|l gifts nag slyly af 6932|red pale firebrick turquoise moccasin|Manufacturer#4|Brand#45|PROMO POLISHED COPPER|30|JUMBO JAR|1838.93|nag carefully theodo 6933|mint smoke white olive powder|Manufacturer#3|Brand#34|STANDARD BURNISHED TIN|39|JUMBO BOX|1839.93|old accounts sleep 6934|tan slate orchid peru red|Manufacturer#1|Brand#13|SMALL BURNISHED STEEL|48|SM CAN|1840.93| quickly f 6935|lawn forest blanched burlywood pale|Manufacturer#2|Brand#24|SMALL PLATED COPPER|14|LG DRUM|1841.93|yly. 6936|dim orange indian drab midnight|Manufacturer#1|Brand#14|STANDARD ANODIZED NICKEL|28|WRAP DRUM|1842.93|es. s 6937|drab beige sky cornsilk firebrick|Manufacturer#2|Brand#21|STANDARD BRUSHED STEEL|33|JUMBO PKG|1843.93|asymptotes: final t 6938|linen burnished lemon drab moccasin|Manufacturer#2|Brand#25|LARGE PLATED STEEL|25|MED PKG|1844.93|nic accounts cajole 6939|mint pale antique rose orange|Manufacturer#1|Brand#15|SMALL PLATED TIN|28|WRAP BAG|1845.93|among the fur 6940|linen cyan chiffon tomato red|Manufacturer#3|Brand#33|PROMO BURNISHED NICKEL|5|SM JAR|1846.94|slyly final, 6941|honeydew plum blush lemon chiffon|Manufacturer#1|Brand#11|PROMO PLATED BRASS|40|WRAP PACK|1847.94|e even, expre 6942|thistle deep dim almond maroon|Manufacturer#3|Brand#31|PROMO BURNISHED STEEL|4|WRAP PKG|1848.94| packag 6943|peru rose lavender light puff|Manufacturer#2|Brand#21|STANDARD ANODIZED BRASS|9|SM BAG|1849.94|ts. bli 6944|sky chocolate olive yellow blanched|Manufacturer#5|Brand#51|MEDIUM POLISHED STEEL|41|JUMBO PACK|1850.94| the pending, final 6945|cornsilk slate peru chiffon orchid|Manufacturer#2|Brand#25|PROMO POLISHED STEEL|34|JUMBO JAR|1851.94|pecial package 6946|chartreuse saddle peach plum burnished|Manufacturer#3|Brand#33|PROMO BURNISHED BRASS|44|SM PACK|1852.94|uests boost furiously 6947|indian dim lace papaya blue|Manufacturer#2|Brand#22|MEDIUM BRUSHED COPPER|18|MED CAN|1853.94| unusu 6948|peach seashell black gainsboro deep|Manufacturer#1|Brand#12|SMALL BRUSHED STEEL|28|WRAP BOX|1854.94|s accou 6949|slate dim cyan navajo white|Manufacturer#5|Brand#55|LARGE BRUSHED NICKEL|26|WRAP PACK|1855.94|lets. 6950|sky ivory cream turquoise aquamarine|Manufacturer#1|Brand#13|ECONOMY BURNISHED STEEL|13|WRAP CAN|1856.95|aggle furiou 6951|tan sandy royal sienna puff|Manufacturer#1|Brand#14|MEDIUM ANODIZED TIN|12|SM BAG|1857.95| accounts wak 6952|gainsboro sienna blush forest lemon|Manufacturer#4|Brand#41|MEDIUM ANODIZED BRASS|37|LG CAN|1858.95|nt packages a 6953|pale plum dark seashell orchid|Manufacturer#3|Brand#34|PROMO BRUSHED NICKEL|12|WRAP PKG|1859.95|. ironic 6954|cornsilk ivory midnight pale blanched|Manufacturer#3|Brand#34|ECONOMY BURNISHED COPPER|27|WRAP BOX|1860.95|regular dep 6955|drab chartreuse cyan wheat snow|Manufacturer#2|Brand#24|SMALL BURNISHED TIN|26|JUMBO CAN|1861.95|ronic, ironic acco 6956|ivory navajo purple blush green|Manufacturer#2|Brand#24|MEDIUM POLISHED COPPER|17|JUMBO DRUM|1862.95| pinto beans sleep fu 6957|salmon pink sky seashell tomato|Manufacturer#5|Brand#54|LARGE PLATED STEEL|5|JUMBO DRUM|1863.95|thely furiously 6958|misty azure floral aquamarine dim|Manufacturer#1|Brand#15|PROMO ANODIZED COPPER|32|SM JAR|1864.95|ully after 6959|puff green deep dark sienna|Manufacturer#5|Brand#51|SMALL PLATED BRASS|28|SM BAG|1865.95| deposits wake care 6960|antique orchid dim linen peru|Manufacturer#5|Brand#53|ECONOMY ANODIZED BRASS|48|JUMBO PKG|1866.96|beans slee 6961|slate drab maroon thistle puff|Manufacturer#3|Brand#35|ECONOMY BURNISHED NICKEL|18|SM JAR|1867.96|s believe furiously! a 6962|green aquamarine azure midnight chiffon|Manufacturer#1|Brand#11|STANDARD BRUSHED BRASS|44|JUMBO JAR|1868.96|e fluffily. expr 6963|maroon deep frosted hot coral|Manufacturer#1|Brand#13|ECONOMY BRUSHED TIN|31|SM PKG|1869.96|arefully carefull 6964|moccasin dodger ivory plum sky|Manufacturer#4|Brand#41|SMALL PLATED TIN|50|LG BOX|1870.96|hless 6965|snow spring peach dodger green|Manufacturer#1|Brand#13|STANDARD BURNISHED COPPER|45|JUMBO CAN|1871.96| nag. q 6966|olive blanched drab smoke light|Manufacturer#4|Brand#42|STANDARD ANODIZED STEEL|22|WRAP BOX|1872.96|sits haggle bold 6967|puff hot brown sky pink|Manufacturer#5|Brand#53|SMALL BURNISHED BRASS|9|SM PACK|1873.96| beans cajole qu 6968|spring red royal turquoise ghost|Manufacturer#3|Brand#35|ECONOMY BRUSHED NICKEL|24|JUMBO BOX|1874.96|indle furiously regula 6969|powder steel lemon aquamarine deep|Manufacturer#2|Brand#21|MEDIUM ANODIZED BRASS|37|JUMBO BAG|1875.96|ily regular requests. 6970|powder hot moccasin deep goldenrod|Manufacturer#5|Brand#52|LARGE POLISHED BRASS|12|JUMBO PKG|1876.97|ly eve 6971|navajo seashell gainsboro light metallic|Manufacturer#4|Brand#45|SMALL BURNISHED NICKEL|6|JUMBO CAN|1877.97|e slyly r 6972|midnight white smoke lavender goldenrod|Manufacturer#2|Brand#23|ECONOMY POLISHED STEEL|20|SM JAR|1878.97|mptotes. depe 6973|grey seashell navajo lace cream|Manufacturer#1|Brand#12|SMALL BRUSHED TIN|16|WRAP PKG|1879.97|t packages inte 6974|beige drab metallic firebrick yellow|Manufacturer#2|Brand#23|MEDIUM ANODIZED STEEL|12|JUMBO BAG|1880.97|ironic ac 6975|smoke maroon peru salmon puff|Manufacturer#4|Brand#41|MEDIUM BRUSHED COPPER|35|JUMBO JAR|1881.97|iously ironic acc 6976|snow blanched midnight smoke olive|Manufacturer#4|Brand#42|PROMO BURNISHED COPPER|20|LG CASE|1882.97|theodolites 6977|papaya lemon olive maroon grey|Manufacturer#4|Brand#42|PROMO POLISHED COPPER|37|LG PACK|1883.97|ld instru 6978|yellow peach lace chiffon linen|Manufacturer#4|Brand#45|MEDIUM ANODIZED BRASS|6|SM PACK|1884.97| abou 6979|dodger saddle thistle drab seashell|Manufacturer#1|Brand#12|STANDARD POLISHED COPPER|2|WRAP PKG|1885.97|e regular, even 6980|thistle linen orange white burlywood|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|35|WRAP PACK|1886.98|along the ironi 6981|cyan blue blanched tan hot|Manufacturer#1|Brand#12|PROMO PLATED TIN|34|WRAP PKG|1887.98|ructions are al 6982|brown wheat cream cornsilk pink|Manufacturer#2|Brand#22|SMALL POLISHED BRASS|12|LG PKG|1888.98|ons wake against th 6983|cornsilk royal olive smoke forest|Manufacturer#3|Brand#32|MEDIUM POLISHED BRASS|39|MED CASE|1889.98|uriously special foxes 6984|lime turquoise lavender floral rosy|Manufacturer#2|Brand#21|PROMO BURNISHED TIN|34|JUMBO PACK|1890.98|ly regular gifts det 6985|white black navy turquoise light|Manufacturer#4|Brand#44|MEDIUM BRUSHED COPPER|48|SM CASE|1891.98| packa 6986|lemon sky chocolate green linen|Manufacturer#2|Brand#21|ECONOMY BRUSHED BRASS|14|LG CAN|1892.98|ges. a 6987|grey burnished white antique beige|Manufacturer#2|Brand#23|ECONOMY POLISHED NICKEL|15|JUMBO PACK|1893.98|ctions wake furiously 6988|cream mint drab tomato ghost|Manufacturer#2|Brand#22|STANDARD ANODIZED BRASS|35|MED PKG|1894.98| deposit 6989|puff bisque wheat dim seashell|Manufacturer#5|Brand#55|STANDARD POLISHED COPPER|4|WRAP PACK|1895.98|s use accordi 6990|red coral violet white medium|Manufacturer#5|Brand#53|STANDARD BURNISHED NICKEL|44|JUMBO CASE|1896.99|ously ironic escapades 6991|drab chartreuse plum white goldenrod|Manufacturer#5|Brand#52|MEDIUM BRUSHED STEEL|33|WRAP BAG|1897.99| the 6992|chocolate lawn mint blanched turquoise|Manufacturer#5|Brand#52|SMALL BURNISHED TIN|21|SM BAG|1898.99|refully silent asympt 6993|thistle blanched peach red navy|Manufacturer#2|Brand#23|SMALL BURNISHED BRASS|1|MED JAR|1899.99|cial theodolites alon 6994|plum smoke cream navajo wheat|Manufacturer#4|Brand#43|PROMO BRUSHED NICKEL|35|JUMBO CASE|1900.99|gle. furi 6995|dim sienna navy smoke coral|Manufacturer#5|Brand#54|LARGE PLATED NICKEL|41|MED CASE|1901.99| according to 6996|metallic pale cornsilk antique tan|Manufacturer#2|Brand#21|LARGE ANODIZED TIN|45|WRAP BOX|1902.99|ly. blithely 6997|peru blanched burlywood lavender black|Manufacturer#5|Brand#54|SMALL BRUSHED STEEL|27|SM BAG|1903.99|gular sheaves! ca 6998|metallic khaki purple cyan cream|Manufacturer#1|Brand#12|STANDARD ANODIZED NICKEL|20|JUMBO BAG|1904.99|equest 6999|moccasin burlywood plum tomato azure|Manufacturer#3|Brand#31|PROMO PLATED TIN|18|JUMBO CAN|1905.99|ular deposits a 7000|beige orchid royal lace burnished|Manufacturer#4|Brand#44|ECONOMY ANODIZED NICKEL|37|WRAP CASE|907.00|. furiously c ================================================ FILE: src/test/regress/data/range_types.csv ================================================ "[1,3)","[1,3)","[1,3)","[""2000-01-02 00:30:00"",""2010-02-03 12:30:00"")" empty,"[1,)","(,)",empty ================================================ FILE: src/test/regress/data/supplier.data ================================================ [File too large to display: 135.4 KB] ================================================ FILE: src/test/regress/data/users_table.data ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/enterprise_failure_schedule ================================================ [File too large to display: 479 B] ================================================ FILE: src/test/regress/enterprise_isolation_logicalrep_1_schedule ================================================ [File too large to display: 587 B] ================================================ FILE: src/test/regress/enterprise_isolation_logicalrep_2_schedule ================================================ [File too large to display: 375 B] ================================================ FILE: src/test/regress/enterprise_isolation_logicalrep_3_schedule ================================================ test: isolation_setup # tests that change node metadata should precede # isolation_cluster_management such that tests # that come later can be parallelized test: isolation_cluster_management test: isolation_logical_replication_with_partitioning test: isolation_logical_replication_binaryless ================================================ FILE: src/test/regress/enterprise_isolation_schedule ================================================ [File too large to display: 668 B] ================================================ FILE: src/test/regress/enterprise_minimal_schedule ================================================ [File too large to display: 168 B] ================================================ FILE: src/test/regress/enterprise_schedule ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/expected/adaptive_executor.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/adaptive_executor_repartition.out ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/expected/add_coordinator.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/adv_lock_permission.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/aggregate_support.out ================================================ [File too large to display: 47.9 KB] ================================================ FILE: src/test/regress/expected/alter_database_owner.out ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/expected/alter_database_propagation.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/alter_distributed_table.out ================================================ [File too large to display: 59.3 KB] ================================================ FILE: src/test/regress/expected/alter_index.out ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/expected/alter_role_propagation.out ================================================ [File too large to display: 17.9 KB] ================================================ FILE: src/test/regress/expected/alter_table_add_column.out ================================================ [File too large to display: 20.3 KB] ================================================ FILE: src/test/regress/expected/alter_table_set_access_method.out ================================================ [File too large to display: 33.6 KB] ================================================ FILE: src/test/regress/expected/alter_table_single_shard_table.out ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/expected/anonymous_columns.out ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_alter_table_add_constraint_without_name.out ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_alter_table_add_constraint_without_name_create.out ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_recurring_outer_join.out ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_router.out ================================================ [File too large to display: 48.0 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_router_create.out ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate.out ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate_cascade.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate_cascade_create.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate_create.out ================================================ [File too large to display: 903 B] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate_partition.out ================================================ [File too large to display: 920 B] ================================================ FILE: src/test/regress/expected/arbitrary_configs_truncate_partition_create.out ================================================ [File too large to display: 648 B] ================================================ FILE: src/test/regress/expected/auto_undist_citus_local.out ================================================ [File too large to display: 59.5 KB] ================================================ FILE: src/test/regress/expected/background_rebalance.out ================================================ [File too large to display: 13.0 KB] ================================================ FILE: src/test/regress/expected/background_rebalance_parallel.out ================================================ [File too large to display: 31.7 KB] ================================================ FILE: src/test/regress/expected/background_rebalance_parallel_reference_tables.out ================================================ [File too large to display: 22.1 KB] ================================================ FILE: src/test/regress/expected/background_task_queue_monitor.out ================================================ [File too large to display: 35.7 KB] ================================================ FILE: src/test/regress/expected/base_enable_mx.out ================================================ [File too large to display: 379 B] ================================================ FILE: src/test/regress/expected/binary_protocol.out ================================================ [File too large to display: 10.2 KB] ================================================ FILE: src/test/regress/expected/bool_agg.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/cdc_library_path.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/ch_bench_having.out ================================================ [File too large to display: 13.3 KB] ================================================ FILE: src/test/regress/expected/ch_bench_having_mx.out ================================================ [File too large to display: 14.0 KB] ================================================ FILE: src/test/regress/expected/ch_bench_subquery_repartition.out ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_1.out ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_2.out ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_3.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_4.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_5.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_6.out ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/expected/ch_benchmarks_create_load.out ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/expected/chbenchmark_all_queries.out ================================================ [File too large to display: 32.2 KB] ================================================ FILE: src/test/regress/expected/check_cluster_state.out ================================================ [File too large to display: 196 B] ================================================ FILE: src/test/regress/expected/check_mx.out ================================================ [File too large to display: 783 B] ================================================ FILE: src/test/regress/expected/citus_aggregated_stats.out ================================================ [File too large to display: 10.7 KB] ================================================ FILE: src/test/regress/expected/citus_copy_shard_placement.out ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/expected/citus_depended_object.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/expected/citus_drain_node.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/citus_internal_access.out ================================================ [File too large to display: 391 B] ================================================ FILE: src/test/regress/expected/citus_local_dist_joins.out ================================================ [File too large to display: 31.2 KB] ================================================ FILE: src/test/regress/expected/citus_local_table_triggers.out ================================================ [File too large to display: 36.0 KB] ================================================ FILE: src/test/regress/expected/citus_local_tables.out ================================================ [File too large to display: 50.1 KB] ================================================ FILE: src/test/regress/expected/citus_local_tables_ent.out ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/expected/citus_local_tables_mx.out ================================================ [File too large to display: 49.3 KB] ================================================ FILE: src/test/regress/expected/citus_local_tables_queries.out ================================================ [File too large to display: 59.8 KB] ================================================ FILE: src/test/regress/expected/citus_local_tables_queries_mx.out ================================================ [File too large to display: 32.2 KB] ================================================ FILE: src/test/regress/expected/citus_locks.out ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/expected/citus_non_blocking_split_columnar.out ================================================ [File too large to display: 80.4 KB] ================================================ FILE: src/test/regress/expected/citus_non_blocking_split_shard_cleanup.out ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/expected/citus_non_blocking_split_shards.out ================================================ [File too large to display: 35.8 KB] ================================================ FILE: src/test/regress/expected/citus_run_command.out ================================================ [File too large to display: 522 B] ================================================ FILE: src/test/regress/expected/citus_schema_distribute_undistribute.out ================================================ [File too large to display: 34.0 KB] ================================================ FILE: src/test/regress/expected/citus_schema_move.out ================================================ [File too large to display: 9.2 KB] ================================================ FILE: src/test/regress/expected/citus_shards.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/citus_split_shard_by_split_points.out ================================================ [File too large to display: 27.6 KB] ================================================ FILE: src/test/regress/expected/citus_split_shard_by_split_points_deferred_drop.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/citus_split_shard_by_split_points_failure.out ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/expected/citus_split_shard_by_split_points_negative.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/citus_split_shard_columnar_partitioned.out ================================================ [File too large to display: 78.9 KB] ================================================ FILE: src/test/regress/expected/citus_stat_tenants.out ================================================ [File too large to display: 44.6 KB] ================================================ FILE: src/test/regress/expected/citus_table_triggers.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/expected/citus_update_table_statistics.out ================================================ [File too large to display: 13.9 KB] ================================================ FILE: src/test/regress/expected/clock.out ================================================ [File too large to display: 11.6 KB] ================================================ FILE: src/test/regress/expected/columnar_alter.out ================================================ [File too large to display: 14.1 KB] ================================================ FILE: src/test/regress/expected/columnar_alter_set_type.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/columnar_analyze.out ================================================ [File too large to display: 874 B] ================================================ FILE: src/test/regress/expected/columnar_chunk_filtering.out ================================================ [File too large to display: 50.6 KB] ================================================ FILE: src/test/regress/expected/columnar_chunk_filtering_0.out ================================================ [File too large to display: 50.6 KB] ================================================ FILE: src/test/regress/expected/columnar_citus_integration.out ================================================ [File too large to display: 38.7 KB] ================================================ FILE: src/test/regress/expected/columnar_clean.out ================================================ [File too large to display: 237 B] ================================================ FILE: src/test/regress/expected/columnar_copyto.out ================================================ [File too large to display: 944 B] ================================================ FILE: src/test/regress/expected/columnar_create.out ================================================ [File too large to display: 11.0 KB] ================================================ FILE: src/test/regress/expected/columnar_cursor.out ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/expected/columnar_data_types.out ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/expected/columnar_drop.out ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/expected/columnar_empty.out ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/expected/columnar_fallback_scan.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/columnar_first_row_number.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/columnar_index_concurrency.out ================================================ [File too large to display: 12.5 KB] ================================================ FILE: src/test/regress/expected/columnar_indexes.out ================================================ [File too large to display: 28.9 KB] ================================================ FILE: src/test/regress/expected/columnar_insert.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/columnar_join.out ================================================ [File too large to display: 12.5 KB] ================================================ FILE: src/test/regress/expected/columnar_load.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/columnar_lz4.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/columnar_lz4_0.out ================================================ [File too large to display: 114 B] ================================================ FILE: src/test/regress/expected/columnar_matview.out ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/expected/columnar_memory.out ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/expected/columnar_partitioning.out ================================================ [File too large to display: 16.6 KB] ================================================ FILE: src/test/regress/expected/columnar_paths.out ================================================ [File too large to display: 18.1 KB] ================================================ FILE: src/test/regress/expected/columnar_paths_0.out ================================================ [File too large to display: 18.2 KB] ================================================ FILE: src/test/regress/expected/columnar_permissions.out ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/expected/columnar_pg15.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/columnar_query.out ================================================ [File too large to display: 8.8 KB] ================================================ FILE: src/test/regress/expected/columnar_recursive.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/columnar_rollback.out ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/expected/columnar_tableoptions.out ================================================ [File too large to display: 14.6 KB] ================================================ FILE: src/test/regress/expected/columnar_temp_tables.out ================================================ [File too large to display: 594 B] ================================================ FILE: src/test/regress/expected/columnar_test_helpers.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/columnar_transactions.out ================================================ [File too large to display: 9.9 KB] ================================================ FILE: src/test/regress/expected/columnar_trigger.out ================================================ [File too large to display: 7.6 KB] ================================================ FILE: src/test/regress/expected/columnar_truncate.out ================================================ [File too large to display: 9.2 KB] ================================================ FILE: src/test/regress/expected/columnar_types_without_comparison.out ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/expected/columnar_update_delete.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/expected/columnar_vacuum.out ================================================ [File too large to display: 10.2 KB] ================================================ FILE: src/test/regress/expected/columnar_vacuum_vs_insert.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/columnar_write_concurrency.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/expected/columnar_write_concurrency_index.out ================================================ [File too large to display: 14.0 KB] ================================================ FILE: src/test/regress/expected/columnar_zstd.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/expected/columnar_zstd_0.out ================================================ [File too large to display: 117 B] ================================================ FILE: src/test/regress/expected/comment_on_database.out ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/expected/comment_on_role.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/connectivity_checks.out ================================================ [File too large to display: 173 B] ================================================ FILE: src/test/regress/expected/coordinator_evaluation.out ================================================ [File too large to display: 19.8 KB] ================================================ FILE: src/test/regress/expected/coordinator_evaluation_modify.out ================================================ [File too large to display: 108.3 KB] ================================================ FILE: src/test/regress/expected/coordinator_evaluation_select.out ================================================ [File too large to display: 87.3 KB] ================================================ FILE: src/test/regress/expected/coordinator_shouldhaveshards.out ================================================ [File too large to display: 86.8 KB] ================================================ FILE: src/test/regress/expected/cpu_priority.out ================================================ [File too large to display: 16.0 KB] ================================================ FILE: src/test/regress/expected/create_citus_local_table_cascade.out ================================================ [File too large to display: 8.8 KB] ================================================ FILE: src/test/regress/expected/create_distributed_table_concurrently.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/create_drop_database_propagation.out ================================================ [File too large to display: 96.3 KB] ================================================ FILE: src/test/regress/expected/create_drop_database_propagation_pg15.out ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/expected/create_drop_database_propagation_pg16.out ================================================ [File too large to display: 836 B] ================================================ FILE: src/test/regress/expected/create_ref_dist_from_citus_local.out ================================================ [File too large to display: 43.8 KB] ================================================ FILE: src/test/regress/expected/create_role_propagation.out ================================================ [File too large to display: 40.4 KB] ================================================ FILE: src/test/regress/expected/create_single_shard_table.out ================================================ [File too large to display: 83.5 KB] ================================================ FILE: src/test/regress/expected/cross_join.out ================================================ [File too large to display: 13.3 KB] ================================================ FILE: src/test/regress/expected/cte_inline.out ================================================ [File too large to display: 82.0 KB] ================================================ FILE: src/test/regress/expected/cte_nested_modification.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/cte_prepared_modify.out ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/expected/cursors.out ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/expected/custom_aggregate_support.out ================================================ [File too large to display: 20.2 KB] ================================================ FILE: src/test/regress/expected/custom_aggregate_support_0.out ================================================ [File too large to display: 9.1 KB] ================================================ FILE: src/test/regress/expected/data_types.out ================================================ [File too large to display: 36.8 KB] ================================================ FILE: src/test/regress/expected/detect_conn_close.out ================================================ [File too large to display: 7.4 KB] ================================================ FILE: src/test/regress/expected/disable_object_propagation.out ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/expected/distributed_collations.out ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/expected/distributed_collations_conflict.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/distributed_domain.out ================================================ [File too large to display: 40.7 KB] ================================================ FILE: src/test/regress/expected/distributed_functions.out ================================================ [File too large to display: 48.1 KB] ================================================ FILE: src/test/regress/expected/distributed_functions_conflict.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/distributed_intermediate_results.out ================================================ [File too large to display: 17.7 KB] ================================================ FILE: src/test/regress/expected/distributed_locks.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/expected/distributed_planning.out ================================================ [File too large to display: 29.9 KB] ================================================ FILE: src/test/regress/expected/distributed_planning_create_load.out ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/expected/distributed_procedure.out ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/expected/distributed_triggers.out ================================================ [File too large to display: 42.0 KB] ================================================ FILE: src/test/regress/expected/distributed_types.out ================================================ [File too large to display: 26.6 KB] ================================================ FILE: src/test/regress/expected/distributed_types_conflict.out ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/expected/distributed_types_xact_add_enum_value.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/dml_recursive.out ================================================ [File too large to display: 16.9 KB] ================================================ FILE: src/test/regress/expected/drop_column_partitioned_table.out ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/expected/drop_database.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/drop_partitioned_table.out ================================================ [File too large to display: 21.7 KB] ================================================ FILE: src/test/regress/expected/dropped_columns_1.out ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/expected/dropped_columns_create_load.out ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/expected/ensure_citus_columnar_not_exists.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/ensure_no_intermediate_data_leak.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/ensure_no_shared_connection_leak.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/escape_extension_name.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/escape_extension_name_0.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/executor_local_failure.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/expression_reference_join.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/failure_add_disable_node.out ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/expected/failure_connection_establishment.out ================================================ [File too large to display: 19.1 KB] ================================================ FILE: src/test/regress/expected/failure_copy_on_hash.out ================================================ [File too large to display: 11.3 KB] ================================================ FILE: src/test/regress/expected/failure_copy_to_reference.out ================================================ [File too large to display: 12.1 KB] ================================================ FILE: src/test/regress/expected/failure_create_database.out ================================================ [File too large to display: 16.4 KB] ================================================ FILE: src/test/regress/expected/failure_create_distributed_table_concurrently.out ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/expected/failure_create_distributed_table_non_empty.out ================================================ [File too large to display: 34.6 KB] ================================================ FILE: src/test/regress/expected/failure_create_index_concurrently.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: src/test/regress/expected/failure_create_reference_table.out ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/expected/failure_create_table.out ================================================ [File too large to display: 19.8 KB] ================================================ FILE: src/test/regress/expected/failure_cte_subquery.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/failure_ddl.out ================================================ [File too large to display: 39.7 KB] ================================================ FILE: src/test/regress/expected/failure_distributed_results.out ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/expected/failure_failover_to_local_execution.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/failure_insert_select_pushdown.out ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/expected/failure_insert_select_repartition.out ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/expected/failure_insert_select_via_coordinator.out ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/expected/failure_multi_dml.out ================================================ [File too large to display: 15.0 KB] ================================================ FILE: src/test/regress/expected/failure_multi_row_insert.out ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/expected/failure_multi_shard_update_delete.out ================================================ [File too large to display: 19.4 KB] ================================================ FILE: src/test/regress/expected/failure_mx_metadata_sync.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/expected/failure_mx_metadata_sync_multi_trans.out ================================================ [File too large to display: 42.0 KB] ================================================ FILE: src/test/regress/expected/failure_non_main_db_2pc.out ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/expected/failure_offline_move_shard_placement.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/failure_on_create_subscription.out ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/expected/failure_online_move_shard_placement.out ================================================ [File too large to display: 17.8 KB] ================================================ FILE: src/test/regress/expected/failure_parallel_connection.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/expected/failure_ref_tables.out ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/expected/failure_replicated_partitions.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/failure_savepoints.out ================================================ [File too large to display: 9.1 KB] ================================================ FILE: src/test/regress/expected/failure_setup.out ================================================ [File too large to display: 541 B] ================================================ FILE: src/test/regress/expected/failure_single_mod.out ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/expected/failure_single_select.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/expected/failure_split_cleanup.out ================================================ [File too large to display: 31.2 KB] ================================================ FILE: src/test/regress/expected/failure_tenant_isolation.out ================================================ [File too large to display: 13.5 KB] ================================================ FILE: src/test/regress/expected/failure_tenant_isolation_nonblocking.out ================================================ [File too large to display: 18.1 KB] ================================================ FILE: src/test/regress/expected/failure_test_helpers.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/failure_truncate.out ================================================ [File too large to display: 34.6 KB] ================================================ FILE: src/test/regress/expected/failure_vacuum.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/expected/fast_path_router_modify.out ================================================ [File too large to display: 23.3 KB] ================================================ FILE: src/test/regress/expected/fkeys_between_local_ref.out ================================================ [File too large to display: 33.1 KB] ================================================ FILE: src/test/regress/expected/follower_single_node.out ================================================ [File too large to display: 13.5 KB] ================================================ FILE: src/test/regress/expected/forcedelegation_functions.out ================================================ [File too large to display: 85.5 KB] ================================================ FILE: src/test/regress/expected/foreign_key_restriction_enforcement.out ================================================ [File too large to display: 73.0 KB] ================================================ FILE: src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/expected/foreign_key_to_reference_table.out ================================================ [File too large to display: 69.5 KB] ================================================ FILE: src/test/regress/expected/foreign_tables_mx.out ================================================ [File too large to display: 19.7 KB] ================================================ FILE: src/test/regress/expected/function_create.out ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/expected/function_propagation.out ================================================ [File too large to display: 63.1 KB] ================================================ FILE: src/test/regress/expected/function_with_case_when.out ================================================ [File too large to display: 781 B] ================================================ FILE: src/test/regress/expected/functions.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/expected/generated_identity.out ================================================ [File too large to display: 25.8 KB] ================================================ FILE: src/test/regress/expected/geqo.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/expected/global_cancel.out ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/expected/grant_on_database_propagation.out ================================================ [File too large to display: 29.0 KB] ================================================ FILE: src/test/regress/expected/grant_on_database_propagation_from_non_maindb.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/grant_on_foreign_server_propagation.out ================================================ [File too large to display: 7.4 KB] ================================================ FILE: src/test/regress/expected/grant_on_function_propagation.out ================================================ [File too large to display: 45.7 KB] ================================================ FILE: src/test/regress/expected/grant_on_schema_propagation.out ================================================ [File too large to display: 18.1 KB] ================================================ FILE: src/test/regress/expected/grant_on_sequence_propagation.out ================================================ [File too large to display: 14.1 KB] ================================================ FILE: src/test/regress/expected/grant_on_table_propagation.out ================================================ [File too large to display: 39.6 KB] ================================================ FILE: src/test/regress/expected/grant_on_table_propagation_0.out ================================================ [File too large to display: 39.6 KB] ================================================ FILE: src/test/regress/expected/grant_role_from_non_maindb.out ================================================ [File too large to display: 12.4 KB] ================================================ FILE: src/test/regress/expected/having_subquery.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/expected/hyperscale_tutorial.out ================================================ [File too large to display: 9.4 KB] ================================================ FILE: src/test/regress/expected/index_create.out ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/expected/insert_select_connection_leak.out ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/expected/insert_select_into_local_table.out ================================================ [File too large to display: 38.8 KB] ================================================ FILE: src/test/regress/expected/insert_select_repartition.out ================================================ [File too large to display: 59.1 KB] ================================================ FILE: src/test/regress/expected/insert_select_single_shard_table.out ================================================ [File too large to display: 63.8 KB] ================================================ FILE: src/test/regress/expected/intermediate_result_pruning.out ================================================ [File too large to display: 73.4 KB] ================================================ FILE: src/test/regress/expected/intermediate_result_pruning_create.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/intermediate_result_pruning_queries_1.out ================================================ [File too large to display: 15.4 KB] ================================================ FILE: src/test/regress/expected/intermediate_result_pruning_queries_2.out ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/expected/intermediate_results.out ================================================ [File too large to display: 26.5 KB] ================================================ FILE: src/test/regress/expected/isolation_acquire_distributed_locks.out ================================================ [File too large to display: 33.4 KB] ================================================ FILE: src/test/regress/expected/isolation_add_coordinator.out ================================================ [File too large to display: 386 B] ================================================ FILE: src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out ================================================ [File too large to display: 18.4 KB] ================================================ FILE: src/test/regress/expected/isolation_add_remove_node.out ================================================ [File too large to display: 17.6 KB] ================================================ FILE: src/test/regress/expected/isolation_append_copy_vs_all.out ================================================ ================================================ FILE: src/test/regress/expected/isolation_blocking_move_multi_shard_commands.out ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/expected/isolation_blocking_move_multi_shard_commands_on_mx.out ================================================ [File too large to display: 8.9 KB] ================================================ FILE: src/test/regress/expected/isolation_blocking_move_single_shard_commands.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/isolation_blocking_move_single_shard_commands_on_mx.out ================================================ [File too large to display: 10.7 KB] ================================================ FILE: src/test/regress/expected/isolation_blocking_shard_split.out ================================================ [File too large to display: 31.7 KB] ================================================ FILE: src/test/regress/expected/isolation_blocking_shard_split_with_fkey_to_reference.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/isolation_cancellation.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/isolation_check_mx.out ================================================ [File too large to display: 399 B] ================================================ FILE: src/test/regress/expected/isolation_citus_dist_activity.out ================================================ [File too large to display: 11.6 KB] ================================================ FILE: src/test/regress/expected/isolation_citus_locks.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/isolation_citus_pause_node.out ================================================ [File too large to display: 9.0 KB] ================================================ FILE: src/test/regress/expected/isolation_citus_pause_node_1.out ================================================ [File too large to display: 9.0 KB] ================================================ FILE: src/test/regress/expected/isolation_citus_schema_distribute_undistribute.out ================================================ [File too large to display: 36.8 KB] ================================================ FILE: src/test/regress/expected/isolation_cluster_management.out ================================================ [File too large to display: 375 B] ================================================ FILE: src/test/regress/expected/isolation_concurrent_dml.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/isolation_concurrent_move_create_table.out ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/expected/isolation_copy_placement_vs_copy_placement.out ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/expected/isolation_copy_placement_vs_modification.out ================================================ [File too large to display: 14.8 KB] ================================================ FILE: src/test/regress/expected/isolation_copy_vs_all_on_mx.out ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/expected/isolation_create_citus_local_table.out ================================================ [File too large to display: 9.3 KB] ================================================ FILE: src/test/regress/expected/isolation_create_distributed_concurrently_after_drop_column.out ================================================ [File too large to display: 21.5 KB] ================================================ FILE: src/test/regress/expected/isolation_create_distributed_table.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/isolation_create_distributed_table_concurrently.out ================================================ [File too large to display: 22.7 KB] ================================================ FILE: src/test/regress/expected/isolation_create_restore_point.out ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/expected/isolation_create_table_vs_add_remove_node.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/isolation_data_migration.out ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/expected/isolation_database_cmd_from_any_node.out ================================================ [File too large to display: 13.1 KB] ================================================ FILE: src/test/regress/expected/isolation_ddl_vs_all.out ================================================ [File too large to display: 36.6 KB] ================================================ FILE: src/test/regress/expected/isolation_delete_vs_all.out ================================================ [File too large to display: 19.6 KB] ================================================ FILE: src/test/regress/expected/isolation_dis2ref_foreign_keys_on_mx.out ================================================ [File too large to display: 21.1 KB] ================================================ FILE: src/test/regress/expected/isolation_distributed_deadlock_detection.out ================================================ [File too large to display: 23.6 KB] ================================================ FILE: src/test/regress/expected/isolation_distributed_transaction_id.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/expected/isolation_drop_alter_index_select_for_update_on_mx.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: src/test/regress/expected/isolation_drop_shards.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/expected/isolation_drop_vs_all.out ================================================ [File too large to display: 18.9 KB] ================================================ FILE: src/test/regress/expected/isolation_dump_global_wait_edges.out ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/expected/isolation_dump_local_wait_edges.out ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/expected/isolation_ensure_dependency_activate_node.out ================================================ [File too large to display: 80.2 KB] ================================================ FILE: src/test/regress/expected/isolation_extension_commands.out ================================================ [File too large to display: 30.7 KB] ================================================ FILE: src/test/regress/expected/isolation_fix_partition_shard_index_names.out ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/expected/isolation_get_all_active_transactions.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/isolation_get_distributed_wait_queries_mx.out ================================================ [File too large to display: 44.0 KB] ================================================ FILE: src/test/regress/expected/isolation_global_pid.out ================================================ [File too large to display: 6.0 KB] ================================================ FILE: src/test/regress/expected/isolation_hash_copy_vs_all.out ================================================ [File too large to display: 39.1 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_select_conflict.out ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_select_repartition.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_select_vs_all.out ================================================ [File too large to display: 77.2 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_select_vs_all_on_mx.out ================================================ [File too large to display: 39.0 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_vs_all.out ================================================ [File too large to display: 49.3 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_vs_all_on_mx.out ================================================ [File too large to display: 30.5 KB] ================================================ FILE: src/test/regress/expected/isolation_insert_vs_vacuum.out ================================================ [File too large to display: 874 B] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_binaryless.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_multi_shard_commands.out ================================================ [File too large to display: 20.8 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_multi_shard_commands_on_mx.out ================================================ [File too large to display: 19.0 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_nonsu_nonbypassrls.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_single_shard_commands.out ================================================ [File too large to display: 16.2 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_single_shard_commands_on_mx.out ================================================ [File too large to display: 23.2 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_skip_fk_validation.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/isolation_logical_replication_with_partitioning.out ================================================ [File too large to display: 14.0 KB] ================================================ FILE: src/test/regress/expected/isolation_master_update_node.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/isolation_master_update_node_0.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/isolation_master_update_node_1.out ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/expected/isolation_max_client_connections.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/isolation_merge.out ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/expected/isolation_merge_replicated.out ================================================ [File too large to display: 778 B] ================================================ FILE: src/test/regress/expected/isolation_metadata_sync_deadlock.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/isolation_metadata_sync_vs_all.out ================================================ [File too large to display: 28.0 KB] ================================================ FILE: src/test/regress/expected/isolation_modify_with_subquery_vs_dml.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/expected/isolation_move_placement_vs_modification.out ================================================ [File too large to display: 17.7 KB] ================================================ FILE: src/test/regress/expected/isolation_move_placement_vs_modification_fk.out ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/expected/isolation_move_placement_vs_move_placement.out ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/expected/isolation_multi_shard_modify_vs_all.out ================================================ [File too large to display: 12.3 KB] ================================================ FILE: src/test/regress/expected/isolation_multiuser_locking.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/expected/isolation_multiuser_locking_0.out ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/expected/isolation_non_blocking_shard_split.out ================================================ [File too large to display: 29.4 KB] ================================================ FILE: src/test/regress/expected/isolation_non_blocking_shard_split_fkey.out ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/expected/isolation_non_blocking_shard_split_with_index_as_replicaIdentity.out ================================================ [File too large to display: 15.7 KB] ================================================ FILE: src/test/regress/expected/isolation_partitioned_copy_vs_all.out ================================================ [File too large to display: 30.3 KB] ================================================ FILE: src/test/regress/expected/isolation_progress_monitoring.out ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/expected/isolation_range_copy_vs_all.out ================================================ [File too large to display: 30.8 KB] ================================================ FILE: src/test/regress/expected/isolation_rebalancer_deferred_drop.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/isolation_ref2ref_foreign_keys.out ================================================ [File too large to display: 32.9 KB] ================================================ FILE: src/test/regress/expected/isolation_ref2ref_foreign_keys_enterprise.out ================================================ [File too large to display: 9.0 KB] ================================================ FILE: src/test/regress/expected/isolation_ref2ref_foreign_keys_on_mx.out ================================================ [File too large to display: 24.3 KB] ================================================ FILE: src/test/regress/expected/isolation_ref_select_for_update_vs_all_on_mx.out ================================================ [File too large to display: 20.1 KB] ================================================ FILE: src/test/regress/expected/isolation_ref_update_delete_upsert_vs_all_on_mx.out ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/expected/isolation_reference_copy_vs_all.out ================================================ [File too large to display: 33.8 KB] ================================================ FILE: src/test/regress/expected/isolation_reference_on_mx.out ================================================ [File too large to display: 22.9 KB] ================================================ FILE: src/test/regress/expected/isolation_reference_table.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/isolation_remove_coordinator.out ================================================ [File too large to display: 386 B] ================================================ FILE: src/test/regress/expected/isolation_replace_wait_function.out ================================================ [File too large to display: 528 B] ================================================ FILE: src/test/regress/expected/isolation_replicate_reference_tables_to_coordinator.out ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/expected/isolation_replicated_dist_on_mx.out ================================================ [File too large to display: 83.5 KB] ================================================ FILE: src/test/regress/expected/isolation_schema_based_sharding.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/expected/isolation_select_for_update.out ================================================ [File too large to display: 8.3 KB] ================================================ FILE: src/test/regress/expected/isolation_select_vs_all.out ================================================ [File too large to display: 103.8 KB] ================================================ FILE: src/test/regress/expected/isolation_select_vs_all_on_mx.out ================================================ [File too large to display: 15.8 KB] ================================================ FILE: src/test/regress/expected/isolation_setup.out ================================================ [File too large to display: 829 B] ================================================ FILE: src/test/regress/expected/isolation_shard_move_vs_start_metadata_sync.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/isolation_shard_rebalancer.out ================================================ [File too large to display: 12.9 KB] ================================================ FILE: src/test/regress/expected/isolation_shard_rebalancer_progress.out ================================================ [File too large to display: 58.5 KB] ================================================ FILE: src/test/regress/expected/isolation_shouldhaveshards.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/isolation_tenant_isolation.out ================================================ [File too large to display: 24.2 KB] ================================================ FILE: src/test/regress/expected/isolation_tenant_isolation_nonblocking.out ================================================ [File too large to display: 33.2 KB] ================================================ FILE: src/test/regress/expected/isolation_tenant_isolation_with_fkey_to_reference.out ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/expected/isolation_transaction_recovery.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/isolation_truncate_vs_all.out ================================================ [File too large to display: 24.0 KB] ================================================ FILE: src/test/regress/expected/isolation_truncate_vs_all_on_mx.out ================================================ [File too large to display: 19.0 KB] ================================================ FILE: src/test/regress/expected/isolation_undistribute_table.out ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/expected/isolation_update_delete_upsert_vs_all_on_mx.out ================================================ [File too large to display: 12.3 KB] ================================================ FILE: src/test/regress/expected/isolation_update_node.out ================================================ [File too large to display: 7.4 KB] ================================================ FILE: src/test/regress/expected/isolation_update_node_lock_writes.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/isolation_update_vs_all.out ================================================ [File too large to display: 48.1 KB] ================================================ FILE: src/test/regress/expected/isolation_upsert_vs_all.out ================================================ [File too large to display: 23.9 KB] ================================================ FILE: src/test/regress/expected/isolation_vacuum_skip_locked.out ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/expected/isolation_validate_vs_insert.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/issue_5099.out ================================================ [File too large to display: 467 B] ================================================ FILE: src/test/regress/expected/issue_5248.out ================================================ [File too large to display: 12.3 KB] ================================================ FILE: src/test/regress/expected/issue_5763.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/issue_6543.out ================================================ [File too large to display: 833 B] ================================================ FILE: src/test/regress/expected/issue_6592.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/expected/issue_6758.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/issue_7477.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/issue_7891.out ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/expected/issue_8243.out ================================================ [File too large to display: 16.3 KB] ================================================ FILE: src/test/regress/expected/join_pushdown.out ================================================ [File too large to display: 13.6 KB] ================================================ FILE: src/test/regress/expected/limit_intermediate_size.out ================================================ [File too large to display: 11.6 KB] ================================================ FILE: src/test/regress/expected/local_dist_join.out ================================================ [File too large to display: 20.2 KB] ================================================ FILE: src/test/regress/expected/local_dist_join_load.out ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/expected/local_dist_join_mixed.out ================================================ [File too large to display: 130.1 KB] ================================================ FILE: src/test/regress/expected/local_dist_join_modifications.out ================================================ [File too large to display: 26.3 KB] ================================================ FILE: src/test/regress/expected/local_execution_local_plan.out ================================================ [File too large to display: 17.2 KB] ================================================ FILE: src/test/regress/expected/local_shard_copy.out ================================================ [File too large to display: 21.2 KB] ================================================ FILE: src/test/regress/expected/local_shard_execution.out ================================================ [File too large to display: 183.4 KB] ================================================ FILE: src/test/regress/expected/local_shard_execution_dropped_column.out ================================================ [File too large to display: 30.8 KB] ================================================ FILE: src/test/regress/expected/local_shard_execution_replicated.out ================================================ [File too large to display: 173.7 KB] ================================================ FILE: src/test/regress/expected/local_shard_utility_command_execution.out ================================================ [File too large to display: 63.7 KB] ================================================ FILE: src/test/regress/expected/local_table_join.out ================================================ [File too large to display: 148.2 KB] ================================================ FILE: src/test/regress/expected/locally_execute_intermediate_results.out ================================================ [File too large to display: 159.6 KB] ================================================ FILE: src/test/regress/expected/logical_rep_consistency.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/expected/logical_replication.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/expected/materialized_view.out ================================================ [File too large to display: 15.1 KB] ================================================ FILE: src/test/regress/expected/merge.out ================================================ [File too large to display: 181.9 KB] ================================================ FILE: src/test/regress/expected/merge_arbitrary.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/expected/merge_arbitrary_create.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/merge_partition_tables.out ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/expected/merge_repartition1.out ================================================ [File too large to display: 50.4 KB] ================================================ FILE: src/test/regress/expected/merge_repartition2.out ================================================ [File too large to display: 13.9 KB] ================================================ FILE: src/test/regress/expected/merge_schema_sharding.out ================================================ [File too large to display: 23.0 KB] ================================================ FILE: src/test/regress/expected/merge_unsupported.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/expected/merge_unsupported_0.out ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/expected/merge_vcore.out ================================================ [File too large to display: 16.9 KB] ================================================ FILE: src/test/regress/expected/metadata_sync_from_non_maindb.out ================================================ [File too large to display: 19.5 KB] ================================================ FILE: src/test/regress/expected/metadata_sync_helpers.out ================================================ [File too large to display: 58.4 KB] ================================================ FILE: src/test/regress/expected/minimal_cluster_management.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/mixed_relkind_tests.out ================================================ [File too large to display: 30.6 KB] ================================================ FILE: src/test/regress/expected/modification_correctness.out ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/expected/multi_add_node_from_backup.out ================================================ [File too large to display: 22.6 KB] ================================================ FILE: src/test/regress/expected/multi_add_node_from_backup_negative.out ================================================ [File too large to display: 19.4 KB] ================================================ FILE: src/test/regress/expected/multi_add_node_from_backup_sync_replica.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/expected/multi_agg_approximate_distinct.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/multi_agg_approximate_distinct_0.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/expected/multi_agg_distinct.out ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/expected/multi_agg_type_conversion.out ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/expected/multi_alias.out ================================================ [File too large to display: 9.7 KB] ================================================ FILE: src/test/regress/expected/multi_alter_table_add_constraints.out ================================================ [File too large to display: 33.6 KB] ================================================ FILE: src/test/regress/expected/multi_alter_table_add_constraints_without_name.out ================================================ [File too large to display: 59.9 KB] ================================================ FILE: src/test/regress/expected/multi_alter_table_add_foreign_key_without_name.out ================================================ [File too large to display: 29.1 KB] ================================================ FILE: src/test/regress/expected/multi_alter_table_row_level_security.out ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/expected/multi_alter_table_row_level_security_escape.out ================================================ [File too large to display: 743 B] ================================================ FILE: src/test/regress/expected/multi_alter_table_statements.out ================================================ [File too large to display: 64.0 KB] ================================================ FILE: src/test/regress/expected/multi_array_agg.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/expected/multi_average_expression.out ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/expected/multi_basic_queries.out ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/expected/multi_behavioral_analytics_basics.out ================================================ [File too large to display: 20.5 KB] ================================================ FILE: src/test/regress/expected/multi_behavioral_analytics_create_table.out ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/expected/multi_behavioral_analytics_create_table_superuser.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/multi_behavioral_analytics_single_shard_queries.out ================================================ [File too large to display: 17.9 KB] ================================================ FILE: src/test/regress/expected/multi_cache_invalidation.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/multi_citus_tools.out ================================================ [File too large to display: 28.0 KB] ================================================ FILE: src/test/regress/expected/multi_cluster_management.out ================================================ [File too large to display: 52.1 KB] ================================================ FILE: src/test/regress/expected/multi_colocated_shard_rebalance.out ================================================ [File too large to display: 37.5 KB] ================================================ FILE: src/test/regress/expected/multi_colocation_utils.out ================================================ [File too large to display: 61.8 KB] ================================================ FILE: src/test/regress/expected/multi_complex_count_distinct.out ================================================ [File too large to display: 39.1 KB] ================================================ FILE: src/test/regress/expected/multi_complex_expressions.out ================================================ [File too large to display: 17.3 KB] ================================================ FILE: src/test/regress/expected/multi_copy.out ================================================ [File too large to display: 39.3 KB] ================================================ FILE: src/test/regress/expected/multi_count_type_conversion.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/multi_create_fdw.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/expected/multi_create_role_dependency.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/multi_create_shards.out ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/expected/multi_create_table.out ================================================ [File too large to display: 16.1 KB] ================================================ FILE: src/test/regress/expected/multi_create_table_constraints.out ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/expected/multi_create_table_superuser.out ================================================ [File too large to display: 24.3 KB] ================================================ FILE: src/test/regress/expected/multi_create_users.out ================================================ [File too large to display: 409 B] ================================================ FILE: src/test/regress/expected/multi_cross_shard.out ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/expected/multi_data_types.out ================================================ [File too large to display: 10.5 KB] ================================================ FILE: src/test/regress/expected/multi_deparse_function.out ================================================ [File too large to display: 28.8 KB] ================================================ FILE: src/test/regress/expected/multi_deparse_procedure.out ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/expected/multi_deparse_shard_query.out ================================================ [File too large to display: 18.2 KB] ================================================ FILE: src/test/regress/expected/multi_distributed_transaction_id.out ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/expected/multi_distribution_metadata.out ================================================ [File too large to display: 20.4 KB] ================================================ FILE: src/test/regress/expected/multi_drop_extension.out ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/expected/multi_dropped_column_aliases.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/expected/multi_explain.out ================================================ [File too large to display: 143.5 KB] ================================================ FILE: src/test/regress/expected/multi_explain_0.out ================================================ [File too large to display: 143.0 KB] ================================================ FILE: src/test/regress/expected/multi_extension.out ================================================ [File too large to display: 144.9 KB] ================================================ FILE: src/test/regress/expected/multi_fix_partition_shard_index_names.out ================================================ [File too large to display: 46.6 KB] ================================================ FILE: src/test/regress/expected/multi_follower_configure_followers.out ================================================ [File too large to display: 941 B] ================================================ FILE: src/test/regress/expected/multi_follower_dml.out ================================================ [File too large to display: 18.4 KB] ================================================ FILE: src/test/regress/expected/multi_follower_sanity_check.out ================================================ [File too large to display: 442 B] ================================================ FILE: src/test/regress/expected/multi_follower_select_statements.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/multi_foreign_key.out ================================================ [File too large to display: 62.2 KB] ================================================ FILE: src/test/regress/expected/multi_foreign_key_relation_graph.out ================================================ [File too large to display: 30.0 KB] ================================================ FILE: src/test/regress/expected/multi_function_evaluation.out ================================================ [File too large to display: 10.0 KB] ================================================ FILE: src/test/regress/expected/multi_function_in_join.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/multi_generate_ddl_commands.out ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/expected/multi_hash_pruning.out ================================================ [File too large to display: 50.7 KB] ================================================ FILE: src/test/regress/expected/multi_having_pushdown.out ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/expected/multi_index_statements.out ================================================ [File too large to display: 37.5 KB] ================================================ FILE: src/test/regress/expected/multi_insert_select.out ================================================ [File too large to display: 223.3 KB] ================================================ FILE: src/test/regress/expected/multi_insert_select_behavioral_analytics_create_table.out ================================================ ================================================ FILE: src/test/regress/expected/multi_insert_select_conflict.out ================================================ [File too large to display: 27.2 KB] ================================================ FILE: src/test/regress/expected/multi_insert_select_non_pushable_queries.out ================================================ [File too large to display: 38.8 KB] ================================================ FILE: src/test/regress/expected/multi_insert_select_window.out ================================================ [File too large to display: 25.1 KB] ================================================ FILE: src/test/regress/expected/multi_join_order_additional.out ================================================ [File too large to display: 7.7 KB] ================================================ FILE: src/test/regress/expected/multi_join_order_tpch_repartition.out ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/expected/multi_join_order_tpch_small.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/multi_join_pruning.out ================================================ [File too large to display: 6.4 KB] ================================================ FILE: src/test/regress/expected/multi_json_agg.out ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/expected/multi_json_object_agg.out ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/expected/multi_jsonb_agg.out ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/expected/multi_jsonb_object_agg.out ================================================ [File too large to display: 11.2 KB] ================================================ FILE: src/test/regress/expected/multi_large_shardid.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/multi_level_recursive_queries.out ================================================ [File too large to display: 24.9 KB] ================================================ FILE: src/test/regress/expected/multi_limit_clause.out ================================================ [File too large to display: 20.9 KB] ================================================ FILE: src/test/regress/expected/multi_limit_clause_approximate.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/multi_load_data.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/expected/multi_load_data_superuser.out ================================================ [File too large to display: 796 B] ================================================ FILE: src/test/regress/expected/multi_load_more_data.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/multi_master_protocol.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/multi_metadata_access.out ================================================ [File too large to display: 829 B] ================================================ FILE: src/test/regress/expected/multi_metadata_attributes.out ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/expected/multi_metadata_sync.out ================================================ [File too large to display: 141.4 KB] ================================================ FILE: src/test/regress/expected/multi_metadata_sync_domain.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/expected/multi_modifications.out ================================================ [File too large to display: 69.5 KB] ================================================ FILE: src/test/regress/expected/multi_modifying_xacts.out ================================================ [File too large to display: 53.3 KB] ================================================ FILE: src/test/regress/expected/multi_move_mx.out ================================================ [File too large to display: 9.3 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser.out ================================================ [File too large to display: 19.2 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_auth.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_basic_queries.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_copy.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_grant.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_load_data.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_master_protocol.out ================================================ [File too large to display: 22.0 KB] ================================================ FILE: src/test/regress/expected/multi_multiuser_master_protocol_0.out ================================================ [File too large to display: 21.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_add_coordinator.out ================================================ [File too large to display: 12.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_alter_distributed_table.out ================================================ [File too large to display: 20.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_call.out ================================================ [File too large to display: 25.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_call_0.out ================================================ [File too large to display: 25.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_copy_data.out ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_create_table.out ================================================ [File too large to display: 93.1 KB] ================================================ FILE: src/test/regress/expected/multi_mx_ddl.out ================================================ [File too large to display: 16.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_explain.out ================================================ [File too large to display: 40.9 KB] ================================================ FILE: src/test/regress/expected/multi_mx_explain_0.out ================================================ [File too large to display: 40.0 KB] ================================================ FILE: src/test/regress/expected/multi_mx_function_call_delegation.out ================================================ [File too large to display: 34.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_function_call_delegation_0.out ================================================ [File too large to display: 34.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_function_table_reference.out ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/expected/multi_mx_hide_shard_names.out ================================================ [File too large to display: 22.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_hide_shard_names_0.out ================================================ [File too large to display: 22.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_insert_select_repartition.out ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_metadata.out ================================================ [File too large to display: 13.4 KB] ================================================ FILE: src/test/regress/expected/multi_mx_modifications.out ================================================ [File too large to display: 18.4 KB] ================================================ FILE: src/test/regress/expected/multi_mx_modifications_to_reference_tables.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_modifying_xacts.out ================================================ [File too large to display: 15.0 KB] ================================================ FILE: src/test/regress/expected/multi_mx_node_metadata.out ================================================ [File too large to display: 29.2 KB] ================================================ FILE: src/test/regress/expected/multi_mx_partitioning.out ================================================ [File too large to display: 13.3 KB] ================================================ FILE: src/test/regress/expected/multi_mx_reference_table.out ================================================ [File too large to display: 24.1 KB] ================================================ FILE: src/test/regress/expected/multi_mx_repartition_join_w1.out ================================================ [File too large to display: 513 B] ================================================ FILE: src/test/regress/expected/multi_mx_repartition_join_w2.out ================================================ [File too large to display: 516 B] ================================================ FILE: src/test/regress/expected/multi_mx_repartition_udt_prepare.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_repartition_udt_w1.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/multi_mx_repartition_udt_w2.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/multi_mx_router_planner.out ================================================ [File too large to display: 55.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_schema_support.out ================================================ [File too large to display: 25.6 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query1.out ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query10.out ================================================ [File too large to display: 15.1 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query12.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query14.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query19.out ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query3.out ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query6.out ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query7.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/multi_mx_tpch_query7_nested.out ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/expected/multi_mx_transaction_recovery.out ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/expected/multi_mx_truncate_from_worker.out ================================================ [File too large to display: 11.7 KB] ================================================ FILE: src/test/regress/expected/multi_name_lengths.out ================================================ [File too large to display: 25.1 KB] ================================================ FILE: src/test/regress/expected/multi_name_resolution.out ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/expected/multi_null_minmax_value_pruning.out ================================================ [File too large to display: 15.8 KB] ================================================ FILE: src/test/regress/expected/multi_orderby_limit_pushdown.out ================================================ [File too large to display: 11.7 KB] ================================================ FILE: src/test/regress/expected/multi_outer_join.out ================================================ [File too large to display: 28.7 KB] ================================================ FILE: src/test/regress/expected/multi_outer_join_columns.out ================================================ [File too large to display: 21.6 KB] ================================================ FILE: src/test/regress/expected/multi_outer_join_reference.out ================================================ [File too large to display: 29.9 KB] ================================================ FILE: src/test/regress/expected/multi_partition_pruning.out ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/expected/multi_partitioning.out ================================================ [File too large to display: 275.4 KB] ================================================ FILE: src/test/regress/expected/multi_partitioning_utils.out ================================================ [File too large to display: 18.7 KB] ================================================ FILE: src/test/regress/expected/multi_poolinfo_usage.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/expected/multi_prepare_plsql.out ================================================ [File too large to display: 32.7 KB] ================================================ FILE: src/test/regress/expected/multi_prepare_sql.out ================================================ [File too large to display: 32.3 KB] ================================================ FILE: src/test/regress/expected/multi_prune_shard_list.out ================================================ [File too large to display: 21.3 KB] ================================================ FILE: src/test/regress/expected/multi_query_directory_cleanup.out ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/expected/multi_read_from_secondaries.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/expected/multi_real_time_transaction.out ================================================ [File too large to display: 20.0 KB] ================================================ FILE: src/test/regress/expected/multi_reference_table.out ================================================ [File too large to display: 50.5 KB] ================================================ FILE: src/test/regress/expected/multi_remove_node_reference_table.out ================================================ [File too large to display: 31.8 KB] ================================================ FILE: src/test/regress/expected/multi_repartition_join_planning.out ================================================ [File too large to display: 23.0 KB] ================================================ FILE: src/test/regress/expected/multi_repartition_join_pruning.out ================================================ [File too large to display: 32.2 KB] ================================================ FILE: src/test/regress/expected/multi_repartition_join_ref.out ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/expected/multi_repartition_join_task_assignment.out ================================================ [File too large to display: 11.4 KB] ================================================ FILE: src/test/regress/expected/multi_repartition_udt.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/expected/multi_repartitioned_subquery_udf.out ================================================ [File too large to display: 772 B] ================================================ FILE: src/test/regress/expected/multi_replicate_reference_table.out ================================================ [File too large to display: 41.7 KB] ================================================ FILE: src/test/regress/expected/multi_rls_join_distribution_key.out ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/expected/multi_router_planner.out ================================================ [File too large to display: 109.6 KB] ================================================ FILE: src/test/regress/expected/multi_router_planner_fast_path.out ================================================ [File too large to display: 79.0 KB] ================================================ FILE: src/test/regress/expected/multi_row_insert.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/expected/multi_row_router_insert.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/expected/multi_schema_support.out ================================================ [File too large to display: 61.4 KB] ================================================ FILE: src/test/regress/expected/multi_select_distinct.out ================================================ [File too large to display: 56.8 KB] ================================================ FILE: src/test/regress/expected/multi_select_for_update.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/expected/multi_sequence_default.out ================================================ [File too large to display: 39.6 KB] ================================================ FILE: src/test/regress/expected/multi_shard_update_delete.out ================================================ [File too large to display: 29.8 KB] ================================================ FILE: src/test/regress/expected/multi_simple_queries.out ================================================ [File too large to display: 35.4 KB] ================================================ FILE: src/test/regress/expected/multi_single_relation_subquery.out ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/expected/multi_size_queries.out ================================================ [File too large to display: 11.0 KB] ================================================ FILE: src/test/regress/expected/multi_sql_function.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/multi_sql_function_0.out ================================================ [File too large to display: 10.9 KB] ================================================ FILE: src/test/regress/expected/multi_subquery.out ================================================ [File too large to display: 38.3 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_behavioral_analytics.out ================================================ [File too large to display: 63.6 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_complex_queries.out ================================================ [File too large to display: 105.1 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_complex_reference_clause.out ================================================ [File too large to display: 91.6 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_in_where_clause.out ================================================ [File too large to display: 16.7 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_in_where_reference_clause.out ================================================ [File too large to display: 15.5 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_misc.out ================================================ [File too large to display: 11.6 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_misc_0.out ================================================ [File too large to display: 11.7 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_union.out ================================================ [File too large to display: 35.3 KB] ================================================ FILE: src/test/regress/expected/multi_subquery_window_functions.out ================================================ [File too large to display: 35.7 KB] ================================================ FILE: src/test/regress/expected/multi_subtransactions.out ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/expected/multi_table_ddl.out ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/expected/multi_task_assignment_policy.out ================================================ [File too large to display: 13.9 KB] ================================================ FILE: src/test/regress/expected/multi_task_string_size.out ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/expected/multi_tenant_isolation.out ================================================ [File too large to display: 52.5 KB] ================================================ FILE: src/test/regress/expected/multi_tenant_isolation_nonblocking.out ================================================ [File too large to display: 55.5 KB] ================================================ FILE: src/test/regress/expected/multi_test_catalog_views.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/expected/multi_test_helpers.out ================================================ [File too large to display: 29.0 KB] ================================================ FILE: src/test/regress/expected/multi_test_helpers_superuser.out ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/expected/multi_tpch_query1.out ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/expected/multi_tpch_query10.out ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/expected/multi_tpch_query12.out ================================================ [File too large to display: 872 B] ================================================ FILE: src/test/regress/expected/multi_tpch_query14.out ================================================ [File too large to display: 525 B] ================================================ FILE: src/test/regress/expected/multi_tpch_query19.out ================================================ [File too large to display: 1006 B] ================================================ FILE: src/test/regress/expected/multi_tpch_query3.out ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/expected/multi_tpch_query6.out ================================================ [File too large to display: 421 B] ================================================ FILE: src/test/regress/expected/multi_tpch_query7.out ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/expected/multi_tpch_query7_nested.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/multi_transaction_recovery.out ================================================ [File too large to display: 15.1 KB] ================================================ FILE: src/test/regress/expected/multi_transaction_recovery_multiple_databases.out ================================================ [File too large to display: 11.1 KB] ================================================ FILE: src/test/regress/expected/multi_transactional_drop_shards.out ================================================ [File too large to display: 23.0 KB] ================================================ FILE: src/test/regress/expected/multi_truncate.out ================================================ [File too large to display: 18.4 KB] ================================================ FILE: src/test/regress/expected/multi_unsupported_worker_operations.out ================================================ [File too large to display: 11.3 KB] ================================================ FILE: src/test/regress/expected/multi_update_select.out ================================================ [File too large to display: 25.6 KB] ================================================ FILE: src/test/regress/expected/multi_upsert.out ================================================ [File too large to display: 10.7 KB] ================================================ FILE: src/test/regress/expected/multi_utilities.out ================================================ [File too large to display: 23.7 KB] ================================================ FILE: src/test/regress/expected/multi_utility_statements.out ================================================ [File too large to display: 17.6 KB] ================================================ FILE: src/test/regress/expected/multi_utility_warnings.out ================================================ [File too large to display: 410 B] ================================================ FILE: src/test/regress/expected/multi_view.out ================================================ [File too large to display: 55.9 KB] ================================================ FILE: src/test/regress/expected/multi_working_columns.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/expected/mx_coordinator_shouldhaveshards.out ================================================ [File too large to display: 22.2 KB] ================================================ FILE: src/test/regress/expected/mx_foreign_key_to_reference_table.out ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/expected/mx_regular_user.out ================================================ [File too large to display: 28.2 KB] ================================================ FILE: src/test/regress/expected/nested_execution.out ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/expected/nested_execution_create.out ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/expected/node_conninfo_reload.out ================================================ [File too large to display: 18.0 KB] ================================================ FILE: src/test/regress/expected/non_colocated_leaf_subquery_joins.out ================================================ [File too large to display: 12.4 KB] ================================================ FILE: src/test/regress/expected/non_colocated_subquery_joins.out ================================================ [File too large to display: 87.5 KB] ================================================ FILE: src/test/regress/expected/non_super_user_cdc_library_path.out ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/expected/non_super_user_object_metadata.out ================================================ [File too large to display: 26.2 KB] ================================================ FILE: src/test/regress/expected/null_parameters.out ================================================ [File too large to display: 29.6 KB] ================================================ FILE: src/test/regress/expected/object_propagation_debug.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/other_databases.out ================================================ [File too large to display: 14.5 KB] ================================================ FILE: src/test/regress/expected/partition_wise_join.out ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/expected/partition_wise_join_0.out ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/expected/partitioned_indexes_create.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/partitioned_intermediate_results.out ================================================ [File too large to display: 19.4 KB] ================================================ FILE: src/test/regress/expected/partitioning_issue_3970.out ================================================ [File too large to display: 10.1 KB] ================================================ FILE: src/test/regress/expected/pg12.out ================================================ [File too large to display: 21.4 KB] ================================================ FILE: src/test/regress/expected/pg13.out ================================================ [File too large to display: 8.9 KB] ================================================ FILE: src/test/regress/expected/pg13_propagate_statistics.out ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/expected/pg13_with_ties.out ================================================ [File too large to display: 8.2 KB] ================================================ FILE: src/test/regress/expected/pg14.out ================================================ [File too large to display: 56.2 KB] ================================================ FILE: src/test/regress/expected/pg15.out ================================================ [File too large to display: 62.3 KB] ================================================ FILE: src/test/regress/expected/pg15_jsonpath.out ================================================ [File too large to display: 29.9 KB] ================================================ FILE: src/test/regress/expected/pg16.out ================================================ [File too large to display: 47.8 KB] ================================================ FILE: src/test/regress/expected/pg17.out ================================================ [File too large to display: 146.0 KB] ================================================ FILE: src/test/regress/expected/pg17_0.out ================================================ [File too large to display: 22.7 KB] ================================================ FILE: src/test/regress/expected/pg17_json.out ================================================ [File too large to display: 26.9 KB] ================================================ FILE: src/test/regress/expected/pg17_json_0.out ================================================ [File too large to display: 751 B] ================================================ FILE: src/test/regress/expected/pg18.out ================================================ [File too large to display: 114.5 KB] ================================================ FILE: src/test/regress/expected/pg18_0.out ================================================ [File too large to display: 710 B] ================================================ FILE: src/test/regress/expected/pg_dump.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/pgmerge.out ================================================ [File too large to display: 66.0 KB] ================================================ FILE: src/test/regress/expected/postgres.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/prepared_statements_1.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/prepared_statements_2.out ================================================ [File too large to display: 17.3 KB] ================================================ FILE: src/test/regress/expected/prepared_statements_3.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/expected/prepared_statements_4.out ================================================ [File too large to display: 9.3 KB] ================================================ FILE: src/test/regress/expected/prepared_statements_create_load.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/expected/propagate_extension_commands.out ================================================ [File too large to display: 24.8 KB] ================================================ FILE: src/test/regress/expected/propagate_foreign_servers.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/expected/propagate_set_commands.out ================================================ [File too large to display: 11.0 KB] ================================================ FILE: src/test/regress/expected/propagate_statistics.out ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/expected/publication.out ================================================ [File too large to display: 29.6 KB] ================================================ FILE: src/test/regress/expected/query_single_shard_table.out ================================================ [File too large to display: 171.8 KB] ================================================ FILE: src/test/regress/expected/reassign_owned.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/expected/recurring_join_pushdown.out ================================================ [File too large to display: 51.1 KB] ================================================ FILE: src/test/regress/expected/recurring_outer_join.out ================================================ [File too large to display: 128.0 KB] ================================================ FILE: src/test/regress/expected/recursive_dml_queries_mx.out ================================================ [File too large to display: 6.4 KB] ================================================ FILE: src/test/regress/expected/recursive_dml_with_different_planners_executors.out ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/expected/recursive_relation_planning_restriction_pushdown.out ================================================ [File too large to display: 38.0 KB] ================================================ FILE: src/test/regress/expected/recursive_view_local_table.out ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/expected/ref_citus_local_fkeys.out ================================================ [File too large to display: 19.5 KB] ================================================ FILE: src/test/regress/expected/relation_access_tracking.out ================================================ [File too large to display: 42.3 KB] ================================================ FILE: src/test/regress/expected/relation_access_tracking_single_node.out ================================================ [File too large to display: 45.5 KB] ================================================ FILE: src/test/regress/expected/remove_coordinator.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/remove_coordinator_from_metadata.out ================================================ [File too large to display: 154 B] ================================================ FILE: src/test/regress/expected/remove_non_default_nodes.out ================================================ [File too large to display: 580 B] ================================================ FILE: src/test/regress/expected/rename_public_to_citus_schema_and_recreate.out ================================================ [File too large to display: 66 B] ================================================ FILE: src/test/regress/expected/replicate_reference_tables_to_coordinator.out ================================================ [File too large to display: 23.1 KB] ================================================ FILE: src/test/regress/expected/replicated_partitioned_table.out ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/expected/replicated_table_disable_node.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/expected/resync_metadata_with_sequences.out ================================================ [File too large to display: 12.6 KB] ================================================ FILE: src/test/regress/expected/role_command_from_any_node.out ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/expected/role_operations_from_non_maindb.out ================================================ [File too large to display: 7.6 KB] ================================================ FILE: src/test/regress/expected/row_types.out ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/expected/run_command_on_all_nodes.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/expected/schema_based_sharding.out ================================================ [File too large to display: 69.6 KB] ================================================ FILE: src/test/regress/expected/schemas.out ================================================ [File too large to display: 370 B] ================================================ FILE: src/test/regress/expected/schemas_create.out ================================================ [File too large to display: 600 B] ================================================ FILE: src/test/regress/expected/seclabel.out ================================================ [File too large to display: 40.4 KB] ================================================ FILE: src/test/regress/expected/seclabel_non_maindb.out ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/expected/sequences.out ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/expected/sequences_create.out ================================================ [File too large to display: 844 B] ================================================ FILE: src/test/regress/expected/sequences_owned_by.out ================================================ [File too large to display: 14.6 KB] ================================================ FILE: src/test/regress/expected/sequences_with_different_types.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/expected/sequential_modifications.out ================================================ [File too large to display: 24.4 KB] ================================================ FILE: src/test/regress/expected/set_operation_and_local_tables.out ================================================ [File too large to display: 27.0 KB] ================================================ FILE: src/test/regress/expected/set_operations.out ================================================ [File too large to display: 76.6 KB] ================================================ FILE: src/test/regress/expected/set_role_in_transaction.out ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/expected/shard_move_constraints.out ================================================ [File too large to display: 26.6 KB] ================================================ FILE: src/test/regress/expected/shard_move_constraints_blocking.out ================================================ [File too large to display: 25.0 KB] ================================================ FILE: src/test/regress/expected/shard_move_deferred_delete.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/shard_rebalancer.out ================================================ [File too large to display: 121.6 KB] ================================================ FILE: src/test/regress/expected/shard_rebalancer_unit.out ================================================ [File too large to display: 43.6 KB] ================================================ FILE: src/test/regress/expected/shared_connection_stats.out ================================================ [File too large to display: 23.7 KB] ================================================ FILE: src/test/regress/expected/shared_connection_waits.out ================================================ [File too large to display: 846 B] ================================================ FILE: src/test/regress/expected/single_hash_repartition_join.out ================================================ [File too large to display: 30.4 KB] ================================================ FILE: src/test/regress/expected/single_node.out ================================================ [File too large to display: 128.5 KB] ================================================ FILE: src/test/regress/expected/single_node_enterprise.out ================================================ [File too large to display: 19.6 KB] ================================================ FILE: src/test/regress/expected/single_node_truncate.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/expected/single_shard_table_prep.out ================================================ [File too large to display: 732 B] ================================================ FILE: src/test/regress/expected/single_shard_table_udfs.out ================================================ [File too large to display: 50.1 KB] ================================================ FILE: src/test/regress/expected/split_shard.out ================================================ [File too large to display: 31.0 KB] ================================================ FILE: src/test/regress/expected/sql_procedure.out ================================================ [File too large to display: 6.4 KB] ================================================ FILE: src/test/regress/expected/sqlancer_failures.out ================================================ [File too large to display: 15.8 KB] ================================================ FILE: src/test/regress/expected/sqlsmith_failures.out ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/expected/ssl_by_default.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/start_stop_metadata_sync.out ================================================ [File too large to display: 21.7 KB] ================================================ FILE: src/test/regress/expected/stat_counters.out ================================================ [File too large to display: 46.4 KB] ================================================ FILE: src/test/regress/expected/stat_statements.out ================================================ [File too large to display: 24.2 KB] ================================================ FILE: src/test/regress/expected/statement_cancel_error_message.out ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/expected/subqueries_deep.out ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/expected/subqueries_not_supported.out ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/expected/subquery_and_cte.out ================================================ [File too large to display: 45.0 KB] ================================================ FILE: src/test/regress/expected/subquery_append.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/expected/subquery_basics.out ================================================ [File too large to display: 28.6 KB] ================================================ FILE: src/test/regress/expected/subquery_complex_target_list.out ================================================ [File too large to display: 31.4 KB] ================================================ FILE: src/test/regress/expected/subquery_executors.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/expected/subquery_in_targetlist.out ================================================ [File too large to display: 21.6 KB] ================================================ FILE: src/test/regress/expected/subquery_in_where.out ================================================ [File too large to display: 55.0 KB] ================================================ FILE: src/test/regress/expected/subquery_local_tables.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: src/test/regress/expected/subquery_partitioning.out ================================================ [File too large to display: 16.5 KB] ================================================ FILE: src/test/regress/expected/subquery_prepared_statements.out ================================================ [File too large to display: 16.0 KB] ================================================ FILE: src/test/regress/expected/subquery_view.out ================================================ [File too large to display: 37.4 KB] ================================================ FILE: src/test/regress/expected/subscripting_op.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/expected/system_queries.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/tableam.out ================================================ [File too large to display: 10.3 KB] ================================================ FILE: src/test/regress/expected/tablespace.out ================================================ [File too large to display: 342 B] ================================================ FILE: src/test/regress/expected/task_tracker_assign_task.out ================================================ ================================================ FILE: src/test/regress/expected/task_tracker_cleanup_job.out ================================================ ================================================ FILE: src/test/regress/expected/task_tracker_create_table.out ================================================ ================================================ FILE: src/test/regress/expected/task_tracker_partition_task.out ================================================ ================================================ FILE: src/test/regress/expected/tdigest_aggregate_support.out ================================================ [File too large to display: 33.4 KB] ================================================ FILE: src/test/regress/expected/tdigest_aggregate_support_0.out ================================================ [File too large to display: 12.6 KB] ================================================ FILE: src/test/regress/expected/tdigest_aggregate_support_1.out ================================================ [File too large to display: 33.4 KB] ================================================ FILE: src/test/regress/expected/text_search.out ================================================ [File too large to display: 34.4 KB] ================================================ FILE: src/test/regress/expected/undistribute_table.out ================================================ [File too large to display: 18.3 KB] ================================================ FILE: src/test/regress/expected/undistribute_table_cascade.out ================================================ [File too large to display: 25.8 KB] ================================================ FILE: src/test/regress/expected/undistribute_table_cascade_mx.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/expected/union_pushdown.out ================================================ [File too large to display: 75.8 KB] ================================================ FILE: src/test/regress/expected/unsupported_lateral_subqueries.out ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/expected/update_colocation_mx.out ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/expected/upgrade_autoconverted_after.out ================================================ [File too large to display: 457 B] ================================================ FILE: src/test/regress/expected/upgrade_autoconverted_before.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/upgrade_basic_after.out ================================================ [File too large to display: 8.2 KB] ================================================ FILE: src/test/regress/expected/upgrade_basic_after_non_mixed.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/upgrade_basic_before.out ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/expected/upgrade_basic_before_non_mixed.out ================================================ ================================================ FILE: src/test/regress/expected/upgrade_citus_finish_citus_upgrade.out ================================================ [File too large to display: 973 B] ================================================ FILE: src/test/regress/expected/upgrade_citus_locks.out ================================================ [File too large to display: 626 B] ================================================ FILE: src/test/regress/expected/upgrade_citus_stat_activity.out ================================================ [File too large to display: 674 B] ================================================ FILE: src/test/regress/expected/upgrade_columnar_after.out ================================================ [File too large to display: 28.5 KB] ================================================ FILE: src/test/regress/expected/upgrade_columnar_before.out ================================================ [File too large to display: 45.2 KB] ================================================ FILE: src/test/regress/expected/upgrade_distributed_function_after.out ================================================ [File too large to display: 791 B] ================================================ FILE: src/test/regress/expected/upgrade_distributed_function_before.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/upgrade_distributed_triggers_after.out ================================================ [File too large to display: 10.5 KB] ================================================ FILE: src/test/regress/expected/upgrade_distributed_triggers_after_0.out ================================================ [File too large to display: 512 B] ================================================ FILE: src/test/regress/expected/upgrade_distributed_triggers_before.out ================================================ [File too large to display: 9.7 KB] ================================================ FILE: src/test/regress/expected/upgrade_distributed_triggers_before_0.out ================================================ [File too large to display: 386 B] ================================================ FILE: src/test/regress/expected/upgrade_list_citus_objects.out ================================================ [File too large to display: 20.9 KB] ================================================ FILE: src/test/regress/expected/upgrade_pg_dist_cleanup_after.out ================================================ [File too large to display: 540 B] ================================================ FILE: src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/upgrade_pg_dist_cleanup_before.out ================================================ [File too large to display: 541 B] ================================================ FILE: src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/upgrade_post_11_after.out ================================================ [File too large to display: 7.7 KB] ================================================ FILE: src/test/regress/expected/upgrade_post_11_after_0.out ================================================ [File too large to display: 436 B] ================================================ FILE: src/test/regress/expected/upgrade_post_11_before.out ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/expected/upgrade_post_11_before_0.out ================================================ [File too large to display: 436 B] ================================================ FILE: src/test/regress/expected/upgrade_post_14_after.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/expected/upgrade_post_14_after_0.out ================================================ [File too large to display: 436 B] ================================================ FILE: src/test/regress/expected/upgrade_post_14_before.out ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/expected/upgrade_post_14_before_0.out ================================================ [File too large to display: 436 B] ================================================ FILE: src/test/regress/expected/upgrade_rebalance_strategy_after.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/upgrade_rebalance_strategy_before.out ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/expected/upgrade_ref2ref_after.out ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/expected/upgrade_ref2ref_before.out ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/expected/upgrade_schema_based_sharding_after.out ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/expected/upgrade_schema_based_sharding_before.out ================================================ [File too large to display: 383 B] ================================================ FILE: src/test/regress/expected/upgrade_single_shard_table_after.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/expected/upgrade_single_shard_table_before.out ================================================ [File too large to display: 277 B] ================================================ FILE: src/test/regress/expected/upgrade_type_after.out ================================================ [File too large to display: 414 B] ================================================ FILE: src/test/regress/expected/upgrade_type_before.out ================================================ [File too large to display: 342 B] ================================================ FILE: src/test/regress/expected/validate_constraint.out ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/expected/values.out ================================================ [File too large to display: 28.3 KB] ================================================ FILE: src/test/regress/expected/view_propagation.out ================================================ [File too large to display: 44.4 KB] ================================================ FILE: src/test/regress/expected/views.out ================================================ [File too large to display: 705 B] ================================================ FILE: src/test/regress/expected/views_create.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/expected/window_functions.out ================================================ [File too large to display: 67.3 KB] ================================================ FILE: src/test/regress/expected/with_basics.out ================================================ [File too large to display: 28.1 KB] ================================================ FILE: src/test/regress/expected/with_dml.out ================================================ [File too large to display: 11.1 KB] ================================================ FILE: src/test/regress/expected/with_executors.out ================================================ [File too large to display: 11.1 KB] ================================================ FILE: src/test/regress/expected/with_join.out ================================================ [File too large to display: 11.1 KB] ================================================ FILE: src/test/regress/expected/with_modifying.out ================================================ [File too large to display: 39.6 KB] ================================================ FILE: src/test/regress/expected/with_nested.out ================================================ [File too large to display: 10.6 KB] ================================================ FILE: src/test/regress/expected/with_partitioning.out ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/expected/with_prepare.out ================================================ [File too large to display: 39.4 KB] ================================================ FILE: src/test/regress/expected/with_set_operations.out ================================================ [File too large to display: 26.4 KB] ================================================ FILE: src/test/regress/expected/with_transactions.out ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/expected/with_where.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/worker_copy_table_to_node.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/expected/worker_split_binary_copy_test.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: src/test/regress/expected/worker_split_copy_test.out ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/expected/worker_split_text_copy_test.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: src/test/regress/failure_base_schedule ================================================ [File too large to display: 270 B] ================================================ FILE: src/test/regress/failure_schedule ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/flaky_tests.md ================================================ [File too large to display: 13.5 KB] ================================================ FILE: src/test/regress/isolation_schedule ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/log_test_times ================================================ [File too large to display: 123 B] ================================================ FILE: src/test/regress/minimal_columnar_schedule ================================================ [File too large to display: 28 B] ================================================ FILE: src/test/regress/minimal_pg_upgrade_schedule ================================================ [File too large to display: 79 B] ================================================ FILE: src/test/regress/minimal_schedule ================================================ [File too large to display: 140 B] ================================================ FILE: src/test/regress/mitmscripts/.gitignore ================================================ [File too large to display: 12 B] ================================================ FILE: src/test/regress/mitmscripts/CONTRIBUTING.md ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/mitmscripts/README.md ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/mitmscripts/fluent.py ================================================ [File too large to display: 14.5 KB] ================================================ FILE: src/test/regress/mitmscripts/structs.py ================================================ [File too large to display: 15.8 KB] ================================================ FILE: src/test/regress/mixed_after_citus_upgrade_schedule ================================================ [File too large to display: 26 B] ================================================ FILE: src/test/regress/mixed_before_citus_upgrade_schedule ================================================ [File too large to display: 27 B] ================================================ FILE: src/test/regress/multi_1_create_citus_schedule ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/multi_1_schedule ================================================ [File too large to display: 11.7 KB] ================================================ FILE: src/test/regress/multi_add_backup_node_schedule ================================================ [File too large to display: 75 B] ================================================ FILE: src/test/regress/multi_follower_schedule ================================================ [File too large to display: 344 B] ================================================ FILE: src/test/regress/multi_mx_schedule ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/multi_schedule ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/multi_schedule_hyperscale ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/multi_schedule_hyperscale_superuser ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/mx_base_schedule ================================================ [File too large to display: 318 B] ================================================ FILE: src/test/regress/mx_minimal_schedule ================================================ [File too large to display: 267 B] ================================================ FILE: src/test/regress/operations_schedule ================================================ [File too large to display: 510 B] ================================================ FILE: src/test/regress/pg_regress_multi.pl ================================================ [File too large to display: 46.9 KB] ================================================ FILE: src/test/regress/postgres_schedule ================================================ [File too large to display: 15 B] ================================================ FILE: src/test/regress/single_shard_table_prep_schedule ================================================ [File too large to display: 30 B] ================================================ FILE: src/test/regress/spec/README.md ================================================ [File too large to display: 628 B] ================================================ FILE: src/test/regress/spec/columnar_index_concurrency.spec ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/spec/columnar_temp_tables.spec ================================================ [File too large to display: 780 B] ================================================ FILE: src/test/regress/spec/columnar_vacuum_vs_insert.spec ================================================ [File too large to display: 763 B] ================================================ FILE: src/test/regress/spec/columnar_write_concurrency.spec ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/spec/columnar_write_concurrency_index.spec ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/spec/isolation_acquire_distributed_locks.spec ================================================ [File too large to display: 8.8 KB] ================================================ FILE: src/test/regress/spec/isolation_add_coordinator.spec ================================================ [File too large to display: 175 B] ================================================ FILE: src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/spec/isolation_add_remove_node.spec ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_move_multi_shard_commands.spec ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_move_multi_shard_commands_on_mx.spec ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_move_single_shard_commands.spec ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_move_single_shard_commands_on_mx.spec ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_shard_split.spec ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/spec/isolation_blocking_shard_split_with_fkey_to_reference.spec ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/spec/isolation_cancellation.spec ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/spec/isolation_check_mx.spec ================================================ [File too large to display: 178 B] ================================================ FILE: src/test/regress/spec/isolation_citus_dist_activity.spec ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/spec/isolation_citus_locks.spec ================================================ [File too large to display: 994 B] ================================================ FILE: src/test/regress/spec/isolation_citus_pause_node.spec ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/spec/isolation_citus_schema_distribute_undistribute.spec ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/spec/isolation_cluster_management.spec ================================================ [File too large to display: 157 B] ================================================ FILE: src/test/regress/spec/isolation_concurrent_dml.spec ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/spec/isolation_concurrent_move_create_table.spec ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/spec/isolation_copy_placement_vs_copy_placement.spec ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/spec/isolation_copy_placement_vs_modification.spec ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/spec/isolation_copy_vs_all_on_mx.spec ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/spec/isolation_create_citus_local_table.spec ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/spec/isolation_create_distributed_concurrently_after_drop_column.spec ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/spec/isolation_create_distributed_table.spec ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/spec/isolation_create_distributed_table_concurrently.spec ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/spec/isolation_create_restore_point.spec ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/spec/isolation_create_table_vs_add_remove_node.spec ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/spec/isolation_data_migration.spec ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/spec/isolation_database_cmd_from_any_node.spec ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/spec/isolation_ddl_vs_all.spec ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/spec/isolation_delete_vs_all.spec ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/spec/isolation_distributed_deadlock_detection.spec ================================================ [File too large to display: 10.4 KB] ================================================ FILE: src/test/regress/spec/isolation_distributed_transaction_id.spec ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/spec/isolation_drop_alter_index_select_for_update_on_mx.spec ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/spec/isolation_drop_shards.spec ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/spec/isolation_drop_vs_all.spec ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/spec/isolation_dump_global_wait_edges.spec ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/spec/isolation_dump_local_wait_edges.spec ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/spec/isolation_ensure_dependency_activate_node.spec ================================================ [File too large to display: 8.8 KB] ================================================ FILE: src/test/regress/spec/isolation_extension_commands.spec ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/spec/isolation_fix_partition_shard_index_names.spec ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/spec/isolation_get_all_active_transactions.spec ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec ================================================ [File too large to display: 10.4 KB] ================================================ FILE: src/test/regress/spec/isolation_global_pid.spec ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/spec/isolation_hash_copy_vs_all.spec ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_select_conflict.spec ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_select_repartition.spec ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_select_vs_all.spec ================================================ [File too large to display: 16.3 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec ================================================ [File too large to display: 7.3 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_vs_all.spec ================================================ [File too large to display: 10.5 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_vs_all_on_mx.spec ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/spec/isolation_insert_vs_vacuum.spec ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_binaryless.spec ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_multi_shard_commands.spec ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_multi_shard_commands_on_mx.spec ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_nonsu_nonbypassrls.spec ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_single_shard_commands.spec ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_single_shard_commands_on_mx.spec ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_skip_fk_validation.spec ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/spec/isolation_logical_replication_with_partitioning.spec ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/spec/isolation_master_update_node.spec ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/spec/isolation_max_client_connections.spec ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/spec/isolation_merge.spec ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/spec/isolation_merge_replicated.spec ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/spec/isolation_metadata_sync_deadlock.spec ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/spec/isolation_metadata_sync_vs_all.spec ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/spec/isolation_modify_with_subquery_vs_dml.spec ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/spec/isolation_move_placement_vs_modification.spec ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/spec/isolation_move_placement_vs_modification_fk.spec ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/spec/isolation_move_placement_vs_move_placement.spec ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/spec/isolation_multi_shard_modify_vs_all.spec ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/spec/isolation_multiuser_locking.spec ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/spec/isolation_mx_common.include.spec ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/spec/isolation_non_blocking_shard_split.spec ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/spec/isolation_non_blocking_shard_split_fkey.spec ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/spec/isolation_non_blocking_shard_split_with_index_as_replicaIdentity.spec ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/spec/isolation_partitioned_copy_vs_all.spec ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/spec/isolation_progress_monitoring.spec ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/spec/isolation_range_copy_vs_all.spec ================================================ [File too large to display: 9.3 KB] ================================================ FILE: src/test/regress/spec/isolation_rebalancer_deferred_drop.spec ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/spec/isolation_ref2ref_foreign_keys.spec ================================================ [File too large to display: 7.6 KB] ================================================ FILE: src/test/regress/spec/isolation_ref2ref_foreign_keys_enterprise.spec ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/spec/isolation_ref2ref_foreign_keys_on_mx.spec ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/spec/isolation_ref_select_for_update_vs_all_on_mx.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/spec/isolation_reference_copy_vs_all.spec ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/spec/isolation_reference_on_mx.spec ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/spec/isolation_reference_table.spec ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/spec/isolation_remove_coordinator.spec ================================================ [File too large to display: 162 B] ================================================ FILE: src/test/regress/spec/isolation_replace_wait_function.spec ================================================ [File too large to display: 866 B] ================================================ FILE: src/test/regress/spec/isolation_replicate_reference_tables_to_coordinator.spec ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/spec/isolation_replicated_dist_on_mx.spec ================================================ [File too large to display: 13.6 KB] ================================================ FILE: src/test/regress/spec/isolation_schema_based_sharding.spec ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/spec/isolation_select_for_update.spec ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/spec/isolation_select_vs_all.spec ================================================ [File too large to display: 15.7 KB] ================================================ FILE: src/test/regress/spec/isolation_select_vs_all_on_mx.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/spec/isolation_setup.spec ================================================ [File too large to display: 695 B] ================================================ FILE: src/test/regress/spec/isolation_shard_move_vs_start_metadata_sync.spec ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/spec/isolation_shard_rebalancer.spec ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/spec/isolation_shard_rebalancer_progress.spec ================================================ [File too large to display: 8.3 KB] ================================================ FILE: src/test/regress/spec/isolation_shouldhaveshards.spec ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/spec/isolation_tenant_isolation.spec ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/spec/isolation_tenant_isolation_nonblocking.spec ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/spec/isolation_tenant_isolation_with_fkey_to_reference.spec ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/spec/isolation_transaction_recovery.spec ================================================ [File too large to display: 718 B] ================================================ FILE: src/test/regress/spec/isolation_truncate_vs_all.spec ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/spec/isolation_truncate_vs_all_on_mx.spec ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/spec/isolation_undistribute_table.spec ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/spec/isolation_update_node.spec ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/spec/isolation_update_node_lock_writes.spec ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/spec/isolation_update_vs_all.spec ================================================ [File too large to display: 10.0 KB] ================================================ FILE: src/test/regress/spec/isolation_upsert_vs_all.spec ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/spec/isolation_vacuum_skip_locked.spec ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/spec/isolation_validate_vs_insert.spec ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/spec/shared_connection_waits.spec ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/split_schedule ================================================ [File too large to display: 997 B] ================================================ FILE: src/test/regress/sql/adaptive_executor.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/adaptive_executor_repartition.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/add_coordinator.sql ================================================ [File too large to display: 656 B] ================================================ FILE: src/test/regress/sql/adv_lock_permission.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/aggregate_support.sql ================================================ [File too large to display: 26.1 KB] ================================================ FILE: src/test/regress/sql/alter_database_owner.sql ================================================ [File too large to display: 6.0 KB] ================================================ FILE: src/test/regress/sql/alter_database_propagation.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/sql/alter_distributed_table.sql ================================================ [File too large to display: 26.3 KB] ================================================ FILE: src/test/regress/sql/alter_index.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/alter_role_propagation.sql ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/sql/alter_table_add_column.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/alter_table_set_access_method.sql ================================================ [File too large to display: 12.5 KB] ================================================ FILE: src/test/regress/sql/alter_table_single_shard_table.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/anonymous_columns.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_alter_table_add_constraint_without_name.sql ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_alter_table_add_constraint_without_name_create.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_recurring_outer_join.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_router.sql ================================================ [File too large to display: 18.5 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_router_create.sql ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate.sql ================================================ [File too large to display: 806 B] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate_cascade.sql ================================================ [File too large to display: 934 B] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate_cascade_create.sql ================================================ [File too large to display: 800 B] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate_create.sql ================================================ [File too large to display: 698 B] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate_partition.sql ================================================ [File too large to display: 365 B] ================================================ FILE: src/test/regress/sql/arbitrary_configs_truncate_partition_create.sql ================================================ [File too large to display: 545 B] ================================================ FILE: src/test/regress/sql/auto_undist_citus_local.sql ================================================ [File too large to display: 38.4 KB] ================================================ FILE: src/test/regress/sql/background_rebalance.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/background_rebalance_parallel.sql ================================================ [File too large to display: 16.7 KB] ================================================ FILE: src/test/regress/sql/background_rebalance_parallel_reference_tables.sql ================================================ [File too large to display: 9.7 KB] ================================================ FILE: src/test/regress/sql/background_task_queue_monitor.sql ================================================ [File too large to display: 21.9 KB] ================================================ FILE: src/test/regress/sql/base_enable_mx.sql ================================================ [File too large to display: 162 B] ================================================ FILE: src/test/regress/sql/binary_protocol.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/sql/bool_agg.sql ================================================ [File too large to display: 894 B] ================================================ FILE: src/test/regress/sql/cdc_library_path.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/ch_bench_having.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/ch_bench_having_mx.sql ================================================ [File too large to display: 6.0 KB] ================================================ FILE: src/test/regress/sql/ch_bench_subquery_repartition.sql ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_1.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_2.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_3.sql ================================================ [File too large to display: 877 B] ================================================ FILE: src/test/regress/sql/ch_benchmarks_4.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_5.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_6.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/ch_benchmarks_create_load.sql ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/sql/chbenchmark_all_queries.sql ================================================ [File too large to display: 21.5 KB] ================================================ FILE: src/test/regress/sql/check_cluster_state.sql ================================================ [File too large to display: 94 B] ================================================ FILE: src/test/regress/sql/check_mx.sql ================================================ [File too large to display: 471 B] ================================================ FILE: src/test/regress/sql/citus_aggregated_stats.sql ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/sql/citus_copy_shard_placement.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/citus_depended_object.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/citus_drain_node.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/citus_internal_access.sql ================================================ [File too large to display: 325 B] ================================================ FILE: src/test/regress/sql/citus_local_dist_joins.sql ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/sql/citus_local_table_triggers.sql ================================================ [File too large to display: 14.4 KB] ================================================ FILE: src/test/regress/sql/citus_local_tables.sql ================================================ [File too large to display: 25.4 KB] ================================================ FILE: src/test/regress/sql/citus_local_tables_ent.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/citus_local_tables_mx.sql ================================================ [File too large to display: 27.9 KB] ================================================ FILE: src/test/regress/sql/citus_local_tables_queries.sql ================================================ [File too large to display: 19.1 KB] ================================================ FILE: src/test/regress/sql/citus_local_tables_queries_mx.sql ================================================ [File too large to display: 19.9 KB] ================================================ FILE: src/test/regress/sql/citus_locks.sql ================================================ [File too large to display: 698 B] ================================================ FILE: src/test/regress/sql/citus_non_blocking_split_columnar.sql ================================================ [File too large to display: 17.3 KB] ================================================ FILE: src/test/regress/sql/citus_non_blocking_split_shard_cleanup.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/citus_non_blocking_split_shards.sql ================================================ [File too large to display: 14.9 KB] ================================================ FILE: src/test/regress/sql/citus_run_command.sql ================================================ [File too large to display: 153 B] ================================================ FILE: src/test/regress/sql/citus_schema_distribute_undistribute.sql ================================================ [File too large to display: 21.7 KB] ================================================ FILE: src/test/regress/sql/citus_schema_move.sql ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/sql/citus_shards.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/sql/citus_split_shard_by_split_points.sql ================================================ [File too large to display: 11.6 KB] ================================================ FILE: src/test/regress/sql/citus_split_shard_by_split_points_deferred_drop.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/citus_split_shard_by_split_points_failure.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/citus_split_shard_by_split_points_negative.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/sql/citus_split_shard_columnar_partitioned.sql ================================================ [File too large to display: 17.0 KB] ================================================ FILE: src/test/regress/sql/citus_stat_tenants.sql ================================================ [File too large to display: 18.8 KB] ================================================ FILE: src/test/regress/sql/citus_table_triggers.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/citus_update_table_statistics.sql ================================================ [File too large to display: 4.4 KB] ================================================ FILE: src/test/regress/sql/clock.sql ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/sql/columnar_alter.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/columnar_alter_set_type.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/columnar_analyze.sql ================================================ [File too large to display: 692 B] ================================================ FILE: src/test/regress/sql/columnar_chunk_filtering.sql ================================================ [File too large to display: 17.5 KB] ================================================ FILE: src/test/regress/sql/columnar_citus_integration.sql ================================================ [File too large to display: 19.1 KB] ================================================ FILE: src/test/regress/sql/columnar_clean.sql ================================================ [File too large to display: 238 B] ================================================ FILE: src/test/regress/sql/columnar_copyto.sql ================================================ [File too large to display: 632 B] ================================================ FILE: src/test/regress/sql/columnar_create.sql ================================================ [File too large to display: 7.5 KB] ================================================ FILE: src/test/regress/sql/columnar_cursor.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/columnar_data_types.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/sql/columnar_drop.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/columnar_empty.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/columnar_fallback_scan.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/columnar_first_row_number.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/columnar_indexes.sql ================================================ [File too large to display: 20.8 KB] ================================================ FILE: src/test/regress/sql/columnar_insert.sql ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/sql/columnar_join.sql ================================================ [File too large to display: 6.0 KB] ================================================ FILE: src/test/regress/sql/columnar_load.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/columnar_lz4.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/columnar_matview.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/columnar_memory.sql ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/sql/columnar_partitioning.sql ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/sql/columnar_paths.sql ================================================ [File too large to display: 9.4 KB] ================================================ FILE: src/test/regress/sql/columnar_permissions.sql ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/sql/columnar_pg15.sql ================================================ [File too large to display: 766 B] ================================================ FILE: src/test/regress/sql/columnar_query.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/sql/columnar_recursive.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/columnar_rollback.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/columnar_tableoptions.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/columnar_test_helpers.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/columnar_transactions.sql ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/sql/columnar_trigger.sql ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/sql/columnar_truncate.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/columnar_types_without_comparison.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/columnar_update_delete.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/columnar_vacuum.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/sql/columnar_zstd.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/comment_on_database.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/comment_on_role.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/connectivity_checks.sql ================================================ [File too large to display: 81 B] ================================================ FILE: src/test/regress/sql/coordinator_evaluation.sql ================================================ [File too large to display: 10.1 KB] ================================================ FILE: src/test/regress/sql/coordinator_evaluation_modify.sql ================================================ [File too large to display: 29.4 KB] ================================================ FILE: src/test/regress/sql/coordinator_evaluation_select.sql ================================================ [File too large to display: 22.2 KB] ================================================ FILE: src/test/regress/sql/coordinator_shouldhaveshards.sql ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/sql/cpu_priority.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/create_citus_local_table_cascade.sql ================================================ [File too large to display: 6.0 KB] ================================================ FILE: src/test/regress/sql/create_distributed_table_concurrently.sql ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/sql/create_drop_database_propagation.sql ================================================ [File too large to display: 28.4 KB] ================================================ FILE: src/test/regress/sql/create_drop_database_propagation_pg15.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/create_drop_database_propagation_pg16.sql ================================================ [File too large to display: 472 B] ================================================ FILE: src/test/regress/sql/create_ref_dist_from_citus_local.sql ================================================ [File too large to display: 27.6 KB] ================================================ FILE: src/test/regress/sql/create_role_propagation.sql ================================================ [File too large to display: 15.1 KB] ================================================ FILE: src/test/regress/sql/create_single_shard_table.sql ================================================ [File too large to display: 52.0 KB] ================================================ FILE: src/test/regress/sql/cross_join.sql ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/sql/cte_inline.sql ================================================ [File too large to display: 15.9 KB] ================================================ FILE: src/test/regress/sql/cte_nested_modification.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/cte_prepared_modify.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/sql/cursors.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/custom_aggregate_support.sql ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/sql/data_types.sql ================================================ [File too large to display: 15.6 KB] ================================================ FILE: src/test/regress/sql/detect_conn_close.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/disable_object_propagation.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/sql/distributed_collations.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/distributed_collations_conflict.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/distributed_domain.sql ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/sql/distributed_functions.sql ================================================ [File too large to display: 28.8 KB] ================================================ FILE: src/test/regress/sql/distributed_functions_conflict.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/distributed_intermediate_results.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/distributed_locks.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/sql/distributed_planning.sql ================================================ [File too large to display: 17.5 KB] ================================================ FILE: src/test/regress/sql/distributed_planning_create_load.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/distributed_procedure.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/distributed_triggers.sql ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/sql/distributed_types.sql ================================================ [File too large to display: 16.9 KB] ================================================ FILE: src/test/regress/sql/distributed_types_conflict.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/sql/distributed_types_xact_add_enum_value.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/dml_recursive.sql ================================================ [File too large to display: 7.7 KB] ================================================ FILE: src/test/regress/sql/drop_column_partitioned_table.sql ================================================ [File too large to display: 8.8 KB] ================================================ FILE: src/test/regress/sql/drop_database.sql ================================================ [File too large to display: 874 B] ================================================ FILE: src/test/regress/sql/drop_partitioned_table.sql ================================================ [File too large to display: 11.9 KB] ================================================ FILE: src/test/regress/sql/dropped_columns_1.sql ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/sql/dropped_columns_create_load.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/sql/ensure_citus_columnar_not_exists.sql ================================================ [File too large to display: 910 B] ================================================ FILE: src/test/regress/sql/ensure_no_intermediate_data_leak.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/ensure_no_shared_connection_leak.sql ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/sql/escape_extension_name.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/executor_local_failure.sql ================================================ [File too large to display: 1017 B] ================================================ FILE: src/test/regress/sql/expression_reference_join.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/failure_add_disable_node.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/failure_connection_establishment.sql ================================================ [File too large to display: 10.1 KB] ================================================ FILE: src/test/regress/sql/failure_copy_on_hash.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: src/test/regress/sql/failure_copy_to_reference.sql ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/sql/failure_create_database.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/failure_create_distributed_table_concurrently.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/failure_create_distributed_table_non_empty.sql ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/sql/failure_create_index_concurrently.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/sql/failure_create_reference_table.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/failure_create_table.sql ================================================ [File too large to display: 10.5 KB] ================================================ FILE: src/test/regress/sql/failure_cte_subquery.sql ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/sql/failure_ddl.sql ================================================ [File too large to display: 20.2 KB] ================================================ FILE: src/test/regress/sql/failure_distributed_results.sql ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/sql/failure_failover_to_local_execution.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/failure_insert_select_pushdown.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/failure_insert_select_repartition.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/sql/failure_insert_select_via_coordinator.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/failure_multi_dml.sql ================================================ [File too large to display: 8.2 KB] ================================================ FILE: src/test/regress/sql/failure_multi_row_insert.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/sql/failure_multi_shard_update_delete.sql ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/sql/failure_mx_metadata_sync.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/failure_mx_metadata_sync_multi_trans.sql ================================================ [File too large to display: 22.6 KB] ================================================ FILE: src/test/regress/sql/failure_offline_move_shard_placement.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/failure_on_create_subscription.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/failure_online_move_shard_placement.sql ================================================ [File too large to display: 9.2 KB] ================================================ FILE: src/test/regress/sql/failure_parallel_connection.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/failure_ref_tables.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/failure_replicated_partitions.sql ================================================ [File too large to display: 983 B] ================================================ FILE: src/test/regress/sql/failure_savepoints.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/sql/failure_setup.sql ================================================ [File too large to display: 225 B] ================================================ FILE: src/test/regress/sql/failure_single_mod.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/sql/failure_single_select.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/sql/failure_split_cleanup.sql ================================================ [File too large to display: 13.7 KB] ================================================ FILE: src/test/regress/sql/failure_tenant_isolation.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/failure_tenant_isolation_nonblocking.sql ================================================ [File too large to display: 10.6 KB] ================================================ FILE: src/test/regress/sql/failure_test_helpers.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/failure_truncate.sql ================================================ [File too large to display: 16.7 KB] ================================================ FILE: src/test/regress/sql/failure_vacuum.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/fast_path_router_modify.sql ================================================ [File too large to display: 6.9 KB] ================================================ FILE: src/test/regress/sql/fkeys_between_local_ref.sql ================================================ [File too large to display: 21.9 KB] ================================================ FILE: src/test/regress/sql/follower_single_node.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/forcedelegation_functions.sql ================================================ [File too large to display: 25.1 KB] ================================================ FILE: src/test/regress/sql/foreign_key_restriction_enforcement.sql ================================================ [File too large to display: 30.6 KB] ================================================ FILE: src/test/regress/sql/foreign_key_to_reference_shard_rebalance.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/foreign_key_to_reference_table.sql ================================================ [File too large to display: 46.5 KB] ================================================ FILE: src/test/regress/sql/foreign_tables_mx.sql ================================================ [File too large to display: 13.5 KB] ================================================ FILE: src/test/regress/sql/function_create.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/function_propagation.sql ================================================ [File too large to display: 37.0 KB] ================================================ FILE: src/test/regress/sql/function_with_case_when.sql ================================================ [File too large to display: 541 B] ================================================ FILE: src/test/regress/sql/functions.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/generated_identity.sql ================================================ [File too large to display: 12.1 KB] ================================================ FILE: src/test/regress/sql/geqo.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/global_cancel.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/grant_on_database_propagation.sql ================================================ [File too large to display: 14.3 KB] ================================================ FILE: src/test/regress/sql/grant_on_foreign_server_propagation.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/grant_on_function_propagation.sql ================================================ [File too large to display: 20.6 KB] ================================================ FILE: src/test/regress/sql/grant_on_schema_propagation.sql ================================================ [File too large to display: 9.0 KB] ================================================ FILE: src/test/regress/sql/grant_on_sequence_propagation.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/grant_on_table_propagation.sql ================================================ [File too large to display: 14.9 KB] ================================================ FILE: src/test/regress/sql/having_subquery.sql ================================================ [File too large to display: 976 B] ================================================ FILE: src/test/regress/sql/hyperscale_tutorial.sql ================================================ [File too large to display: 5.8 KB] ================================================ FILE: src/test/regress/sql/index_create.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/insert_select_connection_leak.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/insert_select_into_local_table.sql ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/sql/insert_select_repartition.sql ================================================ [File too large to display: 19.4 KB] ================================================ FILE: src/test/regress/sql/insert_select_single_shard_table.sql ================================================ [File too large to display: 21.2 KB] ================================================ FILE: src/test/regress/sql/intermediate_result_pruning.sql ================================================ [File too large to display: 19.5 KB] ================================================ FILE: src/test/regress/sql/intermediate_result_pruning_create.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/intermediate_result_pruning_queries_1.sql ================================================ [File too large to display: 12.8 KB] ================================================ FILE: src/test/regress/sql/intermediate_result_pruning_queries_2.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/intermediate_results.sql ================================================ [File too large to display: 14.6 KB] ================================================ FILE: src/test/regress/sql/issue_5099.sql ================================================ [File too large to display: 320 B] ================================================ FILE: src/test/regress/sql/issue_5248.sql ================================================ [File too large to display: 11.4 KB] ================================================ FILE: src/test/regress/sql/issue_5763.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/sql/issue_6543.sql ================================================ [File too large to display: 625 B] ================================================ FILE: src/test/regress/sql/issue_6592.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/issue_6758.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/issue_7477.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/sql/issue_7891.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/issue_8243.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: src/test/regress/sql/join_pushdown.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/sql/limit_intermediate_size.sql ================================================ [File too large to display: 5.9 KB] ================================================ FILE: src/test/regress/sql/local_dist_join.sql ================================================ [File too large to display: 12.1 KB] ================================================ FILE: src/test/regress/sql/local_dist_join_load.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/local_dist_join_mixed.sql ================================================ [File too large to display: 15.0 KB] ================================================ FILE: src/test/regress/sql/local_dist_join_modifications.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/local_execution_local_plan.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/local_shard_copy.sql ================================================ [File too large to display: 8.1 KB] ================================================ FILE: src/test/regress/sql/local_shard_execution.sql ================================================ [File too large to display: 53.4 KB] ================================================ FILE: src/test/regress/sql/local_shard_execution_dropped_column.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/local_shard_execution_replicated.sql ================================================ [File too large to display: 39.3 KB] ================================================ FILE: src/test/regress/sql/local_shard_utility_command_execution.sql ================================================ [File too large to display: 11.1 KB] ================================================ FILE: src/test/regress/sql/local_table_join.sql ================================================ [File too large to display: 25.0 KB] ================================================ FILE: src/test/regress/sql/locally_execute_intermediate_results.sql ================================================ [File too large to display: 25.7 KB] ================================================ FILE: src/test/regress/sql/logical_rep_consistency.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/sql/logical_replication.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/materialized_view.sql ================================================ [File too large to display: 10.3 KB] ================================================ FILE: src/test/regress/sql/merge.sql ================================================ [File too large to display: 72.2 KB] ================================================ FILE: src/test/regress/sql/merge_arbitrary.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/sql/merge_arbitrary_create.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/sql/merge_partition_tables.sql ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/sql/merge_repartition1.sql ================================================ [File too large to display: 16.3 KB] ================================================ FILE: src/test/regress/sql/merge_repartition2.sql ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/sql/merge_schema_sharding.sql ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/sql/merge_unsupported.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/sql/merge_vcore.sql ================================================ [File too large to display: 9.4 KB] ================================================ FILE: src/test/regress/sql/metadata_sync_helpers.sql ================================================ [File too large to display: 42.6 KB] ================================================ FILE: src/test/regress/sql/minimal_cluster_management.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/mixed_relkind_tests.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/modification_correctness.sql ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/sql/multi_add_node_from_backup.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/multi_add_node_from_backup_negative.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/multi_add_node_from_backup_sync_replica.sql ================================================ [File too large to display: 803 B] ================================================ FILE: src/test/regress/sql/multi_agg_approximate_distinct.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/sql/multi_agg_distinct.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: src/test/regress/sql/multi_agg_type_conversion.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/sql/multi_alias.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/multi_alter_table_add_constraints.sql ================================================ [File too large to display: 21.2 KB] ================================================ FILE: src/test/regress/sql/multi_alter_table_add_constraints_without_name.sql ================================================ [File too large to display: 39.7 KB] ================================================ FILE: src/test/regress/sql/multi_alter_table_add_foreign_key_without_name.sql ================================================ [File too large to display: 17.8 KB] ================================================ FILE: src/test/regress/sql/multi_alter_table_row_level_security.sql ================================================ [File too large to display: 14.0 KB] ================================================ FILE: src/test/regress/sql/multi_alter_table_row_level_security_escape.sql ================================================ [File too large to display: 604 B] ================================================ FILE: src/test/regress/sql/multi_alter_table_statements.sql ================================================ [File too large to display: 31.0 KB] ================================================ FILE: src/test/regress/sql/multi_array_agg.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/sql/multi_average_expression.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_basic_queries.sql ================================================ [File too large to display: 548 B] ================================================ FILE: src/test/regress/sql/multi_behavioral_analytics_basics.sql ================================================ [File too large to display: 15.7 KB] ================================================ FILE: src/test/regress/sql/multi_behavioral_analytics_create_table.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/sql/multi_behavioral_analytics_create_table_superuser.sql ================================================ [File too large to display: 9.2 KB] ================================================ FILE: src/test/regress/sql/multi_behavioral_analytics_single_shard_queries.sql ================================================ [File too large to display: 14.2 KB] ================================================ FILE: src/test/regress/sql/multi_cache_invalidation.sql ================================================ [File too large to display: 870 B] ================================================ FILE: src/test/regress/sql/multi_citus_tools.sql ================================================ [File too large to display: 13.5 KB] ================================================ FILE: src/test/regress/sql/multi_cluster_management.sql ================================================ [File too large to display: 25.3 KB] ================================================ FILE: src/test/regress/sql/multi_colocated_shard_rebalance.sql ================================================ [File too large to display: 19.8 KB] ================================================ FILE: src/test/regress/sql/multi_colocation_utils.sql ================================================ [File too large to display: 22.9 KB] ================================================ FILE: src/test/regress/sql/multi_complex_count_distinct.sql ================================================ [File too large to display: 13.2 KB] ================================================ FILE: src/test/regress/sql/multi_complex_expressions.sql ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/sql/multi_copy.sql ================================================ [File too large to display: 25.4 KB] ================================================ FILE: src/test/regress/sql/multi_count_type_conversion.sql ================================================ [File too large to display: 789 B] ================================================ FILE: src/test/regress/sql/multi_create_fdw.sql ================================================ [File too large to display: 855 B] ================================================ FILE: src/test/regress/sql/multi_create_role_dependency.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/multi_create_shards.sql ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/sql/multi_create_table.sql ================================================ [File too large to display: 10.1 KB] ================================================ FILE: src/test/regress/sql/multi_create_table_constraints.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/multi_create_table_superuser.sql ================================================ [File too large to display: 13.0 KB] ================================================ FILE: src/test/regress/sql/multi_create_users.sql ================================================ [File too large to display: 411 B] ================================================ FILE: src/test/regress/sql/multi_cross_shard.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: src/test/regress/sql/multi_data_types.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/multi_deparse_function.sql ================================================ [File too large to display: 11.4 KB] ================================================ FILE: src/test/regress/sql/multi_deparse_procedure.sql ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/sql/multi_deparse_shard_query.sql ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/sql/multi_distributed_transaction_id.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/multi_distribution_metadata.sql ================================================ [File too large to display: 12.7 KB] ================================================ FILE: src/test/regress/sql/multi_drop_extension.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/multi_dropped_column_aliases.sql ================================================ [File too large to display: 929 B] ================================================ FILE: src/test/regress/sql/multi_explain.sql ================================================ [File too large to display: 42.4 KB] ================================================ FILE: src/test/regress/sql/multi_extension.sql ================================================ [File too large to display: 43.3 KB] ================================================ FILE: src/test/regress/sql/multi_fix_partition_shard_index_names.sql ================================================ [File too large to display: 16.2 KB] ================================================ FILE: src/test/regress/sql/multi_follower_configure_followers.sql ================================================ [File too large to display: 651 B] ================================================ FILE: src/test/regress/sql/multi_follower_dml.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/sql/multi_follower_sanity_check.sql ================================================ [File too large to display: 254 B] ================================================ FILE: src/test/regress/sql/multi_follower_select_statements.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/multi_foreign_key.sql ================================================ [File too large to display: 35.4 KB] ================================================ FILE: src/test/regress/sql/multi_foreign_key_relation_graph.sql ================================================ [File too large to display: 15.6 KB] ================================================ FILE: src/test/regress/sql/multi_function_evaluation.sql ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/sql/multi_function_in_join.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/multi_generate_ddl_commands.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/multi_hash_pruning.sql ================================================ [File too large to display: 13.4 KB] ================================================ FILE: src/test/regress/sql/multi_having_pushdown.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/multi_index_statements.sql ================================================ [File too large to display: 19.1 KB] ================================================ FILE: src/test/regress/sql/multi_insert_select.sql ================================================ [File too large to display: 81.0 KB] ================================================ FILE: src/test/regress/sql/multi_insert_select_conflict.sql ================================================ [File too large to display: 9.1 KB] ================================================ FILE: src/test/regress/sql/multi_insert_select_non_pushable_queries.sql ================================================ [File too large to display: 23.1 KB] ================================================ FILE: src/test/regress/sql/multi_insert_select_window.sql ================================================ [File too large to display: 21.6 KB] ================================================ FILE: src/test/regress/sql/multi_join_order_additional.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/multi_join_order_tpch_repartition.sql ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/sql/multi_join_order_tpch_small.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/multi_join_pruning.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/multi_json_agg.sql ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/sql/multi_json_object_agg.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/multi_jsonb_agg.sql ================================================ [File too large to display: 3.3 KB] ================================================ FILE: src/test/regress/sql/multi_jsonb_object_agg.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/multi_large_shardid.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/multi_level_recursive_queries.sql ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/sql/multi_limit_clause.sql ================================================ [File too large to display: 8.6 KB] ================================================ FILE: src/test/regress/sql/multi_limit_clause_approximate.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/multi_load_data.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/multi_load_data_superuser.sql ================================================ [File too large to display: 796 B] ================================================ FILE: src/test/regress/sql/multi_load_more_data.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_master_protocol.sql ================================================ [File too large to display: 288 B] ================================================ FILE: src/test/regress/sql/multi_metadata_access.sql ================================================ [File too large to display: 647 B] ================================================ FILE: src/test/regress/sql/multi_metadata_attributes.sql ================================================ [File too large to display: 902 B] ================================================ FILE: src/test/regress/sql/multi_metadata_sync.sql ================================================ [File too large to display: 34.6 KB] ================================================ FILE: src/test/regress/sql/multi_metadata_sync_domain.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/multi_modifications.sql ================================================ [File too large to display: 47.1 KB] ================================================ FILE: src/test/regress/sql/multi_modifying_xacts.sql ================================================ [File too large to display: 36.4 KB] ================================================ FILE: src/test/regress/sql/multi_move_mx.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser.sql ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_auth.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_basic_queries.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_copy.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_grant.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_load_data.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/multi_multiuser_master_protocol.sql ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/sql/multi_mx_add_coordinator.sql ================================================ [File too large to display: 6.6 KB] ================================================ FILE: src/test/regress/sql/multi_mx_alter_distributed_table.sql ================================================ [File too large to display: 8.3 KB] ================================================ FILE: src/test/regress/sql/multi_mx_call.sql ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/sql/multi_mx_copy_data.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: src/test/regress/sql/multi_mx_create_table.sql ================================================ [File too large to display: 12.8 KB] ================================================ FILE: src/test/regress/sql/multi_mx_ddl.sql ================================================ [File too large to display: 7.7 KB] ================================================ FILE: src/test/regress/sql/multi_mx_explain.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: src/test/regress/sql/multi_mx_function_call_delegation.sql ================================================ [File too large to display: 12.7 KB] ================================================ FILE: src/test/regress/sql/multi_mx_function_table_reference.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_hide_shard_names.sql ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/sql/multi_mx_insert_select_repartition.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_metadata.sql ================================================ [File too large to display: 8.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_modifications.sql ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_modifications_to_reference_tables.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/multi_mx_modifying_xacts.sql ================================================ [File too large to display: 10.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_node_metadata.sql ================================================ [File too large to display: 16.1 KB] ================================================ FILE: src/test/regress/sql/multi_mx_partitioning.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/multi_mx_reference_table.sql ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_repartition_join_w1.sql ================================================ [File too large to display: 514 B] ================================================ FILE: src/test/regress/sql/multi_mx_repartition_join_w2.sql ================================================ [File too large to display: 517 B] ================================================ FILE: src/test/regress/sql/multi_mx_repartition_udt_prepare.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: src/test/regress/sql/multi_mx_repartition_udt_w1.sql ================================================ [File too large to display: 659 B] ================================================ FILE: src/test/regress/sql/multi_mx_repartition_udt_w2.sql ================================================ [File too large to display: 660 B] ================================================ FILE: src/test/regress/sql/multi_mx_router_planner.sql ================================================ [File too large to display: 21.2 KB] ================================================ FILE: src/test/regress/sql/multi_mx_schema_support.sql ================================================ [File too large to display: 13.3 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query1.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query10.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query12.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query14.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query19.sql ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query3.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query6.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query7.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/multi_mx_tpch_query7_nested.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/multi_mx_transaction_recovery.sql ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/sql/multi_mx_truncate_from_worker.sql ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/sql/multi_name_lengths.sql ================================================ [File too large to display: 13.3 KB] ================================================ FILE: src/test/regress/sql/multi_name_resolution.sql ================================================ [File too large to display: 990 B] ================================================ FILE: src/test/regress/sql/multi_null_minmax_value_pruning.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/multi_orderby_limit_pushdown.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/multi_outer_join.sql ================================================ [File too large to display: 16.1 KB] ================================================ FILE: src/test/regress/sql/multi_outer_join_columns.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/multi_outer_join_reference.sql ================================================ [File too large to display: 15.7 KB] ================================================ FILE: src/test/regress/sql/multi_partition_pruning.sql ================================================ [File too large to display: 5.3 KB] ================================================ FILE: src/test/regress/sql/multi_partitioning.sql ================================================ [File too large to display: 91.3 KB] ================================================ FILE: src/test/regress/sql/multi_partitioning_utils.sql ================================================ [File too large to display: 11.2 KB] ================================================ FILE: src/test/regress/sql/multi_poolinfo_usage.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/multi_prepare_plsql.sql ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/sql/multi_prepare_sql.sql ================================================ [File too large to display: 19.9 KB] ================================================ FILE: src/test/regress/sql/multi_prune_shard_list.sql ================================================ [File too large to display: 10.1 KB] ================================================ FILE: src/test/regress/sql/multi_query_directory_cleanup.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: src/test/regress/sql/multi_read_from_secondaries.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/multi_real_time_transaction.sql ================================================ [File too large to display: 11.7 KB] ================================================ FILE: src/test/regress/sql/multi_reference_table.sql ================================================ [File too large to display: 27.1 KB] ================================================ FILE: src/test/regress/sql/multi_remove_node_reference_table.sql ================================================ [File too large to display: 16.8 KB] ================================================ FILE: src/test/regress/sql/multi_repartition_join_planning.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/multi_repartition_join_pruning.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: src/test/regress/sql/multi_repartition_join_ref.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/multi_repartition_join_task_assignment.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_repartition_udt.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/multi_repartitioned_subquery_udf.sql ================================================ [File too large to display: 611 B] ================================================ FILE: src/test/regress/sql/multi_replicate_reference_table.sql ================================================ [File too large to display: 24.1 KB] ================================================ FILE: src/test/regress/sql/multi_rls_join_distribution_key.sql ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/sql/multi_router_planner.sql ================================================ [File too large to display: 40.8 KB] ================================================ FILE: src/test/regress/sql/multi_router_planner_fast_path.sql ================================================ [File too large to display: 26.4 KB] ================================================ FILE: src/test/regress/sql/multi_row_insert.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/multi_row_router_insert.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/multi_schema_support.sql ================================================ [File too large to display: 42.8 KB] ================================================ FILE: src/test/regress/sql/multi_select_distinct.sql ================================================ [File too large to display: 15.3 KB] ================================================ FILE: src/test/regress/sql/multi_select_for_update.sql ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/sql/multi_sequence_default.sql ================================================ [File too large to display: 19.2 KB] ================================================ FILE: src/test/regress/sql/multi_shard_update_delete.sql ================================================ [File too large to display: 22.0 KB] ================================================ FILE: src/test/regress/sql/multi_simple_queries.sql ================================================ [File too large to display: 13.8 KB] ================================================ FILE: src/test/regress/sql/multi_single_relation_subquery.sql ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/sql/multi_size_queries.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/multi_sql_function.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/multi_subquery.sql ================================================ [File too large to display: 21.4 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_behavioral_analytics.sql ================================================ [File too large to display: 46.6 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_complex_queries.sql ================================================ [File too large to display: 76.0 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_complex_reference_clause.sql ================================================ [File too large to display: 61.3 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_in_where_clause.sql ================================================ [File too large to display: 13.1 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_in_where_reference_clause.sql ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_misc.sql ================================================ [File too large to display: 6.8 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_union.sql ================================================ [File too large to display: 27.5 KB] ================================================ FILE: src/test/regress/sql/multi_subquery_window_functions.sql ================================================ [File too large to display: 15.6 KB] ================================================ FILE: src/test/regress/sql/multi_subtransactions.sql ================================================ [File too large to display: 8.0 KB] ================================================ FILE: src/test/regress/sql/multi_table_ddl.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/sql/multi_task_assignment_policy.sql ================================================ [File too large to display: 9.7 KB] ================================================ FILE: src/test/regress/sql/multi_task_string_size.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/multi_tenant_isolation.sql ================================================ [File too large to display: 24.5 KB] ================================================ FILE: src/test/regress/sql/multi_tenant_isolation_nonblocking.sql ================================================ [File too large to display: 25.4 KB] ================================================ FILE: src/test/regress/sql/multi_test_catalog_views.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/multi_test_helpers.sql ================================================ [File too large to display: 29.0 KB] ================================================ FILE: src/test/regress/sql/multi_test_helpers_superuser.sql ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/sql/multi_tpch_query1.sql ================================================ [File too large to display: 593 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query10.sql ================================================ [File too large to display: 592 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query12.sql ================================================ [File too large to display: 654 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query14.sql ================================================ [File too large to display: 410 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query19.sql ================================================ [File too large to display: 906 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query3.sql ================================================ [File too large to display: 475 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query6.sql ================================================ [File too large to display: 321 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query7.sql ================================================ [File too large to display: 840 B] ================================================ FILE: src/test/regress/sql/multi_tpch_query7_nested.sql ================================================ [File too large to display: 1019 B] ================================================ FILE: src/test/regress/sql/multi_transaction_recovery.sql ================================================ [File too large to display: 8.7 KB] ================================================ FILE: src/test/regress/sql/multi_transaction_recovery_multiple_databases.sql ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/sql/multi_transactional_drop_shards.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/multi_truncate.sql ================================================ [File too large to display: 10.2 KB] ================================================ FILE: src/test/regress/sql/multi_unsupported_worker_operations.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/sql/multi_update_select.sql ================================================ [File too large to display: 13.4 KB] ================================================ FILE: src/test/regress/sql/multi_upsert.sql ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/sql/multi_utilities.sql ================================================ [File too large to display: 13.0 KB] ================================================ FILE: src/test/regress/sql/multi_utility_statements.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/sql/multi_utility_warnings.sql ================================================ [File too large to display: 186 B] ================================================ FILE: src/test/regress/sql/multi_view.sql ================================================ [File too large to display: 25.8 KB] ================================================ FILE: src/test/regress/sql/multi_working_columns.sql ================================================ [File too large to display: 631 B] ================================================ FILE: src/test/regress/sql/mx_coordinator_shouldhaveshards.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/mx_foreign_key_to_reference_table.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/mx_regular_user.sql ================================================ [File too large to display: 15.4 KB] ================================================ FILE: src/test/regress/sql/nested_execution.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/nested_execution_create.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/node_conninfo_reload.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/non_colocated_leaf_subquery_joins.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/non_colocated_subquery_joins.sql ================================================ [File too large to display: 28.5 KB] ================================================ FILE: src/test/regress/sql/non_super_user_cdc_library_path.sql ================================================ [File too large to display: 892 B] ================================================ FILE: src/test/regress/sql/non_super_user_object_metadata.sql ================================================ [File too large to display: 15.4 KB] ================================================ FILE: src/test/regress/sql/null_parameters.sql ================================================ [File too large to display: 14.3 KB] ================================================ FILE: src/test/regress/sql/object_propagation_debug.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/sql/partition_wise_join.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/partitioned_indexes_create.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/partitioned_intermediate_results.sql ================================================ [File too large to display: 14.4 KB] ================================================ FILE: src/test/regress/sql/partitioning_issue_3970.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/pg12.sql ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/sql/pg13.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/pg13_propagate_statistics.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: src/test/regress/sql/pg13_with_ties.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/pg14.sql ================================================ [File too large to display: 27.0 KB] ================================================ FILE: src/test/regress/sql/pg15.sql ================================================ [File too large to display: 34.2 KB] ================================================ FILE: src/test/regress/sql/pg15_jsonpath.sql ================================================ [File too large to display: 5.4 KB] ================================================ FILE: src/test/regress/sql/pg16.sql ================================================ [File too large to display: 22.3 KB] ================================================ FILE: src/test/regress/sql/pg17.sql ================================================ [File too large to display: 67.8 KB] ================================================ FILE: src/test/regress/sql/pg17_json.sql ================================================ [File too large to display: 14.0 KB] ================================================ FILE: src/test/regress/sql/pg18.sql ================================================ [File too large to display: 67.6 KB] ================================================ FILE: src/test/regress/sql/pg_dump.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/pgmerge.sql ================================================ [File too large to display: 36.0 KB] ================================================ FILE: src/test/regress/sql/postgres.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/prepared_statements_1.sql ================================================ [File too large to display: 936 B] ================================================ FILE: src/test/regress/sql/prepared_statements_2.sql ================================================ [File too large to display: 10.4 KB] ================================================ FILE: src/test/regress/sql/prepared_statements_3.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: src/test/regress/sql/prepared_statements_4.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/sql/prepared_statements_create_load.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/propagate_extension_commands.sql ================================================ [File too large to display: 16.6 KB] ================================================ FILE: src/test/regress/sql/propagate_foreign_servers.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/sql/propagate_set_commands.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: src/test/regress/sql/propagate_statistics.sql ================================================ [File too large to display: 3.5 KB] ================================================ FILE: src/test/regress/sql/publication.sql ================================================ [File too large to display: 16.8 KB] ================================================ FILE: src/test/regress/sql/query_single_shard_table.sql ================================================ [File too large to display: 57.2 KB] ================================================ FILE: src/test/regress/sql/reassign_owned.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: src/test/regress/sql/recurring_join_pushdown.sql ================================================ [File too large to display: 7.2 KB] ================================================ FILE: src/test/regress/sql/recurring_outer_join.sql ================================================ [File too large to display: 29.3 KB] ================================================ FILE: src/test/regress/sql/recursive_dml_queries_mx.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: src/test/regress/sql/recursive_dml_with_different_planners_executors.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/recursive_relation_planning_restriction_pushdown.sql ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/sql/recursive_view_local_table.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/ref_citus_local_fkeys.sql ================================================ [File too large to display: 8.5 KB] ================================================ FILE: src/test/regress/sql/relation_access_tracking.sql ================================================ [File too large to display: 19.0 KB] ================================================ FILE: src/test/regress/sql/relation_access_tracking_single_node.sql ================================================ [File too large to display: 19.3 KB] ================================================ FILE: src/test/regress/sql/remove_coordinator.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: src/test/regress/sql/remove_coordinator_from_metadata.sql ================================================ [File too large to display: 54 B] ================================================ FILE: src/test/regress/sql/remove_non_default_nodes.sql ================================================ [File too large to display: 489 B] ================================================ FILE: src/test/regress/sql/rename_public_to_citus_schema_and_recreate.sql ================================================ [File too large to display: 66 B] ================================================ FILE: src/test/regress/sql/replicate_reference_tables_to_coordinator.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/replicated_partitioned_table.sql ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/sql/replicated_table_disable_node.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/sql/resync_metadata_with_sequences.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: src/test/regress/sql/role_command_from_any_node.sql ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/sql/row_types.sql ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/sql/run_command_on_all_nodes.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/schema_based_sharding.sql ================================================ [File too large to display: 47.7 KB] ================================================ FILE: src/test/regress/sql/schemas.sql ================================================ [File too large to display: 136 B] ================================================ FILE: src/test/regress/sql/schemas_create.sql ================================================ [File too large to display: 497 B] ================================================ FILE: src/test/regress/sql/seclabel.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/sequences.sql ================================================ [File too large to display: 952 B] ================================================ FILE: src/test/regress/sql/sequences_create.sql ================================================ [File too large to display: 623 B] ================================================ FILE: src/test/regress/sql/sequences_owned_by.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: src/test/regress/sql/sequences_with_different_types.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/regress/sql/sequential_modifications.sql ================================================ [File too large to display: 13.6 KB] ================================================ FILE: src/test/regress/sql/set_operation_and_local_tables.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/set_operations.sql ================================================ [File too large to display: 10.7 KB] ================================================ FILE: src/test/regress/sql/set_role_in_transaction.sql ================================================ [File too large to display: 2.8 KB] ================================================ FILE: src/test/regress/sql/shard_move_constraints.sql ================================================ [File too large to display: 14.3 KB] ================================================ FILE: src/test/regress/sql/shard_move_constraints_blocking.sql ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/sql/shard_move_deferred_delete.sql ================================================ [File too large to display: 7.0 KB] ================================================ FILE: src/test/regress/sql/shard_rebalancer.sql ================================================ [File too large to display: 65.0 KB] ================================================ FILE: src/test/regress/sql/shard_rebalancer_unit.sql ================================================ [File too large to display: 24.0 KB] ================================================ FILE: src/test/regress/sql/shared_connection_stats.sql ================================================ [File too large to display: 15.0 KB] ================================================ FILE: src/test/regress/sql/single_hash_repartition_join.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/single_node.sql ================================================ [File too large to display: 50.6 KB] ================================================ FILE: src/test/regress/sql/single_node_enterprise.sql ================================================ [File too large to display: 11.3 KB] ================================================ FILE: src/test/regress/sql/single_node_truncate.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/single_shard_table_prep.sql ================================================ [File too large to display: 733 B] ================================================ FILE: src/test/regress/sql/single_shard_table_udfs.sql ================================================ [File too large to display: 27.4 KB] ================================================ FILE: src/test/regress/sql/split_shard.sql ================================================ [File too large to display: 22.5 KB] ================================================ FILE: src/test/regress/sql/sql_procedure.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/sqlancer_failures.sql ================================================ [File too large to display: 9.7 KB] ================================================ FILE: src/test/regress/sql/sqlsmith_failures.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: src/test/regress/sql/ssl_by_default.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/sql/start_stop_metadata_sync.sql ================================================ [File too large to display: 11.5 KB] ================================================ FILE: src/test/regress/sql/stat_counters.sql ================================================ [File too large to display: 36.9 KB] ================================================ FILE: src/test/regress/sql/stat_statements.sql ================================================ [File too large to display: 9.6 KB] ================================================ FILE: src/test/regress/sql/statement_cancel_error_message.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/sql/subqueries_deep.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/sql/subqueries_not_supported.sql ================================================ [File too large to display: 3.0 KB] ================================================ FILE: src/test/regress/sql/subquery_and_cte.sql ================================================ [File too large to display: 13.6 KB] ================================================ FILE: src/test/regress/sql/subquery_append.sql ================================================ [File too large to display: 3.8 KB] ================================================ FILE: src/test/regress/sql/subquery_basics.sql ================================================ [File too large to display: 9.5 KB] ================================================ FILE: src/test/regress/sql/subquery_complex_target_list.sql ================================================ [File too large to display: 12.2 KB] ================================================ FILE: src/test/regress/sql/subquery_executors.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/subquery_in_targetlist.sql ================================================ [File too large to display: 13.6 KB] ================================================ FILE: src/test/regress/sql/subquery_in_where.sql ================================================ [File too large to display: 22.5 KB] ================================================ FILE: src/test/regress/sql/subquery_local_tables.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: src/test/regress/sql/subquery_partitioning.sql ================================================ [File too large to display: 5.1 KB] ================================================ FILE: src/test/regress/sql/subquery_prepared_statements.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/subquery_view.sql ================================================ [File too large to display: 10.7 KB] ================================================ FILE: src/test/regress/sql/subscripting_op.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: src/test/regress/sql/system_queries.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: src/test/regress/sql/tableam.sql ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/sql/tablespace.sql ================================================ [File too large to display: 387 B] ================================================ FILE: src/test/regress/sql/tdigest_aggregate_support.sql ================================================ [File too large to display: 7.8 KB] ================================================ FILE: src/test/regress/sql/text_search.sql ================================================ [File too large to display: 20.0 KB] ================================================ FILE: src/test/regress/sql/undistribute_table.sql ================================================ [File too large to display: 6.3 KB] ================================================ FILE: src/test/regress/sql/undistribute_table_cascade.sql ================================================ [File too large to display: 16.5 KB] ================================================ FILE: src/test/regress/sql/undistribute_table_cascade_mx.sql ================================================ [File too large to display: 5.6 KB] ================================================ FILE: src/test/regress/sql/union_pushdown.sql ================================================ [File too large to display: 45.9 KB] ================================================ FILE: src/test/regress/sql/unsupported_lateral_subqueries.sql ================================================ [File too large to display: 4.3 KB] ================================================ FILE: src/test/regress/sql/update_colocation_mx.sql ================================================ [File too large to display: 5.7 KB] ================================================ FILE: src/test/regress/sql/upgrade_autoconverted_after.sql ================================================ [File too large to display: 221 B] ================================================ FILE: src/test/regress/sql/upgrade_autoconverted_before.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/regress/sql/upgrade_basic_after.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: src/test/regress/sql/upgrade_basic_after_non_mixed.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: src/test/regress/sql/upgrade_basic_before.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: src/test/regress/sql/upgrade_basic_before_non_mixed.sql ================================================ ================================================ FILE: src/test/regress/sql/upgrade_citus_finish_citus_upgrade.sql ================================================ [File too large to display: 770 B] ================================================ FILE: src/test/regress/sql/upgrade_citus_locks.sql ================================================ [File too large to display: 406 B] ================================================ FILE: src/test/regress/sql/upgrade_citus_stat_activity.sql ================================================ [File too large to display: 451 B] ================================================ FILE: src/test/regress/sql/upgrade_columnar_after.sql ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/sql/upgrade_columnar_before.sql ================================================ [File too large to display: 20.9 KB] ================================================ FILE: src/test/regress/sql/upgrade_distributed_function_after.sql ================================================ [File too large to display: 401 B] ================================================ FILE: src/test/regress/sql/upgrade_distributed_function_before.sql ================================================ [File too large to display: 762 B] ================================================ FILE: src/test/regress/sql/upgrade_distributed_triggers_after.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: src/test/regress/sql/upgrade_distributed_triggers_before.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: src/test/regress/sql/upgrade_list_citus_objects.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql ================================================ [File too large to display: 970 B] ================================================ FILE: src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/sql/upgrade_post_11_after.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: src/test/regress/sql/upgrade_post_11_before.sql ================================================ [File too large to display: 11.8 KB] ================================================ FILE: src/test/regress/sql/upgrade_post_14_after.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: src/test/regress/sql/upgrade_post_14_before.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/upgrade_rebalance_strategy_after.sql ================================================ [File too large to display: 67 B] ================================================ FILE: src/test/regress/sql/upgrade_rebalance_strategy_before.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: src/test/regress/sql/upgrade_ref2ref_after.sql ================================================ [File too large to display: 672 B] ================================================ FILE: src/test/regress/sql/upgrade_ref2ref_before.sql ================================================ [File too large to display: 907 B] ================================================ FILE: src/test/regress/sql/upgrade_schema_based_sharding_after.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/upgrade_schema_based_sharding_before.sql ================================================ [File too large to display: 387 B] ================================================ FILE: src/test/regress/sql/upgrade_single_shard_table_after.sql ================================================ [File too large to display: 810 B] ================================================ FILE: src/test/regress/sql/upgrade_single_shard_table_before.sql ================================================ [File too large to display: 171 B] ================================================ FILE: src/test/regress/sql/upgrade_type_after.sql ================================================ [File too large to display: 306 B] ================================================ FILE: src/test/regress/sql/upgrade_type_before.sql ================================================ [File too large to display: 236 B] ================================================ FILE: src/test/regress/sql/validate_constraint.sql ================================================ [File too large to display: 4.8 KB] ================================================ FILE: src/test/regress/sql/values.sql ================================================ [File too large to display: 11.0 KB] ================================================ FILE: src/test/regress/sql/view_propagation.sql ================================================ [File too large to display: 25.2 KB] ================================================ FILE: src/test/regress/sql/views.sql ================================================ [File too large to display: 230 B] ================================================ FILE: src/test/regress/sql/views_create.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: src/test/regress/sql/window_functions.sql ================================================ [File too large to display: 16.0 KB] ================================================ FILE: src/test/regress/sql/with_basics.sql ================================================ [File too large to display: 17.1 KB] ================================================ FILE: src/test/regress/sql/with_dml.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: src/test/regress/sql/with_executors.sql ================================================ [File too large to display: 7.1 KB] ================================================ FILE: src/test/regress/sql/with_join.sql ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/sql/with_modifying.sql ================================================ [File too large to display: 20.8 KB] ================================================ FILE: src/test/regress/sql/with_nested.sql ================================================ [File too large to display: 6.2 KB] ================================================ FILE: src/test/regress/sql/with_partitioning.sql ================================================ [File too large to display: 2.9 KB] ================================================ FILE: src/test/regress/sql/with_prepare.sql ================================================ [File too large to display: 6.1 KB] ================================================ FILE: src/test/regress/sql/with_set_operations.sql ================================================ [File too large to display: 6.4 KB] ================================================ FILE: src/test/regress/sql/with_transactions.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: src/test/regress/sql/with_where.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: src/test/regress/sql/worker_copy_table_to_node.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: src/test/regress/sql/worker_split_binary_copy_test.sql ================================================ [File too large to display: 9.4 KB] ================================================ FILE: src/test/regress/sql/worker_split_copy_test.sql ================================================ [File too large to display: 6.7 KB] ================================================ FILE: src/test/regress/sql/worker_split_text_copy_test.sql ================================================ [File too large to display: 9.4 KB] ================================================ FILE: src/test/regress/sql_base_schedule ================================================ ================================================ FILE: src/test/regress/sql_schedule ================================================ [File too large to display: 749 B] ================================================ FILE: src/test/tap/Makefile ================================================ [File too large to display: 1.0 KB] ================================================ FILE: src/test/tap/citus_helpers.pm ================================================ [File too large to display: 2.7 KB] ================================================ FILE: src/test/tap/postgresql.conf ================================================ [File too large to display: 168 B] ================================================ FILE: src/test/tap/t/restore_point_mx.pl ================================================ [File too large to display: 9.9 KB] ================================================ FILE: vendor/README.md ================================================ [File too large to display: 274 B] ================================================ FILE: vendor/safestringlib/.gitattributes ================================================ [File too large to display: 378 B] ================================================ FILE: vendor/safestringlib/.gitignore ================================================ [File too large to display: 2.6 KB] ================================================ FILE: vendor/safestringlib/CMakeLists.txt ================================================ [File too large to display: 10.0 KB] ================================================ FILE: vendor/safestringlib/CODE_OF_CONDUCT.md ================================================ [File too large to display: 3.1 KB] ================================================ FILE: vendor/safestringlib/LICENSE ================================================ [File too large to display: 2.2 KB] ================================================ FILE: vendor/safestringlib/LICENSE©ING.txt ================================================ [File too large to display: 2.3 KB] ================================================ FILE: vendor/safestringlib/README.md ================================================ [File too large to display: 2.4 KB] ================================================ FILE: vendor/safestringlib/include/safe_lib.h ================================================ [File too large to display: 2.3 KB] ================================================ FILE: vendor/safestringlib/include/safe_lib_errno.h ================================================ [File too large to display: 3.0 KB] ================================================ FILE: vendor/safestringlib/include/safe_lib_errno.h.in ================================================ [File too large to display: 2.7 KB] ================================================ FILE: vendor/safestringlib/include/safe_mem_lib.h ================================================ [File too large to display: 4.5 KB] ================================================ FILE: vendor/safestringlib/include/safe_str_lib.h ================================================ [File too large to display: 7.1 KB] ================================================ FILE: vendor/safestringlib/include/safe_types.h ================================================ [File too large to display: 2.1 KB] ================================================ FILE: vendor/safestringlib/include/safe_types.h.in ================================================ [File too large to display: 1.9 KB] ================================================ FILE: vendor/safestringlib/include/snprintf_s.h ================================================ [File too large to display: 1.8 KB] ================================================ FILE: vendor/safestringlib/makefile ================================================ [File too large to display: 3.6 KB] ================================================ FILE: vendor/safestringlib/safeclib/abort_handler_s.c ================================================ [File too large to display: 2.4 KB] ================================================ FILE: vendor/safestringlib/safeclib/ignore_handler_s.c ================================================ [File too large to display: 2.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/mem_primitives_lib.c ================================================ [File too large to display: 21.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/mem_primitives_lib.h ================================================ [File too large to display: 2.4 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcmp16_s.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcmp32_s.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcmp_s.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcpy16_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcpy32_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/memcpy_s.c ================================================ [File too large to display: 5.0 KB] ================================================ FILE: vendor/safestringlib/safeclib/memmove16_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/memmove32_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/memmove_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/memset16_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/memset32_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/memset_s.c ================================================ [File too large to display: 3.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/memzero16_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/memzero32_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/memzero_s.c ================================================ [File too large to display: 3.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/safe_mem_constraint.c ================================================ [File too large to display: 4.5 KB] ================================================ FILE: vendor/safestringlib/safeclib/safe_mem_constraint.h ================================================ [File too large to display: 1.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/safe_str_constraint.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/safe_str_constraint.h ================================================ [File too large to display: 2.6 KB] ================================================ FILE: vendor/safestringlib/safeclib/safeclib_private.h ================================================ [File too large to display: 2.5 KB] ================================================ FILE: vendor/safestringlib/safeclib/snprintf_support.c ================================================ [File too large to display: 9.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/stpcpy_s.c ================================================ [File too large to display: 7.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/stpncpy_s.c ================================================ [File too large to display: 9.5 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcasecmp_s.c ================================================ [File too large to display: 4.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcasestr_s.c ================================================ [File too large to display: 5.0 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcat_s.c ================================================ [File too large to display: 7.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcmp_s.c ================================================ [File too large to display: 4.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcmpfld_s.c ================================================ [File too large to display: 4.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcpy_s.c ================================================ [File too large to display: 5.9 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcpyfld_s.c ================================================ [File too large to display: 5.9 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcpyfldin_s.c ================================================ [File too large to display: 6.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcpyfldout_s.c ================================================ [File too large to display: 6.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strcspn_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/strfirstchar_s.c ================================================ [File too large to display: 3.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/strfirstdiff_s.c ================================================ [File too large to display: 4.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/strfirstsame_s.c ================================================ [File too large to display: 4.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strisalphanumeric_s.c ================================================ [File too large to display: 3.5 KB] ================================================ FILE: vendor/safestringlib/safeclib/strisascii_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/strisdigit_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/strishex_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strislowercase_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strismixedcase_s.c ================================================ [File too large to display: 3.4 KB] ================================================ FILE: vendor/safestringlib/safeclib/strispassword_s.c ================================================ [File too large to display: 4.9 KB] ================================================ FILE: vendor/safestringlib/safeclib/strisuppercase_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strlastchar_s.c ================================================ [File too large to display: 3.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/strlastdiff_s.c ================================================ [File too large to display: 4.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strlastsame_s.c ================================================ [File too large to display: 4.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strljustify_s.c ================================================ [File too large to display: 4.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strncat_s.c ================================================ [File too large to display: 8.0 KB] ================================================ FILE: vendor/safestringlib/safeclib/strncpy_s.c ================================================ [File too large to display: 7.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strnlen_s.c ================================================ [File too large to display: 3.2 KB] ================================================ FILE: vendor/safestringlib/safeclib/strnterminate_s.c ================================================ [File too large to display: 3.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strpbrk_s.c ================================================ [File too large to display: 4.5 KB] ================================================ FILE: vendor/safestringlib/safeclib/strprefix_s.c ================================================ [File too large to display: 3.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/strremovews_s.c ================================================ [File too large to display: 4.4 KB] ================================================ FILE: vendor/safestringlib/safeclib/strspn_s.c ================================================ [File too large to display: 4.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/strstr_s.c ================================================ [File too large to display: 5.0 KB] ================================================ FILE: vendor/safestringlib/safeclib/strtok_s.c ================================================ [File too large to display: 10.1 KB] ================================================ FILE: vendor/safestringlib/safeclib/strtolowercase_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strtouppercase_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/strzero_s.c ================================================ [File too large to display: 2.9 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcpcpy_s.c ================================================ [File too large to display: 6.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcscat_s.c ================================================ [File too large to display: 7.0 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcscpy_s.c ================================================ [File too large to display: 6.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcsncat_s.c ================================================ [File too large to display: 8.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcsncpy_s.c ================================================ [File too large to display: 7.7 KB] ================================================ FILE: vendor/safestringlib/safeclib/wcsnlen_s.c ================================================ [File too large to display: 3.3 KB] ================================================ FILE: vendor/safestringlib/safeclib/wmemcmp_s.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/wmemcpy_s.c ================================================ [File too large to display: 4.8 KB] ================================================ FILE: vendor/safestringlib/safeclib/wmemmove_s.c ================================================ [File too large to display: 4.9 KB] ================================================ FILE: vendor/safestringlib/safeclib/wmemset_s.c ================================================ [File too large to display: 3.2 KB]